From 3be187c89d041805bc93e01a3202fd3533f94e13 Mon Sep 17 00:00:00 2001 From: Justin Pfister <119864532+JJRPF@users.noreply.github.com> Date: Wed, 16 Sep 2026 19:43:09 -0400 Subject: [PATCH 0001/2680] Bluetooth: hci_bcm4377: Add hardware reset recovery and post-init restoration When Bluetooth is stopped during heavy traffic (such as active LE scanning or discovery), transfer ring destruction commands to the Broadcom PCIe controller can time out and fail. Subsequent opens fail because rings cannot be created, leaving the device wedged. Implement hardware reset recovery using PCI Function-Level Reset (FLR) to re-initialize the PCIe controller and restart the firmware. Introduce a synchronous hdev->post_init hook to reload calibration data, regulatory PTB tables, and restore the device BD_ADDR after hardware reset, ensuring the controller is fully restored to operational state. Assisted-by: Antigravity:gemini-3.8-flash Signed-off-by: Justin Pfister <119864532+JJRPF@users.noreply.github.com> --- drivers/bluetooth/hci_bcm4377.c | 259 ++++++++++++++++++++++++++++---- 1 file changed, 229 insertions(+), 30 deletions(-) diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index 0eb137a99add7..5fe775766e781 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -11,11 +11,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include @@ -583,6 +585,10 @@ struct bcm4377_data { char vendor[BCM4377_OTP_MAX_PARAM_LEN]; const char *board_type; + bool needs_reset; + bool cal_needed; + bdaddr_t bdaddr; + struct completion event; struct bcm4377_context *ctx; @@ -1304,9 +1310,86 @@ static int bcm4378_send_ptb(struct bcm4377_data *bcm4377, return 0; } -static int bcm4377_hci_open(struct hci_dev *hdev) +static void bcm4377_disable_aspm(struct bcm4377_data *bcm4377); +static int bcm4377_init_cfg(struct bcm4377_data *bcm4377); +static int bcm4377_boot(struct bcm4377_data *bcm4377); +static int bcm4377_setup_rti(struct bcm4377_data *bcm4377); + +static int bcm4377_pci_reset(struct bcm4377_data *bcm4377) +{ + int ret; + int irq; + + dev_info(&bcm4377->pdev->dev, "performing hardware reset recovery\n"); + + irq = pci_irq_vector(bcm4377->pdev, 0); + if (irq > 0) + disable_irq(irq); + + if (bcm4377->hw->disable_aspm) + bcm4377_disable_aspm(bcm4377); + + ret = pci_reset_function(bcm4377->pdev); + if (ret) + dev_warn(&bcm4377->pdev->dev, + "function level reset failed with %d; trying to continue anyway\n", + ret); + + msleep(100); + + pci_set_master(bcm4377->pdev); + + ret = bcm4377_init_cfg(bcm4377); + if (ret) { + dev_err(&bcm4377->pdev->dev, + "failed to reinitialize PCI config: %d\n", ret); + goto out_irq; + } + + bcm4377->bootstage = 0; + bcm4377->rti_status = 0; + reinit_completion(&bcm4377->event); + + memset(bcm4377->ring_state, 0, sizeof(*bcm4377->ring_state)); + bcm4377->control_h2d_ring.generation = 0; + bcm4377->control_ack_ring.enabled = false; + bitmap_zero(bcm4377->control_h2d_ring.msgids, + bcm4377->control_h2d_ring.n_entries); + if (bcm4377->control_h2d_ring.events) + memset(bcm4377->control_h2d_ring.events, 0, + bcm4377->control_h2d_ring.n_entries * + sizeof(*bcm4377->control_h2d_ring.events)); + + if (irq > 0) + enable_irq(irq); + + ret = bcm4377_boot(bcm4377); + if (ret) { + dev_err(&bcm4377->pdev->dev, "failed to reboot firmware: %d\n", + ret); + return ret; + } + + ret = bcm4377_setup_rti(bcm4377); + if (ret) { + dev_err(&bcm4377->pdev->dev, "failed to re-setup RTI: %d\n", + ret); + return ret; + } + + bcm4377->needs_reset = false; + dev_info(&bcm4377->pdev->dev, + "hardware reset recovery completed successfully\n"); + return 0; + +out_irq: + if (irq > 0) + enable_irq(irq); + return ret; +} + +static int bcm4377_hci_open_rings(struct bcm4377_data *bcm4377) { - struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev); int ret; dev_dbg(&bcm4377->pdev->dev, "creating rings\n"); @@ -1374,23 +1457,92 @@ static int bcm4377_hci_open(struct hci_dev *hdev) return ret; } +static int bcm4377_hci_open(struct hci_dev *hdev) +{ + struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev); + int ret; + + if (bcm4377->needs_reset) { + dev_info(&bcm4377->pdev->dev, + "device marked for reset, reinitializing hardware before open\n"); + ret = bcm4377_pci_reset(bcm4377); + if (ret) + return ret; + bcm4377->cal_needed = true; + } + + ret = bcm4377_hci_open_rings(bcm4377); + if (ret && !bcm4377->needs_reset) { + dev_warn(&bcm4377->pdev->dev, + "creating rings failed (%d); attempting hardware reset recovery\n", + ret); + ret = bcm4377_pci_reset(bcm4377); + if (ret) { + bcm4377->needs_reset = true; + return ret; + } + bcm4377->cal_needed = true; + ret = bcm4377_hci_open_rings(bcm4377); + if (ret) + bcm4377->needs_reset = true; + } + + return ret; +} + static int bcm4377_hci_close(struct hci_dev *hdev) { struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev); dev_dbg(&bcm4377->pdev->dev, "destroying rings in hci_close\n"); - bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring); - bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring); - bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring); - bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring); - bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring); - bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring); + if (!bcm4377->needs_reset) { + if (bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring)) + bcm4377->needs_reset = true; + if (!bcm4377->needs_reset && + bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring)) + bcm4377->needs_reset = true; + if (!bcm4377->needs_reset && + bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring)) + bcm4377->needs_reset = true; + if (!bcm4377->needs_reset && + bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring)) + bcm4377->needs_reset = true; + if (!bcm4377->needs_reset && + bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring)) + bcm4377->needs_reset = true; + if (!bcm4377->needs_reset && + bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring)) + bcm4377->needs_reset = true; + + if (!bcm4377->needs_reset && + bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_event_ring)) + bcm4377->needs_reset = true; + if (!bcm4377->needs_reset && + bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_ack_ring)) + bcm4377->needs_reset = true; + if (!bcm4377->needs_reset && + bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring)) + bcm4377->needs_reset = true; + if (!bcm4377->needs_reset && + bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring)) + bcm4377->needs_reset = true; + } - bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_event_ring); - bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_ack_ring); - bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring); - bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring); + if (bcm4377->needs_reset) { + dev_warn(&bcm4377->pdev->dev, + "ring teardown failed; marking device for reset on next open\n"); + bcm4377->acl_d2h_ring.enabled = false; + bcm4377->acl_h2d_ring.enabled = false; + bcm4377->sco_d2h_ring.enabled = false; + bcm4377->sco_h2d_ring.enabled = false; + bcm4377->hci_d2h_ring.enabled = false; + bcm4377->hci_h2d_ring.enabled = false; + bcm4377->sco_event_ring.enabled = false; + bcm4377->sco_ack_ring.enabled = false; + bcm4377->hci_acl_event_ring.enabled = false; + bcm4377->hci_acl_ack_ring.enabled = false; + } return 0; } @@ -1467,6 +1619,67 @@ static int bcm4377_hci_setup(struct hci_dev *hdev) return bcm4377_check_bdaddr(bcm4377); } +static int bcm4377_hci_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr) +{ + struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev); + struct sk_buff *skb; + int err; + + skb = __hci_cmd_sync(hdev, 0xfc01, 6, bdaddr, HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) { + err = PTR_ERR(skb); + dev_err(&bcm4377->pdev->dev, + "Change address command failed (%d)", err); + return err; + } + kfree_skb(skb); + + bacpy(&bcm4377->bdaddr, bdaddr); + return 0; +} + +static int bcm4377_hci_post_init(struct hci_dev *hdev) +{ + struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev); + const bdaddr_t *bda = NULL; + int ret; + + if (!bcm4377->cal_needed) + return 0; + + dev_info(&bcm4377->pdev->dev, + "reloading calibration and PTB after hardware reset\n"); + + ret = bcm4377_hci_setup(hdev); + if (ret) { + dev_err(&bcm4377->pdev->dev, + "failed to reload calibration/PTB: %d\n", ret); + return ret; + } + + if (bacmp(&bcm4377->bdaddr, BDADDR_ANY)) + bda = &bcm4377->bdaddr; + else if (bacmp(&hdev->setup_addr, BDADDR_ANY)) + bda = &hdev->setup_addr; + else if (bacmp(&hdev->public_addr, BDADDR_ANY)) + bda = &hdev->public_addr; + + if (bda) { + dev_info(&bcm4377->pdev->dev, + "restoring BD_ADDR %pMR after hardware reset\n", bda); + ret = bcm4377_hci_set_bdaddr(hdev, bda); + if (ret) { + dev_err(&bcm4377->pdev->dev, + "failed to restore BD_ADDR: %d\n", ret); + return ret; + } + bacpy(&hdev->bdaddr, bda); + } + + bcm4377->cal_needed = false; + return 0; +} + static int bcm4377_hci_send_frame(struct hci_dev *hdev, struct sk_buff *skb) { struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev); @@ -1504,24 +1717,6 @@ static int bcm4377_hci_send_frame(struct hci_dev *hdev, struct sk_buff *skb) return ret; } -static int bcm4377_hci_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr) -{ - struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev); - struct sk_buff *skb; - int err; - - skb = __hci_cmd_sync(hdev, 0xfc01, 6, bdaddr, HCI_INIT_TIMEOUT); - if (IS_ERR(skb)) { - err = PTR_ERR(skb); - dev_err(&bcm4377->pdev->dev, - "Change address command failed (%d)", err); - return err; - } - kfree_skb(skb); - - return 0; -} - static int bcm4377_alloc_transfer_ring(struct bcm4377_data *bcm4377, struct bcm4377_transfer_ring *ring) { @@ -2241,6 +2436,9 @@ static int bcm4377_probe_of(struct bcm4377_data *bcm4377) return -ENOENT; } + device_property_read_u8_array(&bcm4377->pdev->dev, "local-bd-address", + (u8 *)&bcm4377->bdaddr, 6); + return 0; } @@ -2387,6 +2585,7 @@ static int bcm4377_probe(struct pci_dev *pdev, const struct pci_device_id *id) hdev->send = bcm4377_hci_send_frame; hdev->set_bdaddr = bcm4377_hci_set_bdaddr; hdev->setup = bcm4377_hci_setup; + hdev->post_init = bcm4377_hci_post_init; if (bcm4377->hw->broken_mws_transport_config) hci_set_quirk(hdev, HCI_QUIRK_BROKEN_MWS_TRANSPORT_CONFIG); From e04046f87349de08479f7f4b7332ebf9db5f3e29 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:56:08 +1100 Subject: [PATCH 0002/2680] Bluetooth: hci_bcm4377: stop recovery when the PCI reset fails Reset must quiesce the controller before shared ring state is cleared and firmware is restarted. Preserve the reset error rather than reinitializing DMA state on a running controller. Leave needs_reset set until every recovery stage completes so a failed boot or RTI handshake cannot fall back to ring creation on the next open. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/hci_bcm4377.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index 5fe775766e781..51b3e8bf9c135 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -1320,6 +1320,8 @@ static int bcm4377_pci_reset(struct bcm4377_data *bcm4377) int ret; int irq; + /* Keep retrying a full reset if any recovery stage fails. */ + bcm4377->needs_reset = true; dev_info(&bcm4377->pdev->dev, "performing hardware reset recovery\n"); irq = pci_irq_vector(bcm4377->pdev, 0); @@ -1330,10 +1332,11 @@ static int bcm4377_pci_reset(struct bcm4377_data *bcm4377) bcm4377_disable_aspm(bcm4377); ret = pci_reset_function(bcm4377->pdev); - if (ret) - dev_warn(&bcm4377->pdev->dev, - "function level reset failed with %d; trying to continue anyway\n", - ret); + if (ret) { + dev_err(&bcm4377->pdev->dev, + "function level reset failed: %d\n", ret); + goto out_irq; + } msleep(100); From 960045b5c57186361ff6c30b6574e33bbe439037 Mon Sep 17 00:00:00 2001 From: RJ Skerry-Ryan Date: Fri, 2 Oct 2026 20:21:00 -0700 Subject: [PATCH 0003/2680] Bluetooth: hci_bcm4377: Rebind the driver when the controller stops responding After some suspend-to-idle cycles on a BCM4388 (MacBook Pro 16" M2 Max), the controller no longer answers HCI commands. hci_resume_dev() times out on Set Event Mask (0x0c01), followed by Write Scan Enable, LE address resolution and scan parameter commands. Every later command times out as well, until the module is reloaded, which recovers it. Install an hdev->reset handler, which the HCI core calls on a command timeout, and recover the same way: rebind the driver with device_reprobe(), so probe resets the function and boots the firmware again. The work and its device reference are allocated separately because the reprobe frees bcm4377_data. Timeouts during setup are left alone, as are timeouts while a reprobe is already queued. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: RJ Skerry-Ryan (cherry picked from commit 37e9742a8c798226df0dc219bb3c11e59fc53e5b) --- drivers/bluetooth/hci_bcm4377.c | 55 +++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index 51b3e8bf9c135..793c96939d2e7 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -591,6 +591,9 @@ struct bcm4377_data { struct completion event; + /* Set once a recovery reprobe has been queued for this binding. */ + unsigned long reset_pending; + struct bcm4377_context *ctx; dma_addr_t ctx_dma; @@ -2469,6 +2472,57 @@ static void bcm4377_hci_free_dev(void *data) hci_free_dev(data); } +struct bcm4377_reset_work { + struct work_struct work; + struct pci_dev *pdev; +}; + +static void bcm4377_reset_work(struct work_struct *work) +{ + struct bcm4377_reset_work *rw = + container_of(work, struct bcm4377_reset_work, work); + + /* + * Remove and probe again: probe resets the function and boots the + * firmware from scratch. device_reprobe() frees the old bcm4377_data, + * which is why this work and its device reference live outside it. + */ + if (device_reprobe(&rw->pdev->dev)) + dev_err(&rw->pdev->dev, "reprobe after command timeout failed\n"); + + pci_dev_put(rw->pdev); + kfree(rw); +} + +/* + * Called by the HCI core (from process context) on a command timeout. After + * some system resumes the controller stops answering commands for good: + * every later command times out until the module is reloaded. Recover the + * same way, by rebinding the driver. + */ +static void bcm4377_hci_reset(struct hci_dev *hdev) +{ + struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev); + struct bcm4377_reset_work *rw; + + /* A timeout while still setting up would only loop probe failures. */ + if (hci_dev_test_flag(hdev, HCI_SETUP)) + return; + if (test_and_set_bit(0, &bcm4377->reset_pending)) + return; + + rw = kzalloc_obj(*rw); + if (!rw) { + clear_bit(0, &bcm4377->reset_pending); + return; + } + + bt_dev_err(hdev, "controller not responding, rebinding the driver"); + rw->pdev = pci_dev_get(bcm4377->pdev); + INIT_WORK(&rw->work, bcm4377_reset_work); + schedule_work(&rw->work); +} + static void bcm4377_hci_unregister_dev(void *data) { hci_unregister_dev(data); @@ -2588,6 +2642,7 @@ static int bcm4377_probe(struct pci_dev *pdev, const struct pci_device_id *id) hdev->send = bcm4377_hci_send_frame; hdev->set_bdaddr = bcm4377_hci_set_bdaddr; hdev->setup = bcm4377_hci_setup; + hdev->reset = bcm4377_hci_reset; hdev->post_init = bcm4377_hci_post_init; if (bcm4377->hw->broken_mws_transport_config) From 647d8891d08da067158fa1b793e9b24bb7d7ba19 Mon Sep 17 00:00:00 2001 From: RJ Skerry-Ryan Date: Sat, 3 Oct 2026 12:33:18 -0700 Subject: [PATCH 0004/2680] Bluetooth: hci_bcm4377: Flush the sleep control writes bcm4377_suspend() and bcm4377_resume() write SLEEP_CONTROL and return without reading anything back, so the PCIe write can still be posted when the system enters suspend-to-idle, or when hci_resume_dev() starts sending commands. On a BCM4388 (MacBook Pro 16" M2 Max) about a quarter to half of all resumes then left the controller unresponsive: every resume command timed out, stalling resume by about 20 s. Read the register back after both writes. With a read-back in place, 8 of 8 resumes succeeded, including 10 and 20 minute sleeps. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: RJ Skerry-Ryan (cherry picked from commit 4d662f924baeab3f5080f6e21bdddabeaaa1c691) --- drivers/bluetooth/hci_bcm4377.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index 793c96939d2e7..4cdd619f3e85c 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -2687,6 +2687,8 @@ static int bcm4377_suspend(struct device *dev) iowrite32(BCM4377_BAR0_SLEEP_CONTROL_QUIESCE, bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); + /* Flush the posted write before the system goes to sleep. */ + ioread32(bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); return 0; } @@ -2698,6 +2700,8 @@ static int bcm4377_resume(struct device *dev) iowrite32(BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE, bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); + /* The firmware must see UNQUIESCE before the resume commands. */ + ioread32(bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); return hci_resume_dev(bcm4377->hdev); } From 7c0f6523f6169d2d9a1af5769b0a3369c561bfab Mon Sep 17 00:00:00 2001 From: RJ Skerry-Ryan Date: Sat, 3 Oct 2026 13:41:25 -0700 Subject: [PATCH 0005/2680] Bluetooth: hci_bcm4377: Support remote wakeup The driver has no hdev->wakeup callback, so the HCI core never offers HCI_CONN_FLAG_REMOTE_WAKEUP: bluetoothd's attempt to allow wakeup for a paired mouse fails with "Bad flag given (0x1) vs supported (0xe)", and suspend always disconnects without programming a wake filter. Report device_may_wakeup() as the wakeup state. When wakeup is enabled, keep the firmware running and the function in D0 through system sleep, and arm the interrupt for wakeup, so that a connection attempt or advertisement from a wake-allowed device wakes the system. Otherwise quiesce the firmware as before. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: RJ Skerry-Ryan (cherry picked from commit 8f373b9bfcde0b9ea6aa8243620bc3f6b9e695d5) --- drivers/bluetooth/hci_bcm4377.c | 42 +++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index 4cdd619f3e85c..d62e9799f37d1 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -594,6 +594,10 @@ struct bcm4377_data { /* Set once a recovery reprobe has been queued for this binding. */ unsigned long reset_pending; + /* System sleep: firmware quiesced, or kept running to wake us. */ + bool quiesced; + bool wake_armed; + struct bcm4377_context *ctx; dma_addr_t ctx_dma; @@ -2472,6 +2476,13 @@ static void bcm4377_hci_free_dev(void *data) hci_free_dev(data); } +static bool bcm4377_hci_wakeup(struct hci_dev *hdev) +{ + struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev); + + return device_may_wakeup(&bcm4377->pdev->dev); +} + struct bcm4377_reset_work { struct work_struct work; struct pci_dev *pdev; @@ -2643,6 +2654,7 @@ static int bcm4377_probe(struct pci_dev *pdev, const struct pci_device_id *id) hdev->set_bdaddr = bcm4377_hci_set_bdaddr; hdev->setup = bcm4377_hci_setup; hdev->reset = bcm4377_hci_reset; + hdev->wakeup = bcm4377_hci_wakeup; hdev->post_init = bcm4377_hci_post_init; if (bcm4377->hw->broken_mws_transport_config) @@ -2681,14 +2693,28 @@ static int bcm4377_suspend(struct device *dev) struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev); int ret; + /* With wakeup allowed this programs the event filter and scan. */ ret = hci_suspend_dev(bcm4377->hdev); if (ret) return ret; + if (device_may_wakeup(dev)) { + /* + * Remote wakeup: leave the firmware running and the function + * in D0, so a connection attempt from a wake-allowed device + * raises an interrupt. Saving the state here stops the PCI + * core from putting the function into D3hot. + */ + pci_save_state(pdev); + bcm4377->wake_armed = !enable_irq_wake(pdev->irq); + return 0; + } + iowrite32(BCM4377_BAR0_SLEEP_CONTROL_QUIESCE, bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); /* Flush the posted write before the system goes to sleep. */ ioread32(bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); + bcm4377->quiesced = true; return 0; } @@ -2698,10 +2724,18 @@ static int bcm4377_resume(struct device *dev) struct pci_dev *pdev = to_pci_dev(dev); struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev); - iowrite32(BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE, - bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); - /* The firmware must see UNQUIESCE before the resume commands. */ - ioread32(bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); + if (bcm4377->wake_armed) { + disable_irq_wake(pdev->irq); + bcm4377->wake_armed = false; + } + + if (bcm4377->quiesced) { + iowrite32(BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE, + bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); + /* The firmware must see UNQUIESCE before the resume commands. */ + ioread32(bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); + bcm4377->quiesced = false; + } return hci_resume_dev(bcm4377->hdev); } From df1bcf19c3ce913b40ab9496346ed4e06e2e5415 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:16:43 +1100 Subject: [PATCH 0006/2680] Bluetooth: hci_bcm4377: drain recovery work before module exit Queue recovery on a dedicated workqueue. Unregister the PCI driver and its HCI timeout producers before draining that queue, so no callback can outlive the module text. A module reference released inside a callback alone would not guarantee that the callback has returned before unloading. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/hci_bcm4377.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index d62e9799f37d1..67be6315b0d72 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -2483,6 +2483,8 @@ static bool bcm4377_hci_wakeup(struct hci_dev *hdev) return device_may_wakeup(&bcm4377->pdev->dev); } +static struct workqueue_struct *bcm4377_reset_wq; + struct bcm4377_reset_work { struct work_struct work; struct pci_dev *pdev; @@ -2531,7 +2533,7 @@ static void bcm4377_hci_reset(struct hci_dev *hdev) bt_dev_err(hdev, "controller not responding, rebinding the driver"); rw->pdev = pci_dev_get(bcm4377->pdev); INIT_WORK(&rw->work, bcm4377_reset_work); - schedule_work(&rw->work); + queue_work(bcm4377_reset_wq, &rw->work); } static void bcm4377_hci_unregister_dev(void *data) @@ -2845,7 +2847,29 @@ static struct pci_driver bcm4377_pci_driver = { .probe = bcm4377_probe, .driver.pm = &bcm4377_ops, }; -module_pci_driver(bcm4377_pci_driver); +static int __init bcm4377_init(void) +{ + int ret; + + bcm4377_reset_wq = alloc_workqueue("bcm4377-reset", WQ_UNBOUND, 0); + if (!bcm4377_reset_wq) + return -ENOMEM; + + ret = pci_register_driver(&bcm4377_pci_driver); + if (ret) + destroy_workqueue(bcm4377_reset_wq); + + return ret; +} +module_init(bcm4377_init); + +static void __exit bcm4377_exit(void) +{ + /* Unregister HCI devices before draining the callbacks they queued. */ + pci_unregister_driver(&bcm4377_pci_driver); + destroy_workqueue(bcm4377_reset_wq); +} +module_exit(bcm4377_exit); MODULE_AUTHOR("Sven Peter "); MODULE_DESCRIPTION("Bluetooth support for Broadcom 4377/4378/4387/4388 devices"); From 55c1e6be7771b24477c0447883d84480e5667866 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:34:41 +1100 Subject: [PATCH 0007/2680] Bluetooth: hci_bcm4377: separate the driver and init declarations Add the declaration separator required by strict checkpatch. This is a formatting follow-up to the supplied recovery-work lifetime fix and does not change workqueue or module teardown behavior. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/hci_bcm4377.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index 67be6315b0d72..680fabe9297a6 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -2847,6 +2847,7 @@ static struct pci_driver bcm4377_pci_driver = { .probe = bcm4377_probe, .driver.pm = &bcm4377_ops, }; + static int __init bcm4377_init(void) { int ret; From a6115055fe0c5c21ba9e605baf24a5283876e02d Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Wed, 26 Aug 2026 18:29:18 -0600 Subject: [PATCH 0008/2680] wifi: brcmfmac: add AWDL interface creation for BCM4387 BCM4387 firmware implements AWDL (Apple Wireless Direct Link) and accepts BRCMF_INTERFACE_TYPE_AWDL through interface_create, but nl80211 has no AWDL iftype, so the interface cannot be created via add_virtual_intf(). Drive it from a vendor command instead. Creation is asynchronous, and has to be. nl80211 dispatches vendor commands holding the wiphy mutex: NL80211_CMD_VENDOR sets NL80211_FLAG_NEED_WIPHY without NL80211_FLAG_NO_WIPHY_MTX, unlike NL80211_CMD_NEW_INTERFACE which sets NEED_RTNL | NO_WIPHY_MTX and takes the wiphy mutex itself, in that order. Registering a netdev from the vendor command therefore deadlocks: nl80211_vendor_cmd <- holds wiphy mutex brcmf_awdl_add_vif brcmf_net_attach register_netdev <- takes RTNL register_netdevice call_netdevice_notifiers_info cfg80211_netdev_notifier_call mutex_lock <- wiphy mutex, already held The task blocks in uninterruptible sleep while holding the RTNL, which takes down every netdev operation on the system; recovery needs a reboot. Using cfg80211_register_netdevice() instead is not a fix either: it requires both the RTNL and the wiphy mutex, and the vendor command holds only the latter, so it trips ASSERT_RTNL and registers into an inconsistent state. Taking the RTNL explicitly would invert cfg80211's RTNL-then-wiphy order. So brcmf_awdl_add_vif() only asks firmware for the interface and returns. The netdev is registered later by brcmf_cfg80211_awdl_attach_pending(), called from the fweh event worker, which runs on the system workqueue holding neither lock. The vif event stays armed across the request because brcmf_notify_vif_event() is what links the brcmf_if to the vif; without it ndev->ieee80211_ptr is never set and cfg80211 ignores the netdev. Userspace must wait for awdl0 to appear rather than assume the vendor command created it. Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 141 ++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/cfg80211.h | 5 + .../broadcom/brcm80211/brcmfmac/fweh.c | 7 + .../brcm80211/brcmfmac/interface_create.c | 8 +- .../brcm80211/brcmfmac/interface_create.h | 1 + .../broadcom/brcm80211/brcmfmac/vendor.c | 50 +++++++ .../broadcom/brcm80211/brcmfmac/vendor.h | 12 ++ 7 files changed, 223 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 5d85299dc76ea..ac0392cfdf97a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -605,6 +605,147 @@ brcmf_cfg80211_update_proto_addr_mode(struct wireless_dev *wdev) * * Return: pointer to new vif on success, ERR_PTR(-errno) if not */ +/** + * brcmf_awdl_add_vif() - create an AWDL interface and give it a netdev. + * + * @wiphy: wiphy device of the new interface. + * @name: name of the new interface. + * + * AWDL is Apple Wireless Direct Link, the link layer AirDrop and AirPlay run + * over. BCM4387 firmware implements it and accepts BRCMF_INTERFACE_TYPE_AWDL + * through interface_create, but there is no nl80211 interface type for it, so + * this cannot go through add_virtual_intf(). It is driven from the vendor + * command instead. + * + * The vif is allocated as NL80211_IFTYPE_STATION because cfg80211 needs some + * valid type for its bookkeeping; the firmware side is what actually decides + * the interface behaviour, from the iftype passed to interface_create. + * + * Return: pointer to the new vif on success, ERR_PTR(-errno) on failure. + */ +int brcmf_awdl_add_vif(struct wiphy *wiphy, const char *name) +{ + struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); + struct brcmf_if *ifp = netdev_priv(cfg_to_ndev(cfg)); + struct brcmf_cfg80211_vif *vif; + int err; + + if (brcmf_cfg80211_vif_event_armed(cfg) || cfg->awdl_pending) + return -EBUSY; + + vif = brcmf_alloc_vif(cfg, NL80211_IFTYPE_STATION); + if (IS_ERR(vif)) + return PTR_ERR(vif); + + /* Arm before asking firmware: the BRCMF_E_IF event is what links the + * brcmf_if to this vif (brcmf_notify_vif_event() bails with -EBADF if + * nothing is armed, leaving ndev->ieee80211_ptr unset). + */ + brcmf_cfg80211_arm_vif_event(cfg, vif); + + /* Pass NULL rather than a derived address. Supplying a MAC address + * at interface_create time kills the firmware control channel about + * two seconds after create, so the address is set afterwards via + * cur_etheraddr instead. + * + */ + strscpy(cfg->awdl_ifname, name, sizeof(cfg->awdl_ifname)); + cfg->awdl_pending = true; + + err = brcmf_cfg80211_request_awdl_if(ifp, NULL); + if (err) { + cfg->awdl_pending = false; + brcmf_cfg80211_arm_vif_event(cfg, NULL); + brcmf_free_vif(vif); + return err; + } + + /* Deliberately do NOT wait for the interface here, and do NOT attach + * the netdev. nl80211 calls vendor commands holding the wiphy mutex + * (NL80211_CMD_VENDOR sets NEED_WIPHY without NO_WIPHY_MTX), and + * register_netdev() re-enters that mutex via the netdev notifier + * chain -> cfg80211_netdev_notifier_call(). Doing it here self- + * deadlocks and wedges the RTNL for the whole system. The attach is + * done from brcmf_cfg80211_awdl_attach_pending(), which the fweh + * event worker calls while holding no locks at all. + */ + return 0; +} + +/** + * brcmf_cfg80211_awdl_attach_pending() - finish AWDL interface creation. + * + * @ifp: interface the firmware just reported via BRCMF_E_IF_ADD. + * + * Called from the fweh event worker (brcmf_fweh_handle_if_event()), which + * runs on the system workqueue holding neither the RTNL nor the wiphy mutex. + * That is the only context in which register_netdev() is safe for an + * interface created outside cfg80211's own add_virtual_intf() path. + */ +void brcmf_cfg80211_awdl_attach_pending(struct brcmf_if *ifp) +{ + struct brcmf_cfg80211_info *cfg = ifp->drvr->config; + struct brcmf_cfg80211_vif *vif; + int err; + + if (!cfg || !cfg->awdl_pending) + return; + cfg->awdl_pending = false; + vif = ifp->vif; + + if (!ifp->ndev || !vif) { + bphy_err(ifp->drvr, "AWDL if event without netdev/vif\n"); + goto done; + } + + strscpy(ifp->ndev->name, cfg->awdl_ifname, sizeof(ifp->ndev->name)); + err = brcmf_net_attach(ifp, false); + if (err) { + bphy_err(ifp->drvr, "registering AWDL netdevice failed\n"); + free_netdev(ifp->ndev); + brcmf_free_vif(vif); + goto done; + } + + brcmf_info("AWDL interface %s created (bsscfgidx %d)\n", + ifp->ndev->name, ifp->bsscfgidx); +done: + brcmf_cfg80211_arm_vif_event(cfg, NULL); +} + +/** + * brcmf_awdl_del_vif() - tear down an interface made by brcmf_awdl_add_vif(). + * + * @wiphy: wiphy device of the interface. + * @wdev: wireless device to remove. + * + * Return: 0 on success, negative errno on failure. + */ +int brcmf_awdl_del_vif(struct wiphy *wiphy, struct wireless_dev *wdev) +{ + struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); + struct brcmf_cfg80211_vif *vif = container_of(wdev, + struct brcmf_cfg80211_vif, + wdev); + struct brcmf_if *ifp = vif->ifp; + int err; + + brcmf_cfg80211_arm_vif_event(cfg, vif); + err = brcmf_fil_bsscfg_data_set(ifp, "interface_remove", NULL, 0); + if (err) { + brcmf_cfg80211_arm_vif_event(cfg, NULL); + return err; + } + + err = brcmf_cfg80211_wait_vif_event(cfg, BRCMF_E_IF_DEL, + BRCMF_VIF_EVENT_TIMEOUT); + brcmf_cfg80211_arm_vif_event(cfg, NULL); + if (!err) + return -EIO; + + return 0; +} + static struct wireless_dev *brcmf_apsta_add_vif(struct wiphy *wiphy, const char *name, struct vif_params *params, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index dec5ff1190f94..fdc9f7e627f0a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -397,6 +397,8 @@ struct brcmf_cfg80211_info { struct work_struct escan_timeout_work; struct list_head vif_list; struct brcmf_cfg80211_vif_event vif_event; + bool awdl_pending; + char awdl_ifname[IFNAMSIZ]; struct completion vif_disabled; struct brcmu_d11inf d11inf; struct brcmf_assoclist_le assoclist; @@ -508,6 +510,9 @@ bool brcmf_get_vif_state_any(struct brcmf_cfg80211_info *cfg, void brcmf_cfg80211_arm_vif_event(struct brcmf_cfg80211_info *cfg, struct brcmf_cfg80211_vif *vif); bool brcmf_cfg80211_vif_event_armed(struct brcmf_cfg80211_info *cfg); +int brcmf_awdl_add_vif(struct wiphy *wiphy, const char *name); +void brcmf_cfg80211_awdl_attach_pending(struct brcmf_if *ifp); +int brcmf_awdl_del_vif(struct wiphy *wiphy, struct wireless_dev *wdev); int brcmf_cfg80211_wait_vif_event(struct brcmf_cfg80211_info *cfg, u8 action, ulong timeout); s32 brcmf_notify_escan_complete(struct brcmf_cfg80211_info *cfg, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c index eec4b10e1f674..3e06c23defcb3 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c @@ -183,6 +183,13 @@ static void brcmf_fweh_handle_if_event(struct brcmf_pub *drvr, brcmf_fweh_call_event_handler(drvr, ifp, emsg->event_code, emsg, data); + /* The handler above establishes the vif <-> ifp linkage. Only then can + * the AWDL netdev be registered, and only from here: this worker holds + * no locks, unlike the vendor command that requested the interface. + */ + if (ifp && ifevent->action == BRCMF_E_IF_ADD) + brcmf_cfg80211_awdl_attach_pending(ifp); + if (ifp && ifevent->action == BRCMF_E_IF_DEL) { bool armed = brcmf_cfg80211_vif_event_armed(drvr->config); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c index 1f40ff8d632c2..e9ebb90030575 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c @@ -52,7 +52,7 @@ enum brcmf_interface_type { BRCMF_INTERFACE_TYPE_STA = 0, BRCMF_INTERFACE_TYPE_AP = 1, - /* The missing number here is deliberate */ + BRCMF_INTERFACE_TYPE_AWDL = 2, BRCMF_INTERFACE_TYPE_NAN = 3, BRCMF_INTERFACE_TYPE_P2P_GO = 4, BRCMF_INTERFACE_TYPE_P2P_GC = 5, @@ -240,6 +240,12 @@ int brcmf_cfg80211_request_sta_if(struct brcmf_if *ifp, u8 *macaddr) macaddr); } +int brcmf_cfg80211_request_awdl_if(struct brcmf_if *ifp, u8 *macaddr) +{ + return brcmf_cfg80211_request_if(ifp, BRCMF_INTERFACE_TYPE_AWDL, + macaddr); +} + int brcmf_cfg80211_request_ap_if(struct brcmf_if *ifp) { int err; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.h index 669fa1508b67f..7fadbea590e28 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.h @@ -9,5 +9,6 @@ int brcmf_cfg80211_request_sta_if(struct brcmf_if *ifp, u8 *macaddr); int brcmf_cfg80211_request_ap_if(struct brcmf_if *ifp); +int brcmf_cfg80211_request_awdl_if(struct brcmf_if *ifp, u8 *macaddr); #endif /* _BRCMF_INTERFACE_CREATE_H_ */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c index d07e7c7355d9d..6dd6a3d7231fe 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c @@ -104,6 +104,46 @@ static int brcmf_cfg80211_vndr_cmds_dcmd_handler(struct wiphy *wiphy, return ret; } +#define BRCMF_AWDL_IFNAME "awdl0" + +static int brcmf_cfg80211_vndr_cmds_awdl_handler(struct wiphy *wiphy, + struct wireless_dev *wdev, + const void *data, int len) +{ + struct net_device *ndev; + struct wireless_dev *awdl_wdev; + u32 op; + + if (len < sizeof(op)) + return -EINVAL; + op = *(const u32 *)data; + + switch (op) { + case BRCMF_VNDR_AWDL_OP_CREATE: + ndev = dev_get_by_name(wiphy_net(wiphy), BRCMF_AWDL_IFNAME); + if (ndev) { + dev_put(ndev); + return -EEXIST; + } + /* Asynchronous: returns as soon as firmware has been asked. + * The netdev appears once the fweh worker registers it, so + * userspace must wait for awdl0 rather than assume it exists. + */ + return brcmf_awdl_add_vif(wiphy, BRCMF_AWDL_IFNAME); + case BRCMF_VNDR_AWDL_OP_DESTROY: + ndev = dev_get_by_name(wiphy_net(wiphy), BRCMF_AWDL_IFNAME); + if (!ndev) + return -ENODEV; + awdl_wdev = ndev->ieee80211_ptr; + dev_put(ndev); + if (!awdl_wdev) + return -ENODEV; + return brcmf_awdl_del_vif(wiphy, awdl_wdev); + default: + return -EINVAL; + } +} + const struct wiphy_vendor_command brcmf_vendor_cmds[] = { { { @@ -115,4 +155,14 @@ const struct wiphy_vendor_command brcmf_vendor_cmds[] = { .policy = VENDOR_CMD_RAW_DATA, .doit = brcmf_cfg80211_vndr_cmds_dcmd_handler }, + { + { + .vendor_id = BROADCOM_OUI, + .subcmd = BRCMF_VNDR_CMDS_AWDL + }, + .flags = WIPHY_VENDOR_CMD_NEED_WDEV | + WIPHY_VENDOR_CMD_NEED_NETDEV, + .policy = VENDOR_CMD_RAW_DATA, + .doit = brcmf_cfg80211_vndr_cmds_awdl_handler + }, }; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h index 418f33ea6fd3f..74886de2ba78d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h @@ -11,9 +11,21 @@ enum brcmf_vndr_cmds { BRCMF_VNDR_CMDS_UNSPEC, BRCMF_VNDR_CMDS_DCMD, + BRCMF_VNDR_CMDS_AWDL, BRCMF_VNDR_CMDS_LAST }; +/** + * enum brcmf_vndr_awdl_op - operations for BRCMF_VNDR_CMDS_AWDL + * + * @BRCMF_VNDR_AWDL_OP_CREATE: create the AWDL interface + * @BRCMF_VNDR_AWDL_OP_DESTROY: remove the AWDL interface + */ +enum brcmf_vndr_awdl_op { + BRCMF_VNDR_AWDL_OP_CREATE, + BRCMF_VNDR_AWDL_OP_DESTROY, +}; + /** * enum brcmf_nlattrs - nl80211 message attributes * From 1b67bba9dc460868ae76a43436b80988aefec3d1 Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Wed, 26 Aug 2026 19:02:47 -0600 Subject: [PATCH 0009/2680] wifi: brcmfmac: use AWDL-specific netdev ops for awdl0 Bringing an AWDL interface up through brcmf_netdev_open() stops the firmware answering dcmds. Every subsequent command times out with -EIO, the infrastructure STA link drops with it, and only a brcmfmac reload recovers the device. The standard open path runs station bring-up against the interface: a toe_ol iovar query, brcmf_cfg80211_up(), and then the multicast and promisc configuration that ndo_set_rx_mode schedules via _brcmf_set_multicast_list(). An AWDL bsscfg is not a station, and issuing that sequence against one wedges the firmware control path. Give AWDL interfaces their own net_device_ops. brcmf_netdev_open_awdl() checks the bus is up, clears the 802.1x pending count, and leaves the carrier off, touching no firmware state. ndo_set_rx_mode is omitted entirely so the multicast worker never runs for these interfaces, and ndo_stop skips brcmf_cfg80211_down() for the same reason the open path skips brcmf_cfg80211_up(). awdl0 now comes up with the firmware still responsive and the STA link undisturbed. Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 2 + .../broadcom/brcm80211/brcmfmac/core.c | 52 ++++++++++++++++++- .../broadcom/brcm80211/brcmfmac/core.h | 1 + 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index ac0392cfdf97a..c25000807b610 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -698,6 +698,8 @@ void brcmf_cfg80211_awdl_attach_pending(struct brcmf_if *ifp) goto done; } + /* Must be set before brcmf_net_attach(), which picks netdev_ops. */ + ifp->is_awdl = true; strscpy(ifp->ndev->name, cfg->awdl_ifname, sizeof(ifp->ndev->name)); err = brcmf_net_attach(ifp, false); if (err) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 13cb6cb9655aa..4a137454f8945 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -634,6 +634,55 @@ static int brcmf_netdev_open(struct net_device *ndev) return 0; } +/* AWDL interfaces are not STAs. brcmf_netdev_open() runs the standard station + * bring-up -- a toe_ol query, brcmf_cfg80211_up(), then the multicast and + * promisc configuration driven by ndo_set_rx_mode -- and issuing those against + * an AWDL bsscfg stops the firmware answering dcmds at all: every subsequent + * command times out and only a brcmfmac reload recovers it. Bring the netdev + * up without touching firmware, and deliberately omit ndo_set_rx_mode so the + * multicast worker never runs for these interfaces. + */ +static int brcmf_netdev_open_awdl(struct net_device *ndev) +{ + struct brcmf_if *ifp = netdev_priv(ndev); + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_bus *bus_if = drvr->bus_if; + + brcmf_dbg(TRACE, "Enter, bsscfgidx=%d (AWDL)\n", ifp->bsscfgidx); + + if (bus_if->state != BRCMF_BUS_UP) { + bphy_err(drvr, "failed bus is not ready\n"); + return -EAGAIN; + } + + atomic_set(&ifp->pend_8021x_cnt, 0); + + /* Carrier stays off until AWDL is actually running. */ + netif_carrier_off(ndev); + return 0; +} + +static int brcmf_netdev_stop_awdl(struct net_device *ndev) +{ + struct brcmf_if *ifp = netdev_priv(ndev); + + brcmf_dbg(TRACE, "Enter, bsscfgidx=%d (AWDL)\n", ifp->bsscfgidx); + + /* No brcmf_cfg80211_down(): this vif was never brought up through + * cfg80211, and tearing down the STA state would hit the same + * unresponsive-firmware path as the bring-up. + */ + brcmf_net_setcarrier(ifp, false); + return 0; +} + +static const struct net_device_ops brcmf_netdev_ops_awdl = { + .ndo_open = brcmf_netdev_open_awdl, + .ndo_stop = brcmf_netdev_stop_awdl, + .ndo_start_xmit = brcmf_netdev_start_xmit, + .ndo_set_mac_address = brcmf_netdev_set_mac_address, +}; + static const struct net_device_ops brcmf_netdev_ops_pri = { .ndo_open = brcmf_netdev_open, .ndo_stop = brcmf_netdev_stop, @@ -653,7 +702,8 @@ int brcmf_net_attach(struct brcmf_if *ifp, bool locked) ndev = ifp->ndev; /* set appropriate operations */ - ndev->netdev_ops = &brcmf_netdev_ops_pri; + ndev->netdev_ops = ifp->is_awdl ? &brcmf_netdev_ops_awdl + : &brcmf_netdev_ops_pri; ndev->needed_headroom += drvr->hdrlen; ndev->ethtool_ops = &brcmf_ethtool_ops; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h index 4b52a3aa855de..451bcf85a4bdf 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h @@ -268,6 +268,7 @@ struct brcmf_if { struct in6_addr ipv6_addr_tbl[NDOL_MAX_ENTRIES]; u8 ipv6addr_idx; bool fwil_fwerr; + bool is_awdl; }; int brcmf_netdev_wait_pend8021x(struct brcmf_if *ifp); From f58d4764e70d6976ce3411ab436b2aa22747e5e2 Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Thu, 27 Aug 2026 00:07:12 -0600 Subject: [PATCH 0010/2680] wifi: brcmfmac: return firmware error codes from the vendor dcmd brcmf_fil_cmd_data() collapses every firmware failure to -EBADE unless ifp->fwil_fwerr is set, so a userspace probe driving iovars through the Broadcom vendor dcmd passthrough cannot tell "no such iovar" from "bad argument" from "not up". Set fwil_fwerr around the call and log the raw BCME_* value. This makes the passthrough usable for firmware exploration: the same probe that previously reported EBADE for every failure now distinguishes BCME_BADOPTION, BCME_BUFTOOSHORT, BCME_NOTFOUND and BCME_NOTREADY. Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../wireless/broadcom/brcm80211/brcmfmac/vendor.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c index 6dd6a3d7231fe..d3211d092cf44 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c @@ -64,14 +64,25 @@ static int brcmf_cfg80211_vndr_cmds_dcmd_handler(struct wiphy *wiphy, *(char *)(dcmd_buf + len) = '\0'; } + /* Return the firmware's own BCME_* code instead of the -EBADE that + * brcmf_fil_cmd_data() collapses every failure to. Without this a + * probe cannot tell "no such iovar" from "bad argument" from "not + * up", which is most of what a probe needs to know. + */ + ifp->fwil_fwerr = true; if (cmdhdr->set) ret = brcmf_fil_cmd_data_set(ifp, cmdhdr->cmd, dcmd_buf, ret_len); else ret = brcmf_fil_cmd_data_get(ifp, cmdhdr->cmd, dcmd_buf, ret_len); - if (ret != 0) + ifp->fwil_fwerr = false; + if (ret != 0) { + brcmf_err("dcmd %u ifidx=%d %s: firmware error %d\n", + cmdhdr->cmd, ifp->ifidx, + cmdhdr->set ? "set" : "get", ret); goto exit; + } wr_pointer = dcmd_buf; while (ret_len > 0) { From 7d482e34ccd7d84e09cec78e731421ef6fd37015 Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Thu, 27 Aug 2026 00:07:12 -0600 Subject: [PATCH 0011/2680] wifi: brcmfmac: create the AWDL interface brcmf_cfg80211_request_awdl_if() sent interface_create with MAC_USE alone: no BSSID and no requested interface index. On BCM4387 that yields an interface whose AWDL iovars stay unusable. Send flags 0x1a (MAC_USE | IF_INDEX_USE | BSSID_USE) with an explicit BSSID and if_index 2 instead. Firmware behaviour changes with those arguments: the interface is created at the requested index (wl0.2), awdl_peer_op returns data instead of BCME_NOMEM, and awdl_af_hdr, awdl_config and awdl_af_rssi are accepted where they previously were not. The three values are module parameters so they can be swept without rebuilding. Derive a MAC when the caller passes NULL rather than sending MAC_USE with an all-zero address. Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../brcm80211/brcmfmac/interface_create.c | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c index e9ebb90030575..3a4f6b3be4470 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -22,6 +23,24 @@ #include "common.h" #include "interface_create.h" +/* Creating a usable AWDL interface requires interface_create flags + * MAC_USE | IF_INDEX_USE | BSSID_USE together with an explicit BSSID and + * an interface index; brcmfmac otherwise sends MAC_USE alone, which the + * firmware accepts but leaves the AWDL iovars unusable. These parameters + * exist so the three values can be swept without rebuilding the module. + */ +static uint brcmf_awdl_create_flags = 0x1a; +module_param_named(awdl_create_flags, brcmf_awdl_create_flags, uint, 0644); +MODULE_PARM_DESC(awdl_create_flags, "interface_create flags for AWDL"); + +static uint brcmf_awdl_if_index = 2; +module_param_named(awdl_if_index, brcmf_awdl_if_index, uint, 0644); +MODULE_PARM_DESC(awdl_if_index, "interface_create if_index for AWDL"); + +static char *brcmf_awdl_bssid = "00:25:00:ff:94:73"; +module_param_named(awdl_bssid, brcmf_awdl_bssid, charp, 0644); +MODULE_PARM_DESC(awdl_bssid, "interface_create BSSID for AWDL"); + #define BRCMF_INTERFACE_CREATE_VER_1 1 #define BRCMF_INTERFACE_CREATE_VER_2 2 #define BRCMF_INTERFACE_CREATE_VER_3 3 @@ -130,6 +149,8 @@ static int brcmf_cfg80211_request_if_internal(struct brcmf_if *ifp, u32 version, enum brcmf_interface_type if_type, u8 *macaddr) { + bool awdl = if_type == BRCMF_INTERFACE_TYPE_AWDL; + switch (version) { case BRCMF_INTERFACE_CREATE_VER_1: { struct brcmf_interface_create_v1 iface_v1 = {}; @@ -182,6 +203,27 @@ static int brcmf_cfg80211_request_if_internal(struct brcmf_if *ifp, u32 version, brcmf_set_vif_sta_macaddr(ifp, iface_v3.mac_addr); } + if (awdl) { + flags = brcmf_awdl_create_flags; + /* brcmf_awdl_add_vif() passes NULL, so derive a MAC + * here rather than send MAC_USE with an all-zero + * address. + */ + if ((flags & BRCMF_INTERFACE_MAC_USE) && + is_zero_ether_addr(iface_v3.mac_addr)) + brcmf_set_vif_sta_macaddr(ifp, + iface_v3.mac_addr); + iface_v3.if_index = brcmf_awdl_if_index; + if (mac_pton(brcmf_awdl_bssid, iface_v3.bssid)) { + brcmf_info("awdl interface_create: flags 0x%x, if_index %u, bssid %pM, mac %pM\n", + flags, iface_v3.if_index, + iface_v3.bssid, iface_v3.mac_addr); + } else { + bphy_err(ifp->drvr, "bad awdl_bssid \"%s\"\n", + brcmf_awdl_bssid); + return -EINVAL; + } + } iface_v3.flags = cpu_to_le32(flags); return brcmf_fil_iovar_data_get(ifp, "interface_create", &iface_v3, sizeof(iface_v3)); From dc9e2dadf104d4145ca99d7a23b8824016f24936 Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Thu, 27 Aug 2026 00:07:12 -0600 Subject: [PATCH 0012/2680] wifi: brcmfmac: do not tear down the AWDL netdev from the vendor command nl80211 calls vendor commands holding the wiphy mutex and not the RTNL (NL80211_CMD_VENDOR sets NEED_WIPHY without NO_WIPHY_MTX). Calling brcmf_remove_interface() from brcmf_awdl_del_vif() therefore reaches cfg80211_close_dependents(), which asserts the RTNL: RTNL: assertion failed at net/wireless/core.c (419) cfg80211_close_dependents brcmf_remove_interface brcmf_awdl_del_vif brcmf_cfg80211_vndr_cmds_awdl_handler and then deadlocks the netdev layer against any concurrent RTNL holder, leaving NetworkManager and ip(8) unkillable in D state. Ask the firmware to drop the BSS and return. The BRCMF_E_IF_DEL event lands in brcmf_fweh_event_worker(), a plain work item holding neither lock, and brcmf_fweh_handle_if_event() removes the interface from there. This mirrors brcmf_awdl_add_vif(), which defers brcmf_net_attach() to the same worker for the same reason. Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index c25000807b610..4663f2b42dfcd 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -725,27 +725,33 @@ void brcmf_cfg80211_awdl_attach_pending(struct brcmf_if *ifp) */ int brcmf_awdl_del_vif(struct wiphy *wiphy, struct wireless_dev *wdev) { - struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct brcmf_cfg80211_vif *vif = container_of(wdev, struct brcmf_cfg80211_vif, wdev); struct brcmf_if *ifp = vif->ifp; int err; - brcmf_cfg80211_arm_vif_event(cfg, vif); - err = brcmf_fil_bsscfg_data_set(ifp, "interface_remove", NULL, 0); - if (err) { - brcmf_cfg80211_arm_vif_event(cfg, NULL); - return err; - } - - err = brcmf_cfg80211_wait_vif_event(cfg, BRCMF_E_IF_DEL, - BRCMF_VIF_EVENT_TIMEOUT); - brcmf_cfg80211_arm_vif_event(cfg, NULL); - if (!err) - return -EIO; + /* Asymmetric with the rest of brcmfmac on purpose, for the same reason + * brcmf_awdl_add_vif() does not call brcmf_net_attach(): nl80211 calls + * vendor commands holding the wiphy mutex and NOT the RTNL + * (NL80211_CMD_VENDOR sets NEED_WIPHY without NO_WIPHY_MTX). Tearing + * the netdev down here reaches cfg80211_close_dependents(), which + * asserts the RTNL, and then deadlocks the whole netdev layer against + * any concurrent RTNL holder. Do NOT call brcmf_remove_interface() or + * arm the vif event from this context. + * + * Just ask the firmware to drop the BSS and return. The BRCMF_E_IF_DEL + * event lands in brcmf_fweh_event_worker(), which is a plain work item + * holding neither lock, and brcmf_fweh_handle_if_event() removes the + * interface from there. + */ + err = brcmf_fil_iovar_data_set(ifp, "interface_remove", NULL, 0); + if (err) + err = brcmf_fil_bsscfg_data_set(ifp, "interface_remove", NULL, 0); + if (err) + bphy_err(ifp->drvr, "interface_remove failed %d\n", err); - return 0; + return err; } static From ba3db410cc4161342c60b901c251e094cf66dc52 Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Sun, 6 Sep 2026 14:52:58 -0600 Subject: [PATCH 0013/2680] wifi: brcmfmac: translate AWDL data frames at the awdl0 boundary AWDL data frames on the air are 802.3 + LLC/SNAP (OUI 00:17:f2, protocol 0x0800) + a 6-byte data header (03 04 00 00) + the real ethertype. The kernel only speaks Ethernet II, so for is_awdl interfaces: - RX: brcmf_rx_eth_type_trans() strips the encapsulation in place before eth_type_trans(). It is called from both delivery paths -- brcmf_rx_hdrpull (BCDC) and the msgbuf RX completion (PCIe); a first attempt in hdrpull alone was dead code on PCIe. Frames that are already Ethernet II pass. - TX: brcmf_awdl_encap() wraps Ethernet II in the same header with a per-interface incrementing sequence; needed_headroom grows by 16. Frames that already carry a length field (raw AF_PACKET senders building the encapsulation themselves) pass untouched. Also log the destination MAC on each awdl0 TX completion so per-frame fates can be read per peer (unicast to a MAC without a firmware peer entry completes 0x0003 FW_TOSSED; with one, 0x0000 acked). With this, EUI-64 neighbour pinning and awdl_peer_op ADD, a remote peer completes TCP to us over firmware-native AWDL. Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../broadcom/brcm80211/brcmfmac/core.c | 76 ++++++++++++++++++- .../broadcom/brcm80211/brcmfmac/core.h | 4 + .../broadcom/brcm80211/brcmfmac/msgbuf.c | 50 ++++++++++-- 3 files changed, 121 insertions(+), 9 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 4a137454f8945..a5e0df79191fe 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,17 @@ #define D11_PHY_HDR_LEN 6 +/* AWDL data frames on the air are 802.3 + LLC/SNAP (OUI 00:17:f2, protocol + * 0x0800) + a 4-byte data header, then the real ethertype. The kernel only + * speaks Ethernet II, so the driver translates at the awdl0 boundary in + * both directions. + */ +#define BRCMF_AWDL_ENCAP_LEN 16 +#define BRCMF_AWDL_DATA_MIN_LEN (ETH_HLEN + BRCMF_AWDL_ENCAP_LEN) + +static const u8 brcmf_awdl_snap[] = { 0xaa, 0xaa, 0x03, 0x00, 0x17, 0xf2, + 0x08, 0x00, 0x03, 0x04 }; + struct d11rxhdr_le { __le16 RxFrameSize; u16 PAD; @@ -290,6 +302,55 @@ static bool brcmf_skb_is_iapp(struct sk_buff *skb) #endif } +/* Unwrap SNAP + AWDL data header into Ethernet II, in place. skb->data is + * the 802.3 header. Frames that are not AWDL-encapsulated pass unchanged. + */ +static void brcmf_awdl_decap(struct sk_buff *skb) +{ + const struct ethhdr *eh = (const struct ethhdr *)skb->data; + + if (skb->len < BRCMF_AWDL_DATA_MIN_LEN || + ntohs(eh->h_proto) > ETH_DATA_LEN || + memcmp(skb->data + ETH_HLEN, brcmf_awdl_snap, + sizeof(brcmf_awdl_snap))) + return; + + memmove(skb->data + BRCMF_AWDL_ENCAP_LEN, skb->data, 2 * ETH_ALEN); + skb_pull(skb, BRCMF_AWDL_ENCAP_LEN); +} + +/* Every received data frame passes through here, on the BCDC and the msgbuf + * (PCIe) paths alike, before the kernel classifies it. + */ +__be16 brcmf_rx_eth_type_trans(struct brcmf_if *ifp, struct sk_buff *skb) +{ + if (ifp->is_awdl) + brcmf_awdl_decap(skb); + return eth_type_trans(skb, ifp->ndev); +} + +/* Wrap Ethernet II into 802.3 + SNAP + AWDL data header, in place. Frames + * already carrying a length field (raw AF_PACKET senders that build the + * encapsulation themselves) pass unchanged. Caller guarantees headroom. + */ +static void brcmf_awdl_encap(struct brcmf_if *ifp, struct sk_buff *skb) +{ + const struct ethhdr *eh = (const struct ethhdr *)skb->data; + u8 *hdr; + + if (ntohs(eh->h_proto) < ETH_P_802_3_MIN) + return; + + hdr = skb_push(skb, BRCMF_AWDL_ENCAP_LEN); + memmove(hdr, hdr + BRCMF_AWDL_ENCAP_LEN, 2 * ETH_ALEN); + put_unaligned_be16(skb->len - ETH_HLEN, hdr + 2 * ETH_ALEN); + memcpy(hdr + ETH_HLEN, brcmf_awdl_snap, sizeof(brcmf_awdl_snap)); + put_unaligned_le16(ifp->awdl_tx_seq++, + hdr + ETH_HLEN + sizeof(brcmf_awdl_snap)); + hdr[ETH_HLEN + sizeof(brcmf_awdl_snap) + 2] = 0; + hdr[ETH_HLEN + sizeof(brcmf_awdl_snap) + 3] = 0; +} + static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, struct net_device *ndev) { @@ -298,6 +359,7 @@ static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, struct brcmf_pub *drvr = ifp->drvr; struct ethhdr *eh; int head_delta; + unsigned int hdrlen = drvr->hdrlen; unsigned int tx_bytes = skb->len; brcmf_dbg(DATA, "Enter, bsscfgidx=%d\n", ifp->bsscfgidx); @@ -328,9 +390,12 @@ static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, goto done; } + if (ifp->is_awdl) + hdrlen += BRCMF_AWDL_ENCAP_LEN; + /* Make sure there's enough writeable headroom */ - if (skb_headroom(skb) < drvr->hdrlen || skb_header_cloned(skb)) { - head_delta = max_t(int, drvr->hdrlen - skb_headroom(skb), 0); + if (skb_headroom(skb) < hdrlen || skb_header_cloned(skb)) { + head_delta = max_t(int, hdrlen - skb_headroom(skb), 0); brcmf_dbg(INFO, "%s: %s headroom\n", brcmf_ifname(ifp), head_delta ? "insufficient" : "unmodifiable"); @@ -365,6 +430,9 @@ static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, /* set pacing shift for packet aggregation */ sk_pacing_shift_update(skb->sk, 8); + if (ifp->is_awdl) + brcmf_awdl_encap(ifp, skb); + ret = brcmf_proto_tx_queue_data(drvr, ifp->ifidx, skb); if (ret < 0) brcmf_txfinalize(ifp, skb, false); @@ -495,7 +563,7 @@ static int brcmf_rx_hdrpull(struct brcmf_pub *drvr, struct sk_buff *skb, return -ENODATA; } - skb->protocol = eth_type_trans(skb, (*ifp)->ndev); + skb->protocol = brcmf_rx_eth_type_trans(*ifp, skb); return 0; } @@ -706,6 +774,8 @@ int brcmf_net_attach(struct brcmf_if *ifp, bool locked) : &brcmf_netdev_ops_pri; ndev->needed_headroom += drvr->hdrlen; + if (ifp->is_awdl) + ndev->needed_headroom += BRCMF_AWDL_ENCAP_LEN; ndev->ethtool_ops = &brcmf_ethtool_ops; /* set the mac address & netns */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h index 451bcf85a4bdf..86c9ef09c9090 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h @@ -250,6 +250,8 @@ enum brcmf_netif_stop_reason { * @pend_8021x_cnt: tracks outstanding number of 802.1x frames. * @pend_8021x_wait: used for signalling change in count. * @fwil_fwerr: flag indicating fwil layer should return firmware error codes. + * @is_awdl: interface carries AWDL data frames (SNAP + AWDL data header). + * @awdl_tx_seq: sequence number written into the AWDL data header on TX. */ struct brcmf_if { struct brcmf_pub *drvr; @@ -269,6 +271,7 @@ struct brcmf_if { u8 ipv6addr_idx; bool fwil_fwerr; bool is_awdl; + u16 awdl_tx_seq; }; int brcmf_netdev_wait_pend8021x(struct brcmf_if *ifp); @@ -285,6 +288,7 @@ void brcmf_txflowblock_if(struct brcmf_if *ifp, enum brcmf_netif_stop_reason reason, bool state); void brcmf_txfinalize(struct brcmf_if *ifp, struct sk_buff *txp, bool success); void brcmf_netif_rx(struct brcmf_if *ifp, struct sk_buff *skb); +__be16 brcmf_rx_eth_type_trans(struct brcmf_if *ifp, struct sk_buff *skb); void brcmf_netif_mon_rx(struct brcmf_if *ifp, struct sk_buff *skb); void brcmf_net_detach(struct net_device *ndev, bool locked); int brcmf_net_mon_attach(struct brcmf_if *ifp); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index eb3985a85ca25..869018de9ebf3 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -912,11 +913,22 @@ brcmf_msgbuf_process_ioctl_complete(struct brcmf_msgbuf *msgbuf, void *buf) } +/* Firmware-reported completion status of AWDL data frames. brcmfmac never + * looks at tx_status; it is the WLFC_CTL_PKTFLAG_* fate of the frame + * (0 acked, 4 sent-no-ack i.e. multicast on air, 7 dropped by the firmware). + * Observed: awdl0 multicast completes ~1:3 sent:dropped, template or not. + * Logged for every frame queued on a non-primary flowring. 2000/s so a + * 200-frame burst at 1000/s is logged in full and in order -- the sequence + * of fates is the measurement. + */ +static DEFINE_RATELIMIT_STATE(brcmf_awdl_txs_rs, HZ, 2000); + static void brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) { struct brcmf_commonring *commonring; struct msgbuf_tx_status *tx_status; + struct brcmf_if *ifp; u32 idx; struct sk_buff *skb; u16 flowid; @@ -934,8 +946,23 @@ brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) commonring = msgbuf->flowrings[flowid]; atomic_dec(&commonring->outstanding_tx); - brcmf_txfinalize(brcmf_get_ifp(msgbuf->drvr, tx_status->msg.ifidx), - skb, true); + ifp = brcmf_get_ifp(msgbuf->drvr, tx_status->msg.ifidx); + /* Key on the flowring's ifidx, which the host assigned when it queued + * the frame, not on what the firmware echoes: repeated runs + * logged nothing for 1227 awdl0 frames each, and a completion + * echoed under ifidx 0 would have been invisible to both filters. + */ + if (brcmf_flowring_ifidx_get(msgbuf->flow, flowid) && + __ratelimit(&brcmf_awdl_txs_rs)) + pr_info("brcmfmac: awdl txstatus ring_ifidx=%u msg_ifidx=%u awdl=%d flow=%u status=%d tx_status=0x%04x meta=%u dst=%pM\n", + brcmf_flowring_ifidx_get(msgbuf->flow, flowid), + tx_status->msg.ifidx, ifp ? ifp->is_awdl : -1, flowid, + (int)(s16)le16_to_cpu(tx_status->compl_hdr.status), + le16_to_cpu(tx_status->tx_status), + le16_to_cpu(tx_status->metadata_len), + skb->len >= ETH_ALEN ? skb->data : (u8 *)"\0\0\0\0\0\0"); + + brcmf_txfinalize(ifp, skb, true); } @@ -1236,7 +1263,7 @@ brcmf_msgbuf_process_rx_complete(struct brcmf_msgbuf *msgbuf, void *buf) return; } - skb->protocol = eth_type_trans(skb, ifp->ndev); + skb->protocol = brcmf_rx_eth_type_trans(ifp, skb); brcmf_netif_rx(ifp, skb); } @@ -1584,14 +1611,25 @@ static int brcmf_msgbuf_stats_read(struct seq_file *seq, void *data) if (!msgbuf->flow->rings[i]) continue; ring = msgbuf->flow->rings[i]; - if (ring->status != RING_OPEN) + hash = &msgbuf->flow->hash[ring->hash_id]; + if (ring->status != RING_OPEN) { + /* A ring the firmware has not acknowledged holds every + * frame queued to it, invisibly, while tx_packets + * advances. Show it rather than skip it. + */ + seq_printf(seq, "id %3u: status %d (not open), qlen %4u\n" + " ifidx %u, fifo %u, da %pM\n", + i, ring->status, + skb_queue_len(&ring->skblist), + hash->ifidx, hash->fifo, hash->mac); continue; + } commonring = msgbuf->flowrings[i]; - hash = &msgbuf->flow->hash[ring->hash_id]; - seq_printf(seq, "id %3u: rp %4u, wp %4u, qlen %4u, blocked %u\n" + seq_printf(seq, "id %3u: rp %4u, wp %4u, qlen %4u, blocked %u, outstanding_tx %d\n" " ifidx %u, fifo %u, da %pM\n", i, commonring->r_ptr, commonring->w_ptr, skb_queue_len(&ring->skblist), ring->blocked, + atomic_read(&commonring->outstanding_tx), hash->ifidx, hash->fifo, hash->mac); } From 94112a7e684b8e7520a1e66dfb41ae300f6b49c6 Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Mon, 7 Sep 2026 01:28:47 -0600 Subject: [PATCH 0014/2680] brcmfmac: msgbuf: tolerate txstatus for a freed flowring brcmf_msgbuf_delete_flowring() gives up on outstanding frames after ~75 ms and frees the ring; the firmware then completes them (tx_status 0x0006 EXPIRED) against a flow->rings[] slot that is already NULL. The awdl txstatus log line dereferenced it (oops in brcmf_flowring_ifidx_get from brcmf_msgbuf_process_rx on awdl-if destroy). Look the ring up only if it still exists, and reject a flowring id beyond max_flowrings before indexing. Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../broadcom/brcm80211/brcmfmac/msgbuf.c | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index 869018de9ebf3..1df1a1592ca4f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -932,6 +932,7 @@ brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) u32 idx; struct sk_buff *skb; u16 flowid; + u8 ring_ifidx; tx_status = (struct msgbuf_tx_status *)buf; idx = le32_to_cpu(tx_status->msg.request_id) - 1; @@ -942,6 +943,13 @@ brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) if (!skb) return; + if (flowid >= msgbuf->max_flowrings) { + bphy_err(msgbuf->drvr, "txstatus for flowring %u out of range\n", + flowid); + brcmu_pkt_buf_free_skb(skb); + return; + } + set_bit(flowid, msgbuf->txstatus_done_map); commonring = msgbuf->flowrings[flowid]; atomic_dec(&commonring->outstanding_tx); @@ -951,11 +959,18 @@ brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) * the frame, not on what the firmware echoes: repeated runs * logged nothing for 1227 awdl0 frames each, and a completion * echoed under ifidx 0 would have been invisible to both filters. + * + * The ring may already be gone: brcmf_msgbuf_delete_flowring() gives up + * on outstanding frames after ~75 ms and the firmware completes them + * (0x0006 EXPIRED) after the ring was freed -- awdl0's VO multicast + * never completes while the ring is open (observed as a NULL deref + * here on `awdl-if destroy`). The skb and its pktid are unaffected. */ - if (brcmf_flowring_ifidx_get(msgbuf->flow, flowid) && - __ratelimit(&brcmf_awdl_txs_rs)) + ring_ifidx = msgbuf->flow->rings[flowid] ? + brcmf_flowring_ifidx_get(msgbuf->flow, flowid) : 0; + if (ring_ifidx && __ratelimit(&brcmf_awdl_txs_rs)) pr_info("brcmfmac: awdl txstatus ring_ifidx=%u msg_ifidx=%u awdl=%d flow=%u status=%d tx_status=0x%04x meta=%u dst=%pM\n", - brcmf_flowring_ifidx_get(msgbuf->flow, flowid), + ring_ifidx, tx_status->msg.ifidx, ifp ? ifp->is_awdl : -1, flowid, (int)(s16)le16_to_cpu(tx_status->compl_hdr.status), le16_to_cpu(tx_status->tx_status), From e2aec1038fcd70942585a70db33572855e50fb48 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:43:10 +1100 Subject: [PATCH 0015/2680] wifi: brcmfmac: bound AWDL creation and validate its data path Use the vif-event lock to arbitrate completion and timeout of asynchronous AWDL creation. Defer cleanup of a failed registration to wiphy work, and drain event and wiphy work before protocol teardown. Keep the netdevice reference and validate its wiphy and AWDL role when requesting deletion. Retain the vendor dcmd errno ABI rather than toggling shared fwil error state around commands. Account for the 16-byte AWDL envelope in the MTU and reject oversized Ethernet-II frames before writing an invalid 802.3 length. Route the per-frame measurement trace through dynamic debug. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 150 +++++++++++++++--- .../broadcom/brcm80211/brcmfmac/cfg80211.h | 7 + .../broadcom/brcm80211/brcmfmac/core.c | 30 +++- .../broadcom/brcm80211/brcmfmac/msgbuf.c | 14 +- .../broadcom/brcm80211/brcmfmac/vendor.c | 23 +-- 5 files changed, 177 insertions(+), 47 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 4663f2b42dfcd..f9b390ff8dc09 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -605,6 +605,85 @@ brcmf_cfg80211_update_proto_addr_mode(struct wireless_dev *wdev) * * Return: pointer to new vif on success, ERR_PTR(-errno) if not */ +static void brcmf_awdl_cleanup_locked(struct brcmf_cfg80211_info *cfg) +{ + struct brcmf_cfg80211_vif *vif; + struct brcmf_if *ifp; + + spin_lock(&cfg->vif_event.vif_event_lock); + ifp = cfg->awdl_failed_ifp; + vif = cfg->awdl_vif; + cfg->awdl_failed_ifp = NULL; + cfg->awdl_vif = NULL; + WRITE_ONCE(cfg->awdl_pending, false); + cfg->awdl_attaching = false; + if (cfg->vif_event.vif == vif) { + cfg->vif_event.vif = NULL; + cfg->vif_event.action = 0; + } + spin_unlock(&cfg->vif_event.vif_event_lock); + + if (ifp) { + brcmf_fil_iovar_data_set(ifp, "interface_remove", NULL, 0); + brcmf_proto_del_if(cfg->pub, ifp); + ifp->vif = NULL; + } + if (vif) + brcmf_free_vif(vif); + if (ifp) { + if (ifp->ndev) { + ifp->ndev->ieee80211_ptr = NULL; + free_netdev(ifp->ndev); + } else { + kfree(ifp); + } + } +} + +static void brcmf_awdl_cleanup(struct wiphy *wiphy, struct wiphy_work *work) +{ + struct brcmf_cfg80211_info *cfg = + container_of(work, struct brcmf_cfg80211_info, awdl_cleanup); + + brcmf_awdl_cleanup_locked(cfg); +} + +void brcmf_cfg80211_awdl_stop(struct brcmf_cfg80211_info *cfg) +{ + wiphy_lock(cfg->wiphy); + wiphy_delayed_work_cancel(cfg->wiphy, &cfg->awdl_timeout); + wiphy_work_cancel(cfg->wiphy, &cfg->awdl_cleanup); + brcmf_awdl_cleanup_locked(cfg); + wiphy_unlock(cfg->wiphy); +} + +static void brcmf_awdl_timeout(struct wiphy *wiphy, struct wiphy_work *work) +{ + struct brcmf_cfg80211_info *cfg = + container_of(work, struct brcmf_cfg80211_info, awdl_timeout.work); + struct brcmf_cfg80211_vif *vif; + + spin_lock(&cfg->vif_event.vif_event_lock); + vif = cfg->awdl_vif; + /* IF_ADD has already claimed the vif; its worker owns completion. */ + if (!cfg->awdl_pending || cfg->awdl_attaching || + (vif && vif->ifp)) { + spin_unlock(&cfg->vif_event.vif_event_lock); + return; + } + cfg->awdl_vif = NULL; + WRITE_ONCE(cfg->awdl_pending, false); + if (cfg->vif_event.vif == vif) { + cfg->vif_event.vif = NULL; + cfg->vif_event.action = 0; + } + spin_unlock(&cfg->vif_event.vif_event_lock); + + brcmf_err("AWDL interface creation timed out\n"); + if (vif) + brcmf_free_vif(vif); +} + /** * brcmf_awdl_add_vif() - create an AWDL interface and give it a netdev. * @@ -621,7 +700,7 @@ brcmf_cfg80211_update_proto_addr_mode(struct wireless_dev *wdev) * valid type for its bookkeeping; the firmware side is what actually decides * the interface behaviour, from the iftype passed to interface_create. * - * Return: pointer to the new vif on success, ERR_PTR(-errno) on failure. + * Return: 0 when requested, or a negative error code. */ int brcmf_awdl_add_vif(struct wiphy *wiphy, const char *name) { @@ -630,7 +709,10 @@ int brcmf_awdl_add_vif(struct wiphy *wiphy, const char *name) struct brcmf_cfg80211_vif *vif; int err; - if (brcmf_cfg80211_vif_event_armed(cfg) || cfg->awdl_pending) + if (cfg->pub->bus_if->state != BRCMF_BUS_UP) + return -ENODEV; + if (brcmf_cfg80211_vif_event_armed(cfg) || + READ_ONCE(cfg->awdl_pending)) return -EBUSY; vif = brcmf_alloc_vif(cfg, NL80211_IFTYPE_STATION); @@ -650,13 +732,17 @@ int brcmf_awdl_add_vif(struct wiphy *wiphy, const char *name) * */ strscpy(cfg->awdl_ifname, name, sizeof(cfg->awdl_ifname)); - cfg->awdl_pending = true; + spin_lock(&cfg->vif_event.vif_event_lock); + cfg->awdl_vif = vif; + WRITE_ONCE(cfg->awdl_pending, true); + spin_unlock(&cfg->vif_event.vif_event_lock); + wiphy_delayed_work_queue(wiphy, &cfg->awdl_timeout, + BRCMF_VIF_EVENT_TIMEOUT); err = brcmf_cfg80211_request_awdl_if(ifp, NULL); if (err) { - cfg->awdl_pending = false; - brcmf_cfg80211_arm_vif_event(cfg, NULL); - brcmf_free_vif(vif); + /* An IF_ADD may already be in flight: retire under the same lock. */ + wiphy_delayed_work_queue(wiphy, &cfg->awdl_timeout, 0); return err; } @@ -688,31 +774,54 @@ void brcmf_cfg80211_awdl_attach_pending(struct brcmf_if *ifp) struct brcmf_cfg80211_vif *vif; int err; - if (!cfg || !cfg->awdl_pending) + if (!cfg) return; - cfg->awdl_pending = false; - vif = ifp->vif; + spin_lock(&cfg->vif_event.vif_event_lock); + vif = cfg->awdl_vif; + if (!cfg->awdl_pending || cfg->awdl_attaching || !vif || + ifp->vif != vif) { + spin_unlock(&cfg->vif_event.vif_event_lock); + return; + } + cfg->awdl_attaching = true; + spin_unlock(&cfg->vif_event.vif_event_lock); - if (!ifp->ndev || !vif) { - bphy_err(ifp->drvr, "AWDL if event without netdev/vif\n"); - goto done; + if (!ifp->ndev) { + bphy_err(ifp->drvr, "AWDL if event without netdev\n"); + goto fail; } - /* Must be set before brcmf_net_attach(), which picks netdev_ops. */ ifp->is_awdl = true; strscpy(ifp->ndev->name, cfg->awdl_ifname, sizeof(ifp->ndev->name)); err = brcmf_net_attach(ifp, false); if (err) { bphy_err(ifp->drvr, "registering AWDL netdevice failed\n"); - free_netdev(ifp->ndev); - brcmf_free_vif(vif); - goto done; + goto fail; } + spin_lock(&cfg->vif_event.vif_event_lock); + cfg->awdl_vif = NULL; + WRITE_ONCE(cfg->awdl_pending, false); + cfg->awdl_attaching = false; + if (cfg->vif_event.vif == vif) { + cfg->vif_event.vif = NULL; + cfg->vif_event.action = 0; + } + spin_unlock(&cfg->vif_event.vif_event_lock); brcmf_info("AWDL interface %s created (bsscfgidx %d)\n", ifp->ndev->name, ifp->bsscfgidx); -done: - brcmf_cfg80211_arm_vif_event(cfg, NULL); + return; + +fail: + /* Do not free a vif while an event or wiphy operation can still see it. */ + spin_lock(&cfg->vif_event.vif_event_lock); + if (cfg->pub->iflist[ifp->bsscfgidx] == ifp) + cfg->pub->iflist[ifp->bsscfgidx] = NULL; + cfg->awdl_failed_ifp = ifp; + cfg->vif_event.vif = NULL; + cfg->vif_event.action = 0; + spin_unlock(&cfg->vif_event.vif_event_lock); + wiphy_work_queue(cfg->wiphy, &cfg->awdl_cleanup); } /** @@ -731,6 +840,9 @@ int brcmf_awdl_del_vif(struct wiphy *wiphy, struct wireless_dev *wdev) struct brcmf_if *ifp = vif->ifp; int err; + if (wdev->wiphy != wiphy || !ifp || !ifp->is_awdl) + return -ENODEV; + /* Asymmetric with the rest of brcmfmac on purpose, for the same reason * brcmf_awdl_add_vif() does not call brcmf_net_attach(): nl80211 calls * vendor commands holding the wiphy mutex and NOT the RTNL @@ -8728,6 +8840,8 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, cfg->wiphy = wiphy; cfg->pub = drvr; init_vif_event(&cfg->vif_event); + wiphy_delayed_work_init(&cfg->awdl_timeout, brcmf_awdl_timeout); + wiphy_work_init(&cfg->awdl_cleanup, brcmf_awdl_cleanup); INIT_LIST_HEAD(&cfg->vif_list); cfg->force_band_setup = true; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index fdc9f7e627f0a..ce00102086f67 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -397,7 +397,13 @@ struct brcmf_cfg80211_info { struct work_struct escan_timeout_work; struct list_head vif_list; struct brcmf_cfg80211_vif_event vif_event; + /* Protected by vif_event.vif_event_lock. */ bool awdl_pending; + bool awdl_attaching; + struct brcmf_cfg80211_vif *awdl_vif; + struct brcmf_if *awdl_failed_ifp; + struct wiphy_delayed_work awdl_timeout; + struct wiphy_work awdl_cleanup; char awdl_ifname[IFNAMSIZ]; struct completion vif_disabled; struct brcmu_d11inf d11inf; @@ -510,6 +516,7 @@ bool brcmf_get_vif_state_any(struct brcmf_cfg80211_info *cfg, void brcmf_cfg80211_arm_vif_event(struct brcmf_cfg80211_info *cfg, struct brcmf_cfg80211_vif *vif); bool brcmf_cfg80211_vif_event_armed(struct brcmf_cfg80211_info *cfg); +void brcmf_cfg80211_awdl_stop(struct brcmf_cfg80211_info *cfg); int brcmf_awdl_add_vif(struct wiphy *wiphy, const char *name); void brcmf_cfg80211_awdl_attach_pending(struct brcmf_if *ifp); int brcmf_awdl_del_vif(struct wiphy *wiphy, struct wireless_dev *wdev); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index a5e0df79191fe..891439493a5c0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -333,13 +333,15 @@ __be16 brcmf_rx_eth_type_trans(struct brcmf_if *ifp, struct sk_buff *skb) * already carrying a length field (raw AF_PACKET senders that build the * encapsulation themselves) pass unchanged. Caller guarantees headroom. */ -static void brcmf_awdl_encap(struct brcmf_if *ifp, struct sk_buff *skb) +static int brcmf_awdl_encap(struct brcmf_if *ifp, struct sk_buff *skb) { const struct ethhdr *eh = (const struct ethhdr *)skb->data; u8 *hdr; if (ntohs(eh->h_proto) < ETH_P_802_3_MIN) - return; + return 0; + if (skb->len - ETH_HLEN + BRCMF_AWDL_ENCAP_LEN > ETH_DATA_LEN) + return -EMSGSIZE; hdr = skb_push(skb, BRCMF_AWDL_ENCAP_LEN); memmove(hdr, hdr + BRCMF_AWDL_ENCAP_LEN, 2 * ETH_ALEN); @@ -349,6 +351,7 @@ static void brcmf_awdl_encap(struct brcmf_if *ifp, struct sk_buff *skb) hdr + ETH_HLEN + sizeof(brcmf_awdl_snap)); hdr[ETH_HLEN + sizeof(brcmf_awdl_snap) + 2] = 0; hdr[ETH_HLEN + sizeof(brcmf_awdl_snap) + 3] = 0; + return 0; } static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, @@ -430,8 +433,13 @@ static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, /* set pacing shift for packet aggregation */ sk_pacing_shift_update(skb->sk, 8); - if (ifp->is_awdl) - brcmf_awdl_encap(ifp, skb); + if (ifp->is_awdl) { + ret = brcmf_awdl_encap(ifp, skb); + if (ret) { + brcmf_txfinalize(ifp, skb, false); + goto done; + } + } ret = brcmf_proto_tx_queue_data(drvr, ifp->ifidx, skb); if (ret < 0) @@ -774,8 +782,12 @@ int brcmf_net_attach(struct brcmf_if *ifp, bool locked) : &brcmf_netdev_ops_pri; ndev->needed_headroom += drvr->hdrlen; - if (ifp->is_awdl) + if (ifp->is_awdl) { ndev->needed_headroom += BRCMF_AWDL_ENCAP_LEN; + /* The 802.3 length includes the AWDL envelope. */ + ndev->max_mtu = ETH_DATA_LEN - BRCMF_AWDL_ENCAP_LEN; + ndev->mtu = ndev->max_mtu; + } ndev->ethtool_ops = &brcmf_ethtool_ops; /* set the mac address & netns */ @@ -1614,6 +1626,12 @@ void brcmf_detach(struct device *dev) #endif brcmf_bus_change_state(bus_if, BRCMF_BUS_DOWN); + /* Stop event producers before draining their worker. */ + brcmf_bus_stop(drvr->bus_if); + /* Drain IF events before their interfaces and protocol state go away. */ + brcmf_fweh_detach(drvr); + if (drvr->config) + brcmf_cfg80211_awdl_stop(drvr->config); /* make sure primary interface removed last */ for (i = BRCMF_MAX_IFS - 1; i > -1; i--) { struct brcmf_if *ifp = drvr->iflist[i]; @@ -1642,9 +1660,7 @@ void brcmf_detach(struct device *dev) wiphy_unlock(drvr->wiphy); rtnl_unlock(); } - brcmf_bus_stop(drvr->bus_if); - brcmf_fweh_detach(drvr); brcmf_proto_detach(drvr); if (drvr->mon_if) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index 1df1a1592ca4f..c3dfd76c4dbfa 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -969,13 +969,13 @@ brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) ring_ifidx = msgbuf->flow->rings[flowid] ? brcmf_flowring_ifidx_get(msgbuf->flow, flowid) : 0; if (ring_ifidx && __ratelimit(&brcmf_awdl_txs_rs)) - pr_info("brcmfmac: awdl txstatus ring_ifidx=%u msg_ifidx=%u awdl=%d flow=%u status=%d tx_status=0x%04x meta=%u dst=%pM\n", - ring_ifidx, - tx_status->msg.ifidx, ifp ? ifp->is_awdl : -1, flowid, - (int)(s16)le16_to_cpu(tx_status->compl_hdr.status), - le16_to_cpu(tx_status->tx_status), - le16_to_cpu(tx_status->metadata_len), - skb->len >= ETH_ALEN ? skb->data : (u8 *)"\0\0\0\0\0\0"); + brcmf_dbg(MSGBUF, "awdl txstatus ring_ifidx=%u msg_ifidx=%u awdl=%d flow=%u status=%d tx_status=0x%04x meta=%u dst=%pM\n", + ring_ifidx, tx_status->msg.ifidx, + ifp ? ifp->is_awdl : -1, flowid, + (int)(s16)le16_to_cpu(tx_status->compl_hdr.status), + le16_to_cpu(tx_status->tx_status), + le16_to_cpu(tx_status->metadata_len), + skb->len >= ETH_ALEN ? skb->data : (u8 *)"\0\0\0\0\0\0"); brcmf_txfinalize(ifp, skb, true); } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c index d3211d092cf44..25623533effa7 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c @@ -64,25 +64,14 @@ static int brcmf_cfg80211_vndr_cmds_dcmd_handler(struct wiphy *wiphy, *(char *)(dcmd_buf + len) = '\0'; } - /* Return the firmware's own BCME_* code instead of the -EBADE that - * brcmf_fil_cmd_data() collapses every failure to. Without this a - * probe cannot tell "no such iovar" from "bad argument" from "not - * up", which is most of what a probe needs to know. - */ - ifp->fwil_fwerr = true; if (cmdhdr->set) ret = brcmf_fil_cmd_data_set(ifp, cmdhdr->cmd, dcmd_buf, ret_len); else ret = brcmf_fil_cmd_data_get(ifp, cmdhdr->cmd, dcmd_buf, ret_len); - ifp->fwil_fwerr = false; - if (ret != 0) { - brcmf_err("dcmd %u ifidx=%d %s: firmware error %d\n", - cmdhdr->cmd, ifp->ifidx, - cmdhdr->set ? "set" : "get", ret); + if (ret != 0) goto exit; - } wr_pointer = dcmd_buf; while (ret_len > 0) { @@ -124,6 +113,7 @@ static int brcmf_cfg80211_vndr_cmds_awdl_handler(struct wiphy *wiphy, struct net_device *ndev; struct wireless_dev *awdl_wdev; u32 op; + int err; if (len < sizeof(op)) return -EINVAL; @@ -146,10 +136,13 @@ static int brcmf_cfg80211_vndr_cmds_awdl_handler(struct wiphy *wiphy, if (!ndev) return -ENODEV; awdl_wdev = ndev->ieee80211_ptr; - dev_put(ndev); - if (!awdl_wdev) + if (!awdl_wdev || awdl_wdev->wiphy != wiphy) { + dev_put(ndev); return -ENODEV; - return brcmf_awdl_del_vif(wiphy, awdl_wdev); + } + err = brcmf_awdl_del_vif(wiphy, awdl_wdev); + dev_put(ndev); + return err; default: return -EINVAL; } From 3d0157567facb40c0c8d9168a5bf547de2a1cf37 Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Thu, 8 Oct 2026 18:28:29 -0600 Subject: [PATCH 0016/2680] wifi: brcmfmac: log AWDL tx completions behind a runtime parameter Commit bc1823a897a7 ("wifi: brcmfmac: bound AWDL creation and validate its data path") moved the per-frame AWDL tx-completion line from pr_info() to brcmf_dbg(MSGBUF). brcmf_dbg() only prints with DEBUG or CONFIG_BRCM_TRACING; with neither, as in the linux-aurora config, it is no_printk() and the line is not compiled in at all. That line is not only a measurement trace. omdrop-discoverable sends a burst of frames on awdl0 and counts these completions to decide whether the firmware is transmitting. A firmware that has parked the AWDL data path parses peers, fills its peer table and elects normally while completing nothing, so no other signal tells the two apart. Without the line, every window fails the check. Add an awdl_trace module parameter, off by default and writable at runtime (0644), and print the completion only while it is set. A default build stays quiet, and userspace turns the trace on only for the seconds it needs it. This is the same parameter the out-of-tree omdrop-awdl driver already carries, so userspace works with either module. Fixes: bc1823a897a7 ("wifi: brcmfmac: bound AWDL creation and validate its data path") Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../broadcom/brcm80211/brcmfmac/common.c | 9 +++++++++ .../broadcom/brcm80211/brcmfmac/debug.h | 1 + .../broadcom/brcm80211/brcmfmac/msgbuf.c | 17 +++++++++-------- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index e3a9ca5526c45..2c99a99031568 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -43,6 +43,15 @@ int brcmf_msg_level; module_param_named(debug, brcmf_msg_level, int, 0600); MODULE_PARM_DESC(debug, "Level of debug output"); +/* AWDL tx completions are logged per frame. Userspace counts them to tell a + * radio that is transmitting from one whose firmware has parked the AWDL data + * path: a parked radio completes nothing, and it looks healthy by every other + * measure. Off by default, writable at runtime so no reload is needed. + */ +int brcmf_awdl_trace; +module_param_named(awdl_trace, brcmf_awdl_trace, int, 0644); +MODULE_PARM_DESC(awdl_trace, "Log AWDL tx completions"); + static int brcmf_p2p_enable; module_param_named(p2pon, brcmf_p2p_enable, int, 0); MODULE_PARM_DESC(p2pon, "Enable legacy p2p management functionality"); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h index 432d93ae8fb85..4c7bd41c704c0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h @@ -117,6 +117,7 @@ do { \ } while (0) extern int brcmf_msg_level; +extern int brcmf_awdl_trace; struct brcmf_pub; #ifdef DEBUG diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index c3dfd76c4dbfa..c9dcba08072a3 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -968,14 +968,15 @@ brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) */ ring_ifidx = msgbuf->flow->rings[flowid] ? brcmf_flowring_ifidx_get(msgbuf->flow, flowid) : 0; - if (ring_ifidx && __ratelimit(&brcmf_awdl_txs_rs)) - brcmf_dbg(MSGBUF, "awdl txstatus ring_ifidx=%u msg_ifidx=%u awdl=%d flow=%u status=%d tx_status=0x%04x meta=%u dst=%pM\n", - ring_ifidx, tx_status->msg.ifidx, - ifp ? ifp->is_awdl : -1, flowid, - (int)(s16)le16_to_cpu(tx_status->compl_hdr.status), - le16_to_cpu(tx_status->tx_status), - le16_to_cpu(tx_status->metadata_len), - skb->len >= ETH_ALEN ? skb->data : (u8 *)"\0\0\0\0\0\0"); + if (ring_ifidx && READ_ONCE(brcmf_awdl_trace) && + __ratelimit(&brcmf_awdl_txs_rs)) + pr_info("brcmfmac: awdl txstatus ring_ifidx=%u msg_ifidx=%u awdl=%d flow=%u status=%d tx_status=0x%04x meta=%u dst=%pM\n", + ring_ifidx, tx_status->msg.ifidx, + ifp ? ifp->is_awdl : -1, flowid, + (int)(s16)le16_to_cpu(tx_status->compl_hdr.status), + le16_to_cpu(tx_status->tx_status), + le16_to_cpu(tx_status->metadata_len), + skb->len >= ETH_ALEN ? skb->data : (u8 *)"\0\0\0\0\0\0"); brcmf_txfinalize(ifp, skb, true); } From ad5765d2717562b872b1bd1a53c39d9bc6bdfcbe Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 25 Aug 2026 14:35:38 -0400 Subject: [PATCH 0017/2680] dt-bindings: PCI: apple: add T8103-family PCIe-C Signed-off-by: DjDeveloperr --- .../devicetree/bindings/pci/apple,pcie.yaml | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 0bed9e126fdaa..58ea6eb8d6073 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -21,6 +21,10 @@ description: | separate from the port registers. In that case, there is one PHY register range per port register range. + The PCIe-C variant connects a single root port to a USB4 router instead of + a physical PCIe link. It has no external PHY or PERST# GPIO and follows the + router's cable-controlled power lifetime. + All root ports share a single ECAM space, but separate GPIOs are used to take the PCI devices on those ports out of reset. Therefore the standard "reset-gpios" and "max-link-speed" properties appear on @@ -45,6 +49,11 @@ properties: - const: apple,t8122-pcie - const: apple,t6020-pcie - const: apple,t6020-pcie + - const: apple,t8103-pciec + - items: + - const: apple,t8112-pciec + - const: apple,t8103-pciec + - const: apple,t6000-pciec reg: minItems: 3 @@ -141,6 +150,27 @@ allOf: properties: reg-names: minItems: 10 + - if: + properties: + compatible: + contains: + enum: + - apple,t8103-pciec + - apple,t6000-pciec + then: + properties: + reg: + minItems: 7 + maxItems: 7 + interrupts: + maxItems: 1 + ranges: + minItems: 3 + maxItems: 3 + else: + properties: + ranges: + maxItems: 2 examples: - | From 32a06a5c6e5494c8579c2a1f1ed3fc4e8adcb2c8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 25 Aug 2026 14:16:33 -0400 Subject: [PATCH 0018/2680] PCI: apple: add cable-powered PCIe-C root complexes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — split by topic: the drivers/thunderbolt hunks go to 170-usb] Signed-off-by: DjDeveloperr --- drivers/pci/controller/pcie-apple.c | 251 ++++++++++++++++++++++++---- 1 file changed, 215 insertions(+), 36 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index b972eb05bb85e..bea35fecaa13d 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -159,6 +159,7 @@ struct hw_info { u32 port_rid2sid; u32 port_msimap; u32 max_rid2sid; + bool tunneled; }; static const struct hw_info t8103_hw = { @@ -184,6 +185,14 @@ static const struct hw_info t602x_hw = { .max_rid2sid = 512, }; +static const struct hw_info t6000_pciec_hw = { + .port_msiaddr = PORT_MSIADDR, + .port_perst = PORT_PERST, + .port_rid2sid = PORT_RID2SID, + .max_rid2sid = 64, + .tunneled = true, +}; + struct apple_pcie { struct mutex lock; struct device *dev; @@ -193,6 +202,7 @@ struct apple_pcie { struct list_head ports; struct completion event; struct irq_fwspec fwspec; + struct irq_domain *msi_domain; u32 nvecs; }; @@ -205,8 +215,11 @@ struct apple_pcie_port { struct irq_domain *domain; struct list_head entry; unsigned long *sid_map; + unsigned int irq; + unsigned int link_irqs[2]; int sid_map_sz; int idx; + bool started; }; static void rmw_set(u32 set, void __iomem *addr) @@ -257,8 +270,13 @@ static int apple_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, fwspec.param[fwspec.param_count - 2] += hwirq; ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, &fwspec); - if (ret) + if (ret) { + mutex_lock(&pcie->lock); + bitmap_release_region(pcie->bitmap, hwirq, + order_base_2(nr_irqs)); + mutex_unlock(&pcie->lock); return ret; + } for (i = 0; i < nr_irqs; i++) { irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i, @@ -404,27 +422,29 @@ static int apple_pcie_port_setup_irq(struct apple_pcie_port *port) { struct fwnode_handle *fwnode = &port->np->fwnode; struct apple_pcie *pcie = port->pcie; - unsigned int irq; u32 val = 0; /* FIXME: consider moving each interrupt under each port */ - irq = irq_of_parse_and_map(to_of_node(dev_fwnode(port->pcie->dev)), - port->idx); - if (!irq) + port->irq = irq_of_parse_and_map(to_of_node(dev_fwnode(port->pcie->dev)), + port->idx); + if (!port->irq) return -ENXIO; port->domain = irq_domain_create_linear(fwnode, 32, &apple_port_irq_domain_ops, port); - if (!port->domain) + if (!port->domain) { + irq_dispose_mapping(port->irq); + port->irq = 0; return -ENOMEM; + } /* Disable all interrupts */ writel_relaxed(~0, port->base + PORT_INTMSK); writel_relaxed(~0, port->base + PORT_INTSTAT); writel_relaxed(~0, port->base + PORT_LINKCMDSTS); - irq_set_chained_handler_and_data(irq, apple_port_irq_handler, port); + irq_set_chained_handler_and_data(port->irq, apple_port_irq_handler, port); /* Configure MSI base address */ BUILD_BUG_ON(upper_32_bits(DOORBELL_ADDR)); @@ -489,22 +509,109 @@ static int apple_pcie_port_register_irqs(struct apple_pcie_port *port) [0] = port_irqs[i].hwirq, }, }; - unsigned int irq; - int ret; + int irq, ret; irq = irq_domain_alloc_irqs(port->domain, 1, NUMA_NO_NODE, &fwspec); - if (WARN_ON(!irq)) - continue; + if (irq <= 0) + return irq ?: -ENOMEM; ret = request_irq(irq, apple_pcie_port_irq, 0, port_irqs[i].name, port); - WARN_ON(ret); + if (ret) { + irq_domain_free_irqs(irq, 1); + return ret; + } + + port->link_irqs[i] = irq; } return 0; } +static void apple_pcie_port_unregister_irqs(struct apple_pcie_port *port) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(port->link_irqs); i++) { + if (!port->link_irqs[i]) + continue; + + free_irq(port->link_irqs[i], port); + irq_domain_free_irqs(port->link_irqs[i], 1); + port->link_irqs[i] = 0; + } +} + +static void apple_pcie_tunnel_stop(struct apple_pcie_port *port) +{ + struct apple_pcie *pcie = port->pcie; + u32 stat; + int ret; + + /* + * PCIe-C has no external PERST# line. The USB4 router asserts the + * tunneled reset request, and the root port must acknowledge it before + * ACIO loses power. A cable disconnect requires that same ordering. + */ + rmw_set(PORT_TUNCTRL_PERST_ON, port->base + PORT_TUNCTRL); + ret = readl_relaxed_poll_timeout(port->base + PORT_TUNSTAT, stat, + stat & PORT_TUNSTAT_PERST_ON, + 1000, 100000); + if (ret) + dev_warn(pcie->dev, "port %pOF tunnel reset assertion timed out\n", + port->np); + + rmw_clear(PORT_PERST_OFF, port->base + pcie->hw->port_perst); + rmw_clear(PORT_LTSSMCTL_START, port->base + PORT_LTSSMCTL); + rmw_clear(PORT_APPCLK_EN, port->base + PORT_APPCLK); + + ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, + !(stat & PORT_STATUS_READY), 10, 100000); + if (ret) + dev_warn(pcie->dev, "port %pOF disable timed out\n", port->np); + + rmw_set(PORT_TUNCTRL_PERST_ACK_REQ, port->base + PORT_TUNCTRL); + ret = readl_relaxed_poll_timeout(port->base + PORT_TUNSTAT, stat, + stat & PORT_TUNSTAT_PERST_ACK_PEND, + 1000, 1000000); + if (ret) + dev_warn(pcie->dev, "port %pOF tunnel reset acknowledgment timed out\n", + port->np); + rmw_clear(PORT_TUNCTRL_PERST_ACK_REQ, port->base + PORT_TUNCTRL); + port->started = false; +} + +static void apple_pcie_port_teardown(struct apple_pcie_port *port) +{ + if (port->base) + writel_relaxed(~0, port->base + PORT_INTMSK); + + if (port->pcie->hw->tunneled && port->started) + apple_pcie_tunnel_stop(port); + + apple_pcie_port_unregister_irqs(port); + + if (port->irq) { + irq_set_chained_handler_and_data(port->irq, NULL, NULL); + irq_dispose_mapping(port->irq); + port->irq = 0; + } + + if (port->domain) { + irq_domain_remove(port->domain); + port->domain = NULL; + } + + if (!list_empty(&port->entry)) + list_del_init(&port->entry); + + if (port->np) { + of_node_put(port->np); + port->np = NULL; + } +} + static int apple_pcie_setup_refclk(struct apple_pcie *pcie, struct apple_pcie_port *port) { @@ -650,7 +757,7 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, struct apple_pcie_port *port; struct resource *res; char name[16]; - u32 link_stat, idx; + u32 link_stat, stat, idx; int ret, i; port = devm_kzalloc(pcie->dev, sizeof(*port), GFP_KERNEL); @@ -668,44 +775,85 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, /* Use the first reg entry to work out the port index */ port->idx = idx >> 11; port->pcie = pcie; - port->np = np; + port->np = of_node_get(np); raw_spin_lock_init(&port->lock); + INIT_LIST_HEAD(&port->entry); snprintf(name, sizeof(name), "port%d", port->idx); res = platform_get_resource_byname(platform, IORESOURCE_MEM, name); if (!res) res = platform_get_resource(platform, IORESOURCE_MEM, port->idx + 2); + if (!res) { + ret = -ENODEV; + goto err_teardown; + } port->base = devm_ioremap_resource(&platform->dev, res); - if (IS_ERR(port->base)) - return PTR_ERR(port->base); + if (IS_ERR(port->base)) { + ret = PTR_ERR(port->base); + port->base = NULL; + goto err_teardown; + } - snprintf(name, sizeof(name), "phy%d", port->idx); - res = platform_get_resource_byname(platform, IORESOURCE_MEM, name); - if (res) - port->phy = devm_ioremap_resource(&platform->dev, res); - else - port->phy = pcie->base + CORE_PHY_DEFAULT_BASE(port->idx); + if (!pcie->hw->tunneled) { + snprintf(name, sizeof(name), "phy%d", port->idx); + res = platform_get_resource_byname(platform, IORESOURCE_MEM, name); + if (res) + port->phy = devm_ioremap_resource(&platform->dev, res); + else + port->phy = pcie->base + CORE_PHY_DEFAULT_BASE(port->idx); + if (IS_ERR(port->phy)) { + ret = PTR_ERR(port->phy); + port->phy = NULL; + goto err_teardown; + } + } + if (pcie->hw->tunneled) { + /* PCIe-C is reset through the internal USB4 adapter. */ + rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); + rmw_clear(PORT_TUNCTRL_PERST_ON | + PORT_TUNCTRL_PERST_ACK_REQ, + port->base + PORT_TUNCTRL); + rmw_clear(PORT_PERST_OFF, + port->base + pcie->hw->port_perst); + /* The minimal Tperst-clk value is 100us (PCIe CEM r5.0, 2.9.2). */ + usleep_range(100, 200); - /* link might be already brought up by u-boot, skip setup then */ - link_stat = readl_relaxed(port->base + PORT_LINKSTS); - if (!(link_stat & PORT_LINKSTS_UP)) { - ret = apple_pcie_setup_link(pcie, port, np); - if (ret) - return ret; + rmw_set(PORT_PERST_OFF, port->base + pcie->hw->port_perst); + port->started = true; + + /* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1). */ + msleep(100); + + ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, + stat & PORT_STATUS_READY, + 100, 250000); + if (ret < 0) { + dev_err(pcie->dev, + "port %pOF ready wait timeout\n", np); + goto err_teardown; + } + } else { + /* U-Boot may already have brought up a conventional root port. */ + link_stat = readl_relaxed(port->base + PORT_LINKSTS); + if (!(link_stat & PORT_LINKSTS_UP)) { + ret = apple_pcie_setup_link(pcie, port, np); + if (ret) + goto err_teardown; + } } if (pcie->hw->port_refclk) rmw_clear(PORT_REFCLK_CGDIS, port->base + pcie->hw->port_refclk); - else + else if (port->phy) rmw_set(PHY_LANE_CFG_REFCLKCGEN, port->phy + PHY_LANE_CFG); rmw_clear(PORT_APPCLK_CGDIS, port->base + PORT_APPCLK); ret = apple_pcie_port_setup_irq(port); if (ret) - return ret; + goto err_teardown; /* Reset all RID/SID mappings, and check for RAZ/WI registers */ for (i = 0; i < pcie->hw->max_rid2sid; i++) { @@ -721,11 +869,9 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, list_add_tail(&port->entry, &pcie->ports); init_completion(&pcie->event); - /* In the success path, we keep a reference to np around */ - of_node_get(np); - ret = apple_pcie_port_register_irqs(port); - WARN_ON(ret); + if (ret) + goto err_teardown; link_stat = readl_relaxed(port->base + PORT_LINKSTS); if (!(link_stat & PORT_LINKSTS_UP)) { @@ -744,6 +890,10 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, } return 0; + +err_teardown: + apple_pcie_port_teardown(port); + return ret; } static const struct msi_parent_ops apple_msi_parent_ops = { @@ -777,11 +927,14 @@ static int apple_msi_init(struct apple_pcie *pcie) ret = of_property_read_u32_index(to_of_node(fwnode), "msi-ranges", args.args_count + 1, &pcie->nvecs); - if (ret) + if (ret) { + of_node_put(args.np); return ret; + } of_phandle_args_to_fwspec(args.np, args.args, args.args_count, &pcie->fwspec); + of_node_put(args.np); pcie->bitmap = devm_bitmap_zalloc(pcie->dev, pcie->nvecs, GFP_KERNEL); if (!pcie->bitmap) @@ -793,13 +946,28 @@ static int apple_msi_init(struct apple_pcie *pcie) return -ENXIO; } - if (!msi_create_parent_irq_domain(&info, &apple_msi_parent_ops)) { + pcie->msi_domain = msi_create_parent_irq_domain(&info, &apple_msi_parent_ops); + if (!pcie->msi_domain) { dev_err(pcie->dev, "failed to create IRQ domain\n"); return -ENOMEM; } return 0; } +static void apple_pcie_cleanup(void *data) +{ + struct apple_pcie *pcie = data; + struct apple_pcie_port *port, *tmp; + + list_for_each_entry_safe(port, tmp, &pcie->ports, entry) + apple_pcie_port_teardown(port); + + if (pcie->msi_domain) { + irq_domain_remove(pcie->msi_domain); + pcie->msi_domain = NULL; + } +} + static struct apple_pcie *apple_pcie_lookup(struct device *dev) { return pci_host_bridge_priv(dev_get_drvdata(dev)); @@ -988,10 +1156,20 @@ static int apple_pcie_probe(struct platform_device *pdev) if (ret) return ret; + ret = devm_add_action_or_reset(dev, apple_pcie_cleanup, pcie); + if (ret) + return ret; + return pci_host_common_init(pdev, bridge, &apple_pcie_cfg_ecam_ops); } +static void apple_pcie_remove(struct platform_device *pdev) +{ + pci_host_common_remove(pdev); +} + static const struct of_device_id apple_pcie_of_match[] = { + { .compatible = "apple,t6000-pciec", .data = &t6000_pciec_hw }, { .compatible = "apple,t6020-pcie", .data = &t602x_hw }, { .compatible = "apple,pcie", .data = &t8103_hw }, { } @@ -1000,6 +1178,7 @@ MODULE_DEVICE_TABLE(of, apple_pcie_of_match); static struct platform_driver apple_pcie_driver = { .probe = apple_pcie_probe, + .remove = apple_pcie_remove, .driver = { .name = "pcie-apple", .of_match_table = apple_pcie_of_match, From 561496f2e6a9bc91a277340ccc6ee044065a2648 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 25 Aug 2026 14:35:39 -0400 Subject: [PATCH 0019/2680] PCI: apple: support T8103-family PCIe-C tunnels Signed-off-by: DjDeveloperr --- drivers/pci/controller/pcie-apple.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index bea35fecaa13d..a1a34c74ce47f 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -185,7 +185,7 @@ static const struct hw_info t602x_hw = { .max_rid2sid = 512, }; -static const struct hw_info t6000_pciec_hw = { +static const struct hw_info t8103_pciec_hw = { .port_msiaddr = PORT_MSIADDR, .port_perst = PORT_PERST, .port_rid2sid = PORT_RID2SID, @@ -1169,7 +1169,8 @@ static void apple_pcie_remove(struct platform_device *pdev) } static const struct of_device_id apple_pcie_of_match[] = { - { .compatible = "apple,t6000-pciec", .data = &t6000_pciec_hw }, + { .compatible = "apple,t8103-pciec", .data = &t8103_pciec_hw }, + { .compatible = "apple,t6000-pciec", .data = &t8103_pciec_hw }, { .compatible = "apple,t6020-pcie", .data = &t602x_hw }, { .compatible = "apple,pcie", .data = &t8103_hw }, { } From 569525a0cc00c15b27081bd3549bca75466fb0fd Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 00:05:16 -0400 Subject: [PATCH 0020/2680] PCI: apple: support live tunneled PCIe-C ports Signed-off-by: DjDeveloperr --- drivers/pci/controller/pcie-apple.c | 260 ++++++++++++++++++++-------- drivers/pci/iomap.c | 2 + 2 files changed, 193 insertions(+), 69 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index a1a34c74ce47f..3dc05aadcd465 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -232,6 +232,52 @@ static void rmw_clear(u32 clr, void __iomem *addr) writel_relaxed(readl_relaxed(addr) & ~clr, addr); } +/* + * PCIe-C lives behind the same tunneled fabric as its DART. A live Linux + * /dev/mem replay of the root-port setup sequence showed that the aperture is + * available, but each access needs a full completion barrier. Without it, + * back-to-back relaxed accesses can leave a transaction pending until a later + * access reports an asynchronous SError. Keep conventional root ports on the + * existing fast path. + */ +static inline void apple_pcie_port_writel(struct apple_pcie_port *port, + u32 value, u32 offset) +{ + writel_relaxed(value, port->base + offset); + if (port->pcie->hw->tunneled) { + mb(); + isb(); + } +} + +static inline u32 apple_pcie_port_readl(struct apple_pcie_port *port, + u32 offset) +{ + u32 value = readl_relaxed(port->base + offset); + + if (port->pcie->hw->tunneled) { + mb(); + isb(); + } + + return value; +} + +static void apple_pcie_port_rmw_set(struct apple_pcie_port *port, u32 set, + u32 offset) +{ + apple_pcie_port_writel(port, apple_pcie_port_readl(port, offset) | set, + offset); +} + +static void apple_pcie_port_rmw_clear(struct apple_pcie_port *port, u32 clear, + u32 offset) +{ + apple_pcie_port_writel(port, + apple_pcie_port_readl(port, offset) & ~clear, + offset); +} + static void apple_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) { msg->address_hi = upper_32_bits(DOORBELL_ADDR); @@ -309,7 +355,7 @@ static void apple_port_irq_mask(struct irq_data *data) struct apple_pcie_port *port = irq_data_get_irq_chip_data(data); guard(raw_spinlock_irqsave)(&port->lock); - rmw_set(BIT(data->hwirq), port->base + PORT_INTMSK); + apple_pcie_port_rmw_set(port, BIT(data->hwirq), PORT_INTMSK); } static void apple_port_irq_unmask(struct irq_data *data) @@ -317,7 +363,7 @@ static void apple_port_irq_unmask(struct irq_data *data) struct apple_pcie_port *port = irq_data_get_irq_chip_data(data); guard(raw_spinlock_irqsave)(&port->lock); - rmw_clear(BIT(data->hwirq), port->base + PORT_INTMSK); + apple_pcie_port_rmw_clear(port, BIT(data->hwirq), PORT_INTMSK); } static bool hwirq_is_intx(unsigned int hwirq) @@ -330,7 +376,7 @@ static void apple_port_irq_ack(struct irq_data *data) struct apple_pcie_port *port = irq_data_get_irq_chip_data(data); if (!hwirq_is_intx(data->hwirq)) - writel_relaxed(BIT(data->hwirq), port->base + PORT_INTSTAT); + apple_pcie_port_writel(port, BIT(data->hwirq), PORT_INTSTAT); } static int apple_port_irq_set_type(struct irq_data *data, unsigned int type) @@ -410,7 +456,7 @@ static void apple_port_irq_handler(struct irq_desc *desc) chained_irq_enter(chip, desc); - stat = readl_relaxed(port->base + PORT_INTSTAT); + stat = apple_pcie_port_readl(port, PORT_INTSTAT); for_each_set_bit(i, &stat, 32) generic_handle_domain_irq(port->domain, i); @@ -440,31 +486,32 @@ static int apple_pcie_port_setup_irq(struct apple_pcie_port *port) } /* Disable all interrupts */ - writel_relaxed(~0, port->base + PORT_INTMSK); - writel_relaxed(~0, port->base + PORT_INTSTAT); - writel_relaxed(~0, port->base + PORT_LINKCMDSTS); + apple_pcie_port_writel(port, ~0, PORT_INTMSK); + apple_pcie_port_writel(port, ~0, PORT_INTSTAT); + apple_pcie_port_writel(port, ~0, PORT_LINKCMDSTS); irq_set_chained_handler_and_data(port->irq, apple_port_irq_handler, port); /* Configure MSI base address */ BUILD_BUG_ON(upper_32_bits(DOORBELL_ADDR)); - writel_relaxed(lower_32_bits(DOORBELL_ADDR), - port->base + pcie->hw->port_msiaddr); + apple_pcie_port_writel(port, lower_32_bits(DOORBELL_ADDR), + pcie->hw->port_msiaddr); if (pcie->hw->port_msiaddr_hi) - writel_relaxed(0, port->base + pcie->hw->port_msiaddr_hi); + apple_pcie_port_writel(port, 0, pcie->hw->port_msiaddr_hi); /* Enable MSIs, shared between all ports */ if (pcie->hw->port_msimap) { for (int i = 0; i < pcie->nvecs; i++) - writel_relaxed(FIELD_PREP(PORT_MSIMAP_TARGET, i) | - PORT_MSIMAP_ENABLE, - port->base + pcie->hw->port_msimap + 4 * i); + apple_pcie_port_writel(port, + FIELD_PREP(PORT_MSIMAP_TARGET, i) | + PORT_MSIMAP_ENABLE, + pcie->hw->port_msimap + 4 * i); } else { - writel_relaxed(0, port->base + PORT_MSIBASE); + apple_pcie_port_writel(port, 0, PORT_MSIBASE); val = ilog2(pcie->nvecs) << PORT_MSICFG_L2MSINUM_SHIFT; } - writel_relaxed(val | PORT_MSICFG_EN, port->base + PORT_MSICFG); + apple_pcie_port_writel(port, val | PORT_MSICFG_EN, PORT_MSICFG); return 0; } @@ -543,6 +590,27 @@ static void apple_pcie_port_unregister_irqs(struct apple_pcie_port *port) } } +static int apple_pcie_tunnel_release_reset(struct apple_pcie_port *port) +{ + u32 stat; + int ret; + + /* + * Starting the CIO PCIe tunnel leaves the tunneled root port in reset. + * Releasing it means writing zero to PORT_TUNCTRL, waiting for the + * reset-active indication to clear, and only then enabling the LTSSM. + */ + apple_pcie_port_writel(port, 0, PORT_TUNCTRL); + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + !(stat & PORT_TUNSTAT_PERST_ON), + 1000, 100000, false, port, PORT_TUNSTAT); + if (ret) + dev_err(port->pcie->dev, + "port %pOF tunnel reset release timed out\n", port->np); + + return ret; +} + static void apple_pcie_tunnel_stop(struct apple_pcie_port *port) { struct apple_pcie *pcie = port->pcie; @@ -554,38 +622,41 @@ static void apple_pcie_tunnel_stop(struct apple_pcie_port *port) * tunneled reset request, and the root port must acknowledge it before * ACIO loses power. A cable disconnect requires that same ordering. */ - rmw_set(PORT_TUNCTRL_PERST_ON, port->base + PORT_TUNCTRL); - ret = readl_relaxed_poll_timeout(port->base + PORT_TUNSTAT, stat, - stat & PORT_TUNSTAT_PERST_ON, - 1000, 100000); + apple_pcie_port_rmw_set(port, PORT_TUNCTRL_PERST_ON, PORT_TUNCTRL); + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + stat & PORT_TUNSTAT_PERST_ON, + 1000, 100000, false, port, PORT_TUNSTAT); if (ret) dev_warn(pcie->dev, "port %pOF tunnel reset assertion timed out\n", port->np); - rmw_clear(PORT_PERST_OFF, port->base + pcie->hw->port_perst); - rmw_clear(PORT_LTSSMCTL_START, port->base + PORT_LTSSMCTL); - rmw_clear(PORT_APPCLK_EN, port->base + PORT_APPCLK); + apple_pcie_port_rmw_clear(port, PORT_PERST_OFF, + pcie->hw->port_perst); + apple_pcie_port_rmw_clear(port, PORT_LTSSMCTL_START, PORT_LTSSMCTL); + apple_pcie_port_rmw_clear(port, PORT_APPCLK_EN, PORT_APPCLK); - ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, - !(stat & PORT_STATUS_READY), 10, 100000); + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + !(stat & PORT_STATUS_READY), + 10, 100000, false, port, PORT_STATUS); if (ret) dev_warn(pcie->dev, "port %pOF disable timed out\n", port->np); - rmw_set(PORT_TUNCTRL_PERST_ACK_REQ, port->base + PORT_TUNCTRL); - ret = readl_relaxed_poll_timeout(port->base + PORT_TUNSTAT, stat, - stat & PORT_TUNSTAT_PERST_ACK_PEND, - 1000, 1000000); + apple_pcie_port_rmw_set(port, PORT_TUNCTRL_PERST_ACK_REQ, PORT_TUNCTRL); + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + stat & PORT_TUNSTAT_PERST_ACK_PEND, + 1000, 1000000, false, port, PORT_TUNSTAT); if (ret) dev_warn(pcie->dev, "port %pOF tunnel reset acknowledgment timed out\n", port->np); - rmw_clear(PORT_TUNCTRL_PERST_ACK_REQ, port->base + PORT_TUNCTRL); + apple_pcie_port_rmw_clear(port, PORT_TUNCTRL_PERST_ACK_REQ, + PORT_TUNCTRL); port->started = false; } static void apple_pcie_port_teardown(struct apple_pcie_port *port) { if (port->base) - writel_relaxed(~0, port->base + PORT_INTMSK); + apple_pcie_port_writel(port, ~0, PORT_INTMSK); if (port->pcie->hw->tunneled && port->started) apple_pcie_tunnel_stop(port); @@ -648,17 +719,19 @@ static int apple_pcie_setup_refclk(struct apple_pcie *pcie, return 0; } -static void __iomem *port_rid2sid_addr(struct apple_pcie_port *port, int idx) +static u32 port_rid2sid_offset(struct apple_pcie_port *port, int idx) { - return port->base + port->pcie->hw->port_rid2sid + 4 * idx; + return port->pcie->hw->port_rid2sid + 4 * idx; } static u32 apple_pcie_rid2sid_write(struct apple_pcie_port *port, int idx, u32 val) { - writel_relaxed(val, port_rid2sid_addr(port, idx)); + u32 offset = port_rid2sid_offset(port, idx); + + apple_pcie_port_writel(port, val, offset); /* Read back to ensure completion of the write */ - return readl_relaxed(port_rid2sid_addr(port, idx)); + return apple_pcie_port_readl(port, offset); } static int apple_pcie_setup_link(struct apple_pcie *pcie, @@ -757,7 +830,7 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, struct apple_pcie_port *port; struct resource *res; char name[16]; - u32 link_stat, stat, idx; + u32 link_stat, preinit_status, stat, idx; int ret, i; port = devm_kzalloc(pcie->dev, sizeof(*port), GFP_KERNEL); @@ -789,6 +862,14 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, goto err_teardown; } + /* + * PCIe-C register transactions are non-posted on the tunneled fabric. + * Mark the resource so devm_ioremap_resource() selects ioremap_np() on + * arm64, matching the successful /dev/mem Device-nGnRnE replay. + */ + if (pcie->hw->tunneled) + res->flags |= IORESOURCE_MEM_NONPOSTED; + port->base = devm_ioremap_resource(&platform->dev, res); if (IS_ERR(port->base)) { ret = PTR_ERR(port->base); @@ -810,25 +891,26 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, } } if (pcie->hw->tunneled) { - /* PCIe-C is reset through the internal USB4 adapter. */ - rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); - rmw_clear(PORT_TUNCTRL_PERST_ON | - PORT_TUNCTRL_PERST_ACK_REQ, - port->base + PORT_TUNCTRL); - rmw_clear(PORT_PERST_OFF, - port->base + pcie->hw->port_perst); - /* The minimal Tperst-clk value is 100us (PCIe CEM r5.0, 2.9.2). */ - usleep_range(100, 200); - - rmw_set(PORT_PERST_OFF, port->base + pcie->hw->port_perst); - port->started = true; - - /* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1). */ - msleep(100); + /* + * m1n1 owns PCIe-C cold initialization. Replaying the port reset or + * tunnel-reset handshake against that live handoff raises an + * asynchronous SError on T6020. Accept only an explicitly successful + * handoff and begin with the read-only RUN state check used by m1n1. + */ + ret = of_property_read_u32(pcie->dev->of_node, + "apple,pciec-preinit-status", + &preinit_status); + if (ret || preinit_status != 1) { + dev_err(pcie->dev, + "PCIe-C requires a successful m1n1 preinit handoff\n"); + ret = -ENODEV; + goto err_teardown; + } - ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, - stat & PORT_STATUS_READY, - 100, 250000); + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + stat & PORT_STATUS_READY, + 100, 250000, false, port, + PORT_STATUS); if (ret < 0) { dev_err(pcie->dev, "port %pOF ready wait timeout\n", np); @@ -836,7 +918,7 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, } } else { /* U-Boot may already have brought up a conventional root port. */ - link_stat = readl_relaxed(port->base + PORT_LINKSTS); + link_stat = apple_pcie_port_readl(port, PORT_LINKSTS); if (!(link_stat & PORT_LINKSTS_UP)) { ret = apple_pcie_setup_link(pcie, port, np); if (ret) @@ -849,7 +931,9 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, else if (port->phy) rmw_set(PHY_LANE_CFG_REFCLKCGEN, port->phy + PHY_LANE_CFG); - rmw_clear(PORT_APPCLK_CGDIS, port->base + PORT_APPCLK); + /* The preinitialized PCIe-C APPCLK state is part of the m1n1 handoff. */ + if (!pcie->hw->tunneled) + rmw_clear(PORT_APPCLK_CGDIS, port->base + PORT_APPCLK); ret = apple_pcie_port_setup_irq(port); if (ret) @@ -873,11 +957,18 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, if (ret) goto err_teardown; - link_stat = readl_relaxed(port->base + PORT_LINKSTS); + link_stat = apple_pcie_port_readl(port, PORT_LINKSTS); if (!(link_stat & PORT_LINKSTS_UP)) { unsigned long timeout, left; + + if (pcie->hw->tunneled) { + ret = apple_pcie_tunnel_release_reset(port); + if (ret) + goto err_teardown; + } + /* start link training */ - writel_relaxed(PORT_LTSSMCTL_START, port->base + PORT_LTSSMCTL); + apple_pcie_port_writel(port, PORT_LTSSMCTL_START, PORT_LTSSMCTL); timeout = link_up_timeout * HZ / 1000; left = wait_for_completion_timeout(&pcie->event, timeout); @@ -1001,11 +1092,23 @@ static struct apple_pcie_port *apple_pcie_get_port(struct pci_dev *pdev) static int apple_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_dev *pdev) { + struct apple_pcie *pcie = pci_host_bridge_priv(bridge); + struct resource *res; u32 sid, rid = pci_dev_id(pdev); struct apple_pcie_port *port; struct of_phandle_args iommu_spec = {}; int idx, err; + /* + * Endpoint BARs share PCIe-C's tunneled, non-posted MMIO fabric. Mark + * them before the function driver maps its BAR so pci_iomap() selects + * Device-nGnRnE on arm64. + */ + if (pcie->hw->tunneled) + pci_dev_for_each_resource(pdev, res) + if (res->flags & IORESOURCE_MEM) + res->flags |= IORESOURCE_MEM_NONPOSTED; + port = apple_pcie_get_port(pdev); if (!port) return 0; @@ -1051,7 +1154,8 @@ static void apple_pcie_disable_device(struct pci_host_bridge *bridge, struct pci for_each_set_bit(idx, port->sid_map, port->sid_map_sz) { u32 val; - val = readl_relaxed(port_rid2sid_addr(port, idx)); + val = apple_pcie_port_readl(port, + port_rid2sid_offset(port, idx)); if ((val & 0xffff) == rid) { apple_pcie_rid2sid_write(port, idx, 0); bitmap_release_region(port->sid_map, idx, 0); @@ -1122,17 +1226,31 @@ static int apple_pcie_probe_port(struct device_node *np) static int apple_pcie_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + const struct hw_info *hw; struct pci_host_bridge *bridge; struct device_node *of_port; struct apple_pcie *pcie; int ret; - /* Check for probe dependencies for all ports first */ - for_each_available_child_of_node(dev->of_node, of_port) { - ret = apple_pcie_probe_port(of_port); - if (ret) { - of_node_put(of_port); - return dev_err_probe(dev, ret, "Port %pOF probe fail\n", of_port); + hw = of_device_get_match_data(dev); + if (!hw) + return -ENODEV; + + /* + * A tunneled PCIe-C port has no host GPIO for PERST#: the tunnel + * firmware and m1n1 handoff own its reset state. Requiring a + * reset-gpios provider here rejects the valid synthesized port before + * the tunneled setup path can consume that handoff. + */ + if (!hw->tunneled) { + /* Check for probe dependencies for all ports first */ + for_each_available_child_of_node(dev->of_node, of_port) { + ret = apple_pcie_probe_port(of_port); + if (ret) { + of_node_put(of_port); + return dev_err_probe(dev, ret, + "Port %pOF probe fail\n", of_port); + } } } @@ -1142,9 +1260,7 @@ static int apple_pcie_probe(struct platform_device *pdev) pcie = pci_host_bridge_priv(bridge); pcie->dev = dev; - pcie->hw = of_device_get_match_data(dev); - if (!pcie->hw) - return -ENODEV; + pcie->hw = hw; pcie->base = devm_platform_ioremap_resource(pdev, 1); if (IS_ERR(pcie->base)) return PTR_ERR(pcie->base); @@ -1156,11 +1272,17 @@ static int apple_pcie_probe(struct platform_device *pdev) if (ret) return ret; - ret = devm_add_action_or_reset(dev, apple_pcie_cleanup, pcie); + ret = pci_host_common_init(pdev, bridge, &apple_pcie_cfg_ecam_ops); + if (ret) + apple_pcie_cleanup(pcie); if (ret) return ret; - return pci_host_common_init(pdev, bridge, &apple_pcie_cfg_ecam_ops); + /* + * Port mappings are allocated by the ECAM init callback. Register cleanup + * afterwards so devres runs it before unmapping those registers. + */ + return devm_add_action_or_reset(dev, apple_pcie_cleanup, pcie); } static void apple_pcie_remove(struct platform_device *pdev) diff --git a/drivers/pci/iomap.c b/drivers/pci/iomap.c index ea86c282a3869..2e35f0131fe8b 100644 --- a/drivers/pci/iomap.c +++ b/drivers/pci/iomap.c @@ -50,6 +50,8 @@ void __iomem *pci_iomap_range(struct pci_dev *dev, len = maxlen; if (flags & IORESOURCE_IO) return __pci_ioport_map(dev, start, len); + if (flags & IORESOURCE_MEM_NONPOSTED) + return ioremap_np(start, len) ?: ioremap(start, len); if (flags & IORESOURCE_MEM) return ioremap(start, len); /* What? */ From ad69a11d37785061e7245e28dc0d86b41f748ff9 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 00:07:39 -0400 Subject: [PATCH 0021/2680] thunderbolt: apple: complete PCIe-C build dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — split by topic: only the drivers/pci/controller/Kconfig hunks of this usb commit; the rest is in 170-usb] Signed-off-by: DjDeveloperr --- drivers/pci/controller/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 420db1255346f..e3bb6ff1b6d71 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -46,6 +46,7 @@ config PCIE_APPLE depends on OF depends on PCI_MSI depends on PAGE_SIZE_16KB || COMPILE_TEST + select APPLE_TUNABLE select PCI_HOST_COMMON select IRQ_MSI_LIB help From 2582459cdd20c3f1eb81b865db573f78fa33450f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 22:52:01 -0400 Subject: [PATCH 0022/2680] PCI: apple: quiesce and restore tunneled PCIe-C links Signed-off-by: DjDeveloperr --- drivers/pci/controller/pcie-apple.c | 652 +++++++++++++++++++++++++++- include/linux/pci-apple.h | 9 + 2 files changed, 653 insertions(+), 8 deletions(-) create mode 100644 include/linux/pci-apple.h diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 3dc05aadcd465..707aaf31da6a2 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -29,8 +29,12 @@ #include #include #include +#include +#include #include +#include +#include "../pci.h" #include "pci-host-common.h" static int link_up_timeout = 500; @@ -130,6 +134,11 @@ MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); #define PORT_TUNSTAT_PERST_ON BIT(0) #define PORT_TUNSTAT_PERST_ACK_PEND BIT(1) #define PORT_PREFMEM_ENABLE 0x00994 +#define PORT_COUNTER_CTRL 0x04020 +#define PORT_COUNTER_ENABLE 0x3 + +#define PCIEC_INTR2AXI_CTRL 0x00080 +#define PCIEC_INTR2AXI_ENABLE BIT(0) /* T602x (M2-pro and co) */ #define PORT_T602X_MSIADDR 0x016c @@ -197,6 +206,15 @@ struct apple_pcie { struct mutex lock; struct device *dev; void __iomem *base; + void __iomem *fabric_base; + void __iomem *debug_base; + void __iomem *intr2axi_base; + struct pci_config_window *cfg; + struct apple_tunable *rc_tunable; + struct apple_tunable *fabric_tunable; + struct apple_tunable *debug_tunable; + bool power_retained; + bool bus_stopped; const struct hw_info *hw; unsigned long *bitmap; struct list_head ports; @@ -215,8 +233,11 @@ struct apple_pcie_port { struct irq_domain *domain; struct list_head entry; unsigned long *sid_map; + u32 *saved_rid2sid; + struct apple_tunable *tunable; unsigned int irq; unsigned int link_irqs[2]; + u32 saved_intmask; int sid_map_sz; int idx; bool started; @@ -278,6 +299,40 @@ static void apple_pcie_port_rmw_clear(struct apple_pcie_port *port, u32 clear, offset); } +static inline u32 apple_pcie_tunnel_readl(void __iomem *base, u32 offset) +{ + u32 value = readl_relaxed(base + offset); + + mb(); + isb(); + return value; +} + +static inline void apple_pcie_tunnel_writel(void __iomem *base, u32 value, + u32 offset) +{ + writel_relaxed(value, base + offset); + mb(); + isb(); +} + +static void apple_pcie_tunnel_apply_tunable(void __iomem *base, + struct apple_tunable *tunable) +{ + size_t i; + + for (i = 0; i < tunable->sz; i++) { + u32 old, value; + + old = apple_pcie_tunnel_readl(base, tunable->values[i].offset); + value = (old & ~tunable->values[i].mask) | + tunable->values[i].value; + if (value != old) + apple_pcie_tunnel_writel(base, value, + tunable->values[i].offset); + } +} + static void apple_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) { msg->address_hi = upper_32_bits(DOORBELL_ADDR); @@ -590,6 +645,9 @@ static void apple_pcie_port_unregister_irqs(struct apple_pcie_port *port) } } +static u32 apple_pcie_rid2sid_write(struct apple_pcie_port *port, + int idx, u32 val); + static int apple_pcie_tunnel_release_reset(struct apple_pcie_port *port) { u32 stat; @@ -611,11 +669,257 @@ static int apple_pcie_tunnel_release_reset(struct apple_pcie_port *port) return ret; } -static void apple_pcie_tunnel_stop(struct apple_pcie_port *port) +struct apple_pcie_reset_reg { + u16 offset; + u32 value; +}; + +/* Port hardware reset sequence, identical on T8103 and T600x PCIe-C. */ +static const struct apple_pcie_reset_reg apple_pcie_tunnel_reset_regs[] = { + { 0x08c, 0x00000110 }, + { 0x100, 0xffffffff }, + { 0x148, 0xffffffff }, + { 0x210, 0xffffffff }, + { 0x080, 0x00000000 }, + { 0x084, 0x00000000 }, + { 0x104, 0xffffffff }, + { 0x124, 0x00000000 }, + { 0x128, 0x00000000 }, + { 0x168, 0x00000000 }, + { 0x13c, 0x00000010 }, + { 0x800, 0x00100100 }, + { 0x808, 0x00100045 }, + { 0x810, 0x00000100 }, + { 0x814, 0x00000000 }, + { 0x130, 0x00000208 }, + { 0x140, 0x00000010 }, + { 0x144, 0x00253770 }, + { 0x21c, 0x00000000 }, + { 0x81c, 0x00000000 }, + { 0x824, 0x00000000 }, +}; + +static void apple_pcie_tunnel_reset_hardware(struct apple_pcie_port *port) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(apple_pcie_tunnel_reset_regs); i++) + apple_pcie_port_writel(port, + apple_pcie_tunnel_reset_regs[i].value, + apple_pcie_tunnel_reset_regs[i].offset); + + for (i = 0; i < port->pcie->hw->max_rid2sid; i++) + apple_pcie_rid2sid_write(port, i, 0); +} + +static int apple_pcie_tunnel_reinitialize(struct apple_pcie_port *port) { struct apple_pcie *pcie = port->pcie; u32 stat; - int ret; + int i, ret; + + /* + * A cable disconnect intentionally stops PCIe-C after destroying its + * host bridge. Its always-on power domain retains that stopped state, so + * a later platform reprobe cannot rely on the original m1n1 handoff. + * Replay the port enable sequence, which is the same on T8103 and + * T600x, while ACIO's tunnel and Intr2AXI aperture are live. + */ + apple_pcie_tunnel_apply_tunable(pcie->debug_base, + pcie->debug_tunable); + apple_pcie_tunnel_apply_tunable(pcie->fabric_base, + pcie->fabric_tunable); + apple_pcie_tunnel_reset_hardware(port); + apple_pcie_tunnel_apply_tunable(port->base, port->tunable); + + apple_pcie_port_rmw_set(port, PORT_PERST_OFF, + pcie->hw->port_perst); + apple_pcie_port_rmw_set(port, PORT_APPCLK_EN, PORT_APPCLK); + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + stat & PORT_STATUS_READY, + 10, 250000, false, port, PORT_STATUS); + if (ret) + return dev_err_probe(pcie->dev, ret, + "PCIe-C port did not enter RUN after hot reconnect\n"); + + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + !(stat & PORT_LINKSTS_BUSY), + 10, 250000, false, port, PORT_LINKSTS); + if (ret) + return dev_err_probe(pcie->dev, ret, + "PCIe-C port did not become idle after hot reconnect\n"); + + apple_pcie_port_rmw_clear(port, PORT_APPCLK_CGDIS, PORT_APPCLK); + apple_pcie_tunnel_apply_tunable(pcie->cfg->win, pcie->rc_tunable); + apple_pcie_port_writel(port, PORT_COUNTER_ENABLE, PORT_COUNTER_CTRL); + apple_pcie_tunnel_writel(pcie->intr2axi_base, PCIEC_INTR2AXI_ENABLE, + PCIEC_INTR2AXI_CTRL); + for (i = 0; i < pcie->hw->max_rid2sid; i++) + apple_pcie_rid2sid_write(port, i, 0); + + dev_info(pcie->dev, + "PCIe-C hardware reinitialized after cable reconnect\n"); + return 0; +} + +static void apple_pcie_tunnel_restore_irq_hw(struct apple_pcie_port *port) +{ + struct apple_pcie *pcie = port->pcie; + u32 value = 0; + int i; + + apple_pcie_port_writel(port, ~0, PORT_INTSTAT); + apple_pcie_port_writel(port, ~0, PORT_LINKCMDSTS); + apple_pcie_port_writel(port, lower_32_bits(DOORBELL_ADDR), + pcie->hw->port_msiaddr); + if (pcie->hw->port_msiaddr_hi) + apple_pcie_port_writel(port, 0, pcie->hw->port_msiaddr_hi); + + if (pcie->hw->port_msimap) { + for (i = 0; i < pcie->nvecs; i++) + apple_pcie_port_writel(port, + FIELD_PREP(PORT_MSIMAP_TARGET, i) | + PORT_MSIMAP_ENABLE, + pcie->hw->port_msimap + 4 * i); + } else { + apple_pcie_port_writel(port, 0, PORT_MSIBASE); + value = ilog2(pcie->nvecs) << PORT_MSICFG_L2MSINUM_SHIFT; + } + apple_pcie_port_writel(port, value | PORT_MSICFG_EN, PORT_MSICFG); + apple_pcie_port_writel(port, port->saved_intmask, PORT_INTMSK); +} + +static int apple_pcie_tunnel_start(struct apple_pcie_port *port) +{ + struct apple_pcie *pcie = port->pcie; + u32 stat; + int i, ret; + + if (pcie->power_retained) { + /* + * The ATC PCIe domain remains powered on these systems. Replaying + * resetPortHardware() against that live handoff is not a resume + * operation and raises an asynchronous external abort on T6020. + * Re-activating the ACIO PCIe tunnel closes the Intr2AXI aperture, + * just as it does during cold cable activation, so pulse it before + * releasing the retained port reset. + * Release the completed sleep handshake before re-enabling the + * retained port, which is the inverse of the disable sequence. + */ + apple_pcie_tunnel_writel(pcie->intr2axi_base, + PCIEC_INTR2AXI_ENABLE, + PCIEC_INTR2AXI_CTRL); + ret = apple_pcie_tunnel_release_reset(port); + if (ret) + return ret; + } else { + /* + * A genuinely power-gated PCIe-C controller lost its register + * state. Recreate the enablePortHardware() baseline before start. + */ + apple_pcie_tunnel_apply_tunable(pcie->debug_base, + pcie->debug_tunable); + apple_pcie_tunnel_apply_tunable(pcie->fabric_base, + pcie->fabric_tunable); + apple_pcie_tunnel_reset_hardware(port); + apple_pcie_tunnel_apply_tunable(port->base, port->tunable); + } + + apple_pcie_port_rmw_set(port, PORT_PERST_OFF, + pcie->hw->port_perst); + apple_pcie_port_rmw_set(port, PORT_APPCLK_EN, PORT_APPCLK); + + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + stat & PORT_STATUS_READY, + 10, 250000, false, port, PORT_STATUS); + if (ret) { + dev_err(pcie->dev, "port %pOF resume ready wait timed out\n", + port->np); + return ret; + } + + if (!pcie->power_retained) { + apple_pcie_tunnel_apply_tunable(pcie->cfg->win, + pcie->rc_tunable); + apple_pcie_port_rmw_clear(port, PORT_APPCLK_CGDIS, + PORT_APPCLK); + apple_pcie_port_writel(port, PORT_COUNTER_ENABLE, + PORT_COUNTER_CTRL); + apple_pcie_tunnel_writel(pcie->intr2axi_base, + PCIEC_INTR2AXI_ENABLE, + PCIEC_INTR2AXI_CTRL); + apple_pcie_tunnel_restore_irq_hw(port); + for_each_set_bit(i, port->sid_map, port->sid_map_sz) + apple_pcie_rid2sid_write(port, i, + port->saved_rid2sid[i]); + + ret = apple_pcie_tunnel_release_reset(port); + if (ret) + return ret; + } + + apple_pcie_port_writel(port, PORT_LTSSMCTL_START, PORT_LTSSMCTL); + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + stat & PORT_LINKSTS_UP, + 1000, 500000, false, port, PORT_LINKSTS); + if (ret) { + dev_err(pcie->dev, "port %pOF resume link training timed out\n", + port->np); + return ret; + } + if (pcie->power_retained) + apple_pcie_port_writel(port, port->saved_intmask, PORT_INTMSK); + + port->started = true; + dev_info(pcie->dev, "port %pOF tunnel link restored after suspend\n", + port->np); + return 0; +} + +static bool apple_pcie_tunnel_transactions_idle(struct apple_pcie_port *port) +{ + u32 value; + + value = apple_pcie_port_readl(port, PORT_OUTS_NPREQS); + if (value & (PORT_OUTS_NPREQS_REQ | PORT_OUTS_NPREQS_CPL)) + return false; + if (apple_pcie_port_readl(port, PORT_OUTS_PREQS_HDR) & + PORT_OUTS_PREQS_HDR_MASK) + return false; + if (apple_pcie_port_readl(port, PORT_OUTS_PREQS_DATA) & + PORT_OUTS_PREQS_DATA_MASK) + return false; + if (apple_pcie_port_readl(port, PORT_RXWR_FIFO) & + (PORT_RXWR_FIFO_HDR | PORT_RXWR_FIFO_DATA)) + return false; + if (apple_pcie_port_readl(port, PORT_RXRD_FIFO) & PORT_RXRD_FIFO_REQ) + return false; + if (apple_pcie_port_readl(port, PORT_OUTS_CPLS) & + (PORT_OUTS_CPLS_SHRD | PORT_OUTS_CPLS_WAIT)) + return false; + + return true; +} + +static int apple_pcie_tunnel_stop(struct apple_pcie_port *port) +{ + struct apple_pcie *pcie = port->pcie; + bool idle; + u32 stat; + int err = 0, ret; + + port->saved_intmask = apple_pcie_port_readl(port, PORT_INTMSK); + apple_pcie_port_writel(port, ~0, PORT_INTMSK); + apple_pcie_port_writel(port, ~0, PORT_INTSTAT); + apple_pcie_port_writel(port, ~0, PORT_LINKCMDSTS); + apple_pcie_port_rmw_clear(port, PORT_LTSSMCTL_START, PORT_LTSSMCTL); + ret = read_poll_timeout_atomic(apple_pcie_tunnel_transactions_idle, idle, + idle, 10, 20000, false, port); + if (ret) + dev_warn(pcie->dev, + "port %pOF transactions did not drain before suspend\n", + port->np); + err = ret; /* * PCIe-C has no external PERST# line. The USB4 router asserts the @@ -629,10 +933,11 @@ static void apple_pcie_tunnel_stop(struct apple_pcie_port *port) if (ret) dev_warn(pcie->dev, "port %pOF tunnel reset assertion timed out\n", port->np); + if (ret && !err) + err = ret; apple_pcie_port_rmw_clear(port, PORT_PERST_OFF, pcie->hw->port_perst); - apple_pcie_port_rmw_clear(port, PORT_LTSSMCTL_START, PORT_LTSSMCTL); apple_pcie_port_rmw_clear(port, PORT_APPCLK_EN, PORT_APPCLK); ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, @@ -640,6 +945,8 @@ static void apple_pcie_tunnel_stop(struct apple_pcie_port *port) 10, 100000, false, port, PORT_STATUS); if (ret) dev_warn(pcie->dev, "port %pOF disable timed out\n", port->np); + if (ret && !err) + err = ret; apple_pcie_port_rmw_set(port, PORT_TUNCTRL_PERST_ACK_REQ, PORT_TUNCTRL); ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, @@ -648,9 +955,13 @@ static void apple_pcie_tunnel_stop(struct apple_pcie_port *port) if (ret) dev_warn(pcie->dev, "port %pOF tunnel reset acknowledgment timed out\n", port->np); + if (ret && !err) + err = ret; apple_pcie_port_rmw_clear(port, PORT_TUNCTRL_PERST_ACK_REQ, PORT_TUNCTRL); port->started = false; + + return err; } static void apple_pcie_port_teardown(struct apple_pcie_port *port) @@ -840,6 +1151,10 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, port->sid_map = devm_bitmap_zalloc(pcie->dev, pcie->hw->max_rid2sid, GFP_KERNEL); if (!port->sid_map) return -ENOMEM; + port->saved_rid2sid = devm_kcalloc(pcie->dev, pcie->hw->max_rid2sid, + sizeof(*port->saved_rid2sid), GFP_KERNEL); + if (!port->saved_rid2sid) + return -ENOMEM; ret = of_property_read_u32_index(np, "reg", 0, &idx); if (ret) @@ -876,6 +1191,15 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, port->base = NULL; goto err_teardown; } + if (pcie->hw->tunneled) { + port->tunable = devm_apple_tunable_parse(pcie->dev, np, + "apple,tunable", res); + if (IS_ERR(port->tunable)) { + ret = dev_err_probe(pcie->dev, PTR_ERR(port->tunable), + "port %pOF tunables unavailable\n", np); + goto err_teardown; + } + } if (!pcie->hw->tunneled) { snprintf(name, sizeof(name), "phy%d", port->idx); @@ -912,9 +1236,12 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, 100, 250000, false, port, PORT_STATUS); if (ret < 0) { - dev_err(pcie->dev, - "port %pOF ready wait timeout\n", np); - goto err_teardown; + dev_info(pcie->dev, + "port %pOF stopped; replaying PCIe-C hardware initialization\n", + np); + ret = apple_pcie_tunnel_reinitialize(port); + if (ret) + goto err_teardown; } } else { /* U-Boot may already have brought up a conventional root port. */ @@ -979,6 +1306,8 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, (timeout - left) * 1000 / HZ); } + if (pcie->hw->tunneled) + port->started = true; return 0; @@ -1176,6 +1505,7 @@ static int apple_pcie_init(struct pci_config_window *cfg) pcie = apple_pcie_lookup(dev); if (WARN_ON(!pcie)) return -ENOENT; + pcie->cfg = cfg; for_each_available_child_of_node_scoped(dev->of_node, of_port) { ret = apple_pcie_setup_port(pcie, of_port); @@ -1223,6 +1553,191 @@ static int apple_pcie_probe_port(struct device_node *np) return 0; } +static int apple_pcie_tunnel_init_resources(struct platform_device *pdev, + struct apple_pcie *pcie) +{ + struct resource *config, *debug, *fabric, *intr2axi; + + config = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config"); + debug = platform_get_resource_byname(pdev, IORESOURCE_MEM, "debug"); + fabric = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fabric"); + intr2axi = platform_get_resource_byname(pdev, IORESOURCE_MEM, "intr2axi"); + if (!config || !debug || !fabric || !intr2axi) + return dev_err_probe(pcie->dev, -ENODEV, + "PCIe-C resume resources are incomplete\n"); + + debug->flags |= IORESOURCE_MEM_NONPOSTED; + pcie->debug_base = devm_ioremap_resource(pcie->dev, debug); + if (IS_ERR(pcie->debug_base)) + return PTR_ERR(pcie->debug_base); + + fabric->flags |= IORESOURCE_MEM_NONPOSTED; + pcie->fabric_base = devm_ioremap_resource(pcie->dev, fabric); + if (IS_ERR(pcie->fabric_base)) + return PTR_ERR(pcie->fabric_base); + + intr2axi->flags |= IORESOURCE_MEM_NONPOSTED; + pcie->intr2axi_base = devm_ioremap_resource(pcie->dev, intr2axi); + if (IS_ERR(pcie->intr2axi_base)) + return PTR_ERR(pcie->intr2axi_base); + + pcie->rc_tunable = devm_apple_tunable_parse(pcie->dev, + pcie->dev->of_node, + "apple,tunable-rc", config); + if (IS_ERR(pcie->rc_tunable)) + return dev_err_probe(pcie->dev, PTR_ERR(pcie->rc_tunable), + "PCIe-C root-complex tunables unavailable\n"); + + pcie->debug_tunable = devm_apple_tunable_parse(pcie->dev, + pcie->dev->of_node, + "apple,tunable-debug", debug); + if (IS_ERR(pcie->debug_tunable)) + return dev_err_probe(pcie->dev, PTR_ERR(pcie->debug_tunable), + "PCIe-C debug tunables unavailable\n"); + + pcie->fabric_tunable = devm_apple_tunable_parse(pcie->dev, + pcie->dev->of_node, + "apple,tunable-fabric", fabric); + if (IS_ERR(pcie->fabric_tunable)) + return dev_err_probe(pcie->dev, PTR_ERR(pcie->fabric_tunable), + "PCIe-C fabric tunables unavailable\n"); + + return 0; +} + +static bool apple_pcie_tunnel_power_is_retained(struct device_node *np) +{ + struct device_node *pd_np; + bool retained; + + pd_np = of_parse_phandle(np, "power-domains", 0); + if (!pd_np) + return false; + + retained = of_property_read_bool(pd_np, "apple,always-on"); + of_node_put(pd_np); + return retained; +} + +struct apple_pcie_tunnel_link { + struct device *consumer; + struct device *supplier; +}; + +static void apple_pcie_tunnel_delete_link(void *data) +{ + struct apple_pcie_tunnel_link *link = data; + + /* + * A hot-unplugged endpoint purges its device links during unregister, so + * the struct device_link returned by device_link_add() may already be + * gone by the time PCIe-C releases its devres. Resolve the link by its + * refcounted endpoints instead; device_link_remove() is a no-op after a + * purge and avoids dereferencing a stale link object. + */ + device_link_remove(link->consumer, link->supplier); + put_device(link->consumer); + put_device(link->supplier); +} + +static int apple_pcie_tunnel_add_link(struct apple_pcie *pcie, + struct device *consumer, + struct device *supplier) +{ + struct apple_pcie_tunnel_link *ref; + struct device_link *link; + int ret; + + link = device_link_add(consumer, supplier, DL_FLAG_STATELESS); + if (!link) + return -EINVAL; + + ref = devm_kzalloc(pcie->dev, sizeof(*ref), GFP_KERNEL); + if (!ref) { + device_link_del(link); + return -ENOMEM; + } + + ref->consumer = get_device(consumer); + ref->supplier = get_device(supplier); + ret = devm_add_action_or_reset(pcie->dev, + apple_pcie_tunnel_delete_link, ref); + return ret; +} + +static int apple_pcie_tunnel_keep_d0(struct pci_dev *pdev, void *data) +{ + /* + * The remote USB4 PCIe hierarchy becomes unreachable while its tunnel + * is asleep, so a device left in D3hot cannot receive the config write + * that would bring it back to D0. Quiesce drivers normally, but leave + * PCI power-state ownership to the tunnel across system sleep. + */ + pdev->dev_flags |= PCI_DEV_FLAGS_NO_D3; + return 0; +} + +static int apple_pcie_tunnel_add_links(struct apple_pcie *pcie) +{ + struct platform_device *dart_pdev = NULL; + struct platform_device *nhi_pdev = NULL; + int ret; + + /* + * Apple orders clientDidSleep after disabling the PCIe-C port, and + * clientWillWake before enabling it. NHI and PCIe-C are sibling devices + * under ACIO, so encode that missing edge explicitly: Linux must suspend + * PCIe-C before NHI tears down the router state and resume NHI before the + * host touches its tunneled aperture. + */ + for_each_available_child_of_node_scoped(pcie->dev->parent->of_node, + sibling) { + if (!of_node_name_prefix(sibling, "nhi")) + continue; + + nhi_pdev = of_find_device_by_node(sibling); + break; + } + if (!nhi_pdev) + return dev_err_probe(pcie->dev, -EPROBE_DEFER, + "Apple NHI device is not ready\n"); + + ret = apple_pcie_tunnel_add_link(pcie, pcie->dev, &nhi_pdev->dev); + put_device(&nhi_pdev->dev); + if (ret) + return dev_err_probe(pcie->dev, ret, + "failed to order PCIe-C after NHI resume\n"); + + /* + * The DART and PCIe-C host are synthesized as siblings. Apple restores + * port hardware and RID/SID forwarding before forcing the DART active; + * express the same dependency so Linux suspends DART first and resumes + * PCIe-C first. + */ + for_each_available_child_of_node_scoped(pcie->dev->of_node->parent, + sibling) { + if (!of_property_present(sibling, "#iommu-cells")) + continue; + + dart_pdev = of_find_device_by_node(sibling); + break; + } + if (!dart_pdev) + return dev_err_probe(pcie->dev, -EPROBE_DEFER, + "PCIe-C DART device is not ready\n"); + + ret = apple_pcie_tunnel_add_link(pcie, &dart_pdev->dev, pcie->dev); + put_device(&dart_pdev->dev); + if (ret) + return dev_err_probe(pcie->dev, ret, + "failed to order PCIe-C before DART resume\n"); + + dev_info(pcie->dev, + "PCIe-C ordered after NHI and before DART resume (%s power)\n", + pcie->power_retained ? "retained" : "cycled"); + return 0; +} + static int apple_pcie_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1264,6 +1779,13 @@ static int apple_pcie_probe(struct platform_device *pdev) pcie->base = devm_platform_ioremap_resource(pdev, 1); if (IS_ERR(pcie->base)) return PTR_ERR(pcie->base); + if (pcie->hw->tunneled) { + ret = apple_pcie_tunnel_init_resources(pdev, pcie); + if (ret) + return ret; + pcie->power_retained = + apple_pcie_tunnel_power_is_retained(dev->of_node); + } mutex_init(&pcie->lock); INIT_LIST_HEAD(&pcie->ports); @@ -1282,14 +1804,127 @@ static int apple_pcie_probe(struct platform_device *pdev) * Port mappings are allocated by the ECAM init callback. Register cleanup * afterwards so devres runs it before unmapping those registers. */ - return devm_add_action_or_reset(dev, apple_pcie_cleanup, pcie); + ret = devm_add_action_or_reset(dev, apple_pcie_cleanup, pcie); + if (ret) + return ret; + + if (pcie->hw->tunneled) { + pci_walk_bus(bridge->bus, apple_pcie_tunnel_keep_d0, NULL); + return apple_pcie_tunnel_add_links(pcie); + } + + return 0; +} + +int apple_pcie_tunnel_quiesce(struct device *dev) +{ + struct pci_host_bridge *bridge = dev_get_drvdata(dev); + struct apple_pcie *pcie; + struct apple_pcie_port *port; + int ret = 0; + + if (!bridge || !bridge->bus) + return -ENODEV; + + pcie = pci_host_bridge_priv(bridge); + if (!pcie->hw->tunneled) + return -EINVAL; + + /* + * ACIO's NHI is the control plane for tunneled PERST. Stop PCI + * function drivers first, then complete the port reset handshake while + * the NHI and PCIe-C DART are still alive. The platform device remains + * bound so ACIO can remove the NHI before it finally destroys this host. + */ + pci_lock_rescan_remove(); + if (!pcie->bus_stopped) { + pci_walk_bus(bridge->bus, pci_dev_set_disconnected, NULL); + pci_stop_root_bus(bridge->bus); + pcie->bus_stopped = true; + } + + list_for_each_entry(port, &pcie->ports, entry) { + int err; + + if (!port->started) + continue; + err = apple_pcie_tunnel_stop(port); + if (err && !ret) + ret = err; + } + pci_unlock_rescan_remove(); + + if (!ret) + dev_info(dev, "PCIe-C hierarchy quiesced before NHI shutdown\n"); + + return ret; } +EXPORT_SYMBOL_GPL(apple_pcie_tunnel_quiesce); static void apple_pcie_remove(struct platform_device *pdev) { - pci_host_common_remove(pdev); + struct pci_host_bridge *bridge = platform_get_drvdata(pdev); + struct apple_pcie *pcie = pci_host_bridge_priv(bridge); + + /* + * A USB4 cable pull is a surprise removal: the remote hierarchy is + * already unreachable by the time ACIO depopulates PCIe-C. Mark it + * permanently disconnected before unbinding drivers, matching pciehp's + * surprise-removal path. In particular, this keeps NVMe teardown from + * issuing MMIO to the dead tunneled aperture and wedging the SoC. + */ + pci_lock_rescan_remove(); + if (!pcie->bus_stopped) { + if (pcie->hw->tunneled) + pci_walk_bus(bridge->bus, pci_dev_set_disconnected, NULL); + pci_stop_root_bus(bridge->bus); + } + pci_remove_root_bus(bridge->bus); + pci_unlock_rescan_remove(); +} + +static int apple_pcie_suspend_noirq(struct device *dev) +{ + struct apple_pcie *pcie = apple_pcie_lookup(dev); + struct apple_pcie_port *port; + int i; + + if (!pcie->hw->tunneled) + return 0; + list_for_each_entry(port, &pcie->ports, entry) { + if (port->started) { + for_each_set_bit(i, port->sid_map, port->sid_map_sz) + port->saved_rid2sid[i] = + apple_pcie_port_readl(port, + port_rid2sid_offset(port, i)); + apple_pcie_tunnel_stop(port); + } + } + return 0; +} + +static int apple_pcie_resume_noirq(struct device *dev) +{ + struct apple_pcie *pcie = apple_pcie_lookup(dev); + struct apple_pcie_port *port; + int ret; + + if (!pcie->hw->tunneled) + return 0; + list_for_each_entry(port, &pcie->ports, entry) { + ret = apple_pcie_tunnel_start(port); + if (ret) + return ret; + } + + return 0; } +static const struct dev_pm_ops apple_pcie_pm_ops = { + .suspend_noirq = apple_pcie_suspend_noirq, + .resume_noirq = apple_pcie_resume_noirq, +}; + static const struct of_device_id apple_pcie_of_match[] = { { .compatible = "apple,t8103-pciec", .data = &t8103_pciec_hw }, { .compatible = "apple,t6000-pciec", .data = &t8103_pciec_hw }, @@ -1306,6 +1941,7 @@ static struct platform_driver apple_pcie_driver = { .name = "pcie-apple", .of_match_table = apple_pcie_of_match, .suppress_bind_attrs = true, + .pm = &apple_pcie_pm_ops, }, }; module_platform_driver(apple_pcie_driver); diff --git a/include/linux/pci-apple.h b/include/linux/pci-apple.h new file mode 100644 index 0000000000000..29cff22f9cc0e --- /dev/null +++ b/include/linux/pci-apple.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_PCI_APPLE_H +#define _LINUX_PCI_APPLE_H + +struct device; + +int apple_pcie_tunnel_quiesce(struct device *dev); + +#endif /* _LINUX_PCI_APPLE_H */ From ae3661a7a19093c7ee45539d342d3a68c6abb12d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 29 Aug 2026 04:05:17 -0400 Subject: [PATCH 0023/2680] PCI: apple: order PCIe-C hot-reconnect setup Signed-off-by: DjDeveloperr --- drivers/pci/controller/pcie-apple.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 707aaf31da6a2..f8bde92a4f44e 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -732,6 +732,26 @@ static int apple_pcie_tunnel_reinitialize(struct apple_pcie_port *port) apple_pcie_tunnel_reset_hardware(port); apple_pcie_tunnel_apply_tunable(port->base, port->tunable); + /* + * Keep the port disabled and its downstream reset asserted while the + * root complex is configured. The enable sequence does this explicitly + * after applying the port tunables; releasing either + * one early occasionally leaves a hot-reconnected USB4 endpoint unable to + * train even though the tunnel itself is already live. + */ + apple_pcie_port_rmw_clear(port, PORT_APPCLK_EN, PORT_APPCLK); + apple_pcie_port_rmw_clear(port, PORT_PERST_OFF, + pcie->hw->port_perst); + apple_pcie_tunnel_apply_tunable(pcie->cfg->win, pcie->rc_tunable); + apple_pcie_port_rmw_clear(port, PORT_APPCLK_CGDIS, PORT_APPCLK); + apple_pcie_port_writel(port, PORT_COUNTER_ENABLE, PORT_COUNTER_CTRL); + apple_pcie_port_writel(port, ~0, PORT_INTSTAT); + apple_pcie_port_writel(port, ~0, PORT_LINKCMDSTS); + apple_pcie_tunnel_writel(pcie->intr2axi_base, + PCIEC_INTR2AXI_ENABLE, + PCIEC_INTR2AXI_CTRL); + + /* Release reset immediately before enabling the configured port. */ apple_pcie_port_rmw_set(port, PORT_PERST_OFF, pcie->hw->port_perst); apple_pcie_port_rmw_set(port, PORT_APPCLK_EN, PORT_APPCLK); @@ -749,11 +769,6 @@ static int apple_pcie_tunnel_reinitialize(struct apple_pcie_port *port) return dev_err_probe(pcie->dev, ret, "PCIe-C port did not become idle after hot reconnect\n"); - apple_pcie_port_rmw_clear(port, PORT_APPCLK_CGDIS, PORT_APPCLK); - apple_pcie_tunnel_apply_tunable(pcie->cfg->win, pcie->rc_tunable); - apple_pcie_port_writel(port, PORT_COUNTER_ENABLE, PORT_COUNTER_CTRL); - apple_pcie_tunnel_writel(pcie->intr2axi_base, PCIEC_INTR2AXI_ENABLE, - PCIEC_INTR2AXI_CTRL); for (i = 0; i < pcie->hw->max_rid2sid; i++) apple_pcie_rid2sid_write(port, i, 0); From 7b7c6ce1d7d93e3f5e9c24e05917615f196dbe88 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 31 Aug 2026 16:18:53 -0400 Subject: [PATCH 0024/2680] thunderbolt: apple: tear down PCIe-C when its tunnel goes away MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only an NHI shutdown quiesced the tunneled PCI hierarchy. A tunnel torn down while the router stays up left the endpoint enumerated with its data path gone, so every access to it blocked until the function driver's own timeout fired. For NVMe that is around a minute of hung I/O ending in nvme nvme1: controller is down; CSTS=0xffffffff, PCI_STATUS=0xffff nvme nvme1: Disabling device after reset failure: -19 and the device never came back when the tunnel did, because nothing had removed it for the host to enumerate again. Quiesce PCIe-C from the new deactivate callback, and restore it when the tunnel is activated again. Restoring restarts the ports the way resume does and rescans the bus. Quiescing now removes the hierarchy rather than only stopping it. Stopping unbinds the drivers but leaves the pci_dev objects behind, so a later rescan finds stale devices: the endpoint returns with an unbalanced runtime-PM count and its driver fails to probe. [ryan: adapted to 7.2.10 — split by topic: only the drivers/pci/controller/pcie-apple.c, include/linux/pci-apple.h hunks of this usb commit; the rest is in 170-usb] Signed-off-by: DjDeveloperr --- drivers/pci/controller/pcie-apple.c | 54 ++++++++++++++++++++++++++++- include/linux/pci-apple.h | 1 + 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index f8bde92a4f44e..3102e32162ec4 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -1853,8 +1853,21 @@ int apple_pcie_tunnel_quiesce(struct device *dev) */ pci_lock_rescan_remove(); if (!pcie->bus_stopped) { + struct pci_dev *pdev, *tmp; + pci_walk_bus(bridge->bus, pci_dev_set_disconnected, NULL); - pci_stop_root_bus(bridge->bus); + + /* + * Remove rather than merely stop. Stopping unbinds the drivers + * but leaves the pci_dev objects behind, and a later rescan + * then finds stale devices instead of enumerating fresh ones: + * the endpoint returns with an unbalanced runtime-PM count and + * its driver fails to probe. + */ + list_for_each_entry_safe(pdev, tmp, &bridge->bus->devices, + bus_list) + pci_stop_and_remove_bus_device(pdev); + pcie->bus_stopped = true; } @@ -1876,6 +1889,45 @@ int apple_pcie_tunnel_quiesce(struct device *dev) } EXPORT_SYMBOL_GPL(apple_pcie_tunnel_quiesce); +/* + * Bring the tunneled host back after apple_pcie_tunnel_quiesce(). The ports + * need the restart that resume performs, and the hierarchy below them has to be + * enumerated again because quiescing removed it. + */ +int apple_pcie_tunnel_restore(struct device *dev) +{ + struct pci_host_bridge *bridge = dev_get_drvdata(dev); + struct apple_pcie *pcie; + struct apple_pcie_port *port; + int ret = 0; + + if (!bridge || !bridge->bus) + return -ENODEV; + + pcie = pci_host_bridge_priv(bridge); + if (!pcie->hw->tunneled) + return -EINVAL; + if (!pcie->bus_stopped) + return 0; + + list_for_each_entry(port, &pcie->ports, entry) { + int err = apple_pcie_tunnel_start(port); + + if (err && !ret) + ret = err; + } + + pci_lock_rescan_remove(); + pci_rescan_bus(bridge->bus); + pcie->bus_stopped = false; + pci_unlock_rescan_remove(); + + dev_info(dev, "PCIe-C hierarchy restored after tunnel activation\n"); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_pcie_tunnel_restore); + static void apple_pcie_remove(struct platform_device *pdev) { struct pci_host_bridge *bridge = platform_get_drvdata(pdev); diff --git a/include/linux/pci-apple.h b/include/linux/pci-apple.h index 29cff22f9cc0e..ef664800801fe 100644 --- a/include/linux/pci-apple.h +++ b/include/linux/pci-apple.h @@ -5,5 +5,6 @@ struct device; int apple_pcie_tunnel_quiesce(struct device *dev); +int apple_pcie_tunnel_restore(struct device *dev); #endif /* _LINUX_PCI_APPLE_H */ From 8037cee681ba93efc21ffa1d9136529bdd13eba2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 00:02:34 -0400 Subject: [PATCH 0025/2680] dt-bindings: describe Apple tunneled PCIe-C resources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — split by topic: only the apple,pcie.yaml hunk, which belongs with the PCIe host binding; the apple,dart.yaml hunk is in 020-dart and the apple,t8103-usb4-acio.yaml hunk in 170-usb] Signed-off-by: DjDeveloperr --- .../devicetree/bindings/pci/apple,pcie.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 58ea6eb8d6073..e38960f7607ff 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -160,8 +160,24 @@ allOf: then: properties: reg: - minItems: 7 + minItems: 5 maxItems: 7 + reg-names: + oneOf: + - items: + - const: config + - const: rc + - const: port0 + - const: fabric + - const: debug + - items: + - const: config + - const: rc + - const: port0 + - const: fabric + - const: debug + - const: ltssm + - const: intr2axi interrupts: maxItems: 1 ranges: From f46a99536f5ea629550f2c3088e78bc0ff43ff2f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 29 Aug 2026 00:48:57 -0400 Subject: [PATCH 0026/2680] dt-bindings: apple: validate PCIe-C and USB4 topology MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — split by topic: only the apple,pcie.yaml hunk, which belongs with the PCIe host binding; the apple,dart.yaml hunk is in 020-dart and the apple,t8103-usb4-acio.yaml hunk in 170-usb] Signed-off-by: DjDeveloperr --- .../devicetree/bindings/pci/apple,pcie.yaml | 38 ++++++++++++------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index e38960f7607ff..6a6608dca4d1b 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -61,21 +61,11 @@ properties: reg-names: minItems: 3 - items: - - const: config - - const: rc - - const: port0 - - const: port1 - - const: port2 - - const: port3 - - const: phy0 - - const: phy1 - - const: phy2 - - const: phy3 + maxItems: 10 ranges: minItems: 2 - maxItems: 2 + maxItems: 3 interrupts: description: @@ -94,6 +84,8 @@ properties: power-domains: maxItems: 1 + dma-coherent: true + patternProperties: "^pci@": $ref: /schemas/pci/pci-bus.yaml# @@ -108,13 +100,16 @@ patternProperties: description: Optional GPIO to power on the device maxItems: 1 + interrupt-controller: true + required: - - reset-gpios - interrupt-controller - "#interrupt-cells" - interrupt-map-mask - interrupt-map + unevaluatedProperties: false + required: - compatible - reg @@ -185,8 +180,25 @@ allOf: maxItems: 3 else: properties: + reg-names: + minItems: 3 + items: + - const: config + - const: rc + - const: port0 + - const: port1 + - const: port2 + - const: port3 + - const: phy0 + - const: phy1 + - const: phy2 + - const: phy3 ranges: maxItems: 2 + patternProperties: + "^pci@": + required: + - reset-gpios examples: - | From 3634bf7aad8d467600082f19191e1086192815a9 Mon Sep 17 00:00:00 2001 From: Ace <7362641+Acelogic@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:20:12 -0400 Subject: [PATCH 0027/2680] dt-bindings: PCI: apple: Describe T8140 register resources Describe the T8140 host with its root-controller window and three port/PHY pairs. The resource layout is established by the pinned public m1n1 T8140 initialization and a fresh J700 ADT topology capture. Validate the binding and the J700 DTB with dt-schema. This binding change does not establish downstream configuration or endpoint DMA support. --- .../devicetree/bindings/pci/apple,pcie.yaml | 52 ++++++++++++++----- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 6a6608dca4d1b..e53142bc239b4 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -49,6 +49,7 @@ properties: - const: apple,t8122-pcie - const: apple,t6020-pcie - const: apple,t6020-pcie + - const: apple,t8140-pcie - const: apple,t8103-pciec - items: - const: apple,t8112-pciec @@ -180,21 +181,46 @@ allOf: maxItems: 3 else: properties: - reg-names: - minItems: 3 - items: - - const: config - - const: rc - - const: port0 - - const: port1 - - const: port2 - - const: port3 - - const: phy0 - - const: phy1 - - const: phy2 - - const: phy3 ranges: maxItems: 2 + allOf: + - if: + properties: + compatible: + contains: + const: apple,t8140-pcie + then: + properties: + reg: + minItems: 8 + maxItems: 8 + reg-names: + items: + - const: config + - const: rc + - const: port0 + - const: port1 + - const: port2 + - const: phy0 + - const: phy1 + - const: phy2 + interrupts: + maxItems: 3 + else: + properties: + reg-names: + minItems: 3 + items: + - const: config + - const: rc + - const: port0 + - const: port1 + - const: port2 + - const: port3 + - const: phy0 + - const: phy1 + - const: phy2 + - const: phy3 patternProperties: "^pci@": required: From 8d8c16df436d5a515d24153c93d733ff32981928 Mon Sep 17 00:00:00 2001 From: Ace <7362641+Acelogic@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:22:25 -0400 Subject: [PATCH 0028/2680] PCI: apple: Add a guarded T8140 root-port bring-up path The public T8140 bootloader initializes PCIe, but Linux has no matching host variant. Retain the initialized clocks, power-cycle the endpoint under PERST, and add the T8140 SID-indexed RID table policy. Reject every downstream config access and inactive root slot before mapping ECAM: PIODMA configuration transport and DMA ownership remain unqualified. This is a diagnostic prerequisite, not complete T8140 PCIe support. The first physical test reported link-up after 12ms and enumerated the 106b:100c root bridge; no downstream configuration or DMA was attempted. Register layout uses public m1n1 and fresh ADT observations. Takeover, power timing and RID table behavior also use coordinator-reviewed prior Aurora reference [cc917e8e179862c02160154128092d40d5c5042c]. This is not an independent cleanroom claim. Original Neo research and bring-up are credited to DJ (DjDeveloperr), Ace (Acelogic), and Ryan Murray. --- drivers/pci/controller/pcie-apple.c | 131 ++++++++++++++++++++++++---- 1 file changed, 113 insertions(+), 18 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 3102e32162ec4..7b48de884f141 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -168,6 +168,11 @@ struct hw_info { u32 port_rid2sid; u32 port_msimap; u32 max_rid2sid; + u32 pwren_off_ms; + u32 pwren_on_ms; + bool sid_indexed_rid2sid; + bool retain_clocks; + bool root_bus_only; bool tunneled; }; @@ -194,6 +199,24 @@ static const struct hw_info t602x_hw = { .max_rid2sid = 512, }; +/* + * T8140 uses the newer port register layout and a SID-indexed RID table. + * Keep the PHY and application clock policy established by the bootloader; + * runtime clock gating has not been qualified on this controller. + */ +static const struct hw_info t8140_hw = { + .port_msiaddr = PORT_T602X_MSIADDR, + .port_perst = PORT_T602X_PERST, + .port_rid2sid = PORT_T602X_RID2SID, + .port_msimap = PORT_T602X_MSIMAP, + .max_rid2sid = 19, + .pwren_off_ms = 2, + .pwren_on_ms = 150, + .sid_indexed_rid2sid = true, + .retain_clocks = true, + .root_bus_only = true, +}; + static const struct hw_info t8103_pciec_hw = { .port_msiaddr = PORT_MSIADDR, .port_perst = PORT_PERST, @@ -1114,25 +1137,40 @@ static int apple_pcie_setup_link(struct apple_pcie *pcie, for (u32 idx = 0; idx < num_aux_resets; idx++) gpiod_set_value_cansleep(aux_reset[idx], 1); - /* Power on the device if required */ - gpiod_set_value_cansleep(pwren, 1); - - ret = apple_pcie_setup_refclk(pcie, port); - if (ret < 0) + /* Power-cycle devices which cannot inherit their firmware state. */ + if (pcie->hw->pwren_off_ms) { + if (!pwren) + return -EINVAL; + rmw_clear(PORT_PERST_OFF, port->base + pcie->hw->port_perst); + ret = gpiod_set_value_cansleep(pwren, 0); + if (ret) + return ret; + msleep(pcie->hw->pwren_off_ms); + } + ret = gpiod_set_value_cansleep(pwren, 1); + if (ret) return ret; + if (!pcie->hw->retain_clocks) { + ret = apple_pcie_setup_refclk(pcie, port); + if (ret < 0) + return ret; + } + /* * The minimal Tperst-clk value is 100us (PCIe CEM r5.0, 2.9.2) * If powering up, the minimal Tpvperl is 100ms */ if (pwren) - msleep(100); + msleep(pcie->hw->pwren_on_ms ?: 100); else usleep_range(100, 200); /* Deassert PERST# */ rmw_set(PORT_PERST_OFF, port->base + pcie->hw->port_perst); - gpiod_set_value_cansleep(reset, 0); + ret = gpiod_set_value_cansleep(reset, 0); + if (ret) + return ret; for (u32 idx = 0; idx < num_aux_resets; idx++) gpiod_set_value_cansleep(aux_reset[idx], 0); @@ -1259,6 +1297,15 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, goto err_teardown; } } else { + if (pcie->hw->retain_clocks) + rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); + if (pcie->hw->retain_clocks && + !(apple_pcie_port_readl(port, PORT_STATUS) & PORT_STATUS_READY)) { + ret = dev_err_probe(pcie->dev, -ENODEV, + "port %pOF requires bootloader initialization\n", + np); + goto err_teardown; + } /* U-Boot may already have brought up a conventional root port. */ link_stat = apple_pcie_port_readl(port, PORT_LINKSTS); if (!(link_stat & PORT_LINKSTS_UP)) { @@ -1268,14 +1315,18 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, } } - if (pcie->hw->port_refclk) - rmw_clear(PORT_REFCLK_CGDIS, port->base + pcie->hw->port_refclk); - else if (port->phy) - rmw_set(PHY_LANE_CFG_REFCLKCGEN, port->phy + PHY_LANE_CFG); - - /* The preinitialized PCIe-C APPCLK state is part of the m1n1 handoff. */ - if (!pcie->hw->tunneled) - rmw_clear(PORT_APPCLK_CGDIS, port->base + PORT_APPCLK); + if (!pcie->hw->retain_clocks) { + if (pcie->hw->port_refclk) + rmw_clear(PORT_REFCLK_CGDIS, + port->base + pcie->hw->port_refclk); + else if (port->phy) + rmw_set(PHY_LANE_CFG_REFCLKCGEN, + port->phy + PHY_LANE_CFG); + + /* PCIe-C APPCLK state is part of the m1n1 handoff. */ + if (!pcie->hw->tunneled) + rmw_clear(PORT_APPCLK_CGDIS, port->base + PORT_APPCLK); + } ret = apple_pcie_port_setup_irq(port); if (ret) @@ -1323,6 +1374,11 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, } if (pcie->hw->tunneled) port->started = true; + if (pcie->hw->root_bus_only) + dev_info(pcie->dev, + "port %pOF RUN=%#x link=%#x; downstream config blocked\n", + np, apple_pcie_port_readl(port, PORT_STATUS), + apple_pcie_port_readl(port, PORT_LINKSTS)); return 0; @@ -1468,7 +1524,18 @@ static int apple_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_d sid = iommu_spec.args[0]; mutex_lock(&port->pcie->lock); - idx = bitmap_find_free_region(port->sid_map, port->sid_map_sz, 0); + if (port->pcie->hw->sid_indexed_rid2sid) { + if (sid >= port->sid_map_sz) + idx = -EINVAL; + else if (test_and_set_bit(sid, port->sid_map)) + idx = -EBUSY; + else + idx = sid; + } else { + idx = bitmap_find_free_region(port->sid_map, port->sid_map_sz, 0); + if (idx < 0) + idx = -ENOSPC; + } if (idx >= 0) { apple_pcie_rid2sid_write(port, idx, PORT_RID2SID_VALID | @@ -1480,7 +1547,7 @@ static int apple_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_d mutex_unlock(&port->pcie->lock); - return idx >= 0 ? 0 : -ENOSPC; + return idx >= 0 ? 0 : idx; } static void apple_pcie_disable_device(struct pci_host_bridge *bridge, struct pci_dev *pdev) @@ -1533,12 +1600,37 @@ static int apple_pcie_init(struct pci_config_window *cfg) return 0; } +/* + * The staged T8140 path has no qualified PIODMA config transport yet. + * Reject downstream and inactive root-port accesses before mapping ECAM. + * Both generic config reads and writes use this callback. + */ +static void __iomem *apple_pcie_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) +{ + struct pci_config_window *cfg = bus->sysdata; + struct apple_pcie *pcie = apple_pcie_lookup(cfg->parent); + struct apple_pcie_port *port; + + if (!pcie->hw->root_bus_only) + return pci_ecam_map_bus(bus, devfn, where); + + if (bus->number != cfg->busr.start || PCI_FUNC(devfn)) + return NULL; + + list_for_each_entry(port, &pcie->ports, entry) + if (port->idx == PCI_SLOT(devfn)) + return pci_ecam_map_bus(bus, devfn, where); + + return NULL; +} + static const struct pci_ecam_ops apple_pcie_cfg_ecam_ops = { .init = apple_pcie_init, .enable_device = apple_pcie_enable_device, .disable_device = apple_pcie_disable_device, .pci_ops = { - .map_bus = pci_ecam_map_bus, + .map_bus = apple_pcie_map_bus, .read = pci_generic_config_read, .write = pci_generic_config_write, } @@ -1791,6 +1883,8 @@ static int apple_pcie_probe(struct platform_device *pdev) pcie = pci_host_bridge_priv(bridge); pcie->dev = dev; pcie->hw = hw; + if (hw->root_bus_only) + dev_info(dev, "root-port probe only; downstream config is blocked\n"); pcie->base = devm_platform_ioremap_resource(pdev, 1); if (IS_ERR(pcie->base)) return PTR_ERR(pcie->base); @@ -1996,6 +2090,7 @@ static const struct of_device_id apple_pcie_of_match[] = { { .compatible = "apple,t8103-pciec", .data = &t8103_pciec_hw }, { .compatible = "apple,t6000-pciec", .data = &t8103_pciec_hw }, { .compatible = "apple,t6020-pcie", .data = &t602x_hw }, + { .compatible = "apple,t8140-pcie", .data = &t8140_hw }, { .compatible = "apple,pcie", .data = &t8103_hw }, { } }; From 1d3e50fee6f4b01aa9036d62fdc73533b2aaa108 Mon Sep 17 00:00:00 2001 From: Ace <7362641+Acelogic@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:21:30 -0400 Subject: [PATCH 0029/2680] dt-bindings: pci: describe T8140's experimental PIODMA bootstrap Describe the opt-in retained bootstrap provider and its host phandle. Keep the association restricted to T8140. Schema and J700 DTB checks passed with the integrated public Aurora tree; no runtime register or DMA layout changes. --- .../dma/apple,t8140-piodma-diagnostic.yaml | 56 +++++++++++++++++++ .../devicetree/bindings/pci/apple,pcie.yaml | 16 ++++++ 2 files changed, 72 insertions(+) create mode 100644 Documentation/devicetree/bindings/dma/apple,t8140-piodma-diagnostic.yaml diff --git a/Documentation/devicetree/bindings/dma/apple,t8140-piodma-diagnostic.yaml b/Documentation/devicetree/bindings/dma/apple,t8140-piodma-diagnostic.yaml new file mode 100644 index 0000000000000..a6a8d7d3af77a --- /dev/null +++ b/Documentation/devicetree/bindings/dma/apple,t8140-piodma-diagnostic.yaml @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/dma/apple,t8140-piodma-diagnostic.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 experimental PCIe PIODMA bootstrap engine + +maintainers: + - Ace + +description: | + The J700 diagnostic driver uses this engine for a single retained request + before enabling native downstream ECAM access. It does not expose generic + DMA channels. This experimental binding describes that bounded bootstrap; + it does not establish an engine reset, removal or memory reuse contract. + +properties: + compatible: + const: apple,t8140-piodma-diagnostic + + reg: + items: + - description: Request engine + - description: Interrupt aggregator + - description: PCIe fabric interface + + reg-names: + items: + - const: engine + - const: irq-aggregator + - const: fabric + + interrupts: + maxItems: 1 + + power-domains: + maxItems: 1 + + iommus: + maxItems: 1 + + apple,pcie-host: + $ref: /schemas/types.yaml#/definitions/phandle + description: PCIe host controller whose ECAM access this engine primes. + +required: + - compatible + - reg + - reg-names + - interrupts + - power-domains + - iommus + - apple,pcie-host + +additionalProperties: false diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index e53142bc239b4..48e3ae4ceafbf 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -87,6 +87,11 @@ properties: dma-coherent: true + apple,piodma: + $ref: /schemas/types.yaml#/definitions/phandle + description: + T8140 PIODMA bootstrap engine used before native downstream ECAM access. + patternProperties: "^pci@": $ref: /schemas/pci/pci-bus.yaml# @@ -126,6 +131,17 @@ unevaluatedProperties: false allOf: - $ref: /schemas/pci/pci-host-bridge.yaml# - $ref: /schemas/interrupt-controller/msi-controller.yaml# + - if: + properties: + compatible: + contains: + const: apple,t8140-pcie + then: + properties: + apple,piodma: true + else: + properties: + apple,piodma: false - if: properties: compatible: From b640242650486719942ddffd000a1ae072210550 Mon Sep 17 00:00:00 2001 From: Ace <7362641+Acelogic@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:22:30 -0400 Subject: [PATCH 0030/2680] PCI: apple: enable bounded J700 endpoint bootstrap and enumeration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prime one retained PIODMA request behind an explicit opt-in, validate its completion, then permit typed ECAM access to the two expected radio functions. Program and check the root memory forwarding window and translated endpoint DMA mappings. Add the J700 own-MAC bootloader handoff node. This remains a diagnostic bring-up path. The arena is retained until full reset; removal, suspend, kexec and memory reuse are not qualified. Document those merge blockers rather than treating one completed request as teardown. Built and tested on physical J700/T8140 with public Aurora m1n1/U-Boot/Linux: KDE, WPA2/DHCP/DNS/HTTPS on 2.4 and 5 GHz, Bluetooth discovery and HCI power cycling. Binding and J700 DTB schema checks pass. Executable PCI code matches the tested Image; firmware and unit data are not included. [ryan: adapted to 7.2.10 — split by topic: the device-tree hunks are carried by 000-devicetree] --- Documentation/PCI/controller/apple-t8140.rst | 43 ++ Documentation/PCI/controller/index.rst | 1 + drivers/pci/controller/Kconfig | 11 + drivers/pci/controller/Makefile | 2 + .../pci/controller/pcie-apple-piodma-diag.c | 616 ++++++++++++++++++ .../pci/controller/pcie-apple-piodma-diag.h | 32 + drivers/pci/controller/pcie-apple.c | 203 +++++- 7 files changed, 900 insertions(+), 8 deletions(-) create mode 100644 Documentation/PCI/controller/apple-t8140.rst create mode 100644 drivers/pci/controller/pcie-apple-piodma-diag.c create mode 100644 drivers/pci/controller/pcie-apple-piodma-diag.h diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst new file mode 100644 index 0000000000000..e3d2608d729cf --- /dev/null +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -0,0 +1,43 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +T8140/J700 experimental radio PCIe bootstrap +========================================== + +The J700 path trains port 0, primes native ECAM with one bounded PIODMA +request, checks the completed request and retained arena, and enumerates only +the expected MediaTek Wi-Fi and Bluetooth functions. Typed native ECAM access +is enabled only after this validation. The root bridge memory window covers +the assigned endpoint BARs; DART translates the endpoint DMA streams. + +This is an opt-in bring-up implementation, not general T8140 PCIe support. +Build ``CONFIG_PCIE_APPLE_PIODMA_DIAG=y`` and pass +``pcie_apple_piodma_diag.enumerate=1`` to the kernel. Without the opt-in, the +root-only diagnostic path continues to reject downstream configuration access. +The option excludes suspend and kexec. Failed or uncertain bootstrap state +requires a full external hardware reset; there is no command retry. + +The PIODMA arena remains allocated and owned until that reset. A completed +bootstrap read is not a memory-reuse or engine-reset qualification. Platform +removal, teardown, suspend and arbitrary downstream devices are unqualified; +do not remove the controller or its IOMMU while this experiment is active. +Those lifetime contracts remain merge blockers for production support. + +J700's ``wifi0`` alias and PCI endpoint node allow the public bootloader to +pass the unit's own Wi-Fi address through ``local-mac-address``. The zero +placeholder in the static device tree is not a usable station address. + +Physical validation, 2026-10-02 +------------------------------ + +The public Aurora 7.1.12 base with this platform change booted J700/T8140 into +KDE through public m1n1 and U-Boot. Both endpoint functions enumerated with +translated DMA. The integrated MT7932 fullmac driver subsequently passed +WPA2, DHCP, gateway ping, DNS and verified HTTPS traffic on 2.4 and 5 GHz. +Bluetooth firmware/HCI setup, discovery and a normal HCI off/on cycle passed +while Wi-Fi remained connected. This is evidence of usable radio transport on +the tested platform, not qualification of the excluded lifecycle operations. + +The tested kernel Image SHA256 is +``e9877f3943c5cf04970a1d1b336ea68096377907802063dbce2082b93d20011a``. +Radio source pins, module identity and detailed outcomes belong with the radio +contribution; no firmware, calibration or unit identity is included here. diff --git a/Documentation/PCI/controller/index.rst b/Documentation/PCI/controller/index.rst index c2ce9ccdcfa06..8619be96d0b63 100644 --- a/Documentation/PCI/controller/index.rst +++ b/Documentation/PCI/controller/index.rst @@ -8,3 +8,4 @@ PCI Native Host Bridge and Endpoint Drivers :maxdepth: 2 rcar-pcie-firmware + apple-t8140 diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index e3bb6ff1b6d71..ebdc821fea454 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -56,6 +56,17 @@ config PCIE_APPLE If unsure, say Y if you have an Apple Silicon system. +config PCIE_APPLE_PIODMA_DIAG + bool "J700 PIODMA diagnostic (requires full reset after use)" + depends on PCIE_APPLE=y && ARCH_APPLE && IOMMU_DMA + depends on !PM_SLEEP && !KEXEC_CORE + help + Build the experimental SID17 retained bootstrap provider. It submits + one fixed request, validates typed ECAM and permits enumeration of + the two Neo functions only with the explicit enumerate=1 parameter. + Its arena remains owned until a full reset. This is not production + support, and does not qualify memory reuse or radio operation. + config PCI_VERSATILE bool "ARM Versatile PB PCI controller" depends on ARCH_VERSATILE || COMPILE_TEST diff --git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile index ac8db283f0fea..5984a69a5331e 100644 --- a/drivers/pci/controller/Makefile +++ b/drivers/pci/controller/Makefile @@ -65,3 +65,5 @@ obj-$(CONFIG_ARM64) += pci-thunder-pem.o obj-$(CONFIG_ARM64) += pci-xgene.o endif endif + +obj-$(CONFIG_PCIE_APPLE_PIODMA_DIAG) += pcie-apple-piodma-diag.o diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c new file mode 100644 index 0000000000000..87caa91262e37 --- /dev/null +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -0,0 +1,616 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * J700 PIODMA diagnostic scaffolding, not a production DMA driver. + * + * Public Linux DMA/IOMMU APIs own the SID17 mappings. The register and + * address-view contract is attributed in the accompanying provenance ledger + * to the reviewed Aurora reference and fresh ADT observations. + * + * This built-in driver deliberately retains its coherent arena and device + * power reference until a coordinator-controlled full hardware reset. There + * is no unbind interface, suspend, kexec or retry. Native ECAM is only + * opened by the host after a successful bootstrap and typed read checks. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pcie-apple-piodma-diag.h" + +#define PIODMA_PACKET_MASK DMA_BIT_MASK(36) +#define PIODMA_POINTER_MASK DMA_BIT_MASK(42) +#define PIODMA_POINTER_PREFIX BIT_ULL(40) +#define PIODMA_SID 17 +#define PIODMA_REQUEST_COUNT 1 +#define PIODMA_SLOT_SIZE (3 * PAGE_SIZE) +#define PIODMA_ARENA_SIZE (PIODMA_REQUEST_COUNT * PIODMA_SLOT_SIZE) +#define PIODMA_DATA_OFFSET (2 * PAGE_SIZE) +#define PIODMA_COMPLETION_OFFSET PAGE_SIZE +#define PIODMA_CONFIG_BASE 0x1cb0000000ULL +#define PIODMA_CONFIG_SIZE SZ_256M +#define PIODMA_IRQ_COMPLETE (BIT(0) | BIT(6)) +#define PIODMA_IRQ_MASK GENMASK(9, 0) +#define PIODMA_FIFO_MASK GENMASK(5, 0) +#define PIODMA_IRQ_LIMIT 16 + +static bool enumerate; +module_param(enumerate, bool, 0400); +MODULE_PARM_DESC(enumerate, "Prime once, validate native ECAM, and enumerate the two Neo functions"); + +struct apple_piodma_request { + u16 offset; + u8 function; + u32 mask; + u32 expected; +}; + +static const struct apple_piodma_request requests[PIODMA_REQUEST_COUNT] = { + { PCI_VENDOR_ID, 0, U32_MAX, 0x793214c3 }, +}; + +struct apple_piodma_slot { + struct completion event; + __le32 packet[4]; + __le32 completion[4]; + u32 result; + u32 matched_status; + bool submitted; + bool observed; +}; + +struct apple_piodma_diag { + struct device *dev; + struct iommu_domain *domain; + void __iomem *engine; + void __iomem *fabric; + struct apple_piodma_slot slots[PIODMA_REQUEST_COUNT]; + atomic_t irq_count; + atomic_t attempted; + u32 active_tag; + u64 config_source; + u32 last_irq_status; + u32 last_irq_tag; + int irq; + bool irq_fault; + void *arena; + dma_addr_t dma; + u64 pointer_prefix; + u8 secondary_bus; + bool retained; +}; + +static int apple_piodma_diag_root(struct apple_piodma_diag *diag, struct pci_dev *root) +{ + int ret; + + if (!pci_is_root_bus(root->bus) || pci_domain_nr(root->bus) || + root->bus->number || root->devfn || + root->vendor != PCI_VENDOR_ID_APPLE || root->device != 0x100c || + root->class != PCI_CLASS_BRIDGE_PCI << 8) + return -ENODEV; + ret = pci_read_config_byte(root, PCI_SECONDARY_BUS, &diag->secondary_bus); + if (ret) + return pcibios_err_to_errno(ret); + if (diag->secondary_bus != 1) + return -EINVAL; + diag->config_source += (u64)diag->secondary_bus << 20; + return 0; +} + +static int apple_piodma_diag_iommu(struct apple_piodma_diag *diag) +{ + struct of_phandle_args args; + u64 range[2]; + int ret; + + ret = of_parse_phandle_with_args(diag->dev->of_node, "iommus", + "#iommu-cells", 0, &args); + if (ret) + return ret; + if (args.args_count != 1 || args.args[0] != PIODMA_SID || + !of_device_is_compatible(args.np, "apple,t8140-dart")) { + ret = -EINVAL; + goto out; + } + /* Keep the prior pointer convention with low IAS36 page tables. */ + if (of_property_present(args.np, "apple,dma-offset")) { + ret = -EINVAL; + goto out; + } + diag->pointer_prefix = PIODMA_POINTER_PREFIX; + ret = of_property_read_u64_array(args.np, "apple,dma-range", range, 2); + if (ret) + goto out; + if (diag->pointer_prefix != PIODMA_POINTER_PREFIX || range[0] || + range[1] != BIT_ULL(36)) { + ret = -EINVAL; + goto out; + } + diag->domain = iommu_get_domain_for_dev(diag->dev); + if (!diag->domain || + (diag->domain->type != IOMMU_DOMAIN_DMA && + diag->domain->type != IOMMU_DOMAIN_DMA_FQ)) { + ret = -ENODEV; + goto out; + } + ret = dma_set_mask_and_coherent(diag->dev, PIODMA_PACKET_MASK); +out: + of_node_put(args.np); + return ret; +} + +static int apple_piodma_diag_arena(struct apple_piodma_diag *diag) +{ + u64 last, pointer; + int i; + + diag->arena = dma_alloc_coherent(diag->dev, PIODMA_ARENA_SIZE, &diag->dma, GFP_KERNEL); + if (!diag->arena) + return -ENOMEM; + if (!IS_ALIGNED(diag->dma, SZ_16K) || + check_add_overflow((u64)diag->dma, PIODMA_ARENA_SIZE - 1, &last) || + last > PIODMA_PACKET_MASK || + check_add_overflow(last, diag->pointer_prefix, &pointer) || + pointer > PIODMA_POINTER_MASK || + !diag->domain->geometry.force_aperture || + diag->domain->geometry.aperture_start != 0 || + diag->domain->geometry.aperture_end != PIODMA_PACKET_MASK) + return -ERANGE; + memset(diag->arena, 0, PIODMA_ARENA_SIZE); + for (i = 0; i < PIODMA_REQUEST_COUNT; i++) { + void *slot = diag->arena + i * PIODMA_SLOT_SIZE; + u64 dma = diag->dma + i * PIODMA_SLOT_SIZE; + + memset(slot + PIODMA_DATA_OFFSET, 0xa5, PAGE_SIZE); + *(__le32 *)(slot + PIODMA_DATA_OFFSET) = cpu_to_le32(U32_MAX); + dev_info(diag->dev, + "owned SID17 slot=%d dma=%#llx descriptor=%#llx completion=%#llx data=%#llx\n", + i, dma, dma + diag->pointer_prefix, + dma + diag->pointer_prefix + PIODMA_COMPLETION_OFFSET, + dma + PIODMA_DATA_OFFSET); + } + dma_wmb(); + return 0; +} + + + +static int apple_piodma_diag_resources(struct platform_device *pdev, + struct apple_piodma_diag *diag) +{ + static const struct { + const char *name; + resource_size_t base; + } regions[] = { + { "engine", 0x390030000ULL }, + { "irq-aggregator", 0x390034000ULL }, + { "fabric", 0x390038000ULL }, + }; + struct device_node *host; + struct resource ecam, *res; + int i, ret; + + for (i = 0; i < ARRAY_SIZE(regions); i++) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, regions[i].name); + if (!res || res->start != regions[i].base || resource_size(res) != SZ_16K) + return -EINVAL; + } + host = of_parse_phandle(diag->dev->of_node, "apple,pcie-host", 0); + if (!host) + return -EINVAL; + ret = of_address_to_resource(host, 0, &ecam); + if (!ret && (!of_device_is_compatible(host, "apple,t8140-pcie") || + ecam.start != PIODMA_CONFIG_BASE || + resource_size(&ecam) != PIODMA_CONFIG_SIZE)) + ret = -EINVAL; + of_node_put(host); + if (ret) + return ret; + diag->config_source = ecam.start; + if (diag->config_source + (1 << 12) + sizeof(u32) - 1 > ecam.end) + return -ERANGE; + diag->engine = devm_platform_ioremap_resource_byname(pdev, "engine"); + if (IS_ERR(diag->engine)) + return PTR_ERR(diag->engine); + diag->fabric = devm_platform_ioremap_resource_byname(pdev, "fabric"); + if (IS_ERR(diag->fabric)) + return PTR_ERR(diag->fabric); + diag->irq = platform_get_irq(pdev, 0); + return diag->irq < 0 ? diag->irq : 0; +} + +static int apple_piodma_diag_idle(struct apple_piodma_diag *diag) +{ + static const struct { + u32 offset; + u32 expected; + } engine[] = { + { 0x00, 2 }, { 0x04, 0 }, { 0x08, 0 }, { 0x0c, 16 }, + { 0x44, 0 }, { 0x48, 0 }, { 0x4c, 0 }, { 0x50, 0 }, + { 0x54, 0 }, { 0xd8, 0 }, { 0xdc, 0 }, + }; + int i; + u32 value; + + for (i = 0; i < ARRAY_SIZE(engine); i++) { + value = readl(diag->engine + engine[i].offset); + if (value != engine[i].expected) { + dev_err(diag->dev, "idle mismatch engine[%#x]=%#x expected=%#x\n", + engine[i].offset, value, engine[i].expected); + return -EBUSY; + } + } + if (readl(diag->fabric) || readl(diag->fabric + 4)) + return -EBUSY; + return 0; +} + +static irqreturn_t apple_piodma_diag_irq(int irq, void *data) +{ + struct apple_piodma_diag *diag = data; + struct apple_piodma_slot *slot; + u32 fabric, status, tag, active; + int count; + + /* Acknowledge order follows the attributed prior controller contract. */ + fabric = readl(diag->fabric); + writel(fabric, diag->fabric); + status = readl(diag->engine + 4); + writel(status, diag->engine + 4); + tag = readl(diag->engine + 0xdc) & 0xff; + active = READ_ONCE(diag->active_tag); + count = atomic_inc_return(&diag->irq_count); + WRITE_ONCE(diag->last_irq_status, status); + WRITE_ONCE(diag->last_irq_tag, tag); + dev_info(diag->dev, "irq=%d fabric=%#x status=%#x tag=%#x active=%#x\n", + count, fabric, status, tag, active); + if (count >= PIODMA_IRQ_LIMIT || status & ~PIODMA_IRQ_COMPLETE || + (status && (!tag || tag > active))) { + WRITE_ONCE(diag->irq_fault, true); + disable_irq_nosync(irq); + if (active) + complete(&diag->slots[active - 1].event); + } else if (status & PIODMA_IRQ_COMPLETE && tag == active && active) { + slot = &diag->slots[active - 1]; + WRITE_ONCE(slot->matched_status, READ_ONCE(slot->matched_status) | status); + complete(&slot->event); + } + /* A late old-tag IRQ never wakes or qualifies the current request. */ + return IRQ_HANDLED; +} + +static int apple_piodma_diag_initialize(struct apple_piodma_diag *diag) +{ + static const struct { + u32 offset; + u32 mask; + u32 value; + } tunables[] = { + { 0x00, 0x2, 0 }, + { 0x10, 0x1fffffff, 0x1fbf2 }, + { 0x14, 0x1fffffff, 0x1fbf2 }, + { 0x18, 0x1fffffff, 0x1fbf3 }, + { 0x1c, 0x7ffff, 0x1fbf0 }, + { 0x20, 0x7ffff, 0x1fbf0 }, + }; + int i; + u32 value; + + /* All later failures retain the device, IRQ, domain and arena. */ + diag->retained = true; + writel(PIODMA_IRQ_MASK, diag->engine + 8); + writel(0, diag->fabric + 4); + for (i = 0; i < ARRAY_SIZE(tunables); i++) { + value = readl(diag->engine + tunables[i].offset); + value = (value & ~tunables[i].mask) | tunables[i].value; + writel(value, diag->engine + tunables[i].offset); + } + writel(readl(diag->engine) & ~BIT(0), diag->engine); + /* Exact prior bank values; their architectural meaning is unqualified. */ + for (i = 0; i < 8; i++) { + writel(0, diag->engine + 0x24 + 4 * i); + writel(i == 1 ? 1 : 0, diag->engine + 0xf0 + 4 * i); + } + /* Order the bank writes before their configuration readback. */ + wmb(); + if (readl(diag->engine) || readl(diag->engine + 8) != PIODMA_IRQ_MASK || + readl(diag->engine + 0xf4) != 1 || READ_ONCE(diag->irq_fault)) + return -EIO; + return 0; +} + +static bool apple_piodma_diag_canary(const u8 *memory, size_t size, u8 value) +{ + size_t i; + + for (i = 0; i < size; i++) + if (READ_ONCE(memory[i]) != value) + return false; + return true; +} + +static int apple_piodma_diag_check_slots(struct apple_piodma_diag *diag) +{ + int i, j; + + dma_rmb(); + for (i = 0; i < PIODMA_REQUEST_COUNT; i++) { + struct apple_piodma_slot *slot = &diag->slots[i]; + void *memory = diag->arena + i * PIODMA_SLOT_SIZE; + __le32 *packet = memory; + __le32 *completion = memory + PIODMA_COMPLETION_OFFSET; + __le32 *result = memory + PIODMA_DATA_OFFSET; + + for (j = 0; j < ARRAY_SIZE(slot->packet); j++) { + if (READ_ONCE(packet[j]) != slot->packet[j] || + (slot->observed && READ_ONCE(completion[j]) != slot->completion[j]) || + (!slot->submitted && READ_ONCE(completion[j]))) + goto changed; + } + if ((slot->observed && le32_to_cpu(READ_ONCE(*result)) != slot->result) || + (!slot->submitted && le32_to_cpu(READ_ONCE(*result)) != U32_MAX) || + !apple_piodma_diag_canary(memory + sizeof(slot->packet), + PAGE_SIZE - sizeof(slot->packet), 0) || + !apple_piodma_diag_canary(memory + PIODMA_COMPLETION_OFFSET + + sizeof(slot->completion), + PAGE_SIZE - sizeof(slot->completion), 0) || + !apple_piodma_diag_canary(memory + PIODMA_DATA_OFFSET + sizeof(u32), + PAGE_SIZE - sizeof(u32), 0xa5)) + goto changed; + } + return 0; + +changed: + dev_err(diag->dev, "slot=%d changed outside its permitted response; stop\n", i); + return -EIO; +} + +static int apple_piodma_diag_submit(struct apple_piodma_diag *diag, unsigned int index) +{ + const struct apple_piodma_request *request = &requests[index]; + struct apple_piodma_slot *slot = &diag->slots[index]; + void *memory = diag->arena + index * PIODMA_SLOT_SIZE; + __le32 *packet = memory; + __le32 *result = memory + PIODMA_DATA_OFFSET; + __le32 *completion = memory + PIODMA_COMPLETION_OFFSET; + u64 dma = diag->dma + index * PIODMA_SLOT_SIZE; + u64 source = diag->config_source + ((u64)request->function << 12) + request->offset; + u64 dst = dma + PIODMA_DATA_OFFSET; + u64 src = source & PIODMA_PACKET_MASK; + u64 descriptor = dma + diag->pointer_prefix; + u64 comp = descriptor + PIODMA_COMPLETION_OFFSET; + unsigned long waited; + u32 fifo, command, tag = index + 1; + int i, ret; + + ret = apple_piodma_diag_check_slots(diag); + if (ret || READ_ONCE(diag->irq_fault)) + return ret ?: -EIO; + /* Only these fixed, validated ECAM addresses use aperture1/low36. */ + slot->packet[0] = cpu_to_le32(0xc007 | BIT(18) | ((src & 0xff) << 24)); + slot->packet[1] = cpu_to_le32((src >> 8) | ((dst & 0xf) << 28)); + slot->packet[2] = cpu_to_le32(dst >> 4); + slot->packet[3] = cpu_to_le32(sizeof(u32)); + memcpy(packet, slot->packet, sizeof(slot->packet)); + dma_wmb(); + ret = readl_poll_timeout(diag->engine + 0x0c, fifo, + (fifo & PIODMA_FIFO_MASK) == 16, 10, 50000); + if (ret || READ_ONCE(diag->irq_fault)) + return ret ?: -EIO; + slot->submitted = true; + WRITE_ONCE(diag->active_tag, tag); + dev_info(diag->dev, + "request=%u function=%u offset=%#x source=%#llx packet=%08x:%08x:%08x:%08x tag=%u\n", + index, request->function, request->offset, source, + le32_to_cpu(packet[0]), le32_to_cpu(packet[1]), + le32_to_cpu(packet[2]), le32_to_cpu(packet[3]), tag); + writel(tag, diag->engine + 0xd8); + writel(upper_32_bits(descriptor) & 0x3ff, diag->engine + 0x50); + writel(lower_32_bits(descriptor), diag->engine + 0x4c); + writel(upper_32_bits(comp) & 0x3ff, diag->engine + 0x48); + writel(lower_32_bits(comp), diag->engine + 0x44); + writel(0x411, diag->engine + 0x54); + waited = wait_for_completion_timeout(&slot->event, msecs_to_jiffies(50)); + dma_rmb(); + slot->result = le32_to_cpu(READ_ONCE(*result)); + for (i = 0; i < ARRAY_SIZE(slot->completion); i++) + slot->completion[i] = READ_ONCE(completion[i]); + slot->observed = true; + dev_info(diag->dev, + "response=%u waited=%lu matched_status=%#x word=%#x completion=%08x:%08x:%08x:%08x\n", + index, waited, READ_ONCE(slot->matched_status), slot->result, + le32_to_cpu(slot->completion[0]), le32_to_cpu(slot->completion[1]), + le32_to_cpu(slot->completion[2]), le32_to_cpu(slot->completion[3])); + if (!waited) + return -ETIMEDOUT; + if (READ_ONCE(diag->irq_fault) || !READ_ONCE(slot->matched_status) || + slot->result == U32_MAX || + (slot->result & request->mask) != request->expected) + return -EIO; + /* Admission observations only: all published memory remains retained. */ + ret = readl_poll_timeout(diag->engine + 0x54, command, !(command & BIT(0)), + 10, 50000); + if (ret) + return ret; + ret = readl_poll_timeout(diag->engine + 0x0c, fifo, + (fifo & PIODMA_FIFO_MASK) == 16, 10, 50000); + if (ret) + return ret; + for (i = 0; i < 10; i++) { + usleep_range(1000, 2000); + ret = apple_piodma_diag_check_slots(diag); + if (ret || READ_ONCE(diag->irq_fault)) + return ret ?: -EIO; + } + dev_info(diag->dev, + "READ_VALIDATED request=%u word=%#x FIFO=%#x command=%#x; retained slots stable\n", + index, slot->result, fifo, command); + return 0; +} + +bool apple_piodma_bootstrap_enabled(void) +{ + return enumerate; +} + +static void apple_piodma_put_supplier(void *data) +{ + put_device(data); +} + +int apple_piodma_bootstrap_get(struct device *host, struct device **supplier) +{ + struct device_node *node, *host_node; + struct platform_device *pdev; + struct device_link *link; + bool correct_host; + int ret; + + if (!enumerate) + return -ENODEV; + node = of_parse_phandle(host->of_node, "apple,piodma", 0); + if (!node) + return -EINVAL; + if (!of_device_is_compatible(node, "apple,t8140-piodma-diagnostic")) { + of_node_put(node); + return -EINVAL; + } + host_node = of_parse_phandle(node, "apple,pcie-host", 0); + correct_host = host_node == host->of_node; + of_node_put(host_node); + pdev = correct_host ? of_find_device_by_node(node) : NULL; + of_node_put(node); + if (!correct_host) + return -EINVAL; + if (!pdev) + return -EPROBE_DEFER; + if (!device_is_bound(&pdev->dev) || !platform_get_drvdata(pdev)) { + put_device(&pdev->dev); + return -EPROBE_DEFER; + } + link = device_link_add(host, &pdev->dev, DL_FLAG_AUTOREMOVE_CONSUMER); + if (!link) { + put_device(&pdev->dev); + return -EINVAL; + } + ret = devm_add_action_or_reset(host, apple_piodma_put_supplier, &pdev->dev); + if (ret) + return ret; + *supplier = &pdev->dev; + return 0; +} + +int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root) +{ + struct apple_piodma_diag *diag = dev_get_drvdata(supplier); + int ret; + + /* One attempt for this provider's entire lifetime, including failures. */ + if (atomic_cmpxchg(&diag->attempted, 0, 1)) + return -EALREADY; + ret = apple_piodma_diag_root(diag, root); + if (ret) + return ret; + ret = apple_piodma_diag_idle(diag); + if (ret) + return ret; + ret = apple_piodma_diag_initialize(diag); + if (!ret) + ret = apple_piodma_diag_submit(diag, 0); + if (ret) + dev_err(supplier, "bootstrap failure=%d; retained ownership; no retry\n", ret); + else + dev_info(supplier, "BOOTSTRAP_VALIDATED word=%#x; one retained slot\n", + diag->slots[0].result); + return ret; +} + +static int apple_piodma_diag_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_piodma_diag *diag; + int i, ret; + + if (!enumerate || !of_machine_is_compatible("apple,j700") || + !of_machine_is_compatible("apple,t8140") || PAGE_SIZE != SZ_16K) + return -ENODEV; + diag = kzalloc_obj(*diag); + if (!diag) + return -ENOMEM; + diag->dev = dev; + for (i = 0; i < PIODMA_REQUEST_COUNT; i++) + init_completion(&diag->slots[i].event); + atomic_set(&diag->irq_count, 0); + atomic_set(&diag->attempted, 0); + ret = apple_piodma_diag_iommu(diag); + if (ret) + goto err_free; + pm_runtime_enable(dev); + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) + goto err_pm; + ret = apple_piodma_diag_arena(diag); + if (ret) + goto err_arena; + ret = apple_piodma_diag_resources(pdev, diag); + if (ret) + goto err_arena; + ret = devm_request_irq(dev, diag->irq, apple_piodma_diag_irq, 0, + dev_name(dev), diag); + if (ret) + goto err_arena; + platform_set_drvdata(pdev, diag); + dev_info(dev, "bootstrap supplier ready; no command submitted during probe\n"); + return 0; + +err_arena: + /* No path after initialization may release ownership. */ + if (WARN_ON_ONCE(diag->retained)) + return 0; + if (diag->arena) + dma_free_coherent(dev, PIODMA_ARENA_SIZE, diag->arena, diag->dma); + pm_runtime_put_sync(dev); +err_pm: + pm_runtime_disable(dev); +err_free: + kfree(diag); + return dev_err_probe(dev, ret, "PIODMA diagnostic preflight failed\n"); +} + +static void apple_piodma_diag_shutdown(struct platform_device *pdev) +{ + dev_warn(&pdev->dev, "diagnostic arena retained; require full hardware reset\n"); +} + +static const struct of_device_id apple_piodma_diag_match[] = { + { .compatible = "apple,t8140-piodma-diagnostic" }, + { } +}; + +static struct platform_driver apple_piodma_diag_driver = { + .probe = apple_piodma_diag_probe, + .shutdown = apple_piodma_diag_shutdown, + .driver = { + .name = "apple-piodma-diag", + .of_match_table = apple_piodma_diag_match, + .suppress_bind_attrs = true, + }, +}; +builtin_platform_driver(apple_piodma_diag_driver); +MODULE_DESCRIPTION("T8140 retained one-read PCIe bootstrap experiment"); +MODULE_LICENSE("GPL"); diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.h b/drivers/pci/controller/pcie-apple-piodma-diag.h new file mode 100644 index 0000000000000..46a1b03aa2496 --- /dev/null +++ b/drivers/pci/controller/pcie-apple-piodma-diag.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef PCIE_APPLE_PIODMA_DIAG_H +#define PCIE_APPLE_PIODMA_DIAG_H + +#include +#include +#include + +struct device; +struct pci_dev; + +#if IS_ENABLED(CONFIG_PCIE_APPLE_PIODMA_DIAG) +bool apple_piodma_bootstrap_enabled(void); +int apple_piodma_bootstrap_get(struct device *host, struct device **supplier); +int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root); +#else +static inline bool apple_piodma_bootstrap_enabled(void) +{ + return false; +} + +static inline int apple_piodma_bootstrap_get(struct device *host, struct device **supplier) +{ + return -ENODEV; +} + +static inline int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root) +{ + return -ENODEV; +} +#endif +#endif diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 7b48de884f141..427823d89d741 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -36,6 +36,7 @@ #include "../pci.h" #include "pci-host-common.h" +#include "pcie-apple-piodma-diag.h" static int link_up_timeout = 500; module_param(link_up_timeout, int, 0644); @@ -238,6 +239,9 @@ struct apple_pcie { struct apple_tunable *debug_tunable; bool power_retained; bool bus_stopped; + bool neo_config_ready; + u8 neo_secondary_bus; + struct device *piodma_supplier; const struct hw_info *hw; unsigned long *bitmap; struct list_head ports; @@ -1494,7 +1498,7 @@ static int apple_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_d { struct apple_pcie *pcie = pci_host_bridge_priv(bridge); struct resource *res; - u32 sid, rid = pci_dev_id(pdev); + u32 sid, val, readback, rid = pci_dev_id(pdev); struct apple_pcie_port *port; struct of_phandle_args iommu_spec = {}; int idx, err; @@ -1537,9 +1541,15 @@ static int apple_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_d idx = -ENOSPC; } if (idx >= 0) { - apple_pcie_rid2sid_write(port, idx, - PORT_RID2SID_VALID | - (sid << PORT_RID2SID_SID_SHIFT) | rid); + val = PORT_RID2SID_VALID | (sid << PORT_RID2SID_SID_SHIFT) | rid; + readback = apple_pcie_rid2sid_write(port, idx, val); + if (pcie->hw->sid_indexed_rid2sid && readback != val) { + dev_err(&pdev->dev, "RID-to-SID readback %#x expected %#x\n", + readback, val); + /* Reserve the failed slot; do not enable or recycle it. */ + mutex_unlock(&port->pcie->lock); + return -EIO; + } dev_dbg(&pdev->dev, "mapping RID%x to SID%x (index %d)\n", rid, sid, idx); @@ -1601,9 +1611,9 @@ static int apple_pcie_init(struct pci_config_window *cfg) } /* - * The staged T8140 path has no qualified PIODMA config transport yet. - * Reject downstream and inactive root-port accesses before mapping ECAM. - * Both generic config reads and writes use this callback. + * The T8140 experiment starts with all downstream accesses blocked. + * Only a successful owned bootstrap and typed ECAM checks open functions0/1. + * Generic config callbacks retain their normal native widths and locking. */ static void __iomem *apple_pcie_map_bus(struct pci_bus *bus, unsigned int devfn, int where) @@ -1615,7 +1625,15 @@ static void __iomem *apple_pcie_map_bus(struct pci_bus *bus, if (!pcie->hw->root_bus_only) return pci_ecam_map_bus(bus, devfn, where); - if (bus->number != cfg->busr.start || PCI_FUNC(devfn)) + if (bus->number != cfg->busr.start) { + /* Pair with ready publication before consuming the bus number. */ + if (!smp_load_acquire(&pcie->neo_config_ready) || + bus->number != pcie->neo_secondary_bus || + PCI_SLOT(devfn) || PCI_FUNC(devfn) > 1) + return NULL; + return pci_ecam_map_bus(bus, devfn, where); + } + if (PCI_FUNC(devfn)) return NULL; list_for_each_entry(port, &pcie->ports, entry) @@ -1845,6 +1863,158 @@ static int apple_pcie_tunnel_add_links(struct apple_pcie *pcie) return 0; } +static int apple_pcie_neo_typed_ids(struct apple_pcie *pcie, struct pci_dev *root) +{ + static const u32 ids[] = { 0x793214c3, 0x793b14c3 }; + void __iomem *cfg; + u32 id; + u16 vendor, device; + u8 low, high, header; + int function; + + for (function = 0; function < ARRAY_SIZE(ids); function++) { + /* Bypass the closed guard only for these fixed validation reads. */ + cfg = pci_ecam_map_bus(root->subordinate, PCI_DEVFN(0, function), 0); + if (!cfg) + return -ENODEV; + id = readl(cfg + PCI_VENDOR_ID); + vendor = readw(cfg + PCI_VENDOR_ID); + device = readw(cfg + PCI_DEVICE_ID); + low = readb(cfg + PCI_VENDOR_ID); + high = readb(cfg + PCI_VENDOR_ID + 1); + header = readb(cfg + PCI_HEADER_TYPE); + dev_info(pcie->dev, + "typed ECAM function=%d id=%#x vendor=%#x device=%#x bytes=%02x:%02x header=%#x\n", + function, id, vendor, device, low, high, header); + if (id != ids[function] || vendor != (ids[function] & 0xffff) || + device != ids[function] >> 16 || low != 0xc3 || high != 0x14 || + (header & PCI_HEADER_TYPE_MASK) != PCI_HEADER_TYPE_NORMAL || + (!function && !(header & PCI_HEADER_TYPE_MFD))) + return -EIO; + } + return 0; +} + +static int apple_pcie_neo_check_memory(struct pci_dev *root) +{ + struct pci_dev *endpoint; + struct pci_bus_region region; + struct resource *res; + u32 window = U32_MAX; + u64 base, limit; + u16 command = U16_MAX; + int function, bar, ret = 0; + + ret = pci_read_config_word(root, PCI_COMMAND, &command); + if (!ret) + ret = pci_read_config_dword(root, PCI_MEMORY_BASE, &window); + pci_info(root, "NEO_ROOT_MEMORY_READBACK status=%d command=%#x window=%#x\n", + ret, command, window); + if (ret || command == U16_MAX || !(command & PCI_COMMAND_MEMORY) || window == U32_MAX) + return -EIO; + /* Each packed register field is12 bits; PCI_MEMORY_RANGE_MASK is unsigned long. */ + base = (u64)(window & GENMASK(15, 4)) << 16; + limit = (window & GENMASK(31, 20)) | GENMASK(19, 0); + if (base > limit) + return -EIO; + for (function = 0; function < 2; function++) { + endpoint = pci_get_slot(root->subordinate, PCI_DEVFN(0, function)); + if (!endpoint) + return -ENODEV; + for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) { + res = &endpoint->resource[bar]; + if (!(res->flags & IORESOURCE_MEM)) + continue; + pcibios_resource_to_bus(endpoint->bus, ®ion, res); + if (!res->parent || !resource_size(res) || + res->flags & (IORESOURCE_UNSET | IORESOURCE_DISABLED | IORESOURCE_PREFETCH) || + region.start < base || region.end > limit) { + pci_err(endpoint, "BAR%d outside programmed root memory window\n", bar); + ret = -ERANGE; + break; + } + } + pci_dev_put(endpoint); + if (ret) + return ret; + } + pci_info(root, "NEO_ROOT_MEMORY_ENABLED command=%#x window=%#x bus=%#llx-%#llx\n", + command, window, base, limit); + return 0; +} + +static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) +{ + struct apple_pcie *pcie = pci_host_bridge_priv(bridge); + struct pci_dev *root, *endpoint; + u16 command; + int function, ret; + + root = pci_get_slot(bridge->bus, PCI_DEVFN(0, 0)); + if (!root) + return -ENODEV; + if (!root->subordinate || root->subordinate->number != 1) { + ret = -EINVAL; + goto out; + } + /* No config/raw spinlock is held across this process-context request. */ + ret = apple_piodma_bootstrap_prime(pcie->piodma_supplier, root); + if (ret) + goto out; + ret = apple_pcie_neo_typed_ids(pcie, root); + if (ret) + goto out; + pcie->neo_secondary_bus = root->subordinate->number; + pci_lock_rescan_remove(); + /* Publish validated bootstrap state and bus number before config access. */ + smp_store_release(&pcie->neo_config_ready, true); + /* Root was already enabled by the closed scan. The bridge-specific API + * programs its new windows even when the generic bus rescan would skip + * pci_setup_bridge(). Publish children only after hardware validation. + */ + pci_scan_child_bus(root->subordinate); + pci_assign_unassigned_bridge_resources(root); + ret = apple_pcie_neo_check_memory(root); + if (ret) { + dev_err(pcie->dev, "root memory forwarding unavailable after assignment: %d\n", ret); + goto close_config; + } + pci_bus_add_devices(root->subordinate); + for (function = 0; function < 2; function++) { + endpoint = pci_get_slot(root->subordinate, PCI_DEVFN(0, function)); + if (!endpoint) { + ret = -ENODEV; + break; + } + if (pci_read_config_word(endpoint, PCI_COMMAND, &command)) { + ret = -EIO; + } else { + dev_info(pcie->dev, "enumerated %s id=%04x:%04x command=%#x driver=%s\n", + pci_name(endpoint), endpoint->vendor, endpoint->device, command, + endpoint->driver ? endpoint->driver->name : "unbound"); + if (command & PCI_COMMAND_MASTER) { + /* This experiment never permits endpoint bus mastering. */ + pci_clear_master(endpoint); + ret = -EIO; + } + if (endpoint->driver) + ret = -EBUSY; + } + pci_dev_put(endpoint); + if (ret) + break; + } +close_config: + if (ret) + WRITE_ONCE(pcie->neo_config_ready, false); + pci_unlock_rescan_remove(); + if (!ret) + dev_info(pcie->dev, "NEO_ENUMERATION_ONLY_READY; functions0/1 unbound, bus mastering off\n"); +out: + pci_dev_put(root); + return ret; +} + static int apple_pcie_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1852,6 +2022,7 @@ static int apple_pcie_probe(struct platform_device *pdev) struct pci_host_bridge *bridge; struct device_node *of_port; struct apple_pcie *pcie; + struct device *piodma_supplier = NULL; int ret; hw = of_device_get_match_data(dev); @@ -1876,6 +2047,12 @@ static int apple_pcie_probe(struct platform_device *pdev) } } + if (hw->root_bus_only && apple_piodma_bootstrap_enabled()) { + ret = apple_piodma_bootstrap_get(dev, &piodma_supplier); + if (ret) + return dev_err_probe(dev, ret, "PIODMA supplier unavailable\n"); + } + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); if (!bridge) return -ENOMEM; @@ -1883,6 +2060,7 @@ static int apple_pcie_probe(struct platform_device *pdev) pcie = pci_host_bridge_priv(bridge); pcie->dev = dev; pcie->hw = hw; + pcie->piodma_supplier = piodma_supplier; if (hw->root_bus_only) dev_info(dev, "root-port probe only; downstream config is blocked\n"); pcie->base = devm_platform_ioremap_resource(pdev, 1); @@ -1917,6 +2095,15 @@ static int apple_pcie_probe(struct platform_device *pdev) if (ret) return ret; + if (pcie->piodma_supplier) { + /* All fallible devres setup precedes publication of DMA pointers. */ + ret = apple_pcie_neo_enumerate(bridge); + if (ret) + dev_err(dev, "enumeration experiment failed=%d; host/supplier retained, no retry\n", + ret); + return 0; + } + if (pcie->hw->tunneled) { pci_walk_bus(bridge->bus, apple_pcie_tunnel_keep_d0, NULL); return apple_pcie_tunnel_add_links(pcie); From 193b9fd7985304df7d8d96a04a9b517c7187dbd2 Mon Sep 17 00:00:00 2001 From: Ace <7362641+Acelogic@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:02:56 -0400 Subject: [PATCH 0031/2680] docs: keep T8140 PCIe bring-up instructions focused Keep setup and lifecycle requirements; leave hardware run results in the PR and local qualification artifacts. No hardware code changes. --- Documentation/PCI/controller/apple-t8140.rst | 33 +++++--------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index e3d2608d729cf..90d4eab8f07f4 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -3,11 +3,9 @@ T8140/J700 experimental radio PCIe bootstrap ========================================== -The J700 path trains port 0, primes native ECAM with one bounded PIODMA -request, checks the completed request and retained arena, and enumerates only -the expected MediaTek Wi-Fi and Bluetooth functions. Typed native ECAM access -is enabled only after this validation. The root bridge memory window covers -the assigned endpoint BARs; DART translates the endpoint DMA streams. +The J700 path trains port 0, validates a bounded PIODMA bootstrap request, +and enables native ECAM for the MediaTek Wi-Fi and Bluetooth functions. +The root bridge forwards their BARs and DART translates endpoint DMA. This is an opt-in bring-up implementation, not general T8140 PCIe support. Build ``CONFIG_PCIE_APPLE_PIODMA_DIAG=y`` and pass @@ -16,28 +14,11 @@ root-only diagnostic path continues to reject downstream configuration access. The option excludes suspend and kexec. Failed or uncertain bootstrap state requires a full external hardware reset; there is no command retry. -The PIODMA arena remains allocated and owned until that reset. A completed -bootstrap read is not a memory-reuse or engine-reset qualification. Platform -removal, teardown, suspend and arbitrary downstream devices are unqualified; -do not remove the controller or its IOMMU while this experiment is active. -Those lifetime contracts remain merge blockers for production support. +The PIODMA arena remains allocated until external reset. Controller removal, +teardown, memory reuse and arbitrary downstream devices are unqualified. +Do not remove the controller or its IOMMU while this experiment is active. +These lifecycle contracts require further work before production support. J700's ``wifi0`` alias and PCI endpoint node allow the public bootloader to pass the unit's own Wi-Fi address through ``local-mac-address``. The zero placeholder in the static device tree is not a usable station address. - -Physical validation, 2026-10-02 ------------------------------- - -The public Aurora 7.1.12 base with this platform change booted J700/T8140 into -KDE through public m1n1 and U-Boot. Both endpoint functions enumerated with -translated DMA. The integrated MT7932 fullmac driver subsequently passed -WPA2, DHCP, gateway ping, DNS and verified HTTPS traffic on 2.4 and 5 GHz. -Bluetooth firmware/HCI setup, discovery and a normal HCI off/on cycle passed -while Wi-Fi remained connected. This is evidence of usable radio transport on -the tested platform, not qualification of the excluded lifecycle operations. - -The tested kernel Image SHA256 is -``e9877f3943c5cf04970a1d1b336ea68096377907802063dbce2082b93d20011a``. -Radio source pins, module identity and detailed outcomes belong with the radio -contribution; no firmware, calibration or unit identity is included here. From 293d4518f5af8b7549001e3500c9e1a7fa6815d7 Mon Sep 17 00:00:00 2001 From: Ace <7362641+Acelogic@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:23:14 -0400 Subject: [PATCH 0032/2680] wifi: add existing J700 MT7932 fullmac implementation Import the 26-file GPL-2.0-only MT7932 driver from the user-selected repository: https://github.com/aurora-silicon/linux-neo-cleanroom-eryk-with-wifi branch wifi/width-qualification-20260912 commit 610cb463c03f9bc68a5d020e7ca443af1fda4856 This commit preserves the selected driver files byte-for-byte, including all existing license notices and comments. Register its Kconfig and Makefile on the public Aurora 7.1.12 base. Device firmware, calibration, country policy, credentials and original capture records are not included. This is an import of an existing implementation, not an independent rewrite. Neo radio research and bring-up credit: Eryk Wieliczko (erykwieliczko), DJ (DjDeveloperr), Ace (Acelogic), and Ryan Murray. Source provenance is retained above. Follow-up commits carry the input, scan and NetworkManager compatibility changes tested on the physical J700. Co-authored-by: DJ <43033058+DjDeveloperr@users.noreply.github.com> Co-authored-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Co-authored-by: Eryk Wieliczko --- drivers/net/wireless/mediatek/Kconfig | 1 + drivers/net/wireless/mediatek/Makefile | 1 + drivers/net/wireless/mediatek/mt7932/Kconfig | 19 + drivers/net/wireless/mediatek/mt7932/Makefile | 3 + .../wireless/mediatek/mt7932/association.h | 209 +++ .../net/wireless/mediatek/mt7932/bandwidth.h | 40 + .../wireless/mediatek/mt7932/calibration.h | 402 +++++ .../mediatek/mt7932/calibration_transport.c | 144 ++ .../wireless/mediatek/mt7932/capabilities.h | 136 ++ .../net/wireless/mediatek/mt7932/channels.h | 17 + drivers/net/wireless/mediatek/mt7932/data.c | 267 ++++ .../net/wireless/mediatek/mt7932/firmware.h | 158 ++ .../net/wireless/mediatek/mt7932/frontend.c | 613 ++++++++ .../net/wireless/mediatek/mt7932/lifecycle.c | 69 + drivers/net/wireless/mediatek/mt7932/link.c | 688 +++++++++ drivers/net/wireless/mediatek/mt7932/mt7932.c | 1294 +++++++++++++++++ drivers/net/wireless/mediatek/mt7932/mt7932.h | 226 +++ drivers/net/wireless/mediatek/mt7932/packet.h | 228 +++ drivers/net/wireless/mediatek/mt7932/power.c | 88 ++ .../net/wireless/mediatek/mt7932/protocol.h | 73 + drivers/net/wireless/mediatek/mt7932/rate.h | 97 ++ .../net/wireless/mediatek/mt7932/recovery.c | 291 ++++ .../net/wireless/mediatek/mt7932/regulatory.h | 103 ++ drivers/net/wireless/mediatek/mt7932/reset.c | 114 ++ drivers/net/wireless/mediatek/mt7932/scan.h | 129 ++ .../net/wireless/mediatek/mt7932/startup.c | 491 +++++++ .../net/wireless/mediatek/mt7932/station.c | 105 ++ .../wireless/mediatek/mt7932/station_query.c | 65 + 28 files changed, 6071 insertions(+) create mode 100644 drivers/net/wireless/mediatek/mt7932/Kconfig create mode 100644 drivers/net/wireless/mediatek/mt7932/Makefile create mode 100644 drivers/net/wireless/mediatek/mt7932/association.h create mode 100644 drivers/net/wireless/mediatek/mt7932/bandwidth.h create mode 100644 drivers/net/wireless/mediatek/mt7932/calibration.h create mode 100644 drivers/net/wireless/mediatek/mt7932/calibration_transport.c create mode 100644 drivers/net/wireless/mediatek/mt7932/capabilities.h create mode 100644 drivers/net/wireless/mediatek/mt7932/channels.h create mode 100644 drivers/net/wireless/mediatek/mt7932/data.c create mode 100644 drivers/net/wireless/mediatek/mt7932/firmware.h create mode 100644 drivers/net/wireless/mediatek/mt7932/frontend.c create mode 100644 drivers/net/wireless/mediatek/mt7932/lifecycle.c create mode 100644 drivers/net/wireless/mediatek/mt7932/link.c create mode 100644 drivers/net/wireless/mediatek/mt7932/mt7932.c create mode 100644 drivers/net/wireless/mediatek/mt7932/mt7932.h create mode 100644 drivers/net/wireless/mediatek/mt7932/packet.h create mode 100644 drivers/net/wireless/mediatek/mt7932/power.c create mode 100644 drivers/net/wireless/mediatek/mt7932/protocol.h create mode 100644 drivers/net/wireless/mediatek/mt7932/rate.h create mode 100644 drivers/net/wireless/mediatek/mt7932/recovery.c create mode 100644 drivers/net/wireless/mediatek/mt7932/regulatory.h create mode 100644 drivers/net/wireless/mediatek/mt7932/reset.c create mode 100644 drivers/net/wireless/mediatek/mt7932/scan.h create mode 100644 drivers/net/wireless/mediatek/mt7932/startup.c create mode 100644 drivers/net/wireless/mediatek/mt7932/station.c create mode 100644 drivers/net/wireless/mediatek/mt7932/station_query.c diff --git a/drivers/net/wireless/mediatek/Kconfig b/drivers/net/wireless/mediatek/Kconfig index bd4db128248e8..9abadb928c0ff 100644 --- a/drivers/net/wireless/mediatek/Kconfig +++ b/drivers/net/wireless/mediatek/Kconfig @@ -13,4 +13,5 @@ config WLAN_VENDOR_MEDIATEK if WLAN_VENDOR_MEDIATEK source "drivers/net/wireless/mediatek/mt7601u/Kconfig" source "drivers/net/wireless/mediatek/mt76/Kconfig" +source "drivers/net/wireless/mediatek/mt7932/Kconfig" endif # WLAN_VENDOR_MEDIATEK diff --git a/drivers/net/wireless/mediatek/Makefile b/drivers/net/wireless/mediatek/Makefile index 806172659bf2e..bfb32cf0f41db 100644 --- a/drivers/net/wireless/mediatek/Makefile +++ b/drivers/net/wireless/mediatek/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_MT7601U) += mt7601u/ obj-$(CONFIG_MT76_CORE) += mt76/ +obj-$(CONFIG_MT7932_FULLMAC) += mt7932/ diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig new file mode 100644 index 0000000000000..d435dbe24c32e --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0-only +config MT7932_FULLMAC + tristate "Experimental J700 MT7932 fullmac wireless support" + depends on PCI && OF && ARM64 && IOMMU_DMA + select FW_LOADER + select CFG80211 + help + Native support for the built-in J700 Wi-Fi endpoint: managed + WPA2-CCMP station operation on 2.4 GHz and channels 36..48 at 20 MHz. + Requires original firmware, own calibration and original-derived + country policy packages in addition to the cfg80211 regulatory database. + Requires a trained retained PCI link and a translated SID1 DMA domain. + Firmware, runtime DMA and calibration negotiation start automatically; + cfg80211 and userspace select the country and network. No test boot + parameters are required for ordinary station operation. + Module removal stops host users and performs a checked PCI function + reset before releasing DMA storage. A failed reset blocks removal, + retaining resources until an explicit retry succeeds or system reset. + Other MediaTek devices and other machines are not matched. diff --git a/drivers/net/wireless/mediatek/mt7932/Makefile b/drivers/net/wireless/mediatek/mt7932/Makefile new file mode 100644 index 0000000000000..2ac830d058ca4 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_MT7932_FULLMAC) += mt7932-fullmac.o +mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o station_query.o diff --git a/drivers/net/wireless/mediatek/mt7932/association.h b/drivers/net/wireless/mediatek/mt7932/association.h new file mode 100644 index 0000000000000..eb207921906e1 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/association.h @@ -0,0 +1,209 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef MT7932_ASSOCIATION_H +#define MT7932_ASSOCIATION_H + +#include "protocol.h" +#include "channels.h" + +struct mt7932_peer_ids { + u8 station[2], wtbl[4]; +}; + +/* Firmware owns association IE generation. iwd always declares BSS transition + * and FILS capability bits even for ordinary OPEN authentication. Accept only + * those declarations (or zero subsets), not a FILS operation or arbitrary IE + * carriage request. The generated capabilities are reported separately. + */ +static inline bool mt7932_open_ies(const u8 *ies, size_t length) +{ + unsigned int i; + + if (!length) + return true; + if (!ies || length != 12 || ies[0] != 127 || ies[1] != 10) + return false; + for (i = 0; i < 10; i++) { + u8 allowed = i == 2 ? 8 : i == 9 ? 1 : 0; + + if (ies[i + 2] & ~allowed) + return false; + } + return true; +} + +/* Bounded, nonsecret subset of firmware-generated request IEs. Unknown/vendor + * contents are not exported. These bytes are actual generated capabilities, + * not an echo of the caller's capability declaration or replacement RF IEs. + */ +static inline int mt7932_assoc_capabilities(u8 *out, size_t capacity, + const u8 *ies, size_t length, bool open) +{ + size_t offset = 0, used = 0; + unsigned int seen = 0; + + while (offset < length) { + u8 id, size; + unsigned int bit = 0, expected = 0; + + if (length - offset < 2) + return -EINVAL; + id = ies[offset]; + size = ies[offset + 1]; + if (size > length - offset - 2) + return -EINVAL; + if (open && (id == 48 || (id == 221 && size >= 4 && + !memcmp(ies + offset + 2, "\x00\x50\xf2\x01", 4)))) + return -EINVAL; + switch (id) { + case 45: bit = 1; expected = 26; break; + case 61: bit = 2; expected = 22; break; + case 191: bit = 4; expected = 12; break; + case 192: bit = 8; expected = 5; break; + case 127: bit = 16; expected = size; break; + case WLAN_EID_EXTENSION: + if (!size) + return -EINVAL; + if (ies[offset + 2] != WLAN_EID_EXT_HE_CAPABILITY) + break; + /* Validate the complete standard HE layout, including optional + * wider MCS maps and PPE thresholds, before reporting the + * firmware's actual request. This does not advertise new local + * PHY modes or alter the native association/rate owner. + */ + if (!ieee80211_he_capa_size_ok(ies + offset + 3, size - 1)) + return -EINVAL; + bit = 32; + expected = size; + break; + } + if (bit) { + if ((seen & bit) || size != expected || + (id == 127 && (!size || size > 16)) || + size + 2U > capacity - used) + return -EINVAL; + memcpy(out + used, ies + offset, size + 2); + used += size + 2; + seen |= bit; + } + offset += size + 2; + } + return used; +} + +/* Original25G83 pure WPA2 binary-key result: structured type6, not a string. + * NEO_TYPE6_BINARY_PMK_NATIVE_CONTRACT: standard nl80211 PMK maps directly; + * no passphrase API, key derivation, duplicate independent PMK or extra flags. + */ +static inline int mt7932_connect_binary_pmk(u8 *body, size_t capacity, + const u8 *pmk, size_t length) +{ + if (capacity < 856 || length != 32) + return -EINVAL; + memset(body + 0x40, 0, 0x90); + put_unaligned_le32(32, body + 0x40); + put_unaligned_le32(6, body + 0x44); + memcpy(body + 0x4c, pmk, 32); + /* Clear length, all64 independent storage bytes and explicit allow-auth. + * Policy0 is the coherent original64-hex-input RESULT, not open security. + */ + memset(body + 0x1c8, 0, 4 + 64 + 4); + return 0; +} + +/* Explicit OPEN or PMK-only WPA2/CCMP; never an automatic security fallback. + * Caller owns identities/resources and validated BSS state. + * NEO_WPA2_FULLMAC_ASSOCIATION_AND_ETHERNET_CONTRACT. Never log this body. + */ +static inline int mt7932_connect_body(u8 *body, size_t capacity, const u8 *ssid, + size_t ssid_length, const u8 pmk[32], const u8 bssid[6], + u8 channel, unsigned int listen_interval, u8 connection_flags, + const struct mt7932_peer_ids *ids, bool open) +{ + u8 resources[6]; + unsigned int i, j; + int ret; + + if ((open ? pmk != NULL : pmk == NULL) || + capacity < 856 || !ssid_length || ssid_length > 32 || + !mt7932_channel_band(channel) || connection_flags > 63 || (bssid[0] & 1)) + return -EINVAL; + memcpy(resources, ids->station, 2); + memcpy(resources + 2, ids->wtbl, 4); + for (i = 0; i < 6; i++) { + /* Global0 and per-interface BMC19 remain separately reserved. */ + if (!resources[i] || resources[i] >= (i < 2 ? 15 : 14)) + return -EINVAL; + for (j = 0; j < i; j++) + if (resources[i] == resources[j]) + return -EINVAL; + } + memset(body, 0, 856); + /* NORMAL25G83 original strict keyless OPEN branch, without optional + * automatic-authentication policy. Keep this coherent version/auth/ + * policy tuple together; no dummy keys or stale security state. + * NEO_NORMAL_STRICT_OPEN_PRODUCER_CONTRACT. + */ + if (open) { + body[1] = 1; + put_unaligned_le32(0x78, body + 8); + put_unaligned_le16(1, body + 0x12); + } else { + body[1] = 4; + put_unaligned_le32(0x10, body + 4); + put_unaligned_le32(0x78, body + 8); + put_unaligned_le16(1, body + 0x12); + put_unaligned_le16(8, body + 0x14); + } + put_unaligned_le32(ssid_length, body + 0x18); + memcpy(body + 0x1c, ssid, ssid_length); + /* WCLJoinRequest projects six flags here, NOT an environmental count. */ + body[0x1c0] = connection_flags; + /* Fresh HOST AIS media state1 selects CCMP type5 here. This is not + * structured-key type0x44 or the association event's RSN suite encoding. + * This builder is only used for the first, unassociated attempt. + */ + put_unaligned_le32(open ? 0 : 5, body + 0x220); + put_unaligned_le16(listen_interval > 20 ? 20 : listen_interval, body + 0x228); + memcpy(body + 0x32c, bssid, 6); + body[0x338] = mt7932_channel_band(channel); + body[0x339] = channel; + body[0x33c] = 1; + body[0x33d] = 2; + body[0x33e] = 4; + memcpy(body + 0x340, ids->station, 2); + memcpy(body + 0x348, ids->wtbl, 4); + body[0x354] = 1; + ret = open ? 0 : mt7932_connect_binary_pmk(body, capacity, pmk, 32); + return ret ? ret : 856; +} + +static inline void mt7932_peer_queue(u8 body[32], u8 station, bool bind) +{ + static const u8 tids[] = {1,0,0,1,2,2,3,3}; + + memset(body, 0, 32); + put_unaligned_le16(15, body); + body[2] = 5; + body[14] = 1; + body[15] = station; + put_unaligned_le16(bind ? 15 : 2, body + 16); + memcpy(body + 20, tids, sizeof(tids)); +} + +static inline bool mt7932_assoc_resources(const u8 *body, size_t length, + const struct mt7932_peer_ids *ids, u8 channel, bool open) +{ + unsigned int i; + + if (length < 52 || get_unaligned_le16(body) || body[16] || + body[17] != ids->wtbl[0] || body[20] != ids->station[0] || + !mt7932_channel_band(channel) || body[11] != mt7932_channel_band(channel) || body[12] != channel || + (!open && get_unaligned_le32(body + 48) != 0x000fac04)) + return false; + for (i = 0; i < 4; i++) + if (body[21] == ids->wtbl[i]) + return true; + return false; +} + +#endif diff --git a/drivers/net/wireless/mediatek/mt7932/bandwidth.h b/drivers/net/wireless/mediatek/mt7932/bandwidth.h new file mode 100644 index 0000000000000..c5efe6599e0a4 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/bandwidth.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef MT7932_BANDWIDTH_H +#define MT7932_BANDWIDTH_H + +#include "channels.h" + +/* One startup ceiling: 20/40/80 are 0/1/2 here, not CID0f's encoding. */ +#define MT7932_MAX_5G_BW 2 + +static inline u8 mt7932_domain_5g_bw(void) +{ + return MT7932_MAX_5G_BW == 0 ? 1 : MT7932_MAX_5G_BW == 1 ? 0 : 2; +} + +/* EE4d width 0 combines 20/40; primary and extension select the center. + * Keep all operational primaries inside 36..48, and 2 GHz at 20 MHz. + */ +static inline bool mt7932_channel_layout(u8 primary, u8 width, u8 extension, + u8 maximum, u8 *center, u8 *bw) +{ + u8 band = mt7932_channel_band(primary); + u8 expected = primary == 36 || primary == 44 ? 1 : 3; + + if (!band || maximum > 2 || width > 1) + return false; + if (!width && !extension) { + *center = primary; + *bw = 0; + return true; + } + if (band != 2 || extension != expected) + return false; + *bw = width ? 2 : 1; + if (*bw > maximum) + return false; + *center = width ? 42 : extension == 1 ? primary + 2 : primary - 2; + return true; +} + +#endif diff --git a/drivers/net/wireless/mediatek/mt7932/calibration.h b/drivers/net/wireless/mediatek/mt7932/calibration.h new file mode 100644 index 0000000000000..f008e34146d64 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/calibration.h @@ -0,0 +1,402 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef MT7932_CALIBRATION_H +#define MT7932_CALIBRATION_H + +#include "protocol.h" + +/* Protected by the RX/procedure lock. One active logical procedure only; + * ordinary D6 nulls carry no source-defined transaction/sequence identifier. + */ +struct mt7932_cal_completion { + unsigned int expected, received; + bool active, done; + int error; +}; + +static inline int mt7932_cal_begin(struct mt7932_cal_completion *state, unsigned int expected) +{ + if (state->error) + return state->error; + if (state->active || !expected || expected > 4) + return -EINVAL; + state->expected = expected; + state->received = 0; + state->done = false; + state->active = true; + return 0; +} + +/* Action1 SET only. Caller validates outer/event bounds and dispatches EIDd6 + * independently of pending-query sequence matching. Latch errors until reboot. + */ +static inline int mt7932_cal_null(struct mt7932_cal_completion *state, + const u8 *payload, size_t length, u8 version) +{ + u32 status; + int ret; + + if (state->error) + return state->error; + if (length < 20) { + ret = -EMSGSIZE; + goto fail; + } + if (payload[2] || payload[3] != 1) { + ret = -EPROTO; + goto fail; + } + if (!state->active || !state->expected) { + ret = -EPROTO; + goto fail; + } + status = get_unaligned_le32(payload + 8); + if ((version && status) || (!version && !status)) { + ret = -EREMOTEIO; + goto fail; + } + if (++state->received == state->expected) { + state->active = false; + state->done = true; + } + return state->done; +fail: + state->error = ret; + state->active = false; + return ret; +} + +struct mt7932_cal_segment { + const u8 *data; + size_t length; +}; + +/* Validate the entire original container before selecting any data to send. + * NEO_MT7932_STOCK_ARTIFACTS_AND_CAL_CONVERSION: raw BLOB, not a transcript. + */ +static inline int mt7932_cal_validate(const u8 *data, size_t size) +{ + u32 end, count, i, j; + + if (size < 16 || size > 16 * 1024 * 1024 || memcmp(data, "BLOB", 4) || get_unaligned_be16(data + 8) != 12) + return -EINVAL; + end = get_unaligned_be32(data + 4); + count = get_unaligned_be16(data + 10); + if (!count || end != 16 + 20 * count || end > size) + return -EINVAL; + for (i = 0; i < count; i++) { + const u8 *entry = data + 16 + 20 * i; + u32 offset = get_unaligned_be32(entry + 4); + u32 length = get_unaligned_be32(entry + 8), sum = 0; + + if (get_unaligned_be16(entry + 2) != 12 || offset < end || + offset > size || length < 12 || length > size - offset) + return -EINVAL; + if (memcmp(data + offset, entry, 4) || + get_unaligned_be32(data + offset + 4) != length) + return -EINVAL; + for (j = 0; j < length; j++) + sum += data[offset + j]; + if (sum != get_unaligned_be32(entry + 12)) + return -EINVAL; + for (j = 0; j < i; j++) { + const u8 *prior = data + 16 + 20 * j; + u32 start = get_unaligned_be32(prior + 4); + u32 bytes = get_unaligned_be32(prior + 8); + + if (!memcmp(entry, prior, 2) || (offset < start + bytes && start < offset + length)) + return -EINVAL; + } + } + return 0; +} + +/* Only call after full validation; borrowed storage remains owned by caller. */ +static inline int mt7932_cal_segment(const u8 *data, unsigned int tag, + struct mt7932_cal_segment *segment) +{ + unsigned int i, count = get_unaligned_be16(data + 10); + + for (i = 0; i < count; i++) { + const u8 *entry = data + 16 + 20 * i; + + if (get_unaligned_be16(entry) == tag) { + segment->data = data + get_unaligned_be32(entry + 4) + 8; + segment->length = get_unaligned_be32(entry + 8) - 8; + return 0; + } + } + return -ENOENT; +} + +struct mt7932_cal_piece { + const u8 *data; + u32 length, parameter; + u8 type, fragment; + u8 metadata[8]; +}; + +/* Type1 group0/1, taking the validated operating center, not always primary. + */ +static inline int mt7932_cal_association(const u8 *data, size_t size, u8 channel, + struct mt7932_cal_piece pieces[2]) +{ + struct mt7932_cal_segment segment; + unsigned int group, part; + int ret; + + if (channel >= 1 && channel <= 13) + group = 0; + else if (channel >= 36 && channel <= 48 && !(channel % 2)) + group = 1; + else + return -EOPNOTSUPP; + ret = mt7932_cal_validate(data, size); + if (ret) + return ret; + ret = mt7932_cal_segment(data, 0x2000 | group, &segment); + if (ret) + return ret; + if (!segment.length || segment.length % 2 || segment.length / 2 > 1080) + return -EINVAL; + for (part = 0; part < 2; part++) + pieces[part] = (struct mt7932_cal_piece) { + .data = segment.data + part * (segment.length / 2), + .length = segment.length / 2, + .parameter = 0x04000000 | (channel << 17) | group, + .type = 1, .fragment = 0x20 | part, + }; + return 2; +} + +/* Plan all conditional power-on fragments before the first command is sent. */ +static inline int mt7932_cal_power_on(const u8 *data, size_t size, bool six_ghz, + struct mt7932_cal_piece pieces[14]) +{ + static const unsigned int tags[] = {0x1001, 0x1002, 0x1003, 0x1011, 0x1012, 0x1021, 0x1022}; + static const u8 groups[] = {0, 0, 0, 1, 1, 2, 2}; + static const u8 fragments[] = {0x30, 0x31, 0x32, 0x20, 0x21, 0x20, 0x21}; + static const u32 params[] = {0, 0x10000, 8, 0x10}; + struct mt7932_cal_segment segment; + unsigned int i, part, count, at = 0; + int ret = mt7932_cal_validate(data, size); + + if (ret) + return ret; + for (i = 0; i < (six_ghz ? 7 : 5); i++) { + ret = mt7932_cal_segment(data, tags[i], &segment); + if (ret) + return ret; + if (!segment.length || segment.length > 1080) + return -EINVAL; + pieces[at++] = (struct mt7932_cal_piece) { + .data = segment.data, .length = segment.length, + .parameter = groups[i], .type = 0, .fragment = fragments[i], + }; + } + for (i = 0; i < (six_ghz ? 4 : 3); i++) { + u32 param = params[i]; + unsigned int tag = 0x2000 | (param & 0x7f) | ((param & 0x10000) ? 0x80 : 0); + + ret = mt7932_cal_segment(data, tag, &segment); + if (ret) + return ret; + count = tag & 0x80 ? 1 : 2; + if (!segment.length || segment.length % count || segment.length / count > 1080) + return -EINVAL; + for (part = 0; part < count; part++) + pieces[at++] = (struct mt7932_cal_piece) { + .data = segment.data + part * (segment.length / count), + .length = segment.length / count, .parameter = param, + .type = 1, .fragment = count << 4 | part, + }; + } + return at; +} + +/* Bounded original-own-data5GHz D7 path. Plan all dependencies before send; + * special upper0x1001 needs absent tag2081 and must not send a partial plan. + */ +static inline int mt7932_cal_requested_5g(const u8 *data, size_t size, + u32 upper, u32 channel, u8 version, + u8 module, struct mt7932_cal_piece pieces[7]) +{ + struct mt7932_cal_segment segment; + unsigned int i, part, at = 0, selected; + const u8 types[] = {1, 3, 2}; + u32 params[3]; + unsigned int tags[3]; + int ret; + + if (upper != 0x1000 || version != 12 || module != 0x89 || + channel < 36 || channel > 48 || channel % 2) + return -EOPNOTSUPP; + ret = mt7932_cal_validate(data, size); + if (ret) + return ret; + ret = mt7932_cal_segment(data, 0x501, &segment); + if (ret) + return ret; + if (segment.length != 200) + return -EINVAL; + /* No alternate override/disable bitmap is qualified for this own OCA2. */ + for (i = 0; i < segment.length; i++) + if (segment.data[i]) + return -EOPNOTSUPP; + /* Own row610f07 does not remap center selectors1/2; only primaries. */ + selected = channel % 4 ? channel : channel <= 40 ? 38 : 46; + params[0] = 0x10000001; + params[1] = 0x10001000 | channel; + params[2] = 0x10001000 | (channel << 17) | selected; + tags[0] = 0x2001; + tags[1] = 0x4100 | channel; + tags[2] = 0x3100 | selected; + for (i = 0; i < 3; i++) { + unsigned int count = i == 1 ? 1 : 2; + unsigned int stride = i == 0 ? 1080 : i == 1 ? 20 : 612; + unsigned int meta = i == 0 ? 0 : i == 1 ? 4 : 8; + + ret = mt7932_cal_segment(data, tags[i], &segment); + if (ret) + return ret; + if (segment.length != count * stride) + return -EINVAL; + for (part = 0; part < count; part++) { + const u8 *record = segment.data + part * stride; + + pieces[at] = (struct mt7932_cal_piece) { + .data = record + meta, .length = stride - meta, + .parameter = params[i], .type = types[i], + .fragment = count << 4 | part, + }; + memcpy(pieces[at++].metadata, record, meta); + } + } + return at; +} + +/* A genuine D7 event, not a fabricated channel-based startup trigger. + * Initial PL profile: reject unqualified upper bits and all non-2G requests. + */ +static inline int mt7932_cal_requested(const u8 *data, size_t size, + const u8 *event, size_t length, u8 version, + u8 module, struct mt7932_cal_piece pieces[7]) +{ + static const u8 rules[13][3] = { + {0,15,3}, {0,15,3}, {0x31,7,3}, {0x21,7,3}, + {0}, {0}, {0}, {0}, {0}, {0}, {0}, {0x61,15,7}, {0x61,15,7}, + }; + static const u8 types[] = {1, 3, 2, 2}; + struct mt7932_cal_segment segment; + u32 upper, band, channel, selected, p, action, q, params[4]; + u8 rule[3], disable[2] = {}; + unsigned int width, at = 0, i, part; + int ret; + + if (length != 16 || version > 12) + return -EINVAL; + upper = get_unaligned_le32(event + 4); + band = get_unaligned_le32(event + 8); + channel = get_unaligned_le32(event + 12); + if (band == 1) + return mt7932_cal_requested_5g(data, size, upper, channel, version, module, pieces); + if (band || !channel || channel > 13 || (upper & ~0x1001U)) + return -EOPNOTSUPP; + ret = mt7932_cal_validate(data, size); + if (ret) + return ret; + ret = mt7932_cal_segment(data, 0x501, &segment); + if (ret) + return ret; + if (segment.length != 200) + return -EINVAL; + memcpy(rule, rules[version], 3); + /* Validated file version12: versions2..10 suppress its rule override. */ + if (segment.data[1] && (version < 2 || version >= 11)) { + memcpy(rule, segment.data, 3); + memcpy(disable, segment.data + 3, 2); + } + width = channel == 3 || channel == 8 || channel == 12; + if (!(rule[2] & 1) || !(rule[1] & (1U << width))) + return -EOPNOTSUPP; + selected = channel; + if ((rule[0] & 16) && (rule[0] & (1U << width)) && + !(disable[(channel - 1) / 8] & (1U << ((channel - 1) % 8)))) + selected = channel <= 5 ? 3 : channel <= 10 ? 8 : 12; + p = upper << 16 | channel; + action = p & 0x0e000000; + q = (action >> 25) == 3 ? p : p & 0xf1ffffff; + params[0] = (q & 0x7e000000) | (p & 0x10000); + params[1] = q & 0x7e00ffff; + params[2] = (q & 0x7fff0fff) ^ 0x10000; + params[3] = q | action; + for (i = 0; i < 4; i++) { + u32 param = params[i]; + unsigned int tag, count, stride, meta, raw_count; + + if (types[i] == 1) { + tag = 0x2000 | ((param & 0x10000) ? 0x80 : 0); + count = tag & 0x80 ? 1 : 2; + meta = 0; + } else if (types[i] == 3) { + tag = 0x4000 | channel; + count = 1; + meta = 4; + } else { + param = ((param | (channel << 17)) & ~0xfffU) | selected; + tag = 0x3000 | selected | ((param & 0x10000) ? 0x80 : 0); + raw_count = module & 0x80 ? ((module & 15) == 1 ? 4 : 8) : 2; + if (tag & 0x80) + raw_count /= 2; + count = module & 15 ? (tag & 0x80 ? 1 : 3) : raw_count; + meta = 8; + } + ret = mt7932_cal_segment(data, tag, &segment); + if (ret) + return ret; + if (types[i] == 1) { + if (!segment.length || segment.length % count) + return -EINVAL; + stride = segment.length / count; + } else if (types[i] == 3) { + if (segment.length != 20) + return -EINVAL; + stride = 20; + } else { + if (segment.length != raw_count * 612 || count > raw_count) + return -EINVAL; + stride = 612; + } + if (stride <= meta || stride - meta > 1080 || count > 7 - at) + return -EINVAL; + for (part = 0; part < count; part++) { + const u8 *record = segment.data + part * stride; + + pieces[at] = (struct mt7932_cal_piece) { + .data = record + meta, .length = stride - meta, + .parameter = param, .type = types[i], .fragment = count << 4 | part, + }; + memcpy(pieces[at].metadata, record, meta); + at++; + } + } + return at; +} + +static inline int mt7932_cal_body(u8 *body, size_t capacity, + const struct mt7932_cal_piece *piece, u8 preload_version) +{ + if (capacity < 20 || !piece->length || piece->length > 1080 || piece->length > capacity - 20) + return -EINVAL; + memset(body, 0, 20); + body[0] = piece->type; + body[1] = preload_version ? 0xff : 0; + body[2] = piece->fragment; + body[3] = 1; + put_unaligned_le32(piece->length, body + 4); + put_unaligned_le32(piece->parameter, body + 8); + memcpy(body + 12, piece->metadata, 8); + memcpy(body + 20, piece->data, piece->length); + return 20 + piece->length; +} + +#endif diff --git a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c new file mode 100644 index 0000000000000..7c2dcae0456a6 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include "mt7932.h" + + + +/* Called only by the serialized command owner, never from the RX consumer. */ +int mt_cal_procedure(struct mt7932 *m, struct mt7932_cal_piece *pieces, + unsigned int count, unsigned int logical, u8 context_version) +{ + u8 body[1100]; + unsigned long flags; + unsigned long deadline = jiffies + msecs_to_jiffies(5000); + unsigned int i; + int ret; + + reinit_completion(&m->cal_response); + spin_lock_irqsave(&m->response_lock, flags); + ret = mt7932_cal_begin(&m->cal_state, logical); + spin_unlock_irqrestore(&m->response_lock, flags); + if (ret) + return ret; + for (i = 0; i < count; i++) { + ret = mt7932_cal_body(body, sizeof(body), pieces + i, context_version); + if (ret < 0) + goto fail; + ret = mt_request(m, 0xd6, true, true, false, body, ret); + if (ret) + goto fail; + } + if (time_after_eq(jiffies, deadline) || + !wait_for_completion_timeout(&m->cal_response, deadline - jiffies)) { + ret = -ETIMEDOUT; + goto fail; + } + spin_lock_irqsave(&m->response_lock, flags); + ret = m->cal_state.error ?: (m->cal_state.done ? 0 : -EPROTO); + spin_unlock_irqrestore(&m->response_lock, flags); + if (!ret) + return 0; +fail: + spin_lock_irqsave(&m->response_lock, flags); + m->cal_state.error = ret; + m->cal_state.active = false; + spin_unlock_irqrestore(&m->response_lock, flags); + dev_err(&m->pdev->dev, "CAL_PROCEDURE_FAILED: %d; epoch latched, no replay\n", ret); + return ret; +} + +int mt_calibration_gate(struct mt7932 *m) +{ + const struct firmware *wcal, *oca; + struct mt7932_cal_piece pieces[14]; + u8 body[1028], request[16]; + unsigned long flags; + unsigned int at, count, group; + int ret, planned; + + if (!m->six_ghz_valid || m->smart_version != 12 || !m->preload_version) + return -EOPNOTSUPP; + ret = request_firmware_direct(&wcal, "mediatek/mt7932/wcal.bin", &m->pdev->dev); + if (ret) + return ret; + ret = request_firmware_direct(&oca, "mediatek/mt7932/oca2.bin", &m->pdev->dev); + if (ret) + goto out_wcal; + ret = -EINVAL; + if (!wcal->size || wcal->size > 1024) + goto out; + planned = mt7932_cal_power_on(oca->data, oca->size, m->six_ghz, pieces); + if (planned < 0) { + ret = planned; + goto out; + } + /* Validate every potential PL request locally before changing firmware state. */ + memset(request, 0, sizeof(request)); + put_unaligned_le32(0x1000, request + 4); + for (at = 1; at <= 13; at++) { + struct mt7932_cal_piece check[7]; + + put_unaligned_le32(at, request + 12); + ret = mt7932_cal_requested(oca->data, oca->size, request, sizeof(request), + m->smart_version, m->module_byte, check); + if (ret < 0) + goto out; + } + body[0] = 3; + body[1] = 0; + put_unaligned_le16(wcal->size, body + 2); + memcpy(body + 4, wcal->data, wcal->size); + ret = mt_request_ext(m, 0xed, 0x21, true, true, true, body, 4 + wcal->size); + memzero_explicit(body, sizeof(body)); + if (ret) + goto out; + if (m->reply_length < 44 || m->reply[28] != 0xed || m->reply[32] || + get_unaligned_le32(m->reply + 40)) { + dev_err(&m->pdev->dev, "WCAL_REPLY_REJECTED: length=%zu eid=%02x ext=%02x\n", + m->reply_length, m->reply[28], m->reply[32]); + print_hex_dump(KERN_INFO, "WCAL_RESULT: ", DUMP_PREFIX_OFFSET, 16, 1, + m->reply, min_t(size_t, m->reply_length, 64), false); + ret = -EPROTO; + goto out; + } + dev_info(&m->pdev->dev, "OWN_WCAL_ACCEPTED: %zu bytes\n", wcal->size); + for (at = 0, group = 0; at < planned; at += count, group++) { + count = pieces[at].fragment >> 4; + if (!count || count > planned - at) { + ret = -EINVAL; + goto out; + } + ret = mt_cal_procedure(m, pieces + at, count, 1, m->preload_version); + if (ret) + goto out; + dev_info(&m->pdev->dev, "POWER_ON_CAL_GROUP_OK: %u type=%u parameter=%08x fragments=%u\n", + group, pieces[at].type, pieces[at].parameter, count); + } + dev_info(&m->pdev->dev, "POWER_ON_CAL_COMPLETE: %u groups, %d fragments\n", group, planned); + /* Drain already queued genuine requests without ever manufacturing one. */ + for (at = 0; at < ARRAY_SIZE(m->cal_requests); at++) { + spin_lock_irqsave(&m->response_lock, flags); + count = m->cal_request_count; + if (count) { + memcpy(request, m->cal_requests[m->cal_request_head], 16); + m->cal_request_head = (m->cal_request_head + 1) % ARRAY_SIZE(m->cal_requests); + m->cal_request_count--; + } + spin_unlock_irqrestore(&m->response_lock, flags); + if (!count) + break; + ret = mt7932_cal_requested(oca->data, oca->size, request, sizeof(request), + m->smart_version, m->module_byte, pieces); + if (ret < 0) + goto out; + ret = mt_cal_procedure(m, pieces, ret, 4, m->preload_version); + if (ret) + goto out; + dev_info(&m->pdev->dev, "CAL_REQUEST_SERVICED: channel=%u\n", get_unaligned_le32(request + 12)); + } + ret = 0; +out: + release_firmware(oca); +out_wcal: + release_firmware(wcal); + return ret; +} diff --git a/drivers/net/wireless/mediatek/mt7932/capabilities.h b/drivers/net/wireless/mediatek/mt7932/capabilities.h new file mode 100644 index 0000000000000..54ad6e72fe895 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/capabilities.h @@ -0,0 +1,136 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef MT7932_CAPABILITIES_H +#define MT7932_CAPABILITIES_H +#include "bandwidth.h" + +/* Stock25G83 native IE producers, qualified with J700's two-stream PHY + * response. SGI, STBC and beamforming remain firmware-owned. A different + * profile needs its own accounting, not guessed PHY-TLV fields. + */ +static inline bool mt7932_phy_profile(const u8 phy[12]) +{ + static const u8 qualified[] = {1,1,1,3,2,1,1,1,1,1,15,1}; + + return !memcmp(phy, qualified, sizeof(qualified)); +} + +static inline void mt7932_band_capabilities(struct ieee80211_supported_band *band, + bool band5) +{ + struct ieee80211_sta_ht_cap *ht = &band->ht_cap; + struct ieee80211_sta_vht_cap *vht = &band->vht_cap; + u16 map = 0xffff; + unsigned int i; + + memset(ht, 0, sizeof(*ht)); + memset(vht, 0, sizeof(*vht)); + ht->ht_supported = true; + ht->cap = IEEE80211_HT_CAP_LDPC_CODING | IEEE80211_HT_CAP_SM_PS | + IEEE80211_HT_CAP_SGI_20 | IEEE80211_HT_CAP_TX_STBC | + (1 << IEEE80211_HT_CAP_RX_STBC_SHIFT) | IEEE80211_HT_CAP_MAX_AMSDU; + if (!band5) + ht->cap |= IEEE80211_HT_CAP_40MHZ_INTOLERANT; + ht->ampdu_factor = IEEE80211_HT_MAX_AMPDU_64K; + ht->ampdu_density = 0; + ht->mcs.rx_mask[0] = ht->mcs.rx_mask[1] = 0xff; + ht->mcs.tx_params = IEEE80211_HT_MCS_TX_DEFINED; + if (!band5) + return; + /* NORMAL initializes the shared HT word with DSSS/CCK40 even on + * 5 GHz, clearing it together with width/SGI40 for a narrow peer. + * Account for that generated bit; legacy 5 GHz rates stay OFDM-only. + */ + if (MT7932_MAX_5G_BW) + ht->cap |= IEEE80211_HT_CAP_SUP_WIDTH_20_40 | IEEE80211_HT_CAP_SGI_40 | + IEEE80211_HT_CAP_DSSSCCK40; + vht->vht_supported = true; + vht->cap = IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454 | + IEEE80211_VHT_CAP_RXLDPC | IEEE80211_VHT_CAP_TXSTBC | + IEEE80211_VHT_CAP_RXSTBC_1 | IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE | + IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE | + (3 << IEEE80211_VHT_CAP_BEAMFORMEE_STS_SHIFT) | + (7 << IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_SHIFT); + for (i = 0; i < 2; i++) { + map &= ~(3 << (2 * i)); + map |= IEEE80211_VHT_MCS_SUPPORT_0_9 << (2 * i); + } + vht->vht_mcs.rx_mcs_map = cpu_to_le16(map); + vht->vht_mcs.tx_mcs_map = cpu_to_le16(map); + /* Generated EE5e has this standard capability bit, not a nonzero + * highest-rate value. EXT_NSS_BW bits in cap remain0; no extra widths. + */ + vht->vht_mcs.tx_highest = cpu_to_le16(IEEE80211_VHT_EXT_NSS_BW_CAPABLE); + if (MT7932_MAX_5G_BW >= 2) + vht->cap |= IEEE80211_VHT_CAP_SHORT_GI_80; + /* VHT width bits0 do not mean20MHz: channel flags, domain policy and + * negotiated-width checks remain mandatory. Aggregate maxima are radio + * limits, not host DMA record sizes; never loosen RX bounds to match. + */ +} + +/* Inspect unmodified generated request IEs. Absence permits legacy peers; + * presence must match the registered profile. The metadata exporter separately + * bounds and rejects duplicate capability IEs; repeat these checks here so + * this predicate is safe on any bounded IE list. + */ +static inline bool mt7932_assoc_phy_valid(const struct ieee80211_supported_band *band, + const u8 *ies, size_t length) +{ + size_t offset = 0; + bool ht_seen = false, vht_seen = false; + + while (offset < length) { + const u8 *body; + u8 id, size; + + if (length - offset < 2) + return false; + id = ies[offset]; + size = ies[offset + 1]; + if (size > length - offset - 2) + return false; + body = ies + offset + 2; + if (id == WLAN_EID_HT_CAPABILITY) { + const struct ieee80211_sta_ht_cap *ht = &band->ht_cap; + u16 expected = ht->cap; + + if (ht_seen || size != 26 || !ht->ht_supported) + return false; + /* Native generated IEs may narrow to a20MHz peer. This is + * not permission to add a capability absent from the wiphy. + */ + if (!(get_unaligned_le16(body) & IEEE80211_HT_CAP_SUP_WIDTH_20_40)) + expected &= ~(IEEE80211_HT_CAP_SUP_WIDTH_20_40 | IEEE80211_HT_CAP_SGI_40 | + IEEE80211_HT_CAP_DSSSCCK40); + if (get_unaligned_le16(body) != expected || + body[2] != (ht->ampdu_factor | (ht->ampdu_density << 2)) || + memcmp(body + 3, &ht->mcs, sizeof(ht->mcs))) + return false; + ht_seen = true; + } else if (id == WLAN_EID_VHT_CAPABILITY) { + const struct ieee80211_sta_vht_cap *vht = &band->vht_cap; + u32 cap, sts; + + if (vht_seen || size != 12 || !vht->vht_supported) + return false; + cap = get_unaligned_le32(body); + sts = (cap & IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK) >> + IEEE80211_VHT_CAP_BEAMFORMEE_STS_SHIFT; + /* Native peer-specific reduction3->2 is qualified. It is not + * a payload NSS count or permission for arbitrary reductions. + */ + if ((sts != 2 && sts != 3) || + ((cap ^ vht->cap) & ~(IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK | + IEEE80211_VHT_CAP_SHORT_GI_80)) || + ((cap & IEEE80211_VHT_CAP_SHORT_GI_80) && + !(vht->cap & IEEE80211_VHT_CAP_SHORT_GI_80)) || + memcmp(body + 4, &vht->vht_mcs, sizeof(vht->vht_mcs))) + return false; + vht_seen = true; + } + offset += size + 2; + } + return !vht_seen || ht_seen; +} + +#endif diff --git a/drivers/net/wireless/mediatek/mt7932/channels.h b/drivers/net/wireless/mediatek/mt7932/channels.h new file mode 100644 index 0000000000000..8c4b61e4a3ec7 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/channels.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef MT7932_CHANNELS_H +#define MT7932_CHANNELS_H + +/* Bounded operational whitelist, not the full calibration channel namespace. + * Firmware RF bands1/2 are distinct from single-interface MAC index0. + */ +static inline unsigned int mt7932_channel_band(unsigned int channel) +{ + if (channel >= 1 && channel <= 13) + return 1; + if (channel >= 36 && channel <= 48 && !(channel % 4)) + return 2; + return 0; +} + +#endif diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c new file mode 100644 index 0000000000000..25a7bcbf31238 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -0,0 +1,267 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include "mt7932.h" + +static bool mt_bss_tx_allowed(struct mt7932 *m); +static void mt_bss_tx_account(struct mt7932 *m); + +/* data_lock held: quota is packets admitted while absent, not TX-free credits. + * A stopped netdev leaves unsubmitted packets in the ordinary bounded qdisc. + */ +static bool mt_bss_tx_allowed(struct mt7932 *m) +{ + return !m->bss_absent || m->bss_quota; +} + +static void mt_bss_tx_account(struct mt7932 *m) +{ + if (m->bss_absent) + m->bss_quota--; +} + +void mt_data_clean_locked(struct mt7932 *m) +{ + struct mt7932_ring *q = &m->tx[0]; + + while (q->queued && (le32_to_cpu(READ_ONCE(q->desc[q->tail].control)) & DONE)) { + dma_rmb(); + m->data_retired[q->tail] = true; + if (!m->data_token[q->tail]) + m->data_live[q->tail] = false; + q->tail = (q->tail + 1) % q->count; + q->queued--; + } + if (READ_ONCE(m->connected) && mt_rf_allowed(m) && mt_bss_tx_allowed(m) && + !m->data_live[q->head] && q->queued < q->count - 1) + netif_wake_queue(m->netdev); +} + +void mt_data_clean(struct mt7932 *m) +{ + unsigned long flags; + + if (!READ_ONCE(m->data_ready)) + return; + spin_lock_irqsave(&m->data_lock, flags); + mt_data_clean_locked(m); + spin_unlock_irqrestore(&m->data_lock, flags); +} + +/* response_lock held. EID11 replaces the BSS0 absence/quota state. It is not + * a scan completion or a DMA fence; neither carrier nor tokens change here. + */ +void mt_bss_presence(struct mt7932 *m, const struct mt7932_event *event) +{ + const u8 *body = event->packet + 36; + unsigned long flags; + + if (event->length < 40 || body[0] || !READ_ONCE(m->bss_active)) + return; + spin_lock_irqsave(&m->data_lock, flags); + m->bss_absent = !!body[1]; + m->bss_quota = body[2]; + if (!mt_bss_tx_allowed(m)) + netif_stop_queue(m->netdev); + else if (READ_ONCE(m->data_ready)) + mt_data_clean_locked(m); + spin_unlock_irqrestore(&m->data_lock, flags); + dev_dbg_ratelimited(&m->pdev->dev, "BSS_PRESENCE: absent=%u quota=%u\n", + !!body[1], body[2]); +} + +void mt_data_complete(struct mt7932 *m, const u8 *packet, size_t length) +{ + u16 tokens[512]; + unsigned long flags; + int count, i; + + if (!READ_ONCE(m->data_ready)) + return; + count = mt7932_tx_free(packet, length, tokens, ARRAY_SIZE(tokens)); + if (count < 0) { + dev_warn_ratelimited(&m->pdev->dev, "TX_FREE_INVALID: %d length=%zu\n", count, length); + return; + } + spin_lock_irqsave(&m->data_lock, flags); + mt_data_clean_locked(m); + for (i = 0; i < count; i++) { + unsigned int token = tokens[i]; + + if (token >= ARRAY_SIZE(m->data_live) || !m->data_live[token] || !m->data_token[token]) + continue; + m->data_token[token] = false; + m->tx_released++; + if (m->data_retired[token]) + m->data_live[token] = false; + } + mt_data_clean_locked(m); + spin_unlock_irqrestore(&m->data_lock, flags); + dev_dbg_ratelimited(&m->pdev->dev, "TX_FREE: count=%d total=%llu\n", count, m->tx_released); +} + +netdev_tx_t mt_net_xmit(struct sk_buff *skb, struct net_device *netdev) +{ + struct mt7932 *m = *(struct mt7932 **)netdev_priv(netdev); + struct mt7932_ring *q = &m->tx[0]; + struct mt7932_desc *desc; + u8 *payload, *header; + dma_addr_t dma; + unsigned long flags; + unsigned int slot; + int ret; + + if (!READ_ONCE(m->connected) || !mt_rf_allowed(m) || !READ_ONCE(m->data_ready) || + !READ_ONCE(m->peer_valid) || skb->len < ETH_HLEN || skb->len > ETH_FRAME_LEN + VLAN_HLEN) + goto drop; + spin_lock_irqsave(&m->data_lock, flags); + /* Disconnect closes admission before synchronizing this publisher. */ + if (!READ_ONCE(m->connected) || READ_ONCE(m->disconnecting) || !mt_rf_allowed(m)) { + spin_unlock_irqrestore(&m->data_lock, flags); + goto drop; + } + mt_data_clean_locked(m); + slot = q->head; + if (!mt_bss_tx_allowed(m) || m->data_live[slot] || q->queued >= q->count - 1) { + netif_stop_queue(netdev); + spin_unlock_irqrestore(&m->data_lock, flags); + return NETDEV_TX_BUSY; + } + payload = m->data_payloads + 4096 * slot; + header = m->data_headers + 64 * slot; + ret = skb_copy_bits(skb, 0, payload, skb->len); + if (!ret) + ret = mt7932_data_header(header, payload, skb->len, m->data_payloads_dma + 4096 * slot, + slot, m->peer_wtbl, 0, 0, !m->connect_open); + if (ret) { + spin_unlock_irqrestore(&m->data_lock, flags); + goto drop; + } + dma = m->data_headers_dma + 64 * slot; + desc = &q->desc[slot]; + desc->address = cpu_to_le32(lower_32_bits(dma)); + desc->second = 0; + desc->info = cpu_to_le32(upper_32_bits(dma)); + m->data_live[slot] = m->data_token[slot] = true; + m->data_retired[slot] = false; + dma_wmb(); + desc->control = cpu_to_le32((64 << 16) | BIT(30)); + q->head = (slot + 1) % q->count; + q->queued++; + dma_wmb(); + mt_write(m, q->reg + 8, q->head); + mt_bss_tx_account(m); + m->station_tx_packets++; + m->station_tx_bytes += skb->len; + if (!mt_bss_tx_allowed(m) || m->data_live[q->head] || q->queued >= q->count - 1) + netif_stop_queue(netdev); + spin_unlock_irqrestore(&m->data_lock, flags); + netdev->stats.tx_packets++; + netdev->stats.tx_bytes += skb->len; + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; +drop: + netdev->stats.tx_dropped++; + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; +} + +void mt_data_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame) +{ + struct sk_buff *skb; + unsigned long flags; + int signal; + + if (!READ_ONCE(m->data_ready)) + return; + /* Serialize peer identity and accounting with association retirement. */ + spin_lock_irqsave(&m->response_lock, flags); + if (!m->connected || !m->peer_valid || m->disconnecting || m->stopping) + goto unlock; + if ((frame->unicast && frame->wtbl != m->peer_wtbl) || + (!frame->unicast && frame->wtbl != m->peer_ids.wtbl[0] && frame->wtbl != m->peer_wtbl) || + mt7932_rx_duplicate(m->duplicates, frame)) { + m->netdev->stats.rx_dropped++; + goto unlock; + } + skb = netdev_alloc_skb_ip_align(m->netdev, frame->length); + if (!skb) { + m->netdev->stats.rx_dropped++; + goto unlock; + } + skb_put_data(skb, frame->data, frame->length); + skb->protocol = eth_type_trans(skb, m->netdev); + skb->ip_summed = CHECKSUM_NONE; + m->netdev->stats.rx_packets++; + m->netdev->stats.rx_bytes += frame->length; + m->rx_ethernet++; + spin_lock(&m->data_lock); + m->station_rx_packets++; + m->station_rx_bytes += frame->length; + signal = mt7932_rx_signal(frame, m->antenna_mask); + if (frame->signal_valid && signal > -128) { + m->station_signal = signal; + m->station_signal_valid = true; + } + if (frame->unicast && frame->rate_valid) { + m->station_rx_rate_valid = mt7932_rx_rate(frame->rxv, m->phy_cap[4], + &m->station_rx_rate); + /* Current association contract admits only 20 MHz and no 5 GHz CCK. */ + if (((frame->rxv >> 12) & 7) || + (frame->channel > 14 && !((frame->rxv >> 24) & 15))) + m->station_rx_rate_valid = false; + m->station_rx_rate_time = jiffies; + } + spin_unlock(&m->data_lock); + spin_unlock_irqrestore(&m->response_lock, flags); + netif_rx(skb); + dev_dbg_ratelimited(&m->pdev->dev, "ETHERNET_RX: bytes=%zu peer=%u total=%llu\n", + frame->length, frame->wtbl, m->rx_ethernet); + return; +unlock: + spin_unlock_irqrestore(&m->response_lock, flags); +} + +int mt_data_prepare(struct mt7932 *m) +{ + unsigned int band, i; + u8 protection[12]; + int ret; + + if (m->tx[0].count != ARRAY_SIZE(m->data_live)) + return -EINVAL; + spin_lock_init(&m->data_lock); + m->data_headers = mt_alloc(m, 512 * 64, &m->data_headers_dma); + m->data_payloads = mt_alloc(m, 512 * 4096, &m->data_payloads_dma); + if (!m->data_headers || !m->data_payloads) + return -ENOMEM; + /* Source-defined fresh MAC init; preserve all unrelated register bits. */ + mt_rmw(m, 0xf004, 0xfff8, 0x3000); + mt_rmw(m, 0xf000, 0, 0x88000); + for (i = 0; i < 20; i++) { + mt_rmw(m, 0x34230, 0x3ff, i | BIT(12)); + if (mt_poll(m, 0x34230, BIT(31), 0, 5000)) + return -ETIMEDOUT; + } + for (band = 0; band < 2; band++) { + u32 offset = band ? 0x80000 : 0; + + mt_rmw(m, 0x210f4 + offset, 0x3f, 0x3f | BIT(17) | BIT(18)); + mt_rmw(m, 0x217c4 + offset, 0, BIT(30)); + mt_rmw(m, 0x21780 + offset, 0, BIT(30)); + mt_rmw(m, 0x24804 + offset, 0, BIT(8) | BIT(9)); + mt_rmw(m, 0x21e00 + offset, 0xfff8 | BIT(23), 0x3000); + mt_rmw(m, 0x23408 + offset, 0xc3000000, 0x03000000); + } + mt7932_rts_config(protection); + ret = mt_request_ext(m, 0xed, 0x3e, true, true, true, + protection, sizeof(protection)); + if (ret) + return ret; + /* The selected response parser establishes sequence completion only, + * not a command-specific firmware status. No payload-status invention. + */ + dev_info(&m->pdev->dev, "MAC_RTS_RESPONSE: sequence matched, EID=%02x length=%zu\n", + m->reply[28], m->reply_length); + WRITE_ONCE(m->data_ready, true); + dev_info(&m->pdev->dev, "ETHERNET_PREPARED: TX0 retained coherent slots=512, RX translation enabled\n"); + return 0; +} diff --git a/drivers/net/wireless/mediatek/mt7932/firmware.h b/drivers/net/wireless/mediatek/mt7932/firmware.h new file mode 100644 index 0000000000000..4c08d82bf6735 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/firmware.h @@ -0,0 +1,158 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef MT7932_FIRMWARE_H +#define MT7932_FIRMWARE_H + +#include "protocol.h" +#include "bandwidth.h" + +struct mt7932_region { + u32 offset, length, target, mode; + u8 features; +}; + +/* Source-selected defaults, not regulatory limits or permission to transmit. */ +static inline void mt7932_basic_config(u8 body[24]) +{ + memset(body, 0, 24); + body[3] = MT7932_MAX_5G_BW; + body[12] = 1; + body[14] = 2; + body[15] = 3; + put_unaligned_le16(0x2000, body + 16); + body[20] = 2; +} + +static inline void mt7932_mlme_config(u8 body[4]) +{ + body[0] = 7; + body[1] = 4; + body[2] = 2; + body[3] = 0; +} + +/* Normal band0 MAC initialization, legacy ED/3e SET with a reply waiter. */ +static inline void mt7932_rts_config(u8 body[12]) +{ + memset(body, 0, 12); + body[0] = 1; + put_unaligned_le32(2347, body + 4); + put_unaligned_le32(2, body + 8); +} + +static inline int mt7932_patch_region(const u8 *data, size_t size, unsigned int index, + struct mt7932_region *region) +{ + u32 count, kind, bytes, security; + const u8 *record; + + if (size < 96) + return -EINVAL; + count = get_unaligned_be32(data + 44); + if (!count || count > 4096 || count > (size - 96) / 64 || index >= count) + return -EINVAL; + record = data + 96 + index * 64; + kind = get_unaligned_be32(record); + region->offset = get_unaligned_be32(record + 4); + bytes = get_unaligned_be32(record + 8); + region->target = get_unaligned_be32(record + 12); + region->length = get_unaligned_be32(record + 16); + security = get_unaligned_be32(record + 20); + if ((kind & 0xffff) != 2 || region->offset < 96 + count * 64 || + region->offset > size || bytes > size - region->offset || + !region->length || region->length > bytes || + region->length - 1 > (u32)~0U - region->target) + return -EINVAL; + region->features = 0; + region->mode = 1U << 31; + if (security != (u32)~0U) { + switch (security >> 24) { + case 0: + break; + case 1: + if ((security & 0xff) > 3) + return -EINVAL; + region->mode |= 9 | ((security & 3) << 1); + break; + case 2: + region->mode |= 0x49; + break; + default: + return -EOPNOTSUPP; + } + } + return count; +} + +static inline int mt7932_ram_region(const u8 *data, size_t size, unsigned int index, + struct mt7932_region *region) +{ + size_t metadata, cursor = 0; + unsigned int count, i; + const u8 *record; + u32 length; + + if (size < 36) + return -EINVAL; + count = data[size - 34]; + if (!count || count > (size - 36) / 40 || index >= count) + return -EINVAL; + metadata = size - 36 - count * 40; + for (i = 0; i <= index; i++) { + record = data + metadata + i * 40; + length = get_unaligned_le32(record + 20); + if (!length || cursor > metadata || length > metadata - cursor) + return -EINVAL; + if (i != index) + cursor += length; + } + if (cursor > (u32)~0U) + return -EINVAL; + region->offset = cursor; + region->length = length; + region->target = get_unaligned_le32(record + 16); + region->features = record[24]; + if (length - 1 > (u32)~0U - region->target) + return -EINVAL; + region->mode = 1U << 31; + if (region->features & 1) + region->mode |= 9 | (region->features & 6); + if (region->features & 16) + region->mode |= 64; + return count; +} + +/* Validate the entire capability body before consumers use any individual TLV. */ +static inline int mt7932_capabilities(const u8 *body, size_t length) +{ + unsigned int count, i; + size_t offset = 4; + + if (length < 4) + return -EMSGSIZE; + count = get_unaligned_le16(body); + for (i = 0; i < count; i++) { + u32 type, size, minimum = 0; + + if (length - offset < 8) + return -EMSGSIZE; + type = get_unaligned_le32(body + offset); + size = get_unaligned_le32(body + offset + 4); + offset += 8; + if (size > length - offset) + return -EMSGSIZE; + switch (type) { + case 7: minimum = 6; break; + case 8: minimum = 12; break; + case 0x18: minimum = 4; break; + /* Stock W7932_2 returns four bytes; keep this capability opaque. */ + case 0x20: minimum = 4; break; + } + if (size < minimum) + return -EMSGSIZE; + if (type == 8 && (!body[offset + 4] || body[offset + 4] > 4)) + return -EINVAL; + offset += size; + } + return offset == length ? (int)count : -EPROTO; +} +#endif diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c new file mode 100644 index 0000000000000..464dca910a978 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -0,0 +1,613 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include "mt7932.h" + +static void mt_scan_finish_work(struct work_struct *work); +static void mt_scan_timeout_work(struct work_struct *work); +static int mt_scan_submit_next(struct mt7932 *m); +static int mt_scan_admit(struct mt7932 *m); +static int mt_set_bss(struct mt7932 *m, bool active); +static int mt_net_open(struct net_device *netdev); +static int mt_net_stop(struct net_device *netdev); +static void mt_cal_work(struct work_struct *work); + +/* Standard cfg80211 scan and station interface. */ +struct mt7932 *mt_from_wiphy(struct wiphy *wiphy) +{ + return *(struct mt7932 **)wiphy_priv(wiphy); +} + + + + +static void mt_scan_finish_work(struct work_struct *work) +{ + struct mt7932 *m = container_of(work, struct mt7932, scan_finish_work); + struct cfg80211_scan_request *request; + struct cfg80211_scan_info info = {}; + unsigned long flags; + u8 cancel[4] = {}; + int ret; + + /* Retire the old timer before allowing another request to take ownership. */ + cancel_delayed_work_sync(&m->scan_timeout_work); + mutex_lock(&m->command_mutex); + spin_lock_irqsave(&m->response_lock, flags); + request = m->scan_request; + info.aborted = m->scan_aborted || m->reg_pending || m->stopping; + cancel[0] = m->scan_seq; + spin_unlock_irqrestore(&m->response_lock, flags); + if (request && !info.aborted) { + ret = mt_scan_submit_next(m); + if (!ret) { + mutex_unlock(&m->command_mutex); + return; + } + if (ret < 0) + info.aborted = true; + } + cancel_delayed_work(&m->scan_timeout_work); + spin_lock_irqsave(&m->response_lock, flags); + cancel[0] = m->scan_seq; + if (request && info.aborted && !m->scan_finished) + m->retired_scan_seq = m->scan_seq; + m->scan_request = NULL; + spin_unlock_irqrestore(&m->response_lock, flags); + if (request && info.aborted && !READ_ONCE(m->stopping)) + mt_transport_snapshot(m); + if (request && info.aborted && !READ_ONCE(m->stopping) && READ_ONCE(m->retired_scan_seq)) { + ret = mt_request(m, 0x1b, true, true, false, cancel, sizeof(cancel)); + if (ret) { + WRITE_ONCE(m->link_failed, true); + dev_err(&m->pdev->dev, "SCAN_RETIREMENT_FAILED: %d; recovery required\n", ret); + } + } + mutex_unlock(&m->command_mutex); + if (request) { + dev_info(&m->pdev->dev, "SCAN_FINISHED: seq=%u aborted=%u received-BSS=%llu\n", + cancel[0], info.aborted, m->beacons); + cfg80211_scan_done(request, &info); + } +} + +static void mt_scan_timeout_work(struct work_struct *work) +{ + struct mt7932 *m = container_of(to_delayed_work(work), struct mt7932, scan_timeout_work); + unsigned long flags; + + spin_lock_irqsave(&m->response_lock, flags); + if (!m->stopping && m->scan_request && !m->scan_finished) { + m->scan_aborted = true; + schedule_work(&m->scan_finish_work); + } + spin_unlock_irqrestore(&m->response_lock, flags); +} + +/* Called under response_lock; scan payload sequence differs from MCU sequence. */ +void mt_scan_event(struct mt7932 *m, const struct mt7932_event *event) +{ + if (m->stopping) + return; + /* Abort completes userspace locally, not by an invented cancel ACK. + * Keep the sequence reserved until an actual old-campaign done arrives. + */ + if (event->length >= 37 && m->retired_scan_seq && + event->packet[36] == m->retired_scan_seq) { + m->retired_scan_seq = 0; + complete(&m->scan_done); + if (m->reg_pending) { + m->reg_generation++; + schedule_work(&m->startup_work); + } + return; + } + if (event->length >= 37 && m->discovering && event->packet[36] == m->scan_seq) { + m->discovery_finished = true; + complete(&m->discovery_done); + return; + } + if (event->length < 37 || !m->scan_request || event->packet[36] != m->scan_seq) + return; + m->scan_finished = true; + complete(&m->scan_done); + schedule_work(&m->scan_finish_work); +} + +void mt_abort_scan(struct wiphy *wiphy, struct wireless_dev *wdev) +{ + struct mt7932 *m = mt_from_wiphy(wiphy); + unsigned long flags; + + spin_lock_irqsave(&m->response_lock, flags); + if (!m->stopping && m->scan_request && wdev == &m->wdev) { + m->scan_aborted = true; + schedule_work(&m->scan_finish_work); + } + spin_unlock_irqrestore(&m->response_lock, flags); +} + +/* No command/response lock held. The caller closes new scan admission first. + * Natural EID0d completion fences the campaign; a local cancel alone does not. + */ +int mt_scan_quiesce(struct mt7932 *m) +{ + if (READ_ONCE(m->scan_request) || READ_ONCE(m->retired_scan_seq)) + wait_for_completion_timeout(&m->scan_done, msecs_to_jiffies(5000)); + mt_abort_scan(m->wiphy, &m->wdev); + flush_work(&m->scan_finish_work); + return READ_ONCE(m->retired_scan_seq) ? -EBUSY : 0; +} + +/* command_mutex held: one RF-band/type batch at a time, with matching EID0d + * completion before the next batch. A request without SSIDs is passive; + * active requests must also listen passively on NO_IR channels. + */ +static int mt_scan_submit_next(struct mt7932 *m) +{ + struct cfg80211_scan_request *request = m->scan_request; + const struct cfg80211_ssid *ssid = NULL; + u8 body[0x4d4], channels[13]; + unsigned int count = 0, i; + unsigned long flags; + bool passive = false; + int ret; + + /* 2GHz active/passive, then 5GHz active/passive. Never put a NO_IR + * channel in a firmware request which can emit probe requests. + */ + while (!count && ++m->scan_batch <= 4) { + m->scan_band = (m->scan_batch + 1) / 2; + passive = !(m->scan_batch & 1); + for (i = 0; i < request->n_channels; i++) { + struct ieee80211_channel *channel = request->channels[i]; + bool listen = !request->n_ssids || + (channel->flags & IEEE80211_CHAN_NO_IR); + + if (mt7932_channel_band(channel->hw_value) != m->scan_band || + listen != passive) + continue; + if (channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_20MHZ) || + count == ARRAY_SIZE(channels)) + return -EINVAL; + channels[count++] = channel->hw_value; + } + } + if (!count) + return 1; + if (!passive && request->n_ssids) + ssid = &request->ssids[0]; + spin_lock_irqsave(&m->response_lock, flags); + if (m->scan_aborted || !mt_rf_allowed(m) || m->disconnecting || + (m->scan_home_channel && !m->connected)) { + spin_unlock_irqrestore(&m->response_lock, flags); + return -ECANCELED; + } + m->scan_seq = m->scan_seq % 127 + 1; + m->scan_finished = false; + reinit_completion(&m->scan_done); + spin_unlock_irqrestore(&m->response_lock, flags); + ret = mt7932_scan_body(body, sizeof(body), m->scan_seq, channels, count); + if (ret >= 0 && passive) { + if (m->scan_home_channel) + mt7932_scan_passive(body); + else + mt7932_scan_disconnected_passive(body); + } + if (ret >= 0) + ret = mt7932_scan_home(body, m->scan_home_channel); + if (ret >= 0) + ret = mt7932_scan_options(body, ssid ? ssid->ssid : NULL, + ssid ? ssid->ssid_len : 0, + request->ie, request->ie_len); + if (ret >= 0) { + schedule_delayed_work(&m->scan_timeout_work, msecs_to_jiffies(60000)); + ret = mt_request(m, 3, true, true, false, body, sizeof(body)); + } + if (!ret) + dev_info(&m->pdev->dev, "SCAN_SUBMITTED: seq=%u band=%u channels=%u passive=%u\n", + m->scan_seq, m->scan_band, count, passive); + return ret; +} + +/* Internal, disconnected discovery shares the firmware scan owner, but never + * invents a cfg80211_scan_request or completes a userspace scan. The connect + * epoch is already admitted, so external scans/connects remain excluded. + * command_mutex is held on entry/exit; RX can run during all bounded waits. + */ +int mt_connect_discover(struct mt7932 *m) +{ + u8 body[0x4d4], channels[13], cancel[4] = {}; + unsigned long flags; + unsigned int attempt, band, i, count; + int ret = 0; + + /* A short directed probe can miss a just-restarted AP. One bounded + * passive fallback obtains real beacons, without treating a stale + * userspace candidate as evidence or relaxing association admission. + */ + for (attempt = 0; attempt < 4; attempt++) { + struct ieee80211_supported_band *sband; + bool passive = attempt >= 2; + + band = 1 + attempt % 2; + sband = band == 1 ? &m->band2 : &m->band5; + count = 0; + for (i = 0; i < sband->n_channels; i++) { + struct ieee80211_channel *channel = &sband->channels[i]; + + if (m->connect_channel_req && channel != m->connect_channel_req) + continue; + if (channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_20MHZ)) + continue; + passive |= !!(channel->flags & IEEE80211_CHAN_NO_IR); + channels[count++] = channel->hw_value; + } + if (!count) + continue; + spin_lock_irqsave(&m->response_lock, flags); + if (m->connect_cancelled || m->connect_error || !mt_rf_allowed(m)) { + spin_unlock_irqrestore(&m->response_lock, flags); + return -ECANCELED; + } + reinit_completion(&m->discovery_done); + m->scan_seq = m->scan_seq % 127 + 1; + m->discovering = true; + m->discovery_finished = false; + spin_unlock_irqrestore(&m->response_lock, flags); + ret = mt7932_scan_body(body, sizeof(body), m->scan_seq, channels, count); + if (ret >= 0) { + /* NO_IR forbids probes, not listening. A missing candidate in + * the world domain still needs discovery; the connect worker + * independently rechecks permission before association. + */ + if (passive) + mt7932_scan_disconnected_passive(body); + ret = mt7932_scan_options(body, passive ? NULL : m->connect_ssid, + passive ? 0 : m->connect_ssid_length, NULL, 0); + } + if (ret >= 0) + ret = mt_request(m, 3, true, true, false, body, sizeof(body)); + dev_dbg(&m->pdev->dev, "CONNECT_DISCOVERY_SCAN: seq=%u band=%u channels=%u passive=%u status=%d\n", + m->scan_seq, band, count, passive, ret); + mutex_unlock(&m->command_mutex); + if (!ret && !wait_for_completion_timeout(&m->discovery_done, + msecs_to_jiffies(5000))) + ret = -ETIMEDOUT; + mutex_lock(&m->command_mutex); + /* Cancellation wakes the waiter, but is not a firmware completion. + * Keep ownership closed unless the matching terminal event arrives. + */ + if (!READ_ONCE(m->discovery_finished)) { + cancel[0] = m->scan_seq; + reinit_completion(&m->discovery_done); + mt_request(m, 0x1b, true, true, false, cancel, sizeof(cancel)); + mutex_unlock(&m->command_mutex); + wait_for_completion_timeout(&m->discovery_done, msecs_to_jiffies(1000)); + mutex_lock(&m->command_mutex); + } + spin_lock_irqsave(&m->response_lock, flags); + if (!m->discovery_finished) { + m->link_failed = true; + ret = ret ?: -ETIMEDOUT; + } + m->discovering = false; + if (m->connect_cancelled) + ret = -ECANCELED; + spin_unlock_irqrestore(&m->response_lock, flags); + if (ret) + return ret; + m->connect_bss = mt_connect_find_bss(m); + if (m->connect_bss) + return 0; + } + return -ENOENT; +} + +/* response_lock held. EBUSY means a real scan can later complete, not merely + * that RF is unavailable. iwd waits for that completion before retrying. + */ +static int mt_scan_admit(struct mt7932 *m) +{ + if (m->link_failed || m->retired_scan_seq) + return -EIO; + if (!mt_rf_allowed(m) || !m->bss_active || m->connecting || m->disconnecting) + return -EAGAIN; + if (m->scan_request) + return -EBUSY; + return 0; +} + +int mt_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) +{ + struct mt7932 *m = mt_from_wiphy(wiphy); + unsigned long flags; + unsigned int i; + int ret; + + if (request->wdev != &m->wdev) + return -ENODEV; + dev_info(&m->pdev->dev, "SCAN_REQUEST: channels=%u ssids=%d ies=%zu flags=%08x duration=%u/%u\n", + request->n_channels, request->n_ssids, request->ie_len, request->flags, + request->duration, request->duration_mandatory); + if (!request->n_channels || request->n_channels > 17 || request->ie_len > 600 || + request->duration_mandatory || + (request->flags & ~(NL80211_SCAN_FLAG_FLUSH | NL80211_SCAN_FLAG_COLOCATED_6GHZ)) || request->n_ssids > 1 || + (request->n_ssids && request->ssids[0].ssid_len > 32)) + return -EOPNOTSUPP; + for (i = 0; i < request->n_channels; i++) { + struct ieee80211_channel *channel = request->channels[i]; + + if (!mt7932_channel_band(channel->hw_value) || + (channel->band != NL80211_BAND_2GHZ && + channel->band != NL80211_BAND_5GHZ) || + channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_20MHZ)) + return -EINVAL; + } + mutex_lock(&m->command_mutex); + spin_lock_irqsave(&m->response_lock, flags); + ret = mt_scan_admit(m); + if (ret) { + spin_unlock_irqrestore(&m->response_lock, flags); + goto out; + } + m->scan_band = 0; + m->scan_batch = 0; + m->scan_home_channel = m->connected ? m->connect_channel : 0; + m->scan_request = request; + m->scan_finished = m->scan_aborted = false; + spin_unlock_irqrestore(&m->response_lock, flags); + ret = mt_scan_submit_next(m); + if (ret < 0) { + spin_lock_irqsave(&m->response_lock, flags); + m->scan_request = NULL; + spin_unlock_irqrestore(&m->response_lock, flags); + cancel_delayed_work(&m->scan_timeout_work); + } else { + ret = 0; + } +out: + mutex_unlock(&m->command_mutex); + return ret; +} + +/* command_mutex held; host interface existence is independent of RF policy. */ +static int mt_set_bss(struct mt7932 *m, bool active) +{ + u8 body[12] = {0,1,0,0,0,0,0,0,0,0,0xff,1}; + int ret; + + body[1] = active; + if (active) + memcpy(body + 4, m->mac, 6); + ret = mt_request(m, 0x11, true, true, false, body, sizeof(body)); + if (!ret) + m->bss_active = active; + return ret; +} + +static int mt_net_open(struct net_device *netdev) +{ + struct mt7932 *m = *(struct mt7932 **)netdev_priv(netdev); + int ret = 0; + + mutex_lock(&m->command_mutex); + if (READ_ONCE(m->stopping)) { + mutex_unlock(&m->command_mutex); + return -ESHUTDOWN; + } + if (READ_ONCE(m->rf_ready)) + ret = mt_set_bss(m, true); + if (!ret) + WRITE_ONCE(m->interface_up, true); + mutex_unlock(&m->command_mutex); + netif_carrier_off(netdev); + netif_stop_queue(netdev); + return ret; +} + +static int mt_net_stop(struct net_device *netdev) +{ + struct mt7932 *m = *(struct mt7932 **)netdev_priv(netdev); + WRITE_ONCE(m->interface_up, false); + if (READ_ONCE(m->stopping)) { + netif_carrier_off(netdev); + netif_stop_queue(netdev); + return 0; + } + + mt_disconnect(m->wiphy, netdev, WLAN_REASON_DEAUTH_LEAVING); + flush_work(&m->connect_work); + flush_work(&m->disconnect_work); + mt_abort_scan(m->wiphy, &m->wdev); + flush_work(&m->scan_finish_work); + mutex_lock(&m->command_mutex); + if (m->bss_active) + mt_set_bss(m, false); + mutex_unlock(&m->command_mutex); + netif_carrier_off(netdev); + netif_stop_queue(netdev); + return 0; +} + +static int mt_set_mac_address(struct net_device *netdev, void *address) +{ + struct mt7932 *m = *(struct mt7932 **)netdev_priv(netdev); + unsigned long flags; + int ret; + + /* No live-MAC change: the next BSS activation and connect commands use + * this address together. A failed retirement must not change identity. + */ + mutex_lock(&m->command_mutex); + spin_lock_irqsave(&m->response_lock, flags); + if (m->stopping || m->link_failed || m->bss_active || m->connecting || + m->connected || m->disconnecting || m->peer_valid || + m->scan_request || m->retired_scan_seq || m->discovering) { + ret = -EBUSY; + } else { + ret = eth_mac_addr(netdev, address); + if (!ret) + ether_addr_copy(m->mac, netdev->dev_addr); + } + spin_unlock_irqrestore(&m->response_lock, flags); + mutex_unlock(&m->command_mutex); + return ret; +} + +static const struct net_device_ops mt_net_ops = { + .ndo_open = mt_net_open, .ndo_stop = mt_net_stop, .ndo_start_xmit = mt_net_xmit, + .ndo_set_mac_address = mt_set_mac_address, +}; + +static void mt_cal_work(struct work_struct *work) +{ + struct mt7932 *m = container_of(work, struct mt7932, cal_work); + const struct firmware *oca; + struct mt7932_cal_piece pieces[7]; + u8 request[16]; + unsigned long flags; + unsigned int count; + int ret; + + mutex_lock(&m->command_mutex); + if (!READ_ONCE(m->rf_ready)) + goto unlock; + ret = request_firmware_direct(&oca, "mediatek/mt7932/oca2.bin", &m->pdev->dev); + if (ret) + goto fail; + for (;;) { + spin_lock_irqsave(&m->response_lock, flags); + count = m->cal_request_count; + if (count) { + memcpy(request, m->cal_requests[m->cal_request_head], 16); + m->cal_request_head = (m->cal_request_head + 1) % ARRAY_SIZE(m->cal_requests); + m->cal_request_count--; + } + spin_unlock_irqrestore(&m->response_lock, flags); + if (!count) + break; + ret = mt7932_cal_requested(oca->data, oca->size, request, sizeof(request), + m->smart_version, m->module_byte, pieces); + if (ret < 0) + break; + ret = mt_cal_procedure(m, pieces, ret, + get_unaligned_le32(request + 8) == 1 ? 3 : 4, + m->preload_version); + if (ret) + break; + dev_info(&m->pdev->dev, "D7_CALIBRATION_COMPLETE: channel=%u\n", get_unaligned_le32(request + 12)); + } + release_firmware(oca); + if (!ret) + goto unlock; +fail: + WRITE_ONCE(m->rf_ready, false); + mt_abort_scan(m->wiphy, &m->wdev); + dev_err(&m->pdev->dev, "D7_CALIBRATION_FAILED: %d; RF operations stopped\n", ret); +unlock: + mutex_unlock(&m->command_mutex); +} + +int mt_register_interface(struct mt7932 *m) +{ + struct net_device *netdev; + int ret; + + netdev = alloc_etherdev(sizeof(m)); + if (!netdev) + return -ENOMEM; + *(struct mt7932 **)netdev_priv(netdev) = m; + m->wdev.wiphy = m->wiphy; + m->wdev.iftype = NL80211_IFTYPE_STATION; + m->wdev.netdev = netdev; + netdev->ieee80211_ptr = &m->wdev; + netdev->netdev_ops = &mt_net_ops; + netdev->min_mtu = ETH_MIN_MTU; + netdev->max_mtu = ETH_DATA_LEN; + SET_NETDEV_DEV(netdev, &m->pdev->dev); + eth_hw_addr_set(netdev, m->mac); + ether_addr_copy(netdev->perm_addr, m->wiphy->perm_addr); + strscpy(netdev->name, "wlan%d", IFNAMSIZ); + netif_carrier_off(netdev); + INIT_WORK(&m->scan_finish_work, mt_scan_finish_work); + INIT_WORK(&m->cal_work, mt_cal_work); + INIT_WORK(&m->connect_work, mt_connect_work); + INIT_WORK(&m->disconnect_work, mt_disconnect_work); + INIT_WORK(&m->power_work, mt_power_work); + init_completion(&m->assoc_start); + init_completion(&m->assoc_done); + init_completion(&m->disconnect_done); + init_completion(&m->discovery_done); + init_completion(&m->scan_done); + m->peer_alloc = BIT(0) | BIT(19); + INIT_DELAYED_WORK(&m->scan_timeout_work, mt_scan_timeout_work); + m->netdev = netdev; + ret = register_netdev(netdev); + if (ret) { + m->netdev = NULL; + free_netdev(netdev); + return ret; + } + dev_info(&m->pdev->dev, "CFG80211_INTERFACE_READY: %s, RF awaits country policy\n", netdev->name); + return 0; +} + +int mt_enable_scan(struct mt7932 *m) +{ + int ret = mt_data_prepare(m); + + if (ret) + return ret; + if (READ_ONCE(m->interface_up)) { + ret = mt_set_bss(m, true); + if (ret) + return ret; + } + WRITE_ONCE(m->rf_ready, true); + dev_info(&m->pdev->dev, "CFG80211_SCAN_READY: %s, carrier off until WPA2 authorization\n", m->netdev->name); + schedule_work(&m->cal_work); + return 0; +} + +void mt_packet_receive(struct mt7932 *m, const u8 *packet, size_t length) +{ + struct mt7932_rx_frame frame; + struct cfg80211_bss *bss; + struct ieee80211_channel *channel; + unsigned int band; + int ret = mt7932_rx_frame(packet, length, &frame); + + if (ret || !READ_ONCE(m->rf_ready)) + return; + band = mt7932_channel_band(frame.channel); + channel = ieee80211_get_channel(m->wiphy, band == 1 ? 2407 + 5 * frame.channel : 5000 + 5 * frame.channel); + if (!channel || (channel->flags & IEEE80211_CHAN_DISABLED)) + return; + if (frame.translated) { + mt_data_receive(m, &frame); + return; + } + if (!frame.signal_valid || !mt7932_rx_beacon(&frame)) + return; + if ((frame.frame_control & 0xfc) == IEEE80211_STYPE_BEACON) { + const struct element *tim = cfg80211_find_elem(WLAN_EID_TIM, + frame.data + 36, frame.length - 36); + unsigned long flags; + + spin_lock_irqsave(&m->response_lock, flags); + if (tim && tim->datalen >= 4 && m->connected && !m->disconnecting && + !m->stopping && !m->power_tim && frame.channel == m->connect_channel && + ether_addr_equal(frame.data + 16, m->connect_bssid)) { + m->power_tim = true; + schedule_work(&m->power_work); + } + spin_unlock_irqrestore(&m->response_lock, flags); + } + bss = cfg80211_inform_bss_frame(m->wiphy, channel, + (struct ieee80211_mgmt *)frame.data, frame.length, + mt7932_rx_signal(&frame, m->antenna_mask) * 100, GFP_ATOMIC); + if (bss) { + m->beacons++; + dev_dbg_ratelimited(&m->pdev->dev, "BSS_RECEIVED: channel=%u BSSID=%pM\n", frame.channel, bss->bssid); + cfg80211_put_bss(m->wiphy, bss); + } +} diff --git a/drivers/net/wireless/mediatek/mt7932/lifecycle.c b/drivers/net/wireless/mediatek/mt7932/lifecycle.c new file mode 100644 index 0000000000000..d05d05cd7d7a0 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/lifecycle.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include "mt7932.h" + +/* Join all host users before resetting hardware. No RTNL/wiphy mutex held: + * network callbacks and workers may need command_mutex while being joined. + * DMA storage and Linux objects remain owned until the hardware fence. + */ +void mt_stop_host(struct mt7932 *m) +{ + struct cfg80211_scan_request *scan; + struct cfg80211_scan_info info = { .aborted = true }; + unsigned long flags; + + /* Serialize with command publication and callbacks that queue work. */ + mutex_lock(&m->command_mutex); + spin_lock_irqsave(&m->response_lock, flags); + WRITE_ONCE(m->stopping, true); + WRITE_ONCE(m->rf_ready, false); + m->connect_error = -ESHUTDOWN; + m->cal_state.error = -ESHUTDOWN; + complete_all(&m->response); + complete_all(&m->cal_response); + if (m->interface_registered) { + complete_all(&m->assoc_start); + complete_all(&m->assoc_done); + complete_all(&m->disconnect_done); + complete_all(&m->discovery_done); + complete_all(&m->scan_done); + } + spin_unlock_irqrestore(&m->response_lock, flags); + mutex_unlock(&m->command_mutex); + if (m->netdev) { + netif_carrier_off(m->netdev); + netif_tx_disable(m->netdev); + } + WRITE_ONCE(m->running, false); + mt_write(m, W + 0x204, 0); + if (m->irq_requested) { + devm_free_irq(&m->pdev->dev, pci_irq_vector(m->pdev, 0), m); + m->irq_requested = false; + } + /* A service invocation already in flight may have re-enabled the mask. */ + mt_write(m, W + 0x204, 0); + if (!m->interface_registered) + return; + cancel_work_sync(&m->startup_work); + /* Let an admitted connect publish its one terminal cfg80211 result. */ + flush_work(&m->connect_work); + flush_work(&m->disconnect_work); + cancel_work_sync(&m->cal_work); + cancel_work_sync(&m->power_work); + cancel_delayed_work_sync(&m->scan_timeout_work); + cancel_work_sync(&m->scan_finish_work); + /* With all producers joined, this is the sole owner of any remainder. */ + scan = m->scan_request; + m->scan_request = NULL; + if (scan) + cfg80211_scan_done(scan, &info); + if (m->connect_bss) { + cfg80211_put_bss(m->wiphy, m->connect_bss); + m->connect_bss = NULL; + } + if (m->connected) { + m->connected = false; + cfg80211_disconnected(m->netdev, WLAN_REASON_DEAUTH_LEAVING, + NULL, 0, true, GFP_KERNEL); + } + memzero_explicit(m->connect_pmk, sizeof(m->connect_pmk)); +} diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c new file mode 100644 index 0000000000000..da708907b901e --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -0,0 +1,688 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include "mt7932.h" + +static int mt_peer_reserve(struct mt7932 *m); +static int mt_association_calibration(struct mt7932 *m); +static void mt_peer_release(struct mt7932 *m); +static int mt_retire_connection(struct mt7932 *m); +static bool mt_open_request(const struct cfg80211_connect_params *p); + +/* RX/event lock held. Decode the native layout, then let cfg80211 validate + * every occupied subchannel against the effective regulatory restrictions. + */ +static bool mt_channel_event(struct mt7932 *m, const u8 *body, size_t length) +{ + struct cfg80211_chan_def def = {}; + u8 center, bw; + + if (length < 20 || get_unaligned_le32(body + 12) != mt7932_channel_band(m->connect_channel) || + body[17] != m->connect_channel || + !mt7932_channel_layout(body[17], body[16], body[19], MT7932_MAX_5G_BW, ¢er, &bw)) + return false; + def.chan = ieee80211_get_channel(m->wiphy, body[17] <= 13 ? + 2407 + 5 * body[17] : 5000 + 5 * body[17]); + def.width = bw == 2 ? NL80211_CHAN_WIDTH_80 : bw == 1 ? + NL80211_CHAN_WIDTH_40 : NL80211_CHAN_WIDTH_20; + def.center_freq1 = body[17] <= 13 ? 2407 + 5 * center : 5000 + 5 * center; + if (!def.chan || !cfg80211_chandef_valid(&def) || + !cfg80211_chandef_usable(m->wiphy, &def, IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_RADAR)) + return false; + if (m->connect_center && (m->connect_center != center || m->connect_chandef.width != def.width)) + return false; + m->connect_center = center; + m->connect_chandef = def; + return true; +} + +/* One owned station, firmware WPA2-PSK handshake; no host key fabrication. */ +static int mt_peer_reserve(struct mt7932 *m) +{ + u8 *ids = (u8 *)&m->peer_ids; + unsigned long allocation = m->peer_alloc; + unsigned int i, id; + + for (i = 0; i < 6; i++) { + id = find_first_zero_bit(&allocation, i < 2 ? 15 : 14); + if (id >= (i < 2 ? 15 : 14)) + return -ENOSPC; + __set_bit(id, &allocation); + ids[i] = id; + } + m->peer_alloc = allocation; + return 0; +} + +/* response_lock held by the sole RX event consumer. Never wait here. */ +void mt_link_event(struct mt7932 *m, const struct mt7932_event *event) +{ + const u8 *body; + size_t length; + u32 selector; + + if (event->length < 48 || !m->netdev) + return; + selector = get_unaligned_le32(event->packet + 40); + body = event->packet + 48; + length = event->length - 48; + dev_info(&m->pdev->dev, "LINK_EVENT: selector=%02x length=%zu\n", selector, length); + if (selector == 0x44) { + if (length < 5 || (!m->connecting && !m->connected && !m->disconnecting)) + return; + dev_info(&m->pdev->dev, "DISCONNECT_EVENT: metadata=%u deferred=%u reason-index=%u\n", + body[0], body[3], body[4]); + if (m->disconnect_seen) + return; + /* body3 suppresses stock platform notification; it does not promise + * another EE44. Latch logical termination for either value, but keep + * every independent queue/token/DMA retirement fence below. + */ + m->disconnect_event_received = true; + m->disconnect_seen = true; + complete(&m->disconnect_done); + if (m->connected) { + m->connected = false; + m->disconnecting = true; + m->disconnect_local = false; + m->disconnect_reason = WLAN_REASON_UNSPECIFIED; + netif_carrier_off(m->netdev); + netif_stop_queue(m->netdev); + schedule_work(&m->disconnect_work); + } + if (m->connecting) { + m->disconnecting = true; + if (!m->connect_error) + m->connect_error = -ECONNRESET; + complete(&m->assoc_start); + complete(&m->assoc_done); + complete(&m->discovery_done); + } + return; + } + /* A terminal event/cancellation must exclude late old-peer publication + * while the connect worker is still alive to report and retire it. + */ + if (!m->connecting || m->disconnecting || m->connect_error) + return; + switch (selector) { + case 0x5e: { + int ret = -EINVAL; + u32 count = length >= 4 ? get_unaligned_le32(body) : 0; + + if (length >= 4 && count <= 512 && count <= length - 4) + ret = mt7932_assoc_capabilities(m->assoc_request_ies, + sizeof(m->assoc_request_ies), body + 4, count, + m->connect_open); + if (ret >= 0 && m->band2.ht_cap.ht_supported && + !mt7932_assoc_phy_valid(m->connect_channel <= 14 ? &m->band2 : &m->band5, + m->assoc_request_ies, ret)) + ret = -EPROTO; + if (ret < 0) { + m->connect_error = -EPROTO; + complete(&m->assoc_start); + complete(&m->assoc_done); + } else { + const struct element *he = cfg80211_find_ext_elem( + WLAN_EID_EXT_HE_CAPABILITY, m->assoc_request_ies, ret); + + m->assoc_request_ie_len = ret; + m->assoc_request_seen = true; + /* Generated station request, not the AP's capability or a + * rate-control setting. Bounds were checked by the exporter. + */ + if (he) + dev_info(&m->pdev->dev, + "ASSOC_HE_CAPABILITIES: epoch=%u rx-map=%04x tx-map=%04x\n", + m->connection_generation, + get_unaligned_le16(he->data + 18), + get_unaligned_le16(he->data + 20)); + } + break; + } + case 0x4d: + if (length >= 20) + dev_info(&m->pdev->dev, "ASSOC_START: band=%u width=%u channel=%u extension=%u\n", + get_unaligned_le32(body + 12), body[16], body[17], body[19]); + if (!mt_channel_event(m, body, length)) + m->connect_error = -EPROTO; + complete(&m->assoc_start); + break; + case 0x40: + if (length >= 52) + dev_info(&m->pdev->dev, "ASSOC_REPORT: status=%u band=%u channel=%u bss=%u bmc=%u station=%u peer=%u cipher=%08x\n", + get_unaligned_le16(body), body[11], body[12], body[16], body[17], body[20], body[21], + get_unaligned_le32(body + 48)); + /* OPEN reports retain cipher as numeric metadata only; it neither + * installs a key nor changes the immutable requested security mode. + */ + if (!mt7932_assoc_resources(body, length, &m->peer_ids, m->connect_channel, m->connect_open)) { + m->connect_error = -EPROTO; + complete(&m->assoc_start); + complete(&m->assoc_done); + break; + } + m->peer_wtbl = body[21]; + m->peer_valid = true; + /* HOST decoder confirms the RSN suite namespace, not request enum10. */ + m->event_cipher = get_unaligned_le32(body + 48); + dev_info(&m->pdev->dev, "ASSOCIATED: peer=%u station=%u cipher-raw=%u\n", + m->peer_wtbl, body[20], m->event_cipher); + break; + case 0x42: + if (length >= 2) + dev_info(&m->pdev->dev, "CONNECT_REPORT: status=%u peer-owned=%u\n", get_unaligned_le16(body), m->peer_valid); + if (length < 2 || get_unaligned_le16(body) || !m->peer_valid) + m->connect_error = -ECONNREFUSED; + complete(&m->assoc_done); + break; + } +} + +static int mt_association_calibration(struct mt7932 *m) +{ + const struct firmware *oca; + struct mt7932_cal_piece pieces[2]; + int ret; + + ret = request_firmware_direct(&oca, "mediatek/mt7932/oca2.bin", &m->pdev->dev); + if (ret) + return ret; + ret = mt7932_cal_association(oca->data, oca->size, m->connect_center, pieces); + if (ret == 2) + ret = mt_cal_procedure(m, pieces, 2, 1, 0); + release_firmware(oca); + return ret; +} + +/* command_mutex and response_lock held; all old DMA/readers already retired. */ +static void mt_peer_release(struct mt7932 *m) +{ + const u8 *ids = (const u8 *)&m->peer_ids; + unsigned int i; + + for (i = 0; i < sizeof(m->peer_ids); i++) + __clear_bit(ids[i], &m->peer_alloc); + memset(&m->peer_ids, 0, sizeof(m->peer_ids)); + memset(m->duplicates, 0, sizeof(m->duplicates)); + memzero_explicit(m->connect_pmk, sizeof(m->connect_pmk)); + m->peer_valid = false; + m->peer_wtbl = 0; + m->event_cipher = 0; +} + +/* No lifecycle/command lock held on entry. RX/IRQ and TX-free processing stay + * alive throughout. EE44 and queue removal are NOT a DMA fence. + */ +static int mt_retire_connection(struct mt7932 *m) +{ + u8 body[24] = {}, queue[32]; + unsigned long flags, deadline; + unsigned int i; + bool drained, event_received; + int ret = 0; + + if (READ_ONCE(m->stopping)) + return -ESHUTDOWN; + netif_carrier_off(m->netdev); + netif_tx_disable(m->netdev); + /* ndo_start_xmit rechecks the closed admission gate under data_lock. */ + spin_lock_irqsave(&m->data_lock, flags); + spin_unlock_irqrestore(&m->data_lock, flags); + flush_work(&m->cal_work); + mutex_lock(&m->command_mutex); + spin_lock_irqsave(&m->response_lock, flags); + drained = m->disconnect_seen; + event_received = m->disconnect_event_received; + if (!drained) + reinit_completion(&m->disconnect_done); + spin_unlock_irqrestore(&m->response_lock, flags); + if (!drained) { + put_unaligned_le64(8, body); /* version, not an IEEE reason */ + put_unaligned_le16(1, body + 8); + body[10] = 8; + memcpy(body + 13, m->connect_bssid, 6); + put_unaligned_le16(30, body + 20); + if (!event_received) + ret = mt_request_ext(m, 0xea, 0x42, true, true, false, body, sizeof(body)); + if (!ret && !wait_for_completion_timeout(&m->disconnect_done, msecs_to_jiffies(10000))) + ret = -ETIMEDOUT; + } + if (!ret) { + mt7932_peer_queue(queue, m->peer_ids.station[0], false); + ret = mt_request(m, 0x6b, true, true, false, queue, sizeof(queue)); + } + if (!ret) { + memset(queue, 0, sizeof(queue)); + put_unaligned_le16(15, queue); /* owned BSS0 queue bitmap */ + queue[2] = 1; + queue[14] = 1; + ret = mt_request(m, 0x6b, true, true, false, queue, sizeof(queue)); + } + deadline = jiffies + msecs_to_jiffies(5000); + while (!ret) { + spin_lock_irqsave(&m->data_lock, flags); + mt_data_clean_locked(m); + drained = !m->tx[0].queued; + for (i = 0; i < ARRAY_SIZE(m->data_live); i++) + drained &= !m->data_live[i] && !m->data_token[i]; + spin_unlock_irqrestore(&m->data_lock, flags); + mt_tx_clean(&m->tx[17]); + if (drained && !m->tx[17].queued) + break; + if (time_after_eq(jiffies, deadline)) { + ret = -ETIMEDOUT; + break; + } + usleep_range(1000, 2000); + } + mutex_unlock(&m->command_mutex); + /* Join already queued event consumers, without excluding future IRQs or + * claiming this is a firmware generation fence. Single-flight firmware + * lifecycle ordering remains necessary for asynchronous EE events. + */ + synchronize_irq(m->pdev->irq); + flush_work(&m->cal_work); + mutex_lock(&m->command_mutex); + spin_lock_irqsave(&m->response_lock, flags); + if (!ret && (m->cal_state.active || m->cal_state.error || m->cal_request_count || m->waiting)) + ret = -EPROTO; + if (!ret) + mt_peer_release(m); + else + m->link_failed = true; /* uncertain ownership: recovery required */ + spin_unlock_irqrestore(&m->response_lock, flags); + mutex_unlock(&m->command_mutex); + dev_info(&m->pdev->dev, "CONNECTION_RETIRED: status=%d%s\n", ret, + ret ? "; recovery required, DMA ownership retained" : "; old TX drained, six IDs released"); + return ret; +} + +/* Hints affect preference, not admission. Only real cfg80211 scan entries + * supply channel/BSSID and the reference passed to connect_done(). + */ +struct cfg80211_bss *mt_connect_find_bss(struct mt7932 *m) +{ + const u8 *bssid = is_valid_ether_addr(m->connect_bssid_req) ? m->connect_bssid_req : NULL; + struct cfg80211_bss *bss; + enum ieee80211_privacy privacy = m->connect_open ? + IEEE80211_PRIVACY_OFF : IEEE80211_PRIVACY_ON; + + bss = cfg80211_get_bss(m->wiphy, m->connect_channel_req ?: m->connect_channel_hint, + bssid ?: (is_valid_ether_addr(m->connect_bssid_hint) ? + m->connect_bssid_hint : NULL), + m->connect_ssid, m->connect_ssid_length, + IEEE80211_BSS_TYPE_ESS, privacy); + if (!bss && (m->connect_channel_hint || is_valid_ether_addr(m->connect_bssid_hint))) + bss = cfg80211_get_bss(m->wiphy, m->connect_channel_req, bssid, + m->connect_ssid, m->connect_ssid_length, + IEEE80211_BSS_TYPE_ESS, privacy); + return bss; +} + +void mt_connect_work(struct work_struct *work) +{ + struct mt7932 *m = container_of(work, struct mt7932, connect_work); + struct cfg80211_connect_resp_params response = {}; + u8 *body, mac[8] = {}, queue[32]; + unsigned long flags; + bool submitted = false, peer_owned = false; + int ret; + + mutex_lock(&m->command_mutex); + body = kzalloc(856, GFP_KERNEL); + ret = body ? 0 : -ENOMEM; + if (ret) + goto report; + if (!mt_rf_allowed(m) || READ_ONCE(m->connect_error)) { + ret = -ECANCELED; + goto free; + } + if (!m->connect_bss) { + dev_info(&m->pdev->dev, "CONNECT_DISCOVERY: refreshing missing scan candidate\n"); + ret = mt_connect_discover(m); + if (ret) + goto free; + } + if (m->connect_bss->channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_IR | + IEEE80211_CHAN_NO_20MHZ)) { + ret = -EINVAL; + goto free; + } + spin_lock_irqsave(&m->response_lock, flags); + ret = m->connect_error ?: mt_peer_reserve(m); + if (!ret) { + peer_owned = true; + m->connect_channel = m->connect_bss->channel->hw_value; + memcpy(m->connect_bssid, m->connect_bss->bssid, ETH_ALEN); + } + spin_unlock_irqrestore(&m->response_lock, flags); + if (ret) + goto free; + /* Initial station policy: framework M=0 branch and no optional flags. + * This yields flag2 independently of credentials; no scan-result count + * belongs in this field. The recovered framework uses listen interval20. + * Other platform flag policies remain unsupported, not guessed mappings. + */ + ret = mt7932_connect_body(body, 856, m->connect_ssid, m->connect_ssid_length, + m->connect_open ? NULL : m->connect_pmk, m->connect_bssid, + m->connect_channel, 20, 2, &m->peer_ids, m->connect_open); + if (ret < 0) + goto free; + memcpy(mac + 2, m->mac, ETH_ALEN); + submitted = true; + ret = mt_request(m, 0xb8, true, true, false, mac, sizeof(mac)); + ret = ret ?: READ_ONCE(m->connect_error); + if (!ret) + ret = mt_request(m, 9, true, true, false, NULL, 0); + if (!ret) + ret = mt_power_init(m); + ret = ret ?: READ_ONCE(m->connect_error); + if (!ret) + ret = mt_request_ext(m, 0xea, 0x40, true, true, false, body, 856); +free: + kfree_sensitive(body); + memzero_explicit(m->connect_pmk, sizeof(m->connect_pmk)); + if (ret) + goto report; + dev_info(&m->pdev->dev, "CONNECT_SUBMITTED: BSSID=%pM channel=%u\n", m->connect_bssid, m->connect_channel); + /* Genuine D7 requests must remain serviceable during asynchronous waits. */ + mutex_unlock(&m->command_mutex); + ret = wait_for_completion_timeout(&m->assoc_start, msecs_to_jiffies(5000)) ? 0 : -ETIMEDOUT; + mutex_lock(&m->command_mutex); + if (ret) { + ret = -ETIMEDOUT; + goto report; + } + ret = READ_ONCE(m->connect_error); + if (ret) + goto report; + ret = mt_association_calibration(m); + ret = ret ?: READ_ONCE(m->connect_error); + if (ret) + goto report; + dev_info(&m->pdev->dev, "ASSOCIATION_CALIBRATED: primary=%u center=%u\n", + m->connect_channel, m->connect_center); + mt7932_peer_queue(queue, m->peer_ids.station[0], true); + ret = mt_request(m, 0x6b, true, true, false, queue, sizeof(queue)); + if (ret) + goto report; + mutex_unlock(&m->command_mutex); + if (!wait_for_completion_timeout(&m->assoc_done, msecs_to_jiffies(10000))) + ret = -ETIMEDOUT; + else + ret = READ_ONCE(m->connect_error); + mutex_lock(&m->command_mutex); +report: + ret = ret ?: READ_ONCE(m->connect_error); + if (!ret && (m->connect_open || m->band2.ht_cap.ht_supported) && + !m->assoc_request_seen) + ret = -EPROTO; + if (ret && !READ_ONCE(m->stopping)) + mt_transport_snapshot(m); + memzero_explicit(m->connect_pmk, sizeof(m->connect_pmk)); + if (ret) { + spin_lock_irqsave(&m->response_lock, flags); + m->disconnecting = true; + spin_unlock_irqrestore(&m->response_lock, flags); + mutex_unlock(&m->command_mutex); + if (submitted && !READ_ONCE(m->stopping)) { + mt_retire_connection(m); + } else if (peer_owned && !submitted) { + mutex_lock(&m->command_mutex); + spin_lock_irqsave(&m->response_lock, flags); + mt_peer_release(m); + spin_unlock_irqrestore(&m->response_lock, flags); + mutex_unlock(&m->command_mutex); + } + mutex_lock(&m->command_mutex); + } + /* Serialize the final outcome with cancellation. In particular, a late + * firmware success must not publish carrier/authorization after .deauth. + * cfg80211 queues these notifications; GFP_ATOMIC keeps this IRQ-safe. + */ + spin_lock_irqsave(&m->response_lock, flags); + if (!ret && m->connect_error) { + ret = m->connect_error; + spin_unlock_irqrestore(&m->response_lock, flags); + goto report; + } + response.status = ret ? WLAN_STATUS_UNSPECIFIED_FAILURE : WLAN_STATUS_SUCCESS; + response.links[0].bss = m->connect_bss; + response.links[0].bssid = m->connect_bssid; + if (m->assoc_request_seen) { + response.req_ie = m->assoc_request_ies; + response.req_ie_len = m->assoc_request_ie_len; + } + m->connect_bss = NULL; /* cfg80211 consumes this reference. */ + m->connecting = false; + if (!ret) { + const struct cfg80211_bss_ies *ies; + const struct element *tim; + + rcu_read_lock(); + ies = rcu_dereference(response.links[0].bss->beacon_ies); + tim = ies ? cfg80211_find_elem(WLAN_EID_TIM, ies->data, ies->len) : NULL; + m->power_tim = tim && tim->datalen >= 4; + rcu_read_unlock(); + spin_lock(&m->data_lock); + m->station_tx_bytes = m->station_rx_bytes = 0; + m->station_tx_packets = m->station_rx_packets = 0; + m->station_connected = jiffies; + m->station_signal_valid = false; + m->station_rx_rate_valid = false; + m->station_tx_rate_valid = false; + spin_unlock(&m->data_lock); + } + m->connected = !ret; + m->disconnecting = false; + if (m->connect_cancelled) { + if (response.links[0].bss) + cfg80211_put_bss(m->wiphy, response.links[0].bss); + cfg80211_disconnected(m->netdev, m->disconnect_reason, NULL, 0, true, GFP_ATOMIC); + } else { + cfg80211_connect_done(m->netdev, &response, GFP_ATOMIC); + } + if (!ret) { + schedule_work(&m->power_work); + if (READ_ONCE(m->data_ready)) { + netif_carrier_on(m->netdev); + netif_wake_queue(m->netdev); + } + if (m->connect_open) { + dev_info(&m->pdev->dev, "OPEN_CONNECTED: owned peer, firmware completion; Ethernet=%u\n", m->data_ready); + } else { + cfg80211_port_authorized(m->netdev, m->connect_bssid, NULL, 0, GFP_ATOMIC); + dev_info(&m->pdev->dev, "WPA2_AUTHORIZED: firmware handshake complete; Ethernet=%u\n", m->data_ready); + } + } else { + dev_err(&m->pdev->dev, "CONNECT_FAILED: %d; recovery-required=%u\n", ret, m->link_failed); + } + spin_unlock_irqrestore(&m->response_lock, flags); + mutex_unlock(&m->command_mutex); +} + +/* Narrow ordinary OPEN policy. Firmware generates association capabilities; + * only the documented iwd common-capability declaration is accepted here. + * The standard supplicant's Ethernet EAPOL control-port declaration is allowed; + * nl80211 control-port transport and key/802.1X offloads are not implemented. + */ +static bool mt_open_request(const struct cfg80211_connect_params *p) +{ + return !p->privacy && !p->crypto.wpa_versions && + !p->crypto.n_ciphers_pairwise && !p->crypto.cipher_group && + !p->crypto.n_akm_suites && !p->crypto.psk && + !p->key && !p->key_len && !p->crypto.sae_pwd && !p->crypto.sae_pwd_len && + p->mfp == NL80211_MFP_NO && mt7932_open_ies(p->ie, p->ie_len) && !p->want_1x && + !p->fils_erp_username && !p->fils_erp_username_len && + !p->fils_erp_realm && !p->fils_erp_realm_len && + !p->fils_erp_rrk && !p->fils_erp_rrk_len && + !p->crypto.control_port_over_nl80211 && + p->crypto.control_port_ethertype == cpu_to_be16(ETH_P_PAE) && + (p->auth_type == NL80211_AUTHTYPE_OPEN_SYSTEM || + p->auth_type == NL80211_AUTHTYPE_AUTOMATIC); +} + +int mt_connect(struct wiphy *wiphy, struct net_device *netdev, + struct cfg80211_connect_params *params) +{ + struct mt7932 *m = mt_from_wiphy(wiphy); + unsigned long flags; + bool open = mt_open_request(params); + int ret = 0; + + if (!mt_rf_allowed(m) || !m->bss_active || netdev != m->netdev) + return -EAGAIN; + dev_info(&m->pdev->dev, "CONNECT_REQUEST: WPA=%u pairwise-count=%d group=%08x AKM-count=%d PMK-present=%u MFP=%u IE-bytes=%zu\n", + params->crypto.wpa_versions, params->crypto.n_ciphers_pairwise, params->crypto.cipher_group, + params->crypto.n_akm_suites, !!params->crypto.psk, params->mfp, params->ie_len); + if (!params->ssid_len || params->ssid_len > 32) + return -EINVAL; + if (!open && (params->crypto.wpa_versions != NL80211_WPA_VERSION_2 || + params->crypto.n_ciphers_pairwise != 1 || params->crypto.ciphers_pairwise[0] != WLAN_CIPHER_SUITE_CCMP || + params->crypto.cipher_group != WLAN_CIPHER_SUITE_CCMP || + params->crypto.n_akm_suites != 1 || params->crypto.akm_suites[0] != WLAN_AKM_SUITE_PSK || + !params->crypto.psk || params->mfp == NL80211_MFP_REQUIRED || + !params->ssid_len || params->ssid_len > 32)) + return -EOPNOTSUPP; + if (params->bssid && !is_valid_ether_addr(params->bssid)) + return -EINVAL; + mutex_lock(&m->command_mutex); + spin_lock_irqsave(&m->response_lock, flags); + if (!mt_rf_allowed(m) || m->link_failed || m->connecting || m->connected || m->disconnecting || m->scan_request || m->retired_scan_seq) { + ret = m->link_failed ? -EIO : -EBUSY; + goto unlock; + } + reinit_completion(&m->assoc_start); + m->disconnect_seen = false; + m->disconnect_event_received = false; + reinit_completion(&m->assoc_done); + reinit_completion(&m->disconnect_done); + m->connect_channel_req = params->channel; + m->connect_channel_hint = params->channel_hint; + eth_zero_addr(m->connect_bssid_req); + eth_zero_addr(m->connect_bssid_hint); + if (params->bssid) + ether_addr_copy(m->connect_bssid_req, params->bssid); + if (params->bssid_hint) + ether_addr_copy(m->connect_bssid_hint, params->bssid_hint); + ether_addr_copy(m->connect_bssid, m->connect_bssid_req); + memcpy(m->connect_ssid, params->ssid, params->ssid_len); + m->connect_ssid_length = params->ssid_len; + m->connect_open = open; + memzero_explicit(m->connect_pmk, sizeof(m->connect_pmk)); + if (!open) + memcpy(m->connect_pmk, params->crypto.psk, 32); + m->connect_error = 0; + m->assoc_request_ie_len = 0; + m->assoc_request_seen = false; + m->connect_center = 0; + memset(&m->connect_chandef, 0, sizeof(m->connect_chandef)); + m->connection_generation++; + m->station_rate_query_failed = false; + m->station_rate_query_time = jiffies - HZ; + m->connect_cancelled = false; + m->peer_valid = false; + m->connecting = true; +unlock: + spin_unlock_irqrestore(&m->response_lock, flags); + if (!ret) { + m->connect_bss = mt_connect_find_bss(m); + schedule_work(&m->connect_work); + } + mutex_unlock(&m->command_mutex); + return ret; +} + +void mt_disconnect_work(struct work_struct *work) +{ + struct mt7932 *m = container_of(work, struct mt7932, disconnect_work); + u16 reason = READ_ONCE(m->disconnect_reason); + bool local = READ_ONCE(m->disconnect_local); + unsigned long flags; + int ret = mt_scan_quiesce(m); + + if (!ret) + ret = mt_retire_connection(m); + else + WRITE_ONCE(m->link_failed, true); + + /* Clear admission and queue its terminal event before a new connect can + * start; both paths are serialized by response_lock. + */ + spin_lock_irqsave(&m->response_lock, flags); + m->disconnecting = false; + cfg80211_disconnected(m->netdev, reason, NULL, 0, local, GFP_ATOMIC); + spin_unlock_irqrestore(&m->response_lock, flags); + dev_info(&m->pdev->dev, "DISCONNECTED: teardown=%d local=%u\n", ret, local); +} + +int mt_disconnect(struct wiphy *wiphy, struct net_device *netdev, u16 reason) +{ + struct mt7932 *m = mt_from_wiphy(wiphy); + unsigned long flags; + + if (netdev != m->netdev) + return -ENODEV; + spin_lock_irqsave(&m->response_lock, flags); + if (m->stopping) { + spin_unlock_irqrestore(&m->response_lock, flags); + return 0; + } + if (m->connecting) { + m->disconnecting = true; + if (!m->connect_error) + m->connect_error = -ECANCELED; + m->connect_cancelled = true; + m->disconnect_reason = reason; + complete(&m->assoc_start); + complete(&m->assoc_done); + complete(&m->discovery_done); + } else if (m->connected) { + m->connected = false; + m->disconnecting = true; + m->disconnect_seen = false; + m->disconnect_event_received = false; + m->disconnect_local = true; + m->disconnect_reason = reason; + netif_carrier_off(m->netdev); + netif_stop_queue(m->netdev); + reinit_completion(&m->disconnect_done); + schedule_work(&m->disconnect_work); + } + spin_unlock_irqrestore(&m->response_lock, flags); + /* iwd may issue its next connect as soon as this command is ACKed. + * Both workers finish through queued cfg80211 notifications, never by + * taking the wiphy mutex held by this callback. Drain them without any + * command/response lock so the ACK cannot release a still-owned epoch. + */ + flush_work(&m->connect_work); + flush_work(&m->disconnect_work); + return 0; +} + +/* iwd uses DEAUTHENTICATE to cancel an accepted fullmac connect before its + * association event. Do not advertise softmac auth/assoc or fabricate an RX + * deauth frame; retire that same firmware owner via the disconnect lifecycle. + */ +int mt_deauth(struct wiphy *wiphy, struct net_device *netdev, + struct cfg80211_deauth_request *request) +{ + struct mt7932 *m = mt_from_wiphy(wiphy); + unsigned long flags; + bool matches; + + if (request->ie_len || request->local_state_change) + return -EOPNOTSUPP; + spin_lock_irqsave(&m->response_lock, flags); + matches = netdev == m->netdev && (m->connecting || m->connected) && + ether_addr_equal(request->bssid, m->connect_bssid); + spin_unlock_irqrestore(&m->response_lock, flags); + if (!matches) + return -ENOTCONN; + return mt_disconnect(wiphy, netdev, request->reason_code); +} + +int mt_change_interface(struct wiphy *wiphy, struct net_device *netdev, + enum nl80211_iftype type, struct vif_params *params) +{ + return type == NL80211_IFTYPE_STATION ? 0 : -EOPNOTSUPP; +} diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c new file mode 100644 index 0000000000000..b422d5acb5b9d --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -0,0 +1,1294 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* J700 firmware-owned Wi-Fi: native transport and cfg80211 station support. + * Clean-room contract: NEO_MT7932_DOWNLOAD_AND_NIC_CAP_CONTRACT.md. + * All addresses here are BAR0 offsets, never chip/CPU physical addresses. + */ +#include "mt7932.h" + +static int mt_ring_alloc(struct mt7932 *m, struct mt7932_ring *q, + unsigned int n, bool rx, unsigned int count); +static void mt_rx_post(struct mt7932_ring *q, unsigned int index); +static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length); +static void mt_rx_drain(struct mt7932 *m, struct mt7932_ring *q); +static irqreturn_t mt_irq(int irq, void *data); +static void mt_service(struct mt7932 *m); +static irqreturn_t mt_irq_thread(int irq, void *data); +static int mt_submit(struct mt7932 *m, struct mt7932_ring *q, size_t length); +static int mt_command(struct mt7932 *m, u8 cid, const void *payload, size_t length); +static int mt_ownership(struct mt7932 *m); +static int mt_dma_setup(struct mt7932 *m); +static int mt_rom_gate(struct mt7932 *m); +static int mt_status(struct mt7932 *m, u8 cid, const void *payload, size_t length); +static int mt_scatter(struct mt7932 *m, const u8 *data, size_t length); +static int mt_download(struct mt7932 *m, const u8 *data, struct mt7932_region *r, + bool ram); +static int mt_secure_acquire(struct mt7932 *m); +static void mt_secure_release(struct mt7932 *m); +static int mt_otp_byte(struct mt7932 *m, u32 address, u8 *value); +static int mt_capability_gate(struct mt7932 *m); +static int mt_config_gate(struct mt7932 *m); +static int mt_board_gate(struct mt7932 *m); +static int mt_precal_gate(struct mt7932 *m); +static int mt_runtime_start(struct mt7932 *m); +static int mt_firmware_gate(struct mt7932 *m); +static int mt_initialize(struct mt7932 *m); +static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id); +static void mt_shutdown(struct pci_dev *pdev); + +static bool legacy_irq_test; +module_param(legacy_irq_test, bool, 0444); +MODULE_PARM_DESC(legacy_irq_test, "Explicit diagnostic use of standard PCI INTx instead of MSI"); +static bool poll_test; +module_param(poll_test, bool, 0444); +MODULE_PARM_DESC(poll_test, "Diagnostic ROM/firmware polling with endpoint IRQ generation masked"); +static bool reset_test; +module_param(reset_test, bool, 0444); +MODULE_PARM_DESC(reset_test, "Qualify one FLR and fresh firmware epoch before publishing an interface, retaining old DMA"); + +bool mt_transport_polled(void) +{ + return poll_test; +} + + +bool mt_rf_allowed(struct mt7932 *m) +{ + return !READ_ONCE(m->stopping) && READ_ONCE(m->rf_ready) && !READ_ONCE(m->reg_pending) && + !READ_ONCE(m->policy_failed); +} + +u32 mt_read(struct mt7932 *m, u32 reg) +{ + return readl(m->bar + reg); +} + +void mt_write(struct mt7932 *m, u32 reg, u32 value) +{ + writel(value, m->bar + reg); +} + +void mt_rmw(struct mt7932 *m, u32 reg, u32 mask, u32 value) +{ + mt_write(m, reg, (mt_read(m, reg) & ~mask) | value); +} + +int mt_poll(struct mt7932 *m, u32 reg, u32 mask, u32 expected, u32 timeout) +{ + u32 value; + int ret = readl_poll_timeout(m->bar + reg, value, + value == U32_MAX || (value & mask) == expected, + 1000, timeout); + if (ret || value == U32_MAX) { + dev_err(&m->pdev->dev, "poll %x got %08x wanted %08x/%08x\n", + reg, value, expected, mask); + return ret ?: -ENODEV; + } + return 0; +} + +void *mt_alloc(struct mt7932 *m, size_t size, dma_addr_t *dma) +{ + void *p = dmam_alloc_coherent(&m->pdev->dev, size, dma, GFP_KERNEL); + + if (!p) + return NULL; + if (*dma >= BIT(27) || size > BIT(27) - *dma) { + dev_err(&m->pdev->dev, "DMA allocation outside qualified 27-bit aperture\n"); + return NULL; + } + return p; +} + +static int mt_ring_alloc(struct mt7932 *m, struct mt7932_ring *q, + unsigned int n, bool rx, unsigned int count) +{ + unsigned int i; + + q->count = count; + q->reg = W + (rx ? 0x500 : 0x300) + 16 * n; + q->desc = mt_alloc(m, count * sizeof(*q->desc), &q->dma); + if (!q->desc) + return -ENOMEM; + for (i = 0; i < count; i++) + q->desc[i].control = cpu_to_le32(DONE); + q->slots = rx ? count : n == 16 ? 640 : n == 17 ? 1024 : 0; + if (q->slots) { + q->buffers = mt_alloc(m, q->slots * (rx ? RX_STRIDE : STRIDE), + &q->buffers_dma); + if (!q->buffers) + return -ENOMEM; + } + return 0; +} + +static void mt_rx_post(struct mt7932_ring *q, unsigned int index) +{ + struct mt7932_desc *d = &q->desc[index]; + + d->address = cpu_to_le32(q->buffers_dma + index * RX_STRIDE); + d->second = 0; + d->info = 0; + d->control = cpu_to_le32(MT7932_RX_CAPACITY << 16); +} + +static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) +{ + struct mt7932_event event; + unsigned long flags; + int ret; + + if (length >= 4 && (get_unaligned_le32(packet) >> 27) == 6) { + mt_data_complete(m, packet, length); + return; + } + ret = mt7932_event_parse(packet, length, &event); + + m->packets++; + if (ret) { + if (ret == -ENOMSG) { + mt_packet_receive(m, packet, length); + return; + } + dev_dbg_ratelimited(&m->pdev->dev, "RX framing result %d len %zu word %08x\n", + ret, length, length >= 4 ? get_unaligned_le32(packet) : 0); + return; + } + spin_lock_irqsave(&m->response_lock, flags); + if (event.eid == 0xee) { + mt_link_event(m, &event); + goto unlock; + } + if (event.eid == 0x0d) { + mt_scan_event(m, &event); + goto unlock; + } + if (event.eid == 0x11) { + mt_bss_presence(m, &event); + goto unlock; + } + if (event.eid == 0xd6) { + ret = mt7932_cal_null(&m->cal_state, packet + 36, + event.length >= 36 ? event.length - 36 : 0, + m->smart_version); + dev_info(&m->pdev->dev, "CAL_EVENT: seq=%u length=%zu result=%d credits=%u/%u\n", + event.seq, event.length, ret, m->cal_state.received, m->cal_state.expected); + if (ret) + complete(&m->cal_response); + goto unlock; + } + if (event.eid == 0xd7) { + /* The terminal peer gate and publication share response_lock. Join + * admitted calibration work, but never enqueue another producer + * behind retirement's final join. D6/owned responses still drain. + * An unexpected new request makes ownership uncertain: retain it + * for checked reset, rather than silently declaring retirement safe. + */ + if (m->disconnecting || m->link_failed) { + m->cal_state.error = -EPROTO; + m->link_failed = true; + complete(&m->cal_response); + goto unlock; + } + if (event.length < 52 || m->cal_request_count == ARRAY_SIZE(m->cal_requests)) { + m->cal_state.error = -EPROTO; + complete(&m->cal_response); + } else { + unsigned int at = (m->cal_request_head + m->cal_request_count) % ARRAY_SIZE(m->cal_requests); + + memcpy(m->cal_requests[at], packet + 36, 16); + m->cal_request_seq[at] = event.seq; + m->cal_request_count++; + dev_info(&m->pdev->dev, "CAL_REQUEST: seq=%u words=%08x/%08x/%08x/%08x queued=%u\n", + event.seq, get_unaligned_le32(packet + 36), get_unaligned_le32(packet + 40), + get_unaligned_le32(packet + 44), get_unaligned_le32(packet + 48), m->cal_request_count); + if (READ_ONCE(m->rf_ready)) + schedule_work(&m->cal_work); + } + goto unlock; + } + if (mt7932_event_matches(&event, m->waiting) && !m->reply_length) { + memcpy(m->reply, packet, event.length); + m->reply_length = event.length; + complete(&m->response); + } else { + dev_dbg_ratelimited(&m->pdev->dev, "event eid=%02x seq=%u option=%02x len=%zu\n", + event.eid, event.seq, event.option, event.length); + } +unlock: + spin_unlock_irqrestore(&m->response_lock, flags); +} + +static void mt_rx_drain(struct mt7932 *m, struct mt7932_ring *q) +{ + unsigned int budget = q->count; + + while (budget--) { + u32 ctrl = le32_to_cpu(READ_ONCE(q->desc[q->tail].control)); + u32 length; + + if (!(ctrl & DONE)) + break; + dma_rmb(); + length = (ctrl >> 16) & 0x3fff; + if (length <= MT7932_RX_CAPACITY && (ctrl & BIT(30))) + mt_receive(m, q->buffers + q->tail * RX_STRIDE, length); + else + dev_err_ratelimited(&m->pdev->dev, "RX length overflow %u\n", length); + if (READ_ONCE(m->ram_config) && length >= 33 && length <= MT7932_RX_CAPACITY) { + u8 *packet = q->buffers + q->tail * RX_STRIDE; + + /* Source-specific RAM-config acknowledgment before RX0 refill. */ + if (q == &m->rx[0] && packet[28] == 1 && packet[29] == m->waiting && + !packet[30] && packet[32] == 1) { + mt_write(m, W + 0x204, 0); + mt_write(m, W + 0x200, 0x0c000001); + mt_write(m, W + 0x204, poll_test ? 0 : IRQ_MASK); + mt_write(m, W + 0x1f4, 0xffff); + } + } + /* Runtime owns all N slots; initial download reserves one empty. */ + mt_rx_post(q, m->runtime_epoch ? q->tail : q->head); + q->head = (q->head + 1) % q->count; + q->tail = (q->tail + 1) % q->count; + dma_wmb(); + mt_write(m, q->reg + 8, + m->runtime_epoch ? (q->head + q->count - 1) % q->count : q->head); + } +} + +static irqreturn_t mt_irq(int irq, void *data) +{ + struct mt7932 *m = data; + + if (!READ_ONCE(m->running)) + return IRQ_NONE; + if (!(mt_read(m, W + 0x200) & m->irq_mask)) + return IRQ_NONE; + mt_write(m, W + 0x204, 0); + return IRQ_WAKE_THREAD; +} + +static void mt_service(struct mt7932 *m) +{ + u32 causes = mt_read(m, W + 0x200) & m->irq_mask; + unsigned int i; + + mt_data_clean(m); + for (i = 0; i < ARRAY_SIZE(m->rx); i++) + if (m->rx[i].count) + mt_rx_drain(m, &m->rx[i]); + if (causes & BIT(29)) { + u32 sw = mt_read(m, W + 0x1f0); + + mt_write(m, W + 0x1f0, sw); + } + mt_write(m, W + 0x200, causes); + if (READ_ONCE(m->running) && !poll_test) + mt_write(m, W + 0x204, m->irq_mask); +} + +static irqreturn_t mt_irq_thread(int irq, void *data) +{ + struct mt7932 *m = data; + + m->interrupts++; + mt_service(m); + return IRQ_HANDLED; +} + +void mt_tx_clean(struct mt7932_ring *q) +{ + while (q->queued && (le32_to_cpu(READ_ONCE(q->desc[q->tail].control)) & DONE)) { + dma_rmb(); + q->tail = (q->tail + 1) % q->count; + q->queued--; + } +} + +static int mt_submit(struct mt7932 *m, struct mt7932_ring *q, size_t length) +{ + struct mt7932_desc *d; + unsigned int retry; + + if (q->used >= q->slots || length > STRIDE || !length) + return -ENOSPC; + for (retry = 0; retry < 50; retry++) { + mt_tx_clean(q); + if (q->queued < q->count - 1) + break; + usleep_range(100, 200); + } + if (q->queued == q->count - 1) + return -EBUSY; + d = &q->desc[q->head]; + d->address = cpu_to_le32(q->buffers_dma + q->used * STRIDE); + d->second = 0; + d->info = 0; + dma_wmb(); + d->control = cpu_to_le32((length << 16) | BIT(30)); + q->used++; + q->head = (q->head + 1) % q->count; + q->queued++; + dma_wmb(); + mt_write(m, q->reg + 8, q->head); + return 0; +} + +/* One caller, one outstanding command, no retry after uncertain completion. */ +int mt_request_ext(struct mt7932 *m, u8 cid, u8 ext, bool runtime, bool set, bool wait, + const void *payload, size_t length) +{ + struct mt7932_ring *q = &m->tx[17]; + unsigned long flags; + int ret; + + if (READ_ONCE(m->stopping)) + return -ESHUTDOWN; + if (q->used >= q->slots) { + /* Command buffers can wrap only after every published descriptor has + * retired. Never reclaim by elapsed time or by a reply alone. + */ + mt_tx_clean(q); + if (q->queued) + return -ENOSPC; + q->used = 0; + } + if (!++m->seq) + m->seq = 1; + ret = mt7932_envelope(q->buffers + q->used * STRIDE, STRIDE, + cid, ext, m->seq, runtime, set, wait, payload, length); + if (ret < 0) + return ret; + reinit_completion(&m->response); + spin_lock_irqsave(&m->response_lock, flags); + m->waiting = wait ? m->seq : 0; + m->reply_length = 0; + spin_unlock_irqrestore(&m->response_lock, flags); + ret = mt_submit(m, q, ret); + if (!ret && wait) { + if (poll_test) { + unsigned long deadline = jiffies + msecs_to_jiffies(5000); + + do { + mt_service(m); + if (completion_done(&m->response)) + break; + usleep_range(100, 200); + } while (time_before(jiffies, deadline)); + if (!completion_done(&m->response)) + ret = -ETIMEDOUT; + } else if (!wait_for_completion_timeout(&m->response, msecs_to_jiffies(5000))) { + ret = -ETIMEDOUT; + } + } + if (READ_ONCE(m->stopping)) + ret = -ESHUTDOWN; + if (ret == -ETIMEDOUT) { + /* Diagnostic only: never turn a polled late reply into IRQ success. */ + unsigned int i; + u16 command; + u32 mask, pending; + unsigned int cap = pci_find_capability(m->pdev, PCI_CAP_ID_MSI); + + pci_read_config_word(m->pdev, PCI_COMMAND, &command); + pci_read_config_dword(m->pdev, cap + PCI_MSI_MASK_64, &mask); + pci_read_config_dword(m->pdev, cap + PCI_MSI_PENDING_64, &pending); + dev_info(&m->pdev->dev, "timeout PCI command=%04x MSI mask=%08x pending=%08x\n", + command, mask, pending); + + dev_info(&m->pdev->dev, "timeout gates MAC=%08x WFDMA=%08x SW=%08x\n", + mt_read(m, 0x10188), mt_read(m, W + 0x204), mt_read(m, W + 0x1f4)); + disable_irq(pci_irq_vector(m->pdev, 0)); + for (i = 0; i < ARRAY_SIZE(m->rx); i++) { + struct mt7932_ring *rx = &m->rx[i]; + + if (!rx->count) + continue; + dev_info(&m->pdev->dev, "timeout RX%u ctrl=%08x CPU=%u DMA=%u\n", i, + le32_to_cpu(rx->desc[rx->tail].control), + mt_read(m, rx->reg + 8), mt_read(m, rx->reg + 12)); + if (le32_to_cpu(rx->desc[rx->tail].control) & DONE) { + dma_rmb(); + print_hex_dump(KERN_INFO, "ROM RX prefix: ", DUMP_PREFIX_OFFSET, + 16, 1, rx->buffers + rx->tail * RX_STRIDE, 40, false); + } + mt_rx_drain(m, rx); + } + dev_info(&m->pdev->dev, "late polled reply length=%zu eid=%02x (not IRQ success)\n", + m->reply_length, m->reply_length > 28 ? m->reply[28] : 0); + enable_irq(pci_irq_vector(m->pdev, 0)); + } + spin_lock_irqsave(&m->response_lock, flags); + m->waiting = 0; + spin_unlock_irqrestore(&m->response_lock, flags); + if (ret) + dev_err(&m->pdev->dev, + "command %02x seq %u failed %d IRQ=%llu RX=%llu status=%08x TX=%u/%u\n", + cid, m->seq, ret, m->interrupts, m->packets, mt_read(m, W + 0x200), + mt_read(m, q->reg + 8), mt_read(m, q->reg + 12)); + return ret; +} + +int mt_request(struct mt7932 *m, u8 cid, bool runtime, bool set, bool wait, + const void *payload, size_t length) +{ + return mt_request_ext(m, cid, 0, runtime, set, wait, payload, length); +} + +static int mt_command(struct mt7932 *m, u8 cid, const void *payload, size_t length) +{ + return mt_request(m, cid, false, false, true, payload, length); +} + +static int mt_ownership(struct mt7932 *m) +{ + unsigned int attempt; + int ret = -ETIMEDOUT; + + mt_write(m, 0xe0018, 1); + for (attempt = 0; attempt < 10; attempt++) { + mt_write(m, 0xe0010, 2); + ret = mt_poll(m, 0xe0010, BIT(2), 0, 50000); + if (!ret || ret == -ENODEV) + break; + } + if (ret) + return ret; + mt_write(m, 0xfe250, 0x70001846); + udelay(2); + if ((mt_read(m, 0xfe250) >> 16) != 0x7000) + return -EIO; + dev_info(&m->pdev->dev, "DRIVER_OWN_OK power=%08x\n", mt_read(m, 0xe00f0)); + return 0; +} + +static int mt_dma_setup(struct mt7932 *m) +{ + static const u16 rx_counts[] = {32, 0, 512, 32, 32, 224, 128, 512, 64}; + static const u16 rx_prefetch[] = {0, 0, 0x40, 0xc0, 0x100, 0x140, 0x180, 0x1c0, 0x200}; + static const u16 tx_prefetch[] = {0x240, 0x280, 0x2c0, 0x300, 0x340, 0x400, + 0x440, 0x480, 0x4c0, 0x580, 0x5c0, 0x600, 0x640, 0x700, 0x740, + 0, 0x780, 0x7c0, 0x800}; + unsigned int i, j; + int ret; + u32 glo; + + mt_write(m, W + 0x204, 0); + mt_write(m, 0x10188, poll_test ? 0 : 0xff); + mt_rmw(m, W + 0x208, BIT(0) | BIT(2) | BIT(15) | BIT(21) | BIT(27) | BIT(28), 0); + ret = mt_poll(m, W + 0x208, BIT(1) | BIT(3), 0, 100000); + if (ret) + return ret; + mt_rmw(m, W + 0x2b0, BIT(6), 0); + if (mt_read(m, H + 4) & BIT(28)) + return -EINVAL; + mt_rmw(m, W + 0x100, BIT(4) | BIT(5), 0); + mt_rmw(m, W + 0x100, 0, BIT(4) | BIT(5)); + mt_rmw(m, H + 0x1c, 0x0fff0fff, 1); + mt_rmw(m, H + 0x10, 0xffff0000, 0x80000000); + for (i = 0; i < 16; i++) { + u32 quota = i < 4 ? 0x02000028 : i == 15 ? 0 : + !(i % 4) ? 0x01000014 : 0x01000028; + + mt_write(m, H + 0x20 + 4 * i, quota); + if ((mt_read(m, H + 0x20 + 4 * i) & 0x0fff0fff) != quota) + return -EIO; + } + mt_write(m, H + 0x60, 0x76543210); + mt_write(m, H + 0x64, 0xfedcba98); + mt_write(m, H + 0x68, 0x11111000); + mt_write(m, H + 0x6c, 0x11111000); + mt_write(m, H + 0x70, 0x76543210); + mt_write(m, H + 0x74, 0xfedcba98); + mt_rmw(m, H + 0x0c, 3 << 16, BIT(16)); + mt_rmw(m, H + 4, BIT(28), 0); + for (i = 0; i < ARRAY_SIZE(m->tx); i++) { + if (i == 15) + continue; + ret = mt_ring_alloc(m, &m->tx[i], i, false, i < 15 ? 512 : i == 16 ? 256 : i == 17 ? 24 : 16); + if (ret) + return ret; + } + for (i = 0; i < ARRAY_SIZE(m->rx); i++) { + if (!rx_counts[i]) + continue; + ret = mt_ring_alloc(m, &m->rx[i], i, true, rx_counts[i]); + if (ret) + return ret; + } + m->ipc = mt_alloc(m, 0x300, &m->ipc_dma); + m->aux = mt_alloc(m, 0x320, &m->aux_dma); + if (!m->ipc || !m->aux) + return -ENOMEM; + for (i = 0; i < ARRAY_SIZE(m->tx); i++) { + struct mt7932_ring *q = &m->tx[i]; + + if (!q->count) + continue; + mt_write(m, q->reg, q->dma); + mt_write(m, q->reg + 8, 0); + mt_write(m, q->reg + 4, q->count); + mt_write(m, W + 0x600 + 4 * i, (tx_prefetch[i] << 16) | (i == 4 || i == 8 || i == 12 ? 12 : 4)); + } + for (i = 0; i < ARRAY_SIZE(m->rx); i++) { + struct mt7932_ring *q = &m->rx[i]; + + if (!q->count) + continue; + mt_write(m, q->reg, q->dma); + mt_write(m, q->reg + 8, 0); + mt_write(m, q->reg + 4, q->count); + for (j = 0; j < q->count - 1; j++) + mt_rx_post(q, j); + q->head = q->count - 1; + dma_wmb(); + mt_write(m, q->reg + 8, q->head); + mt_write(m, W + 0x680 + 4 * i, (rx_prefetch[i] << 16) | (i == 2 ? 8 : 4)); + } + mt_write(m, W + 0x20c, U32_MAX); + glo = (mt_read(m, W + 0x208) & ~(GENMASK(5, 4) | BIT(11) | BIT(13) | BIT(20))) | + BIT(6) | BIT(12) | BIT(15) | BIT(21) | BIT(28) | BIT(30); + mt_write(m, W + 0x208, glo); + mt_rmw(m, W + 0x2b0, 0, BIT(6)); + /* All endpoint-visible storage and interrupt state now have an owner. */ + dma_wmb(); + WRITE_ONCE(m->dma_owned, true); + pci_set_master(m->pdev); + WRITE_ONCE(m->running, true); + mt_rmw(m, W + 0x208, 0, BIT(0) | BIT(2)); + mt_write(m, W + 0x298, 0xc); + mt_write(m, C + 0x38, 0x13); + mt_write(m, W + 0x2f0, 0x8032800a); + mt_rmw(m, 0x2120, 0, BIT(1)); + mt_write(m, W + 0x1f4, 0xffff); + mt_write(m, W + 0x204, poll_test ? 0 : IRQ_MASK); + dev_info(&m->pdev->dev, "DMA_PUBLISHED glo=%08x TX17=%pad RX0=%pad IPC=%pad\n", + mt_read(m, W + 0x208), &m->tx[17].dma, &m->rx[0].dma, &m->ipc_dma); + return 0; +} + +static int mt_rom_gate(struct mt7932 *m) +{ + static const u32 addresses[] = {0x70010204, 0x88000004}; + u8 request[12] = {}; + unsigned int i; + int ret = mt_poll(m, 0xe00f0, 7, 1, 1000000); + + if (ret) + return ret; + mt_write(m, 0xfe254, 0x18051848); + udelay(2); + if ((mt_read(m, 0xfe254) >> 16) != 0x1805) + return -EIO; + dma_wmb(); + mt_write(m, 0x93c28, 1); + mt_write(m, 0x93a38, 2); + mt_write(m, 0x93a3c, 0x300); + mt_write(m, 0x93a30, m->ipc_dma); + mt_write(m, 0x93a54, m->aux_dma); + if (mt_read(m, 0x93a30) != m->ipc_dma || mt_read(m, 0x93a54) != m->aux_dma) + return -EIO; + for (i = 0; i < ARRAY_SIZE(addresses); i++) { + put_unaligned_le32(addresses[i], request + 4); + ret = mt_command(m, 3, request, sizeof(request)); + if (ret) + return ret; + if (m->reply_length < 40 || m->reply[28] != 2 || + get_unaligned_le32(m->reply + 32) != addresses[i]) + return -EPROTO; + dev_info(&m->pdev->dev, "ROM_READ_OK %08x=%08x seq=%u IRQ=%llu\n", + addresses[i], get_unaligned_le32(m->reply + 36), m->seq, m->interrupts); + } + dev_info(&m->pdev->dev, "ROM_TRANSPORT_OK mode=%s IRQ=%llu (no firmware download or RF yet)\n", + poll_test ? "polled-test" : "interrupt", m->interrupts); + return 0; +} + +static int mt_status(struct mt7932 *m, u8 cid, const void *payload, size_t length) +{ + int ret = mt_command(m, cid, payload, length); + + if (ret) + return ret; + if (m->reply_length < 33 || m->reply[28] != 1 || m->reply[32] != 1) { + dev_err(&m->pdev->dev, "boot status CID=%02x EID=%02x status=%u len=%zu\n", + cid, m->reply[28], m->reply[32], m->reply_length); + return -EPROTO; + } + return 0; +} + +static int mt_scatter(struct mt7932 *m, const u8 *data, size_t length) +{ + struct mt7932_ring *q = &m->tx[16]; + + while (length) { + size_t bytes = min_t(size_t, length, 2048); + int ret; + + if (q->used >= q->slots) + return -ENOSPC; + memcpy(q->buffers + q->used * STRIDE, data, bytes); + ret = mt_submit(m, q, bytes); + if (ret) + return ret; + mt_tx_clean(q); + data += bytes; + length -= bytes; + } + return 0; +} + +static int mt_download(struct mt7932 *m, const u8 *data, struct mt7932_region *r, + bool ram) +{ + u8 request[12]; + u8 cid = r->target == 0x900000 || r->target == 0x200000 ? 5 : 1; + int ret; + + put_unaligned_le32(r->target, request); + put_unaligned_le32(r->length, request + 4); + put_unaligned_le32(r->mode, request + 8); + WRITE_ONCE(m->ram_config, ram); + ret = mt_status(m, cid, request, sizeof(request)); + WRITE_ONCE(m->ram_config, false); + if (ret) + return ret; + ret = mt_scatter(m, data + r->offset, r->length); + dev_info(&m->pdev->dev, "%s region target=%08x length=%u scatter=%d\n", + ram ? "RAM" : "PATCH", r->target, r->length, ret); + return ret; +} + +static int mt_secure_acquire(struct mt7932 *m) +{ + int ret; + + mt_write(m, 0xfe24c, 0x18451807); + ret = mt_poll(m, 0x40060, 1, 1, 5000000); + if (!ret) + mt_write(m, 0xfe24c, 0x1845184f); + return ret; +} + +static void mt_secure_release(struct mt7932 *m) +{ + mt_write(m, 0xfe24c, 0x18451807); + mt_write(m, 0x40260, 1); + mt_write(m, 0xfe24c, 0x1845184f); +} + +static int mt_otp_byte(struct mt7932 *m, u32 address, u8 *value) +{ + u8 request[4]; + int ret; + + put_unaligned_le32(address, request); + ret = mt_command(m, 0x50, request, sizeof(request)); + if (ret) + return ret; + if (m->reply_length < 40) + return -EPROTO; + *value = m->reply[36]; + return 0; +} + +static int mt_capability_gate(struct mt7932 *m) +{ + u8 entropy[68] = {}; + const u8 *body; + size_t offset = 4; + int ret, count, i; + bool mac = false, phy = false; + + get_random_bytes(entropy + 4, 64); + ret = mt_request(m, 0xe2, false, false, false, entropy, sizeof(entropy)); + memzero_explicit(entropy, sizeof(entropy)); + if (ret) + return ret; + ret = mt_request(m, 0, false, false, false, NULL, 0); + if (ret) + return ret; + ret = mt_request(m, 0x8a, true, false, true, NULL, 0); + if (ret) + return ret; + if (m->reply_length < 40) + return -EPROTO; + body = m->reply + 36; + if (m->reply[28] != 0xec) + return -EPROTO; + count = mt7932_capabilities(body, m->reply_length - 36); + if (count < 0) { + dev_err(&m->pdev->dev, "invalid capability reply len=%zu parse=%d\n", m->reply_length, count); + print_hex_dump(KERN_INFO, "NIC_CAP_REJECTED: ", DUMP_PREFIX_OFFSET, + 16, 1, m->reply, m->reply_length, false); + return count; + } + for (i = 0; i < count; i++) { + u32 type = get_unaligned_le32(body + offset); + u32 length = get_unaligned_le32(body + offset + 4); + const u8 *value = body + offset + 8; + + dev_info(&m->pdev->dev, "capability TLV type=%x bytes=%u\n", type, length); + if (type == 7) { + dev_info(&m->pdev->dev, "firmware MAC %pM\n", value); + mac = true; + } else if (type == 8) { + if (!value[4] || value[4] > 4) + return -EOPNOTSUPP; + memcpy(m->phy_cap, value, 12); + m->antenna_mask = (1U << value[4]) - 1; + dev_info(&m->pdev->dev, "firmware PHY NSS=%u paths=%02x HT=%u VHT=%u HE=%u\n", + value[4], value[10], value[0], value[1], value[11]); + phy = true; + } else if (type == 0x38 && length) { + m->runtime_cal = value[0]; + m->runtime_cal_valid = true; + dev_info(&m->pdev->dev, "RUNTIME_CAL_CAPABILITY: %u\n", value[0]); + } else if (type == 0x18 && length >= 4) { + m->six_ghz = value[0]; + m->six_ghz_valid = value[0] <= 1; + } else if (type == 0x34 && length >= 2) { + m->stof_supported = value[0] || value[1]; + } + offset += 8 + length; + } + if (!mac || !phy) + return -ENODATA; + print_hex_dump(KERN_INFO, "NIC_CAP: ", DUMP_PREFIX_OFFSET, 16, 1, + m->reply, m->reply_length, false); + dev_info(&m->pdev->dev, "MCU_CAPABILITY_OK eid=%02x seq=%u (no calibration or RF yet)\n", + m->reply[28], m->seq); + return 0; +} + +static int mt_config_gate(struct mt7932 *m) +{ + u8 basic[24], mlme[4]; + unsigned long deadline; + int ret; + + mt7932_basic_config(basic); + mt7932_mlme_config(mlme); + ret = mt_request(m, 2, true, true, false, basic, sizeof(basic)); + if (ret) + return ret; + ret = mt_request_ext(m, 0xea, 0x4e, true, true, false, mlme, sizeof(mlme)); + if (ret) + return ret; + deadline = jiffies + msecs_to_jiffies(500); + do { + mt_tx_clean(&m->tx[17]); + if (!m->tx[17].queued) { + dev_info(&m->pdev->dev, "BASIC_MLME_SUBMITTED: TX retired, no semantic ACK or RF startup\n"); + return 0; + } + usleep_range(100, 200); + } while (time_before(jiffies, deadline)); + return -ETIMEDOUT; +} + + +static int mt_board_gate(struct mt7932 *m) +{ + const u32 addresses[] = {0x70, 0x160}; + u8 request[24] = {}, module = 0, board_byte = 0; + u32 status[2]; + unsigned int i, candidate; + int ret; + + if (!m->runtime_epoch) + return -EINVAL; + ret = mt_request_ext(m, 0xed, 0xc1, true, false, true, NULL, 0); + if (ret) + return ret; + if (m->reply_length < 44 || m->reply[28] != 0xed || m->reply[32] != 0xc1) + return -EPROTO; + dev_info(&m->pdev->dev, "CHIP_UID_QUERY_OK: 8 opaque bytes received, not published\n"); + for (i = 0; i < ARRAY_SIZE(addresses); i++) { + put_unaligned_le32(addresses[i], request); + ret = mt_request_ext(m, 0xed, 1, true, false, true, request, sizeof(request)); + if (ret) + return ret; + if (m->reply_length < 60 || m->reply[28] != 0xed || m->reply[32] != 1 || + get_unaligned_le32(m->reply + 36) != addresses[i]) + return -EPROTO; + status[i] = get_unaligned_le32(m->reply + 40); + if (!i) + module = m->reply[54]; + else + board_byte = m->reply[48]; + } + candidate = !(module & 15) ? 2 : !board_byte ? 3 : 1; + m->board_type = candidate; + m->module_byte = module; + dev_info(&m->pdev->dev, "BOARD_CLASSIFICATION_CAPTURED: candidate=%u module=%02x board-byte=%02x status=%08x/%08x; no buffer branch selected\n", + candidate, module, board_byte, status[0], status[1]); + return 0; +} + +static int mt_precal_gate(struct mt7932 *m) +{ + const struct firmware *ppr; + u8 request[416] = {2, 0, 0x9c, 1}; + u32 status; + int ret; + + if (!m->runtime_epoch || m->board_type != 3 || !m->runtime_cal_valid) + return -EINVAL; + /* Stock EfuseBufferModeCal=3, own eFuse board3: original PPR source2. + * This is not the external EEPROM/source0 or own-WCAL/source3 path. + */ + ret = request_firmware_direct(&ppr, "mediatek/mt7932/ppr.bin", &m->pdev->dev); + if (ret) + return ret; + if (ppr->size != sizeof(request) - 4) { + release_firmware(ppr); + return -EINVAL; + } + memcpy(request + 4, ppr->data, ppr->size); + release_firmware(ppr); + ret = mt_request_ext(m, 0xed, 0x21, true, false, true, request, sizeof(request)); + if (ret) + return ret; + if (m->reply_length < 44 || m->reply[28] != 0xed || m->reply[32]) + return -EPROTO; + status = get_unaligned_le32(m->reply + 40); + dev_info(&m->pdev->dev, "PPR_SOURCE2_RESULT: status=%08x\n", status); + if (status) + return -EREMOTEIO; + memset(request, 0, sizeof(request)); + ret = mt_request_ext(m, 0xed, 0xa9, true, false, true, request, 4); + if (ret) + return ret; + if (m->reply_length < 40 || m->reply[28] != 0xed || m->reply[32] != 0xaa) + return -EPROTO; + m->smart_version = m->reply[38] & 15; + dev_info(&m->pdev->dev, "SMART_VERSION_REPLY: eid=%02x ext=%02x version=%u\n", + m->reply[28], m->reply[32], m->smart_version); + request[3] = 9; + ret = mt_request(m, 0xbf, true, false, true, request, 20); + if (ret) + return ret; + if (m->reply_length < 56) + return -EPROTO; + dev_info(&m->pdev->dev, "CALTYPE_REPLY: eid=%02x ext=%02x H=%u P=%u O=%u V=%u marker=%08x\n", + m->reply[28], m->reply[32], m->runtime_cal, m->reply[37], + m->reply[44], m->reply[38], get_unaligned_le32(m->reply + 48)); + /* M=0/T=3 exact host predicate. Stage metadata only after acceptance; + * do not select the legacy absent-marker D6 compatibility fallback. + */ + if (get_unaligned_le32(m->reply + 48) != 0xbf || m->runtime_cal || + m->reply[37] != 1 || m->reply[44] != 1) + return -EPROTO; + m->preload_version = m->reply[38]; + dev_info(&m->pdev->dev, "PRECAL_HANDSHAKE_OK: own calibration and RF policy still required\n"); + return 0; +} + +static int mt_runtime_start(struct mt7932 *m) +{ + int ret; + + ret = mt_capability_gate(m); + if (!ret) + ret = mt_config_gate(m); + if (!ret) + ret = mt_recovery_gate(m); + if (!ret) + ret = mt_board_gate(m); + if (!ret) + ret = mt_precal_gate(m); + + return ret; +} + +static int mt_firmware_gate(struct mt7932 *m) +{ + const struct firmware *patch, *ram; + struct mt7932_region region; + u8 request[8] = {}, value; + u32 entry = 0; + int ret, patches, regions, i; + + ret = request_firmware_direct(&patch, "mediatek/mt7932/IZUBA_WIFI_MT7932_patch_mcu_1_2_hdr.bin", &m->pdev->dev); + if (ret) + return ret; + ret = request_firmware_direct(&ram, "mediatek/mt7932/IZUBA_W7932_2.bin", &m->pdev->dev); + if (ret) + goto out_patch; + /* Validate all regions before emitting the first download command. */ + patches = mt7932_patch_region(patch->data, patch->size, 0, ®ion); + regions = mt7932_ram_region(ram->data, ram->size, 0, ®ion); + ret = -EINVAL; + if (patches < 0 || regions < 0) + goto out; + for (i = 0; i < patches; i++) + if (mt7932_patch_region(patch->data, patch->size, i, ®ion) < 0) + goto out; + for (i = 0; i < regions; i++) + if (mt7932_ram_region(ram->data, ram->size, i, ®ion) < 0) + goto out; + dev_info(&m->pdev->dev, "stock containers patch=%zu/%d regions RAM=%zu/%d regions\n", + patch->size, patches, ram->size, regions); + put_unaligned_le32(2, request); + ret = mt_command(m, 0x10, request, 4); + if (ret) + goto out; + if (m->reply_length < 33 || m->reply[32] < 1 || m->reply[32] > 3) { + ret = -EPROTO; + goto out; + } + if (m->reply[32] != 1) { + for (i = 0; i < patches; i++) { + mt7932_patch_region(patch->data, patch->size, i, ®ion); + ret = mt_download(m, patch->data, ®ion, false); + if (ret) + goto out; + } + ret = mt_secure_acquire(m); + if (ret) + goto out; + memset(request, 0, sizeof(request)); + ret = mt_status(m, 7, request, 4); + if (ret) + goto out; + mt_secure_release(m); + } + dev_info(&m->pdev->dev, "PATCH_READY\n"); + ret = mt_otp_byte(m, 0x7a, &value); + if (!ret) + ret = mt_otp_byte(m, 0x164, &value); + if (!ret) + ret = mt_otp_byte(m, 0x200, &value); + if (ret) + goto out; + if (value != 0x15) { + dev_err(&m->pdev->dev, "OTP selector mismatch: %02x\n", value); + ret = -EPROTO; + goto out; + } + for (i = 0x200; i <= 0x23f; i++) { + ret = mt_otp_byte(m, i, &value); + if (ret) + goto out; + } + for (i = 0; i < regions; i++) { + mt7932_ram_region(ram->data, ram->size, i, ®ion); + if (region.features & BIT(5)) + entry = region.target; + if (region.features & BIT(6)) + continue; + ret = mt_download(m, ram->data, ®ion, true); + if (ret) + goto out; + } + for (i = 0; i < 1000; i++) { + mt_tx_clean(&m->tx[16]); + if (!m->tx[16].queued) + break; + usleep_range(100, 200); + } + if (m->tx[16].queued) { + ret = -ETIMEDOUT; + goto out; + } + usleep_range(20000, 21000); + ret = mt_secure_acquire(m); + if (ret) + goto out; + put_unaligned_le32(!!entry, request); + put_unaligned_le32(entry, request + 4); + ret = mt_status(m, 2, request, sizeof(request)); + if (ret) + goto out; + mt_secure_release(m); + ret = mt_poll(m, 0xe00f0, 3, 3, 5000000); + if (ret) + goto out; + dev_info(&m->pdev->dev, "FIRMWARE_READY entry=%08x\n", entry); + ret = mt_runtime_start(m); +out: + release_firmware(ram); +out_patch: + release_firmware(patch); + return ret; +} + +void mt_transport_snapshot(struct mt7932 *m) +{ + unsigned int i; + struct mt7932_ring *tx = &m->tx[17]; + + dev_info(&m->pdev->dev, "TRANSPORT: interrupts=%llu packets=%llu pending=%08x mask=%08x TX17 head=%u tail=%u queued=%u DMA=%u\n", + m->interrupts, m->packets, mt_read(m, W + 0x200), mt_read(m, W + 0x204), + tx->head, tx->tail, tx->queued, mt_read(m, tx->reg + 12)); + for (i = 0; i < ARRAY_SIZE(m->rx); i++) { + struct mt7932_ring *rx = &m->rx[i]; + + if (rx->count) + dev_info(&m->pdev->dev, "TRANSPORT_RX%u: tail=%u producer=%u DMA=%u control=%08x\n", + i, rx->tail, mt_read(m, rx->reg + 8), mt_read(m, rx->reg + 12), + le32_to_cpu(READ_ONCE(rx->desc[rx->tail].control))); + } +} + + +static int mt_initialize(struct mt7932 *m) +{ + int ret; + + ret = mt_dma_setup(m); + if (!ret) + ret = mt_rom_gate(m); + if (!ret) + ret = mt_firmware_gate(m); + return ret; +} + +static int mt_reset_retained(struct mt7932 **epoch) +{ + struct mt7932 *old = *epoch, *fresh; + struct pci_dev *pdev = old->pdev; + int irq = pci_irq_vector(pdev, 0), ret; + + /* No interface/work producers have been published at this test gate. */ + WRITE_ONCE(old->running, false); + mt_write(old, W + 0x204, 0); + devm_free_irq(&pdev->dev, irq, old); + old->irq_requested = false; + ret = mt_ownership(old); + if (!ret) + ret = mt_dma_stop(old); + if (!ret) + ret = mt_function_reset(old); + if (ret) + return ret; + old->dma_owned = false; + dev_info(&pdev->dev, "RESET_TEST: function reset complete; retaining old DMA\n"); + fresh = devm_kzalloc(&pdev->dev, sizeof(*fresh), GFP_KERNEL); + if (!fresh) + return -ENOMEM; + fresh->pdev = pdev; + fresh->bar = old->bar; + fresh->irq_mask = IRQ_MASK; + fresh->vectors_allocated = true; + ether_addr_copy(fresh->mac, old->mac); + init_completion(&fresh->response); + init_completion(&fresh->cal_response); + init_completion(&fresh->reset_retry); + mutex_init(&fresh->command_mutex); + spin_lock_init(&fresh->response_lock); + *epoch = fresh; + pci_set_drvdata(pdev, fresh); + ret = mt_ownership(fresh); + if (ret) + return ret; + ret = mt_poll(fresh, 0xf0140, BIT(4), BIT(4), 200000); + if (ret) + return ret; + mt_write(fresh, W + 0x204, 0); + ret = devm_request_threaded_irq(&pdev->dev, irq, mt_irq, mt_irq_thread, + IRQF_ONESHOT | IRQF_SHARED, "mt7932", fresh); + if (ret) + return ret; + fresh->irq_requested = true; + ret = mt_initialize(fresh); + if (!ret) + dev_info(&pdev->dev, "RESET_TEST: fresh ROM and firmware gates passed\n"); + return ret; +} + +/* A failed PCI reset cannot be turned into successful removal: devres would + * free still-owned DMA. Keep the remove callback (and therefore its device, + * mappings and module text) alive. This explicit retry does not run hardware + * operations from sysfs or bypass the PCI device lock held by remove. + */ +static ssize_t removal_retry_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct mt7932 *m = dev_get_drvdata(dev); + bool retry; + + if (kstrtobool(buf, &retry) || !retry) + return -EINVAL; + if (!READ_ONCE(m->removal_error)) + return -EBUSY; + complete(&m->reset_retry); + return count; +} +static DEVICE_ATTR_WO(removal_retry); + +static int mt_retire_dma(struct mt7932 *m) +{ + int ret; + + if (!m->dma_owned) + return 0; + ret = mt_ownership(m); + if (!ret) + ret = mt_dma_stop(m); + if (!ret) + ret = mt_function_reset(m); + if (!ret) + m->dma_owned = false; + return ret; +} + +static void mt_remove(struct pci_dev *pdev) +{ + struct mt7932 *m = pci_get_drvdata(pdev); + int ret; + + mt_stop_host(m); + while ((ret = mt_retire_dma(m))) { + WRITE_ONCE(m->removal_error, ret); + dev_crit(&pdev->dev, "removal blocked: reset error %d; DMA remains owned. Write 1 to removal_retry to retry, or reset the system.\n", ret); + /* No automatic reset loop and no free-on-timeout. A permanently + * inaccessible function requires platform recovery, not a fake + * successful rmmod. Unload intentionally remains pending. + */ + wait_for_completion(&m->reset_retry); + WRITE_ONCE(m->removal_error, 0); + } + device_remove_file(&pdev->dev, &dev_attr_removal_retry); + if (m->netdev) { + unregister_netdev(m->netdev); + free_netdev(m->netdev); + m->netdev = NULL; + } + if (m->wiphy_registered) + wiphy_unregister(m->wiphy); + if (m->wiphy) + wiphy_free(m->wiphy); + if (m->vectors_allocated) + pci_free_irq_vectors(pdev); + dev_info(&pdev->dev, "REMOVE_COMPLETE: host users joined, PCI function reset, DMA retired\n"); + /* All coherent allocations/BAR mappings are devres-owned and now safe + * to release. No default-domain ownership manipulation is necessary. + */ +} + +static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct mt7932 *m; + struct iommu_domain *domain; + int ret; + + if (!of_machine_is_compatible("apple,j700") || pdev->devfn != 0) + return -ENODEV; + domain = iommu_get_domain_for_dev(&pdev->dev); + if (!domain || (domain->type != IOMMU_DOMAIN_DMA && domain->type != IOMMU_DOMAIN_DMA_FQ)) + return dev_err_probe(&pdev->dev, -EINVAL, "translated DMA domain required\n"); + ret = pcim_enable_device(pdev); + if (ret) + return ret; + pci_clear_master(pdev); + if (pci_resource_len(pdev, 0) < 0x100000) + return -EINVAL; + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(27)); + if (ret) + return ret; + m = devm_kzalloc(&pdev->dev, sizeof(*m), GFP_KERNEL); + if (!m) + return -ENOMEM; + m->pdev = pdev; + m->irq_mask = IRQ_MASK; + ret = of_get_mac_address(pdev->dev.of_node, m->mac); + if (ret || !is_valid_ether_addr(m->mac)) + return dev_err_probe(&pdev->dev, ret ?: -EINVAL, "valid own-device DT MAC required\n"); + dev_dbg(&pdev->dev, "ADT_WIFI_IDENTITY_OK MAC=%pM\n", m->mac); + m->bar = pcim_iomap_region(pdev, 0, "mt7932"); + if (IS_ERR(m->bar)) + return PTR_ERR(m->bar); + pci_set_drvdata(pdev, m); + init_completion(&m->response); + init_completion(&m->cal_response); + init_completion(&m->reset_retry); + mutex_init(&m->command_mutex); + spin_lock_init(&m->response_lock); + ret = mt_ownership(m); + if (ret) + goto retain; + mt_write(m, W + 0x204, 0); + ret = pci_alloc_irq_vectors(pdev, 1, 1, legacy_irq_test ? PCI_IRQ_INTX : PCI_IRQ_ALL_TYPES); + if (ret < 0) + goto retain; + m->vectors_allocated = true; + ret = devm_request_threaded_irq(&pdev->dev, pci_irq_vector(pdev, 0), + mt_irq, mt_irq_thread, IRQF_ONESHOT | IRQF_SHARED, "mt7932", m); + if (ret) + goto retain; + m->irq_requested = true; + { + struct irq_data *irq = irq_get_irq_data(pci_irq_vector(pdev, 0)); + unsigned int cap = pci_find_capability(pdev, PCI_CAP_ID_MSI); + u16 control, command; + u32 lo, hi, data, mask; + + pci_read_config_word(pdev, PCI_COMMAND, &command); + pci_read_config_word(pdev, cap + PCI_MSI_FLAGS, &control); + pci_read_config_dword(pdev, cap + PCI_MSI_ADDRESS_LO, &lo); + pci_read_config_dword(pdev, cap + PCI_MSI_ADDRESS_HI, &hi); + pci_read_config_dword(pdev, cap + PCI_MSI_DATA_64, &data); + pci_read_config_dword(pdev, cap + PCI_MSI_MASK_64, &mask); + dev_dbg(&pdev->dev, "PCI command=%04x MSI cap=%x control=%04x addr=%08x:%08x data=%08x\n", + command, cap, control, hi, lo, data); + dev_dbg(&pdev->dev, "MSI mask=%08x MSI-X cap=%x chosen=%s\n", mask, + pci_find_capability(pdev, PCI_CAP_ID_MSIX), + pdev->msix_enabled ? "MSI-X" : pdev->msi_enabled ? "MSI" : "INTx"); + for (; irq; irq = irq->parent_data) + dev_dbg(&pdev->dev, "IRQ %u chip=%s hwirq=%lu\n", irq->irq, + irq->chip->name, irq->hwirq); + } + ret = mt_initialize(m); + if (!ret && reset_test) + ret = mt_reset_retained(&m); + if (!ret) + ret = mt_register_regulatory_gate(m); +retain: + if (ret) { + int error = ret; + + mt_stop_host(m); + ret = mt_retire_dma(m); + if (!ret) { + mt_remove(pdev); + return dev_err_probe(&pdev->dev, error, "initialization failed; DMA retired\n"); + } + dev_err(&pdev->dev, "initialization failed: %d, reset error %d; retaining binding and DMA\n", error, ret); + } + ret = device_create_file(&pdev->dev, &dev_attr_removal_retry); + if (ret) + dev_warn(&pdev->dev, "removal retry attribute unavailable: %d\n", ret); + return 0; +} + +static void mt_shutdown(struct pci_dev *pdev) +{ + struct mt7932 *m = pci_get_drvdata(pdev); + + WRITE_ONCE(m->running, false); + mt_write(m, W + 0x204, 0); + pci_clear_master(pdev); + /* Storage remains allocated through reset, even if the engine is stuck. */ +} + +static const struct pci_device_id mt_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x7932) }, + {} +}; +MODULE_DEVICE_TABLE(pci, mt_ids); + +static struct pci_driver mt_driver = { + .name = "mt7932-fullmac", + .id_table = mt_ids, + .probe = mt_probe, + .remove = mt_remove, + .shutdown = mt_shutdown, + .driver.suppress_bind_attrs = true, +}; +module_pci_driver(mt_driver); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("J700 MT7932 firmware-owned cfg80211 station driver"); diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h new file mode 100644 index 0000000000000..3e30ec3aff659 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -0,0 +1,226 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef MT7932_H +#define MT7932_H + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "firmware.h" +#include "calibration.h" +#include "packet.h" +#include "scan.h" +#include "association.h" +#include "regulatory.h" +#include "rate.h" +#include "capabilities.h" + + +#define W 0xd4000 +#define H 0xd6000 +#define C 0xd7000 +#define IRQ_MASK 0xe2c8000d +#define DONE BIT(31) +#define STRIDE 8192 +#define RX_STRIDE ALIGN(MT7932_RX_CAPACITY, 64) + +struct mt7932; +int mt_dma_stop(struct mt7932 *m); +int mt_function_reset(struct mt7932 *m); +void mt_stop_host(struct mt7932 *m); + +struct mt7932_desc { + __le32 address, control, second, info; +}; + +struct mt7932_ring { + struct mt7932_desc *desc; + dma_addr_t dma, buffers_dma; + u8 *buffers; + u32 reg; + u16 count, head, tail, queued; + u16 slots, used; +}; + +struct mt7932 { + struct pci_dev *pdev; + void __iomem *bar; + struct mt7932_ring tx[19], rx[9]; + struct completion response; + struct completion cal_response; + struct mt7932_cal_completion cal_state; + u8 cal_requests[32][16]; + u8 cal_request_seq[32]; + unsigned int cal_request_head, cal_request_count; + spinlock_t response_lock; + u8 seq, waiting; + u8 mac[ETH_ALEN]; + u8 reply[MT7932_RX_CAPACITY]; + size_t reply_length; + u64 interrupts, packets; + bool running; + bool stopping; + bool irq_requested, vectors_allocated, wiphy_registered; + bool dma_owned; + struct completion reset_retry; + int removal_error; + bool ram_config; + bool runtime_epoch; + bool runtime_cal_valid; + u8 runtime_cal, board_type, smart_version, preload_version; + u8 module_byte; + u8 six_ghz; + bool six_ghz_valid; + bool stof_supported; + u8 phy_cap[12]; + u8 antenna_mask; + struct wiphy *wiphy; + struct ieee80211_supported_band band2; + struct cfg80211_chan_def connect_chandef; + u8 connect_center; + struct ieee80211_channel channels[13]; + struct ieee80211_channel channels5[4]; + struct ieee80211_supported_band band5; + u8 scan_band, scan_batch; + struct ieee80211_rate rates[12]; + struct work_struct startup_work; + struct mutex command_mutex; + bool startup_started, rf_ready, reg_pending, policy_failed; + bool interface_registered, interface_up; + u32 reg_generation, reg_attempted; + struct mt7932_reg_snapshot reg_desired; + struct net_device *netdev; + struct wireless_dev wdev; + struct cfg80211_scan_request *scan_request; + struct work_struct scan_finish_work; + struct work_struct cal_work; + struct delayed_work scan_timeout_work; + u8 scan_seq, retired_scan_seq, scan_home_channel; + bool scan_finished, scan_aborted, bss_active; + u64 beacons; + unsigned long peer_alloc; + struct mt7932_peer_ids peer_ids; + struct cfg80211_bss *connect_bss; + struct work_struct connect_work, disconnect_work; + struct work_struct power_work; + /* Desired/submitted PS configuration uses command_mutex; TIM uses response_lock. */ + u8 power_mode, power_applied; + bool power_applied_valid, power_tim; + struct completion assoc_start, assoc_done, disconnect_done; + struct completion discovery_done; + struct completion scan_done; + struct ieee80211_channel *connect_channel_req, *connect_channel_hint; + u8 connect_bssid_req[ETH_ALEN], connect_bssid_hint[ETH_ALEN]; + bool discovering, discovery_finished, connect_cancelled; + u8 connect_ssid[32], connect_pmk[32], connect_bssid[6]; + u8 assoc_request_ies[512]; + u16 assoc_request_ie_len; + bool assoc_request_seen; + u32 connection_generation; + u8 connect_ssid_length, connect_channel, peer_wtbl; + bool connecting, connected, peer_valid, link_failed, connect_open; + bool disconnecting, disconnect_seen, disconnect_event_received, disconnect_local; + u16 disconnect_reason; + int connect_error; + u32 event_cipher; + spinlock_t data_lock; + /* Per-association host Ethernet accounting, protected by data_lock. */ + u64 station_tx_bytes, station_rx_bytes; + u32 station_tx_packets, station_rx_packets; + unsigned long station_connected; + s8 station_signal; + bool station_signal_valid; + struct rate_info station_rx_rate; + unsigned long station_rx_rate_time; + bool station_rx_rate_valid; + struct rate_info station_tx_rate; + unsigned long station_tx_rate_time, station_rate_query_time; + bool station_tx_rate_valid, station_rate_query_failed; + u8 *data_headers, *data_payloads; + dma_addr_t data_headers_dma, data_payloads_dma; + bool data_live[512], data_token[512], data_retired[512]; + struct mt7932_duplicate duplicates[17]; + u64 tx_released, rx_ethernet; + bool data_ready; + bool bss_absent; + u8 bss_quota; + u32 irq_mask; + void *ipc, *aux; + dma_addr_t ipc_dma, aux_dma; +}; + +bool mt_rf_allowed(struct mt7932 *m); +u32 mt_read(struct mt7932 *m, u32 reg); +void mt_write(struct mt7932 *m, u32 reg, u32 value); +void mt_rmw(struct mt7932 *m, u32 reg, u32 mask, u32 value); +int mt_poll(struct mt7932 *m, u32 reg, u32 mask, u32 expected, u32 timeout); +void *mt_alloc(struct mt7932 *m, size_t size, dma_addr_t *dma); +void mt_tx_clean(struct mt7932_ring *q); +int mt_request_ext(struct mt7932 *m, u8 cid, u8 ext, bool runtime, bool set, bool wait, + const void *payload, size_t length); +int mt_request(struct mt7932 *m, u8 cid, bool runtime, bool set, bool wait, + const void *payload, size_t length); +void mt_transport_snapshot(struct mt7932 *m); +int mt_recovery_gate(struct mt7932 *m); +int mt_cal_procedure(struct mt7932 *m, struct mt7932_cal_piece *pieces, + unsigned int count, unsigned int logical, u8 context_version); +int mt_calibration_gate(struct mt7932 *m); +void mt_data_clean_locked(struct mt7932 *m); +void mt_data_clean(struct mt7932 *m); +void mt_bss_presence(struct mt7932 *m, const struct mt7932_event *event); +void mt_data_complete(struct mt7932 *m, const u8 *packet, size_t length); +netdev_tx_t mt_net_xmit(struct sk_buff *skb, struct net_device *netdev); +void mt_data_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame); +int mt_data_prepare(struct mt7932 *m); +struct mt7932 *mt_from_wiphy(struct wiphy *wiphy); +void mt_scan_event(struct mt7932 *m, const struct mt7932_event *event); +void mt_abort_scan(struct wiphy *wiphy, struct wireless_dev *wdev); +int mt_scan_quiesce(struct mt7932 *m); +int mt_connect_discover(struct mt7932 *m); +int mt_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request); +int mt_register_interface(struct mt7932 *m); +int mt_enable_scan(struct mt7932 *m); +void mt_packet_receive(struct mt7932 *m, const u8 *packet, size_t length); +void mt_link_event(struct mt7932 *m, const struct mt7932_event *event); +struct cfg80211_bss *mt_connect_find_bss(struct mt7932 *m); +void mt_connect_work(struct work_struct *work); +int mt_connect(struct wiphy *wiphy, struct net_device *netdev, + struct cfg80211_connect_params *params); +void mt_disconnect_work(struct work_struct *work); +int mt_disconnect(struct wiphy *wiphy, struct net_device *netdev, u16 reason); +int mt_deauth(struct wiphy *wiphy, struct net_device *netdev, + struct cfg80211_deauth_request *request); +int mt_change_interface(struct wiphy *wiphy, struct net_device *netdev, + enum nl80211_iftype type, struct vif_params *params); +int mt_register_regulatory_gate(struct mt7932 *m); +int mt_get_channel(struct wiphy *wiphy, struct wireless_dev *wdev, + unsigned int link_id, struct cfg80211_chan_def *chandef); +int mt_get_station(struct wiphy *wiphy, struct wireless_dev *wdev, + const u8 *mac, struct station_info *sinfo); +int mt_dump_station(struct wiphy *wiphy, struct wireless_dev *wdev, + int idx, u8 *mac, struct station_info *sinfo); +int mt_set_power_mgmt(struct wiphy *wiphy, struct net_device *netdev, + bool enabled, int timeout); +void mt_power_work(struct work_struct *work); +int mt_power_init(struct mt7932 *m); +void mt_station_update_rate(struct mt7932 *m, const u8 *mac); + +bool mt_transport_polled(void); + +#endif diff --git a/drivers/net/wireless/mediatek/mt7932/packet.h b/drivers/net/wireless/mediatek/mt7932/packet.h new file mode 100644 index 0000000000000..303cc87fef8b0 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/packet.h @@ -0,0 +1,228 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef MT7932_PACKET_H +#define MT7932_PACKET_H + +#include "protocol.h" +#include "channels.h" + +struct mt7932_rx_frame { + const u8 *data; + size_t length; + u32 timestamp, rcpi, rxv; + u16 frame_control, sequence_control, qos_control, wtbl; + u8 channel, amsdu; + bool translated, group4, signal_valid, rate_valid, unicast; +}; + +/* NEO_PACKET_RX_GROUPS_AND_DATA_TXP_CONTRACT. Strict non-monitor subset; + * descriptor groups are ordered4,1,2,3,5, not by numerical group identifier. + */ +static inline int mt7932_rx_frame(const u8 *packet, size_t size, struct mt7932_rx_frame *out) +{ + u32 w0, w1, w2, w3, w4, kind; + size_t length, offset = 24, padding; + unsigned int i; + static const u8 groups[] = {4, 1, 2, 3, 5}; + static const u8 sizes[] = {16, 16, 8, 8, 72}; + + if (size < 24) + return -EMSGSIZE; + w0 = get_unaligned_le32(packet); + length = w0 & 0xffff; + kind = w0 >> 27; + if (kind == 7 && ((w0 >> 16) & 15) == 1) + kind = 8; + if (kind != 2 && kind != 8) + return -ENOMSG; + if (length < 24 || length > size) + return -EMSGSIZE; + w1 = get_unaligned_le32(packet + 4); + w2 = get_unaligned_le32(packet + 8); + w3 = get_unaligned_le32(packet + 12); + w4 = get_unaligned_le32(packet + 16); + /* Word2 bit25 is HDR_TRANS_ERROR, not an integrity error. The source + * still parses the original 802.11 frame when translation was not done. + */ + if ((w1 & (0x1eU << 24)) || (w2 & (3U << 23)) || + ((w2 & (1U << 13)) && (w1 & (1U << 23)))) + return -EBADMSG; + if ((w1 & (1U << 15)) && !(w1 & (1U << 13))) + return -EPROTO; + memset(out, 0, sizeof(*out)); + out->translated = !!(w2 & (1U << 13)); + out->channel = (w3 >> 8) & 255; + out->wtbl = w1 & 0x3ff; + out->unicast = ((w3 >> 16) & 3) == 1; + out->amsdu = w4 & 3; + if (!mt7932_channel_band(out->channel)) + return -EOPNOTSUPP; + if ((out->translated && (w2 & (1U << 27))) || (!out->translated && out->amsdu)) + return -EOPNOTSUPP; + for (i = 0; i < 5; i++) { + unsigned int group = groups[i]; + + if (!(w1 & (1U << (10 + group)))) + continue; + if (sizes[i] > length - offset) + return -EMSGSIZE; + if (group == 4) { + out->frame_control = get_unaligned_le16(packet + offset); + out->sequence_control = get_unaligned_le16(packet + offset + 8); + out->qos_control = get_unaligned_le16(packet + offset + 10); + out->group4 = true; + } else if (group == 2) { + out->timestamp = get_unaligned_le32(packet + offset); + } else if (group == 3 || group == 5) { + if (group == 3) { + out->rxv = get_unaligned_le32(packet + offset); + out->rate_valid = true; + } + out->rcpi = get_unaligned_le32(packet + offset + (group == 3 ? 4 : 24)); + out->signal_valid = true; + } + offset += sizes[i]; + } + /* Translated fragments need header reconstruction, which this bounded + * Ethernet path does not implement. RXD word2 bit27 is not equivalent + * to the saved group4 frame-control More Fragments bit. + */ + if (out->translated && out->group4 && (out->frame_control & 0x0400)) + return -EOPNOTSUPP; + padding = 2 * ((w2 >> 14) & 3); + if (padding > length - offset) + return -EMSGSIZE; + offset += padding; + if (length - offset < (out->translated ? 14 : 24)) + return -EMSGSIZE; + out->data = packet + offset; + out->length = length - offset; + if (!out->translated) { + out->frame_control = get_unaligned_le16(out->data); + out->sequence_control = get_unaligned_le16(out->data + 22); + } + return 0; +} + +static inline int mt7932_rx_signal(const struct mt7932_rx_frame *frame, u8 antenna_mask) +{ + int signal = -128; + unsigned int i; + + if (!frame->signal_valid || !antenna_mask || (antenna_mask & ~15)) + return -128; + for (i = 0; i < 4; i++) { + int value = ((frame->rcpi >> (8 * i)) & 255) / 2 - 110; + + if ((antenna_mask & (1U << i)) && value < 0 && value > signal) + signal = value; + } + return signal; +} + +static inline bool mt7932_rx_beacon(const struct mt7932_rx_frame *frame) +{ + size_t offset = 36; + u16 type = frame->frame_control & 0x00fc; + + if (frame->translated || (type != 0x80 && type != 0x50) || frame->length < 36) + return false; + while (offset < frame->length) { + if (frame->length - offset < 2 || frame->data[offset + 1] > frame->length - offset - 2) + return false; + offset += 2 + frame->data[offset + 1]; + } + return true; +} + +/* One linear, protected Ethernet packet. Caller owns token, WTBL and buffers. */ +static inline int mt7932_data_header(u8 out[64], const u8 *ethernet, size_t length, + u64 dma, unsigned int token, unsigned int wtbl, + unsigned int own_mac, unsigned int tid, bool protected_frame) +{ + u32 word1; + + if (length < 14 || length > 4095 || dma >> 35 || token > 0x7fff || + wtbl > 0x3ff || own_mac > 63 || tid > 7) + return -EINVAL; + memset(out, 0, 64); + put_unaligned_le32((16U << 25) | (10U << 16) | (length + 32), out); + word1 = (1U << 31) | (own_mac << 24) | (tid << 20) | wtbl; + if (get_unaligned_be16(ethernet + 12) >= 0x600) + word1 |= 1U << 15; + put_unaligned_le32(word1, out + 4); + put_unaligned_le32((30U << 11) | (protected_frame ? 2 : 0), out + 12); + put_unaligned_le16(token | 0x8000, out + 32); + put_unaligned_le32(dma, out + 40); + put_unaligned_le16(length | ((dma >> 32) << 12) | 0x8000, out + 44); + return 0; +} + +struct mt7932_duplicate { + u16 sequence; + bool valid, retry_amsdu; +}; + +/* RX consumer alone owns this association-local cache. */ +static inline bool mt7932_rx_duplicate(struct mt7932_duplicate cache[17], + const struct mt7932_rx_frame *frame) +{ + struct mt7932_duplicate *entry; + unsigned int index = 16; + u16 sequence; + bool retry; + + if (!frame->group4) { + cache[16] = (struct mt7932_duplicate) { .valid = true }; + return false; + } + if (frame->unicast && (frame->frame_control & 0xfc) == 0x88) + index = frame->qos_control & 15; + entry = &cache[index]; + sequence = frame->sequence_control >> 4; + retry = frame->frame_control & (1U << 11); + if (retry && entry->valid && entry->sequence == sequence) { + if (!entry->retry_amsdu) + return true; + if (frame->amsdu == 1) + entry->retry_amsdu = false; + return false; + } + entry->sequence = sequence; + entry->valid = true; + entry->retry_amsdu = retry && frame->amsdu == 3; + return false; +} + +/* Validate the complete variable walk before returning any completion IDs. */ +static inline int mt7932_tx_free(const u8 *packet, size_t size, u16 *tokens, + unsigned int capacity) +{ + size_t length, offset = 8; + unsigned int version, count, i = 0; + u32 word; + + if (size < 8) + return -EMSGSIZE; + length = get_unaligned_le16(packet); + if (length < 8 || length > size) + return -EMSGSIZE; + word = get_unaligned_le32(packet + 4); + version = (word >> 16) & 7; + if (version > 3) + return -EOPNOTSUPP; + count = get_unaligned_le16(packet + 2) & (version == 3 ? 0x3ff : 0x7f); + if (count > capacity) + return -ENOSPC; + while (i < count) { + if (length - offset < (version ? 4 : 2)) + return -EMSGSIZE; + word = version ? get_unaligned_le32(packet + offset) : get_unaligned_le16(packet + offset); + offset += version ? 4 : 2; + if (version == 3 && (word & (1U << 31))) + continue; + tokens[i++] = version == 3 ? (word >> 16) & 0x7fff : word & (version == 2 ? 0x7fff : 0xffff); + } + return count; +} + +#endif diff --git a/drivers/net/wireless/mediatek/mt7932/power.c b/drivers/net/wireless/mediatek/mt7932/power.c new file mode 100644 index 0000000000000..eeb5071c4fe87 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/power.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include "mt7932.h" + +/* Stock CID05 is a four-byte SET with no requested response. Successful + * submission is accepted configuration, not proof that the radio is asleep. + * cfg80211 implements GET_POWER_SAVE from the accepted requested setting. + */ +static int mt_power_submit(struct mt7932 *m, u8 mode) +{ + u8 body[4] = { 0, mode, 0, 0 }; /* owned BSS0, not a station/WTBL ID */ + int ret; + + ret = mt_request(m, 5, true, true, false, body, sizeof(body)); + m->power_applied_valid = !ret; + if (!ret) + m->power_applied = mode; + return ret; +} + +/* command_mutex held; each new BSS session starts explicitly awake. */ +int mt_power_init(struct mt7932 *m) +{ + unsigned long flags; + + spin_lock_irqsave(&m->response_lock, flags); + m->power_tim = false; + spin_unlock_irqrestore(&m->response_lock, flags); + return mt_power_submit(m, 0); +} + +/* command_mutex fences BSS reuse/retirement while the SET is submitted. */ +static int mt_power_apply(struct mt7932 *m, u8 mode) +{ + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&m->response_lock, flags); + if (m->stopping || !mt_rf_allowed(m)) + ret = -ESHUTDOWN; + else if (!m->connected || m->disconnecting || !m->peer_valid) + ret = -ENOTCONN; + else if (mode && !m->power_tim) + ret = -EAGAIN; + spin_unlock_irqrestore(&m->response_lock, flags); + if (ret) + return ret; + if (m->power_applied_valid && m->power_applied == mode) + return 0; + return mt_power_submit(m, mode); +} + +int mt_set_power_mgmt(struct wiphy *wiphy, struct net_device *netdev, + bool enabled, int timeout) +{ + struct mt7932 *m = mt_from_wiphy(wiphy); + u8 mode = enabled ? (timeout == -1 ? 2 : 1) : 0; + int ret = 0; + + if (netdev != m->netdev) + return -ENODEV; + /* Stock maps positive timeouts to mode1; it does not send a timer value. */ + if (timeout < -1) + return -EINVAL; + mutex_lock(&m->command_mutex); + if (READ_ONCE(m->stopping)) + ret = -ESHUTDOWN; + else if (READ_ONCE(m->disconnecting)) + ret = -EBUSY; + else if (READ_ONCE(m->connected)) + ret = mt_power_apply(m, mode); + /* Disconnected desired policy is applied after association/TIM discovery. */ + if (!ret) + m->power_mode = mode; + mutex_unlock(&m->command_mutex); + return ret; +} + +void mt_power_work(struct work_struct *work) +{ + struct mt7932 *m = container_of(work, struct mt7932, power_work); + int ret; + + mutex_lock(&m->command_mutex); + ret = mt_power_apply(m, m->power_mode); + if (ret && ret != -EAGAIN && ret != -ENOTCONN && ret != -ESHUTDOWN) + dev_warn(&m->pdev->dev, "station power-save submission failed: %d\n", ret); + mutex_unlock(&m->command_mutex); +} diff --git a/drivers/net/wireless/mediatek/mt7932/protocol.h b/drivers/net/wireless/mediatek/mt7932/protocol.h new file mode 100644 index 0000000000000..00751e130ef44 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/protocol.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef MT7932_PROTOCOL_H +#define MT7932_PROTOCOL_H + +#include +#include +#include +#include + +#define MT7932_ENVELOPE 64 +#define MT7932_RX_CAPACITY 0x930 + +/* Byte construction also supports host tests without packed-structure casts. */ +static inline int mt7932_envelope(u8 *out, size_t capacity, u8 cid, u8 ext, + u8 seq, bool runtime, bool set, bool reply, + const void *payload, size_t length) +{ + size_t total = MT7932_ENVELOPE + length; + + if (!seq || length > 8192 - MT7932_ENVELOPE || total > capacity) + return -EINVAL; + memset(out, 0, MT7932_ENVELOPE); + put_unaligned_le32(total | ((cid ? 2U : 3U) << 23), out); + put_unaligned_le32(0x10000, out + 4); + put_unaligned_le16(total - 32, out + 32); + out[36] = cid; + out[37] = 0xa0; + out[38] = cid == 0 || (runtime && set); + out[39] = seq; + out[41] = ext; + out[43] = runtime && reply; + if (length) + memcpy(out + MT7932_ENVELOPE, payload, length); + return total; +} + +struct mt7932_event { + u8 eid, seq, option; + const u8 *packet; + size_t length; +}; + +static inline int mt7932_event_parse(const u8 *packet, size_t length, + struct mt7932_event *event) +{ + u32 word; + size_t outer, inner; + + if (length < 32 || length > MT7932_RX_CAPACITY) + return -EMSGSIZE; + word = get_unaligned_le32(packet); + outer = word & 0xffff; + if (outer < 32 || outer > length) + return -EMSGSIZE; + if ((word >> 27) != 7 || ((word >> 16) & 15) == 1) + return -ENOMSG; + inner = get_unaligned_le16(packet + 24); + if (inner < 8 || inner > outer - 24) + return -EMSGSIZE; + event->eid = packet[28]; + event->seq = packet[29]; + event->option = packet[30]; + event->packet = packet; + event->length = inner + 24; + return 0; +} + +static inline bool mt7932_event_matches(const struct mt7932_event *e, u8 seq) +{ + return seq && e->seq == seq && !(e->option & 4); +} + +#endif diff --git a/drivers/net/wireless/mediatek/mt7932/rate.h b/drivers/net/wireless/mediatek/mt7932/rate.h new file mode 100644 index 0000000000000..d570a69f81f44 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/rate.h @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef MT7932_RATE_H +#define MT7932_RATE_H + +/* Qualified J700 group3 RXV subset. Reject unqualified modes/STBC instead + * of manufacturing a rate from another MediaTek device's encoding. + */ +static inline bool mt7932_rx_rate(u32 rxv, u8 max_nss, struct rate_info *rate) +{ + u8 mcs = rxv & 127, nss = ((rxv >> 7) & 7) + 1; + u8 width = (rxv >> 12) & 7, gi = (rxv >> 15) & 3; + u8 stbc = (rxv >> 22) & 3, mode = (rxv >> 24) & 15; + bool dcm = !!(rxv & (1U << 17)); + static const u16 cck[] = {10, 20, 55, 110, 0, 20, 55, 110}; + static const u16 ofdm[] = {480, 240, 120, 60, 540, 360, 180, 90}; + + memset(rate, 0, sizeof(*rate)); + if (!max_nss || max_nss > 2) + return false; + if (!mode) { + if (mcs > 7 || !cck[mcs]) + return false; + rate->legacy = cck[mcs]; + return true; + } + if (mode == 1) { + if (mcs < 8 || mcs > 15) + return false; + rate->legacy = ofdm[mcs - 8]; + return true; + } + if (nss > max_nss || stbc || width > 3) + return false; + switch (mode) { + case 2: + case 3: + if (width > 1 || gi > 1 || mcs > 15 || nss != mcs / 8 + 1) + return false; + rate->flags = RATE_INFO_FLAGS_MCS; + break; + case 4: + if (gi > 1 || mcs > 9) + return false; + rate->flags = RATE_INFO_FLAGS_VHT_MCS; + break; + case 8: + if (mcs > 11 || gi > 2 || dcm) + return false; + rate->flags = RATE_INFO_FLAGS_HE_MCS; + rate->he_gi = gi; /* nl80211 HE GI0/1/2 = 0.8/1.6/3.2us */ + break; + default: + return false; /* HE ER/TRIG/MU need separately qualified allocation metadata. */ + } + if (mode != 8 && gi) + rate->flags |= RATE_INFO_FLAGS_SHORT_GI; + rate->mcs = mcs; + rate->nss = nss; + switch (width) { + case 0: rate->bw = RATE_INFO_BW_20; break; + case 1: rate->bw = RATE_INFO_BW_40; break; + case 2: rate->bw = RATE_INFO_BW_80; break; + case 3: rate->bw = RATE_INFO_BW_160; break; + } + return true; +} + +/* Stock CID85 reply: exact version1 body, owned BSS/MAC and pending sequence. + * Byte8's station identity and byte0a remain opaque; no invented WTBL test. + */ +static inline bool mt7932_tx_rate_reply(const struct mt7932_event *event, + u8 sequence, const u8 *peer, u8 max_nss, + struct rate_info *rate) +{ + const u8 *body; + u32 a, b, c, rxv; + + if (!mt7932_event_matches(event, sequence) || event->eid != 0x21 || + event->length != 344) + return false; + body = event->packet + 36; + if (body[0] != 1 || + !(get_unaligned_le32(body + 4) & 1) || body[9] || + memcmp(body + 12, peer, 6)) + return false; + a = get_unaligned_le32(body + 0x9c); + b = get_unaligned_le32(body + 0xa0); + c = get_unaligned_le32(body + 0xa4); + if (((a >> 12) & 15) >= 5) + return false; /* TX HE GI/DCM/allocation fields not qualified by this reply. */ + /* Normalize the independently qualified TX words to the shared fields. */ + rxv = (c & 127) | (((c >> 8) & 7) << 7) | + (((a >> 8) & 7) << 12) | (((b >> 26) & 3) << 15) | + (((a >> 6) & 3) << 22) | (((a >> 12) & 15) << 24); + return mt7932_rx_rate(rxv, max_nss, rate); +} +#endif diff --git a/drivers/net/wireless/mediatek/mt7932/recovery.c b/drivers/net/wireless/mediatek/mt7932/recovery.c new file mode 100644 index 0000000000000..f0b7cc9714c24 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/recovery.c @@ -0,0 +1,291 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include "mt7932.h" + +static int mt_tx_retire(struct mt7932 *m); +static int mt_runtime_scheduler(struct mt7932 *m, bool publish); +static int mt_runtime_rebuild(struct mt7932 *m); +static void mt_runtime_publish(struct mt7932 *m); +static int mt_runtime_ack(struct mt7932 *m, u32 bit); +/* First runtime epoch only. Contracts: NEO_IRQ_SELECTION_AND_RUNTIME_HANDSHAKE + * and NEO_RUNTIME_RING_REBUILD_AND_OBSERVED_PRECAL. + */ + +static int mt_tx_retire(struct mt7932 *m) +{ + unsigned long deadline = jiffies + msecs_to_jiffies(500); + unsigned int i; + bool pending; + + do { + pending = false; + for (i = 0; i < ARRAY_SIZE(m->tx); i++) { + mt_tx_clean(&m->tx[i]); + pending |= !!m->tx[i].queued; + } + if (!pending) + return 0; + usleep_range(100, 200); + } while (time_before(jiffies, deadline)); + return -ETIMEDOUT; +} + +static int mt_runtime_scheduler(struct mt7932 *m, bool publish) +{ + static const u32 offsets[] = {0x1c, 0x60, 0x64, 0x68, 0x6c, 0x70, 0x74, 0x0c}; + static const u32 values[] = {1, 0x76543210, 0xfedcba98, 0x11111000, + 0x11111000, 0x76543210, 0xfedcba98, 0x003d1000}; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(offsets); i++) { + if (publish) + mt_write(m, H + offsets[i], values[i]); + if (mt_read(m, H + offsets[i]) != values[i]) + return -EIO; + } + return 0; +} + +static int mt_runtime_rebuild(struct mt7932 *m) +{ + static const u16 counts[] = {32, 0, 512, 32, 32, 224, 128, 512, 64}; + static const u16 rx_pf[] = {0, 0, 0x40, 0xc0, 0x100, 0x140, 0x180, 0x1c0, 0x200}; + static const u16 tx_pf[] = {0x240, 0x280, 0x2c0, 0x300, 0x340, 0x400, + 0x440, 0x480, 0x4c0, 0x580, 0x5c0, 0x600, 0x640, 0x700, 0x740, + 0, 0x780, 0x7c0, 0x800}; + unsigned int i, j; + int ret; + + if (m->runtime_epoch || mt_read(m, W + 0x208) != 0x1010b870 || + mt_read(m, W + 0x2b0) != 0x28c004df || (mt_read(m, H + 4) & BIT(28))) + return -EINVAL; + /* All geometry and device ownership must pass before any descriptor edit. */ + for (i = 0; i < ARRAY_SIZE(m->tx); i++) { + struct mt7932_ring *q = &m->tx[i]; + u16 expected = i == 15 ? 0 : i < 15 ? 512 : i == 16 ? 256 : i == 17 ? 24 : 16; + + mt_tx_clean(q); + if (q->count != expected || q->queued || (q->count && !q->desc)) + return -EBUSY; + } + for (i = 0; i < ARRAY_SIZE(m->rx); i++) { + struct mt7932_ring *q = &m->rx[i]; + + if (q->count != counts[i] || (q->count && + (!q->desc || !q->buffers || q->slots != q->count))) + return -EINVAL; + } + for (i = 0; i < ARRAY_SIZE(m->tx); i++) + for (j = 0; j < m->tx[i].count; j++) + m->tx[i].desc[j].control &= cpu_to_le32(~DONE); + for (i = 0; i < ARRAY_SIZE(m->rx); i++) { + struct mt7932_ring *q = &m->rx[i]; + + for (j = 0; j < q->count; j++) { + struct mt7932_desc *d = &q->desc[j]; + dma_addr_t addr = q->buffers_dma + j * RX_STRIDE; + + d->address = cpu_to_le32(lower_32_bits(addr)); + d->second = cpu_to_le32(upper_32_bits(addr) & 15); + d->info = 0; + d->control = cpu_to_le32((le32_to_cpu(d->control) & (BIT(30) | 0xffff)) | + (MT7932_RX_CAPACITY << 16)); + } + q->head = q->tail = 0; + q->queued = q->count; + } + dma_wmb(); + mt_rmw(m, W + 0x208, BIT(0) | BIT(2) | BIT(15) | BIT(21) | BIT(27) | BIT(28), 0); + ret = mt_poll(m, W + 0x208, BIT(1) | BIT(3), 0, 100000); + if (ret) + return ret; + if (mt_read(m, W + 0x208) != 0x00103870) + return -EIO; + for (i = 0; i < ARRAY_SIZE(m->tx); i++) { + struct mt7932_ring *q = &m->tx[i]; + + if (!q->count) + continue; + /* Final empty runtime TX image, no live submissions in this epoch. */ + for (j = 0; j < q->count; j++) + q->desc[j].control = 0; + dma_wmb(); + mt_write(m, q->reg, q->dma); + mt_write(m, q->reg + 8, 0); + mt_write(m, q->reg + 4, q->count); + } + for (i = 0; i < ARRAY_SIZE(m->rx); i++) { + struct mt7932_ring *q = &m->rx[i]; + + if (!q->count) + continue; + mt_write(m, q->reg, q->dma); + mt_write(m, q->reg + 8, q->count - 1); + mt_write(m, q->reg + 4, q->count); + } + mt_write(m, W + 0x208, 0x00103870); + if (mt_read(m, W + 0x208) != 0x00103870) + return -EIO; + for (i = 0; i < ARRAY_SIZE(m->rx); i++) + if (m->rx[i].count) + mt_write(m, W + 0x680 + 4 * i, (rx_pf[i] << 16) | (i == 2 ? 8 : 4)); + for (i = 0; i < ARRAY_SIZE(m->tx); i++) + if (m->tx[i].count) + mt_write(m, W + 0x600 + 4 * i, (tx_pf[i] << 16) | + (i == 4 || i == 8 || i == 12 ? 12 : 4)); + mt_write(m, W + 0x20c, U32_MAX); + if (mt_read(m, W + 0x2b0) != 0x28c004df) + return -EIO; + mt_write(m, W + 0x208, 0x5030b870); + if ((mt_read(m, W + 0x208) & ~(BIT(1) | BIT(3))) != 0x5030b870) + return -EIO; + dma_wmb(); + mt_write(m, W + 0x208, 0x5030b875); + mt_write(m, W + 0x298, 0x0c); + mt_write(m, C + 0x38, 0x13); + mt_write(m, W + 0x2f0, 0x8032800a); + if ((mt_read(m, W + 0x208) & ~(BIT(1) | BIT(3))) != 0x5030b875 || + mt_read(m, W + 0x298) != 0x0c || mt_read(m, C + 0x38) != 0x13 || + mt_read(m, W + 0x2f0) != 0x8032800a || + mt_read(m, m->tx[17].reg + 8) || mt_read(m, m->tx[17].reg + 12)) + return -EIO; + for (i = 0; i < ARRAY_SIZE(m->tx); i++) { + m->tx[i].head = m->tx[i].tail = 0; + m->tx[i].used = 0; + } + m->runtime_epoch = true; + return 0; +} + +static void mt_runtime_publish(struct mt7932 *m) +{ + m->irq_mask = 0xeec8000d; + mt_write(m, W + 0x200, m->irq_mask); + mt_write(m, W + 0x204, m->irq_mask); + mt_write(m, W + 0x1f4, 0xffff); +} + +static int mt_runtime_ack(struct mt7932 *m, u32 bit) +{ + u32 value; + int ret = readl_poll_timeout(m->bar + W + 0x1f0, value, + value == U32_MAX || (value & bit), 10, 500000); + + if (ret || value == U32_MAX) + return ret ?: -ENODEV; + mt_write(m, W + 0x1f0, bit); + return 0; +} + +int mt_recovery_gate(struct mt7932 *m) +{ + /* Original observed 25F84 CID6b bodies, not PL country tables. + * Explicit experiment; first two action2 field meanings remain opaque. + */ + static const u8 queue_config[3][32] = { + {0x10,0x11,2,0,3,0,7,0,0x2f,0,2}, + {0xe0,0x6e,2,0,0x0f,0,0xff,3,0,0,3}, + {0x0f,0,1}, + }; + static const u8 recover[] = {3, 1, 0, 0}; + u64 start; + unsigned int i, phase = 0; + int ret; + + /* mt_runtime_start() completed configuration before entering this stage. */ + if (mt_transport_polled() || m->runtime_epoch) + return -EINVAL; + for (i = 0; i < ARRAY_SIZE(queue_config); i++) { + ret = mt_request(m, 0x6b, true, true, false, queue_config[i], 32); + if (ret) + return ret; + } + ret = mt_tx_retire(m); + if (ret) + return ret; + disable_irq(pci_irq_vector(m->pdev, 0)); + WRITE_ONCE(m->running, false); + ret = mt_request_ext(m, 0xed, 0x81, true, true, false, recover, sizeof(recover)); + if (ret) + goto out; + ret = mt_tx_retire(m); + if (ret) + goto out; + phase = 1; + mt_write(m, W + 0x204, 0); + ret = mt_runtime_ack(m, 4); + if (ret) + goto out; + mt_write(m, 0x2108, 1); + mt_runtime_publish(m); + phase = 2; + start = ktime_get_ns(); + mt_write(m, 0xe0010, 1); + ret = mt_poll(m, 0xe0010, BIT(2), BIT(2), 500000); + if (ret) + goto out; + while (ktime_get_ns() - start < 1692041) + udelay(10); + mt_write(m, W + 0x204, 0); + while (ktime_get_ns() - start < 2259166) + udelay(10); + mt_write(m, 0xe0010, 2); + ret = mt_poll(m, 0xe0010, BIT(2), 0, 500000); + if (ret) + goto out; + phase = 3; + ret = mt_runtime_ack(m, 8); + if (ret) + goto out; + if (mt_read(m, W + 0x2b0) != 0x28c004df) { + ret = -EIO; + goto out; + } + phase = 4; + ret = mt_runtime_scheduler(m, true); + if (ret) + goto out; + phase = 5; + ret = mt_runtime_rebuild(m); + if (ret) + goto out; + phase = 6; + mt_write(m, 0x2108, 2); + mt_runtime_publish(m); + mt_write(m, W + 0x204, 0); + ret = mt_runtime_ack(m, 0x10); + if (ret) + goto out; + phase = 7; + mt_write(m, 0x2108, 8); + mt_runtime_publish(m); + mt_write(m, W + 0x204, 0); + ret = mt_runtime_ack(m, 0x20); + if (ret) + goto out; + mt_runtime_publish(m); + phase = 8; + ret = mt_runtime_scheduler(m, false); +out: + if (ret) { + mt_write(m, W + 0x204, 0); + dev_err(&m->pdev->dev, "runtime transition phase=%u error=%d SW=%08x GLO=%08x EXT=%08x TX17=%u/%u\n", + phase, ret, mt_read(m, W + 0x1f0), mt_read(m, W + 0x208), + mt_read(m, W + 0x2b0), mt_read(m, m->tx[17].reg + 8), + mt_read(m, m->tx[17].reg + 12)); + } else { + WRITE_ONCE(m->running, true); + dev_info(&m->pdev->dev, "RUNTIME_DMA_READY: full RX epoch, TX17 reset, firmware NORMAL_STATE\n"); + } + enable_irq(pci_irq_vector(m->pdev, 0)); + if (!ret) { + /* New read-only validation transaction on TX17 slot0, not pre-cal replay. */ + ret = mt_request(m, 0x8a, true, false, true, NULL, 0); + if (!ret && (m->reply_length < 40 || m->reply[28] != 0xec || + mt7932_capabilities(m->reply + 36, m->reply_length - 36) < 0)) + ret = -EPROTO; + if (!ret) + dev_info(&m->pdev->dev, "RUNTIME_QUERY_OK: TX17 capability response, IRQ=%llu\n", + m->interrupts); + } + return ret; +} diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory.h b/drivers/net/wireless/mediatek/mt7932/regulatory.h new file mode 100644 index 0000000000000..6f3869fe7200e --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/regulatory.h @@ -0,0 +1,103 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef MT7932_REGULATORY_H +#define MT7932_REGULATORY_H + +/* Private, versioned packaging of original-derived country policy. This is + * not firmware code and does not replace cfg80211's regulatory database. + */ +#define MT7932_POLICY_TABLES 13 +#define MT7932_POLICY_MODES 3 + +struct mt7932_policy { + const u8 *mode[MT7932_POLICY_MODES]; + const u8 *table[MT7932_POLICY_TABLES]; + u16 length[MT7932_POLICY_TABLES]; + u8 modes; +}; + +struct mt7932_reg_snapshot { + /* Requested cfg80211 country; domain carries the selected firmware + * profile. Stock unknown-country startup uses XZ while Linux stays 00. + */ + u8 alpha2[2]; + u8 domain[148]; + u16 length; + /* Keep numeric limits in equality checks even though CID0f contains only + * flags. An unsupported reduction must not be mistaken for no change. + */ + s32 power[17]; + int error; +}; + +static inline int mt7932_policy_parse(struct mt7932_policy *out, + const u8 *data, size_t size, + const u8 alpha2[2]) +{ + size_t at = 16; + unsigned int i, n; + + memset(out, 0, sizeof(*out)); + if (size < at || size > 16384 || memcmp(data, "J7RP", 4) || + get_unaligned_le16(data + 4) != 1 || memcmp(data + 6, alpha2, 2) || + data[8] < 2 || data[8] > MT7932_POLICY_MODES || + data[9] != MT7932_POLICY_TABLES || data[10] || data[11] || + get_unaligned_le32(data + 12) != size) + return -EINVAL; + out->modes = data[8]; + for (i = 0; i < out->modes; i++, at += 328) { + if (size - at < 328 || get_unaligned_le32(data + at) || + !get_unaligned_le32(data + at + 4) || + get_unaligned_le32(data + at + 4) > 320) + return -EINVAL; + out->mode[i] = data + at; + } + for (i = 0; i < MT7932_POLICY_TABLES; i++) { + const u8 *p = data + at; + + if (size - at < 44) + return -EINVAL; + n = get_unaligned_le16(p + 2); + if (n < 44 || n > 1020 || n > size - at) + return -EINVAL; + if (i >= 1 && i <= 9) { + unsigned int count = i == 2 || i == 9 ? 5 : 8; + + /* The stock fallback includes power data for channel 14. + * This is not channel permission: our cfg80211/CID0f lists + * still exclude it, and the driver cannot scan or join it. + */ + if (i == 2 && !memcmp(alpha2, "XZ", 2)) + count = 6; + if (p[0] != 3 || p[1] != 0x0a || p[4] != count || + p[5] != (i <= 2 ? 1 : 2) || + p[6] != (i == 2 || i == 9) || p[7] || + memcmp(p + 8, alpha2, 2) || p[10] || p[11] || + n != 44 + count * 122) + return -EINVAL; + if (!memcmp(alpha2, "XZ", 2)) { + unsigned int j; + + for (j = 0; i <= 2 && j < count; j++) + if (p[44 + j * 122] != (i - 1) * 8 + j + 1) + return -EINVAL; + if (i > 1 && memcmp(p + 12, out->table[1] + 12, 32)) + return -EINVAL; + } + } else { + static const u16 lengths[] = {532, 144, 58, 265}; + static const u8 types[] = {5, 1, 4, 2}; + unsigned int k = i ? i - 9 : 0; + + if (n != lengths[k] || p[0] || p[1] != 2 || p[7] != types[k]) + return -EINVAL; + if (i == 10 && (memcmp(p + 8, alpha2, 2) || p[10] || p[11])) + return -EINVAL; + } + out->table[i] = p; + out->length[i] = n; + at += n; + } + return at == size ? 0 : -EINVAL; +} + +#endif diff --git a/drivers/net/wireless/mediatek/mt7932/reset.c b/drivers/net/wireless/mediatek/mt7932/reset.c new file mode 100644 index 0000000000000..0f384dbdaed83 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/reset.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* J700 MT7932 mapped preparation and function-scoped reset. + * Clean-room contracts: NEO_MT7932_DEVICE_RESET_UNLOAD_CONTRACT.md and + * NEO_MT7932_FLR_PREPARATION_RESOLVED.md. No secondary-bus reset fallback. + */ +#include "mt7932.h" + +int mt_dma_stop(struct mt7932 *m) +{ + u32 value; + + /* Call only after joining software producers and IRQ service. */ + mt_write(m, W + 0x204, 0); + value = mt_read(m, W + 0x208); + if (value == U32_MAX) + return -ENODEV; + mt_write(m, W + 0x208, value & ~0x18208005); + return mt_poll(m, W + 0x208, 0x1820800f, 0, 100000); +} + +static int mt_reset_identity(struct pci_dev *pdev) +{ + u32 id; + + if (pci_read_config_dword(pdev, PCI_VENDOR_ID, &id) || + id != (0x7932 << 16 | PCI_VENDOR_ID_MEDIATEK)) + return -ENODEV; + return 0; +} + +int mt_function_reset(struct mt7932 *m) +{ + struct pci_dev *pdev = m->pdev; + u32 fabric, aer_mask = 0; + u16 command, status; + int aer, ret; + + /* Probe/remove already hold the device lock. Serialize configuration + * access as PCI core reset does, but select FLR explicitly: the generic + * reset-method list can contain a bus reset with a different scope. + */ + device_lock_assert(&pdev->dev); + ret = pcie_reset_flr(pdev, true); + if (ret) + return ret; + ret = mt_reset_identity(pdev); + if (ret) + return ret; + if (mt_read(m, 0xfe250) >> 16 != 0x7000) + return -ENODEV; + aer = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR); + if (aer && aer != 0x200) + return -EOPNOTSUPP; + pci_cfg_access_lock(pdev); + if (aer) { + ret = pci_read_config_dword(pdev, aer + PCI_ERR_UNCOR_MASK, &aer_mask); + if (ret || aer_mask == U32_MAX) { + ret = -EIO; + goto unlock; + } + /* Preserve the OS mask and status evidence, not the vendor's global + * status-clear/enable literals. Only mask the reset-related errors. + */ + ret = pci_write_config_dword(pdev, aer + PCI_ERR_UNCOR_MASK, + aer_mask | 0x0057f010); + if (ret) { + ret = -EIO; + goto unlock; + } + } + mt_write(m, 0x73020, 0); + usleep_range(1000, 1100); + pci_clear_master(pdev); + if (pci_read_config_word(pdev, PCI_COMMAND, &command) || + command == U16_MAX || (command & PCI_COMMAND_MASTER)) { + ret = -EIO; + goto restore_aer; + } + if (!pci_wait_for_pending_transaction(pdev)) { + ret = -ETIMEDOUT; + goto restore_aer; + } + if (pcie_capability_read_word(pdev, PCI_EXP_DEVSTA, &status) || + status == U16_MAX || (status & PCI_EXP_DEVSTA_TRPND)) { + ret = -EIO; + goto restore_aer; + } + ret = pci_save_state(pdev); + if (ret) + goto restore_aer; + ret = pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE); + if (!ret) + ret = pcie_reset_flr(pdev, false); + else + ret = -EIO; + /* Saved BME is already clear. Restore BAR/MSI state, never old DMA. */ + pci_restore_state(pdev); + if (ret) + goto restore_aer; + ret = mt_reset_identity(pdev); + if (ret) + goto restore_aer; + if (pci_read_config_word(pdev, PCI_COMMAND, &command) || + command == U16_MAX || (command & PCI_COMMAND_MASTER) || + pci_read_config_dword(pdev, 0x48c, &fabric) || + fabric == U32_MAX || (fabric & 0x000f0000)) + ret = -EIO; +restore_aer: + if (aer && pci_write_config_dword(pdev, aer + PCI_ERR_UNCOR_MASK, aer_mask)) + ret = ret ?: -EIO; +unlock: + pci_cfg_access_unlock(pdev); + return ret; +} diff --git a/drivers/net/wireless/mediatek/mt7932/scan.h b/drivers/net/wireless/mediatek/mt7932/scan.h new file mode 100644 index 0000000000000..3572129118be2 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/scan.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef MT7932_SCAN_H +#define MT7932_SCAN_H + +#include "protocol.h" +#include "channels.h" + +/* Selected ordinary TX17/fullmac wildcard scan, not a pre-RF gate. + * Caller must own BSS0 and filter cfg80211 DISABLED channels first. NO_IR + * channels require mt7932_scan_passive() before submission. + * NEO_FIRST_CFG80211_SCAN_CONTRACT: 20ms dwell and meaningful MT7932 tail. + */ +static inline int mt7932_scan_body(u8 *body, size_t capacity, u8 seq, + const u8 *channels, unsigned int count) +{ + static const u8 ies[] = {0x7f,0x0a,1,0,8,0x8c,1,0x40,0,0,0,0, + 0x6b,7,0x0f,0xff,0xff,0xff,0xff,0xff,0xff}; + unsigned int i, j; + + if (capacity < 0x4d4 || !seq || seq > 127 || !count || count > 13) + return -EINVAL; + for (i = 0; i < count; i++) { + if (!mt7932_channel_band(channels[i]) || + mt7932_channel_band(channels[i]) != mt7932_channel_band(channels[0])) + return -EINVAL; + for (j = 0; j < i; j++) + if (channels[i] == channels[j]) + return -EINVAL; + } + memset(body, 0, 0x4d4); + body[0] = seq; + body[2] = 1; + body[3] = 1; + body[6] = 0x12; + body[7] = 1; + put_unaligned_le16(20, body + 0x9a); + body[0x9e] = 4; + body[0x9f] = count; + for (i = 0; i < count; i++) { + body[0xa0 + 2 * i] = mt7932_channel_band(channels[i]); + body[0xa1 + 2 * i] = channels[i]; + } + put_unaligned_le16(sizeof(ies), body + 0xe0); + memcpy(body + 0xe2, ies, sizeof(ies)); + put_unaligned_le16(20, body + 0x33c); + memset(body + 0x458, 0xff, 6); + put_unaligned_le32(0x1800, body + 0x4b8); + put_unaligned_le32(45, body + 0x4bc); + put_unaligned_le16(110, body + 0x4c2); + put_unaligned_le16(110, body + 0x4ca); + return 0x4d4; +} + +/* CID03's type selects passive operation; probe-count zero alone does not. + * Retain the source-recorded 40ms normal/minimum and 110ms passive/budget + * inputs. Firmware selects a 97ms passive dwell after its 13ms reserve; + * normal/minimum are not the passive timer. Do not request a random address. + * This still requires normal + * calibrated RF readiness and is not a global transmitter-disable command. + */ +static inline void mt7932_scan_passive(u8 body[0x4d4]) +{ + body[2] = 0; + body[3] = 0; + body[4] = 0; + body[5] = 0; + memset(body + 8, 0, 4 * 36); + put_unaligned_le16(40, body + 0x9a); + put_unaligned_le16(40, body + 0x33c); + memset(body + 0x458, 0, 6); +} + +/* Unassociated discovery has no data home-channel latency to protect. Allow + * two ordinary 100TU beacon periods plus margin, instead of a selected97ms + * timer shorter than one such period. Native normalization reserves13ms from + * the budget;233 retains the requested220ms. This is host timing policy, not + * a promise of uninterrupted reception. Keep connected scans unchanged. + */ +static inline void mt7932_scan_disconnected_passive(u8 body[0x4d4]) +{ + mt7932_scan_passive(body); + put_unaligned_le16(220, body + 0x4c2); + put_unaligned_le16(233, body + 0x4ca); +} + +/* Bounded standard caller-IE carriage, not the reference's silent override. */ +static inline int mt7932_scan_options(u8 body[0x4d4], const u8 *ssid, + size_t ssid_length, const u8 *ies, size_t ie_length) +{ + size_t offset = 0; + + if (ssid_length > 32 || ie_length > 600 || (!body[2] && ssid_length)) + return -EINVAL; + while (offset < ie_length) { + if (ie_length - offset < 2 || ies[offset + 1] > ie_length - offset - 2) + return -EINVAL; + offset += 2 + ies[offset + 1]; + } + if (ssid_length) { + body[3] = 8; + body[4] = 1; + put_unaligned_le32(ssid_length, body + 8); + memcpy(body + 12, ssid, ssid_length); + } + if (ie_length || ssid_length) { + memset(body + 0xe2, 0, 600); + put_unaligned_le16(ie_length, body + 0xe0); + if (ie_length) + memcpy(body + 0xe2, ies, ie_length); + } + return 0; +} + +/* Original online-scan state retains the BSS/keys and supplies its home + * channel at the end of CID03. This never narrows the requested scan list. + * Keep the batch's active/passive dwell settings and native home timing tail. + */ +static inline int mt7932_scan_home(u8 body[0x4d4], u8 channel) +{ + u8 band = mt7932_channel_band(channel); + + if (channel && !band) + return -EINVAL; + body[0x4cc] = channel; + body[0x4cd] = band; + return 0; +} + +#endif diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c new file mode 100644 index 0000000000000..a6f71f80b4b6c --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -0,0 +1,491 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include "mt7932.h" + +static int mt_stock_config(struct mt7932 *m); +static int mt_publish_policy(struct mt7932 *m, const struct mt7932_reg_snapshot *reg, + const struct mt7932_policy *policy); +static int mt_privacy_defaults(struct mt7932 *m); +static int mt_startup_once(struct mt7932 *m); +static void mt_startup_work(struct work_struct *work); +static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request *request); + +struct mt_config_record { + u8 type, key_len, value_len, reserved; + char key[32], value[32]; +}; + +static int mt_stock_config(struct mt7932 *m) +{ + static const struct { const char *key; u8 at, fallback; } updates[] = { + {"StaVHT",1,0}, {"ApVHT",1,1}, {"P2pGoVHT",1,1}, {"P2pGcVHT",1,1}, + {"Nss",4,0}, {"LdpcTx",6,0}, {"LdpcRx",7,0}, {"StbcTx",8,0}, {"StbcRx",9,0}, + }; + struct mt_config_record *records; + const struct firmware *file; + u8 body[284]; + unsigned int count, i, j, batch; + int ret; + + ret = request_firmware_direct(&file, "mediatek/mt7932/config-original.bin", &m->pdev->dev); + if (ret) + return ret; + ret = -EINVAL; + if (file->size != 8 + 65 * 68 || memcmp(file->data, "J7CF", 4) || + get_unaligned_le32(file->data + 4) != 65) + goto out; + /* + * Each absent normalized key appends one record. Reserve for all of + * them, including structurally valid packages missing every such key. + */ + records = kcalloc(65 + ARRAY_SIZE(updates), sizeof(*records), GFP_KERNEL); + if (!records) { + ret = -ENOMEM; + goto out; + } + memcpy(records, file->data + 8, 65 * 68); + count = 65; + for (i = 0; i < count; i++) { + if (records[i].type != 3 || !records[i].key_len || records[i].key_len >= 32 || + records[i].value_len >= 32 || records[i].reserved || + strnlen(records[i].key, 32) != records[i].key_len || + strnlen(records[i].value, 32) != records[i].value_len) + goto free; + for (j = 0; j < i; j++) + if (!strcmp(records[i].key, records[j].key)) + goto free; + } + /* The measured J700 profile has PHY DBDC=1. A different capability needs + * the separately specified conditional DbdcMode insertion, not a default. + */ + if (!m->phy_cap[5]) { + ret = -EOPNOTSUPP; + goto free; + } + for (i = 0; i < ARRAY_SIZE(updates); i++) { + u8 requested = updates[i].fallback, value; + + for (j = 0; j < count; j++) + if (!strcmp(records[j].key, updates[i].key)) + break; + if (j < count) { + if (kstrtou8(records[j].value, 0, &requested)) + goto free; + } else { + count++; + records[j].type = 3; + records[j].key_len = strscpy(records[j].key, updates[i].key, 32); + } + value = updates[i].at == 4 ? min(requested, m->phy_cap[4]) : requested & m->phy_cap[updates[i].at]; + memset(records[j].value, 0, 32); + records[j].value_len = scnprintf(records[j].value, 32, "0x%x", value); + } + { + /* Same ceiling as BASIC_CONFIG, without changing original artifacts. + * CID0f uses a distinct enum; keep Sta2gBw untouched. + */ + for (j = 0; j < count; j++) + if (!strcmp(records[j].key, "Sta5gBw")) + break; + if (j == count) { + ret = -EINVAL; + goto free; + } + memset(records[j].value, 0, sizeof(records[j].value)); + records[j].value[0] = '0' + MT7932_MAX_5G_BW; + records[j].value_len = 1; + for (j = 0; j < count; j++) + if (!strcmp(records[j].key, "DisRoaming")) + break; + if (j == count) { + ret = -EINVAL; + goto free; + } + memset(records[j].value, 0, sizeof(records[j].value)); + records[j].value[0] = '1'; + records[j].value_len = 1; + } + for (i = 0; i < count; i += batch) { + batch = min(4U, count - i); + memset(body, 0, sizeof(body)); + put_unaligned_le32(1, body + 4); + body[8] = batch; + put_unaligned_le16(batch * 68, body + 10); + memcpy(body + 12, records + i, batch * 68); + ret = mt_request(m, 0x70, true, true, false, body, sizeof(body)); + if (ret) + goto free; + } + dev_info(&m->pdev->dev, "STOCK_CONFIG_SUBMITTED: %u original/PHY-derived entries\n", count); +free: + kfree(records); +out: + release_firmware(file); + return ret; +} + +static int mt_publish_policy(struct mt7932 *m, const struct mt7932_reg_snapshot *reg, + const struct mt7932_policy *policy) +{ + u8 dbdc[36] = {0,0,0,0,2,0,36}; + unsigned int i; + int ret; + + if (!m->startup_started) { + ret = mt_request(m, 0x28, true, true, false, dbdc, sizeof(dbdc)); + if (ret) + return ret; + } + for (i = 0; i < policy->modes; i++) { + ret = mt_request(m, 0xca, true, true, false, policy->mode[i], 328); + if (ret) + return ret; + } + ret = mt_request(m, 0x0f, true, true, false, reg->domain, reg->length); + if (ret) + return ret; + for (i = 0; i < MT7932_POLICY_TABLES; i++) { + ret = mt_request(m, 0x5d, true, true, false, policy->table[i], policy->length[i]); + if (ret) + return ret; + } + /* Native SETs have no response/atomic policy-commit event. Command + * submission is not misrepresented as firmware readback or an ACK. + */ + dev_info(&m->pdev->dev, "REGULATORY_POLICY_SUBMITTED: %c%c firmware=%c%c channels=%u (native no-response SETs)\n", + reg->alpha2[0], reg->alpha2[1], reg->domain[0], reg->domain[1], + (reg->length - 12) / 8); + return 0; +} + +static int mt_privacy_defaults(struct mt7932 *m) +{ + static const unsigned int ids[] = {0,1,3,45,50,72,107,108,127,191,221,255}; + u8 body[60] = {}; + unsigned int i; + int ret; + + if (m->stof_supported) { + body[0] = 5; + put_unaligned_le16(32, body + 2); + put_unaligned_le32(6000, body + 4); + put_unaligned_le32(6000, body + 20); + ret = mt_request(m, 0x92, true, true, false, body, 36); + if (ret) + return ret; + } + memset(body, 0, sizeof(body)); + body[4] = 0x27; + for (i = 0; i < ARRAY_SIZE(ids); i++) + body[7 + ids[i] / 8] |= 1U << (ids[i] % 8); + body[43] = 8; + body[46] = 4; + ret = mt_request(m, 0x9b, true, true, false, body, sizeof(body)); + if (ret) + return ret; + memset(body, 0, sizeof(body)); + body[0] = 1; + body[5] = body[6] = 0x14; + ret = mt_request(m, 0x9b, true, true, false, body, sizeof(body)); + if (ret) + return ret; + memset(body, 0, 4); + body[0] = 2; + return mt_request(m, 0xc5, true, true, false, body, 4); +} + +static int mt_startup_once(struct mt7932 *m) +{ + int ret; + u8 rm_cap[44] = {1,0,0,0,0,1,0x70,0,2}; + /* Original fullmac platform roam lock, not a scan-disable/DMA fence. + * Legacy CID3e is distinct from the RTS extended CIDed/ext3e. + */ + u8 roam_lock[12] = {0,0,0,0,0,0,0,1}; + + m->startup_started = true; + ret = mt_request(m, 0x5a, true, true, false, rm_cap, sizeof(rm_cap)); + if (!ret) + ret = mt_stock_config(m); + if (!ret) + ret = mt_privacy_defaults(m); + if (!ret) + ret = mt_calibration_gate(m); + if (!ret) + ret = mt_request(m, 0x3e, true, true, false, roam_lock, sizeof(roam_lock)); + dev_info(&m->pdev->dev, "OWN_CALIBRATION_STARTUP_RESULT: %d\n", ret); + if (!ret) + ret = mt_enable_scan(m); + if (ret) + dev_err(&m->pdev->dev, "SCAN_INTERFACE_SETUP_FAILED: %d\n", ret); + return ret; +} + +static void mt_startup_work(struct work_struct *work) +{ + struct mt7932 *m = container_of(work, struct mt7932, startup_work); + struct mt7932_reg_snapshot reg; + struct mt7932_policy policy; + const struct firmware *file; + unsigned long flags; + u32 generation; + char path[64]; + int ret; + + for (;;) { + spin_lock_irqsave(&m->response_lock, flags); + generation = m->reg_generation; + reg = m->reg_desired; + if (m->stopping || generation == m->reg_attempted) { + spin_unlock_irqrestore(&m->response_lock, flags); + return; + } + m->reg_attempted = generation; + spin_unlock_irqrestore(&m->response_lock, flags); + + /* No command/wiphy lock while joining lifecycle workers. RX, D7 + * calibration and TX-free remain alive until ownership is retired. + */ + if (m->netdev) { + mt_disconnect(m->wiphy, m->netdev, WLAN_REASON_DEAUTH_LEAVING); + /* Prefer natural completion: CID1b has no guaranteed stop ACK. + * reg_pending prevents another band batch being submitted. + */ + if (READ_ONCE(m->scan_request) || READ_ONCE(m->retired_scan_seq)) + wait_for_completion_timeout(&m->scan_done, msecs_to_jiffies(5000)); + mt_abort_scan(m->wiphy, &m->wdev); + flush_work(&m->scan_finish_work); + flush_work(&m->cal_work); + } + ret = reg.error; + if (READ_ONCE(m->retired_scan_seq)) + ret = -EBUSY; + file = NULL; + if (READ_ONCE(m->policy_failed) || READ_ONCE(m->link_failed) || + (m->startup_started && !READ_ONCE(m->rf_ready))) + ret = -EIO; + if (!ret) { + if (!memcmp(reg.alpha2, "00", 2)) + scnprintf(path, sizeof(path), + "mediatek/mt7932/policy/world-XZ.bin"); + else + scnprintf(path, sizeof(path), "mediatek/mt7932/policy/%c%c.bin", + reg.alpha2[0], reg.alpha2[1]); + ret = request_firmware_direct(&file, path, &m->pdev->dev); + } + if (!ret) + ret = mt7932_policy_parse(&policy, file->data, file->size, reg.domain); + mutex_lock(&m->command_mutex); + if (READ_ONCE(m->stopping) || generation != READ_ONCE(m->reg_generation)) + goto next; + if (!ret) { + ret = mt_publish_policy(m, ®, &policy); + if (!ret && !m->startup_started) + ret = mt_startup_once(m); + /* A partial SET sequence is not a rollback. Only missing or + * invalid files BEFORE submission are recoverable by a hint. + */ + if (ret) + WRITE_ONCE(m->policy_failed, true); + } + spin_lock_irqsave(&m->response_lock, flags); + if (!ret && generation == m->reg_generation) + WRITE_ONCE(m->reg_pending, false); + spin_unlock_irqrestore(&m->response_lock, flags); + if (ret) + dev_warn(&m->pdev->dev, "REGULATORY_BLOCKED: %c%c generation=%u error=%d recovery-required=%u\n", + reg.alpha2[0], reg.alpha2[1], generation, ret, + m->policy_failed || m->link_failed); + else + dev_info(&m->pdev->dev, "REGULATORY_READY: %c%c firmware=%c%c generation=%u current=%u\n", + reg.alpha2[0], reg.alpha2[1], reg.domain[0], reg.domain[1], + generation, !m->reg_pending); +next: + mutex_unlock(&m->command_mutex); + if (file) + release_firmware(file); + } +} + +static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request *request) +{ + struct mt7932 *m = mt_from_wiphy(wiphy); + struct mt7932_reg_snapshot reg = {}; + unsigned long irqflags; + unsigned int i, count = 0; + + /* cfg80211 calls after updating channels under RTNL in this kernel. + * Copy, never retain the transient request or take command_mutex here. + */ + if (!request) + return; + memcpy(reg.alpha2, request->alpha2, 2); + if (!((reg.alpha2[0] >= 'A' && reg.alpha2[0] <= 'Z' && + reg.alpha2[1] >= 'A' && reg.alpha2[1] <= 'Z') || + !memcmp(reg.alpha2, "00", 2))) + reg.error = -EOPNOTSUPP; + memcpy(reg.domain, reg.alpha2, 2); + /* Stock database-present startup resolves the absent 00 row to XZ. + * Keep Linux's world channel restrictions and requested identity; only + * the firmware country/domain and original power package use XZ. + * Never fall back for an arbitrary unsupported explicit country. + */ + if (!memcmp(reg.alpha2, "00", 2)) + memcpy(reg.domain, "XZ", 2); + reg.domain[4] = 1; /* 2 GHz remains 20 MHz. */ + reg.domain[5] = mt7932_domain_5g_bw(); + for (i = 0; i < 17; i++) { + struct ieee80211_channel *channel = i < 13 ? &m->channels[i] : &m->channels5[i - 13]; + u32 flags = channel->flags; + + reg.power[i] = channel->max_power; + if (flags & IEEE80211_CHAN_DISABLED) + continue; + if (flags & (IEEE80211_CHAN_RADAR | IEEE80211_CHAN_NO_OFDM | + IEEE80211_CHAN_NO_20MHZ | IEEE80211_CHAN_PSD | IEEE80211_CHAN_IR_CONCURRENT)) + reg.error = -EOPNOTSUPP; + /* Native per-rate units are not a proven dBm conversion. Do not + * silently ignore a numeric reduction below the qualified 20 dBm + * channel envelope or reinterpret sentinel bytes as power caps. + */ + if (channel->max_power < 20) + reg.error = -EOPNOTSUPP; + put_unaligned_le16(channel->hw_value, reg.domain + 12 + count * 8); + /* Driver bandwidth limits are already in channel->flags. Adding + * the old blanket 20 MHz mask here would contradict the wiphy + * and silently force every native 5 GHz association back to 20. + */ + put_unaligned_le32(flags, reg.domain + 16 + count * 8); + count++; + reg.domain[i < 13 ? 8 : 9]++; + } + reg.length = 12 + count * 8; + if (!count) + reg.error = -EOPNOTSUPP; + spin_lock_irqsave(&m->response_lock, irqflags); + if (m->stopping) { + spin_unlock_irqrestore(&m->response_lock, irqflags); + return; + } + if (!memcmp(®, &m->reg_desired, sizeof(reg)) && !m->reg_pending) { + spin_unlock_irqrestore(&m->response_lock, irqflags); + return; + } + m->reg_desired = reg; + m->reg_generation++; + WRITE_ONCE(m->reg_pending, true); + if (m->netdev) { + netif_stop_queue(m->netdev); + if (m->connecting) { + m->connect_error = -ECANCELED; + complete(&m->assoc_start); + complete(&m->assoc_done); + complete(&m->discovery_done); + } + } + if (READ_ONCE(m->interface_registered)) + schedule_work(&m->startup_work); + spin_unlock_irqrestore(&m->response_lock, irqflags); + dev_info(&m->pdev->dev, "REGULATORY_REQUEST: %c%c initiator=%u generation=%u\n", + reg.alpha2[0], reg.alpha2[1], request->initiator, m->reg_generation); +} + +static const struct cfg80211_ops mt_cfg_ops = { + .scan = mt_scan, .abort_scan = mt_abort_scan, + .connect = mt_connect, .disconnect = mt_disconnect, + .deauth = mt_deauth, + .change_virtual_intf = mt_change_interface, + .get_station = mt_get_station, + .get_channel = mt_get_channel, + .dump_station = mt_dump_station, + .set_power_mgmt = mt_set_power_mgmt, +}; + +int mt_register_regulatory_gate(struct mt7932 *m) +{ + static const int rates[] = {10,20,55,110,60,90,120,180,240,360,480,540}; + static const u32 ciphers[] = { WLAN_CIPHER_SUITE_CCMP }; + unsigned int i; + int ret; + + INIT_WORK(&m->startup_work, mt_startup_work); + m->reg_pending = true; + m->wiphy = wiphy_new(&mt_cfg_ops, sizeof(m)); + if (!m->wiphy) + return -ENOMEM; + *(struct mt7932 **)wiphy_priv(m->wiphy) = m; + set_wiphy_dev(m->wiphy, &m->pdev->dev); + ether_addr_copy(m->wiphy->perm_addr, m->mac); + m->wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION); + m->wiphy->signal_type = CFG80211_SIGNAL_TYPE_MBM; + m->wiphy->cipher_suites = ciphers; + m->wiphy->n_cipher_suites = ARRAY_SIZE(ciphers); + wiphy_ext_feature_set(m->wiphy, NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK); + m->wiphy->max_scan_ssids = 1; + m->wiphy->max_scan_ie_len = 600; + m->wiphy->reg_notifier = mt_regulatory_notify; + /* World roaming may relax NO_IR after observing an AP beacon without + * changing alpha2. Request the callback so firmware's CID0f domain is + * updated through the same idle/scan fence as other regulatory changes. + */ + m->wiphy->flags |= WIPHY_FLAG_CHANNEL_CHANGE_ON_BEACON; + /* The native fullmac path applies policy at idle, not seamlessly inside + * an association. Keep the existing explicit user/core-country model: + * accepting this STA's AP hint then disconnecting to apply it would make + * cfg80211 restore the user country and repeat the join forever. + * Global hints from other radios still reach the notifier and are handled. + */ + m->wiphy->regulatory_flags |= REGULATORY_COUNTRY_IE_IGNORE; + for (i = 0; i < 13; i++) { + m->channels[i].band = NL80211_BAND_2GHZ; + m->channels[i].center_freq = 2412 + 5 * i; + m->channels[i].hw_value = i + 1; + m->channels[i].max_power = 20; + m->channels[i].flags = IEEE80211_CHAN_NO_HT40 | IEEE80211_CHAN_NO_80MHZ | IEEE80211_CHAN_NO_160MHZ; + } + for (i = 0; i < ARRAY_SIZE(rates); i++) { + m->rates[i].bitrate = rates[i]; + m->rates[i].hw_value = i; + } + m->band2.channels = m->channels; + m->band2.n_channels = ARRAY_SIZE(m->channels); + m->band2.bitrates = m->rates; + m->band2.n_bitrates = ARRAY_SIZE(m->rates); + m->wiphy->bands[NL80211_BAND_2GHZ] = &m->band2; + { + for (i = 0; i < ARRAY_SIZE(m->channels5); i++) { + m->channels5[i].band = NL80211_BAND_5GHZ; + m->channels5[i].center_freq = 5180 + 20 * i; + m->channels5[i].hw_value = 36 + 4 * i; + m->channels5[i].max_power = 20; + m->channels5[i].flags = IEEE80211_CHAN_NO_160MHZ | + (i % 2 ? IEEE80211_CHAN_NO_HT40PLUS : IEEE80211_CHAN_NO_HT40MINUS); + if (MT7932_MAX_5G_BW < 2) + m->channels5[i].flags |= IEEE80211_CHAN_NO_80MHZ; + if (!MT7932_MAX_5G_BW) + m->channels5[i].flags |= IEEE80211_CHAN_NO_HT40; + } + m->band5.channels = m->channels5; + m->band5.n_channels = ARRAY_SIZE(m->channels5); + m->band5.bitrates = m->rates + 4; + m->band5.n_bitrates = ARRAY_SIZE(m->rates) - 4; + m->wiphy->bands[NL80211_BAND_5GHZ] = &m->band5; + } + if (mt7932_phy_profile(m->phy_cap)) { + mt7932_band_capabilities(&m->band2, false); + mt7932_band_capabilities(&m->band5, true); + } + ret = wiphy_register(m->wiphy); + if (ret) { + wiphy_free(m->wiphy); + m->wiphy = NULL; + return ret; + } + m->wiphy_registered = true; + ret = mt_register_interface(m); + if (ret) + return ret; + WRITE_ONCE(m->interface_registered, true); + schedule_work(&m->startup_work); + dev_info(&m->pdev->dev, "REGULATORY_GATE_REGISTERED: cfg80211 country policy required\n"); + return 0; +} diff --git a/drivers/net/wireless/mediatek/mt7932/station.c b/drivers/net/wireless/mediatek/mt7932/station.c new file mode 100644 index 0000000000000..c9d388e50d8d5 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/station.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include "mt7932.h" + +/* Report the owned, validated EE4d association, not an AP advertisement + * or a transient off-channel scan. + */ +int mt_get_channel(struct wiphy *wiphy, struct wireless_dev *wdev, + unsigned int link_id, struct cfg80211_chan_def *chandef) +{ + struct mt7932 *m = mt_from_wiphy(wiphy); + unsigned long flags; + int ret = -ENODATA; + bool ht; + + if (wdev != &m->wdev) + return -ENODEV; + if (link_id) + return -ENOENT; + spin_lock_irqsave(&m->response_lock, flags); + if (!m->connected || m->disconnecting || m->stopping || !m->peer_valid) + goto out; + if (!m->connect_center || !m->connect_chandef.chan) + goto out; + ht = cfg80211_find_elem(WLAN_EID_HT_CAPABILITY, m->assoc_request_ies, + m->assoc_request_ie_len) != NULL; + *chandef = m->connect_chandef; + if (!ht && chandef->width == NL80211_CHAN_WIDTH_20) + chandef->width = NL80211_CHAN_WIDTH_20_NOHT; + ret = 0; +out: + spin_unlock_irqrestore(&m->response_lock, flags); + return ret; +} + +/* Caller holds response_lock, which serializes peer publication/retirement. + * These are host Ethernet counters, not firmware ACK/retry or MPDU counters. + * Do not invent rate, retry or failed-transmission data from DMA completion. + */ +static int mt_station_snapshot(struct mt7932 *m, struct station_info *sinfo) +{ + if (!m->connected || m->disconnecting || m->stopping || !m->peer_valid) + return -ENOENT; + + spin_lock(&m->data_lock); + sinfo->connected_time = (jiffies - m->station_connected) / HZ; + sinfo->tx_bytes = m->station_tx_bytes; + sinfo->rx_bytes = m->station_rx_bytes; + sinfo->tx_packets = m->station_tx_packets; + sinfo->rx_packets = m->station_rx_packets; + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_CONNECTED_TIME) | + BIT_ULL(NL80211_STA_INFO_TX_BYTES64) | + BIT_ULL(NL80211_STA_INFO_RX_BYTES64) | + BIT_ULL(NL80211_STA_INFO_TX_PACKETS) | + BIT_ULL(NL80211_STA_INFO_RX_PACKETS); + if (m->station_signal_valid) { + sinfo->signal = m->station_signal; + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL); + } + if (m->station_rx_rate_valid && + time_before(jiffies, m->station_rx_rate_time + HZ)) { + sinfo->rxrate = m->station_rx_rate; + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_BITRATE); + } + /* CID85 echoes station identity but its rate words come from a shared + * TX vector without a peer argument, atomicity or freshness attestation. + * Do not publish that vector as this station's current TX bitrate. + */ + spin_unlock(&m->data_lock); + return 0; +} + +int mt_get_station(struct wiphy *wiphy, struct wireless_dev *wdev, + const u8 *mac, struct station_info *sinfo) +{ + struct mt7932 *m = mt_from_wiphy(wiphy); + unsigned long flags; + int ret = -ENOENT; + + if (wdev != &m->wdev) + return -ENODEV; + spin_lock_irqsave(&m->response_lock, flags); + if (ether_addr_equal(mac, m->connect_bssid)) + ret = mt_station_snapshot(m, sinfo); + spin_unlock_irqrestore(&m->response_lock, flags); + return ret; +} + +int mt_dump_station(struct wiphy *wiphy, struct wireless_dev *wdev, + int idx, u8 *mac, struct station_info *sinfo) +{ + struct mt7932 *m = mt_from_wiphy(wiphy); + unsigned long flags; + int ret; + + if (wdev != &m->wdev) + return -ENODEV; + if (idx != 0) + return -ENOENT; + spin_lock_irqsave(&m->response_lock, flags); + ret = mt_station_snapshot(m, sinfo); + if (!ret) + ether_addr_copy(mac, m->connect_bssid); + spin_unlock_irqrestore(&m->response_lock, flags); + return ret; +} diff --git a/drivers/net/wireless/mediatek/mt7932/station_query.c b/drivers/net/wireless/mediatek/mt7932/station_query.c new file mode 100644 index 0000000000000..8382768898172 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/station_query.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include "mt7932.h" + +/* Research helper, deliberately not called by production station reporting: + * the reply identity is per-peer, but its vector is shared and unattributed. + * Retained while qualifying a real per-peer rate source, not a public metric. + * One serialized query for the owned peer. Keep the command owner until the + * reply has been checked; disconnect may close admission in IRQ context but + * cannot reuse BSS/station resources across this critical section. + */ +void mt_station_update_rate(struct mt7932 *m, const u8 *mac) +{ + struct mt7932_event event; + struct rate_info rate; + u8 body[28] = {}, peer[ETH_ALEN]; + u32 generation; + unsigned long flags; + bool valid = false; + int ret; + + mutex_lock(&m->command_mutex); + spin_lock_irqsave(&m->response_lock, flags); + if (!m->connected || m->disconnecting || !m->peer_valid || !mt_rf_allowed(m) || + m->station_rate_query_failed || (mac && !ether_addr_equal(mac, m->connect_bssid)) || + time_before(jiffies, m->station_rate_query_time + HZ / 2)) { + spin_unlock_irqrestore(&m->response_lock, flags); + goto unlock; + } + generation = m->connection_generation; + ether_addr_copy(peer, m->connect_bssid); + body[0] = m->peer_ids.station[0]; + memcpy(body + 4, peer, ETH_ALEN); + m->station_rate_query_time = jiffies; + spin_unlock_irqrestore(&m->response_lock, flags); + + ret = mt_request(m, 0x85, true, false, true, body, sizeof(body)); + if (!ret) + ret = mt7932_event_parse(m->reply, m->reply_length, &event); + if (!ret) + valid = mt7932_tx_rate_reply(&event, m->seq, peer, m->phy_cap[4], &rate); + if (valid && (rate.bw != RATE_INFO_BW_20 || + (m->connect_channel > 14 && !rate.flags && + (rate.legacy == 10 || rate.legacy == 20 || rate.legacy == 55 || rate.legacy == 110)))) + valid = false; + + spin_lock_irqsave(&m->response_lock, flags); + if (generation == m->connection_generation && m->connected && + !m->disconnecting && m->peer_valid && ether_addr_equal(peer, m->connect_bssid)) { + /* A timed-out command must not turn every station dump into another + * five-second wait. A new connection/module epoch may try again. + */ + if (ret) + m->station_rate_query_failed = true; + spin_lock(&m->data_lock); + m->station_tx_rate_valid = valid; + if (valid) { + m->station_tx_rate = rate; + m->station_tx_rate_time = jiffies; + } + spin_unlock(&m->data_lock); + } + spin_unlock_irqrestore(&m->response_lock, flags); +unlock: + mutex_unlock(&m->command_mutex); +} From 5c479df356453c18d2f42807f42337cf2697f0e0 Mon Sep 17 00:00:00 2001 From: Ace <7362641+Acelogic@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:24:21 -0400 Subject: [PATCH 0033/2680] wifi: mt7932: admit Neo config and NetworkManager station requests Accept the original 64-record configuration alongside the validated 65-record variant, preserving record bounds, unique keys and required-key validation. Keep the firmware-native wildcard scan IEs and zero directed-scan IEs. Forwarding normal NetworkManager extras stalled this firmware campaign; arbitrary caller probe-IE carriage remains unsupported. Preserve input TLV validation rather than passing malformed input to firmware. Accept the selected first WPA2 PSK AKM when NetworkManager also supplies trailing alternatives and advertise PSK explicitly. Do not choose PSK if the caller selected another first suite; retain PMK, WPA2, CCMP and MFP checks. Physical J700 module aefb08e7cde9398cdce9ca40e29374950dfe989d138813c203cd2f264e44c7f4: checked module reset/reload, scan, WPA2, DHCP, gateway, DNS and verified HTTPS on 2.4 and 5 GHz; 5 GHz traffic concurrent with Bluetooth discovery and HCI off/on passed 54/54 gateway replies and 18 HTTPS transfers. The 2.4 GHz coexistence run recorded 53/54 replies and is not a zero-loss pass. Co-authored-by: DJ <43033058+DjDeveloperr@users.noreply.github.com> Co-authored-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Co-authored-by: Eryk Wieliczko --- drivers/net/wireless/mediatek/mt7932/link.c | 2 +- drivers/net/wireless/mediatek/mt7932/scan.h | 11 ++++++----- drivers/net/wireless/mediatek/mt7932/startup.c | 14 +++++++++----- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index da708907b901e..8caab22dbbb34 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -539,7 +539,7 @@ int mt_connect(struct wiphy *wiphy, struct net_device *netdev, if (!open && (params->crypto.wpa_versions != NL80211_WPA_VERSION_2 || params->crypto.n_ciphers_pairwise != 1 || params->crypto.ciphers_pairwise[0] != WLAN_CIPHER_SUITE_CCMP || params->crypto.cipher_group != WLAN_CIPHER_SUITE_CCMP || - params->crypto.n_akm_suites != 1 || params->crypto.akm_suites[0] != WLAN_AKM_SUITE_PSK || + !params->crypto.n_akm_suites || params->crypto.akm_suites[0] != WLAN_AKM_SUITE_PSK || !params->crypto.psk || params->mfp == NL80211_MFP_REQUIRED || !params->ssid_len || params->ssid_len > 32)) return -EOPNOTSUPP; diff --git a/drivers/net/wireless/mediatek/mt7932/scan.h b/drivers/net/wireless/mediatek/mt7932/scan.h index 3572129118be2..4899cbb6c1654 100644 --- a/drivers/net/wireless/mediatek/mt7932/scan.h +++ b/drivers/net/wireless/mediatek/mt7932/scan.h @@ -83,7 +83,10 @@ static inline void mt7932_scan_disconnected_passive(u8 body[0x4d4]) put_unaligned_le16(233, body + 0x4ca); } -/* Bounded standard caller-IE carriage, not the reference's silent override. */ +/* This firmware requires its native wildcard template. Directed scans use + * no probe IEs. Forwarding the normal supplicant extras stalled the campaign + * on J700; arbitrary caller probe-IE carriage remains unsupported here. + */ static inline int mt7932_scan_options(u8 body[0x4d4], const u8 *ssid, size_t ssid_length, const u8 *ies, size_t ie_length) { @@ -102,11 +105,9 @@ static inline int mt7932_scan_options(u8 body[0x4d4], const u8 *ssid, put_unaligned_le32(ssid_length, body + 8); memcpy(body + 12, ssid, ssid_length); } - if (ie_length || ssid_length) { + if (ssid_length) { memset(body + 0xe2, 0, 600); - put_unaligned_le16(ie_length, body + 0xe0); - if (ie_length) - memcpy(body + 0xe2, ies, ie_length); + put_unaligned_le16(0, body + 0xe0); } return 0; } diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index a6f71f80b4b6c..d189f4ad13943 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -30,20 +30,21 @@ static int mt_stock_config(struct mt7932 *m) if (ret) return ret; ret = -EINVAL; - if (file->size != 8 + 65 * 68 || memcmp(file->data, "J7CF", 4) || - get_unaligned_le32(file->data + 4) != 65) + if (file->size < 8 || memcmp(file->data, "J7CF", 4)) + goto out; + count = get_unaligned_le32(file->data + 4); + if ((count != 64 && count != 65) || file->size != 8 + count * 68) goto out; /* * Each absent normalized key appends one record. Reserve for all of * them, including structurally valid packages missing every such key. */ - records = kcalloc(65 + ARRAY_SIZE(updates), sizeof(*records), GFP_KERNEL); + records = kcalloc(count + ARRAY_SIZE(updates), sizeof(*records), GFP_KERNEL); if (!records) { ret = -ENOMEM; goto out; } - memcpy(records, file->data + 8, 65 * 68); - count = 65; + memcpy(records, file->data + 8, count * 68); for (i = 0; i < count; i++) { if (records[i].type != 3 || !records[i].key_len || records[i].key_len >= 32 || records[i].value_len >= 32 || records[i].reserved || @@ -404,6 +405,7 @@ int mt_register_regulatory_gate(struct mt7932 *m) { static const int rates[] = {10,20,55,110,60,90,120,180,240,360,480,540}; static const u32 ciphers[] = { WLAN_CIPHER_SUITE_CCMP }; + static const u32 akms[] = { WLAN_AKM_SUITE_PSK }; unsigned int i; int ret; @@ -419,6 +421,8 @@ int mt_register_regulatory_gate(struct mt7932 *m) m->wiphy->signal_type = CFG80211_SIGNAL_TYPE_MBM; m->wiphy->cipher_suites = ciphers; m->wiphy->n_cipher_suites = ARRAY_SIZE(ciphers); + m->wiphy->akm_suites = akms; + m->wiphy->n_akm_suites = ARRAY_SIZE(akms); wiphy_ext_feature_set(m->wiphy, NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK); m->wiphy->max_scan_ssids = 1; m->wiphy->max_scan_ie_len = 600; From a75c5e95db10e595da71a84d880eeb775ac4892f Mon Sep 17 00:00:00 2001 From: Ace <7362641+Acelogic@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:24:23 -0400 Subject: [PATCH 0034/2680] Bluetooth: add opt-in J700 MT7932 PCIe transport experiment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the standalone B1 firmware/IPC/HCI transport exercised on the physical Neo. Firmware, board calibration, PTX and the unit address are loaded from local firmware files, not embedded in source. Keep the probe gate closed until explicitly admitted, retain DMA ownership on terminal failures, and exclude suspend/kexec configurations. Select the firmware loader dependency. The exact C implementation passed HCI setup, discovery, normal HCI power cycling, two bonded AirPods reconnects and an audible AAC test on the earlier 7.1.12 candidate. The integrated Wi-Fi candidate additionally passed Bluetooth discovery/power cycling with simultaneous 5 GHz traffic. Concurrent headset audio was not repeated; no AirPods were present in the new discovery runs. This remains draft: PCI removal lacks a complete teardown contract and the software queue limit is not HCI backpressure. Suppressing bind attributes does not make device removal safe. Do not claim production lifecycle support. Neo radio research and bring-up credit: Eryk Wieliczko (erykwieliczko), DJ (DjDeveloperr), Ace (Acelogic), and Ryan Murray. [ryan: adapted to 7.2.10 — 7.2 changed of_map_id(); look the RID up with of_map_iommu_id() (same iommu-map and iommu-map-mask, cell count from #iommu-cells), as Asahi's apple_pcie_enable_device() now does] Co-authored-by: DJ <43033058+DjDeveloperr@users.noreply.github.com> Co-authored-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Co-authored-by: Eryk Wieliczko --- drivers/bluetooth/Kconfig | 12 + drivers/bluetooth/Makefile | 2 + drivers/bluetooth/mt7932_bt_pcie.c | 1162 ++++++++++++++++++++++++++++ 3 files changed, 1176 insertions(+) create mode 100644 drivers/bluetooth/mt7932_bt_pcie.c diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index 4e8c24d757e9a..7ac07459925dc 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -23,6 +23,18 @@ config BT_MTK tristate select FW_LOADER +config BT_MTK7932_PCIE + bool "J700 MT7932 PCIe Bluetooth experiment" + depends on BT=y && PCI && ARCH_APPLE && IOMMU_DMA + depends on !PM_SLEEP && !KEXEC_CORE + select FW_LOADER + help + Build the standalone J700 MT7932 transport experiment. Its probe gate + defaults closed so board firmware can remain on the real root filesystem. + It retains all DMA memory after activation, including terminal failures, + until a full external reset. Unbind and automatic recovery are excluded. + This does not qualify suspend or physical removal. + config BT_HCIBTUSB tristate "HCI USB driver" depends on USB diff --git a/drivers/bluetooth/Makefile b/drivers/bluetooth/Makefile index bafc26250b636..d6062472f1621 100644 --- a/drivers/bluetooth/Makefile +++ b/drivers/bluetooth/Makefile @@ -50,3 +50,5 @@ hci_uart-$(CONFIG_BT_HCIUART_AG6XX) += hci_ag6xx.o hci_uart-$(CONFIG_BT_HCIUART_MRVL) += hci_mrvl.o hci_uart-$(CONFIG_BT_HCIUART_AML) += hci_aml.o hci_uart-objs := $(hci_uart-y) + +obj-$(CONFIG_BT_MTK7932_PCIE) += mt7932_bt_pcie.o diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c new file mode 100644 index 0000000000000..062522f170c3f --- /dev/null +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -0,0 +1,1162 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * J700 MT7932 Bluetooth experiment, reconstructed from the attributed + * protocol contracts and public Linux interfaces. Original transport research: + * DJ (DjDeveloperr), Ace (Acelogic), and Ryan Murray. + * + * The built-in driver retains every device-visible allocation after mastering + * is enabled, including all terminal failures. No unbind, retry, suspend or + * quiescence claim is made. Firmware and board inputs are supplied locally. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define BT7932_DEPTH 128 +#define BT7932_MASK (BT7932_DEPTH - 1) +#define BT7932_CHANNELS 11 +#define BT7932_CONTROL_SIZE 68 +#define BT7932_BAR_SIZE 0x100000 +#define BT7932_DOORBELL 0x19484 +#define BT7932_STATE 0x33804 +#define BT7932_IMAGE_LO 0x33810 +#define BT7932_IMAGE_HI 0x33814 +#define BT7932_IMAGE_LEN 0x33818 +#define BT7932_IMAGE_RESPONSE 0x3381c +#define BT7932_CONTROL 0x33828 +#define BT7932_SLEEP 0x3382c +#define BT7932_CONTEXT_LO 0x33830 +#define BT7932_CONTEXT_HI 0x33834 +#define BT7932_WINDOW_LO 0x3383c +#define BT7932_WINDOW_HI 0x33840 +#define BT7932_WINDOW_SIZE 0x33844 +#define BT7932_LTR 0x33c28 +#define BT7932_TRANSPORT_SELECTOR 0xfe3b4 +#define BT7932_CONNECTION_SELECTOR 0xfe3b8 +#define BT7932_SEMAPHORE 0xc1060 +#define BT7932_SEMAPHORE_RELEASE 0xc1260 +#define BT7932_CHIP_ID 0xc0200 +#define BT7932_BOOT_CONFIG 0x48c +#define BT7932_CONTROL_POOL_SIZE (BT7932_DEPTH * BT7932_CONTROL_SIZE) +#define BT7932_COMPLETION_SIZE (BT7932_DEPTH * 0x18) +#define BT7932_TX_QUEUE_MAX 128 + +#define BT7932_FW_B0 "mediatek/MT7932B0_OS_TypeB_0.1.44.0_241001003711.bin" +#define BT7932_FW_B1 "mediatek/MT7932B1_OS_TypeB_2.0.177.0_260706180356.bin" +#define BT7932_FW_B1_FALLBACK "mediatek/MT7932B1_OS_TypeB_0.1.133.0_260128190103.bin" +#define BT7932_PTX "mediatek/MT7932_PTB_IzubaA_0.1.0.0_20251021141303.ptx" +#define BT7932_CAL "mediatek/j700-mt7932-btcal.bin" +#define BT7932_ADDR "mediatek/j700-mt7932-bdaddr.bin" + +static bool enable; +module_param(enable, bool, 0600); +MODULE_PARM_DESC(enable, "Diagnostic late-probe gate; default closed, no effect on an already bound device"); + +struct bt7932_geometry { + u16 stride; + u16 payload; + u8 doorbell; + u8 class; + u8 packet; + bool rx; +}; + +static const struct bt7932_geometry geometry[BT7932_CHANNELS] = { + [1] = { 0x11c, 268, 21, 0x43, HCI_COMMAND_PKT, false }, + [2] = { 0x11c, 268, 22, 0x43, HCI_EVENT_PKT, true }, + [3] = { 0x40c, 1020, 23, 0xff, HCI_SCODATA_PKT, false }, + [4] = { 0x40c, 1020, 24, 0xff, HCI_SCODATA_PKT, true }, + [5] = { 0x418, 1032, 25, 0x81, HCI_ACLDATA_PKT, false }, + [6] = { 0x418, 1032, 26, 0x81, HCI_ACLDATA_PKT, true }, + [8] = { 0x418, 1032, 27, 0x81, 0, true }, + [9] = { 0x318, 776, 28, 0xc2, HCI_ISODATA_PKT, false }, + [10] = { 0x318, 776, 29, 0xc2, HCI_ISODATA_PKT, true }, +}; + +struct bt7932_ring { + size_t offset; + u16 host; + u16 previous; +}; + +struct bt7932 { + struct pci_dev *pdev; + void __iomem *bar; + struct hci_dev *hdev; + struct mutex lock; + struct workqueue_struct *tx_wq; + struct work_struct tx_work; + struct sk_buff_head tx_queue; + wait_queue_head_t wake_wait; + struct bt7932_ring rings[BT7932_CHANNELS]; + void *arena; + dma_addr_t arena_dma; + size_t arena_size; + size_t context, peripheral, cr_producer, fw_transfer; + size_t cr_consumer, host_transfer, completions, requests, diagnostic; + void *image; + dma_addr_t image_dma; + size_t image_size, image_body; + u8 *calibration, *ptx; + size_t calibration_size, ptx_size; + u8 address[6]; + u8 rom; + u16 cr_previous; + u32 last_sleep; + int irq; + bool enabled, regions, vectors, irq_requested, retained; + bool ipc_ready, opened, awake, waking, fault, registered; + u64 tx_packets, rx_packets, diagnostics; +}; + +static void bt7932_ring_doorbell(struct bt7932 *bt, unsigned int bit) +{ + dma_wmb(); + writel(BIT(bit), bt->bar + BT7932_DOORBELL); +} + +static u16 bt7932_counter(struct bt7932 *bt, size_t base, unsigned int channel) +{ + __le16 *cell = bt->arena + base + channel * sizeof(*cell); + u16 value = le16_to_cpu(READ_ONCE(*cell)) & BT7932_MASK; + + dma_rmb(); + return value; +} + +static void bt7932_publish_counter(struct bt7932 *bt, size_t base, + unsigned int channel, u16 value) +{ + __le16 *cell = bt->arena + base + channel * sizeof(*cell); + + dma_wmb(); + WRITE_ONCE(*cell, cpu_to_le16(value)); +} + +static void bt7932_fault_locked(struct bt7932 *bt, const char *reason) +{ + int channel; + + if (bt->fault) + return; + WRITE_ONCE(bt->fault, true); + dev_err(&bt->pdev->dev, "terminal transport fault: %s; owned DMA retained, external reset required\n", + reason); + for (channel = 0; channel < BT7932_CHANNELS; channel++) + dev_err(&bt->pdev->dev, "ring%u host=%u firmware=%u previous=%u\n", channel, + bt7932_counter(bt, bt->host_transfer, channel), + bt7932_counter(bt, bt->fw_transfer, channel), bt->rings[channel].previous); + pci_clear_master(bt->pdev); + if (bt->irq_requested) + disable_irq_nosync(bt->irq); + wake_up_all(&bt->wake_wait); +} + +static bool bt7932_hci_length(u8 packet, const u8 *data, size_t len) +{ + size_t expected; + + switch (packet) { + case HCI_COMMAND_PKT: + if (len < HCI_COMMAND_HDR_SIZE) + return false; + expected = HCI_COMMAND_HDR_SIZE + data[2]; + break; + case HCI_EVENT_PKT: + if (len < HCI_EVENT_HDR_SIZE) + return false; + expected = HCI_EVENT_HDR_SIZE + data[1]; + break; + case HCI_ACLDATA_PKT: + if (len < HCI_ACL_HDR_SIZE) + return false; + expected = HCI_ACL_HDR_SIZE + get_unaligned_le16(data + 2); + break; + case HCI_SCODATA_PKT: + if (len < HCI_SCO_HDR_SIZE) + return false; + expected = HCI_SCO_HDR_SIZE + data[2]; + break; + case HCI_ISODATA_PKT: + if (len < HCI_ISO_HDR_SIZE) + return false; + expected = HCI_ISO_HDR_SIZE + (get_unaligned_le16(data + 2) & 0x3fff); + break; + default: + return false; + } + return len == expected; +} + +static int bt7932_validate_ptx(const u8 *data, size_t size) +{ + static const u32 types[] = { 0x101, 0x201, 0x301, 0x401 }; + static const u32 sizes[] = { 14, 44, 44, 0 }; + u32 next = 96; + int i; + + if (size != 198 || memcmp(data, "BLOB", 4) || + get_unaligned_le32(data + 4) != 96 || + get_unaligned_le16(data + 8) != 1 || + get_unaligned_le16(data + 10) != ARRAY_SIZE(types) || + get_unaligned_le32(data + 12)) + return -EINVAL; + for (i = 0; i < ARRAY_SIZE(types); i++) { + const u8 *record = data + 16 + i * 20; + u32 offset = get_unaligned_le32(record + 4); + u32 length = get_unaligned_le32(record + 8); + u32 sum = 0, j; + + if (get_unaligned_le32(record) != types[i] || offset != next || + length != sizes[i] || offset > size || length > size - offset || + get_unaligned_le32(record + 16)) + return -EINVAL; + for (j = 0; j < length; j++) + sum += data[offset + j]; + if (sum != get_unaligned_le32(record + 12)) + return -EINVAL; + next = offset + length; + } + return next == size ? 0 : -EINVAL; +} + +static int bt7932_read_inputs(struct bt7932 *bt) +{ + const struct firmware *fw; + const u8 *trailer; + u64 last; + int i, ret; + + if (!bt->rom) { + ret = request_firmware(&fw, BT7932_FW_B0, &bt->pdev->dev); + } else { + ret = firmware_request_nowarn(&fw, BT7932_FW_B1, &bt->pdev->dev); + if (ret == -ENOENT) + ret = request_firmware(&fw, BT7932_FW_B1_FALLBACK, &bt->pdev->dev); + } + if (ret) + return ret; + if (fw->size <= 32 || fw->size - 32 > U32_MAX - 3) { + ret = -EINVAL; + goto release_image; + } + trailer = fw->data + fw->size - 32; + if (memcmp(trailer + 16, "ALPS\x8a\x10\x8a\x10", 8)) { + ret = -EINVAL; + goto release_image; + } + bt->image_body = fw->size - 32; + bt->image_size = ALIGN(bt->image_body, 4); + bt->image = dma_alloc_coherent(&bt->pdev->dev, bt->image_size, + &bt->image_dma, GFP_KERNEL); + if (!bt->image) { + ret = -ENOMEM; + goto release_image; + } + if (check_add_overflow((u64)bt->image_dma, bt->image_size - 1, &last) || + last > DMA_BIT_MASK(32)) { + ret = -ERANGE; + goto release_image; + } + memset(bt->image, 0, bt->image_size); + memcpy(bt->image, fw->data, bt->image_body); +release_image: + release_firmware(fw); + if (ret) + return ret; + ret = request_firmware(&fw, BT7932_CAL, &bt->pdev->dev); + if (ret) + return ret; + if (!fw->size || fw->size > U16_MAX) { + ret = -EINVAL; + } else { + bt->calibration_size = fw->size; + bt->calibration = kmemdup(fw->data, fw->size, GFP_KERNEL); + if (!bt->calibration) + ret = -ENOMEM; + } + release_firmware(fw); + if (ret) + return ret; + ret = request_firmware(&fw, BT7932_PTX, &bt->pdev->dev); + if (ret) + return ret; + ret = bt7932_validate_ptx(fw->data, fw->size); + if (!ret) { + bt->ptx_size = fw->size; + bt->ptx = kmemdup(fw->data, fw->size, GFP_KERNEL); + if (!bt->ptx) + ret = -ENOMEM; + } + release_firmware(fw); + if (ret) + return ret; + ret = request_firmware(&fw, BT7932_ADDR, &bt->pdev->dev); + if (ret) + return ret; + if (fw->size != sizeof(bt->address) || + !memchr_inv(fw->data, 0, fw->size) || !memchr_inv(fw->data, 0xff, fw->size)) { + ret = -EINVAL; + } else { + for (i = 0; i < sizeof(bt->address); i++) + bt->address[i] = fw->data[sizeof(bt->address) - i - 1]; + } + release_firmware(fw); + return ret; +} + +static int bt7932_reserve(size_t *total, size_t bytes, size_t *offset) +{ + size_t aligned; + + if (check_add_overflow(*total, (size_t)63, &aligned)) + return -EOVERFLOW; + aligned &= ~(size_t)63; + *offset = aligned; + if (check_add_overflow(aligned, bytes, total)) + return -EOVERFLOW; + return 0; +} + +static int bt7932_arena(struct bt7932 *bt) +{ + struct { + size_t bytes; + size_t *offset; + } regions[] = { + { 104, &bt->context }, { 16, &bt->peripheral }, + { 4, &bt->cr_producer }, { 24, &bt->fw_transfer }, + { 4, &bt->cr_consumer }, { 24, &bt->host_transfer }, + { BT7932_COMPLETION_SIZE, &bt->completions }, + { BT7932_CONTROL_POOL_SIZE, &bt->requests }, + { 0x1400, &bt->diagnostic }, + }; + size_t total = 0; + u8 *context; + u64 last; + int i, ret; + + for (i = 0; i < ARRAY_SIZE(regions); i++) { + ret = bt7932_reserve(&total, regions[i].bytes, regions[i].offset); + if (ret) + return ret; + } + for (i = 1; i < BT7932_CHANNELS; i++) { + if (!geometry[i].stride) + continue; + ret = bt7932_reserve(&total, geometry[i].stride * BT7932_DEPTH, + &bt->rings[i].offset); + if (ret) + return ret; + } + if (total > U32_MAX - PAGE_SIZE) + return -EOVERFLOW; + bt->arena_size = PAGE_ALIGN(total); + bt->arena = dma_alloc_coherent(&bt->pdev->dev, bt->arena_size, + &bt->arena_dma, GFP_KERNEL); + if (!bt->arena) + return -ENOMEM; + if (!IS_ALIGNED(bt->arena_dma, 64) || + check_add_overflow((u64)bt->arena_dma, bt->arena_size - 1, &last) || + last > DMA_BIT_MASK(32)) + return -ERANGE; + memset(bt->arena, 0, bt->arena_size); + context = bt->arena + bt->context; + put_unaligned_le64(bt->arena_dma + bt->peripheral, context + 0x08); + put_unaligned_le64(bt->arena_dma + bt->cr_producer, context + 0x10); + put_unaligned_le64(bt->arena_dma + bt->fw_transfer, context + 0x18); + put_unaligned_le64(bt->arena_dma + bt->cr_consumer, context + 0x20); + put_unaligned_le64(bt->arena_dma + bt->host_transfer, context + 0x28); + put_unaligned_le32(0x000b0001, context + 0x30); + put_unaligned_le64(bt->arena_dma + bt->completions, context + 0x34); + put_unaligned_le64(bt->arena_dma + bt->requests, context + 0x3c); + put_unaligned_le16(BT7932_DEPTH, context + 0x44); + put_unaligned_le16(BT7932_DEPTH, context + 0x46); + context[0x51] = 13; + context[0x53] = 2; + dma_wmb(); + dev_info(&bt->pdev->dev, "owned SID2 arena=%pad bytes=%zu image=%pad bytes=%zu, both low32\n", + &bt->arena_dma, bt->arena_size, &bt->image_dma, bt->image_size); + return 0; +} + +/* Caller owns lock; no HCI command wait can run from the IRQ thread. */ +static int bt7932_sleep_locked(struct bt7932 *bt) +{ + __le32 *notification = bt->arena + bt->peripheral + 8; + u32 value; + + dma_rmb(); + value = le32_to_cpu(READ_ONCE(*notification)); + if (value > 1) + return -EPROTO; + if (!value) { + if (bt->last_sleep != value || bt->waking) { + writel(0, bt->bar + BT7932_SLEEP); + bt7932_ring_doorbell(bt, 12); + } + WRITE_ONCE(bt->awake, true); + bt->waking = false; + wake_up_all(&bt->wake_wait); + } else { + WRITE_ONCE(bt->awake, false); + if (!bt->waking && bt->last_sleep != value) { + writel(1, bt->bar + BT7932_SLEEP); + bt7932_ring_doorbell(bt, 12); + } + } + bt->last_sleep = value; + return 0; +} + +/* On success returns with lock held, covering the final check/publication. */ +static int bt7932_wake_and_lock(struct bt7932 *bt) +{ + unsigned long deadline = jiffies + HZ; + unsigned long now; + int ret; + + for (;;) { + mutex_lock(&bt->lock); + if (bt->fault || !bt->opened) { + mutex_unlock(&bt->lock); + return -EIO; + } + ret = bt7932_sleep_locked(bt); + if (ret) + goto fail; + if (bt->awake) + return 0; + if (!bt->waking) { + bt->waking = true; + writel(2, bt->bar + BT7932_SLEEP); + bt7932_ring_doorbell(bt, 12); + } + mutex_unlock(&bt->lock); + now = jiffies; + if (time_after_eq(now, deadline)) { + mutex_lock(&bt->lock); + ret = -ETIMEDOUT; + goto fail; + } + wait_event_timeout(bt->wake_wait, + READ_ONCE(bt->awake) || READ_ONCE(bt->fault) || !READ_ONCE(bt->opened), + min_t(unsigned long, msecs_to_jiffies(10), deadline - now)); + } +fail: + bt7932_fault_locked(bt, "wake handshake failed"); + mutex_unlock(&bt->lock); + return ret; +} + +static int bt7932_tx_channel(u8 packet) +{ + switch (packet) { + case HCI_COMMAND_PKT: return 1; + case HCI_SCODATA_PKT: return 3; + case HCI_ACLDATA_PKT: return 5; + case HCI_ISODATA_PKT: return 9; + default: return -EINVAL; + } +} + +static int bt7932_tx_progress(struct bt7932 *bt, unsigned int channel) +{ + struct bt7932_ring *ring = &bt->rings[channel]; + u16 consumer = bt7932_counter(bt, bt->fw_transfer, channel); + u16 completed = (consumer - ring->previous) & BT7932_MASK; + u16 outstanding = (ring->host - ring->previous) & BT7932_MASK; + + if (completed > outstanding) + return -EPROTO; + ring->previous = consumer; + return 0; +} + +static void bt7932_tx_work(struct work_struct *work) +{ + struct bt7932 *bt = container_of(work, struct bt7932, tx_work); + struct sk_buff *skb; + + while ((skb = skb_dequeue(&bt->tx_queue))) { + int channel = bt7932_tx_channel(hci_skb_pkt_type(skb)); + struct bt7932_ring *ring; + u16 consumer; + u8 *descriptor; + + if (bt7932_wake_and_lock(bt)) { + kfree_skb(skb); + skb_queue_purge(&bt->tx_queue); + return; + } + ring = &bt->rings[channel]; + if (bt7932_tx_progress(bt, channel)) { + bt7932_fault_locked(bt, "TX consumer exceeded outstanding descriptors"); + mutex_unlock(&bt->lock); + kfree_skb(skb); + skb_queue_purge(&bt->tx_queue); + return; + } + consumer = ring->previous; + if (((ring->host + 1) & BT7932_MASK) == consumer) { + skb_queue_head(&bt->tx_queue, skb); + mutex_unlock(&bt->lock); + return; + } + descriptor = bt->arena + ring->offset + ring->host * geometry[channel].stride; + memset(descriptor, 0, geometry[channel].stride); + descriptor[0] = 2; + descriptor[1] = skb->len; + descriptor[2] = skb->len >> 8; + descriptor[3] = skb->len >> 16; + put_unaligned_le16(ring->host, descriptor + 12); + descriptor[15] = 2; + memcpy(descriptor + 16, skb->data, skb->len); + ring->host = (ring->host + 1) & BT7932_MASK; + bt7932_publish_counter(bt, bt->host_transfer, channel, ring->host); + bt7932_ring_doorbell(bt, geometry[channel].doorbell); + bt->tx_packets++; + bt->hdev->stat.byte_tx += skb->len; + mutex_unlock(&bt->lock); + kfree_skb(skb); + } +} + +static void bt7932_offer(struct bt7932 *bt, unsigned int channel, u16 slot) +{ + u8 *descriptor = bt->arena + bt->rings[channel].offset + slot * geometry[channel].stride; + + memset(descriptor, 0, geometry[channel].stride); + put_unaligned_le16(slot, descriptor + 12); +} + +static int bt7932_rx_locked(struct bt7932 *bt, unsigned int channel) +{ + const struct bt7932_geometry *geo = &geometry[channel]; + struct bt7932_ring *ring = &bt->rings[channel]; + u16 completed = bt7932_counter(bt, bt->fw_transfer, channel); + u16 count = (completed - ring->previous) & BT7932_MASK; + u16 offered = (ring->host - ring->previous) & BT7932_MASK; + u16 slot = ring->previous; + int i; + + if (count > offered) + return -EPROTO; + for (i = 0; i < count; i++) { + u8 *descriptor = bt->arena + ring->offset + slot * geo->stride; + u32 length = descriptor[1] | descriptor[2] << 8 | descriptor[3] << 16; + struct sk_buff *skb; + + if (descriptor[0] != 2 || get_unaligned_le16(descriptor + 12) != slot || + descriptor[14] || descriptor[15] != 4 || + memchr_inv(descriptor + 4, 0, 8) || length > geo->payload) + return -EPROTO; + if (geo->packet && !bt7932_hci_length(geo->packet, descriptor + 16, length)) + return -EPROTO; + if (geo->packet && bt->opened) { + skb = bt_skb_alloc(length, GFP_KERNEL); + if (!skb) + return -ENOMEM; + hci_skb_pkt_type(skb) = geo->packet; + skb_put_data(skb, descriptor + 16, length); + bt->hdev->stat.byte_rx += length; + bt->rx_packets++; + hci_recv_frame(bt->hdev, skb); + } else if (!geo->packet) { + bt->diagnostics++; + } + slot = (slot + 1) & BT7932_MASK; + } + for (i = 0; i < count; i++) { + bt7932_offer(bt, channel, ring->host); + ring->host = (ring->host + 1) & BT7932_MASK; + } + ring->previous = completed; + if (count) { + bt7932_publish_counter(bt, bt->host_transfer, channel, ring->host); + bt7932_ring_doorbell(bt, geo->doorbell); + } + return 0; +} + +static irqreturn_t bt7932_irq_thread(int irq, void *data) +{ + struct bt7932 *bt = data; + u16 producer; + int channel, ret; + + mutex_lock(&bt->lock); + if (!bt->ipc_ready || bt->fault) + goto out; + if (READ_ONCE(*(u8 *)(bt->arena + bt->peripheral + 4)) == 4) { + bt7932_fault_locked(bt, "firmware entered fault state4"); + goto out; + } + ret = bt7932_sleep_locked(bt); + if (ret) { + bt7932_fault_locked(bt, "invalid sleep notification"); + goto out; + } + producer = bt7932_counter(bt, bt->cr_producer, 0); + if (producer != bt->cr_previous) { + /* Counter retirement only; no invented per-record ACK interpretation. */ + bt7932_publish_counter(bt, bt->cr_consumer, 0, producer); + bt->cr_previous = producer; + bt7932_ring_doorbell(bt, 19); + } + for (channel = 1; channel < BT7932_CHANNELS; channel++) { + if (!geometry[channel].stride) + continue; + ret = geometry[channel].rx ? bt7932_rx_locked(bt, channel) : + bt7932_tx_progress(bt, channel); + if (ret) { + bt7932_fault_locked(bt, "invalid ring completion or RX allocation failure"); + goto out; + } + } + if (!skb_queue_empty(&bt->tx_queue)) + queue_work(bt->tx_wq, &bt->tx_work); +out: + mutex_unlock(&bt->lock); + return IRQ_HANDLED; +} + +static int bt7932_download(struct bt7932 *bt) +{ + unsigned long deadline; + u32 value, boot; + int ret; + + writel(0x7000188a, bt->bar + BT7932_TRANSPORT_SELECTOR); + writel(1, bt->bar + BT7932_LTR); + dma_wmb(); + writel(lower_32_bits(bt->image_dma), bt->bar + BT7932_IMAGE_LO); + writel(upper_32_bits(bt->image_dma), bt->bar + BT7932_IMAGE_HI); + writel(bt->image_body, bt->bar + BT7932_IMAGE_LEN); + writel(0x188d1807, bt->bar + BT7932_CONNECTION_SELECTOR); + ret = readl_poll_timeout(bt->bar + BT7932_SEMAPHORE, value, + value == U32_MAX || value & 1, 1000, 5000000); + if (ret || value == U32_MAX) + return ret ?: -EIO; + bt7932_ring_doorbell(bt, 11); + deadline = jiffies + HZ; + for (;;) { + value = readl(bt->bar + BT7932_IMAGE_RESPONSE); + ret = pci_read_config_dword(bt->pdev, BT7932_BOOT_CONFIG, &boot); + if (ret || value == U32_MAX || boot == U32_MAX) + return -EIO; + if (value == 1 && (boot & 0xf) == 2) + break; + if (time_after_eq(jiffies, deadline)) + return -ETIMEDOUT; + usleep_range(1000, 2000); + } + writel(0x188d1807, bt->bar + BT7932_CONNECTION_SELECTOR); + writel(1, bt->bar + BT7932_SEMAPHORE_RELEASE); + dev_info(&bt->pdev->dev, "BT_IMAGE_ACCEPTED response1 boot0->2; image remains retained\n"); + return 0; +} + +static int bt7932_ipc_state(struct bt7932 *bt, u8 *state) +{ + u32 value; + + dma_rmb(); + *state = READ_ONCE(*(u8 *)(bt->arena + bt->peripheral + 4)); + if (*state <= 1) { + value = readl(bt->bar + BT7932_STATE); + if (value == U32_MAX) + return -EIO; + /* The reviewed early-state rule replaces memory with the BAR byte. */ + *state = value & 0xff; + } + return 0; +} + +static int bt7932_start_ipc(struct bt7932 *bt) +{ + unsigned long deadline = jiffies + 10 * HZ; + bool init_sent = false, context_sent = false; + u64 context = bt->arena_dma + bt->context; + u8 state; + int ret; + + while (time_before(jiffies, deadline)) { + ret = bt7932_ipc_state(bt, &state); + if (ret) + return ret; + if (!state && !init_sent) { + writel(1, bt->bar + BT7932_CONTROL); + bt7932_ring_doorbell(bt, 13); + init_sent = true; + } else if (state == 1 && !context_sent) { + dma_wmb(); + writel(lower_32_bits(context), bt->bar + BT7932_CONTEXT_LO); + writel(upper_32_bits(context), bt->bar + BT7932_CONTEXT_HI); + writel(lower_32_bits(bt->arena_dma), bt->bar + BT7932_WINDOW_LO); + writel(upper_32_bits(bt->arena_dma), bt->bar + BT7932_WINDOW_HI); + writel(bt->arena_size, bt->bar + BT7932_WINDOW_SIZE); + writel(2, bt->bar + BT7932_CONTROL); + bt7932_ring_doorbell(bt, 13); + context_sent = true; + deadline = min(deadline, jiffies + HZ); + } else if (state == 2) { + return context_sent ? 0 : -EPROTO; + } else if (state > 2) { + return -EPROTO; + } + usleep_range(1000, 2000); + } + return -ETIMEDOUT; +} + +static void bt7932_channels(struct bt7932 *bt) +{ + unsigned int channel, index; + + /* Ten control records remain owned and are never reused. */ + for (channel = 1; channel < BT7932_CHANNELS; channel++) { + const struct bt7932_geometry *geo = &geometry[channel]; + u16 producer = channel - 1; + u8 *request = bt->arena + bt->requests + producer * BT7932_CONTROL_SIZE; + + if (geo->stride) { + put_unaligned_le16(0x34, request + 1); + put_unaligned_le16(producer, request + 0x0c); + request[0x10] = 1; + request[0x12] = geo->class; + request[0x13] = geo->class == 0x81; + put_unaligned_le32(channel | channel << 16, request + 0x14); + put_unaligned_le64(bt->arena_dma + bt->rings[channel].offset, + request + 0x18); + put_unaligned_le32(BT7932_DEPTH, request + 0x28); + put_unaligned_le16(geo->doorbell, request + 0x2c); + put_unaligned_le16(0x59, request + 0x2e); + put_unaligned_le32(4, request + 0x30); + put_unaligned_le32(geo->doorbell, request + 0x34); + put_unaligned_le16(1000, request + 0x3c); + } + bt7932_publish_counter(bt, bt->host_transfer, channel, 0); + bt7932_publish_counter(bt, bt->fw_transfer, channel, 0); + bt7932_publish_counter(bt, bt->host_transfer, 0, producer + 1); + bt7932_ring_doorbell(bt, 20); + } + for (channel = 1; channel < BT7932_CHANNELS; channel++) { + if (!geometry[channel].rx) + continue; + for (index = 0; index < BT7932_DEPTH - 1; index++) + bt7932_offer(bt, channel, index); + bt->rings[channel].host = BT7932_DEPTH - 1; + bt7932_publish_counter(bt, bt->host_transfer, channel, BT7932_DEPTH - 1); + bt7932_ring_doorbell(bt, geometry[channel].doorbell); + } + /* The reviewed cold IPC transition starts ACTIVE; no initial sleep ACK. */ + bt->last_sleep = 0; + WRITE_ONCE(bt->awake, true); + WRITE_ONCE(bt->ipc_ready, true); + dev_info(&bt->pdev->dev, "BT_IPC_READY ten retained advertisements;127 RX offers per channel\n"); +} + +static int bt7932_open(struct hci_dev *hdev) +{ + struct bt7932 *bt = hci_get_drvdata(hdev); + int ret = 0; + + mutex_lock(&bt->lock); + if (bt->fault || !bt->ipc_ready) + ret = -EIO; + else + WRITE_ONCE(bt->opened, true); + mutex_unlock(&bt->lock); + return ret; +} + +static int bt7932_flush(struct hci_dev *hdev) +{ + struct bt7932 *bt = hci_get_drvdata(hdev); + + cancel_work_sync(&bt->tx_work); + skb_queue_purge(&bt->tx_queue); + return 0; +} + +static int bt7932_close(struct hci_dev *hdev) +{ + struct bt7932 *bt = hci_get_drvdata(hdev); + + WRITE_ONCE(bt->opened, false); + wake_up_all(&bt->wake_wait); + return bt7932_flush(hdev); +} + +static int bt7932_send(struct hci_dev *hdev, struct sk_buff *skb) +{ + struct bt7932 *bt = hci_get_drvdata(hdev); + int channel = bt7932_tx_channel(hci_skb_pkt_type(skb)); + unsigned long flags; + int ret; + + if (READ_ONCE(bt->fault) || !READ_ONCE(bt->opened)) + return -EIO; + if (channel < 0 || skb->len > geometry[channel].payload) + return -EMSGSIZE; + ret = skb_linearize(skb); + if (ret) + return ret; + if (!bt7932_hci_length(hci_skb_pkt_type(skb), skb->data, skb->len)) + return -EINVAL; + spin_lock_irqsave(&bt->tx_queue.lock, flags); + if (bt->tx_queue.qlen >= BT7932_TX_QUEUE_MAX) { + spin_unlock_irqrestore(&bt->tx_queue.lock, flags); + return -EBUSY; + } + __skb_queue_tail(&bt->tx_queue, skb); + spin_unlock_irqrestore(&bt->tx_queue.lock, flags); + queue_work(bt->tx_wq, &bt->tx_work); + return 0; +} + +static int bt7932_setup_chunks(struct hci_dev *hdev, u8 kind, + const u8 *data, size_t length) +{ + u8 command[244]; + size_t offset = 0; + int ret; + + while (offset < length) { + size_t chunk = min_t(size_t, length - offset, sizeof(command) - 4); + + command[0] = 1; + command[1] = 4; + command[2] = kind; + command[3] = offset + chunk < length; + memcpy(command + 4, data + offset, chunk); + ret = __hci_cmd_sync_status(hdev, 0xfdd0, chunk + 4, command, HCI_INIT_TIMEOUT); + memzero_explicit(command, sizeof(command)); + if (ret) { + dev_err(&hdev->dev, "BT_SETUP_CHUNK_FAILED kind=%02x offset=%zu status=%d\n", + kind, offset, ret); + return ret; + } + offset += chunk; + } + return 0; +} + +static int bt7932_setup(struct hci_dev *hdev) +{ + struct bt7932 *bt = hci_get_drvdata(hdev); + const char *stage = "Reset"; + u16 opcode = HCI_OP_RESET; + int ret; + + ret = __hci_cmd_sync_status(hdev, HCI_OP_RESET, 0, NULL, HCI_INIT_TIMEOUT); + if (ret) + goto fault; + stage = "calibration"; + opcode = 0xfdd0; + ret = bt7932_setup_chunks(hdev, 0x30, bt->calibration, bt->calibration_size); + if (ret) + goto fault; + stage = "PTX"; + ret = bt7932_setup_chunks(hdev, 0x20, bt->ptx, bt->ptx_size); + if (ret) + goto fault; + stage = "address"; + opcode = 0xfc1a; + ret = __hci_cmd_sync_status(hdev, 0xfc1a, sizeof(bt->address), bt->address, + HCI_INIT_TIMEOUT); + if (ret) + goto fault; + dev_info(&bt->pdev->dev, "BT_HCI_SETUP_OK Reset/calibration/PTX/address completed in order\n"); + return 0; +fault: + dev_err(&bt->pdev->dev, "BT_HCI_SETUP_FAILED stage=%s opcode=%04x status=%d\n", + stage, opcode, ret); + mutex_lock(&bt->lock); + bt7932_fault_locked(bt, "HCI setup command failed; no retry"); + mutex_unlock(&bt->lock); + return ret; +} + +static void bt7932_reset(struct hci_dev *hdev) +{ + struct bt7932 *bt = hci_get_drvdata(hdev); + + mutex_lock(&bt->lock); + bt7932_fault_locked(bt, "HCI requested recovery; reset is not qualified"); + mutex_unlock(&bt->lock); +} + +static void bt7932_hw_error(struct hci_dev *hdev, u8 code) +{ + bt7932_reset(hdev); +} + +static int bt7932_iommu(struct bt7932 *bt) +{ + struct device *dev = &bt->pdev->dev; + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + struct iommu_fwspec *spec = dev_iommu_fwspec_get(dev); + struct pci_bus *root = bt->pdev->bus; + struct of_phandle_args map = {}; + struct device_node *target = NULL; + u32 rid = pci_dev_id(bt->pdev), sid = U32_MAX; + int ret; + + if (!domain || (domain->type != IOMMU_DOMAIN_DMA && + domain->type != IOMMU_DOMAIN_DMA_FQ)) { + dev_err(dev, "BT_IOMMU_REJECT translated domain required type=%u\n", + domain ? domain->type : 0); + return -ENODEV; + } + if (!spec || !is_of_node(spec->iommu_fwnode)) { + dev_err(dev, "BT_IOMMU_REJECT missing OF provider\n"); + return -ENODEV; + } + while (root->parent) + root = root->parent; + if (!of_device_is_compatible(root->dev.of_node, "apple,t8140-pcie")) { + dev_err(dev, "BT_IOMMU_REJECT unexpected host node\n"); + return -ENODEV; + } + /* Apple DART stores IDs in private stream_maps, not fwspec->ids. */ + ret = of_map_iommu_id(root->dev.of_node, rid, &map); + target = map.np; + if (!ret && map.args_count) + sid = map.args[0]; + if (ret || !target || sid != 2 || + of_fwnode_handle(target) != spec->iommu_fwnode || + !of_device_is_compatible(target, "apple,t8140-dart")) { + dev_err(dev, "BT_IOMMU_REJECT RID=%03x SID=%u map_status=%d provider_match=%u\n", + rid, sid, ret, target && of_fwnode_handle(target) == spec->iommu_fwnode); + ret = ret ?: -ENODEV; + } else { + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + if (ret) + dev_err(dev, "BT_IOMMU_REJECT 32-bit DMA mask status=%d\n", ret); + else + dev_info(dev, "BT_IOMMU_ADMITTED RID101->SID2 matched translated DART\n"); + } + of_node_put(target); + return ret; +} + +static void bt7932_unpublished_cleanup(struct bt7932 *bt) +{ + if (WARN_ON_ONCE(bt->retained)) + return; + if (bt->irq_requested) + free_irq(bt->irq, bt); + if (bt->vectors) + pci_free_irq_vectors(bt->pdev); + if (bt->hdev) + hci_free_dev(bt->hdev); + if (bt->tx_wq) + destroy_workqueue(bt->tx_wq); + if (bt->arena) + dma_free_coherent(&bt->pdev->dev, bt->arena_size, bt->arena, bt->arena_dma); + if (bt->image) + dma_free_coherent(&bt->pdev->dev, bt->image_size, bt->image, bt->image_dma); + kfree_sensitive(bt->calibration); + kfree_sensitive(bt->ptx); + if (bt->bar) + pci_iounmap(bt->pdev, bt->bar); + if (bt->regions) + pci_release_regions(bt->pdev); + if (bt->enabled) + pci_disable_device(bt->pdev); + kfree_sensitive(bt); +} + +static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct bt7932 *bt; + u32 chip, boot = U32_MAX, selector; + u16 command; + int ret; + + /* The coordinator opens this only after the real root has its inputs. */ + if (!READ_ONCE(enable)) + return -ENODEV; + if (!of_machine_is_compatible("apple,j700") || + !of_machine_is_compatible("apple,t8140") || + pci_domain_nr(pdev->bus) || pdev->bus->number != 1 || pdev->devfn != PCI_DEVFN(0, 1)) + return -ENODEV; + bt = kzalloc_obj(*bt); + if (!bt) + return -ENOMEM; + bt->pdev = pdev; + bt->last_sleep = U32_MAX; + mutex_init(&bt->lock); + init_waitqueue_head(&bt->wake_wait); + skb_queue_head_init(&bt->tx_queue); + INIT_WORK(&bt->tx_work, bt7932_tx_work); + msleep(100); + ret = pci_enable_device_mem(pdev); + if (ret) + goto fail; + bt->enabled = true; + if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) || + pci_resource_len(pdev, 0) < BT7932_BAR_SIZE) { + ret = -EINVAL; + goto fail; + } + ret = pci_read_config_word(pdev, PCI_COMMAND, &command); + if (ret || command & PCI_COMMAND_MASTER) { + ret = -EBUSY; + goto fail; + } + ret = bt7932_iommu(bt); + if (ret) + goto fail; + ret = pci_request_regions(pdev, "mt7932_bt_pcie"); + if (ret) + goto fail; + bt->regions = true; + bt->bar = pci_iomap(pdev, 0, BT7932_BAR_SIZE); + if (!bt->bar) { + ret = -ENOMEM; + goto fail; + } + dev_info(&pdev->dev, "BT_BAR0 resource=%pR\n", &pdev->resource[0]); + ret = pci_read_config_dword(pdev, BT7932_BOOT_CONFIG, &boot); + if (ret || boot == U32_MAX || (boot & 0xf) || ((boot >> 4) & 7) > 1) { + dev_err(&pdev->dev, "BT_IDENTITY_REJECT config_status=%d boot=%#x\n", ret, boot); + ret = -ENODEV; + goto fail; + } + bt->rom = (boot >> 4) & 7; + writel(0x188d7001, bt->bar + BT7932_CONNECTION_SELECTOR); + selector = readl(bt->bar + BT7932_CONNECTION_SELECTOR); + chip = readl(bt->bar + BT7932_CHIP_ID); + if (chip == U32_MAX || (chip & 0xffff) != 0x7932) { + dev_err(&pdev->dev, "BT_IDENTITY_REJECT boot=%#x selector=%#x chip=%#x\n", + boot, selector, chip); + ret = -ENODEV; + goto fail; + } + dev_info(&pdev->dev, "BT_COLD_ADMISSION ROM=%u bootstate0 chip7932 SID2\n", bt->rom); + ret = bt7932_read_inputs(bt); + if (ret) + goto fail; + ret = bt7932_arena(bt); + if (ret) + goto fail; + bt->tx_wq = alloc_ordered_workqueue("mt7932-bt-tx", WQ_MEM_RECLAIM); + if (!bt->tx_wq) { + ret = -ENOMEM; + goto fail; + } + bt->hdev = hci_alloc_dev(); + if (!bt->hdev) { + ret = -ENOMEM; + goto fail; + } + bt->hdev->bus = HCI_PCI; + SET_HCIDEV_DEV(bt->hdev, &pdev->dev); + hci_set_drvdata(bt->hdev, bt); + bt->hdev->open = bt7932_open; + bt->hdev->close = bt7932_close; + bt->hdev->flush = bt7932_flush; + bt->hdev->send = bt7932_send; + bt->hdev->setup = bt7932_setup; + bt->hdev->reset = bt7932_reset; + bt->hdev->hw_error = bt7932_hw_error; + hci_set_quirk(bt->hdev, HCI_QUIRK_RESET_ON_CLOSE); + hci_set_quirk(bt->hdev, HCI_QUIRK_NON_PERSISTENT_SETUP); + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI); + if (ret < 0) + goto fail; + bt->vectors = true; + bt->irq = pci_irq_vector(pdev, 0); + ret = request_threaded_irq(bt->irq, NULL, bt7932_irq_thread, IRQF_ONESHOT, + "mt7932-bt", bt); + if (ret) + goto fail; + bt->irq_requested = true; + pci_set_drvdata(pdev, bt); + pm_runtime_forbid(&pdev->dev); + /* Every later outcome stays bound and retains all published ownership. */ + bt->retained = true; + pci_set_master(pdev); + ret = bt7932_download(bt); + if (ret) + goto retained_fault; + ret = bt7932_start_ipc(bt); + if (ret) + goto retained_fault; + mutex_lock(&bt->lock); + bt7932_channels(bt); + mutex_unlock(&bt->lock); + ret = hci_register_dev(bt->hdev); + if (ret) + goto retained_fault; + bt->registered = true; + dev_info(&pdev->dev, "BT_HCI_REGISTERED; setup/discovery/connection remain separate gates\n"); + return 0; +retained_fault: + mutex_lock(&bt->lock); + bt7932_fault_locked(bt, "startup failed after bus mastering/publication"); + mutex_unlock(&bt->lock); + dev_err(&pdev->dev, "BT_STARTUP_FAILED error=%d; device bound, no retry\n", ret); + return 0; +fail: + dev_err(&pdev->dev, "BT_ADMISSION_FAILED error=%d; no DMA published\n", ret); + bt7932_unpublished_cleanup(bt); + return ret; +} + +static void bt7932_shutdown(struct pci_dev *pdev) +{ + struct bt7932 *bt = pci_get_drvdata(pdev); + + WRITE_ONCE(bt->opened, false); + wake_up_all(&bt->wake_wait); + cancel_work_sync(&bt->tx_work); + skb_queue_purge(&bt->tx_queue); + mutex_lock(&bt->lock); + bt7932_fault_locked(bt, "shutdown retains DMA until external reset"); + mutex_unlock(&bt->lock); +} + +static const struct pci_device_id bt7932_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x793b) }, + { } +}; +MODULE_DEVICE_TABLE(pci, bt7932_ids); + +static struct pci_driver bt7932_driver = { + .name = "mt7932_bt_pcie", + .id_table = bt7932_ids, + .probe = bt7932_probe, + .shutdown = bt7932_shutdown, + .driver = { + .suppress_bind_attrs = true, + }, +}; +module_pci_driver(bt7932_driver); + +MODULE_DESCRIPTION("Experimental standalone J700 MT7932 Bluetooth PCIe transport"); +MODULE_LICENSE("GPL"); +MODULE_FIRMWARE(BT7932_FW_B0); +MODULE_FIRMWARE(BT7932_FW_B1); +MODULE_FIRMWARE(BT7932_FW_B1_FALLBACK); +MODULE_FIRMWARE(BT7932_PTX); +MODULE_FIRMWARE(BT7932_CAL); +MODULE_FIRMWARE(BT7932_ADDR); From fd06291f3fe08c5acd87ce622a657963ca88fba5 Mon Sep 17 00:00:00 2001 From: Ace <7362641+Acelogic@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:29:45 -0400 Subject: [PATCH 0035/2680] docs: record Neo radio firmware contracts and physical qualification Describe required local firmware/calibration inputs, boot dependencies, source provenance and the exact boundaries of the observed Wi-Fi/Bluetooth results. Include address-free JSON evidence keyed to Image and module hashes. Add a manual physical NetworkManager E2E helper that checks scan, activation, DHCP route, gateway, DNS, verified HTTPS, KDE and kernel health. Its profile selection is explicit and its repeatable JSON artifact omits credentials and unit identities. Install it as test data, not an unattended network-changing selftest. The helper's physical 5 GHz traffic run passed on J700. Retain the failed zero-loss 2.4 GHz coexistence result alongside the passing 5 GHz result. Record PCI/Bluetooth lifetime and queue-saturation blockers; working RF traffic is not a production-removal qualification. Co-authored-by: DJ <43033058+DjDeveloperr@users.noreply.github.com> Co-authored-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Co-authored-by: Eryk Wieliczko --- .../networking/device_drivers/wifi/index.rst | 1 + .../wifi/mt7932-neo-qualification.json | 537 ++++++++++++++++++ .../device_drivers/wifi/mt7932-neo.rst | 139 +++++ tools/testing/selftests/drivers/net/Makefile | 3 + .../selftests/drivers/net/mt7932_e2e.py | 146 +++++ 5 files changed, 826 insertions(+) create mode 100644 Documentation/networking/device_drivers/wifi/mt7932-neo-qualification.json create mode 100644 Documentation/networking/device_drivers/wifi/mt7932-neo.rst create mode 100644 tools/testing/selftests/drivers/net/mt7932_e2e.py diff --git a/Documentation/networking/device_drivers/wifi/index.rst b/Documentation/networking/device_drivers/wifi/index.rst index 29ba9ea64b251..2cd954f5b1fee 100644 --- a/Documentation/networking/device_drivers/wifi/index.rst +++ b/Documentation/networking/device_drivers/wifi/index.rst @@ -10,3 +10,4 @@ Contents: intel/ipw2100 intel/ipw2200 + mt7932-neo diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo-qualification.json b/Documentation/networking/device_drivers/wifi/mt7932-neo-qualification.json new file mode 100644 index 0000000000000..80e3da2f09166 --- /dev/null +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo-qualification.json @@ -0,0 +1,537 @@ +{ + "date_utc": "2026-10-02", + "target": "Apple J700/T8140, Mac17,5", + "kernel_release": "7.1.12-aurora-neo-existing-wifi-20261001-r2+", + "public_base": "1d2904fd3301c63620f07c81ae79f2486a81a9a6", + "driver_source": { + "repository": "https://github.com/aurora-silicon/linux-neo-cleanroom-eryk-with-wifi", + "branch": "wifi/width-qualification-20260912", + "commit": "610cb463c03f9bc68a5d020e7ca443af1fda4856", + "integration_changes": [ + "64-or-65-entry owned config package validation", + "Native scan template and directed zero-IE compatibility", + "Selected WPA2 PSK admission with alternative AKMs; explicit PSK capability", + "Kconfig/Makefile registration", + "J700 PCI Wi-Fi node and bootloader MAC alias" + ] + }, + "image_sha256": "e9877f3943c5cf04970a1d1b336ea68096377907802063dbce2082b93d20011a", + "module_sha256": "aefb08e7cde9398cdce9ca40e29374950dfe989d138813c203cd2f264e44c7f4", + "device_tree_sha256": "8e706f4b443ded0e4dd140a6fe33dd5e4a0f70be852b774254c7f1c462afc4fd", + "config_sha256": "d5a276107754bf8528503a6ceed9954d4eabb6e2d1ebbb116137bf2fa0da29f0", + "checked_wifi_module_removal_reset_reload": "PASS before Bluetooth admission", + "firmware_in_repository": false, + "independent_rewrite_claim": false, + "results": { + "wifi_2g": { + "result": "WIFI_PHYSICAL_NETWORK_E2E_PASS", + "steps": [ + "REQUESTED_NETWORK_SCAN_OBSERVED", + "WPA2_SAVED_PROFILE_ACTIVATED", + "DHCP_ROUTE_OBSERVED", + "GATEWAY_ROUNDTRIPS_VERIFIED", + "DNS_TLS_DOWNLOAD_VERIFIED", + "KDE_RUNNING_NO_KERNEL_FAULT" + ], + "samples": [ + { + "gateway_received": 3, + "gateway_loss_percent": 0, + "dns": true, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "gateway_received": 3, + "gateway_loss_percent": 0, + "dns": true, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "gateway_received": 3, + "gateway_loss_percent": 0, + "dns": true, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + } + ], + "kernel_release": "7.1.12-aurora-neo-existing-wifi-20261001-r2+", + "interface": "wlan0", + "rescan_exit": 0, + "visible_networks": 16, + "requested_network_visible": true, + "activation_exit": 0, + "frequency_mhz": 2437, + "kernel_faults": 0 + }, + "wifi_5g_reconnect_with_bluetooth_enabled": { + "result": "WIFI_PHYSICAL_NETWORK_E2E_PASS", + "steps": [ + "REQUESTED_NETWORK_SCAN_OBSERVED", + "WPA2_SAVED_PROFILE_ACTIVATED", + "DHCP_ROUTE_OBSERVED", + "GATEWAY_ROUNDTRIPS_VERIFIED", + "DNS_TLS_DOWNLOAD_VERIFIED", + "KDE_RUNNING_NO_KERNEL_FAULT" + ], + "samples": [ + { + "gateway_received": 3, + "gateway_loss_percent": 0, + "dns": true, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "gateway_received": 3, + "gateway_loss_percent": 0, + "dns": true, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "gateway_received": 3, + "gateway_loss_percent": 0, + "dns": true, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + } + ], + "kernel_release": "7.1.12-aurora-neo-existing-wifi-20261001-r2+", + "interface": "wlan0", + "rescan_exit": 0, + "visible_networks": 15, + "requested_network_visible": true, + "activation_exit": 0, + "frequency_mhz": 5240, + "kernel_faults": 0 + }, + "coexistence_2g": { + "result": "RADIO_COEXISTENCE_FAILED", + "kernel_release": "7.1.12-aurora-neo-existing-wifi-20261001-r2+", + "samples": [ + { + "elapsed_seconds": 2.178, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 6.366, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 10.557, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 14.737, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 18.921, + "bluetooth_discovering": true, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 23.433, + "bluetooth_discovering": true, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 27.621, + "bluetooth_discovering": true, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 31.831, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 36.094, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 40.288, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 44.477, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 48.667, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 52.858, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 57.033, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 61.21, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 65.406, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 70.585, + "bluetooth_discovering": false, + "gateway_received": 2, + "gateway_loss_percent": 33.3333, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 74.759, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + } + ], + "bluetooth": { + "result": "DISCOVERY_AND_HCI_POWER_CYCLE_PASS", + "steps": [ + "POWERED_ON", + "DISCOVERY_OBSERVED_AND_STOPPED", + "POWERED_OFF", + "POWERED_ON_WITH_NEW_SETUP" + ], + "qualification": "Discovery and normal HCI power cycle; pairing/audio unqualified", + "observed_devices": 18, + "airpods_candidates": 0, + "faults": 0, + "crypto_errors": 0 + }, + "error": "Gateway packet loss; inspect all recorded samples", + "kernel_or_driver_faults": 0 + }, + "coexistence_5g": { + "result": "WIFI_TRAFFIC_BT_DISCOVERY_POWER_CYCLE_COEXISTENCE_PASS", + "kernel_release": "7.1.12-aurora-neo-existing-wifi-20261001-r2+", + "samples": [ + { + "elapsed_seconds": 2.185, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 6.365, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 10.557, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 14.751, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 18.951, + "bluetooth_discovering": true, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 23.136, + "bluetooth_discovering": true, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 27.324, + "bluetooth_discovering": true, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 31.531, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 35.711, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 39.905, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 44.089, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 48.271, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 52.468, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 56.66, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 60.844, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 65.026, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 69.214, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "elapsed_seconds": 73.408, + "bluetooth_discovering": false, + "gateway_received": 3, + "gateway_loss_percent": 0.0, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + } + ], + "start_monotonic_seconds": 1900.801662714, + "bluetooth": { + "result": "DISCOVERY_AND_HCI_POWER_CYCLE_PASS", + "steps": [ + "POWERED_ON", + "DISCOVERY_OBSERVED_AND_STOPPED", + "POWERED_OFF", + "POWERED_ON_WITH_NEW_SETUP" + ], + "qualification": "Discovery and normal HCI power cycle; pairing/audio unqualified", + "observed_devices": 20, + "airpods_candidates": 0, + "faults": 0, + "crypto_errors": 0 + }, + "kernel_or_driver_faults": 0 + }, + "public_network_harness": { + "result": "WIFI_PHYSICAL_NETWORK_E2E_PASS", + "steps": [ + "REQUESTED_NETWORK_SCAN_OBSERVED", + "WPA2_SAVED_PROFILE_ACTIVATED", + "DHCP_ROUTE_OBSERVED", + "GATEWAY_ROUNDTRIPS_VERIFIED", + "DNS_TLS_DOWNLOAD_VERIFIED", + "KDE_RUNNING_NO_KERNEL_FAULT" + ], + "samples": [ + { + "gateway_received": 3, + "gateway_loss_percent": 0, + "dns": true, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "gateway_received": 3, + "gateway_loss_percent": 0, + "dns": true, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + }, + { + "gateway_received": 3, + "gateway_loss_percent": 0, + "dns": true, + "tls_verified": true, + "download_bytes": 713, + "download_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962" + } + ], + "kernel_release": "7.1.12-aurora-neo-existing-wifi-20261001-r2+", + "interface": "wlan0", + "rescan_exit": 0, + "visible_networks": 15, + "requested_network_visible": true, + "activation_exit": 0, + "frequency_mhz": 5240, + "kernel_faults": 0 + } + }, + "limits": [ + "RAM-overlay physical deployment; no persistent installation qualification", + "2.4 GHz coexistence had one missed gateway reply; no zero-loss pass", + "No concurrent headset audio on combined candidate", + "No suspend, kexec, PCI removal, recovery or saturation qualification", + "Earlier Bluetooth-only candidate had an audible AAC pass and two bonded reconnects" + ] +} diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst new file mode 100644 index 0000000000000..c771b0dcab4e6 --- /dev/null +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -0,0 +1,139 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +MT7932 radio bring-up on the MacBook Neo +====================================== + +This draft combines the existing J700 MT7932 fullmac Wi-Fi driver with an +opt-in PCIe Bluetooth transport on the public Aurora platform. Wi-Fi uses +cfg80211 and NetworkManager with firmware-managed WPA2-CCMP authentication. +It is not an mt76/mac80211 softmac implementation. + +The Wi-Fi import is pinned to +``aurora-silicon/linux-neo-cleanroom-eryk-with-wifi``, branch +``wifi/width-qualification-20260912``, commit +``610cb463c03f9bc68a5d020e7ca443af1fda4856``. The import commit preserves all +26 driver files byte-for-byte. A separate commit carries the three tested +compatibility changes: bounded 64/65-record configuration parsing, native scan +IE encoding, and selected WPA2 PSK admission with trailing AKM alternatives. +This contribution imports an existing implementation; it does not claim an +independent rewrite. Existing source notices are retained. Neo radio research +and bring-up credit goes to DJ (DjDeveloperr), Ace (Acelogic), and Ryan Murray. + +Platform and configuration +-------------------------- + +Use the separately reviewed T8140 PCIe bootstrap described in +``Documentation/PCI/controller/apple-t8140.rst``. Its explicit kernel opt-in is +``pcie_apple_piodma_diag.enumerate=1``. Public m1n1 must populate the J700 +``wifi0`` endpoint's own ``local-mac-address``; a zero placeholder is rejected. + +The tested configuration includes ``CONFIG_MT7932_FULLMAC=m``, +``CONFIG_CFG80211=y``, ``CONFIG_BT_MTK7932_PCIE=y``, ``CONFIG_BT_BREDR=y``, +``CONFIG_BT_LE=y``, ``CONFIG_CRYPTO_AES=y`` and ``CONFIG_CRYPTO_CMAC=y``. +The bootstrap and Bluetooth experiment exclude suspend and kexec. +Use the ordinary cfg80211 regulatory database and applicable country policy. +The validated first-admission fallback is kernel country 00 with firmware XZ. + +After the root filesystem and the local firmware packages are available, +select PCIe ASPM performance policy before loading ``mt7932-fullmac``. This +matches the tested admission sequence. Bluetooth's gate defaults closed; +validate the cold, unbound ``14c3:793b`` function, enable +``/sys/module/mt7932_bt_pcie/parameters/enable``, then request its PCI probe. +Do not reprobe after a failed or uncertain Bluetooth admission. The tested +Wi-Fi driver owns function 0 and Bluetooth owns function 1. + +NetworkManager supplies the existing saved WPA2 profile. Use a unique profile +name or UUID and the current interface name; an older ``neo-wifi0`` profile +will not match an interface named ``wlan0``. KDE audio requires BlueZ, +PipeWire, its PulseAudio compatibility service, WirePlumber and the Bluetooth +audio plugins in the logged-in user's session. + +Local firmware inputs +--------------------- + +No firmware, calibration, unit address, credentials or captured command +payloads are distributed in this contribution. Install lawfully obtained +local inputs for the target unit under ``/lib/firmware``. Do not substitute +another unit's calibration or address. A missing or invalid input must remain +a startup failure rather than an excuse to invent calibration or power data. + +Wi-Fi requests the following files under ``mediatek/mt7932/``: + +* ``IZUBA_WIFI_MT7932_patch_mcu_1_2_hdr.bin`` and ``IZUBA_W7932_2.bin``: + the original firmware containers accepted by the driver's parser. +* ``ppr.bin``: the original power-on calibration request payload. +* ``wcal.bin`` and ``oca2.bin``: the unit's factory calibration; OCA2 retains + its original BLOB container, rather than an older replay-stream wrapper. +* ``config-original.bin``: the bounded J7CF package with the original 64 or + 65 configuration records. Required keys and record shapes are validated. +* ``policy/world-XZ.bin`` and, when available, the matching country package + under ``policy/``: J7RP containers with original-derived modes and power + tables. The parser checks the country and table shapes. Relabeling a + different country's payload is not a supported conversion. + +Bluetooth requests these files under ``mediatek/``: + +* ``MT7932B1_OS_TypeB_2.0.177.0_260706180356.bin`` or the supported fallback + ``MT7932B1_OS_TypeB_0.1.133.0_260128190103.bin``; +* ``j700-mt7932-btcal.bin`` (388 bytes); +* ``MT7932_PTB_IzubaA_0.1.0.0_20251021141303.ptx`` (198 bytes); +* ``j700-mt7932-bdaddr.bin`` (the unit's six-byte Bluetooth address). + +Extraction and packaging require the original local assets. The Linux loader +does not grant redistribution rights to them. Do not include those assets or +their unit-specific digests in bug reports, commits or test artifacts. + +Repeatable physical network test +-------------------------------- + +The manual test is installed as a selftest data file rather than an automatic +test: activating a real saved network requires an explicit profile selection. +It selects the profile's requested band and checks an actual scan result, +WPA2 activation, DHCP route, gateway replies, DNS, verified HTTPS response, +KDE processes and fatal kernel/driver diagnostics. It never asks for or prints +the saved password. Run as root, with a fresh attempt name:: + + python3 tools/testing/selftests/drivers/net/mt7932_e2e.py \ + --expected-release "$(uname -r)" --profile "YOUR SAVED PROFILE" \ + --connect --band a --attempt five-ghz + +Use ``--band bg`` for 2.4 GHz. The JSON artifact path is printed on completion; +each of three traffic samples includes the downloaded response's size and +SHA256. Connection names, SSIDs, unit addresses and credentials are omitted +from the result. A successful host serial command is not a test pass: require +``WIFI_PHYSICAL_NETWORK_E2E_PASS`` in the target artifact. + +Qualification and remaining work +-------------------------------- + +The :download:`qualification artifact ` records the exact Image, +module and source identities and the physical results from 2026-10-02. +Both 2.4 and 5 GHz passed scan, WPA2, DHCP, gateway, DNS and verified HTTPS. +Switching bands performed normal peer retirement and fresh authentication. +Wi-Fi's checked module removal/reset/reload also passed before Bluetooth +admission. Five-GHz traffic concurrent with Bluetooth discovery and HCI +off/on passed 54/54 gateway replies and 18 HTTPS downloads with no driver or +kernel faults. The longer 2.4-GHz coexistence run recorded 53/54 replies; +it is explicitly not a zero-loss pass. The lost ping coincided with a +background scan; that timing does not establish a root cause. + +The earlier Bluetooth candidate passed two bonded AirPods reconnects and +30 seconds of AAC output, including a user-confirmed audible tone. Concurrent +headset audio was not repeated on the combined candidate because no AirPods +appeared during its discovery runs. + +This remains a draft, with concrete production merge blockers: + +* PCI bootstrap memory remains retained until external reset; controller + removal, suspend, kexec and reusable-arena lifetime are not qualified. +* Bluetooth lacks a complete PCI removal/quiescence contract. Suppressed + bind/unbind attributes do not make PCI device removal safe. +* Bluetooth's finite software queue limit is not HCI backpressure. Saturation + can drop a frame already accounted by the HCI scheduler; that path needs + correction and saturation qualification before production use. +* Arbitrary caller scan IEs, WPA3/SAE, required MFP, AP/P2P modes, general + country-package generation, roaming and long-duration reliability remain + outside this qualification. Broad compatibility must not be inferred from + the tested station and firmware combination. +* SCO/headset microphone, LE Audio/ISO and simultaneous headset audio remain + unqualified. These limitations are separate from the demonstrated A2DP path. diff --git a/tools/testing/selftests/drivers/net/Makefile b/tools/testing/selftests/drivers/net/Makefile index d5bf4cb638a8c..39e6107cf0269 100644 --- a/tools/testing/selftests/drivers/net/Makefile +++ b/tools/testing/selftests/drivers/net/Makefile @@ -5,6 +5,9 @@ TEST_INCLUDES := $(wildcard lib/py/*.py) \ $(wildcard lib/sh/*.sh) \ ../../net/lib.sh \ +# Manual physical-radio test; requires an explicitly selected saved network. +TEST_FILES += mt7932_e2e.py + TEST_GEN_FILES := \ napi_id_helper \ so_txtime \ diff --git a/tools/testing/selftests/drivers/net/mt7932_e2e.py b/tools/testing/selftests/drivers/net/mt7932_e2e.py new file mode 100644 index 0000000000000..b02a7fc717e06 --- /dev/null +++ b/tools/testing/selftests/drivers/net/mt7932_e2e.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Physical station E2E: explicit Wi-Fi route, gateway round trips, DNS/TLS data. + +Manual hardware test; activates and selects a band on the supplied saved profile. +Run once per named attempt. Output contains no credentials or unit addresses. +The saved NetworkManager profile supplies credentials on the target only. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import time + +parser = argparse.ArgumentParser() +parser.add_argument('--expected-release', required=True) +parser.add_argument('--profile', required=True, help='Unique existing NetworkManager profile name') +parser.add_argument('--connect', action='store_true') +parser.add_argument('--attempt', default='e2e') +parser.add_argument('--band', choices=['bg', 'a'], default='bg') +args = parser.parse_args() +if not re.fullmatch(r'[a-z0-9-]{1,48}', args.attempt): + parser.error('Invalid attempt name') +out = Path('/tmp/neo-wifi-network-' + args.attempt) +result = {'result': 'NOT_STARTED', 'steps': [], 'samples': []} +fault = re.compile(r'translation fault|stale exception latched|Kernel panic|Internal error:|' + r'SError Interrupt|BUG:|Unable to handle kernel|WIFI_TERMINAL|WIFI_STARTUP_FAILED|' + r'qualification stopped:|CAL_PROCEDURE_FAILED|D7_CALIBRATION_FAILED|' + r'SCAN_INTERFACE_SETUP_FAILED|recovery-required=1', re.I) + + +def run(argv, timeout=30): + return subprocess.run(argv, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, timeout=timeout) + + +def require(value, reason): + if not value: + raise RuntimeError(reason) + + +def kernel_ok(): + p = run(['dmesg']) + require(p.returncode == 0 and not fault.search(p.stdout), 'Kernel or driver fault') + return p.stdout + + +require(os.geteuid() == 0 and os.uname().release == args.expected_release, 'Wrong host/kernel') +require(not out.exists(), 'Fresh named E2E attempt required') +compatible = Path('/sys/firmware/devicetree/base/compatible').read_bytes().split(b'\0') +require(b'apple,j700' in compatible and b'apple,t8140' in compatible, 'Wrong device') +out.mkdir(mode=0o700) +try: + kernel_ok() + profile = run(['nmcli', '-g', 'connection.uuid', 'connection', 'show', 'id', args.profile]) + profile_uuid = profile.stdout.strip() + require(profile.returncode == 0 and re.fullmatch(r'[0-9a-f-]{36}', profile_uuid), + 'Missing or ambiguous profile') + ssid_query = run(['nmcli', '-g', '802-11-wireless.ssid', 'connection', 'show', + 'uuid', profile_uuid]) + ssid = ssid_query.stdout.strip() + require(ssid_query.returncode == 0 and ssid, 'Profile has no SSID') + devices = [p.name for p in Path('/sys/class/net').iterdir() if (p / 'wireless').exists()] + require(len(devices) == 1, 'Expected one wireless interface') + iface = devices[0] + result['kernel_release'] = os.uname().release + result['interface'] = iface + require(run(['ip', 'link', 'set', iface, 'up']).returncode == 0, 'Interface up failed') + require(run(['nmcli', 'device', 'set', iface, 'managed', 'yes']).returncode == 0, + 'NetworkManager admission failed') + scan = run(['nmcli', '--wait', '30', 'device', 'wifi', 'rescan', 'ifname', iface], 35) + result['rescan_exit'] = scan.returncode + # NetworkManager may already own a scan when the explicit request arrives. + # Poll its cache without starting a second scan and check firmware health. + names = [] + for attempt in range(20): + time.sleep(2) + kernel_ok() + listings = run(['nmcli', '-t', '-f', 'SSID', 'device', 'wifi', 'list', + '--rescan', 'no', 'ifname', iface]) + require(listings.returncode == 0, 'Scan result read failed') + names = listings.stdout.splitlines() + if ssid in names: + break + result['visible_networks'] = len([name for name in names if name]) + result['requested_network_visible'] = ssid in names + require(result['requested_network_visible'], 'Requested network absent from scan') + result['steps'].append('REQUESTED_NETWORK_SCAN_OBSERVED') + if not args.connect: + result['result'] = 'WIFI_SCAN_OBSERVED' + else: + require(run(['nmcli', 'connection', 'modify', 'uuid', profile_uuid, + '802-11-wireless.band', args.band]).returncode == 0, 'Band selection failed') + connection = run(['nmcli', '--wait', '60', 'connection', 'up', 'uuid', + profile_uuid, 'ifname', iface], 65) + result['activation_exit'] = connection.returncode + link = run(['iw', 'dev', iface, 'link']).stdout + freq = re.search(r'freq: (\d+)', link) + result['frequency_mhz'] = int(freq[1]) if freq else None + require(freq and ((int(freq[1]) < 3000) == (args.band == 'bg')), 'Unexpected associated band') + require(connection.returncode == 0, 'Saved profile activation failed') + for sample in range(3): + addresses = json.loads(run(['ip', '-j', '-4', 'address', 'show', 'dev', iface]).stdout) + require(any(a.get('scope') == 'global' for link in addresses + for a in link.get('addr_info', [])), 'No DHCP IPv4 address') + routes = json.loads(run(['ip', '-j', '-4', 'route', 'show', 'default', 'dev', iface]).stdout) + require(routes and routes[0].get('gateway'), 'No Wi-Fi default gateway') + ping = run(['ping', '-I', iface, '-c', '3', '-W', '3', routes[0]['gateway']], 15) + require(ping.returncode == 0 and re.search(r'\b0(?:\.0+)?% packet loss', ping.stdout) + and re.search(r'\b3 received\b', ping.stdout), 'Gateway packet loss') + dns = run(['getent', 'ahostsv4', 'example.com']) + require(dns.returncode == 0 and dns.stdout.strip(), 'DNS lookup failed') + payload = out / ('https-' + str(sample) + '.html') + transfer = run(['curl', '--fail', '--silent', '--show-error', '--interface', iface, + '--max-time', '20', '--output', str(payload), 'https://example.com/'], 25) + require(transfer.returncode == 0 and payload.is_file(), 'Wi-Fi HTTPS transfer failed') + body = payload.read_bytes() + require(b'Example Domain' in body and len(body) > 100, 'Unexpected HTTPS response') + kernel_ok() + result['samples'].append({'gateway_received': 3, 'gateway_loss_percent': 0, + 'dns': True, 'tls_verified': True, 'download_bytes': len(body), + 'download_sha256': hashlib.sha256(body).hexdigest()}) + if sample != 2: + time.sleep(10) + require(all(run(['pgrep', '-x', name]).returncode == 0 + for name in ('plasmashell', 'kwin_x11')), 'KDE not running') + result['steps'].extend(['WPA2_SAVED_PROFILE_ACTIVATED', 'DHCP_ROUTE_OBSERVED', + 'GATEWAY_ROUNDTRIPS_VERIFIED', 'DNS_TLS_DOWNLOAD_VERIFIED', + 'KDE_RUNNING_NO_KERNEL_FAULT']) + result['result'] = 'WIFI_PHYSICAL_NETWORK_E2E_PASS' +except subprocess.TimeoutExpired: + result['result'] = 'WIFI_NETWORK_E2E_FAILED' + result['error'] = 'External command timed out' +except Exception as error: + result['result'] = 'WIFI_NETWORK_E2E_FAILED' + result['error'] = str(error) +finally: + log = run(['dmesg']).stdout + result['kernel_faults'] = len(fault.findall(log)) + (out / 'result.json').write_text(json.dumps(result, indent=2) + '\n') + print(json.dumps(result, indent=2)) + print('ARTIFACT=' + str(out / 'result.json')) +raise SystemExit(0 if result['result'] in ('WIFI_SCAN_OBSERVED', 'WIFI_PHYSICAL_NETWORK_E2E_PASS') else 1) From e0eb3f7f474207d2e4dda3829eaebdfb07e7e9ea Mon Sep 17 00:00:00 2001 From: Ace <7362641+Acelogic@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:46:37 -0400 Subject: [PATCH 0036/2680] docs: clarify Neo radio co-authorship and review status Use the requested co-author attribution and describe the implementation as experimental now that the PR is ready for review. Preserve source provenance, existing notices, hardware evidence and the production lifecycle limitations. No compiled source or runtime behavior changes. Co-authored-by: DJ <43033058+DjDeveloperr@users.noreply.github.com> Co-authored-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Co-authored-by: Eryk Wieliczko --- .../networking/device_drivers/wifi/mt7932-neo.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index c771b0dcab4e6..0c9c68d82881c 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -3,7 +3,7 @@ MT7932 radio bring-up on the MacBook Neo ====================================== -This draft combines the existing J700 MT7932 fullmac Wi-Fi driver with an +This contribution combines the existing J700 MT7932 fullmac Wi-Fi driver with an opt-in PCIe Bluetooth transport on the public Aurora platform. Wi-Fi uses cfg80211 and NetworkManager with firmware-managed WPA2-CCMP authentication. It is not an mt76/mac80211 softmac implementation. @@ -16,8 +16,8 @@ The Wi-Fi import is pinned to compatibility changes: bounded 64/65-record configuration parsing, native scan IE encoding, and selected WPA2 PSK admission with trailing AKM alternatives. This contribution imports an existing implementation; it does not claim an -independent rewrite. Existing source notices are retained. Neo radio research -and bring-up credit goes to DJ (DjDeveloperr), Ace (Acelogic), and Ryan Murray. +independent rewrite. Existing source notices are retained. Co-authored by +DJ (DjDeveloperr), Ace (Acelogic), and Ryan Murray. Platform and configuration -------------------------- @@ -122,7 +122,7 @@ The earlier Bluetooth candidate passed two bonded AirPods reconnects and headset audio was not repeated on the combined candidate because no AirPods appeared during its discovery runs. -This remains a draft, with concrete production merge blockers: +The experimental implementation has concrete production merge blockers: * PCI bootstrap memory remains retained until external reset; controller removal, suspend, kexec and reusable-arena lifetime are not qualified. From 03cb468d2d7d1cef2a296ca4fcf55bc30d3ad0a9 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 23 Sep 2026 10:56:26 -0700 Subject: [PATCH 0037/2680] PCI: apple: enumerate Thunderbolt PCIe on t8103 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit t8103 boots with the Thunderbolt PCIe port held in reset. Bring the port up when the tunnel activates, and do it before the port's IOMMU probes: that IOMMU's invalidate command does not finish while the port clock is still gated. Program the port tunables and the reset table, release PERST, enable the port clock, then program the root-port configuration tunables. Link training stays in the host driver. If the port is already running, the host does not replay the reset table. Endpoint memory on t8103 is left posted. The PCIe-C IOMMU uses the 64-stream USB4 register block: 16 KiB pages, a 32-bit IOVA and a 36-bit physical address. The 42-bit four-level geometry remains on T8110, which is the hardware that walks it. An xHCI early handoff no longer faults when the capability length read from a BAR is shorter than 16 bytes or not dword-aligned. Tested on a MacBook Pro 13" M1 (j293) with a CalDigit TS3 Plus on USB-1. The tunnel enumerated the dock bridges, four USB host controllers and an I210. The USB devices and a gigabit link came up, and the display on the same port still comes up. [ryan: adapted to 7.2.10 — split by topic: the device-tree hunks are carried by 000-devicetree; the apple-dart hunks go to 020-dart; the drivers/thunderbolt hunks go to 170-usb] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/pci/controller/pcie-apple.c | 477 +++++++++++++++++++++++++--- drivers/usb/host/pci-quirks.c | 15 +- include/linux/pci-apple.h | 2 + 3 files changed, 445 insertions(+), 49 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 427823d89d741..56da7ba9e9afe 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -233,11 +234,15 @@ struct apple_pcie { void __iomem *fabric_base; void __iomem *debug_base; void __iomem *intr2axi_base; + void __iomem *oe_fabric_base; + void __iomem *early_cfg; struct pci_config_window *cfg; struct apple_tunable *rc_tunable; struct apple_tunable *fabric_tunable; struct apple_tunable *debug_tunable; + struct apple_tunable *oe_fabric_tunable; bool power_retained; + bool kernel_init; bool bus_stopped; bool neo_config_ready; u8 neo_secondary_bus; @@ -343,6 +348,16 @@ static inline void apple_pcie_tunnel_writel(void __iomem *base, u32 value, isb(); } +/* Pulse only when the device tree names this bridge. */ +static void apple_pcie_tunnel_pulse_intr2axi(struct apple_pcie *pcie) +{ + if (!pcie->intr2axi_base) + return; + + apple_pcie_tunnel_writel(pcie->intr2axi_base, PCIEC_INTR2AXI_ENABLE, + PCIEC_INTR2AXI_CTRL); +} + static void apple_pcie_tunnel_apply_tunable(void __iomem *base, struct apple_tunable *tunable) { @@ -688,7 +703,9 @@ static int apple_pcie_tunnel_release_reset(struct apple_pcie_port *port) apple_pcie_port_writel(port, 0, PORT_TUNCTRL); ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, !(stat & PORT_TUNSTAT_PERST_ON), - 1000, 100000, false, port, PORT_TUNSTAT); + 1000, + port->pcie->kernel_init ? 250000 : 100000, + false, port, PORT_TUNSTAT); if (ret) dev_err(port->pcie->dev, "port %pOF tunnel reset release timed out\n", port->np); @@ -730,10 +747,15 @@ static void apple_pcie_tunnel_reset_hardware(struct apple_pcie_port *port) { int i; - for (i = 0; i < ARRAY_SIZE(apple_pcie_tunnel_reset_regs); i++) - apple_pcie_port_writel(port, - apple_pcie_tunnel_reset_regs[i].value, - apple_pcie_tunnel_reset_regs[i].offset); + for (i = 0; i < ARRAY_SIZE(apple_pcie_tunnel_reset_regs); i++) { + u32 value = apple_pcie_tunnel_reset_regs[i].value; + + /* t8103 writes 0. 0x208 is the value used when firmware left the port up. */ + if (port->pcie->kernel_init && + apple_pcie_tunnel_reset_regs[i].offset == 0x130) + value = 0; + apple_pcie_port_writel(port, value, apple_pcie_tunnel_reset_regs[i].offset); + } for (i = 0; i < port->pcie->hw->max_rid2sid; i++) apple_pcie_rid2sid_write(port, i, 0); @@ -774,9 +796,7 @@ static int apple_pcie_tunnel_reinitialize(struct apple_pcie_port *port) apple_pcie_port_writel(port, PORT_COUNTER_ENABLE, PORT_COUNTER_CTRL); apple_pcie_port_writel(port, ~0, PORT_INTSTAT); apple_pcie_port_writel(port, ~0, PORT_LINKCMDSTS); - apple_pcie_tunnel_writel(pcie->intr2axi_base, - PCIEC_INTR2AXI_ENABLE, - PCIEC_INTR2AXI_CTRL); + apple_pcie_tunnel_pulse_intr2axi(pcie); /* Release reset immediately before enabling the configured port. */ apple_pcie_port_rmw_set(port, PORT_PERST_OFF, @@ -804,6 +824,74 @@ static int apple_pcie_tunnel_reinitialize(struct apple_pcie_port *port) return 0; } +/* + * t8103 has no firmware handoff for this port. Bring it up from reset: + * tunables and the reset table, then PERST and APPCLK, and only then the + * root-port config-space tunables. Stop before releasing the tunnel reset + * or starting link training. + */ +static int apple_pcie_tunnel_cold_init(struct apple_pcie_port *port) +{ + struct apple_pcie *pcie = port->pcie; + u32 stat; + int ret; + + stat = apple_pcie_port_readl(port, PORT_STATUS); + dev_info(pcie->dev, "port %pOF PCIe-C cold init, status %#x\n", + port->np, stat); + + dev_info(pcie->dev, "port %pOF cold init: debug and fabric tunables\n", + port->np); + apple_pcie_tunnel_apply_tunable(pcie->debug_base, pcie->debug_tunable); + apple_pcie_tunnel_apply_tunable(pcie->fabric_base, pcie->fabric_tunable); + dev_info(pcie->dev, "port %pOF cold init: port reset table\n", port->np); + apple_pcie_tunnel_reset_hardware(port); + apple_pcie_tunnel_apply_tunable(port->base, port->tunable); + + dev_info(pcie->dev, "port %pOF cold init: PERST off, APPCLK on\n", + port->np); + apple_pcie_port_rmw_set(port, PORT_PERST_OFF, pcie->hw->port_perst); + apple_pcie_port_rmw_set(port, PORT_APPCLK_EN, PORT_APPCLK); + apple_pcie_port_rmw_clear(port, PORT_APPCLK_CGDIS, PORT_APPCLK); + + dev_info(pcie->dev, "port %pOF cold init: oe-fabric and root tunables\n", + port->np); + if (pcie->oe_fabric_base) + apple_pcie_tunnel_apply_tunable(pcie->oe_fabric_base, + pcie->oe_fabric_tunable); + apple_pcie_tunnel_apply_tunable(pcie->cfg ? pcie->cfg->win : pcie->early_cfg, + pcie->rc_tunable); + + apple_pcie_port_writel(port, PORT_COUNTER_ENABLE, PORT_COUNTER_CTRL); + apple_pcie_port_writel(port, ~0, PORT_INTSTAT); + apple_pcie_port_writel(port, ~0, PORT_LINKCMDSTS); + apple_pcie_tunnel_pulse_intr2axi(pcie); + + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + stat & PORT_STATUS_READY, + 10, 250000, false, port, PORT_STATUS); + if (ret) + dev_warn(pcie->dev, + "port %pOF cold init: RUN not set (status %#x)\n", + port->np, apple_pcie_port_readl(port, PORT_STATUS)); + + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + !(stat & PORT_LINKSTS_BUSY), + 10, 250000, false, port, PORT_LINKSTS); + if (ret) + dev_warn(pcie->dev, + "port %pOF cold init: link still busy (status %#x)\n", + port->np, apple_pcie_port_readl(port, PORT_LINKSTS)); + + dev_info(pcie->dev, + "port %pOF cold init done, status %#x link %#x tunstat %#x\n", + port->np, + apple_pcie_port_readl(port, PORT_STATUS), + apple_pcie_port_readl(port, PORT_LINKSTS), + apple_pcie_port_readl(port, PORT_TUNSTAT)); + return 0; +} + static void apple_pcie_tunnel_restore_irq_hw(struct apple_pcie_port *port) { struct apple_pcie *pcie = port->pcie; @@ -848,9 +936,7 @@ static int apple_pcie_tunnel_start(struct apple_pcie_port *port) * Release the completed sleep handshake before re-enabling the * retained port, which is the inverse of the disable sequence. */ - apple_pcie_tunnel_writel(pcie->intr2axi_base, - PCIEC_INTR2AXI_ENABLE, - PCIEC_INTR2AXI_CTRL); + apple_pcie_tunnel_pulse_intr2axi(pcie); ret = apple_pcie_tunnel_release_reset(port); if (ret) return ret; @@ -887,9 +973,7 @@ static int apple_pcie_tunnel_start(struct apple_pcie_port *port) PORT_APPCLK); apple_pcie_port_writel(port, PORT_COUNTER_ENABLE, PORT_COUNTER_CTRL); - apple_pcie_tunnel_writel(pcie->intr2axi_base, - PCIEC_INTR2AXI_ENABLE, - PCIEC_INTR2AXI_CTRL); + apple_pcie_tunnel_pulse_intr2axi(pcie); apple_pcie_tunnel_restore_irq_hw(port); for_each_set_bit(i, port->sid_map, port->sid_map_sz) apple_pcie_rid2sid_write(port, i, @@ -1272,33 +1356,47 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, } } if (pcie->hw->tunneled) { + bool preinit_ok; + /* - * m1n1 owns PCIe-C cold initialization. Replaying the port reset or - * tunnel-reset handshake against that live handoff raises an - * asynchronous SError on T6020. Accept only an explicitly successful - * handoff and begin with the read-only RUN state check used by m1n1. + * m1n1 owns PCIe-C cold initialization on T6020. Replaying the + * port reset against that live handoff raises an asynchronous + * SError. t8103 boots with ATC_PCIE off and no handoff, so + * apple,pciec-kernel-init selects the in-kernel sequence. */ ret = of_property_read_u32(pcie->dev->of_node, "apple,pciec-preinit-status", &preinit_status); - if (ret || preinit_status != 1) { + preinit_ok = !ret && preinit_status == 1; + if (pcie->kernel_init && !preinit_ok) { + stat = apple_pcie_port_readl(port, PORT_STATUS); + if (stat & PORT_STATUS_READY) { + dev_info(pcie->dev, + "port %pOF already clocked, status %#x\n", + np, stat); + } else { + ret = apple_pcie_tunnel_cold_init(port); + if (ret) + goto err_teardown; + } + } else if (!preinit_ok) { dev_err(pcie->dev, "PCIe-C requires a successful m1n1 preinit handoff\n"); ret = -ENODEV; goto err_teardown; - } - - ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, - stat & PORT_STATUS_READY, - 100, 250000, false, port, - PORT_STATUS); - if (ret < 0) { - dev_info(pcie->dev, - "port %pOF stopped; replaying PCIe-C hardware initialization\n", - np); - ret = apple_pcie_tunnel_reinitialize(port); - if (ret) - goto err_teardown; + } else { + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + stat & PORT_STATUS_READY, + 100, 250000, false, port, + PORT_STATUS); + if (ret < 0) { + dev_info(pcie->dev, + "port %pOF stopped; replaying PCIe-C hardware initialization\n", + np); + ret = apple_pcie_tunnel_reinitialize(port); + if (ret) + goto err_teardown; + } } } else { if (pcie->hw->retain_clocks) @@ -1356,26 +1454,45 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, link_stat = apple_pcie_port_readl(port, PORT_LINKSTS); if (!(link_stat & PORT_LINKSTS_UP)) { - unsigned long timeout, left; + unsigned long timeout, left = 0; + int attempt, attempts = 1; if (pcie->hw->tunneled) { + if (pcie->kernel_init) { + apple_pcie_tunnel_apply_tunable(port->base, + port->tunable); + udelay(10); + apple_pcie_tunnel_apply_tunable(pcie->cfg->win, + pcie->rc_tunable); + udelay(10); + attempts = 3; + } ret = apple_pcie_tunnel_release_reset(port); if (ret) goto err_teardown; } - /* start link training */ - apple_pcie_port_writel(port, PORT_LTSSMCTL_START, PORT_LTSSMCTL); - timeout = link_up_timeout * HZ / 1000; - left = wait_for_completion_timeout(&pcie->event, timeout); - if (!left) + for (attempt = 0; attempt < attempts; attempt++) { + if (attempt) + reinit_completion(&pcie->event); + apple_pcie_port_writel(port, PORT_LTSSMCTL_START, + PORT_LTSSMCTL); + left = wait_for_completion_timeout(&pcie->event, timeout); + if (left) + break; dev_warn(pcie->dev, "%pOF link didn't come up\n", np); - else + } + if (left) dev_info(pcie->dev, "%pOF link up after %ldms\n", np, (timeout - left) * 1000 / HZ); - } + /* + * A TB3 dock answers CRS for a while after LTSSM start. Wait before + * the bus scan that follows apple_pcie_init(). + */ + if (pcie->hw->tunneled && pcie->kernel_init) + msleep(1000); if (pcie->hw->tunneled) port->started = true; if (pcie->hw->root_bus_only) @@ -1508,7 +1625,11 @@ static int apple_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_d * them before the function driver maps its BAR so pci_iomap() selects * Device-nGnRnE on arm64. */ - if (pcie->hw->tunneled) + /* + * T6020 endpoint BARs fault unless they are mapped non-posted. + * t8103 BARs are posted; a non-posted map there is the SError risk. + */ + if (pcie->hw->tunneled && !of_machine_is_compatible("apple,t8103")) pci_dev_for_each_resource(pdev, res) if (res->flags & IORESOURCE_MEM) res->flags |= IORESOURCE_MEM_NONPOSTED; @@ -1643,6 +1764,28 @@ static void __iomem *apple_pcie_map_bus(struct pci_bus *bus, return NULL; } +static int apple_pcie_config_write(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + struct pci_config_window *cfg = bus->sysdata; + struct apple_pcie *pcie = apple_pcie_lookup(cfg->parent); + struct apple_pcie_port *port; + int ret; + + ret = pci_generic_config_write(bus, devfn, where, size, val); + if (ret || !pcie || !pcie->kernel_init) + return ret; + /* Root-port prefetchable window. The port block has to be told too. */ + if (bus->number != cfg->busr.start || devfn) + return ret; + if (where < 0x24 || where >= 0x30) + return ret; + + list_for_each_entry(port, &pcie->ports, entry) + apple_pcie_port_writel(port, 1, PORT_PREFMEM_ENABLE); + return 0; +} + static const struct pci_ecam_ops apple_pcie_cfg_ecam_ops = { .init = apple_pcie_init, .enable_device = apple_pcie_enable_device, @@ -1650,7 +1793,7 @@ static const struct pci_ecam_ops apple_pcie_cfg_ecam_ops = { .pci_ops = { .map_bus = apple_pcie_map_bus, .read = pci_generic_config_read, - .write = pci_generic_config_write, + .write = apple_pcie_config_write, } }; @@ -1681,15 +1824,19 @@ static int apple_pcie_probe_port(struct device_node *np) static int apple_pcie_tunnel_init_resources(struct platform_device *pdev, struct apple_pcie *pcie) { - struct resource *config, *debug, *fabric, *intr2axi; + struct resource *config, *debug, *fabric, *intr2axi, *oe_fabric; config = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config"); debug = platform_get_resource_byname(pdev, IORESOURCE_MEM, "debug"); fabric = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fabric"); intr2axi = platform_get_resource_byname(pdev, IORESOURCE_MEM, "intr2axi"); - if (!config || !debug || !fabric || !intr2axi) + oe_fabric = platform_get_resource_byname(pdev, IORESOURCE_MEM, "oe-fabric"); + if (!config || !debug || !fabric) return dev_err_probe(pcie->dev, -ENODEV, "PCIe-C resume resources are incomplete\n"); + if (pcie->kernel_init && !oe_fabric) + return dev_err_probe(pcie->dev, -ENODEV, + "PCIe-C oe-fabric region is required\n"); debug->flags |= IORESOURCE_MEM_NONPOSTED; pcie->debug_base = devm_ioremap_resource(pcie->dev, debug); @@ -1701,10 +1848,27 @@ static int apple_pcie_tunnel_init_resources(struct platform_device *pdev, if (IS_ERR(pcie->fabric_base)) return PTR_ERR(pcie->fabric_base); - intr2axi->flags |= IORESOURCE_MEM_NONPOSTED; - pcie->intr2axi_base = devm_ioremap_resource(pcie->dev, intr2axi); - if (IS_ERR(pcie->intr2axi_base)) - return PTR_ERR(pcie->intr2axi_base); + if (intr2axi) { + intr2axi->flags |= IORESOURCE_MEM_NONPOSTED; + pcie->intr2axi_base = devm_ioremap_resource(pcie->dev, intr2axi); + if (IS_ERR(pcie->intr2axi_base)) + return PTR_ERR(pcie->intr2axi_base); + } + + if (oe_fabric) { + oe_fabric->flags |= IORESOURCE_MEM_NONPOSTED; + pcie->oe_fabric_base = devm_ioremap_resource(pcie->dev, oe_fabric); + if (IS_ERR(pcie->oe_fabric_base)) + return PTR_ERR(pcie->oe_fabric_base); + pcie->oe_fabric_tunable = devm_apple_tunable_parse(pcie->dev, + pcie->dev->of_node, + "apple,tunable-oe-fabric", + oe_fabric); + if (IS_ERR(pcie->oe_fabric_tunable)) + return dev_err_probe(pcie->dev, + PTR_ERR(pcie->oe_fabric_tunable), + "PCIe-C oe-fabric tunables unavailable\n"); + } pcie->rc_tunable = devm_apple_tunable_parse(pcie->dev, pcie->dev->of_node, @@ -2067,6 +2231,8 @@ static int apple_pcie_probe(struct platform_device *pdev) if (IS_ERR(pcie->base)) return PTR_ERR(pcie->base); if (pcie->hw->tunneled) { + pcie->kernel_init = of_property_read_bool(dev->of_node, + "apple,pciec-kernel-init"); ret = apple_pcie_tunnel_init_resources(pdev, pcie); if (ret) return ret; @@ -2209,6 +2375,221 @@ int apple_pcie_tunnel_restore(struct device *dev) } EXPORT_SYMBOL_GPL(apple_pcie_tunnel_restore); +struct apple_pcie_map { + void __iomem *base; + struct resource res; +}; + +static void apple_pcie_unmap(struct apple_pcie_map *map) +{ + if (map->base) + iounmap(map->base); + map->base = NULL; +} + +static int apple_pcie_map_named(struct device_node *np, const char *name, + struct apple_pcie_map *map) +{ + int index, ret; + + index = of_property_match_string(np, "reg-names", name); + if (index < 0) + return index; + + ret = of_address_to_resource(np, index, &map->res); + if (ret) + return ret; + + map->base = ioremap_np(map->res.start, resource_size(&map->res)); + if (!map->base) + return -ENOMEM; + return 0; +} + +static struct apple_tunable *apple_pcie_tunable_once(struct device_node *np, + const char *name, + struct resource *res) +{ + struct apple_tunable *tunable; + struct property *prop; + const __be32 *p; + size_t sz; + int i; + + prop = of_find_property(np, name, NULL); + if (!prop) + return ERR_PTR(-ENOENT); + if (prop->length % (3 * sizeof(u32))) + return ERR_PTR(-EINVAL); + + sz = prop->length / (3 * sizeof(u32)); + tunable = kzalloc(struct_size(tunable, values, sz), GFP_KERNEL); + if (!tunable) + return ERR_PTR(-ENOMEM); + tunable->sz = sz; + + for (i = 0, p = NULL; i < tunable->sz; i++) { + p = of_prop_next_u32(prop, p, &tunable->values[i].offset); + p = of_prop_next_u32(prop, p, &tunable->values[i].mask); + p = of_prop_next_u32(prop, p, &tunable->values[i].value); + if (tunable->values[i].offset % 4 || + tunable->values[i].offset > resource_size(res) - 4) { + kfree(tunable); + return ERR_PTR(-EINVAL); + } + } + return tunable; +} + +/* + * Clock the tunneled port before of_platform_populate(). The DART child + * probes immediately and its invalidate command never completes while + * APPCLK is still gated. + */ +static const struct of_device_id apple_pcie_of_match[]; + +int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel) +{ + struct device_node *np, *port_np = NULL; + const struct of_device_id *match; + struct apple_pcie *pcie; + struct apple_pcie_port *port; + struct apple_pcie_map config = {}, debug = {}, fabric = {}, portmap = {}; + struct apple_pcie_map oe = {}; + u32 stat; + int ret; + + np = NULL; + for_each_available_child_of_node(tunnel, np) { + if (of_device_is_compatible(np, "apple,t8103-pciec") && + of_property_read_bool(np, "apple,pciec-kernel-init")) + break; + } + if (!np) + return 0; + + match = of_match_node(apple_pcie_of_match, np); + if (!match) { + ret = -ENODEV; + goto out_np; + } + + ret = apple_pcie_map_named(np, "port0", &portmap); + if (ret) + goto out_np; + + stat = readl(portmap.base + PORT_STATUS); + if (stat & PORT_STATUS_READY) { + dev_info(dev, "PCIe-C port %pOF already clocked, status %#x\n", + np, stat); + ret = 0; + goto out_np; + } + + ret = apple_pcie_map_named(np, "config", &config); + if (ret) + goto out_np; + ret = apple_pcie_map_named(np, "debug", &debug); + if (ret) + goto out_np; + ret = apple_pcie_map_named(np, "fabric", &fabric); + if (ret) + goto out_np; + ret = apple_pcie_map_named(np, "oe-fabric", &oe); + if (ret) + goto out_np; + + port_np = of_get_next_available_child(np, NULL); + if (!port_np) { + ret = -ENODEV; + goto out_np; + } + + pcie = kzalloc_obj(*pcie); + port = kzalloc_obj(*port); + if (!pcie || !port) { + ret = -ENOMEM; + goto out_free; + } + + pcie->dev = dev; + pcie->hw = match->data; + pcie->kernel_init = true; + pcie->early_cfg = config.base; + pcie->debug_base = debug.base; + pcie->fabric_base = fabric.base; + pcie->oe_fabric_base = oe.base; + pcie->debug_tunable = apple_pcie_tunable_once(np, "apple,tunable-debug", + &debug.res); + if (IS_ERR(pcie->debug_tunable)) { + ret = PTR_ERR(pcie->debug_tunable); + pcie->debug_tunable = NULL; + goto out_free; + } + pcie->fabric_tunable = apple_pcie_tunable_once(np, "apple,tunable-fabric", + &fabric.res); + if (IS_ERR(pcie->fabric_tunable)) { + ret = PTR_ERR(pcie->fabric_tunable); + pcie->fabric_tunable = NULL; + goto out_free; + } + pcie->rc_tunable = apple_pcie_tunable_once(np, "apple,tunable-rc", + &config.res); + if (IS_ERR(pcie->rc_tunable)) { + ret = PTR_ERR(pcie->rc_tunable); + pcie->rc_tunable = NULL; + goto out_free; + } + pcie->oe_fabric_tunable = apple_pcie_tunable_once(np, + "apple,tunable-oe-fabric", + &oe.res); + if (IS_ERR(pcie->oe_fabric_tunable)) { + ret = PTR_ERR(pcie->oe_fabric_tunable); + pcie->oe_fabric_tunable = NULL; + goto out_free; + } + + port->pcie = pcie; + port->np = port_np; + port->base = portmap.base; + port->tunable = apple_pcie_tunable_once(port_np, "apple,tunable", + &portmap.res); + if (IS_ERR(port->tunable)) { + ret = PTR_ERR(port->tunable); + port->tunable = NULL; + goto out_free; + } + + dev_info(dev, "PCIe-C clocking %pOF before DART probe\n", np); + ret = apple_pcie_tunnel_cold_init(port); + +out_free: + if (port && !IS_ERR_OR_NULL(port->tunable)) + kfree(port->tunable); + if (pcie) { + if (!IS_ERR_OR_NULL(pcie->debug_tunable)) + kfree(pcie->debug_tunable); + if (!IS_ERR_OR_NULL(pcie->fabric_tunable)) + kfree(pcie->fabric_tunable); + if (!IS_ERR_OR_NULL(pcie->rc_tunable)) + kfree(pcie->rc_tunable); + if (!IS_ERR_OR_NULL(pcie->oe_fabric_tunable)) + kfree(pcie->oe_fabric_tunable); + } + kfree(port); + kfree(pcie); + of_node_put(port_np); +out_np: + apple_pcie_unmap(&config); + apple_pcie_unmap(&debug); + apple_pcie_unmap(&fabric); + apple_pcie_unmap(&portmap); + apple_pcie_unmap(&oe); + of_node_put(np); + return ret; +} +EXPORT_SYMBOL_GPL(apple_pcie_tunnel_prepare); + static void apple_pcie_remove(struct platform_device *pdev) { struct pci_host_bridge *bridge = platform_get_drvdata(pdev); diff --git a/drivers/usb/host/pci-quirks.c b/drivers/usb/host/pci-quirks.c index 0404489c2f6a9..26bba2d42b1bb 100644 --- a/drivers/usb/host/pci-quirks.c +++ b/drivers/usb/host/pci-quirks.c @@ -1224,7 +1224,20 @@ static void quirk_usb_handoff_xhci(struct pci_dev *pdev) if (pdev->vendor == PCI_VENDOR_ID_INTEL) usb_enable_intel_xhci_ports(pdev); - op_reg_base = base + XHCI_HC_LENGTH(readl(base)); + /* + * A dead or unprogrammed BAR reads as all-ones. The capability + * length is then 0xff, and the operational-register read below is + * not 4-byte aligned and faults. A real xHCI length is at least + * 0x10 and dword-aligned. + */ + val = readl(base); + if (XHCI_HC_LENGTH(val) < 0x10 || (XHCI_HC_LENGTH(val) & 3)) { + dev_warn(&pdev->dev, + "xHCI capability length %#x unusable, skipping handoff\n", + XHCI_HC_LENGTH(val)); + goto iounmap; + } + op_reg_base = base + XHCI_HC_LENGTH(val); /* Wait for the host controller to be ready before writing any * operational or runtime registers. Wait 5 seconds and no more. diff --git a/include/linux/pci-apple.h b/include/linux/pci-apple.h index ef664800801fe..057b16bdd42c7 100644 --- a/include/linux/pci-apple.h +++ b/include/linux/pci-apple.h @@ -3,7 +3,9 @@ #define _LINUX_PCI_APPLE_H struct device; +struct device_node; +int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel); int apple_pcie_tunnel_quiesce(struct device *dev); int apple_pcie_tunnel_restore(struct device *dev); From bf2f156e3c7b2984c7f56b9af127b13d7e054020 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 23 Sep 2026 11:38:28 -0700 Subject: [PATCH 0038/2680] PCI: apple: finish tunneled IOMMU commands before the port clock stops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cable removal quiesces the PCIe tunnel and then removes the IOMMU. The IOMMU's runtime resume and remove both issue a command, and that command cannot complete once the port clock is gated. Every dock unplug logged that timeout, then a follow-up quiesce skipped the port and reported success. Tell the IOMMU to stop issuing commands before the clock is turned off, and let it issue them again only after the port is back up. An acknowledgment timeout from a cable that is already gone is no longer a failed quiesce when the local port has left the running state. [ryan: adapted to 7.2.10 — split by topic: the apple-dart hunks go to 020-dart; 020-dart carries apple-dart's final int-returning command API, so the tunnel DART walk callback is typed int from here on; the result stays ignored until the later report of resume failures] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/pci/controller/pcie-apple.c | 55 ++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 56da7ba9e9afe..627768476aca6 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include "../pci.h" @@ -1078,11 +1079,20 @@ static int apple_pcie_tunnel_stop(struct apple_pcie_port *port) ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, stat & PORT_TUNSTAT_PERST_ACK_PEND, 1000, 1000000, false, port, PORT_TUNSTAT); - if (ret) - dev_warn(pcie->dev, "port %pOF tunnel reset acknowledgment timed out\n", + if (ret) { + dev_warn(pcie->dev, + "port %pOF tunnel reset acknowledgment timed out\n", port->np); - if (ret && !err) - err = ret; + /* + * The local port is already down. A removed cable cannot + * finish the router handshake, and reporting that as a + * quiesce failure makes the follow-up call skip this port + * and claim success. + */ + if (err || (apple_pcie_port_readl(port, PORT_STATUS) & + PORT_STATUS_READY)) + err = ret; + } apple_pcie_port_rmw_clear(port, PORT_TUNCTRL_PERST_ACK_REQ, PORT_TUNCTRL); port->started = false; @@ -2278,6 +2288,34 @@ static int apple_pcie_probe(struct platform_device *pdev) return 0; } +typedef int (*apple_pcie_dart_fn)(struct device *dev); + +static void apple_pcie_walk_tunnel_darts(struct apple_pcie *pcie, apple_pcie_dart_fn fn) +{ + struct device_node *parent, *child; + struct platform_device *pdev; + + parent = of_get_parent(pcie->dev->of_node); + if (!parent) + return; + + for_each_available_child_of_node(parent, child) { + if (!of_device_is_compatible(child, "apple,t8103-dart") && + !of_device_is_compatible(child, "apple,t8103-usb4-dart") && + !of_device_is_compatible(child, "apple,t8110-dart") && + !of_device_is_compatible(child, "apple,t6000-dart")) + continue; + + pdev = of_find_device_by_node(child); + if (pdev) { + fn(&pdev->dev); + put_device(&pdev->dev); + } + } + + of_node_put(parent); +} + int apple_pcie_tunnel_quiesce(struct device *dev) { struct pci_host_bridge *bridge = dev_get_drvdata(dev); @@ -2318,6 +2356,12 @@ int apple_pcie_tunnel_quiesce(struct device *dev) pcie->bus_stopped = true; } + /* + * Removing the IOMMU later resumes it and issues a command. Gate + * those commands before APPCLK goes away with the port. + */ + apple_pcie_walk_tunnel_darts(pcie, apple_dart_quiesce_commands); + list_for_each_entry(port, &pcie->ports, entry) { int err; @@ -2362,6 +2406,9 @@ int apple_pcie_tunnel_restore(struct device *dev) if (err && !ret) ret = err; + if (!err) + apple_pcie_walk_tunnel_darts(pcie, + apple_dart_resume_commands); } pci_lock_rescan_remove(); From a85b58a622b838547d9d421a1267a1c689ace7e2 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 21:00:22 -0700 Subject: [PATCH 0039/2680] PCI: apple: preserve failed tunnel recovery and optional DART builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Leave a failed restart quiesced and retryable instead of rescanning and clearing bus_stopped after a reset or link timeout. Resume DART commands only after the ports have restarted. Allow DART-disabled builds through no-op stubs, and constrain Apple PCIe to a module when its optional DART provider is modular. [ryan: adapted to 7.2.10 — split by topic: the apple-dart hunks go to 020-dart] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/pci/controller/Kconfig | 1 + drivers/pci/controller/pcie-apple.c | 13 ++++++------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index ebdc821fea454..d418c2124c565 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -43,6 +43,7 @@ config PCIE_APPLE_MSI_DOORBELL_ADDR config PCIE_APPLE tristate "Apple PCIe controller" depends on ARCH_APPLE || COMPILE_TEST + depends on APPLE_DART || !APPLE_DART depends on OF depends on PCI_MSI depends on PAGE_SIZE_16KB || COMPILE_TEST diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 627768476aca6..e8a7fd0b8915e 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2402,14 +2402,13 @@ int apple_pcie_tunnel_restore(struct device *dev) return 0; list_for_each_entry(port, &pcie->ports, entry) { - int err = apple_pcie_tunnel_start(port); - - if (err && !ret) - ret = err; - if (!err) - apple_pcie_walk_tunnel_darts(pcie, - apple_dart_resume_commands); + if (port->started) + continue; + ret = apple_pcie_tunnel_start(port); + if (ret) + return ret; } + apple_pcie_walk_tunnel_darts(pcie, apple_dart_resume_commands); pci_lock_rescan_remove(); pci_rescan_bus(bridge->bus); From ce0b0b50c5fba8e0a1591c5a4a5870c6e9490d70 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 00:51:47 -0700 Subject: [PATCH 0040/2680] PCI: apple: correct M1 tunneled port restart sequencing The Kensington SD5560T loses its PCIe xHCI hierarchy after s2idle on both ports of j293. Tracing shows all functions left in D0; the host reports link restored immediately before the downstream configuration space becomes inaccessible. A failed software restart also leaves APPCLK enabled while started is false, so later teardown skips the port. Use the M1 activation sequence on retained M1 ports: restore port configuration, APPCLK, PERST and root tunables before releasing tunnel reset. Require a fresh link-up event, retry transient link-down/tunnel errors, and allow downstream reset recovery before resuming config I/O. Wait for M1's reset acknowledgment pending indication to clear, matching Corellium's M1 implementation. Preserve the retained T6020 sequence. Track partially started hardware separately from a successful link so quiesce, teardown and subsequent restore can finish cleanup even after a restart timeout. Preserve the original IRQ mask across such retries. The M1 sequencing changes are a hardware-test candidate; the Kensington suspend/reconnect regression still requires validation on this commit. Link: https://github.com/corellium/linux-m1/blob/master/drivers/pci/controller/pcie-apple-m1.c Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/pci/controller/pcie-apple.c | 79 +++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 3 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index e8a7fd0b8915e..63b5c3266c9b7 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -274,6 +274,7 @@ struct apple_pcie_port { int sid_map_sz; int idx; bool started; + bool needs_stop; }; static void rmw_set(u32 set, void __iomem *addr) @@ -925,6 +926,66 @@ static int apple_pcie_tunnel_start(struct apple_pcie_port *port) struct apple_pcie *pcie = port->pcie; u32 stat; int i, ret; + u32 link_events = BIT(PORT_INT_LINK_UP) | BIT(PORT_INT_LINK_DOWN) | + BIT(PORT_INT_TUNNEL_ERR); + + /* A failed start still owns partially enabled port hardware. */ + port->needs_stop = true; + + if (pcie->kernel_init && pcie->power_retained) { + /* + * M1 requires the same activation sequence after sleep as after + * cable insertion. In particular, release tunneled reset only + * after APPCLK, PERST and the root-port tunables are restored. + */ + apple_pcie_tunnel_pulse_intr2axi(pcie); + apple_pcie_tunnel_apply_tunable(port->base, port->tunable); + apple_pcie_port_rmw_set(port, PORT_APPCLK_EN, PORT_APPCLK); + usleep_range(10, 20); + apple_pcie_port_rmw_set(port, PORT_PERST_OFF, + pcie->hw->port_perst); + usleep_range(10, 20); + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + stat & PORT_STATUS_READY, + 10, 250000, false, port, PORT_STATUS); + if (ret) + return ret; + apple_pcie_tunnel_apply_tunable(pcie->cfg->win, pcie->rc_tunable); + ret = apple_pcie_tunnel_release_reset(port); + if (ret) + return ret; + + /* + * LINKSTS can still contain the pre-suspend link-up indication. + * Require a new link-up event, retrying the LTSSM on a transient + * link-down/tunnel-error event, as during M1 tunnel activation. + */ + for (i = 0; i < 3; i++) { + apple_pcie_port_writel(port, link_events, PORT_INTSTAT); + apple_pcie_port_writel(port, PORT_LTSSMCTL_START, + PORT_LTSSMCTL); + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + stat & link_events, + 1000, 500000, false, port, + PORT_INTSTAT); + if (!ret && (stat & link_events) == BIT(PORT_INT_LINK_UP) && + (apple_pcie_port_readl(port, PORT_LINKSTS) & PORT_LINKSTS_UP)) + break; + } + if (i == 3) { + dev_err(pcie->dev, + "port %pOF tunnel link did not restart (events %#x, link %#x)\n", + port->np, stat, + apple_pcie_port_readl(port, PORT_LINKSTS)); + return -ETIMEDOUT; + } + apple_pcie_port_writel(port, link_events, PORT_INTSTAT); + /* Allow downstream functions to finish reset before config I/O. */ + msleep(100); + if (!(apple_pcie_port_readl(port, PORT_LINKSTS) & PORT_LINKSTS_UP)) + return -ENOLINK; + goto restored; + } if (pcie->power_retained) { /* @@ -994,6 +1055,7 @@ static int apple_pcie_tunnel_start(struct apple_pcie_port *port) port->np); return ret; } +restored: if (pcie->power_retained) apple_pcie_port_writel(port, port->saved_intmask, PORT_INTMSK); @@ -1035,7 +1097,8 @@ static int apple_pcie_tunnel_stop(struct apple_pcie_port *port) u32 stat; int err = 0, ret; - port->saved_intmask = apple_pcie_port_readl(port, PORT_INTMSK); + if (port->started) + port->saved_intmask = apple_pcie_port_readl(port, PORT_INTMSK); apple_pcie_port_writel(port, ~0, PORT_INTMSK); apple_pcie_port_writel(port, ~0, PORT_INTSTAT); apple_pcie_port_writel(port, ~0, PORT_LINKCMDSTS); @@ -1076,7 +1139,10 @@ static int apple_pcie_tunnel_stop(struct apple_pcie_port *port) err = ret; apple_pcie_port_rmw_set(port, PORT_TUNCTRL_PERST_ACK_REQ, PORT_TUNCTRL); + /* M1 reports an outstanding acknowledgment until ACK_PEND clears. */ ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + pcie->kernel_init ? + !(stat & PORT_TUNSTAT_PERST_ACK_PEND) : stat & PORT_TUNSTAT_PERST_ACK_PEND, 1000, 1000000, false, port, PORT_TUNSTAT); if (ret) { @@ -1096,6 +1162,8 @@ static int apple_pcie_tunnel_stop(struct apple_pcie_port *port) apple_pcie_port_rmw_clear(port, PORT_TUNCTRL_PERST_ACK_REQ, PORT_TUNCTRL); port->started = false; + port->needs_stop = apple_pcie_port_readl(port, PORT_STATUS) & + PORT_STATUS_READY; return err; } @@ -1105,7 +1173,7 @@ static void apple_pcie_port_teardown(struct apple_pcie_port *port) if (port->base) apple_pcie_port_writel(port, ~0, PORT_INTMSK); - if (port->pcie->hw->tunneled && port->started) + if (port->pcie->hw->tunneled && (port->started || port->needs_stop)) apple_pcie_tunnel_stop(port); apple_pcie_port_unregister_irqs(port); @@ -2365,7 +2433,7 @@ int apple_pcie_tunnel_quiesce(struct device *dev) list_for_each_entry(port, &pcie->ports, entry) { int err; - if (!port->started) + if (!port->started && !port->needs_stop) continue; err = apple_pcie_tunnel_stop(port); if (err && !ret) @@ -2404,6 +2472,11 @@ int apple_pcie_tunnel_restore(struct device *dev) list_for_each_entry(port, &pcie->ports, entry) { if (port->started) continue; + if (port->needs_stop) { + ret = apple_pcie_tunnel_stop(port); + if (ret) + return ret; + } ret = apple_pcie_tunnel_start(port); if (ret) return ret; From 5e5257c802f0a5bf62c892ef439f32001db36a19 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:03:16 -0700 Subject: [PATCH 0041/2680] PCI: apple: preserve tunneled D0 policy after rescan The initial probe walk leaves tunnel functions in D0 across system sleep, but quiesce removes those pci_dev objects. A later rescan creates new objects without the flag, so a subsequent suspend can put the rebuilt hierarchy into D3. Apply the policy before enabled functions bind and walk the restored hierarchy as well, including functions without a driver. This closes a lifecycle gap; it does not establish recovery of the observed link- training timeout. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/pci/controller/pcie-apple.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 63b5c3266c9b7..dbb336f5733d7 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -1698,6 +1698,10 @@ static int apple_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_d struct of_phandle_args iommu_spec = {}; int idx, err; + /* Also cover functions discovered by a later hotplug or rescan. */ + if (pcie->hw->tunneled) + pdev->dev_flags |= PCI_DEV_FLAGS_NO_D3; + /* * Endpoint BARs share PCIe-C's tunneled, non-posted MMIO fabric. Mark * them before the function driver maps its BAR so pci_iomap() selects @@ -2485,6 +2489,7 @@ int apple_pcie_tunnel_restore(struct device *dev) pci_lock_rescan_remove(); pci_rescan_bus(bridge->bus); + pci_walk_bus(bridge->bus, apple_pcie_tunnel_keep_d0, NULL); pcie->bus_stopped = false; pci_unlock_rescan_remove(); From a811a59c4a5cb2b38a4b0ab46ca8dbdb3467532c Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:03:17 -0700 Subject: [PATCH 0042/2680] PCI: apple: propagate tunneled cold initialization errors Cold initialization currently logs readiness and idle timeouts but always returns success. Its early caller then populates DART even if the port clock did not become ready, although the DART command engine depends on that clock. Return the polling error immediately and retain the last polled register value in the diagnostic. Callers already propagate failures and avoid populating the tunnel children. This changes only the timeout paths. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/pci/controller/pcie-apple.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index dbb336f5733d7..d03f22efd0083 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -873,17 +873,17 @@ static int apple_pcie_tunnel_cold_init(struct apple_pcie_port *port) stat & PORT_STATUS_READY, 10, 250000, false, port, PORT_STATUS); if (ret) - dev_warn(pcie->dev, - "port %pOF cold init: RUN not set (status %#x)\n", - port->np, apple_pcie_port_readl(port, PORT_STATUS)); + return dev_err_probe(pcie->dev, ret, + "port %pOF cold init: RUN not set (status %#x)\n", + port->np, stat); ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, !(stat & PORT_LINKSTS_BUSY), 10, 250000, false, port, PORT_LINKSTS); if (ret) - dev_warn(pcie->dev, - "port %pOF cold init: link still busy (status %#x)\n", - port->np, apple_pcie_port_readl(port, PORT_LINKSTS)); + return dev_err_probe(pcie->dev, ret, + "port %pOF cold init: link still busy (status %#x)\n", + port->np, stat); dev_info(pcie->dev, "port %pOF cold init done, status %#x link %#x tunstat %#x\n", From 9069235cdb4de45c63b9fbfdf2042d3d99531991 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:03:41 -0700 Subject: [PATCH 0043/2680] PCI: apple: unwind host registration before resource cleanup A PM-link failure after pci_host_common_init() leaves a registered PCI bus and possibly bound function drivers behind while probe unwinds its IRQ domains and mappings. Resolve the PM dependencies before registering the host. If the cleanup action allocation fails after host registration, stop and remove the PCI hierarchy before calling apple_pcie_cleanup(). Avoid devm_add_action_or_reset() there because its immediate cleanup would run before the function drivers released their IRQs. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/pci/controller/pcie-apple.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index d03f22efd0083..40a11d5c297c5 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2325,6 +2325,13 @@ static int apple_pcie_probe(struct platform_device *pdev) mutex_init(&pcie->lock); INIT_LIST_HEAD(&pcie->ports); + /* Resolve PM dependencies before publishing the PCI hierarchy. */ + if (pcie->hw->tunneled) { + ret = apple_pcie_tunnel_add_links(pcie); + if (ret) + return ret; + } + ret = apple_msi_init(pcie); if (ret) return ret; @@ -2339,9 +2346,13 @@ static int apple_pcie_probe(struct platform_device *pdev) * Port mappings are allocated by the ECAM init callback. Register cleanup * afterwards so devres runs it before unmapping those registers. */ - ret = devm_add_action_or_reset(dev, apple_pcie_cleanup, pcie); - if (ret) + ret = devm_add_action(dev, apple_pcie_cleanup, pcie); + if (ret) { + /* Drivers must release their IRQs before the domains disappear. */ + pci_host_common_remove(pdev); + apple_pcie_cleanup(pcie); return ret; + } if (pcie->piodma_supplier) { /* All fallible devres setup precedes publication of DMA pointers. */ @@ -2352,10 +2363,8 @@ static int apple_pcie_probe(struct platform_device *pdev) return 0; } - if (pcie->hw->tunneled) { + if (pcie->hw->tunneled) pci_walk_bus(bridge->bus, apple_pcie_tunnel_keep_d0, NULL); - return apple_pcie_tunnel_add_links(pcie); - } return 0; } From f561223f88bcf91d8c31c298e77aab14520856fc Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:11:36 -0700 Subject: [PATCH 0044/2680] iommu/apple-dart: report tunnel command-resume failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resuming commands cleared the gate before runtime resume, then discarded a runtime-PM failure. PCIe-C could consequently rescan functions without a successfully restored IOMMU and without a gate protecting the failed command engine. Return the resume error, restore the gate on failure, and propagate DART errors through the tunneled-host restore path before rescan. Keep the runtime-PM usage count balanced with pm_runtime_resume_and_get(). A KUnit fault case verifies error propagation, gate retention, reference balance and a subsequent successful retry. [ryan: adapted to 7.2.10 — split by topic: only the pcie-apple.c hunk; the apple-dart side is in 020-dart; the int callback type moved into the commit that added the walk] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/pci/controller/pcie-apple.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 40a11d5c297c5..ac7f53fa8f9b1 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2371,14 +2371,15 @@ static int apple_pcie_probe(struct platform_device *pdev) typedef int (*apple_pcie_dart_fn)(struct device *dev); -static void apple_pcie_walk_tunnel_darts(struct apple_pcie *pcie, apple_pcie_dart_fn fn) +static int apple_pcie_walk_tunnel_darts(struct apple_pcie *pcie, apple_pcie_dart_fn fn) { struct device_node *parent, *child; struct platform_device *pdev; + int ret = 0; parent = of_get_parent(pcie->dev->of_node); if (!parent) - return; + return -ENODEV; for_each_available_child_of_node(parent, child) { if (!of_device_is_compatible(child, "apple,t8103-dart") && @@ -2389,12 +2390,16 @@ static void apple_pcie_walk_tunnel_darts(struct apple_pcie *pcie, apple_pcie_dar pdev = of_find_device_by_node(child); if (pdev) { - fn(&pdev->dev); + int err = fn(&pdev->dev); + + if (err && !ret) + ret = err; put_device(&pdev->dev); } } of_node_put(parent); + return ret; } int apple_pcie_tunnel_quiesce(struct device *dev) @@ -2441,7 +2446,7 @@ int apple_pcie_tunnel_quiesce(struct device *dev) * Removing the IOMMU later resumes it and issues a command. Gate * those commands before APPCLK goes away with the port. */ - apple_pcie_walk_tunnel_darts(pcie, apple_dart_quiesce_commands); + ret = apple_pcie_walk_tunnel_darts(pcie, apple_dart_quiesce_commands); list_for_each_entry(port, &pcie->ports, entry) { int err; @@ -2494,7 +2499,9 @@ int apple_pcie_tunnel_restore(struct device *dev) if (ret) return ret; } - apple_pcie_walk_tunnel_darts(pcie, apple_dart_resume_commands); + ret = apple_pcie_walk_tunnel_darts(pcie, apple_dart_resume_commands); + if (ret) + return ret; pci_lock_rescan_remove(); pci_rescan_bus(bridge->bus); From 254ba09dafe30858a9ac96c842e42d196e9998db Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:11:36 -0700 Subject: [PATCH 0045/2680] PCI: apple: describe tunneled PM ordering by Linux dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document the retained-state, doorbell and PCIe-C/DART ordering requirements directly in terms of register access and Linux device dependencies. No functional change. [ryan: adapted to 7.2.10 — split by topic: the apple-dart hunks go to 020-dart] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/pci/controller/pcie-apple.c | 30 +++++++++++++---------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index ac7f53fa8f9b1..a80c607268b74 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -154,11 +154,9 @@ MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); #define PORT_MSIMAP_TARGET GENMASK(7, 0) /* - * The doorbell address is set to 0xfffff000, which by convention - * matches what MacOS does, and it is possible to use any other - * address (in the bottom 4GB, as the base register is only 32bit). - * However, it has to be excluded from the IOVA range, and the DART - * driver has to know about it. + * The doorbell address must be in the bottom 4GB because the base register + * is only 32 bits wide. Exclude it from the IOVA range and share the same + * configured address with the DART driver. */ #define DOORBELL_ADDR CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR @@ -990,8 +988,8 @@ static int apple_pcie_tunnel_start(struct apple_pcie_port *port) if (pcie->power_retained) { /* * The ATC PCIe domain remains powered on these systems. Replaying - * resetPortHardware() against that live handoff is not a resume - * operation and raises an asynchronous external abort on T6020. + * the cold-init reset table against that retained state raises an + * asynchronous external abort on T6020. * Re-activating the ACIO PCIe tunnel closes the Intr2AXI aperture, * just as it does during cold cable activation, so pulse it before * releasing the retained port reset. @@ -1005,7 +1003,7 @@ static int apple_pcie_tunnel_start(struct apple_pcie_port *port) } else { /* * A genuinely power-gated PCIe-C controller lost its register - * state. Recreate the enablePortHardware() baseline before start. + * state. Recreate the cold-init baseline before start. */ apple_pcie_tunnel_apply_tunable(pcie->debug_base, pcie->debug_tunable); @@ -2055,11 +2053,10 @@ static int apple_pcie_tunnel_add_links(struct apple_pcie *pcie) int ret; /* - * Apple orders clientDidSleep after disabling the PCIe-C port, and - * clientWillWake before enabling it. NHI and PCIe-C are sibling devices - * under ACIO, so encode that missing edge explicitly: Linux must suspend - * PCIe-C before NHI tears down the router state and resume NHI before the - * host touches its tunneled aperture. + * NHI and PCIe-C are sibling devices under ACIO, so encode their PM + * dependency explicitly. PCIe-C must complete its reset handshake before + * NHI tears down the router state, and NHI must resume before the host + * touches its tunneled aperture. */ for_each_available_child_of_node_scoped(pcie->dev->parent->of_node, sibling) { @@ -2080,10 +2077,9 @@ static int apple_pcie_tunnel_add_links(struct apple_pcie *pcie) "failed to order PCIe-C after NHI resume\n"); /* - * The DART and PCIe-C host are synthesized as siblings. Apple restores - * port hardware and RID/SID forwarding before forcing the DART active; - * express the same dependency so Linux suspends DART first and resumes - * PCIe-C first. + * The DART and PCIe-C host are synthesized as siblings. Port hardware + * and RID/SID forwarding must be ready before DART access resumes, so + * suspend DART first and resume PCIe-C first. */ for_each_available_child_of_node_scoped(pcie->dev->of_node->parent, sibling) { From e4a288258d1866e9ee0fcca69b818cc8d166c5cf Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 20:35:13 -0700 Subject: [PATCH 0046/2680] thunderbolt: restore M1 tunneled PCIe state and recover late link loss MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kensington USB devices can disappear after s2idle even when the initial port restart and cable health check succeed. A later PCIe link failure must trigger another fresh validation instead of waiting for a cable interrupt or manual replug. Restore the resettable M1 tunneled host and DART state before restarting the downstream link. Gate DART commands and fault handling around clock shutdown, preserve stream selection, and describe the reset and register initialization properties in the binding. Gate Type-C work across initialization, sleep and removal, read fresh cable metadata after resume, and preserve an unchanged healthy session. Propagate late link failures through an atomic switch notification into bounded process-context revalidation. Keep failure history sticky while allowing later notifications from a retained host. Failed reads or health checks request a bounded reconnect; successful checks preserve routing. Keep failed display-route acquisition retryable. Add state-machine and DART regression coverage. The native ARM64 build and 88 KUnit cases pass. Two M1/j293 Kensington USB-only suspend cycles exercised late link-loss recovery and restored usable input without a manual replug. The initiating xHCI resume fault remains unresolved. [ryan: adapted to 7.2.10 — split by topic: only the drivers/pci/controller/pcie-apple.c, include/linux/pci-apple.h, Documentation/devicetree/bindings/pci/apple,pcie.yaml hunks of this usb commit; the rest is in 170-usb] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- .../devicetree/bindings/pci/apple,pcie.yaml | 50 ++++- drivers/pci/controller/pcie-apple.c | 195 ++++++++++++++++-- include/linux/pci-apple.h | 8 + 3 files changed, 237 insertions(+), 16 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 48e3ae4ceafbf..df7ef17ae8d2f 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -36,6 +36,16 @@ description: | distributed over the root ports as the OS sees fit by programming the PCIe controller's port registers. +$defs: + tunable: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Mask of bits to clear + - description: Bits to set + description: Register initialization tuples for one controller region. + properties: compatible: oneOf: @@ -85,8 +95,34 @@ properties: power-domains: maxItems: 1 + resets: + description: + Reset of a tunneled (PCIe-C) controller and its DART, for controllers + the kernel initializes itself. + maxItems: 1 + dma-coherent: true + apple,pciec-kernel-init: + type: boolean + description: The kernel initializes the tunneled controller from reset. + + apple,tunable-rc: + $ref: '#/$defs/tunable' + description: Root-complex register initialization tuples. + + apple,tunable-fabric: + $ref: '#/$defs/tunable' + description: Initialization tuples for the fabric register region. + + apple,tunable-debug: + $ref: '#/$defs/tunable' + description: Initialization tuples for the debug register region. + + apple,tunable-oe-fabric: + $ref: '#/$defs/tunable' + description: Initialization tuples for the oe-fabric register region. + apple,piodma: $ref: /schemas/types.yaml#/definitions/phandle description: @@ -108,6 +144,10 @@ patternProperties: interrupt-controller: true + apple,tunable: + $ref: '#/$defs/tunable' + description: Initialization tuples relative to this port's registers. + required: - interrupt-controller - "#interrupt-cells" @@ -189,7 +229,7 @@ allOf: - const: fabric - const: debug - const: ltssm - - const: intr2axi + - enum: [intr2axi, oe-fabric] interrupts: maxItems: 1 ranges: @@ -197,6 +237,12 @@ allOf: maxItems: 3 else: properties: + resets: false + apple,pciec-kernel-init: false + apple,tunable-rc: false + apple,tunable-fabric: false + apple,tunable-debug: false + apple,tunable-oe-fabric: false ranges: maxItems: 2 allOf: @@ -239,6 +285,8 @@ allOf: - const: phy3 patternProperties: "^pci@": + properties: + apple,tunable: false required: - reset-gpios diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index a80c607268b74..db4410ea35b50 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -28,11 +28,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include @@ -40,6 +42,20 @@ #include "pci-host-common.h" #include "pcie-apple-piodma-diag.h" +static ATOMIC_NOTIFIER_HEAD(apple_pcie_tunnel_notifiers); + +int apple_pcie_tunnel_register_notifier(struct notifier_block *nb) +{ + return atomic_notifier_chain_register(&apple_pcie_tunnel_notifiers, nb); +} +EXPORT_SYMBOL_GPL(apple_pcie_tunnel_register_notifier); + +void apple_pcie_tunnel_unregister_notifier(struct notifier_block *nb) +{ + atomic_notifier_chain_unregister(&apple_pcie_tunnel_notifiers, nb); +} +EXPORT_SYMBOL_GPL(apple_pcie_tunnel_unregister_notifier); + static int link_up_timeout = 500; module_param(link_up_timeout, int, 0644); MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); @@ -243,6 +259,9 @@ struct apple_pcie { bool power_retained; bool kernel_init; bool bus_stopped; + bool reset_on_resume; + bool resume_failed; /* cleared only by teardown and reprobe */ + struct reset_control *reset; bool neo_config_ready; u8 neo_secondary_bus; struct device *piodma_supplier; @@ -273,6 +292,7 @@ struct apple_pcie_port { int idx; bool started; bool needs_stop; + bool link_failed; }; static void rmw_set(u32 set, void __iomem *addr) @@ -554,6 +574,8 @@ static void apple_port_irq_handler(struct irq_desc *desc) chained_irq_enter(chip, desc); stat = apple_pcie_port_readl(port, PORT_INTSTAT); + /* Masked link events can belong to the resume poller. */ + stat &= ~apple_pcie_port_readl(port, PORT_INTMSK); for_each_set_bit(i, &stat, 32) generic_handle_domain_irq(port->domain, i); @@ -624,6 +646,13 @@ static irqreturn_t apple_pcie_port_irq(int irq, void *data) complete_all(&port->pcie->event); break; case PORT_INT_LINK_DOWN: + /* A later link-up alone does not prove downstream state survived. */ + if (port->pcie->hw->tunneled && READ_ONCE(port->started)) { + WRITE_ONCE(port->link_failed, true); + atomic_notifier_call_chain(&apple_pcie_tunnel_notifiers, + APPLE_PCIE_TUNNEL_LINK_DOWN, + port->pcie->dev->parent); + } dev_info_ratelimited(port->pcie->dev, "Link down on %pOF\n", port->np); break; @@ -898,6 +927,8 @@ static void apple_pcie_tunnel_restore_irq_hw(struct apple_pcie_port *port) u32 value = 0; int i; + /* Keep link events pending until the resume poller has consumed them. */ + apple_pcie_port_writel(port, ~0, PORT_INTMSK); apple_pcie_port_writel(port, ~0, PORT_INTSTAT); apple_pcie_port_writel(port, ~0, PORT_LINKCMDSTS); apple_pcie_port_writel(port, lower_32_bits(DOORBELL_ADDR), @@ -916,7 +947,6 @@ static void apple_pcie_tunnel_restore_irq_hw(struct apple_pcie_port *port) value = ilog2(pcie->nvecs) << PORT_MSICFG_L2MSINUM_SHIFT; } apple_pcie_port_writel(port, value | PORT_MSICFG_EN, PORT_MSICFG); - apple_pcie_port_writel(port, port->saved_intmask, PORT_INTMSK); } static int apple_pcie_tunnel_start(struct apple_pcie_port *port) @@ -927,6 +957,9 @@ static int apple_pcie_tunnel_start(struct apple_pcie_port *port) u32 link_events = BIT(PORT_INT_LINK_UP) | BIT(PORT_INT_LINK_DOWN) | BIT(PORT_INT_TUNNEL_ERR); + if (pcie->resume_failed) + return -EIO; + /* A failed start still owns partially enabled port hardware. */ port->needs_stop = true; @@ -1054,10 +1087,12 @@ static int apple_pcie_tunnel_start(struct apple_pcie_port *port) return ret; } restored: - if (pcie->power_retained) - apple_pcie_port_writel(port, port->saved_intmask, PORT_INTMSK); + /* Publish ownership before unmasking a possible link-down event. */ + WRITE_ONCE(port->started, true); + /* The port helper uses a relaxed MMIO write. */ + wmb(); + apple_pcie_port_writel(port, port->saved_intmask, PORT_INTMSK); - port->started = true; dev_info(pcie->dev, "port %pOF tunnel link restored after suspend\n", port->np); return 0; @@ -1159,9 +1194,9 @@ static int apple_pcie_tunnel_stop(struct apple_pcie_port *port) } apple_pcie_port_rmw_clear(port, PORT_TUNCTRL_PERST_ACK_REQ, PORT_TUNCTRL); - port->started = false; - port->needs_stop = apple_pcie_port_readl(port, PORT_STATUS) & - PORT_STATUS_READY; + WRITE_ONCE(port->started, false); + port->needs_stop = err || (apple_pcie_port_readl(port, PORT_STATUS) & + PORT_STATUS_READY); return err; } @@ -1570,7 +1605,7 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, if (pcie->hw->tunneled && pcie->kernel_init) msleep(1000); if (pcie->hw->tunneled) - port->started = true; + WRITE_ONCE(port->started, true); if (pcie->hw->root_bus_only) dev_info(pcie->dev, "port %pOF RUN=%#x link=%#x; downstream config blocked\n", @@ -2316,6 +2351,17 @@ static int apple_pcie_probe(struct platform_device *pdev) return ret; pcie->power_retained = apple_pcie_tunnel_power_is_retained(dev->of_node); + if (pcie->kernel_init) { + /* ACIO releases this line before populating the host. */ + pcie->reset = devm_reset_control_get_optional_exclusive_released(dev, NULL); + if (IS_ERR(pcie->reset)) + return dev_err_probe(dev, PTR_ERR(pcie->reset), + "PCIe-C reset unavailable\n"); + ret = reset_control_acquire(pcie->reset); + if (ret) + return dev_err_probe(dev, ret, + "PCIe-C reset is still in use\n"); + } } mutex_init(&pcie->lock); @@ -2367,10 +2413,12 @@ static int apple_pcie_probe(struct platform_device *pdev) typedef int (*apple_pcie_dart_fn)(struct device *dev); -static int apple_pcie_walk_tunnel_darts(struct apple_pcie *pcie, apple_pcie_dart_fn fn) +static int apple_pcie_walk_tunnel_darts(struct apple_pcie *pcie, + apple_pcie_dart_fn fn, bool required) { struct device_node *parent, *child; struct platform_device *pdev; + bool found = false; int ret = 0; parent = of_get_parent(pcie->dev->of_node); @@ -2388,14 +2436,17 @@ static int apple_pcie_walk_tunnel_darts(struct apple_pcie *pcie, apple_pcie_dart if (pdev) { int err = fn(&pdev->dev); + found = true; if (err && !ret) ret = err; put_device(&pdev->dev); + } else if (required && !ret) { + ret = -ENODEV; } } of_node_put(parent); - return ret; + return ret ?: (required && !found ? -ENODEV : 0); } int apple_pcie_tunnel_quiesce(struct device *dev) @@ -2442,7 +2493,7 @@ int apple_pcie_tunnel_quiesce(struct device *dev) * Removing the IOMMU later resumes it and issues a command. Gate * those commands before APPCLK goes away with the port. */ - ret = apple_pcie_walk_tunnel_darts(pcie, apple_dart_quiesce_commands); + ret = apple_pcie_walk_tunnel_darts(pcie, apple_dart_quiesce_commands, false); list_for_each_entry(port, &pcie->ports, entry) { int err; @@ -2480,6 +2531,8 @@ int apple_pcie_tunnel_restore(struct device *dev) pcie = pci_host_bridge_priv(bridge); if (!pcie->hw->tunneled) return -EINVAL; + if (pcie->resume_failed) + return -EIO; if (!pcie->bus_stopped) return 0; @@ -2495,7 +2548,7 @@ int apple_pcie_tunnel_restore(struct device *dev) if (ret) return ret; } - ret = apple_pcie_walk_tunnel_darts(pcie, apple_dart_resume_commands); + ret = apple_pcie_walk_tunnel_darts(pcie, apple_dart_resume_commands, false); if (ret) return ret; @@ -2511,6 +2564,40 @@ int apple_pcie_tunnel_restore(struct device *dev) } EXPORT_SYMBOL_GPL(apple_pcie_tunnel_restore); +/* + * Check recorded host state without accessing a possibly failed data path. + * The caller serializes tunnel transitions and calls after system PM complete. + * The device lock protects driver data against a concurrent unbind. + */ +int apple_pcie_tunnel_check_state(struct device *dev) +{ + struct pci_host_bridge *bridge; + struct apple_pcie_port *port; + struct apple_pcie *pcie; + + guard(device)(dev); + if (!dev->driver) + return -ENODEV; + bridge = dev_get_drvdata(dev); + if (!bridge || !bridge->bus) + return -ENODEV; + pcie = pci_host_bridge_priv(bridge); + if (!pcie->hw->tunneled) + return -EINVAL; + if (pcie->resume_failed) + return -EIO; + if (pcie->bus_stopped || list_empty(&pcie->ports)) + return -ENOLINK; + list_for_each_entry(port, &pcie->ports, entry) { + /* Keep failure latched until a fresh host is probed. */ + if (!port->started || READ_ONCE(port->link_failed)) + return -ENOLINK; + } + + return 0; +} +EXPORT_SYMBOL_GPL(apple_pcie_tunnel_check_state); + struct apple_pcie_map { void __iomem *base; struct resource res; @@ -2752,19 +2839,77 @@ static int apple_pcie_suspend_noirq(struct device *dev) { struct apple_pcie *pcie = apple_pcie_lookup(dev); struct apple_pcie_port *port; - int i; + bool active = false, can_reset; + int i, ret; if (!pcie->hw->tunneled) return 0; + pcie->reset_on_resume = false; + can_reset = pcie->kernel_init && pcie->power_retained && pcie->reset && + !pcie->bus_stopped && !pcie->resume_failed; + if (can_reset) { + ret = apple_pcie_walk_tunnel_darts(pcie, apple_dart_save_tunnel_state, true); + if (ret) { + dev_warn(dev, "PCIe-C cold resume unavailable: DART save failed: %d\n", + ret); + can_reset = false; + } + } list_for_each_entry(port, &pcie->ports, entry) { if (port->started) { + active = true; for_each_set_bit(i, port->sid_map, port->sid_map_sz) port->saved_rid2sid[i] = apple_pcie_port_readl(port, port_rid2sid_offset(port, i)); - apple_pcie_tunnel_stop(port); } + if (port->started || port->needs_stop) { + ret = apple_pcie_tunnel_stop(port); + if (ret) + can_reset = false; + } + } + pcie->reset_on_resume = can_reset && active; + return 0; +} + +static int apple_pcie_reset_for_resume(struct apple_pcie *pcie) +{ + struct apple_pcie_port *port; + int i, ret; + + /* + * Keep the hierarchy bound and suspended. Reset only after its I/O + * drained, then restore the port and IOMMU before releasing tunneled + * reset. Endpoints must never run with missing address translations. + */ + ret = reset_control_reset(pcie->reset); + if (ret) + return ret; + + list_for_each_entry(port, &pcie->ports, entry) { + u32 stat; + + /* A failed cold setup still owns partially enabled hardware. */ + port->needs_stop = true; + apple_pcie_port_writel(port, PORT_TUNCTRL_PERST_ON, PORT_TUNCTRL); + ret = apple_pcie_tunnel_cold_init(port); + if (ret) + return ret; + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + stat & PORT_TUNSTAT_PERST_ON, + 1000, 100000, false, port, PORT_TUNSTAT); + if (ret) + return ret; + apple_pcie_tunnel_restore_irq_hw(port); + for_each_set_bit(i, port->sid_map, port->sid_map_sz) + apple_pcie_rid2sid_write(port, i, port->saved_rid2sid[i]); } + ret = apple_pcie_walk_tunnel_darts(pcie, apple_dart_restore_tunnel_state, true); + if (ret) + return ret; + + dev_info(pcie->dev, "PCIe-C reset and DART state restored before link resume\n"); return 0; } @@ -2776,13 +2921,33 @@ static int apple_pcie_resume_noirq(struct device *dev) if (!pcie->hw->tunneled) return 0; + if (pcie->resume_failed) + return -EIO; + if (pcie->reset_on_resume) { + pcie->reset_on_resume = false; + ret = apple_pcie_reset_for_resume(pcie); + if (ret) { + dev_err(dev, "PCIe-C cold resume failed: %d\n", ret); + goto failed; + } + } list_for_each_entry(port, &pcie->ports, entry) { ret = apple_pcie_tunnel_start(port); if (ret) - return ret; + goto failed; } return 0; + +failed: + /* Device PM keeps resuming dependents even after this callback fails. */ + pcie->resume_failed = true; + apple_pcie_walk_tunnel_darts(pcie, apple_dart_quiesce_commands, false); + list_for_each_entry(port, &pcie->ports, entry) { + if (port->started || port->needs_stop) + apple_pcie_tunnel_stop(port); + } + return ret; } static const struct dev_pm_ops apple_pcie_pm_ops = { diff --git a/include/linux/pci-apple.h b/include/linux/pci-apple.h index 057b16bdd42c7..552a29357eb6b 100644 --- a/include/linux/pci-apple.h +++ b/include/linux/pci-apple.h @@ -4,9 +4,17 @@ struct device; struct device_node; +struct notifier_block; + +/* Atomic notification; data is the host's parent device, valid in callback. */ +#define APPLE_PCIE_TUNNEL_LINK_DOWN 1 + +int apple_pcie_tunnel_register_notifier(struct notifier_block *nb); +void apple_pcie_tunnel_unregister_notifier(struct notifier_block *nb); int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel); int apple_pcie_tunnel_quiesce(struct device *dev); int apple_pcie_tunnel_restore(struct device *dev); +int apple_pcie_tunnel_check_state(struct device *dev); #endif /* _LINUX_PCI_APPLE_H */ From df70697db1d1d6cf6e27a775bd0690b038619186 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:39:09 -0700 Subject: [PATCH 0047/2680] PCI: apple: resume tunneled functions as if from D3cold Stopping a tunneled port at suspend asserts tunneled PERST, and on t8103 resume also resets the PCIe-C controller. Every function below the port therefore loses its state, even though the host itself only enters suspend-to-idle. The PCI core does not know that: the tunneled functions never enter D3, so in suspend-to-idle it skips their noirq restore and the wait for the secondary bus, and restores config space only in the asynchronous resume phase, with no readiness check. In that window nothing below the root port is configured yet. pciehp on an occupied hotplug port, such as the one in a CalDigit TS3 Plus, reads the device below it through bridges that have not been restored, can see ~0, and then removes the device mid-resume while its driver still accesses its registers. Mark every tunneled function as having lost power when the port is stopped for system sleep. The PCI core then restores config space in the noirq phase, parents first, waits for each secondary bus, and marks a subtree disconnected when it does not come back. When the port fails to resume, mark the hierarchy disconnected instead of gating the port clock under still-bound drivers, as the quiesce path already does. Quiesce removes the hierarchy and stops the port once tasks are thawed. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PA1D59hehGmwnpSxrYcDbk --- drivers/pci/controller/pcie-apple.c | 49 ++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index db4410ea35b50..63477cdf75036 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2835,6 +2835,41 @@ static void apple_pcie_remove(struct platform_device *pdev) pci_unlock_rescan_remove(); } +static int apple_pcie_mark_power_lost(struct pci_dev *pdev, void *unused) +{ + /* + * Tunnel stop asserts tunneled PERST, so the functions below lose their + * state even when the host stays in suspend-to-idle. Make the PCI core + * resume them as if from D3cold: restore config space in the noirq phase, + * parents first, and wait for each secondary bus before touching it. + */ + pdev->skip_bus_pm = false; + pdev->current_state = PCI_D3cold; + return 0; +} + +static void apple_pcie_tunnel_hierarchy_lost(struct device *dev) +{ + struct pci_host_bridge *bridge = dev_get_drvdata(dev); + + if (bridge && bridge->bus) + pci_walk_bus(bridge->bus, apple_pcie_mark_power_lost, NULL); +} + +static void apple_pcie_tunnel_hierarchy_gone(struct device *dev) +{ + struct pci_host_bridge *bridge = dev_get_drvdata(dev); + + /* + * Nothing below a port that failed to resume can be reached. Mark it + * disconnected so that neither the PCI core nor function drivers + * resuming after us issue I/O into the dead aperture; quiesce removes + * the hierarchy once tasks are thawed. + */ + if (bridge && bridge->bus) + pci_walk_bus(bridge->bus, pci_dev_set_disconnected, NULL); +} + static int apple_pcie_suspend_noirq(struct device *dev) { struct apple_pcie *pcie = apple_pcie_lookup(dev); @@ -2870,6 +2905,8 @@ static int apple_pcie_suspend_noirq(struct device *dev) } } pcie->reset_on_resume = can_reset && active; + if (active && !pcie->bus_stopped) + apple_pcie_tunnel_hierarchy_lost(dev); return 0; } @@ -2942,11 +2979,15 @@ static int apple_pcie_resume_noirq(struct device *dev) failed: /* Device PM keeps resuming dependents even after this callback fails. */ pcie->resume_failed = true; + if (!pcie->bus_stopped) + apple_pcie_tunnel_hierarchy_gone(dev); apple_pcie_walk_tunnel_darts(pcie, apple_dart_quiesce_commands, false); - list_for_each_entry(port, &pcie->ports, entry) { - if (port->started || port->needs_stop) - apple_pcie_tunnel_stop(port); - } + /* + * Leave the port clocked with its link down, the state a surprise + * unplug leaves behind, in which stray accesses complete with an + * error. Quiesce removes the hierarchy and then stops the port when + * ACIO is torn down after resume. + */ return ret; } From 1d98f9d08264c4b428d99f6df73d9acf91d07d51 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:03:25 -0700 Subject: [PATCH 0048/2680] thunderbolt: keep M1 PCIe tunnels up through suspend-to-idle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In suspend-to-idle ACIO stays powered and, with the routers kept awake, the Thunderbolt link stays trained. Stopping the tunneled PCIe port anyway holds the dock hierarchy in reset for the whole sleep. After that, every function below it has to be restored and each USB device behind the dock's xHCI reset-resumed one after another, about 5 s before input works again. After a sleep of about 80 s, a Kensington SD5560T on a MacBook Pro 13" M1 (j293) stopped answering altogether shortly after resume: its link trained and its bridges were restored, then every request timed out. When the tunneled port is up and healthy at suspend-to-idle, leave it running. On resume, check that the link is still up and that no link-down event is pending; if so the hierarchy kept its state and resumes like any other PCIe device. If the link was lost while asleep, fall back to the stop, reset and restart path. The Thunderbolt connection manager then leaves the still-active tunnels alone instead of tearing them down and setting them up again. The pcie_apple s2idle_keep_link parameter restores the previous behaviour for comparison. [ryan: adapted to 7.2.10 — split by topic: only the drivers/pci/controller/pcie-apple.c, include/linux/pci-apple.h hunks of this usb commit; the rest is in 170-usb] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PA1D59hehGmwnpSxrYcDbk --- drivers/pci/controller/pcie-apple.c | 81 +++++++++++++++++++++++++++-- include/linux/pci-apple.h | 1 + 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 63477cdf75036..385f16e55feb9 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -37,6 +37,7 @@ #include #include #include +#include #include "../pci.h" #include "pci-host-common.h" @@ -56,6 +57,10 @@ void apple_pcie_tunnel_unregister_notifier(struct notifier_block *nb) } EXPORT_SYMBOL_GPL(apple_pcie_tunnel_unregister_notifier); +static bool s2idle_keep_link = true; +module_param(s2idle_keep_link, bool, 0644); +MODULE_PARM_DESC(s2idle_keep_link, "Keep tunneled links up through suspend-to-idle"); + static int link_up_timeout = 500; module_param(link_up_timeout, int, 0644); MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); @@ -260,6 +265,7 @@ struct apple_pcie { bool kernel_init; bool bus_stopped; bool reset_on_resume; + bool link_kept; bool resume_failed; /* cleared only by teardown and reprobe */ struct reset_control *reset; bool neo_config_ready; @@ -2598,6 +2604,22 @@ int apple_pcie_tunnel_check_state(struct device *dev) } EXPORT_SYMBOL_GPL(apple_pcie_tunnel_check_state); +/* + * Whether the host left its tunnel running for the system sleep in progress. + * The caller serializes against unbind through device PM ordering. + */ +bool apple_pcie_tunnel_link_kept(struct device *dev) +{ + struct pci_host_bridge *bridge = dev_get_drvdata(dev); + struct apple_pcie *pcie; + + if (!dev->driver || !bridge) + return false; + pcie = pci_host_bridge_priv(bridge); + return pcie->hw->tunneled && pcie->link_kept; +} +EXPORT_SYMBOL_GPL(apple_pcie_tunnel_link_kept); + struct apple_pcie_map { void __iomem *base; struct resource res; @@ -2870,16 +2892,47 @@ static void apple_pcie_tunnel_hierarchy_gone(struct device *dev) pci_walk_bus(bridge->bus, pci_dev_set_disconnected, NULL); } -static int apple_pcie_suspend_noirq(struct device *dev) +static bool apple_pcie_tunnel_link_healthy(struct apple_pcie *pcie) +{ + struct apple_pcie_port *port; + + if (list_empty(&pcie->ports)) + return false; + list_for_each_entry(port, &pcie->ports, entry) { + if (!READ_ONCE(port->started) || READ_ONCE(port->link_failed)) + return false; + if (apple_pcie_port_readl(port, PORT_INTSTAT) & + BIT(PORT_INT_LINK_DOWN)) + return false; + if (!(apple_pcie_port_readl(port, PORT_LINKSTS) & PORT_LINKSTS_UP)) + return false; + } + return true; +} + +/* + * In suspend-to-idle ACIO and the router links stay powered, so an up + * tunnel can simply be left running. The functions below keep their state + * and resume like any other PCIe device; nothing has to be re-trained. + */ +static bool apple_pcie_keep_link(struct apple_pcie *pcie) +{ + if (!READ_ONCE(s2idle_keep_link) || !pm_suspend_no_platform()) + return false; + if (!pcie->kernel_init || !pcie->power_retained) + return false; + if (pcie->bus_stopped || pcie->resume_failed) + return false; + return apple_pcie_tunnel_link_healthy(pcie); +} + +static void apple_pcie_stop_for_sleep(struct device *dev) { struct apple_pcie *pcie = apple_pcie_lookup(dev); struct apple_pcie_port *port; bool active = false, can_reset; int i, ret; - if (!pcie->hw->tunneled) - return 0; - pcie->reset_on_resume = false; can_reset = pcie->kernel_init && pcie->power_retained && pcie->reset && !pcie->bus_stopped && !pcie->resume_failed; if (can_reset) { @@ -2907,6 +2960,18 @@ static int apple_pcie_suspend_noirq(struct device *dev) pcie->reset_on_resume = can_reset && active; if (active && !pcie->bus_stopped) apple_pcie_tunnel_hierarchy_lost(dev); +} + +static int apple_pcie_suspend_noirq(struct device *dev) +{ + struct apple_pcie *pcie = apple_pcie_lookup(dev); + + if (!pcie->hw->tunneled) + return 0; + pcie->reset_on_resume = false; + pcie->link_kept = apple_pcie_keep_link(pcie); + if (!pcie->link_kept) + apple_pcie_stop_for_sleep(dev); return 0; } @@ -2960,6 +3025,14 @@ static int apple_pcie_resume_noirq(struct device *dev) return 0; if (pcie->resume_failed) return -EIO; + if (pcie->link_kept) { + pcie->link_kept = false; + if (apple_pcie_tunnel_link_healthy(pcie)) + return 0; + /* Lost while asleep: take the same path as a stopped tunnel. */ + dev_warn(dev, "PCIe-C link lost during suspend-to-idle\n"); + apple_pcie_stop_for_sleep(dev); + } if (pcie->reset_on_resume) { pcie->reset_on_resume = false; ret = apple_pcie_reset_for_resume(pcie); diff --git a/include/linux/pci-apple.h b/include/linux/pci-apple.h index 552a29357eb6b..d7c2d244d209b 100644 --- a/include/linux/pci-apple.h +++ b/include/linux/pci-apple.h @@ -16,5 +16,6 @@ int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel); int apple_pcie_tunnel_quiesce(struct device *dev); int apple_pcie_tunnel_restore(struct device *dev); int apple_pcie_tunnel_check_state(struct device *dev); +bool apple_pcie_tunnel_link_kept(struct device *dev); #endif /* _LINUX_PCI_APPLE_H */ From 686224c78c4a4fb61ff6762f4fc67c3fd66d4927 Mon Sep 17 00:00:00 2001 From: RJ Skerry-Ryan Date: Sun, 27 Sep 2026 23:30:01 -0700 Subject: [PATCH 0049/2680] PCI: apple: Don't restart PCIe-C on resume after the tunnel was lost MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the USB4 router disappears during suspend (e.g. a Thunderbolt 3 dock under s2idle), the NHI's resume tears the PCIe tunnel down and quiesces this host before PCIe-C's own resume_noirq runs. That resume then starts the ports anyway, training against a tunnel that no longer exists; on J416c the port is afterwards unable to link on every later connect until reboot. Skip restarting the ports when the host has been quiesced. They are started again by apple_pcie_tunnel_restore(), or cold-initialized by a fresh populate, once a new tunnel is up. [ryan: adapted to 7.2.10 — the check sits after the resume_failed check that the earlier tunnel-restore commit added here] Assisted-by: Claude:claude-opus-5-5 Signed-off-by: RJ Skerry-Ryan --- drivers/pci/controller/pcie-apple.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 385f16e55feb9..923147b603030 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -3025,6 +3025,20 @@ static int apple_pcie_resume_noirq(struct device *dev) return 0; if (pcie->resume_failed) return -EIO; + + /* + * The NHI resumes first (see apple_pcie_tunnel_add_links()). If the + * USB4 router was lost during suspend, its teardown has already + * quiesced this host. Starting the ports now would train against a + * tunnel that no longer exists, which leaves them unable to link on + * every later connect; apple_pcie_tunnel_restore() starts them once a + * new tunnel is up. + */ + if (pcie->bus_stopped) { + dev_info(dev, "PCIe-C tunnel lost during suspend, not restarting\n"); + return 0; + } + if (pcie->link_kept) { pcie->link_kept = false; if (apple_pcie_tunnel_link_healthy(pcie)) From 57a5e39474cf8a559092b63ae1ab241182da0310 Mon Sep 17 00:00:00 2001 From: Yureka Lilian Date: Fri, 3 Jul 2026 13:13:26 +0200 Subject: [PATCH 0050/2680] arm64: dts: apple: add minimal MacBook Neo device tree Introduce the original T8140/J700 CPU, interrupt, power-management, watchdog and framebuffer description as the base for later SEP device-tree work. Drop an empty aliases node and excess whitespace from the imported commit. Signed-off-by: Yureka Lilian (cherry picked and cleaned from c8c58c8d3b1f5f4a44ef2419b2fb999c64491572) --- arch/arm64/boot/dts/apple/Makefile | 1 + arch/arm64/boot/dts/apple/t8140-j700.dts | 42 ++ arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 756 ++++++++++++++++++++++ arch/arm64/boot/dts/apple/t8140.dtsi | 176 +++++ 4 files changed, 975 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8140-j700.dts create mode 100644 arch/arm64/boot/dts/apple/t8140-pmgr.dtsi create mode 100644 arch/arm64/boot/dts/apple/t8140.dtsi diff --git a/arch/arm64/boot/dts/apple/Makefile b/arch/arm64/boot/dts/apple/Makefile index 62999574a36ba..4f31beeb11ac4 100644 --- a/arch/arm64/boot/dts/apple/Makefile +++ b/arch/arm64/boot/dts/apple/Makefile @@ -109,3 +109,4 @@ dtb-$(CONFIG_ARCH_APPLE) += t8132-j624.dtb dtb-$(CONFIG_ARCH_APPLE) += t8132-j713.dtb dtb-$(CONFIG_ARCH_APPLE) += t8132-j715.dtb dtb-$(CONFIG_ARCH_APPLE) += t8132-j773g.dtb +dtb-$(CONFIG_ARCH_APPLE) += t8140-j700.dtb diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts new file mode 100644 index 0000000000000..76b1537428ee5 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Neo + * + * target-type: J700 + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t8140.dtsi" + +/ { + compatible = "apple,j700", "apple,t8140", "apple,arm-platform"; + model = "Apple MacBook Neo"; + + chosen { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + framebuffer0: framebuffer@0 { + compatible = "apple,simple-framebuffer", "simple-framebuffer"; + reg = <0 0 0 0>; /* To be filled by loader */ + /* Format properties will be added by loader */ + status = "disabled"; + }; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + /* To be filled by loader */ + }; + + memory@800000000 { + device_type = "memory"; + reg = <0x100 0 0x2 0>; /* To be filled by loader */ + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi new file mode 100644 index 0000000000000..73c09c0bce2a9 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -0,0 +1,756 @@ +&pmgr { + ps_sep_aon: power-controller@60 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x60 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sep_aon"; + apple,always-on; + }; + + ps_msg: power-controller@108 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x108 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msg"; + }; + + ps_aic: power-controller@110 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aic"; + apple,always-on; + }; + + ps_dwi: power-controller@118 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x118 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dwi"; + }; + + ps_gpio: power-controller@120 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x120 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gpio"; + }; + + ps_pmc: power-controller@138 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x138 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmc"; + }; + + ps_pms_fpwm0: power-controller@140 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x140 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm0"; + }; + + ps_pms_fpwm1: power-controller@148 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x148 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm1"; + }; + + ps_pms_fpwm2: power-controller@150 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x150 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm2"; + }; + + ps_pms_fpwm3: power-controller@158 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x158 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm3"; + }; + + ps_pms_fpwm4: power-controller@160 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x160 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm4"; + }; + + ps_pms_c1ppt: power-controller@168 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x168 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_c1ppt"; + apple,always-on; + }; + + ps_soc_rc: power-controller@170 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x170 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "soc_rc"; + }; + + ps_soc_dpe: power-controller@178 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x178 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "soc_dpe"; + apple,always-on; + }; + + ps_pmgr_soc_ocla: power-controller@180 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x180 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmgr_soc_ocla"; + }; + + ps_ispsens0: power-controller@188 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x188 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens0"; + }; + + ps_ispsens1: power-controller@190 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x190 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens1"; + }; + + ps_ap_tmm: power-controller@1a0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ap_tmm"; + }; + + ps_afggfxni0: power-controller@1b8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "afggfxni0"; + apple,always-on; + }; + + ps_gfx: power-controller@1f0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gfx"; + }; + + ps_jpg: power-controller@230 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x230 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "jpg"; + }; + + ps_msr: power-controller@238 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x238 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msr"; + }; + + ps_venc0_sys: power-controller@240 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x240 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_sys"; + }; + + ps_prores: power-controller@250 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x250 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "prores"; + }; + + ps_apcie_gp: power-controller@258 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x258 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_gp"; + }; + + ps_isp_sys: power-controller@268 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x268 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_sys"; + }; + + ps_ans: power-controller@278 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x278 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ans"; + }; + + ps_disp_sys: power-controller@280 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x280 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_sys"; + apple,always-on; /* TODO: figure out if we can enable PM here */ + }; + + ps_dispext0_sys: power-controller@288 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x288 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_sys"; + }; + + ps_ane_sys: power-controller@290 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x290 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_sys"; + }; + + ps_msr_ase_core: power-controller@298 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x298 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msr_ase_core"; + power-domains = <&ps_msr>; + }; + + ps_apcie_sys_gp: power-controller@2a0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_sys_gp"; + power-domains = <&ps_apcie_gp>; + }; + + ps_sio_cpu: power-controller@2a8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sio_cpu"; + }; + + ps_fpwm0: power-controller@2b0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm0"; + }; + + ps_fpwm1: power-controller@2b8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm1"; + }; + + ps_fpwm2: power-controller@2c0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm2"; + }; + + ps_i2c0: power-controller@2c8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c0"; + }; + + ps_i2c1: power-controller@2d0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c1"; + }; + + ps_i2c2: power-controller@2d8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c2"; + }; + + ps_i2c3: power-controller@2e0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c3"; + }; + + ps_i2c4: power-controller@2e8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c4"; + }; + + ps_spi_p: power-controller@2f0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi_p"; + }; + + ps_uart_p: power-controller@2f8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart_p"; + }; + + ps_dpa_p: power-controller@300 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x300 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa_p"; + apple,always-on; + }; + + ps_aes: power-controller@308 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x308 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aes"; + }; + + ps_apcie_st: power-controller@310 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x310 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_st"; + power-domains = <&ps_ans>; + }; + + ps_disp_fe: power-controller@318 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x318 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_fe"; + power-domains = <&ps_disp_sys>; + apple,always-on; /* TODO: figure out if we can enable PM here */ + }; + + ps_dispext0_fe: power-controller@320 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x320 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_fe"; + power-domains = <&ps_dispext0_sys>; + }; + + ps_spi0: power-controller@330 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x330 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi0"; + power-domains = <&ps_spi_p>; + }; + + ps_spi1: power-controller@338 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x338 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi1"; + power-domains = <&ps_spi_p>; + }; + + ps_spi2: power-controller@340 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x340 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi2"; + power-domains = <&ps_spi_p>; + }; + + ps_spi3: power-controller@348 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x348 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi3"; + power-domains = <&ps_spi_p>; + }; + + ps_qspi: power-controller@350 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x350 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "qspi"; + power-domains = <&ps_spi_p>; + }; + + ps_uart_n: power-controller@358 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x358 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart_n"; + power-domains = <&ps_uart_p>; + }; + + ps_uart0: power-controller@360 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x360 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart0"; + power-domains = <&ps_uart_p>; + }; + + ps_uart1: power-controller@368 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x368 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart1"; + power-domains = <&ps_uart_p>; + }; + + ps_uart2: power-controller@370 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x370 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart2"; + power-domains = <&ps_uart_p>; + }; + + ps_uart3: power-controller@378 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x378 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart3"; + power-domains = <&ps_uart_p>; + }; + + ps_uart4: power-controller@380 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x380 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart4"; + power-domains = <&ps_uart_p>; + }; + + ps_uart5: power-controller@388 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x388 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart5"; + power-domains = <&ps_uart_p>; + }; + + ps_dpa0: power-controller@390 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x390 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa0"; + power-domains = <&ps_dpa_p>; + }; + + ps_dpa1: power-controller@398 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x398 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa1"; + power-domains = <&ps_dpa_p>; + }; + + ps_apcie_sys_st: power-controller@3a0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_sys_st"; + power-domains = <&ps_ans>, <&ps_apcie_st>; + }; + + ps_apcie_phy_sw: power-controller@3c0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_phy_sw"; + power-domains = <&ps_apcie_sys_st>, <&ps_apcie_sys_gp>; + }; + + ps_pmp: power-controller@3d0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmp"; + apple,always-on; + }; + + ps_pms_sram: power-controller@3d8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_sram"; + apple,always-on; + }; + + ps_atc0_usb_aon: power-controller@3e0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_usb_aon"; + }; + + ps_atc0_common: power-controller@3e8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_common"; + power-domains = <&ps_atc0_usb_aon>; + }; + + ps_atc0_usb: power-controller@3f0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_usb"; + power-domains = <&ps_atc0_common>; + }; + + ps_sep: power-controller@c00 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc00 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sep"; + apple,always-on; + }; + + ps_venc0_dma: power-controller@8000 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_dma"; + power-domains = <&ps_venc0_sys>; + }; + + ps_venc0_pipe4: power-controller@8008 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_pipe4"; + power-domains = <&ps_venc0_dma>; + }; + + ps_venc0_pipe5: power-controller@8010 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_pipe5"; + power-domains = <&ps_venc0_dma>; + }; + + ps_venc0_me0: power-controller@8018 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_me0"; + power-domains = <&ps_venc0_dma>; + }; + + ps_venc0_me1: power-controller@8020 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_me1"; + power-domains = <&ps_venc0_me0>; + }; +}; + +&pmgr_mini { + ps_debug_gated: power-controller@0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "debug_gated"; + apple,always-on; + }; + + ps_nub_spmi0: power-controller@58 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x58 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi0"; + apple,always-on; + }; + + ps_nub_spmi1: power-controller@60 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x60 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi1"; + apple,always-on; + }; + + ps_nub_spmi_a0: power-controller@68 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x68 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi_a0"; + apple,always-on; + }; + + ps_nub_aon_nogate: power-controller@78 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x78 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_aon_nogate"; + apple,always-on; + }; + + ps_nub_ocla: power-controller@80 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x80 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_ocla"; + apple,always-on; + }; + + ps_nub_gpio: power-controller@88 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x88 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_gpio"; + apple,always-on; + }; + + ps_nub_fabric: power-controller@a0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xa0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_fabric"; + apple,always-on; + }; + + ps_nub_sram: power-controller@a8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xa8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_sram"; + apple,always-on; + }; + + ps_debug_switch: power-controller@b0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xb0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "debug_switch"; + apple,always-on; + }; + + ps_smc_i2cm0: power-controller@8028 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "smc_i2cm0"; + }; + + ps_smc_i2cm1: power-controller@8030 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "smc_i2cm1"; + }; + + ps_smc_i2cm2: power-controller@8038 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8038 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "smc_i2cm2"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi new file mode 100644 index 0000000000000..f2199ded4b783 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple T8140 "A18 Pro" SoC + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include + +/ { + compatible = "apple,t8140", "apple,arm-platform"; + + #address-cells = <2>; + #size-cells = <2>; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + cpu-map { + cluster0 { + core0 { + cpu = <&cpu_e0>; + }; + core1 { + cpu = <&cpu_e1>; + }; + core2 { + cpu = <&cpu_e2>; + }; + core3 { + cpu = <&cpu_e3>; + }; + }; + + cluster1 { + core0 { + cpu = <&cpu_p0>; + }; + core1 { + cpu = <&cpu_p1>; + }; + }; + }; + + cpu_e0: cpu@0 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x0>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; + next-level-cache = <&l2_cache_0>; + }; + + cpu_e1: cpu@1 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x1>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; + next-level-cache = <&l2_cache_0>; + }; + cpu_e2: cpu@2 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x2>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; + next-level-cache = <&l2_cache_0>; + }; + + cpu_e3: cpu@3 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x3>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; + next-level-cache = <&l2_cache_0>; + }; + + cpu_p0: cpu@10100 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10100>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; + next-level-cache = <&l2_cache_1>; + }; + + cpu_p1: cpu@10101 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10101>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; + next-level-cache = <&l2_cache_1>; + }; + + l2_cache_0: l2-cache-0 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + cache-size = <0x400000>; + }; + + l2_cache_1: l2-cache-1 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + cache-size = <0x1000000>; + }; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupt-parent = <&aic>; + interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt"; + interrupts = , + , + , + ; + }; + + clkref: clock-ref { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <24000000>; + clock-output-names = "clkref"; + }; + + soc: soc { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + + nonposted-mmio; + ranges; + + pmgr: power-management@300700000 { + compatible = "apple,t8140-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x3 0x700000 0 0x14000>; + /* child nodes are added in t8140-pmgr.dtsi */ + }; + + aic: interrupt-controller@301000000 { + compatible = "apple,t8140-aic3", "apple,t8122-aic3"; + #interrupt-cells = <3>; + interrupt-controller; + reg = <0x00000003 0x01000000 0x00000000 0x1cc000>, + <0x00000003 0x01040000 0x00000000 0x4000>; + reg-names = "core", "event"; + power-domains = <&ps_aic>; + }; + + pmgr_mini: power-management@308280000 { + compatible = "apple,t8140-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x3 0x8280000 0 0xc000>; + /* child nodes are added in t8140-pmgr.dtsi */ + }; + + wdt: watchdog@3082b0000 { + compatible = "apple,t8140-wdt", "apple,wdt"; + reg = <0x3 0x082b0000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + }; + }; +}; + +#include "t8140-pmgr.dtsi" From 76c9423ddc9a780bdf5fe7a0d79cd3a4628e8607 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:45:20 +1000 Subject: [PATCH 0051/2680] arm64: dts: apple: Add T8140/J700 device trees with audio and camera MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add device trees for the A18 Pro (T8140) and the MacBook Neo (J700). t8140.dtsi describes the SoC as far as this series uses it: - six CPUs in two clusters, four efficiency and two performance cores, started through spin tables, with their L2 caches, the architected timer on AIC FIQs and the 24 MHz reference clock; - the two PMGR blocks, the AIC3 and the watchdog; - the GPIO and pin controller with its seven interrupt groups and the reserved pins 50 and 134 in gpio-reserved-ranges, I2C bus 0 and the DockChannel UART; - the Always-On Processor: its ASC mailbox, its setup-port mailbox, its DART, and the AOP node with its ALS and audio children, each DMA consumer with its DART stream and host DMA window; - the three audio ADMACs (base-ns, leap-ns and leap-s), without resets because the AOP firmware owns the audio fabric. leap-s is reached through its proxy aperture at 0x353200000: through the direct window at 0x313200000, the first write to a per-channel register raises an asynchronous SError; - the ISP with its three DARTs, which remap streams 1, 2, 6, 7, 10 and 11 onto stream 0. t8140-pmgr.dtsi adds the power domains those devices use: aic, gpio, i2c0, isp_sys and the thirteen ISP-local domains in the main PMGR block, and the AOP and audio fabric domains, with their parents, in the second one. The AOP firmware is already running at boot, so the AOP domains and the audio fabric parents it relies on are always-on (letting genpd switch audio_p off left the firmware unable to answer power requests); the leaf domains are runtime managed. t8140-j700.dts enables the serial console and adds the framebuffer, memory and reserved-memory placeholders that the bootloader fills in, the IMX558 camera presets and ISP heap from isp-imx558.dtsi together with the ISP platform ID and temporal filter, the CS42L83 headset codec on i2c0, the MAX98360A speaker amplifier, and an apple,j700-macaudio sound card with a headset link and a speaker link to the AOP audio DAIs. The five CS42L83 supplies (VA, VP, VCP, VD_FILT and VL) are left unwired, as on the T8103 boards such as j313. Their board-level rails are not known, and fixed regulators invented for them would describe hardware that has not been established; without them the regulator core substitutes dummy supplies. dtbs_check reports the supplies as missing, as it does for those boards. [ryan: adapted to 7.2.10 — applied on top of Yureka Lilian's minimal MacBook Neo tree, which now comes first in this topic; the J700 files are rewritten to this commit's content] Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 109 +++- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 717 +++++----------------- arch/arm64/boot/dts/apple/t8140.dtsi | 414 ++++++++++++- 3 files changed, 668 insertions(+), 572 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 76b1537428ee5..c52879cab7d4f 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -15,10 +15,15 @@ compatible = "apple,j700", "apple,t8140", "apple,arm-platform"; model = "Apple MacBook Neo"; + aliases { + serial0 = &serial0; + }; + chosen { #address-cells = <2>; #size-cells = <2>; ranges; + stdout-path = "serial0"; framebuffer0: framebuffer@0 { compatible = "apple,simple-framebuffer", "simple-framebuffer"; @@ -35,8 +40,110 @@ /* To be filled by loader */ }; - memory@800000000 { + memory@10000000000 { device_type = "memory"; reg = <0x100 0 0x2 0>; /* To be filled by loader */ }; }; + +#include "isp-imx558.dtsi" + +&serial0 { + status = "okay"; +}; + +&pinctrl_ap { + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + jack_codec: codec@48 { + compatible = "cirrus,cs42l83"; + reg = <0x48>; + reset-gpios = <&pinctrl_ap 103 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <104 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + +&isp { + apple,platform-id = <0x70>; + apple,temporal-filter = <1>; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_mbox { + status = "okay"; +}; + +&aop_setup_mbox { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J700"; + apple,machine-kind = "MacBook Neo"; +}; + +&admac_leap_ns { + status = "okay"; +}; + +&admac_base_ns { + status = "okay"; +}; + +&admac_leap_s { + status = "okay"; +}; + +/ { + speaker_amp: audio-amplifier { + compatible = "maxim,max98360a"; + sdmode-gpios = <&pinctrl_ap 105 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Speaker"; + }; + sound { + compatible = "apple,j700-macaudio", "apple,macaudio"; + model = "MacBook Neo J700"; + #address-cells = <1>; + #size-cells = <0>; + + dai-link@0 { + reg = <0>; + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&aop_audio 0>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + + dai-link@1 { + reg = <1>; + link-name = "Speaker"; + + cpu { + sound-dai = <&aop_audio 4>; + }; + codec { + sound-dai = <&speaker_amp>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 73c09c0bce2a9..482a51c02bd75 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -1,21 +1,9 @@ -&pmgr { - ps_sep_aon: power-controller@60 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x60 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "sep_aon"; - apple,always-on; - }; - - ps_msg: power-controller@108 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x108 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "msg"; - }; +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* Power states used by the T8140 audio/ISP test topology. + * Copyright The Asahi Linux Contributors + */ +&pmgr { ps_aic: power-controller@110 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x110 4>; @@ -25,14 +13,6 @@ apple,always-on; }; - ps_dwi: power-controller@118 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x118 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "dwi"; - }; - ps_gpio: power-controller@120 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x120 4>; @@ -41,169 +21,6 @@ label = "gpio"; }; - ps_pmc: power-controller@138 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x138 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "pmc"; - }; - - ps_pms_fpwm0: power-controller@140 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x140 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "pms_fpwm0"; - }; - - ps_pms_fpwm1: power-controller@148 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x148 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "pms_fpwm1"; - }; - - ps_pms_fpwm2: power-controller@150 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x150 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "pms_fpwm2"; - }; - - ps_pms_fpwm3: power-controller@158 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x158 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "pms_fpwm3"; - }; - - ps_pms_fpwm4: power-controller@160 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x160 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "pms_fpwm4"; - }; - - ps_pms_c1ppt: power-controller@168 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x168 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "pms_c1ppt"; - apple,always-on; - }; - - ps_soc_rc: power-controller@170 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x170 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "soc_rc"; - }; - - ps_soc_dpe: power-controller@178 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x178 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "soc_dpe"; - apple,always-on; - }; - - ps_pmgr_soc_ocla: power-controller@180 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x180 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "pmgr_soc_ocla"; - }; - - ps_ispsens0: power-controller@188 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x188 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "ispsens0"; - }; - - ps_ispsens1: power-controller@190 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x190 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "ispsens1"; - }; - - ps_ap_tmm: power-controller@1a0 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x1a0 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "ap_tmm"; - }; - - ps_afggfxni0: power-controller@1b8 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x1b8 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "afggfxni0"; - apple,always-on; - }; - - ps_gfx: power-controller@1f0 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x1f0 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "gfx"; - }; - - ps_jpg: power-controller@230 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x230 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "jpg"; - }; - - ps_msr: power-controller@238 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x238 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "msr"; - }; - - ps_venc0_sys: power-controller@240 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x240 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "venc0_sys"; - }; - - ps_prores: power-controller@250 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x250 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "prores"; - }; - - ps_apcie_gp: power-controller@258 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x258 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "apcie_gp"; - }; - ps_isp_sys: power-controller@268 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x268 4>; @@ -212,89 +29,6 @@ label = "isp_sys"; }; - ps_ans: power-controller@278 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x278 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "ans"; - }; - - ps_disp_sys: power-controller@280 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x280 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "disp_sys"; - apple,always-on; /* TODO: figure out if we can enable PM here */ - }; - - ps_dispext0_sys: power-controller@288 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x288 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "dispext0_sys"; - }; - - ps_ane_sys: power-controller@290 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x290 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "ane_sys"; - }; - - ps_msr_ase_core: power-controller@298 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x298 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "msr_ase_core"; - power-domains = <&ps_msr>; - }; - - ps_apcie_sys_gp: power-controller@2a0 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x2a0 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "apcie_sys_gp"; - power-domains = <&ps_apcie_gp>; - }; - - ps_sio_cpu: power-controller@2a8 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x2a8 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "sio_cpu"; - }; - - ps_fpwm0: power-controller@2b0 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x2b0 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "fpwm0"; - }; - - ps_fpwm1: power-controller@2b8 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x2b8 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "fpwm1"; - }; - - ps_fpwm2: power-controller@2c0 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x2c0 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "fpwm2"; - }; - ps_i2c0: power-controller@2c8 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x2c8 4>; @@ -303,454 +37,303 @@ label = "i2c0"; }; - ps_i2c1: power-controller@2d0 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x2d0 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "i2c1"; - }; - - ps_i2c2: power-controller@2d8 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x2d8 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "i2c2"; - }; - - ps_i2c3: power-controller@2e0 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x2e0 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "i2c3"; - }; - - ps_i2c4: power-controller@2e8 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x2e8 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "i2c4"; - }; - - ps_spi_p: power-controller@2f0 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x2f0 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "spi_p"; - }; - - ps_uart_p: power-controller@2f8 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x2f8 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "uart_p"; - }; - - ps_dpa_p: power-controller@300 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x300 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "dpa_p"; - apple,always-on; - }; - - ps_aes: power-controller@308 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x308 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "aes"; - }; - - ps_apcie_st: power-controller@310 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x310 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "apcie_st"; - power-domains = <&ps_ans>; - }; - - ps_disp_fe: power-controller@318 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x318 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "disp_fe"; - power-domains = <&ps_disp_sys>; - apple,always-on; /* TODO: figure out if we can enable PM here */ - }; - - ps_dispext0_fe: power-controller@320 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x320 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "dispext0_fe"; - power-domains = <&ps_dispext0_sys>; - }; - - ps_spi0: power-controller@330 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x330 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "spi0"; - power-domains = <&ps_spi_p>; - }; - - ps_spi1: power-controller@338 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x338 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "spi1"; - power-domains = <&ps_spi_p>; - }; - - ps_spi2: power-controller@340 { + /* + * ISP-local power domains (live J700 ADT: ISP_CPU .. ISP_CVM, "no-ps" + * entries in the main PMGR block). As on t8112, the dependency tree is + * left to the ISP driver, which sequences them in the ADT parent-first + * order (ISP_SYS -> ISP_CPU -> cores, ISP_SYS -> ISP_FE -> ...). The + * unnamed slot at 0x4020 is the ISP_DPRX domain, which the firmware + * enables between ISP_FE and ISP_SECURE. Plain power + * gating only: a local domain parked with DEV_DISABLE (the t8112 + * apple,force-disable pattern) never reached ACTIVE again on J700 + * (isp_cpu_core0 stuck at ACTUAL=0 on the second firmware boot). + */ + ps_isp_cpu: power-controller@4000 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x340 4>; + reg = <0x4000 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "spi2"; - power-domains = <&ps_spi_p>; + label = "isp_cpu"; }; - ps_spi3: power-controller@348 { + ps_isp_cpu_core0: power-controller@4008 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x348 4>; + reg = <0x4008 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "spi3"; - power-domains = <&ps_spi_p>; + label = "isp_cpu_core0"; }; - ps_qspi: power-controller@350 { + ps_isp_cpu_core1: power-controller@4010 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x350 4>; + reg = <0x4010 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "qspi"; - power-domains = <&ps_spi_p>; + label = "isp_cpu_core1"; }; - ps_uart_n: power-controller@358 { + ps_isp_fe: power-controller@4018 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x358 4>; + reg = <0x4018 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "uart_n"; - power-domains = <&ps_uart_p>; + label = "isp_fe"; }; - ps_uart0: power-controller@360 { + ps_isp_dprx: power-controller@4020 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x360 4>; + reg = <0x4020 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "uart0"; - power-domains = <&ps_uart_p>; + label = "isp_dprx"; }; - ps_uart1: power-controller@368 { + ps_isp_secure: power-controller@4028 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x368 4>; + reg = <0x4028 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "uart1"; - power-domains = <&ps_uart_p>; + label = "isp_secure"; }; - ps_uart2: power-controller@370 { + ps_isp_sts0: power-controller@4030 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x370 4>; + reg = <0x4030 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "uart2"; - power-domains = <&ps_uart_p>; + label = "isp_sts0"; }; - ps_uart3: power-controller@378 { + ps_isp_sts1: power-controller@4038 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x378 4>; + reg = <0x4038 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "uart3"; - power-domains = <&ps_uart_p>; + label = "isp_sts1"; }; - ps_uart4: power-controller@380 { + ps_isp_pearl: power-controller@4040 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x380 4>; + reg = <0x4040 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "uart4"; - power-domains = <&ps_uart_p>; + label = "isp_pearl"; }; - ps_uart5: power-controller@388 { + ps_isp_be: power-controller@4048 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x388 4>; + reg = <0x4048 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "uart5"; - power-domains = <&ps_uart_p>; + label = "isp_be"; }; - ps_dpa0: power-controller@390 { + ps_isp_clr: power-controller@4050 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x390 4>; + reg = <0x4050 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "dpa0"; - power-domains = <&ps_dpa_p>; + label = "isp_clr"; }; - ps_dpa1: power-controller@398 { + ps_isp_cvd: power-controller@4058 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x398 4>; + reg = <0x4058 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "dpa1"; - power-domains = <&ps_dpa_p>; + label = "isp_cvd"; }; - ps_apcie_sys_st: power-controller@3a0 { + ps_isp_cvm: power-controller@4060 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x3a0 4>; + reg = <0x4060 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "apcie_sys_st"; - power-domains = <&ps_ans>, <&ps_apcie_st>; - }; - - ps_apcie_phy_sw: power-controller@3c0 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x3c0 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "apcie_phy_sw"; - power-domains = <&ps_apcie_sys_st>, <&ps_apcie_sys_gp>; + label = "isp_cvm"; }; +}; - ps_pmp: power-controller@3d0 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x3d0 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "pmp"; +&pmgr_mini { + /* + * AOP and audio-fabric power domains (live J700 ADT /arm-io/pmgr entries + * flagged "no-ps", register slots in this block at 0x4000+). Parents are + * the ADT parents. The speaker path needs them enabled parent-first: + * aop_filter, audio_a, audio_fr, audio_p, audio_leap_mca, audio_leap_a, + * audio_leap_c, audio_leap_ns_a, audio_leap_tx0. The AOP firmware is + * loaded by iBoot and already running when Linux boots, so the AOP + * domains iBoot leaves on are always-on (aop2_cpu, aop2_piodma, + * aop_he_nrcs0/1, aop_i2cm3 and aop_spi1 are off at boot and the HE + * NRCS pair does not power up, so those stay unmanaged). The audio + * fabric parents (audio_a, audio_fr, audio_p, audio_leap_a, + * audio_leap_c) are on at boot and the AOP firmware relies on them: + * letting genpd switch audio_p off during a runtime-PM cycle of + * audio_leap_mca left the firmware unable to answer LPAI power + * requests, so they are always-on; only the leaves (audio_leap_tx0, + * audio_leap_mca, audio_leap_ns_a, ...) are runtime-managed. + */ + ps_aop_filter: power-controller@4000 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_filter"; apple,always-on; }; - ps_pms_sram: power-controller@3d8 { + ps_aop_base: power-controller@4010 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x3d8 4>; + reg = <0x4010 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "pms_sram"; + label = "aop_base"; + power-domains = <&ps_aop_filter>; apple,always-on; }; - ps_atc0_usb_aon: power-controller@3e0 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x3e0 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "atc0_usb_aon"; - }; - - ps_atc0_common: power-controller@3e8 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x3e8 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "atc0_common"; - power-domains = <&ps_atc0_usb_aon>; - }; - - ps_atc0_usb: power-controller@3f0 { + ps_aop_fr: power-controller@4018 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x3f0 4>; + reg = <0x4018 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "atc0_usb"; - power-domains = <&ps_atc0_common>; - }; - - ps_sep: power-controller@c00 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0xc00 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "sep"; + label = "aop_fr"; + power-domains = <&ps_aop_filter>; apple,always-on; }; - ps_venc0_dma: power-controller@8000 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x8000 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "venc0_dma"; - power-domains = <&ps_venc0_sys>; - }; - - ps_venc0_pipe4: power-controller@8008 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x8008 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "venc0_pipe4"; - power-domains = <&ps_venc0_dma>; - }; - - ps_venc0_pipe5: power-controller@8010 { + ps_audio_a: power-controller@4020 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x8010 4>; + reg = <0x4020 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "venc0_pipe5"; - power-domains = <&ps_venc0_dma>; - }; - - ps_venc0_me0: power-controller@8018 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x8018 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "venc0_me0"; - power-domains = <&ps_venc0_dma>; - }; - - ps_venc0_me1: power-controller@8020 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x8020 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "venc0_me1"; - power-domains = <&ps_venc0_me0>; + label = "audio_a"; + /* AOP-owned audio fabric: on at boot, the firmware expects it to stay on */ + apple,always-on; + power-domains = <&ps_aop_filter>; }; -}; -&pmgr_mini { - ps_debug_gated: power-controller@0 { + ps_aop_cpu: power-controller@4098 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x0 4>; + reg = <0x4098 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "debug_gated"; + label = "aop_cpu"; + power-domains = <&ps_aop_base>, <&ps_aop_fr>; apple,always-on; }; - ps_nub_spmi0: power-controller@58 { + ps_audio_fr: power-controller@40a8 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x58 4>; + reg = <0x40a8 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "nub_spmi0"; + label = "audio_fr"; + /* AOP-owned audio fabric: on at boot, the firmware expects it to stay on */ apple,always-on; + power-domains = <&ps_audio_a>; }; - ps_nub_spmi1: power-controller@60 { + ps_audio_p: power-controller@40d0 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x60 4>; + reg = <0x40d0 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "nub_spmi1"; + label = "audio_p"; + /* AOP-owned audio fabric: on at boot, the firmware expects it to stay on */ apple,always-on; + power-domains = <&ps_audio_fr>; }; - ps_nub_spmi_a0: power-controller@68 { + ps_audio_leap_a: power-controller@40d8 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x68 4>; + reg = <0x40d8 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "nub_spmi_a0"; + label = "audio_leap_a"; + /* AOP-owned audio fabric: on at boot, the firmware expects it to stay on */ apple,always-on; + power-domains = <&ps_audio_fr>; }; - ps_nub_aon_nogate: power-controller@78 { + ps_audio_base_ns_a: power-controller@40f0 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x78 4>; + reg = <0x40f0 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "nub_aon_nogate"; - apple,always-on; + label = "audio_base_ns_a"; + power-domains = <&ps_audio_p>; }; - ps_nub_ocla: power-controller@80 { + ps_audio_leap_mca: power-controller@40f8 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x80 4>; + reg = <0x40f8 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "nub_ocla"; - apple,always-on; + label = "audio_leap_mca"; + power-domains = <&ps_audio_p>; }; - ps_nub_gpio: power-controller@88 { + ps_audio_mca0_m: power-controller@4100 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x88 4>; + reg = <0x4100 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "nub_gpio"; - apple,always-on; + label = "audio_mca0_m"; + power-domains = <&ps_audio_p>; }; - ps_nub_fabric: power-controller@a0 { + ps_audio_mca1_m: power-controller@4108 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0xa0 4>; + reg = <0x4108 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "nub_fabric"; - apple,always-on; + label = "audio_mca1_m"; + power-domains = <&ps_audio_p>; }; - ps_nub_sram: power-controller@a8 { + ps_audio_leap_c: power-controller@4130 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0xa8 4>; + reg = <0x4130 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "nub_sram"; + label = "audio_leap_c"; + /* AOP-owned audio fabric: on at boot, the firmware expects it to stay on */ apple,always-on; + power-domains = <&ps_audio_leap_a>; }; - ps_debug_switch: power-controller@b0 { + ps_audio_leap_s_a: power-controller@4138 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0xb0 4>; + reg = <0x4138 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "debug_switch"; - apple,always-on; + label = "audio_leap_s_a"; + power-domains = <&ps_audio_leap_c>; }; - ps_smc_i2cm0: power-controller@8028 { + ps_audio_leap_ns_a: power-controller@4140 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x8028 4>; + reg = <0x4140 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "smc_i2cm0"; + label = "audio_leap_ns_a"; + power-domains = <&ps_audio_leap_c>; }; - ps_smc_i2cm1: power-controller@8030 { + ps_audio_leap_rx0: power-controller@4148 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x8030 4>; + reg = <0x4148 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "smc_i2cm1"; + label = "audio_leap_rx0"; + power-domains = <&ps_audio_leap_c>; }; - ps_smc_i2cm2: power-controller@8038 { + ps_audio_leap_tx0: power-controller@4170 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x8038 4>; + reg = <0x4170 4>; #power-domain-cells = <0>; #reset-cells = <0>; - label = "smc_i2cm2"; + label = "audio_leap_tx0"; + power-domains = <&ps_audio_leap_c>; }; }; diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index f2199ded4b783..2465f73e52b7e 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -7,6 +7,8 @@ #include #include +#include +#include / { compatible = "apple,t8140", "apple,arm-platform"; @@ -61,6 +63,8 @@ cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_0>; }; + + cpu_e2: cpu@2 { compatible = "apple,sawtooth"; device_type = "cpu"; @@ -117,9 +121,9 @@ interrupt-parent = <&aic>; interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt"; interrupts = , - , - , - ; + , + , + ; }; clkref: clock-ref { @@ -164,7 +168,7 @@ }; wdt: watchdog@3082b0000 { - compatible = "apple,t8140-wdt", "apple,wdt"; + compatible = "apple,t8140-wdt", "apple,t8103-wdt"; reg = <0x3 0x082b0000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -173,4 +177,406 @@ }; }; +&soc { + pinctrl_ap: pinctrl@31a000000 { + compatible = "apple,t8140-pinctrl", "apple,t8103-pinctrl"; + reg = <0x3 0x1a000000 0x0 0x100000>; + power-domains = <&ps_gpio>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_ap 0 0 216>; + apple,npins = <216>; + + /* Live J700 ADT /arm-io/gpio0 interrupts (7 groups). */ + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + /* ADT no-restore-gpio-pins entries: 50 and 134. */ + gpio-reserved-ranges = <50 1>, <134 1>; + status = "disabled"; + + i2c0_pins: i2c0-pins { + pinmux = , + ; + }; + + }; + + i2c0: i2c@385010000 { + compatible = "apple,t8140-i2c", "apple,t8103-i2c"; + reg = <0x3 0x85010000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c0_pins>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&ps_i2c0>; + status = "disabled"; + }; + + serial0: serial@30812c000 { + compatible = "apple,dockchannel-uart"; + reg = <0x3 0x0812c000 0x0 0x4000>, + <0x3 0x08128000 0x0 0x4000>, + <0x3 0x0810c000 0x0 0x4000>; + reg-names = "data", "config", "irq"; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + /* + * AOP (always-on processor). Live J700 ADT /arm-io/aop + * (arm-io child + 0x210000000): reg[2] is the AOP MMIO window, + * reg[0] the ASCWrap v6 block whose mailbox sits at +0x8000. + * ADT interrupts [391, 390, 393, 392] = send-not-empty, + * send-empty, recv-not-empty, recv-empty. The firmware + * (j700aop, RTKit 12) is iBoot-loaded and left stopped; Linux + * adopts it and never reloads it. + */ + aop_mbox: mailbox@310608000 { + compatible = "apple,t8140-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x10608000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + /* + * The AOP "setup port": the second mailbox the J700 AOP firmware + * boots through (ADT /arm-io/aop-exclave-mailbox, absolute + * address 0x35067c000, IRQs [397, 394] = recv-not-empty, + * send-empty). Register layout measured on hardware: + * A2I control +0x0 (FULL bit 16), I2A control +0x4 (EMPTY bit + * 17), send +0x180/+0x188, recv +0x1b0/+0x1b8. + */ + aop_setup_mbox: mailbox@35067c000 { + compatible = "apple,t8140-aop-setup-mailbox"; + reg = <0x3 0x5067c000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + /* + * Setup messages are exchanged through dart-aop stream 1, + * whose inherited root has its 64 GiB slot at the ADT + * vm-base-1 (0x20000000000) free for the host arena. + */ + iommus = <&aop_dart 1>; + apple,dma-range = <0x200 0x0 0x10 0x0>; + #mbox-cells = <0>; + status = "disabled"; + }; + + aop_dart: iommu@310fc0000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x3 0x10fc0000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_aop_base>; + /* + * iBoot leaves this DART protected (PROTECT = 0x1e7): every + * stream root is inherited, so each consumer binds exactly + * one stream and the locked-DART path publishes into it. + */ + status = "disabled"; + }; + + aop: aop@310c00000 { + compatible = "apple,t8140-aop"; + reg = <0x3 0x10c00000 0x0 0x1e0000>, + <0x3 0x10600000 0x0 0x88000>; + reg-names = "mmio", "asc"; + mboxes = <&aop_mbox>, <&aop_setup_mbox>; + mbox-names = "mbox", "setup"; + iommus = <&aop_dart 0>; + /* + * Stream 0's inherited root owns the 64 GiB slot at + * 0x10000000000 (the firmware's own mappings); the host's + * AFK buffers go into the next free slot of the ADT + * vm-base-0/vm-size-0 window. + */ + apple,dma-range = <0x110 0x0 0x10 0x0>; + power-domains = <&ps_aop_cpu>; + status = "disabled"; + + /* + * Low-power microphone: the AOP firmware produces 16 kHz + * stereo S32 frames into a 512 KiB host ring bound through + * dart-aop stream 9 (mapper-admac-base-s, ADT vm-base-9 / + * vm-size-9); no ADMAC. + */ + /* + * Ambient light sensor. The AOP firmware announces an + * "als" EPIC service on this machine (the platform device + * appears as als.1.auto and sits unbound without this + * node); the ADT has /arm-io/aop/iop-aop-nub/als with + * als-supports-color = 0, an ALS serial in /product and + * macOS's auto-brightness curve in /exbright. The driver + * matches on the generic compatible only and requires + * the machine-supplied CT817 setup calibration. + */ + aop_als: als { + compatible = "apple,t8140-aop-als", "apple,aop-als"; + /* intentionally empty */ + }; + + aop_audio: audio { + /* no apple,aop-audio fallback: the t8140 capture path + * (setup-port source ring, 16 kHz) is its own driver + */ + compatible = "apple,t8140-aop-audio"; + iommus = <&aop_dart 9>; + apple,dma-range = <0x100 0x40000000 0x0 0xffff0000>; + /* + * Internal speaker: LEAP TX0 (admac-leap-ns channel 0, + * dart-aop stream 10) fed while the AOP spkr service is + * in pwrd; the LEAP power chain is brought up parent- + * first from tx0 and leap_mca. + */ + dmas = <&admac_leap_ns 0>, <&admac_base_ns 4>, + <&admac_base_ns 5>, <&admac_base_ns 3>, + <&admac_leap_s 1>; + /* + * SpeakerTap is the native base-ns RX1 serial monitor. + * "hpai" is the high-quality microphone array: the ADT + * routes audio-hp-mic to leap-s-adma-rx-0, the machine's + * only iop-audio-s stream. The AOP attaches and powers + * the array for an ordinary EPIC client, so the host can + * own this channel directly. + * + * Channel 1, not 0: admac_chan_direction() hardwires + * even channels to MEM_TO_DEV and odd to DEV_TO_MEM, so + * the ADT's tx-N is Linux 2N and its rx-N is 2N+1 -- + * which is why cout (tx-2) is 4 and cin (rx-2) is 5 + * above. rx-0 is therefore 1; asking for 0 hands back + * a transmit channel, and admac_device_config() then + * rejects the capture slave config with -EINVAL. + */ + dma-names = "spkr", "cout", "cin", "spkr-tap", "hpai"; + /* + * 3.5 mm jack: base-ns TX2 (channel 4, dart-aop stream 8) + * into MCA1 while the AOP cout service is in pwrd, the + * headset microphone back from MCA1 on base-ns RX2 + * (channel 5) under the cin service; the serializer + * domain audio_mca1_m is the AP's to power. + */ + /* + * "rx0" is audio_leap_rx0, the exact sibling of + * audio_leap_tx0 that the speaker path holds. A + * LEAP ADMAC's per-channel SRAM registers only + * answer with its own leaf powered, so the + * microphone array's leap-s RX0 channel needs + * this one the way the speaker's leap-ns TX0 + * needs tx0; without it the first write to + * REG_CHAN_SRAM_CARVEOUT takes an asynchronous + * SError and panics the machine. + */ + power-domains = <&ps_audio_leap_tx0>, <&ps_audio_leap_mca>, + <&ps_audio_mca1_m>, <&ps_audio_mca0_m>, + <&ps_audio_leap_rx0>; + power-domain-names = "tx0", "mca", "mca1", "mca0", "rx0"; + #sound-dai-cells = <1>; + }; + }; + + /* + * The three "evolved" ADMAC instances of the audio fabric + * (live J700 ADT admac-base-ns / admac-leap-ns / admac-leap-s, + * compatible admac,t8140, admac-evolution = 1). Register + * ranges are the ADT ones; the per-channel control/descriptor/ + * report windows differ from t8103 and are described by the + * t8140 driver variant, not by older SoC offsets. DMA streams: + * dart-aop mapper-admac-base-ns 8, mapper-admac-leap-ns 10, + * mapper-admac-leap-s 11 (exclave-gated under macOS). + */ + admac_base_ns: dma-controller@313800000 { + compatible = "apple,t8140-admac"; + reg = <0x3 0x13800000 0x0 0x34000>; + dma-channels = <13>; + interrupts-extended = <&aic AIC_IRQ 449 IRQ_TYPE_LEVEL_HIGH>, + <0>, <0>, <0>; + #dma-cells = <1>; + iommus = <&aop_dart 8>; + /* + * Stream 8's inherited root has its slot 16 (IOVA 1 TiB.. + * 1 TiB + 64 GiB) owned by the firmware, and the locked + * publish path never writes into a firmware-owned slot, so + * the AP's window sits in the free slot 17. + */ + apple,dma-range = <0x110 0x00000000 0x0 0x10000000>; + power-domains = <&ps_audio_base_ns_a>; + /* no resets: the AOP owns the audio fabric (see admac_leap_ns) */ + status = "disabled"; + }; + + admac_leap_ns: dma-controller@313300000 { + compatible = "apple,t8140-admac"; + reg = <0x3 0x13300000 0x0 0x34000>; + dma-channels = <9>; + interrupts-extended = <&aic AIC_IRQ 452 IRQ_TYPE_LEVEL_HIGH>, + <0>, <0>, <0>; + #dma-cells = <1>; + iommus = <&aop_dart 10>; + /* stream 10's inherited root is empty; use the generic window */ + apple,dma-range = <0x100 0x40000000 0x0 0xffff0000>; + power-domains = <&ps_audio_leap_ns_a>; + /* + * No resets: the AOP firmware owns the LEAP fabric and a + * second reset pulse (driver reload) wedged the block. + */ + status = "disabled"; + }; + + /* + * Reached through admac-leap-s-rx-0-proxy (0x353200000), not the + * direct window at 0x313200000. The direct one answers its base + * registers -- the controller probes and registers all nine + * channels -- but the first write to a per-channel register + * (REG_CHAN_SRAM_CARVEOUT) takes an asynchronous SError, with + * the block powered every way this driver can power it + * (resets dropped, leap-ns leaves, and leap-s's own rx0 leaf all + * tried and eliminated on hardware). The ADT describes this + * proxy aperture and names admac-leap-s's + * service-exclave_proxy = DMAChannelProxy@admac-leap-s-rx-0-proxy + * precisely because the direct per-channel registers are not the + * non-secure world's to touch. + */ + admac_leap_s: dma-controller@353200000 { + compatible = "apple,t8140-admac"; + reg = <0x3 0x53200000 0x0 0x34000>; + dma-channels = <9>; + interrupts-extended = <&aic AIC_IRQ 451 IRQ_TYPE_LEVEL_HIGH>, + <0>, <0>, <0>; + #dma-cells = <1>; + iommus = <&aop_dart 11>; + power-domains = <&ps_audio_leap_s_a>; + /* + * No resets, for the same reason as admac_leap_ns: the + * AOP firmware owns the LEAP fabric, and apple-admac + * pulses reset_control_reset() at probe. Resetting the + * secure LEAP ADMAC out from under the firmware leaves + * its per-channel/SRAM registers dead, and the first + * dma_request_chan() on it then takes an asynchronous + * SError and panics the machine. + */ + status = "disabled"; + }; + + /* + * ISP (live J700 ADT /arm-io/isp, sensor-type 0x70, H17 "ISP17a" + * firmware, iBoot-loaded). "coproc" is the ISP block, the IRQ + * block sits at +0x2920000 and the firmware GPIO words at + * +0x2921000 (measured on hardware). "wdt" is + * the EIC mmio-wd-isp window: on T8140 the host must kick it at + * 30 Hz while streaming or every frame is the diagnostic fill. + * The three dart-isp instances share one SID0 page-table root. + */ + isp_dart0: iommu@3228c0000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x3 0x228c0000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + /* + * 3-level tables (ias <= 36) as iBoot and the native host + * ran them: the ISP fetches at vm-base 0x10000000000 and the + * walk ignores the high bits; 32 GiB keeps the DAPF MMIO + * identity windows (up to 0x401660000) inside the aperture. + */ + apple,dma-range = <0x0 0x0 0x8 0x0>; + /* ADT "remap": streams 1 2 6 7 10 11 are stream 0 */ + apple,remap-streams = <1 0>, <2 0>, <6 0>, <7 0>, + <10 0>, <11 0>; + status = "disabled"; + }; + + isp_dart1: iommu@3228e0000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x3 0x228e0000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + /* + * 3-level tables (ias <= 36) as iBoot and the native host + * ran them: the ISP fetches at vm-base 0x10000000000 and the + * walk ignores the high bits; 32 GiB keeps the DAPF MMIO + * identity windows (up to 0x401660000) inside the aperture. + */ + apple,dma-range = <0x0 0x0 0x8 0x0>; + /* ADT "remap": streams 1 2 6 7 10 11 are stream 0 */ + apple,remap-streams = <1 0>, <2 0>, <6 0>, <7 0>, + <10 0>, <11 0>; + status = "disabled"; + }; + + isp_dart2: iommu@322900000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x3 0x22900000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + /* + * 3-level tables (ias <= 36) as iBoot and the native host + * ran them: the ISP fetches at vm-base 0x10000000000 and the + * walk ignores the high bits; 32 GiB keeps the DAPF MMIO + * identity windows (up to 0x401660000) inside the aperture. + */ + apple,dma-range = <0x0 0x0 0x8 0x0>; + /* ADT "remap": streams 1 2 6 7 10 11 are stream 0 */ + apple,remap-streams = <1 0>, <2 0>, <6 0>, <7 0>, + <10 0>, <11 0>; + status = "disabled"; + }; + + /* + * ISP17a. The ADT window is 0x320000000+0x3fe8000; "coproc" + * stops short of the secure route table at +0x1804000 (a read + * takes a sticky SError) and of the IRQ block (+0x2920000, + * "mbox": interrupt/enable words, doorbell/ack at +0x410) and + * the GPIO words (+0x2921000). "wdt" is the EIC's mmio-wd-isp + * window the host kicks at 30 Hz while streaming. + */ + isp: isp@320000000 { + compatible = "apple,t8140-isp", "apple,isp"; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + reg-names = "coproc", "mbox", "gpio", "wdt"; + reg = <0x3 0x20000000 0x0 0x1801000>, + <0x3 0x22920000 0x0 0x1000>, + <0x3 0x22921000 0x0 0x100>, + <0x3 0x63b50000 0x0 0x8000>; + interrupt-parent = <&aic>; + /* ADT [323 324 325 326 336]; 323 carries the IPC mailbox */ + interrupts = ; + power-domains = <&ps_isp_sys>, <&ps_isp_cpu>, + <&ps_isp_cpu_core0>, <&ps_isp_cpu_core1>, + <&ps_isp_fe>, <&ps_isp_dprx>, <&ps_isp_secure>, + <&ps_isp_sts0>, <&ps_isp_sts1>, <&ps_isp_pearl>, + <&ps_isp_be>, <&ps_isp_clr>, <&ps_isp_cvd>, + <&ps_isp_cvm>; + apple,dart-vm-size = <0x0 0xa0000000>; + status = "disabled"; + }; +}; + #include "t8140-pmgr.dtsi" From 467f085f7a018efcf5de0d90181928c7b252573c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:19:18 +1100 Subject: [PATCH 0052/2680] PCI: apple: retain preinitialized tunnels when cold init is disabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The board property permits experimental kernel cold initialization, but must not reject an existing successful m1n1 handoff. Admit the explicit preinit certificate in both the population gate and preparation helper without touching cold-init registers. [ryan: adapted to 7.2.10 — split by topic: the drivers/thunderbolt hunks go to 170-usb; aurora-wip carries this check in the PCIe-C prepare helper from commit 57b3f3af33bc ("PCI: apple: enumerate Thunderbolt PCIe on t8103"), not in the parallel helper of the superseded commit a3dca5d9848c ("PCI: apple: Cold-init PCIe-C tunnel ports without an m1n1 handoff"), so it lands there] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pci/controller/pcie-apple.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 923147b603030..0a5dbd73f7a95 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2713,6 +2713,13 @@ int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel) if (!np) return 0; + /* A qualified bootloader handoff needs no cold initialization. */ + if (!of_property_read_u32(np, "apple,pciec-preinit-status", &stat) && + stat == 1) { + ret = 0; + goto out_np; + } + match = of_match_node(apple_pcie_of_match, np); if (!match) { ret = -ENODEV; From 8ceccf56582420b2dcc3cbf3c37d33d608d6be1a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:31:24 +1000 Subject: [PATCH 0053/2680] arm64: dts: apple: t8140-pmgr: Add ANS, SPMI, PMP and SMC I2C domains Add the power domains used by the ANS2 storage controller and the SPMI controller, and the always-on domains of the power management co-processor and of the SMC I2C masters, placed in the existing blocks by register offset: - ans (0x278) and the domains of the PCIe link between the ANS2 controller and the NAND: apcie_gp (0x258), apcie_sys_gp (0x2a0), apcie_st (0x310), apcie_sys_st (0x3a0) and apcie_phy_sw (0x3c0), - pmp (0x3d0) and pms_sram (0x3d8), the power management co-processor and its SRAM, - nub_spmi0 (mini PMGR 0x58), the SPMI controller the PMIC sits on, - smc_i2cm1 and smc_i2cm2 (mini PMGR 0x8030, 0x8038), I2C masters used by the SMC firmware. The offsets and always-on flags match the PMGR description generated from the ADT for Yureka Lilian's A18 Pro series. apcie_sys_gp is given apcie_gp as its parent, as in the T8132 description; the generated A18 Pro description lists no parent for it. The ANS link has only been run with this dependency in place, so it is kept. The ANS co-processor is handed over running and cannot be reset through PMGR (asserting its reset raises an SError), so its domain is always-on. nub_spmi0, pmp, pms_sram and the two SMC I2C masters are always-on too; pmp, pms_sram and the I2C masters have no kernel consumer. The apcie domains are not always-on: the NVMe controller holds them once a board enables it. Also replace the file header with a plain description of the file. Link: https://lore.kernel.org/r/20260911-apple-a18-pro-initial-devicetrees-v2-7-08d06cf011d4@cyberchaos.dev Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 102 +++++++++++++++++++++- 1 file changed, 101 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 482a51c02bd75..7ee4109f2fca0 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0+ OR MIT -/* Power states used by the T8140 audio/ISP test topology. +/* + * PMGR Power domains for the Apple T8140 "A18 Pro" SoC + * * Copyright The Asahi Linux Contributors */ @@ -21,6 +23,14 @@ label = "gpio"; }; + ps_apcie_gp: power-controller@258 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x258 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_gp"; + }; + ps_isp_sys: power-controller@268 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x268 4>; @@ -29,6 +39,24 @@ label = "isp_sys"; }; + ps_ans: power-controller@278 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x278 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ans"; + apple,always-on; /* handed over running, cannot be reset */ + }; + + ps_apcie_sys_gp: power-controller@2a0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_sys_gp"; + power-domains = <&ps_apcie_gp>; + }; + ps_i2c0: power-controller@2c8 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x2c8 4>; @@ -37,6 +65,51 @@ label = "i2c0"; }; + ps_apcie_st: power-controller@310 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x310 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_st"; + power-domains = <&ps_ans>; + }; + + ps_apcie_sys_st: power-controller@3a0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_sys_st"; + power-domains = <&ps_ans>, <&ps_apcie_st>; + }; + + ps_apcie_phy_sw: power-controller@3c0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_phy_sw"; + power-domains = <&ps_apcie_sys_st>, <&ps_apcie_sys_gp>; + }; + + ps_pmp: power-controller@3d0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmp"; + apple,always-on; + }; + + ps_pms_sram: power-controller@3d8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_sram"; + apple,always-on; + }; + /* * ISP-local power domains (live J700 ADT: ISP_CPU .. ISP_CVM, "no-ps" * entries in the main PMGR block). As on t8112, the dependency tree is @@ -154,6 +227,15 @@ }; &pmgr_mini { + ps_nub_spmi0: power-controller@58 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x58 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi0"; + apple,always-on; + }; + /* * AOP and audio-fabric power domains (live J700 ADT /arm-io/pmgr entries * flagged "no-ps", register slots in this block at 0x4000+). Parents are @@ -336,4 +418,22 @@ label = "audio_leap_tx0"; power-domains = <&ps_audio_leap_c>; }; + + ps_smc_i2cm1: power-controller@8030 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "smc_i2cm1"; + apple,always-on; /* used by the SMC firmware */ + }; + + ps_smc_i2cm2: power-controller@8038 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8038 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "smc_i2cm2"; + apple,always-on; /* used by the SMC firmware */ + }; }; From 72a8fe9dbb6518361ab09a2aea1d66aa242630d7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:31:49 +1000 Subject: [PATCH 0054/2680] arm64: dts: apple: t8140: Add SPMI controller and PMIC NVMEM cells Add the nub SPMI controller at 0x308714000 (AIC interrupt 487, power domain nub_spmi0) and the Baku PMIC on it at USID 0xe. Export the PMIC scratch registers that the SMC reboot driver uses as NVMEM cells: the boot stage (0xf701), the boot error and panic counts (low and high nibble of 0xf702) and the shutdown flag (0xf70f, bit 3). The shutdown flag is what makes an SMC power-off request power the machine off instead of rebooting it. The ADT places the PMIC's legacy scratch area at 0xf700. The cell offsets and bit fields within it follow the layout used with the other Apple PMICs, as in the Stowe PMIC description on T8122; they have not been checked against what iBoot or macOS write on this machine. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 43 ++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 2465f73e52b7e..f69aaea240a44 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -9,6 +9,7 @@ #include #include #include +#include / { compatible = "apple,t8140", "apple,arm-platform"; @@ -174,6 +175,48 @@ interrupt-parent = <&aic>; interrupts = ; }; + + nub_spmi0: spmi@308714000 { + compatible = "apple,t8140-spmi", "apple,t8103-spmi"; + reg = <0x3 0x08714000 0x0 0x100>; + #address-cells = <2>; + #size-cells = <0>; + #interrupt-cells = <2>; + interrupt-controller; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_nub_spmi0>; + + pmic@e { + compatible = "apple,baku-pmic", "apple,spmi-nvmem"; + reg = <0xe SPMI_USID>; + + nvmem-layout { + compatible = "fixed-layout"; + #address-cells = <1>; + #size-cells = <1>; + + boot_stage: boot-stage@f701 { + reg = <0xf701 0x1>; + }; + + boot_error_count: boot-error-count@f702,0 { + reg = <0xf702 0x1>; + bits = <0 4>; + }; + + panic_count: panic-count@f702,4 { + reg = <0xf702 0x1>; + bits = <4 4>; + }; + + shutdown_flag: shutdown-flag@f70f,3 { + reg = <0xf70f 0x1>; + bits = <3 1>; + }; + }; + }; + }; }; }; From d33eb60439bda7a42d34d596e294b7dfe3ac8ebf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:32:01 +1000 Subject: [PATCH 0055/2680] arm64: dts: apple: t8140: Add SMC node Add the SMC and its ASC mailbox. The SMC register window at 0x30c600000 is the ADT's smc reg[0] and the SRAM at 0x30de00000 is the iop-smc-nub region-base. The mailbox sits at 0x30c608000; its AIC interrupts 581 to 584 are listed in the binding's send-empty, send-not-empty, recv-empty, recv-not-empty order. The existing macsmc driver attaches through the "apple,t8103-smc" fallback. Children: - gpio, the SMC GPIO controller, - hwmon, with the six CPU cluster die temperatures: Te05 and Te0S for the E cluster, Tp01, Tp05, Tp09 and Tp0D for the P cluster, - reboot, SMC reboot and power-off using the PMIC NVMEM cells. The CPU die channels are the first temperature channels on purpose. J700 carries an interim CPU thermal policy, until its CPU thermal limits are described as DT thermal zones, and that policy reads the first six temperature channels of this node. Other sensors belong to the board files. Both nodes are disabled here; the board enables them. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 76 ++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index f69aaea240a44..6051875d01aaa 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -217,6 +217,82 @@ }; }; }; + + smc: smc@30c600000 { + compatible = "apple,t8140-smc", "apple,t8103-smc"; + reg = <0x3 0x0c600000 0x0 0x4000>, + <0x3 0x0de00000 0x0 0x100000>; + reg-names = "smc", "sram"; + mboxes = <&smc_mbox>; + status = "disabled"; + + smc_gpio: gpio { + compatible = "apple,smc-gpio"; + gpio-controller; + #gpio-cells = <2>; + }; + + smc_hwmon: hwmon { + compatible = "apple,smc-hwmon"; + + /* + * CPU cluster die temperatures. Keep these + * first: the J700 CPU thermal policy reads + * the first six temperature channels. + */ + temperature-Te05 { + apple,key-id = "Te05"; + label = "E-cluster Temperature 1"; + }; + + temperature-Te0S { + apple,key-id = "Te0S"; + label = "E-cluster Temperature 2"; + }; + + temperature-Tp01 { + apple,key-id = "Tp01"; + label = "P-cluster Temperature 1"; + }; + + temperature-Tp05 { + apple,key-id = "Tp05"; + label = "P-cluster Temperature 2"; + }; + + temperature-Tp09 { + apple,key-id = "Tp09"; + label = "P-cluster Temperature 3"; + }; + + temperature-Tp0D { + apple,key-id = "Tp0D"; + label = "P-cluster Temperature 4"; + }; + }; + + smc_reboot: reboot { + compatible = "apple,smc-reboot"; + nvmem-cells = <&shutdown_flag>, <&boot_stage>, + <&boot_error_count>, <&panic_count>; + nvmem-cell-names = "shutdown_flag", "boot_stage", + "boot_error_count", "panic_count"; + }; + }; + + smc_mbox: mailbox@30c608000 { + compatible = "apple,t8140-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x0c608000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; }; }; From 1235072e83b6317fbc27bdb4e5329ec2453d3aa5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:32:14 +1000 Subject: [PATCH 0056/2680] arm64: dts: apple: t8140: Add ANS2 NVMe controller Add the ANS2 storage controller: its ASC mailbox at 0x389608000 (AIC 1007 to 1010), the SART address filter at 0x38dc50000 and the NVMe controller (AIC 1039). As on T8132, the NVMMU has its own register window, so the controller takes three regions: the non-secure controller alias at 0x3cdcc0000, the ANS co-processor at 0x389600000 and the NVMMU at 0x38dcc0000. The ADT sizes the controller alias at 0x10000, but the linear submission queue registers at +0x24908..+0x24910 lie in the same alias, so the region is 0x30000 long. The NVMMU window size, 0x60000, is the ADT's. Besides ans and apcie_sys_st the controller holds apcie_phy_sw, the third power domain the binding requires for T8140: with that domain off, the ANS PCIe link to the NAND falls to L1 idle and the firmware reports a link timeout on its first read. The ANS firmware is started by iBoot and handed over running, and how it is handed over depends on the boot chain, so the nodes are disabled here and enabled by the board. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 48 ++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 6051875d01aaa..f624455bdacf1 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -293,6 +293,54 @@ #mbox-cells = <0>; status = "disabled"; }; + + ans_mbox: mailbox@389608000 { + compatible = "apple,t8140-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x89608000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + power-domains = <&ps_ans>; + #mbox-cells = <0>; + status = "disabled"; + }; + + sart: iommu@38dc50000 { + compatible = "apple,t8140-sart", "apple,t6000-sart"; + reg = <0x3 0x8dc50000 0x0 0x10000>; + power-domains = <&ps_ans>; + status = "disabled"; + }; + + nvme: nvme@3cdcc0000 { + compatible = "apple,t8140-nvme-ans2", "apple,t8132-nvme-ans2"; + /* + * The ADT sizes the controller window at 0x10000, but the + * linear submission queue registers at +0x24908..+0x24910 + * sit in the same non-secure alias. + */ + reg = <0x3 0xcdcc0000 0x0 0x30000>, + <0x3 0x89600000 0x0 0x4000>, + <0x3 0x8dcc0000 0x0 0x60000>; + reg-names = "nvme", "ans", "nvmmu"; + interrupt-parent = <&aic>; + interrupts = ; + mboxes = <&ans_mbox>; + apple,sart = <&sart>; + /* + * With apcie_phy off the ANS PCIe link falls to L1 idle + * and the firmware reports a link timeout. + */ + power-domains = <&ps_ans>, <&ps_apcie_sys_st>, + <&ps_apcie_phy_sw>; + power-domain-names = "ans", "apcie0", "apcie_phy"; + resets = <&ps_ans>; + status = "disabled"; + }; }; }; From de8df941d52e30c01fb3a43de4d090c6f4162bb0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:32:30 +1000 Subject: [PATCH 0057/2680] arm64: dts: apple: t8140: Add CPU frequency scaling Add the DVFS controllers of the E cluster (0x210e20000) and the P cluster (0x211e20000) and their OPP tables, and point the CPU nodes at them. The tables are the ADT's voltage-states1-sram for the E cluster, six states from 816 MHz to 2.424 GHz, and voltage-states5-sram for the P cluster, 17 states from 744 MHz to 4.044 GHz. opp-level is the hardware P-state written to the controller. The controllers carry only the "apple,t8140-cluster-cpufreq" compatible, as the binding documents: the T8140 DVFS status register layout is not known, so no T8112 fallback is claimed. The transition latency has not been measured on this SoC, so the OPPs have no clock-latency-ns and apple-soc-cpufreq reports its DVFS transaction timeout as the transition latency instead. The CPU nodes have no capacity-dmips-mhz either; the core capacities have not been measured. With the interim J700 CPU thermal policy, the cpufreq driver holds both clusters at their lowest state until the board enables that policy. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 134 +++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index f624455bdacf1..8a22ca8b4c216 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -54,6 +54,8 @@ enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_0>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; }; cpu_e1: cpu@1 { @@ -63,6 +65,8 @@ enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_0>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; }; @@ -73,6 +77,8 @@ enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_0>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; }; cpu_e3: cpu@3 { @@ -82,6 +88,8 @@ enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_0>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; }; cpu_p0: cpu@10100 { @@ -91,6 +99,8 @@ enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_1>; + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; }; cpu_p1: cpu@10101 { @@ -100,6 +110,8 @@ enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_1>; + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; }; l2_cache_0: l2-cache-0 { @@ -117,6 +129,116 @@ }; }; + /* + * DVFS states from the ADT: voltage-states1-sram for the E cluster, + * voltage-states5-sram for the P cluster. opp-level is the hardware + * P-state. Transition latencies have not been measured, so there is + * no clock-latency-ns. + */ + ecluster_opp: opp-table-0 { + compatible = "operating-points-v2"; + opp-shared; + + opp01 { + opp-hz = /bits/ 64 <816000000>; + opp-level = <1>; + }; + opp02 { + opp-hz = /bits/ 64 <1140000000>; + opp-level = <2>; + }; + opp03 { + opp-hz = /bits/ 64 <1464000000>; + opp-level = <3>; + }; + opp04 { + opp-hz = /bits/ 64 <1788000000>; + opp-level = <4>; + }; + opp05 { + opp-hz = /bits/ 64 <2112000000>; + opp-level = <5>; + }; + opp06 { + opp-hz = /bits/ 64 <2424000000>; + opp-level = <6>; + }; + }; + + pcluster_opp: opp-table-1 { + compatible = "operating-points-v2"; + opp-shared; + + opp01 { + opp-hz = /bits/ 64 <744000000>; + opp-level = <1>; + }; + opp02 { + opp-hz = /bits/ 64 <1260000000>; + opp-level = <2>; + }; + opp03 { + opp-hz = /bits/ 64 <1620000000>; + opp-level = <3>; + }; + opp04 { + opp-hz = /bits/ 64 <1944000000>; + opp-level = <4>; + }; + opp05 { + opp-hz = /bits/ 64 <2196000000>; + opp-level = <5>; + }; + opp06 { + opp-hz = /bits/ 64 <2472000000>; + opp-level = <6>; + }; + opp07 { + opp-hz = /bits/ 64 <2688000000>; + opp-level = <7>; + }; + opp08 { + opp-hz = /bits/ 64 <2916000000>; + opp-level = <8>; + }; + opp09 { + opp-hz = /bits/ 64 <3144000000>; + opp-level = <9>; + }; + opp10 { + opp-hz = /bits/ 64 <3336000000>; + opp-level = <10>; + }; + opp11 { + opp-hz = /bits/ 64 <3504000000>; + opp-level = <11>; + }; + opp12 { + opp-hz = /bits/ 64 <3648000000>; + opp-level = <12>; + }; + opp13 { + opp-hz = /bits/ 64 <3756000000>; + opp-level = <13>; + }; + opp14 { + opp-hz = /bits/ 64 <3864000000>; + opp-level = <14>; + }; + opp15 { + opp-hz = /bits/ 64 <3924000000>; + opp-level = <15>; + }; + opp16 { + opp-hz = /bits/ 64 <3972000000>; + opp-level = <16>; + }; + opp17 { + opp-hz = /bits/ 64 <4044000000>; + opp-level = <17>; + }; + }; + timer { compatible = "arm,armv8-timer"; interrupt-parent = <&aic>; @@ -142,6 +264,18 @@ nonposted-mmio; ranges; + cpufreq_e: cpufreq@210e20000 { + compatible = "apple,t8140-cluster-cpufreq"; + reg = <0x2 0x10e20000 0x0 0x1000>; + #performance-domain-cells = <0>; + }; + + cpufreq_p: cpufreq@211e20000 { + compatible = "apple,t8140-cluster-cpufreq"; + reg = <0x2 0x11e20000 0x0 0x1000>; + #performance-domain-cells = <0>; + }; + pmgr: power-management@300700000 { compatible = "apple,t8140-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; #address-cells = <1>; From c7eb4ca954594b7edafc22ce642d23c6d9d03d2e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:32:42 +1000 Subject: [PATCH 0058/2680] arm64: dts: apple: t8140: Use the Tahiti CPU compatibles and L1 cache sizes Switch the CPU nodes from the Sawtooth and Everest compatibles to "apple,tahiti-e" and "apple,tahiti-p", the compatibles the CPU binding defines for the A18 Pro efficiency and performance cores, and add the L1 cache sizes so that cacheinfo reports them: 128 KiB instruction and 64 KiB data cache on the E cores, 192 KiB and 128 KiB on the P cores. The compatibles and cache sizes are those of Yureka Lilian's A18 Pro device tree series. Also drop a stray blank line between cpu@1 and cpu@2. Link: https://lore.kernel.org/r/20260911-apple-a18-pro-initial-devicetrees-v2-7-08d06cf011d4@cyberchaos.dev Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 8a22ca8b4c216..4967b4c620a2d 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -48,68 +48,79 @@ }; cpu_e0: cpu@0 { - compatible = "apple,sawtooth"; + compatible = "apple,tahiti-e"; device_type = "cpu"; reg = <0x0 0x0>; enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; operating-points-v2 = <&ecluster_opp>; performance-domains = <&cpufreq_e>; }; cpu_e1: cpu@1 { - compatible = "apple,sawtooth"; + compatible = "apple,tahiti-e"; device_type = "cpu"; reg = <0x0 0x1>; enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; operating-points-v2 = <&ecluster_opp>; performance-domains = <&cpufreq_e>; }; - cpu_e2: cpu@2 { - compatible = "apple,sawtooth"; + compatible = "apple,tahiti-e"; device_type = "cpu"; reg = <0x0 0x2>; enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; operating-points-v2 = <&ecluster_opp>; performance-domains = <&cpufreq_e>; }; cpu_e3: cpu@3 { - compatible = "apple,sawtooth"; + compatible = "apple,tahiti-e"; device_type = "cpu"; reg = <0x0 0x3>; enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; operating-points-v2 = <&ecluster_opp>; performance-domains = <&cpufreq_e>; }; cpu_p0: cpu@10100 { - compatible = "apple,everest"; + compatible = "apple,tahiti-p"; device_type = "cpu"; reg = <0x0 0x10100>; enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; operating-points-v2 = <&pcluster_opp>; performance-domains = <&cpufreq_p>; }; cpu_p1: cpu@10101 { - compatible = "apple,everest"; + compatible = "apple,tahiti-p"; device_type = "cpu"; reg = <0x0 0x10101>; enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; operating-points-v2 = <&pcluster_opp>; performance-domains = <&cpufreq_p>; }; From 15f53131f75346194738745d8aeeb33a7e74e755 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:32:55 +1000 Subject: [PATCH 0059/2680] arm64: dts: apple: t8140-j700: Enable ANS, SMC and CPU frequency scaling Enable the ANS2 NVMe controller and the SMC on the MacBook Neo, and add the SMC sensors shared by the Apple laptops through hwmon-common.dtsi and hwmon-laptop.dtsi, as the other laptop boards do. macsmc-hwmon skips keys that this SMC does not have. Set apple,cpu-thermal-policy on the SMC hwmon node. It enables the interim J700 CPU thermal policy, which stands in until the CPU thermal limits are described as DT thermal zones: the policy throttles on the six CPU die channels that the SoC file lists first, and holds both clusters at their lowest state whenever fresh readings are missing. Without it the T8140 cpufreq driver never leaves the lowest state, so this is what enables CPU frequency scaling here. The driver rejects the property on other machines, which is why it is set in the board file. Also set /chosen/bootargs to "arm64.nowfxt". This is a temporary measure. The secondary cores lose architectural state in WFIT, so SMP needs the kernel to stay off WFxT in its delay loop, and the kernel does not yet do that on T8140 by itself. Until it does, boot chains that pass this DT's /chosen/bootargs through unchanged rely on this property to boot with SMP. A loader that supplies its own command line replaces the property and has to pass arm64.nowfxt itself. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 33 ++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index c52879cab7d4f..70a8a889dbf9d 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -24,6 +24,11 @@ #size-cells = <2>; ranges; stdout-path = "serial0"; + /* + * Temporary: the secondary cores lose state in WFIT and the + * kernel does not yet stop using WFxT on T8140 by itself. + */ + bootargs = "arm64.nowfxt"; framebuffer0: framebuffer@0 { compatible = "apple,simple-framebuffer", "simple-framebuffer"; @@ -147,3 +152,31 @@ }; }; }; + +&smc { + status = "okay"; +}; + +&smc_hwmon { + /* interim J700 CPU thermal policy over the CPU die channels */ + apple,cpu-thermal-policy; +}; + +&smc_mbox { + status = "okay"; +}; + +&ans_mbox { + status = "okay"; +}; + +&sart { + status = "okay"; +}; + +&nvme { + status = "okay"; +}; + +#include "hwmon-common.dtsi" +#include "hwmon-laptop.dtsi" From e62f7838b670b1ddcdcc3b4a40c7b236574964b8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:30:54 +1000 Subject: [PATCH 0060/2680] arm64: dts: apple: t8140-j700: Name the ALS calibration file The AOP firmware of the J700 starts its CT817 ambient light sensor only after the application processor has sent the sensor's calibration over the setup-port mailbox. The calibration is the machine's own and is not distributed with the kernel. Name the file the AOP driver loads it from, so that an owner who has extracted their machine's calibration can install it under that name. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 70a8a889dbf9d..763cf79ffafce 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -97,6 +97,10 @@ status = "okay"; }; +&aop_als { + firmware-name = "apple/j700-aop-als-cal.bin"; +}; + &aop_audio { apple,chassis-name = "J700"; apple,machine-kind = "MacBook Neo"; From 9abf00f20c43bbca27f54a138bbff46382dcf39c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:26:40 +1000 Subject: [PATCH 0061/2680] arm64: dts: apple: t8140-pmgr: Add ATC0 and nub SPMI A0 power domains Add the power domains of the first Type-C port complex, chained as the firmware's power tree chains them, atc0_usb_aon -> atc0_common -> atc0_usb, and the domain of the nub SPMI A0 bus that hosts the USB PD controllers. Like nub SPMI 0 and like the same domain on the T8122, nub SPMI A0 is marked always-on. The register offsets come from the J700 ADT: 0x3e0, 0x3e8 and 0x3f0 in the main PMGR block and 0x68 in the mini PMGR block; none of them was used yet. Compile- and schema-checked (CHECK_DTBS=y apple/t8140-j700.dtb reports only the five inherited CS42L83 supply notes); not run on hardware in this form. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 7ee4109f2fca0..f9058fa8efc30 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -110,6 +110,32 @@ apple,always-on; }; + ps_atc0_usb_aon: power-controller@3e0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_usb_aon"; + }; + + ps_atc0_common: power-controller@3e8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_common"; + power-domains = <&ps_atc0_usb_aon>; + }; + + ps_atc0_usb: power-controller@3f0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_usb"; + power-domains = <&ps_atc0_common>; + }; + /* * ISP-local power domains (live J700 ADT: ISP_CPU .. ISP_CVM, "no-ps" * entries in the main PMGR block). As on t8112, the dependency tree is @@ -236,6 +262,15 @@ apple,always-on; }; + ps_nub_spmi_a0: power-controller@68 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x68 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi_a0"; + apple,always-on; + }; + /* * AOP and audio-fabric power domains (live J700 ADT /arm-io/pmgr entries * flagged "no-ps", register slots in this block at 0x4000+). Parents are From b6ed370ce5e7ea9333786df90efe5987cca05b3f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:27:21 +1000 Subject: [PATCH 0062/2680] arm64: dts: apple: t8140: Add the MTP coprocessor and DockChannel HID nodes The built-in keyboard and trackpad of the J700 hang off the MTP coprocessor, like on the M2 and M3 laptops: an ASC4 RTKit helper with its mailbox and DART, and a DockChannel FIFO pair over which the coprocessor speaks HID. Add the nodes to the SoC file, disabled, in unit-address order. The values come from the J700 ADT: the ASC at 0x3_14600000 with its SRAM window at 0x3_14050000, the mailbox at 0x3_14608000 with AIC lines 1152-1155, the DART at 0x3_14800000 (line 1135) and the DockChannel FIFO at 0x3_14b14000 (line 1137) with its 128 KiB register window at 0x3_14b28000. Unlike the T8112, whose MTP HID transport uses DART stream 1, the T8140 uses stream 0 for both the helper and the transport. The HID transport keeps the register layout, FIFO size and interrupt lines of the T8112 node. Its compatibles (apple,t8140-mtp, apple,t8140-rtk-helper-asc4, apple,t8140-dockchannel, apple,dockchannel-hid) are documented by the DockChannel HID series that is merged separately; until it lands, dtbs_check reports them as undocumented. Compile- and schema-checked; not run on hardware in this form. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 71 ++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 4967b4c620a2d..0d937050afd0a 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -439,6 +439,77 @@ status = "disabled"; }; + mtp: mtp@314600000 { + compatible = "apple,t8140-mtp", "apple,t8140-rtk-helper-asc4", + "apple,mtp", "apple,rtk-helper-asc4"; + reg = <0x3 0x14600000 0x0 0x4000>, + <0x3 0x14050000 0x0 0x4000>; + reg-names = "asc", "sram"; + mboxes = <&mtp_mbox>; + iommus = <&mtp_dart 0>; + #helper-cells = <0>; + status = "disabled"; + }; + + mtp_mbox: mailbox@314608000 { + compatible = "apple,t8140-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x14608000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + mtp_dart: iommu@314800000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x3 0x14800000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x00004000 0x0 0xffff0000>; + status = "disabled"; + }; + + mtp_dockchannel: fifo@314b14000 { + compatible = "apple,t8140-dockchannel", "apple,dockchannel"; + reg = <0x3 0x14b14000 0x0 0x4000>; + reg-names = "irq"; + interrupt-parent = <&aic>; + interrupts = ; + + ranges = <0x0 0x3 0x14b28000 0x20000>; + nonposted-mmio; + #address-cells = <1>; + #size-cells = <1>; + + interrupt-controller; + #interrupt-cells = <2>; + status = "disabled"; + + mtp_hid: input@8000 { + compatible = "apple,dockchannel-hid"; + reg = <0x8000 0x4000>, + <0xc000 0x4000>, + <0x0000 0x4000>, + <0x4000 0x4000>; + reg-names = "config", "data", + "rmt-config", "rmt-data"; + /* The T8140 MTP uses DART stream 0, the T8112 stream 1 */ + iommus = <&mtp_dart 0>; + interrupt-parent = <&mtp_dockchannel>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, + <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + }; + }; + ans_mbox: mailbox@389608000 { compatible = "apple,t8140-asc-mailbox", "apple,asc-mailbox-v4"; reg = <0x3 0x89608000 0x0 0x4000>; From 542e4a925d44d7a02694189ce21018c448847d34 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:29:09 +1000 Subject: [PATCH 0063/2680] arm64: dts: apple: t8140: Add the USB-C controller, ATC PHY and SPMI A0 bus Add the first Type-C port complex of the T8140 and the SPMI bus its PD controllers sit on, disabled, in unit-address order. The DWC3 controller at 0x4_0a280000 keeps the register split of the other Apple SoCs (a core window and the Apple-specific window at +0xcd00) on AIC line 1244, with its two DARTs at 0x4_0af00000 and 0x4_0af80000, both on line 1248 as the ADT lists them. The ATC PHY at 0x4_0b000000 is described as apple,t8140-atcphy with its sixth register region, the secondary eUSB2 bank at 0x4_0a300000; its per-device tunables are filled in by the bootloader, as on every other Apple machine. The controller, the DARTs and the PHY share the atc0_usb power domain. The nub SPMI A0 bus at 0x3_08908000 (AIC line 494) is the bus that carries the Type-C PD controllers on the J700; the bus at 0x3_08714000 carries the PMIC. The graph between the PHY, the controller and the connectors is board specific and stays in the board file. Compile- and schema-checked (CHECK_DTBS=y apple/t8140-j700.dtb reports only the five inherited CS42L83 supply notes and the DockChannel compatibles documented by the separately merged HID series); not run on hardware in this form. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 74 ++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 0d937050afd0a..a19c0b4da20aa 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -363,6 +364,19 @@ }; }; + nub_spmi_a0: spmi@308908000 { + compatible = "apple,t8140-spmi", "apple,t8103-spmi"; + reg = <0x3 0x08908000 0x0 0x100>; + #address-cells = <2>; + #size-cells = <0>; + #interrupt-cells = <2>; + interrupt-controller; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_nub_spmi_a0>; + status = "disabled"; + }; + smc: smc@30c600000 { compatible = "apple,t8140-smc", "apple,t8103-smc"; reg = <0x3 0x0c600000 0x0 0x4000>, @@ -557,6 +571,66 @@ resets = <&ps_ans>; status = "disabled"; }; + + dwc3_0: usb@40a280000 { + compatible = "apple,t8140-dwc3", "apple,t8103-dwc3"; + reg = <0x4 0x0a280000 0x0 0xcd00>, + <0x4 0x0a28cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_0_dart_0 0>, <&dwc3_0_dart_1 1>; + power-domains = <&ps_atc0_usb>; + resets = <&atcphy0>; + phys = <&atcphy0 PHY_TYPE_USB2>, <&atcphy0 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + status = "disabled"; + }; + + dwc3_0_dart_0: iommu@40af00000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x0af00000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_atc0_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x00004000 0x0 0xffff0000>; + status = "disabled"; + }; + + dwc3_0_dart_1: iommu@40af80000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x0af80000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_atc0_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x00004000 0x0 0xffff0000>; + status = "disabled"; + }; + + atcphy0: phy@40b000000 { + compatible = "apple,t8140-atcphy"; + reg = <0x4 0x0b000000 0x0 0x4c000>, + <0x4 0x0b050000 0x0 0x4000>, + <0x4 0x08000000 0x0 0x8000>, + <0x4 0x0aa90000 0x0 0x4000>, + <0x4 0x0aa84000 0x0 0x4000>, + <0x4 0x0a300000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler", "usb2phy-reg"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&ps_atc0_usb>; + status = "disabled"; + }; }; }; From 43cd9871d62f77dbc4dd6d4eeeeb5a6ba3eb3e75 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:29:41 +1000 Subject: [PATCH 0064/2680] arm64: dts: apple: t8140-j700: Enable the keyboard and trackpad Enable the MTP coprocessor, its mailbox and DART and the DockChannel HID transport, and describe the two interfaces the J700 firmware announces. The J700 firmware has no STM interface, so the transport is told not to wait for one. The trackpad is the J700 C1FE variant and uses power method 2: the host sends the firmware, announces each power transition to the coprocessor and drives the reset line of the analog front end itself. That line is SMC pcIO output 19; the SMC drives the reset when the output is set to 1, so the line is described as active high. The keyboard's country code and layout identifier are left at 0, as on the M2 laptops: the bootloader fills them in from the machine's own device tree through the keyboard alias, so one board file serves every keyboard variant. The properties follow the DockChannel HID binding of the separately merged HID series, which also documents apple,j700-multitouch; until that series lands, dtbs_check reports the compatible as undocumented. Compile- and schema-checked; not run on hardware in this form. The boot checks owed are keyboard input and trackpad clicks and scrolling. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 34 ++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 763cf79ffafce..61788854e506f 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -16,6 +16,7 @@ model = "Apple MacBook Neo"; aliases { + keyboard = &keyboard; serial0 = &serial0; }; @@ -182,5 +183,38 @@ status = "okay"; }; +&mtp { + status = "okay"; +}; + +&mtp_mbox { + status = "okay"; +}; + +&mtp_dart { + status = "okay"; +}; + +&mtp_dockchannel { + status = "okay"; +}; + +&mtp_hid { + apple,no-stm; + /* SMC pcIO output 19; level 1 asserts the reset */ + apple,afe-reset-gpios = <&smc_gpio 19 GPIO_ACTIVE_HIGH>; + + multi-touch { + compatible = "apple,j700-multitouch"; + firmware-name = "apple/tpmtfw-j700.bin"; + apple,power-method = <2>; + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; +}; + #include "hwmon-common.dtsi" #include "hwmon-laptop.dtsi" From 4251c0ed97488a6ae92bee5dbd8ba3215108f2aa Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:30:22 +1000 Subject: [PATCH 0065/2680] arm64: dts: apple: t8140-j700: Enable the USB-C ports The J700 has two Type-C connectors on its left side, each with an SN2012 series PD controller on the nub SPMI A0 bus: the front connector at USID 0xa and the back connector at USID 0xc, with the interrupt lines the ADT gives them, in the node shape the M3 MacBook Air uses. The USB topology differs from the earlier laptops. The controller's USB 2.0 pairs do not go to a connector: they go to a VIA Labs VL122 two-port USB 2.0 hub on the board, which enumerates on Linux as USB ID 2109:2122 at the USB 2.0 root port. The hub's port 1 serves the front connector, its port 2 the back connector. Only the back connector has SuperSpeed lanes, and those go straight to the ATC PHY. Describe the hub as a child of the controller with its downstream ports linked to the connectors' high-speed endpoints, and link the back connector's SuperSpeed endpoint and the controller's SuperSpeed port to the PHY as on every other Apple machine. With that graph the PD controllers find no USB role switch on their high-speed path and only drive the PHY's Type-C mux for the back connector's lanes; the dwc3 glue keeps a controller with a hub child in host mode, and the PHY keeps its USB 2.0 path up for the hub. Both connectors are described as host-only data ports for the same reason. The atcphy0 alias lets the bootloader fill in the PHY's per-device tunables. The hub's port assignment comes from the enumeration order observed on a J700: a device at the front connector appears at hub port 1 (bus path 1-1.1). Port 2 for the back connector follows from the hub having two ports. Compile- and schema-checked (CHECK_DTBS=y apple/t8140-j700.dtb reports only the five inherited CS42L83 supply notes); not run on hardware in this form. The boot checks owed are: the front hub enumerating with a device attached, the back connector training at 10 Gb/s in both orientations, an unplug at the back connector during front-hub traffic, and s2idle with a SuperSpeed device at the back connector. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 144 +++++++++++++++++++++++ 1 file changed, 144 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 61788854e506f..259b7c53cf4c9 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -16,6 +16,7 @@ model = "Apple MacBook Neo"; aliases { + atcphy0 = &atcphy0; keyboard = &keyboard; serial0 = &serial0; }; @@ -216,5 +217,148 @@ }; }; +/* USB PD controllers, on the nub SPMI A0 bus */ +&nub_spmi_a0 { + status = "okay"; + + hpm1: usb-pd@a { + compatible = "apple,sn201202x"; + reg = <0xa SPMI_USID>; + interrupts = <37 IRQ_TYPE_EDGE_RISING>, + <39 IRQ_TYPE_EDGE_RISING>, + <43 IRQ_TYPE_EDGE_RISING>, + <45 IRQ_TYPE_EDGE_RISING>; + interrupt-names = "irq", "select", "sleep", "wake"; + + typec1: connector { + compatible = "usb-c-connector"; + label = "USB-C Left-front"; + power-role = "dual"; + data-role = "host"; + + /* USB 2.0 only, through the fixed hub */ + port { + typec1_connector_hs: endpoint { + remote-endpoint = <&hub_front>; + }; + }; + }; + }; + + hpm0: usb-pd@c { + compatible = "apple,sn201202x"; + reg = <0xc SPMI_USID>; + interrupts = <11 IRQ_TYPE_EDGE_RISING>, + <13 IRQ_TYPE_EDGE_RISING>, + <17 IRQ_TYPE_EDGE_RISING>, + <19 IRQ_TYPE_EDGE_RISING>; + interrupt-names = "irq", "select", "sleep", "wake"; + + typec0: connector { + compatible = "usb-c-connector"; + label = "USB-C Left-back"; + power-role = "dual"; + data-role = "host"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + typec0_connector_hs: endpoint { + remote-endpoint = <&hub_rear>; + }; + }; + + port@1 { + reg = <1>; + + typec0_connector_ss: endpoint { + remote-endpoint = <&atcphy0_typec_lanes>; + }; + }; + }; + }; + }; +}; + +/* + * The controller's USB 2.0 pairs go to a VIA VL122 two-port hub on the + * board (USB ID 2109:2122) whose downstream ports serve the two Type-C + * connectors; only the back connector also has SuperSpeed lanes, wired + * straight to the PHY. The controller is therefore always a host. + */ +&dwc3_0 { + status = "okay"; + #address-cells = <1>; + #size-cells = <0>; + + hub@1 { + compatible = "usb2109,2122"; + reg = <1>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + + hub_front: endpoint { + remote-endpoint = <&typec1_connector_hs>; + }; + }; + + port@2 { + reg = <2>; + + hub_rear: endpoint { + remote-endpoint = <&typec0_connector_hs>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + + dwc3_0_ss: endpoint { + remote-endpoint = <&atcphy0_usb3>; + }; + }; + }; +}; + +&atcphy0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + atcphy0_typec_lanes: endpoint { + remote-endpoint = <&typec0_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + + atcphy0_usb3: endpoint { + remote-endpoint = <&dwc3_0_ss>; + }; + }; + }; +}; + #include "hwmon-common.dtsi" #include "hwmon-laptop.dtsi" From ed346d542d1ebfc7199fc237cc3d312734a038b9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 22:33:25 +1000 Subject: [PATCH 0066/2680] arm64: dts: apple: t8140-j700: Enable the USB DARTs The board enables the DWC3 host, whose node references the two USB DARTs of t8140.dtsi, but left both DART nodes disabled. The IOMMU core treats a disabled provider as absent (of_iommu_xlate() returns -ENODEV for it), so the host controller would be set up without its IOMMU, and behind a DART that leaves it unable to DMA. Enable both DARTs on the board next to the host controller, as the other Apple boards do. Verified on the J700 in every qualification candidate of 2026-09-26 (aurora-wip 92257036 onwards): both DARTs probe (40af00000.iommu and 40af80000.iommu), the on-board VL122 hub and a dock enumerate at both connectors, a SuperSpeed SSD at the rear connector reads 1 GiB identically in both orientations (453 and 408 MB/s), and the dock's Ethernet adapter links at 5 Gb/s. Fixes: b8f170b9347b ("arm64: dts: apple: t8140-j700: Enable the USB-C ports") Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 259b7c53cf4c9..b54dacf9aba48 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -335,6 +335,14 @@ }; }; +&dwc3_0_dart_0 { + status = "okay"; +}; + +&dwc3_0_dart_1 { + status = "okay"; +}; + &atcphy0 { status = "okay"; From 52cc3fc90a32d0c9d98959818bce3af9d95806c6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 18:03:49 +1000 Subject: [PATCH 0067/2680] arm64: dts: apple: t8140: Add the SMC RTC Describe the SMC RTC and the PMU NVMEM cell that holds its offset. The t8140 PMU (USID 0xe on nub-spmi0) keeps the offset at 0xf704 in its legacy scratchpad (0xf700), as 8 bytes little-endian of whole seconds; the counter the device tree for M1 and M2 would point at (0xf802 here) holds the running 65536 Hz counter and ignores writes. Mark the cell format with apple,rtc-offset-seconds. Without an RTC the J700 boots with no wall-clock time until the network comes up. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index a19c0b4da20aa..aa51d7a46e6a7 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -356,6 +356,15 @@ bits = <4 4>; }; + /* + * Whole seconds, not 32768 Hz ticks as + * on M1/M2. 0xf802 holds the running + * counter itself. + */ + rtc_offset: rtc-offset@f704 { + reg = <0xf704 0x8>; + }; + shutdown_flag: shutdown-flag@f70f,3 { reg = <0xf70f 0x1>; bits = <3 1>; @@ -437,6 +446,13 @@ nvmem-cell-names = "shutdown_flag", "boot_stage", "boot_error_count", "panic_count"; }; + + rtc { + compatible = "apple,smc-rtc"; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; + apple,rtc-offset-seconds; + }; }; smc_mbox: mailbox@30c608000 { From b43e0043091160b418eaacbc20b10005408589c4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 18:04:07 +1000 Subject: [PATCH 0068/2680] arm64: dts: apple: t8140-j700: Describe the remaining SMC temperatures The J700 SMC hwmon node lists the six CPU die temperatures, plus the Battery Hotspot key from hwmon-laptop.dtsi. The SMC's own key table holds 84 live temperature channels: every read-only float T* key that reads between 10 and 99 C and changes between samples. Add the other 77. Eight get a name after the SMC key group they belong to (storage, Wi-Fi module, USB-C port, ambient, charger and three chassis sensors); the rest are labelled with their raw key, since their placement is not known. They follow the six die temperatures, which the J700 CPU thermal policy expects as the first temperature channels, and are reported only; the policy does not read them. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 393 +++++++++++++++++++++++ 1 file changed, 393 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index b54dacf9aba48..7d863ba06ec05 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -166,6 +166,399 @@ &smc_hwmon { /* interim J700 CPU thermal policy over the CPU die channels */ apple,cpu-thermal-policy; + + /* + * Reporting-only channels. They follow the six CPU die temperatures + * in t8140.dtsi, which the CPU thermal policy samples by position. + * Every key below is in the J700 SMC's key table as a read-only + * float, reads 10-99 C and moves between samples. The names follow + * the SMC key groups and are not otherwise verified. + */ + temperature-TH0p { + apple,key-id = "TH0p"; + label = "Storage (NAND)"; + }; + + temperature-TW0p { + apple,key-id = "TW0p"; + label = "Wi-Fi Module"; + }; + + temperature-TUCp { + apple,key-id = "TUCp"; + label = "USB-C Port"; + }; + + temperature-TaTp { + apple,key-id = "TaTp"; + label = "Ambient"; + }; + + temperature-TCMz { + apple,key-id = "TCMz"; + label = "Charger"; + }; + + temperature-Ts0A { + apple,key-id = "Ts0A"; + label = "Chassis 1"; + }; + + temperature-Ts0I { + apple,key-id = "Ts0I"; + label = "Chassis 2"; + }; + + temperature-Ts0M { + apple,key-id = "Ts0M"; + label = "Chassis 3"; + }; + + /* The remaining live temperature keys; placement unknown */ + temperature-TBLp { + apple,key-id = "TBLp"; + label = "SMC TBLp"; + }; + + temperature-TCMb { + apple,key-id = "TCMb"; + label = "SMC TCMb"; + }; + + temperature-TPD0 { + apple,key-id = "TPD0"; + label = "SMC TPD0"; + }; + + temperature-TPD1 { + apple,key-id = "TPD1"; + label = "SMC TPD1"; + }; + + temperature-TPD2 { + apple,key-id = "TPD2"; + label = "SMC TPD2"; + }; + + temperature-TPD3 { + apple,key-id = "TPD3"; + label = "SMC TPD3"; + }; + + temperature-TPD4 { + apple,key-id = "TPD4"; + label = "SMC TPD4"; + }; + + temperature-TPD5 { + apple,key-id = "TPD5"; + label = "SMC TPD5"; + }; + + temperature-TPD6 { + apple,key-id = "TPD6"; + label = "SMC TPD6"; + }; + + temperature-TR5p { + apple,key-id = "TR5p"; + label = "SMC TR5p"; + }; + + temperature-TRD0 { + apple,key-id = "TRD0"; + label = "SMC TRD0"; + }; + + temperature-TRD1 { + apple,key-id = "TRD1"; + label = "SMC TRD1"; + }; + + temperature-TRD2 { + apple,key-id = "TRD2"; + label = "SMC TRD2"; + }; + + temperature-TRD3 { + apple,key-id = "TRD3"; + label = "SMC TRD3"; + }; + + temperature-TS0p { + apple,key-id = "TS0p"; + label = "SMC TS0p"; + }; + + temperature-TS1p { + apple,key-id = "TS1p"; + label = "SMC TS1p"; + }; + + temperature-TTSp { + apple,key-id = "TTSp"; + label = "SMC TTSp"; + }; + + temperature-TVA0 { + apple,key-id = "TVA0"; + label = "SMC TVA0"; + }; + + temperature-TVA1 { + apple,key-id = "TVA1"; + label = "SMC TVA1"; + }; + + temperature-TVD0 { + apple,key-id = "TVD0"; + label = "SMC TVD0"; + }; + + temperature-TVMR { + apple,key-id = "TVMR"; + label = "SMC TVMR"; + }; + + temperature-TVS0 { + apple,key-id = "TVS0"; + label = "SMC TVS0"; + }; + + temperature-TVS1 { + apple,key-id = "TVS1"; + label = "SMC TVS1"; + }; + + temperature-TVS3 { + apple,key-id = "TVS3"; + label = "SMC TVS3"; + }; + + temperature-TVS4 { + apple,key-id = "TVS4"; + label = "SMC TVS4"; + }; + + temperature-TVmS { + apple,key-id = "TVmS"; + label = "SMC TVmS"; + }; + + temperature-Te04 { + apple,key-id = "Te04"; + label = "SMC Te04"; + }; + + temperature-Te06 { + apple,key-id = "Te06"; + label = "SMC Te06"; + }; + + temperature-Te0R { + apple,key-id = "Te0R"; + label = "SMC Te0R"; + }; + + temperature-Te0T { + apple,key-id = "Te0T"; + label = "SMC Te0T"; + }; + + temperature-Tex0 { + apple,key-id = "Tex0"; + label = "SMC Tex0"; + }; + + temperature-Tex1 { + apple,key-id = "Tex1"; + label = "SMC Tex1"; + }; + + temperature-Tex2 { + apple,key-id = "Tex2"; + label = "SMC Tex2"; + }; + + temperature-Tex3 { + apple,key-id = "Tex3"; + label = "SMC Tex3"; + }; + + temperature-Tp00 { + apple,key-id = "Tp00"; + label = "SMC Tp00"; + }; + + temperature-Tp02 { + apple,key-id = "Tp02"; + label = "SMC Tp02"; + }; + + temperature-Tp04 { + apple,key-id = "Tp04"; + label = "SMC Tp04"; + }; + + temperature-Tp06 { + apple,key-id = "Tp06"; + label = "SMC Tp06"; + }; + + temperature-Tp08 { + apple,key-id = "Tp08"; + label = "SMC Tp08"; + }; + + temperature-Tp0A { + apple,key-id = "Tp0A"; + label = "SMC Tp0A"; + }; + + temperature-Tp0C { + apple,key-id = "Tp0C"; + label = "SMC Tp0C"; + }; + + temperature-Tp0E { + apple,key-id = "Tp0E"; + label = "SMC Tp0E"; + }; + + temperature-Tp0l { + apple,key-id = "Tp0l"; + label = "SMC Tp0l"; + }; + + temperature-Tp0m { + apple,key-id = "Tp0m"; + label = "SMC Tp0m"; + }; + + temperature-Tp0n { + apple,key-id = "Tp0n"; + label = "SMC Tp0n"; + }; + + temperature-Tp0o { + apple,key-id = "Tp0o"; + label = "SMC Tp0o"; + }; + + temperature-Tp0q { + apple,key-id = "Tp0q"; + label = "SMC Tp0q"; + }; + + temperature-Tp0t { + apple,key-id = "Tp0t"; + label = "SMC Tp0t"; + }; + + temperature-Tpx0 { + apple,key-id = "Tpx0"; + label = "SMC Tpx0"; + }; + + temperature-Tpx1 { + apple,key-id = "Tpx1"; + label = "SMC Tpx1"; + }; + + temperature-Tpx2 { + apple,key-id = "Tpx2"; + label = "SMC Tpx2"; + }; + + temperature-Tpx3 { + apple,key-id = "Tpx3"; + label = "SMC Tpx3"; + }; + + temperature-Ts00 { + apple,key-id = "Ts00"; + label = "SMC Ts00"; + }; + + temperature-Ts01 { + apple,key-id = "Ts01"; + label = "SMC Ts01"; + }; + + temperature-Ts02 { + apple,key-id = "Ts02"; + label = "SMC Ts02"; + }; + + temperature-Ts04 { + apple,key-id = "Ts04"; + label = "SMC Ts04"; + }; + + temperature-Ts05 { + apple,key-id = "Ts05"; + label = "SMC Ts05"; + }; + + temperature-Ts06 { + apple,key-id = "Ts06"; + label = "SMC Ts06"; + }; + + temperature-Ts08 { + apple,key-id = "Ts08"; + label = "SMC Ts08"; + }; + + temperature-Ts09 { + apple,key-id = "Ts09"; + label = "SMC Ts09"; + }; + + temperature-Ts0C { + apple,key-id = "Ts0C"; + label = "SMC Ts0C"; + }; + + temperature-Ts0D { + apple,key-id = "Ts0D"; + label = "SMC Ts0D"; + }; + + temperature-Ts0E { + apple,key-id = "Ts0E"; + label = "SMC Ts0E"; + }; + + temperature-Ts0G { + apple,key-id = "Ts0G"; + label = "SMC Ts0G"; + }; + + temperature-Ts0H { + apple,key-id = "Ts0H"; + label = "SMC Ts0H"; + }; + + temperature-Ts0K { + apple,key-id = "Ts0K"; + label = "SMC Ts0K"; + }; + + temperature-Ts0L { + apple,key-id = "Ts0L"; + label = "SMC Ts0L"; + }; + + temperature-Tsx0 { + apple,key-id = "Tsx0"; + label = "SMC Tsx0"; + }; + + temperature-Tsx1 { + apple,key-id = "Tsx1"; + label = "SMC Tsx1"; + }; }; &smc_mbox { From 330b407b3ecc95da14beb9f25e74f65a9e15c94b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 18:04:38 +1000 Subject: [PATCH 0069/2680] arm64: dts: apple: t8140-j700: Describe the SMC current, power and voltage rails hwmon-common.dtsi and hwmon-laptop.dtsi give the J700 its AC input current, voltage and power, total system power and battery voltage. The SMC's key table has 120 read-only float keys in the I*, P* and V* groups; add the 116 not already described, as current, power and voltage channels. The units follow the key prefix, as for the keys the common files already use; on this machine the battery keys agree with each other (VBat x IBat = PBat), as do the AC input keys (VD0R x ID0R = PDTR). Which rail each key measures is not known, so they are labelled with the raw key. Some read zero while the machine is idle. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 586 +++++++++++++++++++++++ 1 file changed, 586 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 7d863ba06ec05..270932d29e966 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -559,6 +559,592 @@ apple,key-id = "Tsx1"; label = "SMC Tsx1"; }; + + /* + * Current, power and voltage rails: every other read-only float + * I*, P* and V* key in the J700 SMC's key table. The units follow + * the prefix (amperes, watts, volts); the rails they measure are not + * known, so they carry the raw key. Some read zero while idle. + */ + current-IBat { + apple,key-id = "IBat"; + label = "SMC IBat"; + }; + + current-IBws { + apple,key-id = "IBws"; + label = "SMC IBws"; + }; + + current-IP0b { + apple,key-id = "IP0b"; + label = "SMC IP0b"; + }; + + current-IP1b { + apple,key-id = "IP1b"; + label = "SMC IP1b"; + }; + + current-IP1l { + apple,key-id = "IP1l"; + label = "SMC IP1l"; + }; + + current-IP2b { + apple,key-id = "IP2b"; + label = "SMC IP2b"; + }; + + current-IP3b { + apple,key-id = "IP3b"; + label = "SMC IP3b"; + }; + + current-IP3l { + apple,key-id = "IP3l"; + label = "SMC IP3l"; + }; + + current-IP4b { + apple,key-id = "IP4b"; + label = "SMC IP4b"; + }; + + current-IP4l { + apple,key-id = "IP4l"; + label = "SMC IP4l"; + }; + + current-IP5b { + apple,key-id = "IP5b"; + label = "SMC IP5b"; + }; + + current-IP5l { + apple,key-id = "IP5l"; + label = "SMC IP5l"; + }; + + current-IP6b { + apple,key-id = "IP6b"; + label = "SMC IP6b"; + }; + + current-IP6l { + apple,key-id = "IP6l"; + label = "SMC IP6l"; + }; + + current-IP7b { + apple,key-id = "IP7b"; + label = "SMC IP7b"; + }; + + current-IP8b { + apple,key-id = "IP8b"; + label = "SMC IP8b"; + }; + + current-IP9l { + apple,key-id = "IP9l"; + label = "SMC IP9l"; + }; + + current-IPMC { + apple,key-id = "IPMC"; + label = "SMC IPMC"; + }; + + current-IPab { + apple,key-id = "IPab"; + label = "SMC IPab"; + }; + + current-IPbb { + apple,key-id = "IPbb"; + label = "SMC IPbb"; + }; + + current-IPbl { + apple,key-id = "IPbl"; + label = "SMC IPbl"; + }; + + current-IPcb { + apple,key-id = "IPcb"; + label = "SMC IPcb"; + }; + + current-IPcl { + apple,key-id = "IPcl"; + label = "SMC IPcl"; + }; + + current-IPdb { + apple,key-id = "IPdb"; + label = "SMC IPdb"; + }; + + current-IPgl { + apple,key-id = "IPgl"; + label = "SMC IPgl"; + }; + + current-IPhl { + apple,key-id = "IPhl"; + label = "SMC IPhl"; + }; + + current-IPil { + apple,key-id = "IPil"; + label = "SMC IPil"; + }; + + current-IR1l { + apple,key-id = "IR1l"; + label = "SMC IR1l"; + }; + + current-IR2l { + apple,key-id = "IR2l"; + label = "SMC IR2l"; + }; + + current-IRil { + apple,key-id = "IRil"; + label = "SMC IRil"; + }; + + current-Ib0f { + apple,key-id = "Ib0f"; + label = "SMC Ib0f"; + }; + + current-Ib1f { + apple,key-id = "Ib1f"; + label = "SMC Ib1f"; + }; + + power-PBat { + apple,key-id = "PBat"; + label = "SMC PBat"; + }; + + power-PBwo { + apple,key-id = "PBwo"; + label = "SMC PBwo"; + }; + + power-PDCP { + apple,key-id = "PDCP"; + label = "SMC PDCP"; + }; + + power-PDTP { + apple,key-id = "PDTP"; + label = "SMC PDTP"; + }; + + power-PP0b { + apple,key-id = "PP0b"; + label = "SMC PP0b"; + }; + + power-PP1b { + apple,key-id = "PP1b"; + label = "SMC PP1b"; + }; + + power-PP1l { + apple,key-id = "PP1l"; + label = "SMC PP1l"; + }; + + power-PP2b { + apple,key-id = "PP2b"; + label = "SMC PP2b"; + }; + + power-PP3b { + apple,key-id = "PP3b"; + label = "SMC PP3b"; + }; + + power-PP3l { + apple,key-id = "PP3l"; + label = "SMC PP3l"; + }; + + power-PP4b { + apple,key-id = "PP4b"; + label = "SMC PP4b"; + }; + + power-PP4l { + apple,key-id = "PP4l"; + label = "SMC PP4l"; + }; + + power-PP5b { + apple,key-id = "PP5b"; + label = "SMC PP5b"; + }; + + power-PP5l { + apple,key-id = "PP5l"; + label = "SMC PP5l"; + }; + + power-PP6b { + apple,key-id = "PP6b"; + label = "SMC PP6b"; + }; + + power-PP6l { + apple,key-id = "PP6l"; + label = "SMC PP6l"; + }; + + power-PP7b { + apple,key-id = "PP7b"; + label = "SMC PP7b"; + }; + + power-PP8b { + apple,key-id = "PP8b"; + label = "SMC PP8b"; + }; + + power-PP9l { + apple,key-id = "PP9l"; + label = "SMC PP9l"; + }; + + power-PPBR { + apple,key-id = "PPBR"; + label = "SMC PPBR"; + }; + + power-PPMC { + apple,key-id = "PPMC"; + label = "SMC PPMC"; + }; + + power-PPSM { + apple,key-id = "PPSM"; + label = "SMC PPSM"; + }; + + power-PPab { + apple,key-id = "PPab"; + label = "SMC PPab"; + }; + + power-PPbb { + apple,key-id = "PPbb"; + label = "SMC PPbb"; + }; + + power-PPbl { + apple,key-id = "PPbl"; + label = "SMC PPbl"; + }; + + power-PPcb { + apple,key-id = "PPcb"; + label = "SMC PPcb"; + }; + + power-PPcl { + apple,key-id = "PPcl"; + label = "SMC PPcl"; + }; + + power-PPdb { + apple,key-id = "PPdb"; + label = "SMC PPdb"; + }; + + power-PPgl { + apple,key-id = "PPgl"; + label = "SMC PPgl"; + }; + + power-PPhl { + apple,key-id = "PPhl"; + label = "SMC PPhl"; + }; + + power-PPil { + apple,key-id = "PPil"; + label = "SMC PPil"; + }; + + power-PR1l { + apple,key-id = "PR1l"; + label = "SMC PR1l"; + }; + + power-PR2l { + apple,key-id = "PR2l"; + label = "SMC PR2l"; + }; + + power-PRil { + apple,key-id = "PRil"; + label = "SMC PRil"; + }; + + power-PZC0 { + apple,key-id = "PZC0"; + label = "SMC PZC0"; + }; + + power-PZC1 { + apple,key-id = "PZC1"; + label = "SMC PZC1"; + }; + + power-PZCB { + apple,key-id = "PZCB"; + label = "SMC PZCB"; + }; + + power-PZCU { + apple,key-id = "PZCU"; + label = "SMC PZCU"; + }; + + power-PZD1 { + apple,key-id = "PZD1"; + label = "SMC PZD1"; + }; + + power-PZT0 { + apple,key-id = "PZT0"; + label = "SMC PZT0"; + }; + + power-Pb0f { + apple,key-id = "Pb0f"; + label = "SMC Pb0f"; + }; + + power-Pb1f { + apple,key-id = "Pb1f"; + label = "SMC Pb1f"; + }; + + power-Pb2f { + apple,key-id = "Pb2f"; + label = "SMC Pb2f"; + }; + + voltage-VBat { + apple,key-id = "VBat"; + label = "SMC VBat"; + }; + + voltage-VP0R { + apple,key-id = "VP0R"; + label = "SMC VP0R"; + }; + + voltage-VP0b { + apple,key-id = "VP0b"; + label = "SMC VP0b"; + }; + + voltage-VP0s { + apple,key-id = "VP0s"; + label = "SMC VP0s"; + }; + + voltage-VP1b { + apple,key-id = "VP1b"; + label = "SMC VP1b"; + }; + + voltage-VP1l { + apple,key-id = "VP1l"; + label = "SMC VP1l"; + }; + + voltage-VP1s { + apple,key-id = "VP1s"; + label = "SMC VP1s"; + }; + + voltage-VP2b { + apple,key-id = "VP2b"; + label = "SMC VP2b"; + }; + + voltage-VP3b { + apple,key-id = "VP3b"; + label = "SMC VP3b"; + }; + + voltage-VP3l { + apple,key-id = "VP3l"; + label = "SMC VP3l"; + }; + + voltage-VP3s { + apple,key-id = "VP3s"; + label = "SMC VP3s"; + }; + + voltage-VP4b { + apple,key-id = "VP4b"; + label = "SMC VP4b"; + }; + + voltage-VP4l { + apple,key-id = "VP4l"; + label = "SMC VP4l"; + }; + + voltage-VP4s { + apple,key-id = "VP4s"; + label = "SMC VP4s"; + }; + + voltage-VP5b { + apple,key-id = "VP5b"; + label = "SMC VP5b"; + }; + + voltage-VP5l { + apple,key-id = "VP5l"; + label = "SMC VP5l"; + }; + + voltage-VP5s { + apple,key-id = "VP5s"; + label = "SMC VP5s"; + }; + + voltage-VP6b { + apple,key-id = "VP6b"; + label = "SMC VP6b"; + }; + + voltage-VP6l { + apple,key-id = "VP6l"; + label = "SMC VP6l"; + }; + + voltage-VP7b { + apple,key-id = "VP7b"; + label = "SMC VP7b"; + }; + + voltage-VP8b { + apple,key-id = "VP8b"; + label = "SMC VP8b"; + }; + + voltage-VP9l { + apple,key-id = "VP9l"; + label = "SMC VP9l"; + }; + + voltage-VPBR { + apple,key-id = "VPBR"; + label = "SMC VPBR"; + }; + + voltage-VPab { + apple,key-id = "VPab"; + label = "SMC VPab"; + }; + + voltage-VPbb { + apple,key-id = "VPbb"; + label = "SMC VPbb"; + }; + + voltage-VPbl { + apple,key-id = "VPbl"; + label = "SMC VPbl"; + }; + + voltage-VPcb { + apple,key-id = "VPcb"; + label = "SMC VPcb"; + }; + + voltage-VPcl { + apple,key-id = "VPcl"; + label = "SMC VPcl"; + }; + + voltage-VPdb { + apple,key-id = "VPdb"; + label = "SMC VPdb"; + }; + + voltage-VPgl { + apple,key-id = "VPgl"; + label = "SMC VPgl"; + }; + + voltage-VPhl { + apple,key-id = "VPhl"; + label = "SMC VPhl"; + }; + + voltage-VPil { + apple,key-id = "VPil"; + label = "SMC VPil"; + }; + + voltage-VR1l { + apple,key-id = "VR1l"; + label = "SMC VR1l"; + }; + + voltage-VR2l { + apple,key-id = "VR2l"; + label = "SMC VR2l"; + }; + + voltage-VR5c { + apple,key-id = "VR5c"; + label = "SMC VR5c"; + }; + + voltage-VRbi { + apple,key-id = "VRbi"; + label = "SMC VRbi"; + }; + + voltage-VRil { + apple,key-id = "VRil"; + label = "SMC VRil"; + }; + + voltage-VRmi { + apple,key-id = "VRmi"; + label = "SMC VRmi"; + }; + + voltage-VRwo { + apple,key-id = "VRwo"; + label = "SMC VRwo"; + }; + + voltage-Vb1f { + apple,key-id = "Vb1f"; + label = "SMC Vb1f"; + }; + + voltage-Vb3f { + apple,key-id = "Vb3f"; + label = "SMC Vb3f"; + }; }; &smc_mbox { From 55fe60ca76cc9e72756efb5e82d780b86b0d8324 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:31:38 +1100 Subject: [PATCH 0070/2680] Bluetooth: mt7932: support modules and system suspend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MT7932 PCIe transport cannot be selected with modular Bluetooth or system sleep enabled. Make the existing transport tristate and coordinate firmware quiescence with HCI suspend before PCI power transitions. Wait for the firmware's system-sleep acknowledgment before completing suspend. Restore the transport before resuming HCI, and undo the sleep request if suspend fails. Keep ring publication stopped while quiesced and preserve the DMA arena across sleep. Invalid notifications and failed restoration close the transport instead of retrying an uncertain state. Unregister HCI, quiesce firmware and synchronize transport work and interrupts on removal. Release device-visible storage only after bus mastering is disabled and pending PCI transactions drain; retain it if that cannot be confirmed. Hibernation, firmware recovery and warm reprobe remain unsupported. This extends the existing MT7932 transport and preserves its contributor notices and firmware/calibration path. [ryan: adapted to 7.2.10 — the documentation conflict against a later rewrite of mt7932-neo.rst is resolved to this commit's text] Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 58 ++-- drivers/bluetooth/Kconfig | 19 +- drivers/bluetooth/mt7932_bt_pcie.c | 265 +++++++++++++++--- 3 files changed, 265 insertions(+), 77 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 0c9c68d82881c..c47d4496e631a 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -28,17 +28,21 @@ Use the separately reviewed T8140 PCIe bootstrap described in ``wifi0`` endpoint's own ``local-mac-address``; a zero placeholder is rejected. The tested configuration includes ``CONFIG_MT7932_FULLMAC=m``, -``CONFIG_CFG80211=y``, ``CONFIG_BT_MTK7932_PCIE=y``, ``CONFIG_BT_BREDR=y``, +``CONFIG_CFG80211=y``, ``CONFIG_BT_MTK7932_PCIE=m``, ``CONFIG_BT_BREDR=y``, ``CONFIG_BT_LE=y``, ``CONFIG_CRYPTO_AES=y`` and ``CONFIG_CRYPTO_CMAC=y``. -The bootstrap and Bluetooth experiment exclude suspend and kexec. +The Bluetooth transport can be built with modular Bluetooth and system +sleep. The separately selected bootstrap provider has its own lifecycle +restrictions. Kexec is excluded while either retained experiment is enabled. Use the ordinary cfg80211 regulatory database and applicable country policy. The validated first-admission fallback is kernel country 00 with firmware XZ. After the root filesystem and the local firmware packages are available, select PCIe ASPM performance policy before loading ``mt7932-fullmac``. This matches the tested admission sequence. Bluetooth's gate defaults closed; -validate the cold, unbound ``14c3:793b`` function, enable -``/sys/module/mt7932_bt_pcie/parameters/enable``, then request its PCI probe. +validate the cold, unbound ``14c3:793b`` function, then load +``mt7932_bt_pcie`` with ``enable=1`` after its local inputs are installed. +A built-in transport can instead open its enable parameter before requesting +the PCI probe. Do not reprobe after a failed or uncertain Bluetooth admission. The tested Wi-Fi driver owns function 0 and Bluetooth owns function 1. @@ -106,34 +110,18 @@ from the result. A successful host serial command is not a test pass: require Qualification and remaining work -------------------------------- -The :download:`qualification artifact ` records the exact Image, -module and source identities and the physical results from 2026-10-02. -Both 2.4 and 5 GHz passed scan, WPA2, DHCP, gateway, DNS and verified HTTPS. -Switching bands performed normal peer retirement and fresh authentication. -Wi-Fi's checked module removal/reset/reload also passed before Bluetooth -admission. Five-GHz traffic concurrent with Bluetooth discovery and HCI -off/on passed 54/54 gateway replies and 18 HTTPS downloads with no driver or -kernel faults. The longer 2.4-GHz coexistence run recorded 53/54 replies; -it is explicitly not a zero-loss pass. The lost ping coincided with a -background scan; that timing does not establish a root cause. - -The earlier Bluetooth candidate passed two bonded AirPods reconnects and -30 seconds of AAC output, including a user-confirmed audible tone. Concurrent -headset audio was not repeated on the combined candidate because no AirPods -appeared during its discovery runs. - -The experimental implementation has concrete production merge blockers: - -* PCI bootstrap memory remains retained until external reset; controller - removal, suspend, kexec and reusable-arena lifetime are not qualified. -* Bluetooth lacks a complete PCI removal/quiescence contract. Suppressed - bind/unbind attributes do not make PCI device removal safe. -* Bluetooth's finite software queue limit is not HCI backpressure. Saturation - can drop a frame already accounted by the HCI scheduler; that path needs - correction and saturation qualification before production use. -* Arbitrary caller scan IEs, WPA3/SAE, required MFP, AP/P2P modes, general - country-package generation, roaming and long-duration reliability remain - outside this qualification. Broad compatibility must not be inferred from - the tested station and firmware combination. -* SCO/headset microphone, LE Audio/ISO and simultaneous headset audio remain - unqualified. These limitations are separate from the demonstrated A2DP path. +* PCI bootstrap memory remains retained until external reset. Controller + removal, memory reuse, suspend and kexec are unqualified. +* Bluetooth system sleep uses a bounded firmware quiesce/restore handshake + around HCI suspend/resume, preserving its DMA arena. Hibernation is rejected. + Removal first quiesces firmware, then stops work and interrupts, disables + bus mastering and drains pending + PCI transactions before freeing DMA. An unconfirmed drain retains storage + until reset. Firmware recovery and warm reprobe are unsupported; use a fresh + cold boot after removing or faulting an active transport. +* The Bluetooth software queue limit does not provide HCI backpressure; + saturation can drop an accounted frame. +* Arbitrary scan IEs, WPA3/SAE, required MFP, AP/P2P, general country-package + generation, roaming and long-duration reliability are unqualified. +* SCO/headset microphone, LE Audio/ISO and simultaneous headset audio are + unqualified. diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index 7ac07459925dc..93c3b591de27b 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -24,16 +24,19 @@ config BT_MTK select FW_LOADER config BT_MTK7932_PCIE - bool "J700 MT7932 PCIe Bluetooth experiment" - depends on BT=y && PCI && ARCH_APPLE && IOMMU_DMA - depends on !PM_SLEEP && !KEXEC_CORE + tristate "J700 MT7932 PCIe Bluetooth transport" + depends on BT && PCI && ARCH_APPLE && IOMMU_DMA + depends on !KEXEC_CORE select FW_LOADER help - Build the standalone J700 MT7932 transport experiment. Its probe gate - defaults closed so board firmware can remain on the real root filesystem. - It retains all DMA memory after activation, including terminal failures, - until a full external reset. Unbind and automatic recovery are excluded. - This does not qualify suspend or physical removal. + Support the J700 Bluetooth PCIe function with locally installed firmware + and calibration. The late-probe enable parameter defaults off. + System suspend uses the firmware sleep handshake and preserves DMA + storage. Hibernation and automatic firmware recovery are unsupported. + Removal drains the PCI function before releasing device-visible memory; + an unconfirmed drain retains it until a full hardware reset. + + To build as a module, choose M. The module is mt7932_bt_pcie. config BT_HCIBTUSB tristate "HCI USB driver" diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 062522f170c3f..8969317d628df 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -4,9 +4,9 @@ * protocol contracts and public Linux interfaces. Original transport research: * DJ (DjDeveloperr), Ace (Acelogic), and Ryan Murray. * - * The built-in driver retains every device-visible allocation after mastering - * is enabled, including all terminal failures. No unbind, retry, suspend or - * quiescence claim is made. Firmware and board inputs are supplied locally. + * Firmware and board inputs are supplied locally. System sleep preserves the + * transport arena and uses the firmware quiesce/restore handshake. Unload + * releases DMA only after the PCI function has stopped bus mastering. */ #include #include @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -94,6 +95,13 @@ struct bt7932_ring { u16 previous; }; +enum bt7932_pm_state { + BT7932_RUNNING, + BT7932_QUIESCING, + BT7932_QUIESCED, + BT7932_RESTORING, +}; + struct bt7932 { struct pci_dev *pdev; void __iomem *bar; @@ -121,6 +129,8 @@ struct bt7932 { int irq; bool enabled, regions, vectors, irq_requested, retained; bool ipc_ready, opened, awake, waking, fault, registered; + bool stopping, runtime_forbidden; + enum bt7932_pm_state pm_state; u64 tx_packets, rx_packets, diagnostics; }; @@ -395,6 +405,13 @@ static int bt7932_arena(struct bt7932 *bt) return 0; } +/* Serialize control requests with sleep notifications and ring publication. */ +static void bt7932_request_sleep(struct bt7932 *bt, u32 control) +{ + writel(control, bt->bar + BT7932_SLEEP); + bt7932_ring_doorbell(bt, 12); +} + /* Caller owns lock; no HCI command wait can run from the IRQ thread. */ static int bt7932_sleep_locked(struct bt7932 *bt) { @@ -403,21 +420,46 @@ static int bt7932_sleep_locked(struct bt7932 *bt) dma_rmb(); value = le32_to_cpu(READ_ONCE(*notification)); + if (value == 3) { + if (bt->pm_state == BT7932_QUIESCING) { + WRITE_ONCE(bt->pm_state, BT7932_QUIESCED); + WRITE_ONCE(bt->awake, false); + wake_up_all(&bt->wake_wait); + } else if (bt->pm_state != BT7932_QUIESCED && + bt->pm_state != BT7932_RESTORING) { + return -EPROTO; + } + return 0; + } if (value > 1) return -EPROTO; - if (!value) { - if (bt->last_sleep != value || bt->waking) { - writel(0, bt->bar + BT7932_SLEEP); - bt7932_ring_doorbell(bt, 12); + /* A previous host-sleep notification must not cancel control 3. */ + if (bt->pm_state == BT7932_QUIESCING) + return 0; + if (bt->pm_state == BT7932_QUIESCED) + return -EPROTO; + if (bt->pm_state == BT7932_RESTORING) { + if (value) { + if (!bt->waking) { + bt->waking = true; + bt7932_request_sleep(bt, 1); + bt7932_request_sleep(bt, 2); + } + return 0; } + bt->waking = true; + WRITE_ONCE(bt->pm_state, BT7932_RUNNING); + } + if (!value) { + if (bt->last_sleep != value || bt->waking) + bt7932_request_sleep(bt, 0); WRITE_ONCE(bt->awake, true); bt->waking = false; wake_up_all(&bt->wake_wait); } else { WRITE_ONCE(bt->awake, false); if (!bt->waking && bt->last_sleep != value) { - writel(1, bt->bar + BT7932_SLEEP); - bt7932_ring_doorbell(bt, 12); + bt7932_request_sleep(bt, 1); } } bt->last_sleep = value; @@ -425,7 +467,7 @@ static int bt7932_sleep_locked(struct bt7932 *bt) } /* On success returns with lock held, covering the final check/publication. */ -static int bt7932_wake_and_lock(struct bt7932 *bt) +static int bt7932_wake_and_lock(struct bt7932 *bt, bool require_open) { unsigned long deadline = jiffies + HZ; unsigned long now; @@ -433,9 +475,10 @@ static int bt7932_wake_and_lock(struct bt7932 *bt) for (;;) { mutex_lock(&bt->lock); - if (bt->fault || !bt->opened) { + if (bt->fault || bt->stopping || bt->pm_state != BT7932_RUNNING || + (require_open && !bt->opened)) { mutex_unlock(&bt->lock); - return -EIO; + return -EHOSTDOWN; } ret = bt7932_sleep_locked(bt); if (ret) @@ -444,8 +487,7 @@ static int bt7932_wake_and_lock(struct bt7932 *bt) return 0; if (!bt->waking) { bt->waking = true; - writel(2, bt->bar + BT7932_SLEEP); - bt7932_ring_doorbell(bt, 12); + bt7932_request_sleep(bt, 2); } mutex_unlock(&bt->lock); now = jiffies; @@ -455,7 +497,9 @@ static int bt7932_wake_and_lock(struct bt7932 *bt) goto fail; } wait_event_timeout(bt->wake_wait, - READ_ONCE(bt->awake) || READ_ONCE(bt->fault) || !READ_ONCE(bt->opened), + READ_ONCE(bt->awake) || READ_ONCE(bt->fault) || + READ_ONCE(bt->stopping) || + (require_open && !READ_ONCE(bt->opened)), min_t(unsigned long, msecs_to_jiffies(10), deadline - now)); } fail: @@ -499,7 +543,7 @@ static void bt7932_tx_work(struct work_struct *work) u16 consumer; u8 *descriptor; - if (bt7932_wake_and_lock(bt)) { + if (bt7932_wake_and_lock(bt, true)) { kfree_skb(skb); skb_queue_purge(&bt->tx_queue); return; @@ -601,7 +645,7 @@ static irqreturn_t bt7932_irq_thread(int irq, void *data) int channel, ret; mutex_lock(&bt->lock); - if (!bt->ipc_ready || bt->fault) + if (!bt->ipc_ready || bt->fault || bt->stopping) goto out; if (READ_ONCE(*(u8 *)(bt->arena + bt->peripheral + 4)) == 4) { bt7932_fault_locked(bt, "firmware entered fault state4"); @@ -612,6 +656,9 @@ static irqreturn_t bt7932_irq_thread(int irq, void *data) bt7932_fault_locked(bt, "invalid sleep notification"); goto out; } + /* Drain the old offers until quiescence, then stop all publication. */ + if (bt->pm_state == BT7932_QUIESCED || bt->pm_state == BT7932_RESTORING) + goto out; producer = bt7932_counter(bt, bt->cr_producer, 0); if (producer != bt->cr_previous) { /* Counter retirement only; no invented per-record ACK interpretation. */ @@ -778,8 +825,9 @@ static int bt7932_open(struct hci_dev *hdev) int ret = 0; mutex_lock(&bt->lock); - if (bt->fault || !bt->ipc_ready) - ret = -EIO; + if (bt->fault || !bt->ipc_ready || bt->stopping || + bt->pm_state != BT7932_RUNNING) + ret = -EHOSTDOWN; else WRITE_ONCE(bt->opened, true); mutex_unlock(&bt->lock); @@ -811,8 +859,9 @@ static int bt7932_send(struct hci_dev *hdev, struct sk_buff *skb) unsigned long flags; int ret; - if (READ_ONCE(bt->fault) || !READ_ONCE(bt->opened)) - return -EIO; + if (READ_ONCE(bt->fault) || !READ_ONCE(bt->opened) || + READ_ONCE(bt->stopping) || READ_ONCE(bt->pm_state) != BT7932_RUNNING) + return -EHOSTDOWN; if (channel < 0 || skb->len > geometry[channel].payload) return -EMSGSIZE; ret = skb_linearize(skb); @@ -821,6 +870,10 @@ static int bt7932_send(struct hci_dev *hdev, struct sk_buff *skb) if (!bt7932_hci_length(hci_skb_pkt_type(skb), skb->data, skb->len)) return -EINVAL; spin_lock_irqsave(&bt->tx_queue.lock, flags); + if (READ_ONCE(bt->stopping) || READ_ONCE(bt->pm_state) != BT7932_RUNNING) { + spin_unlock_irqrestore(&bt->tx_queue.lock, flags); + return -EHOSTDOWN; + } if (bt->tx_queue.qlen >= BT7932_TX_QUEUE_MAX) { spin_unlock_irqrestore(&bt->tx_queue.lock, flags); return -EBUSY; @@ -957,10 +1010,8 @@ static int bt7932_iommu(struct bt7932 *bt) return ret; } -static void bt7932_unpublished_cleanup(struct bt7932 *bt) +static void bt7932_cleanup(struct bt7932 *bt, bool release_dma) { - if (WARN_ON_ONCE(bt->retained)) - return; if (bt->irq_requested) free_irq(bt->irq, bt); if (bt->vectors) @@ -969,9 +1020,9 @@ static void bt7932_unpublished_cleanup(struct bt7932 *bt) hci_free_dev(bt->hdev); if (bt->tx_wq) destroy_workqueue(bt->tx_wq); - if (bt->arena) + if (release_dma && bt->arena) dma_free_coherent(&bt->pdev->dev, bt->arena_size, bt->arena, bt->arena_dma); - if (bt->image) + if (release_dma && bt->image) dma_free_coherent(&bt->pdev->dev, bt->image_size, bt->image, bt->image_dma); kfree_sensitive(bt->calibration); kfree_sensitive(bt->ptx); @@ -979,6 +1030,8 @@ static void bt7932_unpublished_cleanup(struct bt7932 *bt) pci_iounmap(bt->pdev, bt->bar); if (bt->regions) pci_release_regions(bt->pdev); + if (bt->runtime_forbidden) + pm_runtime_allow(&bt->pdev->dev); if (bt->enabled) pci_disable_device(bt->pdev); kfree_sensitive(bt); @@ -1092,7 +1145,8 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) bt->irq_requested = true; pci_set_drvdata(pdev, bt); pm_runtime_forbid(&pdev->dev); - /* Every later outcome stays bound and retains all published ownership. */ + bt->runtime_forbidden = true; + /* Failed admission remains bound until removal or a full reset. */ bt->retained = true; pci_set_master(pdev); ret = bt7932_download(bt); @@ -1118,21 +1172,163 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) return 0; fail: dev_err(&pdev->dev, "BT_ADMISSION_FAILED error=%d; no DMA published\n", ret); - bt7932_unpublished_cleanup(bt); + bt7932_cleanup(bt, true); return ret; } -static void bt7932_shutdown(struct pci_dev *pdev) +/* Poll coherent notifications as well as IRQs, including a lost final edge. */ +static int bt7932_wait_pm(struct bt7932 *bt, enum bt7932_pm_state target) { - struct bt7932 *bt = pci_get_drvdata(pdev); + unsigned long limit = jiffies + HZ; + int ret; - WRITE_ONCE(bt->opened, false); + for (;;) { + mutex_lock(&bt->lock); + ret = bt->fault || bt->stopping ? -EIO : bt7932_sleep_locked(bt); + if (!ret && bt->pm_state == target) { + mutex_unlock(&bt->lock); + return 0; + } + mutex_unlock(&bt->lock); + if (ret) + return ret; + if (time_after_eq(jiffies, limit)) + return -ETIMEDOUT; + usleep_range(1000, 2000); + } +} + +static int bt7932_quiesce(struct bt7932 *bt) +{ + int ret; + + ret = bt7932_wake_and_lock(bt, false); + if (ret) + return ret; + WRITE_ONCE(bt->pm_state, BT7932_QUIESCING); + WRITE_ONCE(bt->awake, false); + bt7932_request_sleep(bt, 3); + mutex_unlock(&bt->lock); wake_up_all(&bt->wake_wait); cancel_work_sync(&bt->tx_work); skb_queue_purge(&bt->tx_queue); + return bt7932_wait_pm(bt, BT7932_QUIESCED); +} + +static int bt7932_restore_transport(struct bt7932 *bt) +{ + int ret; + + mutex_lock(&bt->lock); + if (bt->fault || bt->stopping) { + mutex_unlock(&bt->lock); + return -EIO; + } + WRITE_ONCE(bt->pm_state, BT7932_RESTORING); + bt->waking = false; + bt7932_request_sleep(bt, 0); + mutex_unlock(&bt->lock); + ret = bt7932_wait_pm(bt, BT7932_RUNNING); + if (ret) { + mutex_lock(&bt->lock); + bt7932_fault_locked(bt, "system sleep restoration failed"); + mutex_unlock(&bt->lock); + } + return ret; +} + +static int bt7932_suspend(struct device *dev) +{ + struct bt7932 *bt = pci_get_drvdata(to_pci_dev(dev)); + int ret; + + if (READ_ONCE(bt->fault)) + return -EIO; + if (!bt->registered) + return 0; + ret = hci_suspend_dev(bt->hdev); + if (ret) + return ret; + ret = bt7932_quiesce(bt); + if (!ret) { + dev_dbg(dev, "system sleep quiesce acknowledged\n"); + return 0; + } + dev_warn(dev, "system sleep quiesce failed: %d\n", ret); + /* Undo our request before the PM core rolls back the rest of the bus. */ + if (!READ_ONCE(bt->fault)) + bt7932_restore_transport(bt); + hci_resume_dev(bt->hdev); + return ret; +} + +static int bt7932_resume(struct device *dev) +{ + struct bt7932 *bt = pci_get_drvdata(to_pci_dev(dev)); + int ret; + + if (READ_ONCE(bt->fault)) + return -EIO; + if (!bt->registered) + return 0; + if (READ_ONCE(bt->pm_state) != BT7932_RUNNING) { + ret = bt7932_restore_transport(bt); + if (ret) + return ret; + } + dev_dbg(dev, "transport restored after system sleep\n"); + return hci_resume_dev(bt->hdev); +} + +static int bt7932_freeze(struct device *dev) +{ + struct bt7932 *bt = pci_get_drvdata(to_pci_dev(dev)); + + /* A restored image cannot adopt the running firmware's DMA pointers. */ + return bt->retained ? -EOPNOTSUPP : 0; +} + +static const struct dev_pm_ops bt7932_pm_ops = { + .suspend = bt7932_suspend, + .resume = bt7932_resume, + .freeze = bt7932_freeze, + .thaw = bt7932_resume, + .poweroff = bt7932_freeze, + .restore = bt7932_resume, +}; + +static void bt7932_remove(struct pci_dev *pdev) +{ + struct bt7932 *bt = pci_get_drvdata(pdev); + u16 command = U16_MAX; + bool quiesced, drained; + + /* Closing HCI may send Reset: keep the transport and IRQ alive for it. */ + if (bt->registered) + hci_unregister_dev(bt->hdev); + quiesced = bt->ipc_ready && !READ_ONCE(bt->fault) && !bt7932_quiesce(bt); mutex_lock(&bt->lock); - bt7932_fault_locked(bt, "shutdown retains DMA until external reset"); + WRITE_ONCE(bt->stopping, true); + WRITE_ONCE(bt->opened, false); mutex_unlock(&bt->lock); + wake_up_all(&bt->wake_wait); + cancel_work_sync(&bt->tx_work); + skb_queue_purge(&bt->tx_queue); + if (bt->irq_requested) { + free_irq(bt->irq, bt); + bt->irq_requested = false; + } + pci_clear_master(pdev); + drained = quiesced && !pci_read_config_word(pdev, PCI_COMMAND, &command) && + command != U16_MAX && !(command & PCI_COMMAND_MASTER) && + pci_wait_for_pending_transaction(pdev); + if (!drained && bt->retained) { + /* Keep the DMA allocations and their device alive until reset. */ + get_device(&pdev->dev); + dev_err(&pdev->dev, "PCI drain unconfirmed; DMA retained until reset\n"); + } + pci_set_drvdata(pdev, NULL); + bt7932_cleanup(bt, drained || !bt->retained); } static const struct pci_device_id bt7932_ids[] = { @@ -1145,9 +1341,10 @@ static struct pci_driver bt7932_driver = { .name = "mt7932_bt_pcie", .id_table = bt7932_ids, .probe = bt7932_probe, - .shutdown = bt7932_shutdown, + .remove = bt7932_remove, + .shutdown = bt7932_remove, .driver = { - .suppress_bind_attrs = true, + .pm = pm_sleep_ptr(&bt7932_pm_ops), }, }; module_pci_driver(bt7932_driver); From ce14d5599a4e1b6f8b3427470983bfa368a29ab0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 19:37:23 +1100 Subject: [PATCH 0071/2680] Bluetooth: mt7932: accept nonzero HCI registration indices hci_register_dev returns a nonnegative controller index on success. Treating every nonzero value as an error leaves a successfully published controller with registered=false. Removal then skips hci_unregister_dev and frees its callback state while the HCI core still owns the controller. Reject only negative registration results so every published controller is tracked and unregistered before transport and callback state cleanup. The PCI probe continues to return zero for a successful registration. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/mt7932_bt_pcie.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 8969317d628df..d1c6dc276205d 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -1159,7 +1159,8 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) bt7932_channels(bt); mutex_unlock(&bt->lock); ret = hci_register_dev(bt->hdev); - if (ret) + /* Registration returns the nonnegative HCI index on success. */ + if (ret < 0) goto retained_fault; bt->registered = true; dev_info(&pdev->dev, "BT_HCI_REGISTERED; setup/discovery/connection remain separate gates\n"); From 252d413d17da24aaf03b1eb4975062b3cf951352 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:44:42 +1100 Subject: [PATCH 0072/2680] PCI: apple: preserve the retained bootstrap across suspend-to-idle sleep The retained PIODMA supplier prevents the radio configuration from being built with system sleep. Permit suspend-to-idle once the single bootstrap has completed and the command, FIFO and owned storage validate as idle. Reject sleep from a failed or incomplete bootstrap. Restore the existing register-bank configuration after its power domain resumes, while preserving the arena and never resubmitting its descriptor. Reject hibernation and deeper suspend, which cannot preserve the live endpoint and firmware ownership contract. Keep the supplier and PCI host built in: published DMA and supplier IRQ ownership still lack a removal contract. This retains the existing link boundary and requires no module export or parallel provider. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/PCI/controller/apple-t8140.rst | 10 ++- drivers/pci/controller/Kconfig | 9 ++- .../pci/controller/pcie-apple-piodma-diag.c | 66 ++++++++++++++++++- 3 files changed, 78 insertions(+), 7 deletions(-) diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index 90d4eab8f07f4..24213c2cc3aa7 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -11,11 +11,19 @@ This is an opt-in bring-up implementation, not general T8140 PCIe support. Build ``CONFIG_PCIE_APPLE_PIODMA_DIAG=y`` and pass ``pcie_apple_piodma_diag.enumerate=1`` to the kernel. Without the opt-in, the root-only diagnostic path continues to reject downstream configuration access. -The option excludes suspend and kexec. Failed or uncertain bootstrap state +Build the host and its DART in as well: the supplier retains published DMA +and has no removal contract. Keeping the supplier and host built in avoids +a module dependency on a provider whose lifetime cannot end at unload. + +Suspend-to-idle suspend retains the arena after checking the completed request, +idle FIFO and command state. Resume restores the existing configuration +banks without issuing a second bootstrap. Hibernation, deeper suspend and +kexec are unsupported after activation. Failed or uncertain bootstrap state requires a full external hardware reset; there is no command retry. The PIODMA arena remains allocated until external reset. Controller removal, teardown, memory reuse and arbitrary downstream devices are unqualified. +Fresh hardware qualification must check both radio functions across sleep. Do not remove the controller or its IOMMU while this experiment is active. These lifecycle contracts require further work before production support. diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index d418c2124c565..a3f642fccdb69 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -60,13 +60,16 @@ config PCIE_APPLE config PCIE_APPLE_PIODMA_DIAG bool "J700 PIODMA diagnostic (requires full reset after use)" depends on PCIE_APPLE=y && ARCH_APPLE && IOMMU_DMA - depends on !PM_SLEEP && !KEXEC_CORE + depends on !KEXEC_CORE help Build the experimental SID17 retained bootstrap provider. It submits one fixed request, validates typed ECAM and permits enumeration of the two Neo functions only with the explicit enumerate=1 parameter. - Its arena remains owned until a full reset. This is not production - support, and does not qualify memory reuse or radio operation. + The host and supplier must both be built in: supplier removal cannot + release a published DMA arena safely. Suspend-to-idle sleep retains that + arena and restores the configuration without repeating a request. + Hibernation and deeper suspend are rejected after activation. + This does not qualify memory reuse or arbitrary downstream devices. config PCI_VERSATILE bool "ARM Versatile PB PCI controller" diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index 87caa91262e37..5b1e27ad29e52 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -8,7 +8,9 @@ * * This built-in driver deliberately retains its coherent arena and device * power reference until a coordinator-controlled full hardware reset. There - * is no unbind interface, suspend, kexec or retry. Native ECAM is only + * is no unbind interface, kexec or retry. Suspend-to-idle sleep preserves the + * arena and restores the register banks without resubmitting its request. + * Native ECAM is only * opened by the host after a successful bootstrap and typed read checks. */ #include @@ -29,6 +31,7 @@ #include #include #include +#include #include "pcie-apple-piodma-diag.h" @@ -92,6 +95,7 @@ struct apple_piodma_diag { u64 pointer_prefix; u8 secondary_bus; bool retained; + bool ready; }; static int apple_piodma_diag_root(struct apple_piodma_diag *diag, struct pci_dev *root) @@ -532,11 +536,13 @@ int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root) ret = apple_piodma_diag_initialize(diag); if (!ret) ret = apple_piodma_diag_submit(diag, 0); - if (ret) + if (ret) { dev_err(supplier, "bootstrap failure=%d; retained ownership; no retry\n", ret); - else + } else { + WRITE_ONCE(diag->ready, true); dev_info(supplier, "BOOTSTRAP_VALIDATED word=%#x; one retained slot\n", diag->slots[0].result); + } return ret; } @@ -597,6 +603,59 @@ static void apple_piodma_diag_shutdown(struct platform_device *pdev) dev_warn(&pdev->dev, "diagnostic arena retained; require full hardware reset\n"); } +/* No new request is permitted once the host has admitted its endpoints. */ +static int apple_piodma_diag_sleep_check(struct apple_piodma_diag *diag) +{ + if (!READ_ONCE(diag->ready) || READ_ONCE(diag->irq_fault)) + return -EIO; + if (readl(diag->engine + 0x54) & BIT(0)) + return -EBUSY; + if ((readl(diag->engine + 0x0c) & PIODMA_FIFO_MASK) != 16) + return -EBUSY; + return apple_piodma_diag_check_slots(diag); +} + +static int apple_piodma_diag_suspend_noirq(struct device *dev) +{ + struct apple_piodma_diag *diag = dev_get_drvdata(dev); + + if (!diag->retained) + return 0; + if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE) + return -EOPNOTSUPP; + return apple_piodma_diag_sleep_check(diag); +} + +static int apple_piodma_diag_resume_noirq(struct device *dev) +{ + struct apple_piodma_diag *diag = dev_get_drvdata(dev); + int ret; + + if (!diag->retained) + return 0; + if (!READ_ONCE(diag->ready) || READ_ONCE(diag->irq_fault)) + return -EIO; + /* Genpd may gate this island; restore configuration, never a command. */ + ret = apple_piodma_diag_initialize(diag); + if (ret) + return ret; + return apple_piodma_diag_sleep_check(diag); +} + +static int apple_piodma_diag_freeze(struct device *dev) +{ + struct apple_piodma_diag *diag = dev_get_drvdata(dev); + + return diag->retained ? -EOPNOTSUPP : 0; +} + +static const struct dev_pm_ops apple_piodma_diag_pm_ops = { + .suspend_noirq = apple_piodma_diag_suspend_noirq, + .resume_noirq = apple_piodma_diag_resume_noirq, + .freeze = apple_piodma_diag_freeze, + .poweroff = apple_piodma_diag_freeze, +}; + static const struct of_device_id apple_piodma_diag_match[] = { { .compatible = "apple,t8140-piodma-diagnostic" }, { } @@ -608,6 +667,7 @@ static struct platform_driver apple_piodma_diag_driver = { .driver = { .name = "apple-piodma-diag", .of_match_table = apple_piodma_diag_match, + .pm = pm_sleep_ptr(&apple_piodma_diag_pm_ops), .suppress_bind_attrs = true, }, }; From efdf4ef9f0d1c4382bfe74a5458108d612157b2a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:20:24 +1100 Subject: [PATCH 0073/2680] PCI/DOE: Export pci_doe_disconnected() for modular host controllers pci_dev_set_disconnected() is a static inline in drivers/pci/pci.h that a host controller driver can pass to pci_walk_bus() to mark a hierarchy it has lost as permanently disconnected before tearing it down. Besides setting the error state, it cancels pending DOE exchanges through pci_doe_disconnected(), which is not exported, so modpost rejects a modular caller with 'ERROR: modpost: "pci_doe_disconnected" [drivers/pci/controller/pcie-apple.ko] undefined!'. The Apple PCIe driver is such a caller since it learned to quiesce tunneled PCIe-C links, and it is tristate. Distribution configurations build APPLE_DART as a module, and "depends on APPLE_DART || !APPLE_DART" then makes PCIE_APPLE a module too. Export pci_doe_disconnected() GPL-only, like the rest of the DOE API. Open-coding the walk in the driver would skip the DOE cancellation and leave exchanges waiting on a device that can no longer answer. Fixes: 544b71a5d144 ("PCI: apple: quiesce and restore tunneled PCIe-C links") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pci/doe.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/pci/doe.c b/drivers/pci/doe.c index 7b41da4ec11a3..44be0d513a893 100644 --- a/drivers/pci/doe.c +++ b/drivers/pci/doe.c @@ -891,3 +891,4 @@ void pci_doe_disconnected(struct pci_dev *pdev) xa_for_each(&pdev->doe_mbs, index, doe_mb) pci_doe_cancel_tasks(doe_mb); } +EXPORT_SYMBOL_GPL(pci_doe_disconnected); From 6be32920321ffe456d35465c671553a727af9932 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 01:46:35 +1100 Subject: [PATCH 0074/2680] PCI: apple: run the J700 radio bootstrap by default The J700 Wi-Fi and Bluetooth functions sit behind port 0 of the T8140 PCIe host, which only opens downstream configuration access after the PIODMA bootstrap has completed. The bootstrap provider is already selected per board: it only binds on J700, and only when the device tree points the host's "apple,piodma" property at the engine. It nevertheless also required pcie_apple_piodma_diag.enumerate=1 on the kernel command line, so a kernel built with the provider still left both radio functions undiscovered unless the boot entry carried that option. Make the device tree the switch and run the bootstrap by default. Keep the parameter so enumerate=0 still selects the root-port-only path for diagnosis, and update the Kconfig help and documentation to match. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/PCI/controller/apple-t8140.rst | 11 +++++++---- .../networking/device_drivers/wifi/mt7932-neo.rst | 4 ++-- drivers/pci/controller/Kconfig | 6 ++++-- drivers/pci/controller/pcie-apple-piodma-diag.c | 10 +++++++--- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index 24213c2cc3aa7..fd01165f6cfae 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -7,10 +7,13 @@ The J700 path trains port 0, validates a bounded PIODMA bootstrap request, and enables native ECAM for the MediaTek Wi-Fi and Bluetooth functions. The root bridge forwards their BARs and DART translates endpoint DMA. -This is an opt-in bring-up implementation, not general T8140 PCIe support. -Build ``CONFIG_PCIE_APPLE_PIODMA_DIAG=y`` and pass -``pcie_apple_piodma_diag.enumerate=1`` to the kernel. Without the opt-in, the -root-only diagnostic path continues to reject downstream configuration access. +This is a bring-up implementation, not general T8140 PCIe support. +Build ``CONFIG_PCIE_APPLE_PIODMA_DIAG=y``. The board device tree selects the +bootstrap by pointing the host's ``apple,piodma`` property at the engine, and +the kernel then runs it during the host probe. Passing +``pcie_apple_piodma_diag.enumerate=0`` keeps the root-only diagnostic path, +which rejects downstream configuration access. A kernel built without the +option always uses that root-only path. Build the host and its DART in as well: the supplier retains published DMA and has no removal contract. Keeping the supplier and host built in avoids a module dependency on a provider whose lifetime cannot end at unload. diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index c47d4496e631a..9906e562d4f22 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -23,8 +23,8 @@ Platform and configuration -------------------------- Use the separately reviewed T8140 PCIe bootstrap described in -``Documentation/PCI/controller/apple-t8140.rst``. Its explicit kernel opt-in is -``pcie_apple_piodma_diag.enumerate=1``. Public m1n1 must populate the J700 +``Documentation/PCI/controller/apple-t8140.rst``. It runs during boot when +``CONFIG_PCIE_APPLE_PIODMA_DIAG=y``. Public m1n1 must populate the J700 ``wifi0`` endpoint's own ``local-mac-address``; a zero placeholder is rejected. The tested configuration includes ``CONFIG_MT7932_FULLMAC=m``, diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index a3f642fccdb69..82f0dd1769059 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -63,8 +63,10 @@ config PCIE_APPLE_PIODMA_DIAG depends on !KEXEC_CORE help Build the experimental SID17 retained bootstrap provider. It submits - one fixed request, validates typed ECAM and permits enumeration of - the two Neo functions only with the explicit enumerate=1 parameter. + one fixed request, validates typed ECAM and then enumerates the two + Neo radio functions when the board device tree links the PCIe host + to this engine. The enumerate=0 parameter keeps the host on its + root-port-only path instead. The host and supplier must both be built in: supplier removal cannot release a published DMA arena safely. Suspend-to-idle sleep retains that arena and restores the configuration without repeating a request. diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index 5b1e27ad29e52..c0a85e04b6602 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -7,7 +7,7 @@ * to the reviewed Aurora reference and fresh ADT observations. * * This built-in driver deliberately retains its coherent arena and device - * power reference until a coordinator-controlled full hardware reset. There + * power reference until a full hardware reset. There * is no unbind interface, kexec or retry. Suspend-to-idle sleep preserves the * arena and restores the register banks without resubmitting its request. * Native ECAM is only @@ -51,9 +51,13 @@ #define PIODMA_FIFO_MASK GENMASK(5, 0) #define PIODMA_IRQ_LIMIT 16 -static bool enumerate; +/* + * The board DT opts in by pointing the host's "apple,piodma" at this engine. + * enumerate=0 keeps the host on its root-port-only path for diagnosis. + */ +static bool enumerate = true; module_param(enumerate, bool, 0400); -MODULE_PARM_DESC(enumerate, "Prime once, validate native ECAM, and enumerate the two Neo functions"); +MODULE_PARM_DESC(enumerate, "Prime once, validate native ECAM, and enumerate the two Neo functions (default: on)"); struct apple_piodma_request { u16 offset; From a5e6ceebf9e420e9d3672d84f616886c30d75d41 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 01:47:09 +1100 Subject: [PATCH 0075/2680] PCI: apple: let drivers bind the J700 radio functions After the J700 bootstrap, the host adds both radio functions to the bus and only then checks that neither is bus mastering and that neither has a driver. If either function has a driver by then, the host reports -EBUSY and closes downstream configuration access again, under a driver that has already bound and started using the function. That happens with a built-in driver, or with a module that udev loaded while the bus was being scanned. Check the cold state, with bus mastering off, before the functions are added and their drivers can probe. Treat drivers binding during publication as the normal outcome. Do not report the host as root-port-only once a bootstrap supplier has been found. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pci/controller/pcie-apple.c | 65 ++++++++++++++++------------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 0a5dbd73f7a95..6a8118ceee1aa 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2226,12 +2226,40 @@ static int apple_pcie_neo_check_memory(struct pci_dev *root) return 0; } +/* + * Both functions must come out of the bootstrap cold. Check that before they + * are published, because publication lets their drivers bind and take over. + */ +static int apple_pcie_neo_check_cold(struct apple_pcie *pcie, struct pci_dev *root) +{ + struct pci_dev *endpoint; + u16 command; + int function, ret = 0; + + for (function = 0; function < 2 && !ret; function++) { + endpoint = pci_get_slot(root->subordinate, PCI_DEVFN(0, function)); + if (!endpoint) + return -ENODEV; + if (pci_read_config_word(endpoint, PCI_COMMAND, &command)) { + ret = -EIO; + } else { + dev_info(pcie->dev, "enumerated %s id=%04x:%04x command=%#x\n", + pci_name(endpoint), endpoint->vendor, endpoint->device, command); + if (command & PCI_COMMAND_MASTER) { + pci_clear_master(endpoint); + ret = -EIO; + } + } + pci_dev_put(endpoint); + } + return ret; +} + static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) { struct apple_pcie *pcie = pci_host_bridge_priv(bridge); - struct pci_dev *root, *endpoint; - u16 command; - int function, ret; + struct pci_dev *root; + int ret; root = pci_get_slot(bridge->bus, PCI_DEVFN(0, 0)); if (!root) @@ -2262,37 +2290,16 @@ static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) dev_err(pcie->dev, "root memory forwarding unavailable after assignment: %d\n", ret); goto close_config; } + ret = apple_pcie_neo_check_cold(pcie, root); + if (ret) + goto close_config; pci_bus_add_devices(root->subordinate); - for (function = 0; function < 2; function++) { - endpoint = pci_get_slot(root->subordinate, PCI_DEVFN(0, function)); - if (!endpoint) { - ret = -ENODEV; - break; - } - if (pci_read_config_word(endpoint, PCI_COMMAND, &command)) { - ret = -EIO; - } else { - dev_info(pcie->dev, "enumerated %s id=%04x:%04x command=%#x driver=%s\n", - pci_name(endpoint), endpoint->vendor, endpoint->device, command, - endpoint->driver ? endpoint->driver->name : "unbound"); - if (command & PCI_COMMAND_MASTER) { - /* This experiment never permits endpoint bus mastering. */ - pci_clear_master(endpoint); - ret = -EIO; - } - if (endpoint->driver) - ret = -EBUSY; - } - pci_dev_put(endpoint); - if (ret) - break; - } close_config: if (ret) WRITE_ONCE(pcie->neo_config_ready, false); pci_unlock_rescan_remove(); if (!ret) - dev_info(pcie->dev, "NEO_ENUMERATION_ONLY_READY; functions0/1 unbound, bus mastering off\n"); + dev_info(pcie->dev, "radio functions 0/1 published\n"); out: pci_dev_put(root); return ret; @@ -2344,7 +2351,7 @@ static int apple_pcie_probe(struct platform_device *pdev) pcie->dev = dev; pcie->hw = hw; pcie->piodma_supplier = piodma_supplier; - if (hw->root_bus_only) + if (hw->root_bus_only && !piodma_supplier) dev_info(dev, "root-port probe only; downstream config is blocked\n"); pcie->base = devm_platform_ioremap_resource(pdev, 1); if (IS_ERR(pcie->base)) From 80495250f4757c69c1c1c10c41856a7fd261ba0b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 01:48:09 +1100 Subject: [PATCH 0076/2680] wifi: mt7932: disable ASPM on the J700 radio link On devicetree platforms the PCI core enables ASPM L0s/L1 by default. The station driver was qualified with the PCIe ASPM "performance" policy selected before it was loaded, so the documented admission sequence had userspace change that global policy first. If the module is autoloaded, nothing changes the policy. Disable all ASPM and clock power management states on the link before the driver starts the device, as other MediaTek PCIe Wi-Fi drivers do. This removes the userspace step and leaves the policy for other links alone. The function is probed before the root filesystem is mounted, and the driver reads its firmware while probing, so recommend building it as a module. Document that, and how cfg80211 finds the regulatory database. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../networking/device_drivers/wifi/mt7932-neo.rst | 15 +++++++++++---- drivers/net/wireless/mediatek/mt7932/Kconfig | 2 ++ drivers/net/wireless/mediatek/mt7932/mt7932.c | 2 ++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 9906e562d4f22..07edfea2a922c 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -35,10 +35,17 @@ sleep. The separately selected bootstrap provider has its own lifecycle restrictions. Kexec is excluded while either retained experiment is enabled. Use the ordinary cfg80211 regulatory database and applicable country policy. The validated first-admission fallback is kernel country 00 with firmware XZ. - -After the root filesystem and the local firmware packages are available, -select PCIe ASPM performance policy before loading ``mt7932-fullmac``. This -matches the tested admission sequence. Bluetooth's gate defaults closed; +A modular cfg80211 loads ``regulatory.db`` from the root filesystem; a +built-in one tries before the root filesystem is mounted and then needs +``iw reg reload``. + +The radio functions appear while the kernel boots, before the root +filesystem is mounted, and ``mt7932-fullmac`` reads its firmware while +probing. Build it as a module so that udev loads it from the root +filesystem, or provide its firmware in the initramfs. The driver disables +ASPM and clock power management on its link before starting the device, +matching the tested admission sequence, which used the PCIe ASPM +performance policy. Bluetooth's gate defaults closed; validate the cold, unbound ``14c3:793b`` function, then load ``mt7932_bt_pcie`` with ``enable=1`` after its local inputs are installed. A built-in transport can instead open its enable parameter before requesting diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index d435dbe24c32e..0747ed320c968 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -9,6 +9,8 @@ config MT7932_FULLMAC WPA2-CCMP station operation on 2.4 GHz and channels 36..48 at 20 MHz. Requires original firmware, own calibration and original-derived country policy packages in addition to the cfg80211 regulatory database. + The device is probed before the root filesystem is mounted, so say M + unless those files are in the initramfs. Requires a trained retained PCI link and a translated SID1 DMA domain. Firmware, runtime DMA and calibration negotiation start automatically; cfg80211 and userspace select the country and network. No test boot diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index b422d5acb5b9d..0549ca8575dc1 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -1181,6 +1181,8 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) domain = iommu_get_domain_for_dev(&pdev->dev); if (!domain || (domain->type != IOMMU_DOMAIN_DMA && domain->type != IOMMU_DOMAIN_DMA_FQ)) return dev_err_probe(&pdev->dev, -EINVAL, "translated DMA domain required\n"); + /* Station operation was qualified with ASPM and clock PM off on this link. */ + pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); ret = pcim_enable_device(pdev); if (ret) return ret; From d43b2efaf091d5e5967db0c7a3aa17a8a827a88e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 01:48:39 +1100 Subject: [PATCH 0077/2680] Bluetooth: mt7932: probe the J700 transport by default The transport only probes after userspace loads it with enable=1, so an autoloaded module leaves the Bluetooth function unbound. The gate was meant to keep the probe away until the root filesystem holds the unit's firmware and calibration. A module already loads from the root filesystem, and the probe validates a cold, unbound function before it starts. Open the gate by default and keep enable=0 to leave the function unbound. The documented admission sequence also selected the PCIe ASPM "performance" policy and loaded the Wi-Fi driver first. Disable ASPM and clock power management on the link in this probe as well, so the transport does not depend on the Wi-Fi driver for it. Add a module soft dependency so that modprobe loads mt7932_fullmac, which probes function 0, before the transport, keeping the tested firmware start order when udev loads both. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../networking/device_drivers/wifi/mt7932-neo.rst | 13 ++++++++----- drivers/bluetooth/Kconfig | 3 ++- drivers/bluetooth/mt7932_bt_pcie.c | 10 +++++++--- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 07edfea2a922c..4fbf7b38c817c 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -45,11 +45,14 @@ probing. Build it as a module so that udev loads it from the root filesystem, or provide its firmware in the initramfs. The driver disables ASPM and clock power management on its link before starting the device, matching the tested admission sequence, which used the PCIe ASPM -performance policy. Bluetooth's gate defaults closed; -validate the cold, unbound ``14c3:793b`` function, then load -``mt7932_bt_pcie`` with ``enable=1`` after its local inputs are installed. -A built-in transport can instead open its enable parameter before requesting -the PCI probe. +performance policy. + +``mt7932_bt_pcie`` probes the cold ``14c3:793b`` function when it loads and +rejects a function that is not cold; ``enable=0`` leaves it unbound. Its +module soft dependency loads ``mt7932-fullmac`` first, so the Bluetooth +firmware starts after the Wi-Fi firmware, as in the tested sequence. The +transport also disables ASPM on the shared link. Like the Wi-Fi driver, it +reads its firmware while probing, so build it as a module. Do not reprobe after a failed or uncertain Bluetooth admission. The tested Wi-Fi driver owns function 0 and Bluetooth owns function 1. diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index 93c3b591de27b..70bd7af5a9bd0 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -30,7 +30,8 @@ config BT_MTK7932_PCIE select FW_LOADER help Support the J700 Bluetooth PCIe function with locally installed firmware - and calibration. The late-probe enable parameter defaults off. + and calibration. The function is probed when the driver loads; the + enable=0 parameter leaves it unbound. System suspend uses the firmware sleep handshake and preserves DMA storage. Hibernation and automatic firmware recovery are unsupported. Removal drains the PCI function before releasing device-visible memory; diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index d1c6dc276205d..a0d76eed77f66 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -64,9 +64,9 @@ #define BT7932_CAL "mediatek/j700-mt7932-btcal.bin" #define BT7932_ADDR "mediatek/j700-mt7932-bdaddr.bin" -static bool enable; +static bool enable = true; module_param(enable, bool, 0600); -MODULE_PARM_DESC(enable, "Diagnostic late-probe gate; default closed, no effect on an already bound device"); +MODULE_PARM_DESC(enable, "Probe the transport (default: on); no effect on an already bound device"); struct bt7932_geometry { u16 stride; @@ -1044,13 +1044,15 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) u16 command; int ret; - /* The coordinator opens this only after the real root has its inputs. */ + /* enable=0 leaves the function unbound for diagnosis. */ if (!READ_ONCE(enable)) return -ENODEV; if (!of_machine_is_compatible("apple,j700") || !of_machine_is_compatible("apple,t8140") || pci_domain_nr(pdev->bus) || pdev->bus->number != 1 || pdev->devfn != PCI_DEVFN(0, 1)) return -ENODEV; + /* Admission was qualified with ASPM and clock PM off on this link. */ + pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); bt = kzalloc_obj(*bt); if (!bt) return -ENOMEM; @@ -1352,6 +1354,8 @@ module_pci_driver(bt7932_driver); MODULE_DESCRIPTION("Experimental standalone J700 MT7932 Bluetooth PCIe transport"); MODULE_LICENSE("GPL"); +/* Admission was qualified after the Wi-Fi function had started its firmware. */ +MODULE_SOFTDEP("pre: mt7932_fullmac"); MODULE_FIRMWARE(BT7932_FW_B0); MODULE_FIRMWARE(BT7932_FW_B1); MODULE_FIRMWARE(BT7932_FW_B1_FALLBACK); From 632e63e542aa1e0a58298cd25de711e141530b9d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:15:45 +1100 Subject: [PATCH 0078/2680] wifi: mt7932: depend on cfg80211 instead of selecting it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MT7932_FULLMAC selects CFG80211. Built in, that forces CFG80211=y even when RFKILL=m, and Kconfig then warns about an unmet dependency, because CFG80211 depends on RFKILL || !RFKILL. Wireless drivers normally depend on CFG80211 rather than select it. Do the same, so that cfg80211's own constraints decide whether the driver can be built in. The J700 radio fragment relied on the select. Enable cfg80211 there as a module, which also lets it load regulatory.db from the root filesystem. Record the modular cfg80211 in the tested configuration. [ryan: adapted to 7.2.10 — split by topic: the defconfig fragment hunks are carried by 900-config-tooling] Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/networking/device_drivers/wifi/mt7932-neo.rst | 2 +- drivers/net/wireless/mediatek/mt7932/Kconfig | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 4fbf7b38c817c..282646abb3572 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -28,7 +28,7 @@ Use the separately reviewed T8140 PCIe bootstrap described in ``wifi0`` endpoint's own ``local-mac-address``; a zero placeholder is rejected. The tested configuration includes ``CONFIG_MT7932_FULLMAC=m``, -``CONFIG_CFG80211=y``, ``CONFIG_BT_MTK7932_PCIE=m``, ``CONFIG_BT_BREDR=y``, +``CONFIG_CFG80211=m``, ``CONFIG_BT_MTK7932_PCIE=m``, ``CONFIG_BT_BREDR=y``, ``CONFIG_BT_LE=y``, ``CONFIG_CRYPTO_AES=y`` and ``CONFIG_CRYPTO_CMAC=y``. The Bluetooth transport can be built with modular Bluetooth and system sleep. The separately selected bootstrap provider has its own lifecycle diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index 0747ed320c968..fa18a3357d10c 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -2,8 +2,8 @@ config MT7932_FULLMAC tristate "Experimental J700 MT7932 fullmac wireless support" depends on PCI && OF && ARM64 && IOMMU_DMA + depends on CFG80211 select FW_LOADER - select CFG80211 help Native support for the built-in J700 Wi-Fi endpoint: managed WPA2-CCMP station operation on 2.4 GHz and channels 36..48 at 20 MHz. From 25ba00bba2658fbef2dfdbc93dcc6b3ed46a845b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:35:10 +1100 Subject: [PATCH 0079/2680] wifi: mt7932: support the 5 GHz channels above channel 48 The station only advertises 5 GHz channels 36 to 48, so access points on UNII-2, UNII-2e or UNII-3 channels are invisible. On the MacBook Neo used for testing, the network's 5 GHz radio is on channel 149, and the station falls back to its 2.4 GHz radio. Advertise every 20 MHz primary from 36 to 64, 100 to 144 and 149 to 165, and let the cfg80211 regulatory domain decide which are usable. Each gets its 40 and 80 MHz pairing (165 stays at 20 MHz). The same pairing is used to decode association channel events and to pick the unit's own calibration records, which cover the whole band. Radar channels were previously rejected by refusing the whole regulatory domain. Pass them to the firmware with cfg80211's flags instead, and keep the station passive there. cfg80211 marks them RADAR, not NO_IR, so scans listen on them without probing. Scans made while associated skip them, and connects never target them: DFS client operation is not qualified, and a passive dwell only takes time away from the home channel. A channel whose limits the driver cannot apply, such as a power cap below the qualified 20 dBm, is now disabled on its own rather than blocking every channel. The firmware scan request has 32 channel slots, which holds the larger 5 GHz list in one batch. A scan request left with no eligible channel now completes at once. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 7 ++ drivers/net/wireless/mediatek/mt7932/Kconfig | 4 +- .../net/wireless/mediatek/mt7932/bandwidth.h | 9 +-- .../wireless/mediatek/mt7932/calibration.h | 7 +- .../net/wireless/mediatek/mt7932/channels.h | 66 ++++++++++++++++++- .../net/wireless/mediatek/mt7932/frontend.c | 23 +++++-- drivers/net/wireless/mediatek/mt7932/link.c | 2 +- drivers/net/wireless/mediatek/mt7932/mt7932.h | 4 +- .../net/wireless/mediatek/mt7932/regulatory.h | 6 +- drivers/net/wireless/mediatek/mt7932/scan.h | 5 +- .../net/wireless/mediatek/mt7932/startup.c | 43 ++++++++---- 11 files changed, 142 insertions(+), 34 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 282646abb3572..b2b3540f1d723 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -33,6 +33,13 @@ The tested configuration includes ``CONFIG_MT7932_FULLMAC=m``, The Bluetooth transport can be built with modular Bluetooth and system sleep. The separately selected bootstrap provider has its own lifecycle restrictions. Kexec is excluded while either retained experiment is enabled. +The station uses 2.4 GHz channels 1-13 and the 5 GHz channels 36-64, +100-144 and 149-165 that the cfg80211 regulatory domain permits. Radar (DFS) +channels stay passive: the firmware listens on them while the station is not +associated, but never probes or joins an access point there, and scans made +while associated skip them. A channel whose restrictions the driver cannot +apply (reduced power, no OFDM, PSD limits) is disabled rather than used. + Use the ordinary cfg80211 regulatory database and applicable country policy. The validated first-admission fallback is kernel country 00 with firmware XZ. A modular cfg80211 loads ``regulatory.db`` from the root filesystem; a diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index fa18a3357d10c..32e6608851326 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -6,7 +6,9 @@ config MT7932_FULLMAC select FW_LOADER help Native support for the built-in J700 Wi-Fi endpoint: managed - WPA2-CCMP station operation on 2.4 GHz and channels 36..48 at 20 MHz. + WPA2-CCMP station operation on 2.4 GHz and on the 5 GHz UNII-1, -2, + -2e and -3 channels that the regulatory domain permits. Radar (DFS) + channels are scanned passively but not joined. Requires original firmware, own calibration and original-derived country policy packages in addition to the cfg80211 regulatory database. The device is probed before the root filesystem is mounted, so say M diff --git a/drivers/net/wireless/mediatek/mt7932/bandwidth.h b/drivers/net/wireless/mediatek/mt7932/bandwidth.h index c5efe6599e0a4..07c281e0fc589 100644 --- a/drivers/net/wireless/mediatek/mt7932/bandwidth.h +++ b/drivers/net/wireless/mediatek/mt7932/bandwidth.h @@ -13,13 +13,14 @@ static inline u8 mt7932_domain_5g_bw(void) } /* EE4d width 0 combines 20/40; primary and extension select the center. - * Keep all operational primaries inside 36..48, and 2 GHz at 20 MHz. + * Extension 1 places the secondary above the primary, 3 below it. Keep 2 GHz + * at 20 MHz, and 165, which has no 40 MHz pair, at 20 MHz too. */ static inline bool mt7932_channel_layout(u8 primary, u8 width, u8 extension, u8 maximum, u8 *center, u8 *bw) { u8 band = mt7932_channel_band(primary); - u8 expected = primary == 36 || primary == 44 ? 1 : 3; + u8 pair = mt7932_channel_ht40_centre(primary); if (!band || maximum > 2 || width > 1) return false; @@ -28,12 +29,12 @@ static inline bool mt7932_channel_layout(u8 primary, u8 width, u8 extension, *bw = 0; return true; } - if (band != 2 || extension != expected) + if (band != 2 || !pair || extension != (pair > primary ? 1 : 3)) return false; *bw = width ? 2 : 1; if (*bw > maximum) return false; - *center = width ? 42 : extension == 1 ? primary + 2 : primary - 2; + *center = width ? mt7932_channel_vht80_centre(primary) : pair; return true; } diff --git a/drivers/net/wireless/mediatek/mt7932/calibration.h b/drivers/net/wireless/mediatek/mt7932/calibration.h index f008e34146d64..afb4cdc57ed5d 100644 --- a/drivers/net/wireless/mediatek/mt7932/calibration.h +++ b/drivers/net/wireless/mediatek/mt7932/calibration.h @@ -2,6 +2,7 @@ #ifndef MT7932_CALIBRATION_H #define MT7932_CALIBRATION_H +#include "channels.h" #include "protocol.h" /* Protected by the RX/procedure lock. One active logical procedure only; @@ -146,7 +147,7 @@ static inline int mt7932_cal_association(const u8 *data, size_t size, u8 channel if (channel >= 1 && channel <= 13) group = 0; - else if (channel >= 36 && channel <= 48 && !(channel % 2)) + else if (mt7932_channel_5g_cal(channel)) group = 1; else return -EOPNOTSUPP; @@ -228,7 +229,7 @@ static inline int mt7932_cal_requested_5g(const u8 *data, size_t size, int ret; if (upper != 0x1000 || version != 12 || module != 0x89 || - channel < 36 || channel > 48 || channel % 2) + !mt7932_channel_5g_cal(channel)) return -EOPNOTSUPP; ret = mt7932_cal_validate(data, size); if (ret) @@ -243,7 +244,7 @@ static inline int mt7932_cal_requested_5g(const u8 *data, size_t size, if (segment.data[i]) return -EOPNOTSUPP; /* Own row610f07 does not remap center selectors1/2; only primaries. */ - selected = channel % 4 ? channel : channel <= 40 ? 38 : 46; + selected = mt7932_channel_band(channel) ? mt7932_channel_cal_centre(channel) : channel; params[0] = 0x10000001; params[1] = 0x10001000 | channel; params[2] = 0x10001000 | (channel << 17) | selected; diff --git a/drivers/net/wireless/mediatek/mt7932/channels.h b/drivers/net/wireless/mediatek/mt7932/channels.h index 8c4b61e4a3ec7..bae04c245d72a 100644 --- a/drivers/net/wireless/mediatek/mt7932/channels.h +++ b/drivers/net/wireless/mediatek/mt7932/channels.h @@ -4,14 +4,78 @@ /* Bounded operational whitelist, not the full calibration channel namespace. * Firmware RF bands1/2 are distinct from single-interface MAC index0. + * 5 GHz covers the 20 MHz primaries of UNII-1/2 (36..64), UNII-2e (100..144) + * and UNII-3 (149..165); regulatory flags decide which of them are usable. */ +#define MT7932_CHANNELS_2G 13 +#define MT7932_CHANNELS_5G 25 +#define MT7932_CHANNELS (MT7932_CHANNELS_2G + MT7932_CHANNELS_5G) + static inline unsigned int mt7932_channel_band(unsigned int channel) { if (channel >= 1 && channel <= 13) return 1; - if (channel >= 36 && channel <= 48 && !(channel % 4)) + if (((channel >= 36 && channel <= 64) || (channel >= 100 && channel <= 144)) && + !(channel % 4)) + return 2; + if (channel >= 149 && channel <= 165 && !((channel - 149) % 4)) return 2; return 0; } +/* The nth 5 GHz primary, in the order the wiphy advertises them. */ +static inline unsigned int mt7932_channel_5g(unsigned int index) +{ + if (index < 8) + return 36 + 4 * index; + if (index < 20) + return 100 + 4 * (index - 8); + return 149 + 4 * (index - 20); +} + +/* Offset of a 5 GHz primary within its 80 MHz block numbering. */ +static inline unsigned int mt7932_channel_slot(unsigned int primary, unsigned int *base) +{ + *base = primary >= 149 ? 149 : primary >= 100 ? 100 : 36; + return (primary - *base) / 4; +} + +/* 40 MHz pair centre of a 5 GHz primary; 0 where no pair exists (165). */ +static inline unsigned int mt7932_channel_ht40_centre(unsigned int primary) +{ + unsigned int base, slot; + + if (mt7932_channel_band(primary) != 2 || primary == 165) + return 0; + slot = mt7932_channel_slot(primary, &base); + return base + 4 * (slot & ~1U) + 2; +} + +/* 80 MHz block centre of a 5 GHz primary; 0 where no block exists (165). */ +static inline unsigned int mt7932_channel_vht80_centre(unsigned int primary) +{ + unsigned int base, slot; + + if (mt7932_channel_band(primary) != 2 || primary == 165) + return 0; + slot = mt7932_channel_slot(primary, &base); + return base + 4 * (slot & ~3U) + 6; +} + +/* A 5 GHz primary or a 40/80 MHz centre that calibration can be asked for. */ +static inline bool mt7932_channel_5g_cal(unsigned int channel) +{ + if ((channel >= 36 && channel <= 64) || (channel >= 100 && channel <= 144)) + return !(channel % 2); + return channel >= 149 && channel <= 167 && (channel % 2); +} + +/* Calibration centre for a 5 GHz primary. 165 has no 40 MHz pair; its own + * calibration package carries the record for the 165/169 centre, 167. + */ +static inline unsigned int mt7932_channel_cal_centre(unsigned int primary) +{ + return primary == 165 ? 167 : mt7932_channel_ht40_centre(primary); +} + #endif diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 464dca910a978..0bb2761cf934c 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -145,7 +145,7 @@ static int mt_scan_submit_next(struct mt7932 *m) { struct cfg80211_scan_request *request = m->scan_request; const struct cfg80211_ssid *ssid = NULL; - u8 body[0x4d4], channels[13]; + u8 body[0x4d4], channels[MT7932_CHANNELS_5G]; unsigned int count = 0, i; unsigned long flags; bool passive = false; @@ -159,11 +159,15 @@ static int mt_scan_submit_next(struct mt7932 *m) passive = !(m->scan_batch & 1); for (i = 0; i < request->n_channels; i++) { struct ieee80211_channel *channel = request->channels[i]; - bool listen = !request->n_ssids || + bool radar = channel->flags & IEEE80211_CHAN_RADAR; + bool listen = !request->n_ssids || radar || (channel->flags & IEEE80211_CHAN_NO_IR); + /* Radar channels are never joined; while associated, + * a passive dwell there only delays the home channel. + */ if (mt7932_channel_band(channel->hw_value) != m->scan_band || - listen != passive) + listen != passive || (radar && m->scan_home_channel)) continue; if (channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_20MHZ) || count == ARRAY_SIZE(channels)) @@ -215,7 +219,7 @@ static int mt_scan_submit_next(struct mt7932 *m) */ int mt_connect_discover(struct mt7932 *m) { - u8 body[0x4d4], channels[13], cancel[4] = {}; + u8 body[0x4d4], channels[MT7932_CHANNELS_5G], cancel[4] = {}; unsigned long flags; unsigned int attempt, band, i, count; int ret = 0; @@ -236,7 +240,8 @@ int mt_connect_discover(struct mt7932 *m) if (m->connect_channel_req && channel != m->connect_channel_req) continue; - if (channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_20MHZ)) + if (channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_20MHZ | + IEEE80211_CHAN_RADAR)) continue; passive |= !!(channel->flags & IEEE80211_CHAN_NO_IR); channels[count++] = channel->hw_value; @@ -328,7 +333,10 @@ int mt_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) dev_info(&m->pdev->dev, "SCAN_REQUEST: channels=%u ssids=%d ies=%zu flags=%08x duration=%u/%u\n", request->n_channels, request->n_ssids, request->ie_len, request->flags, request->duration, request->duration_mandatory); - if (!request->n_channels || request->n_channels > 17 || request->ie_len > 600 || + /* cfg80211 lists each advertised channel at most once. */ + if (request->n_channels > MT7932_CHANNELS) + return -EOPNOTSUPP; + if (!request->n_channels || request->ie_len > 600 || request->duration_mandatory || (request->flags & ~(NL80211_SCAN_FLAG_FLUSH | NL80211_SCAN_FLAG_COLOCATED_6GHZ)) || request->n_ssids > 1 || (request->n_ssids && request->ssids[0].ssid_len > 32)) @@ -362,6 +370,9 @@ int mt_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) spin_unlock_irqrestore(&m->response_lock, flags); cancel_delayed_work(&m->scan_timeout_work); } else { + /* Nothing eligible to submit: complete the request. */ + if (ret) + schedule_work(&m->scan_finish_work); ret = 0; } out: diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 8caab22dbbb34..fc2d333519f98 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -343,7 +343,7 @@ void mt_connect_work(struct work_struct *work) goto free; } if (m->connect_bss->channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_IR | - IEEE80211_CHAN_NO_20MHZ)) { + IEEE80211_CHAN_NO_20MHZ | IEEE80211_CHAN_RADAR)) { ret = -EINVAL; goto free; } diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 3e30ec3aff659..f6fc1f235e619 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -94,8 +94,8 @@ struct mt7932 { struct ieee80211_supported_band band2; struct cfg80211_chan_def connect_chandef; u8 connect_center; - struct ieee80211_channel channels[13]; - struct ieee80211_channel channels5[4]; + struct ieee80211_channel channels[MT7932_CHANNELS_2G]; + struct ieee80211_channel channels5[MT7932_CHANNELS_5G]; struct ieee80211_supported_band band5; u8 scan_band, scan_batch; struct ieee80211_rate rates[12]; diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory.h b/drivers/net/wireless/mediatek/mt7932/regulatory.h index 6f3869fe7200e..0ea60d9d7abcb 100644 --- a/drivers/net/wireless/mediatek/mt7932/regulatory.h +++ b/drivers/net/wireless/mediatek/mt7932/regulatory.h @@ -5,6 +5,8 @@ /* Private, versioned packaging of original-derived country policy. This is * not firmware code and does not replace cfg80211's regulatory database. */ +#include "channels.h" + #define MT7932_POLICY_TABLES 13 #define MT7932_POLICY_MODES 3 @@ -20,12 +22,12 @@ struct mt7932_reg_snapshot { * profile. Stock unknown-country startup uses XZ while Linux stays 00. */ u8 alpha2[2]; - u8 domain[148]; + u8 domain[12 + 8 * MT7932_CHANNELS]; u16 length; /* Keep numeric limits in equality checks even though CID0f contains only * flags. An unsupported reduction must not be mistaken for no change. */ - s32 power[17]; + s32 power[MT7932_CHANNELS]; int error; }; diff --git a/drivers/net/wireless/mediatek/mt7932/scan.h b/drivers/net/wireless/mediatek/mt7932/scan.h index 4899cbb6c1654..ae26645cc3693 100644 --- a/drivers/net/wireless/mediatek/mt7932/scan.h +++ b/drivers/net/wireless/mediatek/mt7932/scan.h @@ -5,6 +5,9 @@ #include "protocol.h" #include "channels.h" +/* CID03 carries this many band/channel slots, from 0xa0 up to 0xe0. */ +#define MT7932_SCAN_CHANNELS 32 + /* Selected ordinary TX17/fullmac wildcard scan, not a pre-RF gate. * Caller must own BSS0 and filter cfg80211 DISABLED channels first. NO_IR * channels require mt7932_scan_passive() before submission. @@ -17,7 +20,7 @@ static inline int mt7932_scan_body(u8 *body, size_t capacity, u8 seq, 0x6b,7,0x0f,0xff,0xff,0xff,0xff,0xff,0xff}; unsigned int i, j; - if (capacity < 0x4d4 || !seq || seq > 127 || !count || count > 13) + if (capacity < 0x4d4 || !seq || seq > 127 || !count || count > MT7932_SCAN_CHANNELS) return -EINVAL; for (i = 0; i < count; i++) { if (!mt7932_channel_band(channels[i]) || diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index d189f4ad13943..a24597fe0b369 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -334,22 +334,29 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request memcpy(reg.domain, "XZ", 2); reg.domain[4] = 1; /* 2 GHz remains 20 MHz. */ reg.domain[5] = mt7932_domain_5g_bw(); - for (i = 0; i < 17; i++) { - struct ieee80211_channel *channel = i < 13 ? &m->channels[i] : &m->channels5[i - 13]; + for (i = 0; i < MT7932_CHANNELS; i++) { + struct ieee80211_channel *channel = i < MT7932_CHANNELS_2G ? + &m->channels[i] : &m->channels5[i - MT7932_CHANNELS_2G]; u32 flags = channel->flags; reg.power[i] = channel->max_power; if (flags & IEEE80211_CHAN_DISABLED) continue; - if (flags & (IEEE80211_CHAN_RADAR | IEEE80211_CHAN_NO_OFDM | - IEEE80211_CHAN_NO_20MHZ | IEEE80211_CHAN_PSD | IEEE80211_CHAN_IR_CONCURRENT)) - reg.error = -EOPNOTSUPP; /* Native per-rate units are not a proven dBm conversion. Do not * silently ignore a numeric reduction below the qualified 20 dBm * channel envelope or reinterpret sentinel bytes as power caps. + * A channel with a restriction the driver cannot honour is left + * out of the firmware domain and disabled for cfg80211 instead. + */ + if (flags & (IEEE80211_CHAN_NO_OFDM | IEEE80211_CHAN_NO_20MHZ | + IEEE80211_CHAN_PSD | IEEE80211_CHAN_IR_CONCURRENT) || + channel->max_power < 20) { + channel->flags |= IEEE80211_CHAN_DISABLED; + continue; + } + /* Radar channels carry cfg80211's RADAR flag: the firmware may + * listen there, but the station never probes or joins them. */ - if (channel->max_power < 20) - reg.error = -EOPNOTSUPP; put_unaligned_le16(channel->hw_value, reg.domain + 12 + count * 8); /* Driver bandwidth limits are already in channel->flags. Adding * the old blanket 20 MHz mask here would contradict the wiphy @@ -357,7 +364,7 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request */ put_unaligned_le32(flags, reg.domain + 16 + count * 8); count++; - reg.domain[i < 13 ? 8 : 9]++; + reg.domain[i < MT7932_CHANNELS_2G ? 8 : 9]++; } reg.length = 12 + count * 8; if (!count) @@ -439,7 +446,7 @@ int mt_register_regulatory_gate(struct mt7932 *m) * Global hints from other radios still reach the notifier and are handled. */ m->wiphy->regulatory_flags |= REGULATORY_COUNTRY_IE_IGNORE; - for (i = 0; i < 13; i++) { + for (i = 0; i < MT7932_CHANNELS_2G; i++) { m->channels[i].band = NL80211_BAND_2GHZ; m->channels[i].center_freq = 2412 + 5 * i; m->channels[i].hw_value = i + 1; @@ -457,12 +464,22 @@ int mt_register_regulatory_gate(struct mt7932 *m) m->wiphy->bands[NL80211_BAND_2GHZ] = &m->band2; { for (i = 0; i < ARRAY_SIZE(m->channels5); i++) { + unsigned int primary = mt7932_channel_5g(i); + unsigned int pair = mt7932_channel_ht40_centre(primary); + u32 flags = IEEE80211_CHAN_NO_160MHZ; + + /* The secondary channel sits on the far side of the pair centre. */ + if (!pair) + flags |= IEEE80211_CHAN_NO_HT40 | IEEE80211_CHAN_NO_80MHZ; + else if (pair < primary) + flags |= IEEE80211_CHAN_NO_HT40PLUS; + else + flags |= IEEE80211_CHAN_NO_HT40MINUS; m->channels5[i].band = NL80211_BAND_5GHZ; - m->channels5[i].center_freq = 5180 + 20 * i; - m->channels5[i].hw_value = 36 + 4 * i; + m->channels5[i].center_freq = 5000 + 5 * primary; + m->channels5[i].hw_value = primary; m->channels5[i].max_power = 20; - m->channels5[i].flags = IEEE80211_CHAN_NO_160MHZ | - (i % 2 ? IEEE80211_CHAN_NO_HT40PLUS : IEEE80211_CHAN_NO_HT40MINUS); + m->channels5[i].flags = flags; if (MT7932_MAX_5G_BW < 2) m->channels5[i].flags |= IEEE80211_CHAN_NO_80MHZ; if (!MT7932_MAX_5G_BW) From d8fa6fff8b46b52470402596871899d06211912a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:35:51 +1100 Subject: [PATCH 0080/2680] wifi: mt7932: honour the channel permissions of the country package The country package that the driver sends to the firmware has a per-rate power limit table for every channel. Where a country does not permit a channel, every limit of that channel is undefined. Examples are UNII-3 in Japan, UNII-2e in China, 120 to 128 in Australia and 144 in most of Europe. The driver still listed such channels in the firmware domain whenever the cfg80211 regulatory domain allowed them. Drop channels the loaded package does not permit from the firmware domain, and disable them in cfg80211 too, so that scans and connects do not offer them. cfg80211 recomputes channel flags on each regulatory change, and the package is reloaded for each change, so the two stay in step. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 4 +- .../net/wireless/mediatek/mt7932/regulatory.h | 50 +++++++++++++++++++ .../net/wireless/mediatek/mt7932/startup.c | 28 +++++++++++ 3 files changed, 81 insertions(+), 1 deletion(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index b2b3540f1d723..ea2ebbc5d297e 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -38,7 +38,9 @@ The station uses 2.4 GHz channels 1-13 and the 5 GHz channels 36-64, channels stay passive: the firmware listens on them while the station is not associated, but never probes or joins an access point there, and scans made while associated skip them. A channel whose restrictions the driver cannot -apply (reduced power, no OFDM, PSD limits) is disabled rather than used. +apply (reduced power, no OFDM, PSD limits) is disabled rather than used, and +so is a channel the country package does not permit: every power limit of +that channel in ``policy/.bin`` is left undefined. Use the ordinary cfg80211 regulatory database and applicable country policy. The validated first-admission fallback is kernel country 00 with firmware XZ. diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory.h b/drivers/net/wireless/mediatek/mt7932/regulatory.h index 0ea60d9d7abcb..a49885cdb9788 100644 --- a/drivers/net/wireless/mediatek/mt7932/regulatory.h +++ b/drivers/net/wireless/mediatek/mt7932/regulatory.h @@ -102,4 +102,54 @@ static inline int mt7932_policy_parse(struct mt7932_policy *out, return at == size ? 0 : -EINVAL; } +/* Every entry of a validated package starts with its channel number and is + * followed by per-rate power limits. A country that does not permit a + * channel leaves all of that channel's limits at this byte. + */ +#define MT7932_POLICY_NO_LIMIT 0xc4 + +static inline bool mt7932_policy_permits(const struct mt7932_policy *policy, + unsigned int channel) +{ + unsigned int i, j, k; + + for (i = 1; i <= 9; i++) { + const u8 *table = policy->table[i]; + + for (j = 0; j < table[4]; j++) { + const u8 *entry = table + 44 + j * 122; + + if (entry[0] != channel) + continue; + for (k = 1; k < 122; k++) + if (entry[k] != MT7932_POLICY_NO_LIMIT) + return true; + return false; + } + } + return false; +} + +/* Remove the channels the country package forbids from a CID0f domain. */ +static inline void mt7932_policy_filter(struct mt7932_reg_snapshot *reg, + const struct mt7932_policy *policy) +{ + unsigned int i, kept = 0, count = (reg->length - 12) / 8; + + reg->domain[8] = 0; + reg->domain[9] = 0; + for (i = 0; i < count; i++) { + const u8 *entry = reg->domain + 12 + i * 8; + unsigned int channel = get_unaligned_le16(entry); + + if (!mt7932_policy_permits(policy, channel)) + continue; + memmove(reg->domain + 12 + kept * 8, entry, 8); + reg->domain[channel <= 14 ? 8 : 9]++; + kept++; + } + memset(reg->domain + 12 + kept * 8, 0, (count - kept) * 8); + reg->length = 12 + kept * 8; +} + #endif diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index a24597fe0b369..9b54240339526 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -1,4 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ +#include + #include "mt7932.h" static int mt_stock_config(struct mt7932 *m); @@ -221,6 +223,28 @@ static int mt_startup_once(struct mt7932 *m) return ret; } +/* Keep cfg80211's channel list in step with the channels the country + * package leaves out of the firmware domain. cfg80211 recomputes the flags + * on its next regulatory change, which runs this again. + */ +static void mt_policy_disable(struct mt7932 *m, const struct mt7932_policy *policy) +{ + unsigned int i; + + rtnl_lock(); + wiphy_lock(m->wiphy); + for (i = 0; i < MT7932_CHANNELS; i++) { + struct ieee80211_channel *channel = i < MT7932_CHANNELS_2G ? + &m->channels[i] : &m->channels5[i - MT7932_CHANNELS_2G]; + + if (!(channel->flags & IEEE80211_CHAN_DISABLED) && + !mt7932_policy_permits(policy, channel->hw_value)) + channel->flags |= IEEE80211_CHAN_DISABLED; + } + wiphy_unlock(m->wiphy); + rtnl_unlock(); +} + static void mt_startup_work(struct work_struct *work) { struct mt7932 *m = container_of(work, struct mt7932, startup_work); @@ -275,6 +299,10 @@ static void mt_startup_work(struct work_struct *work) } if (!ret) ret = mt7932_policy_parse(&policy, file->data, file->size, reg.domain); + if (!ret) { + mt7932_policy_filter(®, &policy); + mt_policy_disable(m, &policy); + } mutex_lock(&m->command_mutex); if (READ_ONCE(m->stopping) || generation != READ_ONCE(m->reg_generation)) goto next; From b5ce0b675e5b09347a60769db301d59a078e1e48 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:37:23 +1100 Subject: [PATCH 0081/2680] wifi: mt7932: let an address change wait for a retiring scan NetworkManager sets a per-network station address before each connection by default: Fedora uses a stable per-SSID address. It takes the interface down, changes the address and brings the interface back up. Taking the interface down cancels any running scan, but the firmware completes the cancellation asynchronously. Until then the driver refuses the address change with -EBUSY, so the activation fails with "set-hw-addr: failed to set-cloned MAC address" and NetworkManager moves on to another profile. When the interface is down, wait for the scan to retire, for up to two seconds, before checking whether the address can change. Changes on a running interface are still refused, as eth_mac_addr() requires. Log the busy state at debug level when a change is still refused. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/frontend.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 0bb2761cf934c..47d0c9e399ce6 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -447,12 +447,25 @@ static int mt_set_mac_address(struct net_device *netdev, void *address) /* No live-MAC change: the next BSS activation and connect commands use * this address together. A failed retirement must not change identity. + * NetworkManager takes the interface down before changing the address, + * which cancels any scan; let that scan retire instead of failing. */ + if (!netif_running(netdev)) { + unsigned long deadline = jiffies + msecs_to_jiffies(2000); + + mt_scan_quiesce(m); + while (READ_ONCE(m->retired_scan_seq) && time_before(jiffies, deadline)) + wait_for_completion_timeout(&m->scan_done, msecs_to_jiffies(100)); + } mutex_lock(&m->command_mutex); spin_lock_irqsave(&m->response_lock, flags); if (m->stopping || m->link_failed || m->bss_active || m->connecting || m->connected || m->disconnecting || m->peer_valid || m->scan_request || m->retired_scan_seq || m->discovering) { + dev_dbg(&m->pdev->dev, + "address change refused: bss=%u connecting=%u connected=%u peer=%u scan=%u retired=%u\n", + m->bss_active, m->connecting, m->connected, m->peer_valid, + !!m->scan_request, m->retired_scan_seq); ret = -EBUSY; } else { ret = eth_mac_addr(netdev, address); From 86b28062a5131011581f230fb994e40c82eba938 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 04:02:25 +1100 Subject: [PATCH 0082/2680] PCI: apple: keep T8140 hosts without a bootstrap link root-port-only With the J700 bootstrap on by default, every T8140 host now looks up its "apple,piodma" engine. A host whose device tree has no such property fails its probe with -EINVAL instead of falling back to the root-port-only path. That is the opposite of the intent that the board device tree selects the bootstrap. Only look up a supplier when the property is present. A property that names a missing or wrong engine still fails the probe. Also stop logging "downstream config blocked" for the port of a host whose bootstrap is about to open it. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pci/controller/pcie-apple-piodma-diag.c | 5 +++-- drivers/pci/controller/pcie-apple.c | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index c0a85e04b6602..983b4c7ba706f 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -52,8 +52,9 @@ #define PIODMA_IRQ_LIMIT 16 /* - * The board DT opts in by pointing the host's "apple,piodma" at this engine. - * enumerate=0 keeps the host on its root-port-only path for diagnosis. + * The board DT opts in by pointing the host's "apple,piodma" at this engine; + * a host without that property stays root-port-only. enumerate=0 keeps every + * host on its root-port-only path for diagnosis. */ static bool enumerate = true; module_param(enumerate, bool, 0400); diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 6a8118ceee1aa..6f2f7a57d11ce 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -1613,10 +1613,10 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, if (pcie->hw->tunneled) WRITE_ONCE(port->started, true); if (pcie->hw->root_bus_only) - dev_info(pcie->dev, - "port %pOF RUN=%#x link=%#x; downstream config blocked\n", + dev_info(pcie->dev, "port %pOF RUN=%#x link=%#x; downstream config %s\n", np, apple_pcie_port_readl(port, PORT_STATUS), - apple_pcie_port_readl(port, PORT_LINKSTS)); + apple_pcie_port_readl(port, PORT_LINKSTS), + pcie->piodma_supplier ? "waits for the bootstrap" : "blocked"); return 0; @@ -2337,7 +2337,9 @@ static int apple_pcie_probe(struct platform_device *pdev) } } - if (hw->root_bus_only && apple_piodma_bootstrap_enabled()) { + /* Only a host that the board links to a bootstrap engine opens bus 1. */ + if (hw->root_bus_only && apple_piodma_bootstrap_enabled() && + of_property_present(dev->of_node, "apple,piodma")) { ret = apple_piodma_bootstrap_get(dev, &piodma_supplier); if (ret) return dev_err_probe(dev, ret, "PIODMA supplier unavailable\n"); From 9878ca42bdc6a1083e38406203ccb1a88af948bb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 04:02:43 +1100 Subject: [PATCH 0083/2680] PCI: apple: remove the J700 radio functions when publication fails If the root memory-window check or the cold check fails after the downstream scan, the host closes downstream configuration access but leaves both functions on the bus, scanned and unpublished. A later bus rescan would publish them and let their drivers probe through a closed config window, with memory decoding off. Remove the scanned functions, while still holding the rescan lock and before closing configuration access. A later rescan then finds nothing. Also check and quiesce both functions in the cold check instead of stopping at the first failure, so that a second function left bus mastering is cleared too. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pci/controller/pcie-apple.c | 30 +++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 6f2f7a57d11ce..2ad8b9c11ec90 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2236,18 +2236,21 @@ static int apple_pcie_neo_check_cold(struct apple_pcie *pcie, struct pci_dev *ro u16 command; int function, ret = 0; - for (function = 0; function < 2 && !ret; function++) { + /* Check and quiesce both functions; keep the first error. */ + for (function = 0; function < 2; function++) { endpoint = pci_get_slot(root->subordinate, PCI_DEVFN(0, function)); - if (!endpoint) - return -ENODEV; + if (!endpoint) { + ret = ret ?: -ENODEV; + continue; + } if (pci_read_config_word(endpoint, PCI_COMMAND, &command)) { - ret = -EIO; + ret = ret ?: -EIO; } else { dev_info(pcie->dev, "enumerated %s id=%04x:%04x command=%#x\n", pci_name(endpoint), endpoint->vendor, endpoint->device, command); if (command & PCI_COMMAND_MASTER) { pci_clear_master(endpoint); - ret = -EIO; + ret = ret ?: -EIO; } } pci_dev_put(endpoint); @@ -2255,6 +2258,19 @@ static int apple_pcie_neo_check_cold(struct apple_pcie *pcie, struct pci_dev *ro return ret; } +/* + * Rescan lock held, downstream config still open. Remove functions that were + * scanned but will not be published, so that a later rescan cannot publish + * them after config access has been closed again. + */ +static void apple_pcie_neo_unscan(struct pci_dev *root) +{ + struct pci_dev *dev, *tmp; + + list_for_each_entry_safe_reverse(dev, tmp, &root->subordinate->devices, bus_list) + pci_stop_and_remove_bus_device(dev); +} + static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) { struct apple_pcie *pcie = pci_host_bridge_priv(bridge); @@ -2295,8 +2311,10 @@ static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) goto close_config; pci_bus_add_devices(root->subordinate); close_config: - if (ret) + if (ret) { + apple_pcie_neo_unscan(root); WRITE_ONCE(pcie->neo_config_ready, false); + } pci_unlock_rescan_remove(); if (!ret) dev_info(pcie->dev, "radio functions 0/1 published\n"); From 77dffcdaa98a3c082da8deb6382604f305b49f5a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 04:03:41 +1100 Subject: [PATCH 0084/2680] Bluetooth: mt7932: wait for the Wi-Fi function before starting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bluetooth admission was qualified with the Wi-Fi function already running its firmware. The module soft dependency added for that only orders a modprobe of both modules. A built-in transport, insmod, asynchronous probing or a failed Wi-Fi probe all bypass it. Removal is affected too: "modprobe -r mt7932_bt_pcie" now also unloads the Wi-Fi driver, as kmod removes pre-softdeps after the module. Enforce the order in the driver instead. Defer the probe until mt7932-fullmac is bound to function 0, and add a device link so that the driver core unbinds Bluetooth before Wi-Fi. Drop the soft dependency. Bluetooth now stays unbound while the Wi-Fi driver is missing or failed, which matches the qualified sequence. Also warn when the link's power management cannot be disabled, and update the help text and documentation, which still called the transport opt-in. [ryan: adapted to 7.2.10 — documentation conflicts against the non-linear history of mt7932-neo.rst are resolved to this commit's text] Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 26 +++++++++------- drivers/bluetooth/Kconfig | 6 ++-- drivers/bluetooth/mt7932_bt_pcie.c | 31 +++++++++++++++++-- 3 files changed, 46 insertions(+), 17 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index ea2ebbc5d297e..20a208d1c3ec7 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -3,10 +3,9 @@ MT7932 radio bring-up on the MacBook Neo ====================================== -This contribution combines the existing J700 MT7932 fullmac Wi-Fi driver with an -opt-in PCIe Bluetooth transport on the public Aurora platform. Wi-Fi uses -cfg80211 and NetworkManager with firmware-managed WPA2-CCMP authentication. -It is not an mt76/mac80211 softmac implementation. +J700 MT7932 Wi-Fi uses cfg80211 and NetworkManager with firmware-managed +WPA2-CCMP authentication. Bluetooth uses a transport on the radio's second +PCIe function. The Wi-Fi import is pinned to ``aurora-silicon/linux-neo-cleanroom-eryk-with-wifi``, branch @@ -56,14 +55,17 @@ ASPM and clock power management on its link before starting the device, matching the tested admission sequence, which used the PCIe ASPM performance policy. -``mt7932_bt_pcie`` probes the cold ``14c3:793b`` function when it loads and -rejects a function that is not cold; ``enable=0`` leaves it unbound. Its -module soft dependency loads ``mt7932-fullmac`` first, so the Bluetooth -firmware starts after the Wi-Fi firmware, as in the tested sequence. The -transport also disables ASPM on the shared link. Like the Wi-Fi driver, it -reads its firmware while probing, so build it as a module. -Do not reprobe after a failed or uncertain Bluetooth admission. The tested -Wi-Fi driver owns function 0 and Bluetooth owns function 1. +``mt7932_bt_pcie`` defers its probe until ``mt7932-fullmac`` is bound to +function 0, so the Bluetooth firmware always starts after the Wi-Fi firmware, +as in the tested sequence, however the drivers are loaded. Bluetooth therefore +stays unbound while the Wi-Fi driver is missing or has failed. A device link +makes the driver core unbind Bluetooth before Wi-Fi; unloading +``mt7932_bt_pcie`` does not affect Wi-Fi. The probe rejects a function that is +not cold, which is what now prevents a reprobe after a failed or uncertain +admission; ``enable=0`` leaves the function unbound. The transport also +disables ASPM on the shared link. Like the Wi-Fi driver, it reads its firmware +while probing, so build it as a module. The tested Wi-Fi driver owns function +0 and Bluetooth owns function 1. NetworkManager supplies the existing saved WPA2 profile. Use a unique profile name or UUID and the current interface name; an older ``neo-wifi0`` profile diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index 70bd7af5a9bd0..df22362a3603c 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -30,8 +30,10 @@ config BT_MTK7932_PCIE select FW_LOADER help Support the J700 Bluetooth PCIe function with locally installed firmware - and calibration. The function is probed when the driver loads; the - enable=0 parameter leaves it unbound. + and calibration. The function is probed once the Wi-Fi function is + bound to the mt7932-fullmac driver; the enable=0 parameter leaves it + unbound. Both functions appear before the root filesystem is mounted + and read their firmware when probed, so say M, like the Wi-Fi driver. System suspend uses the firmware sleep handshake and preserves DMA storage. Hibernation and automatic firmware recovery are unsupported. Removal drains the PCI function before releasing device-visible memory; diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index a0d76eed77f66..981759997b991 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -1037,6 +1037,28 @@ static void bt7932_cleanup(struct bt7932 *bt, bool release_dma) kfree_sensitive(bt); } +/* + * Admission was only qualified with the Wi-Fi function already running its + * firmware. Wait until mt7932-fullmac is bound to function 0, then link the + * two functions so that the driver core unbinds this one first. + */ +static int bt7932_wait_for_wifi(struct pci_dev *pdev) +{ + struct pci_dev *wifi = pci_get_slot(pdev->bus, PCI_DEVFN(0, 0)); + int ret = 0; + + if (!wifi) + return -ENODEV; + if (!device_is_bound(&wifi->dev) || + strcmp(dev_driver_string(&wifi->dev), "mt7932-fullmac")) + ret = dev_err_probe(&pdev->dev, -EPROBE_DEFER, + "waiting for the Wi-Fi function\n"); + else if (!device_link_add(&pdev->dev, &wifi->dev, DL_FLAG_AUTOREMOVE_CONSUMER)) + ret = -EINVAL; + pci_dev_put(wifi); + return ret; +} + static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct bt7932 *bt; @@ -1051,8 +1073,13 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) !of_machine_is_compatible("apple,t8140") || pci_domain_nr(pdev->bus) || pdev->bus->number != 1 || pdev->devfn != PCI_DEVFN(0, 1)) return -ENODEV; + ret = bt7932_wait_for_wifi(pdev); + if (ret) + return ret; /* Admission was qualified with ASPM and clock PM off on this link. */ - pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); + ret = pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); + if (ret) + dev_warn(&pdev->dev, "link power management left as found: %d\n", ret); bt = kzalloc_obj(*bt); if (!bt) return -ENOMEM; @@ -1354,8 +1381,6 @@ module_pci_driver(bt7932_driver); MODULE_DESCRIPTION("Experimental standalone J700 MT7932 Bluetooth PCIe transport"); MODULE_LICENSE("GPL"); -/* Admission was qualified after the Wi-Fi function had started its firmware. */ -MODULE_SOFTDEP("pre: mt7932_fullmac"); MODULE_FIRMWARE(BT7932_FW_B0); MODULE_FIRMWARE(BT7932_FW_B1); MODULE_FIRMWARE(BT7932_FW_B1_FALLBACK); From 702d0e3de2349f5f16aa2312c6d7374f266356fe Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 04:03:48 +1100 Subject: [PATCH 0085/2680] wifi: mt7932: warn when link power management cannot be disabled pci_disable_link_state() fails when the OS has no ASPM control, for example with pcie_aspm=off. The link is then left as the firmware set it up, while station operation was qualified with it disabled. Report that instead of ignoring the error. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 0549ca8575dc1..85440558e95a4 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -1182,7 +1182,9 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (!domain || (domain->type != IOMMU_DOMAIN_DMA && domain->type != IOMMU_DOMAIN_DMA_FQ)) return dev_err_probe(&pdev->dev, -EINVAL, "translated DMA domain required\n"); /* Station operation was qualified with ASPM and clock PM off on this link. */ - pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); + ret = pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); + if (ret) + dev_warn(&pdev->dev, "link power management left as found: %d\n", ret); ret = pcim_enable_device(pdev); if (ret) return ret; From 900edbefaf8d5c2475827a4d2a1c30bbd6e0f972 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:03:09 +1100 Subject: [PATCH 0086/2680] wifi: mt7932: skip a country-package pass that a newer request superseded The startup work applies the country package to cfg80211's channel list before it checks whether the request it is serving is still current. If the country changes again while the work is loading the first package, cfg80211 has already recomputed the channel flags for the second country, and the stale pass then disables the first country's forbidden channels on top of them. The next pass adds the second country's own disables, so the union of both stays in place until the following regulatory change. The result is only ever more restrictive than intended, and the boot sequence cannot hit it because the world package forbids nothing, but a user who changes country twice in quick succession loses channels that the final country permits. The regulatory notifier runs under RTNL and starts a new generation there, after cfg80211 has recomputed the flags. Check the generation under RTNL in mt_policy_disable() and leave the channel list alone when it has moved on; the newer generation runs its own pass. Fixes: 6e3838e91a3b ("wifi: mt7932: honour the channel permissions of the country package") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/startup.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 9b54240339526..631a77aa684cd 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -227,11 +227,23 @@ static int mt_startup_once(struct mt7932 *m) * package leaves out of the firmware domain. cfg80211 recomputes the flags * on its next regulatory change, which runs this again. */ -static void mt_policy_disable(struct mt7932 *m, const struct mt7932_policy *policy) +static void mt_policy_disable(struct mt7932 *m, const struct mt7932_policy *policy, + u32 generation) { unsigned int i; rtnl_lock(); + /* cfg80211 recomputes the channel flags for a new country under RTNL + * and only then calls the notifier, which starts a new generation. + * Once that has happened this package belongs to an older request, + * and applying it would disable channels on top of the new country's + * flags until the next regulatory change. The newer generation runs + * its own pass. + */ + if (generation != READ_ONCE(m->reg_generation)) { + rtnl_unlock(); + return; + } wiphy_lock(m->wiphy); for (i = 0; i < MT7932_CHANNELS; i++) { struct ieee80211_channel *channel = i < MT7932_CHANNELS_2G ? @@ -301,7 +313,7 @@ static void mt_startup_work(struct work_struct *work) ret = mt7932_policy_parse(&policy, file->data, file->size, reg.domain); if (!ret) { mt7932_policy_filter(®, &policy); - mt_policy_disable(m, &policy); + mt_policy_disable(m, &policy, generation); } mutex_lock(&m->command_mutex); if (READ_ONCE(m->stopping) || generation != READ_ONCE(m->reg_generation)) From 274abe68fcd6d6c7c56f07f9ba1e0d46b35cc70f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:03:36 +1100 Subject: [PATCH 0087/2680] wifi: mt7932: bound the address-change wait to two seconds An address change on a stopped interface waits for a cancelled scan to retire before it checks whether the address can change. The change runs under RTNL, and the wait is documented as lasting at most two seconds, but it is longer in two cases: - mt_scan_quiesce() first waits up to five seconds for the scan to complete, and the two-second loop only starts after that, so a firmware that never retires the scan holds RTNL for about seven seconds; - once the driver is stopping, mt_stop_host() has completed scan_done for good, so every wait in the loop returns at once and the loop spins for two seconds. The interface is already down at this point, so its scan has been cancelled and only the retirement is left to wait for. Cancel any scan that is still recorded, then wait for the retirement against a single two-second deadline, and stop waiting as soon as the driver is stopping. The address change itself is unchanged: it is still refused with -EBUSY while a retirement is outstanding. Fixes: 22fc65ce6709 ("wifi: mt7932: let an address change wait for a retiring scan") Assisted-by: LLM Signed-off-by: Ryan Murray --- .../net/wireless/mediatek/mt7932/frontend.c | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 47d0c9e399ce6..50f424cdf3c03 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -137,6 +137,24 @@ int mt_scan_quiesce(struct mt7932 *m) return READ_ONCE(m->retired_scan_seq) ? -EBUSY : 0; } +/* Wait at most @timeout in total for a cancelled scan to retire. Callers + * may hold RTNL, so this never waits longer than asked. A stopping driver + * completes scan_done for good, which would turn the wait into a spin. + */ +static void mt_scan_retire_wait(struct mt7932 *m, unsigned long timeout) +{ + unsigned long deadline = jiffies + timeout; + long left; + + mt_abort_scan(m->wiphy, &m->wdev); + flush_work(&m->scan_finish_work); + while (READ_ONCE(m->retired_scan_seq) && !READ_ONCE(m->stopping)) { + left = (long)(deadline - jiffies); + if (left <= 0 || !wait_for_completion_timeout(&m->scan_done, left)) + break; + } +} + /* command_mutex held: one RF-band/type batch at a time, with matching EID0d * completion before the next batch. A request without SSIDs is passive; * active requests must also listen passively on NO_IR channels. @@ -450,13 +468,8 @@ static int mt_set_mac_address(struct net_device *netdev, void *address) * NetworkManager takes the interface down before changing the address, * which cancels any scan; let that scan retire instead of failing. */ - if (!netif_running(netdev)) { - unsigned long deadline = jiffies + msecs_to_jiffies(2000); - - mt_scan_quiesce(m); - while (READ_ONCE(m->retired_scan_seq) && time_before(jiffies, deadline)) - wait_for_completion_timeout(&m->scan_done, msecs_to_jiffies(100)); - } + if (!netif_running(netdev)) + mt_scan_retire_wait(m, msecs_to_jiffies(2000)); mutex_lock(&m->command_mutex); spin_lock_irqsave(&m->response_lock, flags); if (m->stopping || m->link_failed || m->bss_active || m->connecting || From 16e8998cac43f1e7c988a40ec743df7642afa8a2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:03:49 +1100 Subject: [PATCH 0088/2680] wifi: mt7932: name the forbidden-channel marker after what it means A country package marks a channel the country does not permit by setting every per-rate limit of the channel to 0xc4. The constant for that byte is called MT7932_POLICY_NO_LIMIT, which reads as "unrestricted", the opposite of its meaning. Rename it to MT7932_POLICY_NOT_PERMITTED. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/regulatory.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory.h b/drivers/net/wireless/mediatek/mt7932/regulatory.h index a49885cdb9788..fd65e07d9506a 100644 --- a/drivers/net/wireless/mediatek/mt7932/regulatory.h +++ b/drivers/net/wireless/mediatek/mt7932/regulatory.h @@ -104,9 +104,9 @@ static inline int mt7932_policy_parse(struct mt7932_policy *out, /* Every entry of a validated package starts with its channel number and is * followed by per-rate power limits. A country that does not permit a - * channel leaves all of that channel's limits at this byte. + * channel sets all of that channel's limits to this marker. */ -#define MT7932_POLICY_NO_LIMIT 0xc4 +#define MT7932_POLICY_NOT_PERMITTED 0xc4 static inline bool mt7932_policy_permits(const struct mt7932_policy *policy, unsigned int channel) @@ -122,7 +122,7 @@ static inline bool mt7932_policy_permits(const struct mt7932_policy *policy, if (entry[0] != channel) continue; for (k = 1; k < 122; k++) - if (entry[k] != MT7932_POLICY_NO_LIMIT) + if (entry[k] != MT7932_POLICY_NOT_PERMITTED) return true; return false; } From f08bcafe61ed2e2754e3c32e13692d119182f2fa Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:05:50 +1100 Subject: [PATCH 0089/2680] Bluetooth: mt7932: depend on the Wi-Fi driver The Bluetooth transport defers its probe until the Wi-Fi function is bound to mt7932-fullmac. Without that driver it can never bind, yet it can still be selected with MT7932_FULLMAC=n and then sits in the deferred probe list for good. Make the option depend on the Wi-Fi driver being enabled. Use "!= n" rather than a plain dependency so that the transport can still be built in while the Wi-Fi driver is a module; the deferred probe handles that order. Fixes: 920e89a2e3df ("Bluetooth: mt7932: wait for the Wi-Fi function before starting") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index df22362a3603c..5f49313fefe2a 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -27,6 +27,7 @@ config BT_MTK7932_PCIE tristate "J700 MT7932 PCIe Bluetooth transport" depends on BT && PCI && ARCH_APPLE && IOMMU_DMA depends on !KEXEC_CORE + depends on MT7932_FULLMAC != n select FW_LOADER help Support the J700 Bluetooth PCIe function with locally installed firmware From a6bb1981f909ccf960067698b99a8aac2b76b291 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 01:29:48 +1100 Subject: [PATCH 0090/2680] wifi: mt7932: keep the function running across suspend-to-idle The driver has no dev_pm_ops, so the PCI core's default system-suspend callback clears bus mastering on the function while its firmware keeps running. On the J700 a pm_test=devices cycle then hangs the SoC until the watchdog resets it, every time; with the function removed, the same cycle resumes. Stopping the firmware for sleep would take the remove and probe sequence on every suspend: retiring its DMA, a function-level reset and a new firmware start. Leave it running across suspend-to-idle instead. Saving the PCI state in .suspend keeps the PCI core from changing the function's power state or command register, and putting the device on the awake path keeps the PCIe controller's power state on until resume. Refuse any other sleep state. With this change, the pm_test=devices cycle, a one-minute pm_test=platform cycle and real 30 and 90 second suspend-to-idle cycles all resume with the station still associated and passing traffic, and genpd records no power-off of the controller's power state. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.c | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 85440558e95a4..4ca56767f671f 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -5,6 +5,9 @@ */ #include "mt7932.h" +#include +#include + static int mt_ring_alloc(struct mt7932 *m, struct mt7932_ring *q, unsigned int n, bool rx, unsigned int count); static void mt_rx_post(struct mt7932_ring *q, unsigned int index); @@ -1279,6 +1282,25 @@ static void mt_shutdown(struct pci_dev *pdev) /* Storage remains allocated through reset, even if the engine is stuck. */ } +/* + * Stopping the firmware for sleep would take the whole remove and probe + * sequence, so leave it running across suspend-to-idle. A saved state keeps + * the PCI core from changing the function's power state or command register, + * and the awake path keeps the PCIe controller powered until resume. + */ +static int mt_suspend(struct device *dev) +{ + if (!pm_suspend_no_platform()) + return -EOPNOTSUPP; + pci_save_state(to_pci_dev(dev)); + device_set_awake_path(dev); + return 0; +} + +static const struct dev_pm_ops mt_pm_ops = { + .suspend = mt_suspend, +}; + static const struct pci_device_id mt_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x7932) }, {} @@ -1292,6 +1314,7 @@ static struct pci_driver mt_driver = { .remove = mt_remove, .shutdown = mt_shutdown, .driver.suppress_bind_attrs = true, + .driver.pm = pm_sleep_ptr(&mt_pm_ops), }; module_pci_driver(mt_driver); MODULE_LICENSE("GPL"); From e1de22ad53b7247b0d44780aeffa6eb3e895d471 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:23:20 +1100 Subject: [PATCH 0091/2680] wifi: mt7932: report whether the Wi-Fi function initialized MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When initialization fails and the DMA of function 0 cannot be stopped, mt7932-fullmac keeps the binding to retain that DMA. Whether the Wi-Fi firmware is running therefore cannot be told from the binding, which is what the Bluetooth transport on function 1 waits for before it starts its own firmware on the shared radio. Record when the probe has finished initialization and export mt7932_fullmac_ready() to report it. [ryan: adapted to 7.2.10 — split by topic: the MAINTAINERS lines are carried by 900-config-tooling] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.c | 24 +++++++++++++++++++ drivers/net/wireless/mediatek/mt7932/mt7932.h | 2 ++ include/linux/mt7932.h | 14 +++++++++++ 3 files changed, 40 insertions(+) create mode 100644 include/linux/mt7932.h diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 4ca56767f671f..a10e4d2010a10 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -3,6 +3,7 @@ * Clean-room contract: NEO_MT7932_DOWNLOAD_AND_NIC_CAP_CONTRACT.md. * All addresses here are BAR0 offsets, never chip/CPU physical addresses. */ +#include #include "mt7932.h" #include @@ -1254,6 +1255,8 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = mt_reset_retained(&m); if (!ret) ret = mt_register_regulatory_gate(m); + if (!ret) + WRITE_ONCE(m->ready, true); retain: if (ret) { int error = ret; @@ -1316,6 +1319,27 @@ static struct pci_driver mt_driver = { .driver.suppress_bind_attrs = true, .driver.pm = pm_sleep_ptr(&mt_pm_ops), }; + +/** + * mt7932_fullmac_ready() - check that the Wi-Fi function is up + * @pdev: function 0 of the radio + * + * A Wi-Fi function whose initialization failed stays bound when its DMA + * could not be stopped, so being bound does not mean its firmware runs. + * + * Return: true if @pdev is bound to this driver and finished initialization. + */ +bool mt7932_fullmac_ready(struct pci_dev *pdev) +{ + struct mt7932 *m; + + if (!device_is_bound(&pdev->dev) || pdev->dev.driver != &mt_driver.driver) + return false; + m = pci_get_drvdata(pdev); + return m && READ_ONCE(m->ready); +} +EXPORT_SYMBOL_GPL(mt7932_fullmac_ready); + module_pci_driver(mt_driver); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("J700 MT7932 firmware-owned cfg80211 station driver"); diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index f6fc1f235e619..93e7fa5629acb 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -76,6 +76,8 @@ struct mt7932 { u64 interrupts, packets; bool running; bool stopping; + /* Probe finished initialization; the Bluetooth function may start. */ + bool ready; bool irq_requested, vectors_allocated, wiphy_registered; bool dma_owned; struct completion reset_retry; diff --git a/include/linux/mt7932.h b/include/linux/mt7932.h new file mode 100644 index 0000000000000..9ab60efa49786 --- /dev/null +++ b/include/linux/mt7932.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Interface between the functions of the J700 MT7932 radio. + */ +#ifndef __LINUX_MT7932_H +#define __LINUX_MT7932_H + +#include + +struct pci_dev; + +bool mt7932_fullmac_ready(struct pci_dev *pdev); + +#endif /* __LINUX_MT7932_H */ From 3d3f81081a9571fd3ac1d114e2b2b07bf055da8e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:24:23 +1100 Subject: [PATCH 0092/2680] Bluetooth: mt7932: start only after the Wi-Fi function initialized The Bluetooth transport waits until function 0 is bound to mt7932-fullmac, so that its firmware starts after the Wi-Fi firmware as in the tested sequence. Being bound does not mean the Wi-Fi firmware is running: when the Wi-Fi initialization fails and its DMA cannot be stopped, mt7932-fullmac keeps the binding to retain that DMA. Bluetooth then probes anyway and starts its firmware on the shared radio next to a failed Wi-Fi function. Once function 0 is bound, also require mt7932_fullmac_ready(). The Wi-Fi probe finishes initialization before it returns, so a bound function that is not ready has failed, and the Bluetooth probe fails with -ENODEV. The transport now calls into the Wi-Fi module, so it depends on MT7932_FULLMAC plainly: it can no longer be built in while the Wi-Fi driver is a module. Fixes: 920e89a2e3df ("Bluetooth: mt7932: wait for the Wi-Fi function before starting") Assisted-by: LLM Signed-off-by: Ryan Murray --- .../networking/device_drivers/wifi/mt7932-neo.rst | 3 ++- drivers/bluetooth/Kconfig | 11 ++++++----- drivers/bluetooth/mt7932_bt_pcie.c | 5 +++++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 20a208d1c3ec7..0fa20eabe51cc 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -58,7 +58,8 @@ performance policy. ``mt7932_bt_pcie`` defers its probe until ``mt7932-fullmac`` is bound to function 0, so the Bluetooth firmware always starts after the Wi-Fi firmware, as in the tested sequence, however the drivers are loaded. Bluetooth therefore -stays unbound while the Wi-Fi driver is missing or has failed. A device link +stays unbound while the Wi-Fi driver is missing or has failed, including when +a failed Wi-Fi function stays bound to keep DMA it could not stop. A device link makes the driver core unbind Bluetooth before Wi-Fi; unloading ``mt7932_bt_pcie`` does not affect Wi-Fi. The probe rejects a function that is not cold, which is what now prevents a reprobe after a failed or uncertain diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index 5f49313fefe2a..ab57248b9de8c 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -27,14 +27,15 @@ config BT_MTK7932_PCIE tristate "J700 MT7932 PCIe Bluetooth transport" depends on BT && PCI && ARCH_APPLE && IOMMU_DMA depends on !KEXEC_CORE - depends on MT7932_FULLMAC != n + depends on MT7932_FULLMAC select FW_LOADER help Support the J700 Bluetooth PCIe function with locally installed firmware - and calibration. The function is probed once the Wi-Fi function is - bound to the mt7932-fullmac driver; the enable=0 parameter leaves it - unbound. Both functions appear before the root filesystem is mounted - and read their firmware when probed, so say M, like the Wi-Fi driver. + and calibration. The function is probed once the Wi-Fi function has + initialized under the mt7932-fullmac driver; the enable=0 parameter + leaves it unbound. Both functions appear before the root filesystem + is mounted and read their firmware when probed, so say M, like the + Wi-Fi driver. System suspend uses the firmware sleep handshake and preserves DMA storage. Hibernation and automatic firmware recovery are unsupported. Removal drains the PCI function before releasing device-visible memory; diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 981759997b991..99f063b141a0e 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -1053,6 +1054,10 @@ static int bt7932_wait_for_wifi(struct pci_dev *pdev) strcmp(dev_driver_string(&wifi->dev), "mt7932-fullmac")) ret = dev_err_probe(&pdev->dev, -EPROBE_DEFER, "waiting for the Wi-Fi function\n"); + /* Initialization ends before the Wi-Fi probe returns. */ + else if (!mt7932_fullmac_ready(wifi)) + ret = dev_err_probe(&pdev->dev, -ENODEV, + "the Wi-Fi function failed to initialize\n"); else if (!device_link_add(&pdev->dev, &wifi->dev, DL_FLAG_AUTOREMOVE_CONSUMER)) ret = -EINVAL; pci_dev_put(wifi); From c6036c524356f6b8c7b0932b733a38178b31e18f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:27:33 +1100 Subject: [PATCH 0093/2680] wifi: mt7932: keep the country package's channel disables The channels a country package leaves out of the firmware domain are disabled only in cfg80211's channel flags, once per package pass. cfg80211 rebuilds those flags from the original ones on every regulatory update it applies, and then calls the notifier. The notifier skips an update whose snapshot matches the request already applied, so if an update ever leaves the snapshot unchanged, no pass runs and the disables are gone: the driver again offers channels its firmware domain excludes, where scans and joins fail. Do not depend on every such update producing a new snapshot. Record the channels each package pass disables and apply them again when the notifier finds the package still current. A new request clears the set; its own pass records a new one. Fixes: 6e3838e91a3b ("wifi: mt7932: honour the channel permissions of the country package") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.h | 2 ++ .../net/wireless/mediatek/mt7932/startup.c | 29 ++++++++++++++----- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 93e7fa5629acb..261199230ac79 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -107,6 +107,8 @@ struct mt7932 { bool interface_registered, interface_up; u32 reg_generation, reg_attempted; struct mt7932_reg_snapshot reg_desired; + /* Channels the applied country package leaves out; under RTNL. */ + DECLARE_BITMAP(policy_disabled, MT7932_CHANNELS); struct net_device *netdev; struct wireless_dev wdev; struct cfg80211_scan_request *scan_request; diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 631a77aa684cd..225e5a0344fc4 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -223,9 +223,16 @@ static int mt_startup_once(struct mt7932 *m) return ret; } +static struct ieee80211_channel *mt_channel(struct mt7932 *m, unsigned int i) +{ + return i < MT7932_CHANNELS_2G ? &m->channels[i] : + &m->channels5[i - MT7932_CHANNELS_2G]; +} + /* Keep cfg80211's channel list in step with the channels the country - * package leaves out of the firmware domain. cfg80211 recomputes the flags - * on its next regulatory change, which runs this again. + * package leaves out of the firmware domain. cfg80211 rebuilds the flags + * on every regulatory update; the notifier applies the recorded set again + * when the update keeps this package, or starts a new pass otherwise. */ static void mt_policy_disable(struct mt7932 *m, const struct mt7932_policy *policy, u32 generation) @@ -246,11 +253,11 @@ static void mt_policy_disable(struct mt7932 *m, const struct mt7932_policy *poli } wiphy_lock(m->wiphy); for (i = 0; i < MT7932_CHANNELS; i++) { - struct ieee80211_channel *channel = i < MT7932_CHANNELS_2G ? - &m->channels[i] : &m->channels5[i - MT7932_CHANNELS_2G]; + struct ieee80211_channel *channel = mt_channel(m, i); + bool forbidden = !mt7932_policy_permits(policy, channel->hw_value); - if (!(channel->flags & IEEE80211_CHAN_DISABLED) && - !mt7932_policy_permits(policy, channel->hw_value)) + __assign_bit(i, m->policy_disabled, forbidden); + if (forbidden) channel->flags |= IEEE80211_CHAN_DISABLED; } wiphy_unlock(m->wiphy); @@ -375,8 +382,7 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request reg.domain[4] = 1; /* 2 GHz remains 20 MHz. */ reg.domain[5] = mt7932_domain_5g_bw(); for (i = 0; i < MT7932_CHANNELS; i++) { - struct ieee80211_channel *channel = i < MT7932_CHANNELS_2G ? - &m->channels[i] : &m->channels5[i - MT7932_CHANNELS_2G]; + struct ieee80211_channel *channel = mt_channel(m, i); u32 flags = channel->flags; reg.power[i] = channel->max_power; @@ -416,10 +422,17 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request } if (!memcmp(®, &m->reg_desired, sizeof(reg)) && !m->reg_pending) { spin_unlock_irqrestore(&m->response_lock, irqflags); + /* The applied package still holds, but cfg80211 has just + * rebuilt the channel flags without its disables. + */ + for_each_set_bit(i, m->policy_disabled, MT7932_CHANNELS) + mt_channel(m, i)->flags |= IEEE80211_CHAN_DISABLED; return; } m->reg_desired = reg; m->reg_generation++; + /* The next package pass records its own set. */ + bitmap_zero(m->policy_disabled, MT7932_CHANNELS); WRITE_ONCE(m->reg_pending, true); if (m->netdev) { netif_stop_queue(m->netdev); From 8cf19f2e0e65fab2062187b9f66c2b42de9057ba Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 14:36:32 +1100 Subject: [PATCH 0094/2680] Bluetooth: mt7932: don't cancel HCI commands at suspend entry Every system suspend logs Bluetooth: hci0: Opcode 0x0c52 failed: -112 Bluetooth: hci0: unexpected event for opcode 0x0c1a and sometimes the same for 0x0c24. These are Write EIR and Write Class of Device commands that bluetoothd issues at suspend entry: when the session goes inactive, the sound server unregisters its Bluetooth profiles, and removing their UUIDs rewrites the class and EIR. hci_suspend_dev() cancels a command that is still waiting for its completion with -EHOSTDOWN, and the controller's late completion then no longer matches the command that was sent next. The transport calls hci_suspend_dev() twice: the HCI PM notifier calls it at PM_SUSPEND_PREPARE, before tasks are frozen and while bluetoothd is still issuing these commands, and bt7932_suspend() calls it again, which cancels whatever was queued in between. Suspend HCI only from bt7932_suspend(), as the transport's sleep handshake requires anyway, by setting HCI_QUIRK_NO_SUSPEND_NOTIFIER. By then userspace is frozen, so first let the HCI work that it queued finish: queue an empty hci_cmd_sync entry and wait for it, bounded by the HCI command timeout. Leave a user channel device alone, as the notifier does. Fixes: 8896264380c3 ("Bluetooth: mt7932: support modules and system suspend") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/mt7932_bt_pcie.c | 51 +++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 99f063b141a0e..8923e01220f9c 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -8,6 +8,7 @@ * transport arena and uses the firmware quiesce/restore handshake. Unload * releases DMA only after the PCI function has stopped bus mastering. */ +#include #include #include #include @@ -112,6 +113,7 @@ struct bt7932 { struct work_struct tx_work; struct sk_buff_head tx_queue; wait_queue_head_t wake_wait; + struct completion hci_idle; struct bt7932_ring rings[BT7932_CHANNELS]; void *arena; dma_addr_t arena_dma; @@ -1092,6 +1094,7 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) bt->last_sleep = U32_MAX; mutex_init(&bt->lock); init_waitqueue_head(&bt->wake_wait); + init_completion(&bt->hci_idle); skb_queue_head_init(&bt->tx_queue); INIT_WORK(&bt->tx_work, bt7932_tx_work); msleep(100); @@ -1167,6 +1170,8 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) bt->hdev->hw_error = bt7932_hw_error; hci_set_quirk(bt->hdev, HCI_QUIRK_RESET_ON_CLOSE); hci_set_quirk(bt->hdev, HCI_QUIRK_NON_PERSISTENT_SETUP); + /* System sleep suspends HCI from bt7932_suspend(), after the freeze. */ + hci_set_quirk(bt->hdev, HCI_QUIRK_NO_SUSPEND_NOTIFIER); ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI); if (ret < 0) goto fail; @@ -1272,6 +1277,38 @@ static int bt7932_restore_transport(struct bt7932 *bt) return ret; } +static int bt7932_hci_idle(struct hci_dev *hdev, void *data) +{ + return 0; +} + +static void bt7932_hci_idle_done(struct hci_dev *hdev, void *data, int err) +{ + complete(data); +} + +/* + * hci_suspend_dev() cancels a command that is still waiting for its + * completion. Userspace can queue HCI work right up to the freeze (sound + * servers unregister their profiles when the session goes inactive, which + * rewrites the class and EIR), so let that work finish first. + */ +static void bt7932_wait_hci_idle(struct bt7932 *bt) +{ + reinit_completion(&bt->hci_idle); + if (hci_cmd_sync_queue(bt->hdev, bt7932_hci_idle, &bt->hci_idle, + bt7932_hci_idle_done)) + return; + if (!wait_for_completion_timeout(&bt->hci_idle, HCI_CMD_TIMEOUT)) + dev_warn(&bt->pdev->dev, "HCI work still pending at suspend\n"); +} + +/* Like the HCI PM notifier, leave a user channel device to userspace. */ +static bool bt7932_hci_pm(struct bt7932 *bt) +{ + return !hci_dev_test_flag(bt->hdev, HCI_USER_CHANNEL); +} + static int bt7932_suspend(struct device *dev) { struct bt7932 *bt = pci_get_drvdata(to_pci_dev(dev)); @@ -1281,9 +1318,12 @@ static int bt7932_suspend(struct device *dev) return -EIO; if (!bt->registered) return 0; - ret = hci_suspend_dev(bt->hdev); - if (ret) - return ret; + if (bt7932_hci_pm(bt)) { + bt7932_wait_hci_idle(bt); + ret = hci_suspend_dev(bt->hdev); + if (ret) + return ret; + } ret = bt7932_quiesce(bt); if (!ret) { dev_dbg(dev, "system sleep quiesce acknowledged\n"); @@ -1293,7 +1333,8 @@ static int bt7932_suspend(struct device *dev) /* Undo our request before the PM core rolls back the rest of the bus. */ if (!READ_ONCE(bt->fault)) bt7932_restore_transport(bt); - hci_resume_dev(bt->hdev); + if (bt7932_hci_pm(bt)) + hci_resume_dev(bt->hdev); return ret; } @@ -1312,7 +1353,7 @@ static int bt7932_resume(struct device *dev) return ret; } dev_dbg(dev, "transport restored after system sleep\n"); - return hci_resume_dev(bt->hdev); + return bt7932_hci_pm(bt) ? hci_resume_dev(bt->hdev) : 0; } static int bt7932_freeze(struct device *dev) From 3d14c22093100bcbc0439602828caf1e4da6cb28 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 22:12:20 +1100 Subject: [PATCH 0095/2680] treewide: describe hardware behaviour in comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrite a set of comments so they state the hardware and firmware behaviour directly: - soc/apple SEP (sks.rs, image.rs, profile.rs, sep.rs): key-store request layouts, timeouts and endpoint init described as the enclave protocol. - soc/apple mailbox: the ASC wrapper CPU start/stop sequences. - accel/ane: register, handshake, ring and command-format notes for the T602x/T8112 Neural Engine, plus the scratch3_ack parameter text. - drm/asahi G17: the tiler-heap shrink policy. - mt7932, reset-apple-cio and two documentation pages: keep the descriptions self-contained. Every edit keeps its line count, so line-dependent data (panic locations, WARN/BUG and dynamic-debug records) is unchanged. Apart from the scratch3_ack module parameter description, the change is comments only; there is no functional change. [ryan: adapted to 7.2.10 — split by topic: only the Documentation/networking/device_drivers/wifi/mt7932-neo.rst hunks of this ane commit; the rest is in 245-ane; documentation conflicts against the non-linear history of mt7932-neo.rst are resolved to this commit's text] Assisted-by: LLM Signed-off-by: Ryan Murray --- .../networking/device_drivers/wifi/mt7932-neo.rst | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 0fa20eabe51cc..bb382cab2c99a 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -7,16 +7,9 @@ J700 MT7932 Wi-Fi uses cfg80211 and NetworkManager with firmware-managed WPA2-CCMP authentication. Bluetooth uses a transport on the radio's second PCIe function. -The Wi-Fi import is pinned to -``aurora-silicon/linux-neo-cleanroom-eryk-with-wifi``, branch -``wifi/width-qualification-20260912``, commit -``610cb463c03f9bc68a5d020e7ca443af1fda4856``. The import commit preserves all -26 driver files byte-for-byte. A separate commit carries the three tested -compatibility changes: bounded 64/65-record configuration parsing, native scan -IE encoding, and selected WPA2 PSK admission with trailing AKM alternatives. -This contribution imports an existing implementation; it does not claim an -independent rewrite. Existing source notices are retained. Co-authored by -DJ (DjDeveloperr), Ace (Acelogic), and Ryan Murray. +The Wi-Fi driver carries configuration, scan IE and WPA2 AKM compatibility +fixes on top of its original import. Existing source notices are retained. +Co-authored by DJ (DjDeveloperr), Ace (Acelogic), and Ryan Murray. Platform and configuration -------------------------- From 6c77f9a066638becb0a3b675ddc65dd681aa71e4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:25:16 +1100 Subject: [PATCH 0096/2680] wifi: mt7932: report the receive bitrate on 40 and 80 MHz channels The station's receive rate is discarded whenever the RX vector's bandwidth field is not 20 MHz. Associations on 5 GHz use 40 and 80 MHz channels where the access point offers them, and most frames on those channels are received at the full width, so "iw dev wlan0 station dump" shows no rx bitrate on most 5 GHz networks. mt7932_rx_rate() already decodes the bandwidth field into rate_info.bw, and cfg80211 computes the bitrate for every width it produces. Keep the rate unless it is wider than the channel the association uses, which a frame from the associated access point cannot be, and keep refusing a CCK rate on 5 GHz. Fixes: 8709ef5c7d40 ("wifi: add existing J700 MT7932 fullmac implementation") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/data.c | 28 +++++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index 25a7bcbf31238..6008d3fab33ac 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -164,6 +164,23 @@ netdev_tx_t mt_net_xmit(struct sk_buff *skb, struct net_device *netdev) return NETDEV_TX_OK; } +/* Width in MHz of a rate decoded by mt7932_rx_rate(). */ +static int mt_rate_width(enum rate_info_bw bw) +{ + switch (bw) { + case RATE_INFO_BW_20: + return 20; + case RATE_INFO_BW_40: + return 40; + case RATE_INFO_BW_80: + return 80; + case RATE_INFO_BW_160: + return 160; + default: + return INT_MAX; + } +} + void mt_data_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame) { struct sk_buff *skb; @@ -202,11 +219,12 @@ void mt_data_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame) m->station_signal_valid = true; } if (frame->unicast && frame->rate_valid) { - m->station_rx_rate_valid = mt7932_rx_rate(frame->rxv, m->phy_cap[4], - &m->station_rx_rate); - /* Current association contract admits only 20 MHz and no 5 GHz CCK. */ - if (((frame->rxv >> 12) & 7) || - (frame->channel > 14 && !((frame->rxv >> 24) & 15))) + struct rate_info *rate = &m->station_rx_rate; + + m->station_rx_rate_valid = mt7932_rx_rate(frame->rxv, m->phy_cap[4], rate); + /* No CCK on 5 GHz, and no rate wider than the associated channel. */ + if ((frame->channel > 14 && !((frame->rxv >> 24) & 15)) || + mt_rate_width(rate->bw) > cfg80211_chandef_get_width(&m->connect_chandef)) m->station_rx_rate_valid = false; m->station_rx_rate_time = jiffies; } From 5d80f557ac12476b1a2666bcd205faac25be271c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:29:01 +1100 Subject: [PATCH 0097/2680] wifi: mt7932: monitor the connection's signal against a threshold The driver does not implement set_cqm_rssi_config, so userspace cannot ask to be told when the signal crosses a threshold. wpa_supplicant's "simple" background scan module, which NetworkManager enables, then picks its scan interval from one signal reading at association and never revisits it: when that reading is below the threshold it keeps scanning at the short interval, 30 seconds in NetworkManager's configuration, for as long as the long interval lasts. Each scan takes the station off channel for about a second, which shows up as a latency spike every 31 seconds or so. iwd uses the same events to decide when to look for a better access point. The firmware reports no signal threshold events, so run the monitor on the host, the way mac80211 runs it for drivers without one: average the signal of the frames the associated access point sends, both its beacons and its data, and report a crossing of the threshold through cfg80211_cqm_rssi_notify(). Take at most one sample per 100 ms, about a beacon interval, so that the average follows time rather than traffic, and wait for four samples before the first event. As in mac80211, the first event after a new association or configuration reports the side of the threshold the signal is on, and a later event needs the average to be across the threshold and more than the hysteresis away from the level last reported. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/data.c | 1 + .../net/wireless/mediatek/mt7932/frontend.c | 11 ++-- drivers/net/wireless/mediatek/mt7932/link.c | 1 + drivers/net/wireless/mediatek/mt7932/mt7932.h | 13 ++++ .../net/wireless/mediatek/mt7932/startup.c | 1 + .../net/wireless/mediatek/mt7932/station.c | 66 +++++++++++++++++++ 6 files changed, 89 insertions(+), 4 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index 6008d3fab33ac..7a8b5f925b52d 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -217,6 +217,7 @@ void mt_data_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame) if (frame->signal_valid && signal > -128) { m->station_signal = signal; m->station_signal_valid = true; + mt_cqm_signal(m, signal); } if (frame->unicast && frame->rate_valid) { struct rate_info *rate = &m->station_rx_rate; diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 50f424cdf3c03..282404beac17b 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -631,11 +631,14 @@ void mt_packet_receive(struct mt7932 *m, const u8 *packet, size_t length) unsigned long flags; spin_lock_irqsave(&m->response_lock, flags); - if (tim && tim->datalen >= 4 && m->connected && !m->disconnecting && - !m->stopping && !m->power_tim && frame.channel == m->connect_channel && + if (m->connected && !m->disconnecting && !m->stopping && + frame.channel == m->connect_channel && ether_addr_equal(frame.data + 16, m->connect_bssid)) { - m->power_tim = true; - schedule_work(&m->power_work); + mt_cqm_signal(m, mt7932_rx_signal(&frame, m->antenna_mask)); + if (tim && tim->datalen >= 4 && !m->power_tim) { + m->power_tim = true; + schedule_work(&m->power_work); + } } spin_unlock_irqrestore(&m->response_lock, flags); } diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index fc2d333519f98..6869e7ff85c8a 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -471,6 +471,7 @@ void mt_connect_work(struct work_struct *work) m->station_rx_rate_valid = false; m->station_tx_rate_valid = false; spin_unlock(&m->data_lock); + mt_cqm_reset(m); } m->connected = !ret; m->disconnecting = false; diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 261199230ac79..fa312ecca622c 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -3,6 +3,7 @@ #define MT7932_H +#include #include #include #include @@ -40,6 +41,8 @@ #define STRIDE 8192 #define RX_STRIDE ALIGN(MT7932_RX_CAPACITY, 64) +DECLARE_EWMA(mt7932_signal, 4, 4) + struct mt7932; int mt_dma_stop(struct mt7932 *m); int mt_function_reset(struct mt7932 *m); @@ -156,6 +159,12 @@ struct mt7932 { struct rate_info station_tx_rate; unsigned long station_tx_rate_time, station_rate_query_time; bool station_tx_rate_valid, station_rate_query_failed; + /* Connection quality monitor, protected by response_lock. */ + struct ewma_mt7932_signal cqm_signal; + unsigned long cqm_sample_time; + s32 cqm_threshold, cqm_last_event; + u32 cqm_hysteresis; + u8 cqm_samples; u8 *data_headers, *data_payloads; dma_addr_t data_headers_dma, data_payloads_dma; bool data_live[512], data_token[512], data_retired[512]; @@ -219,6 +228,10 @@ int mt_get_station(struct wiphy *wiphy, struct wireless_dev *wdev, const u8 *mac, struct station_info *sinfo); int mt_dump_station(struct wiphy *wiphy, struct wireless_dev *wdev, int idx, u8 *mac, struct station_info *sinfo); +int mt_set_cqm_rssi_config(struct wiphy *wiphy, struct net_device *netdev, + s32 rssi_thold, u32 rssi_hyst); +void mt_cqm_reset(struct mt7932 *m); +void mt_cqm_signal(struct mt7932 *m, int signal); int mt_set_power_mgmt(struct wiphy *wiphy, struct net_device *netdev, bool enabled, int timeout); void mt_power_work(struct work_struct *work); diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 225e5a0344fc4..696b79e48ddfe 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -459,6 +459,7 @@ static const struct cfg80211_ops mt_cfg_ops = { .get_channel = mt_get_channel, .dump_station = mt_dump_station, .set_power_mgmt = mt_set_power_mgmt, + .set_cqm_rssi_config = mt_set_cqm_rssi_config, }; int mt_register_regulatory_gate(struct mt7932 *m) diff --git a/drivers/net/wireless/mediatek/mt7932/station.c b/drivers/net/wireless/mediatek/mt7932/station.c index c9d388e50d8d5..83cee4b650926 100644 --- a/drivers/net/wireless/mediatek/mt7932/station.c +++ b/drivers/net/wireless/mediatek/mt7932/station.c @@ -103,3 +103,69 @@ int mt_dump_station(struct wiphy *wiphy, struct wireless_dev *wdev, spin_unlock_irqrestore(&m->response_lock, flags); return ret; } + +/* + * The firmware reports no signal threshold events, so the connection quality + * monitor runs on the host, as mac80211 runs it for drivers without one: + * average the signal of the frames the associated access point sends and + * report when the average crosses the threshold. Sample at most once per + * beacon interval, so that the average follows time rather than traffic. + */ +#define MT7932_CQM_INTERVAL (HZ / 10) +#define MT7932_CQM_MIN_SAMPLES 4 + +/* Caller holds response_lock; a new association starts a new average. */ +void mt_cqm_reset(struct mt7932 *m) +{ + ewma_mt7932_signal_init(&m->cqm_signal); + m->cqm_samples = 0; + m->cqm_last_event = 0; +} + +/* Caller holds response_lock; @signal is a frame from the associated AP. */ +void mt_cqm_signal(struct mt7932 *m, int signal) +{ + enum nl80211_cqm_rssi_threshold_event event; + int average, last = m->cqm_last_event; + int hysteresis = m->cqm_hysteresis; + + if (signal >= 0 || signal <= -128) + return; + if (m->cqm_samples && + time_before(jiffies, m->cqm_sample_time + MT7932_CQM_INTERVAL)) + return; + m->cqm_sample_time = jiffies; + ewma_mt7932_signal_add(&m->cqm_signal, -signal); + if (m->cqm_samples < MT7932_CQM_MIN_SAMPLES) + m->cqm_samples++; + if (!m->cqm_threshold || m->cqm_samples < MT7932_CQM_MIN_SAMPLES) + return; + + /* The first event after (re)configuration reports the current side. */ + average = -(int)ewma_mt7932_signal_read(&m->cqm_signal); + if (average < m->cqm_threshold && (!last || average < last - hysteresis)) + event = NL80211_CQM_RSSI_THRESHOLD_EVENT_LOW; + else if (average > m->cqm_threshold && (!last || average > last + hysteresis)) + event = NL80211_CQM_RSSI_THRESHOLD_EVENT_HIGH; + else + return; + m->cqm_last_event = average; + cfg80211_cqm_rssi_notify(m->netdev, event, average, GFP_ATOMIC); +} + +int mt_set_cqm_rssi_config(struct wiphy *wiphy, struct net_device *netdev, + s32 rssi_thold, u32 rssi_hyst) +{ + struct mt7932 *m = mt_from_wiphy(wiphy); + unsigned long flags; + + if (netdev != m->netdev) + return -ENODEV; + spin_lock_irqsave(&m->response_lock, flags); + /* A zero threshold disables the monitor. */ + m->cqm_threshold = rssi_thold; + m->cqm_hysteresis = min_t(u32, rssi_hyst, S8_MAX); + m->cqm_last_event = 0; + spin_unlock_irqrestore(&m->response_lock, flags); + return 0; +} From 60d430d7654838fe9c79d010d36717b921143b19 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:35:42 +1100 Subject: [PATCH 0098/2680] Bluetooth: mt7932: reset the controller instead of faulting the transport Any controller error ends the Bluetooth function for the rest of the boot. A command timeout, a Hardware Error event from the controller, the core's own recovery when the controller stops accepting commands, and a write to the device's "reset" sysfs attribute all reach bt7932_reset() or bt7932_hw_error(), which mark the transport faulted, stop bus mastering and leave the adapter dead until a cold boot. A failed setup command does the same. None of these mean that the transport is broken: the DMA rings were consistent when the controller stopped responding. Recover the way other Bluetooth drivers do. On a command timeout, inject a Hardware Error event with hci_reset_dev(); the core then closes the device, which resets the controller, and opens it again, which repeats setup and sends the calibration, PTX and address again. A Hardware Error event from the controller takes the same path. Let a setup that fails return its error, so that a later power-on can try again. Bound it so that a controller that no longer answers cannot cause a reset loop: while a recovery is pending or the device is being reopened, further timeouts do not start another one, and if the setup that follows a recovery fails, the transport faults as before. The next successful setup ends the recovery. Transport faults, where the ring counters or the firmware state are inconsistent, still fault the function: reloading the firmware needs a reset of the PCI function, which this driver does not perform. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 7 ++- drivers/bluetooth/Kconfig | 4 +- drivers/bluetooth/mt7932_bt_pcie.c | 61 ++++++++++++++++++- 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index bb382cab2c99a..3c6323e8d48fc 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -132,8 +132,11 @@ Qualification and remaining work Removal first quiesces firmware, then stops work and interrupts, disables bus mastering and drains pending PCI transactions before freeing DMA. An unconfirmed drain retains storage - until reset. Firmware recovery and warm reprobe are unsupported; use a fresh - cold boot after removing or faulting an active transport. + until reset. A controller that stops answering commands, or reports a + hardware error, is reset over HCI and set up again; if that setup fails, + the transport faults. Firmware reloading and warm reprobe are + unsupported; use a fresh cold boot after removing or faulting an active + transport. * The Bluetooth software queue limit does not provide HCI backpressure; saturation can drop an accounted frame. * Arbitrary scan IEs, WPA3/SAE, required MFP, AP/P2P, general country-package diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index ab57248b9de8c..869861f26f1f6 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -37,7 +37,9 @@ config BT_MTK7932_PCIE is mounted and read their firmware when probed, so say M, like the Wi-Fi driver. System suspend uses the firmware sleep handshake and preserves DMA - storage. Hibernation and automatic firmware recovery are unsupported. + storage. Hibernation and firmware reloading are unsupported. A + controller that stops answering commands is reset over HCI and set + up again; if that setup fails, the transport stays faulted. Removal drains the PCI function before releasing device-visible memory; an unconfirmed drain retains it until a full hardware reset. diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 8923e01220f9c..d20d6257234ad 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -104,6 +104,14 @@ enum bt7932_pm_state { BT7932_RESTORING, }; +enum bt7932_recovery { + BT7932_RECOVERY_NONE, + /* A Hardware Error event was injected and not yet handled. */ + BT7932_RECOVERY_REQUESTED, + /* The core is closing and reopening the device. */ + BT7932_RECOVERY_REOPENING, +}; + struct bt7932 { struct pci_dev *pdev; void __iomem *bar; @@ -133,6 +141,7 @@ struct bt7932 { bool enabled, regions, vectors, irq_requested, retained; bool ipc_ready, opened, awake, waking, fault, registered; bool stopping, runtime_forbidden; + enum bt7932_recovery recovery; enum bt7932_pm_state pm_state; u64 tx_packets, rx_packets, diagnostics; }; @@ -833,6 +842,9 @@ static int bt7932_open(struct hci_dev *hdev) ret = -EHOSTDOWN; else WRITE_ONCE(bt->opened, true); + /* Closing the device drops an injected event that was still queued. */ + if (bt->recovery == BT7932_RECOVERY_REQUESTED) + bt->recovery = BT7932_RECOVERY_NONE; mutex_unlock(&bt->lock); return ret; } @@ -940,28 +952,71 @@ static int bt7932_setup(struct hci_dev *hdev) if (ret) goto fault; dev_info(&bt->pdev->dev, "BT_HCI_SETUP_OK Reset/calibration/PTX/address completed in order\n"); + mutex_lock(&bt->lock); + if (bt->recovery == BT7932_RECOVERY_REOPENING) + dev_info(&bt->pdev->dev, "controller recovered\n"); + bt->recovery = BT7932_RECOVERY_NONE; + mutex_unlock(&bt->lock); return 0; fault: dev_err(&bt->pdev->dev, "BT_HCI_SETUP_FAILED stage=%s opcode=%04x status=%d\n", stage, opcode, ret); + /* + * The transport is intact, so a requested recovery or a later power-on + * may try again. A controller that fails the setup of its recovery, + * though, does not come back without a reset of the PCI function. + */ mutex_lock(&bt->lock); - bt7932_fault_locked(bt, "HCI setup command failed; no retry"); + if (bt->recovery == BT7932_RECOVERY_REOPENING) + bt7932_fault_locked(bt, "controller recovery failed"); mutex_unlock(&bt->lock); return ret; } +/* + * Called when a command times out, and through the "reset" sysfs attribute. + * While the transport rings are consistent, recover the controller the + * standard way: inject a Hardware Error event, so that the core closes the + * device, resetting the controller, and opens it again, repeating setup. + * Allow one attempt until a setup succeeds, so that a controller that no + * longer answers ends in a fault instead of a reset loop. + */ static void bt7932_reset(struct hci_dev *hdev) { struct bt7932 *bt = hci_get_drvdata(hdev); + bool recover; mutex_lock(&bt->lock); - bt7932_fault_locked(bt, "HCI requested recovery; reset is not qualified"); + recover = !bt->fault && !bt->stopping && + bt->recovery == BT7932_RECOVERY_NONE && + bt->pm_state == BT7932_RUNNING; + if (recover) + bt->recovery = BT7932_RECOVERY_REQUESTED; mutex_unlock(&bt->lock); + if (!recover) + return; + dev_warn(&bt->pdev->dev, "resetting the controller\n"); + if (hci_reset_dev(hdev)) { + mutex_lock(&bt->lock); + if (bt->recovery == BT7932_RECOVERY_REQUESTED) + bt->recovery = BT7932_RECOVERY_NONE; + mutex_unlock(&bt->lock); + } } +/* + * The core closes and reopens the device after this, for a Hardware Error + * event from the controller as for one injected above. That reopening is + * the recovery attempt. + */ static void bt7932_hw_error(struct hci_dev *hdev, u8 code) { - bt7932_reset(hdev); + struct bt7932 *bt = hci_get_drvdata(hdev); + + mutex_lock(&bt->lock); + bt->recovery = BT7932_RECOVERY_REOPENING; + mutex_unlock(&bt->lock); + bt_dev_err(hdev, "hardware error 0x%2.2x", code); } static int bt7932_iommu(struct bt7932 *bt) From 74f4b985f766fc84efb2157e6df1a39ea5812384 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:23:41 +1100 Subject: [PATCH 0099/2680] wifi: mt7932: acknowledge interrupts before servicing the rings mt_service() drains the RX rings and retires TX descriptors first and only then writes the causes it read back to the interrupt status register. A completion that the device posts after a ring has been drained but before that write sets a status bit that the write clears again. Its descriptor then waits, unserviced, until some unrelated interrupt arrives; for a command reply that can be the full command timeout. Acknowledge the causes, then service the rings, as other MediaTek PCIe drivers do. Read the status register back after the write so that the acknowledgment has reached the device, and the descriptors written before it are visible, before the rings are read. A completion that arrives while the rings are being serviced now leaves its status bit set, and the device interrupts again once the mask is restored. Fixes: 8709ef5c7d40 ("wifi: add existing J700 MT7932 fullmac implementation") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index a10e4d2010a10..647f6f206763a 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -277,16 +277,27 @@ static void mt_service(struct mt7932 *m) u32 causes = mt_read(m, W + 0x200) & m->irq_mask; unsigned int i; - mt_data_clean(m); - for (i = 0; i < ARRAY_SIZE(m->rx); i++) - if (m->rx[i].count) - mt_rx_drain(m, &m->rx[i]); + /* + * Acknowledge before servicing the rings. A completion that lands + * while they are being drained then sets its status bit again and + * raises a new interrupt once the mask is restored, instead of being + * cleared by a late acknowledgment and left waiting for an unrelated + * interrupt. Read the status back so that the acknowledgment has + * reached the device, and every descriptor it covers is visible, + * before the rings are read. + */ if (causes & BIT(29)) { u32 sw = mt_read(m, W + 0x1f0); mt_write(m, W + 0x1f0, sw); } mt_write(m, W + 0x200, causes); + mt_read(m, W + 0x200); + + mt_data_clean(m); + for (i = 0; i < ARRAY_SIZE(m->rx); i++) + if (m->rx[i].count) + mt_rx_drain(m, &m->rx[i]); if (READ_ONCE(m->running) && !poll_test) mt_write(m, W + 0x204, m->irq_mask); } From 71a826d38c56ba4b55bd8830f1ced330b7d34384 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:31:58 +1100 Subject: [PATCH 0100/2680] wifi: mt7932: ring the TX doorbell once per burst mt_net_xmit() writes the TX0 CPU index after every packet, even when the stack is handing over a burst. Each write is an uncached MMIO store across the PCIe link, and each makes the device fetch descriptors for a single packet. Use netdev_xmit_more() to publish a burst with one doorbell write: ring it when the stack has nothing more queued, when the queue has just stopped, and at least every 64 packets so that the device starts on a long burst early. Descriptors stay published in order behind the doorbell, and the TX0 accounting is unchanged. The queue can also stop from outside ndo_start_xmit(), when the access point goes absent or the link goes down, which ends a burst early. Ring the doorbell in those paths too, before the connection teardown waits for TX0 to drain, and on every interrupt, so that no published descriptor waits behind a doorbell that is never written. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/data.c | 41 +++++++++++++++++-- drivers/net/wireless/mediatek/mt7932/link.c | 6 ++- drivers/net/wireless/mediatek/mt7932/mt7932.h | 3 ++ 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index 7a8b5f925b52d..0427bca2d0de3 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -35,6 +35,25 @@ void mt_data_clean_locked(struct mt7932 *m) netif_wake_queue(m->netdev); } +/* + * The stack marks a packet that has another one right behind it with + * netdev_xmit_more(), so the TX0 doorbell is rung once per burst instead of + * once per packet, and at least every MT7932_TX_BATCH packets so that the + * device starts on a long burst early. + */ +#define MT7932_TX_BATCH 64 + +/* data_lock held: hand every published TX0 descriptor to the device. */ +void mt_data_kick_locked(struct mt7932 *m) +{ + struct mt7932_ring *q = &m->tx[0]; + + if (!m->data_unkicked) + return; + m->data_unkicked = 0; + mt_write(m, q->reg + 8, q->head); +} + void mt_data_clean(struct mt7932 *m) { unsigned long flags; @@ -42,6 +61,10 @@ void mt_data_clean(struct mt7932 *m) if (!READ_ONCE(m->data_ready)) return; spin_lock_irqsave(&m->data_lock, flags); + /* The queue can be stopped from outside ndo_start_xmit() in the middle + * of a burst; no descriptor waits longer than the next interrupt. + */ + mt_data_kick_locked(m); mt_data_clean_locked(m); spin_unlock_irqrestore(&m->data_lock, flags); } @@ -59,10 +82,12 @@ void mt_bss_presence(struct mt7932 *m, const struct mt7932_event *event) spin_lock_irqsave(&m->data_lock, flags); m->bss_absent = !!body[1]; m->bss_quota = body[2]; - if (!mt_bss_tx_allowed(m)) + if (!mt_bss_tx_allowed(m)) { netif_stop_queue(m->netdev); - else if (READ_ONCE(m->data_ready)) + mt_data_kick_locked(m); + } else if (READ_ONCE(m->data_ready)) { mt_data_clean_locked(m); + } spin_unlock_irqrestore(&m->data_lock, flags); dev_dbg_ratelimited(&m->pdev->dev, "BSS_PRESENCE: absent=%u quota=%u\n", !!body[1], body[2]); @@ -122,6 +147,7 @@ netdev_tx_t mt_net_xmit(struct sk_buff *skb, struct net_device *netdev) slot = q->head; if (!mt_bss_tx_allowed(m) || m->data_live[slot] || q->queued >= q->count - 1) { netif_stop_queue(netdev); + mt_data_kick_locked(m); spin_unlock_irqrestore(&m->data_lock, flags); return NETDEV_TX_BUSY; } @@ -147,18 +173,27 @@ netdev_tx_t mt_net_xmit(struct sk_buff *skb, struct net_device *netdev) q->head = (slot + 1) % q->count; q->queued++; dma_wmb(); - mt_write(m, q->reg + 8, q->head); + m->data_unkicked++; mt_bss_tx_account(m); m->station_tx_packets++; m->station_tx_bytes += skb->len; if (!mt_bss_tx_allowed(m) || m->data_live[q->head] || q->queued >= q->count - 1) netif_stop_queue(netdev); + if (!netdev_xmit_more() || netif_queue_stopped(netdev) || + m->data_unkicked >= MT7932_TX_BATCH) + mt_data_kick_locked(m); spin_unlock_irqrestore(&m->data_lock, flags); netdev->stats.tx_packets++; netdev->stats.tx_bytes += skb->len; dev_kfree_skb_any(skb); return NETDEV_TX_OK; drop: + /* The burst ends here: publish what earlier packets queued. */ + if (!netdev_xmit_more() && READ_ONCE(m->data_ready)) { + spin_lock_irqsave(&m->data_lock, flags); + mt_data_kick_locked(m); + spin_unlock_irqrestore(&m->data_lock, flags); + } netdev->stats.tx_dropped++; dev_kfree_skb_any(skb); return NETDEV_TX_OK; diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 6869e7ff85c8a..76bb14db4884f 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -224,8 +224,12 @@ static int mt_retire_connection(struct mt7932 *m) return -ESHUTDOWN; netif_carrier_off(m->netdev); netif_tx_disable(m->netdev); - /* ndo_start_xmit rechecks the closed admission gate under data_lock. */ + /* ndo_start_xmit rechecks the closed admission gate under data_lock. + * The queue may have stopped in the middle of a burst; hand the device + * what it already holds so that the drain below can complete. + */ spin_lock_irqsave(&m->data_lock, flags); + mt_data_kick_locked(m); spin_unlock_irqrestore(&m->data_lock, flags); flush_work(&m->cal_work); mutex_lock(&m->command_mutex); diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index fa312ecca622c..daff6cea97bfa 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -170,6 +170,8 @@ struct mt7932 { bool data_live[512], data_token[512], data_retired[512]; struct mt7932_duplicate duplicates[17]; u64 tx_released, rx_ethernet; + /* TX0 descriptors published since the last doorbell; data_lock. */ + unsigned int data_unkicked; bool data_ready; bool bss_absent; u8 bss_quota; @@ -195,6 +197,7 @@ int mt_cal_procedure(struct mt7932 *m, struct mt7932_cal_piece *pieces, unsigned int count, unsigned int logical, u8 context_version); int mt_calibration_gate(struct mt7932 *m); void mt_data_clean_locked(struct mt7932 *m); +void mt_data_kick_locked(struct mt7932 *m); void mt_data_clean(struct mt7932 *m); void mt_bss_presence(struct mt7932 *m, const struct mt7932_event *event); void mt_data_complete(struct mt7932 *m, const u8 *packet, size_t length); From 902147bad3938ee87aae74ea81daf62a98402f57 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 20:27:16 +1100 Subject: [PATCH 0101/2680] wifi: mt7932: ring the TX doorbell when a regulatory change stops the queue A regulatory change stops the data queue from the cfg80211 notifier, outside ndo_start_xmit() and outside data_lock. If that lands in the middle of a burst, the TX0 descriptors the burst already published stay behind the doorbell until the next interrupt rings it. Ring the doorbell there as well, under data_lock nested inside response_lock in the same order as the receive path, so that every path that stops the queue from outside ndo_start_xmit() hands the device what it already holds. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/startup.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 696b79e48ddfe..ee4b13af441a6 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -436,6 +436,14 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request WRITE_ONCE(m->reg_pending, true); if (m->netdev) { netif_stop_queue(m->netdev); + /* The stop can land in the middle of a burst: hand the device the + * TX0 descriptors already published behind the doorbell. + */ + if (READ_ONCE(m->data_ready)) { + spin_lock(&m->data_lock); + mt_data_kick_locked(m); + spin_unlock(&m->data_lock); + } if (m->connecting) { m->connect_error = -ECANCELED; complete(&m->assoc_start); From 3be3dbe29ed5732dd34603305252263a00a97022 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 20:31:59 +1100 Subject: [PATCH 0102/2680] wifi: mt7932: keep RX0 status in the RAM-config acknowledgment Since "acknowledge interrupts before servicing the rings", mt_service() clears the interrupt causes it is about to service, RX0 included, before it drains the rings, and restores the interrupt mask after the last ring. The RAM-config acknowledgment in mt_rx_drain() still clears RX0's status bit and rewrites the mask in the middle of the drain. The status write can clear the bit of an RX0 completion that arrived after the up-front acknowledgment, so the device no longer shows work that is still pending, and the unmask lets the interrupt fire again while the rings are still being serviced, with the fixed IRQ_MASK rather than m->irq_mask. Clear only the acknowledgment's other two status bits there, and leave the mask to mt_service(). The diagnostic drain on a command timeout runs with the interrupt line disabled; a bit it leaves set is serviced once the line is enabled again. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 647f6f206763a..1b6e1574100b4 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -241,12 +241,17 @@ static void mt_rx_drain(struct mt7932 *m, struct mt7932_ring *q) if (READ_ONCE(m->ram_config) && length >= 33 && length <= MT7932_RX_CAPACITY) { u8 *packet = q->buffers + q->tail * RX_STRIDE; - /* Source-specific RAM-config acknowledgment before RX0 refill. */ + /* + * Source-specific RAM-config acknowledgment before RX0 + * refill. mt_service() has already acknowledged this + * pass's causes, RX0 (bit 0) included, and restores the + * mask once every ring is drained. Leave bit 0 alone, so + * that an RX0 completion arriving now keeps its status, + * and leave the mask alone while the rings are serviced. + */ if (q == &m->rx[0] && packet[28] == 1 && packet[29] == m->waiting && !packet[30] && packet[32] == 1) { - mt_write(m, W + 0x204, 0); - mt_write(m, W + 0x200, 0x0c000001); - mt_write(m, W + 0x204, poll_test ? 0 : IRQ_MASK); + mt_write(m, W + 0x200, 0x0c000000); mt_write(m, W + 0x1f4, 0xffff); } } From 12fac7943498f22e18ebf3a6bd55c32e3644d65f Mon Sep 17 00:00:00 2001 From: Wes Grimes Date: Fri, 2 Oct 2026 19:25:42 -0400 Subject: [PATCH 0103/2680] PCI: apple: Cold-init T600x/T602x PCIe-C ports without an m1n1 handoff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit T600x and T602x PCIe-C (the Thunderbolt PCIe tunnel) is only brought up after an m1n1 handoff ("apple,pciec-preinit-status" = 1), and no m1n1 performs one. The tunnel is always refused, so PCIe devices behind Thunderbolt 3 docks and displays (their xHCI, audio, cameras, network) never appear. Add pcie_apple.tunnel_kernel_init (default off). When set, ports on T6000/T6001/T6020/T6021 (M1 and M2 Pro/Max) without a handoff are cold-initialized by the kernel through the existing t8103 paths: apple_pcie_tunnel_prepare() clocks the port before its DART probes, and apple_pcie_setup_port() cold-initializes a port that is not running. It is a parameter rather than a DT property because m1n1 hands every installed kernel the same DT. Ports whose DART has no "apple,tunable" are refused: m1n1 does not provide them, and the DART cannot probe without them. apple_pcie_tunnel_cold_init() gains one ordering difference for these ports: they must report RUN before the root complex config-space tunables are written, which hard-resets a J416c otherwise. T600x/T602x have no oe-fabric region and pulse Intr2AXI instead, and they keep the 0x208 reset table value. The t8103 power and resume paths, selected by pcie->kernel_init, are not used. Tested on a MacBook Pro 16" M2 Pro (j416s) with an Apple Studio Display on each of its three USB-C ports, plugged in after boot and connected at boot. T600x is untested. [ryan: adapted to 7.2.10 — 7.2 changed of_map_id(); find the tunnel DART with of_map_iommu_id(), as Asahi's apple_pcie_enable_device() now does] Signed-off-by: Wes Grimes --- drivers/pci/controller/pcie-apple.c | 132 +++++++++++++++++++++++----- include/linux/pci-apple.h | 1 + 2 files changed, 110 insertions(+), 23 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 385f16e55feb9..c6e8d25ca3e2c 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -65,6 +65,56 @@ static int link_up_timeout = 500; module_param(link_up_timeout, int, 0644); MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); +static bool tunnel_kernel_init; +module_param(tunnel_kernel_init, bool, 0444); +MODULE_PARM_DESC(tunnel_kernel_init, + "Cold-initialize T600x/T602x PCIe-C ports without an m1n1 handoff (experimental)"); + +static const char *const apple_pcie_tunnel_cold_init_machines[] = { + "apple,t6000", "apple,t6001", "apple,t6020", "apple,t6021", NULL +}; + +/** + * apple_pcie_tunnel_needs_cold_init() - whether the kernel brings up a port + * @np: PCIe-C host node + * + * No m1n1 performs the PCIe-C preinit handoff on T600x/T602x. With the + * opt-in parameter the kernel cold-initializes these ports instead. It is a + * parameter rather than a DT property because m1n1 hands every installed + * kernel the same DT. The tunnel's DART cannot probe without its tunables, + * which m1n1 does not provide, so ports whose DART lacks them are refused. + * + * Unlike "apple,pciec-kernel-init" (t8103), this does not select the t8103 + * power and resume sequences. + */ +bool apple_pcie_tunnel_needs_cold_init(struct device_node *np) +{ + struct of_phandle_args iommu_spec = {}; + struct device_node *dart; + bool tunables; + u32 status; + + if (!tunnel_kernel_init || + !of_device_is_compatible(np, "apple,t6000-pciec") || + !of_machine_compatible_match(apple_pcie_tunnel_cold_init_machines)) + return false; + + if (!of_property_read_u32(np, "apple,pciec-preinit-status", &status) && + status == 1) + return false; + + if (of_map_iommu_id(np, 0, &iommu_spec) || !iommu_spec.np) + return false; + dart = iommu_spec.np; + tunables = of_property_present(dart, "apple,tunable"); + if (!tunables) + pr_warn_once("%pOF: no PCIe-C DART tunables, not cold-initializing\n", + dart); + of_node_put(dart); + return tunables; +} +EXPORT_SYMBOL_GPL(apple_pcie_tunnel_needs_cold_init); + /* T8103 (original M1) and related SoCs */ #define CORE_RC_PHYIF_CTL 0x00024 #define CORE_RC_PHYIF_CTL_RUN BIT(0) @@ -263,6 +313,7 @@ struct apple_pcie { struct apple_tunable *oe_fabric_tunable; bool power_retained; bool kernel_init; + bool tunnel_cold_init; bool bus_stopped; bool reset_on_resume; bool link_kept; @@ -785,7 +836,10 @@ static void apple_pcie_tunnel_reset_hardware(struct apple_pcie_port *port) for (i = 0; i < ARRAY_SIZE(apple_pcie_tunnel_reset_regs); i++) { u32 value = apple_pcie_tunnel_reset_regs[i].value; - /* t8103 writes 0. 0x208 is the value used when firmware left the port up. */ + /* + * t8103 writes 0. 0x208 is the value used when firmware left the + * port up; T600x/T602x cold init without a handoff uses it too. + */ if (port->pcie->kernel_init && apple_pcie_tunnel_reset_regs[i].offset == 0x130) value = 0; @@ -860,10 +914,11 @@ static int apple_pcie_tunnel_reinitialize(struct apple_pcie_port *port) } /* - * t8103 has no firmware handoff for this port. Bring it up from reset: - * tunables and the reset table, then PERST and APPCLK, and only then the - * root-port config-space tunables. Stop before releasing the tunnel reset - * or starting link training. + * Bring up a port that no firmware handoff left running (t8103, or + * T600x/T602x with pcie_apple.tunnel_kernel_init): tunables and the reset + * table, then PERST and APPCLK, and only then the root-port config-space + * tunables. Stop before releasing the tunnel reset or starting link + * training. */ static int apple_pcie_tunnel_cold_init(struct apple_pcie_port *port) { @@ -887,15 +942,33 @@ static int apple_pcie_tunnel_cold_init(struct apple_pcie_port *port) port->np); apple_pcie_port_rmw_set(port, PORT_PERST_OFF, pcie->hw->port_perst); apple_pcie_port_rmw_set(port, PORT_APPCLK_EN, PORT_APPCLK); - apple_pcie_port_rmw_clear(port, PORT_APPCLK_CGDIS, PORT_APPCLK); + /* + * Without an m1n1 handoff, T600x/T602x ports must report RUN before + * the root complex config space is written: writing the RC tunables + * first hard-resets a J416c. + */ + if (!pcie->kernel_init) { + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + stat & PORT_STATUS_READY, + 10, 250000, false, port, + PORT_STATUS); + if (ret) + return dev_err_probe(pcie->dev, ret, + "port %pOF cold init: RUN not set (status %#x)\n", + port->np, stat); + } else { + apple_pcie_port_rmw_clear(port, PORT_APPCLK_CGDIS, PORT_APPCLK); + } - dev_info(pcie->dev, "port %pOF cold init: oe-fabric and root tunables\n", - port->np); + dev_info(pcie->dev, "port %pOF cold init: %sroot tunables\n", + port->np, pcie->oe_fabric_base ? "oe-fabric and " : ""); if (pcie->oe_fabric_base) apple_pcie_tunnel_apply_tunable(pcie->oe_fabric_base, pcie->oe_fabric_tunable); apple_pcie_tunnel_apply_tunable(pcie->cfg ? pcie->cfg->win : pcie->early_cfg, pcie->rc_tunable); + if (!pcie->kernel_init) + apple_pcie_port_rmw_clear(port, PORT_APPCLK_CGDIS, PORT_APPCLK); apple_pcie_port_writel(port, PORT_COUNTER_ENABLE, PORT_COUNTER_CTRL); apple_pcie_port_writel(port, ~0, PORT_INTSTAT); @@ -1479,13 +1552,14 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, * m1n1 owns PCIe-C cold initialization on T6020. Replaying the * port reset against that live handoff raises an asynchronous * SError. t8103 boots with ATC_PCIE off and no handoff, so - * apple,pciec-kernel-init selects the in-kernel sequence. + * apple,pciec-kernel-init selects the in-kernel sequence, as + * pcie_apple.tunnel_kernel_init does on T600x/T602x. */ ret = of_property_read_u32(pcie->dev->of_node, "apple,pciec-preinit-status", &preinit_status); preinit_ok = !ret && preinit_status == 1; - if (pcie->kernel_init && !preinit_ok) { + if ((pcie->kernel_init || pcie->tunnel_cold_init) && !preinit_ok) { stat = apple_pcie_port_readl(port, PORT_STATUS); if (stat & PORT_STATUS_READY) { dev_info(pcie->dev, @@ -1498,7 +1572,7 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, } } else if (!preinit_ok) { dev_err(pcie->dev, - "PCIe-C requires a successful m1n1 preinit handoff\n"); + "PCIe-C requires an m1n1 preinit handoff or pcie_apple.tunnel_kernel_init=1\n"); ret = -ENODEV; goto err_teardown; } else { @@ -2352,6 +2426,8 @@ static int apple_pcie_probe(struct platform_device *pdev) if (pcie->hw->tunneled) { pcie->kernel_init = of_property_read_bool(dev->of_node, "apple,pciec-kernel-init"); + pcie->tunnel_cold_init = + apple_pcie_tunnel_needs_cold_init(dev->of_node); ret = apple_pcie_tunnel_init_resources(pdev, pcie); if (ret) return ret; @@ -2700,14 +2776,16 @@ int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel) struct apple_pcie *pcie; struct apple_pcie_port *port; struct apple_pcie_map config = {}, debug = {}, fabric = {}, portmap = {}; - struct apple_pcie_map oe = {}; + struct apple_pcie_map oe = {}, intr2axi = {}; + bool kernel_init = false; u32 stat; int ret; np = NULL; for_each_available_child_of_node(tunnel, np) { - if (of_device_is_compatible(np, "apple,t8103-pciec") && - of_property_read_bool(np, "apple,pciec-kernel-init")) + kernel_init = of_device_is_compatible(np, "apple,t8103-pciec") && + of_property_read_bool(np, "apple,pciec-kernel-init"); + if (kernel_init || apple_pcie_tunnel_needs_cold_init(np)) break; } if (!np) @@ -2740,7 +2818,11 @@ int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel) ret = apple_pcie_map_named(np, "fabric", &fabric); if (ret) goto out_np; - ret = apple_pcie_map_named(np, "oe-fabric", &oe); + /* t8103 has an oe-fabric region; T600x/T602x pulse Intr2AXI instead. */ + if (kernel_init) + ret = apple_pcie_map_named(np, "oe-fabric", &oe); + else + ret = apple_pcie_map_named(np, "intr2axi", &intr2axi); if (ret) goto out_np; @@ -2759,11 +2841,12 @@ int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel) pcie->dev = dev; pcie->hw = match->data; - pcie->kernel_init = true; + pcie->kernel_init = kernel_init; pcie->early_cfg = config.base; pcie->debug_base = debug.base; pcie->fabric_base = fabric.base; pcie->oe_fabric_base = oe.base; + pcie->intr2axi_base = intr2axi.base; pcie->debug_tunable = apple_pcie_tunable_once(np, "apple,tunable-debug", &debug.res); if (IS_ERR(pcie->debug_tunable)) { @@ -2785,13 +2868,15 @@ int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel) pcie->rc_tunable = NULL; goto out_free; } - pcie->oe_fabric_tunable = apple_pcie_tunable_once(np, - "apple,tunable-oe-fabric", - &oe.res); - if (IS_ERR(pcie->oe_fabric_tunable)) { - ret = PTR_ERR(pcie->oe_fabric_tunable); - pcie->oe_fabric_tunable = NULL; - goto out_free; + if (oe.base) { + pcie->oe_fabric_tunable = + apple_pcie_tunable_once(np, "apple,tunable-oe-fabric", + &oe.res); + if (IS_ERR(pcie->oe_fabric_tunable)) { + ret = PTR_ERR(pcie->oe_fabric_tunable); + pcie->oe_fabric_tunable = NULL; + goto out_free; + } } port->pcie = pcie; @@ -2830,6 +2915,7 @@ int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel) apple_pcie_unmap(&fabric); apple_pcie_unmap(&portmap); apple_pcie_unmap(&oe); + apple_pcie_unmap(&intr2axi); of_node_put(np); return ret; } diff --git a/include/linux/pci-apple.h b/include/linux/pci-apple.h index d7c2d244d209b..b446c5e222902 100644 --- a/include/linux/pci-apple.h +++ b/include/linux/pci-apple.h @@ -12,6 +12,7 @@ struct notifier_block; int apple_pcie_tunnel_register_notifier(struct notifier_block *nb); void apple_pcie_tunnel_unregister_notifier(struct notifier_block *nb); +bool apple_pcie_tunnel_needs_cold_init(struct device_node *np); int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel); int apple_pcie_tunnel_quiesce(struct device *dev); int apple_pcie_tunnel_restore(struct device *dev); From ade998a8a2d9f6564906f34a5135d219bbb0a008 Mon Sep 17 00:00:00 2001 From: Wes Grimes Date: Fri, 2 Oct 2026 19:25:42 -0400 Subject: [PATCH 0104/2680] PCI: apple: Keep cold-initialized PCIe-C functions runtime-active A function behind a cold-initialized T600x/T602x tunnel stays in D0 (NO_D3) when it runtime-suspends, but its wakeup never arrives. When a Studio Display is power-cycled, its Titan Ridge xHCI runtime-suspends before the display's hub reconnects and never sees the hub, so the display's USB devices do not come back. Hold a runtime PM reference on each function for as long as it exists, from a PCI bus notifier registered before the host is scanned, so that neither the driver nor userspace power policy (power/control = auto) can suspend it. A bus notifier rather than a PCI fixup, because fixups in a module are never run. Tested on j416s with an Apple Studio Display: after a display power cycle its hub, camera and audio come back by themselves. Signed-off-by: Wes Grimes --- drivers/pci/controller/pcie-apple.c | 49 +++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index c6e8d25ca3e2c..44b4e189cccac 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -314,6 +315,7 @@ struct apple_pcie { bool power_retained; bool kernel_init; bool tunnel_cold_init; + struct notifier_block tunnel_pci_nb; bool bus_stopped; bool reset_on_resume; bool link_kept; @@ -2149,6 +2151,41 @@ static int apple_pcie_tunnel_add_link(struct apple_pcie *pcie, return ret; } +/* + * A function behind a cold-initialized T600x/T602x tunnel stays in D0 + * (NO_D3) when it runtime-suspends, but its wakeup never arrives: an xHCI + * that suspends before a power-cycled display's hub reconnects never sees + * the hub. Hold a runtime PM reference for as long as each function exists, + * so that neither its driver nor power/control can suspend it. + */ +static int apple_pcie_tunnel_pci_notify(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct apple_pcie *pcie = container_of(nb, struct apple_pcie, + tunnel_pci_nb); + struct pci_dev *pdev = to_pci_dev(data); + + if (pci_host_bridge_priv(pci_find_host_bridge(pdev->bus)) != pcie) + return NOTIFY_DONE; + + switch (action) { + case BUS_NOTIFY_ADD_DEVICE: + pm_runtime_get_noresume(&pdev->dev); + break; + case BUS_NOTIFY_DEL_DEVICE: + pm_runtime_put_noidle(&pdev->dev); + break; + } + return NOTIFY_OK; +} + +static void apple_pcie_tunnel_pci_unregister(void *data) +{ + struct apple_pcie *pcie = data; + + bus_unregister_notifier(&pci_bus_type, &pcie->tunnel_pci_nb); +} + static int apple_pcie_tunnel_keep_d0(struct pci_dev *pdev, void *data) { /* @@ -2456,6 +2493,18 @@ static int apple_pcie_probe(struct platform_device *pdev) return ret; } + /* Every function is added after this and removed before devres runs. */ + if (pcie->tunnel_cold_init) { + pcie->tunnel_pci_nb.notifier_call = apple_pcie_tunnel_pci_notify; + ret = bus_register_notifier(&pci_bus_type, &pcie->tunnel_pci_nb); + if (ret) + return ret; + ret = devm_add_action_or_reset(dev, apple_pcie_tunnel_pci_unregister, + pcie); + if (ret) + return ret; + } + ret = apple_msi_init(pcie); if (ret) return ret; From 6d520a926d1940362471c13f43aca5b1e45c9223 Mon Sep 17 00:00:00 2001 From: Wes Grimes Date: Fri, 2 Oct 2026 19:25:42 -0400 Subject: [PATCH 0105/2680] PCI: apple: Keep cold-initialized PCIe-C links up through suspend-to-idle In suspend-to-idle ACIO and the router links stay powered, and t8103 already leaves a healthy tunnel running instead of stopping the port. Cold-initialized T600x/T602x ports took the stop path. On resume the router rebuilt its tunnels, the in-place restart brought the link up only for it to drop again, the xHCI behind it was lost, and that tunnel instance could not train a link again until reboot. Let these ports keep their link through suspend-to-idle too, so the NHI keeps its tunnels and nothing is retrained. If the link was lost while asleep anyway, fail the host like a surprise unplug rather than restarting it in place; ACIO then revalidates the connection and replaces it. Tested on j416s with an Apple Studio Display: after 20 s and 3 min suspends its hub, camera and audio keep working, and unplugging the display while asleep is handled as an unplug. Signed-off-by: Wes Grimes --- drivers/pci/controller/pcie-apple.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 44b4e189cccac..601da469c09a2 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -3054,7 +3054,8 @@ static bool apple_pcie_keep_link(struct apple_pcie *pcie) { if (!READ_ONCE(s2idle_keep_link) || !pm_suspend_no_platform()) return false; - if (!pcie->kernel_init || !pcie->power_retained) + if (!(pcie->kernel_init || pcie->tunnel_cold_init) || + !pcie->power_retained) return false; if (pcie->bus_stopped || pcie->resume_failed) return false; @@ -3164,8 +3165,18 @@ static int apple_pcie_resume_noirq(struct device *dev) pcie->link_kept = false; if (apple_pcie_tunnel_link_healthy(pcie)) return 0; - /* Lost while asleep: take the same path as a stopped tunnel. */ dev_warn(dev, "PCIe-C link lost during suspend-to-idle\n"); + /* + * A cold-initialized T600x/T602x port does not train again after + * an in-place restart. Fail like a surprise unplug instead: ACIO + * revalidates the connection after resume and replaces this host + * with a freshly cold-initialized one. + */ + if (pcie->tunnel_cold_init) { + ret = -ENOLINK; + goto failed; + } + /* Lost while asleep: take the same path as a stopped tunnel. */ apple_pcie_stop_for_sleep(dev); } if (pcie->reset_on_resume) { From 1f5bf6b63edb95754b8522243c894a7e1d263e0d Mon Sep 17 00:00:00 2001 From: Wes Grimes Date: Fri, 2 Oct 2026 20:28:06 -0400 Subject: [PATCH 0106/2680] PCI: apple: Don't restart a quiesced PCIe-C host on resume When ACIO deactivates the PCIe tunnel while the cable stays connected, apple_pcie_tunnel_quiesce() removes the hierarchy and stops the ports, leaving the host bus_stopped until apple_pcie_tunnel_restore() restarts them on reactivation. resume_noirq() nevertheless restarts every port after a system suspend. Without a tunnel that fails, sets resume_failed, and the host then refuses restore() until it is replaced. A quiesced host has nothing to resume, so leave its ports stopped. Signed-off-by: Wes Grimes --- drivers/pci/controller/pcie-apple.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 601da469c09a2..46e970734d5c8 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -3161,6 +3161,14 @@ static int apple_pcie_resume_noirq(struct device *dev) return 0; if (pcie->resume_failed) return -EIO; + /* + * A host quiesced by a tunnel deactivation has no hierarchy and its + * ports are stopped. Leave them stopped: restarting them without a + * tunnel fails and marks the host dead, while + * apple_pcie_tunnel_restore() restarts them on reactivation. + */ + if (pcie->bus_stopped) + return 0; if (pcie->link_kept) { pcie->link_kept = false; if (apple_pcie_tunnel_link_healthy(pcie)) From 323d28955cedc50a3991867254f1ffd6bcf15789 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 00:03:00 -0700 Subject: [PATCH 0107/2680] PCI: apple: Wait for tunnel functions before every PCIe-C bus scan A Thunderbolt dock's functions come up after the tunnel link. The one second wait before the bus scan only ran on the t8103 path (kernel_init), so a T600x/T602x cold init scanned right after link-up, and the rescan that restores a quiesced hierarchy had no wait at all. On J314s with a CalDigit TS3 Plus, only the switch, the I210 and the Alpine Ridge xHCI were ready at link-up. The dock's two FL1100 and its ASM1142 xHCIs trained their links later, behind switch ports that are not hotplug-capable, so they were never enumerated and the USB ports they serve stayed dead. A J293 on the t8103 path enumerates all five functions with the same dock, scanning one second after link-up. The dock decides how long its functions need, not the Mac. Record when a tunnel link comes up, on first bring-up and on restart, and wait until it has been up for one second before either scan; time already spent counts. This covers every PCIe-C host and replaces the t8103-only condition. Tested with the TS3 Plus on J314s (cold init: all five functions bind, USB keyboard and mouse work) and J293 (t8103: same one second scan timing as before, all five functions bind). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/pci/controller/pcie-apple.c | 42 ++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 46e970734d5c8..383af650b2f12 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -315,6 +315,7 @@ struct apple_pcie { bool power_retained; bool kernel_init; bool tunnel_cold_init; + unsigned long tunnel_ready_at; struct notifier_block tunnel_pci_nb; bool bus_stopped; bool reset_on_resume; @@ -1030,6 +1031,33 @@ static void apple_pcie_tunnel_restore_irq_hw(struct apple_pcie_port *port) apple_pcie_port_writel(port, value | PORT_MSICFG_EN, PORT_MSICFG); } +/* + * Functions behind a Thunderbolt dock come up after the tunnel link does. On + * a CalDigit TS3 Plus the I210 and the Alpine Ridge xHCI answer at link-up, + * but the dock's FL1100 and ASM1142 xHCIs train their links later, behind + * switch ports that are not hotplug-capable, so a scan before then misses + * them for good. PCIe allows a function 1 s after reset before it has to + * answer configuration requests (PCIe r6.0, sec 6.6.1; PCI_RESET_WAIT in the + * PCI core), so the bus behind a tunnel is scanned only once its link has + * been up that long. This applies to every tunnel, on every SoC. + */ +#define APPLE_PCIE_TUNNEL_READY_MS 1000 + +static void apple_pcie_tunnel_link_up(struct apple_pcie *pcie) +{ + WRITE_ONCE(pcie->tunnel_ready_at, + jiffies + msecs_to_jiffies(APPLE_PCIE_TUNNEL_READY_MS)); +} + +/* Call before scanning the bus behind a tunnel; elapsed time counts. */ +static void apple_pcie_tunnel_wait_ready(struct apple_pcie *pcie) +{ + long left = (long)(READ_ONCE(pcie->tunnel_ready_at) - jiffies); + + if (left > 0) + msleep(jiffies_to_msecs(left)); +} + static int apple_pcie_tunnel_start(struct apple_pcie_port *port) { struct apple_pcie *pcie = port->pcie; @@ -1168,6 +1196,7 @@ static int apple_pcie_tunnel_start(struct apple_pcie_port *port) return ret; } restored: + apple_pcie_tunnel_link_up(pcie); /* Publish ownership before unmasking a possible link-down event. */ WRITE_ONCE(port->started, true); /* The port helper uses a relaxed MMIO write. */ @@ -1680,12 +1709,9 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, dev_info(pcie->dev, "%pOF link up after %ldms\n", np, (timeout - left) * 1000 / HZ); } - /* - * A TB3 dock answers CRS for a while after LTSSM start. Wait before - * the bus scan that follows apple_pcie_init(). - */ - if (pcie->hw->tunneled && pcie->kernel_init) - msleep(1000); + if (pcie->hw->tunneled && + (apple_pcie_port_readl(port, PORT_LINKSTS) & PORT_LINKSTS_UP)) + apple_pcie_tunnel_link_up(pcie); if (pcie->hw->tunneled) WRITE_ONCE(port->started, true); if (pcie->hw->root_bus_only) @@ -1925,6 +1951,9 @@ static int apple_pcie_init(struct pci_config_window *cfg) } } + /* The bus scan follows this callback. */ + apple_pcie_tunnel_wait_ready(pcie); + return 0; } @@ -2683,6 +2712,7 @@ int apple_pcie_tunnel_restore(struct device *dev) if (ret) return ret; + apple_pcie_tunnel_wait_ready(pcie); pci_lock_rescan_remove(); pci_rescan_bus(bridge->bus); pci_walk_bus(bridge->bus, apple_pcie_tunnel_keep_d0, NULL); From 6210cd9b7be204f1a025e0f16c45422021c3aaf5 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 00:03:45 -0700 Subject: [PATCH 0108/2680] PCI: apple: Cold-init T600x/T602x PCIe-C ports by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PCIe over Thunderbolt on M1 Pro/Max and M2 Pro/Max no longer needs pcie_apple.tunnel_kernel_init=1. The parameter stays as an off switch: pcie_apple.tunnel_kernel_init=0 keeps these tunnels off. Run on J416s (M2 Pro, Apple Studio Display) and J314s (M1 Pro, CalDigit TS3 Plus: USB keyboard and mouse, camera, audio, Ethernet and a USB SSD behind the dock). M1 Max and M2 Max share those device trees and the same code path but have not run it yet. tb-pcie-report and its README now treat the tunnel as on unless the command line turns it off. [ryan: adapted to 7.2.10 — split by topic: the tools/aurora-tb hunks go to 170-usb with the tool itself] Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/pci/controller/pcie-apple.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 383af650b2f12..b876e0e841019 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -66,10 +66,10 @@ static int link_up_timeout = 500; module_param(link_up_timeout, int, 0644); MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); -static bool tunnel_kernel_init; +static bool tunnel_kernel_init = true; module_param(tunnel_kernel_init, bool, 0444); MODULE_PARM_DESC(tunnel_kernel_init, - "Cold-initialize T600x/T602x PCIe-C ports without an m1n1 handoff (experimental)"); + "Cold-initialize T600x/T602x PCIe-C ports without an m1n1 handoff (default: on)"); static const char *const apple_pcie_tunnel_cold_init_machines[] = { "apple,t6000", "apple,t6001", "apple,t6020", "apple,t6021", NULL @@ -79,11 +79,12 @@ static const char *const apple_pcie_tunnel_cold_init_machines[] = { * apple_pcie_tunnel_needs_cold_init() - whether the kernel brings up a port * @np: PCIe-C host node * - * No m1n1 performs the PCIe-C preinit handoff on T600x/T602x. With the - * opt-in parameter the kernel cold-initializes these ports instead. It is a - * parameter rather than a DT property because m1n1 hands every installed - * kernel the same DT. The tunnel's DART cannot probe without its tunables, - * which m1n1 does not provide, so ports whose DART lacks them are refused. + * No m1n1 performs the PCIe-C preinit handoff on T600x/T602x, so the kernel + * cold-initializes these ports instead; pcie_apple.tunnel_kernel_init=0 turns + * that off. It is a parameter rather than a DT property because m1n1 hands + * every installed kernel the same DT. The tunnel's DART cannot probe without + * its tunables, which m1n1 does not provide, so ports whose DART lacks them + * are refused. * * Unlike "apple,pciec-kernel-init" (t8103), this does not select the t8103 * power and resume sequences. From 508234c96f29075b7b8e04e9800f667012aa49c7 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:04:22 -0700 Subject: [PATCH 0109/2680] PCI: apple: admit J700 radios in the shared kernel with runtime guards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow the built-in bootstrap alongside a modular host and enabled sleep/kexec. Enable only on J700/T8140, install lifetime guards before retaining the arena, and defer probe on transition contention. Restrict link power states to the validated radio link before binding and preserve retained host ownership. [ryan: adapted to 7.2.10 — split by topic: the MAINTAINERS lines are carried by 900-config-tooling; 010-soc carries the final kexec interlock API, so the supplier uses its own struct kexec_blocker from here on (taken forward from "kexec: identify permanent builtin retained-memory interlocks"); refusal codes stay as in this commit] --- Documentation/PCI/controller/apple-t8140.rst | 31 ++++- drivers/net/wireless/mediatek/mt7932/mt7932.c | 8 +- drivers/pci/controller/Kconfig | 13 +- .../pci/controller/pcie-apple-piodma-diag.c | 128 ++++++++++++++++-- drivers/pci/controller/pcie-apple.c | 34 ++++- include/linux/pci-apple-piodma.h | 20 +++ 6 files changed, 208 insertions(+), 26 deletions(-) create mode 100644 include/linux/pci-apple-piodma.h diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index 90d4eab8f07f4..1a82e9447a3a6 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -7,16 +7,33 @@ The J700 path trains port 0, validates a bounded PIODMA bootstrap request, and enables native ECAM for the MediaTek Wi-Fi and Bluetooth functions. The root bridge forwards their BARs and DART translates endpoint DMA. -This is an opt-in bring-up implementation, not general T8140 PCIe support. -Build ``CONFIG_PCIE_APPLE_PIODMA_DIAG=y`` and pass -``pcie_apple_piodma_diag.enumerate=1`` to the kernel. Without the opt-in, the -root-only diagnostic path continues to reject downstream configuration access. -The option excludes suspend and kexec. Failed or uncertain bootstrap state -requires a full external hardware reset; there is no command retry. +This is a bring-up implementation, not general T8140 PCIe support. Build +``CONFIG_PCIE_APPLE_PIODMA_DIAG=y``; the Apple PCIe host driver may be a module. +The bootstrap enables enumeration by default only on machines matching both +``apple,j700`` and ``apple,t8140``, through the matching diagnostic and host +device-tree nodes. No kernel argument is required. To leave the root-only +diagnostic path closed, pass ``pcie_apple_piodma_diag.enumerate=0``. + +The shared kernel can keep suspend, hibernation and kexec support enabled. +Once the supplier retains its arena, a PM notifier refuses sleep transitions +and a kexec interlock refuses loading or executing a replacement kernel, +including an already loaded crash kernel. Image unloading remains permitted. +These interlocks apply only to the active J700 supplier; other machines do not +acquire them. A successful supplier probe retains this protection even if later +radio enumeration fails. Failed or uncertain bootstrap state requires a full +external hardware reset; there is no command retry. + +Before either radio driver can bind, the host disables ASPM, all L1 substates +and Clock PM on the validated radio endpoints' shared PCIe link through the +PCI core. ``CONFIG_PCIEASPM=y`` is required. The global ASPM policy is unchanged; +neither a performance-policy boot argument nor a sysfs policy write is needed. +Booting with ``pcie_aspm=off`` prevents the PCI core from applying this restriction +and radio admission fails closed. The PIODMA arena remains allocated until external reset. Controller removal, teardown, memory reuse and arbitrary downstream devices are unqualified. -Do not remove the controller or its IOMMU while this experiment is active. +The host and activated Bluetooth modules are pinned until reset. Do not remove +the controller, radio PCI devices or their IOMMUs while this experiment is active. These lifecycle contracts require further work before production support. J700's ``wifi0`` alias and PCI endpoint node allow the public bootloader to diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index b422d5acb5b9d..36ba1daad568d 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -1172,11 +1172,17 @@ static void mt_remove(struct pci_dev *pdev) static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) { + struct pci_host_bridge *bridge = pci_find_host_bridge(pdev->bus); struct mt7932 *m; struct iommu_domain *domain; int ret; - if (!of_machine_is_compatible("apple,j700") || pdev->devfn != 0) + if (!of_machine_is_compatible("apple,j700") || + !of_machine_is_compatible("apple,t8140") || pdev->devfn != 0 || + !of_device_is_compatible(pdev->dev.of_node, "pci14c3,7932") || + !bridge->dev.parent || + !of_device_is_compatible(bridge->dev.parent->of_node, + "apple,t8140-pcie")) return -ENODEV; domain = iommu_get_domain_for_dev(&pdev->dev); if (!domain || (domain->type != IOMMU_DOMAIN_DMA && domain->type != IOMMU_DOMAIN_DMA_FQ)) diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index d418c2124c565..830c2c24a68ec 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -59,14 +59,17 @@ config PCIE_APPLE config PCIE_APPLE_PIODMA_DIAG bool "J700 PIODMA diagnostic (requires full reset after use)" - depends on PCIE_APPLE=y && ARCH_APPLE && IOMMU_DMA - depends on !PM_SLEEP && !KEXEC_CORE + depends on PCIE_APPLE != n && ARCH_APPLE && IOMMU_DMA + depends on PCIEASPM help Build the experimental SID17 retained bootstrap provider. It submits one fixed request, validates typed ECAM and permits enumeration of - the two Neo functions only with the explicit enumerate=1 parameter. - Its arena remains owned until a full reset. This is not production - support, and does not qualify memory reuse or radio operation. + the two Neo functions by default on J700/T8140 only. The parameter + pcie_apple_piodma_diag.enumerate=0 disables the bootstrap. + Its arena remains owned until a full reset. While that memory is + retained, suspend, hibernate and kexec are refused at runtime on + the Neo; other machines retain their normal lifecycle support. + The provider is built-in even when the Apple PCIe host is a module. config PCI_VERSATILE bool "ARM Versatile PB PCI controller" diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index 87caa91262e37..997d300229697 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -8,7 +8,8 @@ * * This built-in driver deliberately retains its coherent arena and device * power reference until a coordinator-controlled full hardware reset. There - * is no unbind interface, suspend, kexec or retry. Native ECAM is only + * is no unbind interface or retry. Runtime guards refuse suspend and kexec + * while the arena is held. Native ECAM is only * opened by the host after a successful bootstrap and typed read checks. */ #include @@ -19,16 +20,21 @@ #include #include #include +#include #include +#include #include #include #include #include #include +#include #include #include +#include #include #include +#include #include "pcie-apple-piodma-diag.h" @@ -48,7 +54,7 @@ #define PIODMA_FIFO_MASK GENMASK(5, 0) #define PIODMA_IRQ_LIMIT 16 -static bool enumerate; +static bool enumerate = true; module_param(enumerate, bool, 0400); MODULE_PARM_DESC(enumerate, "Prime once, validate native ECAM, and enumerate the two Neo functions"); @@ -92,8 +98,52 @@ struct apple_piodma_diag { u64 pointer_prefix; u8 secondary_bus; bool retained; + struct notifier_block pm_notifier; + struct kexec_blocker kexec_blocker; }; +/* A successful supplier probe owns its arena and guards until full reset. */ +static struct apple_piodma_diag *active_diag; + +static bool apple_piodma_trylock_sleep(unsigned int *flags) +{ + *flags = 0; + if (!IS_ENABLED(CONFIG_PM_SLEEP)) + return true; + + /* + * Probe holds device_lock(), while system sleep holds the transition + * mutex before waiting for probes and taking device locks. Never wait + * for that mutex here: if sleep won, defer without publishing any DMA. + * If probe won, publish the veto before releasing the transition mutex. + */ + *flags = current->flags; + current->flags |= PF_NOFREEZE; + if (mutex_trylock(&system_transition_mutex)) + return true; + if (!(*flags & PF_NOFREEZE)) + current->flags &= ~PF_NOFREEZE; + return false; +} + +static int apple_piodma_pm_notify(struct notifier_block *nb, + unsigned long action, void *unused) +{ + struct apple_piodma_diag *diag = + container_of(nb, struct apple_piodma_diag, pm_notifier); + + switch (action) { + case PM_SUSPEND_PREPARE: + case PM_HIBERNATION_PREPARE: + case PM_RESTORE_PREPARE: + dev_warn(diag->dev, + "sleep refused: Neo radio bootstrap retains DMA memory until full hardware reset\n"); + return notifier_from_errno(-EBUSY); + default: + return NOTIFY_DONE; + } +} + static int apple_piodma_diag_root(struct apple_piodma_diag *diag, struct pci_dev *root) { int ret; @@ -311,8 +361,8 @@ static int apple_piodma_diag_initialize(struct apple_piodma_diag *diag) int i; u32 value; - /* All later failures retain the device, IRQ, domain and arena. */ - diag->retained = true; + /* Publish validated root/bus fields; all later failures retain ownership. */ + smp_store_release(&diag->retained, true); writel(PIODMA_IRQ_MASK, diag->engine + 8); writel(0, diag->fabric + 4); for (i = 0; i < ARRAY_SIZE(tunables); i++) { @@ -465,8 +515,10 @@ static int apple_piodma_diag_submit(struct apple_piodma_diag *diag, unsigned int bool apple_piodma_bootstrap_enabled(void) { - return enumerate; + return enumerate && of_machine_is_compatible("apple,j700") && + of_machine_is_compatible("apple,t8140"); } +EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_enabled); static void apple_piodma_put_supplier(void *data) { @@ -481,7 +533,8 @@ int apple_piodma_bootstrap_get(struct device *host, struct device **supplier) bool correct_host; int ret; - if (!enumerate) + if (!apple_piodma_bootstrap_enabled() || + !of_device_is_compatible(host->of_node, "apple,t8140-pcie")) return -ENODEV; node = of_parse_phandle(host->of_node, "apple,piodma", 0); if (!node) @@ -514,6 +567,7 @@ int apple_piodma_bootstrap_get(struct device *host, struct device **supplier) *supplier = &pdev->dev; return 0; } +EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_get); int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root) { @@ -539,20 +593,49 @@ int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root) diag->slots[0].result); return ret; } +EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_prime); + +int apple_piodma_radio_check(struct pci_dev *pdev) +{ + /* Acquire the supplier's permanent arena and PM/kexec guard ownership. */ + struct apple_piodma_diag *diag = smp_load_acquire(&active_diag); + struct pci_host_bridge *bridge = pci_find_host_bridge(pdev->bus); + struct device_node *host; + bool matches; + + if (!apple_piodma_bootstrap_enabled() || !diag) + return -ENODEV; + /* Pair with initialization's publication of the validated root/bus. */ + if (!smp_load_acquire(&diag->retained) || !bridge->dev.parent || + pdev->vendor != PCI_VENDOR_ID_MEDIATEK || + pci_domain_nr(pdev->bus) || pdev->bus->number != diag->secondary_bus || + PCI_SLOT(pdev->devfn) || + (pdev->devfn != PCI_DEVFN(0, 0) && pdev->devfn != PCI_DEVFN(0, 1)) || + pdev->device != (PCI_FUNC(pdev->devfn) ? 0x793b : 0x7932)) + return -ENODEV; + + host = of_parse_phandle(diag->dev->of_node, "apple,pcie-host", 0); + matches = host && host == bridge->dev.parent->of_node; + of_node_put(host); + return matches ? 0 : -ENODEV; +} +EXPORT_SYMBOL_GPL(apple_piodma_radio_check); static int apple_piodma_diag_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct apple_piodma_diag *diag; + unsigned int sleep_flags; int i, ret; - if (!enumerate || !of_machine_is_compatible("apple,j700") || - !of_machine_is_compatible("apple,t8140") || PAGE_SIZE != SZ_16K) + if (!apple_piodma_bootstrap_enabled() || PAGE_SIZE != SZ_16K) return -ENODEV; diag = kzalloc_obj(*diag); if (!diag) return -ENOMEM; diag->dev = dev; + diag->kexec_blocker.reason = + "Neo radio bootstrap retains DMA memory until full hardware reset"; for (i = 0; i < PIODMA_REQUEST_COUNT; i++) init_completion(&diag->slots[i].event); atomic_set(&diag->irq_count, 0); @@ -560,6 +643,21 @@ static int apple_piodma_diag_probe(struct platform_device *pdev) ret = apple_piodma_diag_iommu(diag); if (ret) goto err_free; + /* Publish guards and arena atomically with respect to system sleep. */ + if (!apple_piodma_trylock_sleep(&sleep_flags)) { + ret = -EPROBE_DEFER; + goto err_free; + } + diag->pm_notifier.notifier_call = apple_piodma_pm_notify; + ret = register_pm_notifier(&diag->pm_notifier); + if (ret) + goto err_sleep; + ret = kexec_block(&diag->kexec_blocker); + if (ret) { + if (ret == -EBUSY) + ret = -EPROBE_DEFER; + goto err_notifier; + } pm_runtime_enable(dev); ret = pm_runtime_resume_and_get(dev); if (ret < 0) @@ -574,19 +672,31 @@ static int apple_piodma_diag_probe(struct platform_device *pdev) dev_name(dev), diag); if (ret) goto err_arena; + /* The notifier and retained state also keep their device alive. */ + get_device(dev); platform_set_drvdata(pdev, diag); + /* Publish the arena, device reference and both guards together. */ + smp_store_release(&active_diag, diag); + unlock_system_sleep(sleep_flags); dev_info(dev, "bootstrap supplier ready; no command submitted during probe\n"); return 0; err_arena: /* No path after initialization may release ownership. */ - if (WARN_ON_ONCE(diag->retained)) + if (WARN_ON_ONCE(diag->retained)) { + unlock_system_sleep(sleep_flags); return 0; + } if (diag->arena) dma_free_coherent(dev, PIODMA_ARENA_SIZE, diag->arena, diag->dma); pm_runtime_put_sync(dev); err_pm: pm_runtime_disable(dev); + kexec_unblock(&diag->kexec_blocker); +err_notifier: + unregister_pm_notifier(&diag->pm_notifier); +err_sleep: + unlock_system_sleep(sleep_flags); err_free: kfree(diag); return dev_err_probe(dev, ret, "PIODMA diagnostic preflight failed\n"); diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index b876e0e841019..be57aa43c56f7 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2403,7 +2403,6 @@ static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) dev_err(pcie->dev, "root memory forwarding unavailable after assignment: %d\n", ret); goto close_config; } - pci_bus_add_devices(root->subordinate); for (function = 0; function < 2; function++) { endpoint = pci_get_slot(root->subordinate, PCI_DEVFN(0, function)); if (!endpoint) { @@ -2412,28 +2411,48 @@ static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) } if (pci_read_config_word(endpoint, PCI_COMMAND, &command)) { ret = -EIO; + } else if (endpoint->vendor != PCI_VENDOR_ID_MEDIATEK || + endpoint->device != (function ? 0x793b : 0x7932)) { + ret = -ENODEV; } else { dev_info(pcie->dev, "enumerated %s id=%04x:%04x command=%#x driver=%s\n", pci_name(endpoint), endpoint->vendor, endpoint->device, command, endpoint->driver ? endpoint->driver->name : "unbound"); if (command & PCI_COMMAND_MASTER) { - /* This experiment never permits endpoint bus mastering. */ + /* Admission requires cold, unbound endpoints. */ pci_clear_master(endpoint); ret = -EIO; } if (endpoint->driver) ret = -EBUSY; + if (!ret) { + /* + * The radios require the equivalent of ASPM performance + * policy before their drivers start. Keep this restriction + * on their shared link, including L1 substates and Clock + * PM, without changing policy on other PCIe links. The + * PCI core tracks the restriction across policy changes. + */ + ret = pci_disable_link_state(endpoint, + PCIE_LINK_STATE_ALL); + if (ret) + pci_err(endpoint, + "cannot disable radio link power states: %d\n", + ret); + } } pci_dev_put(endpoint); if (ret) break; } + if (!ret) + pci_bus_add_devices(root->subordinate); close_config: if (ret) WRITE_ONCE(pcie->neo_config_ready, false); pci_unlock_rescan_remove(); if (!ret) - dev_info(pcie->dev, "NEO_ENUMERATION_ONLY_READY; functions0/1 unbound, bus mastering off\n"); + dev_info(pcie->dev, "NEO_RADIO_READY; functions0/1 admitted with link power states disabled\n"); out: pci_dev_put(root); return ret; @@ -2471,7 +2490,9 @@ static int apple_pcie_probe(struct platform_device *pdev) } } - if (hw->root_bus_only && apple_piodma_bootstrap_enabled()) { + if (hw->root_bus_only && + of_device_is_compatible(dev->of_node, "apple,t8140-pcie") && + apple_piodma_bootstrap_enabled()) { ret = apple_piodma_bootstrap_get(dev, &piodma_supplier); if (ret) return dev_err_probe(dev, ret, "PIODMA supplier unavailable\n"); @@ -2559,6 +2580,11 @@ static int apple_pcie_probe(struct platform_device *pdev) if (pcie->piodma_supplier) { /* All fallible devres setup precedes publication of DMA pointers. */ + /* Retained bootstrap ownership forbids host module teardown. */ + if (!try_module_get(THIS_MODULE)) { + pci_host_common_remove(pdev); + return -ENODEV; + } ret = apple_pcie_neo_enumerate(bridge); if (ret) dev_err(dev, "enumeration experiment failed=%d; host/supplier retained, no retry\n", diff --git a/include/linux/pci-apple-piodma.h b/include/linux/pci-apple-piodma.h new file mode 100644 index 0000000000000..583da3e643c8a --- /dev/null +++ b/include/linux/pci-apple-piodma.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_PCI_APPLE_PIODMA_H +#define _LINUX_PCI_APPLE_PIODMA_H + +#include +#include + +struct pci_dev; + +#if IS_ENABLED(CONFIG_PCIE_APPLE_PIODMA_DIAG) +/* Verify that this Neo radio's supplier permanently protects retained DMA. */ +int apple_piodma_radio_check(struct pci_dev *pdev); +#else +static inline int apple_piodma_radio_check(struct pci_dev *pdev) +{ + return -ENODEV; +} +#endif + +#endif /* _LINUX_PCI_APPLE_PIODMA_H */ From 7cfa26cdca6c72fc9f7aea1f2311298733bb0bef Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:04:22 -0700 Subject: [PATCH 0110/2680] Bluetooth: build the gated Neo transport with modular Bluetooth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the transport tristate, require the retained supplier interlock before DMA publication, and pin activated ownership. Retire software callbacks on unsupported removal without claiming DMA quiescence. Enable the shared radio configuration and document the runtime contracts. [ryan: adapted to 7.2.10 — split by topic: the defconfig fragment hunks are carried by 900-config-tooling; documentation conflicts against the non-linear history of mt7932-neo.rst are resolved to this commit's text] --- .../device_drivers/wifi/mt7932-neo.rst | 70 ++++++++----------- drivers/bluetooth/Kconfig | 12 ++-- drivers/bluetooth/mt7932_bt_pcie.c | 42 +++++++++-- 3 files changed, 74 insertions(+), 50 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 0c9c68d82881c..c9e5d776036ec 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -23,21 +23,30 @@ Platform and configuration -------------------------- Use the separately reviewed T8140 PCIe bootstrap described in -``Documentation/PCI/controller/apple-t8140.rst``. Its explicit kernel opt-in is -``pcie_apple_piodma_diag.enumerate=1``. Public m1n1 must populate the J700 -``wifi0`` endpoint's own ``local-mac-address``; a zero placeholder is rejected. +``Documentation/PCI/controller/apple-t8140.rst``. Enumeration is enabled by +default only on the J700/T8140; ``pcie_apple_piodma_diag.enumerate=0`` disables +it. Public m1n1 must populate the J700 ``wifi0`` endpoint's own +``local-mac-address``; a zero placeholder is rejected. -The tested configuration includes ``CONFIG_MT7932_FULLMAC=m``, +The original hardware-tested configuration included ``CONFIG_MT7932_FULLMAC=m``, ``CONFIG_CFG80211=y``, ``CONFIG_BT_MTK7932_PCIE=y``, ``CONFIG_BT_BREDR=y``, ``CONFIG_BT_LE=y``, ``CONFIG_CRYPTO_AES=y`` and ``CONFIG_CRYPTO_CMAC=y``. -The bootstrap and Bluetooth experiment exclude suspend and kexec. +The shared build supports ``CONFIG_PCIE_APPLE=m``, +``CONFIG_PCIE_APPLE_PIODMA_DIAG=y``, ``CONFIG_PCIEASPM=y``, +``CONFIG_MT7932_FULLMAC=m`` and ``CONFIG_BT_MTK7932_PCIE=m`` with modular +Bluetooth/rfkill and sleep/kexec enabled. The active J700 supplier refuses sleep +and kexec at runtime while its arena is retained; other Macs keep their normal +behavior. Firmware loading is selected by both radio drivers. This shared +build still requires physical Neo qualification. + Use the ordinary cfg80211 regulatory database and applicable country policy. The validated first-admission fallback is kernel country 00 with firmware XZ. -After the root filesystem and the local firmware packages are available, -select PCIe ASPM performance policy before loading ``mt7932-fullmac``. This -matches the tested admission sequence. Bluetooth's gate defaults closed; -validate the cold, unbound ``14c3:793b`` function, enable +The host now disables link power states on the two radio endpoints before +driver binding; no global PCIe ASPM performance policy is required. Load +``mt7932-fullmac`` after the root filesystem and local firmware packages are +available. Bluetooth's gate defaults closed; load ``mt7932_bt_pcie`` with the +default ``enable=0``, validate the cold, unbound ``14c3:793b`` function, enable ``/sys/module/mt7932_bt_pcie/parameters/enable``, then request its PCI probe. Do not reprobe after a failed or uncertain Bluetooth admission. The tested Wi-Fi driver owns function 0 and Bluetooth owns function 1. @@ -106,34 +115,15 @@ from the result. A successful host serial command is not a test pass: require Qualification and remaining work -------------------------------- -The :download:`qualification artifact ` records the exact Image, -module and source identities and the physical results from 2026-10-02. -Both 2.4 and 5 GHz passed scan, WPA2, DHCP, gateway, DNS and verified HTTPS. -Switching bands performed normal peer retirement and fresh authentication. -Wi-Fi's checked module removal/reset/reload also passed before Bluetooth -admission. Five-GHz traffic concurrent with Bluetooth discovery and HCI -off/on passed 54/54 gateway replies and 18 HTTPS downloads with no driver or -kernel faults. The longer 2.4-GHz coexistence run recorded 53/54 replies; -it is explicitly not a zero-loss pass. The lost ping coincided with a -background scan; that timing does not establish a root cause. - -The earlier Bluetooth candidate passed two bonded AirPods reconnects and -30 seconds of AAC output, including a user-confirmed audible tone. Concurrent -headset audio was not repeated on the combined candidate because no AirPods -appeared during its discovery runs. - -The experimental implementation has concrete production merge blockers: - -* PCI bootstrap memory remains retained until external reset; controller - removal, suspend, kexec and reusable-arena lifetime are not qualified. -* Bluetooth lacks a complete PCI removal/quiescence contract. Suppressed - bind/unbind attributes do not make PCI device removal safe. -* Bluetooth's finite software queue limit is not HCI backpressure. Saturation - can drop a frame already accounted by the HCI scheduler; that path needs - correction and saturation qualification before production use. -* Arbitrary caller scan IEs, WPA3/SAE, required MFP, AP/P2P modes, general - country-package generation, roaming and long-duration reliability remain - outside this qualification. Broad compatibility must not be inferred from - the tested station and firmware combination. -* SCO/headset microphone, LE Audio/ISO and simultaneous headset audio remain - unqualified. These limitations are separate from the demonstrated A2DP path. +* PCI bootstrap memory remains retained until external reset. Sleep and kexec + are refused while it is retained; they are not hardware-qualified on the Neo. + Controller removal and memory reuse remain unqualified. +* Bluetooth PCI removal/quiescence is incomplete. Its software queue limit + does not provide HCI backpressure; saturation can drop an accounted frame. + Both require correction before production use. An activated Bluetooth module + cannot be unloaded; PCI removal only retires software callbacks and does not + establish DMA quiescence or release retained ownership. +* Arbitrary scan IEs, WPA3/SAE, required MFP, AP/P2P, general country-package + generation, roaming and long-duration reliability are unqualified. +* SCO/headset microphone, LE Audio/ISO and simultaneous headset audio are + unqualified. diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index 7ac07459925dc..85f164ed728dd 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -24,16 +24,18 @@ config BT_MTK select FW_LOADER config BT_MTK7932_PCIE - bool "J700 MT7932 PCIe Bluetooth experiment" - depends on BT=y && PCI && ARCH_APPLE && IOMMU_DMA - depends on !PM_SLEEP && !KEXEC_CORE + tristate "J700 MT7932 PCIe Bluetooth experiment" + depends on BT && PCI && ARCH_APPLE && IOMMU_DMA + depends on PCIE_APPLE_PIODMA_DIAG select FW_LOADER help Build the standalone J700 MT7932 transport experiment. Its probe gate defaults closed so board firmware can remain on the real root filesystem. It retains all DMA memory after activation, including terminal failures, - until a full external reset. Unbind and automatic recovery are excluded. - This does not qualify suspend or physical removal. + until a full external reset. After activation the module cannot unload. + The J700 bootstrap blocks suspend, hibernation and kexec at runtime while + DMA is retained. Unbind, automatic recovery and physical removal remain + unqualified. Other Apple machines do not pass the probe admission checks. config BT_HCIBTUSB tristate "HCI USB driver" diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 062522f170c3f..8a1e397ea82e0 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -4,7 +4,7 @@ * protocol contracts and public Linux interfaces. Original transport research: * DJ (DjDeveloperr), Ace (Acelogic), and Ryan Murray. * - * The built-in driver retains every device-visible allocation after mastering + * The driver retains every device-visible allocation after mastering * is enabled, including all terminal failures. No unbind, retry, suspend or * quiescence claim is made. Firmware and board inputs are supplied locally. */ @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -1090,9 +1091,22 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) goto fail; bt->irq_requested = true; + /* The retained supplier owns the runtime suspend/kexec interlock. */ + ret = apple_piodma_radio_check(pdev); + if (ret) + goto fail; + /* Refuse publication if an unload was already committed. */ + if (!try_module_get(THIS_MODULE)) { + ret = -ENODEV; + goto fail; + } pci_set_drvdata(pdev, bt); pm_runtime_forbid(&pdev->dev); - /* Every later outcome stays bound and retains all published ownership. */ + /* + * Every later outcome retains published DMA and its device/module owners, + * including the IRQ and HCI callbacks. Only external reset ends ownership. + */ + pci_dev_get(pdev); bt->retained = true; pci_set_master(pdev); ret = bt7932_download(bt); @@ -1127,12 +1141,29 @@ static void bt7932_shutdown(struct pci_dev *pdev) struct bt7932 *bt = pci_get_drvdata(pdev); WRITE_ONCE(bt->opened, false); - wake_up_all(&bt->wake_wait); - cancel_work_sync(&bt->tx_work); - skb_queue_purge(&bt->tx_queue); mutex_lock(&bt->lock); bt7932_fault_locked(bt, "shutdown retains DMA until external reset"); mutex_unlock(&bt->lock); + /* fault_locked disables without synchronizing while holding bt->lock. */ + synchronize_irq(bt->irq); + cancel_work_sync(&bt->tx_work); + skb_queue_purge(&bt->tx_queue); +} + +static void bt7932_remove(struct pci_dev *pdev) +{ + struct bt7932 *bt = pci_get_drvdata(pdev); + + /* Removal cannot establish DMA quiescence or release retained ownership. */ + bt7932_shutdown(pdev); + if (bt->registered) { + hci_unregister_dev(bt->hdev); + bt->registered = false; + } + /* Flush again after the last HCI callback has been retired. */ + cancel_work_sync(&bt->tx_work); + skb_queue_purge(&bt->tx_queue); + dev_err(&pdev->dev, "removal is unqualified; DMA, device and module retained until external reset\n"); } static const struct pci_device_id bt7932_ids[] = { @@ -1145,6 +1176,7 @@ static struct pci_driver bt7932_driver = { .name = "mt7932_bt_pcie", .id_table = bt7932_ids, .probe = bt7932_probe, + .remove = bt7932_remove, .shutdown = bt7932_shutdown, .driver = { .suppress_bind_attrs = true, From ca23a83bc401b68efe3260b61abcec985583587f Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 2 Oct 2026 22:49:21 -0700 Subject: [PATCH 0111/2680] PCI: apple: veto snapshot transitions after late supplier admission A /dev/snapshot file can send its prepare notifier before the radio supplier probes, then enter device PM through subsequent ioctls without repeating that notification. Add a device prepare veto alongside the notifier so retained DMA remains protected across those transitions. --- Documentation/PCI/controller/apple-t8140.rst | 5 ++++- .../pci/controller/pcie-apple-piodma-diag.c | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index 1a82e9447a3a6..3c1819927c116 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -16,7 +16,10 @@ diagnostic path closed, pass ``pcie_apple_piodma_diag.enumerate=0``. The shared kernel can keep suspend, hibernation and kexec support enabled. Once the supplier retains its arena, a PM notifier refuses sleep transitions -and a kexec interlock refuses loading or executing a replacement kernel, +and its device prepare callback provides a second veto. This also covers +``/dev/snapshot`` files opened before the supplier probes: later image, restore +and suspend ioctls enter device PM without repeating the prepare notifier. +A kexec interlock refuses loading or executing a replacement kernel, including an already loaded crash kernel. Image unloading remains permitted. These interlocks apply only to the active J700 supplier; other machines do not acquire them. A successful supplier probe retains this protection even if later diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index 997d300229697..507dca52932fe 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -144,6 +144,26 @@ static int apple_piodma_pm_notify(struct notifier_block *nb, } } +static int apple_piodma_pm_prepare(struct device *dev) +{ + struct apple_piodma_diag *diag = dev_get_drvdata(dev); + + if (!diag || !diag->arena) + return 0; + /* + * A /dev/snapshot file may have sent its PREPARE notifier before this + * supplier probed. Its later image/restore/suspend ioctls go straight + * to device PM, so retain a veto in the device prepare phase as well. + */ + dev_warn(dev, + "sleep refused: Neo radio bootstrap retains DMA memory until full hardware reset\n"); + return -EBUSY; +} + +static const struct dev_pm_ops apple_piodma_pm_ops = { + .prepare = apple_piodma_pm_prepare, +}; + static int apple_piodma_diag_root(struct apple_piodma_diag *diag, struct pci_dev *root) { int ret; @@ -718,6 +738,7 @@ static struct platform_driver apple_piodma_diag_driver = { .driver = { .name = "apple-piodma-diag", .of_match_table = apple_piodma_diag_match, + .pm = pm_sleep_ptr(&apple_piodma_pm_ops), .suppress_bind_attrs = true, }, }; From dab8d130784f9cd66cf12d3efdb26ecd76944d44 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 2 Oct 2026 22:49:42 -0700 Subject: [PATCH 0112/2680] PCI: apple: require completed radio admission on every probe Failed host validation leaves scanned children that generic PCI rescans can later publish. Publish a separate completed-admission state only after all host checks succeed, and require it before either radio touches hardware and in the host enable callback. Reapply the endpoint link-power restriction on every radio probe so recreated ASPM state cannot inherit an incompatible global policy. --- Documentation/PCI/controller/apple-t8140.rst | 8 ++++++++ drivers/bluetooth/mt7932_bt_pcie.c | 12 ++++++++---- drivers/net/wireless/mediatek/mt7932/mt7932.c | 10 ++++++++++ drivers/pci/controller/pcie-apple-piodma-diag.c | 16 ++++++++++++++-- drivers/pci/controller/pcie-apple-piodma-diag.h | 5 +++++ drivers/pci/controller/pcie-apple.c | 12 +++++++++++- include/linux/pci-apple-piodma.h | 2 +- 7 files changed, 57 insertions(+), 8 deletions(-) diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index 3c1819927c116..af47c90e3cfbd 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -33,6 +33,14 @@ neither a performance-policy boot argument nor a sysfs policy write is needed. Booting with ``pcie_aspm=off`` prevents the PCI core from applying this restriction and radio admission fails closed. +The supplier publishes radio admission only after both functions pass the +host's identity, cold-state, memory-window and link-power checks. Both radio +probes require that completed admission before accessing PCI or MMIO state; +the host's downstream device-enable callback enforces the same gate for other +drivers. This prevents a later generic rescan from enabling cached children of +a failed admission. Each radio probe reapplies the link-power restriction, so +a recreated ASPM link state cannot inherit an incompatible global policy. + The PIODMA arena remains allocated until external reset. Controller removal, teardown, memory reuse and arbitrary downstream devices are unqualified. The host and activated Bluetooth modules are pinned until reset. Do not remove diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 8a1e397ea82e0..9e5df8e10e65f 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -999,6 +999,14 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) !of_machine_is_compatible("apple,t8140") || pci_domain_nr(pdev->bus) || pdev->bus->number != 1 || pdev->devfn != PCI_DEVFN(0, 1)) return -ENODEV; + /* Reject rescanned children of a failed admission before hardware access. */ + ret = apple_piodma_radio_check(pdev); + if (ret) + return ret; + /* A remove/rescan may have recreated link state under the global policy. */ + ret = pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); + if (ret) + return ret; bt = kzalloc_obj(*bt); if (!bt) return -ENOMEM; @@ -1091,10 +1099,6 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) goto fail; bt->irq_requested = true; - /* The retained supplier owns the runtime suspend/kexec interlock. */ - ret = apple_piodma_radio_check(pdev); - if (ret) - goto fail; /* Refuse publication if an unload was already committed. */ if (!try_module_get(THIS_MODULE)) { ret = -ENODEV; diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 36ba1daad568d..0ba692be3bf75 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -3,6 +3,8 @@ * Clean-room contract: NEO_MT7932_DOWNLOAD_AND_NIC_CAP_CONTRACT.md. * All addresses here are BAR0 offsets, never chip/CPU physical addresses. */ +#include + #include "mt7932.h" static int mt_ring_alloc(struct mt7932 *m, struct mt7932_ring *q, @@ -1184,6 +1186,14 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) !of_device_is_compatible(bridge->dev.parent->of_node, "apple,t8140-pcie")) return -ENODEV; + /* Supplier admission and link policy must precede PCI/MMIO/DMA setup. */ + ret = apple_piodma_radio_check(pdev); + if (ret) + return ret; + /* A remove/rescan may have recreated link state under the global policy. */ + ret = pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); + if (ret) + return ret; domain = iommu_get_domain_for_dev(&pdev->dev); if (!domain || (domain->type != IOMMU_DOMAIN_DMA && domain->type != IOMMU_DOMAIN_DMA_FQ)) return dev_err_probe(&pdev->dev, -EINVAL, "translated DMA domain required\n"); diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index 507dca52932fe..7ced530ba7c45 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -98,6 +98,7 @@ struct apple_piodma_diag { u64 pointer_prefix; u8 secondary_bus; bool retained; + bool radio_admitted; struct notifier_block pm_notifier; struct kexec_blocker kexec_blocker; }; @@ -615,6 +616,17 @@ int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root) } EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_prime); +void apple_piodma_bootstrap_admitted(struct device *supplier) +{ + struct apple_piodma_diag *diag = dev_get_drvdata(supplier); + + if (WARN_ON_ONCE(!diag || !diag->retained)) + return; + /* Publish the host's completed cold, resource and ASPM admission checks. */ + smp_store_release(&diag->radio_admitted, true); +} +EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_admitted); + int apple_piodma_radio_check(struct pci_dev *pdev) { /* Acquire the supplier's permanent arena and PM/kexec guard ownership. */ @@ -625,8 +637,8 @@ int apple_piodma_radio_check(struct pci_dev *pdev) if (!apple_piodma_bootstrap_enabled() || !diag) return -ENODEV; - /* Pair with initialization's publication of the validated root/bus. */ - if (!smp_load_acquire(&diag->retained) || !bridge->dev.parent || + /* A retained supplier alone does not prove that host admission succeeded. */ + if (!smp_load_acquire(&diag->radio_admitted) || !bridge->dev.parent || pdev->vendor != PCI_VENDOR_ID_MEDIATEK || pci_domain_nr(pdev->bus) || pdev->bus->number != diag->secondary_bus || PCI_SLOT(pdev->devfn) || diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.h b/drivers/pci/controller/pcie-apple-piodma-diag.h index 46a1b03aa2496..adc0349eeaca1 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.h +++ b/drivers/pci/controller/pcie-apple-piodma-diag.h @@ -13,6 +13,7 @@ struct pci_dev; bool apple_piodma_bootstrap_enabled(void); int apple_piodma_bootstrap_get(struct device *host, struct device **supplier); int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root); +void apple_piodma_bootstrap_admitted(struct device *supplier); #else static inline bool apple_piodma_bootstrap_enabled(void) { @@ -28,5 +29,9 @@ static inline int apple_piodma_bootstrap_prime(struct device *supplier, struct p { return -ENODEV; } + +static inline void apple_piodma_bootstrap_admitted(struct device *supplier) +{ +} #endif #endif diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index be57aa43c56f7..7a0bd3019e815 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -1840,6 +1841,13 @@ static int apple_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_d struct of_phandle_args iommu_spec = {}; int idx, err; + /* A generic rescan must not enable cached children of failed admission. */ + if (pcie->piodma_supplier && !pci_is_root_bus(pdev->bus)) { + err = apple_piodma_radio_check(pdev); + if (err) + return err; + } + /* Also cover functions discovered by a later hotplug or rescan. */ if (pcie->hw->tunneled) pdev->dev_flags |= PCI_DEV_FLAGS_NO_D3; @@ -2445,8 +2453,10 @@ static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) if (ret) break; } - if (!ret) + if (!ret) { + apple_piodma_bootstrap_admitted(pcie->piodma_supplier); pci_bus_add_devices(root->subordinate); + } close_config: if (ret) WRITE_ONCE(pcie->neo_config_ready, false); diff --git a/include/linux/pci-apple-piodma.h b/include/linux/pci-apple-piodma.h index 583da3e643c8a..2026d1f51602e 100644 --- a/include/linux/pci-apple-piodma.h +++ b/include/linux/pci-apple-piodma.h @@ -8,7 +8,7 @@ struct pci_dev; #if IS_ENABLED(CONFIG_PCIE_APPLE_PIODMA_DIAG) -/* Verify that this Neo radio's supplier permanently protects retained DMA. */ +/* Require completed Neo admission and permanent retained-DMA protection. */ int apple_piodma_radio_check(struct pci_dev *pdev); #else static inline int apple_piodma_radio_check(struct pci_dev *pdev) From cfd5b53eb56ef9d883195b83094802469f04d1a4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 00:05:55 -0400 Subject: [PATCH 0113/2680] iommu: apple-dart: support tunneled PCIe-C DMA aliases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — Asahi 7.2 programs only the first stream and remaps the rest (1143d0263e3a) and takes the DMA window from the master cfg (3658355cfaba); a tunnel DART keeps per-stream TCR/TTBR writes and invalidates through apple_dart_remaps_streams(), and the alias offset is applied to the cfg window] Signed-off-by: DjDeveloperr --- drivers/iommu/Kconfig | 1 + drivers/iommu/apple-dart.c | 571 +++++++++++++++++++++++++++++++++---- 2 files changed, 518 insertions(+), 54 deletions(-) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 6e07bd69467a3..f334a6278702c 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -305,6 +305,7 @@ config APPLE_DART tristate "Apple DART IOMMU Support" depends on ARCH_APPLE || COMPILE_TEST depends on !GENERIC_ATOMIC64 # for IOMMU_IO_PGTABLE_DART + select APPLE_TUNABLE select IOMMU_API select IOMMU_IO_PGTABLE_DART help diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 9d1f6cd6649ac..a0cc65c836f9f 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,7 @@ #include #include #include +#include #include #include @@ -39,6 +41,23 @@ #define DART_MAX_STREAMS 256 #define DART_MAX_TTBR 4 #define MAX_DARTS_PER_DEVICE 3 +#define DART_PCIEC_PAGE_SHIFT 14 +#define DART_PCIEC_ADDR_WIDTH 42 +#define DART_PCIEC_STREAMS 64 + +/* + * Temporary bring-up gate for the cable-powered DART behind PCIe-C. The + * normal storage, display and USB4 DARTs never consult this parameter. + */ +static unsigned int apple_dart_debug_stage; +module_param_named(debug_stage, apple_dart_debug_stage, uint, 0644); +MODULE_PARM_DESC(debug_stage, + "Stop tunneled PCIe-C DART setup after the selected debug phase"); + +static unsigned int apple_dart_trace_delay_ms; +module_param_named(tunnel_trace_delay_ms, apple_dart_trace_delay_ms, uint, 0644); +MODULE_PARM_DESC(tunnel_trace_delay_ms, + "Delay after PCIe-C DART trace boundaries (0 disables tracing)"); /* Common registers */ @@ -235,9 +254,12 @@ struct apple_dart { u32 supports_bypass : 1; u32 four_level : 1; u32 locked : 1; + u32 tunneled : 1; + struct apple_tunable *tunables; dma_addr_t dma_min; dma_addr_t dma_max; + dma_addr_t dma_offset; struct iommu_group *sid2group[DART_MAX_STREAMS]; struct iommu_device iommu; @@ -248,6 +270,66 @@ struct apple_dart { u64 *locked_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR]; }; +/* + * PCIe-C is a cable-powered aperture and can report a failed transaction as + * an asynchronous external abort unless each access is completed before the + * next instruction retires. A live /dev/mem diagnostic using the same + * Device-nGnRnE attributes and the exact 64-SID reset sequence is stable when + * every access is followed by dsb sy + isb. ioremap_np() supplies the memory + * type, but writel() only supplies the normal I/O ordering barrier on arm64. + * Match the proven access semantics for the tunneled DART while leaving all + * always-on DARTs on the normal fast path. + */ +static inline void apple_dart_writel(struct apple_dart *dart, u32 value, + u32 offset) +{ + writel(value, dart->regs + offset); + if (unlikely(dart->tunneled)) { + mb(); + isb(); + } +} + +static inline u32 apple_dart_readl(struct apple_dart *dart, u32 offset) +{ + u32 value = readl(dart->regs + offset); + + if (unlikely(dart->tunneled)) { + mb(); + isb(); + } + + return value; +} + +static void apple_dart_apply_tunables(struct apple_dart *dart) +{ + size_t i; + + for (i = 0; i < dart->tunables->sz; ++i) { + u32 old_value, value; + + old_value = apple_dart_readl(dart, + dart->tunables->values[i].offset); + value = old_value & ~dart->tunables->values[i].mask; + value |= dart->tunables->values[i].value; + if (value != old_value) + apple_dart_writel(dart, value, + dart->tunables->values[i].offset); + } +} + +static void apple_dart_tunnel_trace(struct apple_dart *dart, + unsigned int step, const char *operation) +{ + if (!dart->tunneled || !READ_ONCE(apple_dart_trace_delay_ms)) + return; + + dev_info(dart->dev, "PCIe-C DART trace %u armed: %s\n", + step, operation); + msleep(READ_ONCE(apple_dart_trace_delay_ms)); +} + /* * Convenience struct to identify streams. * @@ -286,6 +368,7 @@ struct apple_dart_domain { bool finalized; u64 mask; + dma_addr_t dma_offset; struct mutex init_lock; struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; @@ -336,6 +419,17 @@ static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom) return container_of(dom, struct apple_dart_domain, domain); } +/* + * Asahi's stream remapping programs only the first stream of a map and + * remaps the others onto it. A PCIe-C tunnel DART keeps programming and + * invalidating every stream on its own, with the ordered accesses of + * apple_dart_writel(). + */ +static bool apple_dart_remaps_streams(struct apple_dart *dart) +{ + return dart->hw->enable_remap && !dart->tunneled; +} + static void apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map, int levels) { @@ -349,6 +443,12 @@ apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map, int l WARN_ON(levels != 3 && levels != 4); WARN_ON(levels == 4 && !dart->four_level); + if (!apple_dart_remaps_streams(dart)) { + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) + apple_dart_writel(dart, tcr, DART_TCR(dart, sid)); + return; + } + /* Search for first enabled sid and enable translation. */ sid = find_first_bit(stream_map->sidmap, dart->num_streams); if (sid >= dart->num_streams) @@ -368,7 +468,8 @@ static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map) /* This will also disable remap enabled in apple_dart_hw_enable_translation(). */ for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) - writel(dart->hw->tcr_disabled, dart->regs + DART_TCR(dart, sid)); + apple_dart_writel(dart, dart->hw->tcr_disabled, + DART_TCR(dart, sid)); } static void @@ -379,8 +480,8 @@ apple_dart_hw_enable_bypass(struct apple_dart_stream_map *stream_map) WARN_ON(!stream_map->dart->supports_bypass); for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) - writel(dart->hw->tcr_bypass, - dart->regs + DART_TCR(dart, sid)); + apple_dart_writel(dart, dart->hw->tcr_bypass, + DART_TCR(dart, sid)); } static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map, @@ -390,6 +491,16 @@ static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map, int sid; WARN_ON(paddr & ((1 << dart->hw->ttbr_shift) - 1)); + if (!apple_dart_remaps_streams(dart)) { + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) + apple_dart_writel(dart, + dart->hw->ttbr_valid | + (paddr >> dart->hw->ttbr_shift) << + dart->hw->ttbr_addr_field_shift, + DART_TTBR(dart, sid, idx)); + return; + } + /* Search for the first used stream and set its TTBR. All other sids * of this DART are handled via the remap feature and must not set a * valid TTBR. @@ -410,7 +521,7 @@ static void apple_dart_hw_clear_ttbr(struct apple_dart_stream_map *stream_map, int sid; for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) - writel(0, dart->regs + DART_TTBR(dart, sid, idx)); + apple_dart_writel(dart, 0, DART_TTBR(dart, sid, idx)); } static void @@ -433,7 +544,7 @@ apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) phys_addr_t phys; u64 *l1_tbl; - ttbr = readl(dart->regs + DART_TTBR(dart, sid, idx)); + ttbr = apple_dart_readl(dart, DART_TTBR(dart, sid, idx)); if (!(ttbr & dart->hw->ttbr_valid)) { dev_err(dart->dev, "Invalid ttbr[%u] for locked dart\n", @@ -502,23 +613,25 @@ static int apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map, u32 command) { + struct apple_dart *dart = stream_map->dart; unsigned long flags; int ret, i; u32 command_reg; - spin_lock_irqsave(&stream_map->dart->lock, flags); + spin_lock_irqsave(&dart->lock, flags); - for (i = 0; i < BITS_TO_U32(stream_map->dart->num_streams); i++) - writel(stream_map->sidmap[i], - stream_map->dart->regs + DART_T8020_STREAM_SELECT + 4 * i); - writel(command, stream_map->dart->regs + DART_T8020_STREAM_COMMAND); + for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) + apple_dart_writel(dart, stream_map->sidmap[i], + DART_T8020_STREAM_SELECT + 4 * i); + apple_dart_writel(dart, command, DART_T8020_STREAM_COMMAND); - ret = readl_poll_timeout_atomic( - stream_map->dart->regs + DART_T8020_STREAM_COMMAND, command_reg, + ret = read_poll_timeout_atomic( + apple_dart_readl, command_reg, !(command_reg & DART_T8020_STREAM_COMMAND_BUSY), 1, - DART_STREAM_COMMAND_BUSY_TIMEOUT); + DART_STREAM_COMMAND_BUSY_TIMEOUT, false, dart, + DART_T8020_STREAM_COMMAND); - spin_unlock_irqrestore(&stream_map->dart->lock, flags); + spin_unlock_irqrestore(&dart->lock, flags); if (ret) { dev_err(stream_map->dart->dev, @@ -544,17 +657,19 @@ apple_dart_t8110_hw_tlb_command(struct apple_dart_stream_map *stream_map, for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { u32 val = FIELD_PREP(DART_T8110_TLB_CMD_OP, command) | FIELD_PREP(DART_T8110_TLB_CMD_STREAM, sid); - writel(val, dart->regs + DART_T8110_TLB_CMD); + apple_dart_writel(dart, val, DART_T8110_TLB_CMD); - ret = readl_poll_timeout_atomic( - dart->regs + DART_T8110_TLB_CMD, val, + ret = read_poll_timeout_atomic( + apple_dart_readl, val, !(val & DART_T8110_TLB_CMD_BUSY), 1, - DART_STREAM_COMMAND_BUSY_TIMEOUT); + DART_STREAM_COMMAND_BUSY_TIMEOUT, false, dart, + DART_T8110_TLB_CMD); if (ret) break; /* Since remap is used only the first sid needs to be invalidated. */ - break; + if (apple_dart_remaps_streams(dart)) + break; } spin_unlock_irqrestore(&dart->lock, flags); @@ -612,22 +727,134 @@ static int apple_dart_hw_reset(struct apple_dart *dart) stream_map.dart = dart; bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS); bitmap_set(stream_map.sidmap, 0, dart->num_streams); + + apple_dart_tunnel_trace(dart, 20, "disable DMA for every stream"); apple_dart_hw_disable_dma(&stream_map); + apple_dart_tunnel_trace(dart, 21, "clear every stream TTBR"); apple_dart_hw_clear_all_ttbrs(&stream_map); /* enable all streams globally since TCR is used to control isolation */ + apple_dart_tunnel_trace(dart, 22, "enable all streams globally"); for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) - writel(U32_MAX, dart->regs + dart->hw->enable_streams + 4 * i); + apple_dart_writel(dart, U32_MAX, + dart->hw->enable_streams + 4 * i); /* clear any pending errors before the interrupt is unmasked */ - writel(readl(dart->regs + dart->hw->error), dart->regs + dart->hw->error); + apple_dart_tunnel_trace(dart, 23, "read and clear DART error status"); + apple_dart_writel(dart, apple_dart_readl(dart, dart->hw->error), + dart->hw->error); + apple_dart_tunnel_trace(dart, 24, "clear T8110 error mask"); if (dart->hw->type == DART_T8110) - writel(0, dart->regs + DART_T8110_ERROR_MASK); + apple_dart_writel(dart, 0, DART_T8110_ERROR_MASK); + apple_dart_tunnel_trace(dart, 25, "invalidate every stream TLB"); return dart->hw->invalidate_tlb(&stream_map); } +static int apple_dart_debug_reset(struct apple_dart *dart) +{ + struct apple_dart_stream_map stream_map; + u32 value; + int i, sid; + + if (dart->hw->type != DART_T8110 || apple_dart_debug_stage < 30) + return 0; + + stream_map.dart = dart; + bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS); + set_bit(0, stream_map.sidmap); + + apple_dart_writel(dart, U32_MAX, dart->hw->enable_streams); + dev_info(dart->dev, + "PCIe-C DART reset stage 30: stream word 0 enabled\n"); + if (apple_dart_debug_stage == 30) + return -EAGAIN; + + for (i = 1; i < BITS_TO_U32(dart->num_streams); i++) + apple_dart_writel(dart, U32_MAX, + dart->hw->enable_streams + 4 * i); + dev_info(dart->dev, + "PCIe-C DART reset stage 31: all stream words enabled\n"); + if (apple_dart_debug_stage == 31) + return -EAGAIN; + + value = apple_dart_readl(dart, DART_TCR(dart, 0)); + dev_info(dart->dev, + "PCIe-C DART reset stage 32: SID0 TCR initial value=%#x\n", + value); + if (apple_dart_debug_stage == 32) + return -EAGAIN; + + apple_dart_writel(dart, dart->hw->tcr_disabled, + DART_TCR(dart, 0)); + dev_info(dart->dev, + "PCIe-C DART reset stage 33: SID0 TCR disabled\n"); + if (apple_dart_debug_stage == 33) + return -EAGAIN; + + value = apple_dart_readl(dart, DART_TCR(dart, 0)); + dev_info(dart->dev, + "PCIe-C DART reset stage 34: SID0 TCR readback=%#x\n", + value); + if (apple_dart_debug_stage == 34) + return -EAGAIN; + + for (sid = 1; sid < dart->num_streams; sid++) + apple_dart_writel(dart, dart->hw->tcr_disabled, + DART_TCR(dart, sid)); + dev_info(dart->dev, + "PCIe-C DART reset stage 35: all %u TCRs disabled\n", + dart->num_streams); + if (apple_dart_debug_stage == 35) + return -EAGAIN; + + apple_dart_writel(dart, 0, DART_TTBR(dart, 0, 0)); + dev_info(dart->dev, + "PCIe-C DART reset stage 36: SID0 TTBR0 cleared\n"); + if (apple_dart_debug_stage == 36) + return -EAGAIN; + + value = apple_dart_readl(dart, DART_TTBR(dart, 0, 0)); + dev_info(dart->dev, + "PCIe-C DART reset stage 37: SID0 TTBR0 readback=%#x\n", + value); + if (apple_dart_debug_stage == 37) + return -EAGAIN; + + for (sid = 1; sid < dart->num_streams; sid++) + for (i = 0; i < dart->hw->ttbr_count; i++) + apple_dart_writel(dart, 0, + DART_TTBR(dart, sid, i)); + dev_info(dart->dev, + "PCIe-C DART reset stage 38: all TTBRs cleared\n"); + if (apple_dart_debug_stage == 38) + return -EAGAIN; + + value = apple_dart_readl(dart, dart->hw->error); + dev_info(dart->dev, + "PCIe-C DART reset stage 39: error status=%#x\n", value); + if (apple_dart_debug_stage == 39) + return -EAGAIN; + + apple_dart_writel(dart, value, dart->hw->error); + dev_info(dart->dev, + "PCIe-C DART reset stage 40: error status cleared\n"); + if (apple_dart_debug_stage == 40) + return -EAGAIN; + + apple_dart_writel(dart, 0, DART_T8110_ERROR_MASK); + dev_info(dart->dev, + "PCIe-C DART reset stage 41: error mask cleared\n"); + if (apple_dart_debug_stage == 41) + return -EAGAIN; + + i = dart->hw->invalidate_tlb(&stream_map); + dev_info(dart->dev, + "PCIe-C DART reset stage 42: SID0 TLB flush returned %d\n", i); + return i ?: -EAGAIN; +} + static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain) { int i, j; @@ -679,7 +906,9 @@ static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, if (!ops) return 0; - return ops->iova_to_phys(ops, iova & dart_domain->mask); + return ops->iova_to_phys(ops, + (iova + dart_domain->dma_offset) & + dart_domain->mask); } static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, @@ -693,7 +922,9 @@ static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, if (!ops) return -ENODEV; - return ops->map_pages(ops, iova & dart_domain->mask, paddr, pgsize, + return ops->map_pages(ops, + (iova + dart_domain->dma_offset) & + dart_domain->mask, paddr, pgsize, pgcount, prot, gfp, mapped); } @@ -705,7 +936,9 @@ static size_t apple_dart_unmap_pages(struct iommu_domain *domain, struct apple_dart_domain *dart_domain = to_dart_domain(domain); struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; - return ops->unmap_pages(ops, iova & dart_domain->mask, pgsize, pgcount, + return ops->unmap_pages(ops, + (iova + dart_domain->dma_offset) & + dart_domain->mask, pgsize, pgcount, gather); } @@ -720,7 +953,8 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, /* enable all streams globally since TCR is used to control isolation */ for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) - writel(U32_MAX, dart->regs + dart->hw->enable_streams + 4 * i); + apple_dart_writel(dart, U32_MAX, + dart->hw->enable_streams + 4 * i); for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i) { u64 ttbr = virt_to_phys(pgtbl_cfg->apple_dart_cfg.ttbr[i]); @@ -757,8 +991,9 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, { struct apple_dart *dart = cfg->stream_maps[0].dart; struct io_pgtable_cfg pgtbl_cfg; - u32 ias = min_t(u32, dart->ias, fls64(cfg->dma_max)); dma_addr_t dma_max = cfg->dma_max; + u32 ias = min_t(u32, dart->ias, + fls64(dma_max + dart->dma_offset)); int ret = 0; int i, j; @@ -795,13 +1030,14 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, sid = find_first_bit(sidmap, dart->num_streams); WARN_ON((sid < 0) || bitmap_weight(sidmap, dart->num_streams) > 1); - ttbr = readl(dart->regs + DART_TTBR(dart, sid, 0)); + ttbr = apple_dart_readl(dart, DART_TTBR(dart, sid, 0)); WARN_ON(!(ttbr & dart->hw->ttbr_valid)); /* If the DART is locked, we need to keep the translation level count. */ if (dart->hw->tcr_4level && dart->ias > 36) { - if (readl(dart->regs + DART_TCR(dart, sid)) & dart->hw->tcr_4level) { + if (apple_dart_readl(dart, DART_TCR(dart, sid)) & + dart->hw->tcr_4level) { if (ias < 37) { dev_info(dart->dev, "Expanded to ias=37 due to lock\n"); pgtbl_cfg.ias = 37; @@ -833,6 +1069,7 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, dart_domain->domain.geometry.aperture_start = cfg->dma_min; dart_domain->domain.geometry.aperture_end = dma_max; dart_domain->domain.geometry.force_aperture = true; + dart_domain->dma_offset = dart->dma_offset; dart_domain->finalized = true; @@ -1334,10 +1571,10 @@ static irqreturn_t apple_dart_t8020_irq(int irq, void *dev) { struct apple_dart *dart = dev; const char *fault_name = NULL; - u32 error = readl(dart->regs + DART_T8020_ERROR); + u32 error = apple_dart_readl(dart, DART_T8020_ERROR); u32 error_code = FIELD_GET(DART_T8020_ERROR_CODE, error); - u32 addr_lo = readl(dart->regs + DART_T8020_ERROR_ADDR_LO); - u32 addr_hi = readl(dart->regs + DART_T8020_ERROR_ADDR_HI); + u32 addr_lo = apple_dart_readl(dart, DART_T8020_ERROR_ADDR_LO); + u32 addr_hi = apple_dart_readl(dart, DART_T8020_ERROR_ADDR_HI); u64 addr = addr_lo | (((u64)addr_hi) << 32); u8 stream_idx = FIELD_GET(DART_T8020_ERROR_STREAM, error); @@ -1363,7 +1600,7 @@ static irqreturn_t apple_dart_t8020_irq(int irq, void *dev) "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx", error, stream_idx, error_code, fault_name, addr); - writel(error, dart->regs + DART_T8020_ERROR); + apple_dart_writel(dart, error, DART_T8020_ERROR); return IRQ_HANDLED; } @@ -1371,10 +1608,10 @@ static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) { struct apple_dart *dart = dev; const char *fault_name = NULL; - u32 error = readl(dart->regs + DART_T8110_ERROR); + u32 error = apple_dart_readl(dart, DART_T8110_ERROR); u32 error_code = FIELD_GET(DART_T8110_ERROR_CODE, error); - u32 addr_lo = readl(dart->regs + DART_T8110_ERROR_ADDR_LO); - u32 addr_hi = readl(dart->regs + DART_T8110_ERROR_ADDR_HI); + u32 addr_lo = apple_dart_readl(dart, DART_T8110_ERROR_ADDR_LO); + u32 addr_hi = apple_dart_readl(dart, DART_T8110_ERROR_ADDR_HI); u64 addr = addr_lo | (((u64)addr_hi) << 32); u8 stream_idx = FIELD_GET(DART_T8110_ERROR_STREAM, error); @@ -1402,9 +1639,10 @@ static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx", error, stream_idx, error_code, fault_name, addr); - writel(error, dart->regs + DART_T8110_ERROR); + apple_dart_writel(dart, error, DART_T8110_ERROR); for (int i = 0; i < BITS_TO_U32(dart->num_streams); i++) - writel(U32_MAX, dart->regs + DART_T8110_ERROR_STREAMS + 4 * i); + apple_dart_writel(dart, U32_MAX, + DART_T8110_ERROR_STREAMS + 4 * i); return IRQ_HANDLED; } @@ -1422,7 +1660,43 @@ static irqreturn_t apple_dart_irq(int irq, void *dev) static bool apple_dart_is_locked(struct apple_dart *dart) { - return !!(readl(dart->regs + dart->hw->lock) & dart->hw->lock_bit); + return !!(apple_dart_readl(dart, dart->hw->lock) & dart->hw->lock_bit); +} + +static void apple_dart_iounmap_np(void *data) +{ + iounmap((__force void __iomem *)data); +} + +static void __iomem * +apple_dart_platform_ioremap_np_resource(struct platform_device *pdev, + unsigned int index, + struct resource **res_out) +{ + struct device *dev = &pdev->dev; + struct resource *res; + void __iomem *regs; + int ret; + + res = platform_get_resource(pdev, IORESOURCE_MEM, index); + if (!res) + return ERR_PTR(-EINVAL); + + if (!devm_request_mem_region(dev, res->start, resource_size(res), + dev_name(dev))) + return ERR_PTR(-EBUSY); + + regs = ioremap_np(res->start, resource_size(res)); + if (!regs) + return ERR_PTR(-ENOMEM); + + ret = devm_add_action_or_reset(dev, apple_dart_iounmap_np, + (__force void *)regs); + if (ret) + return ERR_PTR(ret); + + *res_out = res; + return regs; } static int apple_dart_probe(struct platform_device *pdev) @@ -1433,6 +1707,18 @@ static int apple_dart_probe(struct platform_device *pdev) struct apple_dart *dart; struct device *dev = &pdev->dev; u64 dma_range[2]; + bool debug_stop = false; + bool tunneled = dev->of_node->parent && + of_node_name_prefix(dev->of_node->parent, + "usb4-pcie-tunnel-"); + + if (tunneled) { + dev_info(dev, "PCIe-C DART probe entered\n"); + if (apple_dart_debug_stage == 10) { + dev_info(dev, "PCIe-C DART debug stage 10: probe entry\n"); + return -EAGAIN; + } + } dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL); if (!dart) @@ -1440,16 +1726,43 @@ static int apple_dart_probe(struct platform_device *pdev) dart->dev = dev; dart->hw = of_device_get_match_data(dev); + dart->tunneled = tunneled; spin_lock_init(&dart->lock); + platform_set_drvdata(pdev, dart); - dart->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res); + /* + * PCIe-C is a cable-powered aperture whose writes must complete before + * the following transaction. The normal arm64 ioremap() mapping is + * Device-nGnRE and allowed the first posted DART reset write to surface as + * an asynchronous external abort at a later instruction. The identical + * reset sequence is stable through /dev/mem's Device-nGnRnE mapping, and + * PCI configuration forwarding already requires the same non-posted + * ordering. Keep ordinary always-on DARTs on their existing mapping. + */ + if (tunneled) + dart->regs = apple_dart_platform_ioremap_np_resource(pdev, 0, + &res); + else + dart->regs = devm_platform_get_and_ioremap_resource(pdev, 0, + &res); if (IS_ERR(dart->regs)) return PTR_ERR(dart->regs); + if (tunneled && apple_dart_debug_stage == 11) { + dev_info(dev, "PCIe-C DART debug stage 11: registers mapped\n"); + return -EAGAIN; + } if (resource_size(res) < 0x4000) { dev_err(dev, "MMIO region too small (%pr)\n", res); return -EINVAL; } + if (tunneled) { + dart->tunables = devm_apple_tunable_parse(dev, dev->of_node, + "apple,tunable", res); + if (IS_ERR(dart->tunables)) + return dev_err_probe(dev, PTR_ERR(dart->tunables), + "failed to parse PCIe-C DART tunables\n"); + } dart->irq = platform_get_irq(pdev, 0); if (dart->irq < 0) @@ -1460,6 +1773,7 @@ static int apple_dart_probe(struct platform_device *pdev) return ret; dart->num_clks = ret; + apple_dart_tunnel_trace(dart, 10, "enable PCIe-C DART clocks"); ret = clk_bulk_prepare_enable(dart->num_clks, dart->clks); if (ret) return ret; @@ -1471,9 +1785,51 @@ static int apple_dart_probe(struct platform_device *pdev) ret = devm_pm_runtime_enable(dev); if (ret) goto err_clk_disable; + if (tunneled && apple_dart_debug_stage == 12) { + dev_info(dev, + "PCIe-C DART debug stage 12: power and clocks enabled\n"); + ret = -EAGAIN; + debug_stop = true; + goto err_clk_disable; + } + if (tunneled) { + dev_info(dev, "applying %zu firmware PCIe-C DART tunables\n", + dart->tunables->sz); + apple_dart_tunnel_trace(dart, 11, "apply firmware DART tunables"); + apple_dart_apply_tunables(dart); + dev_info(dev, "firmware PCIe-C DART tunables applied\n"); + if (apple_dart_debug_stage == 13) { + dev_info(dev, + "PCIe-C DART debug stage 13: firmware tunables applied\n"); + ret = -EAGAIN; + debug_stop = true; + goto err_clk_disable; + } + } - dart_params[0] = readl(dart->regs + DART_PARAMS1); - dart_params[1] = readl(dart->regs + DART_PARAMS2); + /* + * The cable-powered T8110 DART does not provide a safely probeable + * parameter block at this point in PCIe tunnel activation. The SError + * raised by these reads is asynchronous, which made the following trace + * boundary look guilty on different boots. Apple instead publishes the + * complete immutable topology in ADT: 16 KiB pages, 42-bit addresses and + * 64 SIDs. Use that description before making any DART MMIO access. + */ + if (tunneled) { + dart->pgsize = 1 << DART_PCIEC_PAGE_SHIFT; + dart->supports_bypass = true; + dart->ias = DART_PCIEC_ADDR_WIDTH; + dart->oas = DART_PCIEC_ADDR_WIDTH; + dart->num_streams = DART_PCIEC_STREAMS; + dart->four_level = true; + dev_info(dev, + "PCIe-C DART using fixed topology: pagesize %#x, %u streams, AS %u -> %u\n", + dart->pgsize, dart->num_streams, dart->ias, dart->oas); + goto params_done; + } + + dart_params[0] = apple_dart_readl(dart, DART_PARAMS1); + dart_params[1] = apple_dart_readl(dart, DART_PARAMS2); dart->pgsize = 1 << FIELD_GET(DART_PARAMS1_PAGE_SHIFT, dart_params[0]); dart->supports_bypass = dart_params[1] & DART_PARAMS2_BYPASS_SUPPORT; @@ -1486,17 +1842,20 @@ static int apple_dart_probe(struct platform_device *pdev) break; case DART_T8110: - dart_params[2] = readl(dart->regs + DART_T8110_PARAMS3); - dart_params[3] = readl(dart->regs + DART_T8110_PARAMS4); + dart_params[2] = apple_dart_readl(dart, DART_T8110_PARAMS3); dart->ias = FIELD_GET(DART_T8110_PARAMS3_VA_WIDTH, dart_params[2]); dart->oas = FIELD_GET(DART_T8110_PARAMS3_PA_WIDTH, dart_params[2]); - dart->num_streams = FIELD_GET(DART_T8110_PARAMS4_NUM_SIDS, dart_params[3]); + dart_params[3] = apple_dart_readl(dart, DART_T8110_PARAMS4); + dart->num_streams = FIELD_GET(DART_T8110_PARAMS4_NUM_SIDS, + dart_params[3]); dart->four_level = dart->ias > 36; break; } +params_done: dart->dma_min = 0; dart->dma_max = DMA_BIT_MASK(dart->ias); + dart->dma_offset = 0; ret = of_property_read_u64_array(dev->of_node, "apple,dma-range", dma_range, 2); if (ret == -EINVAL) { @@ -1516,6 +1875,46 @@ static int apple_dart_probe(struct platform_device *pdev) &dart->dma_min, &dart->dma_max); } + ret = of_property_read_u64(dev->of_node, "apple,dma-offset", + &dart->dma_offset); + if (ret == -EINVAL) { + dart->dma_offset = 0; + ret = 0; + } else if (ret) { + goto err_clk_disable; + } else if (dart->dma_offset > DMA_BIT_MASK(dart->ias) || + dart->dma_max > DMA_BIT_MASK(dart->ias) - dart->dma_offset) { + dev_err(dev, "Invalid DMA offset for ias=%u\n", dart->ias); + ret = -EINVAL; + goto err_clk_disable; + } else { + dev_info(dev, + "aliasing DMA range %pad..%pad at DART offset %pad\n", + &dart->dma_min, &dart->dma_max, &dart->dma_offset); + } + + /* + * Older FDTs described the T602x PCIe-C DART-visible 1 TiB window + * directly. PCIe endpoints cannot issue those addresses: the host bridge + * accepts the upper 2 GiB of its 32-bit bus window and adds the 1 TiB + * alias before DART translation. Keep those FDTs bootable while returning + * the low bus address to DMA clients. + */ + if (!dart->dma_offset && tunneled && + (of_machine_is_compatible("apple,t6020") || + of_machine_is_compatible("apple,t6021")) && + ((dart->dma_min == 0 && + dart->dma_max == DMA_BIT_MASK(dart->ias)) || + (dart->dma_min == BIT_ULL(40) && + dart->dma_max == BIT_ULL(40) + BIT_ULL(32) - 1))) { + dart->dma_min = BIT_ULL(31); + dart->dma_max = DMA_BIT_MASK(32); + dart->dma_offset = BIT_ULL(40); + dev_info(dev, + "using T602x PCIe-C bus DMA window %pad..%pad at DART offset %pad\n", + &dart->dma_min, &dart->dma_max, &dart->dma_offset); + } + if (dart->num_streams > DART_MAX_STREAMS) { dev_err(&pdev->dev, "Too many streams (%d > %d)\n", dart->num_streams, DART_MAX_STREAMS); @@ -1523,28 +1922,83 @@ static int apple_dart_probe(struct platform_device *pdev) goto err_clk_disable; } - dart->locked = apple_dart_is_locked(dart); + /* + * The hot-plug PCIe-C DART is freshly made available for this tunnel and + * Apple's ADT does not describe a retained/locked translation context. + * Its T8110 PROTECT register is not readable from this activation state, + * so do not turn that optional observation into a fatal external abort. + */ + if (tunneled) { + dart->locked = false; + dev_info(dev, "PCIe-C DART treating fresh tunnel context as unlocked\n"); + } else { + dart->locked = apple_dart_is_locked(dart); + } + if (tunneled && apple_dart_debug_stage == 17) { + dev_info(dev, "PCIe-C DART debug stage 17: lock=%u\n", + dart->locked); + ret = -EAGAIN; + debug_stop = true; + goto err_clk_disable; + } + if (!dart->locked && tunneled && apple_dart_debug_stage >= 29) { + if (apple_dart_debug_stage == 29) { + dev_info(dev, + "PCIe-C DART debug stage 29: reset boundary reached\n"); + ret = -EAGAIN; + debug_stop = true; + goto err_clk_disable; + } + ret = apple_dart_debug_reset(dart); + debug_stop = ret == -EAGAIN; + goto err_clk_disable; + } if (!dart->locked) { ret = apple_dart_hw_reset(dart); if (ret) goto err_clk_disable; } + if (tunneled && apple_dart_debug_stage == 18) { + dev_info(dev, "PCIe-C DART debug stage 18: reset complete\n"); + ret = -EAGAIN; + debug_stop = true; + goto err_clk_disable; + } + apple_dart_tunnel_trace(dart, 30, "register DART fault IRQ"); ret = request_irq(dart->irq, apple_dart_irq, IRQF_SHARED, "apple-dart fault handler", dart); if (ret) goto err_clk_disable; + if (tunneled && apple_dart_debug_stage == 19) { + dev_info(dev, "PCIe-C DART debug stage 19: IRQ registered\n"); + ret = -EAGAIN; + debug_stop = true; + goto err_free_irq; + } - platform_set_drvdata(pdev, dart); - + apple_dart_tunnel_trace(dart, 31, "register DART sysfs device"); ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart.%s", dev_name(&pdev->dev)); if (ret) goto err_free_irq; + if (tunneled && apple_dart_debug_stage == 20) { + dev_info(dev, "PCIe-C DART debug stage 20: sysfs registered\n"); + ret = -EAGAIN; + debug_stop = true; + goto err_sysfs_remove; + } + apple_dart_tunnel_trace(dart, 32, "register DART with IOMMU core"); ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev); if (ret) goto err_sysfs_remove; + if (tunneled && apple_dart_debug_stage == 21) { + dev_info(dev, "PCIe-C DART debug stage 21: IOMMU registered\n"); + ret = -EAGAIN; + debug_stop = true; + goto err_iommu_unregister; + } pm_runtime_put(dev); @@ -1555,12 +2009,18 @@ static int apple_dart_probe(struct platform_device *pdev) dart->pgsize > PAGE_SIZE, dart->locked, dart->ias, dart->oas); return 0; +err_iommu_unregister: + iommu_device_unregister(&dart->iommu); err_sysfs_remove: iommu_device_sysfs_remove(&dart->iommu); err_free_irq: free_irq(dart->irq, dart); err_clk_disable: - pm_runtime_put(dev); + /* A diagnostic exit must not schedule runtime suspend during teardown. */ + if (debug_stop) + pm_runtime_put_noidle(dev); + else + pm_runtime_put(dev); clk_bulk_disable_unprepare(dart->num_clks, dart->clks); return ret; @@ -1699,10 +2159,12 @@ static __maybe_unused int apple_dart_suspend(struct device *dev) return 0; for (sid = 0; sid < dart->num_streams; sid++) { - dart->save_tcr[sid] = readl(dart->regs + DART_TCR(dart, sid)); + dart->save_tcr[sid] = apple_dart_readl(dart, + DART_TCR(dart, sid)); for (idx = 0; idx < dart->hw->ttbr_count; idx++) dart->save_ttbr[sid][idx] = - readl(dart->regs + DART_TTBR(dart, sid, idx)); + apple_dart_readl(dart, + DART_TTBR(dart, sid, idx)); } return 0; @@ -1726,9 +2188,10 @@ static __maybe_unused int apple_dart_resume(struct device *dev) for (sid = 0; sid < dart->num_streams; sid++) { for (idx = 0; idx < dart->hw->ttbr_count; idx++) - writel(dart->save_ttbr[sid][idx], - dart->regs + DART_TTBR(dart, sid, idx)); - writel(dart->save_tcr[sid], dart->regs + DART_TCR(dart, sid)); + apple_dart_writel(dart, dart->save_ttbr[sid][idx], + DART_TTBR(dart, sid, idx)); + apple_dart_writel(dart, dart->save_tcr[sid], + DART_TCR(dart, sid)); } return 0; From 883932a1bccd3ba7f11660a9597ecb81c0c1e3ca Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 22:48:43 -0400 Subject: [PATCH 0114/2680] iommu: apple-dart: support retained PCIe-C tunnel domains Signed-off-by: DjDeveloperr --- drivers/iommu/apple-dart.c | 240 ++++--------------------------------- 1 file changed, 23 insertions(+), 217 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index a0cc65c836f9f..2caa10bad424e 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -45,20 +45,6 @@ #define DART_PCIEC_ADDR_WIDTH 42 #define DART_PCIEC_STREAMS 64 -/* - * Temporary bring-up gate for the cable-powered DART behind PCIe-C. The - * normal storage, display and USB4 DARTs never consult this parameter. - */ -static unsigned int apple_dart_debug_stage; -module_param_named(debug_stage, apple_dart_debug_stage, uint, 0644); -MODULE_PARM_DESC(debug_stage, - "Stop tunneled PCIe-C DART setup after the selected debug phase"); - -static unsigned int apple_dart_trace_delay_ms; -module_param_named(tunnel_trace_delay_ms, apple_dart_trace_delay_ms, uint, 0644); -MODULE_PARM_DESC(tunnel_trace_delay_ms, - "Delay after PCIe-C DART trace boundaries (0 disables tracing)"); - /* Common registers */ #define DART_PARAMS1 0x00 @@ -255,6 +241,7 @@ struct apple_dart { u32 four_level : 1; u32 locked : 1; u32 tunneled : 1; + u32 power_retained : 1; struct apple_tunable *tunables; dma_addr_t dma_min; @@ -319,17 +306,6 @@ static void apple_dart_apply_tunables(struct apple_dart *dart) } } -static void apple_dart_tunnel_trace(struct apple_dart *dart, - unsigned int step, const char *operation) -{ - if (!dart->tunneled || !READ_ONCE(apple_dart_trace_delay_ms)) - return; - - dev_info(dart->dev, "PCIe-C DART trace %u armed: %s\n", - step, operation); - msleep(READ_ONCE(apple_dart_trace_delay_ms)); -} - /* * Convenience struct to identify streams. * @@ -728,133 +704,24 @@ static int apple_dart_hw_reset(struct apple_dart *dart) bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS); bitmap_set(stream_map.sidmap, 0, dart->num_streams); - apple_dart_tunnel_trace(dart, 20, "disable DMA for every stream"); apple_dart_hw_disable_dma(&stream_map); - apple_dart_tunnel_trace(dart, 21, "clear every stream TTBR"); apple_dart_hw_clear_all_ttbrs(&stream_map); /* enable all streams globally since TCR is used to control isolation */ - apple_dart_tunnel_trace(dart, 22, "enable all streams globally"); for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) apple_dart_writel(dart, U32_MAX, dart->hw->enable_streams + 4 * i); /* clear any pending errors before the interrupt is unmasked */ - apple_dart_tunnel_trace(dart, 23, "read and clear DART error status"); apple_dart_writel(dart, apple_dart_readl(dart, dart->hw->error), dart->hw->error); - apple_dart_tunnel_trace(dart, 24, "clear T8110 error mask"); if (dart->hw->type == DART_T8110) apple_dart_writel(dart, 0, DART_T8110_ERROR_MASK); - apple_dart_tunnel_trace(dart, 25, "invalidate every stream TLB"); return dart->hw->invalidate_tlb(&stream_map); } -static int apple_dart_debug_reset(struct apple_dart *dart) -{ - struct apple_dart_stream_map stream_map; - u32 value; - int i, sid; - - if (dart->hw->type != DART_T8110 || apple_dart_debug_stage < 30) - return 0; - - stream_map.dart = dart; - bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS); - set_bit(0, stream_map.sidmap); - - apple_dart_writel(dart, U32_MAX, dart->hw->enable_streams); - dev_info(dart->dev, - "PCIe-C DART reset stage 30: stream word 0 enabled\n"); - if (apple_dart_debug_stage == 30) - return -EAGAIN; - - for (i = 1; i < BITS_TO_U32(dart->num_streams); i++) - apple_dart_writel(dart, U32_MAX, - dart->hw->enable_streams + 4 * i); - dev_info(dart->dev, - "PCIe-C DART reset stage 31: all stream words enabled\n"); - if (apple_dart_debug_stage == 31) - return -EAGAIN; - - value = apple_dart_readl(dart, DART_TCR(dart, 0)); - dev_info(dart->dev, - "PCIe-C DART reset stage 32: SID0 TCR initial value=%#x\n", - value); - if (apple_dart_debug_stage == 32) - return -EAGAIN; - - apple_dart_writel(dart, dart->hw->tcr_disabled, - DART_TCR(dart, 0)); - dev_info(dart->dev, - "PCIe-C DART reset stage 33: SID0 TCR disabled\n"); - if (apple_dart_debug_stage == 33) - return -EAGAIN; - - value = apple_dart_readl(dart, DART_TCR(dart, 0)); - dev_info(dart->dev, - "PCIe-C DART reset stage 34: SID0 TCR readback=%#x\n", - value); - if (apple_dart_debug_stage == 34) - return -EAGAIN; - - for (sid = 1; sid < dart->num_streams; sid++) - apple_dart_writel(dart, dart->hw->tcr_disabled, - DART_TCR(dart, sid)); - dev_info(dart->dev, - "PCIe-C DART reset stage 35: all %u TCRs disabled\n", - dart->num_streams); - if (apple_dart_debug_stage == 35) - return -EAGAIN; - - apple_dart_writel(dart, 0, DART_TTBR(dart, 0, 0)); - dev_info(dart->dev, - "PCIe-C DART reset stage 36: SID0 TTBR0 cleared\n"); - if (apple_dart_debug_stage == 36) - return -EAGAIN; - - value = apple_dart_readl(dart, DART_TTBR(dart, 0, 0)); - dev_info(dart->dev, - "PCIe-C DART reset stage 37: SID0 TTBR0 readback=%#x\n", - value); - if (apple_dart_debug_stage == 37) - return -EAGAIN; - - for (sid = 1; sid < dart->num_streams; sid++) - for (i = 0; i < dart->hw->ttbr_count; i++) - apple_dart_writel(dart, 0, - DART_TTBR(dart, sid, i)); - dev_info(dart->dev, - "PCIe-C DART reset stage 38: all TTBRs cleared\n"); - if (apple_dart_debug_stage == 38) - return -EAGAIN; - - value = apple_dart_readl(dart, dart->hw->error); - dev_info(dart->dev, - "PCIe-C DART reset stage 39: error status=%#x\n", value); - if (apple_dart_debug_stage == 39) - return -EAGAIN; - - apple_dart_writel(dart, value, dart->hw->error); - dev_info(dart->dev, - "PCIe-C DART reset stage 40: error status cleared\n"); - if (apple_dart_debug_stage == 40) - return -EAGAIN; - - apple_dart_writel(dart, 0, DART_T8110_ERROR_MASK); - dev_info(dart->dev, - "PCIe-C DART reset stage 41: error mask cleared\n"); - if (apple_dart_debug_stage == 41) - return -EAGAIN; - - i = dart->hw->invalidate_tlb(&stream_map); - dev_info(dart->dev, - "PCIe-C DART reset stage 42: SID0 TLB flush returned %d\n", i); - return i ?: -EAGAIN; -} - static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain) { int i, j; @@ -1703,23 +1570,15 @@ static int apple_dart_probe(struct platform_device *pdev) { int ret; u32 dart_params[4]; + struct device_node *pd_np; struct resource *res; struct apple_dart *dart; struct device *dev = &pdev->dev; u64 dma_range[2]; - bool debug_stop = false; bool tunneled = dev->of_node->parent && of_node_name_prefix(dev->of_node->parent, "usb4-pcie-tunnel-"); - if (tunneled) { - dev_info(dev, "PCIe-C DART probe entered\n"); - if (apple_dart_debug_stage == 10) { - dev_info(dev, "PCIe-C DART debug stage 10: probe entry\n"); - return -EAGAIN; - } - } - dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL); if (!dart) return -ENOMEM; @@ -1727,6 +1586,14 @@ static int apple_dart_probe(struct platform_device *pdev) dart->dev = dev; dart->hw = of_device_get_match_data(dev); dart->tunneled = tunneled; + if (tunneled) { + pd_np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (pd_np) { + dart->power_retained = + of_property_read_bool(pd_np, "apple,always-on"); + of_node_put(pd_np); + } + } spin_lock_init(&dart->lock); platform_set_drvdata(pdev, dart); @@ -1747,10 +1614,6 @@ static int apple_dart_probe(struct platform_device *pdev) &res); if (IS_ERR(dart->regs)) return PTR_ERR(dart->regs); - if (tunneled && apple_dart_debug_stage == 11) { - dev_info(dev, "PCIe-C DART debug stage 11: registers mapped\n"); - return -EAGAIN; - } if (resource_size(res) < 0x4000) { dev_err(dev, "MMIO region too small (%pr)\n", res); @@ -1773,7 +1636,6 @@ static int apple_dart_probe(struct platform_device *pdev) return ret; dart->num_clks = ret; - apple_dart_tunnel_trace(dart, 10, "enable PCIe-C DART clocks"); ret = clk_bulk_prepare_enable(dart->num_clks, dart->clks); if (ret) return ret; @@ -1785,26 +1647,11 @@ static int apple_dart_probe(struct platform_device *pdev) ret = devm_pm_runtime_enable(dev); if (ret) goto err_clk_disable; - if (tunneled && apple_dart_debug_stage == 12) { - dev_info(dev, - "PCIe-C DART debug stage 12: power and clocks enabled\n"); - ret = -EAGAIN; - debug_stop = true; - goto err_clk_disable; - } if (tunneled) { dev_info(dev, "applying %zu firmware PCIe-C DART tunables\n", dart->tunables->sz); - apple_dart_tunnel_trace(dart, 11, "apply firmware DART tunables"); apple_dart_apply_tunables(dart); dev_info(dev, "firmware PCIe-C DART tunables applied\n"); - if (apple_dart_debug_stage == 13) { - dev_info(dev, - "PCIe-C DART debug stage 13: firmware tunables applied\n"); - ret = -EAGAIN; - debug_stop = true; - goto err_clk_disable; - } } /* @@ -1930,75 +1777,28 @@ static int apple_dart_probe(struct platform_device *pdev) */ if (tunneled) { dart->locked = false; - dev_info(dev, "PCIe-C DART treating fresh tunnel context as unlocked\n"); } else { dart->locked = apple_dart_is_locked(dart); } - if (tunneled && apple_dart_debug_stage == 17) { - dev_info(dev, "PCIe-C DART debug stage 17: lock=%u\n", - dart->locked); - ret = -EAGAIN; - debug_stop = true; - goto err_clk_disable; - } - if (!dart->locked && tunneled && apple_dart_debug_stage >= 29) { - if (apple_dart_debug_stage == 29) { - dev_info(dev, - "PCIe-C DART debug stage 29: reset boundary reached\n"); - ret = -EAGAIN; - debug_stop = true; - goto err_clk_disable; - } - ret = apple_dart_debug_reset(dart); - debug_stop = ret == -EAGAIN; - goto err_clk_disable; - } if (!dart->locked) { ret = apple_dart_hw_reset(dart); if (ret) goto err_clk_disable; } - if (tunneled && apple_dart_debug_stage == 18) { - dev_info(dev, "PCIe-C DART debug stage 18: reset complete\n"); - ret = -EAGAIN; - debug_stop = true; - goto err_clk_disable; - } - apple_dart_tunnel_trace(dart, 30, "register DART fault IRQ"); ret = request_irq(dart->irq, apple_dart_irq, IRQF_SHARED, "apple-dart fault handler", dart); if (ret) goto err_clk_disable; - if (tunneled && apple_dart_debug_stage == 19) { - dev_info(dev, "PCIe-C DART debug stage 19: IRQ registered\n"); - ret = -EAGAIN; - debug_stop = true; - goto err_free_irq; - } - apple_dart_tunnel_trace(dart, 31, "register DART sysfs device"); ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart.%s", dev_name(&pdev->dev)); if (ret) goto err_free_irq; - if (tunneled && apple_dart_debug_stage == 20) { - dev_info(dev, "PCIe-C DART debug stage 20: sysfs registered\n"); - ret = -EAGAIN; - debug_stop = true; - goto err_sysfs_remove; - } - apple_dart_tunnel_trace(dart, 32, "register DART with IOMMU core"); ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev); if (ret) goto err_sysfs_remove; - if (tunneled && apple_dart_debug_stage == 21) { - dev_info(dev, "PCIe-C DART debug stage 21: IOMMU registered\n"); - ret = -EAGAIN; - debug_stop = true; - goto err_iommu_unregister; - } pm_runtime_put(dev); @@ -2009,18 +1809,12 @@ static int apple_dart_probe(struct platform_device *pdev) dart->pgsize > PAGE_SIZE, dart->locked, dart->ias, dart->oas); return 0; -err_iommu_unregister: - iommu_device_unregister(&dart->iommu); err_sysfs_remove: iommu_device_sysfs_remove(&dart->iommu); err_free_irq: free_irq(dart->irq, dart); err_clk_disable: - /* A diagnostic exit must not schedule runtime suspend during teardown. */ - if (debug_stop) - pm_runtime_put_noidle(dev); - else - pm_runtime_put(dev); + pm_runtime_put(dev); clk_bulk_disable_unprepare(dart->num_clks, dart->clks); return ret; @@ -2154,6 +1948,10 @@ static __maybe_unused int apple_dart_suspend(struct device *dev) struct apple_dart *dart = dev_get_drvdata(dev); unsigned int sid, idx; + /* The tunneled DART has no state to save when its domain stays on. */ + if (dart->power_retained) + return 0; + /* Locked DARTs can't be restored so skip saving their registers. */ if (dart->locked) return 0; @@ -2176,6 +1974,14 @@ static __maybe_unused int apple_dart_resume(struct device *dev) unsigned int sid, idx; int ret; + /* + * PCIe-C DARTs on an apple,always-on domain retain their translation + * state. Resetting one here is both unnecessary and earlier than Apple's + * force-active call at the end of PCIe-C port resume. + */ + if (dart->power_retained) + return 0; + /* Locked DARTs can't be restored, and they should not need it */ if (dart->locked) return 0; From cafa2b7f10b13d9e9c4d277700413a27c3348a51 Mon Sep 17 00:00:00 2001 From: Marcelo Alcantara Date: Mon, 14 Sep 2026 13:29:32 +1000 Subject: [PATCH 0115/2680] iommu: apple-dart: raise MAX_DARTS_PER_DEVICE to 8 A device may reference one DART stream per controller it uses. The Apple display-subsystem lists one scanout DART (stream 0) per DCP; on M2 Max, wiring all four external DCPs (dcpext0..3) plus disp0 needs five. The cap of 3 makes apple_dart_of_xlate() reject the extra streams with -EINVAL, so the later DCPs never get a scanout mapping and their firmware faults on the first frame. Raise the cap to 8. --- drivers/iommu/apple-dart.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 2caa10bad424e..52122ba7391b1 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -40,7 +40,7 @@ #define DART_MAX_STREAMS 256 #define DART_MAX_TTBR 4 -#define MAX_DARTS_PER_DEVICE 3 +#define MAX_DARTS_PER_DEVICE 8 #define DART_PCIEC_PAGE_SHIFT 14 #define DART_PCIEC_ADDR_WIDTH 42 #define DART_PCIEC_STREAMS 64 From 869cd83b5bda984cbce22f7513b25ae2b3655fd4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:26:23 +1000 Subject: [PATCH 0116/2680] iommu/apple-dart: Check devm_memremap() for an error pointer apple_dart_hw_map_locked_ttbr() maps the L1 table of a locked DART stream with devm_memremap() and checks the result for NULL, but devm_memremap() never returns NULL: it returns ERR_PTR(-ENOMEM) or ERR_PTR(-ENXIO) on failure. A failed mapping was therefore stored as the table pointer, and apple_dart_hw_sync_locked() then wrote the page-table entries through it and apple_dart_hw_unmap_locked_ttbr() passed it to memset(). The NULL check was added together with the locked-DART page-table code by commit 0d4ec0744510 ("iommu/dart: Add iommu_ops for locked DARTs"), and became reachable once locked DARTs were allowed to probe. Check the result with IS_ERR() and return the error, so that no error pointer is ever stored. Fixes: 85322ebcc7e8 ("iommu/dart: Support locked DARTs") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 52122ba7391b1..b104bd4973bcf 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -536,8 +536,8 @@ apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) l1_tbl = devm_memremap(dart->dev, phys, dart->pgsize, MEMREMAP_WB); - if (!l1_tbl) - return -ENOMEM; + if (IS_ERR(l1_tbl)) + return PTR_ERR(l1_tbl); dart->locked_ttbr[sid][idx] = l1_tbl; } From d82fa274a96df72842cd638985c45a65c5408e4d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 00:02:34 -0400 Subject: [PATCH 0117/2680] dt-bindings: describe Apple tunneled PCIe-C resources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — split by topic: only the apple,dart.yaml hunk is carried in 020-dart; the apple,pcie.yaml and apple,t8103-usb4-acio.yaml hunks are carried by 170-usb with the Thunderbolt bindings] Signed-off-by: DjDeveloperr --- .../devicetree/bindings/iommu/apple,dart.yaml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index e179199dbd3b5..ae6fd35f548c1 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -54,6 +54,33 @@ properties: power-domains: maxItems: 1 + apple,dma-range: + $ref: /schemas/types.yaml#/definitions/uint64-array + minItems: 2 + maxItems: 2 + description: > + Inclusive device-visible DMA aperture, encoded as a start address and + size. The IOMMU DMA allocator restricts mappings to this range. + + apple,dma-offset: + $ref: /schemas/types.yaml#/definitions/uint64 + description: > + Offset added to device-visible DMA addresses when indexing the DART page + tables. This models host bridges which alias a low PCI bus aperture into + a high DART translation window. + + apple,tunable: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Mask to be applied to the register value + - description: Bits to be set after applying the mask + description: > + List of (register offset, mask, value) tuples copied from Apple's Device + Tree by the bootloader and required to initialize dynamically powered + DART instances. + required: - compatible - reg From 10293e8582cbf6c13a810dbfaa163d1c3dcac6b3 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 29 Aug 2026 00:48:57 -0400 Subject: [PATCH 0118/2680] dt-bindings: apple: validate PCIe-C and USB4 topology MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — split by topic: only the apple,dart.yaml hunk is carried in 020-dart; the apple,pcie.yaml and apple,t8103-usb4-acio.yaml hunks are carried by 170-usb with the Thunderbolt bindings] Signed-off-by: DjDeveloperr --- .../devicetree/bindings/iommu/apple,dart.yaml | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index ae6fd35f548c1..3e93011ca0066 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -28,6 +28,11 @@ properties: - apple,t8103-usb4-dart - apple,t8110-dart - apple,t6000-dart + - items: + - enum: + - apple,t6000-dart + - apple,t8112-dart + - const: apple,t8110-dart - items: - enum: - apple,t6020-dart @@ -46,10 +51,11 @@ properties: Optional since not all IOMMUs are attached to a clock gate. '#iommu-cells': - const: 1 + enum: [1, 5] description: - Has to be one. The single cell describes the stream id emitted by - a master to the IOMMU. + Usually one cell describing the stream id emitted by a master. The + T6020 NHI DART uses five cells to describe a stream range and its + device-visible DMA window. power-domains: maxItems: 1 @@ -87,6 +93,21 @@ required: - '#iommu-cells' - interrupts +allOf: + - if: + properties: + compatible: + contains: + const: apple,t6020-dart + then: + properties: + '#iommu-cells': + enum: [1, 5] + else: + properties: + '#iommu-cells': + const: 1 + additionalProperties: false examples: From bab465989246c62713b770a568b9aa317cab6864 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:26:37 +1000 Subject: [PATCH 0119/2680] dt-bindings: iommu: apple,dart: Add T8140 compatible and stream remaps The DARTs of the A18 Pro (T8140) are T8110-family instances. Add "apple,t8140-dart" with the "apple,t8110-dart" fallback, and correct the description: depending on the generation, a DART handles up to 256 streams, not 16. A T8110-family stream can be remapped: its TCR selects another stream (a 4-bit REMAP field plus an enable bit) whose translation and filtering state it then uses, without a page-table root of its own. Apple's device tree describes this with a "remap" property; on T8140 the ISP DARTs remap streams 1, 2, 6, 7, 10 and 11 onto stream 0. Add apple,remap-streams, a list of pairs. A source may be listed only once, a destination is limited to 15 by the width of the REMAP field and may not itself be remapped, and source and destination must differ. Also state that apple,dma-range must have a nonzero size, and that a consumer binding may specify a window of its own, as the T8140 AOP, AOP audio, AOP setup mailbox and ADMAC bindings do. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/iommu/apple,dart.yaml | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index 3e93011ca0066..bd6524c97087f 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -14,7 +14,7 @@ description: |+ Resolution Table which provides a mandatory layer of address translations for various masters. - Each DART instance is capable of handling up to 16 different streams + Depending on the generation, a DART instance handles up to 256 streams with individual pagetables and page-level read/write protection flags. This DART IOMMU also raises interrupts in response to various @@ -37,6 +37,7 @@ properties: - enum: - apple,t6020-dart - apple,t8122-dart + - apple,t8140-dart - const: apple,t8110-dart reg: @@ -65,8 +66,10 @@ properties: minItems: 2 maxItems: 2 description: > - Inclusive device-visible DMA aperture, encoded as a start address and - size. The IOMMU DMA allocator restricts mappings to this range. + Device-visible DMA aperture encoded as a base and a nonzero size. The + IOMMU DMA allocator restricts mappings to this range; high address bits + may select a firmware-visible alias of the translation aperture. + A consumer may further specify its own aperture in its binding. apple,dma-offset: $ref: /schemas/types.yaml#/definitions/uint64 @@ -87,6 +90,21 @@ properties: Tree by the bootloader and required to initialize dynamically powered DART instances. + apple,remap-streams: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + minItems: 1 + maxItems: 256 + items: + items: + - description: Source stream ID, unique within this list. + maximum: 255 + - description: Destination stream ID, which must not itself be remapped. + maximum: 15 + description: + T8110 stream remaps. A source uses the destination stream's translation + and filtering state without a separate page-table root. Source and + destination must differ, and both must exist on the controller. + required: - compatible - reg From 844d313cbfb44ab6e4376952d06ffdd36e4dbb1c Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 2 Oct 2026 22:50:16 -0700 Subject: [PATCH 0120/2680] PCI: refuse userspace removal of retained host hierarchies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A retained pci_dev reference does not prevent device_del from releasing IOMMU associations during PCI sysfs removal. Let a host permanently prohibit userspace removal before registering its hierarchy, and enable that restriction only for the Neo host with a retained supplier. Read the immutable property through existing bus and bridge references, preserving self-removal and rescan locking order on ordinary hosts. Exercise the actual removal callback with five synthetic-hierarchy KUnit tests; forced platform and IOMMU teardown remain unqualified. [ryan: adapted to 7.2.10 — the new pci_host_bridge flag follows mainline 7.2's broken_l1ss_resume] --- Documentation/PCI/controller/apple-t8140.rst | 11 ++- .../device_drivers/wifi/mt7932-neo.rst | 7 +- drivers/pci/Kconfig | 10 ++ drivers/pci/controller/pcie-apple.c | 2 + drivers/pci/pci-sysfs-test.c | 98 +++++++++++++++++++ drivers/pci/pci-sysfs.c | 18 +++- include/linux/pci.h | 2 + 7 files changed, 139 insertions(+), 9 deletions(-) create mode 100644 drivers/pci/pci-sysfs-test.c diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index af47c90e3cfbd..991f7b68e37a6 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -41,10 +41,13 @@ drivers. This prevents a later generic rescan from enabling cached children of a failed admission. Each radio probe reapplies the link-power restriction, so a recreated ASPM link state cannot inherit an incompatible global policy. -The PIODMA arena remains allocated until external reset. Controller removal, -teardown, memory reuse and arbitrary downstream devices are unqualified. -The host and activated Bluetooth modules are pinned until reset. Do not remove -the controller, radio PCI devices or their IOMMUs while this experiment is active. +The PIODMA arena remains allocated until external reset. Standard PCI sysfs +``remove`` writes for the root port and radio functions return ``-EBUSY`` +while the host has its retained supplier; the removal attributes remain present. +The host and activated Bluetooth modules are pinned until reset. Arbitrary +platform-device removal, IOMMU teardown, forced module removal, memory reuse +and arbitrary downstream devices remain unqualified. Do not remove the +controller or its IOMMUs while this experiment is active. These lifecycle contracts require further work before production support. J700's ``wifi0`` alias and PCI endpoint node allow the public bootloader to diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index c9e5d776036ec..ca917c8faff07 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -117,12 +117,13 @@ Qualification and remaining work * PCI bootstrap memory remains retained until external reset. Sleep and kexec are refused while it is retained; they are not hardware-qualified on the Neo. - Controller removal and memory reuse remain unqualified. + Standard PCI sysfs removal of the root and radio functions is refused; + arbitrary controller/IOMMU teardown and memory reuse remain unqualified. * Bluetooth PCI removal/quiescence is incomplete. Its software queue limit does not provide HCI backpressure; saturation can drop an accounted frame. Both require correction before production use. An activated Bluetooth module - cannot be unloaded; PCI removal only retires software callbacks and does not - establish DMA quiescence or release retained ownership. + cannot be unloaded normally; forced PCI removal only retires software + callbacks and does not establish DMA quiescence or release retained ownership. * Arbitrary scan IEs, WPA3/SAE, required MFP, AP/P2P, general country-package generation, roaming and long-duration reliability are unqualified. * SCO/headset microphone, LE Audio/ISO and simultaneous headset audio are diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig index 0c7408509ba24..0644a5ad860a4 100644 --- a/drivers/pci/Kconfig +++ b/drivers/pci/Kconfig @@ -29,6 +29,16 @@ menuconfig PCI if PCI +config PCI_SYSFS_KUNIT_TEST + bool "Unit tests for PCI sysfs removal restrictions" if !KUNIT_ALL_TESTS + depends on SYSFS && KUNIT=y + default KUNIT_ALL_TESTS + help + Test rejected removal writes for root ports and children under a host + that prohibits userspace removal, plus malformed and no-op writes. + The tests use an unregistered synthetic hierarchy and never remove + a real PCI device or invoke kernfs attribute removal. + config PCI_DOMAINS bool diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 7a0bd3019e815..f5315ca205d40 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2516,6 +2516,8 @@ static int apple_pcie_probe(struct platform_device *pdev) pcie->dev = dev; pcie->hw = hw; pcie->piodma_supplier = piodma_supplier; + /* A retained supplier forbids manual removal of its root or radios. */ + bridge->no_user_remove = !!piodma_supplier; if (hw->root_bus_only) dev_info(dev, "root-port probe only; downstream config is blocked\n"); pcie->base = devm_platform_ioremap_resource(pdev, 1); diff --git a/drivers/pci/pci-sysfs-test.c b/drivers/pci/pci-sysfs-test.c new file mode 100644 index 0000000000000..3875a0f760d23 --- /dev/null +++ b/drivers/pci/pci-sysfs-test.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included by pci-sysfs.c to exercise its actual remove_store callback. */ +#include + +struct pci_sysfs_test_hierarchy { + struct pci_host_bridge bridge; + struct pci_bus root_bus; + struct pci_bus child_bus; + struct pci_dev root_port; + struct pci_dev endpoint; +}; + +static int pci_sysfs_test_init(struct kunit *test) +{ + struct pci_sysfs_test_hierarchy *hierarchy; + + hierarchy = kunit_kzalloc(test, sizeof(*hierarchy), GFP_KERNEL); + if (!hierarchy) + return -ENOMEM; + /* + * Only topology and diagnostic names are needed. These objects are never + * registered, referenced by the PCI core or passed to a removal routine. + * Every tested callback must return before device_remove_file_self(). + */ + hierarchy->root_bus.bridge = &hierarchy->bridge.dev; + hierarchy->child_bus.parent = &hierarchy->root_bus; + hierarchy->root_port.bus = &hierarchy->root_bus; + hierarchy->root_port.dev.init_name = "kunit-root-port"; + hierarchy->endpoint.bus = &hierarchy->child_bus; + hierarchy->endpoint.dev.init_name = "kunit-endpoint"; + test->priv = hierarchy; + return 0; +} + +static void pci_sysfs_remove_blocked_root_test(struct kunit *test) +{ + struct pci_sysfs_test_hierarchy *hierarchy = test->priv; + + hierarchy->bridge.no_user_remove = true; + KUNIT_EXPECT_EQ(test, + remove_store(&hierarchy->root_port.dev, &dev_attr_remove, "1\n", 2), + (ssize_t)-EBUSY); +} + +static void pci_sysfs_remove_blocked_child_test(struct kunit *test) +{ + struct pci_sysfs_test_hierarchy *hierarchy = test->priv; + + hierarchy->bridge.no_user_remove = true; + KUNIT_EXPECT_EQ(test, + remove_store(&hierarchy->endpoint.dev, &dev_attr_remove, "1\n", 2), + (ssize_t)-EBUSY); +} + +static void pci_sysfs_remove_malformed_test(struct kunit *test) +{ + struct pci_sysfs_test_hierarchy *hierarchy = test->priv; + + KUNIT_EXPECT_EQ(test, + remove_store(&hierarchy->endpoint.dev, &dev_attr_remove, "bad\n", 4), + (ssize_t)-EINVAL); +} + +static void pci_sysfs_remove_zero_blocked_test(struct kunit *test) +{ + struct pci_sysfs_test_hierarchy *hierarchy = test->priv; + + hierarchy->bridge.no_user_remove = true; + KUNIT_EXPECT_EQ(test, + remove_store(&hierarchy->endpoint.dev, &dev_attr_remove, "0\n", 2), + (ssize_t)2); +} + +static void pci_sysfs_remove_zero_default_test(struct kunit *test) +{ + struct pci_sysfs_test_hierarchy *hierarchy = test->priv; + + KUNIT_EXPECT_EQ(test, + remove_store(&hierarchy->root_port.dev, &dev_attr_remove, "0\n", 2), + (ssize_t)2); +} + +static struct kunit_case pci_sysfs_test_cases[] = { + KUNIT_CASE(pci_sysfs_remove_blocked_root_test), + KUNIT_CASE(pci_sysfs_remove_blocked_child_test), + KUNIT_CASE(pci_sysfs_remove_malformed_test), + KUNIT_CASE(pci_sysfs_remove_zero_blocked_test), + KUNIT_CASE(pci_sysfs_remove_zero_default_test), + {} +}; + +static struct kunit_suite pci_sysfs_test_suite = { + .name = "pci-sysfs-remove", + .init = pci_sysfs_test_init, + .test_cases = pci_sysfs_test_cases, +}; + +kunit_test_suite(pci_sysfs_test_suite); diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c index 6b016792e0ad8..cf33429556623 100644 --- a/drivers/pci/pci-sysfs.c +++ b/drivers/pci/pci-sysfs.c @@ -514,13 +514,23 @@ static struct device_attribute dev_attr_dev_rescan = __ATTR(rescan, 0200, NULL, static ssize_t remove_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { + struct pci_dev *pdev = to_pci_dev(dev); + struct pci_host_bridge *bridge; unsigned long val; if (kstrtoul(buf, 0, &val) < 0) return -EINVAL; - if (val && device_remove_file_self(dev, attr)) - pci_stop_and_remove_bus_device_locked(to_pci_dev(dev)); + if (val) { + /* Bus/bridge references keep this immutable host property alive. */ + bridge = pci_find_host_bridge(pdev->bus); + if (bridge->no_user_remove) { + pci_warn(pdev, "userspace removal is disabled by the host bridge\n"); + return -EBUSY; + } + if (device_remove_file_self(dev, attr)) + pci_stop_and_remove_bus_device_locked(pdev); + } return count; } static DEVICE_ATTR_IGNORE_LOCKDEP(remove, 0220, NULL, @@ -1923,6 +1933,10 @@ const struct attribute_group *pci_dev_groups[] = { NULL, }; +#ifdef CONFIG_PCI_SYSFS_KUNIT_TEST +#include "pci-sysfs-test.c" +#endif + static const struct attribute_group pci_dev_hp_attr_group = { .attrs = pci_dev_hp_attrs, .is_visible = pci_dev_hp_attrs_are_visible, diff --git a/include/linux/pci.h b/include/linux/pci.h index 66a4fda05793d..8b47b5e42217b 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -666,6 +666,8 @@ struct pci_host_bridge { unsigned int msi_domain:1; /* Bridge wants MSI domain */ unsigned int broken_l1ss_resume:1; /* Resuming from L1SS during system suspend is broken */ + /* Disable sysfs device removal for this hierarchy; set before probing. */ + unsigned int no_user_remove:1; /* Resource alignment requirements */ resource_size_t (*align_resource)(struct pci_dev *dev, From 4745fce1f74df8215923d0acb041bcffb5c6cf26 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 2 Oct 2026 22:50:17 -0700 Subject: [PATCH 0121/2680] wifi: mt7932: join host producers before shutdown Clearing running alone does not close TX, command or regulatory admission. Reuse the existing host-user join path before disabling bus mastering so no admitted worker republishes DMA or unmasks IRQs after shutdown returns. Retain all device-visible allocations without claiming hardware quiescence. --- Documentation/networking/device_drivers/wifi/mt7932-neo.rst | 2 ++ drivers/net/wireless/mediatek/mt7932/mt7932.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index ca917c8faff07..e5ecda0a1aa1a 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -119,6 +119,8 @@ Qualification and remaining work are refused while it is retained; they are not hardware-qualified on the Neo. Standard PCI sysfs removal of the root and radio functions is refused; arbitrary controller/IOMMU teardown and memory reuse remain unqualified. + Wi-Fi shutdown joins host producers before disabling bus mastering and + retains device-visible memory until reset. * Bluetooth PCI removal/quiescence is incomplete. Its software queue limit does not provide HCI backpressure; saturation can drop an accounted frame. Both require correction before production use. An activated Bluetooth module diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 0ba692be3bf75..308664d3e945a 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -1285,8 +1285,8 @@ static void mt_shutdown(struct pci_dev *pdev) { struct mt7932 *m = pci_get_drvdata(pdev); - WRITE_ONCE(m->running, false); - mt_write(m, W + 0x204, 0); + /* Close admission and join software users before disabling bus mastering. */ + mt_stop_host(m); pci_clear_master(pdev); /* Storage remains allocated through reset, even if the engine is stuck. */ } From 69e5ac1da70924f77cf40bf634db5fc923cdaae7 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 00:56:16 -0700 Subject: [PATCH 0122/2680] wifi: mt7932: close RF admission on terminal calibration errors D7 failures and malformed calibration events now fail pending associations and disconnect an established host link under the same lock that guards success publication. Retain uncertain peer and DMA ownership until checked reset, and prevent late startup from reopening RF admission. Name local firmware and policy inputs on load/validation errors, expose fixed Wi-Fi firmware metadata, and document actual Bluetooth variants, modular CMAC providers and post-startup calibration dependencies. --- .../device_drivers/wifi/mt7932-neo.rst | 28 ++++++++- drivers/bluetooth/mt7932_bt_pcie.c | 44 +++++++++----- drivers/net/wireless/mediatek/mt7932/Kconfig | 3 +- .../mediatek/mt7932/calibration_transport.c | 13 ++-- drivers/net/wireless/mediatek/mt7932/data.c | 3 +- .../net/wireless/mediatek/mt7932/frontend.c | 18 ++++-- drivers/net/wireless/mediatek/mt7932/link.c | 59 ++++++++++++++++++- drivers/net/wireless/mediatek/mt7932/mt7932.c | 49 +++++++++++---- drivers/net/wireless/mediatek/mt7932/mt7932.h | 3 + .../net/wireless/mediatek/mt7932/startup.c | 11 +++- 10 files changed, 185 insertions(+), 46 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index e5ecda0a1aa1a..4330af26a9f8c 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -39,6 +39,13 @@ and kexec at runtime while its arena is retained; other Macs keep their normal behavior. Firmware loading is selected by both radio drivers. This shared build still requires physical Neo qualification. +``CONFIG_CRYPTO_CMAC=m`` is supported with the modular Bluetooth core. Generic +Bluetooth SMP allocates ``cmac(aes)`` through the crypto API. Install the crypto +modules and preserve algorithm autoload; if Bluetooth starts in the initramfs, +include its crypto providers there. With ``CONFIG_CRYPTO_AES=y``, this kernel +also registers its AES CMAC implementation when CMAC is configured as a module. +The earlier tested ``CONFIG_CRYPTO_CMAC=y`` is not a Wi-Fi driver requirement. + Use the ordinary cfg80211 regulatory database and applicable country policy. The validated first-admission fallback is kernel country 00 with firmware XZ. @@ -83,8 +90,12 @@ Wi-Fi requests the following files under ``mediatek/mt7932/``: Bluetooth requests these files under ``mediatek/``: * ``MT7932B1_OS_TypeB_2.0.177.0_260706180356.bin`` or the supported fallback - ``MT7932B1_OS_TypeB_0.1.133.0_260128190103.bin``; -* ``j700-mt7932-btcal.bin`` (388 bytes); + ``MT7932B1_OS_TypeB_0.1.133.0_260128190103.bin`` for ROM1. The fallback is + tried only when the preferred file is absent. ROM0 instead requests + ``MT7932B0_OS_TypeB_0.1.44.0_241001003711.bin``; admitting that variant in + software does not qualify it on physical Neo hardware; +* ``j700-mt7932-btcal.bin`` (the measured input is 388 bytes; the driver accepts + nonempty inputs up to 65535 bytes and relies on HCI setup acceptance); * ``MT7932_PTB_IzubaA_0.1.0.0_20251021141303.ptx`` (198 bytes); * ``j700-mt7932-bdaddr.bin`` (the unit's six-byte Bluetooth address). @@ -92,6 +103,19 @@ Extraction and packaging require the original local assets. The Linux loader does not grant redistribution rights to them. Do not include those assets or their unit-specific digests in bug reports, commits or test artifacts. +Keep the installed calibration inputs stable while the Wi-Fi driver is bound: +it requests ``oca2.bin`` again for runtime D7 requests and each association. +``config-original.bin`` and country policies require J7CF/J7RP packaging; copying +unconverted source files to those names does not satisfy the format checks. + +Load the radio drivers after these inputs are available on the real root. +An early Wi-Fi probe already owns DMA before it requests firmware. Only an +explicit ``initialization failed; DMA retired`` message confirms that its +checked reset completed and the unbound driver can be loaded again. A failed +reset retains the binding and DMA; do not reprobe it. Use a full external reset +after a failed or uncertain admission. Installing a missing startup calibration +file does not clear a terminal RF failure in an already bound epoch. + Repeatable physical network test -------------------------------- diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 9e5df8e10e65f..7ab5d30dd458e 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -236,22 +236,33 @@ static int bt7932_validate_ptx(const u8 *data, size_t size) return next == size ? 0 : -EINVAL; } +static int bt7932_input_error(struct bt7932 *bt, const char *name, int error) +{ + dev_err(&bt->pdev->dev, "local input %s rejected: %d\n", name, error); + return error; +} + static int bt7932_read_inputs(struct bt7932 *bt) { const struct firmware *fw; + const char *name; const u8 *trailer; u64 last; int i, ret; if (!bt->rom) { - ret = request_firmware(&fw, BT7932_FW_B0, &bt->pdev->dev); + name = BT7932_FW_B0; + ret = request_firmware(&fw, name, &bt->pdev->dev); } else { - ret = firmware_request_nowarn(&fw, BT7932_FW_B1, &bt->pdev->dev); - if (ret == -ENOENT) - ret = request_firmware(&fw, BT7932_FW_B1_FALLBACK, &bt->pdev->dev); + name = BT7932_FW_B1; + ret = firmware_request_nowarn(&fw, name, &bt->pdev->dev); + if (ret == -ENOENT) { + name = BT7932_FW_B1_FALLBACK; + ret = request_firmware(&fw, name, &bt->pdev->dev); + } } if (ret) - return ret; + return bt7932_input_error(bt, name, ret); if (fw->size <= 32 || fw->size - 32 > U32_MAX - 3) { ret = -EINVAL; goto release_image; @@ -279,10 +290,11 @@ static int bt7932_read_inputs(struct bt7932 *bt) release_image: release_firmware(fw); if (ret) - return ret; - ret = request_firmware(&fw, BT7932_CAL, &bt->pdev->dev); + return bt7932_input_error(bt, name, ret); + name = BT7932_CAL; + ret = request_firmware(&fw, name, &bt->pdev->dev); if (ret) - return ret; + return bt7932_input_error(bt, name, ret); if (!fw->size || fw->size > U16_MAX) { ret = -EINVAL; } else { @@ -293,10 +305,11 @@ static int bt7932_read_inputs(struct bt7932 *bt) } release_firmware(fw); if (ret) - return ret; - ret = request_firmware(&fw, BT7932_PTX, &bt->pdev->dev); + return bt7932_input_error(bt, name, ret); + name = BT7932_PTX; + ret = request_firmware(&fw, name, &bt->pdev->dev); if (ret) - return ret; + return bt7932_input_error(bt, name, ret); ret = bt7932_validate_ptx(fw->data, fw->size); if (!ret) { bt->ptx_size = fw->size; @@ -306,10 +319,11 @@ static int bt7932_read_inputs(struct bt7932 *bt) } release_firmware(fw); if (ret) - return ret; - ret = request_firmware(&fw, BT7932_ADDR, &bt->pdev->dev); + return bt7932_input_error(bt, name, ret); + name = BT7932_ADDR; + ret = request_firmware(&fw, name, &bt->pdev->dev); if (ret) - return ret; + return bt7932_input_error(bt, name, ret); if (fw->size != sizeof(bt->address) || !memchr_inv(fw->data, 0, fw->size) || !memchr_inv(fw->data, 0xff, fw->size)) { ret = -EINVAL; @@ -318,7 +332,7 @@ static int bt7932_read_inputs(struct bt7932 *bt) bt->address[i] = fw->data[sizeof(bt->address) - i - 1]; } release_firmware(fw); - return ret; + return ret ? bt7932_input_error(bt, name, ret) : 0; } static int bt7932_reserve(size_t *total, size_t bytes, size_t *offset) diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index d435dbe24c32e..9c81c31a18f5e 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -6,7 +6,8 @@ config MT7932_FULLMAC select CFG80211 help Native support for the built-in J700 Wi-Fi endpoint: managed - WPA2-CCMP station operation on 2.4 GHz and channels 36..48 at 20 MHz. + WPA2-CCMP station operation on 2.4 GHz at 20 MHz and channels 36..48 + with a software bandwidth limit of 80 MHz. Requires original firmware, own calibration and original-derived country policy packages in addition to the cfg80211 regulatory database. Requires a trained retained PCI link and a translated SID1 DMA domain. diff --git a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c index 7c2dcae0456a6..312315eb5b405 100644 --- a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c +++ b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c @@ -39,8 +39,7 @@ int mt_cal_procedure(struct mt7932 *m, struct mt7932_cal_piece *pieces, return 0; fail: spin_lock_irqsave(&m->response_lock, flags); - m->cal_state.error = ret; - m->cal_state.active = false; + mt_rf_fail_locked(m, ret); spin_unlock_irqrestore(&m->response_lock, flags); dev_err(&m->pdev->dev, "CAL_PROCEDURE_FAILED: %d; epoch latched, no replay\n", ret); return ret; @@ -57,15 +56,17 @@ int mt_calibration_gate(struct mt7932 *m) if (!m->six_ghz_valid || m->smart_version != 12 || !m->preload_version) return -EOPNOTSUPP; - ret = request_firmware_direct(&wcal, "mediatek/mt7932/wcal.bin", &m->pdev->dev); + ret = mt_request_input(m, &wcal, "mediatek/mt7932/wcal.bin"); if (ret) return ret; - ret = request_firmware_direct(&oca, "mediatek/mt7932/oca2.bin", &m->pdev->dev); + ret = mt_request_input(m, &oca, "mediatek/mt7932/oca2.bin"); if (ret) goto out_wcal; ret = -EINVAL; - if (!wcal->size || wcal->size > 1024) + if (!wcal->size || wcal->size > 1024) { + dev_err(&m->pdev->dev, "local input mediatek/mt7932/wcal.bin has invalid size %zu\n", wcal->size); goto out; + } planned = mt7932_cal_power_on(oca->data, oca->size, m->six_ghz, pieces); if (planned < 0) { ret = planned; @@ -137,6 +138,8 @@ int mt_calibration_gate(struct mt7932 *m) } ret = 0; out: + if (ret) + dev_err(&m->pdev->dev, "own calibration using mediatek/mt7932/wcal.bin and mediatek/mt7932/oca2.bin failed: %d\n", ret); release_firmware(oca); out_wcal: release_firmware(wcal); diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index 25a7bcbf31238..19d2e07736eb4 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -261,7 +261,8 @@ int mt_data_prepare(struct mt7932 *m) */ dev_info(&m->pdev->dev, "MAC_RTS_RESPONSE: sequence matched, EID=%02x length=%zu\n", m->reply[28], m->reply_length); - WRITE_ONCE(m->data_ready, true); + /* Publish the initialized data_lock and DMA state to failure handlers. */ + smp_store_release(&m->data_ready, true); dev_info(&m->pdev->dev, "ETHERNET_PREPARED: TX0 retained coherent slots=512, RX translation enabled\n"); return 0; } diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 464dca910a978..174beebc8f6ab 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -471,7 +471,7 @@ static void mt_cal_work(struct work_struct *work) mutex_lock(&m->command_mutex); if (!READ_ONCE(m->rf_ready)) goto unlock; - ret = request_firmware_direct(&oca, "mediatek/mt7932/oca2.bin", &m->pdev->dev); + ret = mt_request_input(m, &oca, "mediatek/mt7932/oca2.bin"); if (ret) goto fail; for (;;) { @@ -500,9 +500,10 @@ static void mt_cal_work(struct work_struct *work) if (!ret) goto unlock; fail: - WRITE_ONCE(m->rf_ready, false); - mt_abort_scan(m->wiphy, &m->wdev); - dev_err(&m->pdev->dev, "D7_CALIBRATION_FAILED: %d; RF operations stopped\n", ret); + spin_lock_irqsave(&m->response_lock, flags); + mt_rf_fail_locked(m, ret); + spin_unlock_irqrestore(&m->response_lock, flags); + dev_err(&m->pdev->dev, "D7_CALIBRATION_FAILED: %d; mediatek/mt7932/oca2.bin RF operations stopped\n", ret); unlock: mutex_unlock(&m->command_mutex); } @@ -553,6 +554,7 @@ int mt_register_interface(struct mt7932 *m) int mt_enable_scan(struct mt7932 *m) { + unsigned long flags; int ret = mt_data_prepare(m); if (ret) @@ -562,9 +564,17 @@ int mt_enable_scan(struct mt7932 *m) if (ret) return ret; } + /* An RX calibration failure must win over late startup publication. */ + spin_lock_irqsave(&m->response_lock, flags); + if (m->stopping || m->link_failed || m->cal_state.error) { + ret = m->cal_state.error ?: -EIO; + spin_unlock_irqrestore(&m->response_lock, flags); + return ret; + } WRITE_ONCE(m->rf_ready, true); dev_info(&m->pdev->dev, "CFG80211_SCAN_READY: %s, carrier off until WPA2 authorization\n", m->netdev->name); schedule_work(&m->cal_work); + spin_unlock_irqrestore(&m->response_lock, flags); return 0; } diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 8caab22dbbb34..53b4e931d605d 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -34,6 +34,55 @@ static bool mt_channel_event(struct mt7932 *m, const u8 *body, size_t length) return true; } +void mt_rf_fail_locked(struct mt7932 *m, int error) +{ + bool first = !m->link_failed; + bool data_ready = smp_load_acquire(&m->data_ready); + + lockdep_assert_held(&m->response_lock); + if (m->stopping) + return; + if (error >= 0) + error = -EIO; + WRITE_ONCE(m->rf_ready, false); + WRITE_ONCE(m->link_failed, true); + if (!m->cal_state.error) + m->cal_state.error = error; + m->cal_state.active = false; + complete_all(&m->cal_response); + if (first) + dev_err(&m->pdev->dev, + "RF_FAILED: %d; host RF admission closed, ownership retained until checked reset\n", + error); + if (!m->interface_registered) + return; + + /* Join a TX completion that may have passed its earlier wake check. */ + if (data_ready) + spin_lock(&m->data_lock); + netif_carrier_off(m->netdev); + netif_stop_queue(m->netdev); + if (data_ready) + spin_unlock(&m->data_lock); + if (m->connecting) { + if (!m->connect_error) + m->connect_error = error; + complete_all(&m->assoc_start); + complete_all(&m->assoc_done); + complete_all(&m->discovery_done); + } else if (m->connected) { + /* Hardware ownership is uncertain: report failure, retain the peer. */ + m->connected = false; + m->disconnecting = true; + cfg80211_disconnected(m->netdev, WLAN_REASON_UNSPECIFIED, + NULL, 0, true, GFP_ATOMIC); + } + if (m->scan_request) { + m->scan_aborted = true; + schedule_work(&m->scan_finish_work); + } +} + /* One owned station, firmware WPA2-PSK handshake; no host key fabrication. */ static int mt_peer_reserve(struct mt7932 *m) { @@ -183,13 +232,15 @@ static int mt_association_calibration(struct mt7932 *m) struct mt7932_cal_piece pieces[2]; int ret; - ret = request_firmware_direct(&oca, "mediatek/mt7932/oca2.bin", &m->pdev->dev); + ret = mt_request_input(m, &oca, "mediatek/mt7932/oca2.bin"); if (ret) return ret; ret = mt7932_cal_association(oca->data, oca->size, m->connect_center, pieces); if (ret == 2) ret = mt_cal_procedure(m, pieces, 2, 1, 0); release_firmware(oca); + if (ret) + dev_err(&m->pdev->dev, "association calibration using mediatek/mt7932/oca2.bin failed: %d\n", ret); return ret; } @@ -413,6 +464,8 @@ void mt_connect_work(struct work_struct *work) mutex_lock(&m->command_mutex); report: ret = ret ?: READ_ONCE(m->connect_error); + if (!ret && !mt_rf_allowed(m)) + ret = -EIO; if (!ret && (m->connect_open || m->band2.ht_cap.ht_supported) && !m->assoc_request_seen) ret = -EPROTO; @@ -440,8 +493,8 @@ void mt_connect_work(struct work_struct *work) * cfg80211 queues these notifications; GFP_ATOMIC keeps this IRQ-safe. */ spin_lock_irqsave(&m->response_lock, flags); - if (!ret && m->connect_error) { - ret = m->connect_error; + if (!ret && (m->connect_error || !mt_rf_allowed(m))) { + ret = m->connect_error ?: -EIO; spin_unlock_irqrestore(&m->response_lock, flags); goto report; } diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 308664d3e945a..75060c4be245d 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -56,7 +56,16 @@ bool mt_transport_polled(void) bool mt_rf_allowed(struct mt7932 *m) { return !READ_ONCE(m->stopping) && READ_ONCE(m->rf_ready) && !READ_ONCE(m->reg_pending) && - !READ_ONCE(m->policy_failed); + !READ_ONCE(m->policy_failed) && !READ_ONCE(m->link_failed); +} + +int mt_request_input(struct mt7932 *m, const struct firmware **fw, const char *name) +{ + int ret = request_firmware_direct(fw, name, &m->pdev->dev); + + if (ret) + dev_err(&m->pdev->dev, "local input %s unavailable: %d\n", name, ret); + return ret; } u32 mt_read(struct mt7932 *m, u32 reg) @@ -174,6 +183,8 @@ static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) m->smart_version); dev_info(&m->pdev->dev, "CAL_EVENT: seq=%u length=%zu result=%d credits=%u/%u\n", event.seq, event.length, ret, m->cal_state.received, m->cal_state.expected); + if (ret < 0) + mt_rf_fail_locked(m, ret); if (ret) complete(&m->cal_response); goto unlock; @@ -186,14 +197,11 @@ static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) * for checked reset, rather than silently declaring retirement safe. */ if (m->disconnecting || m->link_failed) { - m->cal_state.error = -EPROTO; - m->link_failed = true; - complete(&m->cal_response); + mt_rf_fail_locked(m, -EPROTO); goto unlock; } if (event.length < 52 || m->cal_request_count == ARRAY_SIZE(m->cal_requests)) { - m->cal_state.error = -EPROTO; - complete(&m->cal_response); + mt_rf_fail_locked(m, -EPROTO); } else { unsigned int at = (m->cal_request_head + m->cal_request_count) % ARRAY_SIZE(m->cal_requests); @@ -841,10 +849,11 @@ static int mt_precal_gate(struct mt7932 *m) /* Stock EfuseBufferModeCal=3, own eFuse board3: original PPR source2. * This is not the external EEPROM/source0 or own-WCAL/source3 path. */ - ret = request_firmware_direct(&ppr, "mediatek/mt7932/ppr.bin", &m->pdev->dev); + ret = mt_request_input(m, &ppr, "mediatek/mt7932/ppr.bin"); if (ret) return ret; if (ppr->size != sizeof(request) - 4) { + dev_err(&m->pdev->dev, "local input mediatek/mt7932/ppr.bin has invalid size %zu\n", ppr->size); release_firmware(ppr); return -EINVAL; } @@ -913,24 +922,32 @@ static int mt_firmware_gate(struct mt7932 *m) u32 entry = 0; int ret, patches, regions, i; - ret = request_firmware_direct(&patch, "mediatek/mt7932/IZUBA_WIFI_MT7932_patch_mcu_1_2_hdr.bin", &m->pdev->dev); + ret = mt_request_input(m, &patch, "mediatek/mt7932/IZUBA_WIFI_MT7932_patch_mcu_1_2_hdr.bin"); if (ret) return ret; - ret = request_firmware_direct(&ram, "mediatek/mt7932/IZUBA_W7932_2.bin", &m->pdev->dev); + ret = mt_request_input(m, &ram, "mediatek/mt7932/IZUBA_W7932_2.bin"); if (ret) goto out_patch; /* Validate all regions before emitting the first download command. */ patches = mt7932_patch_region(patch->data, patch->size, 0, ®ion); regions = mt7932_ram_region(ram->data, ram->size, 0, ®ion); ret = -EINVAL; - if (patches < 0 || regions < 0) + if (patches < 0 || regions < 0) { + dev_err(&m->pdev->dev, "local input %s has invalid region table\n", + patches < 0 ? "mediatek/mt7932/IZUBA_WIFI_MT7932_patch_mcu_1_2_hdr.bin" : + "mediatek/mt7932/IZUBA_W7932_2.bin"); goto out; + } for (i = 0; i < patches; i++) - if (mt7932_patch_region(patch->data, patch->size, i, ®ion) < 0) + if (mt7932_patch_region(patch->data, patch->size, i, ®ion) < 0) { + dev_err(&m->pdev->dev, "local input mediatek/mt7932/IZUBA_WIFI_MT7932_patch_mcu_1_2_hdr.bin has invalid region %d\n", i); goto out; + } for (i = 0; i < regions; i++) - if (mt7932_ram_region(ram->data, ram->size, i, ®ion) < 0) + if (mt7932_ram_region(ram->data, ram->size, i, ®ion) < 0) { + dev_err(&m->pdev->dev, "local input mediatek/mt7932/IZUBA_W7932_2.bin has invalid region %d\n", i); goto out; + } dev_info(&m->pdev->dev, "stock containers patch=%zu/%d regions RAM=%zu/%d regions\n", patch->size, patches, ram->size, regions); put_unaligned_le32(2, request); @@ -1308,3 +1325,11 @@ static struct pci_driver mt_driver = { module_pci_driver(mt_driver); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("J700 MT7932 firmware-owned cfg80211 station driver"); + +MODULE_FIRMWARE("mediatek/mt7932/IZUBA_WIFI_MT7932_patch_mcu_1_2_hdr.bin"); +MODULE_FIRMWARE("mediatek/mt7932/IZUBA_W7932_2.bin"); +MODULE_FIRMWARE("mediatek/mt7932/ppr.bin"); +MODULE_FIRMWARE("mediatek/mt7932/wcal.bin"); +MODULE_FIRMWARE("mediatek/mt7932/oca2.bin"); +MODULE_FIRMWARE("mediatek/mt7932/config-original.bin"); +MODULE_FIRMWARE("mediatek/mt7932/policy/world-XZ.bin"); diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 3e30ec3aff659..b7af4a04946b3 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -166,6 +166,9 @@ struct mt7932 { }; bool mt_rf_allowed(struct mt7932 *m); +int mt_request_input(struct mt7932 *m, const struct firmware **fw, const char *name); +/* response_lock held; closes RF admission and reports terminal host state. */ +void mt_rf_fail_locked(struct mt7932 *m, int error); u32 mt_read(struct mt7932 *m, u32 reg); void mt_write(struct mt7932 *m, u32 reg, u32 value); void mt_rmw(struct mt7932 *m, u32 reg, u32 mask, u32 value); diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index d189f4ad13943..26967048588fc 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -26,7 +26,7 @@ static int mt_stock_config(struct mt7932 *m) unsigned int count, i, j, batch; int ret; - ret = request_firmware_direct(&file, "mediatek/mt7932/config-original.bin", &m->pdev->dev); + ret = mt_request_input(m, &file, "mediatek/mt7932/config-original.bin"); if (ret) return ret; ret = -EINVAL; @@ -120,6 +120,8 @@ static int mt_stock_config(struct mt7932 *m) free: kfree(records); out: + if (ret) + dev_err(&m->pdev->dev, "local input mediatek/mt7932/config-original.bin processing failed: %d\n", ret); release_firmware(file); return ret; } @@ -271,10 +273,13 @@ static void mt_startup_work(struct work_struct *work) else scnprintf(path, sizeof(path), "mediatek/mt7932/policy/%c%c.bin", reg.alpha2[0], reg.alpha2[1]); - ret = request_firmware_direct(&file, path, &m->pdev->dev); + ret = mt_request_input(m, &file, path); } - if (!ret) + if (!ret) { ret = mt7932_policy_parse(&policy, file->data, file->size, reg.domain); + if (ret) + dev_err(&m->pdev->dev, "local input %s has invalid policy: %d\n", path, ret); + } mutex_lock(&m->command_mutex); if (READ_ONCE(m->stopping) || generation != READ_ONCE(m->reg_generation)) goto next; From f65cbe3ea885894a40b82c8035cd5f23d7114589 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 00:58:56 -0700 Subject: [PATCH 0123/2680] wifi: mt7932: serialize readiness with terminal RF failure Reject terminal errors before data/BSS startup and again before interface readiness publication. Keep pre-policy interface-up behavior while refusing a failed epoch. Publish regulatory completion under the same response lock as RF failure and recheck scan abortion before transferring the request to cfg80211. --- .../net/wireless/mediatek/mt7932/frontend.c | 31 +++++++++++++++++-- .../net/wireless/mediatek/mt7932/startup.c | 11 +++++-- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 174beebc8f6ab..781fb4f60dbca 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -47,6 +47,8 @@ static void mt_scan_finish_work(struct work_struct *work) } cancel_delayed_work(&m->scan_timeout_work); spin_lock_irqsave(&m->response_lock, flags); + /* A terminal failure may have followed the worker's earlier snapshot. */ + info.aborted |= m->scan_aborted || !mt_rf_allowed(m); cancel[0] = m->scan_seq; if (request && info.aborted && !m->scan_finished) m->retired_scan_seq = m->scan_seq; @@ -387,17 +389,28 @@ static int mt_set_bss(struct mt7932 *m, bool active) static int mt_net_open(struct net_device *netdev) { struct mt7932 *m = *(struct mt7932 **)netdev_priv(netdev); + unsigned long flags; int ret = 0; mutex_lock(&m->command_mutex); - if (READ_ONCE(m->stopping)) { + spin_lock_irqsave(&m->response_lock, flags); + if (m->stopping) + ret = -ESHUTDOWN; + else if (m->link_failed || m->cal_state.error) + ret = m->cal_state.error ?: -EIO; + spin_unlock_irqrestore(&m->response_lock, flags); + if (ret) { mutex_unlock(&m->command_mutex); - return -ESHUTDOWN; + return ret; } if (READ_ONCE(m->rf_ready)) ret = mt_set_bss(m, true); + spin_lock_irqsave(&m->response_lock, flags); + if (!ret && (m->stopping || m->link_failed || m->cal_state.error)) + ret = m->cal_state.error ?: (m->stopping ? -ESHUTDOWN : -EIO); if (!ret) WRITE_ONCE(m->interface_up, true); + spin_unlock_irqrestore(&m->response_lock, flags); mutex_unlock(&m->command_mutex); netif_carrier_off(netdev); netif_stop_queue(netdev); @@ -555,8 +568,20 @@ int mt_register_interface(struct mt7932 *m) int mt_enable_scan(struct mt7932 *m) { unsigned long flags; - int ret = mt_data_prepare(m); + int ret; + + spin_lock_irqsave(&m->response_lock, flags); + ret = m->cal_state.error ?: ((m->stopping || m->link_failed) ? -EIO : 0); + spin_unlock_irqrestore(&m->response_lock, flags); + if (ret) + return ret; + ret = mt_data_prepare(m); + if (ret) + return ret; + spin_lock_irqsave(&m->response_lock, flags); + ret = m->cal_state.error ?: ((m->stopping || m->link_failed) ? -EIO : 0); + spin_unlock_irqrestore(&m->response_lock, flags); if (ret) return ret; if (READ_ONCE(m->interface_up)) { diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 26967048588fc..1e4ecc6f065ac 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -232,6 +232,7 @@ static void mt_startup_work(struct work_struct *work) unsigned long flags; u32 generation; char path[64]; + bool submitted; int ret; for (;;) { @@ -263,6 +264,7 @@ static void mt_startup_work(struct work_struct *work) if (READ_ONCE(m->retired_scan_seq)) ret = -EBUSY; file = NULL; + submitted = false; if (READ_ONCE(m->policy_failed) || READ_ONCE(m->link_failed) || (m->startup_started && !READ_ONCE(m->rf_ready))) ret = -EIO; @@ -284,19 +286,21 @@ static void mt_startup_work(struct work_struct *work) if (READ_ONCE(m->stopping) || generation != READ_ONCE(m->reg_generation)) goto next; if (!ret) { + submitted = true; ret = mt_publish_policy(m, ®, &policy); if (!ret && !m->startup_started) ret = mt_startup_once(m); /* A partial SET sequence is not a rollback. Only missing or * invalid files BEFORE submission are recoverable by a hint. */ - if (ret) - WRITE_ONCE(m->policy_failed, true); } spin_lock_irqsave(&m->response_lock, flags); + if (!ret && (m->stopping || m->link_failed || m->cal_state.error)) + ret = m->cal_state.error ?: -EIO; + if (ret && submitted) + WRITE_ONCE(m->policy_failed, true); if (!ret && generation == m->reg_generation) WRITE_ONCE(m->reg_pending, false); - spin_unlock_irqrestore(&m->response_lock, flags); if (ret) dev_warn(&m->pdev->dev, "REGULATORY_BLOCKED: %c%c generation=%u error=%d recovery-required=%u\n", reg.alpha2[0], reg.alpha2[1], generation, ret, @@ -305,6 +309,7 @@ static void mt_startup_work(struct work_struct *work) dev_info(&m->pdev->dev, "REGULATORY_READY: %c%c firmware=%c%c generation=%u current=%u\n", reg.alpha2[0], reg.alpha2[1], reg.domain[0], reg.domain[1], generation, !m->reg_pending); + spin_unlock_irqrestore(&m->response_lock, flags); next: mutex_unlock(&m->command_mutex); if (file) From 657550cc19f6e73713be8ad89d377c3085baab94 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:01:37 -0700 Subject: [PATCH 0124/2680] wifi: mt7932: test terminal RF failure and startup admission --- drivers/net/wireless/mediatek/mt7932/Kconfig | 10 ++ drivers/net/wireless/mediatek/mt7932/Makefile | 1 + .../net/wireless/mediatek/mt7932/rf-test.c | 126 ++++++++++++++++++ 3 files changed, 137 insertions(+) create mode 100644 drivers/net/wireless/mediatek/mt7932/rf-test.c diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index 9c81c31a18f5e..239de549c8947 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -18,3 +18,13 @@ config MT7932_FULLMAC reset before releasing DMA storage. A failed reset blocks removal, retaining resources until an explicit retry succeeds or system reset. Other MediaTek devices and other machines are not matched. + +config MT7932_RF_KUNIT_TEST + bool "KUnit tests for MT7932 terminal RF failure" if !KUNIT_ALL_TESTS + depends on KUNIT=y && MT7932_FULLMAC + default KUNIT_ALL_TESTS + help + Exercise the driver's terminal calibration failure and startup + admission paths with synthetic devices, without radio hardware. + Check association wakeups, retained peer ownership and refusal to + reopen RF admission after a terminal calibration error. diff --git a/drivers/net/wireless/mediatek/mt7932/Makefile b/drivers/net/wireless/mediatek/mt7932/Makefile index 2ac830d058ca4..9e76679c46a8a 100644 --- a/drivers/net/wireless/mediatek/mt7932/Makefile +++ b/drivers/net/wireless/mediatek/mt7932/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_MT7932_FULLMAC) += mt7932-fullmac.o mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o station_query.o +mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += rf-test.o diff --git a/drivers/net/wireless/mediatek/mt7932/rf-test.c b/drivers/net/wireless/mediatek/mt7932/rf-test.c new file mode 100644 index 0000000000000..bf346b26cf6f7 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/rf-test.c @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include + +#include "mt7932.h" + +static void mt_rf_test_device_release(struct device *dev) +{ + /* The enclosing synthetic PCI device is owned by KUnit. */ +} + +static void mt_rf_test_cleanup(void *data) +{ + struct mt7932 *m = data; + + free_netdev(m->netdev); + put_device(&m->pdev->dev); +} + +static int mt_rf_test_init(struct kunit *test) +{ + struct mt7932 *m; + int ret; + + m = kunit_kzalloc(test, sizeof(*m), GFP_KERNEL); + if (!m) + return -ENOMEM; + m->pdev = kunit_kzalloc(test, sizeof(*m->pdev), GFP_KERNEL); + if (!m->pdev) + return -ENOMEM; + device_initialize(&m->pdev->dev); + m->pdev->dev.release = mt_rf_test_device_release; + ret = dev_set_name(&m->pdev->dev, "mt7932-rf-test"); + if (ret) { + put_device(&m->pdev->dev); + return ret; + } + m->netdev = alloc_etherdev(0); + if (!m->netdev) { + put_device(&m->pdev->dev); + return -ENOMEM; + } + ret = kunit_add_action_or_reset(test, mt_rf_test_cleanup, m); + if (ret) + return ret; + spin_lock_init(&m->response_lock); + spin_lock_init(&m->data_lock); + init_completion(&m->cal_response); + init_completion(&m->assoc_start); + init_completion(&m->assoc_done); + init_completion(&m->discovery_done); + m->interface_registered = true; + m->connecting = true; + m->peer_valid = true; + m->rf_ready = true; + m->cal_state.active = true; + netif_carrier_on(m->netdev); + netif_start_queue(m->netdev); + test->priv = m; + return 0; +} + +static void mt_rf_test_fail(struct mt7932 *m, int error) +{ + unsigned long flags; + + spin_lock_irqsave(&m->response_lock, flags); + mt_rf_fail_locked(m, error); + spin_unlock_irqrestore(&m->response_lock, flags); +} + +static void mt_rf_failure_wakes_association_test(struct kunit *test) +{ + struct mt7932 *m = test->priv; + + /* Exercise the data-publisher join, without publishing any DMA. */ + smp_store_release(&m->data_ready, true); + mt_rf_test_fail(m, -ENOENT); + + KUNIT_EXPECT_TRUE(test, completion_done(&m->cal_response)); + KUNIT_EXPECT_TRUE(test, completion_done(&m->assoc_start)); + KUNIT_EXPECT_TRUE(test, completion_done(&m->assoc_done)); + KUNIT_EXPECT_TRUE(test, completion_done(&m->discovery_done)); + KUNIT_EXPECT_EQ(test, m->connect_error, -ENOENT); + KUNIT_EXPECT_FALSE(test, mt_rf_allowed(m)); + KUNIT_EXPECT_FALSE(test, netif_carrier_ok(m->netdev)); + KUNIT_EXPECT_TRUE(test, netif_queue_stopped(m->netdev)); + /* An error must not claim that the firmware-owned peer was retired. */ + KUNIT_EXPECT_TRUE(test, m->peer_valid); +} + +static void mt_rf_failure_blocks_startup_test(struct kunit *test) +{ + struct mt7932 *m = test->priv; + + mt_rf_test_fail(m, -EPROTO); + /* No BAR/DMA exists: startup must reject before touching hardware. */ + KUNIT_EXPECT_EQ(test, mt_enable_scan(m), -EPROTO); + KUNIT_EXPECT_FALSE(test, m->rf_ready); + KUNIT_EXPECT_FALSE(test, m->data_ready); +} + +static void mt_rf_failure_preserves_first_error_test(struct kunit *test) +{ + struct mt7932 *m = test->priv; + + mt_rf_test_fail(m, -ENOENT); + mt_rf_test_fail(m, -EPROTO); + KUNIT_EXPECT_EQ(test, m->cal_state.error, -ENOENT); + KUNIT_EXPECT_EQ(test, m->connect_error, -ENOENT); + KUNIT_EXPECT_EQ(test, mt_enable_scan(m), -ENOENT); +} + +static struct kunit_case mt_rf_test_cases[] = { + KUNIT_CASE(mt_rf_failure_wakes_association_test), + KUNIT_CASE(mt_rf_failure_blocks_startup_test), + KUNIT_CASE(mt_rf_failure_preserves_first_error_test), + {} +}; + +static struct kunit_suite mt_rf_test_suite = { + .name = "mt7932-rf-failure", + .init = mt_rf_test_init, + .test_cases = mt_rf_test_cases, +}; + +kunit_test_suite(mt_rf_test_suite); From 9924c5dc5a990c113e52b068efec97a0847501c5 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:23:17 -0700 Subject: [PATCH 0125/2680] selftests: mt7932: bind DNS to Wi-Fi and select desktop qualification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Query the Wi-Fi connection's IPv4 DNS servers over a device-bound UDP socket, reject malformed or mismatched answers, and use the verified A address for direct interface-bound HTTPS. Keep KDE qualification as the default while allowing an explicit network-only test, and stop qualification on RF_FAILED. Add local parser and real loopback socket regression tests. [ryan: adapted to 7.2.10 — moved from the tooling bucket into the radio topic with the driver it tests; 900-config-tooling skips it] --- tools/testing/selftests/drivers/net/Makefile | 2 +- .../selftests/drivers/net/mt7932_e2e.py | 315 ++++++++++++------ .../selftests/drivers/net/mt7932_e2e_test.py | 131 ++++++++ 3 files changed, 341 insertions(+), 107 deletions(-) create mode 100644 tools/testing/selftests/drivers/net/mt7932_e2e_test.py diff --git a/tools/testing/selftests/drivers/net/Makefile b/tools/testing/selftests/drivers/net/Makefile index 39e6107cf0269..5c93e30014873 100644 --- a/tools/testing/selftests/drivers/net/Makefile +++ b/tools/testing/selftests/drivers/net/Makefile @@ -6,7 +6,7 @@ TEST_INCLUDES := $(wildcard lib/py/*.py) \ ../../net/lib.sh \ # Manual physical-radio test; requires an explicitly selected saved network. -TEST_FILES += mt7932_e2e.py +TEST_FILES += mt7932_e2e.py mt7932_e2e_test.py TEST_GEN_FILES := \ napi_id_helper \ diff --git a/tools/testing/selftests/drivers/net/mt7932_e2e.py b/tools/testing/selftests/drivers/net/mt7932_e2e.py index b02a7fc717e06..ec90e0dd32a1e 100644 --- a/tools/testing/selftests/drivers/net/mt7932_e2e.py +++ b/tools/testing/selftests/drivers/net/mt7932_e2e.py @@ -8,28 +8,20 @@ """ import argparse import hashlib +import ipaddress import json import os from pathlib import Path import re +import socket +import struct import subprocess import time -parser = argparse.ArgumentParser() -parser.add_argument('--expected-release', required=True) -parser.add_argument('--profile', required=True, help='Unique existing NetworkManager profile name') -parser.add_argument('--connect', action='store_true') -parser.add_argument('--attempt', default='e2e') -parser.add_argument('--band', choices=['bg', 'a'], default='bg') -args = parser.parse_args() -if not re.fullmatch(r'[a-z0-9-]{1,48}', args.attempt): - parser.error('Invalid attempt name') -out = Path('/tmp/neo-wifi-network-' + args.attempt) -result = {'result': 'NOT_STARTED', 'steps': [], 'samples': []} fault = re.compile(r'translation fault|stale exception latched|Kernel panic|Internal error:|' r'SError Interrupt|BUG:|Unable to handle kernel|WIFI_TERMINAL|WIFI_STARTUP_FAILED|' r'qualification stopped:|CAL_PROCEDURE_FAILED|D7_CALIBRATION_FAILED|' - r'SCAN_INTERFACE_SETUP_FAILED|recovery-required=1', re.I) + r'SCAN_INTERFACE_SETUP_FAILED|RF_FAILED|recovery-required=1', re.I) def run(argv, timeout=30): @@ -48,99 +40,210 @@ def kernel_ok(): return p.stdout -require(os.geteuid() == 0 and os.uname().release == args.expected_release, 'Wrong host/kernel') -require(not out.exists(), 'Fresh named E2E attempt required') -compatible = Path('/sys/firmware/devicetree/base/compatible').read_bytes().split(b'\0') -require(b'apple,j700' in compatible and b'apple,t8140' in compatible, 'Wrong device') -out.mkdir(mode=0o700) -try: - kernel_ok() - profile = run(['nmcli', '-g', 'connection.uuid', 'connection', 'show', 'id', args.profile]) - profile_uuid = profile.stdout.strip() - require(profile.returncode == 0 and re.fullmatch(r'[0-9a-f-]{36}', profile_uuid), - 'Missing or ambiguous profile') - ssid_query = run(['nmcli', '-g', '802-11-wireless.ssid', 'connection', 'show', - 'uuid', profile_uuid]) - ssid = ssid_query.stdout.strip() - require(ssid_query.returncode == 0 and ssid, 'Profile has no SSID') - devices = [p.name for p in Path('/sys/class/net').iterdir() if (p / 'wireless').exists()] - require(len(devices) == 1, 'Expected one wireless interface') - iface = devices[0] - result['kernel_release'] = os.uname().release - result['interface'] = iface - require(run(['ip', 'link', 'set', iface, 'up']).returncode == 0, 'Interface up failed') - require(run(['nmcli', 'device', 'set', iface, 'managed', 'yes']).returncode == 0, - 'NetworkManager admission failed') - scan = run(['nmcli', '--wait', '30', 'device', 'wifi', 'rescan', 'ifname', iface], 35) - result['rescan_exit'] = scan.returncode - # NetworkManager may already own a scan when the explicit request arrives. - # Poll its cache without starting a second scan and check firmware health. - names = [] - for attempt in range(20): - time.sleep(2) +def dns_name(packet, offset): + """Decode a bounded DNS name, rejecting bad offsets and compression cycles.""" + labels, seen, end, size = [], set(), None, 1 + while True: + require(offset < len(packet) and offset not in seen, 'Invalid DNS name') + seen.add(offset) + length = packet[offset] + if length & 0xc0 == 0xc0: + require(offset + 1 < len(packet), 'Truncated DNS pointer') + if end is None: + end = offset + 2 + offset = ((length & 0x3f) << 8) | packet[offset + 1] + continue + require(not (length & 0xc0), 'Invalid DNS label') + offset += 1 + if not length: + return b'.'.join(labels).lower(), end if end is not None else offset + require(offset + length <= len(packet), 'Truncated DNS label') + size += length + 1 + require(size <= 255, 'DNS name too long') + labels.append(packet[offset:offset + length]) + offset += length + + +def dns_answer(packet, query_id): + """Accept a matching, complete successful A response for example.com.""" + require(len(packet) >= 12, 'Truncated DNS header') + ident, flags, questions, answers, _, _ = struct.unpack_from('!6H', packet) + require(ident == query_id and flags & 0x8000 and not (flags & 0x7a0f), + 'DNS reply rejected') # QR required; opcode, TC and rcode must be zero. + require(questions == 1 and answers > 0, 'DNS positive answer missing') + name, offset = dns_name(packet, 12) + require(offset + 4 <= len(packet) and name == b'example.com' and + struct.unpack_from('!HH', packet, offset) == (1, 1), 'DNS question mismatch') + offset += 4 + addresses, aliases = {}, {} + for _ in range(answers): + owner, offset = dns_name(packet, offset) + require(offset + 10 <= len(packet), 'Truncated DNS record') + kind, cls, _, size = struct.unpack_from('!HHIH', packet, offset) + offset += 10 + require(offset + size <= len(packet), 'Truncated DNS answer') + if cls == 1 and kind == 1: + require(size == 4, 'Invalid DNS A answer') + address = ipaddress.IPv4Address(packet[offset:offset + size]) + require(not (address.is_unspecified or address.is_multicast or address.is_loopback), + 'Unusable DNS A answer') + addresses.setdefault(owner, str(address)) + elif cls == 1 and kind == 5: + target, end = dns_name(packet, offset) + require(end == offset + size, 'Invalid DNS CNAME answer') + aliases[owner] = target + offset += size + name, seen = b'example.com', set() + while name not in seen: + if name in addresses: + return addresses[name] + seen.add(name) + require(name in aliases, 'DNS answer does not resolve question') + name = aliases[name] + raise RuntimeError('DNS CNAME cycle') + + +def dns_query(iface, server, port=53, timeout=5): + """Connect UDP to one DNS peer with both requests and replies bound to iface.""" + query_id = int.from_bytes(os.urandom(2), 'big') + query = (struct.pack('!6H', query_id, 0x100, 1, 0, 0, 0) + + b'\x07example\x03com\0' + struct.pack('!HH', 1, 1)) + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + sock.settimeout(timeout) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_BINDTODEVICE, os.fsencode(iface) + b'\0') + sock.connect((server, port)) + sock.send(query) + return dns_answer(sock.recv(4096), query_id) + + +def wifi_dns(iface): + servers = run(['nmcli', '-g', 'IP4.DNS', 'device', 'show', iface]) + require(servers.returncode == 0, 'Wi-Fi DNS configuration unavailable') + for line in servers.stdout.splitlines(): + try: + address = ipaddress.IPv4Address(line.strip()) + if address.is_loopback or address.is_unspecified or address.is_multicast: + continue + return dns_query(iface, str(address)) + except (OSError, ValueError, RuntimeError): + continue + # Never disclose the unit's server addresses or fall back to a global resolver. + raise RuntimeError('Wi-Fi-bound upstream DNS lookup failed') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--expected-release', required=True) + parser.add_argument('--profile', required=True, help='Unique existing NetworkManager profile name') + parser.add_argument('--connect', action='store_true') + parser.add_argument('--attempt', default='e2e') + parser.add_argument('--band', choices=['bg', 'a'], default='bg') + parser.add_argument('--desktop', choices=['kde', 'none'], default='kde', + help='Desktop qualification; none checks network without desktop requirements') + args = parser.parse_args() + if not re.fullmatch(r'[a-z0-9-]{1,48}', args.attempt): + parser.error('Invalid attempt name') + out = Path('/tmp/neo-wifi-network-' + args.attempt) + result = {'result': 'NOT_STARTED', 'steps': [], 'samples': []} + require(os.geteuid() == 0 and os.uname().release == args.expected_release, 'Wrong host/kernel') + require(not out.exists(), 'Fresh named E2E attempt required') + compatible = Path('/sys/firmware/devicetree/base/compatible').read_bytes().split(b'\0') + require(b'apple,j700' in compatible and b'apple,t8140' in compatible, 'Wrong device') + out.mkdir(mode=0o700) + try: kernel_ok() - listings = run(['nmcli', '-t', '-f', 'SSID', 'device', 'wifi', 'list', - '--rescan', 'no', 'ifname', iface]) - require(listings.returncode == 0, 'Scan result read failed') - names = listings.stdout.splitlines() - if ssid in names: - break - result['visible_networks'] = len([name for name in names if name]) - result['requested_network_visible'] = ssid in names - require(result['requested_network_visible'], 'Requested network absent from scan') - result['steps'].append('REQUESTED_NETWORK_SCAN_OBSERVED') - if not args.connect: - result['result'] = 'WIFI_SCAN_OBSERVED' - else: - require(run(['nmcli', 'connection', 'modify', 'uuid', profile_uuid, - '802-11-wireless.band', args.band]).returncode == 0, 'Band selection failed') - connection = run(['nmcli', '--wait', '60', 'connection', 'up', 'uuid', - profile_uuid, 'ifname', iface], 65) - result['activation_exit'] = connection.returncode - link = run(['iw', 'dev', iface, 'link']).stdout - freq = re.search(r'freq: (\d+)', link) - result['frequency_mhz'] = int(freq[1]) if freq else None - require(freq and ((int(freq[1]) < 3000) == (args.band == 'bg')), 'Unexpected associated band') - require(connection.returncode == 0, 'Saved profile activation failed') - for sample in range(3): - addresses = json.loads(run(['ip', '-j', '-4', 'address', 'show', 'dev', iface]).stdout) - require(any(a.get('scope') == 'global' for link in addresses - for a in link.get('addr_info', [])), 'No DHCP IPv4 address') - routes = json.loads(run(['ip', '-j', '-4', 'route', 'show', 'default', 'dev', iface]).stdout) - require(routes and routes[0].get('gateway'), 'No Wi-Fi default gateway') - ping = run(['ping', '-I', iface, '-c', '3', '-W', '3', routes[0]['gateway']], 15) - require(ping.returncode == 0 and re.search(r'\b0(?:\.0+)?% packet loss', ping.stdout) - and re.search(r'\b3 received\b', ping.stdout), 'Gateway packet loss') - dns = run(['getent', 'ahostsv4', 'example.com']) - require(dns.returncode == 0 and dns.stdout.strip(), 'DNS lookup failed') - payload = out / ('https-' + str(sample) + '.html') - transfer = run(['curl', '--fail', '--silent', '--show-error', '--interface', iface, - '--max-time', '20', '--output', str(payload), 'https://example.com/'], 25) - require(transfer.returncode == 0 and payload.is_file(), 'Wi-Fi HTTPS transfer failed') - body = payload.read_bytes() - require(b'Example Domain' in body and len(body) > 100, 'Unexpected HTTPS response') + profile = run(['nmcli', '-g', 'connection.uuid', 'connection', 'show', 'id', args.profile]) + profile_uuid = profile.stdout.strip() + require(profile.returncode == 0 and re.fullmatch(r'[0-9a-f-]{36}', profile_uuid), + 'Missing or ambiguous profile') + ssid_query = run(['nmcli', '-g', '802-11-wireless.ssid', 'connection', 'show', + 'uuid', profile_uuid]) + ssid = ssid_query.stdout.strip() + require(ssid_query.returncode == 0 and ssid, 'Profile has no SSID') + devices = [p.name for p in Path('/sys/class/net').iterdir() if (p / 'wireless').exists()] + require(len(devices) == 1, 'Expected one wireless interface') + iface = devices[0] + result['kernel_release'] = os.uname().release + result['desktop_qualification'] = args.desktop + result['interface'] = iface + require(run(['ip', 'link', 'set', iface, 'up']).returncode == 0, 'Interface up failed') + require(run(['nmcli', 'device', 'set', iface, 'managed', 'yes']).returncode == 0, + 'NetworkManager admission failed') + scan = run(['nmcli', '--wait', '30', 'device', 'wifi', 'rescan', 'ifname', iface], 35) + result['rescan_exit'] = scan.returncode + # NetworkManager may already own a scan when the explicit request arrives. + # Poll its cache without starting a second scan and check firmware health. + names = [] + for attempt in range(20): + time.sleep(2) kernel_ok() - result['samples'].append({'gateway_received': 3, 'gateway_loss_percent': 0, - 'dns': True, 'tls_verified': True, 'download_bytes': len(body), - 'download_sha256': hashlib.sha256(body).hexdigest()}) - if sample != 2: - time.sleep(10) - require(all(run(['pgrep', '-x', name]).returncode == 0 - for name in ('plasmashell', 'kwin_x11')), 'KDE not running') - result['steps'].extend(['WPA2_SAVED_PROFILE_ACTIVATED', 'DHCP_ROUTE_OBSERVED', - 'GATEWAY_ROUNDTRIPS_VERIFIED', 'DNS_TLS_DOWNLOAD_VERIFIED', - 'KDE_RUNNING_NO_KERNEL_FAULT']) - result['result'] = 'WIFI_PHYSICAL_NETWORK_E2E_PASS' -except subprocess.TimeoutExpired: - result['result'] = 'WIFI_NETWORK_E2E_FAILED' - result['error'] = 'External command timed out' -except Exception as error: - result['result'] = 'WIFI_NETWORK_E2E_FAILED' - result['error'] = str(error) -finally: - log = run(['dmesg']).stdout - result['kernel_faults'] = len(fault.findall(log)) - (out / 'result.json').write_text(json.dumps(result, indent=2) + '\n') - print(json.dumps(result, indent=2)) - print('ARTIFACT=' + str(out / 'result.json')) -raise SystemExit(0 if result['result'] in ('WIFI_SCAN_OBSERVED', 'WIFI_PHYSICAL_NETWORK_E2E_PASS') else 1) + listings = run(['nmcli', '-t', '-f', 'SSID', 'device', 'wifi', 'list', + '--rescan', 'no', 'ifname', iface]) + require(listings.returncode == 0, 'Scan result read failed') + names = listings.stdout.splitlines() + if ssid in names: + break + result['visible_networks'] = len([name for name in names if name]) + result['requested_network_visible'] = ssid in names + require(result['requested_network_visible'], 'Requested network absent from scan') + result['steps'].append('REQUESTED_NETWORK_SCAN_OBSERVED') + if not args.connect: + result['result'] = 'WIFI_SCAN_OBSERVED' + else: + require(run(['nmcli', 'connection', 'modify', 'uuid', profile_uuid, + '802-11-wireless.band', args.band]).returncode == 0, 'Band selection failed') + connection = run(['nmcli', '--wait', '60', 'connection', 'up', 'uuid', + profile_uuid, 'ifname', iface], 65) + result['activation_exit'] = connection.returncode + link = run(['iw', 'dev', iface, 'link']).stdout + freq = re.search(r'freq: (\d+)', link) + result['frequency_mhz'] = int(freq[1]) if freq else None + require(freq and ((int(freq[1]) < 3000) == (args.band == 'bg')), 'Unexpected associated band') + require(connection.returncode == 0, 'Saved profile activation failed') + for sample in range(3): + addresses = json.loads(run(['ip', '-j', '-4', 'address', 'show', 'dev', iface]).stdout) + require(any(a.get('scope') == 'global' for link in addresses + for a in link.get('addr_info', [])), 'No DHCP IPv4 address') + routes = json.loads(run(['ip', '-j', '-4', 'route', 'show', 'default', 'dev', iface]).stdout) + require(routes and routes[0].get('gateway'), 'No Wi-Fi default gateway') + ping = run(['ping', '-I', iface, '-c', '3', '-W', '3', routes[0]['gateway']], 15) + require(ping.returncode == 0 and re.search(r'\b0(?:\.0+)?% packet loss', ping.stdout) + and re.search(r'\b3 received\b', ping.stdout), 'Gateway packet loss') + dns_address = wifi_dns(iface) + payload = out / ('https-' + str(sample) + '.html') + transfer = run(['curl', '--fail', '--silent', '--show-error', '--interface', iface, + '--noproxy', '*', '--resolve', 'example.com:443:' + dns_address, + '--max-time', '20', '--output', str(payload), 'https://example.com/'], 25) + require(transfer.returncode == 0 and payload.is_file(), 'Wi-Fi HTTPS transfer failed') + body = payload.read_bytes() + require(b'Example Domain' in body and len(body) > 100, 'Unexpected HTTPS response') + kernel_ok() + result['samples'].append({'gateway_received': 3, 'gateway_loss_percent': 0, + 'dns': True, 'tls_verified': True, 'download_bytes': len(body), + 'download_sha256': hashlib.sha256(body).hexdigest()}) + if sample != 2: + time.sleep(10) + if args.desktop == 'kde': + require(all(run(['pgrep', '-x', name]).returncode == 0 + for name in ('plasmashell', 'kwin_x11')), 'KDE not running') + result['steps'].append('KDE_RUNNING_NO_KERNEL_FAULT') + result['steps'].extend(['WPA2_SAVED_PROFILE_ACTIVATED', 'DHCP_ROUTE_OBSERVED', + 'GATEWAY_ROUNDTRIPS_VERIFIED', 'WIFI_BOUND_DNS_TLS_DOWNLOAD_VERIFIED']) + result['result'] = 'WIFI_PHYSICAL_NETWORK_E2E_PASS' + except subprocess.TimeoutExpired: + result['result'] = 'WIFI_NETWORK_E2E_FAILED' + result['error'] = 'External command timed out' + except Exception as error: + result['result'] = 'WIFI_NETWORK_E2E_FAILED' + result['error'] = str(error) + finally: + log = run(['dmesg']).stdout + result['kernel_faults'] = len(fault.findall(log)) + (out / 'result.json').write_text(json.dumps(result, indent=2) + '\n') + print(json.dumps(result, indent=2)) + print('ARTIFACT=' + str(out / 'result.json')) + return 0 if result['result'] in ('WIFI_SCAN_OBSERVED', 'WIFI_PHYSICAL_NETWORK_E2E_PASS') else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/testing/selftests/drivers/net/mt7932_e2e_test.py b/tools/testing/selftests/drivers/net/mt7932_e2e_test.py new file mode 100644 index 0000000000000..f58a8eeb9bb71 --- /dev/null +++ b/tools/testing/selftests/drivers/net/mt7932_e2e_test.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Local DNS unit/socket tests; never accesses a physical radio or network.""" +import socket +import struct +import subprocess +import threading +import unittest +from unittest import mock + +import mt7932_e2e as e2e + +QUESTION = b'\x07example\x03com\0' + struct.pack('!HH', 1, 1) +ADDRESS = b'\xcb\x00\x71\x05' # Documentation-only address, never contacted. +IDENT = 1234 + + +def record(owner=b'\xc0\x0c', kind=1, data=ADDRESS): + return owner + struct.pack('!HHIH', kind, 1, 60, len(data)) + data + + +def reply(ident=IDENT, flags=0x8180, question=QUESTION, records=None): + if records is None: + records = [record()] + return (struct.pack('!6H', ident, flags, 1, len(records), 0, 0) + question + + b''.join(records)) + + +class DNSAnswerTest(unittest.TestCase): + def test_positive_compressed_a(self): + self.assertEqual(e2e.dns_answer(reply(), IDENT), '203.0.113.5') + + def test_cname_answer(self): + target = b'\x06target\x07example\x03com\0' + packet = reply(records=[record(kind=5, data=target), record(owner=target)]) + self.assertEqual(e2e.dns_answer(packet, IDENT), '203.0.113.5') + + def test_reject_wrong_transaction_or_flags(self): + for packet in (reply(ident=99), reply(flags=0x0180), reply(flags=0x8380), + reply(flags=0x8183), reply(flags=0x8980)): + with self.subTest(packet=packet[:4]), self.assertRaises(RuntimeError): + e2e.dns_answer(packet, IDENT) + + def test_reject_wrong_question_and_unrelated_answers(self): + for packet in (reply(question=b'\x05other\x03com\0' + struct.pack('!HH', 1, 1)), + reply(question=QUESTION[:-4] + struct.pack('!HH', 28, 1)), + reply(records=[]), reply(records=[record(owner=b'\x05other\0')]), + reply(records=[record(data=b'\x7f\0\0\1')])): + with self.subTest(packet=packet), self.assertRaises(RuntimeError): + e2e.dns_answer(packet, IDENT) + + def test_reject_every_truncation(self): + packet = reply() + for size in range(len(packet)): + with self.subTest(size=size), self.assertRaises(RuntimeError): + e2e.dns_answer(packet[:size], IDENT) + + def test_reject_bad_and_cyclic_name_pointers(self): + owner_offset = 12 + len(QUESTION) + for owner in (b'\xff\xff', struct.pack('!H', 0xc000 | owner_offset), b'\x40'): + with self.subTest(owner=owner), self.assertRaises(RuntimeError): + e2e.dns_answer(reply(records=[record(owner=owner)]), IDENT) + + def test_upstream_selection_and_error_privacy(self): + config = subprocess.CompletedProcess([], 0, '127.0.0.53\n192.0.2.53\n') + with mock.patch.object(e2e, 'run', return_value=config) as run: + with mock.patch.object(e2e, 'dns_query', return_value='203.0.113.5') as query: + self.assertEqual(e2e.wifi_dns('testwifi'), '203.0.113.5') + run.assert_called_once_with(['nmcli', '-g', 'IP4.DNS', 'device', 'show', 'testwifi']) + query.assert_called_once_with('testwifi', '192.0.2.53') + with mock.patch.object(e2e, 'dns_query', side_effect=OSError('private server details')): + with self.assertRaisesRegex(RuntimeError, '^Wi-Fi-bound upstream DNS lookup failed$'): + e2e.wifi_dns('testwifi') + + def test_rf_failed_is_terminal(self): + self.assertIsNotNone(e2e.fault.search('RF_FAILED')) + + +class DNSSocketTest(unittest.TestCase): + def test_bound_loopback_query(self): + # Observe the real kernel socket option before connect, without mocking + # setsockopt/connect/send/recv. All traffic stays on loopback. + bound = [] + + class ObservedSocket(socket.socket): + def connect(self, peer): + bound.append(self.getsockopt(socket.SOL_SOCKET, socket.SO_BINDTODEVICE, 32)) + return super().connect(peer) + + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as server: + server.bind(('127.0.0.1', 0)) + server.settimeout(2) + errors = [] + + def respond(): + try: + query, peer = server.recvfrom(4096) + self.assertEqual(query[12:], QUESTION) + ident = struct.unpack_from('!H', query)[0] + server.sendto(reply(ident=ident), peer) + except Exception as error: + errors.append(error) + + worker = threading.Thread(target=respond) + worker.start() + try: + with mock.patch.object(e2e.socket, 'socket', ObservedSocket): + self.assertEqual(e2e.dns_query('lo', '127.0.0.1', server.getsockname()[1]), + '203.0.113.5') + finally: + worker.join(3) + self.assertFalse(worker.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(bound, [b'lo\0']) + + def test_bound_query_times_out_without_response(self): + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as silent_server: + silent_server.bind(('127.0.0.1', 0)) + with self.assertRaises(TimeoutError): + e2e.dns_query('lo', '127.0.0.1', silent_server.getsockname()[1], timeout=0.03) + silent_server.settimeout(1) + query, _ = silent_server.recvfrom(4096) + self.assertEqual(query[12:], QUESTION) + + def test_invalid_interface_fails_without_fallback(self): + with self.assertRaises(OSError): + e2e.dns_query('no-such-neo-if', '127.0.0.1', timeout=0.03) + + +if __name__ == '__main__': + unittest.main() From dd172c7ec7e6c1f29a4546162526773b22d13973 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:25:54 -0700 Subject: [PATCH 0126/2680] selftests: mt7932: reject late faults in the final kernel log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — moved from the tooling bucket into the radio topic with the driver it tests; 900-config-tooling skips it] --- tools/testing/selftests/drivers/net/mt7932_e2e.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/drivers/net/mt7932_e2e.py b/tools/testing/selftests/drivers/net/mt7932_e2e.py index ec90e0dd32a1e..a29531e9da504 100644 --- a/tools/testing/selftests/drivers/net/mt7932_e2e.py +++ b/tools/testing/selftests/drivers/net/mt7932_e2e.py @@ -237,8 +237,13 @@ def main(): result['result'] = 'WIFI_NETWORK_E2E_FAILED' result['error'] = str(error) finally: - log = run(['dmesg']).stdout + final_log = run(['dmesg']) + log = final_log.stdout result['kernel_faults'] = len(fault.findall(log)) + if result['result'] in ('WIFI_SCAN_OBSERVED', 'WIFI_PHYSICAL_NETWORK_E2E_PASS') and ( + final_log.returncode != 0 or result['kernel_faults']): + result['result'] = 'WIFI_NETWORK_E2E_FAILED' + result['error'] = 'Final kernel log unavailable or contains a driver fault' (out / 'result.json').write_text(json.dumps(result, indent=2) + '\n') print(json.dumps(result, indent=2)) print('ARTIFACT=' + str(out / 'result.json')) From e5f0b3c9832aa74ee526046d08a97dc120c1f62d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:41:45 -0700 Subject: [PATCH 0127/2680] Bluetooth: mt7932: retire IRQ and MSI allocations on removal Clearing bus mastering and disabling the interrupt is insufficient when PCI devres releases MSI device data at unbind. Free the synchronized threaded IRQ before releasing its vectors, after HCI and queued TX callbacks have been retired. Keep the device-visible arena and its owners retained because hardware DMA quiescence remains unqualified. --- drivers/bluetooth/mt7932_bt_pcie.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 7ab5d30dd458e..84d73e14b2f4e 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -1181,6 +1181,19 @@ static void bt7932_remove(struct pci_dev *pdev) /* Flush again after the last HCI callback has been retired. */ cancel_work_sync(&bt->tx_work); skb_queue_purge(&bt->tx_queue); + /* + * shutdown cleared bus mastering and synchronized the disabled IRQ. + * Retaining DMA ownership must not leave an IRQ action attached to MSI + * data which the PCI core releases when this driver is unbound. + */ + if (bt->irq_requested) { + free_irq(bt->irq, bt); + bt->irq_requested = false; + } + if (bt->vectors) { + pci_free_irq_vectors(pdev); + bt->vectors = false; + } dev_err(&pdev->dev, "removal is unqualified; DMA, device and module retained until external reset\n"); } From 16a189ed943c0cae2f998f6bc6434dbd825efeee Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:42:37 -0700 Subject: [PATCH 0128/2680] soc: apple: scope PIODMA radio admission header to its provider The admission API is specific to the Apple SoC bootstrap provider rather than generic PCI. Move its public declaration under soc/apple and update all in-tree clients without changing runtime behavior. --- drivers/bluetooth/mt7932_bt_pcie.c | 2 +- drivers/net/wireless/mediatek/mt7932/mt7932.c | 2 +- drivers/pci/controller/pcie-apple-piodma-diag.c | 2 +- drivers/pci/controller/pcie-apple.c | 2 +- include/linux/{ => soc/apple}/pci-apple-piodma.h | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) rename include/linux/{ => soc/apple}/pci-apple-piodma.h (76%) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 84d73e14b2f4e..a80d91891d5fe 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 75060c4be245d..de910c528b400 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -3,7 +3,7 @@ * Clean-room contract: NEO_MT7932_DOWNLOAD_AND_NIC_CAP_CONTRACT.md. * All addresses here are BAR0 offsets, never chip/CPU physical addresses. */ -#include +#include #include "mt7932.h" diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index 7ced530ba7c45..eb6310f8de1b0 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -28,7 +28,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index f5315ca205d40..ae1d4f203e39a 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -33,7 +33,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/include/linux/pci-apple-piodma.h b/include/linux/soc/apple/pci-apple-piodma.h similarity index 76% rename from include/linux/pci-apple-piodma.h rename to include/linux/soc/apple/pci-apple-piodma.h index 2026d1f51602e..5cb948a60943e 100644 --- a/include/linux/pci-apple-piodma.h +++ b/include/linux/soc/apple/pci-apple-piodma.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ -#ifndef _LINUX_PCI_APPLE_PIODMA_H -#define _LINUX_PCI_APPLE_PIODMA_H +#ifndef _LINUX_SOC_APPLE_PCI_PIODMA_H +#define _LINUX_SOC_APPLE_PCI_PIODMA_H #include #include @@ -17,4 +17,4 @@ static inline int apple_piodma_radio_check(struct pci_dev *pdev) } #endif -#endif /* _LINUX_PCI_APPLE_PIODMA_H */ +#endif /* _LINUX_SOC_APPLE_PCI_PIODMA_H */ From ea2c6e0b8205e423cce862f8e0c489c535e8d260 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:44:01 -0700 Subject: [PATCH 0129/2680] wifi: mt7932: share D7 band reply count during startup A queued 5 GHz D7 request has three logical completions, while startup draining previously waited for four and latched a terminal timeout. Use the runtime band-dependent contract in both paths before waiting for replies. --- drivers/net/wireless/mediatek/mt7932/calibration.h | 8 ++++++++ .../net/wireless/mediatek/mt7932/calibration_transport.c | 3 ++- drivers/net/wireless/mediatek/mt7932/frontend.c | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/calibration.h b/drivers/net/wireless/mediatek/mt7932/calibration.h index f008e34146d64..4bee4ada54df2 100644 --- a/drivers/net/wireless/mediatek/mt7932/calibration.h +++ b/drivers/net/wireless/mediatek/mt7932/calibration.h @@ -65,6 +65,14 @@ static inline int mt7932_cal_null(struct mt7932_cal_completion *state, return ret; } +/* D7 band 1 (5 GHz) requests three logical procedures; 2 GHz needs four. + * Both startup draining and runtime servicing use the same reply contract. + */ +static inline unsigned int mt7932_cal_request_replies(const u8 request[16]) +{ + return get_unaligned_le32(request + 8) == 1 ? 3 : 4; +} + struct mt7932_cal_segment { const u8 *data; size_t length; diff --git a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c index 312315eb5b405..86cf12908de71 100644 --- a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c +++ b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c @@ -131,7 +131,8 @@ int mt_calibration_gate(struct mt7932 *m) m->smart_version, m->module_byte, pieces); if (ret < 0) goto out; - ret = mt_cal_procedure(m, pieces, ret, 4, m->preload_version); + ret = mt_cal_procedure(m, pieces, ret, + mt7932_cal_request_replies(request), m->preload_version); if (ret) goto out; dev_info(&m->pdev->dev, "CAL_REQUEST_SERVICED: channel=%u\n", get_unaligned_le32(request + 12)); diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 781fb4f60dbca..de7bf5f3fd407 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -503,7 +503,7 @@ static void mt_cal_work(struct work_struct *work) if (ret < 0) break; ret = mt_cal_procedure(m, pieces, ret, - get_unaligned_le32(request + 8) == 1 ? 3 : 4, + mt7932_cal_request_replies(request), m->preload_version); if (ret) break; From 531116c9f67f41341243db3e247d0c6b8fefa4cc Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:46:20 -0700 Subject: [PATCH 0130/2680] wifi: mt7932: require Apple bootstrap and cfg80211 dependencies The driver cannot probe without the retained J700 supplier. Express that dependency and ARCH_APPLE in Kconfig, and depend on cfg80211 instead of selecting it across its own dependency constraints. This also prevents a built-in radio driver calling modular cfg80211. --- drivers/net/wireless/mediatek/mt7932/Kconfig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index 239de549c8947..f5a5aa6af5690 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -1,13 +1,13 @@ # SPDX-License-Identifier: GPL-2.0-only config MT7932_FULLMAC tristate "Experimental J700 MT7932 fullmac wireless support" - depends on PCI && OF && ARM64 && IOMMU_DMA + depends on PCI && OF && ARM64 && ARCH_APPLE && IOMMU_DMA + depends on PCIE_APPLE_PIODMA_DIAG && CFG80211 select FW_LOADER - select CFG80211 help Native support for the built-in J700 Wi-Fi endpoint: managed WPA2-CCMP station operation on 2.4 GHz at 20 MHz and channels 36..48 - with a software bandwidth limit of 80 MHz. + at 20/40/80 MHz, with a software bandwidth limit of 80 MHz. Requires original firmware, own calibration and original-derived country policy packages in addition to the cfg80211 regulatory database. Requires a trained retained PCI link and a translated SID1 DMA domain. From 38ba1db0f9bc6f26eb278dc15ecb9c6d166b7701 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:46:20 -0700 Subject: [PATCH 0131/2680] wifi: mt7932: declare runtime country policy firmware pattern Fixed metadata already names all seven fixed Wi-Fi inputs. Add a country-policy glob so compatible initramfs builders (including mkinitcpio) retain installed policy packages for later regulatory hints. It cannot create absent per-unit packages; builders without glob support must explicitly include the requested country policies. --- drivers/net/wireless/mediatek/mt7932/mt7932.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index de910c528b400..084b006a812a9 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -1333,3 +1333,7 @@ MODULE_FIRMWARE("mediatek/mt7932/wcal.bin"); MODULE_FIRMWARE("mediatek/mt7932/oca2.bin"); MODULE_FIRMWARE("mediatek/mt7932/config-original.bin"); MODULE_FIRMWARE("mediatek/mt7932/policy/world-XZ.bin"); +/* Country selection is runtime-dependent. Initramfs builders supporting + * firmware globs must retain every installed policy, including later hints. + */ +MODULE_FIRMWARE("mediatek/mt7932/policy/[A-Z][A-Z].bin"); From ea3ed7ef0960c1c65e83b244bd0ae75ac65f9d69 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:46:57 -0700 Subject: [PATCH 0132/2680] PCI: apple: acquire lifecycle vetoes when bootstrap primes An unpublished coherent arena does not expose pointers to hardware. Leave sleep and kexec available while the supplier waits for its host, and acquire the existing nonblocking lifecycle interlocks in prime before consuming its hardware attempt. Release both guards on pre-publication failures; retain them once initialization makes ownership irreversible. Propagate host deferral and drop its provisional module hold before publication. --- .../pci/controller/pcie-apple-piodma-diag.c | 103 ++++++++++-------- .../pci/controller/pcie-apple-piodma-diag.h | 6 + drivers/pci/controller/pcie-apple.c | 6 + 3 files changed, 69 insertions(+), 46 deletions(-) diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index eb6310f8de1b0..26a1359097b2c 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -103,7 +103,7 @@ struct apple_piodma_diag { struct kexec_blocker kexec_blocker; }; -/* A successful supplier probe owns its arena and guards until full reset. */ +/* A successful supplier probe owns its unpublished arena until full reset. */ static struct apple_piodma_diag *active_diag; static bool apple_piodma_trylock_sleep(unsigned int *flags) @@ -113,10 +113,10 @@ static bool apple_piodma_trylock_sleep(unsigned int *flags) return true; /* - * Probe holds device_lock(), while system sleep holds the transition + * Host probe holds device_lock(), while system sleep holds the transition * mutex before waiting for probes and taking device locks. Never wait * for that mutex here: if sleep won, defer without publishing any DMA. - * If probe won, publish the veto before releasing the transition mutex. + * If prime won, publish the veto before releasing the transition mutex. */ *flags = current->flags; current->flags |= PF_NOFREEZE; @@ -149,7 +149,8 @@ static int apple_piodma_pm_prepare(struct device *dev) { struct apple_piodma_diag *diag = dev_get_drvdata(dev); - if (!diag || !diag->arena) + /* Acquire the prime's retained arena publication before permitting sleep. */ + if (!diag || !smp_load_acquire(&diag->retained)) return 0; /* * A /dev/snapshot file may have sent its PREPARE notifier before this @@ -593,29 +594,63 @@ EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_get); int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root) { struct apple_piodma_diag *diag = dev_get_drvdata(supplier); + unsigned int sleep_flags; int ret; - /* One attempt for this provider's entire lifetime, including failures. */ - if (atomic_cmpxchg(&diag->attempted, 0, 1)) - return -EALREADY; - ret = apple_piodma_diag_root(diag, root); - if (ret) - return ret; - ret = apple_piodma_diag_idle(diag); + /* Contention must not consume the one permitted hardware attempt. */ + if (!apple_piodma_trylock_sleep(&sleep_flags)) + return -EPROBE_DEFER; + ret = kexec_block(&diag->kexec_blocker); + if (ret) { + unlock_system_sleep(sleep_flags); + return ret == -EBUSY ? -EPROBE_DEFER : ret; + } + /* One hardware attempt, including permanent preflight failures. */ + if (atomic_cmpxchg(&diag->attempted, 0, 1)) { + ret = -EALREADY; + goto err_kexec; + } + diag->pm_notifier.notifier_call = apple_piodma_pm_notify; + ret = register_pm_notifier(&diag->pm_notifier); if (ret) - return ret; - ret = apple_piodma_diag_initialize(diag); + goto err_kexec; + ret = apple_piodma_diag_root(diag, root); + if (!ret) + ret = apple_piodma_diag_idle(diag); + if (!ret) + ret = apple_piodma_diag_initialize(diag); if (!ret) ret = apple_piodma_diag_submit(diag, 0); - if (ret) - dev_err(supplier, "bootstrap failure=%d; retained ownership; no retry\n", ret); - else + if (ret) { + dev_err(supplier, "bootstrap failure=%d; %s; no retry\n", ret, + diag->retained ? "retained ownership" : "no hardware publication"); + if (!diag->retained) { + unregister_pm_notifier(&diag->pm_notifier); + goto err_kexec; + } + } else { dev_info(supplier, "BOOTSTRAP_VALIDATED word=%#x; one retained slot\n", diag->slots[0].result); + } + unlock_system_sleep(sleep_flags); + return ret; + +err_kexec: + kexec_unblock(&diag->kexec_blocker); + unlock_system_sleep(sleep_flags); return ret; } EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_prime); +bool apple_piodma_bootstrap_retained(struct device *supplier) +{ + struct apple_piodma_diag *diag = dev_get_drvdata(supplier); + + /* Acquire the prime's irreversible publication and guard ownership. */ + return smp_load_acquire(&diag->retained); +} +EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_retained); + void apple_piodma_bootstrap_admitted(struct device *supplier) { struct apple_piodma_diag *diag = dev_get_drvdata(supplier); @@ -657,7 +692,6 @@ static int apple_piodma_diag_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct apple_piodma_diag *diag; - unsigned int sleep_flags; int i, ret; if (!apple_piodma_bootstrap_enabled() || PAGE_SIZE != SZ_16K) @@ -675,21 +709,6 @@ static int apple_piodma_diag_probe(struct platform_device *pdev) ret = apple_piodma_diag_iommu(diag); if (ret) goto err_free; - /* Publish guards and arena atomically with respect to system sleep. */ - if (!apple_piodma_trylock_sleep(&sleep_flags)) { - ret = -EPROBE_DEFER; - goto err_free; - } - diag->pm_notifier.notifier_call = apple_piodma_pm_notify; - ret = register_pm_notifier(&diag->pm_notifier); - if (ret) - goto err_sleep; - ret = kexec_block(&diag->kexec_blocker); - if (ret) { - if (ret == -EBUSY) - ret = -EPROBE_DEFER; - goto err_notifier; - } pm_runtime_enable(dev); ret = pm_runtime_resume_and_get(dev); if (ret < 0) @@ -704,31 +723,20 @@ static int apple_piodma_diag_probe(struct platform_device *pdev) dev_name(dev), diag); if (ret) goto err_arena; - /* The notifier and retained state also keep their device alive. */ + /* The supplier and unpublished arena keep their device alive. */ get_device(dev); platform_set_drvdata(pdev, diag); - /* Publish the arena, device reference and both guards together. */ + /* Publish the CPU-owned arena and permanent device reference together. */ smp_store_release(&active_diag, diag); - unlock_system_sleep(sleep_flags); dev_info(dev, "bootstrap supplier ready; no command submitted during probe\n"); return 0; err_arena: - /* No path after initialization may release ownership. */ - if (WARN_ON_ONCE(diag->retained)) { - unlock_system_sleep(sleep_flags); - return 0; - } if (diag->arena) dma_free_coherent(dev, PIODMA_ARENA_SIZE, diag->arena, diag->dma); pm_runtime_put_sync(dev); err_pm: pm_runtime_disable(dev); - kexec_unblock(&diag->kexec_blocker); -err_notifier: - unregister_pm_notifier(&diag->pm_notifier); -err_sleep: - unlock_system_sleep(sleep_flags); err_free: kfree(diag); return dev_err_probe(dev, ret, "PIODMA diagnostic preflight failed\n"); @@ -736,7 +744,10 @@ static int apple_piodma_diag_probe(struct platform_device *pdev) static void apple_piodma_diag_shutdown(struct platform_device *pdev) { - dev_warn(&pdev->dev, "diagnostic arena retained; require full hardware reset\n"); + struct apple_piodma_diag *diag = platform_get_drvdata(pdev); + + if (READ_ONCE(diag->retained)) + dev_warn(&pdev->dev, "diagnostic arena retained; require full hardware reset\n"); } static const struct of_device_id apple_piodma_diag_match[] = { diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.h b/drivers/pci/controller/pcie-apple-piodma-diag.h index adc0349eeaca1..907981d97b1d3 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.h +++ b/drivers/pci/controller/pcie-apple-piodma-diag.h @@ -14,6 +14,7 @@ bool apple_piodma_bootstrap_enabled(void); int apple_piodma_bootstrap_get(struct device *host, struct device **supplier); int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root); void apple_piodma_bootstrap_admitted(struct device *supplier); +bool apple_piodma_bootstrap_retained(struct device *supplier); #else static inline bool apple_piodma_bootstrap_enabled(void) { @@ -33,5 +34,10 @@ static inline int apple_piodma_bootstrap_prime(struct device *supplier, struct p static inline void apple_piodma_bootstrap_admitted(struct device *supplier) { } + +static inline bool apple_piodma_bootstrap_retained(struct device *supplier) +{ + return false; +} #endif #endif diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index ae1d4f203e39a..e72905b630654 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2598,6 +2598,12 @@ static int apple_pcie_probe(struct platform_device *pdev) return -ENODEV; } ret = apple_pcie_neo_enumerate(bridge); + if (ret && !apple_piodma_bootstrap_retained(pcie->piodma_supplier)) { + /* No DMA pointer escaped: unwind and allow dependency retry. */ + pci_host_common_remove(pdev); + module_put(THIS_MODULE); + return dev_err_probe(dev, ret, "Neo bootstrap preflight failed\n"); + } if (ret) dev_err(dev, "enumeration experiment failed=%d; host/supplier retained, no retry\n", ret); From 494b2674b9df51847daf00561fb70df794666ce5 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:47:01 -0700 Subject: [PATCH 0133/2680] wifi: mt7932: retry missing policy on userspace admission requests A policy lookup that fails before any SET leaves RF blocked but recoverable. Record only a completed ENOENT attempt, then let interface up, scan, or connect consume that state and reschedule the current country. In-flight attempts and terminal calibration or partial policy failures remain excluded; normal admission can return EAGAIN while the new lookup runs. --- .../net/wireless/mediatek/mt7932/frontend.c | 2 ++ drivers/net/wireless/mediatek/mt7932/link.c | 2 ++ drivers/net/wireless/mediatek/mt7932/mt7932.h | 3 ++- .../net/wireless/mediatek/mt7932/startup.c | 23 +++++++++++++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index de7bf5f3fd407..433e789288403 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -30,6 +30,7 @@ static void mt_scan_finish_work(struct work_struct *work) /* Retire the old timer before allowing another request to take ownership. */ cancel_delayed_work_sync(&m->scan_timeout_work); + mt_retry_missing_policy(m); mutex_lock(&m->command_mutex); spin_lock_irqsave(&m->response_lock, flags); request = m->scan_request; @@ -327,6 +328,7 @@ int mt_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) if (request->wdev != &m->wdev) return -ENODEV; + mt_retry_missing_policy(m); dev_info(&m->pdev->dev, "SCAN_REQUEST: channels=%u ssids=%d ies=%zu flags=%08x duration=%u/%u\n", request->n_channels, request->n_ssids, request->ie_len, request->flags, request->duration, request->duration_mandatory); diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 53b4e931d605d..fe96e04e9aef8 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -582,6 +582,8 @@ int mt_connect(struct wiphy *wiphy, struct net_device *netdev, bool open = mt_open_request(params); int ret = 0; + if (netdev == m->netdev) + mt_retry_missing_policy(m); if (!mt_rf_allowed(m) || !m->bss_active || netdev != m->netdev) return -EAGAIN; dev_info(&m->pdev->dev, "CONNECT_REQUEST: WPA=%u pairwise-count=%d group=%08x AKM-count=%d PMK-present=%u MFP=%u IE-bytes=%zu\n", diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index b7af4a04946b3..c1f0c5fca2551 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -101,7 +101,7 @@ struct mt7932 { struct ieee80211_rate rates[12]; struct work_struct startup_work; struct mutex command_mutex; - bool startup_started, rf_ready, reg_pending, policy_failed; + bool startup_started, rf_ready, reg_pending, policy_failed, reg_retryable; bool interface_registered, interface_up; u32 reg_generation, reg_attempted; struct mt7932_reg_snapshot reg_desired; @@ -166,6 +166,7 @@ struct mt7932 { }; bool mt_rf_allowed(struct mt7932 *m); +void mt_retry_missing_policy(struct mt7932 *m); int mt_request_input(struct mt7932 *m, const struct firmware **fw, const char *name); /* response_lock held; closes RF admission and reports terminal host state. */ void mt_rf_fail_locked(struct mt7932 *m, int error); diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 1e4ecc6f065ac..8f8a2a7c6da84 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -223,6 +223,24 @@ static int mt_startup_once(struct mt7932 *m) return ret; } +/* Retry only a completed file lookup that failed before any policy SET. + * Consume the flag under the same lock as the worker publication, so repeated + * userspace requests cannot continually invalidate an in-flight attempt. + */ +void mt_retry_missing_policy(struct mt7932 *m) +{ + unsigned long flags; + + spin_lock_irqsave(&m->response_lock, flags); + if (m->reg_retryable && m->reg_pending && m->interface_registered && + !m->stopping && !m->policy_failed && !m->link_failed && !m->cal_state.error) { + m->reg_retryable = false; + m->reg_generation++; + schedule_work(&m->startup_work); + } + spin_unlock_irqrestore(&m->response_lock, flags); +} + static void mt_startup_work(struct work_struct *work) { struct mt7932 *m = container_of(work, struct mt7932, startup_work); @@ -244,6 +262,7 @@ static void mt_startup_work(struct work_struct *work) return; } m->reg_attempted = generation; + m->reg_retryable = false; spin_unlock_irqrestore(&m->response_lock, flags); /* No command/wiphy lock while joining lifecycle workers. RX, D7 @@ -299,6 +318,10 @@ static void mt_startup_work(struct work_struct *work) ret = m->cal_state.error ?: -EIO; if (ret && submitted) WRITE_ONCE(m->policy_failed, true); + if (generation == m->reg_generation) + m->reg_retryable = ret == -ENOENT && !submitted && + !m->stopping && !m->policy_failed && !m->link_failed && + !m->cal_state.error; if (!ret && generation == m->reg_generation) WRITE_ONCE(m->reg_pending, false); if (ret) From a65e61bd21977259dad7d2470159cc123324aa63 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:47:01 -0700 Subject: [PATCH 0134/2680] wifi: mt7932: refuse function reset while Bluetooth is live PCI function-scoped FLR does not establish that the radio functions share no firmware or fabric state. Hold the Bluetooth sibling device lock across Wi-Fi reset and reject bound, bus-mastering, or concurrently probed peers. A refusal follows the existing failure path retaining Wi-Fi DMA ownership rather than inventing a cross-function quiescence guarantee. --- drivers/net/wireless/mediatek/mt7932/reset.c | 43 +++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/mediatek/mt7932/reset.c b/drivers/net/wireless/mediatek/mt7932/reset.c index 0f384dbdaed83..3c14e17bb4b21 100644 --- a/drivers/net/wireless/mediatek/mt7932/reset.c +++ b/drivers/net/wireless/mediatek/mt7932/reset.c @@ -28,7 +28,7 @@ static int mt_reset_identity(struct pci_dev *pdev) return 0; } -int mt_function_reset(struct mt7932 *m) +static int mt_function_reset_locked(struct mt7932 *m) { struct pci_dev *pdev = m->pdev; u32 fabric, aer_mask = 0; @@ -112,3 +112,44 @@ int mt_function_reset(struct mt7932 *m) pci_cfg_access_unlock(pdev); return ret; } + +int mt_function_reset(struct mt7932 *m) +{ + struct pci_dev *peer; + u16 command; + int ret; + + device_lock_assert(&m->pdev->dev); + peer = pci_get_slot(m->pdev->bus, PCI_DEVFN(PCI_SLOT(m->pdev->devfn), 1)); + if (!peer) + return mt_function_reset_locked(m); + /* The radio functions may share state beyond PCI's advertised FLR scope. + * No qualified contract permits resetting Wi-Fi while Bluetooth is live. + * Trylock avoids inversion with a concurrent sibling probe/remove; hold + * it across FLR to prevent Bluetooth starting after the admission check. + */ + if (!device_trylock(&peer->dev)) { + ret = -EBUSY; + goto put; + } + if (peer->driver) { + ret = -EBUSY; + goto unlock; + } + if (pci_read_config_word(peer, PCI_COMMAND, &command) || command == U16_MAX) { + ret = -EIO; + goto unlock; + } + if (command & PCI_COMMAND_MASTER) { + ret = -EBUSY; + goto unlock; + } + ret = mt_function_reset_locked(m); +unlock: + device_unlock(&peer->dev); +put: + pci_dev_put(peer); + if (ret) + dev_warn(&m->pdev->dev, "WIFI_FLR_REFUSED_OR_FAILED: %d; Bluetooth sibling must be idle; DMA ownership retained\n", ret); + return ret; +} From 6a954d751ad59958f2d8ae9c6c79f3cd3c83f0b5 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:47:08 -0700 Subject: [PATCH 0135/2680] PCI: test sysfs removal policy without invoking unregistered devices Build the KUnit fixture separately and exercise the same host-policy lookup used by sysfs. Allocate the flexible-array host bridge independently, verify root/descendant and independent-host decisions, and avoid logging or kernfs callbacks on synthetic devices. Rate-limit repeated userspace-removal refusals. --- drivers/pci/Kconfig | 7 +- drivers/pci/Makefile | 1 + drivers/pci/pci-sysfs-test.c | 122 +++++++++++++++++------------------ drivers/pci/pci-sysfs.c | 18 +++--- drivers/pci/pci.h | 2 + 5 files changed, 73 insertions(+), 77 deletions(-) diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig index 0644a5ad860a4..048bc617b42ab 100644 --- a/drivers/pci/Kconfig +++ b/drivers/pci/Kconfig @@ -34,10 +34,9 @@ config PCI_SYSFS_KUNIT_TEST depends on SYSFS && KUNIT=y default KUNIT_ALL_TESTS help - Test rejected removal writes for root ports and children under a host - that prohibits userspace removal, plus malformed and no-op writes. - The tests use an unregistered synthetic hierarchy and never remove - a real PCI device or invoke kernfs attribute removal. + Test userspace-removal policy lookup for root ports and descendants + under independent host bridges. The synthetic objects are separately + allocated and never registered or passed to a sysfs callback. config PCI_DOMAINS bool diff --git a/drivers/pci/Makefile b/drivers/pci/Makefile index 41ebc3b9a5182..9c8fe31506d0c 100644 --- a/drivers/pci/Makefile +++ b/drivers/pci/Makefile @@ -14,6 +14,7 @@ obj-$(CONFIG_PCI) += pwrctrl/ ifdef CONFIG_PCI obj-$(CONFIG_PROC_FS) += proc.o obj-$(CONFIG_SYSFS) += pci-sysfs.o slot.o +obj-$(CONFIG_PCI_SYSFS_KUNIT_TEST) += pci-sysfs-test.o obj-$(CONFIG_ACPI) += pci-acpi.o obj-$(CONFIG_GENERIC_PCI_IOMAP) += iomap.o endif diff --git a/drivers/pci/pci-sysfs-test.c b/drivers/pci/pci-sysfs-test.c index 3875a0f760d23..c166c8659f19d 100644 --- a/drivers/pci/pci-sysfs-test.c +++ b/drivers/pci/pci-sysfs-test.c @@ -1,96 +1,90 @@ // SPDX-License-Identifier: GPL-2.0-only -/* Included by pci-sysfs.c to exercise its actual remove_store callback. */ +/* Policy lookup tests; no sysfs callbacks or hardware operations. */ #include +#include + +#include "pci.h" struct pci_sysfs_test_hierarchy { - struct pci_host_bridge bridge; - struct pci_bus root_bus; - struct pci_bus child_bus; - struct pci_dev root_port; - struct pci_dev endpoint; + struct pci_host_bridge *bridge; + struct pci_bus *root_bus; + struct pci_bus *child_bus; + struct pci_bus *grandchild_bus; + struct pci_dev *root_port; + struct pci_dev *endpoint; + struct pci_dev *grandchild; }; static int pci_sysfs_test_init(struct kunit *test) { - struct pci_sysfs_test_hierarchy *hierarchy; + struct pci_sysfs_test_hierarchy *h; - hierarchy = kunit_kzalloc(test, sizeof(*hierarchy), GFP_KERNEL); - if (!hierarchy) + h = kunit_kzalloc(test, sizeof(*h), GFP_KERNEL); + if (!h) + return -ENOMEM; + h->bridge = kunit_kzalloc(test, sizeof(*h->bridge), GFP_KERNEL); + h->root_bus = kunit_kzalloc(test, sizeof(*h->root_bus), GFP_KERNEL); + h->child_bus = kunit_kzalloc(test, sizeof(*h->child_bus), GFP_KERNEL); + h->grandchild_bus = kunit_kzalloc(test, sizeof(*h->grandchild_bus), GFP_KERNEL); + h->root_port = kunit_kzalloc(test, sizeof(*h->root_port), GFP_KERNEL); + h->endpoint = kunit_kzalloc(test, sizeof(*h->endpoint), GFP_KERNEL); + h->grandchild = kunit_kzalloc(test, sizeof(*h->grandchild), GFP_KERNEL); + if (!h->bridge || !h->root_bus || !h->child_bus || !h->grandchild_bus || + !h->root_port || !h->endpoint || !h->grandchild) return -ENOMEM; - /* - * Only topology and diagnostic names are needed. These objects are never - * registered, referenced by the PCI core or passed to a removal routine. - * Every tested callback must return before device_remove_file_self(). - */ - hierarchy->root_bus.bridge = &hierarchy->bridge.dev; - hierarchy->child_bus.parent = &hierarchy->root_bus; - hierarchy->root_port.bus = &hierarchy->root_bus; - hierarchy->root_port.dev.init_name = "kunit-root-port"; - hierarchy->endpoint.bus = &hierarchy->child_bus; - hierarchy->endpoint.dev.init_name = "kunit-endpoint"; - test->priv = hierarchy; + /* Only the real host lookup's immutable topology is needed. */ + h->root_bus->bridge = &h->bridge->dev; + h->child_bus->parent = h->root_bus; + h->grandchild_bus->parent = h->child_bus; + h->root_port->bus = h->root_bus; + h->endpoint->bus = h->child_bus; + h->grandchild->bus = h->grandchild_bus; + test->priv = h; return 0; } -static void pci_sysfs_remove_blocked_root_test(struct kunit *test) -{ - struct pci_sysfs_test_hierarchy *hierarchy = test->priv; - - hierarchy->bridge.no_user_remove = true; - KUNIT_EXPECT_EQ(test, - remove_store(&hierarchy->root_port.dev, &dev_attr_remove, "1\n", 2), - (ssize_t)-EBUSY); -} - -static void pci_sysfs_remove_blocked_child_test(struct kunit *test) +static void pci_sysfs_remove_default_test(struct kunit *test) { - struct pci_sysfs_test_hierarchy *hierarchy = test->priv; + struct pci_sysfs_test_hierarchy *h = test->priv; - hierarchy->bridge.no_user_remove = true; - KUNIT_EXPECT_EQ(test, - remove_store(&hierarchy->endpoint.dev, &dev_attr_remove, "1\n", 2), - (ssize_t)-EBUSY); + KUNIT_EXPECT_TRUE(test, pci_sysfs_user_remove_allowed(h->root_port)); + KUNIT_EXPECT_TRUE(test, pci_sysfs_user_remove_allowed(h->endpoint)); + KUNIT_EXPECT_TRUE(test, pci_sysfs_user_remove_allowed(h->grandchild)); } -static void pci_sysfs_remove_malformed_test(struct kunit *test) +static void pci_sysfs_remove_retained_test(struct kunit *test) { - struct pci_sysfs_test_hierarchy *hierarchy = test->priv; + struct pci_sysfs_test_hierarchy *h = test->priv; - KUNIT_EXPECT_EQ(test, - remove_store(&hierarchy->endpoint.dev, &dev_attr_remove, "bad\n", 4), - (ssize_t)-EINVAL); + h->bridge->no_user_remove = true; + KUNIT_EXPECT_FALSE(test, pci_sysfs_user_remove_allowed(h->root_port)); + KUNIT_EXPECT_FALSE(test, pci_sysfs_user_remove_allowed(h->endpoint)); + KUNIT_EXPECT_FALSE(test, pci_sysfs_user_remove_allowed(h->grandchild)); } -static void pci_sysfs_remove_zero_blocked_test(struct kunit *test) +static void pci_sysfs_remove_independent_host_test(struct kunit *test) { - struct pci_sysfs_test_hierarchy *hierarchy = test->priv; - - hierarchy->bridge.no_user_remove = true; - KUNIT_EXPECT_EQ(test, - remove_store(&hierarchy->endpoint.dev, &dev_attr_remove, "0\n", 2), - (ssize_t)2); -} - -static void pci_sysfs_remove_zero_default_test(struct kunit *test) -{ - struct pci_sysfs_test_hierarchy *hierarchy = test->priv; - - KUNIT_EXPECT_EQ(test, - remove_store(&hierarchy->root_port.dev, &dev_attr_remove, "0\n", 2), - (ssize_t)2); + struct pci_sysfs_test_hierarchy *h = test->priv; + struct pci_host_bridge *other; + + other = kunit_kzalloc(test, sizeof(*other), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, other); + h->bridge->no_user_remove = true; + KUNIT_ASSERT_FALSE(test, pci_sysfs_user_remove_allowed(h->grandchild)); + /* A descendant of an unrestricted host must not inherit another veto. */ + h->root_bus->bridge = &other->dev; + KUNIT_EXPECT_TRUE(test, pci_sysfs_user_remove_allowed(h->grandchild)); } static struct kunit_case pci_sysfs_test_cases[] = { - KUNIT_CASE(pci_sysfs_remove_blocked_root_test), - KUNIT_CASE(pci_sysfs_remove_blocked_child_test), - KUNIT_CASE(pci_sysfs_remove_malformed_test), - KUNIT_CASE(pci_sysfs_remove_zero_blocked_test), - KUNIT_CASE(pci_sysfs_remove_zero_default_test), + KUNIT_CASE(pci_sysfs_remove_default_test), + KUNIT_CASE(pci_sysfs_remove_retained_test), + KUNIT_CASE(pci_sysfs_remove_independent_host_test), {} }; static struct kunit_suite pci_sysfs_test_suite = { - .name = "pci-sysfs-remove", + .name = "pci-sysfs-policy", .init = pci_sysfs_test_init, .test_cases = pci_sysfs_test_cases, }; diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c index cf33429556623..727ab49a5db0c 100644 --- a/drivers/pci/pci-sysfs.c +++ b/drivers/pci/pci-sysfs.c @@ -511,21 +511,25 @@ static ssize_t dev_rescan_store(struct device *dev, static struct device_attribute dev_attr_dev_rescan = __ATTR(rescan, 0200, NULL, dev_rescan_store); +bool pci_sysfs_user_remove_allowed(struct pci_dev *pdev) +{ + /* Bus/bridge references keep this immutable host property alive. */ + return !pci_find_host_bridge(pdev->bus)->no_user_remove; +} + static ssize_t remove_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct pci_dev *pdev = to_pci_dev(dev); - struct pci_host_bridge *bridge; unsigned long val; if (kstrtoul(buf, 0, &val) < 0) return -EINVAL; if (val) { - /* Bus/bridge references keep this immutable host property alive. */ - bridge = pci_find_host_bridge(pdev->bus); - if (bridge->no_user_remove) { - pci_warn(pdev, "userspace removal is disabled by the host bridge\n"); + if (!pci_sysfs_user_remove_allowed(pdev)) { + dev_warn_ratelimited(&pdev->dev, + "userspace removal is disabled by the host bridge\n"); return -EBUSY; } if (device_remove_file_self(dev, attr)) @@ -1933,10 +1937,6 @@ const struct attribute_group *pci_dev_groups[] = { NULL, }; -#ifdef CONFIG_PCI_SYSFS_KUNIT_TEST -#include "pci-sysfs-test.c" -#endif - static const struct attribute_group pci_dev_hp_attr_group = { .attrs = pci_dev_hp_attrs, .is_visible = pci_dev_hp_attrs_are_visible, diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 62c1b324a9bd0..f3230633caa0a 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -1458,4 +1458,6 @@ static inline int pci_msix_write_tph_tag(struct pci_dev *pdev, unsigned int inde (PCI_CONF1_ADDRESS(bus, dev, func, reg) | \ PCI_CONF1_EXT_REG(reg)) +bool pci_sysfs_user_remove_allowed(struct pci_dev *pdev); + #endif /* DRIVERS_PCI_H */ From f2c7602f9c0047373ab882e5288a800934bea22d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:47:46 -0700 Subject: [PATCH 0136/2680] PCI: apple: retry transient bootstrap guard contention A transient kexec load or sleep transition need not produce another deferred-probe trigger. Retry both nonblocking admissions for up to 50 attempts separated by 20ms, releasing every acquired transition lock before sleeping. This remains bounded under the host device lock and never consumes the hardware attempt on contention. --- .../pci/controller/pcie-apple-piodma-diag.c | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index 26a1359097b2c..b0f1a44b7101b 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -127,6 +127,29 @@ static bool apple_piodma_trylock_sleep(unsigned int *flags) return false; } +static int apple_piodma_lock_transition(struct apple_piodma_diag *diag, + unsigned int *sleep_flags) +{ + unsigned int attempt; + int ret; + + /* Probe cannot wait on either lock while holding its device lock. */ + for (attempt = 0; attempt < 50; attempt++) { + if (apple_piodma_trylock_sleep(sleep_flags)) { + ret = kexec_block(&diag->kexec_blocker); + if (!ret) + return 0; + unlock_system_sleep(*sleep_flags); + if (ret != -EBUSY) + return ret; + } + if (attempt != 49) + msleep(20); + } + /* A short-lived loader may not trigger deferred-probe retries. */ + return -EPROBE_DEFER; +} + static int apple_piodma_pm_notify(struct notifier_block *nb, unsigned long action, void *unused) { @@ -598,13 +621,9 @@ int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root) int ret; /* Contention must not consume the one permitted hardware attempt. */ - if (!apple_piodma_trylock_sleep(&sleep_flags)) - return -EPROBE_DEFER; - ret = kexec_block(&diag->kexec_blocker); - if (ret) { - unlock_system_sleep(sleep_flags); - return ret == -EBUSY ? -EPROBE_DEFER : ret; - } + ret = apple_piodma_lock_transition(diag, &sleep_flags); + if (ret) + return ret; /* One hardware attempt, including permanent preflight failures. */ if (atomic_cmpxchg(&diag->attempted, 0, 1)) { ret = -EALREADY; From bd2c6fe84bedb420199cc8fb67a2b53a430f0063 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:48:00 -0700 Subject: [PATCH 0137/2680] PCI: apple: expose the immutable bootstrap gate read-only Permit unprivileged reading of the boot-time enumerate setting for diagnostics. Keep it immutable at runtime: changing admission after a retained bootstrap cannot undo hardware publication or release ownership. --- drivers/pci/controller/pcie-apple-piodma-diag.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index b0f1a44b7101b..378f649e3ce68 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -55,8 +55,8 @@ #define PIODMA_IRQ_LIMIT 16 static bool enumerate = true; -module_param(enumerate, bool, 0400); -MODULE_PARM_DESC(enumerate, "Prime once, validate native ECAM, and enumerate the two Neo functions"); +module_param(enumerate, bool, 0444); +MODULE_PARM_DESC(enumerate, "Boot-time gate: prime and enumerate Neo radios (read-only, default on)"); struct apple_piodma_request { u16 offset; From a413db6ae2c198dcb7603328ab379fbdb20f892a Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:48:58 -0700 Subject: [PATCH 0138/2680] wifi: mt7932: retain RX rate telemetry for admitted 40 and 80 MHz The qualified RXV decoder supports bounded HT/VHT/HE widths, but a stale 20 MHz guard suppressed all wider RX rates. Bound telemetry by the negotiated chandef and the existing 2 GHz 20 MHz ceiling instead. Reject 5 GHz CCK and widths beyond the current association without changing RF admission. --- drivers/net/wireless/mediatek/mt7932/data.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index 19d2e07736eb4..4d89f8925f86d 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -204,8 +204,14 @@ void mt_data_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame) if (frame->unicast && frame->rate_valid) { m->station_rx_rate_valid = mt7932_rx_rate(frame->rxv, m->phy_cap[4], &m->station_rx_rate); - /* Current association contract admits only 20 MHz and no 5 GHz CCK. */ - if (((frame->rxv >> 12) & 7) || + /* Report only RX widths admitted by the negotiated channel. + * The qualified decoder already bounds mode/MCS/NSS/GI; the + * association remains 20 MHz on 2 GHz and at most 80 MHz on 5 GHz. + */ + if (((frame->rxv >> 12) & 7) > + (m->connect_chandef.width == NL80211_CHAN_WIDTH_80 ? 2 : + m->connect_chandef.width == NL80211_CHAN_WIDTH_40 ? 1 : 0) || + (frame->channel <= 14 && ((frame->rxv >> 12) & 7)) || (frame->channel > 14 && !((frame->rxv >> 24) & 15))) m->station_rx_rate_valid = false; m->station_rx_rate_time = jiffies; From c93b373d52e958ca45feb58f2d59d007cf666004 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:49:29 -0700 Subject: [PATCH 0139/2680] wifi: mt7932: test D7 band-specific completion credits Exercise the real calibration completion machine with the shared startup/runtime reply-count helper. Assert that three 5 GHz replies finish the procedure while 2 GHz requires four; the prior startup count would instead remain active and time out. --- .../net/wireless/mediatek/mt7932/rf-test.c | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/drivers/net/wireless/mediatek/mt7932/rf-test.c b/drivers/net/wireless/mediatek/mt7932/rf-test.c index bf346b26cf6f7..3ac0e6db574a1 100644 --- a/drivers/net/wireless/mediatek/mt7932/rf-test.c +++ b/drivers/net/wireless/mediatek/mt7932/rf-test.c @@ -110,7 +110,33 @@ static void mt_rf_failure_preserves_first_error_test(struct kunit *test) KUNIT_EXPECT_EQ(test, mt_enable_scan(m), -ENOENT); } +static void mt_cal_band_reply_completion_test(struct kunit *test) +{ + struct mt7932_cal_completion state = {}; + u8 request[16] = {}, reply[20] = {}; + unsigned int band, i, expected; + + reply[3] = 1; + for (band = 0; band < 2; band++) { + memset(&state, 0, sizeof(state)); + put_unaligned_le32(band, request + 8); + expected = band ? 3 : 4; + KUNIT_ASSERT_EQ(test, mt7932_cal_begin(&state, + mt7932_cal_request_replies(request)), 0); + /* The actual D6 completion machine must finish on the final + * band-specific reply, not time out waiting for a fourth 5 GHz reply. + */ + for (i = 0; i < expected; i++) { + KUNIT_EXPECT_EQ(test, mt7932_cal_null(&state, reply, + sizeof(reply), 12), i + 1 == expected ? 1 : 0); + KUNIT_EXPECT_EQ(test, state.done, i + 1 == expected); + } + KUNIT_EXPECT_FALSE(test, state.active); + } +} + static struct kunit_case mt_rf_test_cases[] = { + KUNIT_CASE(mt_cal_band_reply_completion_test), KUNIT_CASE(mt_rf_failure_wakes_association_test), KUNIT_CASE(mt_rf_failure_blocks_startup_test), KUNIT_CASE(mt_rf_failure_preserves_first_error_test), From 388c87fc507c5e6c1ff9899e375652f9bf792540 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:50:15 -0700 Subject: [PATCH 0140/2680] wifi: mt7932: name failing OCA2 calibration sections Carry the requested section tag through every startup, association, and D7 planner. Missing sections now identify the exact oca2.bin tag and phase; malformed sections and container/profile failures have separate diagnostics. No calibration payload or unit identity is logged, and planners still validate all dependencies before transmitting fragments. --- .../wireless/mediatek/mt7932/calibration.h | 42 ++++++++++++------- .../mediatek/mt7932/calibration_transport.c | 30 ++++++++++--- .../net/wireless/mediatek/mt7932/frontend.c | 8 ++-- drivers/net/wireless/mediatek/mt7932/link.c | 5 ++- drivers/net/wireless/mediatek/mt7932/mt7932.h | 1 + 5 files changed, 60 insertions(+), 26 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/calibration.h b/drivers/net/wireless/mediatek/mt7932/calibration.h index 4bee4ada54df2..cd53e0be43509 100644 --- a/drivers/net/wireless/mediatek/mt7932/calibration.h +++ b/drivers/net/wireless/mediatek/mt7932/calibration.h @@ -118,12 +118,16 @@ static inline int mt7932_cal_validate(const u8 *data, size_t size) return 0; } -/* Only call after full validation; borrowed storage remains owned by caller. */ +/* Only call after full validation; borrowed storage remains owned by caller. + * Record the requested section for missing/invalid-section diagnostics. + */ static inline int mt7932_cal_segment(const u8 *data, unsigned int tag, - struct mt7932_cal_segment *segment) + struct mt7932_cal_segment *segment, unsigned int *input_tag) { unsigned int i, count = get_unaligned_be16(data + 10); + *input_tag = tag; + for (i = 0; i < count; i++) { const u8 *entry = data + 16 + 20 * i; @@ -146,12 +150,13 @@ struct mt7932_cal_piece { /* Type1 group0/1, taking the validated operating center, not always primary. */ static inline int mt7932_cal_association(const u8 *data, size_t size, u8 channel, - struct mt7932_cal_piece pieces[2]) + struct mt7932_cal_piece pieces[2], unsigned int *input_tag) { struct mt7932_cal_segment segment; unsigned int group, part; int ret; + *input_tag = 0; if (channel >= 1 && channel <= 13) group = 0; else if (channel >= 36 && channel <= 48 && !(channel % 2)) @@ -161,7 +166,7 @@ static inline int mt7932_cal_association(const u8 *data, size_t size, u8 channel ret = mt7932_cal_validate(data, size); if (ret) return ret; - ret = mt7932_cal_segment(data, 0x2000 | group, &segment); + ret = mt7932_cal_segment(data, 0x2000 | group, &segment, input_tag); if (ret) return ret; if (!segment.length || segment.length % 2 || segment.length / 2 > 1080) @@ -178,7 +183,7 @@ static inline int mt7932_cal_association(const u8 *data, size_t size, u8 channel /* Plan all conditional power-on fragments before the first command is sent. */ static inline int mt7932_cal_power_on(const u8 *data, size_t size, bool six_ghz, - struct mt7932_cal_piece pieces[14]) + struct mt7932_cal_piece pieces[14], unsigned int *input_tag) { static const unsigned int tags[] = {0x1001, 0x1002, 0x1003, 0x1011, 0x1012, 0x1021, 0x1022}; static const u8 groups[] = {0, 0, 0, 1, 1, 2, 2}; @@ -186,12 +191,15 @@ static inline int mt7932_cal_power_on(const u8 *data, size_t size, bool six_ghz, static const u32 params[] = {0, 0x10000, 8, 0x10}; struct mt7932_cal_segment segment; unsigned int i, part, count, at = 0; - int ret = mt7932_cal_validate(data, size); + int ret; + + *input_tag = 0; + ret = mt7932_cal_validate(data, size); if (ret) return ret; for (i = 0; i < (six_ghz ? 7 : 5); i++) { - ret = mt7932_cal_segment(data, tags[i], &segment); + ret = mt7932_cal_segment(data, tags[i], &segment, input_tag); if (ret) return ret; if (!segment.length || segment.length > 1080) @@ -205,7 +213,7 @@ static inline int mt7932_cal_power_on(const u8 *data, size_t size, bool six_ghz, u32 param = params[i]; unsigned int tag = 0x2000 | (param & 0x7f) | ((param & 0x10000) ? 0x80 : 0); - ret = mt7932_cal_segment(data, tag, &segment); + ret = mt7932_cal_segment(data, tag, &segment, input_tag); if (ret) return ret; count = tag & 0x80 ? 1 : 2; @@ -226,7 +234,7 @@ static inline int mt7932_cal_power_on(const u8 *data, size_t size, bool six_ghz, */ static inline int mt7932_cal_requested_5g(const u8 *data, size_t size, u32 upper, u32 channel, u8 version, - u8 module, struct mt7932_cal_piece pieces[7]) + u8 module, struct mt7932_cal_piece pieces[7], unsigned int *input_tag) { struct mt7932_cal_segment segment; unsigned int i, part, at = 0, selected; @@ -235,13 +243,14 @@ static inline int mt7932_cal_requested_5g(const u8 *data, size_t size, unsigned int tags[3]; int ret; + *input_tag = 0; if (upper != 0x1000 || version != 12 || module != 0x89 || channel < 36 || channel > 48 || channel % 2) return -EOPNOTSUPP; ret = mt7932_cal_validate(data, size); if (ret) return ret; - ret = mt7932_cal_segment(data, 0x501, &segment); + ret = mt7932_cal_segment(data, 0x501, &segment, input_tag); if (ret) return ret; if (segment.length != 200) @@ -263,7 +272,7 @@ static inline int mt7932_cal_requested_5g(const u8 *data, size_t size, unsigned int stride = i == 0 ? 1080 : i == 1 ? 20 : 612; unsigned int meta = i == 0 ? 0 : i == 1 ? 4 : 8; - ret = mt7932_cal_segment(data, tags[i], &segment); + ret = mt7932_cal_segment(data, tags[i], &segment, input_tag); if (ret) return ret; if (segment.length != count * stride) @@ -283,11 +292,11 @@ static inline int mt7932_cal_requested_5g(const u8 *data, size_t size, } /* A genuine D7 event, not a fabricated channel-based startup trigger. - * Initial PL profile: reject unqualified upper bits and all non-2G requests. + * Reject unqualified upper bits and band-specific profile mismatches. */ static inline int mt7932_cal_requested(const u8 *data, size_t size, const u8 *event, size_t length, u8 version, - u8 module, struct mt7932_cal_piece pieces[7]) + u8 module, struct mt7932_cal_piece pieces[7], unsigned int *input_tag) { static const u8 rules[13][3] = { {0,15,3}, {0,15,3}, {0x31,7,3}, {0x21,7,3}, @@ -300,19 +309,20 @@ static inline int mt7932_cal_requested(const u8 *data, size_t size, unsigned int width, at = 0, i, part; int ret; + *input_tag = 0; if (length != 16 || version > 12) return -EINVAL; upper = get_unaligned_le32(event + 4); band = get_unaligned_le32(event + 8); channel = get_unaligned_le32(event + 12); if (band == 1) - return mt7932_cal_requested_5g(data, size, upper, channel, version, module, pieces); + return mt7932_cal_requested_5g(data, size, upper, channel, version, module, pieces, input_tag); if (band || !channel || channel > 13 || (upper & ~0x1001U)) return -EOPNOTSUPP; ret = mt7932_cal_validate(data, size); if (ret) return ret; - ret = mt7932_cal_segment(data, 0x501, &segment); + ret = mt7932_cal_segment(data, 0x501, &segment, input_tag); if (ret) return ret; if (segment.length != 200) @@ -358,7 +368,7 @@ static inline int mt7932_cal_requested(const u8 *data, size_t size, count = module & 15 ? (tag & 0x80 ? 1 : 3) : raw_count; meta = 8; } - ret = mt7932_cal_segment(data, tag, &segment); + ret = mt7932_cal_segment(data, tag, &segment, input_tag); if (ret) return ret; if (types[i] == 1) { diff --git a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c index 86cf12908de71..32c6837615fa7 100644 --- a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c +++ b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c @@ -45,13 +45,26 @@ int mt_cal_procedure(struct mt7932 *m, struct mt7932_cal_piece *pieces, return ret; } +void mt_cal_input_error(struct mt7932 *m, int error, unsigned int tag, const char *stage) +{ + if (error == -ENOENT) + dev_err(&m->pdev->dev, "local input mediatek/mt7932/oca2.bin missing section 0x%04x during %s: %d\n", + tag, stage, error); + else if (tag) + dev_err(&m->pdev->dev, "local input mediatek/mt7932/oca2.bin section 0x%04x rejected during %s: %d\n", + tag, stage, error); + else + dev_err(&m->pdev->dev, "local input mediatek/mt7932/oca2.bin container/profile rejected during %s: %d\n", + stage, error); +} + int mt_calibration_gate(struct mt7932 *m) { const struct firmware *wcal, *oca; struct mt7932_cal_piece pieces[14]; u8 body[1028], request[16]; unsigned long flags; - unsigned int at, count, group; + unsigned int at, count, group, input_tag; int ret, planned; if (!m->six_ghz_valid || m->smart_version != 12 || !m->preload_version) @@ -67,9 +80,10 @@ int mt_calibration_gate(struct mt7932 *m) dev_err(&m->pdev->dev, "local input mediatek/mt7932/wcal.bin has invalid size %zu\n", wcal->size); goto out; } - planned = mt7932_cal_power_on(oca->data, oca->size, m->six_ghz, pieces); + planned = mt7932_cal_power_on(oca->data, oca->size, m->six_ghz, pieces, &input_tag); if (planned < 0) { ret = planned; + mt_cal_input_error(m, ret, input_tag, "power-on plan"); goto out; } /* Validate every potential PL request locally before changing firmware state. */ @@ -80,9 +94,11 @@ int mt_calibration_gate(struct mt7932 *m) put_unaligned_le32(at, request + 12); ret = mt7932_cal_requested(oca->data, oca->size, request, sizeof(request), - m->smart_version, m->module_byte, check); - if (ret < 0) + m->smart_version, m->module_byte, check, &input_tag); + if (ret < 0) { + mt_cal_input_error(m, ret, input_tag, "D7 plan"); goto out; + } } body[0] = 3; body[1] = 0; @@ -128,9 +144,11 @@ int mt_calibration_gate(struct mt7932 *m) if (!count) break; ret = mt7932_cal_requested(oca->data, oca->size, request, sizeof(request), - m->smart_version, m->module_byte, pieces); - if (ret < 0) + m->smart_version, m->module_byte, pieces, &input_tag); + if (ret < 0) { + mt_cal_input_error(m, ret, input_tag, "D7 plan"); goto out; + } ret = mt_cal_procedure(m, pieces, ret, mt7932_cal_request_replies(request), m->preload_version); if (ret) diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 433e789288403..85f1db0b1b177 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -480,7 +480,7 @@ static void mt_cal_work(struct work_struct *work) struct mt7932_cal_piece pieces[7]; u8 request[16]; unsigned long flags; - unsigned int count; + unsigned int count, input_tag; int ret; mutex_lock(&m->command_mutex); @@ -501,9 +501,11 @@ static void mt_cal_work(struct work_struct *work) if (!count) break; ret = mt7932_cal_requested(oca->data, oca->size, request, sizeof(request), - m->smart_version, m->module_byte, pieces); - if (ret < 0) + m->smart_version, m->module_byte, pieces, &input_tag); + if (ret < 0) { + mt_cal_input_error(m, ret, input_tag, "runtime D7 plan"); break; + } ret = mt_cal_procedure(m, pieces, ret, mt7932_cal_request_replies(request), m->preload_version); diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index fe96e04e9aef8..26eb74b9d1d67 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -230,12 +230,15 @@ static int mt_association_calibration(struct mt7932 *m) { const struct firmware *oca; struct mt7932_cal_piece pieces[2]; + unsigned int input_tag; int ret; ret = mt_request_input(m, &oca, "mediatek/mt7932/oca2.bin"); if (ret) return ret; - ret = mt7932_cal_association(oca->data, oca->size, m->connect_center, pieces); + ret = mt7932_cal_association(oca->data, oca->size, m->connect_center, pieces, &input_tag); + if (ret < 0) + mt_cal_input_error(m, ret, input_tag, "association plan"); if (ret == 2) ret = mt_cal_procedure(m, pieces, 2, 1, 0); release_firmware(oca); diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index c1f0c5fca2551..ecb4fe889d967 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -185,6 +185,7 @@ int mt_recovery_gate(struct mt7932 *m); int mt_cal_procedure(struct mt7932 *m, struct mt7932_cal_piece *pieces, unsigned int count, unsigned int logical, u8 context_version); int mt_calibration_gate(struct mt7932 *m); +void mt_cal_input_error(struct mt7932 *m, int error, unsigned int tag, const char *stage); void mt_data_clean_locked(struct mt7932 *m); void mt_data_clean(struct mt7932 *m); void mt_bss_presence(struct mt7932 *m, const struct mt7932_event *event); From b78c4cb32015e2f99976a9ab8582e5d7950d057a Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:50:23 -0700 Subject: [PATCH 0141/2680] docs: apple: describe Neo sleep refusal and bounded radio support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the accepted release boundaries explicit: Neo sleep and wake are unproven, retained radio admission refuses suspend regardless of installed firmware, Wi-Fi exposes only the current channel and authentication scope, and Bluetooth remains opt-in. Separate checked probe-reset recovery, retryable missing policy and latched startup faults. Record exact Bluetooth probing/address order, mandatory country packages, privileged link-power overrides and retained PCI reset restrictions. [ryan: adapted to 7.2.10 — documentation conflicts against the non-linear history of mt7932-neo.rst are resolved to this commit's text] --- Documentation/PCI/controller/apple-t8140.rst | 32 ++++++-- .../device_drivers/wifi/mt7932-neo.rst | 80 ++++++++++++++----- 2 files changed, 85 insertions(+), 27 deletions(-) diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index 991f7b68e37a6..559ee5cb2f992 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -1,7 +1,7 @@ .. SPDX-License-Identifier: GPL-2.0-only T8140/J700 experimental radio PCIe bootstrap -========================================== +============================================ The J700 path trains port 0, validates a bounded PIODMA bootstrap request, and enables native ECAM for the MediaTek Wi-Fi and Bluetooth functions. @@ -13,18 +13,32 @@ The bootstrap enables enumeration by default only on machines matching both ``apple,j700`` and ``apple,t8140``, through the matching diagnostic and host device-tree nodes. No kernel argument is required. To leave the root-only diagnostic path closed, pass ``pcie_apple_piodma_diag.enumerate=0``. +The read-only ``enumerate`` parameter reports the configured admission request, +not whether the current machine is a Neo or its radios were admitted. Its +default value is also visible on other Macs; the machine and DT checks decide +whether it has any effect. The shared kernel can keep suspend, hibernation and kexec support enabled. -Once the supplier retains its arena, a PM notifier refuses sleep transitions +Immediately before the bootstrap first exposes its arena to hardware, a PM +notifier refuses sleep transitions and its device prepare callback provides a second veto. This also covers ``/dev/snapshot`` files opened before the supplier probes: later image, restore and suspend ioctls enter device PM without repeating the prepare notifier. A kexec interlock refuses loading or executing a replacement kernel, including an already loaded crash kernel. Image unloading remains permitted. -These interlocks apply only to the active J700 supplier; other machines do not -acquire them. A successful supplier probe retains this protection even if later -radio enumeration fails. Failed or uncertain bootstrap state requires a full -external hardware reset; there is no command retry. +These interlocks apply only to the active J700 bootstrap; other machines do not +acquire them. Merely probing its supplier does not block transitions. Once the +bootstrap retains memory, the protection persists even if enumeration or either +radio's firmware loading fails. Thus the block applies on every default J700 +boot which reaches retained radio admission, regardless of installed firmware. +Failed or uncertain bootstrap state requires a full external hardware reset; +there is no command retry. + +Sleep is not supported on the MacBook Neo yet. With radio support active +(the default), the kernel refuses suspend: lid close and ``systemctl suspend`` +do not put it to sleep. Shut down before putting it in a bag. Booting with +``pcie_apple_piodma_diag.enumerate=0`` removes this refusal and disables Wi-Fi +and Bluetooth; sleep and wake remain unproven on this hardware without radios. Before either radio driver can bind, the host disables ASPM, all L1 substates and Clock PM on the validated radio endpoints' shared PCIe link through the @@ -40,10 +54,16 @@ the host's downstream device-enable callback enforces the same gate for other drivers. This prevents a later generic rescan from enabling cached children of a failed admission. Each radio probe reapplies the link-power restriction, so a recreated ASPM link state cannot inherit an incompatible global policy. +Privileged writes to the per-link PCI sysfs ASPM or Clock PM attributes can +override the restriction; it is reapplied only at probe, not continuously. +Do not enable those link power states while the Neo radios are admitted. The PIODMA arena remains allocated until external reset. Standard PCI sysfs ``remove`` writes for the root port and radio functions return ``-EBUSY`` while the host has its retained supplier; the removal attributes remain present. +The same hierarchy refuses userspace ``reset`` and ``reset_subordinate`` +writes, and its host does not hand native PCIe or SHPC slot hot-plug control +to Linux. These restrictions leave other PCI hosts unchanged. The host and activated Bluetooth modules are pinned until reset. Arbitrary platform-device removal, IOMMU teardown, forced module removal, memory reuse and arbitrary downstream devices remain unqualified. Do not remove the diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 4330af26a9f8c..6a1fe4331bfc9 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -1,12 +1,11 @@ .. SPDX-License-Identifier: GPL-2.0-only MT7932 radio bring-up on the MacBook Neo -====================================== +======================================== -This contribution combines the existing J700 MT7932 fullmac Wi-Fi driver with an -opt-in PCIe Bluetooth transport on the public Aurora platform. Wi-Fi uses -cfg80211 and NetworkManager with firmware-managed WPA2-CCMP authentication. -It is not an mt76/mac80211 softmac implementation. +J700 MT7932 Wi-Fi uses cfg80211 and NetworkManager with firmware-managed +WPA2-CCMP authentication or open networks. Bluetooth uses an opt-in PCIe +transport and remains disabled by default pending physical qualification. The Wi-Fi import is pinned to ``aurora-silicon/linux-neo-cleanroom-eryk-with-wifi``, branch @@ -31,10 +30,10 @@ it. Public m1n1 must populate the J700 ``wifi0`` endpoint's own The original hardware-tested configuration included ``CONFIG_MT7932_FULLMAC=m``, ``CONFIG_CFG80211=y``, ``CONFIG_BT_MTK7932_PCIE=y``, ``CONFIG_BT_BREDR=y``, ``CONFIG_BT_LE=y``, ``CONFIG_CRYPTO_AES=y`` and ``CONFIG_CRYPTO_CMAC=y``. -The shared build supports ``CONFIG_PCIE_APPLE=m``, +The shared build supports ``CONFIG_PCIE_APPLE=y`` or ``m``, ``CONFIG_PCIE_APPLE_PIODMA_DIAG=y``, ``CONFIG_PCIEASPM=y``, ``CONFIG_MT7932_FULLMAC=m`` and ``CONFIG_BT_MTK7932_PCIE=m`` with modular -Bluetooth/rfkill and sleep/kexec enabled. The active J700 supplier refuses sleep +Bluetooth/rfkill and sleep/kexec enabled. The active J700 bootstrap refuses sleep and kexec at runtime while its arena is retained; other Macs keep their normal behavior. Firmware loading is selected by both radio drivers. This shared build still requires physical Neo qualification. @@ -54,7 +53,19 @@ driver binding; no global PCIe ASPM performance policy is required. Load ``mt7932-fullmac`` after the root filesystem and local firmware packages are available. Bluetooth's gate defaults closed; load ``mt7932_bt_pcie`` with the default ``enable=0``, validate the cold, unbound ``14c3:793b`` function, enable -``/sys/module/mt7932_bt_pcie/parameters/enable``, then request its PCI probe. +``/sys/module/mt7932_bt_pcie/parameters/enable``, then request its PCI probe:: + + echo 1 > /sys/module/mt7932_bt_pcie/parameters/enable + echo YOUR_BT_PCI_BDF > /sys/bus/pci/drivers_probe + +Use the actual ``14c3:793b`` PCI address for ``YOUR_BT_PCI_BDF``. The driver +suppresses its individual bind/unbind attributes, so there is no driver +``bind`` file. For an intentionally enabled cold boot, a file under +``/etc/modprobe.d/`` may instead contain:: + + options mt7932_bt_pcie enable=1 + +Install the target unit's Bluetooth inputs before opening the gate. Do not reprobe after a failed or uncertain Bluetooth admission. The tested Wi-Fi driver owns function 0 and Bluetooth owns function 1. @@ -81,11 +92,12 @@ Wi-Fi requests the following files under ``mediatek/mt7932/``: * ``wcal.bin`` and ``oca2.bin``: the unit's factory calibration; OCA2 retains its original BLOB container, rather than an older replay-stream wrapper. * ``config-original.bin``: the bounded J7CF package with the original 64 or - 65 configuration records. Required keys and record shapes are validated. -* ``policy/world-XZ.bin`` and, when available, the matching country package - under ``policy/``: J7RP containers with original-derived modes and power - tables. The parser checks the country and table shapes. Relabeling a - different country's payload is not a supported conversion. + 65 configuration records, including all required keys. +* ``policy/world-XZ.bin`` for cfg80211 country 00, or ``policy/.bin`` for + the matching explicit uppercase country code: J7RP containers with + original-derived modes and power tables. A country package is mandatory + whenever that country is set; there is no fallback to the world package. + Relabeling a different country's payload is unsupported. Bluetooth requests these files under ``mediatek/``: @@ -97,7 +109,8 @@ Bluetooth requests these files under ``mediatek/``: * ``j700-mt7932-btcal.bin`` (the measured input is 388 bytes; the driver accepts nonempty inputs up to 65535 bytes and relies on HCI setup acceptance); * ``MT7932_PTB_IzubaA_0.1.0.0_20251021141303.ptx`` (198 bytes); -* ``j700-mt7932-bdaddr.bin`` (the unit's six-byte Bluetooth address). +* ``j700-mt7932-bdaddr.bin`` (the unit's six-byte Bluetooth address, + most-significant byte first; the driver converts it to HCI byte order). Extraction and packaging require the original local assets. The Linux loader does not grant redistribution rights to them. Do not include those assets or @@ -109,12 +122,20 @@ it requests ``oca2.bin`` again for runtime D7 requests and each association. unconverted source files to those names does not satisfy the format checks. Load the radio drivers after these inputs are available on the real root. -An early Wi-Fi probe already owns DMA before it requests firmware. Only an -explicit ``initialization failed; DMA retired`` message confirms that its -checked reset completed and the unbound driver can be loaded again. A failed -reset retains the binding and DMA; do not reprobe it. Use a full external reset -after a failed or uncertain admission. Installing a missing startup calibration -file does not clear a terminal RF failure in an already bound epoch. +There are three distinct Wi-Fi failure classes: + +* An initialization probe failure can occur after DMA is published but before + an interface is registered. Only ``initialization failed; DMA retired`` + confirms that the checked reset completed and the unbound driver may be + loaded again. A failed + reset retains the binding and DMA; do not reprobe it. +* ``REGULATORY_BLOCKED`` with ``error=-2`` and ``recovery-required=0`` means + the requested policy file is absent before policy submission. Install that + exact file and bring the interface up or retry a scan/connection. An identical + ``iw reg set`` request alone does not trigger a retry in cfg80211. +* ``recovery-required=1`` or ``RF_FAILED`` is a latched startup/calibration + failure. Installing a file does not clear it in the bound epoch. A full + external reset is required after a failed or uncertain admission. Repeatable physical network test -------------------------------- @@ -139,17 +160,34 @@ from the result. A successful host serial command is not a test pass: require Qualification and remaining work -------------------------------- +* Wi-Fi station operation is limited to 2.4 GHz channels 1--13 and 5 GHz + channels 36, 40, 44 and 48, subject to the applicable regulatory rules. + 5 GHz supports 20/40/80 MHz channel layouts within this range. Channels + 149--165 and 6 GHz are not exposed. Authentication supports open networks + and WPA2-PSK with CCMP; WPA3/SAE, 802.1X, required MFP and other ciphers + are unsupported. AP/P2P and roaming are unsupported. +* Sleep is not supported on the MacBook Neo yet. With radio support active + (the default), the retained bootstrap makes the kernel refuse suspend: + lid close and ``systemctl suspend`` do not put it to sleep. Shut down before + putting it in a bag. Booting with ``pcie_apple_piodma_diag.enumerate=0`` + removes this refusal and disables Wi-Fi and Bluetooth; sleep and wake are + still unproven on this hardware without the radios. * PCI bootstrap memory remains retained until external reset. Sleep and kexec are refused while it is retained; they are not hardware-qualified on the Neo. Standard PCI sysfs removal of the root and radio functions is refused; arbitrary controller/IOMMU teardown and memory reuse remain unqualified. Wi-Fi shutdown joins host producers before disabling bus mastering and retains device-visible memory until reset. + Userspace PCI ``reset`` and ``reset_subordinate`` are also refused, and + native slot hot-plug is disabled for this host. Per-link sysfs ASPM/Clock PM + writes can lift the power-state restrictions; they are reapplied only at + probe. Do not enable those link power states while the radios are active. * Bluetooth PCI removal/quiescence is incomplete. Its software queue limit does not provide HCI backpressure; saturation can drop an accounted frame. Both require correction before production use. An activated Bluetooth module cannot be unloaded normally; forced PCI removal only retires software - callbacks and does not establish DMA quiescence or release retained ownership. + callbacks, including the threaded IRQ and MSI vectors, and does not establish + DMA quiescence or release retained ownership. * Arbitrary scan IEs, WPA3/SAE, required MFP, AP/P2P, general country-package generation, roaming and long-duration reliability are unqualified. * SCO/headset microphone, LE Audio/ISO and simultaneous headset audio are From 174d92de2c85da1636543dd80e2405f8a20c2121 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:51:17 -0700 Subject: [PATCH 0142/2680] wifi: mt7932: validate missing-policy retries through frontend admission Place interface-up recovery in the actual ndo_open entry point, not scan completion. Exercise open, scan and connect with the real retry helper and a hardware-free work callback; repeated admission requests consume one retry, RF stays blocked, and partial policy or terminal calibration errors do not replay. --- .../net/wireless/mediatek/mt7932/frontend.c | 5 +- drivers/net/wireless/mediatek/mt7932/mt7932.h | 1 + .../net/wireless/mediatek/mt7932/rf-test.c | 68 ++++++++++++++++++- 3 files changed, 70 insertions(+), 4 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 85f1db0b1b177..9104f497e2f4d 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -6,7 +6,6 @@ static void mt_scan_timeout_work(struct work_struct *work); static int mt_scan_submit_next(struct mt7932 *m); static int mt_scan_admit(struct mt7932 *m); static int mt_set_bss(struct mt7932 *m, bool active); -static int mt_net_open(struct net_device *netdev); static int mt_net_stop(struct net_device *netdev); static void mt_cal_work(struct work_struct *work); @@ -30,7 +29,6 @@ static void mt_scan_finish_work(struct work_struct *work) /* Retire the old timer before allowing another request to take ownership. */ cancel_delayed_work_sync(&m->scan_timeout_work); - mt_retry_missing_policy(m); mutex_lock(&m->command_mutex); spin_lock_irqsave(&m->response_lock, flags); request = m->scan_request; @@ -388,13 +386,14 @@ static int mt_set_bss(struct mt7932 *m, bool active) return ret; } -static int mt_net_open(struct net_device *netdev) +int mt_net_open(struct net_device *netdev) { struct mt7932 *m = *(struct mt7932 **)netdev_priv(netdev); unsigned long flags; int ret = 0; mutex_lock(&m->command_mutex); + mt_retry_missing_policy(m); spin_lock_irqsave(&m->response_lock, flags); if (m->stopping) ret = -ESHUTDOWN; diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index ecb4fe889d967..0bbefd9620cf8 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -167,6 +167,7 @@ struct mt7932 { bool mt_rf_allowed(struct mt7932 *m); void mt_retry_missing_policy(struct mt7932 *m); +int mt_net_open(struct net_device *netdev); int mt_request_input(struct mt7932 *m, const struct firmware **fw, const char *name); /* response_lock held; closes RF admission and reports terminal host state. */ void mt_rf_fail_locked(struct mt7932 *m, int error); diff --git a/drivers/net/wireless/mediatek/mt7932/rf-test.c b/drivers/net/wireless/mediatek/mt7932/rf-test.c index 3ac0e6db574a1..2fb0812d09970 100644 --- a/drivers/net/wireless/mediatek/mt7932/rf-test.c +++ b/drivers/net/wireless/mediatek/mt7932/rf-test.c @@ -8,10 +8,18 @@ static void mt_rf_test_device_release(struct device *dev) /* The enclosing synthetic PCI device is owned by KUnit. */ } +static void mt_policy_retry_test_work(struct work_struct *work) +{ + struct mt7932 *m = container_of(work, struct mt7932, startup_work); + + m->reg_attempted++; +} + static void mt_rf_test_cleanup(void *data) { struct mt7932 *m = data; + cancel_work_sync(&m->startup_work); free_netdev(m->netdev); put_device(&m->pdev->dev); } @@ -34,7 +42,8 @@ static int mt_rf_test_init(struct kunit *test) put_device(&m->pdev->dev); return ret; } - m->netdev = alloc_etherdev(0); + INIT_WORK(&m->startup_work, mt_policy_retry_test_work); + m->netdev = alloc_etherdev(sizeof(m)); if (!m->netdev) { put_device(&m->pdev->dev); return -ENOMEM; @@ -42,6 +51,8 @@ static int mt_rf_test_init(struct kunit *test) ret = kunit_add_action_or_reset(test, mt_rf_test_cleanup, m); if (ret) return ret; + *(struct mt7932 **)netdev_priv(m->netdev) = m; + mutex_init(&m->command_mutex); spin_lock_init(&m->response_lock); spin_lock_init(&m->data_lock); init_completion(&m->cal_response); @@ -135,7 +146,62 @@ static void mt_cal_band_reply_completion_test(struct kunit *test) } } +static void mt_missing_policy_admission_retry_test(struct kunit *test) +{ + static const struct cfg80211_ops ops = {}; + struct mt7932 *m = test->priv; + struct cfg80211_scan_request scan = { .wdev = &m->wdev }; + struct cfg80211_connect_params connect = {}; + struct wiphy *wiphy; + unsigned int entry; + + wiphy = wiphy_new(&ops, sizeof(m)); + KUNIT_ASSERT_NOT_NULL(test, wiphy); + *(struct mt7932 **)wiphy_priv(wiphy) = m; + /* Missing policy has not touched hardware. Actual frontend entry points + * must retry that lookup while keeping ordinary RF admission closed. + */ + m->rf_ready = false; + m->reg_pending = true; + for (entry = 0; entry < 3; entry++) { + m->reg_retryable = true; + m->reg_generation = 7; + m->reg_attempted = 7; + switch (entry) { + case 0: + KUNIT_EXPECT_EQ(test, mt_net_open(m->netdev), 0); + break; + case 1: + KUNIT_EXPECT_EQ(test, mt_scan(wiphy, &scan), -EOPNOTSUPP); + break; + default: + KUNIT_EXPECT_EQ(test, mt_connect(wiphy, m->netdev, &connect), -EAGAIN); + break; + } + mt_retry_missing_policy(m); + flush_work(&m->startup_work); + KUNIT_EXPECT_EQ(test, m->reg_generation, 8U); + KUNIT_EXPECT_EQ(test, m->reg_attempted, 8U); + KUNIT_EXPECT_FALSE(test, m->reg_retryable); + KUNIT_EXPECT_TRUE(test, m->reg_pending); + KUNIT_EXPECT_FALSE(test, mt_rf_allowed(m)); + } + /* Neither a partial policy SET nor terminal calibration may be replayed. */ + m->reg_retryable = true; + m->policy_failed = true; + mt_retry_missing_policy(m); + flush_work(&m->startup_work); + KUNIT_EXPECT_EQ(test, m->reg_generation, 8U); + m->policy_failed = false; + mt_rf_test_fail(m, -EPROTO); + KUNIT_EXPECT_EQ(test, mt_net_open(m->netdev), -EPROTO); + flush_work(&m->startup_work); + KUNIT_EXPECT_EQ(test, m->reg_generation, 8U); + wiphy_free(wiphy); +} + static struct kunit_case mt_rf_test_cases[] = { + KUNIT_CASE(mt_missing_policy_admission_retry_test), KUNIT_CASE(mt_cal_band_reply_completion_test), KUNIT_CASE(mt_rf_failure_wakes_association_test), KUNIT_CASE(mt_rf_failure_blocks_startup_test), From c5a8ddec3ced7e43d5068ae59f70340f76131031 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:51:44 -0700 Subject: [PATCH 0143/2680] wifi: mt7932: describe both admitted station security modes The source accepts qualified open networks as well as WPA2-PSK with CCMP. Describe both in the configuration help without claiming wider security or RF support. --- drivers/net/wireless/mediatek/mt7932/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index f5a5aa6af5690..f5a22557f48b6 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -6,7 +6,7 @@ config MT7932_FULLMAC select FW_LOADER help Native support for the built-in J700 Wi-Fi endpoint: managed - WPA2-CCMP station operation on 2.4 GHz at 20 MHz and channels 36..48 + open-network or WPA2-CCMP station operation on 2.4 GHz at 20 MHz and channels 36..48 at 20/40/80 MHz, with a software bandwidth limit of 80 MHz. Requires original firmware, own calibration and original-derived country policy packages in addition to the cfg80211 regulatory database. From e218c1fe676206805be72a949561196de95c2634 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:52:44 -0700 Subject: [PATCH 0144/2680] kexec: identify permanent builtin retained-memory interlocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use a builtin-only guard handle with an immutable boot-lifetime reason and balanced per-guard acquisitions. Drop unused module exports, retain lock-free releases and the existing atomic execution lock, and report the first interlock reason in each nonzero interval without a lifetime-sensitive registry. Permanent kexec and Neo sleep refusals return EOPNOTSUPP; transient admission contention remains EBUSY. [ryan: adapted to 7.2.10 — split by topic: only the pcie-apple-piodma-diag.c hunk; the kexec side is in 010-soc; the kexec_blocker plumbing moved into the commit that first used the interlock] --- drivers/pci/controller/pcie-apple-piodma-diag.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index 378f649e3ce68..cafa1dd4e636c 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -162,7 +162,7 @@ static int apple_piodma_pm_notify(struct notifier_block *nb, case PM_RESTORE_PREPARE: dev_warn(diag->dev, "sleep refused: Neo radio bootstrap retains DMA memory until full hardware reset\n"); - return notifier_from_errno(-EBUSY); + return notifier_from_errno(-EOPNOTSUPP); default: return NOTIFY_DONE; } @@ -182,7 +182,7 @@ static int apple_piodma_pm_prepare(struct device *dev) */ dev_warn(dev, "sleep refused: Neo radio bootstrap retains DMA memory until full hardware reset\n"); - return -EBUSY; + return -EOPNOTSUPP; } static const struct dev_pm_ops apple_piodma_pm_ops = { From c6e482ad70d47d2dac71600d5812e75d11c37e96 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:53:05 -0700 Subject: [PATCH 0145/2680] PCI: protect retained Neo radio hierarchy from user reset and hotplug User function and subordinate-bus resets can invalidate retained bootstrap/radio ownership. Add an immutable per-host reset policy checked before reset or runtime-PM work, keeping driver-owned checked resets available. Disable native PCIe/SHPC hotplug only on the matched Neo host, and withhold their service/probe even when native port handling is forced. Other hosts retain their default policy. Extend standalone hierarchy-policy tests and rate-limit reset refusals. --- drivers/pci/Kconfig | 6 ++--- drivers/pci/controller/pcie-apple.c | 9 +++++-- drivers/pci/hotplug/shpchp_core.c | 4 +++ drivers/pci/pci-sysfs-test.c | 40 +++++++++++++++++++++++++++++ drivers/pci/pci-sysfs.c | 21 ++++++++++++++- drivers/pci/pci.h | 1 + drivers/pci/pcie/portdrv.c | 3 ++- include/linux/pci.h | 2 ++ 8 files changed, 79 insertions(+), 7 deletions(-) diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig index 048bc617b42ab..1542369b6ad41 100644 --- a/drivers/pci/Kconfig +++ b/drivers/pci/Kconfig @@ -30,12 +30,12 @@ menuconfig PCI if PCI config PCI_SYSFS_KUNIT_TEST - bool "Unit tests for PCI sysfs removal restrictions" if !KUNIT_ALL_TESTS + bool "Unit tests for PCI sysfs removal and reset restrictions" if !KUNIT_ALL_TESTS depends on SYSFS && KUNIT=y default KUNIT_ALL_TESTS help - Test userspace-removal policy lookup for root ports and descendants - under independent host bridges. The synthetic objects are separately + Test userspace-removal and reset policy lookup for root ports and + descendants under independent host bridges. The synthetic objects are separately allocated and never registered or passed to a sysfs callback. config PCI_DOMAINS diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index e72905b630654..91a905263be70 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2516,8 +2516,13 @@ static int apple_pcie_probe(struct platform_device *pdev) pcie->dev = dev; pcie->hw = hw; pcie->piodma_supplier = piodma_supplier; - /* A retained supplier forbids manual removal of its root or radios. */ - bridge->no_user_remove = !!piodma_supplier; + /* Only this retained-DMA radio hierarchy forbids user reset/removal. */ + if (piodma_supplier) { + bridge->no_user_remove = true; + bridge->no_user_reset = true; + bridge->native_pcie_hotplug = false; + bridge->native_shpc_hotplug = false; + } if (hw->root_bus_only) dev_info(dev, "root-port probe only; downstream config is blocked\n"); pcie->base = devm_platform_ioremap_resource(pdev, 1); diff --git a/drivers/pci/hotplug/shpchp_core.c b/drivers/pci/hotplug/shpchp_core.c index 8d6f8eea599d5..aea48acb3fb89 100644 --- a/drivers/pci/hotplug/shpchp_core.c +++ b/drivers/pci/hotplug/shpchp_core.c @@ -253,6 +253,10 @@ static int shpc_probe(struct pci_dev *pdev, const struct pci_device_id *ent) int rc; struct controller *ctrl; + /* Do not grant a slot-removal path into a retained-DMA hierarchy. */ + if (pci_find_host_bridge(pdev->bus)->no_user_remove) + return -ENODEV; + if (!shpc_capable(pdev)) return -ENODEV; diff --git a/drivers/pci/pci-sysfs-test.c b/drivers/pci/pci-sysfs-test.c index c166c8659f19d..47714f825ee69 100644 --- a/drivers/pci/pci-sysfs-test.c +++ b/drivers/pci/pci-sysfs-test.c @@ -76,10 +76,50 @@ static void pci_sysfs_remove_independent_host_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, pci_sysfs_user_remove_allowed(h->grandchild)); } +static void pci_sysfs_reset_default_test(struct kunit *test) +{ + struct pci_sysfs_test_hierarchy *h = test->priv; + + KUNIT_EXPECT_TRUE(test, pci_sysfs_user_reset_allowed(h->root_port)); + KUNIT_EXPECT_TRUE(test, pci_sysfs_user_reset_allowed(h->endpoint)); + KUNIT_EXPECT_TRUE(test, pci_sysfs_user_reset_allowed(h->grandchild)); +} + +static void pci_sysfs_reset_retained_test(struct kunit *test) +{ + struct pci_sysfs_test_hierarchy *h = test->priv; + + h->bridge->no_user_reset = true; + KUNIT_EXPECT_FALSE(test, pci_sysfs_user_reset_allowed(h->root_port)); + KUNIT_EXPECT_FALSE(test, pci_sysfs_user_reset_allowed(h->endpoint)); + KUNIT_EXPECT_FALSE(test, pci_sysfs_user_reset_allowed(h->grandchild)); + /* Reset and removal policies are independent for other host users. */ + KUNIT_EXPECT_TRUE(test, pci_sysfs_user_remove_allowed(h->grandchild)); + h->bridge->no_user_reset = false; + h->bridge->no_user_remove = true; + KUNIT_EXPECT_TRUE(test, pci_sysfs_user_reset_allowed(h->grandchild)); +} + +static void pci_sysfs_reset_independent_host_test(struct kunit *test) +{ + struct pci_sysfs_test_hierarchy *h = test->priv; + struct pci_host_bridge *other; + + other = kunit_kzalloc(test, sizeof(*other), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, other); + h->bridge->no_user_reset = true; + KUNIT_ASSERT_FALSE(test, pci_sysfs_user_reset_allowed(h->grandchild)); + h->root_bus->bridge = &other->dev; + KUNIT_EXPECT_TRUE(test, pci_sysfs_user_reset_allowed(h->grandchild)); +} + static struct kunit_case pci_sysfs_test_cases[] = { KUNIT_CASE(pci_sysfs_remove_default_test), KUNIT_CASE(pci_sysfs_remove_retained_test), KUNIT_CASE(pci_sysfs_remove_independent_host_test), + KUNIT_CASE(pci_sysfs_reset_default_test), + KUNIT_CASE(pci_sysfs_reset_retained_test), + KUNIT_CASE(pci_sysfs_reset_independent_host_test), {} }; diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c index 727ab49a5db0c..a554ee24ecb5f 100644 --- a/drivers/pci/pci-sysfs.c +++ b/drivers/pci/pci-sysfs.c @@ -517,6 +517,12 @@ bool pci_sysfs_user_remove_allowed(struct pci_dev *pdev) return !pci_find_host_bridge(pdev->bus)->no_user_remove; } +bool pci_sysfs_user_reset_allowed(struct pci_dev *pdev) +{ + /* User resets can disrupt DMA still owned by a retained host. */ + return !pci_find_host_bridge(pdev->bus)->no_user_reset; +} + static ssize_t remove_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { @@ -577,7 +583,14 @@ static ssize_t reset_subordinate_store(struct device *dev, return -EINVAL; if (val) { - int ret = pci_try_reset_bridge(pdev); + int ret; + + if (!pci_sysfs_user_reset_allowed(pdev)) { + dev_warn_ratelimited(&pdev->dev, + "userspace reset is disabled by the host bridge\n"); + return -EBUSY; + } + ret = pci_try_reset_bridge(pdev); if (ret) return ret; @@ -1588,6 +1601,12 @@ static ssize_t reset_store(struct device *dev, struct device_attribute *attr, if (val != 1) return -EINVAL; + if (!pci_sysfs_user_reset_allowed(pdev)) { + dev_warn_ratelimited(&pdev->dev, + "userspace reset is disabled by the host bridge\n"); + return -EBUSY; + } + pm_runtime_get_sync(dev); result = pci_reset_function(pdev); pm_runtime_put(dev); diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index f3230633caa0a..880d53341c761 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -1459,5 +1459,6 @@ static inline int pci_msix_write_tph_tag(struct pci_dev *pdev, unsigned int inde PCI_CONF1_EXT_REG(reg)) bool pci_sysfs_user_remove_allowed(struct pci_dev *pdev); +bool pci_sysfs_user_reset_allowed(struct pci_dev *pdev); #endif /* DRIVERS_PCI_H */ diff --git a/drivers/pci/pcie/portdrv.c b/drivers/pci/pcie/portdrv.c index 2d6aa488fe7b6..d93f6f201d3d4 100644 --- a/drivers/pci/pcie/portdrv.c +++ b/drivers/pci/pcie/portdrv.c @@ -220,7 +220,8 @@ static int get_port_device_capability(struct pci_dev *dev) struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); int services = 0; - if (dev->is_pciehp && + /* A retained hierarchy must not be removed, even with native forcing. */ + if (!host->no_user_remove && dev->is_pciehp && (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT || pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM) && (pcie_ports_native || host->native_pcie_hotplug)) { diff --git a/include/linux/pci.h b/include/linux/pci.h index 8b47b5e42217b..3f04012108ab7 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -668,6 +668,8 @@ struct pci_host_bridge { system suspend is broken */ /* Disable sysfs device removal for this hierarchy; set before probing. */ unsigned int no_user_remove:1; + /* Disable sysfs function/bridge reset; set before probing. */ + unsigned int no_user_reset:1; /* Resource alignment requirements */ resource_size_t (*align_resource)(struct pci_dev *dev, From 74d6594dc4475b4b3c0f43049faa40e314956615 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:53:31 -0700 Subject: [PATCH 0146/2680] docs: mt7932: require explicit country policy in an early initramfs The runtime country filename is dynamic. Its MODULE_FIRMWARE pattern helps image builders discover installed inputs but does not guarantee they include every match. Keep deployment explicit about the selected mandatory country package so an early load does not rely on root filesystem availability. --- Documentation/networking/device_drivers/wifi/mt7932-neo.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 6a1fe4331bfc9..8a5ceeb6bebf1 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -120,6 +120,10 @@ Keep the installed calibration inputs stable while the Wi-Fi driver is bound: it requests ``oca2.bin`` again for runtime D7 requests and each association. ``config-original.bin`` and country policies require J7CF/J7RP packaging; copying unconverted source files to those names does not satisfy the format checks. +If loading Wi-Fi in the initramfs, include every fixed input and the selected +country package there explicitly. Firmware metadata includes a country-file +pattern, but an initramfs builder's glob handling may select only one match; +inspect the resulting image rather than assuming every country is included. Load the radio drivers after these inputs are available on the real root. There are three distinct Wi-Fi failure classes: From a352e9b98799cf6c7fa1fdc893c42ec7e0ec5d0d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:57:34 -0700 Subject: [PATCH 0147/2680] kexec: test retained-memory load vetoes and unload exceptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exercise the actual legacy and file loader implementations with deliberately invalid inputs while a retained-memory guard is active, and unload only empty image slots. Share the crash selector with tests without installing fake global images or executing a transition. Keep these tests builtin-only and rate limit repeated sleep and normal-kexec refusal diagnostics; crash diagnostics remain emergency messages. [ryan: adapted to 7.2.10 — split by topic: only the pcie-apple-piodma-diag.c hunk; the kexec side is in 010-soc] --- drivers/pci/controller/pcie-apple-piodma-diag.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index cafa1dd4e636c..4caf7a37a5b06 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -160,8 +160,8 @@ static int apple_piodma_pm_notify(struct notifier_block *nb, case PM_SUSPEND_PREPARE: case PM_HIBERNATION_PREPARE: case PM_RESTORE_PREPARE: - dev_warn(diag->dev, - "sleep refused: Neo radio bootstrap retains DMA memory until full hardware reset\n"); + dev_warn_ratelimited(diag->dev, + "sleep refused: Neo radio bootstrap retains DMA memory until full hardware reset\n"); return notifier_from_errno(-EOPNOTSUPP); default: return NOTIFY_DONE; @@ -180,8 +180,8 @@ static int apple_piodma_pm_prepare(struct device *dev) * supplier probed. Its later image/restore/suspend ioctls go straight * to device PM, so retain a veto in the device prepare phase as well. */ - dev_warn(dev, - "sleep refused: Neo radio bootstrap retains DMA memory until full hardware reset\n"); + dev_warn_ratelimited(dev, + "sleep refused: Neo radio bootstrap retains DMA memory until full hardware reset\n"); return -EOPNOTSUPP; } From e31b09083b9e983e8bd525c5dafcb50ff0515acb Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:01:10 -0700 Subject: [PATCH 0148/2680] PCI: withhold AER and DPC recovery on reset-sensitive retained hosts Native error-recovery services can reset the radio link independently of the protected sysfs reset attributes. Suppress AER and DPC service creation when the host forbids user reset, including native and dpc-native overrides. The matched Neo host sets this immutable policy before enumeration; other hosts keep their existing services. --- drivers/pci/pcie/portdrv.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/pci/pcie/portdrv.c b/drivers/pci/pcie/portdrv.c index d93f6f201d3d4..57c5163d779cc 100644 --- a/drivers/pci/pcie/portdrv.c +++ b/drivers/pci/pcie/portdrv.c @@ -238,7 +238,8 @@ static int get_port_device_capability(struct pci_dev *dev) } #ifdef CONFIG_PCIEAER - if ((pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT || + /* Error recovery may reset links and invalidate retained DMA ownership. */ + if (!host->no_user_reset && (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT || pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC) && dev->aer_cap && pci_aer_available() && (pcie_ports_native || host->native_aer)) @@ -263,7 +264,8 @@ static int get_port_device_capability(struct pci_dev *dev) * With dpc-native, allow Linux to use DPC even if it doesn't have * permission to use AER. */ - if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DPC) && + if (!host->no_user_reset && + pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DPC) && pci_aer_available() && (pcie_ports_dpc_native || (services & PCIE_PORT_SERVICE_AER))) services |= PCIE_PORT_SERVICE_DPC; From 92171fc6eb8bc321cba1d0c2c03e01ddee6c3a97 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:01:28 -0700 Subject: [PATCH 0149/2680] docs: apple: record retained host assignment and recovery exclusions VFIO and native AER/DPC recovery can reach reset operations outside the sysfs controls. Document the additional Neo-only admission exclusions and require external reset on a hardware fault instead of suggesting an unqualified bus or link reset. --- Documentation/PCI/controller/apple-t8140.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index 559ee5cb2f992..204685f47eb24 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -64,6 +64,10 @@ while the host has its retained supplier; the removal attributes remain present. The same hierarchy refuses userspace ``reset`` and ``reset_subordinate`` writes, and its host does not hand native PCIe or SHPC slot hot-plug control to Linux. These restrictions leave other PCI hosts unchanged. +VFIO assignment is refused on this host because it exposes reset operations +outside PCI sysfs. Automatic native AER/DPC port recovery is also withheld: +its bus or link reset has no qualified retained-memory contract here. A hardware +fault requires external reset rather than guessed recovery. The host and activated Bluetooth modules are pinned until reset. Arbitrary platform-device removal, IOMMU teardown, forced module removal, memory reuse and arbitrary downstream devices remain unqualified. Do not remove the From 31ea846641070dc431bb4d887f4d5a9d21284b2e Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:36:21 -0700 Subject: [PATCH 0150/2680] selftests: mt7932: restore saved band and report actual authentication MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Record open versus WPA2 from the saved profile instead of declaring every successful connection WPA2. Restore its original band in finally even when selection or activation times out, and reject warning, sanitizer, hung-task and refused-reset diagnostics. Add local tests for restoration failures and both authentication modes. Round-2 S2. This manual helper checks both apple,j700 and apple,t8140 before NetworkManager mutations. No production kernel path changes; all non-Neo behavior remains unchanged. Round-2 S2. Non-Neo runtime behavior remains unchanged. [ryan: adapted to 7.2.10 — moved from the tooling bucket into the radio topic with the driver it tests; 900-config-tooling skips it] --- .../selftests/drivers/net/mt7932_e2e.py | 108 +++++++++++------- .../selftests/drivers/net/mt7932_e2e_test.py | 55 +++++++++ 2 files changed, 123 insertions(+), 40 deletions(-) diff --git a/tools/testing/selftests/drivers/net/mt7932_e2e.py b/tools/testing/selftests/drivers/net/mt7932_e2e.py index a29531e9da504..258baebce12f3 100644 --- a/tools/testing/selftests/drivers/net/mt7932_e2e.py +++ b/tools/testing/selftests/drivers/net/mt7932_e2e.py @@ -7,6 +7,7 @@ The saved NetworkManager profile supplies credentials on the target only. """ import argparse +from contextlib import contextmanager import hashlib import ipaddress import json @@ -21,7 +22,8 @@ fault = re.compile(r'translation fault|stale exception latched|Kernel panic|Internal error:|' r'SError Interrupt|BUG:|Unable to handle kernel|WIFI_TERMINAL|WIFI_STARTUP_FAILED|' r'qualification stopped:|CAL_PROCEDURE_FAILED|D7_CALIBRATION_FAILED|' - r'SCAN_INTERFACE_SETUP_FAILED|RF_FAILED|recovery-required=1', re.I) + r'SCAN_INTERFACE_SETUP_FAILED|RF_FAILED|recovery-required=1|WARNING:|' + r'UBSAN:|hung_task|blocked for more than|WIFI_FLR_REFUSED_OR_FAILED', re.I) def run(argv, timeout=30): @@ -131,6 +133,32 @@ def wifi_dns(iface): raise RuntimeError('Wi-Fi-bound upstream DNS lookup failed') +def profile_security(profile_uuid): + key = run(['nmcli', '-g', '802-11-wireless-security.key-mgmt', + 'connection', 'show', 'uuid', profile_uuid]) + require(key.returncode == 0, 'Profile security unavailable') + mode = key.stdout.strip() + # No wireless-security setting is open; key-mgmt "none" is static WEP. + require(mode in ('', 'wpa-psk'), 'Unsupported profile authentication') + return 'WPA2' if mode == 'wpa-psk' else 'OPEN' + + +@contextmanager +def profile_band(profile_uuid, band): + original = run(['nmcli', '-g', '802-11-wireless.band', 'connection', + 'show', 'uuid', profile_uuid]) + require(original.returncode == 0, 'Original profile band unavailable') + try: + require(run(['nmcli', 'connection', 'modify', 'uuid', profile_uuid, + '802-11-wireless.band', band]).returncode == 0, 'Band selection failed') + yield + finally: + # Also restore after activation errors, timeouts or interrupted testing. + require(run(['nmcli', 'connection', 'modify', 'uuid', profile_uuid, + '802-11-wireless.band', original.stdout.strip()]).returncode == 0, + 'Profile band restoration failed') + + def main(): parser = argparse.ArgumentParser() parser.add_argument('--expected-release', required=True) @@ -156,6 +184,7 @@ def main(): profile_uuid = profile.stdout.strip() require(profile.returncode == 0 and re.fullmatch(r'[0-9a-f-]{36}', profile_uuid), 'Missing or ambiguous profile') + result['authentication'] = profile_security(profile_uuid) ssid_query = run(['nmcli', '-g', '802-11-wireless.ssid', 'connection', 'show', 'uuid', profile_uuid]) ssid = ssid_query.stdout.strip() @@ -190,45 +219,44 @@ def main(): if not args.connect: result['result'] = 'WIFI_SCAN_OBSERVED' else: - require(run(['nmcli', 'connection', 'modify', 'uuid', profile_uuid, - '802-11-wireless.band', args.band]).returncode == 0, 'Band selection failed') - connection = run(['nmcli', '--wait', '60', 'connection', 'up', 'uuid', - profile_uuid, 'ifname', iface], 65) - result['activation_exit'] = connection.returncode - link = run(['iw', 'dev', iface, 'link']).stdout - freq = re.search(r'freq: (\d+)', link) - result['frequency_mhz'] = int(freq[1]) if freq else None - require(freq and ((int(freq[1]) < 3000) == (args.band == 'bg')), 'Unexpected associated band') - require(connection.returncode == 0, 'Saved profile activation failed') - for sample in range(3): - addresses = json.loads(run(['ip', '-j', '-4', 'address', 'show', 'dev', iface]).stdout) - require(any(a.get('scope') == 'global' for link in addresses - for a in link.get('addr_info', [])), 'No DHCP IPv4 address') - routes = json.loads(run(['ip', '-j', '-4', 'route', 'show', 'default', 'dev', iface]).stdout) - require(routes and routes[0].get('gateway'), 'No Wi-Fi default gateway') - ping = run(['ping', '-I', iface, '-c', '3', '-W', '3', routes[0]['gateway']], 15) - require(ping.returncode == 0 and re.search(r'\b0(?:\.0+)?% packet loss', ping.stdout) - and re.search(r'\b3 received\b', ping.stdout), 'Gateway packet loss') - dns_address = wifi_dns(iface) - payload = out / ('https-' + str(sample) + '.html') - transfer = run(['curl', '--fail', '--silent', '--show-error', '--interface', iface, - '--noproxy', '*', '--resolve', 'example.com:443:' + dns_address, - '--max-time', '20', '--output', str(payload), 'https://example.com/'], 25) - require(transfer.returncode == 0 and payload.is_file(), 'Wi-Fi HTTPS transfer failed') - body = payload.read_bytes() - require(b'Example Domain' in body and len(body) > 100, 'Unexpected HTTPS response') - kernel_ok() - result['samples'].append({'gateway_received': 3, 'gateway_loss_percent': 0, - 'dns': True, 'tls_verified': True, 'download_bytes': len(body), - 'download_sha256': hashlib.sha256(body).hexdigest()}) - if sample != 2: - time.sleep(10) - if args.desktop == 'kde': - require(all(run(['pgrep', '-x', name]).returncode == 0 - for name in ('plasmashell', 'kwin_x11')), 'KDE not running') - result['steps'].append('KDE_RUNNING_NO_KERNEL_FAULT') - result['steps'].extend(['WPA2_SAVED_PROFILE_ACTIVATED', 'DHCP_ROUTE_OBSERVED', - 'GATEWAY_ROUNDTRIPS_VERIFIED', 'WIFI_BOUND_DNS_TLS_DOWNLOAD_VERIFIED']) + with profile_band(profile_uuid, args.band): + connection = run(['nmcli', '--wait', '60', 'connection', 'up', 'uuid', + profile_uuid, 'ifname', iface], 65) + result['activation_exit'] = connection.returncode + link = run(['iw', 'dev', iface, 'link']).stdout + freq = re.search(r'freq: (\d+)', link) + result['frequency_mhz'] = int(freq[1]) if freq else None + require(freq and ((int(freq[1]) < 3000) == (args.band == 'bg')), 'Unexpected associated band') + require(connection.returncode == 0, 'Saved profile activation failed') + for sample in range(3): + addresses = json.loads(run(['ip', '-j', '-4', 'address', 'show', 'dev', iface]).stdout) + require(any(a.get('scope') == 'global' for link in addresses + for a in link.get('addr_info', [])), 'No DHCP IPv4 address') + routes = json.loads(run(['ip', '-j', '-4', 'route', 'show', 'default', 'dev', iface]).stdout) + require(routes and routes[0].get('gateway'), 'No Wi-Fi default gateway') + ping = run(['ping', '-I', iface, '-c', '3', '-W', '3', routes[0]['gateway']], 15) + require(ping.returncode == 0 and re.search(r'\b0(?:\.0+)?% packet loss', ping.stdout) + and re.search(r'\b3 received\b', ping.stdout), 'Gateway packet loss') + dns_address = wifi_dns(iface) + payload = out / ('https-' + str(sample) + '.html') + transfer = run(['curl', '--fail', '--silent', '--show-error', '--interface', iface, + '--noproxy', '*', '--resolve', 'example.com:443:' + dns_address, + '--max-time', '20', '--output', str(payload), 'https://example.com/'], 25) + require(transfer.returncode == 0 and payload.is_file(), 'Wi-Fi HTTPS transfer failed') + body = payload.read_bytes() + require(b'Example Domain' in body and len(body) > 100, 'Unexpected HTTPS response') + kernel_ok() + result['samples'].append({'gateway_received': 3, 'gateway_loss_percent': 0, + 'dns': True, 'tls_verified': True, 'download_bytes': len(body), + 'download_sha256': hashlib.sha256(body).hexdigest()}) + if sample != 2: + time.sleep(10) + if args.desktop == 'kde': + require(all(run(['pgrep', '-x', name]).returncode == 0 + for name in ('plasmashell', 'kwin_x11')), 'KDE not running') + result['steps'].append('KDE_RUNNING_NO_KERNEL_FAULT') + result['steps'].extend([result['authentication'] + '_SAVED_PROFILE_ACTIVATED', 'DHCP_ROUTE_OBSERVED', + 'GATEWAY_ROUNDTRIPS_VERIFIED', 'WIFI_BOUND_DNS_TLS_DOWNLOAD_VERIFIED']) result['result'] = 'WIFI_PHYSICAL_NETWORK_E2E_PASS' except subprocess.TimeoutExpired: result['result'] = 'WIFI_NETWORK_E2E_FAILED' diff --git a/tools/testing/selftests/drivers/net/mt7932_e2e_test.py b/tools/testing/selftests/drivers/net/mt7932_e2e_test.py index f58a8eeb9bb71..22b45b154f365 100644 --- a/tools/testing/selftests/drivers/net/mt7932_e2e_test.py +++ b/tools/testing/selftests/drivers/net/mt7932_e2e_test.py @@ -127,5 +127,60 @@ def test_invalid_interface_fails_without_fallback(self): e2e.dns_query('no-such-neo-if', '127.0.0.1', timeout=0.03) +class QualificationTest(unittest.TestCase): + def test_authentication_comes_from_profile(self): + for key, expected in (('', 'OPEN'), ('wpa-psk', 'WPA2')): + with self.subTest(key=key), mock.patch.object( + e2e, 'run', return_value=subprocess.CompletedProcess([], 0, key + '\n')): + self.assertEqual(e2e.profile_security('test-profile'), expected) + with mock.patch.object(e2e, 'run', return_value=subprocess.CompletedProcess([], 0, 'sae\n')): + with self.assertRaisesRegex(RuntimeError, 'Unsupported profile authentication'): + e2e.profile_security('test-profile') + + def test_restore_band_after_success_or_timeout(self): + for original in ('', 'bg', 'a'): + for timeout in (False, True): + with self.subTest(original=original, timeout=timeout): + outputs = [subprocess.CompletedProcess([], 0, original + '\n'), + subprocess.CompletedProcess([], 0, ''), + subprocess.CompletedProcess([], 0, '')] + with mock.patch.object(e2e, 'run', side_effect=outputs) as run: + try: + with e2e.profile_band('test-profile', 'a'): + if timeout: + raise subprocess.TimeoutExpired('activation', 60) + except subprocess.TimeoutExpired: + self.assertTrue(timeout) + self.assertEqual(run.call_args_list[-1], mock.call( + ['nmcli', 'connection', 'modify', 'uuid', 'test-profile', + '802-11-wireless.band', original])) + + def test_restore_band_even_if_selection_times_out(self): + outputs = [subprocess.CompletedProcess([], 0, 'bg\n'), + subprocess.TimeoutExpired('modify', 30), + subprocess.CompletedProcess([], 0, '')] + with mock.patch.object(e2e, 'run', side_effect=outputs) as run: + with self.assertRaises(subprocess.TimeoutExpired): + with e2e.profile_band('test-profile', 'a'): + self.fail('Timed out selection must not activate the profile') + self.assertEqual(run.call_args_list[-1].args[0][-1], 'bg') + + def test_restoration_failure_prevents_a_pass(self): + outputs = [subprocess.CompletedProcess([], 0, 'bg\n'), + subprocess.CompletedProcess([], 0, ''), + subprocess.CompletedProcess([], 1, '')] + with mock.patch.object(e2e, 'run', side_effect=outputs): + with self.assertRaisesRegex(RuntimeError, 'Profile band restoration failed'): + with e2e.profile_band('test-profile', 'a'): + pass + + def test_new_fault_markers_are_terminal(self): + for message in ('WARNING: suspicious RCU usage', 'UBSAN: out-of-bounds', + 'hung_task', 'task worker blocked for more than 120 seconds', + 'WIFI_FLR_REFUSED_OR_FAILED error=-16'): + with self.subTest(message=message): + self.assertIsNotNone(e2e.fault.search(message)) + + if __name__ == '__main__': unittest.main() From 8d976acc1a05da6a482caa0c5892d7879b134692 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:37:34 -0700 Subject: [PATCH 0151/2680] wifi: mt7932: pin module before publishing DMA ownership A live, pinned Bluetooth sibling makes Wi-Fi FLR inadmissible. Refuse normal Wi-Fi unload before it can enter a removal wait holding the function lock and obstruct reboot. Acquire the module hold before the first ring-address write, retain it for live or failed-reset epochs, and release it only after checked FLR while probe/remove framework ownership still protects the callback. The existing exact Neo supplier/host gate remains ahead of this path; off-Neo probing and unload behavior are unchanged. Forced module removal is not supported. Round-2 R2. Non-Neo runtime behavior remains unchanged. --- drivers/net/wireless/mediatek/mt7932/Kconfig | 7 ++++--- drivers/net/wireless/mediatek/mt7932/mt7932.c | 21 +++++++++++++++++-- drivers/net/wireless/mediatek/mt7932/mt7932.h | 2 +- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index f5a22557f48b6..bed6279895a48 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -14,9 +14,10 @@ config MT7932_FULLMAC Firmware, runtime DMA and calibration negotiation start automatically; cfg80211 and userspace select the country and network. No test boot parameters are required for ordinary station operation. - Module removal stops host users and performs a checked PCI function - reset before releasing DMA storage. A failed reset blocks removal, - retaining resources until an explicit retry succeeds or system reset. + Once DMA pointers are published the module is pinned until hardware + reset. Normal unload is refused, including with active Bluetooth. + A failed probe reset retains the binding and DMA; a successful checked + reset retires ownership before releasing probe resources. Other MediaTek devices and other machines are not matched. config MT7932_RF_KUNIT_TEST diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 084b006a812a9..c5ced3f4c1083 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -530,6 +530,14 @@ static int mt_dma_setup(struct mt7932 *m) m->aux = mt_alloc(m, 0x320, &m->aux_dma); if (!m->ipc || !m->aux) return -ENOMEM; + /* Pin before the first endpoint-visible DMA pointer. A live Bluetooth + * sibling forbids FLR, so normal module unload must never enter remove + * and hold a device lock while waiting for an impossible reset. + */ + if (!try_module_get(THIS_MODULE)) + return -ENODEV; + m->module_pinned = true; + WRITE_ONCE(m->dma_owned, true); for (i = 0; i < ARRAY_SIZE(m->tx); i++) { struct mt7932_ring *q = &m->tx[i]; @@ -562,7 +570,6 @@ static int mt_dma_setup(struct mt7932 *m) mt_rmw(m, W + 0x2b0, 0, BIT(6)); /* All endpoint-visible storage and interrupt state now have an owner. */ dma_wmb(); - WRITE_ONCE(m->dma_owned, true); pci_set_master(m->pdev); WRITE_ONCE(m->running, true); mt_rmw(m, W + 0x208, 0, BIT(0) | BIT(2)); @@ -1084,6 +1091,10 @@ static int mt_reset_retained(struct mt7932 **epoch) if (ret) return ret; old->dma_owned = false; + if (old->module_pinned) { + old->module_pinned = false; + module_put(THIS_MODULE); + } dev_info(&pdev->dev, "RESET_TEST: function reset complete; retaining old DMA\n"); fresh = devm_kzalloc(&pdev->dev, sizeof(*fresh), GFP_KERNEL); if (!fresh) @@ -1150,8 +1161,14 @@ static int mt_retire_dma(struct mt7932 *m) ret = mt_dma_stop(m); if (!ret) ret = mt_function_reset(m); - if (!ret) + if (!ret) { m->dma_owned = false; + /* Probe/remove's framework reference still owns this callback. */ + if (m->module_pinned) { + m->module_pinned = false; + module_put(THIS_MODULE); + } + } return ret; } diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 0bbefd9620cf8..ca77ba30b037d 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -77,7 +77,7 @@ struct mt7932 { bool running; bool stopping; bool irq_requested, vectors_allocated, wiphy_registered; - bool dma_owned; + bool dma_owned, module_pinned; struct completion reset_retry; int removal_error; bool ram_config; From 80fb70cebfc37bf9f5a93312b33f04e8861748e6 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:37:42 -0700 Subject: [PATCH 0152/2680] PCI: apple: admit lifecycle guards before publishing Neo children Acquire bounded PM/kexec admission before pci_host_common_init can create children. Publish guarded state for device prepare; remove every child before releasing temporary vetoes on pre-retained failure. Prime performs no late dependency admission and post-child errors cannot defer probing. A supplier-owned delayed attachment retries pre-child contention even when no unrelated driver binds. Only renewed contention rearms it. The matched host reference and supplier stay alive until reset; cancellation never waits while holding device locks. Recheck consumed attempts and shutdown before host setup, and keep irreversible ownership guarded until reset. Round-2 R1. All admission, unwind and retry paths require the exact Neo supplier and reciprocal host. Non-Neo behavior remains unchanged. --- .../pci/controller/pcie-apple-piodma-diag.c | 144 ++++++++++++++---- .../pci/controller/pcie-apple-piodma-diag.h | 13 ++ drivers/pci/controller/pcie-apple.c | 27 +++- 3 files changed, 153 insertions(+), 31 deletions(-) diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index 4caf7a37a5b06..d5f757a66a218 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -8,8 +8,8 @@ * * This built-in driver deliberately retains its coherent arena and device * power reference until a coordinator-controlled full hardware reset. There - * is no unbind interface or retry. Runtime guards refuse suspend and kexec - * while the arena is held. Native ECAM is only + * is no unbind interface or hardware retry. Runtime guards cover host admission + * and retained hardware ownership. Native ECAM is only * opened by the host after a successful bootstrap and typed read checks. */ #include @@ -35,6 +35,7 @@ #include #include #include +#include #include "pcie-apple-piodma-diag.h" @@ -98,9 +99,12 @@ struct apple_piodma_diag { u64 pointer_prefix; u8 secondary_bus; bool retained; + bool guarded; bool radio_admitted; struct notifier_block pm_notifier; struct kexec_blocker kexec_blocker; + struct delayed_work admission_retry; + struct device *retry_host; }; /* A successful supplier probe owns its unpublished arena until full reset. */ @@ -116,7 +120,7 @@ static bool apple_piodma_trylock_sleep(unsigned int *flags) * Host probe holds device_lock(), while system sleep holds the transition * mutex before waiting for probes and taking device locks. Never wait * for that mutex here: if sleep won, defer without publishing any DMA. - * If prime won, publish the veto before releasing the transition mutex. + * If admission won, publish its veto before releasing the mutex. */ *flags = current->flags; current->flags |= PF_NOFREEZE; @@ -172,8 +176,8 @@ static int apple_piodma_pm_prepare(struct device *dev) { struct apple_piodma_diag *diag = dev_get_drvdata(dev); - /* Acquire the prime's retained arena publication before permitting sleep. */ - if (!diag || !smp_load_acquire(&diag->retained)) + /* Acquire admission before either PCI children or DMA pointers escape. */ + if (!diag || !smp_load_acquire(&diag->guarded)) return 0; /* * A /dev/snapshot file may have sent its PREPARE notifier before this @@ -573,6 +577,7 @@ static void apple_piodma_put_supplier(void *data) int apple_piodma_bootstrap_get(struct device *host, struct device **supplier) { struct device_node *node, *host_node; + struct apple_piodma_diag *diag; struct platform_device *pdev; struct device_link *link; bool correct_host; @@ -597,10 +602,21 @@ int apple_piodma_bootstrap_get(struct device *host, struct device **supplier) return -EINVAL; if (!pdev) return -EPROBE_DEFER; - if (!device_is_bound(&pdev->dev) || !platform_get_drvdata(pdev)) { + diag = platform_get_drvdata(pdev); + if (!device_is_bound(&pdev->dev) || !diag) { put_device(&pdev->dev); return -EPROBE_DEFER; } + /* Reject late software attachment before any Neo host hardware setup. */ + if (system_state > SYSTEM_RUNNING) { + put_device(&pdev->dev); + return -ESHUTDOWN; + } + /* Host probe holds its device lock; never rebuild a consumed hierarchy. */ + if (atomic_read(&diag->attempted)) { + put_device(&pdev->dev); + return -EALREADY; + } link = device_link_add(host, &pdev->dev, DL_FLAG_AUTOREMOVE_CONSUMER); if (!link) { put_device(&pdev->dev); @@ -614,25 +630,102 @@ int apple_piodma_bootstrap_get(struct device *host, struct device **supplier) } EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_get); -int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root) +static void apple_piodma_retry_admission(struct work_struct *work) +{ + struct apple_piodma_diag *diag = + container_of(to_delayed_work(work), struct apple_piodma_diag, + admission_retry); + /* Acquire the matched host's permanent reference from the scheduler. */ + struct device *host = smp_load_acquire(&diag->retry_host); + int ret; + + if (system_state > SYSTEM_RUNNING || !host || + !device_is_registered(host) || atomic_read(&diag->attempted) || + READ_ONCE(diag->guarded)) + return; + /* The core rechecks registration and binding under the host device lock. */ + ret = device_attach(host); + if (ret < 0) + dev_dbg(diag->dev, "deferred host attachment returned %d\n", ret); + /* Only fresh pre-child guard contention can schedule another attempt. */ +} + +static void apple_piodma_schedule_admission(struct apple_piodma_diag *diag, + struct device *host) +{ + struct device *previous; + + if (system_state > SYSTEM_RUNNING || !device_is_registered(host)) + return; + /* Publish a boot-lifetime reference; this supplier has one matched host. */ + previous = cmpxchg(&diag->retry_host, NULL, get_device(host)); + if (previous) { + put_device(host); + if (WARN_ON_ONCE(previous != host)) + return; + } + mod_delayed_work(system_unbound_wq, &diag->admission_retry, HZ); +} + +int apple_piodma_bootstrap_guard_begin(struct device *supplier, struct device *host) { struct apple_piodma_diag *diag = dev_get_drvdata(supplier); unsigned int sleep_flags; int ret; - /* Contention must not consume the one permitted hardware attempt. */ + /* Reboot blocks core probing too; reject an attachment already in flight. */ + if (system_state > SYSTEM_RUNNING) + return -ESHUTDOWN; + /* A worker may have waited behind a host probe that consumed its attempt. */ + if (atomic_read(&diag->attempted) || READ_ONCE(diag->guarded)) + return -EALREADY; + /* This is the last dependency admission before PCI children exist. */ ret = apple_piodma_lock_transition(diag, &sleep_flags); - if (ret) + if (ret) { + if (ret == -EPROBE_DEFER) + apple_piodma_schedule_admission(diag, host); return ret; - /* One hardware attempt, including permanent preflight failures. */ - if (atomic_cmpxchg(&diag->attempted, 0, 1)) { - ret = -EALREADY; - goto err_kexec; } + /* Do not wait for a worker that may be waiting on this host device lock. */ + cancel_delayed_work(&diag->admission_retry); diag->pm_notifier.notifier_call = apple_piodma_pm_notify; ret = register_pm_notifier(&diag->pm_notifier); - if (ret) - goto err_kexec; + if (ret) { + kexec_unblock(&diag->kexec_blocker); + } else { + /* Publish both vetoes before permitting system sleep to proceed. */ + smp_store_release(&diag->guarded, true); + } + unlock_system_sleep(sleep_flags); + return ret; +} +EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_guard_begin); + +void apple_piodma_bootstrap_guard_end(struct device *supplier) +{ + struct apple_piodma_diag *diag = dev_get_drvdata(supplier); + + /* Post-publication ownership is irreversible until hardware reset. */ + if (!READ_ONCE(diag->guarded) || smp_load_acquire(&diag->retained)) + return; + /* The host has removed every child before permitting sleep again. */ + smp_store_release(&diag->guarded, false); + unregister_pm_notifier(&diag->pm_notifier); + kexec_unblock(&diag->kexec_blocker); +} +EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_guard_end); + +int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root) +{ + struct apple_piodma_diag *diag = dev_get_drvdata(supplier); + int ret; + + /* Admission was completed before pci_host_common_init published children. */ + if (!smp_load_acquire(&diag->guarded)) + return -EINVAL; + /* One hardware attempt, including permanent preflight failures. */ + if (atomic_cmpxchg(&diag->attempted, 0, 1)) + return -EALREADY; ret = apple_piodma_diag_root(diag, root); if (!ret) ret = apple_piodma_diag_idle(diag); @@ -640,23 +733,12 @@ int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root) ret = apple_piodma_diag_initialize(diag); if (!ret) ret = apple_piodma_diag_submit(diag, 0); - if (ret) { + if (ret) dev_err(supplier, "bootstrap failure=%d; %s; no retry\n", ret, diag->retained ? "retained ownership" : "no hardware publication"); - if (!diag->retained) { - unregister_pm_notifier(&diag->pm_notifier); - goto err_kexec; - } - } else { + else dev_info(supplier, "BOOTSTRAP_VALIDATED word=%#x; one retained slot\n", diag->slots[0].result); - } - unlock_system_sleep(sleep_flags); - return ret; - -err_kexec: - kexec_unblock(&diag->kexec_blocker); - unlock_system_sleep(sleep_flags); return ret; } EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_prime); @@ -719,6 +801,7 @@ static int apple_piodma_diag_probe(struct platform_device *pdev) if (!diag) return -ENOMEM; diag->dev = dev; + INIT_DELAYED_WORK(&diag->admission_retry, apple_piodma_retry_admission); diag->kexec_blocker.reason = "Neo radio bootstrap retains DMA memory until full hardware reset"; for (i = 0; i < PIODMA_REQUEST_COUNT; i++) @@ -765,6 +848,11 @@ static void apple_piodma_diag_shutdown(struct platform_device *pdev) { struct apple_piodma_diag *diag = platform_get_drvdata(pdev); + /* + * Shutdown holds supplier/parent locks: never join a host attachment + * here. Admission rejects shutdown, and the supplier lifetime is retained. + */ + cancel_delayed_work(&diag->admission_retry); if (READ_ONCE(diag->retained)) dev_warn(&pdev->dev, "diagnostic arena retained; require full hardware reset\n"); } diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.h b/drivers/pci/controller/pcie-apple-piodma-diag.h index 907981d97b1d3..c8d29335ed0cb 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.h +++ b/drivers/pci/controller/pcie-apple-piodma-diag.h @@ -12,6 +12,9 @@ struct pci_dev; #if IS_ENABLED(CONFIG_PCIE_APPLE_PIODMA_DIAG) bool apple_piodma_bootstrap_enabled(void); int apple_piodma_bootstrap_get(struct device *host, struct device **supplier); +int apple_piodma_bootstrap_guard_begin(struct device *supplier, + struct device *host); +void apple_piodma_bootstrap_guard_end(struct device *supplier); int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root); void apple_piodma_bootstrap_admitted(struct device *supplier); bool apple_piodma_bootstrap_retained(struct device *supplier); @@ -26,6 +29,16 @@ static inline int apple_piodma_bootstrap_get(struct device *host, struct device return -ENODEV; } +static inline int apple_piodma_bootstrap_guard_begin(struct device *supplier, + struct device *host) +{ + return -ENODEV; +} + +static inline void apple_piodma_bootstrap_guard_end(struct device *supplier) +{ +} + static inline int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root) { return -ENODEV; diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 91a905263be70..74b3496ed812a 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2577,11 +2577,26 @@ static int apple_pcie_probe(struct platform_device *pdev) if (ret) return ret; + /* Only the retained Neo supplier needs admission before child publication. */ + if (pcie->piodma_supplier) { + ret = apple_piodma_bootstrap_guard_begin(pcie->piodma_supplier, dev); + if (ret) { + apple_pcie_cleanup(pcie); + return dev_err_probe(dev, ret, "Neo lifecycle admission failed\n"); + } + } + ret = pci_host_common_init(pdev, bridge, &apple_pcie_cfg_ecam_ops); - if (ret) + if (ret) { apple_pcie_cleanup(pcie); - if (ret) + if (pcie->piodma_supplier) { + apple_piodma_bootstrap_guard_end(pcie->piodma_supplier); + /* Dependency deferral is legal only before publishing children. */ + if (ret == -EPROBE_DEFER) + ret = -EIO; + } return ret; + } /* * Port mappings are allocated by the ECAM init callback. Register cleanup @@ -2592,6 +2607,8 @@ static int apple_pcie_probe(struct platform_device *pdev) /* Drivers must release their IRQs before the domains disappear. */ pci_host_common_remove(pdev); apple_pcie_cleanup(pcie); + if (pcie->piodma_supplier) + apple_piodma_bootstrap_guard_end(pcie->piodma_supplier); return ret; } @@ -2600,13 +2617,17 @@ static int apple_pcie_probe(struct platform_device *pdev) /* Retained bootstrap ownership forbids host module teardown. */ if (!try_module_get(THIS_MODULE)) { pci_host_common_remove(pdev); + apple_piodma_bootstrap_guard_end(pcie->piodma_supplier); return -ENODEV; } ret = apple_pcie_neo_enumerate(bridge); if (ret && !apple_piodma_bootstrap_retained(pcie->piodma_supplier)) { - /* No DMA pointer escaped: unwind and allow dependency retry. */ + /* No DMA pointer escaped: remove children, then release vetoes. */ pci_host_common_remove(pdev); module_put(THIS_MODULE); + apple_piodma_bootstrap_guard_end(pcie->piodma_supplier); + if (ret == -EPROBE_DEFER) + ret = -EIO; return dev_err_probe(dev, ret, "Neo bootstrap preflight failed\n"); } if (ret) From 65e7d0904a90235304a7515e9799f5a726d62a3d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:38:14 -0700 Subject: [PATCH 0153/2680] wifi: mt7932: rate-limit recoverable policy retry diagnostics Repeated interface admission or country hints may safely retry a missing policy before submission. Bound the named loader failure, regulatory refusal and accompanying request logs so ordinary userspace retry loops cannot flood the kernel log. request_firmware_direct already suppresses its core warning; the first driver diagnostics remain visible. Retry and admission semantics are unchanged. These are existing Neo-only driver paths; off-Neo behavior is unchanged. Round-2 R3. Non-Neo runtime behavior remains unchanged. --- drivers/net/wireless/mediatek/mt7932/frontend.c | 2 +- drivers/net/wireless/mediatek/mt7932/link.c | 2 +- drivers/net/wireless/mediatek/mt7932/mt7932.c | 2 +- drivers/net/wireless/mediatek/mt7932/startup.c | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 9104f497e2f4d..f303afe2246f4 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -327,7 +327,7 @@ int mt_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) if (request->wdev != &m->wdev) return -ENODEV; mt_retry_missing_policy(m); - dev_info(&m->pdev->dev, "SCAN_REQUEST: channels=%u ssids=%d ies=%zu flags=%08x duration=%u/%u\n", + dev_info_ratelimited(&m->pdev->dev, "SCAN_REQUEST: channels=%u ssids=%d ies=%zu flags=%08x duration=%u/%u\n", request->n_channels, request->n_ssids, request->ie_len, request->flags, request->duration, request->duration_mandatory); if (!request->n_channels || request->n_channels > 17 || request->ie_len > 600 || diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 26eb74b9d1d67..d8651b626f46e 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -589,7 +589,7 @@ int mt_connect(struct wiphy *wiphy, struct net_device *netdev, mt_retry_missing_policy(m); if (!mt_rf_allowed(m) || !m->bss_active || netdev != m->netdev) return -EAGAIN; - dev_info(&m->pdev->dev, "CONNECT_REQUEST: WPA=%u pairwise-count=%d group=%08x AKM-count=%d PMK-present=%u MFP=%u IE-bytes=%zu\n", + dev_info_ratelimited(&m->pdev->dev, "CONNECT_REQUEST: WPA=%u pairwise-count=%d group=%08x AKM-count=%d PMK-present=%u MFP=%u IE-bytes=%zu\n", params->crypto.wpa_versions, params->crypto.n_ciphers_pairwise, params->crypto.cipher_group, params->crypto.n_akm_suites, !!params->crypto.psk, params->mfp, params->ie_len); if (!params->ssid_len || params->ssid_len > 32) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index c5ced3f4c1083..ac8ac763fbf79 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -64,7 +64,7 @@ int mt_request_input(struct mt7932 *m, const struct firmware **fw, const char *n int ret = request_firmware_direct(fw, name, &m->pdev->dev); if (ret) - dev_err(&m->pdev->dev, "local input %s unavailable: %d\n", name, ret); + dev_err_ratelimited(&m->pdev->dev, "local input %s unavailable: %d\n", name, ret); return ret; } diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 8f8a2a7c6da84..c6035f5cb21c1 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -299,7 +299,7 @@ static void mt_startup_work(struct work_struct *work) if (!ret) { ret = mt7932_policy_parse(&policy, file->data, file->size, reg.domain); if (ret) - dev_err(&m->pdev->dev, "local input %s has invalid policy: %d\n", path, ret); + dev_err_ratelimited(&m->pdev->dev, "local input %s has invalid policy: %d\n", path, ret); } mutex_lock(&m->command_mutex); if (READ_ONCE(m->stopping) || generation != READ_ONCE(m->reg_generation)) @@ -325,7 +325,7 @@ static void mt_startup_work(struct work_struct *work) if (!ret && generation == m->reg_generation) WRITE_ONCE(m->reg_pending, false); if (ret) - dev_warn(&m->pdev->dev, "REGULATORY_BLOCKED: %c%c generation=%u error=%d recovery-required=%u\n", + dev_warn_ratelimited(&m->pdev->dev, "REGULATORY_BLOCKED: %c%c generation=%u error=%d recovery-required=%u\n", reg.alpha2[0], reg.alpha2[1], generation, ret, m->policy_failed || m->link_failed); else @@ -419,7 +419,7 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request if (READ_ONCE(m->interface_registered)) schedule_work(&m->startup_work); spin_unlock_irqrestore(&m->response_lock, irqflags); - dev_info(&m->pdev->dev, "REGULATORY_REQUEST: %c%c initiator=%u generation=%u\n", + dev_info_ratelimited(&m->pdev->dev, "REGULATORY_REQUEST: %c%c initiator=%u generation=%u\n", reg.alpha2[0], reg.alpha2[1], request->initiator, m->reg_generation); } From 6ab25b44707a8788b72f6b517afd1a322fc564fb Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:38:14 -0700 Subject: [PATCH 0154/2680] apple: bound Neo sibling admission and silence unused hotplug events Retry Wi-Fi's Bluetooth sibling trylock for at most 480 ms without ever blocking across function locks or retrying hardware resets. Once acquired, bound or mastering Bluetooth remains an immediate FLR veto. Clear HPIE/CCIE/DLLSCE before withholding retained-host hotplug service, including forced-native handling. Round-2 S3. Radio changes follow exact Neo admission; slot writes require the Neo-only removal flag. Non-Neo behavior remains unchanged. --- drivers/net/wireless/mediatek/mt7932/reset.c | 13 ++++++++++++- drivers/pci/pcie/portdrv.c | 8 ++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/reset.c b/drivers/net/wireless/mediatek/mt7932/reset.c index 3c14e17bb4b21..488a3ba5414b9 100644 --- a/drivers/net/wireless/mediatek/mt7932/reset.c +++ b/drivers/net/wireless/mediatek/mt7932/reset.c @@ -117,6 +117,7 @@ int mt_function_reset(struct mt7932 *m) { struct pci_dev *peer; u16 command; + unsigned int attempt; int ret; device_lock_assert(&m->pdev->dev); @@ -128,7 +129,17 @@ int mt_function_reset(struct mt7932 *m) * Trylock avoids inversion with a concurrent sibling probe/remove; hold * it across FLR to prevent Bluetooth starting after the admission check. */ - if (!device_trylock(&peer->dev)) { + /* Absorb short sibling probe/remove contention without ever blocking + * on its lock while owning Wi-Fi's device lock. Live peers still fail + * immediately once their state can be inspected. + */ + for (attempt = 0; attempt < 25; attempt++) { + if (device_trylock(&peer->dev)) + break; + if (attempt != 24) + msleep(20); + } + if (attempt == 25) { ret = -EBUSY; goto put; } diff --git a/drivers/pci/pcie/portdrv.c b/drivers/pci/pcie/portdrv.c index 57c5163d779cc..1ca9fdb31a618 100644 --- a/drivers/pci/pcie/portdrv.c +++ b/drivers/pci/pcie/portdrv.c @@ -220,8 +220,12 @@ static int get_port_device_capability(struct pci_dev *dev) struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); int services = 0; - /* A retained hierarchy must not be removed, even with native forcing. */ - if (!host->no_user_remove && dev->is_pciehp && + /* Retained hosts have no HP service to consume firmware-enabled events. */ + if (host->no_user_remove) { + pcie_capability_clear_word(dev, PCI_EXP_SLTCTL, + PCI_EXP_SLTCTL_CCIE | PCI_EXP_SLTCTL_HPIE | + PCI_EXP_SLTCTL_DLLSCE); + } else if (dev->is_pciehp && (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT || pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM) && (pcie_ports_native || host->native_pcie_hotplug)) { From 77cea4342b85271cf3ff231088143b798fa2b9e6 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:41:30 -0700 Subject: [PATCH 0155/2680] wifi: mt7932: distinguish input validation from command and profile failures Do not blame config-original.bin or OCA2 formatting for timeouts in submitted firmware commands. Report explicit file-format, host allocation, command/completion, and capability-profile phases, preserving exact missing-section diagnostics. Unsupported board/calibration capability predicates now identify their observed non-identity metadata before refusing admission. Firmware download phases likewise separate local parser rejection from transport/start failures. All changed calls remain in the existing supplier-gated Neo driver; off-Neo paths are unchanged. Round-2 R4. Non-Neo runtime behavior remains unchanged. --- .../mediatek/mt7932/calibration_transport.c | 16 +++++++-- drivers/net/wireless/mediatek/mt7932/link.c | 7 ++-- drivers/net/wireless/mediatek/mt7932/mt7932.c | 35 +++++++++++++++---- .../net/wireless/mediatek/mt7932/startup.c | 13 +++++-- 4 files changed, 57 insertions(+), 14 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c index 32c6837615fa7..0acc017f49156 100644 --- a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c +++ b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c @@ -50,25 +50,32 @@ void mt_cal_input_error(struct mt7932 *m, int error, unsigned int tag, const cha if (error == -ENOENT) dev_err(&m->pdev->dev, "local input mediatek/mt7932/oca2.bin missing section 0x%04x during %s: %d\n", tag, stage, error); + else if (error == -EOPNOTSUPP) + dev_err(&m->pdev->dev, "CAL_PROFILE_UNSUPPORTED: phase=%s section=0x%04x error=%d\n", + stage, tag, error); else if (tag) dev_err(&m->pdev->dev, "local input mediatek/mt7932/oca2.bin section 0x%04x rejected during %s: %d\n", tag, stage, error); else - dev_err(&m->pdev->dev, "local input mediatek/mt7932/oca2.bin container/profile rejected during %s: %d\n", + dev_err(&m->pdev->dev, "local input mediatek/mt7932/oca2.bin container format rejected during %s: %d\n", stage, error); } int mt_calibration_gate(struct mt7932 *m) { const struct firmware *wcal, *oca; + const char *phase = "own input validation"; struct mt7932_cal_piece pieces[14]; u8 body[1028], request[16]; unsigned long flags; unsigned int at, count, group, input_tag; int ret, planned; - if (!m->six_ghz_valid || m->smart_version != 12 || !m->preload_version) + if (!m->six_ghz_valid || m->smart_version != 12 || !m->preload_version) { + dev_err(&m->pdev->dev, "CAL_CAPABILITY_MISMATCH: six-ghz-valid=%u smart-version=%u preload-version=%u\n", + m->six_ghz_valid, m->smart_version, m->preload_version); return -EOPNOTSUPP; + } ret = mt_request_input(m, &wcal, "mediatek/mt7932/wcal.bin"); if (ret) return ret; @@ -100,6 +107,7 @@ int mt_calibration_gate(struct mt7932 *m) goto out; } } + phase = "WCAL command/reply"; body[0] = 3; body[1] = 0; put_unaligned_le16(wcal->size, body + 2); @@ -118,6 +126,7 @@ int mt_calibration_gate(struct mt7932 *m) goto out; } dev_info(&m->pdev->dev, "OWN_WCAL_ACCEPTED: %zu bytes\n", wcal->size); + phase = "power-on D6 command/completion"; for (at = 0, group = 0; at < planned; at += count, group++) { count = pieces[at].fragment >> 4; if (!count || count > planned - at) { @@ -131,6 +140,7 @@ int mt_calibration_gate(struct mt7932 *m) group, pieces[at].type, pieces[at].parameter, count); } dev_info(&m->pdev->dev, "POWER_ON_CAL_COMPLETE: %u groups, %d fragments\n", group, planned); + phase = "queued D7 command/completion"; /* Drain already queued genuine requests without ever manufacturing one. */ for (at = 0; at < ARRAY_SIZE(m->cal_requests); at++) { spin_lock_irqsave(&m->response_lock, flags); @@ -158,7 +168,7 @@ int mt_calibration_gate(struct mt7932 *m) ret = 0; out: if (ret) - dev_err(&m->pdev->dev, "own calibration using mediatek/mt7932/wcal.bin and mediatek/mt7932/oca2.bin failed: %d\n", ret); + dev_err(&m->pdev->dev, "CALIBRATION_FAILED: phase=%s error=%d\n", phase, ret); release_firmware(oca); out_wcal: release_firmware(wcal); diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index d8651b626f46e..281a8e32e23bc 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -231,6 +231,7 @@ static int mt_association_calibration(struct mt7932 *m) const struct firmware *oca; struct mt7932_cal_piece pieces[2]; unsigned int input_tag; + const char *phase = "OCA2 plan"; int ret; ret = mt_request_input(m, &oca, "mediatek/mt7932/oca2.bin"); @@ -239,11 +240,13 @@ static int mt_association_calibration(struct mt7932 *m) ret = mt7932_cal_association(oca->data, oca->size, m->connect_center, pieces, &input_tag); if (ret < 0) mt_cal_input_error(m, ret, input_tag, "association plan"); - if (ret == 2) + if (ret == 2) { + phase = "D6 command/completion"; ret = mt_cal_procedure(m, pieces, 2, 1, 0); + } release_firmware(oca); if (ret) - dev_err(&m->pdev->dev, "association calibration using mediatek/mt7932/oca2.bin failed: %d\n", ret); + dev_err(&m->pdev->dev, "ASSOCIATION_CALIBRATION_FAILED: phase=%s error=%d\n", phase, ret); return ret; } diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index ac8ac763fbf79..d9289cb3001fa 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -751,8 +751,10 @@ static int mt_capability_gate(struct mt7932 *m) dev_info(&m->pdev->dev, "firmware MAC %pM\n", value); mac = true; } else if (type == 8) { - if (!value[4] || value[4] > 4) + if (!value[4] || value[4] > 4) { + dev_err(&m->pdev->dev, "PHY_CAPABILITY_MISMATCH: NSS=%u\n", value[4]); return -EOPNOTSUPP; + } memcpy(m->phy_cap, value, 12); m->antenna_mask = (1U << value[4]) - 1; dev_info(&m->pdev->dev, "firmware PHY NSS=%u paths=%02x HT=%u VHT=%u HE=%u\n", @@ -770,8 +772,10 @@ static int mt_capability_gate(struct mt7932 *m) } offset += 8 + length; } - if (!mac || !phy) + if (!mac || !phy) { + dev_err(&m->pdev->dev, "CAPABILITY_MISSING: MAC=%u PHY=%u\n", mac, phy); return -ENODATA; + } print_hex_dump(KERN_INFO, "NIC_CAP: ", DUMP_PREFIX_OFFSET, 16, 1, m->reply, m->reply_length, false); dev_info(&m->pdev->dev, "MCU_CAPABILITY_OK eid=%02x seq=%u (no calibration or RF yet)\n", @@ -851,8 +855,11 @@ static int mt_precal_gate(struct mt7932 *m) u32 status; int ret; - if (!m->runtime_epoch || m->board_type != 3 || !m->runtime_cal_valid) - return -EINVAL; + if (!m->runtime_epoch || m->board_type != 3 || !m->runtime_cal_valid) { + dev_err(&m->pdev->dev, "PRECAL_CAPABILITY_MISMATCH: runtime-epoch=%u board-type=%u runtime-cal-valid=%u\n", + m->runtime_epoch, m->board_type, m->runtime_cal_valid); + return -EOPNOTSUPP; + } /* Stock EfuseBufferModeCal=3, own eFuse board3: original PPR source2. * This is not the external EEPROM/source0 or own-WCAL/source3 path. */ @@ -867,10 +874,14 @@ static int mt_precal_gate(struct mt7932 *m) memcpy(request + 4, ppr->data, ppr->size); release_firmware(ppr); ret = mt_request_ext(m, 0xed, 0x21, true, false, true, request, sizeof(request)); - if (ret) + if (ret) { + dev_err(&m->pdev->dev, "PPR_SOURCE2_COMMAND_FAILED: %d\n", ret); return ret; - if (m->reply_length < 44 || m->reply[28] != 0xed || m->reply[32]) + } + if (m->reply_length < 44 || m->reply[28] != 0xed || m->reply[32]) { + dev_err(&m->pdev->dev, "PPR_SOURCE2_REPLY_REJECTED: length=%zu\n", m->reply_length); return -EPROTO; + } status = get_unaligned_le32(m->reply + 40); dev_info(&m->pdev->dev, "PPR_SOURCE2_RESULT: status=%08x\n", status); if (status) @@ -925,6 +936,7 @@ static int mt_firmware_gate(struct mt7932 *m) { const struct firmware *patch, *ram; struct mt7932_region region; + const char *phase = "file-format"; u8 request[8] = {}, value; u32 entry = 0; int ret, patches, regions, i; @@ -957,6 +969,7 @@ static int mt_firmware_gate(struct mt7932 *m) } dev_info(&m->pdev->dev, "stock containers patch=%zu/%d regions RAM=%zu/%d regions\n", patch->size, patches, ram->size, regions); + phase = "patch semaphore command/reply"; put_unaligned_le32(2, request); ret = mt_command(m, 0x10, request, 4); if (ret) @@ -966,6 +979,7 @@ static int mt_firmware_gate(struct mt7932 *m) goto out; } if (m->reply[32] != 1) { + phase = "patch download/start"; for (i = 0; i < patches; i++) { mt7932_patch_region(patch->data, patch->size, i, ®ion); ret = mt_download(m, patch->data, ®ion, false); @@ -982,6 +996,7 @@ static int mt_firmware_gate(struct mt7932 *m) mt_secure_release(m); } dev_info(&m->pdev->dev, "PATCH_READY\n"); + phase = "OTP command/profile"; ret = mt_otp_byte(m, 0x7a, &value); if (!ret) ret = mt_otp_byte(m, 0x164, &value); @@ -990,7 +1005,7 @@ static int mt_firmware_gate(struct mt7932 *m) if (ret) goto out; if (value != 0x15) { - dev_err(&m->pdev->dev, "OTP selector mismatch: %02x\n", value); + dev_err(&m->pdev->dev, "OTP_PROFILE_MISMATCH: selector=%02x\n", value); ret = -EPROTO; goto out; } @@ -999,6 +1014,7 @@ static int mt_firmware_gate(struct mt7932 *m) if (ret) goto out; } + phase = "RAM download"; for (i = 0; i < regions; i++) { mt7932_ram_region(ram->data, ram->size, i, ®ion); if (region.features & BIT(5)) @@ -1009,6 +1025,7 @@ static int mt_firmware_gate(struct mt7932 *m) if (ret) goto out; } + phase = "RAM TX retirement"; for (i = 0; i < 1000; i++) { mt_tx_clean(&m->tx[16]); if (!m->tx[16].queued) @@ -1019,6 +1036,7 @@ static int mt_firmware_gate(struct mt7932 *m) ret = -ETIMEDOUT; goto out; } + phase = "firmware start command/readback"; usleep_range(20000, 21000); ret = mt_secure_acquire(m); if (ret) @@ -1033,8 +1051,11 @@ static int mt_firmware_gate(struct mt7932 *m) if (ret) goto out; dev_info(&m->pdev->dev, "FIRMWARE_READY entry=%08x\n", entry); + phase = "runtime capability/transport gates"; ret = mt_runtime_start(m); out: + if (ret) + dev_err(&m->pdev->dev, "FIRMWARE_GATE_FAILED: phase=%s error=%d\n", phase, ret); release_firmware(ram); out_patch: release_firmware(patch); diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index c6035f5cb21c1..fed573cc06b01 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -22,6 +22,7 @@ static int mt_stock_config(struct mt7932 *m) }; struct mt_config_record *records; const struct firmware *file; + const char *phase = "file-format"; u8 body[284]; unsigned int count, i, j, batch; int ret; @@ -41,6 +42,7 @@ static int mt_stock_config(struct mt7932 *m) */ records = kcalloc(count + ARRAY_SIZE(updates), sizeof(*records), GFP_KERNEL); if (!records) { + phase = "host allocation"; ret = -ENOMEM; goto out; } @@ -59,6 +61,8 @@ static int mt_stock_config(struct mt7932 *m) * the separately specified conditional DbdcMode insertion, not a default. */ if (!m->phy_cap[5]) { + phase = "capability profile"; + dev_err(&m->pdev->dev, "STOCK_CONFIG_CAPABILITY_MISMATCH: DBDC expected=1 observed=0\n"); ret = -EOPNOTSUPP; goto free; } @@ -105,6 +109,7 @@ static int mt_stock_config(struct mt7932 *m) records[j].value[0] = '1'; records[j].value_len = 1; } + phase = "native SET submission"; for (i = 0; i < count; i += batch) { batch = min(4U, count - i); memset(body, 0, sizeof(body)); @@ -120,8 +125,12 @@ static int mt_stock_config(struct mt7932 *m) free: kfree(records); out: - if (ret) - dev_err(&m->pdev->dev, "local input mediatek/mt7932/config-original.bin processing failed: %d\n", ret); + if (ret) { + if (!strcmp(phase, "file-format")) + dev_err(&m->pdev->dev, "local input mediatek/mt7932/config-original.bin has invalid format: %d\n", ret); + else + dev_err(&m->pdev->dev, "STOCK_CONFIG_FAILED: phase=%s error=%d\n", phase, ret); + } release_firmware(file); return ret; } From 23cdff4f4c791ee6cb62b1aa3b91af7136fa8fb2 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:44:47 -0700 Subject: [PATCH 0156/2680] wifi: mt7932: test real startup D7 draining and policy admission hooks Exercise the actual startup queue-drain and calibration procedure with synthetic version-12 containers and a test-only transport delivering three logical band-1 completions. A restored hardcoded four-credit startup wait fails the test before touching hardware. Assert open, scan and connect queued missing-policy recovery before invoking the retry helper a second time, so the test cannot mask a missing frontend hook. The production drain is the same bounded serialized path; transport injection is omitted unless RF KUnit is enabled. Existing Neo admission gates remain ahead of runtime entry; off-Neo behavior is unchanged. Round-2 S1. Non-Neo runtime behavior remains unchanged. --- .../mediatek/mt7932/calibration_transport.c | 43 +++++--- drivers/net/wireless/mediatek/mt7932/mt7932.h | 6 ++ .../net/wireless/mediatek/mt7932/rf-test.c | 99 +++++++++++++++++++ 3 files changed, 135 insertions(+), 13 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c index 0acc017f49156..1d6d4d6985a45 100644 --- a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c +++ b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c @@ -23,7 +23,12 @@ int mt_cal_procedure(struct mt7932 *m, struct mt7932_cal_piece *pieces, ret = mt7932_cal_body(body, sizeof(body), pieces + i, context_version); if (ret < 0) goto fail; - ret = mt_request(m, 0xd6, true, true, false, body, ret); +#if IS_ENABLED(CONFIG_MT7932_RF_KUNIT_TEST) + if (m->cal_test_send) + ret = m->cal_test_send(m, body, ret); + else +#endif + ret = mt_request(m, 0xd6, true, true, false, body, ret); if (ret) goto fail; } @@ -67,7 +72,6 @@ int mt_calibration_gate(struct mt7932 *m) const char *phase = "own input validation"; struct mt7932_cal_piece pieces[14]; u8 body[1028], request[16]; - unsigned long flags; unsigned int at, count, group, input_tag; int ret, planned; @@ -140,7 +144,27 @@ int mt_calibration_gate(struct mt7932 *m) group, pieces[at].type, pieces[at].parameter, count); } dev_info(&m->pdev->dev, "POWER_ON_CAL_COMPLETE: %u groups, %d fragments\n", group, planned); - phase = "queued D7 command/completion"; + phase = "queued D7 planning/command/completion"; + ret = mt_cal_drain_requests(m, oca); +out: + if (ret) + dev_err(&m->pdev->dev, "CALIBRATION_FAILED: phase=%s error=%d\n", phase, ret); + release_firmware(oca); +out_wcal: + release_firmware(wcal); + return ret; +} + +/* Shared startup queue drain, entered by the serialized command owner. */ +int mt_cal_drain_requests(struct mt7932 *m, const struct firmware *oca) +{ + struct mt7932_cal_piece pieces[7]; + u8 request[16]; + unsigned long flags; + unsigned int at, count, input_tag; + int ret; + + lockdep_assert_held(&m->command_mutex); /* Drain already queued genuine requests without ever manufacturing one. */ for (at = 0; at < ARRAY_SIZE(m->cal_requests); at++) { spin_lock_irqsave(&m->response_lock, flags); @@ -157,20 +181,13 @@ int mt_calibration_gate(struct mt7932 *m) m->smart_version, m->module_byte, pieces, &input_tag); if (ret < 0) { mt_cal_input_error(m, ret, input_tag, "D7 plan"); - goto out; + return ret; } ret = mt_cal_procedure(m, pieces, ret, mt7932_cal_request_replies(request), m->preload_version); if (ret) - goto out; + return ret; dev_info(&m->pdev->dev, "CAL_REQUEST_SERVICED: channel=%u\n", get_unaligned_le32(request + 12)); } - ret = 0; -out: - if (ret) - dev_err(&m->pdev->dev, "CALIBRATION_FAILED: phase=%s error=%d\n", phase, ret); - release_firmware(oca); -out_wcal: - release_firmware(wcal); - return ret; + return 0; } diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index ca77ba30b037d..f1e4124f4d887 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -65,6 +65,11 @@ struct mt7932 { struct completion response; struct completion cal_response; struct mt7932_cal_completion cal_state; +#if IS_ENABLED(CONFIG_MT7932_RF_KUNIT_TEST) + /* Hardware-free calibration transport seam; production builds omit it. */ + int (*cal_test_send)(struct mt7932 *m, const void *body, size_t length); + void *cal_test_context; +#endif u8 cal_requests[32][16]; u8 cal_request_seq[32]; unsigned int cal_request_head, cal_request_count; @@ -186,6 +191,7 @@ int mt_recovery_gate(struct mt7932 *m); int mt_cal_procedure(struct mt7932 *m, struct mt7932_cal_piece *pieces, unsigned int count, unsigned int logical, u8 context_version); int mt_calibration_gate(struct mt7932 *m); +int mt_cal_drain_requests(struct mt7932 *m, const struct firmware *oca); void mt_cal_input_error(struct mt7932 *m, int error, unsigned int tag, const char *stage); void mt_data_clean_locked(struct mt7932 *m); void mt_data_clean(struct mt7932 *m); diff --git a/drivers/net/wireless/mediatek/mt7932/rf-test.c b/drivers/net/wireless/mediatek/mt7932/rf-test.c index 2fb0812d09970..da078caa2444a 100644 --- a/drivers/net/wireless/mediatek/mt7932/rf-test.c +++ b/drivers/net/wireless/mediatek/mt7932/rf-test.c @@ -178,6 +178,11 @@ static void mt_missing_policy_admission_retry_test(struct kunit *test) KUNIT_EXPECT_EQ(test, mt_connect(wiphy, m->netdev, &connect), -EAGAIN); break; } + /* Assert the actual frontend queued recovery before checking that + * another request cannot consume the same retry a second time. + */ + KUNIT_EXPECT_EQ(test, m->reg_generation, 8U); + KUNIT_EXPECT_FALSE(test, m->reg_retryable); mt_retry_missing_policy(m); flush_work(&m->startup_work); KUNIT_EXPECT_EQ(test, m->reg_generation, 8U); @@ -200,7 +205,101 @@ static void mt_missing_policy_admission_retry_test(struct kunit *test) wiphy_free(wiphy); } +struct mt_cal_drain_test_context { + struct kunit *test; + unsigned int commands, replies; +}; + +static int mt_cal_drain_test_send(struct mt7932 *m, const void *data, size_t length) +{ + struct mt_cal_drain_test_context *ctx = m->cal_test_context; + const u8 *body = data; + u8 reply[20] = {}; + unsigned long flags; + int ret = 0; + + KUNIT_EXPECT_EQ(ctx->test, m->cal_state.expected, 3U); + if (m->cal_state.expected != 3 || length < 20) + return -EPROTO; + ctx->commands++; + /* Firmware completes each logical group after its final fragment. */ + if ((body[2] & 15) + 1 != body[2] >> 4) + return 0; + reply[3] = 1; + spin_lock_irqsave(&m->response_lock, flags); + ret = mt7932_cal_null(&m->cal_state, reply, sizeof(reply), m->smart_version); + if (ret > 0) + complete(&m->cal_response); + spin_unlock_irqrestore(&m->response_lock, flags); + ctx->replies++; + return ret < 0 ? ret : 0; +} + +static void mt_startup_band1_queue_drain_test(struct kunit *test) +{ + /* Synthetic open-container fixtures only: no unit calibration assets. */ + static const u16 tags[] = { 0x501, 0x2001, 0x4124, 0x3126 }; + static const u16 bytes[] = { 200, 2160, 20, 1224 }; + struct mt_cal_drain_test_context ctx = { .test = test }; + struct mt7932 *m = test->priv; + struct firmware oca = {}; + size_t size = 16 + 20 * ARRAY_SIZE(tags), offset; + u8 *data; + unsigned int i, j; + int ret; + + for (i = 0; i < ARRAY_SIZE(tags); i++) + size += 8 + bytes[i]; + data = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, data); + memcpy(data, "BLOB", 4); + offset = 16 + 20 * ARRAY_SIZE(tags); + put_unaligned_be32(offset, data + 4); + put_unaligned_be16(12, data + 8); + put_unaligned_be16(ARRAY_SIZE(tags), data + 10); + for (i = 0; i < ARRAY_SIZE(tags); i++) { + u8 *entry = data + 16 + 20 * i; + u32 sum = 0; + + put_unaligned_be16(tags[i], entry); + put_unaligned_be16(12, entry + 2); + put_unaligned_be32(offset, entry + 4); + put_unaligned_be32(8 + bytes[i], entry + 8); + memcpy(data + offset, entry, 4); + put_unaligned_be32(8 + bytes[i], data + offset + 4); + for (j = 0; j < 8 + bytes[i]; j++) + sum += data[offset + j]; + put_unaligned_be32(sum, entry + 12); + offset += 8 + bytes[i]; + } + oca.data = data; + oca.size = size; + memset(&m->cal_state, 0, sizeof(m->cal_state)); + m->smart_version = 12; + m->preload_version = 1; + m->module_byte = 0x89; + m->cal_request_count = 1; + put_unaligned_le32(0x1000, m->cal_requests[0] + 4); + put_unaligned_le32(1, m->cal_requests[0] + 8); + put_unaligned_le32(36, m->cal_requests[0] + 12); + m->cal_test_send = mt_cal_drain_test_send; + m->cal_test_context = &ctx; + mutex_lock(&m->command_mutex); + ret = mt_cal_drain_requests(m, &oca); + mutex_unlock(&m->command_mutex); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, ctx.commands, 5U); + KUNIT_EXPECT_EQ(test, ctx.replies, 3U); + KUNIT_EXPECT_EQ(test, m->cal_request_count, 0U); + KUNIT_EXPECT_TRUE(test, m->cal_state.done); + KUNIT_EXPECT_FALSE(test, m->cal_state.active); + KUNIT_EXPECT_EQ(test, m->cal_state.error, 0); + m->cal_test_send = NULL; + m->cal_test_context = NULL; +} + static struct kunit_case mt_rf_test_cases[] = { + KUNIT_CASE(mt_startup_band1_queue_drain_test), KUNIT_CASE(mt_missing_policy_admission_retry_test), KUNIT_CASE(mt_cal_band_reply_completion_test), KUNIT_CASE(mt_rf_failure_wakes_association_test), From d4b532248a59d740a139e0a5abfc8e62012f93cd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:30:06 +1000 Subject: [PATCH 0157/2680] iommu/apple-dart: Support T8140 DARTs and firmware-owned stream roots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The DARTs of the A18 Pro (T8140) are T8110-family instances that report version 2.2 or later in PARAMS3. At least the AOP DART is locked by the bootloader (PROTECT reads back 0x1e7), and its stream roots already hold the co-processor firmware's own mappings when Linux takes over. The driver handled neither: - Attaching a domain to a locked stream copied Linux's whole L1 table over the firmware's root table, and releasing it zeroed the table, destroying the firmware's mappings. - From version 2.2 on, per-stream TLB flush commands take an extra command bit, faults are reported through per-stream exception records instead of the legacy error registers, and exceptions latched before Linux took over are still pending at probe. Handle both, and add the device tree inputs the T8140 device tree needs: - Locked roots: publish Linux's L1 entries only into slots that are vacant or that Linux itself filled, keep a shadow copy of what was published, and on release clear only entries that still hold Linux's value. A slot whose owner has changed fails the operation with -EBUSY instead of being overwritten. Publishing is done under the DART lock, and errors now propagate out of the attach path. - TLB maintenance: wait for the busy bit before and after every T8110 TLB command, for up to 2 ms instead of 100 us. From version 2.2 on, set bit 15 in per-stream flush commands, and after a map or unmap flush a locked stream by range instead of invalidating it whole; TLB_START and TLB_END take the first and last page number, derived from the DART page size, shifted left by two. - Faults from version 2.2 on: decode the pending-exception bitmap at 0x4000 (one bit per stream, write 1 to clear) and the per-stream records at 0x8000 + 0x40 * sid (status, access type and fault address), plus the block-level summary at 0x704 on version 2.3. Drain exceptions left over from before probe, logging them at debug level. If the DART is set to lock on error (DIAG_LOCK bit 1), unlock it with TLB command 4 once its exceptions have been acknowledged. - apple,remap-streams: validate the property at probe and program the remaps into the source streams' TCRs when the DART is reset. - apple,dma-range on a consumer node: use that window as the domain's aperture instead of the DART's, so that each consumer's IOVAs land in root slots that are free for it. Consumer and DART windows must have a nonzero size, must not overflow or cross the DART's input address width, and must be aligned to the DART page size. - Power the DART with pm_runtime_resume_and_get() and handle failure instead of warning, both when flushing and when releasing a consumer of a locked DART. When a locked DART's domain is freed, its entries are retired and the stream invalidated first; if the DART cannot be powered or the invalidation fails, the page tables are intentionally leaked rather than freed while the hardware may still cache them. - Return -EINVAL instead of warning when a locked DART is attached with anything other than a single stream. The probe banner and the DMA range and IAS messages move to debug level. [ryan: adapted to 7.2.10 — the consumer apple,dma-range window now overrides Asahi's per-master cfg DMA window (commit 3658355cfaba ("iommu: apple-dart: Support specifying the DMA aperture in the DT")) instead of the DART window; the T8110 TLB loop keeps Asahi's first-stream-only invalidate for remapping DARTs; the static remaps use Asahi's renamed DART_T8110_TCR_REMAP_ENABLE] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 717 +++++++++++++++++++++++++++++++------ 1 file changed, 604 insertions(+), 113 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index b104bd4973bcf..3647d35667eaf 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -11,6 +11,7 @@ * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd. */ +#include #include #include #include @@ -28,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -116,6 +118,13 @@ #define DART_T8110_TLB_CMD_OP GENMASK(10, 8) #define DART_T8110_TLB_CMD_OP_FLUSH_ALL 0 #define DART_T8110_TLB_CMD_OP_FLUSH_SID 1 +#define DART_T8110_TLB_CMD_OP_FLUSH_UNLOCK 4 +#define DART_T8110_TLB_CMD_V2 BIT(15) +#define DART_T8110_TLB_CMD_RANGE BIT(14) +#define DART_T8110_TLB_CMD_STT BIT(13) +#define DART_T8110_TLB_START 0x98 +#define DART_T8110_TLB_END 0xa0 +#define DART_T8110_FLUSH_BUSY_TIMEOUT 2000 #define DART_T8110_TLB_CMD_STREAM GENMASK(7, 0) #define DART_T8110_ERROR 0x100 @@ -137,10 +146,27 @@ #define DART_T8110_ERROR_STREAMS 0x1c0 +#define DART_T8110_EXCEPTION(i) (0x4000 + 4 * (i)) +#define DART_T8110_ERR_SID(sid) (0x8000 + 0x40 * (sid)) +#define DART_T8110_ERR_SID_STATUS(sid) (DART_T8110_ERR_SID(sid) + 0x00) +#define DART_T8110_ERR_SID_STATUS_WRITE BIT(8) +#define DART_T8110_EXCEPTION_CODE GENMASK(3, 0) +#define DART_T8110_EXCEPTION_LEVEL GENMASK(13, 12) + +/* DART v2.3 adds a block-level exception summary. */ +#define DART_T8110_BLOCK_ERROR 0x704 +#define DART_T8110_BLOCK_ERROR_MASK 0x777 + +#define DART_T8110_EXCEPTION_INFO(sid) (0x8000 + 0x40 * (sid)) +#define DART_T8110_EXCEPTION_ADDR_LO(sid) (DART_T8110_ERR_SID(sid) + 0x8) +#define DART_T8110_EXCEPTION_ADDR_HI(sid) (DART_T8110_ERR_SID(sid) + 0xc) + #define DART_T8110_PROTECT 0x200 #define DART_T8110_UNPROTECT 0x204 #define DART_T8110_PROTECT_LOCK 0x208 #define DART_T8110_PROTECT_TTBR_TCR BIT(0) +#define DART_T8110_DIAG_LOCK 0x210 +#define DART_T8110_DIAG_LOCK_ON_ERR BIT(1) #define DART_T8110_ENABLE_STREAMS 0xc00 #define DART_T8110_DISABLE_STREAMS 0xc20 @@ -243,6 +269,10 @@ struct apple_dart { u32 tunneled : 1; u32 power_retained : 1; struct apple_tunable *tunables; + u16 version; + + DECLARE_BITMAP(remapped, DART_MAX_STREAMS); + u8 remap_target[DART_MAX_STREAMS]; dma_addr_t dma_min; dma_addr_t dma_max; @@ -255,6 +285,8 @@ struct apple_dart { u32 save_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR]; u64 *locked_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR]; + /* Values published by this driver into initially vacant root slots. */ + u64 *locked_owned[DART_MAX_STREAMS][DART_MAX_TTBR]; }; /* @@ -343,6 +375,9 @@ struct apple_dart_domain { struct io_pgtable_ops *pgtbl_ops; bool finalized; + bool has_dma_range; + dma_addr_t dma_min; + dma_addr_t dma_max; u64 mask; dma_addr_t dma_offset; struct mutex init_lock; @@ -509,6 +544,56 @@ apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map) apple_dart_hw_clear_ttbr(stream_map, i); } +/* + * A locked root belongs to firmware. Publish only into vacant slots and + * remember exactly what we installed, so teardown cannot erase firmware + * mappings added after Linux attached. + */ +static int apple_dart_publish_root(u64 *live, u64 *owned, const u64 *ours, + size_t entries) +{ + size_t i; + + for (i = 0; i < entries; i++) { + u64 next = READ_ONCE(ours[i]); + u64 previous = READ_ONCE(owned[i]); + + if (!next && !previous) + continue; + if (READ_ONCE(live[i]) != previous) + return -EBUSY; + } + + dma_wmb(); + for (i = 0; i < entries; i++) { + u64 next = READ_ONCE(ours[i]); + u64 previous = READ_ONCE(owned[i]); + + if (next == previous) + continue; + if (cmpxchg(&live[i], previous, next) != previous) + return -EBUSY; + WRITE_ONCE(owned[i], next); + } + return 0; +} + +static void apple_dart_retire_root(u64 *live, u64 *owned, size_t entries) +{ + size_t i; + + for (i = 0; i < entries; i++) { + u64 previous = READ_ONCE(owned[i]); + + if (!previous) + continue; + /* Preserve a slot if its owner changed after we published it. */ + cmpxchg(&live[i], previous, 0); + WRITE_ONCE(owned[i], 0); + } + dma_wmb(); +} + static int apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) { @@ -518,7 +603,10 @@ apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { u32 ttbr; phys_addr_t phys; - u64 *l1_tbl; + u64 *l1_tbl, *owned; + + if (dart->locked_ttbr[sid][idx]) + continue; ttbr = apple_dart_readl(dart, DART_TTBR(dart, sid, idx)); @@ -539,6 +627,12 @@ apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) if (IS_ERR(l1_tbl)) return PTR_ERR(l1_tbl); + owned = devm_kzalloc(dart->dev, dart->pgsize, GFP_KERNEL); + if (!owned) { + devm_memunmap(dart->dev, l1_tbl); + return -ENOMEM; + } + dart->locked_owned[sid][idx] = owned; dart->locked_ttbr[sid][idx] = l1_tbl; } @@ -546,43 +640,54 @@ apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) } static int -apple_dart_hw_unmap_locked_ttbr(struct apple_dart_stream_map *stream_map, - u8 idx) +apple_dart_hw_unmap_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) { struct apple_dart *dart = stream_map->dart; int sid; for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { - /* TODO: locked L1 table might need to be restored to boot state */ - if (dart->locked_ttbr[sid][idx]) { - memset(dart->locked_ttbr[sid][idx], 0, dart->pgsize); - devm_memunmap(dart->dev, dart->locked_ttbr[sid][idx]); - } + u64 *live = dart->locked_ttbr[sid][idx]; + u64 *owned = dart->locked_owned[sid][idx]; + + if (!live) + continue; + apple_dart_retire_root(live, owned, dart->pgsize / sizeof(*live)); + devm_memunmap(dart->dev, live); + devm_kfree(dart->dev, owned); dart->locked_ttbr[sid][idx] = NULL; + dart->locked_owned[sid][idx] = NULL; } - return 0; } static int apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg, - struct apple_dart_stream_map *stream_map) + struct apple_dart_stream_map *stream_map) { struct apple_dart *dart = stream_map->dart; - int sid; + unsigned long flags; + int sid, idx, ret = 0; + spin_lock_irqsave(&dart->lock, flags); for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { - for (int idx = 0; idx < dart->hw->ttbr_count; idx++) { - u64 *ttbrep = dart->locked_ttbr[sid][idx]; - u64 *ptep = cfg->apple_dart_cfg.ttbr[idx]; - if (!ttbrep || !ptep) - continue; - for (int entry = 0; entry < dart->pgsize / sizeof(*ptep); entry++) - ttbrep[entry] = ptep[entry]; + for (idx = 0; idx < cfg->apple_dart_cfg.n_ttbrs; idx++) { + u64 *live = dart->locked_ttbr[sid][idx]; + u64 *owned = dart->locked_owned[sid][idx]; + u64 *ours = cfg->apple_dart_cfg.ttbr[idx]; + + if (!live || !owned || !ours) { + ret = -EIO; + goto out; + } + ret = apple_dart_publish_root(live, owned, ours, + dart->pgsize / sizeof(*live)); + if (ret) + goto out; } } - - return 0; +out: + spin_unlock_irqrestore(&dart->lock, flags); + return ret; } static int @@ -620,44 +725,53 @@ apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map, } static int -apple_dart_t8110_hw_tlb_command(struct apple_dart_stream_map *stream_map, - u32 command) +apple_dart_t8110_hw_tlb_command_range(struct apple_dart_stream_map *stream_map, + u32 command, bool range, u64 first, u64 last) { struct apple_dart *dart = stream_map->dart; unsigned long flags; - int ret = 0; - int sid; + u32 status; + int sid, ret = 0; spin_lock_irqsave(&dart->lock, flags); - for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { u32 val = FIELD_PREP(DART_T8110_TLB_CMD_OP, command) | - FIELD_PREP(DART_T8110_TLB_CMD_STREAM, sid); - apple_dart_writel(dart, val, DART_T8110_TLB_CMD); + FIELD_PREP(DART_T8110_TLB_CMD_STREAM, sid); - ret = read_poll_timeout_atomic( - apple_dart_readl, val, - !(val & DART_T8110_TLB_CMD_BUSY), 1, - DART_STREAM_COMMAND_BUSY_TIMEOUT, false, dart, + ret = read_poll_timeout_atomic(apple_dart_readl, status, + !(status & DART_T8110_TLB_CMD_BUSY), 1, + DART_T8110_FLUSH_BUSY_TIMEOUT, false, dart, + DART_T8110_TLB_CMD); + if (ret) + break; + if (dart->version >= 0x0202 && + command == DART_T8110_TLB_CMD_OP_FLUSH_SID) + val |= DART_T8110_TLB_CMD_V2; + if (range) { + val |= DART_T8110_TLB_CMD_RANGE | DART_T8110_TLB_CMD_STT; + apple_dart_writel(dart, (first >> ilog2(dart->pgsize)) << 2, + DART_T8110_TLB_START); + apple_dart_writel(dart, (last >> ilog2(dart->pgsize)) << 2, + DART_T8110_TLB_END); + } + dma_wmb(); + apple_dart_writel(dart, val, DART_T8110_TLB_CMD); + ret = read_poll_timeout_atomic(apple_dart_readl, status, + !(status & DART_T8110_TLB_CMD_BUSY), 1, + DART_T8110_FLUSH_BUSY_TIMEOUT, false, dart, DART_T8110_TLB_CMD); - if (ret) break; /* Since remap is used only the first sid needs to be invalidated. */ if (apple_dart_remaps_streams(dart)) break; } - spin_unlock_irqrestore(&dart->lock, flags); - - if (ret) { - dev_err(stream_map->dart->dev, - "busy bit did not clear after command %x for stream %d\n", - command, sid); - return ret; - } - - return 0; + if (ret) + dev_err_ratelimited(dart->dev, + "TLB command %#x for stream %d timed out (%#x)\n", + command, sid, status); + return ret; } static int @@ -670,8 +784,122 @@ apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) static int apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) { - return apple_dart_t8110_hw_tlb_command( - stream_map, DART_T8110_TLB_CMD_OP_FLUSH_SID); + return apple_dart_t8110_hw_tlb_command_range( + stream_map, DART_T8110_TLB_CMD_OP_FLUSH_SID, false, 0, 0); +} + +static const char *apple_dart_t8110_v2_fault_name(u32 exception); + +static void apple_dart_drain_stale_exceptions(struct apple_dart *dart) +{ + unsigned int i, drained = 0; + + if (dart->hw->type != DART_T8110 || dart->version < 0x0202) + return; + + for (i = 0; i < DIV_ROUND_UP(dart->num_streams, 32) && i < 8; i++) { + u32 pending = readl(dart->regs + DART_T8110_EXCEPTION(i)); + unsigned long bits = pending; + unsigned int bit; + + if (!pending) + continue; + + for_each_set_bit(bit, &bits, 32) { + unsigned int sid = i * 32 + bit; + u32 status; + u64 addr; + + if (sid >= dart->num_streams) + break; + + status = readl(dart->regs + + DART_T8110_ERR_SID_STATUS(sid)); + addr = readl(dart->regs + + DART_T8110_EXCEPTION_ADDR_LO(sid)) | + (u64)readl(dart->regs + + DART_T8110_EXCEPTION_ADDR_HI(sid)) << 32; + dev_dbg(dart->dev, + "stale exception latched before probe: SID:%u %s %s at %#llx (status %#010x)\n", + sid, + status & DART_T8110_ERR_SID_STATUS_WRITE ? + "write" : "read", + apple_dart_t8110_v2_fault_name(status), + addr, status); + drained++; + } + + writel(pending, dart->regs + DART_T8110_EXCEPTION(i)); + } + + if (drained) { + dsb(sy); + readl(dart->regs + DART_T8110_EXCEPTION(0)); + } + + if (dart->version >= 0x0203) + readl(dart->regs + DART_T8110_BLOCK_ERROR); + + if (drained) + dev_dbg(dart->dev, + "%u stale exception(s) drained; these predate Linux\n", + drained); +} + +static int apple_dart_t8110_v2_unlock(struct apple_dart *dart) +{ + struct apple_dart_stream_map stream_map = { .dart = dart }; + + if (!(readl(dart->regs + DART_T8110_DIAG_LOCK) & DART_T8110_DIAG_LOCK_ON_ERR)) + return 0; + + /* Unlock after acknowledging a lock-on-error exception. */ + __set_bit(0, stream_map.sidmap); + return apple_dart_t8110_hw_tlb_command_range(&stream_map, + DART_T8110_TLB_CMD_OP_FLUSH_UNLOCK, + false, 0, 0); +} + +static int apple_dart_read_remaps(struct apple_dart *dart) +{ + struct device_node *np = dart->dev->of_node; + u32 src, dst; + int count, i, ret; + + if (!of_property_present(np, "apple,remap-streams")) + return 0; + count = of_property_count_u32_elems(np, "apple,remap-streams"); + if (count <= 0 || count > 2 * dart->num_streams || count % 2 || + dart->hw->type != DART_T8110) + return -EINVAL; + for (i = 0; i < count; i += 2) { + ret = of_property_read_u32_index(np, "apple,remap-streams", i, &src); + if (!ret) + ret = of_property_read_u32_index(np, "apple,remap-streams", i + 1, &dst); + if (ret) + return ret; + if (src >= dart->num_streams || dst >= dart->num_streams || + dst > FIELD_MAX(DART_T8110_TCR_REMAP) || src == dst) + return -ERANGE; + if (test_and_set_bit(src, dart->remapped)) + return -EINVAL; + dart->remap_target[src] = dst; + } + /* Only a single remap hop is described by this binding. */ + for_each_set_bit(i, dart->remapped, dart->num_streams) + if (test_bit(dart->remap_target[i], dart->remapped)) + return -EINVAL; + return 0; +} + +static void apple_dart_hw_apply_remaps(struct apple_dart *dart) +{ + int sid; + + for_each_set_bit(sid, dart->remapped, dart->num_streams) + writel(DART_T8110_TCR_REMAP_ENABLE | + FIELD_PREP(DART_T8110_TCR_REMAP, dart->remap_target[sid]), + dart->regs + DART_TCR(dart, sid)); } static void @@ -706,15 +934,26 @@ static int apple_dart_hw_reset(struct apple_dart *dart) apple_dart_hw_disable_dma(&stream_map); apple_dart_hw_clear_all_ttbrs(&stream_map); + apple_dart_hw_apply_remaps(dart); /* enable all streams globally since TCR is used to control isolation */ for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) apple_dart_writel(dart, U32_MAX, dart->hw->enable_streams + 4 * i); - /* clear any pending errors before the interrupt is unmasked */ - apple_dart_writel(dart, apple_dart_readl(dart, dart->hw->error), - dart->hw->error); + /* Newer revisions expose per-stream exception records. */ + if (dart->hw->type == DART_T8110 && dart->version >= 0x0202) { + int ret; + + apple_dart_drain_stale_exceptions(dart); + ret = apple_dart_t8110_v2_unlock(dart); + if (ret) + return ret; + } else { + /* Clear pending errors before the interrupt is unmasked. */ + apple_dart_writel(dart, apple_dart_readl(dart, dart->hw->error), + dart->hw->error); + } if (dart->hw->type == DART_T8110) apple_dart_writel(dart, 0, DART_T8110_ERROR_MASK); @@ -722,28 +961,46 @@ static int apple_dart_hw_reset(struct apple_dart *dart) return dart->hw->invalidate_tlb(&stream_map); } -static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain) +static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, + bool range, u64 first, u64 last) { - int i, j; - struct apple_dart_atomic_stream_map *domain_stream_map; - struct apple_dart_stream_map stream_map; - struct io_pgtable_cfg *pgtbl_cfg = - &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; - - for_each_stream_map(i, domain, domain_stream_map) { - stream_map.dart = domain_stream_map->dart; - - for (j = 0; j < BITS_TO_LONGS(stream_map.dart->num_streams); j++) - stream_map.sidmap[j] = atomic_long_read(&domain_stream_map->sidmap[j]); - - WARN_ON(pm_runtime_get_sync(stream_map.dart->dev) < 0); - - if (stream_map.dart->locked) - apple_dart_hw_sync_locked(pgtbl_cfg, &stream_map); - - stream_map.dart->hw->invalidate_tlb(&stream_map); - pm_runtime_put(stream_map.dart->dev); + struct apple_dart_atomic_stream_map *map; + struct apple_dart_stream_map stream; + struct io_pgtable_cfg *cfg = &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; + int i, j, ret; + + for_each_stream_map(i, domain, map) { + stream.dart = map->dart; + for (j = 0; j < BITS_TO_LONGS(stream.dart->num_streams); j++) + stream.sidmap[j] = atomic_long_read(&map->sidmap[j]); + ret = pm_runtime_resume_and_get(stream.dart->dev); + if (ret < 0) { + dev_err_ratelimited(stream.dart->dev, + "failed to power DART for TLB synchronization: %d\n", ret); + return ret; + } + ret = stream.dart->locked ? apple_dart_hw_sync_locked(cfg, &stream) : 0; + if (ret) + dev_err_ratelimited(stream.dart->dev, + "failed to publish host page-table entries: %d\n", ret); + else { + if (range && stream.dart->locked && stream.dart->version >= 0x0202) + ret = apple_dart_t8110_hw_tlb_command_range(&stream, + DART_T8110_TLB_CMD_OP_FLUSH_SID, true, + first & domain->mask, last & domain->mask); + else + ret = stream.dart->hw->invalidate_tlb(&stream); + } + pm_runtime_put(stream.dart->dev); + if (ret) + return ret; } + return 0; +} + +static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain) +{ + (void)apple_dart_domain_flush_tlb_range(domain, false, 0, 0); } static void apple_dart_flush_iotlb_all(struct iommu_domain *domain) @@ -754,14 +1011,18 @@ static void apple_dart_flush_iotlb_all(struct iommu_domain *domain) static void apple_dart_iotlb_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) { - apple_dart_domain_flush_tlb(to_dart_domain(domain)); + (void)apple_dart_domain_flush_tlb_range(to_dart_domain(domain), + gather->start <= gather->end, + gather->start, gather->end); } static int apple_dart_iotlb_sync_map(struct iommu_domain *domain, - unsigned long iova, size_t size) + unsigned long iova, size_t size) { - apple_dart_domain_flush_tlb(to_dart_domain(domain)); - return 0; + if (!size || iova + size - 1 < iova) + return -EINVAL; + return apple_dart_domain_flush_tlb_range(to_dart_domain(domain), true, + iova, iova + size - 1); } static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, @@ -835,22 +1096,40 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, stream_map->dart->hw->invalidate_tlb(stream_map); } -static void +static int apple_dart_setup_translation_locked(struct apple_dart_domain *domain, struct apple_dart_stream_map *stream_map) { - int i; - struct io_pgtable_cfg *pgtbl_cfg = - &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; + struct io_pgtable_cfg *cfg = &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; + int i, ret; - /* Locked DARTs are set up by the bootloader. */ - for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i) - apple_dart_hw_map_locked_ttbr(stream_map, i); - for (; i < stream_map->dart->hw->ttbr_count; ++i) + for (i = 0; i < cfg->apple_dart_cfg.n_ttbrs; i++) { + ret = apple_dart_hw_map_locked_ttbr(stream_map, i); + if (ret) + goto unmap; + } + ret = apple_dart_hw_sync_locked(cfg, stream_map); + if (!ret) + ret = stream_map->dart->hw->invalidate_tlb(stream_map); + if (!ret) + return 0; +unmap: + while (i--) apple_dart_hw_unmap_locked_ttbr(stream_map, i); + return ret; +} - apple_dart_hw_sync_locked(pgtbl_cfg, stream_map); - stream_map->dart->hw->invalidate_tlb(stream_map); +static int apple_dart_dma_window(u64 base, u64 size, u64 *last) +{ + if (!size || check_add_overflow(base, size - 1, last)) + return -EINVAL; + return 0; +} + +static bool apple_dart_dma_window_aligned(u64 base, u64 last, u32 pgsize) +{ + /* An inclusive end has all page-offset bits set, even at U64_MAX. */ + return IS_ALIGNED(base, pgsize) && IS_ALIGNED(~last, pgsize); } static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, @@ -858,14 +1137,23 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, { struct apple_dart *dart = cfg->stream_maps[0].dart; struct io_pgtable_cfg pgtbl_cfg; - dma_addr_t dma_max = cfg->dma_max; - u32 ias = min_t(u32, dart->ias, - fls64(dma_max + dart->dma_offset)); + dma_addr_t dma_min = dart_domain->has_dma_range ? dart_domain->dma_min : cfg->dma_min; + dma_addr_t dma_max = dart_domain->has_dma_range ? dart_domain->dma_max : cfg->dma_max; + u64 translated_max; + u32 ias; int ret = 0; int i, j; if (dart->pgsize > PAGE_SIZE) return -EINVAL; + if (check_add_overflow((u64)dma_max, (u64)dart->dma_offset, + &translated_max)) + return -ERANGE; + ias = min_t(u32, dart->ias, fls64(translated_max)); + if (!apple_dart_dma_window_aligned(dma_min, dma_max, dart->pgsize)) + return -EINVAL; + if ((dma_min ^ dma_max) & ~DMA_BIT_MASK(dart->ias)) + return -ERANGE; mutex_lock(&dart_domain->init_lock); @@ -896,7 +1184,10 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, sidmap = cfg->stream_maps[0].sidmap; sid = find_first_bit(sidmap, dart->num_streams); - WARN_ON((sid < 0) || bitmap_weight(sidmap, dart->num_streams) > 1); + if (sid >= dart->num_streams || bitmap_weight(sidmap, dart->num_streams) != 1) { + ret = -EINVAL; + goto done; + } ttbr = apple_dart_readl(dart, DART_TTBR(dart, sid, 0)); WARN_ON(!(ttbr & dart->hw->ttbr_valid)); @@ -906,15 +1197,15 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, if (apple_dart_readl(dart, DART_TCR(dart, sid)) & dart->hw->tcr_4level) { if (ias < 37) { - dev_info(dart->dev, "Expanded to ias=37 due to lock\n"); + dev_dbg(dart->dev, "Expanded to ias=37 due to lock\n"); pgtbl_cfg.ias = 37; } } else if (ias > 36) { - dev_info(dart->dev, "Limited to ias=36 due to lock\n"); + dev_dbg(dart->dev, "Limited to ias=36 due to lock\n"); pgtbl_cfg.ias = 36; - if (cfg->dma_min == 0 && dma_max == DMA_BIT_MASK(dart->ias)) { + if (dma_min == 0 && dma_max == DMA_BIT_MASK(dart->ias)) { dma_max = DMA_BIT_MASK(pgtbl_cfg.ias); - } else if ((cfg->dma_min ^ dma_max) & ~DMA_BIT_MASK(36)) { + } else if ((dma_min ^ dma_max) & ~DMA_BIT_MASK(36)) { dev_err(dart->dev, "Invalid DMA range for locked 3-level PT\n"); ret = -ENOMEM; @@ -933,7 +1224,7 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, dart_domain->mask = DMA_BIT_MASK(pgtbl_cfg.ias); dart_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; - dart_domain->domain.geometry.aperture_start = cfg->dma_min; + dart_domain->domain.geometry.aperture_start = dma_min; dart_domain->domain.geometry.aperture_end = dma_max; dart_domain->domain.geometry.force_aperture = true; dart_domain->dma_offset = dart->dma_offset; @@ -1003,9 +1294,11 @@ static int apple_dart_attach_dev_paging(struct iommu_domain *domain, for_each_stream_map(i, cfg, stream_map) { if (!stream_map->dart->locked) apple_dart_setup_translation(dart_domain, stream_map); - else - apple_dart_setup_translation_locked(dart_domain, - stream_map); + else { + ret = apple_dart_setup_translation_locked(dart_domain, stream_map); + if (ret) + goto err; + } } err: @@ -1098,14 +1391,23 @@ static struct iommu_device *apple_dart_probe_device(struct device *dev) static void apple_dart_release_device(struct device *dev) { - int i, j; + int i, j, ret; struct apple_dart_stream_map *stream_map; struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); for_each_stream_map(j, cfg, stream_map) { - if (stream_map->dart->locked) - for (i = 0; i < stream_map->dart->hw->ttbr_count; ++i) - apple_dart_hw_unmap_locked_ttbr(stream_map, i); + if (!stream_map->dart->locked) + continue; + ret = pm_runtime_resume_and_get(stream_map->dart->dev); + if (ret < 0) { + dev_err_ratelimited(stream_map->dart->dev, + "failed to power DART for consumer release: %d\n", ret); + continue; + } + for (i = 0; i < stream_map->dart->hw->ttbr_count; ++i) + apple_dart_hw_unmap_locked_ttbr(stream_map, i); + (void)stream_map->dart->hw->invalidate_tlb(stream_map); + pm_runtime_put(stream_map->dart->dev); } kfree(cfg); @@ -1123,8 +1425,22 @@ static struct iommu_domain *apple_dart_domain_alloc_paging(struct device *dev) if (dev) { struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); + u64 window[2]; int ret; + if (of_property_present(dev->of_node, "apple,dma-range")) { + ret = of_property_read_u64_array(dev->of_node, "apple,dma-range", + window, 2); + if (!ret) + ret = apple_dart_dma_window(window[0], window[1], + &dart_domain->dma_max); + if (ret) { + kfree(dart_domain); + return ERR_PTR(ret); + } + dart_domain->dma_min = window[0]; + dart_domain->has_dma_range = true; + } ret = apple_dart_finalize_domain(dart_domain, cfg); if (ret) { kfree(dart_domain); @@ -1137,9 +1453,34 @@ static struct iommu_domain *apple_dart_domain_alloc_paging(struct device *dev) static void apple_dart_domain_free(struct iommu_domain *domain) { struct apple_dart_domain *dart_domain = to_dart_domain(domain); - + struct apple_dart_atomic_stream_map *map; + struct apple_dart_stream_map stream; + int i, j, idx; + + for_each_stream_map(i, dart_domain, map) { + if (!map->dart->locked) + continue; + stream.dart = map->dart; + for (j = 0; j < BITS_TO_LONGS(stream.dart->num_streams); j++) + stream.sidmap[j] = atomic_long_read(&map->sidmap[j]); + j = pm_runtime_resume_and_get(stream.dart->dev); + if (j < 0) { + dev_err_ratelimited(stream.dart->dev, + "retaining page tables after power failure: %d\n", j); + return; + } + for (idx = 0; idx < stream.dart->hw->ttbr_count; idx++) + apple_dart_hw_unmap_locked_ttbr(&stream, idx); + j = stream.dart->hw->invalidate_tlb(&stream); + pm_runtime_put(stream.dart->dev); + /* Do not free page tables that hardware may still cache. */ + if (j) { + dev_err_ratelimited(stream.dart->dev, + "retaining page tables after TLB invalidation failure: %d\n", j); + return; + } + } free_io_pgtable_ops(dart_domain->pgtbl_ops); - kfree(dart_domain); } @@ -1471,7 +1812,7 @@ static irqreturn_t apple_dart_t8020_irq(int irq, void *dev) return IRQ_HANDLED; } -static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) +static irqreturn_t apple_dart_t8110_legacy_irq(int irq, void *dev) { struct apple_dart *dart = dev; const char *fault_name = NULL; @@ -1514,6 +1855,147 @@ static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) return IRQ_HANDLED; } +static const char *apple_dart_t8110_v2_fault_name(u32 exception) +{ + switch (FIELD_GET(DART_T8110_EXCEPTION_CODE, exception)) { + case 0: + switch (FIELD_GET(DART_T8110_EXCEPTION_LEVEL, exception)) { + case 0: + return "TTBR invalid"; + case 1: + return "CTE invalid"; + case 2: + return "STE invalid"; + case 3: + return "PTE invalid"; + } + break; + case 1: + return "write protect"; + case 2: + return "read protect"; + case 3: + return "drop protect"; + case 4: + return "AXI decode"; + case 5: + return "AXI error"; + case 6: + return "fill region"; + case 7: + return "region protect"; + case 8: + return "CTRR write protect"; + case 9: + return "APF reject"; + case 10: + return "BPF reject"; + case 11: + return "STT mismatch"; + case 12: + return "STT flush"; + case 13: + return "external"; + case 14: + return "illegal SID"; + } + + return "unknown"; +} + +static bool apple_dart_t8110_v2_block_pending(u16 version, u32 block_error) +{ + return version >= 0x0203 && + (block_error & DART_T8110_BLOCK_ERROR_MASK); +} + +static irqreturn_t apple_dart_t8110_v2_irq(int irq, void *dev) +{ + struct apple_dart *dart = dev; + irqreturn_t ret = IRQ_NONE; + unsigned int words = DIV_ROUND_UP(dart->num_streams, 32); + unsigned int i; + u32 block_error; + + /* + * 0x4000 + 4*i is a bitmap of SIDs with a pending exception, one word + * per 32 SIDs -- not a per-SID code. Iterating it per SID (as this + * used to) runs off the end of the array into the register arrays at + * 0x4020/0x4040 for every sid >= ceil(nsid/32). + */ + for (i = 0; i < words && i < 8; i++) { + u32 pending = readl(dart->regs + DART_T8110_EXCEPTION(i)); + unsigned long bits = pending; + unsigned int bit; + + if (!pending) + continue; + + for_each_set_bit(bit, &bits, 32) { + unsigned int sid = i * 32 + bit; + u32 status; + u64 addr; + + if (sid >= dart->num_streams) + break; + + status = readl(dart->regs + + DART_T8110_ERR_SID_STATUS(sid)); + addr = readl(dart->regs + + DART_T8110_EXCEPTION_ADDR_LO(sid)) | + (u64)readl(dart->regs + + DART_T8110_EXCEPTION_ADDR_HI(sid)) << 32; + + dev_err_ratelimited(dart->dev, + "translation fault: SID:%u %s %s at %#llx (status %#010x)\n", + sid, + status & DART_T8110_ERR_SID_STATUS_WRITE ? + "write" : "read", + apple_dart_t8110_v2_fault_name(status), + addr, status); + } + + /* + * Acknowledge: the bitmap is write-1-to-clear. Without this the + * same latched bit re-fires immediately and one historical + * exception looks like a fault storm. + */ + writel(pending, dart->regs + DART_T8110_EXCEPTION(i)); + ret = IRQ_HANDLED; + } + + if (ret == IRQ_HANDLED) { + dsb(sy); + readl(dart->regs + DART_T8110_EXCEPTION(0)); + } + + if (dart->version >= 0x0203) { + block_error = readl(dart->regs + DART_T8110_BLOCK_ERROR); + if (apple_dart_t8110_v2_block_pending(dart->version, block_error)) { + dev_err_ratelimited(dart->dev, + "block exception: status:0x%08x\n", + block_error); + writel(block_error & DART_T8110_BLOCK_ERROR_MASK, + dart->regs + DART_T8110_BLOCK_ERROR); + ret = IRQ_HANDLED; + } + } + if (ret == IRQ_HANDLED) + apple_dart_t8110_v2_unlock(dart); + + return ret; +} + +static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) +{ + struct apple_dart *dart = dev; + + if (dart->version >= 0x0202) + return apple_dart_t8110_v2_irq(irq, dev); + + return apple_dart_t8110_legacy_irq(irq, dev); +} + static irqreturn_t apple_dart_irq(int irq, void *dev) { irqreturn_t ret; @@ -1696,6 +2178,7 @@ static int apple_dart_probe(struct platform_device *pdev) dart->num_streams = FIELD_GET(DART_T8110_PARAMS4_NUM_SIDS, dart_params[3]); dart->four_level = dart->ias > 36; + dart->version = dart_params[2] & GENMASK(15, 0); break; } @@ -1711,15 +2194,22 @@ static int apple_dart_probe(struct platform_device *pdev) goto err_clk_disable; } else { dart->dma_min = dma_range[0]; - dart->dma_max = dma_range[0] + dma_range[1] - 1; + ret = apple_dart_dma_window(dma_range[0], dma_range[1], &dart->dma_max); + if (ret) + goto err_clk_disable; + if (!apple_dart_dma_window_aligned(dart->dma_min, dart->dma_max, + dart->pgsize)) { + ret = -EINVAL; + goto err_clk_disable; + } if ((dart->dma_min ^ dart->dma_max) & ~DMA_BIT_MASK(dart->ias)) { dev_err(&pdev->dev, "Invalid DMA range for ias=%d\n", dart->ias); ret = -EINVAL; goto err_clk_disable; } - dev_info(&pdev->dev, "Limiting DMA range to %pad..%pad\n", - &dart->dma_min, &dart->dma_max); + dev_dbg(&pdev->dev, "Limiting DMA range to %pad..%pad\n", + &dart->dma_min, &dart->dma_max); } ret = of_property_read_u64(dev->of_node, "apple,dma-offset", @@ -1769,23 +2259,24 @@ static int apple_dart_probe(struct platform_device *pdev) goto err_clk_disable; } - /* - * The hot-plug PCIe-C DART is freshly made available for this tunnel and - * Apple's ADT does not describe a retained/locked translation context. - * Its T8110 PROTECT register is not readable from this activation state, - * so do not turn that optional observation into a fatal external abort. - */ - if (tunneled) { + ret = apple_dart_read_remaps(dart); + if (ret) + goto err_clk_disable; + + /* A hot-plug PCIe-C DART has no retained/locked translation context. */ + if (tunneled) dart->locked = false; - } else { + else dart->locked = apple_dart_is_locked(dart); - } if (!dart->locked) { ret = apple_dart_hw_reset(dart); if (ret) goto err_clk_disable; } + if (dart->locked) + apple_dart_drain_stale_exceptions(dart); + ret = request_irq(dart->irq, apple_dart_irq, IRQF_SHARED, "apple-dart fault handler", dart); if (ret) @@ -1802,7 +2293,7 @@ static int apple_dart_probe(struct platform_device *pdev) pm_runtime_put(dev); - dev_info( + dev_dbg( &pdev->dev, "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d, locked: %d, AS %d -> %d] initialized\n", dart->pgsize, dart->num_streams, dart->supports_bypass, From a0abd0694fffea1933cd5cf5592f92467350b7df Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 21:29:31 +1000 Subject: [PATCH 0158/2680] arm64: dts: apple: t8140: Describe observed behaviour, not other implementations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several comments in the T8140 support explained a register value or a behaviour by what another operating system's implementation does, not by what the hardware and firmware do. Reword them to state the observed behaviour directly: the ambient light sensor's ADT properties, the secure ADMAC channel's access path, the AOP calibration message and endpoint set, the ISP capture watchdog, and the speaker unmute value. No functional change. [ryan: adapted to 7.2.10 — split: DT part only; the isp-cam.c, aop.rs and t8140_aop_audio.rs parts go with 240-isp, 010-soc and 070-audio] Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index aa51d7a46e6a7..d9d7b3103115e 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -802,7 +802,7 @@ * appears as als.1.auto and sits unbound without this * node); the ADT has /arm-io/aop/iop-aop-nub/als with * als-supports-color = 0, an ALS serial in /product and - * macOS's auto-brightness curve in /exbright. The driver + * a brightness curve in /exbright. The driver * matches on the generic compatible only and requires * the machine-supplied CT817 setup calibration. */ @@ -878,7 +878,8 @@ * report windows differ from t8103 and are described by the * t8140 driver variant, not by older SoC offsets. DMA streams: * dart-aop mapper-admac-base-ns 8, mapper-admac-leap-ns 10, - * mapper-admac-leap-s 11 (exclave-gated under macOS). + * mapper-admac-leap-s 11 (reached through the proxy aperture + * described below). */ admac_base_ns: dma-controller@313800000 { compatible = "apple,t8140-admac"; From d73f9acd164108fb297b3b3a59ef77cfe335a701 Mon Sep 17 00:00:00 2001 From: Ace <7362641+Acelogic@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:22:25 -0400 Subject: [PATCH 0159/2680] arm64: dts: apple: Add diagnostic J700 PCIe root-port topology Describe the observed T8140 PCIe and DART resources and enable only J700 port0 with its observed reset, clock-request and SMC power controls. Keep the DART disabled and omit endpoint routing. This topology relies on the matching driver's unconditional downstream configuration guard. The apple,t8140-pcie compatible also preserves the pinned public U-Boot pre-MMIO refusal for this host. The candidate kernel/DTB reached root bridge enumeration with link up; downstream devices remain inaccessible. This board enablement is a diagnostic scaffold pending PIODMA support. --- arch/arm64/boot/dts/apple/t8140-j700.dts | 20 +++++++++ arch/arm64/boot/dts/apple/t8140.dtsi | 56 ++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 270932d29e966..4e5bf87f4abe7 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -1349,3 +1349,23 @@ #include "hwmon-common.dtsi" #include "hwmon-laptop.dtsi" + +&pcie0 { + /* Staged root/link probe: the host driver blocks downstream config. */ + pinctrl-0 = <&pcie0_clkreq>; + pinctrl-names = "default"; + status = "okay"; +}; + +&pcie0_port0 { + reset-gpios = <&pinctrl_ap 114 GPIO_ACTIVE_LOW>; + pwren-gpios = <&smc_gpio 8 GPIO_ACTIVE_HIGH>; + max-link-speed = <2>; + status = "okay"; +}; + +&pinctrl_ap { + pcie0_clkreq: pcie0-clkreq { + pinmux = ; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index d9d7b3103115e..5ffe82f64ca3d 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -588,6 +588,62 @@ status = "disabled"; }; + pcie0: pcie@1cb0000000 { + compatible = "apple,t8140-pcie"; + device_type = "pci"; + reg = <0x1c 0xb0000000 0x0 0x10000000>, + <0x3 0x94000000 0x0 0x4000>, + <0x3 0x90028000 0x0 0x8000>, + <0x3 0x91028000 0x0 0x8000>, + <0x3 0x92028000 0x0 0x8000>, + <0x3 0x97020000 0x0 0x4000>, + <0x3 0x97024000 0x0 0x4000>, + <0x3 0x97028000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "port1", "port2", + "phy0", "phy1", "phy2"; + interrupt-parent = <&aic>; + interrupts = , + , + ; + msi-controller; + msi-parent = <&pcie0>; + /* Only port 0 is described; it owns the first 32 vectors. */ + msi-ranges = <&aic AIC_IRQ 1331 IRQ_TYPE_EDGE_RISING 32>; + bus-range = <0 1>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0xb 0xc0000000 0xb 0xc0000000 0x0 0x20000000>, + <0x02000000 0x0 0x80000000 0xb 0x80000000 0x0 0x40000000>; + power-domains = <&ps_apcie_gp>; + status = "disabled"; + + pcie0_port0: pci@0,0 { + device_type = "pci"; + reg = <0 0 0 0 0>; + #address-cells = <3>; + #size-cells = <2>; + ranges; + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie0_port0 0 0 0 0>, + <0 0 0 2 &pcie0_port0 0 0 0 1>, + <0 0 0 3 &pcie0_port0 0 0 0 2>, + <0 0 0 4 &pcie0_port0 0 0 0 3>; + status = "disabled"; + }; + }; + + pcie0_dart: iommu@390000000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x3 0x90000000 0x0 0x20000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_apcie_gp>; + status = "disabled"; + }; + dwc3_0: usb@40a280000 { compatible = "apple,t8140-dwc3", "apple,t8103-dwc3"; reg = <0x4 0x0a280000 0x0 0xcd00>, From 3688970dae75709cdce25fe5863a85507c961d5e Mon Sep 17 00:00:00 2001 From: Ace <7362641+Acelogic@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:22:30 -0400 Subject: [PATCH 0160/2680] arm64: dts: apple: t8140-j700: enable bounded PCIe endpoint bootstrap and enumeration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prime one retained PIODMA request behind an explicit opt-in, validate its completion, then permit typed ECAM access to the two expected radio functions. Program and check the root memory forwarding window and translated endpoint DMA mappings. Add the J700 own-MAC bootloader handoff node. This remains a diagnostic bring-up path. The arena is retained until full reset; removal, suspend, kexec and memory reuse are not qualified. Document those merge blockers rather than treating one completed request as teardown. Built and tested on physical J700/T8140 with public Aurora m1n1/U-Boot/Linux: KDE, WPA2/DHCP/DNS/HTTPS on 2.4 and 5 GHz, Bluetooth discovery and HCI power cycling. Binding and J700 DTB schema checks pass. Executable PCI code matches the tested Image; firmware and unit data are not included. [ryan: adapted to 7.2.10 — split: DT part only; the pcie-apple driver and documentation go with 080-radio] --- arch/arm64/boot/dts/apple/t8140-j700.dts | 37 +++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 4e5bf87f4abe7..cc9f115adb7ab 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -19,6 +19,7 @@ atcphy0 = &atcphy0; keyboard = &keyboard; serial0 = &serial0; + wifi0 = &wifi0; }; chosen { @@ -1351,7 +1352,10 @@ #include "hwmon-laptop.dtsi" &pcie0 { - /* Staged root/link probe: the host driver blocks downstream config. */ + /* Experiment: guard stays closed until the retained bootstrap succeeds. */ + apple,piodma = <&pcie0_piodma_diag>; + iommu-map = <0x100 &pcie0_dart 1 2>; + iommu-map-mask = <0xffff>; pinctrl-0 = <&pcie0_clkreq>; pinctrl-names = "default"; status = "okay"; @@ -1369,3 +1373,34 @@ pinmux = ; }; }; + +/* Experimental SID17 bootstrap; arena ownership lasts until full reset. */ +&pcie0_dart { + status = "okay"; + apple,dma-range = /bits/ 64 <0 0x1000000000>; +}; + +/ { + pcie0_piodma_diag: dma-controller@390030000 { + compatible = "apple,t8140-piodma-diagnostic"; + reg = <0x3 0x90030000 0x0 0x4000>, + <0x3 0x90034000 0x0 0x4000>, + <0x3 0x90038000 0x0 0x4000>; + reg-names = "engine", "irq-aggregator", "fabric"; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_apcie_gp>; + iommus = <&pcie0_dart 17>; + apple,pcie-host = <&pcie0>; + }; +}; + +&pcie0_port0 { + bus-range = <1 1>; + wifi0: wifi@0,0 { + compatible = "pci14c3,7932"; + reg = <0x10000 0 0 0 0>; + /* Public m1n1 supplies this device's address at boot. */ + local-mac-address = [00 00 00 00 00 00]; + }; +}; From 1ae1748f13c2f4d33a5e46c52de9e92b9522b784 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 2 Oct 2026 00:10:51 -0400 Subject: [PATCH 0161/2680] arm64: dts: apple: enable AVD on T8140 Wire the J700 decoder register ranges, interrupts, shared AVD power/reset domain and three DART streams using the existing Neo bring-up by DJ and Ryan. The open v5-t0 firmware boots and /dev/video0 supports verified hardware decoding on the Neo. Co-authored-by: Ryan Murray Co-authored-by: Ace <7362641+Acelogic@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 8 ++++++++ arch/arm64/boot/dts/apple/t8140.dtsi | 24 +++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index f9058fa8efc30..3fc3ea3ad8fb4 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -39,6 +39,14 @@ label = "isp_sys"; }; + ps_avd_sys: power-controller@270 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x270 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "avd_sys"; + }; + ps_ans: power-controller@278 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x278 4>; diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 5ffe82f64ca3d..2dd921b23368c 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -739,6 +739,30 @@ }; + avd: avd@381080000 { + compatible = "apple,t8140-avd"; + interrupt-parent = <&aic>; + interrupts = , + ; + power-domains = <&ps_avd_sys>; + resets = <&ps_avd_sys>; + reg = <0x3 0x81080000 0x0 0x10000>, + <0x3 0x81090000 0x0 0x10000>, + <0x3 0x810a0000 0x0 0x4000>, + <0x3 0x81100000 0x0 0x10000>; + reg-names = "code", "sram", "mbox", "ctrl"; + iommus = <&avd_dart 0>, <&avd_dart 1>, <&avd_dart 2>; + }; + + avd_dart: iommu@381020000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x3 0x81020000 0x0 0x20000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_avd_sys>; + }; + i2c0: i2c@385010000 { compatible = "apple,t8140-i2c", "apple,t8103-i2c"; reg = <0x3 0x85010000 0x0 0x4000>; From b98d34e3bc3c013b57c3b6829e51762815f194b0 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 10:05:51 -0400 Subject: [PATCH 0162/2680] arm64: apple: add initial J813 M5 RAM boot support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the measured J813 CPU affinities, AICv3 windows, DockChannel console and loader-populated framebuffer and memory. Keep the Samsung UART disabled and omit DMA peripherals. Supply a minimal kernel configuration and document the required m1n1/U-Boot handoff and repeatable boot qualification. Adapt the minimal device tree and configuration from the local J813 bring-up at c8f85bcfb03925179bfa6c19c4d28c44efae39ac. Use T8142 CPU compatibles instead of firmware-inherited M3 core names. This initial scope is a single-CPU RAM boot, not complete M5 platform support. [ryan: adapted to 7.2.10 — split: DT part only; documentation and j813.config go with 900-config-tooling] Signed-off-by: Acelogic --- arch/arm64/boot/dts/apple/Makefile | 1 + arch/arm64/boot/dts/apple/t8142-j813.dts | 191 +++++++++++++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813.dts diff --git a/arch/arm64/boot/dts/apple/Makefile b/arch/arm64/boot/dts/apple/Makefile index 4f31beeb11ac4..ef225d6358756 100644 --- a/arch/arm64/boot/dts/apple/Makefile +++ b/arch/arm64/boot/dts/apple/Makefile @@ -110,3 +110,4 @@ dtb-$(CONFIG_ARCH_APPLE) += t8132-j713.dtb dtb-$(CONFIG_ARCH_APPLE) += t8132-j715.dtb dtb-$(CONFIG_ARCH_APPLE) += t8132-j773g.dtb dtb-$(CONFIG_ARCH_APPLE) += t8140-j700.dtb +dtb-$(CONFIG_ARCH_APPLE) += t8142-j813.dtb diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts new file mode 100644 index 0000000000000..3e538bcef76d6 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Air (13-inch, M5, 2026), J813. + * + * Initial RAM boot resources measured from the J813 Apple Device Tree. + * The loader supplies usable memory, CPU release addresses and framebuffer + * geometry. Peripheral power management is left with the boot firmware. + */ +/dts-v1/; + +#include +#include + +/ { + compatible = "apple,j813", "apple,t8142", "apple,arm-platform"; + model = "Apple MacBook Air (13-inch, M5, 2026)"; + + #address-cells = <2>; + #size-cells = <2>; + interrupt-parent = <&aic>; + + /* Request the m1n1 RAM-only handoff, without PCIe/DAPF setup. */ + aurora,minimal-boot; + + aliases { + serial0 = &dockchannel; + serial1 = &serial0; + }; + + chosen { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + stdout-path = "serial0:1500000n8"; + + framebuffer0: framebuffer@0 { + compatible = "apple,simple-framebuffer", "simple-framebuffer"; + reg = <0 0 0 0>; /* Filled by the loader. */ + status = "disabled"; + }; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + /* Filled by the loader. */ + }; + + memory@10000000000 { + device_type = "memory"; + reg = <0x100 0 0x4 0>; /* Filled by the loader. */ + }; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + cpu0: cpu@0 { + compatible = "apple,t8142-e"; + device_type = "cpu"; + reg = <0 0x0>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + + cpu1: cpu@1 { + compatible = "apple,t8142-e"; + device_type = "cpu"; + reg = <0 0x1>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + + cpu2: cpu@2 { + compatible = "apple,t8142-e"; + device_type = "cpu"; + reg = <0 0x2>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + + cpu3: cpu@3 { + compatible = "apple,t8142-e"; + device_type = "cpu"; + reg = <0 0x3>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + + cpu4: cpu@4 { + compatible = "apple,t8142-e"; + device_type = "cpu"; + reg = <0 0x4>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + + cpu5: cpu@5 { + compatible = "apple,t8142-e"; + device_type = "cpu"; + reg = <0 0x5>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + + cpu6: cpu@10100 { + compatible = "apple,t8142-p"; + device_type = "cpu"; + reg = <0 0x10100>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + + cpu7: cpu@10101 { + compatible = "apple,t8142-p"; + device_type = "cpu"; + reg = <0 0x10101>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + + cpu8: cpu@10102 { + compatible = "apple,t8142-p"; + device_type = "cpu"; + reg = <0 0x10102>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + + cpu9: cpu@10103 { + compatible = "apple,t8142-p"; + device_type = "cpu"; + reg = <0 0x10103>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupts = , + , + , + ; + interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt"; + }; + + clkref: clock-ref { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <24000000>; + }; + + soc { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + ranges; + nonposted-mmio; + + aic: interrupt-controller@381000000 { + compatible = "apple,t8142-aic3", "apple,t8122-aic3"; + interrupt-controller; + #interrupt-cells = <3>; + reg = <3 0x81000000 0 0x1cc000>, + <3 0x81040000 0 0x4000>; + reg-names = "core", "event"; + }; + + serial0: serial@3a5200000 { + compatible = "apple,s5l-uart"; + reg = <3 0xa5200000 0 0x4000>; + reg-io-width = <4>; + interrupts = ; + clocks = <&clkref>, <&clkref>; + clock-names = "uart", "clk_uart_baud0"; + status = "disabled"; + }; + + dockchannel: serial@38812c000 { + compatible = "apple,dockchannel-uart"; + reg = <3 0x8812c000 0 0x4000>, + <3 0x88128000 0 0x4000>, + <3 0x8810c000 0 0x1000>; + reg-names = "data", "config", "irq"; + interrupts = ; + }; + }; +}; From a17a9f49c3214f644eb76e1b3c520e0322a64e4c Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 10:42:22 -0400 Subject: [PATCH 0163/2680] arm64: dts: apple: use the native FDT handoff on J813 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — split: DT part only; bindings go with 010-soc and documentation with 900-config-tooling] --- arch/arm64/boot/dts/apple/t8142-j813.dts | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index 3e538bcef76d6..d56cd1dd2f560 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -19,9 +19,6 @@ #size-cells = <2>; interrupt-parent = <&aic>; - /* Request the m1n1 RAM-only handoff, without PCIe/DAPF setup. */ - aurora,minimal-boot; - aliases { serial0 = &dockchannel; serial1 = &serial0; From 633cc25279b4ae3efed7e1cb296cf130c346ce4c Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 09:58:19 -0400 Subject: [PATCH 0164/2680] arm64: dts: apple: Add initial M6 J873g RAM boot support Describe the twelve CPU affinities, architectural timer, AIC3, DockChannel console and loader-filled framebuffer measured on J873g/T8152. Use the existing AIC3 implementation; retained T8152 probes show that the guest-timer mask register is accessible. The qualified m1n1 loader must supply memory reservations and CPU release addresses while preserving firmware peripheral state. Storage, USB, networking, GPU, cpufreq and thermal drivers are not enabled by this initial tree. Signed-off-by: Acelogic --- arch/arm64/boot/dts/apple/Makefile | 1 + arch/arm64/boot/dts/apple/t8152-j873g.dts | 190 ++++++++++++++++++++++ 2 files changed, 191 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8152-j873g.dts diff --git a/arch/arm64/boot/dts/apple/Makefile b/arch/arm64/boot/dts/apple/Makefile index ef225d6358756..3edac61fba143 100644 --- a/arch/arm64/boot/dts/apple/Makefile +++ b/arch/arm64/boot/dts/apple/Makefile @@ -111,3 +111,4 @@ dtb-$(CONFIG_ARCH_APPLE) += t8132-j715.dtb dtb-$(CONFIG_ARCH_APPLE) += t8132-j773g.dtb dtb-$(CONFIG_ARCH_APPLE) += t8140-j700.dtb dtb-$(CONFIG_ARCH_APPLE) += t8142-j813.dtb +dtb-$(CONFIG_ARCH_APPLE) += t8152-j873g.dtb diff --git a/arch/arm64/boot/dts/apple/t8152-j873g.dts b/arch/arm64/boot/dts/apple/t8152-j873g.dts new file mode 100644 index 0000000000000..bd259e3f7e364 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8152-j873g.dts @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple Mac mini (M6), J873g. + * + * Initial RAM boot resources measured on J873g/T8152. + * The loader supplies usable memory, CPU release addresses and framebuffer + * geometry. Peripheral power management is left with the boot firmware. + */ +/dts-v1/; + +#include +#include + +/ { + compatible = "apple,j873g", "apple,t8152", "apple,arm-platform"; + model = "Apple Mac mini (M6)"; + + #address-cells = <2>; + #size-cells = <2>; + interrupt-parent = <&aic>; + + /* Request the m1n1 RAM-only handoff, without PCIe/DAPF setup. */ + aurora,minimal-boot; + + aliases { + serial0 = &dockchannel; + }; + + chosen { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + stdout-path = "serial0:1500000n8"; + + framebuffer0: framebuffer@0 { + compatible = "apple,simple-framebuffer", "simple-framebuffer"; + reg = <0 0 0 0>; /* Filled by the loader. */ + status = "disabled"; + }; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + /* Filled by the loader. */ + }; + + memory@10000000000 { + device_type = "memory"; + reg = <0x100 0 0 0>; /* Filled by the loader. */ + }; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + cpu0: cpu@0 { + compatible = "apple,t8152-e"; + device_type = "cpu"; + reg = <0 0x0>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + + cpu1: cpu@1 { + compatible = "apple,t8152-e"; + device_type = "cpu"; + reg = <0 0x1>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + + cpu2: cpu@2 { + compatible = "apple,t8152-e"; + device_type = "cpu"; + reg = <0 0x2>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + + cpu3: cpu@3 { + compatible = "apple,t8152-e"; + device_type = "cpu"; + reg = <0 0x3>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + + cpu4: cpu@4 { + compatible = "apple,t8152-e"; + device_type = "cpu"; + reg = <0 0x4>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + + cpu5: cpu@5 { + compatible = "apple,t8152-e"; + device_type = "cpu"; + reg = <0 0x5>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + + cpu6: cpu@10100 { + compatible = "apple,t8152-p"; + device_type = "cpu"; + reg = <0 0x10100>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + + cpu7: cpu@10101 { + compatible = "apple,t8152-p"; + device_type = "cpu"; + reg = <0 0x10101>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + + cpu8: cpu@40102 { + compatible = "apple,t8152-m"; + device_type = "cpu"; + reg = <0 0x40102>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + + cpu9: cpu@40103 { + compatible = "apple,t8152-m"; + device_type = "cpu"; + reg = <0 0x40103>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + + cpu10: cpu@40104 { + compatible = "apple,t8152-m"; + device_type = "cpu"; + reg = <0 0x40104>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + + cpu11: cpu@40105 { + compatible = "apple,t8152-m"; + device_type = "cpu"; + reg = <0 0x40105>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* Filled by the loader. */ + }; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupts = , + , + , + ; + interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt"; + }; + + soc { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + ranges; + nonposted-mmio; + + aic: interrupt-controller@301000000 { + compatible = "apple,t8152-aic3", "apple,t8122-aic3"; + interrupt-controller; + #interrupt-cells = <3>; + reg = <3 0x01000000 0 0x1cc000>, + <3 0x01040000 0 0x4000>; + reg-names = "core", "event"; + }; + + dockchannel: serial@30812c000 { + compatible = "apple,dockchannel-uart"; + reg = <3 0x0812c000 0 0x4000>, + <3 0x08128000 0 0x4000>, + <3 0x0810c000 0 0x1000>; + reg-names = "data", "config", "irq"; + interrupts = ; + }; + }; +}; From 52db25473bbfb6772836ccbd225cd34fd25e8a6d Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 12:26:30 -0400 Subject: [PATCH 0165/2680] arm64: dts: apple: t8152-j873g: Add bounded E and P/M performance domains MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — split: DT part only; the cpufreq driver, binding and documentation go with 060-power-thermal] --- arch/arm64/boot/dts/apple/t8152-j873g.dts | 59 +++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8152-j873g.dts b/arch/arm64/boot/dts/apple/t8152-j873g.dts index bd259e3f7e364..6b5e14036c98a 100644 --- a/arch/arm64/boot/dts/apple/t8152-j873g.dts +++ b/arch/arm64/boot/dts/apple/t8152-j873g.dts @@ -62,6 +62,8 @@ reg = <0 0x0>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_e_opp>; + performance-domains = <&m6_e_cpufreq>; }; cpu1: cpu@1 { @@ -70,6 +72,8 @@ reg = <0 0x1>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_e_opp>; + performance-domains = <&m6_e_cpufreq>; }; cpu2: cpu@2 { @@ -78,6 +82,8 @@ reg = <0 0x2>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_e_opp>; + performance-domains = <&m6_e_cpufreq>; }; cpu3: cpu@3 { @@ -86,6 +92,8 @@ reg = <0 0x3>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_e_opp>; + performance-domains = <&m6_e_cpufreq>; }; cpu4: cpu@4 { @@ -94,6 +102,8 @@ reg = <0 0x4>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_e_opp>; + performance-domains = <&m6_e_cpufreq>; }; cpu5: cpu@5 { @@ -102,6 +112,8 @@ reg = <0 0x5>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_e_opp>; + performance-domains = <&m6_e_cpufreq>; }; cpu6: cpu@10100 { @@ -110,6 +122,8 @@ reg = <0 0x10100>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_pm_opp>; + performance-domains = <&m6_pm_cpufreq>; }; cpu7: cpu@10101 { @@ -118,6 +132,8 @@ reg = <0 0x10101>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_pm_opp>; + performance-domains = <&m6_pm_cpufreq>; }; cpu8: cpu@40102 { @@ -126,6 +142,8 @@ reg = <0 0x40102>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_pm_opp>; + performance-domains = <&m6_pm_cpufreq>; }; cpu9: cpu@40103 { @@ -134,6 +152,8 @@ reg = <0 0x40103>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_pm_opp>; + performance-domains = <&m6_pm_cpufreq>; }; cpu10: cpu@40104 { @@ -142,6 +162,8 @@ reg = <0 0x40104>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_pm_opp>; + performance-domains = <&m6_pm_cpufreq>; }; cpu11: cpu@40105 { @@ -150,9 +172,35 @@ reg = <0 0x40105>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_pm_opp>; + performance-domains = <&m6_pm_cpufreq>; }; }; + m6_e_opp: opp-table-e { + compatible = "operating-points-v2"; + opp-shared; + opp-972000000 { + opp-hz = /bits/ 64 <972000000>; + opp-level = <2>; + }; + opp-1152000000 { + opp-hz = /bits/ 64 <1152000000>; + opp-level = <3>; + }; + }; + m6_pm_opp: opp-table-pm { + compatible = "operating-points-v2"; + opp-shared; + opp-1440000000 { + opp-hz = /bits/ 64 <1440000000>; + opp-level = <2>; + }; + opp-1728000000 { + opp-hz = /bits/ 64 <1728000000>; + opp-level = <3>; + }; + }; timer { compatible = "arm,armv8-timer"; interrupts = , @@ -178,6 +226,17 @@ reg-names = "core", "event"; }; + m6_e_cpufreq: performance-controller@210e20000 { + compatible = "apple,t8152-cluster-cpufreq"; + reg = <2 0x10e20000 0 0x1000>; + #performance-domain-cells = <0>; + }; + m6_pm_cpufreq: performance-controller@211e20000 { + compatible = "apple,t8152-cluster-cpufreq"; + reg = <2 0x11e20000 0 0x1000>; + #performance-domain-cells = <0>; + }; + dockchannel: serial@30812c000 { compatible = "apple,dockchannel-uart"; reg = <3 0x0812c000 0 0x4000>, From a8059700ff13e669261d63813889234567fad8f3 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 12:37:37 -0400 Subject: [PATCH 0166/2680] arm64: dts: apple: enable bounded J813 CPU frequency control MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe both shared cluster policies and enable the userspace governor in the J813 configuration. Expose only the states up to the observed firmware handoff: E state 2 at 972 MHz and P state 10 at 3516 MHz. State 1 is a 300 MHz floor before the firmware SRAM table entries. Dependent decrement-loop timing on J813 confirmed every exposed state's frequency. Document repeatable state sweeps and qualification limits. [ryan: adapted to 7.2.10 — split: DT part only; documentation and j813.config go with 900-config-tooling] Signed-off-by: Acelogic --- arch/arm64/boot/dts/apple/t8142-j813.dts | 104 +++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index d56cd1dd2f560..e47839a68dfe4 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -58,6 +58,8 @@ compatible = "apple,t8142-e"; device_type = "cpu"; reg = <0 0x0>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -66,6 +68,8 @@ compatible = "apple,t8142-e"; device_type = "cpu"; reg = <0 0x1>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -74,6 +78,8 @@ compatible = "apple,t8142-e"; device_type = "cpu"; reg = <0 0x2>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -82,6 +88,8 @@ compatible = "apple,t8142-e"; device_type = "cpu"; reg = <0 0x3>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -90,6 +98,8 @@ compatible = "apple,t8142-e"; device_type = "cpu"; reg = <0 0x4>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -98,6 +108,8 @@ compatible = "apple,t8142-e"; device_type = "cpu"; reg = <0 0x5>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -106,6 +118,8 @@ compatible = "apple,t8142-p"; device_type = "cpu"; reg = <0 0x10100>; + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -114,6 +128,8 @@ compatible = "apple,t8142-p"; device_type = "cpu"; reg = <0 0x10101>; + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -122,6 +138,8 @@ compatible = "apple,t8142-p"; device_type = "cpu"; reg = <0 0x10102>; + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -130,6 +148,8 @@ compatible = "apple,t8142-p"; device_type = "cpu"; reg = <0 0x10103>; + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -144,6 +164,78 @@ interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt"; }; + /* State 1 is 300 MHz; firmware SRAM OPP tables begin at state 2. */ + /* Cap each cluster at its measured J813 firmware handoff state. */ + ecluster_opp: opp-table-0 { + compatible = "operating-points-v2"; + opp-shared; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-level = <1>; + }; + + opp-972000000 { + opp-hz = /bits/ 64 <972000000>; + opp-level = <2>; + }; + }; + + pcluster_opp: opp-table-1 { + compatible = "operating-points-v2"; + opp-shared; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-level = <1>; + }; + + opp-1308000000 { + opp-hz = /bits/ 64 <1308000000>; + opp-level = <2>; + }; + + opp-1608000000 { + opp-hz = /bits/ 64 <1608000000>; + opp-level = <3>; + }; + + opp-1932000000 { + opp-hz = /bits/ 64 <1932000000>; + opp-level = <4>; + }; + + opp-2220000000 { + opp-hz = /bits/ 64 <2220000000>; + opp-level = <5>; + }; + + opp-2508000000 { + opp-hz = /bits/ 64 <2508000000>; + opp-level = <6>; + }; + + opp-2820000000 { + opp-hz = /bits/ 64 <2820000000>; + opp-level = <7>; + }; + + opp-3072000000 { + opp-hz = /bits/ 64 <3072000000>; + opp-level = <8>; + }; + + opp-3300000000 { + opp-hz = /bits/ 64 <3300000000>; + opp-level = <9>; + }; + + opp-3516000000 { + opp-hz = /bits/ 64 <3516000000>; + opp-level = <10>; + }; + }; + clkref: clock-ref { compatible = "fixed-clock"; #clock-cells = <0>; @@ -157,6 +249,18 @@ ranges; nonposted-mmio; + cpufreq_e: performance-controller@210e20000 { + compatible = "apple,t8142-cluster-cpufreq"; + reg = <0x2 0x10e20000 0 0x1000>; + #performance-domain-cells = <0>; + }; + + cpufreq_p: performance-controller@211e20000 { + compatible = "apple,t8142-cluster-cpufreq"; + reg = <0x2 0x11e20000 0 0x1000>; + #performance-domain-cells = <0>; + }; + aic: interrupt-controller@381000000 { compatible = "apple,t8142-aic3", "apple,t8122-aic3"; interrupt-controller; From 8c163eab168fcf888fe10fa27a2b6a7915a8df44 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 18:48:20 -0400 Subject: [PATCH 0167/2680] arm64: apple: enable J813 temperatures, input and keyboard backlight MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe the MTP, SMC and PWM resources needed by the existing generic drivers. Request trackpad firmware externally, obtain keyboard layout through the loader, and use runtime sensor discovery with a linear LED brightness scale. [ryan: adapted to 7.2.10 — split: DT part only; j813.config goes with 900-config-tooling] Signed-off-by: Acelogic --- .../dts/apple/t8142-j813-peripherals.dtsi | 137 ++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 2 + 2 files changed, 139 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi new file mode 100644 index 0000000000000..a6708b1c972df --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +#include + +/ { + aliases { + mtp = &mtp; + keyboard = &keyboard; + }; + + led-controller { + compatible = "pwm-leds"; + + led-0 { + pwms = <&fpwm0 0 40000>; + label = "kbd_backlight"; + function = LED_FUNCTION_KBD_BACKLIGHT; + color = ; + max-brightness = <255>; + default-state = "keep"; + }; + }; +}; + +&{/soc} { + pmgr: power-management@380700000 { + compatible = "apple,t8142-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <3 0x80700000 0 0x4000>; + #address-cells = <1>; + #size-cells = <1>; + + ps_fpwm0: power-controller@330 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x330 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm0"; + }; + }; + + fpwm0: pwm@3a5040000 { + compatible = "apple,t8142-fpwm", "apple,s5l-fpwm"; + reg = <3 0xa5040000 0 0x4000>; + power-domains = <&ps_fpwm0>; + clocks = <&clkref>; + #pwm-cells = <2>; + }; + + smc_mbox: mailbox@38c608000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <3 0x8c608000 0 0x4000>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + smc: smc@38c600000 { + compatible = "apple,t8142-smc", "apple,t8103-smc"; + reg = <3 0x8c600000 0 0x4000>, <3 0x8de00000 0 0x120000>; + reg-names = "smc", "sram"; + mboxes = <&smc_mbox>; + + hwmon { + compatible = "apple,smc-hwmon"; + apple,discover-temperatures; + }; + }; + + mtp: mtp@394600000 { + compatible = "apple,t8142-mtp", "apple,t8142-rtk-helper-asc4", + "apple,mtp", "apple,rtk-helper-asc4"; + /* The second region is firmware SRAM, not the ASC control window. */ + reg = <3 0x94600000 0 0x4000>, <3 0x94c00000 0 0x100000>; + reg-names = "asc", "sram"; + mboxes = <&mtp_mbox>; + iommus = <&mtp_dart 0>; + }; + + mtp_mbox: mailbox@394608000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <3 0x94608000 0 0x4000>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + mtp_dart: iommu@394800000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <3 0x94800000 0 0xc000>; + interrupts = ; + #iommu-cells = <1>; + /* Keep host allocations clear of firmware's low IOVA mappings. */ + apple,dma-range = <0 0x02000000 0 0x10000000>; + }; + + mtp_dockchannel: fifo@394b14000 { + compatible = "apple,t8142-dockchannel", "apple,dockchannel"; + reg = <3 0x94b14000 0 0x1000>; + reg-names = "irq"; + interrupts = ; + ranges = <0 3 0x94b28000 0x20000>; + nonposted-mmio; + #address-cells = <1>; + #size-cells = <1>; + interrupt-controller; + #interrupt-cells = <2>; + + mtp_hid: input@8000 { + compatible = "apple,dockchannel-hid"; + reg = <0x8000 0x1000>, <0xc000 0x1000>, + <0x0000 0x1000>, <0x4000 0x1000>; + reg-names = "config", "data", "rmt-config", "rmt-data"; + iommus = <&mtp_dart 0>; + interrupt-parent = <&mtp_dockchannel>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + + stm { }; + + multi-touch { + firmware-name = "apple/tpmtfw-j813.bin"; + apple,power-method = <2>; + apple,firmware-managed-reset; + }; + + /* Layout is filled by the loader using this machine's own data. */ + keyboard: keyboard { }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index e47839a68dfe4..b0a8ac8339955 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -290,3 +290,5 @@ }; }; }; + +#include "t8142-j813-peripherals.dtsi" From fd1c995f0120a1de71e1da4af8a1a3883d53f352 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 21:03:28 -0400 Subject: [PATCH 0168/2680] arm64: dts: apple: t8142-j813: add verified M5 USB3 host support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — split: DT part only; drivers and bindings go with 170-usb, j813.config with 900-config-tooling] --- arch/arm64/boot/dts/apple/t8142-j813-usb.dtsi | 323 ++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + 2 files changed, 324 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-usb.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-usb.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-usb.dtsi new file mode 100644 index 0000000000000..e27e40c04e268 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-usb.dtsi @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +#include +#include + +/ { + aliases { + atcphy0 = &atcphy0; + usb0 = &dwc3_0; + atcphy1 = &atcphy1; + usb1 = &dwc3_1; + }; +}; + +&pmgr { + ps_atc0_common: power-controller@2c8 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0-common"; + }; +}; + +&{/soc} { + pmgr_nub: power-management@388280000 { + compatible = "apple,t8142-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <3 0x88280000 0 0x4000>; + #address-cells = <1>; + #size-cells = <1>; + + ps_nub_spmi_a0: power-controller@70 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x70 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub-spmi-a0"; + }; + + ps_atc0_usb_aon: power-controller@c8 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0-usb-aon"; + }; + + ps_atc0_usb: power-controller@d0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xd0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_atc0_common>, <&ps_atc0_usb_aon>; + label = "atc0-usb"; + }; + }; + + nub_spmi_a0: spmi@388908000 { + compatible = "apple,t8142-spmi"; + reg = <3 0x88908000 0 0x4000>; + #address-cells = <2>; + #size-cells = <0>; + power-domains = <&ps_nub_spmi_a0>; + + hpm0: usb-pd@c { + compatible = "apple,sn201202x"; + reg = <0xc SPMI_USID>; + + typec0: connector { + compatible = "usb-c-connector"; + label = "USB-C Left-front"; + power-role = "dual"; + data-role = "host"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + typec0_hs: endpoint { + remote-endpoint = <&dwc3_0_hs>; + }; + }; + + port@1 { + reg = <1>; + typec0_ss: endpoint { + remote-endpoint = <&atcphy0_typec>; + }; + }; + }; + }; + }; + }; + + dwc3_0: usb@402280000 { + compatible = "apple,t8142-dwc3", "apple,t8103-dwc3"; + reg = <4 0x02280000 0 0xcd00>, <4 0x0228cd00 0 0x4b00>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupts = ; + dr_mode = "host"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_0_dart_0 0>, <&dwc3_0_dart_1 1>; + power-domains = <&ps_atc0_usb>; + resets = <&atcphy0>; + phys = <&atcphy0 PHY_TYPE_USB2>, <&atcphy0 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dwc3_0_hs: endpoint { + remote-endpoint = <&typec0_hs>; + }; + }; + port@1 { + reg = <1>; + dwc3_0_ss: endpoint { + remote-endpoint = <&atcphy0_usb>; + }; + }; + }; + }; + + dwc3_0_dart_0: iommu@402f00000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <4 0x02f00000 0 0xc000>; + interrupts = ; + power-domains = <&ps_atc0_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + dwc3_0_dart_1: iommu@402f80000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <4 0x02f80000 0 0xc000>; + interrupts = ; + power-domains = <&ps_atc0_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + atcphy0: phy@403000000 { + compatible = "apple,t8142-atcphy"; + reg = <4 0x03000000 0 0x4c000>, + <4 0x03050000 0 0x8000>, + <4 0x00000000 0 0x8000>, + <4 0x02a90000 0 0x4000>, + <4 0x02a84000 0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", "pipehandler"; + #phy-cells = <1>; + #reset-cells = <0>; + orientation-switch; + mode-switch; + power-domains = <&ps_atc0_usb>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + atcphy0_typec: endpoint { + remote-endpoint = <&typec0_ss>; + }; + }; + port@1 { + reg = <1>; + atcphy0_usb: endpoint { + remote-endpoint = <&dwc3_0_ss>; + }; + }; + }; + }; +}; + +&pmgr { + ps_atc1_common: power-controller@2d0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1-common"; + }; +}; + +&pmgr_nub { + ps_atc1_usb_aon: power-controller@d8 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xd8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1-usb-aon"; + }; + + ps_atc1_usb: power-controller@e0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xe0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_atc1_common>, <&ps_atc1_usb_aon>; + label = "atc1-usb"; + }; +}; + +&nub_spmi_a0 { + hpm1: usb-pd@a { + compatible = "apple,sn201202x"; + reg = <0xa SPMI_USID>; + + typec1: connector { + compatible = "usb-c-connector"; + label = "USB-C Left-rear"; + power-role = "dual"; + data-role = "host"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + typec1_hs: endpoint { + remote-endpoint = <&dwc3_1_hs>; + }; + }; + + port@1 { + reg = <1>; + typec1_ss: endpoint { + remote-endpoint = <&atcphy1_typec>; + }; + }; + }; + }; + }; +}; + +&{/soc} { + dwc3_1: usb@40a280000 { + compatible = "apple,t8142-dwc3", "apple,t8103-dwc3"; + reg = <4 0x0a280000 0 0xcd00>, <4 0x0a28cd00 0 0x4b00>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupts = ; + dr_mode = "host"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_1_dart_0 0>, <&dwc3_1_dart_1 1>; + power-domains = <&ps_atc1_usb>; + resets = <&atcphy1>; + phys = <&atcphy1 PHY_TYPE_USB2>, <&atcphy1 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dwc3_1_hs: endpoint { + remote-endpoint = <&typec1_hs>; + }; + }; + port@1 { + reg = <1>; + dwc3_1_ss: endpoint { + remote-endpoint = <&atcphy1_usb>; + }; + }; + }; + }; + + dwc3_1_dart_0: iommu@40af00000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <4 0x0af00000 0 0xc000>; + interrupts = ; + power-domains = <&ps_atc1_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + dwc3_1_dart_1: iommu@40af80000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <4 0x0af80000 0 0xc000>; + interrupts = ; + power-domains = <&ps_atc1_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + atcphy1: phy@40b000000 { + compatible = "apple,t8142-atcphy"; + reg = <4 0x0b000000 0 0x4c000>, + <4 0x0b050000 0 0x8000>, + <4 0x08000000 0 0x8000>, + <4 0x0aa90000 0 0x4000>, + <4 0x0aa84000 0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", "pipehandler"; + #phy-cells = <1>; + #reset-cells = <0>; + orientation-switch; + mode-switch; + power-domains = <&ps_atc1_usb>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + atcphy1_typec: endpoint { + remote-endpoint = <&typec1_ss>; + }; + }; + port@1 { + reg = <1>; + atcphy1_usb: endpoint { + remote-endpoint = <&dwc3_1_ss>; + }; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index b0a8ac8339955..be04fd061a885 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -292,3 +292,4 @@ }; #include "t8142-j813-peripherals.dtsi" +#include "t8142-j813-usb.dtsi" From a322797452acc32452afb52eca635da691927a13 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 23:34:51 -0400 Subject: [PATCH 0169/2680] arm64: dts: apple: t8142-j813: add the internal display MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe the J813 display coprocessor (DCP), its mailbox and both display DARTs, with resources read from this machine's Apple Device Tree, plus the dcp/disp0/disp0_piodma aliases the loader uses to hand over the live firmware mappings. The DCP and display DART stay disabled until the loader has reserved those mappings and enables them. Enable the Apple DRM driver in the J813 configuration and document the loader contract and the tested result. Tested on J813 from RAM with public m1n1 and U-Boot plus their J813 display and MTP companion changes: DCP completed startup and drove the internal panel at 2560x1600 (notch hidden) through appledrmfb, DPMS off/on cycles completed, and a KDE Plasma Wayland session ran with the built-in keyboard and trackpad. Rendering is software-only. [ryan: adapted to 7.2.10 — split: DT part only; documentation and j813.config go with 900-config-tooling] --- .../boot/dts/apple/t8142-j813-display.dtsi | 95 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + 2 files changed, 96 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-display.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi new file mode 100644 index 0000000000000..a50fa38fd9265 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * J813 internal display pipeline: the DCP coprocessor and its DARTs. + * + * The bootloader leaves DCP running with its firmware mapped through + * locked DART tables. The loader reserves those mappings and enables the + * nodes below; Linux attaches to the running coprocessor without resetting + * it. Resources were read from this machine's Apple Device Tree. + */ + +/ { + aliases { + dcp = &dcp; + disp0 = &display; + disp0_piodma = &disp0_piodma; + }; + + display: display-subsystem { + compatible = "apple,display-subsystem"; + /* disp0_dart must be first since it is locked */ + iommus = <&disp0_dart 0>; + /* generate phandle explicitly for use in loader */ + phandle = <&display>; + }; +}; + +&{/soc} { + pmgr_dcp: power-management@382720000 { + reg = <3 0x82720000 0 0x8>, <3 0x827a4000 0 0x4>; + reg-names = "dcp-bw-scratch", "dcp-bw-doorbell"; + #apple,bw-scratch-cells = <3>; + #apple,bw-doorbell-cells = <2>; + }; + + dcp_mbox: mailbox@286e08000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <2 0x86e08000 0 0x4000>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + disp0_dart: iommu@286300000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <2 0x86300000 0 0xc000>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + dcp_dart: iommu@286340000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <2 0x86340000 0 0x20000>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + }; + + dcp: dcp@286e00000 { + compatible = "apple,t8142-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 23>; + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4", "iop-vbar"; + reg = <2 0x86e00000 0 0x4000>, + <2 0x84000000 0 0xb1c000>, + <2 0x86240000 0 0x10000>, + <2 0x863ac000 0 0x4000>, + <2 0x86800000 0 0x800000>, + <3 0x82700000 0 0x114000>, + <2 0x86850000 0 0x4000>; + apple,bw-scratch = <&pmgr_dcp 0 5 0>; + apple,bw-doorbell = <&pmgr_dcp 1 6>; + /* Scan out the primary plane on IOMFB surface 2. */ + apple,iomfb-surfaces = <0 0 1 0>; + clocks = <&clkref>; + /* generate phandle explicitly for use in loader */ + phandle = <&dcp>; + status = "disabled"; + /* required bus properties for 'piodma' subdevice */ + #address-cells = <2>; + #size-cells = <2>; + + disp0_piodma: piodma { + iommus = <&disp0_dart 16>; + phandle = <&disp0_piodma>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index be04fd061a885..20df27fa1bfe3 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -291,5 +291,6 @@ }; }; +#include "t8142-j813-display.dtsi" #include "t8142-j813-peripherals.dtsi" #include "t8142-j813-usb.dtsi" From 520be2431c90961ce6a2595f52efb808ee274403 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:53:34 -0400 Subject: [PATCH 0170/2680] arm64: dts: apple: t8140-pmgr: Add the GPU power domains Add the power domain of the GPU and its co-processors, and keep the afggfxni0 domain always on. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit fb7b8edf4ee19a35c07c1a2c6aa6b8e65c71777b) --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 3fc3ea3ad8fb4..d1f53903c9a14 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -23,6 +23,23 @@ label = "gpio"; }; + ps_afggfxni0: power-controller@1b8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "afggfxni0"; + apple,always-on; + }; + + ps_gfx: power-controller@1f0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gfx"; + }; + ps_apcie_gp: power-controller@258 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x258 4>; From c11f633daa8ba4502f6357a26600cdb46082dc77 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:53:45 -0400 Subject: [PATCH 0171/2680] arm64: dts: apple: t8140: Add the GPU Add the GPU, the mailboxes of its two co-processors, its performance states and the reserved memory regions it shares with its firmware. The mailbox nodes describe only the mailbox block of each co-processor wrapper, at offset 0x8000. The bootloader fills in the reserved memory regions and the device-specific performance states. It also reserves uat-l2-pagetables for GPU page tables; the GPU node does not reference that region. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e77d74a6be7645e64b9b50bb51713e72a08c9520) --- arch/arm64/boot/dts/apple/t8140.dtsi | 142 +++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 2dd921b23368c..29e8ad3a19ef0 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -251,6 +251,80 @@ }; }; + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* + * NOTE: The operating points are device-specific and must be + * filled in by the bootloader. Entries keep the order of the + * GPU performance states, which is not sorted by frequency. + */ + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <125000>; + }; + opp01 { + opp-hz = /bits/ 64 <338000000>; + opp-microvolt = <605000>; + }; + opp02 { + opp-hz = /bits/ 64 <492000000>; + opp-microvolt = <630000>; + }; + opp03 { + opp-hz = /bits/ 64 <618000000>; + opp-microvolt = <650000>; + }; + opp04 { + opp-hz = /bits/ 64 <796000000>; + opp-microvolt = <695000>; + }; + opp05 { + opp-hz = /bits/ 64 <928000000>; + opp-microvolt = <745000>; + }; + opp06 { + opp-hz = /bits/ 64 <952000000>; + opp-microvolt = <800000>; + }; + opp07 { + opp-hz = /bits/ 64 <1056000000>; + opp-microvolt = <800000>; + }; + opp08 { + opp-hz = /bits/ 64 <1053000000>; + opp-microvolt = <845000>; + }; + opp09 { + opp-hz = /bits/ 64 <1170000000>; + opp-microvolt = <845000>; + }; + opp10 { + opp-hz = /bits/ 64 <1152000000>; + opp-microvolt = <885000>; + }; + opp11 { + opp-hz = /bits/ 64 <1278000000>; + opp-microvolt = <885000>; + }; + opp12 { + opp-hz = /bits/ 64 <1204000000>; + opp-microvolt = <905000>; + }; + opp13 { + opp-hz = /bits/ 64 <1338000000>; + opp-microvolt = <905000>; + }; + opp14 { + opp-hz = /bits/ 64 <1326000000>; + opp-microvolt = <975000>; + }; + opp15 { + opp-hz = /bits/ 64 <1470000000>; + opp-microvolt = <975000>; + }; + }; + timer { compatible = "arm,armv8-timer"; interrupt-parent = <&aic>; @@ -268,6 +342,29 @@ clock-output-names = "clkref"; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + uat_handoff: uat-handoff { + reg = <0 0 0 0>; + }; + + /* Only reserved: the GPU node does not reference it */ + uat_l2_pagetables: uat-l2-pagetables { + reg = <0 0 0 0>; + }; + + uat_pagetables: uat-pagetables { + reg = <0 0 0 0>; + }; + + uat_ttbs: uat-ttbs { + reg = <0 0 0 0>; + }; + }; + soc: soc { compatible = "simple-bus"; #address-cells = <2>; @@ -703,6 +800,51 @@ power-domains = <&ps_atc0_usb>; status = "disabled"; }; + + gpu: gpu@482600000 { + compatible = "apple,agx-t8140"; + reg = <0x4 0x82600000 0x0 0x4000>, + <0x4 0x82e00000 0x0 0x4000>, + <0x4 0x80000000 0x0 0x4000000>; + reg-names = "asc", "asc1", "sgx"; + mboxes = <&gfx_mbox>, <&gfx1_mbox>; + power-domains = <&ps_gfx>; + memory-region = <&uat_ttbs>, <&uat_pagetables>, + <&uat_handoff>; + memory-region-names = "ttbs", "pagetables", "handoff"; + operating-points-v2 = <&gpu_opp>; + status = "disabled"; + }; + + gfx_mbox: mailbox@482608000 { + compatible = "apple,t8140-gfx-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x82608000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + power-domains = <&ps_gfx>; + #mbox-cells = <0>; + status = "disabled"; + }; + + gfx1_mbox: mailbox@482e08000 { + compatible = "apple,t8140-gfx-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x82e08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + power-domains = <&ps_gfx>; + #mbox-cells = <0>; + status = "disabled"; + }; }; }; From f3c79786009b0769c14c6f0ac3490f4f1159c918 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:54:00 -0400 Subject: [PATCH 0172/2680] arm64: dts: apple: t8140-j700: Enable the GPU Enable the GPU and its mailboxes, and add the gpu alias through which the bootloader finds the GPU node. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit f4a46cd97b2a3e70875d76e3cf2b82f3959fd0f5) --- arch/arm64/boot/dts/apple/t8140-j700.dts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index cc9f115adb7ab..06f4d6385465a 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -17,6 +17,7 @@ aliases { atcphy0 = &atcphy0; + gpu = &gpu; keyboard = &keyboard; serial0 = &serial0; wifi0 = &wifi0; @@ -1348,6 +1349,18 @@ }; }; +&gfx_mbox { + status = "okay"; +}; + +&gfx1_mbox { + status = "okay"; +}; + +&gpu { + status = "okay"; +}; + #include "hwmon-common.dtsi" #include "hwmon-laptop.dtsi" From 0fb8c5be2c3d6bf2e4f9135dcd8d0d010b3df8d0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 20:08:23 -0400 Subject: [PATCH 0173/2680] arm64: dts: apple: t8140: describe the GPU mailboxes as ASCWrap v6 The GPU firmware mailboxes were described as "apple,asc-mailbox-v4" windows at 0x482608000 and 0x482e08000. With these nodes the mailbox driver binds its v4 variant, is_ascwrap_v6 stays false, and the ASCWrap v6 CPU lifecycle helpers the GPU driver calls (get_lifecycle, require_safe_cpu_lifecycle, start_cpu, stop_cpu) return -EOPNOTSUPP. The device tree J700 boots with describes both mailboxes as "apple,t8140-ascwrap-v6": the 0x88000 wrapper at 0x482600000 (0x482e00000 for GFX1) plus the 8-byte IOP vector base at 0x482050000 (0x482850000), the send-empty and recv-not-empty interrupts (993/996, 997/1000), and the firmware role. Use the same description here, and give the GPU node the same "asc" (0x88000 wrapper) and "sgx" regions; the driver maps only those two, so the unused "asc1" entry is dropped. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 9c10d7198c8e56b16da35914ffde0eecb6bfb525) --- arch/arm64/boot/dts/apple/t8140.dtsi | 33 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 29e8ad3a19ef0..4dbec938def51 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -803,10 +803,9 @@ gpu: gpu@482600000 { compatible = "apple,agx-t8140"; - reg = <0x4 0x82600000 0x0 0x4000>, - <0x4 0x82e00000 0x0 0x4000>, + reg = <0x4 0x82600000 0x0 0x88000>, <0x4 0x80000000 0x0 0x4000000>; - reg-names = "asc", "asc1", "sgx"; + reg-names = "asc", "sgx"; mboxes = <&gfx_mbox>, <&gfx1_mbox>; power-domains = <&ps_gfx>; memory-region = <&uat_ttbs>, <&uat_pagetables>, @@ -816,31 +815,31 @@ status = "disabled"; }; - gfx_mbox: mailbox@482608000 { - compatible = "apple,t8140-gfx-asc-mailbox", "apple,asc-mailbox-v4"; - reg = <0x4 0x82608000 0x0 0x4000>; + gfx_mbox: mailbox@482600000 { + compatible = "apple,t8140-ascwrap-v6"; + reg = <0x4 0x82600000 0x0 0x88000>, + <0x4 0x82050000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; interrupt-parent = <&aic>; interrupts = , - , - , ; - interrupt-names = "send-empty", "send-not-empty", - "recv-empty", "recv-not-empty"; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX"; power-domains = <&ps_gfx>; #mbox-cells = <0>; status = "disabled"; }; - gfx1_mbox: mailbox@482e08000 { - compatible = "apple,t8140-gfx-asc-mailbox", "apple,asc-mailbox-v4"; - reg = <0x4 0x82e08000 0x0 0x4000>; + gfx1_mbox: mailbox@482e00000 { + compatible = "apple,t8140-ascwrap-v6"; + reg = <0x4 0x82e00000 0x0 0x88000>, + <0x4 0x82850000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; interrupt-parent = <&aic>; interrupts = , - , - , ; - interrupt-names = "send-empty", "send-not-empty", - "recv-empty", "recv-not-empty"; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX1"; power-domains = <&ps_gfx>; #mbox-cells = <0>; status = "disabled"; From 6755811ab734f4f726cc345a47e0aa23b6f1f3fa Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 7 Oct 2026 15:37:55 +1100 Subject: [PATCH 0174/2680] arm64: dts: apple: t8140: add disabled display power domains The Linux-side ADT record R-PMGR-static identifies six physical display power-state registers and their internal and external parent chains. Describe them using the existing PMGR power-state binding. The common fabric parent and DCP power-gate identifiers are virtual ADT devices without power-state registers, so do not create physical controllers for those identifiers. Keep every new domain disabled. The system domains require firmware notification, whose transition ordering has not yet been qualified. This preserves the inherited display power state while providing the resource descriptions needed for subsequent display integration. Source: Linux ADT /arm-io/pmgr, decoded with public Asahi m1n1. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 58 +++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index d1f53903c9a14..175a64d69e40f 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -73,6 +73,24 @@ apple,always-on; /* handed over running, cannot be reset */ }; + ps_disp_sys: power-controller@280 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x280 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_sys"; + status = "disabled"; + }; + + ps_dispext0_sys: power-controller@288 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x288 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_sys"; + status = "disabled"; + }; + ps_apcie_sys_gp: power-controller@2a0 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x2a0 4>; @@ -99,6 +117,36 @@ power-domains = <&ps_ans>; }; + ps_disp_fe: power-controller@318 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x318 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_fe"; + power-domains = <&ps_disp_sys>; + status = "disabled"; + }; + + ps_dispext0_fe: power-controller@320 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x320 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_fe"; + power-domains = <&ps_dispext0_sys>; + status = "disabled"; + }; + + ps_dispext0_cpu: power-controller@328 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x328 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_cpu"; + power-domains = <&ps_dispext0_fe>; + status = "disabled"; + }; + ps_apcie_sys_st: power-controller@3a0 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x3a0 4>; @@ -275,6 +323,16 @@ #reset-cells = <0>; label = "isp_cvm"; }; + ps_disp_cpu: power-controller@10000 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x10000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_cpu"; + power-domains = <&ps_disp_fe>; + status = "disabled"; + }; + }; &pmgr_mini { From 9350344d2ded915de643630bffff6fcf304d8425 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 7 Oct 2026 16:38:21 +1100 Subject: [PATCH 0175/2680] arm64: dts: apple: t8140: add disabled internal display DARTs Describe the internal display, display GRT and DCP translation controllers from the Linux-exported ADT. Translate their first register windows through the ADT bus ranges and retain the shared interrupt. These controllers use the existing T8110 DART register interface. Only the translation controller windows are needed by the public DART driver. Do not expose the additional ADT windows as DART registers. Keep all three nodes disabled while display power transitions and consumer lifetimes remain unqualified. DMA apertures and consumer stream assignments will be described with their qualified consumers. Source: Linux ADT /arm-io/dart-disp0, dart-dispgrt and dart-dcp, decoded with public Asahi m1n1 (R-ADT). Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 4dbec938def51..4144e0b264fec 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -848,6 +848,33 @@ }; &soc { + disp0_dart: iommu@412300000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x12300000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + status = "disabled"; + }; + + dispgrt_dart: iommu@412320000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x12320000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + status = "disabled"; + }; + + dcp_dart: iommu@412340000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x12340000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + status = "disabled"; + }; + pinctrl_ap: pinctrl@31a000000 { compatible = "apple,t8140-pinctrl", "apple,t8103-pinctrl"; reg = <0x3 0x1a000000 0x0 0x100000>; From 7b8b9f5554b8ec27342b11ceb37248cc1ffc9093 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 7 Oct 2026 17:05:00 +1100 Subject: [PATCH 0176/2680] arm64: dts: apple: t8140: add disabled external display DARTs The Linux-exported ADT describes separate DART controllers for the external display engine and coprocessor. Describe their controller windows and shared interrupt alongside the internal display DARTs, using the existing T8140/T8110 binding. Public m1n1 ADTNode.get_reg() translates /arm-io/dart-dispext0 and /arm-io/dart-dcpext0 through the parent bus ranges. Only the controller windows are register resources here. Keep both nodes disabled until consumer, stream and power integration has been qualified. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 4144e0b264fec..5fd40795096c3 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -875,6 +875,24 @@ status = "disabled"; }; + dispext0_dart: iommu@416300000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x16300000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + status = "disabled"; + }; + + dcpext0_dart: iommu@416340000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x16340000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + status = "disabled"; + }; + pinctrl_ap: pinctrl@31a000000 { compatible = "apple,t8140-pinctrl", "apple,t8103-pinctrl"; reg = <0x3 0x1a000000 0x0 0x100000>; From a1c7c6feae43c403bc2ec777b8294f8d69efcf8b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 01:22:42 +1000 Subject: [PATCH 0177/2680] arm64: dts: apple: t8140: Add the SEP, its DART and mailbox, and SPI2 Describe the Secure Enclave Processor of the Apple A18 Pro (T8140), the DART and ASC mailbox it uses, the SPI2 controller that carries the Touch ID sensor on boards that have one, and the power domains they sit in (sep, sep_aon, spi_p, spi2). Register ranges, interrupts, the SEP DART aperture and the power-domain offsets come from the J700 Apple Device Tree. The power domains are the entries the PMGR generator emits from the ADT; their ADT parents ps_sio and ps_afisocni0 have no PMGR state register in that output and are not described. sep and sep_aon are always on: iBoot starts the SEP and it must never lose power under Linux. The SEP node stays disabled here; boards enable it. Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 35 ++++++++++++++++ arch/arm64/boot/dts/apple/t8140.dtsi | 49 +++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 175a64d69e40f..a31ce36ee968e 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -6,6 +6,15 @@ */ &pmgr { + ps_sep_aon: power-controller@60 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x60 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sep_aon"; + apple,always-on; + }; + ps_aic: power-controller@110 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x110 4>; @@ -108,6 +117,14 @@ label = "i2c0"; }; + ps_spi_p: power-controller@2f0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi_p"; + }; + ps_apcie_st: power-controller@310 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x310 4>; @@ -147,6 +164,15 @@ status = "disabled"; }; + ps_spi2: power-controller@340 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x340 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi2"; + power-domains = <&ps_spi_p>; + }; + ps_apcie_sys_st: power-controller@3a0 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x3a0 4>; @@ -220,6 +246,15 @@ * apple,force-disable pattern) never reached ACTIVE again on J700 * (isp_cpu_core0 stuck at ACTUAL=0 on the second firmware boot). */ + ps_sep: power-controller@c00 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc00 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sep"; + apple,always-on; + }; + ps_isp_cpu: power-controller@4000 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x4000 4>; diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 5fd40795096c3..3588b9ddc03fc 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -385,6 +385,43 @@ #performance-domain-cells = <0>; }; + sep_dart: iommu@280b80000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x2 0x80b80000 0x0 0x20000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + /* + * The SEP only accepts IOVAs in this window, which already + * carries the BIT(40) alias the DART translates. + */ + apple,dma-range = /bits/ 64 <0x10000004000 0xffff0000>; + apple,dma-offset = /bits/ 64 <0x0>; + }; + + sep: sep@282600000 { + compatible = "apple,sep"; + reg = <0x2 0x82600000 0x0 0x88000>; + mboxes = <&sep_mbox>; + mbox-names = "mbox"; + iommus = <&sep_dart 0>; + power-domains = <&ps_sep>; + status = "disabled"; + }; + + sep_mbox: mailbox@282608000 { + compatible = "apple,t8140-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x82608000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + pmgr: power-management@300700000 { compatible = "apple,t8140-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; #address-cells = <1>; @@ -963,6 +1000,18 @@ status = "disabled"; }; + spi2: spi@385108000 { + compatible = "apple,t8140-spi", "apple,t8103-spi"; + reg = <0x3 0x85108000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clkref>; + power-domains = <&ps_spi2>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + serial0: serial@30812c000 { compatible = "apple,dockchannel-uart"; reg = <0x3 0x0812c000 0x0 0x4000>, From e1d2364b1b6f80cc9a6db9748bad5ed9f1e4f5c0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 01:22:43 +1000 Subject: [PATCH 0178/2680] arm64: dts: apple: t8140-j700: Enable the SEP and the Touch ID sensor Enable the SEP on the MacBook Neo, add the "sep" alias the bootloader uses to hand over SEP properties, and describe the Touch ID ("Mesa") sensor on SPI2. Its rails are switched by the SMC through the pcLD key, so the node names the SMC by phandle together with the key and the rail words; its interrupt is AP GPIO 135, rising edge. All values are from the J700 Apple Device Tree. The calibration file is per-device factory data and is not shipped; firmware-name only fixes where the driver looks for it. Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 06f4d6385465a..2abd6ee5d15b2 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -19,6 +19,7 @@ atcphy0 = &atcphy0; gpu = &gpu; keyboard = &keyboard; + sep = &sep; serial0 = &serial0; wifi0 = &wifi0; }; @@ -80,6 +81,34 @@ }; }; +&sep { + status = "okay"; +}; + +/* + * Touch ID sensor. Its rails are switched by the SMC through the pcLD key + * rather than by a GPIO; the rail words are given in power-on order. The + * calibration file is per-device factory data and is not redistributable. + */ +&spi2 { + status = "okay"; + + fingerprint@0 { + compatible = "apple,mesa-fingerprint"; + reg = <0>; + spi-max-frequency = <8000000>; + spi-cpol; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <135 IRQ_TYPE_EDGE_RISING>; + apple,smc = <&smc>; + apple,smc-power-key = "pcLD"; + apple,smc-power-rails = <0x01010000 0x01020000>; + firmware-name = "apple/mesacal-j700.bin"; + }; +}; + &isp { apple,platform-id = <0x70>; apple,temporal-filter = <1>; From 4a993dc45d90e8011df8883276a57fd80f97778d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 02:05:27 +1000 Subject: [PATCH 0179/2680] arm64: dts: apple: t8140: Add CPU energy model and thermal sensor cells Give the thermal framework what it needs to manage the A18 Pro (T8140) CPUs: - opp-microwatt for every OPP of both clusters, so the OPP core registers an energy model for the power allocator governor and the cpufreq cooling devices; - #cooling-cells on every CPU, so the cpufreq cooling devices are registered; - #thermal-sensor-cells on the SMC hwmon node, whose temperature channels the hwmon core already registers as thermal sensors. The power values were measured on a MacBook Neo (J700) following the method used for T8103/T600x and T8112: one busy core per cluster at each operating point, pinned with scaling_min/max_freq, read from the SMC's core-rail power meters. The P cluster's core rail was identified as the one whose voltage steps with the P-state (PP0b, 573 to 967 mV); from 3144 MHz up, the rise of the rail shared with the E cluster (PPab) is added. The E cluster's meter (PPbb) does not register a single busy core below 1788 MHz, so the E values are the cluster's idle reading plus a quarter of the rise with all four cores busy. The top P operating points were measured from a cold die in 2 s windows so that no thermal limit applied during the sample. Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 3588b9ddc03fc..bac39f4515c0e 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -59,6 +59,7 @@ d-cache-size = <0x10000>; operating-points-v2 = <&ecluster_opp>; performance-domains = <&cpufreq_e>; + #cooling-cells = <2>; }; cpu_e1: cpu@1 { @@ -72,6 +73,7 @@ d-cache-size = <0x10000>; operating-points-v2 = <&ecluster_opp>; performance-domains = <&cpufreq_e>; + #cooling-cells = <2>; }; cpu_e2: cpu@2 { @@ -85,6 +87,7 @@ d-cache-size = <0x10000>; operating-points-v2 = <&ecluster_opp>; performance-domains = <&cpufreq_e>; + #cooling-cells = <2>; }; cpu_e3: cpu@3 { @@ -98,6 +101,7 @@ d-cache-size = <0x10000>; operating-points-v2 = <&ecluster_opp>; performance-domains = <&cpufreq_e>; + #cooling-cells = <2>; }; cpu_p0: cpu@10100 { @@ -111,6 +115,7 @@ d-cache-size = <0x20000>; operating-points-v2 = <&pcluster_opp>; performance-domains = <&cpufreq_p>; + #cooling-cells = <2>; }; cpu_p1: cpu@10101 { @@ -124,6 +129,7 @@ d-cache-size = <0x20000>; operating-points-v2 = <&pcluster_opp>; performance-domains = <&cpufreq_p>; + #cooling-cells = <2>; }; l2_cache_0: l2-cache-0 { @@ -154,26 +160,32 @@ opp01 { opp-hz = /bits/ 64 <816000000>; opp-level = <1>; + opp-microwatt = <48800>; }; opp02 { opp-hz = /bits/ 64 <1140000000>; opp-level = <2>; + opp-microwatt = <68900>; }; opp03 { opp-hz = /bits/ 64 <1464000000>; opp-level = <3>; + opp-microwatt = <88100>; }; opp04 { opp-hz = /bits/ 64 <1788000000>; opp-level = <4>; + opp-microwatt = <129600>; }; opp05 { opp-hz = /bits/ 64 <2112000000>; opp-level = <5>; + opp-microwatt = <172600>; }; opp06 { opp-hz = /bits/ 64 <2424000000>; opp-level = <6>; + opp-microwatt = <243500>; }; }; @@ -184,70 +196,87 @@ opp01 { opp-hz = /bits/ 64 <744000000>; opp-level = <1>; + opp-microwatt = <188800>; }; opp02 { opp-hz = /bits/ 64 <1260000000>; opp-level = <2>; + opp-microwatt = <368900>; }; opp03 { opp-hz = /bits/ 64 <1620000000>; opp-level = <3>; + opp-microwatt = <491000>; }; opp04 { opp-hz = /bits/ 64 <1944000000>; opp-level = <4>; + opp-microwatt = <607600>; }; opp05 { opp-hz = /bits/ 64 <2196000000>; opp-level = <5>; + opp-microwatt = <707300>; }; opp06 { opp-hz = /bits/ 64 <2472000000>; opp-level = <6>; + opp-microwatt = <935100>; }; opp07 { opp-hz = /bits/ 64 <2688000000>; opp-level = <7>; + opp-microwatt = <1114000>; }; opp08 { opp-hz = /bits/ 64 <2916000000>; opp-level = <8>; + opp-microwatt = <1324800>; }; opp09 { opp-hz = /bits/ 64 <3144000000>; opp-level = <9>; + opp-microwatt = <1772600>; }; opp10 { opp-hz = /bits/ 64 <3336000000>; opp-level = <10>; + opp-microwatt = <2045700>; }; opp11 { opp-hz = /bits/ 64 <3504000000>; opp-level = <11>; + opp-microwatt = <2329300>; }; opp12 { opp-hz = /bits/ 64 <3648000000>; opp-level = <12>; + opp-microwatt = <2621600>; }; opp13 { opp-hz = /bits/ 64 <3756000000>; opp-level = <13>; + opp-microwatt = <2803500>; }; opp14 { opp-hz = /bits/ 64 <3864000000>; opp-level = <14>; + opp-microwatt = <3064600>; }; opp15 { opp-hz = /bits/ 64 <3924000000>; opp-level = <15>; + opp-microwatt = <3223300>; }; opp16 { opp-hz = /bits/ 64 <3972000000>; opp-level = <16>; + opp-microwatt = <3342600>; }; opp17 { opp-hz = /bits/ 64 <4044000000>; opp-level = <17>; + opp-microwatt = <3424900>; }; }; @@ -536,6 +565,7 @@ smc_hwmon: hwmon { compatible = "apple,smc-hwmon"; + #thermal-sensor-cells = <1>; /* * CPU cluster die temperatures. Keep these From b8df83f46f7afa1683155b9c42a9c750a7c60377 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 02:06:38 +1000 Subject: [PATCH 0180/2680] arm64: dts: apple: t8140-j700: Add CPU thermal zones The MacBook Neo is fanless, so CPU frequency is its only cooling. Describe a thermal zone for each of the six CPU die sensors the SMC reports (E cluster Te05/Te0S, P cluster Tp01/Tp05/Tp09/Tp0D), each throttling both clusters: they share one die and one heat spreader. The power allocator splits each zone's budget by the CPUs' energy models, with the E cluster weighted three to one because its cores do several times more work per watt. Two properties of this machine decide the parameters. The SMC refreshes the readings about once a second and the die settles within that time after a power change, so the loop carries a full sample of delay. And the power a core draws per energy-model watt varies with the workload: a floating-point loop drew 2.7 times the P-core power of an integer loop at the same clock. The governor derives its gains from sustainable-power and the gap between the switch-on and control trips: - 80/95 C with 2.5 W (gains 170/341/34 mW per C) swung the P cluster between its lowest and highest P-state every two seconds; - 58/95 C with 2.5 W (68/135/14) oscillated on the floating-point load; - 33/95 C with 2.5 W (40/80/8) was stable, but the governor only resets its integral term below the switch-on temperature, which this machine never reaches, so time spent above 95 C lowered the budget until the next reboot (measured: -11% sustained P clock after one heavy run); - 58/95 C with 1.5 W (40/81/8) is stable on both loads and resets whenever the machine rests (it idles at 48-53 C). This uses the last. 1.5 W lies inside the sustainable range measured at about 93 C: 1.1 W for the floating-point and 2.5 W for the integer load. The critical trip is 110 C; the zones poll every 500 ms. Measured on a J700 with this device tree (schedutil): 1 P core, burst from idle 4044 MHz for 1 s, then 3504; 83 C 2 P cores, burst from idle 4044 MHz for 1 s, then 1944-2688; 95.6 C 1 P core, integer, 75 s mostly 3336 MHz at 84-88 C all cores, integer, 75 s P 1944 / E 2424 MHz, 85 C, no changes all cores, integer, 10 min P 1620-1944 / E 2424 MHz, 86.5 C mean 1 P core, float, 60 s 1944-2916 MHz, 92.6 C mean all cores, float, 90 s P 979 / E 2424 MHz, 92.5 C all cores, integer, after the float runs and 60 s idle: unchanged The removed walk ran one integer P core at 3504 MHz and all cores at P 2688 / E 1464 MHz, but touched 103.9 C at the start of all-core loads. Sustained P clocks here are 5-20% lower in exchange for a loop that stays stable on every workload tried and never carries wind-up from one load to the next; the power allocator's sysfs parameters (k_po, k_pu, k_i, integral_cutoff) can trade some of that back. Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 305 +++++++++++++++++++++++ 1 file changed, 305 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 2abd6ee5d15b2..2609f70cf452c 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -10,6 +10,7 @@ /dts-v1/; #include "t8140.dtsi" +#include / { compatible = "apple,j700", "apple,t8140", "apple,arm-platform"; @@ -54,6 +55,310 @@ device_type = "memory"; reg = <0x100 0 0x2 0>; /* To be filled by loader */ }; + + /* + * The J700 is fanless: the CPUs are the only cooling devices. Every + * die sensor gets a zone that throttles both clusters, since they share + * one die and one heat spreader. The power allocator splits the budget + * by the energy models, with the E cluster weighted three to one: its + * cores do several times more work per watt, so all-core loads keep + * them at full speed and trim the P cores instead. + * + * The SMC refreshes these readings about once a second and the die + * settles within that time, so the loop carries a full sample of delay, + * and the power a core draws per energy-model watt varies about 2.7x + * between integer and floating-point work. The governor derives its + * gains from sustainable-power and the switch-on to control gap; 1.5 W + * over 37 C gives 40/81/8 mW per C, which held the clock within one + * P-state on both kinds of load (higher gains oscillate on the heavy + * one). 1.5 W lies inside the sustainable range measured at ~93 C: + * 1.1 W for floating-point and 2.5 W for integer loads. The switch-on + * trip sits above the idle temperature (48-53 C), so the controller, + * and with it any integral wind-up, resets whenever the machine rests. + */ + thermal-zones { + cpu-e0-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 0>; /* Te05 */ + sustainable-power = <1500>; + + trips { + cpu_e0_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_e0_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + + trip-critical { + temperature = <110000>; + hysteresis = <0>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_e0_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_e0_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <3072>; + }; + }; + }; + + cpu-e1-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 1>; /* Te0S */ + sustainable-power = <1500>; + + trips { + cpu_e1_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_e1_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + + trip-critical { + temperature = <110000>; + hysteresis = <0>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_e1_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_e1_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <3072>; + }; + }; + }; + + cpu-p0-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 2>; /* Tp01 */ + sustainable-power = <1500>; + + trips { + cpu_p0_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p0_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + + trip-critical { + temperature = <110000>; + hysteresis = <0>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p0_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_p0_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <3072>; + }; + }; + }; + + cpu-p1-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 3>; /* Tp05 */ + sustainable-power = <1500>; + + trips { + cpu_p1_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p1_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + + trip-critical { + temperature = <110000>; + hysteresis = <0>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p1_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_p1_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <3072>; + }; + }; + }; + + cpu-p2-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 4>; /* Tp09 */ + sustainable-power = <1500>; + + trips { + cpu_p2_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p2_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + + trip-critical { + temperature = <110000>; + hysteresis = <0>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p2_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_p2_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <3072>; + }; + }; + }; + + cpu-p3-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 5>; /* Tp0D */ + sustainable-power = <1500>; + + trips { + cpu_p3_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p3_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + + trip-critical { + temperature = <110000>; + hysteresis = <0>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p3_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_p3_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <3072>; + }; + }; + }; + }; }; #include "isp-imx558.dtsi" From 01eecf816f6bb50cd83770ea7e8bc313b0ee7df0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 02:08:14 +1000 Subject: [PATCH 0181/2680] arm64: dts: apple: t8140: Drop the J700 CPU thermal policy property The SMC hwmon node no longer selects a driver-internal thermal policy; the J700's CPU die temperatures feed its device-tree thermal zones. Remove apple,cpu-thermal-policy and reword the comments that described the old positional contract. Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 8 +++----- arch/arm64/boot/dts/apple/t8140.dtsi | 6 +++--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 2609f70cf452c..c2918de44f48f 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -500,12 +500,10 @@ }; &smc_hwmon { - /* interim J700 CPU thermal policy over the CPU die channels */ - apple,cpu-thermal-policy; - /* - * Reporting-only channels. They follow the six CPU die temperatures - * in t8140.dtsi, which the CPU thermal policy samples by position. + * Further channels, reported to userspace. They follow the six CPU + * die temperatures in t8140.dtsi, which the thermal zones above + * refer to by index. * Every key below is in the J700 SMC's key table as a read-only * float, reads 10-99 C and moves between samples. The names follow * the SMC key groups and are not otherwise verified. diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index bac39f4515c0e..6e0125f659ea6 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -568,9 +568,9 @@ #thermal-sensor-cells = <1>; /* - * CPU cluster die temperatures. Keep these - * first: the J700 CPU thermal policy reads - * the first six temperature channels. + * CPU cluster die temperatures. Board thermal + * zones refer to these by channel index, so + * keep them first and in this order. */ temperature-Te05 { apple,key-id = "Te05"; From 68a10d9a1fa32b3df07809dc61c5b11d9ed04521 Mon Sep 17 00:00:00 2001 From: Acelogic <7362641+Acelogic@users.noreply.github.com> Date: Fri, 2 Oct 2026 02:06:57 -0400 Subject: [PATCH 0182/2680] arm64: dts: apple: wire the J700 AVE encoder Add the T8140 encoder power hierarchy, paired DARTs and AVE node. Enable them on J700 so the public Aurora m1n1 firmware handoff can populate its reserved-memory mappings. The register and power-domain description comes from Ace's existing J700 Linux bring-up work. Signed-off-by: Acelogic <7362641+Acelogic@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t8140-j700.dts | 12 ++++++ arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 52 +++++++++++++++++++++++ arch/arm64/boot/dts/apple/t8140.dtsi | 41 ++++++++++++++++++ 3 files changed, 105 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index c2918de44f48f..d6f6b30d4e91e 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -363,6 +363,18 @@ #include "isp-imx558.dtsi" +&ave_dart0 { + status = "okay"; +}; + +&ave_dart1 { + status = "okay"; +}; + +&ave { + status = "okay"; +}; + &serial0 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index a31ce36ee968e..c222bfb392bb3 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -49,6 +49,14 @@ label = "gfx"; }; + ps_venc0_sys: power-controller@240 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x240 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_sys"; + }; + ps_apcie_gp: power-controller@258 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x258 4>; @@ -368,6 +376,50 @@ status = "disabled"; }; + ps_venc0_dma: power-controller@8000 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_dma"; + power-domains = <&ps_venc0_sys>; + }; + + ps_venc0_pipe4: power-controller@8008 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_pipe4"; + power-domains = <&ps_venc0_dma>; + }; + + ps_venc0_pipe5: power-controller@8010 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_pipe5"; + power-domains = <&ps_venc0_dma>; + }; + + ps_venc0_me0: power-controller@8018 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_me0"; + power-domains = <&ps_venc0_dma>; + }; + + ps_venc0_me1: power-controller@8020 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_me1"; + power-domains = <&ps_venc0_me0>; + }; }; &pmgr_mini { diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 6e0125f659ea6..ae9e7d063f4e4 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -752,6 +752,47 @@ status = "disabled"; }; + ave_dart0: iommu@285080000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x2 0x85080000 0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_venc0_sys>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x00004000 0 0x0fffc000>; + status = "disabled"; + }; + + ave_dart1: iommu@2850a0000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x2 0x850a0000 0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_venc0_sys>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x00004000 0 0x0fffc000>; + status = "disabled"; + }; + + ave: video-codec@285100000 { + compatible = "apple,t8140-ave"; + reg = <0x2 0x85100000 0 0x45c000>, + <0x2 0x85050000 0 0x4000>, + <0x2 0x85800000 0 0x800000>; + reg-names = "codec", "mailbox", "coproc"; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + ; + power-domains = <&ps_venc0_dma>, <&ps_venc0_pipe4>, + <&ps_venc0_pipe5>, <&ps_venc0_me0>, <&ps_venc0_me1>; + power-domain-names = "dma", "pipe4", "pipe5", "me0", "me1"; + iommus = <&ave_dart0 0>, <&ave_dart1 0>; + status = "disabled"; + }; + pcie0: pcie@1cb0000000 { compatible = "apple,t8140-pcie"; device_type = "pci"; From bfb9b5a2d8185c3b69b8c6f36c6c465f17f07d69 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 28 Sep 2026 21:24:43 +1000 Subject: [PATCH 0183/2680] arm64: dts: apple: t8140: Add measured CPU capacity values Describe the relative per-MHz throughput of the E and P cores for the scheduler. Without these properties, it treats the two CPU types as equal capacity. On a J700, three alternating CoreMark runs per cluster at each cluster's maximum frequency gave median throughput of 12.6983 iterations/s/MHz on P CPU0 at 4044 MHz and 8.8239 on E CPU1 at 2424 MHz. The E/P ratio is 0.694886. Normalizing P to 1024 and rounding E gives 712. The runs reported valid CoreMark results, stable requested frequencies and no throttling; per-MHz spread was 0.03% for P and 0.17% for E. The earlier nearby-OPP microbenchmarks gave E=912, but did not test the same workload as the CoreMark-derived values used for other Apple SoCs. The expected scheduler capacity at maximum frequency is 1024 for P and about 427 for E; runtime readback of this revision remains to be done. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index ae9e7d063f4e4..1f7cc442d1849 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -58,6 +58,7 @@ i-cache-size = <0x20000>; d-cache-size = <0x10000>; operating-points-v2 = <&ecluster_opp>; + capacity-dmips-mhz = <712>; performance-domains = <&cpufreq_e>; #cooling-cells = <2>; }; @@ -72,6 +73,7 @@ i-cache-size = <0x20000>; d-cache-size = <0x10000>; operating-points-v2 = <&ecluster_opp>; + capacity-dmips-mhz = <712>; performance-domains = <&cpufreq_e>; #cooling-cells = <2>; }; @@ -86,6 +88,7 @@ i-cache-size = <0x20000>; d-cache-size = <0x10000>; operating-points-v2 = <&ecluster_opp>; + capacity-dmips-mhz = <712>; performance-domains = <&cpufreq_e>; #cooling-cells = <2>; }; @@ -100,6 +103,7 @@ i-cache-size = <0x20000>; d-cache-size = <0x10000>; operating-points-v2 = <&ecluster_opp>; + capacity-dmips-mhz = <712>; performance-domains = <&cpufreq_e>; #cooling-cells = <2>; }; @@ -114,6 +118,7 @@ i-cache-size = <0x30000>; d-cache-size = <0x20000>; operating-points-v2 = <&pcluster_opp>; + capacity-dmips-mhz = <1024>; performance-domains = <&cpufreq_p>; #cooling-cells = <2>; }; @@ -128,6 +133,7 @@ i-cache-size = <0x30000>; d-cache-size = <0x20000>; operating-points-v2 = <&pcluster_opp>; + capacity-dmips-mhz = <1024>; performance-domains = <&cpufreq_p>; #cooling-cells = <2>; }; From dd964ced6a21214efe8ea97e9704c191a3a1c33f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 13:52:31 +1100 Subject: [PATCH 0184/2680] arm64: dts: apple: Describe additional T8140 peripheral power states Describe the remaining peripheral power-state registers from the J700 ADT, using its explicit PMGR group, offset and parent identifiers. Expose the supported local-state dependencies through the existing PMGR power-domain binding. Firmware-owned main fabric records marked no-ps are not instantiated as generic power domains. Keep critical states on and preserve the firmware ownership of active AOP peripherals. Describe the unused AOP2 and HE NRCS states disabled: the existing board measurements show that HE NRCS cannot reach ACTIVE. Leave the firmware-owned PMC state unmanaged. The register descriptions are independently derived from the ADT and public bindings. No private device-tree source was copied. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 868 ++++++++++++++++++++++ 1 file changed, 868 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index c222bfb392bb3..afeeeaae0bb32 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -642,3 +642,871 @@ apple,always-on; /* used by the SMC firmware */ }; }; + +&pmgr { + ps_msg: power-controller@108 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x108 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msg"; + status = "okay"; + }; + + ps_dwi: power-controller@118 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x118 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dwi"; + status = "okay"; + }; + + ps_pms_fpwm0: power-controller@140 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x140 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm0"; + status = "okay"; + }; + + ps_pms_fpwm1: power-controller@148 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x148 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm1"; + status = "okay"; + }; + + ps_pms_fpwm2: power-controller@150 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x150 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm2"; + status = "okay"; + }; + + ps_pms_fpwm3: power-controller@158 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x158 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm3"; + status = "okay"; + }; + + ps_pms_fpwm4: power-controller@160 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x160 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm4"; + status = "okay"; + }; + + ps_pms_c1ppt: power-controller@168 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x168 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_c1ppt"; + apple,always-on; + status = "okay"; + }; + + ps_soc_rc: power-controller@170 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x170 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "soc_rc"; + status = "okay"; + }; + + ps_soc_dpe: power-controller@178 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x178 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "soc_dpe"; + apple,always-on; + status = "okay"; + }; + + ps_pmgr_soc_ocla: power-controller@180 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x180 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmgr_soc_ocla"; + status = "okay"; + }; + + ps_ispsens0: power-controller@188 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x188 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens0"; + status = "okay"; + }; + + ps_ispsens1: power-controller@190 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x190 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens1"; + status = "okay"; + }; + + ps_ap_tmm: power-controller@1a0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ap_tmm"; + status = "okay"; + }; + + ps_jpg: power-controller@230 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x230 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "jpg"; + status = "okay"; + }; + + ps_msr: power-controller@238 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x238 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msr"; + status = "okay"; + }; + + ps_prores: power-controller@250 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x250 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "prores"; + status = "okay"; + }; + + ps_dispext0_sys: power-controller@288 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x288 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_sys"; + status = "okay"; + }; + + ps_ane_sys: power-controller@290 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x290 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_sys"; + status = "okay"; + }; + + ps_msr_ase_core: power-controller@298 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x298 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msr_ase_core"; + power-domains = <&ps_msr>; + status = "okay"; + }; + + ps_sio_cpu: power-controller@2a8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sio_cpu"; + status = "okay"; + }; + + ps_fpwm0: power-controller@2b0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm0"; + status = "okay"; + }; + + ps_fpwm1: power-controller@2b8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm1"; + status = "okay"; + }; + + ps_fpwm2: power-controller@2c0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm2"; + status = "okay"; + }; + + ps_i2c1: power-controller@2d0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c1"; + status = "okay"; + }; + + ps_i2c2: power-controller@2d8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c2"; + status = "okay"; + }; + + ps_i2c3: power-controller@2e0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c3"; + status = "okay"; + }; + + ps_i2c4: power-controller@2e8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c4"; + status = "okay"; + }; + + ps_spi_p: power-controller@2f0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi_p"; + status = "okay"; + }; + + ps_uart_p: power-controller@2f8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart_p"; + status = "okay"; + }; + + ps_dpa_p: power-controller@300 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x300 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa_p"; + apple,always-on; + status = "okay"; + }; + + ps_aes: power-controller@308 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x308 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aes"; + status = "okay"; + }; + + ps_dispext0_fe: power-controller@320 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x320 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_fe"; + power-domains = <&ps_dispext0_sys>; + status = "okay"; + }; + + ps_dispext0_cpu: power-controller@328 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x328 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_cpu"; + power-domains = <&ps_dispext0_fe>; + status = "okay"; + }; + + ps_spi0: power-controller@330 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x330 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi0"; + power-domains = <&ps_spi_p>; + status = "okay"; + }; + + ps_spi1: power-controller@338 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x338 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi1"; + power-domains = <&ps_spi_p>; + status = "okay"; + }; + + ps_spi3: power-controller@348 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x348 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi3"; + power-domains = <&ps_spi_p>; + status = "okay"; + }; + + ps_qspi: power-controller@350 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x350 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "qspi"; + power-domains = <&ps_spi_p>; + status = "okay"; + }; + + ps_uart_n: power-controller@358 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x358 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart_n"; + power-domains = <&ps_uart_p>; + status = "okay"; + }; + + ps_uart0: power-controller@360 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x360 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart0"; + power-domains = <&ps_uart_p>; + status = "okay"; + }; + + ps_uart1: power-controller@368 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x368 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart1"; + power-domains = <&ps_uart_p>; + status = "okay"; + }; + + ps_uart2: power-controller@370 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x370 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart2"; + power-domains = <&ps_uart_p>; + status = "okay"; + }; + + ps_uart3: power-controller@378 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x378 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart3"; + power-domains = <&ps_uart_p>; + status = "okay"; + }; + + ps_uart4: power-controller@380 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x380 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart4"; + power-domains = <&ps_uart_p>; + status = "okay"; + }; + + ps_uart5: power-controller@388 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x388 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart5"; + power-domains = <&ps_uart_p>; + status = "okay"; + }; + + ps_dpa0: power-controller@390 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x390 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa0"; + power-domains = <&ps_dpa_p>; + status = "okay"; + }; + + ps_dpa1: power-controller@398 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x398 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa1"; + power-domains = <&ps_dpa_p>; + status = "okay"; + }; + +}; + +&pmgr_mini { + ps_debug_gated: power-controller@0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "debug_gated"; + apple,always-on; + status = "okay"; + }; + + ps_nub_spmi1: power-controller@60 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x60 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi1"; + apple,always-on; + status = "okay"; + }; + + ps_nub_aon_nogate: power-controller@78 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x78 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_aon_nogate"; + apple,always-on; + status = "okay"; + }; + + ps_nub_ocla: power-controller@80 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x80 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_ocla"; + apple,always-on; + status = "okay"; + }; + + ps_nub_gpio: power-controller@88 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x88 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_gpio"; + apple,always-on; + status = "okay"; + }; + + ps_nub_fabric: power-controller@a0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xa0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_fabric"; + apple,always-on; + status = "okay"; + }; + + ps_nub_sram: power-controller@a8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xa8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_sram"; + apple,always-on; + status = "okay"; + }; + + ps_debug_switch: power-controller@b0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xb0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "debug_switch"; + apple,always-on; + status = "okay"; + }; + + ps_aop_gpio: power-controller@4008 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_gpio"; + power-domains = <&ps_aop_filter>; + apple,always-on; + status = "okay"; + }; + + ps_aop_shim: power-controller@4028 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_shim"; + power-domains = <&ps_aop_base>; + apple,always-on; + status = "okay"; + }; + + ps_aop_piodma: power-controller@4030 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_piodma"; + power-domains = <&ps_aop_base>; + apple,always-on; + status = "okay"; + }; + + ps_aop2_piodma: power-controller@4038 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4038 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop2_piodma"; + power-domains = <&ps_aop_base>; + /* Firmware-owned or unable to reach ACTIVE on this board. */ + status = "disabled"; + }; + + ps_aop_i2cm0: power-controller@4040 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4040 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_i2cm0"; + power-domains = <&ps_aop_fr>; + apple,always-on; + status = "okay"; + }; + + ps_aop_i2cm1: power-controller@4048 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4048 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_i2cm1"; + power-domains = <&ps_aop_fr>; + apple,always-on; + status = "okay"; + }; + + ps_aop_i2cm2: power-controller@4050 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4050 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_i2cm2"; + power-domains = <&ps_aop_fr>; + apple,always-on; + status = "okay"; + }; + + ps_aop_i2cm3: power-controller@4058 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4058 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_i2cm3"; + power-domains = <&ps_aop_fr>; + /* Firmware-owned or unable to reach ACTIVE on this board. */ + status = "disabled"; + }; + + ps_aop_scm: power-controller@4060 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4060 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_scm"; + power-domains = <&ps_aop_base>, <&ps_aop_fr>; + apple,always-on; + status = "okay"; + }; + + ps_aop_spi0: power-controller@4068 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4068 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_spi0"; + power-domains = <&ps_aop_fr>; + apple,always-on; + status = "okay"; + }; + + ps_aop_spi1: power-controller@4070 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4070 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_spi1"; + power-domains = <&ps_aop_fr>; + /* Firmware-owned or unable to reach ACTIVE on this board. */ + status = "disabled"; + }; + + ps_aop_spmi0: power-controller@4078 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4078 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_spmi0"; + power-domains = <&ps_aop_filter>, <&ps_aop_fr>; + apple,always-on; + status = "okay"; + }; + + ps_aop_spmi1: power-controller@4080 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4080 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_spmi1"; + power-domains = <&ps_aop_filter>, <&ps_aop_fr>; + apple,always-on; + status = "okay"; + }; + + ps_aop_he_nrcs0: power-controller@4088 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4088 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_he_nrcs0"; + power-domains = <&ps_aop_fr>; + /* Firmware-owned or unable to reach ACTIVE on this board. */ + status = "disabled"; + }; + + ps_aop_he_nrcs1: power-controller@4090 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4090 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_he_nrcs1"; + power-domains = <&ps_aop_fr>; + /* Firmware-owned or unable to reach ACTIVE on this board. */ + status = "disabled"; + }; + + ps_aop2_cpu: power-controller@40a0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x40a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop2_cpu"; + power-domains = <&ps_aop_base>, <&ps_aop_fr>; + /* Firmware-owned or unable to reach ACTIVE on this board. */ + status = "disabled"; + }; + + ps_aop_uart0: power-controller@40b0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x40b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_uart0"; + power-domains = <&ps_aop_shim>; + apple,always-on; + status = "okay"; + }; + + ps_aop_uart1: power-controller@40b8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x40b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_uart1"; + power-domains = <&ps_aop_shim>; + apple,always-on; + status = "okay"; + }; + + ps_aop_uart2: power-controller@40c0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x40c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_uart2"; + power-domains = <&ps_aop_shim>; + apple,always-on; + status = "okay"; + }; + + ps_aop_sram: power-controller@40c8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x40c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_sram"; + power-domains = <&ps_aop_scm>, <&ps_aop_cpu>; + apple,always-on; + status = "okay"; + }; + + ps_audio_sense_pdmc: power-controller@40e0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x40e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_sense_pdmc"; + power-domains = <&ps_audio_fr>; + status = "okay"; + }; + + ps_audio_base_s_a: power-controller@40e8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x40e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_base_s_a"; + power-domains = <&ps_audio_p>; + status = "okay"; + }; + + ps_audio_mca2_m: power-controller@4110 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca2_m"; + power-domains = <&ps_audio_p>; + status = "okay"; + }; + + ps_audio_mca3_m: power-controller@4118 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4118 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca3_m"; + power-domains = <&ps_audio_p>; + status = "okay"; + }; + + ps_audio_mca4_m: power-controller@4120 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4120 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca4_m"; + power-domains = <&ps_audio_p>; + status = "okay"; + }; + + ps_audio_mca5_m: power-controller@4128 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4128 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca5_m"; + power-domains = <&ps_audio_p>; + status = "okay"; + }; + + ps_audio_leap_rx1: power-controller@4150 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4150 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_leap_rx1"; + power-domains = <&ps_audio_leap_c>; + status = "okay"; + }; + + ps_audio_leap_rx2: power-controller@4158 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4158 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_leap_rx2"; + power-domains = <&ps_audio_leap_c>; + status = "okay"; + }; + + ps_audio_leap_rx3: power-controller@4160 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4160 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_leap_rx3"; + power-domains = <&ps_audio_leap_c>; + status = "okay"; + }; + + ps_audio_leap_rx4: power-controller@4168 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4168 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_leap_rx4"; + power-domains = <&ps_audio_leap_c>; + status = "okay"; + }; + + ps_smc_fabric: power-controller@8000 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "smc_fabric"; + status = "okay"; + }; + + ps_smc_i2cm0: power-controller@8028 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "smc_i2cm0"; + power-domains = <&ps_smc_fabric>; + apple,always-on; + status = "okay"; + }; + +}; From 3bfafd12850e40f0747f79b40c785f190f96fc18 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:40:58 +1100 Subject: [PATCH 0185/2680] arm64: dts: apple: Wire up remaining supported J700 peripherals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe the PMP DART, mailbox, SRAM and report windows from the J700 ADT, including the high DMA aperture. Add the QSPIMC controller, SPI NOR NVRAM partition, I2C2 pinmux and bus, and display crossbar. Describe and enable the external display DARTs and mailbox using existing Apple DART and mailbox bindings. Align the existing pinctrl and diagnostic PIODMA node names with their bindings without changing their labels or resources. Add a T8140 cpuidle node and match it in the existing Apple idle driver. Keep WFI as its only idle state and skip legacy device creation when DT describes the controller. Document the idle and crossbar interfaces. The PMP and QSPI consumers need their corresponding driver support. External DCP and Type-C display routes still need a validated T8140 external-display profile; this change does not wire that consumer. The register, interrupt, power-domain and partition descriptions are independently derived from the ADT and public bindings. The QSPI clock selection follows the existing public QSPIMC binding example. No private device-tree source was copied. [ryan: adapted to 7.2.10 — split: DT part only; the cpuidle-apple.c part goes with 010-soc] Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 79 +++++++++++- arch/arm64/boot/dts/apple/t8140.dtsi | 153 +++++++++++++++++++++++ 2 files changed, 230 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index d6f6b30d4e91e..2566b378d5f39 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -17,6 +17,10 @@ model = "Apple MacBook Neo"; aliases { + pmp = &pmp; + nvram = &nvram; + spi0 = &qspi; + i2c2 = &i2c2; atcphy0 = &atcphy0; gpu = &gpu; keyboard = &keyboard; @@ -383,6 +387,77 @@ status = "okay"; }; +&jpeg_dart { + status = "okay"; +}; + +&apr_dart { + status = "okay"; +}; + +&pmp_dart { + status = "okay"; +}; + +&pmp_report { + status = "okay"; +}; + +&pmp_mbox { + status = "okay"; +}; + +&pmp { + status = "okay"; +}; + +&i2c2 { + status = "okay"; +}; + +&qspi { + status = "okay"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <25000000>; + spi-rx-bus-width = <4>; + spi-tx-bus-width = <4>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + nvram: partition@24000 { + label = "nvram"; + reg = <0x24000 0x1d0000>; + }; + }; + }; +}; + +&dispext0_dart { + status = "okay"; +}; + +&dcpext0_dart { + status = "okay"; +}; + +&dcpext_mbox { + status = "okay"; +}; + +&dpxbar0 { + status = "okay"; +}; + +&cpuidle { + status = "okay"; +}; + &i2c0 { status = "okay"; @@ -1726,7 +1801,7 @@ }; &pinctrl_ap { - pcie0_clkreq: pcie0-clkreq { + pcie0_clkreq: pcie0-clkreq-pins { pinmux = ; }; }; @@ -1738,7 +1813,7 @@ }; / { - pcie0_piodma_diag: dma-controller@390030000 { + pcie0_piodma_diag: piodma@390030000 { compatible = "apple,t8140-piodma-diagnostic"; reg = <0x3 0x90030000 0x0 0x4000>, <0x3 0x90034000 0x0 0x4000>, diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 1f7cc442d1849..20a7cf6c7b38b 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -18,6 +18,18 @@ #address-cells = <2>; #size-cells = <2>; + cpuidle: cpuidle { + compatible = "apple,t8140-cpuidle"; + status = "disabled"; + }; + + clk_200m: clock-200m { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <200000000>; + clock-output-names = "clk_200m"; + }; + cpus { #address-cells = <2>; #size-cells = <0>; @@ -408,16 +420,152 @@ nonposted-mmio; ranges; + jpeg_dart: iommu@28d020000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x2 0x8d020000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = /bits/ 64 <0x10000000000 0x30000000000>; + power-domains = <&ps_jpg>; + status = "disabled"; + }; + + apr_dart: iommu@295020000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x2 0x95020000 0x0 0x20000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = /bits/ 64 <0x10000000000 0x30000000000>; + power-domains = <&ps_prores>; + status = "disabled"; + }; + + pmp_dart: iommu@300300000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x3 0x00300000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = /bits/ 64 <0x10000000000 0xbff00000>; + power-domains = <&ps_pmp>; + status = "disabled"; + }; + + pmp_report: pmp-report@3003c0000 { + compatible = "apple,t8140-pmp-v2-report"; + reg = <0x3 0x003c0000 0x0 0x14000>; + power-domains = <&ps_pms_sram>; + status = "disabled"; + }; + + pmp: pmp@300500000 { + compatible = "apple,t8140-pmp-v2", "apple,t6000-pmp-v2"; + reg = <0x3 0x00500000 0x0 0xa0000>, + <0x3 0x00e00000 0x0 0x4000>; + reg-names = "pmp", "asc"; + mboxes = <&pmp_mbox>; + mbox-names = "mbox"; + iommus = <&pmp_dart 0>; + power-domains = <&ps_pmp>; + status = "disabled"; + }; + + pmp_mbox: mailbox@300e08000 { + compatible = "apple,t8140-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x00e08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_pmp>; + status = "disabled"; + }; + + dispext0_dart: iommu@416300000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x16300000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + status = "disabled"; + }; + + dcpext0_dart: iommu@416340000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x16340000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + status = "disabled"; + }; + + dcpext_mbox: mailbox@416e08000 { + compatible = "apple,t8140-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x16e08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_dispext0_cpu>; + status = "disabled"; + }; + + i2c2: i2c@385018000 { + compatible = "apple,t8140-i2c", "apple,t8103-i2c"; + reg = <0x3 0x85018000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c2_pins>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&ps_i2c2>; + status = "disabled"; + }; + + qspi: spi@385110000 { + compatible = "apple,t8140-qspi"; + /* Union of the overlapping controller and FIFO ADT windows. */ + reg = <0x3 0x85110000 0x0 0x4800>; + clocks = <&clk_200m>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&ps_qspi>; + status = "disabled"; + }; + + dpxbar0: display-crossbar@40b04c000 { + compatible = "apple,t6020-display-crossbar"; + reg = <0x4 0x0b04c000 0x0 0x4000>; + #mux-control-cells = <1>; + status = "disabled"; + }; + cpufreq_e: cpufreq@210e20000 { compatible = "apple,t8140-cluster-cpufreq"; reg = <0x2 0x10e20000 0x0 0x1000>; #performance-domain-cells = <0>; + apple,fast-die-effort = <&pmp_report 0>; }; cpufreq_p: cpufreq@211e20000 { compatible = "apple,t8140-cluster-cpufreq"; reg = <0x2 0x11e20000 0x0 0x1000>; #performance-domain-cells = <0>; + apple,fast-die-effort = <&pmp_report 1>; }; sep_dart: iommu@280b80000 { @@ -1032,6 +1180,11 @@ gpio-reserved-ranges = <50 1>, <134 1>; status = "disabled"; + i2c2_pins: i2c2-pins { + pinmux = , + ; + }; + i2c0_pins: i2c0-pins { pinmux = , ; From ce78ba662e48df0f9c0a1f7602d8128d921da7eb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:21:54 +1100 Subject: [PATCH 0186/2680] arm64: dts: apple: enable the PMP fast-die limiter on J700 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe the PMP SRAM, ASC, mailbox, DART and report resources from the Apple device tree. The DART register range includes its v2.3 exception registers. Enable the resources on J700 and connect the cluster effort lanes so cpufreq retains its fallback caps until the firmware runs. Leave the firmware-owned PMC domain untouched. The PMP firmware and fast-die maximum-frequency requests complement the existing thermal power allocator; thermal trips and sustainable power remain unchanged. The bootloader must initialize the PMP DAPF and copy its calibration. [ryan: adapted to 7.2.10 — squashed with commit 19d6664ea33a ("arm64: dts: apple: retain a single J700 PMP alias"), the composition fix for the duplicate pmp alias this commit and commit f75a1041fca4 ("arm64: dts: apple: Wire up remaining supported J700 peripherals") both add] Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 18 +++++++++++++++++- arch/arm64/boot/dts/apple/t8140.dtsi | 3 +++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 2566b378d5f39..05dff1967b376 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -17,7 +17,6 @@ model = "Apple MacBook Neo"; aliases { - pmp = &pmp; nvram = &nvram; spi0 = &qspi; i2c2 = &i2c2; @@ -26,6 +25,7 @@ keyboard = &keyboard; sep = &sep; serial0 = &serial0; + pmp = &pmp; wifi0 = &wifi0; }; @@ -383,6 +383,22 @@ status = "okay"; }; +&pmp_dart { + status = "okay"; +}; + +&pmp_mbox { + status = "okay"; +}; + +&pmp_report { + status = "okay"; +}; + +&pmp { + status = "okay"; +}; + &pinctrl_ap { status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 20a7cf6c7b38b..5f93c24ea6015 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -558,6 +558,7 @@ compatible = "apple,t8140-cluster-cpufreq"; reg = <0x2 0x10e20000 0x0 0x1000>; #performance-domain-cells = <0>; + /* E lane follows the ADT ordering; no changing sample yet. */ apple,fast-die-effort = <&pmp_report 0>; }; @@ -565,6 +566,7 @@ compatible = "apple,t8140-cluster-cpufreq"; reg = <0x2 0x11e20000 0x0 0x1000>; #performance-domain-cells = <0>; + /* P lane has been observed under Linux CPU load. */ apple,fast-die-effort = <&pmp_report 1>; }; @@ -613,6 +615,7 @@ /* child nodes are added in t8140-pmgr.dtsi */ }; + aic: interrupt-controller@301000000 { compatible = "apple,t8140-aic3", "apple,t8122-aic3"; #interrupt-cells = <3>; From e54f44e63b6ed9661c32204fb0122d9a5ef27ded Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:43:26 +1100 Subject: [PATCH 0187/2680] arm64: dts: apple: describe the J700 PMU alarm function Add the RTC function below the existing Baku SPMI PMIC and share the whole-second offset cell with the SMC RTC. Use peripheral interrupt 2 for the retained-power alarm. The driver requires an explicit opt-in. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 5f93c24ea6015..338b8520b4222 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -657,6 +657,15 @@ compatible = "apple,baku-pmic", "apple,spmi-nvmem"; reg = <0xe SPMI_USID>; + pmu_rtc: rtc { + compatible = "apple,j700-pmu-rtc"; + interrupt-parent = <&nub_spmi0>; + interrupts = <2 IRQ_TYPE_EDGE_RISING>; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; + wakeup-source; + }; + nvmem-layout { compatible = "fixed-layout"; #address-cells = <1>; From 256de469fcd0b4a48cd2d7025b3f6575d079bb9b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 17:56:16 +1100 Subject: [PATCH 0188/2680] arm64: dts: apple: describe the disabled T8140 Neural Engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe the ANE cluster, its three DART instances and the ANE_SYS power domain from the device tree. Keep the engine and DARTs disabled until the T8140 firmware backend and child power sequence can initialize them safely. The M2 firmware backend is not compatible with this generation. [ryan: adapted to 7.2.10 — drop the second ps_ane_sys node this added beside the one from commit f898b116beaa ("arm64: dts: apple: Describe additional T8140 peripheral power states"); aurora-wip resolved the same duplicate in a merge] Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 1 + arch/arm64/boot/dts/apple/t8140.dtsi | 46 +++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index afeeeaae0bb32..f44d36a4488e9 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -641,6 +641,7 @@ label = "smc_i2cm2"; apple,always-on; /* used by the SMC firmware */ }; + }; &pmgr { diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 338b8520b4222..7e8ffb3e4b663 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -1608,6 +1608,52 @@ apple,dart-vm-size = <0x0 0xa0000000>; status = "disabled"; }; + + ane_dart0: iommu@401800000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x01800000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + apple,dma-range = <0x100 0x0 0x300 0x0>; + status = "disabled"; + }; + + ane_dart1: iommu@401820000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x01820000 0x0 0x20000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + apple,dma-range = <0x100 0x0 0x300 0x0>; + status = "disabled"; + }; + + ane_dart2: iommu@401840000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x01840000 0x0 0x20000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + apple,dma-range = <0x100 0x0 0x300 0x0>; + status = "disabled"; + }; + + /* The T8140 firmware and child power sequence need their own backend. */ + ane: ane@400000000 { + compatible = "apple,t8140-ane"; + reg = <0x4 0x0 0x0 0x2000000>; + reg-names = "engine"; + interrupt-parent = <&aic>; + interrupts = , + ; + iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; }; #include "t8140-pmgr.dtsi" From 472bc44726d9a4ec2b4b078ec91f1bcd01053b82 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:19:05 +1100 Subject: [PATCH 0189/2680] arm64: dts: apple: t8140: Leave display coprocessor power to firmware Booted with this tree's device tree, the J700 resets about 1.5 seconds into boot: SError Interrupt on CPU5, code 0x00000000be000000 -- SError Call trace: ... apple_pmgr_ps_set apple_pmgr_ps_power_off _genpd_power_off genpd_power_off genpd_sync_state device_links_supplier_sync_state_resume of_platform_sync_state_init The firmware leaves the external display coprocessor running at boot, and the device tree registers its three power states (dispext0_sys, _fe and _cpu) with nothing that keeps them in use: the only consumer is its mailbox, which is runtime suspended because no driver talks to that coprocessor on T8140. Once the mailbox has probed, sync_state powers dispext0_cpu off underneath the running firmware, and that power-state write faults. Stop registering the three external display states, and stop enabling the mailbox and the two DARTs that only that coprocessor would use. The firmware keeps owning them until there is a driver for it. The internal display coprocessor is handed over running as well, and its power states were described disabled, so nothing could depend on them. Register disp_sys, disp_fe and disp_cpu, and mark disp_cpu always-on: genpd then never writes that state, neither at sync_state nor on system suspend, and its parents stay on through it. The bootloader leaves the panel lit, so the domain is on when it probes. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 12 ------------ arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 19 +++++++++++++------ 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 05dff1967b376..146ac921bbbb7 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -454,18 +454,6 @@ }; }; -&dispext0_dart { - status = "okay"; -}; - -&dcpext0_dart { - status = "okay"; -}; - -&dcpext_mbox { - status = "okay"; -}; - &dpxbar0 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index f44d36a4488e9..b49a70355b808 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -96,7 +96,6 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "disp_sys"; - status = "disabled"; }; ps_dispext0_sys: power-controller@288 { @@ -149,7 +148,6 @@ #reset-cells = <0>; label = "disp_fe"; power-domains = <&ps_disp_sys>; - status = "disabled"; }; ps_dispext0_fe: power-controller@320 { @@ -373,7 +371,11 @@ #reset-cells = <0>; label = "disp_cpu"; power-domains = <&ps_disp_fe>; - status = "disabled"; + /* + * The display coprocessor is handed over running and its + * firmware owns this state; writing it faults. + */ + apple,always-on; }; ps_venc0_dma: power-controller@8000 { @@ -800,13 +802,18 @@ status = "okay"; }; + /* + * The external display coprocessor has no Linux consumer yet. Its + * firmware keeps it running, so leave these three states to it: + * powering dispext0_cpu down underneath that firmware faults. + */ ps_dispext0_sys: power-controller@288 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x288 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = "dispext0_sys"; - status = "okay"; + status = "disabled"; }; ps_ane_sys: power-controller@290 { @@ -944,7 +951,7 @@ #reset-cells = <0>; label = "dispext0_fe"; power-domains = <&ps_dispext0_sys>; - status = "okay"; + status = "disabled"; }; ps_dispext0_cpu: power-controller@328 { @@ -954,7 +961,7 @@ #reset-cells = <0>; label = "dispext0_cpu"; power-domains = <&ps_dispext0_fe>; - status = "okay"; + status = "disabled"; }; ps_spi0: power-controller@330 { From 13a5ce73396ba96024d7884591b8d990d7557f6c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:40:09 +1100 Subject: [PATCH 0190/2680] arm64: dts: apple: t8140: Describe the internal display coprocessor The T8140 device tree has the display DARTs and power states but no display coprocessor, so the loader finds nothing to hand the running display over to and Linux is left with the boot framebuffer. Add the internal DCP, its ASC mailbox, the PMGR bandwidth registers it names, its two display clocks and the display subsystem node, following the layout used for the other Apple SoCs: - The DCP register ranges are the coprocessor control block and its I/O vector base from the Apple Device Tree's dcp node, and the first five disp0 ranges. Its DART stream (23) and the scanout and PIODMA streams (0 and 16) come from the matching DART mapper nodes. - The mailbox interrupts are the four dcp interrupts, in the same send-empty, send-not-empty, recv-empty, recv-not-empty order as the external mailbox. - Both display DARTs get the 64 GiB VM window at 0x10000000000 that the Apple Device Tree gives those streams. - The bandwidth scratch and request registers sit 0x20000 and 0x68000 into the display clock window at 0x302800000, the last disp0 range. - The clock rates are disp0's clock IDs 0x154 and 0x194: 300 MHz and 450 MHz. The DCP is handed over running with its firmware mapped through locked DARTs, so it stays disabled here, like its DARTs. The loader enables both once it has reserved those mappings; the display subsystem and the DCP carry explicit phandles because the loader refers to them. A single zero in apple,iomfb-surfaces selects surface 0. Kernels that read the property as a list of surfaces select the same one. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 99 ++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 7e8ffb3e4b663..fd419d1fe8a49 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -389,6 +389,34 @@ clock-output-names = "clkref"; }; + /* disp0 clock ID 0x154; the rate is the Apple Device Tree's */ + clk_disp0_154: clock-disp0-154 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <300000000>; + clock-output-names = "clk_disp0_154"; + }; + + /* disp0 clock ID 0x194 */ + clk_disp0_194: clock-disp0-194 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <450000000>; + clock-output-names = "clk_disp0_194"; + }; + + /* + * Display component master. It has no registers, so it sits outside + * the simple-bus. + */ + display: display-subsystem { + compatible = "apple,display-subsystem"; + /* disp0_dart is locked by the bootloader and must come first */ + iommus = <&disp0_dart 0>; + /* generate a phandle for the loader */ + phandle = <&display>; + }; + reserved-memory { #address-cells = <2>; #size-cells = <2>; @@ -1128,6 +1156,8 @@ interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; + /* VM window of streams 0 and 16 */ + apple,dma-range = <0x100 0x0 0x10 0x0>; status = "disabled"; }; @@ -1146,7 +1176,76 @@ interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; + /* VM window of stream 23 */ + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + dcp_mbox: mailbox@412e08000 { + compatible = "apple,t8140-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x12e08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_disp_cpu>; + status = "disabled"; + }; + + /* + * Bandwidth scratch and request registers, 0x20000 and 0x68000 + * into the display clock window at 0x302800000 (the sixth disp0 + * register range in the Apple Device Tree). + */ + pmgr_dcp: power-management@302820000 { + reg = <0x3 0x02820000 0x0 0x8>, + <0x3 0x02868000 0x0 0x4>; + reg-names = "dcp-bw-scratch", "dcp-bw-doorbell"; + #apple,bw-scratch-cells = <3>; + #apple,bw-doorbell-cells = <2>; + }; + + dcp: dcp@412e00000 { + compatible = "apple,t8140-dcp", "apple,dcp"; + reg = <0x4 0x12e00000 0x0 0x4000>, + <0x4 0x10000000 0x0 0xafc000>, + <0x4 0x12240000 0x0 0x10000>, + <0x4 0x123a8000 0x0 0x4000>, + <0x4 0x123ac000 0x0 0x4000>, + <0x4 0x12800000 0x0 0x800000>, + <0x4 0x12850000 0x0 0x4000>; + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4", "iop-vbar"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 23>; + power-domains = <&ps_disp_cpu>; + clocks = <&clk_disp0_154>, <&clk_disp0_194>; + clock-names = "clock-154", "clock-194"; + apple,bw-scratch = <&pmgr_dcp 0 5 0>; + apple,bw-doorbell = <&pmgr_dcp 1 6>; + /* + * A single zero selects surface 0; J700 kernels that + * read this as a surface list take it the same way. + */ + apple,iomfb-surfaces = <0>; + /* generate a phandle for the loader */ + phandle = <&dcp>; + /* bus properties for the piodma child device */ + #address-cells = <2>; + #size-cells = <2>; + ranges; + /* the loader enables it when it hands the display over */ status = "disabled"; + + disp0_piodma: piodma { + iommus = <&disp0_dart 16>; + phandle = <&disp0_piodma>; + }; }; dispext0_dart: iommu@416300000 { From 1f705a2da2d219222a7cac8cd7ecdd9655624316 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:40:10 +1100 Subject: [PATCH 0191/2680] arm64: dts: apple: t8140: Hand the J700 panel over to the DCP The loader only hands the running internal display over when the device tree has dcp and disp0 aliases. Without them it keeps the boot framebuffer, reserves none of the display memory and leaves the DCP disabled, so the internal panel never reaches the display driver. Add the aliases the loader uses for the hand-off, plus the PIODMA alias in its hyphenated form, enable the DCP mailbox and describe the panel: 2408x1506 at the Apple Device Tree's 219 ppi dot pitch is 279 x 175 mm, and the backlight's 16.16 maximum luminance is 525 nits. Give the boot framebuffer the display CPU power domain, as on the other Apple laptops, so that simpledrm depends on the domain that keeps the display running whether or not the hand-off happens. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 146ac921bbbb7..4742ce8aaca35 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -27,6 +27,10 @@ serial0 = &serial0; pmp = &pmp; wifi0 = &wifi0; + /* used by the loader to hand over the display */ + dcp = &dcp; + disp0 = &display; + disp0-piodma = &disp0_piodma; }; chosen { @@ -43,6 +47,7 @@ framebuffer0: framebuffer@0 { compatible = "apple,simple-framebuffer", "simple-framebuffer"; reg = <0 0 0 0>; /* To be filled by loader */ + power-domains = <&ps_disp_cpu>; /* Format properties will be added by loader */ status = "disabled"; }; @@ -454,6 +459,20 @@ }; }; +&dcp_mbox { + status = "okay"; +}; + +&dcp { + /* 2408x1506 at 219 ppi; 525 nits peak from the backlight node */ + panel: panel { + compatible = "apple,panel-j700", "apple,panel"; + width-mm = <279>; + height-mm = <175>; + apple,max-brightness = <525>; + }; +}; + &dpxbar0 { status = "okay"; }; From 2a8dddb8afcd9bed2d12f97d428c81c200279b5a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:10:10 +1100 Subject: [PATCH 0192/2680] arm64: dts: apple: t8140: describe each power state once t8140-pmgr.dtsi extends &pmgr in two blocks, and four power states are described in both: dispext0_sys, dispext0_fe, dispext0_cpu and spi_p. dtc merges the two copies, and where they disagree the second wins silently. That already bit once: the external display states had to be disabled in the second copy because only the first one said so. Keep the first copy of each state, which is where dtc places the merged node, and drop the second. Carry over the only property that differed, spi_p's explicit status, and the comment explaining why the external display states are left to the firmware. The compiled t8140-j700.dtb is byte-identical before and after. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 50 ++++------------------- 1 file changed, 7 insertions(+), 43 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index b49a70355b808..c2c6d682c5981 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -98,6 +98,12 @@ label = "disp_sys"; }; + /* + * The external display coprocessor has no Linux consumer yet. Its + * firmware keeps it running, so leave its three states, dispext0_sys, + * dispext0_fe and dispext0_cpu, to it: powering dispext0_cpu down + * underneath that firmware faults. + */ ps_dispext0_sys: power-controller@288 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x288 4>; @@ -130,6 +136,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "spi_p"; + status = "okay"; }; ps_apcie_st: power-controller@310 { @@ -802,20 +809,6 @@ status = "okay"; }; - /* - * The external display coprocessor has no Linux consumer yet. Its - * firmware keeps it running, so leave these three states to it: - * powering dispext0_cpu down underneath that firmware faults. - */ - ps_dispext0_sys: power-controller@288 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x288 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "dispext0_sys"; - status = "disabled"; - }; - ps_ane_sys: power-controller@290 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x290 4>; @@ -907,15 +900,6 @@ status = "okay"; }; - ps_spi_p: power-controller@2f0 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x2f0 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "spi_p"; - status = "okay"; - }; - ps_uart_p: power-controller@2f8 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x2f8 4>; @@ -944,26 +928,6 @@ status = "okay"; }; - ps_dispext0_fe: power-controller@320 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x320 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "dispext0_fe"; - power-domains = <&ps_dispext0_sys>; - status = "disabled"; - }; - - ps_dispext0_cpu: power-controller@328 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x328 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "dispext0_cpu"; - power-domains = <&ps_dispext0_fe>; - status = "disabled"; - }; - ps_spi0: power-controller@330 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x330 4>; From b48422d5a0fd8cb389ff076fa1916797e374bef9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:10:11 +1100 Subject: [PATCH 0193/2680] arm64: dts: apple: t8140: describe the external display DARTs once dispext0_dart and dcpext0_dart are described twice, identically: once in the soc node and again in the &soc extension further down. Drop the second copy. The compiled t8140-j700.dtb is byte-identical before and after. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index fd419d1fe8a49..b4f6219510a15 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -1248,24 +1248,6 @@ }; }; - dispext0_dart: iommu@416300000 { - compatible = "apple,t8140-dart", "apple,t8110-dart"; - reg = <0x4 0x16300000 0x0 0xc000>; - interrupt-parent = <&aic>; - interrupts = ; - #iommu-cells = <1>; - status = "disabled"; - }; - - dcpext0_dart: iommu@416340000 { - compatible = "apple,t8140-dart", "apple,t8110-dart"; - reg = <0x4 0x16340000 0x0 0xc000>; - interrupt-parent = <&aic>; - interrupts = ; - #iommu-cells = <1>; - status = "disabled"; - }; - pinctrl_ap: pinctrl@31a000000 { compatible = "apple,t8140-pinctrl", "apple,t8103-pinctrl"; reg = <0x3 0x1a000000 0x0 0x100000>; From e382982edeb5a64d1a8a04bea35c6e714975c33e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:10:12 +1100 Subject: [PATCH 0194/2680] arm64: dts: apple: t8140-j700: enable the PMP nodes once The PMP, its DART, mailbox and report region are enabled twice in the J700 device tree, by two identical sets of status overrides. Drop the second set. The compiled t8140-j700.dtb is byte-identical before and after. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 4742ce8aaca35..216f96dc645b1 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -416,22 +416,6 @@ status = "okay"; }; -&pmp_dart { - status = "okay"; -}; - -&pmp_report { - status = "okay"; -}; - -&pmp_mbox { - status = "okay"; -}; - -&pmp { - status = "okay"; -}; - &i2c2 { status = "okay"; }; From 2a7c8d5ac1d66fb6ccc9715747eba8130bba1030 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 11:51:37 +1100 Subject: [PATCH 0195/2680] arm64: dts: apple: t8140-j700: Keep the idle E cluster out of its lowest OPP The power allocator grants no power to a cooling device that requests none. Above the 58 C switch-on trip it therefore caps an idle E cluster at its lowest OPP, 816 MHz, and work that wakes there runs at that speed until the next evaluation, up to 500 ms later. With the performance cpufreq governor the J700 die idles at 58-70 C, above that trip: in a ten-minute idle the E cooling device sat at state 5 in 39% of the samples, and 35 of 120 short E-core bursts started capped and took 107-114 ms instead of 36 ms. Stop the E-cluster cooling maps at state 3, 1464 MHz. The energy model puts the cost of the cluster's work there within 0.5% of 816 MHz, and with the P cluster at its lowest OPP the floor of both clusters is 0.73 W, well under the 1.1 W sustainable on floating-point loads, so the governor keeps the authority it needs. With this change the same idle never takes the E cluster below 1464 MHz and the slowest burst takes 61 ms, with the P-cluster rail unchanged at 1.21 W. With schedutil the die idles at 41-49 C and the governor never runs, before or after. A ten-minute all-core load keeps the E cluster at 2424 MHz throughout either way, since its weight makes the governor trim the P cores; the P cluster settles at 1260-1370 MHz and the hottest sensor peaks at 93.9 C (95.6 C before). Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 64 ++++++++++++++---------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 216f96dc645b1..f0c6d917595ee 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -82,8 +82,20 @@ * P-state on both kinds of load (higher gains oscillate on the heavy * one). 1.5 W lies inside the sustainable range measured at ~93 C: * 1.1 W for floating-point and 2.5 W for integer loads. The switch-on - * trip sits above the idle temperature (48-53 C), so the controller, - * and with it any integral wind-up, resets whenever the machine rests. + * trip sits above the idle temperature with schedutil (48-53 C), so the + * controller, and with it any integral wind-up, resets whenever the + * machine rests. + * + * The governor grants no power to a cooling device that requests none, + * so above the switch-on trip an idle E cluster is sent to its lowest + * OPP, and work that wakes there runs at 816 MHz until the next + * evaluation. With the performance governor the die idles at 58-70 C, + * above that trip, and in a ten-minute idle the E cluster spent 40% of + * its time capped this way. The E maps therefore stop at cooling + * state 3, 1464 MHz, where the energy model puts the cost of the + * cluster's work within 0.5% of 816 MHz. The lowest power of both + * clusters, 0.73 W, stays well under the 1.1 W sustainable on + * floating-point loads. */ thermal-zones { cpu-e0-thermal { @@ -124,10 +136,10 @@ map1 { trip = <&cpu_e0_control>; cooling-device = - <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; contribution = <3072>; }; }; @@ -171,10 +183,10 @@ map1 { trip = <&cpu_e1_control>; cooling-device = - <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; contribution = <3072>; }; }; @@ -218,10 +230,10 @@ map1 { trip = <&cpu_p0_control>; cooling-device = - <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; contribution = <3072>; }; }; @@ -265,10 +277,10 @@ map1 { trip = <&cpu_p1_control>; cooling-device = - <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; contribution = <3072>; }; }; @@ -312,10 +324,10 @@ map1 { trip = <&cpu_p2_control>; cooling-device = - <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; contribution = <3072>; }; }; @@ -359,10 +371,10 @@ map1 { trip = <&cpu_p3_control>; cooling-device = - <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; contribution = <3072>; }; }; From a98eed4a064ce0be229a148c4d6f80c9c0f1a544 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 15:16:31 +1100 Subject: [PATCH 0196/2680] arm64: dts: apple: t8140: Describe the memory cache controller Add the four system level cache planes of the T8140 memory cache controller, with the display as the device that must take over its bootloader DMA before cache allocation is enabled, and enable it on J700. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 4 ++++ arch/arm64/boot/dts/apple/t8140.dtsi | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index f0c6d917595ee..931fe13b0b17f 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -412,6 +412,10 @@ status = "okay"; }; +&mcc { + status = "okay"; +}; + &pmp { status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index b4f6219510a15..d4ce3b6dddf79 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -598,6 +598,20 @@ apple,fast-die-effort = <&pmp_report 1>; }; + /* + * System level cache planes. Allocation waits for the display, which + * takes over DMA the bootloader left running. + */ + mcc: memory-controller@220000000 { + compatible = "apple,t8140-mcc"; + reg = <0x2 0x20000000 0x0 0x4000>, + <0x2 0x20040000 0x0 0x4000>, + <0x2 0x20080000 0x0 0x4000>, + <0x2 0x200c0000 0x0 0x4000>; + apple,boot-dma-devices = <&display>; + status = "disabled"; + }; + sep_dart: iommu@280b80000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; reg = <0x2 0x80b80000 0x0 0x20000>; From 70d3c43b13911d2a79e27b10c0b8dbcaf1d810be Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 18:11:40 -0400 Subject: [PATCH 0197/2680] arm64: dts: apple: t8140: describe the PMC and report GFX to the PMP Describe the PMC voter interface with the PMP, CLPC and AP agents as static voters, the GPU's dynamic agent as a power domain, and the host performance controller's agent. Add the PMC's PMGR power state. Report the GPU to the PMP through a report entry for PMP soc-device 9, whose type in the Apple device tree requires an acknowledgment. The GPU now sits on the PMC GFX agent, which sits on the report entry, which sits on the GFX PMGR state. Powering the GPU on therefore writes the PMGR state, then reports it to the PMP, then enables its PMC agent, and powering it off undoes them in reverse order, as macOS does. Enable the PMC on J700. Assisted-by: LLM Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t8140-j700.dts | 4 ++ arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 9 +++++ arch/arm64/boot/dts/apple/t8140.dtsi | 45 ++++++++++++++++++++++- 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 931fe13b0b17f..e433db4472941 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -420,6 +420,10 @@ status = "okay"; }; +&pmc { + status = "okay"; +}; + &pinctrl_ap { status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index c2c6d682c5981..0bd81e5f8e6c5 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -32,6 +32,15 @@ label = "gpio"; }; + ps_pmc: power-controller@138 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x138 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmc"; + apple,always-on; + }; + ps_afggfxni0: power-controller@1b8 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x1b8 4>; diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index d4ce3b6dddf79..b643f4dbfa161 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -485,7 +485,19 @@ compatible = "apple,t8140-pmp-v2-report"; reg = <0x3 0x003c0000 0x0 0x14000>; power-domains = <&ps_pms_sram>; + #address-cells = <1>; + #size-cells = <0>; status = "disabled"; + + /* PMP soc-device 9, which the PMP acknowledges */ + pmp_report_gfx: report@9 { + compatible = "apple,t8140-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x9>; + label = "pmp-gfx"; + #power-domain-cells = <0>; + power-domains = <&ps_gfx>; + }; }; pmp: pmp@300500000 { @@ -1123,7 +1135,7 @@ <0x4 0x80000000 0x0 0x4000000>; reg-names = "asc", "sgx"; mboxes = <&gfx_mbox>, <&gfx1_mbox>; - power-domains = <&ps_gfx>; + power-domains = <&pmc_gfx>; memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>; memory-region-names = "ttbs", "pagetables", "handoff"; @@ -1210,6 +1222,37 @@ status = "disabled"; }; + /* + * Perf-state floors of voter agents 0-3, and the rail state and + * interface enable words, in the PMC window at 0x302800000. + */ + pmc: power-management@302800000 { + compatible = "apple,t8140-pmc"; + reg = <0x3 0x02800000 0x0 0x10000>, + <0x3 0x028aa000 0x0 0x3000>; + reg-names = "voters", "rails"; + power-domains = <&ps_pmc>; + #address-cells = <1>; + #size-cells = <0>; + /* PMP, CLPC and AP agents */ + apple,static-voters = <0 1 3>; + status = "disabled"; + + clpc: performance-controller@1 { + compatible = "apple,t8140-clpc"; + reg = <1>; + apple,gpu-power-state = <&ps_gfx>; + }; + + /* Dynamic agent of the GFX PMGR device */ + pmc_gfx: voter@2 { + reg = <2>; + label = "gfx"; + #power-domain-cells = <0>; + power-domains = <&pmp_report_gfx>; + }; + }; + /* * Bandwidth scratch and request registers, 0x20000 and 0x68000 * into the display clock window at 0x302800000 (the sixth disp0 From 15357b7aa2819828cf2afef6aa4bd6f61d054cec Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:41:18 +1100 Subject: [PATCH 0198/2680] arm64: dts: apple: t8140: keep the GPU on its PMGR state With the previous commit the GPU's only power domain is the PMC's GFX voter domain, which only the new PMC driver provides. A kernel without that driver, or built without CONFIG_APPLE_PMC, never probes the GPU: platform 482600000.gpu: deferred probe pending: platform: wait for supplier /soc/power-management@302800000/voter@2 A device tree has to keep working with kernels that predate a driver it describes, so leave the GPU on its PMGR power state as before. The GFX voter does nothing measurable on the J700 anyway: its floor stayed 0 under every GPU load we sampled, and the GPU firmware has no mapping of the PMC window to write it through. The GPU throughput gain comes from the report entry alone. Once the PMP is told that the GPU is on, it raises its own DRAM controller (DCS) vote under GPU load, which the boot firmware already lets through on the DCS rail. With only the report bit set, a kernel without any PMC driver runs glmark2's build scene at 2199 instead of 1453; enabling every PMC agent on every rail on top of that changes it to 2218. Report the GPU as on from probe instead, as t8112 does for its display. The host never turns the GPU's PMGR state off at runtime, and hardware auto-PM already gates the GPU while it stays requested on, so the PMP sees the same thing it does with the report in the GPU's power path: a reported-on GPU that is gated whenever it idles. It votes the DCS up only under GPU load. Every kernel with the PMP report driver supports the entry. Drop the unused GFX voter node. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index b643f4dbfa161..edd81b9b10c22 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -497,6 +497,7 @@ label = "pmp-gfx"; #power-domain-cells = <0>; power-domains = <&ps_gfx>; + apple,always-on; }; }; @@ -1135,7 +1136,7 @@ <0x4 0x80000000 0x0 0x4000000>; reg-names = "asc", "sgx"; mboxes = <&gfx_mbox>, <&gfx1_mbox>; - power-domains = <&pmc_gfx>; + power-domains = <&ps_gfx>; memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>; memory-region-names = "ttbs", "pagetables", "handoff"; @@ -1243,14 +1244,6 @@ reg = <1>; apple,gpu-power-state = <&ps_gfx>; }; - - /* Dynamic agent of the GFX PMGR device */ - pmc_gfx: voter@2 { - reg = <2>; - label = "gfx"; - #power-domain-cells = <0>; - power-domains = <&pmp_report_gfx>; - }; }; /* From e51cc0ed715849f2451cc8d3181e2b3168f4ab83 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:49:16 +1100 Subject: [PATCH 0199/2680] arm64: dts: apple: t8140: describe the PMC's DCS limit The PMC can limit the DRAM controller rail as a cooling device, so give it cooling cells. Also correct the comment on its second register region, which holds the floor the PMC honours for each agent and the interface enable word of each rail rather than per-rail states. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index edd81b9b10c22..db2caf6194a6b 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -1224,8 +1224,9 @@ }; /* - * Perf-state floors of voter agents 0-3, and the rail state and - * interface enable words, in the PMC window at 0x302800000. + * Perf-state floors of voter agents 0-3, and the floor the PMC + * honours for each agent and the interface enable word of each + * rail, in the PMC window at 0x302800000. */ pmc: power-management@302800000 { compatible = "apple,t8140-pmc"; @@ -1235,6 +1236,7 @@ power-domains = <&ps_pmc>; #address-cells = <1>; #size-cells = <0>; + #cooling-cells = <2>; /* PMP, CLPC and AP agents */ apple,static-voters = <0 1 3>; status = "disabled"; From 9966d9d86a0b47f10d7689faca648ea7d2a5d7ee Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:49:17 +1100 Subject: [PATCH 0200/2680] arm64: dts: apple: t8140-j700: limit the DCS vote when the die runs hot Once the PMP knows that the GPU is on, it votes the DRAM controller (DCS) up with GPU load. On a GPU-bound load the CPU zones cannot hold the resulting heat back, because the CPUs request little power: with the performance governor the CPU die sensors passed 95 C within 35 s and reached 104 C before the CPUs throttled. Add a zone that limits that vote through the PMC. Its sensor, the SMC's Tp0t, reads at or above every CPU die sensor under GPU load. Describe the measured system power of each DCS state so that the power allocator can drive the limit, and make the sustainable power the whole DCS range, so that the vote is only trimmed above the control trip. With the zone, three minutes of mixed GL and Vulkan load under the performance governor peak at 97 C on the CPU zones, as the same load does without any DCS vote, and ten minutes of it under schedutil keep within 1-2% of the throughput of the uncontrolled vote. Tp0t becomes channel 6 of the SMC hwmon so that the zone can refer to it by index, as the CPU zones do. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 63 ++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index e433db4472941..1a06bce2f9dc9 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -379,6 +379,52 @@ }; }; }; + + /* + * Once the PMP knows the GPU is on, it votes the DRAM controller + * (DCS) up with GPU load, and the GPU and memory system draw up + * to 2.4 W more. The CPU zones above cannot hold that back: on a + * GPU-bound load the CPUs request little power, so with the + * performance governor the die passed 95 C within 35 s and + * reached 104 C before they throttled. This zone limits the DCS + * vote instead, through the PMC. With it the same load peaks at + * 97 C, as it does without any DCS vote, and a ten-minute + * balanced GPU load loses 1-2% of its throughput. + * + * Tp0t reads at or above every CPU die sensor under GPU load, + * and leads them by about 1 C when the CPUs also run hot. The + * sustainable power is the whole DCS range, so the vote is only + * trimmed above the control trip; the PID gains follow from it + * and the 6 C between the trips. + */ + soc-dcs-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 6>; /* Tp0t */ + sustainable-power = <2407>; + + trips { + soc_dcs_alert: trip-alert { + temperature = <89000>; + hysteresis = <2000>; + type = "passive"; + }; + + soc_dcs_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + }; + + cooling-maps { + map0 { + trip = <&soc_dcs_control>; + cooling-device = + <&pmc THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; }; }; @@ -421,6 +467,11 @@ }; &pmc { + /* + * System power above DCS state 0 at each DCS state, measured with + * glmark2's terrain scene, GPU-bound, on the J700. + */ + apple,dcs-power-microwatt = <0 919000 1453000 2058000 2407000>; status = "okay"; }; @@ -622,6 +673,13 @@ * float, reads 10-99 C and moves between samples. The names follow * the SMC key groups and are not otherwise verified. */ + + /* Channel 6, the sensor of soc-dcs-thermal: keep it first here. */ + temperature-Tp0t { + apple,key-id = "Tp0t"; + label = "SMC Tp0t"; + }; + temperature-TH0p { apple,key-id = "TH0p"; label = "Storage (NAND)"; @@ -898,11 +956,6 @@ label = "SMC Tp0q"; }; - temperature-Tp0t { - apple,key-id = "Tp0t"; - label = "SMC Tp0t"; - }; - temperature-Tpx0 { apple,key-id = "Tpx0"; label = "SMC Tpx0"; From 96d8a4b89d4cff9a8d49876e2581c10c83348744 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 01:29:46 +1100 Subject: [PATCH 0201/2680] arm64: dts: apple: t8140: Keep the GPU power state on across sleep The GPU driver leaves the GPU firmware running across system sleep, but once the GPU device has suspended nothing keeps the gfx power state in use, and genpd switches it off in the noirq phase. The firmware does not survive that for long. On the J700, a pm_test=platform cycle that holds the devices suspended for five seconds comes back with working GPU rendering, but holding them for a minute, or a real 30 or 90 second suspend-to-idle, hangs the SoC during resume until the watchdog resets it. With the GPU driver unbound, the one-minute cycle resumes. Mark the state always-on, as ANS already is. Runtime behaviour does not change: the GPU device never runtime suspends, so genpd only ever switches this state off for system sleep. Keeping it on also matches the PMP report entry for the GPU, which already reports the GPU as on from probe. With the state kept on, the one-minute cycle and real 30 and 90 second suspend-to-idle cycles woken by the RTC all resume, and the GPU renders afterwards. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 0bd81e5f8e6c5..d3c51235542ac 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -56,6 +56,11 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "gfx"; + /* + * The GPU firmware keeps running across system sleep and + * does not survive this state going off for long. + */ + apple,always-on; }; ps_venc0_sys: power-controller@240 { From c26d3493b1c6ce8bd8df26be0a3c438385c60606 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 01:29:49 +1100 Subject: [PATCH 0202/2680] arm64: dts: apple: t8140-j700: give the two RTCs fixed numbers The J700 has two RTCs on the same Baku PMU counter: the SMC RTC, which sets the clock, and the PMU RTC, which provides the wake alarm and cannot set the time. Both drivers are modules and register in whichever order they happen to load, so either one can end up as rtc0. rtc0 is what hctosys, NTP's RTC sync and tools such as hwclock and rtcwake use by default, and setting the time through the PMU RTC fails. Over six boots of the same kernel, the PMU RTC registered first, as rtc0, on four, and set the system clock. Add rtc aliases so that the SMC RTC is always rtc0 and the PMU RTC rtc1. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 2 ++ arch/arm64/boot/dts/apple/t8140.dtsi | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 1a06bce2f9dc9..842e1b2bcf572 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -26,6 +26,8 @@ sep = &sep; serial0 = &serial0; pmp = &pmp; + rtc0 = &smc_rtc; + rtc1 = &pmu_rtc; wifi0 = &wifi0; /* used by the loader to hand over the display */ dcp = &dcp; diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index db2caf6194a6b..cc071157054b8 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -832,7 +832,7 @@ "boot_error_count", "panic_count"; }; - rtc { + smc_rtc: rtc { compatible = "apple,smc-rtc"; nvmem-cells = <&rtc_offset>; nvmem-cell-names = "rtc_offset"; From 3c71b834db19f69918e2b949db499da757099899 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 01:44:16 +1100 Subject: [PATCH 0203/2680] arm64: dts: apple: t8140: Keep the GPIO power state on for the SEP The SEP stops reading its mailbox for good once the GPIO power state is switched off while it runs, which genpd does in the noirq phase of every suspend after the pin controller has suspended. The next message to the enclave, typically from the hwrng fill thread, then wedges it until reboot. Give the SEP the GPIO power state as its second power domain, so that the driver keeps it on for as long as the enclave is in use. Fixes: aec8cb1999f7 ("arm64: dts: apple: t8140: Add the SEP, its DART and mailbox, and SPI2") Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index cc071157054b8..89f12fbd90632 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -645,7 +645,11 @@ mboxes = <&sep_mbox>; mbox-names = "mbox"; iommus = <&sep_dart 0>; - power-domains = <&ps_sep>; + /* + * The enclave stops reading its mailbox for good once + * the GPIO power state is switched off under it. + */ + power-domains = <&ps_sep>, <&ps_gpio>; status = "disabled"; }; From ba2f9da096c294e090a50d44fa751a5bc156c413 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 12 Oct 2026 14:30:09 -0400 Subject: [PATCH 0204/2680] arm64: dts: apple: t8140-j700: keep the DCS rail out of thermal control The soc-dcs-thermal zone let the power allocator cap the DRAM controller (DCS) rail through the PMC whenever Tp0t passed 95 C. Most of that heat comes from the GPU and CPUs rather than from the DCS rail, so the governor could not settle. Under a sustained Minecraft load it stepped the cap between states 1 and 4 every 1-2 s, and at state 4 the rail was held at its lowest state while the PMP and CLPC both voted the highest. The game's GPU fence rate followed the cap (Pearson r = -0.98): about 2,000/s at state 4 and 5,200/s at state 1. In game the frame rate was seen swinging from about 150 FPS down to 30-50 FPS and back every few seconds. With the zone disabled and the cap lifted on the same session, the rate held between 4,500 and 4,900/s. Tp0t settled near 100 C (peak 105 C), and the CPU die sensors stayed at 88-97 C, under the 110 C critical trips of the CPU zones. The PMP already drops its DCS vote when the GPU idles, so the rail only needs to follow demand. Drop the zone. The PMC keeps its cooling cells and power model. Assisted-by: LLM Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t8140-j700.dts | 56 ++++-------------------- 1 file changed, 9 insertions(+), 47 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 842e1b2bcf572..066175edbc83a 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -381,52 +381,6 @@ }; }; }; - - /* - * Once the PMP knows the GPU is on, it votes the DRAM controller - * (DCS) up with GPU load, and the GPU and memory system draw up - * to 2.4 W more. The CPU zones above cannot hold that back: on a - * GPU-bound load the CPUs request little power, so with the - * performance governor the die passed 95 C within 35 s and - * reached 104 C before they throttled. This zone limits the DCS - * vote instead, through the PMC. With it the same load peaks at - * 97 C, as it does without any DCS vote, and a ten-minute - * balanced GPU load loses 1-2% of its throughput. - * - * Tp0t reads at or above every CPU die sensor under GPU load, - * and leads them by about 1 C when the CPUs also run hot. The - * sustainable power is the whole DCS range, so the vote is only - * trimmed above the control trip; the PID gains follow from it - * and the 6 C between the trips. - */ - soc-dcs-thermal { - polling-delay-passive = <500>; - polling-delay = <500>; - thermal-sensors = <&smc_hwmon 6>; /* Tp0t */ - sustainable-power = <2407>; - - trips { - soc_dcs_alert: trip-alert { - temperature = <89000>; - hysteresis = <2000>; - type = "passive"; - }; - - soc_dcs_control: trip-control { - temperature = <95000>; - hysteresis = <2000>; - type = "passive"; - }; - }; - - cooling-maps { - map0 { - trip = <&soc_dcs_control>; - cooling-device = - <&pmc THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - }; - }; }; }; @@ -472,6 +426,14 @@ /* * System power above DCS state 0 at each DCS state, measured with * glmark2's terrain scene, GPU-bound, on the J700. + * + * No thermal zone limits the DCS rail. The PMP already drops its + * DCS vote when the GPU idles, and under load the rail must stay + * where the PMP puts it: a power allocator zone on Tp0t at 95 C + * stepped the limit between states 1 and 4 every 1-2 s in + * Minecraft, swinging the game's GPU submission rate about 2.5x. + * With the limit lifted the rate held within 8% and Tp0t settled + * near 100 C, under the 110 C critical trips of the CPU zones. */ apple,dcs-power-microwatt = <0 919000 1453000 2058000 2407000>; status = "okay"; @@ -676,7 +638,7 @@ * the SMC key groups and are not otherwise verified. */ - /* Channel 6, the sensor of soc-dcs-thermal: keep it first here. */ + /* Channel 6: reads at or above every CPU die sensor under GPU load. */ temperature-Tp0t { apple,key-id = "Tp0t"; label = "SMC Tp0t"; From b888c5478b3100728b777116c6d83baae93d2bfc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:20:49 +1100 Subject: [PATCH 0205/2680] arm64: dts: apple: describe disabled J700 external resources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe the external display controller and four display apertures plus its fifth clock aperture from bounded read-only ADT observations. Select the first observed DCP mapper and the observed PIODMA stream. Add the reciprocal rear Type-C graph while retaining disabled controller and route status. Register names follow existing public bindings. Mailbox, firmware handoff, routing and bandwidth integration remain qualification requirements. This resource description is not a display readiness claim. [ryan: adapted to 7.2.10 — drop the Generated-by/Assisted-by lines from the file header comment, as aurora-wip did in a merge resolution] Assisted-by: LLM Signed-off-by: Ryan Murray --- .../boot/dts/apple/t8140-j700-dcpext.dtsi | 67 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8140-j700.dts | 3 + 2 files changed, 70 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi diff --git a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi new file mode 100644 index 0000000000000..9ab467f65b26a --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * T8140 external display resources measured through the public ADT API. + * + * The controller and route remain disabled. Their live firmware + * profile, bandwidth request, routing selector and handoff must be qualified + * before enabling the controller. The existing T8140 refusal remains in force. + */ + +&{/soc} { + dcpext: dcp@416e00000 { + compatible = "apple,t8140-dcpext"; + reg = <0x4 0x16e00000 0x0 0x4000>, + <0x4 0x14000000 0x0 0xafc000>, + <0x4 0x16240000 0x0 0x10000>, + <0x4 0x163ac000 0x0 0x4000>, + <0x4 0x16800000 0x0 0x800000>, + <0x3 0x02800000 0x0 0xbc000>, + <0x4 0x16850000 0x0 0x4000>; + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4", "iop-vbar"; + mboxes = <&dcpext_mbox>; + mbox-names = "mbox"; + /* First mapper in the measured ADT iommu-parent list. */ + iommus = <&dcpext0_dart 23>; + power-domains = <&ps_dispext0_cpu>; + apple,connector-type = "USB-C"; + status = "disabled"; + + #address-cells = <2>; + #size-cells = <2>; + ranges; + + dispext0_piodma: piodma { + iommus = <&dispext0_dart 16>; + }; + + typec-routes { + #address-cells = <1>; + #size-cells = <0>; + + route@0 { + reg = <0>; + status = "disabled"; + mode-switch; + + port { + dcpext_typec0: endpoint { + remote-endpoint = <&typec0_dcp>; + }; + }; + }; + }; + }; +}; + +&typec0 { + ports { + port@3 { + reg = <3>; + + typec0_dcp: endpoint { + remote-endpoint = <&dcpext_typec0>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 066175edbc83a..beacbf3168957 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -1880,3 +1880,6 @@ local-mac-address = [00 00 00 00 00 00]; }; }; + +/* External resources stay disabled pending the separate firmware profile. */ +#include "t8140-j700-dcpext.dtsi" From a8674c148db6dfc461ae5ee4a48107d29b82f4d7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:30:45 +1100 Subject: [PATCH 0206/2680] arm64: dts: apple: t8140: describe the external binding the display master recognizes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the generic DCP fallback required by the published T8140 external binding and explicitly classify that specific compatible as an external pipeline. Otherwise the display master never discovers a successfully prepared external node. The device and route remain disabled by default. [ryan: adapted to 7.2.10 — split: DT part only; the apple_drv.c part goes with 200-dcp] Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi index 9ab467f65b26a..fc8a1560f78f2 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi @@ -9,7 +9,7 @@ &{/soc} { dcpext: dcp@416e00000 { - compatible = "apple,t8140-dcpext"; + compatible = "apple,t8140-dcpext", "apple,dcp"; reg = <0x4 0x16e00000 0x0 0x4000>, <0x4 0x14000000 0x0 0xafc000>, <0x4 0x16240000 0x0 0x10000>, From a818c5b66018d600908838fc1ff948cc34d2f171 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:31:34 +1100 Subject: [PATCH 0207/2680] arm64: dts: apple: retain the inherited external display supply Keep the external CPU supply on after the loader validates and publishes its power-domain chain. The supply remains disabled in the static tree, so a refused handoff does not register or power it. This preserves firmware ownership while allowing mailbox consumers to bind. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi index fc8a1560f78f2..df95287a0f3aa 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi @@ -65,3 +65,8 @@ }; }; }; + +/* Retain the inherited firmware supply once the loader publishes it. */ +&ps_dispext0_cpu { + apple,always-on; +}; From ba6fdf817594332902ba6fa2cda456fe210d9b22 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:40:12 +1100 Subject: [PATCH 0208/2680] arm64: dts: apple: add an explicit external display trial tree Describe the rear Type-C route and both scanout DARTs in a separate test DTB. Keep the normal J700 tree unchanged. The external DART must be withdrawn by the paired loader when external validation fails. Route selectors and external bandwidth parameters still require hardware qualification. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/Makefile | 1 + .../dts/apple/t8140-j700-external-test.dts | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8140-j700-external-test.dts diff --git a/arch/arm64/boot/dts/apple/Makefile b/arch/arm64/boot/dts/apple/Makefile index 3edac61fba143..7e53575ab7998 100644 --- a/arch/arm64/boot/dts/apple/Makefile +++ b/arch/arm64/boot/dts/apple/Makefile @@ -110,5 +110,6 @@ dtb-$(CONFIG_ARCH_APPLE) += t8132-j713.dtb dtb-$(CONFIG_ARCH_APPLE) += t8132-j715.dtb dtb-$(CONFIG_ARCH_APPLE) += t8132-j773g.dtb dtb-$(CONFIG_ARCH_APPLE) += t8140-j700.dtb +dtb-$(CONFIG_ARCH_APPLE) += t8140-j700-external-test.dtb dtb-$(CONFIG_ARCH_APPLE) += t8142-j813.dtb dtb-$(CONFIG_ARCH_APPLE) += t8152-j873g.dtb diff --git a/arch/arm64/boot/dts/apple/t8140-j700-external-test.dts b/arch/arm64/boot/dts/apple/t8140-j700-external-test.dts new file mode 100644 index 0000000000000..6dde297589d65 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8140-j700-external-test.dts @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* Explicit external-display trial; the ordinary J700 tree is unchanged. */ +#include "t8140-j700.dts" +#include + +&display { + iommus = <&disp0_dart 0>, <&dispext0_dart 0>; +}; + +&dcpext { + clocks = <&clk_disp0_154>, <&clk_disp0_194>; + clock-names = "clock-154", "clock-194"; + phys = <&atcphy0 PHY_TYPE_DP>; + phy-names = "typec0"; + mux-controls = <&dpxbar0 0>; + mux-control-names = "typec0"; + apple,typec-mux-indices = <0>; + apple,typec-dptx-phys = <0>; + apple,dcp-index = <1>; +}; + +&{/soc/dcp@416e00000/typec-routes/route@0} { + status = "okay"; +}; From 6c33e674ce580bb5ff296d457c1e82f59bf14d8d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 12 Oct 2026 19:11:08 -0400 Subject: [PATCH 0209/2680] arm64: dts: apple: t8140-j700: drive the SPI NOR on a single data line With spi-rx-bus-width and spi-tx-bus-width set to 4, the first command spi-nor sends at probe (READ ID) times out on J700 with every byte moved but no completion: "transfer timed out: p=0x100000 c=0x700 ... tx=0 rx=0", so no MTD device and no NVRAM partition appear and asahi-bless cannot read or write the boot volume selection. The same driver probes the flash, creates the nvram partition and lets asahi-bless list volumes when the node describes single data lines, at either 25 or 40 MHz. The ADT's spinor node gives 25 MHz and does not describe quad lanes, and the J700 kernels that probed this flash never declared them. Drop the quad widths and keep the ADT frequency. Tested on J700 (7.1.12, QSPIMC 0x20000): 25 MHz single-line probes and creates "nvram" (0x1d0000 bytes); asahi-bless --list-volumes works. 25 MHz quad fails at the first command. Co-Authored-By: Claude Opus 5.5 --- arch/arm64/boot/dts/apple/t8140-j700.dts | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index beacbf3168957..44143ae11092e 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -462,8 +462,6 @@ compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <25000000>; - spi-rx-bus-width = <4>; - spi-tx-bus-width = <4>; partitions { compatible = "fixed-partitions"; From af1dfd89642a8de3828404a853b2bf3906151023 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 13 Oct 2026 07:38:47 -0400 Subject: [PATCH 0210/2680] arm64: dts: apple: t8140-j700: enable the validated external display profile by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use native PMGR bandwidth scratch and clock-request offsets from the T8140 hardware profile. Keep diagnostic overrides bounded and retain the disable switch and loader firmware validation. Put the existing tested external route and scanout window in the normal J700 tree so users need no manual command line or alternate DTB. [ryan: adapted to 7.2.10 — split: DT part only; the drm/apple and selftest parts go with 200-dcp. t8140-j700-external-test.dts is removed here as aurora-wip did in its merge resolution] --- arch/arm64/boot/dts/apple/Makefile | 1 - .../boot/dts/apple/t8140-j700-dcpext.dtsi | 5 ++-- .../dts/apple/t8140-j700-external-test.dts | 24 ----------------- arch/arm64/boot/dts/apple/t8140-j700.dts | 27 ++++++++++++++++++- 4 files changed, 28 insertions(+), 29 deletions(-) delete mode 100644 arch/arm64/boot/dts/apple/t8140-j700-external-test.dts diff --git a/arch/arm64/boot/dts/apple/Makefile b/arch/arm64/boot/dts/apple/Makefile index 7e53575ab7998..3edac61fba143 100644 --- a/arch/arm64/boot/dts/apple/Makefile +++ b/arch/arm64/boot/dts/apple/Makefile @@ -110,6 +110,5 @@ dtb-$(CONFIG_ARCH_APPLE) += t8132-j713.dtb dtb-$(CONFIG_ARCH_APPLE) += t8132-j715.dtb dtb-$(CONFIG_ARCH_APPLE) += t8132-j773g.dtb dtb-$(CONFIG_ARCH_APPLE) += t8140-j700.dtb -dtb-$(CONFIG_ARCH_APPLE) += t8140-j700-external-test.dtb dtb-$(CONFIG_ARCH_APPLE) += t8142-j813.dtb dtb-$(CONFIG_ARCH_APPLE) += t8152-j873g.dtb diff --git a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi index df95287a0f3aa..69d6eff9aa756 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi @@ -2,9 +2,8 @@ /* * T8140 external display resources measured through the public ADT API. * - * The controller and route remain disabled. Their live firmware - * profile, bandwidth request, routing selector and handoff must be qualified - * before enabling the controller. The existing T8140 refusal remains in force. + * The controller remains disabled until the loader validates and preserves + * the inherited firmware and its complete dependency group. */ &{/soc} { diff --git a/arch/arm64/boot/dts/apple/t8140-j700-external-test.dts b/arch/arm64/boot/dts/apple/t8140-j700-external-test.dts deleted file mode 100644 index 6dde297589d65..0000000000000 --- a/arch/arm64/boot/dts/apple/t8140-j700-external-test.dts +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ OR MIT -/* Explicit external-display trial; the ordinary J700 tree is unchanged. */ -#include "t8140-j700.dts" -#include - -&display { - iommus = <&disp0_dart 0>, <&dispext0_dart 0>; -}; - -&dcpext { - clocks = <&clk_disp0_154>, <&clk_disp0_194>; - clock-names = "clock-154", "clock-194"; - phys = <&atcphy0 PHY_TYPE_DP>; - phy-names = "typec0"; - mux-controls = <&dpxbar0 0>; - mux-control-names = "typec0"; - apple,typec-mux-indices = <0>; - apple,typec-dptx-phys = <0>; - apple,dcp-index = <1>; -}; - -&{/soc/dcp@416e00000/typec-routes/route@0} { - status = "okay"; -}; diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 44143ae11092e..033272cdb32d9 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -1879,5 +1879,30 @@ }; }; -/* External resources stay disabled pending the separate firmware profile. */ +/* The loader publishes external resources after validating their firmware. */ #include "t8140-j700-dcpext.dtsi" + +&display { + iommus = <&disp0_dart 0>, <&dispext0_dart 0>; +}; + +&dcpext { + clocks = <&clk_disp0_154>, <&clk_disp0_194>; + clock-names = "clock-154", "clock-194"; + phys = <&atcphy0 PHY_TYPE_DP>; + phy-names = "typec0"; + mux-controls = <&dpxbar0 0>; + mux-control-names = "typec0"; + apple,typec-mux-indices = <2>; + apple,typec-dptx-phys = <0>; + apple,dcp-index = <1>; +}; + +&{/soc/dcp@416e00000/typec-routes/route@0} { + status = "okay"; +}; + +/* Allocate scanout buffers in the window used by the inherited firmware. */ +&dcpext0_dart { + apple,dma-range = <0x100 0x0 0x10 0x0>; +}; From eb3a7a634b911dafbff96b4ca154e4beded29b20 Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Tue, 13 Oct 2026 02:54:20 +0200 Subject: [PATCH 0211/2680] arm64: dts: apple: t8140: mark firmware-owned power domains MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On MacBook Neo (J700) m1n1 hands the display coprocessor (DCP) over running, and its firmware owns the DISP_CPU power state: writing that register raises an SError. The domain is marked apple,always-on, but the driver still writes it at probe: it sets automatic gating when the domain is on, or forces it on when it is not. The kernel panicked with an asynchronous SError in apple_pmgr_ps_probe before any console. Add apple,firmware-owned: the driver only reads such a domain, keeps it on for its consumers (the boot framebuffer), and never writes its minimum state, power target, automatic gating or reset. Mark J700's disp_cpu with it. [ryan: adapted to 7.2.10 — split: DT part only; the pmgr-pwrstate driver and binding go with 010-soc] Co-Authored-By: Claude Opus 5.5 --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index d3c51235542ac..a51052308a74a 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -397,6 +397,7 @@ * firmware owns this state; writing it faults. */ apple,always-on; + apple,firmware-owned; }; ps_venc0_dma: power-controller@8000 { From 5ba81ddd9c8996c0edc9278028574bc0890c6b36 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:26:58 +1000 Subject: [PATCH 0212/2680] dt-bindings: reserved-memory: Add apple,asc-mem isp-common.dtsi reserves the firmware heap of the Apple ISPs with an "apple,asc-mem" node whose address and size are left zero for the bootloader to fill in, but the compatible has no binding. Document it: a no-map region holding memory that the bootloader set aside for a co-processor it loaded, such as the ISP firmware heap. The bootloader supplies the address and size, and records any IOVA mappings the firmware relies on in iommu-addresses. The region is not a general DMA pool, and the binding fixes no address or size. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../reserved-memory/apple,asc-mem.yaml | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 Documentation/devicetree/bindings/reserved-memory/apple,asc-mem.yaml diff --git a/Documentation/devicetree/bindings/reserved-memory/apple,asc-mem.yaml b/Documentation/devicetree/bindings/reserved-memory/apple,asc-mem.yaml new file mode 100644 index 0000000000000..05540204078be --- /dev/null +++ b/Documentation/devicetree/bindings/reserved-memory/apple,asc-mem.yaml @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/reserved-memory/apple,asc-mem.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple coprocessor reserved memory + +maintainers: + - Ryan Murray + +description: + Memory retained for a bootloader-loaded Apple coprocessor, including the + ISP firmware heap. The bootloader supplies the physical address and size + from its firmware handoff and records any associated device IOVA mappings + in iommu-addresses. Static board sources may carry an empty placeholder + that is filled before entering the kernel. The region is not a generic + allocatable DMA pool and this binding specifies no fixed address or size. + +allOf: + - $ref: reserved-memory.yaml# + +properties: + compatible: + const: apple,asc-mem + + reg: + maxItems: 1 + + no-map: true + + iommu-addresses: true + +required: + - compatible + - reg + +unevaluatedProperties: false From 5b7b6425cbdba1f9cf9b0ed8d985f791be2d0671 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 01:09:44 +1000 Subject: [PATCH 0213/2680] dt-bindings: chosen: document apple,dcp-rtkit-quiesced MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The loader reports whether the DCP core was running at hand-off using a u32 in /chosen. Document both values and allow the property to be absent on a chainload. The kernel uses this label to select its start mode once the H17P path is implemented (DT-5, LIFE-1). [ryan: adapted to 7.2.10 — binding only; the MAINTAINERS hunk goes with 900-config-tooling] Assisted-by: LLM Signed-off-by: Ryan Murray --- .../chosen/apple,dcp-rtkit-quiesced.yaml | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml diff --git a/Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml b/Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml new file mode 100644 index 0000000000000..d086bdc05f5ee --- /dev/null +++ b/Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/chosen/apple,dcp-rtkit-quiesced.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple DCP RTKit loader hand-off + +maintainers: + - Ryan Murray + +description: + The loader records whether it handed a running display coprocessor to + the kernel. It omits this property on a chainload. + +select: + properties: + $nodename: + const: chosen + required: + - $nodename + +properties: + $nodename: + const: chosen + + apple,dcp-rtkit-quiesced: + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1] + description: + One if the DCP core was running at hand-off, zero if it was not. + The name reflects the loader's RTKit hand-off state, not a request + for Linux to quiesce the coprocessor. + +additionalProperties: true +... From ea3a1f82296ac4d3ee31de238813dd64566ab17f Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:59:58 -0700 Subject: [PATCH 0214/2680] vfio/pci: refuse assignment on hosts retaining reset-sensitive DMA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A closed Neo Bluetooth function can otherwise be assigned to vfio-pci, whose open and hot-reset paths bypass the PCI sysfs reset veto. Honor the immutable host reset restriction before VFIO registration, reset-method probing or power management. Other hosts leave the flag clear and retain the existing assignment path. [ryan: adapted to 7.2.10 — moved from the mm-userpages bucket into the radio topic, which adds the reset_sensitive host flag it checks; 220-userpages skips it] --- drivers/vfio/pci/vfio_pci_core.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 6a184588ff234..6d0ecdb2c5b16 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -2222,6 +2222,13 @@ int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev) struct device *dev = &pdev->dev; int ret; + /* Assignment exposes function/bus resets outside PCI sysfs. */ + if (pci_find_host_bridge(pdev->bus)->no_user_reset) { + dev_warn_ratelimited(dev, + "VFIO assignment refused: host bridge forbids userspace reset\n"); + return -EOPNOTSUPP; + } + /* Drivers must set the vfio_pci_core_device to their drvdata */ if (WARN_ON(vdev != dev_get_drvdata(dev))) return -EINVAL; From 562f5f9d5908a6cd9c56c1e0d65feb0170ec3e25 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:42:49 -0700 Subject: [PATCH 0215/2680] apple: give retained-host refusals accurate names and errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename no_user_reset to reset_sensitive because the immutable host flag also protects assignment and recovery. Return EOPNOTSUPP for permanent PCI sysfs policy and test that actual errno without unregistered callbacks. Describe sleep/kexec refusal as an active interlock during admission or retained ownership, so the temporary window is not mislabeled permanent. Round-2 S4. Changed refusals require Neo flags or an active Neo interlock; default-clear hosts keep original policy and transition admission. Non-Neo behavior remains unchanged. [ryan: adapted to 7.2.10 — split by topic: the kernel/kexec*.c and kernel/crash_core.c message renames go to 010-soc, which carries the kexec interlock] --- .../pci/controller/pcie-apple-piodma-diag.c | 8 ++-- drivers/pci/controller/pcie-apple.c | 2 +- drivers/pci/pci-sysfs-test.c | 42 +++++++++---------- drivers/pci/pci-sysfs.c | 26 +++++++----- drivers/pci/pci.h | 4 +- drivers/pci/pcie/portdrv.c | 4 +- drivers/vfio/pci/vfio_pci_core.c | 4 +- include/linux/pci.h | 6 ++- 8 files changed, 51 insertions(+), 45 deletions(-) diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index d5f757a66a218..9fbc04faa5ee8 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -57,7 +57,7 @@ static bool enumerate = true; module_param(enumerate, bool, 0444); -MODULE_PARM_DESC(enumerate, "Boot-time gate: prime and enumerate Neo radios (read-only, default on)"); +MODULE_PARM_DESC(enumerate, "Boot-time gate for J700/T8140 radios only (read-only, default on)"); struct apple_piodma_request { u16 offset; @@ -165,7 +165,7 @@ static int apple_piodma_pm_notify(struct notifier_block *nb, case PM_HIBERNATION_PREPARE: case PM_RESTORE_PREPARE: dev_warn_ratelimited(diag->dev, - "sleep refused: Neo radio bootstrap retains DMA memory until full hardware reset\n"); + "sleep refused: Neo guard active during radio admission or retained DMA lifetime\n"); return notifier_from_errno(-EOPNOTSUPP); default: return NOTIFY_DONE; @@ -185,7 +185,7 @@ static int apple_piodma_pm_prepare(struct device *dev) * to device PM, so retain a veto in the device prepare phase as well. */ dev_warn_ratelimited(dev, - "sleep refused: Neo radio bootstrap retains DMA memory until full hardware reset\n"); + "sleep refused: Neo guard active during radio admission or retained DMA lifetime\n"); return -EOPNOTSUPP; } @@ -803,7 +803,7 @@ static int apple_piodma_diag_probe(struct platform_device *pdev) diag->dev = dev; INIT_DELAYED_WORK(&diag->admission_retry, apple_piodma_retry_admission); diag->kexec_blocker.reason = - "Neo radio bootstrap retains DMA memory until full hardware reset"; + "Neo radio admission or retained DMA ownership"; for (i = 0; i < PIODMA_REQUEST_COUNT; i++) init_completion(&diag->slots[i].event); atomic_set(&diag->irq_count, 0); diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 74b3496ed812a..876eee0f6f93c 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2519,7 +2519,7 @@ static int apple_pcie_probe(struct platform_device *pdev) /* Only this retained-DMA radio hierarchy forbids user reset/removal. */ if (piodma_supplier) { bridge->no_user_remove = true; - bridge->no_user_reset = true; + bridge->reset_sensitive = true; bridge->native_pcie_hotplug = false; bridge->native_shpc_hotplug = false; } diff --git a/drivers/pci/pci-sysfs-test.c b/drivers/pci/pci-sysfs-test.c index 47714f825ee69..f24519991d029 100644 --- a/drivers/pci/pci-sysfs-test.c +++ b/drivers/pci/pci-sysfs-test.c @@ -47,9 +47,9 @@ static void pci_sysfs_remove_default_test(struct kunit *test) { struct pci_sysfs_test_hierarchy *h = test->priv; - KUNIT_EXPECT_TRUE(test, pci_sysfs_user_remove_allowed(h->root_port)); - KUNIT_EXPECT_TRUE(test, pci_sysfs_user_remove_allowed(h->endpoint)); - KUNIT_EXPECT_TRUE(test, pci_sysfs_user_remove_allowed(h->grandchild)); + KUNIT_EXPECT_EQ(test, pci_sysfs_remove_check(h->root_port), 0); + KUNIT_EXPECT_EQ(test, pci_sysfs_remove_check(h->endpoint), 0); + KUNIT_EXPECT_EQ(test, pci_sysfs_remove_check(h->grandchild), 0); } static void pci_sysfs_remove_retained_test(struct kunit *test) @@ -57,9 +57,9 @@ static void pci_sysfs_remove_retained_test(struct kunit *test) struct pci_sysfs_test_hierarchy *h = test->priv; h->bridge->no_user_remove = true; - KUNIT_EXPECT_FALSE(test, pci_sysfs_user_remove_allowed(h->root_port)); - KUNIT_EXPECT_FALSE(test, pci_sysfs_user_remove_allowed(h->endpoint)); - KUNIT_EXPECT_FALSE(test, pci_sysfs_user_remove_allowed(h->grandchild)); + KUNIT_EXPECT_EQ(test, pci_sysfs_remove_check(h->root_port), -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, pci_sysfs_remove_check(h->endpoint), -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, pci_sysfs_remove_check(h->grandchild), -EOPNOTSUPP); } static void pci_sysfs_remove_independent_host_test(struct kunit *test) @@ -70,34 +70,34 @@ static void pci_sysfs_remove_independent_host_test(struct kunit *test) other = kunit_kzalloc(test, sizeof(*other), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, other); h->bridge->no_user_remove = true; - KUNIT_ASSERT_FALSE(test, pci_sysfs_user_remove_allowed(h->grandchild)); + KUNIT_ASSERT_EQ(test, pci_sysfs_remove_check(h->grandchild), -EOPNOTSUPP); /* A descendant of an unrestricted host must not inherit another veto. */ h->root_bus->bridge = &other->dev; - KUNIT_EXPECT_TRUE(test, pci_sysfs_user_remove_allowed(h->grandchild)); + KUNIT_EXPECT_EQ(test, pci_sysfs_remove_check(h->grandchild), 0); } static void pci_sysfs_reset_default_test(struct kunit *test) { struct pci_sysfs_test_hierarchy *h = test->priv; - KUNIT_EXPECT_TRUE(test, pci_sysfs_user_reset_allowed(h->root_port)); - KUNIT_EXPECT_TRUE(test, pci_sysfs_user_reset_allowed(h->endpoint)); - KUNIT_EXPECT_TRUE(test, pci_sysfs_user_reset_allowed(h->grandchild)); + KUNIT_EXPECT_EQ(test, pci_sysfs_reset_check(h->root_port), 0); + KUNIT_EXPECT_EQ(test, pci_sysfs_reset_check(h->endpoint), 0); + KUNIT_EXPECT_EQ(test, pci_sysfs_reset_check(h->grandchild), 0); } static void pci_sysfs_reset_retained_test(struct kunit *test) { struct pci_sysfs_test_hierarchy *h = test->priv; - h->bridge->no_user_reset = true; - KUNIT_EXPECT_FALSE(test, pci_sysfs_user_reset_allowed(h->root_port)); - KUNIT_EXPECT_FALSE(test, pci_sysfs_user_reset_allowed(h->endpoint)); - KUNIT_EXPECT_FALSE(test, pci_sysfs_user_reset_allowed(h->grandchild)); + h->bridge->reset_sensitive = true; + KUNIT_EXPECT_EQ(test, pci_sysfs_reset_check(h->root_port), -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, pci_sysfs_reset_check(h->endpoint), -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, pci_sysfs_reset_check(h->grandchild), -EOPNOTSUPP); /* Reset and removal policies are independent for other host users. */ - KUNIT_EXPECT_TRUE(test, pci_sysfs_user_remove_allowed(h->grandchild)); - h->bridge->no_user_reset = false; + KUNIT_EXPECT_EQ(test, pci_sysfs_remove_check(h->grandchild), 0); + h->bridge->reset_sensitive = false; h->bridge->no_user_remove = true; - KUNIT_EXPECT_TRUE(test, pci_sysfs_user_reset_allowed(h->grandchild)); + KUNIT_EXPECT_EQ(test, pci_sysfs_reset_check(h->grandchild), 0); } static void pci_sysfs_reset_independent_host_test(struct kunit *test) @@ -107,10 +107,10 @@ static void pci_sysfs_reset_independent_host_test(struct kunit *test) other = kunit_kzalloc(test, sizeof(*other), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, other); - h->bridge->no_user_reset = true; - KUNIT_ASSERT_FALSE(test, pci_sysfs_user_reset_allowed(h->grandchild)); + h->bridge->reset_sensitive = true; + KUNIT_ASSERT_EQ(test, pci_sysfs_reset_check(h->grandchild), -EOPNOTSUPP); h->root_bus->bridge = &other->dev; - KUNIT_EXPECT_TRUE(test, pci_sysfs_user_reset_allowed(h->grandchild)); + KUNIT_EXPECT_EQ(test, pci_sysfs_reset_check(h->grandchild), 0); } static struct kunit_case pci_sysfs_test_cases[] = { diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c index a554ee24ecb5f..5dae0ffa366f5 100644 --- a/drivers/pci/pci-sysfs.c +++ b/drivers/pci/pci-sysfs.c @@ -511,16 +511,16 @@ static ssize_t dev_rescan_store(struct device *dev, static struct device_attribute dev_attr_dev_rescan = __ATTR(rescan, 0200, NULL, dev_rescan_store); -bool pci_sysfs_user_remove_allowed(struct pci_dev *pdev) +int pci_sysfs_remove_check(struct pci_dev *pdev) { /* Bus/bridge references keep this immutable host property alive. */ - return !pci_find_host_bridge(pdev->bus)->no_user_remove; + return pci_find_host_bridge(pdev->bus)->no_user_remove ? -EOPNOTSUPP : 0; } -bool pci_sysfs_user_reset_allowed(struct pci_dev *pdev) +int pci_sysfs_reset_check(struct pci_dev *pdev) { - /* User resets can disrupt DMA still owned by a retained host. */ - return !pci_find_host_bridge(pdev->bus)->no_user_reset; + /* Uncoordinated resets can invalidate retained DMA ownership. */ + return pci_find_host_bridge(pdev->bus)->reset_sensitive ? -EOPNOTSUPP : 0; } static ssize_t remove_store(struct device *dev, struct device_attribute *attr, @@ -533,10 +533,12 @@ static ssize_t remove_store(struct device *dev, struct device_attribute *attr, return -EINVAL; if (val) { - if (!pci_sysfs_user_remove_allowed(pdev)) { + int ret = pci_sysfs_remove_check(pdev); + + if (ret) { dev_warn_ratelimited(&pdev->dev, "userspace removal is disabled by the host bridge\n"); - return -EBUSY; + return ret; } if (device_remove_file_self(dev, attr)) pci_stop_and_remove_bus_device_locked(pdev); @@ -585,10 +587,11 @@ static ssize_t reset_subordinate_store(struct device *dev, if (val) { int ret; - if (!pci_sysfs_user_reset_allowed(pdev)) { + ret = pci_sysfs_reset_check(pdev); + if (ret) { dev_warn_ratelimited(&pdev->dev, "userspace reset is disabled by the host bridge\n"); - return -EBUSY; + return ret; } ret = pci_try_reset_bridge(pdev); @@ -1601,10 +1604,11 @@ static ssize_t reset_store(struct device *dev, struct device_attribute *attr, if (val != 1) return -EINVAL; - if (!pci_sysfs_user_reset_allowed(pdev)) { + result = pci_sysfs_reset_check(pdev); + if (result) { dev_warn_ratelimited(&pdev->dev, "userspace reset is disabled by the host bridge\n"); - return -EBUSY; + return result; } pm_runtime_get_sync(dev); diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 880d53341c761..eef0c105a6141 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -1458,7 +1458,7 @@ static inline int pci_msix_write_tph_tag(struct pci_dev *pdev, unsigned int inde (PCI_CONF1_ADDRESS(bus, dev, func, reg) | \ PCI_CONF1_EXT_REG(reg)) -bool pci_sysfs_user_remove_allowed(struct pci_dev *pdev); -bool pci_sysfs_user_reset_allowed(struct pci_dev *pdev); +int pci_sysfs_remove_check(struct pci_dev *pdev); +int pci_sysfs_reset_check(struct pci_dev *pdev); #endif /* DRIVERS_PCI_H */ diff --git a/drivers/pci/pcie/portdrv.c b/drivers/pci/pcie/portdrv.c index 1ca9fdb31a618..b0709e3a529f7 100644 --- a/drivers/pci/pcie/portdrv.c +++ b/drivers/pci/pcie/portdrv.c @@ -243,7 +243,7 @@ static int get_port_device_capability(struct pci_dev *dev) #ifdef CONFIG_PCIEAER /* Error recovery may reset links and invalidate retained DMA ownership. */ - if (!host->no_user_reset && (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT || + if (!host->reset_sensitive && (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT || pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC) && dev->aer_cap && pci_aer_available() && (pcie_ports_native || host->native_aer)) @@ -268,7 +268,7 @@ static int get_port_device_capability(struct pci_dev *dev) * With dpc-native, allow Linux to use DPC even if it doesn't have * permission to use AER. */ - if (!host->no_user_reset && + if (!host->reset_sensitive && pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DPC) && pci_aer_available() && (pcie_ports_dpc_native || (services & PCIE_PORT_SERVICE_AER))) diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 6d0ecdb2c5b16..7f9efea07c27f 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -2223,9 +2223,9 @@ int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev) int ret; /* Assignment exposes function/bus resets outside PCI sysfs. */ - if (pci_find_host_bridge(pdev->bus)->no_user_reset) { + if (pci_find_host_bridge(pdev->bus)->reset_sensitive) { dev_warn_ratelimited(dev, - "VFIO assignment refused: host bridge forbids userspace reset\n"); + "VFIO assignment refused: host bridge retains reset-sensitive DMA\n"); return -EOPNOTSUPP; } diff --git a/include/linux/pci.h b/include/linux/pci.h index 3f04012108ab7..de60fd2de3ef8 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -668,8 +668,10 @@ struct pci_host_bridge { system suspend is broken */ /* Disable sysfs device removal for this hierarchy; set before probing. */ unsigned int no_user_remove:1; - /* Disable sysfs function/bridge reset; set before probing. */ - unsigned int no_user_reset:1; + /* Retained DMA is sensitive to uncoordinated reset/recovery/assignment. + * Immutable after probing; driver-coordinated resets remain permitted. + */ + unsigned int reset_sensitive:1; /* Resource alignment requirements */ resource_size_t (*align_resource)(struct pci_dev *dev, From 55795adfc40bb20ce9dea4c588adaf234365bfca Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:39:31 -0700 Subject: [PATCH 0216/2680] docs: apple: clarify Neo guard and recovery contracts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe pre-child lifecycle admission, self-pinned Wi-Fi ownership, permanent PCI refusal errno and malformed-policy recovery. Treat teardown guards as accident protection: privileged raw resets, root-port unbind and ASPM changes remain outside the contract. Use desktop-session wording and explicit Hyprland test options, remove misleading country fallback wording and track the moved public header. Round-2 S6 and updated R1/R2 contracts. Documentation and MAINTAINERS only: non-Neo runtime behavior is unchanged. Round-2 S6. Non-Neo runtime behavior remains unchanged. [ryan: adapted to 7.2.10 — split by topic: the MAINTAINERS lines are carried by 900-config-tooling; documentation conflicts against the non-linear history of mt7932-neo.rst are resolved to this commit's text] --- Documentation/PCI/controller/apple-t8140.rst | 23 +++++++--- .../device_drivers/wifi/mt7932-neo.rst | 42 ++++++++++++------- drivers/net/wireless/mediatek/mt7932/mt7932.c | 5 ++- .../pci/controller/pcie-apple-piodma-diag.c | 2 +- 4 files changed, 49 insertions(+), 23 deletions(-) diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index 204685f47eb24..ec0d67dfc594c 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -19,15 +19,22 @@ default value is also visible on other Macs; the machine and DT checks decide whether it has any effect. The shared kernel can keep suspend, hibernation and kexec support enabled. -Immediately before the bootstrap first exposes its arena to hardware, a PM -notifier refuses sleep transitions -and its device prepare callback provides a second veto. This also covers +Before the host creates PCI children, a short admission critical section +acquires a kexec interlock and registers a PM notifier that refuses sleep +transitions. Its device prepare callback provides a second veto. This also covers ``/dev/snapshot`` files opened before the supplier probes: later image, restore and suspend ioctls enter device PM without repeating the prepare notifier. A kexec interlock refuses loading or executing a replacement kernel, including an already loaded crash kernel. Image unloading remains permitted. These interlocks apply only to the active J700 bootstrap; other machines do not -acquire them. Merely probing its supplier does not block transitions. Once the +acquire them. Merely probing its supplier does not block transitions. The device +prepare callback also checks this guarded pre-publication window. Admission +contention may defer only before PCI children exist; a later failure is a hard +error. Pre-child contention schedules a delayed attachment of this matched +host, so a long-running kexec loader does not require another driver to bind +before admission can retry. Only renewed contention queues another attempt; +successful admission and hardware publication end these retries. Shutdown +blocks late admission. A failure before memory publication releases the guards. Once the bootstrap retains memory, the protection persists even if enumeration or either radio's firmware loading fails. Thus the block applies on every default J700 boot which reaches retained radio admission, regardless of installed firmware. @@ -59,7 +66,7 @@ override the restriction; it is reapplied only at probe, not continuously. Do not enable those link power states while the Neo radios are admitted. The PIODMA arena remains allocated until external reset. Standard PCI sysfs -``remove`` writes for the root port and radio functions return ``-EBUSY`` +``remove`` writes for the root port and radio functions return ``-EOPNOTSUPP`` while the host has its retained supplier; the removal attributes remain present. The same hierarchy refuses userspace ``reset`` and ``reset_subordinate`` writes, and its host does not hand native PCIe or SHPC slot hot-plug control @@ -68,11 +75,15 @@ VFIO assignment is refused on this host because it exposes reset operations outside PCI sysfs. Automatic native AER/DPC port recovery is also withheld: its bus or link reset has no qualified retained-memory contract here. A hardware fault requires external reset rather than guessed recovery. -The host and activated Bluetooth modules are pinned until reset. Arbitrary +The host and activated Wi-Fi/Bluetooth modules are pinned until reset. Arbitrary platform-device removal, IOMMU teardown, forced module removal, memory reuse and arbitrary downstream devices remain unqualified. Do not remove the controller or its IOMMUs while this experiment is active. These lifecycle contracts require further work before production support. +They protect against accidental teardown and are not a security boundary. +Root can still reset through raw configuration writes, including secondary-bus +reset, FLR and Link Disable, change per-link ASPM controls or unbind the root +port's ``pcieport`` driver. Do not perform these operations on an admitted Neo. J700's ``wifi0`` alias and PCI endpoint node allow the public bootloader to pass the unit's own Wi-Fi address through ``local-mac-address``. The zero diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 8a5ceeb6bebf1..ad89444b13fe7 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -46,7 +46,8 @@ also registers its AES CMAC implementation when CMAC is configured as a module. The earlier tested ``CONFIG_CRYPTO_CMAC=y`` is not a Wi-Fi driver requirement. Use the ordinary cfg80211 regulatory database and applicable country policy. -The validated first-admission fallback is kernel country 00 with firmware XZ. +Kernel country 00 uses firmware XZ and its mandatory ``world-XZ.bin`` package. +An explicit country requires its own package; there is no world fallback. The host now disables link power states on the two radio endpoints before driver binding; no global PCIe ASPM performance policy is required. Load @@ -69,11 +70,10 @@ Install the target unit's Bluetooth inputs before opening the gate. Do not reprobe after a failed or uncertain Bluetooth admission. The tested Wi-Fi driver owns function 0 and Bluetooth owns function 1. -NetworkManager supplies the existing saved WPA2 profile. Use a unique profile -name or UUID and the current interface name; an older ``neo-wifi0`` profile -will not match an interface named ``wlan0``. KDE audio requires BlueZ, -PipeWire, its PulseAudio compatibility service, WirePlumber and the Bluetooth -audio plugins in the logged-in user's session. +Use a saved NetworkManager open or WPA2 profile matching the current interface. +Audio in the desktop session requires BlueZ, PipeWire, its PulseAudio +compatibility service, WirePlumber and Bluetooth audio plugins in the logged-in +user's session. Local firmware inputs --------------------- @@ -126,7 +126,7 @@ pattern, but an initramfs builder's glob handling may select only one match; inspect the resulting image rather than assuming every country is included. Load the radio drivers after these inputs are available on the real root. -There are three distinct Wi-Fi failure classes: +Wi-Fi reports these distinct failure classes: * An initialization probe failure can occur after DMA is published but before an interface is registered. Only ``initialization failed; DMA retired`` @@ -137,6 +137,11 @@ There are three distinct Wi-Fi failure classes: the requested policy file is absent before policy submission. Install that exact file and bring the interface up or retry a scan/connection. An identical ``iw reg set`` request alone does not trigger a retry in cfg80211. +* A malformed country package reports ``error=-22`` before submission and + is not retried on interface up/scan/connect. Replace the exact named package + with a valid one, then shut down and start again. An unchanged + ``iw reg set`` request is ignored by cfg80211. If ``recovery-required=1`` + is also present, use the external-reset recovery below instead. * ``recovery-required=1`` or ``RF_FAILED`` is a latched startup/calibration failure. Installing a file does not clear it in the bound epoch. A full external reset is required after a failed or uncertain admission. @@ -144,16 +149,15 @@ There are three distinct Wi-Fi failure classes: Repeatable physical network test -------------------------------- -The manual test is installed as a selftest data file rather than an automatic -test: activating a real saved network requires an explicit profile selection. -It selects the profile's requested band and checks an actual scan result, -WPA2 activation, DHCP route, gateway replies, DNS, verified HTTPS response, -KDE processes and fatal kernel/driver diagnostics. It never asks for or prints -the saved password. Run as root, with a fresh attempt name:: +The manual test activates a saved profile and checks scan, open or WPA2 +authentication, DHCP, gateway replies, DNS, HTTPS and kernel diagnostics. +For a desktop session using Hyprland, use ``--desktop none`` for this network test. +The helper restores the profile's original band even after a failure. It is +not run automatically. Run as root, with a fresh attempt name:: python3 tools/testing/selftests/drivers/net/mt7932_e2e.py \ --expected-release "$(uname -r)" --profile "YOUR SAVED PROFILE" \ - --connect --band a --attempt five-ghz + --connect --band a --attempt five-ghz --desktop none Use ``--band bg`` for 2.4 GHz. The JSON artifact path is printed on completion; each of three traffic samples includes the downloaded response's size and @@ -186,6 +190,16 @@ Qualification and remaining work native slot hot-plug is disabled for this host. Per-link sysfs ASPM/Clock PM writes can lift the power-state restrictions; they are reapplied only at probe. Do not enable those link power states while the radios are active. + These guards protect against accidental teardown, not privileged access. + Root can still reset through raw PCI configuration writes (SBR, FLR or Link + Disable), override link power states, or unbind the root port's ``pcieport`` + driver. None of these actions is supported while the retained host is active. +* The Wi-Fi module pins itself before publishing its first DMA ring and cannot + be unloaded normally while DMA is retained. A checked reset during probe + failure may release that pin; a failed reset retains it. Never unload or + reprobe Wi-Fi while Bluetooth is active. Shutdown quiesces host producers + without freeing retained memory; use a full shutdown and external reset + after a failed or uncertain admission. * Bluetooth PCI removal/quiescence is incomplete. Its software queue limit does not provide HCI backpressure; saturation can drop an accounted frame. Both require correction before production use. An activated Bluetooth module diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index d9289cb3001fa..fe61ceb0a5f1e 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -1371,7 +1371,8 @@ MODULE_FIRMWARE("mediatek/mt7932/wcal.bin"); MODULE_FIRMWARE("mediatek/mt7932/oca2.bin"); MODULE_FIRMWARE("mediatek/mt7932/config-original.bin"); MODULE_FIRMWARE("mediatek/mt7932/policy/world-XZ.bin"); -/* Country selection is runtime-dependent. Initramfs builders supporting - * firmware globs must retain every installed policy, including later hints. +/* Country selection is runtime-dependent; this pattern is packaging metadata. + * Globs do not guarantee complete inclusion by every initramfs builder. + * Explicitly package world-XZ and every intended current/later country policy. */ MODULE_FIRMWARE("mediatek/mt7932/policy/[A-Z][A-Z].bin"); diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index 9fbc04faa5ee8..62cd323a080e0 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -57,7 +57,7 @@ static bool enumerate = true; module_param(enumerate, bool, 0444); -MODULE_PARM_DESC(enumerate, "Boot-time gate for J700/T8140 radios only (read-only, default on)"); +MODULE_PARM_DESC(enumerate, "Boot-time radio gate (read-only, default on; no effect off J700/T8140)"); struct apple_piodma_request { u16 offset; From df570626f58ac1c734f2e68f0f0cede96df1a90e Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:41:25 -0700 Subject: [PATCH 0217/2680] PCI: withhold native error ownership on the retained Neo host Clear native_aer and native_dpc before creating Neo PCI children, keeping core error reporting consistent with withheld recovery services. Reject forced native AER ownership for this host in pcie_aer_is_native and log once when native/dpc-native recovery forcing is withheld. Round-2 R5. Bridge changes require the matched Neo supplier; core/service refusals require its reset_sensitive flag. Default-clear hosts keep their existing expression; non-Neo behavior remains unchanged. --- drivers/pci/controller/pcie-apple.c | 3 +++ drivers/pci/pcie/aer.c | 2 +- drivers/pci/pcie/portdrv.c | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 876eee0f6f93c..dac9545d54947 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2522,6 +2522,9 @@ static int apple_pcie_probe(struct platform_device *pdev) bridge->reset_sensitive = true; bridge->native_pcie_hotplug = false; bridge->native_shpc_hotplug = false; + bridge->native_aer = false; + bridge->native_dpc = false; + dev_info_once(dev, "Neo radio host: native AER/DPC recovery disabled for retained-DMA bootstrap\n"); } if (hw->root_bus_only) dev_info(dev, "root-port probe only; downstream config is blocked\n"); diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index fbb764ffb916d..99248c3c7fe31 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -243,7 +243,7 @@ int pcie_aer_is_native(struct pci_dev *dev) { struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); - if (!dev->aer_cap) + if (!dev->aer_cap || host->reset_sensitive) return 0; return pcie_ports_native || host->native_aer; diff --git a/drivers/pci/pcie/portdrv.c b/drivers/pci/pcie/portdrv.c index b0709e3a529f7..605a5b9348211 100644 --- a/drivers/pci/pcie/portdrv.c +++ b/drivers/pci/pcie/portdrv.c @@ -220,6 +220,9 @@ static int get_port_device_capability(struct pci_dev *dev) struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); int services = 0; + if (host->reset_sensitive && (pcie_ports_native || pcie_ports_dpc_native)) + dev_info_once(&dev->dev, "native PCIe error-recovery override withheld for reset-sensitive host\n"); + /* Retained hosts have no HP service to consume firmware-enabled events. */ if (host->no_user_remove) { pcie_capability_clear_word(dev, PCI_EXP_SLTCTL, From 0499ec4464da1fd0cc8184ffb08c518932b163ea Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:01:40 -0400 Subject: [PATCH 0218/2680] PCI: apple: gate chained IRQ register access during tunnel teardown The port handler reads its interrupt registers and dispatches child IRQs while ACIO may be quiescing the tunneled hierarchy and stopping APPCLK. Masking PORT_INTMSK does not wait for an already executing chained handler. Chained handlers do not set IRQD_IRQ_INPROGRESS and AIC lacks the active IRQ-state callback, so disable_irq() alone does not drain this path either. Serialize the chained handler with a separate raw spinlock and an explicit IRQ-disabled gate. Close the gate and mask the parent before removing PCI functions and stopping the port. Reopen it after DART restoration and bus rescan. Keep it closed on a failed restore. During final teardown also wait for an RCU grace period after removing the handler to drain callbacks that cached its data before the IRQ domain is destroyed. This hardens an MMIO race during teardown. Mounted SSD removal still needs follow-up: a dock unplug with mounted ext4/Btrfs volumes stopped logging during PCIe-C quiesce and reset the machine on the test build. --- drivers/pci/controller/pcie-apple.c | 51 +++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index dac9545d54947..8776a62fe419b 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -338,6 +339,7 @@ struct apple_pcie { struct apple_pcie_port { raw_spinlock_t lock; + raw_spinlock_t irq_lock; struct apple_pcie *pcie; struct device_node *np; void __iomem *base; @@ -355,6 +357,7 @@ struct apple_pcie_port { bool started; bool needs_stop; bool link_failed; + bool irq_disabled; }; static void rmw_set(u32 set, void __iomem *addr) @@ -635,6 +638,10 @@ static void apple_port_irq_handler(struct irq_desc *desc) chained_irq_enter(chip, desc); + raw_spin_lock(&port->irq_lock); + if (port->irq_disabled) + goto out; + stat = apple_pcie_port_readl(port, PORT_INTSTAT); /* Masked link events can belong to the resume poller. */ stat &= ~apple_pcie_port_readl(port, PORT_INTMSK); @@ -642,6 +649,8 @@ static void apple_port_irq_handler(struct irq_desc *desc) for_each_set_bit(i, &stat, 32) generic_handle_domain_irq(port->domain, i); +out: + raw_spin_unlock(&port->irq_lock); chained_irq_exit(chip, desc); } @@ -778,6 +787,37 @@ static void apple_pcie_port_unregister_irqs(struct apple_pcie_port *port) } } +/* + * A chained handler is not tracked by IRQD_IRQ_INPROGRESS, and AIC does not + * report IRQCHIP_STATE_ACTIVE. Gate and drain MMIO explicitly before masking + * the parent; disable_irq() alone cannot synchronize this chained handler. + */ +static void apple_pcie_port_disable_irq(struct apple_pcie_port *port) +{ + unsigned long flags; + + if (!port->irq || !port->domain || port->irq_disabled) + return; + + raw_spin_lock_irqsave(&port->irq_lock, flags); + port->irq_disabled = true; + raw_spin_unlock_irqrestore(&port->irq_lock, flags); + disable_irq(port->irq); +} + +static void apple_pcie_port_enable_irq(struct apple_pcie_port *port) +{ + unsigned long flags; + + if (!port->irq_disabled) + return; + + raw_spin_lock_irqsave(&port->irq_lock, flags); + port->irq_disabled = false; + raw_spin_unlock_irqrestore(&port->irq_lock, flags); + enable_irq(port->irq); +} + static u32 apple_pcie_rid2sid_write(struct apple_pcie_port *port, int idx, u32 val); @@ -1315,6 +1355,8 @@ static int apple_pcie_tunnel_stop(struct apple_pcie_port *port) static void apple_pcie_port_teardown(struct apple_pcie_port *port) { + apple_pcie_port_disable_irq(port); + if (port->base) apple_pcie_port_writel(port, ~0, PORT_INTMSK); @@ -1325,6 +1367,8 @@ static void apple_pcie_port_teardown(struct apple_pcie_port *port) if (port->irq) { irq_set_chained_handler_and_data(port->irq, NULL, NULL); + /* Drain callbacks that cached the old chained-handler data. */ + synchronize_rcu(); irq_dispose_mapping(port->irq); port->irq = 0; } @@ -1530,6 +1574,7 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, port->np = of_node_get(np); raw_spin_lock_init(&port->lock); + raw_spin_lock_init(&port->irq_lock); INIT_LIST_HEAD(&port->entry); snprintf(name, sizeof(name), "port%d", port->idx); @@ -2707,6 +2752,10 @@ int apple_pcie_tunnel_quiesce(struct device *dev) if (!pcie->bus_stopped) { struct pci_dev *pdev, *tmp; + /* Masking PORT_INTMSK alone does not drain an in-flight handler. */ + list_for_each_entry(port, &pcie->ports, entry) + apple_pcie_port_disable_irq(port); + pci_walk_bus(bridge->bus, pci_dev_set_disconnected, NULL); /* @@ -2791,6 +2840,8 @@ int apple_pcie_tunnel_restore(struct device *dev) pci_rescan_bus(bridge->bus); pci_walk_bus(bridge->bus, apple_pcie_tunnel_keep_d0, NULL); pcie->bus_stopped = false; + list_for_each_entry(port, &pcie->ports, entry) + apple_pcie_port_enable_irq(port); pci_unlock_rescan_remove(); dev_info(dev, "PCIe-C hierarchy restored after tunnel activation\n"); From f2cdfea389a646c71492f7c2ae41e613fd1e670b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 12 Oct 2026 19:11:08 -0400 Subject: [PATCH 0219/2680] PCI: apple: piodma-diag: decide suspend-to-idle admission in prepare Commit 2c815e33ece ("PCI: apple: preserve the retained bootstrap across suspend-to-idle sleep") admits suspend-to-idle once the bootstrap has completed, but its PM_SUSPEND_PREPARE notifier tests pm_suspend_no_platform(). enter_state() clears the suspend flags and runs these notifiers in suspend_prepare() before suspend_devices_and_enter() marks the suspend as suspend-to-idle, so the test is false for every suspend and the guard refuses s2idle too: apple-piodma-diag 390030000.piodma: sleep refused: Neo guard active during radio admission or retained DMA lifetime Leave the suspend decision to apple_piodma_pm_prepare(), which runs after the target state is set, makes the same ready/admitted check and still refuses anything but suspend-to-idle. As a supplier of the radio's PCI devices it is prepared before them. Hibernation and restore remain refused in the notifier. Co-Authored-By: Claude Opus 5.5 --- drivers/pci/controller/pcie-apple-piodma-diag.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index 7684f6aec5eb6..6f5d9bac0e64e 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -168,10 +168,14 @@ static int apple_piodma_pm_notify(struct notifier_block *nb, switch (action) { case PM_SUSPEND_PREPARE: - if (pm_suspend_no_platform() && READ_ONCE(diag->ready) && - smp_load_acquire(&diag->radio_admitted)) - return NOTIFY_DONE; - fallthrough; + /* + * The suspend core marks suspend-to-idle only after these + * notifiers have run, so pm_suspend_no_platform() is still + * false here for every suspend. apple_piodma_pm_prepare() + * makes the same check once the target state is known, and + * still vetoes everything but suspend-to-idle. + */ + return NOTIFY_DONE; case PM_HIBERNATION_PREPARE: case PM_RESTORE_PREPARE: dev_warn_ratelimited(diag->dev, From 0f7d473084e51f183c858245843fff0fc19e405f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 17:45:46 +1100 Subject: [PATCH 0220/2680] iommu: apple-dart: Program T8140 and T8142 streams individually Asahi's DART driver now programs only the first stream of a stream map and remaps the others onto it, invalidates only that first stream on T8110-family DARTs, and reports deferred flush support so the DMA layer uses a flush queue. None of that fits the T8140 and T8142 DARTs. Their locked streams keep the firmware's own roots, so every stream needs its own TTBR and TCR and its own TLB maintenance; their static remaps are described by apple,remap-streams and must not be overwritten by dynamic ones; and a deferred flush ends in a whole-stream invalidate, which can fault the firmware's live accesses through the same stream, as the boot scanout through the T8140 display DART has shown. Mark those DARTs per_stream and keep them on individual stream programming, per-stream invalidation and strict flushes. Every other DART keeps the remapping and the flush queue unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 3647d35667eaf..0d20e47ec241f 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -267,6 +267,7 @@ struct apple_dart { u32 four_level : 1; u32 locked : 1; u32 tunneled : 1; + u32 per_stream : 1; u32 power_retained : 1; struct apple_tunable *tunables; u16 version; @@ -434,11 +435,13 @@ static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom) * Asahi's stream remapping programs only the first stream of a map and * remaps the others onto it. A PCIe-C tunnel DART keeps programming and * invalidating every stream on its own, with the ordered accesses of - * apple_dart_writel(). + * apple_dart_writel(). So do the T8140 and T8142 DARTs (per_stream): their + * locked streams keep firmware-owned roots and filtering state of their own, + * and their static remaps come from apple,remap-streams. */ static bool apple_dart_remaps_streams(struct apple_dart *dart) { - return dart->hw->enable_remap && !dart->tunneled; + return dart->hw->enable_remap && !dart->tunneled && !dart->per_stream; } static void @@ -1704,8 +1707,22 @@ static struct iommu_group *apple_dart_device_group(struct device *dev) static bool apple_dart_capable(struct device *dev, enum iommu_cap cap) { + struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); + struct apple_dart_stream_map *stream_map; + int i; + switch (cap) { case IOMMU_CAP_DEFERRED_FLUSH: + /* + * A deferred flush invalidates whole streams. On a T8140 or + * T8142 DART the firmware may still be translating through + * the same stream, and a whole-stream invalidate can fault + * its live accesses, so keep those DARTs on strict flushes. + */ + if (cfg) + for_each_stream_map(i, cfg, stream_map) + if (stream_map->dart->per_stream) + return false; return true; default: return false; @@ -2068,6 +2085,8 @@ static int apple_dart_probe(struct platform_device *pdev) dart->dev = dev; dart->hw = of_device_get_match_data(dev); dart->tunneled = tunneled; + dart->per_stream = of_device_is_compatible(dev->of_node, "apple,t8140-dart") || + of_device_is_compatible(dev->of_node, "apple,t8142-dart"); if (tunneled) { pd_np = of_parse_phandle(dev->of_node, "power-domains", 0); if (pd_np) { From e0681c2adc86fe7b1ed32583b210a555ae82714c Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:21:53 -0400 Subject: [PATCH 0221/2680] iommu/apple-dart: defer locked root publication until attach The IOMMU core installs firmware-requested direct mappings before attaching the device. apple_dart_finalize_domain() already records its streams, but apple_dart_setup_translation_locked() has not mapped their locked roots or allocated their ownership shadows yet. The new publication check therefore returns -EIO from iotlb_sync_map(), rejecting these early mappings. Skip roots whose live table and ownership shadow are both absent. The attach path maps both and publishes the accumulated host entries before invalidating the stream TLB. Keep rejecting inconsistent mapped-root state and preserve the firmware ownership checks. This addresses the reported M1 Pro DART/DCP binding regression without reverting T8140 support or changing macsmc. Validation: reproduced pre-attach -EIO with the original production functions; the patched functions pass pre-attach, attach publication, firmware preservation, ownership conflict and inconsistent-state tests. The patched driver compiles against the prepared ARM64 kernel build. Hardware validation is pending. Fixes: 79379798f12f ("iommu/apple-dart: Support T8140 DARTs and firmware-owned stream roots") Signed-off-by: Dj <43033058+DjDeveloperr@users.noreply.github.com> --- drivers/iommu/apple-dart.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 0d20e47ec241f..f7794584e9176 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -678,6 +678,13 @@ apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg, u64 *owned = dart->locked_owned[sid][idx]; u64 *ours = cfg->apple_dart_cfg.ttbr[idx]; + /* + * Firmware mappings are created before attach maps the + * locked roots. Attach will publish these entries once + * both the live root and its ownership shadow exist. + */ + if (!live && !owned) + continue; if (!live || !owned || !ours) { ret = -EIO; goto out; From ae85e84d7fb2aa2c3755f4f891e7710b0d95f364 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 23:34:50 -0400 Subject: [PATCH 0222/2680] iommu/apple-dart: accept T8142 firmware slots that mirror Linux mappings On T8142 the display DARTs are locked and their roots already hold the running display coprocessor's own translations. The boot loader re-exports those translations as reserved IOVA regions, which the IOMMU core maps before attach, so Linux's tables end up with entries in root slots the firmware owns. Publishing only into vacant slots then fails the whole attach with -EBUSY. On locked apple,t8142-dart instances with three-level T8110-format roots: - Record the firmware's leaf tables when a locked root is mapped. - Accept a firmware-owned slot once, if its leaf table already translates every page Linux mapped there to the same physical address. The slot is never written; the hardware keeps walking the firmware's table. - Check later Linux mappings into a firmware-owned slot page by page and refuse any the firmware does not already provide. - Resolve IOVAs Linux does not map through those firmware tables, so the host can reach buffers the coprocessor shares later. Other DARTs and other SoCs keep the current behaviour. --- drivers/iommu/apple-dart.c | 272 +++++++++++++++++++++++++++++++++++-- 1 file changed, 261 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index f7794584e9176..c9ebcc5cfab01 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -231,6 +231,17 @@ struct apple_dart_hw { int ttbr_count; }; +/* + * Snapshot of a locked root's firmware-owned slots, taken in process context + * when the root is mapped: the root entry and a mapping of its leaf table. + */ +struct apple_dart_fw_root { + u64 *entry; + u64 **leaf; + /* Linux root entry already verified against each firmware slot. */ + u64 *accepted; +}; + /* * Private structure associated with each DART device. * @@ -269,6 +280,8 @@ struct apple_dart { u32 tunneled : 1; u32 per_stream : 1; u32 power_retained : 1; + /* Locked roots may hold firmware slots that mirror Linux mappings. */ + u32 fw_mirror : 1; struct apple_tunable *tunables; u16 version; @@ -288,6 +301,8 @@ struct apple_dart { u64 *locked_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR]; /* Values published by this driver into initially vacant root slots. */ u64 *locked_owned[DART_MAX_STREAMS][DART_MAX_TTBR]; + /* Leaf tables behind the root slots firmware held when we attached. */ + struct apple_dart_fw_root *locked_fw[DART_MAX_STREAMS][DART_MAX_TTBR]; }; /* @@ -547,12 +562,61 @@ apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map) apple_dart_hw_clear_ttbr(stream_map, i); } +/* APPLE_DART2 page-table entry layout, as in io-pgtable-dart. */ +#define APPLE_DART_PTE_VALID BIT_ULL(0) + +static phys_addr_t apple_dart_pte_to_paddr(const struct apple_dart *dart, + u64 pte) +{ + return (pte & GENMASK_ULL(37, 10)) << 4; +} + +static bool apple_dart_fw_slot_owned(const struct apple_dart_fw_root *fw, + const u64 *live, size_t i) +{ + return fw && fw->leaf[i] && READ_ONCE(live[i]) == fw->entry[i]; +} + +/* + * A root slot firmware already uses can still back Linux mappings when the + * firmware's leaf table translates every page Linux mapped there to the same + * physical address. That is the case for the boot loader's handoff, which + * re-exports live firmware translations as reserved IOVA regions; the + * hardware keeps walking the firmware table and Linux's copy stays private. + */ +static bool apple_dart_fw_slot_mirrors(const struct apple_dart *dart, + const struct apple_dart_fw_root *fw, + u64 next, size_t i, size_t entries) +{ + const u64 *ours_leaf, *fw_leaf; + size_t j; + + ours_leaf = phys_to_virt(apple_dart_pte_to_paddr(dart, next)); + fw_leaf = fw->leaf[i]; + for (j = 0; j < entries; j++) { + u64 ours = READ_ONCE(ours_leaf[j]); + u64 theirs = READ_ONCE(fw_leaf[j]); + + if (!(ours & APPLE_DART_PTE_VALID)) + continue; + if (!(theirs & APPLE_DART_PTE_VALID) || + apple_dart_pte_to_paddr(dart, ours) != + apple_dart_pte_to_paddr(dart, theirs)) + return false; + } + return true; +} + /* * A locked root belongs to firmware. Publish only into vacant slots and * remember exactly what we installed, so teardown cannot erase firmware - * mappings added after Linux attached. + * mappings added after Linux attached. A slot firmware already holds is + * never written; it is accepted once if it mirrors Linux's mappings, and + * apple_dart_check_fw_map() checks later mappings into it page by page. */ -static int apple_dart_publish_root(u64 *live, u64 *owned, const u64 *ours, +static int apple_dart_publish_root(const struct apple_dart *dart, + u64 *live, u64 *owned, const u64 *ours, + struct apple_dart_fw_root *fw, size_t entries) { size_t i; @@ -563,7 +627,12 @@ static int apple_dart_publish_root(u64 *live, u64 *owned, const u64 *ours, if (!next && !previous) continue; - if (READ_ONCE(live[i]) != previous) + if (READ_ONCE(live[i]) == previous) + continue; + if (previous || !apple_dart_fw_slot_owned(fw, live, i)) + return -EBUSY; + if (next != fw->accepted[i] && + !apple_dart_fw_slot_mirrors(dart, fw, next, i, entries)) return -EBUSY; } @@ -574,6 +643,10 @@ static int apple_dart_publish_root(u64 *live, u64 *owned, const u64 *ours, if (next == previous) continue; + if (!previous && apple_dart_fw_slot_owned(fw, live, i)) { + fw->accepted[i] = next; + continue; + } if (cmpxchg(&live[i], previous, next) != previous) return -EBUSY; WRITE_ONCE(owned[i], next); @@ -597,6 +670,67 @@ static void apple_dart_retire_root(u64 *live, u64 *owned, size_t entries) dma_wmb(); } +static void apple_dart_free_fw_root(struct apple_dart *dart, + struct apple_dart_fw_root *fw) +{ + size_t i; + + if (!fw) + return; + if (fw->leaf) + for (i = 0; i < dart->pgsize / sizeof(u64); i++) + if (fw->leaf[i]) + memunmap(fw->leaf[i]); + kfree(fw->accepted); + kfree(fw->leaf); + kfree(fw->entry); + kfree(fw); +} + +/* + * Record the leaf tables behind the slots firmware holds in a three-level + * T8110-format locked root, on DARTs whose loader re-exports firmware + * translations (fw_mirror). Other roots are not snapshotted; their firmware + * slots keep rejecting Linux mappings as before. + */ +static struct apple_dart_fw_root * +apple_dart_snapshot_fw_root(struct apple_dart *dart, int sid, const u64 *live) +{ + size_t i, entries = dart->pgsize / sizeof(u64); + struct apple_dart_fw_root *fw; + + /* Only the T8110-style table layout this walker decodes. */ + if (!dart->fw_mirror || dart->hw->fmt != APPLE_DART2) + return NULL; + if (dart->hw->tcr_4level && + (apple_dart_readl(dart, DART_TCR(dart, sid)) & dart->hw->tcr_4level)) + return NULL; + + fw = kzalloc(sizeof(*fw), GFP_KERNEL); + if (!fw) + return NULL; + fw->entry = kcalloc(entries, sizeof(*fw->entry), GFP_KERNEL); + fw->leaf = kcalloc(entries, sizeof(*fw->leaf), GFP_KERNEL); + fw->accepted = kcalloc(entries, sizeof(*fw->accepted), GFP_KERNEL); + if (!fw->entry || !fw->leaf || !fw->accepted) { + apple_dart_free_fw_root(dart, fw); + return NULL; + } + + for (i = 0; i < entries; i++) { + u64 pte = READ_ONCE(live[i]); + + if (!(pte & APPLE_DART_PTE_VALID)) + continue; + fw->leaf[i] = memremap(apple_dart_pte_to_paddr(dart, pte), + dart->pgsize, MEMREMAP_WB); + if (!fw->leaf[i]) + continue; + fw->entry[i] = pte; + } + return fw; +} + static int apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) { @@ -604,6 +738,8 @@ apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) int sid; for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + struct apple_dart_fw_root *fw; + unsigned long flags; u32 ttbr; phys_addr_t phys; u64 *l1_tbl, *owned; @@ -635,8 +771,12 @@ apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) devm_memunmap(dart->dev, l1_tbl); return -ENOMEM; } + fw = apple_dart_snapshot_fw_root(dart, sid, l1_tbl); + spin_lock_irqsave(&dart->lock, flags); dart->locked_owned[sid][idx] = owned; dart->locked_ttbr[sid][idx] = l1_tbl; + dart->locked_fw[sid][idx] = fw; + spin_unlock_irqrestore(&dart->lock, flags); } return 0; @@ -651,10 +791,17 @@ apple_dart_hw_unmap_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { u64 *live = dart->locked_ttbr[sid][idx]; u64 *owned = dart->locked_owned[sid][idx]; + struct apple_dart_fw_root *fw; + unsigned long flags; if (!live) continue; apple_dart_retire_root(live, owned, dart->pgsize / sizeof(*live)); + spin_lock_irqsave(&dart->lock, flags); + fw = dart->locked_fw[sid][idx]; + dart->locked_fw[sid][idx] = NULL; + spin_unlock_irqrestore(&dart->lock, flags); + apple_dart_free_fw_root(dart, fw); devm_memunmap(dart->dev, live); devm_kfree(dart->dev, owned); dart->locked_ttbr[sid][idx] = NULL; @@ -689,7 +836,8 @@ apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg, ret = -EIO; goto out; } - ret = apple_dart_publish_root(live, owned, ours, + ret = apple_dart_publish_root(dart, live, owned, ours, + dart->locked_fw[sid][idx], dart->pgsize / sizeof(*live)); if (ret) goto out; @@ -1035,18 +1183,111 @@ static int apple_dart_iotlb_sync_map(struct iommu_domain *domain, iova, iova + size - 1); } +/* + * Look up a domain address in the firmware-owned slots of its locked roots. + * Returns true if a firmware slot covers @dva; *phys is then the firmware's + * translation of that page, or 0 if the firmware leaves it unmapped. + */ +static bool apple_dart_fw_lookup(struct apple_dart_domain *domain, u64 dva, + phys_addr_t *phys) +{ + struct apple_dart_atomic_stream_map *map; + bool owned = false; + int i, sid; + + *phys = 0; + for_each_stream_map(i, domain, map) { + struct apple_dart *dart = map->dart; + u32 entries = dart->pgsize / sizeof(u64); + u32 pgshift = ilog2(dart->pgsize); + u32 bits = ilog2(entries); + u64 leaf = (dva >> pgshift) & (entries - 1); + u64 slot = (dva >> (pgshift + bits)) & (entries - 1); + u64 idx = dva >> (pgshift + 2 * bits); + unsigned long flags; + + if (!dart->fw_mirror || idx >= DART_MAX_TTBR) + continue; + + spin_lock_irqsave(&dart->lock, flags); + for (sid = 0; sid < dart->num_streams && !owned; sid++) { + const struct apple_dart_fw_root *fw; + u64 pte; + + if (!(atomic_long_read(&map->sidmap[BIT_WORD(sid)]) & + BIT_MASK(sid))) + continue; + fw = dart->locked_fw[sid][idx]; + if (!dart->locked_ttbr[sid][idx] || + !apple_dart_fw_slot_owned(fw, dart->locked_ttbr[sid][idx], + slot)) + continue; + owned = true; + pte = READ_ONCE(fw->leaf[slot][leaf]); + if (pte & APPLE_DART_PTE_VALID) + *phys = apple_dart_pte_to_paddr(dart, pte); + } + spin_unlock_irqrestore(&dart->lock, flags); + if (owned) + return true; + } + return false; +} + +/* + * Firmware buffers the coprocessor later shares with the host live in the + * firmware's own tables, not in Linux's; resolve them there. + */ +static phys_addr_t apple_dart_fw_iova_to_phys(struct apple_dart_domain *domain, + u64 dva) +{ + phys_addr_t phys; + u32 pgsize = domain->domain.pgsize_bitmap; + + if (!apple_dart_fw_lookup(domain, dva, &phys) || !phys) + return 0; + return phys | (dva & (pgsize - 1)); +} + +/* + * Linux mappings into a firmware-owned slot are never published; the + * hardware keeps walking the firmware's table. Only accept them where the + * firmware already provides the same translation. + */ +static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, + phys_addr_t paddr, size_t size) +{ + u32 pgsize = domain->domain.pgsize_bitmap; + size_t off; + + for (off = 0; off < size; off += pgsize) { + phys_addr_t phys; + + if (apple_dart_fw_lookup(domain, dva + off, &phys) && + phys != paddr + off) + return -EBUSY; + } + return 0; +} + static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { struct apple_dart_domain *dart_domain = to_dart_domain(domain); struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; + phys_addr_t phys; + u64 dva; + if (!ops) return 0; - return ops->iova_to_phys(ops, - (iova + dart_domain->dma_offset) & - dart_domain->mask); + dva = (iova + dart_domain->dma_offset) & dart_domain->mask; + phys = ops->iova_to_phys(ops, dva); + if (phys) + return phys; + + return apple_dart_fw_iova_to_phys(dart_domain, dva); } static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, @@ -1057,13 +1298,19 @@ static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, struct apple_dart_domain *dart_domain = to_dart_domain(domain); struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; + u64 dva; + int ret; + if (!ops) return -ENODEV; - return ops->map_pages(ops, - (iova + dart_domain->dma_offset) & - dart_domain->mask, paddr, pgsize, - pgcount, prot, gfp, mapped); + dva = (iova + dart_domain->dma_offset) & dart_domain->mask; + ret = apple_dart_check_fw_map(dart_domain, dva, paddr, pgsize * pgcount); + if (ret) + return ret; + + return ops->map_pages(ops, dva, paddr, pgsize, pgcount, prot, gfp, + mapped); } static size_t apple_dart_unmap_pages(struct iommu_domain *domain, @@ -2294,6 +2541,9 @@ static int apple_dart_probe(struct platform_device *pdev) dart->locked = false; else dart->locked = apple_dart_is_locked(dart); + /* T8142 loaders re-export the firmware's own display translations. */ + dart->fw_mirror = dart->locked && + of_device_is_compatible(pdev->dev.of_node, "apple,t8142-dart"); if (!dart->locked) { ret = apple_dart_hw_reset(dart); if (ret) From b51378e8fd55af85c2f37f56f3e5a2ae43fc4462 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:52:48 +1000 Subject: [PATCH 0223/2680] iommu/apple-dart: defer publication until locked roots are known The IOMMU core installs firmware-requested mappings before the locked stream is attached. Its root pages are not mapped at that point, so publishing host entries fails with -EIO. Defer that stream during early TLB sync; attach still requires every root and publishes the complete page table (DART-2). Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index c9ebcc5cfab01..0da7693d77528 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -812,7 +812,8 @@ apple_dart_hw_unmap_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx static int apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg, - struct apple_dart_stream_map *stream_map) + struct apple_dart_stream_map *stream_map, + bool defer_unknown) { struct apple_dart *dart = stream_map->dart; unsigned long flags; @@ -820,6 +821,20 @@ apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg, spin_lock_irqsave(&dart->lock, flags); for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + bool ready = true; + + if (defer_unknown) { + for (idx = 0; idx < cfg->apple_dart_cfg.n_ttbrs; idx++) { + if (!dart->locked_ttbr[sid][idx] || + !dart->locked_owned[sid][idx]) { + ready = false; + break; + } + } + if (!ready) + continue; + } + for (idx = 0; idx < cfg->apple_dart_cfg.n_ttbrs; idx++) { u64 *live = dart->locked_ttbr[sid][idx]; u64 *owned = dart->locked_owned[sid][idx]; @@ -1137,7 +1152,8 @@ static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, "failed to power DART for TLB synchronization: %d\n", ret); return ret; } - ret = stream.dart->locked ? apple_dart_hw_sync_locked(cfg, &stream) : 0; + ret = stream.dart->locked ? + apple_dart_hw_sync_locked(cfg, &stream, true) : 0; if (ret) dev_err_ratelimited(stream.dart->dev, "failed to publish host page-table entries: %d\n", ret); @@ -1365,7 +1381,7 @@ apple_dart_setup_translation_locked(struct apple_dart_domain *domain, if (ret) goto unmap; } - ret = apple_dart_hw_sync_locked(cfg, stream_map); + ret = apple_dart_hw_sync_locked(cfg, stream_map, false); if (!ret) ret = stream_map->dart->hw->invalidate_tlb(stream_map); if (!ret) From eb15e30313e825059b6f9f7b0f64e969c86d469a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:58:51 +1000 Subject: [PATCH 0224/2680] iommu/apple-dart: validate software mappings against inherited slots Firmware-owned root slots remain live after Linux attaches. Accept private host mappings only when every applicable stream agrees on the physical address, full-page coverage and read/write/cache attributes encoded by the public APPLE_DART2 format. Apply the same checks to attach-time leaf-table mirroring and to every later mapping, without replacing occupied roots. Keep translated reservations intact so the IOMMU core creates software translations with the correct physical/DVA identity and domain DMA offset, and preserves its translated-domain requirement. RTKit buffer queries use that host table rather than an unchecked first-stream firmware fallback. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 206 +++++++++++++++++++++++-------------- 1 file changed, 128 insertions(+), 78 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 0da7693d77528..e09eb45de8616 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -40,6 +40,10 @@ #include "dma-iommu.h" +#if IS_ENABLED(CONFIG_KUNIT) && !defined(MODULE) +#include +#endif + #define DART_MAX_STREAMS 256 #define DART_MAX_TTBR 4 #define MAX_DARTS_PER_DEVICE 8 @@ -577,6 +581,28 @@ static bool apple_dart_fw_slot_owned(const struct apple_dart_fw_root *fw, return fw && fw->leaf[i] && READ_ONCE(live[i]) == fw->entry[i]; } +static bool apple_dart_fw_leaf_decode(u64 pte, dma_addr_t iova, + phys_addr_t *phys, int *prot) +{ + if (!(pte & BIT_ULL(0)) || (pte & GENMASK_ULL(63, 52)) || + (pte & GENMASK_ULL(51, 40)) != GENMASK_ULL(51, 40)) + return false; + + *phys = ((pte & GENMASK_ULL(37, 10)) << 4) | (iova & (SZ_16K - 1)); + *prot = (!(pte & BIT_ULL(3)) ? IOMMU_READ : 0) | + (!(pte & BIT_ULL(2)) ? IOMMU_WRITE : 0) | + (!(pte & BIT_ULL(1)) ? IOMMU_CACHE : 0); + return true; +} + +static bool apple_dart_fw_mapping_matches(phys_addr_t phys, int prot, + phys_addr_t expected, int required) +{ + const int mask = IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE; + + return phys == expected && (prot & mask) == (required & mask); +} + /* * A root slot firmware already uses can still back Linux mappings when the * firmware's leaf table translates every page Linux mapped there to the same @@ -597,11 +623,16 @@ static bool apple_dart_fw_slot_mirrors(const struct apple_dart *dart, u64 ours = READ_ONCE(ours_leaf[j]); u64 theirs = READ_ONCE(fw_leaf[j]); + phys_addr_t ours_phys, fw_phys; + int ours_prot, fw_prot; + if (!(ours & APPLE_DART_PTE_VALID)) continue; - if (!(theirs & APPLE_DART_PTE_VALID) || - apple_dart_pte_to_paddr(dart, ours) != - apple_dart_pte_to_paddr(dart, theirs)) + + if (!apple_dart_fw_leaf_decode(ours, 0, &ours_phys, &ours_prot) || + !apple_dart_fw_leaf_decode(theirs, 0, &fw_phys, &fw_prot) || + !apple_dart_fw_mapping_matches(fw_phys, fw_prot, + ours_phys, ours_prot)) return false; } return true; @@ -1200,88 +1231,64 @@ static int apple_dart_iotlb_sync_map(struct iommu_domain *domain, } /* - * Look up a domain address in the firmware-owned slots of its locked roots. - * Returns true if a firmware slot covers @dva; *phys is then the firmware's - * translation of that page, or 0 if the firmware leaves it unmapped. + * Hardware continues to use every firmware-owned slot. Accept a private + * software mapping only when every stream using such a slot agrees on the + * full-page translation and its protection/cache attributes. */ -static bool apple_dart_fw_lookup(struct apple_dart_domain *domain, u64 dva, - phys_addr_t *phys) +static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, + phys_addr_t paddr, size_t size, int prot) { struct apple_dart_atomic_stream_map *map; - bool owned = false; - int i, sid; + int i, sid, ret = 0; - *phys = 0; for_each_stream_map(i, domain, map) { struct apple_dart *dart = map->dart; u32 entries = dart->pgsize / sizeof(u64); u32 pgshift = ilog2(dart->pgsize); u32 bits = ilog2(entries); - u64 leaf = (dva >> pgshift) & (entries - 1); - u64 slot = (dva >> (pgshift + bits)) & (entries - 1); - u64 idx = dva >> (pgshift + 2 * bits); unsigned long flags; + size_t off; - if (!dart->fw_mirror || idx >= DART_MAX_TTBR) + if (!dart->fw_mirror) continue; spin_lock_irqsave(&dart->lock, flags); - for (sid = 0; sid < dart->num_streams && !owned; sid++) { - const struct apple_dart_fw_root *fw; - u64 pte; + for (off = 0; off < size; off += dart->pgsize) { + u64 addr = dva + off; + u64 leaf = (addr >> pgshift) & (entries - 1); + u64 slot = (addr >> (pgshift + bits)) & (entries - 1); + u64 idx = addr >> (pgshift + 2 * bits); - if (!(atomic_long_read(&map->sidmap[BIT_WORD(sid)]) & - BIT_MASK(sid))) + if (idx >= DART_MAX_TTBR) continue; - fw = dart->locked_fw[sid][idx]; - if (!dart->locked_ttbr[sid][idx] || - !apple_dart_fw_slot_owned(fw, dart->locked_ttbr[sid][idx], - slot)) - continue; - owned = true; - pte = READ_ONCE(fw->leaf[slot][leaf]); - if (pte & APPLE_DART_PTE_VALID) - *phys = apple_dart_pte_to_paddr(dart, pte); + for (sid = 0; sid < dart->num_streams; sid++) { + const struct apple_dart_fw_root *fw; + phys_addr_t phys; + u64 pte; + int fw_prot; + + if (!(atomic_long_read(&map->sidmap[BIT_WORD(sid)]) & + BIT_MASK(sid))) + continue; + fw = dart->locked_fw[sid][idx]; + if (!dart->locked_ttbr[sid][idx] || + !apple_dart_fw_slot_owned(fw, + dart->locked_ttbr[sid][idx], slot)) + continue; + pte = READ_ONCE(fw->leaf[slot][leaf]); + if (!apple_dart_fw_leaf_decode(pte, addr, + &phys, &fw_prot) || + !apple_dart_fw_mapping_matches(phys, fw_prot, + paddr + off, prot)) { + ret = -EBUSY; + goto unlock; + } + } } + unlock: spin_unlock_irqrestore(&dart->lock, flags); - if (owned) - return true; - } - return false; -} - -/* - * Firmware buffers the coprocessor later shares with the host live in the - * firmware's own tables, not in Linux's; resolve them there. - */ -static phys_addr_t apple_dart_fw_iova_to_phys(struct apple_dart_domain *domain, - u64 dva) -{ - phys_addr_t phys; - u32 pgsize = domain->domain.pgsize_bitmap; - - if (!apple_dart_fw_lookup(domain, dva, &phys) || !phys) - return 0; - return phys | (dva & (pgsize - 1)); -} - -/* - * Linux mappings into a firmware-owned slot are never published; the - * hardware keeps walking the firmware's table. Only accept them where the - * firmware already provides the same translation. - */ -static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, - phys_addr_t paddr, size_t size) -{ - u32 pgsize = domain->domain.pgsize_bitmap; - size_t off; - - for (off = 0; off < size; off += pgsize) { - phys_addr_t phys; - - if (apple_dart_fw_lookup(domain, dva + off, &phys) && - phys != paddr + off) - return -EBUSY; + if (ret) + return ret; } return 0; } @@ -1292,18 +1299,13 @@ static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, struct apple_dart_domain *dart_domain = to_dart_domain(domain); struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; - phys_addr_t phys; - u64 dva; - if (!ops) return 0; - dva = (iova + dart_domain->dma_offset) & dart_domain->mask; - phys = ops->iova_to_phys(ops, dva); - if (phys) - return phys; - - return apple_dart_fw_iova_to_phys(dart_domain, dva); + /* Reserved firmware mappings are also represented in this software table. */ + return ops->iova_to_phys(ops, + (iova + dart_domain->dma_offset) & + dart_domain->mask); } static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, @@ -1321,7 +1323,8 @@ static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, return -ENODEV; dva = (iova + dart_domain->dma_offset) & dart_domain->mask; - ret = apple_dart_check_fw_map(dart_domain, dva, paddr, pgsize * pgcount); + ret = apple_dart_check_fw_map(dart_domain, dva, paddr, pgsize * pgcount, + prot); if (ret) return ret; @@ -2022,8 +2025,9 @@ static int apple_dart_def_domain_type(struct device *dev) static void apple_dart_get_resv_regions(struct device *dev, struct list_head *head) { + struct iommu_resv_region *region; + if (IS_ENABLED(CONFIG_PCIE_APPLE) && dev_is_pci(dev)) { - struct iommu_resv_region *region; int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; region = iommu_alloc_resv_region(DOORBELL_ADDR, @@ -2035,6 +2039,10 @@ static void apple_dart_get_resv_regions(struct device *dev, list_add_tail(®ion->list, head); } + /* Keep translated regions so the core retains their software mappings + * and the device's translated-domain requirement. Locked-root publication + * validates them against every live firmware stream without replacing it. + */ iommu_dma_get_resv_regions(dev, head); } @@ -2810,6 +2818,48 @@ static struct platform_driver apple_dart_driver = { module_platform_driver(apple_dart_driver); +#if IS_ENABLED(CONFIG_KUNIT) && !defined(MODULE) +static void apple_dart_fw_leaf_test(struct kunit *test) +{ + const u64 leaf = GENMASK_ULL(51, 40) | + ((u64)(0x280000 >> 4) & GENMASK_ULL(37, 10)) | + BIT_ULL(0); + const int rwc = IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE; + phys_addr_t phys; + int prot; + bool valid, matched; + + valid = apple_dart_fw_leaf_decode(0, 0x4000, &phys, &prot); + KUNIT_EXPECT_FALSE(test, valid); + valid = apple_dart_fw_leaf_decode(BIT_ULL(0), 0x4000, + &phys, &prot); + KUNIT_EXPECT_FALSE(test, valid); + valid = apple_dart_fw_leaf_decode(leaf, 0x4000, &phys, &prot); + KUNIT_ASSERT_TRUE(test, valid); + matched = apple_dart_fw_mapping_matches(phys, prot, 0x280000, rwc); + KUNIT_EXPECT_TRUE(test, matched); + matched = apple_dart_fw_mapping_matches(phys, prot, 0x284000, rwc); + KUNIT_EXPECT_FALSE(test, matched); + valid = apple_dart_fw_leaf_decode(leaf | BIT_ULL(2), 0x4000, + &phys, &prot); + KUNIT_ASSERT_TRUE(test, valid); + matched = apple_dart_fw_mapping_matches(phys, prot, 0x280000, rwc); + KUNIT_EXPECT_FALSE(test, matched); +} + +static struct kunit_case apple_dart_fw_cases[] = { + KUNIT_CASE(apple_dart_fw_leaf_test), + {} +}; + +static struct kunit_suite apple_dart_fw_suite = { + .name = "apple-dart-firmware-leaf", + .test_cases = apple_dart_fw_cases, +}; + +kunit_test_suite(apple_dart_fw_suite); +#endif + MODULE_DESCRIPTION("IOMMU API for Apple's DART"); MODULE_AUTHOR("Sven Peter "); MODULE_LICENSE("GPL v2"); From dc6bc6f4a9596686eac41ba30e2d72c1997f190d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 7 Oct 2026 08:10:21 +1100 Subject: [PATCH 0225/2680] iommu/apple-dart: import firmware translations on T8140 Select the existing firmware-root import path for locked T8140 DARTs. The loader leaves occupied firmware translations in place; attach-time mirroring and later host maps validate full-page addresses and protection attributes across every applicable stream. Vacant slots remain available for Linux mappings without overwriting inherited roots. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index e09eb45de8616..7633e0620c61b 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -2565,9 +2565,10 @@ static int apple_dart_probe(struct platform_device *pdev) dart->locked = false; else dart->locked = apple_dart_is_locked(dart); - /* T8142 loaders re-export the firmware's own display translations. */ + /* These loaders re-export the firmware's own display translations. */ dart->fw_mirror = dart->locked && - of_device_is_compatible(pdev->dev.of_node, "apple,t8142-dart"); + (of_device_is_compatible(pdev->dev.of_node, "apple,t8140-dart") || + of_device_is_compatible(pdev->dev.of_node, "apple,t8142-dart")); if (!dart->locked) { ret = apple_dart_hw_reset(dart); if (ret) From ae638a672934721aab42143784275b4f9025a4e3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 09:40:34 +1100 Subject: [PATCH 0226/2680] iommu: apple-dart: retain inherited translations before reservation setup A locked display DART already translates the boot firmware regions. When the core builds temporary read-write mappings for those reservations, their cache attributes and the firmware's read-only code region differ from the inherited leaves. Exact mirror validation then rejects the first default-domain attach with EBUSY. Snapshot the supported three-level firmware roots during domain setup, before the core checks translated reservations. Resolve existing firmware translations when the software table has no entry, requiring unchanged root ownership and consistent physical addresses across the domain's streams. The core can then keep the existing mappings without creating replacement leaves or widening firmware permissions. Continue to validate host mappings and publish only into vacant root slots. The lookup borrows mapped firmware tables; it does not install those tables in the software allocator or free them with its page tables. Unqualified layouts and conflicting stream translations remain refused. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 74 +++++++++++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 7633e0620c61b..5359fe7be67df 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -1293,19 +1293,66 @@ static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, return 0; } +static phys_addr_t +apple_dart_inherited_phys(struct apple_dart_domain *domain, dma_addr_t address) +{ + struct apple_dart_atomic_stream_map *map; + phys_addr_t result = 0; + int i, sid; + + for_each_stream_map(i, domain, map) { + struct apple_dart *dart = map->dart; + unsigned long flags; + size_t entries = dart->pgsize / sizeof(u64); + size_t page = (address / dart->pgsize) % entries; + size_t slot = address / dart->pgsize / entries; + + if (!dart->locked || !dart->fw_mirror || slot >= entries) + return 0; + spin_lock_irqsave(&dart->lock, flags); + for (sid = 0; sid < dart->num_streams; sid++) { + struct apple_dart_fw_root *fw; + phys_addr_t phys; + int prot; + + if (!(atomic_long_read(&map->sidmap[BIT_WORD(sid)]) & BIT_MASK(sid))) + continue; + fw = dart->locked_fw[sid][0]; + if (!apple_dart_fw_slot_owned(fw, dart->locked_ttbr[sid][0], slot) || + !apple_dart_fw_leaf_decode(READ_ONCE(fw->leaf[slot][page]), + address, &phys, &prot) || + (result && result != phys)) { + spin_unlock_irqrestore(&dart->lock, flags); + return 0; + } + result = phys; + } + spin_unlock_irqrestore(&dart->lock, flags); + } + return result; +} + static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { struct apple_dart_domain *dart_domain = to_dart_domain(domain); struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; + struct io_pgtable_cfg *cfg; + phys_addr_t phys; + dma_addr_t address; if (!ops) return 0; - /* Reserved firmware mappings are also represented in this software table. */ - return ops->iova_to_phys(ops, - (iova + dart_domain->dma_offset) & - dart_domain->mask); + address = (iova + dart_domain->dma_offset) & dart_domain->mask; + phys = ops->iova_to_phys(ops, address); + if (phys) + return phys; + cfg = &io_pgtable_ops_to_pgtable(ops)->cfg; + if (cfg->apple_dart_cfg.n_levels != 3) + return 0; + /* Existing firmware translations need no replacement software mapping. */ + return apple_dart_inherited_phys(dart_domain, address); } static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, @@ -1505,6 +1552,25 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, dart_domain->domain.geometry.force_aperture = true; dart_domain->dma_offset = dart->dma_offset; + if (pgtbl_cfg.apple_dart_cfg.n_levels == 3) { + struct apple_dart_stream_map *stream; + int index; + + /* Make inherited translations visible before core reservation setup. */ + for_each_stream_map(i, cfg, stream) { + if (!stream->dart->locked || !stream->dart->fw_mirror) + continue; + for (index = 0; index < pgtbl_cfg.apple_dart_cfg.n_ttbrs; index++) { + ret = apple_dart_hw_map_locked_ttbr(stream, index); + if (ret) { + free_io_pgtable_ops(dart_domain->pgtbl_ops); + dart_domain->pgtbl_ops = NULL; + goto done; + } + } + } + } + dart_domain->finalized = true; done: From 05683927aab10e4a3ccbe87a7cc6e59033d08046 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 23:27:26 -0700 Subject: [PATCH 0227/2680] iommu/apple-dart: restore legacy locked display domain handoff Booting the T8140 DART changes on an M1 j293 fails the display DART reservations with -EIO before attach_dev(). Domain finalization copies prospective stream IDs into the active bitmap, so the new strict root synchronization runs before the locked roots have been mapped. Leave the active stream bitmap empty until attachment, and skip TLB synchronization and teardown for domains with no attached streams. The legacy M1/M2 display handoff also requires replacing the firmware root with Linux's page tables after the IOMMU core reconstructs reserved mappings. Requiring vacant root slots instead rejects this handoff with -EBUSY. Restore replacement for legacy DARTs, tracking the installed entries for teardown, while retaining firmware-owned slot protection for T8110 version 2.2 and newer. Add KUnit coverage for pre-attachment reserved mapping, legacy root handoff, and preservation/conflict handling on newer firmware roots. The pre-attachment and handoff tests fail on the unmodified driver and pass with this fix; all 42 Thunderbolt tests also pass. Fixes: 79379798f12f ("iommu/apple-dart: Support T8140 DARTs and firmware-owned stream roots") Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- MAINTAINERS | 1 + drivers/iommu/Kconfig | 10 +++ drivers/iommu/apple-dart-test.c | 146 ++++++++++++++++++++++++++++++++ drivers/iommu/apple-dart.c | 33 +++++++- 4 files changed, 187 insertions(+), 3 deletions(-) create mode 100644 drivers/iommu/apple-dart-test.c diff --git a/MAINTAINERS b/MAINTAINERS index fa4e0b27a6854..da78ae847fc0d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2635,6 +2635,7 @@ F: drivers/i2c/busses/i2c-pasemi-core.c F: drivers/i2c/busses/i2c-pasemi-platform.c F: drivers/input/misc/macsmc-input.c F: drivers/input/touchscreen/apple_z2.c +F: drivers/iommu/apple-dart-test.c F: drivers/iommu/apple-dart.c F: drivers/iommu/io-pgtable-dart.c F: drivers/irqchip/irq-apple-aic.c diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index f334a6278702c..a23c2a126c7ac 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -316,6 +316,16 @@ config APPLE_DART Say Y here if you are using an Apple SoC. +config APPLE_DART_KUNIT_TEST + bool "KUnit tests for Apple DART locked domains" if !KUNIT_ALL_TESTS + depends on APPLE_DART=y && KUNIT=y + default KUNIT_ALL_TESTS + help + Test firmware mapping before attachment and locked page-table + handoff without accessing physical DART hardware. Verify that + legacy display roots can be replaced while newer firmware-owned + roots retain their mappings and reject conflicting host entries. + config S390_IOMMU def_bool y if S390 && PCI depends on S390 && PCI diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c new file mode 100644 index 0000000000000..7f8888bf74115 --- /dev/null +++ b/drivers/iommu/apple-dart-test.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included by apple-dart.c to exercise the locked-domain lifecycle. */ +#include +#include + +static void apple_dart_test_before_attach(struct kunit *test) +{ + struct apple_dart_master_cfg master = {}; + struct apple_dart_domain *domain; + struct apple_dart *dart; + struct io_pgtable_ops *ops; + size_t mapped = 0; + int ret; + + dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart); + domain = kunit_kzalloc(test, sizeof(*domain), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, domain); + dart->dev = kunit_device_register(test, "apple-dart-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dart->dev); + dart->hw = &apple_dart_hw_t8103; + dart->pgsize = PAGE_SIZE; + dart->ias = 32; + dart->oas = 36; + dart->dma_max = DMA_BIT_MASK(32); + dart->num_streams = 16; + dart->locked = true; + dart->regs = (__force void __iomem *)kunit_kzalloc(test, SZ_16K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart->regs); + spin_lock_init(&dart->lock); + mutex_init(&domain->init_lock); + master.stream_maps[0].dart = dart; + __set_bit(0, master.stream_maps[0].sidmap); + writel(dart->hw->ttbr_valid, dart->regs + DART_TTBR(dart, 0, 0)); + KUNIT_ASSERT_EQ(test, apple_dart_finalize_domain(domain, &master), 0); + ops = domain->pgtbl_ops; + pm_runtime_set_active(dart->dev); + pm_runtime_enable(dart->dev); + + /* The IOMMU core installs firmware reservations before attach_dev(). */ + ret = apple_dart_map_pages(&domain->domain, 0, SZ_1M, PAGE_SIZE, 1, + IOMMU_READ | IOMMU_WRITE, GFP_KERNEL, &mapped); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mapped, (size_t)PAGE_SIZE); + KUNIT_EXPECT_EQ(test, apple_dart_iotlb_sync_map(&domain->domain, 0, PAGE_SIZE), 0); + KUNIT_EXPECT_EQ(test, ops->iova_to_phys(ops, 0), (phys_addr_t)SZ_1M); + KUNIT_EXPECT_EQ(test, atomic_long_read(&domain->stream_maps[0].sidmap[0]), 0L); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8020_STREAM_COMMAND), 0U); + + /* Once attached, missing root mappings must still report an error. */ + KUNIT_EXPECT_EQ(test, apple_dart_domain_add_streams(domain, &master), 0); + KUNIT_EXPECT_EQ(test, apple_dart_iotlb_sync_map(&domain->domain, 0, PAGE_SIZE), -EIO); + + pm_runtime_disable(dart->dev); + free_io_pgtable_ops(ops); +} + +static void apple_dart_test_locked_handoff(struct kunit *test) +{ + struct apple_dart_stream_map stream = {}; + struct io_pgtable_cfg cfg = {}; + struct apple_dart *dart; + u64 live[] = { 0x1001, 0, 0x3001, 0 }; + u64 owned[ARRAY_SIZE(live)] = {}; + u64 ours[] = { 0x4001, 0x5001, 0, 0 }; + + dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart); + dart->hw = &apple_dart_hw_t8103; + dart->num_streams = 1; + dart->pgsize = sizeof(live); + spin_lock_init(&dart->lock); + dart->locked_ttbr[0][0] = live; + dart->locked_owned[0][0] = owned; + stream.dart = dart; + __set_bit(0, stream.sidmap); + cfg.apple_dart_cfg.n_ttbrs = 1; + cfg.apple_dart_cfg.ttbr[0] = ours; + + /* Legacy firmware mappings are rebuilt by the IOMMU core at handoff. */ + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream), 0); + KUNIT_EXPECT_MEMEQ(test, live, ours, sizeof(live)); + KUNIT_EXPECT_MEMEQ(test, owned, ours, sizeof(owned)); + apple_dart_retire_root(live, owned, ARRAY_SIZE(live)); + KUNIT_EXPECT_EQ(test, live[0], 0ULL); + KUNIT_EXPECT_EQ(test, live[1], 0ULL); + + /* Older T8110 instances also use the legacy display handoff. */ + dart->hw = &apple_dart_hw_t8110; + dart->version = 0x0200; + live[0] = 0x1001; + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream), 0); + KUNIT_EXPECT_MEMEQ(test, live, ours, sizeof(live)); +} + +static void apple_dart_test_firmware_roots(struct kunit *test) +{ + struct apple_dart_stream_map stream = {}; + struct io_pgtable_cfg cfg = {}; + struct apple_dart *dart; + u64 live[] = { 0x1001, 0, 0, 0 }; + u64 owned[ARRAY_SIZE(live)] = {}; + u64 ours[] = { 0, 0x2001, 0, 0 }; + + dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart); + dart->hw = &apple_dart_hw_t8110; + dart->version = 0x0202; + dart->num_streams = 1; + dart->pgsize = sizeof(live); + spin_lock_init(&dart->lock); + dart->locked_ttbr[0][0] = live; + dart->locked_owned[0][0] = owned; + stream.dart = dart; + __set_bit(0, stream.sidmap); + cfg.apple_dart_cfg.n_ttbrs = 1; + cfg.apple_dart_cfg.ttbr[0] = ours; + + KUNIT_ASSERT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream), 0); + KUNIT_EXPECT_EQ(test, live[0], 0x1001ULL); + KUNIT_EXPECT_EQ(test, live[1], 0x2001ULL); + ours[0] = 0x3001; + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream), -EBUSY); + dart->version = 0x0203; + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream), -EBUSY); + KUNIT_EXPECT_EQ(test, live[0], 0x1001ULL); + /* Teardown must preserve a root which firmware has since replaced. */ + live[1] = 0x4001; + apple_dart_retire_root(live, owned, ARRAY_SIZE(live)); + KUNIT_EXPECT_EQ(test, live[0], 0x1001ULL); + KUNIT_EXPECT_EQ(test, live[1], 0x4001ULL); +} + +static struct kunit_case apple_dart_test_cases[] = { + KUNIT_CASE(apple_dart_test_before_attach), + KUNIT_CASE(apple_dart_test_locked_handoff), + KUNIT_CASE(apple_dart_test_firmware_roots), + {} +}; + +static struct kunit_suite apple_dart_test_suite = { + .name = "apple-dart", + .test_cases = apple_dart_test_cases, +}; + +kunit_test_suite(apple_dart_test_suite); diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 5359fe7be67df..a68701ed359ab 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -882,6 +882,25 @@ apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg, ret = -EIO; goto out; } + /* + * Legacy display DARTs hand their locked root to Linux. + * The IOMMU core has rebuilt the firmware reservations + * before attachment, so replace the old root entries. + * T8110 v2.2+ instead retains firmware-owned entries. + */ + if (dart->hw->type != DART_T8110 || dart->version < 0x0202) { + size_t entry; + + dma_wmb(); + for (entry = 0; entry < dart->pgsize / sizeof(*live); entry++) { + u64 next = READ_ONCE(ours[entry]); + + WRITE_ONCE(live[entry], next); + WRITE_ONCE(owned[entry], next); + } + dma_wmb(); + continue; + } ret = apple_dart_publish_root(dart, live, owned, ours, dart->locked_fw[sid][idx], dart->pgsize / sizeof(*live)); @@ -1177,6 +1196,8 @@ static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, stream.dart = map->dart; for (j = 0; j < BITS_TO_LONGS(stream.dart->num_streams); j++) stream.sidmap[j] = atomic_long_read(&map->sidmap[j]); + if (bitmap_empty(stream.sidmap, stream.dart->num_streams)) + continue; ret = pm_runtime_resume_and_get(stream.dart->dev); if (ret < 0) { dev_err_ratelimited(stream.dart->dev, @@ -1483,11 +1504,11 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, if (dart_domain->finalized) goto done; + /* Reserved mappings may be installed before any device is attached. */ for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { dart_domain->stream_maps[i].dart = cfg->stream_maps[i].dart; - for (j = 0; j < BITS_TO_LONGS(dart->num_streams); j++) - atomic_long_set(&dart_domain->stream_maps[i].sidmap[j], - cfg->stream_maps[i].sidmap[j]); + for (j = 0; j < BITS_TO_LONGS(DART_MAX_STREAMS); j++) + atomic_long_set(&dart_domain->stream_maps[i].sidmap[j], 0); } pgtbl_cfg = (struct io_pgtable_cfg){ @@ -1805,6 +1826,8 @@ static void apple_dart_domain_free(struct iommu_domain *domain) stream.dart = map->dart; for (j = 0; j < BITS_TO_LONGS(stream.dart->num_streams); j++) stream.sidmap[j] = atomic_long_read(&map->sidmap[j]); + if (bitmap_empty(stream.sidmap, stream.dart->num_streams)) + continue; j = pm_runtime_resume_and_get(stream.dart->dev); if (j < 0) { dev_err_ratelimited(stream.dart->dev, @@ -2927,6 +2950,10 @@ static struct kunit_suite apple_dart_fw_suite = { kunit_test_suite(apple_dart_fw_suite); #endif +#if IS_ENABLED(CONFIG_APPLE_DART_KUNIT_TEST) +#include "apple-dart-test.c" +#endif + MODULE_DESCRIPTION("IOMMU API for Apple's DART"); MODULE_AUTHOR("Sven Peter "); MODULE_LICENSE("GPL v2"); From d89120ef990b2bf074835167fa7043c89711f034 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 23 Sep 2026 10:56:26 -0700 Subject: [PATCH 0228/2680] PCI: apple: enumerate Thunderbolt PCIe on t8103 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit t8103 boots with the Thunderbolt PCIe port held in reset. Bring the port up when the tunnel activates, and do it before the port's IOMMU probes: that IOMMU's invalidate command does not finish while the port clock is still gated. Program the port tunables and the reset table, release PERST, enable the port clock, then program the root-port configuration tunables. Link training stays in the host driver. If the port is already running, the host does not replay the reset table. Endpoint memory on t8103 is left posted. The PCIe-C IOMMU uses the 64-stream USB4 register block: 16 KiB pages, a 32-bit IOVA and a 36-bit physical address. The 42-bit four-level geometry remains on T8110, which is the hardware that walks it. An xHCI early handoff no longer faults when the capability length read from a BAR is shorter than 16 bytes or not dword-aligned. Tested on a MacBook Pro 13" M1 (j293) with a CalDigit TS3 Plus on USB-1. The tunnel enumerated the dock bridges, four USB host controllers and an I210. The USB devices and a gigabit link came up, and the display on the same port still comes up. [ryan: adapted to 7.2.10 — split by topic: only the DART hunks (drivers/iommu/apple-dart*.c, include/linux/soc/apple/dart.h) are carried in 020-dart, where the tunnel DART series lives; the remaining hunks stay with 080-radio] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/iommu/apple-dart.c | 48 ++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index a68701ed359ab..f24cb37782d39 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -2490,9 +2490,22 @@ static int apple_dart_probe(struct platform_device *pdev) if (tunneled) { dart->tunables = devm_apple_tunable_parse(dev, dev->of_node, "apple,tunable", res); - if (IS_ERR(dart->tunables)) - return dev_err_probe(dev, PTR_ERR(dart->tunables), - "failed to parse PCIe-C DART tunables\n"); + if (IS_ERR(dart->tunables)) { + /* + * T8110 publishes tunables and the driver requires them. + * A t8103 PCIe-C DART may omit them. Its values include + * the configuration lock, which has to be written after + * the translation context is programmed, so leave them + * off until that ordering exists. + */ + if (PTR_ERR(dart->tunables) == -ENOENT && + dart->hw->type != DART_T8110) { + dart->tunables = NULL; + } else { + return dev_err_probe(dev, PTR_ERR(dart->tunables), + "failed to parse PCIe-C DART tunables\n"); + } + } } dart->irq = platform_get_irq(pdev, 0); @@ -2515,7 +2528,7 @@ static int apple_dart_probe(struct platform_device *pdev) ret = devm_pm_runtime_enable(dev); if (ret) goto err_clk_disable; - if (tunneled) { + if (tunneled && dart->tunables) { dev_info(dev, "applying %zu firmware PCIe-C DART tunables\n", dart->tunables->sz); apple_dart_apply_tunables(dart); @@ -2531,12 +2544,27 @@ static int apple_dart_probe(struct platform_device *pdev) * 64 SIDs. Use that description before making any DART MMIO access. */ if (tunneled) { - dart->pgsize = 1 << DART_PCIEC_PAGE_SHIFT; - dart->supports_bypass = true; - dart->ias = DART_PCIEC_ADDR_WIDTH; - dart->oas = DART_PCIEC_ADDR_WIDTH; - dart->num_streams = DART_PCIEC_STREAMS; - dart->four_level = true; + /* + * PARAMS is not safe to read on a tunneled DART. T8110 is + * 16 KiB pages, 42-bit addresses, 64 streams and four levels. + * A T8020 PCIe-C DART is 16 KiB pages, a 32-bit IOVA, a 36-bit + * PA and as many streams as its register block describes. + */ + if (dart->hw->type == DART_T8110) { + dart->pgsize = 1 << DART_PCIEC_PAGE_SHIFT; + dart->supports_bypass = true; + dart->ias = DART_PCIEC_ADDR_WIDTH; + dart->oas = DART_PCIEC_ADDR_WIDTH; + dart->num_streams = DART_PCIEC_STREAMS; + dart->four_level = true; + } else { + dart->pgsize = 1 << DART_PCIEC_PAGE_SHIFT; + dart->supports_bypass = false; + dart->ias = 32; + dart->oas = dart->hw->oas; + dart->num_streams = dart->hw->max_sid_count; + dart->four_level = false; + } dev_info(dev, "PCIe-C DART using fixed topology: pagesize %#x, %u streams, AS %u -> %u\n", dart->pgsize, dart->num_streams, dart->ias, dart->oas); From 72a2e7679654658e8549229437d57228af5855e8 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 23 Sep 2026 11:38:28 -0700 Subject: [PATCH 0229/2680] PCI: apple: finish tunneled IOMMU commands before the port clock stops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cable removal quiesces the PCIe tunnel and then removes the IOMMU. The IOMMU's runtime resume and remove both issue a command, and that command cannot complete once the port clock is gated. Every dock unplug logged that timeout, then a follow-up quiesce skipped the port and reported success. Tell the IOMMU to stop issuing commands before the clock is turned off, and let it issue them again only after the port is back up. An acknowledgment timeout from a cable that is already gone is no longer a failed quiesce when the local port has left the running state. [ryan: adapted to 7.2.10 — split by topic: only the DART hunks (drivers/iommu/apple-dart*.c, include/linux/soc/apple/dart.h) are carried in 020-dart, where the tunnel DART series lives; the remaining hunks stay with 080-radio] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/iommu/apple-dart.c | 37 ++++++++++++++++++++++++++++++++-- include/linux/soc/apple/dart.h | 14 +++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 include/linux/soc/apple/dart.h diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index f24cb37782d39..ac24e3a308604 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -286,6 +287,7 @@ struct apple_dart { u32 power_retained : 1; /* Locked roots may hold firmware slots that mirror Linux mappings. */ u32 fw_mirror : 1; + u32 commands_gated : 1; struct apple_tunable *tunables; u16 version; @@ -2729,11 +2731,42 @@ static int apple_dart_probe(struct platform_device *pdev) return ret; } +void apple_dart_quiesce_commands(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + + if (dart) + dart->commands_gated = true; +} +EXPORT_SYMBOL_GPL(apple_dart_quiesce_commands); + +void apple_dart_resume_commands(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + int ret; + + if (!dart || !dart->commands_gated) + return; + + dart->commands_gated = false; + ret = pm_runtime_get_sync(dev); + if (ret < 0) + pm_runtime_put_noidle(dev); + else + pm_runtime_put(dev); +} +EXPORT_SYMBOL_GPL(apple_dart_resume_commands); + static void apple_dart_remove(struct platform_device *pdev) { struct apple_dart *dart = platform_get_drvdata(pdev); - if (!dart->locked) + /* + * Cable removal gates the port clock before this device is removed. + * A command issued then never completes. The next probe resets the + * block after that clock is running again. + */ + if (!dart->locked && !dart->commands_gated) apple_dart_hw_reset(dart); free_irq(dart->irq, dart); @@ -2888,7 +2921,7 @@ static __maybe_unused int apple_dart_resume(struct device *dev) * state. Resetting one here is both unnecessary and earlier than Apple's * force-active call at the end of PCIe-C port resume. */ - if (dart->power_retained) + if (dart->power_retained || dart->commands_gated) return 0; /* Locked DARTs can't be restored, and they should not need it */ diff --git a/include/linux/soc/apple/dart.h b/include/linux/soc/apple/dart.h new file mode 100644 index 0000000000000..9902e602c5d11 --- /dev/null +++ b/include/linux/soc/apple/dart.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __LINUX_SOC_APPLE_DART_H__ +#define __LINUX_SOC_APPLE_DART_H__ + +struct device; + +/* + * The tunneled PCIe IOMMU shares the port clock. Stop issuing commands + * before that clock is gated, and allow them again once it is back. + */ +void apple_dart_quiesce_commands(struct device *dev); +void apple_dart_resume_commands(struct device *dev); + +#endif From ddb5a8ec246001eacb27a6a0885705ce599da879 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 21:00:22 -0700 Subject: [PATCH 0230/2680] PCI: apple: preserve failed tunnel recovery and optional DART builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Leave a failed restart quiesced and retryable instead of rescanning and clearing bus_stopped after a reset or link timeout. Resume DART commands only after the ports have restarted. Allow DART-disabled builds through no-op stubs, and constrain Apple PCIe to a module when its optional DART provider is modular. [ryan: adapted to 7.2.10 — split by topic: only the DART hunks (drivers/iommu/apple-dart*.c, include/linux/soc/apple/dart.h) are carried in 020-dart, where the tunnel DART series lives; the remaining hunks stay with 080-radio] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- include/linux/soc/apple/dart.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/include/linux/soc/apple/dart.h b/include/linux/soc/apple/dart.h index 9902e602c5d11..8fca017414d56 100644 --- a/include/linux/soc/apple/dart.h +++ b/include/linux/soc/apple/dart.h @@ -2,13 +2,20 @@ #ifndef __LINUX_SOC_APPLE_DART_H__ #define __LINUX_SOC_APPLE_DART_H__ +#include + struct device; /* * The tunneled PCIe IOMMU shares the port clock. Stop issuing commands * before that clock is gated, and allow them again once it is back. */ +#if IS_ENABLED(CONFIG_APPLE_DART) void apple_dart_quiesce_commands(struct device *dev); void apple_dart_resume_commands(struct device *dev); +#else +static inline void apple_dart_quiesce_commands(struct device *dev) { } +static inline void apple_dart_resume_commands(struct device *dev) { } +#endif #endif From a03413020aa6d442efa83ad77134875c6506bb7c Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:07:40 -0700 Subject: [PATCH 0231/2680] iommu/apple-dart: enforce command gating during tunnel teardown Quiescing a tunneled DART only suppressed the runtime-resume and remove resets. The actual stream and TLB command paths did not check commands_gated, so a late command could still access the command engine after the port clock was disabled. Serialize gating with the command lock, wait for in-flight commands to finish, and reject subsequent T8020/T8110 commands with EHOSTDOWN before any command-register access. Use an independently addressable flag for the lockless PM checks. Add a KUnit case verifying both command engines leave the register image unchanged while gated and accept commands again after restore. This hardens teardown and does not establish a fix for the observed physical PCIe link restart failure. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/iommu/apple-dart-test.c | 55 ++++++++++++++++++++++++++++++++- drivers/iommu/apple-dart.c | 26 +++++++++++----- 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 7f8888bf74115..5892b15c621a2 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only -/* Included by apple-dart.c to exercise the locked-domain lifecycle. */ +/* Included by apple-dart.c to exercise domain and command lifecycles. */ #include #include @@ -131,10 +131,63 @@ static void apple_dart_test_firmware_roots(struct kunit *test) KUNIT_EXPECT_EQ(test, live[1], 0x4001ULL); } +static void apple_dart_test_gated_commands(struct kunit *test) +{ + struct apple_dart_stream_map stream = {}; + struct apple_dart *dart; + u32 sentinel = 0x40; + int ret; + + dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart); + dart->dev = kunit_device_register(test, "apple-dart-gated-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dart->dev); + dev_set_drvdata(dart->dev, dart); + dart->regs = (__force void __iomem *)kunit_kzalloc(test, SZ_16K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart->regs); + dart->num_streams = 1; + dart->pgsize = SZ_16K; + dart->version = 0x0202; + spin_lock_init(&dart->lock); + stream.dart = dart; + __set_bit(0, stream.sidmap); + pm_runtime_set_active(dart->dev); + pm_runtime_enable(dart->dev); + + writel(sentinel, dart->regs + DART_T8020_STREAM_SELECT); + writel(sentinel, dart->regs + DART_T8020_STREAM_COMMAND); + writel(sentinel, dart->regs + DART_T8110_TLB_CMD); + writel(sentinel, dart->regs + DART_T8110_TLB_START); + writel(sentinel, dart->regs + DART_T8110_TLB_END); + apple_dart_quiesce_commands(dart->dev); + + /* Neither engine may touch its command registers after quiesce. */ + KUNIT_EXPECT_EQ(test, apple_dart_t8020_hw_invalidate_tlb(&stream), -EHOSTDOWN); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8020_STREAM_SELECT), sentinel); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8020_STREAM_COMMAND), sentinel); + ret = apple_dart_t8110_hw_tlb_command_range(&stream, + DART_T8110_TLB_CMD_OP_FLUSH_SID, + true, 0, SZ_16K - 1); + KUNIT_EXPECT_EQ(test, ret, -EHOSTDOWN); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_TLB_CMD), sentinel); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_TLB_START), sentinel); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_TLB_END), sentinel); + + /* A restored port may submit commands again. */ + apple_dart_resume_commands(dart->dev); + KUNIT_EXPECT_EQ(test, apple_dart_t8020_hw_invalidate_tlb(&stream), 0); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8020_STREAM_COMMAND), + (u32)DART_T8020_STREAM_COMMAND_INVALIDATE); + KUNIT_EXPECT_EQ(test, apple_dart_t8110_hw_invalidate_tlb(&stream), 0); + KUNIT_EXPECT_NE(test, readl(dart->regs + DART_T8110_TLB_CMD), sentinel); + pm_runtime_disable(dart->dev); +} + static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_before_attach), KUNIT_CASE(apple_dart_test_locked_handoff), KUNIT_CASE(apple_dart_test_firmware_roots), + KUNIT_CASE(apple_dart_test_gated_commands), {} }; diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index ac24e3a308604..d94eae4889879 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -287,7 +287,7 @@ struct apple_dart { u32 power_retained : 1; /* Locked roots may hold firmware slots that mirror Linux mappings. */ u32 fw_mirror : 1; - u32 commands_gated : 1; + bool commands_gated; struct apple_tunable *tunables; u16 version; @@ -925,6 +925,10 @@ apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map, u32 command_reg; spin_lock_irqsave(&dart->lock, flags); + if (dart->commands_gated) { + spin_unlock_irqrestore(&dart->lock, flags); + return -EHOSTDOWN; + } for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) apple_dart_writel(dart, stream_map->sidmap[i], @@ -959,6 +963,10 @@ apple_dart_t8110_hw_tlb_command_range(struct apple_dart_stream_map *stream_map, int sid, ret = 0; spin_lock_irqsave(&dart->lock, flags); + if (dart->commands_gated) { + spin_unlock_irqrestore(&dart->lock, flags); + return -EHOSTDOWN; + } for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { u32 val = FIELD_PREP(DART_T8110_TLB_CMD_OP, command) | FIELD_PREP(DART_T8110_TLB_CMD_STREAM, sid); @@ -2735,8 +2743,11 @@ void apple_dart_quiesce_commands(struct device *dev) { struct apple_dart *dart = dev_get_drvdata(dev); - if (dart) - dart->commands_gated = true; + if (dart) { + /* Finish any in-flight command before the port clock is stopped. */ + guard(spinlock_irqsave)(&dart->lock); + WRITE_ONCE(dart->commands_gated, true); + } } EXPORT_SYMBOL_GPL(apple_dart_quiesce_commands); @@ -2745,10 +2756,11 @@ void apple_dart_resume_commands(struct device *dev) struct apple_dart *dart = dev_get_drvdata(dev); int ret; - if (!dart || !dart->commands_gated) + if (!dart || !READ_ONCE(dart->commands_gated)) return; - dart->commands_gated = false; + scoped_guard(spinlock_irqsave, &dart->lock) + WRITE_ONCE(dart->commands_gated, false); ret = pm_runtime_get_sync(dev); if (ret < 0) pm_runtime_put_noidle(dev); @@ -2766,7 +2778,7 @@ static void apple_dart_remove(struct platform_device *pdev) * A command issued then never completes. The next probe resets the * block after that clock is running again. */ - if (!dart->locked && !dart->commands_gated) + if (!dart->locked && !READ_ONCE(dart->commands_gated)) apple_dart_hw_reset(dart); free_irq(dart->irq, dart); @@ -2921,7 +2933,7 @@ static __maybe_unused int apple_dart_resume(struct device *dev) * state. Resetting one here is both unnecessary and earlier than Apple's * force-active call at the end of PCIe-C port resume. */ - if (dart->power_retained || dart->commands_gated) + if (dart->power_retained || READ_ONCE(dart->commands_gated)) return 0; /* Locked DARTs can't be restored, and they should not need it */ From e19eb85eae327a19616561957d39f7db495e11d5 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:11:36 -0700 Subject: [PATCH 0232/2680] iommu/apple-dart: report tunnel command-resume failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resuming commands cleared the gate before runtime resume, then discarded a runtime-PM failure. PCIe-C could consequently rescan functions without a successfully restored IOMMU and without a gate protecting the failed command engine. Return the resume error, restore the gate on failure, and propagate DART errors through the tunneled-host restore path before rescan. Keep the runtime-PM usage count balanced with pm_runtime_resume_and_get(). A KUnit fault case verifies error propagation, gate retention, reference balance and a subsequent successful retry. [ryan: adapted to 7.2.10 — split by topic: only the DART hunks (drivers/iommu/apple-dart*.c, include/linux/soc/apple/dart.h) are carried in 020-dart, where the tunnel DART series lives; the remaining hunks stay with 080-radio] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/iommu/apple-dart-test.c | 30 ++++++++++++++++++++++++++++++ drivers/iommu/apple-dart.c | 23 ++++++++++++++--------- include/linux/soc/apple/dart.h | 8 ++++---- 3 files changed, 48 insertions(+), 13 deletions(-) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 5892b15c621a2..687d097bd3e5b 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -183,11 +183,41 @@ static void apple_dart_test_gated_commands(struct kunit *test) pm_runtime_disable(dart->dev); } +static void apple_dart_test_resume_commands_failure(struct kunit *test) +{ + struct apple_dart *dart; + + dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart); + dart->dev = kunit_device_register(test, "apple-dart-resume-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dart->dev); + dev_set_drvdata(dart->dev, dart); + spin_lock_init(&dart->lock); + pm_runtime_set_active(dart->dev); + pm_runtime_enable(dart->dev); + apple_dart_quiesce_commands(dart->dev); + + /* A prior runtime-PM error must leave commands gated and report failure. */ + scoped_guard(spinlock_irqsave, &dart->dev->power.lock) + dart->dev->power.runtime_error = -EIO; + KUNIT_EXPECT_EQ(test, apple_dart_resume_commands(dart->dev), -EINVAL); + KUNIT_EXPECT_TRUE(test, dart->commands_gated); + KUNIT_EXPECT_EQ(test, atomic_read(&dart->dev->power.usage_count), 0); + + /* Clearing the PM error permits a later retry without leaking a reference. */ + KUNIT_EXPECT_EQ(test, pm_runtime_set_active(dart->dev), 0); + KUNIT_EXPECT_EQ(test, apple_dart_resume_commands(dart->dev), 0); + KUNIT_EXPECT_FALSE(test, dart->commands_gated); + KUNIT_EXPECT_EQ(test, atomic_read(&dart->dev->power.usage_count), 0); + pm_runtime_disable(dart->dev); +} + static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_before_attach), KUNIT_CASE(apple_dart_test_locked_handoff), KUNIT_CASE(apple_dart_test_firmware_roots), KUNIT_CASE(apple_dart_test_gated_commands), + KUNIT_CASE(apple_dart_test_resume_commands_failure), {} }; diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index d94eae4889879..b175232576711 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -2739,7 +2739,7 @@ static int apple_dart_probe(struct platform_device *pdev) return ret; } -void apple_dart_quiesce_commands(struct device *dev) +int apple_dart_quiesce_commands(struct device *dev) { struct apple_dart *dart = dev_get_drvdata(dev); @@ -2748,24 +2748,29 @@ void apple_dart_quiesce_commands(struct device *dev) guard(spinlock_irqsave)(&dart->lock); WRITE_ONCE(dart->commands_gated, true); } + return 0; } EXPORT_SYMBOL_GPL(apple_dart_quiesce_commands); -void apple_dart_resume_commands(struct device *dev) +int apple_dart_resume_commands(struct device *dev) { struct apple_dart *dart = dev_get_drvdata(dev); int ret; - if (!dart || !READ_ONCE(dart->commands_gated)) - return; + if (!dart) + return -ENODEV; + if (!READ_ONCE(dart->commands_gated)) + return 0; scoped_guard(spinlock_irqsave, &dart->lock) WRITE_ONCE(dart->commands_gated, false); - ret = pm_runtime_get_sync(dev); - if (ret < 0) - pm_runtime_put_noidle(dev); - else - pm_runtime_put(dev); + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) { + apple_dart_quiesce_commands(dev); + return ret; + } + pm_runtime_put(dev); + return 0; } EXPORT_SYMBOL_GPL(apple_dart_resume_commands); diff --git a/include/linux/soc/apple/dart.h b/include/linux/soc/apple/dart.h index 8fca017414d56..423602f02be3b 100644 --- a/include/linux/soc/apple/dart.h +++ b/include/linux/soc/apple/dart.h @@ -11,11 +11,11 @@ struct device; * before that clock is gated, and allow them again once it is back. */ #if IS_ENABLED(CONFIG_APPLE_DART) -void apple_dart_quiesce_commands(struct device *dev); -void apple_dart_resume_commands(struct device *dev); +int apple_dart_quiesce_commands(struct device *dev); +int apple_dart_resume_commands(struct device *dev); #else -static inline void apple_dart_quiesce_commands(struct device *dev) { } -static inline void apple_dart_resume_commands(struct device *dev) { } +static inline int apple_dart_quiesce_commands(struct device *dev) { return 0; } +static inline int apple_dart_resume_commands(struct device *dev) { return 0; } #endif #endif From eb7bc71282a64f6b71728b816b7742db3dc40806 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:11:36 -0700 Subject: [PATCH 0233/2680] PCI: apple: describe tunneled PM ordering by Linux dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document the retained-state, doorbell and PCIe-C/DART ordering requirements directly in terms of register access and Linux device dependencies. No functional change. [ryan: adapted to 7.2.10 — split by topic: only the DART hunks (drivers/iommu/apple-dart*.c, include/linux/soc/apple/dart.h) are carried in 020-dart, where the tunnel DART series lives; the remaining hunks stay with 080-radio] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/iommu/apple-dart.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index b175232576711..23111219734a9 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -2549,9 +2549,9 @@ static int apple_dart_probe(struct platform_device *pdev) * The cable-powered T8110 DART does not provide a safely probeable * parameter block at this point in PCIe tunnel activation. The SError * raised by these reads is asynchronous, which made the following trace - * boundary look guilty on different boots. Apple instead publishes the - * complete immutable topology in ADT: 16 KiB pages, 42-bit addresses and - * 64 SIDs. Use that description before making any DART MMIO access. + * boundary look guilty on different boots. The device tree supplies the + * immutable topology: 16 KiB pages, 42-bit addresses and 64 SIDs. Use that + * description before making any DART MMIO access. */ if (tunneled) { /* @@ -2935,8 +2935,8 @@ static __maybe_unused int apple_dart_resume(struct device *dev) /* * PCIe-C DARTs on an apple,always-on domain retain their translation - * state. Resetting one here is both unnecessary and earlier than Apple's - * force-active call at the end of PCIe-C port resume. + * state. Resetting one here is unnecessary; DART access must remain + * ordered after PCIe-C port resume. */ if (dart->power_retained || READ_ONCE(dart->commands_gated)) return 0; From acb4a65e9767fc4ab735697a8d9aa9b440d397b4 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 20:35:13 -0700 Subject: [PATCH 0234/2680] thunderbolt: restore M1 tunneled PCIe state and recover late link loss MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kensington USB devices can disappear after s2idle even when the initial port restart and cable health check succeed. A later PCIe link failure must trigger another fresh validation instead of waiting for a cable interrupt or manual replug. Restore the resettable M1 tunneled host and DART state before restarting the downstream link. Gate DART commands and fault handling around clock shutdown, preserve stream selection, and describe the reset and register initialization properties in the binding. Gate Type-C work across initialization, sleep and removal, read fresh cable metadata after resume, and preserve an unchanged healthy session. Propagate late link failures through an atomic switch notification into bounded process-context revalidation. Keep failure history sticky while allowing later notifications from a retained host. Failed reads or health checks request a bounded reconnect; successful checks preserve routing. Keep failed display-route acquisition retryable. Add state-machine and DART regression coverage. The native ARM64 build and 88 KUnit cases pass. Two M1/j293 Kensington USB-only suspend cycles exercised late link-loss recovery and restored usable input without a manual replug. The initiating xHCI resume fault remains unresolved. [ryan: adapted to 7.2.10 — split by topic: only the DART hunks (drivers/iommu/apple-dart*.c, include/linux/soc/apple/dart.h) are carried in 020-dart, where the tunnel DART series lives; the remaining hunks stay with 170-usb] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/iommu/apple-dart-test.c | 167 ++++++++++++++++++++++++++++++++ drivers/iommu/apple-dart.c | 96 ++++++++++++++++-- include/linux/soc/apple/dart.h | 5 + 3 files changed, 260 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 687d097bd3e5b..0eba2eca17474 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -212,12 +212,179 @@ static void apple_dart_test_resume_commands_failure(struct kunit *test) pm_runtime_disable(dart->dev); } +static struct apple_dart *apple_dart_test_tunnel(struct kunit *test) +{ + struct apple_dart *dart; + + dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart); + dart->dev = kunit_device_register(test, "apple-dart-tunnel-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dart->dev); + dart->regs = (__force void __iomem *)kunit_kzalloc(test, SZ_16K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart->regs); + dart->hw = &apple_dart_hw_t8103_usb4; + dart->num_streams = dart->hw->max_sid_count; + dart->tunneled = true; + spin_lock_init(&dart->lock); + dev_set_drvdata(dart->dev, dart); + return dart; +} + +static void apple_dart_test_tunnel_restore(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_tunnel(test); + unsigned int sid, idx; + u32 sentinel = 0x40; + + /* M1 tunnel DART nodes have no retained power-domain attachment. */ + KUNIT_ASSERT_FALSE(test, dart->power_retained); + for (sid = 0; sid < dart->num_streams; sid++) { + writel(0x80 + sid, dart->regs + DART_TCR(dart, sid)); + for (idx = 0; idx < dart->hw->ttbr_count; idx++) + writel(0x1000 + 0x10 * sid + idx, + dart->regs + DART_TTBR(dart, sid, idx)); + } + KUNIT_ASSERT_EQ(test, apple_dart_save_tunnel_state(dart->dev), 0); + memset_io(dart->regs, 0, SZ_16K); + KUNIT_ASSERT_EQ(test, apple_dart_restore_tunnel_state(dart->dev), 0); + for (sid = 0; sid < dart->num_streams; sid++) { + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, sid)), + 0x80 + sid); + for (idx = 0; idx < dart->hw->ttbr_count; idx++) + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TTBR(dart, sid, idx)), + 0x1000 + 0x10 * sid + idx); + } + KUNIT_EXPECT_FALSE(test, dart->tunnel_state_saved); + KUNIT_EXPECT_TRUE(test, dart->tunnel_state_restored); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8020_STREAM_SELECT), U32_MAX); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8020_STREAM_SELECT + 4), U32_MAX); + + /* Normal resume must not reset translations after the link is live. */ + writel(sentinel, dart->regs + DART_T8020_STREAM_COMMAND); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8020_STREAM_COMMAND), sentinel); + KUNIT_EXPECT_FALSE(test, dart->tunnel_state_restored); + KUNIT_EXPECT_EQ(test, apple_dart_restore_tunnel_state(dart->dev), -EINVAL); +} + +static void apple_dart_test_tunnel_reject_unsafe(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_tunnel(test); + u32 sentinel = 0x40; + + writel(sentinel, dart->regs + DART_T8020_STREAM_COMMAND); + KUNIT_EXPECT_EQ(test, apple_dart_restore_tunnel_state(dart->dev), -EINVAL); + dart->locked = true; + KUNIT_EXPECT_EQ(test, apple_dart_save_tunnel_state(dart->dev), -EOPNOTSUPP); + dart->locked = false; + dart->commands_gated = true; + KUNIT_EXPECT_EQ(test, apple_dart_save_tunnel_state(dart->dev), -EBUSY); + KUNIT_EXPECT_FALSE(test, dart->tunnel_state_saved); + dart->commands_gated = false; + dart->hw = &apple_dart_hw_t8110; + KUNIT_EXPECT_EQ(test, apple_dart_save_tunnel_state(dart->dev), -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8020_STREAM_COMMAND), sentinel); +} + +static int apple_dart_test_tlb_failure(struct apple_dart_stream_map *stream_map) +{ + return -EIO; +} + +static void apple_dart_test_tunnel_reset_failure(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_tunnel(test); + struct apple_dart_hw hw = apple_dart_hw_t8103_usb4; + + hw.invalidate_tlb = apple_dart_test_tlb_failure; + dart->hw = &hw; + KUNIT_ASSERT_EQ(test, apple_dart_save_tunnel_state(dart->dev), 0); + KUNIT_EXPECT_EQ(test, apple_dart_restore_tunnel_state(dart->dev), -EIO); + KUNIT_EXPECT_FALSE(test, dart->tunnel_state_restored); + KUNIT_EXPECT_TRUE(test, dart->tunnel_state_saved); + + /* A later successful reset can still restore the saved mappings. */ + dart->hw = &apple_dart_hw_t8103_usb4; + KUNIT_EXPECT_EQ(test, apple_dart_restore_tunnel_state(dart->dev), 0); + KUNIT_EXPECT_TRUE(test, dart->tunnel_state_restored); + KUNIT_EXPECT_EQ(test, apple_dart_suspend(dart->dev), 0); + KUNIT_EXPECT_FALSE(test, dart->tunnel_state_restored); +} + +static void apple_dart_test_failed_host_pm(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_tunnel(test); + u32 sentinel = 0x40; + + writel(0x80, dart->regs + DART_TCR(dart, 0)); + KUNIT_ASSERT_EQ(test, apple_dart_save_tunnel_state(dart->dev), 0); + apple_dart_quiesce_commands(dart->dev); + writel(sentinel, dart->regs + DART_T8020_STREAM_COMMAND); + writel(sentinel, dart->regs + DART_TCR(dart, 0)); + + /* PM still resumes dependents after the host's noirq callback fails. */ + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8020_STREAM_COMMAND), sentinel); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 0)), sentinel); + KUNIT_EXPECT_TRUE(test, dart->commands_gated); + + /* Another sleep must neither access the failed port nor lose its snapshot. */ + KUNIT_EXPECT_EQ(test, apple_dart_suspend(dart->dev), 0); + KUNIT_EXPECT_EQ(test, dart->save_tcr[0], 0x80U); + KUNIT_EXPECT_EQ(test, apple_dart_save_tunnel_state(dart->dev), -EBUSY); + KUNIT_EXPECT_TRUE(test, dart->tunnel_state_saved); + KUNIT_EXPECT_EQ(test, dart->save_tcr[0], 0x80U); +} + +static irqreturn_t apple_dart_test_irq_handler(int irq, void *data) +{ + struct apple_dart *dart = data; + + writel(0x1234, dart->regs + DART_TCR(dart, 0)); + return IRQ_HANDLED; +} + +static void apple_dart_test_gated_irq(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_tunnel(test); + struct apple_dart_hw hw = apple_dart_hw_t8103_usb4; + + hw.irq_handler = apple_dart_test_irq_handler; + dart->hw = &hw; + pm_runtime_set_active(dart->dev); + pm_runtime_enable(dart->dev); + apple_dart_quiesce_commands(dart->dev); + KUNIT_EXPECT_EQ(test, apple_dart_irq(0, dart), IRQ_NONE); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 0)), 0U); + KUNIT_EXPECT_EQ(test, atomic_read(&dart->dev->power.usage_count), 0); + + KUNIT_ASSERT_EQ(test, apple_dart_resume_commands(dart->dev), 0); + KUNIT_EXPECT_EQ(test, apple_dart_irq(0, dart), IRQ_HANDLED); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 0)), 0x1234U); + KUNIT_EXPECT_EQ(test, atomic_read(&dart->dev->power.usage_count), 0); + + /* Failed runtime resume must not run the handler or leak its reference. */ + writel(0, dart->regs + DART_TCR(dart, 0)); + scoped_guard(spinlock_irqsave, &dart->dev->power.lock) + dart->dev->power.runtime_error = -EIO; + KUNIT_EXPECT_EQ(test, apple_dart_irq(0, dart), IRQ_NONE); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 0)), 0U); + KUNIT_EXPECT_EQ(test, atomic_read(&dart->dev->power.usage_count), 0); + KUNIT_EXPECT_EQ(test, pm_runtime_set_active(dart->dev), 0); + pm_runtime_disable(dart->dev); +} + static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_before_attach), KUNIT_CASE(apple_dart_test_locked_handoff), KUNIT_CASE(apple_dart_test_firmware_roots), KUNIT_CASE(apple_dart_test_gated_commands), KUNIT_CASE(apple_dart_test_resume_commands_failure), + KUNIT_CASE(apple_dart_test_tunnel_restore), + KUNIT_CASE(apple_dart_test_tunnel_reject_unsafe), + KUNIT_CASE(apple_dart_test_tunnel_reset_failure), + KUNIT_CASE(apple_dart_test_failed_host_pm), + KUNIT_CASE(apple_dart_test_gated_irq), {} }; diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 23111219734a9..f98a15fe8064b 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -287,6 +287,8 @@ struct apple_dart { u32 power_retained : 1; /* Locked roots may hold firmware slots that mirror Linux mappings. */ u32 fw_mirror : 1; + u32 tunnel_state_saved : 1; + u32 tunnel_state_restored : 1; bool commands_gated; struct apple_tunable *tunables; u16 version; @@ -922,6 +924,7 @@ apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map, struct apple_dart *dart = stream_map->dart; unsigned long flags; int ret, i; + u32 sidmap[BITS_TO_U32(DART_MAX_STREAMS)]; u32 command_reg; spin_lock_irqsave(&dart->lock, flags); @@ -930,9 +933,10 @@ apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map, return -EHOSTDOWN; } + bitmap_to_arr32(sidmap, stream_map->sidmap, dart->num_streams); for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) - apple_dart_writel(dart, stream_map->sidmap[i], - DART_T8020_STREAM_SELECT + 4 * i); + apple_dart_writel(dart, sidmap[i], + DART_T8020_STREAM_SELECT + 4 * i); apple_dart_writel(dart, command, DART_T8020_STREAM_COMMAND); ret = read_poll_timeout_atomic( @@ -2392,11 +2396,15 @@ static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) static irqreturn_t apple_dart_irq(int irq, void *dev) { - irqreturn_t ret; + irqreturn_t ret = IRQ_NONE; struct apple_dart *dart = dev; - WARN_ON(pm_runtime_get_sync(dart->dev) < 0); - ret = dart->hw->irq_handler(irq, dev); + if (READ_ONCE(dart->commands_gated)) + return IRQ_NONE; + if (pm_runtime_resume_and_get(dart->dev) < 0) + return IRQ_NONE; + if (!READ_ONCE(dart->commands_gated)) + ret = dart->hw->irq_handler(irq, dev); pm_runtime_put(dart->dev); return ret; } @@ -2745,8 +2753,11 @@ int apple_dart_quiesce_commands(struct device *dev) if (dart) { /* Finish any in-flight command before the port clock is stopped. */ - guard(spinlock_irqsave)(&dart->lock); - WRITE_ONCE(dart->commands_gated, true); + scoped_guard(spinlock_irqsave, &dart->lock) + WRITE_ONCE(dart->commands_gated, true); + /* The fault handler can take dart->lock while acknowledging errors. */ + if (dart->irq > 0) + synchronize_irq(dart->irq); } return 0; } @@ -2774,6 +2785,66 @@ int apple_dart_resume_commands(struct device *dev) } EXPORT_SYMBOL_GPL(apple_dart_resume_commands); +/* + * System sleep has quiesced every downstream device. PCIe-C calls this + * before stopping APPCLK because its domain reset also loses DART state, + * even when ordinary runtime PM treats the domain as retained. + */ +int apple_dart_save_tunnel_state(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned int sid, idx; + + if (!dart || !dart->tunneled || dart->locked || + dart->hw->type != DART_T8020 || dart->tunables) + return -EOPNOTSUPP; + + guard(spinlock_irqsave)(&dart->lock); + if (dart->commands_gated) + return -EBUSY; + dart->tunnel_state_saved = false; + dart->tunnel_state_restored = false; + + for (sid = 0; sid < dart->num_streams; sid++) { + dart->save_tcr[sid] = apple_dart_readl(dart, DART_TCR(dart, sid)); + for (idx = 0; idx < dart->hw->ttbr_count; idx++) + dart->save_ttbr[sid][idx] = + apple_dart_readl(dart, DART_TTBR(dart, sid, idx)); + } + dart->tunnel_state_saved = true; + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_save_tunnel_state); + +/* APPCLK is running again, but PCIe link training has not started. */ +int apple_dart_restore_tunnel_state(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned int sid, idx; + int ret; + + if (!dart || !dart->tunneled || dart->locked || + dart->hw->type != DART_T8020 || dart->tunables) + return -EOPNOTSUPP; + if (!dart->tunnel_state_saved || READ_ONCE(dart->commands_gated)) + return -EINVAL; + + ret = apple_dart_hw_reset(dart); + if (ret) + return ret; + + for (sid = 0; sid < dart->num_streams; sid++) { + for (idx = 0; idx < dart->hw->ttbr_count; idx++) + apple_dart_writel(dart, dart->save_ttbr[sid][idx], + DART_TTBR(dart, sid, idx)); + apple_dart_writel(dart, dart->save_tcr[sid], DART_TCR(dart, sid)); + } + dart->tunnel_state_saved = false; + dart->tunnel_state_restored = true; + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_restore_tunnel_state); + static void apple_dart_remove(struct platform_device *pdev) { struct apple_dart *dart = platform_get_drvdata(pdev); @@ -2907,6 +2978,9 @@ static __maybe_unused int apple_dart_suspend(struct device *dev) struct apple_dart *dart = dev_get_drvdata(dev); unsigned int sid, idx; + dart->tunnel_state_restored = false; + if (READ_ONCE(dart->commands_gated)) + return 0; /* The tunneled DART has no state to save when its domain stays on. */ if (dart->power_retained) return 0; @@ -2933,6 +3007,12 @@ static __maybe_unused int apple_dart_resume(struct device *dev) unsigned int sid, idx; int ret; + /* PCIe-C restored these mappings before starting its downstream link. */ + if (dart->tunnel_state_restored) { + dart->tunnel_state_restored = false; + return 0; + } + /* * PCIe-C DARTs on an apple,always-on domain retain their translation * state. Resetting one here is unnecessary; DART access must remain @@ -2954,7 +3034,7 @@ static __maybe_unused int apple_dart_resume(struct device *dev) for (sid = 0; sid < dart->num_streams; sid++) { for (idx = 0; idx < dart->hw->ttbr_count; idx++) apple_dart_writel(dart, dart->save_ttbr[sid][idx], - DART_TTBR(dart, sid, idx)); + DART_TTBR(dart, sid, idx)); apple_dart_writel(dart, dart->save_tcr[sid], DART_TCR(dart, sid)); } diff --git a/include/linux/soc/apple/dart.h b/include/linux/soc/apple/dart.h index 423602f02be3b..61ec51a342bde 100644 --- a/include/linux/soc/apple/dart.h +++ b/include/linux/soc/apple/dart.h @@ -3,6 +3,7 @@ #define __LINUX_SOC_APPLE_DART_H__ #include +#include struct device; @@ -13,9 +14,13 @@ struct device; #if IS_ENABLED(CONFIG_APPLE_DART) int apple_dart_quiesce_commands(struct device *dev); int apple_dart_resume_commands(struct device *dev); +int apple_dart_save_tunnel_state(struct device *dev); +int apple_dart_restore_tunnel_state(struct device *dev); #else static inline int apple_dart_quiesce_commands(struct device *dev) { return 0; } static inline int apple_dart_resume_commands(struct device *dev) { return 0; } +static inline int apple_dart_save_tunnel_state(struct device *dev) { return -EOPNOTSUPP; } +static inline int apple_dart_restore_tunnel_state(struct device *dev) { return -EOPNOTSUPP; } #endif #endif From 900a7e4d7da7acd8b7f5b2623a956e61efceb072 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 01:30:45 +1100 Subject: [PATCH 0235/2680] iommu: apple-dart: keep inherited firmware translations visible before attach On T8140 the internal display coprocessor's DART stream is locked, and its root already holds firmware translations for the regions the loader re-exports as translated reservations. Domain setup imports those roots so that apple_dart_iova_to_phys() reports the inherited translations, and the IOMMU core then skips building replacement mappings for them. Since domain finalization stopped pre-populating the attached-stream map, because reserved mappings can be installed before any device is attached, that lookup finds no stream and reports nothing. The core then builds read-write, uncached host leaves for the reservations. Those differ from the firmware's cached leaves, and from its read-only code region, so publishing them into the firmware-owned root slots fails, and the core's must-succeed default-domain attach returns -EBUSY: WARNING: drivers/iommu/iommu.c:3243 at iommu_setup_default_domain apple-dart 412340000.iommu: probe with driver apple-dart failed with error -16 The display coprocessor then never binds. Record, per stream map, the streams whose firmware roots finalization imported, and use them in addition to the attached streams when resolving inherited translations and when validating software mappings against firmware-owned slots. The firmware leaves are left untouched, including the read-only region. Also keep DARTs with imported firmware roots out of the legacy path that replaces a locked root's entries wholesale. That path is meant for older display DARTs whose root Linux takes over; on these DARTs it would overwrite live firmware slots. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index f98a15fe8064b..766147a22ec2e 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -393,6 +393,9 @@ struct apple_dart_atomic_stream_map { * @finalized: true if the domain has been completely initialized * @init_lock: protects domain initialization * @stream_maps: streams attached to this domain (valid for DMA/UNMANAGED only) + * @fw_sidmap: streams whose locked firmware roots the domain imported at + * finalization, per entry of @stream_maps; unlike @stream_maps + * they are known before any device is attached * @domain: core iommu domain pointer */ struct apple_dart_domain { @@ -406,6 +409,7 @@ struct apple_dart_domain { dma_addr_t dma_offset; struct mutex init_lock; struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; + DECLARE_BITMAP(fw_sidmap[MAX_DARTS_PER_DEVICE], DART_MAX_STREAMS); struct iommu_domain domain; }; @@ -892,7 +896,8 @@ apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg, * before attachment, so replace the old root entries. * T8110 v2.2+ instead retains firmware-owned entries. */ - if (dart->hw->type != DART_T8110 || dart->version < 0x0202) { + if (!dart->fw_mirror && + (dart->hw->type != DART_T8110 || dart->version < 0x0202)) { size_t entry; dma_wmb(); @@ -1265,6 +1270,20 @@ static int apple_dart_iotlb_sync_map(struct iommu_domain *domain, iova, iova + size - 1); } +/* + * The core installs reserved mappings before it attaches the first device, + * while the attached-stream map is still empty. A locked stream whose + * firmware root the domain imported keeps its inherited translations from + * then on, so treat it as part of the domain for firmware lookups. + */ +static bool apple_dart_domain_uses_fw(struct apple_dart_domain *domain, int i, + struct apple_dart_atomic_stream_map *map, + int sid) +{ + return test_bit(sid, domain->fw_sidmap[i]) || + (atomic_long_read(&map->sidmap[BIT_WORD(sid)]) & BIT_MASK(sid)); +} + /* * Hardware continues to use every firmware-owned slot. Accept a private * software mapping only when every stream using such a slot agrees on the @@ -1302,8 +1321,7 @@ static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, u64 pte; int fw_prot; - if (!(atomic_long_read(&map->sidmap[BIT_WORD(sid)]) & - BIT_MASK(sid))) + if (!apple_dart_domain_uses_fw(domain, i, map, sid)) continue; fw = dart->locked_fw[sid][idx]; if (!dart->locked_ttbr[sid][idx] || @@ -1350,7 +1368,7 @@ apple_dart_inherited_phys(struct apple_dart_domain *domain, dma_addr_t address) phys_addr_t phys; int prot; - if (!(atomic_long_read(&map->sidmap[BIT_WORD(sid)]) & BIT_MASK(sid))) + if (!apple_dart_domain_uses_fw(domain, i, map, sid)) continue; fw = dart->locked_fw[sid][0]; if (!apple_dart_fw_slot_owned(fw, dart->locked_ttbr[sid][0], slot) || @@ -1603,6 +1621,8 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, goto done; } } + bitmap_copy(dart_domain->fw_sidmap[i], stream->sidmap, + stream->dart->num_streams); } } From f9d68c0ce9ad3d7775eadcbd1f2f0f9d8446d746 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 01:12:37 +1100 Subject: [PATCH 0236/2680] iommu: apple-dart: invalidate T8110 ranges by device DVA The range invalidate for locked T8110 v2.2+ streams passes the IOVA masked to the page-table input width. The TLB range registers match the DVA the device actually issues, though. On T8140 the display and DCP streams sit in a window at 1 TiB (apple,dma-range <0x100 0x0 0x10 0x0>), so the masked range never matches a cached translation there: - a range invalidate over [0, 64 GiB) during the boot scanout does nothing observable, even after 2500 repetitions; - the same range over [1 TiB, 1 TiB + 64 GiB) hits the scanout's live translations at once. Every unmap on these streams therefore left its translations in the TLB. Invalidate the DART-visible range, IOVA plus the DART offset, as the map path already uses it. Windows below the table width are unaffected, because masking was a no-op there. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 766147a22ec2e..cb3290b7d3ee0 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -1229,10 +1229,12 @@ static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, dev_err_ratelimited(stream.dart->dev, "failed to publish host page-table entries: %d\n", ret); else { + /* The range registers take the DVA the device issues. */ if (range && stream.dart->locked && stream.dart->version >= 0x0202) ret = apple_dart_t8110_hw_tlb_command_range(&stream, DART_T8110_TLB_CMD_OP_FLUSH_SID, true, - first & domain->mask, last & domain->mask); + first + domain->dma_offset, + last + domain->dma_offset); else ret = stream.dart->hw->invalidate_tlb(&stream); } From 4390cd89cabef7ee2243330a5973c7d1707a7edb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 01:14:04 +1100 Subject: [PATCH 0237/2680] iommu: apple-dart: leave live firmware translations out of invalidates On T8140 the boot loader locks the display DART, and the display controller keeps scanning out the boot framebuffer through the firmware translations while Linux attaches the default domain. The attach ends with a whole-stream TLB invalidate. Now and then the next table walk after that invalidate reports an invalid PTE, although the firmware entry is valid and unchanged: apple-dart 412300000.iommu: translation fault: SID:0 read PTE invalid at 0x10002a20800 (status 0x00003200) The display coprocessor then panics on the failed read, apple-dcp cannot boot RTKit (-ETIME), and the panel stays on simpledrm without backlight control. This hit about one boot in twenty. During the boot scanout, 500 whole-stream invalidates reproduce the fault within 30 ms, with the faulting leaf entry still valid when read back. One invalidate by DVA range over the scanout's window faults as well, with "STT mismatch". So any invalidate of translations that the hardware is using can break the scanout. Linux never changes what a firmware-owned root slot resolves to: it publishes only into vacant slots, and accepts a firmware slot only when its own mappings match it. On DARTs that keep firmware roots (T8140 and T8142), invalidate by DVA range only the parts of the stream's window that are reached through other root slots. That covers the whole-stream invalidate at attach and teardown, and the range invalidates on map and unmap. Slots that Linux retires are invalidated while the root is still known. Clear the root pointer and the firmware snapshot under the same lock, so that an invalidate never sees one without the other. Four-level streams, older DART revisions and other SoCs keep their current invalidates. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 96 +++++++++++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index cb3290b7d3ee0..7bca8b7234be0 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -770,6 +770,9 @@ apple_dart_snapshot_fw_root(struct apple_dart *dart, int sid, const u64 *live) return fw; } +static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, + u64 lo, u64 hi); + static int apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) { @@ -836,14 +839,20 @@ apple_dart_hw_unmap_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx if (!live) continue; apple_dart_retire_root(live, owned, dart->pgsize / sizeof(*live)); + if (dart->fw_mirror && dart->version >= 0x0202 && !idx) { + struct apple_dart_stream_map stream = { .dart = dart }; + + __set_bit(sid, stream.sidmap); + (void)apple_dart_fw_invalidate(&stream, 0, U64_MAX); + } spin_lock_irqsave(&dart->lock, flags); fw = dart->locked_fw[sid][idx]; dart->locked_fw[sid][idx] = NULL; + dart->locked_ttbr[sid][idx] = NULL; spin_unlock_irqrestore(&dart->lock, flags); apple_dart_free_fw_root(dart, fw); devm_memunmap(dart->dev, live); devm_kfree(dart->dev, owned); - dart->locked_ttbr[sid][idx] = NULL; dart->locked_owned[sid][idx] = NULL; } return 0; @@ -1016,6 +1025,84 @@ apple_dart_t8110_hw_tlb_command_range(struct apple_dart_stream_map *stream_map, return ret; } +/* A root table is at most 16 KiB of 64-bit entries. */ +#define DART_MAX_ROOT_ENTRIES (SZ_16K / sizeof(u64)) + +/* + * On DARTs that keep firmware roots (T8140 and T8142), the hardware keeps + * using the translations behind the firmware-owned root slots, e.g. the + * display controller scanning out the boot framebuffer while Linux attaches. + * Invalidating those translations, for the whole stream or by DVA range, can + * make the scanout fail a translation, and the display coprocessor panics on + * the failed read. Linux never changes what those slots resolve to, so only + * invalidate the parts of [lo, hi] reached through other root slots. + */ +static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, + u64 lo, u64 hi) +{ + const u32 op = DART_T8110_TLB_CMD_OP_FLUSH_SID; + struct apple_dart *dart = stream_map->dart; + struct apple_dart_stream_map stream = { .dart = dart }; + size_t entries = dart->pgsize / sizeof(u64); + u32 slot_shift = ilog2(dart->pgsize) + ilog2(entries); + u64 span = BIT_ULL(slot_shift + ilog2(entries)); + u64 base = (dart->dma_min + dart->dma_offset) & ~(span - 1); + DECLARE_BITMAP(fw_slots, DART_MAX_ROOT_ENTRIES); + size_t first, last, i, end; + unsigned long flags; + int sid, ret = 0; + + if (entries > DART_MAX_ROOT_ENTRIES || hi < base || lo > base + span - 1) + return 0; + lo = max(lo, base); + hi = min(hi, base + span - 1); + first = (lo - base) >> slot_shift; + last = (hi - base) >> slot_shift; + + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + const struct apple_dart_fw_root *fw; + const u64 *live; + + __set_bit(sid, stream.sidmap); + if (apple_dart_readl(dart, DART_TCR(dart, sid)) & + dart->hw->tcr_4level) { + ret = apple_dart_t8110_hw_tlb_command_range(&stream, op, + false, 0, 0); + goto next; + } + + /* Nothing of Linux reaches the hardware without the root. */ + bitmap_zero(fw_slots, entries); + spin_lock_irqsave(&dart->lock, flags); + live = dart->locked_ttbr[sid][0]; + fw = dart->locked_fw[sid][0]; + for (i = first; live && i <= last; i++) + if (apple_dart_fw_slot_owned(fw, live, i)) + __set_bit(i, fw_slots); + spin_unlock_irqrestore(&dart->lock, flags); + if (!live) + goto next; + + for (i = find_next_zero_bit(fw_slots, last + 1, first); i <= last; + i = find_next_zero_bit(fw_slots, last + 1, end + 1)) { + u64 start, stop; + + end = find_next_bit(fw_slots, last + 1, i) - 1; + start = max(lo, base + ((u64)i << slot_shift)); + stop = min(hi, base + ((u64)(end + 1) << slot_shift) - 1); + ret = apple_dart_t8110_hw_tlb_command_range(&stream, op, true, + start, stop); + if (ret) + break; + } +next: + __clear_bit(sid, stream.sidmap); + if (ret) + return ret; + } + return 0; +} + static int apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) { @@ -1026,6 +1113,8 @@ apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) static int apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) { + if (stream_map->dart->fw_mirror && stream_map->dart->version >= 0x0202) + return apple_dart_fw_invalidate(stream_map, 0, U64_MAX); return apple_dart_t8110_hw_tlb_command_range( stream_map, DART_T8110_TLB_CMD_OP_FLUSH_SID, false, 0, 0); } @@ -1230,7 +1319,10 @@ static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, "failed to publish host page-table entries: %d\n", ret); else { /* The range registers take the DVA the device issues. */ - if (range && stream.dart->locked && stream.dart->version >= 0x0202) + if (range && stream.dart->fw_mirror && stream.dart->version >= 0x0202) + ret = apple_dart_fw_invalidate(&stream, first + domain->dma_offset, + last + domain->dma_offset); + else if (range && stream.dart->locked && stream.dart->version >= 0x0202) ret = apple_dart_t8110_hw_tlb_command_range(&stream, DART_T8110_TLB_CMD_OP_FLUSH_SID, true, first + domain->dma_offset, From 84f29de3ff143afe1ceac422931b90b0eaec8825 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 17:10:58 +1100 Subject: [PATCH 0238/2680] iommu: apple-dart: invalidate client-ranged T8140 streams again On T8140 the AOP DART has no apple,dma-range of its own. Its clients carry their own windows (apple,dma-range on the AOP, its setup mailbox, the audio node and the base-ns and leap-ns ADMACs, all at or above 1 TiB), and the leap-s ADMAC gets its IOVAs from the top of the DART's 42-bit aperture. apple_dart_fw_invalidate() takes its window from the DART node instead, so it works out [0, 64 GiB), finds every client DVA outside that window and returns before it looks at any stream. That includes the four-level streams (TCR 0x9 on every AOP DART stream Linux uses), which should get a whole-stream invalidate. So no unmap on this DART invalidates the TLB. The IOMMU core then hands the freed IOVA to the next buffer, and the DART still translates it to the pages that were just freed. A capture loop on the high-quality microphone shows it: every new stream's DMA landed in the first, long-freed buffer, never in its own buffer. Once those pages are back in the allocator: - the speaker sense capture zeroes whatever reuses them (struct file and fdtable objects, btrfs tree blocks); - the speaker DMA plays freed memory at full scale. The volume limit only applies to the new buffer, which the DMA no longer reads. Decide per stream instead: - a four-level stream gets the whole-stream invalidate first; - a three-level stream is invalidated by range inside the window of the domain attached to it. The driver records that window when the domain imports the stream's root and when it attaches. Streams whose domain uses the DART's own window, such as the display and DCP streams, keep the window they had before. Fixes: dd5ac6fdffb0 ("iommu: apple-dart: leave live firmware translations out of invalidates") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 50 +++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 7bca8b7234be0..c698e47a4e5bf 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -311,6 +311,12 @@ struct apple_dart { u64 *locked_owned[DART_MAX_STREAMS][DART_MAX_TTBR]; /* Leaf tables behind the root slots firmware held when we attached. */ struct apple_dart_fw_root *locked_fw[DART_MAX_STREAMS][DART_MAX_TTBR]; + /* + * First DVA of the window of the domain using each locked stream. The + * window can be the client's own (apple,dma-range on the client node) + * rather than the DART's. + */ + u64 locked_window[DART_MAX_STREAMS]; }; /* @@ -1046,22 +1052,18 @@ static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, size_t entries = dart->pgsize / sizeof(u64); u32 slot_shift = ilog2(dart->pgsize) + ilog2(entries); u64 span = BIT_ULL(slot_shift + ilog2(entries)); - u64 base = (dart->dma_min + dart->dma_offset) & ~(span - 1); DECLARE_BITMAP(fw_slots, DART_MAX_ROOT_ENTRIES); size_t first, last, i, end; unsigned long flags; int sid, ret = 0; - if (entries > DART_MAX_ROOT_ENTRIES || hi < base || lo > base + span - 1) + if (entries > DART_MAX_ROOT_ENTRIES) return 0; - lo = max(lo, base); - hi = min(hi, base + span - 1); - first = (lo - base) >> slot_shift; - last = (hi - base) >> slot_shift; for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { const struct apple_dart_fw_root *fw; const u64 *live; + u64 base, start_dva, end_dva; __set_bit(sid, stream.sidmap); if (apple_dart_readl(dart, DART_TCR(dart, sid)) & @@ -1071,6 +1073,19 @@ static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, goto next; } + /* + * A three-level stream walks the low bits of the DVA; the TLB is + * tagged with the whole DVA, inside the window of the stream's + * domain. + */ + base = READ_ONCE(dart->locked_window[sid]) & ~(span - 1); + if (hi < base || lo > base + span - 1) + goto next; + start_dva = max(lo, base); + end_dva = min(hi, base + span - 1); + first = (start_dva - base) >> slot_shift; + last = (end_dva - base) >> slot_shift; + /* Nothing of Linux reaches the hardware without the root. */ bitmap_zero(fw_slots, entries); spin_lock_irqsave(&dart->lock, flags); @@ -1088,8 +1103,8 @@ static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, u64 start, stop; end = find_next_bit(fw_slots, last + 1, i) - 1; - start = max(lo, base + ((u64)i << slot_shift)); - stop = min(hi, base + ((u64)(end + 1) << slot_shift) - 1); + start = max(start_dva, base + ((u64)i << slot_shift)); + stop = min(end_dva, base + ((u64)(end + 1) << slot_shift) - 1); ret = apple_dart_t8110_hw_tlb_command_range(&stream, op, true, start, stop); if (ret) @@ -1566,6 +1581,18 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, stream_map->dart->hw->invalidate_tlb(stream_map); } +/* Record the DVA window of @domain for invalidates of @stream_map's streams. */ +static void apple_dart_set_locked_window(struct apple_dart_domain *domain, + struct apple_dart_stream_map *stream_map) +{ + struct apple_dart *dart = stream_map->dart; + u64 start = domain->domain.geometry.aperture_start + domain->dma_offset; + int sid; + + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) + WRITE_ONCE(dart->locked_window[sid], start); +} + static int apple_dart_setup_translation_locked(struct apple_dart_domain *domain, struct apple_dart_stream_map *stream_map) @@ -1573,6 +1600,7 @@ apple_dart_setup_translation_locked(struct apple_dart_domain *domain, struct io_pgtable_cfg *cfg = &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; int i, ret; + apple_dart_set_locked_window(domain, stream_map); for (i = 0; i < cfg->apple_dart_cfg.n_ttbrs; i++) { ret = apple_dart_hw_map_locked_ttbr(stream_map, i); if (ret) @@ -1707,6 +1735,7 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, for_each_stream_map(i, cfg, stream) { if (!stream->dart->locked || !stream->dart->fw_mirror) continue; + apple_dart_set_locked_window(dart_domain, stream); for (index = 0; index < pgtbl_cfg.apple_dart_cfg.n_ttbrs; index++) { ret = apple_dart_hw_map_locked_ttbr(stream, index); if (ret) { @@ -2566,7 +2595,7 @@ apple_dart_platform_ioremap_np_resource(struct platform_device *pdev, static int apple_dart_probe(struct platform_device *pdev) { - int ret; + int ret, sid; u32 dart_params[4]; struct device_node *pd_np; struct resource *res; @@ -2798,6 +2827,9 @@ static int apple_dart_probe(struct platform_device *pdev) &dart->dma_min, &dart->dma_max, &dart->dma_offset); } + for (sid = 0; sid < DART_MAX_STREAMS; sid++) + dart->locked_window[sid] = dart->dma_min + dart->dma_offset; + if (dart->num_streams > DART_MAX_STREAMS) { dev_err(&pdev->dev, "Too many streams (%d > %d)\n", dart->num_streams, DART_MAX_STREAMS); From d359f324274bd4fa0b3becca4c9a547857baf981 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 20:57:31 +1100 Subject: [PATCH 0239/2680] iommu: apple-dart: invalidate four-level T8140 streams by DVA range Since commit 9c512b25f55e ("iommu: apple-dart: invalidate client-ranged T8140 streams again"), every map and unmap on a four-level stream of a DART that keeps firmware roots flushes the whole stream. Every stream Linux uses on the T8140 AOP DART is four-level, so each audio buffer that is allocated or freed flushes its ADMAC's stream whole. That has two problems: - On streams that also hold the AOP firmware's own root slots (stream 0 slot 16, stream 1 slot 33, stream 8 slot 16), it drops translations the firmware set up and may be using. apple_dart_fw_invalidate() exists precisely to leave those alone; only three-level streams got that treatment. - On the MacBook Neo, a capture open/close loop on the microphone array and the headset microphone once left such a flush busy for good: apple-dart 310fc0000.iommu: TLB command 0x1 for stream 11 timed out (0x8700810b) Every later TLB command found the engine busy, buffer mappings failed with -ENOMEM, and the AOP rejected every audio run request until reboot. A whole-stream flush in flight reads 0x8400xxxx and a finished one 0x0400xxxx; the range-invalidate bits 24-25 that the stuck command also showed never appear in a whole-stream flush otherwise. Invalidate four-level streams the way three-level streams already are: by DVA range, through root slots that are not the firmware's. A four-level root translates the whole input address space from DVA 0, so each root slot covers 2^(page shift + 2 * table bits) bytes. Four-level roots are not snapshotted, so a slot is the firmware's when it is valid and does not hold the entry Linux published. Three-level streams, and with them the display and DCP streams, are unchanged. Fixes: 9c512b25f55e ("iommu: apple-dart: invalidate client-ranged T8140 streams again") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 68 ++++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 21 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index c698e47a4e5bf..de3a5fbd3ed7e 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -1034,14 +1034,30 @@ apple_dart_t8110_hw_tlb_command_range(struct apple_dart_stream_map *stream_map, /* A root table is at most 16 KiB of 64-bit entries. */ #define DART_MAX_ROOT_ENTRIES (SZ_16K / sizeof(u64)) +/* + * A slot of a four-level locked root that firmware holds: valid, and not an + * entry Linux published. The loader's four-level roots are not snapshotted, + * so this is the only firmware state Linux knows about them. + */ +static bool apple_dart_fw_slot_foreign(const u64 *live, const u64 *owned, + size_t i) +{ + u64 pte = READ_ONCE(live[i]); + + return (pte & APPLE_DART_PTE_VALID) && (!owned || pte != READ_ONCE(owned[i])); +} + /* * On DARTs that keep firmware roots (T8140 and T8142), the hardware keeps * using the translations behind the firmware-owned root slots, e.g. the - * display controller scanning out the boot framebuffer while Linux attaches. + * display controller scanning out the boot framebuffer while Linux attaches, + * or the AOP firmware's own DMA on the AOP DART streams that Linux shares. * Invalidating those translations, for the whole stream or by DVA range, can - * make the scanout fail a translation, and the display coprocessor panics on - * the failed read. Linux never changes what those slots resolve to, so only - * invalidate the parts of [lo, hi] reached through other root slots. + * break that DMA: the scanout fails a translation and the display + * coprocessor panics on the failed read, and a whole-stream invalidate on a + * busy AOP stream can stay busy for good. Linux never changes what those + * slots resolve to, so only invalidate the parts of [lo, hi] reached through + * other root slots, by DVA range, on three-level and four-level streams alike. */ static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, u64 lo, u64 hi) @@ -1050,8 +1066,7 @@ static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, struct apple_dart *dart = stream_map->dart; struct apple_dart_stream_map stream = { .dart = dart }; size_t entries = dart->pgsize / sizeof(u64); - u32 slot_shift = ilog2(dart->pgsize) + ilog2(entries); - u64 span = BIT_ULL(slot_shift + ilog2(entries)); + u32 table_bits = ilog2(entries); DECLARE_BITMAP(fw_slots, DART_MAX_ROOT_ENTRIES); size_t first, last, i, end; unsigned long flags; @@ -1062,23 +1077,32 @@ static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { const struct apple_dart_fw_root *fw; - const u64 *live; - u64 base, start_dva, end_dva; + const u64 *live, *owned; + u64 base, span, start_dva, end_dva; + bool four_level; + u32 slot_shift; __set_bit(sid, stream.sidmap); - if (apple_dart_readl(dart, DART_TCR(dart, sid)) & - dart->hw->tcr_4level) { - ret = apple_dart_t8110_hw_tlb_command_range(&stream, op, - false, 0, 0); - goto next; + four_level = apple_dart_readl(dart, DART_TCR(dart, sid)) & + dart->hw->tcr_4level; + if (four_level) { + /* + * A four-level stream walks the whole DVA: its root slots + * cover the input address space from zero. + */ + slot_shift = ilog2(dart->pgsize) + 2 * table_bits; + span = BIT_ULL(min(slot_shift + table_bits, dart->ias)); + base = 0; + } else { + /* + * A three-level stream walks the low bits of the DVA; the + * TLB is tagged with the whole DVA, inside the window of + * the stream's domain. + */ + slot_shift = ilog2(dart->pgsize) + table_bits; + span = BIT_ULL(slot_shift + table_bits); + base = READ_ONCE(dart->locked_window[sid]) & ~(span - 1); } - - /* - * A three-level stream walks the low bits of the DVA; the TLB is - * tagged with the whole DVA, inside the window of the stream's - * domain. - */ - base = READ_ONCE(dart->locked_window[sid]) & ~(span - 1); if (hi < base || lo > base + span - 1) goto next; start_dva = max(lo, base); @@ -1090,9 +1114,11 @@ static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, bitmap_zero(fw_slots, entries); spin_lock_irqsave(&dart->lock, flags); live = dart->locked_ttbr[sid][0]; + owned = dart->locked_owned[sid][0]; fw = dart->locked_fw[sid][0]; for (i = first; live && i <= last; i++) - if (apple_dart_fw_slot_owned(fw, live, i)) + if (four_level ? apple_dart_fw_slot_foreign(live, owned, i) : + apple_dart_fw_slot_owned(fw, live, i)) __set_bit(i, fw_slots); spin_unlock_irqrestore(&dart->lock, flags); if (!live) From 02566238f24c92aa7d654cfaac7b0c4c5c65d82a Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 18:48:20 -0400 Subject: [PATCH 0240/2680] dt-bindings: apple: describe J813 monitoring and input peripherals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe the T8142 mailbox, SMC, MTP, DART and PWM resources. Add explicit contracts for runtime temperature discovery and firmware-owned trackpad reset without embedding sensor tables or calibration. [ryan: adapted to 7.2.10 — split by topic: only the apple,dart.yaml hunk is carried in 020-dart; the other binding hunks are in 010-soc, 060-power-thermal and 090-input] Signed-off-by: Acelogic --- Documentation/devicetree/bindings/iommu/apple,dart.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index bd6524c97087f..843ccb6fdddc4 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -38,6 +38,7 @@ properties: - apple,t6020-dart - apple,t8122-dart - apple,t8140-dart + - apple,t8142-dart - const: apple,t8110-dart reg: From b722d94e716eb31773f30a2cb6f1375c6aecfaef Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:49:11 +1100 Subject: [PATCH 0241/2680] iommu: apple-dart: Fix the KUnit test after deferred root publication apple_dart_hw_sync_locked() gained a defer_unknown argument when the early TLB sync learnt to skip locked streams whose roots are not mapped yet, but the KUnit test kept calling it with two arguments, so CONFIG_APPLE_DART_KUNIT_TEST no longer builds. The before-attach case also still expected that early sync to fail with -EIO, which is exactly what the change removed. Pass false where the test exercises the attach-time publication, expect the early sync to succeed, and add a case that checks both sides of the new argument: the early sync leaves a stream with a half-known root alone, while the attach-time sync still refuses it. With this the whole suite passes again under QEMU. Fixes: c361702757bf ("iommu/apple-dart: defer publication until locked roots are known") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart-test.c | 49 ++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 0eba2eca17474..86ed79c8d0c1d 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -47,9 +47,12 @@ static void apple_dart_test_before_attach(struct kunit *test) KUNIT_EXPECT_EQ(test, atomic_long_read(&domain->stream_maps[0].sidmap[0]), 0L); KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8020_STREAM_COMMAND), 0U); - /* Once attached, missing root mappings must still report an error. */ + /* + * A stream whose locked roots are not mapped yet is left to attach, + * which publishes the complete page table. + */ KUNIT_EXPECT_EQ(test, apple_dart_domain_add_streams(domain, &master), 0); - KUNIT_EXPECT_EQ(test, apple_dart_iotlb_sync_map(&domain->domain, 0, PAGE_SIZE), -EIO); + KUNIT_EXPECT_EQ(test, apple_dart_iotlb_sync_map(&domain->domain, 0, PAGE_SIZE), 0); pm_runtime_disable(dart->dev); free_io_pgtable_ops(ops); @@ -78,7 +81,7 @@ static void apple_dart_test_locked_handoff(struct kunit *test) cfg.apple_dart_cfg.ttbr[0] = ours; /* Legacy firmware mappings are rebuilt by the IOMMU core at handoff. */ - KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream), 0); + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream, false), 0); KUNIT_EXPECT_MEMEQ(test, live, ours, sizeof(live)); KUNIT_EXPECT_MEMEQ(test, owned, ours, sizeof(owned)); apple_dart_retire_root(live, owned, ARRAY_SIZE(live)); @@ -89,10 +92,41 @@ static void apple_dart_test_locked_handoff(struct kunit *test) dart->hw = &apple_dart_hw_t8110; dart->version = 0x0200; live[0] = 0x1001; - KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream), 0); + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream, false), 0); KUNIT_EXPECT_MEMEQ(test, live, ours, sizeof(live)); } +static void apple_dart_test_sync_unknown_roots(struct kunit *test) +{ + struct apple_dart_stream_map stream = {}; + struct io_pgtable_cfg cfg = {}; + struct apple_dart *dart; + u64 live[] = { 0x1001, 0, 0, 0 }; + u64 ours[] = { 0, 0x2001, 0, 0 }; + + dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart); + dart->hw = &apple_dart_hw_t8110; + dart->version = 0x0202; + dart->num_streams = 1; + dart->pgsize = sizeof(live); + spin_lock_init(&dart->lock); + /* The live root is mapped, its ownership shadow is not yet. */ + dart->locked_ttbr[0][0] = live; + stream.dart = dart; + __set_bit(0, stream.sidmap); + cfg.apple_dart_cfg.n_ttbrs = 1; + cfg.apple_dart_cfg.ttbr[0] = ours; + + /* An early TLB sync skips the stream and leaves the root alone... */ + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream, true), 0); + KUNIT_EXPECT_EQ(test, live[0], 0x1001ULL); + KUNIT_EXPECT_EQ(test, live[1], 0ULL); + /* ...but publishing at attach still requires both. */ + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream, false), -EIO); + KUNIT_EXPECT_EQ(test, live[1], 0ULL); +} + static void apple_dart_test_firmware_roots(struct kunit *test) { struct apple_dart_stream_map stream = {}; @@ -116,13 +150,13 @@ static void apple_dart_test_firmware_roots(struct kunit *test) cfg.apple_dart_cfg.n_ttbrs = 1; cfg.apple_dart_cfg.ttbr[0] = ours; - KUNIT_ASSERT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream), 0); + KUNIT_ASSERT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream, false), 0); KUNIT_EXPECT_EQ(test, live[0], 0x1001ULL); KUNIT_EXPECT_EQ(test, live[1], 0x2001ULL); ours[0] = 0x3001; - KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream), -EBUSY); + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream, false), -EBUSY); dart->version = 0x0203; - KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream), -EBUSY); + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream, false), -EBUSY); KUNIT_EXPECT_EQ(test, live[0], 0x1001ULL); /* Teardown must preserve a root which firmware has since replaced. */ live[1] = 0x4001; @@ -377,6 +411,7 @@ static void apple_dart_test_gated_irq(struct kunit *test) static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_before_attach), KUNIT_CASE(apple_dart_test_locked_handoff), + KUNIT_CASE(apple_dart_test_sync_unknown_roots), KUNIT_CASE(apple_dart_test_firmware_roots), KUNIT_CASE(apple_dart_test_gated_commands), KUNIT_CASE(apple_dart_test_resume_commands_failure), From 3ae628442b52ad3dea70e1c4a4c3ce14d3e77bef Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 17:56:16 +1100 Subject: [PATCH 0242/2680] iommu: apple-dart: Adapt the KUnit tests to the 7.2 DART changes Two tests stopped matching the driver they test: - apple_dart_test_before_attach builds its master configuration by hand. The DMA window now lives in the master configuration, which of_xlate() starts from the DART's window, so a configuration without it describes an empty window and finalization rejects it with -EINVAL. Fill it in the way of_xlate() does. - apple_dart_test_gated_commands drives the T8110 TLB command path on a DART without hardware description. That path now asks whether the DART remaps its streams, which needs the description, and the test oopses. Describe the DART as the T8110 it exercises. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart-test.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 86ed79c8d0c1d..3280fb0435c6c 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -31,6 +31,9 @@ static void apple_dart_test_before_attach(struct kunit *test) mutex_init(&domain->init_lock); master.stream_maps[0].dart = dart; __set_bit(0, master.stream_maps[0].sidmap); + /* of_xlate() starts each master with the window of its DART. */ + master.dma_min = dart->dma_min; + master.dma_max = dart->dma_max; writel(dart->hw->ttbr_valid, dart->regs + DART_TTBR(dart, 0, 0)); KUNIT_ASSERT_EQ(test, apple_dart_finalize_domain(domain, &master), 0); ops = domain->pgtbl_ops; @@ -179,6 +182,7 @@ static void apple_dart_test_gated_commands(struct kunit *test) dev_set_drvdata(dart->dev, dart); dart->regs = (__force void __iomem *)kunit_kzalloc(test, SZ_16K, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, dart->regs); + dart->hw = &apple_dart_hw_t8110; dart->num_streams = 1; dart->pgsize = SZ_16K; dart->version = 0x0202; From 03f308dc918cbd5c5c07d96a5ae247941348b5f8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 25 Aug 2026 14:35:44 -0400 Subject: [PATCH 0243/2680] arm64: dts: apple: add T8103 and T8112 PCIe-C tunnels Signed-off-by: DjDeveloperr --- arch/arm64/boot/dts/apple/t8103.dtsi | 136 +++++++++++++++++++++++++++ arch/arm64/boot/dts/apple/t8112.dtsi | 136 +++++++++++++++++++++++++++ 2 files changed, 272 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index e3a91f8931073..d99cb8db4c6d1 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -1583,6 +1583,7 @@ <&ps_atc0_cio_usb>; thunderbolt-switch; + apple,pcie-tunnel = <&usb4_0_pcie_tunnel>; pinctrl-0 = <&acio0_pins>; pinctrl-names = "default"; @@ -1645,6 +1646,73 @@ }; }; + usb4_0_pcie_tunnel: usb4-pcie-tunnel-0 { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + apciec0_dart: iommu@3a1008000 { + compatible = "apple,t8103-dart"; + reg = <0x3 0xa1008000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + }; + + apciec0: pcie@3b0000000 { + compatible = "apple,t8103-pciec"; + device_type = "pci"; + + reg = <0x3 0xb0000000 0x0 0x10000000>, + <0x3 0xa0000000 0x0 0x4000>, + <0x3 0xa1000000 0x0 0x8000>, + <0x3 0xa0004000 0x0 0x4000>, + <0x3 0xa0200000 0x0 0x4000>, + <0x3 0xa1010000 0x0 0x4000>, + <0x3 0xa100c000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "common", + "fabric", "debug", "intr2axi"; + + interrupt-parent = <&aic>; + interrupts = ; + + msi-controller; + msi-parent = <&apciec0>; + msi-ranges = <&aic AIC_IRQ 738 IRQ_TYPE_EDGE_RISING 32>; + + iommu-map = <0x0 &apciec0_dart 0x0 0x40>; + iommu-map-mask = <0x3f>; + + bus-range = <0x0 0x7f>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0x4 0x0 + 0x4 0x0 0x0 0x80000000>, + <0x02000000 0x0 0x80000000 + 0x4 0x80000000 0x0 0x40000000>, + <0x42000000 0x0 0xc0000000 + 0x4 0xc0000000 0x0 0x40000000>; + + apciec0_port: pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + external-facing; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &apciec0_port 0 0 0 0>, + <0 0 0 2 &apciec0_port 0 0 0 1>, + <0 0 0 3 &apciec0_port 0 0 0 2>, + <0 0 0 4 &apciec0_port 0 0 0 3>; + }; + }; + }; + dwc3_0: usb@382280000 { compatible = "apple,t8103-dwc3"; reg = <0x3 0x82280000 0x0 0xcd00>, <0x3 0x8228cd00 0x0 0x3200>; @@ -1732,6 +1800,7 @@ <&ps_atc1_cio_usb>; thunderbolt-switch; + apple,pcie-tunnel = <&usb4_1_pcie_tunnel>; pinctrl-0 = <&acio1_pins>; pinctrl-names = "default"; @@ -1794,6 +1863,73 @@ }; }; + usb4_1_pcie_tunnel: usb4-pcie-tunnel-1 { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + apciec1_dart: iommu@521008000 { + compatible = "apple,t8103-dart"; + reg = <0x5 0x21008000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + }; + + apciec1: pcie@530000000 { + compatible = "apple,t8103-pciec"; + device_type = "pci"; + + reg = <0x5 0x30000000 0x0 0x10000000>, + <0x5 0x20000000 0x0 0x4000>, + <0x5 0x21000000 0x0 0x8000>, + <0x5 0x20004000 0x0 0x4000>, + <0x5 0x20200000 0x0 0x4000>, + <0x5 0x21010000 0x0 0x4000>, + <0x5 0x2100c000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "common", + "fabric", "debug", "intr2axi"; + + interrupt-parent = <&aic>; + interrupts = ; + + msi-controller; + msi-parent = <&apciec1>; + msi-ranges = <&aic AIC_IRQ 818 IRQ_TYPE_EDGE_RISING 32>; + + iommu-map = <0x0 &apciec1_dart 0x0 0x40>; + iommu-map-mask = <0x3f>; + + bus-range = <0x0 0x7f>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0x5 0x80000000 + 0x5 0x80000000 0x0 0x80000000>, + <0x02000000 0x0 0x00100000 + 0x6 0x00100000 0x0 0x3ff00000>, + <0x42000000 0x0 0x40000000 + 0x6 0x40000000 0x0 0x40000000>; + + apciec1_port: pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + external-facing; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &apciec1_port 0 0 0 0>, + <0 0 0 2 &apciec1_port 0 0 0 1>, + <0 0 0 3 &apciec1_port 0 0 0 2>, + <0 0 0 4 &apciec1_port 0 0 0 3>; + }; + }; + }; + dwc3_1: usb@502280000 { compatible = "apple,t8103-dwc3"; reg = <0x5 0x02280000 0x0 0xcd00>, <0x5 0x0228cd00 0x0 0x3200>; diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 8ae55040ab18e..18f75c32e266d 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -1848,6 +1848,7 @@ <&ps_atc0_cio_usb>; thunderbolt-switch; + apple,pcie-tunnel = <&usb4_0_pcie_tunnel>; pinctrl-0 = <&acio0_pins>; pinctrl-names = "default"; @@ -1911,6 +1912,73 @@ }; }; + usb4_0_pcie_tunnel: usb4-pcie-tunnel-0 { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + apciec0_dart: iommu@3a1008000 { + compatible = "apple,t8110-dart"; + reg = <0x3 0xa1008000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + }; + + apciec0: pcie@3b0000000 { + compatible = "apple,t8112-pciec", "apple,t8103-pciec"; + device_type = "pci"; + + reg = <0x3 0xb0000000 0x0 0x10000000>, + <0x3 0xa0000000 0x0 0x4000>, + <0x3 0xa1000000 0x0 0x8000>, + <0x3 0xa0004000 0x0 0x4000>, + <0x3 0xa0200000 0x0 0x4000>, + <0x3 0xa1010000 0x0 0x4000>, + <0x3 0xa100c000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "common", + "fabric", "debug", "intr2axi"; + + interrupt-parent = <&aic>; + interrupts = ; + + msi-controller; + msi-parent = <&apciec0>; + msi-ranges = <&aic AIC_IRQ 992 IRQ_TYPE_EDGE_RISING 32>; + + iommu-map = <0x0 &apciec0_dart 0x0 0x40>; + iommu-map-mask = <0x3f>; + + bus-range = <0x0 0x7f>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0x4 0x0 + 0x4 0x0 0x0 0x80000000>, + <0x02000000 0x0 0x80000000 + 0x4 0x80000000 0x0 0x40000000>, + <0x42000000 0x0 0xc0000000 + 0x4 0xc0000000 0x0 0x40000000>; + + apciec0_port: pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + external-facing; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &apciec0_port 0 0 0 0>, + <0 0 0 2 &apciec0_port 0 0 0 1>, + <0 0 0 3 &apciec0_port 0 0 0 2>, + <0 0 0 4 &apciec0_port 0 0 0 3>; + }; + }; + }; + dwc3_0: usb@382280000 { compatible = "apple,t8112-dwc3", "apple,t8103-dwc3"; reg = <0x3 0x82280000 0x0 0xcd00>, <0x3 0x8228cd00 0x0 0x3200>; @@ -1998,6 +2066,7 @@ <&ps_atc1_cio_usb>; thunderbolt-switch; + apple,pcie-tunnel = <&usb4_1_pcie_tunnel>; pinctrl-0 = <&acio1_pins>; pinctrl-names = "default"; @@ -2061,6 +2130,73 @@ }; }; + usb4_1_pcie_tunnel: usb4-pcie-tunnel-1 { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + apciec1_dart: iommu@521008000 { + compatible = "apple,t8110-dart"; + reg = <0x5 0x21008000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + }; + + apciec1: pcie@530000000 { + compatible = "apple,t8112-pciec", "apple,t8103-pciec"; + device_type = "pci"; + + reg = <0x5 0x30000000 0x0 0x10000000>, + <0x5 0x20000000 0x0 0x4000>, + <0x5 0x21000000 0x0 0x8000>, + <0x5 0x20004000 0x0 0x4000>, + <0x5 0x20200000 0x0 0x4000>, + <0x5 0x21010000 0x0 0x4000>, + <0x5 0x2100c000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "common", + "fabric", "debug", "intr2axi"; + + interrupt-parent = <&aic>; + interrupts = ; + + msi-controller; + msi-parent = <&apciec1>; + msi-ranges = <&aic AIC_IRQ 1073 IRQ_TYPE_EDGE_RISING 32>; + + iommu-map = <0x0 &apciec1_dart 0x0 0x40>; + iommu-map-mask = <0x3f>; + + bus-range = <0x0 0x7f>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0x5 0x80000000 + 0x5 0x80000000 0x0 0x80000000>, + <0x02000000 0x0 0x00100000 + 0x6 0x00100000 0x0 0x3ff00000>, + <0x42000000 0x0 0x40000000 + 0x6 0x40000000 0x0 0x40000000>; + + apciec1_port: pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + external-facing; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &apciec1_port 0 0 0 0>, + <0 0 0 2 &apciec1_port 0 0 0 1>, + <0 0 0 3 &apciec1_port 0 0 0 2>, + <0 0 0 4 &apciec1_port 0 0 0 3>; + }; + }; + }; + dwc3_1: usb@502280000 { compatible = "apple,t8112-dwc3", "apple,t8103-dwc3"; reg = <0x5 0x02280000 0x0 0xcd00>, <0x5 0x0228cd00 0x0 0x3200>; From cc79ece100ae26a624ac2013f21bceaa307bb0c6 Mon Sep 17 00:00:00 2001 From: Sasha Finkelstein Date: Sat, 9 Nov 2024 17:55:47 +0100 Subject: [PATCH 0244/2680] soc: apple: rtkit: Add tracekit endpoint. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This system endpoint is advertised by AOP and also needs to be turned on for it to function. [ryan: adapted to 7.2.10 — Asahi 7.2 no longer carries this commit; the J700 AOP (audio, ALS, HQ mic) has always been brought up with TRACEKIT started, so it is carried to keep 7.1 behaviour] Signed-off-by: Sasha Finkelstein --- drivers/soc/apple/rtkit.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index a863ae1668a76..ef0c399ea8523 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -22,6 +22,7 @@ enum { APPLE_RTKIT_EP_DEBUG = 3, APPLE_RTKIT_EP_IOREPORT = 4, APPLE_RTKIT_EP_OSLOG = 8, + APPLE_RTKIT_EP_TRACEKIT = 0xa, }; #define APPLE_RTKIT_MGMT_TYPE GENMASK_ULL(59, 52) @@ -191,6 +192,7 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) case APPLE_RTKIT_EP_DEBUG: case APPLE_RTKIT_EP_IOREPORT: case APPLE_RTKIT_EP_OSLOG: + case APPLE_RTKIT_EP_TRACEKIT: dev_dbg(rtk->dev, "RTKit: Starting system endpoint 0x%02x\n", ep); apple_rtkit_start_ep(rtk, ep); From 25e5e1e085ec5872a4ed27a61d4422da7a86decf Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 23:22:43 -0400 Subject: [PATCH 0245/2680] tty: samsung: avoid suspended Apple UART access Signed-off-by: DjDeveloperr --- drivers/tty/serial/samsung_tty.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/drivers/tty/serial/samsung_tty.c b/drivers/tty/serial/samsung_tty.c index d33a5d848659e..de1741d8d019c 100644 --- a/drivers/tty/serial/samsung_tty.c +++ b/drivers/tty/serial/samsung_tty.c @@ -132,6 +132,7 @@ struct s3c24xx_uart_dma { struct s3c24xx_uart_port { unsigned char rx_enabled; unsigned char tx_enabled; + bool system_sleep_skipped; unsigned int pm_level; unsigned long baudclk_rate; unsigned int min_dma_size; @@ -2091,6 +2092,16 @@ static void s3c24xx_serial_remove(struct platform_device *dev) static int __maybe_unused s3c24xx_serial_suspend(struct device *dev) { struct uart_port *port = s3c24xx_dev_to_port(dev); + struct s3c24xx_uart_port *ourport = to_ourport(port); + + /* + * Runtime PM has already quiesced the port and removed its clocks. Do + * not run the serial-core system-sleep path against an Apple S5L UART + * whose PMGR domain is therefore inaccessible. + */ + ourport->system_sleep_skipped = pm_runtime_suspended(dev); + if (ourport->system_sleep_skipped) + return 0; if (!console_suspend_enabled && uart_console(port)) device_set_wakeup_path(dev); @@ -2106,6 +2117,11 @@ static int __maybe_unused s3c24xx_serial_resume(struct device *dev) struct uart_port *port = s3c24xx_dev_to_port(dev); struct s3c24xx_uart_port *ourport = to_ourport(port); + if (ourport->system_sleep_skipped) { + ourport->system_sleep_skipped = false; + return 0; + } + if (port) { clk_prepare_enable(ourport->clk); if (!IS_ERR(ourport->baudclk)) @@ -2126,6 +2142,9 @@ static int __maybe_unused s3c24xx_serial_resume_noirq(struct device *dev) struct uart_port *port = s3c24xx_dev_to_port(dev); struct s3c24xx_uart_port *ourport = to_ourport(port); + if (ourport->system_sleep_skipped) + return 0; + if (port) { /* restore IRQ mask */ switch (ourport->info->type) { From 6d17edb1b2e38c5fce10264bff65b928890f2ca8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 25 Sep 2026 22:41:52 +1000 Subject: [PATCH 0246/2680] dt-bindings: iommu: apple,sart: Add T8140 compatible The T8140 SoC places a SART in front of its ANS2 NVMe controller that is register compatible with the T6000 generation. Add the SoC-specific compatible with the existing "apple,t6000-sart" fallback so the T8140 device tree can describe it. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/iommu/apple,sart.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/iommu/apple,sart.yaml b/Documentation/devicetree/bindings/iommu/apple,sart.yaml index 8fd7775704ab7..472454c3d24d6 100644 --- a/Documentation/devicetree/bindings/iommu/apple,sart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,sart.yaml @@ -34,6 +34,7 @@ properties: - apple,t6020-sart - apple,t8112-sart - apple,t8122-sart + - apple,t8140-sart - const: apple,t6000-sart - enum: - apple,t6000-sart From d6ee70eadc5181b89e9f236adaa6241674cc3f31 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 25 Sep 2026 22:42:17 +1000 Subject: [PATCH 0247/2680] soc: apple: rtkit: Harden syslog buffer handling Three problems in the syslog receive path can be triggered by the co-processor: - SYSLOG_LOG accepted idx == syslog_n_entries, one past the last entry, and never checked that the entry lies inside the shared buffer, so a bad index read past the end of the ring. - SYSLOG_INIT unconditionally allocated a new message buffer, leaking the previous one when the firmware re-sent INIT after a restart, and a zero message size produced a ZERO_SIZE_PTR that the log path then indexed through strnlen(buf, -1). - The warnings for malformed records were not ratelimited, so a misbehaving firmware could flood the log at interrupt-work rate. Validate the index against both the announced entry count and the buffer size, replace the message buffer on INIT (the ordered RX worker is its only user) and skip the allocation for an empty geometry, and ratelimit the diagnostics. Fixes: 9bd1d9a0d8bb ("soc: apple: Add RTKit IPC library") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit.c | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index ef0c399ea8523..abbb4150a226d 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -422,14 +422,26 @@ static void apple_rtkit_ioreport_rx(struct apple_rtkit *rtk, u64 msg) static void apple_rtkit_syslog_rx_init(struct apple_rtkit *rtk, u64 msg) { - rtk->syslog_n_entries = FIELD_GET(APPLE_RTKIT_SYSLOG_N_ENTRIES, msg); - rtk->syslog_msg_size = FIELD_GET(APPLE_RTKIT_SYSLOG_MSG_SIZE, msg); + size_t entries = FIELD_GET(APPLE_RTKIT_SYSLOG_N_ENTRIES, msg); + size_t size = FIELD_GET(APPLE_RTKIT_SYSLOG_MSG_SIZE, msg); + char *buffer = NULL; - rtk->syslog_msg_buffer = kzalloc(rtk->syslog_msg_size, GFP_KERNEL); + /* + * Firmware may send SYSLOG_INIT again after it restarts. The ordered RX + * worker is the only user of the message buffer, so it can simply be + * replaced here. A zero-sized geometry must not allocate: kzalloc(0) + * returns ZERO_SIZE_PTR, which the log path would then index. + */ + if (entries && size) + buffer = kzalloc(size, GFP_KERNEL); + kfree(rtk->syslog_msg_buffer); + rtk->syslog_msg_buffer = buffer; + rtk->syslog_n_entries = entries; + rtk->syslog_msg_size = size; dev_dbg(rtk->dev, "RTKit: syslog initialized: entries: %zd, msg_size: %zd\n", - rtk->syslog_n_entries, rtk->syslog_msg_size); + entries, size); } static bool should_crop_syslog_char(char c) @@ -445,26 +457,27 @@ static void apple_rtkit_syslog_rx_log(struct apple_rtkit *rtk, u64 msg) int msglen; if (!rtk->syslog_msg_buffer) { - dev_warn( + dev_warn_ratelimited( rtk->dev, "RTKit: received syslog message but no syslog_msg_buffer\n"); goto done; } if (!rtk->syslog_buffer.size) { - dev_warn( + dev_warn_ratelimited( rtk->dev, "RTKit: received syslog message but syslog_buffer.size is zero\n"); goto done; } if (!rtk->syslog_buffer.buffer && !rtk->syslog_buffer.iomem) { - dev_warn( + dev_warn_ratelimited( rtk->dev, "RTKit: received syslog message but no syslog_buffer.buffer or syslog_buffer.iomem\n"); goto done; } - if (idx > rtk->syslog_n_entries) { - dev_warn(rtk->dev, "RTKit: syslog index %d out of range\n", - idx); + if (idx >= rtk->syslog_n_entries || + idx >= rtk->syslog_buffer.size / entry_size) { + dev_warn_ratelimited(rtk->dev, + "RTKit: syslog index %d out of range\n", idx); goto done; } From 6aba2d82300979137c70405f5a8acb8440915fde Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 25 Sep 2026 22:42:22 +1000 Subject: [PATCH 0248/2680] soc: apple: rtkit: Keep a live shared buffer on a repeated request apple_rtkit_common_rx_get_buffer() wrote the requested geometry straight into the live descriptor and allocated a new buffer even when one was already published to the firmware. A repeated request therefore leaked the old allocation and, worse, retired an IOVA the co-processor may still be using. Decode the request into a local descriptor and commit it only once the allocation or mapping has succeeded. If a buffer is already live, re-send the reply for an identical request and refuse a changed one with -EBUSY: a different geometry requires a confirmed firmware restart through apple_rtkit_reinit(), which frees the buffers first. Reject empty requests before calling shmem_setup. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit.c | 56 +++++++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index abbb4150a226d..746704e2209e1 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -262,43 +262,65 @@ static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, struct apple_rtkit_shmem *buffer, u8 ep, u64 msg) { + struct apple_rtkit_shmem request = {}; u64 reply; int err; /* The different size vs. IOVA shifts look odd but are indeed correct this way */ if (ep == APPLE_RTKIT_EP_OSLOG) { - buffer->size = FIELD_GET(APPLE_RTKIT_OSLOG_SIZE, msg); - buffer->iova = FIELD_GET(APPLE_RTKIT_OSLOG_IOVA, msg) << 12; + request.size = FIELD_GET(APPLE_RTKIT_OSLOG_SIZE, msg); + request.iova = FIELD_GET(APPLE_RTKIT_OSLOG_IOVA, msg) << 12; } else { - buffer->size = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg) << 12; - buffer->iova = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg); + request.size = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg) << 12; + request.iova = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg); + } + if (!request.size) { + err = -EINVAL; + goto error; } - buffer->buffer = NULL; - buffer->iomem = NULL; - buffer->is_mapped = false; + /* + * A repeated request must not retire a buffer the firmware may still be + * using. Re-send the reply for an identical request and refuse a changed + * one; a different geometry needs a confirmed firmware restart (reinit) + * first. + */ + if (buffer->size) { + if (request.size != buffer->size || + (request.iova != buffer->iova && + (request.iova || buffer->is_mapped))) { + dev_err(rtk->dev, + "RTKit: buffer request for 0x%zx bytes at %pad while 0x%zx bytes at %pad are live\n", + request.size, &request.iova, buffer->size, + &buffer->iova); + return -EBUSY; + } + goto reply; + } dev_dbg(rtk->dev, "RTKit: buffer request for 0x%zx bytes at %pad\n", - buffer->size, &buffer->iova); + request.size, &request.iova); - if (buffer->iova && !rtk->ops->shmem_setup) { + if (request.iova && !rtk->ops->shmem_setup) { err = -EINVAL; goto error; } if (rtk->ops->shmem_setup) { - err = rtk->ops->shmem_setup(rtk->cookie, buffer); + err = rtk->ops->shmem_setup(rtk->cookie, &request); if (err) goto error; } else { - buffer->buffer = dma_alloc_coherent(rtk->dev, buffer->size, - &buffer->iova, GFP_KERNEL); - if (!buffer->buffer) { + request.buffer = dma_alloc_coherent(rtk->dev, request.size, + &request.iova, GFP_KERNEL); + if (!request.buffer) { err = -ENOMEM; goto error; } } + *buffer = request; +reply: if (!buffer->is_mapped) { /* oslog uses different fields and needs a shifted IOVA instead of size */ if (ep == APPLE_RTKIT_EP_OSLOG) { @@ -322,13 +344,7 @@ static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, error: dev_err(rtk->dev, "RTKit: failed buffer request for 0x%zx bytes (%d)\n", - buffer->size, err); - - buffer->buffer = NULL; - buffer->iomem = NULL; - buffer->iova = 0; - buffer->size = 0; - buffer->is_mapped = false; + request.size, err); return err; } From 3f821825c6fe683ff326028fffc97f1a2eb72a82 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 25 Sep 2026 22:42:27 +1000 Subject: [PATCH 0249/2680] soc: apple: rtkit: Serialize receiver ownership through teardown apple_rtkit_free() and apple_rtkit_reinit() stopped the mailbox and destroyed or flushed the workqueue, but left the mailbox's rx callback and cookie pointing at the instance the whole time. A message popped by a concurrent apple_rtkit_poll() or a work item queued just before the stop could still reach the consumer's callbacks while its state was being torn down, and apple_rtkit_init() left a dangling receiver behind when the workqueue allocation failed. Make the receiver an explicit ownership: claim the mailbox rx callback under rx_lock once the workqueue exists, detach it (with the interrupt already disabled by apple_mbox_stop()) before draining the queue, drop messages that arrive or were queued once teardown has started, and release the cookie only after all host state is gone. apple_rtkit_reinit() re-arms the receiver under the lock and detaches again if the mailbox fails to start. A mailbox poll with no receiver attached returns without touching the FIFO; the entries are drained once a receiver is attached again. Claiming a mailbox that already has a receiver keeps the previous behaviour of taking it over, now with a warning, so existing consumers that never free their instance can still rebind. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/mailbox.c | 9 +++- drivers/soc/apple/rtkit-internal.h | 1 + drivers/soc/apple/rtkit.c | 87 ++++++++++++++++++++++++++---- 3 files changed, 86 insertions(+), 11 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 330015eea2ebe..f0aa511213189 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -186,8 +186,15 @@ static int apple_mbox_poll_locked(struct apple_mbox *mbox) { struct apple_mbox_msg msg; int ret = 0; + u32 mbox_ctrl; - u32 mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->i2a_control); + /* + * A detached receiver (RTKit teardown in progress) leaves firmware + * traffic in the FIFO; it is drained again once a receiver is attached. + */ + if (!mbox->rx) + return 0; + mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->i2a_control); while (!(mbox_ctrl & mbox->hw->control_empty)) { msg.msg0 = readq_relaxed(mbox->regs + mbox->hw->i2a_recv0); diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index c82065a8bf7b0..d5b45b3865381 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -39,6 +39,7 @@ struct apple_rtkit { unsigned int iop_power_state; unsigned int ap_power_state; bool crashed; + bool shutting_down; DECLARE_BITMAP(endpoints, APPLE_RTKIT_MAX_ENDPOINTS); diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 746704e2209e1..071e0477b5eb1 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -560,6 +560,9 @@ static void apple_rtkit_rx_work(struct work_struct *work) container_of(work, struct apple_rtkit_rx_work, work); struct apple_rtkit *rtk = rtk_work->rtk; + if (READ_ONCE(rtk->shutting_down)) + goto out; + switch (rtk_work->ep) { case APPLE_RTKIT_EP_MGMT: apple_rtkit_management_rx(rtk, rtk_work->msg); @@ -592,6 +595,7 @@ static void apple_rtkit_rx_work(struct work_struct *work) rtk_work->ep, rtk_work->msg); } +out: kfree(rtk_work); } @@ -602,6 +606,9 @@ static void apple_rtkit_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg, struct apple_rtkit_rx_work *work; u8 ep = msg.msg1; + if (READ_ONCE(rtk->shutting_down)) + return; + /* * The message was read from a MMIO FIFO and we have to make * sure all reads from buffers sent with that message happen @@ -692,6 +699,53 @@ int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint) } EXPORT_SYMBOL_GPL(apple_rtkit_start_ep); +static void apple_rtkit_claim_rx(struct apple_rtkit *rtk) +{ + struct apple_mbox *mbox = rtk->mbox; + unsigned long flags; + + spin_lock_irqsave(&mbox->rx_lock, flags); + if (mbox->rx || mbox->cookie) + dev_warn(rtk->dev, + "RTKit: mailbox receiver already claimed, taking it over\n"); + mbox->cookie = rtk; + mbox->rx = apple_rtkit_rx; + spin_unlock_irqrestore(&mbox->rx_lock, flags); +} + +static void apple_rtkit_detach_rx(struct apple_rtkit *rtk) +{ + unsigned long flags; + + WRITE_ONCE(rtk->shutting_down, true); + apple_mbox_stop(rtk->mbox); + /* + * IRQ receive and explicit polling both hold rx_lock. Detach before + * draining the queue, but keep the cookie as an ownership reservation + * until all host/buffer teardown is complete. + */ + spin_lock_irqsave(&rtk->mbox->rx_lock, flags); + if (rtk->mbox->cookie == rtk) + rtk->mbox->rx = NULL; + spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); +} + +static void apple_rtkit_stop_rx(struct apple_rtkit *rtk) +{ + apple_rtkit_detach_rx(rtk); + destroy_workqueue(rtk->wq); +} + +static void apple_rtkit_release_rx(struct apple_rtkit *rtk) +{ + unsigned long flags; + + spin_lock_irqsave(&rtk->mbox->rx_lock, flags); + if (rtk->mbox->cookie == rtk) + rtk->mbox->cookie = NULL; + spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); +} + struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, const struct apple_rtkit_ops *ops) @@ -727,9 +781,6 @@ struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, goto free_rtk; } - rtk->mbox->rx = apple_rtkit_rx; - rtk->mbox->cookie = rtk; - rtk->wq = alloc_ordered_workqueue("rtkit-%s", WQ_HIGHPRI | WQ_MEM_RECLAIM, dev_name(rtk->dev)); if (!rtk->wq) { @@ -737,14 +788,17 @@ struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, goto free_rtk; } + apple_rtkit_claim_rx(rtk); + ret = apple_mbox_start(rtk->mbox); if (ret) - goto destroy_wq; + goto stop_rx; return rtk; -destroy_wq: - destroy_workqueue(rtk->wq); +stop_rx: + apple_rtkit_stop_rx(rtk); + apple_rtkit_release_rx(rtk); free_rtk: kfree(rtk); return ERR_PTR(ret); @@ -767,8 +821,11 @@ static int apple_rtkit_wait_for_completion(struct completion *c) int apple_rtkit_reinit(struct apple_rtkit *rtk) { + unsigned long flags; + int ret; + /* make sure we don't handle any messages while reinitializing */ - apple_mbox_stop(rtk->mbox); + apple_rtkit_detach_rx(rtk); flush_workqueue(rtk->wq); apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer); @@ -793,7 +850,17 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk) rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_OFF; rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_OFF; - return apple_mbox_start(rtk->mbox); + spin_lock_irqsave(&rtk->mbox->rx_lock, flags); + WRITE_ONCE(rtk->shutting_down, false); + rtk->mbox->rx = apple_rtkit_rx; + spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); + + ret = apple_mbox_start(rtk->mbox); + if (ret) { + apple_rtkit_detach_rx(rtk); + flush_workqueue(rtk->wq); + } + return ret; } EXPORT_SYMBOL_GPL(apple_rtkit_reinit); @@ -972,8 +1039,7 @@ EXPORT_SYMBOL_GPL(apple_rtkit_wake); void apple_rtkit_free(struct apple_rtkit *rtk) { - apple_mbox_stop(rtk->mbox); - destroy_workqueue(rtk->wq); + apple_rtkit_stop_rx(rtk); apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer); apple_rtkit_free_buffer(rtk, &rtk->crashlog_buffer); @@ -981,6 +1047,7 @@ void apple_rtkit_free(struct apple_rtkit *rtk) apple_rtkit_free_buffer(rtk, &rtk->syslog_buffer); kfree(rtk->syslog_msg_buffer); + apple_rtkit_release_rx(rtk); kfree(rtk); } EXPORT_SYMBOL_GPL(apple_rtkit_free); From f45aff313c05bb5af6989a1b7622f8ffe7396c0c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 25 Sep 2026 22:42:53 +1000 Subject: [PATCH 0250/2680] soc: apple: rtkit: Add apple_rtkit_free_retaining_buffers() A co-processor that could not be quiesced may still DMA into the shared buffers Linux gave it. On T8140 the ANS firmware is handed over running by the bootloader and cannot be reset through PMGR (asserting the reset raises an SError), so its consumer can never prove that the firmware stopped. apple_rtkit_free() would return those buffers to the allocator while they may still be written. Add a teardown variant that stops the callbacks and the workqueue and frees the host-only state, but intentionally leaks the shared buffers and their shmem_setup contexts until reboot, and says so in the log. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit.c | 20 ++++++++++++++++++++ include/linux/soc/apple/rtkit.h | 14 ++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 071e0477b5eb1..c9295a8cabae0 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -1052,6 +1052,26 @@ void apple_rtkit_free(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_free); +void apple_rtkit_free_retaining_buffers(struct apple_rtkit *rtk) +{ + /* No callback or worker may retain the consumer's context after return. */ + apple_rtkit_stop_rx(rtk); + + /* + * A failed shutdown provides no guarantee that firmware stopped DMA. + * Retain shared buffers, including custom allocation contexts, rather + * than returning their storage to an allocator while it may be live. + */ + dev_warn(rtk->dev, + "RTKit: retaining the shared buffers of an unstopped co-processor (ioreport %zu, crashlog %zu, oslog %zu, syslog %zu bytes)\n", + rtk->ioreport_buffer.size, rtk->crashlog_buffer.size, + rtk->oslog_buffer.size, rtk->syslog_buffer.size); + kfree(rtk->syslog_msg_buffer); + apple_rtkit_release_rx(rtk); + kfree(rtk); +} +EXPORT_SYMBOL_GPL(apple_rtkit_free_retaining_buffers); + static void apple_rtkit_free_wrapper(void *data) { apple_rtkit_free(data); diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 42a0666e76fcc..21754efccba33 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -105,6 +105,20 @@ struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, */ void apple_rtkit_free(struct apple_rtkit *rtk); +/* + * Detach a co-processor that could not be quiesced without releasing its + * shared buffers. + * + * Stops mailbox callbacks and drains the private RTKit workqueue before + * freeing the transport state. Firmware-visible mappings and their allocation + * contexts are intentionally retained until reboot; shmem_destroy is not + * called. The caller must also retain any buffers it supplied independently + * and must not treat this as a successful firmware shutdown or as permission + * to reprobe. The caller must exclude concurrent users of rtk, as for + * apple_rtkit_free(). + */ +void apple_rtkit_free_retaining_buffers(struct apple_rtkit *rtk); + /* * Reinitialize internal structures. Must only be called with the co-processor * is held in reset. From 721fc9fdb080f3e5c1ea7e98f08a15fd75469b52 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 25 Sep 2026 22:43:11 +1000 Subject: [PATCH 0251/2680] soc: apple: rtkit: Allow streaming-mapped shared buffers A consumer that validates and maps memory owned by a running firmware, rather than allocating a coherent buffer, hands RTKit a streaming DMA mapping. Reading such a buffer from the CPU without a cache sync returns stale data. Let shmem_setup mark a buffer as needing DMA synchronization and bracket the CPU reads in apple_rtkit_memcpy() with dma_sync_single_range_for_cpu()/for_device(). The flag must not be combined with iomem; warn once and read without syncing if a caller does. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit.c | 10 ++++++++++ include/linux/soc/apple/rtkit.h | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index c9295a8cabae0..53a0287c48706 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -364,16 +364,26 @@ static void apple_rtkit_free_buffer(struct apple_rtkit *rtk, bfr->iova = 0; bfr->size = 0; bfr->is_mapped = false; + bfr->needs_dma_sync = false; + bfr->private = NULL; } static void apple_rtkit_memcpy(struct apple_rtkit *rtk, void *dst, struct apple_rtkit_shmem *bfr, size_t offset, size_t len) { + bool sync = bfr->needs_dma_sync && !WARN_ON_ONCE(bfr->iomem); + + if (sync) + dma_sync_single_range_for_cpu(rtk->dev, bfr->iova, offset, len, + DMA_FROM_DEVICE); if (bfr->iomem) memcpy_fromio(dst, bfr->iomem + offset, len); else memcpy(dst, bfr->buffer + offset, len); + if (sync) + dma_sync_single_range_for_device(rtk->dev, bfr->iova, offset, + len, DMA_FROM_DEVICE); } static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 21754efccba33..14dccde061eab 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -22,6 +22,10 @@ * @size: Size of the shared memory buffer. * @iova: Device VA of shared memory buffer. * @is_mapped: Shared memory buffer is managed by the co-processor. + * @needs_dma_sync: The buffer is a streaming DMA mapping of normal RAM made by + * the parent driver's shmem_setup, with @iova as its DMA + * handle. RTKit synchronizes it for the CPU before reading. + * Never set this for coherent allocations or for iomem. * @private: Private data pointer for the parent driver. */ @@ -31,6 +35,7 @@ struct apple_rtkit_shmem { size_t size; dma_addr_t iova; bool is_mapped; + bool needs_dma_sync; void *private; }; From 259bc3b181cdd8a6e7f02b8765797c29ec924598 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 25 Sep 2026 22:43:29 +1000 Subject: [PATCH 0252/2680] soc: apple: rtkit: Support adopting a running co-processor On T8140 the bootloader leaves the ANS co-processor running: it has already completed HELLO and EPMAP with its previous owner, will not repeat either, and cannot be reset (asserting its PMGR reset raises an SError). apple_rtkit_boot() therefore cannot be used, and the existing handlers would flag every inherited syslog record as an error because the INIT geometry was never observed by this session. Add apple_rtkit_init_adopted(), which primes the system endpoints and power states as already up, acknowledges syslog records without parsing them until the firmware sends a fresh INIT, and otherwise behaves like a booted instance. apple_rtkit_adopt_running() applies the same state to an instance created with apple_rtkit_init(). The caller must verify the device-specific running/ready indicators before adopting. apple_rtkit_reinit() clears the inherited state so that a subsequent real boot negotiates normally. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit-internal.h | 2 + drivers/soc/apple/rtkit.c | 71 ++++++++++++++++++++++++++++-- include/linux/soc/apple/rtkit.h | 29 +++++++++--- 3 files changed, 94 insertions(+), 8 deletions(-) diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index d5b45b3865381..8f68a535ccf9f 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -51,6 +51,8 @@ struct apple_rtkit { char *syslog_msg_buffer; size_t syslog_n_entries; size_t syslog_msg_size; + /* Session inherited from the bootloader: INIT geometry never observed. */ + bool syslog_inherited; struct workqueue_struct *wq; }; diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 53a0287c48706..f889575534d89 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -452,6 +452,8 @@ static void apple_rtkit_syslog_rx_init(struct apple_rtkit *rtk, u64 msg) size_t size = FIELD_GET(APPLE_RTKIT_SYSLOG_MSG_SIZE, msg); char *buffer = NULL; + rtk->syslog_inherited = false; + /* * Firmware may send SYSLOG_INIT again after it restarts. The ordered RX * worker is the only user of the message buffer, so it can simply be @@ -482,6 +484,14 @@ static void apple_rtkit_syslog_rx_log(struct apple_rtkit *rtk, u64 msg) size_t entry_size = 0x20 + rtk->syslog_msg_size; int msglen; + /* + * A session inherited from the bootloader never saw SYSLOG_INIT, so the + * ring cannot be interpreted; the records still have to be acknowledged + * for the co-processor to make progress. A later INIT restores parsing. + */ + if (rtk->syslog_inherited) + goto done; + if (!rtk->syslog_msg_buffer) { dev_warn_ratelimited( rtk->dev, @@ -709,6 +719,30 @@ int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint) } EXPORT_SYMBOL_GPL(apple_rtkit_start_ep); +static void apple_rtkit_mark_running(struct apple_rtkit *rtk) +{ + static const u8 system_endpoints[] = { + APPLE_RTKIT_EP_CRASHLOG, + APPLE_RTKIT_EP_SYSLOG, + APPLE_RTKIT_EP_DEBUG, + APPLE_RTKIT_EP_IOREPORT, + APPLE_RTKIT_EP_OSLOG, + APPLE_RTKIT_EP_TRACEKIT, + }; + int i; + + /* + * The previous owner already completed HELLO and acknowledged this + * EPMAP; the firmware will not repeat either for this session. + */ + for (i = 0; i < ARRAY_SIZE(system_endpoints); i++) + set_bit(system_endpoints[i], rtk->endpoints); + + rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_ON; + rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_ON; + rtk->syslog_inherited = true; +} + static void apple_rtkit_claim_rx(struct apple_rtkit *rtk) { struct apple_mbox *mbox = rtk->mbox; @@ -756,9 +790,10 @@ static void apple_rtkit_release_rx(struct apple_rtkit *rtk) spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); } -struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, - const char *mbox_name, int mbox_idx, - const struct apple_rtkit_ops *ops) +static struct apple_rtkit *__apple_rtkit_init(struct device *dev, void *cookie, + const char *mbox_name, int mbox_idx, + const struct apple_rtkit_ops *ops, + bool adopted) { struct apple_rtkit *rtk; int ret; @@ -798,6 +833,10 @@ struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, goto free_rtk; } + /* Inherited system traffic may arrive as soon as mailbox RX is armed. */ + if (adopted) + apple_rtkit_mark_running(rtk); + apple_rtkit_claim_rx(rtk); ret = apple_mbox_start(rtk->mbox); @@ -813,8 +852,23 @@ struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, kfree(rtk); return ERR_PTR(ret); } + +struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, + const char *mbox_name, int mbox_idx, + const struct apple_rtkit_ops *ops) +{ + return __apple_rtkit_init(dev, cookie, mbox_name, mbox_idx, ops, false); +} EXPORT_SYMBOL_GPL(apple_rtkit_init); +struct apple_rtkit *apple_rtkit_init_adopted(struct device *dev, void *cookie, + const char *mbox_name, int mbox_idx, + const struct apple_rtkit_ops *ops) +{ + return __apple_rtkit_init(dev, cookie, mbox_name, mbox_idx, ops, true); +} +EXPORT_SYMBOL_GPL(apple_rtkit_init_adopted); + static int apple_rtkit_wait_for_completion(struct completion *c) { long t; @@ -848,6 +902,7 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk) rtk->syslog_msg_buffer = NULL; rtk->syslog_n_entries = 0; rtk->syslog_msg_size = 0; + rtk->syslog_inherited = false; bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS); set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints); @@ -945,6 +1000,16 @@ int apple_rtkit_boot(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_boot); +int apple_rtkit_adopt_running(struct apple_rtkit *rtk) +{ + if (rtk->crashed) + return -EINVAL; + + apple_rtkit_mark_running(rtk); + return 0; +} +EXPORT_SYMBOL_GPL(apple_rtkit_adopt_running); + int apple_rtkit_shutdown(struct apple_rtkit *rtk) { int ret; diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 14dccde061eab..0aed8f9904c56 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -51,11 +51,13 @@ struct apple_rtkit_shmem { * should return true if it handled the message. If it * returns false, the message will be passed on to the * worker thread. - * @shmem_setup: Setup shared memory buffer. If bfr.is_iomem is true the - * buffer is managed by the co-processor and needs to be mapped. - * Otherwise the buffer is managed by Linux and needs to be - * allocated. If not specified dma_alloc_coherent is used. - * Called in process context. + * @shmem_setup: Setup shared memory buffer. A nonzero input iova requests a + * mapping of firmware-owned memory; validate the ownership + * before mapping it. Otherwise allocate a Linux-owned buffer. + * On failure undo any partial setup. Do not retain the + * descriptor pointer and do not change size: repeated + * requests are matched against it. If not specified + * dma_alloc_coherent is used. Called in process context. * @shmem_destroy: Undo the shared memory buffer setup in shmem_setup. If not * specified dma_free_coherent is used. Called in process * context. @@ -105,6 +107,17 @@ struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, const struct apple_rtkit_ops *ops); +/* + * Initialize an instance for a co-processor that the bootloader left running + * and that cannot be reset. No HELLO/EPMAP handshake takes place: the system + * endpoints are marked available and syslog records are acknowledged without + * being parsed. The caller must verify the device-specific running/ready + * indicators before using the instance. + */ +struct apple_rtkit *apple_rtkit_init_adopted(struct device *dev, void *cookie, + const char *mbox_name, int mbox_idx, + const struct apple_rtkit_ops *ops); + /* * Free an instance of apple_rtkit. */ @@ -136,6 +149,12 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk); */ int apple_rtkit_boot(struct apple_rtkit *rtk); +/* + * Mark an instance created with apple_rtkit_init() as running without a + * handshake, see apple_rtkit_init_adopted(). + */ +int apple_rtkit_adopt_running(struct apple_rtkit *rtk); + /* * Quiesce the co-processor. */ From 43c0b0bc820b0cfc6addd2965fa726c438fb2de5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 25 Sep 2026 22:43:34 +1000 Subject: [PATCH 0253/2680] soc: apple: rtkit: Report crashes of an adopted co-processor A CRASH message on the crashlog endpoint is a buffer request while the crashlog buffer has not been published yet, and a crash notification afterwards. An adopted session never publishes that buffer (the previous owner did), so a crash of the adopted firmware was parsed as a buffer request: with the default shmem path it failed with -EINVAL and a misleading "failed buffer request", and with a consumer that maps firmware-owned memory it was silently accepted as a mapping. Either way rtk->crashed and ops->crashed never fired and the consumer kept using a dead co-processor. Remember that the crashlog buffer is inherited and treat a CRASH message in that state as the notification it is: log its location, mark the instance crashed and call the consumer's crashed callback without a dump. apple_rtkit_reinit() clears the state. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit-internal.h | 1 + drivers/soc/apple/rtkit.c | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index 8f68a535ccf9f..56d7fea01fade 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -53,6 +53,7 @@ struct apple_rtkit { size_t syslog_msg_size; /* Session inherited from the bootloader: INIT geometry never observed. */ bool syslog_inherited; + bool crashlog_inherited; struct workqueue_struct *wq; }; diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index f889575534d89..1bef9e3e8ef17 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -397,6 +397,23 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) return; } + if (rtk->crashlog_inherited && !rtk->crashlog_buffer.size) { + /* + * The previous owner negotiated the crashlog buffer, so this + * session never learned its geometry. A CRASH message on an + * inherited session is therefore a crash notification, not a + * buffer request, and must not be answered as one. + */ + dev_err(rtk->dev, + "RTKit: adopted co-processor has crashed (crashlog at %#llx, %#llx bytes, not mapped)\n", + (u64)FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg), + (u64)FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg) << 12); + rtk->crashed = true; + if (rtk->ops->crashed) + rtk->ops->crashed(rtk->cookie, NULL, 0); + return; + } + if (!rtk->crashlog_buffer.size) { apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer, APPLE_RTKIT_EP_CRASHLOG, msg); @@ -741,6 +758,7 @@ static void apple_rtkit_mark_running(struct apple_rtkit *rtk) rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_ON; rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_ON; rtk->syslog_inherited = true; + rtk->crashlog_inherited = true; } static void apple_rtkit_claim_rx(struct apple_rtkit *rtk) @@ -903,6 +921,7 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk) rtk->syslog_n_entries = 0; rtk->syslog_msg_size = 0; rtk->syslog_inherited = false; + rtk->crashlog_inherited = false; bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS); set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints); From 3699483ee58cfcf0c2c725cc34fae7112c7516a6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 25 Sep 2026 22:43:38 +1000 Subject: [PATCH 0254/2680] soc: apple: sart: Add apple_sart_is_inherited_region() A driver adopting a co-processor that the bootloader left running has to decide whether a buffer the firmware reports is one the bootloader set up for it. The SART records exactly that: entries found programmed at probe are protected and never rewritten by Linux, including at shutdown. Add a helper that returns true only when a protected allow entry covers the entire, nonempty region. It establishes neither RAM ownership nor cache coherency; callers must validate those separately. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sart.c | 25 +++++++++++++++++++++++++ include/linux/soc/apple/sart.h | 14 ++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/drivers/soc/apple/sart.c b/drivers/soc/apple/sart.c index 9eaf3febb3826..b0b208c92741d 100644 --- a/drivers/soc/apple/sart.c +++ b/drivers/soc/apple/sart.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -245,6 +246,30 @@ struct apple_sart *devm_apple_sart_get(struct device *dev) } EXPORT_SYMBOL_GPL(devm_apple_sart_get); +bool apple_sart_is_inherited_region(struct apple_sart *sart, phys_addr_t paddr, + size_t size) +{ + int i; + + if (!size || paddr > PHYS_ADDR_MAX - (size - 1)) + return false; + for (i = 0; i < APPLE_SART_MAX_ENTRIES; i++) { + phys_addr_t base; + size_t length; + u8 flags; + + /* Linux never writes protected entries, including during shutdown. */ + if (!test_bit(i, &sart->protected_entries)) + continue; + sart->ops->get_entry(sart, i, &flags, &base, &length); + if (flags == sart->ops->flags_allow && paddr >= base && + size <= length && paddr - base <= length - size) + return true; + } + return false; +} +EXPORT_SYMBOL_GPL(apple_sart_is_inherited_region); + static int sart_set_entry(struct apple_sart *sart, int index, u8 flags, phys_addr_t paddr, size_t size) { diff --git a/include/linux/soc/apple/sart.h b/include/linux/soc/apple/sart.h index 2249bf6cde099..f5d0db5022831 100644 --- a/include/linux/soc/apple/sart.h +++ b/include/linux/soc/apple/sart.h @@ -50,4 +50,18 @@ int apple_sart_add_allowed_region(struct apple_sart *sart, phys_addr_t paddr, int apple_sart_remove_allowed_region(struct apple_sart *sart, phys_addr_t paddr, size_t size); +/** + * apple_sart_is_inherited_region - test whether a bootloader-protected SART + * allow entry covers a physical region + * @sart: the SART instance + * @paddr: physical start of the region + * @size: size of the region in bytes, must be nonzero + * + * Returns true only when a protected entry (one Linux never writes, including + * at shutdown) with the allow flags covers the entire region. This says + * nothing about RAM ownership or cache coherency of the region. + */ +bool apple_sart_is_inherited_region(struct apple_sart *sart, phys_addr_t paddr, + size_t size); + #endif /* _LINUX_SOC_APPLE_SART_H_ */ From 6aee3038e5ac57accfb8912de6e97d58f5d9ddfa Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:25:35 +1000 Subject: [PATCH 0255/2680] soc: apple: mailbox: Keep the send-empty IRQ enable balanced The send-empty interrupt is requested with IRQF_NO_AUTOEN. apple_mbox_send() enables it when the A2I FIFO is full and it has to wait, and the interrupt handler disables it again when it fires. When the wait times out or is interrupted by a signal, apple_mbox_send() returns with the interrupt still enabled, so the next send that has to wait calls enable_irq() on a line that is already enabled. The IRQ core warns about the unbalanced enable, and from then on the enable state no longer tracks whether a sender is waiting. On T8140 this shows up as "Unbalanced enable for IRQ" warnings from the SEP hwrng thread, whose sends can wait longer than the 500 ms timeout. Track whether the interrupt is unmasked in struct apple_mbox, protected by tx_lock. The sender enables it only while it is masked, the handler disables it only while it is unmasked, and a sender whose wait failed disables it unless a late interrupt already has. Disabling it unconditionally on the error path would not work: the handler may already be waiting for tx_lock and would then disable it a second time. Fixes: 6e1457fcad3f ("soc: apple: mailbox: Add ASC/M3 mailbox driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/mailbox.c | 24 ++++++++++++++++++------ include/linux/soc/apple/mailbox.h | 3 +++ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index f0aa511213189..4837670501e35 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -137,17 +137,26 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, writel_relaxed(mbox->hw->irq_bit_send_empty, mbox->regs + mbox->hw->irq_ack); } - enable_irq(mbox->irq_send_empty); reinit_completion(&mbox->tx_empty); + if (!mbox->tx_irq_unmasked) { + mbox->tx_irq_unmasked = true; + enable_irq(mbox->irq_send_empty); + } spin_unlock_irqrestore(&mbox->tx_lock, flags); t = wait_for_completion_interruptible_timeout( &mbox->tx_empty, msecs_to_jiffies(APPLE_MBOX_TX_TIMEOUT)); - if (t < 0) - return t; - else if (t == 0) - return -ETIMEDOUT; + if (t <= 0) { + /* A late IRQ may already have masked this under tx_lock. */ + spin_lock_irqsave(&mbox->tx_lock, flags); + if (mbox->tx_irq_unmasked) { + disable_irq_nosync(mbox->irq_send_empty); + mbox->tx_irq_unmasked = false; + } + spin_unlock_irqrestore(&mbox->tx_lock, flags); + return t < 0 ? t : -ETIMEDOUT; + } spin_lock_irqsave(&mbox->tx_lock, flags); mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); @@ -175,7 +184,10 @@ static irqreturn_t apple_mbox_send_empty_irq(int irq, void *data) * it at the main controller again. */ spin_lock(&mbox->tx_lock); - disable_irq_nosync(mbox->irq_send_empty); + if (mbox->tx_irq_unmasked) { + disable_irq_nosync(mbox->irq_send_empty); + mbox->tx_irq_unmasked = false; + } complete(&mbox->tx_empty); spin_unlock(&mbox->tx_lock); diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index f73a8913da951..9cfbbac411b8e 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -34,6 +34,9 @@ struct apple_mbox { /** Receive callback for incoming messages */ void (*rx)(struct apple_mbox *mbox, struct apple_mbox_msg msg, void *cookie); void *cookie; + + /* Protected by tx_lock; send-error and IRQ paths mask this only once. */ + bool tx_irq_unmasked; }; struct apple_mbox *apple_mbox_get(struct device *dev, int index); From fb70e84dab09e2bc7ecb9a16e60e70757299f599 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:26:03 +1000 Subject: [PATCH 0256/2680] pmdomain: apple: Print the read-back state on a power state timeout Commit c300d1b7d25c ("soc: apple: pmgr: Add force-disable/force-reset") changed the transition poll in apple_pmgr_ps_set() to read into cur, but left the timeout message printing reg, which now holds the value that was written. Since then the "now:" field shows what was requested rather than the state the domain is stuck in. Print both the value written and the last value read back. On T8140, audio_leap_mca occasionally misses the 100 us poll window, and the read-back value (0xa4f: BUSY set, ACTUAL clock-gated, TARGET active) is what tells a slow transition from a failed one. Fixes: c300d1b7d25c ("soc: apple: pmgr: Add force-disable/force-reset") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmgr-pwrstate.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index a92db8cf88bb9..3599866382c70 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -114,8 +114,8 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a } if (ret < 0) - dev_err(ps->dev, "PS %s: Failed to reach power state 0x%x (now: 0x%x)\n", - genpd->name, pstate, reg); + dev_err(ps->dev, "PS %s: Failed to reach power state 0x%x (requested 0x%x, now: 0x%x)\n", + genpd->name, pstate, reg, cur); if (auto_enable) { /* Not all devices implement this; this is a no-op where not implemented. */ From 670df56f8f62a237d0c79f801bb390ac0eadd5fb Mon Sep 17 00:00:00 2001 From: Yureka Lilian Date: Fri, 11 Sep 2026 18:58:39 +0200 Subject: [PATCH 0257/2680] dt-bindings: arm: apple: apple,pmgr: Add t8140 compatible The PMGR blocks on the Apple silicon A18 Pro SoC (t8140) is compatible with the M1-M4 SoCs. Add "apple,t8140-pmgr" as A18 Pro specific compatible. Signed-off-by: Yureka Lilian Acked-by: Rob Herring (Arm) Reviewed-by: Sven Peter Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml b/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml index e85c8e9758fba..9e0f5703970d9 100644 --- a/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml +++ b/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml @@ -42,6 +42,7 @@ properties: - apple,t6031-pmgr - apple,t8122-pmgr - apple,t8132-pmgr + - apple,t8140-pmgr - const: apple,t8103-pmgr - const: syscon - const: simple-mfd From f5f612242d1bfbf04159106a5a89ece4515f2188 Mon Sep 17 00:00:00 2001 From: Yureka Lilian Date: Fri, 11 Sep 2026 18:58:40 +0200 Subject: [PATCH 0258/2680] dt-bindings: arm: power: apple,pmgr-pwrstate: Add t8140 compatible The device power state management of the PMGR blocks on the Apple silicon A18 Pro SoC (t8140) is compatible with the M1-M4 SoCs. Add "apple,t8140-pmgr-pwrstate" as A18 Pro specific compatible. Signed-off-by: Yureka Lilian Acked-by: Rob Herring (Arm) Reviewed-by: Sven Peter Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml index 7161f30314ab3..0e9e1fbcd3b10 100644 --- a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml @@ -49,6 +49,7 @@ properties: - apple,t6031-pmgr-pwrstate - apple,t8122-pmgr-pwrstate - apple,t8132-pmgr-pwrstate + - apple,t8140-pmgr-pwrstate - const: apple,t8103-pmgr-pwrstate reg: From e874f9fdba326aee9839a8f73cdf68eb6ba571b4 Mon Sep 17 00:00:00 2001 From: Yureka Lilian Date: Fri, 11 Sep 2026 18:58:41 +0200 Subject: [PATCH 0259/2680] dt-bindings: interrupt-controller: apple,aic2: Add t8140 compatible The Apple t8140 (A18 Pro) SoC uses an AIC3 as interrupt controller, same as the M1-M4 SoCs. Signed-off-by: Yureka Lilian Acked-by: Rob Herring (Arm) Reviewed-by: Sven Peter Signed-off-by: Ryan Murray --- .../devicetree/bindings/interrupt-controller/apple,aic2.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml b/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml index 1cb8867851737..e3cadc5ad39d5 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml @@ -46,6 +46,7 @@ properties: - apple,t6030-aic3 - apple,t6031-aic3 - apple,t8132-aic3 + - apple,t8140-aic3 - const: apple,t8122-aic3 - const: apple,t8122-aic3 From 8ddd6b2a8fbc3d7b347ec2823b1a18e66fe7c035 Mon Sep 17 00:00:00 2001 From: Yureka Lilian Date: Fri, 11 Sep 2026 18:58:42 +0200 Subject: [PATCH 0260/2680] dt-bindings: watchdog: apple,wdt: Add t8140 compatible The watchdog on the Apple t8140 (A18 Pro) SoC is compatible with the M1-M4 SoCs. Add "apple,t8140-wdt" as SoC specific compatible. Signed-off-by: Yureka Lilian Acked-by: Rob Herring (Arm) Reviewed-by: Sven Peter Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/watchdog/apple,wdt.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml b/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml index ca4cf01db59f7..1af924157f7ca 100644 --- a/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml @@ -22,6 +22,7 @@ properties: - apple,t6031-wdt - apple,t8122-wdt - apple,t8132-wdt + - apple,t8140-wdt - const: apple,t8103-wdt - items: - enum: From 6ffe690abb92ba92ff1586451a23884fccdd1845 Mon Sep 17 00:00:00 2001 From: Yureka Lilian Date: Fri, 11 Sep 2026 18:58:43 +0200 Subject: [PATCH 0261/2680] dt-bindings: arm: apple: Add A18 Pro based devices The MacBook Neo is the first MacBook to use an A-Series Apple Silicon SoC. The MacBook Neo is overall similar to the M4 series devices. The iPhone 16 Pro is based on the same SoC, but Apple does not allow users running a custom OS on this device. Signed-off-by: Yureka Lilian Acked-by: Rob Herring (Arm) Reviewed-by: Sven Peter Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/arm/apple.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Documentation/devicetree/bindings/arm/apple.yaml b/Documentation/devicetree/bindings/arm/apple.yaml index c967cfcecdbea..45d0b9fd7013d 100644 --- a/Documentation/devicetree/bindings/arm/apple.yaml +++ b/Documentation/devicetree/bindings/arm/apple.yaml @@ -82,6 +82,10 @@ description: | - iPhone 8 Plus - iPhone X + Devices based on the "A18 Pro" SoC: + + - MacBook Neo (A18 Pro, 2026) + Devices based on the "M1" SoC: - Mac mini (M1, 2020) @@ -291,6 +295,13 @@ properties: - const: apple,t8015 - const: apple,arm-platform + - description: Apple A18 Pro SoC based platforms + items: + - enum: + - apple,j700 # MacBook Neo (A18 Pro, 2026) + - const: apple,t8140 + - const: apple,arm-platform + - description: Apple M1 SoC based platforms items: - enum: From 9114fb4f3fc1e8a847be11ca70c465456cf735cc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:26:31 +1000 Subject: [PATCH 0262/2680] dt-bindings: i2c: apple,i2c: Add T8140 compatible The I2C controllers of the A18 Pro (T8140) are compatible with the T8103 ones; on the MacBook Neo the headset codec sits on i2c0. Add "apple,t8140-i2c" with the "apple,t8103-i2c" fallback. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/i2c/apple,i2c.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/i2c/apple,i2c.yaml b/Documentation/devicetree/bindings/i2c/apple,i2c.yaml index 641f24b0ef4b6..ea4ddc838d860 100644 --- a/Documentation/devicetree/bindings/i2c/apple,i2c.yaml +++ b/Documentation/devicetree/bindings/i2c/apple,i2c.yaml @@ -28,6 +28,7 @@ properties: - apple,t6031-i2c - apple,t8122-i2c - apple,t8132-i2c + - apple,t8140-i2c - const: apple,t8103-i2c - items: - enum: From ccc11ae05e052f682655d60e12ec33431a1a27dd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:26:33 +1000 Subject: [PATCH 0263/2680] dt-bindings: pinctrl: apple,pinctrl: Add T8140 and gpio-reserved-ranges Add "apple,t8140-pinctrl" for the pin controller of the A18 Pro (T8140), in the list of SoC compatibles that use the "apple,t8103-pinctrl" fallback. Some T8140 pins are reserved for other agents: Apple's device tree lists them in no-restore-gpio-pins (pins 50 and 134 on the MacBook Neo), and reading their registers from the application processor raises an SError. Allow the standard gpio-reserved-ranges property so that the device tree can describe them. gpiolib then refuses to hand out those lines, and the driver keeps its T8140 register map away from them. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml index bc6d41b63b5d1..b8b4a24107cd1 100644 --- a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml @@ -24,6 +24,7 @@ properties: - apple,t6031-pinctrl - apple,t8122-pinctrl - apple,t8132-pinctrl + - apple,t8140-pinctrl - const: apple,t8103-pinctrl - items: # Do not add additional SoC to this list. @@ -56,6 +57,8 @@ properties: $ref: /schemas/types.yaml#/definitions/uint32 description: The number of pins in this GPIO controller. + gpio-reserved-ranges: true + interrupts: description: One interrupt for each of the (up to 7) interrupt groups supported by the controller sorted by interrupt group From 82c84ad5ba30777f76ba8f3776fa1cba879c9cba Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:26:40 +1000 Subject: [PATCH 0264/2680] dt-bindings: mailbox: apple,mailbox: Add T8140 mailboxes Add two compatibles for the A18 Pro (T8140): - "apple,t8140-asc-mailbox" joins the ASC v4 list. Its registers follow the v4 layout, but the application processor has to set an enable bit in the I2A control register before the mailbox is used, which the driver keys on this compatible. - "apple,t8140-aop-setup-mailbox" is the second mailbox of the Always-On Processor (AOP), its "setup port". It has a compact register layout of its own and only two interrupts, send-empty and recv-not-empty, so it has no fallback. The setup port carries messages that refer to buffers mapped through stream 1 of the AOP DART, whose root is owned by the AOP firmware. Allow iommus and apple,dma-range, the host's allocation window within that root, on the setup mailbox and require them there. The other variants keep their four interrupts and take neither property. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/mailbox/apple,mailbox.yaml | 60 +++++++++++++++---- 1 file changed, 50 insertions(+), 10 deletions(-) diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index 946c909c69227..ee40e13c9a2b9 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -31,10 +31,15 @@ properties: - apple,t8103-asc-mailbox - apple,t8112-asc-mailbox - apple,t8122-asc-mailbox + - apple,t8140-asc-mailbox - apple,t6000-asc-mailbox - apple,t6020-asc-mailbox - const: apple,asc-mailbox-v4 + - description: + AOP setup-port mailbox with a separate register layout. + const: apple,t8140-aop-setup-mailbox + - description: An older ASC mailbox interface found on T2 and A11 that is also used for the NVMe coprocessor and the system management @@ -57,18 +62,23 @@ properties: maxItems: 1 interrupts: - items: - - description: send fifo is empty interrupt - - description: send fifo is not empty interrupt - - description: receive fifo is empty interrupt - - description: receive fifo is not empty interrupt + minItems: 2 + maxItems: 4 interrupt-names: - items: - - const: send-empty - - const: send-not-empty - - const: recv-empty - - const: recv-not-empty + minItems: 2 + maxItems: 4 + + iommus: + maxItems: 1 + + apple,dma-range: + $ref: /schemas/types.yaml#/definitions/uint64-array + minItems: 2 + maxItems: 2 + description: + Setup-port DMA allocation window, expressed as a base and nonzero size. + This identifies host-available slots in the inherited AOP stream root. "#mbox-cells": const: 0 @@ -76,6 +86,36 @@ properties: power-domains: maxItems: 1 +allOf: + - if: + properties: + compatible: + contains: + const: apple,t8140-aop-setup-mailbox + then: + properties: + interrupts: + maxItems: 2 + interrupt-names: + items: + - const: send-empty + - const: recv-not-empty + required: + - iommus + - apple,dma-range + else: + properties: + interrupts: + minItems: 4 + interrupt-names: + items: + - const: send-empty + - const: send-not-empty + - const: recv-empty + - const: recv-not-empty + iommus: false + apple,dma-range: false + required: - compatible - reg From 3930a82b6ace757d9d42c1fa633116c9f9cb6bf1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:27:12 +1000 Subject: [PATCH 0265/2680] dt-bindings: soc: apple: Add T8140 Always-On Processor The Always-On Processor (AOP) of the A18 Pro (T8140) runs firmware loaded by the bootloader and provides the machine's audio and sensor services. The application processor talks to it over two mailboxes: the primary ASC mailbox and a separate setup-port mailbox. Describe the AOP register window and the ASC wrapper registers, the two mailboxes, the AOP's DART stream together with the host allocation window in that stream's firmware-owned root (apple,dma-range), its power domain, and the ALS and audio child nodes. The AOP nodes of the other Apple SoCs have no schema. The T8140 AOP differs from them in its setup-port mailbox and its firmware-owned DART roots, so it gets a schema of its own. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/soc/apple/apple,t8140-aop.yaml | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml new file mode 100644 index 0000000000000..deeef58299f6d --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,t8140-aop.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 Always-On Processor + +maintainers: + - Ryan Murray + +description: + The Always-On Processor provides audio and sensor services through firmware + loaded by the bootloader. The application processor uses a primary mailbox + and a separate setup-port mailbox. Each mailbox consumer has its own DMA + allocation window in the inherited AOP DART mappings. + +properties: + compatible: + const: apple,t8140-aop + + reg: + items: + - description: AOP register and boot-argument window + - description: ASC wrapper and processor control registers + + reg-names: + items: + - const: mmio + - const: asc + + mboxes: + items: + - description: Primary firmware transport + - description: Firmware setup-port transport + + mbox-names: + items: + - const: mbox + - const: setup + + iommus: + maxItems: 1 + + apple,dma-range: + $ref: /schemas/types.yaml#/definitions/uint64-array + minItems: 2 + maxItems: 2 + description: + Host DMA allocation window as a base and nonzero size. The window must + occupy root slots available to the host on this AOP DART stream. + + power-domains: + maxItems: 1 + + als: + type: object + $ref: /schemas/iio/light/apple,t8140-aop-als.yaml# + + audio: + type: object + $ref: /schemas/sound/apple,t8140-aop-audio.yaml# + +required: + - compatible + - reg + - reg-names + - mboxes + - mbox-names + - iommus + - apple,dma-range + - power-domains + +additionalProperties: false From d0b9eb94f36a4e981944a251eb6082978ffcbebf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:27:16 +1000 Subject: [PATCH 0266/2680] dt-bindings: serial: Add Apple DockChannel UART DockChannel is a FIFO-based transport on Apple SoCs. One of its channels carries a UART-compatible byte stream, which a host can reach through the USB-C debug interface; on the MacBook Neo it is the serial console. A channel has a data window with its TX and RX FIFO ports and fill levels, and a configuration window with the FIFO interrupt thresholds. Its interrupts are masked and acknowledged in an interrupt window that is shared with the other DockChannel functions. Describe the three windows by name and the channel's interrupt. The binding lists Ryan Murray as its maintainer; the driver derives from Yureka Lilian's DockChannel UART work, credited in the driver commit. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../serial/apple,dockchannel-uart.yaml | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml diff --git a/Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml b/Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml new file mode 100644 index 0000000000000..836a73034d443 --- /dev/null +++ b/Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/serial/apple,dockchannel-uart.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple DockChannel UART + +maintainers: + - Ryan Murray + +description: | + Apple SoCs provide UART-compatible byte streams over DockChannel. Each + channel uses separate data and FIFO-configuration windows and shares an + interrupt-controller window with the other DockChannel functions. + +allOf: + - $ref: serial.yaml# + +properties: + compatible: + const: apple,dockchannel-uart + + reg: + items: + - description: Channel data registers + - description: Channel FIFO configuration registers + - description: DockChannel interrupt registers + + reg-names: + items: + - const: data + - const: config + - const: irq + + interrupts: + maxItems: 1 + +required: + - compatible + - reg + - reg-names + - interrupts + +unevaluatedProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + serial@30812c000 { + compatible = "apple,dockchannel-uart"; + reg = <0x3 0x0812c000 0x0 0x4000>, + <0x3 0x08128000 0x0 0x4000>, + <0x3 0x0810c000 0x0 0x4000>; + reg-names = "data", "config", "irq"; + interrupts = ; + }; + }; + +... From fc7184f67adc4f1ecb7dbb276c57fd17eed3b78e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:28:28 +1000 Subject: [PATCH 0267/2680] serial: Add Apple DockChannel UART driver Add a driver for the UART that Apple SoCs expose over DockChannel, a FIFO-based transport that a host reaches through the USB-C debug interface. On the MacBook Neo (T8140) it is the serial console. Each channel has a data window with 8- and 32-bit TX and RX FIFO ports, a TX free-space count (DATA_TX_FREE) and an RX fill count (DATA_RX_COUNT), and a configuration window with the TX and RX interrupt thresholds. The FIFOs are 2 KiB deep, and the 8-bit RX port returns the received byte in bits 15:8. Interrupts are masked in IRQ_MASK and acknowledged in IRQ_FLAG (write 1 to clear), in a window shared with the other DockChannel functions. The driver registers ttyDC ports and a console: - Receive is interrupt driven: the RX threshold is one byte, and the handler drains DATA_RX_COUNT bytes per pass. - Transmit polls DATA_TX_FREE before writing each byte to DATA_TX8. The poll is bounded to 10000 cpu_relax() iterations per byte; when it runs out, the byte is dropped rather than spinning forever with the port lock held. The bound is an iteration count, not a time. - set_termios reports a fixed 115200 baud 8N1 link with CLOCAL, since DockChannel has no baud rate, parity or modem lines. The port uses PORT_GENERIC rather than a new port type. - Probe fails cleanly when a register window or the interrupt is missing. A separate early console (earlycon=dockchannel,, or earlycon with stdout-path pointing at the node, whose first reg entry must be the data window) writes to the data window with the same bounded polling. Known limitations, to be addressed in follow-ups: - start_tx() writes out the whole transmit buffer by polling, with the port lock held and interrupts disabled. It ignores a stopped transmitter and x_char and never calls uart_write_wakeup(). It should use the TX interrupt and threshold with uart_port_tx_limited(). - tx_empty() reports TIOCSER_TEMT as soon as one FIFO slot is free instead of when DATA_TX_FREE equals the FIFO size, so uart_wait_until_sent() can return with data still queued. - Probe masks and acknowledges every source in the shared interrupt window and uses fixed channel-0 bit positions, so only one port can work and other DockChannel users would be disturbed. The UART should become a child of an apple,dockchannel node handled by drivers/soc/apple/dockchannel.c. - Both Kconfig options are bool; the driver should be tristate, with a separate console option. The driver is based on the DockChannel UART driver that Yureka Lilian wrote for the Asahi Linux A18 Pro bring-up. This version bounds the TX polling, propagates resource and interrupt errors from probe, uses PORT_GENERIC instead of a new port type, and is adapted to the serial core of this tree. Her authorship is recorded here rather than as From:/Signed-off-by: because her sign-off has not yet been obtained; it is to be added, with a public Link: to the original commit, once it is. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/tty/serial/Kconfig | 24 ++ drivers/tty/serial/Makefile | 3 + .../tty/serial/apple_dockchannel_early_uart.c | 49 +++ drivers/tty/serial/apple_dockchannel_uart.c | 390 ++++++++++++++++++ 4 files changed, 466 insertions(+) create mode 100644 drivers/tty/serial/apple_dockchannel_early_uart.c create mode 100644 drivers/tty/serial/apple_dockchannel_uart.c diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig index cf7dba473b203..53819706c237d 100644 --- a/drivers/tty/serial/Kconfig +++ b/drivers/tty/serial/Kconfig @@ -97,6 +97,30 @@ config SERIAL_EARLYCON_RISCV_SBI with "earlycon=sbi" on the kernel command line. The console is enabled when early_param is processed. +config SERIAL_APPLE_DOCKCHANNEL_EARLYCON + bool "Apple DockChannel early console" + depends on ARCH_APPLE || COMPILE_TEST + select SERIAL_CORE + select SERIAL_CORE_CONSOLE + select SERIAL_EARLYCON + help + Enable an early console on Apple devices whose serial transport is + a DockChannel FIFO. The firmware supplies the configured channel + and the device tree identifies its data registers. This is useful + for diagnosing boot before the normal console driver is available. + +config SERIAL_APPLE_DOCKCHANNEL + bool "Apple DockChannel UART" + depends on ARCH_APPLE || COMPILE_TEST + select SERIAL_CORE + select SERIAL_CORE_CONSOLE + select SERIAL_EARLYCON + help + Enable the Apple DockChannel FIFO as a serial console. The driver + provides interrupt-driven receive and transmit support on devices + whose firmware has configured this transport, including T8140. + It can take over from the DockChannel early console during boot. + config SERIAL_SB1250_DUART tristate "BCM1xxx on-chip DUART serial support" depends on SIBYTE_SB1xxx_SOC=y diff --git a/drivers/tty/serial/Makefile b/drivers/tty/serial/Makefile index bba7b21a4a1d3..d4edbb8becf79 100644 --- a/drivers/tty/serial/Makefile +++ b/drivers/tty/serial/Makefile @@ -98,3 +98,6 @@ obj-$(CONFIG_SERIAL_MCTRL_GPIO) += serial_mctrl_gpio.o obj-$(CONFIG_KGDB_SERIAL_CONSOLE) += kgdboc.o obj-$(CONFIG_SERIAL_NUVOTON_MA35D1) += ma35d1_serial.o + +obj-$(CONFIG_SERIAL_APPLE_DOCKCHANNEL) += apple_dockchannel_uart.o +obj-$(CONFIG_SERIAL_APPLE_DOCKCHANNEL_EARLYCON) += apple_dockchannel_early_uart.o diff --git a/drivers/tty/serial/apple_dockchannel_early_uart.c b/drivers/tty/serial/apple_dockchannel_early_uart.c new file mode 100644 index 0000000000000..3922684eb1089 --- /dev/null +++ b/drivers/tty/serial/apple_dockchannel_early_uart.c @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple DockChannel UART driver + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include + +#define DOCKCHANNEL_TX8 0x04 +#define DOCKCHANNEL_TX_FREE 0x14 +#define DOCKCHANNEL_TX_RETRIES 10000 + +static void dockchannel_uart_putc(struct uart_port *port, unsigned char c) +{ + unsigned int retries = DOCKCHANNEL_TX_RETRIES; + + while (readl(port->membase + DOCKCHANNEL_TX_FREE) == 0 && --retries) + cpu_relax(); + if (!retries) + return; + + writel(c, port->membase + DOCKCHANNEL_TX8); +} + +static void dockchannel_uart_early_write(struct console *con, const char *s, + unsigned int n) +{ + struct earlycon_device *dev = con->data; + + uart_console_write(&dev->port, s, n, dockchannel_uart_putc); +} + +static int __init dockchannel_uart_early_setup(struct earlycon_device *device, + const char *opt) +{ + struct uart_port *port = &device->port; + + if (!port->mapbase) + return -ENODEV; + + device->con->write = dockchannel_uart_early_write; + return 0; +} + +OF_EARLYCON_DECLARE(dockchannel, "apple,dockchannel-uart", + dockchannel_uart_early_setup); +EARLYCON_DECLARE(dockchannel, dockchannel_uart_early_setup); diff --git a/drivers/tty/serial/apple_dockchannel_uart.c b/drivers/tty/serial/apple_dockchannel_uart.c new file mode 100644 index 0000000000000..30a31b5406020 --- /dev/null +++ b/drivers/tty/serial/apple_dockchannel_uart.c @@ -0,0 +1,390 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple DockChannel UART driver + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include + +#define FIFO_SIZE 0x800 + +#define IRQ_MASK 0x0 +#define IRQ_FLAG 0x4 +#define IRQ_TX BIT(0) +#define IRQ_RX BIT(1) + +#define CONFIG_TX_THRESH 0x0 +#define CONFIG_RX_THRESH 0x4 + +#define DATA_TX8 0x04 +#define DATA_TX32 0x10 +#define DATA_TX_FREE 0x14 +#define DATA_RX8 0x1c +#define DATA_RX32 0x28 +#define DATA_RX_COUNT 0x2c + +#define MAX_PORTS 8 +#define TX_RETRIES 10000 + +struct dcuart_port { + struct uart_port port; + void __iomem *config_base; + void __iomem *irq_base; + u32 irq_mask; + /* Serializes IRQ mask updates and RX FIFO draining. */ + spinlock_t lock; +}; + +static struct uart_driver dcuart_drv; +static struct dcuart_port *dcuart_ports[MAX_PORTS]; + +static inline struct dcuart_port *port_to_dc(struct uart_port *port) +{ + return container_of(port, struct dcuart_port, port); +} + +static void dcuart_irq_update(struct dcuart_port *dcuart, + u32 bits, bool enable) +{ + if (enable) + dcuart->irq_mask |= bits; + else + dcuart->irq_mask &= ~bits; + writel_relaxed(dcuart->irq_mask, dcuart->irq_base + IRQ_MASK); +} + +static void dcuart_irq_enable(struct dcuart_port *dcuart, u32 bits) +{ + /* + * IRQ_FLAG is write-to-clear. Clear stale latched flags before + * unmasking so the next interrupt reflects current FIFO state. + */ + writel_relaxed(bits, dcuart->irq_base + IRQ_FLAG); + dcuart_irq_update(dcuart, bits, true); +} + +static void dcuart_irq_disable(struct dcuart_port *dcuart, u32 bits) +{ + dcuart_irq_update(dcuart, bits, false); +} + +static void dcuart_putc(struct uart_port *port, unsigned char c) +{ + unsigned int retries = TX_RETRIES; + + while (readl(port->membase + DATA_TX_FREE) == 0 && --retries) + cpu_relax(); + if (!retries) + return; + + writel(c, port->membase + DATA_TX8); +} + +static void dcuart_write(struct console *co, const char *s, + unsigned int n) +{ + struct dcuart_port *dcuart = dcuart_ports[co->index]; + + uart_console_write(&dcuart->port, s, n, dcuart_putc); +} + +static int dcuart_console_setup(struct console *co, char *options) +{ + struct dcuart_port *dcuart; + + if (co->index < 0 || co->index >= MAX_PORTS) + return -ENODEV; + + dcuart = dcuart_ports[co->index]; + if (!dcuart) + return -ENODEV; + + return 0; +} + +static struct console dcuart_console = { + .name = "ttyDC", + .device = uart_console_device, + .flags = CON_PRINTBUFFER, + .index = -1, + .write = dcuart_write, + .setup = dcuart_console_setup, + .data = &dcuart_drv, +}; + +static struct uart_driver dcuart_drv = { + .owner = THIS_MODULE, + .driver_name = "dcuart", + .nr = MAX_PORTS, + .cons = &dcuart_console, + .dev_name = "ttyDC", +}; + +static irqreturn_t dcuart_interrupt(int irq, void *data) +{ + struct dcuart_port *dcuart = data; + u32 pending = readl_relaxed(dcuart->irq_base + IRQ_FLAG); + u32 handled = 0; + + if (!(pending & (IRQ_TX | IRQ_RX))) + return IRQ_NONE; + + spin_lock(&dcuart->lock); + + if (pending & IRQ_RX) { + int left; + + while ((left = readl_relaxed(dcuart->port.membase + DATA_RX_COUNT))) { + left = min_t(size_t, left, FIFO_SIZE); + while (left) { + /* + * The byte FIFO register returns the byte in bits [15:8] on + * these instances. + */ + char c; + + c = readl_relaxed(dcuart->port.membase + DATA_RX8) >> 8; + uart_insert_char(&dcuart->port, 0, 0, c, TTY_NORMAL); + dcuart->port.icount.rx++; + left--; + } + } + handled |= IRQ_RX; + } + + if (handled & IRQ_RX) + tty_flip_buffer_push(&dcuart->port.state->port); + writel_relaxed(handled, dcuart->irq_base + IRQ_FLAG); + spin_unlock(&dcuart->lock); + + return IRQ_HANDLED; +} + +static unsigned int dcuart_tx_empty(struct uart_port *port) +{ + return (readl(port->membase + DATA_TX_FREE) != 0) ? + TIOCSER_TEMT : + 0; +} + +static void dcuart_start_tx(struct uart_port *port) +{ + char ch; + struct tty_port *tport = &port->state->port; + + if (!kfifo_is_empty(&tport->xmit_fifo)) { + while (kfifo_get(&tport->xmit_fifo, &ch)) { + dcuart_putc(port, ch); + port->icount.tx++; + } + } +} + +static void dcuart_stop_tx(struct uart_port *port) +{ +} + +static void dcuart_stop_rx(struct uart_port *port) +{ + struct dcuart_port *dcuart = port_to_dc(port); + unsigned long flags; + + spin_lock_irqsave(&dcuart->lock, flags); + dcuart_irq_disable(dcuart, IRQ_RX); + spin_unlock_irqrestore(&dcuart->lock, flags); +} + +static const char *dcuart_type(struct uart_port *port) +{ + return port->type == PORT_GENERIC ? + "Apple DockChannel UART" : NULL; +} + +static int dcuart_startup(struct uart_port *port) +{ + struct dcuart_port *dcuart = port_to_dc(port); + unsigned long flags; + + spin_lock_irqsave(&dcuart->lock, flags); + dcuart_irq_disable(dcuart, IRQ_TX | IRQ_RX); + writel_relaxed(1, dcuart->config_base + CONFIG_RX_THRESH); + writel_relaxed(FIFO_SIZE, dcuart->config_base + CONFIG_TX_THRESH); + dcuart_irq_enable(dcuart, IRQ_RX); + spin_unlock_irqrestore(&dcuart->lock, flags); + + enable_irq(port->irq); + return 0; +} + +static void dcuart_shutdown(struct uart_port *port) +{ + struct dcuart_port *dcuart = port_to_dc(port); + unsigned long flags; + + disable_irq(port->irq); + + spin_lock_irqsave(&dcuart->lock, flags); + dcuart_irq_disable(dcuart, IRQ_TX | IRQ_RX); + spin_unlock_irqrestore(&dcuart->lock, flags); +} + +static void dcuart_set_mctrl(struct uart_port *port, unsigned int mctrl) +{ +} + +static unsigned int dcuart_get_mctrl(struct uart_port *port) +{ + return TIOCM_CAR | TIOCM_DSR | TIOCM_CTS; +} + +static void dcuart_set_termios(struct uart_port *port, struct ktermios *new, + const struct ktermios *old) +{ + new->c_cflag &= ~(CSIZE | PARENB); + new->c_cflag |= CS8 | CLOCAL; + tty_termios_encode_baud_rate(new, 115200, 115200); + uart_update_timeout(port, new->c_cflag, 115200); +} + +static void dcuart_release_port(struct uart_port *port) +{ +} + +static int dcuart_request_port(struct uart_port *port) +{ + return 0; +} + +static void dcuart_config_port(struct uart_port *port, int flags) +{ + if (flags & UART_CONFIG_TYPE) + port->type = PORT_GENERIC; +} + +static const struct uart_ops dcuart_ops = { + .tx_empty = dcuart_tx_empty, + .start_tx = dcuart_start_tx, + .stop_tx = dcuart_stop_tx, + .stop_rx = dcuart_stop_rx, + .type = dcuart_type, + .startup = dcuart_startup, + .shutdown = dcuart_shutdown, + .set_mctrl = dcuart_set_mctrl, + .get_mctrl = dcuart_get_mctrl, + .set_termios = dcuart_set_termios, + .release_port = dcuart_release_port, + .request_port = dcuart_request_port, + .config_port = dcuart_config_port, +}; + +static const struct of_device_id dcuart_id_table[] = { + { .compatible = "apple,dockchannel-uart", }, + {}, +}; +MODULE_DEVICE_TABLE(of, dcuart_id_table); + +static int dcuart_probe(struct platform_device *pdev) +{ + int ret, index; + struct dcuart_port *dcuart; + struct resource *data_res; + + for (index = 0; index < MAX_PORTS; index += 1) { + if (!dcuart_ports[index]) + break; + } + if (index == MAX_PORTS) + return dev_err_probe(&pdev->dev, -ENOSPC, "maximum number of ports reached"); + + dcuart = devm_kzalloc(&pdev->dev, sizeof(*dcuart), GFP_KERNEL); + if (!dcuart) + return -ENOMEM; + + dcuart->port.ops = &dcuart_ops; + dcuart->port.dev = &pdev->dev; + dcuart->port.line = index; + dcuart->port.fifosize = FIFO_SIZE; + dcuart->port.iotype = UPIO_MEM32; + dcuart->port.flags = UPF_BOOT_AUTOCONF; + spin_lock_init(&dcuart->lock); + dcuart->irq_base = devm_platform_ioremap_resource_byname(pdev, "irq"); + if (IS_ERR(dcuart->irq_base)) + return PTR_ERR(dcuart->irq_base); + dcuart->config_base = devm_platform_ioremap_resource_byname(pdev, "config"); + if (IS_ERR(dcuart->config_base)) + return PTR_ERR(dcuart->config_base); + dcuart->port.membase = devm_platform_ioremap_resource_byname(pdev, "data"); + if (IS_ERR(dcuart->port.membase)) + return PTR_ERR(dcuart->port.membase); + + data_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "data"); + if (!data_res) + return -EINVAL; + dcuart->port.mapbase = data_res->start; + + writel_relaxed(0, dcuart->irq_base + IRQ_MASK); + writel_relaxed(~0, dcuart->irq_base + IRQ_FLAG); + + dcuart->port.irq = platform_get_irq(pdev, 0); + if (dcuart->port.irq < 0) + return dcuart->port.irq; + + ret = devm_request_irq(&pdev->dev, dcuart->port.irq, dcuart_interrupt, + IRQF_NO_AUTOEN, dev_name(&pdev->dev), dcuart); + if (ret) + return dev_err_probe(&pdev->dev, ret, "failed to request IRQ\n"); + + dcuart_ports[index] = dcuart; + ret = uart_add_one_port(&dcuart_drv, &dcuart->port); + if (ret) + goto cleanup; + + platform_set_drvdata(pdev, dcuart); + + return 0; +cleanup: + dcuart_ports[index] = NULL; + return ret; +} + +static void dcuart_remove(struct platform_device *pdev) +{ + struct dcuart_port *dcuart = platform_get_drvdata(pdev); + + uart_remove_one_port(&dcuart_drv, &dcuart->port); + dcuart_ports[dcuart->port.line] = NULL; +} + +static struct platform_driver dcuart_platform_drv = { + .probe = dcuart_probe, + .remove = dcuart_remove, + .driver = { + .name = "apple-dockchannel-uart", + .of_match_table = dcuart_id_table, + }, +}; + +static int __init dcuart_init(void) +{ + int ret; + + ret = uart_register_driver(&dcuart_drv); + if (ret) + return ret; + + ret = platform_driver_register(&dcuart_platform_drv); + if (ret) + uart_unregister_driver(&dcuart_drv); + + return ret; +} +device_initcall(dcuart_init); + +MODULE_DESCRIPTION("Apple DockChannel UART driver"); +MODULE_LICENSE("Dual MIT/GPL"); From 152952553f81edddc9a19bfadeaef5a56be10103 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:29:07 +1000 Subject: [PATCH 0268/2680] pinctrl: apple: Keep the T8140 register map off reserved pins The driver's regmap uses a flat cache with 512 raw defaults, so creating it reads all 512 pin register slots back from the hardware, whatever apple,npins says. On T8140 some pins belong to other agents: Apple's device tree lists them in no-restore-gpio-pins (pins 50 and 134 on the MacBook Neo), and reading their registers from the application processor raises an asynchronous SError. A regmap debugfs dump that reached pin 134 (0x31a000218) took the machine down that way. Match apple,t8140-pinctrl with match data that makes the driver create its regmap without a cache and with read and write access tables that allow only the registers of the apple,npins pins, minus the pins listed in the standard gpio-reserved-ranges property. Regmap then refuses accesses to reserved pins from every path, including debugfs, which does not go through gpiolib's reserved-line checks. The ranges are validated against apple,npins before use. The interrupt registers above the pin registers are accessed directly, as before, and the other SoCs keep the cached map. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pinctrl/pinctrl-apple-gpio.c | 77 +++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/drivers/pinctrl/pinctrl-apple-gpio.c b/drivers/pinctrl/pinctrl-apple-gpio.c index 6305b96aaa733..eee7f5014ecc1 100644 --- a/drivers/pinctrl/pinctrl-apple-gpio.c +++ b/drivers/pinctrl/pinctrl-apple-gpio.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -373,6 +374,68 @@ static const struct irq_chip apple_gpio_irqchip = { /* Probe & register */ +struct apple_gpio_hw { + /* + * The register block contains slots that fault when read, so the regmap + * cannot be cache-initialised by reading it back; access it uncached and + * only within the valid GPIO registers minus gpio-reserved-ranges. + */ + bool no_regcache; +}; + +static const struct apple_gpio_hw apple_gpio_t8140_hw = { + .no_regcache = true, +}; + +static int apple_gpio_regmap_access(struct device *dev, unsigned int npins, + struct regmap_config *config) +{ + struct regmap_access_table *access; + struct regmap_range *ranges; + unsigned int start, count; + int nvalues = 0, i, ret; + + if (!npins || npins > 512) + return -EINVAL; + + if (of_property_present(dev->of_node, "gpio-reserved-ranges")) { + nvalues = of_property_count_u32_elems(dev->of_node, + "gpio-reserved-ranges"); + if (nvalues < 0 || nvalues % 2) + return -EINVAL; + } + + access = devm_kzalloc(dev, sizeof(*access), GFP_KERNEL); + ranges = devm_kcalloc(dev, nvalues / 2 + 1, sizeof(*ranges), GFP_KERNEL); + if (!access || !ranges) + return -ENOMEM; + + ranges[0].range_min = 0; + ranges[0].range_max = REG_GPIO(npins - 1); + for (i = 0; i < nvalues / 2; i++) { + ret = of_property_read_u32_index(dev->of_node, "gpio-reserved-ranges", + 2 * i, &start); + if (ret) + return ret; + ret = of_property_read_u32_index(dev->of_node, "gpio-reserved-ranges", + 2 * i + 1, &count); + if (ret) + return ret; + if (!count || start >= npins || count > npins - start) + return -EINVAL; + ranges[i + 1].range_min = REG_GPIO(start); + ranges[i + 1].range_max = REG_GPIO(start + count - 1); + } + access->yes_ranges = ranges; + access->n_yes_ranges = 1; + access->no_ranges = ranges + 1; + access->n_no_ranges = nvalues / 2; + config->rd_table = access; + config->wr_table = access; + config->max_register = REG_GPIO(npins - 1); + return 0; +} + static int apple_gpio_register(struct apple_gpio_pinctrl *pctl) { struct gpio_irq_chip *girq = &pctl->gpio_chip.irq; @@ -434,6 +497,8 @@ static int apple_gpio_register(struct apple_gpio_pinctrl *pctl) static int apple_gpio_pinctrl_probe(struct platform_device *pdev) { struct apple_gpio_pinctrl *pctl; + struct regmap_config pctl_regmap_config = regmap_config; + const struct apple_gpio_hw *hw = device_get_match_data(&pdev->dev); struct pinctrl_pin_desc *pins; unsigned int npins; const char **pin_names; @@ -475,7 +540,16 @@ static int apple_gpio_pinctrl_probe(struct platform_device *pdev) if (IS_ERR(pctl->base)) return PTR_ERR(pctl->base); - pctl->map = devm_regmap_init_mmio(&pdev->dev, pctl->base, ®map_config); + if (hw && hw->no_regcache) { + pctl_regmap_config.cache_type = REGCACHE_NONE; + pctl_regmap_config.num_reg_defaults_raw = 0; + res = apple_gpio_regmap_access(&pdev->dev, npins, &pctl_regmap_config); + if (res) + return dev_err_probe(&pdev->dev, res, "invalid GPIO register ranges\n"); + } + + pctl->map = devm_regmap_init_mmio(&pdev->dev, pctl->base, + &pctl_regmap_config); if (IS_ERR(pctl->map)) return dev_err_probe(&pdev->dev, PTR_ERR(pctl->map), "Failed to create regmap\n"); @@ -522,6 +596,7 @@ static int apple_gpio_pinctrl_probe(struct platform_device *pdev) } static const struct of_device_id apple_gpio_pinctrl_of_match[] = { + { .compatible = "apple,t8140-pinctrl", .data = &apple_gpio_t8140_hw }, { .compatible = "apple,t8103-pinctrl", }, { .compatible = "apple,pinctrl", }, { } From f00ee78d8a53555ff2b9d603e2c770fb80b11370 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:29:42 +1000 Subject: [PATCH 0269/2680] pmdomain: apple: Mask BUSY in power state writes and ratelimit errors apple_pmgr_ps_set() builds the words it writes from a read of the power state register. It clears the WAS_CLKGATED and WAS_PWRGATED flags but not BUSY (bit 11), which the hardware reports while a transition is still in progress: on T8140 audio_leap_mca reads back 0xa4f, with BUSY set, after missing the 100 us poll window. A value read in that state was written back with BUSY set. Clear BUSY in the DEV_DISABLE/PS_RESET write that precedes a power-down and in the target state write, which the auto-enable write reuses. This changes the value written on every SoC that uses this driver. No failure attributable to writing BUSY back was observed; the change is precautionary, as the bit is hardware-owned. The error messages in apple_pmgr_ps_set() and in the reset assert and deassert paths are printed from genpd and reset callbacks that consumers retry, so a domain that keeps missing its poll window, as audio_leap_mca does on T8140, repeats the same message on every attempt. Ratelimit them; their text is unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmgr-pwrstate.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index 3599866382c70..42511ed7ef89c 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -65,11 +65,12 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a /* Resets are synchronous, and only work if the device is powered and clocked. */ if (reg & APPLE_PMGR_RESET && pstate != APPLE_PMGR_PS_ACTIVE) - dev_err(ps->dev, "PS %s: powering off with RESET active\n", + dev_err_ratelimited(ps->dev, "PS %s: powering off with RESET active\n", genpd->name); if (pstate != APPLE_PMGR_PS_ACTIVE && (ps->force_disable || ps->force_reset)) { - u32 reg_pre = reg & ~(APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS); + u32 reg_pre = reg & ~(APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | + APPLE_PMGR_BUSY); if (ps->force_disable) reg_pre |= APPLE_PMGR_DEV_DISABLE; @@ -85,12 +86,14 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a APPLE_PMGR_PS_SET_TIMEOUT); if (ret < 0) - dev_err(ps->dev, "PS %s: Failed to set reset/disable bits (now: 0x%x)\n", + dev_err_ratelimited(ps->dev, "PS %s: Failed to set reset/disable bits (now: 0x%x)\n", genpd->name, reg); } + /* Do not replay the hardware-owned BUSY bit in this whole-word write. */ reg &= ~(APPLE_PMGR_DEV_DISABLE | APPLE_PMGR_PS_RESET | - APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | APPLE_PMGR_PS_TARGET); + APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | APPLE_PMGR_BUSY | + APPLE_PMGR_PS_TARGET); reg |= FIELD_PREP(APPLE_PMGR_PS_TARGET, pstate); dev_dbg(ps->dev, "PS %s: pwrstate = 0x%x: 0x%x\n", genpd->name, pstate, reg); @@ -114,7 +117,7 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a } if (ret < 0) - dev_err(ps->dev, "PS %s: Failed to reach power state 0x%x (requested 0x%x, now: 0x%x)\n", + dev_err_ratelimited(ps->dev, "PS %s: Failed to reach power state 0x%x (requested 0x%x, now: 0x%x)\n", genpd->name, pstate, reg, cur); if (auto_enable) { @@ -158,7 +161,8 @@ static int apple_pmgr_reset_assert(struct reset_controller_dev *rcdev, unsigned spin_lock_irqsave(&ps->genpd.slock, flags); if (ps->genpd.status == GENPD_STATE_OFF) - dev_err(ps->dev, "PS 0x%x: asserting RESET while powered down\n", ps->offset); + dev_err_ratelimited(ps->dev, + "PS 0x%x: asserting RESET while powered down\n", ps->offset); dev_dbg(ps->dev, "PS 0x%x: assert reset\n", ps->offset); /* Quiesce device before asserting reset */ @@ -184,7 +188,8 @@ static int apple_pmgr_reset_deassert(struct reset_controller_dev *rcdev, unsigne regmap_update_bits(ps->regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_DEV_DISABLE, 0); if (ps->genpd.status == GENPD_STATE_OFF) - dev_err(ps->dev, "PS 0x%x: RESET was deasserted while powered down\n", ps->offset); + dev_err_ratelimited(ps->dev, + "PS 0x%x: RESET was deasserted while powered down\n", ps->offset); spin_unlock_irqrestore(&ps->genpd.slock, flags); From 3ed9ec330ad70cbb6f47cf3b11b327e59b9ffbfb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:31:21 +1000 Subject: [PATCH 0270/2680] soc: apple: mailbox: Add T8140 ASC and AOP setup mailboxes The A18 Pro (T8140) needs two new mailbox variants: - apple,t8140-asc-mailbox: the ASC mailboxes use the v4 register layout, but the application processor has to set bit 0 of the I2A (co-processor to AP) control register before that FIFO is used. Set it in apple_mbox_start() for this variant. - apple,t8140-aop-setup-mailbox: the Always-On Processor's second mailbox, its setup port, has a compact layout of its own. The A2I and I2A control registers are at 0x0 and 0x4, with the full and empty flags at bits 16 and 17 as on ASC, A2I send is at 0x180/0x188 and I2A receive at 0x1b0/0x1b8. It has only the send-empty and recv-not-empty interrupts, which is all the driver uses. Also check at probe that the register window covers the variant's last receive register. The setup port's window is much smaller than the ASC one, and a device tree mistake would otherwise make the driver access memory outside its mapping. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/mailbox.c | 56 +++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 4837670501e35..16631fc352f44 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -47,6 +47,14 @@ #define APPLE_ASC_MBOX_I2A_RECV0 0x830 #define APPLE_ASC_MBOX_I2A_RECV1 0x838 +/* T8140 AOP setup-port FIFO/control offsets. */ +#define APPLE_T8140_AOP_SETUP_A2I_CONTROL 0x000 +#define APPLE_T8140_AOP_SETUP_I2A_CONTROL 0x004 +#define APPLE_T8140_AOP_SETUP_A2I_SEND0 0x180 +#define APPLE_T8140_AOP_SETUP_A2I_SEND1 0x188 +#define APPLE_T8140_AOP_SETUP_I2A_RECV0 0x1b0 +#define APPLE_T8140_AOP_SETUP_I2A_RECV1 0x1b8 + #define APPLE_T8015_MBOX_A2I_CONTROL 0x108 #define APPLE_T8015_MBOX_I2A_CONTROL 0x10c @@ -81,6 +89,8 @@ #define APPLE_MBOX_TX_TIMEOUT 500 struct apple_mbox_hw { + bool ap_initializes_mailboxes; + unsigned int control_full; unsigned int control_empty; @@ -261,6 +271,7 @@ EXPORT_SYMBOL(apple_mbox_poll); int apple_mbox_start(struct apple_mbox *mbox) { + u32 control; int ret; if (mbox->active) @@ -270,6 +281,12 @@ int apple_mbox_start(struct apple_mbox *mbox) if (ret) return ret; + if (mbox->hw->ap_initializes_mailboxes) { + /* The primary T8140 outbox requires its enable bit before use. */ + control = readl(mbox->regs + mbox->hw->i2a_control); + writel(control | BIT(0), mbox->regs + mbox->hw->i2a_control); + } + /* * Only some variants of this mailbox HW provide interrupt control * at the mailbox level. We therefore need to handle enabling/disabling @@ -356,6 +373,7 @@ static int apple_mbox_probe(struct platform_device *pdev) char *irqname; struct apple_mbox *mbox; struct device *dev = &pdev->dev; + struct resource *res; mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL); if (!mbox) @@ -366,6 +384,10 @@ static int apple_mbox_probe(struct platform_device *pdev) if (!mbox->hw) return -EINVAL; + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res || resource_size(res) < mbox->hw->i2a_recv1 + sizeof(u64)) + return dev_err_probe(dev, -EINVAL, "mailbox register window is too small\n"); + mbox->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(mbox->regs)) return PTR_ERR(mbox->regs); @@ -441,6 +463,38 @@ static const struct apple_mbox_hw apple_mbox_asc_hw = { .has_irq_controls = false, }; +static const struct apple_mbox_hw apple_mbox_t8140_asc_hw = { + .ap_initializes_mailboxes = true, + + .control_full = APPLE_ASC_MBOX_CONTROL_FULL, + .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, + + .a2i_control = APPLE_ASC_MBOX_A2I_CONTROL, + .a2i_send0 = APPLE_ASC_MBOX_A2I_SEND0, + .a2i_send1 = APPLE_ASC_MBOX_A2I_SEND1, + + .i2a_control = APPLE_ASC_MBOX_I2A_CONTROL, + .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, + .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, + + .has_irq_controls = false, +}; + +static const struct apple_mbox_hw apple_mbox_t8140_aop_setup_hw = { + .control_full = APPLE_ASC_MBOX_CONTROL_FULL, + .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, + + .a2i_control = APPLE_T8140_AOP_SETUP_A2I_CONTROL, + .a2i_send0 = APPLE_T8140_AOP_SETUP_A2I_SEND0, + .a2i_send1 = APPLE_T8140_AOP_SETUP_A2I_SEND1, + + .i2a_control = APPLE_T8140_AOP_SETUP_I2A_CONTROL, + .i2a_recv0 = APPLE_T8140_AOP_SETUP_I2A_RECV0, + .i2a_recv1 = APPLE_T8140_AOP_SETUP_I2A_RECV1, + + .has_irq_controls = false, +}; + static const struct apple_mbox_hw apple_mbox_m3_hw = { .control_full = APPLE_M3_MBOX_CONTROL_FULL, .control_empty = APPLE_M3_MBOX_CONTROL_EMPTY, @@ -461,6 +515,8 @@ static const struct apple_mbox_hw apple_mbox_m3_hw = { }; static const struct of_device_id apple_mbox_of_match[] = { + { .compatible = "apple,t8140-asc-mailbox", .data = &apple_mbox_t8140_asc_hw }, + { .compatible = "apple,t8140-aop-setup-mailbox", .data = &apple_mbox_t8140_aop_setup_hw }, { .compatible = "apple,asc-mailbox-v4", .data = &apple_mbox_asc_hw }, { .compatible = "apple,t8015-asc-mailbox", .data = &apple_mbox_t8015_hw }, { .compatible = "apple,m3-mailbox-v2", .data = &apple_mbox_m3_hw }, From adc3b3467cea1097d5f7d429f25250d34b43f711 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:20:30 +1000 Subject: [PATCH 0271/2680] dt-bindings: nvmem: apple,spmi-nvmem: Add apple,baku-pmic The J700 (T8140) uses a PMIC called Baku. Like the PMICs already listed, it holds the boot-state scratch registers that apple,smc-reboot uses (boot stage, boot error and panic counters, and the shutdown flag) and exposes them over SPMI; on Baku they start at 0xf700. Add its compatible so that it can be described with the apple,spmi-nvmem fallback. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml b/Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml index 4ca75ed07a54f..7a4114c7fbd19 100644 --- a/Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml +++ b/Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml @@ -18,6 +18,7 @@ properties: compatible: items: - enum: + - apple,baku-pmic - apple,maverick-pmic - apple,sera-pmic - apple,stowe-pmic From 179b7403607d97bbec9c1ad40d27a107172e0c4d Mon Sep 17 00:00:00 2001 From: Yureka Lilian Date: Fri, 11 Sep 2026 18:58:38 +0200 Subject: [PATCH 0272/2680] dt-bindings: arm: cpus: Add Apple A18 Pro CPU core compatibles Add "apple,tahiti-e" and "apple,tahiti-p" compatibles for the A18 Pro efficiency and performance CPU cores respectively. As with the M4 cores, there is no longer a generational codename, so we assign the core compatible based on SoC codename ("Tahiti") plus core type (E-Core / P-Core). Acked-by: Krzysztof Kozlowski Signed-off-by: Yureka Lilian Reviewed-by: Sven Peter Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/arm/cpus.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/devicetree/bindings/arm/cpus.yaml b/Documentation/devicetree/bindings/arm/cpus.yaml index c328e9fb98435..5d79da63301be 100644 --- a/Documentation/devicetree/bindings/arm/cpus.yaml +++ b/Documentation/devicetree/bindings/arm/cpus.yaml @@ -95,6 +95,8 @@ properties: - apple,mistral - apple,monsoon - apple,sawtooth + - apple,tahiti-e + - apple,tahiti-p - apple,twister - apple,typhoon - arm,arm710t From 477a7023a7afb6f839b477614f1091e54a0a50c4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 02:50:46 +1000 Subject: [PATCH 0273/2680] dt-bindings: soc: apple: Add the ASC4 RTKit helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MTP coprocessor that serves the keyboard and trackpad of Apple laptops runs RTKit firmware that the operating system only has to boot and keep running; the DockChannel HID transport then talks to it over a FIFO pair. The T8112, T8122, T6020, T6030 and T6031 device trees have carried apple,rtk-helper-asc4 nodes for it without a binding, and the T8140 (J700, MacBook Neo) adds a sixth SoC. Describe the node: the ASC control window, the optional SRAM window from which the firmware may ask the host to map buffers, the mailbox, the IOMMU stream and the #helper-cells property that the existing device trees carry although nothing consumes it. The compatible list covers every in-tree user, including the T6031 node, whose second entry is the T8122 helper compatible, so the existing device trees validate unchanged. No bare apple,rtk-helper-asc4 form is allowed: no device tree uses one, and a node needs its SoC-specific compatibles. Compile-tested only: dt_binding_check on the schema, and dtbs_check on t8112-j413, t8112-j493, t6020-j414s, t6030-j514s, t6031-j514c, t8122-j613 and t8140-j700 with no new warnings. [ryan: adapted to 7.2.10 — the T8140 MAINTAINERS section is created by the 900-config-tooling topic; its F: line is carried there] Assisted-by: LLM Signed-off-by: Ryan Murray --- .../soc/apple/apple,rtk-helper-asc4.yaml | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml diff --git a/Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml b/Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml new file mode 100644 index 0000000000000..6db0197b4e864 --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,rtk-helper-asc4.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple ASC4 RTKit helper coprocessor + +maintainers: + - Ryan Murray + +description: + Apple SoCs contain a number of ASC coprocessors that run RTKit firmware. + For some of them the operating system only has to boot the coprocessor and + keep it running, while the devices it serves are described by other nodes. + The MTP (multi-touch processor) ASC of Apple laptops is one of these. It + serves the built-in keyboard and trackpad over DockChannel, and the + DockChannel HID transport refers to this node through its apple,helper-cpu + property. + +properties: + compatible: + items: + - enum: + - apple,t6020-mtp + - apple,t6030-mtp + - apple,t6031-mtp + - apple,t8112-mtp + - apple,t8122-mtp + - apple,t8140-mtp + - enum: + - apple,t6020-rtk-helper-asc4 + - apple,t6030-rtk-helper-asc4 + - apple,t8112-rtk-helper-asc4 + - apple,t8122-rtk-helper-asc4 + - apple,t8140-rtk-helper-asc4 + - const: apple,mtp + - const: apple,rtk-helper-asc4 + + reg: + minItems: 1 + items: + - description: ASC control registers + - description: + Coprocessor SRAM, from which the firmware may ask the host to map + buffers instead of allocating them from system memory + + reg-names: + minItems: 1 + items: + - const: asc + - const: sram + + mboxes: + maxItems: 1 + + iommus: + maxItems: 1 + + '#helper-cells': + const: 0 + description: + Present in existing device trees. Nothing references the helper with + argument cells; the DockChannel HID transport uses a plain phandle. + +required: + - compatible + - reg + - reg-names + - mboxes + - iommus + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + mtp@314600000 { + compatible = "apple,t8140-mtp", "apple,t8140-rtk-helper-asc4", + "apple,mtp", "apple,rtk-helper-asc4"; + reg = <0x3 0x14600000 0x0 0x4000>, + <0x3 0x14050000 0x0 0x4000>; + reg-names = "asc", "sram"; + mboxes = <&mtp_mbox>; + iommus = <&mtp_dart 0>; + }; + }; From 7cd08ed1407a0b690a02003c1f13fab130a869fe Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 02:50:46 +1000 Subject: [PATCH 0274/2680] dt-bindings: soc: apple: Add the DockChannel FIFO bus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DockChannel is a set of byte FIFOs between the application processor and a coprocessor. Each channel has a host side and a remote side with a configuration window and a data window each, and the channels of one block share an interrupt window whose interrupts signal FIFO threshold events. The apple,dockchannel node maps that interrupt window, translates the channel windows for its children and is their interrupt controller; the HID transport of the MTP coprocessor is its child. The node has been in the T8112, T8122, T6020, T6030 and T6031 device trees without a binding, and the T8140 (J700, MacBook Neo) adds a sixth SoC. Describe it, with every in-tree SoC compatible, so that the existing device trees validate unchanged. Compile-tested only: dt_binding_check on the schema, and dtbs_check on t8112-j413, t8112-j493, t6020-j414s, t6030-j514s, t6031-j514c, t8122-j613 and t8140-j700 with no new warnings. [ryan: adapted to 7.2.10 — the T8140 MAINTAINERS section is created by the 900-config-tooling topic; this commit's MAINTAINERS lines are carried there] Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/soc/apple/apple,dockchannel.yaml | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml diff --git a/Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml b/Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml new file mode 100644 index 0000000000000..40831b082bf28 --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,dockchannel.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple DockChannel FIFO bus + +maintainers: + - Ryan Murray + +description: + DockChannel is a set of byte FIFOs between the application processor and a + coprocessor. Each channel has a host side and a remote side, each with a + configuration window and a data window, and the channels of one block + share an interrupt window whose interrupts signal FIFO threshold events. + This node maps the interrupt window, translates the channel register + windows for its children and is their interrupt controller. Each child + describes one function carried over a channel pair. + +properties: + compatible: + items: + - enum: + - apple,t6020-dockchannel + - apple,t6030-dockchannel + - apple,t6031-dockchannel + - apple,t8112-dockchannel + - apple,t8122-dockchannel + - apple,t8140-dockchannel + - const: apple,dockchannel + + reg: + maxItems: 1 + + reg-names: + const: irq + + interrupts: + maxItems: 1 + + ranges: true + + nonposted-mmio: true + + '#address-cells': + const: 1 + + '#size-cells': + const: 1 + + interrupt-controller: true + + '#interrupt-cells': + const: 2 + description: + The first cell is the FIFO interrupt number within the block and the + second cell the trigger type. + +patternProperties: + '^input@[0-9a-f]+$': + type: object + $ref: /schemas/input/apple,dockchannel-hid.yaml# + description: A DockChannel HID transport. + +required: + - compatible + - reg + - reg-names + - interrupts + - ranges + - '#address-cells' + - '#size-cells' + - interrupt-controller + - '#interrupt-cells' + +additionalProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + mtp_dockchannel: fifo@314b14000 { + compatible = "apple,t8140-dockchannel", "apple,dockchannel"; + reg = <0x3 0x14b14000 0x0 0x4000>; + reg-names = "irq"; + interrupts = ; + ranges = <0x0 0x3 0x14b28000 0x20000>; + nonposted-mmio; + #address-cells = <1>; + #size-cells = <1>; + interrupt-controller; + #interrupt-cells = <2>; + + input@8000 { + compatible = "apple,dockchannel-hid"; + reg = <0x8000 0x4000>, + <0xc000 0x4000>, + <0x0000 0x4000>, + <0x4000 0x4000>; + reg-names = "config", "data", "rmt-config", "rmt-data"; + iommus = <&mtp_dart 0>; + interrupt-parent = <&mtp_dockchannel>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, + <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + }; + }; + }; From 8f0cd42393695d99c2903427d0e6477ef632bcc6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:00:07 +1000 Subject: [PATCH 0275/2680] gpio: macsmc: Fix the valid-mask bound check macsmc_gpio_init_valid_mask() rejects a GPIO number greater than MAX_GPIO, which lets number 64 through to set_bit() on the 64-bit valid mask. The loop stops at the key of GPIO 63 before such a key can be reached, so the out-of-bounds bit is not set in practice; make the check match the mask all the same. Compile-tested only; not run on hardware. Fixes: 9b21051b0885 ("gpio: Add new gpio-macsmc driver for Apple Macs") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpio/gpio-macsmc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpio/gpio-macsmc.c b/drivers/gpio/gpio-macsmc.c index c3ca445a85ac9..261e41312da9a 100644 --- a/drivers/gpio/gpio-macsmc.c +++ b/drivers/gpio/gpio-macsmc.c @@ -222,7 +222,7 @@ static int macsmc_gpio_init_valid_mask(struct gpio_chip *gc, break; gpio_nr = macsmc_gpio_nr(key); - if (gpio_nr < 0 || gpio_nr > MAX_GPIO) { + if (gpio_nr < 0 || gpio_nr >= MAX_GPIO) { dev_err(smcgp->dev, "Bad GPIO key %p4ch\n", &key); continue; } From 67773f29f025a1968b8bd0e445fb764b1ffc2039 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:04:28 +1000 Subject: [PATCH 0276/2680] gpio: macsmc: Add the J700 pcIO AFE reset line The J700 (MacBook Neo) resets its trackpad analog front end through the SMC, but not through a gPxx GPIO key. The machine's Apple device tree describes the reset as the platform function pKW8("pcIO", 0x10000, 0x130000): an eight-byte write to the pcIO key with 0x10000 in the low word, 0x130000 in the high word and the requested level in bits 15:0. Bits 23:16 of the high word hold the pin number, 19. The SMC has no gP13 key for the pin, so the line is invisible to the GPIO provider as it stands. Expose it as GPIO 19 of the apple,smc-gpio chip on J700 and turn writes to it into the pcIO payload. The pcIO key cannot be read back, so the value returned for the line is the last one written, which is 0 after boot whatever the hardware state is. Nothing changes on other machines, and nothing changes for the other lines of the J700 chip. The DockChannel HID driver drives this line during the trackpad's power method 2 transitions. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpio/gpio-macsmc.c | 47 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/drivers/gpio/gpio-macsmc.c b/drivers/gpio/gpio-macsmc.c index 261e41312da9a..f44566d6598b4 100644 --- a/drivers/gpio/gpio-macsmc.c +++ b/drivers/gpio/gpio-macsmc.c @@ -13,9 +13,21 @@ #include #include #include +#include #define MAX_GPIO 64 +/* + * J700 resets its trackpad analog front end through the SMC "pcIO" key + * rather than a gPxx GPIO key. The Apple device tree describes the reset as + * the platform function pKW8("pcIO", 0x10000, 0x130000): an eight-byte write + * with 0x10000 in the low word, 0x130000 in the high word and the requested + * level in bits 15:0. Bits 23:16 of the high word hold the pin number. + */ +#define MACSMC_J700_AFE_RESET_GPIO 19 +#define MACSMC_J700_AFE_RESET_LO 0x10000 +#define MACSMC_J700_AFE_RESET_HI 0x130000 + /* * Commands 0-6 are, presumably, the intended API. * Command 0xff lets you get/set the pin configuration in detail directly, @@ -76,8 +88,17 @@ struct macsmc_gpio { int first_index; smc_key base_key; + + bool j700_afe_reset; + bool j700_afe_reset_level; }; +static bool macsmc_gpio_is_j700_afe_reset(struct macsmc_gpio *smcgp, + unsigned int offset) +{ + return smcgp->j700_afe_reset && offset == MACSMC_J700_AFE_RESET_GPIO; +} + static int macsmc_gpio_nr(smc_key key) { int low = hex_to_bin(key & 0xff); @@ -148,6 +169,9 @@ static int macsmc_gpio_get_direction(struct gpio_chip *gc, unsigned int offset) u32 val; int ret; + if (macsmc_gpio_is_j700_afe_reset(smcgp, offset)) + return GPIO_LINE_DIRECTION_OUT; + /* First try reading the explicit pin mode register */ ret = apple_smc_rw_u32(smcgp->smc, key, CMD_PINMODE, &val); if (!ret) @@ -168,6 +192,10 @@ static int macsmc_gpio_get(struct gpio_chip *gc, unsigned int offset) u32 cmd, val; int ret; + /* pcIO cannot be read back; report the last level written. */ + if (macsmc_gpio_is_j700_afe_reset(smcgp, offset)) + return smcgp->j700_afe_reset_level; + ret = macsmc_gpio_get_direction(gc, offset); if (ret < 0) return ret; @@ -190,6 +218,21 @@ static int macsmc_gpio_set(struct gpio_chip *gc, unsigned int offset, int value) smc_key key = macsmc_gpio_key(smcgp->base_key, offset); int ret; + if (macsmc_gpio_is_j700_afe_reset(smcgp, offset)) { + u64 payload = ((u64)MACSMC_J700_AFE_RESET_HI << 32) | + MACSMC_J700_AFE_RESET_LO | (value ? 1 : 0); + + ret = apple_smc_write_u64(smcgp->smc, SMC_KEY(pcIO), payload); + if (ret < 0) { + dev_err_ratelimited(smcgp->dev, + "pcIO GPIO %u set to %u failed: %d\n", + offset, !!value, ret); + return ret; + } + smcgp->j700_afe_reset_level = !!value; + return 0; + } + value |= CMD_OUTPUT; ret = apple_smc_write_u32(smcgp->smc, key, CMD_OUTPUT | value); if (ret < 0) @@ -209,6 +252,8 @@ static int macsmc_gpio_init_valid_mask(struct gpio_chip *gc, count = min(smcgp->smc->key_count, MAX_GPIO); bitmap_zero(valid_mask, ngpios); + if (smcgp->j700_afe_reset) + set_bit(MACSMC_J700_AFE_RESET_GPIO, valid_mask); for (i = 0; i < count; i++) { int ret, gpio_nr; @@ -252,6 +297,8 @@ static int macsmc_gpio_probe(struct platform_device *pdev) smcgp->dev = &pdev->dev; smcgp->smc = smc; smcgp->base_key = data ? data->base_key : _SMC_KEY("gP\0\0"); + smcgp->j700_afe_reset = of_machine_is_compatible("apple,j700") && + device_is_compatible(&pdev->dev, "apple,smc-gpio"); smcgp->first_index = macsmc_gpio_find_first_gpio_index(smcgp); if (smcgp->first_index < 0) From bbfd8833e69a49ac751ef5cd57a74a8ad862cd7c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 21:25:00 +1000 Subject: [PATCH 0277/2680] serial: apple-dockchannel: Credit Yureka Lilian as the original author The DockChannel UART driver and its early console were added in commit commit ca755c9d7722 ("serial: Add Apple DockChannel UART driver") as an adaptation of the driver Yureka Lilian wrote for the Asahi Linux A18 Pro bring-up. That commit credited her only in its message and left her public links to be added later. Record the attribution where it belongs: name her as the original author in both source files and as the module author, and link her original commits. No functional change. Link: https://cyberchaos.dev/yuka/linux/commit/515a4e3dae7f53bbf1cfbe34486a4a3c7071e93b Link: https://cyberchaos.dev/yuka/linux/commit/d7b9ddde4af5d6d987f8a19e5a20781e16b4d7d9 Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/tty/serial/apple_dockchannel_early_uart.c | 2 ++ drivers/tty/serial/apple_dockchannel_uart.c | 3 +++ 2 files changed, 5 insertions(+) diff --git a/drivers/tty/serial/apple_dockchannel_early_uart.c b/drivers/tty/serial/apple_dockchannel_early_uart.c index 3922684eb1089..1048dde41d794 100644 --- a/drivers/tty/serial/apple_dockchannel_early_uart.c +++ b/drivers/tty/serial/apple_dockchannel_early_uart.c @@ -2,6 +2,8 @@ /* * Apple DockChannel UART driver * Copyright The Asahi Linux Contributors + * + * Based on the DockChannel UART earlycon by Yureka Lilian . */ #include diff --git a/drivers/tty/serial/apple_dockchannel_uart.c b/drivers/tty/serial/apple_dockchannel_uart.c index 30a31b5406020..65c15870e4c83 100644 --- a/drivers/tty/serial/apple_dockchannel_uart.c +++ b/drivers/tty/serial/apple_dockchannel_uart.c @@ -2,6 +2,8 @@ /* * Apple DockChannel UART driver * Copyright The Asahi Linux Contributors + * + * Based on the DockChannel UART driver by Yureka Lilian . */ #include @@ -386,5 +388,6 @@ static int __init dcuart_init(void) } device_initcall(dcuart_init); +MODULE_AUTHOR("Yureka Lilian "); MODULE_DESCRIPTION("Apple DockChannel UART driver"); MODULE_LICENSE("Dual MIT/GPL"); From b3413acf8c44021908b27ecb53e7c89de182ddac Mon Sep 17 00:00:00 2001 From: Ace <7362641+Acelogic@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:20:12 -0400 Subject: [PATCH 0278/2680] gpio: macsmc: Support J700 radio power through pcIO J700 exposes the radio power control as SMC GPIO8 through the pcIO key, so the generic gP08 transaction cannot drive it. Extend the existing J700 pcIO GPIO path with the GPIO8 low/high words from a fresh function-reg_on ADT observation. Keep GPIO19 behavior and cache only successful writes. A coordinator-owned root-port test brought the PCIe link up with this control. No radio firmware or endpoint DMA was exercised. --- drivers/gpio/gpio-macsmc.c | 44 +++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/drivers/gpio/gpio-macsmc.c b/drivers/gpio/gpio-macsmc.c index f44566d6598b4..471bce2e54b4f 100644 --- a/drivers/gpio/gpio-macsmc.c +++ b/drivers/gpio/gpio-macsmc.c @@ -28,6 +28,11 @@ #define MACSMC_J700_AFE_RESET_LO 0x10000 #define MACSMC_J700_AFE_RESET_HI 0x130000 +/* J700 amfm-sunrise function-reg_on: pKW8("pcIO", 0x800000, 0x80000). */ +#define MACSMC_J700_RADIO_REG_ON_GPIO 8 +#define MACSMC_J700_RADIO_REG_ON_LO 0x800000 +#define MACSMC_J700_RADIO_REG_ON_HI 0x80000 + /* * Commands 0-6 are, presumably, the intended API. * Command 0xff lets you get/set the pin configuration in detail directly, @@ -89,14 +94,16 @@ struct macsmc_gpio { int first_index; smc_key base_key; - bool j700_afe_reset; - bool j700_afe_reset_level; + bool j700_pcio; + DECLARE_BITMAP(j700_pcio_level, MAX_GPIO); }; -static bool macsmc_gpio_is_j700_afe_reset(struct macsmc_gpio *smcgp, +static bool macsmc_gpio_is_j700_pcio(struct macsmc_gpio *smcgp, unsigned int offset) { - return smcgp->j700_afe_reset && offset == MACSMC_J700_AFE_RESET_GPIO; + return smcgp->j700_pcio && + (offset == MACSMC_J700_AFE_RESET_GPIO || + offset == MACSMC_J700_RADIO_REG_ON_GPIO); } static int macsmc_gpio_nr(smc_key key) @@ -169,7 +176,7 @@ static int macsmc_gpio_get_direction(struct gpio_chip *gc, unsigned int offset) u32 val; int ret; - if (macsmc_gpio_is_j700_afe_reset(smcgp, offset)) + if (macsmc_gpio_is_j700_pcio(smcgp, offset)) return GPIO_LINE_DIRECTION_OUT; /* First try reading the explicit pin mode register */ @@ -193,8 +200,8 @@ static int macsmc_gpio_get(struct gpio_chip *gc, unsigned int offset) int ret; /* pcIO cannot be read back; report the last level written. */ - if (macsmc_gpio_is_j700_afe_reset(smcgp, offset)) - return smcgp->j700_afe_reset_level; + if (macsmc_gpio_is_j700_pcio(smcgp, offset)) + return test_bit(offset, smcgp->j700_pcio_level); ret = macsmc_gpio_get_direction(gc, offset); if (ret < 0) @@ -218,9 +225,18 @@ static int macsmc_gpio_set(struct gpio_chip *gc, unsigned int offset, int value) smc_key key = macsmc_gpio_key(smcgp->base_key, offset); int ret; - if (macsmc_gpio_is_j700_afe_reset(smcgp, offset)) { - u64 payload = ((u64)MACSMC_J700_AFE_RESET_HI << 32) | - MACSMC_J700_AFE_RESET_LO | (value ? 1 : 0); + if (macsmc_gpio_is_j700_pcio(smcgp, offset)) { + u32 lo, hi; + u64 payload; + + if (offset == MACSMC_J700_RADIO_REG_ON_GPIO) { + lo = MACSMC_J700_RADIO_REG_ON_LO; + hi = MACSMC_J700_RADIO_REG_ON_HI; + } else { + lo = MACSMC_J700_AFE_RESET_LO; + hi = MACSMC_J700_AFE_RESET_HI; + } + payload = ((u64)hi << 32) | lo | !!value; ret = apple_smc_write_u64(smcgp->smc, SMC_KEY(pcIO), payload); if (ret < 0) { @@ -229,7 +245,7 @@ static int macsmc_gpio_set(struct gpio_chip *gc, unsigned int offset, int value) offset, !!value, ret); return ret; } - smcgp->j700_afe_reset_level = !!value; + assign_bit(offset, smcgp->j700_pcio_level, value); return 0; } @@ -252,8 +268,10 @@ static int macsmc_gpio_init_valid_mask(struct gpio_chip *gc, count = min(smcgp->smc->key_count, MAX_GPIO); bitmap_zero(valid_mask, ngpios); - if (smcgp->j700_afe_reset) + if (smcgp->j700_pcio) { set_bit(MACSMC_J700_AFE_RESET_GPIO, valid_mask); + set_bit(MACSMC_J700_RADIO_REG_ON_GPIO, valid_mask); + } for (i = 0; i < count; i++) { int ret, gpio_nr; @@ -297,7 +315,7 @@ static int macsmc_gpio_probe(struct platform_device *pdev) smcgp->dev = &pdev->dev; smcgp->smc = smc; smcgp->base_key = data ? data->base_key : _SMC_KEY("gP\0\0"); - smcgp->j700_afe_reset = of_machine_is_compatible("apple,j700") && + smcgp->j700_pcio = of_machine_is_compatible("apple,j700") && device_is_compatible(&pdev->dev, "apple,smc-gpio"); smcgp->first_index = macsmc_gpio_find_first_gpio_index(smcgp); From 255617b12d486ba327247a369c691629668a1f30 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 10:05:51 -0400 Subject: [PATCH 0279/2680] dt-bindings: apple: describe the M5 J813 minimal boot platform Describe the J813 board, T8142 CPU identifiers and AICv3 compatible. Document the Aurora minimal RAM handoff flag consumed by the J813 m1n1 bring-up stack. Signed-off-by: Acelogic --- .../devicetree/bindings/arm/apple.yaml | 19 +++++++++++++++++++ .../devicetree/bindings/arm/cpus.yaml | 2 ++ .../interrupt-controller/apple,aic2.yaml | 1 + .../devicetree/bindings/vendor-prefixes.yaml | 2 ++ 4 files changed, 24 insertions(+) diff --git a/Documentation/devicetree/bindings/arm/apple.yaml b/Documentation/devicetree/bindings/arm/apple.yaml index 45d0b9fd7013d..3b473dfa6310c 100644 --- a/Documentation/devicetree/bindings/arm/apple.yaml +++ b/Documentation/devicetree/bindings/arm/apple.yaml @@ -107,6 +107,10 @@ description: | - MacBook Pro (14-inch, M3, 2023) - iMac (24-inch, M3, 2023) + Devices based on the "M5" SoC: + + - MacBook Air (13-inch, M5, 2026) + Devices based on the "M1 Pro", "M1 Max" and "M1 Ultra" SoCs: - MacBook Pro (14-inch, M1 Pro, 2021) @@ -346,6 +350,12 @@ properties: - const: apple,t8132 - const: apple,arm-platform + - description: Apple M5 SoC based platforms + items: + - const: apple,j813 # MacBook Air (13-inch, M5, 2026) + - const: apple,t8142 + - const: apple,arm-platform + - description: Apple M1 Pro SoC based platforms items: - enum: @@ -425,6 +435,15 @@ properties: - const: apple,t6032 - const: apple,arm-platform + aurora,minimal-boot: + type: boolean + description: + Request a minimal native RAM boot from the Aurora m1n1 loader. Skip + generic PCIe and DAPF initialization and the legacy EL2 Apple Device + Tree handoff. The loader still supplies the Linux flattened device + tree, usable memory, reserved regions and CPU release addresses. + Used during initial J813 bring-up without DMA peripherals. + additionalProperties: true ... diff --git a/Documentation/devicetree/bindings/arm/cpus.yaml b/Documentation/devicetree/bindings/arm/cpus.yaml index 5d79da63301be..a51ecf598aa2d 100644 --- a/Documentation/devicetree/bindings/arm/cpus.yaml +++ b/Documentation/devicetree/bindings/arm/cpus.yaml @@ -97,6 +97,8 @@ properties: - apple,sawtooth - apple,tahiti-e - apple,tahiti-p + - apple,t8142-e + - apple,t8142-p - apple,twister - apple,typhoon - arm,arm710t diff --git a/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml b/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml index e3cadc5ad39d5..160a4e4ba74d2 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml @@ -47,6 +47,7 @@ properties: - apple,t6031-aic3 - apple,t8132-aic3 - apple,t8140-aic3 + - apple,t8142-aic3 - const: apple,t8122-aic3 - const: apple,t8122-aic3 diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml index 396044f368e7c..da5bf2f0d78c9 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.yaml +++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml @@ -205,6 +205,8 @@ patternProperties: description: Atmel Corporation "^auo,.*": description: AU Optronics Corporation + "^aurora,.*": + description: Aurora Silicon "^auvidea,.*": description: Auvidea GmbH "^avago,.*": From 5ebcb568971391286f0e824e0adb28abaf8a2083 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 10:05:51 -0400 Subject: [PATCH 0280/2680] irqchip/apple-aic: use architectural guest timer masks on T8142 The initial J813 boot traps on writes to VM_TMR_FIQ_ENA_EL2 in aic_init_cpu. Select a T8142 AICv3 profile which masks physical and virtual guest timers through their EL02 control registers and uses their control status in the FIQ handler. Keep the legacy register path for existing SoCs. Port the minimal timer workaround from the local J813 bring-up at c8f85bcfb03925179bfa6c19c4d28c44efae39ac. Native single-CPU boot is the qualification target; KVM guest timer behavior remains unqualified. Signed-off-by: Acelogic --- drivers/irqchip/irq-apple-aic.c | 50 ++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 4a3141d9f9141..eb3a648f2f533 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -260,6 +260,7 @@ struct aic_info { /* Features */ bool fast_ipi; bool local_fast_ipi; + bool vm_timer_fiq_readonly; }; static const struct aic_info aic1_info __initconst = { @@ -306,7 +307,19 @@ static const struct aic_info aic3_info __initconst = { .local_fast_ipi = true, }; +static const struct aic_info aic3_t8142_info __initconst = { + .version = 3, + .irq_cfg = AIC3_IRQ_CFG, + .fast_ipi = true, + .local_fast_ipi = true, + .vm_timer_fiq_readonly = true, +}; + static const struct of_device_id aic_info_match[] = { + { + .compatible = "apple,t8142-aic3", + .data = &aic3_t8142_info, + }, { .compatible = "apple,t8103-aic", .data = &aic1_local_fipi_info, @@ -497,16 +510,38 @@ static unsigned long aic_fiq_get_idx(struct irq_data *d) return AIC_HWIRQ_IRQ(irqd_to_hwirq(d)); } +/* + * T8142 traps writes to VM_TMR_FIQ_ENA_EL2. Mask the guest timers through + * their architectural EL02 controls instead, leaving the older SoCs alone. + */ +static void aic_vm_timer_mask(u64 timers, bool mask) +{ + if (!aic_irqc->info.vm_timer_fiq_readonly) { + sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, + mask ? timers : 0, mask ? 0 : timers); + return; + } + + if (timers & VM_TMR_FIQ_ENABLE_P) + sysreg_clear_set_s(SYS_CNTP_CTL_EL02, + mask ? 0 : ARCH_TIMER_CTRL_IT_MASK, + mask ? ARCH_TIMER_CTRL_IT_MASK : 0); + if (timers & VM_TMR_FIQ_ENABLE_V) + sysreg_clear_set_s(SYS_CNTV_CTL_EL02, + mask ? 0 : ARCH_TIMER_CTRL_IT_MASK, + mask ? ARCH_TIMER_CTRL_IT_MASK : 0); +} + static void aic_fiq_set_mask(struct irq_data *d) { /* Only the guest timers have real mask bits, unfortunately. */ switch (aic_fiq_get_idx(d)) { case AIC_TMR_EL02_PHYS: - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_P, 0); + aic_vm_timer_mask(VM_TMR_FIQ_ENABLE_P, true); isb(); break; case AIC_TMR_EL02_VIRT: - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_V, 0); + aic_vm_timer_mask(VM_TMR_FIQ_ENABLE_V, true); isb(); break; default: @@ -518,11 +553,11 @@ static void aic_fiq_clear_mask(struct irq_data *d) { switch (aic_fiq_get_idx(d)) { case AIC_TMR_EL02_PHYS: - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_P); + aic_vm_timer_mask(VM_TMR_FIQ_ENABLE_P, false); isb(); break; case AIC_TMR_EL02_VIRT: - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_V); + aic_vm_timer_mask(VM_TMR_FIQ_ENABLE_V, false); isb(); break; default: @@ -584,7 +619,9 @@ static void __exception_irq_entry aic_handle_fiq(struct pt_regs *regs) AIC_FIQ_HWIRQ(AIC_TMR_EL0_VIRT)); if (is_kernel_in_hyp_mode()) { - uint64_t enabled = read_sysreg_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2); + uint64_t enabled = aic_irqc->info.vm_timer_fiq_readonly ? + VM_TMR_FIQ_ENABLE_P | VM_TMR_FIQ_ENABLE_V : + read_sysreg_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2); if ((enabled & VM_TMR_FIQ_ENABLE_P) && TIMER_FIRING(read_sysreg_s(SYS_CNTP_CTL_EL02))) @@ -871,8 +908,7 @@ static int aic_init_cpu(unsigned int cpu) /* EL2-only (VHE mode) IRQ sources */ if (is_kernel_in_hyp_mode()) { /* Guest timers */ - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, - VM_TMR_FIQ_ENABLE_V | VM_TMR_FIQ_ENABLE_P, 0); + aic_vm_timer_mask(VM_TMR_FIQ_ENABLE_V | VM_TMR_FIQ_ENABLE_P, true); /* vGIC maintenance IRQ */ sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EL2_En, 0); From 3eb33b6e7673bd1d6c6728f7a59e67783befd6d1 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 10:42:22 -0400 Subject: [PATCH 0281/2680] arm64: dts: apple: use the native FDT handoff on J813 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — Documentation/arch/arm64/apple-j813.rst and the J813 DTS are created by the 000-devicetree topic; their hunks are carried there] --- Documentation/devicetree/bindings/arm/apple.yaml | 9 --------- Documentation/devicetree/bindings/vendor-prefixes.yaml | 2 -- 2 files changed, 11 deletions(-) diff --git a/Documentation/devicetree/bindings/arm/apple.yaml b/Documentation/devicetree/bindings/arm/apple.yaml index 3b473dfa6310c..9239a2e904fb7 100644 --- a/Documentation/devicetree/bindings/arm/apple.yaml +++ b/Documentation/devicetree/bindings/arm/apple.yaml @@ -435,15 +435,6 @@ properties: - const: apple,t6032 - const: apple,arm-platform - aurora,minimal-boot: - type: boolean - description: - Request a minimal native RAM boot from the Aurora m1n1 loader. Skip - generic PCIe and DAPF initialization and the legacy EL2 Apple Device - Tree handoff. The loader still supplies the Linux flattened device - tree, usable memory, reserved regions and CPU release addresses. - Used during initial J813 bring-up without DMA peripherals. - additionalProperties: true ... diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml index da5bf2f0d78c9..396044f368e7c 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.yaml +++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml @@ -205,8 +205,6 @@ patternProperties: description: Atmel Corporation "^auo,.*": description: AU Optronics Corporation - "^aurora,.*": - description: Aurora Silicon "^auvidea,.*": description: Auvidea GmbH "^avago,.*": From d622321927100748fa79bced574c8bcf47608486 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 09:57:49 -0400 Subject: [PATCH 0282/2680] dt-bindings: apple: Describe T8152 and the J873g Mac mini Add the M6 board and E/M/P CPU compatibles, and describe its AIC3 with the existing T8122 fallback. Document the Aurora minimal RAM handoff used during early bring-up without DMA peripherals. Signed-off-by: Acelogic --- .../devicetree/bindings/arm/apple.yaml | 19 +++++++++++++++++++ .../devicetree/bindings/arm/cpus.yaml | 3 +++ .../interrupt-controller/apple,aic2.yaml | 1 + .../devicetree/bindings/vendor-prefixes.yaml | 2 ++ 4 files changed, 25 insertions(+) diff --git a/Documentation/devicetree/bindings/arm/apple.yaml b/Documentation/devicetree/bindings/arm/apple.yaml index 9239a2e904fb7..269909160867d 100644 --- a/Documentation/devicetree/bindings/arm/apple.yaml +++ b/Documentation/devicetree/bindings/arm/apple.yaml @@ -111,6 +111,10 @@ description: | - MacBook Air (13-inch, M5, 2026) + Devices based on the "M6" SoC: + + - Mac mini (M6, J873g) + Devices based on the "M1 Pro", "M1 Max" and "M1 Ultra" SoCs: - MacBook Pro (14-inch, M1 Pro, 2021) @@ -356,6 +360,12 @@ properties: - const: apple,t8142 - const: apple,arm-platform + - description: Apple M6 SoC based platforms + items: + - const: apple,j873g # Mac mini (M6) + - const: apple,t8152 + - const: apple,arm-platform + - description: Apple M1 Pro SoC based platforms items: - enum: @@ -435,6 +445,15 @@ properties: - const: apple,t6032 - const: apple,arm-platform + aurora,minimal-boot: + type: boolean + description: + Request a minimal native RAM boot from the Aurora m1n1 loader. Skip + generic PCIe and DAPF initialization and the legacy EL2 Apple Device + Tree handoff. The loader still supplies the Linux flattened device + tree, usable memory, reserved regions and CPU release addresses. + Used during initial bring-up without DMA peripherals. + additionalProperties: true ... diff --git a/Documentation/devicetree/bindings/arm/cpus.yaml b/Documentation/devicetree/bindings/arm/cpus.yaml index a51ecf598aa2d..bf4142469cb09 100644 --- a/Documentation/devicetree/bindings/arm/cpus.yaml +++ b/Documentation/devicetree/bindings/arm/cpus.yaml @@ -95,6 +95,9 @@ properties: - apple,mistral - apple,monsoon - apple,sawtooth + - apple,t8152-e + - apple,t8152-m + - apple,t8152-p - apple,tahiti-e - apple,tahiti-p - apple,t8142-e diff --git a/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml b/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml index 160a4e4ba74d2..25f31c8fb452d 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml @@ -48,6 +48,7 @@ properties: - apple,t8132-aic3 - apple,t8140-aic3 - apple,t8142-aic3 + - apple,t8152-aic3 - const: apple,t8122-aic3 - const: apple,t8122-aic3 diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml index 396044f368e7c..da5bf2f0d78c9 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.yaml +++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml @@ -205,6 +205,8 @@ patternProperties: description: Atmel Corporation "^auo,.*": description: AU Optronics Corporation + "^aurora,.*": + description: Aurora Silicon "^auvidea,.*": description: Auvidea GmbH "^avago,.*": From 56a06ac404014d8a0f9ba00ed4a1fa55bff97a19 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 18:48:20 -0400 Subject: [PATCH 0283/2680] dt-bindings: apple: describe J813 monitoring and input peripherals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe the T8142 mailbox, SMC, MTP, DART and PWM resources. Add explicit contracts for runtime temperature discovery and firmware-owned trackpad reset without embedding sensor tables or calibration. [ryan: adapted to 7.2.10 — split by topic: the apple,dart.yaml hunk moves to 020-dart (it needs the T8140 DART compatible), the apple,smc-hwmon.yaml and apple,smc.yaml hunks to 060-power-thermal, and the apple,dockchannel-hid.yaml hunk to 090-input, where the context they extend is created] Signed-off-by: Acelogic --- Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml | 1 + Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml | 1 + .../devicetree/bindings/power/apple,pmgr-pwrstate.yaml | 1 + Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml | 1 + .../devicetree/bindings/soc/apple/apple,dockchannel.yaml | 1 + .../devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml | 2 ++ 6 files changed, 7 insertions(+) diff --git a/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml b/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml index 9e0f5703970d9..c5fce7d78bfc6 100644 --- a/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml +++ b/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml @@ -43,6 +43,7 @@ properties: - apple,t8122-pmgr - apple,t8132-pmgr - apple,t8140-pmgr + - apple,t8142-pmgr - const: apple,t8103-pmgr - const: syscon - const: simple-mfd diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index ee40e13c9a2b9..dc4cbb5168f20 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -32,6 +32,7 @@ properties: - apple,t8112-asc-mailbox - apple,t8122-asc-mailbox - apple,t8140-asc-mailbox + - apple,t8142-asc-mailbox - apple,t6000-asc-mailbox - apple,t6020-asc-mailbox - const: apple,asc-mailbox-v4 diff --git a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml index 0e9e1fbcd3b10..d131441214db1 100644 --- a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml @@ -50,6 +50,7 @@ properties: - apple,t8122-pmgr-pwrstate - apple,t8132-pmgr-pwrstate - apple,t8140-pmgr-pwrstate + - apple,t8142-pmgr-pwrstate - const: apple,t8103-pmgr-pwrstate reg: diff --git a/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml b/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml index ae8baa5f7bcdc..3b3cad92b601f 100644 --- a/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml +++ b/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml @@ -19,6 +19,7 @@ properties: - apple,t8103-fpwm - apple,t8112-fpwm - apple,t8122-fpwm + - apple,t8142-fpwm - apple,t8132-fpwm - apple,t6000-fpwm - apple,t6020-fpwm diff --git a/Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml b/Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml index 40831b082bf28..a3362df34e148 100644 --- a/Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml +++ b/Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml @@ -28,6 +28,7 @@ properties: - apple,t8112-dockchannel - apple,t8122-dockchannel - apple,t8140-dockchannel + - apple,t8142-dockchannel - const: apple,dockchannel reg: diff --git a/Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml b/Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml index 6db0197b4e864..da78694e37466 100644 --- a/Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml +++ b/Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml @@ -28,12 +28,14 @@ properties: - apple,t8112-mtp - apple,t8122-mtp - apple,t8140-mtp + - apple,t8142-mtp - enum: - apple,t6020-rtk-helper-asc4 - apple,t6030-rtk-helper-asc4 - apple,t8112-rtk-helper-asc4 - apple,t8122-rtk-helper-asc4 - apple,t8140-rtk-helper-asc4 + - apple,t8142-rtk-helper-asc4 - const: apple,mtp - const: apple,rtk-helper-asc4 From 322328e5c890c887efcada1b811ff8ceb236fcfe Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:52:57 -0400 Subject: [PATCH 0284/2680] soc: apple: rtkit: Support co-processors that place their own buffers Some co-processors place their shared buffers themselves, such as the GPU co-processors of the T8140 (A18 Pro). Their buffer requests carry the IOVA of memory the co-processor already uses, and the request must be acknowledged by sending it back unchanged, whereas RTKit sends nothing for such buffers today. These co-processors also have their crashlog endpoint started while the endpoint map is handled, before the map is acknowledged, instead of together with the other system endpoints after the last map message. Add a flags field to struct apple_rtkit_ops, with an APPLE_RTKIT_COPROC_PLACES_BUFFERS flag selecting this behavior. The buffer is still set up through shmem_setup, which lets the driver provide the memory behind the co-processor's IOVA, for instance to read crash logs from it. Without the flag, RTKit sends exactly the same messages as before. Expose the flag to Rust drivers as Operations::COPROC_PLACES_BUFFERS. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 928641d26be3083dc06c0de072ce99537fa059b7) --- drivers/soc/apple/rtkit.c | 20 ++++++++++++++++++++ include/linux/soc/apple/rtkit.h | 12 ++++++++++++ rust/bindings/bindings_helper.h | 1 + rust/kernel/soc/apple/rtkit.rs | 12 ++++++++++++ 4 files changed, 45 insertions(+) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 1bef9e3e8ef17..1e01e1d3e3a61 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -100,6 +100,11 @@ bool apple_rtkit_is_crashed(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_is_crashed); +static bool apple_rtkit_coproc_places_buffers(struct apple_rtkit *rtk) +{ + return rtk->ops->flags & APPLE_RTKIT_COPROC_PLACES_BUFFERS; +} + static int apple_rtkit_management_send(struct apple_rtkit *rtk, u8 type, u64 msg) { @@ -169,6 +174,14 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) set_bit(ep, rtk->endpoints); } + /* + * For co-processors that place their own buffers, the crashlog + * endpoint is started before the map is acknowledged. + */ + if (apple_rtkit_coproc_places_buffers(rtk) && base == 0 && + test_bit(APPLE_RTKIT_EP_CRASHLOG, &bitmap)) + apple_rtkit_start_ep(rtk, APPLE_RTKIT_EP_CRASHLOG); + reply = FIELD_PREP(APPLE_RTKIT_MGMT_EPMAP_BASE, base); if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST) reply |= APPLE_RTKIT_MGMT_EPMAP_LAST; @@ -193,6 +206,10 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) case APPLE_RTKIT_EP_IOREPORT: case APPLE_RTKIT_EP_OSLOG: case APPLE_RTKIT_EP_TRACEKIT: + /* already started while handling the map */ + if (ep == APPLE_RTKIT_EP_CRASHLOG && + apple_rtkit_coproc_places_buffers(rtk)) + break; dev_dbg(rtk->dev, "RTKit: Starting system endpoint 0x%02x\n", ep); apple_rtkit_start_ep(rtk, ep); @@ -338,6 +355,9 @@ static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, buffer->iova); } apple_rtkit_send_message(rtk, ep, reply, NULL, false); + } else if (apple_rtkit_coproc_places_buffers(rtk)) { + /* acknowledge the co-processor's own buffer with its request */ + apple_rtkit_send_message(rtk, ep, msg, NULL, false); } return 0; diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 0aed8f9904c56..4eafd55c52a4b 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -10,6 +10,7 @@ #ifndef _LINUX_APPLE_RTKIT_H_ #define _LINUX_APPLE_RTKIT_H_ +#include #include #include #include @@ -39,9 +40,19 @@ struct apple_rtkit_shmem { void *private; }; +/* + * The co-processor places its shared buffers itself. A buffer request that + * carries an IOVA is set up with shmem_setup and acknowledged by sending the + * request back unchanged. The crashlog endpoint is started while the endpoint + * map is handled, before the map is acknowledged. + */ +#define APPLE_RTKIT_COPROC_PLACES_BUFFERS BIT(0) + /* * Struct to represent implementation-specific RTKit operations. * + * @flags: Combination of APPLE_RTKIT_* flags describing the + * co-processor. * @crashed: Called when the co-processor has crashed. Runs in process * context. * @recv_message: Function called when a message from RTKit is received @@ -63,6 +74,7 @@ struct apple_rtkit_shmem { * context. */ struct apple_rtkit_ops { + unsigned int flags; void (*crashed)(void *cookie, const void *crashlog, size_t crashlog_size); void (*recv_message)(void *cookie, u8 endpoint, u64 message); bool (*recv_message_early)(void *cookie, u8 endpoint, u64 message); diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index ce864cc200be6..31fe42ed08c4c 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -134,6 +134,7 @@ const gfp_t RUST_CONST_HELPER___GFP_HIGHMEM = ___GFP_HIGHMEM; const gfp_t RUST_CONST_HELPER___GFP_NOWARN = ___GFP_NOWARN; const blk_features_t RUST_CONST_HELPER_BLK_FEAT_ROTATIONAL = BLK_FEAT_ROTATIONAL; const fop_flags_t RUST_CONST_HELPER_FOP_UNSIGNED_OFFSET = FOP_UNSIGNED_OFFSET; +const u32 RUST_CONST_HELPER_APPLE_RTKIT_COPROC_PLACES_BUFFERS = APPLE_RTKIT_COPROC_PLACES_BUFFERS; const u64 BINDINGS_SNDRV_PCM_FMTBIT_FLOAT_LE = SNDRV_PCM_FMTBIT_FLOAT_LE; diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index d68417bb9973a..4b82dbf32129f 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -60,6 +60,13 @@ pub trait Operations { /// Type representing an allocated buffer for RTKit. type Buffer: Buffer; + /// Whether the coprocessor places its shared buffers itself. + /// + /// If set, buffer requests that carry an IOVA are served by [`Operations::shmem_map`] and + /// acknowledged by sending the request back unchanged, and the crashlog endpoint is started + /// before the endpoint map is acknowledged. + const COPROC_PLACES_BUFFERS: bool = false; + /// Called when RTKit crashes. fn crashed(_data: ::Borrowed<'_>, _crashlog: Option<&[u8]>) {} @@ -205,6 +212,11 @@ unsafe extern "C" fn shmem_destroy_callback( impl RtKit { const VTABLE: bindings::apple_rtkit_ops = bindings::apple_rtkit_ops { + flags: if T::COPROC_PLACES_BUFFERS { + bindings::APPLE_RTKIT_COPROC_PLACES_BUFFERS + } else { + 0 + }, crashed: Some(crashed_callback::), recv_message: Some(recv_message_callback::), recv_message_early: Some(recv_message_early_callback::), From a66fdbfe5d46cba12c9d46992b8b94b5f21fefe2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:53:05 -0400 Subject: [PATCH 0285/2680] dt-bindings: mailbox: apple,mailbox: Add the T8140 GPU mailboxes The GPU co-processors of the T8140 (A18 Pro) use the v4 ASC mailbox layout. Unlike the other T8140 ASC mailboxes, described by apple,t8140-asc-mailbox, they do not need the application processor to enable their receive FIFO. Add apple,t8140-gfx-asc-mailbox for them, with apple,asc-mailbox-v4 as the fallback. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 4c5aad430f6283d1d6b3b961a87d1629b397f22f) --- .../devicetree/bindings/mailbox/apple,mailbox.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index dc4cbb5168f20..21dc2231d88ba 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -37,6 +37,14 @@ properties: - apple,t6020-asc-mailbox - const: apple,asc-mailbox-v4 + - description: + ASC mailboxes of the T8140 GPU coprocessors. Unlike the other + T8140 ASC mailboxes, they do not need the application processor + to enable their receive FIFO. + items: + - const: apple,t8140-gfx-asc-mailbox + - const: apple,asc-mailbox-v4 + - description: AOP setup-port mailbox with a separate register layout. const: apple,t8140-aop-setup-mailbox From 30aa95c5aa2030330fbdbd1226d14abd751a79a6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 28 Sep 2026 16:36:16 +1000 Subject: [PATCH 0286/2680] soc: apple: synchronize the ASCWrap mailbox provider with j700 Carry the provider-owned T8140 CPU lifecycle, removal serialization and runtime-PM retry fixes into the cleanroom integration. Both provider files are byte-identical to j700 at 43bb4bbb36ec. This carries the mailbox portions of upstream commits: 10636bd526b981ea233d30d8ad6e7e09b2f9d733 2230686542a49f6d0ebbaf11a9e75fb0981abc20 41afc106aece4a6e319f3b20327d1842174aa0a8 Validation: matching Image/modules build; both GPU firmware processors initialize on local Neo with the rewritten consumer. (cherry picked from commit 29db1d8ab517933c2e0114fb3c6bf9beda5417aa) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/mailbox.c | 394 ++++++++++++++++++++++++++++-- include/linux/soc/apple/mailbox.h | 57 ++++- 2 files changed, 430 insertions(+), 21 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 16631fc352f44..ba8d124e853b1 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -47,7 +47,33 @@ #define APPLE_ASC_MBOX_I2A_RECV0 0x830 #define APPLE_ASC_MBOX_I2A_RECV1 0x838 -/* T8140 AOP setup-port FIFO/control offsets. */ +/* ASCWrap v6 embeds the unchanged ASC mailbox block at this offset. */ +#define APPLE_ASCWRAP_V6_MBOX_OFFSET 0x8000 +#define APPLE_ASCWRAP_V6_WRAPPER_SIZE 0x88000 +#define APPLE_ASCWRAP_V6_IOP_VBAR_SIZE 0x8 +#define APPLE_ASCWRAP_V6_CPU_CONTROL 0x44 +#define APPLE_ASCWRAP_V6_CPU_RUN BIT(4) +#define APPLE_ASCWRAP_V6_CPU_STOP_SECOND_CLEAR BIT(5) + +#define APPLE_T8140_GFX_WRAPPER_BASE 0x482600000ULL +#define APPLE_T8140_GFX_IOP_VBAR_BASE 0x482050000ULL +#define APPLE_T8140_GFX1_WRAPPER_BASE 0x482e00000ULL +#define APPLE_T8140_GFX1_IOP_VBAR_BASE 0x482850000ULL + +static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_A2I_CONTROL == + 0x8110); +static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_CONTROL == + 0x8114); +static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_A2I_SEND0 == + 0x8800); +static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_RECV0 == + 0x8830); + +/* + * The T8140 AOP "setup port" (ADT aop-exclave-mailbox): the second mailbox + * the J700 AOP firmware boots through. Layout measured on hardware by the + * native m1n1 microphone host (artifacts/j700-native-lpmic-20260906). + */ #define APPLE_T8140_AOP_SETUP_A2I_CONTROL 0x000 #define APPLE_T8140_AOP_SETUP_I2A_CONTROL 0x004 #define APPLE_T8140_AOP_SETUP_A2I_SEND0 0x180 @@ -89,6 +115,8 @@ #define APPLE_MBOX_TX_TIMEOUT 500 struct apple_mbox_hw { + unsigned int reg_offset; + bool is_ascwrap_v6; bool ap_initializes_mailboxes; unsigned int control_full; @@ -109,6 +137,140 @@ struct apple_mbox_hw { unsigned int irq_bit_send_empty; }; +int apple_mbox_ascwrap_v6_get_lifecycle( + struct apple_mbox *mbox, + struct apple_mbox_ascwrap_v6_lifecycle *lifecycle) +{ + if (!mbox || !lifecycle) + return -EINVAL; + if (READ_ONCE(mbox->removing)) + return -ENODEV; + if (!mbox->hw->is_ascwrap_v6) + return -EOPNOTSUPP; + + *lifecycle = mbox->ascwrap_v6; + return 0; +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_get_lifecycle); + +int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox) +{ + if (!mbox || !mbox->hw->is_ascwrap_v6) + return -EOPNOTSUPP; + if (READ_ONCE(mbox->removing)) + return -ENODEV; + + /* Future provider revisions may report additional lifecycle work. */ + if (mbox->ascwrap_v6.missing) + return -EOPNOTSUPP; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle); + +static void apple_mbox_ascwrap_v6_stop_cpu_locked(struct apple_mbox *mbox) +{ + void __iomem *cpu_control; + u32 val; + + if (!mbox->ascwrap_v6_cpu_running) + return; + + cpu_control = (u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V6_CPU_CONTROL; + + /* AppleASCWrapV6::_runCPU(false): clear RUN, reread, clear bit 5. */ + val = readl(cpu_control); + writel(val & ~APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); + val = readl(cpu_control); + writel(val & ~APPLE_ASCWRAP_V6_CPU_STOP_SECOND_CLEAR, cpu_control); + + mbox->ascwrap_v6_cpu_running = false; + pm_runtime_mark_last_busy(mbox->dev); + pm_runtime_put_autosuspend(mbox->dev); +} + +/* + * A failed resume leaves the runtime PM error set, and runtime PM then fails + * every later resume with -EINVAL without retrying the power domain. The + * provider keeps no state across a failed resume, so mark it suspended again + * and let the next start retry the domain. + */ +static void apple_mbox_clear_resume_error(struct apple_mbox *mbox) +{ + if (mbox->dev->power.runtime_error) + pm_runtime_set_suspended(mbox->dev); +} + +int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox) +{ + void __iomem *cpu_control; + u32 val; + int ret = 0; + + if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs) + return -EOPNOTSUPP; + + mutex_lock(&mbox->lifecycle_lock); + if (mbox->removing) { + ret = -ENODEV; + goto out_unlock; + } + if (mbox->ascwrap_v6.missing) { + ret = -EOPNOTSUPP; + goto out_unlock; + } + if (mbox->ascwrap_v6_cpu_running) { + ret = -EBUSY; + goto out_unlock; + } + + ret = pm_runtime_resume_and_get(mbox->dev); + if (ret) { + dev_err(mbox->dev, + "ASC CPU start: runtime resume failed: %d (active=%d status=%d disable_depth=%d runtime_error=%d)\n", + ret, mbox->active, mbox->dev->power.runtime_status, + mbox->dev->power.disable_depth, + mbox->dev->power.runtime_error); + apple_mbox_clear_resume_error(mbox); + goto out_unlock; + } + + cpu_control = (u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V6_CPU_CONTROL; + + /* AppleASCWrapV6::_runCPU(true): read, set RUN, write. */ + val = readl(cpu_control); + writel(val | APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); + mbox->ascwrap_v6_cpu_running = true; + +out_unlock: + mutex_unlock(&mbox->lifecycle_lock); + return ret; +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_start_cpu); + +int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox) +{ + int ret = 0; + + if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs) + return -EOPNOTSUPP; + + mutex_lock(&mbox->lifecycle_lock); + if (mbox->removing) { + ret = -ENODEV; + goto out_unlock; + } + + apple_mbox_ascwrap_v6_stop_cpu_locked(mbox); + +out_unlock: + mutex_unlock(&mbox->lifecycle_lock); + return ret; +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_stop_cpu); + int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, bool atomic) { @@ -118,6 +280,10 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, long t; spin_lock_irqsave(&mbox->tx_lock, flags); + if (unlikely(READ_ONCE(mbox->removing))) { + spin_unlock_irqrestore(&mbox->tx_lock, flags); + return -ENODEV; + } mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); while (mbox_ctrl & mbox->hw->control_full) { @@ -169,6 +335,10 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, } spin_lock_irqsave(&mbox->tx_lock, flags); + if (unlikely(READ_ONCE(mbox->removing))) { + spin_unlock_irqrestore(&mbox->tx_lock, flags); + return -ENODEV; + } mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); } @@ -210,10 +380,7 @@ static int apple_mbox_poll_locked(struct apple_mbox *mbox) int ret = 0; u32 mbox_ctrl; - /* - * A detached receiver (RTKit teardown in progress) leaves firmware - * traffic in the FIFO; it is drained again once a receiver is attached. - */ + /* A stopped receiver may still have firmware traffic in its FIFO. */ if (!mbox->rx) return 0; mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->i2a_control); @@ -250,7 +417,8 @@ static irqreturn_t apple_mbox_recv_irq(int irq, void *data) struct apple_mbox *mbox = data; spin_lock(&mbox->rx_lock); - apple_mbox_poll_locked(mbox); + if (likely(!READ_ONCE(mbox->removing))) + apple_mbox_poll_locked(mbox); spin_unlock(&mbox->rx_lock); return IRQ_HANDLED; @@ -262,7 +430,10 @@ int apple_mbox_poll(struct apple_mbox *mbox) int ret; spin_lock_irqsave(&mbox->rx_lock, flags); - ret = apple_mbox_poll_locked(mbox); + if (unlikely(READ_ONCE(mbox->removing))) + ret = -ENODEV; + else + ret = apple_mbox_poll_locked(mbox); spin_unlock_irqrestore(&mbox->rx_lock, flags); return ret; @@ -272,14 +443,26 @@ EXPORT_SYMBOL(apple_mbox_poll); int apple_mbox_start(struct apple_mbox *mbox) { u32 control; - int ret; + int ret = 0; + mutex_lock(&mbox->lifecycle_lock); + if (mbox->removing) { + ret = -ENODEV; + goto out_unlock; + } if (mbox->active) - return 0; + goto out_unlock; ret = pm_runtime_resume_and_get(mbox->dev); - if (ret) - return ret; + if (ret) { + dev_err(mbox->dev, + "mailbox start: runtime resume failed: %d (active=%d status=%d disable_depth=%d runtime_error=%d)\n", + ret, mbox->active, mbox->dev->power.runtime_status, + mbox->dev->power.disable_depth, + mbox->dev->power.runtime_error); + apple_mbox_clear_resume_error(mbox); + goto out_unlock; + } if (mbox->hw->ap_initializes_mailboxes) { /* The primary T8140 outbox requires its enable bit before use. */ @@ -303,11 +486,14 @@ int apple_mbox_start(struct apple_mbox *mbox) enable_irq(mbox->irq_recv_not_empty); mbox->active = true; - return 0; + +out_unlock: + mutex_unlock(&mbox->lifecycle_lock); + return ret; } EXPORT_SYMBOL(apple_mbox_start); -void apple_mbox_stop(struct apple_mbox *mbox) +static void apple_mbox_stop_locked(struct apple_mbox *mbox) { if (!mbox->active) return; @@ -317,8 +503,80 @@ void apple_mbox_stop(struct apple_mbox *mbox) pm_runtime_mark_last_busy(mbox->dev); pm_runtime_put_autosuspend(mbox->dev); } + +void apple_mbox_stop(struct apple_mbox *mbox) +{ + mutex_lock(&mbox->lifecycle_lock); + apple_mbox_stop_locked(mbox); + mutex_unlock(&mbox->lifecycle_lock); +} EXPORT_SYMBOL(apple_mbox_stop); +/* + * Stop the provider-owned CPU, then quiesce every transport edge before + * devres releases IRQs or MMIO mappings. Device links order consumer removal + * first; the callback clear and waiter wake also cover future consumers that + * violate that ownership rule. + */ +static void apple_mbox_quiesce(void *data) +{ + struct apple_mbox *mbox = data; + unsigned long flags; + bool had_rx; + bool was_active; + + mutex_lock(&mbox->lifecycle_lock); + if (mbox->removing) { + mutex_unlock(&mbox->lifecycle_lock); + return; + } + apple_mbox_ascwrap_v6_stop_cpu_locked(mbox); + + /* Serialize the removal fence with every MMIO send critical section. */ + spin_lock_irqsave(&mbox->tx_lock, flags); + WRITE_ONCE(mbox->removing, true); + spin_unlock_irqrestore(&mbox->tx_lock, flags); + + /* + * Publish the inactive state under the lifecycle lock, then drop it + * before disable_irq() waits for a callback. A callback is allowed to + * finish a send or stop request without an rx_lock -> lifecycle_lock + * inversion. The removal fence prevents any matching restart. + */ + was_active = mbox->active; + mbox->active = false; + mutex_unlock(&mbox->lifecycle_lock); + if (was_active) { + disable_irq(mbox->irq_recv_not_empty); + pm_runtime_mark_last_busy(mbox->dev); + pm_runtime_put_autosuspend(mbox->dev); + } + + /* Serialize once-only masking with the IRQ and send-error paths. */ + spin_lock_irqsave(&mbox->tx_lock, flags); + if (mbox->tx_irq_unmasked) { + disable_irq_nosync(mbox->irq_send_empty); + mbox->tx_irq_unmasked = false; + } + spin_unlock_irqrestore(&mbox->tx_lock, flags); + /* The handler takes tx_lock, so drain it after releasing the lock. */ + synchronize_irq(mbox->irq_send_empty); + complete_all(&mbox->tx_empty); + synchronize_irq(mbox->irq_recv_not_empty); + + spin_lock_irqsave(&mbox->rx_lock, flags); + had_rx = mbox->rx; + mbox->rx = NULL; + mbox->cookie = NULL; + spin_unlock_irqrestore(&mbox->rx_lock, flags); + + if (WARN_ON_ONCE(had_rx)) + dev_warn(mbox->dev, + "consumer callback survived provider remove ordering\n"); + + pm_runtime_barrier(mbox->dev); +} + struct apple_mbox *apple_mbox_get(struct device *dev, int index) { struct of_phandle_args args; @@ -371,9 +629,10 @@ static int apple_mbox_probe(struct platform_device *pdev) { int ret; char *irqname; + const char *firmware_role; struct apple_mbox *mbox; struct device *dev = &pdev->dev; - struct resource *res; + struct resource *res, *iop_vbar_res; mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL); if (!mbox) @@ -384,13 +643,75 @@ static int apple_mbox_probe(struct platform_device *pdev) if (!mbox->hw) return -EINVAL; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res || resource_size(res) < mbox->hw->i2a_recv1 + sizeof(u64)) - return dev_err_probe(dev, -EINVAL, "mailbox register window is too small\n"); + if (mbox->hw->is_ascwrap_v6) { + /* + * Validate the entire provider identity before mapping either + * resource. These are fixed T8140 CPU-physical windows and role + * assignments, not generic consumer-provided aliases. + */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "wrapper"); + iop_vbar_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "iop-vbar"); + if (!res || !iop_vbar_res || + resource_size(res) != APPLE_ASCWRAP_V6_WRAPPER_SIZE || + resource_size(iop_vbar_res) != APPLE_ASCWRAP_V6_IOP_VBAR_SIZE) + return dev_err_probe(dev, -EINVAL, + "invalid ASCWrap v6 provider resources\n"); + + ret = of_property_read_string(dev->of_node, "apple,firmware-role", + &firmware_role); + if (ret) + return dev_err_probe(dev, ret, + "missing ASCWrap v6 firmware role\n"); + + if (!strcmp(firmware_role, "GFX")) { + mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX; + if (res->start != APPLE_T8140_GFX_WRAPPER_BASE || + iop_vbar_res->start != APPLE_T8140_GFX_IOP_VBAR_BASE) + return dev_err_probe(dev, -EINVAL, + "GFX ASCWrap v6 resources do not match T8140\n"); + } else if (!strcmp(firmware_role, "GFX1")) { + mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1; + if (res->start != APPLE_T8140_GFX1_WRAPPER_BASE || + iop_vbar_res->start != APPLE_T8140_GFX1_IOP_VBAR_BASE) + return dev_err_probe(dev, -EINVAL, + "GFX1 ASCWrap v6 resources do not match T8140\n"); + } else { + return dev_err_probe(dev, -EINVAL, + "invalid ASCWrap v6 firmware role\n"); + } - mbox->regs = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(mbox->regs)) - return PTR_ERR(mbox->regs); + mbox->ascwrap_v6.wrapper_start = res->start; + mbox->ascwrap_v6.wrapper_size = resource_size(res); + mbox->ascwrap_v6.iop_vbar_start = iop_vbar_res->start; + mbox->ascwrap_v6.iop_vbar_size = resource_size(iop_vbar_res); + /* resetState() is software-only; the matching stop lives above. */ + mbox->ascwrap_v6.missing = 0; + + mbox->wrapper_regs = + devm_platform_ioremap_resource_byname(pdev, "wrapper"); + if (IS_ERR(mbox->wrapper_regs)) + return PTR_ERR(mbox->wrapper_regs); + + mbox->iop_vbar_regs = + devm_platform_ioremap_resource_byname(pdev, "iop-vbar"); + if (IS_ERR(mbox->iop_vbar_regs)) + return PTR_ERR(mbox->iop_vbar_regs); + + mbox->regs = (u8 __iomem *)mbox->wrapper_regs + + mbox->hw->reg_offset; + } else { + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res || resource_size(res) < mbox->hw->reg_offset + + mbox->hw->i2a_recv1 + sizeof(u64)) + return dev_err_probe(dev, -EINVAL, + "mailbox register window is too small\n"); + + mbox->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(mbox->regs)) + return PTR_ERR(mbox->regs); + mbox->regs = (u8 __iomem *)mbox->regs + mbox->hw->reg_offset; + } mbox->irq_recv_not_empty = platform_get_irq_byname(pdev, "recv-not-empty"); @@ -403,6 +724,7 @@ static int apple_mbox_probe(struct platform_device *pdev) spin_lock_init(&mbox->rx_lock); spin_lock_init(&mbox->tx_lock); + mutex_init(&mbox->lifecycle_lock); init_completion(&mbox->tx_empty); irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-recv", dev_name(dev)); @@ -426,6 +748,9 @@ static int apple_mbox_probe(struct platform_device *pdev) return ret; ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; + ret = devm_add_action_or_reset(dev, apple_mbox_quiesce, mbox); if (ret) return ret; @@ -433,6 +758,15 @@ static int apple_mbox_probe(struct platform_device *pdev) return 0; } +static void apple_mbox_remove(struct platform_device *pdev) +{ + struct apple_mbox *mbox = platform_get_drvdata(pdev); + + /* Block new phandle lookups before quiescing the provider. */ + platform_set_drvdata(pdev, NULL); + apple_mbox_quiesce(mbox); +} + static const struct apple_mbox_hw apple_mbox_t8015_hw = { .control_full = APPLE_ASC_MBOX_CONTROL_FULL, .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, @@ -480,6 +814,24 @@ static const struct apple_mbox_hw apple_mbox_t8140_asc_hw = { .has_irq_controls = false, }; +static const struct apple_mbox_hw apple_mbox_ascwrap_v6_hw = { + .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, + .is_ascwrap_v6 = true, + + .control_full = APPLE_ASC_MBOX_CONTROL_FULL, + .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, + + .a2i_control = APPLE_ASC_MBOX_A2I_CONTROL, + .a2i_send0 = APPLE_ASC_MBOX_A2I_SEND0, + .a2i_send1 = APPLE_ASC_MBOX_A2I_SEND1, + + .i2a_control = APPLE_ASC_MBOX_I2A_CONTROL, + .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, + .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, + + .has_irq_controls = false, +}; + static const struct apple_mbox_hw apple_mbox_t8140_aop_setup_hw = { .control_full = APPLE_ASC_MBOX_CONTROL_FULL, .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, @@ -515,6 +867,7 @@ static const struct apple_mbox_hw apple_mbox_m3_hw = { }; static const struct of_device_id apple_mbox_of_match[] = { + { .compatible = "apple,t8140-ascwrap-v6", .data = &apple_mbox_ascwrap_v6_hw }, { .compatible = "apple,t8140-asc-mailbox", .data = &apple_mbox_t8140_asc_hw }, { .compatible = "apple,t8140-aop-setup-mailbox", .data = &apple_mbox_t8140_aop_setup_hw }, { .compatible = "apple,asc-mailbox-v4", .data = &apple_mbox_asc_hw }, @@ -530,6 +883,7 @@ static struct platform_driver apple_mbox_driver = { .of_match_table = apple_mbox_of_match, }, .probe = apple_mbox_probe, + .remove = apple_mbox_remove, }; module_platform_driver(apple_mbox_driver); diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index 9cfbbac411b8e..0d6ad8b16ae4c 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -9,6 +9,7 @@ #define _APPLE_MAILBOX_H_ #include +#include #include /* encodes a single 96bit message sent over the single channel */ @@ -17,11 +18,54 @@ struct apple_mbox_msg { u32 msg1; }; +/** + * enum apple_mbox_ascwrap_v6_role - firmware role owned by an ASCWrap v6 + * mailbox provider + * @APPLE_MBOX_ASCWRAP_V6_ROLE_NONE: not an ASCWrap v6 provider + * @APPLE_MBOX_ASCWRAP_V6_ROLE_GFX: primary GPU firmware role + * @APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1: secondary GPU firmware role + */ +enum apple_mbox_ascwrap_v6_role { + APPLE_MBOX_ASCWRAP_V6_ROLE_NONE, + APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, + APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1, +}; + +/** + * struct apple_mbox_ascwrap_v6_lifecycle - provider-owned lifecycle facts + * @role: firmware role owned by this provider + * @wrapper_start: CPU physical base of the provider's ASCWrap window + * @wrapper_size: size of the provider's ASCWrap window + * @iop_vbar_start: CPU physical base of the provider's IOP VBAR window + * @iop_vbar_size: size of the provider's IOP VBAR window + * @missing: remaining provider lifecycle work; zero for T8140 + * + * The apple-mailbox provider owns both mappings. Consumers request CPU + * transitions through the provider and never map either resource themselves. + * T8140 has a complete matching start and stop transform. AppleA7IOP's + * resetState() only updates software state and performs no reset MMIO. + */ +struct apple_mbox_ascwrap_v6_lifecycle { + enum apple_mbox_ascwrap_v6_role role; + u64 wrapper_start; + u64 wrapper_size; + u64 iop_vbar_start; + u64 iop_vbar_size; + u32 missing; +}; + struct apple_mbox { struct device *dev; void __iomem *regs; + void __iomem *wrapper_regs; + void __iomem *iop_vbar_regs; const struct apple_mbox_hw *hw; + struct apple_mbox_ascwrap_v6_lifecycle ascwrap_v6; + /* Serializes transport start/stop against provider removal. */ + struct mutex lifecycle_lock; bool active; + bool ascwrap_v6_cpu_running; + bool removing; int irq_recv_not_empty; int irq_send_empty; @@ -35,7 +79,11 @@ struct apple_mbox { void (*rx)(struct apple_mbox *mbox, struct apple_mbox_msg msg, void *cookie); void *cookie; - /* Protected by tx_lock; send-error and IRQ paths mask this only once. */ + /* + * Protected by tx_lock. IRQ, send-error and provider-removal paths + * mask an outstanding unmask only once; rebuild every consumer when + * changing this internal structure's layout. + */ bool tx_irq_unmasked; }; @@ -48,4 +96,11 @@ int apple_mbox_poll(struct apple_mbox *mbox); int apple_mbox_send(struct apple_mbox *mbox, struct apple_mbox_msg msg, bool atomic); +int apple_mbox_ascwrap_v6_get_lifecycle( + struct apple_mbox *mbox, + struct apple_mbox_ascwrap_v6_lifecycle *lifecycle); +int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox); +int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox); +int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox); + #endif From e841f72a75076bb8d09f3c0112ca3cb962576f34 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:08:35 -0500 Subject: [PATCH 0287/2680] dt-bindings: mailbox: apple,mailbox: Allow the ANE mailbox variant The T6021 (M2 Max) ANE coprocessor mailbox is an ASC mailbox, but only one of its interrupts is known: the Apple device tree gives the ANE a single interrupt line. The binding requires the four ASC mailbox interrupts in a fixed order, so a node for this mailbox fails dtbs_check. Add the apple,t6021-ane-mailbox compatible with the apple,asc-mailbox-v4 fallback. For it, recv-not-empty is the only required interrupt; send-empty, send-not-empty and recv-empty may follow in that order. Without send-empty, the OS must poll the FIFO status before it sends. The other variants keep their constraints. The four-interrupt rule moves into a condition that excludes the ANE and AOP setup mailboxes, and the AOP setup mailbox states its minimum of two interrupts, which the top-level limit gave it before. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- .../bindings/mailbox/apple,mailbox.yaml | 42 +++++++++++++++++-- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index 21dc2231d88ba..5574069b10d70 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -45,6 +45,13 @@ properties: - const: apple,t8140-gfx-asc-mailbox - const: apple,asc-mailbox-v4 + - description: + The T6021 ANE coprocessor mailbox is an ASC mailbox. Only its + recv-not-empty interrupt is known. + items: + - const: apple,t6021-ane-mailbox + - const: apple,asc-mailbox-v4 + - description: AOP setup-port mailbox with a separate register layout. const: apple,t8140-aop-setup-mailbox @@ -71,11 +78,11 @@ properties: maxItems: 1 interrupts: - minItems: 2 + minItems: 1 maxItems: 4 interrupt-names: - minItems: 2 + minItems: 1 maxItems: 4 iommus: @@ -104,6 +111,7 @@ allOf: then: properties: interrupts: + minItems: 2 maxItems: 2 interrupt-names: items: @@ -113,6 +121,34 @@ allOf: - iommus - apple,dma-range else: + properties: + iommus: false + apple,dma-range: false + + - if: + properties: + compatible: + contains: + const: apple,t6021-ane-mailbox + then: + properties: + interrupt-names: + minItems: 1 + items: + - const: recv-not-empty + - const: send-empty + - const: send-not-empty + - const: recv-empty + + - if: + properties: + compatible: + not: + contains: + enum: + - apple,t6021-ane-mailbox + - apple,t8140-aop-setup-mailbox + then: properties: interrupts: minItems: 4 @@ -122,8 +158,6 @@ allOf: - const: send-not-empty - const: recv-empty - const: recv-not-empty - iommus: false - apple,dma-range: false required: - compatible From b77e6daba4cf1b31a71cb60ff4c39302315a681c Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:08:36 -0500 Subject: [PATCH 0288/2680] soc: apple: mailbox: Make the send-empty IRQ optional (poll TX) The T6021 (M2 Max) ANE coprocessor mailbox has one known AIC line, which the device tree names recv-not-empty. apple-mailbox insists on a send-empty IRQ as well, so it cannot bind that mailbox. Look send-empty up with platform_get_irq_byname_optional(). When the node has none (-ENXIO), do not request it, and let apple_mbox_send() wait for FIFO space by polling the A2I control register, the way its atomic path already does (100 us steps, 500 ms timeout, under tx_lock). Any other lookup error still fails the probe with -ENODEV. A mailbox whose node names send-empty takes the old path unchanged: the IRQ is requested at the same point with the same flags and name, and a non-atomic send still sleeps on tx_empty. All 71 apple-mailbox nodes in arch/arm64/boot/dts/apple name send-empty today. The Rust wrapper (RUST_APPLE_MAILBOX) needs no change. It only calls apple_mbox_get_byname(), apple_mbox_start(), apple_mbox_send() and apple_mbox_stop(), and never reads the IRQ fields, so a Rust client gets the poll through apple_mbox_send(). An equivalent change ran on an M2 Max lab kernel (Asahi 7.1.13 base) from 2026-09-24 to 2026-09-30, where apple-mailbox bound the one-interrupt ANE mailbox. The FIFO-full poll itself did not run there: with the ANE firmware in use (from macOS 13.5), the host sends no mailbox message after probe. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/soc/apple/mailbox.c | 31 ++++++++++++++++++++----------- include/linux/soc/apple/mailbox.h | 2 +- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index ba8d124e853b1..0a9f051a6d9bd 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -287,7 +287,7 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); while (mbox_ctrl & mbox->hw->control_full) { - if (atomic) { + if (atomic || mbox->irq_send_empty < 0) { ret = readl_poll_timeout_atomic( mbox->regs + mbox->hw->a2i_control, mbox_ctrl, !(mbox_ctrl & mbox->hw->control_full), 100, @@ -718,8 +718,13 @@ static int apple_mbox_probe(struct platform_device *pdev) if (mbox->irq_recv_not_empty < 0) return -ENODEV; - mbox->irq_send_empty = platform_get_irq_byname(pdev, "send-empty"); - if (mbox->irq_send_empty < 0) + /* + * Some coprocessors (e.g. the T6021 ANE ASC) have no send-empty line. + * apple_mbox_send then polls the A2I control register instead. + */ + mbox->irq_send_empty = + platform_get_irq_byname_optional(pdev, "send-empty"); + if (mbox->irq_send_empty < 0 && mbox->irq_send_empty != -ENXIO) return -ENODEV; spin_lock_init(&mbox->rx_lock); @@ -737,15 +742,19 @@ static int apple_mbox_probe(struct platform_device *pdev) if (ret) return ret; - irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-send", dev_name(dev)); - if (!irqname) - return -ENOMEM; + if (mbox->irq_send_empty >= 0) { + irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-send", + dev_name(dev)); + if (!irqname) + return -ENOMEM; - ret = devm_request_irq(dev, mbox->irq_send_empty, - apple_mbox_send_empty_irq, - IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, irqname, mbox); - if (ret) - return ret; + ret = devm_request_irq(dev, mbox->irq_send_empty, + apple_mbox_send_empty_irq, + IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, + irqname, mbox); + if (ret) + return ret; + } ret = devm_pm_runtime_enable(dev); if (ret) diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index 0d6ad8b16ae4c..e5697975a50b5 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -68,7 +68,7 @@ struct apple_mbox { bool removing; int irq_recv_not_empty; - int irq_send_empty; + int irq_send_empty; /* < 0: no send-empty IRQ, TX polls */ spinlock_t rx_lock; spinlock_t tx_lock; From e4f97678560bd447439fecefb4db5c53ee6015d0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:32:38 +1100 Subject: [PATCH 0289/2680] soc: apple: mailbox: drain only a present send-empty IRQ PR 65 permits polling when the send-empty IRQ is absent. When composed with the newer provider removal fence, synchronize_irq received that negative optional-resource sentinel even though no IRQ was requested. Drain the TX handler only when its IRQ exists, preserving the RX drain and TX waiter wakeup in both modes. Also return the actual probe IRQ lookup error, including EPROBE_DEFER, while retaining ENXIO as the accepted absent optional TX line. (cherry picked from commit c127de8760a8b66cee210702d6cd16d21959e448) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/mailbox.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 0a9f051a6d9bd..2e22c78f57d69 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -560,7 +560,8 @@ static void apple_mbox_quiesce(void *data) } spin_unlock_irqrestore(&mbox->tx_lock, flags); /* The handler takes tx_lock, so drain it after releasing the lock. */ - synchronize_irq(mbox->irq_send_empty); + if (mbox->irq_send_empty >= 0) + synchronize_irq(mbox->irq_send_empty); complete_all(&mbox->tx_empty); synchronize_irq(mbox->irq_recv_not_empty); @@ -716,7 +717,7 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->irq_recv_not_empty = platform_get_irq_byname(pdev, "recv-not-empty"); if (mbox->irq_recv_not_empty < 0) - return -ENODEV; + return mbox->irq_recv_not_empty; /* * Some coprocessors (e.g. the T6021 ANE ASC) have no send-empty line. @@ -725,7 +726,7 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->irq_send_empty = platform_get_irq_byname_optional(pdev, "send-empty"); if (mbox->irq_send_empty < 0 && mbox->irq_send_empty != -ENXIO) - return -ENODEV; + return mbox->irq_send_empty; spin_lock_init(&mbox->rx_lock); spin_lock_init(&mbox->tx_lock); From 6048520f889bf07c7eb83155c15f240b2421207e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:28:52 +1000 Subject: [PATCH 0290/2680] soc: apple: rtkit: recognise an initial crash log in adopted sessions Map the first crash-buffer request and validate a complete public CLHE header before decoding it. A fresh session's zeroed buffer remains an ordinary allocation request. An inherited session has already negotiated its buffer: mapping failure or an unreadable header must still notify the client of the crash, with an empty log when necessary. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit-crashlog.c | 14 ++-------- drivers/soc/apple/rtkit-internal.h | 11 ++++++++ drivers/soc/apple/rtkit.c | 43 ++++++++++++++++-------------- 3 files changed, 36 insertions(+), 32 deletions(-) diff --git a/drivers/soc/apple/rtkit-crashlog.c b/drivers/soc/apple/rtkit-crashlog.c index 8319e365110b8..5defa12862b15 100644 --- a/drivers/soc/apple/rtkit-crashlog.c +++ b/drivers/soc/apple/rtkit-crashlog.c @@ -8,7 +8,6 @@ #define FOURCC(a, b, c, d) \ (((u32)(a) << 24) | ((u32)(b) << 16) | ((u32)(c) << 8) | ((u32)(d))) -#define APPLE_RTKIT_CRASHLOG_HEADER FOURCC('C', 'L', 'H', 'E') #define APPLE_RTKIT_CRASHLOG_STR FOURCC('C', 's', 't', 'r') #define APPLE_RTKIT_CRASHLOG_VERSION FOURCC('C', 'v', 'e', 'r') #define APPLE_RTKIT_CRASHLOG_MBOX FOURCC('C', 'm', 'b', 'x') @@ -25,15 +24,6 @@ #define PSR_MODE_MASK 0x0000000f #endif -struct apple_rtkit_crashlog_header { - u32 fourcc; - u32 version; - u32 size; - u32 flags; - u8 _unk[16]; -}; -static_assert(sizeof(struct apple_rtkit_crashlog_header) == 0x20); - struct apple_rtkit_crashlog_mbox_entry { u64 msg0; u64 msg1; @@ -190,7 +180,7 @@ void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size) struct apple_rtkit_crashlog_header header; memcpy(&header, bfr, sizeof(header)); - if (header.fourcc != APPLE_RTKIT_CRASHLOG_HEADER) { + if (header.fourcc != APPLE_RTKIT_CRASHLOG_HEADER_FOURCC) { dev_warn(rtk->dev, "RTKit: Expected crashlog header but got %x", header.fourcc); return; @@ -210,7 +200,7 @@ void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size) memcpy(§ion_size, bfr + offset + 12, 4); switch (section_fourcc) { - case APPLE_RTKIT_CRASHLOG_HEADER: + case APPLE_RTKIT_CRASHLOG_HEADER_FOURCC: dev_dbg(rtk->dev, "RTKit: End of crashlog reached"); return; case APPLE_RTKIT_CRASHLOG_STR: diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index 56d7fea01fade..927f33f1642e6 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -21,6 +21,17 @@ #define APPLE_RTKIT_APP_ENDPOINT_START 0x20 #define APPLE_RTKIT_MAX_ENDPOINTS 0x100 +#define APPLE_RTKIT_CRASHLOG_HEADER_FOURCC 0x434c4845 + +struct apple_rtkit_crashlog_header { + u32 fourcc; + u32 version; + u32 size; + u32 flags; + u8 _unk[16]; +}; + +static_assert(sizeof(struct apple_rtkit_crashlog_header) == 0x20); struct apple_rtkit { void *cookie; diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 1e01e1d3e3a61..28532fa437dc8 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -410,6 +410,7 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) { u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg); u8 *bfr __free(kfree) = NULL; + struct apple_rtkit_crashlog_header header; if (type != APPLE_RTKIT_CRASHLOG_CRASH) { dev_warn(rtk->dev, "RTKit: Unknown crashlog message: %llx\n", @@ -417,27 +418,19 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) return; } - if (rtk->crashlog_inherited && !rtk->crashlog_buffer.size) { - /* - * The previous owner negotiated the crashlog buffer, so this - * session never learned its geometry. A CRASH message on an - * inherited session is therefore a crash notification, not a - * buffer request, and must not be answered as one. - */ - dev_err(rtk->dev, - "RTKit: adopted co-processor has crashed (crashlog at %#llx, %#llx bytes, not mapped)\n", - (u64)FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg), - (u64)FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg) << 12); - rtk->crashed = true; - if (rtk->ops->crashed) - rtk->ops->crashed(rtk->cookie, NULL, 0); - return; - } - if (!rtk->crashlog_buffer.size) { - apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer, - APPLE_RTKIT_EP_CRASHLOG, msg); - return; + if (apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer, + APPLE_RTKIT_EP_CRASHLOG, msg)) + goto unmapped; + if (rtk->crashlog_buffer.size < sizeof(header)) + goto unmapped; + + apple_rtkit_memcpy(rtk, &header, &rtk->crashlog_buffer, 0, + sizeof(header)); + if (header.fourcc != APPLE_RTKIT_CRASHLOG_HEADER_FOURCC || + header.size < sizeof(header) || + header.size > rtk->crashlog_buffer.size) + goto unmapped; } dev_err(rtk->dev, "RTKit: co-processor has crashed\n"); @@ -460,6 +453,16 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) rtk->crashed = true; if (rtk->ops->crashed) rtk->ops->crashed(rtk->cookie, bfr, bfr ? rtk->crashlog_buffer.size : 0); + return; + +unmapped: + /* An inherited session already negotiated its crash buffer. */ + if (!rtk->crashlog_inherited) + return; + dev_err(rtk->dev, "RTKit: adopted co-processor crashed without a readable log\n"); + rtk->crashed = true; + if (rtk->ops->crashed) + rtk->ops->crashed(rtk->cookie, NULL, 0); } static void apple_rtkit_ioreport_rx(struct apple_rtkit *rtk, u64 msg) From 0ee52a3560ba40390fd0b7d30fe3bd082478bc70 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 01:21:08 +1000 Subject: [PATCH 0291/2680] dt-bindings: spi: apple,spi: Add t8140-spi compatible The SPI controllers on the Apple A18 Pro (T8140) are compatible with the T8103 ones. Follow the T6020 entry and use "apple,t8103-spi" as the fallback rather than the generic "apple,spi". Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/spi/apple,spi.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/spi/apple,spi.yaml b/Documentation/devicetree/bindings/spi/apple,spi.yaml index 9356b9c337c8c..4b30265686fa1 100644 --- a/Documentation/devicetree/bindings/spi/apple,spi.yaml +++ b/Documentation/devicetree/bindings/spi/apple,spi.yaml @@ -16,7 +16,9 @@ properties: compatible: oneOf: - items: - - const: apple,t6020-spi + - enum: + - apple,t6020-spi + - apple,t8140-spi - const: apple,t8103-spi - items: - enum: From 0a9e16102fc4b2aea7e187c6d4986298fb623d84 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:19:21 +1100 Subject: [PATCH 0292/2680] soc: apple: rtkit: reject an already owned mailbox receiver Keep receiver ownership reserved until teardown finishes. A competing initialization must return EBUSY instead of overwriting the callback and cookie, and must not stop the mailbox owned by the existing receiver. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 28532fa437dc8..ae9589dff3485 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -784,18 +784,21 @@ static void apple_rtkit_mark_running(struct apple_rtkit *rtk) rtk->crashlog_inherited = true; } -static void apple_rtkit_claim_rx(struct apple_rtkit *rtk) +static int apple_rtkit_claim_rx(struct apple_rtkit *rtk) { struct apple_mbox *mbox = rtk->mbox; unsigned long flags; spin_lock_irqsave(&mbox->rx_lock, flags); - if (mbox->rx || mbox->cookie) - dev_warn(rtk->dev, - "RTKit: mailbox receiver already claimed, taking it over\n"); + if (mbox->rx || mbox->cookie) { + spin_unlock_irqrestore(&mbox->rx_lock, flags); + return -EBUSY; + } mbox->cookie = rtk; mbox->rx = apple_rtkit_rx; spin_unlock_irqrestore(&mbox->rx_lock, flags); + + return 0; } static void apple_rtkit_detach_rx(struct apple_rtkit *rtk) @@ -878,7 +881,9 @@ static struct apple_rtkit *__apple_rtkit_init(struct device *dev, void *cookie, if (adopted) apple_rtkit_mark_running(rtk); - apple_rtkit_claim_rx(rtk); + ret = apple_rtkit_claim_rx(rtk); + if (ret) + goto destroy_wq; ret = apple_mbox_start(rtk->mbox); if (ret) @@ -889,6 +894,9 @@ static struct apple_rtkit *__apple_rtkit_init(struct device *dev, void *cookie, stop_rx: apple_rtkit_stop_rx(rtk); apple_rtkit_release_rx(rtk); + goto free_rtk; +destroy_wq: + destroy_workqueue(rtk->wq); free_rtk: kfree(rtk); return ERR_PTR(ret); From 05fa517285fc11607284f9cc524e7b219d3bc17b Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:17:02 -0500 Subject: [PATCH 0293/2680] dt-bindings: mailbox: apple,mailbox: Add the T8112 ANE mailbox The T8112 (M2) ANE coprocessor has an ASC mailbox at the same offset in its block as the T6021 ANE, and the Apple device tree also gives that ANE a single interrupt line (520). Add apple,t8112-ane-mailbox to the ANE mailbox variant, with the same interrupt rules as apple,t6021-ane-mailbox. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- .../devicetree/bindings/mailbox/apple,mailbox.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index 5574069b10d70..e02b57d71f435 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -46,10 +46,12 @@ properties: - const: apple,asc-mailbox-v4 - description: - The T6021 ANE coprocessor mailbox is an ASC mailbox. Only its - recv-not-empty interrupt is known. + The T6021 and T8112 ANE coprocessor mailboxes are ASC mailboxes. + Only their recv-not-empty interrupt is known. items: - - const: apple,t6021-ane-mailbox + - enum: + - apple,t6021-ane-mailbox + - apple,t8112-ane-mailbox - const: apple,asc-mailbox-v4 - description: @@ -129,7 +131,9 @@ allOf: properties: compatible: contains: - const: apple,t6021-ane-mailbox + enum: + - apple,t6021-ane-mailbox + - apple,t8112-ane-mailbox then: properties: interrupt-names: @@ -147,6 +151,7 @@ allOf: contains: enum: - apple,t6021-ane-mailbox + - apple,t8112-ane-mailbox - apple,t8140-aop-setup-mailbox then: properties: From d1dcc1e42f140e6a6072dd6eda803463dea5b196 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 28 Sep 2026 21:24:42 +1000 Subject: [PATCH 0294/2680] cpuidle: apple: Restrict T8140 to WFI until CPU PD is fixed Register the existing Apple cpuidle driver on T8140, but expose only its WFI state. The CPU PD state uses an unverified power-down and wake sequence on this SoC. Marking that state off by default still leaves a per-CPU sysfs control that can enable it at runtime. A guarded CPU PD opt-in on a J700 made the machine lose wired reachability. Recovery required a VDM reset. The target-side log was unavailable after recovery, so the failing core and instruction are unknown. Keep the CPU PD state out of the driver's registered state count until the sequence and wake path have been fixed and qualified. Other Apple SoCs retain both states. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/cpuidle/cpuidle-apple.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/cpuidle/cpuidle-apple.c b/drivers/cpuidle/cpuidle-apple.c index 3d2b804df8a2d..67c7846111cf3 100644 --- a/drivers/cpuidle/cpuidle-apple.c +++ b/drivers/cpuidle/cpuidle-apple.c @@ -124,6 +124,10 @@ static struct cpuidle_driver apple_idle_driver = { static int apple_cpuidle_probe(struct platform_device *pdev) { + /* A T8140 CPU PD opt-in hung J700; expose only WFI. */ + if (of_machine_is_compatible("apple,t8140")) + apple_idle_driver.state_count = STATE_WFI + 1; + return cpuidle_register(&apple_idle_driver, NULL); } @@ -158,7 +162,8 @@ static int __init apple_cpuidle_init(void) of_machine_is_compatible("apple,t6030") || of_machine_is_compatible("apple,t6031") || of_machine_is_compatible("apple,t6032") || - of_machine_is_compatible("apple,t6034"))) + of_machine_is_compatible("apple,t6034") || + of_machine_is_compatible("apple,t8140"))) return 0; pdev = platform_device_register_simple("cpuidle-apple", -1, NULL, 0); From 4093f25cdc30c35a8dc7d08a2a9766b51100aac9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 16:15:31 +1100 Subject: [PATCH 0295/2680] dt-bindings: mailbox: Describe the T8140 GPU ASCWrap interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe the existing GPU coprocessor mailbox interface, with wrapper and vector-base resources, two FIFO interrupts and firmware role names. The public T8140 GPU device tree already supplies these properties; without a binding, validation cannot decode the firmware-role string. The description follows the public GPU interface and J700 ADT resource layout. This does not change the GPU device tree or driver behavior. [ryan: adapted to 7.2.10 — the T8140 MAINTAINERS section is created by the 900-config-tooling topic; this commit's MAINTAINERS lines are carried there] Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/mailbox/apple,ascwrap-v6.yaml | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml diff --git a/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml new file mode 100644 index 0000000000000..4798b62bd8430 --- /dev/null +++ b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mailbox/apple,ascwrap-v6.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 ASCWrap v6 mailbox + +maintainers: + - Ryan Murray + +description: + Mailbox interface of the T8140 GPU coprocessors. The wrapper resource + contains the coprocessor control registers and mailbox FIFOs. A separate + resource provides the coprocessor vector-base register. The firmware role + identifies which of the two GPU firmware instances uses the mailbox. + +properties: + compatible: + const: apple,t8140-ascwrap-v6 + + reg: + items: + - description: ASC wrapper registers + - description: Coprocessor vector-base register + + reg-names: + items: + - const: wrapper + - const: iop-vbar + + interrupts: + items: + - description: Send FIFO empty interrupt + - description: Receive FIFO not-empty interrupt + + interrupt-names: + items: + - const: send-empty + - const: recv-not-empty + + '#mbox-cells': + const: 0 + + power-domains: + maxItems: 1 + + apple,firmware-role: + enum: [GFX, GFX1] + +required: + - compatible + - reg + - reg-names + - interrupts + - interrupt-names + - '#mbox-cells' + - apple,firmware-role + +additionalProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + mailbox@482600000 { + compatible = "apple,t8140-ascwrap-v6"; + reg = <0x4 0x82600000 0x0 0x88000>, + <0x4 0x82050000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX"; + #mbox-cells = <0>; + }; + }; From 14022060211c43467c3e83c0fcfcd1da075fa008 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 13:59:04 +1100 Subject: [PATCH 0296/2680] dt-bindings: spi: describe the T8140 QSPIMC controller Describe the SPI-memory controller's register aperture, clock, interrupt and power domain. QSPIMC uses a packed command FIFO and differs from SPIMC. Originally written by DjDeveloperr; DjDeveloperr's Signed-off-by is outstanding. Port his publicly available controller binding. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/spi/apple,qspi.yaml | 62 +++++++++++++++++++ MAINTAINERS | 1 + 2 files changed, 63 insertions(+) create mode 100644 Documentation/devicetree/bindings/spi/apple,qspi.yaml diff --git a/Documentation/devicetree/bindings/spi/apple,qspi.yaml b/Documentation/devicetree/bindings/spi/apple,qspi.yaml new file mode 100644 index 0000000000000..5f526fe3a7970 --- /dev/null +++ b/Documentation/devicetree/bindings/spi/apple,qspi.yaml @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/spi/apple,qspi.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple ARM SoC QSPIMC controller + +maintainers: + - Asahi Linux + +description: | + Apple QSPIMC is a SPI-memory-only controller with a packed command FIFO. + It is distinct from the ordinary Apple SPIMC controller despite serving a + similar set of SPI NOR devices. + +allOf: + - $ref: spi-controller.yaml# + +properties: + compatible: + const: apple,t8140-qspi + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + interrupts: + maxItems: 1 + + power-domains: + maxItems: 1 + +required: + - compatible + - reg + - clocks + - interrupts + +unevaluatedProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + spi@385110000 { + compatible = "apple,t8140-qspi"; + reg = <0x3 0x85110000 0x0 0x4800>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clk_200m>; + #address-cells = <1>; + #size-cells = <0>; + }; + }; diff --git a/MAINTAINERS b/MAINTAINERS index fa4e0b27a6854..165a386cea40e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2617,6 +2617,7 @@ F: Documentation/devicetree/bindings/power/reset/apple,smc-reboot.yaml F: Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml F: Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml F: Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml +F: Documentation/devicetree/bindings/spi/apple,qspi.yaml F: Documentation/devicetree/bindings/spi/apple,spi.yaml F: Documentation/devicetree/bindings/spmi/apple,spmi.yaml F: Documentation/devicetree/bindings/usb/apple,dwc3.yaml From 04569286c2641d03b9cbd55b37f119786b1d3a1f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 13:59:06 +1100 Subject: [PATCH 0297/2680] spi: apple: add T8140 QSPIMC controller Add PIO spi-mem support for the QSPIMC controller used by T8140. The controller uses packed command descriptors and is not register-compatible with the ordinary Apple SPIMC controller. Use the SPI NOR and MTD stack for flash discovery and access. Originally written by DjDeveloperr; DjDeveloperr's Signed-off-by is outstanding. Port his publicly available GPL controller and binding. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/spi/Kconfig | 9 + drivers/spi/Makefile | 1 + drivers/spi/spi-apple-qspi.c | 618 +++++++++++++++++++++++++++++++++++ 4 files changed, 629 insertions(+) create mode 100644 drivers/spi/spi-apple-qspi.c diff --git a/MAINTAINERS b/MAINTAINERS index 165a386cea40e..b2a4b11fbef78 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2651,6 +2651,7 @@ F: drivers/pwm/pwm-apple.c F: drivers/reset/reset-apple-cio.c F: drivers/rtc/rtc-macsmc.c F: drivers/soc/apple/* +F: drivers/spi/spi-apple-qspi.c F: drivers/spi/spi-apple.c F: drivers/spmi/spmi-apple-controller.c F: drivers/thunderbolt/apple.c diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index 8782514bb89b0..1e54f2848de61 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -129,6 +129,15 @@ config SPI_APPLE to a single device like spi-nor (nvram), input device controller or fingerprint sensor. +config SPI_APPLE_QSPI + tristate "Apple SoC QSPIMC controller" + depends on ARCH_APPLE || COMPILE_TEST + depends on SPI_MEM + help + This enables support for Apple's command-FIFO QSPIMC controller. + Unlike the ordinary Apple SPIMC block, QSPIMC accepts packed SPI + memory descriptors and is used for the boot NOR on T8140 systems. + config SPI_AR934X tristate "Qualcomm Atheros AR934X/QCA95XX SPI controller driver" depends on ATH79 || COMPILE_TEST diff --git a/drivers/spi/Makefile b/drivers/spi/Makefile index 9fa12498ce8c0..3a0f6830961a7 100644 --- a/drivers/spi/Makefile +++ b/drivers/spi/Makefile @@ -23,6 +23,7 @@ obj-$(CONFIG_SPI_AMLOGIC_SPIFC_A1) += spi-amlogic-spifc-a1.o obj-$(CONFIG_SPI_AMLOGIC_SPIFC_A4) += spi-amlogic-spifc-a4.o obj-$(CONFIG_SPI_AMLOGIC_SPISG) += spi-amlogic-spisg.o obj-$(CONFIG_SPI_APPLE) += spi-apple.o +obj-$(CONFIG_SPI_APPLE_QSPI) += spi-apple-qspi.o obj-$(CONFIG_SPI_AR934X) += spi-ar934x.o obj-$(CONFIG_SPI_ARMADA_3700) += spi-armada-3700.o obj-$(CONFIG_SPI_ASPEED_SMC) += spi-aspeed-smc.o diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c new file mode 100644 index 0000000000000..66a4d11dc0a2b --- /dev/null +++ b/drivers/spi/spi-apple-qspi.c @@ -0,0 +1,618 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Apple QSPIMC controller + * + * QSPIMC is not register-compatible with the ordinary Apple SPIMC block. + * It consumes packed transfer descriptors and payload words through a + * command FIFO. This driver intentionally implements the PIO spi-mem path + * only; that is sufficient for SPI NOR discovery and access. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define APPLE_QSPI_C_OFFSET 0x800 +#define APPLE_QSPI_MIN_SIZE (APPLE_QSPI_C_OFFSET + 0xa0) + +/* P (queue) block. */ +#define APPLE_QSPI_P_TXFIFO 0x18 +#define APPLE_QSPI_P_RXFIFO 0x1c +#define APPLE_QSPI_P_QSTATUS 0x20 +#define APPLE_QSPI_P_QSTATUS_TX_USED GENMASK(15, 8) +#define APPLE_QSPI_P_QSTATUS_RX_AVAIL GENMASK(31, 24) +#define APPLE_QSPI_P_IRQ_ENABLE 0x2c +#define APPLE_QSPI_P_IRQ_STATUS 0x30 +#define APPLE_QSPI_P_VERSION 0x3c + +/* C (command) block. */ +#define APPLE_QSPI_C_CONTROL 0x00 +#define APPLE_QSPI_C_CONTROL_START BIT(0) +#define APPLE_QSPI_C_CONTROL_RESET BIT(8) +#define APPLE_QSPI_C_MODE 0x04 +#define APPLE_QSPI_C_MODE_CPHA BIT(8) +#define APPLE_QSPI_C_MODE_CPOL BIT(9) +#define APPLE_QSPI_C_PAYLOAD_COUNT 0x14 +#define APPLE_QSPI_C_RX_COUNT 0x18 +#define APPLE_QSPI_C_QTHRESHOLD 0x24 +#define APPLE_QSPI_C_IRQ_ENABLE 0x28 +#define APPLE_QSPI_C_IRQ_STATUS 0x2c +#define APPLE_QSPI_C_IRQ_COMPLETE BIT(0) +#define APPLE_QSPI_C_QDEPTH 0x38 +#define APPLE_QSPI_C_QDEPTH_TX GENMASK(7, 0) +#define APPLE_QSPI_C_QDEPTH_RX GENMASK(15, 8) +#define APPLE_QSPI_C_CONFIG_48 0x48 +#define APPLE_QSPI_C_CONFIG_58 0x58 +#define APPLE_QSPI_C_CONFIG_5C 0x5c +#define APPLE_QSPI_C_CLKDIV 0x64 +#define APPLE_QSPI_C_CONFIG_68 0x68 +#define APPLE_QSPI_C_CONFIG_6C 0x6c +#define APPLE_QSPI_C_CONFIG_7C 0x7c +#define APPLE_QSPI_C_CONFIG_80 0x80 +#define APPLE_QSPI_C_CONFIG_84 0x84 +#define APPLE_QSPI_C_CONFIG_90 0x90 +#define APPLE_QSPI_C_CONFIG_94 0x94 +#define APPLE_QSPI_C_CONFIG_98 0x98 +#define APPLE_QSPI_C_CONFIG_9C 0x9c + +/* Packed transfer descriptor, word 0. */ +#define APPLE_QSPI_CMD_MODE GENMASK(3, 0) +#define APPLE_QSPI_CMD_FRAME_A BIT(4) +#define APPLE_QSPI_CMD_FRAME_B BIT(5) +#define APPLE_QSPI_CMD_TAG GENMASK(15, 12) +#define APPLE_QSPI_CMD_SHARED_COUNT GENMASK(19, 16) +#define APPLE_QSPI_CMD_MULTI_COUNT GENMASK(23, 20) +#define APPLE_QSPI_CMD_TURNAROUND GENMASK(31, 24) +#define APPLE_QSPI_CMD_RX BIT(3) + +#define APPLE_QSPI_IRQ_MASK 0x3 +#define APPLE_QSPI_MAX_DATA 4096 +#define APPLE_QSPI_MAX_HZ 40000000 +#define APPLE_QSPI_TIMEOUT_MS 1000 + +struct apple_qspi { + void __iomem *base; + void __iomem *p; + void __iomem *c; + struct clk *clk; + struct completion done; + /* Protects the active transfer pointers shared with the IRQ handler. */ + spinlock_t lock; + int irq; + + const u8 *tx; + size_t tx_left; + u8 *rx; + size_t rx_left; + u8 tag; + u8 tx_depth; + u8 rx_depth; + bool active; +}; + +struct apple_qspi_op_shape { + u8 width; + u8 shared_count; + u8 multi_count; + u8 turnaround; +}; + +static inline u32 apple_qspi_p_read(struct apple_qspi *qspi, u32 reg) +{ + return readl_relaxed(qspi->p + reg); +} + +static inline void apple_qspi_p_write(struct apple_qspi *qspi, u32 reg, u32 val) +{ + writel_relaxed(val, qspi->p + reg); +} + +static inline u32 apple_qspi_c_read(struct apple_qspi *qspi, u32 reg) +{ + return readl_relaxed(qspi->c + reg); +} + +static inline void apple_qspi_c_write(struct apple_qspi *qspi, u32 reg, u32 val) +{ + writel_relaxed(val, qspi->c + reg); +} + +static int apple_qspi_width_mode(u8 width) +{ + switch (width) { + case 1: + return 1; + case 2: + return 2; + case 4: + return 3; + default: + return -EINVAL; + } +} + +static int apple_qspi_shape_op(const struct spi_mem_op *op, + struct apple_qspi_op_shape *shape) +{ + u32 dummy_cycles = 0; + u8 width; + + if (op->cmd.nbytes != 1 || op->cmd.buswidth != 1 || op->cmd.dtr) + return -EOPNOTSUPP; + if ((op->addr.nbytes && op->addr.dtr) || + (op->dummy.nbytes && op->dummy.dtr) || + (op->data.nbytes && (op->data.dtr || op->data.ecc || + op->data.swap16))) + return -EOPNOTSUPP; + + if (op->data.nbytes) + width = op->data.buswidth; + else if (op->dummy.nbytes) + width = op->dummy.buswidth; + else if (op->addr.nbytes) + width = op->addr.buswidth; + else + width = 1; + + if (apple_qspi_width_mode(width) < 0) + return -EOPNOTSUPP; + if (op->addr.nbytes && op->addr.buswidth != 1 && + op->addr.buswidth != width) + return -EOPNOTSUPP; + if (op->dummy.nbytes && op->dummy.buswidth != width) + return -EOPNOTSUPP; + if (op->data.nbytes && op->data.buswidth != width) + return -EOPNOTSUPP; + + shape->width = width; + shape->shared_count = op->cmd.nbytes; + shape->multi_count = 0; + if (op->addr.nbytes) { + if (op->addr.buswidth == 1) + shape->shared_count += op->addr.nbytes; + else + shape->multi_count = op->addr.nbytes; + } + + if (shape->shared_count > 15 || shape->multi_count > 15) + return -EOPNOTSUPP; + + if (op->dummy.nbytes) { + dummy_cycles = op->dummy.nbytes * 8 / op->dummy.buswidth; + if (dummy_cycles > 255) + return -EOPNOTSUPP; + } + shape->turnaround = dummy_cycles; + + return 0; +} + +static bool apple_qspi_supports_op(struct spi_mem *mem, + const struct spi_mem_op *op) +{ + struct apple_qspi_op_shape shape; + + if (spi_get_chipselect(mem->spi, 0) || spi_get_csgpiod(mem->spi, 0)) + return false; + if (op->addr.nbytes > 4 || op->data.nbytes > APPLE_QSPI_MAX_DATA) + return false; + if (apple_qspi_shape_op(op, &shape)) + return false; + + return spi_mem_default_supports_op(mem, op); +} + +static int apple_qspi_adjust_op_size(struct spi_mem *mem, struct spi_mem_op *op) +{ + op->data.nbytes = min_t(unsigned int, op->data.nbytes, APPLE_QSPI_MAX_DATA); + return 0; +} + +static void apple_qspi_fill_tx(struct apple_qspi *qspi) +{ + u32 status, used, available; + + status = apple_qspi_p_read(qspi, APPLE_QSPI_P_QSTATUS); + used = FIELD_GET(APPLE_QSPI_P_QSTATUS_TX_USED, status); + available = used < qspi->tx_depth ? qspi->tx_depth - used : 0; + + while (available-- && qspi->tx_left) { + u32 word = 0; + unsigned int shift; + + for (shift = 0; shift < 32 && qspi->tx_left; shift += 8) { + word |= (u32)*qspi->tx++ << shift; + qspi->tx_left--; + } + apple_qspi_p_write(qspi, APPLE_QSPI_P_TXFIFO, word); + } +} + +static void apple_qspi_drain_rx(struct apple_qspi *qspi) +{ + u32 status, available; + + status = apple_qspi_p_read(qspi, APPLE_QSPI_P_QSTATUS); + available = FIELD_GET(APPLE_QSPI_P_QSTATUS_RX_AVAIL, status); + + while (available-- && qspi->rx_left) { + u32 word = apple_qspi_p_read(qspi, APPLE_QSPI_P_RXFIFO); + unsigned int shift; + + for (shift = 0; shift < 32 && qspi->rx_left; shift += 8) { + *qspi->rx++ = word >> shift; + qspi->rx_left--; + } + } +} + +static irqreturn_t apple_qspi_irq(int irq, void *data) +{ + struct apple_qspi *qspi = data; + unsigned long flags; + u32 p_status, c_status; + bool complete_op = false; + + p_status = apple_qspi_p_read(qspi, APPLE_QSPI_P_IRQ_STATUS); + c_status = apple_qspi_c_read(qspi, APPLE_QSPI_C_IRQ_STATUS); + if (!p_status && !c_status) + return IRQ_NONE; + + apple_qspi_p_write(qspi, APPLE_QSPI_P_IRQ_STATUS, p_status); + apple_qspi_c_write(qspi, APPLE_QSPI_C_IRQ_STATUS, c_status); + + spin_lock_irqsave(&qspi->lock, flags); + if (qspi->active) { + apple_qspi_drain_rx(qspi); + apple_qspi_fill_tx(qspi); + if (c_status & APPLE_QSPI_C_IRQ_COMPLETE) + complete_op = true; + } + spin_unlock_irqrestore(&qspi->lock, flags); + + if (complete_op) + complete(&qspi->done); + + return IRQ_HANDLED; +} + +static void apple_qspi_clear_irqs(struct apple_qspi *qspi) +{ + u32 status; + + status = apple_qspi_p_read(qspi, APPLE_QSPI_P_IRQ_STATUS); + apple_qspi_p_write(qspi, APPLE_QSPI_P_IRQ_STATUS, status); + status = apple_qspi_c_read(qspi, APPLE_QSPI_C_IRQ_STATUS); + apple_qspi_c_write(qspi, APPLE_QSPI_C_IRQ_STATUS, status); +} + +static void apple_qspi_init_config(struct apple_qspi *qspi) +{ + u32 val; + + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_7C, 0); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_80, 0); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_58, 11); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_5C, 10); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_94, 0); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_98, 0); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_9C, 0); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_68, 11); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_6C, 10); + + val = apple_qspi_c_read(qspi, APPLE_QSPI_C_CONFIG_48); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_48, val | 0x00300000); + apple_qspi_c_write(qspi, APPLE_QSPI_C_QTHRESHOLD, + 1 | ((qspi->rx_depth >> 1) << 8)); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_90, 4); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_84, 0x00020000); +} + +static int apple_qspi_hw_reset(struct apple_qspi *qspi) +{ + u32 val; + int ret; + + apple_qspi_p_write(qspi, APPLE_QSPI_P_IRQ_ENABLE, 0); + apple_qspi_c_write(qspi, APPLE_QSPI_C_IRQ_ENABLE, 0); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONTROL, + APPLE_QSPI_C_CONTROL_RESET); + ret = readl_poll_timeout(qspi->c + APPLE_QSPI_C_CONTROL, val, + !(val & APPLE_QSPI_C_CONTROL_RESET), 10, 10000); + if (ret) + return ret; + + apple_qspi_clear_irqs(qspi); + apple_qspi_init_config(qspi); + return 0; +} + +static int apple_qspi_set_frequency(struct apple_qspi *qspi, + struct spi_mem *mem, + const struct spi_mem_op *op) +{ + unsigned long input = clk_get_rate(qspi->clk); + u32 requested, divisor; + + requested = op->max_freq ?: mem->spi->max_speed_hz; + requested = min_t(u32, requested, APPLE_QSPI_MAX_HZ); + if (!input || !requested) + return -EINVAL; + + divisor = DIV_ROUND_UP(input, requested); + divisor = clamp_t(u32, divisor, 2, 256); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CLKDIV, divisor - 1); + return 0; +} + +static bool apple_qspi_wait_complete(struct apple_qspi *qspi) +{ + unsigned long deadline = jiffies + + msecs_to_jiffies(APPLE_QSPI_TIMEOUT_MS); + unsigned long flags; + u32 p_status, c_status; + + do { + if (try_wait_for_completion(&qspi->done)) + return true; + + /* + * TX-only commands do not reliably raise the wired interrupt on + * T8140, although the completion bit is latched in the controller. + * Poll both status banks as a fallback and service the FIFOs so a + * page program larger than the TX FIFO can continue making progress. + */ + p_status = apple_qspi_p_read(qspi, APPLE_QSPI_P_IRQ_STATUS); + c_status = apple_qspi_c_read(qspi, APPLE_QSPI_C_IRQ_STATUS); + if (p_status) + apple_qspi_p_write(qspi, APPLE_QSPI_P_IRQ_STATUS, p_status); + if (c_status) + apple_qspi_c_write(qspi, APPLE_QSPI_C_IRQ_STATUS, c_status); + + spin_lock_irqsave(&qspi->lock, flags); + if (qspi->active) { + apple_qspi_drain_rx(qspi); + apple_qspi_fill_tx(qspi); + } + spin_unlock_irqrestore(&qspi->lock, flags); + + if (c_status & APPLE_QSPI_C_IRQ_COMPLETE) + return true; + if (time_after_eq(jiffies, deadline)) + break; + + if (wait_for_completion_timeout(&qspi->done, 1)) + return true; + } while (time_before(jiffies, deadline)); + + return false; +} + +static int apple_qspi_exec_op(struct spi_mem *mem, + const struct spi_mem_op *op) +{ + struct spi_controller *ctlr = mem->spi->controller; + struct apple_qspi *qspi = spi_controller_get_devdata(ctlr); + struct apple_qspi_op_shape shape; + unsigned long flags; + size_t header_len, tx_len; + u8 *tx_buf, *p; + u32 descriptor, mode; + u32 p_status, c_status; + int ret, retries; + bool complete; + + ret = apple_qspi_shape_op(op, &shape); + if (ret) + return ret; + + header_len = op->cmd.nbytes + op->addr.nbytes; + tx_len = header_len; + if (op->data.dir == SPI_MEM_DATA_OUT) + tx_len += op->data.nbytes; + + tx_buf = kmalloc(tx_len, GFP_KERNEL); + if (!tx_buf) + return -ENOMEM; + + p = tx_buf; + *p++ = op->cmd.opcode; + if (op->addr.nbytes) { + int i; + + for (i = op->addr.nbytes - 1; i >= 0; i--) + *p++ = op->addr.val >> (8 * i); + } + if (op->data.dir == SPI_MEM_DATA_OUT && op->data.nbytes) + memcpy(p, op->data.buf.out, op->data.nbytes); + + ret = apple_qspi_set_frequency(qspi, mem, op); + if (ret) + goto out_free; + + mode = (mem->spi->mode & SPI_CPOL ? APPLE_QSPI_C_MODE_CPOL : 0) | + (mem->spi->mode & SPI_CPHA ? APPLE_QSPI_C_MODE_CPHA : 0); + apple_qspi_c_write(qspi, APPLE_QSPI_C_MODE, mode); + + mode = apple_qspi_width_mode(shape.width); + if (op->data.dir == SPI_MEM_DATA_IN) + mode |= APPLE_QSPI_CMD_RX; + descriptor = FIELD_PREP(APPLE_QSPI_CMD_MODE, mode) | + APPLE_QSPI_CMD_FRAME_A | APPLE_QSPI_CMD_FRAME_B | + FIELD_PREP(APPLE_QSPI_CMD_TAG, qspi->tag++ & 0xf) | + FIELD_PREP(APPLE_QSPI_CMD_SHARED_COUNT, + shape.shared_count) | + FIELD_PREP(APPLE_QSPI_CMD_MULTI_COUNT, + shape.multi_count) | + FIELD_PREP(APPLE_QSPI_CMD_TURNAROUND, + shape.turnaround); + + apple_qspi_p_write(qspi, APPLE_QSPI_P_IRQ_ENABLE, 0); + apple_qspi_c_write(qspi, APPLE_QSPI_C_IRQ_ENABLE, 0); + apple_qspi_clear_irqs(qspi); + reinit_completion(&qspi->done); + + spin_lock_irqsave(&qspi->lock, flags); + qspi->tx = tx_buf; + qspi->tx_left = tx_len; + qspi->rx = op->data.dir == SPI_MEM_DATA_IN ? op->data.buf.in : NULL; + qspi->rx_left = op->data.dir == SPI_MEM_DATA_IN ? op->data.nbytes : 0; + qspi->active = true; + apple_qspi_p_write(qspi, APPLE_QSPI_P_TXFIFO, descriptor); + apple_qspi_p_write(qspi, APPLE_QSPI_P_TXFIFO, op->data.nbytes); + apple_qspi_fill_tx(qspi); + spin_unlock_irqrestore(&qspi->lock, flags); + + apple_qspi_c_write(qspi, APPLE_QSPI_C_IRQ_ENABLE, + APPLE_QSPI_IRQ_MASK); + apple_qspi_p_write(qspi, APPLE_QSPI_P_IRQ_ENABLE, + APPLE_QSPI_IRQ_MASK); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONTROL, + APPLE_QSPI_C_CONTROL_START); + + complete = apple_qspi_wait_complete(qspi); + apple_qspi_p_write(qspi, APPLE_QSPI_P_IRQ_ENABLE, 0); + apple_qspi_c_write(qspi, APPLE_QSPI_C_IRQ_ENABLE, 0); + synchronize_irq(qspi->irq); + + spin_lock_irqsave(&qspi->lock, flags); + for (retries = 100; qspi->rx_left && retries; retries--) + apple_qspi_drain_rx(qspi); + qspi->active = false; + ret = qspi->tx_left || qspi->rx_left ? -EIO : 0; + spin_unlock_irqrestore(&qspi->lock, flags); + + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONTROL, 0); + if (!complete) { + p_status = apple_qspi_p_read(qspi, APPLE_QSPI_P_QSTATUS); + c_status = apple_qspi_c_read(qspi, APPLE_QSPI_C_IRQ_STATUS); + dev_err(&mem->spi->dev, + "transfer timed out: p=%#x c=%#x pcnt=%#x prxcnt=%#x tx=%zu rx=%zu\n", + p_status, c_status, + apple_qspi_c_read(qspi, APPLE_QSPI_C_PAYLOAD_COUNT), + apple_qspi_c_read(qspi, APPLE_QSPI_C_RX_COUNT), + qspi->tx_left, qspi->rx_left); + ret = -ETIMEDOUT; + if (apple_qspi_hw_reset(qspi)) + dev_err(&mem->spi->dev, + "controller reset failed after timeout\n"); + } else if (ret) { + dev_err(&mem->spi->dev, + "transfer completed with %zu tx and %zu rx bytes pending\n", + qspi->tx_left, qspi->rx_left); + } + +out_free: + kfree(tx_buf); + return ret; +} + +static const struct spi_controller_mem_ops apple_qspi_mem_ops = { + .adjust_op_size = apple_qspi_adjust_op_size, + .supports_op = apple_qspi_supports_op, + .exec_op = apple_qspi_exec_op, +}; + +static int apple_qspi_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct spi_controller *ctlr; + struct apple_qspi *qspi; + struct resource *res; + u32 depths, version; + int irq, ret; + + ctlr = devm_spi_alloc_host(dev, sizeof(*qspi)); + if (!ctlr) + return -ENOMEM; + + qspi = spi_controller_get_devdata(ctlr); + init_completion(&qspi->done); + spin_lock_init(&qspi->lock); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res || resource_size(res) < APPLE_QSPI_MIN_SIZE) + return dev_err_probe(dev, -EINVAL, + "QSPIMC aperture is too small\n"); + + qspi->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(qspi->base)) + return PTR_ERR(qspi->base); + qspi->p = qspi->base; + qspi->c = qspi->base + APPLE_QSPI_C_OFFSET; + + qspi->clk = devm_clk_get_enabled(dev, NULL); + if (IS_ERR(qspi->clk)) + return dev_err_probe(dev, PTR_ERR(qspi->clk), + "failed to enable controller clock\n"); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + qspi->irq = irq; + ret = devm_request_irq(dev, irq, apple_qspi_irq, 0, + dev_name(dev), qspi); + if (ret) + return dev_err_probe(dev, ret, "failed to request IRQ\n"); + + ret = apple_qspi_hw_reset(qspi); + if (ret) + return dev_err_probe(dev, ret, "controller reset timed out\n"); + + depths = apple_qspi_c_read(qspi, APPLE_QSPI_C_QDEPTH); + qspi->tx_depth = FIELD_GET(APPLE_QSPI_C_QDEPTH_TX, depths); + qspi->rx_depth = FIELD_GET(APPLE_QSPI_C_QDEPTH_RX, depths); + if (qspi->tx_depth < 3 || !qspi->rx_depth) + return dev_err_probe(dev, -ENODEV, + "invalid FIFO depths tx=%u rx=%u\n", + qspi->tx_depth, qspi->rx_depth); + apple_qspi_init_config(qspi); + + ctlr->num_chipselect = 1; + ctlr->max_native_cs = 1; + ctlr->mode_bits = SPI_CPHA | SPI_CPOL | + SPI_RX_DUAL | SPI_RX_QUAD | + SPI_TX_DUAL | SPI_TX_QUAD; + ctlr->bits_per_word_mask = SPI_BPW_MASK(8); + ctlr->mem_ops = &apple_qspi_mem_ops; + + platform_set_drvdata(pdev, ctlr); + ret = devm_spi_register_controller(dev, ctlr); + if (ret) + return dev_err_probe(dev, ret, + "failed to register SPI controller\n"); + + version = apple_qspi_p_read(qspi, APPLE_QSPI_P_VERSION); + dev_info(dev, "QSPIMC version %#x, %lu Hz clock, txq=%u rxq=%u\n", + version, clk_get_rate(qspi->clk), qspi->tx_depth, + qspi->rx_depth); + return 0; +} + +static const struct of_device_id apple_qspi_of_match[] = { + { .compatible = "apple,t8140-qspi" }, + {} +}; +MODULE_DEVICE_TABLE(of, apple_qspi_of_match); + +static struct platform_driver apple_qspi_driver = { + .probe = apple_qspi_probe, + .driver = { + .name = "apple-qspi", + .of_match_table = apple_qspi_of_match, + }, +}; +module_platform_driver(apple_qspi_driver); + +MODULE_AUTHOR("The Asahi Linux Contributors"); +MODULE_DESCRIPTION("Apple QSPIMC controller driver"); +MODULE_LICENSE("GPL"); From 435130268746cd6258571f035a8ebae27f014ce5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:00:14 +1100 Subject: [PATCH 0298/2680] spi: apple-qspi: service transfer FIFOs with bounded microsecond polling T8140 can latch command completion without delivering the wired IRQ, and the RX FIFO threshold interrupt does not reliably fire. Waiting a jiffy between service attempts limits Linux reads to one FIFO per tick. The observed NVRAM read took 55 seconds at about 34 kB/s. Use the polling helper with a microsecond sleep interval to drain and refill the FIFOs. Preserve the one-second deadline and the IRQ completion path. Serialize polling FIFO service with the interrupt handler. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/spi-apple-qspi.c | 66 ++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c index 66a4d11dc0a2b..b831622753d55 100644 --- a/drivers/spi/spi-apple-qspi.c +++ b/drivers/spi/spi-apple-qspi.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -354,47 +353,40 @@ static int apple_qspi_set_frequency(struct apple_qspi *qspi, return 0; } -static bool apple_qspi_wait_complete(struct apple_qspi *qspi) +static bool apple_qspi_poll_complete(struct apple_qspi *qspi) { - unsigned long deadline = jiffies + - msecs_to_jiffies(APPLE_QSPI_TIMEOUT_MS); unsigned long flags; - u32 p_status, c_status; - - do { - if (try_wait_for_completion(&qspi->done)) - return true; - - /* - * TX-only commands do not reliably raise the wired interrupt on - * T8140, although the completion bit is latched in the controller. - * Poll both status banks as a fallback and service the FIFOs so a - * page program larger than the TX FIFO can continue making progress. - */ - p_status = apple_qspi_p_read(qspi, APPLE_QSPI_P_IRQ_STATUS); - c_status = apple_qspi_c_read(qspi, APPLE_QSPI_C_IRQ_STATUS); - if (p_status) - apple_qspi_p_write(qspi, APPLE_QSPI_P_IRQ_STATUS, p_status); - if (c_status) - apple_qspi_c_write(qspi, APPLE_QSPI_C_IRQ_STATUS, c_status); - - spin_lock_irqsave(&qspi->lock, flags); - if (qspi->active) { - apple_qspi_drain_rx(qspi); - apple_qspi_fill_tx(qspi); - } - spin_unlock_irqrestore(&qspi->lock, flags); + u32 queue_irqs, command_irqs; - if (c_status & APPLE_QSPI_C_IRQ_COMPLETE) - return true; - if (time_after_eq(jiffies, deadline)) - break; + spin_lock_irqsave(&qspi->lock, flags); + queue_irqs = apple_qspi_p_read(qspi, APPLE_QSPI_P_IRQ_STATUS); + command_irqs = apple_qspi_c_read(qspi, APPLE_QSPI_C_IRQ_STATUS); + apple_qspi_p_write(qspi, APPLE_QSPI_P_IRQ_STATUS, queue_irqs); + apple_qspi_c_write(qspi, APPLE_QSPI_C_IRQ_STATUS, command_irqs); + if (qspi->active) { + apple_qspi_drain_rx(qspi); + apple_qspi_fill_tx(qspi); + } + spin_unlock_irqrestore(&qspi->lock, flags); - if (wait_for_completion_timeout(&qspi->done, 1)) - return true; - } while (time_before(jiffies, deadline)); + return (command_irqs & APPLE_QSPI_C_IRQ_COMPLETE) || + try_wait_for_completion(&qspi->done); +} - return false; +static bool apple_qspi_wait_complete(struct apple_qspi *qspi) +{ + bool finished; + + /* + * T8140 can latch completion without delivering the wired IRQ, and + * the RX threshold IRQ does not reliably drain the FIFO. A jiffy + * between polls throttles reads to one FIFO per scheduler tick. + * Service the FIFOs at microsecond intervals while retaining the + * bounded transfer deadline and IRQ completion path. + */ + return !read_poll_timeout(apple_qspi_poll_complete, finished, finished, + 20, APPLE_QSPI_TIMEOUT_MS * USEC_PER_MSEC, + false, qspi); } static int apple_qspi_exec_op(struct spi_mem *mem, From f1a1cb527f5f6a8f34ad424c3ef60ea8c49a0ead Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:05:43 +1100 Subject: [PATCH 0299/2680] MAINTAINERS: add a dedicated Apple QSPIMC entry Track the QSPIMC driver and binding together. Keep the existing Apple machine binding list independent of the new SPI controller entry. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index b2a4b11fbef78..851ab4b6c1dad 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2617,7 +2617,6 @@ F: Documentation/devicetree/bindings/power/reset/apple,smc-reboot.yaml F: Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml F: Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml F: Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml -F: Documentation/devicetree/bindings/spi/apple,qspi.yaml F: Documentation/devicetree/bindings/spi/apple,spi.yaml F: Documentation/devicetree/bindings/spmi/apple,spmi.yaml F: Documentation/devicetree/bindings/usb/apple,dwc3.yaml @@ -2651,7 +2650,6 @@ F: drivers/pwm/pwm-apple.c F: drivers/reset/reset-apple-cio.c F: drivers/rtc/rtc-macsmc.c F: drivers/soc/apple/* -F: drivers/spi/spi-apple-qspi.c F: drivers/spi/spi-apple.c F: drivers/spmi/spmi-apple-controller.c F: drivers/thunderbolt/apple.c @@ -2667,6 +2665,13 @@ F: include/uapi/drm/asahi_drm.h F: net/bluetooth/brcm.c F: net/bluetooth/brcm.h +ARM/APPLE QSPIMC DRIVER +M: Ryan Murray +L: asahi@lists.linux.dev +S: Maintained +F: Documentation/devicetree/bindings/spi/apple,qspi.yaml +F: drivers/spi/spi-apple-qspi.c + ARM/ARTPEC MACHINE SUPPORT M: Jesper Nilsson M: Lars Persson From 1d72b1c24356505dbc0c0b23f61d500d44b96556 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 16:12:21 +1100 Subject: [PATCH 0300/2680] MAINTAINERS: identify Aurora as the QSPIMC contact Point the QSPIMC maintenance entry and binding at Ryan Murray and the Aurora Silicon project site. Remove the mailing-list association from this controller's new maintenance entry. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/spi/apple,qspi.yaml | 2 +- MAINTAINERS | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/spi/apple,qspi.yaml b/Documentation/devicetree/bindings/spi/apple,qspi.yaml index 5f526fe3a7970..4d33549e7bf39 100644 --- a/Documentation/devicetree/bindings/spi/apple,qspi.yaml +++ b/Documentation/devicetree/bindings/spi/apple,qspi.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Apple ARM SoC QSPIMC controller maintainers: - - Asahi Linux + - Ryan Murray description: | Apple QSPIMC is a SPI-memory-only controller with a packed command FIFO. diff --git a/MAINTAINERS b/MAINTAINERS index 851ab4b6c1dad..b60cd73cce49d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2667,7 +2667,7 @@ F: net/bluetooth/brcm.h ARM/APPLE QSPIMC DRIVER M: Ryan Murray -L: asahi@lists.linux.dev +W: https://aurora-silicon.github.io S: Maintained F: Documentation/devicetree/bindings/spi/apple,qspi.yaml F: drivers/spi/spi-apple-qspi.c From 07427365b775611e340f4c923f3f759502359d8a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 16:46:49 +1100 Subject: [PATCH 0301/2680] spi: apple-qspi: reject clocks below the divider range Clamping a too-large divider to 256 makes the bus faster than requested. For a 200 MHz input, a 100 kHz request would run at about 781 kHz. Reject unrepresentable slow transfers instead of violating the device limit. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/spi-apple-qspi.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c index b831622753d55..57a572c1e779d 100644 --- a/drivers/spi/spi-apple-qspi.c +++ b/drivers/spi/spi-apple-qspi.c @@ -348,7 +348,10 @@ static int apple_qspi_set_frequency(struct apple_qspi *qspi, return -EINVAL; divisor = DIV_ROUND_UP(input, requested); - divisor = clamp_t(u32, divisor, 2, 256); + /* Never exceed the requested clock when it is below our range. */ + if (divisor > 256) + return -EINVAL; + divisor = max_t(u32, divisor, 2); apple_qspi_c_write(qspi, APPLE_QSPI_C_CLKDIV, divisor - 1); return 0; } From 4ddde0758b74afc346b9e9cac275bcd489b243c6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 13:57:58 +1100 Subject: [PATCH 0302/2680] soc: apple: pmp: use a 42-bit DMA mask on T8140 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The T8140 PMP DART DMA window starts at 1 TiB. The default 32-bit DMA mask leaves no usable IOVA, so the firmware's first buffer request fails with ENOMEM and probe times out. Set the mask before allocating any firmware buffers, using per-SoC match data. Retain the default mask on existing SoCs. Port Ryan Murray's Linux-observed T8140 fix onto the existing Asahi PMP driver by Sasha Finkelstein, including Janne Grunau's fixup. [ryan: adapted to 7.2.10 — Asahi's 7.2 API fixups of the PMP driver changed probe() to the lifetime-bound Data<'bound> form and named io requests; the T8140 DMA mask set-up is applied on that form] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/pmp.rs | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs index 7924881a2046b..82bdf6df6482d 100644 --- a/drivers/soc/apple/pmp.rs +++ b/drivers/soc/apple/pmp.rs @@ -15,7 +15,9 @@ use kernel::{ }, dma::{ Coherent, - CoherentBox, // + CoherentBox, + Device as _, + DmaMask, // }, io::{ io_project, @@ -395,24 +397,47 @@ impl rtkit::Operations for PmpData { #[allow(dead_code)] struct PmpDriver(Arc); +/// Per-SoC configuration. +struct PmpHwConfig { + /// DMA mask to set before the firmware allocates anything, or `None` to + /// keep the default. + dma_mask: Option, +} + +const HW_CFG_DEFAULT: PmpHwConfig = PmpHwConfig { dma_mask: None }; + +/// T8140: the DART translates 42 bits and its DMA window starts at 1 TiB. +const HW_CFG_T8140: PmpHwConfig = PmpHwConfig { + dma_mask: Some(DmaMask::new::<42>()), +}; + kernel::of_device_table!( OF_TABLE, MODULE_OF_TABLE, - (), - [(of::DeviceId::new(c"apple,t6000-pmp-v2"), ())] + ::IdInfo, + [ + (of::DeviceId::new(c"apple,t8140-pmp-v2"), &HW_CFG_T8140), + (of::DeviceId::new(c"apple,t6000-pmp-v2"), &HW_CFG_DEFAULT), + ] ); impl platform::Driver for PmpDriver { - type IdInfo = (); + type IdInfo = &'static PmpHwConfig; type Data<'bound> = PmpDriver; const OF_ID_TABLE: Option> = Some(&OF_TABLE); fn probe<'bound>( pdev: &'bound platform::Device>, - _info: Option<&'bound Self::IdInfo>, + info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { let dev: ARef = pdev.as_ref().into(); + let cfg = info.ok_or(ENODEV)?; + if let Some(mask) = cfg.dma_mask { + // SAFETY: No DMA allocations or mappings exist for this device + // yet. + unsafe { pdev.dma_set_mask_and_coherent(mask)? }; + } let pmp_req = pdev.io_request_by_name(c"pmp").ok_or(EINVAL)?; let pmp_mmio = pmp_req.iomap_sized::()?; let pmp_mmio = pmp_mmio.as_view(); From 79e519594750804598618564c55a4ef4d8bb4a09 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:22:12 +1100 Subject: [PATCH 0303/2680] pmdomain: apple: pmp-report: expose T8140 fast-die effort MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe the T8140 report layout from the Apple device tree and expose its fast-die effort through the existing PMP report mapping. A stopped PMP reads zero, so check the running bit before accepting an effort. Report entry 0x1c1 contains byte 1 for the P cluster, observed under Linux load, and byte 0 for E, assigned from ADT lane order. The E byte has not yet been observed changing. Do not read entry 0x1c2 or scan the unpopulated part of the report aperture. Expose read-only debugfs status and lane effort values for diagnostics. Port the original Linux-derived implementation by Ryan Murray. Retain the Asahi report driver and use it as the sole owner of the mapping. [ryan: adapted to 7.2.10 — kept Asahi's t8122 report offsets next to the T8140 entry; the T8140 MAINTAINERS section is created by the 900-config-tooling topic, so this commit's MAINTAINERS lines are carried there] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmp-report.c | 124 +++++++++++++++++++++++++++ include/linux/soc/apple/pmp-report.h | 26 ++++++ 2 files changed, 150 insertions(+) create mode 100644 include/linux/soc/apple/pmp-report.h diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index 39a10a0b61515..492a724debd11 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -5,21 +5,38 @@ * Copyright The Asahi Linux Contributors */ +#include #include #include +#include #include #include #include #include #include +#include +#include +#include #define PMP_REPORT_READY 0x1 +/** + * struct apple_pmp_report_offsets - per-SoC layout of the report region + * @tgt_read: power state request, read side + * @tgt_write: power state request, write side + * @actual: power state acknowledgment + * @status: PMP status; bit 0 is set while the PMP is running + * @fast_die: 32-bit word holding the fast die-temperature effort of each + * CPU cluster, one byte per lane; 0 if the SoC has none + * @fast_die_lanes: number of lanes at @fast_die, at most 4 + */ struct apple_pmp_report_offsets { u32 tgt_read; u32 tgt_write; u32 actual; u32 status; + u32 fast_die; + u32 fast_die_lanes; }; struct apple_pmp_report { @@ -27,8 +44,89 @@ struct apple_pmp_report { const struct apple_pmp_report_offsets *offsets; void __iomem *base; spinlock_t lock; + struct list_head list; }; +/* + * Bound report regions, for apple_pmp_report_fast_die_effort(). Regions are + * added once they are set up and never removed: the driver cannot be unbound. + */ +static LIST_HEAD(apple_pmp_reports); +static DEFINE_MUTEX(apple_pmp_reports_lock); +static struct dentry *apple_pmp_report_debugfs; + +/** + * apple_pmp_report_fast_die_effort() - read the fast die-temperature effort + * of a CPU cluster + * @np: device tree node of the report region + * @lane: lane of the cluster in the fast die-temperature report + * @effort: set to the effort on success + * + * The PMP of some SoCs runs a control loop on its own die-temperature sensors + * and publishes an 8-bit effort for each CPU cluster: 0 requests no limit, + * and the effort rises towards 255 as the die exceeds the firmware's target. + * The effort is only valid while the PMP reports that it is running; a + * stopped PMP reads 0. + * + * Context: Any context. + * Return: 0 on success, -EPROBE_DEFER if the region is not bound (yet), + * -EOPNOTSUPP if the SoC's PMP publishes no such effort, -EINVAL if @lane is + * out of range, or -EAGAIN if the PMP does not report that it is running. + */ +int apple_pmp_report_fast_die_effort(const struct device_node *np, + unsigned int lane, u8 *effort) +{ + const struct apple_pmp_report_offsets *offs; + const struct apple_pmp_report *rep; + int ret = -EPROBE_DEFER; + + rcu_read_lock(); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->dev->of_node != np) + continue; + + offs = rep->offsets; + if (!offs->fast_die) { + ret = -EOPNOTSUPP; + } else if (lane >= offs->fast_die_lanes) { + ret = -EINVAL; + } else if (!(readl(rep->base + offs->status) & PMP_REPORT_READY)) { + ret = -EAGAIN; + } else { + *effort = readl(rep->base + offs->fast_die) >> + (lane * BITS_PER_BYTE); + ret = 0; + } + break; + } + rcu_read_unlock(); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmp_report_fast_die_effort); + +/* Read only the documented status and fast-die entries, never the aperture. */ +static int apple_pmp_report_status_show(struct seq_file *s, void *unused) +{ + struct apple_pmp_report *rep = s->private; + const struct apple_pmp_report_offsets *offs = rep->offsets; + u32 status = readl(rep->base + offs->status); + unsigned int lane; + u32 effort; + + seq_printf(s, "running: %u\n", !!(status & PMP_REPORT_READY)); + if (!(status & PMP_REPORT_READY) || !offs->fast_die) + return 0; + + effort = readl(rep->base + offs->fast_die); + for (lane = 0; lane < offs->fast_die_lanes; lane++) + seq_printf(s, "lane%u: %u\n", lane, + (effort >> (lane * BITS_PER_BYTE)) & 0xff); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(apple_pmp_report_status); + static int apple_pmp_report_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -51,6 +149,14 @@ static int apple_pmp_report_probe(struct platform_device *pdev) if (ret) return dev_err_probe(dev, ret, "failed to create child devices\n"); + mutex_lock(&apple_pmp_reports_lock); + list_add_tail_rcu(&rep->list, &apple_pmp_reports); + mutex_unlock(&apple_pmp_reports_lock); + + if (rep->offsets->fast_die) + debugfs_create_file(dev_name(dev), 0400, apple_pmp_report_debugfs, + rep, &apple_pmp_report_status_fops); + return 0; } @@ -89,12 +195,23 @@ static const struct apple_pmp_report_offsets apple_pmp_offsets_t8122 = { .status = 0x10, }; +/* The fast die-temperature report is the low two bytes of entry 0x1c1. */ +static const struct apple_pmp_report_offsets apple_pmp_offsets_t8140 = { + .tgt_read = 0x1880, + .tgt_write = 0x10c40, + .actual = 0x18c0, + .status = 0x10, + .fast_die = 0x1c10, + .fast_die_lanes = 2, +}; + static const struct of_device_id apple_pmp_report_of_match[] = { { .compatible = "apple,t6000-pmp-v2-report", .data = &apple_pmp_offsets_t600x }, { .compatible = "apple,t6020-pmp-v2-report", .data = &apple_pmp_offsets_t602x }, { .compatible = "apple,t6030-pmp-v2-report", .data = &apple_pmp_offsets_t6030 }, { .compatible = "apple,t8112-pmp-v2-report", .data = &apple_pmp_offsets_t8112 }, { .compatible = "apple,t8122-pmp-v2-report", .data = &apple_pmp_offsets_t8122 }, + { .compatible = "apple,t8140-pmp-v2-report", .data = &apple_pmp_offsets_t8140 }, {} }; @@ -103,6 +220,11 @@ static struct platform_driver apple_pmp_report_driver = { .driver = { .name = "apple-pmp-report", .of_match_table = apple_pmp_report_of_match, + /* + * Its child power domains and the region list keep using the + * mapping, and nothing tears them down. + */ + .suppress_bind_attrs = true, }, }; @@ -238,6 +360,7 @@ MODULE_DEVICE_TABLE(of, apple_pmp_report_entry_of_match); static int __init apple_pmp_report_init(void) { + apple_pmp_report_debugfs = debugfs_create_dir("apple-pmp-report", NULL); platform_driver_register(&apple_pmp_report_entry_driver); platform_driver_register(&apple_pmp_report_driver); return 0; @@ -245,6 +368,7 @@ static int __init apple_pmp_report_init(void) static void __exit apple_pmp_report_exit(void) { + debugfs_remove_recursive(apple_pmp_report_debugfs); platform_driver_unregister(&apple_pmp_report_entry_driver); platform_driver_unregister(&apple_pmp_report_driver); } diff --git a/include/linux/soc/apple/pmp-report.h b/include/linux/soc/apple/pmp-report.h new file mode 100644 index 0000000000000..31c2e4b17a303 --- /dev/null +++ b/include/linux/soc/apple/pmp-report.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Apple SoC PMP report region + * Copyright (C) The Asahi Linux Contributors + */ + +#ifndef _LINUX_SOC_APPLE_PMP_REPORT_H_ +#define _LINUX_SOC_APPLE_PMP_REPORT_H_ + +#include +#include + +struct device_node; + +#if IS_ENABLED(CONFIG_APPLE_PMP_REPORT) +int apple_pmp_report_fast_die_effort(const struct device_node *np, + unsigned int lane, u8 *effort); +#else +static inline int apple_pmp_report_fast_die_effort(const struct device_node *np, + unsigned int lane, u8 *effort) +{ + return -ENODEV; +} +#endif + +#endif /* _LINUX_SOC_APPLE_PMP_REPORT_H_ */ From 175d6ca4a5d7ae11fb3fa2967c8c452e8701b9e3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:00:11 +1100 Subject: [PATCH 0304/2680] dt-bindings: apple: describe PMP fast-die reporting on T8140 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document the existing PMP firmware driver resources and the report aperture consumed by the fast-die limiter. Define a report phandle and lane index on the cluster DVFS controller, using the measured P lane and ADT E lane ordering. Firmware calibration properties are populated by the bootloader from the Apple device tree. [ryan: adapted to 7.2.10 — the T8140 MAINTAINERS section is created by the 900-config-tooling topic; this commit's MAINTAINERS lines are carried there] Assisted-by: LLM Signed-off-by: Ryan Murray --- .../cpufreq/apple,cluster-cpufreq.yaml | 12 +++ .../power/apple,t8140-pmp-report.yaml | 46 +++++++++ .../bindings/soc/apple/apple,pmp.yaml | 94 +++++++++++++++++++ 3 files changed, 152 insertions(+) create mode 100644 Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml create mode 100644 Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml diff --git a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml index b51913a817917..9c9edb8cb441a 100644 --- a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml +++ b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml @@ -42,6 +42,18 @@ properties: reg: maxItems: 1 + apple,fast-die-effort: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: | + PMP report region and cluster effort lane. Increasing firmware effort + imposes a maximum-frequency QoS constraint alongside thermal cooling. + If the PMP is unavailable, cpufreq retains its fallback ceiling. + items: + - items: + - description: PMP report region + - description: Cluster lane (0 for E, 1 for P on T8140) + enum: [0, 1] + '#performance-domain-cells': const: 0 diff --git a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml new file mode 100644 index 0000000000000..9b515685eef00 --- /dev/null +++ b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/power/apple,t8140-pmp-report.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 PMP report aperture + +maintainers: + - Ryan Murray + +description: | + The PMP report aperture contains a read window and an update window. + The PMP running bit qualifies the per-cluster fast-die effort. Lane 1 + tracks P-cluster load in Linux measurements. Lane 0 is assigned to the + E cluster from the Apple device tree order and has not been observed + changing. Only documented report entries may be read; the aperture is + not a contiguous readable array of registers. + +properties: + compatible: + const: apple,t8140-pmp-v2-report + + reg: + maxItems: 1 + + power-domains: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + pmp-report@3003c0000 { + compatible = "apple,t8140-pmp-v2-report"; + reg = <3 0x003c0000 0 0x14000>; + }; + }; diff --git a/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml b/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml new file mode 100644 index 0000000000000..9784c62fba89e --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,pmp.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple power management co-processor + +maintainers: + - Ryan Murray + +description: | + The PMP runs firmware loaded by the bootloader. Linux starts it through + RTKit and supplies calibration copied from the Apple device tree. The + bootloader must initialize the PMP DART address protection filter before + Linux starts the firmware. Firmware-owned PMC power state must be retained. + +properties: + compatible: + oneOf: + - const: apple,t6000-pmp-v2 + - items: + - const: apple,t8140-pmp-v2 + - const: apple,t6000-pmp-v2 + + reg: + items: + - description: Firmware SRAM + - description: ASC control registers + + reg-names: + items: + - const: pmp + - const: asc + + mboxes: + maxItems: 1 + + mbox-names: + const: mbox + + iommus: + maxItems: 1 + + power-domains: + maxItems: 1 + + apple,board-id: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Board identifier copied by the bootloader. + + apple,dram-vendor-id: + $ref: /schemas/types.yaml#/definitions/uint32 + description: DRAM vendor identifier copied by the bootloader. + + apple,dram-capacity: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Firmware DRAM capacity value copied by the bootloader. + + apple,pio-ranges: + $ref: /schemas/types.yaml#/definitions/uint64-array + minItems: 2 + description: Host physical address and size pairs mapped into firmware PIO space. + +patternProperties: + '^apple,tunable-': + $ref: /schemas/types.yaml#/definitions/uint8-array + description: Opaque firmware calibration copied from the Apple device tree. + +required: + - compatible + - reg + - reg-names + - mboxes + - mbox-names + - iommus + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + pmp@300500000 { + compatible = "apple,t8140-pmp-v2", "apple,t6000-pmp-v2"; + reg = <3 0x00500000 0 0xa0000>, <3 0x00e00000 0 0x4000>; + reg-names = "pmp", "asc"; + mboxes = <&pmp_mbox>; + mbox-names = "mbox"; + iommus = <&pmp_dart 0>; + }; + }; From 3fd88e9c0c8113965c809a804d8d97b507ddf44b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 21:34:23 +1100 Subject: [PATCH 0305/2680] pmdomain: apple: drain PMP report users before teardown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The report mapping and allocation are managed by the platform device, but the global RCU list and fast-die debugfs file outlive driver removal. Withdraw both publications and drain readers before devres releases the mapping. Install the teardown action before publishing the region so probe failures also revoke it. The existing DVFS debugfs action still runs before the mapping is released. Depopulate child providers first. Their power callbacks now look up the report inside an RCU read-side section instead of following a child's unmanaged parent pointer. Remove the child provider and generic domain on teardown, including failed probe paths. If genpd refuses removal of a busy domain, keep its independently allocated storage, name and node references alive, with callbacks returning ENODEV. This bounds the leak to the removed domain and avoids reclaiming memory still owned by genpd. Publish the report before populating children so their initial power requests can use the same lookup. Validate report bit indices and use a 64-bit mask for the 64-bit request register. Fixes: f69f1b76fcb0 ("pmdomain: apple: pmp-report: expose T8140 fast-die effort") Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 206b9b56f4b2bbdfc15efee3f23f5e5167f76bf8) [ryan: adapted to 7.2.10 — commit e3163ca622d6 ("soc: apple: expose the T8140 PMP DVFS report for diagnosis") (the T8140 DVFS debugfs diagnostic) is not carried, so the probe path and context here do not include it] --- drivers/pmdomain/apple/pmp-report.c | 151 ++++++++++++++++++++++------ 1 file changed, 119 insertions(+), 32 deletions(-) diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index 492a724debd11..b193313d98748 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #define PMP_REPORT_READY 0x1 @@ -45,12 +46,10 @@ struct apple_pmp_report { void __iomem *base; spinlock_t lock; struct list_head list; + struct dentry *debugfs; }; -/* - * Bound report regions, for apple_pmp_report_fast_die_effort(). Regions are - * added once they are set up and never removed: the driver cannot be unbound. - */ +/* Bound report regions, protected by RCU until managed teardown drains readers. */ static LIST_HEAD(apple_pmp_reports); static DEFINE_MUTEX(apple_pmp_reports_lock); static struct dentry *apple_pmp_report_debugfs; @@ -127,6 +126,21 @@ static int apple_pmp_report_status_show(struct seq_file *s, void *unused) } DEFINE_SHOW_ATTRIBUTE(apple_pmp_report_status); +static void apple_pmp_report_release(void *data) +{ + struct apple_pmp_report *rep = data; + + /* Withdraw child providers while their power callbacks can still use MMIO. */ + of_platform_depopulate(rep->dev); + mutex_lock(&apple_pmp_reports_lock); + if (!list_empty(&rep->list)) + list_del_rcu(&rep->list); + mutex_unlock(&apple_pmp_reports_lock); + synchronize_rcu(); + /* The debugfs proxy drains active reads and rejects subsequent accesses. */ + debugfs_remove(rep->debugfs); +} + static int apple_pmp_report_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -145,17 +159,24 @@ static int apple_pmp_report_probe(struct platform_device *pdev) return PTR_ERR(rep->base); rep->offsets = of_device_get_match_data(dev); dev_set_drvdata(dev, rep); - ret = of_platform_populate(np, NULL, NULL, dev); + INIT_LIST_HEAD(&rep->list); + ret = devm_add_action_or_reset(dev, apple_pmp_report_release, rep); if (ret) - return dev_err_probe(dev, ret, "failed to create child devices\n"); + return ret; mutex_lock(&apple_pmp_reports_lock); list_add_tail_rcu(&rep->list, &apple_pmp_reports); mutex_unlock(&apple_pmp_reports_lock); if (rep->offsets->fast_die) - debugfs_create_file(dev_name(dev), 0400, apple_pmp_report_debugfs, - rep, &apple_pmp_report_status_fops); + rep->debugfs = debugfs_create_file(dev_name(dev), 0400, + apple_pmp_report_debugfs, rep, + &apple_pmp_report_status_fops); + + /* Publish the region before child power callbacks start looking it up. */ + ret = of_platform_populate(np, NULL, NULL, dev); + if (ret) + return dev_err_probe(dev, ret, "failed to create child devices\n"); return 0; } @@ -220,28 +241,26 @@ static struct platform_driver apple_pmp_report_driver = { .driver = { .name = "apple-pmp-report", .of_match_table = apple_pmp_report_of_match, - /* - * Its child power domains and the region list keep using the - * mapping, and nothing tears them down. - */ + /* Power-domain providers are not intended for manual unbinding. */ .suppress_bind_attrs = true, }, }; struct apple_pmp_report_entry { - struct device *dev; + struct device_node *node; + struct device_node *report_node; struct generic_pm_domain genpd; u32 id; + bool removed; }; #define genpd_to_apple_pmp_report_entry(_genpd) \ container_of(_genpd, struct apple_pmp_report_entry, genpd) -static int apple_pmp_report_set_state(struct generic_pm_domain *genpd, bool enable) +static int apple_pmp_report_set_state(struct apple_pmp_report *rep, u32 id, + bool enable) { - struct apple_pmp_report_entry *ent = genpd_to_apple_pmp_report_entry(genpd); - struct apple_pmp_report *rep = dev_get_drvdata(ent->dev->parent); - u64 bit_val = 1 << ent->id; + u64 bit_val = BIT_ULL(id); u64 val; unsigned long flags; @@ -263,14 +282,56 @@ static int apple_pmp_report_set_state(struct generic_pm_domain *genpd, bool enab 50000); } +static int apple_pmp_report_entry_set_state(struct generic_pm_domain *genpd, bool enable) +{ + struct apple_pmp_report_entry *ent = genpd_to_apple_pmp_report_entry(genpd); + struct apple_pmp_report *rep; + int ret = -ENODEV; + + if (READ_ONCE(ent->removed)) + return ret; + + rcu_read_lock(); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->dev->of_node == ent->report_node) { + ret = apple_pmp_report_set_state(rep, ent->id, enable); + break; + } + } + rcu_read_unlock(); + return ret; +} + static int apple_pmp_report_entry_power_on(struct generic_pm_domain *genpd) { - return apple_pmp_report_set_state(genpd, true); + return apple_pmp_report_entry_set_state(genpd, true); } static int apple_pmp_report_entry_power_off(struct generic_pm_domain *genpd) { - return apple_pmp_report_set_state(genpd, false); + return apple_pmp_report_entry_set_state(genpd, false); +} + +static void apple_pmp_report_entry_free(struct apple_pmp_report_entry *ent) +{ + of_node_put(ent->report_node); + of_node_put(ent->node); + kfree(ent->genpd.name); + kfree(ent); +} + +static void apple_pmp_report_entry_remove(struct platform_device *pdev) +{ + struct apple_pmp_report_entry *ent = platform_get_drvdata(pdev); + + WRITE_ONCE(ent->removed, true); + of_genpd_del_provider(ent->node); + if (pm_genpd_remove(&ent->genpd)) { + /* The core still owns this domain. Keep its now inert storage alive. */ + dev_warn(&pdev->dev, "retaining busy power domain %s\n", ent->genpd.name); + return; + } + apple_pmp_report_entry_free(ent); } static int apple_pmp_report_entry_probe(struct platform_device *pdev) @@ -282,36 +343,53 @@ static int apple_pmp_report_entry_probe(struct platform_device *pdev) const char *name; struct of_phandle_iterator it; - ent = devm_kzalloc(dev, sizeof(*ent), GFP_KERNEL); + ent = kzalloc_obj(*ent); if (!ent) return -ENOMEM; - ent->dev = dev; + ent->node = of_node_get(node); + ent->report_node = of_node_get(dev->parent->of_node); ret = of_property_read_u32(node, "reg", &ent->id); - if (ret) - return dev_err_probe(dev, ret, "missing reg property\n"); + if (ret) { + ret = dev_err_probe(dev, ret, "missing reg property\n"); + goto err_free; + } + if (ent->id >= 64) { + ret = -EINVAL; + goto err_free; + } ret = of_property_read_string(node, "label", &name); - if (ret < 0) - return dev_err_probe(dev, ret, "missing label property\n"); + if (ret < 0) { + ret = dev_err_probe(dev, ret, "missing label property\n"); + goto err_free; + } + ent->genpd.name = kstrdup(name, GFP_KERNEL); + if (!ent->genpd.name) { + ret = -ENOMEM; + goto err_free; + } if (of_property_read_bool(node, "apple,always-on")) { ent->genpd.flags |= GENPD_FLAG_ACTIVE_WAKEUP; - apple_pmp_report_set_state(&ent->genpd, true); + apple_pmp_report_entry_set_state(&ent->genpd, true); } - ent->genpd.name = name; ent->genpd.power_on = apple_pmp_report_entry_power_on; ent->genpd.power_off = apple_pmp_report_entry_power_off; ret = pm_genpd_init(&ent->genpd, NULL, true); - if (ret) - return dev_err_probe(dev, ret, "pm_genpd_init failed\n"); + if (ret) { + ret = dev_err_probe(dev, ret, "pm_genpd_init failed\n"); + goto err_free; + } ret = of_genpd_add_provider_simple(node, &ent->genpd); - if (ret) - return dev_err_probe(dev, ret, "of_genpd_add_provider_simple failed\n"); + if (ret) { + ret = dev_err_probe(dev, ret, "of_genpd_add_provider_simple failed\n"); + goto err_remove_domain; + } of_for_each_phandle(&it, ret, node, "power-domains", "#power-domain-cells", -1) { struct of_phandle_args parent, child; @@ -334,11 +412,19 @@ static int apple_pmp_report_entry_probe(struct platform_device *pdev) } pm_genpd_remove_device(dev); + platform_set_drvdata(pdev, ent); return 0; err_remove: of_genpd_del_provider(node); - pm_genpd_remove(&ent->genpd); +err_remove_domain: + WRITE_ONCE(ent->removed, true); + if (pm_genpd_remove(&ent->genpd)) { + dev_warn(dev, "retaining busy power domain %s\n", ent->genpd.name); + return ret; + } +err_free: + apple_pmp_report_entry_free(ent); return ret; } @@ -349,6 +435,7 @@ static const struct of_device_id apple_pmp_report_entry_of_match[] = { static struct platform_driver apple_pmp_report_entry_driver = { .probe = apple_pmp_report_entry_probe, + .remove = apple_pmp_report_entry_remove, .driver = { .name = "apple-pmp-report-entry", .of_match_table = apple_pmp_report_entry_of_match, From f9371fd3548cafcdcd11155fe94846d663a18f1c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 15:16:11 +1100 Subject: [PATCH 0306/2680] dt-bindings: soc: apple: Add the T8140 memory cache controller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The memory cache controller of the Apple T8140 holds the system level cache in front of DRAM, split into planes. The boot firmware enables and powers the cache ways of every plane but leaves allocation disabled, so the operating system has to allow it. Describe one register block per plane, and the devices that take over DMA the bootloader left running: allocation must wait until they are bound. [ryan: adapted to 7.2.10 — the T8140 MAINTAINERS section is created by the 900-config-tooling topic; this commit's MAINTAINERS lines are carried there] Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/soc/apple/apple,t8140-mcc.yaml | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 Documentation/devicetree/bindings/soc/apple/apple,t8140-mcc.yaml diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t8140-mcc.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t8140-mcc.yaml new file mode 100644 index 0000000000000..e4c5688a01d7f --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,t8140-mcc.yaml @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,t8140-mcc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 memory cache controller + +maintainers: + - Ryan Murray + +description: + The memory cache controller of the Apple T8140 SoC holds the system level + cache in front of DRAM, split into planes. The boot firmware enables and + powers the cache ways of every plane but leaves allocation disabled, so the + cache holds no data until the operating system allows allocation. + +properties: + compatible: + const: apple,t8140-mcc + + reg: + description: The register block of each cache plane. + minItems: 1 + maxItems: 8 + + apple,boot-dma-devices: + $ref: /schemas/types.yaml#/definitions/phandle-array + items: + maxItems: 1 + description: + Devices that take over DMA the bootloader left running. Cache allocation + is enabled only once each of them is bound to its driver. + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + display: display-subsystem { + }; + + memory-controller@220000000 { + compatible = "apple,t8140-mcc"; + reg = <0x2 0x20000000 0x0 0x4000>, + <0x2 0x20040000 0x0 0x4000>, + <0x2 0x20080000 0x0 0x4000>, + <0x2 0x200c0000 0x0 0x4000>; + apple,boot-dma-devices = <&display>; + }; + }; From 8403b4ec002c43f42e43d0623fae872b75963327 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 15:16:21 +1100 Subject: [PATCH 0307/2680] soc: apple: Enable T8140 system level cache allocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On T8140 the boot firmware enables and powers the 12 ways of each of the 4 system level cache planes, but leaves the allocation-disable word of every plane set. The cache therefore holds no data, and every GPU and CPU access that misses the core caches goes to DRAM. Clear the allocation-disable word of each plane once every plane reports as many ways powered as enabled and no flush in progress, and read it back. Do it only after the devices named in the device tree as having taken over DMA from the bootloader are bound, so the cache starts allocating after their hand-off rather than during it. On shutdown, stop allocating and flush the cache before the next boot stage runs. With allocation disabled the cache holds no data, so no maintenance is needed before enabling it. Measured on J700 (MacBook Neo), toggling only this word on a running system: glmark2 build+texture 902 -> 1886, vkcube 2141 -> 5584 fps, glmark2 terrain at 1920x1200 41 -> 66 fps, and the submit-to-completion latency of an isolated GPU job 560 -> 210 us. [ryan: adapted to 7.2.10 — the T8140 MAINTAINERS section is created by the 900-config-tooling topic; this commit's MAINTAINERS lines are carried there] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 12 +++ drivers/soc/apple/Makefile | 3 + drivers/soc/apple/mcc.c | 189 +++++++++++++++++++++++++++++++++++++ 3 files changed, 204 insertions(+) create mode 100644 drivers/soc/apple/mcc.c diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 41bd8fbc87aab..2032de34e6c04 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -25,6 +25,18 @@ config APPLE_MAILBOX Say Y here if you have an Apple SoC. +config APPLE_MCC + tristate "Apple T8140 system level cache" + depends on ARCH_APPLE || COMPILE_TEST + depends on OF + default ARCH_APPLE + help + The boot firmware of the Apple T8140 SoC enables the system level + cache without letting it allocate. This driver allows allocation + once the devices that took over DMA from the bootloader are bound. + + Say 'y' or 'm' here if you have an Apple T8140 SoC. + config APPLE_PMGR_MISC bool "Apple SoC PMGR miscellaneous support" depends on PM diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 1baa6de944958..14f4d8936f1f6 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -25,3 +25,6 @@ obj-$(CONFIG_APPLE_AOP) += aop.o obj-$(CONFIG_APPLE_SEP) += sep.o obj-$(CONFIG_APPLE_PMP) += pmp.o + +obj-$(CONFIG_APPLE_MCC) += apple-mcc.o +apple-mcc-y := mcc.o diff --git a/drivers/soc/apple/mcc.c b/drivers/soc/apple/mcc.c new file mode 100644 index 0000000000000..10f5e6e01e8cd --- /dev/null +++ b/drivers/soc/apple/mcc.c @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple T8140 memory cache controller: system level cache allocation + * + * The boot firmware enables and powers the cache ways of every plane but + * leaves allocation disabled, so the cache holds no data. This driver allows + * allocation once the devices that took over DMA from the bootloader are + * bound, and disables allocation and flushes the cache again on shutdown. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MCC_ALLOC_DISABLE 0x82c +#define MCC_CACHE_WAYS 0x2800 +#define MCC_CACHE_WAYS_ENABLED GENMASK(4, 0) +#define MCC_CACHE_POWER 0x2804 +#define MCC_CACHE_WAYS_POWERED GENMASK(30, 24) +#define MCC_CACHE_FLUSH 0x2844 +#define MCC_CACHE_FLUSH_BUSY BIT(0) + +#define MCC_MAX_PLANES 8 +#define MCC_FLUSH_TIMEOUT_US 100000 + +struct apple_mcc { + struct device *dev; + void __iomem *planes[MCC_MAX_PLANES]; + unsigned int nr_planes; + bool enabled; +}; + +static int apple_mcc_wait_boot_dma(struct device *dev) +{ + struct of_phandle_iterator it; + int ret; + + of_for_each_phandle(&it, ret, dev->of_node, "apple,boot-dma-devices", NULL, 0) { + struct platform_device *pdev = of_find_device_by_node(it.node); + bool bound = pdev && device_is_bound(&pdev->dev); + + if (pdev) + put_device(&pdev->dev); + if (!bound) { + of_node_put(it.node); + return dev_err_probe(dev, -EPROBE_DEFER, "waiting for %pOF to bind\n", + it.node); + } + } + + return ret == -ENOENT ? 0 : ret; +} + +static bool apple_mcc_plane_ready(struct apple_mcc *mcc, unsigned int plane) +{ + void __iomem *regs = mcc->planes[plane]; + u32 ways = FIELD_GET(MCC_CACHE_WAYS_ENABLED, readl(regs + MCC_CACHE_WAYS)); + u32 powered = FIELD_GET(MCC_CACHE_WAYS_POWERED, readl(regs + MCC_CACHE_POWER)); + u32 flush = readl(regs + MCC_CACHE_FLUSH); + + if (ways && powered == ways && !(flush & MCC_CACHE_FLUSH_BUSY)) + return true; + + dev_warn(mcc->dev, "plane %u not ready: %u ways enabled, %u powered, flush %#x\n", + plane, ways, powered, flush); + return false; +} + +static int apple_mcc_enable(struct apple_mcc *mcc) +{ + unsigned int i; + + for (i = 0; i < mcc->nr_planes; i++) + if (!apple_mcc_plane_ready(mcc, i)) + return -EBUSY; + + for (i = 0; i < mcc->nr_planes; i++) { + writel(0, mcc->planes[i] + MCC_ALLOC_DISABLE); + if (readl(mcc->planes[i] + MCC_ALLOC_DISABLE)) { + dev_err(mcc->dev, "plane %u did not enable allocation\n", i); + return -EIO; + } + } + + return 0; +} + +/* Stop allocating, then write back and invalidate the cache. */ +static void apple_mcc_quiesce(struct apple_mcc *mcc) +{ + unsigned int i; + u32 val; + + if (!mcc->enabled) + return; + + for (i = 0; i < mcc->nr_planes; i++) { + writel(1, mcc->planes[i] + MCC_ALLOC_DISABLE); + writel(1, mcc->planes[i] + MCC_CACHE_FLUSH); + } + + for (i = 0; i < mcc->nr_planes; i++) + if (readl_poll_timeout_atomic(mcc->planes[i] + MCC_CACHE_FLUSH, val, + !(val & MCC_CACHE_FLUSH_BUSY), 10, + MCC_FLUSH_TIMEOUT_US)) + dev_err(mcc->dev, "plane %u flush timed out\n", i); + + mcc->enabled = false; +} + +static int apple_mcc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_mcc *mcc; + int ret; + + mcc = devm_kzalloc(dev, sizeof(*mcc), GFP_KERNEL); + if (!mcc) + return -ENOMEM; + + mcc->dev = dev; + while (platform_get_resource(pdev, IORESOURCE_MEM, mcc->nr_planes)) { + if (mcc->nr_planes == MCC_MAX_PLANES) + return dev_err_probe(dev, -EINVAL, "too many planes\n"); + mcc->planes[mcc->nr_planes] = devm_platform_ioremap_resource(pdev, mcc->nr_planes); + if (IS_ERR(mcc->planes[mcc->nr_planes])) + return PTR_ERR(mcc->planes[mcc->nr_planes]); + mcc->nr_planes++; + } + if (!mcc->nr_planes) + return dev_err_probe(dev, -EINVAL, "no planes\n"); + + ret = apple_mcc_wait_boot_dma(dev); + if (ret) + return ret; + + platform_set_drvdata(pdev, mcc); + + /* Allocation is only a performance setting: leave a cache that is not ready alone. */ + ret = apple_mcc_enable(mcc); + if (ret == -EBUSY) + return 0; + if (ret) + return ret; + + mcc->enabled = true; + dev_info(dev, "system level cache allocation enabled on %u planes\n", mcc->nr_planes); + return 0; +} + +static void apple_mcc_remove(struct platform_device *pdev) +{ + apple_mcc_quiesce(platform_get_drvdata(pdev)); +} + +static void apple_mcc_shutdown(struct platform_device *pdev) +{ + struct apple_mcc *mcc = platform_get_drvdata(pdev); + + if (mcc) + apple_mcc_quiesce(mcc); +} + +static const struct of_device_id apple_mcc_of_match[] = { + { .compatible = "apple,t8140-mcc" }, + {} +}; +MODULE_DEVICE_TABLE(of, apple_mcc_of_match); + +static struct platform_driver apple_mcc_driver = { + .driver = { + .name = "apple-mcc", + .of_match_table = apple_mcc_of_match, + }, + .probe = apple_mcc_probe, + .remove = apple_mcc_remove, + .shutdown = apple_mcc_shutdown, +}; +module_platform_driver(apple_mcc_driver); + +MODULE_AUTHOR("Ryan Murray "); +MODULE_DESCRIPTION("Apple T8140 system level cache allocation"); +MODULE_LICENSE("Dual MIT/GPL"); From 0130bfb5b95f7d5d05bb765e7360ee7dca1fdbb4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 11:12:26 +1100 Subject: [PATCH 0308/2680] cpuidle: apple: Explain why T8140 has no CPU PD state Commit 0a1ed8ce6d3c ("cpuidle: apple: Restrict T8140 to WFI until CPU PD is fixed") hid the CPU PD state on T8140 after an opt-in hung a J700, with the cause unknown. The cause is now known, and there is nothing in the sequence to fix, so say why T8140 keeps only WFI. CPU PD requests core power-down by setting bits 25:24 of CYC_OVRD (s3_5_c15_c5_0) before WFI. On a J700 that register reads as zero on both E and P cores, and an MSR to it, even of the value just read, takes an undefined-instruction exception (ESR 0x02000000). In the idle path that exception hits the idle task after cpu_pm_enter(), and the oops becomes a panic. The register is not needed there. Sampling the PMGR power-state registers of the six cores at 1 kHz on an idle J700, the E cores and the second P core read as power-gated in 94% to 99.9% of samples after plain WFI. The boot P core never did, with any idle state tried, and it is not its interrupt load: the second P core stays power-gated in 98.7% of samples while it is woken about 1500 times a second. A CPU PD state without the CYC_OVRD write was also tried: the same WFI between cpu_pm_enter() and cpu_pm_exit(). It ran a 31-minute load and idle soak on all six CPUs without errors, and the governor put 21% to 98% of each CPU's time in it. On the same boot, three paired two-minute idle windows each, against WFI alone: AC input power, display on 7.173 W vs 7.178 W P-cluster rail, median 1.259 W vs 1.259 W hottest P-cluster zone 65.1 C vs 65.0 C It would add CPU PM notifier work and save nothing, so it is not added. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/cpuidle/cpuidle-apple.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/cpuidle/cpuidle-apple.c b/drivers/cpuidle/cpuidle-apple.c index 67c7846111cf3..4d023c877318f 100644 --- a/drivers/cpuidle/cpuidle-apple.c +++ b/drivers/cpuidle/cpuidle-apple.c @@ -124,7 +124,11 @@ static struct cpuidle_driver apple_idle_driver = { static int apple_cpuidle_probe(struct platform_device *pdev) { - /* A T8140 CPU PD opt-in hung J700; expose only WFI. */ + /* + * CPU PD writes CYC_OVRD, which is an undefined instruction on + * T8140. Its WFI already lets the hardware power-gate idle + * secondary cores, so expose only the WFI state. + */ if (of_machine_is_compatible("apple,t8140")) apple_idle_driver.state_count = STATE_WFI + 1; From 4501cf4caa6b02953e00c9c5cb647aa1fdf9bfc8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 18:11:22 -0400 Subject: [PATCH 0309/2680] dt-bindings: power: apple,t8140-pmp-report: describe report entries The PMP chooses DRAM and fabric performance states partly from the power state that the host reports for the devices it tracks. On macOS, ApplePMGR sets a device's bit in the PMP's power-state request word after it powers the device on, clears it before it powers the device off, and waits for the PMP to acknowledge the devices that require it. The pmp-report driver already provides such report entries as power domains on earlier SoCs. Allow them as children of the T8140 report aperture, with the device's PMP soc-device index as the unit address. Assisted-by: LLM Signed-off-by: Diljit Singh --- .../power/apple,t8140-pmp-report.yaml | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml index 9b515685eef00..0cddaf6136379 100644 --- a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml +++ b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml @@ -27,6 +27,51 @@ properties: power-domains: maxItems: 1 + '#address-cells': + const: 1 + + '#size-cells': + const: 0 + +patternProperties: + '^report@[0-9a-f]+$': + type: object + description: | + A device whose power state the host reports to the PMP, provided as a + power domain. The host sets the device's request bit after its parent + domain powers on and clears it before the parent powers off, and waits + for the PMP to acknowledge while the PMP runs. + additionalProperties: false + + properties: + compatible: + items: + - const: apple,t8140-pmp-v2-report-entry + - const: apple,t6000-pmp-v2-report-entry + + reg: + maxItems: 1 + description: PMP soc-device index of the device. + + label: + description: Name of the power domain. + + '#power-domain-cells': + const: 0 + + power-domains: + maxItems: 1 + + apple,always-on: + type: boolean + description: Report the device as on from probe. + + required: + - compatible + - reg + - label + - '#power-domain-cells' + required: - compatible - reg @@ -42,5 +87,15 @@ examples: pmp-report@3003c0000 { compatible = "apple,t8140-pmp-v2-report"; reg = <3 0x003c0000 0 0x14000>; + #address-cells = <1>; + #size-cells = <0>; + + report@9 { + compatible = "apple,t8140-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x9>; + label = "pmp-gfx"; + #power-domain-cells = <0>; + }; }; }; From c5ccd6f2f6dfb52df18283af7308f532b21f774c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 18:11:22 -0400 Subject: [PATCH 0310/2680] dt-bindings: power: add Apple PMC voter interface The PMC arbitrates the performance states of shared rails, such as the DRAM controller and the SoC fabric, between voter agents: the PMP, the host performance controller (CLPC), the GPU and the application processor. Each agent has a perf-state floor word with one 4-bit state per rail, and each rail has an interface enable word with one bit per agent; the PMC ignores the floor of an agent that is not enabled on a rail. Describe the voter and rail windows, the static agents, the dynamic agents as power domains, and the host performance controller's agent. Assisted-by: LLM Signed-off-by: Diljit Singh --- .../devicetree/bindings/power/apple,pmc.yaml | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 Documentation/devicetree/bindings/power/apple,pmc.yaml diff --git a/Documentation/devicetree/bindings/power/apple,pmc.yaml b/Documentation/devicetree/bindings/power/apple,pmc.yaml new file mode 100644 index 0000000000000..69cdc1cb87197 --- /dev/null +++ b/Documentation/devicetree/bindings/power/apple,pmc.yaml @@ -0,0 +1,131 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/power/apple,pmc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple SoC PMC voter interface + +maintainers: + - Diljit Singh + +description: | + The PMC arbitrates the performance states of shared rails, such as the + DRAM controller and the SoC fabric, between voter agents. Each agent has a + perf-state floor word with one 4-bit state per rail, and each rail has an + interface enable word with one bit per agent. The PMC only honours an + agent's floor on rails where its interface is enabled. + + Static agents are enabled for as long as the PMC is bound. A dynamic + agent is a power domain: it is enabled after its parent domain powers on + and disabled before its parent domain powers off. The host performance + controller's agent is described by a child node that raises its floor + while the CPU clusters or the GPU are busy. + +properties: + compatible: + const: apple,t8140-pmc + + reg: + items: + - description: Perf-state floor words of the voter agents + - description: Rail state and interface enable words + + reg-names: + items: + - const: voters + - const: rails + + power-domains: + maxItems: 1 + + '#address-cells': + const: 1 + + '#size-cells': + const: 0 + + apple,static-voters: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: Voter agents to enable on every rail. + +patternProperties: + '^voter@[0-9a-f]+$': + type: object + description: A dynamic voter agent, provided as a power domain. + additionalProperties: false + + properties: + reg: + maxItems: 1 + description: Voter agent index. + + label: + description: Name of the power domain. + + '#power-domain-cells': + const: 0 + + power-domains: + maxItems: 1 + description: Domain that must be powered while the agent is enabled. + + required: + - reg + - '#power-domain-cells' + + '^performance-controller@[0-9a-f]+$': + type: object + description: The host performance controller's voter agent. + additionalProperties: false + + properties: + compatible: + const: apple,t8140-clpc + + reg: + maxItems: 1 + description: Voter agent index. + + apple,gpu-power-state: + $ref: /schemas/types.yaml#/definitions/phandle + description: PMGR power state of the GPU, used as a demand input. + + required: + - compatible + - reg + +required: + - compatible + - reg + - reg-names + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + power-management@302800000 { + compatible = "apple,t8140-pmc"; + reg = <0x3 0x02800000 0x0 0x10000>, + <0x3 0x028aa000 0x0 0x3000>; + reg-names = "voters", "rails"; + #address-cells = <1>; + #size-cells = <0>; + apple,static-voters = <0 1 3>; + + performance-controller@1 { + compatible = "apple,t8140-clpc"; + reg = <1>; + }; + + voter@2 { + reg = <2>; + label = "gfx"; + #power-domain-cells = <0>; + }; + }; + }; From a520d44ea9c556ac46bd20d27e060280eb068797 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 18:11:39 -0400 Subject: [PATCH 0311/2680] pmdomain: apple: add PMC voter interface driver The PMC only honours a voter agent's perf-state floor on rails where that agent's interface is enabled. macOS enables the static agents (PMP, CLPC and AP) on all their rails when it restores the PMGR (ApplePMC::restorePMC), and enables a dynamic agent such as the GPU only while the host keeps the agent's PMGR device powered: after the power-on state write and before the power-off one (ApplePMC::stateChange). Without these enables, the PMC ignores those agents' floors. Enable the static agents listed in the device tree for as long as the PMC is bound, and restore the saved enable words when it is unbound. Provide each dynamic agent as a power domain whose parent is its device's power domain, so that genpd enables the agent after the parent powers on and disables it before the parent powers off, as macOS does. Every enable write is read back. Export the floor words so that a host performance controller can set its agent's floor. Assisted-by: LLM Signed-off-by: Diljit Singh --- drivers/pmdomain/apple/Kconfig | 12 + drivers/pmdomain/apple/Makefile | 1 + drivers/pmdomain/apple/pmc.c | 492 ++++++++++++++++++++++++++++++++ include/linux/soc/apple/pmc.h | 55 ++++ 4 files changed, 560 insertions(+) create mode 100644 drivers/pmdomain/apple/pmc.c create mode 100644 include/linux/soc/apple/pmc.h diff --git a/drivers/pmdomain/apple/Kconfig b/drivers/pmdomain/apple/Kconfig index 6e9deb9de0739..7145a073429aa 100644 --- a/drivers/pmdomain/apple/Kconfig +++ b/drivers/pmdomain/apple/Kconfig @@ -20,4 +20,16 @@ config APPLE_PMP_REPORT depends on PM select PM_GENERIC_DOMAINS +config APPLE_PMC + bool "Apple PMC voter interface" + depends on OF + depends on PM + select PM_GENERIC_DOMAINS + help + The PMC in some Apple SoCs arbitrates the performance states of + shared rails, such as the DRAM controller and the SoC fabric, + between voter agents. This driver enables the static agents and + provides power domains that enable a dynamic agent, such as the + GPU, while its device is powered on. + endif diff --git a/drivers/pmdomain/apple/Makefile b/drivers/pmdomain/apple/Makefile index b098b204d9868..b25665119a2fa 100644 --- a/drivers/pmdomain/apple/Makefile +++ b/drivers/pmdomain/apple/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_APPLE_PMGR_PWRSTATE) += pmgr-pwrstate.o obj-$(CONFIG_APPLE_PMP_REPORT) += pmp-report.o +obj-$(CONFIG_APPLE_PMC) += pmc.o diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c new file mode 100644 index 0000000000000..027fa33a6f8fb --- /dev/null +++ b/drivers/pmdomain/apple/pmc.c @@ -0,0 +1,492 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC PMC voter interface driver + * + * The PMC arbitrates the performance state of shared rails, such as the + * DRAM controller (DCS) and the SoC fabric, between voter agents: the PMP, + * the host performance controller (CLPC), the GPU and the application + * processor. Each agent owns a perf-state floor word holding one 4-bit + * state per rail, and each rail has an interface enable word with one bit + * per agent. The PMC only honours the floor of an agent whose interface is + * enabled on that rail. + * + * At startup macOS enables the static agents (PMP, CLPC and AP) on their + * rails, and enables a dynamic agent such as the GPU only while the PMGR + * device it belongs to is powered on: after the device has been powered + * on, and before it is powered off. This driver does the former at probe + * and models each dynamic agent as a power domain that sits between the + * device and its PMGR power state, which gives the same ordering. + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define APPLE_PMC_MAX_RAILS 8 +#define APPLE_PMC_MAX_AGENTS 32 + +/** + * struct apple_pmc_hw - per-SoC layout of the PMC voter interface + * @agent_stride: distance between the floor words of consecutive agents + * in the "voters" region + * @agents: number of voter agents + * @rails: number of rails + * @rail_state: offset of the per-rail state words in the "rails" region + * @rail_enable: offset of the per-rail interface enable words in the + * "rails" region + */ +struct apple_pmc_hw { + u32 agent_stride; + u32 agents; + u32 rails; + u32 rail_state; + u32 rail_enable; +}; + +struct apple_pmc { + struct device *dev; + const struct apple_pmc_hw *hw; + void __iomem *voters; + void __iomem *rails; + /* Serializes read-modify-write cycles on enable and floor words. */ + raw_spinlock_t lock; + u32 saved_enable[APPLE_PMC_MAX_RAILS]; + u32 static_agents; + struct list_head domains; + struct dentry *debugfs; +}; + +struct apple_pmc_domain { + struct generic_pm_domain genpd; + struct apple_pmc *pmc; + struct device_node *np; + struct list_head list; + unsigned int agent; + bool provider; +}; + +#define genpd_to_apple_pmc_domain(_genpd) \ + container_of(_genpd, struct apple_pmc_domain, genpd) + +static struct dentry *apple_pmc_debugfs_root; + +static u32 apple_pmc_enable_read(struct apple_pmc *pmc, unsigned int rail) +{ + return readl(pmc->rails + pmc->hw->rail_enable + 4 * rail); +} + +/* Caller holds pmc->lock. Every write is read back, as macOS does. */ +static int apple_pmc_enable_update(struct apple_pmc *pmc, unsigned int rail, + u32 set, u32 clear) +{ + void __iomem *reg = pmc->rails + pmc->hw->rail_enable + 4 * rail; + u32 old = readl(reg); + u32 val = (old | set) & ~clear; + + if (val == old) + return 0; + + writel(val, reg); + if (readl(reg) != val) { + dev_err_ratelimited(pmc->dev, + "rail %u enable %#x -> %#x did not stick\n", + rail, old, val); + return -EIO; + } + + return 0; +} + +static int apple_pmc_agent_enable(struct apple_pmc *pmc, unsigned int agent, + bool enable) +{ + unsigned long flags; + unsigned int rail; + int ret = 0; + + raw_spin_lock_irqsave(&pmc->lock, flags); + for (rail = 0; rail < pmc->hw->rails && !ret; rail++) + ret = apple_pmc_enable_update(pmc, rail, + enable ? BIT(agent) : 0, + enable ? 0 : BIT(agent)); + raw_spin_unlock_irqrestore(&pmc->lock, flags); + + return ret; +} + +static struct apple_pmc *apple_pmc_from_dev(struct device *dev) +{ + return dev ? dev_get_drvdata(dev) : NULL; +} + +/** + * apple_pmc_rail_count() - number of rails of a PMC + * @dev: the PMC device + * + * Return: the number of rails, or 0 if @dev is not a bound PMC. + */ +unsigned int apple_pmc_rail_count(struct device *dev) +{ + struct apple_pmc *pmc = apple_pmc_from_dev(dev); + + return pmc ? pmc->hw->rails : 0; +} +EXPORT_SYMBOL_GPL(apple_pmc_rail_count); + +/** + * apple_pmc_voter_enabled() - whether the PMC honours an agent on a rail + * @dev: the PMC device + * @agent: voter agent + * @rail: rail + * + * Context: Any context. + */ +bool apple_pmc_voter_enabled(struct device *dev, unsigned int agent, + unsigned int rail) +{ + struct apple_pmc *pmc = apple_pmc_from_dev(dev); + + if (!pmc || agent >= pmc->hw->agents || rail >= pmc->hw->rails) + return false; + + return apple_pmc_enable_read(pmc, rail) & BIT(agent); +} +EXPORT_SYMBOL_GPL(apple_pmc_voter_enabled); + +/** + * apple_pmc_floor_read() - read the perf-state floor word of an agent + * @dev: the PMC device + * @agent: voter agent + * + * Context: Any context. + * Return: the floor word, or 0 if @agent is out of range. + */ +u32 apple_pmc_floor_read(struct device *dev, unsigned int agent) +{ + struct apple_pmc *pmc = apple_pmc_from_dev(dev); + + if (!pmc || agent >= pmc->hw->agents) + return 0; + + return readl(pmc->voters + agent * pmc->hw->agent_stride); +} +EXPORT_SYMBOL_GPL(apple_pmc_floor_read); + +/** + * apple_pmc_floor_update() - update the perf-state floor of an agent + * @dev: the PMC device + * @agent: voter agent + * @mask: bits of the floor word to change, see APPLE_PMC_RAIL_MASK() + * @value: new value of the bits in @mask + * + * The floor only takes effect on rails where the agent's interface is + * enabled. + * + * Context: Any context. + * Return: 0 on success, -EINVAL if @agent is out of range, or -EIO if the + * PMC did not accept the new floor. + */ +int apple_pmc_floor_update(struct device *dev, unsigned int agent, u32 mask, + u32 value) +{ + struct apple_pmc *pmc = apple_pmc_from_dev(dev); + void __iomem *reg; + unsigned long flags; + u32 old, val; + int ret = 0; + + if (!pmc || agent >= pmc->hw->agents) + return -EINVAL; + + reg = pmc->voters + agent * pmc->hw->agent_stride; + raw_spin_lock_irqsave(&pmc->lock, flags); + old = readl(reg); + val = (old & ~mask) | (value & mask); + if (val != old) { + writel(val, reg); + if ((readl(reg) & mask) != (value & mask)) + ret = -EIO; + } + raw_spin_unlock_irqrestore(&pmc->lock, flags); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmc_floor_update); + +static int apple_pmc_domain_power_on(struct generic_pm_domain *genpd) +{ + struct apple_pmc_domain *dom = genpd_to_apple_pmc_domain(genpd); + + return apple_pmc_agent_enable(dom->pmc, dom->agent, true); +} + +static int apple_pmc_domain_power_off(struct generic_pm_domain *genpd) +{ + struct apple_pmc_domain *dom = genpd_to_apple_pmc_domain(genpd); + + return apple_pmc_agent_enable(dom->pmc, dom->agent, false); +} + +static void apple_pmc_domain_free(struct apple_pmc_domain *dom) +{ + if (dom->provider) + of_genpd_del_provider(dom->np); + if (pm_genpd_remove(&dom->genpd)) { + /* The core still owns this domain. Keep its storage alive. */ + dev_warn(dom->pmc->dev, "retaining busy power domain %s\n", + dom->genpd.name); + return; + } + of_node_put(dom->np); + kfree(dom->genpd.name); + kfree(dom); +} + +static int apple_pmc_domain_add(struct apple_pmc *pmc, struct device_node *np) +{ + struct device *dev = pmc->dev; + struct apple_pmc_domain *dom; + struct of_phandle_args parent; + const char *label; + u32 agent; + int i, ret; + + ret = of_property_read_u32(np, "reg", &agent); + if (ret) + return dev_err_probe(dev, ret, "%pOF: missing reg\n", np); + if (agent >= pmc->hw->agents) + return dev_err_probe(dev, -EINVAL, "%pOF: invalid agent %u\n", + np, agent); + if (pmc->static_agents & BIT(agent)) + return dev_err_probe(dev, -EINVAL, + "%pOF: agent %u is also static\n", np, + agent); + + dom = kzalloc_obj(*dom); + if (!dom) + return -ENOMEM; + + dom->pmc = pmc; + dom->np = of_node_get(np); + dom->agent = agent; + if (of_property_read_string(np, "label", &label)) + label = np->name; + dom->genpd.name = kasprintf(GFP_KERNEL, "pmc-%s", label); + if (!dom->genpd.name) { + of_node_put(dom->np); + kfree(dom); + return -ENOMEM; + } + dom->genpd.power_on = apple_pmc_domain_power_on; + dom->genpd.power_off = apple_pmc_domain_power_off; + + /* + * Start from the hardware state, so that an agent enabled by the + * bootloader is not disabled under a device that is still running. + */ + ret = pm_genpd_init(&dom->genpd, NULL, + !(apple_pmc_enable_read(pmc, 0) & BIT(agent))); + if (ret) { + of_node_put(dom->np); + kfree(dom->genpd.name); + kfree(dom); + return dev_err_probe(dev, ret, "%pOF: genpd init failed\n", np); + } + list_add_tail(&dom->list, &pmc->domains); + + ret = of_genpd_add_provider_simple(np, &dom->genpd); + if (ret) + return dev_err_probe(dev, ret, "%pOF: provider failed\n", np); + dom->provider = true; + + for (i = 0; !of_parse_phandle_with_args(np, "power-domains", + "#power-domain-cells", i, + &parent); i++) { + struct of_phandle_args child = { .np = np }; + + ret = of_genpd_add_subdomain(&parent, &child); + of_node_put(parent.np); + if (ret) + return dev_err_probe(dev, ret, + "%pOF: failed to join parent domain\n", + np); + } + + return 0; +} + +static int apple_pmc_status_show(struct seq_file *s, void *unused) +{ + struct apple_pmc *pmc = s->private; + const struct apple_pmc_hw *hw = pmc->hw; + unsigned int i; + + seq_printf(s, "static agents: %#x\n", pmc->static_agents); + for (i = 0; i < hw->agents; i++) + seq_printf(s, "agent%u floor: %#010x\n", i, + readl(pmc->voters + i * hw->agent_stride)); + for (i = 0; i < hw->rails; i++) + seq_printf(s, "rail%u state: %#010x enable: %#010x (saved %#010x)\n", + i, readl(pmc->rails + hw->rail_state + 4 * i), + apple_pmc_enable_read(pmc, i), pmc->saved_enable[i]); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(apple_pmc_status); + +static void apple_pmc_release(void *data) +{ + struct apple_pmc *pmc = data; + struct apple_pmc_domain *dom, *tmp; + unsigned long flags; + unsigned int rail; + + of_platform_depopulate(pmc->dev); + debugfs_remove_recursive(pmc->debugfs); + list_for_each_entry_safe_reverse(dom, tmp, &pmc->domains, list) { + list_del(&dom->list); + apple_pmc_domain_free(dom); + } + + raw_spin_lock_irqsave(&pmc->lock, flags); + for (rail = 0; rail < pmc->hw->rails; rail++) + writel(pmc->saved_enable[rail], + pmc->rails + pmc->hw->rail_enable + 4 * rail); + raw_spin_unlock_irqrestore(&pmc->lock, flags); +} + +static int apple_pmc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + u32 agents[APPLE_PMC_MAX_AGENTS]; + struct device_node *child; + struct apple_pmc *pmc; + unsigned int rail; + int count, i, ret; + + pmc = devm_kzalloc(dev, sizeof(*pmc), GFP_KERNEL); + if (!pmc) + return -ENOMEM; + + pmc->dev = dev; + pmc->hw = of_device_get_match_data(dev); + raw_spin_lock_init(&pmc->lock); + INIT_LIST_HEAD(&pmc->domains); + + if (pmc->hw->rails > APPLE_PMC_MAX_RAILS || + pmc->hw->agents > APPLE_PMC_MAX_AGENTS) + return -EINVAL; + + pmc->voters = devm_platform_ioremap_resource_byname(pdev, "voters"); + if (IS_ERR(pmc->voters)) + return PTR_ERR(pmc->voters); + pmc->rails = devm_platform_ioremap_resource_byname(pdev, "rails"); + if (IS_ERR(pmc->rails)) + return PTR_ERR(pmc->rails); + + /* An unpowered PMC reads all-ones; do not write to it. */ + for (rail = 0; rail < pmc->hw->rails; rail++) { + pmc->saved_enable[rail] = apple_pmc_enable_read(pmc, rail); + if (pmc->saved_enable[rail] == ~0U) + return dev_err_probe(dev, -ENODEV, + "PMC is not responding\n"); + } + + count = of_property_count_u32_elems(np, "apple,static-voters"); + if (count > 0) { + if (count > ARRAY_SIZE(agents)) + return -EINVAL; + ret = of_property_read_u32_array(np, "apple,static-voters", + agents, count); + if (ret) + return ret; + for (i = 0; i < count; i++) { + if (agents[i] >= pmc->hw->agents) + return dev_err_probe(dev, -EINVAL, + "invalid static voter %u\n", + agents[i]); + pmc->static_agents |= BIT(agents[i]); + } + } + + platform_set_drvdata(pdev, pmc); + ret = devm_add_action_or_reset(dev, apple_pmc_release, pmc); + if (ret) + return ret; + + for (i = 0; i < pmc->hw->agents; i++) { + if (!(pmc->static_agents & BIT(i))) + continue; + ret = apple_pmc_agent_enable(pmc, i, true); + if (ret) + return dev_err_probe(dev, ret, + "failed to enable static voter %d\n", + i); + } + + for_each_available_child_of_node(np, child) { + if (!of_property_present(child, "#power-domain-cells")) + continue; + ret = apple_pmc_domain_add(pmc, child); + if (ret) { + of_node_put(child); + return ret; + } + } + + pmc->debugfs = debugfs_create_dir(dev_name(dev), apple_pmc_debugfs_root); + debugfs_create_file("status", 0400, pmc->debugfs, pmc, + &apple_pmc_status_fops); + + ret = of_platform_populate(np, NULL, NULL, dev); + if (ret) + return dev_err_probe(dev, ret, "failed to create child devices\n"); + + return 0; +} + +static const struct apple_pmc_hw apple_pmc_hw_t8140 = { + .agent_stride = 0x4000, + .agents = 4, + .rails = 4, + .rail_state = 0x0, + .rail_enable = 0x2000, +}; + +static const struct of_device_id apple_pmc_of_match[] = { + { .compatible = "apple,t8140-pmc", .data = &apple_pmc_hw_t8140 }, + {} +}; +MODULE_DEVICE_TABLE(of, apple_pmc_of_match); + +static struct platform_driver apple_pmc_driver = { + .probe = apple_pmc_probe, + .driver = { + .name = "apple-pmc", + .of_match_table = apple_pmc_of_match, + /* Power-domain providers are not intended for manual unbinding. */ + .suppress_bind_attrs = true, + }, +}; + +static int __init apple_pmc_init(void) +{ + apple_pmc_debugfs_root = debugfs_create_dir("apple-pmc", NULL); + return platform_driver_register(&apple_pmc_driver); +} +module_init(apple_pmc_init); + +MODULE_DESCRIPTION("Apple SoC PMC voter interface driver"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/include/linux/soc/apple/pmc.h b/include/linux/soc/apple/pmc.h new file mode 100644 index 0000000000000..be391b83d9752 --- /dev/null +++ b/include/linux/soc/apple/pmc.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Apple SoC PMC voter interface + * Copyright (C) The Asahi Linux Contributors + */ + +#ifndef _LINUX_SOC_APPLE_PMC_H_ +#define _LINUX_SOC_APPLE_PMC_H_ + +#include +#include + +struct device; + +/* + * A PMC perf-state floor word holds one 4-bit performance state per rail, + * rail N in bits [4N+3:4N]. + */ +#define APPLE_PMC_RAIL_SHIFT(rail) (4 * (rail)) +#define APPLE_PMC_RAIL_MASK(rail) (0xfU << APPLE_PMC_RAIL_SHIFT(rail)) + +#if IS_ENABLED(CONFIG_APPLE_PMC) +unsigned int apple_pmc_rail_count(struct device *pmc); +bool apple_pmc_voter_enabled(struct device *pmc, unsigned int agent, + unsigned int rail); +u32 apple_pmc_floor_read(struct device *pmc, unsigned int agent); +int apple_pmc_floor_update(struct device *pmc, unsigned int agent, u32 mask, + u32 value); +#else +static inline unsigned int apple_pmc_rail_count(struct device *pmc) +{ + return 0; +} + +static inline bool apple_pmc_voter_enabled(struct device *pmc, + unsigned int agent, + unsigned int rail) +{ + return false; +} + +static inline u32 apple_pmc_floor_read(struct device *pmc, unsigned int agent) +{ + return 0; +} + +static inline int apple_pmc_floor_update(struct device *pmc, + unsigned int agent, u32 mask, + u32 value) +{ + return -ENODEV; +} +#endif + +#endif /* _LINUX_SOC_APPLE_PMC_H_ */ From e10943767a1b6323e26e31149b3c921894a65c2e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 18:11:39 -0400 Subject: [PATCH 0312/2680] soc: apple: add host performance controller floors On macOS the CLPC raises the DRAM controller (DCS) and SoC fabric floors of its PMC voter agent while the CPU clusters or the GPU are busy (AppleT8140CLPC: clpc::memory::DVFManager and clpc::soc::DVFManager, both ending in the PMC CLPC perf-state floor), and lets them fall at idle. Linux has no equivalent, so CPU-bound work runs with the DRAM controller at whatever state the other agents happen to request. Sample every cpufreq policy every 4 ms, the CLPC sampling interval from the Apple device tree. A policy whose busiest CPU was busy for at least half of the period requests a DCS and fabric state from a table indexed by its frequency quartile. Optionally, while the GPU's PMGR state reports the GPU as actually powered, request a fixed GPU floor. Each rail keeps its last request for 32 ms after demand stops, then returns to 0. The native mapping from CPU and GPU demand to DCS and fabric states is not decoded. The default tables (DCS 1-4 by quartile, no fabric request, no GPU floor) are measured policy and can be changed with module parameters. A debugfs file holds a rail at a fixed state for diagnosis only. Assisted-by: LLM Signed-off-by: Diljit Singh --- drivers/soc/apple/Kconfig | 12 + drivers/soc/apple/Makefile | 2 + drivers/soc/apple/clpc.c | 531 +++++++++++++++++++++++++++++++++++++ 3 files changed, 545 insertions(+) create mode 100644 drivers/soc/apple/clpc.c diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 2032de34e6c04..2e5aaa61ba051 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -134,6 +134,18 @@ config APPLE_PMP Say 'y' here if you have an Apple SoC. +config APPLE_CLPC + tristate "Apple host performance controller floors" + depends on ARCH_APPLE || COMPILE_TEST + depends on APPLE_PMC && CPU_FREQ + select MFD_SYSCON + help + Raises the DRAM controller and SoC fabric performance floors of the + host's PMC voter agent while CPU clusters, and optionally the GPU, + are busy, and releases them at idle. + + Say 'y' or 'm' here if you have an Apple SoC with a PMC. + endmenu endif diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 14f4d8936f1f6..38b59bf10899f 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -28,3 +28,5 @@ obj-$(CONFIG_APPLE_PMP) += pmp.o obj-$(CONFIG_APPLE_MCC) += apple-mcc.o apple-mcc-y := mcc.o + +obj-$(CONFIG_APPLE_CLPC) += clpc.o diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c new file mode 100644 index 0000000000000..32b88c7c968ba --- /dev/null +++ b/drivers/soc/apple/clpc.c @@ -0,0 +1,531 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC host performance controller (CLPC) floors + * + * On macOS the CLPC raises the floor of the memory controller (DCS) and the + * SoC fabric while the CPU clusters or the GPU are busy, through its own + * voter agent in the PMC. This driver keeps that agent's floor in step with + * Linux CPU load and, optionally, GPU activity: + * + * - every sampling period, each cpufreq policy reports how busy its + * busiest CPU was and how fast the policy runs relative to its maximum; + * a busy policy requests a DCS and fabric state from a table indexed by + * that frequency quartile; + * - while the GPU's PMGR power state reports the GPU as actually powered, + * a fixed GPU floor is requested; + * - each rail keeps its last request for a hold time after demand stops, + * then drops back to 0, so idle releases the floor. + * + * The native mapping from CPU and GPU demand to DCS and fabric states has + * not been decoded. The default tables are measured policy, not a copy of + * macOS, and can be changed with module parameters. + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define APPLE_CLPC_MAX_CLUSTERS 8 +#define APPLE_CLPC_LEVELS 4 + +/* PMGR power state: ACTUAL is bits [7:4]; 0xf means powered on. */ +#define APPLE_PMGR_PS_ACTUAL GENMASK(7, 4) +#define APPLE_PMGR_PS_ACTIVE 0xf + +enum apple_clpc_rail { + APPLE_CLPC_DCS, + APPLE_CLPC_SOC, + APPLE_CLPC_RAILS, +}; + +static unsigned int sample_ms = 4; +module_param(sample_ms, uint, 0444); +MODULE_PARM_DESC(sample_ms, "Sampling period in ms"); + +static unsigned int busy_pct = 50; +module_param(busy_pct, uint, 0444); +MODULE_PARM_DESC(busy_pct, "Busy threshold of a CPU cluster, percent of the sampling period"); + +static unsigned int hold_ms = 32; +module_param(hold_ms, uint, 0444); +MODULE_PARM_DESC(hold_ms, "Time a rail keeps its last request after demand stops, ms"); + +static char *dcs_map = "1,2,3,4"; +module_param(dcs_map, charp, 0444); +MODULE_PARM_DESC(dcs_map, "DCS state for a busy cluster below 25, 50, 75 and 100 percent of its maximum frequency"); + +static char *soc_map = "0,0,0,0"; +module_param(soc_map, charp, 0444); +MODULE_PARM_DESC(soc_map, "Fabric state for a busy cluster below 25, 50, 75 and 100 percent of its maximum frequency"); + +static unsigned int gpu_dcs; +module_param(gpu_dcs, uint, 0444); +MODULE_PARM_DESC(gpu_dcs, "DCS state while the GPU is powered (0: none)"); + +static unsigned int gpu_soc; +module_param(gpu_soc, uint, 0444); +MODULE_PARM_DESC(gpu_soc, "Fabric state while the GPU is powered (0: none)"); + +/** + * struct apple_clpc_hw - per-SoC CLPC parameters + * @rail: PMC rail of the DCS and of the fabric + * @max_state: highest state of the DCS and of the fabric + */ +struct apple_clpc_hw { + unsigned int rail[APPLE_CLPC_RAILS]; + unsigned int max_state[APPLE_CLPC_RAILS]; +}; + +struct apple_clpc_cluster { + unsigned int policy_cpu; + cpumask_var_t cpus; + unsigned int busy; + unsigned int level; + unsigned int request[APPLE_CLPC_RAILS]; +}; + +struct apple_clpc_vote { + unsigned int state; + unsigned long last_demand; +}; + +struct apple_clpc { + struct device *dev; + struct device *pmc; + const struct apple_clpc_hw *hw; + unsigned int agent; + struct regmap *gpu_regmap; + u32 gpu_offset; + struct delayed_work work; + struct apple_clpc_cluster clusters[APPLE_CLPC_MAX_CLUSTERS]; + unsigned int nr_clusters; + cpumask_var_t covered; + unsigned long next_scan; + u64 *idle; + u64 *wall; + u8 map[APPLE_CLPC_RAILS][APPLE_CLPC_LEVELS]; + unsigned int gpu_state[APPLE_CLPC_RAILS]; + struct apple_clpc_vote vote[APPLE_CLPC_RAILS]; + int force[APPLE_CLPC_RAILS]; + u32 mask; + u32 saved_floor; + bool gpu_on; + unsigned long samples; + unsigned long update_errors; + struct dentry *debugfs; +}; + +static int apple_clpc_parse_map(struct apple_clpc *clpc, const char *s, + enum apple_clpc_rail rail) +{ + unsigned int v[APPLE_CLPC_LEVELS]; + unsigned int i; + + if (sscanf(s, "%u,%u,%u,%u", &v[0], &v[1], &v[2], &v[3]) != 4) + return -EINVAL; + for (i = 0; i < APPLE_CLPC_LEVELS; i++) { + if (v[i] > clpc->hw->max_state[rail]) + return -EINVAL; + clpc->map[rail][i] = v[i]; + } + + return 0; +} + +/* cpufreq policies can appear after probe; pick up new ones as they do. */ +static void apple_clpc_find_clusters(struct apple_clpc *clpc) +{ + unsigned int cpu, i; + + for_each_possible_cpu(cpu) { + struct cpufreq_policy *policy; + struct apple_clpc_cluster *cl; + + if (cpumask_test_cpu(cpu, clpc->covered)) + continue; + policy = cpufreq_cpu_get(cpu); + if (!policy) + continue; + for (i = 0; i < clpc->nr_clusters; i++) + if (clpc->clusters[i].policy_cpu == policy->cpu) + break; + if (i == clpc->nr_clusters && i < APPLE_CLPC_MAX_CLUSTERS) { + cl = &clpc->clusters[clpc->nr_clusters++]; + cl->policy_cpu = policy->cpu; + } + if (i < clpc->nr_clusters) { + cl = &clpc->clusters[i]; + cpumask_set_cpu(cpu, cl->cpus); + cpumask_set_cpu(cpu, clpc->covered); + clpc->idle[cpu] = get_cpu_idle_time(cpu, &clpc->wall[cpu], 0); + } + cpufreq_cpu_put(policy); + } +} + +static void apple_clpc_sample_cluster(struct apple_clpc *clpc, + struct apple_clpc_cluster *cl) +{ + struct cpufreq_policy *policy; + unsigned int cpu, busy = 0, q, rail; + + for_each_cpu(cpu, cl->cpus) { + u64 wall, idle = get_cpu_idle_time(cpu, &wall, 0); + u64 dw = wall - clpc->wall[cpu], di = idle - clpc->idle[cpu]; + + clpc->wall[cpu] = wall; + clpc->idle[cpu] = idle; + if (dw && di <= dw) + busy = max_t(unsigned int, busy, + div64_u64((dw - di) * 100, dw)); + } + cl->busy = busy; + + cl->level = 0; + policy = cpufreq_cpu_get(cl->policy_cpu); + if (policy) { + if (policy->cpuinfo.max_freq) + cl->level = policy->cur * 100 / policy->cpuinfo.max_freq; + cpufreq_cpu_put(policy); + } + + q = min_t(unsigned int, cl->level / (100 / APPLE_CLPC_LEVELS), + APPLE_CLPC_LEVELS - 1); + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) + cl->request[rail] = busy >= busy_pct ? clpc->map[rail][q] : 0; +} + +static bool apple_clpc_gpu_powered(struct apple_clpc *clpc) +{ + u32 ps; + + /* + * Hardware auto-gating powers the GPU down while the requested state + * stays on, so only the actual state says whether the GPU runs. + */ + if (regmap_read(clpc->gpu_regmap, clpc->gpu_offset, &ps)) + return false; + + return FIELD_GET(APPLE_PMGR_PS_ACTUAL, ps) == APPLE_PMGR_PS_ACTIVE; +} + +static unsigned int apple_clpc_vote(struct apple_clpc_vote *vote, + unsigned int demand, unsigned long now) +{ + if (demand) { + vote->last_demand = now; + vote->state = demand; + } else if (time_after_eq(now, vote->last_demand + + msecs_to_jiffies(hold_ms))) { + vote->state = 0; + } + + return vote->state; +} + +static u32 apple_clpc_floor(struct apple_clpc *clpc, const unsigned int *state) +{ + u32 floor = 0; + unsigned int rail; + + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) + floor |= (state[rail] & 0xf) << + APPLE_PMC_RAIL_SHIFT(clpc->hw->rail[rail]); + + return floor; +} + +static void apple_clpc_work(struct work_struct *work) +{ + struct apple_clpc *clpc = container_of(to_delayed_work(work), + struct apple_clpc, work); + unsigned int demand[APPLE_CLPC_RAILS] = {}; + unsigned int state[APPLE_CLPC_RAILS]; + unsigned long now = jiffies; + unsigned int i, rail; + int force; + + clpc->samples++; + if (!cpumask_equal(clpc->covered, cpu_possible_mask) && + time_after_eq(now, clpc->next_scan)) { + apple_clpc_find_clusters(clpc); + clpc->next_scan = now + HZ; + } + + for (i = 0; i < clpc->nr_clusters; i++) { + struct apple_clpc_cluster *cl = &clpc->clusters[i]; + + apple_clpc_sample_cluster(clpc, cl); + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) + demand[rail] = max(demand[rail], cl->request[rail]); + } + + clpc->gpu_on = clpc->gpu_regmap && + (clpc->gpu_state[APPLE_CLPC_DCS] || + clpc->gpu_state[APPLE_CLPC_SOC]) && + apple_clpc_gpu_powered(clpc); + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) { + if (clpc->gpu_on) + demand[rail] = max(demand[rail], clpc->gpu_state[rail]); + state[rail] = apple_clpc_vote(&clpc->vote[rail], demand[rail], + now); + force = READ_ONCE(clpc->force[rail]); + if (force >= 0) + state[rail] = force; + } + + if (apple_pmc_floor_update(clpc->pmc, clpc->agent, clpc->mask, + apple_clpc_floor(clpc, state))) + clpc->update_errors++; + + queue_delayed_work(system_highpri_wq, &clpc->work, + msecs_to_jiffies(sample_ms)); +} + +static int apple_clpc_status_show(struct seq_file *s, void *unused) +{ + struct apple_clpc *clpc = s->private; + unsigned int i, rail; + + seq_printf(s, "agent: %u floor: %#010x (saved %#010x)\n", clpc->agent, + apple_pmc_floor_read(clpc->pmc, clpc->agent), + clpc->saved_floor); + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) + seq_printf(s, "%s: rail %u state %u enabled %u force %d map %u,%u,%u,%u gpu %u\n", + rail == APPLE_CLPC_DCS ? "dcs" : "soc", + clpc->hw->rail[rail], clpc->vote[rail].state, + apple_pmc_voter_enabled(clpc->pmc, clpc->agent, + clpc->hw->rail[rail]), + READ_ONCE(clpc->force[rail]), clpc->map[rail][0], + clpc->map[rail][1], clpc->map[rail][2], + clpc->map[rail][3], clpc->gpu_state[rail]); + seq_printf(s, "gpu: %s\n", !clpc->gpu_regmap ? "no input" : + !clpc->gpu_state[APPLE_CLPC_DCS] && + !clpc->gpu_state[APPLE_CLPC_SOC] ? "no floor" : + clpc->gpu_on ? "powered" : "off"); + seq_printf(s, "samples: %lu update errors: %lu\n", clpc->samples, + clpc->update_errors); + for (i = 0; i < clpc->nr_clusters; i++) + seq_printf(s, "cluster cpu%u: busy %u%% level %u%% -> dcs %u soc %u\n", + clpc->clusters[i].policy_cpu, clpc->clusters[i].busy, + clpc->clusters[i].level, + clpc->clusters[i].request[APPLE_CLPC_DCS], + clpc->clusters[i].request[APPLE_CLPC_SOC]); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(apple_clpc_status); + +/* + * Diagnostic override: "DCS SOC" holds each rail at a fixed state, -1 + * returns that rail to demand control; "-1" alone clears both. + */ +static ssize_t apple_clpc_force_write(struct file *file, + const char __user *ubuf, size_t len, + loff_t *pos) +{ + struct apple_clpc *clpc = file->private_data; + int v[APPLE_CLPC_RAILS], n; + unsigned int rail; + char buf[32]; + + if (len >= sizeof(buf)) + return -EINVAL; + if (copy_from_user(buf, ubuf, len)) + return -EFAULT; + buf[len] = '\0'; + + n = sscanf(buf, "%d %d", &v[APPLE_CLPC_DCS], &v[APPLE_CLPC_SOC]); + if (n == 1 && v[APPLE_CLPC_DCS] == -1) + v[APPLE_CLPC_SOC] = -1; + else if (n != 2) + return -EINVAL; + + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) + if (v[rail] < -1 || v[rail] > (int)clpc->hw->max_state[rail]) + return -EINVAL; + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) + WRITE_ONCE(clpc->force[rail], v[rail]); + + dev_info(clpc->dev, "diagnostic override: dcs %d soc %d\n", + v[APPLE_CLPC_DCS], v[APPLE_CLPC_SOC]); + return len; +} + +static const struct file_operations apple_clpc_force_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = apple_clpc_force_write, +}; + +static int apple_clpc_gpu_input(struct apple_clpc *clpc) +{ + struct device_node *ps, *pmgr; + int ret; + + ps = of_parse_phandle(clpc->dev->of_node, "apple,gpu-power-state", 0); + if (!ps) + return 0; + + ret = of_property_read_u32(ps, "reg", &clpc->gpu_offset); + pmgr = of_get_parent(ps); + of_node_put(ps); + if (ret) { + of_node_put(pmgr); + return ret; + } + + clpc->gpu_regmap = syscon_node_to_regmap(pmgr); + of_node_put(pmgr); + if (IS_ERR(clpc->gpu_regmap)) { + ret = PTR_ERR(clpc->gpu_regmap); + clpc->gpu_regmap = NULL; + return ret; + } + + return 0; +} + +static void apple_clpc_free_clusters(struct apple_clpc *clpc) +{ + unsigned int i; + + for (i = 0; i < APPLE_CLPC_MAX_CLUSTERS; i++) + free_cpumask_var(clpc->clusters[i].cpus); + free_cpumask_var(clpc->covered); +} + +static void apple_clpc_stop(void *data) +{ + struct apple_clpc *clpc = data; + + cancel_delayed_work_sync(&clpc->work); + debugfs_remove_recursive(clpc->debugfs); + apple_pmc_floor_update(clpc->pmc, clpc->agent, clpc->mask, + clpc->saved_floor); + apple_clpc_free_clusters(clpc); +} + +static int apple_clpc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_clpc *clpc; + unsigned int rail, i; + int ret; + + clpc = devm_kzalloc(dev, sizeof(*clpc), GFP_KERNEL); + if (!clpc) + return -ENOMEM; + + clpc->dev = dev; + clpc->pmc = dev->parent; + clpc->hw = of_device_get_match_data(dev); + if (!apple_pmc_rail_count(clpc->pmc)) + return dev_err_probe(dev, -ENODEV, "parent is not a PMC\n"); + + ret = of_property_read_u32(dev->of_node, "reg", &clpc->agent); + if (ret) + return dev_err_probe(dev, ret, "missing voter agent\n"); + + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) { + if (clpc->hw->rail[rail] >= apple_pmc_rail_count(clpc->pmc)) + return -EINVAL; + clpc->mask |= APPLE_PMC_RAIL_MASK(clpc->hw->rail[rail]); + clpc->force[rail] = -1; + } + + ret = apple_clpc_parse_map(clpc, dcs_map, APPLE_CLPC_DCS) ?: + apple_clpc_parse_map(clpc, soc_map, APPLE_CLPC_SOC); + if (ret) + return dev_err_probe(dev, ret, "invalid state map\n"); + if (gpu_dcs > clpc->hw->max_state[APPLE_CLPC_DCS] || + gpu_soc > clpc->hw->max_state[APPLE_CLPC_SOC]) + return dev_err_probe(dev, -EINVAL, "invalid GPU state\n"); + clpc->gpu_state[APPLE_CLPC_DCS] = gpu_dcs; + clpc->gpu_state[APPLE_CLPC_SOC] = gpu_soc; + + ret = apple_clpc_gpu_input(clpc); + if (ret) + return dev_err_probe(dev, ret, "invalid GPU power state\n"); + + clpc->idle = devm_kcalloc(dev, nr_cpu_ids, sizeof(*clpc->idle), + GFP_KERNEL); + clpc->wall = devm_kcalloc(dev, nr_cpu_ids, sizeof(*clpc->wall), + GFP_KERNEL); + if (!clpc->idle || !clpc->wall) + return -ENOMEM; + + for (i = 0; i < APPLE_CLPC_MAX_CLUSTERS; i++) { + if (!zalloc_cpumask_var(&clpc->clusters[i].cpus, GFP_KERNEL)) { + apple_clpc_free_clusters(clpc); + return -ENOMEM; + } + } + if (!zalloc_cpumask_var(&clpc->covered, GFP_KERNEL)) { + apple_clpc_free_clusters(clpc); + return -ENOMEM; + } + + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) + if (!apple_pmc_voter_enabled(clpc->pmc, clpc->agent, + clpc->hw->rail[rail])) + dev_warn(dev, "voter agent %u is not enabled on rail %u\n", + clpc->agent, clpc->hw->rail[rail]); + + clpc->saved_floor = apple_pmc_floor_read(clpc->pmc, clpc->agent) & + clpc->mask; + apple_clpc_find_clusters(clpc); + clpc->next_scan = jiffies + HZ; + INIT_DELAYED_WORK(&clpc->work, apple_clpc_work); + + clpc->debugfs = debugfs_create_dir(dev_name(dev), NULL); + debugfs_create_file("status", 0400, clpc->debugfs, clpc, + &apple_clpc_status_fops); + debugfs_create_file("force", 0200, clpc->debugfs, clpc, + &apple_clpc_force_fops); + + ret = devm_add_action_or_reset(dev, apple_clpc_stop, clpc); + if (ret) + return ret; + + queue_delayed_work(system_highpri_wq, &clpc->work, + msecs_to_jiffies(sample_ms)); + return 0; +} + +/* T8140: DCS states 0-4 on rail 0, fabric states 0-3 on rail 1. */ +static const struct apple_clpc_hw apple_clpc_hw_t8140 = { + .rail = { [APPLE_CLPC_DCS] = 0, [APPLE_CLPC_SOC] = 1 }, + .max_state = { [APPLE_CLPC_DCS] = 4, [APPLE_CLPC_SOC] = 3 }, +}; + +static const struct of_device_id apple_clpc_of_match[] = { + { .compatible = "apple,t8140-clpc", .data = &apple_clpc_hw_t8140 }, + {} +}; +MODULE_DEVICE_TABLE(of, apple_clpc_of_match); + +static struct platform_driver apple_clpc_driver = { + .probe = apple_clpc_probe, + .driver = { + .name = "apple-clpc", + .of_match_table = apple_clpc_of_match, + }, +}; +module_platform_driver(apple_clpc_driver); + +MODULE_DESCRIPTION("Apple SoC host performance controller floors"); +MODULE_LICENSE("Dual MIT/GPL"); From 69a097c3d62548ff4f7d6e55c51ee7d2388f027d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:42:02 +1100 Subject: [PATCH 0313/2680] pmdomain: apple: pmc: report the honoured floor of each agent The words at the start of the "rails" region are not per-rail states. There is one per voter agent, and each holds that agent's floor as the PMC honours it: masked by the agent's interface enables and clamped to each rail's highest state. On the J700, with agent 1's floor set to 0x40 and its interface enabled only on rail 1, word 1 reads 0x30 (the fabric rail tops out at 3) and word 0 stays 0; enabling agent 1 on rail 0 instead with a floor of 4 makes word 1 read 4. Name the offset after what it holds and print each agent's floor next to what the PMC honours of it, and the rail enables on their own lines. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmc.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 027fa33a6f8fb..fac0b1dfc02b6 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -42,7 +42,9 @@ * in the "voters" region * @agents: number of voter agents * @rails: number of rails - * @rail_state: offset of the per-rail state words in the "rails" region + * @agent_state: offset of the per-agent state words in the "rails" region; + * each holds the agent's floor as the PMC honours it: masked by the + * agent's interface enables and clamped to each rail's highest state * @rail_enable: offset of the per-rail interface enable words in the * "rails" region */ @@ -50,7 +52,7 @@ struct apple_pmc_hw { u32 agent_stride; u32 agents; u32 rails; - u32 rail_state; + u32 agent_state; u32 rail_enable; }; @@ -334,12 +336,12 @@ static int apple_pmc_status_show(struct seq_file *s, void *unused) seq_printf(s, "static agents: %#x\n", pmc->static_agents); for (i = 0; i < hw->agents; i++) - seq_printf(s, "agent%u floor: %#010x\n", i, - readl(pmc->voters + i * hw->agent_stride)); + seq_printf(s, "agent%u floor: %#010x honoured: %#010x\n", i, + readl(pmc->voters + i * hw->agent_stride), + readl(pmc->rails + hw->agent_state + 4 * i)); for (i = 0; i < hw->rails; i++) - seq_printf(s, "rail%u state: %#010x enable: %#010x (saved %#010x)\n", - i, readl(pmc->rails + hw->rail_state + 4 * i), - apple_pmc_enable_read(pmc, i), pmc->saved_enable[i]); + seq_printf(s, "rail%u enable: %#010x (saved %#010x)\n", + i, apple_pmc_enable_read(pmc, i), pmc->saved_enable[i]); return 0; } @@ -461,7 +463,7 @@ static const struct apple_pmc_hw apple_pmc_hw_t8140 = { .agent_stride = 0x4000, .agents = 4, .rails = 4, - .rail_state = 0x0, + .agent_state = 0x0, .rail_enable = 0x2000, }; From 561a027dec8655a43291e46913c458b1bd6082ca Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:48:27 +1100 Subject: [PATCH 0314/2680] dt-bindings: power: apple,pmc: describe the DCS limit The second register region holds one word per voter agent with the floor the PMC honours for it, followed by one interface enable word per rail; say so instead of calling the first words rail states. The PMC can also limit the DRAM controller (DCS) rail on behalf of the host, which makes it a thermal cooling device. Add #cooling-cells, and an optional power model for the power allocator governor: the system power at each DCS state above the lowest one. Bound the static voter list by the 32 agents an enable word can hold. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/power/apple,pmc.yaml | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/power/apple,pmc.yaml b/Documentation/devicetree/bindings/power/apple,pmc.yaml index 69cdc1cb87197..91c77695178c4 100644 --- a/Documentation/devicetree/bindings/power/apple,pmc.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmc.yaml @@ -29,7 +29,9 @@ properties: reg: items: - description: Perf-state floor words of the voter agents - - description: Rail state and interface enable words + - description: | + Per-agent words holding the floor the PMC honours for each agent, + and per-rail interface enable words reg-names: items: @@ -48,6 +50,26 @@ properties: apple,static-voters: $ref: /schemas/types.yaml#/definitions/uint32-array description: Voter agents to enable on every rail. + minItems: 1 + maxItems: 32 + items: + maximum: 31 + + '#cooling-cells': + const: 2 + description: | + The PMC can limit the DRAM controller (DCS) rail. Cooling state 0 leaves + the rail to the agents' votes; each further state lowers the highest + DCS state the PMC honours for them by one, down to the lowest state. + + apple,dcs-power-microwatt: + description: | + Power the system draws at each DCS performance state above what it + draws at the lowest one, from the lowest state up, measured under a + load whose speed follows the DCS state. With it, the DCS limit is a + power actor that the power allocator thermal governor can drive. + minItems: 2 + maxItems: 16 patternProperties: '^voter@[0-9a-f]+$': @@ -115,7 +137,9 @@ examples: reg-names = "voters", "rails"; #address-cells = <1>; #size-cells = <0>; + #cooling-cells = <2>; apple,static-voters = <0 1 3>; + apple,dcs-power-microwatt = <0 919000 1453000 2058000 2407000>; performance-controller@1 { compatible = "apple,t8140-clpc"; From 469b8a334ad14ce8458a098ff375e88a078a9ed6 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 25 Aug 2026 14:43:51 -0400 Subject: [PATCH 0315/2680] arm64: dts: apple: route base M1/M2 displays to both Type-C ports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — Janne's dp-altmode and ps_atc1_common always-on DTS HACKs are not in asahi-wip and stay dropped; carry the dcpext and sio enablement this routing needs here, and keep this commit's own ps_atc0_common always-on] [ryan: adapted to 7.2.10 — the t8103/t8112 display DARTs take five iommu cells in asahi-wip; use their apertures for the display iommus, and keep Asahi's j473 bluetooth0 alias] Signed-off-by: DjDeveloperr --- arch/arm64/boot/dts/apple/t8103-jxxx.dtsi | 82 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8112-j473.dts | 90 ++++++++++++++++++--- arch/arm64/boot/dts/apple/t8112-jxxx.dtsi | 99 +++++++++++++++++++++++ 3 files changed, 261 insertions(+), 10 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi index a3dc78c24b3c5..9ba9b11ae78a1 100644 --- a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi @@ -13,6 +13,7 @@ aliases { bluetooth0 = &bluetooth0; dcp = &dcp; + dcpext = &dcpext; disp0 = &display; disp0_piodma = &disp0_piodma; disp0-piodma = &disp0_piodma; @@ -81,6 +82,7 @@ compatible = "usb-c-connector"; power-role = "dual"; data-role = "dual"; + displayport = <&dcpext>; ports { #address-cells = <1>; @@ -104,6 +106,13 @@ remote-endpoint = <&usb4_0_acio_tbt>; }; }; + + port@3 { + reg = <3>; + typec0_dcp: endpoint { + remote-endpoint = <&dcpext_typec0>; + }; + }; }; }; }; @@ -119,6 +128,7 @@ compatible = "usb-c-connector"; power-role = "dual"; data-role = "dual"; + displayport = <&dcpext>; ports { #address-cells = <1>; @@ -142,11 +152,83 @@ remote-endpoint = <&usb4_1_acio_tbt>; }; }; + + port@3 { + reg = <3>; + typec1_dcp: endpoint { + remote-endpoint = <&dcpext_typec1>; + }; + }; + }; + }; + }; +}; + +&display { + iommus = <&disp0_dart 0 0x0 0x0 0x0 0xfc000000>, + <&dispext0_dart 0 0x0 0x0 0xf 0xffff0000>; +}; + +&dispext0_dart { + status = "okay"; +}; + +&dcpext_dart { + status = "okay"; +}; + +&dcpext_mbox { + status = "okay"; +}; + +&dcpext { + status = "okay"; + apple,connector-type = "USB-C"; + phys = <&atcphy0 PHY_TYPE_DP>, <&atcphy1 PHY_TYPE_DP>; + phy-names = "typec0", "typec1"; + mux-controls = <&atcphy0_xbar 0>, <&atcphy1_xbar 0>; + mux-control-names = "typec0", "typec1"; + apple,typec-mux-indices = <0 0>; + apple,typec-dptx-phys = <0 1>; + + typec-routes { + #address-cells = <1>; + #size-cells = <0>; + + route@0 { + reg = <0>; + mode-switch; + + port { + dcpext_typec0: endpoint { + remote-endpoint = <&typec0_dcp>; + }; + }; + }; + + route@1 { + reg = <1>; + mode-switch; + + port { + dcpext_typec1: endpoint { + remote-endpoint = <&typec1_dcp>; + }; }; }; }; }; +&dpaudio1 { + status = "okay"; +}; + + + +&ps_atc0_common { + apple,always-on; /* Needs to stay on for DP-alt suspend/resume */ +}; + /* USB controllers */ &dwc3_0 { ports { diff --git a/arch/arm64/boot/dts/apple/t8112-j473.dts b/arch/arm64/boot/dts/apple/t8112-j473.dts index b9fa3cc427c29..9c54bbe563693 100644 --- a/arch/arm64/boot/dts/apple/t8112-j473.dts +++ b/arch/arm64/boot/dts/apple/t8112-j473.dts @@ -9,6 +9,8 @@ /dts-v1/; +#define T8112_TYPEC_DCP + #include "t8112.dtsi" #include "t8112-jxxx.dtsi" @@ -19,18 +21,11 @@ aliases { bluetooth0 = &bluetooth0; - /delete-property/ dcp; - dcpext = &dcpext; ethernet0 = ðernet0; - sio = &sio; wifi0 = &wifi0; }; }; -&framebuffer0 { - power-domains = <&ps_dispext_cpu0>, <&ps_dptx_ext_phy>; -}; - &dptxphy { status = "okay"; }; @@ -66,15 +61,90 @@ apple,dptx-phy = <5>; }; -/* remove once m1n1 enables sio nodes after setup */ -&sio { +&display { + /* dispext0_dart needs to be first for m1n1's */ + iommus = <&dispext0_dart 0 0x0 0x0 0xf 0xffff0000>, + <&disp0_dart 0 0x0 0x0 0xf 0xffff0000>; +}; + +&disp0_dart { + status = "okay"; +}; + +&dcp { status = "okay"; + apple,connector-type = "USB-C"; + phys = <&atcphy0 PHY_TYPE_DP>, <&atcphy1 PHY_TYPE_DP>; + phy-names = "typec0", "typec1"; + mux-controls = <&atcphy0_xbar 0>, <&atcphy1_xbar 0>; + mux-control-names = "typec0", "typec1"; + apple,typec-mux-indices = <0 0>; + apple,typec-dptx-phys = <0 1>; + + typec-routes { + #address-cells = <1>; + #size-cells = <0>; + + route@0 { + reg = <0>; + mode-switch; + + port { + dcp_typec0: endpoint { + remote-endpoint = <&typec0_dcp>; + }; + }; + }; + + route@1 { + reg = <1>; + mode-switch; + + port { + dcp_typec1: endpoint { + remote-endpoint = <&typec1_dcp>; + }; + }; + }; + }; }; -&dpaudio1 { +&dcpext { + /delete-property/ mux-controls; + /delete-property/ mux-control-names; + /delete-property/ mux-index; +}; + +&dpaudio0 { status = "okay"; }; +&typec0 { + displayport = <&dcp>; + + ports { + port@3 { + reg = <3>; + typec0_dcp: endpoint { + remote-endpoint = <&dcp_typec0>; + }; + }; + }; +}; + +&typec1 { + displayport = <&dcp>; + + ports { + port@3 { + reg = <3>; + typec1_dcp: endpoint { + remote-endpoint = <&dcp_typec1>; + }; + }; + }; +}; + /* * Keep the power-domains used for the HDMI port on. */ diff --git a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi index 54e423ae5b5f5..cd18fbbb96845 100644 --- a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi @@ -14,11 +14,13 @@ atcphy0 = &atcphy0; atcphy1 = &atcphy1; dcp = &dcp; + dcpext = &dcpext; disp0 = &display; disp0_piodma = &disp0_piodma; disp0-piodma = &disp0_piodma; serial0 = &serial0; serial2 = &serial2; + sio = &sio; usb4-0-acio = &usb4_0_acio; usb4-0-nhi = &usb4_0_nhi; usb4-1-acio = &usb4_1_acio; @@ -74,6 +76,9 @@ compatible = "usb-c-connector"; power-role = "dual"; data-role = "dual"; +#ifndef T8112_TYPEC_DCP + displayport = <&dcpext>; +#endif ports { #address-cells = <1>; @@ -97,6 +102,15 @@ remote-endpoint = <&usb4_0_acio_tbt>; }; }; + +#ifndef T8112_TYPEC_DCP + port@3 { + reg = <3>; + typec0_dcp: endpoint { + remote-endpoint = <&dcpext_typec0>; + }; + }; +#endif }; }; }; @@ -113,6 +127,10 @@ power-role = "dual"; data-role = "dual"; +#ifndef T8112_TYPEC_DCP + displayport = <&dcpext>; +#endif + ports { #address-cells = <1>; #size-cells = <0>; @@ -135,10 +153,91 @@ remote-endpoint = <&usb4_1_acio_tbt>; }; }; + +#ifndef T8112_TYPEC_DCP + port@3 { + reg = <3>; + typec1_dcp: endpoint { + remote-endpoint = <&dcpext_typec1>; + }; + }; +#endif + }; + }; + }; +}; + +&display { + iommus = <&disp0_dart 0 0x0 0x0 0xf 0xffff0000>, + <&dispext0_dart 0 0x0 0x0 0xf 0xffff0000>; +}; + +&dispext0_dart { + status = "okay"; +}; + +&dcpext_dart { + status = "okay"; +}; + +&dcpext_mbox { + status = "okay"; +}; + +#ifndef T8112_TYPEC_DCP +&dcpext { + status = "okay"; + apple,connector-type = "USB-C"; + phys = <&atcphy0 PHY_TYPE_DP>, <&atcphy1 PHY_TYPE_DP>; + phy-names = "typec0", "typec1"; + mux-controls = <&atcphy0_xbar 0>, <&atcphy1_xbar 0>; + mux-control-names = "typec0", "typec1"; + apple,typec-mux-indices = <2 2>; + apple,typec-dptx-phys = <0 1>; + + typec-routes { + #address-cells = <1>; + #size-cells = <0>; + + route@0 { + reg = <0>; + mode-switch; + + port { + dcpext_typec0: endpoint { + remote-endpoint = <&typec0_dcp>; + }; + }; + }; + + route@1 { + reg = <1>; + mode-switch; + + port { + dcpext_typec1: endpoint { + remote-endpoint = <&typec1_dcp>; + }; }; }; }; }; +#endif + +/* remove once m1n1 enables sio nodes after setup */ +&sio { + status = "okay"; +}; + +&dpaudio1 { + status = "okay"; +}; + + + +&ps_atc0_common { + apple,always-on; /* Needs to stay on for DP-alt suspend/resume */ +}; /* USB controllers */ &dwc3_0 { From 84980c14391324ceb62a3a9427826ec6f0936ebd Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 25 Aug 2026 15:01:04 -0400 Subject: [PATCH 0316/2680] arm64: dts: apple: add non-Ultra T600x USB4 tunnel routing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — Asahi now wraps the per-die includes in &SOC_NODE; include t600x-pciec.dtsi inside an &SOC_NODE block, and leave out the ENABLE_DCPEXT_TYPEC blocks of the dropped dp-altmode HACKs] [ryan: adapted to 7.2.10 — the t600x display DARTs take five iommu cells in asahi-wip; use their aperture for the display iommus] Signed-off-by: DjDeveloperr --- arch/arm64/boot/dts/apple/t6001.dtsi | 7 + .../arm64/boot/dts/apple/t600x-j314-j316.dtsi | 153 ++++++++++ arch/arm64/boot/dts/apple/t600x-j375.dtsi | 190 ++++++++++++ arch/arm64/boot/dts/apple/t600x-pciec.dtsi | 274 ++++++++++++++++++ 4 files changed, 624 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t600x-pciec.dtsi diff --git a/arch/arm64/boot/dts/apple/t6001.dtsi b/arch/arm64/boot/dts/apple/t6001.dtsi index 46ce496825dd9..33d8a06d75122 100644 --- a/arch/arm64/boot/dts/apple/t6001.dtsi +++ b/arch/arm64/boot/dts/apple/t6001.dtsi @@ -20,6 +20,8 @@ # define GPU_REPEAT(x) #endif +#define APPLE_T600X_NON_ULTRA + #include "t600x-common.dtsi" / { @@ -45,6 +47,11 @@ #include "t600x-die0.dtsi" #include "t600x-dieX.dtsi" + +&SOC_NODE { +#include "t600x-pciec.dtsi" +}; + #include "t600x-nvme.dtsi" #include "t600x-gpio-pins.dtsi" #include "t600x-pmgr.dtsi" diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index 028f446900979..b95f5d9af85fe 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -22,6 +22,9 @@ bluetooth0 = &bluetooth0; dcp = &dcp; dcpext0 = &dcpext0; +#ifdef APPLE_T600X_NON_ULTRA + dcpext1 = &dcpext1; +#endif disp0 = &display; disp0_piodma = &disp0_piodma; disp0-piodma = &disp0_piodma; @@ -137,6 +140,115 @@ }; }; +#ifdef APPLE_T600X_NON_ULTRA +&display { + iommus = <&disp0_dart 0>, <&dispext0_dart 0>, <&dispext1_dart 0>; +}; + +&dispext1_dart { + status = "okay"; +}; + +&dcpext1_dart { + status = "okay"; +}; + +&dcpext1_mbox { + status = "okay"; +}; + +&dcpext1 { + status = "okay"; + apple,connector-type = "USB-C"; + phys = <&atcphy0 PHY_TYPE_DP>, <&atcphy1 PHY_TYPE_DP>, + <&atcphy2 PHY_TYPE_DP>; + phy-names = "typec0", "typec1", "typec2"; + mux-controls = <&atcphy0_xbar 0>, <&atcphy1_xbar 0>, + <&atcphy2_xbar 0>; + mux-control-names = "typec0", "typec1", "typec2"; + apple,typec-mux-indices = <2 2 2>; + apple,typec-dptx-phys = <0 1 2>; + + typec-routes { + #address-cells = <1>; + #size-cells = <0>; + + route@0 { + reg = <0>; + mode-switch; + + port { + dcpext1_typec0: endpoint { + remote-endpoint = <&typec0_dcp>; + }; + }; + }; + + route@1 { + reg = <1>; + mode-switch; + + port { + dcpext1_typec1: endpoint { + remote-endpoint = <&typec1_dcp>; + }; + }; + }; + + route@2 { + reg = <2>; + mode-switch; + + port { + dcpext1_typec2: endpoint { + remote-endpoint = <&typec2_dcp>; + }; + }; + }; + }; +}; + +&dpaudio2 { + status = "okay"; +}; + +&atcphy0_xbar { + status = "okay"; +}; + +&atcphy1_xbar { + status = "okay"; +}; + +&atcphy2_xbar { + status = "okay"; +}; + +&ps_atc0_common { + apple,always-on; +}; + +&ps_atc1_common { + apple,always-on; +}; + +&ps_atc2_common { + apple,always-on; +}; + +&usb4_0_acio { + apple,pcie-tunnel = <&usb4_0_pcie_tunnel>; +}; + +&usb4_1_acio { + apple,pcie-tunnel = <&usb4_1_pcie_tunnel>; +}; + +&usb4_2_acio { + apple,pcie-tunnel = <&usb4_2_pcie_tunnel>; +}; +#endif + /* remove once m1n1 enables sio nodes after setup */ &sio { status = "okay"; @@ -418,6 +530,47 @@ status = "okay"; }; +#ifdef APPLE_T600X_NON_ULTRA +&typec0 { + displayport = <&dcpext1>; + + ports { + port@3 { + reg = <3>; + typec0_dcp: endpoint { + remote-endpoint = <&dcpext1_typec0>; + }; + }; + }; +}; + +&typec1 { + displayport = <&dcpext1>; + + ports { + port@3 { + reg = <3>; + typec1_dcp: endpoint { + remote-endpoint = <&dcpext1_typec1>; + }; + }; + }; +}; + +&typec2 { + displayport = <&dcpext1>; + + ports { + port@3 { + reg = <3>; + typec2_dcp: endpoint { + remote-endpoint = <&dcpext1_typec2>; + }; + }; + }; +}; +#endif + /* USB controllers */ &dwc3_0 { ports { diff --git a/arch/arm64/boot/dts/apple/t600x-j375.dtsi b/arch/arm64/boot/dts/apple/t600x-j375.dtsi index 2cdd2e2fb5102..5f648f5d2745b 100644 --- a/arch/arm64/boot/dts/apple/t600x-j375.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j375.dtsi @@ -20,6 +20,9 @@ bluetooth0 = &bluetooth0; #ifndef NO_DCP dcp = &dcp; +#ifdef APPLE_T600X_NON_ULTRA + dcpext1 = &dcpext1; +#endif disp0 = &display; disp0_piodma = &disp0_piodma; disp0-piodma = &disp0_piodma; @@ -77,6 +80,139 @@ apple,connector-type = "HDMI-A"; }; +#ifdef APPLE_T600X_NON_ULTRA +&display { + iommus = <&disp0_dart 0 0x0 0x0 0x0 0xfc000000>, + <&dispext1_dart 0 0x0 0x0 0x0 0xfc000000>; +}; + +&dispext1_dart { + status = "okay"; +}; + +&dcpext1_dart { + status = "okay"; +}; + +&dcpext1_mbox { + status = "okay"; +}; + +&dcpext1 { + status = "okay"; + apple,connector-type = "USB-C"; + phys = <&atcphy0 PHY_TYPE_DP>, <&atcphy1 PHY_TYPE_DP>, + <&atcphy2 PHY_TYPE_DP>, <&atcphy3 PHY_TYPE_DP>; + phy-names = "typec0", "typec1", "typec2", "typec3"; + mux-controls = <&atcphy0_xbar 0>, <&atcphy1_xbar 0>, + <&atcphy2_xbar 0>, <&atcphy3_xbar 0>; + mux-control-names = "typec0", "typec1", "typec2", "typec3"; + apple,typec-mux-indices = <2 2 2 2>; + apple,typec-dptx-phys = <0 1 2 3>; + + typec-routes { + #address-cells = <1>; + #size-cells = <0>; + + route@0 { + reg = <0>; + mode-switch; + + port { + dcpext1_typec0: endpoint { + remote-endpoint = <&typec0_dcp>; + }; + }; + }; + + route@1 { + reg = <1>; + mode-switch; + + port { + dcpext1_typec1: endpoint { + remote-endpoint = <&typec1_dcp>; + }; + }; + }; + + route@2 { + reg = <2>; + mode-switch; + + port { + dcpext1_typec2: endpoint { + remote-endpoint = <&typec2_dcp>; + }; + }; + }; + + route@3 { + reg = <3>; + mode-switch; + + port { + dcpext1_typec3: endpoint { + remote-endpoint = <&typec3_dcp>; + }; + }; + }; + }; +}; + +&dpaudio2 { + status = "okay"; +}; + +&atcphy0_xbar { + status = "okay"; +}; + +&atcphy1_xbar { + status = "okay"; +}; + +&atcphy2_xbar { + status = "okay"; +}; + +&atcphy3_xbar { + status = "okay"; +}; + +&ps_atc0_common { + apple,always-on; +}; + +&ps_atc1_common { + apple,always-on; +}; + +&ps_atc2_common { + apple,always-on; +}; + +&ps_atc3_common { + apple,always-on; +}; + +&usb4_0_acio { + apple,pcie-tunnel = <&usb4_0_pcie_tunnel>; +}; + +&usb4_1_acio { + apple,pcie-tunnel = <&usb4_1_pcie_tunnel>; +}; + +&usb4_2_acio { + apple,pcie-tunnel = <&usb4_2_pcie_tunnel>; +}; + +&usb4_3_acio { + apple,pcie-tunnel = <&usb4_3_pcie_tunnel>; +}; +#endif + /* remove once m1n1 enables sio nodes after setup */ &sio { status = "okay"; @@ -213,6 +349,60 @@ }; }; +#ifdef APPLE_T600X_NON_ULTRA +&typec0 { + displayport = <&dcpext1>; + + ports { + port@3 { + reg = <3>; + typec0_dcp: endpoint { + remote-endpoint = <&dcpext1_typec0>; + }; + }; + }; +}; + +&typec1 { + displayport = <&dcpext1>; + + ports { + port@3 { + reg = <3>; + typec1_dcp: endpoint { + remote-endpoint = <&dcpext1_typec1>; + }; + }; + }; +}; + +&typec2 { + displayport = <&dcpext1>; + + ports { + port@3 { + reg = <3>; + typec2_dcp: endpoint { + remote-endpoint = <&dcpext1_typec2>; + }; + }; + }; +}; + +&typec3 { + displayport = <&dcpext1>; + + ports { + port@3 { + reg = <3>; + typec3_dcp: endpoint { + remote-endpoint = <&dcpext1_typec3>; + }; + }; + }; +}; +#endif + /* USB controllers */ &dwc3_0 { ports { diff --git a/arch/arm64/boot/dts/apple/t600x-pciec.dtsi b/arch/arm64/boot/dts/apple/t600x-pciec.dtsi new file mode 100644 index 0000000000000..f4cad7634f588 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t600x-pciec.dtsi @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * PCIe-C root complexes used by USB4 tunnels on T6000/T6001. + * + * Copyright The Asahi Linux Contributors + */ + + usb4_0_pcie_tunnel: usb4-pcie-tunnel-0 { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + apciec0_dart: iommu@721008000 { + compatible = "apple,t6000-dart"; + reg = <0x7 0x21008000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + }; + + apciec0: pcie@730000000 { + compatible = "apple,t6000-pciec"; + device_type = "pci"; + + reg = <0x7 0x30000000 0x0 0x10000000>, + <0x7 0x20000000 0x0 0x4000>, + <0x7 0x21000000 0x0 0x8000>, + <0x7 0x20004000 0x0 0x4000>, + <0x7 0x20200000 0x0 0x4000>, + <0x7 0x21010000 0x0 0x4000>, + <0x7 0x2100c000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "common", + "fabric", "debug", "intr2axi"; + + interrupt-parent = <&aic>; + interrupts = ; + + msi-controller; + msi-parent = <&apciec0>; + msi-ranges = <&aic AIC_IRQ 0 1614 IRQ_TYPE_EDGE_RISING 32>; + + iommu-map = <0x0 &apciec0_dart 0x0 0x40>; + iommu-map-mask = <0x3f>; + + bus-range = <0x0 0x7f>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0x8 0x0 + 0x8 0x0 0x2 0x0>, + <0x02000000 0x0 0x00100000 + 0xa 0x00100000 0x0 0x3ff00000>, + <0x42000000 0x0 0x40000000 + 0xa 0x40000000 0x0 0x40000000>; + + apciec0_port: pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + external-facing; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &apciec0_port 0 0 0 0>, + <0 0 0 2 &apciec0_port 0 0 0 1>, + <0 0 0 3 &apciec0_port 0 0 0 2>, + <0 0 0 4 &apciec0_port 0 0 0 3>; + }; + }; + }; + + usb4_1_pcie_tunnel: usb4-pcie-tunnel-1 { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + apciec1_dart: iommu@b21008000 { + compatible = "apple,t6000-dart"; + reg = <0xb 0x21008000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + }; + + apciec1: pcie@b30000000 { + compatible = "apple,t6000-pciec"; + device_type = "pci"; + + reg = <0xb 0x30000000 0x0 0x10000000>, + <0xb 0x20000000 0x0 0x4000>, + <0xb 0x21000000 0x0 0x8000>, + <0xb 0x20004000 0x0 0x4000>, + <0xb 0x20200000 0x0 0x4000>, + <0xb 0x21010000 0x0 0x4000>, + <0xb 0x2100c000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "common", + "fabric", "debug", "intr2axi"; + + interrupt-parent = <&aic>; + interrupts = ; + + msi-controller; + msi-parent = <&apciec1>; + msi-ranges = <&aic AIC_IRQ 0 1646 IRQ_TYPE_EDGE_RISING 32>; + + iommu-map = <0x0 &apciec1_dart 0x0 0x40>; + iommu-map-mask = <0x3f>; + + bus-range = <0x0 0x7f>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0x8 0x0 + 0xc 0x0 0x2 0x0>, + <0x02000000 0x0 0x00100000 + 0xe 0x00100000 0x0 0x3ff00000>, + <0x42000000 0x0 0x40000000 + 0xe 0x40000000 0x0 0x40000000>; + + apciec1_port: pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + external-facing; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &apciec1_port 0 0 0 0>, + <0 0 0 2 &apciec1_port 0 0 0 1>, + <0 0 0 3 &apciec1_port 0 0 0 2>, + <0 0 0 4 &apciec1_port 0 0 0 3>; + }; + }; + }; + + usb4_2_pcie_tunnel: usb4-pcie-tunnel-2 { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + apciec2_dart: iommu@f21008000 { + compatible = "apple,t6000-dart"; + reg = <0xf 0x21008000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + }; + + apciec2: pcie@f30000000 { + compatible = "apple,t6000-pciec"; + device_type = "pci"; + + reg = <0xf 0x30000000 0x0 0x10000000>, + <0xf 0x20000000 0x0 0x4000>, + <0xf 0x21000000 0x0 0x8000>, + <0xf 0x20004000 0x0 0x4000>, + <0xf 0x20200000 0x0 0x4000>, + <0xf 0x21010000 0x0 0x4000>, + <0xf 0x2100c000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "common", + "fabric", "debug", "intr2axi"; + + interrupt-parent = <&aic>; + interrupts = ; + + msi-controller; + msi-parent = <&apciec2>; + msi-ranges = <&aic AIC_IRQ 0 1678 IRQ_TYPE_EDGE_RISING 32>; + + iommu-map = <0x0 &apciec2_dart 0x0 0x40>; + iommu-map-mask = <0x3f>; + + bus-range = <0x0 0x7f>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0x8 0x0 + 0x10 0x0 0x2 0x0>, + <0x02000000 0x0 0x00100000 + 0x12 0x00100000 0x0 0x3ff00000>, + <0x42000000 0x0 0x40000000 + 0x12 0x40000000 0x0 0x40000000>; + + apciec2_port: pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + external-facing; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &apciec2_port 0 0 0 0>, + <0 0 0 2 &apciec2_port 0 0 0 1>, + <0 0 0 3 &apciec2_port 0 0 0 2>, + <0 0 0 4 &apciec2_port 0 0 0 3>; + }; + }; + }; + + usb4_3_pcie_tunnel: usb4-pcie-tunnel-3 { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + apciec3_dart: iommu@1321008000 { + compatible = "apple,t6000-dart"; + reg = <0x13 0x21008000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + }; + + apciec3: pcie@1330000000 { + compatible = "apple,t6000-pciec"; + device_type = "pci"; + + reg = <0x13 0x30000000 0x0 0x10000000>, + <0x13 0x20000000 0x0 0x4000>, + <0x13 0x21000000 0x0 0x8000>, + <0x13 0x20004000 0x0 0x4000>, + <0x13 0x20200000 0x0 0x4000>, + <0x13 0x21010000 0x0 0x4000>, + <0x13 0x2100c000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "common", + "fabric", "debug", "intr2axi"; + + interrupt-parent = <&aic>; + interrupts = ; + + msi-controller; + msi-parent = <&apciec3>; + msi-ranges = <&aic AIC_IRQ 0 1710 IRQ_TYPE_EDGE_RISING 32>; + + iommu-map = <0x0 &apciec3_dart 0x0 0x40>; + iommu-map-mask = <0x3f>; + + bus-range = <0x0 0x7f>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0x8 0x0 + 0x14 0x0 0x2 0x0>, + <0x02000000 0x0 0x00100000 + 0x16 0x00100000 0x0 0x3ff00000>, + <0x42000000 0x0 0x40000000 + 0x16 0x40000000 0x0 0x40000000>; + + apciec3_port: pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + external-facing; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &apciec3_port 0 0 0 0>, + <0 0 0 2 &apciec3_port 0 0 0 1>, + <0 0 0 3 &apciec3_port 0 0 0 2>, + <0 0 0 4 &apciec3_port 0 0 0 3>; + }; + }; + }; From c74ee7219c0e2eda7bf60fd3669444937c29dc46 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 25 Aug 2026 15:12:56 -0400 Subject: [PATCH 0317/2680] arm64: dts: apple: complete non-Ultra T602x USB4 and DCP routes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — t602x display DARTs take five iommu cells in asahi-wip; convert the new display iommus] [ryan: adapted to 7.2.10 — the &usb4_N_acio apple,pcie-tunnel references move to the PCIe-C wiring commit that defines usb4_N_pcie_tunnel, so this commit builds on its own] Signed-off-by: DjDeveloperr --- arch/arm64/boot/dts/apple/t6020-j474s.dts | 2 +- arch/arm64/boot/dts/apple/t6021-j475c.dts | 2 +- arch/arm64/boot/dts/apple/t6021.dtsi | 2 + arch/arm64/boot/dts/apple/t602x-dieX.dtsi | 70 ++++++ .../arm64/boot/dts/apple/t602x-j414-j416.dtsi | 226 +++++++++++++++++- .../arm64/boot/dts/apple/t602x-j474-j475.dtsi | 178 ++++++++++++++ 6 files changed, 473 insertions(+), 7 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6020-j474s.dts b/arch/arm64/boot/dts/apple/t6020-j474s.dts index 936bd4210a866..1084e0cb71904 100644 --- a/arch/arm64/boot/dts/apple/t6020-j474s.dts +++ b/arch/arm64/boot/dts/apple/t6020-j474s.dts @@ -85,7 +85,7 @@ status = "disabled"; }; &display { - iommus = <&dispext0_dart 0 0x100 0x0 0x10 0x0>; + iommus = <&dispext0_dart 0 0x100 0x0 0x10 0x0>, <&dispext1_dart 0 0x100 0x0 0x10 0x0>; }; &dispext0_dart { status = "okay"; diff --git a/arch/arm64/boot/dts/apple/t6021-j475c.dts b/arch/arm64/boot/dts/apple/t6021-j475c.dts index bff00df1b3404..89ba95bc8cf12 100644 --- a/arch/arm64/boot/dts/apple/t6021-j475c.dts +++ b/arch/arm64/boot/dts/apple/t6021-j475c.dts @@ -81,7 +81,7 @@ status = "disabled"; }; &display { - iommus = <&dispext0_dart 0 0x100 0x0 0x10 0x0>; + iommus = <&dispext0_dart 0 0x100 0x0 0x10 0x0>, <&dispext1_dart 0 0x100 0x0 0x10 0x0>; }; &dispext0_dart { status = "okay"; diff --git a/arch/arm64/boot/dts/apple/t6021.dtsi b/arch/arm64/boot/dts/apple/t6021.dtsi index daffb7170c6d0..a541239222475 100644 --- a/arch/arm64/boot/dts/apple/t6021.dtsi +++ b/arch/arm64/boot/dts/apple/t6021.dtsi @@ -23,6 +23,8 @@ # define GPU_DIE_REPEAT(x) #endif +#define APPLE_T602X_NON_ULTRA + #include "t602x-common.dtsi" / { diff --git a/arch/arm64/boot/dts/apple/t602x-dieX.dtsi b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi index 00f3d7800e14c..28437dd49ef98 100644 --- a/arch/arm64/boot/dts/apple/t602x-dieX.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi @@ -1023,6 +1023,76 @@ }; }; +#ifdef APPLE_T602X_NON_ULTRA + DIE_NODE(usb4_3_pcie_tunnel): usb4-pcie-tunnel-3 { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + DIE_NODE(apciec3_dart): iommu@1321008000 { + compatible = "apple,t8110-dart"; + reg = <0x13 0x21008000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + }; + + DIE_NODE(apciec3): pcie@1330000000 { + compatible = "apple,t6000-pciec"; + device_type = "pci"; + + reg = <0x13 0x30000000 0x0 0x10000000>, + <0x13 0x20000000 0x0 0x4000>, + <0x13 0x21000000 0x0 0x8000>, + <0x13 0x20004000 0x0 0x4000>, + <0x13 0x20200000 0x0 0x4000>, + <0x13 0x21010000 0x0 0x4000>, + <0x13 0x2100c000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "common", + "fabric", "debug", "intr2axi"; + + interrupt-parent = <&aic>; + interrupts = ; + + msi-controller; + msi-parent = <&DIE_NODE(apciec3)>; + msi-ranges = <&aic AIC_IRQ DIE_NO 1929 + IRQ_TYPE_EDGE_RISING 32>; + + iommu-map = <0x0 &DIE_NODE(apciec3_dart) 0x0 0x40>; + iommu-map-mask = <0x3f>; + + bus-range = <0x0 0x7f>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0x8 0x0 + 0x14 0x0 0x2 0x0>, + <0x02000000 0x0 0x00100000 + 0x16 0x00100000 0x0 0x3ff00000>, + <0x42000000 0x0 0x40000000 + 0x16 0x40000000 0x0 0x40000000>; + + DIE_NODE(apciec3_port): pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + external-facing; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &DIE_NODE(apciec3_port) 0 0 0 0>, + <0 0 0 2 &DIE_NODE(apciec3_port) 0 0 0 1>, + <0 0 0 3 &DIE_NODE(apciec3_port) 0 0 0 2>, + <0 0 0 4 &DIE_NODE(apciec3_port) 0 0 0 3>; + }; + }; + }; +#endif + DIE_NODE(dwc3_3): usb@1302280000 { compatible = "apple,t6020-dwc3", "apple,t8103-dwc3"; reg = <0x13 0x02280000 0x0 0xcd00>, <0x13 0x0228cd00 0x0 0x3200>; diff --git a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi index 428e8d009bacf..62e642a129024 100644 --- a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi @@ -19,22 +19,238 @@ / { aliases { + dcp = &dcp; + dcpext0 = &dcpext0; + dcpext1 = &dcpext1; + disp0 = &display; + disp0_piodma = &disp0_piodma; keyboard = &keyboard; + sio = &sio; }; }; &framebuffer0 { power-domains = <&ps_disp0_cpu0>, <&ps_dptx_phy_ps>; + panel = <&panel>; + post-init-providers = <&panel>; +}; + +&dcp { + panel: panel { + compatible = "apple,panel"; + apple,max-brightness = <500>; + }; +}; + +&disp0_dart { + status = "okay"; }; &display { - iommus = <&disp0_dart 0 0x100 0x0 0x10 0x0>, - <&dispext0_dart 0 0x100 0x0 0x10 0x0>; + iommus = <&disp0_dart 0 0x100 0x0 0x10 0x0>, <&dispext0_dart 0 0x100 0x0 0x10 0x0>, <&dispext1_dart 0 0x100 0x0 0x10 0x0>; }; -/* HACK: keep dptx_phy_ps power-domain always-on - * it is unclear how to sequence with dcp for the integrated display - */ +&dispext0_dart { + status = "okay"; +}; + +&dcpext0_dart { + status = "okay"; +}; + +&dcpext0_mbox { + status = "okay"; +}; + +&dcpext0 { + status = "okay"; + apple,connector-type = "HDMI-A"; + hdmi-hpd-gpios = <&pinctrl_aop 25 GPIO_ACTIVE_HIGH>; + hdmi-pwren-gpios = <&smc_gpio 23 GPIO_ACTIVE_HIGH>; + dp2hdmi-pwren-gpios = <&smc_gpio 6 GPIO_ACTIVE_HIGH>; + phys = <&atcphy3 PHY_TYPE_DP>; + phy-names = "dp-phy"; + mux-controls = <&atcphy3_xbar 0>; + mux-control-names = "dp-xbar"; + mux-index = <0>; + apple,dptx-phy = <3>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dcpext0_dpout: endpoint { + remote-endpoint = <&atcphy3_dp>; + }; + }; + }; +}; + +&dispext1_dart { + status = "okay"; +}; + +&dcpext1_dart { + status = "okay"; +}; + +&dcpext1_mbox { + status = "okay"; +}; + +&dcpext1 { + status = "okay"; + apple,connector-type = "USB-C"; + phys = <&atcphy0 PHY_TYPE_DP>, <&atcphy1 PHY_TYPE_DP>, + <&atcphy2 PHY_TYPE_DP>; + phy-names = "typec0", "typec1", "typec2"; + mux-controls = <&atcphy0_xbar 0>, <&atcphy1_xbar 0>, + <&atcphy2_xbar 0>; + mux-control-names = "typec0", "typec1", "typec2"; + apple,typec-mux-indices = <2 2 2>; + apple,typec-dptx-phys = <0 1 2>; + + typec-routes { + #address-cells = <1>; + #size-cells = <0>; + + route@0 { + reg = <0>; + mode-switch; + + port { + dcpext1_typec0: endpoint { + remote-endpoint = <&typec0_dcp>; + }; + }; + }; + + route@1 { + reg = <1>; + mode-switch; + + port { + dcpext1_typec1: endpoint { + remote-endpoint = <&typec1_dcp>; + }; + }; + }; + + route@2 { + reg = <2>; + mode-switch; + + port { + dcpext1_typec2: endpoint { + remote-endpoint = <&typec2_dcp>; + }; + }; + }; + }; +}; + +&typec0 { + displayport = <&dcpext1>; + + ports { + port@3 { + reg = <3>; + typec0_dcp: endpoint { + remote-endpoint = <&dcpext1_typec0>; + }; + }; + }; +}; + +&typec1 { + displayport = <&dcpext1>; + + ports { + port@3 { + reg = <3>; + typec1_dcp: endpoint { + remote-endpoint = <&dcpext1_typec1>; + }; + }; + }; +}; + +&typec2 { + displayport = <&dcpext1>; + + ports { + port@3 { + reg = <3>; + typec2_dcp: endpoint { + remote-endpoint = <&dcpext1_typec2>; + }; + }; + }; +}; + +&sio { + status = "okay"; +}; + +&dpaudio1 { + status = "okay"; +}; + +&dpaudio2 { + status = "okay"; +}; + +&atcphy0_xbar { + status = "okay"; +}; + +&atcphy1_xbar { + status = "okay"; +}; + +&atcphy2_xbar { + status = "okay"; +}; + +&atcphy3_xbar { + status = "okay"; +}; + +&atcphy3 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + reg = <2>; + atcphy3_dp: endpoint { + remote-endpoint = <&dcpext0_dpout>; + }; + }; + }; +}; + +&ps_atc0_common { + apple,always-on; +}; + +&ps_atc1_common { + apple,always-on; +}; + +&ps_atc2_common { + apple,always-on; +}; + +&ps_atc3_common { + apple,always-on; +}; + +/* Keep dptx_phy_ps on until DCP can sequence it during display bring-up. */ &ps_dptx_phy_ps { apple,always-on; }; diff --git a/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi b/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi index 6606e11e2149d..3a22771e3cd62 100644 --- a/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi @@ -21,6 +21,184 @@ power-domains = <&ps_dispext0_cpu0>, <&ps_dptx_phy_ps>; }; +#ifdef APPLE_T602X_NON_ULTRA +/ { + aliases { + dcpext1 = &dcpext1; + }; +}; + +&display { + iommus = <&dispext0_dart 0>, <&dispext1_dart 0>; +}; + +&dispext1_dart { + status = "okay"; +}; + +&dcpext1_dart { + status = "okay"; +}; + +&dcpext1_mbox { + status = "okay"; +}; + +&dcpext1 { + status = "okay"; + apple,connector-type = "USB-C"; + phys = <&atcphy0 PHY_TYPE_DP>, <&atcphy1 PHY_TYPE_DP>, + <&atcphy2 PHY_TYPE_DP>, <&atcphy3 PHY_TYPE_DP>; + phy-names = "typec0", "typec1", "typec2", "typec3"; + mux-controls = <&atcphy0_xbar 0>, <&atcphy1_xbar 0>, + <&atcphy2_xbar 0>, <&atcphy3_xbar 0>; + mux-control-names = "typec0", "typec1", "typec2", "typec3"; + apple,typec-mux-indices = <2 2 2 2>; + apple,typec-dptx-phys = <0 1 2 3>; + + typec-routes { + #address-cells = <1>; + #size-cells = <0>; + + route@0 { + reg = <0>; + mode-switch; + + port { + dcpext1_typec0: endpoint { + remote-endpoint = <&typec0_dcp>; + }; + }; + }; + + route@1 { + reg = <1>; + mode-switch; + + port { + dcpext1_typec1: endpoint { + remote-endpoint = <&typec1_dcp>; + }; + }; + }; + + route@2 { + reg = <2>; + mode-switch; + + port { + dcpext1_typec2: endpoint { + remote-endpoint = <&typec2_dcp>; + }; + }; + }; + + route@3 { + reg = <3>; + mode-switch; + + port { + dcpext1_typec3: endpoint { + remote-endpoint = <&typec3_dcp>; + }; + }; + }; + }; +}; + +&typec0 { + displayport = <&dcpext1>; + + ports { + port@3 { + reg = <3>; + typec0_dcp: endpoint { + remote-endpoint = <&dcpext1_typec0>; + }; + }; + }; +}; + +&typec1 { + displayport = <&dcpext1>; + + ports { + port@3 { + reg = <3>; + typec1_dcp: endpoint { + remote-endpoint = <&dcpext1_typec1>; + }; + }; + }; +}; + +&typec2 { + displayport = <&dcpext1>; + + ports { + port@3 { + reg = <3>; + typec2_dcp: endpoint { + remote-endpoint = <&dcpext1_typec2>; + }; + }; + }; +}; + +&typec3 { + displayport = <&dcpext1>; + + ports { + port@3 { + reg = <3>; + typec3_dcp: endpoint { + remote-endpoint = <&dcpext1_typec3>; + }; + }; + }; +}; + +&dpaudio2 { + status = "okay"; +}; + +&atcphy0_xbar { + status = "okay"; +}; + +&atcphy1_xbar { + status = "okay"; +}; + +&atcphy2_xbar { + status = "okay"; +}; + +&atcphy3_xbar { + status = "okay"; +}; + +&ps_atc0_common { + apple,always-on; +}; + +&ps_atc1_common { + apple,always-on; +}; + +&ps_atc2_common { + apple,always-on; +}; + +&ps_atc3_common { + apple,always-on; +}; + +&usb4_3_acio { + apple,pcie-tunnel = <&usb4_3_pcie_tunnel>; +}; +#endif + &hpm0 { interrupts = <44 IRQ_TYPE_LEVEL_LOW>; }; From f2a2cc3b2b5ab4b93b32263d6b272e311379d155 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 25 Aug 2026 14:16:41 -0400 Subject: [PATCH 0318/2680] arm64: dts: apple: add M2 Pro USB4 and display topology MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — squashed with 82b9fa7601e0 (avoid duplicate M2 display nodes), its immediate fix: asahi-wip already names these nodes mailbox@, so the duplicate labels broke the build in between] [ryan: adapted to 7.2.10 — keep Asahi's clock-dispextN_die1 node names and comment] Signed-off-by: DjDeveloperr --- arch/arm64/boot/dts/apple/t602x-dieX.dtsi | 207 ++++++++++++++++++++++ 1 file changed, 207 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t602x-dieX.dtsi b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi index 28437dd49ef98..f904c29520db1 100644 --- a/arch/arm64/boot/dts/apple/t602x-dieX.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi @@ -500,6 +500,7 @@ <&DIE_NODE(ps_atc0_cio_usb)>; thunderbolt-switch; + apple,pcie-tunnel = <&DIE_NODE(usb4_0_pcie_tunnel)>; pinctrl-0 = <&DIE_NODE(acio0_pins)>; pinctrl-names = "default"; @@ -564,6 +565,74 @@ }; }; + DIE_NODE(usb4_0_pcie_tunnel): usb4-pcie-tunnel-0 { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + DIE_NODE(apciec0_dart): iommu@721008000 { + compatible = "apple,t8110-dart"; + reg = <0x7 0x21008000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + }; + + DIE_NODE(apciec0): pcie@730000000 { + compatible = "apple,t6000-pciec"; + device_type = "pci"; + + reg = <0x7 0x30000000 0x0 0x10000000>, + <0x7 0x20000000 0x0 0x4000>, + <0x7 0x21000000 0x0 0x8000>, + <0x7 0x20004000 0x0 0x4000>, + <0x7 0x20200000 0x0 0x4000>, + <0x7 0x21010000 0x0 0x4000>, + <0x7 0x2100c000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "common", + "fabric", "debug", "intr2axi"; + + interrupt-parent = <&aic>; + interrupts = ; + + msi-controller; + msi-parent = <&DIE_NODE(apciec0)>; + msi-ranges = <&aic AIC_IRQ DIE_NO 1833 + IRQ_TYPE_EDGE_RISING 32>; + + iommu-map = <0x0 &DIE_NODE(apciec0_dart) 0x0 0x40>; + iommu-map-mask = <0x3f>; + + bus-range = <0x0 0x7f>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0x8 0x0 + 0x8 0x0 0x2 0x0>, + <0x02000000 0x0 0x00100000 + 0xa 0x00100000 0x0 0x3ff00000>, + <0x42000000 0x0 0x40000000 + 0xa 0x40000000 0x0 0x40000000>; + + DIE_NODE(apciec0_port): pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + external-facing; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &DIE_NODE(apciec0_port) 0 0 0 0>, + <0 0 0 2 &DIE_NODE(apciec0_port) 0 0 0 1>, + <0 0 0 3 &DIE_NODE(apciec0_port) 0 0 0 2>, + <0 0 0 4 &DIE_NODE(apciec0_port) 0 0 0 3>; + }; + }; + }; + DIE_NODE(dwc3_0): usb@702280000 { compatible = "apple,t6020-dwc3", "apple,t8103-dwc3"; reg = <0x7 0x02280000 0x0 0xcd00>, <0x7 0x0228cd00 0x0 0x3200>; @@ -653,6 +722,7 @@ <&DIE_NODE(ps_atc1_cio_usb)>; thunderbolt-switch; + apple,pcie-tunnel = <&DIE_NODE(usb4_1_pcie_tunnel)>; pinctrl-0 = <&DIE_NODE(acio1_pins)>; pinctrl-names = "default"; @@ -717,6 +787,74 @@ }; }; + DIE_NODE(usb4_1_pcie_tunnel): usb4-pcie-tunnel-1 { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + DIE_NODE(apciec1_dart): iommu@b21008000 { + compatible = "apple,t8110-dart"; + reg = <0xb 0x21008000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + }; + + DIE_NODE(apciec1): pcie@b30000000 { + compatible = "apple,t6000-pciec"; + device_type = "pci"; + + reg = <0xb 0x30000000 0x0 0x10000000>, + <0xb 0x20000000 0x0 0x4000>, + <0xb 0x21000000 0x0 0x8000>, + <0xb 0x20004000 0x0 0x4000>, + <0xb 0x20200000 0x0 0x4000>, + <0xb 0x21010000 0x0 0x4000>, + <0xb 0x2100c000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "common", + "fabric", "debug", "intr2axi"; + + interrupt-parent = <&aic>; + interrupts = ; + + msi-controller; + msi-parent = <&DIE_NODE(apciec1)>; + msi-ranges = <&aic AIC_IRQ DIE_NO 1865 + IRQ_TYPE_EDGE_RISING 32>; + + iommu-map = <0x0 &DIE_NODE(apciec1_dart) 0x0 0x40>; + iommu-map-mask = <0x3f>; + + bus-range = <0x0 0x7f>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0x8 0x0 + 0xc 0x0 0x2 0x0>, + <0x02000000 0x0 0x00100000 + 0xe 0x00100000 0x0 0x3ff00000>, + <0x42000000 0x0 0x40000000 + 0xe 0x40000000 0x0 0x40000000>; + + DIE_NODE(apciec1_port): pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + external-facing; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &DIE_NODE(apciec1_port) 0 0 0 0>, + <0 0 0 2 &DIE_NODE(apciec1_port) 0 0 0 1>, + <0 0 0 3 &DIE_NODE(apciec1_port) 0 0 0 2>, + <0 0 0 4 &DIE_NODE(apciec1_port) 0 0 0 3>; + }; + }; + }; + DIE_NODE(dwc3_1): usb@b02280000 { compatible = "apple,t6020-dwc3", "apple,t8103-dwc3"; reg = <0xb 0x02280000 0x0 0xcd00>, <0xb 0x0228cd00 0x0 0x3200>; @@ -806,6 +944,7 @@ <&DIE_NODE(ps_atc2_cio_usb)>; thunderbolt-switch; + apple,pcie-tunnel = <&DIE_NODE(usb4_2_pcie_tunnel)>; pinctrl-0 = <&DIE_NODE(acio2_pins)>; pinctrl-names = "default"; @@ -870,6 +1009,74 @@ }; }; + DIE_NODE(usb4_2_pcie_tunnel): usb4-pcie-tunnel-2 { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + DIE_NODE(apciec2_dart): iommu@f21008000 { + compatible = "apple,t8110-dart"; + reg = <0xf 0x21008000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + }; + + DIE_NODE(apciec2): pcie@f30000000 { + compatible = "apple,t6000-pciec"; + device_type = "pci"; + + reg = <0xf 0x30000000 0x0 0x10000000>, + <0xf 0x20000000 0x0 0x4000>, + <0xf 0x21000000 0x0 0x8000>, + <0xf 0x20004000 0x0 0x4000>, + <0xf 0x20200000 0x0 0x4000>, + <0xf 0x21010000 0x0 0x4000>, + <0xf 0x2100c000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "common", + "fabric", "debug", "intr2axi"; + + interrupt-parent = <&aic>; + interrupts = ; + + msi-controller; + msi-parent = <&DIE_NODE(apciec2)>; + msi-ranges = <&aic AIC_IRQ DIE_NO 1897 + IRQ_TYPE_EDGE_RISING 32>; + + iommu-map = <0x0 &DIE_NODE(apciec2_dart) 0x0 0x40>; + iommu-map-mask = <0x3f>; + + bus-range = <0x0 0x7f>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0x8 0x0 + 0x10 0x0 0x2 0x0>, + <0x02000000 0x0 0x00100000 + 0x12 0x00100000 0x0 0x3ff00000>, + <0x42000000 0x0 0x40000000 + 0x12 0x40000000 0x0 0x40000000>; + + DIE_NODE(apciec2_port): pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + external-facing; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &DIE_NODE(apciec2_port) 0 0 0 0>, + <0 0 0 2 &DIE_NODE(apciec2_port) 0 0 0 1>, + <0 0 0 3 &DIE_NODE(apciec2_port) 0 0 0 2>, + <0 0 0 4 &DIE_NODE(apciec2_port) 0 0 0 3>; + }; + }; + }; + DIE_NODE(dwc3_2): usb@f02280000 { compatible = "apple,t6020-dwc3", "apple,t8103-dwc3"; reg = <0xf 0x02280000 0x0 0xcd00>, <0xf 0x0228cd00 0x0 0x3200>; From 7b24e68b6556fe9b826db85feb47cb65fb6c6285 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 25 Aug 2026 17:46:43 -0400 Subject: [PATCH 0319/2680] arm64: dts: apple: add complete USB4 handoff aliases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — asahi-wip now wires USB4 on the Ultra (die0 and die1); keep its usb4 aliases unconditional and gate only the PCIe adapter/tunnel aliases this adds to the non-Ultra SoCs that have PCIe-C] Signed-off-by: DjDeveloperr --- arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi | 7 +++++++ arch/arm64/boot/dts/apple/t600x-j375.dtsi | 10 ++++++++++ arch/arm64/boot/dts/apple/t8103-jxxx.dtsi | 4 ++++ arch/arm64/boot/dts/apple/t8112-jxxx.dtsi | 4 ++++ 4 files changed, 25 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index b95f5d9af85fe..209d62ae6eae4 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -32,10 +32,17 @@ sio = &sio; usb4-0-acio = &usb4_0_acio; usb4-0-nhi = &usb4_0_nhi; + usb4-0-pcie-adapter = &usb4_0_acio; + usb4-0-pcie = &apciec0; usb4-1-acio = &usb4_1_acio; usb4-1-nhi = &usb4_1_nhi; + usb4-1-pcie-adapter = &usb4_1_acio; + usb4-1-pcie = &apciec1; usb4-2-acio = &usb4_2_acio; usb4-2-nhi = &usb4_2_nhi; + usb4-2-pcie-adapter = &usb4_2_acio; + usb4-2-pcie = &apciec2; + usb4-3-pcie = &apciec3; wifi0 = &wifi0; usb4_0_rc = &usb4_0_acio; diff --git a/arch/arm64/boot/dts/apple/t600x-j375.dtsi b/arch/arm64/boot/dts/apple/t600x-j375.dtsi index 5f648f5d2745b..a234a0e7b031a 100644 --- a/arch/arm64/boot/dts/apple/t600x-j375.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j375.dtsi @@ -38,6 +38,16 @@ usb4-2-nhi = &usb4_2_nhi; usb4-3-acio = &usb4_3_acio; usb4-3-nhi = &usb4_3_nhi; +#if defined(APPLE_T600X_NON_ULTRA) || defined(APPLE_T602X_NON_ULTRA) + usb4-0-pcie-adapter = &usb4_0_acio; + usb4-0-pcie = &apciec0; + usb4-1-pcie-adapter = &usb4_1_acio; + usb4-1-pcie = &apciec1; + usb4-2-pcie-adapter = &usb4_2_acio; + usb4-2-pcie = &apciec2; + usb4-3-pcie-adapter = &usb4_3_acio; + usb4-3-pcie = &apciec3; +#endif wifi0 = &wifi0; usb4_0_rc = &usb4_0_acio; diff --git a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi index 9ba9b11ae78a1..00b566ed848e4 100644 --- a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi @@ -24,6 +24,8 @@ atcphy1 = &atcphy1; usb4-0-acio = &usb4_0_acio; usb4-0-nhi = &usb4_0_nhi; + usb4-0-pcie-adapter = &usb4_0_acio; + usb4-0-pcie = &apciec0; usb4-1-acio = &usb4_1_acio; usb4-1-nhi = &usb4_1_nhi; @@ -31,6 +33,8 @@ usb4_0_nhi = &usb4_0_nhi; usb4_1_rc = &usb4_1_acio; usb4_1_nhi = &usb4_1_nhi; + usb4-1-pcie-adapter = &usb4_1_acio; + usb4-1-pcie = &apciec1; }; chosen { diff --git a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi index cd18fbbb96845..0346407ed582a 100644 --- a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi @@ -23,6 +23,8 @@ sio = &sio; usb4-0-acio = &usb4_0_acio; usb4-0-nhi = &usb4_0_nhi; + usb4-0-pcie-adapter = &usb4_0_acio; + usb4-0-pcie = &apciec0; usb4-1-acio = &usb4_1_acio; usb4-1-nhi = &usb4_1_nhi; @@ -30,6 +32,8 @@ usb4_0_nhi = &usb4_0_nhi; usb4_1_rc = &usb4_1_acio; usb4_1_nhi = &usb4_1_nhi; + usb4-1-pcie-adapter = &usb4_1_acio; + usb4-1-pcie = &apciec1; }; chosen { From 9e3f666a266b0294f7bf55197c538d37270dda72 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 00:03:23 -0400 Subject: [PATCH 0320/2680] arm64: dts: apple: wire PCIe-C on non-Ultra M1 and M2 SoCs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — also carries the T602x &usb4_N_acio apple,pcie-tunnel references from fc1b73b30e5a, which needed the tunnel nodes defined here] Signed-off-by: DjDeveloperr --- arch/arm64/boot/dts/apple/t600x-dieX.dtsi | 9 ++-- arch/arm64/boot/dts/apple/t600x-pciec.dtsi | 28 +++++----- arch/arm64/boot/dts/apple/t600x-pmgr.dtsi | 12 +++++ arch/arm64/boot/dts/apple/t602x-dieX.dtsi | 53 +++++++++++-------- .../arm64/boot/dts/apple/t602x-j414-j416.dtsi | 12 +++++ .../arm64/boot/dts/apple/t602x-j474-j475.dtsi | 12 +++++ arch/arm64/boot/dts/apple/t602x-pmgr.dtsi | 12 +++++ arch/arm64/boot/dts/apple/t8103-pmgr.dtsi | 2 + arch/arm64/boot/dts/apple/t8103.dtsi | 16 +++--- arch/arm64/boot/dts/apple/t8112-pmgr.dtsi | 2 + arch/arm64/boot/dts/apple/t8112.dtsi | 16 +++--- 11 files changed, 124 insertions(+), 50 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t600x-dieX.dtsi b/arch/arm64/boot/dts/apple/t600x-dieX.dtsi index 9f9ca9d46344e..0b0da1a4b066b 100644 --- a/arch/arm64/boot/dts/apple/t600x-dieX.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-dieX.dtsi @@ -488,7 +488,8 @@ power-domains = <&DIE_NODE(ps_atc0_cio)>, <&DIE_NODE(ps_atc0_cio_pcie)>, - <&DIE_NODE(ps_atc0_cio_usb)>; + <&DIE_NODE(ps_atc0_cio_usb)>, + <&DIE_NODE(ps_atc0_pcie)>; thunderbolt-switch; @@ -641,7 +642,8 @@ power-domains = <&DIE_NODE(ps_atc1_cio)>, <&DIE_NODE(ps_atc1_cio_pcie)>, - <&DIE_NODE(ps_atc1_cio_usb)>; + <&DIE_NODE(ps_atc1_cio_usb)>, + <&DIE_NODE(ps_atc1_pcie)>; thunderbolt-switch; @@ -794,7 +796,8 @@ power-domains = <&DIE_NODE(ps_atc2_cio)>, <&DIE_NODE(ps_atc2_cio_pcie)>, - <&DIE_NODE(ps_atc2_cio_usb)>; + <&DIE_NODE(ps_atc2_cio_usb)>, + <&DIE_NODE(ps_atc2_pcie)>; thunderbolt-switch; diff --git a/arch/arm64/boot/dts/apple/t600x-pciec.dtsi b/arch/arm64/boot/dts/apple/t600x-pciec.dtsi index f4cad7634f588..86cc1ac550e6b 100644 --- a/arch/arm64/boot/dts/apple/t600x-pciec.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-pciec.dtsi @@ -11,7 +11,7 @@ ranges; apciec0_dart: iommu@721008000 { - compatible = "apple,t6000-dart"; + compatible = "apple,t8110-dart"; reg = <0x7 0x21008000 0x0 0x4000>; interrupt-parent = <&aic>; interrupts = ; @@ -29,11 +29,12 @@ <0x7 0x20200000 0x0 0x4000>, <0x7 0x21010000 0x0 0x4000>, <0x7 0x2100c000 0x0 0x4000>; - reg-names = "config", "rc", "port0", "common", - "fabric", "debug", "intr2axi"; + reg-names = "config", "rc", "port0", "fabric", + "debug", "ltssm", "intr2axi"; interrupt-parent = <&aic>; interrupts = ; + power-domains = <&ps_atc0_pcie>; msi-controller; msi-parent = <&apciec0>; @@ -78,7 +79,7 @@ ranges; apciec1_dart: iommu@b21008000 { - compatible = "apple,t6000-dart"; + compatible = "apple,t8110-dart"; reg = <0xb 0x21008000 0x0 0x4000>; interrupt-parent = <&aic>; interrupts = ; @@ -96,11 +97,12 @@ <0xb 0x20200000 0x0 0x4000>, <0xb 0x21010000 0x0 0x4000>, <0xb 0x2100c000 0x0 0x4000>; - reg-names = "config", "rc", "port0", "common", - "fabric", "debug", "intr2axi"; + reg-names = "config", "rc", "port0", "fabric", + "debug", "ltssm", "intr2axi"; interrupt-parent = <&aic>; interrupts = ; + power-domains = <&ps_atc1_pcie>; msi-controller; msi-parent = <&apciec1>; @@ -145,7 +147,7 @@ ranges; apciec2_dart: iommu@f21008000 { - compatible = "apple,t6000-dart"; + compatible = "apple,t8110-dart"; reg = <0xf 0x21008000 0x0 0x4000>; interrupt-parent = <&aic>; interrupts = ; @@ -163,11 +165,12 @@ <0xf 0x20200000 0x0 0x4000>, <0xf 0x21010000 0x0 0x4000>, <0xf 0x2100c000 0x0 0x4000>; - reg-names = "config", "rc", "port0", "common", - "fabric", "debug", "intr2axi"; + reg-names = "config", "rc", "port0", "fabric", + "debug", "ltssm", "intr2axi"; interrupt-parent = <&aic>; interrupts = ; + power-domains = <&ps_atc2_pcie>; msi-controller; msi-parent = <&apciec2>; @@ -212,7 +215,7 @@ ranges; apciec3_dart: iommu@1321008000 { - compatible = "apple,t6000-dart"; + compatible = "apple,t8110-dart"; reg = <0x13 0x21008000 0x0 0x4000>; interrupt-parent = <&aic>; interrupts = ; @@ -230,11 +233,12 @@ <0x13 0x20200000 0x0 0x4000>, <0x13 0x21010000 0x0 0x4000>, <0x13 0x2100c000 0x0 0x4000>; - reg-names = "config", "rc", "port0", "common", - "fabric", "debug", "intr2axi"; + reg-names = "config", "rc", "port0", "fabric", + "debug", "ltssm", "intr2axi"; interrupt-parent = <&aic>; interrupts = ; + power-domains = <&ps_atc3_pcie>; msi-controller; msi-parent = <&apciec3>; diff --git a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi index 833a59ecf3792..8b2c2cacce8c1 100644 --- a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi @@ -509,6 +509,9 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(atc0_pcie); +#ifdef APPLE_T600X_NON_ULTRA + apple,always-on; +#endif power-domains = <&DIE_NODE(ps_atc0_common)>; }; @@ -527,6 +530,9 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(atc1_pcie); +#ifdef APPLE_T600X_NON_ULTRA + apple,always-on; +#endif power-domains = <&DIE_NODE(ps_atc1_common)>; }; @@ -545,6 +551,9 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(atc2_pcie); +#ifdef APPLE_T600X_NON_ULTRA + apple,always-on; +#endif power-domains = <&DIE_NODE(ps_atc2_common)>; }; @@ -563,6 +572,9 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(atc3_pcie); +#ifdef APPLE_T600X_NON_ULTRA + apple,always-on; +#endif power-domains = <&DIE_NODE(ps_atc3_common)>; }; diff --git a/arch/arm64/boot/dts/apple/t602x-dieX.dtsi b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi index f904c29520db1..b171c0ad6d65c 100644 --- a/arch/arm64/boot/dts/apple/t602x-dieX.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi @@ -497,7 +497,8 @@ power-domains = <&DIE_NODE(ps_atc0_cio)>, <&DIE_NODE(ps_atc0_cio_pcie)>, - <&DIE_NODE(ps_atc0_cio_usb)>; + <&DIE_NODE(ps_atc0_cio_usb)>, + <&DIE_NODE(ps_atc0_pcie)>; thunderbolt-switch; apple,pcie-tunnel = <&DIE_NODE(usb4_0_pcie_tunnel)>; @@ -576,6 +577,8 @@ interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; + apple,dma-range = <0x0 0x80000000 0x0 0x80000000>; + apple,dma-offset = <0x100 0x0>; }; DIE_NODE(apciec0): pcie@730000000 { @@ -586,14 +589,13 @@ <0x7 0x20000000 0x0 0x4000>, <0x7 0x21000000 0x0 0x8000>, <0x7 0x20004000 0x0 0x4000>, - <0x7 0x20200000 0x0 0x4000>, - <0x7 0x21010000 0x0 0x4000>, - <0x7 0x2100c000 0x0 0x4000>; - reg-names = "config", "rc", "port0", "common", - "fabric", "debug", "intr2axi"; + <0x7 0x20200000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "fabric", + "debug"; interrupt-parent = <&aic>; interrupts = ; + power-domains = <&DIE_NODE(ps_atc0_pcie)>; msi-controller; msi-parent = <&DIE_NODE(apciec0)>; @@ -719,7 +721,8 @@ power-domains = <&DIE_NODE(ps_atc1_cio)>, <&DIE_NODE(ps_atc1_cio_pcie)>, - <&DIE_NODE(ps_atc1_cio_usb)>; + <&DIE_NODE(ps_atc1_cio_usb)>, + <&DIE_NODE(ps_atc1_pcie)>; thunderbolt-switch; apple,pcie-tunnel = <&DIE_NODE(usb4_1_pcie_tunnel)>; @@ -798,6 +801,8 @@ interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; + apple,dma-range = <0x0 0x80000000 0x0 0x80000000>; + apple,dma-offset = <0x100 0x0>; }; DIE_NODE(apciec1): pcie@b30000000 { @@ -808,14 +813,13 @@ <0xb 0x20000000 0x0 0x4000>, <0xb 0x21000000 0x0 0x8000>, <0xb 0x20004000 0x0 0x4000>, - <0xb 0x20200000 0x0 0x4000>, - <0xb 0x21010000 0x0 0x4000>, - <0xb 0x2100c000 0x0 0x4000>; - reg-names = "config", "rc", "port0", "common", - "fabric", "debug", "intr2axi"; + <0xb 0x20200000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "fabric", + "debug"; interrupt-parent = <&aic>; interrupts = ; + power-domains = <&DIE_NODE(ps_atc1_pcie)>; msi-controller; msi-parent = <&DIE_NODE(apciec1)>; @@ -941,7 +945,8 @@ power-domains = <&DIE_NODE(ps_atc2_cio)>, <&DIE_NODE(ps_atc2_cio_pcie)>, - <&DIE_NODE(ps_atc2_cio_usb)>; + <&DIE_NODE(ps_atc2_cio_usb)>, + <&DIE_NODE(ps_atc2_pcie)>; thunderbolt-switch; apple,pcie-tunnel = <&DIE_NODE(usb4_2_pcie_tunnel)>; @@ -1020,6 +1025,8 @@ interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; + apple,dma-range = <0x0 0x80000000 0x0 0x80000000>; + apple,dma-offset = <0x100 0x0>; }; DIE_NODE(apciec2): pcie@f30000000 { @@ -1030,14 +1037,13 @@ <0xf 0x20000000 0x0 0x4000>, <0xf 0x21000000 0x0 0x8000>, <0xf 0x20004000 0x0 0x4000>, - <0xf 0x20200000 0x0 0x4000>, - <0xf 0x21010000 0x0 0x4000>, - <0xf 0x2100c000 0x0 0x4000>; - reg-names = "config", "rc", "port0", "common", - "fabric", "debug", "intr2axi"; + <0xf 0x20200000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "fabric", + "debug"; interrupt-parent = <&aic>; interrupts = ; + power-domains = <&DIE_NODE(ps_atc2_pcie)>; msi-controller; msi-parent = <&DIE_NODE(apciec2)>; @@ -1242,6 +1248,8 @@ interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; + apple,dma-range = <0x0 0x80000000 0x0 0x80000000>; + apple,dma-offset = <0x100 0x0>; }; DIE_NODE(apciec3): pcie@1330000000 { @@ -1252,14 +1260,13 @@ <0x13 0x20000000 0x0 0x4000>, <0x13 0x21000000 0x0 0x8000>, <0x13 0x20004000 0x0 0x4000>, - <0x13 0x20200000 0x0 0x4000>, - <0x13 0x21010000 0x0 0x4000>, - <0x13 0x2100c000 0x0 0x4000>; - reg-names = "config", "rc", "port0", "common", - "fabric", "debug", "intr2axi"; + <0x13 0x20200000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "fabric", + "debug"; interrupt-parent = <&aic>; interrupts = ; + power-domains = <&DIE_NODE(ps_atc3_pcie)>; msi-controller; msi-parent = <&DIE_NODE(apciec3)>; diff --git a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi index 62e642a129024..0f1f984fca875 100644 --- a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi @@ -250,6 +250,18 @@ apple,always-on; }; +&usb4_0_acio { + apple,pcie-tunnel = <&usb4_0_pcie_tunnel>; +}; + +&usb4_1_acio { + apple,pcie-tunnel = <&usb4_1_pcie_tunnel>; +}; + +&usb4_2_acio { + apple,pcie-tunnel = <&usb4_2_pcie_tunnel>; +}; + /* Keep dptx_phy_ps on until DCP can sequence it during display bring-up. */ &ps_dptx_phy_ps { apple,always-on; diff --git a/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi b/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi index 3a22771e3cd62..0da6de6e960e2 100644 --- a/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi @@ -194,6 +194,18 @@ apple,always-on; }; +&usb4_0_acio { + apple,pcie-tunnel = <&usb4_0_pcie_tunnel>; +}; + +&usb4_1_acio { + apple,pcie-tunnel = <&usb4_1_pcie_tunnel>; +}; + +&usb4_2_acio { + apple,pcie-tunnel = <&usb4_2_pcie_tunnel>; +}; + &usb4_3_acio { apple,pcie-tunnel = <&usb4_3_pcie_tunnel>; }; diff --git a/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi b/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi index 4ead781fea689..fffd0ad79b9bb 100644 --- a/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi @@ -362,6 +362,9 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(atc0_pcie); +#ifdef APPLE_T602X_NON_ULTRA + apple,always-on; +#endif power-domains = <&DIE_NODE(ps_atc0_common)>; }; @@ -380,6 +383,9 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(atc1_pcie); +#ifdef APPLE_T602X_NON_ULTRA + apple,always-on; +#endif power-domains = <&DIE_NODE(ps_atc1_common)>; }; @@ -398,6 +404,9 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(atc2_pcie); +#ifdef APPLE_T602X_NON_ULTRA + apple,always-on; +#endif power-domains = <&DIE_NODE(ps_atc2_common)>; }; @@ -416,6 +425,9 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(atc3_pcie); +#ifdef APPLE_T602X_NON_ULTRA + apple,always-on; +#endif power-domains = <&DIE_NODE(ps_atc3_common)>; }; diff --git a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi index 5d3846d44e357..4a399bc8f0276 100644 --- a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi @@ -864,6 +864,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "atc0_pcie"; + apple,always-on; power-domains = <&ps_atc0_common>; }; @@ -908,6 +909,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "atc1_pcie"; + apple,always-on; power-domains = <&ps_atc1_common>; }; diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index d99cb8db4c6d1..daa23f4ab489e 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -1580,7 +1580,8 @@ power-domains = <&ps_atc0_cio>, <&ps_atc0_cio_pcie>, - <&ps_atc0_cio_usb>; + <&ps_atc0_cio_usb>, + <&ps_atc0_pcie>; thunderbolt-switch; apple,pcie-tunnel = <&usb4_0_pcie_tunnel>; @@ -1670,11 +1671,12 @@ <0x3 0xa0200000 0x0 0x4000>, <0x3 0xa1010000 0x0 0x4000>, <0x3 0xa100c000 0x0 0x4000>; - reg-names = "config", "rc", "port0", "common", - "fabric", "debug", "intr2axi"; + reg-names = "config", "rc", "port0", "fabric", + "debug", "ltssm", "intr2axi"; interrupt-parent = <&aic>; interrupts = ; + power-domains = <&ps_atc0_pcie>; msi-controller; msi-parent = <&apciec0>; @@ -1797,7 +1799,8 @@ power-domains = <&ps_atc1_cio>, <&ps_atc1_cio_pcie>, - <&ps_atc1_cio_usb>; + <&ps_atc1_cio_usb>, + <&ps_atc1_pcie>; thunderbolt-switch; apple,pcie-tunnel = <&usb4_1_pcie_tunnel>; @@ -1887,11 +1890,12 @@ <0x5 0x20200000 0x0 0x4000>, <0x5 0x21010000 0x0 0x4000>, <0x5 0x2100c000 0x0 0x4000>; - reg-names = "config", "rc", "port0", "common", - "fabric", "debug", "intr2axi"; + reg-names = "config", "rc", "port0", "fabric", + "debug", "ltssm", "intr2axi"; interrupt-parent = <&aic>; interrupts = ; + power-domains = <&ps_atc1_pcie>; msi-controller; msi-parent = <&apciec1>; diff --git a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi index ab8ec9bd4e440..128a560dbac73 100644 --- a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi @@ -866,6 +866,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "atc0_pcie"; + apple,always-on; power-domains = <&ps_atc0_common>; }; @@ -911,6 +912,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "atc1_pcie"; + apple,always-on; power-domains = <&ps_atc1_common>; }; diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 18f75c32e266d..cdc02310a9334 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -1845,7 +1845,8 @@ power-domains = <&ps_atc0_cio>, <&ps_atc0_cio_pcie>, - <&ps_atc0_cio_usb>; + <&ps_atc0_cio_usb>, + <&ps_atc0_pcie>; thunderbolt-switch; apple,pcie-tunnel = <&usb4_0_pcie_tunnel>; @@ -1936,11 +1937,12 @@ <0x3 0xa0200000 0x0 0x4000>, <0x3 0xa1010000 0x0 0x4000>, <0x3 0xa100c000 0x0 0x4000>; - reg-names = "config", "rc", "port0", "common", - "fabric", "debug", "intr2axi"; + reg-names = "config", "rc", "port0", "fabric", + "debug", "ltssm", "intr2axi"; interrupt-parent = <&aic>; interrupts = ; + power-domains = <&ps_atc0_pcie>; msi-controller; msi-parent = <&apciec0>; @@ -2063,7 +2065,8 @@ power-domains = <&ps_atc1_cio>, <&ps_atc1_cio_pcie>, - <&ps_atc1_cio_usb>; + <&ps_atc1_cio_usb>, + <&ps_atc1_pcie>; thunderbolt-switch; apple,pcie-tunnel = <&usb4_1_pcie_tunnel>; @@ -2154,11 +2157,12 @@ <0x5 0x20200000 0x0 0x4000>, <0x5 0x21010000 0x0 0x4000>, <0x5 0x2100c000 0x0 0x4000>; - reg-names = "config", "rc", "port0", "common", - "fabric", "debug", "intr2axi"; + reg-names = "config", "rc", "port0", "fabric", + "debug", "ltssm", "intr2axi"; interrupt-parent = <&aic>; interrupts = ; + power-domains = <&ps_atc1_pcie>; msi-controller; msi-parent = <&apciec1>; From 35f6323e8e2418da8b4d1d6953f1c487a7c669c9 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 23:21:32 -0400 Subject: [PATCH 0321/2680] arm64: dts: apple: t602x: describe the tunneled PCIe lifecycle resources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — split: DT part only; the thunderbolt driver part goes with 170-usb] Signed-off-by: DjDeveloperr --- arch/arm64/boot/dts/apple/t602x-dieX.dtsi | 24 +++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t602x-dieX.dtsi b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi index b171c0ad6d65c..41c924dc7c779 100644 --- a/arch/arm64/boot/dts/apple/t602x-dieX.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi @@ -589,9 +589,11 @@ <0x7 0x20000000 0x0 0x4000>, <0x7 0x21000000 0x0 0x8000>, <0x7 0x20004000 0x0 0x4000>, - <0x7 0x20200000 0x0 0x4000>; + <0x7 0x20200000 0x0 0x4000>, + <0x7 0x21100000 0x0 0x4000>, + <0x7 0x2100c000 0x0 0x4000>; reg-names = "config", "rc", "port0", "fabric", - "debug"; + "debug", "ltssm", "intr2axi"; interrupt-parent = <&aic>; interrupts = ; @@ -813,9 +815,11 @@ <0xb 0x20000000 0x0 0x4000>, <0xb 0x21000000 0x0 0x8000>, <0xb 0x20004000 0x0 0x4000>, - <0xb 0x20200000 0x0 0x4000>; + <0xb 0x20200000 0x0 0x4000>, + <0xb 0x21100000 0x0 0x4000>, + <0xb 0x2100c000 0x0 0x4000>; reg-names = "config", "rc", "port0", "fabric", - "debug"; + "debug", "ltssm", "intr2axi"; interrupt-parent = <&aic>; interrupts = ; @@ -1037,9 +1041,11 @@ <0xf 0x20000000 0x0 0x4000>, <0xf 0x21000000 0x0 0x8000>, <0xf 0x20004000 0x0 0x4000>, - <0xf 0x20200000 0x0 0x4000>; + <0xf 0x20200000 0x0 0x4000>, + <0xf 0x21100000 0x0 0x4000>, + <0xf 0x2100c000 0x0 0x4000>; reg-names = "config", "rc", "port0", "fabric", - "debug"; + "debug", "ltssm", "intr2axi"; interrupt-parent = <&aic>; interrupts = ; @@ -1260,9 +1266,11 @@ <0x13 0x20000000 0x0 0x4000>, <0x13 0x21000000 0x0 0x8000>, <0x13 0x20004000 0x0 0x4000>, - <0x13 0x20200000 0x0 0x4000>; + <0x13 0x20200000 0x0 0x4000>, + <0x13 0x21100000 0x0 0x4000>, + <0x13 0x2100c000 0x0 0x4000>; reg-names = "config", "rc", "port0", "fabric", - "debug"; + "debug", "ltssm", "intr2axi"; interrupt-parent = <&aic>; interrupts = ; From 59033a633245a4b6fdb53b89f6937f24c8a2bec5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 29 Aug 2026 19:30:52 -0400 Subject: [PATCH 0322/2680] arm64: dts: apple: describe multi-DCP Type-C display topology With the DCP display fabric in place, describe the routes the hardware actually has instead of pinning every Type-C port to one external DCP. On the M2 Pro/Max laptops dcpext0 drives the fixed HDMI output but can also reach all three Type-C PHYs, so give it those PHYs, crossbar controls and typec-routes alongside its fixed dp-phy, and point each Type-C connector at both external DCPs. External DCP 0 uses crossbar state 0 and external DCP 1 state 2, matching the ADT. Keep this in the t602x file rather than the shared t600x one: on the M1 Pro/Max laptops dcpext0 is disabled, and a route node inherits no status from its parent, so describing routes there would leave typec_mux_match() deferring forever on a mux that never registers and take Type-C DP down on those machines entirely. Drop the stale mux-index from dcpext1, which selects its crossbar per route, and remove the static displayport phandles wherever a typec-routes provider now owns hotplug notification, so a connector cannot be notified twice or through the wrong pipeline. On the M2 Pro mini and M2 Max Studio that phandle is guarded by APPLE_T600X_NON_ULTRA, which only t6001.dtsi defines, so it survives into t602x and has to be deleted explicitly. Boards without such a provider keep their route. Signed-off-by: DjDeveloperr --- .../arm64/boot/dts/apple/t600x-j314-j316.dtsi | 25 +++-- arch/arm64/boot/dts/apple/t600x-j375.dtsi | 8 -- .../arm64/boot/dts/apple/t602x-j414-j416.dtsi | 91 ++++++++++++++++--- .../arm64/boot/dts/apple/t602x-j474-j475.dtsi | 13 ++- arch/arm64/boot/dts/apple/t8103-jxxx.dtsi | 2 - arch/arm64/boot/dts/apple/t8112-j473.dts | 4 - arch/arm64/boot/dts/apple/t8112-jxxx.dtsi | 8 -- 7 files changed, 100 insertions(+), 51 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index 209d62ae6eae4..36c96a312d484 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -145,6 +145,7 @@ }; }; }; + }; #ifdef APPLE_T600X_NON_ULTRA @@ -539,12 +540,14 @@ #ifdef APPLE_T600X_NON_ULTRA &typec0 { - displayport = <&dcpext1>; - ports { port@3 { reg = <3>; - typec0_dcp: endpoint { + #address-cells = <1>; + #size-cells = <0>; + + typec0_dcp: endpoint@0 { + reg = <0>; remote-endpoint = <&dcpext1_typec0>; }; }; @@ -552,12 +555,14 @@ }; &typec1 { - displayport = <&dcpext1>; - ports { port@3 { reg = <3>; - typec1_dcp: endpoint { + #address-cells = <1>; + #size-cells = <0>; + + typec1_dcp: endpoint@0 { + reg = <0>; remote-endpoint = <&dcpext1_typec1>; }; }; @@ -565,12 +570,14 @@ }; &typec2 { - displayport = <&dcpext1>; - ports { port@3 { reg = <3>; - typec2_dcp: endpoint { + #address-cells = <1>; + #size-cells = <0>; + + typec2_dcp: endpoint@0 { + reg = <0>; remote-endpoint = <&dcpext1_typec2>; }; }; diff --git a/arch/arm64/boot/dts/apple/t600x-j375.dtsi b/arch/arm64/boot/dts/apple/t600x-j375.dtsi index a234a0e7b031a..2b16c93d410bd 100644 --- a/arch/arm64/boot/dts/apple/t600x-j375.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j375.dtsi @@ -361,8 +361,6 @@ #ifdef APPLE_T600X_NON_ULTRA &typec0 { - displayport = <&dcpext1>; - ports { port@3 { reg = <3>; @@ -374,8 +372,6 @@ }; &typec1 { - displayport = <&dcpext1>; - ports { port@3 { reg = <3>; @@ -387,8 +383,6 @@ }; &typec2 { - displayport = <&dcpext1>; - ports { port@3 { reg = <3>; @@ -400,8 +394,6 @@ }; &typec3 { - displayport = <&dcpext1>; - ports { port@3 { reg = <3>; diff --git a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi index 0f1f984fca875..0594a7045d351 100644 --- a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi @@ -68,12 +68,16 @@ hdmi-hpd-gpios = <&pinctrl_aop 25 GPIO_ACTIVE_HIGH>; hdmi-pwren-gpios = <&smc_gpio 23 GPIO_ACTIVE_HIGH>; dp2hdmi-pwren-gpios = <&smc_gpio 6 GPIO_ACTIVE_HIGH>; - phys = <&atcphy3 PHY_TYPE_DP>; - phy-names = "dp-phy"; - mux-controls = <&atcphy3_xbar 0>; - mux-control-names = "dp-xbar"; + phys = <&atcphy3 PHY_TYPE_DP>, <&atcphy0 PHY_TYPE_DP>, + <&atcphy1 PHY_TYPE_DP>, <&atcphy2 PHY_TYPE_DP>; + phy-names = "dp-phy", "typec0", "typec1", "typec2"; + mux-controls = <&atcphy3_xbar 0>, <&atcphy0_xbar 0>, + <&atcphy1_xbar 0>, <&atcphy2_xbar 0>; + mux-control-names = "dp-xbar", "typec0", "typec1", "typec2"; mux-index = <0>; apple,dptx-phy = <3>; + apple,typec-mux-indices = <0 0 0>; + apple,typec-dptx-phys = <0 1 2>; ports { #address-cells = <1>; @@ -86,6 +90,44 @@ }; }; }; + + typec-routes { + #address-cells = <1>; + #size-cells = <0>; + + route@0 { + reg = <0>; + mode-switch; + + port { + dcpext0_typec0: endpoint { + remote-endpoint = <&typec0_dcpext0>; + }; + }; + }; + + route@1 { + reg = <1>; + mode-switch; + + port { + dcpext0_typec1: endpoint { + remote-endpoint = <&typec1_dcpext0>; + }; + }; + }; + + route@2 { + reg = <2>; + mode-switch; + + port { + dcpext0_typec2: endpoint { + remote-endpoint = <&typec2_dcpext0>; + }; + }; + }; + }; }; &dispext1_dart { @@ -101,6 +143,7 @@ }; &dcpext1 { + /delete-property/ mux-index; status = "okay"; apple,connector-type = "USB-C"; phys = <&atcphy0 PHY_TYPE_DP>, <&atcphy1 PHY_TYPE_DP>, @@ -152,40 +195,62 @@ }; &typec0 { - displayport = <&dcpext1>; - ports { port@3 { reg = <3>; - typec0_dcp: endpoint { + #address-cells = <1>; + #size-cells = <0>; + + typec0_dcp: endpoint@0 { + reg = <0>; remote-endpoint = <&dcpext1_typec0>; }; + + typec0_dcpext0: endpoint@1 { + reg = <1>; + remote-endpoint = <&dcpext0_typec0>; + }; }; }; }; &typec1 { - displayport = <&dcpext1>; - + /delete-property/ displayport; ports { port@3 { reg = <3>; - typec1_dcp: endpoint { + #address-cells = <1>; + #size-cells = <0>; + + typec1_dcp: endpoint@0 { + reg = <0>; remote-endpoint = <&dcpext1_typec1>; }; + + typec1_dcpext0: endpoint@1 { + reg = <1>; + remote-endpoint = <&dcpext0_typec1>; + }; }; }; }; &typec2 { - displayport = <&dcpext1>; - ports { port@3 { reg = <3>; - typec2_dcp: endpoint { + #address-cells = <1>; + #size-cells = <0>; + + typec2_dcp: endpoint@0 { + reg = <0>; remote-endpoint = <&dcpext1_typec2>; }; + + typec2_dcpext0: endpoint@1 { + reg = <1>; + remote-endpoint = <&dcpext0_typec2>; + }; }; }; }; diff --git a/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi b/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi index 0da6de6e960e2..0e25a4fd62681 100644 --- a/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi @@ -107,8 +107,6 @@ }; &typec0 { - displayport = <&dcpext1>; - ports { port@3 { reg = <3>; @@ -120,7 +118,12 @@ }; &typec1 { - displayport = <&dcpext1>; + /* + * t600x-j375.dtsi guards this on APPLE_T600X_NON_ULTRA, which only + * t6001.dtsi defines, so the legacy phandle survives on t602x. The + * typec-routes provider owns hotplug notification here. + */ + /delete-property/ displayport; ports { port@3 { @@ -133,8 +136,6 @@ }; &typec2 { - displayport = <&dcpext1>; - ports { port@3 { reg = <3>; @@ -146,8 +147,6 @@ }; &typec3 { - displayport = <&dcpext1>; - ports { port@3 { reg = <3>; diff --git a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi index 00b566ed848e4..627c8cac8d15c 100644 --- a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi @@ -86,7 +86,6 @@ compatible = "usb-c-connector"; power-role = "dual"; data-role = "dual"; - displayport = <&dcpext>; ports { #address-cells = <1>; @@ -132,7 +131,6 @@ compatible = "usb-c-connector"; power-role = "dual"; data-role = "dual"; - displayport = <&dcpext>; ports { #address-cells = <1>; diff --git a/arch/arm64/boot/dts/apple/t8112-j473.dts b/arch/arm64/boot/dts/apple/t8112-j473.dts index 9c54bbe563693..46992c6868411 100644 --- a/arch/arm64/boot/dts/apple/t8112-j473.dts +++ b/arch/arm64/boot/dts/apple/t8112-j473.dts @@ -120,8 +120,6 @@ }; &typec0 { - displayport = <&dcp>; - ports { port@3 { reg = <3>; @@ -133,8 +131,6 @@ }; &typec1 { - displayport = <&dcp>; - ports { port@3 { reg = <3>; diff --git a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi index 0346407ed582a..f99a508e8e620 100644 --- a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi @@ -80,10 +80,6 @@ compatible = "usb-c-connector"; power-role = "dual"; data-role = "dual"; -#ifndef T8112_TYPEC_DCP - displayport = <&dcpext>; -#endif - ports { #address-cells = <1>; #size-cells = <0>; @@ -131,10 +127,6 @@ power-role = "dual"; data-role = "dual"; -#ifndef T8112_TYPEC_DCP - displayport = <&dcpext>; -#endif - ports { #address-cells = <1>; #size-cells = <0>; From 293f17ddb7009adfa09d25e6c3d9420ac0e7f8eb Mon Sep 17 00:00:00 2001 From: Marcelo Alcantara Date: Mon, 14 Sep 2026 13:29:35 +1000 Subject: [PATCH 0323/2680] arm64: dts: apple: t6021: add dcpext2/dcpext3 external displays The M2 Max die has four external display controllers, but only dcpext0 (HDMI + Type-C) and dcpext1 (Type-C) were described. Add dcpext2 and dcpext3 as further Type-C route candidates in a new t6021-display2.dtsi, included by the M2 Max laptop boards (j414c/j416c) only -- M2 Pro (t6020) has two external DCPs. This lets the machine drive its internal panel plus HDMI and three Type-C displays at once. Needs MAX_DARTS_PER_DEVICE >= 5 (previous patch): the display-subsystem now lists five scanout DARTs. --- arch/arm64/boot/dts/apple/t6021-display2.dtsi | 308 ++++++++++++++++++ arch/arm64/boot/dts/apple/t6021-j414c.dts | 1 + arch/arm64/boot/dts/apple/t6021-j416c.dts | 1 + 3 files changed, 310 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t6021-display2.dtsi diff --git a/arch/arm64/boot/dts/apple/t6021-display2.dtsi b/arch/arm64/boot/dts/apple/t6021-display2.dtsi new file mode 100644 index 0000000000000..c518139f1fb4b --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6021-display2.dtsi @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Extra external display controllers (dcpext2 / dcpext3) for M2 Max laptops. + * + * The M2 Max die has four external DCPs; the base support wires only dcpext0 + * (HDMI + Type-C) and dcpext1 (Type-C). This adds dcpext2 and dcpext3 as + * further Type-C route candidates so all three Type-C ports (plus HDMI) can + * drive a display at once. M2 Pro (t6020) has only two external DCPs, so this + * is included by the t6021 (Max) laptop boards only. + * + * Requires MAX_DARTS_PER_DEVICE >= 5 in drivers/iommu/apple-dart.c: the + * display-subsystem lists one scanout DART (SID 0) per DCP, and dcpext2/3 + * push that list past the stock cap of 3. + * + * Copyright The Asahi Linux Contributors + */ + +/ { + aliases { + dcpext2 = &dcpext2; + dcpext3 = &dcpext3; + }; + + clk_dispext2: clock-dispext2 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext2"; + }; + + clk_dispext3: clock-dispext3 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext3"; + }; +}; + +&{/soc} { + + dispext2_dart: iommu@48d304000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x4 0x8d304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_dispext2_cpu0>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "okay"; + }; + + dcpext2_dart: iommu@48d30c000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x4 0x8d30c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_dispext2_cpu0>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "okay"; + }; + + dcpext2_mbox: mbox@48dc08000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x8dc08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_dispext2_cpu0>; + resets = <&ps_dispext2_cpu0>; + status = "okay"; + }; + + dcpext2: dcp@48dc00000 { + compatible = "apple,t6020-dcpext", "apple,dcpext"; + mboxes = <&dcpext2_mbox>; + mbox-names = "mbox"; + iommus = <&dcpext2_dart 5>; + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x4 0x8dc00000 0x0 0x4000>, + <0x4 0x8c000000 0x0 0x4000000>, + <0x4 0x8d320000 0x0 0x4000>, + <0x4 0x8d344000 0x0 0x4000>, + <0x4 0x8d800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x1220>; + apple,iomfb-surfaces = <1 1 0 1>; + power-domains = <&ps_dispext2_cpu0>; + resets = <&ps_dispext2_cpu0>; + clocks = <&clk_dispext2>; + apple,dcp-index = <3>; + status = "okay"; + apple,connector-type = "USB-C"; + apple,dptx-phy = <1>; + phys = <&atcphy0 PHY_TYPE_DP>, <&atcphy1 PHY_TYPE_DP>, + <&atcphy2 PHY_TYPE_DP>; + phy-names = "typec0", "typec1", "typec2"; + mux-controls = <&atcphy0_xbar 0>, <&atcphy1_xbar 0>, + <&atcphy2_xbar 0>; + mux-control-names = "typec0", "typec1", "typec2"; + apple,typec-mux-indices = <4 4 4>; + apple,typec-dptx-phys = <0 1 2>; + #address-cells = <2>; + #size-cells = <2>; + + piodma { + iommus = <&dispext2_dart 4>; + }; + + typec-routes { + #address-cells = <1>; + #size-cells = <0>; + + route@0 { + reg = <0>; + mode-switch; + port { + dcpext2_typec0: endpoint { + remote-endpoint = <&typec0_dcpext2>; + }; + }; + }; + + route@1 { + reg = <1>; + mode-switch; + port { + dcpext2_typec1: endpoint { + remote-endpoint = <&typec1_dcpext2>; + }; + }; + }; + + route@2 { + reg = <2>; + mode-switch; + port { + dcpext2_typec2: endpoint { + remote-endpoint = <&typec2_dcpext2>; + }; + }; + }; + }; + }; + + dispext3_dart: iommu@50d304000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x5 0x0d304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_dispext3_cpu0>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "okay"; + }; + + dcpext3_dart: iommu@50d30c000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x5 0x0d30c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_dispext3_cpu0>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "okay"; + }; + + dcpext3_mbox: mbox@50dc08000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x5 0x0dc08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_dispext3_cpu0>; + resets = <&ps_dispext3_cpu0>; + status = "okay"; + }; + + dcpext3: dcp@50dc00000 { + compatible = "apple,t6020-dcpext", "apple,dcpext"; + mboxes = <&dcpext3_mbox>; + mbox-names = "mbox"; + iommus = <&dcpext3_dart 5>; + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x5 0x0dc00000 0x0 0x4000>, + <0x5 0x0c000000 0x0 0x4000000>, + <0x5 0x0d320000 0x0 0x4000>, + <0x5 0x0d344000 0x0 0x4000>, + <0x5 0x0d800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x1228>; + apple,iomfb-surfaces = <1 1 0 1>; + power-domains = <&ps_dispext3_cpu0>; + resets = <&ps_dispext3_cpu0>; + clocks = <&clk_dispext3>; + apple,dcp-index = <4>; + status = "okay"; + apple,connector-type = "USB-C"; + apple,dptx-phy = <1>; + phys = <&atcphy0 PHY_TYPE_DP>, <&atcphy1 PHY_TYPE_DP>, + <&atcphy2 PHY_TYPE_DP>; + phy-names = "typec0", "typec1", "typec2"; + mux-controls = <&atcphy0_xbar 0>, <&atcphy1_xbar 0>, + <&atcphy2_xbar 0>; + mux-control-names = "typec0", "typec1", "typec2"; + apple,typec-mux-indices = <6 6 6>; + apple,typec-dptx-phys = <0 1 2>; + #address-cells = <2>; + #size-cells = <2>; + + piodma { + iommus = <&dispext3_dart 4>; + }; + + typec-routes { + #address-cells = <1>; + #size-cells = <0>; + + route@0 { + reg = <0>; + mode-switch; + port { + dcpext3_typec0: endpoint { + remote-endpoint = <&typec0_dcpext3>; + }; + }; + }; + + route@1 { + reg = <1>; + mode-switch; + port { + dcpext3_typec1: endpoint { + remote-endpoint = <&typec1_dcpext3>; + }; + }; + }; + + route@2 { + reg = <2>; + mode-switch; + port { + dcpext3_typec2: endpoint { + remote-endpoint = <&typec2_dcpext3>; + }; + }; + }; + }; + }; +}; + +&display { + iommus = <&disp0_dart 0>, <&dispext0_dart 0>, <&dispext1_dart 0>, + <&dispext2_dart 0>, <&dispext3_dart 0>; +}; + +&typec0 { + ports { + port@3 { + typec0_dcpext2: endpoint@2 { + reg = <2>; + remote-endpoint = <&dcpext2_typec0>; + }; + typec0_dcpext3: endpoint@3 { + reg = <3>; + remote-endpoint = <&dcpext3_typec0>; + }; + }; + }; +}; + +&typec1 { + ports { + port@3 { + typec1_dcpext2: endpoint@2 { + reg = <2>; + remote-endpoint = <&dcpext2_typec1>; + }; + typec1_dcpext3: endpoint@3 { + reg = <3>; + remote-endpoint = <&dcpext3_typec1>; + }; + }; + }; +}; + +&typec2 { + ports { + port@3 { + typec2_dcpext2: endpoint@2 { + reg = <2>; + remote-endpoint = <&dcpext2_typec2>; + }; + typec2_dcpext3: endpoint@3 { + reg = <3>; + remote-endpoint = <&dcpext3_typec2>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6021-j414c.dts b/arch/arm64/boot/dts/apple/t6021-j414c.dts index fab3b03ff3c45..ef90ffe8423f4 100644 --- a/arch/arm64/boot/dts/apple/t6021-j414c.dts +++ b/arch/arm64/boot/dts/apple/t6021-j414c.dts @@ -11,6 +11,7 @@ #include "t6021.dtsi" #include "t602x-j414-j416.dtsi" +#include "t6021-display2.dtsi" / { compatible = "apple,j414c", "apple,t6021", "apple,arm-platform"; diff --git a/arch/arm64/boot/dts/apple/t6021-j416c.dts b/arch/arm64/boot/dts/apple/t6021-j416c.dts index b476e235639ff..1c95fce375000 100644 --- a/arch/arm64/boot/dts/apple/t6021-j416c.dts +++ b/arch/arm64/boot/dts/apple/t6021-j416c.dts @@ -11,6 +11,7 @@ #include "t6021.dtsi" #include "t602x-j414-j416.dtsi" +#include "t6021-display2.dtsi" / { compatible = "apple,j416c", "apple,t6021", "apple,arm-platform"; From 03fac165cb4632a8d8b31a12e9bab48501f038d9 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Sun, 20 Sep 2026 10:30:19 +0200 Subject: [PATCH 0324/2680] arm64: dts: apple: t600x: add m1n1 v1.6.1 USB4 aliases m1n1 v1.6.1 copies ACIO tunables and the USB4 DROM using aliases named usb4_%d_rc and usb4_%d_nhi (underscores). The DT only provided usb4-%d-acio (hyphens). On that m1n1 the tunable copy is skipped, so thunderbolt-apple-acio fails with "Unable to load rc tunable" and the Type-C ports that wait on ACIO never come up. Newer m1n1 falls back to the hyphen names. Ship both so v1.6.1 and current m1n1 work. t602x J414/J416 include this file. Tested on MacBook Pro 16-inch M2 Pro (apple,j416s). Signed-off-by: Oliver Lukschander --- arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index 36c96a312d484..b7efe3826fe9c 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -34,14 +34,26 @@ usb4-0-nhi = &usb4_0_nhi; usb4-0-pcie-adapter = &usb4_0_acio; usb4-0-pcie = &apciec0; + usb4_0_rc = &usb4_0_acio; + usb4_0_nhi = &usb4_0_nhi; + usb4_0_pcie_adapter = &usb4_0_acio; + usb4_0_pcie = &apciec0; usb4-1-acio = &usb4_1_acio; usb4-1-nhi = &usb4_1_nhi; usb4-1-pcie-adapter = &usb4_1_acio; usb4-1-pcie = &apciec1; + usb4_1_rc = &usb4_1_acio; + usb4_1_nhi = &usb4_1_nhi; + usb4_1_pcie_adapter = &usb4_1_acio; + usb4_1_pcie = &apciec1; usb4-2-acio = &usb4_2_acio; usb4-2-nhi = &usb4_2_nhi; usb4-2-pcie-adapter = &usb4_2_acio; usb4-2-pcie = &apciec2; + usb4_2_rc = &usb4_2_acio; + usb4_2_nhi = &usb4_2_nhi; + usb4_2_pcie_adapter = &usb4_2_acio; + usb4_2_pcie = &apciec2; usb4-3-pcie = &apciec3; wifi0 = &wifi0; From 064ba7d0a735eddf26e28a28078633aea7efd2ab Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 23 Sep 2026 10:56:26 -0700 Subject: [PATCH 0325/2680] arm64: dts: apple: t8103: describe Thunderbolt PCIe enumeration resources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit t8103 boots with the Thunderbolt PCIe port held in reset. Bring the port up when the tunnel activates, and do it before the port's IOMMU probes: that IOMMU's invalidate command does not finish while the port clock is still gated. Program the port tunables and the reset table, release PERST, enable the port clock, then program the root-port configuration tunables. Link training stays in the host driver. If the port is already running, the host does not replay the reset table. Endpoint memory on t8103 is left posted. The PCIe-C IOMMU uses the 64-stream USB4 register block: 16 KiB pages, a 32-bit IOVA and a 36-bit physical address. The 42-bit four-level geometry remains on T8110, which is the hardware that walks it. An xHCI early handoff no longer faults when the capability length read from a BAR is shorter than 16 bytes or not dword-aligned. Tested on a MacBook Pro 13" M1 (j293) with a CalDigit TS3 Plus on USB-1. The tunnel enumerated the dock bridges, four USB host controllers and an I210. The USB devices and a gigabit link came up, and the display on the same port still comes up. [ryan: adapted to 7.2.10 — split: DT part only; the PCI, DART, thunderbolt and xhci parts go with 080-radio and 170-usb] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- .../boot/dts/apple/t8103-pciec-tunables.dtsi | 50 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8103.dtsi | 32 ++++++++---- 2 files changed, 72 insertions(+), 10 deletions(-) create mode 100644 arch/arm64/boot/dts/apple/t8103-pciec-tunables.dtsi diff --git a/arch/arm64/boot/dts/apple/t8103-pciec-tunables.dtsi b/arch/arm64/boot/dts/apple/t8103-pciec-tunables.dtsi new file mode 100644 index 0000000000000..2265bdab1cba3 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8103-pciec-tunables.dtsi @@ -0,0 +1,50 @@ +/* t8103 Thunderbolt PCIe configuration, shared by both ports. + * Offsets are relative to the region named by each property. + */ +apple,pciec-kernel-init; +apple,tunable-rc = + <0x718 0x7c000 0x0>, + <0x8bc 0x1 0x0>, + <0x78 0x7000 0x0>, + <0x814 0xffffffff 0xf060>; +apple,tunable-fabric = + <0x4 0x1f001f 0x180018>, + <0x8 0x1 0x1>, + <0xc 0x1 0x1>, + <0x10 0x801f 0x4>, + <0x18 0x3ff 0x280>, + <0x44 0x1 0x1>, + <0x94 0x801f 0x4>, + <0x9c 0x801f 0x4>, + <0x20 0x801f 0x4>, + <0x28 0x3f 0x20>, + <0x2c 0x801f 0x4>, + <0x34 0x3f 0x20>, + <0x38 0x801f 0x4>, + <0x40 0x10001 0x10001>, + <0x48 0x1 0x1>, + <0x4c 0x801f 0x4>, + <0x54 0x7f007f 0x4b004b>, + <0x58 0x801f 0x4>, + <0x60 0x801f 0x4>, + <0x68 0x1 0x1>, + <0x6c 0x801f 0x4>, + <0x74 0x1 0x1>, + <0x78 0x801f 0x4>, + <0x84 0x801f 0x4>, + <0x8c 0x1 0x1>, + <0x90 0x1 0x1>, + <0xa4 0x801f 0x4>; +apple,tunable-debug = + <0x1000 0x1 0x0>; +apple,tunable-oe-fabric = + <0x4 0x70007 0x40004>, + <0x8 0x1 0x1>, + <0xc 0x1 0x1>, + <0x10 0x801f 0x4>, + <0x18 0x7f007f 0x450045>, + <0x1c 0x801f 0x4>, + <0x24 0x801f 0x4>, + <0x2c 0x1 0x1>, + <0x30 0x1 0x1>, + <0x34 0x801f 0x4>; diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index daa23f4ab489e..ba13c756900c5 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -1572,8 +1572,11 @@ usb4_0_acio: cio@381ac0000 { compatible = "apple,t8103-usb4-acio"; reg = <0x3 0x81ac0000 0x0 0xc000>, - <0x3 0x81080000 0x0 0x20000>; - reg-names = "rc", "sram"; + <0x3 0x81080000 0x0 0x20000>, + <0x3 0x81e44000 0x0 0x4000>; + reg-names = "rc", "sram", "pcie-adapter"; + /* PCIe adapter configuration. One record, bits [2:1] at +0x1008. */ + apple,tunable-pcie-adapter = <0x1008 0x6 0x6>; mboxes = <&usb4_0_mbox>; resets = <&cio_reset 0>; @@ -1653,7 +1656,7 @@ ranges; apciec0_dart: iommu@3a1008000 { - compatible = "apple,t8103-dart"; + compatible = "apple,t8103-usb4-dart"; reg = <0x3 0xa1008000 0x0 0x4000>; interrupt-parent = <&aic>; interrupts = ; @@ -1670,9 +1673,10 @@ <0x3 0xa0004000 0x0 0x4000>, <0x3 0xa0200000 0x0 0x4000>, <0x3 0xa1010000 0x0 0x4000>, - <0x3 0xa100c000 0x0 0x4000>; + <0x3 0xa0008000 0x0 0x4000>; reg-names = "config", "rc", "port0", "fabric", - "debug", "ltssm", "intr2axi"; + "debug", "ltssm", "oe-fabric"; +#include "t8103-pciec-tunables.dtsi" interrupt-parent = <&aic>; interrupts = ; @@ -1699,6 +1703,8 @@ device_type = "pci"; reg = <0x0 0x0 0x0 0x0 0x0>; external-facing; + /* Port configuration. One record at +0x140. */ + apple,tunable = <0x140 0x1 0x1>; #address-cells = <3>; #size-cells = <2>; @@ -1791,8 +1797,11 @@ usb4_1_acio: cio@501ac0000 { compatible = "apple,t8103-usb4-acio"; reg = <0x5 0x01ac0000 0x0 0xc000>, - <0x5 0x01080000 0x0 0x20000>; - reg-names = "rc", "sram"; + <0x5 0x01080000 0x0 0x20000>, + <0x5 0x01e44000 0x0 0x4000>; + reg-names = "rc", "sram", "pcie-adapter"; + /* PCIe adapter configuration. One record, bits [2:1] at +0x1008. */ + apple,tunable-pcie-adapter = <0x1008 0x6 0x6>; mboxes = <&usb4_1_mbox>; resets = <&cio_reset 1>; @@ -1872,7 +1881,7 @@ ranges; apciec1_dart: iommu@521008000 { - compatible = "apple,t8103-dart"; + compatible = "apple,t8103-usb4-dart"; reg = <0x5 0x21008000 0x0 0x4000>; interrupt-parent = <&aic>; interrupts = ; @@ -1889,9 +1898,10 @@ <0x5 0x20004000 0x0 0x4000>, <0x5 0x20200000 0x0 0x4000>, <0x5 0x21010000 0x0 0x4000>, - <0x5 0x2100c000 0x0 0x4000>; + <0x5 0x20008000 0x0 0x4000>; reg-names = "config", "rc", "port0", "fabric", - "debug", "ltssm", "intr2axi"; + "debug", "ltssm", "oe-fabric"; +#include "t8103-pciec-tunables.dtsi" interrupt-parent = <&aic>; interrupts = ; @@ -1918,6 +1928,8 @@ device_type = "pci"; reg = <0x0 0x0 0x0 0x0 0x0>; external-facing; + /* Port configuration. One record at +0x140. */ + apple,tunable = <0x140 0x1 0x1>; #address-cells = <3>; #size-cells = <2>; From 1a78bbb475723fc7e231b4fecd425e3f93a42c2a Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Thu, 24 Sep 2026 16:47:27 +0200 Subject: [PATCH 0326/2680] arm64: dts: apple: t602x: add USB4 tunnel mux-control entries Add a second mux-control entry (index 1) per Type-C PHY crossbar to both dcpext0's and dcpext1's mux-controls/mux-control-names, named "typecN-usb4" alongside the existing "typecN" (direct alt-mode) entry. These are the devicetree handles the display crossbar driver resolves by name to select a USB4/Thunderbolt DP tunnel route (dpin0/dpin1) rather than a direct Type-C alt-mode route, on T602X (M2 Pro) j414/j416 machines. Co-Authored-By: Claude Sonnet 5 --- arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi index 0594a7045d351..50f97498f58af 100644 --- a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi @@ -72,8 +72,11 @@ <&atcphy1 PHY_TYPE_DP>, <&atcphy2 PHY_TYPE_DP>; phy-names = "dp-phy", "typec0", "typec1", "typec2"; mux-controls = <&atcphy3_xbar 0>, <&atcphy0_xbar 0>, - <&atcphy1_xbar 0>, <&atcphy2_xbar 0>; - mux-control-names = "dp-xbar", "typec0", "typec1", "typec2"; + <&atcphy1_xbar 0>, <&atcphy2_xbar 0>, + <&atcphy0_xbar 1>, <&atcphy1_xbar 1>, + <&atcphy2_xbar 1>; + mux-control-names = "dp-xbar", "typec0", "typec1", "typec2", + "typec0-usb4", "typec1-usb4", "typec2-usb4"; mux-index = <0>; apple,dptx-phy = <3>; apple,typec-mux-indices = <0 0 0>; @@ -150,8 +153,11 @@ <&atcphy2 PHY_TYPE_DP>; phy-names = "typec0", "typec1", "typec2"; mux-controls = <&atcphy0_xbar 0>, <&atcphy1_xbar 0>, - <&atcphy2_xbar 0>; - mux-control-names = "typec0", "typec1", "typec2"; + <&atcphy2_xbar 0>, + <&atcphy0_xbar 1>, <&atcphy1_xbar 1>, + <&atcphy2_xbar 1>; + mux-control-names = "typec0", "typec1", "typec2", + "typec0-usb4", "typec1-usb4", "typec2-usb4"; apple,typec-mux-indices = <2 2 2>; apple,typec-dptx-phys = <0 1 2>; From 0d3c07f439b10e78766bffda94695bf7cf32fc75 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 20:35:13 -0700 Subject: [PATCH 0327/2680] arm64: dts: apple: t8103: describe tunneled PCIe state restore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kensington USB devices can disappear after s2idle even when the initial port restart and cable health check succeed. A later PCIe link failure must trigger another fresh validation instead of waiting for a cable interrupt or manual replug. Restore the resettable M1 tunneled host and DART state before restarting the downstream link. Gate DART commands and fault handling around clock shutdown, preserve stream selection, and describe the reset and register initialization properties in the binding. Gate Type-C work across initialization, sleep and removal, read fresh cable metadata after resume, and preserve an unchanged healthy session. Propagate late link failures through an atomic switch notification into bounded process-context revalidation. Keep failure history sticky while allowing later notifications from a retained host. Failed reads or health checks request a bounded reconnect; successful checks preserve routing. Keep failed display-route acquisition retryable. Add state-machine and DART regression coverage. The native ARM64 build and 88 KUnit cases pass. Two M1/j293 Kensington USB-only suspend cycles exercised late link-loss recovery and restored usable input without a manual replug. The initiating xHCI resume fault remains unresolved. [ryan: adapted to 7.2.10 — split: DT part only; driver and binding parts go with 170-usb] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t8103.dtsi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index ba13c756900c5..ea073c207c014 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -1681,6 +1681,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_atc0_pcie>; + resets = <&ps_atc0_pcie>; msi-controller; msi-parent = <&apciec0>; @@ -1906,6 +1907,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_atc1_pcie>; + resets = <&ps_atc1_pcie>; msi-controller; msi-parent = <&apciec1>; From 00ee2936728b6c6dcd1d7ccf2dc9734a9aa9d337 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 28 Sep 2026 13:02:15 -0700 Subject: [PATCH 0328/2680] arm64: dts: apple: t8103: alias the USB4 controllers for m1n1 m1n1 copies each machine's apple,tunable-rc onto the node named by usb4-N-acio, or usb4_N_rc on the 1.6.1 bootloader. The t8103 CIO nodes were only labels, so a boot image rebuilt from this tree never received the calibration and the Thunderbolt driver refused to probe. --- arch/arm64/boot/dts/apple/t8103.dtsi | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index ea073c207c014..3f13a03f86964 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -22,6 +22,21 @@ aliases { gpu = &gpu; + /* m1n1 copies this machine's Thunderbolt calibration onto the + * node these names point at. Current m1n1 looks up usb4-N-acio + * and usb4-N-nhi; the packaged 1.6.1 bootloader looks up + * usb4_N_rc and usb4_N_nhi. A label on the node is not enough. + */ + usb4-0-acio = &usb4_0_acio; + usb4_0_rc = &usb4_0_acio; + usb4-0-nhi = &usb4_0_nhi; + usb4_0_nhi = &usb4_0_nhi; + usb4_0_pcie_adapter = &usb4_0_acio; + usb4-1-acio = &usb4_1_acio; + usb4_1_rc = &usb4_1_acio; + usb4-1-nhi = &usb4_1_nhi; + usb4_1_nhi = &usb4_1_nhi; + usb4_1_pcie_adapter = &usb4_1_acio; }; cpus { From dcbbed223739aa83578d89beea79ddafa8fd074a Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 28 Sep 2026 13:10:57 -0700 Subject: [PATCH 0329/2680] arm64: dts: apple: t602x: alias the USB4 controllers for m1n1 The 14-inch and 16-inch M2 Pro/Max boards and the M2 Pro/Max mini enable the CIO nodes without the usb4-N-acio aliases m1n1 searches for, so a rebuilt boot image never receives apple,tunable-rc. --- .../arm64/boot/dts/apple/t602x-j414-j416.dtsi | 27 ++++++++++++++ .../arm64/boot/dts/apple/t602x-j474-j475.dtsi | 35 +++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi index 50f97498f58af..90dc63a27163f 100644 --- a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi @@ -26,6 +26,33 @@ disp0_piodma = &disp0_piodma; keyboard = &keyboard; sio = &sio; + /* m1n1 writes this machine's Thunderbolt calibration onto these + * aliases. usb4-N-* is current m1n1; usb4_N_* is the 1.6.1 bootloader. + */ + usb4-0-acio = &usb4_0_acio; + usb4-0-nhi = &usb4_0_nhi; + usb4-0-pcie-adapter = &usb4_0_acio; + usb4-0-pcie = &apciec0; + usb4_0_rc = &usb4_0_acio; + usb4_0_nhi = &usb4_0_nhi; + usb4_0_pcie_adapter = &usb4_0_acio; + usb4_0_pcie = &apciec0; + usb4-1-acio = &usb4_1_acio; + usb4-1-nhi = &usb4_1_nhi; + usb4-1-pcie-adapter = &usb4_1_acio; + usb4-1-pcie = &apciec1; + usb4_1_rc = &usb4_1_acio; + usb4_1_nhi = &usb4_1_nhi; + usb4_1_pcie_adapter = &usb4_1_acio; + usb4_1_pcie = &apciec1; + usb4-2-acio = &usb4_2_acio; + usb4-2-nhi = &usb4_2_nhi; + usb4-2-pcie-adapter = &usb4_2_acio; + usb4-2-pcie = &apciec2; + usb4_2_rc = &usb4_2_acio; + usb4_2_nhi = &usb4_2_nhi; + usb4_2_pcie_adapter = &usb4_2_acio; + usb4_2_pcie = &apciec2; }; }; diff --git a/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi b/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi index 0e25a4fd62681..64adaec8f3f11 100644 --- a/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi @@ -25,6 +25,41 @@ / { aliases { dcpext1 = &dcpext1; + /* Same Thunderbolt aliases as the 14/16-inch boards. This mini + * brings up a fourth controller. + */ + usb4-0-acio = &usb4_0_acio; + usb4-0-nhi = &usb4_0_nhi; + usb4-0-pcie-adapter = &usb4_0_acio; + usb4-0-pcie = &apciec0; + usb4_0_rc = &usb4_0_acio; + usb4_0_nhi = &usb4_0_nhi; + usb4_0_pcie_adapter = &usb4_0_acio; + usb4_0_pcie = &apciec0; + usb4-1-acio = &usb4_1_acio; + usb4-1-nhi = &usb4_1_nhi; + usb4-1-pcie-adapter = &usb4_1_acio; + usb4-1-pcie = &apciec1; + usb4_1_rc = &usb4_1_acio; + usb4_1_nhi = &usb4_1_nhi; + usb4_1_pcie_adapter = &usb4_1_acio; + usb4_1_pcie = &apciec1; + usb4-2-acio = &usb4_2_acio; + usb4-2-nhi = &usb4_2_nhi; + usb4-2-pcie-adapter = &usb4_2_acio; + usb4-2-pcie = &apciec2; + usb4_2_rc = &usb4_2_acio; + usb4_2_nhi = &usb4_2_nhi; + usb4_2_pcie_adapter = &usb4_2_acio; + usb4_2_pcie = &apciec2; + usb4-3-acio = &usb4_3_acio; + usb4-3-nhi = &usb4_3_nhi; + usb4-3-pcie-adapter = &usb4_3_acio; + usb4-3-pcie = &apciec3; + usb4_3_rc = &usb4_3_acio; + usb4_3_nhi = &usb4_3_nhi; + usb4_3_pcie_adapter = &usb4_3_acio; + usb4_3_pcie = &apciec3; }; }; From 660132860dd6bbf63b55d5303a9e4cd0fdef0e83 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 28 Sep 2026 13:23:29 -0700 Subject: [PATCH 0330/2680] arm64: dts: apple: t8112: alias USB4 for the 1.6.1 bootloader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The base M2 trees already have usb4-N-acio. m1n1 still copies the PCIe tunnel calibration through usb4_N_pcie, and the 1.6.1 bootloader still looks up usb4_N_rc. Add those names for both ports. [ryan: adapted to 7.2.10 — Asahi's WIP m1n1-compat aliases already provide usb4_N_rc/usb4_N_nhi; add only the PCIe adapter and tunnel aliases on top] --- arch/arm64/boot/dts/apple/t8112-jxxx.dtsi | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi index f99a508e8e620..e448f76c0fce3 100644 --- a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi @@ -25,15 +25,19 @@ usb4-0-nhi = &usb4_0_nhi; usb4-0-pcie-adapter = &usb4_0_acio; usb4-0-pcie = &apciec0; + usb4_0_rc = &usb4_0_acio; + usb4_0_nhi = &usb4_0_nhi; + usb4_0_pcie_adapter = &usb4_0_acio; + usb4_0_pcie = &apciec0; usb4-1-acio = &usb4_1_acio; usb4-1-nhi = &usb4_1_nhi; - usb4_0_rc = &usb4_0_acio; - usb4_0_nhi = &usb4_0_nhi; usb4_1_rc = &usb4_1_acio; usb4_1_nhi = &usb4_1_nhi; usb4-1-pcie-adapter = &usb4_1_acio; usb4-1-pcie = &apciec1; + usb4_1_pcie_adapter = &usb4_1_acio; + usb4_1_pcie = &apciec1; }; chosen { From d26c40d48c5460a093c0faf5adc04663d4ab0661 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 15 Sep 2026 17:05:00 -0400 Subject: [PATCH 0331/2680] arm64: dts: apple: t8112: add spi2 for the Touch ID sensor The MacBook Air (M2) and MacBook Pro 13" (M2) carry the Mesa fingerprint sensor on spi2, chip-select 0, like every other Apple laptop. t8112 only described spi1, spi3 and spi4, so the Apple SEP driver had no controller to bring the sensor up on these machines. Add the spi2 controller node, mirroring the sibling spi1/spi3 controllers (reg base + 0x8000, AIC IRQ 750, ps_spi2 power domain), and the spi2_pins pinmux group. The pad numbers (64-67: SDI, SDO, SCK and the hardware chip select) were read back from the live AP pinctrl on a J415 and cross-checked against the T8112 ADT, where spi2 sits between spi1's CS pad 49 and spi3's CS pad 96. The node stays disabled; the SEP driver enables it when it brings up the sensor. Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 --- arch/arm64/boot/dts/apple/t8112.dtsi | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index cdc02310a9334..29980b6e35b09 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -886,6 +886,20 @@ status = "disabled"; }; + spi2: spi@235108000 { + compatible = "apple,t8112-spi", "apple,spi"; + reg = <0x2 0x35108000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clkref>; + pinctrl-0 = <&spi2_pins>; + pinctrl-names = "default"; + power-domains = <&ps_spi2>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + spi3: spi@23510c000 { compatible = "apple,t8112-spi", "apple,spi"; reg = <0x2 0x3510c000 0x0 0x4000>; @@ -1295,6 +1309,13 @@ ; }; + spi2_pins: spi2-pins { + /* Three signal pads; SPI2 uses the controller's native chip select. */ + pinmux = , + , + ; + }; + spi3_pins: spi3-pins { pinmux = , , From 2ee9de298bff2f0a6b0849035b8beb8f9cb712b2 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 13:42:48 -0400 Subject: [PATCH 0332/2680] arm64: dts: apple: t8103: bring up the Mesa fingerprint sensor Describe the SPI2 controller and the Mesa fingerprint sensor on T8103, and enable them on the MacBook Air (M1, J313). The controller sits at the SPI cluster base + 0x8000 (reg 0x235108000, AIC IRQ 616, ps_spi2 power domain), mirroring the sibling SPI nodes. Its pin group is the three signal pads (128 CLK, 129 MOSI, 130 MISO), read back from live hardware as peripheral function 1; SPI2 has no chip-select pad and uses the controller's native chip select. The sensor's power line is GPIO 108 (active high); the data-ready line is left out because the driver polls. Signed-off-by: Chromatischer [dj: adapted for aurora feat/sep, clean-room] Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 --- arch/arm64/boot/dts/apple/t8103-j313.dts | 28 +++++++++++++ arch/arm64/boot/dts/apple/t8103.dtsi | 51 ++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103-j313.dts b/arch/arm64/boot/dts/apple/t8103-j313.dts index 5e658fd35c19e..299227ea2b9a0 100644 --- a/arch/arm64/boot/dts/apple/t8103-j313.dts +++ b/arch/arm64/boot/dts/apple/t8103-j313.dts @@ -94,6 +94,34 @@ }; }; +/* + * Mesa fingerprint sensor on SPI2. The power line (pin 108, active high) comes + * from the J313 platform device tree, which also lists an interrupt on pin 104. The + * interrupt is left out on purpose: the driver polls, and the trigger flags in + * the ADT (3) are not decoded with any certainty here. Pin 104 reads back as a + * plain input with a pull-down, which rules out the two active-low readings of + * that value but does not decide between the remaining ones. + * + * Do not add cs-gpios. The sensor needs the controller's native chip select, + * otherwise the SPI core skips the hardware CS timing. + */ +&spi2 { + status = "okay"; +}; + +&mesa { + status = "okay"; + enable-gpios = <&pinctrl_ap 108 GPIO_ACTIVE_HIGH>; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; + /* + * Per-device factory data written at manufacture. It is not + * redistributable and has to be extracted from the machine's own macOS + * install, so the name only fixes where the driver looks for it. + */ + firmware-name = "apple/mesacal-j313.bin"; +}; + &i2c1 { speaker_left: codec@31 { compatible = "ti,tas5770l", "ti,tas2770"; diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 3f13a03f86964..d6afbdb483015 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -874,6 +874,51 @@ status = "disabled"; }; + /* + * The J313 platform device tree puts the Mesa fingerprint sensor + * on SPI2. The 120 MHz parent was confirmed by timing transfers: + * a 22158-byte transfer requested at 8 MHz, divider 15, ran at + * 7.45 Mbit/s. A 200 MHz parent would make that divider + * 13.3 MHz, and the measurement could not then have come out + * below 8 MHz. + * + * The pin group was read out of the hardware rather than taken + * from a schematic. iBoot leaves every SPI pad it uses at + * peripheral function 1, and the three buses whose groups are + * already described here have the same register signature: two + * pads with no pull for CLK and MOSI, one with a pull-down for + * MISO. Pins 128, 129 and 130 are the only remaining pads with + * that signature, and they are contiguous, so they are SPI2. + * Pin 131 is a different pad type and is not part of the group. + * + * There is no chip-select pin. The platform device tree gives + * SPI1, SPI3 and SPI4 a GPIO chip select (pins 45, 49 and 24, each the + * last pad of its group) and gives SPI2 the string "null", so + * SPI2 uses the controller's own chip select. That is also why + * SPI0, which has no GPIO chip select either, lists three pins. + */ + spi2: spi@235108000 { + compatible = "apple,t8103-spi", "apple,spi"; + reg = <0x2 0x35108000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clk_120m>; + pinctrl-0 = <&spi2_pins>; + pinctrl-names = "default"; + power-domains = <&ps_spi2>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + mesa: fingerprint@0 { + compatible = "apple,mesa-fingerprint"; + reg = <0>; + spi-max-frequency = <8000000>; + spi-cpha; + status = "disabled"; + }; + }; + spi3: spi@23510c000 { compatible = "apple,t8103-spi", "apple,spi"; reg = <0x2 0x3510c000 0x0 0x4000>; @@ -1139,6 +1184,12 @@ ; }; + spi2_pins: spi2-pins { + pinmux = , /* CLK */ + , /* MOSI */ + ; /* MISO */ + }; + spi3_pins: spi3-pins { pinmux = , , From 72036cc478e76bd551398e6638c780e545ece31b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 25 Sep 2026 17:03:59 -0400 Subject: [PATCH 0333/2680] arm64: dts: apple: t602x-j414-j416: J414s SEP enrollment support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Carry only the SEP-related part of 95731b2390, including the SEP device-tree alias, calibration tooling, and fingerprint protocol changes. Exclude Thunderbolt and display modifications from this branch. [ryan: adapted to 7.2.10 — split: DT part only; the SEP driver and tools parts go with 100-sep] Source-Commit: 95731b2390 --- arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi index 90dc63a27163f..dea2ac54c6d8b 100644 --- a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi @@ -25,6 +25,7 @@ disp0 = &display; disp0_piodma = &disp0_piodma; keyboard = &keyboard; + sep = &sep; sio = &sio; /* m1n1 writes this machine's Thunderbolt calibration onto these * aliases. usb4-N-* is current m1n1; usb4_N_* is the 1.6.1 bootloader. From de175e3070ec27dfc7423e88169eca963e371797 Mon Sep 17 00:00:00 2001 From: RJ Skerry-Ryan Date: Sun, 27 Sep 2026 13:03:09 -0700 Subject: [PATCH 0334/2680] arm64: dts: apple: t602x-j414-j416: Enable PCIe-C kernel init No m1n1 performs the PCIe-C preinit handoff on these machines. Mark apciec0-2 "apple,pciec-kernel-init" so the kernel cold-initializes the Thunderbolt PCIe tunnel ports itself. The tunneled DART driver also requires "apple,tunable", but iBoot adds the corresponding "dart-tunables-instance-0" to the ADT only at boot and m1n1 does not copy it for dart-apciec*. Without it the DART fails to probe and devices behind the tunnel time out. Add the values from the live J416c ADT (macOS ioreg), identical for all three ports; the same decoding reproduces entry for entry the apciec tunables m1n1 already copies into the DT. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: RJ Skerry-Ryan --- .../arm64/boot/dts/apple/t602x-j414-j416.dtsi | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi index dea2ac54c6d8b..7affa3cb0aec4 100644 --- a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi @@ -361,6 +361,51 @@ apple,pcie-tunnel = <&usb4_2_pcie_tunnel>; }; +/* + * No m1n1 performs the PCIe-C preinit handoff on these machines. Let the + * PCIe driver cold-initialize the Thunderbolt PCIe tunnel ports itself, so + * PCIe devices behind Thunderbolt 3 docks (e.g. their xHCI) come up. + */ +&apciec0 { + apple,pciec-kernel-init; +}; + +&apciec1 { + apple,pciec-kernel-init; +}; + +&apciec2 { + apple,pciec-kernel-init; +}; + +/* + * PCIe-C DART tunables. iBoot adds "dart-tunables-instance-0" to the ADT at + * boot and m1n1 does not copy it for these DARTs. Values taken from the live + * J416c ADT (macOS ioreg), identical for all three ports; offsets are + * relative to the DART's reg. + */ +#define APCIEC_DART_TUNABLES \ + apple,tunable = <0x20c 0xff0000b7 0xe40000b7>, \ + <0x220 0x000f0f0f 0x000f0f0f>, \ + <0x224 0x00ffffff 0x00080808>, \ + <0x300 0x00001f31 0x00000001>, \ + <0x308 0x3ffffffc 0x10000000>, \ + <0x310 0x3ffffffc 0x3ffffffc> + +&apciec0_dart { + APCIEC_DART_TUNABLES; +}; + +&apciec1_dart { + APCIEC_DART_TUNABLES; +}; + +&apciec2_dart { + APCIEC_DART_TUNABLES; +}; + +#undef APCIEC_DART_TUNABLES + /* Keep dptx_phy_ps on until DCP can sequence it during display bring-up. */ &ps_dptx_phy_ps { apple,always-on; From b67a96e808e33c7854a108486f22969916af218f Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:32:31 -0700 Subject: [PATCH 0335/2680] arm64: dts: apple: t8103-j293: enable the Mesa fingerprint sensor Enable SPI2 and the Mesa fingerprint sensor on the MacBook Pro (13-inch, M1, 2020), mirroring the J313 node. The J293 platform device tree puts the sensor on the same controller (0x235108000), at the same 8 MHz clock, with power on AP GPIO pin 108 (argument 0x101) and the interrupt on pin 104, which is left out because the driver polls. The calibration blob is per device, so the name is J293-specific. Co-Authored-By: Claude Opus 5.5 (1M context) --- arch/arm64/boot/dts/apple/t8103-j293.dts | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103-j293.dts b/arch/arm64/boot/dts/apple/t8103-j293.dts index 2c63d056522c6..bf6d9f00bdc66 100644 --- a/arch/arm64/boot/dts/apple/t8103-j293.dts +++ b/arch/arm64/boot/dts/apple/t8103-j293.dts @@ -95,6 +95,32 @@ }; }; +/* + * Mesa fingerprint sensor on SPI2. The J293 platform device tree wires it the + * same way as J313: controller at 0x235108000, 8 MHz, power on pin 108 with + * argument 0x101, and an interrupt on pin 104. The interrupt is left out for + * the same reason as on J313; the driver polls. + * + * Do not add cs-gpios. The sensor needs the controller's native chip select, + * otherwise the SPI core skips the hardware CS timing. + */ +&spi2 { + status = "okay"; +}; + +&mesa { + status = "okay"; + enable-gpios = <&pinctrl_ap 108 GPIO_ACTIVE_HIGH>; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; + /* + * Per-device factory data written at manufacture. It is not + * redistributable and has to be extracted from the machine's own macOS + * install, so the name only fixes where the driver looks for it. + */ + firmware-name = "apple/mesacal-j293.bin"; +}; + &i2c1 { speaker_left_rear: codec@31 { compatible = "ti,tas5770l", "ti,tas2770"; From a5f2067295534d02326b5dd8b23af68f4f56c35f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:48:30 +1100 Subject: [PATCH 0336/2680] pmdomain: apple: pmc: limit the DCS rail as a cooling device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Once the PMP is told that the GPU is on, it votes the DRAM controller (DCS) up with GPU load. On the J700 that is worth up to 2.4x in GPU throughput, but it also lets the GPU and the memory system draw up to 2.4 W more, and nothing on the host can hold that back: the CPU cooling devices do not touch it. Register the PMC as a cooling device that limits the DCS rail. At cooling state N, only the AP's own agent stays enabled on the DCS rail, and it votes what the other enabled agents ask for, capped N states below the top. A delayed work re-reads their votes every 20 ms while a limit is in place, so the rail still follows demand, including down to 0 when the GPU idles. The capped vote is in place before the others stop being honoured and is withdrawn only once they are honoured again, so the rail never dips on a change of state. Enables that the static and dynamic agents request while a limit is in place are recorded and take effect once it is lifted. With a power model in the device tree, the cooling device is a power actor for the power allocator governor: it requests the power of the state the other agents ask for and converts a granted power into the highest DCS state that fits. [ryan: adapted to 7.2.10 — 7.2 thermal/of (commit 3570cb58e317 ("thermal/of: Pass cdev_id and introduce devm registration helper")) takes a cooling device index instead of a node in devm_thermal_of_cooling_device_register(); the PMC registers on its own node with index 0, as the old call did] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmc.c | 289 ++++++++++++++++++++++++++++++++++- 1 file changed, 283 insertions(+), 6 deletions(-) diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index fac0b1dfc02b6..8510750d125aa 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -17,6 +17,11 @@ * and models each dynamic agent as a power domain that sits between the * device and its PMGR power state, which gives the same ordering. * + * Under thermal pressure the driver can also limit the DRAM controller + * (DCS) rail: it stops honouring the other agents on that rail and votes + * on their behalf through the AP's own agent, capped. This bounds the heat + * of the PMP's DCS vote, which follows GPU load. + * * Copyright The Asahi Linux Contributors */ @@ -32,9 +37,15 @@ #include #include #include +#include +#include #define APPLE_PMC_MAX_RAILS 8 #define APPLE_PMC_MAX_AGENTS 32 +#define APPLE_PMC_MAX_STATES 16 + +/* How often a DCS limit re-reads the votes it stands in for. */ +#define APPLE_PMC_DCS_POLL_MS 20 /** * struct apple_pmc_hw - per-SoC layout of the PMC voter interface @@ -47,6 +58,10 @@ * agent's interface enables and clamped to each rail's highest state * @rail_enable: offset of the per-rail interface enable words in the * "rails" region + * @dcs_rail: rail of the DRAM controller (DCS) + * @dcs_max: highest DCS performance state + * @ap_agent: voter agent of the application processor, which a DCS limit + * votes through */ struct apple_pmc_hw { u32 agent_stride; @@ -54,6 +69,9 @@ struct apple_pmc_hw { u32 rails; u32 agent_state; u32 rail_enable; + u32 dcs_rail; + u32 dcs_max; + u32 ap_agent; }; struct apple_pmc { @@ -61,12 +79,22 @@ struct apple_pmc { const struct apple_pmc_hw *hw; void __iomem *voters; void __iomem *rails; - /* Serializes read-modify-write cycles on enable and floor words. */ + /* + * Serializes read-modify-write cycles on enable and floor words, and + * protects @enable and @dcs_state. + */ raw_spinlock_t lock; u32 saved_enable[APPLE_PMC_MAX_RAILS]; + /* Agents enabled on each rail, before any DCS limit is applied. */ + u32 enable[APPLE_PMC_MAX_RAILS]; u32 static_agents; struct list_head domains; struct dentry *debugfs; + /* Cooling state of the DCS limit: 0 is none, dcs_max holds DCS at 0. */ + unsigned long dcs_state; + /* Power cost of each DCS state in mW, if a model is described. */ + u32 dcs_power[APPLE_PMC_MAX_STATES]; + struct delayed_work dcs_work; }; struct apple_pmc_domain { @@ -88,14 +116,18 @@ static u32 apple_pmc_enable_read(struct apple_pmc *pmc, unsigned int rail) return readl(pmc->rails + pmc->hw->rail_enable + 4 * rail); } -/* Caller holds pmc->lock. Every write is read back, as macOS does. */ -static int apple_pmc_enable_update(struct apple_pmc *pmc, unsigned int rail, - u32 set, u32 clear) +/* + * Caller holds pmc->lock. While the DCS rail is limited, only the AP's agent + * is enabled on it. Every write is read back, as macOS does. + */ +static int apple_pmc_enable_write(struct apple_pmc *pmc, unsigned int rail) { void __iomem *reg = pmc->rails + pmc->hw->rail_enable + 4 * rail; u32 old = readl(reg); - u32 val = (old | set) & ~clear; + u32 val = pmc->enable[rail]; + if (pmc->dcs_state && rail == pmc->hw->dcs_rail) + val = BIT(pmc->hw->ap_agent); if (val == old) return 0; @@ -110,6 +142,15 @@ static int apple_pmc_enable_update(struct apple_pmc *pmc, unsigned int rail, return 0; } +/* Caller holds pmc->lock. */ +static int apple_pmc_enable_update(struct apple_pmc *pmc, unsigned int rail, + u32 set, u32 clear) +{ + pmc->enable[rail] = (pmc->enable[rail] | set) & ~clear; + + return apple_pmc_enable_write(pmc, rail); +} + static int apple_pmc_agent_enable(struct apple_pmc *pmc, unsigned int agent, bool enable) { @@ -226,6 +267,226 @@ int apple_pmc_floor_update(struct device *dev, unsigned int agent, u32 mask, } EXPORT_SYMBOL_GPL(apple_pmc_floor_update); +/* + * Caller holds pmc->lock. The highest DCS state that the agents enabled on + * the DCS rail ask for, the AP's own agent excepted. + */ +static unsigned int apple_pmc_dcs_demand(struct apple_pmc *pmc) +{ + const struct apple_pmc_hw *hw = pmc->hw; + unsigned long agents = pmc->enable[hw->dcs_rail] & ~BIT(hw->ap_agent); + unsigned int agent, demand = 0; + u32 floor; + + for_each_set_bit(agent, &agents, hw->agents) { + floor = readl(pmc->voters + agent * hw->agent_stride); + demand = max(demand, (floor & APPLE_PMC_RAIL_MASK(hw->dcs_rail)) >> + APPLE_PMC_RAIL_SHIFT(hw->dcs_rail)); + } + + return min(demand, hw->dcs_max); +} + +/* + * Caller holds pmc->lock. Vote through the AP's agent for what the other + * agents ask for, capped by the limit; vote nothing without a limit. + */ +static void apple_pmc_dcs_vote(struct apple_pmc *pmc) +{ + const struct apple_pmc_hw *hw = pmc->hw; + void __iomem *reg = pmc->voters + hw->ap_agent * hw->agent_stride; + u32 mask = APPLE_PMC_RAIL_MASK(hw->dcs_rail); + unsigned int state = 0; + u32 old, val; + + if (pmc->dcs_state) + state = min_t(unsigned int, apple_pmc_dcs_demand(pmc), + hw->dcs_max - pmc->dcs_state); + + old = readl(reg); + val = (old & ~mask) | (state << APPLE_PMC_RAIL_SHIFT(hw->dcs_rail)); + if (val != old) + writel(val, reg); +} + +static void apple_pmc_dcs_work(struct work_struct *work) +{ + struct apple_pmc *pmc = container_of(to_delayed_work(work), + struct apple_pmc, dcs_work); + unsigned long flags; + bool limited; + + raw_spin_lock_irqsave(&pmc->lock, flags); + apple_pmc_dcs_vote(pmc); + limited = pmc->dcs_state; + raw_spin_unlock_irqrestore(&pmc->lock, flags); + + if (limited) + queue_delayed_work(system_freezable_power_efficient_wq, + &pmc->dcs_work, + msecs_to_jiffies(APPLE_PMC_DCS_POLL_MS)); +} + +static int apple_pmc_dcs_get_max_state(struct thermal_cooling_device *cdev, + unsigned long *state) +{ + struct apple_pmc *pmc = cdev->devdata; + + *state = pmc->hw->dcs_max; + return 0; +} + +static int apple_pmc_dcs_get_cur_state(struct thermal_cooling_device *cdev, + unsigned long *state) +{ + struct apple_pmc *pmc = cdev->devdata; + + *state = READ_ONCE(pmc->dcs_state); + return 0; +} + +static int apple_pmc_dcs_set_cur_state(struct thermal_cooling_device *cdev, + unsigned long state) +{ + struct apple_pmc *pmc = cdev->devdata; + unsigned int rail = pmc->hw->dcs_rail; + unsigned long flags; + int ret = 0; + + if (state > pmc->hw->dcs_max) + return -EINVAL; + + raw_spin_lock_irqsave(&pmc->lock, flags); + if (state != pmc->dcs_state) { + /* + * Vote the capped demand before the other agents stop being + * honoured, and drop that vote only once they are honoured + * again, so that the rail never dips below either. + */ + WRITE_ONCE(pmc->dcs_state, state); + if (state) { + apple_pmc_dcs_vote(pmc); + ret = apple_pmc_enable_write(pmc, rail); + } else { + ret = apple_pmc_enable_write(pmc, rail); + apple_pmc_dcs_vote(pmc); + } + } + raw_spin_unlock_irqrestore(&pmc->lock, flags); + + if (state) + queue_delayed_work(system_freezable_power_efficient_wq, + &pmc->dcs_work, + msecs_to_jiffies(APPLE_PMC_DCS_POLL_MS)); + else + cancel_delayed_work(&pmc->dcs_work); + + return ret; +} + +static int apple_pmc_dcs_get_requested_power(struct thermal_cooling_device *cdev, + u32 *power) +{ + struct apple_pmc *pmc = cdev->devdata; + unsigned long flags; + unsigned int demand; + + raw_spin_lock_irqsave(&pmc->lock, flags); + demand = apple_pmc_dcs_demand(pmc); + raw_spin_unlock_irqrestore(&pmc->lock, flags); + + *power = pmc->dcs_power[demand]; + return 0; +} + +static int apple_pmc_dcs_state2power(struct thermal_cooling_device *cdev, + unsigned long state, u32 *power) +{ + struct apple_pmc *pmc = cdev->devdata; + + if (state > pmc->hw->dcs_max) + return -EINVAL; + + *power = pmc->dcs_power[pmc->hw->dcs_max - state]; + return 0; +} + +static int apple_pmc_dcs_power2state(struct thermal_cooling_device *cdev, + u32 power, unsigned long *state) +{ + struct apple_pmc *pmc = cdev->devdata; + unsigned long s; + + for (s = 0; s < pmc->hw->dcs_max; s++) + if (pmc->dcs_power[pmc->hw->dcs_max - s] <= power) + break; + + *state = s; + return 0; +} + +static const struct thermal_cooling_device_ops apple_pmc_dcs_ops = { + .get_max_state = apple_pmc_dcs_get_max_state, + .get_cur_state = apple_pmc_dcs_get_cur_state, + .set_cur_state = apple_pmc_dcs_set_cur_state, +}; + +static const struct thermal_cooling_device_ops apple_pmc_dcs_power_ops = { + .get_max_state = apple_pmc_dcs_get_max_state, + .get_cur_state = apple_pmc_dcs_get_cur_state, + .set_cur_state = apple_pmc_dcs_set_cur_state, + .get_requested_power = apple_pmc_dcs_get_requested_power, + .state2power = apple_pmc_dcs_state2power, + .power2state = apple_pmc_dcs_power2state, +}; + +/* + * Register the DCS limit as a cooling device. With a power model it is a + * power actor that the power allocator governor can drive. + */ +static int apple_pmc_dcs_init(struct apple_pmc *pmc) +{ + const struct thermal_cooling_device_ops *ops = &apple_pmc_dcs_ops; + const struct apple_pmc_hw *hw = pmc->hw; + struct device *dev = pmc->dev; + struct device_node *np = dev->of_node; + struct thermal_cooling_device *cdev; + u32 uw[APPLE_PMC_MAX_STATES]; + unsigned int i; + int count, ret; + + if (!IS_ENABLED(CONFIG_THERMAL) || + !of_property_present(np, "#cooling-cells")) + return 0; + + count = of_property_count_u32_elems(np, "apple,dcs-power-microwatt"); + if (count > 0) { + if (count != hw->dcs_max + 1) + return dev_err_probe(dev, -EINVAL, + "need %u DCS power values\n", + hw->dcs_max + 1); + ret = of_property_read_u32_array(np, "apple,dcs-power-microwatt", + uw, count); + if (ret) + return ret; + for (i = 0; i < count; i++) { + pmc->dcs_power[i] = uw[i] / 1000; + if (i && pmc->dcs_power[i] < pmc->dcs_power[i - 1]) + return dev_err_probe(dev, -EINVAL, + "DCS power must not decrease\n"); + } + ops = &apple_pmc_dcs_power_ops; + } + + cdev = devm_thermal_of_cooling_device_register(dev, 0, "pmc-dcs", pmc, + ops); + if (IS_ERR(cdev)) + return dev_err_probe(dev, PTR_ERR(cdev), + "failed to register the DCS limit\n"); + + return 0; +} + static int apple_pmc_domain_power_on(struct generic_pm_domain *genpd) { struct apple_pmc_domain *dom = genpd_to_apple_pmc_domain(genpd); @@ -342,6 +603,8 @@ static int apple_pmc_status_show(struct seq_file *s, void *unused) for (i = 0; i < hw->rails; i++) seq_printf(s, "rail%u enable: %#010x (saved %#010x)\n", i, apple_pmc_enable_read(pmc, i), pmc->saved_enable[i]); + seq_printf(s, "dcs limit: state %lu of %u\n", READ_ONCE(pmc->dcs_state), + hw->dcs_max); return 0; } @@ -361,10 +624,14 @@ static void apple_pmc_release(void *data) apple_pmc_domain_free(dom); } + /* The cooling device is gone by now; drop any limit it left. */ + cancel_delayed_work_sync(&pmc->dcs_work); raw_spin_lock_irqsave(&pmc->lock, flags); for (rail = 0; rail < pmc->hw->rails; rail++) writel(pmc->saved_enable[rail], pmc->rails + pmc->hw->rail_enable + 4 * rail); + pmc->dcs_state = 0; + apple_pmc_dcs_vote(pmc); raw_spin_unlock_irqrestore(&pmc->lock, flags); } @@ -388,8 +655,10 @@ static int apple_pmc_probe(struct platform_device *pdev) INIT_LIST_HEAD(&pmc->domains); if (pmc->hw->rails > APPLE_PMC_MAX_RAILS || - pmc->hw->agents > APPLE_PMC_MAX_AGENTS) + pmc->hw->agents > APPLE_PMC_MAX_AGENTS || + pmc->hw->dcs_max >= APPLE_PMC_MAX_STATES) return -EINVAL; + INIT_DELAYED_WORK(&pmc->dcs_work, apple_pmc_dcs_work); pmc->voters = devm_platform_ioremap_resource_byname(pdev, "voters"); if (IS_ERR(pmc->voters)) @@ -404,6 +673,7 @@ static int apple_pmc_probe(struct platform_device *pdev) if (pmc->saved_enable[rail] == ~0U) return dev_err_probe(dev, -ENODEV, "PMC is not responding\n"); + pmc->enable[rail] = pmc->saved_enable[rail]; } count = of_property_count_u32_elems(np, "apple,static-voters"); @@ -448,6 +718,10 @@ static int apple_pmc_probe(struct platform_device *pdev) } } + ret = apple_pmc_dcs_init(pmc); + if (ret) + return ret; + pmc->debugfs = debugfs_create_dir(dev_name(dev), apple_pmc_debugfs_root); debugfs_create_file("status", 0400, pmc->debugfs, pmc, &apple_pmc_status_fops); @@ -465,6 +739,9 @@ static const struct apple_pmc_hw apple_pmc_hw_t8140 = { .rails = 4, .agent_state = 0x0, .rail_enable = 0x2000, + .dcs_rail = 0, + .dcs_max = 4, + .ap_agent = 3, }; static const struct of_device_id apple_pmc_of_match[] = { From 7de391a58fa9254a4e1c3edbf259c857a814474e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:48:31 +1100 Subject: [PATCH 0337/2680] pmdomain: apple: build the PMC driver by default The J700 device tree relies on the PMC driver for its DCS thermal limit and for letting the host performance controller's floors through, so a kernel built without it runs hotter under GPU load. Build it by default on ARCH_APPLE, as the T8140 memory cache controller driver is; it binds to nothing on SoCs without a PMC node. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/Kconfig | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/pmdomain/apple/Kconfig b/drivers/pmdomain/apple/Kconfig index 7145a073429aa..2494e7aece065 100644 --- a/drivers/pmdomain/apple/Kconfig +++ b/drivers/pmdomain/apple/Kconfig @@ -25,11 +25,13 @@ config APPLE_PMC depends on OF depends on PM select PM_GENERIC_DOMAINS + default ARCH_APPLE help The PMC in some Apple SoCs arbitrates the performance states of shared rails, such as the DRAM controller and the SoC fabric, between voter agents. This driver enables the static agents and provides power domains that enable a dynamic agent, such as the - GPU, while its device is powered on. + GPU, while its device is powered on. It can also limit the DRAM + controller rail as a thermal cooling device. endif From 36c1ae5ac43a72cf489e27f9d65e8259100f9bfb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:48:58 +1100 Subject: [PATCH 0338/2680] soc: apple: clpc: do not count offline CPUs as busy An offline CPU accrues no idle time while the wall clock keeps running, so its idle delta is zero and the sampler counted it as 100% busy. With any CPU of a policy offline, that policy then requested its top DCS state on every sample for as long as the CPU stayed down. Skip offline CPUs, and take a fresh baseline for a CPU when it comes back instead of charging it with the whole time it was away. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/clpc.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index 32b88c7c968ba..02a9df787a642 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -183,12 +183,26 @@ static void apple_clpc_sample_cluster(struct apple_clpc *clpc, unsigned int cpu, busy = 0, q, rail; for_each_cpu(cpu, cl->cpus) { - u64 wall, idle = get_cpu_idle_time(cpu, &wall, 0); - u64 dw = wall - clpc->wall[cpu], di = idle - clpc->idle[cpu]; + u64 wall, idle, dw, di; + bool valid; + + /* + * An offline CPU accrues no idle time while the wall clock runs + * on, so it would look fully busy. Skip it, and take a fresh + * baseline once it is back. + */ + if (!cpu_online(cpu)) { + clpc->wall[cpu] = 0; + continue; + } + idle = get_cpu_idle_time(cpu, &wall, 0); + dw = wall - clpc->wall[cpu]; + di = idle - clpc->idle[cpu]; + valid = clpc->wall[cpu]; clpc->wall[cpu] = wall; clpc->idle[cpu] = idle; - if (dw && di <= dw) + if (valid && dw && di <= dw) busy = max_t(unsigned int, busy, div64_u64((dw - di) * 100, dw)); } From 991edac1d5a0798263a65a6e1984cedd78af7730 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:49:13 +1100 Subject: [PATCH 0339/2680] soc: apple: clpc: stop waking idle CPUs and release the floor in sleep The sampler ran every 4 ms on a high-priority, non-freezable workqueue with a normal timer. That woke an idle CPU 250 times a second even when no vote was held, kept running through system sleep, and, once tasks are frozen and sampling stops, would leave the agent's floor at whatever the last sample before suspend asked for, typically a busy one, for the whole sleep. Sample on a freezable, power-efficient workqueue. While a vote is held, keep the normal timer so the vote still drops on time after demand stops; with no vote held, use a deferrable timer that waits for a CPU that is awake anyway. Before system sleep, stop sampling and give the floor back; resume sampling from fresh baselines afterwards. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/clpc.c | 94 +++++++++++++++++++++++++++++++++++----- 1 file changed, 84 insertions(+), 10 deletions(-) diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index 02a9df787a642..2d66b98c3a9be 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -109,7 +109,10 @@ struct apple_clpc { unsigned int agent; struct regmap *gpu_regmap; u32 gpu_offset; + /* Samples while a vote is held; idle_work defers to busy CPUs. */ struct delayed_work work; + struct delayed_work idle_work; + bool stopped; struct apple_clpc_cluster clusters[APPLE_CLPC_MAX_CLUSTERS]; unsigned int nr_clusters; cpumask_var_t covered; @@ -262,10 +265,23 @@ static u32 apple_clpc_floor(struct apple_clpc *clpc, const unsigned int *state) return floor; } -static void apple_clpc_work(struct work_struct *work) +static void apple_clpc_queue(struct apple_clpc *clpc, bool idle) +{ + if (READ_ONCE(clpc->stopped)) + return; + + /* + * A held vote has to be sampled on time so that it drops once demand + * stops; with no vote held, sampling can wait for a CPU that wakes up + * anyway rather than waking an idle one every period. + */ + queue_delayed_work(system_freezable_power_efficient_wq, + idle ? &clpc->idle_work : &clpc->work, + msecs_to_jiffies(sample_ms)); +} + +static void apple_clpc_sample(struct apple_clpc *clpc) { - struct apple_clpc *clpc = container_of(to_delayed_work(work), - struct apple_clpc, work); unsigned int demand[APPLE_CLPC_RAILS] = {}; unsigned int state[APPLE_CLPC_RAILS]; unsigned long now = jiffies; @@ -305,8 +321,19 @@ static void apple_clpc_work(struct work_struct *work) apple_clpc_floor(clpc, state))) clpc->update_errors++; - queue_delayed_work(system_highpri_wq, &clpc->work, - msecs_to_jiffies(sample_ms)); + apple_clpc_queue(clpc, !apple_clpc_floor(clpc, state)); +} + +static void apple_clpc_work(struct work_struct *work) +{ + apple_clpc_sample(container_of(to_delayed_work(work), struct apple_clpc, + work)); +} + +static void apple_clpc_idle_work(struct work_struct *work) +{ + apple_clpc_sample(container_of(to_delayed_work(work), struct apple_clpc, + idle_work)); } static int apple_clpc_status_show(struct seq_file *s, void *unused) @@ -422,14 +449,33 @@ static void apple_clpc_free_clusters(struct apple_clpc *clpc) free_cpumask_var(clpc->covered); } +/* Stop sampling and give the floor back, before suspend and on removal. */ +static void apple_clpc_halt(struct apple_clpc *clpc) +{ + unsigned int rail, i; + + /* + * Each pass queues one of the two works unless stopped. A pass that + * began before the flag was set can still queue the other work once, + * which the second round cancels. + */ + WRITE_ONCE(clpc->stopped, true); + for (i = 0; i < 2; i++) { + cancel_delayed_work_sync(&clpc->work); + cancel_delayed_work_sync(&clpc->idle_work); + } + apple_pmc_floor_update(clpc->pmc, clpc->agent, clpc->mask, + clpc->saved_floor); + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) + clpc->vote[rail].state = 0; +} + static void apple_clpc_stop(void *data) { struct apple_clpc *clpc = data; - cancel_delayed_work_sync(&clpc->work); + apple_clpc_halt(clpc); debugfs_remove_recursive(clpc->debugfs); - apple_pmc_floor_update(clpc->pmc, clpc->agent, clpc->mask, - clpc->saved_floor); apple_clpc_free_clusters(clpc); } @@ -504,6 +550,7 @@ static int apple_clpc_probe(struct platform_device *pdev) apple_clpc_find_clusters(clpc); clpc->next_scan = jiffies + HZ; INIT_DELAYED_WORK(&clpc->work, apple_clpc_work); + INIT_DEFERRABLE_WORK(&clpc->idle_work, apple_clpc_idle_work); clpc->debugfs = debugfs_create_dir(dev_name(dev), NULL); debugfs_create_file("status", 0400, clpc->debugfs, clpc, @@ -515,11 +562,37 @@ static int apple_clpc_probe(struct platform_device *pdev) if (ret) return ret; - queue_delayed_work(system_highpri_wq, &clpc->work, - msecs_to_jiffies(sample_ms)); + platform_set_drvdata(pdev, clpc); + apple_clpc_queue(clpc, true); return 0; } +/* + * Tasks are frozen before devices suspend, and with them the sampling, so + * the floor would otherwise stay at whatever the last busy sample asked for + * through the whole sleep. + */ +static int apple_clpc_suspend(struct device *dev) +{ + apple_clpc_halt(dev_get_drvdata(dev)); + return 0; +} + +static int apple_clpc_resume(struct device *dev) +{ + struct apple_clpc *clpc = dev_get_drvdata(dev); + unsigned int cpu; + + for_each_possible_cpu(cpu) + clpc->wall[cpu] = 0; + WRITE_ONCE(clpc->stopped, false); + apple_clpc_queue(clpc, true); + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(apple_clpc_pm_ops, apple_clpc_suspend, + apple_clpc_resume); + /* T8140: DCS states 0-4 on rail 0, fabric states 0-3 on rail 1. */ static const struct apple_clpc_hw apple_clpc_hw_t8140 = { .rail = { [APPLE_CLPC_DCS] = 0, [APPLE_CLPC_SOC] = 1 }, @@ -537,6 +610,7 @@ static struct platform_driver apple_clpc_driver = { .driver = { .name = "apple-clpc", .of_match_table = apple_clpc_of_match, + .pm = pm_sleep_ptr(&apple_clpc_pm_ops), }, }; module_platform_driver(apple_clpc_driver); From 5dc276d9a299c1dcc45e68ae2995c06dd270c957 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 22:24:11 +1100 Subject: [PATCH 0340/2680] soc: apple: build the CLPC floors by default Without a floor from the host, the J700's DRAM controller stays at its lowest state for CPU work: nothing else votes it up unless the GPU is busy. A single thread copying a 256 MiB buffer moves 6.88 GB/s (sd 0.02, three runs of 12) without the CLPC floors and 28.4 GB/s (sd 0.1-0.3) with them, the floor following the copy to DCS 4. Build the floors by default on ARCH_APPLE; the driver binds only to a host performance controller node under a PMC. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 2e5aaa61ba051..62f2f33e6b56c 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -139,6 +139,7 @@ config APPLE_CLPC depends on ARCH_APPLE || COMPILE_TEST depends on APPLE_PMC && CPU_FREQ select MFD_SYSCON + default ARCH_APPLE help Raises the DRAM controller and SoC fabric performance floors of the host's PMC voter agent while CPU clusters, and optionally the GPU, From e8edf365db4ddc39ccfa8d22bc8fbeeb4063c265 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:06:36 +1100 Subject: [PATCH 0341/2680] pmdomain: apple: pmp-report: keep entry domains bound Each PMP report entry is a power-domain provider, but unlike the report driver itself the entry driver still exposes the sysfs bind and unbind attributes. Unbinding an entry whose domain has consumers (the display, camera or video decoder, or the GPU on t8112) cannot remove the domain: pm_genpd_remove() fails while devices are attached, so the entry keeps an inert domain whose power callbacks return -ENODEV, and the consumers can no longer runtime-resume until the next boot. Before commit commit 4f3e0ed14a9c ("pmdomain: apple: drain PMP report users before teardown") added a .remove callback, the same unbind left the domain pointing at freed memory. Suppress the bind attributes, as the report driver already does. The .remove callback stays for OF depopulation of the entries. Fixes: e5f8c27073d6 ("pmdomain: apple: Add PMP reporting interface") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmp-report.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index b193313d98748..285815327fb11 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -439,6 +439,8 @@ static struct platform_driver apple_pmp_report_entry_driver = { .driver = { .name = "apple-pmp-report-entry", .of_match_table = apple_pmp_report_entry_of_match, + /* Power-domain providers are not intended for manual unbinding. */ + .suppress_bind_attrs = true, }, }; From 35738358610a189a7c0de209e759a4723fc29345 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 20:53:24 +1100 Subject: [PATCH 0342/2680] soc: apple: dockchannel: Let received data wake the system The DockChannel FIFO carries the MTP coprocessor's keyboard and trackpad reports on the M2 and later laptops. The coprocessor keeps running across suspend-to-idle, so a key press does reach the FIFO, but it cannot end the sleep: suspend_device_irqs() disables the receive interrupt like any other device interrupt, and enable_irq_wake() on it fails with -ENXIO because the DockChannel interrupt controller implements neither irq_set_wake() nor IRQCHIP_SKIP_SET_WAKE. Nothing needs configuring for a FIFO interrupt to wake the system. The controller is chained to a single AIC interrupt, which the IRQ core never disables for system sleep, and the AIC keeps delivering it in suspend-to-idle. Mark the irqchip IRQCHIP_SKIP_SET_WAKE. Add dockchannel_init_wakeup() for clients whose remote side is a wakeup source. It makes the client device wakeup capable and enabled, and registers the receive interrupt as its wake interrupt, so that the PM core arms it across system sleep whenever wakeup is enabled in sysfs. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/dockchannel.c | 33 +++++++++++++++++++++++++++ include/linux/soc/apple/dockchannel.h | 1 + 2 files changed, 34 insertions(+) diff --git a/drivers/soc/apple/dockchannel.c b/drivers/soc/apple/dockchannel.c index 3557b1e4e0558..3072284a911a7 100644 --- a/drivers/soc/apple/dockchannel.c +++ b/drivers/soc/apple/dockchannel.c @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include #include @@ -263,6 +265,30 @@ struct dockchannel *dockchannel_init(struct platform_device *pdev) } EXPORT_SYMBOL(dockchannel_init); +/** + * dockchannel_init_wakeup() - Let data from the remote side wake the system + * @dockchannel: FIFO returned by dockchannel_init() + * + * Make the FIFO's device a wakeup source, enabled by default, with the + * receive interrupt as its wake interrupt. While wakeup is enabled for the + * device, the PM core keeps that interrupt armed across system sleep, and + * data that arrives then wakes the system. The data itself is read after + * resume, as usual. + * + * Return: 0 on success, or a negative error code. + */ +int dockchannel_init_wakeup(struct dockchannel *dockchannel) +{ + int ret; + + ret = devm_device_init_wakeup(dockchannel->dev); + if (ret) + return ret; + + return devm_pm_set_wake_irq(dockchannel->dev, dockchannel->rx_irq); +} +EXPORT_SYMBOL(dockchannel_init_wakeup); + /* Dockchannel IRQchip */ @@ -308,11 +334,18 @@ static void dockchannel_irq_unmask(struct irq_data *data) writel_relaxed(val | BIT(hwirq), dcc->irq_base + IRQ_MASK); } +/* + * The controller hangs off a single chained AIC interrupt, which the IRQ core + * never disables for system sleep and which keeps being delivered in + * suspend-to-idle. A FIFO interrupt left unmasked can therefore end a + * suspend-to-idle without configuring anything, here or in the AIC. + */ static const struct irq_chip dockchannel_irqchip = { .name = "dockchannel-irqc", .irq_ack = dockchannel_irq_ack, .irq_mask = dockchannel_irq_mask, .irq_unmask = dockchannel_irq_unmask, + .flags = IRQCHIP_SKIP_SET_WAKE, }; static int dockchannel_irq_domain_map(struct irq_domain *d, unsigned int virq, diff --git a/include/linux/soc/apple/dockchannel.h b/include/linux/soc/apple/dockchannel.h index 0b7093935ddf4..759b5dda2be1f 100644 --- a/include/linux/soc/apple/dockchannel.h +++ b/include/linux/soc/apple/dockchannel.h @@ -15,6 +15,7 @@ struct dockchannel; struct dockchannel *dockchannel_init(struct platform_device *pdev); +int dockchannel_init_wakeup(struct dockchannel *dockchannel); int dockchannel_send(struct dockchannel *dockchannel, const void *buf, size_t count); int dockchannel_recv(struct dockchannel *dockchannel, void *buf, size_t count); From c65e78dd561e5127aa277130cd43059e6b849f70 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 15:42:14 +1100 Subject: [PATCH 0343/2680] pmdomain: apple: Undo a power state transition that times out apple_pmgr_ps_set() writes the new target state and polls the actual state for 100 us. When the poll times out it logs and returns the error, but leaves the new target requested, and a power-on also sets AUTO_ENABLE. genpd then keeps the domain in its previous state: off after a failed ->power_on(), on after a failed ->power_off(). From then on the register and genpd disagree, and genpd does not write the domain again until its next transition. On T8140 (J700), rebinding the ISP after an unbind fails because the resident firmware does not boot a second time, and the CPU core domain stops short of ACTIVE: [ 760.990199] apple-pmgr-pwrstate 300700000.power-management:power-controller@4008: PS isp_cpu_core0: Failed to reach power state 0xf (requested 0xf00000f, now: 0xf000b4f) genpd and the ISP driver unwind as if the domain were off, but its register keeps requesting ACTIVE with auto-PM enabled, so the core sits clock-gated instead of power-gated until the next boot. A failed power-off is the opposite case: genpd still treats the domain as on and will not power it on again for its next consumer, while the register asks for it to be power-gated. Follow a failed transition with the opposite one: power the domain back off after a failed power-on, and back on after a failed power-off. Both are the normal sequences, so a power-off still sets the force-disable and force-reset bits first, and the domain is down before genpd powers off its parents. genpd ignores power-on errors in the noirq and syscore phases of system sleep (genpd_sync_power_on()) and records the domain as on there, so powering it back off would leave a domain off that genpd and its consumers believe is on. Slow domains do time out there: the gfx domain misses the poll window on resume from s2idle on M1 Pro (t6000), and did on T8140 in genpd's noirq resume before it became always-on, then reaches ACTIVE, and the GPU keeps working. The PM core runs those phases between the suspend_late and resume_early callbacks of every device. Add these callbacks to mark the window and keep requesting the domain on inside it, as before. The probe-time power-on of an always-on domain also keeps the current behaviour, as genpd never powers those domains. Transitions that succeed are unchanged. Fixes: 6df9d38f9146 ("soc: apple: Add driver for Apple PMGR power state controls") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmgr-pwrstate.c | 68 ++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index 42511ed7ef89c..c5699abae3d93 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -48,6 +48,7 @@ struct apple_pmgr_ps { bool force_disable; bool force_reset; bool externally_clocked; + bool system_sleep; }; #define genpd_to_apple_pmgr_ps(_genpd) container_of(_genpd, struct apple_pmgr_ps, genpd) @@ -143,14 +144,62 @@ static bool apple_pmgr_ps_is_active(struct apple_pmgr_ps *ps) reg & APPLE_PMGR_AUTO_ENABLE)); } +/* + * genpd keeps a domain off when ->power_on() fails and on when ->power_off() + * fails. Follow a failed transition with the opposite one, so the domain is + * not left requesting a state genpd does not know it is in. + * + * The exception is a failed power-on in the noirq and syscore phases of a + * system sleep transition: genpd ignores the error there and records the + * domain as on. The PM core runs those phases after the suspend_late callback + * of every device and before any resume_early callback, so keep requesting + * the domain on between this device's two callbacks. + */ static int apple_pmgr_ps_power_on(struct generic_pm_domain *genpd) { - return apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_ACTIVE, true); + struct apple_pmgr_ps *ps = genpd_to_apple_pmgr_ps(genpd); + int ret; + + ret = apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_ACTIVE, true); + if (ret && !ps->system_sleep) + apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_PWRGATE, false); + + return ret; } static int apple_pmgr_ps_power_off(struct generic_pm_domain *genpd) { - return apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_PWRGATE, false); + int ret; + + ret = apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_PWRGATE, false); + if (ret) + apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_ACTIVE, true); + + return ret; +} + +static void apple_pmgr_ps_set_system_sleep(struct device *dev, bool sleep) +{ + struct apple_pmgr_ps *ps = dev_get_drvdata(dev); + unsigned long flags; + + spin_lock_irqsave(&ps->genpd.slock, flags); + ps->system_sleep = sleep; + spin_unlock_irqrestore(&ps->genpd.slock, flags); +} + +static int apple_pmgr_ps_suspend_late(struct device *dev) +{ + apple_pmgr_ps_set_system_sleep(dev, true); + + return 0; +} + +static int apple_pmgr_ps_resume_early(struct device *dev) +{ + apple_pmgr_ps_set_system_sleep(dev, false); + + return 0; } static int apple_pmgr_reset_assert(struct reset_controller_dev *rcdev, unsigned long id) @@ -253,6 +302,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) ps->dev = dev; ps->regmap = regmap; + platform_set_drvdata(pdev, ps); ret = of_property_read_string(node, "label", &name); if (ret < 0) { @@ -290,8 +340,13 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) ps->genpd.flags |= GENPD_FLAG_ALWAYS_ON; if (!active) { dev_warn(dev, "always-on domain %s is not on at boot\n", name); - /* Turn it on so pm_genpd_init does not fail */ - active = apple_pmgr_ps_power_on(&ps->genpd) == 0; + /* + * Turn it on so pm_genpd_init does not fail. genpd never + * powers an always-on domain, so keep requesting it on + * even if it does not get there in time. + */ + active = apple_pmgr_ps_set(&ps->genpd, APPLE_PMGR_PS_ACTIVE, + true) == 0; } } else if (active) { ps->genpd.flags |= GENPD_FLAG_DEFER_OFF | GENPD_FLAG_ACTIVE_WAKEUP; @@ -366,11 +421,16 @@ static const struct of_device_id apple_pmgr_ps_of_match[] = { MODULE_DEVICE_TABLE(of, apple_pmgr_ps_of_match); +static const struct dev_pm_ops apple_pmgr_ps_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(apple_pmgr_ps_suspend_late, apple_pmgr_ps_resume_early) +}; + static struct platform_driver apple_pmgr_ps_driver = { .probe = apple_pmgr_ps_probe, .driver = { .name = "apple-pmgr-pwrstate", .of_match_table = apple_pmgr_ps_of_match, + .pm = pm_sleep_ptr(&apple_pmgr_ps_pm_ops), }, }; From c84b6c82bc18ea51d72a533a06f4b7beb691144f Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:04:21 -0700 Subject: [PATCH 0344/2680] kexec: interlock load and execution with retained device DMA Add counted blockers serialized with the kexec lock. Refuse both load APIs and normal or crash execution while a device owns memory that cannot be quiesced; keep image unloading available. Cover balanced/nested ownership, lock contention and safe execution refusal with KUnit. --- include/linux/kexec.h | 5 ++ kernel/Kconfig.kexec | 11 +++ kernel/Makefile | 1 + kernel/crash_core.c | 2 +- kernel/kexec.c | 5 ++ kernel/kexec_block_test.c | 136 ++++++++++++++++++++++++++++++++++++++ kernel/kexec_core.c | 48 ++++++++++++++ kernel/kexec_file.c | 5 ++ kernel/kexec_internal.h | 1 + 9 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 kernel/kexec_block_test.c diff --git a/include/linux/kexec.h b/include/linux/kexec.h index 8a22bc9b8c6c8..111e79cfafe18 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -449,6 +449,9 @@ extern struct kimage *kexec_crash_image; bool kexec_load_permitted(int kexec_image_type); +int kexec_block(void); +void kexec_unblock(void); + #ifndef kexec_flush_icache_page #define kexec_flush_icache_page(page) #endif @@ -540,6 +543,8 @@ static inline void __crash_kexec(struct pt_regs *regs) { } static inline void crash_kexec(struct pt_regs *regs) { } static inline int kexec_should_crash(struct task_struct *p) { return 0; } static inline int kexec_crash_loaded(void) { return 0; } +static inline int kexec_block(void) { return 0; } +static inline void kexec_unblock(void) { } static inline void *kimage_map_segment(struct kimage *image, int idx) { return NULL; } static inline void kimage_unmap_segment(void *buffer) { } diff --git a/kernel/Kconfig.kexec b/kernel/Kconfig.kexec index 15632358bcf71..37c347d8fc8b4 100644 --- a/kernel/Kconfig.kexec +++ b/kernel/Kconfig.kexec @@ -11,6 +11,17 @@ config VMCORE_INFO config KEXEC_CORE bool +config KEXEC_BLOCK_KUNIT_TEST + bool "Unit tests for retained-memory kexec blockers" if !KUNIT_ALL_TESTS + depends on KEXEC_CORE && KUNIT=y + default KUNIT_ALL_TESTS + help + Test balanced and nested retained-memory blockers and acquisition + failure when the kexec lock is held, plus execution refusal with no + image loaded. These tests never load or execute a kernel, or modify + a loaded normal or crash image. + Tests are skipped if a real device already blocks kexec. + config KEXEC_ELF bool diff --git a/kernel/Makefile b/kernel/Makefile index 1e1a31673577d..81eb70048fe7f 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -79,6 +79,7 @@ obj-$(CONFIG_BSD_PROCESS_ACCT) += acct.o obj-$(CONFIG_VMCORE_INFO) += vmcore_info.o elfcorehdr.o obj-$(CONFIG_CRASH_RESERVE) += crash_reserve.o obj-$(CONFIG_KEXEC_CORE) += kexec_core.o +obj-$(CONFIG_KEXEC_BLOCK_KUNIT_TEST) += kexec_block_test.o obj-$(CONFIG_CRASH_DUMP) += crash_core.o obj-$(CONFIG_CRASH_DM_CRYPT) += crash_dump_dm_crypt.o obj-$(CONFIG_CRASH_DUMP_KUNIT_TEST) += crash_core_test.o diff --git a/kernel/crash_core.c b/kernel/crash_core.c index 4f21fc3b108b8..8a4c56df5efb4 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -135,7 +135,7 @@ void __noclone __crash_kexec(struct pt_regs *regs) * sufficient. But since I reuse the memory... */ if (kexec_trylock()) { - if (kexec_crash_image) { + if (kexec_crash_image && !kexec_blocked()) { struct pt_regs fixed_regs; crash_setup_regs(&fixed_regs, regs); diff --git a/kernel/kexec.c b/kernel/kexec.c index 90756dc6339b0..45a639752ebe2 100644 --- a/kernel/kexec.c +++ b/kernel/kexec.c @@ -102,6 +102,11 @@ static int do_kexec_load(unsigned long entry, unsigned long nr_segments, */ if (!kexec_trylock()) return -EBUSY; + if (nr_segments && kexec_blocked()) { + pr_warn("kexec load refused: a device retains memory until hardware reset\n"); + ret = -EBUSY; + goto out_unlock; + } #ifdef CONFIG_CRASH_DUMP if (flags & KEXEC_ON_CRASH) { diff --git a/kernel/kexec_block_test.c b/kernel/kexec_block_test.c new file mode 100644 index 0000000000000..476faf03f7bca --- /dev/null +++ b/kernel/kexec_block_test.c @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include + +#include "kexec_internal.h" + +struct kexec_block_test_state { + unsigned int acquisitions; +}; + +static void kexec_block_test_cleanup(void *data) +{ + struct kexec_block_test_state *state = data; + + while (state->acquisitions) { + kexec_unblock(); + state->acquisitions--; + } +} + +static int kexec_block_test_init(struct kunit *test) +{ + struct kexec_block_test_state *state; + bool blocked; + + /* Never disturb a real device's retained-memory acquisition. */ + if (!kexec_trylock()) + kunit_skip(test, "a kexec operation is in progress"); + blocked = kexec_blocked(); + kexec_unlock(); + if (blocked) + kunit_skip(test, "a device already blocks kexec"); + state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + if (!state) + return -ENOMEM; + test->priv = state; + return kunit_add_action_or_reset(test, kexec_block_test_cleanup, state); +} + +static void kexec_test_acquire(struct kunit *test) +{ + struct kexec_block_test_state *state = test->priv; + int ret = kexec_block(); + + KUNIT_ASSERT_EQ(test, ret, 0); + state->acquisitions++; +} + +static void kexec_test_release(struct kunit *test) +{ + struct kexec_block_test_state *state = test->priv; + + KUNIT_ASSERT_GT(test, state->acquisitions, 0); + kexec_unblock(); + state->acquisitions--; +} + +static void kexec_test_expect_blocked(struct kunit *test, bool expected) +{ + bool blocked; + + KUNIT_ASSERT_TRUE(test, kexec_trylock()); + blocked = kexec_blocked(); + kexec_unlock(); + KUNIT_EXPECT_EQ(test, blocked, expected); +} + +static void kexec_block_balanced_test(struct kunit *test) +{ + kexec_test_acquire(test); + kexec_test_expect_blocked(test, true); + kexec_test_release(test); + kexec_test_expect_blocked(test, false); +} + +static void kexec_block_nested_test(struct kunit *test) +{ + kexec_test_acquire(test); + kexec_test_acquire(test); + kexec_test_release(test); + kexec_test_expect_blocked(test, true); + kexec_test_release(test); + kexec_test_expect_blocked(test, false); +} + +static void kexec_block_busy_test(struct kunit *test) +{ + int ret; + + kexec_test_acquire(test); + KUNIT_ASSERT_TRUE(test, kexec_trylock()); + ret = kexec_block(); + kexec_unlock(); + KUNIT_EXPECT_EQ(test, ret, -EBUSY); + /* A failed acquisition must not leave an extra reference behind. */ + kexec_test_release(test); + kexec_test_expect_blocked(test, false); +} + +static void kexec_block_execution_test(struct kunit *test) +{ + bool image_loaded; + + kexec_test_acquire(test); + KUNIT_ASSERT_TRUE(test, kexec_trylock()); + image_loaded = !!kexec_image; + kexec_unlock(); + /* + * New loads are blocked during this test. Require an empty image so + * even a broken execution guard cannot transition to another kernel. + * The blocker must take precedence over the no-image -EINVAL result. + */ + if (image_loaded) + kunit_skip(test, "a normal kexec image is already loaded"); + KUNIT_EXPECT_EQ(test, kernel_kexec(), -EBUSY); + kexec_test_release(test); +} + +static struct kunit_case kexec_block_test_cases[] = { + KUNIT_CASE(kexec_block_balanced_test), + KUNIT_CASE(kexec_block_nested_test), + KUNIT_CASE(kexec_block_busy_test), + KUNIT_CASE(kexec_block_execution_test), + {} +}; + +static struct kunit_suite kexec_block_test_suite = { + .name = "kexec-block", + .init = kexec_block_test_init, + .test_cases = kexec_block_test_cases, +}; + +kunit_test_suite(kexec_block_test_suite); + +MODULE_LICENSE("GPL"); diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index dc770b9a6d053..2e7004d0550a3 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -51,6 +51,49 @@ atomic_t __kexec_lock = ATOMIC_INIT(0); +/* Acquisition is serialized with load and execution by __kexec_lock. */ +static atomic_t kexec_blockers = ATOMIC_INIT(0); + +/** + * kexec_block - prevent loading and executing kernels while memory is retained + * + * Acquire before publishing memory to hardware that cannot be quiesced for + * kexec. Existing normal and crash images remain loaded, but cannot execute. + * Acquisition is serialized with load and execution, including crash/NMI + * execution, and returns -EBUSY if one of those operations is in progress. + * Callers must leave hardware untouched on acquisition failure. A successful + * acquisition must be paired with kexec_unblock() only after DMA has stopped. + * + * Return: 0 on success, or -EBUSY if load or execution is in progress. + */ +int kexec_block(void) +{ + if (!kexec_trylock()) + return -EBUSY; + atomic_inc(&kexec_blockers); + kexec_unlock(); + return 0; +} +EXPORT_SYMBOL_GPL(kexec_block); + +/** + * kexec_unblock - release a successful kexec_block() acquisition + * + * Only call after the retained memory is no longer accessible by hardware. + */ +void kexec_unblock(void) +{ + /* Releasing after DMA stops cannot make any transition unsafe. */ + WARN_ON_ONCE(atomic_dec_if_positive(&kexec_blockers) < 0); +} +EXPORT_SYMBOL_GPL(kexec_unblock); + +/* All callers hold __kexec_lock, so checking and execution cannot race. */ +bool kexec_blocked(void) +{ + return atomic_read(&kexec_blockers) != 0; +} + /* Flag to indicate we are going to kexec a new kernel */ bool kexec_in_progress = false; @@ -1141,6 +1184,11 @@ int kernel_kexec(void) if (!kexec_trylock()) return -EBUSY; + if (kexec_blocked()) { + pr_warn("kexec refused: a device retains memory until hardware reset\n"); + kexec_unlock(); + return -EBUSY; + } if (!kexec_image) { error = -EINVAL; goto Unlock; diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index 2bfbb2d144e69..84a220e08b0cb 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -382,6 +382,11 @@ SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd, if (!kexec_trylock()) return -EBUSY; + if (!(flags & KEXEC_FILE_UNLOAD) && kexec_blocked()) { + pr_warn("kexec file load refused: a device retains memory until hardware reset\n"); + kexec_unlock(); + return -EBUSY; + } #ifdef CONFIG_CRASH_DUMP if (image_type == KEXEC_TYPE_CRASH) { diff --git a/kernel/kexec_internal.h b/kernel/kexec_internal.h index 228bb88c018bc..5e36092a2f8dc 100644 --- a/kernel/kexec_internal.h +++ b/kernel/kexec_internal.h @@ -21,6 +21,7 @@ int kimage_is_destination_range(struct kimage *image, * "simple" atomic variable that is acquired with a cmpxchg(). */ extern atomic_t __kexec_lock; +bool kexec_blocked(void); static inline bool kexec_trylock(void) { int old = 0; From 798b97e47a04dde403db1a2410fe10b968b04643 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:48:51 -0700 Subject: [PATCH 0345/2680] kexec: diagnose crash transitions refused by retained ownership Keep the conservative crash-image veto, including for preloaded images, but emit an emergency diagnostic rather than silently skipping the dump. The existing NMI-safe kexec lock remains the only admission synchronization. --- kernel/crash_core.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/crash_core.c b/kernel/crash_core.c index 8a4c56df5efb4..780e0447e7359 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -135,7 +135,9 @@ void __noclone __crash_kexec(struct pt_regs *regs) * sufficient. But since I reuse the memory... */ if (kexec_trylock()) { - if (kexec_crash_image && !kexec_blocked()) { + if (kexec_crash_image && kexec_blocked()) { + pr_emerg("crash kexec refused: a device retains memory until hardware reset\n"); + } else if (kexec_crash_image) { struct pt_regs fixed_regs; crash_setup_regs(&fixed_regs, regs); From c0459606e17fc6aa332932376d0cdd2fa5a54091 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:52:44 -0700 Subject: [PATCH 0346/2680] kexec: identify permanent builtin retained-memory interlocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use a builtin-only guard handle with an immutable boot-lifetime reason and balanced per-guard acquisitions. Drop unused module exports, retain lock-free releases and the existing atomic execution lock, and report the first interlock reason in each nonzero interval without a lifetime-sensitive registry. Permanent kexec and Neo sleep refusals return EOPNOTSUPP; transient admission contention remains EBUSY. [ryan: adapted to 7.2.10 — the drivers/pci/controller/pcie-apple-piodma-diag.c hunk is carried by the 080-radio topic, which creates that file] --- include/linux/kexec.h | 15 +++++++++++---- kernel/crash_core.c | 3 ++- kernel/kexec.c | 5 +++-- kernel/kexec_block_test.c | 13 ++++++++----- kernel/kexec_core.c | 35 ++++++++++++++++++++++++++++------- kernel/kexec_file.c | 5 +++-- kernel/kexec_internal.h | 1 + 7 files changed, 56 insertions(+), 21 deletions(-) diff --git a/include/linux/kexec.h b/include/linux/kexec.h index 111e79cfafe18..af41936297f7c 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -15,6 +15,7 @@ #if !defined(__ASSEMBLY__) +#include #include #include #include @@ -23,6 +24,12 @@ #include #include +/* Built-in users only: reason must remain immutable for the whole boot. */ +struct kexec_blocker { + atomic_t users; + const char *reason; +}; + extern note_buf_t __percpu *crash_notes; #ifdef CONFIG_CRASH_DUMP @@ -449,8 +456,8 @@ extern struct kimage *kexec_crash_image; bool kexec_load_permitted(int kexec_image_type); -int kexec_block(void); -void kexec_unblock(void); +int kexec_block(struct kexec_blocker *blocker); +void kexec_unblock(struct kexec_blocker *blocker); #ifndef kexec_flush_icache_page #define kexec_flush_icache_page(page) @@ -543,8 +550,8 @@ static inline void __crash_kexec(struct pt_regs *regs) { } static inline void crash_kexec(struct pt_regs *regs) { } static inline int kexec_should_crash(struct task_struct *p) { return 0; } static inline int kexec_crash_loaded(void) { return 0; } -static inline int kexec_block(void) { return 0; } -static inline void kexec_unblock(void) { } +static inline int kexec_block(struct kexec_blocker *blocker) { return 0; } +static inline void kexec_unblock(struct kexec_blocker *blocker) { } static inline void *kimage_map_segment(struct kimage *image, int idx) { return NULL; } static inline void kimage_unmap_segment(void *buffer) { } diff --git a/kernel/crash_core.c b/kernel/crash_core.c index 780e0447e7359..cb0cdacd65188 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -136,7 +136,8 @@ void __noclone __crash_kexec(struct pt_regs *regs) */ if (kexec_trylock()) { if (kexec_crash_image && kexec_blocked()) { - pr_emerg("crash kexec refused: a device retains memory until hardware reset\n"); + pr_emerg("crash kexec refused permanently; first interlock: %s\n", + kexec_block_reason()); } else if (kexec_crash_image) { struct pt_regs fixed_regs; diff --git a/kernel/kexec.c b/kernel/kexec.c index 45a639752ebe2..23f58d28ac459 100644 --- a/kernel/kexec.c +++ b/kernel/kexec.c @@ -103,8 +103,9 @@ static int do_kexec_load(unsigned long entry, unsigned long nr_segments, if (!kexec_trylock()) return -EBUSY; if (nr_segments && kexec_blocked()) { - pr_warn("kexec load refused: a device retains memory until hardware reset\n"); - ret = -EBUSY; + pr_warn("kexec load refused permanently; first interlock: %s\n", + kexec_block_reason()); + ret = -EOPNOTSUPP; goto out_unlock; } diff --git a/kernel/kexec_block_test.c b/kernel/kexec_block_test.c index 476faf03f7bca..21abb223e81ed 100644 --- a/kernel/kexec_block_test.c +++ b/kernel/kexec_block_test.c @@ -7,6 +7,7 @@ struct kexec_block_test_state { unsigned int acquisitions; + struct kexec_blocker blocker; }; static void kexec_block_test_cleanup(void *data) @@ -14,7 +15,7 @@ static void kexec_block_test_cleanup(void *data) struct kexec_block_test_state *state = data; while (state->acquisitions) { - kexec_unblock(); + kexec_unblock(&state->blocker); state->acquisitions--; } } @@ -34,6 +35,7 @@ static int kexec_block_test_init(struct kunit *test) state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); if (!state) return -ENOMEM; + state->blocker.reason = "KUnit retained-memory guard"; test->priv = state; return kunit_add_action_or_reset(test, kexec_block_test_cleanup, state); } @@ -41,7 +43,7 @@ static int kexec_block_test_init(struct kunit *test) static void kexec_test_acquire(struct kunit *test) { struct kexec_block_test_state *state = test->priv; - int ret = kexec_block(); + int ret = kexec_block(&state->blocker); KUNIT_ASSERT_EQ(test, ret, 0); state->acquisitions++; @@ -52,7 +54,7 @@ static void kexec_test_release(struct kunit *test) struct kexec_block_test_state *state = test->priv; KUNIT_ASSERT_GT(test, state->acquisitions, 0); - kexec_unblock(); + kexec_unblock(&state->blocker); state->acquisitions--; } @@ -86,11 +88,12 @@ static void kexec_block_nested_test(struct kunit *test) static void kexec_block_busy_test(struct kunit *test) { + struct kexec_block_test_state *state = test->priv; int ret; kexec_test_acquire(test); KUNIT_ASSERT_TRUE(test, kexec_trylock()); - ret = kexec_block(); + ret = kexec_block(&state->blocker); kexec_unlock(); KUNIT_EXPECT_EQ(test, ret, -EBUSY); /* A failed acquisition must not leave an extra reference behind. */ @@ -113,7 +116,7 @@ static void kexec_block_execution_test(struct kunit *test) */ if (image_loaded) kunit_skip(test, "a normal kexec image is already loaded"); - KUNIT_EXPECT_EQ(test, kernel_kexec(), -EBUSY); + KUNIT_EXPECT_EQ(test, kernel_kexec(), -EOPNOTSUPP); kexec_test_release(test); } diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 2e7004d0550a3..bf21701b473ce 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -53,10 +53,14 @@ atomic_t __kexec_lock = ATOMIC_INIT(0); /* Acquisition is serialized with load and execution by __kexec_lock. */ static atomic_t kexec_blockers = ATOMIC_INIT(0); +/* An immutable built-in reason remains valid after its guard is released. */ +static const char *kexec_first_reason; /** * kexec_block - prevent loading and executing kernels while memory is retained * + * @blocker: built-in guard with a boot-lifetime immutable reason + * * Acquire before publishing memory to hardware that cannot be quiesced for * kexec. Existing normal and crash images remain loaded, but cannot execute. * Acquisition is serialized with load and execution, including crash/NMI @@ -66,27 +70,33 @@ static atomic_t kexec_blockers = ATOMIC_INIT(0); * * Return: 0 on success, or -EBUSY if load or execution is in progress. */ -int kexec_block(void) +int kexec_block(struct kexec_blocker *blocker) { + if (!blocker || !blocker->reason || !*blocker->reason) + return -EINVAL; if (!kexec_trylock()) return -EBUSY; - atomic_inc(&kexec_blockers); + if (atomic_inc_return(&kexec_blockers) == 1) + kexec_first_reason = blocker->reason; + atomic_inc(&blocker->users); kexec_unlock(); return 0; } -EXPORT_SYMBOL_GPL(kexec_block); /** * kexec_unblock - release a successful kexec_block() acquisition * + * @blocker: guard from a successful kexec_block() acquisition + * * Only call after the retained memory is no longer accessible by hardware. */ -void kexec_unblock(void) +void kexec_unblock(struct kexec_blocker *blocker) { + if (WARN_ON_ONCE(atomic_dec_if_positive(&blocker->users) < 0)) + return; /* Releasing after DMA stops cannot make any transition unsafe. */ WARN_ON_ONCE(atomic_dec_if_positive(&kexec_blockers) < 0); } -EXPORT_SYMBOL_GPL(kexec_unblock); /* All callers hold __kexec_lock, so checking and execution cannot race. */ bool kexec_blocked(void) @@ -94,6 +104,16 @@ bool kexec_blocked(void) return atomic_read(&kexec_blockers) != 0; } +/* + * Called under __kexec_lock. With several guards, report the first reason in + * the current nonzero interval, even if that guard has since been released. + * The reason has boot lifetime; release remains lock-free and NMI-safe. + */ +const char *kexec_block_reason(void) +{ + return kexec_first_reason; +} + /* Flag to indicate we are going to kexec a new kernel */ bool kexec_in_progress = false; @@ -1185,9 +1205,10 @@ int kernel_kexec(void) if (!kexec_trylock()) return -EBUSY; if (kexec_blocked()) { - pr_warn("kexec refused: a device retains memory until hardware reset\n"); + pr_warn("kexec refused permanently; first interlock: %s\n", + kexec_block_reason()); kexec_unlock(); - return -EBUSY; + return -EOPNOTSUPP; } if (!kexec_image) { error = -EINVAL; diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index 84a220e08b0cb..80f2dfae02649 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -383,9 +383,10 @@ SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd, if (!kexec_trylock()) return -EBUSY; if (!(flags & KEXEC_FILE_UNLOAD) && kexec_blocked()) { - pr_warn("kexec file load refused: a device retains memory until hardware reset\n"); + pr_warn("kexec file load refused permanently; first interlock: %s\n", + kexec_block_reason()); kexec_unlock(); - return -EBUSY; + return -EOPNOTSUPP; } #ifdef CONFIG_CRASH_DUMP diff --git a/kernel/kexec_internal.h b/kernel/kexec_internal.h index 5e36092a2f8dc..140b8f814f32a 100644 --- a/kernel/kexec_internal.h +++ b/kernel/kexec_internal.h @@ -22,6 +22,7 @@ int kimage_is_destination_range(struct kimage *image, */ extern atomic_t __kexec_lock; bool kexec_blocked(void); +const char *kexec_block_reason(void); static inline bool kexec_trylock(void) { int old = 0; From e8867e4e3f828250f5d52d4e6c054fbb47ddf57e Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:57:34 -0700 Subject: [PATCH 0347/2680] kexec: test retained-memory load vetoes and unload exceptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exercise the actual legacy and file loader implementations with deliberately invalid inputs while a retained-memory guard is active, and unload only empty image slots. Share the crash selector with tests without installing fake global images or executing a transition. Keep these tests builtin-only and rate limit repeated sleep and normal-kexec refusal diagnostics; crash diagnostics remain emergency messages. [ryan: adapted to 7.2.10 — the drivers/pci/controller/pcie-apple-piodma-diag.c hunk is carried by the 080-radio topic, which creates that file] --- kernel/crash_core.c | 9 +++-- kernel/kexec.c | 8 ++-- kernel/kexec_block_test.c | 84 +++++++++++++++++++++++++++++++++++++-- kernel/kexec_core.c | 4 +- kernel/kexec_file.c | 17 +++++--- kernel/kexec_internal.h | 15 +++++++ 6 files changed, 119 insertions(+), 18 deletions(-) diff --git a/kernel/crash_core.c b/kernel/crash_core.c index cb0cdacd65188..103e69570e3f3 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -135,10 +135,11 @@ void __noclone __crash_kexec(struct pt_regs *regs) * sufficient. But since I reuse the memory... */ if (kexec_trylock()) { - if (kexec_crash_image && kexec_blocked()) { - pr_emerg("crash kexec refused permanently; first interlock: %s\n", - kexec_block_reason()); - } else if (kexec_crash_image) { + if (!kexec_crash_image_allowed(!!kexec_crash_image)) { + if (kexec_crash_image) + pr_emerg("crash kexec refused permanently; first interlock: %s\n", + kexec_block_reason()); + } else { struct pt_regs fixed_regs; crash_setup_regs(&fixed_regs, regs); diff --git a/kernel/kexec.c b/kernel/kexec.c index 23f58d28ac459..e8335aab1f69b 100644 --- a/kernel/kexec.c +++ b/kernel/kexec.c @@ -88,8 +88,8 @@ static int kimage_alloc_init(struct kimage **rimage, unsigned long entry, return ret; } -static int do_kexec_load(unsigned long entry, unsigned long nr_segments, - struct kexec_segment *segments, unsigned long flags) +int do_kexec_load(unsigned long entry, unsigned long nr_segments, + struct kexec_segment *segments, unsigned long flags) { struct kimage **dest_image, *image; unsigned long i; @@ -103,8 +103,8 @@ static int do_kexec_load(unsigned long entry, unsigned long nr_segments, if (!kexec_trylock()) return -EBUSY; if (nr_segments && kexec_blocked()) { - pr_warn("kexec load refused permanently; first interlock: %s\n", - kexec_block_reason()); + pr_warn_ratelimited("kexec load refused permanently; first interlock: %s\n", + kexec_block_reason()); ret = -EOPNOTSUPP; goto out_unlock; } diff --git a/kernel/kexec_block_test.c b/kernel/kexec_block_test.c index 21abb223e81ed..22ef87ee92f5c 100644 --- a/kernel/kexec_block_test.c +++ b/kernel/kexec_block_test.c @@ -1,7 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only #include #include -#include #include "kexec_internal.h" @@ -120,11 +119,92 @@ static void kexec_block_execution_test(struct kunit *test) kexec_test_release(test); } +static void kexec_test_require_empty_images(struct kunit *test) +{ + bool loaded; + + KUNIT_ASSERT_TRUE(test, kexec_trylock()); + loaded = kexec_image || kexec_crash_image; + kexec_unlock(); + /* The test guard prevents new loads; never unload a real image. */ + if (loaded) + kunit_skip(test, "a normal or crash image is already loaded"); +} + +#ifdef CONFIG_KEXEC +static void kexec_block_legacy_load_test(struct kunit *test) +{ + /* Bad alignment also makes a missing guard fail safely in validation. */ + struct kexec_segment segment = { .mem = 1, .memsz = PAGE_SIZE }; + + kexec_test_acquire(test); + kexec_test_require_empty_images(test); + KUNIT_EXPECT_EQ(test, do_kexec_load(0, 1, &segment, 0), -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, do_kexec_load(0, 1, &segment, KEXEC_ON_CRASH), + -EOPNOTSUPP); + /* Unload an empty slot through the actual loader while still blocked. */ + KUNIT_EXPECT_EQ(test, do_kexec_load(0, 0, NULL, 0), 0); + KUNIT_EXPECT_EQ(test, do_kexec_load(0, 0, NULL, KEXEC_ON_CRASH), 0); + kexec_test_release(test); +} +#endif + +#ifdef CONFIG_KEXEC_FILE +static void kexec_block_file_load_test(struct kunit *test) +{ + int ret; + + kexec_test_acquire(test); + kexec_test_require_empty_images(test); + /* Invalid fds cannot create an image even if the veto regresses. */ + ret = do_kexec_file_load(-1, -1, 0, NULL, 0); + if (ret == -EPERM) + kunit_skip(test, "file loads disabled by permissions or attempt limit"); + KUNIT_EXPECT_EQ(test, ret, -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, + do_kexec_file_load(-1, -1, 0, NULL, KEXEC_FILE_ON_CRASH), + -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, do_kexec_file_load(-1, -1, 0, NULL, KEXEC_FILE_UNLOAD), 0); + KUNIT_EXPECT_EQ(test, + do_kexec_file_load(-1, -1, 0, NULL, + KEXEC_FILE_UNLOAD | KEXEC_FILE_ON_CRASH), 0); + kexec_test_release(test); +} +#endif + +static void kexec_block_crash_policy_test(struct kunit *test) +{ + bool empty_allowed, loaded_allowed; + + kexec_test_acquire(test); + KUNIT_ASSERT_TRUE(test, kexec_trylock()); + /* Exercise the real crash selector without installing any fake image. */ + empty_allowed = kexec_crash_image_allowed(false); + loaded_allowed = kexec_crash_image_allowed(true); + kexec_unlock(); + KUNIT_EXPECT_FALSE(test, empty_allowed); + KUNIT_EXPECT_FALSE(test, loaded_allowed); + kexec_test_release(test); + KUNIT_ASSERT_TRUE(test, kexec_trylock()); + empty_allowed = kexec_crash_image_allowed(false); + loaded_allowed = kexec_crash_image_allowed(true); + kexec_unlock(); + KUNIT_EXPECT_FALSE(test, empty_allowed); + KUNIT_EXPECT_TRUE(test, loaded_allowed); +} + static struct kunit_case kexec_block_test_cases[] = { KUNIT_CASE(kexec_block_balanced_test), KUNIT_CASE(kexec_block_nested_test), KUNIT_CASE(kexec_block_busy_test), KUNIT_CASE(kexec_block_execution_test), +#ifdef CONFIG_KEXEC + KUNIT_CASE(kexec_block_legacy_load_test), +#endif +#ifdef CONFIG_KEXEC_FILE + KUNIT_CASE(kexec_block_file_load_test), +#endif + KUNIT_CASE(kexec_block_crash_policy_test), {} }; @@ -135,5 +215,3 @@ static struct kunit_suite kexec_block_test_suite = { }; kunit_test_suite(kexec_block_test_suite); - -MODULE_LICENSE("GPL"); diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index bf21701b473ce..0874986e83824 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1205,8 +1205,8 @@ int kernel_kexec(void) if (!kexec_trylock()) return -EBUSY; if (kexec_blocked()) { - pr_warn("kexec refused permanently; first interlock: %s\n", - kexec_block_reason()); + pr_warn_ratelimited("kexec refused permanently; first interlock: %s\n", + kexec_block_reason()); kexec_unlock(); return -EOPNOTSUPP; } diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index 80f2dfae02649..9bcc465252667 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -361,9 +361,8 @@ kimage_file_alloc_init(struct kimage **rimage, int kernel_fd, return ret; } -SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd, - unsigned long, cmdline_len, const char __user *, cmdline_ptr, - unsigned long, flags) +int do_kexec_file_load(int kernel_fd, int initrd_fd, unsigned long cmdline_len, + const char __user *cmdline_ptr, unsigned long flags) { int image_type = (flags & KEXEC_FILE_ON_CRASH) ? KEXEC_TYPE_CRASH : KEXEC_TYPE_DEFAULT; @@ -383,8 +382,8 @@ SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd, if (!kexec_trylock()) return -EBUSY; if (!(flags & KEXEC_FILE_UNLOAD) && kexec_blocked()) { - pr_warn("kexec file load refused permanently; first interlock: %s\n", - kexec_block_reason()); + pr_warn_ratelimited("kexec file load refused permanently; first interlock: %s\n", + kexec_block_reason()); kexec_unlock(); return -EOPNOTSUPP; } @@ -475,6 +474,14 @@ SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd, return ret; } +SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd, + unsigned long, cmdline_len, const char __user *, cmdline_ptr, + unsigned long, flags) +{ + return do_kexec_file_load(kernel_fd, initrd_fd, cmdline_len, + cmdline_ptr, flags); +} + static int locate_mem_hole_top_down(unsigned long start, unsigned long end, struct kexec_buf *kbuf) { diff --git a/kernel/kexec_internal.h b/kernel/kexec_internal.h index 140b8f814f32a..5d458f83af0ad 100644 --- a/kernel/kexec_internal.h +++ b/kernel/kexec_internal.h @@ -23,6 +23,21 @@ int kimage_is_destination_range(struct kimage *image, extern atomic_t __kexec_lock; bool kexec_blocked(void); const char *kexec_block_reason(void); +/* Caller holds the existing NMI-safe kexec lock; no image is dereferenced. */ +static inline bool kexec_crash_image_allowed(bool image_loaded) +{ + return image_loaded && !kexec_blocked(); +} + +#ifdef CONFIG_KEXEC +int do_kexec_load(unsigned long entry, unsigned long nr_segments, + struct kexec_segment *segments, unsigned long flags); +#endif +#ifdef CONFIG_KEXEC_FILE +int do_kexec_file_load(int kernel_fd, int initrd_fd, unsigned long cmdline_len, + const char __user *cmdline_ptr, unsigned long flags); +#endif + static inline bool kexec_trylock(void) { int old = 0; From 5e5b2ecbe3e37df7548dc5c2554784ace5f280c8 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:00:26 -0700 Subject: [PATCH 0348/2680] kexec: require a file table for load-veto regression tests Invalid descriptors fail safely only when the test task has a file table. Skip the file-loader case in an unusual kernel-thread context without one, so a regressed guard cannot enter fd lookup with a null files_struct. --- kernel/kexec_block_test.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernel/kexec_block_test.c b/kernel/kexec_block_test.c index 22ef87ee92f5c..23adbc89163d4 100644 --- a/kernel/kexec_block_test.c +++ b/kernel/kexec_block_test.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only #include #include +#include #include "kexec_internal.h" @@ -154,6 +155,8 @@ static void kexec_block_file_load_test(struct kunit *test) { int ret; + if (!current->files) + kunit_skip(test, "no file table for safe invalid-fd validation"); kexec_test_acquire(test); kexec_test_require_empty_images(test); /* Invalid fds cannot create an image even if the veto regresses. */ From b3cd873ea02e4bcc35fa983297cecae91160ee7a Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:42:37 -0700 Subject: [PATCH 0349/2680] kexec: keep crash interlock reads untraced and harden test admission Mark crash-visible interlock accessors notrace and force the selector inline. Skip regression cases when either load-attempt budget is finite, and compare crash selection with the blocker state under the same kexec lock rather than assuming no real guard appeared. The production interlock remains active only through the Neo supplier; these annotations do not change non-Neo transition admission. Budget checks and revised expectations compile only into the builtin KUnit suite. Round-2 S5. Non-Neo runtime behavior remains unchanged. --- kernel/kexec_block_test.c | 8 ++++++-- kernel/kexec_core.c | 20 ++++++++++++++++++-- kernel/kexec_internal.h | 9 ++++++--- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/kernel/kexec_block_test.c b/kernel/kexec_block_test.c index 23adbc89163d4..495e37612fe0c 100644 --- a/kernel/kexec_block_test.c +++ b/kernel/kexec_block_test.c @@ -25,6 +25,8 @@ static int kexec_block_test_init(struct kunit *test) struct kexec_block_test_state *state; bool blocked; + if (!kexec_test_load_limits_unlimited()) + kunit_skip(test, "a finite load-attempt budget is configured"); /* Never disturb a real device's retained-memory acquisition. */ if (!kexec_trylock()) kunit_skip(test, "a kexec operation is in progress"); @@ -177,7 +179,7 @@ static void kexec_block_file_load_test(struct kunit *test) static void kexec_block_crash_policy_test(struct kunit *test) { - bool empty_allowed, loaded_allowed; + bool empty_allowed, loaded_allowed, blocked; kexec_test_acquire(test); KUNIT_ASSERT_TRUE(test, kexec_trylock()); @@ -189,11 +191,13 @@ static void kexec_block_crash_policy_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, loaded_allowed); kexec_test_release(test); KUNIT_ASSERT_TRUE(test, kexec_trylock()); + /* A real guard may have appeared since this case's initial skip check. */ + blocked = kexec_blocked(); empty_allowed = kexec_crash_image_allowed(false); loaded_allowed = kexec_crash_image_allowed(true); kexec_unlock(); KUNIT_EXPECT_FALSE(test, empty_allowed); - KUNIT_EXPECT_TRUE(test, loaded_allowed); + KUNIT_EXPECT_EQ(test, loaded_allowed, !blocked); } static struct kunit_case kexec_block_test_cases[] = { diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 0874986e83824..934dbf225695e 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -99,7 +99,7 @@ void kexec_unblock(struct kexec_blocker *blocker) } /* All callers hold __kexec_lock, so checking and execution cannot race. */ -bool kexec_blocked(void) +bool notrace kexec_blocked(void) { return atomic_read(&kexec_blockers) != 0; } @@ -109,7 +109,7 @@ bool kexec_blocked(void) * the current nonzero interval, even if that guard has since been released. * The reason has boot lifetime; release remains lock-free and NMI-safe. */ -const char *kexec_block_reason(void) +const char *notrace kexec_block_reason(void) { return kexec_first_reason; } @@ -1093,6 +1093,22 @@ static struct kexec_load_limit load_limit_panic = { .limit = -1, }; +#ifdef CONFIG_KEXEC_BLOCK_KUNIT_TEST +bool kexec_test_load_limits_unlimited(void) +{ + bool unlimited; + + /* Never let a test consume a deliberately finite load-attempt budget. */ + mutex_lock(&load_limit_reboot.mutex); + unlimited = load_limit_reboot.limit == -1; + mutex_unlock(&load_limit_reboot.mutex); + mutex_lock(&load_limit_panic.mutex); + unlimited &= load_limit_panic.limit == -1; + mutex_unlock(&load_limit_panic.mutex); + return unlimited; +} +#endif + struct kimage *kexec_image; struct kimage *kexec_crash_image; static int kexec_load_disabled; diff --git a/kernel/kexec_internal.h b/kernel/kexec_internal.h index 5d458f83af0ad..a42b95f65a43f 100644 --- a/kernel/kexec_internal.h +++ b/kernel/kexec_internal.h @@ -21,10 +21,13 @@ int kimage_is_destination_range(struct kimage *image, * "simple" atomic variable that is acquired with a cmpxchg(). */ extern atomic_t __kexec_lock; -bool kexec_blocked(void); -const char *kexec_block_reason(void); +bool notrace kexec_blocked(void); +const char *notrace kexec_block_reason(void); +#ifdef CONFIG_KEXEC_BLOCK_KUNIT_TEST +bool kexec_test_load_limits_unlimited(void); +#endif /* Caller holds the existing NMI-safe kexec lock; no image is dereferenced. */ -static inline bool kexec_crash_image_allowed(bool image_loaded) +static __always_inline bool kexec_crash_image_allowed(bool image_loaded) { return image_loaded && !kexec_blocked(); } From 9e1e7bac03e2e8877df4975dce21879285282ee7 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 18:48:20 -0400 Subject: [PATCH 0350/2680] soc: apple: rtkit: Allow AP readiness before the IOP power acknowledgement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Support power method 2 when the MTP firmware owns reset, reject conflicting GPIO ownership, and announce AP readiness before the T8142 MTP power acknowledgment. Preserve the existing sequence for other coprocessors and host-controlled trackpads. [ryan: adapted to 7.2.10 — split out of e179042ff067 ("HID: dockchannel: support firmware-managed M5 trackpad power"), retitled for the RTKit part: only the RTKit and RTKit-helper hunks are carried in 010-soc, because the RTKit API belongs to this topic and later RTKit commits build on it; the drivers/hid/dockchannel-hid/dockchannel-hid.c hunk stays with 090-input] Signed-off-by: Acelogic --- drivers/soc/apple/rtkit-helper.c | 5 ++++- drivers/soc/apple/rtkit.c | 32 ++++++++++++++++++++++++++++---- include/linux/soc/apple/rtkit.h | 3 +++ 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c index 080d083ed9bd2..89e5642444367 100644 --- a/drivers/soc/apple/rtkit-helper.c +++ b/drivers/soc/apple/rtkit-helper.c @@ -113,7 +113,10 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) helper->asc_base + APPLE_ASC_CPU_CONTROL); /* Works for both wake and boot */ - ret = apple_rtkit_wake(helper->rtk); + if (of_device_is_compatible(dev->of_node, "apple,t8142-mtp")) + ret = apple_rtkit_wake_early_ap(helper->rtk); + else + ret = apple_rtkit_wake(helper->rtk); if (ret != 0) return dev_err_probe(dev, ret, "Failed to wake up coprocessor"); diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index ae9589dff3485..a68f845d94cc5 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -1025,7 +1025,7 @@ static int apple_rtkit_set_iop_power_state(struct apple_rtkit *rtk, return 0; } -int apple_rtkit_boot(struct apple_rtkit *rtk) +static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power) { int ret; @@ -1041,12 +1041,25 @@ int apple_rtkit_boot(struct apple_rtkit *rtk) if (rtk->boot_result) return rtk->boot_result; + /* Some helpers need AP readiness before acknowledging IOP power. */ + if (early_ap_power) { + ret = apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); + if (ret) + return ret; + } + dev_dbg(rtk->dev, "RTKit: waiting for IOP power state ACK\n"); ret = apple_rtkit_wait_for_completion(&rtk->iop_pwr_ack_completion); if (ret) return ret; - return apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); + return early_ap_power ? 0 : + apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); +} + +int apple_rtkit_boot(struct apple_rtkit *rtk) +{ + return apple_rtkit_boot_common(rtk, false); } EXPORT_SYMBOL_GPL(apple_rtkit_boot); @@ -1138,7 +1151,7 @@ int apple_rtkit_quiesce(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_quiesce); -int apple_rtkit_wake(struct apple_rtkit *rtk) +static int apple_rtkit_wake_common(struct apple_rtkit *rtk, bool early_ap_power) { u64 msg; int ret; @@ -1158,10 +1171,21 @@ int apple_rtkit_wake(struct apple_rtkit *rtk) if (ret) return ret; - return apple_rtkit_boot(rtk); + return apple_rtkit_boot_common(rtk, early_ap_power); +} + +int apple_rtkit_wake(struct apple_rtkit *rtk) +{ + return apple_rtkit_wake_common(rtk, false); } EXPORT_SYMBOL_GPL(apple_rtkit_wake); +int apple_rtkit_wake_early_ap(struct apple_rtkit *rtk) +{ + return apple_rtkit_wake_common(rtk, true); +} +EXPORT_SYMBOL_GPL(apple_rtkit_wake_early_ap); + void apple_rtkit_free(struct apple_rtkit *rtk) { apple_rtkit_stop_rx(rtk); diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 4eafd55c52a4b..e4b38a8f6e1ba 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -177,6 +177,9 @@ int apple_rtkit_quiesce(struct apple_rtkit *rtk); */ int apple_rtkit_wake(struct apple_rtkit *rtk); +/* Wake helpers which require AP readiness before their IOP power acknowledgment. */ +int apple_rtkit_wake_early_ap(struct apple_rtkit *rtk); + /* * Shutdown the co-processor */ From 36ed0a70a5a9df57dceb40b5545488a143368763 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:47:51 +1100 Subject: [PATCH 0351/2680] soc: apple: rtkit: identify failed boot handshake stages Report which bounded startup wait failed so endpoint discovery, host readiness and IOP acknowledgment failures can be distinguished without logging message payloads. Successful handshakes remain unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index a68f845d94cc5..6de523df0a0b1 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -1036,22 +1036,28 @@ static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power) dev_dbg(rtk->dev, "RTKit: waiting for boot to finish\n"); ret = apple_rtkit_wait_for_completion(&rtk->epmap_completion); - if (ret) + if (ret) { + dev_err(rtk->dev, "RTKit: endpoint discovery failed: %d\n", ret); return ret; + } if (rtk->boot_result) return rtk->boot_result; /* Some helpers need AP readiness before acknowledging IOP power. */ if (early_ap_power) { ret = apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); - if (ret) + if (ret) { + dev_err(rtk->dev, "RTKit: early AP readiness failed: %d\n", ret); return ret; + } } dev_dbg(rtk->dev, "RTKit: waiting for IOP power state ACK\n"); ret = apple_rtkit_wait_for_completion(&rtk->iop_pwr_ack_completion); - if (ret) + if (ret) { + dev_err(rtk->dev, "RTKit: IOP power acknowledgment failed: %d\n", ret); return ret; + } return early_ap_power ? 0 : apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); From c6790132f5ada05f9b3be7a8a2e4ebdd39d7c23c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 12 Oct 2026 16:39:47 -0400 Subject: [PATCH 0352/2680] soc: apple: rtkit-helper: adopt a running T8140 MTP session T8140 MTP firmware faults or times out when it is woken after a boot stage has taken it to QUIESCED or SLEEP, so a boot stage that drives the keyboard has to hand the session over running. When the node carries apple,rtkit-handoff = "mtp-live-v1" and the ASC CPU is still running, adopt the session with apple_rtkit_init_adopted() instead of setting RUN and waking the coprocessor. The boot stage reserves the buffers the firmware holds and maps them 1:1 through the DART (memory-region with iommu-addresses), so they stay valid; the RTKit core acknowledges the inherited system endpoints without a new handshake. An adopted session cannot be restarted, so remove() leaves it running and the transport is released with apple_rtkit_free_retaining_buffers(). Without the property, or with a stopped CPU, the cold-boot path is unchanged. Co-Authored-By: Claude Opus 5.5 --- drivers/soc/apple/rtkit-helper.c | 60 ++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c index 89e5642444367..3dd941241893b 100644 --- a/drivers/soc/apple/rtkit-helper.c +++ b/drivers/soc/apple/rtkit-helper.c @@ -12,13 +12,19 @@ #include #include #include +#include #define APPLE_ASC_CPU_CONTROL 0x44 #define APPLE_ASC_CPU_CONTROL_RUN BIT(4) +/* A running T8140 MTP session described by the boot stage that started it. */ +#define APPLE_RTKIT_HELPER_HANDOFF "mtp-live-v1" + struct apple_rtkit_helper { struct device *dev; struct apple_rtkit *rtk; + /* Inherited from the boot stage; it cannot be restarted. */ + bool adopted; void __iomem *asc_base; @@ -73,6 +79,39 @@ static const struct apple_rtkit_ops apple_rtkit_helper_ops = { .shmem_destroy = apple_rtkit_helper_shmem_destroy, }; +static void apple_rtkit_helper_release(void *rtk) +{ + apple_rtkit_free_retaining_buffers(rtk); +} + +/* + * T8140 MTP firmware cannot be restarted once it has been taken to QUIESCED + * or SLEEP, so a boot stage that used it hands the session over running. It + * reserves the buffers the firmware holds and maps them 1:1 through the + * DART. Only adopt a session it described whose CPU still runs. + */ +static bool apple_rtkit_helper_live(struct apple_rtkit_helper *helper) +{ + const char *handoff; + + if (of_property_read_string(helper->dev->of_node, "apple,rtkit-handoff", + &handoff)) + return false; + + if (strcmp(handoff, APPLE_RTKIT_HELPER_HANDOFF)) { + dev_warn(helper->dev, "unknown RTKit handoff \"%s\"\n", handoff); + return false; + } + + if (!(readl_relaxed(helper->asc_base + APPLE_ASC_CPU_CONTROL) & + APPLE_ASC_CPU_CONTROL_RUN)) { + dev_warn(helper->dev, "RTKit handoff without a running CPU\n"); + return false; + } + + return true; +} + static int apple_rtkit_helper_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -103,6 +142,23 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) "Failed to map SRAM region"); } + if (apple_rtkit_helper_live(helper)) { + helper->rtk = apple_rtkit_init_adopted(dev, helper, NULL, 0, + &apple_rtkit_helper_ops); + if (IS_ERR(helper->rtk)) + return dev_err_probe(dev, PTR_ERR(helper->rtk), + "Failed to adopt RTKit"); + + ret = devm_add_action_or_reset(dev, apple_rtkit_helper_release, + helper->rtk); + if (ret) + return ret; + + helper->adopted = true; + dev_info(dev, "adopted the running RTKit session\n"); + return 0; + } + helper->rtk = devm_apple_rtkit_init(dev, helper, NULL, 0, &apple_rtkit_helper_ops); if (IS_ERR(helper->rtk)) @@ -127,6 +183,10 @@ static void apple_rtkit_helper_remove(struct platform_device *pdev) { struct apple_rtkit_helper *helper = platform_get_drvdata(pdev); + /* An inherited session would not come back from QUIESCED. */ + if (helper->adopted) + return; + if (apple_rtkit_is_running(helper->rtk)) apple_rtkit_quiesce(helper->rtk); From a2f7b0ed8efff5d7d57895f6a1a2482e6769ca74 Mon Sep 17 00:00:00 2001 From: Bartosz Nitka Date: Mon, 12 Oct 2026 21:32:58 +0200 Subject: [PATCH 0353/2680] cpuidle: apple: retain a selectable WFI state for T8140 s2idle T8140 exposes only the index-0 WFI fallback to avoid the unsupported CPU power-down path. The generic s2idle selector skips index 0, so this prevents tick_freeze() and timekeeping suspension from being reached. Copy the safe WFI state over the T8140 power-down entry instead of reducing state_count. This preserves an eligible nonzero s2idle entry without using CYC_OVRD, while leaving other Apple SoCs unchanged. On MacBook Neo, three 20-second RTC sleeps accounted for approximately zero suspended time before this change and 20.198, 19.808 and 20.419 seconds afterward using the identical BOOTTIME-minus-MONOTONIC test. Signed-off-by: Bartosz Nitka --- drivers/cpuidle/cpuidle-apple.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/cpuidle/cpuidle-apple.c b/drivers/cpuidle/cpuidle-apple.c index 4d023c877318f..dde13bc173a0b 100644 --- a/drivers/cpuidle/cpuidle-apple.c +++ b/drivers/cpuidle/cpuidle-apple.c @@ -127,10 +127,12 @@ static int apple_cpuidle_probe(struct platform_device *pdev) /* * CPU PD writes CYC_OVRD, which is an undefined instruction on * T8140. Its WFI already lets the hardware power-gate idle - * secondary cores, so expose only the WFI state. + * secondary cores, so both entries must use WFI. Keep index 1: + * the generic s2idle selector skips fallback index 0, otherwise + * tick_freeze() and timekeeping suspension are never reached. */ if (of_machine_is_compatible("apple,t8140")) - apple_idle_driver.state_count = STATE_WFI + 1; + apple_idle_driver.states[STATE_PWRDOWN] = apple_idle_driver.states[STATE_WFI]; return cpuidle_register(&apple_idle_driver, NULL); } From 30f1874018875d67d80b357f6a1d4ee635c4aa89 Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Tue, 13 Oct 2026 02:54:20 +0200 Subject: [PATCH 0354/2680] pmdomain: apple: leave firmware-owned power domains untouched On MacBook Neo (J700) m1n1 hands the display coprocessor (DCP) over running, and its firmware owns the DISP_CPU power state: writing that register raises an SError. The domain is marked apple,always-on, but the driver still writes it at probe: it sets automatic gating when the domain is on, or forces it on when it is not. The kernel panicked with an asynchronous SError in apple_pmgr_ps_probe before any console. Add apple,firmware-owned: the driver only reads such a domain, keeps it on for its consumers (the boot framebuffer), and never writes its minimum state, power target, automatic gating or reset. Mark J700's disp_cpu with it. Co-Authored-By: Claude Opus 5.5 --- .../bindings/power/apple,pmgr-pwrstate.yaml | 7 ++++ drivers/pmdomain/apple/pmgr-pwrstate.c | 33 +++++++++++++++++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml index d131441214db1..d86c8b68d1b48 100644 --- a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml @@ -79,6 +79,13 @@ properties: Forces this power domain to always be powered up. type: boolean + apple,firmware-owned: + description: + The boot loader hands over the coprocessor in this domain running, and its + firmware owns the domain's power state; writing it faults. The OS keeps the + domain on and never changes its power target, automatic gating or reset. + type: boolean + apple,min-state: description: Specifies the minimum power state for auto-PM. diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index c5699abae3d93..47f9593ae1221 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -49,6 +49,7 @@ struct apple_pmgr_ps { bool force_reset; bool externally_clocked; bool system_sleep; + bool firmware_owned; }; #define genpd_to_apple_pmgr_ps(_genpd) container_of(_genpd, struct apple_pmgr_ps, genpd) @@ -160,6 +161,10 @@ static int apple_pmgr_ps_power_on(struct generic_pm_domain *genpd) struct apple_pmgr_ps *ps = genpd_to_apple_pmgr_ps(genpd); int ret; + /* Its coprocessor's firmware owns the state; Linux only holds a vote. */ + if (ps->firmware_owned) + return 0; + ret = apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_ACTIVE, true); if (ret && !ps->system_sleep) apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_PWRGATE, false); @@ -171,6 +176,9 @@ static int apple_pmgr_ps_power_off(struct generic_pm_domain *genpd) { int ret; + if (genpd_to_apple_pmgr_ps(genpd)->firmware_owned) + return 0; + ret = apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_PWRGATE, false); if (ret) apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_ACTIVE, true); @@ -207,6 +215,9 @@ static int apple_pmgr_reset_assert(struct reset_controller_dev *rcdev, unsigned struct apple_pmgr_ps *ps = rcdev_to_apple_pmgr_ps(rcdev); unsigned long flags; + if (ps->firmware_owned) + return -EPERM; + spin_lock_irqsave(&ps->genpd.slock, flags); if (ps->genpd.status == GENPD_STATE_OFF) @@ -230,6 +241,9 @@ static int apple_pmgr_reset_deassert(struct reset_controller_dev *rcdev, unsigne struct apple_pmgr_ps *ps = rcdev_to_apple_pmgr_ps(rcdev); unsigned long flags; + if (ps->firmware_owned) + return -EPERM; + spin_lock_irqsave(&ps->genpd.slock, flags); dev_dbg(ps->dev, "PS 0x%x: deassert reset\n", ps->offset); @@ -321,8 +335,15 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) ps->genpd.power_on = apple_pmgr_ps_power_on; ps->genpd.power_off = apple_pmgr_ps_power_off; + /* + * A domain whose coprocessor the boot loader hands over running (the DCP) is + * owned by that coprocessor's firmware: writing its state faults. Only read + * it, keep it on and never change its target or automatic gating. + */ + ps->firmware_owned = of_property_read_bool(node, "apple,firmware-owned"); + ret = of_property_read_u32(node, "apple,min-state", &ps->min_state); - if (ret == 0 && ps->min_state <= APPLE_PMGR_PS_ACTIVE) + if (ret == 0 && ps->min_state <= APPLE_PMGR_PS_ACTIVE && !ps->firmware_owned) regmap_update_bits(regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_PS_MIN, FIELD_PREP(APPLE_PMGR_PS_MIN, ps->min_state)); @@ -336,7 +357,13 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) ps->externally_clocked = true; active = apple_pmgr_ps_is_active(ps); - if (of_property_read_bool(node, "apple,always-on")) { + if (ps->firmware_owned) { + /* Consumers (the boot framebuffer) need it; never try to turn it on. */ + if (!active) + dev_warn(dev, "firmware-owned domain %s does not read as on\n", name); + ps->genpd.flags |= GENPD_FLAG_ALWAYS_ON; + active = true; + } else if (of_property_read_bool(node, "apple,always-on")) { ps->genpd.flags |= GENPD_FLAG_ALWAYS_ON; if (!active) { dev_warn(dev, "always-on domain %s is not on at boot\n", name); @@ -353,7 +380,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) } /* Turn on auto-PM if the domain is already on */ - if (active) + if (active && !ps->firmware_owned) regmap_update_bits(regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_AUTO_ENABLE, APPLE_PMGR_AUTO_ENABLE); From 18e5f1d31615dbd517bb3bd2eddcc95a25014bd3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 17:37:36 +1100 Subject: [PATCH 0355/2680] soc: apple: rtkit: Recognise opaque OSLog events 3 to 5 The OSLog endpoint sends message types 3, 4 and 5 that need no acknowledgment and carry no buffer request. They are currently reported as unknown messages with a warning on every occurrence. Log them at info level with their raw payload, without interpreting the payload as a buffer address or a firmware error. This change reached aurora-wip only as content of the integration merge commit 08e0d0a90be3 ("drm/apple: integrate reviewed fixed-host lifecycle and scoped boot fixes"), which no single commit carries; it is restated here as a commit of its own. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 6de523df0a0b1..3402df716c97f 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -608,6 +608,17 @@ static void apple_rtkit_oslog_rx(struct apple_rtkit *rtk, u64 msg) apple_rtkit_common_rx_get_buffer(rtk, &rtk->oslog_buffer, APPLE_RTKIT_EP_OSLOG, msg); break; + case 3: + case 4: + case 5: + /* + * OpenBSD's public RTKit consumer recognizes these opaque events + * without an ACK. Keep their payload visible without interpreting + * it as a buffer address or a decoded firmware error. + */ + dev_info(rtk->dev, "RTKit: OSLog event %u: %016llx (undecoded)\n", + type, msg); + break; default: dev_warn(rtk->dev, "RTKit: Unknown oslog message: %llx\n", msg); From d52d79ebc45a8c650542826da001d11ef827bca4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:40:58 +1100 Subject: [PATCH 0356/2680] cpuidle: apple: Bind to the T8140 cpuidle node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe the PMP DART, mailbox, SRAM and report windows from the J700 ADT, including the high DMA aperture. Add the QSPIMC controller, SPI NOR NVRAM partition, I2C2 pinmux and bus, and display crossbar. Describe and enable the external display DARTs and mailbox using existing Apple DART and mailbox bindings. Align the existing pinctrl and diagnostic PIODMA node names with their bindings without changing their labels or resources. Add a T8140 cpuidle node and match it in the existing Apple idle driver. Keep WFI as its only idle state and skip legacy device creation when DT describes the controller. Document the idle and crossbar interfaces. The PMP and QSPI consumers need their corresponding driver support. External DCP and Type-C display routes still need a validated T8140 external-display profile; this change does not wire that consumer. The register, interrupt, power-domain and partition descriptions are independently derived from the ADT and public bindings. The QSPI clock selection follows the existing public QSPIMC binding example. No private device-tree source was copied. [ryan: adapted to 7.2.10 — split out of commit f75a1041fca4 ("arm64: dts: apple: Wire up remaining supported J700 peripherals"), retitled for the cpuidle part: only the drivers/cpuidle/cpuidle-apple.c hunk (the apple,t8140-cpuidle match) is carried in 010-soc, after the other T8140 cpuidle commits; the T8140/J700 DTS hunks are carried by 000-devicetree] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/cpuidle/cpuidle-apple.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/cpuidle/cpuidle-apple.c b/drivers/cpuidle/cpuidle-apple.c index dde13bc173a0b..64b45dcbafb31 100644 --- a/drivers/cpuidle/cpuidle-apple.c +++ b/drivers/cpuidle/cpuidle-apple.c @@ -137,9 +137,15 @@ static int apple_cpuidle_probe(struct platform_device *pdev) return cpuidle_register(&apple_idle_driver, NULL); } +static const struct of_device_id apple_cpuidle_of_match[] = { + { .compatible = "apple,t8140-cpuidle" }, + { } +}; + static struct platform_driver apple_cpuidle_driver = { .driver = { .name = "cpuidle-apple", + .of_match_table = apple_cpuidle_of_match, }, .probe = apple_cpuidle_probe, }; @@ -147,12 +153,20 @@ static struct platform_driver apple_cpuidle_driver = { static int __init apple_cpuidle_init(void) { struct platform_device *pdev; + struct device_node *np; int ret; ret = platform_driver_register(&apple_cpuidle_driver); if (ret) return ret; + /* DT-described systems are populated by the OF platform code. */ + np = of_find_matching_node(NULL, apple_cpuidle_of_match); + if (np) { + of_node_put(np); + return 0; + } + if (!of_machine_is_compatible("apple,arm-platform")) return 0; From a4e437e5b39f7e931ac40b4c9031c6d81844ff2e Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:42:49 -0700 Subject: [PATCH 0357/2680] apple: give retained-host refusals accurate names and errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename no_user_reset to reset_sensitive because the immutable host flag also protects assignment and recovery. Return EOPNOTSUPP for permanent PCI sysfs policy and test that actual errno without unregistered callbacks. Describe sleep/kexec refusal as an active interlock during admission or retained ownership, so the temporary window is not mislabeled permanent. Round-2 S4. Changed refusals require Neo flags or an active Neo interlock; default-clear hosts keep original policy and transition admission. Non-Neo behavior remains unchanged. [ryan: adapted to 7.2.10 — split by topic: only the kernel/kexec*.c and kernel/crash_core.c message renames, for 010-soc, which carries the kexec interlock; the PCI, VFIO and piodma-diag hunks are in 080-radio] --- kernel/crash_core.c | 2 +- kernel/kexec.c | 2 +- kernel/kexec_core.c | 2 +- kernel/kexec_file.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/crash_core.c b/kernel/crash_core.c index 103e69570e3f3..82b5e3471713e 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -137,7 +137,7 @@ void __noclone __crash_kexec(struct pt_regs *regs) if (kexec_trylock()) { if (!kexec_crash_image_allowed(!!kexec_crash_image)) { if (kexec_crash_image) - pr_emerg("crash kexec refused permanently; first interlock: %s\n", + pr_emerg("crash kexec refused by active interlock; first reason: %s\n", kexec_block_reason()); } else { struct pt_regs fixed_regs; diff --git a/kernel/kexec.c b/kernel/kexec.c index e8335aab1f69b..34c9e8a79ba49 100644 --- a/kernel/kexec.c +++ b/kernel/kexec.c @@ -103,7 +103,7 @@ int do_kexec_load(unsigned long entry, unsigned long nr_segments, if (!kexec_trylock()) return -EBUSY; if (nr_segments && kexec_blocked()) { - pr_warn_ratelimited("kexec load refused permanently; first interlock: %s\n", + pr_warn_ratelimited("kexec load refused by active interlock; first reason: %s\n", kexec_block_reason()); ret = -EOPNOTSUPP; goto out_unlock; diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 934dbf225695e..b1fa971110a75 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1221,7 +1221,7 @@ int kernel_kexec(void) if (!kexec_trylock()) return -EBUSY; if (kexec_blocked()) { - pr_warn_ratelimited("kexec refused permanently; first interlock: %s\n", + pr_warn_ratelimited("kexec refused by active interlock; first reason: %s\n", kexec_block_reason()); kexec_unlock(); return -EOPNOTSUPP; diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index 9bcc465252667..b39fa9de095d3 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -382,7 +382,7 @@ int do_kexec_file_load(int kernel_fd, int initrd_fd, unsigned long cmdline_len, if (!kexec_trylock()) return -EBUSY; if (!(flags & KEXEC_FILE_UNLOAD) && kexec_blocked()) { - pr_warn_ratelimited("kexec file load refused permanently; first interlock: %s\n", + pr_warn_ratelimited("kexec file load refused by active interlock; first reason: %s\n", kexec_block_reason()); kexec_unlock(); return -EOPNOTSUPP; From 405ef46e0adb6fa6c7bc0f0c3be92c0c6c1cdff3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:25:03 +1000 Subject: [PATCH 0358/2680] dt-bindings: nvme: apple,nvme-ans: Add T8140 support Starting with T8132 (M4), the NVMMU registers of the ANS2 controller are no longer part of the NVMe register block but a region of their own, and T8140 (A18 Pro) uses the same layout. Add "apple,t8132-nvme-ans2" for this generation and "apple,t8140-nvme-ans2" with it as fallback, and a third "nvmmu" register region that these compatibles require and the older ones must not have. T8140 needs a third power domain, for the PCIe PHY of the link to the storage. Nothing else references that domain, and while it is off the link drops to L1 idle and the ANS firmware fails its first read with a link timeout. Allow "apcie_phy" as the name of the third domain and require three domains on T8140, as on T6000 and T6020. The ANS2 reset stays required, but on T8140 asserting it raises an SError, so it cannot be used to restart the co-processor there. Say so in its description. A bootloader can hand the ANS2 firmware over to the operating system still running, with the NVMe controller stopped, instead of shutting it down. Add the "apple,rtkit-handoff" string property with which it states this at boot. The property describes the state the bootloader left behind rather than the hardware, so only that bootloader adds it, at runtime; it must not appear in a board device tree source. It is only valid for T8140. (cherry picked from commit e1f783581eeb0e45d7f686afb270ede007b0b393) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/nvme/apple,nvme-ans.yaml | 104 ++++++++++++++---- 1 file changed, 81 insertions(+), 23 deletions(-) diff --git a/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml b/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml index 352251c801f5a..5449db3b25080 100644 --- a/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml +++ b/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml @@ -18,6 +18,10 @@ properties: - apple,t6020-nvme-ans2 - apple,t8122-nvme-ans2 - const: apple,t8103-nvme-ans2 + - const: apple,t8132-nvme-ans2 + - items: + - const: apple,t8140-nvme-ans2 + - const: apple,t8132-nvme-ans2 - items: - enum: # Do not add additional SoC to this list. @@ -27,34 +31,45 @@ properties: - const: apple,nvme-ans2 reg: + minItems: 2 items: - - description: NVMe and NVMMU registers + - description: NVMe controller registers, including the NVMMU + registers on SoCs without a separate NVMMU region - description: ANS2 co-processor control registers + - description: NVMMU registers, a separate region since T8132 reg-names: + minItems: 2 items: - const: nvme - const: ans + - const: nvmmu resets: maxItems: 1 + description: + Reset of the ANS2 co-processor. On T8140 asserting it raises an + SError, so it cannot be used to restart the co-processor there. power-domains: - # two domains for t8103, three for t6000 + # two domains for t8103, three for t6000, t6020 and t8140 minItems: 2 items: - description: power domain for the NVMe controller. - description: power domain for the first PCIe bus connecting the NVMe controller to the storage modules. - description: optional power domain for the second PCIe bus - connecting the NVMe controller to the storage modules. + connecting the NVMe controller to the storage modules (T6000, + T6020), or for the PCIe PHY of the link to the storage (T8140). power-domain-names: minItems: 2 items: - const: ans - const: apcie0 - - const: apcie1 + - enum: + - apcie1 + - apcie_phy mboxes: maxItems: 1 @@ -70,25 +85,68 @@ properties: The SART address filter is documented in iommu/apple,sart.yaml. -if: - properties: - compatible: - contains: - enum: - - apple,t6000-nvme-ans2 - - apple,t6020-nvme-ans2 -then: - properties: - power-domains: - minItems: 3 - power-domain-names: - minItems: 3 -else: - properties: - power-domains: - maxItems: 2 - power-domain-names: - maxItems: 2 + apple,rtkit-handoff: + $ref: /schemas/types.yaml#/definitions/string + const: nvme-disabled-v1 + description: | + Added at boot by a bootloader that ran the ANS2 firmware and hands it + over still running instead of shutting it down. It describes the + state that bootloader left behind, not the hardware, and must not be + written into a board device tree source. Its presence states that the + firmware keeps a negotiated RTKit session, that all NVMe controller + DMA has stopped (CC.EN and CSTS.RDY clear), that the SART entries the + bootloader programmed are kept, and that every buffer shared with the + firmware is reserved in the memory map without no-map. + +allOf: + - if: + properties: + compatible: + not: + contains: + const: apple,t8140-nvme-ans2 + then: + properties: + apple,rtkit-handoff: false + + - if: + properties: + compatible: + contains: + enum: + - apple,t6000-nvme-ans2 + - apple,t6020-nvme-ans2 + - apple,t8140-nvme-ans2 + then: + properties: + power-domains: + minItems: 3 + power-domain-names: + minItems: 3 + else: + properties: + power-domains: + maxItems: 2 + power-domain-names: + maxItems: 2 + + - if: + properties: + compatible: + contains: + const: apple,t8132-nvme-ans2 + then: + properties: + reg: + minItems: 3 + reg-names: + minItems: 3 + else: + properties: + reg: + maxItems: 2 + reg-names: + maxItems: 2 required: - compatible From a47d72830a0543bbaaf0b8c2f5baf248a479b45c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:25:11 +1000 Subject: [PATCH 0359/2680] nvme-apple: Support the post-M4 register layout Starting with T8132 (M4), the ANS2 NVMMU registers are no longer part of the NVMe register block, and the I/O queues are set up differently. T8140 (A18 Pro) uses the same layout. Add hardware data for it: - Map the NVMMU from its own "nvmmu" register region. - After Create SQ, also program the I/O CQ and SQ base addresses into the registers at 0x1208 and 0x1200. Without them the firmware crashes and its crash log reports null I/O SQ and CQ addresses. - The register at 0x1210, which takes queue entry counts on older SoCs, takes zero-based sizes here, like AQA. Programming 64 for the 64-entry queue makes the firmware treat it as 65 entries and reject CQ head 63 once the host wraps through the queue. - The queue and NVMMU TCB base registers are pairs of 32-bit registers; write them low word first with lo_hi_writeq(), as m1n1 and U-Boot do. - Do not send Delete SQ/CQ. The firmware rejects both with BAD_CMD, and the controller disable tears the I/O queues down. - Do not send Set Features (Number of Queues). These controllers provide exactly one I/O queue pair, and m1n1 brings them up without that command; the driver keeps sending it on the older SoCs. On these SoCs, read LSQ_CTRL and the queue size register back after programming them and fail the reset if either did not take. Older SoCs keep their existing sequence, whose registers have never been read back. Only T8140 has been tested here. The T8132 entry follows m1n1, which programs both generations the same way, and has not been run. (cherry picked from commit 3028c4021c61be6d4a135c2c2ad4e056baf81ea2) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 136 +++++++++++++++++++++++++++++++------- 1 file changed, 113 insertions(+), 23 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index c932a38056fc1..97f6e7a602c87 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -42,6 +42,8 @@ #define APPLE_ANS_ACQ_DB 0x1004 #define APPLE_ANS_IOCQ_DB 0x100c +#define APPLE_ANS_IOSQ_REGISTER 0x1200 +#define APPLE_ANS_IOCQ_REGISTER 0x1208 #define APPLE_ANS_MAX_PEND_CMDS_CTRL 0x1210 #define APPLE_ANS_BOOT_STATUS 0x1300 @@ -184,6 +186,9 @@ struct apple_nvme_iod { struct apple_nvme_hw { bool has_lsq_nvmmu; + bool has_queue_count; + bool has_separate_nvmmu; + bool needs_ioq_registers; u32 max_queue_depth; }; @@ -192,6 +197,7 @@ struct apple_nvme { void __iomem *mmio_coproc; void __iomem *mmio_nvme; + void __iomem *mmio_nvmmu; const struct apple_nvme_hw *hw; struct device **pd_dev; @@ -227,6 +233,21 @@ struct apple_nvme { struct delayed_work flush_dwork; }; +static inline void apple_nvme_writeq(struct apple_nvme *anv, u64 value, + void __iomem *addr) +{ + /* + * Post-M4 ANS consumes the queue and NVMMU TCB base addresses as paired + * 32-bit registers. Match the access sequence used by Apple firmware, + * m1n1, and U-Boot; older ANS generations retain their native 64-bit + * access. + */ + if (anv->hw->needs_ioq_registers) + lo_hi_writeq(value, addr); + else + writeq(value, addr); +} + unsigned int flush_interval = 1000; module_param(flush_interval, uint, 0644); MODULE_PARM_DESC(flush_interval, "Grace period in msecs between flushes"); @@ -310,8 +331,8 @@ static void apple_nvmmu_inval(struct apple_nvme_queue *q, unsigned int tag) { struct apple_nvme *anv = queue_to_apple_nvme(q); - writel(tag, anv->mmio_nvme + APPLE_NVMMU_TCB_INVAL); - if (readl(anv->mmio_nvme + APPLE_NVMMU_TCB_STAT)) + writel(tag, anv->mmio_nvmmu + APPLE_NVMMU_TCB_INVAL); + if (readl(anv->mmio_nvmmu + APPLE_NVMMU_TCB_STAT)) dev_warn_ratelimited(anv->dev, "NVMMU TCB invalidation failed\n"); } @@ -916,7 +937,13 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) nvme_quiesce_io_queues(&anv->ctrl); if (!dead) { - if (apple_nvme_queue_enabled(&anv->ioq)) { + /* + * Post-M4 firmware rejects Delete SQ and Delete CQ with BAD_CMD. + * Its I/O queues are registered through the dedicated IOSQ/IOCQ + * registers and are torn down by the controller disable below. + */ + if (apple_nvme_queue_enabled(&anv->ioq) && + !anv->hw->needs_ioq_registers) { apple_nvme_remove_sq(anv); apple_nvme_remove_cq(anv); } @@ -1076,7 +1103,7 @@ static void apple_nvme_reset_work(struct work_struct *work) { unsigned int nr_io_queues = 1; int ret; - u32 boot_status, aqa; + u32 boot_status, aqa, ioqa, ioqa_depth; struct apple_nvme *anv = container_of(work, struct apple_nvme, ctrl.reset_work); enum nvme_ctrl_state state = nvme_ctrl_state(&anv->ctrl); @@ -1168,30 +1195,58 @@ static void apple_nvme_reset_work(struct work_struct *work) */ writel(APPLE_ANS_LINEAR_SQ_EN, anv->mmio_nvme + APPLE_ANS_LINEAR_SQ_CTRL); + if (anv->hw->needs_ioq_registers) { + u32 lsq_ctrl = readl(anv->mmio_nvme + + APPLE_ANS_LINEAR_SQ_CTRL); + + if (!(lsq_ctrl & APPLE_ANS_LINEAR_SQ_EN)) { + dev_err(anv->dev, + "failed to enable linear submission queues: LSQ_CTRL=0x%08x\n", + lsq_ctrl); + ret = -EIO; + goto out; + } + } - /* Allow as many pending command as possible for both queues */ - writel(anv->hw->max_queue_depth - | (anv->hw->max_queue_depth << 16), anv->mmio_nvme - + APPLE_ANS_MAX_PEND_CMDS_CTRL); + /* + * The legacy register takes queue entry counts. On post-M4 + * controllers the same offset is IOQA and both fields contain the + * zero-based queue size, like NVMe AQA. Programming 64 for a + * 64-entry queue makes ANS treat it as 65 entries and reject CQ head + * 63 once the host wraps through the queue. + */ + ioqa_depth = anv->hw->max_queue_depth; + if (anv->hw->needs_ioq_registers) + ioqa_depth--; + ioqa = ioqa_depth | (ioqa_depth << 16); + writel(ioqa, anv->mmio_nvme + APPLE_ANS_MAX_PEND_CMDS_CTRL); + if (anv->hw->needs_ioq_registers && + readl(anv->mmio_nvme + APPLE_ANS_MAX_PEND_CMDS_CTRL) != ioqa) { + dev_err(anv->dev, "failed to program I/O queue aperture\n"); + ret = -EIO; + goto out; + } /* Setup the NVMMU for the maximum admin and IO queue depth */ writel(anv->hw->max_queue_depth - 1, - anv->mmio_nvme + APPLE_NVMMU_NUM_TCBS); + anv->mmio_nvmmu + APPLE_NVMMU_NUM_TCBS); } /* Setup the admin queue */ aqa = APPLE_NVME_AQ_DEPTH - 1; aqa |= aqa << 16; writel(aqa, anv->mmio_nvme + NVME_REG_AQA); - writeq(anv->adminq.sq_dma_addr, anv->mmio_nvme + NVME_REG_ASQ); - writeq(anv->adminq.cq_dma_addr, anv->mmio_nvme + NVME_REG_ACQ); + apple_nvme_writeq(anv, anv->adminq.sq_dma_addr, + anv->mmio_nvme + NVME_REG_ASQ); + apple_nvme_writeq(anv, anv->adminq.cq_dma_addr, + anv->mmio_nvme + NVME_REG_ACQ); if (anv->hw->has_lsq_nvmmu) { /* Setup NVMMU for both queues */ - writeq(anv->adminq.tcb_dma_addr, - anv->mmio_nvme + APPLE_NVMMU_ASQ_TCB_BASE); - writeq(anv->ioq.tcb_dma_addr, - anv->mmio_nvme + APPLE_NVMMU_IOSQ_TCB_BASE); + apple_nvme_writeq(anv, anv->adminq.tcb_dma_addr, + anv->mmio_nvmmu + APPLE_NVMMU_ASQ_TCB_BASE); + apple_nvme_writeq(anv, anv->ioq.tcb_dma_addr, + anv->mmio_nvmmu + APPLE_NVMMU_IOSQ_TCB_BASE); } anv->ctrl.sqsize = @@ -1228,13 +1283,26 @@ static void apple_nvme_reset_work(struct work_struct *work) goto out_remove_cq; apple_nvme_init_queue(&anv->ioq); + if (anv->hw->needs_ioq_registers) { + /* + * Post-M4 firmware also takes the I/O queue addresses through + * dedicated registers. Without them it crashes and its crash log + * reports null I/O SQ and CQ addresses. + */ + apple_nvme_writeq(anv, anv->ioq.cq_dma_addr, + anv->mmio_nvme + APPLE_ANS_IOCQ_REGISTER); + apple_nvme_writeq(anv, anv->ioq.sq_dma_addr, + anv->mmio_nvme + APPLE_ANS_IOSQ_REGISTER); + } nr_io_queues = 1; - ret = nvme_set_queue_count(&anv->ctrl, &nr_io_queues); - if (ret) - goto out_remove_sq; - if (nr_io_queues != 1) { - ret = -ENXIO; - goto out_remove_sq; + if (anv->hw->has_queue_count) { + ret = nvme_set_queue_count(&anv->ctrl, &nr_io_queues); + if (ret) + goto out_remove_sq; + if (nr_io_queues != 1) { + ret = -ENXIO; + goto out_remove_sq; + } } anv->ctrl.queue_count = nr_io_queues + 1; @@ -1257,9 +1325,11 @@ static void apple_nvme_reset_work(struct work_struct *work) return; out_remove_sq: - apple_nvme_remove_sq(anv); + if (!anv->hw->needs_ioq_registers) + apple_nvme_remove_sq(anv); out_remove_cq: - apple_nvme_remove_cq(anv); + if (!anv->hw->needs_ioq_registers) + apple_nvme_remove_cq(anv); out: dev_warn(anv->ctrl.device, "Reset failure status: %d\n", ret); nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_DELETING); @@ -1559,6 +1629,16 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) ret = PTR_ERR(anv->mmio_nvme); goto put_dev; } + if (anv->hw->has_separate_nvmmu) { + anv->mmio_nvmmu = + devm_platform_ioremap_resource_byname(pdev, "nvmmu"); + if (IS_ERR(anv->mmio_nvmmu)) { + ret = PTR_ERR(anv->mmio_nvmmu); + goto put_dev; + } + } else { + anv->mmio_nvmmu = anv->mmio_nvme; + } if (anv->hw->has_lsq_nvmmu) { anv->adminq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_ASQ_DB; @@ -1769,17 +1849,27 @@ static DEFINE_SIMPLE_DEV_PM_OPS(apple_nvme_pm_ops, apple_nvme_suspend, static const struct apple_nvme_hw apple_nvme_t8015_hw = { .has_lsq_nvmmu = false, + .has_queue_count = true, .max_queue_depth = 16, }; static const struct apple_nvme_hw apple_nvme_t8103_hw = { .has_lsq_nvmmu = true, + .has_queue_count = true, + .max_queue_depth = 64, +}; + +static const struct apple_nvme_hw apple_nvme_t8132_hw = { + .has_lsq_nvmmu = true, + .has_separate_nvmmu = true, + .needs_ioq_registers = true, .max_queue_depth = 64, }; static const struct of_device_id apple_nvme_of_match[] = { { .compatible = "apple,t8015-nvme-ans2", .data = &apple_nvme_t8015_hw }, { .compatible = "apple,t8103-nvme-ans2", .data = &apple_nvme_t8103_hw }, + { .compatible = "apple,t8132-nvme-ans2", .data = &apple_nvme_t8132_hw }, { .compatible = "apple,nvme-ans2", .data = &apple_nvme_t8103_hw }, {}, }; From 14338a5e38f9951c33e20cf287f8a61b28885613 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:25:19 +1000 Subject: [PATCH 0360/2680] nvme-apple: Restart a stopped resident ANS without a PMGR reset When a previous boot stage has run the ANS and stopped it cleanly, the driver finds CPU_CONTROL.RUN clear and takes its soft-reset path, which asserts the ANS reset. On post-M4 SoCs that assert raises an SError. The stopped firmware stays resident, so on these SoCs set RUN and run a fresh RTKit boot handshake instead, as U-Boot does, and keep the reset path for the older generations. This was tested on T8140 (MacBook Neo, J700) booting Linux from the internal NVMe through a chain whose first m1n1 stage runs the ANS and stops it (CPU_CONTROL.RUN clear, firmware still resident) before chainloading the stage that boots Linux. It does not recover an ANS that Linux itself has stopped: once Linux has shut RTKit down and cleared RUN, as suspend did, the RTKit boot after setting RUN again times out. Satya wrote this change for Aurora's tree on 2026-09-18, after the reset path raised the SError on his J700, and has been booting Omarchy from the internal NVMe with it since 2026-09-16. It is carried here with the restart message demoted to dev_dbg; his Signed-off-by is outstanding. (cherry picked from commit 711f8f6287ce8374336372a104e46d6f427dd44e) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 97f6e7a602c87..0e848eb2987e0 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -1142,6 +1142,22 @@ static void apple_nvme_reset_work(struct work_struct *work) if (!(readl(anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL) & APPLE_ANS_COPROC_CPU_CONTROL_RUN)) { + if (anv->hw->needs_ioq_registers) { + /* + * Post-M4 ANS cannot be reset through PMGR: the reset + * assert raises an SError. A previous boot stage that + * stopped the coprocessor cleanly leaves its firmware + * resident, so mirror the qualified U-Boot handoff: + * set RUN and run a fresh RTKit boot handshake. + */ + dev_dbg(anv->dev, + "post-M4 firmware handoff: restarting stopped ANS without reset\n"); + writel(APPLE_ANS_COPROC_CPU_CONTROL_RUN, + anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); + ret = apple_rtkit_boot(anv->rtk); + goto booted; + } + ret = reset_control_assert(anv->reset); if (ret) goto out; @@ -1162,6 +1178,7 @@ static void apple_nvme_reset_work(struct work_struct *work) ret = apple_rtkit_wake(anv->rtk); } +booted: if (ret) { dev_err(anv->dev, "ANS did not boot"); goto out; From 36c78ad8a82c3558bfa5c58e285bf763298164bb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:29:03 +1000 Subject: [PATCH 0361/2680] nvme-apple: Adopt a live inherited RTKit session on post-M4 controllers On post-M4 SoCs the ANS2 co-processor cannot be reset (asserting its PMGR reset raises an SError), and a bootloader that used the internal storage may hand its firmware over still running. That firmware has completed HELLO and EPMAP with its previous owner and will not repeat them, so the driver has to adopt the running RTKit session instead of booting it. Adopt the session when the controller shows a healthy firmware that still has an owner: - CPU_CONTROL.RUN is set, BOOT_STATUS reads 0xde71ce55 and CSTS.CFS is clear, and - either CC.EN and CSTS.RDY are both set, because the bootloader left the controller running (U-Boot does so until the session has a new owner), or both are clear and the device tree carries "apple,rtkit-handoff" = "nvme-disabled-v1", with which the bootloader states that it stopped the controller but kept the session. RUN and BOOT_STATUS alone do not identify a live session: both stay set after the firmware has gone inactive, and adopting that state marks the RTKit power states as on without the wake handshake, which leaves CSTS.RDY stuck at zero. A controller with neither CC.EN nor the property therefore takes the existing wake path. If the property is present but the controller state does not match it, probe fails rather than guess. A controller whose RUN bit is clear is restarted as before. The adopted firmware keeps using the shared buffers its previous owner gave it, so its buffer requests arrive with a non-zero IOVA. Map such a buffer only if a SART entry programmed by the bootloader covers it, it is cache-line aligned, and every page of it is reserved System RAM in the linear map. Map it with DMA_ATTR_SKIP_CPU_SYNC and reject it unless the DMA address equals the firmware's IOVA, so that a bounce buffer is never substituted for live data, and let RTKit sync it before reading. The inherited SART entries are never removed. Create the RTKit instance as adopted before mailbox RX starts, so that inherited syslog traffic is not dropped as coming from undiscovered endpoints. In the reset work, adopt the session, read CAP and CC from the controller and clear CC.EN if it is set; only then program the admin queue and NVMMU base registers. No command is ever submitted to queue or TCB memory that the driver did not allocate. Because mailbox RX now runs from probe, an RTKit crash can be reported before the NVMe controller is initialized, so the crash callback only schedules a reset once probe has armed it. The adoption predicate, apple_nvme_can_adopt_rtkit(), is Acelogic's work for Aurora's tree (2026-09-09), where it let a J700 booted directly through m1n1 with the controller left enabled complete initialization (CC=0x460001, CSTS=1); it is extended here with the CSTS.CFS check and the bootloader-asserted stopped state. His Signed-off-by is outstanding. The enabled-controller path is the one the J700 boots through today: the m1n1 builds used by its boot chain hand the controller over running. The stopped-controller path is produced by the m1n1 builds from 2026-09-24 on, which retire the controller's DMA, reserve the retained heap and only then publish apple,rtkit-handoff. Booting through U-Boot has not been tested here. (cherry picked from commit e828e9ca80c155f1ca977bc82f6cbc025da08400) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 241 +++++++++++++++++++++++++++++++++++++- 1 file changed, 236 insertions(+), 5 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 0e848eb2987e0..e60ba5fe191f2 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -18,9 +18,12 @@ #include #include #include +#include #include #include #include +#include +#include #include #include #include @@ -194,6 +197,10 @@ struct apple_nvme_hw { struct apple_nvme { struct device *dev; + /* Protects reset_ready against RTKit crash callbacks. */ + spinlock_t event_lock; + bool reset_ready; + bool handoff_disabled; void __iomem *mmio_coproc; void __iomem *mmio_nvme; @@ -207,6 +214,8 @@ struct apple_nvme { struct apple_sart *sart; struct apple_rtkit *rtk; struct reset_control *reset; + bool owns_rtkit; + bool inherited_rtkit; struct dma_pool *prp_page_pool; struct dma_pool *prp_small_pool; @@ -260,6 +269,46 @@ static inline struct apple_nvme *ctrl_to_apple_nvme(struct nvme_ctrl *ctrl) return container_of(ctrl, struct apple_nvme, ctrl); } +static bool apple_nvme_can_adopt_rtkit(struct apple_nvme *anv) +{ + u32 cc, csts; + + if (!anv->hw->needs_ioq_registers || + !(readl(anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL) & + APPLE_ANS_COPROC_CPU_CONTROL_RUN) || + readl(anv->mmio_nvme + APPLE_ANS_BOOT_STATUS) != APPLE_ANS_BOOT_STATUS_OK) + return false; + cc = readl(anv->mmio_nvme + NVME_REG_CC); + csts = readl(anv->mmio_nvme + NVME_REG_CSTS); + if (csts & NVME_CSTS_CFS) + return false; + + /* + * CPU_RUN/BOOT_STATUS alone survive inactive sessions. A disabled + * controller requires the producer's explicit live-session contract. + * Borrowed buffers still require reserved RAM and protected SART ranges. + */ + if (anv->handoff_disabled) + return !(cc & NVME_CC_ENABLE) && !(csts & NVME_CSTS_RDY); + return (cc & NVME_CC_ENABLE) && (csts & NVME_CSTS_RDY); +} + +static int apple_nvme_read_handoff(struct apple_nvme *anv) +{ + const char *handoff; + int ret; + + ret = of_property_read_string(anv->dev->of_node, "apple,rtkit-handoff", + &handoff); + if (ret == -EINVAL) + return 0; + if (ret || !anv->hw->needs_ioq_registers || + strcmp(handoff, "nvme-disabled-v1")) + return -EINVAL; + anv->handoff_disabled = true; + return apple_nvme_can_adopt_rtkit(anv) ? 0 : -EIO; +} + static inline struct apple_nvme *queue_to_apple_nvme(struct apple_nvme_queue *q) { if (q->is_adminq) @@ -278,24 +327,128 @@ static unsigned int apple_nvme_queue_depth(struct apple_nvme_queue *q) return anv->hw->max_queue_depth; } +static void apple_nvme_set_reset_ready(struct apple_nvme *anv, bool ready) +{ + unsigned long flags; + + spin_lock_irqsave(&anv->event_lock, flags); + anv->reset_ready = ready; + spin_unlock_irqrestore(&anv->event_lock, flags); +} + static void apple_nvme_rtkit_crashed(void *cookie, const void *crashlog, size_t crashlog_size) { struct apple_nvme *anv = cookie; + unsigned long flags; dev_warn(anv->dev, "RTKit crashed; unable to recover without a reboot"); - nvme_reset_ctrl(&anv->ctrl); + /* + * RX may run inside RTKit initialization, before ctrl/admin_q exist. + * Probe's initial reset observes RTKit's latched crash state. Serialize + * callback scheduling with closing this gate before work cancellation. + */ + spin_lock_irqsave(&anv->event_lock, flags); + if (anv->reset_ready) + nvme_reset_ctrl(&anv->ctrl); + spin_unlock_irqrestore(&anv->event_lock, flags); } +static bool apple_nvme_in_inherited_rtkit_range(struct apple_nvme *anv, + struct apple_rtkit_shmem *bfr) +{ + phys_addr_t last; + unsigned long pfn, last_pfn; + + if (!anv->inherited_rtkit || !bfr->size || + check_add_overflow((phys_addr_t)bfr->iova, + (phys_addr_t)bfr->size - 1, &last) || + !IS_ALIGNED(bfr->iova, dma_get_cache_alignment()) || + !IS_ALIGNED(bfr->size, dma_get_cache_alignment()) || + !apple_sart_is_inherited_region(anv->sart, bfr->iova, bfr->size)) + return false; + + /* + * A range property or a partial System RAM intersection does not prove + * ownership. Every page must be reserved and present in the linear map. + * Native m1n1 reserves the predecessor's inherited allocation prefix. + * RAM excluded by a resident EL2 stage needs a separate cache contract. + */ + if (region_intersects(bfr->iova, bfr->size, IORESOURCE_SYSTEM_RAM, + IORES_DESC_NONE) != REGION_INTERSECTS) + return false; + last_pfn = PHYS_PFN(last); + for (pfn = PHYS_PFN(bfr->iova); pfn <= last_pfn; pfn++) { + struct page *page; + + if (!pfn_valid(pfn)) + return false; + page = pfn_to_page(pfn); + if (!PageReserved(page) || PageHighMem(page) || + !virt_addr_valid(page_to_virt(page))) + return false; + } + return true; +} + +/* A retained DMA mapping must also retain its device context. */ +struct apple_nvme_rtkit_mapping { + struct device *dev; +}; + static int apple_nvme_sart_dma_setup(void *cookie, struct apple_rtkit_shmem *bfr) { struct apple_nvme *anv = cookie; int ret; - if (bfr->iova) - return -EINVAL; if (!bfr->size) return -EINVAL; + if (bfr->iova) { + struct apple_nvme_rtkit_mapping *mapping; + dma_addr_t dma; + void *buffer; + + if (!apple_nvme_in_inherited_rtkit_range(anv, bfr)) + return -EINVAL; + mapping = kzalloc_obj(*mapping); + if (!mapping) + return -ENOMEM; + buffer = memremap(bfr->iova, bfr->size, MEMREMAP_WB); + if (!buffer) { + kfree(mapping); + return -ENOMEM; + } + + /* + * Firmware keeps the old identity IOVA. A bounce/remapped address + * cannot replace it. Skip synchronization until that is verified, + * and never copy a rejected bounce buffer over the live data. + */ + dma = dma_map_single_attrs(anv->dev, buffer, bfr->size, + DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); + if (dma_mapping_error(anv->dev, dma)) { + ret = -ENOMEM; + goto unmap_buffer; + } + if (dma != bfr->iova) { + dma_unmap_single_attrs(anv->dev, dma, bfr->size, + DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); + ret = -EINVAL; + goto unmap_buffer; + } + dma_sync_single_for_device(anv->dev, dma, bfr->size, DMA_FROM_DEVICE); + mapping->dev = get_device(anv->dev); + bfr->buffer = buffer; + bfr->is_mapped = true; + bfr->needs_dma_sync = true; + bfr->private = mapping; + return 0; + +unmap_buffer: + memunmap(buffer); + kfree(mapping); + return ret; + } bfr->buffer = dma_alloc_coherent(anv->dev, bfr->size, &bfr->iova, GFP_KERNEL); @@ -316,7 +469,15 @@ static void apple_nvme_sart_dma_destroy(void *cookie, struct apple_rtkit_shmem *bfr) { struct apple_nvme *anv = cookie; + struct apple_nvme_rtkit_mapping *mapping = bfr->private; + if (mapping) { + dma_unmap_single(mapping->dev, bfr->iova, bfr->size, DMA_FROM_DEVICE); + memunmap(bfr->buffer); + put_device(mapping->dev); + kfree(mapping); + return; + } apple_sart_remove_allowed_region(anv->sart, bfr->iova, bfr->size); dma_free_coherent(anv->dev, bfr->size, bfr->buffer, bfr->iova); } @@ -1122,6 +1283,36 @@ static void apple_nvme_reset_work(struct work_struct *work) goto out; } + /* + * Post-M4 firmware can be handed over live, but cannot renegotiate HELLO + * and EPMAP across boot stages. Adopt the first healthy inherited session. + */ + if (!anv->owns_rtkit && apple_nvme_can_adopt_rtkit(anv)) { + ret = apple_rtkit_adopt_running(anv->rtk); + if (ret) + goto out; + dev_dbg(anv->dev, "adopted inherited post-M4 RTKit session\n"); + + /* + * U-Boot keeps the controller enabled until the mailbox session has + * an owner. Disable it only after adoption, so that the admin queue + * and NVMMU pointers below are replaced while CC.EN is clear. The + * I/O queue registers are written after Create SQ, as on a cold + * start. + */ + anv->ctrl.cap = readq(anv->mmio_nvme + NVME_REG_CAP); + anv->ctrl.ctrl_config = + readl(anv->mmio_nvme + NVME_REG_CC); + if (anv->ctrl.ctrl_config & NVME_CC_ENABLE) { + ret = nvme_disable_ctrl(&anv->ctrl, false); + if (ret) + goto out; + dev_dbg(anv->dev, + "disabled inherited post-M4 controller after RTKit adoption\n"); + } + goto rtkit_ready; + } + /* RTKit must be shut down cleanly for the (soft)-reset to work */ if (apple_rtkit_is_running(anv->rtk)) { /* reset the controller if it is enabled */ @@ -1184,6 +1375,9 @@ static void apple_nvme_reset_work(struct work_struct *work) goto out; } +rtkit_ready: + anv->owns_rtkit = true; + ret = readl_poll_timeout(anv->mmio_nvme + APPLE_ANS_BOOT_STATUS, boot_status, boot_status == APPLE_ANS_BOOT_STATUS_OK, @@ -1574,6 +1768,14 @@ static void devm_apple_nvme_mempool_destroy(void *data) mempool_destroy(data); } +static void apple_nvme_free_rtkit(void *data) +{ + struct apple_nvme *anv = data; + + apple_nvme_set_reset_ready(anv, false); + apple_rtkit_free(anv->rtk); +} + static void apple_nvme_flush_work(struct work_struct *work) { struct nvme_command c = { }; @@ -1600,6 +1802,7 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct apple_nvme *anv; + bool inherited_rtkit; int ret; anv = devm_kzalloc(dev, sizeof(*anv), GFP_KERNEL); @@ -1607,6 +1810,7 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) return ERR_PTR(-ENOMEM); anv->dev = get_device(dev); + spin_lock_init(&anv->event_lock); anv->adminq.is_adminq = true; platform_set_drvdata(pdev, anv); @@ -1657,6 +1861,12 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) anv->mmio_nvmmu = anv->mmio_nvme; } + ret = apple_nvme_read_handoff(anv); + if (ret) { + dev_err_probe(dev, ret, "Invalid live firmware handoff\n"); + goto put_dev; + } + if (anv->hw->has_lsq_nvmmu) { anv->adminq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_ASQ_DB; anv->adminq.cq_db = anv->mmio_nvme + APPLE_ANS_ACQ_DB; @@ -1732,14 +1942,34 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) goto put_dev; } - anv->rtk = - devm_apple_rtkit_init(dev, anv, NULL, 0, &apple_nvme_rtkit_ops); + /* + * A post-M4 boot stage may leave RTKit fully running. Prime the known + * system endpoints before mailbox RX starts, otherwise inherited syslog + * traffic can race the later reset work and be dropped as undiscovered. + */ + inherited_rtkit = apple_nvme_can_adopt_rtkit(anv); + /* Publish admission before the initializer enables mailbox RX. */ + anv->inherited_rtkit = inherited_rtkit; + if (anv->hw->needs_ioq_registers) + dev_dbg(dev, "post-M4 firmware handoff: CC=%#x CSTS=%#x adopt=%d\n", + readl(anv->mmio_nvme + NVME_REG_CC), + readl(anv->mmio_nvme + NVME_REG_CSTS), inherited_rtkit); + if (inherited_rtkit) + anv->rtk = apple_rtkit_init_adopted(dev, anv, NULL, 0, + &apple_nvme_rtkit_ops); + else + anv->rtk = apple_rtkit_init(dev, anv, NULL, 0, + &apple_nvme_rtkit_ops); if (IS_ERR(anv->rtk)) { ret = dev_err_probe(dev, PTR_ERR(anv->rtk), "Failed to initialize RTKit"); goto put_dev; } + ret = devm_add_action_or_reset(dev, apple_nvme_free_rtkit, anv); + if (ret) + goto put_dev; + ret = nvme_init_ctrl(&anv->ctrl, anv->dev, &nvme_ctrl_ops, NVME_QUIRK_SKIP_CID_GEN | NVME_QUIRK_IDENTIFY_CNS | NVME_QUIRK_ADMIN_PAGE_ALIGN); @@ -1783,6 +2013,7 @@ static int apple_nvme_probe(struct platform_device *pdev) INIT_DELAYED_WORK(&anv->flush_dwork, apple_nvme_flush_work); + apple_nvme_set_reset_ready(anv, true); nvme_reset_ctrl(&anv->ctrl); async_schedule(apple_nvme_async_probe, anv); From ea0e79d7d74df6d0d01d8810aa1494e3a0cce5a4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:29:13 +1000 Subject: [PATCH 0362/2680] nvme-apple: Serialize post-M4 disable, timeout recovery and reset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A post-M4 ANS2 cannot be restarted once it has been stopped. On J700, after Linux has shut RTKit down and cleared CPU_CONTROL.RUN, the RTKit boot that follows setting RUN again times out, and the PMGR reset that older SoCs use to reload the firmware raises an SError. The RTKit shutdown handshake itself crashes the firmware at system shutdown. A controller reset on these SoCs therefore keeps the owned RTKit session and only stops and rebuilds the NVMe queues, and only a completed CC.EN clear (CSTS.RDY reading 0) stops the controller's DMA: neither CSTS.CFS, a firmware crash nor quiescing the block layer queues does. Give post-M4 its own disable: freeze and quiesce, mark both queues disabled, clear CC.EN and wait for CSTS.RDY, then reap the completion queues and cancel what is left. It sends no Delete SQ/CQ, which the firmware rejects, no shutdown notification and no RTKit sleep. Whether the T8140 firmware completes a shutdown notification has not been measured here; m1n1 sends one before it hands the controller over and tolerates a timeout. The legacy disable is unchanged. disable_lock serializes this disable with the register programming in the reset work and with RTKit teardown. The reset work drops the lock while it waits for admin commands (Identify, Create CQ/SQ), so that a timeout of one of them can still stop the controller, and re-checks for removal or an abort after each unlocked phase. A post-M4 request timeout never completes a request before the controller has stopped. On a live controller the handler moves it to RESETTING and hands the stop to recovery_work, which schedules the reset once the controller has stopped. A timeout of the reset's own admin command, while CONNECTING, marks that reset aborted and lets recovery_work stop the controller. During removal the handler stops the controller itself, as pci.c does. Outstanding requests are cancelled only after the stop. On all controllers the timeout handler now polls the completion queue before the non-live fast-fail, so that a lost interrupt during the first Identify no longer turns a completed command into a reset, and its warnings include CC, CSTS and the completion queue state. The NVMe character device can outlive the platform device, so struct apple_nvme is now reference counted: devres holds one reference and the nvme_ctrl the other, and the register accessors fail with -ENODEV once the device is detached. RTKit teardown on post-M4 uses apple_rtkit_free_retaining_buffers(), because the firmware keeps running after unbind and may still write the shared buffers. remove() keeps its order: stop the controller, remove the namespaces, which drains outstanding I/O, then disable. On all controllers remove() now cancels a pending reset or recovery instead of waiting for it, and shutdown() cancels them too; both leave a post-M4 RTKit session running. suspend() moves a post-M4 controller to RESETTING, so that neither timeout recovery nor an RTKit crash can start a reset while it is suspended, and disables only the NVMe controller; resume() schedules the reset that rebuilds the queues on the owned session. [ryan: adapted to 7.2.10 — nvme_wait_freeze_timeout() lost its timeout argument (commit 23b6d2cbf75f ("nvme: remove redundant timeout argument from nvme_wait_freeze_timeout")) and now waits ctrl->io_timeout, which nvme_init_ctrl() sets to NVME_IO_TIMEOUT, the value passed here before; the only difference is that it now follows the controller's io_timeout sysfs attribute.] (cherry picked from commit 7296c586542325e840514bdb1713c7563dd28579) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 426 ++++++++++++++++++++++++++++++++------ 1 file changed, 360 insertions(+), 66 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index e60ba5fe191f2..5be769506be4c 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -21,10 +21,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -197,10 +199,19 @@ struct apple_nvme_hw { struct apple_nvme { struct device *dev; + struct kref ref; + /* Serializes the post-M4 controller disable with reset and teardown. */ + struct mutex disable_lock; /* Protects reset_ready against RTKit crash callbacks. */ spinlock_t event_lock; bool reset_ready; bool handoff_disabled; + bool detached; + bool removing; + bool abort_reset; + bool recover_to_live; + bool suspended; + unsigned long recovery_flags; void __iomem *mmio_coproc; void __iomem *mmio_nvme; @@ -223,6 +234,7 @@ struct apple_nvme { struct nvme_ctrl ctrl; struct work_struct remove_work; + struct work_struct recovery_work; struct apple_nvme_queue adminq; struct apple_nvme_queue ioq; @@ -242,6 +254,9 @@ struct apple_nvme { struct delayed_work flush_dwork; }; +/* Bit in recovery_flags: recovery_work owns the next controller stop. */ +#define APPLE_NVME_RECOVERY_PENDING 0 + static inline void apple_nvme_writeq(struct apple_nvme *anv, u64 value, void __iomem *addr) { @@ -269,6 +284,26 @@ static inline struct apple_nvme *ctrl_to_apple_nvme(struct nvme_ctrl *ctrl) return container_of(ctrl, struct apple_nvme, ctrl); } +/* + * The controller's character device can outlive platform-driver unbind. + * Keep the container until both devres and the last NVMe reference are gone. + */ +static void apple_nvme_release(struct kref *ref) +{ + struct apple_nvme *anv = container_of(ref, struct apple_nvme, ref); + + put_device(anv->dev); + kfree(anv); +} + +static void apple_nvme_put_resource_owner(void *data) +{ + struct apple_nvme *anv = data; + + WRITE_ONCE(anv->detached, true); + kref_put(&anv->ref, apple_nvme_release); +} + static bool apple_nvme_can_adopt_rtkit(struct apple_nvme *anv) { u32 cc, csts; @@ -341,6 +376,8 @@ static void apple_nvme_rtkit_crashed(void *cookie, const void *crashlog, size_t struct apple_nvme *anv = cookie; unsigned long flags; + if (READ_ONCE(anv->removing)) + return; dev_warn(anv->dev, "RTKit crashed; unable to recover without a reboot"); /* * RX may run inside RTKit initialization, before ctrl/admin_q exist. @@ -348,7 +385,7 @@ static void apple_nvme_rtkit_crashed(void *cookie, const void *crashlog, size_t * callback scheduling with closing this gate before work cancellation. */ spin_lock_irqsave(&anv->event_lock, flags); - if (anv->reset_ready) + if (anv->reset_ready && !READ_ONCE(anv->removing)) nvme_reset_ctrl(&anv->ctrl); spin_unlock_irqrestore(&anv->event_lock, flags); } @@ -401,6 +438,8 @@ static int apple_nvme_sart_dma_setup(void *cookie, struct apple_nvme *anv = cookie; int ret; + if (READ_ONCE(anv->removing)) + return -ESHUTDOWN; if (!bfr->size) return -EINVAL; if (bfr->iova) { @@ -1068,7 +1107,7 @@ static int apple_nvme_init_request(struct blk_mq_tag_set *set, return 0; } -static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) +static void apple_nvme_disable_legacy(struct apple_nvme *anv, bool shutdown) { enum nvme_ctrl_state state = nvme_ctrl_state(&anv->ctrl); u32 csts = readl(anv->mmio_nvme + NVME_REG_CSTS); @@ -1098,13 +1137,7 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) nvme_quiesce_io_queues(&anv->ctrl); if (!dead) { - /* - * Post-M4 firmware rejects Delete SQ and Delete CQ with BAD_CMD. - * Its I/O queues are registered through the dedicated IOSQ/IOCQ - * registers and are torn down by the controller disable below. - */ - if (apple_nvme_queue_enabled(&anv->ioq) && - !anv->hw->needs_ioq_registers) { + if (apple_nvme_queue_enabled(&anv->ioq)) { apple_nvme_remove_sq(anv); apple_nvme_remove_cq(anv); } @@ -1153,15 +1186,158 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) } } +static int apple_nvme_disable_locked(struct apple_nvme *anv, bool shutdown) +{ + enum nvme_ctrl_state state; + unsigned long flags; + int ret; + + lockdep_assert_held(&anv->disable_lock); + + state = nvme_ctrl_state(&anv->ctrl); + if ((state == NVME_CTRL_LIVE || state == NVME_CTRL_RESETTING) && + !test_bit(NVME_CTRL_FROZEN, &anv->ctrl.flags)) + nvme_start_freeze(&anv->ctrl); + if (shutdown && test_bit(NVME_CTRL_FROZEN, &anv->ctrl.flags) && + !apple_rtkit_is_crashed(anv->rtk)) + nvme_wait_freeze_timeout(&anv->ctrl); + + /* Quiescing dispatch does not stop already submitted DMA. */ + nvme_quiesce_io_queues(&anv->ctrl); + nvme_quiesce_admin_queue(&anv->ctrl); + apple_nvme_disable_queue(&anv->ioq); + apple_nvme_disable_queue(&anv->adminq); + + /* + * CFS and firmware crashes are not DMA fences. Post-M4 does not use + * Delete SQ/CQ, SHN or RTKit sleep; only a completed CC.EN clear stops + * the controller's queue DMA. + */ + ret = nvme_disable_ctrl(&anv->ctrl, false); + if (ret) + dev_err(anv->dev, "controller did not stop: %d\n", ret); + + spin_lock_irqsave(&anv->lock, flags); + apple_nvme_handle_cq(&anv->ioq, true); + apple_nvme_handle_cq(&anv->adminq, true); + spin_unlock_irqrestore(&anv->lock, flags); + nvme_cancel_tagset(&anv->ctrl); + nvme_cancel_admin_tagset(&anv->ctrl); + if (shutdown) { + nvme_unquiesce_io_queues(&anv->ctrl); + nvme_unquiesce_admin_queue(&anv->ctrl); + } + return ret; +} + +static int apple_nvme_disable(struct apple_nvme *anv, bool shutdown) +{ + if (!anv->hw->needs_ioq_registers) { + apple_nvme_disable_legacy(anv, shutdown); + return 0; + } + guard(mutex)(&anv->disable_lock); + return apple_nvme_disable_locked(anv, shutdown); +} + +static void apple_nvme_recovery_work(struct work_struct *work) +{ + struct apple_nvme *anv = container_of(work, struct apple_nvme, recovery_work); + bool restart = anv->recover_to_live; + int ret; + + /* + * The timeout callback must return before cancellation waits for the + * block layer's reference to that request. This worker owns the stop. + */ + if (READ_ONCE(anv->removing)) + goto done; + ret = apple_nvme_disable(anv, false); + if (ret) + goto done; + clear_bit(APPLE_NVME_RECOVERY_PENDING, &anv->recovery_flags); + if (restart && !READ_ONCE(anv->removing) && !READ_ONCE(anv->suspended)) + nvme_try_sched_reset(&anv->ctrl); + return; +done: + clear_bit(APPLE_NVME_RECOVERY_PENDING, &anv->recovery_flags); +} + static enum blk_eh_timer_return apple_nvme_timeout(struct request *req) { struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); struct apple_nvme_queue *q = iod->q; struct apple_nvme *anv = queue_to_apple_nvme(q); + struct nvme_completion *cqe = &q->cqes[q->cq_head]; + enum nvme_ctrl_state state = nvme_ctrl_state(&anv->ctrl); unsigned long flags; - u32 csts = readl(anv->mmio_nvme + NVME_REG_CSTS); + u32 cc, csts; + + if (blk_mq_request_completed(req)) + return BLK_EH_DONE; + cc = readl(anv->mmio_nvme + NVME_REG_CC); + csts = readl(anv->mmio_nvme + NVME_REG_CSTS); + + /* + * The first Identify runs while the controller is CONNECTING. Poll the + * completion queue before applying the non-live fast-fail rule so a lost + * interrupt cannot turn a completed initialization command into a reset. + */ + if (!apple_rtkit_is_crashed(anv->rtk) && !(csts & NVME_CSTS_CFS)) { + spin_lock_irqsave(&anv->lock, flags); + apple_nvme_handle_cq(q, false); + spin_unlock_irqrestore(&anv->lock, flags); + if (blk_mq_request_completed(req)) { + dev_warn(anv->dev, + "I/O %d(aq:%d) timeout in state %d: completion polled\n", + req->tag, q->is_adminq, state); + return BLK_EH_DONE; + } + } - if (nvme_ctrl_state(&anv->ctrl) != NVME_CTRL_LIVE) { + if (anv->hw->needs_ioq_registers) { + /* + * remove() drains the namespaces through the live controller and + * stops it afterwards. A request that times out meanwhile stops + * the controller here, as pci.c does for a deleting controller. + */ + if ((state == NVME_CTRL_DELETING || + state == NVME_CTRL_DELETING_NOIO) && + READ_ONCE(anv->removing)) { + nvme_req(req)->flags |= NVME_REQ_CANCELLED; + apple_nvme_disable(anv, true); + return BLK_EH_DONE; + } + /* + * An existing reset/removal owns the stop of its old requests. + * CONNECTING is different: reset_work may be waiting for this + * very admin command and needs the separate recovery worker. + */ + if (state != NVME_CTRL_LIVE && state != NVME_CTRL_CONNECTING) + return BLK_EH_RESET_TIMER; + if (!READ_ONCE(anv->removing) && + !test_and_set_bit(APPLE_NVME_RECOVERY_PENDING, &anv->recovery_flags)) { + /* + * Reserve RESETTING before scheduling recovery. An existing + * initialization instead receives an abort indication; its + * synchronous command is cancelled only after DMA stops. + */ + anv->recover_to_live = + nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_RESETTING); + if (!anv->recover_to_live) { + if (nvme_ctrl_state(&anv->ctrl) != NVME_CTRL_CONNECTING) { + clear_bit(APPLE_NVME_RECOVERY_PENDING, + &anv->recovery_flags); + return BLK_EH_RESET_TIMER; + } + WRITE_ONCE(anv->abort_reset, true); + } + queue_work(system_unbound_wq, &anv->recovery_work); + } + return BLK_EH_RESET_TIMER; + } + + if (state != NVME_CTRL_LIVE) { /* * From rdma.c: * If we are resetting, connecting or deleting we should @@ -1177,8 +1353,10 @@ static enum blk_eh_timer_return apple_nvme_timeout(struct request *req) * recovery work, so it's fine that we fail it here. */ dev_warn(anv->dev, - "I/O %d(aq:%d) timeout while not in live state\n", - req->tag, q->is_adminq); + "I/O %d(aq:%d) timeout in state %d CC=0x%08x CSTS=0x%08x CQ=%u/%u status=0x%04x cid=%u\n", + req->tag, q->is_adminq, state, cc, csts, q->cq_head, + q->cq_phase, le16_to_cpu(READ_ONCE(cqe->status)), + READ_ONCE(cqe->command_id)); if (blk_mq_request_started(req) && !blk_mq_request_completed(req)) { nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD; @@ -1188,25 +1366,15 @@ static enum blk_eh_timer_return apple_nvme_timeout(struct request *req) return BLK_EH_DONE; } - /* check if we just missed an interrupt if we're still alive */ - if (!apple_rtkit_is_crashed(anv->rtk) && !(csts & NVME_CSTS_CFS)) { - spin_lock_irqsave(&anv->lock, flags); - apple_nvme_handle_cq(q, false); - spin_unlock_irqrestore(&anv->lock, flags); - if (blk_mq_request_completed(req)) { - dev_warn(anv->dev, - "I/O %d(aq:%d) timeout: completion polled\n", - req->tag, q->is_adminq); - return BLK_EH_DONE; - } - } - /* * aborting commands isn't supported which leaves a full reset as our * only option here */ - dev_warn(anv->dev, "I/O %d(aq:%d) timeout: resetting controller\n", - req->tag, q->is_adminq); + dev_warn(anv->dev, + "I/O %d(aq:%d) timeout: resetting controller CC=0x%08x CSTS=0x%08x CQ=%u/%u status=0x%04x cid=%u\n", + req->tag, q->is_adminq, cc, csts, q->cq_head, q->cq_phase, + le16_to_cpu(READ_ONCE(cqe->status)), + READ_ONCE(cqe->command_id)); nvme_req(req)->flags |= NVME_REQ_CANCELLED; apple_nvme_disable(anv, false); nvme_reset_ctrl(&anv->ctrl); @@ -1221,6 +1389,8 @@ static int apple_nvme_poll(struct blk_mq_hw_ctx *hctx, bool found; unsigned long flags; + if (READ_ONCE(anv->detached)) + return 0; spin_lock_irqsave(&anv->lock, flags); found = apple_nvme_poll_cq(q, iob); spin_unlock_irqrestore(&anv->lock, flags); @@ -1263,12 +1433,23 @@ static void apple_nvme_init_queue(struct apple_nvme_queue *q) static void apple_nvme_reset_work(struct work_struct *work) { unsigned int nr_io_queues = 1; + bool phase_locked = false; int ret; u32 boot_status, aqa, ioqa, ioqa_depth; struct apple_nvme *anv = container_of(work, struct apple_nvme, ctrl.reset_work); enum nvme_ctrl_state state = nvme_ctrl_state(&anv->ctrl); + if (anv->hw->needs_ioq_registers) { + mutex_lock(&anv->disable_lock); + phase_locked = true; + } + if (READ_ONCE(anv->removing)) { + if (phase_locked) + mutex_unlock(&anv->disable_lock); + return; + } + WRITE_ONCE(anv->abort_reset, false); if (state != NVME_CTRL_RESETTING) { dev_warn(anv->dev, "ctrl state %d is not RESETTING\n", state); ret = -ENODEV; @@ -1285,7 +1466,8 @@ static void apple_nvme_reset_work(struct work_struct *work) /* * Post-M4 firmware can be handed over live, but cannot renegotiate HELLO - * and EPMAP across boot stages. Adopt the first healthy inherited session. + * and EPMAP across boot stages. Adopt the first healthy inherited session; + * subsequent post-M4 controller resets retain the owned RTKit session. */ if (!anv->owns_rtkit && apple_nvme_can_adopt_rtkit(anv)) { ret = apple_rtkit_adopt_running(anv->rtk); @@ -1313,11 +1495,25 @@ static void apple_nvme_reset_work(struct work_struct *work) goto rtkit_ready; } - /* RTKit must be shut down cleanly for the (soft)-reset to work */ + /* + * RTKit must be shut down cleanly for the (soft)-reset to work, except + * on post-M4: that firmware stays live and only its queue DMA is + * stopped and drained, under the register-programming lock. + */ if (apple_rtkit_is_running(anv->rtk)) { + if (anv->hw->needs_ioq_registers) { + if (!anv->owns_rtkit) { + ret = -EIO; + goto out; + } + ret = apple_nvme_disable_locked(anv, false); + if (ret) + goto out; + goto rtkit_ready; + } /* reset the controller if it is enabled */ if (anv->ctrl.ctrl_config & NVME_CC_ENABLE) - apple_nvme_disable(anv, false); + apple_nvme_disable_legacy(anv, false); dev_dbg(anv->dev, "Trying to shut down RTKit before reset."); ret = apple_rtkit_shutdown(anv->rtk); if (ret) @@ -1376,6 +1572,11 @@ static void apple_nvme_reset_work(struct work_struct *work) } rtkit_ready: + if (anv->hw->needs_ioq_registers && + (READ_ONCE(anv->removing) || READ_ONCE(anv->abort_reset))) { + ret = -ECANCELED; + goto out; + } anv->owns_rtkit = true; ret = readl_poll_timeout(anv->mmio_nvme + APPLE_ANS_BOOT_STATUS, @@ -1480,6 +1681,14 @@ static void apple_nvme_reset_work(struct work_struct *work) goto out; } + /* + * Admin commands may need the recovery worker to cancel them. Never + * hold disable_lock while waiting for their completions. + */ + if (phase_locked) { + mutex_unlock(&anv->disable_lock); + phase_locked = false; + } ret = nvme_init_ctrl_finish(&anv->ctrl, false); if (ret) goto out; @@ -1493,6 +1702,14 @@ static void apple_nvme_reset_work(struct work_struct *work) if (ret) goto out_remove_cq; + if (anv->hw->needs_ioq_registers) { + mutex_lock(&anv->disable_lock); + phase_locked = true; + if (READ_ONCE(anv->removing) || READ_ONCE(anv->abort_reset)) { + ret = -ECANCELED; + goto out; + } + } apple_nvme_init_queue(&anv->ioq); if (anv->hw->needs_ioq_registers) { /* @@ -1533,6 +1750,8 @@ static void apple_nvme_reset_work(struct work_struct *work) nvme_start_ctrl(&anv->ctrl); dev_dbg(anv->dev, "ANS boot and NVMe init completed."); + if (phase_locked) + mutex_unlock(&anv->disable_lock); return; out_remove_sq: @@ -1542,10 +1761,14 @@ static void apple_nvme_reset_work(struct work_struct *work) if (!anv->hw->needs_ioq_registers) apple_nvme_remove_cq(anv); out: + if (phase_locked) + mutex_unlock(&anv->disable_lock); dev_warn(anv->ctrl.device, "Reset failure status: %d\n", ret); nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_DELETING); - nvme_get_ctrl(&anv->ctrl); apple_nvme_disable(anv, false); + if (READ_ONCE(anv->removing)) + return; + nvme_get_ctrl(&anv->ctrl); nvme_mark_namespaces_dead(&anv->ctrl); if (!queue_work(nvme_wq, &anv->remove_work)) nvme_put_ctrl(&anv->ctrl); @@ -1562,19 +1785,31 @@ static void apple_nvme_remove_dead_ctrl_work(struct work_struct *work) static int apple_nvme_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val) { - *val = readl(ctrl_to_apple_nvme(ctrl)->mmio_nvme + off); + struct apple_nvme *anv = ctrl_to_apple_nvme(ctrl); + + if (READ_ONCE(anv->detached)) + return -ENODEV; + *val = readl(anv->mmio_nvme + off); return 0; } static int apple_nvme_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val) { - writel(val, ctrl_to_apple_nvme(ctrl)->mmio_nvme + off); + struct apple_nvme *anv = ctrl_to_apple_nvme(ctrl); + + if (READ_ONCE(anv->detached)) + return -ENODEV; + writel(val, anv->mmio_nvme + off); return 0; } static int apple_nvme_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val) { - *val = readq(ctrl_to_apple_nvme(ctrl)->mmio_nvme + off); + struct apple_nvme *anv = ctrl_to_apple_nvme(ctrl); + + if (READ_ONCE(anv->detached)) + return -ENODEV; + *val = readq(anv->mmio_nvme + off); return 0; } @@ -1587,7 +1822,9 @@ static int apple_nvme_get_address(struct nvme_ctrl *ctrl, char *buf, int size) static void apple_nvme_free_ctrl(struct nvme_ctrl *ctrl) { - put_device(ctrl->dev); + struct apple_nvme *anv = ctrl_to_apple_nvme(ctrl); + + kref_put(&anv->ref, apple_nvme_release); } static const struct nvme_ctrl_ops nvme_ctrl_ops = { @@ -1773,7 +2010,12 @@ static void apple_nvme_free_rtkit(void *data) struct apple_nvme *anv = data; apple_nvme_set_reset_ready(anv, false); - apple_rtkit_free(anv->rtk); + guard(mutex)(&anv->disable_lock); + if (anv->hw->needs_ioq_registers) + apple_rtkit_free_retaining_buffers(anv->rtk); + else + apple_rtkit_free(anv->rtk); + anv->rtk = NULL; } static void apple_nvme_flush_work(struct work_struct *work) @@ -1805,57 +2047,62 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) bool inherited_rtkit; int ret; - anv = devm_kzalloc(dev, sizeof(*anv), GFP_KERNEL); + anv = kzalloc_obj(*anv); if (!anv) return ERR_PTR(-ENOMEM); anv->dev = get_device(dev); + kref_init(&anv->ref); + mutex_init(&anv->disable_lock); spin_lock_init(&anv->event_lock); + ret = devm_add_action_or_reset(dev, apple_nvme_put_resource_owner, anv); + if (ret) + return ERR_PTR(ret); anv->adminq.is_adminq = true; platform_set_drvdata(pdev, anv); anv->hw = of_device_get_match_data(&pdev->dev); if (!anv->hw) { ret = -ENODEV; - goto put_dev; + goto out_detach_genpd; } ret = apple_nvme_attach_genpd(anv); if (ret < 0) { dev_err_probe(dev, ret, "Failed to attach power domains"); - goto put_dev; + goto out_detach_genpd; } if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64))) { ret = -ENXIO; - goto put_dev; + goto out_detach_genpd; } anv->irq = platform_get_irq(pdev, 0); if (anv->irq < 0) { ret = anv->irq; - goto put_dev; + goto out_detach_genpd; } if (!anv->irq) { ret = -ENXIO; - goto put_dev; + goto out_detach_genpd; } anv->mmio_coproc = devm_platform_ioremap_resource_byname(pdev, "ans"); if (IS_ERR(anv->mmio_coproc)) { ret = PTR_ERR(anv->mmio_coproc); - goto put_dev; + goto out_detach_genpd; } anv->mmio_nvme = devm_platform_ioremap_resource_byname(pdev, "nvme"); if (IS_ERR(anv->mmio_nvme)) { ret = PTR_ERR(anv->mmio_nvme); - goto put_dev; + goto out_detach_genpd; } if (anv->hw->has_separate_nvmmu) { anv->mmio_nvmmu = devm_platform_ioremap_resource_byname(pdev, "nvmmu"); if (IS_ERR(anv->mmio_nvmmu)) { ret = PTR_ERR(anv->mmio_nvmmu); - goto put_dev; + goto out_detach_genpd; } } else { anv->mmio_nvmmu = anv->mmio_nvme; @@ -1864,7 +2111,7 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) ret = apple_nvme_read_handoff(anv); if (ret) { dev_err_probe(dev, ret, "Invalid live firmware handoff\n"); - goto put_dev; + goto out_detach_genpd; } if (anv->hw->has_lsq_nvmmu) { @@ -1883,40 +2130,41 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) if (IS_ERR(anv->sart)) { ret = dev_err_probe(dev, PTR_ERR(anv->sart), "Failed to initialize SART"); - goto put_dev; + goto out_detach_genpd; } anv->reset = devm_reset_control_array_get_exclusive(anv->dev); if (IS_ERR(anv->reset)) { ret = dev_err_probe(dev, PTR_ERR(anv->reset), "Failed to get reset control"); - goto put_dev; + goto out_detach_genpd; } INIT_WORK(&anv->ctrl.reset_work, apple_nvme_reset_work); INIT_WORK(&anv->remove_work, apple_nvme_remove_dead_ctrl_work); + INIT_WORK(&anv->recovery_work, apple_nvme_recovery_work); spin_lock_init(&anv->lock); ret = apple_nvme_queue_alloc(anv, &anv->adminq); if (ret) - goto put_dev; + goto out_detach_genpd; ret = apple_nvme_queue_alloc(anv, &anv->ioq); if (ret) - goto put_dev; + goto out_detach_genpd; anv->prp_page_pool = dmam_pool_create("prp list page", anv->dev, NVME_CTRL_PAGE_SIZE, NVME_CTRL_PAGE_SIZE, 0); if (!anv->prp_page_pool) { ret = -ENOMEM; - goto put_dev; + goto out_detach_genpd; } anv->prp_small_pool = dmam_pool_create("prp list 256", anv->dev, 256, 256, 0); if (!anv->prp_small_pool) { ret = -ENOMEM; - goto put_dev; + goto out_detach_genpd; } WARN_ON_ONCE(apple_nvme_iod_alloc_size() > PAGE_SIZE); @@ -1924,22 +2172,22 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) mempool_create_kmalloc_pool(1, apple_nvme_iod_alloc_size()); if (!anv->iod_mempool) { ret = -ENOMEM; - goto put_dev; + goto out_detach_genpd; } ret = devm_add_action_or_reset(anv->dev, devm_apple_nvme_mempool_destroy, anv->iod_mempool); if (ret) - goto put_dev; + goto out_detach_genpd; ret = apple_nvme_alloc_tagsets(anv); if (ret) - goto put_dev; + goto out_detach_genpd; ret = devm_request_irq(anv->dev, anv->irq, apple_nvme_irq, 0, "nvme-apple", anv); if (ret) { dev_err_probe(dev, ret, "Failed to request IRQ"); - goto put_dev; + goto out_detach_genpd; } /* @@ -1963,25 +2211,27 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) if (IS_ERR(anv->rtk)) { ret = dev_err_probe(dev, PTR_ERR(anv->rtk), "Failed to initialize RTKit"); - goto put_dev; + goto out_detach_genpd; } ret = devm_add_action_or_reset(dev, apple_nvme_free_rtkit, anv); if (ret) - goto put_dev; + goto out_detach_genpd; ret = nvme_init_ctrl(&anv->ctrl, anv->dev, &nvme_ctrl_ops, NVME_QUIRK_SKIP_CID_GEN | NVME_QUIRK_IDENTIFY_CNS | NVME_QUIRK_ADMIN_PAGE_ALIGN); if (ret) { dev_err_probe(dev, ret, "Failed to initialize nvme_ctrl"); - goto put_dev; + goto out_detach_genpd; } + /* Released by apple_nvme_free_ctrl(), including asynchronous users. */ + kref_get(&anv->ref); + return anv; -put_dev: +out_detach_genpd: apple_nvme_detach_genpd(anv); - put_device(anv->dev); return ERR_PTR(ret); } @@ -2031,9 +2281,13 @@ static void apple_nvme_remove(struct platform_device *pdev) { struct apple_nvme *anv = platform_get_drvdata(pdev); + WRITE_ONCE(anv->removing, true); + apple_nvme_set_reset_ready(anv, false); nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_DELETING); - flush_work(&anv->ctrl.reset_work); + cancel_work_sync(&anv->recovery_work); + cancel_work_sync(&anv->ctrl.reset_work); nvme_stop_ctrl(&anv->ctrl); + cancel_delayed_work_sync(&anv->flush_dwork); nvme_remove_namespaces(&anv->ctrl); apple_nvme_disable(anv, true); if (anv->ctrl.admin_q && !blk_queue_dying(anv->ctrl.admin_q)) { @@ -2047,12 +2301,14 @@ static void apple_nvme_remove(struct platform_device *pdev) } nvme_uninit_ctrl(&anv->ctrl); - if (apple_rtkit_is_running(anv->rtk)) { + /* Post-M4 firmware stays live; see apple_nvme_shutdown(). */ + if (!anv->hw->needs_ioq_registers && apple_rtkit_is_running(anv->rtk)) { apple_rtkit_shutdown(anv->rtk); writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); } + WRITE_ONCE(anv->detached, true); apple_nvme_detach_genpd(anv); } @@ -2060,8 +2316,25 @@ static void apple_nvme_shutdown(struct platform_device *pdev) { struct apple_nvme *anv = platform_get_drvdata(pdev); + WRITE_ONCE(anv->removing, true); + apple_nvme_set_reset_ready(anv, false); flush_delayed_work(&anv->flush_dwork); apple_nvme_disable(anv, true); + cancel_work_sync(&anv->recovery_work); + cancel_work_sync(&anv->ctrl.reset_work); + /* + * Post-M4 ANS firmware crashes when asked to perform the RTKit shutdown + * handshake during final system shutdown, even after all Linux queues are + * quiesced and the controller is left enabled. Leave the co-processor + * running until the imminent platform reset. The controller queues are + * stopped separately; firmware-owned shared buffers remain retained. + */ + if (anv->hw->needs_ioq_registers) { + dev_dbg(anv->dev, + "post-M4 shutdown: leaving RTKit running for system reset\n"); + return; + } + if (apple_rtkit_is_running(anv->rtk)) { apple_rtkit_shutdown(anv->rtk); @@ -2073,15 +2346,36 @@ static int apple_nvme_resume(struct device *dev) { struct apple_nvme *anv = dev_get_drvdata(dev); + if (anv->hw->needs_ioq_registers) { + if (!anv->suspended) + return 0; + WRITE_ONCE(anv->suspended, false); + return nvme_try_sched_reset(&anv->ctrl); + } return nvme_reset_ctrl(&anv->ctrl); } static int apple_nvme_suspend(struct device *dev) { struct apple_nvme *anv = dev_get_drvdata(dev); - int ret = 0; + int ret; - apple_nvme_disable(anv, true); + if (anv->hw->needs_ioq_registers) { + if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_RESETTING)) + return -EBUSY; + WRITE_ONCE(anv->suspended, true); + } + ret = apple_nvme_disable(anv, true); + if (ret) + return ret; + /* + * Only the NVMe controller was disabled. Its owned RTKit session stays + * live, so resume rebuilds queues without clearing CPU_CONTROL.RUN. + */ + if (anv->hw->needs_ioq_registers) { + cancel_work_sync(&anv->recovery_work); + return 0; + } if (apple_rtkit_is_running(anv->rtk)) { ret = apple_rtkit_shutdown(anv->rtk); From 8466b98b64615dee46988db9e664a8f297f637b0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:29:22 +1000 Subject: [PATCH 0363/2680] nvme-apple: Quarantine a controller that fails to stop If clearing CC.EN on a post-M4 controller does not complete, the controller may still read and write its queue memory, the NVMMU TCBs, the PRP lists and the data pages of every request in flight. It cannot be reset, so nothing can make that memory safe to reuse before the SoC itself is reset. Quarantine such a controller instead of tearing it down: disable its interrupt, keep its power domains on, move it to DELETING, mark its namespaces dead, and keep every allocation it may still access. The queue memory and PRP pools move from managed allocations to explicit devres actions, and those actions, like the ones for the IOD mempool and the tag sets (which own the requests in flight), skip the free while the controller is quarantined. The power domains, the RTKit buffers and the devres reference on struct apple_nvme are kept as well. The requests in flight are never completed; their timeouts re-arm and their pages stay with the device. Register access, polling, timeout recovery and RTKit crash handling stop, resume fails with -EIO, and remove() and shutdown() return as soon as the stop has failed. A quarantined controller is recorded by the address of its NVMe registers, and a later probe of the same controller fails with -EBUSY until the next boot. The retained requests keep the block layer calling into this driver, so take a module reference when a controller is quarantined. It is never dropped. It cannot stop a module unload that is already in progress: a quarantine during module removal still leaves the retained state pointing at code that is about to be freed, so kernels that must rule this out need the driver built in. All of this memory is leaked on purpose until reboot. (cherry picked from commit f3f2311bcaeebaea054573cd8ac240f2e93e76af) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 189 +++++++++++++++++++++++++++++--------- 1 file changed, 145 insertions(+), 44 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 5be769506be4c..0cc557bd31791 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -206,6 +207,9 @@ struct apple_nvme { spinlock_t event_lock; bool reset_ready; bool handoff_disabled; + struct list_head quarantine_node; + resource_size_t controller_start; + bool quarantined; bool detached; bool removing; bool abort_reset; @@ -257,6 +261,14 @@ struct apple_nvme { /* Bit in recovery_flags: recovery_work owns the next controller stop. */ #define APPLE_NVME_RECOVERY_PENDING 0 +/* + * Controllers that failed to stop. Unknown DMA ownership survives driver + * unbind; never admit the same controller again until the platform has + * reset. Entries retain their owner. + */ +static DEFINE_MUTEX(apple_nvme_quarantine_lock); +static LIST_HEAD(apple_nvme_quarantines); + static inline void apple_nvme_writeq(struct apple_nvme *anv, u64 value, void __iomem *addr) { @@ -301,7 +313,8 @@ static void apple_nvme_put_resource_owner(void *data) struct apple_nvme *anv = data; WRITE_ONCE(anv->detached, true); - kref_put(&anv->ref, apple_nvme_release); + if (!READ_ONCE(anv->quarantined)) + kref_put(&anv->ref, apple_nvme_release); } static bool apple_nvme_can_adopt_rtkit(struct apple_nvme *anv) @@ -376,7 +389,7 @@ static void apple_nvme_rtkit_crashed(void *cookie, const void *crashlog, size_t struct apple_nvme *anv = cookie; unsigned long flags; - if (READ_ONCE(anv->removing)) + if (READ_ONCE(anv->quarantined) || READ_ONCE(anv->removing)) return; dev_warn(anv->dev, "RTKit crashed; unable to recover without a reboot"); /* @@ -385,7 +398,8 @@ static void apple_nvme_rtkit_crashed(void *cookie, const void *crashlog, size_t * callback scheduling with closing this gate before work cancellation. */ spin_lock_irqsave(&anv->event_lock, flags); - if (anv->reset_ready && !READ_ONCE(anv->removing)) + if (anv->reset_ready && !READ_ONCE(anv->quarantined) && + !READ_ONCE(anv->removing)) nvme_reset_ctrl(&anv->ctrl); spin_unlock_irqrestore(&anv->event_lock, flags); } @@ -438,7 +452,7 @@ static int apple_nvme_sart_dma_setup(void *cookie, struct apple_nvme *anv = cookie; int ret; - if (READ_ONCE(anv->removing)) + if (READ_ONCE(anv->quarantined) || READ_ONCE(anv->removing)) return -ESHUTDOWN; if (!bfr->size) return -EINVAL; @@ -1186,6 +1200,33 @@ static void apple_nvme_disable_legacy(struct apple_nvme *anv, bool shutdown) } } +static void apple_nvme_quarantine(struct apple_nvme *anv, int error) +{ + int i; + + /* Called under disable_lock after both dispatch queues are quiesced. */ + WRITE_ONCE(anv->quarantined, true); + /* + * The retained requests keep blk-mq timeouts and completions pointing + * into this module until reboot. + */ + __module_get(THIS_MODULE); + apple_nvme_set_reset_ready(anv, false); + disable_irq(anv->irq); + if (anv->pd_count > 1) { + for (i = 0; i < anv->pd_count; i++) + pm_runtime_get_noresume(anv->pd_dev[i]); + } + mutex_lock(&apple_nvme_quarantine_lock); + list_add_tail(&anv->quarantine_node, &apple_nvme_quarantines); + mutex_unlock(&apple_nvme_quarantine_lock); + nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_DELETING); + nvme_mark_namespaces_dead(&anv->ctrl); + dev_crit(anv->dev, + "controller did not stop (%d); retaining DMA and requests until platform reset\n", + error); +} + static int apple_nvme_disable_locked(struct apple_nvme *anv, bool shutdown) { enum nvme_ctrl_state state; @@ -1193,6 +1234,8 @@ static int apple_nvme_disable_locked(struct apple_nvme *anv, bool shutdown) int ret; lockdep_assert_held(&anv->disable_lock); + if (READ_ONCE(anv->quarantined)) + return -EIO; state = nvme_ctrl_state(&anv->ctrl); if ((state == NVME_CTRL_LIVE || state == NVME_CTRL_RESETTING) && @@ -1210,12 +1253,14 @@ static int apple_nvme_disable_locked(struct apple_nvme *anv, bool shutdown) /* * CFS and firmware crashes are not DMA fences. Post-M4 does not use - * Delete SQ/CQ, SHN or RTKit sleep; only a completed CC.EN clear stops - * the controller's queue DMA. + * Delete SQ/CQ, SHN or RTKit sleep; only a completed CC.EN clear makes + * the queue and request mappings eligible for release. */ ret = nvme_disable_ctrl(&anv->ctrl, false); - if (ret) - dev_err(anv->dev, "controller did not stop: %d\n", ret); + if (ret) { + apple_nvme_quarantine(anv, ret); + return ret; + } spin_lock_irqsave(&anv->lock, flags); apple_nvme_handle_cq(&anv->ioq, true); @@ -1227,7 +1272,7 @@ static int apple_nvme_disable_locked(struct apple_nvme *anv, bool shutdown) nvme_unquiesce_io_queues(&anv->ctrl); nvme_unquiesce_admin_queue(&anv->ctrl); } - return ret; + return 0; } static int apple_nvme_disable(struct apple_nvme *anv, bool shutdown) @@ -1250,7 +1295,7 @@ static void apple_nvme_recovery_work(struct work_struct *work) * The timeout callback must return before cancellation waits for the * block layer's reference to that request. This worker owns the stop. */ - if (READ_ONCE(anv->removing)) + if (READ_ONCE(anv->quarantined) || READ_ONCE(anv->removing)) goto done; ret = apple_nvme_disable(anv, false); if (ret) @@ -1275,6 +1320,8 @@ static enum blk_eh_timer_return apple_nvme_timeout(struct request *req) if (blk_mq_request_completed(req)) return BLK_EH_DONE; + if (READ_ONCE(anv->quarantined)) + return BLK_EH_RESET_TIMER; cc = readl(anv->mmio_nvme + NVME_REG_CC); csts = readl(anv->mmio_nvme + NVME_REG_CSTS); @@ -1305,7 +1352,8 @@ static enum blk_eh_timer_return apple_nvme_timeout(struct request *req) state == NVME_CTRL_DELETING_NOIO) && READ_ONCE(anv->removing)) { nvme_req(req)->flags |= NVME_REQ_CANCELLED; - apple_nvme_disable(anv, true); + if (apple_nvme_disable(anv, true)) + return BLK_EH_RESET_TIMER; return BLK_EH_DONE; } /* @@ -1389,7 +1437,7 @@ static int apple_nvme_poll(struct blk_mq_hw_ctx *hctx, bool found; unsigned long flags; - if (READ_ONCE(anv->detached)) + if (READ_ONCE(anv->quarantined) || READ_ONCE(anv->detached)) return 0; spin_lock_irqsave(&anv->lock, flags); found = apple_nvme_poll_cq(q, iob); @@ -1444,7 +1492,7 @@ static void apple_nvme_reset_work(struct work_struct *work) mutex_lock(&anv->disable_lock); phase_locked = true; } - if (READ_ONCE(anv->removing)) { + if (READ_ONCE(anv->quarantined) || READ_ONCE(anv->removing)) { if (phase_locked) mutex_unlock(&anv->disable_lock); return; @@ -1705,7 +1753,8 @@ static void apple_nvme_reset_work(struct work_struct *work) if (anv->hw->needs_ioq_registers) { mutex_lock(&anv->disable_lock); phase_locked = true; - if (READ_ONCE(anv->removing) || READ_ONCE(anv->abort_reset)) { + if (READ_ONCE(anv->quarantined) || READ_ONCE(anv->removing) || + READ_ONCE(anv->abort_reset)) { ret = -ECANCELED; goto out; } @@ -1765,8 +1814,7 @@ static void apple_nvme_reset_work(struct work_struct *work) mutex_unlock(&anv->disable_lock); dev_warn(anv->ctrl.device, "Reset failure status: %d\n", ret); nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_DELETING); - apple_nvme_disable(anv, false); - if (READ_ONCE(anv->removing)) + if (apple_nvme_disable(anv, false) || READ_ONCE(anv->removing)) return; nvme_get_ctrl(&anv->ctrl); nvme_mark_namespaces_dead(&anv->ctrl); @@ -1787,7 +1835,7 @@ static int apple_nvme_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val) { struct apple_nvme *anv = ctrl_to_apple_nvme(ctrl); - if (READ_ONCE(anv->detached)) + if (READ_ONCE(anv->quarantined) || READ_ONCE(anv->detached)) return -ENODEV; *val = readl(anv->mmio_nvme + off); return 0; @@ -1797,7 +1845,7 @@ static int apple_nvme_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val) { struct apple_nvme *anv = ctrl_to_apple_nvme(ctrl); - if (READ_ONCE(anv->detached)) + if (READ_ONCE(anv->quarantined) || READ_ONCE(anv->detached)) return -ENODEV; writel(val, anv->mmio_nvme + off); return 0; @@ -1807,7 +1855,7 @@ static int apple_nvme_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val) { struct apple_nvme *anv = ctrl_to_apple_nvme(ctrl); - if (READ_ONCE(anv->detached)) + if (READ_ONCE(anv->quarantined) || READ_ONCE(anv->detached)) return -ENODEV; *val = readq(anv->mmio_nvme + off); return 0; @@ -1850,7 +1898,11 @@ static void apple_nvme_async_probe(void *data, async_cookie_t cookie) static void devm_apple_nvme_put_tag_set(void *data) { - blk_mq_free_tag_set(data); + struct blk_mq_tag_set *set = data; + struct apple_nvme *anv = queue_to_apple_nvme(set->driver_data); + + if (!READ_ONCE(anv->quarantined)) + blk_mq_free_tag_set(set); } static int apple_nvme_alloc_tagsets(struct apple_nvme *anv) @@ -1902,15 +1954,40 @@ static int apple_nvme_alloc_tagsets(struct apple_nvme *anv) return 0; } +static void apple_nvme_free_queue(void *data) +{ + struct apple_nvme_queue *q = data; + struct apple_nvme *anv = queue_to_apple_nvme(q); + unsigned int depth = apple_nvme_queue_depth(q); + size_t sq_size = anv->hw->has_lsq_nvmmu ? + depth * sizeof(struct nvme_command) : depth << APPLE_NVME_IOSQES; + + if (READ_ONCE(anv->quarantined)) + return; + if (q->tcbs) + dma_free_coherent(anv->dev, + anv->hw->max_queue_depth * sizeof(struct apple_nvmmu_tcb), + q->tcbs, q->tcb_dma_addr); + if (q->sqes) + dma_free_coherent(anv->dev, sq_size, q->sqes, q->sq_dma_addr); + if (q->cqes) + dma_free_coherent(anv->dev, depth * sizeof(struct nvme_completion), + q->cqes, q->cq_dma_addr); +} + static int apple_nvme_queue_alloc(struct apple_nvme *anv, struct apple_nvme_queue *q) { unsigned int depth = apple_nvme_queue_depth(q); size_t iosq_size; + int ret; - q->cqes = dmam_alloc_coherent(anv->dev, - depth * sizeof(struct nvme_completion), - &q->cq_dma_addr, GFP_KERNEL); + ret = devm_add_action_or_reset(anv->dev, apple_nvme_free_queue, q); + if (ret) + return ret; + q->cqes = dma_alloc_coherent(anv->dev, + depth * sizeof(struct nvme_completion), + &q->cq_dma_addr, GFP_KERNEL); if (!q->cqes) return -ENOMEM; @@ -1919,8 +1996,8 @@ static int apple_nvme_queue_alloc(struct apple_nvme *anv, else iosq_size = depth << APPLE_NVME_IOSQES; - q->sqes = dmam_alloc_coherent(anv->dev, iosq_size, - &q->sq_dma_addr, GFP_KERNEL); + q->sqes = dma_alloc_coherent(anv->dev, iosq_size, + &q->sq_dma_addr, GFP_KERNEL); if (!q->sqes) return -ENOMEM; @@ -1929,10 +2006,10 @@ static int apple_nvme_queue_alloc(struct apple_nvme *anv, * We need the maximum queue depth here because the NVMMU only * has a single depth configuration shared between both queues. */ - q->tcbs = dmam_alloc_coherent(anv->dev, - anv->hw->max_queue_depth * - sizeof(struct apple_nvmmu_tcb), - &q->tcb_dma_addr, GFP_KERNEL); + q->tcbs = dma_alloc_coherent(anv->dev, + anv->hw->max_queue_depth * + sizeof(struct apple_nvmmu_tcb), + &q->tcb_dma_addr, GFP_KERNEL); if (!q->tcbs) return -ENOMEM; } @@ -1949,7 +2026,7 @@ static void apple_nvme_detach_genpd(struct apple_nvme *anv) { int i; - if (anv->pd_count <= 1) + if (anv->pd_count <= 1 || READ_ONCE(anv->quarantined)) return; for (i = anv->pd_count - 1; i >= 0; i--) { @@ -2000,9 +2077,15 @@ static int apple_nvme_attach_genpd(struct apple_nvme *anv) return 0; } -static void devm_apple_nvme_mempool_destroy(void *data) +static void apple_nvme_free_pools(void *data) { - mempool_destroy(data); + struct apple_nvme *anv = data; + + if (READ_ONCE(anv->quarantined)) + return; + mempool_destroy(anv->iod_mempool); + dma_pool_destroy(anv->prp_small_pool); + dma_pool_destroy(anv->prp_page_pool); } static void apple_nvme_free_rtkit(void *data) @@ -2043,10 +2126,23 @@ static void apple_nvme_flush_work(struct work_struct *work) static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct apple_nvme *anv; + struct resource *resource; + struct apple_nvme *anv, *held; bool inherited_rtkit; int ret; + resource = platform_get_resource_byname(pdev, IORESOURCE_MEM, "nvme"); + if (!resource) + return ERR_PTR(-EINVAL); + mutex_lock(&apple_nvme_quarantine_lock); + list_for_each_entry(held, &apple_nvme_quarantines, quarantine_node) { + if (held->controller_start == resource->start) { + mutex_unlock(&apple_nvme_quarantine_lock); + return ERR_PTR(dev_err_probe(dev, -EBUSY, + "DMA ownership is uncertain until platform reset\n")); + } + } + mutex_unlock(&apple_nvme_quarantine_lock); anv = kzalloc_obj(*anv); if (!anv) return ERR_PTR(-ENOMEM); @@ -2055,6 +2151,8 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) kref_init(&anv->ref); mutex_init(&anv->disable_lock); spin_lock_init(&anv->event_lock); + INIT_LIST_HEAD(&anv->quarantine_node); + anv->controller_start = resource->start; ret = devm_add_action_or_reset(dev, apple_nvme_put_resource_owner, anv); if (ret) return ERR_PTR(ret); @@ -2152,16 +2250,19 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) if (ret) goto out_detach_genpd; - anv->prp_page_pool = dmam_pool_create("prp list page", anv->dev, - NVME_CTRL_PAGE_SIZE, - NVME_CTRL_PAGE_SIZE, 0); + ret = devm_add_action_or_reset(dev, apple_nvme_free_pools, anv); + if (ret) + goto out_detach_genpd; + anv->prp_page_pool = dma_pool_create("prp list page", anv->dev, + NVME_CTRL_PAGE_SIZE, + NVME_CTRL_PAGE_SIZE, 0); if (!anv->prp_page_pool) { ret = -ENOMEM; goto out_detach_genpd; } anv->prp_small_pool = - dmam_pool_create("prp list 256", anv->dev, 256, 256, 0); + dma_pool_create("prp list 256", anv->dev, 256, 256, 0); if (!anv->prp_small_pool) { ret = -ENOMEM; goto out_detach_genpd; @@ -2174,11 +2275,6 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) ret = -ENOMEM; goto out_detach_genpd; } - ret = devm_add_action_or_reset(anv->dev, - devm_apple_nvme_mempool_destroy, anv->iod_mempool); - if (ret) - goto out_detach_genpd; - ret = apple_nvme_alloc_tagsets(anv); if (ret) goto out_detach_genpd; @@ -2289,7 +2385,9 @@ static void apple_nvme_remove(struct platform_device *pdev) nvme_stop_ctrl(&anv->ctrl); cancel_delayed_work_sync(&anv->flush_dwork); nvme_remove_namespaces(&anv->ctrl); - apple_nvme_disable(anv, true); + /* A quarantined controller keeps everything it may still DMA into. */ + if (apple_nvme_disable(anv, true)) + return; if (anv->ctrl.admin_q && !blk_queue_dying(anv->ctrl.admin_q)) { /* * If the controller was reset during removal, it's possible @@ -2319,7 +2417,8 @@ static void apple_nvme_shutdown(struct platform_device *pdev) WRITE_ONCE(anv->removing, true); apple_nvme_set_reset_ready(anv, false); flush_delayed_work(&anv->flush_dwork); - apple_nvme_disable(anv, true); + if (apple_nvme_disable(anv, true)) + return; cancel_work_sync(&anv->recovery_work); cancel_work_sync(&anv->ctrl.reset_work); /* @@ -2347,6 +2446,8 @@ static int apple_nvme_resume(struct device *dev) struct apple_nvme *anv = dev_get_drvdata(dev); if (anv->hw->needs_ioq_registers) { + if (READ_ONCE(anv->quarantined)) + return -EIO; if (!anv->suspended) return 0; WRITE_ONCE(anv->suspended, false); From ece22e77f90a92c9cf1be1551983044be27a1fdf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:29:29 +1000 Subject: [PATCH 0364/2680] nvme-apple: Do not defer flushes on post-M4 controllers The driver completes a flush at once, and sends the real Flush command later from a delayed work, when the previous flush was less than flush_interval (1000 ms by default) ago. Until that work runs, the filesystem treats data as stable that may still be in the controller's volatile write cache. Keep that behaviour on the older controllers, where it was added because their flushes are slow, but always send flushes to post-M4 controllers and complete them when the controller does. The FUA emulation added later in this series turns every FUA write on T8140 into a write followed by a flush, and completing that flush early would defeat it. The module parameter description now says that it is ignored on post-M4 controllers. (cherry picked from commit bb41ad2b07216dfcb40fcd339a01148bc428f9a3) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 0cc557bd31791..e681f0875080a 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -286,7 +286,8 @@ static inline void apple_nvme_writeq(struct apple_nvme *anv, u64 value, unsigned int flush_interval = 1000; module_param(flush_interval, uint, 0644); -MODULE_PARM_DESC(flush_interval, "Grace period in msecs between flushes"); +MODULE_PARM_DESC(flush_interval, + "Grace period in msecs between flushes (ignored on post-M4 ANS)"); static_assert(sizeof(struct nvme_command) == 64); static_assert(sizeof(struct apple_nvmmu_tcb) == 128); @@ -1028,7 +1029,13 @@ static int apple_nvme_remove_sq(struct apple_nvme *anv) static bool apple_nvme_delayed_flush(struct apple_nvme *anv, struct nvme_ns *ns, struct request *req) { - if (!anv->flush_interval || req_op(req) != REQ_OP_FLUSH) + /* + * A flush completion is a persistence boundary for the filesystem. + * Post-M4 internal-root storage must submit it to the controller and + * wait for its completion, even when the legacy grace period is set. + */ + if (anv->hw->needs_ioq_registers || + !anv->flush_interval || req_op(req) != REQ_OP_FLUSH) return false; if (delayed_work_pending(&anv->flush_dwork)) return true; From dbdd4090b95381851666cd2bccb485a294f521bb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:29:38 +1000 Subject: [PATCH 0365/2680] nvme: add NVME_QUIRK_BROKEN_FUA Some controllers report a volatile write cache and accept writes with the FUA bit set, but a FUA write is not reliably durable when it completes, while explicit Flush commands are. Add a quirk that keeps the write cache advertised but clears BLK_FEAT_FUA. The block layer then sends each FUA write as a plain write followed by a flush. The first user is the Apple ANS2 controller on T8140. (cherry picked from commit 6072c70267f5fe704e53fb8f70d69fd1c2e34f34) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/core.c | 2 ++ drivers/nvme/host/nvme.h | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index a085e47f0d03f..a20eabc7a5ed4 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -2465,6 +2465,8 @@ static int nvme_update_ns_info_block(struct nvme_ns *ns, lim.features |= BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA; else lim.features &= ~(BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA); + if (ns->ctrl->quirks & NVME_QUIRK_BROKEN_FUA) + lim.features &= ~BLK_FEAT_FUA; if (info->is_rotational) lim.features |= BLK_FEAT_ROTATIONAL; diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 47e52a2b61c84..ba40d93750f69 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -183,6 +183,11 @@ enum nvme_quirks { * Admin queue DMA buffers must be page aligned */ NVME_QUIRK_ADMIN_PAGE_ALIGN = (1 << 23), + + /* + * Emulate Force Unit Access with explicit flush commands. + */ + NVME_QUIRK_BROKEN_FUA = (1 << 24), }; static inline char *nvme_quirk_name(enum nvme_quirks q) @@ -236,6 +241,8 @@ static inline char *nvme_quirk_name(enum nvme_quirks q) return "dmapool_align_512"; case NVME_QUIRK_ADMIN_PAGE_ALIGN: return "admin_page_align"; + case NVME_QUIRK_BROKEN_FUA: + return "broken_fua"; } return "unknown"; From 30b319fc9ba2c8a147c45923b22dd9a2b6b90aa0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:29:46 +1000 Subject: [PATCH 0366/2680] nvme-apple: Emulate FUA with post-flushes on T8140 In testing on T8140 (J700) with a Btrfs root on the internal NVMe, a 16 MiB overwrite that had been fsynced reverted to its previous contents after an abrupt system reset (a reset over USB-C VDM, which gives the controller no shutdown), while data written before explicit Flush commands survived. With FUA writes turned into write plus flush the same overwrite survived the same reset. The number of trials was not recorded, and the controller's VWC field was not captured; if it reports no volatile write cache the quirk changes nothing. Apply NVME_QUIRK_BROKEN_FUA to T8140 through its hardware data, so that the block layer follows every FUA write with a flush. Flushes are no longer deferred on post-M4 controllers, so these reach the controller. This is not a claim about other post-M4 controllers; T8132 keeps native FUA. An abrupt reset is also not a power loss, which was not tested. (cherry picked from commit 5390b50a8bd16f13bf59b6e4adaab1abeaf0936f) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index e681f0875080a..c7d09f1980583 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -196,6 +196,7 @@ struct apple_nvme_hw { bool has_separate_nvmmu; bool needs_ioq_registers; u32 max_queue_depth; + unsigned long quirks; }; struct apple_nvme { @@ -2323,7 +2324,7 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) ret = nvme_init_ctrl(&anv->ctrl, anv->dev, &nvme_ctrl_ops, NVME_QUIRK_SKIP_CID_GEN | NVME_QUIRK_IDENTIFY_CNS | - NVME_QUIRK_ADMIN_PAGE_ALIGN); + NVME_QUIRK_ADMIN_PAGE_ALIGN | anv->hw->quirks); if (ret) { dev_err_probe(dev, ret, "Failed to initialize nvme_ctrl"); goto out_detach_genpd; @@ -2516,10 +2517,24 @@ static const struct apple_nvme_hw apple_nvme_t8132_hw = { .max_queue_depth = 64, }; +static const struct apple_nvme_hw apple_nvme_t8140_hw = { + .has_lsq_nvmmu = true, + .has_separate_nvmmu = true, + .needs_ioq_registers = true, + .max_queue_depth = 64, + /* + * An fsynced overwrite completed and later reverted after an abrupt + * reset; explicit flushes persisted the preceding data. Let the block + * layer follow each FUA write with a flush. + */ + .quirks = NVME_QUIRK_BROKEN_FUA, +}; + static const struct of_device_id apple_nvme_of_match[] = { { .compatible = "apple,t8015-nvme-ans2", .data = &apple_nvme_t8015_hw }, { .compatible = "apple,t8103-nvme-ans2", .data = &apple_nvme_t8103_hw }, { .compatible = "apple,t8132-nvme-ans2", .data = &apple_nvme_t8132_hw }, + { .compatible = "apple,t8140-nvme-ans2", .data = &apple_nvme_t8140_hw }, { .compatible = "apple,nvme-ans2", .data = &apple_nvme_t8103_hw }, {}, }; From 16a2fbc0da4117b8688575b91ffbcdee0ecdb79f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 02:55:40 +1000 Subject: [PATCH 0367/2680] nvme-apple: Let post-M4 reset recovery finish while I/O is frozen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the I/O queue is re-created, reset_work unquiesces the I/O queues and waits in nvme_wait_freeze() for the requests that entered before the reset, while holding disable_lock. If one of them times out while the controller is CONNECTING, the timeout handler marks the reset aborted and queues recovery_work, which needs disable_lock to stop the controller and cancel the request, so neither side can proceed. A timeout during removal and apple_nvme_shutdown() also stop the controller under this lock, so they wait behind reset_work as well. Release disable_lock while waiting and take it again before resizing the tag set and publishing LIVE. If the reset was aborted, or the controller was removed or quarantined meanwhile, unfreeze and take the reset failure path. Releasing the lock alone does not let the wait finish. recovery_work stops the controller without unquiescing the queues, and the requests it cancels, which are retryable, are requeued onto the quiesced queues. An aborted initialization gets no follow-up reset, so after its stop, unquiesce the queues under the same lock: the requeued requests then reach the disabled queues and fail with BLK_STS_IOERR, as the PCI driver lets them fail after a timeout while connecting. The failure path removes the controller, so these requests could not have completed anyway. A controller whose stop fails is quarantined and keeps its requests until reboot, so the wait could still never finish. Wait in steps of the I/O timeout, and stop waiting once the controller is quarantined or the reset aborted. The failure path then stops the controller, or leaves a quarantined one as it is. Removing a controller that cannot be stopped still hangs: the quarantine keeps its requests, and del_gendisk() waits for them. That needs a separate decision. Legacy controllers are unchanged. Compile-tested only; not run on hardware. [ryan: adapted to 7.2.10 — nvme_wait_freeze_timeout() lost its timeout argument (commit 23b6d2cbf75f ("nvme: remove redundant timeout argument from nvme_wait_freeze_timeout")); each wait still lasts ctrl->io_timeout, NVME_IO_TIMEOUT by default, so the loop keeps its period.] Fixes: 7296c5865423 ("nvme-apple: Serialize post-M4 disable, timeout recovery and reset") (cherry picked from commit 517d6a19771bf4f3c7b5eb9d5b89d39c7efdacf3) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 48 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index c7d09f1980583..b074c129e3948 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -1305,7 +1305,19 @@ static void apple_nvme_recovery_work(struct work_struct *work) */ if (READ_ONCE(anv->quarantined) || READ_ONCE(anv->removing)) goto done; - ret = apple_nvme_disable(anv, false); + mutex_lock(&anv->disable_lock); + ret = apple_nvme_disable_locked(anv, false); + /* + * Without restart, this stop aborts an initialization, and no reset + * follows. The requests it cancelled have been requeued; let them + * reach the stopped queues and fail, so that reset_work does not wait + * for them forever. + */ + if (!ret && !restart) { + nvme_unquiesce_io_queues(&anv->ctrl); + nvme_unquiesce_admin_queue(&anv->ctrl); + } + mutex_unlock(&anv->disable_lock); if (ret) goto done; clear_bit(APPLE_NVME_RECOVERY_PENDING, &anv->recovery_flags); @@ -1486,6 +1498,19 @@ static void apple_nvme_init_queue(struct apple_nvme_queue *q) apple_nvme_enable_queue(q); } +/* + * Wait for the requests that entered before a post-M4 reset. Give up once + * the reset has been aborted, whose failure path stops the controller, or + * the controller has been quarantined, which keeps its requests. + */ +static void apple_nvme_wait_freeze(struct apple_nvme *anv) +{ + while (!nvme_wait_freeze_timeout(&anv->ctrl)) { + if (READ_ONCE(anv->abort_reset) || READ_ONCE(anv->quarantined)) + return; + } +} + static void apple_nvme_reset_work(struct work_struct *work) { unsigned int nr_io_queues = 1; @@ -1793,7 +1818,26 @@ static void apple_nvme_reset_work(struct work_struct *work) anv->ctrl.queue_count = nr_io_queues + 1; nvme_unquiesce_io_queues(&anv->ctrl); - nvme_wait_freeze(&anv->ctrl); + if (phase_locked) { + /* + * Do not hold disable_lock while waiting for the requests that + * entered before the reset: if one of them times out, stopping + * the controller needs the lock. That stop fails the requests, + * unless it quarantines the controller, which + * apple_nvme_wait_freeze() gives up on. + */ + mutex_unlock(&anv->disable_lock); + apple_nvme_wait_freeze(anv); + mutex_lock(&anv->disable_lock); + if (READ_ONCE(anv->quarantined) || READ_ONCE(anv->removing) || + READ_ONCE(anv->abort_reset)) { + nvme_unfreeze(&anv->ctrl); + ret = -ECANCELED; + goto out; + } + } else { + nvme_wait_freeze(&anv->ctrl); + } blk_mq_update_nr_hw_queues(&anv->tagset, 1); nvme_unfreeze(&anv->ctrl); From 0270e5597925e46a7fb68d1a771e6ada8da5bcba Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 02:56:41 +1000 Subject: [PATCH 0368/2680] nvme-apple: Release disable_lock before nvme_start_ctrl() reset_work publishes LIVE and then calls nvme_start_ctrl() with disable_lock held. When the controller reports supported asynchronous events in OAES, nvme_start_ctrl() sends Set Features (Asynchronous Event Configuration) and waits for it. If that command times out, the controller is LIVE, so the timeout handler moves it to RESETTING and queues recovery_work, which blocks on disable_lock while reset_work waits for the command that recovery_work would cancel. A timeout during removal blocks on the lock the same way. This driver has no submit_async_event() callback, so a controller that reported asynchronous events would also fault in nvme_async_event_work(). The OAES field of the post-M4 controllers has not been read here, though, so do not rely on the command never being sent: release disable_lock once LIVE has been published, before nvme_start_ctrl(). Nothing in nvme_start_ctrl() programs the controller, which is what the lock keeps apart from a stop. After the lock is released, a recovery can stop the controller before nvme_start_ctrl() unquiesces the I/O queues. The unquiesce then undoes the stop's quiesce, and requests fail on the stopped queues until the next reset instead of waiting for it. The PCI driver has the same window. A request can time out in it only while nvme_start_ctrl() waits for the command above: the requests that entered before the reset have completed by then, and later ones have been in flight for less than the I/O timeout. Legacy controllers never take disable_lock and are unchanged. Compile-tested only; not run on hardware. Fixes: 7296c5865423 ("nvme-apple: Serialize post-M4 disable, timeout recovery and reset") (cherry picked from commit f5be53a3aa91ac071d155e41e64903713177938c) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index b074c129e3948..91e03440691e7 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -1848,11 +1848,19 @@ static void apple_nvme_reset_work(struct work_struct *work) goto out_remove_sq; } + /* + * nvme_start_ctrl() sends Set Features if the controller reports + * asynchronous events. If that times out, the controller is live and + * recovery_work needs disable_lock to stop it. As in pci.c, a stop + * that lands before nvme_start_ctrl() unquiesces the I/O queues loses + * its quiesce, and requests then fail on the stopped queues until the + * next reset instead of waiting for it. + */ + if (phase_locked) + mutex_unlock(&anv->disable_lock); nvme_start_ctrl(&anv->ctrl); dev_dbg(anv->dev, "ANS boot and NVMe init completed."); - if (phase_locked) - mutex_unlock(&anv->disable_lock); return; out_remove_sq: From 16f83d0e45264f638a356ff47508d70b10d910e1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 21:25:01 +1000 Subject: [PATCH 0369/2680] nvme-apple: Credit Yureka Lilian's T8132 bring-up Commit 3028c4021c61 ("nvme-apple: Support the post-M4 register layout") states that its T8132 entry follows m1n1, but does not say that m1n1's T8132 NVMe support, including the I/O queue register sequence the commit adopts, is Yureka Lilian's work. Credit her next to the hardware data it informs, and link her m1n1 commit. She has also posted the equivalent Linux support for T8132; it is linked here so the two can be reconciled when that series lands. No functional change. Link: https://github.com/AsahiLinux/m1n1/commit/53f8ee9b54ba52b7f87e607e8206a8c827f06b04 Link: https://cyberchaos.dev/yuka/linux/commit/42839287f85c8c36b63579371af533fcd1b8c970 (cherry picked from commit f35bab7d5983e50b5555b355f99dc084396e7165) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 91e03440691e7..4421617009540 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -2562,6 +2562,10 @@ static const struct apple_nvme_hw apple_nvme_t8103_hw = { .max_queue_depth = 64, }; +/* + * The post-M4 (T8132 and later) I/O queue setup follows Yureka Lilian's T8132 + * support in m1n1 (commit 53f8ee9b54ba, "nvme: support T8132"). + */ static const struct apple_nvme_hw apple_nvme_t8132_hw = { .has_lsq_nvmmu = true, .has_separate_nvmmu = true, From 9f98b5fbd3d640d93f75960a9ff1119a5dcab6ca Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 00:06:24 -0400 Subject: [PATCH 0370/2680] nvme-pci: order tunneled non-posted queue accesses Signed-off-by: DjDeveloperr Signed-off-by: Ryan Murray (cherry picked from commit fa72d43fb7d709c89bf6472c2fa0671f923ccd9f) --- drivers/nvme/host/pci.c | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 8c9826e72bd23..3464dc27a473d 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -728,8 +729,12 @@ static inline void nvme_write_sq_db(struct nvme_queue *nvmeq, bool write_sq) } if (nvme_dbbuf_update_and_check_event(nvmeq->sq_tail, - nvmeq->dbbuf_sq_db, nvmeq->dbbuf_sq_ei)) + nvmeq->dbbuf_sq_db, nvmeq->dbbuf_sq_ei)) { writel(nvmeq->sq_tail, nvmeq->q_db); + if (pci_resource_flags(to_pci_dev(nvmeq->dev->dev), 0) & + IORESOURCE_MEM_NONPOSTED) + readl(nvmeq->dev->bar + NVME_REG_CSTS); + } nvmeq->last_sq_tail = nvmeq->sq_tail; } @@ -1549,8 +1554,12 @@ static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq) u16 head = nvmeq->cq_head; if (nvme_dbbuf_update_and_check_event(head, nvmeq->dbbuf_cq_db, - nvmeq->dbbuf_cq_ei)) + nvmeq->dbbuf_cq_ei)) { writel(head, nvmeq->q_db + nvmeq->dev->db_stride); + if (pci_resource_flags(to_pci_dev(nvmeq->dev->dev), 0) & + IORESOURCE_MEM_NONPOSTED) + readl(nvmeq->dev->bar + NVME_REG_CSTS); + } } static inline struct blk_mq_tags *nvme_queue_tagset(struct nvme_queue *nvmeq) @@ -1632,6 +1641,21 @@ static irqreturn_t nvme_irq(int irq, void *data) { struct nvme_queue *nvmeq = data; DEFINE_IO_COMP_BATCH(iob); + unsigned int retry; + + /* + * PCIe-C can deliver the tunneled MSI before the preceding completion + * write is visible to the CPU. Wait briefly for the CQ phase update on + * BARs marked non-posted by the Apple host controller. + */ + if (pci_resource_flags(to_pci_dev(nvmeq->dev->dev), 0) & + IORESOURCE_MEM_NONPOSTED) { + for (retry = 0; retry < 1000 && !nvme_cqe_pending(nvmeq); + retry++) { + udelay(1); + dma_rmb(); + } + } if (nvme_poll_cq(nvmeq, &iob)) { if (!rq_list_empty(&iob.req_list)) @@ -2319,7 +2343,10 @@ static int nvme_remap_bar(struct nvme_dev *dev, unsigned long size) return -ENOMEM; if (dev->bar) iounmap(dev->bar); - dev->bar = ioremap(pci_resource_start(pdev, 0), size); + if (pci_resource_flags(pdev, 0) & IORESOURCE_MEM_NONPOSTED) + dev->bar = ioremap_np(pci_resource_start(pdev, 0), size); + else + dev->bar = ioremap(pci_resource_start(pdev, 0), size); if (!dev->bar) { dev->bar_mapped_size = 0; return -ENOMEM; From bc917dbae2fa6006158c95b87bbfcb9f2eae55f5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 14:57:22 -0400 Subject: [PATCH 0371/2680] nvme-apple: Drive the NVMMU inline AES-XTS with wrapped FileVault keys Drive the storage controller's inline AES-XTS with the wrapped key so an encrypted APFS volume is read and written with no plaintext key material on the application processor. Register a blk-crypto profile for hardware-wrapped AES-256-XTS keys with 4096-byte data units on controllers with the linear submission queue NVMMU, load the wrapped key and the DUN into the request's TCB, and scrub them from the TCB when the request is unmapped. Let NVMe core register a controller's crypto profile on each namespace queue. Export bio_crypt_dun_is_contiguous() for modular FileVault users. [ryan: split out of commit e7db648a8064 ("soc: apple: add production SEP FileVault integration"): this commit carries its nvme-apple, nvme core and blk-crypto parts so that 100-sep can stack on 050-nvme; the SEP parts stay with 100-sep. Adapted to 7.2.10: bio_crypt_set_ctx(), blk_crypto_init_key() and blk_crypto_start_using_key() are now exported by the base, so only the bio_crypt_dun_is_contiguous() export remains. blk_crypto_register() and the profile fields are unchanged in 7.2.] Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh Signed-off-by: Ryan Murray --- block/blk-crypto.c | 1 + drivers/nvme/host/Kconfig | 1 + drivers/nvme/host/apple.c | 67 +++++++++++++++++++++++++++++++++++++-- drivers/nvme/host/core.c | 5 +++ drivers/nvme/host/nvme.h | 3 ++ 5 files changed, 75 insertions(+), 2 deletions(-) diff --git a/block/blk-crypto.c b/block/blk-crypto.c index 15e25e41b1669..c0dd0725da5b5 100644 --- a/block/blk-crypto.c +++ b/block/blk-crypto.c @@ -187,6 +187,7 @@ bool bio_crypt_dun_is_contiguous(const struct bio_crypt_ctx *bc, /* If the DUN wrapped through 0, don't treat it as contiguous. */ return carry == 0; } +EXPORT_SYMBOL_GPL(bio_crypt_dun_is_contiguous); /* * Checks that two bio crypt contexts are compatible - i.e. that diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig index 31974c7dd20c9..de0fa14096286 100644 --- a/drivers/nvme/host/Kconfig +++ b/drivers/nvme/host/Kconfig @@ -127,6 +127,7 @@ config NVME_APPLE depends on OF && BLOCK depends on APPLE_RTKIT && APPLE_SART depends on ARCH_APPLE || COMPILE_TEST + select BLK_INLINE_ENCRYPTION select NVME_CORE help This provides support for the NVMe controller embedded in Apple SoCs diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 4421617009540..8adaf00e015f0 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -10,6 +10,8 @@ */ #include +#include +#include #include #include #include @@ -257,6 +259,7 @@ struct apple_nvme { unsigned long flush_interval; unsigned long last_flush; struct delayed_work flush_dwork; + struct blk_crypto_profile crypto_profile; }; /* Bit in recovery_flags: recovery_work owns the next controller stop. */ @@ -293,6 +296,9 @@ MODULE_PARM_DESC(flush_interval, static_assert(sizeof(struct nvme_command) == 64); static_assert(sizeof(struct apple_nvmmu_tcb) == 128); +#define APPLE_NVME_CRYPTO_KEY_SIZE 64 +#define APPLE_NVME_CRYPTO_DATA_UNIT_SIZE 4096 + static inline struct apple_nvme *ctrl_to_apple_nvme(struct nvme_ctrl *ctrl) { return container_of(ctrl, struct apple_nvme, ctrl); @@ -575,9 +581,12 @@ static void apple_nvme_submit_cmd_t8015(struct apple_nvme_queue *q, static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, - struct nvme_command *cmd) + struct nvme_command *cmd, + struct request *req) { struct apple_nvme *anv = queue_to_apple_nvme(q); + const u8 *key = NULL; + u64 dun = 0; u32 tag = nvme_tag_from_cid(cmd->common.command_id); struct apple_nvmmu_tcb *tcb = &q->tcbs[tag]; @@ -594,6 +603,22 @@ static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, else tcb->dma_flags = APPLE_ANS_TCB_DMA_FROM_DEVICE; + if (unlikely(req->crypt_ctx)) { + const struct blk_crypto_key *blk_key = req->crypt_ctx->bc_key; + + key = blk_key->bytes; + dun = req->crypt_ctx->bc_dun[0]; + } + + if (key) { + __le64 dun_le = cpu_to_le64(dun); + + tcb->dma_flags |= BIT(2); + memcpy(tcb->aes_iv, &dun_le, sizeof(dun_le)); + memset(tcb->_aes_unk, 0, sizeof(tcb->_aes_unk)); + memcpy(tcb->_aes_unk, key, 4); + memcpy(tcb->_aes_unk + 16, key + 16, 48); + } memcpy(&q->sqes[tag], cmd, sizeof(*cmd)); /* @@ -846,9 +871,18 @@ static __always_inline void apple_nvme_unmap_rq(struct request *req) { struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); struct apple_nvme *anv = queue_to_apple_nvme(iod->q); + struct apple_nvmmu_tcb *tcb; + u32 tag; if (blk_rq_nr_phys_segments(req)) apple_nvme_unmap_data(anv, req); + tag = nvme_tag_from_cid(iod->cmd.common.command_id); + tcb = &iod->q->tcbs[tag]; + if (unlikely(tcb->dma_flags & BIT(2))) { + memzero_explicit(tcb->aes_iv, sizeof(tcb->aes_iv)); + memzero_explicit(tcb->_aes_unk, sizeof(tcb->_aes_unk)); + tcb->dma_flags &= ~BIT(2); + } } static void apple_nvme_complete_rq(struct request *req) @@ -1080,6 +1114,20 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, ret = nvme_setup_cmd(ns, req); if (ret) return ret; + if (unlikely(req->crypt_ctx)) { + const struct blk_crypto_key *key = req->crypt_ctx->bc_key; + + if (WARN_ON_ONCE(key->size != APPLE_NVME_CRYPTO_KEY_SIZE || + key->crypto_cfg.crypto_mode != + BLK_ENCRYPTION_MODE_AES_256_XTS || + key->crypto_cfg.key_type != + BLK_CRYPTO_KEY_TYPE_HW_WRAPPED || + key->crypto_cfg.data_unit_size != + APPLE_NVME_CRYPTO_DATA_UNIT_SIZE)) { + ret = BLK_STS_NOTSUPP; + goto out_free_cmd; + } + } if (blk_rq_nr_phys_segments(req)) { ret = apple_nvme_map_data(anv, req, cmnd); @@ -1095,7 +1143,7 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, } if (anv->hw->has_lsq_nvmmu) - apple_nvme_submit_cmd_t8103(q, cmnd); + apple_nvme_submit_cmd_t8103(q, cmnd, req); else apple_nvme_submit_cmd_t8015(q, cmnd); @@ -2374,6 +2422,18 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) if (ret) goto out_detach_genpd; + if (anv->hw->has_lsq_nvmmu) { + ret = devm_blk_crypto_profile_init(dev, &anv->crypto_profile, 0); + if (ret) + goto out_detach_genpd; + anv->crypto_profile.max_dun_bytes_supported = sizeof(u64); + anv->crypto_profile.key_types_supported = + BLK_CRYPTO_KEY_TYPE_HW_WRAPPED; + anv->crypto_profile.modes_supported[BLK_ENCRYPTION_MODE_AES_256_XTS] = + BIT(ilog2(APPLE_NVME_CRYPTO_DATA_UNIT_SIZE)); + anv->crypto_profile.dev = dev; + } + ret = nvme_init_ctrl(&anv->ctrl, anv->dev, &nvme_ctrl_ops, NVME_QUIRK_SKIP_CID_GEN | NVME_QUIRK_IDENTIFY_CNS | NVME_QUIRK_ADMIN_PAGE_ALIGN | anv->hw->quirks); @@ -2385,6 +2445,9 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) /* Released by apple_nvme_free_ctrl(), including asynchronous users. */ kref_get(&anv->ref); + if (anv->hw->has_lsq_nvmmu) + anv->ctrl.crypto_profile = &anv->crypto_profile; + return anv; out_detach_genpd: apple_nvme_detach_genpd(anv); diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index a20eabc7a5ed4..31d217901caee 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -4257,6 +4257,11 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, struct nvme_ns_info *info) ns->ctrl = ctrl; kref_init(&ns->kref); +#ifdef CONFIG_BLK_INLINE_ENCRYPTION + if (ctrl->crypto_profile) + blk_crypto_register(ctrl->crypto_profile, ns->queue); +#endif + if (nvme_init_ns_head(ns, info)) goto out_cleanup_disk; diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index ba40d93750f69..49ad643625967 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -361,6 +361,9 @@ struct nvme_ctrl { int numa_node; struct blk_mq_tag_set *tagset; struct blk_mq_tag_set *admin_tagset; +#ifdef CONFIG_BLK_INLINE_ENCRYPTION + struct blk_crypto_profile *crypto_profile; +#endif struct list_head namespaces; struct mutex namespaces_lock; struct srcu_struct srcu; From 19872093ae89de499789569992ed4fd869d60e23 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:43:48 +1100 Subject: [PATCH 0372/2680] nvme: apple: guard crypto cleanup on controllers without TCBs T8015 does not allocate NVMMU transaction control blocks. Complete its ordinary DMA unmapping before returning, and access crypto state only on controllers with LSQ NVMMU support. (cherry picked from commit f527149b2521930d037f8fead78c054f6563d4ec) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 8adaf00e015f0..4628332103be6 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -876,6 +876,9 @@ static __always_inline void apple_nvme_unmap_rq(struct request *req) if (blk_rq_nr_phys_segments(req)) apple_nvme_unmap_data(anv, req); + if (!anv->hw->has_lsq_nvmmu) + return; + tag = nvme_tag_from_cid(iod->cmd.common.command_id); tcb = &iod->q->tcbs[tag]; if (unlikely(tcb->dma_flags & BIT(2))) { From 161d8dcb321a2014fdbd22dba79ff07ad16d8f96 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 01:29:44 +1100 Subject: [PATCH 0373/2680] nvme: apple: keep the post-M4 ANS link domains powered across sleep A post-M4 ANS cannot be started again once CPU_CONTROL.RUN is cleared, so apple_nvme_suspend() only disables the NVMe controller and leaves the coprocessor running. The running firmware keeps using its PCIe link, whose power domains on T8140 are apcie_sys_st and apcie_phy_sw. genpd powers a domain off in the noirq phase once every device attached to it has suspended, whatever the runtime PM state. The controller holds its domains through per-domain virtual devices, which suspend right after the controller, so both link domains went down underneath the running ANS. On the J700, a pm_test=platform cycle, which holds the devices suspended for five seconds, then hung the SoC during resume until the watchdog reset it. Put the devices through which the controller holds its domains on the awake path when the ANS is left running. The pmgr power-state driver marks these boot-active domains GENPD_FLAG_ACTIVE_WAKEUP, so genpd keeps them powered until resume, as it already does for the domains of the display coprocessor. With this change the same cycle resumes with the root filesystem intact, and genpd records no power-off of either link domain across it or across real suspend-to-idle cycles. Controllers before M4 shut the ANS down and are not affected. (cherry picked from commit 3ec50f1cc9dd81b526f19f300e8b244ab7a69c3f) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 4628332103be6..1b072f75f1fc5 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -2582,6 +2582,28 @@ static int apple_nvme_resume(struct device *dev) return nvme_reset_ctrl(&anv->ctrl); } +/* + * The ANS stays up across system sleep on post-M4 hardware, and it keeps + * using its PCIe link while it runs. genpd, however, switches a domain off + * in the noirq phase once every device attached to it has suspended, no + * matter what runtime PM says. Put the devices that hold the controller's + * domains on the awake path so that genpd leaves those domains powered + * until resume. + */ +static void apple_nvme_keep_link_powered(struct apple_nvme *anv) +{ + int i; + + /* A single domain is attached to the controller device itself. */ + if (anv->pd_count <= 1) { + device_set_awake_path(anv->dev); + return; + } + + for (i = 0; i < anv->pd_count; i++) + device_set_awake_path(anv->pd_dev[i]); +} + static int apple_nvme_suspend(struct device *dev) { struct apple_nvme *anv = dev_get_drvdata(dev); @@ -2601,6 +2623,7 @@ static int apple_nvme_suspend(struct device *dev) */ if (anv->hw->needs_ioq_registers) { cancel_work_sync(&anv->recovery_work); + apple_nvme_keep_link_powered(anv); return 0; } From 8ce0f091ccffe2aa189d9a92ff456f1da857e327 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:38:03 +1100 Subject: [PATCH 0374/2680] nvme-apple: Drop the unused stopped-controller handoff Post-M4 ANS2 firmware can only be adopted while it still runs. The driver accepts two hand-over states: a controller the bootloader left enabled and ready, and a stopped controller that the bootloader vouches for with the "apple,rtkit-handoff" = "nvme-disabled-v1" device tree property. No bootloader produces that property. U-Boot, which the T8140 boot chain uses to load Linux, deletes its I/O queues and hands the controller over enabled and ready, and m1n1 does not publish the property either. The second state is therefore unreachable, and its property is device tree ABI that nothing produces. Remove the property parser and the stopped-controller branch of apple_nvme_can_adopt_rtkit(). The live-controller predicate is unchanged: RUN, BOOT_STATUS, CC.EN and CSTS.RDY must be set and CSTS.CFS clear. A device tree without the property, which is every device tree in use, takes exactly the same paths as before. (cherry picked from commit 2b07321f3689a2fc0043dcba26c71887a12b107d) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 37 ++++++------------------------------- 1 file changed, 6 insertions(+), 31 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 1b072f75f1fc5..c6f97f57866d5 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -209,7 +209,6 @@ struct apple_nvme { /* Protects reset_ready against RTKit crash callbacks. */ spinlock_t event_lock; bool reset_ready; - bool handoff_disabled; struct list_head quarantine_node; resource_size_t controller_start; bool quarantined; @@ -336,33 +335,15 @@ static bool apple_nvme_can_adopt_rtkit(struct apple_nvme *anv) return false; cc = readl(anv->mmio_nvme + NVME_REG_CC); csts = readl(anv->mmio_nvme + NVME_REG_CSTS); - if (csts & NVME_CSTS_CFS) - return false; /* - * CPU_RUN/BOOT_STATUS alone survive inactive sessions. A disabled - * controller requires the producer's explicit live-session contract. - * Borrowed buffers still require reserved RAM and protected SART ranges. + * CPU_RUN and BOOT_STATUS alone survive a session that has gone + * inactive. Only a controller that the previous owner left enabled + * and ready identifies a live session. Borrowed buffers still require + * reserved RAM and protected SART ranges. */ - if (anv->handoff_disabled) - return !(cc & NVME_CC_ENABLE) && !(csts & NVME_CSTS_RDY); - return (cc & NVME_CC_ENABLE) && (csts & NVME_CSTS_RDY); -} - -static int apple_nvme_read_handoff(struct apple_nvme *anv) -{ - const char *handoff; - int ret; - - ret = of_property_read_string(anv->dev->of_node, "apple,rtkit-handoff", - &handoff); - if (ret == -EINVAL) - return 0; - if (ret || !anv->hw->needs_ioq_registers || - strcmp(handoff, "nvme-disabled-v1")) - return -EINVAL; - anv->handoff_disabled = true; - return apple_nvme_can_adopt_rtkit(anv) ? 0 : -EIO; + return (cc & NVME_CC_ENABLE) && (csts & NVME_CSTS_RDY) && + !(csts & NVME_CSTS_CFS); } static inline struct apple_nvme *queue_to_apple_nvme(struct apple_nvme_queue *q) @@ -2317,12 +2298,6 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) anv->mmio_nvmmu = anv->mmio_nvme; } - ret = apple_nvme_read_handoff(anv); - if (ret) { - dev_err_probe(dev, ret, "Invalid live firmware handoff\n"); - goto out_detach_genpd; - } - if (anv->hw->has_lsq_nvmmu) { anv->adminq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_ASQ_DB; anv->adminq.cq_db = anv->mmio_nvme + APPLE_ANS_ACQ_DB; From 686f990f6920e6a9a4b5724b4a2546c613dae1d2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:38:20 +1100 Subject: [PATCH 0375/2680] dt-bindings: nvme: apple,nvme-ans: Drop apple,rtkit-handoff The "apple,rtkit-handoff" property was meant to be added at boot by a bootloader that stops the ANS2 NVMe controller but leaves its firmware running. No bootloader does that: U-Boot hands the T8140 controller over enabled and ready, which the driver recognises from the hardware state alone, and the driver no longer parses the property. Remove the property and the condition that limited it to T8140 before it gains a user. No board device tree source carries it. (cherry picked from commit 7bface2e329fe1a1dad8e47d1339ce06e0426d97) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/nvme/apple,nvme-ans.yaml | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml b/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml index 5449db3b25080..fc8a4afd5add4 100644 --- a/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml +++ b/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml @@ -85,30 +85,7 @@ properties: The SART address filter is documented in iommu/apple,sart.yaml. - apple,rtkit-handoff: - $ref: /schemas/types.yaml#/definitions/string - const: nvme-disabled-v1 - description: | - Added at boot by a bootloader that ran the ANS2 firmware and hands it - over still running instead of shutting it down. It describes the - state that bootloader left behind, not the hardware, and must not be - written into a board device tree source. Its presence states that the - firmware keeps a negotiated RTKit session, that all NVMe controller - DMA has stopped (CC.EN and CSTS.RDY clear), that the SART entries the - bootloader programmed are kept, and that every buffer shared with the - firmware is reserved in the memory map without no-map. - allOf: - - if: - properties: - compatible: - not: - contains: - const: apple,t8140-nvme-ans2 - then: - properties: - apple,rtkit-handoff: false - - if: properties: compatible: From 573e276623f4d3b5da1342e7c6c4b23356547f17 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:38:45 +1100 Subject: [PATCH 0376/2680] nvme-apple: Name the NVMMU TCB inline encryption flag The TCB flag that enables inline AES for a command is open-coded as BIT(2) where the command is submitted and where its key material is wiped on completion. Define it next to the DMA direction flags of the same field, and fold the key and DUN lookup into the block that programs them. No functional change. (cherry picked from commit b59cf281c0deae7864fb5fe18bfc4aedd7bf8348) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index c6f97f57866d5..6ef14f7bf96c2 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -116,6 +116,7 @@ struct apple_nvmmu_tcb { #define APPLE_ANS_TCB_DMA_FROM_DEVICE BIT(0) #define APPLE_ANS_TCB_DMA_TO_DEVICE BIT(1) +#define APPLE_ANS_TCB_AES BIT(2) u8 dma_flags; u8 command_id; @@ -566,8 +567,6 @@ static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, struct request *req) { struct apple_nvme *anv = queue_to_apple_nvme(q); - const u8 *key = NULL; - u64 dun = 0; u32 tag = nvme_tag_from_cid(cmd->common.command_id); struct apple_nvmmu_tcb *tcb = &q->tcbs[tag]; @@ -585,16 +584,10 @@ static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, tcb->dma_flags = APPLE_ANS_TCB_DMA_FROM_DEVICE; if (unlikely(req->crypt_ctx)) { - const struct blk_crypto_key *blk_key = req->crypt_ctx->bc_key; + const u8 *key = req->crypt_ctx->bc_key->bytes; + __le64 dun_le = cpu_to_le64(req->crypt_ctx->bc_dun[0]); - key = blk_key->bytes; - dun = req->crypt_ctx->bc_dun[0]; - } - - if (key) { - __le64 dun_le = cpu_to_le64(dun); - - tcb->dma_flags |= BIT(2); + tcb->dma_flags |= APPLE_ANS_TCB_AES; memcpy(tcb->aes_iv, &dun_le, sizeof(dun_le)); memset(tcb->_aes_unk, 0, sizeof(tcb->_aes_unk)); memcpy(tcb->_aes_unk, key, 4); @@ -862,10 +855,10 @@ static __always_inline void apple_nvme_unmap_rq(struct request *req) tag = nvme_tag_from_cid(iod->cmd.common.command_id); tcb = &iod->q->tcbs[tag]; - if (unlikely(tcb->dma_flags & BIT(2))) { + if (unlikely(tcb->dma_flags & APPLE_ANS_TCB_AES)) { memzero_explicit(tcb->aes_iv, sizeof(tcb->aes_iv)); memzero_explicit(tcb->_aes_unk, sizeof(tcb->_aes_unk)); - tcb->dma_flags &= ~BIT(2); + tcb->dma_flags &= ~APPLE_ANS_TCB_AES; } } From 0bf79a1c36c11ff5b5ad826f0ff3c5e229bb1a53 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:00:04 +1000 Subject: [PATCH 0377/2680] cpufreq: apple-soc: Use related_cpus when removing OPP tables at exit apple_soc_cpufreq_exit() drops the static OPP tables with dev_pm_opp_of_cpumask_remove_table(policy->cpus), but policy->cpus is always empty by then. __cpufreq_offline() clears the outgoing CPU from the mask first and calls ->exit() only once the last CPU of the policy has gone, and unbinding the driver takes the same path. The call removes nothing, so every time a cluster goes fully offline and comes back, or the driver is rebound, the OPP table of each of its CPUs is left with one more reference and is never freed. Remove the tables of policy->related_cpus instead, which still lists every CPU the policy covered, as qcom-cpufreq-hw does. The error path of apple_soc_cpufreq_init() keeps using policy->cpus, because the core fills in related_cpus only after ->init() has succeeded. Fixes: d87cb889dc7a ("cpufreq: apple-soc: Fix OPP table cleanup") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/cpufreq/apple-soc-cpufreq.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 7c50ce74f9117..55beec07370ca 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -324,7 +324,11 @@ static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy) struct apple_cpu_priv *priv = policy->driver_data; dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); - dev_pm_opp_of_cpumask_remove_table(policy->cpus); + /* + * The core clears the last CPU from policy->cpus before calling + * ->exit(), so remove the tables of every CPU the policy covered. + */ + dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); iounmap(priv->reg_base); kfree(priv); } From b1cdae9328d3d83c7de94298374938406f2e9c7f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:00:34 +1000 Subject: [PATCH 0378/2680] hwmon: (macsmc-hwmon) Reject NaN/Inf and decode subnormal floats macsmc_hwmon_read_f32_scaled() decodes the IEEE-754 single-precision values that the SMC returns for most sensor keys, and it mishandles the special encodings: - An exponent field of 0xff (infinity or NaN) is decoded as a huge finite number and saturated, so a sensor without a valid reading is reported as LONG_MAX or LONG_MIN millidegrees, millivolts, milliamps or microwatts. The chip registers HWMON_C_REGISTER_TZ, so a thermal zone built on such a sensor would see an absurd temperature. - The implicit leading one is also added when the exponent field is 0, so zero and subnormal inputs are decoded with the wrong mantissa and exponent. With the scales the driver uses (1000 and 1000000) the result still truncates to 0, so no reported value changes; the decoder now follows IEEE-754 for every exponent. - A negative value whose magnitude is exactly LONG_MAX + 1 was negated after the conversion to long, which is signed overflow. Return -ENODATA for NaN and infinity so that such readings show up as unavailable rather than as extreme values, decode exponent field 0 without the implicit bit and with the minimum exponent, and produce LONG_MIN without negating it. A scaled value that does not fit in a long is now reported as -ERANGE instead of being clamped to LONG_MAX or LONG_MIN: nothing the SMC measures gets anywhere near 9.2e15 degrees, volts or amps, so such a value is a decoding or transport problem, and a clamped number would only hide it from the thermal core and from userspace. Ordinary readings are unchanged and still truncate towards zero. Eryk Wieliczko wrote this change for Aurora's tree (2026-09-08, under the file's GPL-2.0-only OR MIT licence) and verified it against 9045 exact-integer oracle cases under UBSan and on a J700. It is carried here unchanged, without its host-side Python test; his Signed-off-by is outstanding. Fixes: 785205fd8139 ("hwmon: Add Apple Silicon SMC hwmon driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hwmon/macsmc-hwmon.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index 1500ec2cc9f83..13e38b25c75fb 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -151,31 +151,42 @@ static int macsmc_hwmon_read_f32_scaled(struct apple_smc *smc, smc_key key, if (ret < 0) return ret; - val = ((u64)((fval & FLT_MANT_MASK) | BIT(23))); - exp = ((fval >> 23) & 0xff) - FLT_EXP_BIAS - FLT_MANT_BIAS; + exp = FIELD_GET(FLT_EXP_MASK, fval); + /* NaN and infinity are unavailable readings, not extreme temperatures. */ + if (exp == 0xff) + return -ENODATA; + + val = fval & FLT_MANT_MASK; + if (exp) + val |= BIT(23); + else + exp = 1; /* Zero and subnormal values have no implicit leading one. */ + exp -= FLT_EXP_BIAS + FLT_MANT_BIAS; /* We never have negatively scaled SMC floats */ val *= scale; if (exp > 63) - val = U64_MAX; + return -ERANGE; else if (exp < -63) val = 0; else if (exp < 0) val >>= -exp; else if (exp != 0 && (val & ~((1ULL << (64 - exp)) - 1))) /* overflow */ - val = U64_MAX; + return -ERANGE; else val <<= exp; if (fval & FLT_SIGN_MASK) { if (val > (u64)LONG_MAX + 1) + return -ERANGE; + if (val == (u64)LONG_MAX + 1) *p = LONG_MIN; else *p = -(long)val; } else { if (val > (u64)LONG_MAX) - *p = LONG_MAX; + return -ERANGE; else *p = (long)val; } From 9cb99d03aeb0a9a4dc8ca54a5a336fd3fabf6188 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:03:18 +1000 Subject: [PATCH 0379/2680] dt-bindings: cpufreq: apple,cluster-cpufreq: Add T8140 compatible The CPU clusters of the Apple T8140 (A18 Pro) use the same command register layout as T8112, with a 5-bit state field, but Linux has to drive them differently: every transition must be verified, the state left by the bootloader must be adopted rather than overwritten, and states above 2 may only be used under a thermal policy. Add the compatible without the generic "apple,cluster-cpufreq" fallback. A kernel that only knows the fallback must not program these clusters: it would drive them without verifying any transition and expose the whole ladder without thermal coupling. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml index 9c9edb8cb441a..d44c3bc580b72 100644 --- a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml +++ b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml @@ -35,6 +35,7 @@ properties: - const: apple,t7000-cluster-cpufreq - const: apple,s5l8960x-cluster-cpufreq - const: apple,s5l8960x-cluster-cpufreq + - const: apple,t8140-cluster-cpufreq - items: - const: apple,t6020-cluster-cpufreq - const: apple,t8112-cluster-cpufreq From 684927ba19893a3fe5114dc2091e9e986b69d460 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:14:55 +1000 Subject: [PATCH 0380/2680] cpufreq: apple-soc: Add T8140 support The Apple T8140 (A18 Pro) clusters take P-state requests through the same command register layout as T8112, but this driver cannot assume that it owns them from the start: the bootloader leaves a state of its own choosing, and nothing guarantees that no other agent writes the register later. Add them with a per-SoC verify_transition mode: - At init, wait for the controller to be idle, read the state left in the command register and adopt it as policy->cur. A state that is not in the OPP table fails init with -ERANGE rather than being replaced by one whose effect is unknown. - Before each request, check that the register still holds the state this driver last set or adopted; anything else means another agent has changed it. After writing, poll until the controller is idle and reports the requested state. A request for the current state is skipped. - A busy timeout, an unexpected state or a completion timeout latches the cluster as failed. The driver logs the command register once and then returns -EIO for every later request without touching the hardware, so the cluster stays wherever the controller stopped and no governor, user limit or thermal cap can move it until the policy is torn down and initialised again. The driver deliberately does not guess a rollback after an unexplained failure. - Fast switching is disabled, because a verified request can busy-wait for up to twice APPLE_DVFS_TRANSITION_TIMEOUT, and that doubled bound is the transition latency the policy advertises when the DT gives none. - T8140 state IDs start at 1, so the info data gains min_pstate. Its DVFS_STATUS layout is not known, so ->get() uses the existing command-register fallback and reports the last accepted request, not a measured frequency. The T8140 ladders go up to P17 (4.044 GHz) on the P cluster and E6 (2.424 GHz) on the E cluster. States above 2 need a thermal policy that can cap them; until one is coupled to this driver, a T8140 OPP table containing them is refused. Three changes apply to every SoC the driver handles: - OPP levels outside [min_pstate, max_pstate] fail init, and apple_soc_cpufreq_set_target() rejects them. The old "fallback" clamp compared the table index rather than the level and ran after the level had been read, so an out-of-range level was shifted unmasked into the command register, over the neighbouring bits. No in-tree Apple DT has such a level. - A cluster node with status = "disabled" is left alone. - The cluster node reference is dropped after of_iomap() instead of before it. All SoCs keep sharing one cpufreq_driver; the match data alone selects the T8140 behaviour. As on the other SoCs, T8140 clusters are set to their lowest state for system suspend. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/cpufreq/apple-soc-cpufreq.c | 119 ++++++++++++++++++++++++++-- 1 file changed, 112 insertions(+), 7 deletions(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 55beec07370ca..106645cedb191 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -62,6 +62,8 @@ struct apple_soc_cpufreq_info { bool has_ps2; + bool verify_transition; + u64 min_pstate; u64 max_pstate; u64 cur_pstate_mask; u64 cur_pstate_shift; @@ -73,6 +75,8 @@ struct apple_cpu_priv { struct device *cpu_dev; void __iomem *reg_base; const struct apple_soc_cpufreq_info *info; + bool transition_failed; + unsigned int expected_pstate; }; static struct cpufreq_driver apple_soc_cpufreq_driver; @@ -104,6 +108,20 @@ static const struct apple_soc_cpufreq_info soc_t8112_info = { .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, }; +/* + * T8140 uses 5-bit PS1 state IDs starting at 1. The command register reads + * back the last accepted request, not a measured frequency; every transition + * is verified, see apple_soc_cpufreq_set_target(). Voltage, PLL and firmware + * throttling configuration are left as the bootloader set them. + */ +static const struct apple_soc_cpufreq_info soc_t8140_info = { + .verify_transition = true, + .min_pstate = 1, + .max_pstate = 31, + .ps1_mask = APPLE_DVFS_CMD_PS1, + .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, +}; + static const struct apple_soc_cpufreq_info soc_default_info = { .has_ps2 = false, .max_pstate = 15, @@ -129,6 +147,10 @@ static const struct of_device_id apple_soc_cpufreq_of_match[] __maybe_unused = { .compatible = "apple,cluster-cpufreq", .data = &soc_default_info, }, + { + .compatible = "apple,t8140-cluster-cpufreq", + .data = &soc_t8140_info, + }, {} }; @@ -175,16 +197,37 @@ static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, unsigned int pstate = policy->freq_table[index].driver_data; u64 reg; - /* Fallback for newer SoCs */ - if (index > priv->info->max_pstate) - index = priv->info->max_pstate; + /* OPP level is a hardware ID, not the frequency-table array index. */ + if (pstate < priv->info->min_pstate || pstate > priv->info->max_pstate) + return -EINVAL; + if (priv->transition_failed) + return -EIO; if (readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg, !(reg & APPLE_DVFS_CMD_BUSY), 2, APPLE_DVFS_TRANSITION_TIMEOUT)) { + if (priv->info->verify_transition) { + priv->transition_failed = true; + dev_err(priv->cpu_dev, + "DVFS busy timeout, command=%#llx; stopping requests\n", reg); + } return -EIO; } + if (priv->info->verify_transition) { + unsigned int previous = FIELD_GET(APPLE_DVFS_CMD_PS1, reg); + + /* Do not fight an unexpected retained policy owner. */ + if (previous != priv->expected_pstate) { + priv->transition_failed = true; + dev_err(priv->cpu_dev, "unexpected DVFS state %u (expected %u); stopping requests\n", + previous, priv->expected_pstate); + return -EIO; + } + if (previous == pstate) + return 0; + } + reg &= ~priv->info->ps1_mask; reg |= pstate << priv->info->ps1_shift; if (priv->info->has_ps2) @@ -193,6 +236,20 @@ static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD); + if (priv->info->verify_transition && + readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg, + !(reg & APPLE_DVFS_CMD_BUSY) && + FIELD_GET(APPLE_DVFS_CMD_PS1, reg) == pstate, + 2, APPLE_DVFS_TRANSITION_TIMEOUT)) { + /* Do not issue another request or guess a rollback after failure. */ + priv->transition_failed = true; + dev_err(priv->cpu_dev, + "DVFS completion timeout, command=%#llx; stopping requests\n", reg); + return -ETIMEDOUT; + } + if (priv->info->verify_transition) + priv->expected_pstate = pstate; + return 0; } @@ -220,13 +277,15 @@ static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy, return ret; match = of_match_node(apple_soc_cpufreq_of_match, args.np); - of_node_put(args.np); - if (!match) + if (!match || !of_device_is_available(args.np)) { + of_node_put(args.np); return -ENODEV; + } *info = match->data; *reg_base = of_iomap(args.np, 0); + of_node_put(args.np); if (!*reg_base) return -ENOMEM; @@ -235,6 +294,7 @@ static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy, static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) { + struct cpufreq_frequency_table *p; int ret, i; unsigned int transition_latency; void __iomem *reg_base; @@ -289,6 +349,33 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) } freq_table[i].driver_data = dev_pm_opp_get_level(opp); dev_pm_opp_put(opp); + if (freq_table[i].driver_data < info->min_pstate || + freq_table[i].driver_data > info->max_pstate) { + ret = -EINVAL; + goto out_free_cpufreq_table; + } + } + + if (info->verify_transition) { + u64 cmd; + unsigned int state; + + /* Registration must not silently reset an unknown inherited state. */ + ret = readq_poll_timeout_atomic(reg_base + APPLE_DVFS_CMD, cmd, + !(cmd & APPLE_DVFS_CMD_BUSY), 2, + APPLE_DVFS_TRANSITION_TIMEOUT); + if (ret) + goto out_free_cpufreq_table; + state = FIELD_GET(APPLE_DVFS_CMD_PS1, cmd); + for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) + if (freq_table[i].driver_data == state) + break; + if (freq_table[i].frequency == CPUFREQ_TABLE_END) { + ret = -ERANGE; + goto out_free_cpufreq_table; + } + policy->cur = freq_table[i].frequency; + priv->expected_pstate = state; } priv->cpu_dev = cpu_dev; @@ -298,13 +385,31 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) policy->freq_table = freq_table; transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev); - if (!transition_latency) + if (!transition_latency) { + /* Conservative transaction bound, not a measured transition time. */ transition_latency = APPLE_DVFS_TRANSITION_TIMEOUT * NSEC_PER_USEC; + if (info->verify_transition) + transition_latency *= 2; + } policy->cpuinfo.transition_latency = transition_latency; policy->dvfs_possible_from_any_cpu = true; - policy->fast_switch_possible = true; + policy->fast_switch_possible = !info->verify_transition; policy->suspend_freq = freq_table[0].frequency; + if (info->verify_transition) { + /* + * P-states above 2 need a thermal policy that can cap them; + * refuse such a table rather than run it unthrottled. + */ + cpufreq_for_each_valid_entry(p, policy->freq_table) { + if (p->driver_data > 2) { + dev_err(cpu_dev, "P-state %u requires a CPU thermal policy\n", + p->driver_data); + ret = -ENODEV; + goto out_free_cpufreq_table; + } + } + } return 0; From 8d98ea130262a92ea2f2fb7dec07ef98ce156a32 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:18:39 +1000 Subject: [PATCH 0381/2680] dt-bindings: mfd: apple,smc: Add apple,t8140-smc The SMC of the Apple T8140 (A18 Pro) is driven like the T8103 one, through the same mailbox protocol and the same "smc" and "sram" register regions; the J700 device tree describes it as "apple,t8140-smc", "apple,t8103-smc". Add the SoC compatible to the list that falls back to apple,t8103-smc. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/mfd/apple,smc.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/mfd/apple,smc.yaml b/Documentation/devicetree/bindings/mfd/apple,smc.yaml index d5d30c0538458..ed6b02086c33c 100644 --- a/Documentation/devicetree/bindings/mfd/apple,smc.yaml +++ b/Documentation/devicetree/bindings/mfd/apple,smc.yaml @@ -20,6 +20,7 @@ properties: - enum: - apple,t6020-smc - apple,t8122-smc + - apple,t8140-smc - const: apple,t8103-smc - items: - enum: From 4972dda8bc35a85f9dea21ea30533e4533884929 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:19:44 +1000 Subject: [PATCH 0382/2680] dt-bindings: spmi: apple,spmi: Add apple,t8140-spmi The SPMI controller of the Apple T8140 (A18 Pro), which connects the J700's Baku PMIC, is register compatible with the T8103 one. Add the SoC compatible with the apple,t8103-spmi fallback so that the J700 device tree can describe it. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/spmi/apple,spmi.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/spmi/apple,spmi.yaml b/Documentation/devicetree/bindings/spmi/apple,spmi.yaml index 1d14025b44c5b..a6bc22832c633 100644 --- a/Documentation/devicetree/bindings/spmi/apple,spmi.yaml +++ b/Documentation/devicetree/bindings/spmi/apple,spmi.yaml @@ -25,6 +25,7 @@ properties: - apple,t8012-spmi - apple,t8015-spmi - apple,t8122-spmi + - apple,t8140-spmi - const: apple,t8103-spmi - items: - enum: From da78fe980d25a7741909f28634510b8292f8c47d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 02:50:02 +1000 Subject: [PATCH 0383/2680] power: supply: macsmc: Only register optional properties whose keys exist VOLTAGE_MIN_DESIGN, CHARGE_TERM_CURRENT, CONSTANT_CHARGE_CURRENT_MAX and CONSTANT_CHARGE_VOLTAGE are read from the BITV, B0RC, B0RI and B0RV keys, and the driver registers them on every machine. When such a read fails with an SMC error (-EIO), the power supply core logs "driver failed to report" (rate-limited) for each read of the property, and power_supply_uevent() gives up at the first failing property, so no change uevent is sent for the battery at all. Register these four properties only if their key exists at probe time. This touches every machine the driver supports; firmware that has the keys keeps the same properties. On J700, reads of all four keys fail and the VOLTAGE_MIN_DESIGN errors flood the kernel log. It has not been confirmed there that the keys are missing from the key index, which is what apple_smc_key_exists() checks, rather than present but unreadable. macsmc_power_critical_work() forces an emergency shutdown when B0AV drops below BITV. Without a readable BITV that check never fires, as before this change, and only the BCF0 empty flag starts the orderly shutdown. This applies to J700, where no key that could replace BITV is known. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/power/supply/macsmc-power.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c index addb4348954f5..41116868bfe3c 100644 --- a/drivers/power/supply/macsmc-power.c +++ b/drivers/power/supply/macsmc-power.c @@ -888,13 +888,17 @@ static int macsmc_power_probe(struct platform_device *pdev) /* Extended properties usually present */ props[nprops++] = POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW; props[nprops++] = POWER_SUPPLY_PROP_TIME_TO_FULL_NOW; - props[nprops++] = POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN; + if (apple_smc_key_exists(smc, SMC_KEY(BITV))) + props[nprops++] = POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN; props[nprops++] = POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN; props[nprops++] = POWER_SUPPLY_PROP_VOLTAGE_MIN; props[nprops++] = POWER_SUPPLY_PROP_VOLTAGE_MAX; - props[nprops++] = POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT; - props[nprops++] = POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX; - props[nprops++] = POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE; + if (apple_smc_key_exists(smc, SMC_KEY(B0RC))) + props[nprops++] = POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT; + if (apple_smc_key_exists(smc, SMC_KEY(B0RI))) + props[nprops++] = POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX; + if (apple_smc_key_exists(smc, SMC_KEY(B0RV))) + props[nprops++] = POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE; props[nprops++] = POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN; props[nprops++] = POWER_SUPPLY_PROP_CHARGE_FULL; props[nprops++] = POWER_SUPPLY_PROP_CHARGE_NOW; From cfd4ad76a47111f61c50537f6426bc029785cc68 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 02:50:45 +1000 Subject: [PATCH 0384/2680] power: supply: macsmc: Read B0RM in native byte order on J700 Commit 57694663d658 ("power: supply: macsmc: Support macOS 27 SMC firmware") infers two firmware changes from one fact: when the BCF0 key shrinks from four bytes to one, B0RM is taken to switch from the gas gauge's big-endian pass-through to native byte order. J700's SMC firmware reports a four-byte BCF0 but a native-endian B0RM. Swapping it made CHARGE_NOW and ENERGY_NOW move in 256 mAh steps as the low byte changed, and gave values above CHARGE_FULL or below zero. Track the B0RM byte order in its own flag, b0rm_native, set at probe with the one-byte BCF0 layout as before and additionally on J700. fw_ge_27 now means only that BCF0 is one byte. The J700 case is keyed on the machine compatible because nothing in the key information the driver reads distinguishes it: BCF0 has the older size, and the B0RM key type has not been compared with other firmware. The helper now returns u16 instead of s16 on every machine. B0RM is an unsigned mAh count, so a remaining charge below 32768 mAh reads as before; a larger value, such as a wrongly swapped one, no longer turns negative. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/power/supply/macsmc-power.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c index 41116868bfe3c..7f45d95b5ad62 100644 --- a/drivers/power/supply/macsmc-power.c +++ b/drivers/power/supply/macsmc-power.c @@ -86,11 +86,14 @@ struct macsmc_power { bool has_ch0i; /* Force discharge (Older firmware) */ bool has_ch0c; /* Inhibit charge (Older firmware) */ bool has_chte; /* Inhibit charge (Modern firmware) */ + /* Battery critical key (BCF0) is 1 byte (Modern firmware) */ + bool fw_ge_27; /* - * Battery critical key is 1 byte and charge key is little endian - * (Modern firmware) + * Remaining charge key (B0RM) is in native byte order rather than the + * gas gauge's big-endian pass-through. This comes with the 1-byte BCF0 + * firmware, but J700 firmware has a native B0RM and a 4-byte BCF0. */ - bool fw_ge_27; + bool b0rm_native; u8 num_cells; int nominal_voltage_mv; @@ -421,10 +424,9 @@ static int macsmc_battery_get_capacity_level(struct macsmc_power *power) return POWER_SUPPLY_CAPACITY_LEVEL_NORMAL; } -static s16 macsmc_swap_b0rm(struct macsmc_power *power, s16 b0rm) +static u16 macsmc_b0rm_to_mah(struct macsmc_power *power, u16 b0rm) { - /* B0RM was Big Endian, likely pass through from TI gas gauge */ - return power->fw_ge_27 ? b0rm : (s16)swab16(b0rm); + return power->b0rm_native ? b0rm : swab16(b0rm); } static int macsmc_battery_get_property(struct power_supply *psy, @@ -521,7 +523,7 @@ static int macsmc_battery_get_property(struct power_supply *psy, break; case POWER_SUPPLY_PROP_CHARGE_NOW: ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RM), &vu16); - val->intval = macsmc_swap_b0rm(power, vu16) * 1000; + val->intval = macsmc_b0rm_to_mah(power, vu16) * 1000; break; case POWER_SUPPLY_PROP_ENERGY_FULL_DESIGN: ret = apple_smc_read_u16(power->smc, SMC_KEY(B0DC), &vu16); @@ -533,7 +535,7 @@ static int macsmc_battery_get_property(struct power_supply *psy, break; case POWER_SUPPLY_PROP_ENERGY_NOW: ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RM), &vu16); - val->intval = macsmc_swap_b0rm(power, vu16) * power->nominal_voltage_mv; + val->intval = macsmc_b0rm_to_mah(power, vu16) * power->nominal_voltage_mv; break; case POWER_SUPPLY_PROP_TEMP: ret = apple_smc_read_u16(power->smc, SMC_KEY(B0AT), &vu16); @@ -929,6 +931,9 @@ static int macsmc_power_probe(struct platform_device *pdev) return -EIO; } + power->b0rm_native = power->fw_ge_27 || + of_machine_is_compatible("apple,j700"); + /* Reset "Optimised Battery Charging" flags to default state */ if (power->has_chte) apple_smc_write_u32(smc, SMC_KEY(CHTE), 0); From df5608a2d2466d60d7a7ba3e4df629fd5283a453 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 02:51:48 +1000 Subject: [PATCH 0385/2680] power: supply: macsmc: Report net discharge on J700 as discharging A J700 on a low-power USB-C source can have a charge-capable adapter online, with CHCE and CHCC set, while the system draws more than the adapter supplies. The battery current B0AC is then negative and the remaining charge falls. CHCE and CHCC describe the adapter, not the battery, so where the other SMC flags then lead the driver to report CHARGING, the status contradicts the battery. On J700, report DISCHARGING instead of CHARGING when B0AC is negative. Only the three CHARGING results change: CHNC clear, CHNC holding only the BMS-busy bit below the charge limit, and the CHSC fallback when CHNC cannot be read. FULL, NOT_CHARGING for a charge limit or an inhibit, and the existing DISCHARGING cases are decided first and stay as they are, so a battery held at its charge limit keeps reporting NOT_CHARGING. Whether CHNC reports an inhibit in the low-power state has not been recorded; if it does, the status there stays NOT_CHARGING. The quirk is decided once at probe, and other machines keep the current status logic. B0AC is compared against zero. No noise floor around 0 mA has been measured, so a net current that hovers around zero can make the status alternate between CHARGING and DISCHARGING. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/power/supply/macsmc-power.c | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c index 7f45d95b5ad62..3eab305357cd9 100644 --- a/drivers/power/supply/macsmc-power.c +++ b/drivers/power/supply/macsmc-power.c @@ -94,6 +94,8 @@ struct macsmc_power { * firmware, but J700 firmware has a native B0RM and a 4-byte BCF0. */ bool b0rm_native; + /* Charging with a negative battery current is discharging (J700) */ + bool status_from_current; u8 num_cells; int nominal_voltage_mv; @@ -203,6 +205,23 @@ static void macsmc_do_dbg(struct macsmc_power *power) #undef FD3 } +/* + * The status to report where the SMC flags say the battery is charging. On + * J700 a charge-capable adapter can be online while the system draws more + * than it supplies, and the battery then drains. + */ +static int macsmc_battery_charging_status(struct macsmc_power *power) +{ + s16 current_ma; + + if (power->status_from_current && + !apple_smc_read_s16(power->smc, SMC_KEY(B0AC), ¤t_ma) && + current_ma < 0) + return POWER_SUPPLY_STATUS_DISCHARGING; + + return POWER_SUPPLY_STATUS_CHARGING; +} + static int macsmc_battery_get_status(struct macsmc_power *power) { u64 nocharge_flags; @@ -280,11 +299,11 @@ static int macsmc_battery_get_status(struct macsmc_power *power) return POWER_SUPPLY_STATUS_FULL; /* BMS busy shows up as inhibit, but we treat it as charging */ else if (nocharge_flags == CHNC_BMS_BUSY && !limited) - return POWER_SUPPLY_STATUS_CHARGING; + return macsmc_battery_charging_status(power); else if (nocharge_flags) return POWER_SUPPLY_STATUS_NOT_CHARGING; else - return POWER_SUPPLY_STATUS_CHARGING; + return macsmc_battery_charging_status(power); } /* Fallback: System charging flag */ @@ -294,7 +313,7 @@ static int macsmc_battery_get_status(struct macsmc_power *power) if (!flag) return POWER_SUPPLY_STATUS_NOT_CHARGING; - return POWER_SUPPLY_STATUS_CHARGING; + return macsmc_battery_charging_status(power); } static int macsmc_battery_get_charge_behaviour(struct macsmc_power *power) @@ -933,6 +952,7 @@ static int macsmc_power_probe(struct platform_device *pdev) power->b0rm_native = power->fw_ge_27 || of_machine_is_compatible("apple,j700"); + power->status_from_current = of_machine_is_compatible("apple,j700"); /* Reset "Optimised Battery Charging" flags to default state */ if (power->has_chte) From 750bc892e4c82833b5868328c1e36ef0816fe92e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 18:02:50 +1000 Subject: [PATCH 0386/2680] dt-bindings: rtc: apple,smc-rtc: Add apple,rtc-offset-seconds The SMC RTC is a free-running counter in the PMU (read through the SMC CLKM key at 32768 Hz) plus an offset kept in PMU NVMEM. On M1 and M2 the offset is a 48-bit count of 32768 Hz ticks. The t8140 (A18 Pro, MacBook Neo J700) PMU keeps it differently: the system firmware stores a 64-bit little-endian count of whole seconds in the PMU's legacy scratchpad, and the register the device tree for older SoCs would point at holds the running counter itself, which ignores writes. Add a boolean property selecting the whole-second format for the rtc_offset cell, and describe both formats in the cell description. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/rtc/apple,smc-rtc.yaml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml b/Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml index 607b610665a28..2d468e3c302c6 100644 --- a/Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml +++ b/Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml @@ -21,12 +21,23 @@ properties: nvmem-cells: items: - - description: 48bit RTC offset, specified in 32768 (2^15) Hz clock ticks + - description: + RTC offset. By default a 48bit value, specified in 32768 (2^15) Hz + clock ticks. With apple,rtc-offset-seconds, a 64bit little-endian + count of whole seconds. nvmem-cell-names: items: - const: rtc_offset + apple,rtc-offset-seconds: + type: boolean + description: + The RTC offset cell holds whole seconds rather than 32768 Hz ticks. The + current time is the SMC RTC counter's integer seconds plus the offset. + Used where the system firmware keeps the offset in a PMU scratchpad + register in this format, such as on the t8140 (A18 Pro). + required: - compatible - nvmem-cells From 920c53a4c74feebddbfda45eb421f0cfe89e6487 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 18:03:28 +1000 Subject: [PATCH 0387/2680] rtc: macsmc: Support an RTC offset stored in whole seconds On the t8140 (A18 Pro, MacBook Neo J700) the system firmware keeps the RTC offset in the PMU's legacy scratchpad as a 64-bit little-endian count of whole seconds, not as the 48-bit count of 32768 Hz ticks used on M1 and M2. The wall-clock time is the counter's integer seconds plus that offset. On a J700 today the scratchpad at 0xf704 holds 1773779187, and that plus the PMU counter's seconds matched the NTP-synchronised system clock to within 0.01 s. When the device tree sets apple,rtc-offset-seconds, read and write the offset in that format. Writing now minus the counter's whole seconds stores the same value the system firmware keeps, so the two operating systems agree on the time. Report an all-zero offset as an invalid time: nothing has been stored yet and the counter alone would put the clock in 1970. The representable range starts at the epoch. The tick format and its behaviour are unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/rtc/rtc-macsmc.c | 59 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/drivers/rtc/rtc-macsmc.c b/drivers/rtc/rtc-macsmc.c index 8fe8830669569..25d5af5cfbbcd 100644 --- a/drivers/rtc/rtc-macsmc.c +++ b/drivers/rtc/rtc-macsmc.c @@ -12,6 +12,7 @@ #include #include #include +#include /* 48-bit RTC */ #define RTC_BYTES 6 @@ -20,13 +21,47 @@ /* 32768 Hz clock */ #define RTC_SEC_SHIFT 15 +/* Offset kept as a 64-bit little-endian count of whole seconds */ +#define RTC_OFFSET_SECS_BYTES 8 + struct macsmc_rtc { struct device *dev; struct apple_smc *smc; struct rtc_device *rtc_dev; struct nvmem_cell *rtc_offset; + bool offset_in_seconds; }; +static int macsmc_rtc_get_time_secs(struct macsmc_rtc *rtc, u64 ctr, + struct rtc_time *tm) +{ + void *p_off; + size_t len; + u64 off; + + p_off = nvmem_cell_read(rtc->rtc_offset, &len); + if (IS_ERR(p_off)) + return PTR_ERR(p_off); + if (len < RTC_OFFSET_SECS_BYTES) { + kfree(p_off); + return -EIO; + } + + off = get_unaligned_le64(p_off); + kfree(p_off); + + /* + * Nothing has been stored yet: the counter alone is the time since + * the PMU first powered up, not the wall-clock time. + */ + if (!off) + return -EINVAL; + + rtc_time64_to_tm((time64_t)(ctr >> RTC_SEC_SHIFT) + (s64)off, tm); + + return 0; +} + static int macsmc_rtc_get_time(struct device *dev, struct rtc_time *tm) { struct macsmc_rtc *rtc = dev_get_drvdata(dev); @@ -42,6 +77,9 @@ static int macsmc_rtc_get_time(struct device *dev, struct rtc_time *tm) if (ret != RTC_BYTES) return -EIO; + if (rtc->offset_in_seconds) + return macsmc_rtc_get_time_secs(rtc, ctr, tm); + p_off = nvmem_cell_read(rtc->rtc_offset, &len); if (IS_ERR(p_off)) return PTR_ERR(p_off); @@ -72,6 +110,15 @@ static int macsmc_rtc_set_time(struct device *dev, struct rtc_time *tm) if (ret != RTC_BYTES) return -EIO; + if (rtc->offset_in_seconds) { + __le64 off_secs; + + /* The counter's fraction of a second carries into the new time */ + off_secs = cpu_to_le64(rtc_tm_to_time64(tm) - (ctr >> RTC_SEC_SHIFT)); + ret = nvmem_cell_write(rtc->rtc_offset, &off_secs, sizeof(off_secs)); + return ret < 0 ? ret : 0; + } + /* This sets the offset such that the set second begins now */ off = (rtc_tm_to_time64(tm) << RTC_SEC_SHIFT) - ctr; return nvmem_cell_write(rtc->rtc_offset, &off, RTC_BYTES); @@ -111,9 +158,17 @@ static int macsmc_rtc_probe(struct platform_device *pdev) if (IS_ERR(rtc->rtc_dev)) return PTR_ERR(rtc->rtc_dev); + rtc->offset_in_seconds = of_property_read_bool(pdev->dev.of_node, + "apple,rtc-offset-seconds"); + rtc->rtc_dev->ops = &macsmc_rtc_ops; - rtc->rtc_dev->range_min = S64_MIN >> (RTC_SEC_SHIFT + (64 - RTC_BITS)); - rtc->rtc_dev->range_max = S64_MAX >> (RTC_SEC_SHIFT + (64 - RTC_BITS)); + if (rtc->offset_in_seconds) { + rtc->rtc_dev->range_min = 0; + rtc->rtc_dev->range_max = RTC_TIMESTAMP_END_9999; + } else { + rtc->rtc_dev->range_min = S64_MIN >> (RTC_SEC_SHIFT + (64 - RTC_BITS)); + rtc->rtc_dev->range_max = S64_MAX >> (RTC_SEC_SHIFT + (64 - RTC_BITS)); + } platform_set_drvdata(pdev, rtc); From a07220d1b6531b2ebe8bef81782664968948a8c4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 03:20:32 +1000 Subject: [PATCH 0388/2680] mfd: macsmc: Only register the second GPIO bank where it is described Commit 77cb8f24c238 ("mfd: macsmc: Add second gpio subdevice for 'gp00' keys") added the "apple,smc-low-gpio" cell to the cells every SMC registers. Only some SMCs have that bank; the device trees describe it for T6030 and T6031. On every other machine the MFD core finds no node, warns macsmc-low-gpio: Failed to locate of_node [id: -1] and still creates a platform device that nothing can bind. On the MacBook Neo (T8140) a test node confirmed there is no such bank: the SMC has no keys in the gp00-gp1f range and gpio-macsmc declines to probe. Register the cell separately, and only when the SMC node has a child with that compatible. Fixes: 77cb8f24c238 ("mfd: macsmc: Add second gpio subdevice for 'gp00' keys") Signed-off-by: Ryan Murray --- drivers/mfd/macsmc.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 31540f67ddbd8..8cdc59a7dd88c 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -48,12 +48,15 @@ static const struct mfd_cell apple_smc_devs[] = { MFD_CELL_NAME("macsmc-input"), MFD_CELL_NAME("macsmc-power"), MFD_CELL_OF("macsmc-gpio", NULL, NULL, 0, 0, "apple,smc-gpio"), - MFD_CELL_OF("macsmc-low-gpio", NULL, NULL, 0, 0, "apple,smc-low-gpio"), MFD_CELL_OF("macsmc-hwmon", NULL, NULL, 0, 0, "apple,smc-hwmon"), MFD_CELL_OF("macsmc-reboot", NULL, NULL, 0, 0, "apple,smc-reboot"), MFD_CELL_OF("macsmc-rtc", NULL, NULL, 0, 0, "apple,smc-rtc"), }; +/* The second GPIO bank ('gp' keys) only exists on some SMCs. */ +static const struct mfd_cell apple_smc_low_gpio_dev = + MFD_CELL_OF("macsmc-low-gpio", NULL, NULL, 0, 0, "apple,smc-low-gpio"); + static int apple_smc_cmd_locked(struct apple_smc *smc, u64 cmd, u64 arg, u64 size, u64 wsize, u32 *ret_data) { @@ -410,6 +413,7 @@ static void apple_smc_disable_notifications(void *data) static int apple_smc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + struct device_node *np; struct apple_smc *smc; __be32 count; int ret; @@ -481,6 +485,20 @@ static int apple_smc_probe(struct platform_device *pdev) if (ret) return dev_err_probe(smc->dev, ret, "Failed to register sub-devices"); + /* + * Register the second GPIO bank only where the device tree describes + * it: the MFD core would otherwise create an unbound device and warn + * that it found no node on every SMC without one. + */ + np = of_get_compatible_child(dev->of_node, "apple,smc-low-gpio"); + if (np) { + of_node_put(np); + ret = devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, + &apple_smc_low_gpio_dev, 1, NULL, 0, NULL); + if (ret) + return dev_err_probe(smc->dev, ret, + "Failed to register the second GPIO bank"); + } return 0; } From c94fdf1e32bfd90e84eabb22a25ec0cfd642d5d8 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 12:26:30 -0400 Subject: [PATCH 0389/2680] cpufreq: apple: Add bounded T8152 E and P/M performance domains MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — the original sat on the since-removed SMC CPU thermal policy hooks; the unmanaged-ceiling check now refuses P-states above max_unmanaged_pstate the way the T8140 check refuses those above 2. The t8152-j873g.dts hunk moves to aurora/bits/000-devicetree.] --- Documentation/arch/arm64/apple-m6-cpu.rst | 29 ++++++++++++++ .../cpufreq/apple,cluster-cpufreq.yaml | 1 + drivers/cpufreq/apple-soc-cpufreq.c | 38 +++++++++++++++---- 3 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 Documentation/arch/arm64/apple-m6-cpu.rst diff --git a/Documentation/arch/arm64/apple-m6-cpu.rst b/Documentation/arch/arm64/apple-m6-cpu.rst new file mode 100644 index 0000000000000..046872e83e09a --- /dev/null +++ b/Documentation/arch/arm64/apple-m6-cpu.rst @@ -0,0 +1,29 @@ +M6 CPU bring-up +============== + +J873g has six E CPUs and a shared performance domain containing two P CPUs +and four M CPUs. The T8152 ACC request uses a five-bit hardware state, with +state 2 as the first ADT operating point. Do not use the older voltage or PLL +initialization sequences. + +Failure cases and qualification +------------------------------ + +Before testing, account for incorrect MPIDRs, incomplete spin-table release, +broken cross-CPU interrupts, a stalled architectural timer, mismatched OPP +indices, an unknown inherited request, a stuck busy bit, an unacknowledged +transition, and another agent changing the request. A transition failure must +latch until the policy is recreated. A requested frequency is not a measured +frequency. + +The initial tables expose only states 2 and 3. Higher states require an SMC +thermal policy; this port does not qualify them. The loader must hand over +within the advertised table. Keep firmware voltage, PLL and throttling +configuration intact. + +On physical J873g, capture the source revision, image and DTB hashes, boot log, +online CPU list, per-CPU workload completion, interrupt counters and both +cpufreq policies. Exercise both exposed states in each domain, compare timed +dependent workloads, restore the low states and check the kernel log. Save +the commands and a machine-readable receipt. Native m1n1 SMP results alone +do not qualify Linux scheduling. diff --git a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml index d44c3bc580b72..0a81abfbd8b09 100644 --- a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml +++ b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml @@ -36,6 +36,7 @@ properties: - const: apple,s5l8960x-cluster-cpufreq - const: apple,s5l8960x-cluster-cpufreq - const: apple,t8140-cluster-cpufreq + - const: apple,t8152-cluster-cpufreq - items: - const: apple,t6020-cluster-cpufreq - const: apple,t8112-cluster-cpufreq diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 106645cedb191..38f7112287bf2 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -63,6 +63,8 @@ struct apple_soc_cpufreq_info { bool has_ps2; bool verify_transition; + u32 transition_timeout_us; + u32 max_unmanaged_pstate; u64 min_pstate; u64 max_pstate; u64 cur_pstate_mask; @@ -116,12 +118,24 @@ static const struct apple_soc_cpufreq_info soc_t8112_info = { */ static const struct apple_soc_cpufreq_info soc_t8140_info = { .verify_transition = true, + .max_unmanaged_pstate = 2, .min_pstate = 1, .max_pstate = 31, .ps1_mask = APPLE_DVFS_CMD_PS1, .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, }; +/* T8152 ACC uses state 2 as the first OPP and shares P/M requests. */ +static const struct apple_soc_cpufreq_info soc_t8152_info = { + .verify_transition = true, + .transition_timeout_us = 2000, + .max_unmanaged_pstate = 3, + .min_pstate = 2, + .max_pstate = 31, + .ps1_mask = APPLE_DVFS_CMD_PS1, + .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, +}; + static const struct apple_soc_cpufreq_info soc_default_info = { .has_ps2 = false, .max_pstate = 15, @@ -151,9 +165,18 @@ static const struct of_device_id apple_soc_cpufreq_of_match[] __maybe_unused = { .compatible = "apple,t8140-cluster-cpufreq", .data = &soc_t8140_info, }, + { + .compatible = "apple,t8152-cluster-cpufreq", + .data = &soc_t8152_info, + }, {} }; +static u32 apple_soc_cpufreq_timeout(const struct apple_soc_cpufreq_info *info) +{ + return info->transition_timeout_us ?: APPLE_DVFS_TRANSITION_TIMEOUT; +} + static unsigned int apple_soc_cpufreq_get_rate(unsigned int cpu) { struct cpufreq_policy *policy; @@ -205,7 +228,7 @@ static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, if (readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg, !(reg & APPLE_DVFS_CMD_BUSY), 2, - APPLE_DVFS_TRANSITION_TIMEOUT)) { + apple_soc_cpufreq_timeout(priv->info))) { if (priv->info->verify_transition) { priv->transition_failed = true; dev_err(priv->cpu_dev, @@ -240,7 +263,7 @@ static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg, !(reg & APPLE_DVFS_CMD_BUSY) && FIELD_GET(APPLE_DVFS_CMD_PS1, reg) == pstate, - 2, APPLE_DVFS_TRANSITION_TIMEOUT)) { + 2, apple_soc_cpufreq_timeout(priv->info))) { /* Do not issue another request or guess a rollback after failure. */ priv->transition_failed = true; dev_err(priv->cpu_dev, @@ -363,7 +386,7 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) /* Registration must not silently reset an unknown inherited state. */ ret = readq_poll_timeout_atomic(reg_base + APPLE_DVFS_CMD, cmd, !(cmd & APPLE_DVFS_CMD_BUSY), 2, - APPLE_DVFS_TRANSITION_TIMEOUT); + apple_soc_cpufreq_timeout(info)); if (ret) goto out_free_cpufreq_table; state = FIELD_GET(APPLE_DVFS_CMD_PS1, cmd); @@ -387,7 +410,7 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev); if (!transition_latency) { /* Conservative transaction bound, not a measured transition time. */ - transition_latency = APPLE_DVFS_TRANSITION_TIMEOUT * NSEC_PER_USEC; + transition_latency = apple_soc_cpufreq_timeout(info) * NSEC_PER_USEC; if (info->verify_transition) transition_latency *= 2; } @@ -398,11 +421,12 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) policy->suspend_freq = freq_table[0].frequency; if (info->verify_transition) { /* - * P-states above 2 need a thermal policy that can cap them; - * refuse such a table rather than run it unthrottled. + * P-states above the unmanaged ceiling need a thermal policy + * that can cap them; refuse such a table rather than run it + * unthrottled. */ cpufreq_for_each_valid_entry(p, policy->freq_table) { - if (p->driver_data > 2) { + if (p->driver_data > info->max_unmanaged_pstate) { dev_err(cpu_dev, "P-state %u requires a CPU thermal policy\n", p->driver_data); ret = -ENODEV; From 45486aa32e99521ccb2481583b03257927b2522b Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 12:37:37 -0400 Subject: [PATCH 0390/2680] dt-bindings: cpufreq: apple: describe the T8142 cluster controller Add the M5 compatible for its five-bit P-state command interface. The controller retains firmware voltage and CLPC/PMP configuration. Signed-off-by: Acelogic --- .../devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml index 0a81abfbd8b09..a27f4314e0387 100644 --- a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml +++ b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml @@ -36,6 +36,7 @@ properties: - const: apple,s5l8960x-cluster-cpufreq - const: apple,s5l8960x-cluster-cpufreq - const: apple,t8140-cluster-cpufreq + - const: apple,t8142-cluster-cpufreq - const: apple,t8152-cluster-cpufreq - items: - const: apple,t6020-cluster-cpufreq From 0925f606df443412218f341f5b1e7879de22db1a Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 12:37:37 -0400 Subject: [PATCH 0391/2680] cpufreq: apple: add T8142 command-state transitions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use five-bit P-state requests with the existing busy timeout, expected state and completion checks. Preserve firmware voltage, PLL and CLPC/PMP configuration; stop requests after an unexpected state or timeout. Separate T8140's SMC thermal-policy requirement from command verification so T8142 can use verified transitions with its bounded initial OPP table. T8140 retains its thermal requirement and behavior. [ryan: adapted to 7.2.10 — the original split the SMC CPU thermal policy hook out under needs_thermal_policy; that policy has since been removed, so needs_thermal_policy now gates the unmanaged-ceiling check instead, and T8152 keeps it as on aurora-wip.] Signed-off-by: Acelogic --- drivers/cpufreq/apple-soc-cpufreq.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 38f7112287bf2..62b88c3aeecb3 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -63,6 +63,7 @@ struct apple_soc_cpufreq_info { bool has_ps2; bool verify_transition; + bool needs_thermal_policy; u32 transition_timeout_us; u32 max_unmanaged_pstate; u64 min_pstate; @@ -118,6 +119,7 @@ static const struct apple_soc_cpufreq_info soc_t8112_info = { */ static const struct apple_soc_cpufreq_info soc_t8140_info = { .verify_transition = true, + .needs_thermal_policy = true, .max_unmanaged_pstate = 2, .min_pstate = 1, .max_pstate = 31, @@ -125,9 +127,19 @@ static const struct apple_soc_cpufreq_info soc_t8140_info = { .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, }; +/* T8142 command-state readback; firmware retains CLPC/PMP ownership. */ +static const struct apple_soc_cpufreq_info soc_t8142_info = { + .verify_transition = true, + .min_pstate = 1, + .max_pstate = 31, + .ps1_mask = APPLE_DVFS_CMD_PS1, + .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, +}; + /* T8152 ACC uses state 2 as the first OPP and shares P/M requests. */ static const struct apple_soc_cpufreq_info soc_t8152_info = { .verify_transition = true, + .needs_thermal_policy = true, .transition_timeout_us = 2000, .max_unmanaged_pstate = 3, .min_pstate = 2, @@ -165,6 +177,10 @@ static const struct of_device_id apple_soc_cpufreq_of_match[] __maybe_unused = { .compatible = "apple,t8140-cluster-cpufreq", .data = &soc_t8140_info, }, + { + .compatible = "apple,t8142-cluster-cpufreq", + .data = &soc_t8142_info, + }, { .compatible = "apple,t8152-cluster-cpufreq", .data = &soc_t8152_info, @@ -419,7 +435,7 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) policy->dvfs_possible_from_any_cpu = true; policy->fast_switch_possible = !info->verify_transition; policy->suspend_freq = freq_table[0].frequency; - if (info->verify_transition) { + if (info->needs_thermal_policy) { /* * P-states above the unmanaged ceiling need a thermal policy * that can cap them; refuse such a table rather than run it From 0eed25b980c1c371635ac1b18095b6eed6c7029d Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 18:48:20 -0400 Subject: [PATCH 0392/2680] hwmon: macsmc: discover firmware temperature keys on opt-in boards Enumerate readable float temperature keys from the running SMC and label them by key identity. Keep dynamically assigned channels separate from static thermal policy, and leave existing board sensor definitions unchanged. Signed-off-by: Acelogic --- drivers/hwmon/macsmc-hwmon.c | 64 ++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index 13e38b25c75fb..36598f06ecaf2 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -501,6 +501,66 @@ static int macsmc_hwmon_create_sensor(struct device *dev, struct apple_smc *smc, return 0; } +/* Discover reporting-only temperatures without a board-specific key table. */ +static int macsmc_hwmon_discover_temperatures(struct macsmc_hwmon *hwmon) +{ + struct apple_smc *smc = hwmon->smc; + struct macsmc_hwmon_sensor *sensor; + struct apple_smc_key_info info; + smc_key key, previous = 0; + u32 i; + int ret; + + if (!of_property_read_bool(hwmon->dev->of_node, "apple,discover-temperatures")) + return 0; + + /* Dynamic channel numbers must not replace a DT-described thermal policy. */ + if (hwmon->temp.count || + of_property_read_bool(hwmon->dev->of_node, "apple,cpu-thermal-policy")) + return -EINVAL; + + /* Bound both allocation and mailbox transactions for invalid firmware data. */ + if (!smc->key_count || smc->key_count > U16_MAX) + return -EINVAL; + + hwmon->temp.sensors = devm_kcalloc(hwmon->dev, smc->key_count, + sizeof(*sensor), GFP_KERNEL); + if (!hwmon->temp.sensors) + return -ENOMEM; + + for (i = 0; i < smc->key_count; i++) { + ret = apple_smc_get_key_by_index(smc, i, &key); + if (ret < 0) + return ret; + /* SMC key tables are strictly ordered; reject duplicates as well. */ + if (i && key <= previous) + return -EINVAL; + previous = key; + if ((key >> 24) < 'T') + continue; + if ((key >> 24) > 'T') + break; + + ret = apple_smc_get_key_info(smc, key, &info); + if (ret < 0) + return ret; + if (!(info.flags & APPLE_SMC_READABLE) || + (info.flags & APPLE_SMC_FUNCTION) || + info.type_code != __SMC_KEY('f', 'l', 't', ' ') || info.size != 4) + continue; + + sensor = &hwmon->temp.sensors[hwmon->temp.count++]; + sensor->macsmc_key = key; + sensor->info = info; + sensor->attrs = HWMON_T_INPUT | HWMON_T_LABEL; + /* Key identity only: do not invent locations or calibration. */ + snprintf(sensor->label, sizeof(sensor->label), "%c%c%c%c", + key >> 24, (key >> 16) & 0xff, (key >> 8) & 0xff, key & 0xff); + } + + return 0; +} + /* * Fan data is exposed by the SMC as multiple sensors. * @@ -810,6 +870,10 @@ static int macsmc_hwmon_probe(struct platform_device *pdev) return ret; } + ret = macsmc_hwmon_discover_temperatures(hwmon); + if (ret) + return dev_err_probe(hwmon->dev, ret, "Temperature discovery failed\n"); + if (!hwmon->curr.count && !hwmon->fan.count && !hwmon->power.count && !hwmon->temp.count && !hwmon->volt.count) { From a74a2b1ad059f351b6896f89c2bfe846c1faddce Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 01:30:05 +1000 Subject: [PATCH 0393/2680] dt-bindings: hwmon: apple,smc-hwmon: Allow #thermal-sensor-cells The hwmon core registers every temperature channel of the SMC hwmon device as a thermal sensor (the driver sets HWMON_C_REGISTER_TZ), so a board can back thermal zones with SMC temperatures. The binding does not allow the property that makes the node a sensor provider, though. Allow #thermal-sensor-cells with one cell, the index of the temperature channel in child-node order. Signed-off-by: Ryan Murray --- .../devicetree/bindings/hwmon/apple,smc-hwmon.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml index 64149ef4cc59c..098ad1cdbc484 100644 --- a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml +++ b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml @@ -40,6 +40,12 @@ properties: compatible: const: apple,smc-hwmon + "#thermal-sensor-cells": + const: 1 + description: + The cell is the zero-based index of a temperature channel, counting + the temperature-* child nodes in order. + patternProperties: "^current-[A-Za-z0-9]{4}$": $ref: "#/$defs/sensor" From e0547a304b6f721fa81572bd59c3e262f088fd60 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 02:06:39 +1000 Subject: [PATCH 0394/2680] thermal: gov_power_allocator: Only warn about sustainable power when it is used power_allocator_bind() warns that sustainable power will be estimated for every zone that does not provide it, including zones without any passive or active trip point. The governor never controls such a zone: power_allocator_manage() returns early when there is no trip_max, and the estimate is never computed. With power_allocator as the default governor every trip-less zone, such as the one each power supply registers for its battery, logs this warning at boot. Warn only when the zone has a trip the governor controls. Signed-off-by: Ryan Murray --- drivers/thermal/gov_power_allocator.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/thermal/gov_power_allocator.c b/drivers/thermal/gov_power_allocator.c index 37f2e22a999e6..31f778dd49ed2 100644 --- a/drivers/thermal/gov_power_allocator.c +++ b/drivers/thermal/gov_power_allocator.c @@ -728,10 +728,10 @@ static int power_allocator_bind(struct thermal_zone_device *tz) params->allocated_tzp = true; } - if (!tz->tzp->sustainable_power) - dev_warn(&tz->device, "power_allocator: sustainable_power will be estimated\n"); - else + if (tz->tzp->sustainable_power) params->sustainable_power = tz->tzp->sustainable_power; + else if (params->trip_max) + dev_warn(&tz->device, "power_allocator: sustainable_power will be estimated\n"); if (params->trip_max) estimate_pid_constants(tz, tz->tzp->sustainable_power, From a0330be1a9327f88e4c5e32a3fb6826e5b31affe Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 28 Sep 2026 21:24:40 +1000 Subject: [PATCH 0395/2680] power: supply: macsmc: Report unavailable battery times as ENODATA The SMC reports 0xffff in B0TE and B0TF when it has no time-to-empty or time-to-full estimate, for example B0TE while on AC and B0TF with a full battery. The driver already recognises this sentinel but reports it as 0 seconds, which userspace cannot tell apart from a real "empty now" or "full now" estimate. This affects every machine the driver supports; it was observed on J700 (T8140), where both keys read 0xffff with the battery full on AC. Return -ENODATA instead, as max17042_battery does for its own 0xffff time sentinel. The power-supply core logs -ENODATA only at debug level and leaves the property out of the uevent. While here, return the SMC read error directly instead of computing intval from vu16 when the read failed. Hardware: candidate 15 and 17 included this change. On J700 with a full battery on AC, status was Full and capacity 100%; both unavailable time properties were absent. No charging or discharging transition was tested on this exact stack. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/power/supply/macsmc-power.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c index 3eab305357cd9..900d76a305dce 100644 --- a/drivers/power/supply/macsmc-power.c +++ b/drivers/power/supply/macsmc-power.c @@ -475,11 +475,19 @@ static int macsmc_battery_get_property(struct power_supply *psy, break; case POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW: ret = apple_smc_read_u16(power->smc, SMC_KEY(B0TE), &vu16); - val->intval = vu16 == 0xffff ? 0 : vu16 * 60; + if (ret) + break; + if (vu16 == U16_MAX) + return -ENODATA; + val->intval = vu16 * 60; break; case POWER_SUPPLY_PROP_TIME_TO_FULL_NOW: ret = apple_smc_read_u16(power->smc, SMC_KEY(B0TF), &vu16); - val->intval = vu16 == 0xffff ? 0 : vu16 * 60; + if (ret) + break; + if (vu16 == U16_MAX) + return -ENODATA; + val->intval = vu16 * 60; break; case POWER_SUPPLY_PROP_CAPACITY: ret = apple_smc_read_u8(power->smc, SMC_KEY(BUIC), &vu8); From 5df649a389730c4be770565ca7907bbad58435b3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:46:56 +1100 Subject: [PATCH 0396/2680] cpufreq: apple: retain the unmanaged P-state ceiling on T8152 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR 43 replaces the private J700 thermal controller with thermal zones and cpufreq cooling. On the current base, T8152 also uses the private-policy prerequisite. Removing that check globally allows its high P-states even though no replacement policy has been provided for that SoC. Stop requiring the private policy on T8140, whose thermal zones this series supplies. Keep rejecting tables above the unmanaged ceiling for other SoCs that still require qualified thermal protection. Preserve the current SoC-specific transition timeout and verification paths. [ryan: adapted to 7.2.10 — the unmanaged-ceiling check already sits under needs_thermal_policy here, so this only takes T8140 out of it and rewords the check.] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/cpufreq/apple-soc-cpufreq.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 62b88c3aeecb3..7577d318ce12e 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -119,8 +119,6 @@ static const struct apple_soc_cpufreq_info soc_t8112_info = { */ static const struct apple_soc_cpufreq_info soc_t8140_info = { .verify_transition = true, - .needs_thermal_policy = true, - .max_unmanaged_pstate = 2, .min_pstate = 1, .max_pstate = 31, .ps1_mask = APPLE_DVFS_CMD_PS1, @@ -435,19 +433,14 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) policy->dvfs_possible_from_any_cpu = true; policy->fast_switch_possible = !info->verify_transition; policy->suspend_freq = freq_table[0].frequency; + /* Unqualified SoCs still require protection above their safe ceiling. */ if (info->needs_thermal_policy) { - /* - * P-states above the unmanaged ceiling need a thermal policy - * that can cap them; refuse such a table rather than run it - * unthrottled. - */ cpufreq_for_each_valid_entry(p, policy->freq_table) { - if (p->driver_data > info->max_unmanaged_pstate) { - dev_err(cpu_dev, "P-state %u requires a CPU thermal policy\n", - p->driver_data); - ret = -ENODEV; - goto out_free_cpufreq_table; - } + if (p->driver_data <= info->max_unmanaged_pstate) + continue; + dev_err(cpu_dev, "higher P-states require a qualified thermal policy\n"); + ret = -ENODEV; + goto out_free_cpufreq_table; } } From c48a41bc4059643b4fb8ea7089ad380c2bfea607 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:57:15 +1100 Subject: [PATCH 0397/2680] cpufreq: apple-soc: enforce PMP fast-die limits through QoS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PMP publishes a per-cluster effort faster than the SMC reports die temperature. Sample the effort and enforce a maximum-frequency QoS request. The cpufreq core combines it with thermal cooling limits and reports reduced capacity to the scheduler. Keep the existing IPA policy. Use the OPP power table to map increasing effort to a monotonic power ceiling. Until the PMP is running, retain a ceiling of one third of the maximum OPP power. Deferrable timers sample busy CPUs without waking idle clusters; an hrtimer follows a reduced ceiling so idle capacity can recover. Drain timers and update work before removing the QoS request. Port Ryan Murray's original Linux-tested direct limiter. Apply the public IPA series' removal of the obsolete SMC policy hooks while preserving command-state transition verification. Keep the firmware- limit lifecycle separate from the generic policy cleanup so the public thermal-series removals merge without overlapping new hooks. No additional thermal policy or experimental smoothing is introduced. [ryan: adapted to 7.2.10 — the SMC CPU thermal policy hooks this removed were never ported, so only the limiter remains; the unmanaged-ceiling check moves ahead of the limiter set-up so its error path cannot leak the QoS request. The MAINTAINERS hunk is left to aurora/bits/900-config-tooling with the J700 section it edits. Needs from aurora/bits/010-soc.] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/cpufreq/apple-soc-cpufreq.c | 534 +++++++++++++++++++++++++++- 1 file changed, 518 insertions(+), 16 deletions(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 7577d318ce12e..359abc45e14e6 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -7,6 +7,18 @@ * Based on scpi-cpufreq.c */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + #include #include #include @@ -60,6 +72,118 @@ #define APPLE_DVFS_TRANSITION_TIMEOUT 400 +/* + * Fast die-temperature limit + * + * The power management processor (PMP) of some SoCs runs a control loop on + * its own die-temperature sensors and publishes, for each CPU cluster, an + * 8-bit effort: 0 requests no limit, and the effort rises towards 255 as the + * die exceeds the firmware's target. The firmware refreshes the effort every + * few milliseconds, far faster than the SMC temperature sensors, so a cluster + * can run at full speed until the die hot spot reaches the firmware's target. + * + * The firmware requests the limit and the OS enforces it, as scmi-cpufreq + * does with SCMI performance limits: the limit is a FREQ_QOS_MAX request on + * the policy, so the cpufreq core applies it to the governor and reports it + * to the scheduler as cpufreq pressure. apple_fast_die_limit() maps an + * effort to that limit. + * + * The limit fails safe. The effort is only valid while the PMP reports that + * it is running, and the PMP may start late or never, for instance when its + * driver is not loaded or the bootloader did not prepare it. Until it + * reports that it is running, the cluster is held at a fallback limit: the + * fastest state whose power is at most a third of the fastest state's. If + * it has not reported running 30 s after the limit was set up, a warning + * says that the fast limit is unavailable; the fallback limit stays until + * the PMP runs. + * + * The effort has to be sampled, since no interrupt is known to signal a new + * one under Linux. While the limit is at the cluster's maximum, every CPU + * of the cluster has a pinned, deferrable timer that samples the effort once + * a tick, and the first CPU to run in a tick takes the sample. Deferrable + * timers do not wake idle CPUs, so only busy CPUs sample, and an idle + * cluster, which produces no heat, costs nothing. Once the limit is below + * the maximum, a per-cluster hrtimer samples every 5 ms whether the cluster + * is busy or not, until the limit is back at the maximum: an idle cluster + * would otherwise keep a stale limit, which the scheduler sees as reduced + * capacity and so keeps work away from the cluster. While the PMP is not + * running, the hrtimer only checks every 100 ms, and every second once the + * warning has been given. A nohz_full CPU can run a task with its tick + * stopped, and so without sampling, so a cluster with nohz_full CPUs + * samples from the hrtimer all the time. + * + * A sample reads the effort through the PMP report driver; only when the + * limit it maps to changes is a work item queued to update the QoS request, + * which must be done in process context. + */ +#define APPLE_FAST_DIE_EFFORT_MAX 255 +#define APPLE_FAST_DIE_POLL_MS 5 +#define APPLE_FAST_DIE_WAIT_MS 100 +#define APPLE_FAST_DIE_GAVE_UP_MS 1000 +#define APPLE_FAST_DIE_WARN_S 30 + +/** + * struct apple_fast_die - fast die-temperature limit of one cluster + * @policy: policy of the cluster + * @report: PMP report region, or NULL if the limit is misdescribed + * @lane: lane of the cluster in the fast die-temperature report + * @lock: serialises samples and protects the fields below it + * @limit: last limit derived from a sample, in kHz; 0 forces an update + * @polling: @poll is armed + * @waiting: the last sample found no running PMP + * @warned: the fast limit has been reported unavailable + * @per_cpu: the per-CPU deferrable timers sample while the limit is at @max + * @stopping: ->exit() is tearing the limit down; do not arm @poll again + * @max: frequency of the fastest state, in kHz + * @fallback: limit while the PMP is not running, in kHz + * @start: jiffies when the limit was set up + * @last_poll: jiffies of the last sample by a per-CPU timer + * @poll: non-deferrable sampling timer, armed while the limit is below @max + * @work: updates @req to @limit + * @req: FREQ_QOS_MAX request that enforces @limit + * @cpus: every CPU of the cluster, online or not + * @nr_states: number of frequency-table entries + * @power: power of each frequency-table entry, in microwatts from the OPP + * table, or the frequency when the OPPs do not describe their power + */ +struct apple_fast_die { + struct cpufreq_policy *policy; + struct device_node *report; + unsigned int lane; + spinlock_t lock; /* serialises samples; protects the fields below */ + unsigned int limit; + bool polling; + bool waiting; + bool warned; + bool per_cpu; + bool stopping; + unsigned int max; + unsigned int fallback; + unsigned long start; + unsigned long last_poll; + struct hrtimer poll; + struct work_struct work; + struct freq_qos_request req; + cpumask_var_t cpus; + unsigned int nr_states; + unsigned long power[] __counted_by(nr_states); +}; + +/** + * struct apple_fast_die_timer - per-CPU sampling timer + * @timer: pinned, deferrable timer that samples the effort + * @fd: limit of the cluster of this CPU, NULL when it has none; written with + * the CPU hotplug lock held, like the hotplug callbacks that read it + */ +struct apple_fast_die_timer { + struct timer_list timer; + struct apple_fast_die *fd; +}; + +static DEFINE_PER_CPU(struct apple_fast_die_timer, apple_fast_die_timers); +/* Dynamic CPU hotplug state of the per-CPU timers, 0 if there is none. */ +static int apple_fast_die_hp_state; + struct apple_soc_cpufreq_info { bool has_ps2; bool verify_transition; @@ -75,6 +199,7 @@ struct apple_soc_cpufreq_info { }; struct apple_cpu_priv { + struct apple_fast_die *fast_die; struct device *cpu_dev; void __iomem *reg_base; const struct apple_soc_cpufreq_info *info; @@ -83,6 +208,7 @@ struct apple_cpu_priv { }; static struct cpufreq_driver apple_soc_cpufreq_driver; +static void apple_soc_cpufreq_exit_with_limits(struct cpufreq_policy *policy); static const struct apple_soc_cpufreq_info soc_s5l8960x_info = { .has_ps2 = false, @@ -301,7 +427,8 @@ static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy, static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy, void __iomem **reg_base, - const struct apple_soc_cpufreq_info **info) + const struct apple_soc_cpufreq_info **info, + struct device_node **cluster) { struct of_phandle_args args; const struct of_device_id *match; @@ -322,13 +449,338 @@ static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy, *info = match->data; *reg_base = of_iomap(args.np, 0); - of_node_put(args.np); - if (!*reg_base) + if (!*reg_base) { + of_node_put(args.np); return -ENOMEM; + } + *cluster = args.np; + + return 0; +} + +/* The fastest state whose power is at most @budget, or the slowest. */ +static unsigned int apple_fast_die_fit(const struct apple_fast_die *fd, + u64 budget) +{ + unsigned int i, idx = 0; + + for (i = 0; i < fd->nr_states; i++) + if (fd->power[i] <= budget) + idx = i; + + return fd->policy->freq_table[idx].frequency; +} + +/* + * Map an effort to a frequency limit, in kHz: the fastest state whose power + * fits a budget of (255 - effort) / 255 of the fastest state's power. Any + * mapping that lowers the limit monotonically as the effort rises would do, + * since the firmware's loop is closed on the die temperature and corrects + * the error; power is the natural axis for the output of a temperature + * controller. + */ +static unsigned int apple_fast_die_limit(const struct apple_fast_die *fd, + unsigned int effort) +{ + /* + * Multiply first: dividing the fastest state's power by 255 first + * rounds it down, which would limit an unlimited cluster one state low. + */ + return apple_fast_die_fit(fd, + div_u64((u64)fd->power[fd->nr_states - 1] * + (APPLE_FAST_DIE_EFFORT_MAX - effort), + APPLE_FAST_DIE_EFFORT_MAX)); +} + +static ktime_t apple_fast_die_interval(const struct apple_fast_die *fd) +{ + if (!fd->waiting) + return ms_to_ktime(APPLE_FAST_DIE_POLL_MS); + if (!fd->warned) + return ms_to_ktime(APPLE_FAST_DIE_WAIT_MS); + return ms_to_ktime(APPLE_FAST_DIE_GAVE_UP_MS); +} + +/* Arm the non-deferrable timer if the limit needs it. Call with @lock held. */ +static void apple_fast_die_arm(struct apple_fast_die *fd) +{ + if (fd->stopping || fd->polling || + (fd->per_cpu && fd->limit >= fd->max)) + return; + + fd->polling = true; + hrtimer_start(&fd->poll, apple_fast_die_interval(fd), + HRTIMER_MODE_REL_SOFT); +} + +/* Take a sample and queue an update if the limit changes. Call with @lock held. */ +static void apple_fast_die_sample(struct apple_fast_die *fd) +{ + struct apple_cpu_priv *priv = fd->policy->driver_data; + unsigned int limit; + u8 effort; + int ret; + + ret = apple_pmp_report_fast_die_effort(fd->report, fd->lane, &effort); + if (!ret) { + limit = apple_fast_die_limit(fd, effort); + fd->waiting = false; + } else { + limit = fd->fallback; + fd->waiting = true; + if (!fd->warned && + time_after(jiffies, fd->start + APPLE_FAST_DIE_WARN_S * HZ)) { + fd->warned = true; + dev_warn(priv->cpu_dev, + "PMP not running after %d s (%d): fast die-temperature limit unavailable, CPUs %*pbl stay limited to %u kHz\n", + APPLE_FAST_DIE_WARN_S, ret, + cpumask_pr_args(fd->cpus), limit); + } + } + + if (limit != fd->limit) { + WRITE_ONCE(fd->limit, limit); + schedule_work(&fd->work); + } +} + +static void apple_fast_die_work(struct work_struct *work) +{ + struct apple_fast_die *fd = container_of(work, struct apple_fast_die, work); + struct apple_cpu_priv *priv = fd->policy->driver_data; + unsigned int limit = READ_ONCE(fd->limit); + int ret; + + ret = freq_qos_update_request(&fd->req, limit); + if (ret >= 0) + return; + + dev_warn_ratelimited(priv->cpu_dev, + "failed to update the fast die-temperature limit: %d\n", ret); + /* Make the next sample queue the update again, and make sure it comes. */ + spin_lock_bh(&fd->lock); + if (fd->limit == limit) { + fd->limit = 0; + apple_fast_die_arm(fd); + } + spin_unlock_bh(&fd->lock); +} + +static enum hrtimer_restart apple_fast_die_poll(struct hrtimer *t) +{ + struct apple_fast_die *fd = container_of(t, struct apple_fast_die, poll); + enum hrtimer_restart restart = HRTIMER_NORESTART; + + spin_lock(&fd->lock); + apple_fast_die_sample(fd); + if (!fd->stopping && (!fd->per_cpu || fd->limit < fd->max)) { + hrtimer_forward_now(t, apple_fast_die_interval(fd)); + restart = HRTIMER_RESTART; + } else { + fd->polling = false; + } + spin_unlock(&fd->lock); + + return restart; +} + +static void apple_fast_die_tick(struct timer_list *t) +{ + struct apple_fast_die_timer *ft = timer_container_of(ft, t, timer); + struct apple_fast_die *fd = ft->fd; + unsigned long now = jiffies; + + /* + * The first busy CPU of the cluster to run in a tick takes the sample, + * unless the non-deferrable timer is sampling. + */ + if (xchg(&fd->last_poll, now) != now && spin_trylock(&fd->lock)) { + if (!fd->polling) { + apple_fast_die_sample(fd); + apple_fast_die_arm(fd); + } + spin_unlock(&fd->lock); + } + mod_timer(t, now + 1); +} + +static int apple_fast_die_cpu_online(unsigned int cpu) +{ + struct apple_fast_die_timer *ft = per_cpu_ptr(&apple_fast_die_timers, cpu); + + if (!ft->fd) + return 0; + + /* Wait for a running sample, which rearms the timer, before arming it. */ + timer_delete_sync(&ft->timer); + ft->timer.expires = jiffies + 1; + add_timer_on(&ft->timer, cpu); + + return 0; +} + +static int apple_fast_die_cpu_offline(unsigned int cpu) +{ + struct apple_fast_die_timer *ft = per_cpu_ptr(&apple_fast_die_timers, cpu); + + if (ft->fd) + timer_delete_sync(&ft->timer); return 0; } +/* + * Called at the end of ->init(), with the CPU hotplug lock held. The core + * fills policy->related_cpus only after ->init() returns, and may call ->exit() + * before it does, so the limit keeps its own copy of the cluster's CPUs. + */ +static int apple_fast_die_init(struct cpufreq_policy *policy, + struct device_node *cluster) +{ + struct apple_cpu_priv *priv = policy->driver_data; + struct of_phandle_args args; + struct apple_fast_die *fd; + unsigned int cpu, i, n = 0; + bool have_power = true; + int parsed, ret; + + parsed = of_parse_phandle_with_fixed_args(cluster, "apple,fast-die-effort", + 1, 0, &args); + if (parsed == -ENOENT) + return 0; + + while (policy->freq_table[n].frequency != CPUFREQ_TABLE_END) + n++; + + fd = kzalloc_flex(*fd, power, n); + if (!fd) { + ret = -ENOMEM; + goto err_put; + } + fd->nr_states = n; + if (!zalloc_cpumask_var(&fd->cpus, GFP_KERNEL)) { + ret = -ENOMEM; + goto err_free; + } + + for (i = 0; i < n; i++) { + unsigned long rate = policy->freq_table[i].frequency * 1000UL + 999; + struct dev_pm_opp *opp = dev_pm_opp_find_freq_floor(priv->cpu_dev, &rate); + + if (IS_ERR(opp)) { + ret = PTR_ERR(opp); + goto err_free; + } + fd->power[i] = dev_pm_opp_get_power(opp); + dev_pm_opp_put(opp); + if (!fd->power[i]) + have_power = false; + } + /* Without an energy model, frequency stands in for power. */ + if (!have_power) + for (i = 0; i < n; i++) + fd->power[i] = policy->freq_table[i].frequency; + + fd->policy = policy; + spin_lock_init(&fd->lock); + INIT_WORK(&fd->work, apple_fast_die_work); + hrtimer_setup(&fd->poll, apple_fast_die_poll, CLOCK_MONOTONIC, + HRTIMER_MODE_REL_SOFT); + cpumask_copy(fd->cpus, policy->cpus); + fd->per_cpu = apple_fast_die_hp_state > 0; + for_each_cpu(cpu, fd->cpus) + if (tick_nohz_full_cpu(cpu)) + fd->per_cpu = false; + fd->max = policy->freq_table[n - 1].frequency; + fd->fallback = apple_fast_die_fit(fd, div_u64(fd->power[n - 1], 3)); + fd->start = jiffies; + + if (parsed) { + /* The limit is meant to be there, so keep the fallback. */ + dev_warn(priv->cpu_dev, + "invalid apple,fast-die-effort (%d): CPUs %*pbl stay limited to %u kHz\n", + parsed, cpumask_pr_args(fd->cpus), fd->fallback); + fd->warned = true; + } else { + fd->report = args.np; + fd->lane = args.args[0]; + } + + /* Hold the fallback limit until a sample finds the PMP running. */ + fd->limit = fd->fallback; + ret = freq_qos_add_request(&policy->constraints, &fd->req, FREQ_QOS_MAX, + fd->limit); + if (ret < 0) + goto err_free; + + priv->fast_die = fd; + + if (fd->per_cpu) { + for_each_cpu(cpu, fd->cpus) { + struct apple_fast_die_timer *ft = + per_cpu_ptr(&apple_fast_die_timers, cpu); + + timer_setup(&ft->timer, apple_fast_die_tick, + TIMER_DEFERRABLE | TIMER_PINNED); + ft->fd = fd; + if (cpu_online(cpu)) + apple_fast_die_cpu_online(cpu); + } + } + + spin_lock_bh(&fd->lock); + apple_fast_die_arm(fd); + spin_unlock_bh(&fd->lock); + + dev_dbg(priv->cpu_dev, "fast die-temperature limit set up, fallback %u kHz\n", + fd->fallback); + + return 0; + +err_free: + free_cpumask_var(fd->cpus); + kfree(fd); +err_put: + if (!parsed) + of_node_put(args.np); + return ret; +} + +static void apple_fast_die_exit(struct apple_cpu_priv *priv) +{ + struct apple_fast_die *fd = priv->fast_die; + unsigned int cpu; + + if (!fd) + return; + + /* + * Keep the update work from arming the hrtimer again, stop the per-CPU + * timers, which can arm it too, then the hrtimer, and then drain the + * update any of them may have queued. + */ + spin_lock_bh(&fd->lock); + fd->stopping = true; + spin_unlock_bh(&fd->lock); + if (fd->per_cpu) { + for_each_cpu(cpu, fd->cpus) { + struct apple_fast_die_timer *ft = + per_cpu_ptr(&apple_fast_die_timers, cpu); + + timer_shutdown_sync(&ft->timer); + ft->fd = NULL; + } + } + hrtimer_cancel(&fd->poll); + cancel_work_sync(&fd->work); + freq_qos_remove_request(&fd->req); + + priv->fast_die = NULL; + of_node_put(fd->report); + free_cpumask_var(fd->cpus); + kfree(fd); +} + static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) { struct cpufreq_frequency_table *p; @@ -339,6 +791,7 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) struct apple_cpu_priv *priv; const struct apple_soc_cpufreq_info *info; struct cpufreq_frequency_table *freq_table; + struct device_node *cluster; cpu_dev = get_cpu_device(policy->cpu); if (!cpu_dev) { @@ -350,7 +803,7 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) if (!priv) return -ENOMEM; - ret = apple_soc_cpufreq_find_cluster(policy, ®_base, &info); + ret = apple_soc_cpufreq_find_cluster(policy, ®_base, &info, &cluster); if (ret) { dev_err(cpu_dev, "%s: failed to get cluster info: %d\n", __func__, ret); goto out_free_priv; @@ -415,12 +868,30 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) priv->expected_pstate = state; } + /* Unqualified SoCs still require protection above their safe ceiling. */ + if (info->needs_thermal_policy) { + cpufreq_for_each_valid_entry(p, freq_table) { + if (p->driver_data <= info->max_unmanaged_pstate) + continue; + dev_err(cpu_dev, "higher P-states require a qualified thermal policy\n"); + ret = -ENODEV; + goto out_free_cpufreq_table; + } + } + priv->cpu_dev = cpu_dev; priv->reg_base = reg_base; priv->info = info; policy->driver_data = priv; policy->freq_table = freq_table; + ret = apple_fast_die_init(policy, cluster); + if (ret) { + dev_err(cpu_dev, "failed to set up the fast die-temperature limit: %d\n", ret); + goto out_clear_policy; + } + of_node_put(cluster); + transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev); if (!transition_latency) { /* Conservative transaction bound, not a measured transition time. */ @@ -433,25 +904,19 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) policy->dvfs_possible_from_any_cpu = true; policy->fast_switch_possible = !info->verify_transition; policy->suspend_freq = freq_table[0].frequency; - /* Unqualified SoCs still require protection above their safe ceiling. */ - if (info->needs_thermal_policy) { - cpufreq_for_each_valid_entry(p, policy->freq_table) { - if (p->driver_data <= info->max_unmanaged_pstate) - continue; - dev_err(cpu_dev, "higher P-states require a qualified thermal policy\n"); - ret = -ENODEV; - goto out_free_cpufreq_table; - } - } return 0; +out_clear_policy: + policy->driver_data = NULL; + policy->freq_table = NULL; out_free_cpufreq_table: dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); out_free_table: dev_pm_opp_of_cpumask_remove_table(policy->cpus); out_iounmap: iounmap(reg_base); + of_node_put(cluster); out_free_priv: kfree(priv); return ret; @@ -478,7 +943,7 @@ static struct cpufreq_driver apple_soc_cpufreq_driver = { .verify = cpufreq_generic_frequency_table_verify, .get = apple_soc_cpufreq_get_rate, .init = apple_soc_cpufreq_init, - .exit = apple_soc_cpufreq_exit, + .exit = apple_soc_cpufreq_exit_with_limits, .target_index = apple_soc_cpufreq_set_target, .fast_switch = apple_soc_cpufreq_fast_switch, .register_em = cpufreq_register_em_with_opp, @@ -486,18 +951,55 @@ static struct cpufreq_driver apple_soc_cpufreq_driver = { .suspend = cpufreq_generic_suspend, }; +/* Drain firmware-limit updates before releasing the policy's OPP table. */ +static void apple_soc_cpufreq_exit_with_limits(struct cpufreq_policy *policy) +{ + apple_fast_die_exit(policy->driver_data); + apple_soc_cpufreq_exit(policy); +} + static int __init apple_soc_cpufreq_module_init(void) { + struct device_node *np; + int ret; + if (!of_machine_is_compatible("apple,arm-platform")) return -ENODEV; - return cpufreq_register_driver(&apple_soc_cpufreq_driver); + /* + * Only SoCs with a fast die-temperature limit need the per-CPU timers + * to follow CPU hotplug. Either order against the cpufreq core's own + * hotplug state is fine: ->init() arms the timers of the CPUs that are + * online, and ->exit() stops them all. Without the state the limit + * still works, sampling from the hrtimer all the time. + */ + np = of_find_node_with_property(NULL, "apple,fast-die-effort"); + if (np) { + of_node_put(np); + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "cpufreq/apple-soc:online", + apple_fast_die_cpu_online, + apple_fast_die_cpu_offline); + if (ret < 0) + pr_warn("apple-cpufreq: no CPU hotplug state (%d), fast die-temperature limit always polls\n", + ret); + else + apple_fast_die_hp_state = ret; + } + + ret = cpufreq_register_driver(&apple_soc_cpufreq_driver); + if (ret && apple_fast_die_hp_state > 0) + cpuhp_remove_state_nocalls(apple_fast_die_hp_state); + + return ret; } module_init(apple_soc_cpufreq_module_init); static void __exit apple_soc_cpufreq_module_exit(void) { cpufreq_unregister_driver(&apple_soc_cpufreq_driver); + if (apple_fast_die_hp_state > 0) + cpuhp_remove_state_nocalls(apple_fast_die_hp_state); } module_exit(apple_soc_cpufreq_module_exit); From f3b9dff56171bf0360a803fb890e1ab03d62869d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 17:04:00 +1100 Subject: [PATCH 0398/2680] cpufreq: apple-soc: keep required fallback limits with an older DT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new limiter replaces the old thermal admission hooks. A missing fast-die property must not skip the fallback on SoCs that require thermal policy. Treat that omission like other invalid limit descriptions while leaving optional legacy platforms unchanged. [ryan: adapted to 7.2.10 — the earlier ceiling change took T8140 out of needs_thermal_policy, which would leave the J700 without the fail-closed fallback this adds. Mark T8140 as needing a thermal policy again and exempt it from the unmanaged ceiling by name instead, as the aurora-wip merge of the thermal series did (commit 026b4131cf05 ("Merge pull request #43 from aurora-silicon/feat/j700-thermal-framework-v2")).] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/cpufreq/apple-soc-cpufreq.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 359abc45e14e6..37b44cccb9b87 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -245,6 +245,8 @@ static const struct apple_soc_cpufreq_info soc_t8112_info = { */ static const struct apple_soc_cpufreq_info soc_t8140_info = { .verify_transition = true, + .needs_thermal_policy = true, + .max_unmanaged_pstate = 2, .min_pstate = 1, .max_pstate = 31, .ps1_mask = APPLE_DVFS_CMD_PS1, @@ -646,7 +648,8 @@ static int apple_fast_die_init(struct cpufreq_policy *policy, parsed = of_parse_phandle_with_fixed_args(cluster, "apple,fast-die-effort", 1, 0, &args); - if (parsed == -ENOENT) + /* Required thermal admission must fail closed with an older DT. */ + if (parsed == -ENOENT && !priv->info->needs_thermal_policy) return 0; while (policy->freq_table[n].frequency != CPUFREQ_TABLE_END) @@ -868,8 +871,8 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) priv->expected_pstate = state; } - /* Unqualified SoCs still require protection above their safe ceiling. */ - if (info->needs_thermal_policy) { + /* T8140 has IPA and the PMP limit; other SoCs keep their safe ceiling. */ + if (info->needs_thermal_policy && info != &soc_t8140_info) { cpufreq_for_each_valid_entry(p, freq_table) { if (p->driver_data <= info->max_unmanaged_pstate) continue; From 8df55e620ae10b416e98d5e70fb1285b78978b75 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 13:58:05 +1100 Subject: [PATCH 0399/2680] dt-bindings: rtc: describe the J700 Baku PMU alarm Describe the RTC function as a child of the existing Baku SPMI NVMEM provider. It shares the whole-second offset cell with the SMC RTC and has a dedicated alarm interrupt. Add the child to the PMIC schema. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/nvmem/apple,spmi-nvmem.yaml | 3 + .../bindings/rtc/apple,j700-pmu-rtc.yaml | 63 +++++++++++++++++++ MAINTAINERS | 1 + 3 files changed, 67 insertions(+) create mode 100644 Documentation/devicetree/bindings/rtc/apple,j700-pmu-rtc.yaml diff --git a/Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml b/Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml index 7a4114c7fbd19..09cdd776f8d8b 100644 --- a/Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml +++ b/Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml @@ -27,6 +27,9 @@ properties: reg: maxItems: 1 + rtc: + $ref: /schemas/rtc/apple,j700-pmu-rtc.yaml# + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/rtc/apple,j700-pmu-rtc.yaml b/Documentation/devicetree/bindings/rtc/apple,j700-pmu-rtc.yaml new file mode 100644 index 0000000000000..b1e289c06fe33 --- /dev/null +++ b/Documentation/devicetree/bindings/rtc/apple,j700-pmu-rtc.yaml @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/rtc/apple,j700-pmu-rtc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple J700 Baku PMU real-time clock + +maintainers: + - Ryan Murray + +description: | + Baku exposes a 65536 Hz counter and an alarm comparator over SPMI. + The clock's whole-second offset is shared with the SMC RTC through an + eight-byte little-endian NVMEM cell. This function is a child of the + PMIC NVMEM provider and uses that provider's SPMI register map. + +allOf: + - $ref: rtc.yaml# + +properties: + compatible: + const: apple,j700-pmu-rtc + + interrupts: + maxItems: 1 + + nvmem-cells: + maxItems: 1 + description: The shared eight-byte whole-second RTC offset. + + nvmem-cell-names: + items: + - const: rtc_offset + + wakeup-source: true + +required: + - compatible + - interrupts + - nvmem-cells + - nvmem-cell-names + +unevaluatedProperties: false + +examples: + - | + #include + #include + + pmic@e { + compatible = "apple,baku-pmic", "apple,spmi-nvmem"; + reg = <0xe SPMI_USID>; + + rtc { + compatible = "apple,j700-pmu-rtc"; + interrupt-parent = <&spmi>; + interrupts = <2 IRQ_TYPE_EDGE_RISING>; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; + wakeup-source; + }; + }; diff --git a/MAINTAINERS b/MAINTAINERS index b60cd73cce49d..2ca5fdac9e4e5 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2616,6 +2616,7 @@ F: Documentation/devicetree/bindings/power/apple* F: Documentation/devicetree/bindings/power/reset/apple,smc-reboot.yaml F: Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml F: Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml +F: Documentation/devicetree/bindings/rtc/apple,j700-pmu-rtc.yaml F: Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml F: Documentation/devicetree/bindings/spi/apple,spi.yaml F: Documentation/devicetree/bindings/spmi/apple,spmi.yaml From f3ad473b872637ef91bc06b00243ed71f7bcbfb0 Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Thu, 8 Oct 2026 14:43:19 +1100 Subject: [PATCH 0400/2680] rtc: add opt-in J700 Baku PMU alarms with retained s2idle interrupts Expose the Baku counter and alarm as a child of the existing SPMI PMIC provider. Share its regmap and NVMEM offset with the SMC RTC. Read the 65536 Hz counter as whole seconds and obtain the 64-bit seconds offset on each operation so that SMC clock updates are visible immediately. Keep time-setting on the existing SMC RTC and expose the PMU alarm here. The original opt-in alarm design was written by Eryk Wieliczko; Eryk's Signed-off-by is outstanding. Independently rewrite the implementation using register facts from private research and Linux observations, without copying private implementation code. Retain his authorship for the design. Retained SPMI/AIC power is required. Opt-in is explicit and alarms are limited to one hour until longer retention is qualified. Reject deeper suspend states and an alarm inherited from firmware. Acknowledge alarms in a threaded interrupt and fail closed on acknowledgment I/O errors. No off-domain wake or power-saving claim is made. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/nvmem/apple-spmi-nvmem.c | 10 +- drivers/rtc/Kconfig | 11 + drivers/rtc/Makefile | 1 + drivers/rtc/rtc-apple-pmu.c | 369 +++++++++++++++++++++++++++++++ 5 files changed, 391 insertions(+), 1 deletion(-) create mode 100644 drivers/rtc/rtc-apple-pmu.c diff --git a/MAINTAINERS b/MAINTAINERS index 2ca5fdac9e4e5..e3e20f7aca64a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2649,6 +2649,7 @@ F: drivers/power/reset/macsmc-reboot.c F: drivers/power/supply/macsmc-power.c F: drivers/pwm/pwm-apple.c F: drivers/reset/reset-apple-cio.c +F: drivers/rtc/rtc-apple-pmu.c F: drivers/rtc/rtc-macsmc.c F: drivers/soc/apple/* F: drivers/spi/spi-apple.c diff --git a/drivers/nvmem/apple-spmi-nvmem.c b/drivers/nvmem/apple-spmi-nvmem.c index 7acb0c07d6abe..338e2e72f803d 100644 --- a/drivers/nvmem/apple-spmi-nvmem.c +++ b/drivers/nvmem/apple-spmi-nvmem.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -37,6 +38,7 @@ static int apple_spmi_nvmem_write(void *priv, unsigned int offset, void *val, static int apple_spmi_nvmem_probe(struct spmi_device *sdev) { struct regmap *regmap; + struct nvmem_device *provider; struct nvmem_config nvmem_cfg = { .dev = &sdev->dev, .name = "spmi_nvmem", @@ -54,7 +56,13 @@ static int apple_spmi_nvmem_probe(struct spmi_device *sdev) nvmem_cfg.priv = regmap; - return PTR_ERR_OR_ZERO(devm_nvmem_register(&sdev->dev, &nvmem_cfg)); + provider = devm_nvmem_register(&sdev->dev, &nvmem_cfg); + if (IS_ERR(provider)) + return PTR_ERR(provider); + if (!of_device_is_compatible(sdev->dev.of_node, "apple,baku-pmic")) + return 0; + + return devm_of_platform_populate(&sdev->dev); } static const struct of_device_id apple_spmi_nvmem_id_table[] = { diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig index 01def82318731..afff28ce32b94 100644 --- a/drivers/rtc/Kconfig +++ b/drivers/rtc/Kconfig @@ -2093,6 +2093,17 @@ config RTC_DRV_WILCO_EC This can also be built as a module. If so, the module will be named "rtc_wilco_ec". +config RTC_DRV_APPLE_PMU + tristate "Apple J700 Baku PMU RTC alarm" + depends on ARCH_APPLE && NVMEM_APPLE_SPMI && PM_SLEEP + help + Read the Baku PMU clock and use alarms with retained-power + suspend-to-idle. The module requires experimental=1. Alarms are + limited to one hour; deeper suspend states are not supported. + Continue setting the shared clock through the SMC RTC. + + This driver can be built as the module rtc-apple-pmu. + config RTC_DRV_MACSMC tristate "Apple Mac System Management Controller RTC" depends on MFD_MACSMC diff --git a/drivers/rtc/Makefile b/drivers/rtc/Makefile index 0347645b021f8..d998775d52f67 100644 --- a/drivers/rtc/Makefile +++ b/drivers/rtc/Makefile @@ -94,6 +94,7 @@ obj-$(CONFIG_RTC_DRV_M48T35) += rtc-m48t35.o obj-$(CONFIG_RTC_DRV_M48T59) += rtc-m48t59.o obj-$(CONFIG_RTC_DRV_M48T86) += rtc-m48t86.o obj-$(CONFIG_RTC_DRV_MA35D1) += rtc-ma35d1.o +obj-$(CONFIG_RTC_DRV_APPLE_PMU) += rtc-apple-pmu.o obj-$(CONFIG_RTC_DRV_MACSMC) += rtc-macsmc.o obj-$(CONFIG_RTC_DRV_MAX31335) += rtc-max31335.o obj-$(CONFIG_RTC_DRV_MAX6900) += rtc-max6900.o diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c new file mode 100644 index 0000000000000..f398a1a802165 --- /dev/null +++ b/drivers/rtc/rtc-apple-pmu.c @@ -0,0 +1,369 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Apple Baku PMU clock and retained-power alarm. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define BAKU_RTC_CONTROL 0xf800 +#define BAKU_RTC_COUNTER 0xf802 +#define BAKU_RTC_COMPARE 0xf808 +#define BAKU_RTC_EVENT 0xf80c +#define BAKU_RTC_MASK 0xf80e +#define BAKU_RTC_RUN BIT(0) +#define BAKU_RTC_ARM BIT(6) +#define BAKU_RTC_ALARM BIT(0) +#define BAKU_RTC_HZ_SHIFT 16 +#define BAKU_RTC_MAX_DELAY 3600 + +static bool experimental; +module_param(experimental, bool, 0444); +MODULE_PARM_DESC(experimental, "Enable alarms with retained SPMI/AIC power in s2idle"); + +struct baku_rtc { + struct regmap *map; + struct nvmem_cell *offset; + struct rtc_device *clock; + /* Serialize comparator updates and interrupt acknowledgment. */ + struct mutex transaction; + int interrupt; + bool io_failed; +}; + +static int baku_rtc_seconds(struct baku_rtc *baku, u64 *seconds) +{ + u8 count[8] = {}; + int error; + + error = regmap_bulk_read(baku->map, BAKU_RTC_COUNTER, count, 6); + if (error) + return error; + *seconds = get_unaligned_le64(count) >> BAKU_RTC_HZ_SHIFT; + return 0; +} + +static int baku_rtc_offset(struct baku_rtc *baku, s64 *seconds) +{ + size_t bytes; + void *value; + + /* The SMC RTC can update this cell, so do not cache it at probe. */ + value = nvmem_cell_read(baku->offset, &bytes); + if (IS_ERR(value)) + return PTR_ERR(value); + if (bytes != sizeof(__le64)) { + kfree(value); + return -EINVAL; + } + *seconds = get_unaligned_le64(value); + kfree(value); + if (*seconds < 0 || *seconds > RTC_TIMESTAMP_END_9999 - U32_MAX) + return -ERANGE; + return 0; +} + +static int baku_rtc_read_time(struct device *dev, struct rtc_time *time) +{ + struct baku_rtc *baku = dev_get_drvdata(dev); + u64 counter; + s64 epoch; + int error; + + guard(mutex)(&baku->transaction); + error = baku_rtc_offset(baku, &epoch); + if (error) + return error; + error = baku_rtc_seconds(baku, &counter); + if (error) + return error; + rtc_time64_to_tm(epoch + counter, time); + return 0; +} + +/* All comparator/control accesses are serialized with the IRQ thread. */ +static int baku_rtc_stop_alarm(struct baku_rtc *baku) +{ + int error; + + error = regmap_update_bits(baku->map, BAKU_RTC_MASK, + BAKU_RTC_ALARM, BAKU_RTC_ALARM); + if (error) + return error; + error = regmap_update_bits(baku->map, BAKU_RTC_CONTROL, BAKU_RTC_ARM, 0); + if (error) + return error; + return regmap_write(baku->map, BAKU_RTC_EVENT, BAKU_RTC_ALARM); +} + +static int baku_rtc_start_alarm(struct baku_rtc *baku) +{ + int error; + + error = regmap_write(baku->map, BAKU_RTC_EVENT, BAKU_RTC_ALARM); + if (!error) + error = regmap_update_bits(baku->map, BAKU_RTC_MASK, BAKU_RTC_ALARM, 0); + if (!error) + error = regmap_update_bits(baku->map, BAKU_RTC_CONTROL, + BAKU_RTC_RUN | BAKU_RTC_ARM, + BAKU_RTC_RUN | BAKU_RTC_ARM); + if (error) + baku_rtc_stop_alarm(baku); + return error; +} + +static int baku_rtc_validate_alarm(struct baku_rtc *baku, u64 comparator) +{ + u64 counter; + int error; + + error = baku_rtc_seconds(baku, &counter); + if (error) + return error; + if (comparator <= counter || comparator - counter > BAKU_RTC_MAX_DELAY) + return -ETIME; + return 0; +} + +static int baku_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alarm) +{ + struct baku_rtc *baku = dev_get_drvdata(dev); + unsigned int control, event, mask; + __le32 comparator; + s64 epoch; + int error; + + guard(mutex)(&baku->transaction); + error = baku_rtc_offset(baku, &epoch); + if (error) + return error; + error = regmap_bulk_read(baku->map, BAKU_RTC_COMPARE, + &comparator, sizeof(comparator)); + if (!error) + error = regmap_read(baku->map, BAKU_RTC_CONTROL, &control); + if (!error) + error = regmap_read(baku->map, BAKU_RTC_EVENT, &event); + if (!error) + error = regmap_read(baku->map, BAKU_RTC_MASK, &mask); + if (error) + return error; + rtc_time64_to_tm(epoch + le32_to_cpu(comparator), &alarm->time); + alarm->enabled = (control & BAKU_RTC_ARM) && !(mask & BAKU_RTC_ALARM); + alarm->pending = !!(event & BAKU_RTC_ALARM); + return 0; +} + +static int baku_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alarm) +{ + struct baku_rtc *baku = dev_get_drvdata(dev); + __le32 comparator, readback; + s64 epoch, deadline; + int error; + + guard(mutex)(&baku->transaction); + if (baku->io_failed) + return -EIO; + error = baku_rtc_offset(baku, &epoch); + if (error) + return error; + deadline = rtc_tm_to_time64(&alarm->time) - epoch; + if (deadline < 0 || deadline > U32_MAX) + return -ERANGE; + error = baku_rtc_validate_alarm(baku, deadline); + if (error) + return error; + error = baku_rtc_stop_alarm(baku); + if (error) + return error; + comparator = cpu_to_le32(deadline); + error = regmap_bulk_write(baku->map, BAKU_RTC_COMPARE, + &comparator, sizeof(comparator)); + if (error) + return error; + error = regmap_bulk_read(baku->map, BAKU_RTC_COMPARE, + &readback, sizeof(readback)); + if (error) + return error; + if (readback != comparator) + return -EIO; + if (alarm->enabled) + return baku_rtc_start_alarm(baku); + return 0; +} + +static int baku_rtc_alarm_enable(struct device *dev, unsigned int enable) +{ + struct baku_rtc *baku = dev_get_drvdata(dev); + __le32 comparator; + int error; + + guard(mutex)(&baku->transaction); + if (!enable) + return baku_rtc_stop_alarm(baku); + if (baku->io_failed) + return -EIO; + error = regmap_bulk_read(baku->map, BAKU_RTC_COMPARE, + &comparator, sizeof(comparator)); + if (error) + return error; + error = baku_rtc_validate_alarm(baku, le32_to_cpu(comparator)); + return error ?: baku_rtc_start_alarm(baku); +} + +static irqreturn_t baku_rtc_interrupt(int irq, void *context) +{ + struct device *dev = context; + struct baku_rtc *baku = dev_get_drvdata(dev); + unsigned int event; + int error; + + mutex_lock(&baku->transaction); + error = regmap_read(baku->map, BAKU_RTC_EVENT, &event); + if (!error && (event & BAKU_RTC_ALARM)) + error = baku_rtc_stop_alarm(baku); + if (error) { + baku->io_failed = true; + disable_irq_nosync(irq); + } + mutex_unlock(&baku->transaction); + if (error) { + dev_err(dev, "cannot acknowledge RTC interrupt: %d\n", error); + pm_wakeup_dev_event(dev, 0, true); + return IRQ_HANDLED; + } + if (!(event & BAKU_RTC_ALARM)) + return IRQ_NONE; + if (device_may_wakeup(dev)) + pm_wakeup_dev_event(dev, 0, true); + rtc_update_irq(baku->clock, 1, RTC_IRQF | RTC_AF); + return IRQ_HANDLED; +} + +static const struct rtc_class_ops baku_rtc_ops = { + .read_time = baku_rtc_read_time, + .read_alarm = baku_rtc_read_alarm, + .set_alarm = baku_rtc_set_alarm, + .alarm_irq_enable = baku_rtc_alarm_enable, +}; + +static int baku_rtc_prepare(struct device *dev) +{ + struct baku_rtc *baku = dev_get_drvdata(dev); + + if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE || baku->io_failed) + return -EBUSY; + return 0; +} + +static const struct dev_pm_ops baku_rtc_pm = { + .prepare = baku_rtc_prepare, +}; + +static void baku_rtc_release(void *context) +{ + struct device *dev = context; + struct baku_rtc *baku = dev_get_drvdata(dev); + + disable_irq(baku->interrupt); + mutex_lock(&baku->transaction); + if (baku_rtc_stop_alarm(baku)) + dev_err(dev, "cannot disable RTC alarm on removal\n"); + mutex_unlock(&baku->transaction); + device_init_wakeup(dev, false); +} + +static int baku_rtc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct baku_rtc *baku; + unsigned int control; + int error; + + /* Retention has only been measured on J700 with the explicit opt-in. */ + if (!experimental) + return -ENODEV; + baku = devm_kzalloc(dev, sizeof(*baku), GFP_KERNEL); + if (!baku) + return -ENOMEM; + baku->map = dev_get_regmap(dev->parent, NULL); + if (!baku->map) + return dev_err_probe(dev, -EPROBE_DEFER, "PMIC regmap unavailable\n"); + baku->offset = devm_nvmem_cell_get(dev, "rtc_offset"); + if (IS_ERR(baku->offset)) + return dev_err_probe(dev, PTR_ERR(baku->offset), "RTC offset unavailable\n"); + baku->interrupt = platform_get_irq(pdev, 0); + if (baku->interrupt < 0) + return baku->interrupt; + mutex_init(&baku->transaction); + platform_set_drvdata(pdev, baku); + error = regmap_read(baku->map, BAKU_RTC_CONTROL, &control); + if (error) + return error; + /* Respect an alarm left armed by firmware. */ + if (control & BAKU_RTC_ARM) + return -EBUSY; + baku->clock = devm_rtc_allocate_device(dev); + if (IS_ERR(baku->clock)) + return PTR_ERR(baku->clock); + baku->clock->ops = &baku_rtc_ops; + baku->clock->range_min = 0; + baku->clock->range_max = RTC_TIMESTAMP_END_9999; + clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, baku->clock->features); + error = device_init_wakeup(dev, true); + if (error) + return error; + /* + * This interrupt remains live during retained-power s2idle. The + * thread uses the parent PMIC's SPMI regmap to acknowledge the alarm. + * Off-domain wake routing is not established by this driver. + */ + error = devm_request_threaded_irq(dev, baku->interrupt, NULL, + baku_rtc_interrupt, + IRQF_NO_AUTOEN | IRQF_NO_SUSPEND | IRQF_ONESHOT, + dev_name(dev), dev); + if (error) + goto disable_wakeup; + error = baku_rtc_stop_alarm(baku); + if (error) + goto disable_wakeup; + error = devm_rtc_register_device(baku->clock); + if (error) + goto disable_wakeup; + error = devm_add_action_or_reset(dev, baku_rtc_release, dev); + if (error) + return error; + enable_irq(baku->interrupt); + return 0; + +disable_wakeup: + device_init_wakeup(dev, false); + return error; +} + +static const struct of_device_id baku_rtc_match[] = { + { .compatible = "apple,j700-pmu-rtc" }, + { } +}; +MODULE_DEVICE_TABLE(of, baku_rtc_match); + +static struct platform_driver baku_rtc_driver = { + .probe = baku_rtc_probe, + .driver = { + .name = "apple-pmu-rtc", + .of_match_table = baku_rtc_match, + .pm = &baku_rtc_pm, + }, +}; +module_platform_driver(baku_rtc_driver); + +MODULE_AUTHOR("Eryk Wieliczko "); +MODULE_DESCRIPTION("Apple Baku PMU RTC alarm with retained-power s2idle"); +MODULE_LICENSE("GPL"); From 1bef966af1d8d2d13c5cc957e31cc435bfba48fb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:50:15 +1100 Subject: [PATCH 0401/2680] rtc: apple-pmu: leave the counter control state to firmware The alarm arm mask is described by the Linux-captured ADT. Setting an additional RUN bit is unnecessary for programming the comparator and changes the shared time counter's state. Update only the alarm arm bit, leaving the clock path and unrelated control bits unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/rtc/rtc-apple-pmu.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c index f398a1a802165..7853b72a1b552 100644 --- a/drivers/rtc/rtc-apple-pmu.c +++ b/drivers/rtc/rtc-apple-pmu.c @@ -19,7 +19,6 @@ #define BAKU_RTC_COMPARE 0xf808 #define BAKU_RTC_EVENT 0xf80c #define BAKU_RTC_MASK 0xf80e -#define BAKU_RTC_RUN BIT(0) #define BAKU_RTC_ARM BIT(6) #define BAKU_RTC_ALARM BIT(0) #define BAKU_RTC_HZ_SHIFT 16 @@ -113,8 +112,7 @@ static int baku_rtc_start_alarm(struct baku_rtc *baku) error = regmap_update_bits(baku->map, BAKU_RTC_MASK, BAKU_RTC_ALARM, 0); if (!error) error = regmap_update_bits(baku->map, BAKU_RTC_CONTROL, - BAKU_RTC_RUN | BAKU_RTC_ARM, - BAKU_RTC_RUN | BAKU_RTC_ARM); + BAKU_RTC_ARM, BAKU_RTC_ARM); if (error) baku_rtc_stop_alarm(baku); return error; From 04d50784288ffb6bb7be71ae43e06a9a5b627653 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 21:34:26 +1100 Subject: [PATCH 0402/2680] rtc: apple-pmu: distinguish alarm range errors from expiration The RTC core treats ETIME as an expired timer, retaining it on enqueue and retrying from irqwork without using its bounded error retry count. Returning it for a future alarm beyond the one-hour limit therefore keeps the worker under ops_lock until the alarm enters that horizon. Reserve ETIME for deadlines at or before the current counter. Return ERANGE for unsupported future deadlines so enqueue removes the timer and returns an error, and worker reprogramming uses its bounded error path. Advertise the same one-hour horizon through alarm_offset_max. Fixes: ff07618c82bb ("rtc: add opt-in J700 Baku PMU alarms with retained s2idle interrupts") (cherry picked from commit d3bf9897f2d5cbb6a9169d776b7d946fafb927c1) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/rtc/rtc-apple-pmu.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c index 7853b72a1b552..a684233205803 100644 --- a/drivers/rtc/rtc-apple-pmu.c +++ b/drivers/rtc/rtc-apple-pmu.c @@ -126,8 +126,10 @@ static int baku_rtc_validate_alarm(struct baku_rtc *baku, u64 comparator) error = baku_rtc_seconds(baku, &counter); if (error) return error; - if (comparator <= counter || comparator - counter > BAKU_RTC_MAX_DELAY) + if (comparator <= counter) return -ETIME; + if (comparator - counter > BAKU_RTC_MAX_DELAY) + return -ERANGE; return 0; } @@ -314,6 +316,7 @@ static int baku_rtc_probe(struct platform_device *pdev) baku->clock->ops = &baku_rtc_ops; baku->clock->range_min = 0; baku->clock->range_max = RTC_TIMESTAMP_END_9999; + baku->clock->alarm_offset_max = BAKU_RTC_MAX_DELAY; clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, baku->clock->features); error = device_init_wakeup(dev, true); if (error) From 844977b4bb78d490ca047d86ea6880062a78d496 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 01:29:51 +1100 Subject: [PATCH 0403/2680] rtc: apple-pmu: take over an alarm that firmware left armed The probe refused to bind with -EBUSY whenever it found the alarm armed, so that it would not overwrite a wake scheduled by firmware. On the J700 the alarm is armed at every boot, with the comparator at about 2^32 - 1 day in wall-clock seconds, a time in early 2106. Nothing is due then, but the check means the driver never binds. Keep the alarm only when it is unmasked and due within the range the driver supports, so that the RTC core picks it up at registration. Otherwise disarm it, as probe already did for an alarm it found disarmed. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/rtc/rtc-apple-pmu.c | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c index a684233205803..8c7e4156b015f 100644 --- a/drivers/rtc/rtc-apple-pmu.c +++ b/drivers/rtc/rtc-apple-pmu.c @@ -247,6 +247,32 @@ static irqreturn_t baku_rtc_interrupt(int irq, void *context) return IRQ_HANDLED; } +/* + * Firmware leaves the alarm armed even when it has no wake scheduled, with + * the comparator set close to the end of 32-bit time. Keep an alarm that is + * still pending within the supported range, so that the RTC core picks it + * up at registration, and disarm anything else. + */ +static int baku_rtc_init_alarm(struct baku_rtc *baku) +{ + unsigned int control, mask; + __le32 comparator; + int error; + + error = regmap_read(baku->map, BAKU_RTC_CONTROL, &control); + if (!error) + error = regmap_read(baku->map, BAKU_RTC_MASK, &mask); + if (!error) + error = regmap_bulk_read(baku->map, BAKU_RTC_COMPARE, + &comparator, sizeof(comparator)); + if (error) + return error; + if ((control & BAKU_RTC_ARM) && !(mask & BAKU_RTC_ALARM) && + !baku_rtc_validate_alarm(baku, le32_to_cpu(comparator))) + return 0; + return baku_rtc_stop_alarm(baku); +} + static const struct rtc_class_ops baku_rtc_ops = { .read_time = baku_rtc_read_time, .read_alarm = baku_rtc_read_alarm, @@ -284,7 +310,6 @@ static int baku_rtc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct baku_rtc *baku; - unsigned int control; int error; /* Retention has only been measured on J700 with the explicit opt-in. */ @@ -304,12 +329,6 @@ static int baku_rtc_probe(struct platform_device *pdev) return baku->interrupt; mutex_init(&baku->transaction); platform_set_drvdata(pdev, baku); - error = regmap_read(baku->map, BAKU_RTC_CONTROL, &control); - if (error) - return error; - /* Respect an alarm left armed by firmware. */ - if (control & BAKU_RTC_ARM) - return -EBUSY; baku->clock = devm_rtc_allocate_device(dev); if (IS_ERR(baku->clock)) return PTR_ERR(baku->clock); @@ -332,7 +351,7 @@ static int baku_rtc_probe(struct platform_device *pdev) dev_name(dev), dev); if (error) goto disable_wakeup; - error = baku_rtc_stop_alarm(baku); + error = baku_rtc_init_alarm(baku); if (error) goto disable_wakeup; error = devm_rtc_register_device(baku->clock); From 0672e81d5975f607f58da4ce5fa60ee6dd5f3465 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 01:29:52 +1100 Subject: [PATCH 0404/2680] rtc: apple-pmu: enable the alarm without a module parameter The driver only probed with experimental=1, because alarm delivery during suspend-to-idle had not been measured. Nothing on that path is switched off by s2idle on the J700: the SPMI controller and the AIC sit in always-on power domains, the SPMI controller's chained interrupt is never disabled by the suspend core, and the alarm interrupt is requested with IRQF_NO_SUSPEND and reports a hard wakeup. Measured on the J700: alarms set 30 and 90 seconds ahead ended suspend-to-idle 29.8 to 30.5 and 90.3 seconds after it began. Drop the parameter so that the PMU RTC binds by default. It is the only wake source on the J700 that does not need a person at the machine. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/rtc/Kconfig | 8 ++++---- drivers/rtc/rtc-apple-pmu.c | 7 ------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig index afff28ce32b94..dedf3bceee1ce 100644 --- a/drivers/rtc/Kconfig +++ b/drivers/rtc/Kconfig @@ -2097,10 +2097,10 @@ config RTC_DRV_APPLE_PMU tristate "Apple J700 Baku PMU RTC alarm" depends on ARCH_APPLE && NVMEM_APPLE_SPMI && PM_SLEEP help - Read the Baku PMU clock and use alarms with retained-power - suspend-to-idle. The module requires experimental=1. Alarms are - limited to one hour; deeper suspend states are not supported. - Continue setting the shared clock through the SMC RTC. + Read the Baku PMU clock and use its alarm to wake the system + from suspend-to-idle. Alarms are limited to one hour; deeper + suspend states are not supported. Continue setting the shared + clock through the SMC RTC. This driver can be built as the module rtc-apple-pmu. diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c index 8c7e4156b015f..2e3161a914c8a 100644 --- a/drivers/rtc/rtc-apple-pmu.c +++ b/drivers/rtc/rtc-apple-pmu.c @@ -24,10 +24,6 @@ #define BAKU_RTC_HZ_SHIFT 16 #define BAKU_RTC_MAX_DELAY 3600 -static bool experimental; -module_param(experimental, bool, 0444); -MODULE_PARM_DESC(experimental, "Enable alarms with retained SPMI/AIC power in s2idle"); - struct baku_rtc { struct regmap *map; struct nvmem_cell *offset; @@ -312,9 +308,6 @@ static int baku_rtc_probe(struct platform_device *pdev) struct baku_rtc *baku; int error; - /* Retention has only been measured on J700 with the explicit opt-in. */ - if (!experimental) - return -ENODEV; baku = devm_kzalloc(dev, sizeof(*baku), GFP_KERNEL); if (!baku) return -ENOMEM; From bf0d96f91362b8c66fdd19a33d56146bee01639a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:16:31 +1100 Subject: [PATCH 0405/2680] hwmon: macsmc: drop the check for the undefined apple,cpu-thermal-policy Temperature discovery refuses to run when the hwmon node has an apple,cpu-thermal-policy property. No binding defines that property and no device tree sets it, so the check can never fire. The real guard is the one next to it: discovered channels are not mixed with channels the device tree describes, which thermal zones refer to by index. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hwmon/macsmc-hwmon.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index 36598f06ecaf2..b0172bbb76e98 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -514,9 +514,8 @@ static int macsmc_hwmon_discover_temperatures(struct macsmc_hwmon *hwmon) if (!of_property_read_bool(hwmon->dev->of_node, "apple,discover-temperatures")) return 0; - /* Dynamic channel numbers must not replace a DT-described thermal policy. */ - if (hwmon->temp.count || - of_property_read_bool(hwmon->dev->of_node, "apple,cpu-thermal-policy")) + /* Dynamic channel numbers must not replace DT-described channels. */ + if (hwmon->temp.count) return -EINVAL; /* Bound both allocation and mailbox transactions for invalid firmware data. */ From 6086ef38d9edc29bb2f49a6fd239788c9e9b9c1b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:17:04 +1100 Subject: [PATCH 0406/2680] cpufreq: apple-soc: tidy includes and the ->exit() callback The headers the fast die-temperature limit needs were added as a second, separately sorted block. Merge them into the existing alphabetical list. ->exit() is a wrapper that stops the limit and then calls apple_soc_cpufreq_exit(), which needs a forward declaration. Stop the limit at the start of apple_soc_cpufreq_exit() instead and use that as the callback. The order of teardown is unchanged. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/cpufreq/apple-soc-cpufreq.c | 36 ++++++++++++----------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 37b44cccb9b87..e0e84b85db244 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -7,32 +7,31 @@ * Based on scpi-cpufreq.c */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include #include #include #include +#include #include #include #include +#include #include #include +#include +#include +#include #include #include #include #include +#include #include +#include +#include +#include +#include +#include #define APPLE_DVFS_CMD 0x20 #define APPLE_DVFS_CMD_BUSY BIT(31) @@ -208,7 +207,6 @@ struct apple_cpu_priv { }; static struct cpufreq_driver apple_soc_cpufreq_driver; -static void apple_soc_cpufreq_exit_with_limits(struct cpufreq_policy *policy); static const struct apple_soc_cpufreq_info soc_s5l8960x_info = { .has_ps2 = false, @@ -929,6 +927,9 @@ static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy) { struct apple_cpu_priv *priv = policy->driver_data; + /* Drain firmware-limit updates before releasing the OPP tables. */ + apple_fast_die_exit(priv); + dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); /* * The core clears the last CPU from policy->cpus before calling @@ -946,7 +947,7 @@ static struct cpufreq_driver apple_soc_cpufreq_driver = { .verify = cpufreq_generic_frequency_table_verify, .get = apple_soc_cpufreq_get_rate, .init = apple_soc_cpufreq_init, - .exit = apple_soc_cpufreq_exit_with_limits, + .exit = apple_soc_cpufreq_exit, .target_index = apple_soc_cpufreq_set_target, .fast_switch = apple_soc_cpufreq_fast_switch, .register_em = cpufreq_register_em_with_opp, @@ -954,13 +955,6 @@ static struct cpufreq_driver apple_soc_cpufreq_driver = { .suspend = cpufreq_generic_suspend, }; -/* Drain firmware-limit updates before releasing the policy's OPP table. */ -static void apple_soc_cpufreq_exit_with_limits(struct cpufreq_policy *policy) -{ - apple_fast_die_exit(policy->driver_data); - apple_soc_cpufreq_exit(policy); -} - static int __init apple_soc_cpufreq_module_init(void) { struct device_node *np; From fd8ef4ff005685b95acca98be2a73c2b6bca4938 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:18:45 +1100 Subject: [PATCH 0407/2680] cpufreq: apple-soc: describe the unmanaged P-state ceiling as data SoCs whose higher P-states need a thermal policy that is not qualified yet refuse an OPP table that goes above max_unmanaged_pstate. T8140 has one (power_allocator zones and the PMP fast die-temperature limit), so init skips the check for it by comparing the info pointer with &soc_t8140_info, while soc_t8140_info still carries a ceiling that is never used. Drop the unused T8140 ceiling and run the check whenever a SoC has one, so the per-SoC table says which SoCs are capped. Document both fields. The check runs for exactly the same SoCs as before (T8152 only). No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/cpufreq/apple-soc-cpufreq.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index e0e84b85db244..5f16b80b746b9 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -186,8 +186,10 @@ static int apple_fast_die_hp_state; struct apple_soc_cpufreq_info { bool has_ps2; bool verify_transition; + /* Fail closed when the DT describes no fast die-temperature limit. */ bool needs_thermal_policy; u32 transition_timeout_us; + /* Highest P-state allowed without a thermal policy; 0 if not capped. */ u32 max_unmanaged_pstate; u64 min_pstate; u64 max_pstate; @@ -244,7 +246,6 @@ static const struct apple_soc_cpufreq_info soc_t8112_info = { static const struct apple_soc_cpufreq_info soc_t8140_info = { .verify_transition = true, .needs_thermal_policy = true, - .max_unmanaged_pstate = 2, .min_pstate = 1, .max_pstate = 31, .ps1_mask = APPLE_DVFS_CMD_PS1, @@ -869,8 +870,8 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) priv->expected_pstate = state; } - /* T8140 has IPA and the PMP limit; other SoCs keep their safe ceiling. */ - if (info->needs_thermal_policy && info != &soc_t8140_info) { + /* SoCs without a qualified thermal policy keep their safe ceiling. */ + if (info->max_unmanaged_pstate) { cpufreq_for_each_valid_entry(p, freq_table) { if (p->driver_data <= info->max_unmanaged_pstate) continue; From c4d51dda48f03a8c7509955b7d53d70a8c4db308 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:27:48 +1100 Subject: [PATCH 0408/2680] rtc: macsmc: correct the comment on the whole-second offset With apple,rtc-offset-seconds the offset is stored in whole seconds. set_time() subtracts the counter's whole seconds from the requested time, so the fraction of a second the counter had reached is dropped and the next second starts early: the clock runs up to a second ahead. The comment says the fraction "carries into the new time", which reads as the opposite. Say what happens. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/rtc/rtc-macsmc.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/rtc/rtc-macsmc.c b/drivers/rtc/rtc-macsmc.c index 25d5af5cfbbcd..1c1c529df8296 100644 --- a/drivers/rtc/rtc-macsmc.c +++ b/drivers/rtc/rtc-macsmc.c @@ -113,7 +113,10 @@ static int macsmc_rtc_set_time(struct device *dev, struct rtc_time *tm) if (rtc->offset_in_seconds) { __le64 off_secs; - /* The counter's fraction of a second carries into the new time */ + /* + * The offset holds whole seconds, so the counter's fraction of a + * second is lost: the clock may run up to a second ahead. + */ off_secs = cpu_to_le64(rtc_tm_to_time64(tm) - (ctr >> RTC_SEC_SHIFT)); ret = nvmem_cell_write(rtc->rtc_offset, &off_secs, sizeof(off_secs)); return ret < 0 ? ret : 0; From 26d7f8a9a45b790735ab56bebdb01b32d7a5283e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:05:55 +1100 Subject: [PATCH 0409/2680] hwmon: macsmc: account for temperature channels in one helper Move the code that turns a temperature node into a channel into macsmc_hwmon_add_temp_channel(), so that the next patch can change how a node that fails to set up is accounted for in one place, and a unit test can exercise it without an SMC. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hwmon/macsmc-hwmon.c | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index b0172bbb76e98..bfdd357849c61 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -501,6 +501,21 @@ static int macsmc_hwmon_create_sensor(struct device *dev, struct apple_smc *smc, return 0; } +/* Account for the temperature sensor just set up at the end of @temp. */ +static void macsmc_hwmon_add_temp_channel(struct macsmc_hwmon_sensors *temp, + int create_ret) +{ + struct macsmc_hwmon_sensor *sensor = &temp->sensors[temp->count]; + + if (create_ret) + return; + + sensor->attrs = HWMON_T_INPUT; + if (*sensor->label) + sensor->attrs |= HWMON_T_LABEL; + temp->count++; +} + /* Discover reporting-only temperatures without a board-specific key table. */ static int macsmc_hwmon_discover_temperatures(struct macsmc_hwmon *hwmon) { @@ -637,6 +652,7 @@ static int macsmc_hwmon_populate_sensors(struct macsmc_hwmon *hwmon, struct device_node *key_node __maybe_unused; struct macsmc_hwmon_sensor *sensor; u32 n_current = 0, n_fan = 0, n_power = 0, n_temperature = 0, n_voltage = 0; + int ret; for_each_child_of_node_with_prefix(hwmon_node, key_node, "current-") { n_current++; @@ -713,14 +729,9 @@ static int macsmc_hwmon_populate_sensors(struct macsmc_hwmon *hwmon, for_each_child_of_node_with_prefix(hwmon_node, key_node, "temperature-") { sensor = &hwmon->temp.sensors[hwmon->temp.count]; - if (!macsmc_hwmon_create_sensor(hwmon->dev, hwmon->smc, key_node, sensor)) { - sensor->attrs = HWMON_T_INPUT; - - if (*sensor->label) - sensor->attrs |= HWMON_T_LABEL; - - hwmon->temp.count++; - } + ret = macsmc_hwmon_create_sensor(hwmon->dev, hwmon->smc, + key_node, sensor); + macsmc_hwmon_add_temp_channel(&hwmon->temp, ret); } } From 8bc5b3545cf344300d48d2970ee581e1e12a4d7a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:09:26 +1100 Subject: [PATCH 0410/2680] hwmon: macsmc: keep the channel of a temperature sensor that fails to set up Since the SMC temperatures became thermal sensors, thermal zones refer to them by channel index, and the binding defines that index as the position of the node among the temperature-* nodes. The driver only counts nodes whose key it could look up, so when one fails, for instance because the SMC firmware does not have the key, every later channel moves down by one. Every zone that refers to a later channel then silently reads a different sensor. On T8140 the six CPU die zones use the first six channels, so a missing die key would move every later CPU zone onto its neighbour's sensor and the last one onto a sensor outside the CPU clusters. On a fanless machine that can mean running hot without throttling. Give every temperature node its channel. A sensor that fails to set up keeps its slot with no key; is_visible() hides it, so hwmon creates no attributes and no thermal sensor for it, and only the zones that use that channel fail to register. The config entry stays non-zero, since a zero entry would end the channel list. Say which node failed. The KUnit test sets up three channels with the middle one failing and checks that the third keeps its index and key and that the middle one is hidden. It fails before this change (two channels, the third key in slot 1) and passes after it. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 2 +- drivers/hwmon/Kconfig | 10 ++++ drivers/hwmon/macsmc-hwmon-test.c | 87 +++++++++++++++++++++++++++++++ drivers/hwmon/macsmc-hwmon.c | 28 ++++++++-- 4 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 drivers/hwmon/macsmc-hwmon-test.c diff --git a/MAINTAINERS b/MAINTAINERS index e3e20f7aca64a..0a4251663b43a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2630,7 +2630,7 @@ F: drivers/clk/clk-apple-nco.c F: drivers/cpufreq/apple-soc-cpufreq.c F: drivers/dma/apple-admac.c F: drivers/gpio/gpio-macsmc.c -F: drivers/hwmon/macsmc-hwmon.c +F: drivers/hwmon/macsmc-hwmon*.c F: drivers/pmdomain/apple/ F: drivers/i2c/busses/i2c-pasemi-core.c F: drivers/i2c/busses/i2c-pasemi-platform.c diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index 2bfbcc033d599..70c71a9148d32 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -1208,6 +1208,16 @@ config SENSORS_MACSMC_HWMON on Apple Silicon devices. Say Y here if you have an Apple Silicon device. +config SENSORS_MACSMC_HWMON_KUNIT_TEST + bool "KUnit tests for the Apple SMC hwmon driver" if !KUNIT_ALL_TESTS + depends on SENSORS_MACSMC_HWMON && KUNIT=y + default KUNIT_ALL_TESTS + help + Unit tests for the channel numbering of the Apple SMC hwmon driver. + Thermal zones refer to temperature channels by index, so a sensor + that cannot be set up must keep its channel; the tests check that. + No SMC is needed. Only useful for kernel developers. + This driver can also be built as a module. If so, the module will be called macsmc-hwmon. diff --git a/drivers/hwmon/macsmc-hwmon-test.c b/drivers/hwmon/macsmc-hwmon-test.c new file mode 100644 index 0000000000000..e7239986ce3c9 --- /dev/null +++ b/drivers/hwmon/macsmc-hwmon-test.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * KUnit tests for the Apple SMC hwmon driver: channel numbering of the + * temperature sensors, which thermal zones depend on. + * + * Included by macsmc-hwmon.c. + */ + +#include + +/* What macsmc_hwmon_create_sensor() leaves behind for one node. */ +static void macsmc_hwmon_test_add(struct macsmc_hwmon_sensors *temp, + const char *key, int create_ret) +{ + struct macsmc_hwmon_sensor *sensor = &temp->sensors[temp->count]; + + sensor->macsmc_key = _SMC_KEY(key); + strscpy_pad(sensor->label, key, sizeof(sensor->label)); + macsmc_hwmon_add_temp_channel(temp, create_ret); +} + +/* A node that fails to set up keeps its channel; later channels do not move. */ +static void macsmc_hwmon_test_failed_temp_keeps_slot(struct kunit *test) +{ + struct macsmc_hwmon_sensor sensors[3] = {}; + struct macsmc_hwmon hwmon = { + .temp = { .sensors = sensors }, + }; + u32 configs[ARRAY_SIZE(sensors) + 1] = {}; + + macsmc_hwmon_test_add(&hwmon.temp, "Te05", 0); + macsmc_hwmon_test_add(&hwmon.temp, "Te0S", -EINVAL); + macsmc_hwmon_test_add(&hwmon.temp, "Tp01", 0); + + KUNIT_ASSERT_EQ(test, hwmon.temp.count, 3); + KUNIT_EXPECT_EQ(test, sensors[0].macsmc_key, _SMC_KEY("Te05")); + KUNIT_EXPECT_EQ(test, sensors[2].macsmc_key, _SMC_KEY("Tp01")); + + KUNIT_EXPECT_EQ(test, macsmc_hwmon_is_visible(&hwmon, hwmon_temp, + hwmon_temp_input, 0), 0444); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_is_visible(&hwmon, hwmon_temp, + hwmon_temp_input, 1), 0); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_is_visible(&hwmon, hwmon_temp, + hwmon_temp_label, 1), 0); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_is_visible(&hwmon, hwmon_temp, + hwmon_temp_input, 2), 0444); + + /* A zero entry would end the channel list at the failed sensor. */ + macsmc_hwmon_populate_configs(configs, &hwmon.temp); + KUNIT_EXPECT_NE(test, configs[0], 0); + KUNIT_EXPECT_NE(test, configs[1], 0); + KUNIT_EXPECT_NE(test, configs[2], 0); + KUNIT_EXPECT_EQ(test, configs[3], 0); +} + +static void macsmc_hwmon_test_temp_label(struct kunit *test) +{ + struct macsmc_hwmon_sensor sensors[2] = {}; + struct macsmc_hwmon hwmon = { + .temp = { .sensors = sensors }, + }; + + macsmc_hwmon_test_add(&hwmon.temp, "Tp05", 0); + sensors[1].macsmc_key = _SMC_KEY("Tp09"); + macsmc_hwmon_add_temp_channel(&hwmon.temp, 0); + + KUNIT_ASSERT_EQ(test, hwmon.temp.count, 2); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_is_visible(&hwmon, hwmon_temp, + hwmon_temp_label, 0), 0444); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_is_visible(&hwmon, hwmon_temp, + hwmon_temp_label, 1), 0); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_is_visible(&hwmon, hwmon_temp, + hwmon_temp_input, 1), 0444); +} + +static struct kunit_case macsmc_hwmon_test_cases[] = { + KUNIT_CASE(macsmc_hwmon_test_failed_temp_keeps_slot), + KUNIT_CASE(macsmc_hwmon_test_temp_label), + {} +}; + +static struct kunit_suite macsmc_hwmon_test_suite = { + .name = "macsmc-hwmon", + .test_cases = macsmc_hwmon_test_cases, +}; + +kunit_test_suite(macsmc_hwmon_test_suite); diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index bfdd357849c61..40b3b0b5fb41e 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -436,6 +436,10 @@ static umode_t macsmc_hwmon_is_visible(const void *data, return 0; } + /* A sensor without a key could not be set up; see macsmc_hwmon_add_temp_channel(). */ + if (!sensor->macsmc_key) + return 0; + /* Sensors only register ro attributes */ if (sensor->attrs & BIT(attr)) return 0444; @@ -501,19 +505,28 @@ static int macsmc_hwmon_create_sensor(struct device *dev, struct apple_smc *smc, return 0; } -/* Account for the temperature sensor just set up at the end of @temp. */ +/* + * Account for the temperature sensor just set up at the end of @temp. + * + * Thermal zones refer to a temperature channel by the index of its node + * among the temperature-* nodes, so every node takes a channel. A sensor + * that could not be set up keeps its channel with no key, which hides it: + * hwmon creates no attributes and no thermal sensor for it, and only the + * zones that use it fail to register, instead of every later channel + * moving onto the wrong sensor. + */ static void macsmc_hwmon_add_temp_channel(struct macsmc_hwmon_sensors *temp, int create_ret) { - struct macsmc_hwmon_sensor *sensor = &temp->sensors[temp->count]; + struct macsmc_hwmon_sensor *sensor = &temp->sensors[temp->count++]; if (create_ret) - return; + memset(sensor, 0, sizeof(*sensor)); + /* A zero entry would end the list of channels here. */ sensor->attrs = HWMON_T_INPUT; if (*sensor->label) sensor->attrs |= HWMON_T_LABEL; - temp->count++; } /* Discover reporting-only temperatures without a board-specific key table. */ @@ -731,6 +744,9 @@ static int macsmc_hwmon_populate_sensors(struct macsmc_hwmon *hwmon, sensor = &hwmon->temp.sensors[hwmon->temp.count]; ret = macsmc_hwmon_create_sensor(hwmon->dev, hwmon->smc, key_node, sensor); + if (ret) + dev_warn(hwmon->dev, "%pOFn unavailable (%d), temp%u hidden\n", + key_node, ret, hwmon->temp.count + 1); macsmc_hwmon_add_temp_channel(&hwmon->temp, ret); } } @@ -932,6 +948,10 @@ static struct platform_driver macsmc_hwmon_driver = { }; module_platform_driver(macsmc_hwmon_driver); +#if IS_ENABLED(CONFIG_SENSORS_MACSMC_HWMON_KUNIT_TEST) +#include "macsmc-hwmon-test.c" +#endif + MODULE_DESCRIPTION("Apple Silicon SMC hwmon driver"); MODULE_AUTHOR("James Calligeros "); MODULE_LICENSE("Dual MIT/GPL"); From 786a0f32e520b29582db8884d4b1e48c486e5df3 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 18:48:20 -0400 Subject: [PATCH 0411/2680] dt-bindings: apple: describe J813 monitoring and input peripherals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe the T8142 mailbox, SMC, MTP, DART and PWM resources. Add explicit contracts for runtime temperature discovery and firmware-owned trackpad reset without embedding sensor tables or calibration. [ryan: adapted to 7.2.10 — split by topic: this is the apple,smc-hwmon.yaml and apple,smc.yaml part; the rest is in aurora/bits/010-soc (fcf4f03692ca), 020-dart and 090-input. The temperature-discovery property now sits beside #thermal-sensor-cells instead of the since-removed apple,cpu-thermal-policy property.] Signed-off-by: Acelogic --- .../bindings/hwmon/apple,smc-hwmon.yaml | 20 +++++++++++++++++++ .../devicetree/bindings/mfd/apple,smc.yaml | 1 + 2 files changed, 21 insertions(+) diff --git a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml index 098ad1cdbc484..3bb7931d648bd 100644 --- a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml +++ b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml @@ -40,6 +40,16 @@ properties: compatible: const: apple,smc-hwmon + apple,discover-temperatures: + type: boolean + description: + Enumerate readable, non-function temperature keys from the running SMC + instead of providing a board-specific temperature table. Only keys + beginning with T and reporting a four-byte floating-point value are + exposed. Labels are the key identifiers, without inferred locations. + Channels are reporting-only; their indices depend on the firmware and + must not be used by a static thermal policy. + "#thermal-sensor-cells": const: 1 description: @@ -88,4 +98,14 @@ patternProperties: $ref: "#/$defs/sensor" unevaluatedProperties: false +allOf: + - if: + required: + - apple,discover-temperatures + then: + properties: + apple,cpu-thermal-policy: false + patternProperties: + "^temperature-": false + additionalProperties: false diff --git a/Documentation/devicetree/bindings/mfd/apple,smc.yaml b/Documentation/devicetree/bindings/mfd/apple,smc.yaml index ed6b02086c33c..31d1940f9d3dc 100644 --- a/Documentation/devicetree/bindings/mfd/apple,smc.yaml +++ b/Documentation/devicetree/bindings/mfd/apple,smc.yaml @@ -21,6 +21,7 @@ properties: - apple,t6020-smc - apple,t8122-smc - apple,t8140-smc + - apple,t8142-smc - const: apple,t8103-smc - items: - enum: From 238941d31eb9a525da58e7c145277616a11c0d08 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:16:31 +1100 Subject: [PATCH 0412/2680] dt-bindings: hwmon: apple,smc-hwmon: drop apple,cpu-thermal-policy The conditional for apple,discover-temperatures forbids an apple,cpu-thermal-policy property, but the binding never defines that property and additionalProperties is false, so it is rejected anyway. No device tree uses it. Remove the clause. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml index 3bb7931d648bd..df155f9c34a65 100644 --- a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml +++ b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml @@ -103,8 +103,6 @@ allOf: required: - apple,discover-temperatures then: - properties: - apple,cpu-thermal-policy: false patternProperties: "^temperature-": false From de0f63dd38ac858f339288a22277ece4c42a290f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 23 Sep 2026 19:38:22 +1000 Subject: [PATCH 0413/2680] ASoC: cs42l42: handle jack interrupt I/O and resume errors A failed status or mask read leaves part of the interrupt snapshot uninitialized, and the handler can then interpret that data as headset or button events. It also carries on with register access after a failed runtime resume. Check the runtime-PM acquisition and each snapshot read before processing any events, returning IRQ_NONE otherwise and logging failed reads. Acquire the device with RPM_TRANSPARENT: the I2C transport never enables runtime PM, and a plain pm_runtime_resume_and_get() would fail with -EACCES there and drop every jack interrupt. Release the IRQ mutex and PM reference on every exit path through a single error label. Should the codec stop responding altogether, IRQ_NONE lets the spurious-interrupt detector disable the level-triggered line instead of leaving it storming. Fixes: 2c394ca79604 ("ASoC: Add support for CS42L42 codec") Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/codecs/cs42l42.c | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/sound/soc/codecs/cs42l42.c b/sound/soc/codecs/cs42l42.c index 1f2efce071912..6a6c49510ab2d 100644 --- a/sound/soc/codecs/cs42l42.c +++ b/sound/soc/codecs/cs42l42.c @@ -1666,8 +1666,12 @@ irqreturn_t cs42l42_irq_thread(int irq, void *data) unsigned int current_plug_status; unsigned int current_button_status; unsigned int i; + int ret; - pm_runtime_get_sync(cs42l42->dev); + /* The I2C transport never enables runtime PM; treat it as powered. */ + ret = pm_runtime_get_active(cs42l42->dev, RPM_TRANSPARENT); + if (ret < 0) + return IRQ_NONE; mutex_lock(&cs42l42->irq_lock); if (cs42l42->suspended || !cs42l42->init_done) { mutex_unlock(&cs42l42->irq_lock); @@ -1677,10 +1681,14 @@ irqreturn_t cs42l42_irq_thread(int irq, void *data) /* Read sticky registers to clear interrupt */ for (i = 0; i < ARRAY_SIZE(stickies); i++) { - regmap_read(cs42l42->regmap, irq_params_table[i].status_addr, - &(stickies[i])); - regmap_read(cs42l42->regmap, irq_params_table[i].mask_addr, - &(masks[i])); + ret = regmap_read(cs42l42->regmap, irq_params_table[i].status_addr, + &stickies[i]); + if (ret) + goto out_error; + ret = regmap_read(cs42l42->regmap, irq_params_table[i].mask_addr, + &masks[i]); + if (ret) + goto out_error; stickies[i] = stickies[i] & (~masks[i]) & irq_params_table[i].mask; } @@ -1777,6 +1785,12 @@ irqreturn_t cs42l42_irq_thread(int irq, void *data) pm_runtime_put_autosuspend(cs42l42->dev); return IRQ_HANDLED; + +out_error: + mutex_unlock(&cs42l42->irq_lock); + pm_runtime_put_autosuspend(cs42l42->dev); + dev_err_ratelimited(cs42l42->dev, "Failed to read jack interrupt state: %d\n", ret); + return IRQ_NONE; } EXPORT_SYMBOL_NS_GPL(cs42l42_irq_thread, "SND_SOC_CS42L42_CORE"); From 965aa81912114de0f3418e3d8c1e1e3014689ac2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 23 Sep 2026 19:39:16 +1000 Subject: [PATCH 0414/2680] ASoC: cs42l42: support Apple headset remotes on CS42L83 Apple wired headset volume buttons use a microphone-bias signalling protocol instead of the resistor levels handled by the analogue detector. The CS42L83 can identify these remotes and decode their button indications without an audio stream or an external serial clock. After CTIA or OMTP detection, perform a bounded identification exchange on CS42L83. Wait for the receive acknowledgment rather than the earlier transmit-complete indication, which can expose an incomplete identifier. Validate all three identifier symbols before enabling remote reception. Restore the analogue detector on timeout, a short, an invalid identifier, or a register-access error. Preserve the receiver's raw volume indications before applying the analogue interrupt masks. Report volume indications as press/release pairs and use short-detect transitions for the centre button. Discard pre-identification button state, and clear events latched while the detector is reconfigured before unmasking its interrupts again. Restore the saved detector settings on unplug, redetection and suspend under the existing IRQ mutex, releasing any remote button still held. Select this capability from the CS42L83 device identity. No board-specific property, clock provider, work item or extra power reference is needed. The CS42L42 continues to use its analogue detector. The undocumented CS42L83 sequence was independently implemented from register-level reverse engineering and validated on a J700 with a B0 codec and Apple wired earbuds. Other boards and codec revisions have not been hardware-tested. The machine driver must expose the appropriate SND_JACK_BTN capabilities and key mappings. Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/codecs/cs42l42.c | 322 +++++++++++++++++++++++++++++++++++++ sound/soc/codecs/cs42l42.h | 16 ++ 2 files changed, 338 insertions(+) diff --git a/sound/soc/codecs/cs42l42.c b/sound/soc/codecs/cs42l42.c index 6a6c49510ab2d..55455b91a7885 100644 --- a/sound/soc/codecs/cs42l42.c +++ b/sound/soc/codecs/cs42l42.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -1256,6 +1257,297 @@ static void cs42l42_manual_hs_type_detect(struct cs42l42_private *cs42l42) (CS42L42_HSDET_COMP2_LVL_DEFAULT << CS42L42_HSDET_COMP2_LVL_SHIFT)); } +/* + * CS42L83 receives Apple headset remote data over the microphone-bias line. + * The receiver operates on the internal oscillator, including while audio + * is idle. These control values and status bits are specific to CS42L83. + */ + +/* MISC_DET_CTL */ +#define CS42L83_REMOTE_TX_START BIT(7) +/* Upper HSBIAS_CTL bit: set for the 2.7 V setting, clear when grounded. */ +#define CS42L83_HSBIAS_CTL_ON (2 << CS42L42_HSBIAS_CTL_SHIFT) + +/* HS_BIAS_CTL: undocumented field cleared before a bias transition */ +#define CS42L83_HS_BIAS_CTL_TRANSITION GENMASK(3, 2) + +/* DET_INT_STATUS1: volume pulses, outside the CS42L42 interrupt masks */ +#define CS42L83_REMOTE_BUTTONS GENMASK(4, 0) +#define CS42L83_REMOTE_VOLUME_DOWN BIT(0) +#define CS42L83_REMOTE_VOLUME_UP BIT(1) + +/* DET_INT_STATUS2 / DET_INT2_MASK */ +#define CS42L83_REMOTE_RX_ACK BIT(3) +#define CS42L83_REMOTE_BUTTON_IRQ BIT(5) + +/* Identifier: three 3-bit symbols in DET_STATUS1[4:0]:DET_STATUS2[7:4] */ +#define CS42L83_REMOTE_ID_LO GENMASK(7, 4) +#define CS42L83_REMOTE_ID_HI GENMASK(4, 0) +#define CS42L83_REMOTE_ID_SYMBOLS 3 +#define CS42L83_REMOTE_ID_SYMBOL GENMASK(2, 0) + +/* Complete receiver control values; these are not CS42L42 detect modes. */ +#define CS42L83_REMOTE_BIAS_RESET 0x03 +#define CS42L83_REMOTE_BIAS_IDENTIFY 0x9e +#define CS42L83_REMOTE_BIAS_BUTTONS 0x5f +#define CS42L83_REMOTE_RX_CONFIG 0x8f +#define CS42L83_REMOTE_LEVEL_IDENTIFY 0x36 +#define CS42L83_REMOTE_LEVEL_BUTTONS 0x3c +#define CS42L83_REMOTE_SENSE_TRIP (3 << CS42L42_HSBIAS_SENSE_TRIP_SHIFT) + +#define CS42L83_REMOTE_BIAS_SETTLE_US 10000 +#define CS42L83_REMOTE_SETTLE_MS 35 +#define CS42L83_REMOTE_RX_POLL_US 5000 +#define CS42L83_REMOTE_RX_TIMEOUT_US 500000 + +static const unsigned int cs42l83_remote_regs[] = { + [CS42L83_REMOTE_SAVED_HSBIAS_SC_AUTOCTL] = CS42L42_HSBIAS_SC_AUTOCTL, + [CS42L83_REMOTE_SAVED_WAKE_CTL] = CS42L42_WAKE_CTL, + [CS42L83_REMOTE_SAVED_MISC_DET_CTL] = CS42L42_MISC_DET_CTL, + [CS42L83_REMOTE_SAVED_MIC_DET_CTL1] = CS42L42_MIC_DET_CTL1, + [CS42L83_REMOTE_SAVED_MIC_DET_CTL2] = CS42L42_MIC_DET_CTL2, + [CS42L83_REMOTE_SAVED_HS_BIAS_CTL] = CS42L42_HS_BIAS_CTL, + [CS42L83_REMOTE_SAVED_HSDET_CTL2] = CS42L42_HSDET_CTL2, + [CS42L83_REMOTE_SAVED_DET_INT1_MASK] = CS42L42_DET_INT1_MASK, + [CS42L83_REMOTE_SAVED_DET_INT2_MASK] = CS42L42_DET_INT2_MASK, +}; + +static_assert(ARRAY_SIZE(cs42l83_remote_regs) == CS42L83_REMOTE_NUM_SAVED_REGS); + +static void cs42l83_remote_restore_reg(struct cs42l42_private *cs42l42, int i) +{ + int ret; + + ret = regmap_write(cs42l42->regmap, cs42l83_remote_regs[i], + cs42l42->remote_saved[i]); + if (ret) + dev_warn(cs42l42->dev, "Failed to restore headset detection register %#x: %d\n", + cs42l83_remote_regs[i], ret); +} + +/* + * Restore the analogue detector. @settle lets the level detector settle + * before its button interrupts are unmasked again, for when it stays in use. + */ +static void cs42l83_remote_restore(struct cs42l42_private *cs42l42, bool settle) +{ + unsigned int int_status; + int i; + + /* Nothing reports a release for a held remote button once disabled. */ + snd_soc_jack_report(cs42l42->jack, 0, + SND_JACK_BTN_0 | SND_JACK_BTN_1 | SND_JACK_BTN_2); + + for (i = 0; i < CS42L83_REMOTE_SAVED_DET_INT1_MASK; i++) + cs42l83_remote_restore_reg(cs42l42, i); + + if (settle) + msleep(cs42l42->btn_det_init_dbnce); + + /* Discard events latched while the detector was reconfigured. */ + regmap_read(cs42l42->regmap, CS42L42_DET_INT_STATUS1, &int_status); + regmap_read(cs42l42->regmap, CS42L42_DET_INT_STATUS2, &int_status); + + for (; i < CS42L83_REMOTE_NUM_SAVED_REGS; i++) + cs42l83_remote_restore_reg(cs42l42, i); + + cs42l42->remote_active = false; +} + +/* + * Hold the bias reference during a bias-mode change and allow it to settle + * before starting transmission. Apply the TX bit only after the bias bits. + */ +static int cs42l83_remote_set_detect(struct cs42l42_private *cs42l42, unsigned int value) +{ + unsigned int old; + bool changed; + int ret; + + ret = regmap_read(cs42l42->regmap, CS42L42_MISC_DET_CTL, &old); + if (ret) + return ret; + changed = (old & CS42L42_HSBIAS_CTL_MASK) != (value & CS42L42_HSBIAS_CTL_MASK); + if (changed) { + ret = regmap_update_bits(cs42l42->regmap, CS42L42_HS_BIAS_CTL, + CS42L83_HS_BIAS_CTL_TRANSITION, 0); + if (ret) + return ret; + ret = regmap_update_bits(cs42l42->regmap, CS42L42_HSDET_CTL2, + CS42L42_HSBIAS_REF_MASK, CS42L42_HSBIAS_REF_MASK); + if (ret) + return ret; + } + ret = regmap_update_bits(cs42l42->regmap, CS42L42_MISC_DET_CTL, + 0xff & ~(value & CS42L83_REMOTE_TX_START), value); + if (ret) + return ret; + if ((old ^ value) & CS42L83_HSBIAS_CTL_ON) + usleep_range(CS42L83_REMOTE_BIAS_SETTLE_US, + CS42L83_REMOTE_BIAS_SETTLE_US + 1000); + if (value & CS42L83_REMOTE_TX_START) { + ret = regmap_write(cs42l42->regmap, CS42L42_MISC_DET_CTL, value); + if (ret) + return ret; + } + if (changed) + return regmap_update_bits(cs42l42->regmap, CS42L42_HSDET_CTL2, + CS42L42_HSBIAS_REF_MASK, 0); + return 0; +} + +/* Called with irq_lock held; every attempt is bounded and has analogue fallback. */ +static void cs42l83_remote_start(struct cs42l42_private *cs42l42) +{ + static const struct reg_sequence identify[] = { + { CS42L42_DET_INT2_MASK, 0xff }, + { CS42L42_HSBIAS_SC_AUTOCTL, CS42L83_REMOTE_SENSE_TRIP }, + { CS42L42_WAKE_CTL, CS42L42_M_HP_WAKE_MASK | CS42L42_M_MIC_WAKE_MASK }, + { CS42L42_MIC_DET_CTL2, CS42L83_REMOTE_RX_CONFIG }, + }; + unsigned int int_status, status1, status2, id; + int i, ret; + + if (cs42l42->devid != CS42L83_CHIP_ID || + (cs42l42->hs_type != CS42L42_PLUG_CTIA && + cs42l42->hs_type != CS42L42_PLUG_OMTP)) + return; + + for (i = 0; i < ARRAY_SIZE(cs42l83_remote_regs); i++) { + ret = regmap_read(cs42l42->regmap, cs42l83_remote_regs[i], + &cs42l42->remote_saved[i]); + if (ret) { + dev_dbg(cs42l42->dev, "Failed to save headset detection state: %d\n", ret); + return; + } + } + + ret = regmap_multi_reg_write(cs42l42->regmap, identify, ARRAY_SIZE(identify)); + if (ret) + goto fallback; + /* Reset the bias before applying the identification threshold. */ + ret = cs42l83_remote_set_detect(cs42l42, CS42L83_REMOTE_BIAS_RESET); + if (ret) + goto fallback; + usleep_range(CS42L83_REMOTE_BIAS_SETTLE_US, CS42L83_REMOTE_BIAS_SETTLE_US + 1000); + ret = regmap_update_bits(cs42l42->regmap, CS42L42_MIC_DET_CTL1, + CS42L42_HS_DET_LEVEL_MASK, CS42L83_REMOTE_LEVEL_IDENTIFY); + if (ret) + goto fallback; + /* Discard stale sticky events before identification starts. */ + ret = regmap_read(cs42l42->regmap, CS42L42_DET_INT_STATUS1, &int_status); + if (ret) + goto fallback; + ret = regmap_read(cs42l42->regmap, CS42L42_DET_INT_STATUS2, &int_status); + if (ret) + goto fallback; + /* Start identification after the controlled bias rise. */ + ret = cs42l83_remote_set_detect(cs42l42, CS42L83_REMOTE_BIAS_IDENTIFY); + if (ret) + goto fallback; + + /* + * DET_INT_STATUS2 bit 4 signals TX completion, before the remote reply + * is ready. Wait for RX acknowledgment instead, with a bounded timeout. + */ + msleep(CS42L83_REMOTE_SETTLE_MS); + ret = regmap_read_poll_timeout(cs42l42->regmap, CS42L42_DET_INT_STATUS2, + int_status, int_status & CS42L83_REMOTE_RX_ACK, + CS42L83_REMOTE_RX_POLL_US, + CS42L83_REMOTE_RX_TIMEOUT_US); + if (ret) + goto fallback; + ret = regmap_read(cs42l42->regmap, CS42L42_DET_INT_STATUS1, &int_status); + if (ret) + goto fallback; + ret = regmap_read(cs42l42->regmap, CS42L42_DET_STATUS1, &status1); + if (ret) + goto fallback; + ret = regmap_read(cs42l42->regmap, CS42L42_DET_STATUS2, &status2); + if (ret) + goto fallback; + /* A short or an invalid identifier must retain the analogue detector. */ + if (status2 & CS42L42_SHORT_TRUE_MASK) { + ret = -ENODEV; + goto fallback; + } + id = FIELD_GET(CS42L83_REMOTE_ID_LO, status2) | + FIELD_GET(CS42L83_REMOTE_ID_HI, status1) << 4; + for (i = 0; i < CS42L83_REMOTE_ID_SYMBOLS; i++) { + unsigned int symbol = (id >> (i * 3)) & CS42L83_REMOTE_ID_SYMBOL; + + /* Valid identifier symbols are 1 to 4. */ + if (symbol < 1 || symbol > 4) { + ret = -ENODEV; + goto fallback; + } + } + + /* Use the remote-button threshold only after a valid identification. */ + ret = regmap_update_bits(cs42l42->regmap, CS42L42_MIC_DET_CTL1, + CS42L42_HS_DET_LEVEL_MASK, CS42L83_REMOTE_LEVEL_BUTTONS); + if (ret) + goto fallback; + + /* The receiver now decodes remote button pulses autonomously. */ + ret = cs42l83_remote_set_detect(cs42l42, CS42L83_REMOTE_BIAS_BUTTONS); + if (ret) + goto fallback; + msleep(CS42L83_REMOTE_SETTLE_MS); + ret = regmap_write(cs42l42->regmap, CS42L42_HSBIAS_SC_AUTOCTL, + (cs42l42->hs_bias_sense_en << CS42L42_HSBIAS_SENSE_EN_SHIFT) | + CS42L83_REMOTE_SENSE_TRIP); + if (ret) + goto fallback; + /* Discard events latched while switching to button reception. */ + ret = regmap_read(cs42l42->regmap, CS42L42_DET_INT_STATUS1, &int_status); + if (ret) + goto fallback; + ret = regmap_read(cs42l42->regmap, CS42L42_DET_INT_STATUS2, &int_status); + if (ret) + goto fallback; + ret = regmap_write(cs42l42->regmap, CS42L42_DET_INT1_MASK, + cs42l42->remote_saved[CS42L83_REMOTE_SAVED_DET_INT1_MASK] & + ~(cs42l42->hs_bias_sense_en << CS42L42_HSBIAS_SENSE_SHIFT)); + if (ret) + goto fallback; + ret = regmap_write(cs42l42->regmap, CS42L42_DET_INT2_MASK, + 0xff & ~(CS42L83_REMOTE_BUTTON_IRQ | + CS42L42_M_SHORT_DET_MASK | CS42L42_M_SHORT_RLS_MASK)); + if (ret) + goto fallback; + cs42l42->remote_active = true; + dev_dbg(cs42l42->dev, "Apple headset remote enabled\n"); + return; + +fallback: + dev_dbg(cs42l42->dev, "Headset remote identification failed: %d\n", ret); + cs42l83_remote_restore(cs42l42, true); +} + +static void cs42l83_remote_buttons(struct cs42l42_private *cs42l42, + unsigned int int1, unsigned int int2) +{ + unsigned int buttons = 0; + + /* Volume indications are pulses, without a separate release event. */ + if (int1 & CS42L83_REMOTE_BUTTONS) { + if (int1 & CS42L83_REMOTE_VOLUME_DOWN) + buttons |= SND_JACK_BTN_2; + if (int1 & CS42L83_REMOTE_VOLUME_UP) + buttons |= SND_JACK_BTN_1; + if (buttons) + snd_soc_jack_report(cs42l42->jack, buttons, + SND_JACK_BTN_1 | SND_JACK_BTN_2); + snd_soc_jack_report(cs42l42->jack, 0, + SND_JACK_BTN_1 | SND_JACK_BTN_2); + } + if (int2 & CS42L42_M_SHORT_DET_MASK) + snd_soc_jack_report(cs42l42->jack, SND_JACK_BTN_0, SND_JACK_BTN_0); + if (int2 & CS42L42_M_SHORT_RLS_MASK) + snd_soc_jack_report(cs42l42->jack, 0, SND_JACK_BTN_0); +} + static void cs42l42_process_hs_type_detect(struct cs42l42_private *cs42l42) { unsigned int hs_det_status; @@ -1397,6 +1689,9 @@ static void cs42l42_process_hs_type_detect(struct cs42l42_private *cs42l42) static void cs42l42_init_hs_type_detect(struct cs42l42_private *cs42l42) { + if (cs42l42->remote_active) + cs42l83_remote_restore(cs42l42, false); + /* Mask tip sense interrupts */ regmap_update_bits(cs42l42->regmap, CS42L42_TSRS_PLUG_INT_MASK, @@ -1484,6 +1779,9 @@ static void cs42l42_init_hs_type_detect(struct cs42l42_private *cs42l42) static void cs42l42_cancel_hs_type_detect(struct cs42l42_private *cs42l42) { + if (cs42l42->remote_active) + cs42l83_remote_restore(cs42l42, false); + /* Mask button detect interrupts */ regmap_update_bits(cs42l42->regmap, CS42L42_DET_INT2_MASK, @@ -1663,6 +1961,7 @@ irqreturn_t cs42l42_irq_thread(int irq, void *data) struct cs42l42_private *cs42l42 = (struct cs42l42_private *)data; unsigned int stickies[12]; unsigned int masks[12]; + unsigned int raw_remote_buttons = 0; unsigned int current_plug_status; unsigned int current_button_status; unsigned int i; @@ -1689,6 +1988,10 @@ irqreturn_t cs42l42_irq_thread(int irq, void *data) &masks[i]); if (ret) goto out_error; + /* Capture remote volume pulses before the CS42L42 masks drop them. */ + if (cs42l42->remote_active && + irq_params_table[i].status_addr == CS42L42_DET_INT_STATUS1) + raw_remote_buttons = stickies[i] & CS42L83_REMOTE_BUTTONS; stickies[i] = stickies[i] & (~masks[i]) & irq_params_table[i].mask; } @@ -1712,6 +2015,17 @@ irqreturn_t cs42l42_irq_thread(int irq, void *data) if ((~masks[5]) & irq_params_table[5].mask) { if (stickies[5] & CS42L42_HSDET_AUTO_DONE_MASK) { cs42l42_process_hs_type_detect(cs42l42); + if (cs42l42->devid == CS42L83_CHIP_ID) { + cs42l83_remote_start(cs42l42); + /* + * Identification toggles the mic bias, so button events + * latched until now are not key presses, even if the + * analogue detector is retained. + */ + current_button_status = 0; + stickies[7] = 0; + raw_remote_buttons = 0; + } switch (cs42l42->hs_type) { case CS42L42_PLUG_CTIA: case CS42L42_PLUG_OMTP: @@ -1762,6 +2076,11 @@ irqreturn_t cs42l42_irq_thread(int irq, void *data) } } + if (cs42l42->remote_active && cs42l42->plug_state == CS42L42_TS_PLUG) { + cs42l83_remote_buttons(cs42l42, raw_remote_buttons, stickies[7]); + goto out_unlock; + } + /* Check button detect status */ if (cs42l42->plug_state == CS42L42_TS_PLUG && ((~masks[7]) & irq_params_table[7].mask)) { if (!(current_button_status & @@ -1781,6 +2100,7 @@ irqreturn_t cs42l42_irq_thread(int irq, void *data) } } +out_unlock: mutex_unlock(&cs42l42->irq_lock); pm_runtime_put_autosuspend(cs42l42->dev); @@ -2177,6 +2497,8 @@ int cs42l42_suspend(struct device *dev) * is shared. */ mutex_lock(&cs42l42->irq_lock); + if (cs42l42->remote_active) + cs42l83_remote_restore(cs42l42, false); cs42l42->suspended = true; /* Save register values that will be overwritten by shutdown sequence */ diff --git a/sound/soc/codecs/cs42l42.h b/sound/soc/codecs/cs42l42.h index 3d85ebc594896..654491c42b3fa 100644 --- a/sound/soc/codecs/cs42l42.h +++ b/sound/soc/codecs/cs42l42.h @@ -24,6 +24,20 @@ #include #include +/* Restored in this order; the interrupt masks must come last. */ +enum cs42l83_remote_saved_reg { + CS42L83_REMOTE_SAVED_HSBIAS_SC_AUTOCTL, + CS42L83_REMOTE_SAVED_WAKE_CTL, + CS42L83_REMOTE_SAVED_MISC_DET_CTL, + CS42L83_REMOTE_SAVED_MIC_DET_CTL1, + CS42L83_REMOTE_SAVED_MIC_DET_CTL2, + CS42L83_REMOTE_SAVED_HS_BIAS_CTL, + CS42L83_REMOTE_SAVED_HSDET_CTL2, + CS42L83_REMOTE_SAVED_DET_INT1_MASK, + CS42L83_REMOTE_SAVED_DET_INT2_MASK, + CS42L83_REMOTE_NUM_SAVED_REGS, +}; + struct cs42l42_private { struct regmap *regmap; struct device *dev; @@ -51,6 +65,8 @@ struct cs42l42_private { u8 hs_bias_ramp_time; u8 hs_bias_sense_en; u8 stream_use; + bool remote_active; + unsigned int remote_saved[CS42L83_REMOTE_NUM_SAVED_REGS]; bool hp_adc_up_pending; bool suspended; bool sdw_waiting_first_unattach; From 2c152da6315e916b81db3fe1a9952653e1fcf8b9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 23 Sep 2026 19:33:54 +1000 Subject: [PATCH 0415/2680] ASoC: cs42l84: support Apple headset remotes Apple wired headset volume buttons and the centre play/pause button use a microphone-bias signalling protocol rather than the CTIA/OMTP resistor levels used for plug-type detection. The CS42L84 already exposes status bits for this over its microphone-detect block, but they are undocumented, unlike CS42L83's, and CS42L84 raises no known interrupt for them. Poll CS42L84_REMOTE_BUTTON_STATUS1 and the short-detect bit of HS_DET_STATUS2 at a fixed cadence while a mic-equipped headset is attached, and report press/release pairs on each rising edge, matching how the preceding CS42L83 patch reports its volume pulses. The centre button is therefore reported as a momentary press; a long press is not distinguished. The first sample after polling starts only establishes a baseline, so a line that is already shorted at detection is not reported as a press. Start polling once plug detection has settled and stop it on unplug or redetection, guarded by the same irq_lock used elsewhere for plug-state transitions. The poll runs on the freezable workqueue so it does not access the codec across system suspend, and is registered as resource-managed delayed work so it is cancelled if probe fails after a plug interrupt has already started it. The undocumented CS42L84 bits were identified by reverse engineering against a running CS42L84 on a J316 (MacBook Pro 16"), reading raw register state live via I2C while triggering the remote's buttons, and cross-checking candidate bits against the equivalent CS42L83 mechanism. Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/codecs/cs42l84.c | 101 +++++++++++++++++++++++++++++++++++++ sound/soc/codecs/cs42l84.h | 23 +++++++++ 2 files changed, 124 insertions(+) diff --git a/sound/soc/codecs/cs42l84.c b/sound/soc/codecs/cs42l84.c index f2a58163de0ed..48d8e95a53405 100644 --- a/sound/soc/codecs/cs42l84.c +++ b/sound/soc/codecs/cs42l84.c @@ -15,9 +15,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -51,6 +53,10 @@ struct cs42l84_private { u32 srate; u8 stream_use; int hs_type; + bool remote_active; + unsigned int remote_btn1_prev; + unsigned int remote_short_prev; + struct delayed_work remote_poll_work; }; static bool cs42l84_volatile_register(struct device *dev, unsigned int reg) @@ -61,6 +67,7 @@ static bool cs42l84_volatile_register(struct device *dev, unsigned int reg) case CS42L84_PLL_LOCK_STATUS: case CS42L84_TSRS_PLUG_STATUS: case CS42L84_HS_DET_STATUS2: + case CS42L84_REMOTE_BUTTON_STATUS1: return true; default: return false; @@ -710,10 +717,89 @@ static const struct cs42l84_irq_params irq_params_table[] = { CS42L84_TSRS_PLUG_VAL_MASK} }; +/* + * Poll for Apple remote button pulses while a mic-equipped headset is + * attached. CS42L84 has no known interrupt source for these (the only + * entry in irq_params_table is TSRS plug/unplug), so this polls the two + * empirically-discovered status bits at CS42L84_REMOTE_POLL_MS cadence and + * reports a press/release pair on each rising edge, matching how the + * CS42L83 reports its volume pulses. The work is freezable so it does not + * touch the codec across system suspend. + */ +static void cs42l84_remote_poll(struct work_struct *work) +{ + struct cs42l84_private *cs42l84 = container_of(to_delayed_work(work), + struct cs42l84_private, + remote_poll_work); + unsigned int btn1 = 0, short_status = 0, short_bit; + int ret; + + mutex_lock(&cs42l84->irq_lock); + + if (!cs42l84->remote_active) + goto out_unlock; + + ret = regmap_read(cs42l84->regmap, CS42L84_REMOTE_BUTTON_STATUS1, &btn1); + if (ret) + goto reschedule; + + ret = regmap_read(cs42l84->regmap, CS42L84_HS_DET_STATUS2, &short_status); + if (ret) + goto reschedule; + + short_bit = short_status & CS42L84_HS_DET_STATUS2_SHORT_TRUE; + + if ((btn1 & CS42L84_REMOTE_VOLUME_DOWN) && + !(cs42l84->remote_btn1_prev & CS42L84_REMOTE_VOLUME_DOWN)) { + snd_soc_jack_report(cs42l84->jack, SND_JACK_BTN_2, SND_JACK_BTN_2); + snd_soc_jack_report(cs42l84->jack, 0, SND_JACK_BTN_2); + } + if ((btn1 & CS42L84_REMOTE_VOLUME_UP) && + !(cs42l84->remote_btn1_prev & CS42L84_REMOTE_VOLUME_UP)) { + snd_soc_jack_report(cs42l84->jack, SND_JACK_BTN_1, SND_JACK_BTN_1); + snd_soc_jack_report(cs42l84->jack, 0, SND_JACK_BTN_1); + } + if (short_bit && !cs42l84->remote_short_prev) { + snd_soc_jack_report(cs42l84->jack, SND_JACK_BTN_0, SND_JACK_BTN_0); + snd_soc_jack_report(cs42l84->jack, 0, SND_JACK_BTN_0); + } + + cs42l84->remote_btn1_prev = btn1; + cs42l84->remote_short_prev = short_bit; + +reschedule: + queue_delayed_work(system_freezable_wq, &cs42l84->remote_poll_work, + msecs_to_jiffies(CS42L84_REMOTE_POLL_MS)); +out_unlock: + mutex_unlock(&cs42l84->irq_lock); +} + +/* Called with irq_lock held. */ +static void cs42l84_remote_start(struct cs42l84_private *cs42l84) +{ + cs42l84->remote_active = true; + /* + * Treat every input as already asserted so the first poll only takes + * a baseline; a line that is shorted at detection is not a key press. + */ + cs42l84->remote_btn1_prev = CS42L84_REMOTE_VOLUME_DOWN | CS42L84_REMOTE_VOLUME_UP; + cs42l84->remote_short_prev = CS42L84_HS_DET_STATUS2_SHORT_TRUE; + queue_delayed_work(system_freezable_wq, &cs42l84->remote_poll_work, + msecs_to_jiffies(CS42L84_REMOTE_POLL_MS)); +} + +/* Called with irq_lock held. */ +static void cs42l84_remote_stop(struct cs42l84_private *cs42l84) +{ + cs42l84->remote_active = false; +} + static void cs42l84_detect_hs(struct cs42l84_private *cs42l84) { unsigned int reg; + cs42l84_remote_stop(cs42l84); + /* Power up HSBIAS */ regmap_update_bits(cs42l84->regmap, CS42L84_MISC_DET_CTL, @@ -773,6 +859,7 @@ static void cs42l84_detect_hs(struct cs42l84_private *cs42l84) cs42l84->hs_type = SND_JACK_HEADSET; snd_soc_jack_report(cs42l84->jack, SND_JACK_HEADSET, SND_JACK_HEADSET); + cs42l84_remote_start(cs42l84); break; case 0b00: /* open */ @@ -790,6 +877,8 @@ static void cs42l84_detect_hs(struct cs42l84_private *cs42l84) static void cs42l84_revert_hs(struct cs42l84_private *cs42l84) { + cs42l84_remote_stop(cs42l84); + /* Power down HSBIAS */ regmap_update_bits(cs42l84->regmap, CS42L84_MISC_DET_CTL, @@ -1003,6 +1092,16 @@ static int cs42l84_i2c_probe(struct i2c_client *i2c_client) return ret; } + /* + * A plug IRQ during probe can start polling; cancel it on any probe + * failure, and before the regmap it uses is released. + */ + ret = devm_delayed_work_autocancel(&i2c_client->dev, + &cs42l84->remote_poll_work, + cs42l84_remote_poll); + if (ret) + return ret; + /* Reset the Device */ cs42l84->reset_gpio = devm_gpiod_get_optional(&i2c_client->dev, "reset", GPIOD_OUT_LOW); @@ -1091,6 +1190,8 @@ static void cs42l84_i2c_remove(struct i2c_client *i2c_client) if (i2c_client->irq) free_irq(i2c_client->irq, cs42l84); + cancel_delayed_work_sync(&cs42l84->remote_poll_work); + gpiod_set_value_cansleep(cs42l84->reset_gpio, 0); } diff --git a/sound/soc/codecs/cs42l84.h b/sound/soc/codecs/cs42l84.h index dbf778a902b9c..e8aa3e8b8ad3a 100644 --- a/sound/soc/codecs/cs42l84.h +++ b/sound/soc/codecs/cs42l84.h @@ -106,6 +106,26 @@ #define CS42L84_MIC_DET_CTL4_LATCH_TO_VP BIT(1) #define CS42L84_HS_DET_STATUS2 0x147d +/* + * Bit 0 pulses briefly on a momentary short of the mic/remote line, which is + * how a 3-button Apple remote signals its centre (play/pause) button. This + * mirrors CS42L42_SHORT_TRUE_MASK / M_SHORT_DET on the CS42L42/CS42L83. + * Empirically observed on real hardware (16" MacBook Pro, M1 Pro, J316); + * there is no public register reference for CS42L84. + */ +#define CS42L84_HS_DET_STATUS2_SHORT_TRUE BIT(0) + +/* + * Undocumented, empirically-discovered register that pulses a bit when the + * Apple remote's volume buttons are pressed, one bit per direction. Found by + * polling raw I2C reads while pressing physical buttons on wired Apple + * earbuds. Bit assignment follows the CS42L83 remote convention + * (CS42L83_REMOTE_VOLUME_DOWN/UP in cs42l42.c) since CS42L84 is a close + * relative of that chip family; not confirmed against a datasheet. + */ +#define CS42L84_REMOTE_BUTTON_STATUS1 0x1484 +#define CS42L84_REMOTE_VOLUME_DOWN BIT(0) +#define CS42L84_REMOTE_VOLUME_UP BIT(1) #define CS42L84_MSM_BLOCK_EN1 0x1800 #define CS42L84_MSM_BLOCK_EN2 0x1801 @@ -207,4 +227,7 @@ #define CS42L84_PLL_LOCK_POLL_US 250 #define CS42L84_PLL_LOCK_TIMEOUT_US 1250 +/* Poll interval while a mic-equipped headset is attached, looking for remote button pulses. */ +#define CS42L84_REMOTE_POLL_MS 30 + #endif /* __CS42L84_H__ */ From b817ad032a7d327f5eba56b77f5c003caf0f2fea Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 23 Sep 2026 19:33:54 +1000 Subject: [PATCH 0416/2680] ASoC: macaudio: report headset remote buttons CS42L83 and CS42L84 now decode Apple wired-headset remote presses (see the preceding cs42l42 and cs42l84 patches) and report them as SND_JACK_BTN_0..2 on the codec's jack. When CS42L83 falls back to its analogue detector for other headsets, it can also report SND_JACK_BTN_3. Request those bits on the headphone jack and map them to the usual keys so userspace sees ordinary input events: centre button to KEY_PLAYPAUSE, volume up/down to KEY_VOLUMEUP and KEY_VOLUMEDOWN, and a fourth button to KEY_VOICECOMMAND, matching other CS42L42-family machine drivers. Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/macaudio.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index 2f26adc5b1be9..6f4df9de67378 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -1085,7 +1085,9 @@ static int macaudio_probe(struct snd_soc_card *card) dev_dbg(card->dev, "%s!\n", __func__); ret = snd_soc_card_jack_new_pins(card, "Headphone Jack", - SND_JACK_HEADSET | SND_JACK_HEADPHONE, + SND_JACK_HEADSET | SND_JACK_HEADPHONE | + SND_JACK_BTN_0 | SND_JACK_BTN_1 | + SND_JACK_BTN_2 | SND_JACK_BTN_3, &ma->jack, macaudio_jack_pins, ARRAY_SIZE(macaudio_jack_pins)); if (ret < 0) { @@ -1093,6 +1095,15 @@ static int macaudio_probe(struct snd_soc_card *card) return ret; } + /* + * Apple remotes use BTN_0..2; CS42L83's analogue fallback can also + * report BTN_3 for the fourth button of other headsets. + */ + snd_jack_set_key(ma->jack.jack, SND_JACK_BTN_0, KEY_PLAYPAUSE); + snd_jack_set_key(ma->jack.jack, SND_JACK_BTN_1, KEY_VOLUMEUP); + snd_jack_set_key(ma->jack.jack, SND_JACK_BTN_2, KEY_VOLUMEDOWN); + snd_jack_set_key(ma->jack.jack, SND_JACK_BTN_3, KEY_VOICECOMMAND); + return ret; } From caf818af61c1b2a42a2349f2cbf5434348aac58b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:26:46 +1000 Subject: [PATCH 0417/2680] dt-bindings: dma: apple,admac: Add T8140 compatible The ADMACs of the A18 Pro (T8140) audio fabric keep the register layout within each channel block but move the RX blocks: TX channel blocks sit at 0x8000 + n * 0x200 and RX channel blocks at 0xc000 + n * 0x200, where the earlier ADMACs interleave all channels from 0x8000. A driver that only knows apple,admac or apple,t8122-admac would program the RX channels at the wrong offsets, so add "apple,t8140-admac" without a fallback. The T8140 ADMACs sit behind the AOP DART, whose stream roots are owned by the AOP firmware. Add an optional apple,dma-range property (base and nonzero size) giving the host's allocation window for the controller's stream; without it, the DART's aperture applies. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/dma/apple,admac.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Documentation/devicetree/bindings/dma/apple,admac.yaml b/Documentation/devicetree/bindings/dma/apple,admac.yaml index 5b4fd8348f99f..adc94ff3a3b69 100644 --- a/Documentation/devicetree/bindings/dma/apple,admac.yaml +++ b/Documentation/devicetree/bindings/dma/apple,admac.yaml @@ -39,6 +39,7 @@ properties: - apple,t6031-admac - const: apple,t8122-admac - const: apple,t8122-admac + - const: apple,t8140-admac reg: maxItems: 1 @@ -65,6 +66,15 @@ properties: minItems: 1 maxItems: 2 + apple,dma-range: + $ref: /schemas/types.yaml#/definitions/uint64-array + minItems: 2 + maxItems: 2 + description: + Host DMA allocation window as a base and nonzero size. T8140 ADMAC + instances share firmware-owned AOP DART roots and use windows assigned + to their own streams. If absent, the DART allocation aperture applies. + power-domains: maxItems: 1 From 5f4a53837af18059258ebe53fca454acf4feff18 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:26:53 +1000 Subject: [PATCH 0418/2680] dt-bindings: iio: light: Add Apple T8140 AOP ambient light sensor On the A18 Pro (T8140) the ambient light sensor is read by the Always-On Processor (AOP) firmware and exposed to the host as an AOP service. Describe it as a child of the AOP node without resources of its own: it has no MMIO window or interrupt, and its calibration is loaded separately through the firmware loader rather than stored in the device tree. The "apple,aop-als" fallback is the compatible that the ALS nodes of the other Apple SoCs (T8103, T8112, T600x, T602x, T603x and T8122) already use. Those nodes have no schema, so a select keeps this T8140 schema from being applied to them through the shared fallback. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../iio/light/apple,t8140-aop-als.yaml | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml diff --git a/Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml b/Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml new file mode 100644 index 0000000000000..5b6ad07fa6f84 --- /dev/null +++ b/Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iio/light/apple,t8140-aop-als.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 AOP ambient light sensor + +maintainers: + - Ryan Murray + +description: + Ambient light measurements are reported by a service of the Always-On + Processor. This child node identifies that service; it has no separate + application-processor MMIO or interrupt resource. Machine calibration is + supplied separately through the firmware loader, not inline in this node. + +select: + properties: + compatible: + contains: + const: apple,t8140-aop-als + required: + - compatible + +properties: + compatible: + items: + - const: apple,t8140-aop-als + - const: apple,aop-als + +required: + - compatible + +additionalProperties: false From 9926d998d9746a689ba438d6ad337c1936f83fa5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:27:05 +1000 Subject: [PATCH 0419/2680] dt-bindings: sound: cirrus,cs42l42: Allow DAI cells and name prefix Apple machines use the CS42L83, which this schema covers, as their headset codec. Their device trees reference it as a DAI, with "#sound-dai-cells = <0>", and prefix its controls with sound-name-prefix = "Jack", but the schema has additionalProperties: false and allows neither property, so dtbs_check rejects those nodes (the T8103 Mac mini, MacBook and iMac boards, and now the MacBook Neo). Allow both. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml b/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml index af599d8735e26..28a5303cf7204 100644 --- a/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml +++ b/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml @@ -26,6 +26,11 @@ properties: The I2C address of the CS42L42. maxItems: 1 + '#sound-dai-cells': + const: 0 + + sound-name-prefix: true + VP-supply: description: VP power supply. From 5448def045ad6c2ad5f42913a008126dbfb79292 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:27:07 +1000 Subject: [PATCH 0420/2680] dt-bindings: sound: apple,macaudio: Add J700 compatible The MacBook Neo (J700) sound card links the audio DAIs of the Always-On Processor to a CS42L83 headset codec and a MAX98360A speaker amplifier. Add "apple,j700-macaudio" with the "apple,macaudio" fallback. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/sound/apple,macaudio.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/sound/apple,macaudio.yaml b/Documentation/devicetree/bindings/sound/apple,macaudio.yaml index 8fe22dec3015d..8e986cca62594 100644 --- a/Documentation/devicetree/bindings/sound/apple,macaudio.yaml +++ b/Documentation/devicetree/bindings/sound/apple,macaudio.yaml @@ -20,6 +20,7 @@ properties: - apple,j274-macaudio - apple,j293-macaudio - apple,j314-macaudio + - apple,j700-macaudio - const: apple,macaudio "#address-cells": From b8ccbe1d65977a4eb981246a23f337235fa9b482 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:27:09 +1000 Subject: [PATCH 0421/2680] dt-bindings: sound: Add Apple T8140 AOP audio On the A18 Pro (T8140) the Always-On Processor (AOP) firmware owns the audio paths: the internal speaker, the headset jack in both directions, a serial monitor tap on the speaker output, the high-quality microphone array and the low-power microphone. Add a binding for the audio child of the AOP node, which gives the host: - five ADMAC channels (spkr, cout, cin, spkr-tap and hpai), used while the matching AOP service is active; - the power domains of the LEAP transmit and receive channels and of the serializers (tx0, mca, mca1, mca0 and rx0); - the AOP DART stream and IOVA window of the low-power microphone, which the firmware fills through a shared ring instead of an ADMAC channel; - one DAI cell selecting the AOP audio DAI (on the MacBook Neo, 0 is the headset link and 4 the speaker link); - apple,chassis-name and apple,machine-kind, which name the sound card as on the existing AOP audio nodes. There is no "apple,aop-audio" fallback: the T8140 capture path, a 16 kHz source ring on the setup port, is not compatible with the existing AOP audio driver. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/sound/apple,t8140-aop-audio.yaml | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml diff --git a/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml b/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml new file mode 100644 index 0000000000000..a4e10cbb4deea --- /dev/null +++ b/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/apple,t8140-aop-audio.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 AOP audio + +maintainers: + - Ryan Murray + +description: | + The Always-On Processor on Apple T8140 SoCs owns the machine's low-power + audio paths. This node is a child of the AOP node and describes the DMA + channels and power domains the host needs in order to drive them, namely + the internal speaker, the 3.5 mm headset jack in both directions, a serial + monitor tap on the speaker output, and the high-quality microphone array. + + The low-power microphone uses a shared ring on this node's DART stream. + The remaining audio data moves over the named ADMAC channels while its AOP + service is active. The parent AOP binding describes the firmware transport. + +properties: + compatible: + const: apple,t8140-aop-audio + + dmas: + items: + - description: Internal speaker output + - description: Headset jack output + - description: Headset microphone input + - description: Speaker output monitor input + - description: High-quality microphone array input + + dma-names: + items: + - const: spkr + - const: cout + - const: cin + - const: spkr-tap + - const: hpai + + power-domains: + items: + - description: LEAP speaker transmit channel + - description: LEAP serializer + - description: Headset serializer + - description: Speaker monitor serializer + - description: LEAP microphone receive channel + + power-domain-names: + items: + - const: tx0 + - const: mca + - const: mca1 + - const: mca0 + - const: rx0 + + iommus: + maxItems: 1 + + apple,dma-range: + $ref: /schemas/types.yaml#/definitions/uint64-array + minItems: 2 + maxItems: 2 + description: | + IOVA window the AOP may use for this device's audio buffers, expressed + as a base followed by its size. + + "#sound-dai-cells": + const: 1 + description: + AOP audio DAI identifier. The J700 headset link uses 0 and its speaker + link uses 4. + + apple,chassis-name: + $ref: /schemas/types.yaml#/definitions/string + description: + Chassis identifier used to name the sound card, matching the + apple,aop-audio convention. + + apple,machine-kind: + $ref: /schemas/types.yaml#/definitions/string + description: + Human-readable machine name used in the sound card's description. + +required: + - compatible + - dmas + - dma-names + - power-domains + - power-domain-names + - iommus + - apple,dma-range + - "#sound-dai-cells" + - apple,chassis-name + - apple,machine-kind + +additionalProperties: false + +examples: + - | + aop { + audio { + compatible = "apple,t8140-aop-audio"; + iommus = <&aop_dart 9>; + apple,dma-range = <0x100 0x40000000 0x0 0xffff0000>; + dmas = <&admac_leap_ns 0>, <&admac_base_ns 4>, + <&admac_base_ns 5>, <&admac_base_ns 3>, + <&admac_leap_s 1>; + dma-names = "spkr", "cout", "cin", "spkr-tap", "hpai"; + power-domains = <&ps_audio_leap_tx0>, <&ps_audio_leap_mca>, + <&ps_audio_mca1_m>, <&ps_audio_mca0_m>, + <&ps_audio_leap_rx0>; + power-domain-names = "tx0", "mca", "mca1", "mca0", "rx0"; + #sound-dai-cells = <1>; + apple,chassis-name = "J700"; + apple,machine-kind = "MacBook Neo"; + }; + }; From 464a9301a697750a6f4fdd834c0df93716c1bcfd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:32:08 +1000 Subject: [PATCH 0422/2680] rust: soc: apple: mailbox: Fix receiver teardown and expose the device Mailbox::new_byname() installs the receive callback and its cookie in struct apple_mbox before starting the mailbox. If apple_mbox_start() fails, the scope guard frees the cookie but leaves both pointers installed, and Drop likewise frees the cookie after stopping the mailbox without clearing them. Anything that reaches the provider's receive path afterwards, such as apple_mbox_poll(), then calls into freed memory. Clear the callback and the cookie on the error path, and in Drop before the cookie is freed; apple_mbox_poll() returns without touching the FIFO while no receiver is attached. Add Mailbox::device(), which returns the mailbox provider's device. A consumer uses it to allocate DMA buffers against the provider's own IOMMU stream, as the T8140 AOP driver does for the setup-port mailbox, whose node carries that stream. Also point the module documentation at include/linux/soc/apple/mailbox.h instead of include/linux/gpio/driver.h. Fixes: 0de2ff394e19 ("rust: soc: apple: Add Apple mailbox abstractions") Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/kernel/soc/apple/mailbox.rs | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/rust/kernel/soc/apple/mailbox.rs b/rust/kernel/soc/apple/mailbox.rs index f477ddc0afb00..c4f86930e3c65 100644 --- a/rust/kernel/soc/apple/mailbox.rs +++ b/rust/kernel/soc/apple/mailbox.rs @@ -2,7 +2,7 @@ //! Support for Apple ASC Mailbox. //! -//! C header: [`include/linux/soc/apple/mailbox.h`](../../../../include/linux/gpio/driver.h) +//! C header: [`include/linux/soc/apple/mailbox.h`](../../../../include/linux/soc/apple/mailbox.h) use crate::{ bindings, @@ -72,7 +72,11 @@ impl Mailbox { unsafe { (*mbox).cookie = ptr; (*mbox).rx = Some(mailbox_rx_callback::); - to_result(bindings::apple_mbox_start(mbox))?; + if let Err(err) = to_result(bindings::apple_mbox_start(mbox)) { + (*mbox).rx = None; + (*mbox).cookie = core::ptr::null_mut(); + return Err(err); + } } guard.dismiss(); Ok(Mailbox { @@ -80,6 +84,13 @@ impl Mailbox { _p: PhantomData, }) } + /// The mailbox provider's device (DMA through its own IOMMU stream). + pub fn device(&self) -> &device::Device { + // SAFETY: `mbox` is a valid pointer for the life of `self` and its + // `dev` is the provider's device, which outlives the client. + unsafe { device::Device::from_raw((*self.mbox).dev) } + } + /// Sends the specified message pub fn send(&self, msg: Message, atomic: bool) -> Result<()> { // SAFETY: Calling the c function, `mbox` is a valid pointer @@ -91,8 +102,15 @@ impl Drop for Mailbox { fn drop(&mut self) { // SAFETY: mbox is a valid pointer unsafe { bindings::apple_mbox_stop(self.mbox) }; - // SAFETY: `cookie` came from `into_foreign` - unsafe { T::Data::from_foreign((*self.mbox).cookie.cast()) }; + // SAFETY: RX is synchronously disabled above, and `cookie` came from + // `into_foreign`. Clear the provider-visible callback before freeing + // its context so provider removal can never observe a dangling owner. + unsafe { + let cookie = (*self.mbox).cookie; + (*self.mbox).rx = None; + (*self.mbox).cookie = core::ptr::null_mut(); + T::Data::from_foreign(cookie.cast()); + } } } From 54c074fcc08804be7983b5b2272ce197ea619235 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:44:27 +1000 Subject: [PATCH 0423/2680] rust: sync: condvar: Add wait_timeout() CondVar has an uninterruptible wait without a timeout (wait()) and interruptible waits with and without one, but no uninterruptible wait with a timeout. A driver that waits a bounded time for a co-processor reply, and must not have that wait cut short by a signal, has no suitable primitive. Add wait_timeout(), which sleeps in TASK_UNINTERRUPTIBLE for at most the given number of jiffies and returns CondVarTimeoutResult::Timeout, or CondVarTimeoutResult::Woken with the remaining jiffies. It never returns CondVarTimeoutResult::Signal. Like the other waits it can wake up spuriously, so callers have to check their condition again. The T8140 AOP driver, submitted separately, uses it for its bounded firmware waits. Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/kernel/sync/condvar.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/rust/kernel/sync/condvar.rs b/rust/kernel/sync/condvar.rs index 69d58dfbad7b1..f60b6f20135de 100644 --- a/rust/kernel/sync/condvar.rs +++ b/rust/kernel/sync/condvar.rs @@ -177,6 +177,27 @@ impl CondVar { crate::current!().signal_pending() } + /// Releases the lock and waits for a notification in uninterruptible mode. + /// + /// Atomically releases the given lock (whose ownership is proven by the guard) and puts the + /// thread to sleep. It wakes up when notified by [`CondVar::notify_one`] or + /// [`CondVar::notify_all`], or when a timeout occurs. Signals do not wake it up, so the + /// result is never [`CondVarTimeoutResult::Signal`]. + #[must_use = "wait_timeout may wake up spuriously, so the caller must check the return value"] + pub fn wait_timeout( + &self, + guard: &mut Guard<'_, T, B>, + jiffies: Jiffies, + ) -> CondVarTimeoutResult { + let jiffies = jiffies.try_into().unwrap_or(MAX_SCHEDULE_TIMEOUT); + let res = self.wait_internal(TASK_UNINTERRUPTIBLE, guard, jiffies); + + match res as Jiffies { + 0 => CondVarTimeoutResult::Timeout, + jiffies => CondVarTimeoutResult::Woken { jiffies }, + } + } + /// Releases the lock and waits for a notification in interruptible mode. /// /// Atomically releases the given lock (whose ownership is proven by the guard) and puts the From 8d9546b5f8e7e978c510175e48db2e5da73c7b69 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:44:28 +1000 Subject: [PATCH 0424/2680] rust: helpers: Add runtime PM and ASoC helpers for the AOP drivers The T8140 AOP drivers, submitted separately, need a few C interfaces that Rust cannot reach through bindgen alone: - pm_runtime_get_sync(), pm_runtime_put_sync() and pm_runtime_put_noidle() are static inlines. Add helpers for them in a new rust/helpers/pm_domain.c, and generate bindings for linux/pm_domain.h and linux/pm_runtime.h, which the audio driver needs for its named power domains. - The ASoC core and DPCM interfaces (sound/soc.h and sound/soc-dpcm.h). SNDRV_PCM_FMTBIT_S32_LE, SNDRV_PCM_FMTBIT_S24_LE and SNDRV_PCM_SUBFMTBIT_STD are macros that bindgen cannot evaluate, so export them as constants, as SNDRV_PCM_FMTBIT_FLOAT_LE already is. Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/bindings/bindings_helper.h | 7 +++++++ rust/helpers/helpers.c | 1 + rust/helpers/pm_domain.c | 19 +++++++++++++++++++ 3 files changed, 27 insertions(+) create mode 100644 rust/helpers/pm_domain.c diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 31fe42ed08c4c..79187dc7ce63f 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -107,7 +107,11 @@ #include #include #include +#include +#include #include +#include +#include #include /* @@ -137,6 +141,9 @@ const fop_flags_t RUST_CONST_HELPER_FOP_UNSIGNED_OFFSET = FOP_UNSIGNED_OFFSET; const u32 RUST_CONST_HELPER_APPLE_RTKIT_COPROC_PLACES_BUFFERS = APPLE_RTKIT_COPROC_PLACES_BUFFERS; const u64 BINDINGS_SNDRV_PCM_FMTBIT_FLOAT_LE = SNDRV_PCM_FMTBIT_FLOAT_LE; +const u64 BINDINGS_SNDRV_PCM_FMTBIT_S32_LE = SNDRV_PCM_FMTBIT_S32_LE; +const u64 BINDINGS_SNDRV_PCM_FMTBIT_S24_LE = SNDRV_PCM_FMTBIT_S24_LE; +const u32 BINDINGS_SNDRV_PCM_SUBFMTBIT_STD = SNDRV_PCM_SUBFMTBIT_STD; const u32 BINDINGS_IIO_CHAN_INFO_RAW = IIO_CHAN_INFO_RAW; const u32 BINDINGS_IIO_CHAN_INFO_PROCESSED = IIO_CHAN_INFO_PROCESSED; diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 8a4e1c8354074..0e8664be6e80c 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -85,6 +85,7 @@ #include "poll.c" #include "processor.c" #include "property.c" +#include "pm_domain.c" #include "pwm.c" #include "rbtree.c" #include "rcu.c" diff --git a/rust/helpers/pm_domain.c b/rust/helpers/pm_domain.c new file mode 100644 index 0000000000000..f60252c81bb7b --- /dev/null +++ b/rust/helpers/pm_domain.c @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +__rust_helper int rust_helper_pm_runtime_get_sync(struct device *dev) +{ + return pm_runtime_get_sync(dev); +} + +__rust_helper int rust_helper_pm_runtime_put_sync(struct device *dev) +{ + return pm_runtime_put_sync(dev); +} + +__rust_helper void rust_helper_pm_runtime_put_noidle(struct device *dev) +{ + pm_runtime_put_noidle(dev); +} From 1311d9fb4cc2e83232c31a5b2f9c4fd16695c438 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:45:13 +1000 Subject: [PATCH 0425/2680] rust: soc: apple: rtkit: Add shutdown() and a buffer-retaining drop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Rust RTKit client has no way to shut its co-processor down, and when a shutdown cannot be confirmed, dropping the RtKit still calls apple_rtkit_free(), which returns the shared buffers to their allocator while the firmware may still DMA into them. Add RtKit::shutdown(), which calls apple_rtkit_shutdown(). The C core bounds its waits and drains the RTKit workqueue, so the caller must not hold a lock that its receive callbacks take. Add RtKit::retain_shared_buffers_on_drop(), after which Drop calls apple_rtkit_free_retaining_buffers() instead of apple_rtkit_free(): the callbacks and the workqueue are still torn down, but the shared buffers and their shmem_setup contexts stay allocated until reboot. The owner must keep its own DMA buffers as well and must not re-probe the device. Implement Unpin for RtKit regardless of T. The wrapper holds only raw pointers and a PhantomData, and neither the C instance nor the callback context points back into it, so moving it is sound; without this, RtKit is !Unpin whenever T is. [ryan: adapted to 7.2.10 — dropped the IoSysMap I/O-memory rejection, Asahi's rtkit.rs now matches IoSysMap::Sys itself] Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/kernel/soc/apple/rtkit.rs | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index 4b82dbf32129f..5f7e572a3a462 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -117,9 +117,14 @@ pub trait Operations { pub struct RtKit { rtk: *mut bindings::apple_rtkit, data: *mut core::ffi::c_void, + retain_shared_buffers: bool, _p: PhantomData, } +// The wrapper owns opaque pointers. Neither the C allocation nor the foreign +// callback context points back to this Rust wrapper, so moving it is safe. +impl Unpin for RtKit {} + unsafe extern "C" fn crashed_callback( cookie: *mut core::ffi::c_void, crashlog: *const core::ffi::c_void, @@ -264,10 +269,29 @@ impl RtKit { Ok(Self { rtk, data: ptr, + retain_shared_buffers: false, _p: PhantomData, }) } + /// Requests acknowledged RTKit shutdown using the C core's bounded waits. + /// + /// The caller must not hold a lock needed by its receive callbacks: the + /// shutdown path also drains the RTKit workqueue. + pub fn shutdown(self: Pin<&mut Self>) -> Result { + // SAFETY: `rtk` is valid and the mutable borrow excludes other callers. + to_result(unsafe { bindings::apple_rtkit_shutdown(self.rtk) }) + } + + /// Retains firmware-visible buffers when a shutdown cannot be confirmed. + /// + /// Drop will still close callbacks and drain the private RTKit queue, but + /// shared mappings/allocation contexts remain retained until reboot. The + /// owner must retain its own DMA buffers too. This does not permit reprobe. + pub fn retain_shared_buffers_on_drop(&mut self) { + self.retain_shared_buffers = true; + } + /// Boots (wakes up) the RTKit coprocessor. pub fn wake(self: Pin<&mut Self>) -> Result { // SAFETY: `rtk` is valid per the type invariant. @@ -309,7 +333,13 @@ unsafe impl Send for RtKit {} impl Drop for RtKit { fn drop(&mut self) { // SAFETY: The pointer is valid by the type invariant. - unsafe { bindings::apple_rtkit_free(self.rtk) }; + unsafe { + if self.retain_shared_buffers { + bindings::apple_rtkit_free_retaining_buffers(self.rtk); + } else { + bindings::apple_rtkit_free(self.rtk); + } + } // Free context data. // From 79fad70347b7a703e44fc719c45d8138607eb6b6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:03:13 +1000 Subject: [PATCH 0426/2680] rust: workqueue: Add OwnedQueue for driver-private workqueues The workqueue abstraction only hands out the system queues. A driver that needs its work items to run one at a time in queueing order, or that has to wait for all of its outstanding work before it tears down the state the work uses, has no safe way to get a workqueue of its own and ends up calling alloc_workqueue(), drain_workqueue() and destroy_workqueue() through raw bindings. Add OwnedQueue, a wrapper around a workqueue allocated with alloc_ordered_workqueue(). It dereferences to Queue, so work items are enqueued on it like on the system queues. drain() exposes drain_workqueue(); its documentation states the rule that every other producer has to be stopped first, because the kernel refuses (with a warning) to queue work from outside the queue while it drains. Drop calls destroy_workqueue(), which drains the queue before freeing it. alloc_ordered_workqueue() is a variadic macro, so a helper with a fixed "%s" format takes the name; the kernel copies it into the workqueue. The T8140 AOP driver uses it to serialize the registration of its service devices with its own teardown and to run its setup-port protocol in order. Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/helpers/workqueue.c | 6 +++ rust/kernel/workqueue.rs | 104 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/rust/helpers/workqueue.c b/rust/helpers/workqueue.c index ce1c3a5b21502..7cd3b000a5b6b 100644 --- a/rust/helpers/workqueue.c +++ b/rust/helpers/workqueue.c @@ -14,3 +14,9 @@ __rust_helper void rust_helper_init_work_with_key(struct work_struct *work, INIT_LIST_HEAD(&work->entry); work->func = func; } + +__rust_helper struct workqueue_struct * +rust_helper_alloc_ordered_workqueue(const char *name, unsigned int flags) +{ + return alloc_ordered_workqueue("%s", flags, name); +} diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs index 7e253b6f299ce..d2791beb6d15b 100644 --- a/rust/kernel/workqueue.rs +++ b/rust/kernel/workqueue.rs @@ -200,7 +200,7 @@ use crate::{ time::Jiffies, types::Opaque, }; -use core::{marker::PhantomData, ptr::NonNull}; +use core::{marker::PhantomData, ops::Deref, ptr::NonNull}; /// Creates a [`Work`] initialiser with the given name and a newly-created lock class. #[macro_export] @@ -372,6 +372,108 @@ impl WorkItem for ClosureWork { } } +/// A work queue owned by the caller. +/// +/// Wraps a `struct workqueue_struct` allocated with `alloc_ordered_workqueue()`. It dereferences +/// to a [`Queue`], so work items are enqueued on it exactly like on the system queues. Dropping it +/// calls `destroy_workqueue()`, which waits for every pending work item to finish first. +/// +/// An owned queue gives a driver two things the system queues do not: its work items run one at a +/// time in queueing order, and the driver can wait for all of them at once (see [`drain`]) before +/// it tears down the state they use. +/// +/// # Invariants +/// +/// The pointer is valid and points at a workqueue allocated by `alloc_ordered_workqueue()`. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::Arc; +/// use kernel::workqueue::{self, impl_has_work, new_work, OwnedQueue, Work, WorkItem}; +/// +/// #[pin_data] +/// struct MyStruct { +/// value: i32, +/// #[pin] +/// work: Work, +/// } +/// +/// impl_has_work! { +/// impl HasWork for MyStruct { self.work } +/// } +/// +/// impl WorkItem for MyStruct { +/// type Pointer = Arc; +/// +/// fn run(this: Arc) { +/// pr_info!("The value is: {}\n", this.value); +/// } +/// } +/// +/// let queue = OwnedQueue::new_ordered(c"my_queue")?; +/// let item = Arc::pin_init(pin_init!(MyStruct { +/// value: 42, +/// work <- new_work!("MyStruct::work"), +/// }), GFP_KERNEL)?; +/// let _ = queue.enqueue(item); +/// // Dropping `queue` waits for the work item to finish. +/// # Ok::<(), Error>(()) +/// ``` +/// +/// [`drain`]: OwnedQueue::drain +pub struct OwnedQueue(NonNull); + +impl OwnedQueue { + /// Allocates an ordered workqueue named `name`. + /// + /// The work items of an ordered workqueue are executed one at a time, in the order in which + /// they were queued, by an unbound worker. The name is copied by the kernel and appears in the + /// worker's thread name. + pub fn new_ordered(name: &CStr) -> Result { + // SAFETY: `name` is a valid NUL-terminated string; the C side copies it. + let ptr = unsafe { bindings::alloc_ordered_workqueue(name.as_char_ptr(), 0) }; + + // INVARIANT: `alloc_ordered_workqueue()` returns a valid workqueue or NULL. + Ok(Self(NonNull::new(ptr).ok_or(ENOMEM)?)) + } + + /// Waits until the workqueue is empty. + /// + /// Work items that are pending or running when this is called may still enqueue further work + /// on this queue and that work is waited for too, but anything else that enqueues on a + /// draining queue is refused (with a warning) and its work item is not run. Callers therefore + /// stop every other producer before draining. Must be called from a context that may sleep. + pub fn drain(&self) { + // SAFETY: The pointer is valid by the type invariant. + unsafe { bindings::drain_workqueue(self.0.as_ptr()) }; + } +} + +impl Deref for OwnedQueue { + type Target = Queue; + + fn deref(&self) -> &Queue { + // SAFETY: The pointer is valid by the type invariant and stays valid for as long as `self` + // does, which bounds the returned reference. + unsafe { Queue::from_raw(self.0.as_ptr()) } + } +} + +impl Drop for OwnedQueue { + fn drop(&mut self) { + // SAFETY: The pointer is valid by the type invariant. `destroy_workqueue()` drains the + // queue before freeing it, so no work item can run on a freed queue. + unsafe { bindings::destroy_workqueue(self.0.as_ptr()) }; + } +} + +// SAFETY: The workqueue API is thread-safe; an `OwnedQueue` only holds a pointer to the C object. +unsafe impl Send for OwnedQueue {} + +// SAFETY: The workqueue API is thread-safe; an `OwnedQueue` only holds a pointer to the C object. +unsafe impl Sync for OwnedQueue {} + /// A raw work item. /// /// This is the low-level trait that is designed for being as general as possible. From 10416b088c44d66ee3c740fbfa616fedcd6c96f4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:04:58 +1000 Subject: [PATCH 0427/2680] rust: soc: apple: aop: Require Send + Sync listeners The AOP driver stores every fake-HID listener as an Arc in a list that its RTKit receive worker walks, while the child driver that registered the listener keeps its own reference and reads the same object from the IIO read path. FakehidListener has no Send or Sync bound, so the driver's list entry carries an unsafe impl Send with no argument for why the trait object may cross threads. Make Send + Sync supertraits of FakehidListener, and of the sensors' MessageProcessor, which is the only non-trivially-Sync field of the type that implements it. The driver's unsafe impl then goes away: an Arc is Send and Sync on its own. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 2 -- rust/kernel/iio/common/aop_sensors.rs | 9 ++++++--- rust/kernel/soc/apple/aop.rs | 7 +++++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 29aea83dbd384..a57940369989d 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -625,8 +625,6 @@ struct ListenerEntry { listener: Arc, } -unsafe impl Send for ListenerEntry {} - #[pin_data] struct AopData { dev: ARef, diff --git a/rust/kernel/iio/common/aop_sensors.rs b/rust/kernel/iio/common/aop_sensors.rs index 3161174d9c9a2..1e294ccd89dd6 100644 --- a/rust/kernel/iio/common/aop_sensors.rs +++ b/rust/kernel/iio/common/aop_sensors.rs @@ -21,9 +21,12 @@ use kernel::{ ThisModule, // }; -/// TODO: add documentation -pub trait MessageProcessor { - /// TODO: add documentation +/// Decodes one fake-HID report of a sensor into the value the IIO channel exposes. +/// +/// It is called from the AOP driver's receive worker while the IIO device may be read from +/// another thread, hence `Send + Sync`. +pub trait MessageProcessor: Send + Sync { + /// Returns the channel value carried by `message`. fn process(&self, message: &[u8]) -> u32; } diff --git a/rust/kernel/soc/apple/aop.rs b/rust/kernel/soc/apple/aop.rs index da46ce0bcb027..aa15e13f845da 100644 --- a/rust/kernel/soc/apple/aop.rs +++ b/rust/kernel/soc/apple/aop.rs @@ -14,8 +14,11 @@ pub struct EPICService { pub endpoint: u8, } -/// Listener for the "HID" events sent by aop -pub trait FakehidListener { +/// Listener for the "HID" events sent by aop. +/// +/// The AOP driver keeps the listener in an [`Arc`] shared with the child driver that registered +/// it and invokes it from its own RTKit receive worker, so listeners have to be `Send + Sync`. +pub trait FakehidListener: Send + Sync { /// Process the event. fn process_fakehid_report(&self, data: &[u8]) -> Result<()>; } From 5652dc0cdcb75db07996c959095fe2fb27876dfb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:08:14 +1000 Subject: [PATCH 0428/2680] soc: apple: aop: Fix the reply tag bound check A reply's tag selects the slot of the call it answers. The check that guards the slot lookup allows tag == AOP_MAX_CALLS + 1, and the lookup then indexes one past the end of the calls array. Tags come from the firmware, so a stray reply would take the RTKit receive worker down. Reject tags above AOP_MAX_CALLS. Fixes: dc97e135ec26 ("soc: apple: Add support for the AOP co-processor") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index a57940369989d..650150f16d889 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -469,7 +469,7 @@ impl AFKEndpoint { } let retcode = u32::from_ne_bytes(data[..4].try_into().unwrap()); let tag = ehdr.tag as usize; - if tag == 0 || tag - 1 > self.calls.len() || self.calls[tag - 1].is_none() { + if tag == 0 || tag > self.calls.len() || self.calls[tag - 1].is_none() { dev_err!( client.dev, "Got a retcode with invalid tag {:?} on endpoint {}", From a92df872440d1a48e5a8faa80fe81f1fa691fcd1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:09:35 +1000 Subject: [PATCH 0429/2680] soc: apple: aop: Check message lengths before decoding headers The receive path splits every ring entry at the size of the EPIC header without checking that the entry is that long, and it reads a service announcement through a struct pointer without checking that the payload holds the struct. Both lengths come from the firmware; a short entry panics in split_at() and a short announcement reads past the payload. Check both lengths and report a short message instead. While at it, decode the header fields and the announcement from the byte slices, so that no pointer casts remain in the receive path. The announcement is a NUL-padded 32-byte name followed by the channel at byte 32; the old struct split the name at 20 bytes, which made no difference for the names the firmware uses but is not the layout of the message. Fixes: dc97e135ec26 ("soc: apple: Add support for the AOP co-processor") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 90 ++++++++++++++++++++++++++++------------ 1 file changed, 64 insertions(+), 26 deletions(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 650150f16d889..7c4d77af897d4 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -104,6 +104,16 @@ fn align_up(v: usize, a: usize) -> usize { (v + a - 1) & !(a - 1) } +/// Reads the little-endian `u16` at `off`; `b` must hold it. +fn le_u16(b: &[u8], off: usize) -> u16 { + u16::from_le_bytes([b[off], b[off + 1]]) +} + +/// Reads the little-endian `u32` at `off`; `b` must hold it. +fn le_u32(b: &[u8], off: usize) -> u32 { + u32::from_le_bytes([b[off], b[off + 1], b[off + 2], b[off + 3]]) +} + #[inline(always)] fn mem_sync() { unsafe { @@ -139,17 +149,37 @@ struct EPICHeader { inline_len: u32, } -#[repr(C, packed)] -struct EPICServiceAnnounce { - name: [u8; 20], - _unk0: u32, - retcode: u32, - _unk1: u32, - channel: u32, - _unk2: u32, - _unk3: u32, +/// The EPIC header fields that message dispatch needs. +struct EPICHeaderFields { + category: u8, + subtype: u16, + tag: u16, } +impl EPICHeaderFields { + /// Splits `msg` into its decoded header and its payload, or returns `None` + /// when `msg` is too short to hold a header. + fn decode(msg: &[u8]) -> Option<(EPICHeaderFields, &[u8])> { + if msg.len() < mem::size_of::() { + return None; + } + let (hdr, data) = msg.split_at(mem::size_of::()); + let fields = EPICHeaderFields { + category: hdr[mem::offset_of!(EPICHeader, category)], + subtype: le_u16(hdr, mem::offset_of!(EPICHeader, subtype)), + tag: le_u16(hdr, mem::offset_of!(EPICHeader, tag)), + }; + Some((fields, data)) + } +} + +/// A service announcement (report subtype [`EPIC_SUBTYPE_STD_SERVICE`]): +/// a NUL-padded 32-byte service name followed by the channel the service +/// answers on. Its trailing 8 bytes are not used here. +const EPIC_ANNOUNCE_NAME_LEN: usize = 32; +const EPIC_ANNOUNCE_CHANNEL_OFFSET: usize = 32; +const EPIC_ANNOUNCE_LEN: usize = 44; + #[pin_data] struct FutureValue { #[pin] @@ -410,9 +440,16 @@ impl AFKEndpoint { } msg_buf.resize(qeh.size as usize, 0, GFP_KERNEL)?; self.memcpy_from_iomem(base + rptr + QEH_SIZE, &mut msg_buf)?; - let (hdr_bytes, msg) = msg_buf.split_at(mem::size_of::()); - let header = unsafe { &*(hdr_bytes.as_ptr() as *const EPICHeader) }; - self.handle_ipc(client, qeh, header, msg)?; + let Some((header, msg)) = EPICHeaderFields::decode(&msg_buf) else { + dev_err!( + client.dev, + "Short EPIC message ({} bytes) on ep {}", + msg_buf.len(), + self.index + ); + return Err(EIO); + }; + self.handle_ipc(client, qeh, &header, msg)?; rptr = align_up(rptr + QEH_SIZE + qeh.size as usize, block_size) % buf_size; mem_sync(); self.iomem_write32(buf_offset + block_size, rptr as u32)?; @@ -425,24 +462,25 @@ impl AFKEndpoint { &mut self, client: ArcBorrow<'_, AopData>, qhdr: &QEHeader, - ehdr: &EPICHeader, + ehdr: &EPICHeaderFields, data: &[u8], ) -> Result<()> { let subtype = ehdr.subtype; if ehdr.category == EPIC_CATEGORY_REPORT { if subtype == EPIC_SUBTYPE_STD_SERVICE { - let announce = unsafe { &*(data.as_ptr() as *const EPICServiceAnnounce) }; - let chan = announce.channel; - let name_len = announce - .name - .iter() - .position(|x| *x == 0) - .unwrap_or(announce.name.len()); - return Into::>::into(client).register_service( - self, - chan, - &announce.name[..name_len], - ); + if data.len() < EPIC_ANNOUNCE_LEN { + dev_err!( + client.dev, + "Short service announcement ({} bytes) on endpoint {}", + data.len(), + self.index + ); + return Err(EIO); + } + let name = &data[..EPIC_ANNOUNCE_NAME_LEN]; + let name = &name[..name.iter().position(|x| *x == 0).unwrap_or(name.len())]; + let chan = le_u32(data, EPIC_ANNOUNCE_CHANNEL_OFFSET); + return Into::>::into(client).register_service(self, chan, name); } else if subtype == EPIC_SUBTYPE_FAKEHID_REPORT { return client.process_fakehid_report(self, qhdr.channel, data); } else { @@ -467,7 +505,7 @@ impl AFKEndpoint { ); return Err(EIO); } - let retcode = u32::from_ne_bytes(data[..4].try_into().unwrap()); + let retcode = le_u32(data, 0); let tag = ehdr.tag as usize; if tag == 0 || tag > self.calls.len() || self.calls[tag - 1].is_none() { dev_err!( From 27ea498cfdecdd68e774ee8a0a8c43bc82fcedf6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:11:33 +1000 Subject: [PATCH 0430/2680] soc: apple: aop: Bounds-check ring accesses and access pointers volatile The ring buffer accessors index the endpoint's shared buffer with offsets and sizes that come from the firmware: the ring geometry from the InitRX/InitTX messages, the entry sizes from the ring itself. Since the switch from CoherentAllocation::as_slice() to plain slicing, an offset outside the allocation is no longer reported as an error but panics in the RTKit receive worker, as does a ring whose stated buffer size exceeds the ring size or whose entry size exceeds the room left before the read pointer. Route every access through one bounds-checked helper and return EIO for anything outside the allocation. Validate the ring geometry when it is announced: the header blocks have to fit, the block size has to be a nonzero power of two because the entry alignment relies on it, and the buffer size has to be nonzero. Check the wrap arithmetic in the receive and send paths for underflow. The read and write pointer words are written by one side and polled by the other at any time, so a plain slice over them does not meet Coherent::as_ref()'s condition that the device stays away from the memory. Access them with volatile loads and stores, the kernel's READ_ONCE()/WRITE_ONCE(), and copy entries with raw pointer copies whose comments state the ring rule that makes them race-free: an entry is only read between the read and the write pointer and only written between the write and the read pointer. Fixes: 7a7c13773b68 ("fixup! soc: apple: Add support for the AOP co-processor") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 89 ++++++++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 26 deletions(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 7c4d77af897d4..7a37c94cdf3b9 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -317,53 +317,89 @@ impl AFKEndpoint { Ok(()) } + /// Decodes an InitRX/InitTX message: a ring of `size` bytes at `offset` + /// in the shared buffer, laid out as three header blocks (buffer size, + /// read pointer, write pointer) followed by `buf_size` bytes of entries. fn parse_ring_buf(&self, msg: u64) -> Result { let msg = msg as usize; let size = ((msg >> 16) & 0xFFFF) * AFK_RB_BLOCK_STEP; let offset = ((msg >> 32) & 0xFFFF) * AFK_RB_BLOCK_STEP; + let iomem_size = self.iomem.as_ref().ok_or(ENXIO)?.size(); let buf_size = self.iomem_read32(offset)? as usize; - let block_size = (size - buf_size) / 3; + let block_size = size.checked_sub(buf_size).ok_or(EIO)? / 3; + let end = offset.checked_add(size).ok_or(EIO)?; + if end > iomem_size || buf_size == 0 || block_size == 0 || !block_size.is_power_of_two() { + return Err(EIO); + } Ok(AFKRingBuffer { offset, block_size, buf_size, }) } + + /// Returns a pointer to `len` bytes at `off` in the shared buffer after + /// checking that they lie inside it. + fn iomem_ptr(&self, off: usize, len: usize) -> Result<*mut u8> { + let iomem = self.iomem.as_ref().ok_or(ENXIO)?; + let end = off.checked_add(len).ok_or(EIO)?; + if end > iomem.size() { + return Err(EIO); + } + // SAFETY: `off + len` does not exceed the size of the allocation, so + // the offset pointer stays inside it. + Ok(unsafe { iomem.as_mut_ptr().cast::().add(off) }) + } + + /// Writes one of the ring's pointer words. The firmware polls the word + /// concurrently, so the write is volatile. fn iomem_write32(&mut self, off: usize, data: u32) -> Result<()> { - let size = core::mem::size_of::(); - let data = data.to_le_bytes(); - let iomem = &self.iomem.as_mut().ok_or(ENXIO)?; - let buf = unsafe { &mut iomem.as_mut()[off..off + size] }; - buf.copy_from_slice(&data); + if off % mem::align_of::() != 0 { + return Err(EIO); + } + let ptr = self.iomem_ptr(off, mem::size_of::())?; + // SAFETY: `ptr` is valid for four bytes and aligned for a `u32`. A + // volatile write is the kernel's WRITE_ONCE() for a word the device + // reads at any time. + unsafe { ptr.cast::().write_volatile(data.to_le()) }; Ok(()) } + /// Reads one of the ring's pointer words. The firmware updates the word + /// concurrently, so the read is volatile. fn iomem_read32(&self, off: usize) -> Result { - let size = core::mem::size_of::(); - let iomem = &self.iomem.as_ref().ok_or(ENXIO)?; - let buf = unsafe { &iomem.as_ref()[off..off + size] }; - Ok(u32::from_le_bytes(buf.try_into().unwrap())) + if off % mem::align_of::() != 0 { + return Err(EIO); + } + let ptr = self.iomem_ptr(off, mem::size_of::())?; + // SAFETY: `ptr` is valid for four bytes and aligned for a `u32`. A + // volatile read is the kernel's READ_ONCE() for a word the device + // writes at any time. + Ok(u32::from_le(unsafe { ptr.cast::().read_volatile() })) } + /// Copies a ring entry out of the shared buffer. Callers only read + /// entries between the read and the write pointer, which the firmware + /// has finished writing and does not touch again until the read pointer + /// passes them. fn memcpy_from_iomem(&self, off: usize, target: &mut [u8]) -> Result<()> { - let iomem = &self.iomem.as_ref().ok_or(ENXIO)?; - // SAFETY: - // as_slice() checks that off and target.len() are whithin iomem's limits. - unsafe { - let src = &iomem.as_ref()[off..off + target.len()]; - target.copy_from_slice(src); - } + let src = self.iomem_ptr(off, target.len())?; + // SAFETY: `src` is valid for `target.len()` bytes, `target` is a + // distinct allocation, and by the ring protocol (see above) the device + // does not write the entry while it is copied. + unsafe { ptr::copy_nonoverlapping(src, target.as_mut_ptr(), target.len()) }; Ok(()) } + /// Copies a ring entry into the shared buffer. Callers only write between + /// the write and the read pointer, which the firmware does not read until + /// the write pointer is advanced past the entry. fn memcpy_to_iomem(&mut self, off: usize, src: &[u8]) -> Result<()> { - let iomem = &self.iomem.as_mut().ok_or(ENXIO)?; - // SAFETY: - // as_slice_mut() checks that off and src.len() are whithin iomem's limits. - unsafe { - let target = &mut iomem.as_mut()[off..off + src.len()]; - target.copy_from_slice(src); - } + let dst = self.iomem_ptr(off, src.len())?; + // SAFETY: `dst` is valid for `src.len()` bytes, `src` is a distinct + // allocation, and by the ring protocol (see above) the device does + // not access the entry while it is written. + unsafe { ptr::copy_nonoverlapping(src.as_ptr(), dst, src.len()) }; Ok(()) } @@ -422,7 +458,8 @@ impl AFKEndpoint { ); return Err(EIO); } - if qeh.size as usize > (buf_size - rptr - QEH_SIZE) { + let room = buf_size.checked_sub(rptr + QEH_SIZE).ok_or(EIO)?; + if qeh.size as usize > room { rptr = 0; self.memcpy_from_iomem(base + rptr, &mut qeh_bytes)?; qeh = unsafe { &*(qeh_bytes.as_ptr() as *const QEHeader) }; @@ -592,7 +629,7 @@ impl AFKEndpoint { ) }; self.memcpy_to_iomem(base + wptr, qeh_bytes)?; - if payload_len > buf_size - wptr - QEH_SIZE { + if payload_len > buf_size.checked_sub(wptr + QEH_SIZE).ok_or(EIO)? { wptr = 0; self.memcpy_to_iomem(base + wptr, qeh_bytes)?; } From 1587f1982d0c4a99cca5afeb231f146dbd1ca616 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:12:12 +1000 Subject: [PATCH 0431/2680] soc: apple: aop: Check the endpoint range in the RTKit receive callback The RTKit receive callback subtracts AFK_ENDPOINT_START from the endpoint number and indexes the endpoint table with the result without checking either step. Overflow checks are enabled in this configuration, so a message on an endpoint below 0x20 panics in the subtraction and one on an endpoint above 0x2e panics in the indexing. The endpoint number comes from the mailbox. Look the endpoint up through one checked conversion and ignore messages on endpoints the driver does not drive, with a warning. Use the same conversion for the endpoint of a service in the call paths, so that a malformed service record returns EINVAL instead of panicking. Fixes: dc97e135ec26 ("soc: apple: Add support for the AOP co-processor") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 7a37c94cdf3b9..85d6989847c65 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -104,6 +104,14 @@ fn align_up(v: usize, a: usize) -> usize { (v + a - 1) & !(a - 1) } +/// Index into the AFK endpoint tables for RTKit endpoint `ep`, if it is one +/// of the endpoints this driver drives. +fn afk_endpoint_index(ep: u8) -> Option { + ep.checked_sub(AFK_ENDPOINT_START) + .filter(|i| *i < AFK_ENDPOINT_COUNT) + .map(usize::from) +} + /// Reads the little-endian `u16` at `off`; `b` must hold it. fn le_u16(b: &[u8], off: usize) -> u16 { u16::from_le_bytes([b[off], b[off + 1]]) @@ -915,11 +923,11 @@ impl AopData { impl AOP for AopData { fn epic_call(&self, svc: &EPICService, subtype: u16, msg_bytes: &[u8]) -> Result { - let ep_idx = svc.endpoint - AFK_ENDPOINT_START; + let ep_idx = afk_endpoint_index(svc.endpoint).ok_or(EINVAL)?; let call = { let mut rtk_guard = self.rtkit.lock(); let mut rtk = rtk_guard.as_mut().as_pin_mut().unwrap(); - let mut ep_guard = self.endpoints[ep_idx as usize].lock(); + let mut ep_guard = self.endpoints[ep_idx].lock(); ep_guard.epic_notify(self, rtk.as_mut(), svc.channel, subtype, msg_bytes, None)? }; Ok(call.wait().retcode) @@ -931,11 +939,11 @@ impl AOP for AopData { msg_bytes: &[u8], ret_len: usize, ) -> Result<(u32, KVec)> { - let ep_idx = svc.endpoint - AFK_ENDPOINT_START; + let ep_idx = afk_endpoint_index(svc.endpoint).ok_or(EINVAL)?; let call = { let mut rtk_guard = self.rtkit.lock(); let mut rtk = rtk_guard.as_mut().as_pin_mut().unwrap(); - let mut ep_guard = self.endpoints[ep_idx as usize].lock(); + let mut ep_guard = self.endpoints[ep_idx].lock(); let mut ret_buf = KVec::new(); ret_buf.resize(ret_len, 0, GFP_KERNEL)?; ep_guard.epic_notify( @@ -998,9 +1006,18 @@ impl rtkit::Operations for AopData { type Buffer = NoBuffer; fn recv_message(data: ::Borrowed<'_>, ep: u8, msg: u64) { + let Some(index) = afk_endpoint_index(ep) else { + dev_warn!( + data.dev, + "Message {:#x} on unexpected endpoint {:#04x}", + msg, + ep + ); + return; + }; let mut guard = data.rtkit.lock(); let mut rtk = guard.as_mut().as_pin_mut().unwrap(); - let mut ep_guard = data.endpoints[(ep - AFK_ENDPOINT_START) as usize].lock(); + let mut ep_guard = data.endpoints[index].lock(); let ret = ep_guard.recv_message(data, rtk.as_mut(), msg); if let Err(e) = ret { dev_err!(data.dev, "Failed to handle rtkit message, error: {:?}", e); From b4dc168ca841e3d7029729a7975e3a390be36f84 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:12:35 +1000 Subject: [PATCH 0432/2680] soc: apple: aop: Keep the child fwnode referenced until it is registered The service registration work looks up the child firmware node by name, which returns a counted reference, and passes the raw pointer to platform_device_register_full(). The reference is dropped inside the map() closure that extracts the pointer, before the registration runs, so the platform device is created from a node the driver no longer holds. It only works because the parent's own node keeps its children alive. Keep the reference until platform_device_register_full() has returned; the registration takes a reference of its own on success. Fixes: dc97e135ec26 ("soc: apple: Add support for the AOP co-processor") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 85d6989847c65..db2704a0c665a 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -759,6 +759,8 @@ impl WorkItem for AopServiceRegisterWork { type Pointer = Pin>; fn run(this: Pin>) { + // Held until the device has been registered: the registration takes + // its own reference on the node only then. let fwnode = this .data .dev @@ -773,12 +775,16 @@ impl WorkItem for AopServiceRegisterWork { data: &this.service as *const EPICService as *const _, size_data: mem::size_of::(), dma_mask: 0, - fwnode: fwnode.map(|x| x.as_raw()).unwrap_or(ptr::null_mut()), + fwnode: fwnode + .as_ref() + .map(|x| x.as_raw()) + .unwrap_or(ptr::null_mut()), swnode: ptr::null_mut(), properties: ptr::null_mut(), of_node_reused: false, }; let pdev = unsafe { from_err_ptr(bindings::platform_device_register_full(&info)) }; + drop(fwnode); match pdev { Err(e) => { dev_err!( From 281cb4564edbf3bb8ca83ee3a6770db96208b31d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:14:32 +1000 Subject: [PATCH 0433/2680] soc: apple: aop: Serialize service registration with removal Service devices are registered from work items on the system workqueue, one per service announcement, and remove() walks the list of registered children without waiting for that work. A registration that is still queued or running when the AOP is removed registers its child after the list was walked, so the child is never unregistered and outlives the parent's driver data that its probe reads. The work item also pushes the new device onto the list only after registering it and merely logs when the push fails, which leaks a registered device the same way. Run the registrations on a workqueue owned by the driver, and have remove() set a removal flag under a gate that register_service() takes while queueing, then drain the queue. After the drain the list of children is complete. Reserve the list slot before registering the device so that a registered device is always tracked, and unregister it on the spot if the push fails after all rather than leaking it. Wrap the child pointers in a type whose only operation is to unregister the device once. With that, and with Send + Sync listeners, AopData is Send and Sync by construction and its unsafe impls go, as does the one on AFKEndpoint, which never had a field that needed it. Fixes: dc97e135ec26 ("soc: apple: Add support for the AOP co-processor") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 96 +++++++++++++++++++++++++++++++--------- 1 file changed, 76 insertions(+), 20 deletions(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index db2704a0c665a..649bceaa0e9f9 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -10,7 +10,7 @@ use core::{ cmp, marker::Unpin, mem, - ptr, + ptr::{self, NonNull}, slice, // }; @@ -51,6 +51,11 @@ use kernel::{ }, sync::{ aref::ARef, + atomic::{ + Acquire, + Atomic, + Release, // + }, Arc, ArcBorrow, CondVar, @@ -58,9 +63,9 @@ use kernel::{ }, types::ForeignOwnable, workqueue::{ - self, impl_has_work, new_work, + OwnedQueue, Work, WorkItem, // }, // @@ -242,8 +247,6 @@ struct AFKEndpoint { call_returns: [Option>; AOP_MAX_CALLS], } -unsafe impl Send for AFKEndpoint {} - impl AFKEndpoint { fn new(index: u8) -> AFKEndpoint { AFKEndpoint { @@ -708,9 +711,33 @@ struct ListenerEntry { listener: Arc, } +/// A service platform device registered by this driver. It is unregistered +/// explicitly by [`AopData::remove`]; nothing else touches the pointer. +struct ChildDevice(NonNull); + +// SAFETY: The only operation on the pointer is `platform_device_unregister()`, +// which may be called from any thread. +unsafe impl Send for ChildDevice {} + +impl ChildDevice { + fn unregister(self) { + // SAFETY: The pointer came from a successful + // `platform_device_register_full()` and is unregistered exactly once, + // here, because this consumes `self`. + unsafe { bindings::platform_device_unregister(self.0.as_ptr()) }; + } +} + #[pin_data] struct AopData { dev: ARef, + /// Runs the service registrations one at a time; drained on removal. + registration_queue: OwnedQueue, + /// Serializes queueing a registration with the start of removal, so that + /// nothing is queued once the queue drains. Never held across a drain. + #[pin] + registration_gate: Mutex<()>, + removing: Atomic, #[pin] rtkit: Mutex>>, #[pin] @@ -720,12 +747,9 @@ struct AopData { #[pin] hid_listeners: Mutex>, #[pin] - subdevices: Mutex>, + subdevices: Mutex>, } -unsafe impl Send for AopData {} -unsafe impl Sync for AopData {} - #[pin_data] struct AopServiceRegisterWork { name: &'static CStr, @@ -783,9 +807,24 @@ impl WorkItem for AopServiceRegisterWork { properties: ptr::null_mut(), of_node_reused: false, }; + // The slot is reserved before the device exists, so that a device + // that was registered is always tracked and gets unregistered. The + // child's probe runs inside the registration and calls back into the + // transport, but nothing on that path takes this lock. + let mut subdevices = this.data.subdevices.lock(); + if subdevices.reserve(1, GFP_KERNEL).is_err() { + dev_err!( + this.data.dev, + "Failed to allocate the device slot for service {:?}", + this.name + ); + return; + } + // SAFETY: `info` is a valid, fully initialized `platform_device_info` + // whose pointers outlive the call. let pdev = unsafe { from_err_ptr(bindings::platform_device_register_full(&info)) }; drop(fwnode); - match pdev { + match pdev.and_then(|pdev| NonNull::new(pdev).ok_or(EINVAL)) { Err(e) => { dev_err!( this.data.dev, @@ -795,9 +834,10 @@ impl WorkItem for AopServiceRegisterWork { ); } Ok(pdev) => { - let res = this.data.subdevices.lock().push(pdev, GFP_KERNEL); - if res.is_err() { - dev_err!(this.data.dev, "Failed to store subdevice"); + if let Err(child) = subdevices.push_within_capacity(ChildDevice(pdev)) { + // Cannot happen after the reservation above; never leave + // a registered device untracked. + child.0.unregister(); } } } @@ -806,10 +846,14 @@ impl WorkItem for AopServiceRegisterWork { impl AopData { fn new(dev: &platform::Device>) -> Result> { + let registration_queue = OwnedQueue::new_ordered(c_str!("apple-aop"))?; Arc::pin_init( pin_init!( AopData { dev: dev.as_ref().into(), + registration_queue, + registration_gate <- new_mutex!(()), + removing: Atomic::new(false), rtkit <- new_mutex!(None), endpoints <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(AFKEndpoint::new(AFK_ENDPOINT_START + i as u8)) @@ -859,9 +903,16 @@ impl AopData { return Ok(()); } }; - // probe can call back into us, run it with locks dropped. - let work = AopServiceRegisterWork::new(dev_name, self, svc)?; - workqueue::system().enqueue(work); + // The child's probe calls back into the transport, so it runs from a + // work item with the endpoint lock dropped. The gate orders queueing + // it against removal, which drains the queue after setting the flag. + let gate = self.registration_gate.lock(); + if self.removing.load(Acquire) { + return Ok(()); + } + let work = AopServiceRegisterWork::new(dev_name, self.clone(), svc)?; + self.registration_queue.enqueue(work); + drop(gate); Ok(()) } @@ -983,15 +1034,20 @@ impl AOP for AopData { false } fn remove(&self) { + { + let _gate = self.registration_gate.lock(); + self.removing.store(true, Release); + } + // No registration is queued after this point, so once the queue is + // empty the list of children is complete. + self.registration_queue.drain(); if let Err(e) = self.stop() { dev_err!(self.dev, "Failed to stop AOP {:?}", e); } self.rtkit.lock().as_mut().set(None); - let guard = self.subdevices.lock(); - for pdev in &*guard { - unsafe { - bindings::platform_device_unregister(*pdev); - } + let children = mem::replace(&mut *self.subdevices.lock(), KVec::new()); + for child in children { + child.unregister(); } } } From 890df2cada3ff95f682749c05202848d3b730163 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:16:43 +1000 Subject: [PATCH 0434/2680] soc: apple: aop: Bound the EPIC call wait A caller of epic_call() waits for the reply in TASK_UNINTERRUPTIBLE with no timeout. If the firmware never answers, which happens when it has crashed or when a request concerns a fabric the AP has powered off, the caller is stuck in D state for good, and so is anything that then waits for that task, such as an unbind of the child driver. Wait at most five seconds, using CondVar::wait_timeout() with the remaining time carried across spurious wakeups, and return ETIMEDOUT. The wait stays uninterruptible on purpose: a signal must not abandon a transaction whose reply is about to arrive. A reply carries the tag of its call, and the tag is the index of the call's slot. Freeing the slot on timeout would let the next call reuse the tag, and a reply that arrives late would then complete that call with the wrong result. Instead, a timed-out call leaves its slot marked as abandoned; the slot is reserved until the reply arrives and is dropped. A doorbell that fails after the entry was written to the ring is handled the same way, since the next doorbell delivers the entry. Merge the two arrays that tracked a call and its return buffer into one slot enum, and replace the unwrap() of the return buffer in epic_call_ret() with an error. Fixes: dc97e135ec26 ("soc: apple: Add support for the AOP co-processor") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 167 +++++++++++++++++++++++++++++---------- 1 file changed, 124 insertions(+), 43 deletions(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 649bceaa0e9f9..5f943f99f13c8 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -59,8 +59,10 @@ use kernel::{ Arc, ArcBorrow, CondVar, + CondVarTimeoutResult, Mutex, // }, + time::msecs_to_jiffies, types::ForeignOwnable, workqueue::{ impl_has_work, @@ -104,6 +106,9 @@ const EPIC_SUBTYPE_RETCODE_PAYLOAD: u16 = 0xa0; const EPIC_SUBTYPE_STRING: u16 = 0x8a; const QE_MAGIC1: u32 = from_fourcc(b" POI"); const QE_MAGIC2: u32 = from_fourcc(b" POA"); +/// Bound on the wait for the reply to an EPIC call. The firmware can stop +/// answering, and a caller must not be left in D state forever. +const EPIC_CALL_TIMEOUT_MS: u32 = 5000; fn align_up(v: usize, a: usize) -> usize { (v + a - 1) & !(a - 1) @@ -221,6 +226,23 @@ impl FutureValue { } ret_guard.take().unwrap() } + /// Waits at most `timeout_ms` for the value; `None` if it did not arrive. + /// + /// The wait is uninterruptible: a firmware transaction must not be + /// abandoned because the calling task has a signal pending, or its reply + /// would arrive for a call that no longer exists. + fn wait_timeout(&self, timeout_ms: u32) -> Option { + let mut ret_guard = self.val.lock(); + let mut left = msecs_to_jiffies(timeout_ms); + while ret_guard.is_none() { + match self.completion.wait_timeout(&mut ret_guard, left) { + CondVarTimeoutResult::Timeout => break, + CondVarTimeoutResult::Woken { jiffies } + | CondVarTimeoutResult::Signal { jiffies } => left = jiffies, + } + } + ret_guard.take() + } fn reset(&self) { *self.val.lock() = None; } @@ -237,14 +259,24 @@ struct CallResult { extra_data: Option>, } +/// An EPIC call in flight on an endpoint. The slot index plus one is the tag +/// the firmware echoes in the reply. +enum CallSlot { + /// The caller waits for the reply; the buffer receives the reply payload. + Pending(Arc>, Option>), + /// The caller gave up waiting. The slot stays reserved, so that its tag is + /// not handed to a later call that the late reply would then complete, + /// until that reply arrives and is dropped. + Abandoned, +} + struct AFKEndpoint { index: u8, iomem: Option>, txbuf: Option, rxbuf: Option, seq: u16, - calls: [Option>>; AOP_MAX_CALLS], - call_returns: [Option>; AOP_MAX_CALLS], + calls: [Option; AOP_MAX_CALLS], } impl AFKEndpoint { @@ -256,7 +288,6 @@ impl AFKEndpoint { rxbuf: None, seq: 0, calls: [const { None }; AOP_MAX_CALLS], - call_returns: [const { None }; AOP_MAX_CALLS], } } @@ -555,7 +586,7 @@ impl AFKEndpoint { } let retcode = le_u32(data, 0); let tag = ehdr.tag as usize; - if tag == 0 || tag > self.calls.len() || self.calls[tag - 1].is_none() { + if tag == 0 || tag > self.calls.len() { dev_err!( client.dev, "Got a retcode with invalid tag {:?} on endpoint {}", @@ -564,15 +595,35 @@ impl AFKEndpoint { ); return Err(EIO); } - let future = self.calls[tag - 1].take().unwrap(); - let extra_data = if let Some(mut ret) = self.call_returns[tag - 1].take() { + let (future, ret) = match self.calls[tag - 1].take() { + Some(CallSlot::Pending(future, ret)) => (future, ret), + Some(CallSlot::Abandoned) => { + // The late reply to a call that timed out; its slot + // is free again. + dev_warn!( + client.dev, + "Late reply for tag {} on endpoint {}", + tag, + self.index + ); + return Ok(()); + } + None => { + dev_err!( + client.dev, + "Got a retcode with no call in flight (tag {}) on endpoint {}", + tag, + self.index + ); + return Err(EIO); + } + }; + let extra_data = ret.map(|mut ret| { let len = cmp::min(data.len() - 4, ret.len()); ret[..len].copy_from_slice(&data[4..(len + 4)]); ret.truncate(len); - Some(ret) - } else { - None - }; + ret + }); future.complete(CallResult { retcode, extra_data, @@ -597,15 +648,17 @@ impl AFKEndpoint { ); Err(EIO) } - fn send_rb( + /// Writes one entry into the transmit ring and returns the doorbell + /// message that announces it. Nothing is visible to the firmware until + /// that message is sent. + fn write_entry( &mut self, client: &AopData, - rtkit: Pin<&mut rtkit::RtKit>, channel: u32, ty: u32, header: &[u8], data: &[u8], - ) -> Result<()> { + ) -> Result { let (buf_offset, block_size, buf_size) = match self.txbuf.as_ref() { Some(b) => (b.offset, b.block_size, b.buf_size), None => { @@ -648,8 +701,7 @@ impl AFKEndpoint { self.memcpy_to_iomem(base + wptr + QEH_SIZE + header.len(), data)?; wptr = align_up(wptr + QEH_SIZE + payload_len, block_size) % buf_size; self.iomem_write32(buf_offset + block_size * 2, wptr as u32)?; - let msg = wptr as u64 | (AFK_OPC_SEND << 48); - rtkit.send_message(self.index, msg) + Ok(wptr as u64 | (AFK_OPC_SEND << 48)) } fn epic_notify( &mut self, @@ -660,21 +712,14 @@ impl AFKEndpoint { data: &[u8], ret: Option>, ) -> Result>> { - let mut tag = 0; - for i in 0..self.calls.len() { - if self.calls[i].is_none() { - tag = i + 1; - break; - } - } - if tag == 0 { + let Some(slot) = self.calls.iter().position(|c| c.is_none()) else { dev_err!( client.dev, "Too many inflight calls on endpoint {}", self.index ); return Err(EIO); - } + }; let call = Arc::pin_init(FutureValue::pin_init(), GFP_KERNEL)?; let hdr = EPICHeader { version: 2, @@ -683,27 +728,39 @@ impl AFKEndpoint { sub_version: 2, category: EPIC_CATEGORY_NOTIFY, subtype, - tag: tag as u16, + tag: (slot + 1) as u16, ..EPICHeader::default() }; - self.call_returns[tag - 1] = ret; - self.send_rb( - client, - rtkit, - channel, - EPIC_TYPE_NOTIFY, - unsafe { - slice::from_raw_parts( - &hdr as *const EPICHeader as *const u8, - mem::size_of::(), - ) - }, - data, - )?; + // SAFETY: `hdr` is a packed plain-data struct that outlives the slice, + // which covers exactly its bytes. + let hdr_bytes = unsafe { + slice::from_raw_parts( + &hdr as *const EPICHeader as *const u8, + mem::size_of::(), + ) + }; + let doorbell = self.write_entry(client, channel, EPIC_TYPE_NOTIFY, hdr_bytes, data)?; self.seq = self.seq.wrapping_add(1); - self.calls[tag - 1] = Some(call.clone()); + self.calls[slot] = Some(CallSlot::Pending(call.clone(), ret)); + if let Err(e) = rtkit.send_message(self.index, doorbell) { + // The entry is in the ring and the next doorbell delivers it, so + // its reply still arrives; keep the tag reserved until it does. + self.calls[slot] = Some(CallSlot::Abandoned); + return Err(e); + } Ok(call) } + /// Gives up on `call`: its slot stays reserved until the reply arrives. + fn abandon_call(&mut self, call: &Arc>) { + for slot in self.calls.iter_mut() { + if let Some(CallSlot::Pending(pending, _)) = slot { + if Arc::ptr_eq(pending, call) { + *slot = Some(CallSlot::Abandoned); + return; + } + } + } + } } struct ListenerEntry { @@ -978,6 +1035,30 @@ impl AopData { } } +impl AopData { + /// Waits a bounded time for the reply to `call` on endpoint `ep_idx`. + fn wait_call( + &self, + ep_idx: usize, + svc: &EPICService, + subtype: u16, + call: Arc>, + ) -> Result { + if let Some(res) = call.wait_timeout(EPIC_CALL_TIMEOUT_MS) { + return Ok(res); + } + self.endpoints[ep_idx].lock().abandon_call(&call); + dev_err!( + self.dev, + "EPIC call {:#x} on channel {} timed out after {} ms", + subtype, + svc.channel, + EPIC_CALL_TIMEOUT_MS + ); + Err(ETIMEDOUT) + } +} + impl AOP for AopData { fn epic_call(&self, svc: &EPICService, subtype: u16, msg_bytes: &[u8]) -> Result { let ep_idx = afk_endpoint_index(svc.endpoint).ok_or(EINVAL)?; @@ -987,7 +1068,7 @@ impl AOP for AopData { let mut ep_guard = self.endpoints[ep_idx].lock(); ep_guard.epic_notify(self, rtk.as_mut(), svc.channel, subtype, msg_bytes, None)? }; - Ok(call.wait().retcode) + Ok(self.wait_call(ep_idx, svc, subtype, call)?.retcode) } fn epic_call_ret( &self, @@ -1012,8 +1093,8 @@ impl AOP for AopData { Some(ret_buf), )? }; - let res = call.wait(); - Ok((res.retcode, res.extra_data.unwrap())) + let res = self.wait_call(ep_idx, svc, subtype, call)?; + Ok((res.retcode, res.extra_data.ok_or(EIO)?)) } fn add_fakehid_listener( &self, From 24eb80ffc7851c452f58192a9025c26849ff983f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:17:42 +1000 Subject: [PATCH 0435/2680] soc: apple: aop: Bound the endpoint shutdown wait stop() sends a shutdown to every endpoint the firmware advertises and waits for the acknowledgment without a timeout. Two things go wrong with that. An endpoint whose AFK handshake was never started, because start() failed part way through, is shut down anyway and never answers, so the removal hangs. And a firmware that has stopped answering leaves the unbind in D state for good, which is exactly when one would want to unbind. One shared completion serves all endpoints, too, so a late acknowledgment from one endpoint completes the wait for the next. Record which endpoints were started and shut down only those. Give each endpoint its own completion, wait at most five seconds for it, and on a timeout warn, remember the error and carry on with the remaining endpoints, so that stop() always returns. Also return ENODEV instead of unwrapping a missing RTKit handle in start() and stop(). Fixes: dc97e135ec26 ("soc: apple: Add support for the AOP co-processor") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 70 +++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 5f943f99f13c8..15f427b7c744d 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -109,6 +109,8 @@ const QE_MAGIC2: u32 = from_fourcc(b" POA"); /// Bound on the wait for the reply to an EPIC call. The firmware can stop /// answering, and a caller must not be left in D state forever. const EPIC_CALL_TIMEOUT_MS: u32 = 5000; +/// Bound on the wait for an endpoint's shutdown acknowledgment. +const AFK_SHUTDOWN_TIMEOUT_MS: u32 = 5000; fn align_up(v: usize, a: usize) -> usize { (v + a - 1) & !(a - 1) @@ -219,13 +221,6 @@ impl FutureValue { *self.val.lock() = Some(val); self.completion.notify_all(); } - fn wait(&self) -> T { - let mut ret_guard = self.val.lock(); - while ret_guard.is_none() { - self.completion.wait(&mut ret_guard); - } - ret_guard.take().unwrap() - } /// Waits at most `timeout_ms` for the value; `None` if it did not arrive. /// /// The wait is uninterruptible: a firmware transaction must not be @@ -272,6 +267,8 @@ enum CallSlot { struct AFKEndpoint { index: u8, + /// The AFK handshake was started; only such endpoints are shut down. + started: bool, iomem: Option>, txbuf: Option, rxbuf: Option, @@ -283,6 +280,7 @@ impl AFKEndpoint { fn new(index: u8) -> AFKEndpoint { AFKEndpoint { index, + started: false, iomem: None, txbuf: None, rxbuf: None, @@ -347,7 +345,7 @@ impl AFKEndpoint { self.recv_rb(client)?; } AFK_OPC_SHUTDOWN_ACK => { - client.shutdown_complete(); + client.shutdown_complete(self.index); } _ => dev_err!( client.dev, @@ -800,7 +798,7 @@ struct AopData { #[pin] endpoints: [Mutex; AFK_ENDPOINT_COUNT as usize], #[pin] - ep_shutdown: FutureValue<()>, + ep_shutdown: [FutureValue<()>; AFK_ENDPOINT_COUNT as usize], #[pin] hid_listeners: Mutex>, #[pin] @@ -915,7 +913,7 @@ impl AopData { endpoints <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(AFKEndpoint::new(AFK_ENDPOINT_START + i as u8)) }), - ep_shutdown <- FutureValue::pin_init(), + ep_shutdown <- pin_init::pin_init_array_from_fn(|_| FutureValue::pin_init()), hid_listeners <- new_mutex!(KVec::new()), subdevices <- new_mutex!(KVec::new()), } @@ -926,19 +924,20 @@ impl AopData { fn start(&self) -> Result<()> { { let mut guard = self.rtkit.lock(); - let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + let mut rtk = guard.as_mut().as_pin_mut().ok_or(ENODEV)?; rtk.as_mut().wake()?; } - for ep in 0..AFK_ENDPOINT_COUNT { - let rtk_ep_num = AFK_ENDPOINT_START + ep; + for ep in 0..AFK_ENDPOINT_COUNT as usize { + let rtk_ep_num = AFK_ENDPOINT_START + ep as u8; let mut guard = self.rtkit.lock(); - let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + let mut rtk = guard.as_mut().as_pin_mut().ok_or(ENODEV)?; if !rtk.as_mut().has_endpoint(rtk_ep_num) { continue; } rtk.as_mut().start_endpoint(rtk_ep_num)?; - let ep_guard = self.endpoints[ep as usize].lock(); + let mut ep_guard = self.endpoints[ep].lock(); ep_guard.start(rtk.as_mut())?; + ep_guard.started = true; } Ok(()) } @@ -983,26 +982,45 @@ impl AopData { Ok(()) } - fn shutdown_complete(&self) { - self.ep_shutdown.complete(()); + fn shutdown_complete(&self, endpoint: u8) { + if let Some(index) = afk_endpoint_index(endpoint) { + self.ep_shutdown[index].complete(()); + } } + /// Shuts down every started endpoint, waiting a bounded time for each + /// acknowledgment. Returns the first error but still tries the rest. fn stop(&self) -> Result<()> { - for ep in 0..AFK_ENDPOINT_COUNT { + let mut ret = Ok(()); + for ep in 0..AFK_ENDPOINT_COUNT as usize { { - let rtk_ep_num = AFK_ENDPOINT_START + ep; let mut guard = self.rtkit.lock(); - let mut rtk = guard.as_mut().as_pin_mut().unwrap(); - if !rtk.as_mut().has_endpoint(rtk_ep_num) { + let mut rtk = guard.as_mut().as_pin_mut().ok_or(ENODEV)?; + let mut ep_guard = self.endpoints[ep].lock(); + if !ep_guard.started { + continue; + } + // Whatever happens below, the endpoint is not started again. + ep_guard.started = false; + self.ep_shutdown[ep].reset(); + if let Err(e) = ep_guard.stop(rtk.as_mut()) { + ret = ret.and(Err(e)); continue; } - let ep_guard = self.endpoints[ep as usize].lock(); - ep_guard.stop(rtk.as_mut())?; } - self.ep_shutdown.wait(); - self.ep_shutdown.reset(); + if self.ep_shutdown[ep] + .wait_timeout(AFK_SHUTDOWN_TIMEOUT_MS) + .is_none() + { + dev_warn!( + self.dev, + "Endpoint {:#04x} did not acknowledge its shutdown", + AFK_ENDPOINT_START + ep as u8 + ); + ret = ret.and(Err(ETIMEDOUT)); + } } - Ok(()) + ret } fn patch_bootargs( From f8e83de83fd963f7d92113b81b16f2bc9a3eb156 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:19:37 +1000 Subject: [PATCH 0436/2680] soc: apple: aop: Tear down in unbind and unwind a failed probe The AOP is only taken down from Drop, which the driver core runs after the device is unbound, and nothing at all cleans up after a probe that fails once the co-processor runs. Several things follow from that: - The shared ring buffers are dma_alloc_coherent() allocations on the device; freeing them after unbind frees them on a device whose DMA configuration is already gone. - A probe that fails in start() returns with the CPU running, the RTKit handle dropped while endpoints are live, and service devices still being registered by the work items the announcements queued. Those children stay registered, and their probes read the parent's driver data, which never gets published. - The RTKit handle is dropped while the rtkit lock is held. Dropping it waits for the RTKit receive worker, and that worker takes the same lock to deliver a message, so a message arriving during removal deadlocks the unbind. - epic_call() and the receive callback unwrap the RTKit handle, which is None once removal has started while a child's call is in flight. Move the teardown to unbind, where the device is still bound, and make it idempotent so that Drop stays as a fallback. Run the same teardown from a scope guard when probe fails. Unbind the children first, while the transport still works and their unbind can talk to their services, then drop any listener a child left behind, mark the transport as closing so no further call is started, stop the endpoints, take the RTKit handle out of the shared state and drop it with the lock released, and unregister the children last. A missing handle is now an error (ENODEV) in the call paths and ends the receive callback. [ryan: adapted to 7.2.10 - unbind() takes Device>; the probe path uses the ASC_CPU_CONTROL register update already in asahi-wip instead of the removed start_cpu()] Fixes: dc97e135ec26 ("soc: apple: Add support for the AOP co-processor") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 66 +++++++++++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 7 deletions(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 15f427b7c744d..1f0cb00c05547 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -63,7 +63,10 @@ use kernel::{ Mutex, // }, time::msecs_to_jiffies, - types::ForeignOwnable, + types::{ + ForeignOwnable, + ScopeGuard, // + }, workqueue::{ impl_has_work, new_work, @@ -775,6 +778,14 @@ struct ChildDevice(NonNull); unsafe impl Send for ChildDevice {} impl ChildDevice { + /// Unbinds the child's driver while the child stays registered. + fn release_driver(&self) { + // SAFETY: The pointer came from a successful + // `platform_device_register_full()` and the device is still + // registered, so its embedded `struct device` is valid. + unsafe { bindings::device_release_driver(ptr::addr_of_mut!((*self.0.as_ptr()).dev)) }; + } + fn unregister(self) { // SAFETY: The pointer came from a successful // `platform_device_register_full()` and is unregistered exactly once, @@ -792,7 +803,10 @@ struct AopData { /// nothing is queued once the queue drains. Never held across a drain. #[pin] registration_gate: Mutex<()>, + /// Set once by the first teardown; later teardowns return at once. removing: Atomic, + /// Set when the transport starts closing; no call is started after it. + transport_closing: Atomic, #[pin] rtkit: Mutex>>, #[pin] @@ -909,6 +923,7 @@ impl AopData { registration_queue, registration_gate <- new_mutex!(()), removing: Atomic::new(false), + transport_closing: Atomic::new(false), rtkit <- new_mutex!(None), endpoints <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(AFKEndpoint::new(AFK_ENDPOINT_START + i as u8)) @@ -1079,10 +1094,13 @@ impl AopData { impl AOP for AopData { fn epic_call(&self, svc: &EPICService, subtype: u16, msg_bytes: &[u8]) -> Result { + if self.transport_closing.load(Acquire) { + return Err(ENODEV); + } let ep_idx = afk_endpoint_index(svc.endpoint).ok_or(EINVAL)?; let call = { let mut rtk_guard = self.rtkit.lock(); - let mut rtk = rtk_guard.as_mut().as_pin_mut().unwrap(); + let mut rtk = rtk_guard.as_mut().as_pin_mut().ok_or(ENODEV)?; let mut ep_guard = self.endpoints[ep_idx].lock(); ep_guard.epic_notify(self, rtk.as_mut(), svc.channel, subtype, msg_bytes, None)? }; @@ -1095,10 +1113,13 @@ impl AOP for AopData { msg_bytes: &[u8], ret_len: usize, ) -> Result<(u32, KVec)> { + if self.transport_closing.load(Acquire) { + return Err(ENODEV); + } let ep_idx = afk_endpoint_index(svc.endpoint).ok_or(EINVAL)?; let call = { let mut rtk_guard = self.rtkit.lock(); - let mut rtk = rtk_guard.as_mut().as_pin_mut().unwrap(); + let mut rtk = rtk_guard.as_mut().as_pin_mut().ok_or(ENODEV)?; let mut ep_guard = self.endpoints[ep_idx].lock(); let mut ret_buf = KVec::new(); ret_buf.resize(ret_len, 0, GFP_KERNEL)?; @@ -1120,6 +1141,9 @@ impl AOP for AopData { listener: Arc, ) -> Result<()> { let mut guard = self.hid_listeners.lock(); + if self.removing.load(Acquire) { + return Err(ENODEV); + } Ok(guard.push(ListenerEntry { svc, listener }, GFP_KERNEL)?) } fn remove_fakehid_listener(&self, svc: &EPICService) -> bool { @@ -1132,19 +1156,35 @@ impl AOP for AopData { } false } + /// Takes the AOP down: from unbind, from a failed probe, or as a fallback + /// from Drop. Only the first call does anything. fn remove(&self) { { let _gate = self.registration_gate.lock(); - self.removing.store(true, Release); + if self.removing.xchg(true, Acquire) { + return; + } } // No registration is queued after this point, so once the queue is // empty the list of children is complete. self.registration_queue.drain(); + // Unbind the children while the transport still works, so that their + // unbind can talk to their services. Then drop any listener a child + // left behind: no report may reach a driver that is going away. + let children = mem::take(&mut *self.subdevices.lock()); + for child in &children { + child.release_driver(); + } + self.hid_listeners.lock().clear(); + self.transport_closing.store(true, Release); if let Err(e) = self.stop() { dev_err!(self.dev, "Failed to stop AOP {:?}", e); } - self.rtkit.lock().as_mut().set(None); - let children = mem::replace(&mut *self.subdevices.lock(), KVec::new()); + // Take the handle out of the shared state before dropping it: the + // drop waits for the RTKit receive worker, which takes the same lock. + // After it, no callback runs and the device may be unbound. + let rtkit = self.rtkit.lock().take(); + drop(rtkit); for child in children { child.unregister(); } @@ -1177,7 +1217,9 @@ impl rtkit::Operations for AopData { return; }; let mut guard = data.rtkit.lock(); - let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + let Some(mut rtk) = guard.as_mut().as_pin_mut() else { + return; + }; let mut ep_guard = data.endpoints[index].lock(); let ret = ep_guard.recv_message(data, rtk.as_mut(), msg); if let Err(e) = ret { @@ -1250,6 +1292,9 @@ impl platform::Driver for AopDriver { let asc_mmio = asc_req.iomap_sized::()?; let asc_mmio = asc_mmio.as_view().relaxed(); let data = AopData::new(pdev)?; + // Whatever fails below leaves the AOP running and its children + // registering; the same teardown as unbind's cleans that up. + let probe_guard = ScopeGuard::new_with_data(data.clone(), |data| data.remove()); let aop_mmio = aop_mmio.as_view(); data.patch_bootargs( &aop_mmio, @@ -1264,9 +1309,16 @@ impl platform::Driver for AopDriver { data.rtkit.lock().as_mut().set(Some(rtkit)); asc_mmio.update(ASC_CPU_CONTROL, |r| r.with_const_cpu_run::<1>()); data.start()?; + probe_guard.dismiss(); let data = data as Arc; Ok(Self(data)) } + + fn unbind(_dev: &platform::Device>, this: Pin<&Self>) { + // The device is still bound here, which the DMA allocations the + // teardown frees require. Drop only repeats it as a no-op. + this.0.remove(); + } } impl Drop for AopDriver { From 442890f728e4734b24c800d60f1c927bc1a58be5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:20:09 +1000 Subject: [PATCH 0437/2680] soc: apple: aop: Keep the shared buffer before publishing its address recv_get_buf() allocates the endpoint's shared buffer, sends its address to the firmware and only then stores the allocation in the endpoint. If the send fails, the allocation is dropped on the error path and the memory returns to the allocator, although the message may already have reached the firmware, which then DMAs into memory the kernel reuses. Store the buffer first and send the address afterwards. A failed send then leaves the buffer owned by the endpoint until the driver is torn down, which is the conservative outcome. Also replace the placeholder safety comment on the bound-device conversion with the actual argument: the callback can only run while the RTKit handle exists, and the handle is dropped before the device is unbound. Fixes: dc97e135ec26 ("soc: apple: Add support for the AOP co-processor") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 1f0cb00c05547..b0d21de6c53ce 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -461,11 +461,19 @@ impl AFKEndpoint { ); return Err(EIO); } - // SAFETY: TODO + // SAFETY: This runs from the RTKit receive worker. The RTKit handle + // is dropped by `AopData::remove()`, which runs from unbind or from a + // failed probe while the device is still bound, and no callback runs + // once that drop has returned. So the device is bound for as long as + // this callback runs. let bound_dev = unsafe { dev.as_bound() }; let iomem = Coherent::::zeroed_slice(bound_dev, size, GFP_KERNEL)?; - rtkit.send_message(self.index, AFK_MSG_GET_BUF_ACK | iomem.dma_handle())?; + let iova = iomem.dma_handle(); + // Own the buffer before its address leaves the host: should the + // doorbell fail after the firmware has seen the message, the buffer + // must not go back to the allocator while the firmware uses it. self.iomem = Some(iomem); + rtkit.send_message(self.index, AFK_MSG_GET_BUF_ACK | iova)?; Ok(()) } From 1dfade1580d5d8abb0f04056bd1f8ceebe8952a8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:22:30 +1000 Subject: [PATCH 0438/2680] soc: apple: aop: Skip messages the endpoint cannot handle When a ring entry cannot be handled, because it is too short, carries an unknown category or subtype, or is a reply with no call in flight, the receive loop returns before it advances the read pointer. The entry is then read again on the next doorbell, with the same result, so one bad message stalls the endpoint for the rest of the boot and prints two error lines per doorbell. Skip such messages instead: log them and advance past them, since the ring position is still known. Only errors that make the ring position untrustworthy, an invalid entry magic or an entry outside the buffer, still end the loop. Throttle the log to the powers of two of a per-endpoint drop count, so that a stream of unhandled messages cannot flood it, and print the category, subtype, tag and channel of the dropped message together with the reason in that one line. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 115 ++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 62 deletions(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index b0d21de6c53ce..3663d75421b2a 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -25,6 +25,7 @@ use kernel::{ DmaMask, // }, error::from_err_ptr, + fmt, io::{ io_project, Io, @@ -277,6 +278,8 @@ struct AFKEndpoint { rxbuf: Option, seq: u16, calls: [Option; AOP_MAX_CALLS], + /// Messages this endpoint could not handle, for throttling their logging. + dropped: u32, } impl AFKEndpoint { @@ -289,6 +292,7 @@ impl AFKEndpoint { rxbuf: None, seq: 0, calls: [const { None }; AOP_MAX_CALLS], + dropped: 0, } } @@ -528,16 +532,30 @@ impl AFKEndpoint { } msg_buf.resize(qeh.size as usize, 0, GFP_KERNEL)?; self.memcpy_from_iomem(base + rptr + QEH_SIZE, &mut msg_buf)?; - let Some((header, msg)) = EPICHeaderFields::decode(&msg_buf) else { - dev_err!( - client.dev, - "Short EPIC message ({} bytes) on ep {}", - msg_buf.len(), - self.index - ); - return Err(EIO); - }; - self.handle_ipc(client, qeh, &header, msg)?; + // A message the endpoint cannot handle is skipped. The ring + // position is still good, and not advancing past the entry would + // read it again on every doorbell and stall the endpoint for good. + match EPICHeaderFields::decode(&msg_buf) { + None => { + self.drop_message(&client.dev, fmt!("short message ({} bytes)", msg_buf.len())) + } + Some((header, msg)) => { + if let Err(e) = self.handle_ipc(client, qeh, &header, msg) { + let channel = qeh.channel; + self.drop_message( + &client.dev, + fmt!( + "category {:#x} subtype {:#x} tag {} on channel {}: {:?}", + header.category, + header.subtype, + header.tag, + channel, + e + ), + ); + } + } + } rptr = align_up(rptr + QEH_SIZE + qeh.size as usize, block_size) % buf_size; mem_sync(); self.iomem_write32(buf_offset + block_size, rptr as u32)?; @@ -546,6 +564,23 @@ impl AFKEndpoint { } Ok(()) } + /// Counts a message the endpoint could not handle and logs it. The log + /// is throttled to the powers of two of the count, so that a stream of + /// such messages cannot flood it. + fn drop_message(&mut self, dev: &device::Device, why: fmt::Arguments<'_>) { + self.dropped = self.dropped.saturating_add(1); + if self.dropped.is_power_of_two() { + dev_warn!( + dev, + "Endpoint {:#04x} dropped a message: {} ({} so far)", + self.index, + why, + self.dropped + ); + } + } + /// Dispatches one message. An error means the message was not handled; + /// the caller logs it and skips the message. fn handle_ipc( &mut self, client: ArcBorrow<'_, AopData>, @@ -557,13 +592,7 @@ impl AFKEndpoint { if ehdr.category == EPIC_CATEGORY_REPORT { if subtype == EPIC_SUBTYPE_STD_SERVICE { if data.len() < EPIC_ANNOUNCE_LEN { - dev_err!( - client.dev, - "Short service announcement ({} bytes) on endpoint {}", - data.len(), - self.index - ); - return Err(EIO); + return Err(EMSGSIZE); } let name = &data[..EPIC_ANNOUNCE_NAME_LEN]; let name = &name[..name.iter().position(|x| *x == 0).unwrap_or(name.len())]; @@ -571,38 +600,20 @@ impl AFKEndpoint { return Into::>::into(client).register_service(self, chan, name); } else if subtype == EPIC_SUBTYPE_FAKEHID_REPORT { return client.process_fakehid_report(self, qhdr.channel, data); - } else { - dev_err!( - client.dev, - "Unexpected EPIC report subtype {:x} on endpoint {}", - subtype, - self.index - ); - return Err(EIO); } + return Err(EINVAL); } else if ehdr.category == EPIC_CATEGORY_REPLY { if subtype == EPIC_SUBTYPE_RETCODE_PAYLOAD || subtype == EPIC_SUBTYPE_RETCODE || subtype == EPIC_SUBTYPE_STRING { if data.len() < mem::size_of::() { - dev_err!( - client.dev, - "Retcode data too short on endpoint {}", - self.index - ); - return Err(EIO); + return Err(EMSGSIZE); } let retcode = le_u32(data, 0); let tag = ehdr.tag as usize; if tag == 0 || tag > self.calls.len() { - dev_err!( - client.dev, - "Got a retcode with invalid tag {:?} on endpoint {}", - tag, - self.index - ); - return Err(EIO); + return Err(EINVAL); } let (future, ret) = match self.calls[tag - 1].take() { Some(CallSlot::Pending(future, ret)) => (future, ret), @@ -617,15 +628,8 @@ impl AFKEndpoint { ); return Ok(()); } - None => { - dev_err!( - client.dev, - "Got a retcode with no call in flight (tag {}) on endpoint {}", - tag, - self.index - ); - return Err(EIO); - } + // A reply with no call in flight. + None => return Err(ENOENT), }; let extra_data = ret.map(|mut ret| { let len = cmp::min(data.len() - 4, ret.len()); @@ -639,23 +643,10 @@ impl AFKEndpoint { }); return Ok(()); - } else { - dev_err!( - client.dev, - "Unexpected EPIC reply subtype {:x} on endpoint {}", - subtype, - self.index - ); - return Err(EIO); } + return Err(EINVAL); } - dev_err!( - client.dev, - "Unexpected EPIC category {:x} on endpoint {}", - ehdr.category, - self.index - ); - Err(EIO) + Err(EINVAL) } /// Writes one entry into the transmit ring and returns the doorbell /// message that announces it. Nothing is visible to the firmware until From 2bb9bf604a82240c2797b088f5754ab007fc04c3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:24:05 +1000 Subject: [PATCH 0439/2680] iio: common: aop_sensors: Remove the fake-HID listener on unbind The ALS and lid-angle drivers register their sensor data as a fake-HID listener with the AOP core and never remove it. When the child driver is unbound or its module unloaded, the AOP keeps calling the listener for every report: the sensor data stays alive through the Arc the AOP holds, but its vtable points into module text that may be gone, and the IIO device it feeds has been unregistered. Add FakehidRegistration to the AOP client API: it owns one subscription and removes it on unregister() or drop. Removing a listener takes the same lock as report dispatch, so it returns only once no callback into the listener is running. Have both drivers hold a registration as their first field, unregister it from unbind while the IIO device still exists, and let drop take care of a probe that fails after subscribing. Since a service can now only be subscribed once per bound driver, the AOP core rejects a second listener for the same service with EBUSY instead of shadowing the first. [ryan: adapted to 7.2.10 - unbind() takes Device>] Fixes: 81556e2106e3 ("iio: common: Add AOP sensor drivers") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iio/common/aop_sensors/aop_als.rs | 28 +++++++----- drivers/iio/common/aop_sensors/aop_las.rs | 28 +++++++----- drivers/soc/apple/aop.rs | 3 ++ rust/kernel/soc/apple/aop.rs | 56 +++++++++++++++++++++-- 4 files changed, 90 insertions(+), 25 deletions(-) diff --git a/drivers/iio/common/aop_sensors/aop_als.rs b/drivers/iio/common/aop_sensors/aop_als.rs index 1f0424f358bb3..b7d8813f4eb29 100644 --- a/drivers/iio/common/aop_sensors/aop_als.rs +++ b/drivers/iio/common/aop_sensors/aop_als.rs @@ -11,7 +11,7 @@ use kernel::{ iio::common::aop_sensors::{AopSensorData, IIORegistration, MessageProcessor}, module_platform_driver, of, platform, prelude::*, - soc::apple::aop::{EPICService, AOP}, + soc::apple::aop::{EPICService, FakehidRegistration, AOP}, sync::Arc, types::ForeignOwnable, }; @@ -96,8 +96,11 @@ impl MessageProcessor for MsgProc { } } -#[repr(transparent)] -struct IIOAopAlsDriver(IIORegistration); +struct IIOAopAlsDriver { + /// Declared first: the subscription ends before the IIO device goes. + listener: FakehidRegistration, + _iio: IIORegistration, +} kernel::of_device_table!( OF_TABLE, @@ -126,16 +129,19 @@ impl platform::Driver for IIOAopAlsDriver { let ty = bindings::BINDINGS_IIO_LIGHT; let offset = get_lux_offset(adata.as_ref(), pdev, &service)?; let data = AopSensorData::new(dev.into(), ty, MsgProc(offset))?; - adata.add_fakehid_listener(service, data.clone())?; + let listener = FakehidRegistration::new(adata.clone(), service, data.clone())?; enable_als(adata.as_ref(), pdev, &service)?; let info_mask = 1 << bindings::BINDINGS_IIO_CHAN_INFO_PROCESSED; - Ok(IIOAopAlsDriver(IIORegistration::::new( - data, - c"aop-sensors-als", - ty, - info_mask, - &THIS_MODULE, - )?)) + let iio = + IIORegistration::::new(data, c"aop-sensors-als", ty, info_mask, &THIS_MODULE)?; + Ok(IIOAopAlsDriver { + listener, + _iio: iio, + }) + } + + fn unbind(_dev: &platform::Device>, this: Pin<&Self>) { + this.listener.unregister(); } } diff --git a/drivers/iio/common/aop_sensors/aop_las.rs b/drivers/iio/common/aop_sensors/aop_las.rs index d500d8441e1d5..4fb0db3372b9e 100644 --- a/drivers/iio/common/aop_sensors/aop_las.rs +++ b/drivers/iio/common/aop_sensors/aop_las.rs @@ -10,7 +10,7 @@ use kernel::{ iio::common::aop_sensors::{AopSensorData, IIORegistration, MessageProcessor}, module_platform_driver, of, platform, prelude::*, - soc::apple::aop::{EPICService, AOP}, + soc::apple::aop::{EPICService, FakehidRegistration, AOP}, sync::Arc, types::ForeignOwnable, }; @@ -23,8 +23,11 @@ impl MessageProcessor for MsgProc { } } -#[repr(transparent)] -struct IIOAopLasDriver(IIORegistration); +struct IIOAopLasDriver { + /// Declared first: the subscription ends before the IIO device goes. + listener: FakehidRegistration, + _iio: IIORegistration, +} kernel::of_device_table!( OF_TABLE, @@ -53,15 +56,18 @@ impl platform::Driver for IIOAopLasDriver { let ty = bindings::BINDINGS_IIO_ANGL; let data = AopSensorData::new(dev.into(), ty, MsgProc)?; - adata.add_fakehid_listener(service, data.clone())?; + let listener = FakehidRegistration::new(adata, service, data.clone())?; let info_mask = 1 << bindings::BINDINGS_IIO_CHAN_INFO_RAW; - Ok(IIOAopLasDriver(IIORegistration::::new( - data, - c"aop-sensors-las", - ty, - info_mask, - &THIS_MODULE, - )?)) + let iio = + IIORegistration::::new(data, c"aop-sensors-las", ty, info_mask, &THIS_MODULE)?; + Ok(IIOAopLasDriver { + listener, + _iio: iio, + }) + } + + fn unbind(_dev: &platform::Device>, this: Pin<&Self>) { + this.listener.unregister(); } } diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 3663d75421b2a..aebc5972b8d2f 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -1143,6 +1143,9 @@ impl AOP for AopData { if self.removing.load(Acquire) { return Err(ENODEV); } + if guard.iter().any(|entry| entry.svc == svc) { + return Err(EBUSY); + } Ok(guard.push(ListenerEntry { svc, listener }, GFP_KERNEL)?) } fn remove_fakehid_listener(&self, svc: &EPICService) -> bool { diff --git a/rust/kernel/soc/apple/aop.rs b/rust/kernel/soc/apple/aop.rs index aa15e13f845da..f6267e371a550 100644 --- a/rust/kernel/soc/apple/aop.rs +++ b/rust/kernel/soc/apple/aop.rs @@ -2,7 +2,13 @@ //! Common code for AOP endpoint drivers -use kernel::{prelude::*, sync::Arc}; +use kernel::{ + prelude::*, + sync::{ + atomic::{Atomic, Relaxed}, + Arc, // + }, +}; /// Representation of an "EPIC" service. #[derive(Clone, Copy, PartialEq, Eq)] @@ -36,18 +42,62 @@ pub trait AOP: Send + Sync { ret_len: usize, ) -> Result<(u32, KVec)>; - /// Adds the listener for the specified service + /// Adds the listener for the specified service. A service takes one listener at a time; + /// a second registration fails with `EBUSY`. fn add_fakehid_listener( &self, svc: EPICService, listener: Arc, ) -> Result<()>; - /// Remove the listener for the specified service + /// Removes the listener for the specified service. Returns once no callback into the + /// listener is running any more. fn remove_fakehid_listener(&self, svc: &EPICService) -> bool; /// Internal method to detach the device. fn remove(&self); } +/// A child driver's fake-HID subscription on an AOP service. +/// +/// The AOP core calls the listener from its receive worker for as long as the subscription +/// exists, so the child has to end it before it releases anything the listener uses. Call +/// [`FakehidRegistration::unregister`] from the driver's `unbind`, while those resources are +/// still alive; dropping the registration unregisters as well, as a fallback for a failed probe. +pub struct FakehidRegistration { + aop: Arc, + service: EPICService, + registered: Atomic, +} + +impl FakehidRegistration { + /// Subscribes `listener` to the fake-HID reports of `service`. + pub fn new( + aop: Arc, + service: EPICService, + listener: Arc, + ) -> Result { + aop.add_fakehid_listener(service, listener)?; + Ok(Self { + aop, + service, + registered: Atomic::new(true), + }) + } + + /// Ends the subscription. Returns once no callback into the listener is running any more; + /// repeated calls do nothing. + pub fn unregister(&self) { + if self.registered.xchg(false, Relaxed) { + self.aop.remove_fakehid_listener(&self.service); + } + } +} + +impl Drop for FakehidRegistration { + fn drop(&mut self) { + self.unregister(); + } +} + /// Converts a text representation of a FourCC to u32 pub const fn from_fourcc(b: &[u8]) -> u32 { b[3] as u32 | (b[2] as u32) << 8 | (b[1] as u32) << 16 | (b[0] as u32) << 24 From 20fa712c62eb19f5b9e8dae91bf1742e1a343c62 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:24:26 +1000 Subject: [PATCH 0440/2680] iio: common: aop_sensors: Check iio_device_alloc() for failure IIORegistration::new() writes the private data, name, channels and callbacks into the device returned by iio_device_alloc() without checking for NULL, so an allocation failure dereferences a null pointer. Return ENOMEM instead. The Drop impl already skips a null device, so the partially built registration unwinds cleanly. Fixes: 81556e2106e3 ("iio: common: Add AOP sensor drivers") Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/kernel/iio/common/aop_sensors.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rust/kernel/iio/common/aop_sensors.rs b/rust/kernel/iio/common/aop_sensors.rs index 1e294ccd89dd6..a772a6026116a 100644 --- a/rust/kernel/iio/common/aop_sensors.rs +++ b/rust/kernel/iio/common/aop_sensors.rs @@ -140,6 +140,9 @@ impl IIORegistration { _p: PhantomData, }; this.dev = unsafe { bindings::iio_device_alloc(data.dev.as_raw(), 0) }; + if this.dev.is_null() { + return Err(ENOMEM); + } unsafe { (*this.dev).priv_ = data.clone().into_foreign().cast(); (*this.dev).name = name.as_ptr() as _; From d63cf46186dd60de2f573f76941ddc64471f96f1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:25:51 +1000 Subject: [PATCH 0441/2680] iio: common: aop_sensors: Defer probing until the AOP has published A sensor's probe reads the AOP core's driver data from its parent device. The core registers the sensor devices from a workqueue while it is still probing, and the driver core publishes a driver's data only once its probe returns, so a sensor that probes early finds no data and borrows a null pointer as an Arc. Add AOP::from_child() to the client API: it returns the core behind a service device, or EPROBE_DEFER while the core has not published yet, in which case the driver core retries the probe once the core has bound. The lookup relies on the core's driver type being repr(transparent) over the Arc; the contract is now documented in both places instead of being assumed in every child. Also check that the service record the core attaches as platform data is present. Fixes: 81556e2106e3 ("iio: common: Add AOP sensor drivers") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iio/common/aop_sensors/aop_als.rs | 19 ++++++++------- drivers/iio/common/aop_sensors/aop_las.rs | 19 ++++++++------- drivers/soc/apple/aop.rs | 3 +++ rust/kernel/soc/apple/aop.rs | 29 +++++++++++++++++++++++ 4 files changed, 54 insertions(+), 16 deletions(-) diff --git a/drivers/iio/common/aop_sensors/aop_als.rs b/drivers/iio/common/aop_sensors/aop_als.rs index b7d8813f4eb29..4129e7defe05a 100644 --- a/drivers/iio/common/aop_sensors/aop_als.rs +++ b/drivers/iio/common/aop_sensors/aop_als.rs @@ -12,8 +12,6 @@ use kernel::{ module_platform_driver, of, platform, prelude::*, soc::apple::aop::{EPICService, FakehidRegistration, AOP}, - sync::Arc, - types::ForeignOwnable, }; const EPIC_SUBTYPE_GET_AOP_PROPERTY: u16 = 0xa; @@ -120,12 +118,17 @@ impl platform::Driver for IIOAopAlsDriver { _info: Option<&'bound ()>, ) -> impl PinInit, Error> + 'bound { let dev = pdev.as_ref(); - let parent = dev.parent().unwrap(); - // SAFETY: our parent is AOP, and AopDriver is repr(transparent) for Arc - let adata_ptr = unsafe { Pin::>>::borrow(parent.get_drvdata()) }; - let adata = (&*adata_ptr).clone(); - // SAFETY: AOP sets the platform data correctly - let service = unsafe { *((*dev.as_raw()).platform_data as *const EPICService) }; + // SAFETY: This driver binds only to the service devices the AOP core + // registers, and `pdev` is being probed. + let adata = unsafe { ::from_child(dev) }?; + // SAFETY: `dev` is a valid device; the AOP core attached its service + // record as the platform data of every service device it registers. + let service = unsafe { (*dev.as_raw()).platform_data.cast::() }; + if service.is_null() { + return Err(ENODEV); + } + // SAFETY: The record lives as long as the device it is attached to. + let service = unsafe { *service }; let ty = bindings::BINDINGS_IIO_LIGHT; let offset = get_lux_offset(adata.as_ref(), pdev, &service)?; let data = AopSensorData::new(dev.into(), ty, MsgProc(offset))?; diff --git a/drivers/iio/common/aop_sensors/aop_las.rs b/drivers/iio/common/aop_sensors/aop_las.rs index 4fb0db3372b9e..1f93e23d580c5 100644 --- a/drivers/iio/common/aop_sensors/aop_las.rs +++ b/drivers/iio/common/aop_sensors/aop_las.rs @@ -11,8 +11,6 @@ use kernel::{ module_platform_driver, of, platform, prelude::*, soc::apple::aop::{EPICService, FakehidRegistration, AOP}, - sync::Arc, - types::ForeignOwnable, }; struct MsgProc; @@ -47,12 +45,17 @@ impl platform::Driver for IIOAopLasDriver { _info: Option<&'bound ()>, ) -> impl PinInit, Error> + 'bound { let dev = pdev.as_ref(); - let parent = dev.parent().unwrap(); - // SAFETY: our parent is AOP, and AopDriver is repr(transparent) for Arc - let adata_ptr = unsafe { Pin::>>::borrow(parent.get_drvdata()) }; - let adata = (&*adata_ptr).clone(); - // SAFETY: AOP sets the platform data correctly - let service = unsafe { *((*dev.as_raw()).platform_data as *const EPICService) }; + // SAFETY: This driver binds only to the service devices the AOP core + // registers, and `pdev` is being probed. + let adata = unsafe { ::from_child(dev) }?; + // SAFETY: `dev` is a valid device; the AOP core attached its service + // record as the platform data of every service device it registers. + let service = unsafe { (*dev.as_raw()).platform_data.cast::() }; + if service.is_null() { + return Err(ENODEV); + } + // SAFETY: The record lives as long as the device it is attached to. + let service = unsafe { *service }; let ty = bindings::BINDINGS_IIO_ANGL; let data = AopSensorData::new(dev.into(), ty, MsgProc)?; diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index aebc5972b8d2f..4c7b621d3e197 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -1234,6 +1234,9 @@ impl rtkit::Operations for AopData { } } +/// The driver data of the AOP. Children reach the `Arc` through +/// `AOP::from_child()`, which relies on this being `repr(transparent)` over +/// it. #[repr(transparent)] struct AopDriver(Arc); diff --git a/rust/kernel/soc/apple/aop.rs b/rust/kernel/soc/apple/aop.rs index f6267e371a550..0de0d9acf6372 100644 --- a/rust/kernel/soc/apple/aop.rs +++ b/rust/kernel/soc/apple/aop.rs @@ -3,11 +3,13 @@ //! Common code for AOP endpoint drivers use kernel::{ + device::Device, prelude::*, sync::{ atomic::{Atomic, Relaxed}, Arc, // }, + types::ForeignOwnable, }; /// Representation of an "EPIC" service. @@ -56,6 +58,33 @@ pub trait AOP: Send + Sync { fn remove(&self); } +impl dyn AOP { + /// Returns the AOP core that registered the service device `child`. + /// + /// The core registers its service devices from a workqueue while it is still probing, and the + /// driver core publishes a driver's data only once its probe has returned, so a child can be + /// probed before the core's data exists. This returns `EPROBE_DEFER` in that case; the child's + /// probe is retried once the core has bound. + /// + /// # Safety + /// + /// `child` must be a platform device that the AOP core driver registered for one of its + /// services, and the caller must be probing it or be bound to it. The core unregisters its + /// children before it releases its driver data, so that data is valid for the lookup. + pub unsafe fn from_child(child: &Device) -> Result> { + let parent = child.parent().ok_or(ENODEV)?; + let ptr = parent.get_drvdata::(); + if ptr.is_null() { + return Err(EPROBE_DEFER); + } + // SAFETY: By this function's contract `ptr` is the AOP core's driver data: a pinned box + // holding the core's driver type, which is `#[repr(transparent)]` over an `Arc`, + // and it stays valid while `child` is registered. + let aop = unsafe { Pin::>>::borrow(ptr) }; + Ok((*aop).clone()) + } +} + /// A child driver's fake-HID subscription on an AOP service. /// /// The AOP core calls the listener from its receive worker for as long as the subscription From 0f381ae07a115abd112b6b3126fef89c2a6a86a0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:27:36 +1000 Subject: [PATCH 0442/2680] iio: common: aop_sensors: Report readings with micro resolution The processed illuminance channel is reported as IIO_VAL_INT after the lux value, which the sensor delivers as a single-precision float, has been truncated to an integer. Anything below one lux reads as zero, which in a dim room is the whole range of interest for auto-brightness. A report too short to hold the value also stores a fake zero, and the lid-angle decoder indexes its report without checking its length. Have the message processors return the reading in micro-units, or None when the report carries none, and store it as such. read_raw() then reports processed values as IIO_VAL_INT_PLUS_MICRO and raw values as IIO_VAL_INT. Readings are capped so that both parts fit the i32 values IIO passes around; a report without a reading leaves the previous one in place. The float conversion is done in integer arithmetic, since floating point is not available in the kernel: negative values clamp to zero, NaN and the infinities carry no reading. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iio/common/aop_sensors/aop_als.rs | 48 ++++++++++--------- drivers/iio/common/aop_sensors/aop_las.rs | 6 +-- rust/kernel/iio/common/aop_sensors.rs | 57 ++++++++++++++++------- 3 files changed, 68 insertions(+), 43 deletions(-) diff --git a/drivers/iio/common/aop_sensors/aop_als.rs b/drivers/iio/common/aop_sensors/aop_als.rs index 4129e7defe05a..2711da61f51e3 100644 --- a/drivers/iio/common/aop_sensors/aop_als.rs +++ b/drivers/iio/common/aop_sensors/aop_als.rs @@ -8,7 +8,7 @@ use kernel::{ bindings, c_str, device::Core, firmware::Firmware, - iio::common::aop_sensors::{AopSensorData, IIORegistration, MessageProcessor}, + iio::common::aop_sensors::{AopSensorData, IIORegistration, MessageProcessor, MICRO}, module_platform_driver, of, platform, prelude::*, soc::apple::aop::{EPICService, FakehidRegistration, AOP}, @@ -63,34 +63,38 @@ fn set_als_property(aop: &dyn AOP, svc: &EPICService, tag: u32, data: &[u8]) -> aop.epic_call(svc, EPIC_SUBTYPE_SET_ALS_PROPERTY, &buf) } -fn f32_to_u32(f: u32) -> u32 { - if f & 0x80000000 != 0 { - return 0; +/// Converts the bits of an IEEE 754 single to microlux, rounding down. +/// Negative values clamp to zero; NaN and the infinities carry no reading. +/// Values beyond `MAX_MICRO` saturate when they are stored. +fn f32_to_micro(f: u32) -> Option { + let exp = ((f >> 23) & 0xff) as i32; + if exp == 0xff { + return None; } - let exp = ((f & 0x7f800000) >> 23) as i32 - 127; - if exp < 0 { - return 0; + if f & 0x8000_0000 != 0 || exp == 0 { + return Some(0); } - if exp == 128 && f & 0x7fffff != 0 { - return 0; - } - let mant = f & 0x7fffff | 0x800000; - if exp <= 23 { - return mant >> (23 - exp); - } - if exp >= 32 { - return u32::MAX; - } - mant << (exp - 23) + // The value is mant * 2^shift; scaled to microlux it is below 2^44 + // before the shift, so a left shift of up to 19 cannot overflow. + let mant = u64::from(f & 0x7f_ffff | 0x80_0000) * MICRO; + let shift = exp - 127 - 23; + Some(if shift >= 20 { + u64::MAX + } else if shift >= 0 { + mant << shift + } else if shift <= -64 { + 0 + } else { + mant >> -shift + }) } struct MsgProc(usize); impl MessageProcessor for MsgProc { - fn process(&self, message: &[u8]) -> u32 { - let offset = self.0; - let raw = u32::from_le_bytes(message[offset..offset + 4].try_into().unwrap()); - f32_to_u32(raw) + fn process(&self, message: &[u8]) -> Option { + let raw = message.get(self.0..self.0 + 4)?; + f32_to_micro(u32::from_le_bytes([raw[0], raw[1], raw[2], raw[3]])) } } diff --git a/drivers/iio/common/aop_sensors/aop_las.rs b/drivers/iio/common/aop_sensors/aop_las.rs index 1f93e23d580c5..b33d28fcfa1f7 100644 --- a/drivers/iio/common/aop_sensors/aop_las.rs +++ b/drivers/iio/common/aop_sensors/aop_las.rs @@ -7,7 +7,7 @@ use kernel::{ bindings, c_str, device::Core, - iio::common::aop_sensors::{AopSensorData, IIORegistration, MessageProcessor}, + iio::common::aop_sensors::{AopSensorData, IIORegistration, MessageProcessor, MICRO}, module_platform_driver, of, platform, prelude::*, soc::apple::aop::{EPICService, FakehidRegistration, AOP}, @@ -16,8 +16,8 @@ use kernel::{ struct MsgProc; impl MessageProcessor for MsgProc { - fn process(&self, message: &[u8]) -> u32 { - message[1] as u32 + fn process(&self, message: &[u8]) -> Option { + message.get(1).map(|angle| u64::from(*angle) * MICRO) } } diff --git a/rust/kernel/iio/common/aop_sensors.rs b/rust/kernel/iio/common/aop_sensors.rs index a772a6026116a..6a6ec04b8315d 100644 --- a/rust/kernel/iio/common/aop_sensors.rs +++ b/rust/kernel/iio/common/aop_sensors.rs @@ -6,7 +6,6 @@ use core::marker::{PhantomData, PhantomPinned}; use core::ptr; -use core::sync::atomic::{AtomicU32, Ordering}; use kernel::{ bindings, @@ -15,37 +14,47 @@ use kernel::{ soc::apple::aop::FakehidListener, sync::{ aref::ARef, + atomic::{Atomic, Relaxed}, Arc, // }, types::ForeignOwnable, ThisModule, // }; +/// Micro-units per unit of a channel value. +pub const MICRO: u64 = 1_000_000; + +/// The largest reading that can be stored: both the integer part and the micro part have to fit +/// the `i32` values IIO reports. Larger readings saturate. +pub const MAX_MICRO: u64 = i32::MAX as u64 * MICRO + (MICRO - 1); + /// Decodes one fake-HID report of a sensor into the value the IIO channel exposes. /// /// It is called from the AOP driver's receive worker while the IIO device may be read from /// another thread, hence `Send + Sync`. pub trait MessageProcessor: Send + Sync { - /// Returns the channel value carried by `message`. - fn process(&self, message: &[u8]) -> u32; + /// Returns the reading carried by `message` in micro-units of the channel (millionths of a + /// lux, of a degree, ...), or `None` when the message carries no reading. + fn process(&self, message: &[u8]) -> Option; } -/// TODO: add documentation +/// The state shared between a sensor's report listener and its IIO device: the latest reading. pub struct AopSensorData { dev: ARef, ty: u32, - value: AtomicU32, + /// The latest reading in micro-units, at most [`MAX_MICRO`]. + value: Atomic, msg_proc: T, } impl AopSensorData { - /// TODO: add documentation + /// Creates the state for one channel of type `ty` whose reports `msg_proc` decodes. pub fn new(dev: ARef, ty: u32, msg_proc: T) -> Result>> { Ok(Arc::new( AopSensorData { dev, ty, - value: AtomicU32::new(0), + value: Atomic::new(0), msg_proc, }, GFP_KERNEL, @@ -55,34 +64,46 @@ impl AopSensorData { impl FakehidListener for AopSensorData { fn process_fakehid_report(&self, data: &[u8]) -> Result<()> { - self.value - .store(self.msg_proc.process(data), Ordering::Relaxed); + // A report without a reading leaves the last reading in place. + if let Some(value) = self.msg_proc.process(data) { + self.value.store(value.min(MAX_MICRO), Relaxed); + } Ok(()) } } +/// The `read_raw` callback: processed values are reported with their micro part, raw values as +/// integers. unsafe extern "C" fn aop_read_raw( dev: *mut bindings::iio_dev, chan: *const bindings::iio_chan_spec, val: *mut i32, - _: *mut i32, + val2: *mut i32, mask: isize, ) -> i32 { + // SAFETY: The IIO core calls this with the device and channel that `IIORegistration::new()` + // registered; `priv_` is the `Arc` it stored there. let data = unsafe { Arc::>::borrow((*dev).priv_.cast()) }; + // SAFETY: `chan` is one of the registered channel specs. let ty = unsafe { (*chan).type_ }; - if mask != bindings::BINDINGS_IIO_CHAN_INFO_PROCESSED as isize - && mask != bindings::BINDINGS_IIO_CHAN_INFO_RAW as isize - { - return EINVAL.to_errno(); - } if data.ty != ty { return EINVAL.to_errno(); } - let value = data.value.load(Ordering::Relaxed); + let micro = data.value.load(Relaxed); + // Both parts fit an `i32` because stored readings never exceed `MAX_MICRO`. + let (int, frac) = ((micro / MICRO) as i32, (micro % MICRO) as i32); + // SAFETY: `val` and `val2` point to the caller's result variables. unsafe { - *val = value as i32; + *val = int; + *val2 = frac; + } + if mask == bindings::BINDINGS_IIO_CHAN_INFO_PROCESSED as isize { + bindings::IIO_VAL_INT_PLUS_MICRO as i32 + } else if mask == bindings::BINDINGS_IIO_CHAN_INFO_RAW as isize { + bindings::IIO_VAL_INT as i32 + } else { + EINVAL.to_errno() } - bindings::IIO_VAL_INT as i32 } struct IIOSpec { From da0195e3c7b1ed33feaa35abaa59265dc19145ea Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:28:07 +1000 Subject: [PATCH 0443/2680] iio: common: aop_sensors: Return ENODATA until the first reading A sensor's channel reads as zero from the moment the IIO device is registered until the first report arrives, and stays at zero if reports never come, as on an ambient light sensor whose calibration is missing and which the firmware therefore never starts. Userspace cannot tell that apart from darkness. Track whether a reading has arrived and return ENODATA from read_raw() until one has. The reading is published with release ordering and the flag read with acquire ordering, so a reader that sees the flag sees the reading. Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/kernel/iio/common/aop_sensors.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/rust/kernel/iio/common/aop_sensors.rs b/rust/kernel/iio/common/aop_sensors.rs index 6a6ec04b8315d..7c80cef56b92f 100644 --- a/rust/kernel/iio/common/aop_sensors.rs +++ b/rust/kernel/iio/common/aop_sensors.rs @@ -14,7 +14,12 @@ use kernel::{ soc::apple::aop::FakehidListener, sync::{ aref::ARef, - atomic::{Atomic, Relaxed}, + atomic::{ + Acquire, + Atomic, + Relaxed, + Release, // + }, Arc, // }, types::ForeignOwnable, @@ -44,6 +49,8 @@ pub struct AopSensorData { ty: u32, /// The latest reading in micro-units, at most [`MAX_MICRO`]. value: Atomic, + /// Whether a reading has arrived at all; until then reads return `ENODATA`. + valid: Atomic, msg_proc: T, } @@ -55,6 +62,7 @@ impl AopSensorData { dev, ty, value: Atomic::new(0), + valid: Atomic::new(false), msg_proc, }, GFP_KERNEL, @@ -67,6 +75,8 @@ impl FakehidListener for AopSensorData { // A report without a reading leaves the last reading in place. if let Some(value) = self.msg_proc.process(data) { self.value.store(value.min(MAX_MICRO), Relaxed); + // Publishes the reading above to readers that see `valid`. + self.valid.store(true, Release); } Ok(()) } @@ -89,6 +99,11 @@ unsafe extern "C" fn aop_read_raw( if data.ty != ty { return EINVAL.to_errno(); } + // A sensor that has not reported yet, for instance because it lacks its + // calibration, has no reading rather than a reading of zero. + if !data.valid.load(Acquire) { + return ENODATA.to_errno(); + } let micro = data.value.load(Relaxed); // Both parts fit an `i32` because stored readings never exceed `MAX_MICRO`. let (int, frac) = ((micro / MICRO) as i32, (micro % MICRO) as i32); From dfea7b3deaa26e45ebc48041c116af418116dabe Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:30:53 +1000 Subject: [PATCH 0444/2680] dt-bindings: iio: light: apple,t8140-aop-als: Add firmware-name The T8140 AOP firmware does not start the ambient light sensor until the application processor has handed it the sensor's calibration over the AOP's setup-port mailbox: without it the sensor answers property reads and accepts its reporting interval, and then never reports. The calibration is an 80-byte setup-port message that the machine's own firmware sends the AOP; it is specific to the machine and is not part of the operating system, so the operating system has to load it from a file the machine's owner provides. Add an optional firmware-name property naming that file, so that the name is chosen per machine rather than hard-coded per SoC. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/iio/light/apple,t8140-aop-als.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml b/Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml index 5b6ad07fa6f84..5e919e8dc2b6c 100644 --- a/Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml +++ b/Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml @@ -15,6 +15,12 @@ description: application-processor MMIO or interrupt resource. Machine calibration is supplied separately through the firmware loader, not inline in this node. + The AOP firmware does not start the sensor until the application processor + has handed it the sensor's calibration over the AOP's setup-port mailbox. + The calibration is an 80-byte message that is specific to the machine and + is not part of the operating system; the machine's own firmware sends the + same message to the AOP. + select: properties: compatible: @@ -29,6 +35,13 @@ properties: - const: apple,t8140-aop-als - const: apple,aop-als + firmware-name: + maxItems: 1 + description: + Name of the file that holds the sensor's 80-byte setup-port calibration + message, to be loaded through the firmware loader and passed to the AOP + unchanged. Without it the sensor stays unstarted and reports nothing. + required: - compatible From 9e10b1100da5c653463121bf13a51c5fa407a117 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:31:58 +1000 Subject: [PATCH 0445/2680] soc: apple: aop: Add EPIC version 4 headers The T8140 AOP firmware speaks EPIC with version 4 sub-headers. The sub-header is the same size as the version 2 one but carries a 64-bit timestamp between the subtype and the tag, so the tag of a reply sits eight bytes further in. The firmware also expects a call to advertise the size of the reply it can take in the inline_len field: macOS sets it on every call it makes to this firmware, and while a property read happens to be answered with zero there, other calls are not guaranteed to be. Add the version 4 layout, select the tag offset by the sub-header version the message itself declares, and build version 4 headers, with inline_len set to the return buffer size, on firmware that a new epic_v4 flag in the match data marks as speaking it. No compatible sets the flag yet, and the header sent to the existing SoCs is unchanged. [ryan: adapted to 7.2.10 - AopData::new() keeps the Device> signature from asahi-wip] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 84 ++++++++++++++++++++++++++++++++++------ 1 file changed, 72 insertions(+), 12 deletions(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 4c7b621d3e197..7fb17dc7e5461 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -173,6 +173,30 @@ struct EPICHeader { inline_len: u32, } +/// The EPIC header of the T8140 firmware (sub-header version 4): the +/// sub-header carries a timestamp ahead of the tag, and the reply capacity +/// of a call goes into `inline_len`. Same size as the version 2 header. +#[repr(C, packed)] +#[derive(Clone, Copy, Default)] +struct EPICHeaderV4 { + version: u8, + seq: u16, + _pad0: u8, + _unk0: u32, + timestamp: u64, + // Subheader + length: u32, + sub_version: u8, + category: u8, + subtype: u16, + sub_timestamp: u64, + tag: u16, + _unk1: u16, + inline_len: u32, +} + +const _: () = assert!(mem::size_of::() == mem::size_of::()); + /// The EPIC header fields that message dispatch needs. struct EPICHeaderFields { category: u8, @@ -182,16 +206,22 @@ struct EPICHeaderFields { impl EPICHeaderFields { /// Splits `msg` into its decoded header and its payload, or returns `None` - /// when `msg` is too short to hold a header. + /// when `msg` is too short to hold a header. The sub-header version in the + /// message selects the layout; both layouts share the leading fields. fn decode(msg: &[u8]) -> Option<(EPICHeaderFields, &[u8])> { if msg.len() < mem::size_of::() { return None; } let (hdr, data) = msg.split_at(mem::size_of::()); + let tag_offset = if hdr[mem::offset_of!(EPICHeader, sub_version)] >= 4 { + mem::offset_of!(EPICHeaderV4, tag) + } else { + mem::offset_of!(EPICHeader, tag) + }; let fields = EPICHeaderFields { category: hdr[mem::offset_of!(EPICHeader, category)], subtype: le_u16(hdr, mem::offset_of!(EPICHeader, subtype)), - tag: le_u16(hdr, mem::offset_of!(EPICHeader, tag)), + tag: le_u16(hdr, tag_offset), }; Some((fields, data)) } @@ -721,23 +751,44 @@ impl AFKEndpoint { return Err(EIO); }; let call = Arc::pin_init(FutureValue::pin_init(), GFP_KERNEL)?; - let hdr = EPICHeader { + let tag = (slot + 1) as u16; + let hdr_v2 = EPICHeader { version: 2, seq: self.seq, length: data.len() as u32, sub_version: 2, category: EPIC_CATEGORY_NOTIFY, subtype, - tag: (slot + 1) as u16, + tag, ..EPICHeader::default() }; - // SAFETY: `hdr` is a packed plain-data struct that outlives the slice, - // which covers exactly its bytes. + // The version 4 firmware wants the reply capacity advertised; a + // property read is answered even with zero, other calls may not be. + let hdr_v4 = EPICHeaderV4 { + version: 2, + seq: self.seq, + length: data.len() as u32, + sub_version: 4, + category: EPIC_CATEGORY_NOTIFY, + subtype, + tag, + inline_len: ret.as_ref().map_or(0, |ret| ret.len() as u32), + ..EPICHeaderV4::default() + }; + // SAFETY: Both headers are packed plain-data structs that outlive the + // slice, which covers exactly the bytes of the one selected. let hdr_bytes = unsafe { - slice::from_raw_parts( - &hdr as *const EPICHeader as *const u8, - mem::size_of::(), - ) + if client.epic_v4 { + slice::from_raw_parts( + ptr::from_ref(&hdr_v4).cast::(), + mem::size_of::(), + ) + } else { + slice::from_raw_parts( + ptr::from_ref(&hdr_v2).cast::(), + mem::size_of::(), + ) + } }; let doorbell = self.write_entry(client, channel, EPIC_TYPE_NOTIFY, hdr_bytes, data)?; self.seq = self.seq.wrapping_add(1); @@ -796,6 +847,8 @@ impl ChildDevice { #[pin_data] struct AopData { dev: ARef, + /// The firmware speaks EPIC with version 4 sub-headers. + epic_v4: bool, /// Runs the service registrations one at a time; drained on removal. registration_queue: OwnedQueue, /// Serializes queueing a registration with the start of removal, so that @@ -913,12 +966,13 @@ impl WorkItem for AopServiceRegisterWork { } impl AopData { - fn new(dev: &platform::Device>) -> Result> { + fn new(dev: &platform::Device>, cfg: &AopHwConfig) -> Result> { let registration_queue = OwnedQueue::new_ordered(c_str!("apple-aop"))?; Arc::pin_init( pin_init!( AopData { dev: dev.as_ref().into(), + epic_v4: cfg.epic_v4, registration_queue, registration_gate <- new_mutex!(()), removing: Atomic::new(false), @@ -1244,27 +1298,33 @@ struct AopHwConfig { ec0p: u64, alig: u64, aopt: u64, + /// The firmware speaks EPIC with version 4 sub-headers. + epic_v4: bool, } const HW_CFG_T8103: AopHwConfig = AopHwConfig { ec0p: 0x020000, aopt: 1, alig: 128, + epic_v4: false, }; const HW_CFG_T8112: AopHwConfig = AopHwConfig { ec0p: 0x020000, aopt: 0, alig: 128, + epic_v4: false, }; const HW_CFG_T6000: AopHwConfig = AopHwConfig { ec0p: 0x020000, aopt: 0, alig: 64, + epic_v4: false, }; const HW_CFG_T6020: AopHwConfig = AopHwConfig { ec0p: 0x0100_00000000, aopt: 0, alig: 64, + epic_v4: false, }; kernel::of_device_table!( @@ -1296,7 +1356,7 @@ impl platform::Driver for AopDriver { let asc_req = pdev.io_request_by_index(1).ok_or(EINVAL)?; let asc_mmio = asc_req.iomap_sized::()?; let asc_mmio = asc_mmio.as_view().relaxed(); - let data = AopData::new(pdev)?; + let data = AopData::new(pdev, cfg)?; // Whatever fails below leaves the AOP running and its children // registering; the same teardown as unbind's cleans that up. let probe_guard = ScopeGuard::new_with_data(data.clone(), |data| data.remove()); From 85b175f2e0b3a57cc400088ad2bdd5ee1ab91265 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:33:02 +1000 Subject: [PATCH 0446/2680] soc: apple: aop: Serialize calls on endpoints whose replies carry no tag The replies of the T8140 AOP firmware do not echo the tag of the call they answer, so a reply cannot be matched to its call by tag. With two calls in flight on one endpoint the results could be swapped. On firmware marked epic_v4, let each endpoint carry one call at a time: callers take a per-endpoint turn that they hold until the reply or the timeout. A reply whose tag matches no call in flight is then attributed to the endpoint's single occupied slot. When a call times out, its slot stays abandoned and the endpoint refuses further calls, because a new call's reply could not be told from the late one; the late reply, if it comes, frees the slot and the endpoint is usable again. If it never comes, the endpoint stays closed until the driver is rebound, which the timeout message says. Tagged firmware is not affected: it keeps its concurrent calls and its tag matching. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 60 ++++++++++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 8 deletions(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 7fb17dc7e5461..ca202ce8c7981 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -61,7 +61,8 @@ use kernel::{ ArcBorrow, CondVar, CondVarTimeoutResult, - Mutex, // + Mutex, + MutexGuard, // }, time::msecs_to_jiffies, types::{ @@ -642,10 +643,20 @@ impl AFKEndpoint { } let retcode = le_u32(data, 0); let tag = ehdr.tag as usize; - if tag == 0 || tag > self.calls.len() { - return Err(EINVAL); - } - let (future, ret) = match self.calls[tag - 1].take() { + let slot = match tag.checked_sub(1) { + Some(slot) if slot < self.calls.len() && self.calls[slot].is_some() => slot, + // The version 4 firmware does not echo the tag. Calls on + // such an endpoint are serialized, and none is started + // while an abandoned one is outstanding, so at most one + // slot is in use and the reply belongs to it. + _ if client.epic_v4 => { + self.calls.iter().position(|c| c.is_some()).ok_or(ENOENT)? + } + _ if tag == 0 || tag > self.calls.len() => return Err(EINVAL), + _ => return Err(ENOENT), + }; + let tag = slot + 1; + let (future, ret) = match self.calls[slot].take() { Some(CallSlot::Pending(future, ret)) => (future, ret), Some(CallSlot::Abandoned) => { // The late reply to a call that timed out; its slot @@ -658,7 +669,6 @@ impl AFKEndpoint { ); return Ok(()); } - // A reply with no call in flight. None => return Err(ENOENT), }; let extra_data = ret.map(|mut ret| { @@ -742,6 +752,21 @@ impl AFKEndpoint { data: &[u8], ret: Option>, ) -> Result>> { + if client.epic_v4 + && self + .calls + .iter() + .any(|c| matches!(c, Some(CallSlot::Abandoned))) + { + // Replies carry no tag here: until the reply to the abandoned call + // has arrived, a new call's reply could not be told from it. + dev_dbg!( + client.dev, + "Endpoint {:#04x} waits for a late reply, call refused", + self.index + ); + return Err(EIO); + } let Some(slot) = self.calls.iter().position(|c| c.is_none()) else { dev_err!( client.dev, @@ -863,6 +888,10 @@ struct AopData { rtkit: Mutex>>, #[pin] endpoints: [Mutex; AFK_ENDPOINT_COUNT as usize], + /// Version 4 firmware answers without the request tag, so an endpoint + /// carries one call at a time: callers queue here for their turn. + #[pin] + call_turn: [Mutex<()>; AFK_ENDPOINT_COUNT as usize], #[pin] ep_shutdown: [FutureValue<()>; AFK_ENDPOINT_COUNT as usize], #[pin] @@ -981,6 +1010,7 @@ impl AopData { endpoints <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(AFKEndpoint::new(AFK_ENDPOINT_START + i as u8)) }), + call_turn <- pin_init::pin_init_array_from_fn(|_| new_mutex!(())), ep_shutdown <- pin_init::pin_init_array_from_fn(|_| FutureValue::pin_init()), hid_listeners <- new_mutex!(KVec::new()), subdevices <- new_mutex!(KVec::new()), @@ -1122,6 +1152,13 @@ impl AopData { } impl AopData { + /// On firmware whose replies carry no tag, takes the endpoint's turn: the + /// guard is held until the reply or the timeout, so that a reply can only + /// belong to the one call in flight. + fn take_call_turn(&self, ep_idx: usize) -> Option> { + self.epic_v4.then(|| self.call_turn[ep_idx].lock()) + } + /// Waits a bounded time for the reply to `call` on endpoint `ep_idx`. fn wait_call( &self, @@ -1136,10 +1173,15 @@ impl AopData { self.endpoints[ep_idx].lock().abandon_call(&call); dev_err!( self.dev, - "EPIC call {:#x} on channel {} timed out after {} ms", + "EPIC call {:#x} on channel {} timed out after {} ms{}", subtype, svc.channel, - EPIC_CALL_TIMEOUT_MS + EPIC_CALL_TIMEOUT_MS, + if self.epic_v4 { + "; the endpoint takes no call until the reply arrives" + } else { + "" + } ); Err(ETIMEDOUT) } @@ -1151,6 +1193,7 @@ impl AOP for AopData { return Err(ENODEV); } let ep_idx = afk_endpoint_index(svc.endpoint).ok_or(EINVAL)?; + let _turn = self.take_call_turn(ep_idx); let call = { let mut rtk_guard = self.rtkit.lock(); let mut rtk = rtk_guard.as_mut().as_pin_mut().ok_or(ENODEV)?; @@ -1170,6 +1213,7 @@ impl AOP for AopData { return Err(ENODEV); } let ep_idx = afk_endpoint_index(svc.endpoint).ok_or(EINVAL)?; + let _turn = self.take_call_turn(ep_idx); let call = { let mut rtk_guard = self.rtkit.lock(); let mut rtk = rtk_guard.as_mut().as_pin_mut().ok_or(ENODEV)?; From 3f61460b2c5cb4a5c7b2dbb0f50e27d5262ea2a4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:37:51 +1000 Subject: [PATCH 0447/2680] soc: apple: aop: Add the T8140 setup-port handshake The T8140 AOP firmware boots through a second mailbox besides the RTKit one, its "setup port". Messages on it are a 64-bit word plus an endpoint number. Endpoint 0 runs the management handshake: the firmware sends HELLO (answered with the same version word), an endpoint map in fragments (each echoed back, the last one flagged in bit 51), and a few messages that want a fixed answer or none. Once the RTKit side has reached AP power on, the host requests the setup port's AP power state and the firmware reports it. Five service endpoints (0x20, 0x21, 0x23, 0x30 and 0x32) then each request a host-to-AOP message page and an AOP-to-host reply window of 0x400 or 0x1000 16-byte entries, which the host hands out of an arena of 13 pages that it maps through the setup mailbox's own DART stream and describes to the firmware as (5 << 60) | (direction << 54) | (pages << 48) | (iova >> 4). Boot is complete when the power state is on and all five endpoints have their buffers; the AFK endpoints are started only then. Add this handshake behind a setup_port flag in the match data, which no compatible sets yet. The mailbox interrupt handler may not sleep, so it allocates a small work item atomically for each message, the same way the RTKit core defers its own messages, and queues it on an ordered workqueue that handles the messages one at a time under the setup lock; a message it fails to queue is lost, and the port is marked as failed rather than left waiting for a reply that cannot come. All sends happen from that worker or from process context and may sleep for FIFO room. Waits on the port are uninterruptible, bounded, and carry the remaining time across wakeups; they end early once the port has failed. Unknown management messages and words on endpoints without buffers are logged and ignored, not treated as fatal. The arena is allocated on the mailbox provider's device, since its node carries the DART stream. The mailbox lookup adds a device link to that provider, so it stays bound for as long as the AOP is, and the arena is released while the AOP is still bound. The port is closed on removal by dropping the mailbox and draining the queue, both outside the setup lock. Compile-tested only; not run on hardware. [ryan: adapted to 7.2.10 - the setup-port bring-up is ordered around the ASC_CPU_CONTROL register update and pinned rtkit handle that asahi-wip uses in probe] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 1 + drivers/soc/apple/aop.rs | 441 +++++++++++++++++++++++++++++++++++++- 2 files changed, 435 insertions(+), 7 deletions(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 62f2f33e6b56c..0a92213f76c46 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -102,6 +102,7 @@ config APPLE_AOP depends on PM depends on RUST select RUST_APPLE_RTKIT + select RUST_APPLE_MAILBOX help A co-processor persent on certain Apple SoCs controlling accelerometers, gyros, ambient light sensors and microphones. Is not actually always on. diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index ca202ce8c7981..0fe4741307fe7 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -24,7 +24,10 @@ use kernel::{ Device, DmaMask, // }, - error::from_err_ptr, + error::{ + from_err_ptr, + to_result, // + }, fmt, io::{ io_project, @@ -46,6 +49,7 @@ use kernel::{ FakehidListener, AOP, // }, + soc::apple::mailbox, soc::apple::rtkit::{ self, ASC_CPU_CONTROL, // @@ -117,6 +121,40 @@ const EPIC_CALL_TIMEOUT_MS: u32 = 5000; /// Bound on the wait for an endpoint's shutdown acknowledgment. const AFK_SHUTDOWN_TIMEOUT_MS: u32 = 5000; +// The T8140 AOP boots through a second mailbox, its "setup port", besides the +// RTKit one. Its management endpoint runs a HELLO / endpoint map / power +// handshake, after which five service endpoints each request a host-to-AOP +// message page and an AOP-to-host reply window, which the host hands out of +// an arena it maps through the setup mailbox's own DART stream. Messages are +// a 64-bit word plus the endpoint number. +const SETUP_PAGE: usize = 0x4000; +/// Five message pages plus the reply windows: the endpoints request either +/// 0x400 or 0x1000 16-byte reply entries, one or four pages, and the five of +/// them take eight pages in total. +const SETUP_ARENA_PAGES: usize = 13; +const SETUP_MGMT_EP: u8 = 0; +/// Management message types, in bits 52..60 of the word. +const SETUP_TYPE_HELLO: u64 = 1; +const SETUP_TYPE_HELLO_REPLY: u64 = 2; +const SETUP_TYPE_UNK3: u64 = 3; +const SETUP_TYPE_UNK3_REPLY: u64 = 4; +const SETUP_TYPE_PWR_ACK: u64 = 7; +const SETUP_TYPE_EPMAP: u64 = 8; +const SETUP_TYPE_AP_PWR: u64 = 0xb; +const SETUP_HELLO_VERSION: u64 = 0xc000c; +const SETUP_EPMAP_LAST: u64 = 1 << 51; +const SETUP_AP_PWR_INIT: u64 = 0x220; +const SETUP_AP_PWR_ON: u64 = 0x20; +/// A service endpoint's buffer request: the type in bits 56..64, the number +/// of reply entries in the low 32 bits. +const SETUP_BUFFER_REQUEST: u64 = 0x12; +const SETUP_BUFFER_REQUEST_ACK: u64 = SETUP_BUFFER_REQUEST << 56; +const SETUP_BUFFER_ENTRY_SIZE: usize = 16; +/// The service endpoints that request buffers; boot completes once all of +/// them have theirs. +const SETUP_ENDPOINTS: [u8; 5] = [0x20, 0x21, 0x23, 0x30, 0x32]; +const SETUP_BOOT_TIMEOUT_MS: u32 = 15000; + fn align_up(v: usize, a: usize) -> usize { (v + a - 1) & !(a - 1) } @@ -844,6 +882,136 @@ struct ListenerEntry { listener: Arc, } +/// One of the setup port's service endpoints, once it has its buffers. +#[derive(Clone, Copy)] +struct SetupEndpoint { + ep: u8, + /// The last word received on the endpoint that was not a buffer request. + reply: Option, +} + +struct SetupState { + /// `None` until the port is opened and again once it is closed. + mbox: Option>, + arena: Option>, + /// The next unassigned page of the arena. + next_page: usize, + endpoints: [Option; SETUP_ENDPOINTS.len()], + map_done: bool, + ap_ready: bool, + power_sent: bool, + /// The protocol broke down; waiters give up instead of timing out. + failed: bool, +} + +impl SetupState { + fn new() -> SetupState { + SetupState { + mbox: None, + arena: None, + next_page: 0, + endpoints: [None; SETUP_ENDPOINTS.len()], + map_done: false, + ap_ready: false, + power_sent: false, + failed: false, + } + } + + fn endpoint_mut(&mut self, ep: u8) -> Option<&mut SetupEndpoint> { + self.endpoints.iter_mut().flatten().find(|e| e.ep == ep) + } + + fn endpoint_count(&self) -> usize { + self.endpoints.iter().flatten().count() + } + + fn arena_iova(&self) -> Result { + Ok(self.arena.as_ref().ok_or(ENXIO)?.dma_handle()) + } + + /// Boot is complete once the AP power state is on and every service + /// endpoint has its buffers. + fn ready(&self) -> bool { + self.ap_ready && self.endpoint_count() == SETUP_ENDPOINTS.len() + } + + /// Sends one word to a setup-port endpoint. Callers run in process + /// context, so the send may sleep for room in the mailbox FIFO. + fn send(&self, ep: u8, word: u64) -> Result<()> { + let mbox = self.mbox.as_ref().ok_or(ENXIO)?; + mbox.send( + mailbox::Message { + msg0: word, + msg1: u32::from(ep), + }, + false, + ) + } + + /// The word that hands a buffer to the firmware: + /// (5 << 60) | (host_to_aop << 54) | (pages << 48) | (iova >> 4). + fn shared_descriptor(iova: u64, pages: usize, host_to_aop: bool) -> u64 { + (5u64 << 60) | (u64::from(host_to_aop) << 54) | ((pages as u64) << 48) | (iova >> 4) + } +} + +struct SetupPortCallback; + +impl mailbox::MailCallback for SetupPortCallback { + type Data = Arc; + + /// Runs in hard IRQ context: hands the message to the ordered setup + /// queue, which is all that may be done here. A message that cannot be + /// queued is lost, and the protocol state with it. + fn recv_message(data: ::Borrowed<'_>, msg: mailbox::Message) { + let queued = SetupRxWork::new(data.into(), msg).map(|work| { + if let Some(queue) = data.setup_queue.as_ref() { + queue.enqueue(work); + } + }); + if queued.is_err() { + data.setup_lost.store(true, Release); + data.setup_cv.notify_all(); + } + } +} + +/// One received setup-port message on its way to the setup queue. +#[pin_data] +struct SetupRxWork { + data: Arc, + msg: mailbox::Message, + #[pin] + work: Work, +} + +impl_has_work! { + impl HasWork for SetupRxWork { self.work } +} + +impl SetupRxWork { + /// Allocates atomically: the caller is the mailbox interrupt handler. + fn new(data: Arc, msg: mailbox::Message) -> Result>> { + KBox::pin_init( + pin_init!(SetupRxWork { + data, + msg, + work <- new_work!("SetupRxWork::work"), + }), + GFP_ATOMIC, + ) + } +} + +impl WorkItem for SetupRxWork { + type Pointer = Pin>; + + fn run(this: Pin>) { + this.data.setup_receive(this.msg); + } +} + /// A service platform device registered by this driver. It is unregistered /// explicitly by [`AopData::remove`]; nothing else touches the pointer. struct ChildDevice(NonNull); @@ -884,6 +1052,16 @@ struct AopData { removing: Atomic, /// Set when the transport starts closing; no call is started after it. transport_closing: Atomic, + /// Runs the setup-port messages in order; only on firmware with a setup + /// port. + setup_queue: Option, + /// A setup-port message could not be queued from the interrupt handler. + setup_lost: Atomic, + #[pin] + setup: Mutex, + /// Signalled after every setup-port message. + #[pin] + setup_cv: CondVar, #[pin] rtkit: Mutex>>, #[pin] @@ -997,6 +1175,11 @@ impl WorkItem for AopServiceRegisterWork { impl AopData { fn new(dev: &platform::Device>, cfg: &AopHwConfig) -> Result> { let registration_queue = OwnedQueue::new_ordered(c_str!("apple-aop"))?; + let setup_queue = if cfg.setup_port { + Some(OwnedQueue::new_ordered(c_str!("apple-aop-setup"))?) + } else { + None + }; Arc::pin_init( pin_init!( AopData { @@ -1006,6 +1189,10 @@ impl AopData { registration_gate <- new_mutex!(()), removing: Atomic::new(false), transport_closing: Atomic::new(false), + setup_queue, + setup_lost: Atomic::new(false), + setup <- new_mutex!(SetupState::new()), + setup_cv <- new_condvar!(), rtkit <- new_mutex!(None), endpoints <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(AFKEndpoint::new(AFK_ENDPOINT_START + i as u8)) @@ -1020,11 +1207,17 @@ impl AopData { ) } fn start(&self) -> Result<()> { - { - let mut guard = self.rtkit.lock(); - let mut rtk = guard.as_mut().as_pin_mut().ok_or(ENODEV)?; - rtk.as_mut().wake()?; - } + self.wake()?; + self.start_afk() + } + /// Runs the RTKit handshake up to AP power on. + fn wake(&self) -> Result<()> { + let mut guard = self.rtkit.lock(); + let mut rtk = guard.as_mut().as_pin_mut().ok_or(ENODEV)?; + rtk.as_mut().wake() + } + /// Starts the AFK handshake on every advertised endpoint. + fn start_afk(&self) -> Result<()> { for ep in 0..AFK_ENDPOINT_COUNT as usize { let rtk_ep_num = AFK_ENDPOINT_START + ep as u8; let mut guard = self.rtkit.lock(); @@ -1285,12 +1478,228 @@ impl AOP for AopData { // After it, no callback runs and the device may be unbound. let rtkit = self.rtkit.lock().take(); drop(rtkit); + // Close the setup port: dropping the mailbox stops its interrupt, and + // draining the queue finishes the messages that were already taken. + // Neither may happen under the setup lock, which the queue's work + // takes. The arena is freed while the device is still bound. + let setup_mbox = self.setup.lock().mbox.take(); + drop(setup_mbox); + if let Some(queue) = self.setup_queue.as_ref() { + queue.drain(); + } + let arena = self.setup.lock().arena.take(); + drop(arena); for child in children { child.unregister(); } } } +impl AopData { + /// Opens the setup port: the mailbox and the arena its endpoints get + /// their buffers from. Done before the co-processor runs, since its + /// first message has to be answered. + fn setup_open(this: &Arc, dev: &device::Device) -> Result<()> { + let mbox = + mailbox::Mailbox::::new_byname(dev, c_str!("setup"), this.clone())?; + // The arena is DMA of the mailbox provider's device: its node carries + // the DART stream the firmware reaches the setup buffers through. + let mbox_dev = mbox.device(); + // SAFETY: `mbox_dev` is a valid device and nothing of ours has DMA in + // flight on it yet. + unsafe { + to_result(bindings::dma_set_mask_and_coherent( + mbox_dev.as_raw(), + DmaMask::new::<42>().value(), + ))?; + } + // SAFETY: Getting the mailbox added a device link from this device to + // the provider, so the provider stays bound for as long as this device + // is, and the arena is freed by `remove()` while this device is bound. + let bound = unsafe { mbox_dev.as_bound() }; + let arena = + Coherent::::zeroed_slice(bound, SETUP_ARENA_PAGES * SETUP_PAGE, GFP_KERNEL)?; + let mut st = this.setup.lock(); + st.arena = Some(arena); + st.mbox = Some(mbox); + Ok(()) + } + + /// Handles one received setup-port message, in order, on the setup queue. + fn setup_receive(&self, msg: mailbox::Message) { + let mut st = self.setup.lock(); + if st.mbox.is_none() { + return; + } + if self.setup_lost.load(Acquire) { + st.failed = true; + } else { + let ep = (msg.msg1 & 0xff) as u8; + if let Err(e) = self.setup_handle(&mut st, ep, msg.msg0) { + dev_err!( + self.dev, + "setup port: protocol error on endpoint {:#x}, message {:#x}: {:?}", + ep, + msg.msg0, + e + ); + st.failed = true; + } + } + drop(st); + self.setup_cv.notify_all(); + } + + fn setup_handle(&self, st: &mut SetupState, ep: u8, word: u64) -> Result<()> { + if ep == SETUP_MGMT_EP { + return match (word >> 52) & 0xff { + SETUP_TYPE_HELLO => { + if word & 0xffff_ffff != SETUP_HELLO_VERSION { + dev_warn!( + self.dev, + "setup port: HELLO version {:#x}", + word & 0xffff_ffff + ); + } + st.send( + SETUP_MGMT_EP, + (SETUP_TYPE_HELLO_REPLY << 52) | SETUP_HELLO_VERSION, + ) + } + SETUP_TYPE_EPMAP => { + // Each fragment of the endpoint map is acknowledged by + // echoing it. + st.send(SETUP_MGMT_EP, word)?; + if word & SETUP_EPMAP_LAST != 0 { + st.map_done = true; + } + Ok(()) + } + SETUP_TYPE_AP_PWR => { + st.ap_ready = (word & 0xffff) == SETUP_AP_PWR_ON; + Ok(()) + } + SETUP_TYPE_UNK3 => st.send(SETUP_MGMT_EP, SETUP_TYPE_UNK3_REPLY << 52), + SETUP_TYPE_PWR_ACK => Ok(()), + ty => { + dev_warn!( + self.dev, + "setup port: ignoring management message type {:#x} ({:#x})", + ty, + word + ); + Ok(()) + } + }; + } + if word >> 56 == SETUP_BUFFER_REQUEST { + return self.setup_endpoint_request(st, ep, word); + } + if let Some(endpoint) = st.endpoint_mut(ep) { + endpoint.reply = Some(word); + return Ok(()); + } + dev_warn!( + self.dev, + "setup port: ignoring message {:#x} on endpoint {:#x}", + word, + ep + ); + Ok(()) + } + + /// Answers a service endpoint's buffer request with one message page + /// and a reply window of the requested number of entries, both taken + /// from the arena. + fn setup_endpoint_request(&self, st: &mut SetupState, ep: u8, word: u64) -> Result<()> { + let requested = (word & 0xffff_ffff) as usize; + let slot = SETUP_ENDPOINTS + .iter() + .position(|e| *e == ep) + .ok_or(EINVAL)?; + if st.endpoints[slot].is_some() || (requested != 0x400 && requested != 0x1000) { + return Err(EINVAL); + } + let rx_pages = (requested * SETUP_BUFFER_ENTRY_SIZE).div_ceil(SETUP_PAGE); + if st.next_page + 1 + rx_pages > SETUP_ARENA_PAGES { + return Err(ENOMEM); + } + let base = st.arena_iova()?; + let tx_iova = base + (st.next_page * SETUP_PAGE) as u64; + let rx_iova = tx_iova + SETUP_PAGE as u64; + st.next_page += 1 + rx_pages; + st.send(ep, SETUP_BUFFER_REQUEST_ACK)?; + st.send(ep, SetupState::shared_descriptor(tx_iova, 1, true))?; + st.send(ep, SetupState::shared_descriptor(rx_iova, rx_pages, false))?; + st.endpoints[slot] = Some(SetupEndpoint { ep, reply: None }); + dev_dbg!( + self.dev, + "setup port: endpoint {:#x} tx {:#x} rx {:#x} ({} pages)", + ep, + tx_iova, + rx_iova, + rx_pages + ); + Ok(()) + } + + /// Waits, in TASK_UNINTERRUPTIBLE and for at most `timeout_ms`, until + /// `done` holds for the setup state. Fails at once when the setup port + /// has failed. Spurious wakeups continue the wait with the time left. + fn setup_wait( + &self, + st: &mut MutexGuard<'_, SetupState>, + timeout_ms: u32, + done: impl Fn(&SetupState) -> bool, + ) -> Result<()> { + let mut left = msecs_to_jiffies(timeout_ms); + loop { + if done(st) { + return Ok(()); + } + if st.failed || self.setup_lost.load(Acquire) { + return Err(EIO); + } + match self.setup_cv.wait_timeout(st, left) { + CondVarTimeoutResult::Timeout => { + return if done(st) { Ok(()) } else { Err(ETIMEDOUT) }; + } + CondVarTimeoutResult::Woken { jiffies } + | CondVarTimeoutResult::Signal { jiffies } => left = jiffies, + } + } + } + + /// After the RTKit side has reached AP power on: requests the setup + /// port's AP power state and waits for it and for the five endpoint + /// buffers. + fn setup_finish_boot(&self) -> Result<()> { + let mut st = self.setup.lock(); + if !st.map_done { + dev_warn!( + self.dev, + "setup port: endpoint map incomplete before the AP power request" + ); + } + if !st.power_sent { + st.send(SETUP_MGMT_EP, (SETUP_TYPE_AP_PWR << 52) | SETUP_AP_PWR_INIT)?; + st.power_sent = true; + } + if let Err(e) = self.setup_wait(&mut st, SETUP_BOOT_TIMEOUT_MS, SetupState::ready) { + dev_err!( + self.dev, + "setup port: boot incomplete (AP power on: {}, endpoints: {}/{}): {:?}", + st.ap_ready, + st.endpoint_count(), + SETUP_ENDPOINTS.len(), + e + ); + return Err(e); + } + Ok(()) + } +} + struct NoBuffer; impl rtkit::Buffer for NoBuffer { fn iova(&self) -> Result { @@ -1344,6 +1753,8 @@ struct AopHwConfig { aopt: u64, /// The firmware speaks EPIC with version 4 sub-headers. epic_v4: bool, + /// The firmware boots through a second, "setup", mailbox as well. + setup_port: bool, } const HW_CFG_T8103: AopHwConfig = AopHwConfig { @@ -1351,24 +1762,28 @@ const HW_CFG_T8103: AopHwConfig = AopHwConfig { aopt: 1, alig: 128, epic_v4: false, + setup_port: false, }; const HW_CFG_T8112: AopHwConfig = AopHwConfig { ec0p: 0x020000, aopt: 0, alig: 128, epic_v4: false, + setup_port: false, }; const HW_CFG_T6000: AopHwConfig = AopHwConfig { ec0p: 0x020000, aopt: 0, alig: 64, epic_v4: false, + setup_port: false, }; const HW_CFG_T6020: AopHwConfig = AopHwConfig { ec0p: 0x0100_00000000, aopt: 0, alig: 64, epic_v4: false, + setup_port: false, }; kernel::of_device_table!( @@ -1416,8 +1831,20 @@ impl platform::Driver for AopDriver { )?; let rtkit = rtkit::RtKit::::new(pdev.as_ref(), None, 0, data.clone())?; data.rtkit.lock().as_mut().set(Some(rtkit)); + if cfg.setup_port { + AopData::setup_open(&data, pdev.as_ref())?; + } asc_mmio.update(ASC_CPU_CONTROL, |r| r.with_const_cpu_run::<1>()); - data.start()?; + if cfg.setup_port { + // The RTKit handshake up to AP power on comes first, then the + // setup port's own power state and endpoint buffers, and only + // then the AFK endpoints. + data.wake()?; + data.setup_finish_boot()?; + data.start_afk()?; + } else { + data.start()?; + } probe_guard.dismiss(); let data = data as Arc; Ok(Self(data)) From b771b46944af3e9d6592005c80b1ca3145abff42 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:39:49 +1000 Subject: [PATCH 0448/2680] soc: apple: aop: Shut the T8140 firmware down on unbind and retain DMA Removal stops the AFK endpoints and frees the RTKit handle, but never shuts the co-processor down. Its firmware keeps running on the shared ring buffers and, on the T8140, on the setup arena, all of which return to the allocator when the driver data is dropped. The existing SoCs have always been removed this way and are left as they are; the T8140, whose firmware also owns DMA windows the host handed it over the setup port, gets a quiesce_on_unbind flag in its match data. With the flag set, removal asks the RTKit core for a shutdown after the endpoints have stopped. If the firmware does not acknowledge it, the buffers it may still write to cannot be freed: the endpoint buffers and the setup arena are leaked, the RTKit handle is dropped with its shared buffers retained, and the service devices stay registered so that their IOMMU domains keep the mappings the firmware may use, but are given a driver override that no driver matches so that nothing binds to them again. A static flag then keeps this device from probing again until a reboot, since the firmware's state is unknown. Nothing of this happens unless the co-processor was actually started. Compile-tested only; not run on hardware. [ryan: adapted to 7.2.10 - start_cpu() no longer exists (asahi-wip starts the CPU through the ASC_CPU_CONTROL register update), so cpu_started is set right after that update in probe] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 96 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 95 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 0fe4741307fe7..d60a26af959f5 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -1020,7 +1020,32 @@ struct ChildDevice(NonNull); // which may be called from any thread. unsafe impl Send for ChildDevice {} +/// A driver override that no driver matches: keeps a service device from +/// binding again once its transport is gone but it has to stay registered. +const RETIRED_DRIVER_OVERRIDE: &CStr = c_str!("apple-aop-retired"); + +/// Set once a shutdown could not be confirmed. The firmware may still be +/// running on the retained buffers, so the device is not brought up again +/// before a reboot. +static RETIRED: Atomic = Atomic::new(false); + impl ChildDevice { + /// Keeps the child from binding to any driver again; it stays registered. + fn retire(&self) { + // SAFETY: The device is registered, so its embedded `struct device` is + // valid, and the override string is NUL-terminated. + let ret = unsafe { + bindings::__device_set_driver_override( + ptr::addr_of_mut!((*self.0.as_ptr()).dev), + RETIRED_DRIVER_OVERRIDE.as_char_ptr(), + RETIRED_DRIVER_OVERRIDE.to_bytes().len(), + ) + }; + // Only an allocation failure; the device is unbound either way and + // the retired flag keeps the parent from coming back. + let _ = ret; + } + /// Unbinds the child's driver while the child stays registered. fn release_driver(&self) { // SAFETY: The pointer came from a successful @@ -1052,6 +1077,12 @@ struct AopData { removing: Atomic, /// Set when the transport starts closing; no call is started after it. transport_closing: Atomic, + /// The co-processor was started; only then is there anything to shut + /// down and to retain. + cpu_started: Atomic, + /// Shut the co-processor down on removal, and if that cannot be + /// confirmed, retain everything it may still DMA to. + quiesce_on_unbind: bool, /// Runs the setup-port messages in order; only on firmware with a setup /// port. setup_queue: Option, @@ -1189,6 +1220,8 @@ impl AopData { registration_gate <- new_mutex!(()), removing: Atomic::new(false), transport_closing: Atomic::new(false), + cpu_started: Atomic::new(false), + quiesce_on_unbind: cfg.quiesce_on_unbind, setup_queue, setup_lost: Atomic::new(false), setup <- new_mutex!(SetupState::new()), @@ -1342,6 +1375,19 @@ impl AopData { bootargs.copy_from_slice(&arg_bytes); Ok(()) } + + /// Leaks every allocation the firmware may still access. Called when the + /// shutdown could not be confirmed; the memory is lost until reboot. + fn retain_dma_buffers(&self) { + for endpoint in &self.endpoints { + if let Some(buffer) = endpoint.lock().iomem.take() { + mem::forget(buffer); + } + } + if let Some(arena) = self.setup.lock().arena.take() { + mem::forget(arena); + } + } } impl AopData { @@ -1477,7 +1523,24 @@ impl AOP for AopData { // drop waits for the RTKit receive worker, which takes the same lock. // After it, no callback runs and the device may be unbound. let rtkit = self.rtkit.lock().take(); - drop(rtkit); + let mut quiesced = true; + if let Some(mut rtkit) = rtkit { + if self.quiesce_on_unbind && self.cpu_started.load(Acquire) { + // The co-processor DMAs into the shared buffers until it has + // acknowledged the shutdown. If it does not, nothing it may + // still write to can be freed. + if let Err(e) = Pin::new(&mut rtkit).shutdown() { + dev_err!( + self.dev, + "AOP shutdown unconfirmed ({:?}); retaining its buffers until reboot", + e + ); + rtkit.retain_shared_buffers_on_drop(); + quiesced = false; + } + } + drop(rtkit); + } // Close the setup port: dropping the mailbox stops its interrupt, and // draining the queue finishes the messages that were already taken. // Neither may happen under the setup lock, which the queue's work @@ -1487,6 +1550,22 @@ impl AOP for AopData { if let Some(queue) = self.setup_queue.as_ref() { queue.drain(); } + if !quiesced { + // The children stay registered: their IOMMU domains hold the + // mappings the firmware may still use. Keep them from binding + // again, and this device from probing again, until a reboot. + self.retain_dma_buffers(); + for child in &children { + child.retire(); + } + RETIRED.store(true, Release); + dev_err!( + self.dev, + "keeping {} unbound service devices and their DMA mappings", + children.len() + ); + return; + } let arena = self.setup.lock().arena.take(); drop(arena); for child in children { @@ -1755,6 +1834,9 @@ struct AopHwConfig { epic_v4: bool, /// The firmware boots through a second, "setup", mailbox as well. setup_port: bool, + /// Shut the co-processor down on unbind and retain its buffers if the + /// shutdown cannot be confirmed. + quiesce_on_unbind: bool, } const HW_CFG_T8103: AopHwConfig = AopHwConfig { @@ -1763,6 +1845,7 @@ const HW_CFG_T8103: AopHwConfig = AopHwConfig { alig: 128, epic_v4: false, setup_port: false, + quiesce_on_unbind: false, }; const HW_CFG_T8112: AopHwConfig = AopHwConfig { ec0p: 0x020000, @@ -1770,6 +1853,7 @@ const HW_CFG_T8112: AopHwConfig = AopHwConfig { alig: 128, epic_v4: false, setup_port: false, + quiesce_on_unbind: false, }; const HW_CFG_T6000: AopHwConfig = AopHwConfig { ec0p: 0x020000, @@ -1777,6 +1861,7 @@ const HW_CFG_T6000: AopHwConfig = AopHwConfig { alig: 64, epic_v4: false, setup_port: false, + quiesce_on_unbind: false, }; const HW_CFG_T6020: AopHwConfig = AopHwConfig { ec0p: 0x0100_00000000, @@ -1784,6 +1869,7 @@ const HW_CFG_T6020: AopHwConfig = AopHwConfig { alig: 64, epic_v4: false, setup_port: false, + quiesce_on_unbind: false, }; kernel::of_device_table!( @@ -1809,6 +1895,13 @@ impl platform::Driver for AopDriver { info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { let cfg = info.ok_or(ENODEV)?; + if RETIRED.load(Acquire) { + dev_err!( + pdev.as_ref(), + "an earlier instance could not be shut down; reboot before probing again" + ); + return Err(ENODEV); + } unsafe { pdev.dma_set_mask_and_coherent(DmaMask::new::<42>())? }; let aop_req = pdev.io_request_by_index(0).ok_or(EINVAL)?; let aop_mmio = aop_req.iomap_sized::()?; @@ -1835,6 +1928,7 @@ impl platform::Driver for AopDriver { AopData::setup_open(&data, pdev.as_ref())?; } asc_mmio.update(ASC_CPU_CONTROL, |r| r.with_const_cpu_run::<1>()); + data.cpu_started.store(true, Release); if cfg.setup_port { // The RTKit handshake up to AP power on comes first, then the // setup port's own power state and endpoint buffers, and only From 958dd8354aa4a9cc14493536f2f0816a846c6b30 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:41:00 +1000 Subject: [PATCH 0449/2680] soc: apple: aop: Add T8140 support Bind to apple,t8140-aop. The T8140 AOP firmware is loaded by the bootloader and left stopped with its boot arguments in place, so the EC0p/nCal/alig/AOPt patching that the other SoCs need is not applied to it. It boots through the setup port and speaks EPIC with version 4 sub-headers, and it is shut down on unbind. The firmware advertises more AFK endpoints than it starts. Its application map is 0x20 misc, 0x21 aop-audio, 0x22 aop-voicetrigger, 0x23 als and 0x2b aop-audprov, which are also the endpoints macOS starts on this AOP. Those are required; any other advertised endpoint that fails to start is skipped with a warning instead of failing the probe. The existing SoCs keep requiring every advertised endpoint. Compile-tested only; not run on hardware. [ryan: adapted to 7.2.10 - the bootargs patching it makes conditional takes the Mmio view asahi-wip introduced] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 78 ++++++++++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 14 deletions(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index d60a26af959f5..b272b921ee664 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -1083,6 +1083,8 @@ struct AopData { /// Shut the co-processor down on removal, and if that cannot be /// confirmed, retain everything it may still DMA to. quiesce_on_unbind: bool, + /// The endpoints that have to start; empty means all advertised ones. + required_endpoints: &'static [u8], /// Runs the setup-port messages in order; only on firmware with a setup /// port. setup_queue: Option, @@ -1222,6 +1224,7 @@ impl AopData { transport_closing: Atomic::new(false), cpu_started: Atomic::new(false), quiesce_on_unbind: cfg.quiesce_on_unbind, + required_endpoints: cfg.required_endpoints, setup_queue, setup_lost: Atomic::new(false), setup <- new_mutex!(SetupState::new()), @@ -1249,7 +1252,8 @@ impl AopData { let mut rtk = guard.as_mut().as_pin_mut().ok_or(ENODEV)?; rtk.as_mut().wake() } - /// Starts the AFK handshake on every advertised endpoint. + /// Starts the AFK handshake on every advertised endpoint. An endpoint + /// the match data does not require may fail to start; it is skipped. fn start_afk(&self) -> Result<()> { for ep in 0..AFK_ENDPOINT_COUNT as usize { let rtk_ep_num = AFK_ENDPOINT_START + ep as u8; @@ -1258,10 +1262,26 @@ impl AopData { if !rtk.as_mut().has_endpoint(rtk_ep_num) { continue; } - rtk.as_mut().start_endpoint(rtk_ep_num)?; - let mut ep_guard = self.endpoints[ep].lock(); - ep_guard.start(rtk.as_mut())?; - ep_guard.started = true; + let required = + self.required_endpoints.is_empty() || self.required_endpoints.contains(&rtk_ep_num); + let started = rtk.as_mut().start_endpoint(rtk_ep_num).and_then(|()| { + let mut ep_guard = self.endpoints[ep].lock(); + ep_guard.start(rtk.as_mut())?; + ep_guard.started = true; + Ok(()) + }); + match started { + Ok(()) => {} + Err(e) if !required => { + dev_warn!( + self.dev, + "Endpoint {:#04x} did not start ({:?}); skipping it", + rtk_ep_num, + e + ); + } + Err(e) => return Err(e), + } } Ok(()) } @@ -1830,6 +1850,8 @@ struct AopHwConfig { ec0p: u64, alig: u64, aopt: u64, + /// Complete the firmware's boot arguments before starting it. + patch_bootargs: bool, /// The firmware speaks EPIC with version 4 sub-headers. epic_v4: bool, /// The firmware boots through a second, "setup", mailbox as well. @@ -1837,39 +1859,64 @@ struct AopHwConfig { /// Shut the co-processor down on unbind and retain its buffers if the /// shutdown cannot be confirmed. quiesce_on_unbind: bool, + /// The endpoints that have to start; empty means all advertised ones. + required_endpoints: &'static [u8], } const HW_CFG_T8103: AopHwConfig = AopHwConfig { ec0p: 0x020000, aopt: 1, alig: 128, + patch_bootargs: true, epic_v4: false, setup_port: false, quiesce_on_unbind: false, + required_endpoints: &[], }; const HW_CFG_T8112: AopHwConfig = AopHwConfig { ec0p: 0x020000, aopt: 0, alig: 128, + patch_bootargs: true, epic_v4: false, setup_port: false, quiesce_on_unbind: false, + required_endpoints: &[], }; const HW_CFG_T6000: AopHwConfig = AopHwConfig { ec0p: 0x020000, aopt: 0, alig: 64, + patch_bootargs: true, epic_v4: false, setup_port: false, quiesce_on_unbind: false, + required_endpoints: &[], }; const HW_CFG_T6020: AopHwConfig = AopHwConfig { ec0p: 0x0100_00000000, aopt: 0, alig: 64, + patch_bootargs: true, epic_v4: false, setup_port: false, quiesce_on_unbind: false, + required_endpoints: &[], +}; +/// T8140: the firmware is started with the boot arguments the bootloader +/// left, boots through the setup port and speaks EPIC version 4. Of the +/// advertised AFK endpoints, the application map is 0x20 misc, 0x21 +/// aop-audio, 0x22 aop-voicetrigger, 0x23 als and 0x2b aop-audprov, which +/// are the ones macOS starts as well; the rest are started if they will. +const HW_CFG_T8140: AopHwConfig = AopHwConfig { + ec0p: 0, + aopt: 0, + alig: 0, + patch_bootargs: false, + epic_v4: true, + setup_port: true, + quiesce_on_unbind: true, + required_endpoints: &[0x20, 0x21, 0x22, 0x23, 0x2b], }; kernel::of_device_table!( @@ -1881,6 +1928,7 @@ kernel::of_device_table!( (of::DeviceId::new(c_str!("apple,t8112-aop")), &HW_CFG_T8112), (of::DeviceId::new(c_str!("apple,t6000-aop")), &HW_CFG_T6000), (of::DeviceId::new(c_str!("apple,t6020-aop")), &HW_CFG_T6020), + (of::DeviceId::new(c_str!("apple,t8140-aop")), &HW_CFG_T8140), ] ); @@ -1913,15 +1961,17 @@ impl platform::Driver for AopDriver { // registering; the same teardown as unbind's cleans that up. let probe_guard = ScopeGuard::new_with_data(data.clone(), |data| data.remove()); let aop_mmio = aop_mmio.as_view(); - data.patch_bootargs( - &aop_mmio, - &[ - (from_fourcc(b"EC0p"), cfg.ec0p), - (from_fourcc(b"nCal"), 0x0), - (from_fourcc(b"alig"), cfg.alig), - (from_fourcc(b"AOPt"), cfg.aopt), - ], - )?; + if cfg.patch_bootargs { + data.patch_bootargs( + &aop_mmio, + &[ + (from_fourcc(b"EC0p"), cfg.ec0p), + (from_fourcc(b"nCal"), 0x0), + (from_fourcc(b"alig"), cfg.alig), + (from_fourcc(b"AOPt"), cfg.aopt), + ], + )?; + } let rtkit = rtkit::RtKit::::new(pdev.as_ref(), None, 0, data.clone())?; data.rtkit.lock().as_mut().set(Some(rtkit)); if cfg.setup_port { From bf5be56eb7b001a3c2993f3bb16630e43987e573 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:43:46 +1000 Subject: [PATCH 0450/2680] soc: apple: aop: Send the T8140 ALS its calibration over the setup port The T8140 AOP firmware does not start the ambient light sensor until the application processor has sent the sensor's calibration to setup-port endpoint 0x21. Without it the sensor answers property reads and accepts its reporting interval, and then never reports anything. Add the setup port's request interface: a payload is copied into the endpoint's message page, (3 << 60) | length announces it, the firmware answers with a result word after writing a status byte at the start of the reply window, and 4 << 60 releases the request. An endpoint takes one request at a time. A request that times out leaves the endpoint busy until its late reply arrives, so that a later request cannot be completed by it; a result word other than the ones the caller accepts is an error and the request is not released. Waits are uninterruptible and bounded. Load the calibration through the firmware loader from the file named by the ALS node's firmware-name property and send it before the AFK endpoints start, so it precedes any client of the ALS service. The file is an 80-byte message: a 64-bit operation code, a 64-bit body length of 56, and 64 bytes of data, which are checked for shape only and sent as captured; the firmware answers it with 0x2000000000000004 rather than the 0x2000000000000001 of other requests. The calibration is specific to the machine and cannot be distributed with the kernel, so no MODULE_FIRMWARE names it. A node without the property or a file that is not installed leaves the sensor off with a message, not the AOP down. Select RUST_FW_LOADER_ABSTRACTIONS, which kernel::firmware needs; APPLE_AOP=m with IIO_AOP_SENSOR_ALS=n builds again. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 1 + drivers/soc/apple/aop.rs | 225 +++++++++++++++++++++++++++++++++++++- 2 files changed, 224 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 0a92213f76c46..20b972dac8fcd 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -103,6 +103,7 @@ config APPLE_AOP depends on RUST select RUST_APPLE_RTKIT select RUST_APPLE_MAILBOX + select RUST_FW_LOADER_ABSTRACTIONS help A co-processor persent on certain Apple SoCs controlling accelerometers, gyros, ambient light sensors and microphones. Is not actually always on. diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index b272b921ee664..1ca6b643d6ec5 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -28,6 +28,7 @@ use kernel::{ from_err_ptr, to_result, // }, + firmware::Firmware, fmt, io::{ io_project, @@ -54,6 +55,7 @@ use kernel::{ self, ASC_CPU_CONTROL, // }, + str::CString, sync::{ aref::ARef, atomic::{ @@ -154,6 +156,26 @@ const SETUP_BUFFER_ENTRY_SIZE: usize = 16; /// them have theirs. const SETUP_ENDPOINTS: [u8; 5] = [0x20, 0x21, 0x23, 0x30, 0x32]; const SETUP_BOOT_TIMEOUT_MS: u32 = 15000; +// A request to a service endpoint: the payload goes into the endpoint's +// message page, (3 << 60) | length announces it, the firmware answers with a +// result word after writing a status byte at the start of the reply window, +// and 4 << 60 releases the request. +const SETUP_REQUEST_TYPE: u64 = 3 << 60; +const SETUP_REQUEST_DONE: u64 = 4 << 60; +/// The result word of a completed request. +const SETUP_REPLY_READY: u64 = 0x2000000000000001; +/// The result word the ALS calibration is answered with instead. +const SETUP_REPLY_READY_ALT: u64 = 0x2000000000000004; +const SETUP_REPLY_TIMEOUT_MS: u32 = 5000; +/// The ambient light sensor's setup-port endpoint. The firmware does not +/// start the sensor until its calibration has been sent here. +const SETUP_ALS_EP: u8 = 0x21; +/// The calibration is an 80-byte message: a 64-bit operation code, a 64-bit +/// body length of 56, and 64 bytes of data that are sent as captured (the +/// last eight lie beyond the stated body length). +const ALS_CALIBRATION_LEN: usize = 80; +const ALS_CALIBRATION_OPERATION: u64 = 0x0746_b8d6_6515_2e31; +const ALS_CALIBRATION_BODY_LEN: u64 = 56; fn align_up(v: usize, a: usize) -> usize { (v + a - 1) & !(a - 1) @@ -177,6 +199,11 @@ fn le_u32(b: &[u8], off: usize) -> u32 { u32::from_le_bytes([b[off], b[off + 1], b[off + 2], b[off + 3]]) } +/// Reads the little-endian `u64` at `off`; `b` must hold it. +fn le_u64(b: &[u8], off: usize) -> u64 { + u64::from(le_u32(b, off)) | u64::from(le_u32(b, off + 4)) << 32 +} + #[inline(always)] fn mem_sync() { unsafe { @@ -882,10 +909,18 @@ struct ListenerEntry { listener: Arc, } -/// One of the setup port's service endpoints, once it has its buffers. +/// One of the setup port's service endpoints, once it has its buffers: a +/// host-to-AOP message page and an AOP-to-host reply window in the arena. #[derive(Clone, Copy)] struct SetupEndpoint { ep: u8, + tx_iova: u64, + rx_iova: u64, + /// A request is outstanding; the endpoint takes one at a time. + busy: bool, + /// The outstanding request timed out. The endpoint stays busy until the + /// reply arrives, so that it cannot be taken for a later request's. + abandoned: bool, /// The last word received on the endpoint that was not a buffer request. reply: Option, } @@ -930,6 +965,25 @@ impl SetupState { Ok(self.arena.as_ref().ok_or(ENXIO)?.dma_handle()) } + /// Returns a pointer to `len` bytes at `off` in the arena after checking + /// that they lie inside it. + fn arena_ptr(&self, off: usize, len: usize) -> Result<*mut u8> { + let arena = self.arena.as_ref().ok_or(ENXIO)?; + let end = off.checked_add(len).ok_or(EINVAL)?; + if end > arena.size() { + return Err(EINVAL); + } + // SAFETY: `off + len` does not exceed the size of the allocation, so + // the offset pointer stays inside it. + Ok(unsafe { arena.as_mut_ptr().cast::().add(off) }) + } + + /// The offset of an endpoint buffer in the arena. + fn arena_offset(&self, iova: u64) -> Result { + let off = iova.checked_sub(self.arena_iova()?).ok_or(EINVAL)?; + usize::try_from(off).map_err(|_| EINVAL) + } + /// Boot is complete once the AP power state is on and every service /// endpoint has its buffers. fn ready(&self) -> bool { @@ -1695,6 +1749,19 @@ impl AopData { return self.setup_endpoint_request(st, ep, word); } if let Some(endpoint) = st.endpoint_mut(ep) { + if endpoint.abandoned { + // The late reply to a request that timed out; the endpoint + // takes requests again. + dev_warn!( + self.dev, + "setup port: late reply {:#x} on endpoint {:#x}", + word, + ep + ); + endpoint.abandoned = false; + endpoint.busy = false; + return Ok(()); + } endpoint.reply = Some(word); return Ok(()); } @@ -1730,7 +1797,14 @@ impl AopData { st.send(ep, SETUP_BUFFER_REQUEST_ACK)?; st.send(ep, SetupState::shared_descriptor(tx_iova, 1, true))?; st.send(ep, SetupState::shared_descriptor(rx_iova, rx_pages, false))?; - st.endpoints[slot] = Some(SetupEndpoint { ep, reply: None }); + st.endpoints[slot] = Some(SetupEndpoint { + ep, + tx_iova, + rx_iova, + busy: false, + abandoned: false, + reply: None, + }); dev_dbg!( self.dev, "setup port: endpoint {:#x} tx {:#x} rx {:#x} ({} pages)", @@ -1769,6 +1843,148 @@ impl AopData { } } + /// Sends `payload` as a request on setup-port endpoint `ep`, waits for + /// the firmware's result word, and returns the status byte the firmware + /// wrote to the reply window if the result is one of `accepted`. An + /// endpoint takes one request at a time (EBUSY otherwise). After a + /// timeout the endpoint stays busy until the late reply arrives. + fn setup_request(&self, ep: u8, payload: &[u8], accepted: &[u64]) -> Result { + if payload.len() > SETUP_PAGE { + return Err(EMSGSIZE); + } + let mut st = self.setup.lock(); + let endpoint = st.endpoint_mut(ep).ok_or(ENXIO)?; + if endpoint.busy { + return Err(EBUSY); + } + endpoint.busy = true; + endpoint.reply = None; + let (tx_iova, rx_iova) = (endpoint.tx_iova, endpoint.rx_iova); + let ret = self.setup_request_locked(&mut st, ep, tx_iova, rx_iova, payload, accepted); + if let Some(endpoint) = st.endpoint_mut(ep) { + if ret == Err(ETIMEDOUT) { + endpoint.abandoned = true; + } else { + endpoint.busy = false; + } + } + ret + } + + fn setup_request_locked( + &self, + st: &mut MutexGuard<'_, SetupState>, + ep: u8, + tx_iova: u64, + rx_iova: u64, + payload: &[u8], + accepted: &[u64], + ) -> Result { + let tx_off = st.arena_offset(tx_iova)?; + let rx_off = st.arena_offset(rx_iova)?; + let tx = st.arena_ptr(tx_off, payload.len())?; + // SAFETY: `tx` is valid for `payload.len()` bytes of the arena, and + // the message page is the host's until the request word below is + // sent; the endpoint is busy, so nothing else writes it. + unsafe { ptr::copy_nonoverlapping(payload.as_ptr(), tx, payload.len()) }; + mem_sync(); + st.send(ep, SETUP_REQUEST_TYPE | payload.len() as u64)?; + let replied = |st: &SetupState| { + st.endpoints + .iter() + .flatten() + .any(|e| e.ep == ep && e.reply.is_some()) + }; + if let Err(e) = self.setup_wait(st, SETUP_REPLY_TIMEOUT_MS, replied) { + dev_err!( + self.dev, + "setup port: no reply to the request on endpoint {:#x}: {:?}", + ep, + e + ); + return Err(e); + } + let reply = st + .endpoint_mut(ep) + .and_then(|e| e.reply.take()) + .ok_or(EIO)?; + if !accepted.contains(&reply) { + dev_err!( + self.dev, + "setup port: request on endpoint {:#x} answered with {:#x}", + ep, + reply + ); + return Err(EIO); + } + mem_sync(); + let rx = st.arena_ptr(rx_off, 1)?; + // SAFETY: `rx` is valid for one byte of the arena. The firmware wrote + // the status before it sent the reply and does not touch the window + // again until the next request. + let status = unsafe { rx.read_volatile() }; + st.send(ep, SETUP_REQUEST_DONE)?; + Ok(status) + } + + /// Sends the ambient light sensor its calibration, named by the ALS + /// node's firmware-name, before the AFK endpoints start. Without it the + /// sensor answers property reads and accepts its reporting interval but + /// never reports; the trusted side does not start it. A missing file is + /// not an error: the AOP's other services work without it. + fn setup_als_calibration(&self) -> Result<()> { + let Some(name) = self + .dev + .fwnode() + .and_then(|node| node.get_child_by_name(c_str!("als"))) + .and_then(|als| { + als.property_read::(c_str!("firmware-name")) + .optional() + }) + else { + dev_info!( + self.dev, + "no ALS calibration named in the device tree; the sensor stays off" + ); + return Ok(()); + }; + let calibration = match Firmware::request_nowarn(&name, &self.dev) { + Ok(calibration) => calibration, + Err(e) => { + dev_warn!( + self.dev, + "ALS calibration {:?} not available ({:?}); the sensor stays off", + &*name, + e + ); + return Ok(()); + } + }; + let data = calibration.data(); + if data.len() != ALS_CALIBRATION_LEN + || le_u64(data, 0) != ALS_CALIBRATION_OPERATION + || le_u64(data, 8) != ALS_CALIBRATION_BODY_LEN + { + dev_err!( + self.dev, + "ALS calibration {:?} is not an {}-byte calibration message", + &*name, + ALS_CALIBRATION_LEN + ); + return Err(EINVAL); + } + let status = self.setup_request( + SETUP_ALS_EP, + data, + &[SETUP_REPLY_READY, SETUP_REPLY_READY_ALT], + )?; + if status != 0 { + dev_err!(self.dev, "ALS calibration rejected (status {:#x})", status); + return Err(EIO); + } + Ok(()) + } + /// After the RTKit side has reached AP power on: requests the setup /// port's AP power state and waits for it and for the five endpoint /// buffers. @@ -1985,6 +2201,11 @@ impl platform::Driver for AopDriver { // then the AFK endpoints. data.wake()?; data.setup_finish_boot()?; + // Before any client reaches the ALS service: a failure here costs + // the sensor, not the AOP. + if let Err(e) = data.setup_als_calibration() { + dev_warn!(pdev.as_ref(), "ALS calibration failed ({:?})", e); + } data.start_afk()?; } else { data.start()?; From a7ce0e97d89f4373c95763dab242b8bcf517c337 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:44:04 +1000 Subject: [PATCH 0451/2680] iio: common: aop_sensors: Add CT817 support The T8140 (J700) ambient light sensor is a CT817. The AOP reports it by its part number rather than by a code name, and its report differs from the CT720's despite the shared family: it is 32 bytes with two raw channel floats at +0x08 and +0x0c and the lux float at +0x1c, one byte below the CT720's offset. Its calibration is not an ALS property but a setup-port message that the AOP driver sends before the endpoints start, so the driver does not request the legacy calibration file for it. What arms the sensor is the zero to non-zero transition of its report interval, which the sensor accepts only after its batch and raw interval properties have been written (with the values 1 and 0 that it runs with on the T8140) and its verbosity set to 6. The sensor answers each of these writes with a not-permitted status yet applies them, so the status is logged and only a transport failure aborts the probe. Fetching the HID report descriptor on this endpoint is never answered, and writing the MODE (0xd7) or 0xe4 properties wedges the AOP for the rest of the boot, so neither is done. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iio/common/aop_sensors/aop_als.rs | 76 +++++++++++++++++++++-- 1 file changed, 70 insertions(+), 6 deletions(-) diff --git a/drivers/iio/common/aop_sensors/aop_als.rs b/drivers/iio/common/aop_sensors/aop_als.rs index 2711da61f51e3..91170c2c1cd06 100644 --- a/drivers/iio/common/aop_sensors/aop_als.rs +++ b/drivers/iio/common/aop_sensors/aop_als.rs @@ -16,14 +16,44 @@ use kernel::{ const EPIC_SUBTYPE_GET_AOP_PROPERTY: u16 = 0xa; const EPIC_SUBTYPE_SET_ALS_PROPERTY: u16 = 0x4; + +/* ALS properties. */ +const ALS_PROP_INTERVAL: u32 = 0x00; +const ALS_PROP_CALIBRATION: u32 = 0x0b; +const ALS_PROP_BATCH_INTERVAL: u32 = 0x16; +const ALS_PROP_RAW_INTERVAL: u32 = 0x5c; +const ALS_PROP_VERBOSITY: u32 = 0xe1; + +/* + * The CT817 starts reporting on the zero to non-zero transition of its + * report interval, and it accepts that transition only once its batch and + * raw interval properties have been written. Despite their names these two + * are not intervals; they are the values the sensor is configured with on + * the T8140. The sensor answers every one of these writes with a + * not-permitted status and applies them regardless, so the status is only + * logged. + */ +const ALS_BATCH_INTERVAL: u32 = 1; +const ALS_RAW_INTERVAL: u32 = 0; +const ALS_VERBOSITY: u32 = 6; +/// The reporting interval in microseconds. +const ALS_INTERVAL_US: u32 = 200000; const LUX_OFFSET_CT720: usize = 0x1d; const LUX_OFFSET_VD6286: usize = 0x28; +/* + * The CT817 sends a 32-byte report: two raw channel floats at +0x08 and + * +0x0c and the lux float at +0x1c, one byte below the CT720's, so the two + * parts do not share a layout. + */ +const LUX_OFFSET_CT817: usize = 0x1c; fn get_lux_offset(aop: &dyn AOP, dev: &platform::Device, svc: &EPICService) -> Result { let name = get_aop_property(aop, svc, 0xf, 16)?.1; match name.as_slice() { b"Redbird\0" => Ok(LUX_OFFSET_VD6286), b"FireFish2\0" => Ok(LUX_OFFSET_CT720), + // The T8140 AOP reports the part number rather than a code name. + b"CT817\0" => Ok(LUX_OFFSET_CT817), _ => { dev_warn!( dev.as_ref(), @@ -35,12 +65,46 @@ fn get_lux_offset(aop: &dyn AOP, dev: &platform::Device, svc: &EPICService) -> R } } -fn enable_als(aop: &dyn AOP, dev: &platform::Device, svc: &EPICService) -> Result<()> { - let fw = Firmware::request(c_str!("apple/aop-als-cal.bin"), dev.as_ref())?; - set_als_property(aop, svc, 0xb, fw.data())?; - set_als_property(aop, svc, 0, &200000u32.to_le_bytes())?; +fn enable_als( + aop: &dyn AOP, + dev: &platform::Device, + svc: &EPICService, + offset: usize, +) -> Result<()> { + if offset != LUX_OFFSET_CT817 { + let fw = Firmware::request(c_str!("apple/aop-als-cal.bin"), dev.as_ref())?; + set_als_property(aop, svc, ALS_PROP_CALIBRATION, fw.data())?; + set_als_property(aop, svc, ALS_PROP_INTERVAL, &ALS_INTERVAL_US.to_le_bytes())?; + return Ok(()); + } - Ok(()) + // The CT817 gets its calibration from the AOP driver over the setup port, + // not as an ALS property. Only the transport can fail here; the sensor's + // status words are logged, see the property comments above. Do not fetch + // the HID report descriptor (subtype 0x01) on this endpoint: the AOP never + // answers it, and do not write the MODE (0xd7) or 0xe4 properties: either + // wedges the AOP for the rest of the boot. + let set = |tag: u32, val: u32, what: &str| -> Result<()> { + let status = set_als_property(aop, svc, tag, &val.to_le_bytes())?; + dev_dbg!( + dev.as_ref(), + "{} = {:#x}: status {:#x}\n", + what, + val, + status + ); + Ok(()) + }; + set(ALS_PROP_VERBOSITY, ALS_VERBOSITY, "verbosity")?; + set( + ALS_PROP_BATCH_INTERVAL, + ALS_BATCH_INTERVAL, + "batch interval", + )?; + set(ALS_PROP_RAW_INTERVAL, ALS_RAW_INTERVAL, "raw interval")?; + // The interval is zero at probe, so this is the transition that arms the + // sensor. + set(ALS_PROP_INTERVAL, ALS_INTERVAL_US, "interval") } fn get_aop_property( @@ -137,7 +201,7 @@ impl platform::Driver for IIOAopAlsDriver { let offset = get_lux_offset(adata.as_ref(), pdev, &service)?; let data = AopSensorData::new(dev.into(), ty, MsgProc(offset))?; let listener = FakehidRegistration::new(adata.clone(), service, data.clone())?; - enable_als(adata.as_ref(), pdev, &service)?; + enable_als(adata.as_ref(), pdev, &service, offset)?; let info_mask = 1 << bindings::BINDINGS_IIO_CHAN_INFO_PROCESSED; let iio = IIORegistration::::new(data, c"aop-sensors-als", ty, info_mask, &THIS_MODULE)?; From 667f2b3fbbaa8533c3ff7e18f51fe9bf5e5ac6b5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:46:10 +1000 Subject: [PATCH 0452/2680] soc: apple: aop: Dispatch EPIC reports to registered listeners The AOP delivers only fake-HID reports to child drivers; any other report subtype is dropped. The T8140 low-power microphone service reports its producer position as EPIC reports of subtype 0x20, which the AOP audio driver needs to follow the firmware's writes into the source ring. Add ReportListener to the AOP client API together with add_report_listener() and remove_report_listener(), keyed by service and subtype, with one listener per key. Reports that no listener claims keep being dropped and counted like other unhandled messages. The documentation of both listener traits now states the constraints the callbacks run under: the RTKit lock, the endpoint lock and the listener list lock are held, so a callback may neither sleep for long nor call back into the AOP, and removal returns only once no callback into the listener runs any more. Listeners are cleared on removal after the children have been unbound. The T8140 AOP audio driver, submitted separately, is the user. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 64 +++++++++++++++++++++++++++++++++++- rust/kernel/soc/apple/aop.rs | 25 ++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 1ca6b643d6ec5..5ac2d104a77ba 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -48,6 +48,7 @@ use kernel::{ from_fourcc, EPICService, FakehidListener, + ReportListener, AOP, // }, soc::apple::mailbox, @@ -696,8 +697,11 @@ impl AFKEndpoint { return Into::>::into(client).register_service(self, chan, name); } else if subtype == EPIC_SUBTYPE_FAKEHID_REPORT { return client.process_fakehid_report(self, qhdr.channel, data); + } else if client.process_report(self, qhdr.channel, subtype, data)? { + return Ok(()); } - return Err(EINVAL); + // A report nobody listens for. + return Err(ENOENT); } else if ehdr.category == EPIC_CATEGORY_REPLY { if subtype == EPIC_SUBTYPE_RETCODE_PAYLOAD || subtype == EPIC_SUBTYPE_RETCODE @@ -909,6 +913,12 @@ struct ListenerEntry { listener: Arc, } +struct ReportListenerEntry { + svc: EPICService, + subtype: u16, + listener: Arc, +} + /// One of the setup port's service endpoints, once it has its buffers: a /// host-to-AOP message page and an AOP-to-host reply window in the arena. #[derive(Clone, Copy)] @@ -1162,6 +1172,8 @@ struct AopData { #[pin] hid_listeners: Mutex>, #[pin] + report_listeners: Mutex>, + #[pin] subdevices: Mutex>, } @@ -1290,6 +1302,7 @@ impl AopData { call_turn <- pin_init::pin_init_array_from_fn(|_| new_mutex!(())), ep_shutdown <- pin_init::pin_init_array_from_fn(|_| FutureValue::pin_init()), hid_listeners <- new_mutex!(KVec::new()), + report_listeners <- new_mutex!(KVec::new()), subdevices <- new_mutex!(KVec::new()), } ), @@ -1370,6 +1383,19 @@ impl AopData { Ok(()) } + /// Hands a report to its listener; `Ok(false)` when there is none. + fn process_report(&self, ep: &AFKEndpoint, ch: u32, subtype: u16, data: &[u8]) -> Result { + let guard = self.report_listeners.lock(); + for entry in &*guard { + if entry.svc.endpoint == ep.index && entry.svc.channel == ch && entry.subtype == subtype + { + entry.listener.process_report(subtype, data)?; + return Ok(true); + } + } + Ok(false) + } + fn process_fakehid_report(&self, ep: &AFKEndpoint, ch: u32, data: &[u8]) -> Result<()> { let guard = self.hid_listeners.lock(); for entry in &*guard { @@ -1569,6 +1595,41 @@ impl AOP for AopData { } false } + fn add_report_listener( + &self, + svc: EPICService, + subtype: u16, + listener: Arc, + ) -> Result<()> { + let mut guard = self.report_listeners.lock(); + if self.removing.load(Acquire) { + return Err(ENODEV); + } + if guard + .iter() + .any(|entry| entry.svc == svc && entry.subtype == subtype) + { + return Err(EBUSY); + } + Ok(guard.push( + ReportListenerEntry { + svc, + subtype, + listener, + }, + GFP_KERNEL, + )?) + } + fn remove_report_listener(&self, svc: &EPICService, subtype: u16) -> bool { + let mut guard = self.report_listeners.lock(); + for i in 0..guard.len() { + if guard[i].svc == *svc && guard[i].subtype == subtype { + guard.swap_remove(i); + return true; + } + } + false + } /// Takes the AOP down: from unbind, from a failed probe, or as a fallback /// from Drop. Only the first call does anything. fn remove(&self) { @@ -1589,6 +1650,7 @@ impl AOP for AopData { child.release_driver(); } self.hid_listeners.lock().clear(); + self.report_listeners.lock().clear(); self.transport_closing.store(true, Release); if let Err(e) = self.stop() { dev_err!(self.dev, "Failed to stop AOP {:?}", e); diff --git a/rust/kernel/soc/apple/aop.rs b/rust/kernel/soc/apple/aop.rs index 0de0d9acf6372..52a77f61f03af 100644 --- a/rust/kernel/soc/apple/aop.rs +++ b/rust/kernel/soc/apple/aop.rs @@ -26,11 +26,25 @@ pub struct EPICService { /// /// The AOP driver keeps the listener in an [`Arc`] shared with the child driver that registered /// it and invokes it from its own RTKit receive worker, so listeners have to be `Send + Sync`. +/// The callback runs with the AOP's RTKit lock, the endpoint's lock and the listener list lock +/// held: it must not sleep for long, must not call into the AOP (no [`AOP::epic_call`] and no +/// listener registration), and it keeps running until the listener is removed. pub trait FakehidListener: Send + Sync { /// Process the event. fn process_fakehid_report(&self, data: &[u8]) -> Result<()>; } +/// Listener for the EPIC reports of a service other than the fake-HID ones, such as the producer +/// reports (subtype 0x20) of the T8140 low-power microphone service. +/// +/// The same rules as for [`FakehidListener`] apply: the callback runs from the AOP's RTKit +/// receive worker with the RTKit lock, the endpoint's lock and the listener list lock held, so +/// it must not sleep for long or call into the AOP, and it runs until the listener is removed. +pub trait ReportListener: Send + Sync { + /// Process the report payload (everything after the EPIC headers). + fn process_report(&self, subtype: u16, data: &[u8]) -> Result<()>; +} + /// AOP communications manager. pub trait AOP: Send + Sync { /// Calls a method on a specified service @@ -54,6 +68,17 @@ pub trait AOP: Send + Sync { /// Removes the listener for the specified service. Returns once no callback into the /// listener is running any more. fn remove_fakehid_listener(&self, svc: &EPICService) -> bool; + /// Adds a report listener for one report subtype of the specified service. A service and + /// subtype take one listener at a time; a second registration fails with `EBUSY`. + fn add_report_listener( + &self, + svc: EPICService, + subtype: u16, + listener: Arc, + ) -> Result<()>; + /// Removes the report listener for the specified service and subtype. Returns once no + /// callback into the listener is running any more. + fn remove_report_listener(&self, svc: &EPICService, subtype: u16) -> bool; /// Internal method to detach the device. fn remove(&self); } From aa36276cd76ff897dd47e0d324e8f116b63773fe Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:47:03 +1000 Subject: [PATCH 0453/2680] soc: apple: aop: Add the T8140 low-power microphone source ring The T8140 AOP firmware produces the low-power microphone's audio frames into a ring in host memory that the application processor hands it over setup-port endpoint 0x20: a 24-byte request of the operation code 0x6b803ce492bd9547, the ring's IOVA and its size. The ring is DMA of the audio child device, whose node carries the DART stream the firmware reaches it through. The firmware accepts one such binding per boot; a second one is answered with 0x2000000000000009. So the ring cannot belong to the audio driver, which may be bound and unbound repeatedly: the AOP core owns it and hands the same ring to every probe. Add SourceRing to the AOP client API and AOP::source_ring(), which allocates the ring on the audio child on first use, binds it, and returns it, or the existing ring, thereafter. The binding is recorded before the firmware learns the address; if the request fails, the ring is kept and never bound again, since even a timeout may mean the firmware took it, and later calls fail with EIO. The ring's buffer is not exposed: SourceRing::read() copies a span out, and its documentation states that only a span the firmware has published through its producer report is meaningful. On removal the ring is freed after the audio child is unbound and before it is unregistered, while its IOMMU domain still exists, and it is retained when the shutdown is unconfirmed. The T8140 AOP audio driver, submitted separately, is the user. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 70 +++++++++++++++++++++++++++++++++++- rust/kernel/soc/apple/aop.rs | 54 +++++++++++++++++++++++++++- 2 files changed, 122 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 5ac2d104a77ba..90cbfc95eefa2 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -18,7 +18,7 @@ use kernel::{ bindings, c_str, device, - device::Core, + device::{Bound, Core}, dma::{ Coherent, Device, @@ -49,6 +49,7 @@ use kernel::{ EPICService, FakehidListener, ReportListener, + SourceRing, AOP, // }, soc::apple::mailbox, @@ -171,6 +172,11 @@ const SETUP_REPLY_TIMEOUT_MS: u32 = 5000; /// The ambient light sensor's setup-port endpoint. The firmware does not /// start the sensor until its calibration has been sent here. const SETUP_ALS_EP: u8 = 0x21; +/// The low-power microphone's setup-port endpoint, which takes the source +/// ring binding: a 24-byte request of this operation code, the ring's IOVA +/// and its size. +const SETUP_SOURCE_EP: u8 = 0x20; +const SETUP_SET_SOURCE_BUFFER: u64 = 0x6b80_3ce4_92bd_9547; /// The calibration is an 80-byte message: a 64-bit operation code, a 64-bit /// body length of 56, and 64 bytes of data that are sent as captured (the /// last eight lie beyond the stated body length). @@ -919,6 +925,14 @@ struct ReportListenerEntry { listener: Arc, } +/// The source ring and whether the firmware is known to have taken it. +struct SourceRingBinding { + ring: Arc, + /// The bind request completed. Until then the firmware may or may not + /// hold the ring, so it is neither freed nor bound again. + confirmed: bool, +} + /// One of the setup port's service endpoints, once it has its buffers: a /// host-to-AOP message page and an AOP-to-host reply window in the arena. #[derive(Clone, Copy)] @@ -1174,6 +1188,8 @@ struct AopData { #[pin] report_listeners: Mutex>, #[pin] + source_ring: Mutex>, + #[pin] subdevices: Mutex>, } @@ -1303,6 +1319,7 @@ impl AopData { ep_shutdown <- pin_init::pin_init_array_from_fn(|_| FutureValue::pin_init()), hid_listeners <- new_mutex!(KVec::new()), report_listeners <- new_mutex!(KVec::new()), + source_ring <- new_mutex!(None), subdevices <- new_mutex!(KVec::new()), } ), @@ -1487,6 +1504,9 @@ impl AopData { if let Some(arena) = self.setup.lock().arena.take() { mem::forget(arena); } + if let Some(binding) = self.source_ring.lock().take() { + mem::forget(binding); + } } } @@ -1630,6 +1650,36 @@ impl AOP for AopData { } false } + fn source_ring(&self, dev: &device::Device, size: usize) -> Result> { + if self.setup_queue.is_none() || self.transport_closing.load(Acquire) { + return Err(ENODEV); + } + let mut guard = self.source_ring.lock(); + if let Some(binding) = guard.as_ref() { + if !binding.confirmed { + return Err(EIO); + } + if binding.ring.size() < size { + return Err(EBUSY); + } + return Ok(binding.ring.clone()); + } + let buf = Coherent::::zeroed_slice(dev, size, GFP_KERNEL)?; + let ring = Arc::new(SourceRing::new(buf), GFP_KERNEL)?; + let iova = ring.iova; + // The binding is recorded before the firmware learns the address: + // whatever the request's outcome, the ring is kept and not bound + // again, since even a timeout may mean the firmware took it. + *guard = Some(SourceRingBinding { + ring: ring.clone(), + confirmed: false, + }); + self.setup_bind_source(iova, size as u64)?; + if let Some(binding) = &mut *guard { + binding.confirmed = true; + } + Ok(ring) + } /// Takes the AOP down: from unbind, from a failed probe, or as a fallback /// from Drop. Only the first call does anything. fn remove(&self) { @@ -1704,6 +1754,10 @@ impl AOP for AopData { } let arena = self.setup.lock().arena.take(); drop(arena); + // The ring is DMA of the audio child; free it while the child is + // still registered and keeps its IOMMU domain. + let source = self.source_ring.lock().take(); + drop(source); for child in children { child.unregister(); } @@ -1989,6 +2043,20 @@ impl AopData { Ok(status) } + /// Binds the source ring to the firmware: one bind per boot. + fn setup_bind_source(&self, iova: u64, size: u64) -> Result<()> { + let mut req = [0u8; 24]; + req[..8].copy_from_slice(&SETUP_SET_SOURCE_BUFFER.to_le_bytes()); + req[8..16].copy_from_slice(&iova.to_le_bytes()); + req[16..].copy_from_slice(&size.to_le_bytes()); + let status = self.setup_request(SETUP_SOURCE_EP, &req, &[SETUP_REPLY_READY])?; + if status != 0 { + dev_err!(self.dev, "source ring bind rejected (status {:#x})", status); + return Err(EIO); + } + Ok(()) + } + /// Sends the ambient light sensor its calibration, named by the ALS /// node's firmware-name, before the AFK endpoints start. Without it the /// sensor answers property reads and accepts its reporting interval but diff --git a/rust/kernel/soc/apple/aop.rs b/rust/kernel/soc/apple/aop.rs index 52a77f61f03af..d49146db2c71a 100644 --- a/rust/kernel/soc/apple/aop.rs +++ b/rust/kernel/soc/apple/aop.rs @@ -3,7 +3,8 @@ //! Common code for AOP endpoint drivers use kernel::{ - device::Device, + device::{Bound, Device}, + dma::Coherent, prelude::*, sync::{ atomic::{Atomic, Relaxed}, @@ -45,6 +46,51 @@ pub trait ReportListener: Send + Sync { fn process_report(&self, subtype: u16, data: &[u8]) -> Result<()>; } +/// The T8140 low-power microphone source ring: a buffer the AOP firmware produces audio frames +/// into, mapped through the audio child's IOMMU stream. +/// +/// The firmware accepts one ring per boot (a second binding is answered with +/// 0x2000000000000009), so the AOP core owns the allocation and hands the same ring to every +/// probe of the audio driver. +pub struct SourceRing { + buf: Coherent<[u8]>, + /// The IOVA the firmware produces into. + pub iova: u64, +} + +impl SourceRing { + /// Wraps a coherent allocation as the source ring. + pub fn new(buf: Coherent<[u8]>) -> Self { + let iova = buf.dma_handle(); + Self { buf, iova } + } + + /// Size of the ring in bytes. + pub fn size(&self) -> usize { + self.buf.size() + } + + /// Copies `dst.len()` bytes from offset `offset` of the ring into `dst`. + /// + /// The firmware writes the ring while it runs, so the copy is only meaningful for a span the + /// firmware has published through its producer report and has not yet reused; the caller + /// keeps within it and treats the data as untrusted. + pub fn read(&self, offset: usize, dst: &mut [u8]) -> Result<()> { + let end = offset.checked_add(dst.len()).ok_or(EINVAL)?; + if end > self.buf.size() { + return Err(EINVAL); + } + // SAFETY: `offset + dst.len()` does not exceed the allocation, so the source pointer and + // the copy stay inside it; `dst` is a distinct allocation. By the protocol above the + // firmware does not write the span while it is copied. + unsafe { + let src = self.buf.as_ptr().cast::().add(offset); + core::ptr::copy_nonoverlapping(src, dst.as_mut_ptr(), dst.len()); + } + Ok(()) + } +} + /// AOP communications manager. pub trait AOP: Send + Sync { /// Calls a method on a specified service @@ -79,6 +125,12 @@ pub trait AOP: Send + Sync { /// Removes the report listener for the specified service and subtype. Returns once no /// callback into the listener is running any more. fn remove_report_listener(&self, svc: &EPICService, subtype: u16) -> bool; + /// The T8140 low-power microphone source ring: `size` bytes allocated on and mapped through + /// `dev`, the audio child, and bound to the firmware over the setup port on first use, then + /// shared for the rest of the AOP's lifetime. Fails with `ENODEV` on firmware without a setup + /// port, with `EBUSY` when the bound ring is smaller than `size`, and with `EIO` after a + /// binding whose outcome is unknown. + fn source_ring(&self, dev: &Device, size: usize) -> Result>; /// Internal method to detach the device. fn remove(&self); } From 60631eb8a536a33140f1513bdfa0b78a29235955 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:10:50 +1000 Subject: [PATCH 0454/2680] ASoC: cs42l42: Add DSP_A support The CS42L42 and CS42L83 ASP supports a pulse-framed serial interface besides I2S: the frame starts on the rising edge of a one-SCLK LRCLK pulse, the first data bit follows 1.0 SCLKs later and the channels sit back to back within the frame. On Apple's T8140 (MacBook Neo, J700) the AOP firmware runs the CS42L83 jack link that way, with two 32-bit slots per frame, and the host cannot change the bus format. Accept SND_SOC_DAIFMT_DSP_A: program ASP_FRM_CFG for pulse mode with the frame start on the LRCLK rising edge (STP) and a 1.0 SCLK frame start delay, and place channel 2 one physical sample width after the frame start in the same LRCLK phase instead of in the high phase. Remember the format so hw_params can tell the two layouts apart. The channel bit-position registers are now written for I2S as well, with their power-on values, so that a codec that changes format between streams does not keep the DSP_A offsets. The bit-clock and frame inversion flags keep their meaning: a link that launches data on the rising SCLK edge and samples it on the falling edge is described as SND_SOC_DAIFMT_DSP_A | SND_SOC_DAIFMT_IB_NF. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/codecs/cs42l42.c | 51 +++++++++++++++++++++++++++++++++++--- sound/soc/codecs/cs42l42.h | 1 + 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/sound/soc/codecs/cs42l42.c b/sound/soc/codecs/cs42l42.c index 55455b91a7885..8fdf9f854aad9 100644 --- a/sound/soc/codecs/cs42l42.c +++ b/sound/soc/codecs/cs42l42.c @@ -828,6 +828,7 @@ static int cs42l42_asp_config(struct snd_soc_component *component, static int cs42l42_set_dai_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt) { struct snd_soc_component *component = codec_dai->component; + struct cs42l42_private *cs42l42 = snd_soc_component_get_drvdata(component); u32 asp_cfg_val = 0; switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { @@ -859,10 +860,27 @@ static int cs42l42_set_dai_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt) (CS42L42_ASP_FSD_1_0 << CS42L42_ASP_FSD_SHIFT)); break; + case SND_SOC_DAIFMT_DSP_A: + /* + * Pulse mode, frame starts on the rising edge of LRCLK, + * data delayed by 1.0 SCLKs. Both channels sit in the same + * LRCLK phase, one slot apart; hw_params places them. + */ + snd_soc_component_update_bits(component, + CS42L42_ASP_FRM_CFG, + CS42L42_ASP_STP_MASK | + CS42L42_ASP_5050_MASK | + CS42L42_ASP_FSD_MASK, + CS42L42_ASP_STP_MASK | + (CS42L42_ASP_FSD_1_0 << + CS42L42_ASP_FSD_SHIFT)); + break; default: return -EINVAL; } + cs42l42->dai_format = fmt & SND_SOC_DAIFMT_FORMAT_MASK; + /* Bitclock/frame inversion */ switch (fmt & SND_SOC_DAIFMT_INV_MASK) { case SND_SOC_DAIFMT_NB_NF: @@ -916,6 +934,7 @@ static int cs42l42_pcm_hw_params(struct snd_pcm_substream *substream, unsigned int width = (params_width(params) / 8) - 1; unsigned int sample_rate = params_rate(params); unsigned int slot_width = 0; + unsigned int ch2_bit = 0; unsigned int val = 0; unsigned int bclk; int ret; @@ -938,16 +957,32 @@ static int cs42l42_pcm_hw_params(struct snd_pcm_substream *substream, bclk = snd_soc_tdm_params_to_bclk(params, slot_width, 0, 2); } + /* + * In DSP_A both channels share the LRCLK phase and channel 2 starts + * one slot after channel 1. In I2S each channel sits at the start of + * its own LRCLK phase. + */ + if (cs42l42->dai_format == SND_SOC_DAIFMT_DSP_A) + ch2_bit = params_physical_width(params); + switch (substream->stream) { case SNDRV_PCM_STREAM_CAPTURE: - /* channel 2 on high LRCLK */ - val = CS42L42_ASP_TX_CH2_AP_MASK | - (width << CS42L42_ASP_TX_CH2_RES_SHIFT) | + val = (width << CS42L42_ASP_TX_CH2_RES_SHIFT) | (width << CS42L42_ASP_TX_CH1_RES_SHIFT); + /* channel 2 on high LRCLK */ + if (cs42l42->dai_format != SND_SOC_DAIFMT_DSP_A) + val |= CS42L42_ASP_TX_CH2_AP_MASK; snd_soc_component_update_bits(component, CS42L42_ASP_TX_CH_AP_RES, CS42L42_ASP_TX_CH1_AP_MASK | CS42L42_ASP_TX_CH2_AP_MASK | CS42L42_ASP_TX_CH2_RES_MASK | CS42L42_ASP_TX_CH1_RES_MASK, val); + + snd_soc_component_write(component, CS42L42_ASP_TX_CH1_BIT_MSB, 0); + snd_soc_component_write(component, CS42L42_ASP_TX_CH1_BIT_LSB, 0); + snd_soc_component_write(component, CS42L42_ASP_TX_CH2_BIT_MSB, + ch2_bit >> 8); + snd_soc_component_write(component, CS42L42_ASP_TX_CH2_BIT_LSB, + ch2_bit & 0xff); break; case SNDRV_PCM_STREAM_PLAYBACK: val |= width << CS42L42_ASP_RX_CH_RES_SHIFT; @@ -956,11 +991,19 @@ static int cs42l42_pcm_hw_params(struct snd_pcm_substream *substream, CS42L42_ASP_RX_CH_AP_MASK | CS42L42_ASP_RX_CH_RES_MASK, val); /* Channel 2 on high LRCLK */ - val |= CS42L42_ASP_RX_CH_AP_HI << CS42L42_ASP_RX_CH_AP_SHIFT; + if (cs42l42->dai_format != SND_SOC_DAIFMT_DSP_A) + val |= CS42L42_ASP_RX_CH_AP_HI << CS42L42_ASP_RX_CH_AP_SHIFT; snd_soc_component_update_bits(component, CS42L42_ASP_RX_DAI0_CH2_AP_RES, CS42L42_ASP_RX_CH_AP_MASK | CS42L42_ASP_RX_CH_RES_MASK, val); + snd_soc_component_write(component, CS42L42_ASP_RX_DAI0_CH1_BIT_MSB, 0); + snd_soc_component_write(component, CS42L42_ASP_RX_DAI0_CH1_BIT_LSB, 0); + snd_soc_component_write(component, CS42L42_ASP_RX_DAI0_CH2_BIT_MSB, + ch2_bit >> 8); + snd_soc_component_write(component, CS42L42_ASP_RX_DAI0_CH2_BIT_LSB, + ch2_bit & 0xff); + /* Channel B comes from the last active channel */ snd_soc_component_update_bits(component, CS42L42_SP_RX_CH_SEL, CS42L42_SP_RX_CHB_SEL_MASK, diff --git a/sound/soc/codecs/cs42l42.h b/sound/soc/codecs/cs42l42.h index 654491c42b3fa..7a9f2a5be5ef8 100644 --- a/sound/soc/codecs/cs42l42.h +++ b/sound/soc/codecs/cs42l42.h @@ -65,6 +65,7 @@ struct cs42l42_private { u8 hs_bias_ramp_time; u8 hs_bias_sense_en; u8 stream_use; + unsigned int dai_format; bool remote_active; unsigned int remote_saved[CS42L83_REMOTE_NUM_SAVED_REGS]; bool hp_adc_up_pending; From dd68aa9a48d9c8e4e6c39762b654cdc88678111e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:11:24 +1000 Subject: [PATCH 0455/2680] ASoC: cs42l42: Accept odd SCLK/LRCLK ratios in DSP modes cs42l42_asp_config() rejects any SCLK that is not an even multiple of the sample rate, because an I2S frame is split into two equal LRCLK phases. A pulse-framed DSP_A link has no such requirement: the LRCLK is a pulse and the channels are placed by their bit positions, so the frame may hold any whole number of SCLKs. Apple's T8140 AOP firmware clocks the CS42L83 jack link at 125 SCLKs per 48 kHz frame (6 MHz). Accept an odd ratio when the DAI is in DSP_A, and keep the pulse width at half the frame rounded down (62 SCLKs for that link), which is how the vendor driver programs it. I2S keeps rejecting odd ratios, as an odd frame cannot give it a 50/50 duty cycle. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/codecs/cs42l42.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sound/soc/codecs/cs42l42.c b/sound/soc/codecs/cs42l42.c index 8fdf9f854aad9..08ab73ef96560 100644 --- a/sound/soc/codecs/cs42l42.c +++ b/sound/soc/codecs/cs42l42.c @@ -788,10 +788,16 @@ EXPORT_SYMBOL_NS_GPL(cs42l42_src_config, "SND_SOC_CS42L42_CORE"); static int cs42l42_asp_config(struct snd_soc_component *component, unsigned int sclk, unsigned int sample_rate) { + struct cs42l42_private *cs42l42 = snd_soc_component_get_drvdata(component); u32 fsync = sclk / sample_rate; - /* Set up the LRCLK */ - if (((fsync * sample_rate) != sclk) || ((fsync % 2) != 0)) { + /* + * Set up the LRCLK. A 50/50 duty cycle needs an even number of + * SCLKs per frame; a pulse-framed (DSP) link has no such + * constraint and takes a pulse of half the frame, rounded down. + */ + if ((fsync * sample_rate) != sclk || + (cs42l42->dai_format != SND_SOC_DAIFMT_DSP_A && (fsync % 2) != 0)) { dev_err(component->dev, "Unsupported sclk %d/sample rate %d\n", sclk, From 2712153729795735d187157f98f788f7b5472878 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:12:34 +1000 Subject: [PATCH 0456/2680] ASoC: macaudio: Cancel the lock update work after the card is gone The speaker back-end trigger queues lock_update_work, and that work arms lock_timeout_work. remove() cancels only the delayed work, and it does so while the card is still registered: devres unregisters the card after remove() returns. A stream that is still running can therefore queue either work after the cancellation, and the work then runs on driver data that devres has freed. Register the card without devres and unregister it at the start of remove(), which closes every stream, so that no trigger can queue work any more. Then cancel the update work, which is the one that arms the timeout, and the timeout work after it. Compile-tested only; not run on hardware. Fixes: bad95a548bb7 ("macaudio: speaker volume safety interlocks") Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/macaudio.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index 6f4df9de67378..b9e83b7f91501 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -1729,13 +1729,19 @@ static int macaudio_snd_platform_probe(struct platform_device *pdev) INIT_WORK(&data->lock_update_work, macaudio_vlimit_update_work); INIT_DELAYED_WORK(&data->lock_timeout_work, macaudio_vlimit_timeout_work); - return devm_snd_soc_register_card(dev, card); + return snd_soc_register_card(card); } static void macaudio_snd_platform_remove(struct platform_device *pdev) { struct macaudio_snd_data *ma = dev_get_drvdata(&pdev->dev); + /* + * Unregister first: a back-end trigger queues lock_update_work, which + * arms lock_timeout_work, and neither may run once the card is gone. + */ + snd_soc_unregister_card(&ma->card); + cancel_work_sync(&ma->lock_update_work); cancel_delayed_work_sync(&ma->lock_timeout_work); } From 233aaa32bfc4ada75316cbe2b8cd806e1ab0c8c3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:13:16 +1000 Subject: [PATCH 0457/2680] ASoC: macaudio: Serialise the speaker lock control ops volume_lock_mutex guards the speaker lock state, but two of the "Speaker Volume Unlock" control ops still touch that state outside it. The put op reads speaker_lock_owner and clears speaker_volume_was_locked before taking the mutex, so a lock loss recorded by the timeout work between the check and the mutex can be consumed without ever being reported to the daemon, and the unlock op clears the owner and the timeout with no lock at all, racing the timeout and update works that read them. Take the mutex for the whole of both ops. Compile-tested only; not run on hardware. Fixes: d856390cce84 ("macaudio: Use an explicit mutex for the speaker volume lock") Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/macaudio.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index b9e83b7f91501..24e0090e1a374 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -1423,20 +1423,23 @@ static int macaudio_slk_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_v struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); - if (!ma->speaker_lock_owner) - return -EPERM; - if (uvalue->value.integer.value[0] != SPEAKER_MAGIC_VALUE) return -EINVAL; + mutex_lock(&ma->volume_lock_mutex); + + if (!ma->speaker_lock_owner) { + mutex_unlock(&ma->volume_lock_mutex); + return -EPERM; + } + /* Serves as a notification that the lock was lost at some point */ if (ma->speaker_volume_was_locked) { ma->speaker_volume_was_locked = false; + mutex_unlock(&ma->volume_lock_mutex); return -ETIMEDOUT; } - mutex_lock(&ma->volume_lock_mutex); - cancel_delayed_work(&ma->lock_timeout_work); ma->speaker_lock_remain = ms_to_ktime(SPEAKER_LOCK_TIMEOUT); @@ -1491,9 +1494,11 @@ static void macaudio_slk_unlock(struct snd_kcontrol *kcontrol) struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + mutex_lock(&ma->volume_lock_mutex); ma->speaker_lock_owner = NULL; ma->speaker_lock_timeout = 0; macaudio_vlimit_update(ma); + mutex_unlock(&ma->volume_lock_mutex); } /* From 00f4bc7bd1675a68869b9c10908b26def53a34a9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:13:45 +1000 Subject: [PATCH 0458/2680] ASoC: macaudio: Do not hand a lease to a new speaker lock owner The speaker volume is unlocked while the lock owner keeps pinging the "Speaker Volume Unlock" control; each ping grants a lease of SPEAKER_LOCK_TIMEOUT. Unlocking the control clears the owner but not the remaining lease, so a process that locks the control next inherits whatever was left and gets the volume unlocked before it has pinged once, for up to a full lease. Clear the lease and cancel its timeout when the control is unlocked and again when it is locked, so that a new owner starts locked and must ping to unlock. Compile-tested only; not run on hardware. Fixes: bad95a548bb7 ("macaudio: speaker volume safety interlocks") Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/macaudio.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index 24e0090e1a374..27a933fd4814c 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -1473,6 +1473,16 @@ static int macaudio_slk_lock(struct snd_kcontrol *kcontrol, struct snd_ctl_file struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); mutex_lock(&ma->volume_lock_mutex); + + /* + * A lease belongs to the owner that pinged for it. Whatever the + * previous owner had left must not unlock the volume for a new one + * before it has pinged itself. + */ + cancel_delayed_work(&ma->lock_timeout_work); + ma->speaker_lock_remain = 0; + ma->speaker_lock_timeout = 0; + ma->speaker_lock_owner = owner; macaudio_vlimit_update(ma); @@ -1495,7 +1505,9 @@ static void macaudio_slk_unlock(struct snd_kcontrol *kcontrol) struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); mutex_lock(&ma->volume_lock_mutex); + cancel_delayed_work(&ma->lock_timeout_work); ma->speaker_lock_owner = NULL; + ma->speaker_lock_remain = 0; ma->speaker_lock_timeout = 0; macaudio_vlimit_update(ma); mutex_unlock(&ma->volume_lock_mutex); From d55e37702b8c2dfb7dd64b66764e5f273c0f530f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:14:14 +1000 Subject: [PATCH 0459/2680] ASoC: macaudio: Keep a lease that a ping renewed under the timeout work macaudio_vlimit_timeout_work() expires the speaker lock lease as soon as it runs. A ping that arrives while the work is already running cannot cancel it: cancel_delayed_work() returns without waiting, the ping installs a fresh lease and re-arms the timer, and the running work then takes the mutex and expires the lease that was just renewed. The daemon sees an unexplained lock loss and the volume is clamped until its next ping. Disabling the timeout while the work is running has the same effect on the remaining lease. Check the lease deadline under the mutex: when it lies in the future, re-arm the work for the remainder instead of expiring, and do nothing when the timeout was disabled or the owner went away in the meantime. Compile-tested only; not run on hardware. Fixes: bad95a548bb7 ("macaudio: speaker volume safety interlocks") Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/macaudio.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index 27a933fd4814c..d29d6bef467c8 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -376,12 +376,31 @@ static void macaudio_vlimit_timeout_work(struct work_struct *wrk) { struct macaudio_snd_data *ma = container_of(to_delayed_work(wrk), struct macaudio_snd_data, lock_timeout_work); + ktime_t now, left; mutex_lock(&ma->volume_lock_mutex); + /* + * A ping or a disabled timeout can have cancelled this work after it + * had already started running; neither must expire the lease. + */ + if (!ma->speaker_lock_timeout_enabled || !ma->speaker_lock_owner || + ma->speaker_lock_remain <= 0) + goto out; + + /* A ping renewed the lease while this work was already running */ + now = ktime_get(); + if (ktime_before(now, ma->speaker_lock_timeout)) { + left = ktime_sub(ma->speaker_lock_timeout, now); + schedule_delayed_work(&ma->lock_timeout_work, + usecs_to_jiffies(ktime_to_us(left))); + goto out; + } + ma->speaker_lock_remain = 0; macaudio_vlimit_update(ma); +out: mutex_unlock(&ma->volume_lock_mutex); } From 283effb42d907c65ad3e0bc6beb0070f7f0d5af9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:19:04 +1000 Subject: [PATCH 0460/2680] rust: helpers: Add arm_arch_timer helpers arch_timer_read_counter() is a function pointer and arch_timer_get_rate() is not exported, so a Rust module cannot use either through bindgen alone. Add helpers for both, built into the kernel like the other helpers, so that a driver can read the architected counter through the arch timer layer (including its erratum workarounds) instead of reading the system registers itself. Both fall back to the header's stubs on architectures without the arch timer, so the helpers compile everywhere. The T8140 AOP audio driver, submitted separately, needs them: the AOP firmware stamps power requests with the always-on 24 MHz timebase, which is derived from the counter and its rate. Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/helpers/arm_arch_timer.c | 18 ++++++++++++++++++ rust/helpers/helpers.c | 1 + 2 files changed, 19 insertions(+) create mode 100644 rust/helpers/arm_arch_timer.c diff --git a/rust/helpers/arm_arch_timer.c b/rust/helpers/arm_arch_timer.c new file mode 100644 index 0000000000000..5810dbdca439e --- /dev/null +++ b/rust/helpers/arm_arch_timer.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +/* + * Named apart from the C symbols: the arm64 headers already declare + * arch_timer_read_counter (a function pointer) and arch_timer_get_rate + * in the generated bindings. + */ +__rust_helper u64 rust_helper_arch_timer_counter(void) +{ + return arch_timer_read_counter(); +} + +__rust_helper u32 rust_helper_arch_timer_rate(void) +{ + return arch_timer_get_rate(); +} diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 0e8664be6e80c..f5aec520173f7 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -39,6 +39,7 @@ #endif #include "acpi.c" +#include "arm_arch_timer.c" #include "atomic.c" #include "atomic_ext.c" #include "auxiliary.c" From 7492f596f8404085bb8f879d5658fc978b1656f8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:19:59 +1000 Subject: [PATCH 0461/2680] rust: bindings: Add ALSA TLV and control interface constants Generate bindings for sound/tlv.h so that a Rust driver can build a dB-scale TLV from SNDRV_CTL_TLVT_DB_SCALE and TLV_DB_SCALE_MUTE rather than from bare numbers, and export SNDRV_CTL_ELEM_IFACE_MIXER, a __force cast that bindgen cannot evaluate, as a constant in the same way as the PCM format bits. The T8140 AOP audio driver, submitted separately, uses them for its speaker volume control. Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/bindings/bindings_helper.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 79187dc7ce63f..e721b01b79f0a 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -112,6 +112,7 @@ #include #include #include +#include #include /* @@ -144,6 +145,7 @@ const u64 BINDINGS_SNDRV_PCM_FMTBIT_FLOAT_LE = SNDRV_PCM_FMTBIT_FLOAT_LE; const u64 BINDINGS_SNDRV_PCM_FMTBIT_S32_LE = SNDRV_PCM_FMTBIT_S32_LE; const u64 BINDINGS_SNDRV_PCM_FMTBIT_S24_LE = SNDRV_PCM_FMTBIT_S24_LE; const u32 BINDINGS_SNDRV_PCM_SUBFMTBIT_STD = SNDRV_PCM_SUBFMTBIT_STD; +const snd_ctl_elem_iface_t BINDINGS_SNDRV_CTL_ELEM_IFACE_MIXER = SNDRV_CTL_ELEM_IFACE_MIXER; const u32 BINDINGS_IIO_CHAN_INFO_RAW = IIO_CHAN_INFO_RAW; const u32 BINDINGS_IIO_CHAN_INFO_PROCESSED = IIO_CHAN_INFO_PROCESSED; From e8107beb7de4ec5c9b33b79e9039675a63dab911 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:37:04 +1000 Subject: [PATCH 0462/2680] ASoC: apple: Add T8140 AOP audio driver On Apple's T8140 (J700, MacBook Neo) the AOP firmware owns the audio fabric: the ADMAC channels, the MCA serializers and the LEAP block are run by the firmware, and the host reaches each path through an audio EPIC service. Add a driver for the AOP's audio child that exposes them. The driver registers a card of its own with two capture PCMs. The low-power microphone (lpai) has no host DMA at all: the firmware produces 16 kHz stereo S32 frames into a 512 KiB ring that the AOP driver binds over the setup port, and reports its cursor every 3,200 frames; the report listener copies each reported span into the ALSA buffer. The high-quality microphone array (hpai) is a PDM part behind the AOP that comes back as float32 over leap-s ADMAC RX0; it has no codec and no serializer of ours, so like the other AOP microphones it is a plain dmaengine PCM on the driver's card, advertising FLOAT_LE. For the headphone jack and the internal speaker the driver is an ASoC component: back-end DAIs "cout" (the CS42L83 on MCA1, base-ns ADMAC TX2/RX2) and "spkr" (the MAX98360A amplifiers on LEAP TX0, with the SpeakerTap on base-ns RX1 as their capture side), and three DPCM front-end PCMs on the corresponding ADMAC channels, which the J700 macaudio variant links to the codecs. A back-end attaches its service at startup, requests "pw0 ", holds its PMGR leaves through runtime PM and requests "pwrd", after which the firmware runs the serializer clocks. The speaker and the microphone array start from a work item queued by the trigger: "pwrd" makes the LEAP move samples at once, so the DMA must already be running, and the firmware call sleeps. A PCM underrun stops the DMA but not the service, so the speaker's prepare retires the previous run before the stream is started again. The speaker front-end is copy-only. The LEAP consumes float32 and the words on the wire are 2 * value * 2^31, so 0.5 is full scale and anything above it wraps: every S32 sample is scaled by "Speaker Playback Volume" (0.5 dB steps below that ceiling, the top entry asserted at compile time) and converted in the copy path, mmap is not offered, and the buffer is bounded to 341 ms so that a gain change is not stuck behind a long queue. macaudio's volume lock keeps the control 20 dB down until a speaker-protection daemon takes it. The tap that macaudio exposes as "Speaker Sense" is a digital tap: it returns the words the amplifier received, after this gain, and senses no current or voltage. The LEAP ADMAC's channel registers only answer while the channel's own PMGR leaf is on, so the speaker and array channels are requested per open with their leaves powered and released before the leaves are. A leaf that fails to power up fails that open; recovering a domain from a failed resume is the pmdomain driver's job, not a consumer's. Power requests carry a timestamp in the AOP's 24 MHz timebase, derived from the architected counter through the arch timer helpers. Compile-tested only; not run on hardware. [ryan: adapted to 7.2.10 - platform::Driver gains type Data<'bound> and the bound-lifetime probe()/unbind() signatures] Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/Kconfig | 13 + sound/soc/apple/Makefile | 3 + sound/soc/apple/t8140_aop_audio.rs | 2673 ++++++++++++++++++++++++++++ 3 files changed, 2689 insertions(+) create mode 100644 sound/soc/apple/t8140_aop_audio.rs diff --git a/sound/soc/apple/Kconfig b/sound/soc/apple/Kconfig index eac1b416d2bd6..a57eadd07a1ba 100644 --- a/sound/soc/apple/Kconfig +++ b/sound/soc/apple/Kconfig @@ -12,6 +12,19 @@ config SND_SOC_APPLE_AOP_AUDIO co-processor on ARM Macs. This includes the built-in microphone on those machines. +config SND_SOC_APPLE_T8140_AOP_AUDIO + tristate "Apple T8140 AOP audio driver" + depends on ARCH_APPLE || COMPILE_TEST + depends on RUST + select APPLE_AOP + select SND_DMAENGINE_PCM + help + This option enables a driver for the audio paths owned by the AOP + co-processor on Apple T8140 SoCs: a card with the low-power and the + high-quality microphones, and ASoC back-end DAIs for the headphone + jack and the internal speaker, which the macaudio driver links into + its card. + config SND_SOC_APPLE_MCA tristate "Apple Silicon MCA driver" depends on ARCH_APPLE || COMPILE_TEST diff --git a/sound/soc/apple/Makefile b/sound/soc/apple/Makefile index 9e6f245f934b8..df3c2264f6279 100644 --- a/sound/soc/apple/Makefile +++ b/sound/soc/apple/Makefile @@ -3,6 +3,9 @@ snd-soc-aop-y := aop_audio.o obj-$(CONFIG_SND_SOC_APPLE_AOP_AUDIO) += snd-soc-aop.o +snd-soc-t8140-aop-audio-y := t8140_aop_audio.o +obj-$(CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO) += snd-soc-t8140-aop-audio.o + snd-soc-apple-mca-y := mca.o obj-$(CONFIG_SND_SOC_APPLE_MCA) += snd-soc-apple-mca.o diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs new file mode 100644 index 0000000000000..e6be972f7ab77 --- /dev/null +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -0,0 +1,2673 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![recursion_limit = "2048"] + +//! Apple T8140 (J700, MacBook Neo) AOP audio +//! +//! The AOP firmware owns the audio fabric on this SoC, so the paths the AP +//! sees are its services: +//! +//! * `lpai`, the low-power microphone: 16 kHz stereo S32 frames (24 +//! meaningful bits) produced into a 512 KiB host ring mapped through +//! dart-aop stream 9, with a cursor report every 3,200 frames. There is +//! no DMA engine on the host side: the ring is bound to the firmware over +//! the AOP setup port (the AOP driver owns it, the firmware binds it once +//! per boot, and hands out reads of it) and the driver copies each +//! reported span into the ALSA buffer of the first PCM of its own card. +//! * `hpai`, the high-quality microphone array: a PDM part behind the AOP, +//! read back as float32 over leap-s ADMAC RX0. It has no codec and no +//! serializer of ours, so it is the second PCM of the driver's own card, +//! as the other AOP microphones are on the machines with an `aop_audio` +//! node. +//! * `cout`/`cin` (the CS42L83 jack on MCA1, base-ns ADMAC TX2/RX2), `spkr` +//! (the MAX98360A speakers on LEAP TX0, leap-ns ADMAC TX0) and `tap ` +//! (the speaker tap, base-ns RX1): ASoC back-end DAIs that attach the +//! service, request its power states (`pw0 ` idle, `pwrd` running, the +//! firmware then runs the serializer clocks) and runtime-power the +//! AP-side PMGR leaves, with DPCM front-end PCMs on the ADMAC channels. +//! The macaudio J700 variant links them to the codecs. +//! +//! The speaker front-end is copy-only: the LEAP consumes float32 and the +//! driver converts each S32 sample through the amplifier's full scale and +//! the "Speaker Playback Volume" control, which the card's volume lock holds +//! 20 dB down whenever no speaker-protection daemon owns it. +//! +//! Copyright (C) The Asahi Linux Contributors +//! Copyright (C) 2026 The Aurora Silicon Contributors + +use core::{ + mem, + ptr, + slice, // +}; + +use kernel::{ + bindings, + c_str, + device, + device::property::FwNode, + device::Core, + dma::{ + Device as DmaDevice, + DmaMask, // + }, + error::from_err_ptr, + module_platform_driver, + new_spinlock, + of, + platform, + prelude::*, + soc::apple::aop::{ + from_fourcc, + EPICService, + ReportListener, + SourceRing, + AOP, // + }, + str::CString, + sync::{ + aref::ARef, + atomic::{ + Atomic, + AtomicFlag, + Relaxed, // + }, + Arc, + ArcBorrow, + SpinLock, // + }, + types::{ForeignOwnable, Opaque}, + workqueue::{ + self, + impl_has_work, + new_work, + Work, + WorkItem, // + }, // +}; + +const EPIC_SUBTYPE_WRAPPED_CALL: u16 = 0x20; +const EPIC_SUBTYPE_PRODUCER_REPORT: u16 = 0x20; +const CALLTYPE_AUDIO_ATTACH_DEVICE: u32 = 0xc3000002; +const CALLTYPE_AUDIO_GET_PROP: u32 = 0xc3000004; +const CALLTYPE_AUDIO_SET_PROP: u32 = 0xc3000005; +const AUDIO_DEV_LPAI: u32 = from_fourcc(b"lpai"); +const PROP_POWER_STATE: u32 = 200; +const PROP_POWER_REQUEST: u32 = 202; +const PROP_CHANNEL_CONTROL: u32 = 300; +const PROP_STREAM_FORMAT: u32 = 301; +const PROP_RING_GEOMETRY: u32 = 302; +/// Output services: 4-byte boolean, 1 mutes (with the firmware's ramp). +const PROP_OUTPUT_MUTE: u32 = 700; +const POWER_STATE_RUN: u32 = from_fourcc(b"runn"); +const POWER_STATE_IDLE: u32 = from_fourcc(b"idle"); + +const AUDIO_DEV_SPKR: u32 = from_fourcc(b"spkr"); +const AUDIO_DEV_TAP: u32 = from_fourcc(b"tap "); +const AUDIO_DEV_COUT: u32 = from_fourcc(b"cout"); +const AUDIO_DEV_CIN: u32 = from_fourcc(b"cin "); +/// The high-quality microphone array (ADT /arm-io/iop-audio-controller/ +/// audio-hp-mic, identifier 'iaph' -- ADT FourCCs are byte-reversed). Unlike +/// `lpai` it is the machine's only `iop-audio-s` stream: its data comes back +/// over leap-s ADMAC RX0, not the AOP source ring. +const AUDIO_DEV_HPAI: u32 = from_fourcc(b"hpai"); +/// kIOReturnBusy from the firmware: the device is already attached. +const AUDIO_RET_BUSY: u32 = 0xe00002d5; +const SPKR_POWER_STATE_PW0: u32 = from_fourcc(b"pw0 "); +const SPKR_POWER_STATE_PWRD: u32 = from_fourcc(b"pwrd"); + +/// The low-power microphone stream: 16 kHz stereo S32 in a 512 KiB ring +/// reported every 3,200 frames. +const LPAI_RATE: u32 = 16000; +const LPAI_CHANNELS: u32 = 2; +const LPAI_FRAME_BYTES: usize = 8; +const RING_BYTES: usize = 0x80000; +const LPAI_PERIOD_FRAMES: usize = 3200; +const LPAI_PERIOD_BYTES: usize = LPAI_PERIOD_FRAMES * LPAI_FRAME_BYTES; +const LPAI_PERIODS_MAX: u32 = 16; +const REPORT_MIN_LEN: usize = 0x68; + +/// The high-quality microphone array: 48 kHz stereo float32 over the LEAP +/// ADMAC. +const HPAI_RATE: u32 = 48000; +const HPAI_CHANNELS: u32 = 2; + +/// Firmware power requests carry a timestamp in the 24 MHz always-on +/// timebase; the architected counter runs at a different rate (1 GHz on +/// T8140), so its reading is converted. +const AOP_TIMEBASE_HZ: u64 = 24_000_000; + +fn aop_counter_ticks(counter: u64, frequency: u64) -> Result { + if frequency == 0 || frequency > u64::from(u32::MAX) { + return Err(EINVAL); + } + // Split whole seconds from the remainder to avoid overflowing a u64 + // multiplication on an otherwise ordinary long-running system. + let seconds = counter / frequency; + let remainder = (counter % frequency) * AOP_TIMEBASE_HZ / frequency; + seconds + .checked_mul(AOP_TIMEBASE_HZ) + .and_then(|ticks| ticks.checked_add(remainder)) + .ok_or(EOVERFLOW) +} + +fn aop_timestamp() -> Result { + // SAFETY: both helpers read the architected timer through the arch timer + // layer and take no arguments. + let (counter, rate) = unsafe { (bindings::arch_timer_counter(), bindings::arch_timer_rate()) }; + aop_counter_ticks(counter, u64::from(rate)) +} + +/// Little-endian field readers for firmware replies and reports. +fn le_u32(buf: &[u8], off: usize) -> Result { + let b = buf.get(off..off + 4).ok_or(EINVAL)?; + Ok(u32::from_le_bytes([b[0], b[1], b[2], b[3]])) +} + +fn le_u64(buf: &[u8], off: usize) -> Result { + let b = buf.get(off..off + 8).ok_or(EINVAL)?; + Ok(u64::from_le_bytes([ + b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], + ])) +} + +#[repr(C, packed(4))] +#[derive(Clone, Copy, Default)] +struct AudioAttachDevice { + _zero0: u32, + unk0: u32, + calltype: u32, + _zero1: [u8; 16], + cookie: u32, + len: u64, + dev_id: u32, + _zero2: u32, +} + +impl AudioAttachDevice { + fn new(dev_id: u32) -> AudioAttachDevice { + AudioAttachDevice { + unk0: 0xffffffff, + calltype: CALLTYPE_AUDIO_ATTACH_DEVICE, + len: 0x2c, + dev_id, + ..AudioAttachDevice::default() + } + } +} + +#[repr(C, packed(4))] +#[derive(Clone, Copy, Default)] +struct AudioGetDeviceProp { + _zero0: u32, + unk0: u32, + calltype: u32, + _zero1: [u8; 16], + _pad0: u32, + len: u64, + dev_id: u32, + modifier: u32, + unk6: u32, +} + +impl AudioGetDeviceProp { + fn new(dev_id: u32, modifier: u32) -> AudioGetDeviceProp { + AudioGetDeviceProp { + unk0: 0xffffffff, + calltype: CALLTYPE_AUDIO_GET_PROP, + len: 0x30, + dev_id, + modifier, + unk6: 1, + ..AudioGetDeviceProp::default() + } + } +} + +#[repr(C, packed(4))] +#[derive(Clone, Copy, Default)] +struct AudioSetDeviceProp { + _zero0: u32, + unk0: u32, + calltype: u32, + _zero1: [u8; 16], + _pad0: u32, + len: u64, + dev_id: u32, + modifier: u32, + len2: u32, + data: T, +} + +impl AudioSetDeviceProp { + fn new(dev_id: u32, modifier: u32, data: T) -> AudioSetDeviceProp { + AudioSetDeviceProp { + unk0: 0xffffffff, + calltype: CALLTYPE_AUDIO_SET_PROP, + len: mem::size_of::() as u64 + 0x30, + dev_id, + modifier, + len2: mem::size_of::() as u32, + data, + ..AudioSetDeviceProp::default() + } + } +} + +/// Property 300: supported / enabled / history channel masks. +#[repr(C, packed(4))] +#[derive(Clone, Copy, Default)] +struct ChannelControl { + supported: u32, + enabled: u32, + history: u32, +} + +/// Property 202, the power request: the device (the FourCC for the output +/// services, 0 for lpai), a per-device request sequence, the request time +/// in the firmware's 24 MHz timebase, the target state FourCC and a +/// parameter (1 for lpai, 0 for the output services). +#[repr(C, packed(4))] +#[derive(Clone, Copy, Default)] +struct PowerRequest { + dev_id: u32, + sequence: u32, + timestamp: u64, + _zero0: u32, + state: u32, + param: u32, + _zero1: [u8; 20], +} + +impl PowerRequest { + fn new(dev_id: u32, sequence: u32, state: u32, param: u32) -> Result { + Ok(PowerRequest { + dev_id, + sequence, + timestamp: aop_timestamp()?, + state, + param, + ..PowerRequest::default() + }) + } +} + +/// A named power domain of this device (`dev_pm_domain_attach_by_name()`): +/// the virtual device the PM core hands out for one of several domains, +/// runtime-powered around the streams that need it and detached when the +/// driver data goes away. +struct PowerDomain { + dev: ptr::NonNull, + name: &'static str, +} + +// SAFETY: the virtual domain device belongs to the PM core, which serializes +// its runtime-PM state itself; this handle only passes the pointer to the +// runtime-PM calls, which may be made from any thread, until `Drop` detaches +// it after the last user is gone. +unsafe impl Send for PowerDomain {} +// SAFETY: as for `Send`; a shared `PowerDomain` exposes nothing but those +// calls. +unsafe impl Sync for PowerDomain {} + +impl PowerDomain { + /// Attaches the domain named `name`; `Ok(None)` when the device has no + /// domain of that name. + fn attach( + dev: &device::Device, + name: &'static str, + dt_name: &'static CStr, + ) -> Result> { + // SAFETY: `dev` is live for the call and `dt_name` is NUL-terminated. + let pd = from_err_ptr(unsafe { + bindings::dev_pm_domain_attach_by_name(dev.as_raw(), dt_name.as_char_ptr()) + })?; + Ok(ptr::NonNull::new(pd).map(|dev| PowerDomain { dev, name })) + } + + /// Takes a runtime-PM reference and waits for the domain to be on. + fn get(&self) -> Result<()> { + // SAFETY: the domain device stays attached for the lifetime of `self`. + let rc = unsafe { bindings::pm_runtime_get_sync(self.dev.as_ptr()) }; + if rc < 0 { + // SAFETY: as above; a failed resume still took the usage count. + unsafe { bindings::pm_runtime_put_noidle(self.dev.as_ptr()) }; + return Err(Error::from_errno(rc)); + } + Ok(()) + } + + /// Drops a reference taken by [`Self::get`]. + fn put(&self) { + // SAFETY: the domain device stays attached for the lifetime of `self`. + unsafe { bindings::pm_runtime_put_sync(self.dev.as_ptr()) }; + } +} + +impl Drop for PowerDomain { + fn drop(&mut self) { + // SAFETY: attached in `attach`; the driver data that owns this handle + // is dropped only after every stream, and with it every `get`, has + // been balanced by a `put`. + unsafe { bindings::dev_pm_domain_detach(self.dev.as_ptr(), true) }; + } +} + +/// One firmware audio service driven as an ASoC back-end (or, for the +/// microphone array, by its own PCM): attached once, requested into `pw0 ` +/// (idle) at startup and `pwrd` (running) when the stream starts, back to +/// `pw0 ` at shutdown, with its serializer's PMGR leaves runtime-powered for +/// the stream's lifetime. +struct Service { + dev_id: u32, + name: &'static str, + /// Request sequence of the property 202 messages. + sequence: Atomic, + /// In `pwrd`. + running: AtomicFlag, +} + +impl Service { + const fn new(dev_id: u32, name: &'static str) -> Service { + Service { + dev_id, + name, + sequence: Atomic::new(0), + running: AtomicFlag::new(false), + } + } +} + +/// The low-power microphone stream, guarded by a spinlock that is never taken +/// with the ALSA stream lock held (trigger only touches the atomic run +/// flag), so the report path may call snd_pcm_period_elapsed() while holding +/// it and close cannot free the substream under a report. +struct StreamState { + substream: *mut bindings::snd_pcm_substream, + /// absolute producer byte count already copied into the ALSA buffer + copied: Option, + hw_ptr_bytes: usize, + period_acc: usize, + buffer_bytes: usize, +} + +// SAFETY: the raw substream pointer is only dereferenced under the spinlock +// that holds this state, and only while ALSA keeps the substream alive: open +// sets it and close clears it under the same lock. +unsafe impl Send for StreamState {} + +#[pin_data] +struct SndSocT8140AopData { + dev: ARef, + adata: Arc, + service: EPICService, + ring: Arc, + ring_bytes: Atomic, + sequence: Atomic, + /// lpai is in `runn`. + powered: AtomicFlag, + /// The low-power microphone stream is triggered (set and cleared under + /// the ALSA stream lock, read by the report path). + running: AtomicFlag, + hw_ptr_frames: Atomic, + /// One warning per producer-report fault class and driver instance. + report_warnings: [AtomicFlag; 5], + /// The back-end services: the jack's output and headset microphone, + /// the speakers and their tap, and the microphone array. + cout: Service, + cin: Service, + spkr: Service, + tap: Service, + hpai: Service, + /// The serializer leaves, runtime-powered around streams: audio_mca1_m + /// (jack), audio_mca0_m (tap and the LEAP's serializer), audio_leap_tx0 + /// and audio_leap_mca (speakers), audio_leap_rx0 (microphone array). + pd_mca1: Option, + pd_mca0: Option, + pd_tx0: Option, + pd_leap_mca: Option, + pd_rx0: Option, + /// A speaker stream has been triggered and not yet stopped. + spkr_want_run: AtomicFlag, + /// A microphone-array stream has been triggered and not yet stopped. + hpai_want_run: AtomicFlag, + /// "Speaker Playback Volume" (REG_SPKR_VOLUME), 0..=SPKR_VOLUME_MAX. + spkr_volume: Atomic, + /// The sleeping firmware starts queued from the atomic triggers. + #[pin] + spkr_start_work: Work, + #[pin] + hpai_start_work: Work, + #[pin] + stream: SpinLock, +} + +const SPKR_START_WORK: u64 = 0; +const HPAI_START_WORK: u64 = 1; + +impl_has_work! { + impl HasWork for SndSocT8140AopData { self.spkr_start_work } + impl HasWork for SndSocT8140AopData { self.hpai_start_work } +} + +/// The speaker stream start (pwrd + unmute), queued by the back-end DAI's +/// trigger START: those are sleeping firmware calls, and the DMA must already +/// be running when the LEAP starts consuming. +impl WorkItem for SndSocT8140AopData { + type Pointer = Arc; + + fn run(this: Arc) { + if let Err(e) = this.spkr_go() { + dev_err!(this.dev, "spkr: start failed: {:?}\n", e); + } + } +} + +/// The microphone-array stream start (`pwrd`), queued by its PCM trigger +/// START for the same reason as the speaker's: the ADMAC must already be +/// running before the LEAP starts producing. +impl WorkItem for SndSocT8140AopData { + type Pointer = Arc; + + fn run(this: Arc) { + if let Err(e) = this.hpai_go() { + dev_err!(this.dev, "hpai: start failed: {:?}\n", e); + } + } +} + +impl SndSocT8140AopData { + fn epic_wrapped_call(&self, data: &T) -> Result { + // SAFETY: T is a plain packed firmware record. + let msg_bytes = + unsafe { slice::from_raw_parts(ptr::from_ref(data).cast::(), mem::size_of::()) }; + self.adata + .epic_call(&self.service, EPIC_SUBTYPE_WRAPPED_CALL, msg_bytes) + } + + fn epic_wrapped_call_ret(&self, data: &T, ret_len: usize) -> Result> { + // SAFETY: T is a plain packed firmware record. + let msg_bytes = + unsafe { slice::from_raw_parts(ptr::from_ref(data).cast::(), mem::size_of::()) }; + let (retcode, ret) = self.adata.epic_call_ret( + &self.service, + EPIC_SUBTYPE_WRAPPED_CALL, + msg_bytes, + ret_len, + )?; + if retcode != 0 { + dev_err!( + self.dev, + "audio wrapped call failed, return code {:#x}\n", + retcode + ); + return Err(EIO); + } + Ok(ret) + } + + /// Attach an audio device. The firmware keeps attachments for its + /// lifetime (there is no detach in the protocol we know), so a reload of + /// this module sees kIOReturnBusy for a device it attached earlier. + fn attach_device(&self, dev_id: u32, name: &str) -> Result<()> { + let ret = self.epic_wrapped_call(&AudioAttachDevice::new(dev_id))?; + match ret { + 0 | AUDIO_RET_BUSY => Ok(()), + _ => { + dev_err!( + self.dev, + "unable to attach {}, return code {:#x}\n", + name, + ret + ); + Err(EIO) + } + } + } + + fn set_channel_control(&self) -> Result<()> { + let ctl = ChannelControl { + supported: 3, + enabled: 3, + history: 3, + }; + let ret = self.epic_wrapped_call(&AudioSetDeviceProp::new( + AUDIO_DEV_LPAI, + PROP_CHANNEL_CONTROL, + ctl, + ))?; + if ret != 0 { + dev_err!( + self.dev, + "unable to set lpai channels, return code {:#x}\n", + ret + ); + return Err(EIO); + } + Ok(()) + } + + /// Reply layout after the retcode: u32 length, then the property bytes. + fn get_prop(&self, modifier: u32, len: usize) -> Result> { + let ret = self + .epic_wrapped_call_ret(&AudioGetDeviceProp::new(AUDIO_DEV_LPAI, modifier), 4 + len)?; + if ret.len() < 4 + len { + dev_err!( + self.dev, + "lpai property {} reply too short ({})\n", + modifier, + ret.len() + ); + return Err(EIO); + } + let reported = le_u32(&ret, 0)? as usize; + if reported != len { + dev_err!( + self.dev, + "lpai property {} length {} (expected {})\n", + modifier, + reported, + len + ); + return Err(EIO); + } + Ok(ret) + } + + fn read_geometry(&self) -> Result<()> { + // 302: FourCC/pad, ring frames u64, report frames u64 + let geo = self.get_prop(PROP_RING_GEOMETRY, 24)?; + let frames = le_u64(&geo, 12)?; + let quantum = le_u64(&geo, 20)?; + if frames == 0 + || frames > (RING_BYTES / LPAI_FRAME_BYTES) as u64 + || quantum == 0 + || quantum > frames + { + dev_err!( + self.dev, + "invalid lpai ring geometry: {} frames, {} per report\n", + frames, + quantum + ); + return Err(EIO); + } + self.ring_bytes + .store((frames as usize * LPAI_FRAME_BYTES) as u32, Relaxed); + // 301: stream format, sample rate at +4 + let fmt = self.get_prop(PROP_STREAM_FORMAT, 16)?; + let rate = le_u32(&fmt, 8)?; + if rate != LPAI_RATE { + dev_err!(self.dev, "unexpected lpai sample rate {}\n", rate); + return Err(EIO); + } + dev_dbg!( + self.dev, + "lpai: {} Hz, ring {} frames, {} frames per report\n", + rate, + frames, + quantum + ); + Ok(()) + } + + fn set_power(&self, state: u32) -> Result<()> { + let sequence = self.sequence.fetch_add(1, Relaxed) + 1; + let req = PowerRequest::new(0, sequence, state, 1)?; + let ret = self.epic_wrapped_call(&AudioSetDeviceProp::new( + AUDIO_DEV_LPAI, + PROP_POWER_REQUEST, + req, + ))?; + if ret != 0 { + dev_err!( + self.dev, + "lpai power request {:#x} rejected: {:#x}\n", + state, + ret + ); + return Err(EIO); + } + let st = self.get_prop(PROP_POWER_STATE, 4)?; + let now = le_u32(&st, 4)?; + if now != state { + dev_err!( + self.dev, + "lpai power state {:#x} after requesting {:#x}\n", + now, + state + ); + return Err(EIO); + } + Ok(()) + } + + // ---- the back-end services ---- + + /// Property 202 with the service's sequence and a 24 MHz timestamp; + /// `pw0 ` is confirmed through property 200 like the native host does. + fn service_set_power(&self, svc: &Service, state: u32) -> Result<()> { + let sequence = svc.sequence.fetch_add(1, Relaxed) + 1; + let req = PowerRequest::new(svc.dev_id, sequence, state, 0)?; + let ret = self.epic_wrapped_call(&AudioSetDeviceProp::new( + svc.dev_id, + PROP_POWER_REQUEST, + req, + ))?; + if ret != 0 { + dev_err!( + self.dev, + "{} power request {:#x} rejected: {:#x}\n", + svc.name, + state, + ret + ); + return Err(EIO); + } + if state == SPKR_POWER_STATE_PW0 { + let ret = self + .epic_wrapped_call_ret(&AudioGetDeviceProp::new(svc.dev_id, PROP_POWER_STATE), 8)?; + let now = le_u32(&ret, 4)?; + if now != state { + dev_err!( + self.dev, + "{} power state {:#x} after requesting {:#x}\n", + svc.name, + now, + state + ); + return Err(EIO); + } + } + Ok(()) + } + + /// Runtime-power serializer leaves, parents first. A leaf that fails to + /// power up fails the stream; recovering a domain from that state is the + /// power-domain driver's business, not a consumer's. + fn power_up(&self, name: &str, pds: &[&Option]) -> Result<()> { + for (i, slot) in pds.iter().enumerate() { + let Some(pd) = slot else { + dev_err!(self.dev, "{}: power domain not attached\n", name); + self.power_down(&pds[..i]); + return Err(ENODEV); + }; + if let Err(e) = pd.get() { + dev_err!( + self.dev, + "{}: power domain {} did not power up: {:?}\n", + name, + pd.name, + e + ); + self.power_down(&pds[..i]); + return Err(e); + } + } + Ok(()) + } + + /// Release leaves powered by [`Self::power_up`], children first. + fn power_down(&self, pds: &[&Option]) { + for pd in pds.iter().rev().filter_map(|pd| pd.as_ref()) { + pd.put(); + } + } + + /// Back-end startup: attach the service, power its leaves, `pw0 `. + fn service_startup(&self, svc: &Service, pds: &[&Option]) -> Result<()> { + self.attach_device(svc.dev_id, svc.name)?; + self.power_up(svc.name, pds)?; + if let Err(e) = self.service_set_power(svc, SPKR_POWER_STATE_PW0) { + self.power_down(pds); + return Err(e); + } + svc.running.store(false, Relaxed); + Ok(()) + } + + /// `pwrd`: the firmware starts the serializer clocks and the stream. + fn service_run(&self, svc: &Service) -> Result<()> { + if svc.running.load(Relaxed) { + return Ok(()); + } + self.service_set_power(svc, SPKR_POWER_STATE_PWRD)?; + svc.running.store(true, Relaxed); + Ok(()) + } + + /// Back-end shutdown: `pw0 ` if running, then the leaves. + fn service_shutdown(&self, svc: &Service, pds: &[&Option]) { + if svc.running.load(Relaxed) { + if let Err(e) = self.service_set_power(svc, SPKR_POWER_STATE_PW0) { + dev_err!(self.dev, "{}: pw0 failed: {:?}\n", svc.name, e); + } + svc.running.store(false, Relaxed); + } + self.power_down(pds); + } + + // ---- 3.5 mm jack: cout (MCA1, base-ns TX2) and cin (the headset + // microphone, base-ns RX2), one MCA1 power reference per open direction ---- + + fn jack_service(&self, playback: bool) -> &Service { + if playback { + &self.cout + } else { + &self.cin + } + } + + fn jack_startup(&self, playback: bool) -> Result<()> { + self.service_startup(self.jack_service(playback), &[&self.pd_mca1]) + } + + /// pwrd starts the MCA1 clocks; the codec's PLL locks to them when ASoC + /// unmutes it right after this. + fn jack_prepare(&self, playback: bool) -> Result<()> { + self.service_run(self.jack_service(playback)) + } + + fn jack_shutdown(&self, playback: bool) { + self.service_shutdown(self.jack_service(playback), &[&self.pd_mca1]) + } + + // ---- speaker tap: the native SpeakerTap, base-ns RX1 under the tap + // service on MCA0. The words the amplifier receives come back on this + // stream (the card's "Speaker Sense" capture): it observes the serial + // data after the copy-time gain, it senses no current or voltage. ---- + + fn sense_startup(&self) -> Result<()> { + self.service_startup(&self.tap, &[&self.pd_mca0]) + } + + fn sense_prepare(&self) -> Result<()> { + self.service_run(&self.tap) + } + + fn sense_shutdown(&self) { + self.service_shutdown(&self.tap, &[&self.pd_mca0]) + } + + // ---- the high-quality microphone array: the hpai service feeding + // leap-s ADMAC RX0. The host holds its RX0 leaf and the shared LEAP MCA + // domain while the firmware service owns the microphone fabric. ---- + + /// The array's LEAP leaves: its own RX0 channel leaf, plus the LEAP MCA + /// the speaker path also holds. + fn hpai_domains(&self) -> [&Option; 2] { + [&self.pd_rx0, &self.pd_leap_mca] + } + + fn hpai_startup(&self) -> Result<()> { + self.service_startup(&self.hpai, &self.hpai_domains()) + } + + /// The array's start (`pwrd`), from the system workqueue right after + /// trigger START. Started earlier, from prepare, the LEAP produces into + /// a ring the ADMAC is not servicing yet and the capture is noise from + /// then on; started here, the DMA is already running. + fn hpai_go(&self) -> Result<()> { + if !self.hpai_want_run.load(Relaxed) || self.hpai.running.load(Relaxed) { + return Ok(()); + } + self.service_run(&self.hpai) + } + + /// Called from sleeping close paths: a queued start must finish before + /// the service is retired. + fn flush_hpai_start(&self) { + self.hpai_want_run.store(false, Relaxed); + // SAFETY: the work item lives in `self` for the driver's lifetime; + // flushing waits for a queued `hpai_go` to finish. + unsafe { bindings::flush_work(Work::raw_get(&self.hpai_start_work)) }; + } + + fn hpai_shutdown(&self) { + self.flush_hpai_start(); + self.service_shutdown(&self.hpai, &self.hpai_domains()) + } + + // ---- internal speakers: the spkr service on LEAP TX0 (leap-ns ADMAC + // channel 0); the leaves are tx0 (-> leap_c -> leap_a -> audio_fr -> + // audio_a), leap_mca (-> audio_p) and the MCA0 serializer the LEAP + // drives the amplifiers through -- shared with the tap capture, so + // both hold it and neither can power it off under the other ---- + + /// Output mute (property 700) is independent of the amplifier's SD_MODE; + /// the amplifier stays off across a failure here. + fn spkr_set_mute(&self, muted: bool) -> Result<()> { + let ret = self.epic_wrapped_call(&AudioSetDeviceProp::new( + AUDIO_DEV_SPKR, + PROP_OUTPUT_MUTE, + u32::from(muted), + ))?; + if ret != 0 { + dev_err!(self.dev, "speaker mute request rejected: {:#x}\n", ret); + return Err(EIO); + } + Ok(()) + } + + fn spkr_domains(&self) -> [&Option; 3] { + [&self.pd_tx0, &self.pd_leap_mca, &self.pd_mca0] + } + + fn spkr_startup(&self) -> Result<()> { + self.service_startup(&self.spkr, &self.spkr_domains()) + } + + /// A PCM underrun stops the DMA without closing the back-end, so the + /// service may still be in `pwrd` when the stream is prepared again. + /// Retire that run here so the next trigger queues a fresh `pwrd` and + /// unmute once the DMA has restarted. + fn spkr_prepare(&self) -> Result<()> { + self.flush_spkr_start(); + if !self.spkr.running.load(Relaxed) { + return Ok(()); + } + self.spkr_set_mute(true)?; + self.service_set_power(&self.spkr, SPKR_POWER_STATE_PW0)?; + self.spkr.running.store(false, Relaxed); + Ok(()) + } + + /// Speaker stream start, from the system workqueue right after trigger + /// START: the DMA must be running before pwrd starts the LEAP consuming + /// (issued from prepare, before the DMA, the wire stays silent), then the + /// firmware unmute, the value the native amplifier driver writes after + /// enabling its output. + fn spkr_go(&self) -> Result<()> { + if !self.spkr_want_run.load(Relaxed) || self.spkr.running.load(Relaxed) { + return Ok(()); + } + self.service_run(&self.spkr)?; + if let Err(e) = self.spkr_set_mute(false) { + let _ = self.service_set_power(&self.spkr, SPKR_POWER_STATE_PW0); + self.spkr.running.store(false, Relaxed); + return Err(e); + } + Ok(()) + } + + /// Called from sleeping prepare/shutdown paths while ALSA serializes + /// this stream: a queued start must finish before the mute and pw0. + fn flush_spkr_start(&self) { + self.spkr_want_run.store(false, Relaxed); + // SAFETY: the work item lives in `self` for the driver's lifetime; + // flushing waits for a queued `spkr_go` to finish. + unsafe { bindings::flush_work(Work::raw_get(&self.spkr_start_work)) }; + } + + /// Speaker back-end shutdown: mute, then the service; the amplifier is + /// already off (codec trigger STOP) and a queued start was flushed. + fn spkr_shutdown(&self) { + self.flush_spkr_start(); + if self.spkr.running.load(Relaxed) { + if let Err(e) = self.spkr_set_mute(true) { + dev_err!(self.dev, "spkr: mute failed: {:?}\n", e); + } + } + self.service_shutdown(&self.spkr, &self.spkr_domains()); + } +} + +/// Copy `len` bytes from the source ring (absolute producer offset `from`) +/// into the ALSA buffer at `hw_ptr`, both wrapping. Only spans the firmware +/// has announced through a producer report are copied, so the producer is +/// past them; `ring_bytes` is the geometry the firmware reported and never +/// exceeds the ring, so `read()` cannot fail on range. +fn copy_span( + ring: &SourceRing, + dma_area: *mut u8, + buffer_bytes: usize, + ring_bytes: usize, + from: u64, + len: usize, + hw_ptr: usize, +) -> Result { + let mut src = (from % ring_bytes as u64) as usize; + let mut dst = hw_ptr; + let mut left = len; + while left > 0 { + let n = left.min(ring_bytes - src).min(buffer_bytes - dst); + // SAFETY: dst + n <= buffer_bytes, and the ALSA buffer stays mapped + // while the substream is set in the stream state, whose lock the + // caller holds. + let span = unsafe { slice::from_raw_parts_mut(dma_area.add(dst), n) }; + ring.read(src, span)?; + src = (src + n) % ring_bytes; + dst = (dst + n) % buffer_bytes; + left -= n; + } + Ok(dst) +} + +impl ReportListener for SndSocT8140AopData { + fn process_report(&self, _subtype: u16, report: &[u8]) -> Result<()> { + if report.len() < REPORT_MIN_LEN { + if !self.report_warnings[0].xchg(true, Relaxed) { + dev_warn!( + self.dev, + "short lpai producer report ({} bytes)\n", + report.len() + ); + } + return Ok(()); + } + let count = le_u64(report, 0x40)?; + let Some(absolute) = count + .checked_add(1) + .and_then(|frames| frames.checked_mul(LPAI_FRAME_BYTES as u64)) + else { + if !self.report_warnings[1].xchg(true, Relaxed) { + dev_warn!(self.dev, "overflowing lpai producer counter\n"); + } + return Ok(()); + }; + let frames = absolute / LPAI_FRAME_BYTES as u64; + let cursor = le_u64(report, 0x60)?; + let ring_bytes = self.ring_bytes.load(Relaxed) as usize; + if ring_bytes == 0 || cursor != absolute % ring_bytes as u64 { + if !self.report_warnings[2].xchg(true, Relaxed) { + dev_warn!( + self.dev, + "inconsistent lpai report: frames {} cursor {:#x}\n", + frames, + cursor + ); + } + return Ok(()); + } + + let mut st = self.stream.lock(); + if !self.running.load(Relaxed) || st.substream.is_null() { + return Ok(()); + } + let copied = match st.copied { + Some(c) => c, + None => { + // start at a completed producer position, not at ring zero + st.copied = Some(absolute); + return Ok(()); + } + }; + if absolute < copied { + if !self.report_warnings[3].xchg(true, Relaxed) { + dev_warn!(self.dev, "lpai producer moved backwards\n"); + } + st.copied = Some(absolute); + return Ok(()); + } + let delta = (absolute - copied) as usize; + if delta >= ring_bytes { + st.copied = Some(absolute); + if !self.report_warnings[4].xchg(true, Relaxed) { + dev_warn!(self.dev, "lpai source overrun ({} bytes)\n", delta); + } + return Ok(()); + } + if delta == 0 { + return Ok(()); + } + // SAFETY: the substream is registered in the stream state, which + // close clears under this lock, so it is live; its runtime and DMA + // buffer exist between hw_params and hw_free, which bracket prepare + // (where `buffer_bytes` was taken) and every triggered state. + let dma_area = unsafe { (*(*st.substream).runtime).dma_area }; + let buffer_bytes = st.buffer_bytes; + let new_ptr = copy_span( + &self.ring, + dma_area, + buffer_bytes, + ring_bytes, + copied, + delta, + st.hw_ptr_bytes, + )?; + st.hw_ptr_bytes = new_ptr; + st.copied = Some(absolute); + st.period_acc += delta; + self.hw_ptr_frames + .store((new_ptr / LPAI_FRAME_BYTES) as u32, Relaxed); + if st.period_acc >= LPAI_PERIOD_BYTES { + st.period_acc %= LPAI_PERIOD_BYTES; + // SAFETY: the substream is valid while it is registered in + // the stream state, which close clears under this lock. + unsafe { bindings::snd_pcm_period_elapsed(st.substream) }; + } + Ok(()) + } +} + +// ---- the driver's own card: the low-power microphone (PCM 0) and the +// microphone array (PCM 1) ---- + +/// The driver data behind one of this card's PCMs. +/// +/// # Safety +/// +/// `substream` must be a live substream of a PCM whose private data is the +/// `Arc` handed to ALSA at probe (released in `pcm_free_private` after the +/// last op). +unsafe fn pcm_data<'a>( + substream: *mut bindings::snd_pcm_substream, +) -> ArcBorrow<'a, SndSocT8140AopData> { + // SAFETY: by the caller's contract. + unsafe { Arc::::borrow((*substream).private_data.cast()) } +} + +/// # Safety +/// +/// Called by ALSA with a live substream of this card's PCM, whose private +/// data is the driver's `Arc`. +unsafe extern "C" fn lpai_pcm_open(substream: *mut bindings::snd_pcm_substream) -> i32 { + // SAFETY: ALSA calls this op with a live substream of the LPAI PCM. + let data = unsafe { pcm_data(substream) }; + let hw = bindings::snd_pcm_hardware { + info: bindings::SNDRV_PCM_INFO_MMAP + | bindings::SNDRV_PCM_INFO_MMAP_VALID + | bindings::SNDRV_PCM_INFO_INTERLEAVED, + formats: bindings::BINDINGS_SNDRV_PCM_FMTBIT_S32_LE, + subformats: 0, + rates: bindings::SNDRV_PCM_RATE_16000, + rate_min: LPAI_RATE, + rate_max: LPAI_RATE, + channels_min: LPAI_CHANNELS, + channels_max: LPAI_CHANNELS, + buffer_bytes_max: LPAI_PERIOD_BYTES * LPAI_PERIODS_MAX as usize, + period_bytes_min: LPAI_PERIOD_BYTES, + period_bytes_max: LPAI_PERIOD_BYTES, + periods_min: 2, + periods_max: LPAI_PERIODS_MAX, + fifo_size: 0, + }; + // SAFETY: ALSA calls this op with a live substream whose runtime exists + // for the op's duration. + unsafe { + (*(*substream).runtime).hw = hw; + } + data.running.store(false, Relaxed); + let mut st = data.stream.lock(); + st.substream = substream; + st.copied = None; + st.hw_ptr_bytes = 0; + st.period_acc = 0; + data.hw_ptr_frames.store(0, Relaxed); + 0 +} + +/// # Safety +/// +/// Called by ALSA with a live substream of this card's PCM, whose private +/// data is the driver's `Arc`. +unsafe extern "C" fn lpai_pcm_close(substream: *mut bindings::snd_pcm_substream) -> i32 { + // SAFETY: ALSA calls this op with a live substream of the LPAI PCM. + let data = unsafe { pcm_data(substream) }; + data.running.store(false, Relaxed); + { + let mut st = data.stream.lock(); + st.substream = ptr::null_mut(); + } + if data.powered.load(Relaxed) { + if let Err(e) = data.set_power(POWER_STATE_IDLE) { + dev_err!(data.dev, "unable to return lpai to idle\n"); + return e.to_errno(); + } + data.powered.store(false, Relaxed); + } + 0 +} + +/// # Safety +/// +/// Called by ALSA with a live substream of this card's PCM, whose private +/// data is the driver's `Arc`. +unsafe extern "C" fn lpai_pcm_prepare(substream: *mut bindings::snd_pcm_substream) -> i32 { + // SAFETY: ALSA calls this op with a live substream of the LPAI PCM. + let data = unsafe { pcm_data(substream) }; + { + let mut st = data.stream.lock(); + // SAFETY: ALSA calls this op with a live substream; its runtime and + // DMA buffer exist between hw_params and hw_free, which bracket + // prepare. + st.buffer_bytes = unsafe { (*(*substream).runtime).dma_bytes }; + st.hw_ptr_bytes = 0; + st.period_acc = 0; + st.copied = None; + data.hw_ptr_frames.store(0, Relaxed); + } + if !data.powered.load(Relaxed) { + if let Err(e) = data.set_power(POWER_STATE_RUN) { + dev_err!(data.dev, "unable to run lpai\n"); + return e.to_errno(); + } + data.powered.store(true, Relaxed); + } + 0 +} + +/// Called with the ALSA stream lock held: only the run flag changes here, +/// the counters were reset by prepare. +/// +/// # Safety +/// +/// Called by ALSA with a live substream of this card's PCM, whose private +/// data is the driver's `Arc`. +unsafe extern "C" fn lpai_pcm_trigger( + substream: *mut bindings::snd_pcm_substream, + cmd: i32, +) -> i32 { + // SAFETY: ALSA calls this op with a live substream of the LPAI PCM. + let data = unsafe { pcm_data(substream) }; + match cmd as u32 { + bindings::SNDRV_PCM_TRIGGER_START | bindings::SNDRV_PCM_TRIGGER_RESUME => { + data.running.store(true, Relaxed); + 0 + } + bindings::SNDRV_PCM_TRIGGER_STOP | bindings::SNDRV_PCM_TRIGGER_SUSPEND => { + data.running.store(false, Relaxed); + 0 + } + _ => EINVAL.to_errno(), + } +} + +/// # Safety +/// +/// Called by ALSA with a live substream of this card's PCM, whose private +/// data is the driver's `Arc`. +unsafe extern "C" fn lpai_pcm_pointer( + substream: *mut bindings::snd_pcm_substream, +) -> bindings::snd_pcm_uframes_t { + // SAFETY: ALSA calls this op with a live substream of the LPAI PCM. + let data = unsafe { pcm_data(substream) }; + data.hw_ptr_frames.load(Relaxed) as bindings::snd_pcm_uframes_t +} + +/// The microphone array's capture: the LEAP produces float32 into the ADMAC +/// channel requested here; the channel's SRAM registers only answer while +/// its own LEAP leaf is powered, so the service (and with it the leaves) +/// comes up before the request and goes down after the release. +/// +/// # Safety +/// +/// Called by ALSA with a live substream of this card's PCM, whose private +/// data is the driver's `Arc`. +unsafe extern "C" fn hpai_pcm_open(substream: *mut bindings::snd_pcm_substream) -> i32 { + // SAFETY: ALSA calls this op with a live substream of the HPAI PCM. + let data = unsafe { pcm_data(substream) }; + if let Err(e) = data.hpai_startup() { + return e.to_errno(); + } + // SAFETY: the platform device is live and the channel name is + // NUL-terminated. + let chan = match from_err_ptr(unsafe { + bindings::dma_request_chan(data.dev.as_raw(), c"hpai".as_char_ptr()) + }) { + Ok(chan) => chan, + Err(e) => { + dev_err!(data.dev, "hpai: DMA channel unavailable: {:?}\n", e); + data.hpai_shutdown(); + return e.to_errno(); + } + }; + let mut hw = dmaengine_hw(); + hw.rates = bindings::SNDRV_PCM_RATE_48000; + hw.rate_min = HPAI_RATE; + hw.rate_max = HPAI_RATE; + hw.channels_min = HPAI_CHANNELS; + hw.channels_max = HPAI_CHANNELS; + let ret = pcm_open_dmaengine( + substream, + chan, + hw, + bindings::BINDINGS_SNDRV_PCM_FMTBIT_FLOAT_LE, + ); + if ret < 0 { + // SAFETY: `chan` was requested above and is not attached to the + // substream when the open failed. + unsafe { bindings::dma_release_channel(chan) }; + data.hpai_shutdown(); + } + ret +} + +/// # Safety +/// +/// Called by ALSA with a live substream of this card's PCM, whose private +/// data is the driver's `Arc`. +unsafe extern "C" fn hpai_pcm_close(substream: *mut bindings::snd_pcm_substream) -> i32 { + // SAFETY: ALSA calls this op with a live substream of the HPAI PCM. + let data = unsafe { pcm_data(substream) }; + // SAFETY: the substream is live and its dmaengine runtime was set up by + // `snd_dmaengine_pcm_open` in `hpai_pcm_open`. + let ret = unsafe { bindings::snd_dmaengine_pcm_close_release_chan(substream) }; + data.hpai_shutdown(); + ret +} + +/// Trigger, with the ALSA stream lock held: the DMA first, then the +/// sleeping firmware start queued behind it. +/// +/// # Safety +/// +/// Called by ALSA with a live substream of this card's PCM, whose private +/// data is the driver's `Arc`. +unsafe extern "C" fn hpai_pcm_trigger( + substream: *mut bindings::snd_pcm_substream, + cmd: i32, +) -> i32 { + // SAFETY: ALSA calls this op with a live substream of the HPAI PCM. + let data = unsafe { pcm_data(substream) }; + // SAFETY: the substream is live and its dmaengine runtime was set up by + // `snd_dmaengine_pcm_open` in `hpai_pcm_open`. + let ret = unsafe { bindings::snd_dmaengine_pcm_trigger(substream, cmd) }; + if ret < 0 { + return ret; + } + match cmd as u32 { + bindings::SNDRV_PCM_TRIGGER_START + | bindings::SNDRV_PCM_TRIGGER_RESUME + | bindings::SNDRV_PCM_TRIGGER_PAUSE_RELEASE => { + data.hpai_want_run.store(true, Relaxed); + // Err = already queued: the pending run will see want_run. + let _ = workqueue::system().enqueue::<_, HPAI_START_WORK>(Arc::from(data)); + } + _ => data.hpai_want_run.store(false, Relaxed), + } + 0 +} + +/// # Safety +/// +/// Called once by ALSA when the PCM goes away, with the `Arc` leaked into +/// its private data at probe. +unsafe extern "C" fn pcm_free_private(pcm: *mut bindings::snd_pcm) { + // SAFETY: this is the `Arc` leaked into the PCM's private data at probe; + // ALSA calls this once, when the PCM goes away. + unsafe { + Arc::::from_foreign((*pcm).private_data.cast()); + } +} + +/// The dmaengine PCMs' common description: mmap-able, interleaved, a 1 MiB +/// buffer in periods from 256 bytes to the ADMAC's segment size. Rates, +/// channels and formats are left to the caller (or, for a DPCM front-end, +/// to the DAI streams: ASoC fills in what is still zero here). +fn dmaengine_hw() -> bindings::snd_pcm_hardware { + bindings::snd_pcm_hardware { + info: bindings::SNDRV_PCM_INFO_MMAP + | bindings::SNDRV_PCM_INFO_MMAP_VALID + | bindings::SNDRV_PCM_INFO_INTERLEAVED, + formats: 0, + subformats: 0, + rates: 0, + rate_min: 0, + rate_max: 0, + channels_min: 0, + channels_max: 0, + buffer_bytes_max: PCM_BUFFER_MAX, + period_bytes_min: PCM_PERIOD_BYTES_MIN, + period_bytes_max: PCM_PERIOD_BYTES_MAX, + periods_min: 2, + periods_max: (PCM_BUFFER_MAX / PCM_PERIOD_BYTES_MIN) as u32, + fifo_size: 16, + } +} + +/// Set up a dmaengine-backed substream on `chan`: a managed buffer on the +/// DMA device (once per substream), `hw` refined by the channel's +/// capabilities and its formats narrowed to `formats`, then the dmaengine +/// runtime. On failure the channel is left with the caller. +fn pcm_open_dmaengine( + substream: *mut bindings::snd_pcm_substream, + chan: *mut bindings::dma_chan, + mut hw: bindings::snd_pcm_hardware, + formats: u64, +) -> i32 { + // SAFETY: the callers hand over a live substream and a channel they + // requested. + let unallocated = + unsafe { (*substream).dma_buffer.dev.type_ == bindings::SNDRV_DMA_TYPE_UNKNOWN as i32 }; + if unallocated { + // SAFETY: as above; the channel's device is the DMA controller and + // outlives the buffer, which ALSA frees with the PCM. + let ret = unsafe { + bindings::snd_pcm_set_managed_buffer( + substream, + bindings::SNDRV_DMA_TYPE_DEV as i32, + (*(*chan).device).dev, + 0, + 0, + ) + }; + if ret < 0 { + return ret; + } + } + let mut dma_data = bindings::snd_dmaengine_dai_dma_data::default(); + // SAFETY: the substream and the channel are live for the call. + let ret = unsafe { + bindings::snd_dmaengine_pcm_refine_runtime_hwparams(substream, &mut dma_data, &mut hw, chan) + }; + if ret < 0 { + return ret; + } + // The refinement describes the bus widths the channel can move, not the + // sample representation; the LEAP produces and consumes one encoding. + hw.formats &= formats; + if hw.formats == 0 { + return EINVAL.to_errno(); + } + // SAFETY: the substream's runtime exists for the op's duration; the + // dmaengine open attaches `chan` to it on success. + unsafe { + (*(*substream).runtime).hw = hw; + bindings::snd_dmaengine_pcm_open(substream, chan) + } +} + +/// Buffer limits of the dmaengine PCMs. A period is bounded by the ADMAC's +/// segment size (the channel's capabilities refine it further). +const PCM_BUFFER_MAX: usize = 1024 * 1024; +const PCM_PERIOD_BYTES_MIN: usize = 256; +const PCM_PERIOD_BYTES_MAX: usize = 0x10000; +/// A copy-time gain change must not sit behind a long queue of converted +/// speaker samples: at 48 kHz stereo S32 this bounds the queued audio to +/// 16384 frames (341 ms), while still allowing the largest PipeWire quantum. +const SPKR_BUFFER_MAX: usize = 128 * 1024; + +fn copy_str(target: &mut [u8], source: &[u8]) { + target[..source.len()].copy_from_slice(source) +} + +// ---- ASoC component: DPCM front-end PCMs on the ADMAC channels, back-end +// DAIs on the AOP output services (macaudio's J700 variant links them) ---- + +/// Front-end PCM DAIs ("j700-pcm-N"): 0 the jack (playback and the headset +/// microphone), 1 the speaker (playback only), 2 the speaker tap (capture). +const FE_COUNT: usize = 3; +/// DMA channel names per front-end and stream (dma-names on the audio +/// child): [playback, capture]. The primary front-end is the jack: cout on +/// base-ns TX2, cin (the headset microphone) on base-ns RX2. The speaker +/// front-end's "spkr" channel is on the LEAP ADMAC: requesting it touches +/// the channel's SRAM registers, which only answer with its leaf powered. +const FE_DMA_NAMES: [[Option<&core::ffi::CStr>; 2]; FE_COUNT] = [ + [Some(c"cout"), Some(c"cin")], + [Some(c"spkr"), None], + [None, Some(c"spkr-tap")], +]; +/// The front-end that plays the internal speaker (j700-pcm-1, the card's +/// "Secondary" PCM): its data goes through `spkr_copy` below. +const FE_SPKR: usize = 1; + +/// Digital ceiling of the speaker path, as a Q31 gain applied to every +/// sample userspace writes: 0.5, the amplifier's full scale. +/// +/// The words on the speaker wire are `2 × float × 2^31`, as read back on +/// the SpeakerTap: a float of 6 791 000/2^31 came back as a clean 1 kHz +/// sine with peak 13 582 000 and nothing else in the path. A float of 0.5 +/// is therefore the I2S full scale (0 dBFS on the wire, 7.3 V peak at the +/// MAX98360A's output on the J700) and anything above it would wrap. +/// Protection is the userspace daemon's job, exactly as on the other Apple +/// laptops: it owns "Speaker Playback Volume" and governs it with the +/// speaker's thermal model, fed by the tap; when no daemon holds the lock, +/// macaudio keeps the control 20 dB down. +const SPKR_CEILING_Q31: i64 = 1 << 30; + +/// "Speaker Playback Volume": the software gain below the ceiling that the +/// speaker-protection daemon and the card's volume lock drive, 0.5 dB per +/// step from -63.0 dB (1) to 0 dB (127) relative to SPKR_CEILING_Q31, 0 = +/// mute; entry i = round(SPKR_CEILING_Q31 * 10^(-(127 - i) / 40)). +const SPKR_VOLUME_MAX: u32 = 127; +const _: () = assert!(SPKR_VOLUME_Q31[SPKR_VOLUME_MAX as usize] as i64 == SPKR_CEILING_Q31); +const SPKR_VOLUME_Q31: [u32; 128] = [ + 0, 760151, 805193, 852903, 903441, 956973, 1013678, 1073742, 1137365, 1204758, 1276145, + 1351761, 1431858, 1516701, 1606571, 1701766, 1802602, 1909413, 2022553, 2142397, 2269342, + 2403809, 2546243, 2697118, 2856932, 3026216, 3205530, 3395470, 3596664, 3809780, 4035523, + 4274643, 4527932, 4796229, 5080423, 5381457, 5700328, 6038094, 6395874, 6774853, 7176288, + 7601510, 8051928, 8529034, 9034412, 9569734, 10136776, 10737418, 11373650, 12047581, 12761445, + 13517609, 14318577, 15167006, 16065708, 17017661, 18026021, 19094130, 20225528, 21423966, + 22693416, 24038085, 25462431, 26971175, 28569318, 30262156, 32055302, 33954698, 35966640, + 38097798, 40355234, 42746432, 45279317, 47962285, 50804230, 53814569, 57003283, 60380940, + 63958736, 67748529, 71762882, 76015100, 80519278, 85290345, 90344115, 95697341, 101367765, + 107374182, 113736503, 120475814, 127614455, 135176087, 143185773, 151670064, 160657080, + 170176611, 180260209, 190941298, 202255281, 214239660, 226934158, 240380852, 254624313, + 269711752, 285693178, 302621563, 320553018, 339546978, 359666402, 380977976, 403552340, + 427464319, 452793173, 479622855, 508042296, 538145694, 570032831, 603809400, 639587356, + 677485290, 717628817, 760150998, 805192776, 852903448, 903441154, 956973408, 1013677647, + 1073741824, +]; +/// The component's only "register": the speaker volume. +const REG_SPKR_VOLUME: u32 = 0; +/// DECLARE_TLV_DB_SCALE(-6350, 50, 1): a dB scale from -63.50 dB in 0.50 dB +/// steps whose lowest value mutes. +static SPKR_VOLUME_TLV: [u32; 4] = [ + bindings::SNDRV_CTL_TLVT_DB_SCALE, + 2 * mem::size_of::() as u32, + (-6350i32) as u32, + 50 | bindings::TLV_DB_SCALE_MUTE, +]; + +/// Audio data from a live ASoC component's separately owned context. +/// +/// # Safety +/// +/// `component` must belong to this driver's live per-instance descriptor. +/// The descriptor/context outlives synchronous registration and unregister. +unsafe fn component_data<'a>( + component: *mut bindings::snd_soc_component, +) -> &'a SndSocT8140AopData { + // SAFETY: by the caller's contract. + unsafe { &component_context(component).data } +} + +/// The stable context behind this component's descriptor. +/// +/// # Safety +/// +/// As for [`component_data`]. +unsafe fn component_context<'a>(component: *mut bindings::snd_soc_component) -> &'a AsocContext { + // SAFETY: by the caller's contract. + let descriptor = unsafe { (*component).driver }; + // SAFETY: AsocContext is repr(C), with the descriptor as its first field. + // C keeps the exact descriptor pointer passed to registration. + unsafe { &*descriptor.cast::() } +} + +/// The ASoC runtime of a substream. +/// +/// # Safety +/// +/// `substream` must be live and belong to one of this component's links. +unsafe fn substream_rtd( + substream: *mut bindings::snd_pcm_substream, +) -> *mut bindings::snd_soc_pcm_runtime { + // SAFETY: by the caller's contract. + unsafe { + (*substream) + .private_data + .cast::() + } +} + +/// Whether a runtime is a DPCM back-end (no PCM of its own). +/// +/// # Safety +/// +/// `rtd` must be a live ASoC runtime. +unsafe fn rtd_is_be(rtd: *mut bindings::snd_soc_pcm_runtime) -> bool { + // SAFETY: by the caller's contract. + unsafe { (*(*rtd).dai_link).no_pcm() != 0 } +} + +/// The first CPU DAI of a runtime. +/// +/// # Safety +/// +/// `rtd` must be a live ASoC runtime. +unsafe fn rtd_cpu_dai(rtd: *mut bindings::snd_soc_pcm_runtime) -> *mut bindings::snd_soc_dai { + // SAFETY: by the caller's contract. + unsafe { *(*rtd).dais } +} + +/// DAI ids: 0 means "assign one" to ASoC, so the front-ends carry 1..=3 +/// (index + 1) and the back-ends start at 10. +const FE_DAI_ID_BASE: u32 = 1; +const BE_DAI_ID_COUT: u32 = 10; +const BE_DAI_ID_SPKR: u32 = 11; + +/// The front-end index of a runtime, `None` for a back-end. +/// +/// # Safety +/// +/// `rtd` must be a live ASoC runtime of one of this component's links. +unsafe fn fe_index(rtd: *mut bindings::snd_soc_pcm_runtime) -> Option { + // SAFETY: by the caller's contract. + let id = unsafe { (*rtd_cpu_dai(rtd)).id } as u32; + if id >= FE_DAI_ID_BASE && id < FE_DAI_ID_BASE + FE_COUNT as u32 { + Some((id - FE_DAI_ID_BASE) as usize) + } else { + None + } +} + +/// # Safety +/// +/// `substream` must be live. +unsafe fn substream_is_playback(substream: *mut bindings::snd_pcm_substream) -> bool { + // SAFETY: by the caller's contract. + let stream = unsafe { (*substream).stream }; + stream == bindings::SNDRV_PCM_STREAM_PLAYBACK as i32 +} + +/// Front-end open: request the stream's ADMAC channel and set the substream +/// up on it. The speaker channel is on the LEAP ADMAC, whose channel +/// registers only answer with the LEAP leaves powered, so those are held +/// from here to close (the back-end holds its own references while the +/// service runs). +/// +/// # Safety +/// +/// Called by ASoC on this driver's registered component, with live +/// arguments belonging to one of its links. +unsafe extern "C" fn asoc_pcm_open( + component: *mut bindings::snd_soc_component, + substream: *mut bindings::snd_pcm_substream, +) -> i32 { + // SAFETY: ASoC calls this op on a component whose descriptor/context stays live + // throughout registration, use and unregister. + let data = unsafe { component_data(component) }; + // SAFETY: ASoC hands its ops a live substream whose private data is the + // `snd_soc_pcm_runtime` of one of this component's links. + let rtd = unsafe { substream_rtd(substream) }; + // SAFETY: as above. + if unsafe { rtd_is_be(rtd) } { + return 0; + } + // SAFETY: as above. + let Some(fe) = (unsafe { fe_index(rtd) }) else { + return EINVAL.to_errno(); + }; + // SAFETY: the substream is live for the duration of the op. + let playback = unsafe { substream_is_playback(substream) }; + let Some(name) = FE_DMA_NAMES[fe][usize::from(!playback)] else { + return ENODEV.to_errno(); + }; + let leap = fe == FE_SPKR; + if leap { + if let Err(e) = data.power_up(data.spkr.name, &data.spkr_domains()) { + return e.to_errno(); + } + } + // SAFETY: the platform device is live and the channel name is + // NUL-terminated. + let chan = match from_err_ptr(unsafe { + bindings::dma_request_chan(data.dev.as_raw(), name.as_char_ptr()) + }) { + Ok(chan) => chan, + Err(e) => { + dev_err!( + data.dev, + "front-end {}: DMA channel {:?} unavailable: {:?}\n", + fe, + name, + e + ); + if leap { + data.power_down(&data.spkr_domains()); + } + return e.to_errno(); + } + }; + let mut hw = dmaengine_hw(); + let mut formats = u64::MAX; + if fe == FE_SPKR { + // S32 in, converted by `asoc_pcm_copy` through the ceiling into the + // float32 the LEAP consumes: no mmap, and a bounded queue. + hw.info = bindings::SNDRV_PCM_INFO_INTERLEAVED; + hw.buffer_bytes_max = SPKR_BUFFER_MAX; + hw.periods_max = (SPKR_BUFFER_MAX / PCM_PERIOD_BYTES_MIN) as u32; + formats = bindings::BINDINGS_SNDRV_PCM_FMTBIT_S32_LE; + } + let ret = pcm_open_dmaengine(substream, chan, hw, formats); + if ret < 0 { + // SAFETY: `chan` was requested above and is not attached to the + // substream when the open failed. + unsafe { bindings::dma_release_channel(chan) }; + if leap { + data.power_down(&data.spkr_domains()); + } + } + ret +} + +/// # Safety +/// +/// Called by ASoC on this driver's registered component, with live +/// arguments belonging to one of its links. +unsafe extern "C" fn asoc_pcm_close( + component: *mut bindings::snd_soc_component, + substream: *mut bindings::snd_pcm_substream, +) -> i32 { + // SAFETY: ASoC calls this op on a component whose descriptor/context stays live + // throughout registration, use and unregister. + let data = unsafe { component_data(component) }; + // SAFETY: ASoC hands its ops a live substream whose private data is the + // `snd_soc_pcm_runtime` of one of this component's links. + let rtd = unsafe { substream_rtd(substream) }; + // SAFETY: as above. + if unsafe { rtd_is_be(rtd) } { + return 0; + } + // SAFETY: the substream is live and its dmaengine runtime was set up by + // `snd_dmaengine_pcm_open` in `asoc_pcm_open`. + let ret = unsafe { bindings::snd_dmaengine_pcm_close_release_chan(substream) }; + // SAFETY: as for `rtd_is_be`. + if unsafe { fe_index(rtd) } == Some(FE_SPKR) { + data.power_down(&data.spkr_domains()); + } + ret +} + +/// # Safety +/// +/// Called by ASoC on this driver's registered component, with live +/// arguments belonging to one of its links. +unsafe extern "C" fn asoc_pcm_hw_params( + _component: *mut bindings::snd_soc_component, + substream: *mut bindings::snd_pcm_substream, + params: *mut bindings::snd_pcm_hw_params, +) -> i32 { + // SAFETY: ASoC hands its ops a live substream whose private data is the + // `snd_soc_pcm_runtime` of one of this component's links. + if unsafe { rtd_is_be(substream_rtd(substream)) } { + return 0; + } + // SAFETY: the substream is live and its dmaengine runtime was set up by + // `snd_dmaengine_pcm_open` in `asoc_pcm_open`. + let chan = unsafe { bindings::snd_dmaengine_pcm_get_chan(substream) }; + let mut cfg = bindings::dma_slave_config::default(); + // SAFETY: the substream and the hardware parameters are live for the op. + let rc = unsafe { bindings::snd_hwparams_to_dma_slave_config(substream, params, &mut cfg) }; + if rc < 0 { + return rc; + } + // Interval query for the channel count: bindgen has no inline helper. + // SAFETY: ASoC calls hw_params with a live `snd_pcm_hw_params`. + let channels = unsafe { + let iv = &(*params).intervals[(bindings::SNDRV_PCM_HW_PARAM_CHANNELS + - bindings::SNDRV_PCM_HW_PARAM_FIRST_INTERVAL) + as usize]; + iv.min + }; + // The ADMAC moves one frame per burst, up to four words wide. + let window = core::cmp::min(channels, 4); + // SAFETY: the substream is live for the duration of the op. + if unsafe { substream_is_playback(substream) } { + cfg.dst_port_window_size = window; + } else { + cfg.src_port_window_size = window; + } + // SAFETY: `chan` is the live channel behind this substream's dmaengine runtime. + unsafe { + match (*(*chan).device).device_config { + Some(f) => f(chan, &mut cfg), + None => ENOSYS.to_errno(), + } + } +} + +/// Clear old words before a new speaker stream or recovery from a revoked +/// protection lease; the queued firmware start follows the DMA trigger. +/// +/// # Safety +/// +/// Called by ASoC on this driver's registered component, with live +/// arguments belonging to one of its links. +unsafe extern "C" fn asoc_pcm_prepare( + _component: *mut bindings::snd_soc_component, + substream: *mut bindings::snd_pcm_substream, +) -> i32 { + // SAFETY: ASoC hands its ops a live substream whose private data is the + // `snd_soc_pcm_runtime` of one of this component's links. + let rtd = unsafe { substream_rtd(substream) }; + // SAFETY: as above. + if unsafe { rtd_is_be(rtd) } || unsafe { fe_index(rtd) } != Some(FE_SPKR) { + return 0; + } + // SAFETY: ALSA calls this op with a live substream; its runtime and DMA + // buffer exist between hw_params and hw_free, which bracket prepare. + let (area, bytes) = unsafe { + let runtime = (*substream).runtime; + ((*runtime).dma_area, (*runtime).dma_bytes) + }; + if !area.is_null() && bytes != 0 { + // SAFETY: the managed buffer is dma_bytes long and no DMA runs before + // trigger START. + unsafe { ptr::write_bytes(area, 0, bytes) }; + } + 0 +} + +/// The float32 bit pattern of `v / 2^31`, built with integer arithmetic (the +/// kernel has no floating point). +fn f32_bits_of_q31(v: i32) -> u32 { + if v == 0 { + return 0; + } + let sign = if v < 0 { 1u32 << 31 } else { 0 }; + let m = v.unsigned_abs(); + let p = 31 - m.leading_zeros(); // position of the leading one, 0..=31 + // value = m * 2^-31 = 1.f * 2^(p - 31) + let exp = (p as i32 - 31 + 127) as u32; + let mant = if p >= 23 { + (m >> (p - 23)) & 0x7f_ffff + } else { + (m << (23 - p)) & 0x7f_ffff + }; + sign | (exp << 23) | mant +} + +/// One S32 sample -> the float32 word the LEAP consumes, through the volume +/// table (whose top entry is the ceiling). +fn spkr_f32_bits(sample: i32, gain_q31: u32) -> u32 { + let scaled = (i64::from(sample) * i64::from(gain_q31)) >> 31; // |scaled| < SPKR_CEILING_Q31 + f32_bits_of_q31(scaled as i32) +} + +/// Copy between userspace and the front-end buffers. The speaker front-end +/// converts and limits; the jack front-ends copy verbatim (their buffers are +/// also mmap-able, this path serves read()/write()). +/// +/// # Safety +/// +/// Called by ASoC on this driver's registered component, with live +/// arguments belonging to one of its links. +unsafe extern "C" fn asoc_pcm_copy( + component: *mut bindings::snd_soc_component, + substream: *mut bindings::snd_pcm_substream, + _channel: i32, + pos: usize, + iter: *mut bindings::iov_iter, + bytes: usize, +) -> i32 { + // SAFETY: ASoC hands its ops a live substream whose private data is the + // `snd_soc_pcm_runtime` of one of this component's links. + let rtd = unsafe { substream_rtd(substream) }; + // SAFETY: as above. + if unsafe { rtd_is_be(rtd) } { + return EINVAL.to_errno(); + } + // SAFETY: ALSA calls this op with a live substream; its runtime and DMA + // buffer exist between hw_params and hw_free, which bracket every copy. + let (area, dma_bytes) = unsafe { + let runtime = (*substream).runtime; + ((*runtime).dma_area, (*runtime).dma_bytes) + }; + if area.is_null() || pos.checked_add(bytes).is_none_or(|end| end > dma_bytes) { + return EINVAL.to_errno(); + } + // SAFETY: bounds checked against the managed buffer above; ALSA + // serializes the copies of a substream. + let dst = unsafe { slice::from_raw_parts_mut(area.add(pos), bytes) }; + // SAFETY: the substream is live for the duration of the op. + let playback = unsafe { substream_is_playback(substream) }; + if !playback { + // SAFETY: ALSA hands us a live iov_iter for the whole call. + let iov = unsafe { kernel::iov::IovIterDest::from_raw(iter) }; + return if iov.copy_to_iter(dst) == bytes { + 0 + } else { + EFAULT.to_errno() + }; + } + // SAFETY: as above. + let iov = unsafe { kernel::iov::IovIterSource::from_raw(iter) }; + // SAFETY: as for `rtd_is_be`. + if unsafe { fe_index(rtd) } != Some(FE_SPKR) { + return if iov.copy_from_iter(dst) == bytes { + 0 + } else { + EFAULT.to_errno() + }; + } + // S32_LE interleaved stereo in, float32 LE out, same byte count. + // SAFETY: ASoC calls this op on a component whose descriptor/context stays live + // throughout registration, use and unregister. + let data = unsafe { component_data(component) }; + let volume = core::cmp::min(data.spkr_volume.load(Relaxed), SPKR_VOLUME_MAX) as usize; + let gain = SPKR_VOLUME_Q31[volume]; + let mut chunk = [0u8; 512]; + let mut off = 0; + while off < bytes { + let n = core::cmp::min(chunk.len(), bytes - off) & !3; + if n == 0 { + return EINVAL.to_errno(); + } + if iov.copy_from_iter(&mut chunk[..n]) != n { + return EFAULT.to_errno(); + } + for (i, word) in chunk[..n].chunks_exact(4).enumerate() { + let sample = i32::from_le_bytes([word[0], word[1], word[2], word[3]]); + let bits = spkr_f32_bits(sample, gain); + dst[off + i * 4..off + i * 4 + 4].copy_from_slice(&bits.to_le_bytes()); + } + off += n; + } + 0 +} + +/// # Safety +/// +/// Called by ASoC on this driver's registered component, with live +/// arguments belonging to one of its links. +unsafe extern "C" fn asoc_pcm_trigger( + _component: *mut bindings::snd_soc_component, + substream: *mut bindings::snd_pcm_substream, + cmd: i32, +) -> i32 { + // SAFETY: ASoC hands its ops a live substream whose private data is the + // `snd_soc_pcm_runtime` of one of this component's links. + if unsafe { rtd_is_be(substream_rtd(substream)) } { + return 0; + } + // SAFETY: the substream is live and its dmaengine runtime was set up by + // `snd_dmaengine_pcm_open` in `asoc_pcm_open`. + unsafe { bindings::snd_dmaengine_pcm_trigger(substream, cmd) } +} + +/// # Safety +/// +/// Called by ASoC on this driver's registered component, with live +/// arguments belonging to one of its links. +unsafe extern "C" fn asoc_pcm_pointer( + _component: *mut bindings::snd_soc_component, + substream: *mut bindings::snd_pcm_substream, +) -> bindings::snd_pcm_uframes_t { + // SAFETY: ASoC hands its ops a live substream whose private data is the + // `snd_soc_pcm_runtime` of one of this component's links. + if unsafe { rtd_is_be(substream_rtd(substream)) } { + // -ENOTSUPP as the pointer callback returns it (mca does the same). + return (-i64::from(bindings::ENOTSUPP)) as bindings::snd_pcm_uframes_t; + } + // SAFETY: the substream is live and its dmaengine runtime was set up by + // `snd_dmaengine_pcm_open` in `asoc_pcm_open`. + unsafe { bindings::snd_dmaengine_pcm_pointer(substream) } +} + +/// The component's register file: the speaker volume, read and written by +/// the standard volsw control helpers. +/// +/// # Safety +/// +/// Called by ASoC on this driver's registered component, with live +/// arguments belonging to one of its links. +unsafe extern "C" fn asoc_component_read( + component: *mut bindings::snd_soc_component, + reg: u32, +) -> u32 { + // SAFETY: ASoC calls this op on a component whose descriptor/context stays live + // throughout registration, use and unregister. + let data = unsafe { component_data(component) }; + match reg { + REG_SPKR_VOLUME => data.spkr_volume.load(Relaxed), + _ => 0, + } +} + +/// # Safety +/// +/// Called by ASoC on this driver's registered component, with live +/// arguments belonging to one of its links. +unsafe extern "C" fn asoc_component_write( + component: *mut bindings::snd_soc_component, + reg: u32, + val: u32, +) -> i32 { + // SAFETY: ASoC calls this op on a component whose descriptor/context stays live + // throughout registration, use and unregister. + let data = unsafe { component_data(component) }; + match reg { + REG_SPKR_VOLUME => { + data.spkr_volume + .store(core::cmp::min(val, SPKR_VOLUME_MAX), Relaxed); + 0 + } + _ => EINVAL.to_errno(), + } +} + +/// Driver data from one of this component's DAIs. +/// +/// # Safety +/// +/// `dai` must be a live DAI of this driver's registered component. +unsafe fn dai_data<'a>(dai: *mut bindings::snd_soc_dai) -> &'a SndSocT8140AopData { + // SAFETY: by the caller's contract. + unsafe { component_data((*dai).component) } +} + +/// The ASoC context of one of this component's DAIs. +/// +/// # Safety +/// +/// As for [`dai_data`]. +unsafe fn dai_context<'a>(dai: *mut bindings::snd_soc_dai) -> &'a AsocContext { + // SAFETY: by the caller's contract. + unsafe { component_context((*dai).component) } +} + +/// # Safety +/// +/// Called by ASoC with a live DAI of this driver's registered component +/// (and a live substream where one is passed). +unsafe extern "C" fn dai_accept_fmt(_dai: *mut bindings::snd_soc_dai, _fmt: u32) -> i32 { + // The AOP runs the serializers: the format, slots and clocks are the + // firmware's, whatever the machine driver asks for. + 0 +} + +/// # Safety +/// +/// Called by ASoC with a live DAI of this driver's registered component +/// (and a live substream where one is passed). +unsafe extern "C" fn dai_accept_sysclk( + _dai: *mut bindings::snd_soc_dai, + _id: i32, + _freq: u32, + _dir: i32, +) -> i32 { + 0 +} + +/// # Safety +/// +/// Called by ASoC with a live DAI of this driver's registered component +/// (and a live substream where one is passed). +unsafe extern "C" fn dai_accept_bclk_ratio(_dai: *mut bindings::snd_soc_dai, _ratio: u32) -> i32 { + 0 +} + +/// # Safety +/// +/// Called by ASoC with a live DAI of this driver's registered component +/// (and a live substream where one is passed). +unsafe extern "C" fn cout_dai_startup( + substream: *mut bindings::snd_pcm_substream, + dai: *mut bindings::snd_soc_dai, +) -> i32 { + // SAFETY: ASoC calls DAI ops with a DAI of our registered component. + let data = unsafe { dai_data(dai) }; + // SAFETY: the substream is live for the duration of the op. + match data.jack_startup(unsafe { substream_is_playback(substream) }) { + Ok(()) => 0, + Err(e) => e.to_errno(), + } +} + +/// # Safety +/// +/// Called by ASoC with a live DAI of this driver's registered component +/// (and a live substream where one is passed). +unsafe extern "C" fn cout_dai_prepare( + substream: *mut bindings::snd_pcm_substream, + dai: *mut bindings::snd_soc_dai, +) -> i32 { + // SAFETY: ASoC calls DAI ops with a DAI of our registered component. + let data = unsafe { dai_data(dai) }; + // SAFETY: the substream is live for the duration of the op. + match data.jack_prepare(unsafe { substream_is_playback(substream) }) { + Ok(()) => 0, + Err(e) => e.to_errno(), + } +} + +/// # Safety +/// +/// Called by ASoC with a live DAI of this driver's registered component +/// (and a live substream where one is passed). +unsafe extern "C" fn cout_dai_shutdown( + substream: *mut bindings::snd_pcm_substream, + dai: *mut bindings::snd_soc_dai, +) { + // SAFETY: ASoC calls DAI ops with a DAI of our registered component. + let data = unsafe { dai_data(dai) }; + // SAFETY: the substream is live for the duration of the op. + data.jack_shutdown(unsafe { substream_is_playback(substream) }); +} + +/// # Safety +/// +/// Called by ASoC with a live DAI of this driver's registered component +/// (and a live substream where one is passed). +unsafe extern "C" fn spkr_dai_startup( + substream: *mut bindings::snd_pcm_substream, + dai: *mut bindings::snd_soc_dai, +) -> i32 { + // SAFETY: ASoC calls DAI ops with a DAI of our registered component. + let data = unsafe { dai_data(dai) }; + // SAFETY: the substream is live for the duration of the op. + let ret = if unsafe { substream_is_playback(substream) } { + data.spkr_startup() + } else { + data.sense_startup() + }; + match ret { + Ok(()) => 0, + Err(e) => e.to_errno(), + } +} + +/// The tap (capture) side runs from prepare: the RX DMA is armed by then +/// and a capture may start before or after the speaker plays. +/// +/// # Safety +/// +/// Called by ASoC with a live DAI of this driver's registered component +/// (and a live substream where one is passed). +unsafe extern "C" fn spkr_dai_prepare( + substream: *mut bindings::snd_pcm_substream, + dai: *mut bindings::snd_soc_dai, +) -> i32 { + // SAFETY: ASoC calls DAI ops with a DAI of our registered component. + let data = unsafe { dai_data(dai) }; + // SAFETY: the substream is live for the duration of the op. + let ret = if unsafe { substream_is_playback(substream) } { + data.spkr_prepare() + } else { + data.sense_prepare() + }; + match ret { + Ok(()) => 0, + Err(e) => e.to_errno(), + } +} + +/// Atomic context: queue the sleeping start, or note the stop (playback +/// only; the tap side needs nothing here). +/// +/// # Safety +/// +/// Called by ASoC with a live DAI of this driver's registered component +/// (and a live substream where one is passed). +unsafe extern "C" fn spkr_dai_trigger( + substream: *mut bindings::snd_pcm_substream, + cmd: i32, + dai: *mut bindings::snd_soc_dai, +) -> i32 { + // SAFETY: the substream is live for the duration of the op. + if !unsafe { substream_is_playback(substream) } { + return 0; + } + // SAFETY: ASoC calls DAI ops with a DAI of our registered component. + let context = unsafe { dai_context(dai) }; + match cmd as u32 { + bindings::SNDRV_PCM_TRIGGER_START + | bindings::SNDRV_PCM_TRIGGER_RESUME + | bindings::SNDRV_PCM_TRIGGER_PAUSE_RELEASE => { + context.data.spkr_want_run.store(true, Relaxed); + // Err = already queued: the pending run will see want_run. + let _ = workqueue::system().enqueue::<_, SPKR_START_WORK>(context.data.clone()); + 0 + } + bindings::SNDRV_PCM_TRIGGER_STOP + | bindings::SNDRV_PCM_TRIGGER_SUSPEND + | bindings::SNDRV_PCM_TRIGGER_PAUSE_PUSH => { + context.data.spkr_want_run.store(false, Relaxed); + 0 + } + _ => EINVAL.to_errno(), + } +} + +/// # Safety +/// +/// Called by ASoC with a live DAI of this driver's registered component +/// (and a live substream where one is passed). +unsafe extern "C" fn spkr_dai_shutdown( + substream: *mut bindings::snd_pcm_substream, + dai: *mut bindings::snd_soc_dai, +) { + // SAFETY: ASoC calls DAI ops with a DAI of our registered component. + let data = unsafe { dai_data(dai) }; + // SAFETY: the substream is live for the duration of the op. + if unsafe { substream_is_playback(substream) } { + data.spkr_shutdown(); + } else { + data.sense_shutdown(); + } +} + +/// The DAI operation tables, built once per instance (the C descriptors +/// hold function pointers, which have no constant zero form in Rust). +struct DaiOps { + cout: bindings::snd_soc_dai_ops, + spkr: bindings::snd_soc_dai_ops, + fe: bindings::snd_soc_dai_ops, +} + +impl DaiOps { + fn new() -> Self { + let fe = bindings::snd_soc_dai_ops { + set_fmt: Some(dai_accept_fmt), + set_sysclk: Some(dai_accept_sysclk), + set_bclk_ratio: Some(dai_accept_bclk_ratio), + ..Default::default() + }; + DaiOps { + cout: bindings::snd_soc_dai_ops { + startup: Some(cout_dai_startup), + prepare: Some(cout_dai_prepare), + shutdown: Some(cout_dai_shutdown), + ..fe + }, + spkr: bindings::snd_soc_dai_ops { + startup: Some(spkr_dai_startup), + prepare: Some(spkr_dai_prepare), + trigger: Some(spkr_dai_trigger), + shutdown: Some(spkr_dai_shutdown), + ..fe + }, + fe, + } + } +} + +fn pcm_stream( + name: &'static core::ffi::CStr, + formats: u64, + channels: u32, +) -> bindings::snd_soc_pcm_stream { + bindings::snd_soc_pcm_stream { + stream_name: name.as_char_ptr(), + formats, + rates: bindings::SNDRV_PCM_RATE_48000, + rate_min: 48000, + rate_max: 48000, + channels_min: channels, + channels_max: channels, + ..Default::default() + } +} + +/// The AOP-configured serializers take 24-bit samples from the low bits +/// of each 32-bit DMA word: S24_LE. +const OUT_FORMATS: u64 = bindings::BINDINGS_SNDRV_PCM_FMTBIT_S24_LE; +const S32_FORMATS: u64 = bindings::BINDINGS_SNDRV_PCM_FMTBIT_S32_LE; + +/// The DAI table: [0] the cout back-end, [1..=3] the front-end PCMs, [4] the +/// speaker back-end. The order is ABI: `sound-dai = <&aop_audio N>` in the +/// machine DTS indexes this array, so `<&aop_audio 0>` is cout and +/// `<&aop_audio 4>` is spkr. New DAIs go at the end; inserting one +/// renumbers every DAI after it and silently repoints the existing links. +const DAI_COUNT: usize = 2 + FE_COUNT; + +fn dai_drivers(ops: &DaiOps) -> [bindings::snd_soc_dai_driver; DAI_COUNT] { + [ + bindings::snd_soc_dai_driver { + name: c"cout".as_char_ptr(), + id: BE_DAI_ID_COUT, + ops: &ops.cout, + playback: pcm_stream(c"COUT TX", OUT_FORMATS, 2), + // cin: the headset microphone, one 32-bit slot per frame + capture: pcm_stream(c"CIN RX", OUT_FORMATS, 1), + ..Default::default() + }, + bindings::snd_soc_dai_driver { + name: c"j700-pcm-0".as_char_ptr(), + id: FE_DAI_ID_BASE, + ops: &ops.fe, + playback: pcm_stream(c"PCM0 TX", OUT_FORMATS, 2), + // macaudio routes "Headset Capture" into PCM0 RX (mono cin) + capture: pcm_stream(c"PCM0 RX", OUT_FORMATS, 1), + ..Default::default() + }, + bindings::snd_soc_dai_driver { + name: c"j700-pcm-1".as_char_ptr(), + id: FE_DAI_ID_BASE + 1, + ops: &ops.fe, + // the speaker front-end: S32 in, converted through the ceiling + playback: pcm_stream(c"PCM1 TX", S32_FORMATS, 2), + ..Default::default() + }, + bindings::snd_soc_dai_driver { + name: c"j700-pcm-2".as_char_ptr(), + id: FE_DAI_ID_BASE + 2, + ops: &ops.fe, + // the speaker tap front-end: the words on the speaker wire + capture: pcm_stream(c"PCM2 RX", S32_FORMATS, 2), + ..Default::default() + }, + // The internal speaker back-end: LEAP TX0 under the AOP spkr service, + // the MAX98360A's SD_MODE driven by its own codec driver; its capture + // side is the SpeakerTap (base-ns RX1, the tap service). + bindings::snd_soc_dai_driver { + name: c"spkr".as_char_ptr(), + id: BE_DAI_ID_SPKR, + ops: &ops.spkr, + playback: pcm_stream(c"SPKR TX", S32_FORMATS, 2), + capture: pcm_stream(c"SPKR RX", S32_FORMATS, 2), + ..Default::default() + }, + ] +} + +/// The ASoC component and everything the core keeps pointers into. C +/// retains the address of `component`, so this is allocated before +/// registration and kept until unregistration has finished all callbacks. +#[repr(C)] +struct AsocContext { + component: bindings::snd_soc_component_driver, + data: Arc, + dais: KBox>, + _ops: KBox, + _controls: KBox<[bindings::snd_kcontrol_new; 1]>, + // The C mixer helpers modify platform_max under their control locks. + _volume: KBox>, +} + +const _: () = assert!(mem::offset_of!(AsocContext, component) == 0); + +impl AsocContext { + fn new(data: Arc) -> Result> { + let ops = KBox::new(DaiOps::new(), GFP_KERNEL)?; + let dais = KBox::new(Opaque::new(dai_drivers(&ops)), GFP_KERNEL)?; + let volume = KBox::new( + Opaque::new(bindings::soc_mixer_control { + min: 0, + max: SPKR_VOLUME_MAX as i32, + platform_max: SPKR_VOLUME_MAX as i32, + reg: REG_SPKR_VOLUME as i32, + rreg: REG_SPKR_VOLUME as i32, + ..Default::default() + }), + GFP_KERNEL, + )?; + // SOC_SINGLE_TLV("Speaker Playback Volume", REG_SPKR_VOLUME, 0, 127, 0, tlv) + let controls = KBox::new( + [bindings::snd_kcontrol_new { + iface: bindings::BINDINGS_SNDRV_CTL_ELEM_IFACE_MIXER, + name: c"Speaker Playback Volume".as_char_ptr(), + access: bindings::SNDRV_CTL_ELEM_ACCESS_TLV_READ + | bindings::SNDRV_CTL_ELEM_ACCESS_READWRITE, + info: Some(bindings::snd_soc_info_volsw), + get: Some(bindings::snd_soc_get_volsw), + put: Some(bindings::snd_soc_put_volsw), + tlv: bindings::snd_kcontrol_new__bindgen_ty_1 { + p: SPKR_VOLUME_TLV.as_ptr(), + }, + private_value: volume.get() as usize, + ..Default::default() + }], + GFP_KERNEL, + )?; + let component = bindings::snd_soc_component_driver { + name: c"t8140-aop-audio".as_char_ptr(), + read: Some(asoc_component_read), + write: Some(asoc_component_write), + controls: controls.as_ptr(), + num_controls: 1, + open: Some(asoc_pcm_open), + close: Some(asoc_pcm_close), + hw_params: Some(asoc_pcm_hw_params), + prepare: Some(asoc_pcm_prepare), + trigger: Some(asoc_pcm_trigger), + pointer: Some(asoc_pcm_pointer), + copy: Some(asoc_pcm_copy), + ..Default::default() + }; + Ok(KBox::new( + Self { + component, + data, + dais, + _ops: ops, + _controls: controls, + _volume: volume, + }, + GFP_KERNEL, + )?) + } +} + +struct SndSocT8140AopDriver { + card: *mut bindings::snd_card, + data: Arc, + asoc: KBox, + asoc_registered: bool, + closed: AtomicFlag, +} + +// SAFETY: the raw card pointer is only used from probe, unbind and drop, +// which the driver core serializes; the card's own state is guarded by +// ALSA. Everything else is `Send + Sync` on its own. +unsafe impl Send for SndSocT8140AopDriver {} +// SAFETY: as for `Send`. +unsafe impl Sync for SndSocT8140AopDriver {} + +impl SndSocT8140AopDriver { + const LPAI_OPS: bindings::snd_pcm_ops = bindings::snd_pcm_ops { + open: Some(lpai_pcm_open), + close: Some(lpai_pcm_close), + prepare: Some(lpai_pcm_prepare), + trigger: Some(lpai_pcm_trigger), + pointer: Some(lpai_pcm_pointer), + ioctl: None, + hw_params: None, + hw_free: None, + sync_stop: None, + get_time_info: None, + fill_silence: None, + copy: None, + page: None, + mmap: None, + ack: None, + }; + + const HPAI_OPS: bindings::snd_pcm_ops = bindings::snd_pcm_ops { + open: Some(hpai_pcm_open), + close: Some(hpai_pcm_close), + prepare: None, + trigger: Some(hpai_pcm_trigger), + pointer: Some(bindings::snd_dmaengine_pcm_pointer), + ioctl: None, + hw_params: Some(hpai_pcm_hw_params), + hw_free: None, + sync_stop: None, + get_time_info: None, + fill_silence: None, + copy: None, + page: None, + mmap: None, + ack: None, + }; + + fn new(data: Arc, fwnode: &FwNode) -> Result { + // From here on a failure drops `this`, whose teardown undoes + // whatever was set up. + let mut this = SndSocT8140AopDriver { + card: ptr::null_mut(), + asoc: AsocContext::new(data.clone())?, + data: data.clone(), + asoc_registered: false, + closed: AtomicFlag::new(false), + }; + + // The low-power microphone: its ring is reported to us by the + // firmware for the AOP's lifetime. + data.attach_device(AUDIO_DEV_LPAI, "lpai")?; + data.set_channel_control()?; + data.read_geometry()?; + let listener: Arc = data.clone(); + data.adata + .add_report_listener(data.service, EPIC_SUBTYPE_PRODUCER_REPORT, listener)?; + + // SAFETY: FFI call with the live device and the module; `this.card` receives the new card. + let ret = unsafe { + bindings::snd_card_new( + data.dev.as_raw(), + -1, + ptr::null(), + THIS_MODULE.as_ptr(), + 0, + &mut this.card, + ) + }; + if ret < 0 { + dev_err!(data.dev, "unable to allocate the sound card\n"); + return Err(Error::from_errno(ret)); + } + // Named like the other AOP audio cards: Apple. + let chassis = fwnode + .property_read::(c_str!("apple,chassis-name")) + .required_by(&data.dev)?; + let machine_kind = fwnode + .property_read::(c_str!("apple,machine-kind")) + .required_by(&data.dev)?; + let id_str = CString::try_from_fmt(fmt!("Apple{}AOP", *chassis))?; + let shortname = CString::try_from_fmt(fmt!("{} {} AOP", *machine_kind, *chassis))?; + let longname = CString::try_from_fmt(fmt!("{} {} AOP Audio", *machine_kind, *chassis))?; + // SAFETY: `card` was created by `snd_card_new` above; the fixed-size name fields are filled + // with NUL-terminated strings that fit. + unsafe { + copy_str( + &mut (*this.card).driver, + c"t8140-aop-audio".to_bytes_with_nul(), + ); + copy_str(&mut (*this.card).id, id_str.to_bytes_with_nul()); + copy_str(&mut (*this.card).shortname, shortname.to_bytes_with_nul()); + copy_str(&mut (*this.card).longname, longname.to_bytes_with_nul()); + } + + this.new_pcm(0, c"lpai", c"Low-Power Microphone", &Self::LPAI_OPS)?; + this.new_pcm(1, c"hpai", c"High-Quality Microphone", &Self::HPAI_OPS)?; + + // SAFETY: `card` was created by `snd_card_new` above and is not registered yet. + let ret = unsafe { bindings::snd_card_register(this.card) }; + if ret < 0 { + dev_err!(data.dev, "unable to register the sound card\n"); + return Err(Error::from_errno(ret)); + } + + // The ASoC side: the jack (cout) and speaker back-ends, the + // front-end PCMs and the speaker volume control. + // SAFETY: the descriptor, the DAI table and the controls are heap + // allocations owned by `this.asoc`, which outlives the component. + let ret = unsafe { + bindings::snd_soc_register_component( + data.dev.as_raw(), + &this.asoc.component, + this.asoc.dais.get().cast(), + DAI_COUNT as i32, + ) + }; + if ret < 0 { + dev_err!(data.dev, "unable to register the ASoC component: {}\n", ret); + return Err(Error::from_errno(ret)); + } + this.asoc_registered = true; + Ok(this) + } + + /// One capture PCM of the card, with the driver data as its private + /// data. + fn new_pcm( + &mut self, + device: i32, + id: &'static core::ffi::CStr, + name: &'static core::ffi::CStr, + ops: &'static bindings::snd_pcm_ops, + ) -> Result<()> { + let mut pcm = ptr::null_mut(); + // SAFETY: `card` was created by `snd_card_new`; `pcm` receives the new PCM. + let ret = + unsafe { bindings::snd_pcm_new(self.card, id.as_char_ptr(), device, 0, 1, &mut pcm) }; + if ret < 0 { + dev_err!(self.data.dev, "unable to allocate PCM {}\n", device); + return Err(Error::from_errno(ret)); + } + // SAFETY: `pcm` was created by `snd_pcm_new` above; the ops are static and the private + // data is an `Arc` released by `pcm_free_private`; `name` fits the fixed-size field. + unsafe { + bindings::snd_pcm_set_ops(pcm, bindings::SNDRV_PCM_STREAM_CAPTURE as i32, ops); + (*pcm).private_data = self.data.clone().into_foreign().cast(); + (*pcm).private_free = Some(pcm_free_private); + (*pcm).info_flags = 0; + copy_str(&mut (*pcm).name, name.to_bytes_with_nul()); + } + if device == 0 { + // The low-power microphone's buffer is filled by the driver from + // the source ring; the array's is set up on its DMA device at + // open. + // SAFETY: `pcm` is live; a vmalloc buffer needs no device. + unsafe { + bindings::snd_pcm_set_managed_buffer_all( + pcm, + bindings::SNDRV_DMA_TYPE_VMALLOC as i32, + ptr::null_mut(), + LPAI_PERIOD_BYTES * 4, + LPAI_PERIOD_BYTES * LPAI_PERIODS_MAX as usize, + ); + } + } + Ok(()) + } + + /// Undo probe: stop the reports, unregister the ASoC component and free + /// the card (which closes every stream, releasing its channel and power + /// references), then return lpai to idle. Runs from unbind, and again + /// (as a no-op) from Drop; also on a failed probe, with whatever had + /// been set up. + fn teardown(&self) { + if self.closed.xchg(true, Relaxed) { + return; + } + self.data.running.store(false, Relaxed); + // Stop reports before ALSA destroys any substream or module callback. + self.data + .adata + .remove_report_listener(&self.data.service, EPIC_SUBTYPE_PRODUCER_REPORT); + if self.asoc_registered { + // SAFETY: the context remains owned until after unregister + // completes PCM, DAI and control callbacks. + unsafe { + bindings::snd_soc_unregister_component_by_driver( + self.data.dev.as_raw(), + &self.asoc.component, + ) + }; + } + self.data.flush_spkr_start(); + self.data.flush_hpai_start(); + if !self.card.is_null() { + // SAFETY: the card is freed once, while the binding remains live. + unsafe { bindings::snd_card_free(self.card) }; + } + if self.data.powered.load(Relaxed) && self.data.set_power(POWER_STATE_IDLE).is_ok() { + self.data.powered.store(false, Relaxed); + } + } +} + +impl Drop for SndSocT8140AopDriver { + fn drop(&mut self) { + self.teardown(); + } +} + +/// # Safety +/// +/// Called by ALSA with a live substream of this card's PCM, whose private +/// data is the driver's `Arc`. +unsafe extern "C" fn hpai_pcm_hw_params( + substream: *mut bindings::snd_pcm_substream, + params: *mut bindings::snd_pcm_hw_params, +) -> i32 { + // SAFETY: the substream is live and its dmaengine runtime was set up by + // `snd_dmaengine_pcm_open` in `hpai_pcm_open`. + let chan = unsafe { bindings::snd_dmaengine_pcm_get_chan(substream) }; + let mut cfg = bindings::dma_slave_config::default(); + // SAFETY: the substream and the hardware parameters are live for the op. + let rc = unsafe { bindings::snd_hwparams_to_dma_slave_config(substream, params, &mut cfg) }; + if rc < 0 { + return rc; + } + // One stereo frame per burst. + cfg.src_port_window_size = HPAI_CHANNELS; + // SAFETY: `chan` is the live channel behind this substream's dmaengine runtime. + unsafe { + match (*(*chan).device).device_config { + Some(f) => f(chan, &mut cfg), + None => ENOSYS.to_errno(), + } + } +} + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + ::IdInfo, + [(of::DeviceId::new(c_str!("apple,t8140-aop-audio")), ())] +); + +impl platform::Driver for SndSocT8140AopDriver { + type IdInfo = (); + type Data<'bound> = SndSocT8140AopDriver; + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe<'bound>( + pdev: &'bound platform::Device>, + _info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound { + let dev = ARef::::from(pdev.as_ref()); + // SAFETY: this is a service device the AOP core registered, being + // probed here; the lookup defers the probe until the core has + // published its driver data. + let adata = unsafe { ::from_child(pdev.as_ref()) }?; + // SAFETY: the AOP core sets this child's platform data to the EPIC + // service it created the child for; it lives as long as the child. + let service = unsafe { (*dev.as_raw()).platform_data as *const EPICService }; + if service.is_null() { + return Err(ENODEV); + } + // SAFETY: the pointer was checked above and stays valid with the child. + let svc = unsafe { *service }; + + // The source ring: 512 KiB through our own IOMMU stream (dart-aop 9), + // owned by the AOP driver because the firmware binds it once per boot. + // SAFETY: no DMA is in flight on this device yet. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::new::<42>())? }; + let ring = adata.source_ring(pdev.as_ref(), RING_BYTES)?; + + // The serializer leaves. A missing one only disables the paths that + // need it; those fail to open with ENODEV. + let mut pds: [Option; 5] = [None, None, None, None, None]; + for (slot, (name, dt_name)) in pds.iter_mut().zip([ + ("mca1", c"mca1"), + ("mca0", c"mca0"), + ("tx0", c"tx0"), + ("leap mca", c"mca"), + ("rx0", c"rx0"), + ]) { + match PowerDomain::attach(&dev, name, dt_name) { + Ok(Some(pd)) => *slot = Some(pd), + Ok(None) => dev_warn!(dev, "no {} power domain\n", name), + Err(e) => dev_warn!(dev, "{} power domain attach failed: {:?}\n", name, e), + } + } + let [pd_mca1, pd_mca0, pd_tx0, pd_leap_mca, pd_rx0] = pds; + + let fwnode = pdev.as_ref().fwnode().ok_or(ENOENT)?; + let data = Arc::pin_init( + pin_init!(SndSocT8140AopData { + dev, + adata, + service: svc, + ring, + ring_bytes: Atomic::new(0), + sequence: Atomic::new(0), + powered: AtomicFlag::new(false), + running: AtomicFlag::new(false), + hw_ptr_frames: Atomic::new(0), + report_warnings: [const { AtomicFlag::new(false) }; 5], + cout: Service::new(AUDIO_DEV_COUT, "cout"), + cin: Service::new(AUDIO_DEV_CIN, "cin"), + spkr: Service::new(AUDIO_DEV_SPKR, "spkr"), + tap: Service::new(AUDIO_DEV_TAP, "tap"), + hpai: Service::new(AUDIO_DEV_HPAI, "hpai"), + pd_mca1, + pd_mca0, + pd_tx0, + pd_leap_mca, + pd_rx0, + spkr_want_run: AtomicFlag::new(false), + hpai_want_run: AtomicFlag::new(false), + spkr_volume: Atomic::new(SPKR_VOLUME_MAX), + spkr_start_work <- new_work!("SndSocT8140AopData::spkr_start_work"), + hpai_start_work <- new_work!("SndSocT8140AopData::hpai_start_work"), + stream <- new_spinlock!(StreamState { + substream: ptr::null_mut(), + copied: None, + hw_ptr_bytes: 0, + period_acc: 0, + buffer_bytes: 0, + }), + }), + GFP_KERNEL, + )?; + Self::new(data, fwnode) + } + + fn unbind(_dev: &platform::Device>, this: Pin<&Self>) { + this.teardown(); + } +} + +module_platform_driver! { + type: SndSocT8140AopDriver, + name: "snd_soc_t8140_aop_audio", + description: "Apple T8140 AOP audio: microphones, headphone jack and speakers", + license: "Dual MIT/GPL", +} From 3db140ed0afbaace61226a06968bccdaf9e9f1cc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:40:43 +1000 Subject: [PATCH 0463/2680] ASoC: macaudio: Support a software-gain speaker amplifier Every speaker amplifier macaudio knows has a volume control of its own, which the speaker lock limits. The MAX98360A on the J700 has none: it is a fixed-gain part fed by the AOP's LEAP, and the CPU DAI applies the volume as a software gain when it converts the samples in its copy path. Add AMP_MAX98360A for that arrangement. The lock machinery works as before, through the CPU DAI's "Speaker Playback Volume" (0.5 dB steps, 127 = the amplifier's full scale, so the -20 dB safe level is 87). Two things differ. If the limit cannot be applied at all, the front-end refuses to play: with a software gain the limit is the whole protection. And a lower gain cannot change samples that were converted before it was set, so a lock event bumps an epoch and stops the software-gain front-end stream with an XRUN; the stream is refused until it has been prepared again under the new epoch, which lets the CPU DAI clear its buffer. The refusal is made in the front-end's trigger, which runs before the CPU DAI and the back-ends are triggered (the front-end links now spell out the default SND_SOC_DPCM_TRIGGER_PRE order that guarantees it), so a refused start leaves the DMA untouched. The platform config names the CPU DAI that applies the gain, and the speaker codec's output widget, since the MAX98360A codec driver calls it "Speaker" rather than "OUT". An amplifier without a TDM interface passes the slot assignment like the headphones do, and a speaker CPU DAI without a capture side gets no sense route. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/macaudio.c | 175 +++++++++++++++++++++++++++++++++++-- 1 file changed, 169 insertions(+), 6 deletions(-) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index d29d6bef467c8..c29609dffff18 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -23,6 +23,7 @@ /* #define DEBUG */ #include +#include #include #include #include @@ -64,6 +65,11 @@ enum macaudio_amp_type { AMP_TAS5770, AMP_SN012776, AMP_SSM3515, + /* + * A fixed-gain amplifier with no controls of its own; the volume is + * a software gain the CPU DAI applies before the samples reach it. + */ + AMP_MAX98360A, }; enum macaudio_spkr_config { @@ -81,12 +87,23 @@ struct macaudio_platform_cfg { bool stereo; int amp_gain; int safe_vol; + /* + * The speaker codec's output widget, "OUT" (the TAS and SSM + * amplifiers) unless set. + */ + const char *spk_out_widget; + /* + * Software-gain amplifier: the CPU DAI whose front-end converts the + * speaker samples through the volume control at copy time. + */ + const char *sw_gain_dai; }; static const char *volume_control_names[] = { [AMP_TAS5770] = "* Speaker Playback Volume", [AMP_SN012776] = "* Speaker Volume", [AMP_SSM3515] = "* DAC Playback Volume", + [AMP_MAX98360A] = "* Speaker Playback Volume", }; #define SN012776_0DB 201 @@ -98,6 +115,10 @@ static const char *volume_control_names[] = { #define SSM3515_0DB (255 - 64) /* +24dB max, steps of 3/8 dB */ #define SSM3515_DB(x) (SSM3515_0DB + (8 * (x) / 3)) +/* Software gain of the CPU DAI: 0.5 dB steps, 127 = the amplifier's full scale */ +#define MAX98360A_0DB 127 +#define MAX98360A_DB(x) (MAX98360A_0DB + 2 * (x)) + struct ma_codec_idle { int idle_mode; u32 idle_mask; @@ -118,6 +139,8 @@ struct macaudio_snd_data { /* frontend props */ unsigned int bclk_ratio; bool is_sense; + /* the front-end that applies the software speaker gain */ + bool sw_gain; /* backend props */ bool is_speakers; @@ -141,6 +164,16 @@ struct macaudio_snd_data { struct delayed_work lock_timeout_work; struct work_struct lock_update_work; + /* + * Software-gain amplifier: a lower gain cannot change samples that + * were converted before it was set, so every lock event bumps the + * epoch and stops the front-end stream; it is refused until it has + * been prepared (and its buffer cleared) under the new epoch. + */ + atomic_t sw_gain_lock_epoch; + atomic_t sw_gain_prepared_epoch; + atomic_t sw_gain_limit_failed; + struct work_struct sw_gain_stop_work; }; static int please_blow_up_my_speakers; @@ -162,6 +195,11 @@ SND_SOC_DAILINK_DEFS(sense, DAILINK_COMP_ARRAY(COMP_DUMMY()), // CODEC DAILINK_COMP_ARRAY(COMP_EMPTY())); +/* + * The front-ends are triggered before their back-ends (the default order, + * made explicit here): macaudio_fe_trigger() runs first of all on START, + * so a start it refuses leaves the DMA and the back-ends untouched. + */ static struct snd_soc_dai_link macaudio_fe_links[] = { { .name = "Primary", @@ -171,6 +209,7 @@ static struct snd_soc_dai_link macaudio_fe_links[] = { .dpcm_merged_chan = 1, .dpcm_merged_format = 1, .dai_fmt = MACAUDIO_DAI_FMT, + .trigger = { SND_SOC_DPCM_TRIGGER_PRE, SND_SOC_DPCM_TRIGGER_PRE }, SND_SOC_DAILINK_REG(primary), }, { @@ -182,6 +221,7 @@ static struct snd_soc_dai_link macaudio_fe_links[] = { .dpcm_merged_format = 1, .dai_fmt = MACAUDIO_DAI_FMT, .playback_only = 1, + .trigger = { SND_SOC_DPCM_TRIGGER_PRE, SND_SOC_DPCM_TRIGGER_PRE }, SND_SOC_DAILINK_REG(secondary), }, { @@ -260,12 +300,24 @@ static void macaudio_vlimit_unlock(struct macaudio_snd_data *ma, bool unlock) else max = SSM3515_DB(ma->cfg->safe_vol); break; + case AMP_MAX98360A: + if (unlock) + max = MAX98360A_0DB; + else + max = MAX98360A_DB(ma->cfg->safe_vol); + break; } ret = snd_soc_limit_volume(&ma->card, name, max); if (ret < 0) dev_err(ma->card.dev, "Failed to %slock volume %s: %d\n", unlock ? "un" : "", name, ret); + /* + * With a software gain the limit is the whole protection: refuse to + * play rather than run the amplifier at full scale unlimited. + */ + if (ma->cfg->amp == AMP_MAX98360A) + atomic_set(&ma->sw_gain_limit_failed, ret < 0); } static void macaudio_vlimit_update(struct macaudio_snd_data *ma) @@ -314,6 +366,11 @@ static void macaudio_vlimit_update(struct macaudio_snd_data *ma) } macaudio_vlimit_unlock(ma, unlock); + if (!unlock && ma->cfg->amp == AMP_MAX98360A) { + /* Samples already converted at the old gain must not play */ + atomic_inc(&ma->sw_gain_lock_epoch); + schedule_work(&ma->sw_gain_stop_work); + } ma->speaker_volume_unlocked = unlock; snd_ctl_notify(ma->card.snd_card, SNDRV_CTL_EVENT_MASK_VALUE, &ma->speaker_lock_kctl->id); @@ -415,6 +472,49 @@ static void macaudio_vlimit_update_work(struct work_struct *wrk) macaudio_vlimit_disable_timeout(ma); } +static bool macaudio_sw_gain_pcm(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + + return props->sw_gain && substream->stream == SNDRV_PCM_STREAM_PLAYBACK; +} + +static void macaudio_sw_gain_stop_work(struct work_struct *work) +{ + struct macaudio_snd_data *ma = container_of(work, struct macaudio_snd_data, + sw_gain_stop_work); + struct snd_soc_pcm_runtime *rtd; + struct snd_pcm_substream *substream; + unsigned long flags; + + /* No volume or control lock may be held while taking the stream lock */ + for_each_card_rtds(&ma->card, rtd) { + if (!rtd->pcm) + continue; + substream = rtd->pcm->streams[SNDRV_PCM_STREAM_PLAYBACK].substream; + if (!substream || !macaudio_sw_gain_pcm(substream)) + continue; + snd_pcm_stream_lock_irqsave(substream, flags); + if (substream->runtime && + atomic_read(&ma->sw_gain_prepared_epoch) != + atomic_read(&ma->sw_gain_lock_epoch)) { + switch (substream->runtime->state) { + case SNDRV_PCM_STATE_PREPARED: + case SNDRV_PCM_STATE_RUNNING: + case SNDRV_PCM_STATE_DRAINING: + case SNDRV_PCM_STATE_PAUSED: + snd_pcm_stop(substream, SNDRV_PCM_STATE_XRUN); + break; + default: + break; + } + } + snd_pcm_stream_unlock_irqrestore(substream, flags); + } +} + static int macaudio_copy_link(struct device *dev, struct snd_soc_dai_link *target, struct snd_soc_dai_link *source) { @@ -594,6 +694,9 @@ static int macaudio_parse_of(struct macaudio_snd_data *ma) goto err_free; memcpy(link_props, &macaudio_fe_link_props[i], sizeof(struct macaudio_link_props)); + if (ma->cfg->sw_gain_dai) + link_props->sw_gain = !strcmp(link->cpus[0].dai_name, + ma->cfg->sw_gain_dai); link++; link_props++; } @@ -977,10 +1080,57 @@ static int macaudio_be_trigger(struct snd_pcm_substream *substream, int cmd) return 0; } +static int macaudio_fe_prepare(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + + if (!macaudio_sw_gain_pcm(substream)) + return 0; + + if (atomic_read(&ma->sw_gain_limit_failed)) + return -EIO; + /* The CPU DAI's prepare clears the old samples out of the buffer next */ + atomic_set(&ma->sw_gain_prepared_epoch, + atomic_read(&ma->sw_gain_lock_epoch)); + return 0; +} + +/* + * Runs before the CPU DAI and the back-ends are triggered (see the + * front-end links): a lock event since prepare means the buffer holds + * samples converted at the old gain, so the start is refused and the + * stream has to be prepared again. + */ +static int macaudio_fe_trigger(struct snd_pcm_substream *substream, int cmd) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + + if (!macaudio_sw_gain_pcm(substream)) + return 0; + + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: + if (atomic_read(&ma->sw_gain_limit_failed) || + atomic_read(&ma->sw_gain_prepared_epoch) != + atomic_read(&ma->sw_gain_lock_epoch)) + return -EPIPE; + break; + default: + break; + } + return 0; +} + static const struct snd_soc_ops macaudio_fe_ops = { .startup = macaudio_fe_startup, .shutdown = macaudio_dpcm_shutdown, .hw_params = macaudio_fe_hw_params, + .prepare = macaudio_fe_prepare, + .trigger = macaudio_fe_trigger, }; static const struct snd_soc_ops macaudio_be_ops = { @@ -1011,9 +1161,11 @@ static int macaudio_be_assign_tdm(struct snd_soc_pcm_runtime *rtd) /* * Headphones get a pass on -ENOTSUPP (see the comment - * around bclk_ratio value for primary FE). + * around bclk_ratio value for primary FE), as does an + * amplifier without a TDM interface behind a software gain. */ - if (ret == -ENOTSUPP && props->is_headphones) + if (ret == -ENOTSUPP && + (props->is_headphones || ma->cfg->amp == AMP_MAX98360A)) return 0; return ret; @@ -1152,8 +1304,8 @@ static int macaudio_add_backend_dai_route(struct snd_soc_card *card, struct snd_ r->sink = "Headset Capture"; } - /* If speakers, add sense capture path */ - if (is_speakers) { + /* If speakers, add sense capture path (a playback-only CPU DAI has none) */ + if (is_speakers && dai->stream[SNDRV_PCM_STREAM_CAPTURE].widget) { r = &routes[nroutes++]; r->source = dai->stream[SNDRV_PCM_STREAM_CAPTURE].widget->name; r->sink = "Speaker Sense Capture"; @@ -1169,6 +1321,7 @@ static int macaudio_add_backend_dai_route(struct snd_soc_card *card, struct snd_ static int macaudio_add_pin_routes(struct snd_soc_card *card, struct snd_soc_component *component, bool is_speakers) { + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); struct snd_soc_dapm_route routes[2]; struct snd_soc_dapm_route *r; int nroutes = 0; @@ -1179,10 +1332,12 @@ static int macaudio_add_pin_routes(struct snd_soc_card *card, struct snd_soc_com /* Connect the far ends of CODECs to pins */ if (is_speakers) { + const char *out = ma->cfg->spk_out_widget ?: "OUT"; + r = &routes[nroutes++]; - r->source = "OUT"; + r->source = out; if (component->name_prefix) { - snprintf(buf, sizeof(buf) - 1, "%s OUT", component->name_prefix); + snprintf(buf, sizeof(buf) - 1, "%s %s", component->name_prefix, out); r->source = buf; } r->sink = "Speaker"; @@ -1310,6 +1465,9 @@ static int macaudio_set_speaker(struct snd_soc_card *card, const char *prefix, b /* TODO: HPF, needs new call to set */ break; + case AMP_MAX98360A: + /* No controls of its own; the gain sits on the CPU DAI */ + break; default: return -EINVAL; } @@ -1764,6 +1922,10 @@ static int macaudio_snd_platform_probe(struct platform_device *pdev) INIT_WORK(&data->lock_update_work, macaudio_vlimit_update_work); INIT_DELAYED_WORK(&data->lock_timeout_work, macaudio_vlimit_timeout_work); + INIT_WORK(&data->sw_gain_stop_work, macaudio_sw_gain_stop_work); + atomic_set(&data->sw_gain_lock_epoch, 0); + atomic_set(&data->sw_gain_prepared_epoch, -1); + atomic_set(&data->sw_gain_limit_failed, 0); return snd_soc_register_card(card); } @@ -1779,6 +1941,7 @@ static void macaudio_snd_platform_remove(struct platform_device *pdev) snd_soc_unregister_card(&ma->card); cancel_work_sync(&ma->lock_update_work); cancel_delayed_work_sync(&ma->lock_timeout_work); + cancel_work_sync(&ma->sw_gain_stop_work); } static struct platform_driver macaudio_snd_driver = { From 58c32b5b70d0ec9b61f47c1e7aa43e92855a6c4e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:41:42 +1000 Subject: [PATCH 0464/2680] ASoC: macaudio: Add J700 routing Route the MacBook Neo's (T8140, J700) audio through macaudio: the CS42L83 headset jack on the AOP's "cout" back-end and one MAX98360A per channel on its "spkr" back-end, with the T8140 AOP audio driver as the CPU component and its three "j700-pcm-N" front-ends in place of the MCA's. The AOP firmware fixes the jack link: 125 SCLKs per 48 kHz frame (6 MHz), a pulse frame with two 32-bit slots, data launched on the rising SCLK edge. That is DSP_A with an inverted bit clock, so the J700 config carries its own DAI format for the links that reach a codec and a primary bclk ratio of 125, which the codec's PLL table accepts. The speaker back-end has no sense lines and no amplifier controls; the volume is the CPU DAI's software gain, held 20 dB down by the lock until a protection daemon takes it. Compile-tested only; not run on hardware. The jack link was brought up with the codec driven by a non-standard register setting (a half SCLK frame delay and the SDOUT edge flipped under a normal bit clock); the inverted-bit-clock description is the standard form of that timing and needs confirming on the machine. Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/macaudio.c | 48 +++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index c29609dffff18..295be26aba9de 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -57,6 +57,8 @@ #define MACAUDIO_MAX_BCLK_FREQ 24576000 #define SPEAKER_MAGIC_VALUE (s32)0xdec1be15 +/* Primary, Secondary and Speaker Sense */ +#define MACAUDIO_NUM_FE_LINKS 3 /* milliseconds */ #define SPEAKER_LOCK_TIMEOUT 250 @@ -97,6 +99,15 @@ struct macaudio_platform_cfg { * speaker samples through the volume control at copy time. */ const char *sw_gain_dai; + /* + * A CPU component other than the MCA: its front-end DAI names in + * place of "mca-pcm-N", the DAI format its serial links run (the + * default MACAUDIO_DAI_FMT when 0) and the primary front-end's + * bclk ratio when the link's frame length is not ours to choose. + */ + const char *fe_dai_names[MACAUDIO_NUM_FE_LINKS]; + unsigned int dai_fmt; + unsigned int primary_bclk_ratio; }; static const char *volume_control_names[] = { @@ -237,6 +248,8 @@ static struct snd_soc_dai_link macaudio_fe_links[] = { }, }; +static_assert(ARRAY_SIZE(macaudio_fe_links) == MACAUDIO_NUM_FE_LINKS); + static struct macaudio_link_props macaudio_fe_link_props[] = { { /* @@ -567,7 +580,7 @@ static int macaudio_parse_of_be_dai_link(struct macaudio_snd_data *ma, link->no_pcm = 1; - link->dai_fmt = MACAUDIO_DAI_FMT; + link->dai_fmt = ma->cfg->dai_fmt ?: MACAUDIO_DAI_FMT; link->num_codecs = ncodecs_per_be; link->codecs = devm_kcalloc(dev, ncodecs_per_be, @@ -694,6 +707,12 @@ static int macaudio_parse_of(struct macaudio_snd_data *ma) goto err_free; memcpy(link_props, &macaudio_fe_link_props[i], sizeof(struct macaudio_link_props)); + if (ma->cfg->fe_dai_names[i]) + link->cpus[0].dai_name = ma->cfg->fe_dai_names[i]; + if (ma->cfg->dai_fmt && !link_props->is_sense) + link->dai_fmt = ma->cfg->dai_fmt; + if (i == 0 && ma->cfg->primary_bclk_ratio) + link_props->bclk_ratio = ma->cfg->primary_bclk_ratio; if (ma->cfg->sw_gain_dai) link_props->sw_gain = !strcmp(link->cpus[0].dai_name, ma->cfg->sw_gain_dai); @@ -1761,6 +1780,31 @@ struct macaudio_platform_cfg macaudio_j313_cfg = { true, AMP_TAS5770, SPKR_1W, true, 10, -20, }; +/* + * J700 (T8140, MacBook Neo): the AOP firmware owns the audio fabric, so the + * CPU component is the T8140 AOP audio driver rather than the MCA. The jack + * is a CS42L83 on the AOP's "cout" back-end, which the firmware runs as a + * 125-SCLK pulse frame at 48 kHz (6 MHz) with two 32-bit slots, data + * launched on the rising SCLK edge: DSP_A with an inverted bit clock. The + * speakers are one MAX98360A per channel on the "spkr" back-end, with no + * sense lines and no amplifier controls; the volume is the CPU DAI's + * software gain, which the lock holds 20 dB down until a protection daemon + * takes it. The daemon models the speaker from the SpeakerTap, a digital + * tap of the words the amplifier receives. + */ +struct macaudio_platform_cfg macaudio_j700_cfg = { + .enable_speakers = true, + .amp = AMP_MAX98360A, + .speakers = SPKR_1W, + .stereo = true, + .safe_vol = -20, + .spk_out_widget = "Speaker", + .sw_gain_dai = "j700-pcm-1", + .fe_dai_names = { "j700-pcm-0", "j700-pcm-1", "j700-pcm-2" }, + .dai_fmt = SND_SOC_DAIFMT_DSP_A | SND_SOC_DAIFMT_CBC_CFC | SND_SOC_DAIFMT_IB_NF, + .primary_bclk_ratio = 125, +}; + struct macaudio_platform_cfg macaudio_j314_cfg = { true, AMP_SN012776, SPKR_2W1T, true, 15, -20, }; @@ -1848,6 +1892,8 @@ static const struct of_device_id macaudio_snd_device_id[] = { /* j575 AID33 sn012776 15 1x 1W Compat: apple,j375-macaudio */ /* j613 AID20 sn012776 15 2x 1W+1T Compat: apple,j413-macaudio */ /* j615 AID21 sn012776 15 2x 2W+1T Compat: apple,j415-macaudio */ + /* j700 AID44 max98360a - 2x 1W (AOP-fed, software gain) */ + { .compatible = "apple,j700-macaudio", .data = &macaudio_j700_cfg }, /* Fallback, jack only */ { .compatible = "apple,macaudio"}, { } From c821b4f9f1501dca4c4243c4524e04c524fef8c3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:23:01 +0000 Subject: [PATCH 0465/2680] ASoC: apple: t8140-aop-audio: give the hpai PCM a prepare op ALSA calls snd_pcm_ops.prepare unconditionally from snd_pcm_do_prepare(). The high-quality microphone array PCM left it NULL, so the first SNDRV_PCM_IOCTL_PREPARE on hw:AppleJ700AOP,1 branched to address 0: Unable to handle kernel NULL pointer dereference at virtual address 0 pc : 0x0 lr : snd_pcm_do_prepare+0x6c/0x88 [snd_pcm] ... snd_pcm_prepare+0xc0/0xd0 [snd_pcm] Nothing needs arming before the trigger: the dmaengine runtime is set up at open, and trigger starts the DMA and queues the firmware run. Provide an empty prepare so the stream can be prepared at all. Found on the first hardware boot of aurora-wip 92257036 + PR 33 + PR 34 on the J700 (arecord -D hw:0,1 Oops, 2026-09-26). Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index e6be972f7ab77..c5c93e67f8438 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -1246,6 +1246,18 @@ unsafe extern "C" fn hpai_pcm_close(substream: *mut bindings::snd_pcm_substream) ret } +/// Nothing to arm before the trigger: the dmaengine runtime was set up at +/// open, and the DMA start followed by the queued firmware run belong to +/// trigger. ALSA calls the prepare op unconditionally, so it must exist: +/// without it the first hpai prepare jumped to address 0. +/// +/// # Safety +/// +/// Called by ALSA with a live substream of this card's PCM. +unsafe extern "C" fn hpai_pcm_prepare(_substream: *mut bindings::snd_pcm_substream) -> i32 { + 0 +} + /// Trigger, with the ALSA stream lock held: the DMA first, then the /// sleeping firmware start queued behind it. /// @@ -2348,7 +2360,7 @@ impl SndSocT8140AopDriver { const HPAI_OPS: bindings::snd_pcm_ops = bindings::snd_pcm_ops { open: Some(hpai_pcm_open), close: Some(hpai_pcm_close), - prepare: None, + prepare: Some(hpai_pcm_prepare), trigger: Some(hpai_pcm_trigger), pointer: Some(bindings::snd_dmaengine_pcm_pointer), ioctl: None, From 2c0b878551272262afb66736ecb24b1eb22de3b8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:10:13 +1100 Subject: [PATCH 0466/2680] soc: apple: rtkit: Handle oslog notifications quietly Besides the buffer request, RTKit firmware sends oslog messages of types 2 to 5 during normal operation. Types 3 to 5 arrive in a burst when SMC, DCP and AOP firmware boot, and were logged at info level, four or five lines per co-processor on every boot. Type 2 arrives once per boot from the AOP and the MTP, and was logged as an unknown message with a warning. None of them expects a reply: the co-processors keep running, and keyboard, trackpad and AOP sensors keep working after they arrive. The payload is not decoded, so the lines say nothing actionable. Log all four types at debug level and keep the warning for types that have not been seen. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 3402df716c97f..4390ba10767b9 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -608,16 +608,13 @@ static void apple_rtkit_oslog_rx(struct apple_rtkit *rtk, u64 msg) apple_rtkit_common_rx_get_buffer(rtk, &rtk->oslog_buffer, APPLE_RTKIT_EP_OSLOG, msg); break; - case 3: - case 4: - case 5: + case 2 ... 5: /* - * OpenBSD's public RTKit consumer recognizes these opaque events - * without an ACK. Keep their payload visible without interpreting - * it as a buffer address or a decoded firmware error. + * Undocumented notifications sent during normal operation. They + * expect no reply, and the co-processor keeps running without + * one. */ - dev_info(rtk->dev, "RTKit: OSLog event %u: %016llx (undecoded)\n", - type, msg); + dev_dbg(rtk->dev, "RTKit: oslog notification: %016llx\n", msg); break; default: dev_warn(rtk->dev, "RTKit: Unknown oslog message: %llx\n", From f34f34b069b0d603030564a597eeb61d04a7d886 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 04:40:03 +0000 Subject: [PATCH 0467/2680] ASoC: apple: t8140-aop-audio: bring the firmware service to pw0 before the leaves The AOP owns the audio fabric and power-gates it (audio_p and the LEAP domains under it) whenever none of its services is held in pw0 or pwrd, for instance after a low-power microphone capture. Back-end startup powered the AP-side serializer leaves first and only then asked for pw0, so on a gated fabric the first leaf request under audio_p found its parent off and never completed: PS audio_leap_mca: Failed to reach power state 0xf (requested 0xf, now: 0xa4f) The failed resume then left the domain in the runtime-PM error state, which took the speaker down with the microphone array for the rest of the boot. Ask the firmware for pw0 first, then power the leaves. Observed on the J700 with aurora-wip 92257036 + PR 33 + PR 34: an LPAI capture leaves audio_p at 0x200 and the next hpai open fails every time; the previous hybrid kernel only avoided it because speakerguardd held the speaker service in pw0 throughout. Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index c5c93e67f8438..c9530f1b05c1c 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -713,14 +713,18 @@ impl SndSocT8140AopData { } } - /// Back-end startup: attach the service, power its leaves, `pw0 `. + /// Back-end startup: attach the service, `pw0 `, then power its leaves. + /// + /// The firmware owns the audio fabric and power-gates it (`audio_p` and + /// the LEAP domains under it) whenever none of its services is held in + /// `pw0 ` or `pwrd`, for instance after a low-power microphone capture. + /// Requesting `pw0 ` first brings the fabric back; only then can the + /// serializer leaves under it be powered from the AP side, otherwise the + /// PMGR reports the parent off and the leaf request never completes. fn service_startup(&self, svc: &Service, pds: &[&Option]) -> Result<()> { self.attach_device(svc.dev_id, svc.name)?; + self.service_set_power(svc, SPKR_POWER_STATE_PW0)?; self.power_up(svc.name, pds)?; - if let Err(e) = self.service_set_power(svc, SPKR_POWER_STATE_PW0) { - self.power_down(pds); - return Err(e); - } svc.running.store(false, Relaxed); Ok(()) } From 041bb941bf1915cdc07496ce1eec8954ce595861 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 04:40:07 +0000 Subject: [PATCH 0468/2680] ASoC: apple: t8140-aop-audio: retry a leaf power-up the firmware delayed After the service's pw0 request the firmware brings the audio fabric back asynchronously, so the first serializer leaf request under audio_p can still miss the PMGR driver's 100 us poll window with the parent coming up: PS audio_leap_mca: Failed to reach power state 0xf (requested 0xf, now: 0xa4f) The failed resume leaves the virtual domain device in the runtime-PM error state, and every later request for it fails with EINVAL until reboot, taking the speaker down together with the microphone array. Clear the error state and retry the request a few times, 5 ms apart, before failing the stream. The previous hybrid kernel carried the same recovery, with a single 1 ms retry; the bound here covers the measured fabric power-up after pw0. Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 38 ++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index c9530f1b05c1c..88d9d8322e918 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -115,6 +115,10 @@ const AUDIO_RET_BUSY: u32 = 0xe00002d5; const SPKR_POWER_STATE_PW0: u32 = from_fourcc(b"pw0 "); const SPKR_POWER_STATE_PWRD: u32 = from_fourcc(b"pwrd"); +/// Leaf power-up retries while the firmware raises the fabric (see `PowerDomain::get`). +const PD_GET_ATTEMPTS: u32 = 60; +const PD_GET_RETRY_MS: i64 = 10; + /// The low-power microphone stream: 16 kHz stereo S32 in a 512 KiB ring /// reported every 3,200 frames. const LPAI_RATE: u32 = 16000; @@ -325,15 +329,39 @@ impl PowerDomain { } /// Takes a runtime-PM reference and waits for the domain to be on. + /// + /// The firmware brings the audio fabric back asynchronously after the + /// service's `pw0 ` request, so the first leaf request under it can miss + /// the PMGR driver's poll window with the parent still coming up. A + /// failed resume leaves the virtual domain device in the runtime-PM error + /// state, where every later request fails with EINVAL; clear that state + /// and retry a bounded number of times before giving up. fn get(&self) -> Result<()> { - // SAFETY: the domain device stays attached for the lifetime of `self`. - let rc = unsafe { bindings::pm_runtime_get_sync(self.dev.as_ptr()) }; - if rc < 0 { + let mut rc = 0; + for attempt in 0..PD_GET_ATTEMPTS { + // SAFETY: the domain device stays attached for the lifetime of `self`. + rc = unsafe { bindings::pm_runtime_get_sync(self.dev.as_ptr()) }; + if rc >= 0 { + return Ok(()); + } // SAFETY: as above; a failed resume still took the usage count. unsafe { bindings::pm_runtime_put_noidle(self.dev.as_ptr()) }; - return Err(Error::from_errno(rc)); + // SAFETY: as above; the device is suspended from runtime PM's point + // of view once the failed resume has been undone. Clearing the + // error state also lets a later open try again after a final + // failure here. + unsafe { + bindings::__pm_runtime_set_status( + self.dev.as_ptr(), + bindings::rpm_status_RPM_SUSPENDED as u32, + ) + }; + if Error::from_errno(rc) != ETIMEDOUT || attempt + 1 == PD_GET_ATTEMPTS { + break; + } + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(PD_GET_RETRY_MS)); } - Ok(()) + Err(Error::from_errno(rc)) } /// Drops a reference taken by [`Self::get`]. From 5d253032a39fdf3d178b6b0b30c7e8121d187993 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 04:40:12 +0000 Subject: [PATCH 0469/2680] ASoC: apple: t8140-aop-audio: raise a gated audio fabric before the leaves The firmware power-gates the audio fabric (audio_p and the LEAP domains under it) once no service is in pw0 or pwrd, for instance after a low-power microphone capture. A service's pw0 request does not raise it again: with the fabric gated, a microphone-array open leaves audio_p at 0x200 for the whole leaf retry window and the serializer leaf under it never completes. Only a running stream raises the fabric, after which a service held in pw0 keeps it up. When the leaf request times out, run the low-power microphone briefly around the retry: it is attached for the firmware's lifetime and its running state is what raises the fabric. The earlier hybrid kernel never saw this because its qualification-era power-domain driver kept the inherited leaves powered; with runtime power management of the leaves the first hpai or speaker open after an LPAI capture failed every time. Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 61 ++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index 88d9d8322e918..6f4075278d9ba 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -116,7 +116,9 @@ const SPKR_POWER_STATE_PW0: u32 = from_fourcc(b"pw0 "); const SPKR_POWER_STATE_PWRD: u32 = from_fourcc(b"pwrd"); /// Leaf power-up retries while the firmware raises the fabric (see `PowerDomain::get`). -const PD_GET_ATTEMPTS: u32 = 60; +/// Blind attempts before the fabric is raised, and attempts while it rises. +const PD_GET_ATTEMPTS: u32 = 3; +const PD_GET_RAISED_ATTEMPTS: u32 = 60; const PD_GET_RETRY_MS: i64 = 10; /// The low-power microphone stream: 16 kHz stereo S32 in a 512 KiB ring @@ -336,9 +338,9 @@ impl PowerDomain { /// failed resume leaves the virtual domain device in the runtime-PM error /// state, where every later request fails with EINVAL; clear that state /// and retry a bounded number of times before giving up. - fn get(&self) -> Result<()> { + fn get(&self, attempts: u32) -> Result<()> { let mut rc = 0; - for attempt in 0..PD_GET_ATTEMPTS { + for attempt in 0..attempts { // SAFETY: the domain device stays attached for the lifetime of `self`. rc = unsafe { bindings::pm_runtime_get_sync(self.dev.as_ptr()) }; if rc >= 0 { @@ -356,7 +358,7 @@ impl PowerDomain { bindings::rpm_status_RPM_SUSPENDED as u32, ) }; - if Error::from_errno(rc) != ETIMEDOUT || attempt + 1 == PD_GET_ATTEMPTS { + if Error::from_errno(rc) != ETIMEDOUT || attempt + 1 == attempts { break; } kernel::time::delay::fsleep(kernel::time::Delta::from_millis(PD_GET_RETRY_MS)); @@ -712,21 +714,23 @@ impl SndSocT8140AopData { /// Runtime-power serializer leaves, parents first. A leaf that fails to /// power up fails the stream; recovering a domain from that state is the /// power-domain driver's business, not a consumer's. - fn power_up(&self, name: &str, pds: &[&Option]) -> Result<()> { + fn power_up(&self, name: &str, pds: &[&Option], attempts: u32) -> Result<()> { for (i, slot) in pds.iter().enumerate() { let Some(pd) = slot else { dev_err!(self.dev, "{}: power domain not attached\n", name); self.power_down(&pds[..i]); return Err(ENODEV); }; - if let Err(e) = pd.get() { - dev_err!( - self.dev, - "{}: power domain {} did not power up: {:?}\n", - name, - pd.name, - e - ); + if let Err(e) = pd.get(attempts) { + if e != ETIMEDOUT || attempts > PD_GET_ATTEMPTS { + dev_err!( + self.dev, + "{}: power domain {} did not power up: {:?}\n", + name, + pd.name, + e + ); + } self.power_down(&pds[..i]); return Err(e); } @@ -752,11 +756,38 @@ impl SndSocT8140AopData { fn service_startup(&self, svc: &Service, pds: &[&Option]) -> Result<()> { self.attach_device(svc.dev_id, svc.name)?; self.service_set_power(svc, SPKR_POWER_STATE_PW0)?; - self.power_up(svc.name, pds)?; + self.power_up_raising(svc.name, pds)?; svc.running.store(false, Relaxed); Ok(()) } + /// [`Self::power_up`], raising the fabric when the leaves cannot come up. + fn power_up_raising(&self, name: &str, pds: &[&Option]) -> Result<()> { + match self.power_up(name, pds, PD_GET_ATTEMPTS) { + Err(e) if e == ETIMEDOUT => { + self.with_fabric_raised(|| self.power_up(name, pds, PD_GET_RAISED_ATTEMPTS)) + } + other => other, + } + } + + /// Raise the audio fabric the firmware gated. A service's `pw0 ` request + /// does not do that (measured: `audio_p` stays gated for the whole leaf + /// retry window), only a running stream does, after which a service in + /// `pw0 ` keeps it up. The low-power microphone is attached for the + /// firmware's lifetime, so run it briefly around `f` when it is idle. + fn with_fabric_raised(&self, f: impl FnOnce() -> Result) -> Result { + if self.powered.load(Relaxed) { + return f(); + } + self.set_power(POWER_STATE_RUN)?; + let ret = f(); + if let Err(e) = self.set_power(POWER_STATE_IDLE) { + dev_warn!(self.dev, "lpai: unable to idle after raising the fabric: {:?}\n", e); + } + ret + } + /// `pwrd`: the firmware starts the serializer clocks and the stream. fn service_run(&self, svc: &Service) -> Result<()> { if svc.running.load(Relaxed) { @@ -1620,7 +1651,7 @@ unsafe extern "C" fn asoc_pcm_open( }; let leap = fe == FE_SPKR; if leap { - if let Err(e) = data.power_up(data.spkr.name, &data.spkr_domains()) { + if let Err(e) = data.power_up_raising(data.spkr.name, &data.spkr_domains()) { return e.to_errno(); } } From 9563ce4affe3c5a80234864aceaab83ad00b1715 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 15:59:16 +1000 Subject: [PATCH 0470/2680] ASoC: apple: t8140-aop-audio: raise the fabric before every leaf request The first back-end or speaker open after a low-power microphone capture finds the AOP's audio fabric gated; a service left in pw0 after its stream does not keep it up either. The blind leaf pass then times out before the raised pass succeeds, and the raised pass itself retried every 10 ms while the fabric came up, so every open in that state left a run of PMGR error lines ("PS audio_leap_mca: Failed to reach power state 0xf"), six to nine per boot on the J700 with the automatic microphone checks. Always go through the raised path, and wait for the low-power microphone's first report after the run request before the first leaf request: the report proves the stream, and so the fabric, is up (45 to 200 ms measured at boot). With the fabric already up the cost is one brief low-power microphone run. The bounded leaf retry stays as the fallback. Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 40 +++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index 6f4075278d9ba..ceae8e27110c7 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -120,6 +120,13 @@ const SPKR_POWER_STATE_PWRD: u32 = from_fourcc(b"pwrd"); const PD_GET_ATTEMPTS: u32 = 3; const PD_GET_RAISED_ATTEMPTS: u32 = 60; const PD_GET_RETRY_MS: i64 = 10; +/// The fabric comes up some tens to hundreds of milliseconds after the +/// low-power microphone is asked to run (45–200 ms measured at boot); its +/// first report proves the stream, and so the fabric, is up. Poll for it +/// before the first leaf request so that request does not time out, and +/// log, in the PMGR driver. +const FABRIC_RAISE_POLL_MS: i64 = 5; +const FABRIC_RAISE_POLLS: u32 = 100; /// The low-power microphone stream: 16 kHz stereo S32 in a 512 KiB ring /// reported every 3,200 frames. @@ -435,6 +442,8 @@ struct SndSocT8140AopData { sequence: Atomic, /// lpai is in `runn`. powered: AtomicFlag, + /// A low-power microphone report arrived since the last run request. + reported: AtomicFlag, /// The low-power microphone stream is triggered (set and cleared under /// the ALSA stream lock, read by the report path). running: AtomicFlag, @@ -761,26 +770,37 @@ impl SndSocT8140AopData { Ok(()) } - /// [`Self::power_up`], raising the fabric when the leaves cannot come up. + /// [`Self::power_up`] with the fabric raised first. + /// + /// The fabric may be gated at any open (a low-power capture is enough, + /// and a service left in `pw0 ` after its stream does not keep it up), + /// and a blind leaf request against it fails noisily in the PMGR + /// driver, so always go through the raised path; with the fabric up it + /// costs one brief low-power microphone run. fn power_up_raising(&self, name: &str, pds: &[&Option]) -> Result<()> { - match self.power_up(name, pds, PD_GET_ATTEMPTS) { - Err(e) if e == ETIMEDOUT => { - self.with_fabric_raised(|| self.power_up(name, pds, PD_GET_RAISED_ATTEMPTS)) - } - other => other, - } + self.with_fabric_raised(|| self.power_up(name, pds, PD_GET_RAISED_ATTEMPTS)) } /// Raise the audio fabric the firmware gated. A service's `pw0 ` request /// does not do that (measured: `audio_p` stays gated for the whole leaf /// retry window), only a running stream does, after which a service in - /// `pw0 ` keeps it up. The low-power microphone is attached for the - /// firmware's lifetime, so run it briefly around `f` when it is idle. + /// `pw0 ` keeps it up only while its stream runs. The low-power + /// microphone is attached for the firmware's lifetime, so run it briefly + /// around `f` when it is idle, once its first report has arrived. fn with_fabric_raised(&self, f: impl FnOnce() -> Result) -> Result { if self.powered.load(Relaxed) { return f(); } + self.reported.store(false, Relaxed); self.set_power(POWER_STATE_RUN)?; + let mut polls = 0; + while !self.reported.load(Relaxed) && polls < FABRIC_RAISE_POLLS { + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(FABRIC_RAISE_POLL_MS)); + polls += 1; + } + if polls == FABRIC_RAISE_POLLS { + dev_warn!(self.dev, "lpai: no report while raising the fabric\n"); + } let ret = f(); if let Err(e) = self.set_power(POWER_STATE_IDLE) { dev_warn!(self.dev, "lpai: unable to idle after raising the fabric: {:?}\n", e); @@ -1028,6 +1048,7 @@ impl ReportListener for SndSocT8140AopData { return Ok(()); }; let frames = absolute / LPAI_FRAME_BYTES as u64; + self.reported.store(true, Relaxed); let cursor = le_u64(report, 0x60)?; let ring_bytes = self.ring_bytes.load(Relaxed) as usize; if ring_bytes == 0 || cursor != absolute % ring_bytes as u64 { @@ -2704,6 +2725,7 @@ impl platform::Driver for SndSocT8140AopDriver { ring_bytes: Atomic::new(0), sequence: Atomic::new(0), powered: AtomicFlag::new(false), + reported: AtomicFlag::new(false), running: AtomicFlag::new(false), hw_ptr_frames: Atomic::new(0), report_warnings: [const { AtomicFlag::new(false) }; 5], From b0ee56449cc8a8c23888711e9503aba0396742bc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 15:59:22 +1000 Subject: [PATCH 0471/2680] ASoC: apple: macaudio: do not read a substream's runtime in the stop work The software-gain stop work walks every runtime on the card and, for each playback substream, went through snd_soc_substream_to_rtd() to find the link's properties. Not every playback substream on the card has a runtime as its private data, so the walk dereferenced NULL: Unable to handle kernel NULL pointer dereference at virtual address 0000000000000008 Workqueue: events macaudio_sw_gain_stop_work [snd_soc_macaudio] pc : macaudio_sw_gain_stop_work+0x68/0xf0 [snd_soc_macaudio] seen on the J700 right after "Speaker volumes locked: Not all child controls locked by the same process" when a speaker stream closed. The runtime being walked already names the link; take the properties from it and only then look at its playback substream. Signed-off-by: Ryan Murray --- sound/soc/apple/macaudio.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index 295be26aba9de..095a0391ea411 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -502,12 +502,16 @@ static void macaudio_sw_gain_stop_work(struct work_struct *work) struct snd_pcm_substream *substream; unsigned long flags; - /* No volume or control lock may be held while taking the stream lock */ + /* + * No volume or control lock may be held while taking the stream lock. + * Pick the links from the runtime being walked: not every playback + * substream on the card carries a runtime as its private data. + */ for_each_card_rtds(&ma->card, rtd) { - if (!rtd->pcm) + if (!rtd->pcm || !ma->link_props[rtd->dai_link->id].sw_gain) continue; substream = rtd->pcm->streams[SNDRV_PCM_STREAM_PLAYBACK].substream; - if (!substream || !macaudio_sw_gain_pcm(substream)) + if (!substream) continue; snd_pcm_stream_lock_irqsave(substream, flags); if (substream->runtime && From d1ec9a7ce55c7a1fb5ec3e59b856ba3bc79609a0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 21:29:31 +1000 Subject: [PATCH 0472/2680] soc: apple: aop, ASoC: apple: t8140-aop-audio: Describe observed behaviour, not other implementations Several comments in the T8140 support explained a register value or a behaviour by what another operating system's implementation does, not by what the hardware and firmware do. Reword them to state the observed behaviour directly: the ambient light sensor's ADT properties, the secure ADMAC channel's access path, the AOP calibration message and endpoint set, the ISP capture watchdog, and the speaker unmute value. No functional change. [ryan: split from commit 1ce75ee9e797 ("treewide: apple: t8140: Describe observed behaviour, not other implementations") for the 7.2.10 topics: the AOP and audio hunks only; the DT and ISP hunks travel with 000-devicetree and 240-isp] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 6 +++--- sound/soc/apple/t8140_aop_audio.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 90cbfc95eefa2..69123e033e9db 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -178,7 +178,7 @@ const SETUP_ALS_EP: u8 = 0x21; const SETUP_SOURCE_EP: u8 = 0x20; const SETUP_SET_SOURCE_BUFFER: u64 = 0x6b80_3ce4_92bd_9547; /// The calibration is an 80-byte message: a 64-bit operation code, a 64-bit -/// body length of 56, and 64 bytes of data that are sent as captured (the +/// body length of 56, and 64 bytes of calibration data sent unchanged (the /// last eight lie beyond the stated body length). const ALS_CALIBRATION_LEN: usize = 80; const ALS_CALIBRATION_OPERATION: u64 = 0x0746_b8d6_6515_2e31; @@ -2252,8 +2252,8 @@ const HW_CFG_T6020: AopHwConfig = AopHwConfig { /// T8140: the firmware is started with the boot arguments the bootloader /// left, boots through the setup port and speaks EPIC version 4. Of the /// advertised AFK endpoints, the application map is 0x20 misc, 0x21 -/// aop-audio, 0x22 aop-voicetrigger, 0x23 als and 0x2b aop-audprov, which -/// are the ones macOS starts as well; the rest are started if they will. +/// aop-audio, 0x22 aop-voicetrigger, 0x23 als and 0x2b aop-audprov; the rest +/// are started if they will. const HW_CFG_T8140: AopHwConfig = AopHwConfig { ec0p: 0, aopt: 0, diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index ceae8e27110c7..37fed754b0a13 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -957,8 +957,8 @@ impl SndSocT8140AopData { /// Speaker stream start, from the system workqueue right after trigger /// START: the DMA must be running before pwrd starts the LEAP consuming /// (issued from prepare, before the DMA, the wire stays silent), then the - /// firmware unmute, the value the native amplifier driver writes after - /// enabling its output. + /// firmware unmute, the value observed on the bus once the amplifier + /// output is enabled. fn spkr_go(&self) -> Result<()> { if !self.spkr_want_run.load(Relaxed) || self.spkr.running.load(Relaxed) { return Ok(()); From 5e96c4cdd1998189c6bf555386d694bfdaf02227 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 19:05:02 +1100 Subject: [PATCH 0473/2680] ASoC: apple: t8140-aop-audio: count the holders of the lpai run state The low-power microphone's runn state is requested by its own PCM and, around every back-end and microphone-array startup, by with_fabric_raised(), which runs the stream to bring a gated audio fabric back. Each request reads the state back from the firmware and fails on a mismatch, but nothing orders the requests. When two services start at the same time, one raise returns lpai to idle between the other's run request and its readback, and that open fails with EIO: audio.3.auto: lpai power state 0x69646c65 after requesting 0x72756e6e A raise that finishes while the PCM starts likewise idles the capture the PCM just started, and a raise that skips the run request because the PCM holds it powers the leaves while the PCM's close idles the stream under it. Count the holders of runn under a mutex: the first holder requests runn, the last one requests idle, and every lpai power request is made by one caller at a time. Fixes: 656867bfe3ec ("ASoC: apple: t8140-aop-audio: raise a gated audio fabric before the leaves") Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 78 ++++++++++++++++++++++++------ 1 file changed, 63 insertions(+), 15 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index 37fed754b0a13..79ee7ea8ccbea 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -52,6 +52,7 @@ use kernel::{ }, error::from_err_ptr, module_platform_driver, + new_mutex, new_spinlock, of, platform, @@ -73,6 +74,7 @@ use kernel::{ }, Arc, ArcBorrow, + Mutex, SpinLock, // }, types::{ForeignOwnable, Opaque}, @@ -432,6 +434,14 @@ struct StreamState { // sets it and close clears it under the same lock. unsafe impl Send for StreamState {} +/// The holders of the low-power microphone's `runn` state: its own PCM +/// while prepared, and every service startup that raises the fabric. +struct LpaiRun { + users: u32, + /// `runn` is the last state the firmware confirmed. + running: bool, +} + #[pin_data] struct SndSocT8140AopData { dev: ARef, @@ -440,7 +450,7 @@ struct SndSocT8140AopData { ring: Arc, ring_bytes: Atomic, sequence: Atomic, - /// lpai is in `runn`. + /// The low-power microphone PCM holds a reference on `lpai_run`. powered: AtomicFlag, /// A low-power microphone report arrived since the last run request. reported: AtomicFlag, @@ -478,6 +488,10 @@ struct SndSocT8140AopData { hpai_start_work: Work, #[pin] stream: SpinLock, + /// Every lpai power request goes through here, see + /// [`SndSocT8140AopData::lpai_run_get`]. + #[pin] + lpai_run: Mutex, } const SPKR_START_WORK: u64 = 0; @@ -680,6 +694,39 @@ impl SndSocT8140AopData { Ok(()) } + /// Takes a reference on the low-power microphone's `runn` state, + /// requesting it for the first holder. + /// + /// The PCM and the fabric raises share the stream, and every power + /// request is checked against the state the firmware reports back: a + /// raise that returns lpai to `idle` while another service starts up + /// makes that one read `idle` after requesting `runn`, and idles a + /// capture the PCM just started. So they count their holds here and + /// only the first and last holder change the state. + fn lpai_run_get(&self) -> Result<()> { + let mut run = self.lpai_run.lock(); + if !run.running { + self.reported.store(false, Relaxed); + self.set_power(POWER_STATE_RUN)?; + run.running = true; + } + run.users += 1; + Ok(()) + } + + /// Drops a reference taken by [`Self::lpai_run_get`], returning lpai to + /// `idle` after the last holder. If that fails, the next holder finds + /// it still running. + fn lpai_run_put(&self) -> Result<()> { + let mut run = self.lpai_run.lock(); + run.users -= 1; + if run.users == 0 && run.running { + self.set_power(POWER_STATE_IDLE)?; + run.running = false; + } + Ok(()) + } + // ---- the back-end services ---- /// Property 202 with the service's sequence and a 24 MHz timestamp; @@ -785,14 +832,10 @@ impl SndSocT8140AopData { /// does not do that (measured: `audio_p` stays gated for the whole leaf /// retry window), only a running stream does, after which a service in /// `pw0 ` keeps it up only while its stream runs. The low-power - /// microphone is attached for the firmware's lifetime, so run it briefly - /// around `f` when it is idle, once its first report has arrived. + /// microphone is attached for the firmware's lifetime, so hold it in + /// `runn` around `f`, once its first report has arrived. fn with_fabric_raised(&self, f: impl FnOnce() -> Result) -> Result { - if self.powered.load(Relaxed) { - return f(); - } - self.reported.store(false, Relaxed); - self.set_power(POWER_STATE_RUN)?; + self.lpai_run_get()?; let mut polls = 0; while !self.reported.load(Relaxed) && polls < FABRIC_RAISE_POLLS { kernel::time::delay::fsleep(kernel::time::Delta::from_millis(FABRIC_RAISE_POLL_MS)); @@ -802,7 +845,7 @@ impl SndSocT8140AopData { dev_warn!(self.dev, "lpai: no report while raising the fabric\n"); } let ret = f(); - if let Err(e) = self.set_power(POWER_STATE_IDLE) { + if let Err(e) = self.lpai_run_put() { dev_warn!(self.dev, "lpai: unable to idle after raising the fabric: {:?}\n", e); } ret @@ -1192,12 +1235,11 @@ unsafe extern "C" fn lpai_pcm_close(substream: *mut bindings::snd_pcm_substream) let mut st = data.stream.lock(); st.substream = ptr::null_mut(); } - if data.powered.load(Relaxed) { - if let Err(e) = data.set_power(POWER_STATE_IDLE) { + if data.powered.xchg(false, Relaxed) { + if let Err(e) = data.lpai_run_put() { dev_err!(data.dev, "unable to return lpai to idle\n"); return e.to_errno(); } - data.powered.store(false, Relaxed); } 0 } @@ -1221,7 +1263,7 @@ unsafe extern "C" fn lpai_pcm_prepare(substream: *mut bindings::snd_pcm_substrea data.hw_ptr_frames.store(0, Relaxed); } if !data.powered.load(Relaxed) { - if let Err(e) = data.set_power(POWER_STATE_RUN) { + if let Err(e) = data.lpai_run_get() { dev_err!(data.dev, "unable to run lpai\n"); return e.to_errno(); } @@ -2620,8 +2662,10 @@ impl SndSocT8140AopDriver { // SAFETY: the card is freed once, while the binding remains live. unsafe { bindings::snd_card_free(self.card) }; } - if self.data.powered.load(Relaxed) && self.data.set_power(POWER_STATE_IDLE).is_ok() { - self.data.powered.store(false, Relaxed); + // Every stream is closed now; retry an idle request that failed then. + let mut run = self.data.lpai_run.lock(); + if run.running && self.data.set_power(POWER_STATE_IDLE).is_ok() { + run.running = false; } } } @@ -2751,6 +2795,10 @@ impl platform::Driver for SndSocT8140AopDriver { period_acc: 0, buffer_bytes: 0, }), + lpai_run <- new_mutex!(LpaiRun { + users: 0, + running: false, + }), }), GFP_KERNEL, )?; From 59050b226b627a06ecbb9c175e7ef3fbaf5109ba Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:35:02 +1100 Subject: [PATCH 0474/2680] ASoC: apple: t8140-aop-audio: wait for stopped streams in sync_stop None of the driver's PCMs has a sync_stop op, so after a stop ALSA moves on to hw_params, hw_free or prepare without waiting for the stream to have really stopped. The microphone array and the front-ends stop their ADMAC channel from the trigger with dmaengine_terminate_async(), and the channel is only synchronized at close. hw_free comes first and releases the buffer, which these PCMs allocate at every hw_params, so the channel's last burst and a pending period callback can land after the buffer is gone. Give them the dmaengine helper's sync_stop, as the generic dmaengine PCM does. The low-power microphone has no DMA: its report handler copies each reported span into the ALSA buffer under the stream state lock, after checking the run flag, and the trigger only clears that flag. A report that passed the check before the stop can still be copying when hw_free releases a buffer larger than the preallocated one. Its sync_stop takes the state lock once, which waits for such a copy; any later report sees the flag clear. Fixes: eeb112648f2a ("ASoC: apple: Add T8140 AOP audio driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 58 ++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index 79ee7ea8ccbea..9794acdd789fc 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -1298,6 +1298,24 @@ unsafe extern "C" fn lpai_pcm_trigger( } } +/// Called after a stop, without the ALSA stream lock, before the buffer is +/// reallocated or freed (hw_params, hw_free) or the counters are reset +/// (prepare). The trigger only clears the run flag, so a report that saw +/// the stream running may still be copying into the buffer: it holds the +/// stream lock for the whole copy, so taking that lock waits for it, and +/// every report after it sees the flag clear. +/// +/// # Safety +/// +/// Called by ALSA with a live substream of this card's PCM, whose private +/// data is the driver's `Arc`. +unsafe extern "C" fn lpai_pcm_sync_stop(substream: *mut bindings::snd_pcm_substream) -> i32 { + // SAFETY: ALSA calls this op with a live substream of the LPAI PCM. + let data = unsafe { pcm_data(substream) }; + drop(data.stream.lock()); + 0 +} + /// # Safety /// /// Called by ALSA with a live substream of this card's PCM, whose private @@ -1416,6 +1434,19 @@ unsafe extern "C" fn hpai_pcm_trigger( 0 } +/// The trigger stops the DMA asynchronously: wait for the channel to stop +/// and for its completion callbacks to finish before ALSA reuses or frees +/// the buffer. +/// +/// # Safety +/// +/// Called by ALSA with a live substream of this card's PCM. +unsafe extern "C" fn hpai_pcm_sync_stop(substream: *mut bindings::snd_pcm_substream) -> i32 { + // SAFETY: the substream is live and its dmaengine runtime was set up by + // `snd_dmaengine_pcm_open` in `hpai_pcm_open`. + unsafe { bindings::snd_dmaengine_pcm_sync_stop(substream) } +} + /// # Safety /// /// Called once by ALSA when the PCM goes away, with the `Arc` leaked into @@ -1995,6 +2026,28 @@ unsafe extern "C" fn asoc_pcm_trigger( unsafe { bindings::snd_dmaengine_pcm_trigger(substream, cmd) } } +/// Front-end stop synchronization, as for the microphone array: the trigger +/// stops the DMA asynchronously, so wait for the channel to stop and for its +/// completion callbacks to finish before ALSA reuses or frees the buffer. +/// +/// # Safety +/// +/// Called by ASoC on this driver's registered component, with live +/// arguments belonging to one of its links. +unsafe extern "C" fn asoc_pcm_sync_stop( + _component: *mut bindings::snd_soc_component, + substream: *mut bindings::snd_pcm_substream, +) -> i32 { + // SAFETY: ASoC hands its ops a live substream whose private data is the + // `snd_soc_pcm_runtime` of one of this component's links. + if unsafe { rtd_is_be(substream_rtd(substream)) } { + return 0; + } + // SAFETY: the substream is live and its dmaengine runtime was set up by + // `snd_dmaengine_pcm_open` in `asoc_pcm_open`. + unsafe { bindings::snd_dmaengine_pcm_sync_stop(substream) } +} + /// # Safety /// /// Called by ASoC on this driver's registered component, with live @@ -2431,6 +2484,7 @@ impl AsocContext { hw_params: Some(asoc_pcm_hw_params), prepare: Some(asoc_pcm_prepare), trigger: Some(asoc_pcm_trigger), + sync_stop: Some(asoc_pcm_sync_stop), pointer: Some(asoc_pcm_pointer), copy: Some(asoc_pcm_copy), ..Default::default() @@ -2474,7 +2528,7 @@ impl SndSocT8140AopDriver { ioctl: None, hw_params: None, hw_free: None, - sync_stop: None, + sync_stop: Some(lpai_pcm_sync_stop), get_time_info: None, fill_silence: None, copy: None, @@ -2492,7 +2546,7 @@ impl SndSocT8140AopDriver { ioctl: None, hw_params: Some(hpai_pcm_hw_params), hw_free: None, - sync_stop: None, + sync_stop: Some(hpai_pcm_sync_stop), get_time_info: None, fill_silence: None, copy: None, From 84aa9bdcece3e3001813d6a1bed9203da8727395 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:40:33 +1100 Subject: [PATCH 0475/2680] dt-bindings: sound: cirrus,cs42l42: Allow platform-managed CS42L83 supplies Keep CS42L42 supply phandles mandatory while permitting the established CS42L83 board model with platform-managed supplies. Existing public Apple board descriptions already use that model. [ryan: split from commit 691ba4cea259 ("dt-bindings: apple: Describe T8140 idle and display crossbar controllers") for the 7.2.10 topics: the cs42l42 binding hunk only; the cpuidle and display crossbar bindings stay with their own topics] Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/sound/cirrus,cs42l42.yaml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml b/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml index 28a5303cf7204..1d839e28c0f8a 100644 --- a/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml +++ b/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml @@ -14,6 +14,8 @@ description: It provides a high-dynamic range, stereo DAC for audio playback and a mono high-dynamic-range ADC for audio capture. There is an integrated headset detection block. + CS42L83 systems may have platform-managed supplies without regulator + phandles, as on Apple Silicon boards. properties: compatible: @@ -192,11 +194,18 @@ properties: required: - compatible - reg - - VP-supply - - VCP-supply - - VD_FILT-supply - - VL-supply - - VA-supply +allOf: + - if: + properties: + compatible: + const: cirrus,cs42l42 + then: + required: + - VP-supply + - VCP-supply + - VD_FILT-supply + - VL-supply + - VA-supply additionalProperties: false From 7afdf72bced97953a5c2bef75a46c2fef26ade40 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:22:40 +1100 Subject: [PATCH 0476/2680] ASoC: apple: t8140-aop-audio: Format with rustfmt make rustfmtcheck flags one dev_warn!() call that is wider than the kernel's rustfmt configuration allows. Reformat it. Also move the comment on the float conversion's exponent onto its own line above the statement. rustfmt had aligned its continuation line under the end of the statement, where it reads as a stray comment. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index 9794acdd789fc..3ba08ee5597c3 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -846,7 +846,11 @@ impl SndSocT8140AopData { } let ret = f(); if let Err(e) = self.lpai_run_put() { - dev_warn!(self.dev, "lpai: unable to idle after raising the fabric: {:?}\n", e); + dev_warn!( + self.dev, + "lpai: unable to idle after raising the fabric: {:?}\n", + e + ); } ret } @@ -1907,8 +1911,8 @@ fn f32_bits_of_q31(v: i32) -> u32 { } let sign = if v < 0 { 1u32 << 31 } else { 0 }; let m = v.unsigned_abs(); - let p = 31 - m.leading_zeros(); // position of the leading one, 0..=31 - // value = m * 2^-31 = 1.f * 2^(p - 31) + // The position of the leading one, 0..=31: |v| / 2^31 = 1.f * 2^(p - 31). + let p = 31 - m.leading_zeros(); let exp = (p as i32 - 31 + 127) as u32; let mant = if p >= 23 { (m >> (p - 23)) & 0x7f_ffff From 70ac9c48cbbd99fdfd419d192d2ef4ee6fd3ebf4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:44:17 +1100 Subject: [PATCH 0477/2680] ASoC: dt-bindings: cirrus,cs42l42: Reference dai-common.yaml The binding lists '#sound-dai-cells' and sound-name-prefix by hand. Reference dai-common.yaml for the common DAI properties instead, as the other codec bindings do, and use unevaluatedProperties so that what it defines is accepted. While at it, put the note on the CS42L83 supplies of Apple Silicon machines in a paragraph of its own and say why they are optional there. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/sound/cirrus,cs42l42.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml b/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml index 1d839e28c0f8a..31cdd309f33ee 100644 --- a/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml +++ b/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml @@ -14,8 +14,9 @@ description: It provides a high-dynamic range, stereo DAC for audio playback and a mono high-dynamic-range ADC for audio capture. There is an integrated headset detection block. - CS42L83 systems may have platform-managed supplies without regulator - phandles, as on Apple Silicon boards. + + On Apple Silicon machines the supplies of the CS42L83 are platform-managed + rails that are not described as regulators, so they are optional there. properties: compatible: @@ -31,8 +32,6 @@ properties: '#sound-dai-cells': const: 0 - sound-name-prefix: true - VP-supply: description: VP power supply. @@ -194,7 +193,9 @@ properties: required: - compatible - reg + allOf: + - $ref: dai-common.yaml# - if: properties: compatible: @@ -207,7 +208,7 @@ allOf: - VL-supply - VA-supply -additionalProperties: false +unevaluatedProperties: false examples: - | From 2715ed7bb929b8d2b673e840a9b7507aaffa8040 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:44:18 +1100 Subject: [PATCH 0478/2680] ASoC: dt-bindings: apple,t8140-aop-audio: Reference dai-common.yaml The node is a DAI provider, like the MCA. Reference dai-common.yaml as apple,mca.yaml does, and use unevaluatedProperties so that the common DAI properties it defines are accepted. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/sound/apple,t8140-aop-audio.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml b/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml index a4e10cbb4deea..b023080e48c7f 100644 --- a/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml +++ b/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml @@ -20,6 +20,9 @@ description: | The remaining audio data moves over the named ADMAC channels while its AOP service is active. The parent AOP binding describes the firmware transport. +allOf: + - $ref: dai-common.yaml# + properties: compatible: const: apple,t8140-aop-audio @@ -96,7 +99,7 @@ required: - apple,chassis-name - apple,machine-kind -additionalProperties: false +unevaluatedProperties: false examples: - | From 42f0b7dbd9c0c2557a4843f8793b6d1e02611b0a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 14:08:54 +1100 Subject: [PATCH 0479/2680] ASoC: apple: t8140-aop-audio: test the speaker ceiling with KUnit The copy-time conversion is the only digital limit between userspace and the J700 speaker amplifiers: the top of "Speaker Playback Volume" must be the amplifier's full scale (a float of 0.5 on the LEAP; above it the words wrap), and macaudio's lock level (87) must sit 20 dB below it. Nothing checked either outside the compile-time assertion on the table's top entry. Add a KUnit suite, snd_soc_t8140_aop_audio_speaker: - the top of the volume range is the amplifier full scale; - the table falls in 0.5 dB steps to mute; - the locked level is 20 dB below full scale; - no S32 sample at any volume converts above full scale (with the exact results for i32::MAX/MIN at full volume and at the lock); - the float conversion is exact on powers of two. kunit.py on arm64 qemu: 5/5 pass. With the conversion's shift changed from 31 to 30 (6 dB over the ceiling) the full-scale test fails. Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 87 ++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index 3ba08ee5597c3..596f717d02b36 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -2874,3 +2874,90 @@ module_platform_driver! { description: "Apple T8140 AOP audio: microphones, headphone jack and speakers", license: "Dual MIT/GPL", } + +/// The speaker ceiling and the copy-path conversion are the whole digital +/// protection of the J700 speakers below the daemon: check them where they +/// are built, without the hardware. +#[kunit_tests(snd_soc_t8140_aop_audio_speaker)] +mod tests { + use super::*; + + /// macaudio's level for the J700 while no daemon holds the lock: + /// MAX98360A_DB(safe_vol = -20). + const LOCKED_VOLUME: usize = 87; + /// 10^(0.5 / 20) in parts per 10^9. + const HALF_DB_PPB: u128 = 1_059_253_725; + /// 0.5 as a float32: the amplifier's full scale on the wire. + const F32_HALF: u32 = 0x3f00_0000; + + #[test] + fn the_top_of_the_volume_range_is_the_amplifier_full_scale() { + assert_eq!( + SPKR_VOLUME_Q31[SPKR_VOLUME_MAX as usize] as i64, + SPKR_CEILING_Q31 + ); + assert_eq!(SPKR_CEILING_Q31, 1 << 30); + assert_eq!(f32_bits_of_q31(SPKR_CEILING_Q31 as i32), F32_HALF); + } + + #[test] + fn the_volume_table_falls_in_half_db_steps_to_mute() { + assert_eq!(SPKR_VOLUME_Q31[0], 0); + for i in 2..=SPKR_VOLUME_MAX as usize { + let (lo, hi) = (SPKR_VOLUME_Q31[i - 1] as u128, SPKR_VOLUME_Q31[i] as u128); + assert!(hi > lo); + // hi / lo == 10^(0.5/20) to within the table's rounding + let expect = lo * HALF_DB_PPB / 1_000_000_000; + assert!(hi.abs_diff(expect) <= 2); + } + } + + #[test] + fn the_locked_level_is_20_db_below_full_scale() { + let locked = SPKR_VOLUME_Q31[LOCKED_VOLUME] as i64; + // 10 * locked == ceiling to within the table's rounding (0.1 ppm) + assert!((10 * locked - SPKR_CEILING_Q31).abs() <= 10); + } + + #[test] + fn no_sample_at_any_volume_exceeds_the_full_scale() { + for &gain in SPKR_VOLUME_Q31.iter() { + for sample in [ + i32::MAX, + i32::MIN, + i32::MAX - 1, + i32::MIN + 1, + 1 << 30, + -(1 << 30), + 1, + -1, + 0, + ] { + let bits = spkr_f32_bits(sample, gain); + assert!(bits & 0x7fff_ffff <= F32_HALF); + } + } + // the extremes at full volume: just under and exactly at full scale + assert_eq!( + spkr_f32_bits(i32::MAX, SPKR_CEILING_Q31 as u32), + 0x3eff_ffff + ); + assert_eq!( + spkr_f32_bits(i32::MIN, SPKR_CEILING_Q31 as u32), + 0x8000_0000 | F32_HALF + ); + // and 20 dB down at the locked level + let locked = spkr_f32_bits(i32::MIN, SPKR_VOLUME_Q31[LOCKED_VOLUME]); + // 0.05 as a float32 is 0x3d4c_cccd; the conversion truncates + assert!(locked & 0x7fff_ffff <= 0x3d4c_cccd && locked & 0x7fff_ffff > 0x3d4c_cccd - 64); + } + + #[test] + fn the_float_conversion_is_exact_on_powers_of_two() { + assert_eq!(f32_bits_of_q31(0), 0); + assert_eq!(f32_bits_of_q31(1 << 29), 0x3e80_0000); // 0.25 + assert_eq!(f32_bits_of_q31(-(1 << 30)), 0xbf00_0000); // -0.5 + assert_eq!(f32_bits_of_q31(1), 0x3000_0000); // 2^-31 + assert_eq!(f32_bits_of_q31(i32::MIN), 0xbf80_0000); // -1.0 + } +} From 7b2202062e98ac1b9db09af22174a33f0d608411 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 02:50:46 +1000 Subject: [PATCH 0480/2680] dt-bindings: input: Add the Apple DockChannel HID transport Apple laptops from the M2 generation on connect the built-in keyboard and trackpad to the MTP coprocessor, which announces them as HID interfaces over a DockChannel FIFO pair. The apple,dockchannel-hid node has been in the T8112, T8122, T6020, T6030 and T6031 device trees without a binding. Describe it with the properties those trees use: the four FIFO windows, in either of the two orders the trees have; the tx/rx interrupts; the IOMMU stream; the FIFO size; the helper phandle; the AFE and STM reset lines; and the keyboard, multi-touch, stm, actuator and tp_accel children. Add what the T8140 (J700, MacBook Neo) needs on top. apple,no-stm says that the firmware has no STM interface to report the machine identifiers, so the transport must not wait for one. The apple,j700-multitouch compatible on the multi-touch child says that the trackpad sends a different report layout (a 32-byte header instead of 38 bytes, and an 8-byte trailer). apple,power-method selects the power sequencing protocol of the firmware: with method 1, used by every earlier laptop and therefore the default, the coprocessor sequences the trackpad and asks the host to pulse the reset line; with method 2 the host announces each transition, drives the AFE reset line itself and confirms the transition. The reset line is described with the level that asserts the reset as its active level, so that the driver can use logical GPIO values for both methods. Compile-tested only: dt_binding_check on the schema, and dtbs_check on t8112-j413, t8112-j493, t6020-j414s, t6030-j514s, t6031-j514c, t8122-j613 and t8140-j700 with no new warnings. [ryan: adapted to 7.2.10 - the MAINTAINERS hunk moved to the T8140 section carried by 900-config-tooling] Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/input/apple,dockchannel-hid.yaml | 246 ++++++++++++++++++ 1 file changed, 246 insertions(+) create mode 100644 Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml diff --git a/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml b/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml new file mode 100644 index 0000000000000..3b82308e0b568 --- /dev/null +++ b/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml @@ -0,0 +1,246 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/input/apple,dockchannel-hid.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple DockChannel HID transport + +maintainers: + - Ryan Murray + +description: | + Apple laptops from the M2 generation on connect the built-in keyboard and + trackpad to the MTP (multi-touch processor) coprocessor, which exposes them + as HID devices over a DockChannel FIFO pair. The coprocessor announces its + interfaces by name when it boots. Each interface that the operating system + should use has a child node of the same name; announced interfaces without + a node are ignored. + + The multi-touch interface may need firmware that the host sends after the + coprocessor has booted, followed by a power cycle of the trackpad. Two + power methods exist. With power method 1 the coprocessor sequences the + trackpad itself and asks the host to pulse a reset line when it needs one. + With power method 2 the host announces each power transition to the + coprocessor, drives the reset line of the trackpad analog front end (AFE) + itself and then confirms the transition. + +properties: + compatible: + const: apple,dockchannel-hid + + reg: + minItems: 4 + maxItems: 4 + description: + Host and remote FIFO configuration and data windows, in the order + given by reg-names. + + reg-names: + oneOf: + - items: + - const: config + - const: data + - const: rmt-config + - const: rmt-data + - items: + - const: rmt-config + - const: rmt-data + - const: config + - const: data + + interrupts: + items: + - description: Transmit FIFO threshold + - description: Receive FIFO threshold + + interrupt-names: + items: + - const: tx + - const: rx + + iommus: + maxItems: 1 + + apple,fifo-size: + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + description: Capacity in bytes of each DockChannel FIFO. + + apple,helper-cpu: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The RTKit helper node of the coprocessor at the remote end of the FIFO. + + apple,afe-reset-gpios: + maxItems: 1 + description: + Reset line of the trackpad analog front end, described with the level + that asserts the reset as its active level. With power method 1 the + coprocessor asks the host to pulse it. With power method 2 the host + asserts it while the multi-touch interface is powered off and releases + it when the interface is powered on. + + apple,stm-reset-gpios: + maxItems: 1 + description: + Reset line of the STM interface, pulsed on the coprocessor's request. + + apple,no-stm: + type: boolean + description: + The coprocessor firmware has no STM interface. The STM normally reports + the machine's USB vendor and product identifiers, keyboard type and + serial number before the other interfaces are started; without it the + HID devices are registered with the Apple vendor identifier and no + product identifier. + + keyboard: + type: object + additionalProperties: false + description: The built-in keyboard. + + properties: + hid-country-code: + $ref: /schemas/types.yaml#/definitions/uint32 + maximum: 255 + description: + HID country code of the keyboard, which the coprocessor does not + report. The bootloader fills it in from the machine's own device + tree. + + apple,keyboard-layout-id: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Apple's keyboard layout identifier, exposed to user space through + the HID device's apple_layout_id attribute. + + multi-touch: + type: object + additionalProperties: false + description: The built-in trackpad. + + properties: + compatible: + const: apple,j700-multitouch + description: + The trackpad sends the report layout of the J700 (MacBook Neo), a + 32-byte header followed by the contacts and an 8-byte trailer, + instead of the 38-byte header layout of the earlier laptops. + + firmware-name: + maxItems: 1 + description: + Name of the trackpad firmware that the host sends to the + coprocessor when the interface is first opened. + + apple,power-method: + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [1, 2] + default: 1 + description: + The power sequencing protocol of the coprocessor firmware, as + described above. Power method 2 requires apple,afe-reset-gpios. + + stm: + type: object + additionalProperties: false + description: + The system management interface, which reports the machine + identifiers. + + actuator: + type: object + additionalProperties: false + description: The trackpad haptic actuator. + + tp_accel: + type: object + additionalProperties: false + description: The trackpad accelerometer. + +required: + - compatible + - reg + - reg-names + - interrupts + - interrupt-names + - iommus + - apple,fifo-size + - apple,helper-cpu + +additionalProperties: false + +examples: + - | + #include + #include + + // MacBook Air (M2, 2022) + input@8000 { + compatible = "apple,dockchannel-hid"; + reg = <0x8000 0x4000>, + <0xc000 0x4000>, + <0x0000 0x4000>, + <0x4000 0x4000>; + reg-names = "config", "data", "rmt-config", "rmt-data"; + iommus = <&mtp_dart 1>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, + <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; + + multi-touch { + firmware-name = "apple/tpmtfw-j413.bin"; + }; + + keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; + }; + - | + #include + #include + + // MacBook Neo (J700) + input@8000 { + compatible = "apple,dockchannel-hid"; + reg = <0x8000 0x4000>, + <0xc000 0x4000>, + <0x0000 0x4000>, + <0x4000 0x4000>; + reg-names = "config", "data", "rmt-config", "rmt-data"; + iommus = <&mtp_dart 0>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, + <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + apple,no-stm; + /* SMC pcIO output 19; level 1 asserts the reset */ + apple,afe-reset-gpios = <&smc_gpio 19 GPIO_ACTIVE_HIGH>; + + multi-touch { + compatible = "apple,j700-multitouch"; + firmware-name = "apple/tpmtfw-j700.bin"; + apple,power-method = <2>; + }; + + keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + }; From 5f4b4ef03deae64a113e6acd03e3a4cbc94b8ce4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 02:54:13 +1000 Subject: [PATCH 0481/2680] HID: dockchannel: Fix out-of-bounds interface index on received packets dchid_handle_packet() checks the interface number of a received packet against MAX_INTERFACES, logs an error and then indexes ifaces[] with it anyway. The number is a byte from the coprocessor, so any value from 16 to 255 reads past the 16-entry array and uses whatever pointer it finds there. Drop the packet after logging the error, and re-arm the receive callback as for every other malformed packet. Compile-tested only; not run on hardware. Fixes: 18cedb745333 ("HID: Add Apple DockChannel HID transport driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 5f28b1317a889..5bb9398ebe7f5 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -1053,6 +1053,7 @@ static void dchid_handle_packet(void *cookie, size_t avail) if (hdr.iface >= MAX_INTERFACES) { dev_err(dchid->dev, "Bad iface %d\n", hdr.iface); + goto done; } iface = dchid->ifaces[hdr.iface]; From f5b50f5a5d09f6582402c62c1a1ba9f3d9a4c3de Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 02:54:15 +1000 Subject: [PATCH 0482/2680] HID: dockchannel: Check that a sub-header fits before reading it Every packet body starts with an eight-byte sub-header, but nothing checked that the body was that long. dchid_packet_work() copies the body into a work item sized exactly by the header's length field, so a packet shorter than eight bytes made it read the sub-header past the end of the allocation. dchid_handle_ack() read the sub-header from the stale contents of the packet buffer. On top of that, dchid_packet_work() returned without freeing the work item when the sub-header's length did not fit the packet. Reject bodies shorter than a sub-header in both paths, and free the work item on every exit. Compile-tested only; not run on hardware. Fixes: 18cedb745333 ("HID: Add Apple DockChannel HID transport driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 5bb9398ebe7f5..84154b0777218 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -953,15 +953,23 @@ static void dchid_packet_work(struct work_struct *ws) struct dchid_work *work = container_of(ws, struct dchid_work, work); struct dchid_subhdr *shdr = (void *)work->data; struct dockchannel_hid *dchid = work->iface->dchid; - int type = FIELD_GET(FLAGS_GROUP, shdr->flags); u8 *payload = work->data + sizeof(*shdr); + int type; + + if (work->hdr.length < sizeof(*shdr)) { + dev_err(dchid->dev, "Short packet (%u bytes) for iface %d\n", + work->hdr.length, work->hdr.iface); + goto out; + } if (shdr->length + sizeof(*shdr) > work->hdr.length) { dev_err(dchid->dev, "Bad sub header length (%hu > %zu)\n", shdr->length, work->hdr.length - sizeof(*shdr)); - return; + goto out; } + type = FIELD_GET(FLAGS_GROUP, shdr->flags); + switch (type) { case HID_INPUT_REPORT: if (work->hdr.iface == IFACE_COMM) @@ -974,6 +982,7 @@ static void dchid_packet_work(struct work_struct *ws) break; } +out: kfree(work); } @@ -982,6 +991,11 @@ static void dchid_handle_ack(struct dchid_iface *iface, struct dchid_hdr *hdr, v struct dchid_subhdr *shdr = (void *)data; u8 *payload = data + sizeof(*shdr); + if (hdr->length < sizeof(*shdr)) { + dev_err(iface->dchid->dev, "Short ACK packet (%u bytes)\n", + hdr->length); + return; + } if (shdr->length + sizeof(*shdr) > hdr->length) { dev_err(iface->dchid->dev, "Bad sub header length (%hu > %zu)\n", shdr->length, hdr->length - sizeof(*shdr)); From d123e13df526d1328427e1aeed685de32034d572 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 02:54:18 +1000 Subject: [PATCH 0483/2680] HID: dockchannel: Terminate the interface name before using it The init packet carries the interface name in a 16-byte field that the firmware need not NUL-terminate. dchid_get_interface() compares it with strcmp(), duplicates it with devm_kstrdup() and passes it to alloc_ordered_workqueue(), all of which read on until they find a NUL, so a 16-character name makes them read past the packet. Copy the name into a terminated local buffer first. Compile-tested only; not run on hardware. Fixes: 18cedb745333 ("HID: Add Apple DockChannel HID transport driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 84154b0777218..5db92bcca302b 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -781,11 +781,15 @@ static void dchid_handle_init(struct dockchannel_hid *dchid, void *data, size_t struct dchid_init_hdr *hdr = data; struct dchid_iface *iface; struct dchid_init_block_hdr *blk; + char name[sizeof(hdr->name) + 1]; if (length < sizeof(*hdr)) return; - iface = dchid_get_interface(dchid, hdr->iface, hdr->name); + /* The name field is not necessarily NUL-terminated. */ + memcpy(name, hdr->name, sizeof(hdr->name)); + name[sizeof(hdr->name)] = '\0'; + iface = dchid_get_interface(dchid, hdr->iface, name); if (!iface) return; From c813892e008975315e16c8bf8b8810e6f863ca46 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 02:54:21 +1000 Subject: [PATCH 0484/2680] HID: dockchannel: Check event and init block lengths before reading them Three places in the receive path trusted a length that the coprocessor supplies. dchid_handle_event() reads the event type byte before checking that the payload has one, so an empty payload read one byte past the end of the work item. The GPIO request handler checked the request structure against the bytes left in the packet instead of against its own block length, so a short block was parsed from the bytes of the blocks that follow it. The product name handler indexed product[blk->length - 1], which for an empty block is the byte before the name. Return on an empty event payload, ignore a GPIO request block that is shorter than the request structure, and treat an empty product name as unterminated. Compile-tested only; not run on hardware. Fixes: 18cedb745333 ("HID: Add Apple DockChannel HID transport driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 5db92bcca302b..23978afc2d849 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -812,8 +812,11 @@ static void dchid_handle_init(struct dockchannel_hid *dchid, void *data, size_t case INIT_GPIO_REQUEST: { struct dchid_gpio_request *req = data; - if (sizeof(*req) > length) + if (blk->length < sizeof(*req)) { + dev_warn(dchid->dev, "Short GPIO request for %s\n", + iface->name); break; + } if (iface->gpio_id) { dev_err(dchid->dev, @@ -832,7 +835,7 @@ static void dchid_handle_init(struct dockchannel_hid *dchid, void *data, size_t case INIT_PRODUCT_NAME: { char *product = data; - if (product[blk->length - 1] != 0) { + if (!blk->length || product[blk->length - 1] != 0) { dev_warn(dchid->dev, "Unterminated product name for %s\n", iface->name); } else { @@ -924,6 +927,10 @@ static void dchid_handle_gpio(struct dockchannel_hid *dchid, void *data, size_t static void dchid_handle_event(struct dockchannel_hid *dchid, void *data, size_t length) { u8 *p = data; + + if (!length) + return; + switch (*p) { case EVENT_INIT: dchid_handle_init(dchid, data, length); From d21ec3e2a70097576f9450d0d04e59631cc87263 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 02:54:24 +1000 Subject: [PATCH 0485/2680] HID: dockchannel: Validate the firmware size before reading its header dchid_get_firmware() casts the start of the firmware file to its 20-byte header and checks the header's fields against the file size, but never checks that the file holds a whole header. A truncated file made it read past the end of the firmware data. Reject files shorter than the header. Compile-tested only; not run on hardware. Fixes: 18cedb745333 ("HID: Add Apple DockChannel HID transport driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 23978afc2d849..450e697fef5ec 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -424,6 +424,13 @@ static int dchid_get_firmware(struct dchid_iface *iface, void **firmware, size_t if (ret) return ret; + if (fw->size < sizeof(*hdr)) { + dev_warn(iface->dchid->dev, "%s: firmware too short for its header\n", + fw_name); + ret = -EINVAL; + goto done; + } + hdr = (struct fw_header *)fw->data; if (hdr->magic != FW_MAGIC || hdr->version != FW_VER || From 93c7c2652d1dd258aba7401ece173e5f97d02964 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 02:57:25 +1000 Subject: [PATCH 0486/2680] HID: dockchannel: Return only the bytes copied into a GET_REPORT buffer dchid_handle_ack() copies at most the caller's buffer size out of an acknowledgment, but then records the full wire length of the reply as the response size, which dchid_cmd() returns to its callers. A GET_REPORT through dchid_raw_request() whose reply is longer than the buffer therefore reports more bytes than it delivered. __hid_request() passes that count to hid_input_report() as the size of a buffer it allocated for the report's own length, so the excess is read from beyond the buffer. Record the number of bytes copied, plus the report ID byte, as the response size instead. Callers that deliberately read a prefix of a longer report, such as the 11-byte STM device ID read, still get the bytes they asked for. Compile-tested only; not run on hardware. Fixes: 18cedb745333 ("HID: Add Apple DockChannel HID transport driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 450e697fef5ec..d920d098a2408 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -1041,10 +1041,20 @@ static void dchid_handle_ack(struct dchid_iface *iface, struct dchid_hdr *hdr, v return; } - if (iface->resp_buf && iface->resp_size) - memcpy(iface->resp_buf, payload + 1, min((size_t)shdr->length - 1, iface->resp_size)); + /* + * Report the response length as the bytes actually delivered, so that + * a caller with a small buffer is not told about bytes that were never + * copied. The report ID byte counts as delivered. + */ + if (iface->resp_buf && iface->resp_size) { + size_t copied = min((size_t)shdr->length - 1, iface->resp_size); + + memcpy(iface->resp_buf, payload + 1, copied); + iface->resp_size = copied + 1; + } else { + iface->resp_size = shdr->length; + } - iface->resp_size = shdr->length; iface->out_report = -1; iface->retcode = shdr->retcode; complete(&iface->out_complete); From e350afe64727136871733a0cc954a118583c3b09 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 02:57:27 +1000 Subject: [PATCH 0487/2680] HID: dockchannel: Keep receiving after an allocation failure dchid_handle_packet() re-arms the receive callback at the end of every packet, whether it was handled or dropped, except when the work item for a report cannot be allocated: that path returns without re-arming, and the transport never receives another packet. One failed allocation therefore leaves the keyboard and trackpad dead until the next boot. Drop the packet and re-arm the callback like every other error path. Compile-tested only; not run on hardware. Fixes: 18cedb745333 ("HID: Add Apple DockChannel HID transport driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index d920d098a2408..2c7e2ddb2b3b7 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -1119,7 +1119,7 @@ static void dchid_handle_packet(void *cookie, size_t avail) work = kzalloc(sizeof(*work) + hdr.length, GFP_KERNEL); if (!work) - return; + goto done; work->hdr = hdr; work->iface = iface; From 9e0263d43628405182735d323b47cf24fce9af90 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 02:57:30 +1000 Subject: [PATCH 0488/2680] HID: dockchannel: Fix the interface allocation error paths dchid_get_interface() allocates an interface's ordered workqueue before it looks for the interface's device tree node, and returns without destroying the workqueue when there is no node. Every interface that the coprocessor announces and the device tree does not describe leaks a workqueue on every boot. The result of devm_kstrdup() is not checked either, the device tree node reference is never dropped on failure, and dockchannel_hid_probe() leaks the creation workqueue when the comm interface cannot be set up. Look the node up first, so that nothing is allocated for an interface that is going to be ignored, check the name duplication, drop the node reference on the remaining failure paths and destroy the creation workqueue on the probe failure path. Compile-tested only; not run on hardware. Fixes: 18cedb745333 ("HID: Add Apple DockChannel HID transport driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 2c7e2ddb2b3b7..b9a520d8059e2 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -210,6 +210,7 @@ static DEVICE_ATTR_RO(apple_layout_id); static struct dchid_iface * dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) { + struct device_node *of_node = NULL; struct dchid_iface *iface; if (index >= MAX_INTERFACES) { @@ -220,35 +221,39 @@ dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) if (dchid->ifaces[index]) return dchid->ifaces[index]; + /* Comm is not a HID subdevice */ + if (strcmp(name, "comm")) { + of_node = of_get_child_by_name(dchid->dev->of_node, name); + if (!of_node) { + dev_warn(dchid->dev, "No OF node for subdevice %s, ignoring.", name); + return NULL; + } + } + iface = devm_kzalloc(dchid->dev, sizeof(struct dchid_iface), GFP_KERNEL); if (!iface) - return NULL; + goto err_put_node; iface->index = index; iface->name = devm_kstrdup(dchid->dev, name, GFP_KERNEL); + if (!iface->name) + goto err_put_node; iface->dchid = dchid; - iface->out_report= -1; + iface->of_node = of_node; + iface->out_report = -1; init_completion(&iface->out_complete); init_completion(&iface->ready); mutex_init(&iface->out_mutex); iface->wq = alloc_ordered_workqueue("dchid-%s", WQ_MEM_RECLAIM, iface->name); if (!iface->wq) - return NULL; - - /* Comm is not a HID subdevice */ - if (!strcmp(name, "comm")) { - dchid->ifaces[index] = iface; - return iface; - } - - iface->of_node = of_get_child_by_name(dchid->dev->of_node, name); - if (!iface->of_node) { - dev_warn(dchid->dev, "No OF node for subdevice %s, ignoring.", name); - return NULL; - } + goto err_put_node; dchid->ifaces[index] = iface; return iface; + +err_put_node: + of_node_put(of_node); + return NULL; } static u32 dchid_checksum(void *p, size_t length) @@ -1224,6 +1229,7 @@ static int dockchannel_hid_probe(struct platform_device *pdev) dchid->comm = dchid_get_interface(dchid, IFACE_COMM, "comm"); if (!dchid->comm) { dev_err(dchid->dev, "Failed to initialize comm interface"); + destroy_workqueue(dchid->new_iface_wq); return -EIO; } From 338ec8e4ace3728a6f5e7f02520d32efd734b9d8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 02:57:32 +1000 Subject: [PATCH 0489/2680] HID: dockchannel: Refuse to send messages longer than the header allows The packet header's length field and the sub-header's length field are both 16 bits wide. dchid_send() stores the padded message length plus the sub-header size in the first and the message length in the second without checking that they fit, so an oversized message goes out with wrapped lengths and the coprocessor sees a corrupt packet. The GPIO acknowledgment echoes the command it answers at whatever length the coprocessor sent it, so such a message is not out of reach. Return -EMSGSIZE for a message the header cannot describe. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index b9a520d8059e2..ae3d6d75bd570 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -281,6 +281,10 @@ static int dchid_send(struct dchid_iface *iface, u32 flags, void *msg, size_t si struct dchid_subhdr sub; } __packed h; + /* Both length fields are 16 bits wide. */ + if (size > U16_MAX - sizeof(h.sub) - 3) + return -EMSGSIZE; + memset(&h, 0, sizeof(h)); h.hdr.hdr_len = sizeof(h.hdr); h.hdr.channel = DCHID_CHANNEL_CMD; From 19e511386534303c5621012f95926e44a2b581c2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:00:01 +1000 Subject: [PATCH 0490/2680] HID: dockchannel: Implement driver removal dockchannel_hid_remove() is BUG_ON(1): unloading the module or removing the device crashes the machine. Replace it with an ordered teardown. Mark the transport as stopping and disable the receive interrupt, so that no new packet is parsed, no new work is queued and a receive callback that is still running does not re-arm the interrupt behind the teardown's back. Then flush and destroy the per-interface workqueues, on which packets are parsed and interface creation is queued, then the creation workqueue, and only then destroy the HID devices and drop the device tree node references. Opens and raw requests that arrive during the teardown fail with -ESHUTDOWN instead of waiting for acknowledgments that cannot come. The MTP coprocessor announces its interfaces only when it boots. After an unbind the driver binds again, but the coprocessor is already running and announces nothing, so the keyboard and trackpad stay absent until the next boot. Disable the sysfs bind and unbind attributes, and say in the Kconfig help that unloading and reloading the module has the same effect. Restarting the coprocessor from Linux, which would make both work, is separate work. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/Kconfig | 4 ++ drivers/hid/dockchannel-hid/dockchannel-hid.c | 57 ++++++++++++++++++- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/drivers/hid/dockchannel-hid/Kconfig b/drivers/hid/dockchannel-hid/Kconfig index 254961ad15e19..34472470a9ccc 100644 --- a/drivers/hid/dockchannel-hid/Kconfig +++ b/drivers/hid/dockchannel-hid/Kconfig @@ -10,4 +10,8 @@ config HID_DOCKCHANNEL The keyboard and touchpad are HID based devices connected via the proprietary DockChannel interface. + The MTP coprocessor announces the keyboard and touchpad only when + it boots, so the driver cannot be unbound and rebound, or built as + a module and reloaded, without losing both until the next boot. + endmenu diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index ae3d6d75bd570..cd53d312a3e48 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -151,7 +151,7 @@ struct dchid_iface { int index; const char *name; - const struct device_node *of_node; + struct device_node *of_node; uint8_t tx_seq; bool deferred; @@ -181,6 +181,7 @@ struct dockchannel_hid { struct device *dev; struct dockchannel *dc; struct device_link *helper_link; + bool stopping; bool id_ready; struct dchid_stm_id device_id; @@ -213,6 +214,9 @@ dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) struct device_node *of_node = NULL; struct dchid_iface *iface; + if (READ_ONCE(dchid->stopping)) + return NULL; + if (index >= MAX_INTERFACES) { dev_err(dchid->dev, "Interface index %d out of range\n", index); return NULL; @@ -572,6 +576,9 @@ static int dchid_open(struct hid_device *hdev) struct dchid_iface *iface = hdev->driver_data; int ret; + if (READ_ONCE(iface->dchid->stopping)) + return -ESHUTDOWN; + if (!completion_done(&iface->ready)) { ret = dchid_start_interface(iface); if (ret < 0) @@ -621,6 +628,9 @@ static int dchid_raw_request(struct hid_device *hdev, { struct dchid_iface *iface = hdev->driver_data; + if (READ_ONCE(iface->dchid->stopping)) + return -ESHUTDOWN; + switch (reqtype) { case HID_REQ_GET_REPORT: buf[0] = reportnum; @@ -650,6 +660,9 @@ static void dchid_create_interface_work(struct work_struct *ws) struct hid_device *hid; int ret; + if (READ_ONCE(dchid->stopping)) + return; + if (iface->hid) { dev_warn(dchid->dev, "Interface %s already created!\n", iface->name); @@ -715,6 +728,8 @@ static void dchid_create_interface_work(struct work_struct *ws) static int dchid_create_interface(struct dchid_iface *iface) { + if (READ_ONCE(iface->dchid->stopping)) + return -ESHUTDOWN; if (iface->creating) return -EBUSY; @@ -983,6 +998,9 @@ static void dchid_packet_work(struct work_struct *ws) u8 *payload = work->data + sizeof(*shdr); int type; + if (READ_ONCE(dchid->stopping)) + goto out; + if (work->hdr.length < sizeof(*shdr)) { dev_err(dchid->dev, "Short packet (%u bytes) for iface %d\n", work->hdr.length, work->hdr.iface); @@ -1077,6 +1095,9 @@ static void dchid_handle_packet(void *cookie, size_t avail) struct dchid_iface *iface; u32 checksum; + if (READ_ONCE(dchid->stopping)) + return; + if (dockchannel_recv(dchid->dc, &hdr, sizeof(hdr)) != sizeof(hdr)) { dev_err(dchid->dev, "Read failed (header)\n"); return; @@ -1138,7 +1159,9 @@ static void dchid_handle_packet(void *cookie, size_t avail) queue_work(iface->wq, &work->work); done: - dockchannel_await(dchid->dc, dchid_handle_packet, dchid, sizeof(struct dchid_hdr)); + if (!READ_ONCE(dchid->stopping)) + dockchannel_await(dchid->dc, dchid_handle_packet, dchid, + sizeof(struct dchid_hdr)); } static int dockchannel_hid_probe(struct platform_device *pdev) @@ -1238,6 +1261,7 @@ static int dockchannel_hid_probe(struct platform_device *pdev) } dev_info(dchid->dev, "Initialized, awaiting packets\n"); + platform_set_drvdata(pdev, dchid); dockchannel_await(dchid->dc, dchid_handle_packet, dchid, sizeof(struct dchid_hdr)); return 0; @@ -1245,7 +1269,32 @@ static int dockchannel_hid_probe(struct platform_device *pdev) static void dockchannel_hid_remove(struct platform_device *pdev) { - BUG_ON(1); + struct dockchannel_hid *dchid = platform_get_drvdata(pdev); + int i; + + /* + * Stop the receive path first: no new packets, no new work, and a + * receive callback that is still running does not re-arm itself. + */ + WRITE_ONCE(dchid->stopping, true); + dockchannel_await(dchid->dc, NULL, NULL, 0); + + /* Packet work can still queue interface creation; drain it first. */ + for (i = 0; i < MAX_INTERFACES; i++) { + if (dchid->ifaces[i]) + destroy_workqueue(dchid->ifaces[i]->wq); + } + destroy_workqueue(dchid->new_iface_wq); + + for (i = 0; i < MAX_INTERFACES; i++) { + struct dchid_iface *iface = dchid->ifaces[i]; + + if (!iface) + continue; + if (iface->hid) + hid_destroy_device(iface->hid); + of_node_put(iface->of_node); + } } static const struct of_device_id dockchannel_hid_of_match[] = { @@ -1259,6 +1308,8 @@ static struct platform_driver dockchannel_hid_driver = { .driver = { .name = "dockchannel-hid", .of_match_table = dockchannel_hid_of_match, + /* The coprocessor announces its interfaces only once per boot. */ + .suppress_bind_attrs = true, }, .probe = dockchannel_hid_probe, .remove = dockchannel_hid_remove, From d59c3d9b5bca2f6388f781aa72429f414bde5f7b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:00:03 +1000 Subject: [PATCH 0491/2680] HID: dockchannel: Accept GPIO id 0 in firmware GPIO requests The coprocessor's init packet can announce one GPIO per interface, with an identifier that its GPIO commands then refer to. The driver used the identifier itself as the marker that a GPIO was announced, so an announcement with identifier 0 was forgotten: the interface start did not request the line, a second announcement was not rejected, and the first GPIO command requested the line on the spot. Track the announcement in a flag of its own, so that identifier 0 is handled like any other and a GPIO command for an interface that announced no GPIO is rejected. For a GPIO announced with identifier 0 the line is now requested when the interface starts, as it already was for every other identifier, instead of when the first GPIO command arrives; the request leaves the line at its inactive level in both cases. dchid_request_gpio() also returns the error it got instead of -1, which is -EPERM. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index cd53d312a3e48..010d7ecaf4947 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -165,6 +165,7 @@ struct dchid_iface { struct gpio_desc *gpio; char gpio_name[MAX_GPIO_NAME]; int gpio_id; + bool gpio_announced; struct mutex out_mutex; u32 out_flags; @@ -477,6 +478,7 @@ static int dchid_get_firmware(struct dchid_iface *iface, void **firmware, size_t static int dchid_request_gpio(struct dchid_iface *iface) { char prop_name[MAX_GPIO_NAME + 16]; + int ret; if (iface->gpio) return 0; @@ -488,10 +490,12 @@ static int dchid_request_gpio(struct dchid_iface *iface) iface->gpio = devm_gpiod_get_index(iface->dchid->dev, prop_name, 0, GPIOD_OUT_LOW); - if (IS_ERR_OR_NULL(iface->gpio)) { - dev_err(iface->dchid->dev, "Failed to request GPIO %s-gpios\n", prop_name); + if (IS_ERR(iface->gpio)) { + ret = PTR_ERR(iface->gpio); iface->gpio = NULL; - return -1; + dev_err(iface->dchid->dev, "Failed to request GPIO %s-gpios: %d\n", + prop_name, ret); + return ret; } return 0; @@ -518,7 +522,7 @@ static int dchid_start_interface(struct dchid_iface *iface) goto err; /* If we need a GPIO, make sure we have it. */ - if (iface->gpio_id) { + if (iface->gpio_announced) { ret = dchid_request_gpio(iface); if (ret < 0) goto err; @@ -849,7 +853,7 @@ static void dchid_handle_init(struct dockchannel_hid *dchid, void *data, size_t break; } - if (iface->gpio_id) { + if (iface->gpio_announced) { dev_err(dchid->dev, "Cannot request more than one GPIO per interface!\n"); break; @@ -857,6 +861,7 @@ static void dchid_handle_init(struct dockchannel_hid *dchid, void *data, size_t strscpy(iface->gpio_name, req->name, MAX_GPIO_NAME); iface->gpio_id = req->id; + iface->gpio_announced = true; break; } @@ -915,6 +920,12 @@ static void dchid_handle_gpio(struct dockchannel_hid *dchid, void *data, size_t goto err; } + if (!iface->gpio_announced) { + dev_err(dchid->dev, "Got GPIO command for %s, which announced no GPIO\n", + iface->name); + goto err; + } + if (dchid_request_gpio(iface) < 0) goto err; From bd2b73cfa980d8b2e6604eeb2003e839ce64f6c1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:00:05 +1000 Subject: [PATCH 0492/2680] HID: Remove the duplicate dockchannel-hid Makefile entry drivers/hid/Makefile adds the dockchannel-hid directory twice, four lines apart. Keep the first entry. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile index 01c6cfa52a495..f0796a041eceb 100644 --- a/drivers/hid/Makefile +++ b/drivers/hid/Makefile @@ -183,8 +183,6 @@ obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid/ obj-$(CONFIG_SPI_HID_APPLE_CORE) += spi-hid/ -obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid/ - obj-$(CONFIG_SURFACE_HID_CORE) += surface-hid/ obj-$(CONFIG_INTEL_THC_HID) += intel-thc-hid/ From 90916bb6642e4751499e343e5a92e835b52f1497 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:09:27 +1000 Subject: [PATCH 0493/2680] HID: dockchannel: Support MTP power method 2 The MTP firmware of the M2 and M3 laptops powers the trackpad with what Apple calls power method 1: after the host has sent the trackpad firmware it sends a four-byte RESET_INTERFACE command for the OFF state and one for the ON state, and the coprocessor sequences the trackpad itself, asking the host to pulse the AFE reset line when it needs a pulse. The J700 (MacBook Neo) firmware uses power method 2 and accepts only the nine-byte form of the command: the method in byte 1, the interface in byte 2, the target state in byte 3 and a has-changed flag in byte 4. Each transition is announced with the flag clear; the host then drives the AFE reset line itself, asserting it for a power-off and holding it for 50 ms, or releasing it for a power-on; and the transition is confirmed with the flag set. After the firmware has been sent the trackpad is taken through OFF and then ON, as with method 1. This is the sequence recorded on the J700 during its bring-up. Select the method with the apple,power-method property of the multi-touch node, which defaults to 1, so that nothing changes on the existing machines: they keep their two four-byte commands and keep ignoring the commands' results. With method 2 the AFE reset line is requested in probe, as soon as the device tree asks for the method, so that a GPIO provider that has not probed yet defers the probe instead of failing the first open; it is requested as-is, so that its level does not change before the first transition has been announced. The line is driven with logical values, and the J700 device tree describes it with the level that asserts the reset as its active level. Should the firmware also announce the line as a GPIO request, the same descriptor serves the request. Move the command encoders into a header of their own and add KUnit tests for them, so that the wire format is pinned down without Apple hardware. The method 2 encoder and the first two test cases are based on work by Eryk Wieliczko for Aurora's tree (September 2026); Eryk Wieliczko's Signed-off-by is outstanding. Compile-tested only, including the KUnit test object; not run on hardware. [ryan: adapted to 7.2.10 - the MAINTAINERS hunk moved to the T8140 section carried by 900-config-tooling] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/Kconfig | 12 +++ drivers/hid/dockchannel-hid/Makefile | 1 + .../dockchannel-hid-protocol-test.c | 81 ++++++++++++++++ .../dockchannel-hid-protocol.h | 55 +++++++++++ drivers/hid/dockchannel-hid/dockchannel-hid.c | 93 ++++++++++++++++++- 5 files changed, 238 insertions(+), 4 deletions(-) create mode 100644 drivers/hid/dockchannel-hid/dockchannel-hid-protocol-test.c create mode 100644 drivers/hid/dockchannel-hid/dockchannel-hid-protocol.h diff --git a/drivers/hid/dockchannel-hid/Kconfig b/drivers/hid/dockchannel-hid/Kconfig index 34472470a9ccc..a19b543c066cb 100644 --- a/drivers/hid/dockchannel-hid/Kconfig +++ b/drivers/hid/dockchannel-hid/Kconfig @@ -14,4 +14,16 @@ config HID_DOCKCHANNEL it boots, so the driver cannot be unbound and rebound, or built as a module and reloaded, without losing both until the next boot. +config HID_DOCKCHANNEL_KUNIT_TEST + tristate "KUnit tests for the DockChannel HID protocol helpers" if !KUNIT_ALL_TESTS + depends on KUNIT && APPLE_DOCKCHANNEL + default KUNIT_ALL_TESTS + help + Build the KUnit tests for the DockChannel HID protocol helpers, + which check the encoding of the MTP power commands. They need no + Apple hardware. For more information on KUnit and unit tests in + general, refer to Documentation/dev-tools/kunit/. + + If unsure, say N. + endmenu diff --git a/drivers/hid/dockchannel-hid/Makefile b/drivers/hid/dockchannel-hid/Makefile index 7dba766b047fc..825ccb27db94b 100644 --- a/drivers/hid/dockchannel-hid/Makefile +++ b/drivers/hid/dockchannel-hid/Makefile @@ -4,3 +4,4 @@ # obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid.o +obj-$(CONFIG_HID_DOCKCHANNEL_KUNIT_TEST) += dockchannel-hid-protocol-test.o diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid-protocol-test.c b/drivers/hid/dockchannel-hid/dockchannel-hid-protocol-test.c new file mode 100644 index 0000000000000..2c4d18a9177a2 --- /dev/null +++ b/drivers/hid/dockchannel-hid/dockchannel-hid-protocol-test.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * KUnit tests for the DockChannel HID MTP power commands + */ +#include +#include + +#include "dockchannel-hid-protocol.h" + +static void dchid_pm1_command_test(struct kunit *test) +{ + const u8 expected[DCHID_PM1_CMD_SIZE] = { + DCHID_CMD_RESET_INTERFACE, DCHID_POWER_METHOD_1, 3, + DCHID_POWER_STATE_ON, + }; + u8 cmd[DCHID_PM1_CMD_SIZE]; + + dchid_pm1_command(3, DCHID_POWER_STATE_ON, cmd); + + KUNIT_EXPECT_MEMEQ(test, cmd, expected, sizeof(expected)); +} + +static void dchid_pm2_will_change_test(struct kunit *test) +{ + const u8 expected[DCHID_PM2_CMD_SIZE] = { + DCHID_CMD_RESET_INTERFACE, DCHID_POWER_METHOD_2, 0x5a, + DCHID_POWER_STATE_ON, 0, 0, 0, 0, 0, + }; + u8 cmd[DCHID_PM2_CMD_SIZE]; + + memset(cmd, 0xff, sizeof(cmd)); + dchid_pm2_command(0x5a, DCHID_POWER_STATE_ON, false, cmd); + + KUNIT_EXPECT_MEMEQ(test, cmd, expected, sizeof(expected)); +} + +static void dchid_pm2_has_changed_test(struct kunit *test) +{ + const u8 expected[DCHID_PM2_CMD_SIZE] = { + DCHID_CMD_RESET_INTERFACE, DCHID_POWER_METHOD_2, 0xff, + DCHID_POWER_STATE_ON, 1, 0, 0, 0, 0, + }; + u8 cmd[DCHID_PM2_CMD_SIZE]; + + memset(cmd, 0xff, sizeof(cmd)); + dchid_pm2_command(0xff, DCHID_POWER_STATE_ON, true, cmd); + + KUNIT_EXPECT_MEMEQ(test, cmd, expected, sizeof(expected)); +} + +static void dchid_pm2_power_off_test(struct kunit *test) +{ + const u8 expected[DCHID_PM2_CMD_SIZE] = { + DCHID_CMD_RESET_INTERFACE, DCHID_POWER_METHOD_2, 1, + DCHID_POWER_STATE_OFF, 1, 0, 0, 0, 0, + }; + u8 cmd[DCHID_PM2_CMD_SIZE]; + + memset(cmd, 0xff, sizeof(cmd)); + dchid_pm2_command(1, DCHID_POWER_STATE_OFF, true, cmd); + + KUNIT_EXPECT_MEMEQ(test, cmd, expected, sizeof(expected)); +} + +static struct kunit_case dchid_protocol_test_cases[] = { + KUNIT_CASE(dchid_pm1_command_test), + KUNIT_CASE(dchid_pm2_will_change_test), + KUNIT_CASE(dchid_pm2_has_changed_test), + KUNIT_CASE(dchid_pm2_power_off_test), + {} +}; + +static struct kunit_suite dchid_protocol_test_suite = { + .name = "dockchannel-hid-protocol", + .test_cases = dchid_protocol_test_cases, +}; + +kunit_test_suite(dchid_protocol_test_suite); + +MODULE_DESCRIPTION("Apple DockChannel HID protocol KUnit tests"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid-protocol.h b/drivers/hid/dockchannel-hid/dockchannel-hid-protocol.h new file mode 100644 index 0000000000000..c73c911862ac0 --- /dev/null +++ b/drivers/hid/dockchannel-hid/dockchannel-hid-protocol.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Apple DockChannel HID transport: MTP interface power commands + */ +#ifndef _DOCKCHANNEL_HID_PROTOCOL_H +#define _DOCKCHANNEL_HID_PROTOCOL_H + +#include +#include + +/* + * Interface power commands go to the comm interface as SET_REPORT feature + * reports. Byte 0 is the command, byte 1 the power method, byte 2 the + * interface number and byte 3 the target power state. + * + * Power method 1 has nothing more. The coprocessor sequences the device + * itself and asks the host to pulse a reset line when it needs one. + * + * Power method 2 adds a has-changed flag in byte 4 and four bytes of + * padding. Each transition is announced with the flag clear, the host then + * drives the AFE reset line, and the transition is confirmed with the flag + * set. The J700 MTP firmware accepts only this nine-byte form. + */ +#define DCHID_CMD_RESET_INTERFACE 0x40 + +#define DCHID_POWER_METHOD_1 1 +#define DCHID_POWER_METHOD_2 2 + +#define DCHID_POWER_STATE_OFF 0 +#define DCHID_POWER_STATE_ON 2 + +#define DCHID_PM1_CMD_SIZE 4 +#define DCHID_PM2_CMD_SIZE 9 + +static inline void dchid_pm1_command(u8 iface, u8 state, + u8 cmd[DCHID_PM1_CMD_SIZE]) +{ + cmd[0] = DCHID_CMD_RESET_INTERFACE; + cmd[1] = DCHID_POWER_METHOD_1; + cmd[2] = iface; + cmd[3] = state; +} + +static inline void dchid_pm2_command(u8 iface, u8 state, bool has_changed, + u8 cmd[DCHID_PM2_CMD_SIZE]) +{ + memset(cmd, 0, DCHID_PM2_CMD_SIZE); + cmd[0] = DCHID_CMD_RESET_INTERFACE; + cmd[1] = DCHID_POWER_METHOD_2; + cmd[2] = iface; + cmd[3] = state; + cmd[4] = has_changed; +} + +#endif /* _DOCKCHANNEL_HID_PROTOCOL_H */ diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 010d7ecaf4947..5746b4681e540 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -19,6 +19,7 @@ #include #include #include "../hid-ids.h" +#include "dockchannel-hid-protocol.h" #define COMMAND_TIMEOUT_MS 1000 #define START_TIMEOUT_MS 2000 @@ -74,7 +75,6 @@ struct dchid_init_hdr { #define INIT_TERMINATOR 2 #define INIT_PRODUCT_NAME 7 -#define CMD_RESET_INTERFACE 0x40 #define CMD_SEND_FIRMWARE 0x95 #define CMD_ENABLE_INTERFACE 0xb4 #define CMD_ACK_GPIO_CMD 0xa1 @@ -176,6 +176,7 @@ struct dchid_iface { struct completion out_complete; u32 keyboard_layout_id; + u32 power_method; }; struct dockchannel_hid { @@ -184,6 +185,9 @@ struct dockchannel_hid { struct device_link *helper_link; bool stopping; + /* Held from probe on when the multi-touch interface uses power method 2 */ + struct gpio_desc *afe_reset; + bool id_ready; struct dchid_stm_id device_id; char serial[64]; @@ -212,6 +216,7 @@ static DEVICE_ATTR_RO(apple_layout_id); static struct dchid_iface * dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) { + u32 power_method = DCHID_POWER_METHOD_1; struct device_node *of_node = NULL; struct dchid_iface *iface; @@ -233,6 +238,19 @@ dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) dev_warn(dchid->dev, "No OF node for subdevice %s, ignoring.", name); return NULL; } + + of_property_read_u32(of_node, "apple,power-method", &power_method); + if (power_method != DCHID_POWER_METHOD_1 && + power_method != DCHID_POWER_METHOD_2) { + dev_err(dchid->dev, "Unsupported power method %u for %s\n", + power_method, name); + goto err_put_node; + } + if (power_method == DCHID_POWER_METHOD_2 && !dchid->afe_reset) { + dev_err(dchid->dev, "Power method 2 for %s needs the AFE reset line\n", + name); + goto err_put_node; + } } iface = devm_kzalloc(dchid->dev, sizeof(struct dchid_iface), GFP_KERNEL); @@ -245,6 +263,7 @@ dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) goto err_put_node; iface->dchid = dchid; iface->of_node = of_node; + iface->power_method = power_method; iface->out_report = -1; init_completion(&iface->out_complete); init_completion(&iface->ready); @@ -384,11 +403,39 @@ static int dchid_enable_interface(struct dchid_iface *iface) static int dchid_reset_interface(struct dchid_iface *iface, int state) { - u8 msg[] = { CMD_RESET_INTERFACE, 1, iface->index, state }; + u8 msg[DCHID_PM1_CMD_SIZE]; + dchid_pm1_command(iface->index, state, msg); return dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); } +/* + * Power method 2: announce the transition, drive the AFE reset line to the + * new state and confirm the transition. The reset is asserted for a power-off + * and held for 50 ms, and released for a power-on. + */ +static int dchid_pm2_set_power(struct dchid_iface *iface, u8 state) +{ + struct dockchannel_hid *dchid = iface->dchid; + u8 msg[DCHID_PM2_CMD_SIZE]; + int ret; + + dchid_pm2_command(iface->index, state, false, msg); + ret = dchid_comm_cmd(dchid, msg, sizeof(msg)); + if (ret < 0) + return ret; + + ret = gpiod_set_value_cansleep(dchid->afe_reset, + state == DCHID_POWER_STATE_OFF); + if (ret < 0) + return ret; + if (state == DCHID_POWER_STATE_OFF) + fsleep(50 * USEC_PER_MSEC); + + dchid_pm2_command(iface->index, state, true, msg); + return dchid_comm_cmd(dchid, msg, sizeof(msg)); +} + static int dchid_send_firmware(struct dchid_iface *iface, void *firmware, size_t size) { struct { @@ -486,6 +533,13 @@ static int dchid_request_gpio(struct dchid_iface *iface) dev_info(iface->dchid->dev, "Requesting GPIO %s#%d: %s\n", iface->name, iface->gpio_id, iface->gpio_name); + /* A power method 2 interface holds its reset line from probe on. */ + if (iface->power_method == DCHID_POWER_METHOD_2 && + !strcmp(iface->gpio_name, "afe-reset")) { + iface->gpio = iface->dchid->afe_reset; + return 0; + } + snprintf(prop_name, sizeof(prop_name), "apple,%s", iface->gpio_name); iface->gpio = devm_gpiod_get_index(iface->dchid->dev, prop_name, 0, GPIOD_OUT_LOW); @@ -541,8 +595,17 @@ static int dchid_start_interface(struct dchid_iface *iface) /* After loading firmware, multi-touch needs a reset */ dev_info(iface->dchid->dev, "Resetting %s\n", iface->name); - dchid_reset_interface(iface, 0); - dchid_reset_interface(iface, 2); + if (iface->power_method == DCHID_POWER_METHOD_2) { + ret = dchid_pm2_set_power(iface, DCHID_POWER_STATE_OFF); + if (ret < 0) + goto err; + ret = dchid_pm2_set_power(iface, DCHID_POWER_STATE_ON); + if (ret < 0) + goto err; + } else { + dchid_reset_interface(iface, DCHID_POWER_STATE_OFF); + dchid_reset_interface(iface, DCHID_POWER_STATE_ON); + } } return 0; @@ -1227,6 +1290,28 @@ static int dockchannel_hid_probe(struct platform_device *pdev) } } + /* + * With power method 2 the host drives the multi-touch AFE reset line + * during every power transition, so the line must be there before the + * coprocessor announces the interface. Request it now, while a + * -EPROBE_DEFER can still be honoured, and leave its level alone until + * the first transition has been announced. + */ + child = of_get_child_by_name(dev->of_node, "multi-touch"); + if (child) { + u32 power_method = DCHID_POWER_METHOD_1; + + of_property_read_u32(child, "apple,power-method", &power_method); + of_node_put(child); + if (power_method == DCHID_POWER_METHOD_2) { + dchid->afe_reset = devm_gpiod_get(dev, "apple,afe-reset", + GPIOD_ASIS); + if (IS_ERR(dchid->afe_reset)) + return dev_err_probe(dev, PTR_ERR(dchid->afe_reset), + "Failed to request the AFE reset line\n"); + } + } + /* * Make sure we also have the MTP coprocessor available, and * defer probe if the helper hasn't probed yet. From e7d24f9330a9fa853a2e1c097ec8d7724083ff2d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:09:30 +1000 Subject: [PATCH 0494/2680] HID: dockchannel: Support MTP firmware without an STM interface The transport starts the keyboard and trackpad interfaces only after the STM interface has reported the machine's vendor and product IDs, keyboard type and serial number, which go into the HID devices. The J700 (MacBook Neo) MTP firmware announces no STM interface, so nothing is ever started. Let the device tree say so with apple,no-stm: the HID devices then get Apple's vendor ID and no product ID, version or serial, and the interfaces start as soon as they are announced. hid-apple and hid-magicmouse match Apple's vendor ID with any product on the host bus, so both still bind. The J700 firmware also announces an interface named "mtp" that the device tree does not describe. Interfaces without a node are ignored by design, so log that at debug level instead of warning on every boot. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 5746b4681e540..0579fcface540 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -235,7 +235,7 @@ dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) if (strcmp(name, "comm")) { of_node = of_get_child_by_name(dchid->dev->of_node, name); if (!of_node) { - dev_warn(dchid->dev, "No OF node for subdevice %s, ignoring.", name); + dev_dbg(dchid->dev, "No OF node for subdevice %s, ignoring\n", name); return NULL; } @@ -1262,6 +1262,17 @@ static int dockchannel_hid_probe(struct platform_device *pdev) dchid->id_ready = !stm || !fwnode_device_is_available(stm); fwnode_handle_put(stm); + /* + * The STM interface reports the machine's vendor and product IDs, + * keyboard type and serial number, and the other interfaces wait for + * it. Firmware without an STM interface gets Apple's vendor ID and + * nothing else. + */ + if (of_property_read_bool(dev->of_node, "apple,no-stm")) { + dchid->device_id.vendor_id = HOST_VENDOR_ID_APPLE; + dchid->id_ready = true; + } + /* * First make sure all the GPIOs are available, in cased we need to defer. * This is necessary because MTP will request them by name later, and by then From abefa3bddddec81e05f8544c122b6c0ab5fc7737 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:09:32 +1000 Subject: [PATCH 0495/2680] HID: magicmouse: Fix malformed kernel-doc comments The comments on struct tp_header and struct tp_mouse_report open with "/**" but name no structure and describe only some of the members, so a W=1 build warns "expecting prototype for struct vendor" and "expecting prototype for struct standard". Give them the structure names and the missing members. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/hid-magicmouse.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index f2683800bca24..ef3dbaa56c472 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -712,10 +712,12 @@ struct tp_finger { } __attribute__((packed, aligned(2))); /** - * struct vendor trackpad report + * struct tp_header - vendor trackpad report header * - * @num_fingers: the number of fingers being reported in @fingers + * @unknown: unknown + * @num_fingers: the number of finger records that follow * @buttons: same as HID buttons + * @unknown3: unknown */ struct tp_header { // HID vendor part, up to 1751 bytes @@ -726,10 +728,13 @@ struct tp_header { }; /** - * struct standard HID mouse report + * struct tp_mouse_report - standard HID mouse report * - * @report_id: reportid + * @report_id: report ID * @buttons: HID Usage Buttons 3 1-bit reports + * @rel_x: relative X movement + * @rel_y: relative Y movement + * @padding: padding */ struct tp_mouse_report { // HID mouse report From 6f78105967e1846e9dab87661ed64a430b13ca77 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:09:35 +1000 Subject: [PATCH 0496/2680] HID: magicmouse: Decode the J700 C1FE multitouch report The trackpad of the J700 (MacBook Neo) is driven by Apple's C1FE controller, and its MTP firmware sends report 0x75 in a layout the driver does not accept: a 32-byte header instead of the 38-byte one, then the contact records, then an 8-byte trailer. The header carries the contact section length at offset 16, the trailer length at offset 20, the contact count at offset 22 and the button in bit 0 of offset 23; the contact records have the known 30-byte layout, of which only the positions are known to be meaningful. The touch area field is zero on some real contacts, and the controller has no force sensor. Select the layout from the apple,j700-multitouch compatible of the transport's multi-touch node, so that nothing changes for the other laptops. Check the whole shape of a report before decoding it, and drop a report that fails the check rather than fall through to the other decoder; report every declared contact instead of filtering on the touch area; and advertise only the position and button capabilities, so that user space does not take a missing pressure for a touch size. The report layout and the decoding rules are based on LiamRay11's description of the C1FE report for Aurora's tree, from a capture taken on 2026-09-07; LiamRay11's Signed-off-by is outstanding. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/hid-magicmouse.c | 140 +++++++++++++++++++++++++++++------ 1 file changed, 116 insertions(+), 24 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index ef3dbaa56c472..d6c43cc967e3b 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -15,7 +15,9 @@ #include #include #include +#include #include +#include #include #include "hid-ids.h" @@ -143,6 +145,7 @@ struct magicmouse_input_ops { * @input: Input device through which we report events. * @quirks: Currently unused. * @query_dimensions: Whether to query and update dimensions on first open + * @mtp_c1fe: Whether the MTP trackpad sends the J700 C1FE report layout. * @ntouches: Number of touches in most recent touch report. * @scroll_accel: Number of consecutive scroll motions. * @scroll_jiffies: Time of last scroll motion. @@ -158,6 +161,7 @@ struct magicmouse_sc { struct input_dev *input; unsigned long quirks; bool query_dimensions; + bool mtp_c1fe; int ntouches; int scroll_accel; @@ -745,6 +749,58 @@ struct tp_mouse_report { u8 padding[4]; }; +/* + * The J700 trackpad (Apple's C1FE controller) sends report 0x75 with a + * 32-byte header instead of the 38-byte one above, and an 8-byte trailer + * after the contact records. Offsets count from the report ID; multi-byte + * fields are little-endian: + * + * offset size contents + * 0 1 report ID, 0x75 + * 2 1 header length, 32 + * 3 1 4, meaning unknown + * 16 4 contact section length, 30 * N + * 20 2 trailer length, 8 + * 22 1 contact count N + * 23 1 bit 0: button pressed + * 32 30 * N contact records, in the struct tp_finger layout + * 32 + 30 * N 8 trailer, not decoded + * + * Only the position fields of a record are known to be meaningful; the + * touch area fields are zero on some real contacts, and the controller has + * no force sensor. + */ +#define MAGICMOUSE_C1FE_REPORT_ID 0x75 +#define MAGICMOUSE_C1FE_HEADER_SIZE 32 +#define MAGICMOUSE_C1FE_TRAILER_SIZE 8 + +static bool magicmouse_c1fe_report_valid(const u8 *data, int size) +{ + const size_t touch_sz = sizeof(struct tp_finger); + unsigned int count; + + if (size < MAGICMOUSE_C1FE_HEADER_SIZE || + data[0] != MAGICMOUSE_C1FE_REPORT_ID || + data[2] != MAGICMOUSE_C1FE_HEADER_SIZE || data[3] != 4) + return false; + + count = data[22]; + return count <= MAX_CONTACTS && + get_unaligned_le32(data + 16) == count * touch_sz && + get_unaligned_le16(data + 20) == MAGICMOUSE_C1FE_TRAILER_SIZE && + (size_t)size == MAGICMOUSE_C1FE_HEADER_SIZE + count * touch_sz + + MAGICMOUSE_C1FE_TRAILER_SIZE; +} + +static void report_c1fe_finger_data(struct input_dev *input, int slot, + const struct input_mt_pos *pos) +{ + input_mt_slot(input, slot); + input_mt_report_slot_state(input, MT_TOOL_FINGER, true); + input_report_abs(input, ABS_MT_POSITION_X, pos->x); + input_report_abs(input, ABS_MT_POSITION_Y, pos->y); +} + static void report_finger_data(struct input_dev *input, int slot, const struct input_mt_pos *pos, const struct tp_finger *f) @@ -776,7 +832,7 @@ static int magicmouse_raw_event_mtp(struct hid_device *hdev, struct tp_finger *f; int i, n; u32 npoints; - const size_t hdr_sz = sizeof(struct tp_header); + size_t hdr_sz = sizeof(struct tp_header); const size_t touch_sz = sizeof(struct tp_finger); u8 map_contacs[MAX_CONTACTS]; @@ -784,12 +840,19 @@ static int magicmouse_raw_event_mtp(struct hid_device *hdev, // print_hex_dump_debug("appleft ev: ", DUMP_PREFIX_OFFSET, 16, 1, data, // size, false); - /* Expect 46 bytes of prefix, and N * 30 bytes of touch data. */ - if (size < hdr_sz || ((size - hdr_sz) % touch_sz) != 0) + if (msc->mtp_c1fe) { + if (!magicmouse_c1fe_report_valid(data, size)) + return 0; + hdr_sz = MAGICMOUSE_C1FE_HEADER_SIZE; + } else if (size < hdr_sz || ((size - hdr_sz) % touch_sz) != 0) { + /* Expect 46 bytes of prefix, and N * 30 bytes of touch data. */ return 0; + } + /* The count and button bytes sit at the same offsets in both headers. */ tp_hdr = (struct tp_header *)data; + /* The C1FE trailer is shorter than a record, so this is N there too. */ npoints = (size - hdr_sz) / touch_sz; if (npoints < tp_hdr->num_fingers || npoints > MAX_CONTACTS) { hid_warn(hdev, @@ -802,7 +865,7 @@ static int magicmouse_raw_event_mtp(struct hid_device *hdev, n = 0; for (i = 0; i < tp_hdr->num_fingers; i++) { f = (struct tp_finger *)(data + hdr_sz + i * touch_sz); - if (le16_to_int(f->touch_major) == 0) + if (!msc->mtp_c1fe && le16_to_int(f->touch_major) == 0) continue; hid_dbg(hdev, "ev x:%04x y:%04x\n", le16_to_int(f->abs_x), @@ -818,7 +881,10 @@ static int magicmouse_raw_event_mtp(struct hid_device *hdev, for (i = 0; i < n; i++) { int idx = map_contacs[i]; f = (struct tp_finger *)(data + hdr_sz + idx * touch_sz); - report_finger_data(input, msc->tracking_ids[i], &msc->pos[i], f); + if (msc->mtp_c1fe) + report_c1fe_finger_data(input, msc->tracking_ids[i], &msc->pos[i]); + else + report_finger_data(input, msc->tracking_ids[i], &msc->pos[i], f); } input_mt_sync_frame(input); @@ -1070,13 +1136,33 @@ static int magicmouse_setup_input_mtp(struct input_dev *input, mt_flags = INPUT_MT_POINTER | INPUT_MT_DROP_UNUSED | INPUT_MT_TRACK; - /* finger touch area */ - input_set_abs_params(input, ABS_MT_TOUCH_MAJOR, 0, 5000, 0, 0); - input_set_abs_params(input, ABS_MT_TOUCH_MINOR, 0, 5000, 0, 0); + /* + * The C1FE report carries positions and the button only; advertise + * nothing that user space would take for a touch size or a pressure. + */ + if (!msc->mtp_c1fe) { + /* finger touch area */ + input_set_abs_params(input, ABS_MT_TOUCH_MAJOR, 0, 5000, 0, 0); + input_set_abs_params(input, ABS_MT_TOUCH_MINOR, 0, 5000, 0, 0); + + /* finger approach area */ + input_set_abs_params(input, ABS_MT_WIDTH_MAJOR, 0, 5000, 0, 0); + input_set_abs_params(input, ABS_MT_WIDTH_MINOR, 0, 5000, 0, 0); - /* finger approach area */ - input_set_abs_params(input, ABS_MT_WIDTH_MAJOR, 0, 5000, 0, 0); - input_set_abs_params(input, ABS_MT_WIDTH_MINOR, 0, 5000, 0, 0); + input_set_abs_params(input, ABS_MT_PRESSURE, 0, 6000, 0, 0); + + /* + * This makes libinput recognize this as a PressurePad and + * stop trying to use pressure for touch size. Pressure unit + * seems to be ~grams on these touchpads. + */ + input_abs_set_res(input, ABS_MT_PRESSURE, 1); + + /* finger orientation */ + input_set_abs_params(input, ABS_MT_ORIENTATION, + -J314_TP_MAX_FINGER_ORIENTATION, + J314_TP_MAX_FINGER_ORIENTATION, 0, 0); + } /* Note: Touch Y position from the device is inverted relative * to how pointer motion is reported (and relative to how USB @@ -1085,19 +1171,6 @@ static int magicmouse_setup_input_mtp(struct input_dev *input, * inverse of the reported Y. */ - input_set_abs_params(input, ABS_MT_PRESSURE, 0, 6000, 0, 0); - - /* - * This makes libinput recognize this as a PressurePad and - * stop trying to use pressure for touch size. Pressure unit - * seems to be ~grams on these touchpads. - */ - input_abs_set_res(input, ABS_MT_PRESSURE, 1); - - /* finger orientation */ - input_set_abs_params(input, ABS_MT_ORIENTATION, -J314_TP_MAX_FINGER_ORIENTATION, - J314_TP_MAX_FINGER_ORIENTATION, 0, 0); - /* finger position */ input_set_abs_params(input, ABS_MT_POSITION_X, J314_TP_MIN_X, J314_TP_MAX_X, 0, 0); @@ -1120,6 +1193,18 @@ static int magicmouse_setup_input_mtp(struct input_dev *input, */ __clear_bit(EV_REP, input->evbit); + if (msc->mtp_c1fe) { + /* hid-input may have set these from the report descriptor. */ + __clear_bit(ABS_MT_PRESSURE, input->absbit); + __clear_bit(ABS_PRESSURE, input->absbit); + __clear_bit(ABS_MT_TOUCH_MAJOR, input->absbit); + __clear_bit(ABS_MT_TOUCH_MINOR, input->absbit); + __clear_bit(ABS_MT_WIDTH_MAJOR, input->absbit); + __clear_bit(ABS_MT_WIDTH_MINOR, input->absbit); + __clear_bit(ABS_MT_ORIENTATION, input->absbit); + __clear_bit(ABS_TOOL_WIDTH, input->absbit); + } + error = input_mt_init_slots(input, MAX_CONTACTS, mt_flags); if (error) return error; @@ -1334,6 +1419,13 @@ static int magicmouse_probe(struct hid_device *hdev, // internal trackpad use a data format use input ops to avoid // conflicts with the report ID. if (id->bus == BUS_HOST) { + struct device_node *np; + + /* The J700 trackpad sends a different report layout. */ + np = of_get_child_by_name(dev_of_node(hdev->dev.parent), "multi-touch"); + msc->mtp_c1fe = of_device_is_compatible(np, "apple,j700-multitouch"); + of_node_put(np); + msc->input_ops.raw_event = magicmouse_raw_event_mtp; msc->input_ops.setup_input = magicmouse_setup_input_mtp; } else if (id->bus == BUS_SPI) { From 21579429275be68787659793e3c029248c37667e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:10:31 +1000 Subject: [PATCH 0497/2680] Documentation: hid: Describe the Apple J700 C1FE multitouch report Write down the layout of the J700 trackpad's report 0x75 as hid-magicmouse decodes it: the 32-byte header with its length, count and button fields, the 30-byte contact records, of which only the positions are decoded, and the 8-byte trailer, together with the checks a report has to pass and what happens to one that fails them. This is the format the C1FE controller's firmware was captured sending on 2026-09-07; the earlier Apple Silicon laptops use a different header, which is described only in the driver. Based on LiamRay11's description of the C1FE report for Aurora's tree; LiamRay11's Signed-off-by is outstanding. [ryan: adapted to 7.2.10 - the MAINTAINERS hunk moved to the T8140 section carried by 900-config-tooling] Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/hid/apple-j700-multitouch.rst | 61 +++++++++++++++++++++ Documentation/hid/index.rst | 1 + 2 files changed, 62 insertions(+) create mode 100644 Documentation/hid/apple-j700-multitouch.rst diff --git a/Documentation/hid/apple-j700-multitouch.rst b/Documentation/hid/apple-j700-multitouch.rst new file mode 100644 index 0000000000000..493dc43b065f4 --- /dev/null +++ b/Documentation/hid/apple-j700-multitouch.rst @@ -0,0 +1,61 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later + +================================= +Apple J700 C1FE multitouch report +================================= + +The trackpad of the Apple J700 (MacBook Neo) is driven by Apple's C1FE +controller. Its MTP firmware delivers touch data as HID report 0x75 over +the DockChannel HID transport, in a layout that differs from the one the +earlier Apple Silicon laptops use. hid-magicmouse selects this layout when +the transport's ``multi-touch`` device tree node is compatible with +``apple,j700-multitouch``. + +Report layout +============= + +Offsets count from the report ID byte. Multi-byte fields are +little-endian. A report is a 32-byte header, N contact records of 30 bytes +each and an 8-byte trailer, so its total length is 32 + 30 * N + 8. + +=================== ================================================== +Offset / size Contents +=================== ================================================== +0 / 1 Report ID, 0x75 +2 / 1 Header length, 32 +3 / 1 4; meaning unknown +16 / 4 Contact section length, 30 * N +20 / 2 Trailer length, 8 +22 / 1 Contact count N, at most 16 +23 / 1 Bit 0: the button is pressed +32 + 30 * i / 30 Contact record i, for i < N +32 + 30 * N / 8 Trailer; not decoded +=================== ================================================== + +Every report is checked against this shape before anything is decoded. A +report that does not match produces no events at all, neither touches nor +releases. The firmware also sends reports 0x60 and 0x52 when it starts; +they carry no touch data. + +Contact records +=============== + +A contact record has the layout of ``struct tp_finger`` in hid-magicmouse: +signed 16-bit X at offset 4 and Y at offset 6 of the record. Y is negated, +as for the other MTP trackpads, and the axis ranges come from the same +dimensions query (feature report 0xd9). The touch area, tool area, +orientation and pressure fields of the record are not known to carry +meaningful values on this controller, which has no force sensor; they are +not decoded, and the input device does not advertise them. The touch area +field is zero on some genuine contacts, so unlike the earlier layout a +record is never dropped because of it: every one of the N declared +contacts is reported. + +Contacts are matched to slots by position, as for the other MTP trackpads. +Each valid report is a complete frame, so a report with N = 0 releases +every contact. The button is reported independently of the contact count. + +The limit of 16 contacts is the driver's slot capacity, not a measured +property of the hardware. The header checks describe the reports captured +from one J700; firmware that sends other header values is rejected until +its layout is understood. diff --git a/Documentation/hid/index.rst b/Documentation/hid/index.rst index baf156b44b58a..b2651af21ace0 100644 --- a/Documentation/hid/index.rst +++ b/Documentation/hid/index.rst @@ -17,6 +17,7 @@ Human Interface Devices (HID) uhid hid-alps + apple-j700-multitouch intel-ish-hid intel-thc-hid amd-sfh-hid From 472033c4212f51eef38251ca3d227593eb5bc3c5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 22:33:25 +1000 Subject: [PATCH 0498/2680] HID: dockchannel: Bound empty GET_REPORT response buffers The HID core accepts a GET_REPORT request whose buffer holds only the report ID (hid_hw_raw_request() requires len >= 1), which reaches dchid_raw_request() with room for the ID and nothing else, so dchid_cmd() is given a response buffer of size zero. The ACK handler treated a zero-capacity buffer like the SET_REPORT case, which has no buffer at all, and returned the firmware's full ACK length although nothing beyond the ID had been delivered: the caller was told about bytes that were never copied. Take the GET_REPORT path whenever a response pointer is present: copy nothing when there is no room, and report one byte, the report ID. Carried in every J700 qualification candidate of 2026-09-26, from aurora-wip 92257036 + this + the USB DART enable onwards: keyboard and trackpad confirmed by the operator across the candidates, no DockChannel HID errors in the boot logs. The one-byte request itself was not exercised on the machine. Fixes: e945dcb763fa ("HID: dockchannel: Return only the bytes copied into a GET_REPORT buffer") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 0579fcface540..e7e5231ab0da8 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -1147,10 +1147,11 @@ static void dchid_handle_ack(struct dchid_iface *iface, struct dchid_hdr *hdr, v * a caller with a small buffer is not told about bytes that were never * copied. The report ID byte counts as delivered. */ - if (iface->resp_buf && iface->resp_size) { + if (iface->resp_buf) { size_t copied = min((size_t)shdr->length - 1, iface->resp_size); - memcpy(iface->resp_buf, payload + 1, copied); + if (copied) + memcpy(iface->resp_buf, payload + 1, copied); iface->resp_size = copied + 1; } else { iface->resp_size = shdr->length; From c15edd5f86eb2291b641849555d9905187e3bdce Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 18:48:20 -0400 Subject: [PATCH 0499/2680] HID: dockchannel: support firmware-managed M5 trackpad power Support power method 2 when the MTP firmware owns reset, reject conflicting GPIO ownership, and announce AP readiness before the T8142 MTP power acknowledgment. Preserve the existing sequence for other coprocessors and host-controlled trackpads. [ryan: adapted to 7.2.10 - only the drivers/hid/dockchannel-hid/dockchannel-hid.c hunk is carried here; the RTKit and RTKit-helper hunks (apple_rtkit_wake_early_ap()) are carried by 010-soc as 3fddde3a32ed] Signed-off-by: Acelogic --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 41 +++++++++++++++---- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index e7e5231ab0da8..4a5e346097400 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -177,6 +177,7 @@ struct dchid_iface { u32 keyboard_layout_id; u32 power_method; + bool firmware_managed_reset; }; struct dockchannel_hid { @@ -217,6 +218,7 @@ static struct dchid_iface * dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) { u32 power_method = DCHID_POWER_METHOD_1; + bool firmware_managed_reset = false; struct device_node *of_node = NULL; struct dchid_iface *iface; @@ -240,13 +242,19 @@ dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) } of_property_read_u32(of_node, "apple,power-method", &power_method); + firmware_managed_reset = + of_property_read_bool(of_node, "apple,firmware-managed-reset"); + if (firmware_managed_reset && + (strcmp(name, "multi-touch") || power_method != DCHID_POWER_METHOD_2)) + goto err_put_node; if (power_method != DCHID_POWER_METHOD_1 && power_method != DCHID_POWER_METHOD_2) { dev_err(dchid->dev, "Unsupported power method %u for %s\n", power_method, name); goto err_put_node; } - if (power_method == DCHID_POWER_METHOD_2 && !dchid->afe_reset) { + if (power_method == DCHID_POWER_METHOD_2 && + !firmware_managed_reset && !dchid->afe_reset) { dev_err(dchid->dev, "Power method 2 for %s needs the AFE reset line\n", name); goto err_put_node; @@ -264,6 +272,7 @@ dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) iface->dchid = dchid; iface->of_node = of_node; iface->power_method = power_method; + iface->firmware_managed_reset = firmware_managed_reset; iface->out_report = -1; init_completion(&iface->out_complete); init_completion(&iface->ready); @@ -425,12 +434,18 @@ static int dchid_pm2_set_power(struct dchid_iface *iface, u8 state) if (ret < 0) return ret; - ret = gpiod_set_value_cansleep(dchid->afe_reset, - state == DCHID_POWER_STATE_OFF); - if (ret < 0) - return ret; - if (state == DCHID_POWER_STATE_OFF) - fsleep(50 * USEC_PER_MSEC); + if (iface->firmware_managed_reset) { + /* Firmware owns the reset; allow its power-on transition to settle. */ + if (state == DCHID_POWER_STATE_ON) + fsleep(50 * USEC_PER_MSEC); + } else { + ret = gpiod_set_value_cansleep(dchid->afe_reset, + state == DCHID_POWER_STATE_OFF); + if (ret < 0) + return ret; + if (state == DCHID_POWER_STATE_OFF) + fsleep(50 * USEC_PER_MSEC); + } dchid_pm2_command(iface->index, state, true, msg); return dchid_comm_cmd(dchid, msg, sizeof(msg)); @@ -527,6 +542,9 @@ static int dchid_request_gpio(struct dchid_iface *iface) char prop_name[MAX_GPIO_NAME + 16]; int ret; + if (iface->firmware_managed_reset) + return -EINVAL; + if (iface->gpio) return 0; @@ -1312,10 +1330,17 @@ static int dockchannel_hid_probe(struct platform_device *pdev) child = of_get_child_by_name(dev->of_node, "multi-touch"); if (child) { u32 power_method = DCHID_POWER_METHOD_1; + bool firmware_managed_reset; of_property_read_u32(child, "apple,power-method", &power_method); + firmware_managed_reset = + of_property_read_bool(child, "apple,firmware-managed-reset"); of_node_put(child); - if (power_method == DCHID_POWER_METHOD_2) { + if (firmware_managed_reset && + (power_method != DCHID_POWER_METHOD_2 || + of_property_present(dev->of_node, "apple,afe-reset-gpios"))) + return dev_err_probe(dev, -EINVAL, "Conflicting touch reset ownership\n"); + if (power_method == DCHID_POWER_METHOD_2 && !firmware_managed_reset) { dchid->afe_reset = devm_gpiod_get(dev, "apple,afe-reset", GPIOD_ASIS); if (IS_ERR(dchid->afe_reset)) From dc39f1f3d3561269a5513eb0f00797438c54c311 Mon Sep 17 00:00:00 2001 From: Yicong Hui Date: Fri, 21 Aug 2026 20:18:49 +0200 Subject: [PATCH 0500/2680] HID: dockchannel-hid: forward correct report type on SET_REPORT dchid_set_report() always sends HID_OUTPUT_REPORT regardless of the report type requested by the caller. Inline it into its sole caller and forward rtype directly, so SET_REPORT works correctly for feature reports too, not just output reports, and to simplify function calls. Signed-off-by: Yicong Hui --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 4a5e346097400..d5720bae6560d 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -701,12 +701,6 @@ static int dchid_get_report_cmd(struct dchid_iface *iface, u8 reportnum, void *b return ret <= 0 ? ret : ret - 1; } -/* Note: buf includes report number! */ -static int dchid_set_report(struct dchid_iface *iface, void *buf, size_t len) -{ - return dchid_cmd(iface, HID_OUTPUT_REPORT, REQ_SET_REPORT, buf, len, NULL, 0); -} - static int dchid_raw_request(struct hid_device *hdev, unsigned char reportnum, __u8 *buf, size_t len, unsigned char rtype, int reqtype) @@ -721,7 +715,7 @@ static int dchid_raw_request(struct hid_device *hdev, buf[0] = reportnum; return dchid_cmd(iface, rtype, REQ_GET_REPORT, &reportnum, 1, buf + 1, len - 1); case HID_REQ_SET_REPORT: - return dchid_set_report(iface, buf, len); + return dchid_cmd(iface, rtype, REQ_SET_REPORT, buf, len, NULL, 0); default: return -EIO; } From a4c58d6c07b07899e4be9b3ec4c5747f5acd0872 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 21:22:24 +0200 Subject: [PATCH 0501/2680] HID: spi-hid-apple: enumerate the fourth HID interface The J313 trackpad firmware reports five HID interfaces, but the driver clamps the count to three and only ever queries mngt, kbd and tp. The fourth one is the trackpad's haptic actuator: iface 3: name 'Actuator', country 0x0, in 16 out 64 ctrl 1024 Its report descriptor declares output report 0x53, which is the report a waveform is played with, so dropping the interface puts the taptic engine out of reach entirely. Raise the cap to four and give interface 3 a spihid_interface of its own so that it is queried, registered as a HID device and torn down with the others. The "expecting at least 3" sanity check tested against the same constant, so give it its own SPIHID_MIN_DEVICES; otherwise raising the cap would reject a machine that reports exactly three interfaces. Interface 4 is still not enumerated. What it is has not been established. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/spi-hid/spi-hid-apple-core.c | 34 ++++++++++++++++-------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/drivers/hid/spi-hid/spi-hid-apple-core.c b/drivers/hid/spi-hid/spi-hid-apple-core.c index 2ed909895391c..ba4b4b23e6fbc 100644 --- a/drivers/hid/spi-hid/spi-hid-apple-core.c +++ b/drivers/hid/spi-hid/spi-hid-apple-core.c @@ -35,12 +35,14 @@ #define SPIHID_MAX_INPUT_REPORT_SIZE 0x800 -/* support only keyboard, trackpad and management dev for now */ -#define SPIHID_MAX_DEVICES 3 +/* management, keyboard, trackpad and the trackpad's actuator */ +#define SPIHID_MIN_DEVICES 3 +#define SPIHID_MAX_DEVICES 4 #define SPIHID_DEVICE_ID_MNGT 0x0 #define SPIHID_DEVICE_ID_KBD 0x1 #define SPIHID_DEVICE_ID_TP 0x2 +#define SPIHID_DEVICE_ID_AUX 0x3 #define SPIHID_DEVICE_ID_INFO 0xd0 #define SPIHID_READ_PACKET 0x20 @@ -86,6 +88,7 @@ struct spihid_apple { struct spihid_interface mngt; struct spihid_interface kbd; struct spihid_interface tp; + struct spihid_interface aux; wait_queue_head_t wait; struct mutex tx_lock; //< protects against concurrent SPI writes @@ -279,6 +282,8 @@ static struct spihid_apple *spihid_get_data(struct spihid_interface *idev) return container_of(idev, struct spihid_apple, kbd); case SPIHID_DEVICE_ID_TP: return container_of(idev, struct spihid_apple, tp); + case SPIHID_DEVICE_ID_AUX: + return container_of(idev, struct spihid_apple, aux); default: return NULL; } @@ -408,6 +413,8 @@ static struct spihid_interface *spihid_get_iface(struct spihid_apple *spihid, return &spihid->kbd; case SPIHID_DEVICE_ID_TP: return &spihid->tp; + case SPIHID_DEVICE_ID_AUX: + return &spihid->aux; default: return NULL; } @@ -494,18 +501,17 @@ static bool spihid_process_device_info(struct spihid_apple *spihid, u32 iface, num_devices = __le16_to_cpu(info->num_devices); - if (num_devices < SPIHID_MAX_DEVICES) { + if (num_devices < SPIHID_MIN_DEVICES) { dev_err(dev, - "Device info reports %u devices, expecting at least 3", - num_devices); + "Device info reports %u devices, expecting at least %u", + num_devices, SPIHID_MIN_DEVICES); return false; } spihid->num_devices = num_devices; if (spihid->num_devices > SPIHID_MAX_DEVICES) { - dev_info( - dev, - "limiting the number of devices to mngt, kbd and mouse"); + dev_info(dev, "limiting the number of devices to %u", + SPIHID_MAX_DEVICES); spihid->num_devices = SPIHID_MAX_DEVICES; } @@ -592,8 +598,12 @@ static bool spihid_process_iface_info(struct spihid_apple *spihid, u32 num, iface->name[name_len] = '\0'; } - dev_dbg(&spihid->spidev->dev, "Info for %s, country code: 0x%x", - iface->name, iface->country); + dev_dbg(&spihid->spidev->dev, + "iface %u: name '%s', country 0x%x, in %u out %u ctrl %u", + num, iface->name, iface->country, + iface->max_input_report_len, + iface->max_output_report_len, + iface->max_control_report_len); wake_up_interruptible(&spihid->wait); } @@ -1004,9 +1014,10 @@ int spihid_apple_core_probe(struct spi_device *spi, struct spihid_apple_ops *ops spihid->kbd.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); spihid->tp.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); + spihid->aux.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); if (!spihid->report.buf || !spihid->kbd.hid_desc || - !spihid->tp.hid_desc) + !spihid->tp.hid_desc || !spihid->aux.hid_desc) return -ENOMEM; init_waitqueue_head(&spihid->wait); @@ -1081,6 +1092,7 @@ void spihid_apple_core_remove(struct spi_device *spi) /* destroy input devices */ + spihid_destroy_hid_device(&spihid->aux); spihid_destroy_hid_device(&spihid->tp); spihid_destroy_hid_device(&spihid->kbd); From 58aaae3f8f115b694e184f7a5d1d448623ea2462 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 21:22:24 +0200 Subject: [PATCH 0502/2680] HID: spi-hid-apple: address reports by report id, not interface id apple_ll_raw_request() refused any SET_REPORT whose report number differed from the interface id, and apple_ll_output_report() tagged its request with the interface id rather than the report id in buf[0]. The comment on the latter already suspected it: "second idev->id should maybe be buf[0]?". Both survived only because the report number and the interface id happened to coincide for every report the driver sent: the keyboard's LED report is report 1 on interface 1, and the SPI trackpad's multitouch enable is report 2 on interface 2. The actuator on interface 3 takes reports 0x21 and 0x53, so neither held. Its mode switch was rejected before it was sent, spi-hid-apple-of spi2.0: device:3 reportnum:33 mismatch and its waveforms went out labelled report 3, which the firmware ignored silently: every write was accepted and nothing moved. With both corrected the actuator latches host-controlled mode, feature report 0x21 reads back 01, and waveforms play on a J313. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/spi-hid/spi-hid-apple-core.c | 25 ++++++++++++------------ 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/drivers/hid/spi-hid/spi-hid-apple-core.c b/drivers/hid/spi-hid/spi-hid-apple-core.c index ba4b4b23e6fbc..0f8b13994f349 100644 --- a/drivers/hid/spi-hid/spi-hid-apple-core.c +++ b/drivers/hid/spi-hid/spi-hid-apple-core.c @@ -362,15 +362,12 @@ static int apple_ll_raw_request(struct hid_device *hdev, return idev->reply_len; case HID_REQ_SET_REPORT: - if (buf[0] != reportnum) + if (!len || buf[0] != reportnum) return -EINVAL; - if (reportnum != idev->id) { - dev_warn(&spihid->spidev->dev, - "device:%u reportnum:" - "%hhu mismatch", - idev->id, reportnum); - return -EINVAL; - } + /* + * The report number is not the interface id: the keyboard's LED + * report is 1 on interface 1, the actuator's is 0x21 on 3. + */ return spihid_apple_request(spihid, idev->id, 0x52, reportnum, 0x00, 2, buf, len); default: return -EIO; @@ -385,11 +382,13 @@ static int apple_ll_output_report(struct hid_device *hdev, __u8 *buf, if (!spihid) return -1; - dev_dbg(&spihid->spidev->dev, - "apple_ll_output_report: device:%u len:%zu:", - idev->id, len); - // second idev->id should maybe be buf[0]? - return spihid_apple_request(spihid, idev->id, 0x51, idev->id, 0x00, 0, buf, len); + if (!len) + return -EINVAL; + + dev_dbg(&spihid->spidev->dev, "device:%u report:%u len:%zu", + idev->id, buf[0], len); + /* The request carries the report number, not the interface id. */ + return spihid_apple_request(spihid, idev->id, 0x51, buf[0], 0x00, 0, buf, len); } static struct hid_ll_driver apple_hid_ll = { From c690908adf05a96e614743711c9a38a88c766992 Mon Sep 17 00:00:00 2001 From: Yicong Hui Date: Fri, 21 Aug 2026 20:29:06 +0200 Subject: [PATCH 0503/2680] HID: apple-mtp-haptic: add driver for the MTP haptic touchpad actuator Add a driver to control the actuator of the MTP trackpad, for M2 and later devices. It plays a waveform with output report 0x53 and switches the actuator between device-controlled and host-controlled mode with feature report 0x21. The specific hardware commands and their fields have been reverse engineered on my own machine (Macbook Pro 2023 M2 Max) using the m1n1 hypervisor. Signed-off-by: Yicong Hui Co-developed-by: Chromatischer Signed-off-by: Chromatischer --- arch/arm64/configs/asahi.config | 1 + drivers/hid/Kconfig | 10 +++ drivers/hid/Makefile | 1 + drivers/hid/hid-apple-mtp-haptic.c | 106 +++++++++++++++++++++++++++++ include/linux/soc/apple/actuator.h | 19 ++++++ 5 files changed, 137 insertions(+) create mode 100644 drivers/hid/hid-apple-mtp-haptic.c create mode 100644 include/linux/soc/apple/actuator.h diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 77b97e2a92b02..43bcb1e98684e 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -40,6 +40,7 @@ CONFIG_SND_SOC_TAS2764=m CONFIG_SND_SOC_TAS2770=m CONFIG_HID_APPLE=m CONFIG_HID_MAGICMOUSE=m +CONFIG_HID_APPLE_MTP_HAPTIC=m CONFIG_SERIAL_SAMSUNG=y CONFIG_SERIAL_SAMSUNG_CONSOLE=y CONFIG_HID_DOCKCHANNEL=m diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index 0a85b4161b6d4..34f217f82481e 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -146,6 +146,16 @@ config HID_APPLE Say Y here if you want support for keyboards of Apple iBooks, PowerBooks, MacBooks, MacBook Pros and Apple Aluminum. +config HID_APPLE_MTP_HAPTIC + tristate "Apple MTP Haptic Actuator support" + depends on HID + help + Support for the haptic actuator of the internal trackpads of Apple + Silicon laptops. + + This driver can also be built as a module. If so, it will be called + hid-apple-mtp-haptic. + config HID_APPLEIR tristate "Apple infrared receiver" depends on (USB_HID) diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile index f0796a041eceb..4750e03a98b0e 100644 --- a/drivers/hid/Makefile +++ b/drivers/hid/Makefile @@ -29,6 +29,7 @@ obj-$(CONFIG_HID_ACCUTOUCH) += hid-accutouch.o obj-$(CONFIG_HID_ALPS) += hid-alps.o obj-$(CONFIG_HID_ACRUX) += hid-axff.o obj-$(CONFIG_HID_APPLE) += hid-apple.o +obj-$(CONFIG_HID_APPLE_MTP_HAPTIC) += hid-apple-mtp-haptic.o obj-$(CONFIG_HID_APPLEIR) += hid-appleir.o obj-$(CONFIG_HID_APPLETB_BL) += hid-appletb-bl.o obj-$(CONFIG_HID_APPLETB_KBD) += hid-appletb-kbd.o diff --git a/drivers/hid/hid-apple-mtp-haptic.c b/drivers/hid/hid-apple-mtp-haptic.c new file mode 100644 index 0000000000000..e4dca10249df6 --- /dev/null +++ b/drivers/hid/hid-apple-mtp-haptic.c @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#include +#include + +#include "hid-ids.h" + +int apple_taptic_send(struct hid_device *haptic_hdev, u16 effect_type, u8 strength, u8 softness) +{ + u8 event_click[] = {0x53, 0x01, strength, softness, 0x03, 0x02, 0x22, 0x48, 0x49, + 0x01, 0x04, 0x60, 0x17, 0x2D, 0x03, 0x02, 0x18, 0x32, 0x1E}; + u8 event_deep_click[] = {0x53, 0x01, strength, softness, 0x01, 0x07, 0x18, 0x18, 0x4F, + 0x02, 0x02, 0x60, 0x0C, 0x25}; + u8 event_release[] = {0x53, 0x01, strength, softness}; + u8 *msg; + int msg_length; + int ret; + + switch (effect_type) { + case HID_HP_WAVEFORMPRESS & HID_USAGE: + msg = event_click; + msg_length = sizeof(event_click); + break; + case APPLE_HP_WAVEFORMDEEPCLICK & HID_USAGE: + msg = event_deep_click; + msg_length = sizeof(event_deep_click); + break; + case HID_HP_WAVEFORMRELEASE & HID_USAGE: + msg = event_release; + msg_length = sizeof(event_release); + break; + default: + return -EINVAL; + } + + ret = hid_hw_raw_request(haptic_hdev, msg[0], msg, msg_length, + HID_OUTPUT_REPORT, HID_REQ_SET_REPORT); + + if (ret < 0) + return ret; + return 0; +} +EXPORT_SYMBOL_GPL(apple_taptic_send); + +int apple_taptic_switch_modes(struct hid_device *haptic_hdev, bool enable_host_controlled) +{ + u8 msg[] = { 0x21, enable_host_controlled }; + int ret; + + ret = hid_hw_raw_request(haptic_hdev, msg[0], msg, sizeof(msg), + HID_FEATURE_REPORT, HID_REQ_SET_REPORT); + + if (ret < 0) + return ret; + return 0; +} +EXPORT_SYMBOL_GPL(apple_taptic_switch_modes); + +static int apple_actuator_probe(struct hid_device *haptic_hdev, const struct hid_device_id *id) +{ + int ret; + + ret = hid_parse(haptic_hdev); + if (ret) { + hid_err(haptic_hdev, "hid parse failed\n"); + return ret; + } + + ret = hid_hw_start(haptic_hdev, 0); + if (ret) { + hid_err(haptic_hdev, "hw start failed\n"); + return ret; + } + + return 0; +} + +static bool actuator_match(struct hid_device *hdev, bool ignore_special_drivers) +{ + return (strcmp(hdev->name, "Apple MTP actuator") == 0); +} + +static void actuator_remove(struct hid_device *haptic_hdev) +{ + apple_taptic_switch_modes(haptic_hdev, false); + hid_hw_stop(haptic_hdev); +} + +static const struct hid_device_id apple_haptic_devices[] = { + { HID_DEVICE(BUS_HOST, HID_GROUP_ANY, HOST_VENDOR_ID_APPLE, + HID_ANY_ID), .driver_data = 0 }, + { } +}; +MODULE_DEVICE_TABLE(hid, apple_haptic_devices); + +static struct hid_driver apple_actuator_driver = { + .name = "hid-apple-mtp-haptic", + .id_table = apple_haptic_devices, + .probe = apple_actuator_probe, + .match = actuator_match, + .remove = actuator_remove +}; +module_hid_driver(apple_actuator_driver); + +MODULE_DESCRIPTION("Apple MTP Haptic Actuator driver"); +MODULE_LICENSE("GPL"); diff --git a/include/linux/soc/apple/actuator.h b/include/linux/soc/apple/actuator.h new file mode 100644 index 0000000000000..614f50d952424 --- /dev/null +++ b/include/linux/soc/apple/actuator.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#ifndef __APPLE_MTP_ACTUATOR_H__ +#define __APPLE_MTP_ACTUATOR_H__ +#include + +#define APPLE_HP_WAVEFORMDEEPCLICK 0x000e2001 + +#if IS_ENABLED(CONFIG_HID_APPLE_MTP_HAPTIC) +int apple_taptic_send(struct hid_device *hdev, u16 effect_type, u8 strength, u8 softness); +int apple_taptic_switch_modes(struct hid_device *hdev, bool currently_host_controlled); + +#else +static inline int apple_taptic_send(struct hid_device *hdev, u16 effect_type, + u8 strength, u8 softness) { return -ENODEV; } +static inline int apple_taptic_switch_modes(struct hid_device *hdev, + bool enable_host_controlled) { return -ENODEV; } +#endif +#endif /* __APPLE_MTP_ACTUATOR_H__ */ From 060912c2272b9f07317ea7c28bb9f5e345fcb9f9 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 21:22:29 +0200 Subject: [PATCH 0504/2680] HID: apple-mtp-haptic: bind the actuator on SPI machines The actuator on the SPI machines takes the same output report 0x53 and the same mode-switch feature report 0x21 as the MTP one, so the driver needs no protocol change to drive it; it simply never matched. Its id table covers BUS_HOST only, and its .match insists on the name "Apple MTP actuator", while spi-hid names the interface "Apple SPI Actuator". Nothing else claims the device either: hid-apple's id table matches every Apple SPI device, which makes hid_generic_match() stand aside, and hid-apple's probe then rejects anything that is not HID_TYPE_SPI_KEYBOARD. The actuator is left bound to no driver at all. Match BUS_SPI Apple devices as well and accept either name, via a helper in the actuator header so hid-magicmouse can use the same test. Start with HID_CONNECT_HIDRAW so the waveform parameters can be exercised from userspace: the strength and softness that suit this engine are not known yet, and the MTP defaults feel harsh on a J313. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-apple-mtp-haptic.c | 6 ++++-- include/linux/soc/apple/actuator.h | 28 ++++++++++++++++++++++++---- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/drivers/hid/hid-apple-mtp-haptic.c b/drivers/hid/hid-apple-mtp-haptic.c index e4dca10249df6..0678953acbb40 100644 --- a/drivers/hid/hid-apple-mtp-haptic.c +++ b/drivers/hid/hid-apple-mtp-haptic.c @@ -66,7 +66,7 @@ static int apple_actuator_probe(struct hid_device *haptic_hdev, const struct hid return ret; } - ret = hid_hw_start(haptic_hdev, 0); + ret = hid_hw_start(haptic_hdev, HID_CONNECT_HIDRAW); if (ret) { hid_err(haptic_hdev, "hw start failed\n"); return ret; @@ -77,7 +77,7 @@ static int apple_actuator_probe(struct hid_device *haptic_hdev, const struct hid static bool actuator_match(struct hid_device *hdev, bool ignore_special_drivers) { - return (strcmp(hdev->name, "Apple MTP actuator") == 0); + return apple_taptic_is_actuator(hdev); } static void actuator_remove(struct hid_device *haptic_hdev) @@ -89,6 +89,8 @@ static void actuator_remove(struct hid_device *haptic_hdev) static const struct hid_device_id apple_haptic_devices[] = { { HID_DEVICE(BUS_HOST, HID_GROUP_ANY, HOST_VENDOR_ID_APPLE, HID_ANY_ID), .driver_data = 0 }, + { HID_DEVICE(BUS_SPI, HID_GROUP_ANY, SPI_VENDOR_ID_APPLE, + HID_ANY_ID), .driver_data = 0 }, { } }; MODULE_DEVICE_TABLE(hid, apple_haptic_devices); diff --git a/include/linux/soc/apple/actuator.h b/include/linux/soc/apple/actuator.h index 614f50d952424..0b565d5567eb0 100644 --- a/include/linux/soc/apple/actuator.h +++ b/include/linux/soc/apple/actuator.h @@ -2,18 +2,38 @@ #ifndef __APPLE_MTP_ACTUATOR_H__ #define __APPLE_MTP_ACTUATOR_H__ + #include +#include +#include #define APPLE_HP_WAVEFORMDEEPCLICK 0x000e2001 -#if IS_ENABLED(CONFIG_HID_APPLE_MTP_HAPTIC) +/* + * The actuator is a HID device of its own, named by the transport: "Apple + * MTP actuator" on the MTP machines, "Apple SPI Actuator" on the SPI ones. + */ +static inline bool apple_taptic_is_actuator(struct hid_device *hdev) +{ + return !strcmp(hdev->name, "Apple MTP actuator") || + !strcmp(hdev->name, "Apple SPI Actuator"); +} + +#if IS_REACHABLE(CONFIG_HID_APPLE_MTP_HAPTIC) int apple_taptic_send(struct hid_device *hdev, u16 effect_type, u8 strength, u8 softness); -int apple_taptic_switch_modes(struct hid_device *hdev, bool currently_host_controlled); +int apple_taptic_switch_modes(struct hid_device *hdev, bool host_controlled); #else static inline int apple_taptic_send(struct hid_device *hdev, u16 effect_type, - u8 strength, u8 softness) { return -ENODEV; } + u8 strength, u8 softness) +{ + return -ENODEV; +} + static inline int apple_taptic_switch_modes(struct hid_device *hdev, - bool enable_host_controlled) { return -ENODEV; } + bool host_controlled) +{ + return -ENODEV; +} #endif #endif /* __APPLE_MTP_ACTUATOR_H__ */ From 29a992069c00b58e61006212c5ecda3c46e74765 Mon Sep 17 00:00:00 2001 From: Yicong Hui Date: Fri, 21 Aug 2026 21:23:49 +0200 Subject: [PATCH 0505/2680] HID: magicmouse: add support for controlling MTP touchpad haptics Integrate the MTP actuator driver with the MTP trackpad driver to expose new functionality that allows userspace to control the haptic effects of the touchpad actuator as a force feedback device, following the convention for haptic touchpad functionality as specified in [1]. Set the INPUT_PROP_PRESSUREPAD property within the MTP trackpad device indicating that this device is a haptic touchpad, and set the FF_HAPTIC capability to indicate that force feedback haptic effects can be uploaded to the trackpad [2]. Userspace can thus choose to upload force feedback effects to the touchpad, which would trigger it to switch into "Host controlled" mode, where the touchpad will no longer click unless signalled to do so by userspace. This patch uses an unconventional means of doing so, by writing a separate loadable module for the haptic actuator. This is because the haptic actuator is considered a separate device on MTP trackpads, whereas haptics for HUTRR63 (and the other kernel drivers) considers the haptic actuator and the touchpad to be the very same device. Therefore, to conform to the standardized interface in [1] this patch creates a separate driver for the separate HID device, and is designed such that commands to the actuator are sent through the touchpad. This changeset is relevant because allowing userspace control of the haptics allows for user control of: 1. Click feedback intensity 2. Sensitivity of when clicks trigger 3. Configuring "Force Touch"-like deep click functionality The actuator is a sibling HID device that the transport can register after the trackpad, so it is resolved lazily on the first effect upload. The force feedback device is created before the input device is registered and destroyed explicitly before the driver state goes away, and the whole integration is compiled out when the actuator driver is not reachable, so this driver does not depend on it. [1] https://lore.kernel.org/all/20250818-support-forcepads-v3-0-e4f9ab0add84@google.com/T/#m12f9be6691015fb9b2e834248c368e5f90 [2] https://docs.kernel.org/input/event-codes.html#input-prop-pressurepad Signed-off-by: Yicong Hui Co-developed-by: Chromatischer Signed-off-by: Chromatischer --- drivers/hid/hid-magicmouse.c | 347 ++++++++++++++++++++++++++++++++++- 1 file changed, 346 insertions(+), 1 deletion(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index d6c43cc967e3b..35fbe163e0beb 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -19,7 +19,9 @@ #include #include #include +#include +#include "hid-haptic.h" #include "hid-ids.h" static bool emulate_3button = true; @@ -140,6 +142,14 @@ struct magicmouse_input_ops { int (*setup_input)(struct input_dev *input, struct hid_device *hdev); }; +struct effect_job { + struct work_struct work; + struct hid_device *taptic_hdev; + u16 effect_type; + u8 strength; + u8 softness; +}; + /** * struct magicmouse_sc - Tracks Magic Mouse-specific data. * @input: Input device through which we report events. @@ -152,6 +162,9 @@ struct magicmouse_input_ops { * @pos: multi touch position data of the last report. * @touches: Most recent data for a touch, indexed by tracking ID. * @tracking_ids: Mapping of current touch input data to @touches. + * @haptics: Haptic device state of an MTP trackpad, NULL otherwise. + * @taptic_hdev: Actuator HID device of an MTP trackpad, if resolved. + * @haptic_effects: Uploaded haptic effects of an MTP trackpad. * @hdev: Pointer to the underlying HID device. * @work: Workqueue to handle initialization retry for quirky devices. * @battery_timer: Timer for obtaining battery level information. @@ -179,6 +192,10 @@ struct magicmouse_sc { } touches[MAX_CONTACTS]; int tracking_ids[MAX_CONTACTS]; + struct hid_haptic_device *haptics; + struct hid_device *taptic_hdev; + struct effect_job *haptic_effects; + struct hid_device *hdev; struct delayed_work work; struct timer_list battery_timer; @@ -1127,6 +1144,7 @@ static int magicmouse_setup_input_mtp(struct input_dev *input, struct magicmouse_sc *msc = hid_get_drvdata(hdev); __set_bit(INPUT_PROP_BUTTONPAD, input->propbit); + __set_bit(INPUT_PROP_PRESSUREPAD, input->propbit); __clear_bit(BTN_0, input->keybit); __clear_bit(BTN_RIGHT, input->keybit); __clear_bit(BTN_MIDDLE, input->keybit); @@ -1134,7 +1152,7 @@ static int magicmouse_setup_input_mtp(struct input_dev *input, __clear_bit(REL_X, input->relbit); __clear_bit(REL_Y, input->relbit); - mt_flags = INPUT_MT_POINTER | INPUT_MT_DROP_UNUSED | INPUT_MT_TRACK; + mt_flags = INPUT_MT_POINTER | INPUT_MT_DROP_UNUSED | INPUT_MT_TRACK | INPUT_MT_TOTAL_FORCE; /* * The C1FE report carries positions and the button only; advertise @@ -1254,6 +1272,57 @@ static int magicmouse_input_mapping(struct hid_device *hdev, return 0; } +static int magicmouse_init_haptics(struct magicmouse_sc *msc, + struct hid_device *hdev); + +#if IS_REACHABLE(CONFIG_HID_APPLE_MTP_HAPTIC) +static int match_actuator(struct device *dev, const void *data) +{ + struct hid_device *hdev; + + if (dev->bus != &hid_bus_type) + return 0; + + hdev = to_hid_device(dev); + + return !strcmp(hdev->name, "Apple MTP actuator"); +} + +static struct hid_device *magicmouse_get_actuator(struct magicmouse_sc *msc) +{ + struct device *dev; + + if (msc->taptic_hdev) + return msc->taptic_hdev; + + if (!msc->hdev->dev.parent) + return NULL; + + dev = device_find_child(msc->hdev->dev.parent, NULL, match_actuator); + if (!dev) + return NULL; + + msc->taptic_hdev = to_hid_device(dev); + hid_dbg(msc->hdev, "bound to the MTP actuator\n"); + + return msc->taptic_hdev; +} + +static int magicmouse_switch_mode(struct input_dev *trackpad_idev, int mode) +{ + struct hid_device *trackpad_hdev = input_get_drvdata(trackpad_idev); + struct magicmouse_sc *msc = hid_get_drvdata(trackpad_hdev); + struct hid_device *actuator = magicmouse_get_actuator(msc); + + if (!actuator) + return -ENODEV; + + hid_dbg(trackpad_hdev, "switching haptics to mode %d\n", mode); + + return apple_taptic_switch_modes(actuator, mode); +} +#endif + static int magicmouse_input_configured(struct hid_device *hdev, struct hid_input *hi) @@ -1274,6 +1343,16 @@ static int magicmouse_input_configured(struct hid_device *hdev, return ret; } + /* + * This runs before the input device is registered, and the force + * feedback device has to exist by then for evdev clients to see + * FF_HAPTIC. + */ + ret = magicmouse_init_haptics(msc, hdev); + if (ret) + hid_warn(hdev, "haptics unavailable (%d), continuing without force feedback\n", + ret); + return 0; } @@ -1399,6 +1478,262 @@ static void magicmouse_battery_timer_tick(struct timer_list *t) } } +#if IS_REACHABLE(CONFIG_HID_APPLE_MTP_HAPTIC) +static int apple_upload_effects(struct input_dev *trackpad_idev, + struct ff_effect *effect, struct ff_effect *old) +{ + struct hid_device *trackpad_hdev = input_get_drvdata(trackpad_idev); + struct magicmouse_sc *msc = hid_get_drvdata(trackpad_hdev); + struct hid_haptic_device *haptics = msc->haptics; + struct hid_device *actuator; + int ret; + + switch (effect->u.haptic.hid_usage) { + case (HID_HP_WAVEFORMPRESS & HID_USAGE): + case (HID_HP_WAVEFORMRELEASE & HID_USAGE): + case (APPLE_HP_WAVEFORMDEEPCLICK & HID_USAGE): + break; + default: + return -EINVAL; + } + + actuator = magicmouse_get_actuator(msc); + if (!actuator) + return -ENODEV; + + msc->haptic_effects[effect->id].taptic_hdev = actuator; + msc->haptic_effects[effect->id].effect_type = (effect->u.haptic.hid_usage) & HID_USAGE; + msc->haptic_effects[effect->id].strength = + (min(100, (effect->u.haptic.intensity)) * 255) / 100; + msc->haptic_effects[effect->id].softness = 0x90; + /* A future extension could add configurability to softness. */ + + /* The actuator may have been rebound or reset since the last upload. */ + ret = magicmouse_switch_mode(trackpad_idev, HID_HAPTIC_MODE_HOST); + if (ret) { + dev_err(&msc->hdev->dev, + "failed to switch to host-controlled mode: %d\n", ret); + msc->haptic_effects[effect->id].effect_type = 0; + return ret; + } + + haptics->mode = HID_HAPTIC_MODE_HOST; + + hid_dbg(trackpad_hdev, "uploaded haptic effect %d\n", effect->id); + return 0; +} + +static void haptic_playback_worker(struct work_struct *ws) +{ + struct effect_job *job = container_of(ws, struct effect_job, work); + + if (job->effect_type && + apple_taptic_send(job->taptic_hdev, job->effect_type, + job->strength, job->softness)) + hid_warn(job->taptic_hdev, "cannot play the haptic effect\n"); +} + +static int apple_taptic_playback(struct input_dev *trackpad_idev, int effect_id, int value) +{ + struct hid_device *trackpad_hdev = input_get_drvdata(trackpad_idev); + struct magicmouse_sc *msc = hid_get_drvdata(trackpad_hdev); + + if (value) + queue_work(msc->haptics->wq, &msc->haptic_effects[effect_id].work); + + return 0; +} + +static int apple_taptic_erase(struct input_dev *trackpad_idev, int effect_id) +{ + struct hid_device *trackpad_hdev = input_get_drvdata(trackpad_idev); + struct magicmouse_sc *msc = hid_get_drvdata(trackpad_hdev); + int i, ret = 0; + + msc->haptic_effects[effect_id].effect_type = 0; + + for (i = 0; i < FF_MAX_EFFECTS; i++) { + if (msc->haptic_effects[i].effect_type != 0) + return 0; + } + + /* Return to device-controlled mode if there are no effects left */ + if (msc->haptics->mode == HID_HAPTIC_MODE_HOST) { + flush_workqueue(msc->haptics->wq); + ret = magicmouse_switch_mode(trackpad_idev, HID_HAPTIC_MODE_DEVICE); + + if (ret) { + dev_err(&msc->hdev->dev, + "failed to switch back to device-controlled mode: %d\n", + ret); + return ret; + } + + msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; + } + + return 0; +} + +static void apple_taptic_destroy(struct ff_device *ff) +{ + struct hid_haptic_device *haptic_dev = ff->private; + struct magicmouse_sc *msc = hid_get_drvdata(haptic_dev->hdev); + int ret; + + /* ff->private is devm-allocated; do not let ff-core kfree() it. */ + ff->private = NULL; + + if (msc->haptics && msc->haptics->mode == HID_HAPTIC_MODE_HOST) { + flush_workqueue(msc->haptics->wq); + ret = magicmouse_switch_mode(msc->haptics->input_dev, HID_HAPTIC_MODE_DEVICE); + if (ret) + hid_warn(msc->hdev, + "failed to switch back to device-controlled mode: %d\n", + ret); + else + msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; + } + + destroy_workqueue(haptic_dev->wq); + haptic_dev->wq = NULL; + + kfree(msc->haptic_effects); + msc->haptic_effects = NULL; +} + +static int apple_taptic_init_mtp(struct magicmouse_sc *msc, + struct hid_haptic_device *haptic_dev) +{ + struct hid_device *trackpad_hdev = msc->hdev; + struct ff_device *ff; + int ret, i; + + haptic_dev->hdev = trackpad_hdev; + haptic_dev->input_dev = msc->input; + + msc->haptic_effects = kzalloc_objs(struct effect_job, FF_MAX_EFFECTS); + if (!msc->haptic_effects) { + dev_err(&trackpad_hdev->dev, "Cannot allocate haptic effects\n"); + return -ENOMEM; + } + + haptic_dev->wq = create_singlethread_workqueue("Apple trackpad haptics workqueue"); + if (!haptic_dev->wq) { + dev_err(&trackpad_hdev->dev, "Cannot allocate haptic workqueue\n"); + kfree(msc->haptic_effects); + msc->haptic_effects = NULL; + return -ENOMEM; + } + + for (i = 0; i < FF_MAX_EFFECTS; i++) + INIT_WORK(&msc->haptic_effects[i].work, haptic_playback_worker); + + /* FF_HAPTIC has to be in ffbit before the FF device is created. */ + input_set_capability(msc->input, EV_FF, FF_HAPTIC); + + ret = input_ff_create(msc->input, FF_MAX_EFFECTS); + if (ret) { + /* undo the capability set above, no FF device was created */ + __clear_bit(EV_FF, msc->input->evbit); + __clear_bit(FF_HAPTIC, msc->input->ffbit); + + kfree(msc->haptic_effects); + msc->haptic_effects = NULL; + + destroy_workqueue(haptic_dev->wq); + haptic_dev->wq = NULL; + + dev_err(&trackpad_hdev->dev, "Failed to create force-feedback device.\n"); + return ret; + } + + ff = msc->input->ff; + ff->private = haptic_dev; + ff->upload = apple_upload_effects; + ff->playback = apple_taptic_playback; + ff->erase = apple_taptic_erase; + ff->destroy = apple_taptic_destroy; + + /* + * Hold the input device until magicmouse_teardown_haptics(), which + * destroys the force feedback device while msc is still valid. + */ + input_get_device(msc->input); + + hid_info(trackpad_hdev, "initialised MTP haptics\n"); + return 0; +} + +static int magicmouse_init_haptics(struct magicmouse_sc *msc, struct hid_device *trackpad_hdev) +{ + struct hid_haptic_device *haptics; + int ret; + + if (msc->haptics) + return 0; + + /* + * Only the internal trackpads have an actuator. It is a separate HID + * device that may be registered after the trackpad, so it is resolved + * lazily when the first effect is uploaded. + */ + if (trackpad_hdev->bus != BUS_HOST || + trackpad_hdev->type != HID_TYPE_SPI_MOUSE) + return 0; + + haptics = devm_kzalloc(&trackpad_hdev->dev, sizeof(*haptics), GFP_KERNEL); + if (!haptics) + return -ENOMEM; + + haptics->hdev = trackpad_hdev; + + ret = apple_taptic_init_mtp(msc, haptics); + if (ret) { + devm_kfree(&trackpad_hdev->dev, haptics); + return ret; + } + + msc->haptics = haptics; + return 0; +} + +static void magicmouse_teardown_haptics(struct magicmouse_sc *msc) +{ + if (!msc->haptics) + return; + + /* + * Tear the force feedback device down while msc is still valid: an + * open evdev fd would otherwise run it from input_dev_release() after + * devres has freed the driver state. + */ + if (msc->haptics->input_dev) { + input_ff_destroy(msc->haptics->input_dev); + input_put_device(msc->haptics->input_dev); + msc->haptics->input_dev = NULL; + } + + msc->haptics = NULL; + msc->haptic_effects = NULL; + + if (msc->taptic_hdev) { + put_device(&msc->taptic_hdev->dev); + msc->taptic_hdev = NULL; + } +} +#else +static int magicmouse_init_haptics(struct magicmouse_sc *msc, + struct hid_device *hdev) +{ + return 0; +} + +static void magicmouse_teardown_haptics(struct magicmouse_sc *msc) +{ +} +#endif + static int magicmouse_probe(struct hid_device *hdev, const struct hid_device_id *id) { @@ -1452,6 +1787,11 @@ static int magicmouse_probe(struct hid_device *hdev, ret = hid_hw_start(hdev, HID_CONNECT_DEFAULT); if (ret) { hid_err(hdev, "magicmouse hw start failed\n"); + /* + * hid_connect() cannot fail once hidinput_connect() has + * succeeded, so no input device is registered here. + */ + magicmouse_teardown_haptics(msc); return ret; } @@ -1554,6 +1894,8 @@ static int magicmouse_probe(struct hid_device *hdev, timer_delete_sync(&msc->battery_timer); hid_hw_stop(hdev); + magicmouse_teardown_haptics(msc); + return ret; } @@ -1569,6 +1911,9 @@ static void magicmouse_remove(struct hid_device *hdev) } hid_hw_stop(hdev); + + if (msc) + magicmouse_teardown_haptics(msc); } #ifdef CONFIG_PM From 0a18b11b3e6ce6b9af2edb21547e2af4dd53a2b6 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 21:22:29 +0200 Subject: [PATCH 0506/2680] HID: magicmouse: allow haptics on SPI trackpads magicmouse_init_haptics() requires BUS_HOST, and match_actuator() looks for the MTP actuator by name, so the SPI trackpads were left without haptics even though their machines have an actuator that speaks the same protocol. Accept BUS_SPI as well and use the shared name test. The lookup itself needs no change: the actuator is a sibling HID device under the same SPI parent, which is what magicmouse_get_actuator() already searches. This does not change how a click feels by default. The actuator stays in device-controlled mode, where its firmware provides the click feedback, and only moves to host-controlled mode when userspace uploads an effect. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-magicmouse.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 35fbe163e0beb..556be8b1bb0e8 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -1285,7 +1285,7 @@ static int match_actuator(struct device *dev, const void *data) hdev = to_hid_device(dev); - return !strcmp(hdev->name, "Apple MTP actuator"); + return apple_taptic_is_actuator(hdev); } static struct hid_device *magicmouse_get_actuator(struct magicmouse_sc *msc) @@ -1674,11 +1674,12 @@ static int magicmouse_init_haptics(struct magicmouse_sc *msc, struct hid_device return 0; /* - * Only the internal trackpads have an actuator. It is a separate HID - * device that may be registered after the trackpad, so it is resolved - * lazily when the first effect is uploaded. + * Only the internal trackpads have an actuator: the MTP ones on + * BUS_HOST and the SPI ones, whose firmware exposes it as a separate + * "Actuator" interface. It may be registered after the trackpad, so + * it is resolved lazily on the first effect upload. */ - if (trackpad_hdev->bus != BUS_HOST || + if ((trackpad_hdev->bus != BUS_HOST && trackpad_hdev->bus != BUS_SPI) || trackpad_hdev->type != HID_TYPE_SPI_MOUSE) return 0; From b873a97e4f03a71af677a54fe20c517c62b8a3a4 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 21:42:42 +0200 Subject: [PATCH 0507/2680] HID: magicmouse: make the haptic waveform shape configurable The waveform sent to the actuator carries a strength and a softness. Both were fixed: softness at 0x90, and strength at the full 255 scaled by the effect's intensity. Those suit the MTP actuator, but on a J313 the same numbers produce a hard knock rather than a click. Judged by feel there, a press wants roughly strength 0x28 at softness 0xf0, far weaker than the MTP default, while a deep click wants full strength 0xff at a softness of 0xd0. Add four parameters for the two waveforms, each defaulting per transport so the MTP path keeps the values its firmware was tuned with. Strength now names what 100% intensity maps to, so an effect asking for half intensity still gets half of it. They are writable at runtime and take effect on the next effect upload, which is what makes tuning by feel practical: the values that suit a given actuator cannot be derived, only tried. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-magicmouse.c | 62 ++++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 556be8b1bb0e8..7727cd87668d9 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -55,6 +55,27 @@ static bool report_undeciphered; module_param(report_undeciphered, bool, 0644); MODULE_PARM_DESC(report_undeciphered, "Report undeciphered multi-touch state field using a MSC_RAW event"); +/* + * Waveform shaping, writable at runtime and applied on the next effect + * upload. Zero means "use the per-transport default": the MTP machines keep + * the values their firmware was tuned with, the SPI actuator in a J313 needs + * a much softer, weaker waveform. The SPI numbers were judged by feel there. + */ +static u8 haptic_softness; +module_param(haptic_softness, byte, 0644); +MODULE_PARM_DESC(haptic_softness, "softness of the press and release waveforms, 1-255 (0: 0xf0 on SPI, 0x90 on MTP)"); + +static u8 haptic_deep_softness; +module_param(haptic_deep_softness, byte, 0644); +MODULE_PARM_DESC(haptic_deep_softness, "softness of the deep-click waveform, 1-255 (0: 0xd0 on SPI, 0x90 on MTP)"); + +static u8 haptic_strength; +module_param(haptic_strength, byte, 0644); +MODULE_PARM_DESC(haptic_strength, "strength at 100% intensity for press and release, 1-255 (0: 0x28 on SPI, 0xff on MTP)"); + +static u8 haptic_deep_strength; +module_param(haptic_deep_strength, byte, 0644); +MODULE_PARM_DESC(haptic_deep_strength, "strength at 100% intensity for a deep click, 1-255 (0: 0xff on SPI, 0xff on MTP)"); #define TRACKPAD2_2021_BT_VERSION 0x110 #define TRACKPAD_2024_BT_VERSION 0x314 @@ -1479,6 +1500,40 @@ static void magicmouse_battery_timer_tick(struct timer_list *t) } #if IS_REACHABLE(CONFIG_HID_APPLE_MTP_HAPTIC) +static bool magicmouse_effect_is_deep(const struct ff_effect *effect) +{ + return effect->u.haptic.hid_usage == + (APPLE_HP_WAVEFORMDEEPCLICK & HID_USAGE); +} + +static u8 magicmouse_haptic_softness(struct hid_device *hdev, + const struct ff_effect *effect) +{ + bool deep = magicmouse_effect_is_deep(effect); + u8 configured = deep ? haptic_deep_softness : haptic_softness; + + if (configured) + return configured; + if (hdev->bus != BUS_SPI) + return 0x90; + + return deep ? 0xd0 : 0xf0; +} + +static u8 magicmouse_haptic_strength(struct hid_device *hdev, + const struct ff_effect *effect) +{ + bool deep = magicmouse_effect_is_deep(effect); + u8 configured = deep ? haptic_deep_strength : haptic_strength; + + if (configured) + return configured; + if (hdev->bus != BUS_SPI) + return 0xff; + + return deep ? 0xff : 0x28; +} + static int apple_upload_effects(struct input_dev *trackpad_idev, struct ff_effect *effect, struct ff_effect *old) { @@ -1504,9 +1559,10 @@ static int apple_upload_effects(struct input_dev *trackpad_idev, msc->haptic_effects[effect->id].taptic_hdev = actuator; msc->haptic_effects[effect->id].effect_type = (effect->u.haptic.hid_usage) & HID_USAGE; msc->haptic_effects[effect->id].strength = - (min(100, (effect->u.haptic.intensity)) * 255) / 100; - msc->haptic_effects[effect->id].softness = 0x90; - /* A future extension could add configurability to softness. */ + (min(100, (effect->u.haptic.intensity)) * + magicmouse_haptic_strength(trackpad_hdev, effect)) / 100; + msc->haptic_effects[effect->id].softness = + magicmouse_haptic_softness(trackpad_hdev, effect); /* The actuator may have been rebound or reset since the last upload. */ ret = magicmouse_switch_mode(trackpad_idev, HID_HAPTIC_MODE_HOST); From 84460497ded2a42411d8c2d7c6809ce8fbe93135 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 22:13:13 +0200 Subject: [PATCH 0508/2680] HID: magicmouse: drive click feedback from the host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The actuator's firmware provides the feedback for a physical press, and how that press feels cannot be changed while it does. Playing the waveform from the host instead makes it adjustable, but it cannot be done by reacting to the button the trackpad reports: in host-controlled mode the firmware stops detecting clicks altogether. It keeps reporting fingers and never sets the button again, because deciding that a press happened belongs to whoever owns the actuator. Hooking the button bit therefore works exactly once — for the press already in flight when the mode changed — and then the pad has no click feedback and no button at all. So derive the click here. Sum the pressure of the fingers in each frame, call a press when it rises past haptic_press_threshold and a release when it falls below haptic_release_threshold, report BTN_MOUSE from that, and play the waveform from the haptics workqueue, which is where it can sleep. The thresholds default to 120 and 75, measured on a J313 against its own firmware: it asserts the button between 130 and 243 and releases between 26 and 73. The gap between the two is what stops a click chattering. This is off by default. Turning it off returns the actuator to the firmware on the next frame, and any failure while playing does the same, so a pad that cannot play a waveform falls back to working feedback rather than none. What the parameters do on an SPI actuator is narrower than the report layout suggests. The fifteen waveform bytes are ignored: zeroing them feels identical. Strength is close to on or off, with 0x00 silent and the values above it differing little. Softness is what shapes the pulse; swept from 0x10 to 0xff at a fixed strength it runs from a faint tick to a hard knock. A click that wants more weight than softness gives is made of more pulses, which is what haptic_press_pulses is for. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer [Satya Benson: rebased onto J700 C1FE support; C1FE reports carry no pressure, so those pads keep reporting the firmware's button] Assisted-by: Claude:claude-opus-5-5 --- drivers/hid/hid-magicmouse.c | 238 +++++++++++++++++++++++++++++++++-- 1 file changed, 227 insertions(+), 11 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 7727cd87668d9..b7049a888e023 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -11,6 +11,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -76,6 +77,42 @@ MODULE_PARM_DESC(haptic_strength, "strength at 100% intensity for press and rele static u8 haptic_deep_strength; module_param(haptic_deep_strength, byte, 0644); MODULE_PARM_DESC(haptic_deep_strength, "strength at 100% intensity for a deep click, 1-255 (0: 0xff on SPI, 0xff on MTP)"); + +/* + * Host-driven click feedback: hold the actuator in host-controlled mode and + * play the pulses from here, which is the only way to change how a press + * feels. With this off, the firmware provides the feedback itself. + */ +static bool haptic_press; +module_param(haptic_press, bool, 0644); +MODULE_PARM_DESC(haptic_press, "play click feedback from the host on physical press (default off: the actuator's firmware does it)"); + +static u8 haptic_press_pulses = 1; +module_param(haptic_press_pulses, byte, 0644); +MODULE_PARM_DESC(haptic_press_pulses, "pulses played on press (default 1)"); + +static u8 haptic_release_pulses = 1; +module_param(haptic_release_pulses, byte, 0644); +MODULE_PARM_DESC(haptic_release_pulses, "release waveforms played on release, minimum 1 (default 1)"); + +/* + * Click thresholds, in the units the trackpad reports per finger summed over + * the fingers in a frame. On a J313 the firmware asserts the button between + * 130 and 243 and releases between 26 and 73; these sit just inside that, with + * enough hysteresis that a click cannot chatter. + */ +static u16 haptic_press_threshold = 120; +module_param(haptic_press_threshold, ushort, 0644); +MODULE_PARM_DESC(haptic_press_threshold, "summed finger pressure at which a click starts (default 120)"); + +static u16 haptic_release_threshold = 75; +module_param(haptic_release_threshold, ushort, 0644); +MODULE_PARM_DESC(haptic_release_threshold, "summed finger pressure at which a click ends (default 75)"); + +static u8 haptic_pulse_gap_ms = 25; +module_param(haptic_pulse_gap_ms, byte, 0644); +MODULE_PARM_DESC(haptic_pulse_gap_ms, "milliseconds between pulses when more than one (default 25)"); + #define TRACKPAD2_2021_BT_VERSION 0x110 #define TRACKPAD_2024_BT_VERSION 0x314 @@ -216,6 +253,9 @@ struct magicmouse_sc { struct hid_haptic_device *haptics; struct hid_device *taptic_hdev; struct effect_job *haptic_effects; + struct work_struct haptic_press_work; + struct work_struct haptic_release_work; + bool haptic_button_down; struct hid_device *hdev; struct delayed_work work; @@ -839,6 +879,9 @@ static void report_c1fe_finger_data(struct input_dev *input, int slot, input_report_abs(input, ABS_MT_POSITION_Y, pos->y); } +static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, + bool firmware_button); + static void report_finger_data(struct input_dev *input, int slot, const struct input_mt_pos *pos, const struct tp_finger *f) @@ -868,6 +911,7 @@ static int magicmouse_raw_event_mtp(struct hid_device *hdev, struct input_dev *input = msc->input; struct tp_header *tp_hdr; struct tp_finger *f; + u32 pressure = 0; int i, n; u32 npoints; size_t hdr_sz = sizeof(struct tp_header); @@ -919,14 +963,20 @@ static int magicmouse_raw_event_mtp(struct hid_device *hdev, for (i = 0; i < n; i++) { int idx = map_contacs[i]; f = (struct tp_finger *)(data + hdr_sz + idx * touch_sz); - if (msc->mtp_c1fe) + if (msc->mtp_c1fe) { report_c1fe_finger_data(input, msc->tracking_ids[i], &msc->pos[i]); - else + } else { report_finger_data(input, msc->tracking_ids[i], &msc->pos[i], f); + pressure += le16_to_int(f->pressure); + } } input_mt_sync_frame(input); - input_report_key(input, BTN_MOUSE, tp_hdr->buttons & 1); + /* The C1FE report carries no pressure to derive a click from. */ + input_report_key(input, BTN_MOUSE, + msc->mtp_c1fe ? tp_hdr->buttons & 1 : + magicmouse_haptic_click(msc, pressure, + tp_hdr->buttons & 1)); input_sync(input); return 1; @@ -1342,6 +1392,168 @@ static int magicmouse_switch_mode(struct input_dev *trackpad_idev, int mode) return apple_taptic_switch_modes(actuator, mode); } + +static u8 magicmouse_haptic_softness(struct hid_device *hdev, bool deep); +static u8 magicmouse_haptic_strength(struct hid_device *hdev, bool deep); +static void magicmouse_haptic_release_mode(struct magicmouse_sc *msc); + +/* + * Play the waveform @pulses times, @haptic_pulse_gap_ms apart. A failure here + * would leave host-controlled mode without any click feedback, so hand the + * actuator back to the firmware and turn the feature off instead. + */ +static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, + u8 pulses) +{ + struct hid_device *actuator = magicmouse_get_actuator(msc); + u8 strength, softness; + unsigned int i; + int ret; + + if (!pulses) + return; + + if (!actuator) { + hid_warn(msc->hdev, + "no actuator for host click feedback; using the firmware's\n"); + haptic_press = false; + return; + } + + if (msc->haptics->mode != HID_HAPTIC_MODE_HOST) { + ret = apple_taptic_switch_modes(actuator, HID_HAPTIC_MODE_HOST); + if (ret) { + hid_warn(msc->hdev, + "cannot take host-controlled mode (%d); using the firmware's\n", + ret); + haptic_press = false; + return; + } + msc->haptics->mode = HID_HAPTIC_MODE_HOST; + hid_info(msc->hdev, "host-driven click feedback active\n"); + } + + strength = magicmouse_haptic_strength(msc->hdev, false); + softness = magicmouse_haptic_softness(msc->hdev, false); + + for (i = 0; i < pulses; i++) { + ret = apple_taptic_send(actuator, usage, strength, softness); + if (ret) { + hid_warn(msc->hdev, + "click waveform failed (%d); returning click feedback to the firmware\n", + ret); + haptic_press = false; + magicmouse_haptic_release_mode(msc); + return; + } + hid_dbg(msc->hdev, "played %s pulse %u/%u at strength 0x%02x\n", + usage == (HID_HP_WAVEFORMRELEASE & HID_USAGE) ? + "release" : "press", + i + 1, pulses, strength); + if (i + 1 < pulses) + msleep(haptic_pulse_gap_ms); + } +} + +/* + * Hand click feedback back to the actuator's firmware, unless userspace still + * has effects uploaded and so still wants host-controlled mode. + */ +static void magicmouse_haptic_release_mode(struct magicmouse_sc *msc) +{ + struct hid_device *actuator; + unsigned int i; + + if (msc->haptics->mode != HID_HAPTIC_MODE_HOST) + return; + + if (msc->haptic_effects) { + for (i = 0; i < FF_MAX_EFFECTS; i++) + if (msc->haptic_effects[i].effect_type) + return; + } + + actuator = magicmouse_get_actuator(msc); + if (actuator && !apple_taptic_switch_modes(actuator, + HID_HAPTIC_MODE_DEVICE)) + msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; +} + +static void magicmouse_press_worker(struct work_struct *ws) +{ + struct magicmouse_sc *msc = + container_of(ws, struct magicmouse_sc, haptic_press_work); + + if (!haptic_press) { + magicmouse_haptic_release_mode(msc); + return; + } + + magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMPRESS & HID_USAGE, + haptic_press_pulses); +} + +static void magicmouse_release_worker(struct work_struct *ws) +{ + struct magicmouse_sc *msc = + container_of(ws, struct magicmouse_sc, haptic_release_work); + + if (!haptic_press) { + magicmouse_haptic_release_mode(msc); + return; + } + + /* + * The bare header tells the firmware that the click gesture is over, so + * send it even when no extra pulses were asked for. + */ + magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMRELEASE & HID_USAGE, + max_t(u8, haptic_release_pulses, 1)); +} + +/* + * Decide whether the pad is clicked from the summed finger pressure, and + * queue the feedback for it: in host-controlled mode the firmware no longer + * sets the button itself. The wide gap between the press and release + * thresholds keeps a single click from chattering. + * + * Returns the button state to report. Playing the waveform sleeps, so it is + * left to the workqueue. + */ +static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, + bool firmware_button) +{ + if (!msc->haptics || !msc->haptics->wq) + return firmware_button; + + if (!haptic_press) { + /* Hand the actuator back and report its button again. */ + if (msc->haptics->mode == HID_HAPTIC_MODE_HOST) { + msc->haptic_button_down = false; + queue_work(msc->haptics->wq, &msc->haptic_release_work); + } + return firmware_button; + } + + if (!msc->haptic_button_down && pressure >= haptic_press_threshold) { + msc->haptic_button_down = true; + hid_dbg(msc->hdev, "click at pressure %u\n", pressure); + queue_work(msc->haptics->wq, &msc->haptic_press_work); + } else if (msc->haptic_button_down && + pressure <= haptic_release_threshold) { + msc->haptic_button_down = false; + hid_dbg(msc->hdev, "release at pressure %u\n", pressure); + queue_work(msc->haptics->wq, &msc->haptic_release_work); + } + + return msc->haptic_button_down; +} +#else +static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, + bool firmware_button) +{ + return firmware_button; +} #endif static int magicmouse_input_configured(struct hid_device *hdev, @@ -1506,10 +1718,8 @@ static bool magicmouse_effect_is_deep(const struct ff_effect *effect) (APPLE_HP_WAVEFORMDEEPCLICK & HID_USAGE); } -static u8 magicmouse_haptic_softness(struct hid_device *hdev, - const struct ff_effect *effect) +static u8 magicmouse_haptic_softness(struct hid_device *hdev, bool deep) { - bool deep = magicmouse_effect_is_deep(effect); u8 configured = deep ? haptic_deep_softness : haptic_softness; if (configured) @@ -1520,10 +1730,8 @@ static u8 magicmouse_haptic_softness(struct hid_device *hdev, return deep ? 0xd0 : 0xf0; } -static u8 magicmouse_haptic_strength(struct hid_device *hdev, - const struct ff_effect *effect) +static u8 magicmouse_haptic_strength(struct hid_device *hdev, bool deep) { - bool deep = magicmouse_effect_is_deep(effect); u8 configured = deep ? haptic_deep_strength : haptic_strength; if (configured) @@ -1560,9 +1768,11 @@ static int apple_upload_effects(struct input_dev *trackpad_idev, msc->haptic_effects[effect->id].effect_type = (effect->u.haptic.hid_usage) & HID_USAGE; msc->haptic_effects[effect->id].strength = (min(100, (effect->u.haptic.intensity)) * - magicmouse_haptic_strength(trackpad_hdev, effect)) / 100; + magicmouse_haptic_strength(trackpad_hdev, + magicmouse_effect_is_deep(effect))) / 100; msc->haptic_effects[effect->id].softness = - magicmouse_haptic_softness(trackpad_hdev, effect); + magicmouse_haptic_softness(trackpad_hdev, + magicmouse_effect_is_deep(effect)); /* The actuator may have been rebound or reset since the last upload. */ ret = magicmouse_switch_mode(trackpad_idev, HID_HAPTIC_MODE_HOST); @@ -1685,6 +1895,9 @@ static int apple_taptic_init_mtp(struct magicmouse_sc *msc, for (i = 0; i < FF_MAX_EFFECTS; i++) INIT_WORK(&msc->haptic_effects[i].work, haptic_playback_worker); + INIT_WORK(&msc->haptic_press_work, magicmouse_press_worker); + INIT_WORK(&msc->haptic_release_work, magicmouse_release_worker); + /* FF_HAPTIC has to be in ffbit before the FF device is created. */ input_set_capability(msc->input, EV_FF, FF_HAPTIC); @@ -1760,6 +1973,9 @@ static void magicmouse_teardown_haptics(struct magicmouse_sc *msc) if (!msc->haptics) return; + cancel_work_sync(&msc->haptic_press_work); + cancel_work_sync(&msc->haptic_release_work); + /* * Tear the force feedback device down while msc is still valid: an * open evdev fd would otherwise run it from input_dev_release() after From 3c1b53dfcf0f432f2a6aca01bf087fc7a0f5286b Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 22:25:12 +0200 Subject: [PATCH 0509/2680] HID: magicmouse: add a deep click The pad reports finger pressure well past the point where a click is asserted, so a second, firmer press inside one click can be told apart from an ordinary one. That is what Force Touch does, and the hardware already carries the signal. Fire once per click at haptic_deep_threshold, with its own pulse count, gap and waveform shape so it can be made to feel distinct from the first click, and clear the flag only on release so holding at depth does not repeat it. Linux has no key for a force click, so haptic_deep_keycode names one and nothing is reported unless it is set; the capability has to be advertised at probe, so that parameter is read-only. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-magicmouse.c | 75 ++++++++++++++++++++++++++++++++---- 1 file changed, 68 insertions(+), 7 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index b7049a888e023..c490f4fcbd363 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -109,6 +109,32 @@ static u16 haptic_release_threshold = 75; module_param(haptic_release_threshold, ushort, 0644); MODULE_PARM_DESC(haptic_release_threshold, "summed finger pressure at which a click ends (default 75)"); +/* + * A deep click is a second, firmer press within one click, the way Force Touch + * behaves. On a J313 an ordinary click peaks around 250 and a firm press + * reaches past 1000, so the threshold sits well clear of the first one. + */ +static u16 haptic_deep_threshold = 500; +module_param(haptic_deep_threshold, ushort, 0644); +MODULE_PARM_DESC(haptic_deep_threshold, "summed finger pressure for a deep click, 0 disables (default 500)"); + +static u8 haptic_deep_pulses = 1; +module_param(haptic_deep_pulses, byte, 0644); +MODULE_PARM_DESC(haptic_deep_pulses, "pulses played for a deep click (default 1)"); + +static u8 haptic_deep_gap_ms = 15; +module_param(haptic_deep_gap_ms, byte, 0644); +MODULE_PARM_DESC(haptic_deep_gap_ms, "milliseconds between deep-click pulses, 0 to use haptic_pulse_gap_ms (default 15)"); + +/* + * Linux has no key for a force click, so which one to send is the user's + * choice; nothing is sent unless one is named. It is read at probe because the + * capability has to be advertised when the input device is set up. + */ +static u16 haptic_deep_keycode; +module_param(haptic_deep_keycode, ushort, 0444); +MODULE_PARM_DESC(haptic_deep_keycode, "key code reported on a deep click, 0 for none (default 0)"); + static u8 haptic_pulse_gap_ms = 25; module_param(haptic_pulse_gap_ms, byte, 0644); MODULE_PARM_DESC(haptic_pulse_gap_ms, "milliseconds between pulses when more than one (default 25)"); @@ -255,7 +281,9 @@ struct magicmouse_sc { struct effect_job *haptic_effects; struct work_struct haptic_press_work; struct work_struct haptic_release_work; + struct work_struct haptic_deep_work; bool haptic_button_down; + bool haptic_deep_fired; struct hid_device *hdev; struct delayed_work work; @@ -1403,10 +1431,10 @@ static void magicmouse_haptic_release_mode(struct magicmouse_sc *msc); * actuator back to the firmware and turn the feature off instead. */ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, - u8 pulses) + bool deep, u8 pulses) { struct hid_device *actuator = magicmouse_get_actuator(msc); - u8 strength, softness; + u8 strength, softness, gap; unsigned int i; int ret; @@ -1433,8 +1461,10 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, hid_info(msc->hdev, "host-driven click feedback active\n"); } - strength = magicmouse_haptic_strength(msc->hdev, false); - softness = magicmouse_haptic_softness(msc->hdev, false); + strength = magicmouse_haptic_strength(msc->hdev, deep); + softness = magicmouse_haptic_softness(msc->hdev, deep); + gap = (deep && haptic_deep_gap_ms) ? haptic_deep_gap_ms + : haptic_pulse_gap_ms; for (i = 0; i < pulses; i++) { ret = apple_taptic_send(actuator, usage, strength, softness); @@ -1451,7 +1481,7 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, "release" : "press", i + 1, pulses, strength); if (i + 1 < pulses) - msleep(haptic_pulse_gap_ms); + msleep(gap); } } @@ -1489,10 +1519,22 @@ static void magicmouse_press_worker(struct work_struct *ws) return; } - magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMPRESS & HID_USAGE, + magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMPRESS & HID_USAGE, false, haptic_press_pulses); } +static void magicmouse_deep_worker(struct work_struct *ws) +{ + struct magicmouse_sc *msc = + container_of(ws, struct magicmouse_sc, haptic_deep_work); + + if (!haptic_press) + return; + + magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMPRESS & HID_USAGE, true, + haptic_deep_pulses); +} + static void magicmouse_release_worker(struct work_struct *ws) { struct magicmouse_sc *msc = @@ -1507,7 +1549,7 @@ static void magicmouse_release_worker(struct work_struct *ws) * The bare header tells the firmware that the click gesture is over, so * send it even when no extra pulses were asked for. */ - magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMRELEASE & HID_USAGE, + magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMRELEASE & HID_USAGE, false, max_t(u8, haptic_release_pulses, 1)); } @@ -1537,13 +1579,27 @@ static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, if (!msc->haptic_button_down && pressure >= haptic_press_threshold) { msc->haptic_button_down = true; + msc->haptic_deep_fired = false; hid_dbg(msc->hdev, "click at pressure %u\n", pressure); queue_work(msc->haptics->wq, &msc->haptic_press_work); } else if (msc->haptic_button_down && pressure <= haptic_release_threshold) { msc->haptic_button_down = false; + msc->haptic_deep_fired = false; hid_dbg(msc->hdev, "release at pressure %u\n", pressure); queue_work(msc->haptics->wq, &msc->haptic_release_work); + } else if (msc->haptic_button_down && !msc->haptic_deep_fired && + haptic_deep_threshold && + pressure >= haptic_deep_threshold) { + /* Pressing harder without lifting: once per click, not while held. */ + msc->haptic_deep_fired = true; + hid_dbg(msc->hdev, "deep click at pressure %u\n", pressure); + queue_work(msc->haptics->wq, &msc->haptic_deep_work); + + if (haptic_deep_keycode) { + input_report_key(msc->input, haptic_deep_keycode, 1); + input_report_key(msc->input, haptic_deep_keycode, 0); + } } return msc->haptic_button_down; @@ -1897,6 +1953,10 @@ static int apple_taptic_init_mtp(struct magicmouse_sc *msc, INIT_WORK(&msc->haptic_press_work, magicmouse_press_worker); INIT_WORK(&msc->haptic_release_work, magicmouse_release_worker); + INIT_WORK(&msc->haptic_deep_work, magicmouse_deep_worker); + + if (haptic_deep_keycode) + input_set_capability(msc->input, EV_KEY, haptic_deep_keycode); /* FF_HAPTIC has to be in ffbit before the FF device is created. */ input_set_capability(msc->input, EV_FF, FF_HAPTIC); @@ -1975,6 +2035,7 @@ static void magicmouse_teardown_haptics(struct magicmouse_sc *msc) cancel_work_sync(&msc->haptic_press_work); cancel_work_sync(&msc->haptic_release_work); + cancel_work_sync(&msc->haptic_deep_work); /* * Tear the force feedback device down while msc is still valid: an From 9d7b04bce25544ae5c68f43457fd84d49e32a9e0 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Thu, 17 Sep 2026 09:39:53 +0200 Subject: [PATCH 0510/2680] HID: magicmouse: derive the click while the host holds the actuator The firmware reports the button only while it owns the actuator. The derived click covered haptic_press, but a force-feedback effect uploaded by userspace also takes host-controlled mode, and the mode cannot be released while an effect is loaded. In that state the driver reported the firmware's button, which never changes again, so the pad stopped clicking until every effect was erased. Derive the button whenever the actuator is in host-controlled mode, whatever put it there, and keep asking for it back when the waveform is not ours to play. Found by a review bot on the pull request. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-magicmouse.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index c490f4fcbd363..8ac997a743538 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -1565,18 +1565,31 @@ static void magicmouse_release_worker(struct work_struct *ws) static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, bool firmware_button) { + bool host_mode; + if (!msc->haptics || !msc->haptics->wq) return firmware_button; - if (!haptic_press) { - /* Hand the actuator back and report its button again. */ - if (msc->haptics->mode == HID_HAPTIC_MODE_HOST) { - msc->haptic_button_down = false; - queue_work(msc->haptics->wq, &msc->haptic_release_work); - } + host_mode = msc->haptics->mode == HID_HAPTIC_MODE_HOST; + + if (!haptic_press && !host_mode) { + /* The firmware owns the actuator and reports the button. */ + msc->haptic_button_down = false; + msc->haptic_deep_fired = false; return firmware_button; } + /* + * Host-controlled mode without host-driven feedback means userspace + * uploaded an effect and owns the waveform. Ask for the actuator back, + * but keep deriving the button until it is: the firmware reports no + * button while it does not hold the actuator, and the mode cannot be + * released while an effect is loaded, so trusting it here would leave + * the pad unable to click until every effect is erased. + */ + if (!haptic_press) + queue_work(msc->haptics->wq, &msc->haptic_release_work); + if (!msc->haptic_button_down && pressure >= haptic_press_threshold) { msc->haptic_button_down = true; msc->haptic_deep_fired = false; From ec9bc6d5e8da3ea0ea177448ae37828c2fc7da1f Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Thu, 17 Sep 2026 09:39:57 +0200 Subject: [PATCH 0511/2680] HID: magicmouse: forget the actuator's mode across system sleep The actuator leaves system sleep in device-controlled mode. The transport resumes the keyboard and the trackpad, not the actuator, and nothing tells the driver the mode changed, so msc->haptics->mode still reads host-controlled. The next press then skips the switch and plays a pulse while the firmware plays its own, which is felt as two clicks per press. On a J313 across an s2idle cycle the driver says nothing between 'host-driven click feedback active' before the suspend and the first press after it: it never re-took the mode because it believed it still held it. Drop the cached mode in reset_resume, which the transport does call for the trackpad, and drop the click state with it. The next press takes host-controlled mode again if it needs it. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-magicmouse.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 8ac997a743538..e67e4ce60ab92 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -1617,12 +1617,34 @@ static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, return msc->haptic_button_down; } + +/* + * The actuator leaves system sleep in device-controlled mode whatever it was + * in before: the transport resumes the keyboard and the trackpad, not the + * actuator, and nothing reports the change. A cached host-controlled mode then + * makes the next press skip the switch and play a pulse while the firmware is + * playing its own, which is felt as two clicks. Forget it instead; the next + * press takes the mode again if it needs it. + */ +static void magicmouse_haptic_forget_mode(struct magicmouse_sc *msc) +{ + if (!msc->haptics) + return; + + msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; + msc->haptic_button_down = false; + msc->haptic_deep_fired = false; +} #else static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, bool firmware_button) { return firmware_button; } + +static void magicmouse_haptic_forget_mode(struct magicmouse_sc *msc) +{ +} #endif static int magicmouse_input_configured(struct hid_device *hdev, @@ -2275,6 +2297,9 @@ static int magicmouse_reset_resume(struct hid_device *hdev) if (msc && (hdev->type == HID_TYPE_USBMOUSE || hdev->bus == BUS_SPI)) schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); + if (msc) + magicmouse_haptic_forget_mode(msc); + return 0; } #endif From 4b1f9bdf97606727e6908ec88ba3a26902b1c368 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Thu, 17 Sep 2026 10:06:10 +0200 Subject: [PATCH 0512/2680] HID: apple-mtp-haptic: put the actuator back in device-controlled mode The actuator keeps its mode across a warm reboot. Every user of it starts out believing it is device-controlled, so one left in host-controlled mode by a previous boot is silent: the firmware plays nothing because it no longer owns the actuator, and the host that did is gone. The trackpad then has no click feedback at all, with nothing in the log to say why. Only a cold boot clears it. Assert device-controlled mode at probe instead of inheriting whatever the last boot left. The switch is not fatal if it fails, since that leaves the actuator exactly where it already was. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-apple-mtp-haptic.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/hid/hid-apple-mtp-haptic.c b/drivers/hid/hid-apple-mtp-haptic.c index 0678953acbb40..42ab019f7a27a 100644 --- a/drivers/hid/hid-apple-mtp-haptic.c +++ b/drivers/hid/hid-apple-mtp-haptic.c @@ -72,6 +72,21 @@ static int apple_actuator_probe(struct hid_device *haptic_hdev, const struct hid return ret; } + /* + * The actuator keeps its mode across a warm reboot, and every user of + * it starts out believing it is device-controlled. One left in + * host-controlled mode by a previous boot is silent: its firmware plays + * nothing, and the host that was driving it is gone. Assert the state + * the drivers assume rather than inheriting the last one. A failure + * here is not fatal; it only means the actuator keeps whatever mode it + * had, which is what would have happened anyway. + */ + ret = apple_taptic_switch_modes(haptic_hdev, false); + if (ret) + hid_warn(haptic_hdev, + "cannot return the actuator to device-controlled mode (%d)\n", + ret); + return 0; } From 38fadde52c072d8f41f0068a1e16e45b36b6fe87 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 7 Oct 2026 11:15:00 -0400 Subject: [PATCH 0513/2680] HID: magicmouse: click through each force stage on the way up A deep click fired once per click and nothing marked leaving it: pressing gave a click and then a deep click, but easing off went straight from the deep stage to the final release with no feedback in between. macOS treats Force Touch as two stages and clicks at each boundary in both directions: press, deep, back to the first stage, release. Track the deep stage as a level rather than a one-shot flag. It ends at haptic_deep_release_threshold, well below where it starts, for the same reason the first stage has a gap between press and release: a finger held near the threshold must not chatter. Leaving it plays the release waveform, and the stage can be entered again within the same click. The default of 350 sits above the 150-270 that ordinary clicks peak at on a J613 and below its 500 deep threshold. Play the actuator's own force-click waveform for the deep stage on MTP (haptic_deep_waveform, on by default) instead of the press waveform with deep-click shaping, which on MTP has the same defaults as a press and so felt identical. Report the deep-click key from a separate "Force Click" input device and hold it for as long as the deep stage lasts. On the trackpad's own device it never reached userspace: libinput drives a touchpad with a dispatcher that drops every key that is not a button. Assisted-by: Claude:claude-opus-5-5 --- drivers/hid/hid-magicmouse.c | 162 ++++++++++++++++++++++++++++------- 1 file changed, 133 insertions(+), 29 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index e67e4ce60ab92..8eb69d187ce87 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -118,6 +118,22 @@ static u16 haptic_deep_threshold = 500; module_param(haptic_deep_threshold, ushort, 0644); MODULE_PARM_DESC(haptic_deep_threshold, "summed finger pressure for a deep click, 0 disables (default 500)"); +/* + * Force Touch has two stages, and macOS clicks on the way back up through + * each of them as well as on the way down: easing off a deep click to an + * ordinary press is felt, and so is the final release. The deep stage ends + * well below where it starts, for the same reason the first one does: a + * finger held near the threshold must not chatter. J613 ordinary clicks peak + * at 150-270 and firm presses at 700-910, so this sits above the former. + */ +static u16 haptic_deep_release_threshold = 350; +module_param(haptic_deep_release_threshold, ushort, 0644); +MODULE_PARM_DESC(haptic_deep_release_threshold, "summed finger pressure at which a deep click ends, below haptic_deep_threshold (default 350)"); + +static bool haptic_deep_waveform = true; +module_param(haptic_deep_waveform, bool, 0644); +MODULE_PARM_DESC(haptic_deep_waveform, "play the actuator's force-click waveform for a deep click (default on; off: the press waveform)"); + static u8 haptic_deep_pulses = 1; module_param(haptic_deep_pulses, byte, 0644); MODULE_PARM_DESC(haptic_deep_pulses, "pulses played for a deep click (default 1)"); @@ -128,12 +144,15 @@ MODULE_PARM_DESC(haptic_deep_gap_ms, "milliseconds between deep-click pulses, 0 /* * Linux has no key for a force click, so which one to send is the user's - * choice; nothing is sent unless one is named. It is read at probe because the - * capability has to be advertised when the input device is set up. + * choice; nothing is sent unless one is named. It is held down for as long as + * the deep stage lasts. The key comes from a separate input device: libinput + * drives a touchpad with its own dispatcher, which drops every key that is not + * a button, so a key on the trackpad's own device never reaches userspace. It + * is read at probe because that device is set up there. */ static u16 haptic_deep_keycode; module_param(haptic_deep_keycode, ushort, 0444); -MODULE_PARM_DESC(haptic_deep_keycode, "key code reported on a deep click, 0 for none (default 0)"); +MODULE_PARM_DESC(haptic_deep_keycode, "key code held during a deep click on a separate \"Force Click\" input device, 0 for none (default 0)"); static u8 haptic_pulse_gap_ms = 25; module_param(haptic_pulse_gap_ms, byte, 0644); @@ -282,8 +301,10 @@ struct magicmouse_sc { struct work_struct haptic_press_work; struct work_struct haptic_release_work; struct work_struct haptic_deep_work; + struct work_struct haptic_deep_release_work; + struct input_dev *force_input; bool haptic_button_down; - bool haptic_deep_fired; + bool haptic_deep_down; struct hid_device *hdev; struct delayed_work work; @@ -1478,7 +1499,7 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, } hid_dbg(msc->hdev, "played %s pulse %u/%u at strength 0x%02x\n", usage == (HID_HP_WAVEFORMRELEASE & HID_USAGE) ? - "release" : "press", + "release" : deep ? "deep" : "press", i + 1, pulses, strength); if (i + 1 < pulses) msleep(gap); @@ -1531,8 +1552,32 @@ static void magicmouse_deep_worker(struct work_struct *ws) if (!haptic_press) return; - magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMPRESS & HID_USAGE, true, - haptic_deep_pulses); + magicmouse_haptic_pulses(msc, haptic_deep_waveform ? + APPLE_HP_WAVEFORMDEEPCLICK & HID_USAGE : + HID_HP_WAVEFORMPRESS & HID_USAGE, + true, haptic_deep_pulses); +} + +/* Easing off a deep click to an ordinary press: the first of two releases. */ +static void magicmouse_deep_release_worker(struct work_struct *ws) +{ + struct magicmouse_sc *msc = + container_of(ws, struct magicmouse_sc, haptic_deep_release_work); + + if (!haptic_press) + return; + + magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMRELEASE & HID_USAGE, false, + max_t(u8, haptic_release_pulses, 1)); +} + +static void magicmouse_force_key(struct magicmouse_sc *msc, bool down) +{ + if (!msc->force_input) + return; + + input_report_key(msc->force_input, haptic_deep_keycode, down); + input_sync(msc->force_input); } static void magicmouse_release_worker(struct work_struct *ws) @@ -1575,7 +1620,10 @@ static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, if (!haptic_press && !host_mode) { /* The firmware owns the actuator and reports the button. */ msc->haptic_button_down = false; - msc->haptic_deep_fired = false; + if (msc->haptic_deep_down) { + msc->haptic_deep_down = false; + magicmouse_force_key(msc, false); + } return firmware_button; } @@ -1590,29 +1638,39 @@ static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, if (!haptic_press) queue_work(msc->haptics->wq, &msc->haptic_release_work); - if (!msc->haptic_button_down && pressure >= haptic_press_threshold) { - msc->haptic_button_down = true; - msc->haptic_deep_fired = false; - hid_dbg(msc->hdev, "click at pressure %u\n", pressure); - queue_work(msc->haptics->wq, &msc->haptic_press_work); - } else if (msc->haptic_button_down && - pressure <= haptic_release_threshold) { + /* + * Three levels, up, clicked and deep, each step with its own click on + * the way down and on the way back up. One step per frame: a press hard + * enough to reach the deep stage at once still clicks twice, a frame + * apart. + */ + if (!msc->haptic_button_down) { + if (pressure >= haptic_press_threshold) { + msc->haptic_button_down = true; + hid_dbg(msc->hdev, "click at pressure %u\n", pressure); + queue_work(msc->haptics->wq, &msc->haptic_press_work); + } + } else if (pressure <= haptic_release_threshold) { msc->haptic_button_down = false; - msc->haptic_deep_fired = false; + if (msc->haptic_deep_down) { + msc->haptic_deep_down = false; + magicmouse_force_key(msc, false); + } hid_dbg(msc->hdev, "release at pressure %u\n", pressure); queue_work(msc->haptics->wq, &msc->haptic_release_work); - } else if (msc->haptic_button_down && !msc->haptic_deep_fired && - haptic_deep_threshold && - pressure >= haptic_deep_threshold) { - /* Pressing harder without lifting: once per click, not while held. */ - msc->haptic_deep_fired = true; - hid_dbg(msc->hdev, "deep click at pressure %u\n", pressure); - queue_work(msc->haptics->wq, &msc->haptic_deep_work); - - if (haptic_deep_keycode) { - input_report_key(msc->input, haptic_deep_keycode, 1); - input_report_key(msc->input, haptic_deep_keycode, 0); + } else if (!msc->haptic_deep_down) { + if (haptic_deep_threshold && pressure >= haptic_deep_threshold) { + msc->haptic_deep_down = true; + hid_dbg(msc->hdev, "deep click at pressure %u\n", pressure); + queue_work(msc->haptics->wq, &msc->haptic_deep_work); + magicmouse_force_key(msc, true); } + } else if (pressure <= min(haptic_deep_release_threshold, + haptic_deep_threshold)) { + msc->haptic_deep_down = false; + hid_dbg(msc->hdev, "deep release at pressure %u\n", pressure); + queue_work(msc->haptics->wq, &msc->haptic_deep_release_work); + magicmouse_force_key(msc, false); } return msc->haptic_button_down; @@ -1633,7 +1691,10 @@ static void magicmouse_haptic_forget_mode(struct magicmouse_sc *msc) msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; msc->haptic_button_down = false; - msc->haptic_deep_fired = false; + if (msc->haptic_deep_down) { + msc->haptic_deep_down = false; + magicmouse_force_key(msc, false); + } } #else static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, @@ -1959,6 +2020,46 @@ static void apple_taptic_destroy(struct ff_device *ff) msc->haptic_effects = NULL; } +/* + * The deep-click key lives on its own input device (see haptic_deep_keycode). + * It is managed by devres, so it goes away with the trackpad. Failing to set + * it up costs only the key, not the click feedback. + */ +static void magicmouse_setup_force_input(struct magicmouse_sc *msc) +{ + struct hid_device *hdev = msc->hdev; + struct input_dev *input; + int ret; + + if (haptic_deep_keycode > KEY_MAX) + return; + + input = devm_input_allocate_device(&hdev->dev); + if (!input) + return; + + input->name = devm_kasprintf(&hdev->dev, GFP_KERNEL, "%s Force Click", + hdev->name); + if (!input->name) + return; + input->phys = hdev->phys; + input->uniq = hdev->uniq; + input->id.bustype = hdev->bus; + input->id.vendor = hdev->vendor; + input->id.product = hdev->product; + input->id.version = hdev->version; + input->dev.parent = &hdev->dev; + input_set_capability(input, EV_KEY, haptic_deep_keycode); + + ret = input_register_device(input); + if (ret) { + hid_warn(hdev, "no force click key (%d)\n", ret); + return; + } + + msc->force_input = input; +} + static int apple_taptic_init_mtp(struct magicmouse_sc *msc, struct hid_haptic_device *haptic_dev) { @@ -1989,9 +2090,11 @@ static int apple_taptic_init_mtp(struct magicmouse_sc *msc, INIT_WORK(&msc->haptic_press_work, magicmouse_press_worker); INIT_WORK(&msc->haptic_release_work, magicmouse_release_worker); INIT_WORK(&msc->haptic_deep_work, magicmouse_deep_worker); + INIT_WORK(&msc->haptic_deep_release_work, + magicmouse_deep_release_worker); if (haptic_deep_keycode) - input_set_capability(msc->input, EV_KEY, haptic_deep_keycode); + magicmouse_setup_force_input(msc); /* FF_HAPTIC has to be in ffbit before the FF device is created. */ input_set_capability(msc->input, EV_FF, FF_HAPTIC); @@ -2071,6 +2174,7 @@ static void magicmouse_teardown_haptics(struct magicmouse_sc *msc) cancel_work_sync(&msc->haptic_press_work); cancel_work_sync(&msc->haptic_release_work); cancel_work_sync(&msc->haptic_deep_work); + cancel_work_sync(&msc->haptic_deep_release_work); /* * Tear the force feedback device down while msc is still valid: an From 8e65b5d6284931fa3b334695340d517d64e426a8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:30:44 +1100 Subject: [PATCH 0514/2680] HID: magicmouse: serialize haptic effect updates and retain C1FE clicks Publish complete effect parameters under a per-effect spinlock only after the mode switch succeeds. Workers take one coherent snapshot and erase drains the affected job before clearing it. Failed updates leave the previous effect intact. Serialize actuator lookup, mode transactions and cached state with the existing haptic auto-trigger mutex. Drain workers before taking that mutex during erase and destroy, and retake host mode on playback after resume invalidates the mode cache. C1FE reports have no pressure, so they cannot emulate buttons when firmware click handling is disabled by host-controlled haptics. Keep those pads in firmware-controlled mode and omit force feedback and pressure-pad capabilities. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/hid-magicmouse.c | 140 ++++++++++++++++++++++++----------- 1 file changed, 97 insertions(+), 43 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 8eb69d187ce87..aa0e6cee6d8fd 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -247,6 +248,9 @@ struct magicmouse_input_ops { struct effect_job { struct work_struct work; + struct magicmouse_sc *msc; + /* Protects effect parameters shared with the worker. */ + spinlock_t lock; struct hid_device *taptic_hdev; u16 effect_type; u8 strength; @@ -1264,7 +1268,8 @@ static int magicmouse_setup_input_mtp(struct input_dev *input, struct magicmouse_sc *msc = hid_get_drvdata(hdev); __set_bit(INPUT_PROP_BUTTONPAD, input->propbit); - __set_bit(INPUT_PROP_PRESSUREPAD, input->propbit); + if (!msc->mtp_c1fe) + __set_bit(INPUT_PROP_PRESSUREPAD, input->propbit); __clear_bit(BTN_0, input->keybit); __clear_bit(BTN_RIGHT, input->keybit); __clear_bit(BTN_MIDDLE, input->keybit); @@ -1272,7 +1277,9 @@ static int magicmouse_setup_input_mtp(struct input_dev *input, __clear_bit(REL_X, input->relbit); __clear_bit(REL_Y, input->relbit); - mt_flags = INPUT_MT_POINTER | INPUT_MT_DROP_UNUSED | INPUT_MT_TRACK | INPUT_MT_TOTAL_FORCE; + mt_flags = INPUT_MT_POINTER | INPUT_MT_DROP_UNUSED | INPUT_MT_TRACK; + if (!msc->mtp_c1fe) + mt_flags |= INPUT_MT_TOTAL_FORCE; /* * The C1FE report carries positions and the button only; advertise @@ -1445,6 +1452,7 @@ static int magicmouse_switch_mode(struct input_dev *trackpad_idev, int mode) static u8 magicmouse_haptic_softness(struct hid_device *hdev, bool deep); static u8 magicmouse_haptic_strength(struct hid_device *hdev, bool deep); static void magicmouse_haptic_release_mode(struct magicmouse_sc *msc); +static void magicmouse_haptic_release_mode_locked(struct magicmouse_sc *msc); /* * Play the waveform @pulses times, @haptic_pulse_gap_ms apart. A failure here @@ -1454,7 +1462,7 @@ static void magicmouse_haptic_release_mode(struct magicmouse_sc *msc); static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, bool deep, u8 pulses) { - struct hid_device *actuator = magicmouse_get_actuator(msc); + struct hid_device *actuator; u8 strength, softness, gap; unsigned int i; int ret; @@ -1462,6 +1470,9 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, if (!pulses) return; + guard(mutex)(&msc->haptics->auto_trigger_mutex); + actuator = magicmouse_get_actuator(msc); + if (!actuator) { hid_warn(msc->hdev, "no actuator for host click feedback; using the firmware's\n"); @@ -1469,7 +1480,7 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, return; } - if (msc->haptics->mode != HID_HAPTIC_MODE_HOST) { + if (READ_ONCE(msc->haptics->mode) != HID_HAPTIC_MODE_HOST) { ret = apple_taptic_switch_modes(actuator, HID_HAPTIC_MODE_HOST); if (ret) { hid_warn(msc->hdev, @@ -1478,7 +1489,7 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, haptic_press = false; return; } - msc->haptics->mode = HID_HAPTIC_MODE_HOST; + WRITE_ONCE(msc->haptics->mode, HID_HAPTIC_MODE_HOST); hid_info(msc->hdev, "host-driven click feedback active\n"); } @@ -1494,7 +1505,7 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, "click waveform failed (%d); returning click feedback to the firmware\n", ret); haptic_press = false; - magicmouse_haptic_release_mode(msc); + magicmouse_haptic_release_mode_locked(msc); return; } hid_dbg(msc->hdev, "played %s pulse %u/%u at strength 0x%02x\n", @@ -1510,24 +1521,30 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, * Hand click feedback back to the actuator's firmware, unless userspace still * has effects uploaded and so still wants host-controlled mode. */ -static void magicmouse_haptic_release_mode(struct magicmouse_sc *msc) +static void magicmouse_haptic_release_mode_locked(struct magicmouse_sc *msc) { struct hid_device *actuator; unsigned int i; - if (msc->haptics->mode != HID_HAPTIC_MODE_HOST) + if (READ_ONCE(msc->haptics->mode) != HID_HAPTIC_MODE_HOST) return; if (msc->haptic_effects) { for (i = 0; i < FF_MAX_EFFECTS; i++) - if (msc->haptic_effects[i].effect_type) + if (READ_ONCE(msc->haptic_effects[i].effect_type)) return; } actuator = magicmouse_get_actuator(msc); if (actuator && !apple_taptic_switch_modes(actuator, HID_HAPTIC_MODE_DEVICE)) - msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; + WRITE_ONCE(msc->haptics->mode, HID_HAPTIC_MODE_DEVICE); +} + +static void magicmouse_haptic_release_mode(struct magicmouse_sc *msc) +{ + guard(mutex)(&msc->haptics->auto_trigger_mutex); + magicmouse_haptic_release_mode_locked(msc); } static void magicmouse_press_worker(struct work_struct *ws) @@ -1615,7 +1632,7 @@ static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, if (!msc->haptics || !msc->haptics->wq) return firmware_button; - host_mode = msc->haptics->mode == HID_HAPTIC_MODE_HOST; + host_mode = READ_ONCE(msc->haptics->mode) == HID_HAPTIC_MODE_HOST; if (!haptic_press && !host_mode) { /* The firmware owns the actuator and reports the button. */ @@ -1689,7 +1706,7 @@ static void magicmouse_haptic_forget_mode(struct magicmouse_sc *msc) if (!msc->haptics) return; - msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; + WRITE_ONCE(msc->haptics->mode, HID_HAPTIC_MODE_DEVICE); msc->haptic_button_down = false; if (msc->haptic_deep_down) { msc->haptic_deep_down = false; @@ -1901,6 +1918,10 @@ static int apple_upload_effects(struct input_dev *trackpad_idev, struct magicmouse_sc *msc = hid_get_drvdata(trackpad_hdev); struct hid_haptic_device *haptics = msc->haptics; struct hid_device *actuator; + struct effect_job *job = &msc->haptic_effects[effect->id]; + unsigned long flags; + u8 strength, softness; + bool deep; int ret; switch (effect->u.haptic.hid_usage) { @@ -1912,30 +1933,31 @@ static int apple_upload_effects(struct input_dev *trackpad_idev, return -EINVAL; } + guard(mutex)(&haptics->auto_trigger_mutex); actuator = magicmouse_get_actuator(msc); if (!actuator) return -ENODEV; - msc->haptic_effects[effect->id].taptic_hdev = actuator; - msc->haptic_effects[effect->id].effect_type = (effect->u.haptic.hid_usage) & HID_USAGE; - msc->haptic_effects[effect->id].strength = - (min(100, (effect->u.haptic.intensity)) * - magicmouse_haptic_strength(trackpad_hdev, - magicmouse_effect_is_deep(effect))) / 100; - msc->haptic_effects[effect->id].softness = - magicmouse_haptic_softness(trackpad_hdev, - magicmouse_effect_is_deep(effect)); + deep = magicmouse_effect_is_deep(effect); + strength = min_t(u16, 100, effect->u.haptic.intensity) * + magicmouse_haptic_strength(trackpad_hdev, deep) / 100; + softness = magicmouse_haptic_softness(trackpad_hdev, deep); /* The actuator may have been rebound or reset since the last upload. */ ret = magicmouse_switch_mode(trackpad_idev, HID_HAPTIC_MODE_HOST); if (ret) { dev_err(&msc->hdev->dev, "failed to switch to host-controlled mode: %d\n", ret); - msc->haptic_effects[effect->id].effect_type = 0; return ret; } - haptics->mode = HID_HAPTIC_MODE_HOST; + spin_lock_irqsave(&job->lock, flags); + job->taptic_hdev = actuator; + job->strength = strength; + job->softness = softness; + WRITE_ONCE(job->effect_type, effect->u.haptic.hid_usage & HID_USAGE); + spin_unlock_irqrestore(&job->lock, flags); + WRITE_ONCE(haptics->mode, HID_HAPTIC_MODE_HOST); hid_dbg(trackpad_hdev, "uploaded haptic effect %d\n", effect->id); return 0; @@ -1944,11 +1966,32 @@ static int apple_upload_effects(struct input_dev *trackpad_idev, static void haptic_playback_worker(struct work_struct *ws) { struct effect_job *job = container_of(ws, struct effect_job, work); + struct hid_device *actuator; + unsigned long flags; + u16 type; + u8 strength, softness; + int ret; + + spin_lock_irqsave(&job->lock, flags); + actuator = job->taptic_hdev; + type = READ_ONCE(job->effect_type); + strength = job->strength; + softness = job->softness; + spin_unlock_irqrestore(&job->lock, flags); - if (job->effect_type && - apple_taptic_send(job->taptic_hdev, job->effect_type, - job->strength, job->softness)) - hid_warn(job->taptic_hdev, "cannot play the haptic effect\n"); + if (!type) + return; + guard(mutex)(&job->msc->haptics->auto_trigger_mutex); + if (READ_ONCE(job->msc->haptics->mode) != HID_HAPTIC_MODE_HOST) { + ret = apple_taptic_switch_modes(actuator, HID_HAPTIC_MODE_HOST); + if (ret) { + hid_warn(actuator, "cannot take host-controlled mode (%d)\n", ret); + return; + } + WRITE_ONCE(job->msc->haptics->mode, HID_HAPTIC_MODE_HOST); + } + if (apple_taptic_send(actuator, type, strength, softness)) + hid_warn(actuator, "cannot play the haptic effect\n"); } static int apple_taptic_playback(struct input_dev *trackpad_idev, int effect_id, int value) @@ -1968,16 +2011,19 @@ static int apple_taptic_erase(struct input_dev *trackpad_idev, int effect_id) struct magicmouse_sc *msc = hid_get_drvdata(trackpad_hdev); int i, ret = 0; - msc->haptic_effects[effect_id].effect_type = 0; + cancel_work_sync(&msc->haptic_effects[effect_id].work); + WRITE_ONCE(msc->haptic_effects[effect_id].effect_type, 0); for (i = 0; i < FF_MAX_EFFECTS; i++) { - if (msc->haptic_effects[i].effect_type != 0) + if (READ_ONCE(msc->haptic_effects[i].effect_type) != 0) return 0; } + /* Drain workers before taking the mutex they need. */ + flush_workqueue(msc->haptics->wq); + guard(mutex)(&msc->haptics->auto_trigger_mutex); /* Return to device-controlled mode if there are no effects left */ - if (msc->haptics->mode == HID_HAPTIC_MODE_HOST) { - flush_workqueue(msc->haptics->wq); + if (READ_ONCE(msc->haptics->mode) == HID_HAPTIC_MODE_HOST) { ret = magicmouse_switch_mode(trackpad_idev, HID_HAPTIC_MODE_DEVICE); if (ret) { @@ -1987,7 +2033,7 @@ static int apple_taptic_erase(struct input_dev *trackpad_idev, int effect_id) return ret; } - msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; + WRITE_ONCE(msc->haptics->mode, HID_HAPTIC_MODE_DEVICE); } return 0; @@ -2002,15 +2048,18 @@ static void apple_taptic_destroy(struct ff_device *ff) /* ff->private is devm-allocated; do not let ff-core kfree() it. */ ff->private = NULL; - if (msc->haptics && msc->haptics->mode == HID_HAPTIC_MODE_HOST) { - flush_workqueue(msc->haptics->wq); - ret = magicmouse_switch_mode(msc->haptics->input_dev, HID_HAPTIC_MODE_DEVICE); - if (ret) - hid_warn(msc->hdev, - "failed to switch back to device-controlled mode: %d\n", - ret); - else - msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; + flush_workqueue(haptic_dev->wq); + { + guard(mutex)(&haptic_dev->auto_trigger_mutex); + if (msc->haptics && READ_ONCE(msc->haptics->mode) == HID_HAPTIC_MODE_HOST) { + ret = magicmouse_switch_mode(msc->haptics->input_dev, HID_HAPTIC_MODE_DEVICE); + if (ret) + hid_warn(msc->hdev, + "failed to switch back to device-controlled mode: %d\n", + ret); + else + WRITE_ONCE(msc->haptics->mode, HID_HAPTIC_MODE_DEVICE); + } } destroy_workqueue(haptic_dev->wq); @@ -2069,6 +2118,7 @@ static int apple_taptic_init_mtp(struct magicmouse_sc *msc, haptic_dev->hdev = trackpad_hdev; haptic_dev->input_dev = msc->input; + mutex_init(&haptic_dev->auto_trigger_mutex); msc->haptic_effects = kzalloc_objs(struct effect_job, FF_MAX_EFFECTS); if (!msc->haptic_effects) { @@ -2084,8 +2134,11 @@ static int apple_taptic_init_mtp(struct magicmouse_sc *msc, return -ENOMEM; } - for (i = 0; i < FF_MAX_EFFECTS; i++) + for (i = 0; i < FF_MAX_EFFECTS; i++) { + msc->haptic_effects[i].msc = msc; + spin_lock_init(&msc->haptic_effects[i].lock); INIT_WORK(&msc->haptic_effects[i].work, haptic_playback_worker); + } INIT_WORK(&msc->haptic_press_work, magicmouse_press_worker); INIT_WORK(&msc->haptic_release_work, magicmouse_release_worker); @@ -2137,7 +2190,8 @@ static int magicmouse_init_haptics(struct magicmouse_sc *msc, struct hid_device struct hid_haptic_device *haptics; int ret; - if (msc->haptics) + /* C1FE has no pressure data to emulate clicks in host mode. */ + if (msc->haptics || msc->mtp_c1fe) return 0; /* From f84538c678b2a67856dde1f86da7f6dad8bebad9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:34:39 +1100 Subject: [PATCH 0515/2680] HID: magicmouse: wrap the reviewed haptic mode restoration call Wrap the mode-restoration call to satisfy the kernel line-length check. This is a formatting follow-up to the supplied review fix; the haptic locking and firmware-controlled C1FE behavior are unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/hid-magicmouse.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index aa0e6cee6d8fd..c382f940ff094 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -2052,7 +2052,8 @@ static void apple_taptic_destroy(struct ff_device *ff) { guard(mutex)(&haptic_dev->auto_trigger_mutex); if (msc->haptics && READ_ONCE(msc->haptics->mode) == HID_HAPTIC_MODE_HOST) { - ret = magicmouse_switch_mode(msc->haptics->input_dev, HID_HAPTIC_MODE_DEVICE); + ret = magicmouse_switch_mode(msc->haptics->input_dev, + HID_HAPTIC_MODE_DEVICE); if (ret) hid_warn(msc->hdev, "failed to switch back to device-controlled mode: %d\n", From 53a856abb468c435c47951dda186e17ea64a643a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:17:16 +1100 Subject: [PATCH 0516/2680] HID: magicmouse: depend on a reachable MTP haptic actuator driver hid-magicmouse drives the trackpad actuator through apple_taptic_send() and apple_taptic_switch_modes(), which hid-apple-mtp-haptic exports, and guards that code with IS_REACHABLE(). With HID_MAGICMOUSE=y and HID_APPLE_MTP_HAPTIC=m the helpers are not reachable from built-in code, so the driver silently compiles the haptics out: the actuator driver binds, but the trackpad never gets a force feedback device or FF_HAPTIC, and nothing in the log says why. Use the usual optional-dependency idiom so that Kconfig limits hid-magicmouse to a module whenever the actuator driver is one, and test the helpers with IS_ENABLED(), which now matches what can be linked. Every valid configuration gets haptics when the actuator driver is enabled, and none when it is not. Fixes: cf248ccdaabc ("HID: magicmouse: add support for controlling MTP touchpad haptics") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/Kconfig | 1 + drivers/hid/hid-magicmouse.c | 4 ++-- include/linux/soc/apple/actuator.h | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index 34f217f82481e..9fee16e37fef0 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -749,6 +749,7 @@ config LOGIWHEELS_FF config HID_MAGICMOUSE tristate "Apple Magic Mouse/Trackpad multi-touch support" + depends on HID_APPLE_MTP_HAPTIC || !HID_APPLE_MTP_HAPTIC help Support for the Apple Magic Mouse/Trackpad multi-touch. diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index c382f940ff094..70d066a43279b 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -1402,7 +1402,7 @@ static int magicmouse_input_mapping(struct hid_device *hdev, static int magicmouse_init_haptics(struct magicmouse_sc *msc, struct hid_device *hdev); -#if IS_REACHABLE(CONFIG_HID_APPLE_MTP_HAPTIC) +#if IS_ENABLED(CONFIG_HID_APPLE_MTP_HAPTIC) static int match_actuator(struct device *dev, const void *data) { struct hid_device *hdev; @@ -1880,7 +1880,7 @@ static void magicmouse_battery_timer_tick(struct timer_list *t) } } -#if IS_REACHABLE(CONFIG_HID_APPLE_MTP_HAPTIC) +#if IS_ENABLED(CONFIG_HID_APPLE_MTP_HAPTIC) static bool magicmouse_effect_is_deep(const struct ff_effect *effect) { return effect->u.haptic.hid_usage == diff --git a/include/linux/soc/apple/actuator.h b/include/linux/soc/apple/actuator.h index 0b565d5567eb0..18e1bee0e266c 100644 --- a/include/linux/soc/apple/actuator.h +++ b/include/linux/soc/apple/actuator.h @@ -19,7 +19,7 @@ static inline bool apple_taptic_is_actuator(struct hid_device *hdev) !strcmp(hdev->name, "Apple SPI Actuator"); } -#if IS_REACHABLE(CONFIG_HID_APPLE_MTP_HAPTIC) +#if IS_ENABLED(CONFIG_HID_APPLE_MTP_HAPTIC) int apple_taptic_send(struct hid_device *hdev, u16 effect_type, u8 strength, u8 softness); int apple_taptic_switch_modes(struct hid_device *hdev, bool host_controlled); From f1e217a9e3e08d408687789afac7e9ea4e68842e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:08:47 +1100 Subject: [PATCH 0517/2680] HID: magicmouse: shorten the haptics workqueue name Workqueue names are limited to 31 characters. "Apple trackpad haptics workqueue" is 32, so the first MTP trackpad probe of every boot warns that the name exceeds WQ_NAME_LEN and truncates it to "Apple trackpad haptics workqueu". Name it "apple-trackpad-haptics" instead. Fixes: cf248ccdaabc ("HID: magicmouse: add support for controlling MTP touchpad haptics") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/hid-magicmouse.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 70d066a43279b..e417a65acd7e7 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -2127,7 +2127,7 @@ static int apple_taptic_init_mtp(struct magicmouse_sc *msc, return -ENOMEM; } - haptic_dev->wq = create_singlethread_workqueue("Apple trackpad haptics workqueue"); + haptic_dev->wq = create_singlethread_workqueue("apple-trackpad-haptics"); if (!haptic_dev->wq) { dev_err(&trackpad_hdev->dev, "Cannot allocate haptic workqueue\n"); kfree(msc->haptic_effects); From 541ee161eb36b0f20929648edf0d767642d93a78 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 20:58:39 +1100 Subject: [PATCH 0518/2680] HID: dockchannel: Wake the system on keyboard and trackpad input A key press or a trackpad click on the built-in keyboard of an M2 or later laptop does not wake it from suspend-to-idle. The MTP coprocessor keeps running and sends the report, but the DockChannel receive interrupt is not a wake interrupt, so it is disabled for the sleep like any other device interrupt and the report waits in the FIFO until something else wakes the system. Register the receive interrupt as the device's wake interrupt with dockchannel_init_wakeup(). Wakeup is enabled by default, as for other keyboards, and can be turned off through the device's power/wakeup attribute. Reports that arrive during sleep are processed after resume, as before. All interfaces share the FIFO, so any trackpad input wakes the system too, a touch as well as a click. On the J700 a key press, a trackpad click and a trackpad touch each end suspend-to-idle at once, with "Triggering wakeup from IRQ" naming the DockChannel receive interrupt. With the lid closed the machine stays asleep until the lid is opened. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index d5720bae6560d..133c97673176a 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -1376,6 +1376,17 @@ static int dockchannel_hid_probe(struct platform_device *pdev) if (IS_ERR_OR_NULL(dchid->dc)) { return PTR_ERR(dchid->dc); } + + /* + * Keyboard and trackpad input wakes the system. The coprocessor keeps + * running across suspend-to-idle and keeps sending reports through + * the FIFO, so its receive interrupt is the wake interrupt. Without + * it the machine still works, it just cannot be woken from here. + */ + ret = dockchannel_init_wakeup(dchid->dc); + if (ret) + dev_warn(dev, "Failed to set up wakeup: %d\n", ret); + dchid->new_iface_wq = alloc_ordered_workqueue("dchid-new", 0); if (!dchid->new_iface_wq) return -ENOMEM; From 85a3a632c0c4a737a3e30fde3295e38fd45dc857 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:23:53 +1100 Subject: [PATCH 0519/2680] Input: macsmc-input - unregister the SMC event notifier on unbind Probe adds a notifier block to the SMC's event chain, but nothing ever takes it off again. The block is embedded in devm-allocated driver data, so after an unbind the chain still links freed memory, and every SMC notification walks through it: a power button press, a lid change or a charger event is then a use-after-free, and once the module has been unloaded the chain calls into unloaded module text. A rebind can also reuse the freed memory for the new block, which zeroes the old link and cuts any handler registered after it, such as macsmc-power's, off the chain. Unregister the notifier from a devm action added right after the registration, so it comes off the chain before the input device and the driver data are released, and stop ignoring the registration result. Fixes: e89b625fdbce ("input: macsmc-input: New driver to handle the Apple Mac SMC buttons/lid") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/input/misc/macsmc-input.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/drivers/input/misc/macsmc-input.c b/drivers/input/misc/macsmc-input.c index 2cead3b7f45fe..e149b043fef82 100644 --- a/drivers/input/misc/macsmc-input.c +++ b/drivers/input/misc/macsmc-input.c @@ -106,6 +106,13 @@ static int macsmc_input_event(struct notifier_block *nb, unsigned long event, vo } } +static void macsmc_input_unregister_notifier(void *data) +{ + struct macsmc_input *smcin = data; + + blocking_notifier_chain_unregister(&smcin->smc->event_handlers, &smcin->nb); +} + static int macsmc_input_probe(struct platform_device *pdev) { struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent); @@ -168,7 +175,19 @@ static int macsmc_input_probe(struct platform_device *pdev) input_sync(smcin->input); smcin->nb.notifier_call = macsmc_input_event; - blocking_notifier_chain_register(&smc->event_handlers, &smcin->nb); + error = blocking_notifier_chain_register(&smc->event_handlers, &smcin->nb); + if (error) + return error; + + /* + * The notifier block lives in devm-allocated memory and the callback + * reports to a devm-managed input device: take it off the SMC's chain + * before either goes away on unbind. + */ + error = devm_add_action_or_reset(&pdev->dev, + macsmc_input_unregister_notifier, smcin); + if (error) + return error; device_init_wakeup(&pdev->dev, true); From a0d4687fc25402cf36ed4b0fb83fe2372e59ad3d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:09:59 +1100 Subject: [PATCH 0520/2680] HID: dockchannel: Drop reports that arrive before the device is open The MTP firmware starts sending reports for an interface as soon as the host enables it. dchid_create_interface_work() enables the interface first and only then allocates and registers the HID device, while the reports are handled on the interface's own workqueue. The keyboard sends its first report in that window on every boot, so the driver logs Report received but keyboard is not initialized! and reads iface->hid while the create work is still writing it. A report that arrives before the HID device is open has nobody to go to either way: the existing open check already dropped it once the device was registered. Make the open flag the only gate, so this expected state no longer warns, and use release/acquire on it so a reader that sees the device open also sees iface->hid. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 133c97673176a..2cb317e2bacb9 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -675,7 +675,8 @@ static int dchid_open(struct hid_device *hdev) } } - iface->open = true; + /* Pairs with the acquire in dchid_handle_report(); publishes iface->hid. */ + smp_store_release(&iface->open, true); return 0; } @@ -683,7 +684,7 @@ static void dchid_close(struct hid_device *hdev) { struct dchid_iface *iface = hdev->driver_data; - iface->open = false; + WRITE_ONCE(iface->open, false); } static int dchid_parse(struct hid_device *hdev) @@ -1063,14 +1064,12 @@ static void dchid_handle_event(struct dockchannel_hid *dchid, void *data, size_t static void dchid_handle_report(struct dchid_iface *iface, void *data, size_t length) { - struct dockchannel_hid *dchid = iface->dchid; - - if (!iface->hid) { - dev_warn(dchid->dev, "Report received but %s is not initialized!\n", iface->name); - return; - } - - if (!iface->open) + /* + * An interface starts reporting as soon as it is enabled, which is + * before its HID device has been registered and opened. Until then + * there is nobody to deliver the reports to. + */ + if (!smp_load_acquire(&iface->open)) return; hid_input_report(iface->hid, HID_INPUT_REPORT, data, length, 1); From b1436a3bf8633c34f9189bae859f9959b94915f4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:10:28 +1100 Subject: [PATCH 0521/2680] HID: dockchannel: Terminate log messages and fix two typos Eight messages lack a trailing newline, so the next message can be appended to the same console line, and two carry typos ("firmwareS", "expFected"). No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 2cb317e2bacb9..b079f6a9ded21 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -580,7 +580,7 @@ static int dchid_start_interface(struct dchid_iface *iface) int ret; if (iface->starting) { - dev_warn(iface->dchid->dev, "Interface %s is already starting", iface->name); + dev_warn(iface->dchid->dev, "Interface %s is already starting\n", iface->name); return -EINPROGRESS; } @@ -607,7 +607,7 @@ static int dchid_start_interface(struct dchid_iface *iface) dev_info(iface->dchid->dev, "Sending firmware for %s\n", iface->name); ret = dchid_send_firmware(iface, fw, size); if (ret < 0) { - dev_err(iface->dchid->dev, "Failed to send %s firmwareS", iface->name); + dev_err(iface->dchid->dev, "Failed to send %s firmware\n", iface->name); goto err; } @@ -640,7 +640,7 @@ static int dchid_start(struct hid_device *hdev) if (iface->keyboard_layout_id) { int ret = device_create_file(&hdev->dev, &dev_attr_apple_layout_id); if (ret) { - dev_warn(iface->dchid->dev, "Failed to create apple_layout_id: %d", ret); + dev_warn(iface->dchid->dev, "Failed to create apple_layout_id: %d\n", ret); iface->keyboard_layout_id = 0; } } @@ -802,7 +802,7 @@ static void dchid_create_interface_work(struct work_struct *ws) if (ret < 0) { iface->hid = NULL; hid_destroy_device(hid); - dev_warn(iface->dchid->dev, "Failed to register hid device %s", iface->name); + dev_warn(iface->dchid->dev, "Failed to register hid device %s\n", iface->name); } } @@ -1133,7 +1133,7 @@ static void dchid_handle_ack(struct dchid_iface *iface, struct dchid_hdr *hdr, v } if (shdr->flags != iface->out_flags) { dev_err(iface->dchid->dev, - "Received unexpected flags 0x%x on ACK channel (expFected 0x%x)\n", + "Received unexpected flags 0x%x on ACK channel (expected 0x%x)\n", shdr->flags, iface->out_flags); return; } @@ -1348,14 +1348,14 @@ static int dockchannel_hid_probe(struct platform_device *pdev) */ helper = of_parse_phandle(dev->of_node, "apple,helper-cpu", 0); if (!helper) { - dev_err(dev, "Missing apple,helper-cpu property"); + dev_err(dev, "Missing apple,helper-cpu property\n"); return -EINVAL; } helper_pdev = of_find_device_by_node(helper); of_node_put(helper); if (!helper_pdev) { - dev_err(dev, "Failed to find helper device"); + dev_err(dev, "Failed to find helper device\n"); return -EINVAL; } @@ -1363,7 +1363,7 @@ static int dockchannel_hid_probe(struct platform_device *pdev) DL_FLAG_AUTOREMOVE_CONSUMER); put_device(&helper_pdev->dev); if (!dchid->helper_link) { - dev_err(dev, "Failed to link to helper device"); + dev_err(dev, "Failed to link to helper device\n"); return -EINVAL; } @@ -1392,7 +1392,7 @@ static int dockchannel_hid_probe(struct platform_device *pdev) dchid->comm = dchid_get_interface(dchid, IFACE_COMM, "comm"); if (!dchid->comm) { - dev_err(dchid->dev, "Failed to initialize comm interface"); + dev_err(dchid->dev, "Failed to initialize comm interface\n"); destroy_workqueue(dchid->new_iface_wq); return -EIO; } From ccf288c657a95bcab59d4a24b1592340bd3400c6 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 9 Sep 2026 03:59:31 -0400 Subject: [PATCH 0522/2680] soc: apple: add Apple SEP driver (Touch ID and key sealing) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a Rust driver for the Apple SEP (Secure Enclave Processor) on Apple silicon Macs, built as the CONFIG_APPLE_SEP module apple-sep. It attaches to a running SEP over the AP mailbox, drives the Touch ID fingerprint sensor, and exposes the enclave's key services to Linux. The driver is organised by SEP endpoint: - sep.rs transport, endpoint bring-up, mailbox and probe (crate root) - sbio.rs Touch ID: enrol, verify and match over /dev/sep-bio - sks.rs the key store: key-bag recovery and lock state - refkey.rs machine ref-key sealing, signing and attestation - fv.rs the device-bound FileVault key hierarchy - proto.rs the SEP wire protocol Touch ID enrol and verify are exposed through a /dev/sep-bio character device; the enclave matches and no biometric image ever crosses to userspace. A SEP-backed trusted key source lets keyctl seal keys to the enclave, which needs the trusted-keys core change included here to register a runtime trusted-key source. The C shims bridge kernel interfaces whose ABI is awkward to restate in Rust (struct hwrng, the file and crypto APIs, the trusted-key framework); no protocol logic lives in them. [ryan: adapted to 7.2.10 — this rewrite replaces the body of Asahi's boot-only drivers/soc/apple/sep.rs, whose original commit and 7.2 fixups stay in the base; the driver-core probe uses the 7.2 lifetime-bound Core<'_>/Data<'bound> form, as Asahi's fixup does; trusted-type.h keeps mainline's CONFIG_TRUSTED_KEYS_DEBUG dump_payload()] Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 24 +- drivers/soc/apple/Makefile | 4 +- drivers/soc/apple/bio.rs | 914 ++++++ drivers/soc/apple/bio_shim.c | 203 ++ drivers/soc/apple/catacomb.rs | 73 + drivers/soc/apple/control.rs | 183 ++ drivers/soc/apple/crypto_shim.c | 294 ++ drivers/soc/apple/der.rs | 158 + drivers/soc/apple/dt.rs | 407 +++ drivers/soc/apple/fv.rs | 158 + drivers/soc/apple/hwrng.rs | 81 + drivers/soc/apple/hwrng_shim.c | 72 + drivers/soc/apple/image.rs | 243 ++ drivers/soc/apple/keybag.rs | 293 ++ drivers/soc/apple/p256_shim.c | 164 + drivers/soc/apple/proto.rs | 301 ++ drivers/soc/apple/refkey.rs | 275 ++ drivers/soc/apple/refkey_seal.rs | 237 ++ drivers/soc/apple/rxring.rs | 78 + drivers/soc/apple/sbio.rs | 3316 +++++++++++++++++++++ drivers/soc/apple/scrd.rs | 144 + drivers/soc/apple/seed.rs | 113 + drivers/soc/apple/sensor.rs | 705 +++++ drivers/soc/apple/sensor_shim.c | 401 +++ drivers/soc/apple/sep-bio.h | 127 + drivers/soc/apple/sep.rs | 2419 +++++++++++++-- drivers/soc/apple/sha_shim.c | 44 + drivers/soc/apple/shim.h | 144 + drivers/soc/apple/shim.rs | 231 ++ drivers/soc/apple/shmem.rs | 235 ++ drivers/soc/apple/sks.rs | 1115 +++++++ drivers/soc/apple/store.rs | 334 +++ drivers/soc/apple/store_shim.c | 105 + drivers/soc/apple/transfer.rs | 389 +++ drivers/soc/apple/trusted.rs | 268 ++ drivers/soc/apple/trusted_shim.c | 176 ++ drivers/soc/apple/xarm.rs | 365 +++ include/keys/trusted-type.h | 3 + security/keys/trusted-keys/trusted_core.c | 140 +- 39 files changed, 14634 insertions(+), 302 deletions(-) create mode 100644 drivers/soc/apple/bio.rs create mode 100644 drivers/soc/apple/bio_shim.c create mode 100644 drivers/soc/apple/catacomb.rs create mode 100644 drivers/soc/apple/control.rs create mode 100644 drivers/soc/apple/crypto_shim.c create mode 100644 drivers/soc/apple/der.rs create mode 100644 drivers/soc/apple/dt.rs create mode 100644 drivers/soc/apple/fv.rs create mode 100644 drivers/soc/apple/hwrng.rs create mode 100644 drivers/soc/apple/hwrng_shim.c create mode 100644 drivers/soc/apple/image.rs create mode 100644 drivers/soc/apple/keybag.rs create mode 100644 drivers/soc/apple/p256_shim.c create mode 100644 drivers/soc/apple/proto.rs create mode 100644 drivers/soc/apple/refkey.rs create mode 100644 drivers/soc/apple/refkey_seal.rs create mode 100644 drivers/soc/apple/rxring.rs create mode 100644 drivers/soc/apple/sbio.rs create mode 100644 drivers/soc/apple/scrd.rs create mode 100644 drivers/soc/apple/seed.rs create mode 100644 drivers/soc/apple/sensor.rs create mode 100644 drivers/soc/apple/sensor_shim.c create mode 100644 drivers/soc/apple/sep-bio.h create mode 100644 drivers/soc/apple/sha_shim.c create mode 100644 drivers/soc/apple/shim.h create mode 100644 drivers/soc/apple/shim.rs create mode 100644 drivers/soc/apple/shmem.rs create mode 100644 drivers/soc/apple/sks.rs create mode 100644 drivers/soc/apple/store.rs create mode 100644 drivers/soc/apple/store_shim.c create mode 100644 drivers/soc/apple/transfer.rs create mode 100644 drivers/soc/apple/trusted.rs create mode 100644 drivers/soc/apple/trusted_shim.c create mode 100644 drivers/soc/apple/xarm.rs diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 62f2f33e6b56c..0386028537565 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -95,6 +95,7 @@ config RUST_APPLE_RTKIT depends on PM depends on RUST select APPLE_RTKIT + select RUST_APPLE_MAILBOX config APPLE_AOP tristate "Apple \"Always-on\" Processor" @@ -109,17 +110,24 @@ config APPLE_AOP Say 'y' here if you have an Apple laptop. config APPLE_SEP - tristate "Apple Secure Element Processor" - depends on ARCH_APPLE || COMPILE_TEST - depends on PM + tristate "Apple SEP (Secure Enclave Processor)" + depends on ARCH_APPLE depends on RUST - select RUST_APPLE_RTKIT - select RUST_APPLE_MAILBOX + depends on HW_RANDOM + depends on TRUSTED_KEYS + depends on CRYPTO + select CRYPTO_HMAC + select CRYPTO_SHA256 + select CRYPTO_AES + select CRYPTO_GCM + select CRYPTO_ECDH help - A security co-processor persent on Apple SoCs, controlling transparent - disk encryption, secure boot, HDCP, biometric auth and probably more. + Driver for the Apple SEP (Secure Enclave Processor) on Apple silicon. + It drives the Touch ID fingerprint sensor over a /dev/sep-bio character + device (enrol and match), exposes the SEP hardware RNG, and registers a + SEP-backed trusted key source so keyctl can seal keys to the enclave. - Say 'y' here if you have an Apple SoC. + Say Y here if you have an Apple silicon Mac. config APPLE_PMP tristate "Apple Power Management Processor" diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 38b59bf10899f..b8cd56e097439 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -22,7 +22,9 @@ apple-tunable-y = tunable.o obj-$(CONFIG_APPLE_AOP) += aop.o -obj-$(CONFIG_APPLE_SEP) += sep.o +obj-$(CONFIG_APPLE_SEP) += apple-sep.o +apple-sep-y := sep.o hwrng_shim.o store_shim.o bio_shim.o sha_shim.o \ + crypto_shim.o sensor_shim.o p256_shim.o trusted_shim.o obj-$(CONFIG_APPLE_PMP) += pmp.o diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs new file mode 100644 index 0000000000000..27d7bd25d1110 --- /dev/null +++ b/drivers/soc/apple/bio.rs @@ -0,0 +1,914 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +use crate::store::{Key, Store}; +use kernel::ioctl::{_IO, _IOR, _IOW, _IOWR}; +use kernel::prelude::*; +use kernel::uaccess::{UserPtr, UserSlice}; + +pub(crate) const IFACE_VERSION: u32 = 4; + +pub(crate) const ENROL_USER_ID: i32 = 1000; + +pub(crate) const UUID_LEN: usize = 16; +const LABEL_LEN: usize = 128; +const NONCE_LEN: usize = 32; +pub(crate) const TOKEN_LEN: usize = 32; +const MAX_IDENTITIES: usize = 32; + +pub(crate) const DEVICE_NAME: &CStr = c"sep-bio"; +pub(crate) const DEVICE_MODE: u16 = 0o600; + +#[derive(Clone, Copy, Default)] +#[repr(u32)] +enum State { + #[default] + Idle = 0, + Pending = 1, + Progress = 2, + Done = 3, + Failed = 4, +} + +#[derive(Clone, Copy, Default, PartialEq)] +#[repr(u32)] +pub(crate) enum Guidance { + #[default] + None = 0, + Place = 1, + LiftAndMove = 2, + HoldStill = 3, +} + +#[derive(Clone, Copy, Default)] +#[repr(u32)] +enum MatchResult { + #[default] + NoMatch = 0, + Match = 1, + NotCompared = 2, +} + +pub(crate) const ENROL_STAGES: u32 = 8; + +const TOKEN_LIFETIME_NS: u64 = 10 * 1_000_000_000; + +const SUSPEND_SLACK_NS: u64 = 1_000_000; + +const STATUS_LOCAL: u32 = 0; + +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct Identity { + uuid: [u8; UUID_LEN], + label: [u8; LABEL_LEN], +} + +impl Identity { + const ZERO: Identity = Identity { + uuid: [0; UUID_LEN], + label: [0; LABEL_LEN], + }; +} + +#[repr(C)] +#[derive(Default)] +struct Info { + version: u32, + sensor_present: u32, + enrolled: u32, + capacity: u32, + enroll_stages: u32, + reserved: [u32; 3], +} + +#[repr(C)] +struct List { + count: u32, + reserved: u32, + id: [Identity; MAX_IDENTITIES], +} + +#[repr(C)] +struct EnrolStart { + flags: u32, + reserved: u32, + label: [u8; LABEL_LEN], +} + +#[repr(C)] +#[derive(Default)] +struct EnrolPoll { + state: State, + stage: u32, + stages_total: u32, + status: u32, + uuid: [u8; UUID_LEN], + guidance: Guidance, + progress_percent: u32, +} + +#[repr(C)] +struct VerifyStart { + flags: u32, + reserved: u32, + nonce: [u8; NONCE_LEN], +} + +#[repr(C)] +#[derive(Default)] +struct VerifyPoll { + state: State, + result: MatchResult, + status: u32, + reserved: u32, + uuid: [u8; UUID_LEN], + token: [u8; TOKEN_LEN], + deadline_ns: u64, +} + +#[repr(C)] +struct Delete { + uuid: [u8; UUID_LEN], +} + +pub(crate) const ATTEST_CHALLENGE_LEN: usize = 32; +/// Uncompressed P-256 public point (`04‖X‖Y`). +pub(crate) const ATTEST_PUB_LEN: usize = 65; +/// Max DER `SEQUENCE { INTEGER r, INTEGER s }` for P-256. +pub(crate) const ATTEST_SIG_MAX: usize = 72; + +#[repr(C)] +pub(crate) struct Attest { + pub(crate) sig_len: u32, + /// in: challenge to sign. + pub(crate) challenge: [u8; ATTEST_CHALLENGE_LEN], + /// out: ref-key public point. + pub(crate) public: [u8; ATTEST_PUB_LEN], + /// out: DER signature, zero-padded to `ATTEST_SIG_MAX`. + pub(crate) signature: [u8; ATTEST_SIG_MAX], + pub(crate) reserved: [u8; 3], +} + +// Sizes are baked into the ioctl numbers. +static_assert!(core::mem::size_of::() == 144); +static_assert!(core::mem::size_of::() == 32); +static_assert!(core::mem::size_of::() == 4616); +static_assert!(core::mem::size_of::() == 136); +static_assert!(core::mem::size_of::() == 40); +static_assert!(core::mem::size_of::() == 40); +static_assert!(core::mem::size_of::() == 72); +static_assert!(core::mem::size_of::() == 16); +static_assert!(core::mem::size_of::() == 176); + +macro_rules! ioctl_pod { + (to_user: $($t:ty),*; from_user: $($u:ty),*) => { + $( + // SAFETY: no padding, so the value is a faithful byte image. + unsafe impl kernel::transmute::AsBytes for $t {} + )* + $( + // SAFETY: every bit pattern of the members is a valid value. + unsafe impl kernel::transmute::FromBytes for $u {} + )* + }; +} +ioctl_pod! { + to_user: Info, List, EnrolPoll, VerifyPoll, Attest; + from_user: EnrolStart, VerifyStart, Delete, Attest +} + +const MAGIC: u32 = 0xB1; + +const IOC_GET_INFO: u32 = _IOR::(MAGIC, 0x01); +const IOC_LIST: u32 = _IOR::(MAGIC, 0x02); +const IOC_ENROL_START: u32 = _IOW::(MAGIC, 0x03); +const IOC_ENROL_POLL: u32 = _IOR::(MAGIC, 0x04); +const IOC_VERIFY_START: u32 = _IOW::(MAGIC, 0x05); +const IOC_VERIFY_POLL: u32 = _IOR::(MAGIC, 0x06); +const IOC_CANCEL: u32 = _IO(MAGIC, 0x07); +const IOC_DELETE: u32 = _IOW::(MAGIC, 0x08); +const IOC_DELETE_ALL: u32 = _IO(MAGIC, 0x09); +pub(crate) const IOC_ATTEST: u32 = _IOWR::(MAGIC, 0x0a); + +pub(crate) struct MatchEvidence { + identity: [u8; UUID_LEN], +} + +impl MatchEvidence { + pub(crate) fn from_enclave_reply(identity: [u8; UUID_LEN]) -> MatchEvidence { + MatchEvidence { identity } + } +} + +pub(crate) enum VerifyOutcome { + Failed(u32), + NoMatch, + Matched(MatchEvidence), +} + +struct ResultToken { + bytes: [u8; TOKEN_LEN], + nonce: [u8; NONCE_LEN], + identity: [u8; UUID_LEN], + deadline_ns: u64, + minted_mono_ns: u64, + minted_boot_ns: u64, +} + +impl ResultToken { + fn mint( + evidence: &MatchEvidence, + nonce: &[u8; NONCE_LEN], + bytes: [u8; TOKEN_LEN], + ) -> ResultToken { + let mono = crate::shim::monotonic_ns(); + ResultToken { + bytes, + nonce: *nonce, + identity: evidence.identity, + deadline_ns: mono.saturating_add(TOKEN_LIFETIME_NS), + minted_mono_ns: mono, + minted_boot_ns: crate::shim::boottime_ns(), + } + } + + fn usable(&self, nonce: &[u8; NONCE_LEN], identity: &[u8; UUID_LEN]) -> bool { + let mono = crate::shim::monotonic_ns(); + if mono > self.deadline_ns { + return false; + } + let suspended = crate::shim::boottime_ns().saturating_sub(self.minted_boot_ns); + let awake = mono.saturating_sub(self.minted_mono_ns); + if suspended.saturating_sub(awake) > SUSPEND_SLACK_NS { + return false; + } + self.nonce == *nonce && self.identity == *identity + } +} + +const PRIVATE_TYPE_IDENTITIES: u8 = 0xF1; + +fn identity_key() -> Key { + Key::root(PRIVATE_TYPE_IDENTITIES) +} + +const INDEX_FORMAT: u32 = 2; + +pub(crate) struct IdentityIndex { + entries: KVec, +} + +impl IdentityIndex { + pub(crate) fn new() -> IdentityIndex { + IdentityIndex { + entries: KVec::new(), + } + } + + pub(crate) fn load(store: &mut Store) -> Result { + let mut index = IdentityIndex::new(); + let Some(raw) = store.read(&identity_key())? else { + return Ok(index); + }; + if raw.len() < 4 { + return Ok(index); + } + if u32::from_le_bytes([raw[0], raw[1], raw[2], raw[3]]) != INDEX_FORMAT { + return Err(EINVAL); + } + if raw.len() < 8 { + return Ok(index); + } + let count = u32::from_le_bytes([raw[4], raw[5], raw[6], raw[7]]) as usize; + let stride = UUID_LEN + LABEL_LEN; + for i in 0..count.min(MAX_IDENTITIES) { + let at = 8 + i * stride; + if at + stride > raw.len() { + break; + } + let mut entry = Identity::ZERO; + entry.uuid.copy_from_slice(&raw[at..at + UUID_LEN]); + entry + .label + .copy_from_slice(&raw[at + UUID_LEN..at + stride]); + index.entries.push(entry, GFP_KERNEL)?; + } + Ok(index) + } + + fn save(&self, store: &mut Store) -> Result<()> { + let mut raw = KVec::new(); + raw.extend_from_slice(&INDEX_FORMAT.to_le_bytes(), GFP_KERNEL)?; + raw.extend_from_slice(&(self.entries.len() as u32).to_le_bytes(), GFP_KERNEL)?; + for entry in self.entries.iter() { + raw.extend_from_slice(&entry.uuid, GFP_KERNEL)?; + raw.extend_from_slice(&entry.label, GFP_KERNEL)?; + } + store.write(&identity_key(), &raw) + } + + pub(crate) fn total(&self) -> usize { + self.entries.len() + } + + pub(crate) fn insert(&mut self, id: Identity) -> Result<()> { + for existing in self.entries.iter_mut() { + if existing.uuid == id.uuid { + *existing = id; + return Ok(()); + } + } + if self.entries.len() >= MAX_IDENTITIES { + return Err(ENOSPC); + } + self.entries.push(id, GFP_KERNEL)?; + Ok(()) + } + + pub(crate) fn persist(&self, store: &mut Store) -> Result<()> { + self.save(store) + } + + pub(crate) fn identity_v1_for( + &self, + uuid: &[u8; UUID_LEN], + user_id: i32, + ) -> Option { + if !self.contains(uuid) { + return None; + } + Some(crate::sbio::IdentityV1::from_index(user_id, *uuid)) + } + + pub(crate) fn reconcile_to( + &mut self, + listed: &[[u8; UUID_LEN]], + ) -> Result> { + let mut dropped = KVec::new(); + for entry in self.entries.iter() { + if !listed.contains(&entry.uuid) { + dropped.push(entry.uuid, GFP_KERNEL)?; + } + } + self.entries.retain(|e| listed.contains(&e.uuid)); + for uuid in listed.iter() { + if !self.contains(uuid) { + self.insert(Identity { + uuid: *uuid, + label: [0u8; LABEL_LEN], + })?; + } + } + Ok(dropped) + } + + pub(crate) fn remove(&mut self, uuid: &[u8; UUID_LEN]) { + self.entries.retain(|e| e.uuid != *uuid); + } + + pub(crate) fn contains_uuid(&self, uuid: &[u8; UUID_LEN]) -> bool { + self.contains(uuid) + } + + fn contains(&self, uuid: &[u8; UUID_LEN]) -> bool { + self.entries.iter().any(|e| e.uuid == *uuid) + } +} + +enum Op { + Idle, + Enrol { + label: [u8; LABEL_LEN], + stage: u32, + terminal: Option, + guidance: Guidance, + percent: u32, + }, + Verify { + nonce: [u8; NONCE_LEN], + terminal: Option, + }, +} + +enum EnrolOutcome { + Failed(u32), + Done([u8; UUID_LEN]), +} + +impl Op { + fn is_terminal(&self) -> bool { + match self { + Op::Idle => false, + Op::Enrol { terminal, .. } => terminal.is_some(), + Op::Verify { terminal, .. } => terminal.is_some(), + } + } + + fn may_start(&self) -> bool { + matches!(self, Op::Idle) || self.is_terminal() + } +} + +pub(crate) struct Session { + open: bool, + op: Op, + unseen: bool, + token: Option, +} + +impl Session { + pub(crate) fn new() -> Session { + Session { + open: false, + op: Op::Idle, + unseen: false, + token: None, + } + } + + fn reset(&mut self) { + self.open = false; + self.op = Op::Idle; + self.unseen = false; + self.token = None; + } + + pub(crate) fn ready(&self) -> bool { + self.unseen + } +} + +pub(crate) struct Context<'a> { + pub(crate) session: &'a mut Session, + pub(crate) index: &'a IdentityIndex, + pub(crate) sensor_present: bool, +} + +pub(crate) struct Handled { + pub(crate) ret: isize, + pub(crate) wake: bool, + pub(crate) start_enrol: bool, + pub(crate) start_verify: bool, + pub(crate) delete_identity: Option, + pub(crate) delete_identities: KVec, +} + +fn ok() -> Result { + Ok(Handled { + ret: 0, + wake: false, + start_enrol: false, + start_verify: false, + delete_identity: None, + delete_identities: KVec::new(), + }) +} + +fn require_admin() -> Result<()> { + if crate::shim::capable_admin() { + Ok(()) + } else { + Err(EPERM) + } +} + +fn require_sensor(present: bool) -> Result<()> { + if present { + Ok(()) + } else { + Err(ENODEV) + } +} + +pub(crate) fn ioctl(ctx: &mut Context<'_>, cmd: u32, arg: usize) -> Result { + let user = UserPtr::from_addr(arg); + + match cmd { + IOC_GET_INFO => get_info(ctx, user), + IOC_LIST => list(ctx, user), + IOC_ENROL_START => enrol_start(ctx, user), + IOC_ENROL_POLL => enrol_poll(ctx, user), + IOC_VERIFY_START => verify_start(ctx, user), + IOC_VERIFY_POLL => verify_poll(ctx, user), + IOC_CANCEL => cancel(ctx), + IOC_DELETE => delete(ctx, user), + IOC_DELETE_ALL => delete_all(ctx), + _ => Err(ENOTTY), + } +} + +fn get_info(ctx: &mut Context<'_>, user: UserPtr) -> Result { + let info = Info { + version: IFACE_VERSION, + sensor_present: u32::from(ctx.sensor_present), + enrolled: ctx.index.total() as u32, + capacity: MAX_IDENTITIES as u32, + enroll_stages: ENROL_STAGES, + reserved: [0; 3], + }; + UserSlice::new(user, core::mem::size_of::()) + .writer() + .write(&info)?; + ok() +} + +fn list(ctx: &mut Context<'_>, user: UserPtr) -> Result { + let mut out = List { + count: 0, + reserved: 0, + id: [Identity::ZERO; MAX_IDENTITIES], + }; + for entry in ctx.index.entries.iter() { + if out.count as usize >= MAX_IDENTITIES { + break; + } + out.id[out.count as usize] = *entry; + out.count += 1; + } + + UserSlice::new(user, core::mem::size_of::()) + .writer() + .write(&out)?; + ok() +} + +fn enrol_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { + require_admin()?; + let request: EnrolStart = UserSlice::new(user, core::mem::size_of::()) + .reader() + .read()?; + if request.flags != 0 || request.reserved != 0 { + return Err(EINVAL); + } + require_sensor(ctx.sensor_present)?; + + if !ctx.session.op.may_start() { + return Err(EBUSY); + } + if ctx.index.total() >= MAX_IDENTITIES { + return Err(ENOSPC); + } + + ctx.session.op = Op::Enrol { + label: request.label, + stage: 0, + terminal: None, + guidance: Guidance::Place, + percent: 0, + }; + Ok(Handled { + ret: 0, + wake: false, + start_enrol: true, + start_verify: false, + delete_identity: None, + delete_identities: KVec::new(), + }) +} + +pub(crate) fn enrol_advance( + session: &mut Session, + completed: u32, + percent_now: u32, + guide: Guidance, +) -> bool { + if let Op::Enrol { + stage, + terminal, + guidance, + percent, + .. + } = &mut session.op + { + if terminal.is_none() { + *stage = completed; + *percent = percent_now; + *guidance = guide; + session.unseen = true; + return true; + } + } + false +} + +pub(crate) fn enrol_guide(session: &mut Session, guide: Guidance) -> bool { + if let Op::Enrol { + terminal, guidance, .. + } = &mut session.op + { + if terminal.is_none() && *guidance != guide { + *guidance = guide; + session.unseen = true; + return true; + } + } + false +} + +pub(crate) fn enrol_finish( + session: &mut Session, + index: &mut IdentityIndex, + outcome: core::result::Result<[u8; UUID_LEN], u32>, +) -> bool { + let Op::Enrol { + label, terminal, .. + } = &mut session.op + else { + return false; + }; + if terminal.is_some() { + return false; + } + match outcome { + Ok(uuid) => { + match index.insert(Identity { + uuid, + label: *label, + }) { + Ok(()) => *terminal = Some(EnrolOutcome::Done(uuid)), + Err(_) => *terminal = Some(EnrolOutcome::Failed(STATUS_LOCAL)), + } + } + Err(status) => *terminal = Some(EnrolOutcome::Failed(status)), + } + session.unseen = true; + true +} + +pub(crate) fn enrol_is_live(session: &Session) -> bool { + matches!(&session.op, Op::Enrol { terminal: None, .. }) && session.open +} + +fn enrol_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { + let mut out = EnrolPoll { + stages_total: ENROL_STAGES, + ..EnrolPoll::default() + }; + + match &ctx.session.op { + Op::Idle => out.state = State::Idle, + Op::Verify { .. } => return Err(EBUSY), + Op::Enrol { + stage, + terminal, + guidance, + percent, + .. + } => { + out.stage = *stage; + out.guidance = *guidance; + out.progress_percent = *percent; + match terminal { + None => { + out.state = if ctx.session.unseen { + State::Progress + } else { + State::Pending + } + } + Some(EnrolOutcome::Failed(status)) => { + out.state = State::Failed; + out.status = *status; + } + Some(EnrolOutcome::Done(uuid)) => { + out.state = State::Done; + out.uuid = *uuid; + } + } + } + } + + UserSlice::new(user, core::mem::size_of::()) + .writer() + .write(&out)?; + ctx.session.unseen = false; + ok() +} + +fn verify_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { + let request: VerifyStart = UserSlice::new(user, core::mem::size_of::()) + .reader() + .read()?; + if request.flags != 0 || request.reserved != 0 { + return Err(EINVAL); + } + require_sensor(ctx.sensor_present)?; + + if !ctx.session.op.may_start() { + return Err(EBUSY); + } + + if ctx.index.total() == 0 { + ctx.session.token = None; + ctx.session.op = Op::Verify { + nonce: request.nonce, + terminal: Some(VerifyOutcome::NoMatch), + }; + return Ok(Handled { + ret: 0, + wake: true, + start_enrol: false, + start_verify: false, + delete_identity: None, + delete_identities: KVec::new(), + }); + } + + ctx.session.token = None; + ctx.session.op = Op::Verify { + nonce: request.nonce, + terminal: None, + }; + Ok(Handled { + ret: 0, + wake: false, + start_enrol: false, + start_verify: true, + delete_identity: None, + delete_identities: KVec::new(), + }) +} + +pub(crate) fn verify_finish( + session: &mut Session, + outcome: VerifyOutcome, + token_bytes: [u8; TOKEN_LEN], +) -> bool { + let Op::Verify { nonce, terminal } = &mut session.op else { + return false; + }; + if terminal.is_some() { + return false; + } + + let minted = match &outcome { + VerifyOutcome::Matched(evidence) => Some(ResultToken::mint(evidence, nonce, token_bytes)), + VerifyOutcome::NoMatch | VerifyOutcome::Failed(_) => None, + }; + + *terminal = Some(outcome); + session.token = minted; + session.unseen = true; + true +} + +pub(crate) fn verify_is_live(session: &Session) -> bool { + matches!(&session.op, Op::Verify { terminal: None, .. }) && session.open +} + +pub(crate) fn capture_is_live(session: &Session) -> bool { + enrol_is_live(session) || verify_is_live(session) +} + +fn verify_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { + let mut out = VerifyPoll::default(); + // Spend the token only after the copy to user succeeds; a fault must not lose a match. + let mut consume_token = false; + + match &ctx.session.op { + Op::Idle => out.state = State::Idle, + Op::Enrol { .. } => return Err(EBUSY), + Op::Verify { nonce, terminal } => match terminal { + None => { + out.state = if ctx.session.unseen { + State::Progress + } else { + State::Pending + } + } + Some(VerifyOutcome::Failed(status)) => { + out.state = State::Failed; + out.result = MatchResult::NotCompared; + out.status = *status; + } + Some(VerifyOutcome::NoMatch) => { + out.state = State::Done; + out.result = MatchResult::NoMatch; + } + Some(VerifyOutcome::Matched(evidence)) => { + out.state = State::Done; + match ctx.session.token.as_ref() { + Some(token) if token.usable(nonce, &evidence.identity) => { + out.result = MatchResult::Match; + out.uuid = evidence.identity; + out.token = token.bytes; + out.deadline_ns = token.deadline_ns; + consume_token = true; + } + _ => { + out.result = MatchResult::NotCompared; + } + } + } + }, + } + + UserSlice::new(user, core::mem::size_of::()) + .writer() + .write(&out)?; + if consume_token { + ctx.session.token = None; + } + ctx.session.unseen = false; + ok() +} + +fn cancel(ctx: &mut Context<'_>) -> Result { + if ctx.session.op.is_terminal() { + return Err(ENOENT); + } + + match &mut ctx.session.op { + Op::Idle => return Err(ENOENT), + Op::Enrol { terminal, .. } => *terminal = Some(EnrolOutcome::Failed(STATUS_LOCAL)), + Op::Verify { terminal, .. } => *terminal = Some(VerifyOutcome::Failed(STATUS_LOCAL)), + } + + ctx.session.token = None; + ctx.session.unseen = true; + + Ok(Handled { + ret: 0, + wake: true, + start_enrol: false, + start_verify: false, + delete_identity: None, + delete_identities: KVec::new(), + }) +} + +fn delete(ctx: &mut Context<'_>, user: UserPtr) -> Result { + require_admin()?; + let request: Delete = UserSlice::new(user, core::mem::size_of::()) + .reader() + .read()?; + + if !ctx.index.contains(&request.uuid) { + pr_info!( + "sep_bio: DELETE of an identity not held; reporting success (nothing to delete)\n" + ); + return ok(); + } + + // `0x57` takes one `identity_v1_t` (signed user id + 16-byte UUID). + let Some(identity) = ctx.index.identity_v1_for(&request.uuid, ENROL_USER_ID) else { + pr_warn!( + "sep_bio: DELETE of a held identity could not be expressed as an identity_v1_t\n" + ); + return Err(ENOENT); + }; + + Ok(Handled { + ret: 0, + wake: false, + start_enrol: false, + start_verify: false, + delete_identity: Some(identity), + delete_identities: KVec::new(), + }) +} + +fn delete_all(ctx: &mut Context<'_>) -> Result { + require_admin()?; + + if ctx.index.total() == 0 { + pr_info!( + "sep_bio: DELETE_ALL on empty index; reporting success (nothing to delete)\n" + ); + return ok(); + } + + let mut ids = KVec::new(); + for entry in ctx.index.entries.iter() { + let Some(identity) = ctx.index.identity_v1_for(&entry.uuid, ENROL_USER_ID) else { + pr_warn!( + "sep_bio: DELETE_ALL of a held identity could not be expressed as an identity_v1_t\n" + ); + return Err(ENOENT); + }; + ids.push(identity, GFP_KERNEL)?; + } + + Ok(Handled { + ret: 0, + wake: false, + start_enrol: false, + start_verify: false, + delete_identity: None, + delete_identities: ids, + }) +} + +pub(crate) fn open(session: &mut Session) -> Result<()> { + if session.open { + return Err(EBUSY); + } + session.reset(); + session.open = true; + Ok(()) +} + +pub(crate) fn release(session: &mut Session) { + session.reset(); +} diff --git a/drivers/soc/apple/bio_shim.c b/drivers/soc/apple/bio_shim.c new file mode 100644 index 0000000000000..50f7e7ddf61ea --- /dev/null +++ b/drivers/soc/apple/bio_shim.c @@ -0,0 +1,203 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "shim.h" + +struct sep_bio_chardev { + struct miscdevice misc; + wait_queue_head_t wq; + struct mutex lock; + void *ctx; /* NULL once detached */ + int (*f_open)(void *ctx); + void (*f_release)(void *ctx); + long (*f_ioctl)(void *ctx, unsigned int cmd, unsigned long arg); + int (*f_ready)(void *ctx); + bool registered; + int open_count; +}; + +static struct sep_bio_chardev *from_file(struct file *file) +{ + return container_of(file->private_data, struct sep_bio_chardev, misc); +} + +static int sep_bio_fop_open(struct inode *inode, struct file *file) +{ + struct sep_bio_chardev *d = from_file(file); + int ret; + + mutex_lock(&d->lock); + if (!d->ctx) { + mutex_unlock(&d->lock); + return -ENODEV; + } + ret = d->f_open(d->ctx); + if (!ret) + d->open_count++; + mutex_unlock(&d->lock); + + return ret; +} + +static int sep_bio_fop_release(struct inode *inode, struct file *file) +{ + struct sep_bio_chardev *d = from_file(file); + bool last; + + mutex_lock(&d->lock); + if (d->ctx) + d->f_release(d->ctx); + d->open_count--; + last = (d->open_count == 0) && !d->ctx; + mutex_unlock(&d->lock); + + /* Detached while this file was open: nobody else can reach it now. */ + if (last) { + mutex_destroy(&d->lock); + kfree(d); + } + + return 0; +} + +static long sep_bio_fop_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct sep_bio_chardev *d = from_file(file); + long ret; + + mutex_lock(&d->lock); + ret = d->ctx ? d->f_ioctl(d->ctx, cmd, arg) : -ENODEV; + mutex_unlock(&d->lock); + + return ret; +} + +/* + * Readable when a *_POLL ioctl has something the caller has not seen yet; quiet + * once that ioctl consumes it. + */ +static __poll_t sep_bio_fop_poll(struct file *file, + struct poll_table_struct *wait) +{ + struct sep_bio_chardev *d = from_file(file); + + __poll_t mask; + + poll_wait(file, &d->wq, wait); + + mutex_lock(&d->lock); + if (!d->ctx) + mask = EPOLLERR | EPOLLHUP; + else + mask = d->f_ready(d->ctx) ? (EPOLLIN | EPOLLRDNORM) : 0; + mutex_unlock(&d->lock); + + return mask; +} + +static const struct file_operations sep_bio_fops = { + .owner = THIS_MODULE, + .open = sep_bio_fop_open, + .release = sep_bio_fop_release, + .unlocked_ioctl = sep_bio_fop_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .poll = sep_bio_fop_poll, +}; + +/* @name must outlive the registration; the Rust side passes a &'static CStr. */ +void *sep_bio_register(const char *name, unsigned short mode, void *ctx, + int (*f_open)(void *), + void (*f_release)(void *), + long (*f_ioctl)(void *, unsigned int, unsigned long), + int (*f_ready)(void *)) +{ + struct sep_bio_chardev *d; + int ret; + + d = kzalloc(sizeof(*d), GFP_KERNEL); + if (!d) + return NULL; + + init_waitqueue_head(&d->wq); + mutex_init(&d->lock); + d->ctx = ctx; + d->f_open = f_open; + d->f_release = f_release; + d->f_ioctl = f_ioctl; + d->f_ready = f_ready; + + d->misc.minor = MISC_DYNAMIC_MINOR; + d->misc.name = name; + d->misc.fops = &sep_bio_fops; + d->misc.mode = mode; + + ret = misc_register(&d->misc); + if (ret) { + mutex_destroy(&d->lock); + kfree(d); + return NULL; + } + d->registered = true; + + return d; +} + +void sep_bio_unregister(void *dev) +{ + struct sep_bio_chardev *d = dev; + bool free_now; + + if (!d) + return; + + if (d->registered) + misc_deregister(&d->misc); + + mutex_lock(&d->lock); + d->ctx = NULL; + free_now = (d->open_count == 0); + mutex_unlock(&d->lock); + + wake_up_interruptible(&d->wq); + + if (free_now) { + mutex_destroy(&d->lock); + kfree(d); + } +} + +void sep_bio_wake(void *dev) +{ + struct sep_bio_chardev *d = dev; + + if (d) + wake_up_interruptible(&d->wq); +} + +/* In C so the CAP_SYS_ADMIN number stays a kernel header constant, not a Rust literal. */ +int sep_bio_capable_admin(void) +{ + return capable(CAP_SYS_ADMIN) ? 1 : 0; +} + +__u64 sep_bio_monotonic_ns(void) +{ + return ktime_get_ns(); +} + +__u64 sep_bio_boottime_ns(void) +{ + return ktime_get_boottime_ns(); +} diff --git a/drivers/soc/apple/catacomb.rs b/drivers/soc/apple/catacomb.rs new file mode 100644 index 0000000000000..8b18efb4ead77 --- /dev/null +++ b/drivers/soc/apple/catacomb.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! File-backed persistence for the SEP catacombs. +//! +//! A catacomb is the opaque identity blob the enclave returns from `0x6c`; each +//! lives in its own variable-length file, too large for the block store. + +use crate::shim; +use crate::store::crc16_ccitt_false; +use kernel::prelude::*; + +const MAGIC: [u8; 8] = *b"SEPCTMB1"; +/// magic(8) + len(4, LE) + crc(2, LE) + pad(2) +const HEADER: usize = 16; +const MAX_CATACOMB: usize = 1 << 20; + +fn path_for(kind: u8) -> Option<&'static CStr> { + Some(match kind { + crate::PRIVATE_TYPE_CATACOMB_MASTER => c"/var/lib/apple-sep-catacomb-master.bin", + crate::PRIVATE_TYPE_CATACOMB_OWNER => c"/var/lib/apple-sep-catacomb-owner.bin", + crate::PRIVATE_TYPE_CATACOMB_USER => c"/var/lib/apple-sep-catacomb-user.bin", + _ => return None, + }) +} + +pub(crate) fn is_kind(kind: u8) -> bool { + path_for(kind).is_some() +} + +pub(crate) fn write(kind: u8, blob: &[u8]) -> Result<()> { + let path = path_for(kind).ok_or(EINVAL)?; + if blob.len() > MAX_CATACOMB { + return Err(ENOSPC); + } + let mut head = [0u8; HEADER]; + head[..MAGIC.len()].copy_from_slice(&MAGIC); + head[8..12].copy_from_slice(&(blob.len() as u32).to_le_bytes()); + head[12..14].copy_from_slice(&crc16_ccitt_false(blob).to_le_bytes()); + + let file = shim::StoreFile::open_trunc(path)?; + file.write_all(0, &head)?; + if !blob.is_empty() { + file.write_all(HEADER as u64, blob)?; + } + file.sync() +} + +pub(crate) fn read(kind: u8) -> Option> { + let path = path_for(kind)?; + let file = shim::StoreFile::open_readonly(path).ok()?; + let size = file.size().ok()?; + if size < HEADER as u64 { + return None; + } + let mut head = [0u8; HEADER]; + file.read_exact(0, &mut head).ok()?; + if head[..MAGIC.len()] != MAGIC { + return None; + } + let len = u32::from_le_bytes([head[8], head[9], head[10], head[11]]) as usize; + let crc = u16::from_le_bytes([head[12], head[13]]); + if len == 0 || len > MAX_CATACOMB || size != (HEADER + len) as u64 { + return None; + } + let mut blob: KVec = KVec::with_capacity(len, GFP_KERNEL).ok()?; + blob.resize(len, 0, GFP_KERNEL).ok()?; + file.read_exact(HEADER as u64, &mut blob).ok()?; + if crc16_ccitt_false(&blob) != crc { + return None; + } + Some(blob) +} diff --git a/drivers/soc/apple/control.rs b/drivers/soc/apple/control.rs new file mode 100644 index 0000000000000..be81aa1de38df --- /dev/null +++ b/drivers/soc/apple/control.rs @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Control-endpoint bookkeeping: the in-flight request table, the tag +//! allocator, and the reserved-tag entropy sink. + +use crate::proto; +use kernel::prelude::*; + +const MAX_INFLIGHT: usize = 8; + +const POOL_LEN: usize = (proto::TAG_POOL_LAST - proto::TAG_POOL_FIRST + 1) as usize; + +#[derive(Clone, Copy)] +struct Slot { + used: bool, + tag: u8, + reply: Option, +} + +impl Slot { + const FREE: Slot = Slot { + used: false, + tag: 0, + reply: None, + }; +} + +struct EntropySink { + value: Option, + msg1: u32, + unclaimed: u32, + busy: bool, +} + +pub(crate) enum Delivery { + Entropy, + Matched, + Unmatched, +} + +pub(crate) struct ControlState { + slots: [Slot; MAX_INFLIGHT], + next_tag: u8, + retired: [u64; 4], + retired_count: u32, + // SEP-initiated messages (type != 0x01). + unsolicited: u32, + entropy: EntropySink, + // After the persistent-state exchange the SEP parks this endpoint; refuse rather than time out. + closed: bool, +} + +impl ControlState { + pub(crate) fn new() -> Self { + ControlState { + slots: [Slot::FREE; MAX_INFLIGHT], + next_tag: proto::TAG_POOL_FIRST, + retired: [0; 4], + retired_count: 0, + unsolicited: 0, + entropy: EntropySink { + value: None, + msg1: 0, + unclaimed: 0, + busy: false, + }, + closed: false, + } + } + + fn is_retired(&self, tag: u8) -> bool { + self.retired[(tag >> 6) as usize] & (1u64 << (tag & 0x3f)) != 0 + } + + fn retire(&mut self, tag: u8) { + if !self.is_retired(tag) { + self.retired[(tag >> 6) as usize] |= 1u64 << (tag & 0x3f); + self.retired_count += 1; + } + } + + fn tag_in_flight(&self, tag: u8) -> bool { + self.slots.iter().any(|s| s.used && s.tag == tag) + } + + pub(crate) fn alloc(&mut self) -> Result<(usize, u8)> { + if self.closed { + return Err(EPIPE); + } + let idx = self.slots.iter().position(|s| !s.used).ok_or(EBUSY)?; + + for _ in 0..POOL_LEN { + let tag = self.next_tag; + self.next_tag = if tag >= proto::TAG_POOL_LAST { + proto::TAG_POOL_FIRST + } else { + tag + 1 + }; + if !self.tag_in_flight(tag) && !self.is_retired(tag) { + self.slots[idx] = Slot { + used: true, + tag, + reply: None, + }; + return Ok((idx, tag)); + } + } + Err(EBUSY) + } + + pub(crate) fn take_reply(&mut self, idx: usize) -> Option { + self.slots[idx].reply.take() + } + + pub(crate) fn release(&mut self, idx: usize) { + self.slots[idx] = Slot::FREE; + } + + // Retire the tag so a late reply cannot be mistaken for a later request's answer. + pub(crate) fn abandon(&mut self, idx: usize) -> u8 { + let tag = self.slots[idx].tag; + self.retire(tag); + self.slots[idx] = Slot::FREE; + tag + } + + pub(crate) fn retired_count(&self) -> u32 { + self.retired_count + } + + pub(crate) fn deliver(&mut self, reply: proto::ControlReply) -> Delivery { + // The reserved entropy tag is claimed first, outstanding request or not. + if reply.tag == proto::TAG_ENTROPY { + if self.entropy.value.is_some() { + self.entropy.unclaimed = self.entropy.unclaimed.wrapping_add(1); + } + self.entropy.value = Some(reply.data_lo); + self.entropy.msg1 = reply.msg1; + return Delivery::Entropy; + } + + if let Some(slot) = self + .slots + .iter_mut() + .find(|s| s.used && s.tag == reply.tag && s.reply.is_none()) + { + slot.reply = Some(reply); + return Delivery::Matched; + } + + // No match: drop it, never hand it to another waiter. + Delivery::Unmatched + } + + pub(crate) fn note_unsolicited(&mut self) -> u32 { + self.unsolicited = self.unsolicited.wrapping_add(1); + self.unsolicited + } + + pub(crate) fn entropy_begin(&mut self) -> Result<()> { + if self.closed { + return Err(EPIPE); + } + if self.entropy.busy { + return Err(EBUSY); + } + self.entropy.busy = true; + if self.entropy.value.take().is_some() { + self.entropy.unclaimed = self.entropy.unclaimed.wrapping_add(1); + } + Ok(()) + } + + pub(crate) fn entropy_take(&mut self) -> Option<(u32, u32)> { + self.entropy.value.take().map(|v| (v, self.entropy.msg1)) + } + + pub(crate) fn entropy_end(&mut self) { + self.entropy.busy = false; + } + +} diff --git a/drivers/soc/apple/crypto_shim.c b/drivers/soc/apple/crypto_shim.c new file mode 100644 index 0000000000000..e4d0863cce01b --- /dev/null +++ b/drivers/soc/apple/crypto_shim.c @@ -0,0 +1,294 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "shim.h" + +#define SEP_GCM_TAG_LEN 16 + +/* + * AES-GCM with a 16-byte (non-96-bit) IV, over the raw AES block cipher. The + * kernel's gcm(aes) only takes a 12-byte IV; the SEP's ECIES uses a 16-byte IV, + * whose J0 must be derived as GHASH_H(IV ‖ len-block), not IV‖0x00000001. + */ + +/* GF(2^128) multiply, GCM bit convention (big-endian block, R = 0xe1‖0^120). */ +static void sep_gf_mult(const u8 *X, const u8 *Y, u8 *out) +{ + u8 Z[16] = {0}; + u8 V[16]; + int i, j; + + memcpy(V, Y, 16); + for (i = 0; i < 128; i++) { + if ((X[i >> 3] >> (7 - (i & 7))) & 1) + for (j = 0; j < 16; j++) + Z[j] ^= V[j]; + { + int lsb = V[15] & 1; + + for (j = 15; j > 0; j--) + V[j] = (V[j] >> 1) | ((V[j - 1] & 1) << 7); + V[0] >>= 1; + if (lsb) + V[0] ^= 0xe1; + } + } + memcpy(out, Z, 16); +} + +/* GHASH: fold `len` bytes of `data` (zero-padded to 16) into accumulator Y. */ +static void sep_ghash(const u8 H[16], u8 Y[16], const u8 *data, size_t len) +{ + u8 blk[16], t[16]; + size_t i, n, j; + + for (i = 0; i < len; i += 16) { + n = (len - i >= 16) ? 16 : (len - i); + memset(blk, 0, 16); + memcpy(blk, data + i, n); + for (j = 0; j < 16; j++) + Y[j] ^= blk[j]; + sep_gf_mult(Y, H, t); + memcpy(Y, t, 16); + } +} + +/* Increment the low 32 bits (big-endian) of a counter block, in place. */ +static void sep_inc32(u8 cb[16]) +{ + u32 c = ((u32)cb[12] << 24) | ((u32)cb[13] << 16) | + ((u32)cb[14] << 8) | (u32)cb[15]; + c++; + cb[12] = c >> 24; + cb[13] = c >> 16; + cb[14] = c >> 8; + cb[15] = c; +} + +/* + * AES-GCM in place over `buf` = [aadlen AAD][datalen payload][16 tag], 16-byte + * IV, keylen 16 or 32. Returns 0, or -EBADMSG on a decrypt tag mismatch. + */ +static int sep_gcm16(int encrypt, const void *key, size_t keylen, + const u8 *iv, size_t aadlen, u8 *buf, size_t datalen) +{ + struct aes_enckey aes; + u8 H[16], J0[16], EJ0[16], S[16], cb[16], ks[16], lb[16], zero[16]; + u8 *ct; + u64 aadbits, ctbits; + size_t i, n, j; + int rc; + + rc = aes_prepareenckey(&aes, key, keylen); + if (rc) + return rc; + + /* H = AES_K(0^128) */ + memset(zero, 0, 16); + aes_encrypt(&aes, H, zero); + + /* J0 = GHASH_H(IV ‖ [0^64 ‖ len(IV)_64]) for a non-96-bit IV. */ + memset(J0, 0, 16); + sep_ghash(H, J0, iv, 16); + memset(lb, 0, 16); + lb[15] = 0x80; /* 128 bits, big-endian in the low 64 bits */ + for (j = 0; j < 16; j++) + J0[j] ^= lb[j]; + { + u8 t[16]; + + sep_gf_mult(J0, H, t); + memcpy(J0, t, 16); + } + aes_encrypt(&aes, EJ0, J0); + + ct = buf + aadlen; + + /* GHASH over AAD, then ciphertext (before CTR on decrypt). */ + memset(S, 0, 16); + sep_ghash(H, S, buf, aadlen); + if (!encrypt) + sep_ghash(H, S, ct, datalen); + + /* CTR keystream from inc32(J0). */ + memcpy(cb, J0, 16); + sep_inc32(cb); + for (i = 0; i < datalen; i += 16) { + aes_encrypt(&aes, ks, cb); + n = (datalen - i >= 16) ? 16 : (datalen - i); + for (j = 0; j < n; j++) + ct[i + j] ^= ks[j]; + sep_inc32(cb); + } + + if (encrypt) + sep_ghash(H, S, ct, datalen); + + /* lengths block: (aadbits)_64 ‖ (ctbits)_64, big-endian. */ + aadbits = (u64)aadlen * 8; + ctbits = (u64)datalen * 8; + memset(lb, 0, 16); + for (j = 0; j < 8; j++) + lb[7 - j] = (u8)(aadbits >> (8 * j)); + for (j = 0; j < 8; j++) + lb[15 - j] = (u8)(ctbits >> (8 * j)); + for (j = 0; j < 16; j++) + S[j] ^= lb[j]; + { + u8 t[16]; + + sep_gf_mult(S, H, t); + memcpy(S, t, 16); + } + for (j = 0; j < 16; j++) + S[j] ^= EJ0[j]; /* S is now the computed tag */ + + if (encrypt) { + memcpy(ct + datalen, S, 16); + rc = 0; + } else { + u8 diff = 0; + + for (j = 0; j < 16; j++) + diff |= S[j] ^ ct[datalen + j]; + rc = diff ? -EBADMSG : 0; + } + + memzero_explicit(ks, sizeof(ks)); + memzero_explicit(EJ0, sizeof(EJ0)); + memzero_explicit(H, sizeof(H)); + memzero_explicit(&aes, sizeof(aes)); + return rc; +} + +/* + * Fills `buf` from the kernel CSPRNG, waiting for the seed first so it never + * returns unseeded bytes. Host-side entropy for key-bag secrets the host must + * reproduce; anything measuring the enclave's own entropy stays on SEP. + */ +int sep_random_bytes(void *buf, size_t len) +{ + int ret = wait_for_random_bytes(); + + if (ret) + return ret; + get_random_bytes(buf, len); + return 0; +} + +/* HMAC-SHA256 over one message; writes 32 bytes to `out`, untouched on error. */ +int sep_hmac_sha256(const void *key, size_t keylen, + const void *data, size_t datalen, u8 *out) +{ + struct crypto_shash *tfm; + struct shash_desc *desc; + int rc; + + tfm = crypto_alloc_shash("hmac(sha256)", 0, 0); + if (IS_ERR(tfm)) + return PTR_ERR(tfm); + + rc = crypto_shash_setkey(tfm, key, keylen); + if (rc) + goto out_tfm; + + desc = kzalloc(sizeof(*desc) + crypto_shash_descsize(tfm), GFP_KERNEL); + if (!desc) { + rc = -ENOMEM; + goto out_tfm; + } + desc->tfm = tfm; + + rc = crypto_shash_digest(desc, data, datalen, out); + + kfree_sensitive(desc); +out_tfm: + crypto_free_shash(tfm); + return rc; +} + +/* + * AES-256-GCM in place over `buf` = [aadlen AAD][payload][tag]. Encrypt appends + * the tag; decrypt expects it and leaves plaintext in the first `datalen` bytes + * after the AAD. keylen 16 or 32, ivlen 12 or 16 (a 16-byte IV is routed to the + * GHASH J0 derivation, not truncated). + */ +int sep_gcm(int encrypt, const void *key, size_t keylen, + const void *iv, size_t ivlen, size_t aadlen, + void *buf, size_t buflen, size_t datalen) +{ + struct crypto_aead *tfm; + struct aead_request *req; + struct scatterlist sg; + DECLARE_CRYPTO_WAIT(wait); + size_t cryptlen; + u8 ivcopy[16]; + int rc; + + if ((keylen != 16 && keylen != 32) || + (ivlen != 12 && ivlen != 16) || ivlen > sizeof(ivcopy)) + return -EINVAL; + + /* The whole AAD-plus-payload-plus-tag extent must be inside the buffer. */ + if (aadlen + datalen + SEP_GCM_TAG_LEN < aadlen || + aadlen + datalen + SEP_GCM_TAG_LEN > buflen) + return -EINVAL; + + /* A 16-byte IV needs the GHASH J0 derivation; gcm(aes) would silently + * use only its first 12 bytes. */ + if (ivlen == 16) + return sep_gcm16(encrypt, key, keylen, iv, aadlen, buf, + datalen); + + tfm = crypto_alloc_aead("gcm(aes)", 0, 0); + if (IS_ERR(tfm)) + return PTR_ERR(tfm); + + rc = crypto_aead_setkey(tfm, key, keylen); + if (rc) + goto out_tfm; + rc = crypto_aead_setauthsize(tfm, SEP_GCM_TAG_LEN); + if (rc) + goto out_tfm; + + req = aead_request_alloc(tfm, GFP_KERNEL); + if (!req) { + rc = -ENOMEM; + goto out_tfm; + } + + /* The API may modify the IV buffer, so it never sees the caller's. */ + memcpy(ivcopy, iv, ivlen); + + sg_init_one(&sg, buf, aadlen + datalen + SEP_GCM_TAG_LEN); + + /* AEAD API quirk: cryptlen includes the tag on decrypt but not on encrypt. */ + cryptlen = encrypt ? datalen : datalen + SEP_GCM_TAG_LEN; + + aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + aead_request_set_ad(req, aadlen); + aead_request_set_crypt(req, &sg, &sg, cryptlen, ivcopy); + + rc = crypto_wait_req(encrypt ? crypto_aead_encrypt(req) + : crypto_aead_decrypt(req), &wait); + + aead_request_free(req); + memzero_explicit(ivcopy, sizeof(ivcopy)); +out_tfm: + crypto_free_aead(tfm); + return rc; +} diff --git a/drivers/soc/apple/der.rs b/drivers/soc/apple/der.rs new file mode 100644 index 0000000000000..bc3db32343eae --- /dev/null +++ b/drivers/soc/apple/der.rs @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! The key-store configuration blob, which is DER. + +use kernel::prelude::*; + +const TAG_SET: u8 = 0x31; +const TAG_SEQUENCE: u8 = 0x30; +const TAG_UTF8: u8 = 0x0c; +pub(crate) const TAG_INTEGER: u8 = 0x02; +pub(crate) const TAG_OCTET_STRING: u8 = 0x04; + +const MAX_ITEM: usize = 4096; + +fn put_header(out: &mut KVec, tag: u8, len: usize) -> Result<()> { + out.push(tag, GFP_KERNEL)?; + if len < 0x80 { + out.push(len as u8, GFP_KERNEL)?; + } else if len <= 0xff { + out.push(0x81, GFP_KERNEL)?; + out.push(len as u8, GFP_KERNEL)?; + } else if len <= 0xffff { + out.push(0x82, GFP_KERNEL)?; + out.push((len >> 8) as u8, GFP_KERNEL)?; + out.push((len & 0xff) as u8, GFP_KERNEL)?; + } else { + return Err(EINVAL); + } + Ok(()) +} + +fn integer_bytes(v: u32) -> Result> { + let mut raw = KVec::new(); + let be = v.to_be_bytes(); + let mut i = 0usize; + while i < 3 && be[i] == 0 { + i += 1; + } + if be[i] & 0x80 != 0 { + raw.push(0, GFP_KERNEL)?; + } + raw.extend_from_slice(&be[i..], GFP_KERNEL)?; + Ok(raw) +} + +#[derive(Clone, Copy)] +pub(crate) enum RefKeyValue<'a> { + /// op mnemonic: `oc`/`ow`/`ouw`. + Utf8(&'a [u8]), + Integer(u32), + Octets(&'a [u8]), + Der(&'a [u8]), +} + +/// Canonical DER `SET OF SEQUENCE`, members sorted by key — the shape the +/// enclave parser requires. +pub(crate) fn encode_refkey_set(items: &[(&[u8], RefKeyValue<'_>)]) -> Result> { + let mut members: KVec<(&[u8], KVec)> = KVec::new(); + for (key, value) in items { + if key.is_empty() || key.len() > MAX_ITEM { + return Err(EINVAL); + } + let mut inner = KVec::new(); + put_header(&mut inner, TAG_UTF8, key.len())?; + inner.extend_from_slice(key, GFP_KERNEL)?; + match value { + RefKeyValue::Utf8(s) => { + if s.len() > MAX_ITEM { + return Err(EINVAL); + } + put_header(&mut inner, TAG_UTF8, s.len())?; + inner.extend_from_slice(s, GFP_KERNEL)?; + } + RefKeyValue::Integer(v) => { + let raw = integer_bytes(*v)?; + put_header(&mut inner, TAG_INTEGER, raw.len())?; + inner.extend_from_slice(&raw, GFP_KERNEL)?; + } + RefKeyValue::Octets(b) => { + if b.len() > MAX_ITEM { + return Err(EINVAL); + } + put_header(&mut inner, TAG_OCTET_STRING, b.len())?; + inner.extend_from_slice(b, GFP_KERNEL)?; + } + RefKeyValue::Der(d) => inner.extend_from_slice(d, GFP_KERNEL)?, + } + let mut seq = KVec::new(); + put_header(&mut seq, TAG_SEQUENCE, inner.len())?; + seq.extend_from_slice(&inner, GFP_KERNEL)?; + members.push((key, seq), GFP_KERNEL)?; + } + members.sort_unstable_by(|a, b| a.0.cmp(b.0)); + let mut body = KVec::new(); + for (_, seq) in members.iter() { + body.extend_from_slice(seq, GFP_KERNEL)?; + } + let mut out = KVec::new(); + put_header(&mut out, TAG_SET, body.len())?; + out.extend_from_slice(&body, GFP_KERNEL)?; + Ok(out) +} + +fn take_tlv(buf: &[u8]) -> Option<(u8, &[u8], &[u8])> { + let tag = *buf.first()?; + let first = *buf.get(1)?; + let (len, off) = if first < 0x80 { + (first as usize, 2usize) + } else if first == 0x81 { + (*buf.get(2)? as usize, 3usize) + } else if first == 0x82 { + let hi = *buf.get(2)? as usize; + let lo = *buf.get(3)? as usize; + ((hi << 8) | lo, 4usize) + } else { + return None; + }; + if len > MAX_ITEM { + return None; + } + let end = off.checked_add(len)?; + if buf.len() < end { + return None; + } + Some((tag, &buf[off..end], &buf[end..])) +} + +pub(crate) fn refkey_find<'a>(set_blob: &'a [u8], key: &[u8]) -> Option<&'a [u8]> { + let mut body = match take_tlv(set_blob) { + Some((TAG_SET, body, [])) => body, + _ => return None, + }; + while !body.is_empty() { + let (seq_tag, seq, next) = take_tlv(body)?; + if seq_tag != TAG_SEQUENCE { + return None; + } + let (key_tag, this_key, after_key) = take_tlv(seq)?; + if key_tag != TAG_UTF8 { + return None; + } + let (_val_tag, _value, after_val) = take_tlv(after_key)?; + if this_key == key { + return Some(&after_key[..after_key.len() - after_val.len()]); + } + body = next; + } + None +} + +pub(crate) fn octet_string_body(tlv: &[u8]) -> Option<&[u8]> { + match take_tlv(tlv) { + Some((TAG_OCTET_STRING, body, [])) => Some(body), + _ => None, + } +} + diff --git a/drivers/soc/apple/dt.rs b/drivers/soc/apple/dt.rs new file mode 100644 index 0000000000000..7fc5be540c654 --- /dev/null +++ b/drivers/soc/apple/dt.rs @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Device-tree work: bringing the SEP and its DART out of `status = "disabled"` +//! at module init, and the boot-persistent marker that stops the one-shot +//! registration from being sent twice. + +use kernel::bindings; +use kernel::error::to_result; +use kernel::prelude::*; + +const SEP_COMPATIBLE: &CStr = c"apple,sep"; +const IOMMUS_PROP: &CStr = c"iommus"; + +const STATUS_PROP: &CStr = c"status"; +const STATUS_OKAY: &CStr = c"okay"; + +// SEP DMA is offset BIT(40) above the DART input; one 64-bit value = two cells. +const DMA_OFFSET_PROP: &CStr = c"apple,dma-offset"; +// [0x100, 0] = BIT(40); [0x1, 0] would be BIT(32) and miss the page tables. +const DMA_OFFSET_CELLS: [u32; 2] = [0x100, 0x0]; + +const DMA_RANGE_PROP: &CStr = c"apple,dma-range"; +const DMA_RANGE_CELLS: [u32; 4] = [0, 0, 1, 0]; + +const REGISTERED_PROP: &CStr = c"apple,sep-shmem-registered-iova"; + +pub(crate) struct DtNode(*mut bindings::device_node); + +impl DtNode { + fn find_compatible(compatible: &CStr) -> Option { + // SAFETY: `of_find_compatible_node` accepts NULL for `from` and `type`, + // takes a reference on the node it returns, and returns NULL when there + // is no match. + let np = unsafe { + bindings::of_find_compatible_node( + core::ptr::null_mut(), + core::ptr::null(), + compatible.as_char_ptr(), + ) + }; + (!np.is_null()).then_some(DtNode(np)) + } + + pub(crate) fn of_device(dev: &kernel::device::Device) -> Option { + // SAFETY: `dev` is valid for the duration of the borrow, and its + // `of_node` is either NULL or a live node the device holds a reference + // to. + let np = unsafe { (*dev.as_raw()).of_node }; + if np.is_null() { + return None; + } + // SAFETY: `np` is a live node; take our own reference for `DtNode`. + unsafe { bindings::of_node_get(np) }; + Some(DtNode(np)) + } + + fn parse_phandle(&self, name: &CStr, index: i32) -> Option { + // SAFETY: `self.0` is a valid node per the type invariant; + // `of_parse_phandle` takes a reference on what it returns and returns + // NULL when the property or the index is absent. + let np = unsafe { bindings::of_parse_phandle(self.0, name.as_char_ptr(), index) }; + (!np.is_null()).then_some(DtNode(np)) + } + + fn as_ptr(&self) -> *mut bindings::device_node { + self.0 + } + + fn is_available(&self) -> bool { + // SAFETY: `self.0` is a valid node per the type invariant. + unsafe { bindings::of_device_is_available(self.0) } + } + + pub(crate) fn has_property(&self, name: &CStr) -> bool { + // SAFETY: `self.0` is a valid node per the type invariant; a NULL + // length pointer is accepted. + let p = unsafe { + bindings::of_find_property(self.0, name.as_char_ptr(), core::ptr::null_mut()) + }; + !p.is_null() + } + +} + +impl Drop for DtNode { + fn drop(&mut self) { + // SAFETY: we own one reference, taken by the function that produced + // `self.0`. + unsafe { bindings::of_node_put(self.0) }; + } +} + +fn with_changeset(f: F) -> Result<()> +where + F: FnOnce(*mut bindings::of_changeset) -> Result, +{ + let mut cs = core::mem::MaybeUninit::::uninit(); + let csp = cs.as_mut_ptr(); + + // SAFETY: `csp` points at valid uninitialised storage of the right type and + // `of_changeset_init` initialises it in place. + unsafe { bindings::of_changeset_init(csp) }; + + let res = match f(csp) { + // SAFETY: `csp` is an initialised changeset. + Ok(true) => to_result(unsafe { bindings::of_changeset_apply(csp) }), + Ok(false) => Ok(()), + Err(e) => Err(e), + }; + + // SAFETY: `csp` is an initialised changeset. + unsafe { bindings::of_changeset_destroy(csp) }; + + res +} + +fn queue_u32_array( + cs: *mut bindings::of_changeset, + node: &DtNode, + name: &CStr, + cells: &[u32], +) -> Result<()> { + // SAFETY: `cs` is an initialised changeset, `node` is a valid node, `name` + // is NUL-terminated, and `cells` is valid for `cells.len()` reads. The + // callee copies both the name and the value. + to_result(unsafe { + bindings::of_changeset_add_prop_u32_array( + cs, + node.as_ptr(), + name.as_char_ptr(), + cells.as_ptr(), + cells.len(), + ) + }) +} + +fn queue_status_okay(cs: *mut bindings::of_changeset, node: &DtNode) -> Result<()> { + // SAFETY: as above; `of_changeset_update_prop_string` duplicates the string. + to_result(unsafe { + bindings::of_changeset_update_prop_string( + cs, + node.as_ptr(), + STATUS_PROP.as_char_ptr(), + STATUS_OKAY.as_char_ptr(), + ) + }) +} + +pub(crate) fn sep_node() -> Option { + DtNode::find_compatible(SEP_COMPATIBLE) +} + +pub(crate) fn enable_sep_and_dart() -> Result<()> { + let sep = sep_node().ok_or_else(|| { + pr_err!( + "apple_sep: no device-tree node with compatible '{}'\n", + SEP_COMPATIBLE + ); + ENODEV + })?; + + let dart = sep.parse_phandle(IOMMUS_PROP, 0).ok_or_else(|| { + pr_err!( + "apple_sep: SEP node has no '{}' phandle; cannot find its DART\n", + IOMMUS_PROP + ); + ENODEV + })?; + + let dart_available = dart.is_available(); + let need_offset = !dart.has_property(DMA_OFFSET_PROP); + let need_range = !dart.has_property(DMA_RANGE_PROP); + + pr_info!( + "apple_sep: SEP DART state: enabled={}, apple,dma-offset present={}, apple,dma-range present={}\n", + dart_available, + !need_offset, + !need_range + ); + + if dart_available && (need_offset || need_range) { + pr_err!( + "apple_sep: SEP DART already enabled but missing apple,dma-offset/apple,dma-range; the BIT(40) offset applies only at DART probe. Reboot and load this module first.\n" + ); + return Err(EBUSY); + } + + if need_offset || need_range || !dart_available { + with_changeset(|cs| { + let mut queued = false; + // Properties before status: the DART probes inside of_changeset_apply(). + if need_offset { + queue_u32_array(cs, &dart, DMA_OFFSET_PROP, &DMA_OFFSET_CELLS)?; + queued = true; + } + if need_range { + queue_u32_array(cs, &dart, DMA_RANGE_PROP, &DMA_RANGE_CELLS)?; + queued = true; + } + if !dart_available { + queue_status_okay(cs, &dart)?; + queued = true; + } + Ok(queued) + })?; + pr_info!( + "apple_sep: SEP DART changeset applied (offset BIT(40), range 0..4GiB, status okay)\n" + ); + } else { + pr_info!("apple_sep: SEP DART already enabled and configured\n"); + } + + if sep.is_available() { + pr_info!("apple_sep: SEP node already enabled\n"); + } else { + with_changeset(|cs| { + queue_status_okay(cs, &sep)?; + Ok(true) + })?; + pr_info!("apple_sep: SEP node enabled; platform device should now exist\n"); + } + + Ok(()) +} + +pub(crate) fn registration_already_sent(sep: &DtNode) -> bool { + sep.has_property(REGISTERED_PROP) +} + +pub(crate) fn mark_registration_sent(sep: &DtNode, iova: u64) -> Result<()> { + let cells = [(iova >> 32) as u32, iova as u32]; + with_changeset(|cs| { + queue_u32_array(cs, sep, REGISTERED_PROP, &cells)?; + Ok(true) + }) +} + +impl DtNode { + fn reg_base(&self) -> Option { + // SAFETY: `struct resource` is plain integers and pointers, so an + // all-zero value is valid. Zeroed rather than `default()` because + // bindgen's generated structs do not derive `Default`. + let mut res: bindings::resource = unsafe { core::mem::zeroed() }; + // SAFETY: `self.0` is a live node and `res` is a valid out-parameter. + let rc = unsafe { bindings::of_address_to_resource(self.0, 0, &mut res) }; + if rc != 0 { + None + } else { + Some(res.start) + } + } + + fn child_with_reg(&self, value: u32) -> Option { + let mut child: *mut bindings::device_node = core::ptr::null_mut(); + loop { + // SAFETY: `of_get_next_child` takes a live parent and the previous + // child, which it drops for us; NULL starts the iteration. + child = unsafe { bindings::of_get_next_child(self.0, child) }; + if child.is_null() { + return None; + } + let mut got: u32 = 0; + // SAFETY: `child` is live for this iteration and `got` is a valid + // out-parameter. + let rc = unsafe { + bindings::of_property_read_variable_u32_array( + child, + c"reg".as_char_ptr(), + &mut got, + 1, + 1, + ) + }; + if rc >= 0 && got == value { + return Some(DtNode(child)); + } + } + } +} + +fn node_at_address(base: u64) -> Option { + let mut np: *mut bindings::device_node = core::ptr::null_mut(); + loop { + // SAFETY: `of_find_node_with_property` accepts NULL to start and drops + // the reference to the previous node for us. + np = unsafe { bindings::of_find_node_with_property(np, c"reg".as_char_ptr()) }; + if np.is_null() { + return None; + } + let node = DtNode(np); + if node.reg_base() == Some(base) { + return Some(node); + } + } +} + +pub(crate) const SENSOR_COMPATIBLE: &CStr = c"apple,mesa-fingerprint"; + +const SENSOR_NODE_NAME: &CStr = c"mesa@0"; + +fn queue_sensor_node(cs_handle: *mut bindings::of_changeset, parent: &DtNode) -> Result<()> { + // SAFETY: `cs_handle` is a live changeset and `parent` a live node; the + // returned node belongs to the changeset, which owns it until applied. + let node = unsafe { + bindings::of_changeset_create_node(cs_handle, parent.0, SENSOR_NODE_NAME.as_char_ptr()) + }; + if node.is_null() { + pr_err!("apple_sep: could not create the sensor node\n"); + return Err(ENOMEM); + } + + // SAFETY: all four calls take the live changeset, the node just created and + // a NUL-terminated name; the u32 form takes a slice it copies. + let rc = unsafe { + let mut rc = bindings::of_changeset_add_prop_string( + cs_handle, + node, + c"compatible".as_char_ptr(), + SENSOR_COMPATIBLE.as_char_ptr(), + ); + // reg on an SPI child is the chip select. + if rc == 0 { + rc = add_u32(cs_handle, node, c"reg", 0); + } + if rc == 0 { + rc = add_u32(cs_handle, node, c"spi-max-frequency", 8_000_000); + } + // 20 ns setup/hold the controller does not apply itself. + if rc == 0 { + rc = add_u32(cs_handle, node, c"spi-cs-setup-delay-ns", 20); + } + if rc == 0 { + rc = add_u32(cs_handle, node, c"spi-cs-hold-delay-ns", 20); + } + // SPI mode 2 (CPOL=1, CPHA=0); absent spi-cpha is how CPHA=0 is spelled. + if rc == 0 { + rc = bindings::of_changeset_add_prop_bool(cs_handle, node, c"spi-cpol".as_char_ptr()); + } + rc + }; + if rc != 0 { + pr_err!("apple_sep: could not describe the sensor node: {}\n", rc); + return Err(Error::from_errno(rc)); + } + Ok(()) +} + +/// # Safety +unsafe fn add_u32( + cs_handle: *mut bindings::of_changeset, + node: *mut bindings::device_node, + name: &CStr, + value: u32, +) -> i32 { + // of_changeset_add_prop_u32 is a static inline missing from the bindings; wrap the array form. + // SAFETY: per this function's contract; `value` is read as one element. + unsafe { + bindings::of_changeset_add_prop_u32_array(cs_handle, node, name.as_char_ptr(), &value, 1) + } +} + +pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { + let controller = node_at_address(base).ok_or_else(|| { + pr_err!( + "apple_sep: no device-tree node with reg base 0x{:x}; the sensor's SPI bus is not in this tree\n", + base + ); + ENODEV + })?; + + let controller_ok = controller.is_available(); + let existing = controller.child_with_reg(cs); + pr_info!( + "apple_sep: sensor SPI bus at 0x{:x}: controller enabled={}, chip-select {} child present={}\n", + base, + controller_ok, + cs, + existing.is_some() + ); + + if controller_ok && existing.is_some() { + return Ok(()); + } + + if existing.is_some() && !controller_ok { + // Enable only the bus; a second child would be two devices at one chip select. + pr_info!("apple_sep: sensor node already present; enabling only the bus\n"); + return with_changeset(|cs_handle| { + queue_status_okay(cs_handle, &controller)?; + Ok(true) + }); + } + + pr_info!( + "apple_sep: creating the sensor node (absent from Linux's tree; Apple has it at /arm-io/spi2/mesa)\n" + ); + + with_changeset(|cs_handle| { + // Controller before child: the SPI core's notifier needs the bus to exist first. + if !controller_ok { + queue_status_okay(cs_handle, &controller)?; + } + queue_sensor_node(cs_handle, &controller)?; + Ok(true) + }) +} diff --git a/drivers/soc/apple/fv.rs b/drivers/soc/apple/fv.rs new file mode 100644 index 0000000000000..ba05aba7d854f --- /dev/null +++ b/drivers/soc/apple/fv.rs @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj +//! Native FileVault key hierarchy: device-bound volume-key provisioning, +//! reverse-engineered and kept as capability. Not wired to a caller. +#![allow(dead_code)] + +use crate::{image, proto, shim}; +use crate::{LockState, SepData, SksRequest}; +use kernel::prelude::*; + +impl SepData { + const FV_LABEL: &'static [u8; 16] = b"AppleSEPvek00001"; + const FV_PKH: &'static [u8; 16] = b"AppleSEP-KEK-001"; + const FV_PARAM_LEN: usize = 0x130; + + fn fv_param(&self) -> Result> { + let mut p: KVec = KVec::new(); + p.resize(Self::FV_PARAM_LEN, 0u8, GFP_KERNEL)?; + p[0x10..0x20].copy_from_slice(Self::FV_LABEL); + Ok(p) + } + + fn sks_req_fv(&self, selector: u8, body: &image::Body) -> Result { + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_fv(selector, self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_PERFORM_OP_NAME, + msg, + img, + }) + } + + fn fv_send( + &self, + selector: u8, + body: image::Body, + _what: &str, + ) -> Option<(i32, Option, Option>)> { + let out = self.sks_send(self.sks_req_fv(selector, &body))?; + let mailbox: i32 = out.reply.status.into(); + let (mut opst, mut blob) = (None, None); + if mailbox == 0 { + if let Some(rbody) = self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &out) { + let mut f = proto::FieldCursor::new(rbody); + opst = f.i32(); + if let Some(b) = f.blob() { + let mut owned: KVec = KVec::new(); + owned.extend_from_slice(b, GFP_KERNEL).ok()?; + blob = Some(owned); + } + } + } + Some((mailbox, opst, blob)) + } + + /// `0x42` mint the device-bound key-encryption key. + fn fv_new_kek(&self, handle: u64, param: &[u8]) -> Option> { + let mut b = image::Body::new(); + b.put_u32(0).ok()?; + b.put_u64(handle).ok()?; + b.put_blob(param).ok()?; + b.put_u32(0).ok()?; + b.put_blob(&[]).ok()?; + b.put_blob(Self::FV_PKH).ok()?; + let (mb, op, blob) = self.fv_send(crate::sks::OP_SKS_FV_NEW_KEK, b, "new_kek")?; + (mb == 0 && op == Some(0)).then_some(())?; + blob + } + + /// `0x40` mint the wrapped, device-bound volume key. + fn fv_new_vek(&self, handle: u64, param: &[u8]) -> Option> { + let mut b = image::Body::new(); + b.put_u32(0).ok()?; + b.put_u64(handle).ok()?; + b.put_blob(param).ok()?; + b.put_blob(&[]).ok()?; + b.put_blob(&[]).ok()?; + b.put_blob(Self::FV_PKH).ok()?; + let (mb, op, blob) = self.fv_send(crate::sks::OP_SKS_FV_NEW_VEK, b, "new_vek")?; + (mb == 0 && op == Some(0)).then_some(())?; + blob + } + + /// `0x41` install the volume key into our collection (empty KEK slot = self-derive). + fn fv_unwrap_vek(&self, handle: u64, param: &[u8], wrapped_vek: &[u8]) -> Option> { + let mut b = image::Body::new(); + b.put_u32(0).ok()?; + b.put_u64(handle).ok()?; + b.put_blob(param).ok()?; + b.put_u32(0).ok()?; + b.put_blob(&[]).ok()?; + b.put_blob(&[]).ok()?; + b.put_blob(wrapped_vek).ok()?; + b.put_blob(&[]).ok()?; + let (mb, op, blob) = self.fv_send(crate::sks::OP_SKS_FV_UNWRAP_VEK, b, "unwrap_vek")?; + (mb == 0 && op == Some(0)).then_some(())?; + blob.or_else(|| Some(KVec::new())) + } + + fn fv_dump(path: &CStr, data: &[u8]) { + if let Ok(f) = shim::StoreFile::open(path) { + let _ = f.write_all(0, data); + let _ = f.sync(); + } + } + + fn fv_load(path: &CStr) -> Option> { + let f = shim::StoreFile::open_readonly(path).ok()?; + let sz = f.size().unwrap_or(0); + if !(1..=8192).contains(&sz) { + return None; + } + let mut buf: KVec = KVec::new(); + buf.resize(sz as usize, 0u8, GFP_KERNEL).ok()?; + f.read_exact(0, &mut buf).ok()?; + Some(buf) + } + + /// Provisions the device-bound FileVault key hierarchy and installs the volume + /// key. The destructive clear (`0x47`) is never emitted; the installed VEK is + /// consumed by the storage inline-AES engine (ANS), so on its own this seals + /// nothing on Linux. + pub(crate) fn fv_provision(&self, handle: crate::sks::KeyBagHandle, secret: &[u8]) -> Option { + const KEK_PATH: &CStr = c"/var/lib/apple-sep-fv-kek.bin"; + const VEK_PATH: &CStr = c"/var/lib/apple-sep-fv-vek.bin"; + + if let Some(healthy) = self.sks_health_check(c"fv provision") { + let _ = self.sks_send(self.sks_req_change_lock_state( + handle, + LockState::Unlocked, + secret, + healthy, + )); + } + let param = self.fv_param().ok()?; + let h = handle.value() as u64; + + let (_kek, vek) = match (Self::fv_load(KEK_PATH), Self::fv_load(VEK_PATH)) { + (Some(kek), Some(vek)) => (kek, vek), + _ => { + let kek = self.fv_new_kek(h, ¶m)?; + let vek = self.fv_new_vek(h, ¶m)?; + Self::fv_dump(KEK_PATH, &kek); + Self::fv_dump(VEK_PATH, &vek); + (kek, vek) + } + }; + + let installed = self.fv_unwrap_vek(h, ¶m, &vek)?; + let vek_handle = if installed.len() >= 4 { + u64::from(u32::from_le_bytes([installed[0], installed[1], installed[2], installed[3]])) + } else { + 0 + }; + Some(vek_handle) + } +} diff --git a/drivers/soc/apple/hwrng.rs b/drivers/soc/apple/hwrng.rs new file mode 100644 index 0000000000000..92f3ac7043a78 --- /dev/null +++ b/drivers/soc/apple/hwrng.rs @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Ownership of the hwrng-core registration. + +use kernel::prelude::*; + +pub(crate) type ReadFn = + unsafe extern "C" fn(ctx: *mut c_void, data: *mut c_void, max: usize, wait: bool) -> c_int; + +extern "C" { + fn sep_hwrng_alloc() -> *mut c_void; + fn sep_hwrng_free(mem: *mut c_void); + fn sep_hwrng_register( + mem: *mut c_void, + name: *const c_char, + quality: c_ushort, + ctx: *mut c_void, + read: Option, + ) -> c_int; + fn sep_hwrng_unregister(mem: *mut c_void); +} + +pub(crate) const QUALITY: c_ushort = 1024; + +pub(crate) const NAME: &CStr = c"apple-sep"; + +pub(crate) struct HwRngHandle { + mem: *mut c_void, + registered: bool, +} + +// SAFETY: `mem` is a plain heap allocation with no thread affinity; all access +// goes through the C shim, which locks inside the hwrng core. +unsafe impl Send for HwRngHandle {} + +impl HwRngHandle { + pub(crate) fn new() -> Result { + // SAFETY: no preconditions; returns NULL on allocation failure. + let mem = unsafe { sep_hwrng_alloc() }; + if mem.is_null() { + return Err(ENOMEM); + } + Ok(HwRngHandle { + mem, + registered: false, + }) + } + + /// # Safety + /// `ctx` must stay valid and safe to pass to `read` until [`Self::unregister`] + /// returns or this handle is dropped. + pub(crate) unsafe fn register(&mut self, ctx: *mut c_void, read: ReadFn) -> Result<()> { + if self.registered { + return Err(EBUSY); + } + // SAFETY: `mem` is live, `NAME` is a static NUL-terminated string, and + // the caller guarantees `ctx` outlives the registration. + let ret = + unsafe { sep_hwrng_register(self.mem, NAME.as_char_ptr(), QUALITY, ctx, Some(read)) }; + kernel::error::to_result(ret)?; + self.registered = true; + Ok(()) + } + + pub(crate) fn unregister(&mut self) { + if self.registered { + self.registered = false; + // SAFETY: `mem` is live and was registered. + unsafe { sep_hwrng_unregister(self.mem) }; + } + } +} + +impl Drop for HwRngHandle { + fn drop(&mut self) { + self.unregister(); + // SAFETY: `mem` is live and now unregistered, so nothing else refers to it. + unsafe { sep_hwrng_free(self.mem) }; + } +} diff --git a/drivers/soc/apple/hwrng_shim.c b/drivers/soc/apple/hwrng_shim.c new file mode 100644 index 0000000000000..e0507f935d0f7 --- /dev/null +++ b/drivers/soc/apple/hwrng_shim.c @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ +/* + * hwrng core shim: struct hwrng lives here where the C compiler owns its + * layout, and Rust sees an opaque pointer plus four entry points. + */ + +#include +#include +#include + +#include "shim.h" + +struct sep_hwrng { + struct hwrng rng; + /* Opaque Rust-side context, handed back to the read callback. */ + void *ctx; + int (*read)(void *ctx, void *data, size_t max, bool wait); +}; + +static int sep_hwrng_read(struct hwrng *rng, void *data, size_t max, + bool wait) +{ + struct sep_hwrng *h = + container_of(rng, struct sep_hwrng, rng); + + return h->read(h->ctx, data, max, wait); +} + +void *sep_hwrng_alloc(void) +{ + return kzalloc(sizeof(struct sep_hwrng), GFP_KERNEL); +} + +/* Must not be called while registered. */ +void sep_hwrng_free(void *mem) +{ + kfree(mem); +} + +/* + * @name must outlive the registration; the Rust side passes a &'static CStr. + * @quality is bits of entropy per 1024 bits of input; 0 takes the core default, + * which is also 1024. + */ +int sep_hwrng_register(void *mem, const char *name, + unsigned short quality, void *ctx, + int (*read)(void *ctx, void *data, size_t max, + bool wait)) +{ + struct sep_hwrng *h = mem; + + h->ctx = ctx; + h->read = read; + h->rng.name = name; + h->rng.read = sep_hwrng_read; + h->rng.quality = quality; + + return hwrng_register(&h->rng); +} + +/* + * Blocks until the core is done with the device, so any read in flight must + * return promptly: the Rust side sets its shutdown flag and wakes its waiters + * before calling this. + */ +void sep_hwrng_unregister(void *mem) +{ + struct sep_hwrng *h = mem; + + hwrng_unregister(&h->rng); +} diff --git a/drivers/soc/apple/image.rs b/drivers/soc/apple/image.rs new file mode 100644 index 0000000000000..b81f8130fe602 --- /dev/null +++ b/drivers/soc/apple/image.rs @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Request and response images for the key-store endpoint. + +use kernel::prelude::*; + +extern "C" { + fn sep_sha256( + a: *const c_void, + alen: usize, + b: *const c_void, + blen: usize, + out: *mut u8, + ) -> c_int; +} + +pub(crate) const HEADER_SIZE: u32 = 0x50; + +pub(crate) const HEADER_WIRE: usize = 0x54; + +const DIGEST_OFF: usize = 0x00; +const DIGEST_LEN: usize = 16; + +const DIGEST_FROM: usize = 0x10; + +pub(crate) const SHA256_LEN: usize = 32; + +pub(crate) fn sha256(bytes: &[u8]) -> Result<[u8; SHA256_LEN]> { + let mut digest = [0u8; SHA256_LEN]; + // SAFETY: `bytes` is a live slice for the duration of the call and `digest` + // is exactly the 32 bytes the shim writes. The shim reads the second + // segment only when its length is nonzero, and it is zero here. + let rc = unsafe { + sep_sha256( + bytes.as_ptr().cast(), + bytes.len(), + core::ptr::null(), + 0, + digest.as_mut_ptr(), + ) + }; + if rc != 0 { + return Err(Error::from_errno(rc)); + } + Ok(digest) +} + +const OFF_VERSION: usize = 0x10; +const OFF_TIMESTAMP: usize = 0x14; +const OFF_FLAGS: usize = 0x1c; +const OFF_RESERVED: usize = 0x20; +const OFF_PROC_ID: usize = 0x28; +const OFF_PID: usize = 0x30; +const OFF_CDHASH: usize = 0x34; +const OFF_TRAILER: usize = 0x48; + +static_assert!(OFF_VERSION == DIGEST_OFF + DIGEST_LEN); +static_assert!(OFF_TIMESTAMP == OFF_VERSION + 4); +static_assert!(OFF_FLAGS == OFF_TIMESTAMP + 8); +static_assert!(OFF_RESERVED == OFF_FLAGS + 4); +static_assert!(OFF_PROC_ID == OFF_RESERVED + 8); +static_assert!(OFF_PID == OFF_PROC_ID + 8); +static_assert!(OFF_CDHASH == OFF_PID + 4); +static_assert!(OFF_TRAILER == OFF_CDHASH + 20); +static_assert!(OFF_TRAILER + 8 == HEADER_SIZE as usize); +static_assert!(HEADER_WIRE == 4 + HEADER_SIZE as usize); + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Version { + V1, +} + +impl Version { + pub(crate) const fn wire(self) -> u32 { + match self { + Version::V1 => 1, + } + } + + const fn digest_end(self) -> usize { + match self { + Version::V1 => OFF_TRAILER, + } + } +} + +pub(crate) struct Body { + bytes: KVec, +} + +impl Body { + pub(crate) fn new() -> Body { + Body { bytes: KVec::new() } + } + + pub(crate) fn put_u32(&mut self, v: u32) -> Result<()> { + self.bytes.extend_from_slice(&v.to_le_bytes(), GFP_KERNEL)?; + Ok(()) + } + + pub(crate) fn put_i32(&mut self, v: i32) -> Result<()> { + self.bytes.extend_from_slice(&v.to_le_bytes(), GFP_KERNEL)?; + Ok(()) + } + + pub(crate) fn put_u64(&mut self, v: u64) -> Result<()> { + self.bytes.extend_from_slice(&v.to_le_bytes(), GFP_KERNEL)?; + Ok(()) + } + + pub(crate) fn put_blob(&mut self, bytes: &[u8]) -> Result<()> { + let len = u32::try_from(bytes.len()).map_err(|_| EINVAL)?; + self.put_u32(len)?; + self.bytes.extend_from_slice(bytes, GFP_KERNEL)?; + let pad = bytes.len().wrapping_neg() % 4; + for _ in 0..pad { + self.bytes.push(0, GFP_KERNEL)?; + } + Ok(()) + } + + fn as_slice(&self) -> &[u8] { + &self.bytes + } +} + +const fn pad_of(len: usize) -> usize { + len.wrapping_neg() % 4 +} +static_assert!(pad_of(0) == 0); +static_assert!(pad_of(1) == 3); +static_assert!(pad_of(2) == 2); +static_assert!(pad_of(3) == 1); +static_assert!(pad_of(4) == 0); + +pub(crate) struct RequestImage { + bytes: KVec, +} + +impl RequestImage { + pub(crate) fn as_slice(&self) -> &[u8] { + &self.bytes + } + + pub(crate) fn len(&self) -> usize { + self.bytes.len() + } +} + +pub(crate) fn build_request( + version: Version, + timestamp_us: u64, + body: &Body, +) -> Result { + let mut bytes = KVec::new(); + bytes.extend_from_slice(&HEADER_SIZE.to_le_bytes(), GFP_KERNEL)?; + + let header_at = bytes.len(); + for _ in 0..HEADER_SIZE { + bytes.push(0, GFP_KERNEL)?; + } + + let put = |bytes: &mut KVec, off: usize, src: &[u8]| { + bytes[header_at + off..header_at + off + src.len()].copy_from_slice(src); + }; + put(&mut bytes, OFF_VERSION, &version.wire().to_le_bytes()); + put(&mut bytes, OFF_TIMESTAMP, ×tamp_us.to_le_bytes()); + // Flags, reserved, proc_id, pid, cdhash and trailer stay zero; the enclave accepts that. + + bytes.extend_from_slice(body.as_slice(), GFP_KERNEL)?; + + static_assert!(OFF_VERSION == DIGEST_FROM); + let end = version.digest_end(); + let head = &bytes[header_at + DIGEST_FROM..header_at + end]; + let tail = &bytes[header_at + HEADER_SIZE as usize..]; + + let mut digest = [0u8; SHA256_LEN]; + // SAFETY: `head` and `tail` are live slices of `bytes` for the duration of + // the call, and `digest` is exactly the 32 bytes the shim writes. The shim + // reads only the two segments it is given and writes only the output. + let rc = unsafe { + sep_sha256( + head.as_ptr().cast(), + head.len(), + tail.as_ptr().cast(), + tail.len(), + digest.as_mut_ptr(), + ) + }; + if rc != 0 { + return Err(Error::from_errno(rc)); + } + + bytes[header_at + DIGEST_OFF..header_at + DIGEST_OFF + DIGEST_LEN] + .copy_from_slice(&digest[..DIGEST_LEN]); + + Ok(RequestImage { bytes }) +} + +pub(crate) struct ResponseImage<'a> { + pub(crate) body: &'a [u8], +} + +pub(crate) fn parse_response(bytes: &[u8]) -> Result> { + if bytes.len() < 4 { + return Err(EINVAL); + } + let header_size = u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); + + let header = header_size as usize; + if header < OFF_VERSION + 4 { + return Err(EINVAL); + } + let body_at = 4usize.checked_add(header).ok_or(EINVAL)?; + if bytes.len() < body_at { + return Err(EINVAL); + } + + Ok(ResponseImage { + body: &bytes[body_at..], + }) +} + +pub(crate) fn read_blob(body: &[u8], off: usize) -> Option<(&[u8], usize)> { + let len_end = off.checked_add(4)?; + if body.len() < len_end { + return None; + } + let len = u32::from_le_bytes([body[off], body[off + 1], body[off + 2], body[off + 3]]) as usize; + let end = len_end.checked_add(len)?; + if body.len() < end { + return None; + } + Some((&body[len_end..end], end + pad_of(len))) +} + +pub(crate) fn operation_status(body: &[u8]) -> Option { + if body.len() < 4 { + return None; + } + Some(i32::from_le_bytes([body[0], body[1], body[2], body[3]])) +} diff --git a/drivers/soc/apple/keybag.rs b/drivers/soc/apple/keybag.rs new file mode 100644 index 0000000000000..133a1ad5769db --- /dev/null +++ b/drivers/soc/apple/keybag.rs @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Host-side persistence for the key bag. +//! +//! `0x02` copies the bag out enclave-wrapped, `0x03` reloads it. NEVER create a +//! second bag while a stored blob exists — it orphans the previous one in the +//! enclave with no recovery; [`NoStoredKeyBag`] enforces this by type. + +use crate::shim; +use crate::store::crc16_ccitt_false; +use kernel::prelude::*; + +pub(crate) const KEYBAG_PATH: &CStr = c"/var/lib/apple-sep-keybag.bin"; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Slot { + Identity, +} + +impl Slot { + pub(crate) fn path(self) -> &'static CStr { + match self { + Slot::Identity => KEYBAG_PATH, + } + } + +} + +const MAGIC: [u8; 16] = *b"APPLE-SEP-KBAG01"; +/// v2 stores the bag secret; a v1 record is refused, not reinterpreted. +const VERSION: u32 = 2; + +const STATE_COMMITTED: u32 = 1; +/// Create refused by the enclave: nothing exists, retry allowed (unlike an +/// intent record, where a bag may exist unnamed — block retry). +const STATE_REFUSED: u32 = 2; +/// UUID field is the bag's own, read back via `0x06`; state 1 holds the +/// host-generated one, and for an identity bag these differ. +const STATE_COMMITTED_BAG_UUID: u32 = 3; + +/// Only an `AsGenerated` UUID may be repaired on a mismatch; adopting one for a +/// read-back record would point it at a different bag and strand the real one. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum UuidProvenance { + AsGenerated, + ReadBackFromBag, +} + +impl UuidProvenance { + fn state(self) -> u32 { + match self { + Self::AsGenerated => STATE_COMMITTED, + Self::ReadBackFromBag => STATE_COMMITTED_BAG_UUID, + } + } + + fn from_state(state: u32) -> Option { + match state { + STATE_COMMITTED => Some(Self::AsGenerated), + STATE_COMMITTED_BAG_UUID => Some(Self::ReadBackFromBag), + _ => None, + } + } +} + +const OFF_VERSION: usize = 0x10; +const OFF_STATE: usize = 0x14; +const OFF_LEN: usize = 0x18; +const OFF_CRC: usize = 0x1c; +const OFF_UUID: usize = 0x20; +const OFF_SECRET_LEN: usize = 0x30; +const HEADER: usize = 0x34; + +static_assert!(OFF_VERSION == MAGIC.len()); +static_assert!(OFF_STATE == OFF_VERSION + 4); +static_assert!(OFF_LEN == OFF_STATE + 4); +static_assert!(OFF_CRC == OFF_LEN + 4); +static_assert!(OFF_UUID == OFF_CRC + 4); +static_assert!(OFF_SECRET_LEN == OFF_UUID + UUID_LEN); +static_assert!(HEADER == OFF_SECRET_LEN + 4); + +pub(crate) const UUID_LEN: usize = 16; + +const MAX_WRAPPED: usize = crate::store::MAX_VALUE; + +/// Proof the host holds no bag; create takes one by value, so it is unreachable +/// without it and unrepeatable with it. +pub(crate) struct NoStoredKeyBag; + +pub(crate) struct StoredKeyBag { + wrapped: KVec, + uuid: [u8; UUID_LEN], + secret: crate::Secret, + provenance: UuidProvenance, +} + +impl StoredKeyBag { + pub(crate) fn wrapped(&self) -> &[u8] { + &self.wrapped + } + + pub(crate) fn uuid(&self) -> &[u8; UUID_LEN] { + &self.uuid + } + + pub(crate) fn uuid_provenance(&self) -> UuidProvenance { + self.provenance + } + + pub(crate) fn secret(&self) -> &[u8] { + &self.secret + } +} + +pub(crate) enum State { + Present(StoredKeyBag), + Absent(NoStoredKeyBag), +} + +/// CRC preimage; this byte order is the on-disk contract — changing it breaks +/// records written by an earlier build. +fn checksum_input( + state: u32, + wrapped: &[u8], + uuid: &[u8; UUID_LEN], + secret: &[u8], +) -> Result> { + let mut v = KVec::new(); + v.extend_from_slice(&VERSION.to_le_bytes(), GFP_KERNEL)?; + v.extend_from_slice(&state.to_le_bytes(), GFP_KERNEL)?; + v.extend_from_slice(&(wrapped.len() as u32).to_le_bytes(), GFP_KERNEL)?; + v.extend_from_slice(&(secret.len() as u32).to_le_bytes(), GFP_KERNEL)?; + v.extend_from_slice(uuid, GFP_KERNEL)?; + v.extend_from_slice(wrapped, GFP_KERNEL)?; + v.extend_from_slice(secret, GFP_KERNEL)?; + Ok(v) +} + +/// Absent only when the file is missing; every other unreadable state errors — +/// "cannot tell" must never read as "nothing here", which would orphan a bag. +pub(crate) fn read(slot: Slot) -> Result { + let file = match shim::StoreFile::open_readonly(slot.path()) { + Ok(f) => f, + Err(e) if e == ENOENT => { + return Ok(State::Absent(NoStoredKeyBag)); + } + Err(e) => return Err(e), + }; + + let size = file.size()?; + if size < HEADER as u64 { + return Err(EINVAL); + } + + let mut head = [0u8; HEADER]; + file.read_exact(0, &mut head)?; + if head[..MAGIC.len()] != MAGIC { + return Err(EINVAL); + } + let version = u32::from_le_bytes([ + head[OFF_VERSION], + head[OFF_VERSION + 1], + head[OFF_VERSION + 2], + head[OFF_VERSION + 3], + ]); + if version != VERSION { + return Err(ENOTSYNC); + } + let state = u32::from_le_bytes([ + head[OFF_STATE], + head[OFF_STATE + 1], + head[OFF_STATE + 2], + head[OFF_STATE + 3], + ]); + if state == STATE_REFUSED { + return Ok(State::Absent(NoStoredKeyBag)); + } + let Some(provenance) = UuidProvenance::from_state(state) else { + // Intent or unknown state: a bag may exist unnamed; creating again would + // orphan it permanently. + return Err(EEXIST); + }; + + let len = u32::from_le_bytes([ + head[OFF_LEN], + head[OFF_LEN + 1], + head[OFF_LEN + 2], + head[OFF_LEN + 3], + ]) as usize; + if len == 0 || len > MAX_WRAPPED || size < (HEADER + len) as u64 { + return Err(EINVAL); + } + let crc = u16::from_le_bytes([head[OFF_CRC], head[OFF_CRC + 1]]); + + let mut uuid = [0u8; UUID_LEN]; + uuid.copy_from_slice(&head[OFF_UUID..OFF_UUID + UUID_LEN]); + + let secret_len = u32::from_le_bytes([ + head[OFF_SECRET_LEN], + head[OFF_SECRET_LEN + 1], + head[OFF_SECRET_LEN + 2], + head[OFF_SECRET_LEN + 3], + ]) as usize; + if secret_len > MAX_WRAPPED || size < (HEADER + len + secret_len) as u64 { + return Err(EINVAL); + } + + let mut wrapped = KVec::with_capacity(len, GFP_KERNEL)?; + wrapped.resize(len, 0, GFP_KERNEL)?; + file.read_exact(HEADER as u64, &mut wrapped)?; + + let mut secret_bytes = KVec::with_capacity(secret_len, GFP_KERNEL)?; + secret_bytes.resize(secret_len, 0, GFP_KERNEL)?; + if secret_len > 0 { + file.read_exact((HEADER + len) as u64, &mut secret_bytes)?; + } + + if size != (HEADER + len + secret_len) as u64 { + return Err(EINVAL); + } + let secret = crate::Secret(secret_bytes); + + if crc16_ccitt_false(&checksum_input(state, &wrapped, &uuid, &secret)?) != crc { + return Err(EINVAL); + } + + Ok(State::Present(StoredKeyBag { + wrapped, + uuid, + secret, + provenance, + })) +} + +/// Replaces only the wrapped blob. The enclave ratchets bag material while a bag +/// is active, so this snapshot must be re-taken at the catacomb-save commit or +/// the enclave answers a later restore empty. +pub(crate) fn replace_wrapped( + slot: Slot, + fresh: &[u8], + snapshot_uuid: &[u8; UUID_LEN], +) -> Result<()> { + if fresh.is_empty() || fresh.len() > MAX_WRAPPED { + return Err(EINVAL); + } + let stored = match read(slot)? { + State::Present(stored) => stored, + State::Absent(_) => return Err(ENOENT), + }; + if stored.uuid() != snapshot_uuid { + // A snapshot of a different bag would strand the stored one. + return Err(EPERM); + } + write_record( + slot, + stored.uuid_provenance().state(), + fresh, + stored.uuid(), + stored.secret(), + ) +} + +fn write_record( + slot: Slot, + state: u32, + wrapped: &[u8], + uuid: &[u8; UUID_LEN], + secret: &[u8], +) -> Result<()> { + let crc = crc16_ccitt_false(&checksum_input(state, wrapped, uuid, secret)?); + + let mut head = [0u8; HEADER]; + head[..MAGIC.len()].copy_from_slice(&MAGIC); + head[OFF_VERSION..OFF_VERSION + 4].copy_from_slice(&VERSION.to_le_bytes()); + head[OFF_STATE..OFF_STATE + 4].copy_from_slice(&state.to_le_bytes()); + head[OFF_LEN..OFF_LEN + 4].copy_from_slice(&(wrapped.len() as u32).to_le_bytes()); + head[OFF_CRC..OFF_CRC + 2].copy_from_slice(&crc.to_le_bytes()); + head[OFF_UUID..OFF_UUID + UUID_LEN].copy_from_slice(uuid); + head[OFF_SECRET_LEN..OFF_SECRET_LEN + 4].copy_from_slice(&(secret.len() as u32).to_le_bytes()); + + let file = shim::StoreFile::open_trunc(slot.path())?; + // Body first, then the header that vouches for it: a crash between leaves a + // bad checksum, which `read` refuses — the safe side. + if !wrapped.is_empty() { + file.write_all(HEADER as u64, wrapped)?; + } + if !secret.is_empty() { + file.write_all((HEADER + wrapped.len()) as u64, secret)?; + } + file.write_all(0, &head)?; + file.sync() +} diff --git a/drivers/soc/apple/p256_shim.c b/drivers/soc/apple/p256_shim.c new file mode 100644 index 0000000000000..bcf1323e3c58f --- /dev/null +++ b/drivers/soc/apple/p256_shim.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Dj */ +/* + * Apple SEP driver — P-256 ECDH sender shim for the ref-key ECIES seal. + * + * The ref-key (op 0x22) SE-seal is an ECIES scheme: the AP encrypts to the + * ref-key's public point and the enclave decrypts (op "od"). The enclave holds + * the recipient private key and never exposes it; the encrypt half touches only + * the public key, so it runs here in the kernel with no secret crossing into + * the SEP request but the (public) ephemeral point. This shim does the sender's + * key agreement: generate an ephemeral P-256 keypair, return its public point + * for the envelope, and compute the ECDH shared secret against the recipient's. + * + * Byte order: the kernel's `ecdh-nist-p256` kpp is big-endian throughout + * (ecdh_set_secret decodes the key via ecc_digits_from_bytes; ecc_swap_digits + * reads/writes coordinates as __be64). So the private key, both public points + * (X||Y), and the shared secret are plain big-endian byte strings, matching the + * ECIES/SEP convention: this shim does no byte swapping. It only marshals + * through kmalloc'd buffers, since a scatterlist entry must live in the linear + * map and the caller's may not. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "shim.h" + +#define P256_COORD_LEN 32 +#define P256_POINT_LEN 64 /* X || Y, big-endian, no 0x04 prefix */ + +/* + * Core key agreement. `priv_be` is a 32-byte big-endian private scalar, or NULL + * to generate a fresh ephemeral key. `pub_be_out` receives the 64-byte + * big-endian public point (X||Y), `shared_be_out` the 32-byte big-endian shared + * secret (X coordinate of priv*peer). Scatterlist buffers are kmalloc'd since + * sg_init_one requires the linear map. + */ +static int p256_agree(const u8 *priv_be, const u8 *peer_pub_be, + u8 *pub_be_out, u8 *shared_be_out) +{ + struct crypto_kpp *tfm; + struct kpp_request *req; + struct ecdh params; + struct scatterlist src, dst; + DECLARE_CRYPTO_WAIT(wait); + unsigned int enc_len; + char *enc = NULL; + u8 *pub_buf = NULL; + u8 *peer_buf = NULL; + u8 *shared_buf = NULL; + int rc; + + tfm = crypto_alloc_kpp("ecdh-nist-p256", 0, 0); + if (IS_ERR(tfm)) { + rc = PTR_ERR(tfm); + pr_err("apple_sep p256: alloc_kpp rc=%d\n", rc); + return rc; + } + + memset(¶ms, 0, sizeof(params)); + if (priv_be) { + params.key = (void *)priv_be; /* big-endian, as the kpp wants */ + params.key_size = P256_COORD_LEN; + } else { + params.key = NULL; + params.key_size = 0; /* kpp generates a valid random scalar */ + } + + enc_len = crypto_ecdh_key_len(¶ms); + enc = kmalloc(enc_len, GFP_KERNEL); + if (!enc) { + rc = -ENOMEM; + goto out_tfm; + } + rc = crypto_ecdh_encode_key(enc, enc_len, ¶ms); + if (rc) { + pr_err("apple_sep p256: encode_key rc=%d\n", rc); + goto out_enc; + } + rc = crypto_kpp_set_secret(tfm, enc, enc_len); + if (rc) { + pr_err("apple_sep p256: set_secret rc=%d\n", rc); + goto out_enc; + } + + req = kpp_request_alloc(tfm, GFP_KERNEL); + if (!req) { + rc = -ENOMEM; + goto out_enc; + } + + pub_buf = kmalloc(P256_POINT_LEN, GFP_KERNEL); + if (!pub_buf) { + rc = -ENOMEM; + goto out_req; + } + sg_init_one(&dst, pub_buf, P256_POINT_LEN); + kpp_request_set_input(req, NULL, 0); + kpp_request_set_output(req, &dst, P256_POINT_LEN); + kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + rc = crypto_wait_req(crypto_kpp_generate_public_key(req), &wait); + if (rc) { + pr_err("apple_sep p256: generate_public_key rc=%d\n", rc); + goto out_pub; + } + memcpy(pub_be_out, pub_buf, P256_POINT_LEN); + + peer_buf = kmalloc(P256_POINT_LEN, GFP_KERNEL); + shared_buf = kmalloc(P256_COORD_LEN, GFP_KERNEL); + if (!peer_buf || !shared_buf) { + rc = -ENOMEM; + goto out_pub; + } + memcpy(peer_buf, peer_pub_be, P256_POINT_LEN); + sg_init_one(&src, peer_buf, P256_POINT_LEN); + sg_init_one(&dst, shared_buf, P256_COORD_LEN); + kpp_request_set_input(req, &src, P256_POINT_LEN); + kpp_request_set_output(req, &dst, P256_COORD_LEN); + kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + rc = crypto_wait_req(crypto_kpp_compute_shared_secret(req), &wait); + if (rc) { + pr_err("apple_sep p256: compute_shared_secret rc=%d\n", rc); + goto out_pub; + } + memcpy(shared_be_out, shared_buf, P256_COORD_LEN); + rc = 0; + +out_pub: + if (shared_buf) { + memzero_explicit(shared_buf, P256_COORD_LEN); + kfree(shared_buf); + } + kfree(peer_buf); + kfree(pub_buf); +out_req: + kpp_request_free(req); +out_enc: + kfree_sensitive(enc); +out_tfm: + crypto_free_kpp(tfm); + return rc; +} + +/* + * ECIES sender key agreement with a fresh ephemeral key. `peer_pub_be` is the + * recipient's 64-byte big-endian public point (0x04 prefix already stripped by + * the caller). `eph_pub_be_out` receives the 64-byte big-endian ephemeral + * public point (caller prepends 0x04 for the envelope), `shared_be_out` the + * 32-byte big-endian shared secret for the KDF. + */ +int sep_p256_sender(const void *peer_pub_be, void *eph_pub_be_out, + void *shared_be_out) +{ + return p256_agree(NULL, peer_pub_be, eph_pub_be_out, shared_be_out); +} + diff --git a/drivers/soc/apple/proto.rs b/drivers/soc/apple/proto.rs new file mode 100644 index 0000000000000..d084a889109c5 --- /dev/null +++ b/drivers/soc/apple/proto.rs @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! The SEP wire protocol, reverse-engineered: opcode tables, request encoders +//! and reply decoders. +#![allow(dead_code)] + +use kernel::prelude::*; +use kernel::soc::apple::mailbox::Message; + +pub(crate) const EP_CONTROL: u8 = 0x00; +pub(crate) const EP_DISCOVER: u8 = 0xFD; +pub(crate) const EP_SHMEM: u8 = 0xFE; +pub(crate) const EP_BOOT: u8 = 0xFF; +pub(crate) const EP_XARM: u8 = 0x13; +pub(crate) const EP_SBIO: u8 = 0x08; +pub(crate) const EP_XARS: u8 = 0x10; +pub(crate) const EP_SCRD: u8 = 0x0a; +pub(crate) const EP_SKS: u8 = 0x12; + +pub(crate) const DISCOVER_TYPE_DESCRIPTOR: u8 = 0x00; +pub(crate) const DISCOVER_TYPE_CONFIG: u8 = 0x01; + +pub(crate) const MSG_TAG_SHIFT: u32 = 8; +pub(crate) const MSG_TYPE_SHIFT: u32 = 16; +pub(crate) const MSG_PARAM_SHIFT: u32 = 24; +pub(crate) const MSG_DATA_SHIFT: u32 = 32; + +// 4 KiB units even though CPU pages are 16 KiB +pub(crate) const IOVA_SHIFT: u32 = 12; + +#[derive(Clone, Copy)] +pub(crate) struct Fields { + pub(crate) ep: u8, + pub(crate) tag: u8, + pub(crate) ty: u8, + pub(crate) param: u8, + pub(crate) data_lo: u32, +} + +pub(crate) fn decode(msg: &Message) -> Fields { + Fields { + ep: msg.msg0 as u8, + tag: (msg.msg0 >> MSG_TAG_SHIFT) as u8, + ty: (msg.msg0 >> MSG_TYPE_SHIFT) as u8, + param: (msg.msg0 >> MSG_PARAM_SHIFT) as u8, + data_lo: (msg.msg0 >> MSG_DATA_SHIFT) as u32, + } +} + +pub(crate) const fn encode_registration_msg0(iova: u64, size: usize) -> u64 { + (EP_SHMEM as u64) + | (((size as u64) >> IOVA_SHIFT) << MSG_TYPE_SHIFT) + | ((iova >> IOVA_SHIFT) << MSG_DATA_SHIFT) +} + +static_assert!(encode_registration_msg0(0xBEE0_0000, 0x4_0000) == 0x000b_ee00_0040_00fe); + +pub(crate) fn shmem_registration(iova: u64, size: usize) -> Result { + let unit = 1u64 << IOVA_SHIFT; + + if size == 0 || (size as u64) & (unit - 1) != 0 { + return Err(EINVAL); + } + if iova & (unit - 1) != 0 { + return Err(EINVAL); + } + + let size_field = (size as u64) >> IOVA_SHIFT; + if size_field > 0xFF { + return Err(EINVAL); + } + + let iova_field = iova >> IOVA_SHIFT; + if iova_field > u64::from(u32::MAX) { + return Err(EINVAL); + } + + Ok(Message { + msg0: encode_registration_msg0(iova, size), + msg1: 0, + }) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct Fourcc(pub(crate) [u8; 4]); + +impl Fourcc { + pub(crate) const ZERO: Fourcc = Fourcc([0; 4]); +} + +impl kernel::fmt::Display for Fourcc { + fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + use core::fmt::Write; + for c in self.0 { + let c = if (0x20..0x7f).contains(&c) { c } else { b'.' }; + f.write_char(c as char)?; + } + Ok(()) + } +} + +pub(crate) fn fourcc(msg: &Message) -> Fourcc { + Fourcc(((msg.msg0 >> MSG_DATA_SHIFT) as u32).to_be_bytes()) +} + +pub(crate) const CONTROL_REPLY_TYPE: u8 = 0x01; + +pub(crate) const TAG_UNSOLICITED: u8 = 0x00; + +pub(crate) const TAG_ENTROPY: u8 = 0xE7; + +pub(crate) const CONTROL_TIMEOUT_MS: u32 = 2000; + +pub(crate) const TAG_POOL_FIRST: u8 = 0x01; +pub(crate) const TAG_POOL_LAST: u8 = 0x7e; + +static_assert!(TAG_POOL_FIRST > TAG_UNSOLICITED); +static_assert!(TAG_POOL_LAST < TAG_ENTROPY); + +pub(crate) struct ControlOp { + ty: u8, + param: u8, + data: u32, + expect_reply: bool, + timeout_ms: u32, + reserved_tag: bool, + name: &'static CStr, +} + +impl ControlOp { + pub(crate) fn expects_reply(&self) -> bool { + self.expect_reply + } + pub(crate) fn timeout_ms(&self) -> u32 { + self.timeout_ms + } + pub(crate) fn uses_reserved_tag(&self) -> bool { + self.reserved_tag + } + pub(crate) fn name(&self) -> &'static CStr { + self.name + } +} + +pub(crate) fn op_nop(param: u8) -> ControlOp { + ControlOp { + ty: 0x00, + param, + data: 0, + expect_reply: true, + timeout_ms: CONTROL_TIMEOUT_MS, + reserved_tag: false, + name: c"NOP", + } +} + +pub(crate) fn op_security_mode() -> ControlOp { + ControlOp { + ty: 0x14, + param: 0x00, + data: 0, + expect_reply: true, + timeout_ms: CONTROL_TIMEOUT_MS, + reserved_tag: false, + name: c"SECMODE", + } +} + +// no op for control 0x18: it wedges the control endpoint + +pub(crate) fn op_get_entropy() -> ControlOp { + ControlOp { + ty: 0x36, + param: 0x00, + data: 0, + expect_reply: true, + timeout_ms: CONTROL_TIMEOUT_MS, + reserved_tag: true, + name: c"GET_ENTROPY", + } +} + +pub(crate) fn encode_control(op: &ControlOp, tag: u8) -> Message { + Message { + msg0: u64::from(EP_CONTROL) + | (u64::from(tag) << MSG_TAG_SHIFT) + | (u64::from(op.ty) << MSG_TYPE_SHIFT) + | (u64::from(op.param) << MSG_PARAM_SHIFT) + | (u64::from(op.data) << MSG_DATA_SHIFT), + msg1: 0, + } +} + +const OP_OOL_INBOUND_SIZE: u8 = 0x04; +const OP_OOL_INBOUND_ADDR: u8 = 0x02; +const OP_OOL_OUTBOUND_SIZE: u8 = 0x05; +const OP_OOL_OUTBOUND_ADDR: u8 = 0x03; + +fn op_ool(ty: u8, endpoint: u8, data: u32, name: &'static CStr) -> ControlOp { + ControlOp { + ty, + param: endpoint, + data, + expect_reply: true, + timeout_ms: CONTROL_TIMEOUT_MS, + reserved_tag: false, + name, + } +} + +const OP_DMA_RING_PAGES: u8 = 0x19; +const OP_DMA_RING_ADDR: u8 = 0x1a; + +pub(crate) const DMA_RING_PAGES: u32 = 4; + +pub(crate) fn op_dma_ring_pages(endpoint: u8, pages: u32) -> ControlOp { + op_ool(OP_DMA_RING_PAGES, endpoint, pages, c"RING_PAGES") +} + +pub(crate) fn op_dma_ring_addr(endpoint: u8, iova: u64) -> ControlOp { + op_ool( + OP_DMA_RING_ADDR, + endpoint, + (iova >> IOVA_SHIFT) as u32, + c"RING_ADDR", + ) +} + +pub(crate) fn op_ool_inbound_size(endpoint: u8, len: u32) -> ControlOp { + op_ool(OP_OOL_INBOUND_SIZE, endpoint, len, c"OOL_IN_SIZE") +} + +pub(crate) fn op_ool_inbound_addr(endpoint: u8, iova: u64) -> ControlOp { + op_ool( + OP_OOL_INBOUND_ADDR, + endpoint, + (iova >> IOVA_SHIFT) as u32, + c"OOL_IN_ADDR", + ) +} + +pub(crate) fn op_ool_outbound_size(endpoint: u8, len: u32) -> ControlOp { + op_ool(OP_OOL_OUTBOUND_SIZE, endpoint, len, c"OOL_OUT_SIZE") +} + +pub(crate) fn op_ool_outbound_addr(endpoint: u8, iova: u64) -> ControlOp { + op_ool( + OP_OOL_OUTBOUND_ADDR, + endpoint, + (iova >> IOVA_SHIFT) as u32, + c"OOL_OUT_ADDR", + ) +} + +pub(crate) struct FieldCursor<'a> { + body: &'a [u8], + at: usize, +} + +impl<'a> FieldCursor<'a> { + pub(crate) fn new(body: &'a [u8]) -> FieldCursor<'a> { + FieldCursor { body, at: 0 } + } + + pub(crate) fn i32(&mut self) -> Option { + let end = self.at.checked_add(4)?; + let v = i32::from_le_bytes(self.body.get(self.at..end)?.try_into().ok()?); + self.at = end; + Some(v) + } + + // blob = u32 len + bytes + zero pad to 4-byte boundary; 0x45 reply has two in a row + pub(crate) fn blob(&mut self) -> Option<&'a [u8]> { + let len_end = self.at.checked_add(4)?; + let len = u32::from_le_bytes(self.body.get(self.at..len_end)?.try_into().ok()?) as usize; + let end = len_end.checked_add(len)?; + let bytes = self.body.get(len_end..end)?; + let pad = len.wrapping_neg() % 4; + self.at = end.checked_add(pad)?; + Some(bytes) + } +} + +#[derive(Clone, Copy)] +pub(crate) struct ControlReply { + pub(crate) tag: u8, + pub(crate) data_lo: u32, + pub(crate) msg1: u32, +} + +impl ControlReply { + pub(crate) fn from_message(msg: &Message) -> ControlReply { + let f = decode(msg); + ControlReply { + tag: f.tag, + data_lo: f.data_lo, + msg1: msg.msg1, + } + } +} diff --git a/drivers/soc/apple/refkey.rs b/drivers/soc/apple/refkey.rs new file mode 100644 index 0000000000000..c0456a604b38d --- /dev/null +++ b/drivers/soc/apple/refkey.rs @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj +//! Machine ref-key: Secure-Enclave key sealing, signing, and attestation. +//! +//! A ref-key (op `0x22`) is an EC keypair the enclave keeps; the private half +//! never leaves the SEP. The host-side ECIES encrypt lives in [`refkey_seal`]; +//! every half that needs the private key runs in the enclave. + +use crate::{image, keybag, proto, refkey_seal, shim}; +use crate::{LockState, MachineRefKey, SepData, SksRequest}; +use kernel::prelude::*; + +pub(crate) const REFKEY_ENVELOPE_VERSION: u32 = 2; + +impl SepData { + fn sks_req_refkey(&self, keybag: i32, der_set: &[u8]) -> Result { + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(keybag)?; + body.put_u32(crate::refkey::REFKEY_ENVELOPE_VERSION)?; + body.put_blob(der_set)?; + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_perform_operation(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_PERFORM_OP_NAME, + msg, + img, + }) + } + + fn sks_create_refkey( + &self, + handle: crate::sks::KeyBagHandle, + secret: &[u8], + ) -> Option> { + use crate::der::RefKeyValue as RV; + + // Class 9 = non-extractable (private stays enclave-resident); key type 5 = P-256. + const PROTECTION_CLASS: u32 = 9; + const KEY_TYPE: u32 = 5; + + // Unlock the bag first; an unbound create is refused. + if let Some(healthy) = self.sks_health_check(c"ref-key create") { + let _ = self.sks_send(self.sks_req_change_lock_state( + handle, + LockState::Unlocked, + secret, + healthy, + )); + } + + let create_der = { + let mut items: KVec<(&[u8], RV<'_>)> = KVec::new(); + if items.push((b"o", RV::Utf8(b"oc")), GFP_KERNEL).is_err() + || items.push((b"bc", RV::Integer(PROTECTION_CLASS)), GFP_KERNEL).is_err() + || items.push((b"kt", RV::Integer(KEY_TYPE)), GFP_KERNEL).is_err() + { + return None; + } + crate::der::encode_refkey_set(&items).ok()? + }; + let cout = self.sks_send(self.sks_req_refkey(handle.value(), &create_der))?; + if cout.reply.status != 0 { + return None; + } + let cbody = + self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &cout)?; + let mut cf = proto::FieldCursor::new(cbody); + let (Some(0), Some(cblob)) = (cf.i32(), cf.blob()) else { + return None; + }; + let mut refkey_blob: KVec = KVec::new(); + refkey_blob.extend_from_slice(cblob, GFP_KERNEL).ok()?; + Some(refkey_blob) + } + + fn refkey_pub(blob: &[u8]) -> Option<&[u8]> { + let rk_tlv = crate::der::refkey_find(blob, b"rk")?; + let pub_raw = crate::der::refkey_find(rk_tlv, b"pub") + .and_then(crate::der::octet_string_body)?; + (pub_raw.len() == refkey_seal::POINT_LEN).then_some(pub_raw) + } + + pub(crate) fn sks_machine_refkey(&self, handle: crate::sks::KeyBagHandle, secret: &[u8]) { + const MACHINE_REFKEY_PATH: &CStr = c"/var/lib/apple-sep-refkey.bin"; + + if self.machine_refkey.lock().is_some() { + return; + } + + if let Ok(f) = shim::StoreFile::open_readonly(MACHINE_REFKEY_PATH) { + let sz = f.size().unwrap_or(0); + if (1..=8192).contains(&sz) { + let mut blob: KVec = KVec::new(); + if blob.resize(sz as usize, 0u8, GFP_KERNEL).is_ok() + && f.read_exact(0, &mut blob).is_ok() + && self.cache_machine_refkey(blob) + { + return; + } + } + } + + let Some(blob) = self.sks_create_refkey(handle, secret) else { + dev_warn!( + self.dev, + "sks: could not create the machine ref-key; trusted-key sealing is unavailable this boot.\n" + ); + return; + }; + if let Ok(f) = shim::StoreFile::open(MACHINE_REFKEY_PATH) { + if f.write_all(0, &blob).is_ok() { + let _ = f.sync(); + } + } + let _ = self.cache_machine_refkey(blob); + } + + fn cache_machine_refkey(&self, blob: KVec) -> bool { + let Some(pubv) = Self::refkey_pub(&blob) else { + return false; + }; + let mut pub_raw: KVec = KVec::new(); + if pub_raw.extend_from_slice(pubv, GFP_KERNEL).is_err() { + return false; + } + *self.machine_refkey.lock() = Some(MachineRefKey { blob, pub_raw }); + true + } + + fn ensure_machine_refkey(&self) -> Result<()> { + if self.machine_refkey.lock().is_some() { + return Ok(()); + } + if !self.sks_ready() { + return Err(ENODEV); + } + let keybag::State::Present(stored) = keybag::read(keybag::Slot::Identity)? else { + return Err(ENODEV); + }; + let (handle, _uuid) = self.sks_recover(&stored).ok_or(EIO)?; + self.sks_machine_refkey(handle, stored.secret()); + let _ = self.sks_send(self.sks_req_unload_keybag(handle)); + if self.machine_refkey.lock().is_some() { + Ok(()) + } else { + Err(EIO) + } + } + + pub(crate) fn refkey_seal_trusted(&self, key: &[u8]) -> Result> { + self.ensure_machine_refkey()?; + let guard = self.machine_refkey.lock(); + let mk = guard.as_ref().ok_or(ENODEV)?; + refkey_seal::ecies_seal(&mk.pub_raw, key) + } + + pub(crate) fn refkey_unseal_trusted(&self, sealed: &[u8]) -> Result> { + self.ensure_machine_refkey()?; + let blob = { + let guard = self.machine_refkey.lock(); + let mk = guard.as_ref().ok_or(ENODEV)?; + let mut b: KVec = KVec::new(); + b.extend_from_slice(&mk.blob, GFP_KERNEL)?; + b + }; + let keybag::State::Present(stored) = keybag::read(keybag::Slot::Identity)? else { + return Err(ENODEV); + }; + let (handle, _uuid) = self.sks_recover(&stored).ok_or(EIO)?; + if let Some(healthy) = self.sks_health_check(c"trusted-key unseal") { + let _ = self.sks_send(self.sks_req_change_lock_state( + handle, + LockState::Unlocked, + stored.secret(), + healthy, + )); + } + let recovered = self.sks_refkey_unseal(handle, &blob, sealed); + let _ = self.sks_send(self.sks_req_unload_keybag(handle)); + recovered.ok_or(EIO) + } + + /// Op `osgn`: ECDSA-P256 over `challenge` as the pre-computed digest. + fn sks_refkey_sign( + &self, + handle: crate::sks::KeyBagHandle, + refkey_blob: &[u8], + challenge: &[u8], + ) -> Option> { + use crate::der::RefKeyValue as RV; + let mut items: KVec<(&[u8], RV<'_>)> = KVec::new(); + items.push((b"o", RV::Utf8(b"osgn")), GFP_KERNEL).ok()?; + items.push((b"d", RV::Octets(challenge)), GFP_KERNEL).ok()?; + items.push((b"rk", RV::Der(refkey_blob)), GFP_KERNEL).ok()?; + let der = crate::der::encode_refkey_set(&items).ok()?; + let out = self.sks_send(self.sks_req_refkey(handle.value(), &der))?; + if out.reply.status != 0 { + dev_warn!(self.dev, "sks: ref-key attest sign failed (status {})\n", out.reply.status); + return None; + } + let body = self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &out)?; + let mut f = proto::FieldCursor::new(body); + let (Some(0), Some(sig)) = (f.i32(), f.blob()) else { + return None; + }; + // Enclave wraps the sig in OCTET STRING { SEQUENCE { r, s } }; strip to inner DER. + let der_sig = crate::der::octet_string_body(sig).unwrap_or(sig); + let mut owned: KVec = KVec::new(); + owned.extend_from_slice(der_sig, GFP_KERNEL).ok()?; + Some(owned) + } + + /// Attestation of key possession by signing proof; op `oa` (Apple-CA-chained) + /// needs the SEP device attestation key, absent on a Linux-attached SEP. + pub(crate) fn refkey_attest_sign(&self, challenge: &[u8]) -> Result<(KVec, KVec)> { + self.ensure_machine_refkey()?; + let (blob, pubk) = { + let guard = self.machine_refkey.lock(); + let mk = guard.as_ref().ok_or(ENODEV)?; + let mut b: KVec = KVec::new(); + b.extend_from_slice(&mk.blob, GFP_KERNEL)?; + let mut p: KVec = KVec::new(); + p.extend_from_slice(&mk.pub_raw, GFP_KERNEL)?; + (b, p) + }; + let keybag::State::Present(stored) = keybag::read(keybag::Slot::Identity)? else { + return Err(ENODEV); + }; + let (handle, _uuid) = self.sks_recover(&stored).ok_or(EIO)?; + if let Some(healthy) = self.sks_health_check(c"ref-key attest") { + let _ = self.sks_send(self.sks_req_change_lock_state( + handle, + LockState::Unlocked, + stored.secret(), + healthy, + )); + } + let sig = self.sks_refkey_sign(handle, &blob, challenge); + let _ = self.sks_send(self.sks_req_unload_keybag(handle)); + Ok((sig.ok_or(EIO)?, pubk)) + } + + /// Enclave ECIES-decrypt (o = "oecd"); takes the ephemeral from the front of `d`. + fn sks_refkey_unseal( + &self, + handle: crate::sks::KeyBagHandle, + refkey_blob: &[u8], + sealed: &[u8], + ) -> Option> { + use crate::der::RefKeyValue as RV; + let mut items: KVec<(&[u8], RV<'_>)> = KVec::new(); + items.push((b"o", RV::Utf8(b"oecd")), GFP_KERNEL).ok()?; + items.push((b"d", RV::Octets(sealed)), GFP_KERNEL).ok()?; + items.push((b"rk", RV::Der(refkey_blob)), GFP_KERNEL).ok()?; + let der = crate::der::encode_refkey_set(&items).ok()?; + let out = self.sks_send(self.sks_req_refkey(handle.value(), &der))?; + if out.reply.status != 0 { + dev_warn!(self.dev, "trusted-keys: ref-key unseal failed (status {})\n", out.reply.status); + return None; + } + let body = self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &out)?; + let mut f = proto::FieldCursor::new(body); + match (f.i32(), f.blob()) { + (Some(0), Some(rec)) => { + let mut owned = KVec::new(); + owned.extend_from_slice(rec, GFP_KERNEL).ok()?; + Some(owned) + } + _ => None, + } + } +} diff --git a/drivers/soc/apple/refkey_seal.rs b/drivers/soc/apple/refkey_seal.rs new file mode 100644 index 0000000000000..f1d95935690b5 --- /dev/null +++ b/drivers/soc/apple/refkey_seal.rs @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj +//! Ref-key (op 0x22) ECIES **encrypt** — the AP half of the SE seal. +//! +//! The SEP has no scalar×G, so it cannot make the sender ephemeral (in-SEP `ow` +//! returns -10); `od` imports the one this module produces, ECDHs it against the +//! in-SEP private key and AES-GCM-decrypts. The encrypt runs here by construction. + +use kernel::prelude::*; + +extern "C" { + fn sep_p256_sender( + peer_pub_be: *const c_void, + eph_pub_be_out: *mut u8, + shared_be_out: *mut u8, + ) -> c_int; + + fn sep_gcm( + encrypt: c_int, + key: *const c_void, + keylen: usize, + iv: *const c_void, + ivlen: usize, + aadlen: usize, + buf: *mut c_void, + buflen: usize, + datalen: usize, + ) -> c_int; +} + +// The enclave's SE seal is an ECIES scheme: +// ECDH P-256 (shared = X), X9.63-SHA256 KDF, KDF-derived key‖IV, AES-GCM. + +pub(crate) const TAG_LEN: usize = 16; +/// 16-byte IV (KDF-derived "variable IV"), not the 12-byte GCM default. +pub(crate) const IV_LEN: usize = 16; +pub(crate) const AES256_KEY_LEN: usize = 32; +pub(crate) const AES128_KEY_LEN: usize = 16; +/// Uncompressed P-256 point: `0x04 ‖ X ‖ Y`. +pub(crate) const POINT_LEN: usize = 65; +/// Raw `X ‖ Y` (no prefix), as the ECDH shim exchanges. +const COORDS_LEN: usize = 64; + +fn wipe(bytes: &mut [u8]) { + for b in bytes.iter_mut() { + // SAFETY: `b` is a valid, uniquely borrowed byte for this write. + unsafe { core::ptr::write_volatile(b, 0) }; + } +} + +fn p256_sender(peer_pub64: &[u8]) -> Result<([u8; COORDS_LEN], [u8; 32])> { + if peer_pub64.len() != COORDS_LEN { + return Err(EINVAL); + } + let mut eph = [0u8; COORDS_LEN]; + let mut shared = [0u8; 32]; + // SAFETY: `peer_pub64` is 64 live bytes; `eph`/`shared` are exactly the 64 + // and 32 bytes the shim writes. + let rc = unsafe { + sep_p256_sender( + peer_pub64.as_ptr().cast(), + eph.as_mut_ptr(), + shared.as_mut_ptr(), + ) + }; + if rc != 0 { + return Err(Error::from_errno(rc)); + } + Ok((eph, shared)) +} + +fn kdf_sha256_mode(z: &[u8], shared_info: &[u8], out_len: usize, mode: u8) -> Result> { + let mut out = KVec::new(); + let mut counter: u32 = 1; + while out.len() < out_len { + let ctr = counter.to_be_bytes(); + let mut buf = KVec::new(); + match mode { + 0 => { + buf.extend_from_slice(z, GFP_KERNEL)?; + buf.extend_from_slice(&ctr, GFP_KERNEL)?; + buf.extend_from_slice(shared_info, GFP_KERNEL)?; + } + 1 => { + buf.extend_from_slice(&ctr, GFP_KERNEL)?; + buf.extend_from_slice(z, GFP_KERNEL)?; + buf.extend_from_slice(shared_info, GFP_KERNEL)?; + } + 2 => { + buf.extend_from_slice(z, GFP_KERNEL)?; + buf.extend_from_slice(shared_info, GFP_KERNEL)?; + buf.extend_from_slice(&ctr, GFP_KERNEL)?; + } + 3 => { + buf.extend_from_slice(shared_info, GFP_KERNEL)?; + buf.extend_from_slice(z, GFP_KERNEL)?; + buf.extend_from_slice(&ctr, GFP_KERNEL)?; + } + _ => { + buf.extend_from_slice(z, GFP_KERNEL)?; + buf.extend_from_slice(shared_info, GFP_KERNEL)?; + } + } + let digest = crate::image::sha256(&buf)?; + wipe(&mut buf); + let take = core::cmp::min(digest.len(), out_len - out.len()); + out.extend_from_slice(&digest[..take], GFP_KERNEL)?; + counter = counter.checked_add(1).ok_or(EINVAL)?; + } + Ok(out) +} + +fn gcm_encrypt(key: &[u8], iv: &[u8], aad: &[u8], plaintext: &[u8]) -> Result> { + let mut buf = KVec::new(); + buf.extend_from_slice(aad, GFP_KERNEL)?; + buf.extend_from_slice(plaintext, GFP_KERNEL)?; + buf.resize(aad.len() + plaintext.len() + TAG_LEN, 0u8, GFP_KERNEL)?; + // SAFETY: `key`/`iv` are live; `buf` holds aad‖plaintext‖tag-space and its + // length is passed as both the buffer size and the extent to authenticate. + let rc = unsafe { + sep_gcm( + 1, + key.as_ptr().cast(), + key.len(), + iv.as_ptr().cast(), + iv.len(), + aad.len(), + buf.as_mut_ptr().cast(), + buf.len(), + plaintext.len(), + ) + }; + if rc != 0 { + return Err(Error::from_errno(rc)); + } + let mut out = KVec::new(); + out.extend_from_slice(&buf[aad.len()..], GFP_KERNEL)?; + Ok(out) +} + +pub(crate) struct EciesParts { + /// Ephemeral point `0x04 ‖ X ‖ Y`; also the KDF sharedInfo. + pub(crate) eph_point: KVec, + pub(crate) ciphertext: KVec, + pub(crate) tag: KVec, +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn ecies_encrypt_parts( + recipient_pub: &[u8], + plaintext: &[u8], + key_len: usize, + variable_iv: bool, + iv_len: usize, + aad_eph: bool, + si_mode: u8, + kdf_mode: u8, +) -> Result { + if recipient_pub.len() != POINT_LEN || recipient_pub[0] != 0x04 { + return Err(EINVAL); + } + if key_len != AES256_KEY_LEN && key_len != AES128_KEY_LEN { + return Err(EINVAL); + } + if iv_len != 12 && iv_len != 16 { + return Err(EINVAL); + } + let (eph_coords, mut shared) = p256_sender(&recipient_pub[1..])?; + + let mut eph_point = KVec::new(); + eph_point.push(0x04, GFP_KERNEL)?; + eph_point.extend_from_slice(&eph_coords, GFP_KERNEL)?; + + let mut si = KVec::new(); + match si_mode { + 0 => si.extend_from_slice(&eph_point, GFP_KERNEL)?, + 1 => {} + 2 => si.extend_from_slice(&eph_point[1..], GFP_KERNEL)?, + 3 => si.extend_from_slice(&eph_point[1..33], GFP_KERNEL)?, + 4 => si.extend_from_slice(recipient_pub, GFP_KERNEL)?, + _ => { + si.extend_from_slice(&eph_point, GFP_KERNEL)?; + si.extend_from_slice(recipient_pub, GFP_KERNEL)?; + } + } + + let kdf_len = if variable_iv { + key_len + iv_len + } else { + key_len + }; + let mut km = kdf_sha256_mode(&shared, &si, kdf_len, kdf_mode)?; + wipe(&mut shared); + let mut iv = KVec::new(); + if variable_iv { + iv.extend_from_slice(&km[key_len..key_len + iv_len], GFP_KERNEL)?; + } else { + iv.resize(iv_len, 0u8, GFP_KERNEL)?; + } + let aad: &[u8] = if aad_eph { &eph_point } else { &[] }; + let ct_tag = gcm_encrypt(&km[..key_len], &iv, aad, plaintext); + wipe(&mut km); + let ct_tag = ct_tag?; + if ct_tag.len() != plaintext.len() + TAG_LEN { + return Err(EINVAL); + } + + let mut ciphertext = KVec::new(); + ciphertext.extend_from_slice(&ct_tag[..plaintext.len()], GFP_KERNEL)?; + let mut tag = KVec::new(); + tag.extend_from_slice(&ct_tag[plaintext.len()..], GFP_KERNEL)?; + + Ok(EciesParts { + eph_point, + ciphertext, + tag, + }) +} + +/// Returns the enclave-ready blob: `ephemeral_point ‖ ciphertext ‖ tag`. +pub(crate) fn ecies_seal(recipient_pub: &[u8], secret: &[u8]) -> Result> { + let parts = ecies_encrypt_parts( + recipient_pub, + secret, + AES256_KEY_LEN, + true, + IV_LEN, + false, + 0, + 0, + )?; + let mut blob = KVec::new(); + blob.extend_from_slice(&parts.eph_point, GFP_KERNEL)?; + blob.extend_from_slice(&parts.ciphertext, GFP_KERNEL)?; + blob.extend_from_slice(&parts.tag, GFP_KERNEL)?; + Ok(blob) +} diff --git a/drivers/soc/apple/rxring.rs b/drivers/soc/apple/rxring.rs new file mode 100644 index 0000000000000..4aee7f407fa76 --- /dev/null +++ b/drivers/soc/apple/rxring.rs @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +use core::cell::UnsafeCell; +use kernel::soc::apple::mailbox::Message; +use kernel::sync::atomic::{Acquire, Atomic, Relaxed, Release}; + +const RING_LEN: u32 = 64; +const RING_MASK: u32 = RING_LEN - 1; + +pub(crate) struct RxRing { + slots: [UnsafeCell; RING_LEN as usize], + head: Atomic, + tail: Atomic, + dropped: Atomic, +} + +// SAFETY: ownership of each slot passes between the two sides via the +// release/acquire pairing on `head`/`tail`, so no slot is touched by both at +// once; single-producer/consumer are enforced by rx_lock and workqueue non-reentrancy. +unsafe impl Sync for RxRing {} +// SAFETY: `Message` is a plain data struct with no thread affinity. +unsafe impl Send for RxRing {} + +impl RxRing { + pub(crate) fn new() -> Self { + RxRing { + slots: core::array::from_fn(|_| UnsafeCell::new(Message { msg0: 0, msg1: 0 })), + head: Atomic::new(0), + tail: Atomic::new(0), + dropped: Atomic::new(0), + } + } + + pub(crate) fn push(&self, msg: Message) -> bool { + let head = self.head.load(Relaxed); + // Acquire pairs with the consumer's release of `tail`, so the reused slot is free. + let tail = self.tail.load(Acquire); + + if head.wrapping_sub(tail) >= RING_LEN { + self.dropped + .store(self.dropped.load(Relaxed).wrapping_add(1), Relaxed); + return false; + } + + let slot = &self.slots[(head & RING_MASK) as usize]; + // SAFETY: `head` is not published yet, so the consumer cannot be looking at + // this slot, and we are the only producer. + unsafe { slot.get().write(msg) }; + + // Release pairs with the consumer's acquire of `head`, publishing the slot write. + self.head.store(head.wrapping_add(1), Release); + true + } + + pub(crate) fn pop(&self) -> Option { + let tail = self.tail.load(Relaxed); + // Acquire pairs with the producer's release of `head`. + let head = self.head.load(Acquire); + + if head == tail { + return None; + } + + let slot = &self.slots[(tail & RING_MASK) as usize]; + // SAFETY: `head` is past `tail`, so the producer finished this slot and + // will not touch it again until we publish `tail`. + let msg = unsafe { slot.get().read() }; + + // Release pairs with the producer's acquire of `tail`, handing the slot back. + self.tail.store(tail.wrapping_add(1), Release); + Some(msg) + } + + pub(crate) fn dropped(&self) -> u32 { + self.dropped.load(Relaxed) + } +} diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs new file mode 100644 index 0000000000000..308f3b76c01b9 --- /dev/null +++ b/drivers/soc/apple/sbio.rs @@ -0,0 +1,3316 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj +//! Touch ID: the biometric endpoint (SBIO) and its `/dev/sep-bio` interface. +//! The enclave matches; no biometric image ever crosses to userspace. + +#![allow(dead_code)] + +use super::*; +use kernel::prelude::*; +use kernel::soc::apple::mailbox::Message; +use crate::proto::*; +use crate::sks::SKS_AUTH_TOKEN_LEN; + +impl SepData { + fn sbio_expect_ok(&self, op: &crate::sbio::SbioOp) -> Option> { + match self.sbio_call(op) { + SbioOutcome::Ok(payload) => Some(payload), + _ => None, + } + } + + pub(crate) fn sbio_call(&self, op: &crate::sbio::SbioOp) -> SbioOutcome { + let done = match self.sbio_transfer(op) { + Ok(done) => done, + Err(_) => { + return SbioOutcome::Other; + } + }; + + let Some(err) = done.status.answered() else { + return SbioOutcome::Other; + }; + + match err as u16 { + crate::sbio::SBIO_STATUS_OK => SbioOutcome::Ok(done.payload), + crate::sbio::SBIO_STATUS_PREREQUISITE => { + SbioOutcome::PrerequisiteMissing + } + crate::sbio::SBIO_STATUS_16 => { + SbioOutcome::Status16 + } + _ => { + SbioOutcome::Other + } + } + } + + fn sbio_relay(&self, relay: &crate::sbio::SbioRelay<'_>) -> Option> { + match self.sbio_transfer_raw(relay.opcode(), relay.name(), relay.payload()) { + Ok(done) if done.status.is_ok() => Some(done.payload), + Ok(_) => None, + Err(_) => None, + } + } + + fn sync_device_view(&self) -> bool { + let synced = matches!( + self.sbio_call(&crate::sbio::sbio_update_device_list()), + SbioOutcome::Ok(_) + ); + + let policy_ok = match self.sbio_call(&crate::sbio::sbio_match_policy()) { + SbioOutcome::Ok(policy) if policy.len() == crate::sbio::SBIO_MATCH_POLICY_LEN => { + true + } + SbioOutcome::Ok(_) => { + false + } + _ => { + false + } + }; + + synced && policy_ok + } + + fn note_capture_end(&self) { + self.last_capture_end_ns.store(shim::boottime_ns(), Relaxed); + } + + fn settle_before_capture(&self) { + let last = self.last_capture_end_ns.load(Relaxed); + if last == 0 { + return; + } + let elapsed_ms = shim::boottime_ns().saturating_sub(last) / 1_000_000; + if elapsed_ms >= u64::from(MATCH_SETTLE_MS) { + return; + } + let remaining = MATCH_SETTLE_MS - elapsed_ms as u32; + let _ = sensor::idle(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(remaining))); + } + + fn pace_between_captures(&self) { + let _ = sensor::idle(); + + if !self.await_sensor_state(sensor::STATE_IDLE, c"idle, between captures") { + dev_warn!( + self.dev, + "enrol: sensor did not idle within {} ms; continuing the reposition wait anyway (pause is for the person)\n", + ENROL_IDLE_TIMEOUT_MS + ); + } + + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(ENROL_REPOSITION_MS))); + } + + fn stash_enrol_identity(&self, record: &[u8]) { + let mut found: KVec<([u8; bio::UUID_LEN], u32)> = KVec::new(); + crate::sbio::enrol_identity_candidates(record, SBIO_PROBE_USER_ID, |at, uuid| { + let _ = found.push((uuid, at as u32), GFP_KERNEL); + }); + for (_uuid, _at) in found.iter() { + } + *self.enrol_identity_candidates.lock() = found; + } + + fn identity_just_enrolled(&self) -> Option<[u8; bio::UUID_LEN]> { + let candidates = core::mem::take(&mut *self.enrol_identity_candidates.lock()); + let listed = self.enclave_identities_for(SBIO_PROBE_USER_ID)?; + + let mut confirmed: KVec<([u8; bio::UUID_LEN], u32)> = KVec::new(); + for (uuid, at) in candidates.iter() { + if listed.iter().any(|u| u == uuid) && !confirmed.iter().any(|(u, _)| u == uuid) { + if confirmed.push((*uuid, *at), GFP_KERNEL).is_err() { + break; + } + } + } + + match confirmed.len() { + 1 => { + let (uuid, _at) = confirmed[0]; + Some(uuid) + } + 0 => { + None + } + _ => { + None + } + } + } + + fn reconcile_identities(&self) { + let Some(listed) = self.enclave_identities_for(SBIO_PROBE_USER_ID) else { + return; + }; + + if listed.is_empty() { + return; + } + + let dropped = { + let mut index = self.bio_index.lock(); + match index.reconcile_to(&listed) { + Ok(dropped) => dropped, + Err(_) => { + return; + } + } + }; + + let held = { + let index = self.bio_index.lock(); + index.total() + }; + for _uuid in listed.iter() { + } + + if dropped.is_empty() && held == listed.len() { + return; + } + let persisted = { + let index = self.bio_index.lock(); + self.with_store(|store| index.persist(store)) + }; + match persisted { + Some(Ok(())) => {}, + _ => dev_warn!( + self.dev, + "sbio: the reconciled index could not be written; it is correct in memory for this boot and will be rebuilt at the next attach\n" + ), + } + } + + fn enclave_identities_for(&self, user_id: i32) -> Option> { + let op = crate::sbio::sbio_list_identities(); + let SbioOutcome::Ok(reply) = self.sbio_call(&op) else { + return None; + }; + + let records = crate::sbio::IdentityRecords::new(&reply)?; + + let mut out = KVec::new(); + let mut failed = false; + records.each_built_in_for(user_id, |identity| { + if out.push(*identity.uuid(), GFP_KERNEL).is_err() { + failed = true; + } + }); + if failed { + return None; + } + Some(out) + } + + fn finish_enrolment(&self, outcome: core::result::Result<[u8; bio::UUID_LEN], u32>) { + let filed = { + let mut session = self.bio_session.lock(); + let mut index = self.bio_index.lock(); + bio::enrol_finish(&mut session, &mut index, outcome) + }; + if !filed { + return; + } + // lock ordering: store lock outside the session lock (UAF otherwise) + if outcome.is_ok() { + let index = self.bio_index.lock(); + let saved = self.with_store(|store| index.persist(store)); + match saved { + Some(Ok(())) => {}, + Some(Err(e)) => dev_err!( + self.dev, + "enrol: enclave holds a new template but the host index could not be written ({:?})\n", + e + ), + None => {} + } + } + self.bio_wake(); + } + + fn bio_wake(&self) { + if let Some(dev) = self.bio_dev.lock().as_ref() { + dev.wake(); + } + } + + pub(crate) fn attach_sensor(&self) { + if let Err(e) = sensor::register_driver() { + self.sensor_present.store(false, Relaxed); + dev_err!( + self.dev, + "sensor: could not register the SPI driver: {:?}\n", + e + ); + return; + } + + if let Err(e) = dt::enable_spi_sensor(sensor::CONTROLLER_BASE, sensor::CHIP_SELECT) { + self.sensor_present.store(false, Relaxed); + dev_warn!( + self.dev, + "sensor: could not enable the SPI bus at 0x{:x} or create the sensor node ({:?}); no sensor\n", + sensor::CONTROLLER_BASE, + e + ); + return; + } + + let bound = sensor::is_bound(); + self.sensor_present.store(bound, Relaxed); + if bound && sensor::power_line().is_none() { + dev_warn!( + self.dev, + "sensor: no power line ({}); an unpowered sensor answers sixteen zero bytes like a dead bus\n", + sensor::power_source().name() + ); + } + } + + fn open_enrolment_context(&self, user: crate::sbio::UserId) -> bool { + let listed = match self.enclave_identities_for(SBIO_PROBE_USER_ID) { + Some(list) => list.len(), + None => { + return self.enrol_into_existing_context(user); + } + }; + + let Some(proof) = crate::sbio::NoExistingCatacomb::from_zero_identities(listed) else { + return self.enrol_into_existing_context(user); + }; + + self.open_fresh_context(user, &proof) + } + + fn enrol_into_existing_context(&self, user: crate::sbio::UserId) -> bool { + if !self.activate_protected_config(user) { + return false; + } + true + } + + fn open_fresh_context(&self, user: crate::sbio::UserId, proof: &crate::sbio::NoExistingCatacomb) -> bool { + let system = crate::sbio::sbio_select_context(crate::sbio::ContextScope::SYSTEM, proof); + if self.sbio_expect_ok(&system).is_none() { + return false; + } + + for (who, kind, what) in [ + ( + crate::sbio::CatacombUser::MASTER, + PRIVATE_TYPE_CATACOMB_MASTER, + c"master catacomb", + ), + ( + crate::sbio::CatacombUser::OWNER, + PRIVATE_TYPE_CATACOMB_OWNER, + c"owner catacomb", + ), + ] { + if !self.save_catacomb(who, kind, what) { + return false; + } + } + + let per_user = crate::sbio::sbio_select_context(crate::sbio::ContextScope::user(user), proof); + if self.sbio_expect_ok(&per_user).is_none() { + return false; + } + + if !self.activate_protected_config(user) { + return false; + } + + true + } + + fn activate_protected_config(&self, user: crate::sbio::UserId) -> bool { + let op = crate::sbio::sbio_protected_config(user); + let Some(config) = self.sbio_expect_ok(&op) else { + return false; + }; + if config.len() != crate::sbio::SBIO_PROTECTED_CONFIG_LEN { + return false; + } + true + } + + fn begin_enrolment_on_enclave(&self) -> Option> { + if self.enrol_open.load(Relaxed) { + self.enrol_open.store(false, Relaxed); + let _ = self.sbio_call(&crate::sbio::sbio_cancel_operation()); + } + + let user = crate::sbio::UserId::new(SBIO_PROBE_USER_ID)?; + + // 0x03 answers -3 until a user key bag is designated + if !self.keybag_designated.load(Relaxed) { + return None; + } + + if self.enrol_material.lock().is_none() { + return None; + } + + // 0x03 answers 0x1 until an enrolment context exists + if !self.open_enrolment_context(user) { + return None; + } + + let Some(acm_handle) = self.establish_passcode_validated_context(user) else { + dev_err!( + self.dev, + "enrol: could not establish the PasscodeValidated SCRD credential; not falling back to a transient SKS token (a catacomb sealed under one does not survive a reboot)\n" + ); + return None; + }; + let op = crate::sbio::sbio_begin_enrol(user, crate::sbio::BE_AUTH_TYPE_ACM_CONTEXT, &acm_handle); + + match self.sbio_call(&op) { + SbioOutcome::Ok(_payload) => { + self.enrol_open.store(true, Relaxed); + Some(OpenEnrolment { + sep: self, + armed: true, + }) + } + _ => { + None + } + } + } + + fn restore_all_components(&self) -> bool { + // 0x6b is re-read before every component, not once up front. + let user_id = SBIO_PROBE_USER_ID; + let mut any = false; + let mut missing_files = false; + let mut user_outcome = RestoreOutcome::NoStoredFile; + + // 0x8002 (cold transition) is a success only on the owner component. + for (id, kind, what, cold_ok) in [ + ( + crate::sbio::CatacombUser::MASTER.value(), + PRIVATE_TYPE_CATACOMB_MASTER, + c"master catacomb", + false, + ), + ( + crate::sbio::CatacombUser::OWNER.value(), + PRIVATE_TYPE_CATACOMB_OWNER, + c"owner catacomb", + true, + ), + (user_id, PRIVATE_TYPE_CATACOMB_USER, c"user catacomb", false), + ] { + let Some(reply) = self.read_component_states() else { + return false; + }; + let states = crate::sbio::ComponentStates::new(&reply); + let outcome = self.restore_catacomb(states.state_for(id), id, kind, what, cold_ok); + match outcome { + RestoreOutcome::Restored | RestoreOutcome::AlreadyActive => any = true, + RestoreOutcome::NoStoredFile => missing_files = true, + RestoreOutcome::Failed | RestoreOutcome::EmptyTolerated => {} + } + if id == user_id { + user_outcome = outcome; + } + } + + let user_ok = matches!( + user_outcome, + RestoreOutcome::Restored | RestoreOutcome::AlreadyActive + ); + let lockout = match self.restore_lockout() { + RestoreOutcome::Restored | RestoreOutcome::AlreadyActive => true, + RestoreOutcome::EmptyTolerated => true, + RestoreOutcome::NoStoredFile => { + missing_files = true; + false + } + RestoreOutcome::Failed => false, + }; + + if missing_files { + dev_warn!( + self.dev, + "sbio: at least one of four artefacts has no file on disk; if enrolled before the four-artefact save, re-enrol once to write all four\n" + ); + } + + let _ = any; + user_ok && lockout + } + + fn read_component_states(&self) -> Option> { + let op = crate::sbio::sbio_context_state(); + let SbioOutcome::Ok(reply) = self.sbio_call(&op) else { + return None; + }; + + let states = crate::sbio::ComponentStates::new(&reply); + if states.count() == 0 { + return None; + } + Some(reply) + } + + fn restore_catacomb( + &self, + state: Option, + id: i32, + kind: u8, + what: &CStr, + cold_ok: bool, + ) -> RestoreOutcome { + let Some(state) = state else { + return RestoreOutcome::Failed; + }; + match crate::sbio::component_action(state) { + crate::sbio::ComponentAction::AlreadyActive => { + RestoreOutcome::AlreadyActive + } + crate::sbio::ComponentAction::Unsupported(_) => { + RestoreOutcome::Failed + } + crate::sbio::ComponentAction::Load => { + let Some(blob) = self.read_stored(kind, what) else { + return RestoreOutcome::NoStoredFile; + }; + + let at = crate::sbio::SBIO_SAVED_USER_ID_AT; + if blob.len() < at + 4 { + return RestoreOutcome::Failed; + } + let carried = + i32::from_le_bytes([blob[at], blob[at + 1], blob[at + 2], blob[at + 3]]); + if carried != id { + return RestoreOutcome::Failed; + } + + let Some(request) = crate::sbio::SbioLoadCatacomb::new(&blob) else { + return RestoreOutcome::Failed; + }; + + match self.sbio_transfer_raw(request.opcode(), request.name(), request.payload()) { + Ok(done) if done.status.is_ok() => { + self.confirm_active(id, what, LoadAnswer::StatusZero) + } + Ok(done) + if done.status.answered() == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) + && cold_ok => + { + self.confirm_active(id, what, LoadAnswer::ColdTransition) + } + Ok(done) + if done.status.answered() == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) => + { + RestoreOutcome::Failed + } + // 0x101 ALREADY_ACTIVE is a success + Ok(done) + if done.status.answered() + == Some(crate::sbio::SBIO_STATUS_ALREADY_ACTIVE) => + { + self.confirm_active(id, what, LoadAnswer::AlreadyActive) + } + _ => { + RestoreOutcome::Failed + } + } + } + } + } + + fn confirm_active(&self, id: i32, what: &CStr, answer: LoadAnswer) -> RestoreOutcome { + let Some(reply) = self.read_component_states() else { + dev_err!( + self.dev, + "sbio: the {} answered {} and 0x6b could not be re-read; treating unknown as failure\n", + what, + answer.describe() + ); + return RestoreOutcome::Failed; + }; + let states = crate::sbio::ComponentStates::new(&reply); + + match states.state_for(id) { + Some(state) if state & crate::sbio::COMPONENT_STATE_ACTIVE != 0 => { + RestoreOutcome::Restored + } + Some(state) if state & crate::sbio::COMPONENT_STATE_COLD != 0 => { + RestoreOutcome::Failed + } + Some(_) => { + RestoreOutcome::Failed + } + None => { + RestoreOutcome::Failed + } + } + } + + fn restore_lockout(&self) -> RestoreOutcome { + let Some(blob) = self.read_stored(PRIVATE_TYPE_LOCKOUT, c"bio lockout record") else { + return RestoreOutcome::NoStoredFile; + }; + let Some(request) = crate::sbio::SbioLoadLockout::new(&blob) else { + return RestoreOutcome::Failed; + }; + match self.sbio_transfer_raw(request.opcode(), request.name(), request.payload()) { + Ok(done) if done.status.is_ok() => { + RestoreOutcome::Restored + } + Ok(done) + if matches!( + done.status, + transfer::DeviceStatus::Answered(crate::sbio::SBIO_STATUS_COLD_TRANSITION) + ) => + { + RestoreOutcome::EmptyTolerated + } + Ok(_) => { + RestoreOutcome::Failed + } + Err(_) => { + RestoreOutcome::Failed + } + } + } + + fn read_stored(&self, kind: u8, _what: &CStr) -> Option> { + if crate::catacomb::is_kind(kind) { + if let Some(blob) = crate::catacomb::read(kind) { + return Some(blob); + } + } + match self.with_store(|store| store.read(&store::Key::root(kind))) { + Some(Ok(Some(blob))) => Some(blob), + Some(Ok(None)) => None, + Some(Err(_)) => { + None + } + None => None, + } + } + + fn save_all_components(&self, user: crate::sbio::UserId) -> bool { + let mut all = true; + for (who, kind, what) in [ + ( + crate::sbio::CatacombUser::MASTER, + PRIVATE_TYPE_CATACOMB_MASTER, + c"master catacomb (user -1)", + ), + ( + crate::sbio::CatacombUser::OWNER, + PRIVATE_TYPE_CATACOMB_OWNER, + c"owner catacomb (user 501)", + ), + ( + crate::sbio::CatacombUser::enrolling(user), + PRIVATE_TYPE_CATACOMB_USER, + c"user catacomb", + ), + ] { + if !self.save_catacomb(who, kind, what) { + all = false; + } + } + if !self.save_lockout() { + all = false; + } + all + } + + fn save_after_match(&self, user: crate::sbio::UserId) -> bool { + + if !self.save_lockout() { + dev_err!( + self.dev, + "verify: post-match lockout persistence failed; refusing to claim the AP and SEP anti-replay state are synchronized\n" + ); + return false; + } + + if !self.save_catacomb( + crate::sbio::CatacombUser::enrolling(user), + PRIVATE_TYPE_CATACOMB_USER, + c"post-match user catacomb", + ) { + dev_err!( + self.dev, + "verify: post-match user-catacomb persistence failed; the next boot may reject the older AP blob\n" + ); + return false; + } + + if !self.save_catacomb( + crate::sbio::CatacombUser::MASTER, + PRIVATE_TYPE_CATACOMB_MASTER, + c"post-match system (id -1) material snapshot", + ) { + dev_err!( + self.dev, + "verify: post-match system material snapshot failed; next boot may reinstall pre-match material and the user load could cold-transition (0x8002)\n" + ); + return false; + } + + if !self.resnapshot_identity_keybag() { + dev_err!( + self.dev, + "verify: post-match identity-bag snapshot failed; the newly confirmed catacomb has no matching durable bag image\n" + ); + return false; + } + + true + } + + fn save_catacomb(&self, who: crate::sbio::CatacombUser, kind: u8, _what: &CStr) -> bool { + let selector = crate::sbio::SaveSelector::new(who); + + let Some(blob) = self.sbio_expect_ok(&crate::sbio::sbio_save_catacomb(&selector)) else { + return false; + }; + if blob.len() < crate::sbio::SBIO_SAVED_MIN || blob.len() > crate::sbio::SBIO_SAVED_MAX { + return false; + } + + let at = crate::sbio::SBIO_SAVED_USER_ID_AT; + let carried = i32::from_le_bytes([blob[at], blob[at + 1], blob[at + 2], blob[at + 3]]); + if carried != who.value() { + return false; + } + + if crate::catacomb::write(kind, &blob).is_err() { + return false; + } + + if self + .sbio_expect_ok(&crate::sbio::sbio_confirm_save(&selector)) + .is_none() + { + return false; + } + true + } + + fn save_lockout(&self) -> bool { + let Some(blob) = self.sbio_expect_ok(&crate::sbio::sbio_save_lockout()) else { + return false; + }; + if blob.is_empty() { + return false; + } + match self.with_store(|store| store.write(&store::Key::root(PRIVATE_TYPE_LOCKOUT), &blob)) { + Some(Ok(())) => { + true + } + Some(Err(_)) => { + false + } + None => false, + } + } + + fn enable_sbio(&self) -> Result<()> { + if self.sbio_ready.load(Relaxed) { + return Ok(()); + } + + self.register_ool(&self.ool_sbio)?; + + self.sbio_ready.store(true, Relaxed); + Ok(()) + } + + fn attach_bringup(&self) { + if !self.sensor_present.load(Relaxed) { + dev_warn!( + self.dev, + "sbio: no sensor bound at attach; catacomb restore ran but capture is unavailable\n" + ); + return; + } + let Some(patch) = self.wake_sensor() else { + return; + }; + if !self.complete_bringup(patch) { + dev_warn!( + self.dev, + "sbio: attach-time sensor bring-up did not complete; matching remains unavailable until a later bring-up succeeds.\n" + ); + } + } + + // ordering: the catacomb restore must complete before the sensor is woken + pub(crate) fn run_bringup(&self) { + if self.bringup_started.xchg(true, Relaxed) { + return; + } + if let Err(e) = self.enable_sbio() { + dev_err!(self.dev, "bringup: could not enable the biometric transport ({:?}); Touch ID is unavailable this boot\n", e); + return; + } + if let Err(e) = self.enable_sks() { + dev_warn!(self.dev, "bringup: could not enable the key store ({:?}); keybag and ref-key operations are unavailable\n", e); + } + if let Ok(keybag::State::Present(stored)) = keybag::read(keybag::Slot::Identity) { + if let Some((handle, uuid)) = self.sks_recover(&stored) { + self.sks_designate_user_keybag(handle, stored.secret()); + self.sks_machine_refkey(handle, stored.secret()); + let prepared = self.cold_match_continue(handle, uuid); + self.ensure_restored_after(prepared); + } + } + self.attach_bringup(); + } + + pub(crate) fn run_verify(&self) { + if !self.templates_restored.load(Relaxed) { + dev_err!( + self.dev, + "verify: refusing before touching the sensor; cold-match prep or restore did not complete, enclave holds no template (every match refused 0x1). A restore failure, not a non-matching finger\n" + ); + self.finish_verify( + bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE), + [0u8; bio::TOKEN_LEN], + ); + return; + } + + let Some(token_bytes) = self.mint_token_bytes() else { + self.finish_verify( + bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE), + [0u8; bio::TOKEN_LEN], + ); + return; + }; + + self.settle_before_capture(); + + let Some(patch) = self.wake_sensor() else { + self.finish_verify(bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), token_bytes); + let _ = sensor::idle(); + return; + }; + + // enclave refuses a capture from an uncalibrated sensor (0x65 answers 1) + let already_calibrated = self.sensor_calibrated.load(Relaxed); + self.sensor_calibrated.store(true, Relaxed); + let patch = if already_calibrated { + patch + } else { + if !self.calibrate_sensor() { + self.finish_verify(bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), token_bytes); + let _ = sensor::idle(); + return; + } + let Some(patch) = self.wake_sensor() else { + self.finish_verify(bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), token_bytes); + let _ = sensor::idle(); + return; + }; + patch + }; + if !self.complete_bringup(patch) { + self.finish_verify(bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), token_bytes); + let _ = sensor::idle(); + return; + } + + // The match uses the plain 0x23->0x06->0x65 pipeline, with no SCRD step. + + let outcome = self.verify_one_image(); + let _ = sensor::idle(); + sensor::power(false); + self.note_capture_end(); + + let definitive = matches!( + &outcome, + bio::VerifyOutcome::Matched(_) | bio::VerifyOutcome::NoMatch + ); + if definitive { + let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + self.finish_verify( + bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE), + token_bytes, + ); + return; + }; + if !self.save_after_match(user) { + self.finish_verify( + bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE), + token_bytes, + ); + return; + } + } + self.finish_verify(outcome, token_bytes); + } + + fn verify_one_image(&self) -> bio::VerifyOutcome { + let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + + if sensor::start_capture().is_err() { + return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR); + } + let advertised = match self.await_capture() { + CaptureWait::Ready(n) => n, + CaptureWait::Timeout => { + return bio::VerifyOutcome::Failed(ENROL_STATUS_TIMEOUT); + } + CaptureWait::Fault(_state) => { + return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR); + } + CaptureWait::Abandon => return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), + }; + + let capture = match sensor::read_capture(advertised) { + Ok(capture) => capture, + Err(sensor::CaptureError::Checksum { .. }) | Err(sensor::CaptureError::Length(_)) => { + return bio::VerifyOutcome::Failed(ENROL_STATUS_RETRY); + } + Err(sensor::CaptureError::Bus(_)) => { + return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR); + } + Err(sensor::CaptureError::NoMemory) => { + return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR) + } + }; + + if self + .sbio_expect_ok(&crate::sbio::sbio_prepare_image_processing()) + .is_none() + { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + let _image = ImageContext { sep: self }; + + // MATCHING: byte 0x12 set, 0x13 clear; last-image flag at byte 9 (enrol byte 8) + let init = crate::sbio::sbio_image_processing_init( + crate::sbio::ImagePurpose::Matching, + true, + true, + 0, + user, + crate::shim::monotonic_ns(), + ); + if self.sbio_expect_ok(&init).is_none() { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + + let relay = crate::sbio::SbioRelay::new(&capture); + if self.sbio_relay(&relay).is_none() { + dev_err!( + self.dev, + "verify: enclave refused the capture (status above); no comparison performed, not a non-match. Reported as a failure so userspace does not tell the user their finger was rejected\n" + ); + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + drop(capture); + + let Some(assessment) = self.sbio_expect_ok(&crate::sbio::sbio_image_assessment()) else { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + if assessment.len() <= crate::sbio::ASSESS_USABLE_MATCH { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + + if assessment[crate::sbio::ASSESS_USABLE_MATCH] == 0 { + return bio::VerifyOutcome::Failed(ENROL_STATUS_RETRY); + } + + let Some(result) = self.sbio_expect_ok(&crate::sbio::sbio_match_result()) else { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + let Some(parsed) = crate::sbio::MatchResult::parse(&result) else { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + + if !parsed.matches(user) { + return bio::VerifyOutcome::NoMatch; + } + + // match result: user id groups templates, UUID at offset 0x04 identifies one + let identity = *parsed.identity_uuid(); + let known = self.bio_index.lock().contains_uuid(&identity); + + if !known { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + + bio::VerifyOutcome::Matched(bio::MatchEvidence::from_enclave_reply(identity)) + } + + pub(crate) fn run_enrolment(&self) { + let Some(patch) = self.wake_sensor() else { + self.finish_enrolment(Err(ENROL_STATUS_SENSOR)); + let _ = sensor::idle(); + return; + }; + + let already_calibrated = self.sensor_calibrated.load(Relaxed); + self.sensor_calibrated.store(true, Relaxed); + let patch = if already_calibrated { + patch + } else { + if !self.calibrate_sensor() { + self.finish_enrolment(Err(ENROL_STATUS_SENSOR)); + let _ = sensor::idle(); + return; + } + let Some(patch) = self.wake_sensor() else { + self.finish_enrolment(Err(ENROL_STATUS_SENSOR)); + let _ = sensor::idle(); + return; + }; + patch + }; + + if !self.complete_bringup(patch) { + self.finish_enrolment(Err(ENROL_STATUS_SENSOR)); + let _ = sensor::idle(); + return; + } + + let Some(open) = self.begin_enrolment_on_enclave() else { + self.finish_enrolment(Err(ENROL_STATUS_ENCLAVE)); + let _ = sensor::idle(); + return; + }; + + let mut counter: u32 = 0; + let mut enrolment_completed = false; + let mut identity: Option<[u8; bio::UUID_LEN]> = None; + + let outcome = loop { + if !bio::enrol_is_live(&self.bio_session.lock()) { + break None; + } + if counter >= ENROL_MAX_CAPTURES { + break Some(Err(ENROL_STATUS_TOO_MANY)); + } + + match self.enrol_one_image(counter) { + ImageOutcome::Progress { + stage, + percent, + complete, + has_template, + } => { + counter = counter.saturating_add(1); + let woke = bio::enrol_advance( + &mut self.bio_session.lock(), + stage, + percent, + bio::Guidance::LiftAndMove, + ); + if woke { + self.bio_wake(); + } + // completion is the flag at offset 0xbfe, not a derived stage count + if complete { + if !has_template { + break Some(Err(ENROL_STATUS_ENCLAVE)); + } + enrolment_completed = true; + break Some(Ok(())); + } + } + ImageOutcome::Retry => { + counter = counter.saturating_add(1); + } + ImageOutcome::NoFinger => { + break Some(Err(ENROL_STATUS_TIMEOUT)); + } + ImageOutcome::Failed(status) => break Some(Err(status)), + } + + self.pace_between_captures(); + }; + + if enrolment_completed { + open.completed(); + + if let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) { + if !self.save_all_components(user) { + dev_warn!( + self.dev, + "enrol: enrolment succeeded but at least one of four artefacts was not persisted; works until the next reboot (see which component above)\n" + ); + } else { + self.templates_restored.store(true, Relaxed); + if !self.resnapshot_identity_keybag() { + dev_warn!( + self.dev, + "enrol: catacombs saved but post-commit identity-bag snapshot was not; template works this boot but is not reboot-persistent\n" + ); + } + } + } + + identity = self.identity_just_enrolled(); + + self.reconcile_identities(); + } + + let _ = sensor::idle(); + sensor::power(false); + self.note_capture_end(); + + if let Some(result) = outcome { + let filed = match (result, identity) { + (Ok(()), Some(uuid)) => Ok(uuid), + (Ok(()), None) => Err(ENROL_STATUS_UNFILED), + (Err(status), _) => Err(status), + }; + self.finish_enrolment(filed); + } + } + + fn register_sensor(&self, id: &sensor::Identifier) -> bool { + let stage = self.bringup.load(Relaxed); + + if stage >= BRINGUP_ESTABLISHED { + if self.sbio_expect_ok(&crate::sbio::sbio_clear_state()).is_none() { + return false; + } + } + + if stage == BRINGUP_FRESH { + let op = crate::sbio::sbio_register_sensor(id); + match self.sbio_call(&op) { + SbioOutcome::Ok(_payload) => { + self.bringup.store(BRINGUP_IDENTIFIED, Relaxed); + } + _ => { + return false; + } + } + } + + let serial = match sensor::read_sensor_serial() { + Ok(serial) => serial, + Err(_) => { + return false; + } + }; + + let op = crate::sbio::sbio_register_sensor_serial(&serial); + match self.sbio_call(&op) { + SbioOutcome::Ok(_payload) => { + self.bringup.store(BRINGUP_ESTABLISHED, Relaxed); + true + } + _ => { + false + } + } + } + + fn wake_sensor(&self) -> Option { + let _source = sensor::power_source(); + let cycled = sensor::power_cycle(); + if !sensor::cs_timing().is_hardware() { + dev_warn!( + self.dev, + "sensor: chip-select timing not reaching hardware; software emulation cannot time this sensor, silent status expected (mode 2 is the other half)\n" + ); + } + + let mut answered_without_identifier = false; + + for delay in sensor::POWER_ON_READ_DELAYS_MS { + if delay > 0 { + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(delay))); + } + let st = match sensor::status() { + Ok(st) => st, + Err(_) => { + return None; + } + }; + if st.is_silent() { + continue; + } + + if let sensor::Offset12::Identifier(id) = st.offset12() { + if id != 0 && id != sensor::EXPECTED_IDENTIFIER { + return None; + } + } + + let Some(id) = st.identifier_to_register() else { + answered_without_identifier = true; + continue; + }; + if !self.register_sensor(&id) { + return None; + } + + // The patch decision is made on status[8], not on state 9. + if st.patch_ack() == sensor::PATCH_ACCEPTED { + return Some(PatchLoaded(())); + } + return self.apply_sensor_patch(st.patch_ack()); + } + + dev_err!( + self.dev, + "sensor gated: all {} wake reads returned sixteen zero bytes. Power {}. CS timing: {}. Mode 2 (CPOL=1 CPHA=0)\n", + sensor::POWER_ON_READ_DELAYS_MS.len(), + if cycled { c"was cycled" } else { c"was NOT cycled" }, + sensor::cs_timing().name() + ); + if answered_without_identifier { + return None; + } + None + } + + fn calibrate_sensor(&self) -> bool { + let blob = match kernel::firmware::Firmware::request(CALIBRATION_FIRMWARE, &self.dev) { + Ok(fw) => CalibrationBlob::new(fw), + Err(e) => { + dev_err!( + self.dev, + "sensor: calibration blob {} could not be loaded ({:?}); per-device factory data, cannot be synthesised, stopping\n", + CALIBRATION_FIRMWARE, + e + ); + return false; + } + }; + + let Some(challenge) = self.sbio_expect_ok(&crate::sbio::sbio_module_challenge()) else { + return false; + }; + if challenge.len() != crate::sbio::SBIO_MODULE_CHALLENGE_LEN { + return false; + } + + let challenge = sensor::Params::new(challenge); + if sensor::send_module_challenge(&challenge).is_err() { + return false; + } + + let reply = match sensor::read_module_reply() { + Ok(reply) => reply, + Err(_) => { + return false; + } + }; + + // commit the sensor's reply, not the challenge; both are 64 bytes at count 0x4b + let Some(commit) = crate::sbio::sbio_module_commit(&reply) else { + return false; + }; + let Some(_serial) = self.sbio_expect_ok(&commit) else { + return false; + }; + + let request = match crate::sbio::SbioCalibration::new(&blob) { + Ok(request) => request, + Err(_) => { + return false; + } + }; + if self + .sbio_transfer_raw(request.opcode(), request.name(), request.payload()) + .ok() + .filter(|done| done.status.is_ok()) + .is_none() + { + dev_err!( + self.dev, + "sensor: 0x5b LOAD_CALIBRATION failed. Without it every capture is refused with status 1.\n" + ); + return false; + } + + let _ = self.sbio_call(&crate::sbio::sbio_diagnostics()); + + true + } + + fn complete_bringup(&self, patch: PatchLoaded) -> bool { + + let Some(params) = self.apply_sensor_parameters() else { + return false; + }; + + let op = crate::sbio::sbio_complete_init(patch, params); + if self.sbio_expect_ok(&op).is_none() { + dev_err!(self.dev, "sensor: 0x01 COMPLETE_INIT failed — status above\n"); + return false; + } + + let listed = self.sync_device_view(); + + if listed && !self.device_view_synced.xchg(true, Relaxed) { + let count = self.log_identity_count(c"after device-view synchronisation"); + match count { + Some(0) => { + self.templates_restored.store(false, Relaxed); + } + Some(n) if self.templates_restored.load(Relaxed) && self.prove_restore() => { + dev_warn!( + self.dev, + "Touch ID: restored {} enrolled identity/identities; enrolment survived reboot\n", + n + ); + } + Some(_) => { + self.templates_restored.store(false, Relaxed); + } + None => { + self.templates_restored.store(false, Relaxed); + } + } + if self.templates_restored.load(Relaxed) { + self.reconcile_identities(); + } + } + + true + } + + fn apply_sensor_parameters(&self) -> Option { + // 0x5d and 0x5c: an empty blob is a failure. + self.relay_parameters( + &crate::sbio::sbio_coverage_params(), + &sensor::Geometry::COVERAGE, + false, + )?; + self.relay_parameters( + &crate::sbio::sbio_operation_params(), + &sensor::Geometry::OPERATION, + false, + )?; + // 0x6a: an empty blob is success, and relaying nothing is correct. + self.relay_parameters( + &crate::sbio::sbio_transparent_channel(), + &sensor::Geometry::TRANSPARENT, + true, + )?; + + Some(ParametersApplied(())) + } + + fn relay_parameters( + &self, + op: &crate::sbio::SbioOp, + geom: &sensor::Geometry, + empty_is_success: bool, + ) -> Option { + let blob = match self.sbio_call(op) { + SbioOutcome::Ok(b) => sensor::Params::new(b), + _ => { + return None; + } + }; + + if blob.is_empty() { + if empty_is_success { + return Some(0); + } + return None; + } + + let relayed = blob.len(); + match sensor::send_encrypted_parameters(&blob, geom) { + Ok(()) => { + Some(relayed) + } + Err(sensor::ParamsError::Empty) => { + None + } + Err(sensor::ParamsError::TooLong(n, capacity)) => { + dev_err!( + self.dev, + "sensor: {}'s blob is {} bytes and this frame holds {} (declared 0x{:x} less the 9 bytes of header and CRC)\n", + op.name(), + n, + capacity, + geom.declared() + ); + None + } + Err(sensor::ParamsError::Transfer(_e)) => { + None + } + } + } + + fn establish_session(&self) -> bool { + + let share = match self.sbio_call(&crate::sbio::sbio_request_session_share()) { + SbioOutcome::Ok(sh) => sh, + SbioOutcome::Status16 => { + return false; + } + SbioOutcome::PrerequisiteMissing => { + return false; + } + SbioOutcome::Other => { + return false; + } + }; + + // 0x30 reserved for this reply; length is a floor, take the first 40 + if share.len() < sensor::SESSION_SHARE_LEN { + return false; + } + let mut out = [0u8; sensor::SESSION_SHARE_LEN]; + out.copy_from_slice(&share[..sensor::SESSION_SHARE_LEN]); + + // relay it: class 0x72, no CRC, no padding + if sensor::send_session_share(&out).is_err() { + return false; + } + + let reply = match sensor::read_session_reply() { + Ok(r) => r, + Err(_) => { + return false; + } + }; + + match self.sbio_transfer(&crate::sbio::sbio_commit_session_share(&reply)) { + Ok(done) if done.status.is_ok() => { + true + } + Ok(_) => { + false + } + Err(_) => { + false + } + } + } + + fn init_sequence_counter(&self) -> bool { + + let challenge = match self.sbio_call(&crate::sbio::sbio_request_challenge()) { + SbioOutcome::Ok(c) => c, + SbioOutcome::PrerequisiteMissing => { + if !self.establish_session() { + return false; + } + match self.sbio_call(&crate::sbio::sbio_request_challenge()) { + SbioOutcome::Ok(c) => c, + _ => { + return false; + } + } + } + SbioOutcome::Status16 => { + return false; + } + SbioOutcome::Other => { + return false; + } + }; + if challenge.len() < sensor::CHALLENGE_LEN { + return false; + } + let mut out = [0u8; sensor::CHALLENGE_LEN]; + out.copy_from_slice(&challenge[..sensor::CHALLENGE_LEN]); + + if sensor::send_challenge(&out).is_err() { + return false; + } + + let reply = match sensor::read_challenge_reply() { + Ok(r) => r, + Err(_) => { + return false; + } + }; + + match self.sbio_transfer(&crate::sbio::sbio_commit_challenge(&reply)) { + Ok(done) if done.status.is_ok() => { + true + } + Ok(_) => { + false + } + Err(_) => { + false + } + } + } + + fn apply_sensor_patch(&self, _ack_before: u8) -> Option { + let blob = match self.sbio_call(&crate::sbio::sbio_fetch_patch()) { + SbioOutcome::Ok(b) => b, + SbioOutcome::PrerequisiteMissing => { + if !self.init_sequence_counter() { + return None; + } + match self.sbio_call(&crate::sbio::sbio_fetch_patch()) { + SbioOutcome::Ok(b) => b, + _ => { + return None; + } + } + } + SbioOutcome::Status16 => { + return None; + } + SbioOutcome::Other => { + return None; + } + }; + if blob.is_empty() { + return None; + } + + // enable command is mandatory; skipping it leaves the sensor in state 9 + if sensor::setup_patch_enable().is_err() { + return None; + } + + if !self.await_sensor_state(sensor::STATE_IDLE, c"idle, before sending the patch") { + return None; + } + + if sensor::send_patch(&blob).is_err() { + return None; + } + + // acceptance is at status[8], not at the state + for _attempt in 0..PATCH_POLL_ATTEMPTS { + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(PATCH_POLL_MS))); + let st = match sensor::status() { + Ok(st) => st, + Err(_) => { + return None; + } + }; + if st.patch_ack() == sensor::PATCH_ACCEPTED { + + if let Ok(after) = sensor::status() { + if after.state == sensor::STATE_NEEDS_PATCH { + return None; + } + } + return Some(PatchLoaded(())); + } + } + + let _ = sensor::status(); + None + } + + fn await_sensor_state(&self, want: u8, _why: &CStr) -> bool { + for _attempt in 0..PATCH_POLL_ATTEMPTS { + match sensor::status() { + Ok(st) if st.state == want => { + return true; + } + Ok(_) => {} + Err(_) => { + return false; + } + } + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(PATCH_POLL_MS))); + } + false + } + + fn enrol_one_image(&self, counter: u32) -> ImageOutcome { + if sensor::start_capture().is_err() { + return ImageOutcome::Failed(ENROL_STATUS_SENSOR); + } + + let available = match self.await_capture() { + CaptureWait::Ready(n) => n, + CaptureWait::Timeout => return ImageOutcome::NoFinger, + CaptureWait::Fault(_) => { + return ImageOutcome::Failed(ENROL_STATUS_SENSOR); + } + CaptureWait::Abandon => return ImageOutcome::Failed(ENROL_STATUS_SENSOR), + }; + + let capture = match sensor::read_capture(available) { + Ok(c) => c, + Err(sensor::CaptureError::Checksum { advertised: _advertised, computed: _computed }) => { + return ImageOutcome::Retry; + } + Err(sensor::CaptureError::Length(_n)) => { + return ImageOutcome::Retry; + } + Err(sensor::CaptureError::Bus(_e)) => { + return ImageOutcome::Failed(ENROL_STATUS_SENSOR); + } + Err(sensor::CaptureError::NoMemory) => { + return ImageOutcome::Failed(ENROL_STATUS_SENSOR) + } + }; + + let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + if self + .sbio_expect_ok(&crate::sbio::sbio_prepare_image_processing()) + .is_none() + { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + let _image = ImageContext { sep: self }; + + let init = crate::sbio::sbio_image_processing_init( + crate::sbio::ImagePurpose::Enrolment, + counter == 0, + false, + counter, + user, + crate::shim::monotonic_ns(), + ); + if self.sbio_expect_ok(&init).is_none() { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + + let relay = crate::sbio::SbioRelay::new(&capture); + if self.sbio_relay(&relay).is_none() { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + drop(capture); + + let Some(assessment) = self.sbio_expect_ok(&crate::sbio::sbio_image_assessment()) else { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + if assessment.len() < crate::sbio::ASSESS_MIN_LEN { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + let usable = assessment[crate::sbio::ASSESS_USABLE_ENROL] != 0; + if !usable { + return ImageOutcome::Retry; + } + + let Some(result) = self.sbio_expect_ok(&crate::sbio::sbio_enrolment_result()) else { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + let Some(parsed) = crate::sbio::EnrolmentResult::parse(&result) else { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + + if parsed.complete() { + self.stash_enrol_identity(&result); + } + + let percent = parsed.progress_percent(); + + ImageOutcome::Progress { + stage: (percent * bio::ENROL_STAGES) + .div_ceil(100) + .min(bio::ENROL_STAGES), + percent, + complete: parsed.complete(), + has_template: parsed.has_template(), + } + } + + fn await_capture(&self) -> CaptureWait { + let mut previous: Option<[u8; sensor::STATUS_LEN]> = None; + let mut armed_reported = false; + + for attempt in 0..ENROL_POLL_ATTEMPTS { + if !bio::capture_is_live(&self.bio_session.lock()) { + return CaptureWait::Abandon; + } + + let st = match sensor::status() { + Ok(st) => st, + Err(_) => { + return CaptureWait::Abandon; + } + }; + + if attempt == 0 || previous != Some(st.raw) { + previous = Some(st.raw); + } + + if st.state == sensor::STATE_NEEDS_PATCH { + return CaptureWait::Fault(st.state); + } + + let guide = match st.state { + sensor::STATE_ARMED => Some(bio::Guidance::Place), + sensor::STATE_READING => Some(bio::Guidance::HoldStill), + _ => None, + }; + if let Some(guide) = guide { + if bio::enrol_guide(&mut self.bio_session.lock(), guide) { + self.bio_wake(); + } + } + + if st.state == sensor::STATE_ARMED && !armed_reported { + armed_reported = true; + } + + if let sensor::Offset12::Available(count) = st.offset12() { + if count == 0 { + return CaptureWait::Timeout; + } + return CaptureWait::Ready(count); + } + + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(ENROL_POLL_MS))); + } + let _ = sensor::status(); + CaptureWait::Timeout + } + + pub(crate) fn on_sbio(&self, msg: Message) { + let f = proto::decode(&msg); + let marker = f.tag; + + if !self.sbio_ready.load(Relaxed) { + return; + } + + // notification and the enclave's DMA are not ordered; buffer may still be poison + if !self.ool_await_written(&self.ool_sbio, 0, transfer::HEADER_LEN) { + self.sbio_fail_unwritten(); + return; + } + + let header = match self.ool_read(&self.ool_sbio, 0, transfer::HEADER_LEN) { + Ok(h) => h, + Err(_) => { + self.sbio_fail(); + return; + } + }; + let packet = match transfer::Packet::decode(&header) { + Ok(p) => p, + Err(_) => { + self.sbio_fail(); + return; + } + }; + + let _repeated_header = { + let mut bytes = [0u8; transfer::HEADER_LEN]; + bytes.copy_from_slice(&header[..transfer::HEADER_LEN]); + let mut last = self.sbio_last_header.lock(); + let same = last.as_ref().is_some_and(|prev| *prev == bytes); + *last = Some(bytes); + same + }; + + let chunk = packet.chunk as usize; + let payload = if chunk == 0 { + KVec::new() + } else if !self.ool_await_written(&self.ool_sbio, transfer::HEADER_LEN, chunk) { + self.sbio_fail_unwritten(); + return; + } else { + match self.ool_read(&self.ool_sbio, transfer::HEADER_LEN, chunk) { + Ok(p) => p, + Err(_) => { + self.sbio_fail(); + return; + } + } + }; + + let progress = self.sbio_rx.lock().on_chunk(marker, &packet, &payload); + + match progress { + transfer::Progress::NeedMore(cont) => { + self.sbio_continue(&cont); + } + transfer::Progress::Complete => { + // 0xFE requests the peer's next packet and acks the final one; nothing to send + self.sbio_wq.notify_all(); + } + transfer::Progress::Ignored(_why) => {}, + transfer::Progress::Grant => { + self.sbio_wq.notify_all(); + } + transfer::Progress::Notification { tag: _tag, opcode: _opcode } => { + let _ = self.sbio_rx.lock().awaiting(); + } + transfer::Progress::Failed(_why) => { + self.sbio_wq.notify_all(); + } + } + } + + fn sbio_continue(&self, cont: &transfer::Continuation) { + let mut header = [0u8; transfer::HEADER_LEN]; + if cont.packet().encode(&mut header).is_err() { + self.sbio_fail(); + return; + } + if self.ool_write(&self.ool_sbio, 0, &header).is_err() { + self.sbio_fail(); + return; + } + if self.send(crate::sbio::encode_sbio_continue(cont)).is_err() { + self.sbio_fail(); + } + } + + fn sbio_fail(&self) { + self.sbio_rx.lock().abort(); + self.sbio_wq.notify_all(); + } + + fn sbio_fail_unwritten(&self) { + self.sbio_rx + .lock() + .abort_with(transfer::DeviceStatus::BufferNeverWritten); + self.sbio_wq.notify_all(); + } + + fn sbio_transfer(&self, op: &crate::sbio::SbioOp) -> Result { + self.sbio_transfer_raw(op.opcode(), op.name(), op.payload()) + } + + fn sbio_transfer_raw( + &self, + opcode: u16, + name: &'static CStr, + payload: &[u8], + ) -> Result { + if !self.sbio_ready.load(Relaxed) { + return Err(ENODEV); + } + + self.sbio_rx.lock().begin(u32::from(opcode))?; + + if let Err(e) = self.sbio_start(opcode, name, payload) { + let mut guard = self.sbio_rx.lock(); + guard.abort(); + let _ = guard.take_done(); + return Err(e); + } + + let mut remaining = time::msecs_to_jiffies(SBIO_TIMEOUT_MS); + let mut guard = self.sbio_rx.lock(); + loop { + if let Some(done) = guard.take_done() { + let _ = done.opcode; + return Ok(done); + } + if remaining == 0 { + guard.abort(); + let _ = guard.take_done(); + drop(guard); + return Err(ETIMEDOUT); + } + match self + .sbio_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + } + } + + fn sbio_start(&self, opcode: u16, name: &'static CStr, payload: &[u8]) -> Result<()> { + if crate::sbio::encode_sbio_raw(opcode, 0).is_none() { + return Err(EPERM); + } + + let total = payload.len(); + if total > transfer::MAX_TRANSACTION as usize { + return Err(EINVAL); + } + + let capacity = OOL_SIZE_SBIO - transfer::HEADER_LEN; + + self.sbio_rx.lock().begin_send(u32::from(opcode)); + let result = self.sbio_send_chunks(opcode, name, payload, capacity); + self.sbio_rx.lock().end_send(); + result + } + + fn sbio_send_chunks( + &self, + opcode: u16, + name: &'static CStr, + payload: &[u8], + capacity: usize, + ) -> Result<()> { + let total = payload.len(); + let mut offset = 0usize; + let mut first = true; + let mut seq: u16 = 0; + let mut request = KVec::new(); + + loop { + let chunk = core::cmp::min(total - offset, capacity); + + let packet = transfer::Packet { + version: 1, + total: total as u32, + offset: offset as u32, + flags: 0, + err: 0, + opcode: u32::from(opcode), + chunk: chunk as u32, + }; + + request.clear(); + request.resize(transfer::HEADER_LEN + chunk, 0u8, GFP_KERNEL)?; + packet.encode(&mut request)?; + request[transfer::HEADER_LEN..].copy_from_slice(&payload[offset..offset + chunk]); + + self.ool_write(&self.ool_sbio, 0, &request)?; + + let msg = if first { + crate::sbio::encode_sbio_raw(opcode, seq) + } else { + crate::sbio::encode_sbio_next(opcode, seq) + } + .ok_or(EPERM)?; + self.send(msg)?; + + offset += chunk; + first = false; + seq = seq.wrapping_add(1); + if offset >= total { + return Ok(()); + } + + self.sbio_await_grant(name, offset, total)?; + } + } + + fn sbio_await_grant(&self, _name: &'static CStr, _sent: usize, _total: usize) -> Result<()> { + let mut remaining = time::msecs_to_jiffies(SBIO_TIMEOUT_MS); + let mut guard = self.sbio_rx.lock(); + loop { + if guard.take_grant() { + return Ok(()); + } + if remaining == 0 { + drop(guard); + return Err(ETIMEDOUT); + } + match self + .sbio_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + } + } + + pub(crate) fn register_bio(&self) -> Result<()> { + let mut guard = self.bio_dev.lock(); + if guard.is_some() { + return Ok(()); + } + + let ctx = core::ptr::from_ref(self).cast_mut().cast::(); + + // SAFETY: `ctx` is this `SepData`, kept alive by the `Arc` in the + // driver's private data. `remove()` drops the registration before that + // `Arc` can go, and the shim's unregister clears its stored context + // under the same lock every callback takes, so once it returns, no + // callback can observe this pointer again. + let dev = unsafe { + shim::BioChardev::register( + bio::DEVICE_NAME, + bio::DEVICE_MODE, + ctx, + bio_open_trampoline, + bio_release_trampoline, + bio_ioctl_trampoline, + bio_ready_trampoline, + ) + }?; + + *guard = Some(dev); + Ok(()) + } + + pub(crate) fn bio_open(&self) -> Result<()> { + bio::open(&mut self.bio_session.lock()) + } + + pub(crate) fn bio_release(&self) { + bio::release(&mut self.bio_session.lock()); + if let Some(dev) = self.bio_dev.lock().as_ref() { + dev.wake(); + } + } + + pub(crate) fn queue_enrolment(this: Arc) { + if workqueue::system() + .enqueue::, ENROL_WORK_ID>(this.clone()) + .is_err() + { + this.finish_enrolment(Err(ENROL_STATUS_SENSOR)); + } + } + + pub(crate) fn queue_verify(this: Arc) { + if workqueue::system() + .enqueue::, VERIFY_WORK_ID>(this.clone()) + .is_err() + { + this.finish_verify( + bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), + [0u8; bio::TOKEN_LEN], + ); + } + } + + fn finish_verify(&self, outcome: bio::VerifyOutcome, token_bytes: [u8; bio::TOKEN_LEN]) { + let filed = bio::verify_finish(&mut self.bio_session.lock(), outcome, token_bytes); + if filed { + self.bio_wake(); + } + } + + fn mint_token_bytes(&self) -> Option<[u8; bio::TOKEN_LEN]> { + let mut bytes = [0u8; bio::TOKEN_LEN]; + for chunk in bytes.chunks_mut(4) { + match self.get_entropy_word() { + Ok(word) => chunk.copy_from_slice(&word.to_le_bytes()[..chunk.len()]), + Err(_) => { + return None; + } + } + } + Some(bytes) + } + + pub(crate) fn bio_ioctl(&self, cmd: u32, arg: usize) -> Result { + if cmd == bio::IOC_ATTEST { + return self.bio_attest(arg); + } + let handled = { + let mut session = self.bio_session.lock(); + let index = self.bio_index.lock(); + let mut ctx = bio::Context { + session: &mut session, + index: &index, + sensor_present: self.sensor_present.load(Relaxed), + }; + bio::ioctl(&mut ctx, cmd, arg)? + }; + + // 0x57 sent with the session lock released (it waits on a mailbox reply) + if let Some(identity) = handled.delete_identity { + self.delete_identity(&identity)?; + } + + let mut delete_all_err: Option = None; + for identity in handled.delete_identities.iter() { + if let Err(e) = self.delete_identity(identity) { + delete_all_err = delete_all_err.or(Some(e)); + } + } + if let Some(e) = delete_all_err { + return Err(e); + } + + if handled.wake { + self.bio_wake(); + } + Ok(handled) + } + + fn bio_attest(&self, arg: usize) -> Result { + let user = kernel::uaccess::UserPtr::from_addr(arg); + let req: bio::Attest = kernel::uaccess::UserSlice::new(user, core::mem::size_of::()) + .reader() + .read()?; + let (sig, pubk) = self.refkey_attest_sign(&req.challenge)?; + if sig.len() > bio::ATTEST_SIG_MAX || pubk.len() != bio::ATTEST_PUB_LEN { + return Err(EIO); + } + let mut out = bio::Attest { + sig_len: sig.len() as u32, + challenge: req.challenge, + public: [0u8; bio::ATTEST_PUB_LEN], + signature: [0u8; bio::ATTEST_SIG_MAX], + reserved: [0u8; 3], + }; + out.public.copy_from_slice(&pubk); + out.signature[..sig.len()].copy_from_slice(&sig); + kernel::uaccess::UserSlice::new(user, core::mem::size_of::()) + .writer() + .write(&out)?; + Ok(bio::Handled { + ret: 0, + wake: false, + start_enrol: false, + start_verify: false, + delete_identity: None, + delete_identities: KVec::new(), + }) + } + + fn ensure_restored_after(&self, _prepared: bool) { + let restored = self.restore_all_components(); + self.templates_restored.store(restored, Relaxed); + if !restored { + dev_err!( + self.dev, + "matching unavailable this boot: restore did not complete, enclave holds no template (every match refused 0x1). Not the sensor or the finger; on-disk enrolments intact\n" + ); + return; + } + + if self.keybag_designated.load(Relaxed) { + if let Some(user) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) { + let special = user.special_handle(); + if let Ok(keybag::State::Present(stored)) = keybag::read(keybag::Slot::Identity) { + let _ = self.sks_step(crate::sks::SKS_LOCK_STATE_NAME, |healthy| { + self.sks_req_unlock_special(special, stored.secret(), healthy) + }); + } + } + } + + // enclave's match arm asserts a non-empty ACM context (<= 32 bytes) + if let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) { + self.establish_scrd_match_context(user); + } + } + + fn cold_match_continue( + &self, + source: crate::sks::KeyBagHandle, + uuid: [u8; keybag::UUID_LEN], + ) -> bool { + if !self.keybag_designated.load(Relaxed) { + return false; + } + + let Some(user) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) else { + return false; + }; + let special = user.special_handle(); + self.log_identity_count(c"before the cold-match preparation"); + + let uuid_ok = self + .sks_send(self.sks_req_copy_uuid_special(special)) + .and_then(|out| self.sks_uuid_from_reply(&out)); + match uuid_ok { + Some(got) if got == uuid => {}, + Some(_) => { + return false; + } + None => { + return false; + } + } + + if self.sks_send(self.sks_req_unload_keybag(source)).is_none() { + return false; + } + + // unlock/0x18 is the destructive-template path (bag stays designated, locked) + self.cold_prepared.store(true, Relaxed); + true + } + + pub(crate) fn sep_random(&self, buf: &mut [u8]) -> Result<()> { + for chunk in buf.chunks_mut(4) { + let word = self.get_entropy_word()?; + let bytes = word.to_le_bytes(); + let take = chunk.len(); + chunk.copy_from_slice(&bytes[..take]); + } + Ok(()) + } + + pub(crate) fn wrapped_from_copy_reply(&self, out: &SksOutcome, _from: &CStr) -> Option> { + let body = self.sks_report_response(c"COPY_KEYBAG", out)?; + if out.reply.status != 0 || image::operation_status(body).unwrap_or(-1) != 0 { + return None; + } + let (blob, _) = image::read_blob(body, 4)?; + if blob.is_empty() { + return None; + } + let mut copy = KVec::new(); + if copy.extend_from_slice(blob, GFP_KERNEL).is_err() { + return None; + } + Some(copy) + } + + fn prove_restore(&self) -> bool { + let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + return false; + }; + let op = crate::sbio::sbio_free_identity_count(user); + let Some(reply) = self.sbio_expect_ok(&op) else { + return false; + }; + if reply.len() != crate::sbio::SBIO_FREE_COUNT_REPLY_LEN { + return false; + } + let count = u32::from_le_bytes([reply[0], reply[1], reply[2], reply[3]]); + if count > crate::sbio::SBIO_FREE_COUNT_MAX { + return false; + } + if count == crate::sbio::SBIO_FREE_COUNT_MAX { + return true; + } + true + } + + fn log_identity_count(&self, _when: &CStr) -> Option { + let op = crate::sbio::sbio_list_identities(); + let SbioOutcome::Ok(reply) = self.sbio_call(&op) else { + return None; + }; + let records = crate::sbio::IdentityRecords::new(&reply)?; + Some(records.count()) + } + + fn enclave_lists(&self, uuid: &[u8; bio::UUID_LEN]) -> Option { + let op = crate::sbio::sbio_list_identities(); + let SbioOutcome::Ok(reply) = self.sbio_call(&op) else { + return None; + }; + let records = crate::sbio::IdentityRecords::new(&reply)?; + Some(records.lists_uuid(uuid)) + } + + fn delete_identity(&self, identity: &crate::sbio::IdentityV1) -> Result<()> { + if self.enclave_lists(identity.uuid()) == Some(false) { + self.forget_identity(identity.uuid(), c"a host entry the enclave never had"); + return Ok(()); + } + + let op = crate::sbio::sbio_delete_identity(identity); + + if self.sbio_expect_ok(&op).is_none() { + dev_err!( + self.dev, + "bio: 0x57 failed, status above; host index left unchanged (forgetting a template the enclave still holds would report a deletion that did not happen)\n" + ); + return Err(EIO); + } + + self.forget_identity(identity.uuid(), c"removed from the enclave"); + Ok(()) + } + + fn forget_identity(&self, uuid: &[u8; bio::UUID_LEN], why: &CStr) { + let persisted = { + let mut index = self.bio_index.lock(); + index.remove(uuid); + self.with_store(|store| index.persist(store)) + }; + match persisted { + Some(Ok(())) => {}, + _ => dev_warn!( + self.dev, + "bio: {} ({}) but the host index could not be written; correct in memory this boot, rebuilt at next attach\n", + Hex(uuid), + why + ), + } + } + + pub(crate) fn bio_ready(&self) -> bool { + self.bio_session.lock().ready() + } +} + +pub(crate) struct SbioOp { + opcode: u16, + payload: [u8; SBIO_MAX_PAYLOAD], + payload_len: usize, + name: &'static CStr, +} + +const SBIO_MAX_PAYLOAD: usize = 0x97; + +pub(crate) const SBIO_IMAGE_INIT_LEN: usize = 0x97; +static_assert!(SBIO_IMAGE_INIT_LEN <= SBIO_MAX_PAYLOAD); + +const fn pad_payload(src: &[u8]) -> [u8; SBIO_MAX_PAYLOAD] { + let mut out = [0u8; SBIO_MAX_PAYLOAD]; + let mut i = 0; + while i < src.len() { + out[i] = src[i]; + i += 1; + } + out +} + +impl SbioOp { + pub(crate) fn opcode(&self) -> u16 { + self.opcode + } + pub(crate) fn payload(&self) -> &[u8] { + &self.payload[..self.payload_len] + } + pub(crate) fn name(&self) -> &'static CStr { + self.name + } +} + +pub(crate) const SBIO_PROTOCOL_GENERATION: u32 = 1; + +const OP_SBIO_INIT_COMMS: u16 = 0x73; + +const OP_SBIO_REGISTER_SENSOR: u16 = 0x80; +pub(crate) fn sbio_register_sensor(id: &crate::sensor::Identifier) -> SbioOp { + SbioOp { + opcode: OP_SBIO_REGISTER_SENSOR, + payload: pad_payload(&id.value().to_le_bytes()), + payload_len: 2, + name: c"REGISTER_SENSOR", + } +} + +const OP_SBIO_SEND_SERIAL: u16 = 0x48; + +pub(crate) const SBIO_SIGNAL_QUALITY: u32 = 0; + +const OP_SBIO_COVERAGE_PARAMS: u16 = 0x5d; +pub(crate) fn sbio_coverage_params() -> SbioOp { + SbioOp { + opcode: OP_SBIO_COVERAGE_PARAMS, + payload: pad_payload(&SBIO_SIGNAL_QUALITY.to_le_bytes()), + payload_len: 4, + name: c"COVERAGE_PARAMS", + } +} + +const OP_SBIO_OPERATION_PARAMS: u16 = 0x5c; + +pub(crate) const SBIO_OPERATION_PARAMS_LEN: usize = match SBIO_PROTOCOL_GENERATION { + 1 => 4, + 6 => 8, + _ => 0, +}; +static_assert!(SBIO_OPERATION_PARAMS_LEN != 0); +static_assert!(SBIO_OPERATION_PARAMS_LEN <= SBIO_MAX_PAYLOAD); + +pub(crate) fn sbio_operation_params() -> SbioOp { + SbioOp { + opcode: OP_SBIO_OPERATION_PARAMS, + payload: pad_payload(&[]), + payload_len: SBIO_OPERATION_PARAMS_LEN, + name: c"OPERATION_PARAMS", + } +} + +const OP_SBIO_TRANSPARENT_CHANNEL: u16 = 0x6a; +pub(crate) fn sbio_transparent_channel() -> SbioOp { + SbioOp { + opcode: OP_SBIO_TRANSPARENT_CHANNEL, + payload: pad_payload(&[]), + payload_len: 0, + name: c"TRANSPARENT_CHANNEL", + } +} + +const OP_SBIO_COMPLETE_INIT: u16 = 0x01; +pub(crate) fn sbio_complete_init( + patch: crate::PatchLoaded, + params: crate::ParametersApplied, +) -> SbioOp { + let (_, _) = (patch, params); + let mut body = [0u8; 8]; + body[0..4].copy_from_slice(&1u32.to_le_bytes()); + body[4..8].copy_from_slice(&1u32.to_le_bytes()); + SbioOp { + opcode: OP_SBIO_COMPLETE_INIT, + payload: pad_payload(&body), + payload_len: 8, + name: c"COMPLETE_INIT", + } +} + +const CONTEXT_SCOPE_SYSTEM: i32 = -1; +static_assert!(CONTEXT_SCOPE_SYSTEM != 0); +static_assert!(CONTEXT_SCOPE_SYSTEM < 0); + +#[derive(Clone, Copy)] +pub(crate) struct ContextScope(i32); + +impl ContextScope { + pub(crate) const SYSTEM: ContextScope = ContextScope(CONTEXT_SCOPE_SYSTEM); + + pub(crate) fn user(id: UserId) -> ContextScope { + ContextScope(id.value()) + } + + pub(crate) fn value(&self) -> i32 { + self.0 + } +} + +const OP_SBIO_CONTEXT_STATE: u16 = 0x6b; +pub(crate) fn sbio_context_state() -> SbioOp { + SbioOp { + opcode: OP_SBIO_CONTEXT_STATE, + payload: pad_payload(&[]), + payload_len: 0, + name: c"CONTEXT_STATE", + } +} + +// 0x2d destroys the catacomb; only send with zero existing identities +pub(crate) struct NoExistingCatacomb { + _seal: (), +} + +impl NoExistingCatacomb { + pub(crate) fn from_zero_identities(count: usize) -> Option { + match count { + 0 => Some(NoExistingCatacomb { _seal: () }), + _ => None, + } + } +} + +const OP_SBIO_SELECT_CONTEXT: u16 = 0x2d; +pub(crate) fn sbio_select_context(scope: ContextScope, proof: &NoExistingCatacomb) -> SbioOp { + let NoExistingCatacomb { _seal: () } = proof; + SbioOp { + opcode: OP_SBIO_SELECT_CONTEXT, + payload: pad_payload(&scope.value().to_le_bytes()), + payload_len: 4, + name: c"SELECT_CONTEXT", + } +} + +pub(crate) const SBIO_PROTECTED_CONFIG_LEN: usize = 32; + +const OP_SBIO_PROTECTED_CONFIG: u16 = 0x2b; +pub(crate) fn sbio_protected_config(id: UserId) -> SbioOp { + SbioOp { + opcode: OP_SBIO_PROTECTED_CONFIG, + payload: pad_payload(&id.value().to_le_bytes()), + payload_len: 4, + name: c"PROTECTED_CONFIG", + } +} + +const OP_SBIO_BEGIN_ENROL: u16 = 0x03; + +// type 0 (ACM context) survives a reboot; type 1 (SKS token) does not +pub(crate) const BE_AUTH_TYPE_ACM_CONTEXT: u32 = 0; +pub(crate) const BE_AUTH_TYPE_SKS_TOKEN: u32 = 1; +static_assert!(BE_AUTH_TYPE_ACM_CONTEXT != BE_AUTH_TYPE_SKS_TOKEN); + +pub(crate) const SBIO_BEGIN_ENROL_LEN: usize = 0x44; +static_assert!(SBIO_BEGIN_ENROL_LEN <= SBIO_MAX_PAYLOAD); + +const BE_FLAGS: usize = 0; +const BE_USER_ID: usize = 4; +const BE_AUTH_TYPE: usize = 8; +const BE_TOKEN_LEN: usize = 12; +const BE_TOKEN: usize = 16; +const BE_SELECTOR: usize = 48; +static_assert!(BE_TOKEN + SKS_AUTH_TOKEN_LEN <= BE_SELECTOR); +const BE_SELECTOR_LEN: usize = 20; +static_assert!(BE_SELECTOR + BE_SELECTOR_LEN == SBIO_BEGIN_ENROL_LEN); + +pub(crate) const SBIO_BEGIN_ENROL_COPIED: usize = match SBIO_PROTOCOL_GENERATION { + 1 => 0x30, + 6 => SBIO_BEGIN_ENROL_LEN, + _ => 0, +}; +static_assert!(SBIO_BEGIN_ENROL_COPIED != 0); +static_assert!(SBIO_BEGIN_ENROL_COPIED <= SBIO_BEGIN_ENROL_LEN); + +const BE_SELECTOR_IN_RECORD: bool = SBIO_BEGIN_ENROL_COPIED >= BE_SELECTOR + BE_SELECTOR_LEN; + +static_assert!(!BE_SELECTOR_IN_RECORD); + +pub(crate) fn sbio_begin_enrol( + user: UserId, + auth_type: u32, + token: &[u8; SKS_AUTH_TOKEN_LEN], +) -> SbioOp { + let mut body = [0u8; SBIO_BEGIN_ENROL_LEN]; + body[BE_FLAGS..BE_FLAGS + 4].copy_from_slice(&0u32.to_le_bytes()); + body[BE_USER_ID..BE_USER_ID + 4].copy_from_slice(&user.value().to_le_bytes()); + body[BE_AUTH_TYPE..BE_AUTH_TYPE + 4].copy_from_slice(&auth_type.to_le_bytes()); + body[BE_TOKEN_LEN..BE_TOKEN_LEN + 4] + .copy_from_slice(&(SKS_AUTH_TOKEN_LEN as u32).to_le_bytes()); + body[BE_TOKEN..BE_TOKEN + SKS_AUTH_TOKEN_LEN].copy_from_slice(token); + SbioOp { + opcode: OP_SBIO_BEGIN_ENROL, + payload: pad_payload(&body), + payload_len: SBIO_BEGIN_ENROL_LEN, + name: c"BEGIN_ENROL", + } +} + +const OP_SBIO_MODULE_CHALLENGE: u16 = 0x31; +pub(crate) fn sbio_module_challenge() -> SbioOp { + SbioOp { + opcode: OP_SBIO_MODULE_CHALLENGE, + payload: pad_payload(&[]), + payload_len: 0, + name: c"MODULE_CHALLENGE", + } +} + +pub(crate) const SBIO_MODULE_CHALLENGE_LEN: usize = 0x40; + +const OP_SBIO_MODULE_COMMIT: u16 = 0x32; +pub(crate) fn sbio_module_commit(reply: &[u8]) -> Option { + if reply.is_empty() || reply.len() > SBIO_MAX_PAYLOAD { + return None; + } + Some(SbioOp { + opcode: OP_SBIO_MODULE_COMMIT, + payload: pad_payload(reply), + payload_len: reply.len(), + name: c"MODULE_COMMIT", + }) +} + +const OP_SBIO_LOAD_CALIBRATION: u16 = 0x5b; + +pub(crate) const SBIO_CALIBRATION_SOURCE: u32 = 3; + +pub(crate) struct SbioCalibration(KVec); + +impl SbioCalibration { + pub(crate) fn new(blob: &crate::CalibrationBlob) -> Result { + let bytes = blob.bytes(); + let mut body = KVec::new(); + body.extend_from_slice(&SBIO_CALIBRATION_SOURCE.to_le_bytes(), GFP_KERNEL)?; + body.extend_from_slice(bytes, GFP_KERNEL)?; + Ok(SbioCalibration(body)) + } + + pub(crate) fn opcode(&self) -> u16 { + OP_SBIO_LOAD_CALIBRATION + } + + pub(crate) fn payload(&self) -> &[u8] { + &self.0 + } + + pub(crate) fn name(&self) -> &'static CStr { + c"LOAD_CALIBRATION" + } +} + +const OP_SBIO_UPDATE_DEVICE_LIST: u16 = 0x7b; +pub(crate) fn sbio_update_device_list() -> SbioOp { + SbioOp { + opcode: OP_SBIO_UPDATE_DEVICE_LIST, + payload: pad_payload(&[]), + payload_len: 0, + name: c"UPDATE_DEVICE_LIST", + } +} + +pub(crate) const SBIO_MATCH_POLICY_LEN: usize = 2; + +const OP_SBIO_MATCH_POLICY: u16 = 0x1f; +pub(crate) fn sbio_match_policy() -> SbioOp { + SbioOp { + opcode: OP_SBIO_MATCH_POLICY, + payload: pad_payload(&[]), + payload_len: 0, + name: c"MATCH_POLICY", + } +} + +pub(crate) const SBIO_ENROL_RESULT_LEN: usize = 0xc98; + +const ER_STATUS: usize = 0x000; +const ER_ERROR: usize = 0x002; +const ER_PROGRESS: usize = 0x004; +const ER_HAS_TEMPLATE: usize = 0x006; +const ER_COMPLETE: usize = 0xbfe; + +static_assert!(ER_STATUS + 2 <= SBIO_ENROL_RESULT_LEN); +static_assert!(ER_ERROR + 2 <= SBIO_ENROL_RESULT_LEN); +static_assert!(ER_PROGRESS < SBIO_ENROL_RESULT_LEN); +static_assert!(ER_HAS_TEMPLATE + 4 <= SBIO_ENROL_RESULT_LEN); +static_assert!(ER_COMPLETE + 4 <= SBIO_ENROL_RESULT_LEN); +static_assert!(ER_COMPLETE > ER_HAS_TEMPLATE + 4); + +pub(crate) const ER_IDENTITY_LEN: usize = 4 + IDENTITY_UUID_LEN; +static_assert!(ER_IDENTITY_LEN == 20); +static_assert!(ER_IDENTITY_LEN <= SBIO_ENROL_RESULT_LEN); + +pub(crate) fn enrol_identity_candidates( + bytes: &[u8], + user_id: i32, + mut visit: impl FnMut(usize, [u8; IDENTITY_UUID_LEN]), +) { + if bytes.len() < ER_IDENTITY_LEN { + return; + } + let wanted = user_id.to_le_bytes(); + for at in 0..=bytes.len() - ER_IDENTITY_LEN { + if bytes[at..at + 4] != wanted { + continue; + } + let mut uuid = [0u8; IDENTITY_UUID_LEN]; + uuid.copy_from_slice(&bytes[at + 4..at + ER_IDENTITY_LEN]); + if uuid.iter().all(|&b| b == 0) { + continue; + } + visit(at, uuid); + } +} + +pub(crate) struct EnrolmentResult { + progress_raw: u8, + has_template: u32, + complete: u32, +} + +impl EnrolmentResult { + pub(crate) fn parse(bytes: &[u8]) -> Option { + if bytes.len() < SBIO_ENROL_RESULT_LEN { + return None; + } + Some(EnrolmentResult { + progress_raw: bytes[ER_PROGRESS], + has_template: u32::from_le_bytes([ + bytes[ER_HAS_TEMPLATE], + bytes[ER_HAS_TEMPLATE + 1], + bytes[ER_HAS_TEMPLATE + 2], + bytes[ER_HAS_TEMPLATE + 3], + ]), + complete: u32::from_le_bytes([ + bytes[ER_COMPLETE], + bytes[ER_COMPLETE + 1], + bytes[ER_COMPLETE + 2], + bytes[ER_COMPLETE + 3], + ]), + }) + } + + pub(crate) fn progress_percent(&self) -> u32 { + (u32::from(self.progress_raw) * 100 + 127) / 255 + } + + pub(crate) fn has_template(&self) -> bool { + self.has_template != 0 + } + + pub(crate) fn complete(&self) -> bool { + self.complete != 0 + } +} + +#[derive(Clone, Copy)] +pub(crate) struct CatacombUser(i32); + +impl CatacombUser { + pub(crate) const MASTER: CatacombUser = CatacombUser(-1); + pub(crate) const OWNER: CatacombUser = CatacombUser(501); + + pub(crate) fn enrolling(user: UserId) -> CatacombUser { + CatacombUser(user.value()) + } + + pub(crate) const fn value(&self) -> i32 { + self.0 + } +} +static_assert!(CatacombUser::MASTER.value() == -1); +static_assert!(CatacombUser::OWNER.value() == 501); +static_assert!(CatacombUser::MASTER.value() != CatacombUser::OWNER.value()); +static_assert!(CatacombUser::MASTER.value() < 0); + +pub(crate) const SBIO_SAVED_USER_ID_AT: usize = 8; +static_assert!(SBIO_SAVED_USER_ID_AT + 4 <= SBIO_SAVED_MIN); + +pub(crate) const SBIO_SAVE_SELECTOR_LEN: usize = 24; +const SS_USER_ID: usize = 0; +const SS_DEVICE: usize = 4; +const SS_DEVICE_LEN: usize = 20; +static_assert!(SS_DEVICE + SS_DEVICE_LEN == SBIO_SAVE_SELECTOR_LEN); +static_assert!(SBIO_SAVE_SELECTOR_LEN <= SBIO_MAX_PAYLOAD); + +pub(crate) struct SaveSelector([u8; SBIO_SAVE_SELECTOR_LEN]); + +impl SaveSelector { + pub(crate) fn new(user: CatacombUser) -> SaveSelector { + let mut out = [0u8; SBIO_SAVE_SELECTOR_LEN]; + out[SS_USER_ID..SS_USER_ID + 4].copy_from_slice(&user.value().to_le_bytes()); + SaveSelector(out) + } + + pub(crate) fn bytes(&self) -> &[u8; SBIO_SAVE_SELECTOR_LEN] { + &self.0 + } +} + +pub(crate) const SBIO_STATUS_COLD_TRANSITION: u32 = 0x8002; +static_assert!(SBIO_STATUS_COLD_TRANSITION != 0); + +// 0x101 from a 0x6d load = already active; a success answer, not an error +pub(crate) const SBIO_STATUS_ALREADY_ACTIVE: u32 = 0x101; +static_assert!(SBIO_STATUS_ALREADY_ACTIVE != 0); +static_assert!(SBIO_STATUS_ALREADY_ACTIVE != SBIO_STATUS_COLD_TRANSITION); + +pub(crate) const COMPONENT_STATE_COLD: u32 = 0x1; +pub(crate) const COMPONENT_STATE_ACTIVE: u32 = 0x2; +static_assert!(COMPONENT_STATE_COLD != COMPONENT_STATE_ACTIVE); + +pub(crate) const COMPONENT_PAIR_LEN: usize = 8; + +pub(crate) struct ComponentStates<'a>(&'a [u8]); + +impl<'a> ComponentStates<'a> { + pub(crate) fn new(reply: &'a [u8]) -> ComponentStates<'a> { + ComponentStates(reply) + } + + pub(crate) fn count(&self) -> usize { + self.0.len() / COMPONENT_PAIR_LEN + } + + pub(crate) fn trailing(&self) -> usize { + self.0.len() % COMPONENT_PAIR_LEN + } + + pub(crate) fn pair(&self, i: usize) -> Option<(i32, u32)> { + let at = i.checked_mul(COMPONENT_PAIR_LEN)?; + let bytes = self.0.get(at..at + COMPONENT_PAIR_LEN)?; + Some(( + i32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]), + u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]), + )) + } + + pub(crate) fn state_for(&self, id: i32) -> Option { + (0..self.count()) + .filter_map(|i| self.pair(i)) + .find(|(pair_id, _)| *pair_id == id) + .map(|(_, state)| state) + } +} + +pub(crate) enum ComponentAction { + AlreadyActive, + Load, + Unsupported(u32), +} + +pub(crate) fn component_action(state: u32) -> ComponentAction { + if state & COMPONENT_STATE_ACTIVE != 0 { + ComponentAction::AlreadyActive + } else if state & COMPONENT_STATE_COLD != 0 { + ComponentAction::Load + } else { + ComponentAction::Unsupported(state) + } +} + +const OP_SBIO_SAVE_LOCKOUT: u16 = 0x70; +pub(crate) fn sbio_save_lockout() -> SbioOp { + SbioOp { + opcode: OP_SBIO_SAVE_LOCKOUT, + payload: pad_payload(&[]), + payload_len: 0, + name: c"SAVE_LOCKOUT", + } +} + +pub(crate) struct SbioLoadLockout<'a>(&'a [u8]); + +impl<'a> SbioLoadLockout<'a> { + pub(crate) fn new(blob: &'a [u8]) -> Option> { + if blob.is_empty() || blob.len() > SBIO_SAVED_MAX { + return None; + } + Some(SbioLoadLockout(blob)) + } + + pub(crate) fn opcode(&self) -> u16 { + OP_SBIO_LOAD_LOCKOUT + } + + pub(crate) fn payload(&self) -> &[u8] { + self.0 + } + + pub(crate) fn name(&self) -> &'static CStr { + c"LOAD_LOCKOUT" + } +} +const OP_SBIO_LOAD_LOCKOUT: u16 = 0x71; + +const OP_SBIO_SAVE_CATACOMB: u16 = 0x6c; +pub(crate) fn sbio_save_catacomb(selector: &SaveSelector) -> SbioOp { + SbioOp { + opcode: OP_SBIO_SAVE_CATACOMB, + payload: pad_payload(selector.bytes()), + payload_len: SBIO_SAVE_SELECTOR_LEN, + name: c"SAVE_CATACOMB", + } +} + +const OP_SBIO_CONFIRM_SAVE: u16 = 0x37; +pub(crate) fn sbio_confirm_save(selector: &SaveSelector) -> SbioOp { + SbioOp { + opcode: OP_SBIO_CONFIRM_SAVE, + payload: pad_payload(selector.bytes()), + payload_len: SBIO_SAVE_SELECTOR_LEN, + name: c"CONFIRM_SAVE", + } +} + +pub(crate) const SBIO_SAVED_MIN: usize = 0x22; +pub(crate) const SBIO_SAVED_MAX: usize = 0x4b000; +static_assert!(SBIO_SAVED_MIN < SBIO_SAVED_MAX); +static_assert!(SBIO_SAVED_MAX as u32 <= transfer::MAX_TRANSACTION); + +pub(crate) struct SbioLoadCatacomb<'a>(&'a [u8]); + +impl<'a> SbioLoadCatacomb<'a> { + pub(crate) fn new(blob: &'a [u8]) -> Option> { + if blob.len() < SBIO_SAVED_MIN || blob.len() > SBIO_SAVED_MAX { + return None; + } + Some(SbioLoadCatacomb(blob)) + } + + pub(crate) fn opcode(&self) -> u16 { + OP_SBIO_LOAD_CATACOMB + } + + pub(crate) fn payload(&self) -> &[u8] { + self.0 + } + + pub(crate) fn name(&self) -> &'static CStr { + c"LOAD_CATACOMB" + } +} +const OP_SBIO_LOAD_CATACOMB: u16 = 0x6d; + +pub(crate) const IDENTITY_RECORD_LEN: usize = 40; +const IR_SELECTOR: usize = 20; +const IR_SELECTOR_BUILTIN: u32 = 1; +static_assert!(IR_SELECTOR == IDENTITY_V1_LEN); +static_assert!(IR_SELECTOR + 4 <= IDENTITY_RECORD_LEN); + +const OP_SBIO_LIST_IDENTITIES: u16 = 0x7a; +pub(crate) fn sbio_list_identities() -> SbioOp { + SbioOp { + opcode: OP_SBIO_LIST_IDENTITIES, + payload: pad_payload(&[]), + payload_len: 0, + name: c"LIST_IDENTITIES", + } +} + +const OP_SBIO_GROUP_STATE: u16 = 0x79; + +const OP_SBIO_LIST_IDENTITIES_SCOPED: u16 = 0x6e; + +pub(crate) struct IdentityRecords<'a>(&'a [u8]); + +impl<'a> IdentityRecords<'a> { + pub(crate) fn new(reply: &'a [u8]) -> Option> { + if reply.len() % IDENTITY_RECORD_LEN != 0 { + return None; + } + Some(IdentityRecords(reply)) + } + + pub(crate) fn count(&self) -> usize { + self.0.len() / IDENTITY_RECORD_LEN + } + + pub(crate) fn record(&self, i: usize) -> Option<(IdentityV1, bool)> { + let at = i.checked_mul(IDENTITY_RECORD_LEN)?; + let bytes = self.0.get(at..at + IDENTITY_RECORD_LEN)?; + let user_id = i32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); + let mut uuid = [0u8; IDENTITY_UUID_LEN]; + uuid.copy_from_slice(&bytes[4..IDENTITY_V1_LEN]); + let selector = u32::from_le_bytes([ + bytes[IR_SELECTOR], + bytes[IR_SELECTOR + 1], + bytes[IR_SELECTOR + 2], + bytes[IR_SELECTOR + 3], + ]); + Some(( + IdentityV1 { user_id, uuid }, + selector == IR_SELECTOR_BUILTIN, + )) + } + + pub(crate) fn lists_uuid(&self, uuid: &[u8; IDENTITY_UUID_LEN]) -> bool { + (0..self.count()) + .filter_map(|i| self.record(i)) + .any(|(identity, _)| identity.uuid == *uuid) + } + + pub(crate) fn each_built_in_for(&self, user_id: i32, mut visit: impl FnMut(IdentityV1)) { + for i in 0..self.count() { + if let Some((identity, built_in)) = self.record(i) { + if built_in && identity.user_id == user_id { + visit(identity); + } + } + } + } +} + +const FIC_DEVICE_BUILTIN: u32 = 1; +pub(crate) const SBIO_FREE_COUNT_LEN: usize = 4 + 20; +static_assert!(SBIO_FREE_COUNT_LEN == SBIO_SAVE_SELECTOR_LEN); +pub(crate) const SBIO_FREE_COUNT_REPLY_LEN: usize = 4; +pub(crate) const SBIO_FREE_COUNT_MAX: u32 = 3; + +const OP_SBIO_FREE_IDENTITY_COUNT: u16 = 0x38; +pub(crate) fn sbio_free_identity_count(user: UserId) -> SbioOp { + let mut body = [0u8; SBIO_FREE_COUNT_LEN]; + body[0..4].copy_from_slice(&user.value().to_le_bytes()); + body[4..8].copy_from_slice(&FIC_DEVICE_BUILTIN.to_le_bytes()); + SbioOp { + opcode: OP_SBIO_FREE_IDENTITY_COUNT, + payload: pad_payload(&body), + payload_len: SBIO_FREE_COUNT_LEN, + name: c"FREE_IDENTITY_COUNT", + } +} + +const OP_SBIO_DELETE_IDENTITY: u16 = 0x57; +pub(crate) fn sbio_delete_identity(identity: &IdentityV1) -> SbioOp { + SbioOp { + opcode: OP_SBIO_DELETE_IDENTITY, + payload: pad_payload(&identity.wire()), + payload_len: IDENTITY_V1_LEN, + name: c"DELETE_IDENTITY", + } +} + +const OP_SBIO_MATCH_RESULT: u16 = 0x09; +pub(crate) fn sbio_match_result() -> SbioOp { + SbioOp { + opcode: OP_SBIO_MATCH_RESULT, + payload: pad_payload(&[]), + payload_len: 0, + name: c"MATCH_RESULT", + } +} + +pub(crate) const SBIO_MATCH_RESULT_LEN: usize = 0xca2; + +const MR_USER_ID: usize = 0x000; +const MR_IDENTITY: usize = 0x004; +const MR_CANDIDATES: usize = 0x014; +const MR_FLAGS: usize = 0xc8a; +const MR_SECOND_IDENTITY_UNUSED: usize = 0xc8e; +static_assert!(MR_IDENTITY != MR_SECOND_IDENTITY_UNUSED); +static_assert!(MR_USER_ID + 4 <= SBIO_MATCH_RESULT_LEN); +static_assert!(MR_IDENTITY + IDENTITY_UUID_LEN <= SBIO_MATCH_RESULT_LEN); +static_assert!(MR_CANDIDATES + 4 <= SBIO_MATCH_RESULT_LEN); +static_assert!(MR_FLAGS + 4 <= SBIO_MATCH_RESULT_LEN); +static_assert!(MR_SECOND_IDENTITY_UNUSED + IDENTITY_UUID_LEN <= SBIO_MATCH_RESULT_LEN); +static_assert!(MR_IDENTITY == MR_USER_ID + 4); +static_assert!(MR_IDENTITY + IDENTITY_UUID_LEN == MR_CANDIDATES); +static_assert!(MR_FLAGS > MR_USER_ID + 4); +static_assert!(SBIO_MATCH_RESULT_LEN != SBIO_ENROL_RESULT_LEN); + +pub(crate) const IDENTITY_UUID_LEN: usize = 16; +pub(crate) const IDENTITY_V1_LEN: usize = 4 + IDENTITY_UUID_LEN; +static_assert!(IDENTITY_V1_LEN == 0x14); + +#[derive(Clone, Copy)] +pub(crate) struct IdentityV1 { + user_id: i32, + uuid: [u8; IDENTITY_UUID_LEN], +} + +impl IdentityV1 { + pub(crate) fn from_index(user_id: i32, uuid: [u8; IDENTITY_UUID_LEN]) -> IdentityV1 { + IdentityV1 { user_id, uuid } + } + + pub(crate) fn uuid(&self) -> &[u8; IDENTITY_UUID_LEN] { + &self.uuid + } + + pub(crate) fn wire(&self) -> [u8; IDENTITY_V1_LEN] { + let mut out = [0u8; IDENTITY_V1_LEN]; + out[0..4].copy_from_slice(&self.user_id.to_le_bytes()); + out[4..IDENTITY_V1_LEN].copy_from_slice(&self.uuid); + out + } +} + +pub(crate) struct MatchResult { + user_id: i32, + identity: [u8; IDENTITY_UUID_LEN], +} + +impl MatchResult { + pub(crate) fn parse(bytes: &[u8]) -> Option { + if bytes.len() < SBIO_MATCH_RESULT_LEN { + return None; + } + let mut identity = [0u8; IDENTITY_UUID_LEN]; + identity.copy_from_slice(&bytes[MR_IDENTITY..MR_IDENTITY + IDENTITY_UUID_LEN]); + Some(MatchResult { + user_id: i32::from_le_bytes([ + bytes[MR_USER_ID], + bytes[MR_USER_ID + 1], + bytes[MR_USER_ID + 2], + bytes[MR_USER_ID + 3], + ]), + identity, + }) + } + + pub(crate) fn identity_uuid(&self) -> &[u8; IDENTITY_UUID_LEN] { + &self.identity + } + + pub(crate) fn matches(&self, user: UserId) -> bool { + self.user_id == user.value() + } + +} + +const OP_SBIO_IMAGE_CLEANUP: u16 = 0x22; +pub(crate) fn sbio_image_cleanup() -> SbioOp { + SbioOp { + opcode: OP_SBIO_IMAGE_CLEANUP, + payload: pad_payload(&[]), + payload_len: 0, + name: c"IMAGE_CLEANUP", + } +} + +const OP_SBIO_CANCEL: u16 = 0x05; +pub(crate) fn sbio_cancel_operation() -> SbioOp { + SbioOp { + opcode: OP_SBIO_CANCEL, + payload: pad_payload(&[]), + payload_len: 0, + name: c"CANCEL_OPERATION", + } +} + +const OP_SBIO_CLEAR_STATE: u16 = 0x19; +pub(crate) fn sbio_clear_state() -> SbioOp { + SbioOp { + opcode: OP_SBIO_CLEAR_STATE, + payload: pad_payload(&[]), + payload_len: 0, + name: c"CLEAR_STATE", + } +} + +const OP_SBIO_REGISTER_SERIAL: u16 = 0x17; +pub(crate) fn sbio_register_sensor_serial(serial: &crate::sensor::SensorSerial) -> SbioOp { + SbioOp { + opcode: OP_SBIO_REGISTER_SERIAL, + payload: pad_payload(serial.bytes()), + payload_len: crate::sensor::SENSOR_SERIAL_LEN, + name: c"REGISTER_SERIAL", + } +} + +const OP_SBIO_ENUMERATE: u16 = 0x7c; +pub(crate) const fn sbio_enumerate() -> SbioOp { + SbioOp { + opcode: OP_SBIO_ENUMERATE, + payload: [0; SBIO_MAX_PAYLOAD], + payload_len: 0, + name: c"ENUMERATE", + } +} + +const OP_SBIO_DIAGNOSTICS: u16 = 0x63; +pub(crate) const fn sbio_diagnostics() -> SbioOp { + SbioOp { + opcode: OP_SBIO_DIAGNOSTICS, + payload: [0; SBIO_MAX_PAYLOAD], + payload_len: 0, + name: c"DIAGNOSTICS", + } +} + +const SURVEY_ENUMERATE: SbioOp = sbio_enumerate(); +const SURVEY_DIAGNOSTICS: SbioOp = sbio_diagnostics(); + +static_assert!(SURVEY_ENUMERATE.payload_len == 0); +static_assert!(SURVEY_DIAGNOSTICS.payload_len == 0); + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum ImagePurpose { + Enrolment, + Matching, +} + +impl ImagePurpose { + const fn offset(self) -> usize { + match self { + ImagePurpose::Matching => 0x12, + ImagePurpose::Enrolment => 0x13, + } + } + + const fn last_image_offset(self) -> usize { + match self { + ImagePurpose::Enrolment => 0x08, + ImagePurpose::Matching => 0x09, + } + } + +} + +static_assert!(ImagePurpose::Enrolment.offset() != ImagePurpose::Matching.offset()); +static_assert!(ImagePurpose::Matching.offset() == 0x12); +static_assert!(ImagePurpose::Enrolment.offset() == 0x13); +static_assert!(ImagePurpose::Enrolment.last_image_offset() == 0x08); +static_assert!(ImagePurpose::Matching.last_image_offset() == 0x09); +static_assert!( + ImagePurpose::Enrolment.last_image_offset() != ImagePurpose::Matching.last_image_offset() +); +static_assert!(ImagePurpose::Matching.last_image_offset() < ImagePurpose::Matching.offset()); + +#[derive(Clone, Copy)] +pub(crate) struct UserId(i32); + +impl UserId { + pub(crate) fn new(value: i32) -> Option { + if value > 0 { + Some(UserId(value)) + } else { + None + } + } + + pub(crate) const fn value(&self) -> i32 { + self.0 + } +} + +const IPI_FIRST_IMAGE: usize = 0x07; +const IPI_TIMESTAMP: usize = 0x40; +const IPI_CAPTURE_COUNTER: usize = 0x48; +const IPI_DEVICE_KIND: usize = 0x4f; +const IPI_USER_ID: usize = 0x67; + +const IPI_DEVICE_BUILTIN: u32 = 1; + +static_assert!(IPI_USER_ID + 4 <= SBIO_MAX_PAYLOAD); +static_assert!(IPI_DEVICE_KIND + 4 <= SBIO_MAX_PAYLOAD); +static_assert!(IPI_TIMESTAMP + 8 <= SBIO_MAX_PAYLOAD); +static_assert!(IPI_CAPTURE_COUNTER + 4 <= SBIO_MAX_PAYLOAD); +static_assert!(IPI_FIRST_IMAGE < ImagePurpose::Matching.offset()); +static_assert!(ImagePurpose::Enrolment.offset() < IPI_TIMESTAMP); + +const OP_SBIO_PREPARE_IMAGE: u16 = 0x23; +pub(crate) fn sbio_prepare_image_processing() -> SbioOp { + SbioOp { + opcode: OP_SBIO_PREPARE_IMAGE, + payload: [0; SBIO_MAX_PAYLOAD], + payload_len: 0, + name: c"PREPARE_IMAGE_PROCESSING", + } +} + +const OP_SBIO_IMAGE_INIT: u16 = 0x06; +pub(crate) fn sbio_image_processing_init( + purpose: ImagePurpose, + first_image: bool, + last_image: bool, + capture_counter: u32, + user: UserId, + monotonic_ns: u64, +) -> SbioOp { + let mut payload = [0u8; SBIO_MAX_PAYLOAD]; + + if first_image { + payload[IPI_FIRST_IMAGE] = 1; + } + if last_image { + payload[purpose.last_image_offset()] = 1; + } + payload[purpose.offset()] = 1; + + // ns*24/1000 split to dodge u64 overflow and the kernel's absent __udivti3 + let scaled = (monotonic_ns / 1000) * 24 + ((monotonic_ns % 1000) * 24) / 1000; + payload[IPI_TIMESTAMP..IPI_TIMESTAMP + 8].copy_from_slice(&scaled.to_le_bytes()); + payload[IPI_CAPTURE_COUNTER..IPI_CAPTURE_COUNTER + 4] + .copy_from_slice(&capture_counter.to_le_bytes()); + payload[IPI_DEVICE_KIND..IPI_DEVICE_KIND + 4] + .copy_from_slice(&IPI_DEVICE_BUILTIN.to_le_bytes()); + payload[IPI_USER_ID..IPI_USER_ID + 4].copy_from_slice(&user.value().to_le_bytes()); + + SbioOp { + opcode: OP_SBIO_IMAGE_INIT, + payload, + payload_len: SBIO_IMAGE_INIT_LEN, + name: c"IMAGE_PROCESSING_INIT", + } +} + +const OP_SBIO_RELAY_CAPTURE: u16 = 0x65; + +pub(crate) struct SbioRelay<'a> { + capture: &'a crate::sensor::Capture, +} + +impl<'a> SbioRelay<'a> { + pub(crate) fn new(capture: &'a crate::sensor::Capture) -> SbioRelay<'a> { + SbioRelay { capture } + } + + pub(crate) fn opcode(&self) -> u16 { + OP_SBIO_RELAY_CAPTURE + } + + pub(crate) fn payload(&self) -> &[u8] { + self.capture.bytes() + } + + pub(crate) fn name(&self) -> &'static CStr { + c"RELAY_CAPTURE" + } +} + +const OP_SBIO_ASSESSMENT: u16 = 0x07; +pub(crate) fn sbio_image_assessment() -> SbioOp { + SbioOp { + opcode: OP_SBIO_ASSESSMENT, + payload: [0; SBIO_MAX_PAYLOAD], + payload_len: 0, + name: c"IMAGE_ASSESSMENT", + } +} + +const OP_SBIO_ENROL_RESULT: u16 = 0x08; +pub(crate) fn sbio_enrolment_result() -> SbioOp { + SbioOp { + opcode: OP_SBIO_ENROL_RESULT, + payload: [0; SBIO_MAX_PAYLOAD], + payload_len: 0, + name: c"ENROLMENT_RESULT", + } +} + +pub(crate) const ASSESS_MIN_LEN: usize = 0x89; +pub(crate) const ASSESS_ERROR: usize = 0x00; +pub(crate) const ASSESS_USABLE_MATCH: usize = 0x06; +pub(crate) const ASSESS_USABLE_ENROL: usize = 0x07; +pub(crate) const ASSESS_FEEDBACK: usize = 0x0e; +pub(crate) const ASSESS_DIRTY: usize = 0x51; + +static_assert!(ASSESS_ERROR + 2 <= ASSESS_MIN_LEN); +static_assert!(ASSESS_USABLE_MATCH < ASSESS_USABLE_ENROL); +static_assert!(ASSESS_FEEDBACK + 4 <= ASSESS_MIN_LEN); +static_assert!(ASSESS_DIRTY < ASSESS_MIN_LEN); + +pub(crate) const SBIO_SESSION_SHARE_LEN: usize = 40; + +pub(crate) const SBIO_STATUS_OK: u16 = 0x00; +pub(crate) const SBIO_STATUS_PREREQUISITE: u16 = 0x01; +pub(crate) const SBIO_STATUS_16: u16 = 0x16; +static_assert!(SBIO_STATUS_PREREQUISITE != SBIO_STATUS_16); +pub(crate) const SBIO_SESSION_MODE: u32 = 1; +static_assert!(SBIO_SESSION_SHARE_LEN <= SBIO_MAX_PAYLOAD); + +const OP_SBIO_SESSION_SHARE: u16 = 0x15; +pub(crate) fn sbio_request_session_share() -> SbioOp { + SbioOp { + opcode: OP_SBIO_SESSION_SHARE, + payload: pad_payload(&SBIO_SESSION_MODE.to_le_bytes()), + payload_len: 4, + name: c"SESSION_SHARE", + } +} + +const OP_SBIO_COMMIT_SESSION: u16 = 0x16; +pub(crate) fn sbio_commit_session_share(share: &[u8; SBIO_SESSION_SHARE_LEN]) -> SbioOp { + SbioOp { + opcode: OP_SBIO_COMMIT_SESSION, + payload: pad_payload(share), + payload_len: SBIO_SESSION_SHARE_LEN, + name: c"SESSION_COMMIT", + } +} +pub(crate) const SBIO_CHALLENGE_LEN: usize = 64; +static_assert!(SBIO_CHALLENGE_LEN <= SBIO_MAX_PAYLOAD); + +const OP_SBIO_CHALLENGE: u16 = 0x42; +pub(crate) fn sbio_request_challenge() -> SbioOp { + SbioOp { + opcode: OP_SBIO_CHALLENGE, + payload: [0; SBIO_MAX_PAYLOAD], + payload_len: 0, + name: c"SEQUENCE_CHALLENGE", + } +} + +const OP_SBIO_COMMIT_CHALLENGE: u16 = 0x18; +pub(crate) fn sbio_commit_challenge(reply: &[u8; SBIO_CHALLENGE_LEN]) -> SbioOp { + SbioOp { + opcode: OP_SBIO_COMMIT_CHALLENGE, + payload: pad_payload(reply), + payload_len: SBIO_CHALLENGE_LEN, + name: c"SEQUENCE_COMMIT", + } +} +const OP_SBIO_FETCH_PATCH: u16 = 0x5f; + +const SBIO_PATCH_SUBTYPE: u16 = 2; + +pub(crate) fn sbio_fetch_patch() -> SbioOp { + SbioOp { + opcode: OP_SBIO_FETCH_PATCH, + payload: pad_payload(&SBIO_PATCH_SUBTYPE.to_le_bytes()), + payload_len: 2, + name: c"FETCH_SENSOR_PATCH", + } +} + +const fn encode_sbio(opcode: u16, marker: u8, seq: u16) -> Message { + Message { + msg0: (EP_SBIO as u64) + | ((marker as u64) << MSG_TAG_SHIFT) + | ((opcode as u64) << MSG_TYPE_SHIFT) + | ((seq as u64) << 48), + msg1: 0, + } +} + +static_assert!(encode_sbio(OP_SBIO_INIT_COMMS, transfer::MARKER_FIRST, 0).msg0 == 0x0000_0073_fc08); + +pub(crate) fn encode_sbio_raw(opcode: u16, seq: u16) -> Option { + Some(encode_sbio(opcode, transfer::MARKER_FIRST, seq)) +} + +pub(crate) fn encode_sbio_next(opcode: u16, seq: u16) -> Option { + Some(encode_sbio(opcode, transfer::MARKER_NEXT, seq)) +} + +pub(crate) fn encode_sbio_continue(cont: &transfer::Continuation) -> Message { + encode_sbio(cont.opcode() as u16, transfer::MARKER_REQUEST, cont.seq()) +} diff --git a/drivers/soc/apple/scrd.rs b/drivers/soc/apple/scrd.rs new file mode 100644 index 0000000000000..0326308651ba4 --- /dev/null +++ b/drivers/soc/apple/scrd.rs @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! SEP credential endpoint (SCRD, EP 0x0a) wire protocol. + +#![allow(dead_code)] + +use kernel::prelude::*; +use kernel::soc::apple::mailbox::Message; +use crate::proto::*; +use crate::sks::SKS_AUTH_TOKEN_LEN; + +pub(crate) const SCRD_ACM_HANDLE_LEN: usize = SKS_AUTH_TOKEN_LEN; + +pub(crate) const SCRD_REQUEST_COMMAND: u8 = 1; + +const SCRD_MAGIC: [u8; 4] = *b"DRCS"; + +const SCRD_CMD_INITIALIZE: u8 = 0x0a; +const SCRD_CMD_CONTEXT_CREATE_TRACKED: u8 = 0x24; +const SCRD_CMD_CONTEXT_EXTERNALIZE: u8 = 0x13; +const SCRD_CMD_VERIFY_POLICY: u8 = 0x03; + +const SCRD_INIT_LOG_LEVEL: u8 = 0x28; + +const SCRD_POLICY_TOUCHID_ENROLLMENT: &[u8] = b"TouchIdEnrollment"; + +pub(crate) const SCRD_CONTEXT_CREATE_REPLY_LEN: usize = SCRD_ACM_HANDLE_LEN + 1 + 4; + +const SCRD_MAX_LOGICAL: usize = 64; +static_assert!( + 8 + SCRD_ACM_HANDLE_LEN + SCRD_POLICY_TOUCHID_ENROLLMENT.len() + 1 + 1 + 8 <= SCRD_MAX_LOGICAL +); + +pub(crate) struct ScrdCommand { + request: u8, + payload: [u8; SCRD_MAX_LOGICAL], + payload_len: usize, + name: &'static CStr, +} + +impl ScrdCommand { + pub(crate) fn request(&self) -> u8 { + self.request + } + pub(crate) fn payload(&self) -> &[u8] { + &self.payload[..self.payload_len] + } + pub(crate) fn name(&self) -> &'static CStr { + self.name + } +} + +fn scrd_header(out: &mut [u8; SCRD_MAX_LOGICAL], command: u8, byte5: u8, version: u8) { + out[0..4].copy_from_slice(&SCRD_MAGIC); + out[4] = command; + out[5] = byte5; + out[6] = 0; + out[7] = version; +} + +pub(crate) fn scrd_initialize() -> ScrdCommand { + let mut payload = [0u8; SCRD_MAX_LOGICAL]; + scrd_header(&mut payload, SCRD_CMD_INITIALIZE, SCRD_INIT_LOG_LEVEL, 0); + ScrdCommand { + request: SCRD_REQUEST_COMMAND, + payload, + payload_len: 8, + name: c"SCRD_INITIALIZE", + } +} + +pub(crate) fn scrd_context_create_tracked(session_uid: i32) -> ScrdCommand { + let mut payload = [0u8; SCRD_MAX_LOGICAL]; + scrd_header(&mut payload, SCRD_CMD_CONTEXT_CREATE_TRACKED, 0, 1); + payload[8..12].copy_from_slice(&session_uid.to_le_bytes()); + ScrdCommand { + request: SCRD_REQUEST_COMMAND, + payload, + payload_len: 12, + name: c"SCRD_CONTEXT_CREATE", + } +} + +pub(crate) fn scrd_context_externalize(handle: &[u8; SCRD_ACM_HANDLE_LEN]) -> ScrdCommand { + let mut payload = [0u8; SCRD_MAX_LOGICAL]; + scrd_header(&mut payload, SCRD_CMD_CONTEXT_EXTERNALIZE, 0, 1); + payload[8..8 + SCRD_ACM_HANDLE_LEN].copy_from_slice(handle); + ScrdCommand { + request: SCRD_REQUEST_COMMAND, + payload, + payload_len: 8 + SCRD_ACM_HANDLE_LEN, + name: c"SCRD_EXTERNALIZE", + } +} + +pub(crate) fn scrd_verify_touchid_enrollment(handle: &[u8; SCRD_ACM_HANDLE_LEN]) -> ScrdCommand { + let mut payload = [0u8; SCRD_MAX_LOGICAL]; + scrd_header(&mut payload, SCRD_CMD_VERIFY_POLICY, 0, 1); + let mut n = 8; + payload[n..n + SCRD_ACM_HANDLE_LEN].copy_from_slice(handle); + n += SCRD_ACM_HANDLE_LEN; + payload[n..n + SCRD_POLICY_TOUCHID_ENROLLMENT.len()] + .copy_from_slice(SCRD_POLICY_TOUCHID_ENROLLMENT); + n += SCRD_POLICY_TOUCHID_ENROLLMENT.len(); + // then NUL, preflight(1), u32 flags, u32 param count -- all zero + n += 1 + 1 + 4 + 4; + ScrdCommand { + request: SCRD_REQUEST_COMMAND, + payload, + payload_len: n, + name: c"SCRD_VERIFY_POLICY", + } +} + +pub(crate) struct ScrdReply { + pub(crate) request: u8, + pub(crate) response_size: u16, + pub(crate) status: i32, +} + +pub(crate) fn decode_scrd_reply(msg: &Message) -> ScrdReply { + let b = msg.msg0.to_le_bytes(); + ScrdReply { + request: b[1], + response_size: u16::from_le_bytes([b[2], b[3]]), + status: i32::from_le_bytes([b[4], b[5], b[6], b[7]]), + } +} + +pub(crate) fn encode_scrd(request: u8, len: usize) -> Message { + let len = (len & 0xffff) as u16; + let len = len.to_le_bytes(); + Message { + msg0: u64::from_le_bytes([EP_SCRD, request, len[0], len[1], 0, 0, 0, 0]), + msg1: 0, + } +} + +static_assert!(EP_SCRD == 0x0a); +static_assert!(SCRD_CMD_INITIALIZE == 0x0a); +static_assert!(SCRD_CMD_CONTEXT_CREATE_TRACKED == 0x24); +static_assert!(SCRD_CMD_CONTEXT_EXTERNALIZE == 0x13); +static_assert!(SCRD_CMD_VERIFY_POLICY == 0x03); diff --git a/drivers/soc/apple/seed.rs b/drivers/soc/apple/seed.rs new file mode 100644 index 0000000000000..d1793d8ebe361 --- /dev/null +++ b/drivers/soc/apple/seed.rs @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Importing the enclave's existing records. + +use crate::shim; +use crate::store::{Key, Store}; +use kernel::prelude::*; + +pub(crate) const SEED_PATH: &CStr = c"/var/lib/apple-sep-seed.bin"; + +const MAGIC: [u8; 4] = *b"AXRT"; +const HEADER_LEN: usize = 8; +const RECORD_HEADER_LEN: usize = 1 + 16 + 4; + +const MIN_RECORD_LEN: u32 = 1; +const MAX_RECORD_LEN: u32 = 0x8000; + +const MAX_SEED_BYTES: u64 = 1 << 20; + +const MAX_RECORDS: u32 = 64; + +const TYPE_ROOT_LOW: u8 = 1; +const TYPE_ROOT_HIGH: u8 = 2; +const TYPE_SESSION_HIGH: u8 = 4; + +pub(crate) struct Imported { + pub(crate) records: usize, + pub(crate) roots: usize, + pub(crate) sessions: usize, + pub(crate) bytes: usize, +} + +fn le32(buf: &[u8], off: usize) -> u32 { + u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]]) +} + +fn read_seed() -> Result> { + let file = shim::StoreFile::open_readonly(SEED_PATH)?; + let size = file.size()?; + if size < HEADER_LEN as u64 || size > MAX_SEED_BYTES { + return Err(EINVAL); + } + let mut buf = KVec::with_capacity(size as usize, GFP_KERNEL)?; + buf.resize(size as usize, 0, GFP_KERNEL)?; + file.read_exact(0, &mut buf)?; + Ok(buf) +} + +pub(crate) fn import(store: &mut Store) -> Result { + let buf = read_seed()?; + + if buf[..4] != MAGIC { + return Err(EINVAL); + } + let count = le32(&buf, 4); + if count == 0 || count > MAX_RECORDS { + return Err(EINVAL); + } + + let mut offsets: KVec<(u8, [u8; 16], usize, usize)> = KVec::new(); + let mut pos = HEADER_LEN; + for _ in 0..count { + if pos + RECORD_HEADER_LEN > buf.len() { + return Err(EINVAL); + } + let kind = buf[pos]; + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&buf[pos + 1..pos + 17]); + let len = le32(&buf, pos + 17); + pos += RECORD_HEADER_LEN; + + if !(TYPE_ROOT_LOW..=TYPE_SESSION_HIGH).contains(&kind) { + return Err(EINVAL); + } + if !(MIN_RECORD_LEN..=MAX_RECORD_LEN).contains(&len) { + return Err(EINVAL); + } + // Root records carry an all-zero UUID; anything else would be unreachable. + let is_root = kind == TYPE_ROOT_LOW || kind == TYPE_ROOT_HIGH; + if is_root && uuid != [0u8; 16] { + return Err(EINVAL); + } + let len = len as usize; + if pos + len > buf.len() { + return Err(EINVAL); + } + offsets.push((kind, uuid, pos, len), GFP_KERNEL)?; + pos += len; + } + + // Writes are keyed, so re-running after a crash simply replaces. + let mut result = Imported { + records: 0, + roots: 0, + sessions: 0, + bytes: 0, + }; + for (kind, uuid, at, len) in offsets { + store.write(&Key::new(kind, uuid), &buf[at..at + len])?; + result.records += 1; + result.bytes += len; + if kind == TYPE_ROOT_LOW || kind == TYPE_ROOT_HIGH { + result.roots += 1; + } else { + result.sessions += 1; + } + } + + // Mark consumed last: a crash above leaves the flag clear and the import reruns. + store.mark_seeded()?; + Ok(result) +} diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs new file mode 100644 index 0000000000000..44eeb912868c7 --- /dev/null +++ b/drivers/soc/apple/sensor.rs @@ -0,0 +1,705 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! The fingerprint sensor over SPI. + +use kernel::prelude::*; +use kernel::time::delay::fsleep; +use kernel::time::Delta; + +extern "C" { + fn sep_sensor_register() -> c_int; + fn sep_sensor_unregister(); + fn sep_sensor_bound() -> c_int; + fn sep_sensor_power_line() -> c_int; + fn sep_sensor_cs_timing_mode() -> c_int; + fn sep_sensor_power_cycle() -> c_int; + fn sep_sensor_power_source() -> c_int; + fn sep_sensor_power(on: c_int) -> c_int; + fn sep_sensor_xfer(tx: *const c_void, rx: *mut c_void, len: usize) -> c_int; + fn sep_sensor_xfer_tx(tx: *const c_void, len: usize) -> c_int; + fn sep_sensor_xfer2( + tx: *const c_void, + tx_len: usize, + rx: *mut c_void, + rx_len: usize, + ) -> c_int; +} + +// Spi2. +pub(crate) const CONTROLLER_BASE: u64 = 0x3_9b10_8000; +pub(crate) const CHIP_SELECT: u32 = 0; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum CsTiming { + Software, + HookBypassed, + Hardware, +} + +impl CsTiming { + fn from_wire(v: c_int) -> CsTiming { + match v { + 2 => CsTiming::Hardware, + 1 => CsTiming::HookBypassed, + _ => CsTiming::Software, + } + } + + pub(crate) fn name(self) -> &'static CStr { + match self { + CsTiming::Hardware => c"HARDWARE — programmed into the controller, which is what this sensor needs", + CsTiming::HookBypassed => { + c"emulated — the controller has a set_cs_timing hook but a GPIO chip select bypasses it" + } + CsTiming::Software => c"emulated in software — no set_cs_timing hook, and the sensor does not accept this", + } + } + + pub(crate) fn is_hardware(self) -> bool { + matches!(self, CsTiming::Hardware) + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum PowerSource { + None, + NodeProperty, + ChipLine, +} + +impl PowerSource { + fn from_wire(v: c_int) -> PowerSource { + match v { + 1 => PowerSource::NodeProperty, + 2 => PowerSource::ChipLine, + _ => PowerSource::None, + } + } + + pub(crate) fn name(self) -> &'static CStr { + match self { + PowerSource::None => c"none — no power line was taken", + PowerSource::NodeProperty => c"the sensor node's own gpios property", + PowerSource::ChipLine => c"/soc/pinctrl@39b028000 line 122, taken by device-tree node", + } + } +} + +pub(crate) const POWER_ON_READ_DELAYS_MS: [u32; 4] = [0, 3, 10, 50]; + +pub(crate) const STATUS_IDENTIFIER: usize = 12; +pub(crate) const EXPECTED_IDENTIFIER: u16 = 0x3352; +static_assert!(STATUS_IDENTIFIER + 2 <= STATUS_LEN); + +const CMD_LEN: usize = 7; +const CMD_GET_STATUS: [u8; CMD_LEN] = [0x80, 0x10, 0x00, 0x07, 0x00, 0x00, 0x00]; +const CMD_IDLE: [u8; CMD_LEN] = [0x80, 0x30, 0x00, 0x07, 0x00, 0x00, 0x00]; +const CMD_START_CAPTURE: [u8; CMD_LEN] = [0x80, 0x40, 0x00, 0x07, 0x00, 0x00, 0x00]; +// Precedes the patch blob. +const CMD_SETUP_PATCH_ENABLE: [u8; CMD_LEN] = [0x80, 0x60, 0x00, 0x07, 0x00, 0x00, 0x00]; +const CMD_GET_SENSOR_SERIAL: [u8; CMD_LEN] = [0x80, 0x70, 0x00, 0x07, 0x00, 0x00, 0x00]; + +const READ_CMD_LEN: usize = 11; +const CMD_READ_PREFIX: [u8; CMD_LEN] = [0x80, 0x13, 0x00, 0x0b, 0x00, 0x00, 0x00]; + +const STATUS_XFER_LEN: usize = 23; +const STATUS_AT: usize = 7; +pub(crate) const STATUS_LEN: usize = 16; +static_assert!(STATUS_AT + STATUS_LEN == STATUS_XFER_LEN); + +// Offsets index the 16-byte status, not the 23-byte transfer. +const STATUS_STATE: usize = STATUS_AT + 7; +const STATUS_COUNT: usize = STATUS_AT + 12; +const STATUS_STATE_TRANSFER_REL: usize = 7; +const STATUS_COUNT_TRANSFER_REL: usize = 12; + +static_assert!(STATUS_STATE == 14); +static_assert!(STATUS_COUNT == 19); +static_assert!(STATUS_COUNT + 4 == STATUS_XFER_LEN); +static_assert!(STATUS_STATE != STATUS_STATE_TRANSFER_REL); +static_assert!(STATUS_COUNT != STATUS_COUNT_TRANSFER_REL); +static_assert!(STATUS_STATE >= STATUS_AT && STATUS_STATE < STATUS_AT + STATUS_LEN); +static_assert!(STATUS_COUNT >= STATUS_AT && STATUS_COUNT + 4 <= STATUS_AT + STATUS_LEN); + +pub(crate) const STATE_DATA_READY: u8 = 7; +pub(crate) const STATE_READING: u8 = 19; +pub(crate) const STATE_IDLE: u8 = 0; +pub(crate) const STATE_ARMED: u8 = 17; +static_assert!(STATE_ARMED != STATE_READING); +static_assert!(STATE_ARMED != STATE_DATA_READY); +static_assert!(STATE_ARMED != STATE_IDLE); +pub(crate) const STATE_NEEDS_PATCH: u8 = 9; + +// status[8], not the state byte status[7]. +pub(crate) const STATUS_PATCH_ACK: usize = 8; +pub(crate) const PATCH_ACCEPTED: u8 = 0x29; +static_assert!(STATUS_PATCH_ACK != 7); +static_assert!(STATUS_PATCH_ACK < STATUS_LEN); + +pub(crate) const MAX_CAPTURE: usize = 0x10000; + +const CRC_LEN: usize = 2; + +const FRAME_HEADER: usize = 7; +const FRAME_LEN_AT: usize = 3; +static_assert!(FRAME_LEN_AT + 2 <= FRAME_HEADER); + +const TYPE_SESSION: u8 = 0x72; +const TYPE_SEQUENCE: u8 = 0x73; +const TYPE_ENCRYPTED: u8 = 0x55; + +pub(crate) const SESSION_SHARE_LEN: usize = 40; +pub(crate) const CHALLENGE_LEN: usize = 64; + +const MAX_FRAME_PAYLOAD: usize = CHALLENGE_LEN; +static_assert!(SESSION_SHARE_LEN <= MAX_FRAME_PAYLOAD); + +pub(crate) const SESSION_REPLY_LEN: usize = 0x33; +pub(crate) const SESSION_REPLY_AT: usize = 9; +pub(crate) const CHALLENGE_REPLY_LEN: usize = 0x4b; +pub(crate) const CHALLENGE_REPLY_AT: usize = 9; + +static_assert!(SESSION_REPLY_AT + SESSION_SHARE_LEN <= SESSION_REPLY_LEN); +static_assert!(CHALLENGE_REPLY_AT + CHALLENGE_LEN <= CHALLENGE_REPLY_LEN); + +fn check(rc: c_int) -> Result<()> { + if rc < 0 { + Err(Error::from_errno(rc)) + } else { + Ok(()) + } +} + +pub(crate) fn register_driver() -> Result<()> { + // SAFETY: takes no arguments and is idempotent. + check(unsafe { sep_sensor_register() }) +} + +pub(crate) fn unregister_driver() { + // SAFETY: idempotent, and safe when nothing was ever registered. + unsafe { sep_sensor_unregister() } +} + +pub(crate) fn is_bound() -> bool { + // SAFETY: reads one pointer for nullness. + unsafe { sep_sensor_bound() != 0 } +} + +pub(crate) fn power_line() -> Option { + // SAFETY: no arguments; returns a negative errno when unbound. + let n = unsafe { sep_sensor_power_line() }; + if n < 0 { + None + } else { + Some(n) + } +} + +pub(crate) fn power_cycle() -> bool { + // SAFETY: the shim holds the descriptor and applies its delays. + unsafe { sep_sensor_power_cycle() == 0 } +} + +pub(crate) fn cs_timing() -> CsTiming { + // SAFETY: reads one int. + CsTiming::from_wire(unsafe { sep_sensor_cs_timing_mode() }) +} + +pub(crate) fn power_source() -> PowerSource { + // SAFETY: reads one int. + PowerSource::from_wire(unsafe { sep_sensor_power_source() }) +} + +pub(crate) fn power(on: bool) -> bool { + // SAFETY: the shim holds the descriptor and applies its delays. + unsafe { sep_sensor_power(if on { 1 } else { 0 }) == 0 } +} + +fn command(cmd: &[u8; CMD_LEN]) -> Result<()> { + let mut rx = [0u8; CMD_LEN]; + // SAFETY: both buffers are `CMD_LEN` bytes and live across the call. + check(unsafe { sep_sensor_xfer(cmd.as_ptr().cast(), rx.as_mut_ptr().cast(), CMD_LEN) }) +} + +pub(crate) fn idle() -> Result<()> { + command(&CMD_IDLE) +} + +pub(crate) fn start_capture() -> Result<()> { + command(&CMD_START_CAPTURE) +} + +pub(crate) fn setup_patch_enable() -> Result<()> { + command(&CMD_SETUP_PATCH_ENABLE) +} + +fn send_framed(frame_type: u8, payload: &[u8]) -> Result<()> { + let total = FRAME_HEADER + payload.len(); + if payload.len() > MAX_FRAME_PAYLOAD || total > u16::MAX as usize { + return Err(EINVAL); + } + + let mut frame = [0u8; FRAME_HEADER + MAX_FRAME_PAYLOAD]; + frame[0] = 0x80; + frame[1] = frame_type; + frame[FRAME_LEN_AT..FRAME_LEN_AT + 2].copy_from_slice(&(total as u16).to_le_bytes()); + frame[FRAME_HEADER..total].copy_from_slice(payload); + + // SAFETY: `frame` is at least `total` bytes and lives across the call; the + // shim only reads from it and there is no receive buffer. + check(unsafe { sep_sensor_xfer_tx(frame.as_ptr().cast(), total) }) +} + +const ENCRYPTED_DECLARED_MAX: usize = 0x12c; +const ENCRYPTED_OVERHEAD: usize = FRAME_HEADER + CRC_LEN; +static_assert!(ENCRYPTED_OVERHEAD == 9); +const ENCRYPTED_TRANSFER_MAX: usize = 0x12b; +static_assert!(ENCRYPTED_TRANSFER_MAX < ENCRYPTED_DECLARED_MAX); + +pub(crate) struct Geometry { + declared: usize, + transfer: usize, +} + +impl Geometry { + // 0x5d. + pub(crate) const COVERAGE: Geometry = Geometry { + declared: 0x40, + transfer: 0x40, + }; + // 0x5c. + pub(crate) const OPERATION: Geometry = Geometry { + declared: 0x40, + transfer: 0x40, + }; + pub(crate) const MODULE_CHALLENGE: Geometry = Geometry { + declared: 0x49, + transfer: 0x49, + }; + + // 0x6a, transmitted zero-padded past declared. + pub(crate) const TRANSPARENT: Geometry = Geometry { + declared: 0x4d, + transfer: 0x12b, + }; + + pub(crate) const fn payload_capacity(&self) -> usize { + self.declared - ENCRYPTED_OVERHEAD + } + + pub(crate) fn declared(&self) -> usize { + self.declared + } +} + +static_assert!(Geometry::MODULE_CHALLENGE.declared < ENCRYPTED_DECLARED_MAX); +static_assert!(Geometry::MODULE_CHALLENGE.declared > ENCRYPTED_OVERHEAD); +static_assert!(Geometry::MODULE_CHALLENGE.transfer >= Geometry::MODULE_CHALLENGE.declared); +static_assert!(Geometry::MODULE_CHALLENGE.transfer <= ENCRYPTED_TRANSFER_MAX); +static_assert!(Geometry::MODULE_CHALLENGE.payload_capacity() == 0x40); +static_assert!(Geometry::COVERAGE.declared < ENCRYPTED_DECLARED_MAX); +static_assert!(Geometry::OPERATION.declared < ENCRYPTED_DECLARED_MAX); +static_assert!(Geometry::TRANSPARENT.declared < ENCRYPTED_DECLARED_MAX); +static_assert!(Geometry::COVERAGE.declared > ENCRYPTED_OVERHEAD); +static_assert!(Geometry::OPERATION.declared > ENCRYPTED_OVERHEAD); +static_assert!(Geometry::TRANSPARENT.declared > ENCRYPTED_OVERHEAD); +static_assert!(Geometry::COVERAGE.transfer >= Geometry::COVERAGE.declared); +static_assert!(Geometry::OPERATION.transfer >= Geometry::OPERATION.declared); +static_assert!(Geometry::TRANSPARENT.transfer >= Geometry::TRANSPARENT.declared); +static_assert!(Geometry::COVERAGE.transfer <= ENCRYPTED_TRANSFER_MAX); +static_assert!(Geometry::OPERATION.transfer <= ENCRYPTED_TRANSFER_MAX); +static_assert!(Geometry::TRANSPARENT.transfer <= ENCRYPTED_TRANSFER_MAX); + +fn crc16_ansi(data: &[u8]) -> u16 { + let mut crc: u16 = 0; + for &byte in data { + crc ^= u16::from(byte); + for _ in 0..8 { + if crc & 1 != 0 { + crc = (crc >> 1) ^ 0xA001; + } else { + crc >>= 1; + } + } + } + crc +} + +pub(crate) struct Params(KVec); + +impl Params { + pub(crate) fn new(blob: KVec) -> Params { + Params(blob) + } + + pub(crate) fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub(crate) fn len(&self) -> usize { + self.0.len() + } +} + +impl Drop for Params { + fn drop(&mut self) { + for b in self.0.iter_mut() { + // SAFETY: a valid, uniquely borrowed byte. + unsafe { core::ptr::write_volatile(b, 0) }; + } + } +} + +pub(crate) enum ParamsError { + Empty, + TooLong(usize, usize), + Transfer(Error), +} + +pub(crate) fn send_encrypted_parameters( + params: &Params, + geom: &Geometry, +) -> core::result::Result<(), ParamsError> { + if params.0.is_empty() { + return Err(ParamsError::Empty); + } + let capacity = geom.payload_capacity(); + if params.0.len() > capacity { + return Err(ParamsError::TooLong(params.0.len(), capacity)); + } + + let declared = geom.declared; + let transfer = geom.transfer; + let mut frame = [0u8; ENCRYPTED_TRANSFER_MAX]; + + frame[0] = 0x80; + frame[1] = TYPE_ENCRYPTED; + // Declared size, not the transferred count; the tail past it is zero padding. + frame[FRAME_LEN_AT..FRAME_LEN_AT + 2].copy_from_slice(&(declared as u16).to_le_bytes()); + frame[FRAME_HEADER..FRAME_HEADER + params.0.len()].copy_from_slice(¶ms.0); + + let crc_at = declared - CRC_LEN; + let crc = crc16_ansi(&frame[..crc_at]); + frame[crc_at..declared].copy_from_slice(&crc.to_le_bytes()); + + // SAFETY: `frame` is `ENCRYPTED_TRANSFER_MAX` bytes, `transfer` is no + // larger (asserted for every `Geometry`), and it lives across the call; the + // shim only reads from it and there is no receive buffer. + check(unsafe { sep_sensor_xfer_tx(frame.as_ptr().cast(), transfer) }) + .map_err(ParamsError::Transfer) +} + +pub(crate) const MODULE_REPLY_LENS: [usize; 2] = [0x4b, 0x4f]; +static_assert!(MODULE_REPLY_LENS[0] <= MAX_REPLY); +static_assert!(MODULE_REPLY_LENS[1] <= MAX_REPLY); +static_assert!(MODULE_REPLY_LENS[0] != MODULE_REPLY_LENS[1]); + +const MODULE_REPLY_ADJUST: usize = 11; +const MODULE_REPLY_AT: usize = 9; +static_assert!(MODULE_REPLY_AT == SESSION_REPLY_AT); +static_assert!(MODULE_REPLY_AT == CHALLENGE_REPLY_AT); +static_assert!(MODULE_REPLY_AT == SENSOR_SERIAL_AT); +static_assert!(MODULE_REPLY_ADJUST == MODULE_REPLY_AT + CRC_LEN); +static_assert!(MODULE_REPLY_LENS[0] > MODULE_REPLY_ADJUST); + +pub(crate) fn send_module_challenge(challenge: &Params) -> core::result::Result<(), ParamsError> { + send_encrypted_parameters(challenge, &Geometry::MODULE_CHALLENGE) +} + +pub(crate) fn read_module_reply() -> Result> { + let count = poll_until_ready_any(&MODULE_REPLY_LENS)?; + + let mut buf = [0xffu8; MAX_REPLY]; + read_reply_ready(count, &mut buf)?; + + // Relayed unparsed to the enclave, op 0x32. + let len = count - MODULE_REPLY_ADJUST; + let mut out = KVec::new(); + out.extend_from_slice(&buf[MODULE_REPLY_AT..MODULE_REPLY_AT + len], GFP_KERNEL)?; + Ok(out) +} + +fn poll_until_ready_any(accepted: &[usize]) -> Result { + for _ in 0..READ_POLL_ATTEMPTS { + let st = status()?; + if let Offset12::Available(count) = st.offset12() { + let count = count as usize; + if accepted.contains(&count) { + return Ok(count); + } + } + fsleep(Delta::from_millis(READ_POLL_MS)); + } + Err(ETIMEDOUT) +} + +pub(crate) fn send_session_share(share: &[u8; SESSION_SHARE_LEN]) -> Result<()> { + send_framed(TYPE_SESSION, share) +} + +pub(crate) fn send_challenge(challenge: &[u8; CHALLENGE_LEN]) -> Result<()> { + send_framed(TYPE_SEQUENCE, challenge) +} + +static_assert!(FRAME_HEADER + SESSION_SHARE_LEN == 0x2f); +static_assert!(FRAME_HEADER + CHALLENGE_LEN == 0x47); + +fn read_framed(total: usize, at: usize, out: &mut [u8]) -> Result<()> { + let mut buf = [0xffu8; MAX_REPLY]; + read_reply(total, &mut buf)?; + if at + out.len() > total { + return Err(EINVAL); + } + out.copy_from_slice(&buf[at..at + out.len()]); + Ok(()) +} + +fn poll_until_ready(expected: usize) -> Result<()> { + for _ in 0..READ_POLL_ATTEMPTS { + let st = status()?; + if let Offset12::Available(count) = st.offset12() { + if count as usize == expected { + return Ok(()); + } + } + fsleep(Delta::from_millis(READ_POLL_MS)); + } + Err(ETIMEDOUT) +} + +const READ_POLL_MS: i64 = 5; +const READ_POLL_ATTEMPTS: usize = 200; + +fn read_reply(total: usize, buf: &mut [u8; MAX_REPLY]) -> Result<()> { + poll_until_ready(total)?; + read_reply_ready(total, buf) +} + +fn read_reply_ready(total: usize, buf: &mut [u8; MAX_REPLY]) -> Result<()> { + if total > MAX_REPLY { + return Err(EINVAL); + } + + let mut cmd = [0u8; READ_CMD_LEN]; + cmd[..CMD_LEN].copy_from_slice(&CMD_READ_PREFIX); + cmd[CMD_LEN..].copy_from_slice(&(total as u32).to_le_bytes()); + + // SAFETY: `cmd` is `READ_CMD_LEN` bytes, `buf` is `MAX_REPLY` and `total` + // is bounded by it; both live across the call and the shim writes only into + // the receive buffer. + check(unsafe { + sep_sensor_xfer2( + cmd.as_ptr().cast(), + READ_CMD_LEN, + buf.as_mut_ptr().cast(), + total, + ) + }) +} + +fn read_framed_verified(total: usize, at: usize, out: &mut [u8]) -> Result<()> { + if total < CRC_LEN || at + out.len() > total - CRC_LEN { + return Err(EINVAL); + } + let mut buf = [0xffu8; MAX_REPLY]; + read_reply(total, &mut buf)?; + + let split = total - CRC_LEN; + let expected = u16::from_le_bytes([buf[split], buf[split + 1]]); + let computed = crc16_ansi(&buf[..split]); + if expected != computed { + return Err(EIO); + } + + out.copy_from_slice(&buf[at..at + out.len()]); + Ok(()) +} + +const MAX_REPLY: usize = 0x4f; +static_assert!(SESSION_REPLY_LEN <= MAX_REPLY); +static_assert!(CHALLENGE_REPLY_LEN <= MAX_REPLY); +static_assert!(SENSOR_SERIAL_REPLY_LEN <= MAX_REPLY); + +pub(crate) const SENSOR_SERIAL_REPLY_LEN: usize = 0x1b; +pub(crate) const SENSOR_SERIAL_LEN: usize = 16; +pub(crate) const SENSOR_SERIAL_AT: usize = 9; +static_assert!(SENSOR_SERIAL_AT == SESSION_REPLY_AT); +static_assert!(SENSOR_SERIAL_AT + SENSOR_SERIAL_LEN <= SENSOR_SERIAL_REPLY_LEN - CRC_LEN); + +pub(crate) struct SensorSerial([u8; SENSOR_SERIAL_LEN]); + +impl SensorSerial { + pub(crate) fn bytes(&self) -> &[u8; SENSOR_SERIAL_LEN] { + &self.0 + } +} + +pub(crate) fn read_sensor_serial() -> Result { + command(&CMD_GET_SENSOR_SERIAL)?; + let mut serial = [0u8; SENSOR_SERIAL_LEN]; + read_framed_verified(SENSOR_SERIAL_REPLY_LEN, SENSOR_SERIAL_AT, &mut serial)?; + Ok(SensorSerial(serial)) +} + +pub(crate) fn read_session_reply() -> Result<[u8; SESSION_SHARE_LEN]> { + let mut share = [0u8; SESSION_SHARE_LEN]; + read_framed(SESSION_REPLY_LEN, SESSION_REPLY_AT, &mut share)?; + Ok(share) +} + +pub(crate) fn read_challenge_reply() -> Result<[u8; CHALLENGE_LEN]> { + let mut reply = [0u8; CHALLENGE_LEN]; + read_framed(CHALLENGE_REPLY_LEN, CHALLENGE_REPLY_AT, &mut reply)?; + Ok(reply) +} + +pub(crate) fn send_patch(blob: &[u8]) -> Result<()> { + if blob.is_empty() { + return Err(EINVAL); + } + // SAFETY: `blob` is a live slice for the duration of the call and the shim + // only reads from it; there is no receive buffer to write into. + check(unsafe { sep_sensor_xfer_tx(blob.as_ptr().cast(), blob.len()) }) +} + +pub(crate) struct Status { + pub(crate) state: u8, + offset12: u32, + pub(crate) raw: [u8; STATUS_LEN], +} + +pub(crate) enum Offset12 { + Identifier(u16), + Available(u32), + Undefined, +} + +pub(crate) struct Identifier(u16); + +impl Identifier { + pub(crate) fn value(&self) -> u16 { + self.0 + } +} + +impl Status { + pub(crate) fn offset12(&self) -> Offset12 { + match self.state { + STATE_IDLE => Offset12::Identifier(u16::from_le_bytes([ + self.raw[STATUS_IDENTIFIER], + self.raw[STATUS_IDENTIFIER + 1], + ])), + STATE_DATA_READY => Offset12::Available(self.offset12), + _ => Offset12::Undefined, + } + } + + pub(crate) fn identifier_to_register(&self) -> Option { + match self.offset12() { + Offset12::Identifier(0) => None, + Offset12::Identifier(value) => Some(Identifier(value)), + Offset12::Available(_) | Offset12::Undefined => None, + } + } + + pub(crate) fn patch_ack(&self) -> u8 { + self.raw[STATUS_PATCH_ACK] + } + + pub(crate) fn is_silent(&self) -> bool { + self.raw.iter().all(|b| *b == 0) + } +} + +pub(crate) fn status() -> Result { + let mut tx = [0xffu8; STATUS_XFER_LEN]; + tx[..CMD_LEN].copy_from_slice(&CMD_GET_STATUS); + let mut rx = [0u8; STATUS_XFER_LEN]; + + // SAFETY: both buffers are `STATUS_XFER_LEN` bytes and live across the call. + check(unsafe { + sep_sensor_xfer(tx.as_ptr().cast(), rx.as_mut_ptr().cast(), STATUS_XFER_LEN) + })?; + + let mut raw = [0u8; STATUS_LEN]; + raw.copy_from_slice(&rx[STATUS_AT..STATUS_AT + STATUS_LEN]); + + Ok(Status { + state: rx[STATUS_STATE], + offset12: u32::from_le_bytes([ + rx[STATUS_COUNT], + rx[STATUS_COUNT + 1], + rx[STATUS_COUNT + 2], + rx[STATUS_COUNT + 3], + ]), + raw, + }) +} + +pub(crate) struct Capture(KVec); + +impl Capture { + pub(crate) fn bytes(&self) -> &[u8] { + &self.0 + } + +} + +impl Drop for Capture { + fn drop(&mut self) { + for b in self.0.iter_mut() { + // SAFETY: a valid, uniquely borrowed byte. + unsafe { core::ptr::write_volatile(b, 0) }; + } + } +} + +pub(crate) enum CaptureError { + Length(u32), + Bus(Error), + NoMemory, + Checksum { advertised: u16, computed: u16 }, +} + +pub(crate) fn read_capture(length: u32) -> core::result::Result { + let len = length as usize; + if length == 0 || len > MAX_CAPTURE || len <= CRC_LEN { + return Err(CaptureError::Length(length)); + } + + let mut cmd = [0u8; READ_CMD_LEN]; + cmd[..CMD_LEN].copy_from_slice(&CMD_READ_PREFIX); + cmd[CMD_LEN..].copy_from_slice(&length.to_le_bytes()); + + let mut buf = KVec::with_capacity(len, GFP_KERNEL).map_err(|_| CaptureError::NoMemory)?; + buf.resize(len, 0xff, GFP_KERNEL) + .map_err(|_| CaptureError::NoMemory)?; + + // SAFETY: `cmd` is `READ_CMD_LEN` bytes and `buf` is `len` bytes; both live + // across the call, and the shim writes only into the receive buffer. + let rc = unsafe { + sep_sensor_xfer2( + cmd.as_ptr().cast(), + READ_CMD_LEN, + buf.as_mut_ptr().cast(), + len, + ) + }; + if rc < 0 { + return Err(CaptureError::Bus(Error::from_errno(rc))); + } + + let split = len - CRC_LEN; + let advertised = u16::from_le_bytes([buf[split], buf[split + 1]]); + // CRC-16/ANSI, not CCITT-FALSE. + let computed = crc16_ansi(&buf[..split]); + if advertised != computed { + return Err(CaptureError::Checksum { + advertised, + computed, + }); + } + + Ok(Capture(buf)) +} diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c new file mode 100644 index 0000000000000..a182d6ac04817 --- /dev/null +++ b/drivers/soc/apple/sensor_shim.c @@ -0,0 +1,401 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ +/* Fingerprint sensor SPI shim: moves bytes over the bus and toggles power. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "shim.h" + +/* + * 8 MHz, SPI mode 2 (CPOL=1 CPHA=0), 8-bit words. Mode 3 returns sixteen zero + * bytes, else identical. Must agree with the node's spi-cpol / absent spi-cpha, + * which is what spi_setup() applies. + */ +#define SEP_SENSOR_HZ 8000000 +#define SEP_SENSOR_BITS 8 +static unsigned int sep_sensor_mode = SPI_MODE_2; +module_param_named(sensor_spi_mode, sep_sensor_mode, uint, 0444); +MODULE_PARM_DESC(sensor_spi_mode, + "SPI mode for the sensor: 2 (CPOL=1 CPHA=0, the default and the only mode this sensor answers under) or 3 (CPOL=1 CPHA=1, reads all-zero). For comparison only."); +#define SEP_SENSOR_MODE sep_sensor_mode + +/* Chip-select setup and hold, in ns. */ +#define SEP_SENSOR_CS_NS 20 + +#define SEP_SENSOR_OFF_DELAY_MS 10 +#define SEP_SENSOR_ON_DELAY_MS 7 + +static struct spi_device *sep_spi; +static struct gpio_desc *sep_power; +static bool sep_registered; + +/* + * Power line resolved by DT node path, not gpiochip index: registration order + * shifts, so an index would silently pick the wrong chip. + */ +#define SEP_SENSOR_GPIO_NODE "/soc/pinctrl@39b028000" +#define SEP_SENSOR_GPIO_LINE 122 + +#define SEP_POWER_NONE 0 +#define SEP_POWER_NODE_PROPERTY 1 +#define SEP_POWER_CHIP_LINE 2 + +static int sep_power_source; + +/* + * Takes the power line as an output driven low: the power cycle begins with an + * off phase, so the line must be actively driven off, not merely read as low. + */ +static void sep_acquire_power(struct spi_device *spi) +{ + struct device_node *np; + struct gpio_device *gdev; + struct gpio_chip *gc; + + sep_power = gpiod_get_index(&spi->dev, NULL, 0, GPIOD_OUT_LOW); + if (!IS_ERR(sep_power)) { + sep_power_source = SEP_POWER_NODE_PROPERTY; + dev_info(&spi->dev, + "sep sensor: power line from the device node, driven low\n"); + return; + } + sep_power = NULL; + + np = of_find_node_by_path(SEP_SENSOR_GPIO_NODE); + if (!np) { + dev_warn(&spi->dev, + "sep sensor: no DT node at %s, no power line\n", + SEP_SENSOR_GPIO_NODE); + return; + } + + gdev = gpio_device_find_by_fwnode(of_fwnode_handle(np)); + of_node_put(np); + if (!gdev) { + dev_warn(&spi->dev, + "sep sensor: %s has no registered GPIO device\n", + SEP_SENSOR_GPIO_NODE); + return; + } + + gc = gpio_device_get_chip(gdev); + if (gc) + sep_power = gpiochip_request_own_desc(gc, + SEP_SENSOR_GPIO_LINE, + "apple-mesa-power", + GPIO_LOOKUP_FLAGS_DEFAULT, + GPIOD_OUT_LOW); + if (!gc || IS_ERR(sep_power)) { + sep_power = NULL; + dev_warn(&spi->dev, + "sep sensor: could not take line %d on %s (%s)\n", + SEP_SENSOR_GPIO_LINE, SEP_SENSOR_GPIO_NODE, + gpio_device_get_label(gdev)); + gpio_device_put(gdev); + return; + } + + sep_power_source = SEP_POWER_CHIP_LINE; + dev_info(&spi->dev, + "sep sensor: power line %s line %d (%s), driven low\n", + SEP_SENSOR_GPIO_NODE, SEP_SENSOR_GPIO_LINE, + gpio_device_get_label(gdev)); + gpio_device_put(gdev); +} + +static void sep_release_power(void) +{ + if (!sep_power) + return; + gpiod_set_value_cansleep(sep_power, 0); + if (sep_power_source == SEP_POWER_CHIP_LINE) + gpiochip_free_own_desc(sep_power); + else + gpiod_put(sep_power); + sep_power = NULL; + sep_power_source = SEP_POWER_NONE; +} + +#define SEP_CS_TIMING_SOFTWARE 0 +#define SEP_CS_TIMING_HOOK_BYPASSED 1 +#define SEP_CS_TIMING_HARDWARE 2 + +static int sep_cs_timing_mode; + +/* + * Programs the 20 ns chip-select setup/hold and applies mode/speed via + * spi_setup(), which reaches the controller's set_cs_timing hook. + * + * This sensor needs hardware timing. The hook runs only for a native chip + * select; with a GPIO chip select the core silently emulates the delays in + * software, which this sensor rejects but which looks like success. + */ +static int sep_apply_cs_timing(struct spi_device *spi) +{ + struct spi_controller *ctlr = spi->controller; + int rc; + + if (!ctlr->set_cs_timing) + sep_cs_timing_mode = SEP_CS_TIMING_SOFTWARE; + else if (spi_get_csgpiod(spi, 0)) + sep_cs_timing_mode = SEP_CS_TIMING_HOOK_BYPASSED; + else + sep_cs_timing_mode = SEP_CS_TIMING_HARDWARE; + + /* + * Hold the bus lock: these fields are read when a transfer asserts chip + * select. spi_setup() takes the controller io_mutex, not the bus lock, + * so no deadlock. + */ + rc = spi_bus_lock(ctlr); + if (rc) + return rc; + + spi->mode = SEP_SENSOR_MODE; + spi->bits_per_word = SEP_SENSOR_BITS; + spi->max_speed_hz = SEP_SENSOR_HZ; + spi->cs_setup.value = SEP_SENSOR_CS_NS; + spi->cs_setup.unit = SPI_DELAY_UNIT_NSECS; + spi->cs_hold.value = SEP_SENSOR_CS_NS; + spi->cs_hold.unit = SPI_DELAY_UNIT_NSECS; + + rc = spi_setup(spi); + + spi_bus_unlock(ctlr); + + if (rc) + return rc; + + switch (sep_cs_timing_mode) { + case SEP_CS_TIMING_HARDWARE: + dev_info(&spi->dev, + "sep sensor: CS timing %u ns programmed in hardware\n", + SEP_SENSOR_CS_NS); + break; + case SEP_CS_TIMING_HOOK_BYPASSED: + dev_warn(&spi->dev, + "sep sensor: GPIO chip select, CS timing emulated in software (sensor needs hardware timing)\n"); + break; + default: + dev_warn(&spi->dev, + "sep sensor: no set_cs_timing hook, %u ns emulated in software (sensor needs hardware timing)\n", + SEP_SENSOR_CS_NS); + break; + } + return 0; +} + +static int sep_sensor_probe(struct spi_device *spi) +{ + int rc; + + rc = sep_apply_cs_timing(spi); + if (rc) + return rc; + + sep_spi = spi; + sep_acquire_power(spi); + /* Read the mode back: a mode that did not take must not be logged as + * the one requested. */ + dev_info(&spi->dev, + "sep sensor: bound, %u Hz, mode %u (CPOL=%d CPHA=%d)\n", + spi->max_speed_hz, + (unsigned int)(spi->mode & (SPI_CPOL | SPI_CPHA)), + !!(spi->mode & SPI_CPOL), !!(spi->mode & SPI_CPHA)); + return 0; +} + +static void sep_sensor_remove(struct spi_device *spi) +{ + sep_release_power(); + sep_spi = NULL; +} + +/* + * Its own compatible, deliberately not spidev's: capture must stay in the + * kernel, and a spidev node would leak an image to userspace. No + * MODULE_DEVICE_TABLE, so udev cannot auto-load this module. + */ +static const struct of_device_id sep_sensor_of_match[] = { + { .compatible = "apple,mesa-fingerprint" }, + { } +}; + +/* + * Legacy ID table only to silence the SPI core's "no spi_device_id" note. No + * MODULE_DEVICE_TABLE: it quiets a message, it does not advertise a binding. + */ +static const struct spi_device_id sep_sensor_spi_ids[] = { + { "mesa-fingerprint", 0 }, + { } +}; + +static struct spi_driver sep_sensor_driver = { + .driver = { + .name = "apple-mesa", + .of_match_table = sep_sensor_of_match, + }, + .id_table = sep_sensor_spi_ids, + .probe = sep_sensor_probe, + .remove = sep_sensor_remove, +}; + +/* + * Registered before the device tree gains the sensor node, so the bind happens + * when the SPI core's notifier creates the device. + */ +int sep_sensor_register(void) +{ + int rc; + + if (sep_registered) + return 0; + rc = spi_register_driver(&sep_sensor_driver); + if (rc) + return rc; + sep_registered = true; + return 0; +} + +void sep_sensor_unregister(void) +{ + if (!sep_registered) + return; + spi_unregister_driver(&sep_sensor_driver); + sep_registered = false; +} + +int sep_sensor_bound(void) +{ + return sep_spi != NULL; +} + +/* Whether those delays reach hardware, are emulated, or are silently bypassed. */ +int sep_sensor_cs_timing_mode(void) +{ + return sep_cs_timing_mode; +} + +/* Power cycle: off, 10 ms, on, 7 ms. -ENODEV if there is no power line. */ +int sep_sensor_power_cycle(void) +{ + if (!sep_power) + return -ENODEV; + + gpiod_set_value_cansleep(sep_power, 0); + msleep(SEP_SENSOR_OFF_DELAY_MS); + gpiod_set_value_cansleep(sep_power, 1); + msleep(SEP_SENSOR_ON_DELAY_MS); + return 0; +} + +/* How the power line was obtained: none, the node's property, or chip+line. */ +int sep_sensor_power_source(void) +{ + return sep_power_source; +} + +int sep_sensor_power_line(void) +{ + if (!sep_power) + return -ENODEV; + return desc_to_gpio(sep_power); +} + +/* Powers the sensor on/off, holding the hardware settling delay. */ +int sep_sensor_power(int on) +{ + if (!sep_power) + return -ENODEV; + + gpiod_set_value_cansleep(sep_power, on ? 1 : 0); + if (on) + msleep(SEP_SENSOR_ON_DELAY_MS); + else + msleep(SEP_SENSOR_OFF_DELAY_MS); + return 0; +} + +/* + * One chip-select assertion, `len` clocks, full duplex. The caller supplies all + * `len` transmit bytes (trailing ones as 0xff) rather than relying on the + * controller running its transmit buffer dry. + */ +int sep_sensor_xfer(const void *tx, void *rx, size_t len) +{ + struct spi_transfer xfer = { + .tx_buf = tx, + .rx_buf = rx, + .len = len, + .speed_hz = SEP_SENSOR_HZ, + .bits_per_word = SEP_SENSOR_BITS, + }; + + if (!sep_spi) + return -ENODEV; + if (!len) + return -EINVAL; + + return spi_sync_transfer(sep_spi, &xfer, 1); +} + +/* + * One chip-select assertion, transmit only, rx_buf NULL (not a discard buffer). + * A simultaneous receive changes the long transfer's pacing and the sensor + * rejects the patch blob, so the receive must be absent entirely. + */ +int sep_sensor_xfer_tx(const void *tx, size_t len) +{ + struct spi_transfer xfer = { + .tx_buf = tx, + .rx_buf = NULL, + .len = len, + .speed_hz = SEP_SENSOR_HZ, + .bits_per_word = SEP_SENSOR_BITS, + }; + + if (!sep_spi) + return -ENODEV; + if (!len) + return -EINVAL; + + return spi_sync_transfer(sep_spi, &xfer, 1); +} + +/* + * Two transfers in one chip-select assertion: command out, then read in. + * Chip-select stays asserted because neither transfer sets cs_change. + */ +int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len) +{ + struct spi_transfer xfers[2] = { + { + .tx_buf = tx, + .len = tx_len, + .speed_hz = SEP_SENSOR_HZ, + .bits_per_word = SEP_SENSOR_BITS, + }, + { + .rx_buf = rx, + .len = rx_len, + .speed_hz = SEP_SENSOR_HZ, + .bits_per_word = SEP_SENSOR_BITS, + }, + }; + + if (!sep_spi) + return -ENODEV; + if (!tx_len || !rx_len) + return -EINVAL; + + return spi_sync_transfer(sep_spi, xfers, 2); +} diff --git a/drivers/soc/apple/sep-bio.h b/drivers/soc/apple/sep-bio.h new file mode 100644 index 0000000000000..7ad0b814af648 --- /dev/null +++ b/drivers/soc/apple/sep-bio.h @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ +/* + * Userspace interface for the SEP biometric device. + * + * The enclave does the matching. Nothing biometric crosses this interface: only + * an operation, a stage, a status, an opaque identity UUID, and an opaque host + * label userspace chooses. + */ + +#pragma once + +#include +#include + +#define SEP_BIO_IFACE_VERSION 4 + +#define SEP_BIO_UUID_LEN 16 +#define SEP_BIO_LABEL_LEN 128 +#define SEP_BIO_NONCE_LEN 32 +#define SEP_BIO_TOKEN_LEN 32 +#define SEP_BIO_MAX_IDENTITIES 32 +#define SEP_BIO_CHALLENGE_LEN 32 +#define SEP_BIO_ATTEST_PUB_LEN 65 +#define SEP_BIO_ATTEST_SIG_MAX 72 + +enum { + SEP_BIO_STATE_IDLE = 0, + SEP_BIO_STATE_PENDING = 1, + SEP_BIO_STATE_PROGRESS = 2, + SEP_BIO_STATE_DONE = 3, + SEP_BIO_STATE_FAILED = 4, +}; + + +enum { + SEP_BIO_NO_MATCH = 0, + SEP_BIO_MATCH = 1, + SEP_BIO_NOT_COMPARED = 2, +}; + +struct sep_bio_identity { + __u8 uuid[SEP_BIO_UUID_LEN]; + __u8 label[SEP_BIO_LABEL_LEN]; +}; + +struct sep_bio_info { + __u32 version; + __u32 sensor_present; + __u32 enrolled; + __u32 capacity; + __u32 enroll_stages; + __u32 reserved[3]; +}; + +struct sep_bio_list { + __u32 count; + __u32 reserved; + struct sep_bio_identity id[SEP_BIO_MAX_IDENTITIES]; +}; + +struct sep_bio_enrol_start { + __u32 flags; + __u32 reserved; + __u8 label[SEP_BIO_LABEL_LEN]; +}; + +#define SEP_BIO_GUIDANCE_NONE 0 +#define SEP_BIO_GUIDANCE_PLACE 1 +#define SEP_BIO_GUIDANCE_LIFT_AND_MOVE 2 +#define SEP_BIO_GUIDANCE_HOLD_STILL 3 + +struct sep_bio_enrol_poll { + __u32 state; + __u32 stage; + __u32 stages_total; + __u32 status; + __u8 uuid[SEP_BIO_UUID_LEN]; + __u32 guidance; + __u32 progress_percent; +}; +struct sep_bio_verify_start { + __u32 flags; + __u32 reserved; + __u8 nonce[SEP_BIO_NONCE_LEN]; +}; + +struct sep_bio_verify_poll { + __u32 state; + __u32 result; + __u32 status; + __u32 reserved; + __u8 uuid[SEP_BIO_UUID_LEN]; + __u8 token[SEP_BIO_TOKEN_LEN]; /* single use, bound to the nonce */ + __u64 deadline_ns; /* CLOCK_MONOTONIC; past this the token is void */ +}; + +struct sep_bio_delete { + __u8 uuid[SEP_BIO_UUID_LEN]; +}; + +/* + * Device attestation of key possession: the enclave signs 'challenge' with the + * machine ref-key (ECDSA-P256 over the challenge as the pre-computed digest) and + * returns the DER signature and public point. The private key never leaves the + * enclave. + */ +struct sep_bio_attest { + __u32 sig_len; /* out: DER signature length */ + __u8 challenge[SEP_BIO_CHALLENGE_LEN]; /* in */ + __u8 public[SEP_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ + __u8 signature[SEP_BIO_ATTEST_SIG_MAX]; /* out: DER SEQUENCE{r,s} */ + __u8 reserved[3]; +}; + +#define SEP_BIO_IOC_MAGIC 0xB1 + +#define SEP_BIO_GET_INFO _IOR (SEP_BIO_IOC_MAGIC, 0x01, struct sep_bio_info) +#define SEP_BIO_LIST _IOR (SEP_BIO_IOC_MAGIC, 0x02, struct sep_bio_list) +#define SEP_BIO_ENROL_START _IOW (SEP_BIO_IOC_MAGIC, 0x03, struct sep_bio_enrol_start) +#define SEP_BIO_ENROL_POLL _IOR (SEP_BIO_IOC_MAGIC, 0x04, struct sep_bio_enrol_poll) +#define SEP_BIO_VERIFY_START _IOW (SEP_BIO_IOC_MAGIC, 0x05, struct sep_bio_verify_start) +#define SEP_BIO_VERIFY_POLL _IOR (SEP_BIO_IOC_MAGIC, 0x06, struct sep_bio_verify_poll) +#define SEP_BIO_CANCEL _IO (SEP_BIO_IOC_MAGIC, 0x07) +#define SEP_BIO_DELETE _IOW (SEP_BIO_IOC_MAGIC, 0x08, struct sep_bio_delete) +#define SEP_BIO_DELETE_ALL _IO (SEP_BIO_IOC_MAGIC, 0x09) +#define SEP_BIO_ATTEST _IOWR(SEP_BIO_IOC_MAGIC, 0x0a, struct sep_bio_attest) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index afa722240047b..ea8f95c5121cc 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -1,15 +1,42 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj +//! Driver for the Apple SEP (Secure Enclave Processor): attaches to a running +//! SEP over the AP mailbox, drives Touch ID enrol/verify via `/dev/sep-bio`, +//! and registers a hwrng and a SEP-backed `trusted` key source. #![recursion_limit = "2048"] -//! Apple SEP driver -//! -//! Copyright (C) The Asahi Linux Contributors +#[cfg(not(CONFIG_OF_DYNAMIC))] +compile_error!("apple_sep requires CONFIG_OF_DYNAMIC: the SEP and DART nodes ship disabled and are enabled with a device-tree changeset"); + +mod bio; +mod catacomb; +mod control; +mod der; +mod dt; +mod fv; +mod hwrng; +mod image; +mod keybag; +mod proto; +mod refkey; +mod refkey_seal; +mod rxring; +mod sbio; +mod scrd; +mod seed; +mod sensor; +mod shim; +mod shmem; +mod sks; +mod store; +mod transfer; +mod trusted; +mod xarm; use kernel::{ - bindings, device, dma, - module_platform_driver, + driver, new_mutex, of, platform, @@ -25,351 +52,2161 @@ use kernel::{ Atomic, Relaxed, // }, + new_condvar, Arc, + CondVar, + CondVarTimeoutResult, Mutex, // }, + time, types::ForeignOwnable, workqueue::{ self, + impl_has_delayed_work, impl_has_work, + new_delayed_work, new_work, + DelayedWork, Work, WorkItem, // }, // }; -const SHMEM_SIZE: usize = 0x30000; -const MSG_BOOT_TZ0: u64 = 0x5; -const MSG_BOOT_IMG4: u64 = 0x6; -const MSG_SET_SHMEM: u64 = 0x18; -const MSG_BOOT_TZ0_ACK1: u64 = 0x69; -const MSG_BOOT_TZ0_ACK2: u64 = 0xD2; -const MSG_BOOT_IMG4_ACK: u64 = 0x6A; -const MSG_ADVERTISE_EP: u64 = 0; -const EP_DISCOVER: u64 = 0xFD; -const EP_SHMEM: u64 = 0xFE; -const EP_BOOT: u64 = 0xFF; - -const MSG_TYPE_SHIFT: u32 = 16; -const MSG_TYPE_MASK: u64 = 0xFF; -//const MSG_PARAM_SHIFT: u32 = 24; -//const MSG_PARAM_MASK: u64 = 0xFF; - -const MSG_EP_MASK: u64 = 0xFF; -const MSG_DATA_SHIFT: u32 = 32; - -const IOVA_SHIFT: u32 = 0xC; - -type ShMem = dma::Coherent<[u8]>; - -fn align_up(v: usize, a: usize) -> usize { - (v + a - 1) & !(a - 1) -} - -fn memcpy_to_iomem(iomem: &mut ShMem, off: usize, src: &[u8]) -> Result<()> { - // SAFETY: - // as_slice_mut() checks that off and src.len() are whithin iomem's limits. - // memcpy_to_iomem is only called from within probe() ansuring there are no - // concurrent read and write accesses to the same region while the slice is - // alive per as_slice_mut()'s requiremnts. - unsafe { - let target = &mut iomem.as_mut()[off..off + src.len()]; - target.copy_from_slice(src); - } - Ok(()) -} - -fn build_shmem(dev: &platform::Device>) -> Result { - let fwnode = dev.as_ref().fwnode().ok_or(EIO)?; - let mut iomem = dma::Coherent::::zeroed_slice(dev.as_ref(), SHMEM_SIZE, GFP_KERNEL)?; - - let panic_offset = 0x4000; - let panic_size = 0x8000; - memcpy_to_iomem(&mut iomem, panic_offset, &1u32.to_le_bytes())?; - - let lpol_offset = panic_offset + panic_size; - let lpol_prop_name = c"local-policy-manifest"; - let lpol_prop_size = fwnode.property_count_elem::(lpol_prop_name)?; - let lpol = fwnode - .property_read_array_vec(lpol_prop_name, lpol_prop_size)? - .required_by(dev.as_ref())?; - memcpy_to_iomem( - &mut iomem, - lpol_offset, - &(lpol_prop_size as u32).to_le_bytes(), - )?; - memcpy_to_iomem(&mut iomem, lpol_offset + 4, &lpol)?; - let lpol_size = align_up(lpol_prop_size + 4, 0x4000); - - let ibot_offset = lpol_offset + lpol_size; - let ibot_prop_name = c"iboot-manifest"; - let ibot_prop_size = fwnode.property_count_elem::(ibot_prop_name)?; - let ibot = fwnode - .property_read_array_vec(ibot_prop_name, ibot_prop_size)? - .required_by(dev.as_ref())?; - memcpy_to_iomem( - &mut iomem, - ibot_offset, - &(ibot_prop_size as u32).to_le_bytes(), - )?; - memcpy_to_iomem(&mut iomem, ibot_offset + 4, &ibot)?; - let ibot_size = align_up(ibot_prop_size + 4, 0x4000); - - memcpy_to_iomem(&mut iomem, 0, b"CNIP")?; - memcpy_to_iomem(&mut iomem, 4, &(panic_size as u32).to_le_bytes())?; - memcpy_to_iomem(&mut iomem, 8, &(panic_offset as u32).to_le_bytes())?; - - memcpy_to_iomem(&mut iomem, 16, b"OPLA")?; - memcpy_to_iomem(&mut iomem, 16 + 4, &(lpol_size as u32).to_le_bytes())?; - memcpy_to_iomem(&mut iomem, 16 + 8, &(lpol_offset as u32).to_le_bytes())?; - - memcpy_to_iomem(&mut iomem, 32, b"IPIS")?; - memcpy_to_iomem(&mut iomem, 32 + 4, &(ibot_size as u32).to_le_bytes())?; - memcpy_to_iomem(&mut iomem, 32 + 8, &(ibot_offset as u32).to_le_bytes())?; - - memcpy_to_iomem(&mut iomem, 48, b"llun")?; - Ok(iomem) -} +const SETTLE_MS: time::Msecs = 200; +const FIRST_RESPONSE_MS: time::Msecs = 3000; -#[pin_data] -struct SepReceiveWork { - data: Arc, - msg: Message, - #[pin] - work: Work, +const SETTLE_WORK_ID: u64 = 1; + +const ENROL_WORK_ID: u64 = 2; +static_assert!(ENROL_WORK_ID != SETTLE_WORK_ID && ENROL_WORK_ID != 0); +const VERIFY_WORK_ID: u64 = 3; +static_assert!(VERIFY_WORK_ID != ENROL_WORK_ID && VERIFY_WORK_ID != SETTLE_WORK_ID); +static_assert!(VERIFY_WORK_ID != 0); + +const EXCHANGE_TIMEOUT_MS: time::Msecs = 15000; + +const PHASE_ATTACH: u32 = 0; +const PHASE_EXCHANGE: u32 = 1; +const PHASE_READY: u32 = 2; + +const ENDPOINTS_BEFORE_EXCHANGE: usize = 7; + +const OOL_SIZE_XARM: usize = 0x8000; + +const OOL_SIZE_XARS: usize = 0x8000; + +const OOL_SIZE_SBIO: usize = 0x4000; + +const OOL_SIZE_SCRD: usize = 0x4000; + +const SBIO_TIMEOUT_MS: time::Msecs = 5000; + +const SKS_ALLOC: usize = 0x8000; + + +const SKS_MAX_CAPTURE: usize = 8; + +const XARS_MAX_CAPTURE: usize = 8; + +const SCRD_MAX_CAPTURE: usize = 8; + +const SCRD_TIMEOUT_MS: time::Msecs = 2000; + +const SKS_TIMEOUT_MS: time::Msecs = 2000; + +const SKS_TIMEOUT_PER_KIB_MS: time::Msecs = 6000; + +const SKS_TIMEOUT_MAX_MS: time::Msecs = 30_000; + +const SKS_MAX_ABANDONED: usize = 8; + +const fn sks_timeout_for(image_len: usize) -> time::Msecs { + let kib = image_len / 1024; + let kib = if kib > u16::MAX as usize { + u16::MAX as time::Msecs + } else { + kib as time::Msecs + }; + let scaled = SKS_TIMEOUT_MS.saturating_add(SKS_TIMEOUT_PER_KIB_MS.saturating_mul(kib)); + if scaled > SKS_TIMEOUT_MAX_MS { + SKS_TIMEOUT_MAX_MS + } else { + scaled + } } -impl_has_work! { - impl HasWork for SepReceiveWork { self.work } +static_assert!(sks_timeout_for(64) == SKS_TIMEOUT_MS); +static_assert!(sks_timeout_for(1508) > SKS_TIMEOUT_MS); +static_assert!(sks_timeout_for(usize::MAX) == SKS_TIMEOUT_MAX_MS); + +const SKS_MAX_SET_ASIDE: u32 = 16; + +const DMA_RING_SIZE: usize = 4 * (1 << 12); + +const OOL_WRITE_POLL_MS: u32 = 1; +const OOL_WRITE_POLL_ATTEMPTS: u32 = 200; + +const OOL_POISON_INBOUND: u8 = 0xA5; +const OOL_POISON_OUTBOUND: u8 = 0x5A; + +const PROTECTED_DATA_AVAILABLE: bool = true; + +const RNG_MAX_WORDS_PER_READ: usize = 16; + +const SECMODE_UNKNOWN: u32 = u32::MAX; + +const MAX_ENDPOINTS: usize = 64; + + +#[derive(Clone, Copy)] +struct Endpoint { + id: u8, + fourcc: proto::Fourcc, + have_descriptor: bool, + have_config: bool, + descriptor_msg0: u64, + descriptor_msg1: u32, + config_msg0: u64, + config_msg1: u32, } -impl SepReceiveWork { - fn new(data: Arc, msg: Message) -> Result> { - Arc::pin_init( - pin_init!(SepReceiveWork { - data, - msg, - work <- new_work!("SepReceiveWork::work"), - }), - GFP_ATOMIC, - ) +impl Endpoint { + fn new(id: u8) -> Self { + Endpoint { + id, + fourcc: proto::Fourcc::ZERO, + have_descriptor: false, + have_config: false, + descriptor_msg0: 0, + descriptor_msg1: 0, + config_msg0: 0, + config_msg1: 0, + } } } -impl WorkItem for SepReceiveWork { - type Pointer = Arc; +struct OolPair { + endpoint: u8, + allocated: usize, + declared_in: usize, + declared_out: usize, + inbound: shmem::ShMem, + outbound: shmem::ShMem, + registered: bool, +} + +fn sks_declared_sizes() -> (usize, usize) { + (0x8000, 0x4000) +} + +const SKS_MALFORMED: i8 = crate::sks::SKS_STATUS_MALFORMED; + +struct Hex<'a>(&'a [u8]); - fn run(this: Arc) { - this.data.process_message(this.msg); +impl kernel::fmt::Display for Hex<'_> { + fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + use core::fmt::Write; + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + for (i, byte) in self.0.iter().enumerate() { + if i > 0 { + f.write_char(' ')?; + } + f.write_char(DIGITS[(*byte >> 4) as usize] as char)?; + f.write_char(DIGITS[(*byte & 0x0f) as usize] as char)?; + } + Ok(()) } } -struct FwRegionParams { - addr: u64, - size: usize, +const SKS_SECRET_LEN: usize = 32; +static_assert!(SKS_SECRET_LEN % 4 == 0); + +const SKS_LOCK_STATE_VARIANT: u32 = 1; + +#[derive(Clone, Copy)] +enum LockState { + Unlocked, + Locked, } -#[pin_data] -struct SepData { - dev: ARef, - #[pin] - mbox: Mutex>>, - shmem: ShMem, - region_params: FwRegionParams, - fw_mapped: Atomic, +impl LockState { + const fn wire(self) -> i32 { + match self { + LockState::Unlocked => 0, + LockState::Locked => 1, + } + } } -impl SepData { - fn new( - dev: &platform::Device>, - region_params: FwRegionParams, - ) -> Result> { - Arc::pin_init( - try_pin_init!(SepData { - shmem: build_shmem(dev)?, - dev: ARef::::from(dev.as_ref()), - mbox <- new_mutex!(None), - region_params, - fw_mapped: Atomic::new(false), - }), - GFP_KERNEL, - ) +static_assert!(LockState::Unlocked.wire() == 0); +static_assert!(LockState::Locked.wire() == 1); + +const SKS_LOCK_STATE_FLAGS: u64 = 0; + +enum SbioOutcome { + Ok(KVec), + // status 0x01 + PrerequisiteMissing, + // status 0x16; NOT "malformed" — that mapping is another service's status space + Status16, + Other, +} + +const BRINGUP_FRESH: u32 = 0; +const BRINGUP_IDENTIFIED: u32 = 1; +const BRINGUP_ESTABLISHED: u32 = 2; + +pub(crate) struct PatchLoaded(()); + +pub(crate) struct ParametersApplied(()); + +pub(crate) struct CalibrationBlob(kernel::firmware::Firmware); + +impl CalibrationBlob { + fn new(fw: kernel::firmware::Firmware) -> CalibrationBlob { + CalibrationBlob(fw) } - fn start(&self) -> Result<()> { - self.mbox.lock().as_ref().unwrap().send( - Message { - msg0: EP_BOOT | (MSG_BOOT_TZ0 << MSG_TYPE_SHIFT), - msg1: 0, - }, - false, - ) + + pub(crate) fn bytes(&self) -> &[u8] { + self.0.data() } - fn load_fw_and_shmem(&self) -> Result<()> { - let fw_addr = unsafe { - let res = bindings::dma_map_resource( - self.dev.as_raw(), - self.region_params.addr, - self.region_params.size, - bindings::dma_data_direction_DMA_TO_DEVICE, - 0, - ); - if bindings::dma_mapping_error(self.dev.as_raw(), res) != 0 { - dev_err!(self.dev, "Failed to map firmware"); - return Err(ENOMEM); - } - self.fw_mapped.store(true, Relaxed); - res >> IOVA_SHIFT - }; - let guard = self.mbox.lock(); - let mbox = guard.as_ref().unwrap(); - mbox.send( - Message { - msg0: EP_BOOT | (MSG_BOOT_IMG4 << MSG_TYPE_SHIFT) | (fw_addr << MSG_DATA_SHIFT), - msg1: 0, - }, - false, - )?; - let shm_addr = self.shmem.dma_handle() >> IOVA_SHIFT; - mbox.send( - Message { - msg0: EP_SHMEM | (MSG_SET_SHMEM << MSG_TYPE_SHIFT) | (shm_addr << MSG_DATA_SHIFT), - msg1: 0, - }, - false, - )?; - Ok(()) +} + +struct EnrolMaterial { + special: crate::sks::SpecialHandle, + secret: Secret, +} + +struct ImageContext<'a> { + sep: &'a SepData, +} + +impl Drop for ImageContext<'_> { + fn drop(&mut self) { + let op = crate::sbio::sbio_image_cleanup(); + let _ = self.sep.sbio_call(&op); } - fn process_boot_msg(&self, msg: Message) { - let ty = (msg.msg0 >> MSG_TYPE_SHIFT) & MSG_TYPE_MASK; - match ty { - MSG_BOOT_TZ0_ACK1 => {} - MSG_BOOT_TZ0_ACK2 => { - let res = self.load_fw_and_shmem(); - if let Err(e) = res { - dev_err!(self.dev, "Unable to load firmware: {:?}", e); - } - } - MSG_BOOT_IMG4_ACK => {} - _ => { - dev_err!(self.dev, "Unknown boot message type: {}", ty); - } +} + +struct OpenEnrolment<'a> { + sep: &'a SepData, + armed: bool, +} + +impl OpenEnrolment<'_> { + fn completed(mut self) { + self.armed = false; + self.sep.enrol_open.store(false, Relaxed); + } +} + +impl Drop for OpenEnrolment<'_> { + fn drop(&mut self) { + if !self.armed { + return; } + let _ = self.sep.sbio_call(&crate::sbio::sbio_cancel_operation()); + self.sep.enrol_open.store(false, Relaxed); } - fn process_discover_msg(&self, msg: Message) { - let ty = (msg.msg0 >> MSG_TYPE_SHIFT) & MSG_TYPE_MASK; - //let data = (msg.msg0 >> MSG_DATA_SHIFT) as u32; - //let param = (msg.msg0 >> MSG_PARAM_SHIFT) & MSG_PARAM_MASK; - match ty { - MSG_ADVERTISE_EP => { - /*dev_info!( - self.dev, - "Got endpoint {:?} at {}", - core::str::from_utf8(&data.to_be_bytes()), - param - );*/ - } - _ => { - //dev_warn!(self.dev, "Unknown discovery message type: {}", ty); - } +} + +enum ImageOutcome { + // `complete` flag from 0xbfe + Progress { + stage: u32, + percent: u32, + complete: bool, + has_template: bool, + }, + Retry, + NoFinger, + Failed(u32), +} + +enum CaptureWait { + Ready(u32), + Timeout, + Fault(u8), + Abandon, +} + +const ENROL_STATUS_SENSOR: u32 = 1; +const ENROL_STATUS_ENCLAVE: u32 = 2; +const ENROL_STATUS_TOO_MANY: u32 = 3; +const ENROL_STATUS_TIMEOUT: u32 = 4; +#[derive(Clone, Copy)] +enum LoadAnswer { + StatusZero, + ColdTransition, + // status 0x101: SEP re-activated the component from its own xART records + AlreadyActive, +} + +impl LoadAnswer { + fn describe(self) -> &'static CStr { + match self { + LoadAnswer::StatusZero => c"status 0", + LoadAnswer::ColdTransition => c"the cold-transition status 0x8002", + LoadAnswer::AlreadyActive => c"the already-active status 0x101", } } - fn process_message(&self, msg: Message) { - let ep = msg.msg0 & MSG_EP_MASK; - match ep { - EP_BOOT => self.process_boot_msg(msg), - EP_DISCOVER => self.process_discover_msg(msg), - _ => {} // dev_warn!(self.dev, "Message from unknown endpoint: {}", ep), +} + +enum RestoreOutcome { + Restored, + AlreadyActive, + NoStoredFile, + Failed, + // 0x8002 cold-transition; tolerated only for the lockout record, never a catacomb + EmptyTolerated, +} + +const PRIVATE_TYPE_CATACOMB_MASTER: u8 = 0xF2; +const PRIVATE_TYPE_CATACOMB_OWNER: u8 = 0xF3; +const PRIVATE_TYPE_CATACOMB_USER: u8 = 0xF4; +const PRIVATE_TYPE_LOCKOUT: u8 = 0xF5; +// 0xF1 is the identity index's type; a colliding type silently mis-restores +static_assert!(PRIVATE_TYPE_CATACOMB_MASTER != 0xF1); +static_assert!(PRIVATE_TYPE_CATACOMB_OWNER != PRIVATE_TYPE_CATACOMB_MASTER); +static_assert!(PRIVATE_TYPE_CATACOMB_USER != PRIVATE_TYPE_CATACOMB_OWNER); +static_assert!(PRIVATE_TYPE_CATACOMB_USER != PRIVATE_TYPE_CATACOMB_MASTER); +static_assert!(PRIVATE_TYPE_LOCKOUT != PRIVATE_TYPE_CATACOMB_USER); +static_assert!(PRIVATE_TYPE_LOCKOUT != PRIVATE_TYPE_CATACOMB_OWNER); +static_assert!(PRIVATE_TYPE_LOCKOUT != PRIVATE_TYPE_CATACOMB_MASTER); + +const ENROL_STATUS_UNFILED: u32 = 6; +static_assert!(ENROL_STATUS_UNFILED != ENROL_STATUS_RETRY); +static_assert!(ENROL_STATUS_UNFILED != ENROL_STATUS_ENCLAVE); +static_assert!(ENROL_STATUS_UNFILED != ENROL_STATUS_SENSOR); +static_assert!(ENROL_STATUS_UNFILED != ENROL_STATUS_TIMEOUT); +static_assert!(ENROL_STATUS_UNFILED != ENROL_STATUS_TOO_MANY); + +const ENROL_STATUS_RETRY: u32 = 5; +static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_TIMEOUT); +static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_ENCLAVE); +static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_SENSOR); + +const ENROL_MAX_CAPTURES: u32 = 12; + +const ENROL_POLL_MS: u32 = 2; + +const ENROL_CAPTURE_TIMEOUT_MS: u32 = 60_000; +const ENROL_POLL_ATTEMPTS: u32 = ENROL_CAPTURE_TIMEOUT_MS / ENROL_POLL_MS; +static_assert!(ENROL_POLL_ATTEMPTS * ENROL_POLL_MS == ENROL_CAPTURE_TIMEOUT_MS); + +const ENROL_REPOSITION_MS: u32 = 1800; + +const MATCH_SETTLE_MS: u32 = ENROL_REPOSITION_MS; + +const ENROL_IDLE_TIMEOUT_MS: u32 = 2000; + +const PATCH_POLL_MS: u32 = 20; +const PATCH_POLL_ATTEMPTS: u32 = 250; + +const SEAL_VECTOR: &[u8] = b"apple-sep seal round trip v1"; + +const SEAL_STATUS_SHAPE: i8 = SKS_MALFORMED; + +const SEAL_STATUS_NAMES_NOTHING: i8 = -11; + +const SEAL_STATUS_WRONG_KIND: i8 = -12; + +const SEAL_STATUS_MISSING_PREREQUISITE: i8 = -3; +static_assert!(SEAL_STATUS_MISSING_PREREQUISITE != SEAL_STATUS_WRONG_KIND); +static_assert!(SEAL_STATUS_MISSING_PREREQUISITE != SEAL_STATUS_NAMES_NOTHING); + +const SEAL_STATUS_BACKUP_WRAP: i8 = -14; + +static_assert!(SEAL_STATUS_WRONG_KIND != SEAL_STATUS_NAMES_NOTHING); +static_assert!(SEAL_STATUS_WRONG_KIND != SEAL_STATUS_SHAPE); +static_assert!(SEAL_STATUS_BACKUP_WRAP != SEAL_STATUS_WRONG_KIND); +static_assert!(SEAL_STATUS_BACKUP_WRAP != SEAL_STATUS_SHAPE); + +const SKS_WRAP_OVERHEAD: usize = 256; + +const SKS_WRAP_PRODUCTION_CAPACITY: u32 = SEAL_VECTOR.len() as u32 + SKS_WRAP_OVERHEAD as u32; + +const SKS_CAPACITY_MAP: [u32; 23] = [ + 128, 192, 255, 256, 257, 258, 300, 320, 384, 512, 513, 640, 768, 1023, 1024, 1025, 1026, 1280, + 1536, 2048, 2049, 3072, 4096, +]; + +const fn capacity_map_ascends() -> bool { + let mut i = 1; + while i < SKS_CAPACITY_MAP.len() { + if SKS_CAPACITY_MAP[i - 1] >= SKS_CAPACITY_MAP[i] { + return false; } + i += 1; } - fn remove(&self) { - self.mbox.lock().as_mut().set(None); - if self.fw_mapped.load(Relaxed) { - unsafe { - bindings::dma_unmap_resource( - self.dev.as_raw(), - self.region_params.addr, - self.region_params.size, - bindings::dma_data_direction_DMA_TO_DEVICE, - 0, - ); - } + true +} +static_assert!(capacity_map_ascends()); +static_assert!(SKS_CAPACITY_MAP[0] > 0); +static_assert!(SKS_CAPACITY_MAP[5] < SKS_WRAP_PRODUCTION_CAPACITY); +static_assert!(SKS_WRAP_PRODUCTION_CAPACITY < SKS_CAPACITY_MAP[6]); +static_assert!(SKS_CAPACITY_MAP[2] + 1 == SKS_CAPACITY_MAP[3]); +static_assert!(SKS_CAPACITY_MAP[13] + 1 == SKS_CAPACITY_MAP[14]); + +const CALIBRATION_FIRMWARE: &CStr = c"apple/mesa_calibration.bin"; + +const SKS_CONFIG_MASKS: [u32; 17] = [ + 0xffff_ffff, + 0x1, + 0x3, + 0x5, + 0x9, + 0x11, + 0x21, + 0x41, + 0x81, + 0x101, + 0x201, + 0x401, + 0x801, + 0x1001, + 0x2001, + 0x4001, + 0x8001, +]; + +const fn config_masks_are_single_bit_probes() -> bool { + if SKS_CONFIG_MASKS[0] != u32::MAX || SKS_CONFIG_MASKS[1] != 1 { + return false; + } + let mut i = 2; + while i < SKS_CONFIG_MASKS.len() { + let m = SKS_CONFIG_MASKS[i]; + if m & 1 != 1 || (m & !1u32).count_ones() != 1 { + return false; } + if i > 2 && SKS_CONFIG_MASKS[i - 1] >= m { + return false; + } + i += 1; } + true } +static_assert!(config_masks_are_single_bit_probes()); -impl MailCallback for SepData { - type Data = Arc; - fn recv_message(data: ::Borrowed<'_>, msg: Message) { - let work = SepReceiveWork::new(data.into(), msg); - if let Ok(work) = work { - let res = workqueue::system().enqueue(work); - if res.is_err() { - dev_err!( - data.dev, - "Unable to schedule work item for message {}", - msg.msg0 - ); - } +const CONFIG_STAGE_NONE: u32 = 0; +const SBIO_PROBE_USER_ID: i32 = 1000; +static_assert!(SBIO_PROBE_USER_ID >= crate::sks::SKS_DESIGNATE_USER_MIN); +static_assert!(SBIO_PROBE_USER_ID == bio::ENROL_USER_ID); +static_assert!(SBIO_PROBE_USER_ID > 0); +static_assert!(SBIO_PROBE_USER_ID == crate::sks::SKS_IDENTITY_USER_ID); + +const SKS_LOAD_REPLY_LEN: usize = 8; + +// StoreType: the u32 at +0x60 of the create body +#[derive(Clone, Copy, PartialEq, Eq)] +struct StoreType(u32); + +const STORE_TYPE_MAX: u32 = 8; + +impl StoreType { + const IDENTITY: StoreType = StoreType(0); + const SEALING: StoreType = StoreType(1); + + const fn new(value: u32) -> Option { + if value <= STORE_TYPE_MAX { + Some(StoreType(value)) } else { - dev_err!( - data.dev, - "Unable to allocate work item for message {}", - msg.msg0 - ); + None } } + + const fn wire(self) -> u32 { + self.0 + } } -unsafe impl Send for SepData {} -unsafe impl Sync for SepData {} +static_assert!(StoreType::IDENTITY.wire() == 0); +static_assert!(StoreType::SEALING.wire() == 1); +static_assert!(StoreType::new(STORE_TYPE_MAX).is_some()); +static_assert!(StoreType::new(STORE_TYPE_MAX + 1).is_none()); -struct SepDriver(Arc); +#[derive(Clone, Copy)] -kernel::of_device_table!( - OF_TABLE, - MODULE_OF_TABLE, - (), - [(of::DeviceId::new(c"apple,sep"), ())] -); +#[must_use] +struct Healthy(()); -impl platform::Driver for SepDriver { - type IdInfo = (); - type Data<'bound> = SepDriver; +struct SksRequest { + name: &'static CStr, + msg: Message, + img: image::RequestImage, +} - const OF_ID_TABLE: Option> = Some(&OF_TABLE); +struct SksOutcome { + reply: crate::sks::SksReply, + response: Secret, +} - fn probe<'bound>( - pdev: &'bound platform::Device>, - _info: Option<&'bound Self::IdInfo>, - ) -> impl PinInit, Error> + 'bound { - let of = pdev.as_ref().of_node().ok_or(EIO)?; - let res = of.reserved_mem_region_to_resource_byname(c"sepfw")?; - let data = SepData::new( - pdev, - FwRegionParams { - addr: res.start(), - size: res.size().try_into()?, - }, - )?; - data.mbox.lock().as_mut().set(Some(Mailbox::new_byname( - pdev.as_ref(), - c"mbox", - data.clone(), - )?)); - data.start()?; - Ok(Self(data)) +pub(crate) struct Secret(pub(crate) KVec); + +impl Secret { + pub(crate) fn empty() -> Secret { + Secret(KVec::new()) } } -impl Drop for SepDriver { +impl core::ops::Deref for Secret { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + &self.0 + } +} + +impl Drop for Secret { fn drop(&mut self) { - self.0.remove(); + for b in self.0.iter_mut() { + // SAFETY: `b` is a valid, uniquely borrowed byte for this write, and + // a volatile store is what stops the compiler discarding a wipe of + // memory nothing reads afterwards. + unsafe { core::ptr::write_volatile(b, 0) }; + } + } +} + +#[derive(Clone, Copy)] +struct Abandoned { + selector: u8, + seq: u8, + label: &'static CStr, + waited_ms: u64, +} + +struct SksProbe { + active: bool, + captured: KVec, + label: Option<&'static CStr>, + unsolicited: u32, + abandoned: [Option; SKS_MAX_ABANDONED], + abandoned_next: usize, +} + +impl SksProbe { + fn new() -> Self { + SksProbe { + active: false, + captured: KVec::new(), + label: None, + unsolicited: 0, + abandoned: [None; SKS_MAX_ABANDONED], + abandoned_next: 0, + } + } +} + +struct XarsProbe { + active: bool, + captured: KVec, +} + +impl XarsProbe { + fn new() -> Self { + XarsProbe { + active: false, + captured: KVec::new(), + } + } +} + +struct ScrdProbe { + active: bool, + captured: KVec, +} + +impl ScrdProbe { + fn new() -> Self { + ScrdProbe { + active: false, + captured: KVec::new(), + } + } +} + +struct XarmState { + deferred_query: Option, + serviced: u32, + refused: u32, + os_uuid: Option<[u8; 16]>, +} + +impl XarmState { + fn new() -> Self { + XarmState { + deferred_query: None, + serviced: 0, + refused: 0, + os_uuid: None, + } + } +} + +struct EndpointTable { + eps: KVec, + discovery_msgs: u32, + unknown_types: u32, + dirty: bool, +} + +impl EndpointTable { + fn new() -> Self { + EndpointTable { + eps: KVec::new(), + discovery_msgs: 0, + unknown_types: 0, + dirty: false, + } + } + + fn slot(&mut self, id: u8) -> Result { + if let Some(i) = self.eps.iter().position(|e| e.id == id) { + return Ok(i); + } + if self.eps.len() >= MAX_ENDPOINTS { + return Err(ENOSPC); + } + self.eps.push(Endpoint::new(id), GFP_KERNEL)?; + self.dirty = true; + Ok(self.eps.len() - 1) + } +} + + +struct MachineRefKey { + blob: KVec, + pub_raw: KVec, +} + +#[pin_data] +struct SepData { + dev: ARef, + + #[pin] + mbox: Mutex>>, + + #[pin] + shmem: Mutex>, + + #[pin] + endpoints: Mutex, + + #[pin] + control: Mutex, + #[pin] + control_wq: CondVar, + + security_mode: Atomic, + + sks_wedged: Atomic, + + sks_seq: Atomic, + + // u32, not u8: the kernel's Rust atomics have no u8 AtomicType + phase: Atomic, + + #[pin] + ool_xarm: Mutex>, + + #[pin] + ool_sbio: Mutex>, + + #[pin] + sbio_last_header: Mutex>, + + #[pin] + dma_ring: Mutex>, + + #[pin] + sbio_rx: Mutex, + + #[pin] + sbio_wq: CondVar, + + sbio_ready: Atomic, + + templates_restored: Atomic, + + restore_attempted: Atomic, + + sensor_calibrated: Atomic, + + enrol_open: Atomic, + + #[pin] + enrol_material: Mutex>, + + #[pin] + enrol_identity_candidates: Mutex>, + + last_capture_end_ns: Atomic, + + device_view_synced: Atomic, + + backup_bag_other: Atomic, + + borrowed_sealing_handle: Atomic, + sealing_designations: Atomic, + config_stage: Atomic, + config_writes_sent: Atomic, + + keybag_designated: Atomic, + + cold_prepared: Atomic, + bringup_started: Atomic, + + bringup: Atomic, + + #[pin] + ool_sks: Mutex>, + + #[pin] + sks_probe: Mutex, + #[pin] + sks_wq: CondVar, + + #[pin] + ool_xars: Mutex>, + #[pin] + xars_probe: Mutex, + #[pin] + xars_wq: CondVar, + + #[pin] + ool_scrd: Mutex>, + #[pin] + scrd_probe: Mutex, + #[pin] + scrd_wq: CondVar, + + sensor_present: Atomic, + + #[pin] + bio_session: Mutex, + + #[pin] + bio_index: Mutex, + + #[pin] + bio_dev: Mutex>, + + #[pin] + store: Mutex>, + + #[pin] + xarm: Mutex, + + #[pin] + rng: Mutex>, + + #[pin] + machine_refkey: Mutex>, + + rng_shutdown: Atomic, + + rng_failures: Atomic, + + rx: rxring::RxRing, + + rx_count: Atomic, + settle_mark: Atomic, + settle_idle_ticks: Atomic, + + registered: Atomic, + + #[pin] + rx_work: Work, + + #[pin] + settle_work: DelayedWork, + + #[pin] + enrol_work: Work, + + #[pin] + verify_work: Work, +} + +impl_has_work! { + impl HasWork for SepData { self.rx_work } + impl HasWork for SepData { self.enrol_work } + impl HasWork for SepData { self.verify_work } +} + +impl_has_delayed_work! { + impl HasDelayedWork for SepData { self.settle_work } +} + +// SAFETY: every field is either internally synchronised (the mutexes, the +// atomics, the lock-free ring) or immutable after construction. The DMA buffer +// is only touched in probe, before the SEP has been told it exists, and at +// unbind, after the mailbox has been stopped. +unsafe impl Send for SepData {} +// SAFETY: see above. +unsafe impl Sync for SepData {} + +impl SepData { + fn new(pdev: &platform::Device>) -> Result> { + let built = shmem::build(pdev)?; + let dev: &device::Device> = pdev.as_ref(); + + let buf = built.buf; + + let ool_xarm = Self::alloc_ool( + dev, + xarm::EP_XARM, + OOL_SIZE_XARM, + (OOL_SIZE_XARM, OOL_SIZE_XARM), + )?; + let ool_sbio = Self::alloc_ool( + dev, + proto::EP_SBIO, + OOL_SIZE_SBIO, + (OOL_SIZE_SBIO, OOL_SIZE_SBIO), + )?; + let sks_declared = sks_declared_sizes(); + let ool_sks = Self::alloc_ool(dev, proto::EP_SKS, SKS_ALLOC, sks_declared)?; + let ool_xars = Self::alloc_ool( + dev, + xarm::EP_XARS, + OOL_SIZE_XARS, + (OOL_SIZE_XARS, OOL_SIZE_XARS), + )?; + let ool_scrd = Self::alloc_ool( + dev, + proto::EP_SCRD, + OOL_SIZE_SCRD, + (OOL_SIZE_SCRD, OOL_SIZE_SCRD), + )?; + let dma_ring = dma::Coherent::::zeroed_slice(dev, DMA_RING_SIZE, GFP_KERNEL)?; + + let mut store = match store::Store::open() { + Ok(mut store) => { + + if !store.seeded() { + match seed::import(&mut store) { + Ok(_imported) => {}, + Err(e) => dev_err!( + dev, + "seed '{}' could not be imported ({:?}); store was never seeded, so the exchange stops after the first ROOT_READ and endpoint count stays at {}\n", + seed::SEED_PATH, + e, + ENDPOINTS_BEFORE_EXCHANGE + ), + } + } + + Some(store) + } + Err(e) => { + dev_err!( + dev, + "could not open the backing store '{}': {:?}; persistent-state exchange cannot run\n", + store::STORE_PATH, + e + ); + None + } + }; + + let bio_index = match store.as_mut().map(bio::IdentityIndex::load) { + Some(Ok(index)) => index, + Some(Err(_)) => bio::IdentityIndex::new(), + None => bio::IdentityIndex::new(), + }; + + Arc::pin_init( + try_pin_init!(SepData { + dev: ARef::::from(dev), + mbox <- new_mutex!(None), + shmem <- new_mutex!(Some(buf)), + endpoints <- new_mutex!(EndpointTable::new()), + control <- new_mutex!(control::ControlState::new()), + control_wq <- new_condvar!("SepData::control_wq"), + security_mode: Atomic::new(SECMODE_UNKNOWN), + sks_seq: Atomic::new(0), + sks_wedged: Atomic::new(0), + bringup: Atomic::new(BRINGUP_FRESH), + keybag_designated: Atomic::new(false), + cold_prepared: Atomic::new(false), + bringup_started: Atomic::new(false), + enrol_open: Atomic::new(false), + sensor_calibrated: Atomic::new(false), + templates_restored: Atomic::new(false), + restore_attempted: Atomic::new(false), + enrol_material <- new_mutex!(None), + enrol_identity_candidates <- new_mutex!(KVec::new()), + last_capture_end_ns: Atomic::new(0), + device_view_synced: Atomic::new(false), + backup_bag_other: Atomic::new(0), + borrowed_sealing_handle: Atomic::new(0), + sealing_designations: Atomic::new(0), + config_writes_sent: Atomic::new(0), + config_stage: Atomic::new(CONFIG_STAGE_NONE), + phase: Atomic::new(PHASE_ATTACH), + ool_xarm <- new_mutex!(Some(ool_xarm)), + ool_sbio <- new_mutex!(Some(ool_sbio)), + sbio_last_header <- new_mutex!(None::<[u8; transfer::HEADER_LEN]>), + ool_sks <- new_mutex!(Some(ool_sks)), + sks_probe <- new_mutex!(SksProbe::new()), + sks_wq <- new_condvar!("SepData::sks_wq"), + ool_xars <- new_mutex!(Some(ool_xars)), + xars_probe <- new_mutex!(XarsProbe::new()), + xars_wq <- new_condvar!("SepData::xars_wq"), + ool_scrd <- new_mutex!(Some(ool_scrd)), + scrd_probe <- new_mutex!(ScrdProbe::new()), + scrd_wq <- new_condvar!("SepData::scrd_wq"), + dma_ring <- new_mutex!(Some(dma_ring)), + sbio_rx <- new_mutex!(transfer::Reassembly::new()), + sbio_wq <- new_condvar!("SepData::sbio_wq"), + sbio_ready: Atomic::new(false), + sensor_present: Atomic::new(false), + bio_session <- new_mutex!(bio::Session::new()), + bio_index <- new_mutex!(bio_index), + bio_dev <- new_mutex!(None), + store <- new_mutex!(store), + xarm <- new_mutex!(XarmState::new()), + rng <- new_mutex!(None), + machine_refkey <- new_mutex!(None), + rng_shutdown: Atomic::new(false), + rng_failures: Atomic::new(0), + rx: rxring::RxRing::new(), + rx_count: Atomic::new(0), + settle_mark: Atomic::new(0), + settle_idle_ticks: Atomic::new(0), + registered: Atomic::new(false), + rx_work <- new_work!("SepData::rx_work"), + enrol_work <- new_work!("SepData::enrol_work"), + verify_work <- new_work!("SepData::verify_work"), + settle_work <- new_delayed_work!("SepData::settle_work"), + }), + GFP_KERNEL, + ) + } + + fn attach(&self, sep_node: &dt::DtNode) -> Result<()> { + let (iova, size) = { + let guard = self.shmem.lock(); + let buf = guard.as_ref().ok_or(EINVAL)?; + (buf.dma_handle(), buf.len()) + }; + + let msg = proto::shmem_registration(iova, size)?; + + + if dt::registration_already_sent(sep_node) { + return Err(EBUSY); + } + + if let Err(e) = dt::mark_registration_sent(sep_node, iova) { + dev_err!( + self.dev, + "could not record the registration marker ({:?}); refusing to send, since without it an rmmod/insmod cycle could send it twice and park the SEP\n", + e + ); + return Err(e); + } + self.registered.store(true, Relaxed); + + self.mbox.lock().as_ref().ok_or(EINVAL)?.send(msg, false)?; + + // no ack on 0xFE; success is the discovery burst on 0xFD + Ok(()) + } + + fn send(&self, msg: Message) -> Result<()> { + self.mbox.lock().as_ref().ok_or(ENODEV)?.send(msg, false) + } + + + fn control_request(&self, op: &proto::ControlOp) -> Result> { + let (idx, tag) = self.control.lock().alloc()?; + let msg = proto::encode_control(op, tag); + + if let Err(e) = self.send(msg) { + self.control.lock().abandon(idx); + return Err(e); + } + + let mut remaining = time::msecs_to_jiffies(op.timeout_ms()); + let mut guard = self.control.lock(); + loop { + if let Some(reply) = guard.take_reply(idx) { + guard.release(idx); + drop(guard); + if reply.tag != tag { + return Err(EIO); + } + return Ok(Some(reply)); + } + + if remaining == 0 { + let tag = guard.abandon(idx); + let retired = guard.retired_count(); + drop(guard); + if op.expects_reply() { + dev_warn!( + self.dev, + "control <- {}: no reply within {} ms; tag 0x{:02x} retired ({} retired in total)\n", + op.name(), + op.timeout_ms(), + tag, + retired + ); + } + return Ok(None); + } + + match self + .control_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + } + } + + fn get_entropy_word(&self) -> Result { + let op = proto::op_get_entropy(); + if !op.uses_reserved_tag() { + return Err(EINVAL); + } + + self.control.lock().entropy_begin()?; + + let msg = proto::encode_control(&op, proto::TAG_ENTROPY); + if let Err(e) = self.send(msg) { + self.control.lock().entropy_end(); + return Err(e); + } + + let mut remaining = time::msecs_to_jiffies(op.timeout_ms()); + let mut guard = self.control.lock(); + let result = loop { + if let Some((value, _msg1)) = guard.entropy_take() { + break Ok(value); + } + if self.rng_shutdown.load(Relaxed) { + break Err(ECANCELED); + } + if remaining == 0 { + break Err(ETIMEDOUT); + } + match self + .control_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + }; + guard.entropy_end(); + result + } + + fn control_survey(&self) { + + for param in [0x00u8, 0x01, 0xff] { + let _ = self.control_request(&proto::op_nop(param)); + } + + match self.control_request(&proto::op_security_mode()) { + Ok(Some(reply)) => { + self.security_mode.store(reply.data_lo, Relaxed); + } + Ok(None) => {} + Err(_) => {} + } + + let mut words = [0u32; 4]; + let mut got = 0; + for w in words.iter_mut() { + match self.get_entropy_word() { + Ok(v) => { + *w = v; + got += 1; + } + Err(_) => { + break; + } + } + } + + if got == words.len() { + match self.register_hwrng() { + Ok(()) => {}, + Err(e) => dev_err!(self.dev, "hwrng: registration failed: {:?}\n", e), + } + } else { + dev_err!( + self.dev, + "hwrng: not registering, the source did not answer during the survey\n" + ); + } + + let _state = self.control.lock(); + } + + + fn alloc_ool( + dev: &device::Device>, + endpoint: u8, + allocated: usize, + declared: (usize, usize), + ) -> Result { + if declared.0 > allocated || declared.1 > allocated { + return Err(EINVAL); + } + + let inbound = dma::Coherent::::zeroed_slice(dev, allocated, GFP_KERNEL)?; + let outbound = dma::Coherent::::zeroed_slice(dev, allocated, GFP_KERNEL)?; + + // SAFETY: nothing is registered yet, so the SEP does not know these + // buffers exist and cannot be touching them. + unsafe { + inbound.as_mut().fill(OOL_POISON_INBOUND); + outbound.as_mut().fill(OOL_POISON_OUTBOUND); + } + + Ok(OolPair { + endpoint, + allocated, + declared_in: declared.0, + declared_out: declared.1, + inbound, + outbound, + registered: false, + }) + } + + fn control_request_required(&self, op: &proto::ControlOp) -> Result { + match self.control_request(op)? { + Some(reply) => Ok(reply), + None => Err(ETIMEDOUT), + } + } + + fn register_ool(&self, slot: &Mutex>) -> Result<()> { + let guard = slot.lock(); + if guard.as_ref().is_some_and(|b| b.registered) { + return Ok(()); + } + + let (endpoint, allocated, declared_in, declared_out, in_iova, out_iova) = { + let b = guard.as_ref().ok_or(EINVAL)?; + ( + b.endpoint, + b.allocated, + b.declared_in, + b.declared_out, + b.inbound.dma_handle(), + b.outbound.dma_handle(), + ) + }; + + if declared_in > allocated || declared_out > allocated { + return Err(EINVAL); + } + + // drop the buffer lock before blocking control requests; the drain path takes it + drop(guard); + + self.control_request_required(&proto::op_ool_inbound_size(endpoint, declared_in as u32))?; + self.control_request_required(&proto::op_ool_inbound_addr(endpoint, in_iova))?; + self.control_request_required(&proto::op_ool_outbound_size(endpoint, declared_out as u32))?; + self.control_request_required(&proto::op_ool_outbound_addr(endpoint, out_iova))?; + + { + // deref first, or the guard's inherent as_mut wins over Option::as_mut + let mut guard = slot.lock(); + let pair: &mut Option = &mut guard; + if let Some(b) = pair.as_mut() { + b.registered = true; + } + } + Ok(()) + } + + fn with_store(&self, f: impl FnOnce(&mut store::Store) -> R) -> Option { + let mut guard = self.store.lock(); + let store: &mut Option = &mut guard; + store.as_mut().map(f) + } + + // pre-generate: an entropy draw on the drain path would deadlock on its own reply + fn prepare_os_uuid(&self) { + let key = xarm::os_uuid_key(); + + let existing = match self.with_store(|store| store.read(&key)) { + Some(result) => result, + None => return, + }; + + if let Ok(Some(value)) = existing { + if value.len() == 16 { + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&value); + self.xarm.lock().os_uuid = Some(uuid); + return; + } + } + + let mut uuid = [0u8; 16]; + for word in 0..4 { + match self.get_entropy_word() { + Ok(v) => uuid[word * 4..word * 4 + 4].copy_from_slice(&v.to_le_bytes()), + Err(_) => { + return; + } + } + } + xarm::make_uuid_v4(&mut uuid); + + match self.with_store(|store| store.write(&key, &uuid)) { + Some(Ok(())) => {} + Some(Err(_)) => { + return; + } + None => { + return; + } + } + self.xarm.lock().os_uuid = Some(uuid); + } + + fn on_xarm(&self, msg: Message) { + let req = crate::xarm::decode_xarm(&msg); + + if xarm::is_silent(req.opcode) { + self.xarm.lock().serviced += 1; + return; + } + + let ready = self.ool_xarm.lock().as_ref().is_some_and(|b| b.registered); + + if !ready { + if !xarm::needs_buffers(req.opcode) { + self.xarm.lock().deferred_query = Some(req.tag); + return; + } + + let mut state = self.xarm.lock(); + state.refused += 1; + drop(state); + self.fail_xarm(req.tag); + return; + } + + self.service_xarm(&req); + } + + fn service_xarm(&self, req: &crate::xarm::XarmRequest) { + let want = req.length as usize; + if want > OOL_SIZE_XARM { + self.fail_xarm(req.tag); + return; + } + + if want > 0 && !self.ool_await_written(&self.ool_xarm, 0, want) { + self.fail_xarm(req.tag); + return; + } + let payload = match self.ool_read(&self.ool_xarm, 0, want) { + Ok(p) => p, + Err(_) => { + self.fail_xarm(req.tag); + return; + } + }; + + let mut staging = KVec::new(); + if staging.resize(OOL_SIZE_XARM, 0u8, GFP_KERNEL).is_err() { + self.fail_xarm(req.tag); + return; + } + + let os_uuid = self.xarm.lock().os_uuid; + let serviced = self.with_store(|store| { + xarm::service( + req, + &payload, + &mut staging, + store, + PROTECTED_DATA_AVAILABLE, + os_uuid, + ) + }); + let Some(done) = serviced else { + self.fail_xarm(req.tag); + return; + }; + + if done.reply_bytes > 0 { + if self.ool_write(&self.ool_xarm, 0, &staging[..done.reply_bytes]).is_err() { + self.fail_xarm(req.tag); + return; + } + } + + self.xarm.lock().serviced += 1; + + self.send_xarm_reply(&done.reply); + } + + fn ool_read(&self, slot: &Mutex>, off: usize, len: usize) -> Result> { + let mut guard = slot.lock(); + let pair: &mut Option = &mut guard; + let buffers = pair.as_mut().ok_or(ENODEV)?; + if off + len > buffers.allocated { + return Err(EINVAL); + } + + let mut payload = KVec::new(); + if len > 0 { + // SAFETY: the SEP fills this buffer while preparing a request and + // then waits for the host's reply, so it is quiescent here, and the + // driver is the only other accessor. + let src = unsafe { &buffers.outbound.as_ref()[off..off + len] }; + payload.extend_from_slice(src, GFP_KERNEL)?; + } + + Ok(payload) + } + + fn ool_await_written(&self, slot: &Mutex>, off: usize, len: usize) -> bool { + for _ in 0..OOL_WRITE_POLL_ATTEMPTS { + { + let guard = slot.lock(); + let Some(buffers) = guard.as_ref() else { + return false; + }; + if off + len > buffers.allocated { + return true; + } + // SAFETY: the region is within the allocation as checked above, + // and the driver is the only host-side accessor. + let region = unsafe { &buffers.outbound.as_ref()[off..off + len] }; + if region.iter().any(|&b| b != OOL_POISON_OUTBOUND) { + return true; + } + } + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(OOL_WRITE_POLL_MS))); + } + false + } + + fn ool_write(&self, slot: &Mutex>, off: usize, bytes: &[u8]) -> Result<()> { + let guard = slot.lock(); + let buffers = guard.as_ref().ok_or(ENODEV)?; + if off + bytes.len() > buffers.allocated { + return Err(ENOSPC); + } + // SAFETY: the SEP reads this buffer only after the reply message is + // sent, which happens after this returns; the driver is the only writer. + unsafe { buffers.inbound.as_mut()[off..off + bytes.len()].copy_from_slice(bytes) }; + Ok(()) + } + + fn fail_xarm(&self, tag: u8) { + self.send_xarm_reply(&crate::xarm::XarmReply { + tag, + status: xarm::STATUS_FAILED, + length: 0, + args: [0; 3], + }); + } + + fn send_xarm_reply(&self, reply: &crate::xarm::XarmReply) { + let msg = crate::xarm::encode_xarm_reply(reply); + let _ = self.send(msg); + } + + fn release_deferred_query(&self) { + let Some(tag) = self.xarm.lock().deferred_query.take() else { + return; + }; + let mut reply = crate::xarm::XarmReply { + tag, + status: xarm::STATUS_OK, + length: 0, + args: [0; 3], + }; + reply.args[0] = u8::from(PROTECTED_DATA_AVAILABLE); + self.xarm.lock().serviced += 1; + self.send_xarm_reply(&reply); + } + + + fn on_xars(&self, msg: Message) { + let mut probe = self.xars_probe.lock(); + if probe.active { + if probe.captured.len() < XARS_MAX_CAPTURE { + let _ = probe.captured.push(msg, GFP_KERNEL); + } + drop(probe); + self.xars_wq.notify_all(); + } + } + + + fn on_scrd(&self, msg: Message) { + let mut probe = self.scrd_probe.lock(); + if probe.active { + if probe.captured.len() < SCRD_MAX_CAPTURE { + let _ = probe.captured.push(msg, GFP_KERNEL); + } + drop(probe); + self.scrd_wq.notify_all(); + } + } + + fn scrd_arm(&self) { + let mut probe = self.scrd_probe.lock(); + probe.captured.clear(); + probe.active = true; + } + + fn scrd_disarm(&self) -> KVec { + let mut probe = self.scrd_probe.lock(); + probe.active = false; + core::mem::take(&mut probe.captured) + } + + fn scrd_wait(&self, ms: time::Msecs, until: usize) { + let mut remaining = time::msecs_to_jiffies(ms); + let mut guard = self.scrd_probe.lock(); + loop { + if guard.captured.len() >= until || remaining == 0 { + return; + } + match self + .scrd_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + } + } + + fn scrd_zero_buffers(&self) -> Result<()> { + let mut guard = self.ool_scrd.lock(); + let pair: &mut Option = &mut guard; + let buffers = pair.as_mut().ok_or(ENODEV)?; + // SAFETY: nothing is outstanding on this endpoint here; the enclave + // touches these only between a request going out and its reply, and the + // driver is the only other accessor. + unsafe { + buffers.inbound.as_mut()[..buffers.allocated].fill(0); + buffers.outbound.as_mut()[..buffers.allocated].fill(0); + } + Ok(()) + } + + fn scrd_transact( + &self, + _name: &'static CStr, + request: u8, + payload: &[u8], + timeout_ms: time::Msecs, + ) -> Option<(crate::scrd::ScrdReply, KVec)> { + if !self.ool_registered(&self.ool_scrd) { + return None; + } + if self.scrd_zero_buffers().is_err() { + return None; + } + if !payload.is_empty() { + if self.ool_write(&self.ool_scrd, 0, payload).is_err() { + return None; + } + } + + let msg = crate::scrd::encode_scrd(request, payload.len()); + self.scrd_arm(); + if self.send(msg).is_err() { + let _ = self.scrd_disarm(); + return None; + } + + let mut examined = 0usize; + let correlating = loop { + self.scrd_wait(timeout_ms, examined + 1); + let next = { + let probe = self.scrd_probe.lock(); + let m = probe.captured.get(examined).copied(); + if m.is_some() { + examined += 1; + } + m + }; + let Some(candidate) = next else { break None }; + let decoded = crate::scrd::decode_scrd_reply(&candidate); + if decoded.request == request { + break Some(candidate); + } + if examined >= SCRD_MAX_CAPTURE { + break None; + } + }; + + let leftover = self.scrd_disarm(); + let correlating = match correlating { + Some(m) => Some(m), + None => leftover + .iter() + .skip(examined) + .copied() + .find(|m| crate::scrd::decode_scrd_reply(m).request == request), + }; + + let raw = correlating?; + + let reply = crate::scrd::decode_scrd_reply(&raw); + let mut response = KVec::new(); + if reply.response_size > 0 { + if let Ok(bytes) = self.ool_read(&self.ool_scrd, 0, reply.response_size as usize) { + response = bytes; + } + } + Some((reply, response)) + } + + fn scrd_ready(&self) -> bool { + if self.ool_registered(&self.ool_scrd) { + return true; + } + if !self.endpoint_present(proto::EP_SCRD) { + return false; + } + if self.register_ool(&self.ool_scrd).is_err() { + return false; + } + true + } + + fn scrd_command(&self, cmd: &crate::scrd::ScrdCommand) -> Option<(i32, KVec)> { + let (reply, out) = + self.scrd_transact(cmd.name(), cmd.request(), cmd.payload(), SCRD_TIMEOUT_MS)?; + Some((reply.status, out)) + } + + fn establish_passcode_validated_context( + &self, + user: crate::sbio::UserId, + ) -> Option<[u8; crate::scrd::SCRD_ACM_HANDLE_LEN]> { + if !self.scrd_ready() { + return None; + } + + let (special, secret) = { + let guard = self.enrol_material.lock(); + let material = guard.as_ref()?; + let mut copy = KVec::new(); + copy.extend_from_slice(&material.secret, GFP_KERNEL).ok()?; + (material.special, Secret(copy)) + }; + + let (status, _) = self.scrd_command(&crate::scrd::scrd_initialize())?; + if status != 0 { + return None; + } + + let (status, out) = self.scrd_command(&crate::scrd::scrd_context_create_tracked(user.value()))?; + if status != 0 || out.len() < crate::scrd::SCRD_ACM_HANDLE_LEN { + return None; + } + let mut acm_handle = [0u8; crate::scrd::SCRD_ACM_HANDLE_LEN]; + acm_handle.copy_from_slice(&out[..crate::scrd::SCRD_ACM_HANDLE_LEN]); + + let (status, _) = self.scrd_command(&crate::scrd::scrd_context_externalize(&acm_handle))?; + if status != 0 { + return None; + } + + let out = self.sks_send(self.sks_req_verify_secret(special, &secret, &acm_handle))?; + if out.reply.status != 0 { + return None; + } + + let (status, out) = self.scrd_command(&crate::scrd::scrd_verify_touchid_enrollment(&acm_handle))?; + let satisfied = out.len() >= 4 + && u32::from_le_bytes([out[0], out[1], out[2], out[3]]) != 0; + if status != 0 || !satisfied { + return None; + } + + Some(acm_handle) + } + + fn establish_scrd_match_context(&self, user: crate::sbio::UserId) { + if !self.scrd_ready() { + return; + } + let Some(du) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) else { + return; + }; + let special = du.special_handle(); + let stored = match keybag::read(keybag::Slot::Identity) { + Ok(keybag::State::Present(s)) => s, + _ => { + return; + } + }; + + let _ = (|| -> Option<[u8; crate::scrd::SCRD_ACM_HANDLE_LEN]> { + let (status, _) = self.scrd_command(&crate::scrd::scrd_initialize())?; + if status != 0 { + return None; + } + let (status, out) = + self.scrd_command(&crate::scrd::scrd_context_create_tracked(user.value()))?; + if status != 0 || out.len() < crate::scrd::SCRD_ACM_HANDLE_LEN { + return None; + } + let mut acm_handle = [0u8; crate::scrd::SCRD_ACM_HANDLE_LEN]; + acm_handle.copy_from_slice(&out[..crate::scrd::SCRD_ACM_HANDLE_LEN]); + let (status, _) = self.scrd_command(&crate::scrd::scrd_context_externalize(&acm_handle))?; + if status != 0 { + return None; + } + let out = self.sks_send(self.sks_req_verify_secret(special, stored.secret(), &acm_handle))?; + if out.reply.status != 0 { + return None; + } + self.scrd_command(&crate::scrd::scrd_verify_touchid_enrollment(&acm_handle))?; + Some(acm_handle) + })(); + } + + + fn hwrng_fill(&self, buf: *mut u8, max: usize, wait: bool) -> c_int { + if !wait { + return 0; + } + + let words = core::cmp::min(max / 4, RNG_MAX_WORDS_PER_READ); + let mut written: usize = 0; + let mut failure: Option = None; + + for i in 0..words { + if self.rng_shutdown.load(Relaxed) { + failure = Some(ENODEV); + break; + } + match self.get_entropy_word() { + Ok(value) => { + // SAFETY: the core guarantees `buf` is valid for `max` + // bytes and aligned for any type, and `i < max / 4`. + unsafe { buf.add(i * 4).cast::().write_unaligned(value) }; + written += 4; + } + Err(e) => { + failure = Some(e); + break; + } + } + } + + if written > 0 { + return written as c_int; + } + + // must return a negative errno, not 0: a blocking reader would spin on 0 + let e = failure.unwrap_or(EIO); + let n = self.rng_failures.load(Relaxed).wrapping_add(1); + self.rng_failures.store(n, Relaxed); + if n == 1 { + dev_err!( + self.dev, + "hwrng: read produced nothing ({:?}); further failures will not be logged\n", + e + ); + } + e.to_errno() + } + + fn register_hwrng(&self) -> Result<()> { + let mut guard = self.rng.lock(); + if guard.is_some() { + return Ok(()); + } + + let mut handle = hwrng::HwRngHandle::new()?; + let ctx = core::ptr::from_ref(self).cast_mut().cast::(); + + // SAFETY: `ctx` is this `SepData`, which is kept alive by the `Arc` held + // in the driver's private data. `remove()` sets `rng_shutdown`, wakes + // any blocked draw, and unregisters, and `hwrng_unregister()` blocks + // until the core has finished with the device, all before that `Arc` + // can be dropped. So the pointer cannot outlive the registration. + unsafe { handle.register(ctx, hwrng_read_trampoline) }?; + + *guard = Some(handle); + Ok(()) + } + + + fn drain(&self) -> u32 { + let mut n: u32 = 0; + while let Some(msg) = self.rx.pop() { + self.dispatch(msg); + n += 1; + } + if n > 0 { + // single writer, single instance: a load/store pair suffices + let total = self.rx_count.load(Relaxed).wrapping_add(u64::from(n)); + self.rx_count.store(total, Relaxed); + } + n + } + + + fn arm_settle(this: &Arc) { + let delay = time::msecs_to_jiffies(SETTLE_MS); + let _ = workqueue::system() + .enqueue_delayed::, SETTLE_WORK_ID>(this.clone(), delay); + } + + fn settle_tick(this: &Arc) { + match this.phase.load(Relaxed) { + PHASE_ATTACH => Self::tick_attach(this), + PHASE_EXCHANGE => Self::tick_exchange(this), + _ => {} + } + } + + fn tick_attach(this: &Arc) { + let now = this.rx_count.load(Relaxed); + + // HW: the SEP takes ~265 ms to answer a registration + if now == 0 { + let ticks = this.settle_idle_ticks.load(Relaxed).wrapping_add(1); + this.settle_idle_ticks.store(ticks, Relaxed); + if ticks.saturating_mul(u64::from(SETTLE_MS)) < u64::from(FIRST_RESPONSE_MS) { + Self::arm_settle(this); + return; + } + } + + if now != this.settle_mark.load(Relaxed) { + this.settle_mark.store(now, Relaxed); + Self::arm_settle(this); + return; + } + + if this.endpoint_count() == 0 { + this.phase.store(PHASE_READY, Relaxed); + return; + } + + this.prepare_os_uuid(); + + this.control_survey(); + + if let Err(e) = this.register_ool(&this.ool_xarm) { + dev_err!( + this.dev, + "xarm: out-of-line buffer registration failed ({:?}); the persistent-state exchange cannot run\n", + e + ); + this.phase.store(PHASE_READY, Relaxed); + return; + } + + this.phase.store(PHASE_EXCHANGE, Relaxed); + this.settle_mark.store(this.rx_count.load(Relaxed), Relaxed); + this.settle_idle_ticks.store(0, Relaxed); + this.release_deferred_query(); + Self::arm_settle(this); + } + + fn tick_exchange(this: &Arc) { + let now = this.rx_count.load(Relaxed); + + if now != this.settle_mark.load(Relaxed) { + this.settle_mark.store(now, Relaxed); + this.settle_idle_ticks.store(0, Relaxed); + Self::arm_settle(this); + return; + } + + let endpoints = this.endpoint_count(); + if endpoints <= ENDPOINTS_BEFORE_EXCHANGE { + let ticks = this.settle_idle_ticks.load(Relaxed).wrapping_add(1); + this.settle_idle_ticks.store(ticks, Relaxed); + if ticks.saturating_mul(u64::from(SETTLE_MS)) < u64::from(EXCHANGE_TIMEOUT_MS) { + Self::arm_settle(this); + return; + } + } + + this.phase.store(PHASE_READY, Relaxed); + this.run_bringup(); + } + + fn endpoint_present(&self, id: u8) -> bool { + self.endpoints.lock().eps.iter().any(|e| e.id == id) + } + + fn endpoint_count(&self) -> usize { + self.endpoints.lock().eps.len() + } + + fn dispatch(&self, msg: Message) { + let f = proto::decode(&msg); + + match f.ep { + proto::EP_DISCOVER => self.on_discovery(msg, f), + + proto::EP_SHMEM => {}, + + proto::EP_CONTROL => self.on_control(msg, f), + + xarm::EP_XARM => self.on_xarm(msg), + + proto::EP_SBIO => self.on_sbio(msg), + + proto::EP_SKS => self.on_sks(msg), + + // xars: only 0x08 and 0x04 are sent; other opcodes are destructive + xarm::EP_XARS => self.on_xars(msg), + + proto::EP_SCRD => self.on_scrd(msg), + + proto::EP_BOOT => dev_warn!( + self.dev, + "unexpected message from boot endpoint 0xff: type 0x{:02x} param 0x{:02x} msg0 {:#018x} msg1 {:#010x}\n", + f.ty, + f.param, + msg.msg0, + msg.msg1 + ), + + _ep => {}, + } + } + + fn on_control(&self, msg: Message, f: proto::Fields) { + if f.ty != proto::CONTROL_REPLY_TYPE { + self.control.lock().note_unsolicited(); + return; + } + + let reply = proto::ControlReply::from_message(&msg); + let disposition = self.control.lock().deliver(reply); + + match disposition { + control::Delivery::Entropy => { + self.control_wq.notify_all(); + } + control::Delivery::Matched => self.control_wq.notify_all(), + control::Delivery::Unmatched => {} + } + } + + fn on_discovery(&self, msg: Message, f: proto::Fields) { + let mut table = self.endpoints.lock(); + table.discovery_msgs += 1; + + let id = f.param; + + let idx = match table.slot(id) { + Ok(i) => i, + Err(_) => { + return; + } + }; + + match f.ty { + proto::DISCOVER_TYPE_DESCRIPTOR => { + let cc = proto::fourcc(&msg); + let e = &mut table.eps[idx]; + e.have_descriptor = true; + e.descriptor_msg0 = msg.msg0; + e.descriptor_msg1 = msg.msg1; + e.fourcc = cc; + table.dirty = true; + } + proto::DISCOVER_TYPE_CONFIG => { + let e = &mut table.eps[idx]; + e.have_config = true; + e.config_msg0 = msg.msg0; + e.config_msg1 = msg.msg1; + table.dirty = true; + } + _ => { + table.unknown_types += 1; + } + } + } + + + fn remove(&self) { + // first: resets the keyring static calls before freeing, so no keyctl op hits freed data + trusted::unregister(); + + let dev = self.bio_dev.lock().take(); + if dev.is_some() { + bio::release(&mut self.bio_session.lock()); + sensor::power(false); + sensor::unregister_driver(); + } + drop(dev); + + self.rng_shutdown.store(true, Relaxed); + self.control_wq.notify_all(); + if let Some(mut handle) = self.rng.lock().take() { + handle.unregister(); + } + + // stop the mailbox last: no callbacks after this + *self.mbox.lock() = None; + + let _ = self.store.lock().take(); + + for slot in [&self.ool_xarm, &self.ool_sbio, &self.ool_sks] { + if let Some(buffers) = slot.lock().take() { + if buffers.registered { + core::mem::forget(buffers); + } + } + } + + if let Some(ring) = self.dma_ring.lock().take() { + if self.sbio_ready.load(Relaxed) { + core::mem::forget(ring); + } + } + + let mut guard = self.shmem.lock(); + if let Some(buf) = guard.take() { + if self.registered.load(Relaxed) { + core::mem::forget(buf); + } else { + drop(buf); + } + } + } +} + +/// # Safety +/// +/// Each is called only by `bio_shim.c`, with the `*const SepData` handed to +/// `BioChardev::register`, which stays valid for the whole registered window. +unsafe extern "C" fn bio_open_trampoline(ctx: *mut c_void) -> c_int { + // SAFETY: per the contract above. + let this = unsafe { &*ctx.cast::() }; + match this.bio_open() { + Ok(()) => 0, + Err(e) => e.to_errno(), + } +} + +/// # Safety +unsafe extern "C" fn bio_release_trampoline(ctx: *mut c_void) { + // SAFETY: per the contract above. + let this = unsafe { &*ctx.cast::() }; + this.bio_release(); +} + +/// # Safety +unsafe extern "C" fn bio_ioctl_trampoline(ctx: *mut c_void, cmd: c_uint, arg: c_ulong) -> c_long { + // SAFETY: per the contract above. + let this = unsafe { &*ctx.cast::() }; + let handled = match this.bio_ioctl(cmd, arg) { + Ok(h) => h, + Err(e) => return e.to_errno() as c_long, + }; + + if handled.start_verify { + // SAFETY: as for `start_enrol` below: same pointer, same guarantee. + let borrow = unsafe { kernel::sync::ArcBorrow::::from_raw(ctx.cast::()) }; + SepData::queue_verify(borrow.into()); + } + + if handled.start_enrol { + // SAFETY: `ctx` is the pointer the driver's own `Arc` was made + // from and that `Arc` outlives every callback; see the contract on the + // registration. `ArcBorrow` is what turns that guarantee into an owned + // reference with the refcount incremented rather than a second owner of + // the same count. + let borrow = unsafe { kernel::sync::ArcBorrow::::from_raw(ctx.cast::()) }; + SepData::queue_enrolment(borrow.into()); + } + handled.ret as c_long +} + +/// # Safety +unsafe extern "C" fn bio_ready_trampoline(ctx: *mut c_void) -> c_int { + // SAFETY: per the contract above. + let this = unsafe { &*ctx.cast::() }; + c_int::from(this.bio_ready()) +} + +/// # Safety +/// +/// Only ever called by the hwrng core, through `hwrng_shim.c`. `ctx` is the +/// `*const SepData` handed to `HwRngHandle::register`, which stays valid for +/// the whole registered window (see the ordering note in `remove()`), and +/// `data` is valid for `max` bytes. +unsafe extern "C" fn hwrng_read_trampoline( + ctx: *mut c_void, + data: *mut c_void, + max: usize, + wait: bool, +) -> c_int { + // SAFETY: per the contract above. + let this = unsafe { &*ctx.cast::() }; + this.hwrng_fill(data.cast::(), max, wait) +} + +impl MailCallback for SepData { + type Data = Arc; + + fn recv_message(data: ::Borrowed<'_>, msg: Message) { + if !data.rx.push(msg) && data.rx.dropped() == 1 { + dev_err!( + data.dev, + "receive ring full; dropping messages (first was msg0 {:#018x})\n", + msg.msg0 + ); + } + + let this: Arc = data.into(); + let _ = workqueue::system().enqueue::, 0>(this); + } +} + +impl WorkItem for SepData { + type Pointer = Arc; + + fn run(this: Arc) { + if this.drain() > 0 { + SepData::arm_settle(&this); + } + } +} + +impl WorkItem for SepData { + type Pointer = Arc; + + fn run(this: Arc) { + SepData::settle_tick(&this); + } +} + +impl WorkItem for SepData { + type Pointer = Arc; + + fn run(this: Arc) { + this.run_enrolment(); + } +} + +impl WorkItem for SepData { + type Pointer = Arc; + + fn run(this: Arc) { + this.run_verify(); + } +} + + +struct SepDriver(Arc); + +const OF_TABLE: kernel::device_id::IdArray = + kernel::device_id::IdArray::new([(of::DeviceId::new(c"apple,sep"), ())]); + +impl platform::Driver for SepDriver { + type IdInfo = (); + type Data<'bound> = SepDriver; + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe<'bound>( + pdev: &'bound platform::Device>, + _info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound { + let dev: &device::Device> = pdev.as_ref(); + let sep_node = dt::DtNode::of_device(dev).ok_or(ENODEV)?; + + if dt::registration_already_sent(&sep_node) { + dev_err!( + dev, + "the shared-memory registration was already sent on this boot (marker property present). It is one-shot per AP reset and cannot be repeated or withdrawn: reboot to attach again. Not probing.\n" + ); + return Err(EBUSY); + } + + let data = SepData::new(pdev)?; + + *data.mbox.lock() = Some(Mailbox::new_byname(dev, c"mbox", data.clone())?); + + if let Err(e) = data.register_bio() { + dev_err!( + dev, + "could not register /dev/{}: {:?}\n", + bio::DEVICE_NAME, + e + ); + } + + data.attach(&sep_node)?; + + data.attach_sensor(); + + if let Err(e) = trusted::register(data.clone()) { + dev_warn!(data.dev, "trusted-keys: registration failed ({:?})\n", e); + } + + if data.registered.load(Relaxed) { + SepData::arm_settle(&data); + } + + Ok(SepDriver(data)) + } +} + +impl Drop for SepDriver { + fn drop(&mut self) { + self.0.remove(); + } +} + + +#[pin_data] +struct SepModule { + // field-init order enables the DT nodes before the driver registers; a failure here aborts module init + _dt: (), + + #[pin] + _driver: driver::Registration>, +} + +impl kernel::InPlaceModule for SepModule { + fn init(module: &'static ThisModule) -> impl PinInit { + try_pin_init!(Self { + _dt: dt::enable_sep_and_dart()?, + + _driver <- driver::Registration::new( + ::NAME, + module, + ), + }) } } -module_platform_driver! { - type: SepDriver, +module! { + type: SepModule, name: "apple_sep", - description: "Secure enclave processor stub driver", + description: "Apple SEP coprocessor: warm attach, endpoint discovery and tag-correlated control endpoint", license: "Dual MIT/GPL", } diff --git a/drivers/soc/apple/sha_shim.c b/drivers/soc/apple/sha_shim.c new file mode 100644 index 0000000000000..2e823df021dde --- /dev/null +++ b/drivers/soc/apple/sha_shim.c @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ +/* SHA-256 shim: hashes two caller-chosen segments; no protocol logic here. */ + +#include +#include +#include + +#include "shim.h" + +/* + * Hashes `a` then `b`, writes 32 bytes to `out`. Allocates a transform per call + * and may sleep. Returns 0 or a negative errno; `out` invalid on failure. + */ +int sep_sha256(const void *a, size_t alen, const void *b, size_t blen, + unsigned char *out) +{ + struct crypto_shash *tfm; + int rc; + + tfm = crypto_alloc_shash("sha256", 0, 0); + if (IS_ERR(tfm)) + return PTR_ERR(tfm); + + { + SHASH_DESC_ON_STACK(desc, tfm); + + desc->tfm = tfm; + + rc = crypto_shash_init(desc); + if (!rc && alen) + rc = crypto_shash_update(desc, a, alen); + if (!rc && blen) + rc = crypto_shash_update(desc, b, blen); + if (!rc) + rc = crypto_shash_final(desc, out); + + /* The digest is over a request that may contain a secret. */ + shash_desc_zero(desc); + } + + crypto_free_shash(tfm); + return rc; +} diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h new file mode 100644 index 0000000000000..2abb13765bad7 --- /dev/null +++ b/drivers/soc/apple/shim.h @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ +/* + * Apple SEP driver — private declarations for the C shim. + * + * Glue between the Rust driver and kernel interfaces whose ABI is awkward or + * unsafe to restate in Rust: `struct hwrng`, whose layout moves with + * CONFIG_LOCKDEP; the file interface, whose O_* flags are macro-derived and so + * absent from the Rust bindings; and the crypto API, whose shash descriptor is + * sized by a macro at its call site. + * + * No protocol logic lives in the shim. Included only by the shim translation + * units. + */ + +#ifndef SEP_SHIM_H +#define SEP_SHIM_H + +#include + +/* -- hwrng_shim.c ------------------------------------------------------- */ + +void *sep_hwrng_alloc(void); +void sep_hwrng_free(void *mem); +int sep_hwrng_register(void *mem, const char *name, + unsigned short quality, void *ctx, + int (*read)(void *ctx, void *data, size_t max, + bool wait)); +void sep_hwrng_unregister(void *mem); + +/* -- store_shim.c ------------------------------------------------------- */ + +void *sep_store_open(const char *path); +void *sep_store_open_trunc(const char *path); +void *sep_store_open_ro(const char *path); +void sep_store_close(void *handle); +long long sep_store_size(void *handle); +long sep_store_read(void *handle, long long off, void *buf, size_t len); +long sep_store_write(void *handle, long long off, const void *buf, + size_t len); +int sep_store_sync(void *handle); + +/* -- sha_shim.c --------------------------------------------------------- */ + +int sep_sha256(const void *a, size_t alen, const void *b, size_t blen, + unsigned char *out); + +/* -- crypto_shim.c ------------------------------------------------------ */ + +/* HMAC-SHA256 over one message, 32 bytes out. */ +int sep_hmac_sha256(const void *key, size_t keylen, + const void *data, size_t datalen, u8 *out); + +/* + * AES-256-GCM in place over `buf` = `aadlen` bytes of AAD followed by the + * payload. Returns -EBADMSG on auth failure, which the caller must treat as + * fatal. + */ +int sep_gcm(int encrypt, const void *key, size_t keylen, + const void *iv, size_t ivlen, size_t aadlen, + void *buf, size_t buflen, size_t datalen); + +/* + * Fill `buf` from the kernel CSPRNG, seeded, or fail. For key-bag secrets only; + * anything measuring the enclave's own entropy (0x59 device-key probe, + * /dev/hwrng) stays on SEP. + */ +int sep_random_bytes(void *buf, size_t len); + +/* -- p256_shim.c ------------------------------------------------------- */ + +/* + * ECIES sender key agreement for the ref-key (op 0x22) SE-seal, with a fresh + * ephemeral P-256 keypair. `peer_pub_be` is the recipient's 64-byte public + * point (0x04 prefix stripped), `eph_pub_be_out` receives the 64-byte ephemeral + * public point, `shared_be_out` the 32-byte shared secret. All big-endian. + */ +int sep_p256_sender(const void *peer_pub_be, void *eph_pub_be_out, + void *shared_be_out); + +/* -- sensor_shim.c ------------------------------------------------------ */ + +int sep_sensor_register(void); +void sep_sensor_unregister(void); +int sep_sensor_bound(void); +int sep_sensor_power_line(void); +int sep_sensor_cs_timing_mode(void); +int sep_sensor_power_cycle(void); +int sep_sensor_power_source(void); +int sep_sensor_power(int on); +int sep_sensor_xfer(const void *tx, void *rx, size_t len); +int sep_sensor_xfer_tx(const void *tx, size_t len); +int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len); + +/* -- bio_shim.c --------------------------------------------------------- */ + +void *sep_bio_register(const char *name, unsigned short mode, void *ctx, + int (*f_open)(void *), + void (*f_release)(void *), + long (*f_ioctl)(void *, unsigned int, unsigned long), + int (*f_ready)(void *)); +void sep_bio_unregister(void *dev); +void sep_bio_wake(void *dev); +int sep_bio_capable_admin(void); +__u64 sep_bio_monotonic_ns(void); +__u64 sep_bio_boottime_ns(void); + +/* -- trusted_shim.c ---------------------------------------------------- */ + +/* + * The kernel trusted-key source (keyctl). The shim owns the framework structs + * and registration; the sealing policy is in Rust (trusted.rs), reached through + * the callbacks passed to sep_tk_register(). The payload struct is only + * forward-declared here since these prototypes take it by pointer; trusted_shim.c + * pulls in for the definition. + */ +struct trusted_key_payload; + +typedef int (*sep_tk_init_fn)(void); +typedef int (*sep_tk_seal_fn)(struct trusted_key_payload *p, char *datablob); +typedef int (*sep_tk_unseal_fn)(struct trusted_key_payload *p, char *datablob); +typedef int (*sep_tk_random_fn)(unsigned char *key, size_t key_len); +typedef void (*sep_tk_exit_fn)(void); + +int sep_tk_register(sep_tk_init_fn init, sep_tk_seal_fn seal, + sep_tk_unseal_fn unseal, sep_tk_random_fn random, + sep_tk_exit_fn exit); +void sep_tk_unregister(void); + +int sep_tk_register_key_type(void); +void sep_tk_unregister_key_type(void); + +size_t sep_tk_max_key_size(void); +size_t sep_tk_max_blob_size(void); + +unsigned char *sep_tk_key_ptr(struct trusted_key_payload *p); +unsigned int sep_tk_key_len(const struct trusted_key_payload *p); +void sep_tk_set_key_len(struct trusted_key_payload *p, unsigned int n); + +unsigned char *sep_tk_blob_ptr(struct trusted_key_payload *p); +unsigned int sep_tk_blob_len(const struct trusted_key_payload *p); +void sep_tk_set_blob_len(struct trusted_key_payload *p, unsigned int n); + +#endif /* SEP_SHIM_H */ diff --git a/drivers/soc/apple/shim.rs b/drivers/soc/apple/shim.rs new file mode 100644 index 0000000000000..2968a04943e99 --- /dev/null +++ b/drivers/soc/apple/shim.rs @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Rust side of `store_shim.c`: the backing-store file. + +use kernel::prelude::*; + +extern "C" { + fn sep_store_open(path: *const c_char) -> *mut c_void; + fn sep_store_open_trunc(path: *const c_char) -> *mut c_void; + fn sep_store_open_ro(path: *const c_char) -> *mut c_void; + fn sep_store_close(handle: *mut c_void); + fn sep_store_size(handle: *mut c_void) -> i64; + fn sep_store_read(handle: *mut c_void, off: i64, buf: *mut c_void, len: usize) + -> c_long; + fn sep_store_write( + handle: *mut c_void, + off: i64, + buf: *const c_void, + len: usize, + ) -> c_long; + fn sep_store_sync(handle: *mut c_void) -> c_int; +} + +/// Backing-store file handle. +/// +/// # Invariants +/// `handle` is a live `struct file *` from `sep_store_open`. +pub(crate) struct StoreFile { + handle: *mut c_void, +} + +// SAFETY: a `struct file *` has no thread affinity; every access goes through +// the C shim, and the driver keeps this behind a mutex. +unsafe impl Send for StoreFile {} + +fn result_of(ret: c_long) -> Result { + if ret < 0 { + Err(Error::from_errno(ret as c_int)) + } else { + Ok(ret as usize) + } +} + +impl StoreFile { + pub(crate) fn open(path: &CStr) -> Result { + // SAFETY: `path` is NUL-terminated; the shim returns NULL on failure. + let handle = unsafe { sep_store_open(path.as_char_ptr()) }; + if handle.is_null() { + return Err(ENOENT); + } + Ok(StoreFile { handle }) + } + + pub(crate) fn open_trunc(path: &CStr) -> Result { + // SAFETY: `path` is NUL-terminated; the shim returns NULL on failure. + let handle = unsafe { sep_store_open_trunc(path.as_char_ptr()) }; + if handle.is_null() { + return Err(ENOENT); + } + Ok(StoreFile { handle }) + } + + pub(crate) fn open_readonly(path: &CStr) -> Result { + // SAFETY: `path` is NUL-terminated; the shim returns NULL on failure. + let handle = unsafe { sep_store_open_ro(path.as_char_ptr()) }; + if handle.is_null() { + return Err(ENOENT); + } + Ok(StoreFile { handle }) + } + + pub(crate) fn size(&self) -> Result { + // SAFETY: `handle` is live per the type invariant. + let n = unsafe { sep_store_size(self.handle) }; + if n < 0 { + return Err(EIO); + } + Ok(n as u64) + } + + pub(crate) fn read_exact(&self, off: u64, buf: &mut [u8]) -> Result<()> { + let mut done = 0usize; + while done < buf.len() { + // SAFETY: `handle` is live; the pointer and length describe the + // remaining tail of `buf`, which we own mutably. + let n = result_of(unsafe { + sep_store_read( + self.handle, + (off + done as u64) as i64, + buf[done..].as_mut_ptr().cast::(), + buf.len() - done, + ) + })?; + if n == 0 { + return Err(EIO); + } + done += n; + } + Ok(()) + } + + pub(crate) fn write_all(&self, off: u64, buf: &[u8]) -> Result<()> { + let mut done = 0usize; + while done < buf.len() { + // SAFETY: as above, for a shared borrow. + let n = result_of(unsafe { + sep_store_write( + self.handle, + (off + done as u64) as i64, + buf[done..].as_ptr().cast::(), + buf.len() - done, + ) + })?; + if n == 0 { + return Err(EIO); + } + done += n; + } + Ok(()) + } + + pub(crate) fn sync(&self) -> Result<()> { + // SAFETY: `handle` is live per the type invariant. + kernel::error::to_result(unsafe { sep_store_sync(self.handle) }) + } +} + +impl Drop for StoreFile { + fn drop(&mut self) { + // SAFETY: `handle` is live per the type invariant and is not used again. + unsafe { sep_store_close(self.handle) }; + } +} + +extern "C" { + fn sep_bio_register( + name: *const c_char, + mode: u16, + ctx: *mut c_void, + f_open: Option c_int>, + f_release: Option, + f_ioctl: Option c_long>, + f_ready: Option c_int>, + ) -> *mut c_void; + fn sep_bio_unregister(dev: *mut c_void); + fn sep_bio_wake(dev: *mut c_void); + fn sep_bio_capable_admin() -> c_int; + fn sep_bio_monotonic_ns() -> u64; + fn sep_bio_boottime_ns() -> u64; +} + +extern "C" { +} + +pub(crate) fn capable_admin() -> bool { + // SAFETY: no preconditions; reads the current task's credentials. + unsafe { sep_bio_capable_admin() != 0 } +} + +pub(crate) fn monotonic_ns() -> u64 { + // SAFETY: no preconditions. + unsafe { sep_bio_monotonic_ns() } +} + +/// `CLOCK_BOOTTIME`; differs from monotonic by suspend time — how "tokens don't +/// survive a suspend" is detected. +pub(crate) fn boottime_ns() -> u64 { + // SAFETY: no preconditions. + unsafe { sep_bio_boottime_ns() } +} + +/// The registered character device. +/// +/// # Invariants +/// `dev` is a live registration from `sep_bio_register`. +pub(crate) struct BioChardev { + dev: *mut c_void, +} + +// SAFETY: the registration has no thread affinity; every access goes through +// the C shim, which locks internally. +unsafe impl Send for BioChardev {} + +impl BioChardev { + /// Registers `/dev/`. + /// + /// # Safety + /// `ctx` must stay valid and safe to pass to the four callbacks until this + /// is dropped; the callbacks may run on any task at any time. + pub(crate) unsafe fn register( + name: &'static CStr, + mode: u16, + ctx: *mut c_void, + f_open: unsafe extern "C" fn(*mut c_void) -> c_int, + f_release: unsafe extern "C" fn(*mut c_void), + f_ioctl: unsafe extern "C" fn(*mut c_void, c_uint, c_ulong) -> c_long, + f_ready: unsafe extern "C" fn(*mut c_void) -> c_int, + ) -> Result { + // SAFETY: `name` is a static NUL-terminated string that outlives the + // registration, and the caller guarantees the same of `ctx`. + let dev = unsafe { + sep_bio_register( + name.as_char_ptr(), + mode, + ctx, + Some(f_open), + Some(f_release), + Some(f_ioctl), + Some(f_ready), + ) + }; + if dev.is_null() { + return Err(ENODEV); + } + Ok(BioChardev { dev }) + } + + pub(crate) fn wake(&self) { + // SAFETY: `dev` is live per the type invariant. + unsafe { sep_bio_wake(self.dev) }; + } +} + +impl Drop for BioChardev { + fn drop(&mut self) { + // SAFETY: `dev` is live per the type invariant and not used again. + // Deregistration waits for open files, so no callback runs once this returns. + unsafe { sep_bio_unregister(self.dev) }; + } +} diff --git a/drivers/soc/apple/shmem.rs b/drivers/soc/apple/shmem.rs new file mode 100644 index 0000000000000..1d9a7ba724d00 --- /dev/null +++ b/drivers/soc/apple/shmem.rs @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +use kernel::device; +use kernel::dma; +use kernel::platform; +use kernel::prelude::*; + +pub(crate) const SHMEM_SIZE: usize = 0x40000; + +const ENTRY_SIZE: usize = 16; +const ENTRY_OFF_FOURCC: usize = 0; +const ENTRY_OFF_SIZE: usize = 4; +const ENTRY_OFF_OFFSET: usize = 8; + +// Buffer-format constant, not PAGE_SIZE despite the coincidence. +const PAYLOAD_BASE: usize = 0x4000; +const PAYLOAD_ALIGN: usize = 0x4000; + +const CINP_MIN_SIZE: usize = 0x8000; + +const CINP_PAYLOAD: [u8; 1] = [0]; + +// Wire byte order, not byte-reversed; llun is the terminator spelling. +const FOURCC_CINP: &[u8; 4] = b"CINP"; +const FOURCC_OPLA: &[u8; 4] = b"OPLA"; +const FOURCC_IPIS: &[u8; 4] = b"IPIS"; +const FOURCC_TERM: &[u8; 4] = b"llun"; + +const PROP_LOCAL_POLICY: &CStr = c"local-policy-manifest"; +const PROP_IBOOT: &CStr = c"iboot-manifest"; + +pub(crate) type ShMem = dma::Coherent<[u8]>; + +#[derive(Clone, Copy)] +pub(crate) struct Region { + pub(crate) offset: usize, + // Entry size field = allocation size, not payload length. + pub(crate) size: usize, + pub(crate) payload_len: usize, +} + +impl Region { + fn place(offset: usize, payload_len: usize, min_size: usize) -> Region { + let mut size = align_up(payload_len + 4, PAYLOAD_ALIGN); + if size < min_size { + size = min_size; + } + Region { + offset, + size, + payload_len, + } + } + + fn end(&self) -> usize { + self.offset + self.size + } +} + +pub(crate) struct Shmem { + pub(crate) buf: ShMem, +} + +const fn align_up(v: usize, a: usize) -> usize { + (v + a - 1) & !(a - 1) +} + +fn write_at(buf: &mut ShMem, off: usize, src: &[u8]) -> Result<()> { + let end = off.checked_add(src.len()).ok_or(EINVAL)?; + if end > SHMEM_SIZE { + return Err(EINVAL); + } + // SAFETY: runs in probe before the SEP is told the buffer exists, and probe + // is the only writer, so nothing else accesses it. + unsafe { + buf.as_mut()[off..end].copy_from_slice(src); + } + Ok(()) +} + +fn write_entry( + buf: &mut ShMem, + index: usize, + fourcc: &[u8; 4], + size: usize, + offset: usize, +) -> Result<()> { + let base = index * ENTRY_SIZE; + if base + ENTRY_SIZE > PAYLOAD_BASE { + return Err(EINVAL); + } + write_at(buf, base + ENTRY_OFF_FOURCC, fourcc)?; + let size32: u32 = size.try_into().map_err(|_| EINVAL)?; + write_at(buf, base + ENTRY_OFF_SIZE, &size32.to_le_bytes())?; + // u32 offset at byte 8 (written as u64 with zero pad), never byte 12. + let offset64: u64 = offset.try_into().map_err(|_| EINVAL)?; + write_at(buf, base + ENTRY_OFF_OFFSET, &offset64.to_le_bytes())?; + Ok(()) +} + +fn read_blob(dev: &device::Device, name: &CStr) -> Result> { + let fwnode = dev.fwnode().ok_or(EIO)?; + let len = fwnode.property_count_elem::(name)?; + if len == 0 { + return Err(ENODATA); + } + fwnode + .property_read_array_vec::(name, len)? + .required_by(dev) +} + +fn read_manifest(dev: &device::Device>, name: &CStr) -> Result> { + match read_blob(dev, name) { + Ok(v) => Ok(v), + Err(e) => { + dev_err!( + dev, + "refusing to attach: device-tree property '{}' missing or unreadable ({:?}); a CINP-only table would burn the one-shot registration and fault\n", + name, + e + ); + Err(e) + } + } +} + +fn verify_layout( + dev: &device::Device>, + cinp: &Region, + opla: &Region, + ipis: &Region, + used: usize, +) -> Result<()> { + let regions = [ + (FOURCC_CINP, cinp), + (FOURCC_OPLA, opla), + (FOURCC_IPIS, ipis), + ]; + + for (name, r) in regions { + let bad = r.offset < PAYLOAD_BASE + || r.offset % PAYLOAD_ALIGN != 0 + || r.size % PAYLOAD_ALIGN != 0 + || r.size < r.payload_len + 4 + || r.end() > SHMEM_SIZE; + if bad { + dev_err!( + dev, + "refusing to attach: item '{}' region is malformed (offset 0x{:x}, size 0x{:x}, payload {} bytes)\n", + core::str::from_utf8(name).unwrap_or("????"), + r.offset, + r.size, + r.payload_len + ); + return Err(EINVAL); + } + } + + if cinp.offset != PAYLOAD_BASE + || opla.offset != cinp.end() + || ipis.offset != opla.end() + || used != ipis.end() + { + dev_err!( + dev, + "refusing to attach: regions do not tile from 0x{:x} (CINP 0x{:x}+0x{:x}, OPLA 0x{:x}+0x{:x}, IPIS 0x{:x}+0x{:x}, used 0x{:x})\n", + PAYLOAD_BASE, + cinp.offset, + cinp.size, + opla.offset, + opla.size, + ipis.offset, + ipis.size, + used + ); + return Err(EINVAL); + } + + if cinp.size < CINP_MIN_SIZE { + return Err(EINVAL); + } + + if 4 * ENTRY_SIZE > PAYLOAD_BASE || used > SHMEM_SIZE { + return Err(ENOSPC); + } + + Ok(()) +} + +pub(crate) fn build(pdev: &platform::Device>) -> Result { + let dev: &device::Device> = pdev.as_ref(); + + // Read manifests before allocating: a CINP-only registration would burn the one-shot and fault. + let opla_blob = read_manifest(dev, PROP_LOCAL_POLICY)?; + let ipis_blob = read_manifest(dev, PROP_IBOOT)?; + + let cinp = Region::place(PAYLOAD_BASE, CINP_PAYLOAD.len(), CINP_MIN_SIZE); + let opla = Region::place(cinp.end(), opla_blob.len(), 0); + let ipis = Region::place(opla.end(), ipis_blob.len(), 0); + let used = ipis.end(); + + verify_layout(dev, &cinp, &opla, &ipis, used)?; + + let mut buf = dma::Coherent::::zeroed_slice(dev, SHMEM_SIZE, GFP_KERNEL)?; + + // Payloads before entries: a failure leaves an all-zero table, not a valid-looking one. + write_at( + &mut buf, + cinp.offset, + &(cinp.payload_len as u32).to_le_bytes(), + )?; + write_at(&mut buf, cinp.offset + 4, &CINP_PAYLOAD)?; + + write_at( + &mut buf, + opla.offset, + &(opla.payload_len as u32).to_le_bytes(), + )?; + write_at(&mut buf, opla.offset + 4, &opla_blob)?; + + write_at( + &mut buf, + ipis.offset, + &(ipis.payload_len as u32).to_le_bytes(), + )?; + write_at(&mut buf, ipis.offset + 4, &ipis_blob)?; + + write_entry(&mut buf, 0, FOURCC_CINP, cinp.size, cinp.offset)?; + write_entry(&mut buf, 1, FOURCC_OPLA, opla.size, opla.offset)?; + write_entry(&mut buf, 2, FOURCC_IPIS, ipis.size, ipis.offset)?; + write_entry(&mut buf, 3, FOURCC_TERM, 0, 0)?; + + Ok(Shmem { buf }) +} diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs new file mode 100644 index 0000000000000..f30f4870db076 --- /dev/null +++ b/drivers/soc/apple/sks.rs @@ -0,0 +1,1115 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj +//! SEP key store (SKS, endpoint `0x12`): key-bag and key-management request/reply +//! framing, DER-imaged request builders, and lock-state control. + +#![allow(dead_code)] + +use super::*; +use kernel::prelude::*; +use kernel::soc::apple::mailbox::Message; +use crate::proto::*; + +impl SepData { + pub(crate) fn on_sks(&self, msg: Message) { + let r = crate::sks::decode_sks(&msg); + let mut probe = self.sks_probe.lock(); + if probe.active { + if probe.captured.len() < SKS_MAX_CAPTURE { + let _ = probe.captured.push(msg, GFP_KERNEL); + } + drop(probe); + self.sks_wq.notify_all(); + } else { + probe.unsolicited = probe.unsolicited.wrapping_add(1); + let matched = probe + .abandoned + .iter() + .flatten() + .find(|a| a.selector == r.selector && a.seq == r.seq) + .copied(); + drop(probe); + if let Some(a) = matched { + self.sks_wedged.store(0, Relaxed); + dev_warn!(self.dev, "sks: late answer to {} after {} ms; wedge lifted\n", a.label, a.waited_ms); + } + } + } + + fn sks_note_abandoned(&self, selector: u8, seq: u8, label: &'static CStr, waited_ms: u64) { + let mut probe = self.sks_probe.lock(); + let slot = probe.abandoned_next; + probe.abandoned[slot] = Some(Abandoned { + selector, + seq, + label, + waited_ms, + }); + probe.abandoned_next = (slot + 1) % SKS_MAX_ABANDONED; + } + + fn sks_read_buffer(&self, inbound: bool, len: usize) -> Result> { + let guard = self.ool_sks.lock(); + let pair = guard.as_ref().ok_or(ENODEV)?; + let src = if inbound { + &pair.inbound + } else { + &pair.outbound + }; + if len > pair.allocated { + return Err(EINVAL); + } + let mut out = KVec::new(); + // SAFETY: the enclave writes this buffer then signals the mailbox, so it is + // quiescent when we read it; the driver is the only other accessor. + out.extend_from_slice(unsafe { &src.as_ref()[..len] }, GFP_KERNEL)?; + Ok(out) + } + + fn sks_zero_buffers(&self) -> Result<()> { + let mut guard = self.ool_sks.lock(); + let pair: &mut Option = &mut guard; + let buffers = pair.as_mut().ok_or(ENODEV)?; + // SAFETY: the enclave touches these only between a request word and its + // reply, and none is outstanding here; the driver is the only other accessor. + unsafe { + buffers.inbound.as_mut()[..buffers.allocated].fill(0); + buffers.outbound.as_mut()[..buffers.allocated].fill(0); + } + Ok(()) + } + + pub(crate) fn ool_registered(&self, slot: &Mutex>) -> bool { + slot.lock().as_ref().is_some_and(|b| b.registered) + } + + pub(crate) fn sks_next_seq(&self) -> crate::sks::Sequence { + let n = self.sks_seq.load(Relaxed); + self.sks_seq.store(n.wrapping_add(1), Relaxed); + crate::sks::Sequence::from_counter(n as u8) + } + + fn sks_exchange( + &self, + label: &'static CStr, + msg: Message, + img: &image::RequestImage, + ) -> Option { + self.sks_exchange_patient(label, msg, img, 0, true) + } + + fn sks_exchange_patient( + &self, + label: &'static CStr, + msg: Message, + img: &image::RequestImage, + floor_ms: time::Msecs, + condemn: bool, + ) -> Option { + if !self.ool_registered(&self.ool_sks) { + return None; + } + + if self.sks_wedged.load(Relaxed) != 0 { + return None; + } + + if self.sks_zero_buffers().is_err() { + return None; + } + if self.ool_write(&self.ool_sks, 0, img.as_slice()).is_err() { + return None; + } + + let sized_ms = sks_timeout_for(img.len()); + let timeout_ms = if sized_ms < floor_ms { + floor_ms + } else { + sized_ms + }; + self.sks_arm(label); + if self.send(msg).is_err() { + self.sks_disarm(); + return None; + } + let started_ns = crate::shim::boottime_ns(); + + let sent = crate::sks::decode_sks(&msg); + let mut examined = 0usize; + let mut set_aside = 0u32; + let correlating = loop { + self.sks_wait(timeout_ms, examined + 1); + + let next = { + let probe = self.sks_probe.lock(); + let m = probe.captured.get(examined).copied(); + if m.is_some() { + examined += 1; + } + m + }; + + let Some(candidate) = next else { break None }; + + let decoded = crate::sks::decode_sks(&candidate); + if decoded.seq == sent.seq && decoded.selector == sent.selector { + break Some(candidate); + } + + set_aside = set_aside.saturating_add(1); + if set_aside >= SKS_MAX_SET_ASIDE { + break None; + } + }; + + let leftover = self.sks_disarm(); + let correlating = match correlating { + Some(m) => Some(m), + None => leftover.iter().skip(examined).copied().find(|m| { + let d = crate::sks::decode_sks(m); + d.seq == sent.seq && d.selector == sent.selector + }), + }; + + let Some(raw) = correlating else { + let waited_ms = crate::shim::boottime_ns().saturating_sub(started_ns) / 1_000_000; + self.sks_note_abandoned(sent.selector, sent.seq, label, waited_ms); + if condemn { + self.sks_wedged.store(1, Relaxed); + } + dev_err!(self.dev, "sks: {} got no reply after {} ms\n", label, waited_ms); + return None; + }; + + let reply = crate::sks::decode_sks(&raw); + // reply.status is signed here on (e.g. -13, not 0xf3). + + let mut response = Secret::empty(); + if reply.response_size > 0 { + if let Ok(bytes) = self.sks_read_buffer(false, reply.response_size as usize) { + response = Secret(bytes); + } + } + + Some(SksOutcome { reply, response }) + } + + pub(crate) fn sks_report_response<'a>( + &self, + _label: &CStr, + out: &'a SksOutcome, + ) -> Option<&'a [u8]> { + if out.response.is_empty() { + return None; + } + + let parsed = match image::parse_response(&out.response) { + Ok(p) => p, + Err(_) => { + return None; + } + }; + Some(parsed.body) + } + + pub(crate) fn sks_timestamp_us(&self) -> u64 { + crate::shim::boottime_ns() / 1000 + } + + pub(crate) fn sks_image_len(&self, img: &image::RequestImage) -> Result { + let declared_in = sks_declared_sizes().0; + if img.len() > declared_in { + return Err(ENOSPC); + } + crate::sks::ImageLen::of(img.len()).ok_or(EINVAL) + } + + fn sks_seal(&self, op: &crate::sks::SksOp, body: &image::Body) -> Result { + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), body)?; + let len = self.sks_image_len(&img)?; + Ok(SksRequest { + name: op.name(), + msg: crate::sks::encode_sks_read(op, self.sks_next_seq(), len), + img, + }) + } + + /// `0x4d` get capabilities. + fn sks_req_get_capabilities(&self) -> Result { + let op = crate::sks::sks_get_capabilities(); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(1)?; + body.put_blob(&[])?; + self.sks_seal(&op, &body) + } + + /// `0x0d` designate. + fn sks_req_designate(&self, d: &crate::sks::Designation, secret: &[u8]) -> Result { + let mut body = image::Body::new(); + body.put_u32(crate::sks::SKS_DESIGNATE_VARIANT)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(d.source().value())?; + body.put_i32(d.user().special_handle().value())?; + body.put_blob(secret)?; + // Flags is a u64, not u32; a u32 leaves the body short -> enclave answers -13. + body.put_u64(crate::sks::SKS_DESIGNATE_FLAGS)?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_designate(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_DESIGNATE_NAME, + msg, + img, + }) + } + + /// `0x05` unload the source handle. + pub(crate) fn sks_req_unload_keybag(&self, handle: crate::sks::KeyBagHandle) -> Result { + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(handle.value())?; + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_unload_keybag(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_UNLOAD_NAME, + msg, + img, + }) + } + + /// `0x06` UUID read, against the special handle. + pub(crate) fn sks_req_copy_uuid_special(&self, special: crate::sks::SpecialHandle) -> Result { + let op = crate::sks::sks_copy_keybag_uuid(); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(special.value())?; + self.sks_seal(&op, &body) + } + + pub(crate) fn sks_req_unlock_special( + &self, + special: crate::sks::SpecialHandle, + secret: &[u8], + healthy: Healthy, + ) -> Result { + let Healthy(()) = healthy; + let mut body = image::Body::new(); + body.put_u32(SKS_LOCK_STATE_VARIANT)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(special.value())?; + body.put_i32(LockState::Unlocked.wire())?; + body.put_blob(secret)?; + body.put_u64(SKS_LOCK_STATE_FLAGS)?; + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_change_lock_state(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_LOCK_STATE_NAME, + msg, + img, + }) + } + + /// `0x06` copy key-bag UUID. + fn sks_req_copy_keybag_uuid(&self, handle: crate::sks::KeyBagHandle) -> Result { + let op = crate::sks::sks_copy_keybag_uuid(); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(handle.value())?; + self.sks_seal(&op, &body) + } + + /// `0x02` copy the designated biometric identity bag. + fn sks_req_copy_keybag_special(&self, handle: crate::sks::SpecialHandle) -> Result { + let op = crate::sks::sks_copy_keybag(); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(handle.value())?; + self.sks_seal(&op, &body) + } + + /// `0x03` load key bag. + fn sks_req_load_keybag(&self, wrapped: &[u8]) -> Result { + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_blob(wrapped)?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_load(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_LOAD_NAME, + msg, + img, + }) + } + + pub(crate) fn sks_send(&self, req: Result) -> Option { + match req { + Ok(r) => self.sks_exchange(r.name, r.msg, &r.img), + Err(_) => None, + } + } + + pub(crate) fn sks_designate_user_keybag(&self, handle: crate::sks::KeyBagHandle, secret: &[u8]) { + let Some(user) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) else { + return; + }; + + let designation = crate::sks::Designation::new(handle, user); + + let Some(out) = self.sks_send(self.sks_req_designate(&designation, secret)) else { + dev_warn!(self.dev, "sks: DESIGNATE_KEYBAG did not complete; enrolment will refuse\n"); + return; + }; + let Some(body) = + self.sks_report_response(crate::sks::SKS_DESIGNATE_NAME, &out) + else { + return; + }; + + if body.len() != crate::sks::SKS_DESIGNATE_REPLY_LEN { + return; + } + let variant = u32::from_le_bytes([body[0], body[1], body[2], body[3]]); + if variant != crate::sks::SKS_DESIGNATE_VARIANT { + return; + } + + self.keybag_designated.store(true, Relaxed); + + self.sks_remember_enrolment_material( + designation.user().special_handle(), + secret, + ); + } + + fn sks_remember_enrolment_material( + &self, + special: crate::sks::SpecialHandle, + secret: &[u8], + ) { + let mut copy = KVec::new(); + if copy.extend_from_slice(secret, GFP_KERNEL).is_err() { + return; + } + *self.enrol_material.lock() = Some(EnrolMaterial { + special, + secret: Secret(copy), + }); + } + + /// `0x04` change lock state (v1). + pub(crate) fn sks_req_change_lock_state( + &self, + handle: crate::sks::KeyBagHandle, + state: LockState, + secret: &[u8], + healthy: Healthy, + ) -> Result { + let Healthy(()) = healthy; + let mut body = image::Body::new(); + body.put_u32(SKS_LOCK_STATE_VARIANT)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(handle.value())?; + body.put_i32(state.wire())?; + body.put_blob(secret)?; + // Trailing u64 goes after the blob (`0x18` puts it before). + body.put_u64(SKS_LOCK_STATE_FLAGS)?; + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_change_lock_state(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_LOCK_STATE_NAME, + msg, + img, + }) + } + + /// `0x21` verify-secret (variant 1). + pub(crate) fn sks_req_verify_secret( + &self, + special: crate::sks::SpecialHandle, + secret: &[u8], + acm_context: &[u8; crate::scrd::SCRD_ACM_HANDLE_LEN], + ) -> Result { + let mut body = image::Body::new(); + body.put_u32(crate::sks::SKS_VERIFY_SECRET_VARIANT)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(special.value())?; + body.put_blob(secret)?; + body.put_blob(acm_context)?; + body.put_u64(0)?; + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_verify_secret(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_VERIFY_SECRET_NAME, + msg, + img, + }) + } + + pub(crate) fn sks_step( + &self, + label: &CStr, + build: impl FnOnce(Healthy) -> Result, + ) -> Option { + let healthy = self.sks_health_check(label)?; + let out = self.sks_send(build(healthy))?; + if out.reply.status != 0 { + return None; + } + Some(out) + } + + pub(crate) fn sks_health_check(&self, why: &CStr) -> Option { + let Some(out) = self.sks_send(self.sks_req_get_capabilities()) else { + dev_err!(self.dev, "sks: health check ({}) got no reply; endpoint gone for this boot\n", why); + return None; + }; + if out.reply.status != 0 { + dev_err!(self.dev, "sks: health check ({}) returned status {}\n", why, out.reply.status); + return None; + } + Some(Healthy(())) + } + + pub(crate) fn sks_recover( + &self, + stored: &keybag::StoredKeyBag, + ) -> Option<(crate::sks::KeyBagHandle, [u8; keybag::UUID_LEN])> { + let out = self.sks_send(self.sks_req_load_keybag(stored.wrapped()))?; + let body = self.sks_report_response(crate::sks::SKS_LOAD_NAME, &out)?; + + if out.reply.status != 0 { + dev_warn!( + self.dev, + "sks: LOAD_KEYBAG mailbox status {}; no handle\n", + out.reply.status + ); + return None; + } + if body.len() != SKS_LOAD_REPLY_LEN { + return None; + } + let status = i32::from_le_bytes([body[0], body[1], body[2], body[3]]); + let handle = i32::from_le_bytes([body[4], body[5], body[6], body[7]]); + if status != 0 || handle < 0 { + return None; + } + let handle = crate::sks::KeyBagHandle::from_load_reply(handle); + + let uuid = self.sks_read_uuid(handle)?; + + if uuid == *stored.uuid() { + Some((handle, uuid)) + } else { + dev_err!( + self.dev, + "sks: load succeeded but bag identity mismatch: stored UUID {}, loaded UUID {}\n", + Hex(stored.uuid()), + Hex(&uuid) + ); + None + } + } + + pub(crate) fn resnapshot_identity_keybag(&self) -> bool { + let special = { + let material = self.enrol_material.lock(); + let Some(material) = material.as_ref() else { + return false; + }; + material.special + }; + + let Some(out) = self.sks_send(self.sks_req_copy_keybag_special(special)) else { + return false; + }; + let Some(wrapped) = self.wrapped_from_copy_reply(&out, c"the designated identity bag") + else { + return false; + }; + + let Some(uuid) = self + .sks_send(self.sks_req_copy_uuid_special(special)) + .and_then(|out| self.sks_uuid_from_reply(&out)) + else { + return false; + }; + + match keybag::replace_wrapped(keybag::Slot::Identity, &wrapped, &uuid) { + Ok(()) => true, + Err(e) => { + dev_err!( + self.dev, + "enrol: could not commit the identity-bag snapshot ({:?}); may not survive reboot\n", + e + ); + false + } + } + } + + fn sks_read_uuid(&self, handle: crate::sks::KeyBagHandle) -> Option<[u8; keybag::UUID_LEN]> { + let out = self.sks_send(self.sks_req_copy_keybag_uuid(handle))?; + self.sks_uuid_from_reply(&out) + } + + pub(crate) fn sks_uuid_from_reply(&self, out: &SksOutcome) -> Option<[u8; keybag::UUID_LEN]> { + let body = self.sks_report_response(c"COPY_KEYBAG_UUID", out)?; + + if out.reply.status != 0 || image::operation_status(body).unwrap_or(-1) != 0 { + return None; + } + let (blob, _) = image::read_blob(body, 4)?; + if blob.len() != keybag::UUID_LEN { + return None; + } + let mut uuid = [0u8; keybag::UUID_LEN]; + uuid.copy_from_slice(blob); + Some(uuid) + } + + pub(crate) fn enable_sks(&self) -> Result<()> { + self.register_ool(&self.ool_sks) + } + + pub(crate) fn sks_ready(&self) -> bool { + if self.ool_registered(&self.ool_sks) { + return true; + } + if !self.endpoint_present(proto::EP_SKS) { + dev_err!( + self.dev, + "sks: endpoint 0x{:02x} (key store) was not advertised on this boot; keybag and ref-key operations cannot run\n", + proto::EP_SKS + ); + return false; + } + if let Err(e) = self.enable_sks() { + dev_err!(self.dev, "sks: could not register out-of-line buffers ({:?})\n", e); + return false; + } + true + } + + fn sks_arm(&self, label: &'static CStr) { + let mut probe = self.sks_probe.lock(); + probe.captured.clear(); + probe.label = Some(label); + probe.active = true; + } + + fn sks_disarm(&self) -> KVec { + let mut probe = self.sks_probe.lock(); + probe.active = false; + probe.label = None; + core::mem::take(&mut probe.captured) + } + + fn sks_wait(&self, ms: time::Msecs, until: usize) { + let mut remaining = time::msecs_to_jiffies(ms); + let mut guard = self.sks_probe.lock(); + loop { + if guard.captured.len() >= until || remaining == 0 { + return; + } + match self + .sks_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + } + } +} + +const SKS_SELECTOR_MAX: u8 = 0x5f; + +pub(crate) const SKS_REPLY_BIT: u8 = 0x80; + +const OP_SKS_REWRAP_FORBIDDEN: u8 = 0x0f; + +pub(crate) struct SksOp { + opcode: u8, + name: &'static CStr, +} + +impl SksOp { + pub(crate) fn opcode(&self) -> u8 { + self.opcode + } + pub(crate) fn name(&self) -> &'static CStr { + self.name + } +} + +const OP_SKS_DEVICE_STATE: u8 = 0x19; +pub(crate) fn sks_get_device_state() -> SksOp { + SksOp { + opcode: OP_SKS_DEVICE_STATE, + name: c"GET_DEVICE_STATE", + } +} + +const OP_SKS_GET_CONFIGURATION: u8 = 0x23; + +const OP_SKS_SET_CONFIGURATION: u8 = 0x24; +static_assert!(OP_SKS_SET_CONFIGURATION != OP_SKS_GET_CONFIGURATION); + +const OP_SKS_NEW_PFK: u8 = 0x10; +static_assert!(OP_SKS_NEW_PFK != 0x0f); +static_assert!(OP_SKS_NEW_PFK != 0x09); + +// FileVault seal order: 0x42 KEK, then 0x40 VEK, then 0x41 install; 0x47 clear forbidden +pub(crate) const OP_SKS_FV_NEW_VEK: u8 = 0x40; +pub(crate) const OP_SKS_FV_UNWRAP_VEK: u8 = 0x41; +pub(crate) const OP_SKS_FV_NEW_KEK: u8 = 0x42; + +const OP_SKS_GENERIC_OPERATION: u8 = 0x1a; +const OP_SKS_PERFORM_OPERATION: u8 = 0x22; +const OP_SKS_IDENTITY_OPERATION: u8 = 0x51; +pub(crate) const SKS_PERFORM_OP_NAME: &CStr = c"PERFORM_OPERATION"; +static_assert!(OP_SKS_PERFORM_OPERATION != OP_SKS_GET_CONFIGURATION); +static_assert!(OP_SKS_PERFORM_OPERATION != OP_SKS_SET_CONFIGURATION); + +pub(crate) fn encode_sks_perform_operation(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_PERFORM_OPERATION, seq.value(), len.value()); + Some(msg) +} + +pub(crate) fn encode_sks_fv(selector: u8, seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(selector, seq.value(), len.value()); + Some(msg) +} +const OP_SKS_LAST_USER_OPERATION: u8 = 0x53; +static_assert!(OP_SKS_LAST_USER_OPERATION != 0x56); + +const OP_SKS_CAPABILITIES: u8 = 0x4d; +pub(crate) fn sks_get_capabilities() -> SksOp { + SksOp { + opcode: OP_SKS_CAPABILITIES, + name: c"GET_CAPABILITIES", + } +} + +const OP_SKS_COPY_UUID: u8 = 0x06; +pub(crate) fn sks_copy_keybag_uuid() -> SksOp { + SksOp { + opcode: OP_SKS_COPY_UUID, + name: c"COPY_KEYBAG_UUID", + } +} + +const OP_SKS_COPY_KEYBAG: u8 = 0x02; +pub(crate) fn sks_copy_keybag() -> SksOp { + SksOp { + opcode: OP_SKS_COPY_KEYBAG, + name: c"COPY_KEYBAG", + } +} + +const OP_SKS_CREATE_KEYBAG: u8 = 0x01; + +const OP_SKS_LOAD_KEYBAG: u8 = 0x03; + +pub(crate) const SKS_LOAD_NAME: &CStr = c"LOAD_KEYBAG"; + +const OP_SKS_CHANGE_LOCK_STATE: u8 = 0x04; + +pub(crate) const SKS_LOCK_STATE_NAME: &CStr = c"CHANGE_LOCK_STATE"; + +const OP_SKS_TOKEN_CREATE: u8 = 0x1c; + +pub(crate) const SKS_TOKEN_CREATE_NAME: &CStr = c"AUTH_TOKEN_CREATE"; + +const OP_SKS_TOKEN_VERIFY: u8 = 0x1d; + +static_assert!(OP_SKS_TOKEN_VERIFY == OP_SKS_TOKEN_CREATE + 1); + +#[derive(Clone, Copy)] +pub(crate) struct NewDeviceState(i32); + +impl NewDeviceState { + pub(crate) const UNLOCKED: NewDeviceState = NewDeviceState(0); + + pub(crate) const fn value(&self) -> i32 { + self.0 + } +} +static_assert!(NewDeviceState::UNLOCKED.value() == 0); + +const OP_SKS_DEVICE_STATE_TRANSITION: u8 = 0x18; +static_assert!(OP_SKS_DEVICE_STATE_TRANSITION + 1 == 0x19); + +pub(crate) const SKS_DEVICE_STATE_REPLY_LEN: usize = 20; +static_assert!(SKS_DEVICE_STATE_REPLY_LEN == 4 + 8 + 8); + +const OP_SKS_VERIFY_SECRET: u8 = 0x21; +pub(crate) const SKS_VERIFY_SECRET_NAME: &CStr = c"VERIFY_SECRET"; +pub(crate) const SKS_VERIFY_SECRET_VARIANT: u32 = 1; + +pub(crate) fn encode_sks_verify_secret(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_VERIFY_SECRET, seq.value(), len.value()); + Some(msg) +} + +pub(crate) const SKS_VERIFY_SECRET_REPLY_LEN: usize = 12; +static_assert!(SKS_VERIFY_SECRET_REPLY_LEN == 4 + 8); + +const SKS_SELECTOR_BITS: u8 = 0x7f; + +static_assert!(SKS_SELECTOR_MAX <= SKS_SELECTOR_BITS); + +pub(crate) const SKS_CLIENT_ID: u64 = u64::from_be_bytes(*b"LINUXSKS"); +static_assert!(SKS_CLIENT_ID == 0x4c49_4e55_5853_4b53); + +pub(crate) const SKS_CLIENT_ID_SEALING: u64 = u64::from_be_bytes(*b"LINUXSLS"); +static_assert!(SKS_CLIENT_ID_SEALING == 0x4c49_4e55_5853_4c53); +static_assert!(SKS_CLIENT_ID_SEALING != SKS_CLIENT_ID); + +#[derive(Clone, Copy)] +pub(crate) struct KeyBagHandle(i32); + +impl KeyBagHandle { + + pub(crate) const fn from_load_reply(v: i32) -> KeyBagHandle { + KeyBagHandle(v) + } + + pub(crate) const fn value(&self) -> i32 { + self.0 + } +} + +const OP_SKS_DESIGNATE_KEYBAG: u8 = 0x0d; + +pub(crate) const SKS_DESIGNATE_VARIANT: u32 = 1; + +pub(crate) const SKS_DESIGNATE_VARIANT_GENERIC: u32 = 0; + +static_assert!(SKS_DESIGNATE_VARIANT == 1); +static_assert!(SKS_DESIGNATE_VARIANT_GENERIC == 0); +static_assert!(SKS_DESIGNATE_VARIANT != SKS_DESIGNATE_VARIANT_GENERIC); + +pub(crate) const SKS_CREATE_VARIANT_IDENTITY: u32 = 5; +static_assert!(SKS_CREATE_VARIANT_IDENTITY != 1); + +pub(crate) const SKS_IDENTITY_UUID_LEN: usize = 16; +static_assert!(SKS_IDENTITY_UUID_LEN == crate::sbio::IDENTITY_UUID_LEN); + +pub(crate) const SKS_IDENTITY_USER_ID: i32 = 1000; + +pub(crate) const SKS_DESIGNATE_USER_MIN: i32 = 10; +static_assert!(SKS_DESIGNATE_USER_MIN > 0); + +#[derive(Clone, Copy)] +pub(crate) struct DesignateUser(i32); + +impl DesignateUser { + pub(crate) fn new(value: i32) -> Option { + if value >= SKS_DESIGNATE_USER_MIN { + Some(DesignateUser(value)) + } else { + None + } + } + + pub(crate) const fn special_handle(&self) -> SpecialHandle { + SpecialHandle(-self.0) + } +} + +#[derive(Clone, Copy)] +pub(crate) struct SpecialHandle(i32); + +impl SpecialHandle { +} + +impl SpecialHandle { + pub(crate) const fn value(&self) -> i32 { + self.0 + } +} + +pub(crate) const SKS_AUTH_TOKEN_LEN: usize = 16; + +pub(crate) struct AuthToken([u8; SKS_AUTH_TOKEN_LEN]); + +impl AuthToken { + pub(crate) fn from_reply(body: &[u8]) -> Option { + if body.len() < 8 { + return None; + } + let status = i32::from_le_bytes([body[0], body[1], body[2], body[3]]); + if status != 0 { + return None; + } + let len = u32::from_le_bytes([body[4], body[5], body[6], body[7]]) as usize; + if len != SKS_AUTH_TOKEN_LEN || body.len() < 8 + len { + return None; + } + let mut out = [0u8; SKS_AUTH_TOKEN_LEN]; + out.copy_from_slice(&body[8..8 + len]); + Some(AuthToken(out)) + } + +} + +impl Drop for AuthToken { + fn drop(&mut self) { + for b in self.0.iter_mut() { + // SAFETY: a valid, uniquely borrowed byte; volatile so the wipe is not elided. + unsafe { core::ptr::write_volatile(b, 0) }; + } + } +} + +pub(crate) struct Designation { + source: KeyBagHandle, + user: DesignateUser, +} + +impl Designation { + pub(crate) fn new(source: KeyBagHandle, user: DesignateUser) -> Designation { + Designation { source, user } + } + + pub(crate) const fn source(&self) -> KeyBagHandle { + self.source + } + + pub(crate) const fn user(&self) -> DesignateUser { + self.user + } +} + +pub(crate) const SKS_DESIGNATE_REPLY_LEN: usize = 4; + +pub(crate) const SKS_DESIGNATE_FLAGS: u64 = 0; +pub(crate) const SKS_DESIGNATE_FLAGS_DEVICE_KEYBAG: u64 = 0x100; +static_assert!(SKS_DESIGNATE_FLAGS == 0); +static_assert!(SKS_DESIGNATE_FLAGS != SKS_DESIGNATE_FLAGS_DEVICE_KEYBAG); + +// keybag_flags sits at +0x64 in a 0x01 body +pub(crate) struct CreateFlags(u32); + +pub(crate) const SKS_KEYBAG_FLAG_MAX: u32 = 0xff; +static_assert!((SKS_KEYBAG_FLAG_MAX as u64) < SKS_DESIGNATE_FLAGS_DEVICE_KEYBAG); + +impl CreateFlags { + pub(crate) const fn new(bits: u32) -> Option { + if bits <= SKS_KEYBAG_FLAG_MAX { + Some(CreateFlags(bits)) + } else { + None + } + } + + pub(crate) const fn none() -> CreateFlags { + CreateFlags(0) + } + + pub(crate) const fn value(&self) -> u32 { + self.0 + } +} + +static_assert!(CreateFlags::new(SKS_DESIGNATE_FLAGS_DEVICE_KEYBAG as u32).is_none()); +static_assert!(CreateFlags::new(0x100).is_none()); +static_assert!(CreateFlags::new(0x101).is_none()); +static_assert!(CreateFlags::new(0x1ff).is_none()); +static_assert!(CreateFlags::new(u32::MAX).is_none()); +static_assert!(CreateFlags::new(0x80).is_some()); +static_assert!(CreateFlags::none().value() == 0); + +// device keybag = 0x0d with a u64 flags of 0x100 at +0x78 + +pub(crate) fn encode_sks_designate(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_DESIGNATE_KEYBAG, seq.value(), len.value()); + Some(msg) +} + +pub(crate) const SKS_DESIGNATE_NAME: &CStr = c"DESIGNATE_KEYBAG"; + +const OP_SKS_WRAP: u8 = 0x08; +const OP_SKS_UNWRAP: u8 = 0x45; + +// 0x08/0x45 carry no keybag-handle field + +const OP_SKS_CHANGE_SECRET: u8 = 0x07; +const OP_SKS_DRAIN_BACKUP_KEYS: u8 = 0x17; +const OP_SKS_ESCROW_ENABLE: u8 = 0x29; +const OP_SKS_ESCROW_CREATE: u8 = 0x13; +const OP_SKS_ESCROW_PERSIST: u8 = 0x2b; +// 0x13 reply is status only, no handle + +const OP_SKS_PUBLIC_BACKUP_HANDLE: u8 = 0x2d; +const OP_SKS_UNLOAD_PUBLIC_BACKUP: u8 = 0x2e; + +static_assert!(SKS_DESIGNATE_FLAGS_DEVICE_KEYBAG == 0x100); +static_assert!(SKS_DESIGNATE_FLAGS != SKS_DESIGNATE_FLAGS_DEVICE_KEYBAG); + +const OP_SKS_MAKE_BACKUP_BAG: u8 = 0x11; +const OP_SKS_SET_BACKUP_BAG: u8 = 0x0e; +const OP_SKS_BACKUP_WRAP: u8 = 0x54; +const OP_SKS_BACKUP_UNWRAP: u8 = 0x55; + +pub(crate) const SKS_SET_BACKUP_NAME: &CStr = c"SET_BACKUP_BAG"; + +pub(crate) fn encode_sks_set_backup_bag(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_SET_BACKUP_BAG, seq.value(), len.value()); + Some(msg) +} + +const OP_SKS_SET_ENV: u8 = 0x2a; + +pub(crate) const SKS_CLIENT_ID_ALT: u64 = u64::from_be_bytes(*b"LINUXSKT"); +static_assert!(SKS_CLIENT_ID_ALT != SKS_CLIENT_ID); +static_assert!(SKS_CLIENT_ID_ALT != SKS_CLIENT_ID_SEALING); +const fn client_id_byte_distance(a: u64, b: u64) -> u32 { + let (x, y) = (a.to_be_bytes(), b.to_be_bytes()); + let mut differing = 0; + let mut i = 0; + while i < 8 { + if x[i] != y[i] { + differing += 1; + } + i += 1; + } + differing +} +static_assert!(client_id_byte_distance(SKS_CLIENT_ID, SKS_CLIENT_ID_ALT) == 1); + +const OP_SKS_UNLOAD_KEYBAG: u8 = 0x05; +pub(crate) const SKS_UNLOAD_NAME: &CStr = c"UNLOAD_KEYBAG"; + +pub(crate) fn encode_sks_unload_keybag(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_UNLOAD_KEYBAG, seq.value(), len.value()); + Some(msg) +} + +#[derive(Clone, Copy)] +pub(crate) struct Sequence(u8); + +const SEQ_MIN: u8 = 0x60; + +impl Sequence { + pub(crate) const fn from_counter(n: u8) -> Sequence { + Sequence(SEQ_MIN | (n & !SEQ_MIN)) + } + + pub(crate) const fn value(&self) -> u8 { + self.0 + } +} + +pub(crate) const SEQ_SPACE: usize = 64; + +pub(crate) const SEQ_FIRST_REUSE: u8 = 32; + +static_assert!(Sequence::from_counter(0).value() != Sequence::from_counter(1).value()); +static_assert!( + Sequence::from_counter(0).value() == Sequence::from_counter(SEQ_FIRST_REUSE).value() +); +static_assert!(SEQ_SPACE == 64); + +const fn every_sequence_is_above_the_selector_range() -> bool { + let mut n = 0u8; + loop { + if Sequence::from_counter(n).value() <= SKS_SELECTOR_MAX { + return false; + } + if n == 0xff { + return true; + } + n += 1; + } +} +static_assert!(every_sequence_is_above_the_selector_range()); + +#[derive(Clone, Copy)] +pub(crate) struct ImageLen(u16); + +impl ImageLen { + pub(crate) fn of(len: usize) -> Option { + if len < crate::image::HEADER_WIRE { + return None; + } + match u16::try_from(len) { + Ok(v) => Some(ImageLen(v)), + Err(_) => None, + } + } + + pub(crate) const fn value(&self) -> u16 { + self.0 + } +} + +const fn encode_sks_raw(selector: u8, seq: u8, len: u16) -> Message { + Message { + msg0: (EP_SKS as u64) + | ((selector as u64) << MSG_TAG_SHIFT) + | ((seq as u64) << MSG_TYPE_SHIFT) + | ((len as u64) << 48), + msg1: 0, + } +} + +pub(crate) fn encode_sks_read(op: &SksOp, seq: Sequence, len: ImageLen) -> Message { + encode_sks_raw(op.opcode(), seq.value(), len.value()) +} + +pub(crate) fn encode_sks_load(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_LOAD_KEYBAG, seq.value(), len.value()); + Some(msg) +} + +pub(crate) fn encode_sks_change_lock_state(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_CHANGE_LOCK_STATE, seq.value(), len.value()); + Some(msg) +} + +pub(crate) fn encode_sks_token_create(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_TOKEN_CREATE, seq.value(), len.value()); + Some(msg) +} + +pub(crate) struct SksReply { + pub(crate) selector: u8, + pub(crate) seq: u8, + pub(crate) status: i8, + pub(crate) flags: u16, + pub(crate) response_size: u16, +} + +pub(crate) fn decode_sks(msg: &Message) -> SksReply { + let b = msg.msg0.to_le_bytes(); + SksReply { + selector: b[1] & !SKS_REPLY_BIT, + seq: b[2], + status: b[3] as i8, + flags: u16::from_le_bytes([b[4], b[5]]), + response_size: u16::from_le_bytes([b[6], b[7]]), + } +} + +pub(crate) const SKS_STATUS_MALFORMED: i8 = -13; +static_assert!(SKS_STATUS_MALFORMED as u8 == 0xf3); + +pub(crate) const SKS_STATUS_REFUSED: i8 = -19; +static_assert!(SKS_STATUS_REFUSED != SKS_STATUS_MALFORMED); +static_assert!(SKS_STATUS_REFUSED != 0); diff --git a/drivers/soc/apple/store.rs b/drivers/soc/apple/store.rs new file mode 100644 index 0000000000000..ef4140ec781dc --- /dev/null +++ b/drivers/soc/apple/store.rs @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Host-owned slotted backing store for the SEP's persistent records: a +//! compile-time path exclusive to this driver, re-initialised and reseeded on a +//! magic mismatch. Re-initialised is never served empty — an empty store halts the SEP. + +use crate::shim; +use kernel::prelude::*; + +pub(crate) const STORE_PATH: &CStr = c"/var/lib/apple-sep-state.bin"; + +const BLOCK_SIZE: usize = 0x8000; +const BLOCK_COUNT: usize = 72; +pub(crate) const STORE_SIZE: usize = BLOCK_SIZE * BLOCK_COUNT; +const SLOT_COUNT: usize = BLOCK_COUNT - 1; +pub(crate) const MAX_VALUE: usize = BLOCK_SIZE; + +const MAGIC: [u8; 16] = *b"APPLE-SEP-STOR01"; +const VERSION: u32 = 1; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct Key { + pub(crate) kind: u8, + pub(crate) uuid: [u8; 16], +} + +impl Key { + pub(crate) const fn new(kind: u8, uuid: [u8; 16]) -> Key { + Key { kind, uuid } + } + + pub(crate) const fn root(kind: u8) -> Key { + Key { + kind, + uuid: [0u8; 16], + } + } +} + +#[derive(Clone, Copy)] +struct Slot { + used: bool, + kind: u8, + uuid: [u8; 16], + len: u32, +} + +impl Slot { + const FREE: Slot = Slot { + used: false, + kind: 0, + uuid: [0; 16], + len: 0, + }; + + fn matches(&self, key: &Key) -> bool { + self.used && self.kind == key.kind && self.uuid == key.uuid + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Intent { + None, + Write { + slot: u16, + kind: u8, + }, + Delete { + slot: u16, + kind: u8, + }, +} + +const SB_MAGIC: usize = 0; +const SB_VERSION: usize = 16; +const SB_SLOT_COUNT: usize = 20; +const SB_BLOCK_SIZE: usize = 24; +const SB_GENERATION: usize = 28; +const SB_INTENT_KIND: usize = 36; +const SB_INTENT_SLOT: usize = 37; +const SB_INTENT_TYPE: usize = 39; +// In previously reserved space, so an older store reads it as zero (unseeded). +const SB_SEEDED: usize = 40; +const SB_SLOT_TABLE: usize = 64; +const SLOT_ENTRY_SIZE: usize = 24; + +const INTENT_NONE: u8 = 0; +const INTENT_WRITE: u8 = 1; +const INTENT_DELETE: u8 = 2; + +// Not internally locked; the caller holds a mutex around all access. +pub(crate) struct Store { + file: shim::StoreFile, + slots: [Slot; SLOT_COUNT], + generation: u64, + pub(crate) recovered: bool, + pub(crate) fresh: bool, + seeded: bool, +} + +fn le32(buf: &[u8], off: usize) -> u32 { + u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]]) +} + +fn le64(buf: &[u8], off: usize) -> u64 { + let mut v = [0u8; 8]; + v.copy_from_slice(&buf[off..off + 8]); + u64::from_le_bytes(v) +} + +impl Store { + pub(crate) fn open() -> Result { + let file = shim::StoreFile::open(STORE_PATH)?; + + let mut store = Store { + file, + slots: [Slot::FREE; SLOT_COUNT], + generation: 0, + recovered: false, + fresh: false, + seeded: false, + }; + + let size = store.file.size()?; + if size < STORE_SIZE as u64 { + store.create()?; + return Ok(store); + } + + let mut sb = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; + sb.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; + store.file.read_exact(0, &mut sb)?; + + if sb[SB_MAGIC..SB_MAGIC + 16] != MAGIC + || le32(&sb, SB_VERSION) != VERSION + || le32(&sb, SB_SLOT_COUNT) as usize != SLOT_COUNT + || le32(&sb, SB_BLOCK_SIZE) as usize != BLOCK_SIZE + { + store.create()?; + return Ok(store); + } + + store.generation = le64(&sb, SB_GENERATION); + store.seeded = sb[SB_SEEDED] != 0; + + for i in 0..SLOT_COUNT { + let off = SB_SLOT_TABLE + i * SLOT_ENTRY_SIZE; + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&sb[off + 2..off + 18]); + store.slots[i] = Slot { + used: sb[off] != 0, + kind: sb[off + 1], + uuid, + len: le32(&sb, off + 18), + }; + } + + // Crash recovery: writes are copy-on-write, so discard the intent, never complete it. + let intent_kind = sb[SB_INTENT_KIND]; + if intent_kind != INTENT_NONE { + store.recovered = true; + let slot = u16::from_le_bytes([sb[SB_INTENT_SLOT], sb[SB_INTENT_SLOT + 1]]) as usize; + if intent_kind == INTENT_WRITE && slot < SLOT_COUNT { + store.slots[slot] = Slot::FREE; + } + store.commit()?; + } + + Ok(store) + } + + fn create(&mut self) -> Result<()> { + self.slots = [Slot::FREE; SLOT_COUNT]; + self.generation = 1; + self.fresh = true; + self.seeded = false; + + let mut zero = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; + zero.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; + for b in 0..BLOCK_COUNT { + self.file.write_all((b * BLOCK_SIZE) as u64, &zero)?; + } + + self.commit()?; + Ok(()) + } + + fn commit(&mut self) -> Result<()> { + self.write_superblock(Intent::None)?; + self.file.sync() + } + + fn write_superblock(&mut self, intent: Intent) -> Result<()> { + let mut sb = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; + sb.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; + + sb[SB_MAGIC..SB_MAGIC + 16].copy_from_slice(&MAGIC); + sb[SB_VERSION..SB_VERSION + 4].copy_from_slice(&VERSION.to_le_bytes()); + sb[SB_SLOT_COUNT..SB_SLOT_COUNT + 4].copy_from_slice(&(SLOT_COUNT as u32).to_le_bytes()); + sb[SB_BLOCK_SIZE..SB_BLOCK_SIZE + 4].copy_from_slice(&(BLOCK_SIZE as u32).to_le_bytes()); + sb[SB_GENERATION..SB_GENERATION + 8].copy_from_slice(&self.generation.to_le_bytes()); + + let (ikind, islot, itype) = match intent { + Intent::None => (INTENT_NONE, 0u16, 0u8), + Intent::Write { slot, kind } => (INTENT_WRITE, slot, kind), + Intent::Delete { slot, kind } => (INTENT_DELETE, slot, kind), + }; + sb[SB_SEEDED] = u8::from(self.seeded); + sb[SB_INTENT_KIND] = ikind; + sb[SB_INTENT_SLOT..SB_INTENT_SLOT + 2].copy_from_slice(&islot.to_le_bytes()); + sb[SB_INTENT_TYPE] = itype; + + for (i, slot) in self.slots.iter().enumerate() { + let off = SB_SLOT_TABLE + i * SLOT_ENTRY_SIZE; + sb[off] = u8::from(slot.used); + sb[off + 1] = slot.kind; + sb[off + 2..off + 18].copy_from_slice(&slot.uuid); + sb[off + 18..off + 22].copy_from_slice(&slot.len.to_le_bytes()); + } + + self.file.write_all(0, &sb) + } + + fn find(&self, key: &Key) -> Option { + self.slots.iter().position(|s| s.matches(key)) + } + + fn find_free(&self) -> Option { + self.slots.iter().position(|s| !s.used) + } + + fn block_offset(slot: usize) -> u64 { + ((slot + 1) * BLOCK_SIZE) as u64 + } + + pub(crate) fn read(&mut self, key: &Key) -> Result>> { + let Some(idx) = self.find(key) else { + return Ok(None); + }; + let len = self.slots[idx].len as usize; + let mut value = KVec::with_capacity(len, GFP_KERNEL)?; + value.resize(len, 0, GFP_KERNEL)?; + if len > 0 { + self.file.read_exact(Self::block_offset(idx), &mut value)?; + } + Ok(Some(value)) + } + + pub(crate) fn write(&mut self, key: &Key, value: &[u8]) -> Result<()> { + if value.len() > MAX_VALUE { + return Err(ENOSPC); + } + + let old = self.find(key); + let fresh = self.find_free().ok_or(ENOSPC)?; + + // 1. Record the intent, durably, before touching any data block. + self.write_superblock(Intent::Write { + slot: fresh as u16, + kind: key.kind, + })?; + self.file.sync()?; + + // 2. Fill the new block and make it durable. + if !value.is_empty() { + self.file.write_all(Self::block_offset(fresh), value)?; + } + self.file.sync()?; + + // 3. Switch the slot table, release the old block, clear the intent. + self.slots[fresh] = Slot { + used: true, + kind: key.kind, + uuid: key.uuid, + len: value.len() as u32, + }; + if let Some(old) = old { + self.slots[old] = Slot::FREE; + } + self.generation = self.generation.wrapping_add(1); + self.commit() + } + + pub(crate) fn delete(&mut self, key: &Key) -> Result { + let Some(idx) = self.find(key) else { + return Ok(false); + }; + + self.write_superblock(Intent::Delete { + slot: idx as u16, + kind: key.kind, + })?; + self.file.sync()?; + + self.slots[idx] = Slot::FREE; + self.generation = self.generation.wrapping_add(1); + self.commit()?; + Ok(true) + } + + // Gates import, not store-emptiness: a store emptied after the SEP moved on must not be reseeded. + pub(crate) fn seeded(&self) -> bool { + self.seeded + } + + pub(crate) fn mark_seeded(&mut self) -> Result<()> { + self.seeded = true; + self.commit() + } + +} + +pub(crate) const fn crc16_ccitt_false(data: &[u8]) -> u16 { + let mut crc: u16 = 0xffff; + let mut i = 0; + while i < data.len() { + crc ^= (data[i] as u16) << 8; + let mut bit = 0; + while bit < 8 { + crc = if crc & 0x8000 != 0 { + (crc << 1) ^ 0x1021 + } else { + crc << 1 + }; + bit += 1; + } + i += 1; + } + crc +} + +// Canonical CCITT-FALSE check value; guards against a reflected/XOR variant. +static_assert!(crc16_ccitt_false(b"123456789") == 0x29B1); diff --git a/drivers/soc/apple/store_shim.c b/drivers/soc/apple/store_shim.c new file mode 100644 index 0000000000000..33ed552f54798 --- /dev/null +++ b/drivers/soc/apple/store_shim.c @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ +/* + * Backing-store file access. In C because the O_* flags are macro-derived and + * absent from the Rust bindings. + * + * These files are the driver's own. Opens run under kernel credentials so + * access never depends on which task triggered the operation: a keyctl(2) call + * from an unprivileged process reaches unseal in that process's context, and + * without this the driver could not read its own root-only store. + */ + +#include +#include +#include +#include +#include + +#include "shim.h" + +static struct file *open_as_kernel(const char *path, int flags, umode_t mode) +{ + const struct cred *old; + struct cred *kern; + struct file *f; + + kern = prepare_kernel_cred(&init_task); + if (!kern) + return ERR_PTR(-ENOMEM); + + old = override_creds(kern); + f = filp_open(path, flags, mode); + put_cred(revert_creds(old)); + + return f; +} + +/* Opens the store, creating if absent. Mode 0600: the file is the driver's own. */ +void *sep_store_open(const char *path) +{ + struct file *f = open_as_kernel(path, O_RDWR | O_CREAT | O_LARGEFILE, 0600); + + return IS_ERR(f) ? NULL : f; +} + +/* + * Opens truncated to empty, for callers that rewrite the whole file each time, + * so a shorter record cannot leave stale trailing bytes from a larger one. + */ +void *sep_store_open_trunc(const char *path) +{ + struct file *f = open_as_kernel(path, O_RDWR | O_CREAT | O_TRUNC | O_LARGEFILE, 0600); + + return IS_ERR(f) ? NULL : f; +} + +/* + * Opens an existing file read-only, NULL if absent. O_RDONLY and no O_CREAT so + * the seed cannot be created, truncated or written by mistake. + */ +void *sep_store_open_ro(const char *path) +{ + struct file *f = open_as_kernel(path, O_RDONLY | O_LARGEFILE, 0); + + return IS_ERR(f) ? NULL : f; +} + +void sep_store_close(void *handle) +{ + if (handle) + filp_close((struct file *)handle, NULL); +} + +/* Current length in bytes, or a negative errno. */ +long long sep_store_size(void *handle) +{ + struct file *f = handle; + + return i_size_read(file_inode(f)); +} + +/* Returns bytes read, 0 at end of file, or a negative errno. */ +long sep_store_read(void *handle, long long off, void *buf, size_t len) +{ + struct file *f = handle; + loff_t pos = off; + + return kernel_read(f, buf, len, &pos); +} + +/* Returns bytes written, or a negative errno. */ +long sep_store_write(void *handle, long long off, const void *buf, + size_t len) +{ + struct file *f = handle; + loff_t pos = off; + + return kernel_write(f, buf, len, &pos); +} + +/* Flushes data and metadata to durable storage. */ +int sep_store_sync(void *handle) +{ + return vfs_fsync((struct file *)handle, 0); +} diff --git a/drivers/soc/apple/transfer.rs b/drivers/soc/apple/transfer.rs new file mode 100644 index 0000000000000..67cf7ed13c312 --- /dev/null +++ b/drivers/soc/apple/transfer.rs @@ -0,0 +1,389 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! The generic transfer layer. + +use kernel::prelude::*; + +pub(crate) const MARKER_FIRST: u8 = 0xFC; +pub(crate) const MARKER_NEXT: u8 = 0xFD; +/// Requests the peer's next packet and acks the final one; same marker for both. +pub(crate) const MARKER_REQUEST: u8 = 0xFE; +pub(crate) const MARKER_ERROR: u8 = 0xFF; + +// A tag below MARKER_FIRST is not a marker; sound only while every marker >= it. +static_assert!(MARKER_NEXT >= MARKER_FIRST); +static_assert!(MARKER_REQUEST >= MARKER_FIRST); + +#[derive(Clone, Copy)] +pub(crate) enum DeviceStatus { + /// `0xffffffff` is a real refusal (signed −1), not the absence of an answer. + Answered(u32), + ReportedWithoutStatus, + NotAnswered, + BufferNeverWritten, +} + +impl DeviceStatus { + pub(crate) fn answered(&self) -> Option { + match self { + DeviceStatus::Answered(err) => Some(*err), + DeviceStatus::ReportedWithoutStatus + | DeviceStatus::NotAnswered + | DeviceStatus::BufferNeverWritten => None, + } + } + + pub(crate) fn is_ok(&self) -> bool { + matches!(self, DeviceStatus::Answered(0)) + } +} + +impl kernel::fmt::Display for DeviceStatus { + fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + match self { + DeviceStatus::Answered(err) => write!(f, "0x{:x}", err), + DeviceStatus::ReportedWithoutStatus => write!( + f, + "an error report from the device carrying no status word (the device DID answer)" + ), + DeviceStatus::NotAnswered => write!( + f, + "NONE — no message arrived at all (host sentinel, not an enclave value)" + ), + DeviceStatus::BufferNeverWritten => write!( + f, + "NONE — a payload notification arrived but the outbound buffer was never written (a race with the enclave's DMA, not a silent enclave)" + ), + } + } +} + +pub(crate) const HEADER_WORDS: usize = 7; +pub(crate) const HEADER_LEN: usize = HEADER_WORDS * 4; + +const VERSION: u32 = 1; + +pub(crate) const MAX_TRANSACTION: u32 = 0x4B000; + +#[derive(Clone, Copy)] +pub(crate) struct Packet { + pub(crate) version: u32, + pub(crate) total: u32, + pub(crate) offset: u32, + pub(crate) flags: u32, + pub(crate) err: u32, + pub(crate) opcode: u32, + pub(crate) chunk: u32, +} + +fn le32(buf: &[u8], word: usize) -> u32 { + let o = word * 4; + u32::from_le_bytes([buf[o], buf[o + 1], buf[o + 2], buf[o + 3]]) +} + +impl Packet { + pub(crate) fn decode(buf: &[u8]) -> Result { + if buf.len() < HEADER_LEN { + return Err(EINVAL); + } + Ok(Packet { + version: le32(buf, 0), + total: le32(buf, 1), + offset: le32(buf, 2), + flags: le32(buf, 3), + err: le32(buf, 4), + opcode: le32(buf, 5), + chunk: le32(buf, 6), + }) + } + + pub(crate) fn encode(&self, buf: &mut [u8]) -> Result<()> { + if buf.len() < HEADER_LEN { + return Err(EINVAL); + } + for (word, value) in [ + self.version, + self.total, + self.offset, + self.flags, + self.err, + self.opcode, + self.chunk, + ] + .into_iter() + .enumerate() + { + buf[word * 4..word * 4 + 4].copy_from_slice(&value.to_le_bytes()); + } + Ok(()) + } + + pub(crate) fn request_next(opcode: u32, received: u32, total: u32) -> Packet { + Packet { + version: VERSION, + total, + offset: received, + flags: 0, + err: 0, + opcode, + chunk: 0, + } + } +} + +struct Active { + opcode: u32, + total: u32, + payload: KVec, + seq: u16, +} + +pub(crate) struct Completed { + pub(crate) opcode: u32, + pub(crate) status: DeviceStatus, + pub(crate) payload: KVec, +} + +pub(crate) struct Continuation { + opcode: u32, + received: u32, + total: u32, + seq: u16, +} + +impl Continuation { + pub(crate) fn opcode(&self) -> u32 { + self.opcode + } + pub(crate) fn seq(&self) -> u16 { + self.seq + } + pub(crate) fn packet(&self) -> Packet { + Packet::request_next(self.opcode, self.received, self.total) + } +} + +pub(crate) enum Progress { + NeedMore(Continuation), + Complete, + /// A stray chunk, dropped without touching transfer state — distinct from + /// `Failed`, which kills the transfer. + Ignored(&'static CStr), + Grant, + Notification { tag: u8, opcode: u32 }, + Failed(&'static CStr), +} + +pub(crate) struct Reassembly { + active: Option, + done: Option, + sending: Option, + grants: u32, +} + +impl Reassembly { + pub(crate) fn new() -> Reassembly { + Reassembly { + active: None, + done: None, + sending: None, + grants: 0, + } + } + + pub(crate) fn begin(&mut self, opcode: u32) -> Result<()> { + if self.active.is_some() { + return Err(EBUSY); + } + self.done = None; + self.active = Some(Active { + opcode, + total: 0, + payload: KVec::new(), + seq: 0, + }); + Ok(()) + } + + pub(crate) fn awaiting(&self) -> Option { + self.active.as_ref().map(|a| a.opcode) + } + + pub(crate) fn begin_send(&mut self, opcode: u32) { + self.sending = Some(opcode); + self.grants = 0; + } + + pub(crate) fn end_send(&mut self) { + self.sending = None; + self.grants = 0; + } + + pub(crate) fn take_grant(&mut self) -> bool { + if self.grants > 0 { + self.grants -= 1; + return true; + } + false + } + + pub(crate) fn take_done(&mut self) -> Option { + self.done.take() + } + + pub(crate) fn abort(&mut self) { + self.abort_with(DeviceStatus::NotAnswered); + } + + pub(crate) fn abort_with(&mut self, status: DeviceStatus) { + if let Some(active) = self.active.take() { + self.done = Some(Completed { + opcode: active.opcode, + status, + payload: KVec::new(), + }); + } + } + + fn fail(&mut self) { + self.abort(); + } + + pub(crate) fn on_chunk(&mut self, marker: u8, packet: &Packet, payload: &[u8]) -> Progress { + if marker < MARKER_FIRST { + return Progress::Notification { + tag: marker, + opcode: packet.opcode, + }; + } + + // 0xFE is flow control for the request being sent, answered from `sending`. + if marker == MARKER_REQUEST { + return match self.sending { + Some(_) => { + // Counted under the caller's lock, closing the race with the waiter. + self.grants = self.grants.saturating_add(1); + Progress::Grant + } + None => Progress::Ignored(c"a 0xFE arrived with nothing being sent"), + }; + } + + let Some(active_opcode) = self.active.as_ref().map(|a| a.opcode) else { + return Progress::Ignored(c"no transfer outstanding"); + }; + + // Checked before the opcode test below, deliberately. + if marker == MARKER_ERROR { + let status = if packet.err != 0 { + DeviceStatus::Answered(packet.err) + } else { + DeviceStatus::ReportedWithoutStatus + }; + self.finish(status, KVec::new()); + return Progress::Complete; + } + + // The device echoes the opcode on a data chunk. + if packet.opcode != active_opcode { + return Progress::Ignored(c"chunk belongs to a different opcode"); + } + + if packet.version != VERSION { + self.fail(); + return Progress::Failed(c"header version is not 1"); + } + if packet.chunk as usize != payload.len() { + self.fail(); + return Progress::Failed(c"chunk length disagrees with the payload taken"); + } + if packet.total > MAX_TRANSACTION { + self.fail(); + return Progress::Failed(c"total length exceeds the maximum transaction size"); + } + + match marker { + MARKER_FIRST => { + let Some(active) = self.active.as_ref() else { + return Progress::Ignored(c"transfer vanished"); + }; + if !active.payload.is_empty() { + self.fail(); + return Progress::Failed(c"second first-chunk for one transfer"); + } + if packet.offset != 0 { + self.fail(); + return Progress::Failed(c"first chunk is not at offset zero"); + } + if let Some(active) = self.active.as_mut() { + active.total = packet.total; + } + } + MARKER_NEXT => { + let Some(active) = self.active.as_ref() else { + return Progress::Ignored(c"transfer vanished"); + }; + if packet.offset as usize != active.payload.len() { + self.fail(); + return Progress::Failed(c"continuation chunk is not at the expected offset"); + } + if packet.total != active.total { + self.fail(); + return Progress::Failed(c"continuation chunk changed the total length"); + } + } + _ => { + self.fail(); + return Progress::Failed(c"unexpected marker at or above 0xFC"); + } + } + + let Some(active) = self.active.as_mut() else { + return Progress::Ignored(c"transfer vanished"); + }; + + if packet.err != 0 { + let status = DeviceStatus::Answered(packet.err); + let collected = core::mem::take(&mut active.payload); + self.finish(status, collected); + return Progress::Complete; + } + + if active + .payload + .extend_from_slice(payload, GFP_KERNEL) + .is_err() + { + self.fail(); + return Progress::Failed(c"out of memory reassembling"); + } + + let received = active.payload.len() as u32; + let total = active.total; + + if received >= total { + let collected = core::mem::take(&mut active.payload); + self.finish(DeviceStatus::Answered(0), collected); + Progress::Complete + } else { + active.seq = active.seq.wrapping_add(1); + Progress::NeedMore(Continuation { + opcode: active_opcode, + received, + total, + seq: active.seq, + }) + } + } + + fn finish(&mut self, status: DeviceStatus, payload: KVec) { + let opcode = self.active.as_ref().map_or(0, |a| a.opcode); + self.active = None; + self.done = Some(Completed { + opcode, + status, + payload, + }); + } +} diff --git a/drivers/soc/apple/trusted.rs b/drivers/soc/apple/trusted.rs new file mode 100644 index 0000000000000..6162bc8af9d3c --- /dev/null +++ b/drivers/soc/apple/trusted.rs @@ -0,0 +1,268 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! A Linux trusted-key source backed by the SEP reference-key seal. +//! +//! Seal is a host-side ECIES encrypt to the machine ref-key's public point (no +//! enclave round trip). Unseal asks the enclave to ECIES-decrypt (op 0x22 +//! `"oecd"`) with the in-enclave private, which needs the login key bag reloaded +//! and unlocked first. The ref-key lifecycle lives on [`SepData`]; this is the +//! keyring glue. The config-dependent payload ABI is in `trusted_shim.c`. + +use core::sync::atomic::{AtomicPtr, Ordering}; + +use kernel::prelude::*; +use kernel::sync::Arc; +use kernel::types::ForeignOwnable; + +use crate::{Secret, SepData}; + +/// One opaque answer for every unseal rejection: telling a wrong blob, a foreign +/// machine's blob and a tampered blob apart would be an oracle. No name in the +/// Rust error set, hence the literal. +const EBADMSG: c_int = -74; + +/// The framework dispatches the ops with no context pointer, so `SepData` is +/// reached through this global. Holds the [`ForeignOwnable::into_foreign`] +/// pointer while registered; `null` otherwise, and the cmpxchg once-guard. +static SEP: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); + +extern "C" { + fn sep_tk_register( + init: unsafe extern "C" fn() -> c_int, + seal: unsafe extern "C" fn(*mut c_void, *mut c_char) -> c_int, + unseal: unsafe extern "C" fn(*mut c_void, *mut c_char) -> c_int, + random: unsafe extern "C" fn(*mut u8, usize) -> c_int, + exit: unsafe extern "C" fn(), + ) -> c_int; + fn sep_tk_unregister(); + + fn sep_tk_register_key_type() -> c_int; + fn sep_tk_unregister_key_type(); + + fn sep_tk_max_key_size() -> usize; + fn sep_tk_max_blob_size() -> usize; + + fn sep_tk_key_ptr(p: *mut c_void) -> *mut u8; + fn sep_tk_key_len(p: *const c_void) -> c_uint; + fn sep_tk_set_key_len(p: *mut c_void, n: c_uint); + + fn sep_tk_blob_ptr(p: *mut c_void) -> *mut u8; + fn sep_tk_blob_len(p: *const c_void) -> c_uint; + fn sep_tk_set_blob_len(p: *mut c_void, n: c_uint); +} + +/// Soundness rests on [`unregister`] resetting the framework's static calls +/// before it reclaims the `Arc`: no op runs after that, and while one runs the +/// pointer is live. +fn with_sep(f: impl FnOnce(&SepData) -> T) -> Option { + let ptr = SEP.load(Ordering::Acquire); + if ptr.is_null() { + return None; + } + // SAFETY: produced by `into_foreign` in `register`, reclaimed only by + // `unregister` after ops stop dispatching, so live for this borrow. + let borrow = unsafe { as ForeignOwnable>::borrow(ptr) }; + Some(f(&borrow)) +} + +/// # Safety +/// Called by the key-type core as the `init` op; takes no arguments. +unsafe extern "C" fn tk_init() -> c_int { + let rc = with_sep(|sep| { + dev_info!( + sep.dev, + "trusted-keys: registering \"trusted\" key type backed by the SEP ref-key seal\n" + ); + + // SAFETY: no preconditions; -EEXIST if already registered. + let krc = unsafe { sep_tk_register_key_type() }; + if krc != 0 { + dev_err!( + sep.dev, + "trusted-keys: register_key_type failed ({}); registration aborted\n", + krc + ); + } + krc + }); + rc.unwrap_or_else(|| ENODEV.to_errno()) +} + +/// `datablob` options (keyhandle/hash/policy) are ignored: no migration or +/// policy binding, so a key is bound only to the machine ref-key. +/// +/// # Safety +/// Called by the framework with a live `struct trusted_key_payload *p`. +unsafe extern "C" fn tk_seal(p: *mut c_void, _datablob: *mut c_char) -> c_int { + let rc = with_sep(|sep| { + // SAFETY: `p` is a live payload; key_len <= MAX_KEY_SIZE bytes at p->key. + let key_len = unsafe { sep_tk_key_len(p.cast_const()) } as usize; + // SAFETY: `p` is a live payload. + let key_ptr = unsafe { sep_tk_key_ptr(p) }; + // SAFETY: `key_ptr`/`key_len` describe `p->key`; read-only. + let key = unsafe { core::slice::from_raw_parts(key_ptr.cast_const(), key_len) }; + + let blob = match sep.refkey_seal_trusted(key) { + Ok(blob) => blob, + Err(e) => { + dev_warn!( + sep.dev, + "trusted-keys: seal failed ({:?}); the machine ref-key may not be established yet\n", + e + ); + return e.to_errno(); + } + }; + + // SAFETY: no preconditions. + let max_blob = unsafe { sep_tk_max_blob_size() }; + if blob.len() > max_blob { + dev_warn!( + sep.dev, + "trusted-keys: sealed blob {} byte(s) exceeds MAX_BLOB_SIZE {} byte(s); refusing\n", + blob.len(), + max_blob + ); + return E2BIG.to_errno(); + } + + // SAFETY: `p` is a live payload. + let dst = unsafe { sep_tk_blob_ptr(p) }; + // SAFETY: `dst` is `p->blob` (MAX_BLOB_SIZE bytes), blob.len() <= max_blob, + // no overlap. + unsafe { core::ptr::copy_nonoverlapping(blob.as_ptr(), dst, blob.len()) }; + // SAFETY: `p` is live. + unsafe { sep_tk_set_blob_len(p, blob.len() as c_uint) }; + + // Lengths only, never the key or blob bytes. + dev_info!( + sep.dev, + "trusted-keys: sealed {}-byte key into {}-byte enclave blob\n", + key_len, + blob.len() + ); + 0 + }); + rc.unwrap_or_else(|| ENODEV.to_errno()) +} + +/// # Safety +/// Called by the framework with a live `struct trusted_key_payload *p`. +unsafe extern "C" fn tk_unseal(p: *mut c_void, _datablob: *mut c_char) -> c_int { + let rc = with_sep(|sep| { + // SAFETY: `p` is a live payload; blob_len <= MAX_BLOB_SIZE bytes at p->blob. + let blob_len = unsafe { sep_tk_blob_len(p.cast_const()) } as usize; + // SAFETY: `p` is a live payload. + let blob_ptr = unsafe { sep_tk_blob_ptr(p) }; + // SAFETY: `blob_ptr`/`blob_len` describe `p->blob`; read-only. + let blob = unsafe { core::slice::from_raw_parts(blob_ptr.cast_const(), blob_len) }; + + // Every failure past here is one opaque EBADMSG. + let Ok(plain) = sep.refkey_unseal_trusted(blob) else { + return EBADMSG; + }; + let plain = Secret(plain); + + // SAFETY: no preconditions. + let max_key = unsafe { sep_tk_max_key_size() }; + if plain.is_empty() || plain.len() > max_key { + return EBADMSG; + } + + // SAFETY: `p` is a live payload. + let dst = unsafe { sep_tk_key_ptr(p) }; + // SAFETY: `dst` is `p->key` (MAX_KEY_SIZE + 1 bytes), plain.len() <= max_key, + // no overlap. + unsafe { core::ptr::copy_nonoverlapping(plain.as_ptr(), dst, plain.len()) }; + // SAFETY: `p` is live. + unsafe { sep_tk_set_key_len(p, plain.len() as c_uint) }; + + dev_info!( + sep.dev, + "trusted-keys: unsealed {}-byte enclave blob to {}-byte key\n", + blob_len, + plain.len() + ); + 0 + }); + rc.unwrap_or_else(|| ENODEV.to_errno()) +} + +/// Returns the byte count on success (not 0) per the framework contract, so the +/// caller's `ret != key_len` check passes. +/// +/// # Safety +/// Called by the framework with `key` writable for `key_len` bytes. +unsafe extern "C" fn tk_get_random(key: *mut u8, key_len: usize) -> c_int { + let rc = with_sep(|sep| { + if key_len == 0 { + return 0; + } + // SAFETY: the framework guarantees `key` is writable for `key_len` bytes. + let buf = unsafe { core::slice::from_raw_parts_mut(key, key_len) }; + match sep.sep_random(buf) { + Ok(()) => key_len as c_int, + Err(e) => e.to_errno(), + } + }); + rc.unwrap_or_else(|| ENODEV.to_errno()) +} + +/// # Safety +/// Called by the framework at most once per successful `init()`. +unsafe extern "C" fn tk_exit() { + // SAFETY: no preconditions; the key type was registered by `tk_init`. + unsafe { sep_tk_unregister_key_type() }; +} + +/// Idempotent: the cmpxchg claims the global slot once. On any failure the `Arc` +/// is reclaimed and the slot left clear, so a later call can retry. +pub(crate) fn register(sep: Arc) -> Result<()> { + let ptr = sep.into_foreign(); + + if SEP + .compare_exchange( + core::ptr::null_mut(), + ptr, + Ordering::AcqRel, + Ordering::Acquire, + ) + .is_err() + { + // SAFETY: `ptr` was just produced by `into_foreign` and never published. + drop(unsafe { as ForeignOwnable>::from_foreign(ptr) }); + return Err(EBUSY); + } + + // SAFETY: the shim stores these callbacks and calls + // `register_trusted_key_source`; valid for the module's lifetime. + let rc = unsafe { sep_tk_register(tk_init, tk_seal, tk_unseal, tk_get_random, tk_exit) }; + if rc != 0 { + // Nothing was wired, so no op can dispatch; clear the slot and reclaim. + let raw = SEP.swap(core::ptr::null_mut(), Ordering::AcqRel); + if !raw.is_null() { + // SAFETY: `raw` is the pointer we published; the ops are not wired. + drop(unsafe { as ForeignOwnable>::from_foreign(raw) }); + } + return Err(Error::from_errno(rc)); + } + Ok(()) +} + +/// Safe to call even if [`register`] never ran. Resets the framework's static +/// calls and runs `exit()` before reclaiming the `Arc`, so no op is in flight +/// against freed data once this returns. +pub(crate) fn unregister() { + if SEP.load(Ordering::Acquire).is_null() { + return; + } + // SAFETY: no preconditions; matched with the `sep_tk_register` above. + unsafe { sep_tk_unregister() }; + + let ptr = SEP.swap(core::ptr::null_mut(), Ordering::AcqRel); + if !ptr.is_null() { + // SAFETY: `ptr` came from `into_foreign` in `register`, and no op runs now. + drop(unsafe { as ForeignOwnable>::from_foreign(ptr) }); + } +} diff --git a/drivers/soc/apple/trusted_shim.c b/drivers/soc/apple/trusted_shim.c new file mode 100644 index 0000000000000..ed34794031f47 --- /dev/null +++ b/drivers/soc/apple/trusted_shim.c @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Dj */ +/* + * Apple SEP driver — trusted-key source shim (device-rooted sealing wired to + * the kernel trusted-key framework). + * + * An ABI the Rust bindings do not cover. 's struct + * trusted_key_payload is sized by MAX_KEY_SIZE / MAX_BLOB_SIZE and embeds an + * rcu_head, so its layout moves with the kernel config; and + * register_trusted_key_source / unregister_trusted_key_source, key_type_trusted + * and register_key_type() are C symbols absent from the Rust bindings. + * + * No sealing policy lives here: this file owns the struct handling and the + * registration only. init/seal/unseal/get_random/exit live in trusted.rs and + * are reached through the callbacks stored below. No key material is touched. + */ + +#include +#include +#include +#include + +#include "shim.h" + +/* + * register_trusted_key_source / unregister_trusted_key_source are declared by + * and exported by the trusted-key core. Not re-declared + * here: restating them risks a conflicting type (unregister returns void, not + * int). register validates src->{name,ops->{init,seal,unseal}}, honours + * trusted.source=, returns -EBUSY if another source is active, calls + * src->ops->init(), then wires the static calls; unregister resets those static + * calls, runs src->ops->exit(), and clears the active source. + */ + +/* Rust callbacks (trusted.rs); the sep_tk_*_fn types are declared in shim.h. */ +static sep_tk_init_fn rs_init; +static sep_tk_seal_fn rs_seal; +static sep_tk_unseal_fn rs_unseal; +static sep_tk_random_fn rs_random; +static sep_tk_exit_fn rs_exit; + +/* + * Trampolines for the framework's ops table, each forwarding to the stored Rust + * callback. Keeping these pointers in C means no Rust symbol is exported by + * name and the ops table stays a plain static initialiser. + */ +static int tk_init(void) +{ + return rs_init ? rs_init() : -ENODEV; +} + +static int tk_seal(struct trusted_key_payload *p, char *datablob) +{ + return rs_seal ? rs_seal(p, datablob) : -ENODEV; +} + +static int tk_unseal(struct trusted_key_payload *p, char *datablob) +{ + return rs_unseal ? rs_unseal(p, datablob) : -ENODEV; +} + +static int tk_get_random(unsigned char *key, size_t key_len) +{ + return rs_random ? rs_random(key, key_len) : -ENODEV; +} + +static void tk_exit(void) +{ + if (rs_exit) + rs_exit(); +} + +/* + * A writable array, not a string literal, so .name assigns cleanly whether the + * framework declares it char * or const char *. + */ +static char sep_tk_name[] = "applesep"; + +static struct trusted_key_ops sep_tk_ops = { + .migratable = 0, + .init = tk_init, + .seal = tk_seal, + .unseal = tk_unseal, + .get_random = tk_get_random, + .exit = tk_exit, +}; + +static struct trusted_key_source sep_tk_source = { + .name = sep_tk_name, + .ops = &sep_tk_ops, +}; + +/* + * Store the callbacks, then register. They must be in place first because + * register_trusted_key_source() calls init() synchronously. + */ +int sep_tk_register(sep_tk_init_fn init, sep_tk_seal_fn seal, + sep_tk_unseal_fn unseal, sep_tk_random_fn random, + sep_tk_exit_fn exit) +{ + rs_init = init; + rs_seal = seal; + rs_unseal = unseal; + rs_random = random; + rs_exit = exit; + return register_trusted_key_source(&sep_tk_source); +} + +void sep_tk_unregister(void) +{ + unregister_trusted_key_source(&sep_tk_source); + rs_init = NULL; + rs_seal = NULL; + rs_unseal = NULL; + rs_random = NULL; + rs_exit = NULL; +} + +/* + * Key-type lifecycle, owned by the source's init()/exit(), as the in-tree + * trusted-key sources (dcp, pkwm, tpm) do. + */ +int sep_tk_register_key_type(void) +{ + return register_key_type(&key_type_trusted); +} + +void sep_tk_unregister_key_type(void) +{ + unregister_key_type(&key_type_trusted); +} + +/* Compile-time facts the Rust side bounds-checks against. */ +size_t sep_tk_max_key_size(void) +{ + return MAX_KEY_SIZE; +} + +size_t sep_tk_max_blob_size(void) +{ + return MAX_BLOB_SIZE; +} + +/* + * Payload accessors. The struct layout tracks MAX_KEY_SIZE / MAX_BLOB_SIZE, so + * these wrappers keep the offsets in C where the header defines them. + */ +unsigned char *sep_tk_key_ptr(struct trusted_key_payload *p) +{ + return p->key; +} + +unsigned int sep_tk_key_len(const struct trusted_key_payload *p) +{ + return p->key_len; +} + +void sep_tk_set_key_len(struct trusted_key_payload *p, unsigned int n) +{ + p->key_len = n; +} + +unsigned char *sep_tk_blob_ptr(struct trusted_key_payload *p) +{ + return p->blob; +} + +unsigned int sep_tk_blob_len(const struct trusted_key_payload *p) +{ + return p->blob_len; +} + +void sep_tk_set_blob_len(struct trusted_key_payload *p, unsigned int n) +{ + p->blob_len = n; +} diff --git a/drivers/soc/apple/xarm.rs b/drivers/soc/apple/xarm.rs new file mode 100644 index 0000000000000..1c39467149b41 --- /dev/null +++ b/drivers/soc/apple/xarm.rs @@ -0,0 +1,365 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! The persistent-state service, `xarm`, endpoint `0x13`. + +#![allow(dead_code)] + +use crate::xarm::{XarmReply as Reply, XarmRequest as Request}; +use kernel::soc::apple::mailbox::Message; +use crate::store::{crc16_ccitt_false, Key, Store, MAX_VALUE}; +use kernel::prelude::*; + +pub(crate) use crate::proto::EP_XARM; +pub(crate) use crate::proto::EP_XARS; + +const OP_ROOT_READ: u8 = 0x00; +const OP_ROOT_WRITE: u8 = 0x01; +const OP_SESSION_READ: u8 = 0x05; +const OP_SESSION_WRITE: u8 = 0x06; +const OP_SESSION_DELETE: u8 = 0x07; +pub(crate) const OP_QUERY_PROTECTED: u8 = 0x0e; +const OP_GET_OS_UUID: u8 = 0x13; +const OP_NOTIFY_DISABLE_FIRST: u8 = 0x1d; +const OP_NOTIFY_DISABLE_LAST: u8 = 0x1f; + +pub(crate) const STATUS_OK: u8 = 0x00; +const STATUS_UNAVAILABLE: u8 = 0x02; +pub(crate) const STATUS_FAILED: u8 = 0x16; + +const ROOT_TYPE_BASE: u8 = 1; +const SESSION_TYPE_BASE: u8 = 3; + +/// Root read prefixes its value with seventeen zero bytes. +const ROOT_READ_PREFIX: usize = 17; + +/// `0x01` rejects when `(length >> 4) >= 0x7ff`. +const ROOT_WRITE_MAX: usize = 0x7fef; + +pub(crate) const PRIVATE_TYPE_OS_UUID: u8 = 0xF0; + +impl Reply { + fn ok(req: &Request, length: u16) -> Reply { + Reply { + tag: req.tag, + status: STATUS_OK, + length, + args: [0; 3], + } + } + + fn fail(req: &Request, status: u8) -> Reply { + Reply { + tag: req.tag, + status, + length: 0, + args: [0; 3], + } + } +} + +pub(crate) fn is_silent(opcode: u8) -> bool { + (OP_NOTIFY_DISABLE_FIRST..=OP_NOTIFY_DISABLE_LAST).contains(&opcode) +} + +pub(crate) fn needs_buffers(opcode: u8) -> bool { + opcode != OP_QUERY_PROTECTED +} + +pub(crate) fn opcode_name(opcode: u8) -> &'static CStr { + match opcode { + OP_ROOT_READ => c"ROOT_READ", + OP_ROOT_WRITE => c"ROOT_WRITE", + OP_SESSION_READ => c"SESSION_READ", + OP_SESSION_WRITE => c"SESSION_WRITE", + OP_SESSION_DELETE => c"SESSION_DELETE", + OP_QUERY_PROTECTED => c"QUERY_PROTECTED", + OP_GET_OS_UUID => c"GET_OS_UUID", + OP_NOTIFY_DISABLE_FIRST..=OP_NOTIFY_DISABLE_LAST => c"NOTIFY_DISABLE", + _ => c"UNKNOWN", + } +} + +fn root_key(args: &[u8; 3]) -> Key { + Key::root(ROOT_TYPE_BASE + (args[0] & 1)) +} + +fn session_type(args: &[u8; 3]) -> u8 { + SESSION_TYPE_BASE + (args[0] & 1) +} + +fn expected_crc(args: &[u8; 3]) -> u16 { + u16::from_le_bytes([args[1], args[2]]) +} + +fn uuid_from(bytes: &[u8]) -> [u8; 16] { + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&bytes[..16]); + uuid +} + +pub(crate) struct Serviced { + pub(crate) reply: Reply, + pub(crate) reply_bytes: usize, +} + +pub(crate) fn service( + req: &Request, + outbound: &[u8], + inbound: &mut [u8], + store: &mut Store, + protected_data: bool, + os_uuid: Option<[u8; 16]>, +) -> Serviced { + match req.opcode { + OP_ROOT_READ => { + let key = root_key(&req.args); + let value = match store.read(&key) { + Ok(v) => v, + Err(_) => { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + } + } + }; + + let Some(value) = value else { + return Serviced { + reply: Reply::ok(req, 0), + reply_bytes: 0, + }; + }; + let total = ROOT_READ_PREFIX + value.len(); + if total > inbound.len() { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + inbound[..ROOT_READ_PREFIX].fill(0); + inbound[ROOT_READ_PREFIX..total].copy_from_slice(&value); + + Serviced { + reply: Reply::ok(req, total as u16), + reply_bytes: total, + } + } + + OP_ROOT_WRITE => { + if (req.length as usize) > ROOT_WRITE_MAX || outbound.len() > MAX_VALUE { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + if crc16_ccitt_false(outbound) != expected_crc(&req.args) { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + match store.write(&root_key(&req.args), outbound) { + Ok(()) => Serviced { + reply: Reply::ok(req, 0), + reply_bytes: 0, + }, + Err(_) => Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }, + } + } + + OP_SESSION_READ => { + if outbound.len() < 16 { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + let key = Key::new(session_type(&req.args), uuid_from(outbound)); + match store.read(&key) { + Ok(Some(value)) => { + if value.len() > inbound.len() { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + inbound[..value.len()].copy_from_slice(&value); + Serviced { + reply: Reply::ok(req, value.len() as u16), + reply_bytes: value.len(), + } + } + Ok(None) => Serviced { + reply: Reply::ok(req, 0), + reply_bytes: 0, + }, + Err(_) => Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }, + } + } + + OP_SESSION_WRITE => { + if outbound.len() < 16 { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + if crc16_ccitt_false(outbound) != expected_crc(&req.args) { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + let key = Key::new(session_type(&req.args), uuid_from(outbound)); + match store.write(&key, &outbound[16..]) { + Ok(()) => Serviced { + reply: Reply::ok(req, 0), + reply_bytes: 0, + }, + Err(_) => Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }, + } + } + + OP_SESSION_DELETE => { + if outbound.len() < 16 { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + let key = Key::new(session_type(&req.args), uuid_from(outbound)); + match store.delete(&key) { + Ok(true) => Serviced { + reply: Reply::ok(req, 0), + reply_bytes: 0, + }, + Ok(false) => Serviced { + reply: Reply::ok(req, 0), + reply_bytes: 0, + }, + Err(_) => Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }, + } + } + + OP_QUERY_PROTECTED => { + let mut reply = Reply::ok(req, 0); + reply.args[0] = u8::from(protected_data); + Serviced { + reply, + reply_bytes: 0, + } + } + + OP_GET_OS_UUID => match os_uuid { + Some(uuid) => { + if inbound.len() < 16 { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + inbound[..16].copy_from_slice(&uuid); + // Reply length must echo the request, not the 16 flushed bytes; setting 16 is wrong. + Serviced { + reply: Reply::ok(req, req.length), + reply_bytes: 16, + } + } + None => Serviced { + reply: Reply::fail(req, STATUS_UNAVAILABLE), + reply_bytes: 0, + }, + }, + + _ => Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }, + } +} + +pub(crate) fn make_uuid_v4(bytes: &mut [u8; 16]) { + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; +} + +pub(crate) fn os_uuid_key() -> Key { + Key::root(PRIVATE_TYPE_OS_UUID) +} + +pub(crate) struct XarmRequest { + pub(crate) tag: u8, + pub(crate) opcode: u8, + pub(crate) length: u16, + pub(crate) args: [u8; 3], +} + +pub(crate) struct XarmReply { + pub(crate) tag: u8, + pub(crate) status: u8, + pub(crate) length: u16, + pub(crate) args: [u8; 3], +} + +pub(crate) fn decode_xarm(msg: &Message) -> XarmRequest { + let b = msg.msg0.to_le_bytes(); + XarmRequest { + tag: b[1], + opcode: b[2], + length: u16::from_le_bytes([b[3], b[4]]), + args: [b[5], b[6], b[7]], + } +} + +pub(crate) fn encode_xarm_reply(reply: &XarmReply) -> Message { + let len = reply.length.to_le_bytes(); + Message { + msg0: u64::from_le_bytes([ + EP_XARM, + reply.tag, + reply.status, + len[0], + len[1], + reply.args[0], + reply.args[1], + reply.args[2], + ]), + msg1: 0, + } +} + +static_assert!(EP_XARM == 0x13); + +const OP_XARS_SETUP_OS_SESSION: u8 = 0x08; + +const OP_XARS_FETCH_KNOWN_SESSIONS: u8 = 0x04; + +pub(crate) struct XarsReply { + pub(crate) tag: u8, + pub(crate) status: u8, +} + +pub(crate) fn decode_xars_reply(msg: &Message) -> XarsReply { + let b = msg.msg0.to_le_bytes(); + XarsReply { + tag: b[1], + status: b[2], + } +} + +static_assert!(EP_XARS == 0x10); +static_assert!(OP_XARS_SETUP_OS_SESSION == 0x08); +static_assert!(OP_XARS_FETCH_KNOWN_SESSIONS == 0x04); diff --git a/include/keys/trusted-type.h b/include/keys/trusted-type.h index 9f9940482da43..e95387b11f925 100644 --- a/include/keys/trusted-type.h +++ b/include/keys/trusted-type.h @@ -83,6 +83,9 @@ struct trusted_key_source { extern struct key_type key_type_trusted; +int register_trusted_key_source(struct trusted_key_source *src); +void unregister_trusted_key_source(struct trusted_key_source *src); + #ifdef CONFIG_TRUSTED_KEYS_DEBUG extern bool trusted_debug; diff --git a/security/keys/trusted-keys/trusted_core.c b/security/keys/trusted-keys/trusted_core.c index 0509d9955f2a6..e43e95009fbc5 100644 --- a/security/keys/trusted-keys/trusted_core.c +++ b/security/keys/trusted-keys/trusted_core.c @@ -67,6 +67,126 @@ DEFINE_STATIC_CALL_NULL(trusted_key_get_random, static void (*trusted_key_exit)(void); static unsigned char migratable __ro_after_init; +static DEFINE_MUTEX(trusted_key_source_lock); +static bool trusted_key_source_active; +static struct trusted_key_source *trusted_key_source_registered; + +/* Defined below; registration needs it before its definition. */ +static int kernel_get_random(unsigned char *key, size_t key_len); + +/* + * Dispatch targets for when no source is registered. The call sites use + * static_call() directly, so the pointers must always be valid code; a NULL + * target would fault instead of returning an error. + */ +static int trusted_key_absent_seal(struct trusted_key_payload *p, char *datablob) +{ + return -ENODEV; +} + +static int trusted_key_absent_unseal(struct trusted_key_payload *p, char *datablob) +{ + return -ENODEV; +} + +static int trusted_key_absent_get_random(unsigned char *key, size_t key_len) +{ + return -ENODEV; +} + +/** + * register_trusted_key_source - offer a trust source from a module + * @src: the source. Must outlive its registration. + * + * Returns 0 on success, -EBUSY if a source is already active, -ENODEV if + * trusted.source= names a different one, or the source's own init() error. + */ +int register_trusted_key_source(struct trusted_key_source *src) +{ + int (*get_random)(unsigned char *key, size_t key_len); + int ret; + + if (!src || !src->name || !src->ops || !src->ops->init || + !src->ops->seal || !src->ops->unseal) + return -EINVAL; + + /* An explicit trusted.source= names one source; honour it. */ + if (trusted_key_source && strcmp(trusted_key_source, src->name)) + return -ENODEV; + + mutex_lock(&trusted_key_source_lock); + + if (trusted_key_source_active) { + mutex_unlock(&trusted_key_source_lock); + return -EBUSY; + } + + get_random = src->ops->get_random; + if (trusted_rng && strcmp(trusted_rng, "default")) { + if (!strcmp(trusted_rng, "kernel")) { + get_random = kernel_get_random; + } else if (strcmp(trusted_rng, src->name) || !get_random) { + mutex_unlock(&trusted_key_source_lock); + return -EINVAL; + } + } + if (!get_random) + get_random = kernel_get_random; + + ret = src->ops->init(); + if (ret) { + mutex_unlock(&trusted_key_source_lock); + return ret; + } + + static_call_update(trusted_key_seal, src->ops->seal); + static_call_update(trusted_key_unseal, src->ops->unseal); + static_call_update(trusted_key_get_random, get_random); + trusted_key_exit = src->ops->exit; + migratable = src->ops->migratable; + trusted_key_source_active = true; + trusted_key_source_registered = src; + + mutex_unlock(&trusted_key_source_lock); + pr_info("trusted_key: source '%s' registered\n", src->name); + return 0; +} +EXPORT_SYMBOL_GPL(register_trusted_key_source); + +/** + * unregister_trusted_key_source - withdraw a previously registered source + * @src: the source that was registered. + * + * Keys sealed by @src stay in the keyring and stop being usable, which is the + * intended outcome: the thing that could unseal them is gone. + */ +void unregister_trusted_key_source(struct trusted_key_source *src) +{ + mutex_lock(&trusted_key_source_lock); + + /* Only the source that registered may withdraw itself. */ + if (!trusted_key_source_active || src != trusted_key_source_registered) { + mutex_unlock(&trusted_key_source_lock); + return; + } + + static_call_update(trusted_key_seal, trusted_key_absent_seal); + static_call_update(trusted_key_unseal, trusted_key_absent_unseal); + static_call_update(trusted_key_get_random, trusted_key_absent_get_random); + + if (trusted_key_exit) + trusted_key_exit(); + trusted_key_exit = NULL; + migratable = 0; + trusted_key_source_active = false; + trusted_key_source_registered = NULL; + + mutex_unlock(&trusted_key_source_lock); + pr_info("trusted_key: source '%s' unregistered\n", + src && src->name ? src->name : "?"); +} +EXPORT_SYMBOL_GPL(unregister_trusted_key_source); + enum { Opt_err, Opt_new, Opt_load, Opt_update, @@ -343,7 +463,10 @@ static int kernel_get_random(unsigned char *key, size_t key_len) static int __init init_trusted(void) { int (*get_random)(unsigned char *key, size_t key_len); - int i, ret = 0; + /* -ENODEV when the built-in array is empty or nothing matches, so a + * later module source can still register instead of hitting -EBUSY. + */ + int i, ret = -ENODEV; for (i = 0; i < ARRAY_SIZE(trusted_key_sources); i++) { if (trusted_key_source && @@ -391,8 +514,21 @@ static int __init init_trusted(void) * encrypted_keys.ko depends on successful load of this module even if * trusted key implementation is not found. */ - if (ret == -ENODEV) + if (ret == -ENODEV) { + /* + * No built-in source. Leave the dispatch pointing at code that + * fails rather than at NULL, so a seal attempted before any + * module registers returns an error instead of faulting. + */ + static_call_update(trusted_key_seal, trusted_key_absent_seal); + static_call_update(trusted_key_unseal, trusted_key_absent_unseal); + static_call_update(trusted_key_get_random, + trusted_key_absent_get_random); return 0; + } + + if (!ret) + trusted_key_source_active = true; return ret; } From c88aa43fa8e424c9162dc1d8c30345460a2413ce Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:24:37 +1000 Subject: [PATCH 0523/2680] usb: typec: tipd: Descend into tipd/ whenever the core is enabled The TPS6598x core (CONFIG_TYPEC_TPS6598X_CORE) is a separate tristate that both bus transports select: the I2C one (CONFIG_TYPEC_TPS6598X) and, since commit 550afd1aa62b ("usb: typec: tipd: Add sn201202x support"), the SPMI one (CONFIG_TYPEC_SN201202X). tipd/Makefile builds the core and each transport from its own symbol, but drivers/usb/typec/Makefile still descends into tipd/ only for CONFIG_TYPEC_TPS6598X. With CONFIG_TYPEC_SN201202X=y and CONFIG_TYPEC_TPS6598X=m the core is built in, but tipd/ is entered only as a module directory: the built-in core and SPMI transport never reach vmlinux, and modpost fails on the tps6598x module's references to tipd_init() and the other symbols the core exports. With CONFIG_TYPEC_TPS6598X=n the SPMI transport is not built at all. Descend into tipd/ on CONFIG_TYPEC_TPS6598X_CORE, which is set whenever either transport is enabled. Fixes: da16cc966d15 ("usb: typec: tipd: Factor out i2c specifics") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/usb/typec/Makefile b/drivers/usb/typec/Makefile index 8a6a1c663eb69..eebe658de6f8c 100644 --- a/drivers/usb/typec/Makefile +++ b/drivers/usb/typec/Makefile @@ -5,7 +5,7 @@ typec-$(CONFIG_ACPI) += port-mapper.o obj-$(CONFIG_TYPEC) += altmodes/ obj-$(CONFIG_TYPEC_TCPM) += tcpm/ obj-$(CONFIG_TYPEC_UCSI) += ucsi/ -obj-$(CONFIG_TYPEC_TPS6598X) += tipd/ +obj-$(CONFIG_TYPEC_TPS6598X_CORE) += tipd/ obj-$(CONFIG_TYPEC_ANX7411) += anx7411.o obj-$(CONFIG_TYPEC_HD3SS3220) += hd3ss3220.o obj-$(CONFIG_TYPEC_STUSB160X) += stusb160x.o From f0342d16ba731c65ba89a15da9a6d942206b0a06 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 11 Sep 2025 00:01:39 +0200 Subject: [PATCH 0524/2680] usb: typec: tipd: Track data_status changes for CD321x HDP status for DisplayPort alt-mode is signaled data_status. Track changes to have a debounced HPD to forward to the DRM KMS driver. Signed-off-by: Janne Grunau --- drivers/usb/typec/tipd/core.c | 2 ++ drivers/usb/typec/tipd/tps6598x.h | 1 + 2 files changed, 3 insertions(+) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 3d357b178581c..d858c454fe106 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -675,6 +675,7 @@ static void cd321x_update_work(struct work_struct *work) st = cd321x->update_status; cd321x->update_status.status_changed = 0; + cd321x->update_status.data_status_changed = 0; bool old_connected = !!tps->partner; bool new_connected = st.status & TPS_STATUS_PLUG_PRESENT; @@ -773,6 +774,7 @@ static void cd321x_update_work(struct work_struct *work) static void cd321x_queue_status(struct cd321x *cd321x) { cd321x->update_status.status_changed |= cd321x->update_status.status ^ cd321x->tps.status; + cd321x->update_status.data_status_changed |= cd321x->update_status.data_status ^ cd321x->tps.data_status; cd321x->update_status.status = cd321x->tps.status; cd321x->update_status.pwr_status = cd321x->tps.pwr_status; diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 6cf669418a504..4dfca6aefd0b4 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -358,6 +358,7 @@ struct cd321x_status { u32 pwr_status; u32 data_status; u32 status_changed; + u32 data_status_changed; struct usb_pd_identity partner_identity; struct tps6598x_dp_sid_status_reg dp_sid_status; struct tps6598x_intel_vid_status_reg intel_vid_status; From b4d2a00dcd2fe2baef981c613afc1d2d25529ee1 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 22 Dec 2025 22:08:35 +0100 Subject: [PATCH 0525/2680] usb: typec: tipd: HACK: Use drm oob hotplug event MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is not how dp-altmode support should be implemented but it works for new. Requires a "displayport" property in the connector node with a phandle of the connector. [ryan: adapted to 7.2.10 — conflict with the tbt switch hookup Asahi already carries (fc76d0f20c29 is in the base): kept the connector lookup ahead of the switch lookup, as on aurora-wip] Signed-off-by: Janne Grunau --- drivers/usb/typec/tipd/core.c | 17 +++++++++++++++++ drivers/usb/typec/tipd/tps6598x.h | 2 ++ 2 files changed, 19 insertions(+) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index d858c454fe106..a7ed74dabfbf4 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -6,6 +6,8 @@ * Author: Heikki Krogerus */ +#include + #include #include #include @@ -684,6 +686,9 @@ static void cd321x_update_work(struct work_struct *work) bool usb_connection = st.data_status & (TPS_DATA_STATUS_USB2_CONNECTION | TPS_DATA_STATUS_USB3_CONNECTION); + bool dp_hpd = st.data_status & CD321X_DATA_STATUS_HPD_LEVEL; + bool dp_hpd_changed = st.data_status_changed & CD321X_DATA_STATUS_HPD_LEVEL; + enum usb_role old_role = usb_role_switch_get_role(tps->role_sw); enum usb_role new_role = USB_ROLE_NONE; enum typec_pwr_opmode pwr_opmode = TYPEC_PWR_MODE_USB; @@ -712,6 +717,10 @@ static void cd321x_update_work(struct work_struct *work) if (old_role != USB_ROLE_NONE && (new_role != old_role || was_disconnected)) usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); + if (cd321x->connector_fwnode && (!dp_hpd || dp_hpd_changed)) { + drm_connector_oob_hotplug_event(cd321x->connector_fwnode, connector_status_disconnected); + } + /* Process partner disconnection or change */ if (!new_connected || partner_changed) { if (!IS_ERR(tps->partner)) @@ -768,6 +777,9 @@ static void cd321x_update_work(struct work_struct *work) /* Launch the USB role switch */ usb_role_switch_set_role(tps->role_sw, new_role); + if (cd321x->connector_fwnode && dp_hpd) + drm_connector_oob_hotplug_event(cd321x->connector_fwnode, connector_status_connected); + power_supply_changed(tps->psy); } @@ -1203,6 +1215,7 @@ static int cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) { struct cd321x *cd321x = container_of(tps, struct cd321x, tps); + struct fwnode_handle *connector_fwnode = NULL; int ret; INIT_DELAYED_WORK(&cd321x->update_work, cd321x_update_work); @@ -1221,6 +1234,10 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) goto err_unregister_altmodes; } + if (fwnode_property_present(fwnode, "displayport")) + connector_fwnode = fwnode_find_reference(fwnode, "displayport", 0); + if (!IS_ERR_OR_NULL(connector_fwnode)) + cd321x->connector_fwnode = connector_fwnode; cd321x->tbt_switch = fwnode_typec_thunderbolt_switch_get(fwnode); if (IS_ERR(cd321x->tbt_switch)) { ret = PTR_ERR(cd321x->tbt_switch); diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 4dfca6aefd0b4..17b01d2c55c9b 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -383,6 +383,8 @@ struct cd321x { struct cd321x_status update_status; struct delayed_work update_work; struct usb_pd_identity cur_partner_identity; + + struct fwnode_handle *connector_fwnode; }; struct sn201202x { From c004d5a9b9f3ec32bd020012666ad8433a77c8c1 Mon Sep 17 00:00:00 2001 From: Sven Peter Date: Mon, 24 Aug 2026 18:07:56 +0200 Subject: [PATCH 0526/2680] dt-bindings: thunderbolt: Add Apple USB4/Thunderbolt NHI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Native Host Interface (NHI) provides the TX and RX rings used by the software connection manager on Apple Silicon SoCs. The NHI is part of the ACIO host router and is exposed to the main SoC bus while the ACIO co-processor is running. T6000 and T6020 use one DART stream for the TX and RX control rings and one shared by all other rings, which are used for XDomain connections. This likely improves the isolation of those host-to-host connections. The M3 generation uses a slightly insane set of 24 IOMMU entries. These are probably one DART stream for each TX and RX ring. [ryan: adapted to 7.2.10 — split by topic: the MAINTAINERS lines are carried by 900-config-tooling] Signed-off-by: Sven Peter --- .../thunderbolt/apple,t8103-usb4-nhi.yaml | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml diff --git a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml new file mode 100644 index 0000000000000..4f95132cebf30 --- /dev/null +++ b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml @@ -0,0 +1,149 @@ +# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/thunderbolt/apple,t8103-usb4-nhi.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple Silicon USB4/Thunderbolt Native Host Interface (NHI) + +maintainers: + - Sven Peter + +description: | + On Apple Silicon SoCs the Native Host Interface (NHI) is part of the ACIO + host router. It provides the transmit and receive rings used by the software + connection manager. + + The ACIO co-processor exposes its registers to the main SoC bus while it is + running. The NHI is therefore represented as a child of the ACIO node. + +properties: + compatible: + oneOf: + - const: apple,t8103-usb4-nhi + - items: + - enum: + - apple,t6000-usb4-nhi + - apple,t6020-usb4-nhi + - apple,t6030-usb4-nhi + - apple,t6031-usb4-nhi + - apple,t8112-usb4-nhi + - apple,t8122-usb4-nhi + - const: apple,t8103-usb4-nhi + + reg: + items: + - description: NHI registers with the transmit and receive rings + - description: + Protocol Defined Field (PDF) bitmasks selecting which frames are + accepted by each receive ring + + reg-names: + items: + - const: nhi + - const: pdf + + interrupts: + description: + One interrupt for each of the 12 transmit rings followed by one for each + of the 12 receive rings. + maxItems: 24 + + interrupt-names: + items: + - const: txring0 + - const: txring1 + - const: txring2 + - const: txring3 + - const: txring4 + - const: txring5 + - const: txring6 + - const: txring7 + - const: txring8 + - const: txring9 + - const: txring10 + - const: txring11 + - const: rxring0 + - const: rxring1 + - const: rxring2 + - const: rxring3 + - const: rxring4 + - const: rxring5 + - const: rxring6 + - const: rxring7 + - const: rxring8 + - const: rxring9 + - const: rxring10 + - const: rxring11 + + iommus: + minItems: 1 + maxItems: 24 + description: | + T6000 and T6020 use two IOMMUs: The first is used for transmit ring 0 + and receive ring 0 which carry control traffic. The second is shared by + all other rings which are only used for XDomain connections to other + hosts. + + T6030, T6031 and T8122 use 24 entries. These are most likely one DART + stream for each of the 12 transmit and 12 receive rings. + + apple,thunderbolt-drom: + $ref: /schemas/types.yaml#/definitions/uint8-array + description: + DROM of the host router built by the bootloader from Apple's Device Tree. + + apple,tunable-nhi: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Mask of bits to clear + - description: Bits to set + description: | + List of (register offset, mask, value) tuples copied from Apple's Device + Tree, converted by the bootloader and used to configure the NHI after + the co-processor has booted. + +required: + - compatible + - reg + - reg-names + - interrupts + - interrupt-names + - iommus + - apple,thunderbolt-drom + +allOf: + - if: + properties: + compatible: + contains: + enum: + - apple,t6030-usb4-nhi + - apple,t6031-usb4-nhi + - apple,t8122-usb4-nhi + then: + properties: + iommus: + minItems: 24 + maxItems: 24 + else: + if: + properties: + compatible: + contains: + enum: + - apple,t6000-usb4-nhi + - apple,t6020-usb4-nhi + then: + properties: + iommus: + minItems: 2 + maxItems: 2 + else: + properties: + iommus: + maxItems: 1 + +additionalProperties: false From 4a0e803dd1d1ff617c34689b70bc321f74ab52ef Mon Sep 17 00:00:00 2001 From: Sven Peter Date: Mon, 24 Aug 2026 18:07:56 +0200 Subject: [PATCH 0527/2680] dt-bindings: thunderbolt: Add Apple USB4/Thunderbolt host router MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each Type-C port with USB4/Thunderbolt support on Apple Silicon SoCs comes with a host router called ACIO. It contains a Cortex-M3 co-processor and the hardware blocks implementing the USB4 router. The co-processor exposes the NHI and its DART to the main SoC bus while it is running. ACIO can only be powered on after the Type-C PHY has been switched to USB4/Thunderbolt mode. The Type-C controller provides the cable information required to boot it through the thunderbolt-switch connection. [ryan: adapted to 7.2.10 — split by topic: the MAINTAINERS lines are carried by 900-config-tooling] Signed-off-by: Sven Peter --- .../thunderbolt/apple,t8103-usb4-acio.yaml | 218 ++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml diff --git a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml new file mode 100644 index 0000000000000..6674c7dde2b51 --- /dev/null +++ b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml @@ -0,0 +1,218 @@ +# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/thunderbolt/apple,t8103-usb4-acio.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple Silicon USB4/Thunderbolt Host Router (ACIO) + +maintainers: + - Sven Peter + +description: | + Each Type-C port with USB4/Thunderbolt support on Apple Silicon SoCs has a + host router complex called ACIO (Apple Converged I/O). It contains a + Cortex-M3 co-processor, the hardware blocks implementing the USB4 router + and its PCIe, USB3 and DisplayPort adapters. + + The co-processor exposes the Native Host Interface (NHI) and its DART IOMMU + to the main SoC bus once it is running. They are therefore represented as + child nodes. All child registers are located in a single 16 MiB MMIO window + and use addresses relative to this window. + + ACIO can only be powered on after the Type-C PHY has been switched to + USB4/Thunderbolt mode. Before the connection to the remote device can be + established the Type-C PD controller has to pass the cable information + through the thunderbolt-switch connection. + +properties: + compatible: + oneOf: + - const: apple,t8103-usb4-acio + - items: + - enum: + - apple,t6000-usb4-acio + - apple,t6020-usb4-acio + - apple,t6030-usb4-acio + - apple,t6031-usb4-acio + - apple,t8112-usb4-acio + - apple,t8122-usb4-acio + - const: apple,t8103-usb4-acio + + reg: + items: + - description: Root complex and co-processor control registers + - description: SRAM used for communication with the co-processor + + reg-names: + items: + - const: rc + - const: sram + + mboxes: + maxItems: 1 + description: Mailbox used for RTKit communication with the co-processor + + resets: + maxItems: 1 + description: Reset used to start the ACIO block before booting the + co-processor + + power-domains: + minItems: 3 + maxItems: 3 + + thunderbolt-switch: true + + apple,tunable-rc: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Mask of bits to clear + - description: Bits to set + description: | + List of (register offset, mask, value) tuples copied from Apple's Device + Tree, converted by the bootloader and used to configure the root complex + after the co-processor has booted. + + '#address-cells': + const: 1 + + '#size-cells': + const: 1 + + ranges: + maxItems: 1 + description: Single 16 MiB MMIO window containing all child registers + + nonposted-mmio: true + + ports: + $ref: /schemas/graph.yaml#/properties/ports + description: + OF graph ports for the USB4 router. The port numbers correspond to the + USB4 router adapter numbers. + properties: + port@1: + $ref: /schemas/graph.yaml#/properties/port + description: USB4 port connected to the USB Type-C connector + + # port@2 is the second, unused USB4 port of the host router + + port@3: + $ref: /schemas/graph.yaml#/properties/port + description: PCIe adapter + + port@4: + $ref: /schemas/graph.yaml#/properties/port + description: USB3 adapter + + port@5: + $ref: /schemas/graph.yaml#/properties/port + description: First DP IN adapter + + port@6: + $ref: /schemas/graph.yaml#/properties/port + description: Second DP IN adapter + + required: + - port@1 + + iommu@a80000: + $ref: /schemas/iommu/apple,dart.yaml# + description: DART IOMMU exposed by ACIO and used for DMA from the NHI + + nhi@f00000: + $ref: /schemas/thunderbolt/apple,t8103-usb4-nhi.yaml# + description: Native Host Interface exposed by ACIO + +required: + - compatible + - reg + - reg-names + - mboxes + - power-domains + - resets + - thunderbolt-switch + - '#address-cells' + - '#size-cells' + - ranges + - nonposted-mmio + - ports + - iommu@a80000 + - nhi@f00000 + +allOf: + - $ref: /schemas/usb/usb-switch.yaml# + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + cio@501ac0000 { + compatible = "apple,t8103-usb4-acio"; + reg = <0x5 0x01ac0000 0x0 0xc000>, + <0x5 0x01080000 0x0 0x20000>; + reg-names = "rc", "sram"; + mboxes = <&usb4_1_mbox>; + power-domains = <&ps_atc1_cio>, + <&ps_atc1_cio_pcie>, + <&ps_atc1_cio_usb>; + resets = <&cio_reset 1>; + thunderbolt-switch; + pinctrl-0 = <&acio1_pins>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x5 0x01000000 0x1000000>; + nonposted-mmio; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + endpoint { + remote-endpoint = <&typec1_con_tbt>; + }; + }; + }; + + usb4_1_dart: iommu@a80000 { + compatible = "apple,t8103-dart"; + reg = <0xa80000 0x4000>; + interrupt-parent = <&aic>; + interrupts = <1 890 4>; + #iommu-cells = <1>; + }; + + nhi@f00000 { + compatible = "apple,t8103-usb4-nhi"; + reg = <0xf00000 0x100000>, + <0xdb0000 0x30004>; + reg-names = "nhi", "pdf"; + interrupt-parent = <&aic>; + interrupts = <1 863 4>, <1 864 4>, <1 865 4>, <1 866 4>, + <1 867 4>, <1 868 4>, <1 869 4>, <1 870 4>, + <1 871 4>, <1 872 4>, <1 873 4>, <1 874 4>, + <1 875 4>, <1 876 4>, <1 877 4>, <1 878 4>, + <1 879 4>, <1 880 4>, <1 881 4>, <1 882 4>, + <1 883 4>, <1 884 4>, <1 885 4>, <1 886 4>; + interrupt-names = "txring0", "txring1", "txring2", "txring3", + "txring4", "txring5", "txring6", "txring7", + "txring8", "txring9", "txring10", "txring11", + "rxring0", "rxring1", "rxring2", "rxring3", + "rxring4", "rxring5", "rxring6", "rxring7", + "rxring8", "rxring9", "rxring10", "rxring11"; + iommus = <&usb4_1_dart 1>; + /* To be filled by the loader */ + apple,thunderbolt-drom = [00]; + }; + }; + }; From 18a7b0adcb81fddd30655a821cf416be58b998dc Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 24 Aug 2026 13:14:09 -0400 Subject: [PATCH 0528/2680] phy: apple: atc: Don't treat the pipehandler lock as fatal in USB4 mode atcphy_configure_pipehandler_usb4() aborts the mux switch when atcphy_pipehandler_lock() times out. That lock cannot succeed in this path: the LOCK_PIPE_IF handshake FSM is clocked by the DWC3 PIPE clock, and once the connector's lanes are routed to the USB4 controller that clock is no longer sourced the way a USB3 bring-up sources it. The ACK never asserts, so every routed-USB4 mux switch fails before PIPEHANDLER_MUX_CTRL is ever written and the port silently stays on the previous backend. The handshake is advisory in this path. It reports that the PIPE interface has quiesced, but the mux switch does not depend on that once the lanes have already moved to the USB4 controller. Linux already encodes that policy everywhere else, just through control flow rather than a shared fall-through. The dummy path only warns on both the lock and the unlock, and the sole fatal acquire is the USB3 host BIST sequence, which genuinely needs the PIPE quiesced. Make the USB4 path warn and continue like the dummy path, and say why in a comment so the next refactor doesn't "fix" the missing error check back in. Observed on a MacBook Pro 14" (M2 Pro, J414s, T6020), where the ACK poll deadlocked the mux switch deterministically. Signed-off-by: DjDeveloperr --- drivers/phy/apple/atc.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 0fe15742e7817..f6981b79e6e02 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1225,11 +1225,17 @@ static int atcphy_configure_pipehandler_usb4(struct apple_atcphy *atcphy) set32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, PIPEHANDLER_OVERRIDE_RXVALID); set32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, PIPEHANDLER_OVERRIDE_RXDETECT); + /* + * The lock is best-effort here: the LOCK_PIPE_IF handshake is clocked by the + * DWC3 PIPE clock, which is not running once the lanes are routed to the USB4 + * controller, so the ACK legitimately never asserts. The handshake is + * advisory in this path, and atcphy_configure_pipehandler_dummy() below + * already relies on the same warn-and-continue behaviour. Only the USB3 host + * BIST sequence genuinely needs the PIPE quiesced and keeps the lock fatal. + */ ret = atcphy_pipehandler_lock(atcphy); - if (ret) { - dev_err(atcphy->dev, "Failed to lock pipehandler\n"); - return ret; - } + if (ret) + dev_warn(atcphy->dev, "Failed to lock pipehandler\n"); /* Configure PIPE mux to the USB4/Thunderbolt controller */ atcphy_pipehandler_set_mux(atcphy, PIPEHANDLER_MUX_CTRL_DATA_USB4, From cada415fb7e27d69c9b8acf783f823859153cbf1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 24 Aug 2026 13:02:12 -0400 Subject: [PATCH 0529/2680] thunderbolt: apple: Preserve the other M3 control bits when starting the core apple_cio_start() releases the ACIO Cortex-M3 with a bare store of APPLE_CIO_M3_CTRL_START, which clears every other bit of the RC-window control register at 0x0c. Apple's own enableEmbeddedCPU reads that register first, logs its current value ("Current LSTX State") and then does a read-modify-write, so the register is known to carry at least one other live field. On t6020 it has been observed reading 0x3 before the kernel touches it, i.e. iBoot leaves bit 0 set and the bare store clears it. Nothing here needs those bits cleared, and their meaning is undocumented, so read-modify-write instead: unknown-and-preserved beats unknown-and-cleared. Signed-off-by: DjDeveloperr --- drivers/thunderbolt/apple.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 530008cc8ec5d..50333b29dbb63 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -702,7 +702,7 @@ static int apple_cio_start(struct apple_cio *acio) { struct device_link *link; int i, ret; - u32 state; + u32 state, val; lockdep_assert_held(&acio->lock); @@ -727,8 +727,16 @@ static int apple_cio_start(struct apple_cio *acio) goto remove_links; } - /* Start and wait for the co-processor to boot */ - writel(APPLE_CIO_M3_CTRL_START, acio->rc_base + APPLE_CIO_M3_CTRL); + /* + * Start and wait for the co-processor to boot. + * + * iBoot can leave other bits of this register set (0x3 has been observed + * on t6020 before the kernel touches it). The + * remaining fields are undocumented, so preserve them rather than clearing + * them with a bare store. + */ + val = readl(acio->rc_base + APPLE_CIO_M3_CTRL); + writel(val | APPLE_CIO_M3_CTRL_START, acio->rc_base + APPLE_CIO_M3_CTRL); acio->rtk = apple_rtkit_init(acio->dev, acio, NULL, 0, &apple_cio_rtkit_ops); if (IS_ERR(acio->rtk)) { ret = PTR_ERR(acio->rtk); From 06a1581d731fd1d4fa5cdaf0b52cf2f920d0caa3 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 24 Aug 2026 13:10:17 -0400 Subject: [PATCH 0530/2680] thunderbolt: apple: Reject NHIs with more ring IRQs than status bits apple_cio_ring_index() maps TX ring n to bit n and RX ring n to bit n + n_rings, and apple_nhi_ring_interrupt_active() and apple_cio_ring_irq() then use BIT(idx) against the 32-bit IRQ enable and status registers. The index is derived from the interrupt count in the device tree, which is only checked for being non-zero and even; the later hop_count comparison is against that same firmware-provided number and masks off ten bits, so it does not bound it either. Nothing in the path stops a firmware description with more than sixteen ring pairs from shifting a u32 by 32 or more, which is undefined behaviour and would quietly enable and acknowledge the wrong ring. Reject it at probe instead. Real hardware reports twelve pairs, so this only rules out descriptions the register layout cannot express. Signed-off-by: DjDeveloperr --- drivers/thunderbolt/apple.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 50333b29dbb63..7aca8ddafb110 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -289,6 +289,17 @@ static int apple_nhi_probe_irqs(struct apple_nhi *anhi) dev_err(anhi->dev, "Invalid number of interrupts: %d must be even\n", n_irqs); return -EINVAL; } + /* + * TX ring n is addressed as bit n and RX ring n as bit n + n_rings of the 32 bit + * IRQ status and enable registers, so both halves have to fit into a single + * register. hop_count is only checked against n_rings later on, and both come + * from firmware, so bound this before anything shifts by the derived index. + */ + if (n_irqs > 32) { + dev_err(anhi->dev, "Invalid number of interrupts: %d exceeds 32 status bits\n", + n_irqs); + return -EINVAL; + } anhi->n_rings = n_irqs / 2; anhi->rx_irqs = devm_kcalloc(anhi->dev, anhi->n_rings, sizeof(*anhi->rx_irqs), GFP_KERNEL); From 4b2805c87e33d9dde65574771749e7162cfab5c1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 24 Aug 2026 13:39:35 -0400 Subject: [PATCH 0531/2680] reset: apple-cio: Wait for the shared request register before reconfiguring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On t600x and t602x every port on a die shares one CIO reconfigure register, so a request issued for a different port can still be in flight when this one runs. The driver's mutex serialises its own callers but says nothing about the hardware state machine, and a request stored while the block is still busy is dropped. The block needs two waits rather than one: poll the busy bits until they clear, store the request bit, then poll again. The same shape applies across t600x and t602x, with only the register offset and the die split changing. Keep the first wait advisory and store the request either way. A timeout there means the block was busy for 192 ms, which is worth a message, but failing the reset would turn a slow reconfigure into a probe failure of the whole USB4 complex. Worth noting for later: t8103, t8112 and t8122 have no PMGR CIO reconfigure at all, which is why the t8103 variant here drives a dedicated register page instead. [ryan: adapted to 7.2.10 — Asahi reset-apple-cio.c is the base (newer revision of the driver we carried); kept its NULL-variant check and comment, the pre-request busy wait goes on top] Signed-off-by: DjDeveloperr --- drivers/reset/reset-apple-cio.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/drivers/reset/reset-apple-cio.c b/drivers/reset/reset-apple-cio.c index 5727c608f82f8..7a8e0340fc516 100644 --- a/drivers/reset/reset-apple-cio.c +++ b/drivers/reset/reset-apple-cio.c @@ -41,6 +41,7 @@ struct apple_cio_reset_variant { struct apple_cio_reset { struct reset_controller_dev rcdev; const struct apple_cio_reset_variant *variant; + struct device *dev; struct regmap *regmap; u32 offset; struct mutex lock; /* serializes access to the shared t600x register */ @@ -85,6 +86,20 @@ static int t6000_cio_deassert(struct apple_cio_reset *priv, unsigned long id) */ guard(mutex)(&priv->lock); + /* + * Every port on the die shares this register, so a reconfiguration requested for + * another port can still be running. The busy bits must clear before the request + * is stored, and the result of that wait is advisory, so only + * warn here: waiting can only help, while failing would turn a slow reconfigure + * into a probe failure. + */ + ret = regmap_read_poll_timeout(priv->regmap, priv->offset, val, + !(val & T6000_CIO_CTRL_INIT_BUSY(id)), + APPLE_CIO_RESET_POLL_US, + APPLE_CIO_RESET_TIMEOUT_US); + if (ret) + dev_warn(priv->dev, "CIO %lu still busy before reconfigure request\n", id); + ret = regmap_write(priv->regmap, priv->offset, T6000_CIO_CTRL_INIT_REQ(id)); if (ret) return ret; @@ -137,6 +152,7 @@ static int apple_cio_reset_probe(struct platform_device *pdev) priv->variant = of_device_get_match_data(dev); if (!priv->variant) return -ENODEV; + priv->dev = dev; ret = devm_mutex_init(dev, &priv->lock); if (ret) From 2f35ee3eaaf9b692cbfe261865463182f9c41a45 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 24 Aug 2026 14:53:34 -0400 Subject: [PATCH 0532/2680] reset: apple-cio: Wait before the request on t8103 too, and record the layouts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The t8103 variant has the same problem the t600x one did: it stores the request without first checking whether the block is still busy from an earlier one. Poll bit 1 of the port's control word until it clears, then set the request bit, keeping the wait advisory for the reason given on the t600x side. Bit 1 had no name here because nothing read it. Call it INIT_BUSY. On t8103 and t8112 the request can also be set with a read-modify-write rather than a plain store. Deliberately not done here: the existing poll waits for the register to read exactly INIT_DONE, which works on every machine this has run on, and preserving unknown bits could stop that value ever appearing. Noted in the code so the difference reads as a decision rather than an oversight. Record the known register layouts while they are in hand. The two variants here cover four SoCs and give no hint what the next one needs, so anyone extending this would otherwise redo the work. Three things in that table matter: The first argument selects a PMGR register map rather than an address. Only map 0 is pinned down, the PMGR block itself, which is why t8103 resolves to 0x23b784000 and t6020 to the pmgr node plus 0xa02c. t8122 and t6030 select map 0x62 and t8112 map 0x63, and those are unresolved. M3 is therefore blocked on identifying one map index rather than on finding a register: its offset and bit layout are already known and match t6030's. t6050 uses two bits per port instead of a request bit and a busy bit sixteen apart, so it needs a third variant rather than a new offset. t8132 and t8142 index a per-port table rather than forming an offset, and are not decoded. [ryan: adapted to 7.2.10 — kept Asahi t8103 deassert description and added the pre-request wait after it] Signed-off-by: DjDeveloperr --- drivers/reset/reset-apple-cio.c | 49 +++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/drivers/reset/reset-apple-cio.c b/drivers/reset/reset-apple-cio.c index 7a8e0340fc516..39b20eadbc1e1 100644 --- a/drivers/reset/reset-apple-cio.c +++ b/drivers/reset/reset-apple-cio.c @@ -9,6 +9,38 @@ * t8103 each port comes with a dedicated register page while t600x * uses a single register shared by all ports of a die inside the PMGR * MMIO region. + * + * Only the deassert direction is implemented: a request to assert returns + * immediately on every SoC, hence the missing .assert op here. The register + * layout is not shared between generations: + * + * SoC regmap offset request busy + * t8103 0 0x84000 + 0x4000 * port BIT(0), rmw BIT(1) + * t8112 0x63 0x8000 / 0x4000 BIT(0), rmw BIT(1) + * t8122 0x62 0x8 BIT(slot) BIT(16 + slot) + * t6000 0 0x1a034 BIT(port) BIT(16 + port) + * t6020 0 0xa02c BIT(port) BIT(16 + port) + * t6030 0x62 0x8 BIT(slot) BIT(16 + slot) + * t6031 ? 0x2028 BIT(port) BIT(16 + port) + * t6041 ? 0x2024 BIT(slot) BIT(16 + slot) + * t6050 ? 0x20060 BIT(2 * port) BIT(2 * port + 1) + * + * t6000, t6020, t6031 and t6050 split slot into a die index and a port with + * "port = slot % 4", which the device tree expresses as one controller node + * per die instead. The others index by slot directly. + * + * "regmap" is an ApplePMGR::RegMap enum rather than an address. Only 0 is + * pinned down, as the PMGR block itself: t8103 resolves to 0x23b784000 and + * t6020 to the pmgr node plus 0xa02c, both of which match this driver. The + * non-zero selectors are unresolved, so t8122 and t6030 cannot be wired up + * yet even though their offset and bit layout are known. + * + * t8132 and t8142 have a different shape again, indexing a per-port table out + * of a driver-private structure rather than forming an offset, and have not + * been decoded. + * + * Every implementation polls the busy bits, stores the request, then polls + * again, with a 192 ms budget. The first poll's result is discarded. */ #include @@ -25,6 +57,7 @@ #define T8103_CIO_CTRL_STRIDE 0x4000 #define T8103_CIO_CTRL_INIT_REQ BIT(0) +#define T8103_CIO_CTRL_INIT_BUSY BIT(1) #define T8103_CIO_CTRL_INIT_DONE BIT(2) #define T6000_CIO_CTRL_INIT_REQ(port) BIT(port) @@ -58,7 +91,23 @@ static int t8103_cio_deassert(struct apple_cio_reset *priv, unsigned long id) * Once the sequence is done BIT(0) is cleared automatically and then BIT(2) * is set. There's no way to assert the reset again other than cutting the power * to the entire CIO block. Writing the bit again has no effect. + * + * Wait for a reconfiguration already in flight, as Apple does, and like Apple + * carry on regardless of the result. See the t6000 variant for why this is only + * a warning. + * + * Apple also sets the request bit with a read-modify-write rather than a plain + * store. That is deliberately not copied here: the poll below waits for the + * whole register to read exactly INIT_DONE, which works on every machine this + * has run on, and preserving unknown bits could stop that value ever appearing. */ + ret = regmap_read_poll_timeout(priv->regmap, offset, val, + !(val & T8103_CIO_CTRL_INIT_BUSY), + APPLE_CIO_RESET_POLL_US, + APPLE_CIO_RESET_TIMEOUT_US); + if (ret) + dev_warn(priv->dev, "CIO %lu still busy before reconfigure request\n", id); + ret = regmap_write(priv->regmap, offset, T8103_CIO_CTRL_INIT_REQ); if (ret) return ret; From d82b73fb9526233a200bbf65db6e0bc6a3008269 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 24 Aug 2026 13:02:51 -0400 Subject: [PATCH 0533/2680] usb: typec: tipd: Use the CD321x active-cable bit when rebuilding the TBT VDO The reconstructed cable Discover Mode VDO takes TBT_CABLE_ACTIVE_PASSIVE from TPS_DATA_STATUS_ACTIVE_CABLE, which is TI's bit 2. CD321x does not use TI's layout for this register: the upper half is already known to differ, which is why CD321X_DATA_STATUS_USB4_CONNECTION sits at bit 23 where TI documents FORCE_LSX, and why HPD_IRQ/HPD_LEVEL need their own defines. On CD321x the active-cable state is bit 22 of the same register, and the cable speed field is bits [27:25], matching TPS_DATA_STATUS_TBT_CABLE_SPEED_MASK as already used here. This matters beyond the VDO itself. TBT_CABLE_ACTIVE_PASSIVE becomes TB_VSE_CAP_APPLE_CABLE_INFO_ACTIVE_CABLE in the host router's cable word and gates BIDIR_LSRX, so describing a passive cable as active programs the router incorrectly. Add a CD321x-specific define and use it in the cd321x path only; tps6598x keeps TI's bit 2. Verified passive on a J414s (T6020), where DATA_STATUS reads 0x96800011: bit 22 clear, speed field 3. An active cable is still worth testing against. Signed-off-by: DjDeveloperr --- drivers/usb/typec/tipd/core.c | 2 +- drivers/usb/typec/tipd/tps6598x.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index a7ed74dabfbf4..b2c25bb05cecf 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -611,7 +611,7 @@ static void cd321x_typec_update_mode(struct tps6598x *tps, struct cd321x_status tbt_data.cable_mode |= TBT_CABLE_OPTICAL; if (st->data_status & TPS_DATA_STATUS_ACTIVE_LINK_TRAIN) tbt_data.cable_mode |= TBT_CABLE_LINK_TRAINING; - if (st->data_status & TPS_DATA_STATUS_ACTIVE_CABLE) + if (st->data_status & CD321X_DATA_STATUS_ACTIVE_CABLE) tbt_data.cable_mode |= TBT_CABLE_ACTIVE_PASSIVE; tbt_data.device_mode = TBT_MODE | (u32)le16_to_cpu(st->intel_vid_status.device_mode) << 16; diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 17b01d2c55c9b..d03b9ae7fa58e 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -204,6 +204,7 @@ /* modified TPS_REG_DATA_STATUS bits for CD321x (and likely also TPS65987DDK) */ #define CD321X_DATA_STATUS_HPD_IRQ BIT(14) #define CD321X_DATA_STATUS_HPD_LEVEL BIT(15) +#define CD321X_DATA_STATUS_ACTIVE_CABLE BIT(22) #define CD321X_DATA_STATUS_USB4_CONNECTION BIT(23) #define TPS_DATA_STATUS_DP_PIN_ASSIGNMENT_MASK GENMASK(11, 10) From 30fb85847f528e2d76e5cc32056ad03071f55b64 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 00:02:34 -0400 Subject: [PATCH 0534/2680] dt-bindings: describe Apple tunneled PCIe-C resources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — split by topic: only the Documentation/devicetree/bindings/thunderbolt hunks of this soc-platform commit; the rest is in 010-soc] Signed-off-by: DjDeveloperr --- .../bindings/thunderbolt/apple,t8103-usb4-acio.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml index 6674c7dde2b51..34b8a8fa7c473 100644 --- a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml +++ b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml @@ -60,7 +60,11 @@ properties: power-domains: minItems: 3 - maxItems: 3 + maxItems: 4 + description: | + The first three domains power the ACIO co-processor complex. When + apple,pcie-tunnel is present, the fourth domain powers its PCIe-C + controller and is enabled only after the USB4 PCIe tunnel is active. thunderbolt-switch: true @@ -162,7 +166,8 @@ examples: mboxes = <&usb4_1_mbox>; power-domains = <&ps_atc1_cio>, <&ps_atc1_cio_pcie>, - <&ps_atc1_cio_usb>; + <&ps_atc1_cio_usb>, + <&ps_atc1_pcie>; resets = <&cio_reset 1>; thunderbolt-switch; pinctrl-0 = <&acio1_pins>; From 1c0d23d40baa166b5c14bf2322591783c17f71e0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 29 Aug 2026 00:48:57 -0400 Subject: [PATCH 0535/2680] dt-bindings: apple: validate PCIe-C and USB4 topology MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — split by topic: only the Documentation/devicetree/bindings/thunderbolt hunks of this soc-platform commit; the rest is in 010-soc] Signed-off-by: DjDeveloperr --- .../bindings/thunderbolt/apple,t8103-usb4-acio.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml index 34b8a8fa7c473..c83e00f446314 100644 --- a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml +++ b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml @@ -68,6 +68,12 @@ properties: thunderbolt-switch: true + apple,pcie-tunnel: + $ref: /schemas/types.yaml#/definitions/phandle + description: + PCIe-C tunnel container populated after the USB4 router has enabled its + PCIe adapter. + apple,tunable-rc: $ref: /schemas/types.yaml#/definitions/uint32-matrix items: From d0fbccacd942ae94ead1150465e3fc70b0669ba6 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 25 Aug 2026 14:16:33 -0400 Subject: [PATCH 0536/2680] PCI: apple: add cable-powered PCIe-C root complexes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — split by topic: only the drivers/thunderbolt hunks of this radio commit; the rest is in 080-radio] [ryan: adapted to 7.2.10 — Asahi apple.c now documents struct apple_nhi; the helper goes above that comment] Signed-off-by: DjDeveloperr --- drivers/thunderbolt/apple.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 7aca8ddafb110..d8b20e49a9a27 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -162,6 +162,7 @@ struct apple_cio { struct device *dev; struct device_node *np; + struct device_node *pcie_tunnel_np; struct apple_rtkit *rtk; void __iomem *rc_base; @@ -185,6 +186,11 @@ struct apple_cio { struct typec_thunderbolt_switch_dev *tbt_switch; }; +static void apple_cio_of_node_put(void *data) +{ + of_node_put(data); +} + /** * struct apple_nhi - Apple Native Host Interface * @dev: NHI device @@ -792,6 +798,23 @@ static int apple_cio_start(struct apple_cio *acio) goto err_depopulate; } + /* + * The PCIe-C root complex is outside the ACIO MMIO window but shares + * its cable-controlled power lifetime. Instantiate it only after the + * router and NHI are live; parenting it to ACIO makes depopulation tear + * it down before the co-processor and power domains disappear. + */ + if (acio->pcie_tunnel_np) { + ret = of_platform_populate(acio->pcie_tunnel_np, NULL, NULL, + acio->dev); + if (ret) { + dev_err(acio->dev, + "failed to populate tunneled PCIe controller: %d\n", + ret); + goto err_depopulate; + } + } + acio->current_cable_info = acio->target_cable_info; return 0; @@ -916,6 +939,14 @@ static int apple_cio_probe(struct platform_device *pdev) init_completion(&acio->nhi_boot_completion); acio->dev = &pdev->dev; acio->np = dev->of_node; + acio->pcie_tunnel_np = + of_parse_phandle(dev->of_node, "apple,pcie-tunnel", 0); + if (acio->pcie_tunnel_np) { + ret = devm_add_action_or_reset(dev, apple_cio_of_node_put, + acio->pcie_tunnel_np); + if (ret) + return ret; + } acio->sram_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "sram"); if (!acio->sram_res) From 73ea63e48554c78133c1477f95d93ebcfb529ce1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:27:00 +1000 Subject: [PATCH 0537/2680] dt-bindings: media: Add Apple T8140 image signal processor Add a binding for the image signal processor of the A18 Pro (T8140), which drives the MacBook Neo camera through firmware loaded by the bootloader. The node describes: - four register windows: the co-processor block, the interrupt and doorbell registers ("mbox"), the firmware handshake registers ("gpio") and the capture watchdog ("wdt"), which the host has to service while streaming, or the frames carry a diagnostic fill instead of sensor pixels; - one interrupt, three DART streams and 14 power domains; - the bootloader-filled firmware heap (memory-region "heap"); - apple,dart-vm-size, apple,platform-id, apple,temporal-filter and the sensor-presets node, which the existing Apple ISP nodes and the isp-*.dtsi sensor includes already use. The "apple,isp" fallback is the compatible of the ISP nodes on T8103, T8112, T600x and T602x, none of which has a schema; a select keeps this T8140 schema from being applied to them. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/media/apple,t8140-isp.yaml | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 Documentation/devicetree/bindings/media/apple,t8140-isp.yaml diff --git a/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml b/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml new file mode 100644 index 0000000000000..dea138b415b7a --- /dev/null +++ b/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml @@ -0,0 +1,147 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/media/apple,t8140-isp.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 Image Signal Processor + +maintainers: + - Ryan Murray + +description: + The T8140 ISP controls the attached camera sensor and processes its frames + through bootloader-loaded firmware. The host exchanges commands and image + buffers through three DART instances. A separate watchdog register window + must be serviced while capturing to receive sensor pixels. + +select: + properties: + compatible: + contains: + const: apple,t8140-isp + required: + - compatible + +properties: + compatible: + items: + - const: apple,t8140-isp + - const: apple,isp + + reg: + items: + - description: Coprocessor control registers + - description: Mailbox interrupt and doorbell registers + - description: Firmware handshake registers + - description: Capture watchdog registers + + reg-names: + items: + - const: coproc + - const: mbox + - const: gpio + - const: wdt + + interrupts: + maxItems: 1 + + iommus: + items: + - description: First ISP DART stream + - description: Second ISP DART stream + - description: Third ISP DART stream + + power-domains: + items: + - description: ISP system domain + - description: ISP CPU domain + - description: ISP CPU core 0 domain + - description: ISP CPU core 1 domain + - description: ISP FE domain + - description: ISP DPRX domain + - description: ISP secure domain + - description: ISP STS0 domain + - description: ISP STS1 domain + - description: ISP Pearl domain + - description: ISP BE domain + - description: ISP CLR domain + - description: ISP CVD domain + - description: ISP CVM domain + + memory-region: + maxItems: 1 + description: Firmware heap region provided by the bootloader + + memory-region-names: + items: + - const: heap + + apple,dart-vm-size: + $ref: /schemas/types.yaml#/definitions/uint64 + description: + Size in bytes of the host allocation aperture used by the ISP driver. + + apple,platform-id: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Platform identifier passed in the firmware boot descriptor. + + apple,temporal-filter: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Firmware temporal-filter strength selected for the attached sensor. + + sensor-presets: + type: object + description: + Sensor firmware presets and the host output geometry associated with them. + patternProperties: + "^preset[0-9]+$": + type: object + properties: + apple,config-index: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Sensor configuration index understood by the firmware + apple,input-size: + $ref: /schemas/types.yaml#/definitions/uint32-array + items: + - description: Input width in pixels + - description: Input height in pixels + description: Sensor input dimensions + apple,output-size: + $ref: /schemas/types.yaml#/definitions/uint32-array + items: + - description: Output width in pixels + - description: Output height in pixels + description: Delivered frame dimensions + apple,crop: + $ref: /schemas/types.yaml#/definitions/uint32-array + items: + - description: Horizontal crop origin in pixels + - description: Vertical crop origin in pixels + - description: Crop width in pixels + - description: Crop height in pixels + description: Cropped sensor rectangle + required: + - apple,config-index + - apple,input-size + - apple,output-size + - apple,crop + additionalProperties: false + additionalProperties: false + +required: + - compatible + - reg + - reg-names + - interrupts + - iommus + - power-domains + - memory-region + - memory-region-names + - apple,dart-vm-size + - apple,platform-id + - sensor-presets + +additionalProperties: false From 040539b2358a9945ec71180518770a61279d27bd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:12:01 +1000 Subject: [PATCH 0538/2680] media: apple: isp: Fix the power domain unwind on power-up failure apple_isp_power_up_domains() takes a runtime PM reference on each auxiliary power domain, index 1 and up; index 0 follows the ISP device through its device link. When one of them fails to resume, the unwind loop `while (--i != 1)` releases the wrong set of references: a failure on domain 2 releases nothing and leaks domain 1, a failure on domain 3 releases domain 2 and leaks domain 1, and a failure on domain 1 releases domain 0, which was never taken here, and then keeps going below zero, indexing pd_dev[-1] and beyond. pm_runtime_get_sync() also leaves the usage count of the domain that failed incremented. Release every domain from i - 1 down to 1, and use pm_runtime_resume_and_get(), which drops the reference of the domain that failed to resume. Compile-tested only; not run on hardware. Fixes: b11332c267da ("media: apple: isp: Make sub-pmdomain handling explicit") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-fw.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 87433669526b1..587c079e7f815 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -51,11 +51,11 @@ static int apple_isp_power_up_domains(struct apple_isp *isp) return 0; for (int i = 1; i < isp->pd_count; i++) { - ret = pm_runtime_get_sync(isp->pd_dev[i]); + ret = pm_runtime_resume_and_get(isp->pd_dev[i]); if (ret < 0) { dev_err(isp->dev, "Failed to power up power domain %d: %d\n", i, ret); - while (--i != 1) + while (--i >= 1) pm_runtime_put_sync(isp->pd_dev[i]); return ret; } From 41dd581c5cfe32a05ba8e65a39737a923b4289f1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:12:37 +1000 Subject: [PATCH 0539/2680] media: apple: isp: Power down the domains when boot stage 1 fails Boot stage 1 powers up the auxiliary power domains, resets the coprocessor, starts it and waits for the first firmware handshake. When the reset or the handshake fails, isp_firmware_boot_stage1() returns without undoing any of that, and isp_firmware_boot() does not call isp_firmware_shutdown_stage1() for the stage that failed. The domains stay powered with pds_active set, and after a handshake timeout the coprocessor is left running with its control register at 0x10. Nothing turns either off until a later boot succeeds and is shut down. Undo stage 1 on its own error paths: stop the coprocessor and power the domains down. Compile-tested only; not run on hardware. Fixes: b11332c267da ("media: apple: isp: Make sub-pmdomain handling explicit") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-fw.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 587c079e7f815..9e3888e560021 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -292,7 +292,7 @@ static int isp_firmware_boot_stage1(struct apple_isp *isp) err = isp_reset_coproc(isp); if (err < 0) - return err; + goto shutdown; isp_gpio_write32(isp, ISP_GPIO_0, 0x0); isp_gpio_write32(isp, ISP_GPIO_1, 0x0); @@ -322,10 +322,15 @@ static int isp_firmware_boot_stage1(struct apple_isp *isp) if (retries >= ISP_FIRMWARE_MAX_TRIES) { isp_err(isp, "never received first magic number from firmware\n"); - return -ENODEV; + err = -ENODEV; + goto shutdown; } return 0; + +shutdown: + isp_firmware_shutdown_stage1(isp); + return err; } int apple_isp_alloc_firmware_surface(struct apple_isp *isp) From e444f637e2f6423ac598416746633586f9d39d54 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:13:19 +1000 Subject: [PATCH 0540/2680] media: apple: isp: Check iommu_map_sgtable() for errors iommu_map_sgtable() returns the number of bytes it mapped or a negative error code. isp_surf_iommu_map() stores the result in an unsigned long, and apple_isp_iommu_map_sgt() compares the ssize_t against the u64 surface size. Either way a negative error becomes a huge value that passes the "mapped < size" check, so the failure is treated as success, and apple_isp_iommu_map_sgt() records the error code as the surface size that apple_isp_iommu_unmap_sgt() later unmaps. Treat a negative return as an error and propagate it. Should a short mapping ever be returned, unmap it before failing. Compile-tested only; not run on hardware. Fixes: 329bf83de183 ("media: apple: Add Apple ISP driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-iommu.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-iommu.c b/drivers/media/platform/apple/isp/isp-iommu.c index 9e64b25de5c99..ecb467b9801e6 100644 --- a/drivers/media/platform/apple/isp/isp-iommu.c +++ b/drivers/media/platform/apple/isp/isp-iommu.c @@ -101,7 +101,7 @@ static void isp_surf_iommu_unmap(struct apple_isp *isp, struct isp_surf *surf) static int isp_surf_iommu_map(struct apple_isp *isp, struct isp_surf *surf) { - unsigned long size; + ssize_t mapped; int err; err = sg_alloc_table_from_pages(&surf->sgt, surf->pages, @@ -112,13 +112,15 @@ static int isp_surf_iommu_map(struct apple_isp *isp, struct isp_surf *surf) return err; } - size = iommu_map_sgtable(isp->domain, surf->iova, &surf->sgt, - IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE); - if (size < surf->size) { + mapped = iommu_map_sgtable(isp->domain, surf->iova, &surf->sgt, + IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE); + if (mapped < 0 || (u64)mapped < surf->size) { dev_err(isp->dev, "failed to iommu_map sgt to iova %pad\n", &surf->iova); + if (mapped > 0) + iommu_unmap(isp->domain, surf->iova, mapped); sg_free_table(&surf->sgt); - return -ENXIO; + return mapped < 0 ? mapped : -ENXIO; } return 0; @@ -233,11 +235,13 @@ int apple_isp_iommu_map_sgt(struct apple_isp *isp, struct isp_surf *surf, mapped = iommu_map_sgtable(isp->domain, surf->iova, sgt, IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE); - if (mapped < surf->size) { + if (mapped < 0 || (u64)mapped < surf->size) { dev_err(isp->dev, "failed to iommu_map sgt to iova %pad\n", &surf->iova); + if (mapped > 0) + iommu_unmap(isp->domain, surf->iova, mapped); isp_surf_unreserve_iova(isp, surf); - return -ENXIO; + return mapped < 0 ? mapped : -ENXIO; } surf->size = mapped; From db5fa37aea4cd1280320526eb0db483e46cdc32f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:14:20 +1000 Subject: [PATCH 0541/2680] media: apple: isp: Validate the boot arguments offset In boot stage 2 the firmware reports through GPIO 1 where in the IPC surface the host should place the boot arguments. The driver adds that offset to the surface's IOVA and to its kernel mapping without any check, writes the 0x180-byte boot arguments there and places the command area, which every later command and buffer batch uses, right behind them. A bad offset makes the host write outside the IPC surface's vmap. Before writing anything, check that the boot arguments and the start of the command area lie inside the IPC surface, and fail the boot otherwise. The command area must hold at least the 0x280 bytes the firmware reads for a buffer batch, which is more than any command needs. Compile-tested only; not run on hardware. Fixes: 5eca30dc648f ("media: apple: isp: Remove ioread/iowrite and stop doing raw address translation") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-fw.c | 33 ++++++++++++++++++----- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 9e3888e560021..ad54a058f1021 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -370,8 +371,8 @@ static void isp_firmware_shutdown_stage2(struct apple_isp *isp) static int isp_firmware_boot_stage2(struct apple_isp *isp) { struct isp_firmware_bootargs args; - dma_addr_t args_iova; - void *args_virt; + dma_addr_t args_iova, cmd_iova; + void *args_virt, *cmd_virt; int err, retries; u32 num_ipc_chans = isp_gpio_read32(isp, ISP_GPIO_0); @@ -388,16 +389,36 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) dev_warn(isp->dev, "unexpected channel count (%d)\n", num_ipc_chans); + /* + * The firmware picks the offset of the boot arguments in the IPC + * surface, and the command area follows them. Both have to lie + * inside the surface; the command area must hold the 0x280 bytes the + * firmware reads for a buffer batch, more than any command needs. + */ + args_virt = NULL; + cmd_virt = NULL; + if (!check_add_overflow(isp->ipc_surf->iova, (u64)args_offset + 0x40, + &args_iova) && + !check_add_overflow(args_iova, sizeof(args) + 0x40, &cmd_iova)) { + args_virt = apple_isp_ipc_translate(isp, args_iova, + sizeof(args)); + cmd_virt = apple_isp_ipc_translate(isp, cmd_iova, + ISP_IPC_BUFEXC_STAT_SIZE); + } + if (!args_virt || !cmd_virt) { + dev_err(isp->dev, "invalid boot arguments offset 0x%x\n", + args_offset); + return -EIO; + } + isp->extra_surf = isp_alloc_surface_vmap(isp, extra_size); if (!isp->extra_surf) { isp_err(isp, "failed to alloc surface for extra heap\n"); return -ENOMEM; } - args_iova = isp->ipc_surf->iova + args_offset + 0x40; - args_virt = isp->ipc_surf->virt + args_offset + 0x40; - isp->cmd_iova = args_iova + sizeof(args) + 0x40; - isp->cmd_virt = args_virt + sizeof(args) + 0x40; + isp->cmd_iova = cmd_iova; + isp->cmd_virt = cmd_virt; memset(&args, 0, sizeof(args)); args.ipc_iova = isp->ipc_surf->iova; From ffebd81d7f2e559752b1249c9596c9f74a50a2bf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:15:27 +1000 Subject: [PATCH 0542/2680] media: apple: isp: Validate the IPC channel table After the second handshake the firmware publishes a table of channel descriptors in the IPC surface, and isp_fill_channel_info() trusts it: - desc.num * ISP_IPC_MESSAGE_SIZE is computed in 32 bits, so a large message count wraps to a small ring size that passes the bounds check in apple_isp_ipc_translate(), and boot stage 3 then writes desc.num 64-byte messages past the IPC surface's vmap; - desc.num == 0 makes chan_update_cursor() compare the cursor against num - 1 == UINT_MAX, so the cursor walks out of the ring; - 1 << desc.src is undefined for a source of 32 or more; - kstrdup() reads the 64-byte name field up to a NUL the firmware does not have to provide, and a failed allocation leaves a NULL name that isp_get_chan_index() hands to strcasecmp(); - the channel count read from GPIO 0 in stage 2 is stored in an int and only checked against zero. Accept at most 32 channels, one per doorbell bit, compute the ring size in 64 bits, require at least one message slot and a source below 32, copy at most the 64 bytes of the name field and check that allocation. Check the channel type before translating the ring, return -EIO instead of -ENOMEM for a malformed table, and make isp_free_channel_info() cope with a failed allocation of the channel array. Compile-tested only; not run on hardware. Fixes: 329bf83de183 ("media: apple: Add Apple ISP driver") Fixes: 5eca30dc648f ("media: apple: isp: Remove ioread/iowrite and stop doing raw address translation") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-fw.c | 51 +++++++++++++++------- drivers/media/platform/apple/isp/isp-ipc.h | 3 ++ 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index ad54a058f1021..1582b5f959672 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -380,8 +380,9 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) u32 extra_size = isp_gpio_read32(isp, ISP_GPIO_3); isp->num_ipc_chans = num_ipc_chans; - if (!isp->num_ipc_chans) { - dev_err(isp->dev, "No IPC channels found\n"); + if (!num_ipc_chans || num_ipc_chans > ISP_IPC_MAX_CHANNELS) { + dev_err(isp->dev, "invalid IPC channel count %u\n", + num_ipc_chans); return -ENODEV; } @@ -481,6 +482,9 @@ static inline struct isp_channel *isp_get_chan_index(struct apple_isp *isp, static void isp_free_channel_info(struct apple_isp *isp) { + if (!isp->ipc_chans) + return; + for (int i = 0; i < isp->num_ipc_chans; i++) { struct isp_channel *chan = isp->ipc_chans[i]; if (!chan) @@ -499,7 +503,8 @@ static int isp_fill_channel_info(struct apple_isp *isp) ((u64)isp_gpio_read32(isp, ISP_GPIO_1)) << 32; void *table_virt = apple_isp_ipc_translate( isp, table_iova, - sizeof(struct isp_chan_desc) * isp->num_ipc_chans); + array_size(sizeof(struct isp_chan_desc), isp->num_ipc_chans)); + int err = -EIO; if (!table_virt) { dev_err(isp->dev, "Failed to find channel table\n"); @@ -509,35 +514,34 @@ static int isp_fill_channel_info(struct apple_isp *isp) isp->ipc_chans = kcalloc(isp->num_ipc_chans, sizeof(struct isp_channel *), GFP_KERNEL); if (!isp->ipc_chans) - goto out; + return -ENOMEM; for (int i = 0; i < isp->num_ipc_chans; i++) { struct isp_chan_desc desc; void *desc_virt = table_virt + (i * sizeof(desc)); struct isp_channel *chan = kzalloc(sizeof(struct isp_channel), GFP_KERNEL); - if (!chan) + if (!chan) { + err = -ENOMEM; goto out; + } isp->ipc_chans[i] = chan; memcpy(&desc, desc_virt, sizeof(desc)); - chan->name = kstrdup(desc.name, GFP_KERNEL); + /* The firmware does not have to NUL-terminate the name. */ + chan->name = kstrndup(desc.name, sizeof(desc.name), GFP_KERNEL); + if (!chan->name) { + err = -ENOMEM; + goto out; + } chan->type = desc.type; chan->src = desc.src; - chan->doorbell = 1 << chan->src; chan->num = desc.num; - chan->size = desc.num * ISP_IPC_MESSAGE_SIZE; + chan->size = (u64)desc.num * ISP_IPC_MESSAGE_SIZE; chan->iova = desc.iova; - chan->virt = - apple_isp_ipc_translate(isp, desc.iova, chan->size); chan->cursor = 0; mutex_init(&chan->lock); - if (!chan->virt) { - dev_err(isp->dev, "Failed to find channel buffer\n"); - goto out; - } - if ((chan->type != ISP_IPC_CHAN_TYPE_COMMAND) && (chan->type != ISP_IPC_CHAN_TYPE_REPLY) && (chan->type != ISP_IPC_CHAN_TYPE_REPORT)) { @@ -546,6 +550,21 @@ static int isp_fill_channel_info(struct apple_isp *isp) goto out; } + /* Each channel has its own doorbell bit and at least one slot. */ + if (chan->src >= ISP_IPC_MAX_CHANNELS || !chan->num) { + isp_err(isp, "invalid ipc chan %s (src %u, num %u)\n", + chan->name, chan->src, chan->num); + goto out; + } + chan->doorbell = BIT(chan->src); + + chan->virt = + apple_isp_ipc_translate(isp, chan->iova, chan->size); + if (!chan->virt) { + dev_err(isp->dev, "Failed to find channel buffer\n"); + goto out; + } + isp_dbg(isp, "chan: %s type: %d src: %d num: %d iova: %pad\n", chan->name, chan->type, chan->src, chan->num, &chan->iova); @@ -572,7 +591,7 @@ static int isp_fill_channel_info(struct apple_isp *isp) return 0; out: isp_free_channel_info(isp); - return -ENOMEM; + return err; } static void isp_firmware_shutdown_stage3(struct apple_isp *isp) diff --git a/drivers/media/platform/apple/isp/isp-ipc.h b/drivers/media/platform/apple/isp/isp-ipc.h index 0c1d681835c72..7bc95a2005465 100644 --- a/drivers/media/platform/apple/isp/isp-ipc.h +++ b/drivers/media/platform/apple/isp/isp-ipc.h @@ -10,6 +10,9 @@ #define ISP_IPC_CHAN_TYPE_REPLY 1 #define ISP_IPC_CHAN_TYPE_REPORT 2 +/* each channel owns one bit of the 32-bit doorbell register */ +#define ISP_IPC_MAX_CHANNELS 32 + #define ISP_IPC_BUFEXC_STAT_SIZE 0x280 #define ISP_IPC_BUFEXC_FLAG_RENDER 0x10000000 #define ISP_IPC_BUFEXC_FLAG_COMMAND 0x30000000 From 333b9b3591accc335a82f688b00ba1ad64c97215 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:16:10 +1000 Subject: [PATCH 0543/2680] media: apple: isp: Bound firmware-requested surface sizes The firmware chooses the size of two kinds of host allocations: the extra heap it asks for through GPIO 3 in boot stage 2, and the surfaces it requests on the SHAREDMALLOC channel. __isp_alloc_surface() accepts any u64 for them. A zero size produces an empty surface, and a size whose page count does not fit the u32 page counter truncates it, after which the page array and the scatterlist built from it no longer describe the size that is reserved and mapped. Reject a zero size and any size larger than the ISP's IOVA range, which no allocation could satisfy anyway; this also keeps the page count within 32 bits. The limit applies to the driver's own allocations as well, all of which are far smaller. Compile-tested only; not run on hardware. Fixes: 329bf83de183 ("media: apple: Add Apple ISP driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.c | 4 ++-- drivers/media/platform/apple/isp/isp-drv.h | 1 + drivers/media/platform/apple/isp/isp-iommu.c | 13 ++++++++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 8cd1dcc91676b..5ed54cd2c272b 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -128,8 +128,8 @@ static int apple_isp_init_iommu(struct apple_isp *isp) } // FIXME: refactor this, maybe use regular iova stuff? - drm_mm_init(&isp->iovad, isp->fw.heap_top, - vm_size - (heap_base & 0xffffffff)); + isp->iova_size = vm_size - (heap_base & 0xffffffff); + drm_mm_init(&isp->iovad, isp->fw.heap_top, isp->iova_size); return 0; } diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 227e5af4cde7a..abdab66f6b42c 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -224,6 +224,7 @@ struct apple_isp { struct iommu_domain *domain; unsigned long shift; struct drm_mm iovad; /* TODO iova.c can't allocate bottom-up */ + u64 iova_size; /* size of the iovad range */ struct mutex iovad_lock; struct isp_firmware { diff --git a/drivers/media/platform/apple/isp/isp-iommu.c b/drivers/media/platform/apple/isp/isp-iommu.c index ecb467b9801e6..68abaa4b33c83 100644 --- a/drivers/media/platform/apple/isp/isp-iommu.c +++ b/drivers/media/platform/apple/isp/isp-iommu.c @@ -138,9 +138,20 @@ static void __isp_surf_init(struct apple_isp *isp, struct isp_surf *surf, struct isp_surf *__isp_alloc_surface(struct apple_isp *isp, u64 size, bool gc) { + struct isp_surf *surf; int err; - struct isp_surf *surf = kzalloc(sizeof(struct isp_surf), GFP_KERNEL); + /* + * The firmware chooses the size of the extra heap and of SHAREDMALLOC + * surfaces. Nothing larger than the IOVA range can be mapped, and the + * bound also keeps the page count within 32 bits. + */ + if (!size || size > isp->iova_size) { + dev_err(isp->dev, "invalid surface size 0x%llx\n", size); + return NULL; + } + + surf = kzalloc(sizeof(*surf), GFP_KERNEL); if (!surf) return NULL; From 5421390e380e040fd29bc7e2e730316674e04625 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:17:08 +1000 Subject: [PATCH 0544/2680] media: apple: isp: Fix the buffer return list validation ipc_bt_handle() parses the list of returned buffers that the firmware sends on the BUF_T2H channel. It does not check whether apple_isp_translate() found the list, so an address outside the buffer surface, or a buffer surface that was never identified, leads to a NULL dereference. Its bound on the number of entries subtracts the size of struct isp_buffer, the host's vb2 buffer wrapper, instead of the 16-byte list header; for lists shorter than that wrapper the subtraction wraps and every count passes, and the loop reads descriptors past the list and past the surface. The 64-bit count is also truncated to 32 bits before the check. Check the length against the buffer surface and the translation result against NULL, derive the bound from the list header, and keep the count in 64 bits. Compile-tested only; not run on hardware. Fixes: 903629ad2f73 ("media: apple: isp: Rework meta surface handling & buffer return") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-v4l2.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index e06b4f1d55c1a..66669cd99dbb0 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -47,22 +47,25 @@ int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan) struct isp_message *req = &chan->req, *rsp = &chan->rsp; struct isp_buffer *tmp, *buf; struct isp_buflist *bl; - u32 count; + u64 count; int err = 0; /* printk("H2T: 0x%llx 0x%llx 0x%llx\n", (long long)req->arg0, (long long)req->arg1, (long long)req->arg2); */ - if (req->arg1 < sizeof(struct isp_buflist)) { + if (!isp->bt_surf || req->arg1 < sizeof(*bl) || + req->arg1 > isp->bt_surf->size) { dev_err(isp->dev, "%s: Bad length 0x%llx\n", chan->name, req->arg1); return -EIO; } bl = apple_isp_translate(isp, isp->bt_surf, req->arg0, req->arg1); + if (!bl) + return -EIO; count = bl->num_buffers; - if (count > (req->arg1 - sizeof(struct isp_buffer)) / + if (count > (req->arg1 - sizeof(*bl)) / sizeof(struct isp_buflist_buffer)) { dev_err(isp->dev, "%s: Bad length 0x%llx\n", chan->name, req->arg1); @@ -70,7 +73,7 @@ int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan) } spin_lock(&isp->buf_lock); - for (int i = 0; i < count; i++) { + for (u64 i = 0; i < count; i++) { struct isp_buflist_buffer *bufd = &bl->buffers[i]; /* printk("Return: 0x%llx (%d)\n", bufd->iovas[0], From c201cd2f760cfbcc2d9dd133fa378b1af221fddf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:18:23 +1000 Subject: [PATCH 0545/2680] media: apple: isp: Bound buffer batches to the command area isp_submit_buffers() builds each BUF_H2T batch in the command area behind the boot arguments: a 16-byte header, then one 64-byte descriptor for every metadata surface the firmware does not hold and one for every pending capture buffer. Nothing limits the number of pending buffers here, since vb2 can hold more than the 16 buffers the driver asks for through VIDIOC_CREATE_BUFS, and nothing checks that the batch fits in the IPC surface, so a long pending list is written past its end. Metadata descriptors are also only partly written, so their unused planes, flags and tag keep whatever an earlier command or batch left in the command area. Define the command area as the largest batch the driver builds, all metadata surfaces plus one rendered pool of ISP_MAX_BUFFERS, and make boot stage 2 require that much room in the IPC surface. Put at most ISP_MAX_BUFFERS capture buffers into a batch and leave the rest pending for the next one, and clear the batch before filling it. Compile-tested only; not run on hardware. Fixes: 903629ad2f73 ("media: apple: isp: Rework meta surface handling & buffer return") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.h | 11 +++++++++++ drivers/media/platform/apple/isp/isp-fw.c | 6 +++--- drivers/media/platform/apple/isp/isp-v4l2.c | 18 ++++++++++++------ 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index abdab66f6b42c..3fd27ebb7f31b 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -28,6 +28,17 @@ /* used to limit the user space buffers to the buffer_pool_config */ #define ISP_MAX_BUFFERS 16 +/* + * The command area holds one command or one buffer batch at a time. A + * batch is a 16-byte header and a 64-byte descriptor per buffer, and + * carries at most every metadata buffer and one full rendered pool. + */ +#define ISP_BUFLIST_HDR_SIZE 0x10 +#define ISP_BUFLIST_DESC_SIZE 0x40 +#define ISP_CMD_AREA_SIZE \ + (ISP_BUFLIST_HDR_SIZE + \ + ISP_BUFLIST_DESC_SIZE * (ISP_MAX_BUFFERS + ISP_MAX_BUFFERS)) + enum isp_generation { ISP_GEN_T8103, ISP_GEN_T8112, diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 1582b5f959672..409c167056d87 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -393,8 +393,8 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) /* * The firmware picks the offset of the boot arguments in the IPC * surface, and the command area follows them. Both have to lie - * inside the surface; the command area must hold the 0x280 bytes the - * firmware reads for a buffer batch, more than any command needs. + * inside the surface, and the command area must hold the largest + * buffer batch, which is more than any command needs. */ args_virt = NULL; cmd_virt = NULL; @@ -404,7 +404,7 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) args_virt = apple_isp_ipc_translate(isp, args_iova, sizeof(args)); cmd_virt = apple_isp_ipc_translate(isp, cmd_iova, - ISP_IPC_BUFEXC_STAT_SIZE); + ISP_CMD_AREA_SIZE); } if (!args_virt || !cmd_virt) { dev_err(isp->dev, "invalid boot arguments offset 0x%x\n", diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 66669cd99dbb0..4423b1655da57 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -34,13 +34,16 @@ struct isp_buflist_buffer { u32 tag; u32 pad; } __packed; -static_assert(sizeof(struct isp_buflist_buffer) == 0x40); +static_assert(sizeof(struct isp_buflist_buffer) == ISP_BUFLIST_DESC_SIZE); struct isp_buflist { u64 type; u64 num_buffers; struct isp_buflist_buffer buffers[]; -}; +} __packed; +static_assert(sizeof(struct isp_buflist) == ISP_BUFLIST_HDR_SIZE); +/* the firmware reads at least ISP_IPC_BUFEXC_STAT_SIZE bytes of a batch */ +static_assert(ISP_CMD_AREA_SIZE >= ISP_IPC_BUFEXC_STAT_SIZE); int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan) { @@ -124,6 +127,7 @@ static int isp_submit_buffers(struct apple_isp *isp) struct isp_channel *chan = isp->chan_bh; struct isp_message *req = &chan->req; struct isp_buffer *buf, *tmp; + unsigned int num_rendered = 0; unsigned long flags; size_t offset; int err; @@ -131,8 +135,9 @@ static int isp_submit_buffers(struct apple_isp *isp) struct isp_buflist *bl = isp->cmd_virt; struct isp_buflist_buffer *bufd = &bl->buffers[0]; + /* Clear what earlier commands and batches left in the reserved fields. */ + memset(bl, 0, ISP_CMD_AREA_SIZE); bl->type = 1; - bl->num_buffers = 0; spin_lock_irqsave(&isp->buf_lock, flags); for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) { @@ -154,10 +159,10 @@ static int isp_submit_buffers(struct apple_isp *isp) meta->submitted = true; } - while ((buf = list_first_entry_or_null(&isp->bufs_pending, + /* One rendered pool per batch; the rest stays pending for the next. */ + while (num_rendered < ISP_MAX_BUFFERS && + (buf = list_first_entry_or_null(&isp->bufs_pending, struct isp_buffer, link))) { - memset(bufd, 0, sizeof(*bufd)); - bufd->num_planes = fmt->num_planes; bufd->pool_type = isp->hw->scl1 ? CISP_POOL_TYPE_RENDERED_SCL1 : CISP_POOL_TYPE_RENDERED; @@ -173,6 +178,7 @@ static int isp_submit_buffers(struct apple_isp *isp) buf->surfs[0].iova + buf->surfs[0].size); */ bufd++; bl->num_buffers++; + num_rendered++; /* * Queue the buffer as submitted and release the lock for now. From 2306d06a5e394df5448fbf90a98fc25184ec2d9e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:19:03 +1000 Subject: [PATCH 0546/2680] media: apple: isp: Free the channel info reply for unsupported sensors isp_ch_cache_sensor_info() allocates the CH_INFO_GET reply buffer and frees it on the common exit path, but returns -ENODEV directly when the sensor is unknown or not supported, leaking the buffer. Take the common exit path. Compile-tested only; not run on hardware. Fixes: 329bf83de183 ("media: apple: Add Apple ISP driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-cam.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index f84f0603260d4..b3c613d3e2986 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -223,7 +223,8 @@ static int isp_ch_cache_sensor_info(struct apple_isp *isp, u32 ch) dev_err(isp->dev, "ch %d: unsupported sensor. Please file a bug report with hardware info & dmesg trace.\n", ch); - return -ENODEV; + err = -ENODEV; + goto exit; } exit: From c1179ec29ba85402dce2b2446bba2c1287b783e1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:20:48 +1000 Subject: [PATCH 0547/2680] media: apple: isp: Keep the stream's buffers across system sleep apple_isp_video_suspend() hands every buffer the driver holds back to vb2 with VB2_BUF_STATE_QUEUED before stopping the stream. Outside of start_streaming that state requeues nothing: vb2 marks the buffers as queued but never passes them to the driver again, so the stream that apple_isp_video_resume() restarts has no capture buffers, no frame is ever completed and VIDIOC_DQBUF blocks forever. The resume path also ignores a failed restart, which leaves the queue in the same state, and a later VIDIOC_STREAMOFF then stops a stream that is not running: it sends CH_STOP to a firmware that was shut down, through the channel structures that were freed with it. Keep the buffers with the driver across sleep. Before stopping, move the buffers given to the firmware to the pending list, so that the buffer return for the stop does not complete them, and submit them again when the stream restarts on resume. If the restart fails, complete them with VB2_BUF_STATE_ERROR and flag the queue with vb2_queue_error(), so that userspace gets -EIO instead of waiting. Only stop the firmware stream from stop_streaming when it is running, and hold the video lock in the sleep callbacks as the vb2 callbacks do. Compile-tested only; not run on hardware. Fixes: a2a90c284cf1 ("media: apple: isp: Support system sleep") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-v4l2.c | 54 ++++++++++++++++----- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 4423b1655da57..5991cc7b1e3d3 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -353,7 +353,7 @@ static int apple_isp_start_streaming(struct apple_isp *isp) err = apple_isp_start_camera(isp); if (err) { dev_err(isp->dev, "failed to start camera: %d\n", err); - goto release_buffers; + return err; } err = isp_submit_buffers(isp); @@ -374,14 +374,15 @@ static int apple_isp_start_streaming(struct apple_isp *isp) stop_camera: apple_isp_stop_camera(isp); -release_buffers: - isp_vb2_release_buffers(isp, VB2_BUF_STATE_QUEUED); return err; } static void apple_isp_stop_streaming(struct apple_isp *isp) { - clear_bit(ISP_STATE_STREAMING, &isp->state); + /* Not running if restarting it after system sleep failed. */ + if (!test_and_clear_bit(ISP_STATE_STREAMING, &isp->state)) + return; + apple_isp_stop_capture(isp); apple_isp_stop_camera(isp); } @@ -389,10 +390,15 @@ static void apple_isp_stop_streaming(struct apple_isp *isp) static int isp_vb2_start_streaming(struct vb2_queue *q, unsigned int count) { struct apple_isp *isp = vb2_get_drv_priv(q); + int err; isp->sequence = 0; - return apple_isp_start_streaming(isp); + err = apple_isp_start_streaming(isp); + if (err) + isp_vb2_release_buffers(isp, VB2_BUF_STATE_QUEUED); + + return err; } static void isp_vb2_stop_streaming(struct vb2_queue *q) @@ -405,26 +411,50 @@ static void isp_vb2_stop_streaming(struct vb2_queue *q) int apple_isp_video_suspend(struct apple_isp *isp) { - /* Swap into STATE_SLEEPING as isp_vb2_buf_queue() submits on - * STATE_STREAMING. + unsigned long flags; + + mutex_lock(&isp->video_lock); + + /* + * Stop the stream but keep its buffers queued to the driver. First + * move the buffers given to the firmware to the pending list, so that + * the buffer return for the stop does not complete them; they are + * submitted again when the stream restarts on resume. + * isp_vb2_buf_queue() does not submit while the stream is stopped. */ if (test_bit(ISP_STATE_STREAMING, &isp->state)) { - /* Signal buffers to be recycled for clean shutdown */ - isp_vb2_release_buffers(isp, VB2_BUF_STATE_QUEUED); + spin_lock_irqsave(&isp->buf_lock, flags); + list_splice_init(&isp->bufs_submitted, &isp->bufs_pending); + spin_unlock_irqrestore(&isp->buf_lock, flags); + apple_isp_stop_streaming(isp); set_bit(ISP_STATE_SLEEPING, &isp->state); } + mutex_unlock(&isp->video_lock); + return 0; } int apple_isp_video_resume(struct apple_isp *isp) { - if (test_bit(ISP_STATE_SLEEPING, &isp->state)) { - clear_bit(ISP_STATE_SLEEPING, &isp->state); - apple_isp_start_streaming(isp); + int err; + + mutex_lock(&isp->video_lock); + + if (test_and_clear_bit(ISP_STATE_SLEEPING, &isp->state)) { + err = apple_isp_start_streaming(isp); + if (err) { + dev_err(isp->dev, + "failed to restart streaming after resume: %d\n", + err); + isp_vb2_release_buffers(isp, VB2_BUF_STATE_ERROR); + vb2_queue_error(&isp->vbq); + } } + mutex_unlock(&isp->video_lock); + return 0; } From 0918a5e8e497236ac80d35cb7c8e26d995c45f6e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:21:30 +1000 Subject: [PATCH 0548/2680] media: apple: isp: Drop the probe-time firmware dumps and preset queries Every probe prints the raw CONFIG_GET and CH_INFO_GET replies as hex dumps at KERN_INFO, the latter including the camera module's serial number, and then sends CH_CAMERA_CONFIG_GET for each preset the firmware reports, only to hex-dump the replies. The driver uses none of it; it is bring-up output. The probe also announces itself and prints the IOVAs of the IPC and data surfaces at info level. Remove the dumps and the preset queries, together with isp_cmd_ch_camera_config_get(), which has no other user. Log the sensor version at debug level and without the serial number, and demote the surface addresses to debug. On every SoC this removes one firmware command per preset from probe. Those commands only read state, so the firmware's configuration is not affected. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-cam.c | 38 +--------------------- drivers/media/platform/apple/isp/isp-cmd.c | 9 ----- drivers/media/platform/apple/isp/isp-cmd.h | 2 -- drivers/media/platform/apple/isp/isp-drv.c | 2 -- drivers/media/platform/apple/isp/isp-fw.c | 8 ++--- 5 files changed, 5 insertions(+), 54 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index b3c613d3e2986..c5642de57b873 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -166,29 +166,6 @@ static int isp_ch_get_sensor_id(struct apple_isp *isp, u32 ch) return err; } -static int isp_ch_get_camera_preset(struct apple_isp *isp, u32 ch, u32 ps) -{ - int err = 0; - - struct cmd_ch_camera_config *args; /* Too big to allocate on stack */ - args = kzalloc(sizeof(*args), GFP_KERNEL); - if (!args) - return -ENOMEM; - - err = isp_cmd_ch_camera_config_get(isp, ch, ps, args); - if (err) - goto exit; - - pr_info("apple-isp: ps: CISP_CMD_CH_CAMERA_CONFIG_GET: %d\n", ps); - print_hex_dump(KERN_INFO, "apple-isp: ps: ", DUMP_PREFIX_NONE, 32, 4, - args, sizeof(*args), false); - -exit: - kfree(args); - - return err; -} - static int isp_ch_cache_sensor_info(struct apple_isp *isp, u32 ch) { struct isp_format *fmt = isp_get_format(isp, ch); @@ -203,19 +180,10 @@ static int isp_ch_cache_sensor_info(struct apple_isp *isp, u32 ch) if (err) goto exit; - dev_info(isp->dev, "found sensor %x %s on ch %d\n", args->version, - args->module_sn, ch); + dev_dbg(isp->dev, "found sensor %x on ch %d\n", args->version, ch); fmt->version = args->version; - pr_info("apple-isp: ch: CISP_CMD_CH_INFO_GET: %d\n", ch); - print_hex_dump(KERN_INFO, "apple-isp: ch: ", DUMP_PREFIX_NONE, 32, 4, - args, sizeof(*args), false); - - for (u32 ps = 0; ps < args->num_presets; ps++) { - isp_ch_get_camera_preset(isp, ch, ps); - } - err = isp_ch_get_sensor_id(isp, ch); if (err || (fmt->id != ISP_IMX248_1820_01 && fmt->id != ISP_IMX558_1921_01 && @@ -244,10 +212,6 @@ static int isp_detect_camera(struct apple_isp *isp) if (err) return err; - pr_info("apple-isp: CISP_CMD_CONFIG_GET: \n"); - print_hex_dump(KERN_INFO, "apple-isp: ", DUMP_PREFIX_NONE, 32, 4, &args, - sizeof(args), false); - if (!args.num_channels) { dev_err(isp->dev, "did not detect any channels\n"); return -ENODEV; diff --git a/drivers/media/platform/apple/isp/isp-cmd.c b/drivers/media/platform/apple/isp/isp-cmd.c index ee491d2cb42c5..48aa139a81ccf 100644 --- a/drivers/media/platform/apple/isp/isp-cmd.c +++ b/drivers/media/platform/apple/isp/isp-cmd.c @@ -223,15 +223,6 @@ int isp_cmd_ch_info_get(struct apple_isp *isp, u32 chan, return CISP_SEND_OUT(isp, args); } -int isp_cmd_ch_camera_config_get(struct apple_isp *isp, u32 chan, u32 preset, - struct cmd_ch_camera_config *args) -{ - args->opcode = CISP_OPCODE(CISP_CMD_CH_CAMERA_CONFIG_GET); - args->preset = preset; - args->chan = chan; - return CISP_SEND_OUT(isp, args); -} - int isp_cmd_ch_camera_config_current_get(struct apple_isp *isp, u32 chan, struct cmd_ch_camera_config *args) { diff --git a/drivers/media/platform/apple/isp/isp-cmd.h b/drivers/media/platform/apple/isp/isp-cmd.h index 5a3c8cd9177e4..3301ccf633ff0 100644 --- a/drivers/media/platform/apple/isp/isp-cmd.h +++ b/drivers/media/platform/apple/isp/isp-cmd.h @@ -471,8 +471,6 @@ int isp_cmd_ch_start(struct apple_isp *isp, u32 chan); int isp_cmd_ch_stop(struct apple_isp *isp, u32 chan); int isp_cmd_ch_info_get(struct apple_isp *isp, u32 chan, struct cmd_ch_info *args); -int isp_cmd_ch_camera_config_get(struct apple_isp *isp, u32 chan, u32 preset, - struct cmd_ch_camera_config *args); int isp_cmd_ch_camera_config_current_get(struct apple_isp *isp, u32 chan, struct cmd_ch_camera_config *args); int isp_cmd_ch_camera_config_select(struct apple_isp *isp, u32 chan, diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 5ed54cd2c272b..8c083424fd3f1 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -409,8 +409,6 @@ static int apple_isp_probe(struct platform_device *pdev) goto free_surface; } - dev_info(dev, "apple-isp probe!\n"); - return 0; free_surface: diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 409c167056d87..2e86880c0184f 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -342,8 +342,8 @@ int apple_isp_alloc_firmware_surface(struct apple_isp *isp) isp_err(isp, "failed to alloc shared surface for ipc\n"); return -ENOMEM; } - dev_info(isp->dev, "IPC surface iova: 0x%llx\n", - (long long)isp->ipc_surf->iova); + dev_dbg(isp->dev, "IPC surface iova: 0x%llx\n", + (long long)isp->ipc_surf->iova); isp->data_surf = isp_alloc_surface_vmap(isp, ISP_FIRMWARE_DATA_SIZE); if (!isp->data_surf) { @@ -351,8 +351,8 @@ int apple_isp_alloc_firmware_surface(struct apple_isp *isp) isp_free_surface(isp, isp->ipc_surf); return -ENOMEM; } - dev_info(isp->dev, "Data surface iova: 0x%llx\n", - (long long)isp->data_surf->iova); + dev_dbg(isp->dev, "Data surface iova: 0x%llx\n", + (long long)isp->data_surf->iova); return 0; } From c9efd9c9b3f3c02dcdb78b5837865dbc482b2b4c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:22:10 +1000 Subject: [PATCH 0549/2680] media: apple: isp: Print firmware log messages through dynamic debug The firmware sends its log output on the TERMINAL channel. The handler only decodes it when APPLE_ISP_DEBUG is defined in isp-drv.h at build time, and that define also turns every isp_dbg() message into an unconditional info message; without it the text is dropped. Always decode the log text and print it with dev_dbg(), so that it can be enabled at runtime through dynamic debug, and remove APPLE_ISP_DEBUG. Print the text directly from the log surface instead of copying it to the stack first, and cut lines at 512 bytes rather than dropping them. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.h | 6 ----- drivers/media/platform/apple/isp/isp-ipc.c | 26 +++++++++------------- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 3fd27ebb7f31b..717f67da997db 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -14,7 +14,6 @@ #include #include -/* #define APPLE_ISP_DEBUG */ #define APPLE_ISP_DEVICE_NAME "apple-isp" #define APPLE_ISP_CARD_NAME "FaceTime HD Camera" @@ -289,13 +288,8 @@ enum { ISP_STATE_SLEEPING, }; -#ifdef APPLE_ISP_DEBUG -#define isp_dbg(isp, fmt, ...) \ - dev_info((isp)->dev, "[%s] " fmt, __func__, ##__VA_ARGS__) -#else #define isp_dbg(isp, fmt, ...) \ dev_dbg((isp)->dev, "[%s] " fmt, __func__, ##__VA_ARGS__) -#endif #define isp_err(isp, fmt, ...) \ dev_err((isp)->dev, "[%s] " fmt, __func__, ##__VA_ARGS__) diff --git a/drivers/media/platform/apple/isp/isp-ipc.c b/drivers/media/platform/apple/isp/isp-ipc.c index a1948717a3196..e43e7ad5ba91c 100644 --- a/drivers/media/platform/apple/isp/isp-ipc.c +++ b/drivers/media/platform/apple/isp/isp-ipc.c @@ -7,6 +7,8 @@ #include "isp-fw.h" #define ISP_IPC_FLAG_TERMINAL_ACK 0x3 +/* longer firmware log lines are cut */ +#define ISP_IPC_TERMINAL_MAX_LEN 512 #define ISP_IPC_BUFEXC_STAT_META_OFFSET 0x10 struct isp_sm_deferred_work { @@ -196,23 +198,17 @@ int ipc_chan_send(struct apple_isp *isp, struct isp_channel *chan, int ipc_tm_handle(struct apple_isp *isp, struct isp_channel *chan) { - struct isp_message *rsp = &chan->rsp; - -#ifdef APPLE_ISP_DEBUG - struct isp_message *req = &chan->req; - char buf[512]; + struct isp_message *req = &chan->req, *rsp = &chan->rsp; dma_addr_t iova = req->arg0 & ~ISP_IPC_FLAG_TERMINAL_ACK; - u32 size = req->arg1; - if (iova && size && size < sizeof(buf) && - isp->log_surf) { - void *p = apple_isp_translate(isp, isp->log_surf, iova, size); - if (p) { - size = min_t(u32, size, 512); - memcpy(buf, p, size); - isp_dbg(isp, "ISPASC: %.*s", size, buf); - } + size_t size = min_t(u64, req->arg1, ISP_IPC_TERMINAL_MAX_LEN); + const char *line; + + /* Firmware log output, available through dynamic debug */ + if (iova && size && isp->log_surf) { + line = apple_isp_translate(isp, isp->log_surf, iova, size); + if (line) + dev_dbg(isp->dev, "ISPASC: %.*s", (int)size, line); } -#endif rsp->arg0 = ISP_IPC_FLAG_ACK; rsp->arg1 = 0x0; From d84dacca91435ad4455795e605e4a1802ea097e4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:23:04 +1000 Subject: [PATCH 0550/2680] media: apple: isp: Warn once about a missing setfile The per-sensor setfile carries the camera module's calibration and is optional: without it the driver streams uncalibrated. Yet every stream start without the file logs three messages: the firmware loader's warning, "failed to request setfile" and "calibration data not loaded". Request the file with firmware_request_nowarn() and warn once, naming the file. A setfile that is present but too short or has the wrong magic is still an error each time, and now says which file and magic it found. Read the magic with get_unaligned_be32() rather than through a cast, and refuse a table entry larger than the data surface instead of copying past it. Document the file name and search path in the Kconfig help, which the driver did not have. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/Kconfig | 7 ++++ drivers/media/platform/apple/isp/isp-cam.c | 37 ++++++++++++---------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/drivers/media/platform/apple/isp/Kconfig b/drivers/media/platform/apple/isp/Kconfig index 8e339db43bc41..5d818e55448a4 100644 --- a/drivers/media/platform/apple/isp/Kconfig +++ b/drivers/media/platform/apple/isp/Kconfig @@ -10,3 +10,10 @@ config VIDEO_APPLE_ISP depends on OF_ADDRESS depends on V4L_PLATFORM_DRIVERS depends on VIDEO_DEV + help + Driver for the image signal processor that runs the built-in + camera of Apple silicon machines. The ISP firmware is loaded by + the bootloader. When a stream starts, the driver loads the + calibration file for the detected camera module, + apple/isp__XX.dat, from the firmware search + path; if it is missing, the stream runs without calibration. diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index c5642de57b873..784ae3e1bd9ab 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -2,6 +2,7 @@ /* Copyright 2023 Eileen Yoon */ #include +#include #include "isp-cam.h" #include "isp-cmd.h" @@ -274,28 +275,33 @@ static int isp_ch_load_setfile(struct apple_isp *isp, u32 ch) u32 magic; int err; - err = request_firmware(&fw, setfile->path, isp->dev); + if (WARN_ON_ONCE(setfile->size > isp->data_surf->size)) + return -ENOSPC; + + err = firmware_request_nowarn(&fw, setfile->path, isp->dev); if (err) { - dev_err(isp->dev, "failed to request setfile '%s': %d\n", - setfile->path, err); + dev_warn_once(isp->dev, + "setfile '%s' not loaded (%d), streaming without calibration\n", + setfile->path, err); return err; } if (fw->size < setfile->size) { - dev_err(isp->dev, "setfile too small (0x%zx/0x%zx)\n", fw->size, - setfile->size); + dev_err(isp->dev, "setfile '%s' too small (0x%zx/0x%zx)\n", + setfile->path, fw->size, setfile->size); release_firmware(fw); return -EINVAL; } - magic = be32_to_cpup((__be32 *)fw->data); + magic = get_unaligned_be32(fw->data); if (magic != setfile->magic) { - dev_err(isp->dev, "setfile '%s' corrupted?\n", setfile->path); + dev_err(isp->dev, "setfile '%s' has magic 0x%08x, expected 0x%08x\n", + setfile->path, magic, setfile->magic); release_firmware(fw); return -EINVAL; } - memcpy(isp->data_surf->virt, (void *)fw->data, setfile->size); + memcpy(isp->data_surf->virt, fw->data, setfile->size); release_firmware(fw); return isp_cmd_ch_set_file_load(isp, ch, isp->data_surf->iova, @@ -309,16 +315,13 @@ static int isp_ch_configure_capture(struct apple_isp *isp, u32 ch) isp_cmd_flicker_sensor_set(isp, 0); - /* The setfile isn't requisite but then we don't get calibration */ + /* + * The setfile isn't requisite but then we don't get calibration. If + * loading it was interrupted by a signal, give up on the stream. + */ err = isp_ch_load_setfile(isp, ch); - if (err) { - dev_err(isp->dev, "warning: calibration data not loaded: %d\n", - err); - - /* If this failed due to a signal, propagate */ - if (err == -EINTR) - return err; - } + if (err == -EINTR) + return err; if (isp->hw->lpdp) { err = isp_cmd_ch_lpdp_hs_receiver_tuning_set(isp, ch, 1, 15); From 0a4e60f0c0b115d23369fc562386acfa1e512f85 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:24:09 +1000 Subject: [PATCH 0551/2680] media: apple: isp: Sleep while waiting for the firmware The coprocessor reset and the four firmware handshakes poll their registers with mdelay(1), up to 1000 times each, so a firmware that does not answer keeps a CPU spinning for a second per wait. That happens on every failed boot, and a boot that fails during probe is retried through deferred probing. All of these waits run in process context. Use readl_poll_timeout(), which sleeps between reads, with the same 1 ms interval and one second limit. The two short read loops on the reset acknowledge registers do not delay and stay as they are. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-fw.c | 86 +++++++---------------- 1 file changed, 26 insertions(+), 60 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 2e86880c0184f..faac21ffd6fa3 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -16,8 +17,8 @@ #include "isp-regs.h" #include "isp-v4l2.h" -#define ISP_FIRMWARE_MDELAY 1 -#define ISP_FIRMWARE_MAX_TRIES 1000 +#define ISP_FIRMWARE_POLL_US 1000 +#define ISP_FIRMWARE_TIMEOUT_US 1000000 #define ISP_FIRMWARE_IPC_SIZE 0x1c000 #define ISP_FIRMWARE_DATA_SIZE 0x28000 @@ -44,6 +45,16 @@ static inline void isp_gpio_write32(struct apple_isp *isp, u32 reg, u32 val) writel(val, isp->gpio + reg); } +/* Wait for the firmware to write @expected to a GPIO word. */ +static int isp_gpio_wait(struct apple_isp *isp, u32 reg, u32 expected) +{ + u32 val; + + return readl_poll_timeout(isp->gpio + reg, val, val == expected, + ISP_FIRMWARE_POLL_US, + ISP_FIRMWARE_TIMEOUT_US); +} + static int apple_isp_power_up_domains(struct apple_isp *isp) { int ret; @@ -245,19 +256,13 @@ static int isp_reset_coproc(struct apple_isp *isp) break; } - for (retries = 0; retries < ISP_FIRMWARE_MAX_TRIES; retries++) { - status = isp_coproc_read32(isp, ISP_COPROC_STATUS); - if (status & ISP_COPROC_IN_WFI) { - isp_dbg(isp, "%d: coproc in WFI (status: 0x%x)\n", - retries, status); - break; - } - mdelay(ISP_FIRMWARE_MDELAY); - } - if (retries >= ISP_FIRMWARE_MAX_TRIES) { + if (readl_poll_timeout(isp->coproc + ISP_COPROC_STATUS, status, + status & ISP_COPROC_IN_WFI, + ISP_FIRMWARE_POLL_US, ISP_FIRMWARE_TIMEOUT_US)) { isp_err(isp, "coproc NOT in WFI (status: 0x%x)\n", status); return -ENODEV; } + isp_dbg(isp, "coproc in WFI (status: 0x%x)\n", status); return 0; } @@ -271,7 +276,7 @@ static void isp_firmware_shutdown_stage1(struct apple_isp *isp) static int isp_firmware_boot_stage1(struct apple_isp *isp) { - int err, retries; + int err; // u32 val; err = apple_isp_power_up_domains(isp); @@ -310,22 +315,13 @@ static int isp_firmware_boot_stage1(struct apple_isp *isp) isp_coproc_write32(isp, ISP_COPROC_CONTROL, 0x10); /* Wait for ISP_GPIO_7 to 0x0 -> 0x8042006 */ - for (retries = 0; retries < ISP_FIRMWARE_MAX_TRIES; retries++) { - u32 val = isp_gpio_read32(isp, ISP_GPIO_7); - if (val == 0x8042006) { - isp_dbg(isp, - "got first magic number (0x%x) from firmware\n", - val); - break; - } - mdelay(ISP_FIRMWARE_MDELAY); - } - if (retries >= ISP_FIRMWARE_MAX_TRIES) { + if (isp_gpio_wait(isp, ISP_GPIO_7, 0x8042006)) { isp_err(isp, "never received first magic number from firmware\n"); err = -ENODEV; goto shutdown; } + isp_dbg(isp, "got first magic number from firmware\n"); return 0; @@ -373,7 +369,7 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) struct isp_firmware_bootargs args; dma_addr_t args_iova, cmd_iova; void *args_virt, *cmd_virt; - int err, retries; + int err; u32 num_ipc_chans = isp_gpio_read32(isp, ISP_GPIO_0); u32 args_offset = isp_gpio_read32(isp, ISP_GPIO_1); @@ -446,22 +442,13 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) /* Wait for ISP_GPIO_7 to 0xf7fbdff9 -> 0x8042006 */ isp_gpio_write32(isp, ISP_GPIO_7, 0xf7fbdff9); - for (retries = 0; retries < ISP_FIRMWARE_MAX_TRIES; retries++) { - u32 val = isp_gpio_read32(isp, ISP_GPIO_7); - if (val == 0x8042006) { - isp_dbg(isp, - "got second magic number (0x%x) from firmware\n", - val); - break; - } - mdelay(ISP_FIRMWARE_MDELAY); - } - if (retries >= ISP_FIRMWARE_MAX_TRIES) { + if (isp_gpio_wait(isp, ISP_GPIO_7, 0x8042006)) { isp_err(isp, "never received second magic number from firmware\n"); err = -ENODEV; goto free_extra; } + isp_dbg(isp, "got second magic number from firmware\n"); return 0; @@ -601,7 +588,7 @@ static void isp_firmware_shutdown_stage3(struct apple_isp *isp) static int isp_firmware_boot_stage3(struct apple_isp *isp) { - int err, retries; + int err; err = isp_fill_channel_info(isp); if (err < 0) @@ -626,17 +613,7 @@ static int isp_firmware_boot_stage3(struct apple_isp *isp) /* Wait for ISP_GPIO_3 to 0x8042006 -> 0x0 */ isp_gpio_write32(isp, ISP_GPIO_3, 0x8042006); - for (retries = 0; retries < ISP_FIRMWARE_MAX_TRIES; retries++) { - u32 val = isp_gpio_read32(isp, ISP_GPIO_3); - if (val == 0x0) { - isp_dbg(isp, - "got third magic number (0x%x) from firmware\n", - val); - break; - } - mdelay(ISP_FIRMWARE_MDELAY); - } - if (retries >= ISP_FIRMWARE_MAX_TRIES) { + if (isp_gpio_wait(isp, ISP_GPIO_3, 0x0)) { isp_err(isp, "never received third magic number from firmware\n"); isp_free_channel_info(isp); @@ -650,8 +627,6 @@ static int isp_firmware_boot_stage3(struct apple_isp *isp) static int isp_stop_command_processor(struct apple_isp *isp) { - int retries; - #if 0 int res = isp_cmd_stop(isp, 0); if (res) { @@ -673,16 +648,7 @@ static int isp_stop_command_processor(struct apple_isp *isp) } #endif - for (retries = 0; retries < ISP_FIRMWARE_MAX_TRIES; retries++) { - u32 val = isp_gpio_read32(isp, ISP_GPIO_0); - if (val == 0x8042006) { - isp_dbg(isp, "got magic number (0x%x) from firmware\n", - val); - break; - } - mdelay(ISP_FIRMWARE_MDELAY); - } - if (retries >= ISP_FIRMWARE_MAX_TRIES) { + if (isp_gpio_wait(isp, ISP_GPIO_0, 0x8042006)) { isp_err(isp, "never received magic number from firmware\n"); return -ENODEV; } From 9f33929707d27411ebc983101b5902b749d980ef Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:24:47 +1000 Subject: [PATCH 0552/2680] media: apple: isp: Rate-limit errors caused by firmware messages Firmware messages arrive continuously while the camera streams. A malformed one makes apple_isp_translate(), a channel handler, the buffer return parser or the surface allocator print an error, and a firmware that keeps sending such messages floods the log at the frame rate. Use the rate-limited variants for these messages, and drop the stray parenthesis from the handler failure message. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-fw.c | 18 +++++++++--------- drivers/media/platform/apple/isp/isp-iommu.c | 3 ++- drivers/media/platform/apple/isp/isp-ipc.c | 8 +++++--- drivers/media/platform/apple/isp/isp-v4l2.c | 8 ++++---- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index faac21ffd6fa3..d68fbb14394c1 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -100,24 +100,24 @@ void *apple_isp_translate(struct apple_isp *isp, struct isp_surf *surf, { dma_addr_t end = iova + size; if (!surf) { - dev_err(isp->dev, - "Failed to translate IPC iova 0x%llx (0x%zx): No surface\n", - (long long)iova, size); + dev_err_ratelimited(isp->dev, + "Failed to translate IPC iova 0x%llx (0x%zx): No surface\n", + (long long)iova, size); return NULL; } if (end < iova || iova < surf->iova || end > (surf->iova + surf->size)) { - dev_err(isp->dev, - "Failed to translate IPC iova 0x%llx (0x%zx): Out of bounds\n", - (long long)iova, size); + dev_err_ratelimited(isp->dev, + "Failed to translate IPC iova 0x%llx (0x%zx): Out of bounds\n", + (long long)iova, size); return NULL; } if (!surf->virt) { - dev_err(isp->dev, - "Failed to translate IPC iova 0x%llx (0x%zx): No VMap\n", - (long long)iova, size); + dev_err_ratelimited(isp->dev, + "Failed to translate IPC iova 0x%llx (0x%zx): No VMap\n", + (long long)iova, size); return NULL; } diff --git a/drivers/media/platform/apple/isp/isp-iommu.c b/drivers/media/platform/apple/isp/isp-iommu.c index 68abaa4b33c83..5649ba4139097 100644 --- a/drivers/media/platform/apple/isp/isp-iommu.c +++ b/drivers/media/platform/apple/isp/isp-iommu.c @@ -147,7 +147,8 @@ struct isp_surf *__isp_alloc_surface(struct apple_isp *isp, u64 size, bool gc) * bound also keeps the page count within 32 bits. */ if (!size || size > isp->iova_size) { - dev_err(isp->dev, "invalid surface size 0x%llx\n", size); + dev_err_ratelimited(isp->dev, "invalid surface size 0x%llx\n", + size); return NULL; } diff --git a/drivers/media/platform/apple/isp/isp-ipc.c b/drivers/media/platform/apple/isp/isp-ipc.c index e43e7ad5ba91c..56d5c1d7f2376 100644 --- a/drivers/media/platform/apple/isp/isp-ipc.c +++ b/drivers/media/platform/apple/isp/isp-ipc.c @@ -114,7 +114,8 @@ static int chan_handle_once(struct apple_isp *isp, struct isp_channel *chan) err = chan->ops->handle(isp, chan); if (err < 0) { - dev_err(isp->dev, "%s: handler failed: %d)\n", chan->name, err); + dev_err_ratelimited(isp->dev, "%s: handler failed: %d\n", + chan->name, err); return err; } @@ -227,8 +228,9 @@ int ipc_sm_handle(struct apple_isp *isp, struct isp_channel *chan) surf = isp_alloc_surface_gc(isp, req->arg1); if (!surf) { - isp_err(isp, "failed to alloc requested size 0x%llx\n", - req->arg1); + dev_err_ratelimited(isp->dev, + "failed to alloc requested size 0x%llx\n", + req->arg1); return -ENOMEM; } surf->type = req->arg2; diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 5991cc7b1e3d3..8ac21c7f612f0 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -58,8 +58,8 @@ int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan) if (!isp->bt_surf || req->arg1 < sizeof(*bl) || req->arg1 > isp->bt_surf->size) { - dev_err(isp->dev, "%s: Bad length 0x%llx\n", chan->name, - req->arg1); + dev_err_ratelimited(isp->dev, "%s: Bad length 0x%llx\n", + chan->name, req->arg1); return -EIO; } @@ -70,8 +70,8 @@ int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan) count = bl->num_buffers; if (count > (req->arg1 - sizeof(*bl)) / sizeof(struct isp_buflist_buffer)) { - dev_err(isp->dev, "%s: Bad length 0x%llx\n", chan->name, - req->arg1); + dev_err_ratelimited(isp->dev, "%s: Bad length 0x%llx\n", + chan->name, req->arg1); return -EIO; } From 711f6fd2b6722697cce1d36534551475bf2d8b28 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:25:31 +1000 Subject: [PATCH 0553/2680] media: apple: isp: Warn once when the firmware version is not provided The bootloader describes the ISP firmware with the apple,firmware-version and apple,firmware-compat properties. When it provides neither, probe prints "could not parse" with the -EINVAL of the property lookup for each of them, followed by an info line with two unknown versions. Stay quiet about an absent property in isp_read_fw_version(), which still warns about values it cannot parse or does not know, and warn once from isp_check_firmware_version() when the compatibility version, the one the driver uses to pick its command layouts, is missing. The firmware version itself is only informative and is not reported. Demote the summary line to debug level. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 8c083424fd3f1..a950361d22349 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -244,6 +244,9 @@ static enum isp_firmware_version isp_read_fw_version(struct device *dev, ISP_FW_VERSION_MAX_LEN); switch (len) { + case -EINVAL: + /* not provided, see isp_check_firmware_version() */ + break; case 3: if (ver[0] == 12 && ver[1] == 3 && ver[2] <= 1) return ISP_FIRMWARE_V_12_3; @@ -280,8 +283,11 @@ static enum isp_firmware_version isp_check_firmware_version(struct device *dev) version = isp_read_fw_version(dev, "apple,firmware-version"); compat = isp_read_fw_version(dev, "apple,firmware-compat"); - dev_info(dev, "ISP firmware-compat: %s (FW: %s)\n", isp_fw2str(compat), - isp_fw2str(version)); + dev_dbg(dev, "ISP firmware-compat: %s (FW: %s)\n", isp_fw2str(compat), + isp_fw2str(version)); + + if (!of_property_present(dev->of_node, "apple,firmware-compat")) + dev_warn(dev, "firmware compatibility version not provided, assuming 12.x\n"); return compat; } From ddca973ad7ad2ff76276605b15cbe05c08b4d260 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:26:13 +1000 Subject: [PATCH 0554/2680] media: apple: isp: Check the sensor presets against the firmware The sensor presets come from the device tree, and the driver passes their apple,config-index to CH_CAMERA_CONFIG_SELECT unchecked. CH_INFO_GET reports how many presets the firmware has for the sensor, numbered from zero, so an index at or beyond that count names a preset the firmware does not have, and selecting it can only fail when a stream starts. Ignore such presets at probe, with a warning, and fail the probe if none is left. A firmware that reports no presets at all is not checked, as before. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-cam.c | 39 ++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index 784ae3e1bd9ab..af68d2c7cfd15 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -167,6 +167,43 @@ static int isp_ch_get_sensor_id(struct apple_isp *isp, u32 ch) return err; } +/* + * The presets come from the device tree. The firmware numbers its own + * presets from zero, so a config index at or beyond the count it reports + * names a preset it does not have, and selecting it can only fail. + */ +static int isp_check_presets(struct apple_isp *isp, u32 num_fw_presets) +{ + int num = 0; + + /* Nothing to check against */ + if (!num_fw_presets) + return 0; + + for (int i = 0; i < isp->num_presets; i++) { + struct isp_preset *preset = &isp->presets[i]; + + if (preset->index >= num_fw_presets) { + dev_warn(isp->dev, + "ignoring %ux%u preset: config %u, firmware has %u\n", + preset->output_dim.x, preset->output_dim.y, + preset->index, num_fw_presets); + continue; + } + + isp->presets[num++] = *preset; + } + + if (!num) { + dev_err(isp->dev, "no usable sensor presets\n"); + return -ENODEV; + } + + isp->num_presets = num; + + return 0; +} + static int isp_ch_cache_sensor_info(struct apple_isp *isp, u32 ch) { struct isp_format *fmt = isp_get_format(isp, ch); @@ -196,6 +233,8 @@ static int isp_ch_cache_sensor_info(struct apple_isp *isp, u32 ch) goto exit; } + err = isp_check_presets(isp, args->num_presets); + exit: kfree(args); From a1ac78797ec36181951e0047ab99a78b4ba68744 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:26:55 +1000 Subject: [PATCH 0555/2680] media: apple: isp: Stop the coprocessor before freeing its memory isp_firmware_shutdown() asks the firmware to suspend, then frees the channel state and the extra heap, and only then stops the coprocessor and powers its domains down in stage 1. It ignores the result of the suspend request, so a firmware that did not acknowledge it keeps running while memory it uses is released. The pages are unmapped from the DART before they are freed, so late accesses fault instead of reaching freed memory, but they need not happen at all. Stop the coprocessor and power its domains down right after releasing the interrupt, before any memory is freed, and warn when this happens without the firmware having acknowledged the suspend request. The DART stays powered through the ISP's own runtime PM reference until the surfaces are unmapped. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-fw.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index d68fbb14394c1..569cafe6c3a4e 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -770,11 +770,18 @@ static int isp_firmware_boot(struct apple_isp *isp) static void isp_firmware_shutdown(struct apple_isp *isp) { flush_workqueue(isp->wq); - isp_stop_command_processor(isp); + + /* + * Stop the coprocessor before releasing any memory the firmware + * uses, also when it did not acknowledge the suspend request. + */ + if (isp_stop_command_processor(isp)) + dev_warn(isp->dev, "firmware did not suspend, stopping it\n"); isp_disable_irq(isp); + isp_firmware_shutdown_stage1(isp); + isp_firmware_shutdown_stage3(isp); isp_firmware_shutdown_stage2(isp); - isp_firmware_shutdown_stage1(isp); isp_collect_gc_surface(isp); } From 19d8e1847853bdcc7a819ab3d7f98ba311cdc241 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:28:12 +1000 Subject: [PATCH 0556/2680] media: apple: isp: Strip untranslated bits from firmware addresses On T8140 the ISP DARTs translate 36-bit addresses, and the firmware reports the addresses it hands to the host with its vm-base, bit 40, set: the channel table and ring addresses after boot, the addresses in command acknowledgements, log messages and buffer return lists. The DART ignores the upper bits, and the host allocates its surfaces at the low addresses, so these values have to be masked before the driver compares them or translates them to its own mappings. Add a per-SoC mask of the address bits the DART translates, apply it wherever the driver takes an address from the firmware, and leave the addresses unmasked on SoCs that do not set it, which are all of them so far. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.c | 1 + drivers/media/platform/apple/isp/isp-drv.h | 10 ++++++++++ drivers/media/platform/apple/isp/isp-fw.c | 4 ++-- drivers/media/platform/apple/isp/isp-ipc.c | 6 ++++-- drivers/media/platform/apple/isp/isp-v4l2.c | 3 ++- 5 files changed, 19 insertions(+), 5 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index a950361d22349..5200e7348c6fd 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -95,6 +95,7 @@ static int apple_isp_init_iommu(struct apple_isp *isp) if (!isp->domain) return -ENODEV; isp->shift = __ffs(isp->domain->pgsize_bitmap); + isp->fw_iova_mask = isp->hw->fw_iova_mask ?: U64_MAX; idx = of_property_match_string(dev->of_node, "memory-region-names", "heap"); diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 717f67da997db..84e0ef42188df 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -135,6 +135,9 @@ struct apple_isp_hw { u32 meta_size; bool scl1; bool lpdp; + + /* address bits the ISP DARTs translate, 0 for all */ + u64 fw_iova_mask; }; enum isp_sensor_id { @@ -235,6 +238,7 @@ struct apple_isp { unsigned long shift; struct drm_mm iovad; /* TODO iova.c can't allocate bottom-up */ u64 iova_size; /* size of the iovad range */ + u64 fw_iova_mask; /* see isp_fw_iova() */ struct mutex iovad_lock; struct isp_firmware { @@ -294,6 +298,12 @@ enum { #define isp_err(isp, fmt, ...) \ dev_err((isp)->dev, "[%s] " fmt, __func__, ##__VA_ARGS__) +/* + * Addresses from the firmware may carry bits above the ones the DART + * translates (the vm-base on T8140); strip them before comparing or + * translating. + */ +#define isp_fw_iova(isp, x) ((x) & (isp)->fw_iova_mask) #define isp_get_format(isp, ch) (&(isp)->fmts[(ch)]) #define isp_get_current_format(isp) (isp_get_format(isp, isp->current_ch)) diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 569cafe6c3a4e..3c537a80b7ed4 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -489,7 +489,7 @@ static int isp_fill_channel_info(struct apple_isp *isp) u64 table_iova = isp_gpio_read32(isp, ISP_GPIO_0) | ((u64)isp_gpio_read32(isp, ISP_GPIO_1)) << 32; void *table_virt = apple_isp_ipc_translate( - isp, table_iova, + isp, isp_fw_iova(isp, table_iova), array_size(sizeof(struct isp_chan_desc), isp->num_ipc_chans)); int err = -EIO; @@ -525,7 +525,7 @@ static int isp_fill_channel_info(struct apple_isp *isp) chan->src = desc.src; chan->num = desc.num; chan->size = (u64)desc.num * ISP_IPC_MESSAGE_SIZE; - chan->iova = desc.iova; + chan->iova = isp_fw_iova(isp, desc.iova); chan->cursor = 0; mutex_init(&chan->lock); diff --git a/drivers/media/platform/apple/isp/isp-ipc.c b/drivers/media/platform/apple/isp/isp-ipc.c index 56d5c1d7f2376..82e1907dbf68c 100644 --- a/drivers/media/platform/apple/isp/isp-ipc.c +++ b/drivers/media/platform/apple/isp/isp-ipc.c @@ -163,7 +163,8 @@ static inline bool chan_tx_done(struct apple_isp *isp, struct isp_channel *chan) dma_rmb(); chan_read_msg(isp, chan, &chan->rsp); - if ((chan->rsp.arg0) == (chan->req.arg0 | ISP_IPC_FLAG_ACK)) { + if (isp_fw_iova(isp, chan->rsp.arg0) == + isp_fw_iova(isp, chan->req.arg0 | ISP_IPC_FLAG_ACK)) { chan_update_cursor(chan); return true; } @@ -200,7 +201,8 @@ int ipc_chan_send(struct apple_isp *isp, struct isp_channel *chan, int ipc_tm_handle(struct apple_isp *isp, struct isp_channel *chan) { struct isp_message *req = &chan->req, *rsp = &chan->rsp; - dma_addr_t iova = req->arg0 & ~ISP_IPC_FLAG_TERMINAL_ACK; + dma_addr_t iova = + isp_fw_iova(isp, req->arg0 & ~ISP_IPC_FLAG_TERMINAL_ACK); size_t size = min_t(u64, req->arg1, ISP_IPC_TERMINAL_MAX_LEN); const char *line; diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 8ac21c7f612f0..9f3c2674a64d5 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -63,7 +63,8 @@ int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan) return -EIO; } - bl = apple_isp_translate(isp, isp->bt_surf, req->arg0, req->arg1); + bl = apple_isp_translate(isp, isp->bt_surf, isp_fw_iova(isp, req->arg0), + req->arg1); if (!bl) return -EIO; From 06671c9a41620540cccb2d418c3832138a6893e4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:29:18 +1000 Subject: [PATCH 0557/2680] media: apple: isp: Support the ISP17a mailbox and control registers On the ISP17a in T8140 the doorbell and acknowledge registers are not a window of their own: they sit at offset 0x410 of the interrupt block that the binding describes as "mbox", which is why the T8140 binding has no "mbox2" entry. The interrupt block also has two more enable words, at 0x18 and 0x38, and an interrupt routing word at 0x73c, which are set together with the enable word at 0x8. The coprocessor control register moved from 0x1400044 to 0x1600044. Add match data for these differences: the offset of the doorbell block inside the "mbox" window, checked against the window's size; a flag for the additional interrupt words, which are set to the values observed while the firmware runs on this SoC and cleared with the enable word; and the offset of the control register. SoCs that set none of them behave as before. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.c | 23 ++++++++++++++++----- drivers/media/platform/apple/isp/isp-drv.h | 7 +++++++ drivers/media/platform/apple/isp/isp-fw.c | 23 ++++++++++++++++++--- drivers/media/platform/apple/isp/isp-regs.h | 15 ++++++++++++++ 4 files changed, 60 insertions(+), 8 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 5200e7348c6fd..e3aff088f160e 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -297,6 +297,7 @@ static int apple_isp_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct apple_isp *isp; + struct resource *res; int err; err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); @@ -348,7 +349,8 @@ static int apple_isp_probe(struct platform_device *pdev) goto detach_genpd; } - isp->mbox = devm_platform_ioremap_resource_byname(pdev, "mbox"); + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mbox"); + isp->mbox = devm_ioremap_resource(dev, res); if (IS_ERR(isp->mbox)) { err = PTR_ERR(isp->mbox); goto detach_genpd; @@ -360,10 +362,21 @@ static int apple_isp_probe(struct platform_device *pdev) goto detach_genpd; } - isp->mbox2 = devm_platform_ioremap_resource_byname(pdev, "mbox2"); - if (IS_ERR(isp->mbox2)) { - err = PTR_ERR(isp->mbox2); - goto detach_genpd; + if (isp->hw->mbox2_offset) { + if (resource_size(res) < + isp->hw->mbox2_offset + ISP_MBOX2_SIZE) { + dev_err(dev, "mbox window too small for the doorbells\n"); + err = -EINVAL; + goto detach_genpd; + } + isp->mbox2 = isp->mbox + isp->hw->mbox2_offset; + } else { + isp->mbox2 = devm_platform_ioremap_resource_byname(pdev, + "mbox2"); + if (IS_ERR(isp->mbox2)) { + err = PTR_ERR(isp->mbox2); + goto detach_genpd; + } } isp->irq = platform_get_irq(pdev, 0); diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 84e0ef42188df..c496dff6ba8af 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -138,6 +138,13 @@ struct apple_isp_hw { /* address bits the ISP DARTs translate, 0 for all */ u64 fw_iova_mask; + + /* doorbell block inside "mbox" instead of an "mbox2" window */ + u32 mbox2_offset; + /* further interrupt enable and routing words (ISP17a) */ + bool mbox_irq_route; + /* coprocessor control register, 0 for ISP_COPROC_CONTROL */ + u32 coproc_control; }; enum isp_sensor_id { diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 3c537a80b7ed4..05ebdd2e836fe 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -45,6 +45,11 @@ static inline void isp_gpio_write32(struct apple_isp *isp, u32 reg, u32 val) writel(val, isp->gpio + reg); } +static inline u32 isp_coproc_control(struct apple_isp *isp) +{ + return isp->hw->coproc_control ?: ISP_COPROC_CONTROL; +} + /* Wait for the firmware to write @expected to a GPIO word. */ static int isp_gpio_wait(struct apple_isp *isp, u32 reg, u32 expected) { @@ -202,6 +207,10 @@ static irqreturn_t apple_isp_isr_thread(int irq, void *dev) static void isp_disable_irq(struct apple_isp *isp) { isp_mbox_write32(isp, isp->hw->mbox_irq_enable, 0x0); + if (isp->hw->mbox_irq_route) { + isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE1_T8140, 0x0); + isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE2_T8140, 0x0); + } free_irq(isp->irq, isp); isp_gpio_write32(isp, ISP_GPIO_1, 0xfeedbabe); /* real funny */ } @@ -220,6 +229,14 @@ static int isp_enable_irq(struct apple_isp *isp) isp_dbg(isp, "about to enable interrupts...\n"); isp_mbox_write32(isp, isp->hw->mbox_irq_enable, 0xf); + if (isp->hw->mbox_irq_route) { + isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE1_T8140, + ISP_MBOX_IRQ_ENABLE1_T8140_VAL); + isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE2_T8140, + ISP_MBOX_IRQ_ENABLE2_T8140_VAL); + isp_mbox_write32(isp, ISP_MBOX_IRQ_ROUTE_T8140, + ISP_MBOX_IRQ_ROUTE_T8140_VAL); + } return 0; } @@ -269,7 +286,7 @@ static int isp_reset_coproc(struct apple_isp *isp) static void isp_firmware_shutdown_stage1(struct apple_isp *isp) { - isp_coproc_write32(isp, ISP_COPROC_CONTROL, 0x0); + isp_coproc_write32(isp, isp_coproc_control(isp), 0x0); apple_isp_power_down_domains(isp); } @@ -311,8 +328,8 @@ static int isp_firmware_boot_stage1(struct apple_isp *isp) isp_mbox_write32(isp, isp->hw->mbox_irq_enable, 0x0); - isp_coproc_write32(isp, ISP_COPROC_CONTROL, 0x0); - isp_coproc_write32(isp, ISP_COPROC_CONTROL, 0x10); + isp_coproc_write32(isp, isp_coproc_control(isp), 0x0); + isp_coproc_write32(isp, isp_coproc_control(isp), 0x10); /* Wait for ISP_GPIO_7 to 0x0 -> 0x8042006 */ if (isp_gpio_wait(isp, ISP_GPIO_7, 0x8042006)) { diff --git a/drivers/media/platform/apple/isp/isp-regs.h b/drivers/media/platform/apple/isp/isp-regs.h index 2cf902028a6a4..dddf24e803e5f 100644 --- a/drivers/media/platform/apple/isp/isp-regs.h +++ b/drivers/media/platform/apple/isp/isp-regs.h @@ -23,11 +23,26 @@ #define ISP_COPROC_IRQ_MASK_4 0x1400a10 #define ISP_COPROC_IRQ_MASK_5 0x1400a14 +#define ISP_COPROC_CONTROL_T8140 0x1600044 + #define ISP_MBOX_IRQ_INTERRUPT 0x00 #define ISP_MBOX_IRQ_ENABLE 0x04 #define ISP_MBOX_IRQ_ENABLE_T6031 0x08 #define ISP_MBOX2_IRQ_DOORBELL 0x00 #define ISP_MBOX2_IRQ_ACK 0x0c +#define ISP_MBOX2_SIZE 0x10 + +/* + * ISP17a (T8140) interrupt enable and routing words next to + * ISP_MBOX_IRQ_ENABLE_T6031, and the values observed while the firmware + * runs. + */ +#define ISP_MBOX_IRQ_ENABLE1_T8140 0x18 +#define ISP_MBOX_IRQ_ENABLE2_T8140 0x38 +#define ISP_MBOX_IRQ_ROUTE_T8140 0x73c +#define ISP_MBOX_IRQ_ENABLE1_T8140_VAL 0x70000000 +#define ISP_MBOX_IRQ_ENABLE2_T8140_VAL 0x700000 +#define ISP_MBOX_IRQ_ROUTE_T8140_VAL 0x8 #define ISP_GPIO_0 0x00 #define ISP_GPIO_1 0x04 From 18e925049d841dad9c9198cbd98dd387e1b36f25 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:30:22 +1000 Subject: [PATCH 0558/2680] media: apple: isp: Map firmware MMIO windows at their physical addresses The T8140 firmware accesses registers of other blocks through its DART at their physical addresses: the DSID broadcast-clear windows of the memory fabric, the PMGR scratch registers the driver names in PMP_CTRL_SET, some ISP power-state registers in the PMGR, and a few more. They are the ranges the bootloader programs into the DART's address filter for the ISP, rounded out to the 16 KiB DART page size. Add a per-SoC table of such windows to the match data and map them 1:1 into the ISP's IOMMU domain along with the rest of its address space, unmapping them on teardown. They are mapped cacheable: with the DART's no-cache attribute the ISP took an SError on its first write to the PMP scratch register when a stream started. A window that overlaps the range the driver allocates its surfaces from is refused. The windows are SoC constants, like the PMU, DSID and PMP scratch addresses the driver already keeps in its match data and passes to the firmware, so they are not described in the device tree; the T8140 ISP binding covers only the ISP's own registers. Mapping whole pages exposes more than the firmware needs: each window gives the ISP DMA access to every register in its pages, and a window can be much wider than the registers the firmware is known to use. The DART cannot restrict it further. The mappings are also not reserved in the DMA API's IOVA allocator for the device; the driver does not use that allocator for them, and on T8140 it hands out addresses downwards from the 32 GiB end of the ISP's DART aperture, far above the windows. SoCs without a table are not affected. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.c | 57 ++++++++++++++++++++++ drivers/media/platform/apple/isp/isp-drv.h | 10 ++++ 2 files changed, 67 insertions(+) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index e3aff088f160e..01177ab87b9c2 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -79,6 +79,56 @@ static int apple_isp_attach_genpd(struct apple_isp *isp) return 0; } +static void apple_isp_unmap_fw_mmio(struct apple_isp *isp, unsigned int count) +{ + for (unsigned int i = 0; i < count; i++) + iommu_unmap(isp->domain, isp->hw->fw_mmio[i].base, + isp->hw->fw_mmio[i].size); +} + +/* + * Some firmware accesses registers of other blocks, such as the PMGR + * scratch registers named in PMP_CTRL_SET, through its DART at their + * physical addresses. Map those windows 1:1. + */ +static int apple_isp_map_fw_mmio(struct apple_isp *isp) +{ + u64 start = isp->fw.heap_top, end = start + isp->iova_size; + + for (unsigned int i = 0; i < isp->hw->num_fw_mmio; i++) { + const struct isp_mmio_window *w = &isp->hw->fw_mmio[i]; + int err; + + /* Keep them out of the range the surfaces come from. */ + if (w->base < end && w->base + w->size > start) { + dev_err(isp->dev, + "MMIO window 0x%llx+0x%llx overlaps the IOVA range\n", + w->base, w->size); + err = -EINVAL; + } else { + /* + * Cacheable: with the no-cache attribute, the ISP took + * an SError on its first write to the PMP scratch + * register when streaming started. + */ + err = iommu_map(isp->domain, w->base, w->base, w->size, + IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE, + GFP_KERNEL); + if (err) + dev_err(isp->dev, + "failed to map MMIO window 0x%llx+0x%llx: %d\n", + w->base, w->size, err); + } + + if (err) { + apple_isp_unmap_fw_mmio(isp, i); + return err; + } + } + + return 0; +} + static int apple_isp_init_iommu(struct apple_isp *isp) { struct device *dev = isp->dev; @@ -132,11 +182,18 @@ static int apple_isp_init_iommu(struct apple_isp *isp) isp->iova_size = vm_size - (heap_base & 0xffffffff); drm_mm_init(&isp->iovad, isp->fw.heap_top, isp->iova_size); + err = apple_isp_map_fw_mmio(isp); + if (err) { + drm_mm_takedown(&isp->iovad); + return err; + } + return 0; } static void apple_isp_free_iommu(struct apple_isp *isp) { + apple_isp_unmap_fw_mmio(isp, isp->hw->num_fw_mmio); drm_mm_takedown(&isp->iovad); } diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index c496dff6ba8af..3113137811f7a 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -100,6 +100,12 @@ struct coord { u32 y; }; +/* MMIO window the firmware accesses at its physical address */ +struct isp_mmio_window { + u64 base; + u64 size; +}; + struct isp_preset { u32 index; struct coord input_dim; @@ -145,6 +151,10 @@ struct apple_isp_hw { bool mbox_irq_route; /* coprocessor control register, 0 for ISP_COPROC_CONTROL */ u32 coproc_control; + + /* windows mapped 1:1 for the firmware */ + const struct isp_mmio_window *fw_mmio; + unsigned int num_fw_mmio; }; enum isp_sensor_id { From 7b724b09c8723edce7e5085c928fc3da58be002c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:32:30 +1000 Subject: [PATCH 0559/2680] media: apple: isp: Add the ISP17a reset and the H16 boot descriptor T8140 has an ISP17a coprocessor and runs the H17 generation of the ISP firmware. Both boot differently from the existing SoCs: - The reset request in EDPRCR is followed by all-ones writes to four fabric words at 0x748, 0x848, 0x948 and 0xa50, the last of which breaks the stride of the others in the observed sequence, and to four interrupt mask words from 0x1600a00 to 0x1600a0c. Then come eight alternating reads of the reset acknowledge words at 0x1600818 and 0x160081c, with a 50 ms wait after the reset request clears and another after the reads. This is the sequence observed when the firmware boots on this SoC. Only the reset request bit is known to signal completion, so it is the only one polled; the acknowledge words change between reads, but no condition on them is known, and the coprocessor status is not polled for WFI. Which of the steps are required has not been established. - There is no clock enable word at GPIO offset 0x20. - GPIO 6 is a boot mode field, and the boot the driver performs uses mode 1. - At the first handshake the firmware sets bit 1 of GPIO 2 to request a 0x290-byte boot descriptor instead of the 0x180-byte boot arguments. The IPC queue size at 0x50 is 64 bits wide and holds the boot argument offset plus one, the word at 0x68 is 0x40, the shared span is 0xee000000 bytes from the heap top, and of the 0x200-byte connection descriptor at 0x70 only the flag at its offset 0xb0, which says that there is no optical device card, is set. These are the values the firmware boots with; how they derive from the rest of the setup is not known. Select the reset sequence by the ISP generation and the boot descriptor by a firmware interface field in the match data, which also takes the boot mode. When the H17 interface is selected, refuse to boot firmware that does not request the H16 descriptor rather than give it a layout it did not ask for, and do not warn about a missing apple,firmware-compat property, which that interface does not use. The existing boot arguments move to a helper, unchanged. Existing SoCs are not affected. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.c | 10 +- drivers/media/platform/apple/isp/isp-drv.h | 15 ++ drivers/media/platform/apple/isp/isp-fw.c | 155 +++++++++++++++++--- drivers/media/platform/apple/isp/isp-regs.h | 15 ++ 4 files changed, 172 insertions(+), 23 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 01177ab87b9c2..24e4c4786e286 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -333,8 +333,10 @@ static enum isp_firmware_version isp_read_fw_version(struct device *dev, return ISP_FIRMWARE_V_UNKNOWN; } -static enum isp_firmware_version isp_check_firmware_version(struct device *dev) +static enum isp_firmware_version +isp_check_firmware_version(struct apple_isp *isp) { + struct device *dev = isp->dev; enum isp_firmware_version version, compat; /* firmware version is just informative */ @@ -344,7 +346,9 @@ static enum isp_firmware_version isp_check_firmware_version(struct device *dev) dev_dbg(dev, "ISP firmware-compat: %s (FW: %s)\n", isp_fw2str(compat), isp_fw2str(version)); - if (!of_property_present(dev->of_node, "apple,firmware-compat")) + /* The H17 interface does not depend on it. */ + if (isp->hw->fw_abi == ISP_FW_ABI_LEGACY && + !of_property_present(dev->of_node, "apple,firmware-compat")) dev_warn(dev, "firmware compatibility version not provided, assuming 12.x\n"); return compat; @@ -373,7 +377,7 @@ static int apple_isp_probe(struct platform_device *pdev) /* Differences between firmware versions are rather minor so try to work * with unknown firmware. */ - isp->fw_compat = isp_check_firmware_version(dev); + isp->fw_compat = isp_check_firmware_version(isp); err = of_property_read_u32(dev->of_node, "apple,platform-id", &isp->platform_id); diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 3113137811f7a..f6797b90e98bb 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -42,6 +42,17 @@ enum isp_generation { ISP_GEN_T8103, ISP_GEN_T8112, ISP_GEN_T6031, + ISP_GEN_T8140, +}; + +/* + * Firmware interface generation. The existing SoCs pick their command + * layouts by apple,firmware-compat; T8140 only runs the H17 firmware, so + * its match data selects that interface. + */ +enum isp_fw_abi { + ISP_FW_ABI_LEGACY, + ISP_FW_ABI_H17, }; enum isp_firmware_version { @@ -116,6 +127,7 @@ struct isp_preset { struct apple_isp_hw { enum isp_generation gen; + enum isp_fw_abi fw_abi; u64 pmu_base; int dsid_count; @@ -155,6 +167,9 @@ struct apple_isp_hw { /* windows mapped 1:1 for the firmware */ const struct isp_mmio_window *fw_mmio; unsigned int num_fw_mmio; + + /* ISP_GPIO_6 boot mode */ + u32 boot_mode; }; enum isp_sensor_id { diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 05ebdd2e836fe..ee866ccd7a78c 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -154,6 +154,39 @@ struct isp_firmware_bootargs { } __packed; static_assert(sizeof(struct isp_firmware_bootargs) == 0x180); +/* + * Boot descriptor of the H17 firmware. The values the driver fills in are + * the ones this firmware boots with; how they derive from the rest of the + * setup is not known. + */ +#define ISP_H16_DESCRIPTOR_SIZE 0x200 +#define ISP_H16_SHARED_SIZE 0xee000000ULL +#define ISP_H16_DESC_NO_OPTICAL_CARD_ID 0xb0 + +struct isp_firmware_bootargs_h16 { + u32 pad_0[2]; + u64 ipc_iova; + u64 shared_base; + u64 shared_size; + u64 extra_iova; + u64 extra_size; + u32 platform_id; + u32 pad_34; + u64 logbuf_addr; + u64 logbuf_size; + u64 logbuf_entsize; + u64 ipc_queue_size; + u32 pad_58[4]; + u32 unk_68; + u32 pad_6c; + u8 descriptor[ISP_H16_DESCRIPTOR_SIZE]; + u32 pad_270[8]; +} __packed; +static_assert(sizeof(struct isp_firmware_bootargs_h16) == 0x290); + +/* set in ISP_GPIO_2 by firmware that takes the H16 boot descriptor */ +#define ISP_GPIO2_H16_DESCRIPTOR BIT(1) + struct isp_chan_desc { char name[64]; u32 type; @@ -241,12 +274,54 @@ static int isp_enable_irq(struct apple_isp *isp) return 0; } +/* + * The ISP17a reset as observed when the firmware boots on T8140. Only the + * reset request bit is known to signal completion. The acknowledge words + * change between the eight reads, but no condition on them is known, and + * the coprocessor status is not polled for WFI on this generation. Which + * of the writes, reads and waits are required has not been established. + */ +static int isp_reset_coproc_t8140(struct apple_isp *isp) +{ + u32 val; + + isp_coproc_write32(isp, ISP_COPROC_EDPRCR, 0x2); + if (readl_poll_timeout(isp->coproc + ISP_COPROC_EDPRCR, val, + !(val & 0x2), ISP_FIRMWARE_POLL_US, + 100 * USEC_PER_MSEC)) { + isp_err(isp, "coprocessor reset request did not clear\n"); + return -ETIMEDOUT; + } + msleep(50); + + isp_coproc_write32(isp, ISP_COPROC_FABRIC_0_T8140, 0xffffffff); + isp_coproc_write32(isp, ISP_COPROC_FABRIC_1_T8140, 0xffffffff); + isp_coproc_write32(isp, ISP_COPROC_FABRIC_2_T8140, 0xffffffff); + isp_coproc_write32(isp, ISP_COPROC_FABRIC_3_T8140, 0xffffffff); + + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_0_T8140, 0xffffffff); + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_1_T8140, 0xffffffff); + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_2_T8140, 0xffffffff); + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_3_T8140, 0xffffffff); + + for (int i = 0; i < 8; i++) { + isp_coproc_read32(isp, ISP_COPROC_RESET_ACK_0_T8140); + isp_coproc_read32(isp, ISP_COPROC_RESET_ACK_1_T8140); + } + msleep(50); + + return 0; +} + static int isp_reset_coproc(struct apple_isp *isp) { int retries; u32 status; u32 val; + if (isp->hw->gen == ISP_GEN_T8140) + return isp_reset_coproc_t8140(isp); + isp_coproc_write32(isp, ISP_COPROC_EDPRCR, 0x2); isp_coproc_write32(isp, ISP_COPROC_FABRIC_0, 0xff00ff); @@ -300,8 +375,9 @@ static int isp_firmware_boot_stage1(struct apple_isp *isp) if (err < 0) return err; - - isp_gpio_write32(isp, ISP_GPIO_CLOCK_EN, 0x1); + /* ISP17a has no clock enable word */ + if (isp->hw->gen != ISP_GEN_T8140) + isp_gpio_write32(isp, ISP_GPIO_CLOCK_EN, 0x1); #if 0 /* This doesn't work well with system sleep */ @@ -323,7 +399,7 @@ static int isp_firmware_boot_stage1(struct apple_isp *isp) isp_gpio_write32(isp, ISP_GPIO_3, 0x0); isp_gpio_write32(isp, ISP_GPIO_4, 0x0); isp_gpio_write32(isp, ISP_GPIO_5, 0x0); - isp_gpio_write32(isp, ISP_GPIO_6, 0x0); + isp_gpio_write32(isp, ISP_GPIO_6, isp->hw->boot_mode); isp_gpio_write32(isp, ISP_GPIO_7, 0x0); isp_mbox_write32(isp, isp->hw->mbox_irq_enable, 0x0); @@ -381,15 +457,56 @@ static void isp_firmware_shutdown_stage2(struct apple_isp *isp) isp_free_surface(isp, isp->extra_surf); } -static int isp_firmware_boot_stage2(struct apple_isp *isp) +static void isp_write_bootargs(struct apple_isp *isp, void *virt, + dma_addr_t iova) { struct isp_firmware_bootargs args; + + memset(&args, 0, sizeof(args)); + args.ipc_iova = isp->ipc_surf->iova; + args.ipc_size = isp->ipc_surf->size; + args.shared_base = isp->fw.heap_top & 0xffffffff; + args.shared_size = 0x10000000UL - args.shared_base; + args.extra_iova = isp->extra_surf->iova; + args.extra_size = isp->extra_surf->size; + args.platform_id = isp->platform_id; + args.unk5 = 0x40; + args.unk7 = 0x1; // 0? + args.unk_iova1 = iova + sizeof(args) - 0xc; + args.unk9 = 0x3; + memcpy(virt, &args, sizeof(args)); +} + +static void isp_write_bootargs_h16(struct apple_isp *isp, void *virt, + u32 args_offset) +{ + struct isp_firmware_bootargs_h16 args; + + memset(&args, 0, sizeof(args)); + args.ipc_iova = isp->ipc_surf->iova; + args.shared_base = isp->fw.heap_top & 0xffffffff; + args.shared_size = ISP_H16_SHARED_SIZE; + args.extra_iova = isp->extra_surf->iova; + args.extra_size = isp->extra_surf->size; + args.platform_id = isp->platform_id; + args.ipc_queue_size = (u64)args_offset + 1; + args.unk_68 = 0x40; + args.descriptor[ISP_H16_DESC_NO_OPTICAL_CARD_ID] = 1; + memcpy(virt, &args, sizeof(args)); +} + +static int isp_firmware_boot_stage2(struct apple_isp *isp) +{ + bool h16 = isp->hw->fw_abi == ISP_FW_ABI_H17; + size_t args_size = h16 ? sizeof(struct isp_firmware_bootargs_h16) : + sizeof(struct isp_firmware_bootargs); dma_addr_t args_iova, cmd_iova; void *args_virt, *cmd_virt; int err; u32 num_ipc_chans = isp_gpio_read32(isp, ISP_GPIO_0); u32 args_offset = isp_gpio_read32(isp, ISP_GPIO_1); + u32 desc_flags = isp_gpio_read32(isp, ISP_GPIO_2); u32 extra_size = isp_gpio_read32(isp, ISP_GPIO_3); isp->num_ipc_chans = num_ipc_chans; @@ -403,6 +520,14 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) dev_warn(isp->dev, "unexpected channel count (%d)\n", num_ipc_chans); + /* Only hand the H16 descriptor to firmware that asks for it. */ + if (h16 && !(desc_flags & ISP_GPIO2_H16_DESCRIPTOR)) { + dev_err(isp->dev, + "firmware did not request the H16 boot descriptor (0x%x)\n", + desc_flags); + return -ENODEV; + } + /* * The firmware picks the offset of the boot arguments in the IPC * surface, and the command area follows them. Both have to lie @@ -413,9 +538,8 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) cmd_virt = NULL; if (!check_add_overflow(isp->ipc_surf->iova, (u64)args_offset + 0x40, &args_iova) && - !check_add_overflow(args_iova, sizeof(args) + 0x40, &cmd_iova)) { - args_virt = apple_isp_ipc_translate(isp, args_iova, - sizeof(args)); + !check_add_overflow(args_iova, args_size + 0x40, &cmd_iova)) { + args_virt = apple_isp_ipc_translate(isp, args_iova, args_size); cmd_virt = apple_isp_ipc_translate(isp, cmd_iova, ISP_CMD_AREA_SIZE); } @@ -434,19 +558,10 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) isp->cmd_iova = cmd_iova; isp->cmd_virt = cmd_virt; - memset(&args, 0, sizeof(args)); - args.ipc_iova = isp->ipc_surf->iova; - args.ipc_size = isp->ipc_surf->size; - args.shared_base = isp->fw.heap_top & 0xffffffff; - args.shared_size = 0x10000000UL - args.shared_base; - args.extra_iova = isp->extra_surf->iova; - args.extra_size = isp->extra_surf->size; - args.platform_id = isp->platform_id; - args.unk5 = 0x40; - args.unk7 = 0x1; // 0? - args.unk_iova1 = args_iova + sizeof(args) - 0xc; - args.unk9 = 0x3; - memcpy(args_virt, &args, sizeof(args)); + if (h16) + isp_write_bootargs_h16(isp, args_virt, args_offset); + else + isp_write_bootargs(isp, args_virt, args_iova); isp_gpio_write32(isp, ISP_GPIO_0, args_iova); /* TODO: handle this via Kconfig depends? hardware is only present on diff --git a/drivers/media/platform/apple/isp/isp-regs.h b/drivers/media/platform/apple/isp/isp-regs.h index dddf24e803e5f..bf6878ef7b16d 100644 --- a/drivers/media/platform/apple/isp/isp-regs.h +++ b/drivers/media/platform/apple/isp/isp-regs.h @@ -25,6 +25,21 @@ #define ISP_COPROC_CONTROL_T8140 0x1600044 +/* + * ISP17a (T8140) reset. The fourth fabric word is at 0xa50, off the 0x100 + * stride of the other three, in the observed sequence. + */ +#define ISP_COPROC_FABRIC_0_T8140 0x748 +#define ISP_COPROC_FABRIC_1_T8140 0x848 +#define ISP_COPROC_FABRIC_2_T8140 0x948 +#define ISP_COPROC_FABRIC_3_T8140 0xa50 +#define ISP_COPROC_IRQ_MASK_0_T8140 0x1600a00 +#define ISP_COPROC_IRQ_MASK_1_T8140 0x1600a04 +#define ISP_COPROC_IRQ_MASK_2_T8140 0x1600a08 +#define ISP_COPROC_IRQ_MASK_3_T8140 0x1600a0c +#define ISP_COPROC_RESET_ACK_0_T8140 0x1600818 +#define ISP_COPROC_RESET_ACK_1_T8140 0x160081c + #define ISP_MBOX_IRQ_INTERRUPT 0x00 #define ISP_MBOX_IRQ_ENABLE 0x04 #define ISP_MBOX_IRQ_ENABLE_T6031 0x08 From 8a0af4bf02f6288946ac3d68417fc4e2228f9ec5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:34:04 +1000 Subject: [PATCH 0560/2680] media: apple: isp: Add the H17 start-up and channel info commands The H17 firmware on T8140 differs from the older firmware in three commands the driver sends while booting it and detecting the sensor: - After PRINT_ENABLE it takes no SET_ISP_PMU_BASE, and it takes its single DSID broadcast-clear window through SET_DSID_CLR_MULTI_BC_REG_BASE (0x3206), a count followed by the window, instead of SET_DSID_CLR_REG_BASE. - CH_INFO_GET exchanges 0x190 bytes. The first 0x118 match the older reply; in them the word at 0x68 is the size of the per-frame metadata record. - CH_SET_FILE_LOAD uses the 64-bit layout and returns the 0x18-byte record. Send these forms when the match data selects the H17 interface. Refuse a firmware whose metadata size differs from the one the driver allocates its metadata buffers for, since it would write past them; the older firmware reports a different value in that word, so it is only checked for H17. What the SET_FILE_LOAD reply contains is not known, and it is not checked. The PMU base and DSID setup move to a helper, unchanged for the older firmware. Existing SoCs are not affected. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-cam.c | 9 ++++ drivers/media/platform/apple/isp/isp-cmd.c | 41 +++++++++++++++++- drivers/media/platform/apple/isp/isp-cmd.h | 24 +++++++++++ drivers/media/platform/apple/isp/isp-fw.c | 49 +++++++++++++++------- 4 files changed, 105 insertions(+), 18 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index af68d2c7cfd15..131ded5d4a795 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -220,6 +220,15 @@ static int isp_ch_cache_sensor_info(struct apple_isp *isp, u32 ch) dev_dbg(isp->dev, "found sensor %x on ch %d\n", args->version, ch); + /* The metadata buffers are allocated for the size the driver knows. */ + if (isp->hw->fw_abi == ISP_FW_ABI_H17 && + args->unk_68 != isp->hw->meta_size) { + dev_err(isp->dev, "metadata size 0x%x, expected 0x%x\n", + args->unk_68, isp->hw->meta_size); + err = -ENODEV; + goto exit; + } + fmt->version = args->version; err = isp_ch_get_sensor_id(isp, ch); diff --git a/drivers/media/platform/apple/isp/isp-cmd.c b/drivers/media/platform/apple/isp/isp-cmd.c index 48aa139a81ccf..8ee46b3271fff 100644 --- a/drivers/media/platform/apple/isp/isp-cmd.c +++ b/drivers/media/platform/apple/isp/isp-cmd.c @@ -151,6 +151,18 @@ int isp_cmd_set_dsid_clr_req_base(struct apple_isp *isp, u64 dsid_clr_base, return CISP_SEND_IN(isp, args); } +int isp_cmd_set_dsid_clr_multi_bc(struct apple_isp *isp, u64 dsid_clr_base, + u32 dsid_clr_range) +{ + struct cmd_set_dsid_clr_multi_bc_reg_base args = { + .opcode = CISP_OPCODE(CISP_CMD_SET_DSID_CLR_MULTI_BC_REG_BASE), + .count = 1, + .dsid_clr_range = dsid_clr_range, + .dsid_clr_base = dsid_clr_base, + }; + return CISP_SEND_IN(isp, args); +} + int isp_cmd_pmp_ctrl_set(struct apple_isp *isp, u64 clock_scratch, u64 clock_base, u8 clock_bit, u8 clock_size, u64 bandwidth_scratch, u64 bandwidth_base, @@ -218,9 +230,26 @@ int isp_cmd_flicker_sensor_set(struct apple_isp *isp, u32 mode) int isp_cmd_ch_info_get(struct apple_isp *isp, u32 chan, struct cmd_ch_info *args) { + struct cmd_ch_info_h17 *h17; + int err; + args->opcode = CISP_OPCODE(CISP_CMD_CH_INFO_GET); args->chan = chan; - return CISP_SEND_OUT(isp, args); + if (isp->hw->fw_abi != ISP_FW_ABI_H17) + return CISP_SEND_OUT(isp, args); + + /* Exchange the longer H17 record and keep the common part. */ + h17 = kzalloc_obj(*h17); + if (!h17) + return -ENOMEM; + + h17->info = *args; + err = CISP_SEND_OUT(isp, h17); + if (!err) + *args = h17->info; + kfree(h17); + + return err; } int isp_cmd_ch_camera_config_current_get(struct apple_isp *isp, u32 chan, @@ -253,13 +282,21 @@ int isp_cmd_ch_buffer_return(struct apple_isp *isp, u32 chan) int isp_cmd_ch_set_file_load(struct apple_isp *isp, u32 chan, u64 addr, u32 size) { - if (isp->fw_compat >= ISP_FIRMWARE_V_13_5) { + bool h17 = isp->hw->fw_abi == ISP_FW_ABI_H17; + + if (h17 || isp->fw_compat >= ISP_FIRMWARE_V_13_5) { struct cmd_ch_set_file_load64 args = { .opcode = CISP_OPCODE(CISP_CMD_CH_SET_FILE_LOAD), .chan = chan, .addr = addr, .size = size, }; + /* + * The H17 firmware returns the record; what its reply + * contains is not known, so it is not checked. + */ + if (h17) + return CISP_SEND_INOUT(isp, args); return CISP_SEND_IN(isp, args); } else { struct cmd_ch_set_file_load args = { diff --git a/drivers/media/platform/apple/isp/isp-cmd.h b/drivers/media/platform/apple/isp/isp-cmd.h index 3301ccf633ff0..2da0752fe8355 100644 --- a/drivers/media/platform/apple/isp/isp-cmd.h +++ b/drivers/media/platform/apple/isp/isp-cmd.h @@ -86,6 +86,7 @@ #define CISP_CMD_IPC_ENDPOINT_UNSET2 0x300d #define CISP_CMD_SET_DSID_CLR_REG_BASE2 0x3204 #define CISP_CMD_SET_DSID_CLR_REG_BASE 0x3205 +#define CISP_CMD_SET_DSID_CLR_MULTI_BC_REG_BASE 0x3206 #define CISP_CMD_APPLE_CH_AE_METERING_MODE_SET 0x8206 #define CISP_CMD_APPLE_CH_AE_FD_SCENE_METERING_CONFIG_SET 0x820e #define CISP_CMD_APPLE_CH_AE_FLICKER_FREQ_UPDATE_CURRENT_SET 0x8212 @@ -195,6 +196,16 @@ struct cmd_set_dsid_clr_req_base { } __packed; static_assert(sizeof(struct cmd_set_dsid_clr_req_base) == 0x14); +/* H17: a count of broadcast-clear windows and the first window */ +struct cmd_set_dsid_clr_multi_bc_reg_base { + u64 opcode; + u8 count; + u8 pad_9[3]; + u32 dsid_clr_range; + u64 dsid_clr_base; +} __packed; +static_assert(sizeof(struct cmd_set_dsid_clr_multi_bc_reg_base) == 0x18); + struct cmd_pmp_ctrl_set { u64 opcode; u64 clock_scratch; @@ -248,6 +259,8 @@ int isp_cmd_config_get(struct apple_isp *isp, struct cmd_config_get *args); int isp_cmd_set_isp_pmu_base(struct apple_isp *isp, u64 pmu_base); int isp_cmd_set_dsid_clr_req_base(struct apple_isp *isp, u64 dsid_clr_base, u32 dsid_clr_range); +int isp_cmd_set_dsid_clr_multi_bc(struct apple_isp *isp, u64 dsid_clr_base, + u32 dsid_clr_range); int isp_cmd_set_dsid_clr_req_base2(struct apple_isp *isp, u64 dsid_clr_base0, u64 dsid_clr_base1, u64 dsid_clr_base2, u64 dsid_clr_base3, u32 dsid_clr_range0, @@ -326,6 +339,17 @@ struct cmd_ch_info { } __packed; static_assert(sizeof(struct cmd_ch_info) == 0x118); +/* + * The H17 firmware exchanges a 0x190-byte CH_INFO_GET record whose first + * 0x118 bytes match struct cmd_ch_info; there unk_68 is the size of the + * per-frame metadata record. + */ +struct cmd_ch_info_h17 { + struct cmd_ch_info info; + u32 unk_118[0x1e]; +} __packed; +static_assert(sizeof(struct cmd_ch_info_h17) == 0x190); + struct cmd_ch_camera_config { u64 opcode; u32 chan; diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index ee866ccd7a78c..f4f15c5b0d719 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -788,6 +788,38 @@ static int isp_stop_command_processor(struct apple_isp *isp) return 0; } +/* Pass the PMU base and the DSID broadcast-clear windows. */ +static int isp_set_dsid_clr(struct apple_isp *isp) +{ + const struct apple_isp_hw *hw = isp->hw; + int err; + + /* + * The H17 firmware takes no PMU base, and its single broadcast-clear + * window in the multi-window form. + */ + if (hw->fw_abi == ISP_FW_ABI_H17) + return isp_cmd_set_dsid_clr_multi_bc(isp, hw->dsid_clr_base0, + hw->dsid_clr_range0); + + err = isp_cmd_set_isp_pmu_base(isp, hw->pmu_base); + if (err) + return err; + + if (hw->dsid_count == 1) + return isp_cmd_set_dsid_clr_req_base(isp, hw->dsid_clr_base0, + hw->dsid_clr_range0); + + return isp_cmd_set_dsid_clr_req_base2(isp, hw->dsid_clr_base0, + hw->dsid_clr_base1, + hw->dsid_clr_base2, + hw->dsid_clr_base3, + hw->dsid_clr_range0, + hw->dsid_clr_range1, + hw->dsid_clr_range2, + hw->dsid_clr_range3); +} + static int isp_start_command_processor(struct apple_isp *isp) { int err; @@ -796,25 +828,10 @@ static int isp_start_command_processor(struct apple_isp *isp) if (err) return err; - err = isp_cmd_set_isp_pmu_base(isp, isp->hw->pmu_base); + err = isp_set_dsid_clr(isp); if (err) return err; - if (isp->hw->dsid_count == 1) { - err = isp_cmd_set_dsid_clr_req_base( - isp, isp->hw->dsid_clr_base0, isp->hw->dsid_clr_range0); - if (err) - return err; - } else { - err = isp_cmd_set_dsid_clr_req_base2( - isp, isp->hw->dsid_clr_base0, isp->hw->dsid_clr_base1, - isp->hw->dsid_clr_base2, isp->hw->dsid_clr_base3, - isp->hw->dsid_clr_range0, isp->hw->dsid_clr_range1, - isp->hw->dsid_clr_range2, isp->hw->dsid_clr_range3); - if (err) - return err; - } - if (isp->hw->clock_scratch) { err = isp_cmd_pmp_ctrl_set( isp, isp->hw->clock_scratch, isp->hw->clock_base, From da45dbf8daa39a811191882a558640e61754bdf9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:36:00 +1000 Subject: [PATCH 0561/2680] media: apple: isp: Configure capture channels for the H17 firmware The H17 firmware on T8140 expects a different channel configuration from the older firmware: - The frame rate limits and all buffer pools are configured before the preview stream is enabled, and the rendered pool configuration carries the plane sizes and strides of the output buffers, in the places of the metadata sizes and of the first two reserved words, with two data blocks. - CH_LOCAL_RAW_BUFFER_ENABLE is required: the sensor data takes the ISP-local raw path, and without the local buffers the firmware waits for a raw pool from the host. - APPLE_CH_AE_FD_SCENE_METERING_CONFIG_SET with the constants the older firmware takes makes this firmware fault, so it is not sent. - The configuration ends with CH_MASTER_SLAVE_SYNC_MODE_SET(0). - The IMX558 module needs the setfile with header version 0x21, isp_1925_03XX.dat; the firmware faults while parsing the lens shading data of the version 0x20 files the other SoCs use. The file holds 0xcba6 bytes, and the firmware is given 0xcbc0, the length rounded up to 64 bytes, with the tail cleared. Add the local raw buffer, master/slave sync and rendered pool commands, the latter with its own structure rather than reusing the metadata fields, and follow the H17 order when the match data selects that interface. The order for the older firmware does not change: the frame rate and pool steps move into helpers that both orders share. The setfile table gains an optional length of the file's contents; the other setfiles still have to be present in full. Existing SoCs are not affected. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-cam.c | 111 ++++++++++++++++----- drivers/media/platform/apple/isp/isp-cmd.c | 42 ++++++++ drivers/media/platform/apple/isp/isp-cmd.h | 48 +++++++++ drivers/media/platform/apple/isp/isp-drv.h | 1 + 4 files changed, 178 insertions(+), 24 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index 131ded5d4a795..ddcf24eb55075 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -16,6 +16,8 @@ struct isp_setfile { u32 magic; const char *path; size_t size; + /* bytes present in the file if fewer than size; the rest is zero */ + size_t file_size; }; // clang-format off @@ -45,6 +47,7 @@ static const struct isp_setfile isp_setfiles[] = { [ISP_IMX514_2820_04] = {0x514, 0x28200405, "apple/isp_2820_04XX.dat", 0xa198}, [ISP_IMX558_1921_01] = {0x558, 0x19210106, "apple/isp_1921_01XX.dat", 0xad40}, [ISP_IMX558_1922_02] = {0x558, 0x19220201, "apple/isp_1922_02XX.dat", 0xad40}, + [ISP_IMX558_1925_03] = {0x558, 0x19250306, "apple/isp_1925_03XX.dat", 0xcbc0, 0xcba6}, [ISP_IMX603_7920_01] = {0x603, 0x79200109, "apple/isp_7920_01XX.dat", 0xad2c}, [ISP_IMX603_7920_02] = {0x603, 0x79200205, "apple/isp_7920_02XX.dat", 0xad2c}, [ISP_IMX603_7921_01] = {0x603, 0x79210104, "apple/isp_7921_01XX.dat", 0xad90}, @@ -115,7 +118,11 @@ static int isp_ch_get_sensor_id(struct apple_isp *isp, u32 ch) id = ISP_IMX514_2820_01; break; case 0x558: - id = ISP_IMX558_1921_01; + /* The H17 firmware only parses header version 0x21 setfiles. */ + if (isp->hw->fw_abi == ISP_FW_ABI_H17) + id = ISP_IMX558_1925_03; + else + id = ISP_IMX558_1921_01; break; case 0x603: id = ISP_IMX603_7920_01; @@ -234,7 +241,7 @@ static int isp_ch_cache_sensor_info(struct apple_isp *isp, u32 ch) err = isp_ch_get_sensor_id(isp, ch); if (err || (fmt->id != ISP_IMX248_1820_01 && fmt->id != ISP_IMX558_1921_01 && - fmt->id != ISP_IMX364_8720_01)) { + fmt->id != ISP_IMX558_1925_03 && fmt->id != ISP_IMX364_8720_01)) { dev_err(isp->dev, "ch %d: unsupported sensor. Please file a bug report with hardware info & dmesg trace.\n", ch); @@ -319,6 +326,7 @@ static int isp_ch_load_setfile(struct apple_isp *isp, u32 ch) { struct isp_format *fmt = isp_get_format(isp, ch); const struct isp_setfile *setfile = &isp_setfiles[fmt->id]; + size_t len = setfile->file_size ?: setfile->size; const struct firmware *fw; u32 magic; int err; @@ -334,9 +342,9 @@ static int isp_ch_load_setfile(struct apple_isp *isp, u32 ch) return err; } - if (fw->size < setfile->size) { + if (fw->size < len) { dev_err(isp->dev, "setfile '%s' too small (0x%zx/0x%zx)\n", - setfile->path, fw->size, setfile->size); + setfile->path, fw->size, len); release_firmware(fw); return -EINVAL; } @@ -349,16 +357,55 @@ static int isp_ch_load_setfile(struct apple_isp *isp, u32 ch) return -EINVAL; } - memcpy(isp->data_surf->virt, fw->data, setfile->size); + memcpy(isp->data_surf->virt, fw->data, len); + memset(isp->data_surf->virt + len, 0, setfile->size - len); release_firmware(fw); return isp_cmd_ch_set_file_load(isp, ch, isp->data_surf->iova, setfile->size); } +static int isp_ch_configure_frame_rate(struct apple_isp *isp, u32 ch) +{ + int err; + + err = isp_cmd_ch_ae_frame_rate_max_set(isp, ch, ISP_FRAME_RATE_DEN); + if (err) + return err; + + return isp_cmd_ch_ae_frame_rate_min_set(isp, ch, ISP_FRAME_RATE_DEN2); +} + +static int isp_ch_configure_pools(struct apple_isp *isp, u32 ch) +{ + struct isp_format *fmt = isp_get_format(isp, ch); + int err; + + err = isp_cmd_ch_buffer_pool_config_set(isp, ch, CISP_POOL_TYPE_META); + if (err) + return err; + + err = isp_cmd_ch_buffer_pool_config_set(isp, ch, + CISP_POOL_TYPE_META_CAPTURE); + if (err) + return err; + + /* The H17 firmware also takes the geometry of the output buffers. */ + if (isp->hw->fw_abi != ISP_FW_ABI_H17) + return 0; + + return isp_cmd_ch_buffer_pool_config_set_rendered(isp, ch, + ISP_MAX_BUFFERS, + fmt->plane_size[0], + fmt->strides[0], + fmt->plane_size[1], + fmt->strides[1]); +} + static int isp_ch_configure_capture(struct apple_isp *isp, u32 ch) { struct isp_format *fmt = isp_get_format(isp, ch); + bool h17 = isp->hw->fw_abi == ISP_FW_ABI_H17; int err; isp_cmd_flicker_sensor_set(isp, 0); @@ -408,6 +455,26 @@ static int isp_ch_configure_capture(struct apple_isp *isp, u32 ch) if (err) return err; + /* + * The H17 firmware takes the frame rate limits and the buffer pools + * before the preview stream is enabled. It also needs its local raw + * buffers enabled: the sensor data takes the ISP-local raw path, and + * without them the firmware waits for a raw pool from the host. + */ + if (h17) { + err = isp_ch_configure_frame_rate(isp, ch); + if (err) + return err; + + err = isp_ch_configure_pools(isp, ch); + if (err) + return err; + + err = isp_cmd_ch_local_raw_buffer_enable(isp, ch, 1); + if (err) + return err; + } + err = isp_cmd_ch_preview_stream_set(isp, ch, 1); if (err) return err; @@ -420,9 +487,13 @@ static int isp_ch_configure_capture(struct apple_isp *isp, u32 ch) if (err) return err; - err = isp_cmd_apple_ch_ae_fd_scene_metering_config_set(isp, ch); - if (err) - return err; + /* The H17 firmware faults on this face-detection metering setup. */ + if (!h17) { + err = isp_cmd_apple_ch_ae_fd_scene_metering_config_set(isp, + ch); + if (err) + return err; + } err = isp_cmd_apple_ch_ae_metering_mode_set(isp, ch, 3); if (err) @@ -440,13 +511,11 @@ static int isp_ch_configure_capture(struct apple_isp *isp, u32 ch) if (err) return err; - err = isp_cmd_ch_ae_frame_rate_max_set(isp, ch, ISP_FRAME_RATE_DEN); - if (err) - return err; - - err = isp_cmd_ch_ae_frame_rate_min_set(isp, ch, ISP_FRAME_RATE_DEN2); - if (err) - return err; + if (!h17) { + err = isp_ch_configure_frame_rate(isp, ch); + if (err) + return err; + } err = isp_cmd_apple_ch_temporal_filter_start(isp, ch, isp->temporal_filter); if (err) @@ -460,16 +529,10 @@ static int isp_ch_configure_capture(struct apple_isp *isp, u32 ch) if (err) return err; - err = isp_cmd_ch_buffer_pool_config_set(isp, ch, CISP_POOL_TYPE_META); - if (err) - return err; + if (h17) + return isp_cmd_ch_master_slave_sync_mode_set(isp, ch, 0); - err = isp_cmd_ch_buffer_pool_config_set(isp, ch, - CISP_POOL_TYPE_META_CAPTURE); - if (err) - return err; - - return 0; + return isp_ch_configure_pools(isp, ch); } static int isp_configure_capture(struct apple_isp *isp) diff --git a/drivers/media/platform/apple/isp/isp-cmd.c b/drivers/media/platform/apple/isp/isp-cmd.c index 8ee46b3271fff..1d20b9fd49f68 100644 --- a/drivers/media/platform/apple/isp/isp-cmd.c +++ b/drivers/media/platform/apple/isp/isp-cmd.c @@ -319,6 +319,28 @@ int isp_cmd_ch_sbs_enable(struct apple_isp *isp, u32 chan, u32 enable) return CISP_SEND_IN(isp, args); } +int isp_cmd_ch_local_raw_buffer_enable(struct apple_isp *isp, u32 chan, + u16 enable) +{ + struct cmd_ch_local_raw_buffer_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_LOCAL_RAW_BUFFER_ENABLE), + .chan = chan, + .enable = enable, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_master_slave_sync_mode_set(struct apple_isp *isp, u32 chan, + u32 mode) +{ + struct cmd_ch_master_slave_sync_mode_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_MASTER_SLAVE_SYNC_MODE_SET), + .chan = chan, + .mode = mode, + }; + return CISP_SEND_IN(isp, args); +} + int isp_cmd_ch_crop_set(struct apple_isp *isp, u32 chan, u32 x1, u32 y1, u32 x2, u32 y2) { @@ -446,6 +468,26 @@ int isp_cmd_ch_buffer_pool_config_set(struct apple_isp *isp, u32 chan, u16 type) return CISP_SEND_INOUT(isp, args); } +int isp_cmd_ch_buffer_pool_config_set_rendered(struct apple_isp *isp, u32 chan, + u16 count, u32 plane0_size, + u32 plane0_stride, + u32 plane1_size, + u32 plane1_stride) +{ + struct cmd_ch_buffer_pool_config_set_rendered args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_BUFFER_POOL_CONFIG_SET), + .chan = chan, + .type = CISP_POOL_TYPE_RENDERED, + .count = count, + .plane0_size = plane0_size, + .plane0_stride = plane0_stride, + .plane1_size = plane1_size, + .plane1_stride = plane1_stride, + .data_blocks = 2, + }; + return CISP_SEND_INOUT(isp, args); +} + int isp_cmd_ch_buffer_pool_return(struct apple_isp *isp, u32 chan) { struct cmd_ch_buffer_pool_return args = { diff --git a/drivers/media/platform/apple/isp/isp-cmd.h b/drivers/media/platform/apple/isp/isp-cmd.h index 2da0752fe8355..1b01eb043f4d9 100644 --- a/drivers/media/platform/apple/isp/isp-cmd.h +++ b/drivers/media/platform/apple/isp/isp-cmd.h @@ -41,6 +41,7 @@ #define CISP_CMD_CH_LOCAL_RAW_BUFFER_ENABLE 0x0125 #define CISP_CMD_CH_META_DATA_ENABLE 0x0126 #define CISP_CMD_CH_CAMERA_MIPI_FREQUENCY_TOTAL_GET 0x0133 +#define CISP_CMD_CH_MASTER_SLAVE_SYNC_MODE_SET 0x0138 #define CISP_CMD_CH_SBS_ENABLE 0x013b #define CISP_CMD_CH_LSC_POLYNOMIAL_COEFF_GET 0x0142 #define CISP_CMD_CH_SET_META_DATA_REQUIRED 0x014f @@ -410,6 +411,21 @@ struct cmd_ch_sbs_enable { } __packed; static_assert(sizeof(struct cmd_ch_sbs_enable) == 0x10); +struct cmd_ch_local_raw_buffer_enable { + u64 opcode; + u32 chan; + u16 enable; + u16 pad_e; +} __packed; +static_assert(sizeof(struct cmd_ch_local_raw_buffer_enable) == 0x10); + +struct cmd_ch_master_slave_sync_mode_set { + u64 opcode; + u32 chan; + u32 mode; +} __packed; +static_assert(sizeof(struct cmd_ch_master_slave_sync_mode_set) == 0x10); + struct cmd_ch_crop_set { u64 opcode; u32 chan; @@ -503,6 +519,10 @@ int isp_cmd_ch_set_file_load(struct apple_isp *isp, u32 chan, u64 addr, u32 size); int isp_cmd_ch_buffer_return(struct apple_isp *isp, u32 chan); int isp_cmd_ch_sbs_enable(struct apple_isp *isp, u32 chan, u32 enable); +int isp_cmd_ch_local_raw_buffer_enable(struct apple_isp *isp, u32 chan, + u16 enable); +int isp_cmd_ch_master_slave_sync_mode_set(struct apple_isp *isp, u32 chan, + u32 mode); int isp_cmd_ch_crop_set(struct apple_isp *isp, u32 chan, u32 x1, u32 y1, u32 x2, u32 y2); int isp_cmd_ch_output_config_set(struct apple_isp *isp, u32 chan, u32 width, @@ -553,6 +573,29 @@ struct cmd_ch_buffer_pool_config_set { } __packed; static_assert(sizeof(struct cmd_ch_buffer_pool_config_set) == 0x9c); +/* + * The H17 firmware takes the output geometry with the rendered pool: the + * plane sizes and strides in the places of the metadata sizes and the + * first two reserved words. + */ +struct cmd_ch_buffer_pool_config_set_rendered { + u64 opcode; + u32 chan; + u16 type; + u16 count; + u32 plane0_size; + u32 plane0_stride; + u64 unk0; + u64 unk1; + u64 unk2; + u32 plane1_size; + u32 plane1_stride; + u32 zero[0x17]; + u32 data_blocks; + u32 compress; +} __packed; +static_assert(sizeof(struct cmd_ch_buffer_pool_config_set_rendered) == 0x9c); + struct cmd_ch_buffer_pool_return { u64 opcode; u32 chan; @@ -564,6 +607,11 @@ int isp_cmd_ch_buffer_recycle_mode_set(struct apple_isp *isp, u32 chan, int isp_cmd_ch_buffer_recycle_start(struct apple_isp *isp, u32 chan); int isp_cmd_ch_buffer_pool_config_set(struct apple_isp *isp, u32 chan, u16 type); +int isp_cmd_ch_buffer_pool_config_set_rendered(struct apple_isp *isp, u32 chan, + u16 count, u32 plane0_size, + u32 plane0_stride, + u32 plane1_size, + u32 plane1_stride); int isp_cmd_ch_buffer_pool_config_get(struct apple_isp *isp, u32 chan, u16 type); int isp_cmd_ch_buffer_pool_return(struct apple_isp *isp, u32 chan); diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index f6797b90e98bb..4661f8cddf54a 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -198,6 +198,7 @@ enum isp_sensor_id { ISP_IMX514_2820_04, ISP_IMX558_1921_01, ISP_IMX558_1922_02, + ISP_IMX558_1925_03, ISP_IMX603_7920_01, ISP_IMX603_7920_02, ISP_IMX603_7921_01, From 629eee0d771497b62968a2236d4849d79ab5e129 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:48:26 +1000 Subject: [PATCH 0562/2680] media: apple: isp: Add the capture metadata pool Besides the per-frame metadata buffers, the H17 firmware on T8140 uses a pool of capture metadata buffers. CH_INFO_GET reports their size, 0x29fc0 bytes, and the pool is configured as CISP_POOL_TYPE_META_CAPTURE with that size rather than the metadata size the older firmware takes there. The firmware is given the metadata buffers before CH_START and the capture metadata and capture buffers right after it, the order it was run with when streaming worked. When the match data gives a capture metadata size: - configure the capture metadata pool with it, and refuse a firmware that reports a different size, as for the metadata size; - allocate eight capture metadata buffers, as many as the firmware was run with, without a kernel mapping, since only the firmware accesses them; - send the metadata buffers in a batch of their own before CH_START and the capture metadata and capture buffers after it, and count the capture metadata buffers in the largest batch, which boot stage 2 requires the command area to hold; SoCs without the pool keep the requirement they had; - recognise returned capture metadata buffers. In the buffer lists the driver tags these buffers with pool type 2 and recognises returned ones by that tag, as the code the firmware was first brought up with did, although the pool is configured as type 8. Why the two differ is not known. Existing SoCs are not affected. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-cam.c | 11 ++- drivers/media/platform/apple/isp/isp-cmd.c | 8 +- drivers/media/platform/apple/isp/isp-cmd.h | 2 +- drivers/media/platform/apple/isp/isp-drv.h | 18 +++- drivers/media/platform/apple/isp/isp-fw.c | 4 +- drivers/media/platform/apple/isp/isp-v4l2.c | 103 +++++++++++++++++--- 6 files changed, 118 insertions(+), 28 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index ddcf24eb55075..f3600500ceac9 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -227,11 +227,14 @@ static int isp_ch_cache_sensor_info(struct apple_isp *isp, u32 ch) dev_dbg(isp->dev, "found sensor %x on ch %d\n", args->version, ch); - /* The metadata buffers are allocated for the size the driver knows. */ + /* The metadata buffers are allocated for the sizes the driver knows. */ if (isp->hw->fw_abi == ISP_FW_ABI_H17 && - args->unk_68 != isp->hw->meta_size) { - dev_err(isp->dev, "metadata size 0x%x, expected 0x%x\n", - args->unk_68, isp->hw->meta_size); + (args->unk_68 != isp->hw->meta_size || + args->unk_78 != isp->hw->capture_meta_size)) { + dev_err(isp->dev, + "metadata sizes 0x%x/0x%x, expected 0x%x/0x%x\n", + args->unk_68, args->unk_78, isp->hw->meta_size, + isp->hw->capture_meta_size); err = -ENODEV; goto exit; } diff --git a/drivers/media/platform/apple/isp/isp-cmd.c b/drivers/media/platform/apple/isp/isp-cmd.c index 1d20b9fd49f68..42c448fd0f30c 100644 --- a/drivers/media/platform/apple/isp/isp-cmd.c +++ b/drivers/media/platform/apple/isp/isp-cmd.c @@ -452,13 +452,17 @@ int isp_cmd_ch_buffer_recycle_start(struct apple_isp *isp, u32 chan) int isp_cmd_ch_buffer_pool_config_set(struct apple_isp *isp, u32 chan, u16 type) { + u32 size = type == CISP_POOL_TYPE_META_CAPTURE && + isp->hw->capture_meta_size ? + isp->hw->capture_meta_size : + isp->hw->meta_size; struct cmd_ch_buffer_pool_config_set args = { .opcode = CISP_OPCODE(CISP_CMD_CH_BUFFER_POOL_CONFIG_SET), .chan = chan, .type = type, .count = ISP_MAX_BUFFERS, - .meta_size0 = isp->hw->meta_size, - .meta_size1 = isp->hw->meta_size, + .meta_size0 = size, + .meta_size1 = size, .unk0 = 0, .unk1 = 0, .unk2 = 0, diff --git a/drivers/media/platform/apple/isp/isp-cmd.h b/drivers/media/platform/apple/isp/isp-cmd.h index 1b01eb043f4d9..ab0c85023bebf 100644 --- a/drivers/media/platform/apple/isp/isp-cmd.h +++ b/drivers/media/platform/apple/isp/isp-cmd.h @@ -343,7 +343,7 @@ static_assert(sizeof(struct cmd_ch_info) == 0x118); /* * The H17 firmware exchanges a 0x190-byte CH_INFO_GET record whose first * 0x118 bytes match struct cmd_ch_info; there unk_68 is the size of the - * per-frame metadata record. + * per-frame metadata record and unk_78 that of the capture metadata. */ struct cmd_ch_info_h17 { struct cmd_ch_info info; diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 4661f8cddf54a..c590877f09ef8 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -27,16 +27,20 @@ /* used to limit the user space buffers to the buffer_pool_config */ #define ISP_MAX_BUFFERS 16 +/* capture metadata buffers, for firmware that has that pool */ +#define ISP_CAPMETA_BUFFERS 8 + /* * The command area holds one command or one buffer batch at a time. A * batch is a 16-byte header and a 64-byte descriptor per buffer, and - * carries at most every metadata buffer and one full rendered pool. + * carries at most every metadata buffer, the @capmeta capture metadata + * buffers and one full rendered pool. */ #define ISP_BUFLIST_HDR_SIZE 0x10 #define ISP_BUFLIST_DESC_SIZE 0x40 -#define ISP_CMD_AREA_SIZE \ - (ISP_BUFLIST_HDR_SIZE + \ - ISP_BUFLIST_DESC_SIZE * (ISP_MAX_BUFFERS + ISP_MAX_BUFFERS)) +#define ISP_CMD_AREA_SIZE(capmeta) \ + (ISP_BUFLIST_HDR_SIZE + ISP_BUFLIST_DESC_SIZE * \ + (ISP_MAX_BUFFERS + (capmeta) + ISP_MAX_BUFFERS)) enum isp_generation { ISP_GEN_T8103, @@ -170,6 +174,9 @@ struct apple_isp_hw { /* ISP_GPIO_6 boot mode */ u32 boot_mode; + + /* size of a capture metadata buffer, 0 for no such pool */ + u32 capture_meta_size; }; enum isp_sensor_id { @@ -284,6 +291,7 @@ struct apple_isp { struct isp_surf *log_surf; struct isp_surf *bt_surf; struct isp_surf *meta_surfs[ISP_MAX_BUFFERS]; + struct isp_surf *capmeta_surfs[ISP_CAPMETA_BUFFERS]; struct list_head gc; struct workqueue_struct *wq; @@ -338,6 +346,8 @@ enum { */ #define isp_fw_iova(isp, x) ((x) & (isp)->fw_iova_mask) #define isp_get_format(isp, ch) (&(isp)->fmts[(ch)]) +#define isp_num_capmeta(isp) \ + ((isp)->hw->capture_meta_size ? ISP_CAPMETA_BUFFERS : 0) #define isp_get_current_format(isp) (isp_get_format(isp, isp->current_ch)) #endif /* __ISP_DRV_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index f4f15c5b0d719..e80180df56ccf 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -500,6 +500,7 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) bool h16 = isp->hw->fw_abi == ISP_FW_ABI_H17; size_t args_size = h16 ? sizeof(struct isp_firmware_bootargs_h16) : sizeof(struct isp_firmware_bootargs); + size_t cmd_size = ISP_CMD_AREA_SIZE(isp_num_capmeta(isp)); dma_addr_t args_iova, cmd_iova; void *args_virt, *cmd_virt; int err; @@ -540,8 +541,7 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) &args_iova) && !check_add_overflow(args_iova, args_size + 0x40, &cmd_iova)) { args_virt = apple_isp_ipc_translate(isp, args_iova, args_size); - cmd_virt = apple_isp_ipc_translate(isp, cmd_iova, - ISP_CMD_AREA_SIZE); + cmd_virt = apple_isp_ipc_translate(isp, cmd_iova, cmd_size); } if (!args_virt || !cmd_virt) { dev_err(isp->dev, "invalid boot arguments offset 0x%x\n", diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 9f3c2674a64d5..a64e16a60ab62 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -43,7 +43,13 @@ struct isp_buflist { } __packed; static_assert(sizeof(struct isp_buflist) == ISP_BUFLIST_HDR_SIZE); /* the firmware reads at least ISP_IPC_BUFEXC_STAT_SIZE bytes of a batch */ -static_assert(ISP_CMD_AREA_SIZE >= ISP_IPC_BUFEXC_STAT_SIZE); +static_assert(ISP_CMD_AREA_SIZE(0) >= ISP_IPC_BUFEXC_STAT_SIZE); + +/* + * Buffer list tag of capture metadata buffers. The pool itself is set up + * as CISP_POOL_TYPE_META_CAPTURE; why the tags differ is not known. + */ +#define ISP_BUFLIST_POOL_CAPTURE_META 2 int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan) { @@ -91,6 +97,14 @@ int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan) meta->submitted = false; } } + } else if (bufd->pool_type == ISP_BUFLIST_POOL_CAPTURE_META && + isp_num_capmeta(isp)) { + for (int j = 0; j < isp_num_capmeta(isp); j++) { + struct isp_surf *meta = isp->capmeta_surfs[j]; + + if (meta && (u32)bufd->iovas[0] == (u32)meta->iova) + meta->submitted = false; + } } else { list_for_each_entry_safe_reverse( buf, tmp, &isp->bufs_submitted, link) { @@ -122,7 +136,12 @@ int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan) return err; } -static int isp_submit_buffers(struct apple_isp *isp) +/* + * With a capture metadata pool, the firmware takes the metadata buffers + * before CH_START and the capture metadata and capture buffers after it, + * so the batch before the start (@pre_start) has only the former. + */ +static int isp_submit_buffers(struct apple_isp *isp, bool pre_start) { struct isp_format *fmt = isp_get_current_format(isp); struct isp_channel *chan = isp->chan_bh; @@ -137,7 +156,7 @@ static int isp_submit_buffers(struct apple_isp *isp) struct isp_buflist_buffer *bufd = &bl->buffers[0]; /* Clear what earlier commands and batches left in the reserved fields. */ - memset(bl, 0, ISP_CMD_AREA_SIZE); + memset(bl, 0, ISP_CMD_AREA_SIZE(isp_num_capmeta(isp))); bl->type = 1; spin_lock_irqsave(&isp->buf_lock, flags); @@ -160,6 +179,25 @@ static int isp_submit_buffers(struct apple_isp *isp) meta->submitted = true; } + if (isp_num_capmeta(isp) && pre_start) + goto send; + + for (int i = 0; i < isp_num_capmeta(isp); i++) { + struct isp_surf *meta = isp->capmeta_surfs[i]; + + if (meta->submitted) + continue; + + bufd->num_planes = 1; + bufd->pool_type = ISP_BUFLIST_POOL_CAPTURE_META; + bufd->iovas[0] = meta->iova; + bufd->flags[0] = 0x40000000; + bufd++; + bl->num_buffers++; + + meta->submitted = true; + } + /* One rendered pool per batch; the rest stays pending for the next. */ while (num_rendered < ISP_MAX_BUFFERS && (buf = list_first_entry_or_null(&isp->bufs_pending, @@ -189,6 +227,7 @@ static int isp_submit_buffers(struct apple_isp *isp) list_move_tail(&buf->link, &isp->bufs_submitted); } +send: spin_unlock_irqrestore(&isp->buf_lock, flags); req->arg0 = isp->cmd_iova; @@ -224,6 +263,12 @@ static int isp_submit_buffers(struct apple_isp *isp) meta->submitted = false; } } + for (int j = 0; j < isp_num_capmeta(isp); j++) { + struct isp_surf *meta = isp->capmeta_surfs[j]; + + if (bufd->iovas[0] == meta->iova) + meta->submitted = false; + } } spin_unlock_irqrestore(&isp->buf_lock, flags); @@ -344,7 +389,7 @@ static void isp_vb2_buf_queue(struct vb2_buffer *vb) spin_unlock_irqrestore(&isp->buf_lock, flags); if (test_bit(ISP_STATE_STREAMING, &isp->state) && !empty) - isp_submit_buffers(isp); + isp_submit_buffers(isp, false); } static int apple_isp_start_streaming(struct apple_isp *isp) @@ -357,7 +402,7 @@ static int apple_isp_start_streaming(struct apple_isp *isp) return err; } - err = isp_submit_buffers(isp); + err = isp_submit_buffers(isp, true); if (err) { dev_err(isp->dev, "failed to send initial batch: %d\n", err); goto stop_camera; @@ -369,6 +414,16 @@ static int apple_isp_start_streaming(struct apple_isp *isp) goto stop_camera; } + if (isp_num_capmeta(isp)) { + err = isp_submit_buffers(isp, false); + if (err) { + dev_err(isp->dev, "failed to send the capture batch: %d\n", + err); + apple_isp_stop_capture(isp); + goto stop_camera; + } + } + set_bit(ISP_STATE_STREAMING, &isp->state); return 0; @@ -835,6 +890,21 @@ static const struct media_device_ops isp_media_device_ops = { .link_notify = v4l2_pipeline_link_notify, }; +static void isp_free_meta_surfaces(struct apple_isp *isp) +{ + for (int i = 0; i < ARRAY_SIZE(isp->capmeta_surfs); i++) { + if (isp->capmeta_surfs[i]) + isp_free_surface(isp, isp->capmeta_surfs[i]); + isp->capmeta_surfs[i] = NULL; + } + + for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) { + if (isp->meta_surfs[i]) + isp_free_surface(isp, isp->meta_surfs[i]); + isp->meta_surfs[i] = NULL; + } +} + int apple_isp_setup_video(struct apple_isp *isp) { struct video_device *vdev = &isp->vdev; @@ -858,6 +928,17 @@ int apple_isp_setup_video(struct apple_isp *isp) } } + /* Only the firmware reads and writes these, so they need no vmap. */ + for (int i = 0; i < isp_num_capmeta(isp); i++) { + isp->capmeta_surfs[i] = + isp_alloc_surface(isp, isp->hw->capture_meta_size); + if (!isp->capmeta_surfs[i]) { + isp_err(isp, "failed to alloc capture meta surface\n"); + err = -ENOMEM; + goto surf_cleanup; + } + } + media_device_init(&isp->mdev); isp->v4l2_dev.mdev = &isp->mdev; isp->mdev.ops = &isp_media_device_ops; @@ -926,11 +1007,7 @@ int apple_isp_setup_video(struct apple_isp *isp) media_cleanup: media_device_cleanup(&isp->mdev); surf_cleanup: - for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) { - if (isp->meta_surfs[i]) - isp_free_surface(isp, isp->meta_surfs[i]); - isp->meta_surfs[i] = NULL; - } + isp_free_meta_surfaces(isp); return err; } @@ -941,9 +1018,5 @@ void apple_isp_remove_video(struct apple_isp *isp) v4l2_device_unregister(&isp->v4l2_dev); media_device_unregister(&isp->mdev); media_device_cleanup(&isp->mdev); - for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) { - if (isp->meta_surfs[i]) - isp_free_surface(isp, isp->meta_surfs[i]); - isp->meta_surfs[i] = NULL; - } + isp_free_meta_surfaces(isp); } From 192c7ff80bbe4333da7ec88e9295710c7720fd63 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:40:05 +1000 Subject: [PATCH 0563/2680] media: apple: isp: Service the T8140 capture watchdog On T8140 the ISP firmware gates the image on a watchdog outside the ISP, the window the binding names "wdt". While that watchdog is not serviced, the firmware keeps running but replaces every frame with a flat fill. Servicing it means writing 0 to its clear register at 0xc, 0x244140 to its reload register at 0x8 and 1 to its kick register at 0x0, every 33 ms, from before CH_START until the channel is stopped and its buffers are returned: the sequence and rate observed on macOS. macOS services this watchdog only while its camera-in-use indicator is shown, which makes it a privacy interlock: frames carry an image only while the user can see that the camera is on. That indicator is drawn by the display pipeline for secure firmware that Linux does not run, and no indicator the driver could drive instead is known, so the driver cannot tie the watchdog to one. It services the watchdog whenever it captures; the stream, as seen by userspace, is then the only sign that the camera is on. Map the "wdt" window when the device tree provides one and service it from an hrtimer while capturing. writel() already orders the three writes, so no barrier follows them. The timer does not need the hard interrupt mode: a late kick can only mask frames. Other SoCs have no "wdt" window and are not affected. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-cam.c | 67 ++++++++++++++++++++- drivers/media/platform/apple/isp/isp-cam.h | 3 + drivers/media/platform/apple/isp/isp-drv.c | 11 ++++ drivers/media/platform/apple/isp/isp-drv.h | 4 ++ drivers/media/platform/apple/isp/isp-regs.h | 7 +++ 5 files changed, 91 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index f3600500ceac9..3585b43c2e1f9 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -8,6 +8,7 @@ #include "isp-cmd.h" #include "isp-fw.h" #include "isp-iommu.h" +#include "isp-regs.h" #define ISP_MAX_PRESETS 32 @@ -568,13 +569,77 @@ void apple_isp_stop_camera(struct apple_isp *isp) apple_isp_firmware_shutdown(isp); } +/* + * Capture watchdog. While it is not serviced, the ISP firmware replaces + * every frame with a flat fill. macOS services it only while its + * camera-in-use indicator is shown, which makes it a privacy interlock. + * That indicator is drawn by the display pipeline for secure firmware + * that Linux does not run, so the driver services the watchdog whenever + * it captures, with the register sequence and rate observed on macOS. + */ +static void apple_isp_wdt_kick(struct apple_isp *isp) +{ + writel(0, isp->wdt + ISP_WDT_CLEAR); + writel(ISP_WDT_RELOAD_VAL, isp->wdt + ISP_WDT_RELOAD); + writel(1, isp->wdt + ISP_WDT_KICK); +} + +static enum hrtimer_restart apple_isp_wdt_timer(struct hrtimer *timer) +{ + struct apple_isp *isp = container_of(timer, struct apple_isp, + wdt_timer); + + apple_isp_wdt_kick(isp); + hrtimer_forward_now(timer, ns_to_ktime(ISP_WDT_PERIOD_NS)); + + return HRTIMER_RESTART; +} + +void apple_isp_wdt_init(struct apple_isp *isp) +{ + hrtimer_setup(&isp->wdt_timer, apple_isp_wdt_timer, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); +} + +static void apple_isp_wdt_start(struct apple_isp *isp) +{ + if (!isp->wdt || isp->wdt_running) + return; + + isp->wdt_running = true; + apple_isp_wdt_kick(isp); + hrtimer_start(&isp->wdt_timer, ns_to_ktime(ISP_WDT_PERIOD_NS), + HRTIMER_MODE_REL); +} + +void apple_isp_wdt_stop(struct apple_isp *isp) +{ + if (!isp->wdt_running) + return; + + hrtimer_cancel(&isp->wdt_timer); + isp->wdt_running = false; +} + int apple_isp_start_capture(struct apple_isp *isp) { - return isp_cmd_ch_start(isp, 0); // TODO channel mask + int err; + + /* Serviced from before the channel starts... */ + apple_isp_wdt_start(isp); + + err = isp_cmd_ch_start(isp, 0); // TODO channel mask + if (err) + apple_isp_wdt_stop(isp); + + return err; } void apple_isp_stop_capture(struct apple_isp *isp) { isp_cmd_ch_stop(isp, 0); // TODO channel mask isp_cmd_ch_buffer_return(isp, isp->current_ch); + + /* ... until its buffers are back. */ + apple_isp_wdt_stop(isp); } diff --git a/drivers/media/platform/apple/isp/isp-cam.h b/drivers/media/platform/apple/isp/isp-cam.h index f4fa4224c7a93..97b3940139663 100644 --- a/drivers/media/platform/apple/isp/isp-cam.h +++ b/drivers/media/platform/apple/isp/isp-cam.h @@ -18,4 +18,7 @@ void apple_isp_stop_camera(struct apple_isp *isp); int apple_isp_start_capture(struct apple_isp *isp); void apple_isp_stop_capture(struct apple_isp *isp); +void apple_isp_wdt_init(struct apple_isp *isp); +void apple_isp_wdt_stop(struct apple_isp *isp); + #endif /* __ISP_CAM_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 24e4c4786e286..0a0eb66f0780b 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -440,6 +440,17 @@ static int apple_isp_probe(struct platform_device *pdev) } } + /* Capture watchdog, where the ISP gates its frames on one */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "wdt"); + if (res) { + isp->wdt = devm_ioremap_resource(dev, res); + if (IS_ERR(isp->wdt)) { + err = PTR_ERR(isp->wdt); + goto detach_genpd; + } + apple_isp_wdt_init(isp); + } + isp->irq = platform_get_irq(pdev, 0); if (isp->irq < 0) { err = isp->irq; diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index c590877f09ef8..7f4b8681b696d 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -4,6 +4,7 @@ #ifndef __ISP_DRV_H__ #define __ISP_DRV_H__ +#include #include #include #include @@ -273,6 +274,9 @@ struct apple_isp { void __iomem *mbox; void __iomem *gpio; void __iomem *mbox2; + void __iomem *wdt; + struct hrtimer wdt_timer; + bool wdt_running; struct iommu_domain *domain; unsigned long shift; diff --git a/drivers/media/platform/apple/isp/isp-regs.h b/drivers/media/platform/apple/isp/isp-regs.h index bf6878ef7b16d..f2f6f1c9489f4 100644 --- a/drivers/media/platform/apple/isp/isp-regs.h +++ b/drivers/media/platform/apple/isp/isp-regs.h @@ -59,6 +59,13 @@ #define ISP_MBOX_IRQ_ENABLE2_T8140_VAL 0x700000 #define ISP_MBOX_IRQ_ROUTE_T8140_VAL 0x8 +/* capture watchdog ("wdt") */ +#define ISP_WDT_KICK 0x00 +#define ISP_WDT_RELOAD 0x08 +#define ISP_WDT_CLEAR 0x0c +#define ISP_WDT_RELOAD_VAL 0x244140 +#define ISP_WDT_PERIOD_NS (NSEC_PER_SEC / 30) + #define ISP_GPIO_0 0x00 #define ISP_GPIO_1 0x04 #define ISP_GPIO_2 0x08 From cf45611a6dd295eb5b626b5729a9bfbdfe6e4f49 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:43:20 +1000 Subject: [PATCH 0564/2680] media: apple: isp: Keep resident firmware running between streams The T8140 firmware is loaded by the bootloader and cannot be started a second time: once it has been stopped with CISP_CMD_SUSPEND, the coprocessor reset and its power domains cycled, it never completes the first handshake of another boot. The driver boots and stops the firmware for every stream and for the camera detection at probe, so on T8140 only the detection would ever work. Add a match data flag for such resident firmware, and track the firmware state explicitly as off, running or dead: - Resident firmware is booted once, for the detection at probe, and keeps running between streams. Every stream configures the channel again, including its buffer pools, so metadata buffers still marked as submitted are reset, under the buffer lock, when a stream starts. - Resident firmware that has been stopped, or that failed to boot, is dead. Starting a stream then fails at once with -EIO instead of running the reset and waiting for a handshake that cannot come, and a stream that was running at system sleep fails its restart, which flags the vb2 queue. - A failed boot at probe returns its error instead of -EPROBE_DEFER, because a deferred retry would fail the same way. - Unbind stops the firmware after the video device is unregistered and before the metadata surfaces are freed, and a probe failure after the boot stops it as well. For system sleep the firmware is stopped cleanly before the system sleeps, not kept alive. Keeping it alive would need the ISP power domains to stay on, which genpd only does for a device on an awake path whose domain allows it, and would need the ISP DARTs to keep their state, while the DART driver resets unlocked DARTs on resume. Neither has been tried with the firmware running, and firmware that loses power without being told may have DMA in flight. Stopping it first keeps the ISP quiet while its DARTs and domains go down; the price is that the camera is unavailable after the first system sleep until the next boot, which the driver reports. While the driver is bound, resident firmware holds the runtime PM reference, and with it the ISP power domains, because it has to keep running. SHAREDMALLOC surfaces the firmware releases are still only reclaimed when it stops, which for resident firmware means at unbind. Existing SoCs still boot their firmware for each stream. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-cam.c | 10 ++++- drivers/media/platform/apple/isp/isp-drv.c | 20 +++++++++- drivers/media/platform/apple/isp/isp-drv.h | 15 ++++++++ drivers/media/platform/apple/isp/isp-fw.c | 42 +++++++++++++++++++++ drivers/media/platform/apple/isp/isp-fw.h | 1 + drivers/media/platform/apple/isp/isp-v4l2.c | 20 ++++++++++ drivers/media/platform/apple/isp/isp-v4l2.h | 1 + 7 files changed, 107 insertions(+), 2 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index 3585b43c2e1f9..d35e8041894b4 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -311,11 +311,19 @@ int apple_isp_detect_camera(struct apple_isp *isp) dev_err(isp->dev, "failed to boot firmware for initial sensor detection: %d\n", err); - return -EPROBE_DEFER; + /* Resident firmware would fail a deferred retry the same way. */ + return isp->hw->resident_fw ? err : -EPROBE_DEFER; } err = isp_detect_camera(isp); + /* Resident firmware stays up; the channel is set up per stream. */ + if (isp->hw->resident_fw) { + if (err) + apple_isp_firmware_halt(isp); + return err; + } + isp_cmd_flicker_sensor_set(isp, 0); isp_cmd_ch_stop(isp, 0); diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 0a0eb66f0780b..0d6864c728906 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -498,11 +498,13 @@ static int apple_isp_probe(struct platform_device *pdev) err = apple_isp_setup_video(isp); if (err) { dev_err(dev, "failed to register video device: %d\n", err); - goto free_surface; + goto halt_firmware; } return 0; +halt_firmware: + apple_isp_firmware_halt(isp); free_surface: pm_runtime_disable(dev); apple_isp_free_firmware_surface(isp); @@ -520,6 +522,9 @@ static void apple_isp_remove(struct platform_device *pdev) struct apple_isp *isp = platform_get_drvdata(pdev); apple_isp_remove_video(isp); + /* Stop resident firmware before freeing what it may still use. */ + apple_isp_firmware_halt(isp); + apple_isp_free_video(isp); pm_runtime_disable(isp->dev); apple_isp_free_firmware_surface(isp); apple_isp_free_iommu(isp); @@ -725,6 +730,19 @@ static __maybe_unused int apple_isp_suspend(struct device *dev) */ apple_isp_video_suspend(isp); + /* + * The ISP's power domains go off during system sleep, and resident + * firmware would not survive that. Stop it cleanly instead; resuming + * a stream then fails with an error until the next system boot. + */ + if (isp->hw->resident_fw) { + mutex_lock(&isp->video_lock); + if (isp->fw_state == ISP_FW_RUNNING) + dev_warn(dev, "stopping the firmware for system sleep, the camera is unavailable until the next boot\n"); + apple_isp_firmware_halt(isp); + mutex_unlock(&isp->video_lock); + } + return 0; } diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 7f4b8681b696d..f7d99c97a096f 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -60,6 +60,13 @@ enum isp_fw_abi { ISP_FW_ABI_H17, }; +enum isp_fw_state { + ISP_FW_OFF, + ISP_FW_RUNNING, + /* resident firmware that stopped: it cannot be started again */ + ISP_FW_DEAD, +}; + enum isp_firmware_version { ISP_FIRMWARE_V_UNKNOWN, ISP_FIRMWARE_V_12_3, @@ -178,6 +185,12 @@ struct apple_isp_hw { /* size of a capture metadata buffer, 0 for no such pool */ u32 capture_meta_size; + + /* + * The firmware can be started only once per system boot, so it is + * started at probe and kept running while the driver is bound. + */ + bool resident_fw; }; enum isp_sensor_id { @@ -288,6 +301,8 @@ struct apple_isp { struct isp_firmware { u64 heap_top; } fw; + /* changes under video_lock once the video device is registered */ + enum isp_fw_state fw_state; struct isp_surf *ipc_surf; struct isp_surf *extra_surf; diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index e80180df56ccf..7ece3219b2ea0 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -10,6 +10,7 @@ #include #include +#include "isp-cam.h" #include "isp-cmd.h" #include "isp-fw.h" #include "isp-iommu.h" @@ -934,10 +935,30 @@ static void isp_firmware_shutdown(struct apple_isp *isp) isp_collect_gc_surface(isp); } +/* + * Resident firmware (hw->resident_fw) is loaded by the bootloader and + * does not start again once it has been stopped: after CISP_CMD_SUSPEND, + * a coprocessor reset and a power cycle of its domains it never completes + * the first handshake. It is booted once, at probe, and runs until the + * driver is unbound, probe fails or the system goes to sleep; after that + * the device stays unusable until the next system boot. + */ int apple_isp_firmware_boot(struct apple_isp *isp) { int err; + switch (isp->fw_state) { + case ISP_FW_OFF: + break; + case ISP_FW_RUNNING: + /* Resident firmware keeps running between streams. */ + return 0; + case ISP_FW_DEAD: + dev_err_ratelimited(isp->dev, + "firmware was stopped and cannot be restarted before the next boot\n"); + return -EIO; + } + /* Needs to be power cycled for IOMMU to behave correctly */ err = pm_runtime_resume_and_get(isp->dev); if (err < 0) { @@ -949,14 +970,35 @@ int apple_isp_firmware_boot(struct apple_isp *isp) if (err) { dev_err(isp->dev, "failed to boot firmware: %d\n", err); pm_runtime_put_sync(isp->dev); + if (isp->hw->resident_fw) + isp->fw_state = ISP_FW_DEAD; return err; } + isp->fw_state = ISP_FW_RUNNING; + return 0; } +/* The end of a stream or of the camera detection at probe */ void apple_isp_firmware_shutdown(struct apple_isp *isp) { + if (isp->hw->resident_fw) + return; + + apple_isp_firmware_halt(isp); +} + +/* Stop the firmware; resident firmware cannot be started again. */ +void apple_isp_firmware_halt(struct apple_isp *isp) +{ + /* Only a capture services the watchdog, but be sure. */ + apple_isp_wdt_stop(isp); + + if (isp->fw_state != ISP_FW_RUNNING) + return; + isp_firmware_shutdown(isp); pm_runtime_put_sync(isp->dev); + isp->fw_state = isp->hw->resident_fw ? ISP_FW_DEAD : ISP_FW_OFF; } diff --git a/drivers/media/platform/apple/isp/isp-fw.h b/drivers/media/platform/apple/isp/isp-fw.h index 974216f0989f9..71e97a77d25c4 100644 --- a/drivers/media/platform/apple/isp/isp-fw.h +++ b/drivers/media/platform/apple/isp/isp-fw.h @@ -11,6 +11,7 @@ void apple_isp_free_firmware_surface(struct apple_isp *isp); int apple_isp_firmware_boot(struct apple_isp *isp); void apple_isp_firmware_shutdown(struct apple_isp *isp); +void apple_isp_firmware_halt(struct apple_isp *isp); void *apple_isp_translate(struct apple_isp *isp, struct isp_surf *surf, dma_addr_t iova, size_t size); diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index a64e16a60ab62..6753a60fa8838 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -394,6 +394,7 @@ static void isp_vb2_buf_queue(struct vb2_buffer *vb) static int apple_isp_start_streaming(struct apple_isp *isp) { + unsigned long flags; int err; err = apple_isp_start_camera(isp); @@ -402,6 +403,20 @@ static int apple_isp_start_streaming(struct apple_isp *isp) return err; } + /* + * Resident firmware was not restarted, but the channel's pools are + * configured again for every stream. Metadata buffers still marked + * as submitted to the previous pools would never reach the new ones. + */ + if (isp->hw->resident_fw) { + spin_lock_irqsave(&isp->buf_lock, flags); + for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) + isp->meta_surfs[i]->submitted = false; + for (int i = 0; i < isp_num_capmeta(isp); i++) + isp->capmeta_surfs[i]->submitted = false; + spin_unlock_irqrestore(&isp->buf_lock, flags); + } + err = isp_submit_buffers(isp, true); if (err) { dev_err(isp->dev, "failed to send initial batch: %d\n", err); @@ -1018,5 +1033,10 @@ void apple_isp_remove_video(struct apple_isp *isp) v4l2_device_unregister(&isp->v4l2_dev); media_device_unregister(&isp->mdev); media_device_cleanup(&isp->mdev); +} + +/* After apple_isp_remove_video() and with the firmware stopped */ +void apple_isp_free_video(struct apple_isp *isp) +{ isp_free_meta_surfaces(isp); } diff --git a/drivers/media/platform/apple/isp/isp-v4l2.h b/drivers/media/platform/apple/isp/isp-v4l2.h index 4d47deeb83b05..622288296aaae 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.h +++ b/drivers/media/platform/apple/isp/isp-v4l2.h @@ -8,6 +8,7 @@ int apple_isp_setup_video(struct apple_isp *isp); void apple_isp_remove_video(struct apple_isp *isp); +void apple_isp_free_video(struct apple_isp *isp); int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan); int apple_isp_video_suspend(struct apple_isp *isp); From 0cc89b7e75794093239c1ce3a3d9c12414efd2f5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:44:34 +1000 Subject: [PATCH 0565/2680] media: apple: isp: Add T8140 support Add the match data for the ISP of the A18 Pro (T8140), which runs the camera of the MacBook Neo (J700), and match the apple,t8140-isp compatible: - an ISP17a coprocessor with resident H17 firmware, which takes the 0x290-byte boot descriptor with boot mode 1 and gates its frames on the capture watchdog that the device tree describes as "wdt"; - no PMU base, one DSID broadcast-clear window of 0x2fc bytes at 0x220114000, and the PMP clock and bandwidth scratch registers at 0x3003d0ca0 and 0x302824000, both 8 bytes wide; - metadata records of 0x4d00 bytes and capture metadata records of 0x29fc0 bytes, the sizes CH_INFO_GET reports; - the doorbell block at offset 0x410 of the "mbox" window, the interrupt enable word at 0x8 together with the additional enable and routing words, and the coprocessor control register at 0x1600044; - 36-bit DART addresses, which the firmware reports with its vm-base, bit 40, set; - twelve MMIO windows the firmware accesses at their physical addresses. These are the values the firmware was booted and streamed with on J700. The windows are the bootloader's DART address filter ranges for the ISP rounded out to 16 KiB pages, and several of them are much wider than the registers the firmware is known to use: the 160 KiB window at 0x3003c0000 contains the PMP clock scratch word among other PMGR registers, and the page at 0x300704000 holds the ISP power-state registers and the rest of that 16 KiB of the PMGR. The ISP can write every register in these pages. Compile-tested only; not run on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.c | 55 ++++++++++++++++++++++ drivers/media/platform/apple/isp/isp-drv.h | 2 + 2 files changed, 57 insertions(+) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 0d6864c728906..aaa809aeb1680 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -698,6 +698,60 @@ static const struct apple_isp_hw apple_isp_hw_t6031 = { .meta_size = ISP_META_SIZE_T6031, }; +/* + * Registers the T8140 firmware accesses at their physical addresses: the + * bootloader's DART address filter ranges for the ISP, rounded out to + * 16 KiB pages. + */ +static const struct isp_mmio_window apple_isp_fw_mmio_t8140[] = { + { 0x220004000, 0x14000 }, /* DSID broadcast-clear windows */ + { 0x220044000, 0x14000 }, + { 0x220084000, 0x14000 }, + { 0x2200c4000, 0x14000 }, + { 0x220104000, 0x14000 }, + { 0x3003c0000, 0x28000 }, /* PMGR, with the PMP clock scratch */ + { 0x300704000, 0x4000 }, /* PMGR, ISP power states */ + { 0x300730000, 0x4000 }, + { 0x300e3c000, 0x4000 }, + { 0x302824000, 0x4000 }, /* PMP bandwidth scratch */ + { 0x31062c000, 0x20000 }, + { 0x401660000, 0x4000 }, +}; + +static const struct apple_isp_hw apple_isp_hw_t8140 = { + .gen = ISP_GEN_T8140, + .fw_abi = ISP_FW_ABI_H17, + .pmu_base = 0x0, + + .dsid_count = 1, + .dsid_clr_base0 = 0x220114000, + .dsid_clr_range0 = 0x2fc, + + .clock_scratch = 0x3003d0ca0, + .clock_base = 0x0, + .clock_bit = 0x0, + .clock_size = 0x8, + .bandwidth_scratch = 0x302824000, + .bandwidth_base = 0x0, + .bandwidth_bit = 0x0, + .bandwidth_size = 0x8, + .mbox_irq_enable = ISP_MBOX_IRQ_ENABLE_T6031, + + .scl1 = false, + .lpdp = true, + .meta_size = ISP_META_SIZE_T8140, + + .fw_iova_mask = GENMASK_ULL(35, 0), + .mbox2_offset = 0x410, + .mbox_irq_route = true, + .coproc_control = ISP_COPROC_CONTROL_T8140, + .fw_mmio = apple_isp_fw_mmio_t8140, + .num_fw_mmio = ARRAY_SIZE(apple_isp_fw_mmio_t8140), + .boot_mode = 1, + .capture_meta_size = ISP_CAPTURE_META_SIZE_T8140, + .resident_fw = true, +}; + static const struct of_device_id apple_isp_of_match[] = { { .compatible = "apple,t8103-isp", .data = &apple_isp_hw_t8103 }, { .compatible = "apple,t8112-isp", .data = &apple_isp_hw_t8112 }, @@ -706,6 +760,7 @@ static const struct of_device_id apple_isp_of_match[] = { { .compatible = "apple,t6020-isp", .data = &apple_isp_hw_t6020 }, { .compatible = "apple,t6030-isp", .data = &apple_isp_hw_t6030 }, { .compatible = "apple,t6031-isp", .data = &apple_isp_hw_t6031 }, + { .compatible = "apple,t8140-isp", .data = &apple_isp_hw_t8140 }, {}, }; MODULE_DEVICE_TABLE(of, apple_isp_of_match); diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index f7d99c97a096f..10dc441a030d1 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -24,6 +24,8 @@ #define ISP_META_SIZE_T8103 0x4640 #define ISP_META_SIZE_T8112 0x4840 #define ISP_META_SIZE_T6031 0x4a40 +#define ISP_META_SIZE_T8140 0x4d00 +#define ISP_CAPTURE_META_SIZE_T8140 0x29fc0 /* used to limit the user space buffers to the buffer_pool_config */ #define ISP_MAX_BUFFERS 16 From c7e49c478068250683e6f447a465666bd8ae98bd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 21:29:31 +1000 Subject: [PATCH 0566/2680] media: apple: isp: Describe the capture watchdog by observed behaviour The comment on the T8140 capture watchdog explained the servicing rule by what another operating system's implementation does, not by what the hardware and firmware do. Reword it to state the observed behaviour directly. No functional change. [ryan: split from commit 1ce75ee9e797 ("treewide: apple: t8140: Describe observed behaviour, not other implementations") for the 7.2.10 topics: the ISP hunk only; the DT hunk travels with 000-devicetree and the AOP and audio hunks with 070-audio] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-cam.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index d35e8041894b4..6dc262912d3e5 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -579,11 +579,12 @@ void apple_isp_stop_camera(struct apple_isp *isp) /* * Capture watchdog. While it is not serviced, the ISP firmware replaces - * every frame with a flat fill. macOS services it only while its - * camera-in-use indicator is shown, which makes it a privacy interlock. - * That indicator is drawn by the display pipeline for secure firmware - * that Linux does not run, so the driver services the watchdog whenever - * it captures, with the register sequence and rate observed on macOS. + * every frame with a flat fill. The firmware expects it to be serviced + * only while a camera-in-use indicator is shown, which makes it a privacy + * interlock. That indicator is drawn by the display pipeline for secure + * firmware that Linux does not run, so the driver services the watchdog + * whenever it captures, with the register sequence and rate observed on + * the running hardware. */ static void apple_isp_wdt_kick(struct apple_isp *isp) { From 772d9583f53d5687774f4165dc9b94b1ad93b38f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 11:46:12 +1100 Subject: [PATCH 0567/2680] media: apple: isp: Let userspace select the capture frame rate The driver reports a single 30 fps frame interval and ignores VIDIOC_S_PARM, so applications cannot ask for a lower rate. Offer 30, 25, 24, 20 and 15 fps as discrete intervals for every frame size. VIDIOC_S_PARM picks the closest of them, a zero interval selects the nominal 30 fps, and a change is refused while streaming because the rate is configured when the stream starts. VIDIOC_G_PARM reports the selected interval. Frame interval enumeration now also rejects pixel formats and frame sizes that the driver does not offer. The selected rate becomes the auto-exposure frame-rate ceiling. The firmware takes rates in units of 1/256 frame per second, as the existing 30 and 15 fps values (7680 and 3840) show. The auto-exposure floor stays at 15 fps, so the default behaviour is unchanged. On the J700 (T8140) 1080p NV12 streams run at 30.01, 25.00, 24.01, 20.01 and 15.00 fps for the five intervals, a 1001/30000 s request selects 30 fps, and VIDIOC_S_PARM returns -EBUSY during a stream. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-cam.c | 8 ++- drivers/media/platform/apple/isp/isp-cam.h | 9 ++- drivers/media/platform/apple/isp/isp-drv.h | 2 + drivers/media/platform/apple/isp/isp-v4l2.c | 70 ++++++++++++++++++--- 4 files changed, 76 insertions(+), 13 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index 6dc262912d3e5..834faf0c57b3e 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -381,11 +381,15 @@ static int isp_ch_configure_frame_rate(struct apple_isp *isp, u32 ch) { int err; - err = isp_cmd_ch_ae_frame_rate_max_set(isp, ch, ISP_FRAME_RATE_DEN); + err = isp_cmd_ch_ae_frame_rate_max_set(isp, ch, + isp->frame_rate * + ISP_FRAME_RATE_SCALE); if (err) return err; - return isp_cmd_ch_ae_frame_rate_min_set(isp, ch, ISP_FRAME_RATE_DEN2); + return isp_cmd_ch_ae_frame_rate_min_set(isp, ch, + ISP_FRAME_RATE_MIN * + ISP_FRAME_RATE_SCALE); } static int isp_ch_configure_pools(struct apple_isp *isp, u32 ch) diff --git a/drivers/media/platform/apple/isp/isp-cam.h b/drivers/media/platform/apple/isp/isp-cam.h index 97b3940139663..51c2f2ec6c079 100644 --- a/drivers/media/platform/apple/isp/isp-cam.h +++ b/drivers/media/platform/apple/isp/isp-cam.h @@ -6,9 +6,12 @@ #include "isp-drv.h" -#define ISP_FRAME_RATE_NUM 256 -#define ISP_FRAME_RATE_DEN 7680 -#define ISP_FRAME_RATE_DEN2 3840 +/* The firmware takes frame rates in units of 1/256 frame per second. */ +#define ISP_FRAME_RATE_SCALE 256 +/* The slowest rate auto exposure may fall to, in frames per second */ +#define ISP_FRAME_RATE_MIN 15 +/* The nominal capture rate, in frames per second */ +#define ISP_FRAME_RATE_DEFAULT 30 int apple_isp_detect_camera(struct apple_isp *isp); diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 10dc441a030d1..96ab4a8a24343 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -277,6 +277,8 @@ struct apple_isp { struct mutex video_lock; unsigned int sequence; bool multiplanar; + /* capture rate in frames per second, changes under video_lock */ + unsigned int frame_rate; int pd_count; struct device **pd_dev; diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 6753a60fa8838..6468c4e5e95a7 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright 2023 Eileen Yoon */ +#include #include #include @@ -22,6 +23,9 @@ #define ISP_BUFFER_TIMEOUT msecs_to_jiffies(1500) #define ISP_STRIDE_ALIGNMENT 64 +/* Capture rates offered to userspace, in frames per second */ +static const unsigned int isp_frame_rates[] = { 30, 25, 24, 20, 15 }; + static bool multiplanar = false; module_param(multiplanar, bool, 0644); MODULE_PARM_DESC(multiplanar, "Enable multiplanar API"); @@ -641,18 +645,56 @@ static int isp_vidioc_enum_framesizes(struct file *file, void *fh, return 0; } -static int isp_vidioc_enum_frameintervals(struct file *filp, void *priv, +static int isp_vidioc_enum_frameintervals(struct file *file, void *fh, struct v4l2_frmivalenum *interval) { - if (interval->index != 0) + struct apple_isp *isp = video_drvdata(file); + int i; + + if (interval->index >= ARRAY_SIZE(isp_frame_rates)) + return -EINVAL; + + if (interval->pixel_format != V4L2_PIX_FMT_NV12 && + interval->pixel_format != V4L2_PIX_FMT_NV12M) + return -EINVAL; + + for (i = 0; i < isp->num_presets; i++) { + if (isp->presets[i].output_dim.x == interval->width && + isp->presets[i].output_dim.y == interval->height) + break; + } + if (i == isp->num_presets) return -EINVAL; interval->type = V4L2_FRMIVAL_TYPE_DISCRETE; interval->discrete.numerator = 1; - interval->discrete.denominator = 30; + interval->discrete.denominator = isp_frame_rates[interval->index]; return 0; } +/* Returns the supported rate whose frame interval is closest to @tpf. */ +static unsigned int isp_closest_frame_rate(const struct v4l2_fract *tpf) +{ + unsigned int best = isp_frame_rates[0]; + + /* + * |n/d - 1/r| = |n * r - d| / (d * r), so the closest rate has the + * smallest |n * r - d| / r; compare those by cross-multiplying. + */ + for (int i = 1; i < ARRAY_SIZE(isp_frame_rates); i++) { + unsigned int rate = isp_frame_rates[i]; + u64 err = abs_diff((u64)tpf->numerator * rate, + (u64)tpf->denominator); + u64 best_err = abs_diff((u64)tpf->numerator * best, + (u64)tpf->denominator); + + if (err * best < best_err * rate) + best = rate; + } + + return best; +} + static inline void isp_get_sp_pix_format(struct apple_isp *isp, struct v4l2_format *f, struct isp_format *fmt) @@ -836,8 +878,8 @@ static int isp_vidioc_get_param(struct file *file, void *fh, a->parm.capture.capability = V4L2_CAP_TIMEPERFRAME; a->parm.capture.readbuffers = ISP_MIN_FRAMES; - a->parm.capture.timeperframe.numerator = ISP_FRAME_RATE_NUM; - a->parm.capture.timeperframe.denominator = ISP_FRAME_RATE_DEN; + a->parm.capture.timeperframe.numerator = 1; + a->parm.capture.timeperframe.denominator = isp->frame_rate; return 0; } @@ -846,17 +888,27 @@ static int isp_vidioc_set_param(struct file *file, void *fh, struct v4l2_streamparm *a) { struct apple_isp *isp = video_drvdata(file); + struct v4l2_fract *tpf = &a->parm.capture.timeperframe; if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE && (!isp->multiplanar || a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)) return -EINVAL; - /* Not supporting frame rate sets. No use. Plus floats. */ + /* The rate is configured when the stream starts. */ + if (vb2_is_streaming(&isp->vbq)) + return -EBUSY; + + /* A zero interval selects the nominal rate. */ + if (!tpf->numerator || !tpf->denominator) + isp->frame_rate = ISP_FRAME_RATE_DEFAULT; + else + isp->frame_rate = isp_closest_frame_rate(tpf); + a->parm.capture.capability = V4L2_CAP_TIMEPERFRAME; a->parm.capture.readbuffers = ISP_MIN_FRAMES; - a->parm.capture.timeperframe.numerator = ISP_FRAME_RATE_NUM; - a->parm.capture.timeperframe.denominator = ISP_FRAME_RATE_DEN; + tpf->numerator = 1; + tpf->denominator = isp->frame_rate; return 0; } @@ -933,6 +985,8 @@ int apple_isp_setup_video(struct apple_isp *isp) return err; } + isp->frame_rate = ISP_FRAME_RATE_DEFAULT; + for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) { isp->meta_surfs[i] = isp_alloc_surface_vmap(isp, isp->hw->meta_size); From 6dfc5baf61b962657d623a2df9b4c72286b0e04f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 11:48:31 +1100 Subject: [PATCH 0568/2680] media: apple: isp: Hold the frame rate in low light by default Auto exposure may lower the frame rate to 15 fps to lengthen the exposure. On the J700 (T8140) a 1080p stream runs at 30 fps in a lit room but falls to between 23.5 and 15 fps in the dark, while V4L2 still reports a 30 fps interval. V4L2 has a control for this choice. V4L2_CID_EXPOSURE_AUTO_PRIORITY, which UVC webcams also expose, says whether auto exposure may vary the frame rate, and it is off by default so that the rate stays constant. Add it, and set the auto-exposure frame-rate floor to the selected rate unless it is on, in which case the floor stays at 15 fps. The firmware takes the frame-rate limits before the stream starts, so the control is locked while streaming. This changes the default low-light behaviour on every supported ISP: auto exposure now raises the gain instead of slowing down, which keeps motion smooth at the cost of more noise. Setting the control to 1 restores the previous behaviour. On the J700 the control reads 0 by default, is reported as grabbed with VIDIOC_S_EXT_CTRLS returning -EBUSY during a stream, and 1080p streams run at 30.0 fps with it at 0 or 1 in a 60 lux room. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-cam.c | 17 ++++++---- drivers/media/platform/apple/isp/isp-drv.h | 3 ++ drivers/media/platform/apple/isp/isp-v4l2.c | 35 +++++++++++++++++++-- 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index 834faf0c57b3e..5090ed2ed5b98 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -379,17 +379,22 @@ static int isp_ch_load_setfile(struct apple_isp *isp, u32 ch) static int isp_ch_configure_frame_rate(struct apple_isp *isp, u32 ch) { + u32 rate = isp->frame_rate * ISP_FRAME_RATE_SCALE; + u32 min_rate = rate; int err; - err = isp_cmd_ch_ae_frame_rate_max_set(isp, ch, - isp->frame_rate * - ISP_FRAME_RATE_SCALE); + /* + * Auto exposure lengthens the frame time in low light only if it may + * vary the frame rate; otherwise it holds the rate and raises the gain. + */ + if (v4l2_ctrl_g_ctrl(isp->exposure_priority)) + min_rate = ISP_FRAME_RATE_MIN * ISP_FRAME_RATE_SCALE; + + err = isp_cmd_ch_ae_frame_rate_max_set(isp, ch, rate); if (err) return err; - return isp_cmd_ch_ae_frame_rate_min_set(isp, ch, - ISP_FRAME_RATE_MIN * - ISP_FRAME_RATE_SCALE); + return isp_cmd_ch_ae_frame_rate_min_set(isp, ch, min_rate); } static int isp_ch_configure_pools(struct apple_isp *isp, u32 ch) diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 96ab4a8a24343..ac93aba409840 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -273,6 +274,8 @@ struct apple_isp { struct video_device vdev; struct media_device mdev; struct v4l2_device v4l2_dev; + struct v4l2_ctrl_handler ctrl_handler; + struct v4l2_ctrl *exposure_priority; struct vb2_queue vbq; struct mutex video_lock; unsigned int sequence; diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 6468c4e5e95a7..0465834ebe622 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -6,6 +6,8 @@ #include #include +#include +#include #include #include #include @@ -469,9 +471,14 @@ static int isp_vb2_start_streaming(struct vb2_queue *q, unsigned int count) isp->sequence = 0; + /* The exposure priority is applied when the stream starts. */ + v4l2_ctrl_grab(isp->exposure_priority, true); + err = apple_isp_start_streaming(isp); - if (err) + if (err) { + v4l2_ctrl_grab(isp->exposure_priority, false); isp_vb2_release_buffers(isp, VB2_BUF_STATE_QUEUED); + } return err; } @@ -482,6 +489,7 @@ static void isp_vb2_stop_streaming(struct vb2_queue *q) apple_isp_stop_streaming(isp); isp_vb2_release_buffers(isp, VB2_BUF_STATE_ERROR); + v4l2_ctrl_grab(isp->exposure_priority, false); } int apple_isp_video_suspend(struct apple_isp *isp) @@ -941,6 +949,10 @@ static const struct v4l2_ioctl_ops isp_v4l2_ioctl_ops = { .vidioc_prepare_buf = vb2_ioctl_prepare_buf, .vidioc_streamon = vb2_ioctl_streamon, .vidioc_streamoff = vb2_ioctl_streamoff, + + .vidioc_log_status = v4l2_ctrl_log_status, + .vidioc_subscribe_event = v4l2_ctrl_subscribe_event, + .vidioc_unsubscribe_event = v4l2_event_unsubscribe, }; static const struct v4l2_file_operations isp_v4l2_fops = { @@ -1023,10 +1035,25 @@ int apple_isp_setup_video(struct apple_isp *isp) isp->multiplanar = multiplanar; + /* + * Auto exposure is always on. By default it holds the frame rate, as + * V4L2 specifies; with exposure priority it may slow down in low light. + */ + v4l2_ctrl_handler_init(&isp->ctrl_handler, 1); + isp->exposure_priority = + v4l2_ctrl_new_std(&isp->ctrl_handler, NULL, + V4L2_CID_EXPOSURE_AUTO_PRIORITY, 0, 1, 1, 0); + if (isp->ctrl_handler.error) { + err = isp->ctrl_handler.error; + dev_err(isp->dev, "failed to create controls: %d\n", err); + goto ctrl_free; + } + isp->v4l2_dev.ctrl_handler = &isp->ctrl_handler; + err = v4l2_device_register(isp->dev, &isp->v4l2_dev); if (err) { dev_err(isp->dev, "failed to register v4l2 device: %d\n", err); - goto media_unregister; + goto ctrl_free; } vbq->drv_priv = isp; @@ -1071,7 +1098,8 @@ int apple_isp_setup_video(struct apple_isp *isp) v4l2_unregister: v4l2_device_unregister(&isp->v4l2_dev); -media_unregister: +ctrl_free: + v4l2_ctrl_handler_free(&isp->ctrl_handler); media_device_unregister(&isp->mdev); media_cleanup: media_device_cleanup(&isp->mdev); @@ -1085,6 +1113,7 @@ void apple_isp_remove_video(struct apple_isp *isp) { vb2_video_unregister_device(&isp->vdev); v4l2_device_unregister(&isp->v4l2_dev); + v4l2_ctrl_handler_free(&isp->ctrl_handler); media_device_unregister(&isp->mdev); media_device_cleanup(&isp->mdev); } From 8f2ca2652f4c9f1dbb45748ffe275498d74db448 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 05:43:54 +1100 Subject: [PATCH 0569/2680] media: apple: isp: Keep system sleep off resident firmware's domains The driver switches the power domains of resident firmware through runtime PM alone, because the firmware does not start again once they have been off. Their virtual genpd devices still take part in system sleep, though: genpd counts them as suspended in the noirq phase of suspend, switches the domains off when nothing else holds them, and switches every one of them back on in the noirq phase of resume. The firmware is stopped for system sleep, so on T8140 the CPU core domains of the stopped coprocessor fail to reach the active state on every resume, and genpd turns them off again when the resume completes: PS isp_cpu_core0: Failed to reach power state 0xf (now: 0xf000b4f) PS isp_cpu_core1: Failed to reach power state 0xf (now: 0xf000b4f) Mark the domain devices as syscore devices for as long as they are attached, so that system sleep leaves the domains to runtime PM. The Apple ACIO driver does the same for the domains of its coprocessor. SoCs whose firmware is started for each stream are unchanged. On J700 (T8140) with this change, suspend-to-idle with a PMU RTC wake no longer logs the errors; the camera is still stopped for sleep. Fixes: 62b659e9e5c7 ("media: apple: isp: Keep resident firmware running between streams") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index aaa809aeb1680..436ffb78af943 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -28,8 +28,11 @@ static void apple_isp_detach_genpd(struct apple_isp *isp) for (int i = isp->pd_count - 1; i >= 0; i--) { if (isp->pd_link[i]) device_link_del(isp->pd_link[i]); - if (!IS_ERR_OR_NULL(isp->pd_dev[i])) - dev_pm_domain_detach(isp->pd_dev[i], true); + if (IS_ERR_OR_NULL(isp->pd_dev[i])) + continue; + if (isp->hw->resident_fw) + dev_pm_syscore_device(isp->pd_dev[i], false); + dev_pm_domain_detach(isp->pd_dev[i], true); } return; @@ -74,6 +77,18 @@ static int apple_isp_attach_genpd(struct apple_isp *isp) apple_isp_detach_genpd(isp); return -EINVAL; } + + /* + * The domains of resident firmware are switched by the + * driver alone, through runtime PM: the firmware does not + * start again once they have been off. System sleep would + * otherwise also have genpd switch them off in the noirq + * phase of suspend and back on in the noirq phase of resume, + * where the CPU core domains of a stopped coprocessor fail + * to reach the active state. + */ + if (isp->hw->resident_fw) + dev_pm_syscore_device(isp->pd_dev[i], true); } return 0; From e3c62694446d083d29147da5f0f14ff23b7a9710 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 05:43:54 +1100 Subject: [PATCH 0570/2680] media: apple: isp: Keep resident firmware running across suspend Resident firmware is stopped for every system sleep, because its power domains used to go off under it. It does not start again, so on T8140, where suspend-to-idle now works, the camera is unavailable after the first suspend until the next boot, and every VIDIOC_STREAMON fails with -EIO. System sleep now leaves those domains to the driver, which keeps them on while the firmware runs. Keep the firmware running, idle, across suspend. The stream is stopped before the system sleeps and restarted when it resumes, as before, with its buffers kept. The awake path, which keeps the NVMe controller's link domains on across sleep, would not be enough here: genpd honours it only for a domain that was on when it was registered, and nine of the fourteen ISP domains are first switched on by this driver. Keeping the domains on has a cost. With the display off and the firmware idle, the J700 draws 30 to 50 mW more than with the firmware stopped and its domains off (3.553 W against 3.523 W, and 3.564 W against 3.512 W, means of 300 and 360 samples). The idle firmware raised no interrupt in 30 seconds, and the ISP DARTs, which are not locked, are reset and restored by their own resume callback while it is idle. SoCs whose firmware is started for each stream are unchanged. Hibernation still stops the firmware: the image is restored by another kernel, and firmware left running would go on using the memory of the kernel that started it. Tested on J700 with a PMU RTC wake: five suspend-to-idle cycles of 30 and 90 seconds. 1080p NV12 capture runs at 30 fps before and after each cycle. A capture left running across two of the cycles resumes by itself at 30 fps and completes all its frames. No ISP, power domain or DART errors are logged, and the ISP domains stay on throughout. Fixes: 62b659e9e5c7 ("media: apple: isp: Keep resident firmware running between streams") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.c | 29 ++++++++++++++++++---- drivers/media/platform/apple/isp/isp-fw.c | 5 ++-- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 436ffb78af943..9f5f8f58b3748 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -801,14 +801,28 @@ static __maybe_unused int apple_isp_suspend(struct device *dev) apple_isp_video_suspend(isp); /* - * The ISP's power domains go off during system sleep, and resident - * firmware would not survive that. Stop it cleanly instead; resuming - * a stream then fails with an error until the next system boot. + * Resident firmware stays up, idle, across suspend-to-idle and + * suspend-to-RAM: its domains stay on, see apple_isp_attach_genpd(). + */ + return 0; +} + +static __maybe_unused int apple_isp_freeze(struct device *dev) +{ + struct apple_isp *isp = dev_get_drvdata(dev); + + apple_isp_video_suspend(isp); + + /* + * Hibernation hands the system from one kernel to another, and + * firmware left running would go on using the memory of the kernel + * that started it. Stop it cleanly instead; the camera is then + * unavailable until the next system boot. */ if (isp->hw->resident_fw) { mutex_lock(&isp->video_lock); if (isp->fw_state == ISP_FW_RUNNING) - dev_warn(dev, "stopping the firmware for system sleep, the camera is unavailable until the next boot\n"); + dev_warn(dev, "stopping the firmware for hibernation, the camera is unavailable until the next boot\n"); apple_isp_firmware_halt(isp); mutex_unlock(&isp->video_lock); } @@ -826,7 +840,12 @@ static __maybe_unused int apple_isp_resume(struct device *dev) } static const struct dev_pm_ops apple_isp_pm_ops = { - SYSTEM_SLEEP_PM_OPS(apple_isp_suspend, apple_isp_resume) + .suspend = pm_sleep_ptr(apple_isp_suspend), + .resume = pm_sleep_ptr(apple_isp_resume), + .freeze = pm_sleep_ptr(apple_isp_freeze), + .thaw = pm_sleep_ptr(apple_isp_resume), + .poweroff = pm_sleep_ptr(apple_isp_freeze), + .restore = pm_sleep_ptr(apple_isp_resume), RUNTIME_PM_OPS(apple_isp_runtime_suspend, apple_isp_runtime_resume, NULL) }; diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 7ece3219b2ea0..ed35f4701c04a 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -940,8 +940,9 @@ static void isp_firmware_shutdown(struct apple_isp *isp) * does not start again once it has been stopped: after CISP_CMD_SUSPEND, * a coprocessor reset and a power cycle of its domains it never completes * the first handshake. It is booted once, at probe, and runs until the - * driver is unbound, probe fails or the system goes to sleep; after that - * the device stays unusable until the next system boot. + * driver is unbound, probe fails or the system hibernates, also across + * suspend; after it has stopped the device stays unusable until the next + * system boot. */ int apple_isp_firmware_boot(struct apple_isp *isp) { From eeb56dd5b0dce96ab67ae94f5be8152671e06971 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:31:42 +1100 Subject: [PATCH 0571/2680] media: apple: isp: Stop the coprocessor before unwinding a failed boot A firmware boot that fails after the first handshake unwinds stage by stage: it frees the channel table and the extra heap, and only then stops the coprocessor and powers its domains down. Stage 2 also frees the extra heap itself when the second handshake does not come. The firmware is running from the first handshake on, so it can still access the heap while its pages are released; the pages are unmapped from the DART first, so a late access faults instead of reaching freed memory, but it should not happen at all. isp_firmware_shutdown() already stops the coprocessor before freeing anything. Share that order with the failed boot: stop the coprocessor and power its domains down, then free the channel table, the extra heap and the surfaces the firmware requested. Stage 2 leaves the heap to the caller, and the heap pointer is cleared once it has been freed, so an unwind that runs before the heap is allocated has nothing to free. The runtime PM reference of the ISP and the references on its other power domains are dropped as before. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-fw.c | 57 +++++++++++++---------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index ed35f4701c04a..65c760de616f1 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -455,7 +455,11 @@ void apple_isp_free_firmware_surface(struct apple_isp *isp) static void isp_firmware_shutdown_stage2(struct apple_isp *isp) { + if (!isp->extra_surf) + return; + isp_free_surface(isp, isp->extra_surf); + isp->extra_surf = NULL; } static void isp_write_bootargs(struct apple_isp *isp, void *virt, @@ -504,7 +508,6 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) size_t cmd_size = ISP_CMD_AREA_SIZE(isp_num_capmeta(isp)); dma_addr_t args_iova, cmd_iova; void *args_virt, *cmd_virt; - int err; u32 num_ipc_chans = isp_gpio_read32(isp, ISP_GPIO_0); u32 args_offset = isp_gpio_read32(isp, ISP_GPIO_1); @@ -575,19 +578,18 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) /* Wait for ISP_GPIO_7 to 0xf7fbdff9 -> 0x8042006 */ isp_gpio_write32(isp, ISP_GPIO_7, 0xf7fbdff9); + /* + * The firmware may already use the extra heap: it is freed only once + * the coprocessor has stopped, see isp_firmware_boot(). + */ if (isp_gpio_wait(isp, ISP_GPIO_7, 0x8042006)) { isp_err(isp, "never received second magic number from firmware\n"); - err = -ENODEV; - goto free_extra; + return -ENODEV; } isp_dbg(isp, "got second magic number from firmware\n"); return 0; - -free_extra: - isp_free_surface(isp, isp->extra_surf); - return err; } static inline struct isp_channel *isp_get_chan_index(struct apple_isp *isp, @@ -866,32 +868,49 @@ static void isp_collect_gc_surface(struct apple_isp *isp) } } +/* + * Stop the coprocessor and power its domains down, then release the memory + * the firmware may still be using: its channel table, extra heap and the + * surfaces it requested. + */ +static void isp_firmware_stop(struct apple_isp *isp) +{ + isp_firmware_shutdown_stage1(isp); + + isp_firmware_shutdown_stage3(isp); + isp_firmware_shutdown_stage2(isp); + isp_collect_gc_surface(isp); +} + static int isp_firmware_boot(struct apple_isp *isp) { int err; err = isp_firmware_boot_stage1(isp); if (err < 0) { + /* Stage 1 stops the coprocessor and powers it down itself. */ isp_err(isp, "failed firmware boot stage 1: %d\n", err); - goto garbage_collect; + isp_collect_gc_surface(isp); + return err; } + /* From here on the firmware runs, and may use what it was given. */ err = isp_firmware_boot_stage2(isp); if (err < 0) { isp_err(isp, "failed firmware boot stage 2: %d\n", err); - goto shutdown_stage1; + goto stop; } err = isp_firmware_boot_stage3(isp); if (err < 0) { isp_err(isp, "failed firmware boot stage 3: %d\n", err); - goto shutdown_stage2; + goto stop; } err = isp_enable_irq(isp); if (err < 0) { isp_err(isp, "failed to enable interrupts: %d\n", err); - goto shutdown_stage3; + goto stop; } err = isp_start_command_processor(isp); @@ -906,14 +925,8 @@ static int isp_firmware_boot(struct apple_isp *isp) disable_irqs: isp_disable_irq(isp); -shutdown_stage3: - isp_firmware_shutdown_stage3(isp); -shutdown_stage2: - isp_firmware_shutdown_stage2(isp); -shutdown_stage1: - isp_firmware_shutdown_stage1(isp); -garbage_collect: - isp_collect_gc_surface(isp); +stop: + isp_firmware_stop(isp); return err; } @@ -928,11 +941,7 @@ static void isp_firmware_shutdown(struct apple_isp *isp) if (isp_stop_command_processor(isp)) dev_warn(isp->dev, "firmware did not suspend, stopping it\n"); isp_disable_irq(isp); - isp_firmware_shutdown_stage1(isp); - - isp_firmware_shutdown_stage3(isp); - isp_firmware_shutdown_stage2(isp); - isp_collect_gc_surface(isp); + isp_firmware_stop(isp); } /* From 172aa5d1f38b1c80ab072f776053457d1305cab7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:32:41 +1100 Subject: [PATCH 0572/2680] media: apple: isp: Report a power domain that fails to go down as such apple_isp_power_down_domains() reports a failed runtime PM put of an ISP power domain as a failure to power it up, which points at the wrong transition when a domain stays on after the firmware has stopped. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-fw.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 65c760de616f1..46b4513b39df3 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -95,7 +95,7 @@ static void apple_isp_power_down_domains(struct apple_isp *isp) ret = pm_runtime_put_sync(isp->pd_dev[i]); if (ret < 0) dev_err(isp->dev, - "Failed to power up power domain %d: %d\n", i, ret); + "Failed to power down power domain %d: %d\n", i, ret); } isp->pds_active = false; From a31155a33914c0ecea575b565676c2aad805059c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 14:03:58 +1100 Subject: [PATCH 0573/2680] media: apple: isp: Keep the driver state until the last file is closed The driver state, which embeds the video device, its queue and lock, the control handler and the media device, is allocated with devm_kzalloc() and freed as soon as the driver is unbound. A process that still has the video device open at unbind keeps a reference to the video device, and when it closes the file, vb2_fop_release() takes the queue lock and v4l2_fh_release() walks the file handle lists, all in freed memory. A control event subscription also reaches the freed control handler. Tie the state to the v4l2_device reference count as V4L2 drivers do: allocate it with kzalloc(), free it, the control handler and the media device from the v4l2_device release callback, and drop the driver's own reference at the end of remove. The video device holds the other one while it is registered or open, so the state lives until the last file is closed. Before the video device is registered nothing else can reach the state, so a failed probe still frees it directly. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.c | 17 ++++++++++---- drivers/media/platform/apple/isp/isp-v4l2.c | 25 +++++++++++++++++++-- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 9f5f8f58b3748..71ba9c7939470 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -380,7 +380,11 @@ static int apple_isp_probe(struct platform_device *pdev) if (err) return err; - isp = devm_kzalloc(dev, sizeof(*isp), GFP_KERNEL); + /* + * Not devres: an open video device file keeps this until it is + * closed, also after unbind. See apple_isp_release(). + */ + isp = kzalloc_obj(*isp); if (!isp) return -ENOMEM; @@ -399,7 +403,7 @@ static int apple_isp_probe(struct platform_device *pdev) if (err) { dev_err(dev, "failed to get 'apple,platform-id' property: %d\n", err); - return err; + goto free_isp; } err = of_property_read_u32(dev->of_node, "apple,temporal-filter", @@ -410,13 +414,13 @@ static int apple_isp_probe(struct platform_device *pdev) err = apple_isp_init_presets(isp); if (err) { dev_err(dev, "failed to initialize presets\n"); - return err; + goto free_isp; } err = apple_isp_attach_genpd(isp); if (err) { dev_err(dev, "failed to attatch power domains\n"); - return err; + goto free_isp; } isp->coproc = devm_platform_ioremap_resource_byname(pdev, "coproc"); @@ -529,6 +533,9 @@ static int apple_isp_probe(struct platform_device *pdev) destroy_workqueue(isp->wq); detach_genpd: apple_isp_detach_genpd(isp); +free_isp: + /* Nothing else can reach it before the video device is registered. */ + kfree(isp); return err; } @@ -545,6 +552,8 @@ static void apple_isp_remove(struct platform_device *pdev) apple_isp_free_iommu(isp); destroy_workqueue(isp->wq); apple_isp_detach_genpd(isp); + /* Freed now, or when the last open file is closed. */ + v4l2_device_put(&isp->v4l2_dev); } static const struct apple_isp_hw apple_isp_hw_t8103 = { diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 0465834ebe622..451afc6dba45d 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -969,6 +969,22 @@ static const struct media_device_ops isp_media_device_ops = { .link_notify = v4l2_pipeline_link_notify, }; +/* + * The video device's file operations can run until its last open file is + * closed, after the driver has been unbound, so the state they reach is freed + * only with the last reference to the v4l2_device: the one remove drops, or + * the one the video device holds while it is open. + */ +static void apple_isp_release(struct v4l2_device *v4l2_dev) +{ + struct apple_isp *isp = container_of(v4l2_dev, struct apple_isp, + v4l2_dev); + + v4l2_ctrl_handler_free(&isp->ctrl_handler); + media_device_cleanup(&isp->mdev); + kfree(isp); +} + static void isp_free_meta_surfaces(struct apple_isp *isp) { for (int i = 0; i < ARRAY_SIZE(isp->capmeta_surfs); i++) { @@ -1094,6 +1110,9 @@ int apple_isp_setup_video(struct apple_isp *isp) goto v4l2_unregister; } + /* From here on, the last v4l2_device reference frees the state. */ + isp->v4l2_dev.release = apple_isp_release; + return 0; v4l2_unregister: @@ -1109,13 +1128,15 @@ int apple_isp_setup_video(struct apple_isp *isp) return err; } +/* + * Unregister the device nodes. An open file keeps the video device, and + * with it the driver state, until it is closed: see apple_isp_release(). + */ void apple_isp_remove_video(struct apple_isp *isp) { vb2_video_unregister_device(&isp->vdev); v4l2_device_unregister(&isp->v4l2_dev); - v4l2_ctrl_handler_free(&isp->ctrl_handler); media_device_unregister(&isp->mdev); - media_device_cleanup(&isp->mdev); } /* After apple_isp_remove_video() and with the firmware stopped */ From c8f3856a44ca3abf1b2e8547acb58e4f7a69ba93 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 15:52:10 +0000 Subject: [PATCH 0574/2680] media: apple: isp: Parse the firmware version for the H17 interface The H17 command layouts come from the match data, not from the legacy firmware version table, so a T8140 version property does not need to match an entry of that table. Report it as the H17 interface instead of warning about an unknown version, and pass the firmware ABI to isp_read_fw_version() for that. of_property_read_variable_u32_array() silently drops trailing bytes that do not fill a cell. Reject a version property whose length is not a multiple of a cell instead of parsing a truncated value. [ryan: split from merge commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate") for the 7.2.10 topics: its isp-drv.c content only. The merge carried this as the "ISP metadata parsing corrections"; neither parent has it as a commit of its own] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.c | 39 ++++++++++++++++++---- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 71ba9c7939470..76c28c0dc9f7c 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -309,12 +309,39 @@ static const char * isp_fw2str(enum isp_firmware_version version) #define ISP_FW_VERSION_MAX_LEN 5 static enum isp_firmware_version isp_read_fw_version(struct device *dev, - const char *name) + const char *name, + enum isp_fw_abi abi) { u32 ver[ISP_FW_VERSION_MAX_LEN]; - int len = of_property_read_variable_u32_array(dev->of_node, name, ver, - ISP_FW_VERSION_MIN_LEN, - ISP_FW_VERSION_MAX_LEN); + int size, len; + + /* The array reader truncates trailing bytes; require complete cells. */ + if (of_find_property(dev->of_node, name, &size) && size % sizeof(u32)) { + dev_warn(dev, "could not parse %s: invalid length %d\n", name, size); + return ISP_FIRMWARE_V_UNKNOWN; + } + + len = of_property_read_variable_u32_array(dev->of_node, name, ver, + ISP_FW_VERSION_MIN_LEN, + ISP_FW_VERSION_MAX_LEN); + + /* H17 command layouts come from match data, not the legacy version table. */ + if (abi == ISP_FW_ABI_H17) { + switch (len) { + case 3: + dev_info(dev, "%s: %u.%u.%u (H17 interface)\n", + name, ver[0], ver[1], ver[2]); + return ISP_FIRMWARE_V_UNKNOWN; + case 4: + dev_info(dev, "%s: %u.%u.%u.%u (H17 interface)\n", + name, ver[0], ver[1], ver[2], ver[3]); + return ISP_FIRMWARE_V_UNKNOWN; + case 5: + dev_info(dev, "%s: %u.%u.%u.%u.%u (H17 interface)\n", + name, ver[0], ver[1], ver[2], ver[3], ver[4]); + return ISP_FIRMWARE_V_UNKNOWN; + } + } switch (len) { case -EINVAL: @@ -355,8 +382,8 @@ isp_check_firmware_version(struct apple_isp *isp) enum isp_firmware_version version, compat; /* firmware version is just informative */ - version = isp_read_fw_version(dev, "apple,firmware-version"); - compat = isp_read_fw_version(dev, "apple,firmware-compat"); + version = isp_read_fw_version(dev, "apple,firmware-version", isp->hw->fw_abi); + compat = isp_read_fw_version(dev, "apple,firmware-compat", isp->hw->fw_abi); dev_dbg(dev, "ISP firmware-compat: %s (FW: %s)\n", isp_fw2str(compat), isp_fw2str(version)); From d92fa66a68802dccafec7dee68cc6c7eea53ff0d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:15:16 +1100 Subject: [PATCH 0575/2680] media: apple: isp: Do not report read() buffers in the stream parameters The driver fills readbuffers in G_PARM and S_PARM with the minimum number of queued buffers. That field is the number of buffers used for read(), and the device does not support read(): it does not set V4L2_CAP_READWRITE and its queue only allows MMAP. A non-zero value there makes v4l2-compliance fail the G_PARM and S_PARM tests. G_PARM is cleared by the V4L2 core before the driver fills it, so it only has to leave the field alone. S_PARM passes in what the caller wrote, so set it to zero there. Fixes: 329bf83de183 ("media: apple: Add Apple ISP driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-v4l2.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 451afc6dba45d..e649981398055 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -885,7 +885,6 @@ static int isp_vidioc_get_param(struct file *file, void *fh, return -EINVAL; a->parm.capture.capability = V4L2_CAP_TIMEPERFRAME; - a->parm.capture.readbuffers = ISP_MIN_FRAMES; a->parm.capture.timeperframe.numerator = 1; a->parm.capture.timeperframe.denominator = isp->frame_rate; @@ -914,7 +913,7 @@ static int isp_vidioc_set_param(struct file *file, void *fh, isp->frame_rate = isp_closest_frame_rate(tpf); a->parm.capture.capability = V4L2_CAP_TIMEPERFRAME; - a->parm.capture.readbuffers = ISP_MIN_FRAMES; + a->parm.capture.readbuffers = 0; tpf->numerator = 1; tpf->denominator = isp->frame_rate; From 5e5d70c09316dc6108b3bc0da6858f8c4082fa09 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:15:34 +1100 Subject: [PATCH 0576/2680] media: apple: isp: Refuse format changes while buffers are allocated S_FMT changes the format of the current channel and the type of the vb2 queue at any time, also while buffers are allocated or the stream runs, for example from a second open file. The running stream keeps the preset it was configured with, but buffer preparation and submission then use the new frame size. With the multiplanar API enabled, an S_FMT for the other buffer type also changes the queue type under the allocated buffers. Return -EBUSY from S_FMT while the queue has buffers, as V4L2 specifies and as uvcvideo does. TRY_FMT and G_FMT are unchanged. Fixes: 329bf83de183 ("media: apple: Add Apple ISP driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-v4l2.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index e649981398055..dfd836f8df2fc 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -757,6 +757,10 @@ static int isp_vidioc_set_format(struct file *file, void *fh, struct isp_preset *preset; int err; + /* The buffers and the queue type depend on the format. */ + if (vb2_is_busy(&isp->vbq)) + return -EBUSY; + preset = isp_select_preset(isp, f->fmt.pix.width, f->fmt.pix.height); err = isp_set_preset(isp, fmt, preset); if (err) @@ -812,6 +816,9 @@ static int isp_vidioc_set_format_mplane(struct file *file, void *fh, if (!isp->multiplanar) return -ENOTTY; + if (vb2_is_busy(&isp->vbq)) + return -EBUSY; + preset = isp_select_preset(isp, f->fmt.pix_mp.width, f->fmt.pix_mp.height); err = isp_set_preset(isp, fmt, preset); From 1a392f81ff2aa41e33dfe980058c53789a128ea8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 17:41:51 +1100 Subject: [PATCH 0577/2680] media: apple: isp: Count existing buffers against the buffer pool size The firmware's buffer pools are configured for ISP_MAX_BUFFERS buffers, and queue_setup() caps the number of buffers to allocate to that many. The cap only looks at the current request. REQBUFS frees the old buffers before it calls queue_setup(), but CREATE_BUFS adds to them: REQBUFS for 16 buffers followed by CREATE_BUFS gives the queue more buffers than the pool holds, and once they are all queued the driver hands them all to the firmware. Count the buffers the queue already has. CREATE_BUFS on a full queue now fails with -ENOBUFS, and one that asks for more than the remaining space gets what fits. REQBUFS behaves as before. vb2_queue.max_num_buffers cannot express the limit: vb2_queue_init() refuses any value below VB2_MAX_FRAME (32). Fixes: 6f0201787b3a ("media: apple: isp: Limit maximal number of buffers") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-v4l2.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index dfd836f8df2fc..69ddb1f229991 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -292,13 +292,20 @@ static int isp_vb2_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, { struct apple_isp *isp = vb2_get_drv_priv(vq); struct isp_format *fmt = isp_get_current_format(isp); + unsigned int q_num_bufs = vb2_get_num_buffers(vq); /* This is not strictly neccessary but makes it easy to enforce that * at most 16 buffers are submitted at once. ISP on t6001 (FW 12.3) * times out if more buffers are submitted than set in the buffer pool * config before streaming is started. + * + * REQBUFS frees the old buffers first, CREATE_BUFS adds to them, so + * count the buffers the queue already has. */ - *nbuffers = min_t(unsigned int, *nbuffers, ISP_MAX_BUFFERS); + if (q_num_bufs >= ISP_MAX_BUFFERS) + return -ENOBUFS; + *nbuffers = min_t(unsigned int, *nbuffers, + ISP_MAX_BUFFERS - q_num_bufs); if (*num_planes) { if (sizes[0] < fmt->total_size) From bc87a6d57c0322322b3a5b4a64e88b1af90efab5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:16:11 +1100 Subject: [PATCH 0578/2680] media: apple: isp: Use V4L2_CAP_VIDEO_CAPTURE in the device capabilities device_caps is built from V4L2_BUF_TYPE_VIDEO_CAPTURE, a buffer type, instead of the V4L2_CAP_VIDEO_CAPTURE capability flag. Both are 1, so the reported capabilities are right, but only by coincidence. Use the capability flag. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-v4l2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 69ddb1f229991..54edc098fc0d4 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -1105,7 +1105,7 @@ int apple_isp_setup_video(struct apple_isp *isp) vdev->queue = vbq; vdev->fops = &isp_v4l2_fops; vdev->ioctl_ops = &isp_v4l2_ioctl_ops; - vdev->device_caps = V4L2_BUF_TYPE_VIDEO_CAPTURE | V4L2_CAP_STREAMING; + vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING; if (isp->multiplanar) vdev->device_caps |= V4L2_CAP_VIDEO_CAPTURE_MPLANE; vdev->entity.flags = MEDIA_ENT_FL_DEFAULT; From 4bdaf186554a382315597c4c16378d9f293e7677 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:16:39 +1100 Subject: [PATCH 0579/2680] media: apple: isp: Skip the firmware version check for the H17 interface On T8140 the command layouts come from the match data, and the firmware version properties are never compared against the version table. isp_read_fw_version() still parsed them, then returned "unknown" through a separate H17 branch that printed each property at info level, two lines on every boot that the other SoCs log only at debug level. Return from isp_check_firmware_version() before reading the properties on H17. isp_read_fw_version() loses its interface argument and the H17 branch. fw_compat stays ISP_FIRMWARE_V_UNKNOWN on T8140, as before; the other SoCs are unchanged. The bootloader-provided versions remain readable in the device tree. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.c | 33 ++++++---------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 76c28c0dc9f7c..d11ce3eff54ba 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -309,8 +309,7 @@ static const char * isp_fw2str(enum isp_firmware_version version) #define ISP_FW_VERSION_MAX_LEN 5 static enum isp_firmware_version isp_read_fw_version(struct device *dev, - const char *name, - enum isp_fw_abi abi) + const char *name) { u32 ver[ISP_FW_VERSION_MAX_LEN]; int size, len; @@ -325,24 +324,6 @@ static enum isp_firmware_version isp_read_fw_version(struct device *dev, ISP_FW_VERSION_MIN_LEN, ISP_FW_VERSION_MAX_LEN); - /* H17 command layouts come from match data, not the legacy version table. */ - if (abi == ISP_FW_ABI_H17) { - switch (len) { - case 3: - dev_info(dev, "%s: %u.%u.%u (H17 interface)\n", - name, ver[0], ver[1], ver[2]); - return ISP_FIRMWARE_V_UNKNOWN; - case 4: - dev_info(dev, "%s: %u.%u.%u.%u (H17 interface)\n", - name, ver[0], ver[1], ver[2], ver[3]); - return ISP_FIRMWARE_V_UNKNOWN; - case 5: - dev_info(dev, "%s: %u.%u.%u.%u.%u (H17 interface)\n", - name, ver[0], ver[1], ver[2], ver[3], ver[4]); - return ISP_FIRMWARE_V_UNKNOWN; - } - } - switch (len) { case -EINVAL: /* not provided, see isp_check_firmware_version() */ @@ -381,16 +362,18 @@ isp_check_firmware_version(struct apple_isp *isp) struct device *dev = isp->dev; enum isp_firmware_version version, compat; + /* The H17 interface is selected by match data, not by version. */ + if (isp->hw->fw_abi == ISP_FW_ABI_H17) + return ISP_FIRMWARE_V_UNKNOWN; + /* firmware version is just informative */ - version = isp_read_fw_version(dev, "apple,firmware-version", isp->hw->fw_abi); - compat = isp_read_fw_version(dev, "apple,firmware-compat", isp->hw->fw_abi); + version = isp_read_fw_version(dev, "apple,firmware-version"); + compat = isp_read_fw_version(dev, "apple,firmware-compat"); dev_dbg(dev, "ISP firmware-compat: %s (FW: %s)\n", isp_fw2str(compat), isp_fw2str(version)); - /* The H17 interface does not depend on it. */ - if (isp->hw->fw_abi == ISP_FW_ABI_LEGACY && - !of_property_present(dev->of_node, "apple,firmware-compat")) + if (!of_property_present(dev->of_node, "apple,firmware-compat")) dev_warn(dev, "firmware compatibility version not provided, assuming 12.x\n"); return compat; From 74568c088ab436b25383778549fbdd6f838107f8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:16:54 +1100 Subject: [PATCH 0580/2680] media: apple: isp: Name the H17 flag in boot stage 2 after the interface isp_firmware_boot_stage2() keeps "is this the H17 interface" in a variable called h16, after the H16 boot descriptor that interface takes. Everywhere else the same test is called h17. Rename it so that the condition reads the same in every function; the descriptor keeps its H16 name. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-fw.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 46b4513b39df3..5c3026502e0e5 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -502,8 +502,8 @@ static void isp_write_bootargs_h16(struct apple_isp *isp, void *virt, static int isp_firmware_boot_stage2(struct apple_isp *isp) { - bool h16 = isp->hw->fw_abi == ISP_FW_ABI_H17; - size_t args_size = h16 ? sizeof(struct isp_firmware_bootargs_h16) : + bool h17 = isp->hw->fw_abi == ISP_FW_ABI_H17; + size_t args_size = h17 ? sizeof(struct isp_firmware_bootargs_h16) : sizeof(struct isp_firmware_bootargs); size_t cmd_size = ISP_CMD_AREA_SIZE(isp_num_capmeta(isp)); dma_addr_t args_iova, cmd_iova; @@ -526,7 +526,7 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) num_ipc_chans); /* Only hand the H16 descriptor to firmware that asks for it. */ - if (h16 && !(desc_flags & ISP_GPIO2_H16_DESCRIPTOR)) { + if (h17 && !(desc_flags & ISP_GPIO2_H16_DESCRIPTOR)) { dev_err(isp->dev, "firmware did not request the H16 boot descriptor (0x%x)\n", desc_flags); @@ -562,7 +562,7 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) isp->cmd_iova = cmd_iova; isp->cmd_virt = cmd_virt; - if (h16) + if (h17) isp_write_bootargs_h16(isp, args_virt, args_offset); else isp_write_bootargs(isp, args_virt, args_iova); From 8912c479ebfeb7a3bbaac65864bdce00efbbdc29 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:22:56 +1100 Subject: [PATCH 0581/2680] dt-bindings: media: apple,t8140-isp: Add an example The binding has no example, so dt_binding_check never validates a node against it. Add one that follows the T8140 device tree, with a single sensor preset. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/media/apple,t8140-isp.yaml | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml b/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml index dea138b415b7a..59c6c91fb19d7 100644 --- a/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml +++ b/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml @@ -145,3 +145,45 @@ required: - sensor-presets additionalProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + isp@320000000 { + compatible = "apple,t8140-isp", "apple,isp"; + reg = <0x3 0x20000000 0x0 0x1801000>, + <0x3 0x22920000 0x0 0x1000>, + <0x3 0x22921000 0x0 0x100>, + <0x3 0x63b50000 0x0 0x8000>; + reg-names = "coproc", "mbox", "gpio", "wdt"; + interrupt-parent = <&aic>; + interrupts = ; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + power-domains = <&ps_isp_sys>, <&ps_isp_cpu>, + <&ps_isp_cpu_core0>, <&ps_isp_cpu_core1>, + <&ps_isp_fe>, <&ps_isp_dprx>, <&ps_isp_secure>, + <&ps_isp_sts0>, <&ps_isp_sts1>, <&ps_isp_pearl>, + <&ps_isp_be>, <&ps_isp_clr>, <&ps_isp_cvd>, + <&ps_isp_cvm>; + memory-region = <&isp_heap>; + memory-region-names = "heap"; + apple,dart-vm-size = <0x0 0xa0000000>; + apple,platform-id = <0x70>; + apple,temporal-filter = <1>; + + sensor-presets { + preset0 { + apple,config-index = <1>; + apple,input-size = <1920 1080>; + apple,output-size = <1920 1080>; + apple,crop = <0 0 1920 1080>; + }; + }; + }; + }; From ee75f0fca2d2b8897d8e6618fce5b38e73d90464 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 17:45:34 +1100 Subject: [PATCH 0582/2680] media: apple: isp: Add KUnit tests for the videobuf2 queue Move the vb2 queue set-up out of apple_isp_setup_video() into isp_vb2_queue_init() and test it, together with the buffer limit in queue_setup(), without hardware: - the queue initialises with the parameters the driver probes with, so a parameter that vb2_queue_init() rejects fails here instead of at probe, where it leaves the machine without a camera; - REQBUFS gets at most ISP_MAX_BUFFERS buffers; - CREATE_BUFS counts the buffers the queue already has, fails with -ENOBUFS on a full queue and still refuses planes smaller than a frame. The tests are built into isp-v4l2.c, as they call its static functions, and run with tools/testing/kunit/kunit.py run --arch=arm64 \ --kunitconfig= No functional change to the driver. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/Kconfig | 12 ++ .../media/platform/apple/isp/isp-v4l2-test.c | 122 ++++++++++++++++++ drivers/media/platform/apple/isp/isp-v4l2.c | 35 +++-- 3 files changed, 157 insertions(+), 12 deletions(-) create mode 100644 drivers/media/platform/apple/isp/isp-v4l2-test.c diff --git a/drivers/media/platform/apple/isp/Kconfig b/drivers/media/platform/apple/isp/Kconfig index 5d818e55448a4..cc2c59b51a441 100644 --- a/drivers/media/platform/apple/isp/Kconfig +++ b/drivers/media/platform/apple/isp/Kconfig @@ -17,3 +17,15 @@ config VIDEO_APPLE_ISP calibration file for the detected camera module, apple/isp__XX.dat, from the firmware search path; if it is missing, the stream runs without calibration. + +config VIDEO_APPLE_ISP_KUNIT_TEST + bool "KUnit tests for the Apple ISP driver" if !KUNIT_ALL_TESTS + depends on VIDEO_APPLE_ISP && KUNIT + depends on KUNIT=y || VIDEO_APPLE_ISP=m + default KUNIT_ALL_TESTS + help + Builds unit tests for the videobuf2 queue set-up of the Apple ISP + driver into the driver. They need no hardware. Only useful for + kernel developers running KUnit. + + If unsure, say N. diff --git a/drivers/media/platform/apple/isp/isp-v4l2-test.c b/drivers/media/platform/apple/isp/isp-v4l2-test.c new file mode 100644 index 0000000000000..895a6d1299c22 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-v4l2-test.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for the videobuf2 queue of the Apple ISP driver, built into + * isp-v4l2.c. They check the queue parameters vb2 accepts and the buffer + * limit queue_setup() enforces, without any hardware. + */ + +#include +#include +#include + +#define ISP_TEST_FRAME_SIZE 0x1000 + +static struct apple_isp *isp_test_alloc(struct kunit *test) +{ + struct apple_isp *isp; + + isp = kunit_kzalloc(test, sizeof(*isp), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, isp); + + mutex_init(&isp->video_lock); + isp_get_current_format(isp)->total_size = ISP_TEST_FRAME_SIZE; + + return isp; +} + +/* The queue must initialise with the parameters the driver probes with. */ +static void isp_test_queue_init(struct kunit *test) +{ + struct apple_isp *isp = isp_test_alloc(test); + + isp->dev = kunit_device_register(test, "apple-isp-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, isp->dev); + + KUNIT_ASSERT_EQ(test, isp_vb2_queue_init(isp), 0); + KUNIT_EXPECT_GE(test, isp->vbq.max_num_buffers, VB2_MAX_FRAME); + vb2_queue_release(&isp->vbq); +} + +/* Pretend the queue already has @num buffers, as vb2_get_num_buffers() sees. */ +static void isp_test_set_num_buffers(struct kunit *test, struct vb2_queue *q, + unsigned int num) +{ + q->max_num_buffers = VB2_MAX_FRAME; + q->bufs_bitmap = kunit_kzalloc(test, + BITS_TO_LONGS(VB2_MAX_FRAME) * + sizeof(unsigned long), + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, q->bufs_bitmap); + bitmap_set(q->bufs_bitmap, 0, num); +} + +static int isp_test_setup(struct kunit *test, unsigned int existing, + unsigned int *nbuffers, unsigned int *num_planes, + unsigned int sizes[]) +{ + struct apple_isp *isp = isp_test_alloc(test); + + isp->vbq.drv_priv = isp; + isp_test_set_num_buffers(test, &isp->vbq, existing); + + return isp_vb2_queue_setup(&isp->vbq, nbuffers, num_planes, sizes, + NULL); +} + +/* REQBUFS: an empty queue gets at most one pool of buffers. */ +static void isp_test_reqbufs_limit(struct kunit *test) +{ + unsigned int sizes[VB2_MAX_PLANES] = {}; + unsigned int nbuffers = VB2_MAX_FRAME, num_planes = 0; + + KUNIT_ASSERT_EQ(test, isp_test_setup(test, 0, &nbuffers, &num_planes, + sizes), 0); + KUNIT_EXPECT_EQ(test, nbuffers, ISP_MAX_BUFFERS); + KUNIT_EXPECT_EQ(test, num_planes, 1); + KUNIT_EXPECT_EQ(test, sizes[0], ISP_TEST_FRAME_SIZE); + + nbuffers = 4; + num_planes = 0; + KUNIT_ASSERT_EQ(test, isp_test_setup(test, 0, &nbuffers, &num_planes, + sizes), 0); + KUNIT_EXPECT_EQ(test, nbuffers, 4); +} + +/* CREATE_BUFS: the buffers the queue has count against the pool. */ +static void isp_test_create_bufs_limit(struct kunit *test) +{ + unsigned int sizes[VB2_MAX_PLANES] = { ISP_TEST_FRAME_SIZE }; + unsigned int nbuffers, num_planes; + + nbuffers = 10; + num_planes = 1; + KUNIT_ASSERT_EQ(test, isp_test_setup(test, 10, &nbuffers, &num_planes, + sizes), 0); + KUNIT_EXPECT_EQ(test, nbuffers, ISP_MAX_BUFFERS - 10); + + nbuffers = 1; + num_planes = 1; + KUNIT_EXPECT_EQ(test, isp_test_setup(test, ISP_MAX_BUFFERS, &nbuffers, + &num_planes, sizes), -ENOBUFS); + + /* A plane smaller than a frame is still refused. */ + nbuffers = 1; + num_planes = 1; + sizes[0] = ISP_TEST_FRAME_SIZE - 1; + KUNIT_EXPECT_EQ(test, isp_test_setup(test, 0, &nbuffers, &num_planes, + sizes), -EINVAL); +} + +static struct kunit_case isp_v4l2_test_cases[] = { + KUNIT_CASE(isp_test_queue_init), + KUNIT_CASE(isp_test_reqbufs_limit), + KUNIT_CASE(isp_test_create_bufs_limit), + {} +}; + +static struct kunit_suite isp_v4l2_test_suite = { + .name = "apple-isp-v4l2", + .test_cases = isp_v4l2_test_cases, +}; + +kunit_test_suite(isp_v4l2_test_suite); diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 54edc098fc0d4..23eab4bf498a9 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -1013,6 +1013,24 @@ static void isp_free_meta_surfaces(struct apple_isp *isp) } } +static int isp_vb2_queue_init(struct apple_isp *isp) +{ + struct vb2_queue *vbq = &isp->vbq; + + vbq->drv_priv = isp; + vbq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + vbq->io_modes = VB2_MMAP; + vbq->dev = isp->dev; + vbq->ops = &isp_vb2_ops; + vbq->mem_ops = &vb2_dma_sg_memops; + vbq->buf_struct_size = sizeof(struct isp_buffer); + vbq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; + vbq->min_queued_buffers = ISP_MIN_FRAMES; + vbq->lock = &isp->video_lock; + + return vb2_queue_init(vbq); +} + int apple_isp_setup_video(struct apple_isp *isp) { struct video_device *vdev = &isp->vdev; @@ -1085,18 +1103,7 @@ int apple_isp_setup_video(struct apple_isp *isp) goto ctrl_free; } - vbq->drv_priv = isp; - vbq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - vbq->io_modes = VB2_MMAP; - vbq->dev = isp->dev; - vbq->ops = &isp_vb2_ops; - vbq->mem_ops = &vb2_dma_sg_memops; - vbq->buf_struct_size = sizeof(struct isp_buffer); - vbq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; - vbq->min_queued_buffers = ISP_MIN_FRAMES; - vbq->lock = &isp->video_lock; - - err = vb2_queue_init(vbq); + err = isp_vb2_queue_init(isp); if (err) { dev_err(isp->dev, "failed to init vb2 queue: %d\n", err); goto v4l2_unregister; @@ -1157,3 +1164,7 @@ void apple_isp_free_video(struct apple_isp *isp) { isp_free_meta_surfaces(isp); } + +#if IS_ENABLED(CONFIG_VIDEO_APPLE_ISP_KUNIT_TEST) +#include "isp-v4l2-test.c" +#endif From 6d27adb5d66602606b84471141b72ede9e9eb291 Mon Sep 17 00:00:00 2001 From: Aaron Date: Sun, 13 Sep 2026 13:52:49 +0100 Subject: [PATCH 0583/2680] media: apple: avd: allow cacheable (non-coherent) MMAP buffers The capture buffers are allocated with vb2_dma_contig_memops on a device that is not marked dma-coherent, so both the kernel and the user space mapping are Normal-NC (pgprot_dmacoherent()). Reading decoded frames back through such a mapping is extremely slow on Apple silicon: every load is a serialised uncached DRAM transaction, and neither wider loads nor non-temporal loads help. Measured on an M1 Mac mini (T8103) with one 2304x1296 NV12 capture buffer (13.2 MB plane, 4.5 MB of it image data) mapped by user space: mapping memcpy NEON ld1x4 ldnp/stnp read-only u64 coherent (Normal-NC) 240 MB/s 250 MB/s 246 MB/s 124 MB/s non-coherent (cacheable) 17.9 GB/s 23.8 GB/s 23.9 GB/s 18.8 GB/s For a client that copies the frames to system memory (ffmpeg's v4l2request hwaccel without a DRM-PRIME consumer, as used by Frigate) that is ~18 ms of CPU time per 2304x1296 frame just for the copy, which made hardware decode cost 4-5x more CPU than software decode. Let user space opt into cacheable buffers with the standard videobuf2 mechanism: set allow_cache_hints on both queues so that V4L2_MEMORY_FLAG_NON_COHERENT (REQBUFS/CREATE_BUFS) and V4L2_BUF_FLAG_NO_CACHE_{INVALIDATE,CLEAN} (QBUF) are honoured, and advertise V4L2_BUF_CAP_SUPPORTS_MMAP_CACHE_HINTS. videobuf2 then allocates with dma_alloc_noncontiguous(), maps the buffers cacheable and does the cache maintenance in prepare/finish (clean before the hardware runs, invalidate at DQBUF), so the data is still correct without any assumption about the fabric's coherency. DMA_ATTR_NO_KERNEL_MAPPING has to go from the capture queue: dma_alloc_noncontiguous() rejects every attribute other than DMA_ATTR_ALLOC_SINGLE_PAGES (WARN_ON_ONCE in kernel/dma/mapping.c) and the allocation fails with -ENOMEM. The attribute was ineffective anyway: with an IOMMU and a blocking allocation iommu_dma_alloc() always goes through iommu_dma_alloc_remap(), which creates the vmap mapping regardless of the attribute. Clients that do not pass the flag are unaffected (same coherent allocation and mapping as before). With ffmpeg requesting V4L2_MEMORY_FLAG_NON_COHERENT for its capture buffers, the Frigate-like chain "-hwaccel v4l2request -i 2304x1296.h264 -vf fps=5,scale=640:360 -f rawvideo -pix_fmt yuv420p" for 150 frames goes from 2.97 s user + 0.09 s sys to 0.36-0.40 s user + 0.11-0.14 s sys (software decode: 0.53 s user single-threaded, 0.76 s with the default thread count), output bit-identical. Decode + read-back alone ("-f null -") goes from 2.75 s to 0.08 s user + 0.12 s sys, the remaining sys time being the cache maintenance over the 13.2 MB planes. Verified with the full clip matrix (gst v4l2slh264dec, coherent buffers, unchanged) and repeated per-frame md5 comparison against software decode with the non-coherent buffers. --- drivers/media/platform/apple/avd/avd-drv.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 1a376cccb5472..5f0e4a03b23aa 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -322,6 +322,7 @@ static int avd_queue_init(void *priv, struct vb2_queue *src_vq, src_vq->lock = &ctx->dev->vdev_lock; src_vq->dev = ctx->dev->v4l2_dev.dev; src_vq->supports_requests = true; + src_vq->allow_cache_hints = true; ret = vb2_queue_init(src_vq); if (ret) @@ -338,6 +339,7 @@ static int avd_queue_init(void *priv, struct vb2_queue *src_vq, dst_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; dst_vq->lock = &ctx->dev->vdev_lock; dst_vq->dev = ctx->dev->v4l2_dev.dev; + dst_vq->allow_cache_hints = true; return vb2_queue_init(dst_vq); } From 882691e3b599dac604983e1adf92814a8d71656c Mon Sep 17 00:00:00 2001 From: Aaron Date: Sun, 13 Sep 2026 13:52:36 +0100 Subject: [PATCH 0584/2680] media: apple: avd: h264: reject slices that reference invalid DPB entries stream_refs() pushes a reference header and its RVRA addresses only for the DPB entries flagged V4L2_H264_DPB_ENTRY_FLAG_VALID, and the slice commands index the references by DPB index. Nothing checks that the indices in ref_pic_list0/1 point at VALID entries, so a slice that references any other entry makes the VP dereference an uninitialised reference slot: apple-dart 269010000.iommu: translation fault: status:0x80000002 stream:0 code:0x2 (NO PMD FOR IOVA) at 0x8080800 avd 269080000.avd: Frame processing timed out! Vp: 255 (00) i.e. a DART fault, a frame that never completes and, two seconds later, a watchdog reset of the VP, which also disturbs the other contexts that are decoding at the time. This is what happens every time a client starts decoding on a non-IDR frame, e.g. ffmpeg's v4l2request hwaccel joining a live RTSP stream mid-GOP: the decoder has no reference pictures yet, the DPB control is empty, and the hwaccel leaves the index of a reference it cannot find at 0. Validate the ref lists of P and B slices before anything is pushed to the hardware and fail the request with -EINVAL, so the client gets the capture buffer back with V4L2_BUF_FLAG_ERROR just like for corrupt input and the hardware is never programmed with the bogus references. Other stateless drivers (hantro, rkvdec) handle missing references by substituting the destination buffer; here the safe choice is to reject the slice, since the reference slot table cannot be filled for an entry that has no buffer. [7.1.13 port: the check runs right after avd_run_postamble(), so the media request of a rejected slice is still completed (returning before it leaves the request pending and every later request on the context fails with EBUSY); the slice already copied is freed by avd_h264_done().] --- drivers/media/platform/apple/avd/avd-h264.c | 54 +++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index 7fe88c4483389..bf7be37996816 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -714,6 +714,54 @@ static int avd_h264_realloc_slices(struct avd_ctx *ctx) return 0; } +/* + * Check that every reference in the slice's ref lists points at a DPB entry + * that is VALID. stream_refs() only pushes a reference header (and its RVRA + * addresses) for VALID entries and the slice command indexes those by DPB + * index, so a reference to any other entry makes the VP dereference an + * uninitialised reference slot: that ends in a DART translation fault, a + * frame that never completes and a watchdog reset of the VP, which also + * disturbs the other contexts. It happens whenever a client starts decoding + * on a non-IDR frame (joining a live RTSP stream mid-GOP): ffmpeg's + * v4l2request hwaccel leaves the index of a missing reference at 0 and the + * DPB is empty. Refuse such slices before anything is pushed; the client gets + * the capture buffer back with V4L2_BUF_FLAG_ERROR, as for corrupt input. + */ +static bool avd_h264_refs_valid(const struct avd_h264_run *run) +{ + const struct v4l2_ctrl_h264_slice_params *sl = run->slice_params; + const struct v4l2_h264_dpb_entry *dpb = run->decode_params->dpb; + const struct v4l2_h264_reference *lists[2] = { + sl->ref_pic_list0, sl->ref_pic_list1 + }; + u32 num[2] = { + sl->num_ref_idx_l0_active_minus1 + 1, + sl->num_ref_idx_l1_active_minus1 + 1 + }; + int nlists; + + if (sl->slice_type == V4L2_H264_SLICE_TYPE_P) + nlists = 1; + else if (sl->slice_type == V4L2_H264_SLICE_TYPE_B) + nlists = 2; + else + return true; + + for (int l = 0; l < nlists; l++) { + if (num[l] > V4L2_H264_REF_LIST_LEN) + return false; + for (u32 i = 0; i < num[l]; i++) { + u8 idx = lists[l][i].index; + + if (idx >= V4L2_H264_NUM_DPB_ENTRIES || + !(dpb[idx].flags & V4L2_H264_DPB_ENTRY_FLAG_VALID)) + return false; + } + } + + return true; +} + static int avd_h264_run(struct avd_ctx *ctx) { struct avd_h264_ctx *h264_ctx = ctx->priv; @@ -753,6 +801,12 @@ static int avd_h264_run(struct avd_ctx *ctx) avd_run_postamble(ctx, &run.base); + if (!avd_h264_refs_valid(&run)) { + dev_dbg_ratelimited(ctx->dev->dev, + "slice references an invalid DPB entry, dropping frame\n"); + return -EINVAL; + } + if (is_new_frame(run.slice_params)) { ret = avd_init_job(ctx, AVD_CODEC_H264, h264_ctx->alloc_slice_num + 1); From 2549e97ac45b14d5b8381611b28c7728aa2d2b8c Mon Sep 17 00:00:00 2001 From: Aaron Date: Sun, 13 Sep 2026 11:43:04 +0100 Subject: [PATCH 0585/2680] media: apple: avd: allocate the job segment table with kvcalloc avd_init_job() allocates MAX_SLICES (4096 for H.264) struct avd_segment entries of 2056 bytes each - about 8.4 MB, an order-10 allocation on a 16K-page kernel - with kzalloc() for every frame. Once the page cache is full of file pages (an NVR writing recordings) that contiguous allocation fails, the frame is returned with an error and the output silently loses frames: with four concurrent 2304x1296 decodes on T6000 the page allocation failure was logged in avd_init_job() and all four outputs differed from the single-stream reference. The table is only read by the CPU (its instructions are written to MMIO with writel()), so it does not need physically contiguous memory. Use kvcalloc()/kvfree(), which also checks the multiplication for overflow. --- drivers/media/platform/apple/avd/avd-drv.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 5f0e4a03b23aa..4c7bfb97404f6 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -139,7 +139,7 @@ int avd_init_job(struct avd_ctx *ctx, enum avd_codec codec, size_t segments) job->codec = codec; job->num = 0; - job->segments = kzalloc(sizeof(*job->segments) * segments, GFP_KERNEL); + job->segments = kvcalloc(segments, sizeof(*job->segments), GFP_KERNEL); if (!job->segments) ret = -ENOMEM; return ret; @@ -187,7 +187,7 @@ int avd_submit_job(struct avd_ctx *ctx) reg); } - kfree(sub->segments); + kvfree(sub->segments); sub->segments = NULL; return 0; } From 52d1aef4137cb09477f93c6b8148cd9cf30f0a06 Mon Sep 17 00:00:00 2001 From: Aaron Date: Sun, 13 Sep 2026 12:54:30 +0100 Subject: [PATCH 0586/2680] media: apple: avd: h264: reset slice_num when a frame is done avd_h264_run() appends every slice to h264_ctx->slices[] and increments slice_num, and avd_h264_done() frees slices[0..slice_num) once the capture buffer is no longer held, but slice_num is only ever zeroed by the kzalloc() in avd_h264_start(). On a long-running stream it therefore reaches MAX_SLICES (4096) after 4096 slices - about four and a half minutes of a 15 fps single-slice camera - and every later frame is rejected with "slice_num > 4096, stream was rejected!". done() also walks an ever-growing range of already-freed entries on every frame. Reset slice_num after the slices are freed. Also complete the media request (avd_run_postamble) on the two early returns that precede it (MAX_SLICES, slice buffer allocation failure); otherwise the rejected request is never completed and every following request on the context fails with EBUSY. [ryan: adapted to 7.2.10 - asahi-wip's fixup b00e2172cbb5 ("fixup! media: apple: avd: add a dedicated "submit" step") already resets slice_num in avd_h264_done() and replaced the MAX_SLICES rejection with a growing slice table; what remains is completing the request on the error paths, now in its err_free_jobs exit] --- drivers/media/platform/apple/avd/avd-h264.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index bf7be37996816..23a213beb21c6 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -830,6 +830,7 @@ static int avd_h264_run(struct avd_ctx *ctx) err_free_jobs: kfree(ctx->job.segments); + avd_run_postamble(ctx, &run.base); return ret; } From da7e04a044b93c271fcac7e0b87b0548e8eb92fb Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:17:01 -0500 Subject: [PATCH 0587/2680] dt-bindings: npu: apple,ane: Add the Apple Neural Engine Add a binding for the Neural Engine (ANE) of the Apple T8103 (M1), T6000 (M1 Pro, Max and Ultra), T6020, T6021, T6022 (M2 Pro, Max and Ultra) and T8112 (M2). These are the compatibles that the ane and ane_t6021 drivers match. On T8103 and T6000 the node has the engine register window, one interrupt, the three ANE DARTs and five power domains. On T602x a coprocessor runs the ANE firmware, so the node also has the PMGR and power state windows, eight power domains, the coprocessor reset and mailbox, and an IOVA-only reservation for the firmware entry alias. The T8112 node has the T602x windows plus the two eFuse words that give the SoC revision to the firmware, seven power domains and the mailbox. The schema goes in npu/ with the other neural processing units, and in the ARM/APPLE MACHINE SUPPORT entry with the other Apple bindings. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- .../devicetree/bindings/npu/apple,ane.yaml | 198 ++++++++++++++++++ MAINTAINERS | 1 + 2 files changed, 199 insertions(+) create mode 100644 Documentation/devicetree/bindings/npu/apple,ane.yaml diff --git a/Documentation/devicetree/bindings/npu/apple,ane.yaml b/Documentation/devicetree/bindings/npu/apple,ane.yaml new file mode 100644 index 0000000000000..ebca460a81bc1 --- /dev/null +++ b/Documentation/devicetree/bindings/npu/apple,ane.yaml @@ -0,0 +1,198 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/npu/apple,ane.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple Neural Engine + +maintainers: + - Joshua Warren <816217+joshuaswarren@users.noreply.github.com> + +description: + The Apple Neural Engine (ANE) is the neural network accelerator of Apple + SoCs. It accesses memory through three DARTs. On T8103 and T6000 the host + drives the engine through its register window. On T602x and T8112 a + coprocessor runs the ANE firmware and the host talks to it through an ASC + mailbox. + +properties: + compatible: + enum: + - apple,t8103-ane + - apple,t6000-ane + - apple,t6020-ane + - apple,t6021-ane + - apple,t6022-ane + - apple,t8112-ane + + reg: + minItems: 1 + items: + - description: Engine registers. On T602x and T8112 this is the whole + ANE block, including the coprocessor. + - description: Power manager (PMGR) registers of the ANE + - description: ANE power state registers in the PMGR + - description: eFuse words that give the SoC revision to the ANE + firmware + + reg-names: + minItems: 1 + items: + - const: engine + - const: pmgr + - const: set + - const: fuse + + interrupts: + maxItems: 1 + + interrupt-names: + items: + - const: ane + + iommus: + maxItems: 3 + + power-domains: + description: + The power states the engine needs, five on T8103 and T6000, eight on + T602x and seven on T8112. + minItems: 5 + maxItems: 8 + + resets: + maxItems: 1 + description: Reset of the ANE coprocessor + + mboxes: + maxItems: 1 + description: Mailbox of the ANE coprocessor + + memory-region: + maxItems: 1 + description: + IOVA-only reservation of the firmware entry alias in the ANE DART + address space. + +required: + - compatible + - reg + - reg-names + - interrupts + - interrupt-names + - iommus + - power-domains + +allOf: + - if: + properties: + compatible: + contains: + enum: + - apple,t8103-ane + - apple,t6000-ane + then: + properties: + reg: + maxItems: 1 + reg-names: + maxItems: 1 + power-domains: + maxItems: 5 + resets: false + mboxes: false + memory-region: false + + - if: + properties: + compatible: + contains: + enum: + - apple,t6020-ane + - apple,t6021-ane + - apple,t6022-ane + then: + properties: + reg: + minItems: 3 + maxItems: 3 + reg-names: + minItems: 3 + maxItems: 3 + power-domains: + minItems: 8 + required: + - resets + - mboxes + - memory-region + + - if: + properties: + compatible: + contains: + const: apple,t8112-ane + then: + properties: + reg: + minItems: 4 + reg-names: + minItems: 4 + power-domains: + minItems: 7 + maxItems: 7 + resets: false + memory-region: false + required: + - mboxes + +additionalProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + ane@26bc04000 { + compatible = "apple,t8103-ane"; + reg = <0x2 0x6bc04000 0x0 0x24000>; + reg-names = "engine"; + interrupt-parent = <&aic>; + interrupts = ; + interrupt-names = "ane"; + iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; + power-domains = <&ps_ane_set1>, <&ps_ane_set2>, + <&ps_ane_set3>, <&ps_ane_set4>, <&ps_ane_set5>; + }; + }; + + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + ane@284000000 { + compatible = "apple,t6021-ane"; + reg = <0x2 0x84000000 0x0 0x2000000>, + <0x2 0x8e080000 0x0 0x4034>, + <0x2 0x8e08c000 0x0 0x4000>; + reg-names = "engine", "pmgr", "set"; + interrupt-parent = <&aic>; + interrupts = ; + interrupt-names = "ane"; + iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; + power-domains = <&ps_ane_sys_mpm>, <&ps_ane_td>, + <&ps_ane_base>, <&ps_ane_set1>, <&ps_ane_set2>, + <&ps_ane_set3>, <&ps_ane_set4>, <&ps_ane_cpu>; + resets = <&ps_ane_cpu>; + mboxes = <&ane_mbox>; + memory-region = <&ane_alias_iova>; + }; + }; diff --git a/MAINTAINERS b/MAINTAINERS index b60cd73cce49d..dce950f9b9ccd 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2606,6 +2606,7 @@ F: Documentation/devicetree/bindings/leds/backlight/apple,dwi-bl.yaml F: Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml F: Documentation/devicetree/bindings/mfd/apple,smc.yaml F: Documentation/devicetree/bindings/net/bluetooth/brcm,bcm4377-bluetooth.yaml +F: Documentation/devicetree/bindings/npu/apple,ane.yaml F: Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml F: Documentation/devicetree/bindings/nvmem/apple,efuses.yaml F: Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml From d139b54c353435244d2de99300c444894cbb3206 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:22:08 -0500 Subject: [PATCH 0588/2680] accel/ane: Add the Apple Neural Engine driver Add a DRM accel driver for the Apple Neural Engine (ANE). One Kconfig symbol, DRM_ACCEL_ANE, builds two modules from this directory: - ane binds apple,t8103-ane and apple,t6000-ane (M1, and die 0 of the M1 Pro, Max and Ultra). Linux starts no ANE firmware on these SoCs: the host writes task descriptors to the engine's task queue and maps the buffers through the three ANE DARTs (ABI 1, DRM_IOCTL_ANE_SUBMIT). - ane_t6021 binds apple,t6020-ane, apple,t6021-ane, apple,t6022-ane and apple,t8112-ane (M2 Pro, Max and Ultra die 0, and M2). On these SoCs Linux must start the ANE coprocessor. The driver loads the firmware with request_firmware(), checks it against a SHA-256 pin, copies it to its own DMA memory, starts the coprocessor, and sends LOAD, CREATE and CALL commands on the firmware's command ring (ABI 2, DRM_IOCTL_ANE_PROG_LOAD, DRM_IOCTL_ANE_PROC_CREATE and DRM_IOCTL_ANE_EXEC). Both register /dev/accel/accel*. include/uapi/drm/ane_accel.h has the ioctls of both ABIs, and the DRM major version tells userspace which ABI the device has. The sources are those of the out-of-tree omarchy-ane driver at tag v0.4.0 (16d13ef52dc6), which started from the ane driver of Eileen Yoon. The move changes only the build glue: the SPDX comment style, the include path of the UAPI header, an include of drm.h and the Linux-syscall-note in the UAPI header, and the module version, which the out-of-tree build took from git describe (MODULE_VERSION and one log line). Compiled for arm64, every object has the same code as the out-of-tree build, except for the argument list of that log line. The out-of-tree build of this code ran on T8103 (M1), T6001 (M1 Max) and T6021 (M2 Max) with linux-asahi 7.1.13. T6000, T6002, T6020, T6022 and T8112 have not run it. ane also matches apple,t6020-ane and apple,t6021-ane and refuses them at probe; ane_t6021 binds them. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- MAINTAINERS | 9 + drivers/accel/Kconfig | 1 + drivers/accel/Makefile | 1 + drivers/accel/ane/Kconfig | 23 + drivers/accel/ane/Makefile | 14 + drivers/accel/ane/ane.h | 160 ++ drivers/accel/ane/ane_boost.c | 145 ++ drivers/accel/ane/ane_dart.c | 271 +++ drivers/accel/ane/ane_drv.c | 1234 ++++++++++ drivers/accel/ane/ane_fw_validate.h | 480 ++++ drivers/accel/ane/ane_t6021.h | 796 +++++++ drivers/accel/ane/ane_t6021_boot.c | 557 +++++ drivers/accel/ane/ane_t6021_boot.h | 832 +++++++ drivers/accel/ane/ane_t6021_fwload.c | 645 +++++ drivers/accel/ane/ane_t6021_rtclient_main.c | 2363 +++++++++++++++++++ drivers/accel/ane/ane_tm.c | 652 +++++ drivers/accel/ane/ane_tm.h | 20 + include/uapi/drm/ane_accel.h | 130 + 18 files changed, 8333 insertions(+) create mode 100644 drivers/accel/ane/Kconfig create mode 100644 drivers/accel/ane/Makefile create mode 100644 drivers/accel/ane/ane.h create mode 100644 drivers/accel/ane/ane_boost.c create mode 100644 drivers/accel/ane/ane_dart.c create mode 100644 drivers/accel/ane/ane_drv.c create mode 100644 drivers/accel/ane/ane_fw_validate.h create mode 100644 drivers/accel/ane/ane_t6021.h create mode 100644 drivers/accel/ane/ane_t6021_boot.c create mode 100644 drivers/accel/ane/ane_t6021_boot.h create mode 100644 drivers/accel/ane/ane_t6021_fwload.c create mode 100644 drivers/accel/ane/ane_t6021_rtclient_main.c create mode 100644 drivers/accel/ane/ane_tm.c create mode 100644 drivers/accel/ane/ane_tm.h create mode 100644 include/uapi/drm/ane_accel.h diff --git a/MAINTAINERS b/MAINTAINERS index dce950f9b9ccd..f478d6dc548ed 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7945,6 +7945,15 @@ S: Maintained F: drivers/soc/ti/smartreflex.c F: include/linux/power/smartreflex.h +DRM ACCEL DRIVER FOR APPLE NEURAL ENGINE +M: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> +L: asahi@lists.linux.dev +L: dri-devel@lists.freedesktop.org +S: Maintained +F: Documentation/devicetree/bindings/npu/apple,ane.yaml +F: drivers/accel/ane/ +F: include/uapi/drm/ane_accel.h + DRM ACCEL DRIVERS FOR INTEL VPU M: Karol Wachowski M: Andrzej Kacprowski diff --git a/drivers/accel/Kconfig b/drivers/accel/Kconfig index bdf48ccafcf21..9e641c2c30ae1 100644 --- a/drivers/accel/Kconfig +++ b/drivers/accel/Kconfig @@ -25,6 +25,7 @@ menuconfig DRM_ACCEL and debugfs). source "drivers/accel/amdxdna/Kconfig" +source "drivers/accel/ane/Kconfig" source "drivers/accel/ethosu/Kconfig" source "drivers/accel/habanalabs/Kconfig" source "drivers/accel/ivpu/Kconfig" diff --git a/drivers/accel/Makefile b/drivers/accel/Makefile index 1d3a7251b950f..235ebf723025c 100644 --- a/drivers/accel/Makefile +++ b/drivers/accel/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_DRM_ACCEL_AMDXDNA) += amdxdna/ +obj-$(CONFIG_DRM_ACCEL_ANE) += ane/ obj-$(CONFIG_DRM_ACCEL_ARM_ETHOSU) += ethosu/ obj-$(CONFIG_DRM_ACCEL_HABANALABS) += habanalabs/ obj-$(CONFIG_DRM_ACCEL_IVPU) += ivpu/ diff --git a/drivers/accel/ane/Kconfig b/drivers/accel/ane/Kconfig new file mode 100644 index 0000000000000..a1e8766340621 --- /dev/null +++ b/drivers/accel/ane/Kconfig @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config DRM_ACCEL_ANE + tristate "Apple Neural Engine (ANE)" + depends on DRM_ACCEL + depends on ARCH_APPLE || COMPILE_TEST + depends on APPLE_RTKIT + depends on APPLE_DART || COMPILE_TEST + depends on MMU + select CRYPTO_LIB_SHA256 + help + Choose this option if you have an Apple SoC with a Neural Engine: + M1 (T8103), M1 Pro, Max and Ultra (T6000, T6001, T6002), M2 + (T8112), or M2 Pro, Max and Ultra (T6020, T6021, T6022). + + The interface exposed to userspace is described in + include/uapi/drm/ane_accel.h. + + If unsure, say N. + + To compile this driver as modules, choose M here: the module for + the M1 family will be called ane, and the module for the M2 family + will be called ane_t6021. diff --git a/drivers/accel/ane/Makefile b/drivers/accel/ane/Makefile new file mode 100644 index 0000000000000..c0ad4a43bc0c3 --- /dev/null +++ b/drivers/accel/ane/Makefile @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_DRM_ACCEL_ANE) += ane.o ane_t6021.o + +ane-y := \ + ane_drv.o \ + ane_tm.o \ + ane_dart.o \ + ane_boost.o + +ane_t6021-y := \ + ane_t6021_rtclient_main.o \ + ane_t6021_fwload.o \ + ane_t6021_boot.o diff --git a/drivers/accel/ane/ane.h b/drivers/accel/ane/ane.h new file mode 100644 index 0000000000000..10fee2864728c --- /dev/null +++ b/drivers/accel/ane/ane.h @@ -0,0 +1,160 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2022 Eileen Yoon */ + +#ifndef __ANE_H__ +#define __ANE_H__ + +#include +#include +#include +#include +#include + +#include +#include + +#include + +struct ane_device { + struct drm_device drm; + struct device *dev; + + struct device **pd_dev; + struct device_link **pd_link; + int pd_count; + + void __iomem *engine; + + /* The ane SET block (m1n1 ANE.ps_map): pmgr power-state words for + * this engine, mapped at probe on every SoC. Read-only, always: + * pmgr reads give recovery its ACTUAL evidence, while direct SET + * writes are firmware-locked and external-abort the SoC (T6001 + * netconsole-named 2026-09-16; T8103 same mechanism). + */ + phys_addr_t ps_base; + void __iomem *ps; + + struct drm_mm mm; + struct iommu_domain *domain; + unsigned long shift; + struct ane_dart { + void __iomem *regs; + int irq; + u32 sid; + bool masked; + } darts[3]; + int dart_count; + + int irq; + struct mutex iommu_lock; + struct mutex engine_lock; + struct list_head bo_list; + + /* + * Wedge-preserved BO ranges, tied to their still-inserted drm_mm + * nodes (see ane_reclaim_preserved). Registered under engine_lock. + */ + struct list_head preserved_list; + bool removed; + + /* + * Set under engine_lock once a task manager timeout leaves the queue + * state unknown and DMA possibly still active. A wedged engine refuses + * new work while recovery (ane_tm_recover) power-cycles it; the + * preserve-until-reboot behaviour applies only when that reset fails. + */ + atomic_t wedged; + + /* + * Set under engine_lock for the duration of a recovery power cycle, + * so runtime suspend may gate an engine that the reset itself is + * quiescing. + */ + bool recovering; + + /* + * TM_STATUS sampled right after the first tm_enable at probe: the + * engine's fresh, idle-state signature. Recovery accepts a reset + * whose post-cycle status matches it, since the register reset + * value is not documented. + */ + u32 tm_status_fresh; + bool tm_status_known; + + /* + * SoC descriptor fact: the set0/base islands hold the tm/tq + * register file in retention through any genpd power cycle + * (T6001 bisect evidence 2026-09-16: TQ_EN still reads 0x3000 + * after a 2 s gate), so recovery must drain retained task-manager + * state itself instead of expecting a power-on reset to clear it. + * False on T8103, where the cycle is a full POR of the file. + */ + bool tm_retention; + + /* + * Engine-busy CPU cluster boost (ane_boost.c): min-frequency QoS on + * every cpufreq policy while a submit runs and until boost_idle_ms + * after the last one completes: on T8103 bandwidth-bound programs + * run ~2x slower while schedutil parks the idle clusters low. + */ + struct ane_boost { + struct mutex lock; + struct delayed_work off; + struct freq_qos_request *legs; + int nlegs; + int held; + unsigned long last_end; + bool busy; + bool on; + } boost; +}; + +struct ane_request { + int qid; + u32 nid; + u32 td_size; + u32 td_count; + u32 btsp_iova; + u32 bar[ANE_TILE_COUNT]; +}; + +/* + * Drop the wedge pin (module refcount) without requiring a successful + * recovery. Used by ane_drm_postclose so a wedged engine cannot leave + * the module un-unloadable after a clean session close. Recovery stays + * safe to call afterwards; if it returns error, the postclose path has + * already unlocked the module count so the operator can swap the ko. + */ +void ane_wedge_clear(struct ane_device *ane); + +/* + * Reclaim wedge-preserved BO ranges (unmap, release nodes and pages). + * Only legal once DMA is provably quiescent: after a successful + * recovery power cycle, or at driver remove. Callers hold engine_lock. + */ +void ane_reclaim_preserved(struct ane_device *ane); + +int ane_boost_init(struct ane_device *ane); +void ane_boost_exit(struct ane_device *ane); +void ane_boost_begin(struct ane_device *ane); +void ane_boost_end(struct ane_device *ane); + +#define ANE_DART_MAX 3 +#define ANE_DART_SCRATCH_MAX 16 + +struct ane_dart_scratch { + u64 iova; + struct page *page; + struct drm_mm_node *node; +}; + +int ane_dart_init(struct ane_device *ane); +void ane_dart_mask(struct ane_device *ane); +void ane_dart_unmask(struct ane_device *ane); +bool ane_dart_faulted(struct ane_device *ane, u64 *iova, u32 *status); +int ane_dart_drain_fault(struct ane_device *ane, u64 fault_iova, + struct ane_dart_scratch *scratch, int *pages); +void ane_dart_release_scratch(struct ane_device *ane, + struct ane_dart_scratch *scratch, int pages); + +#endif /* __ANE_H__ */ diff --git a/drivers/accel/ane/ane_boost.c b/drivers/accel/ane/ane_boost.c new file mode 100644 index 0000000000000..a13d706f4bc3c --- /dev/null +++ b/drivers/accel/ane/ane_boost.c @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Engine-busy CPU cluster boost. + * + * Bandwidth-bound programs run at a speed set by the CPU cluster + * p-states. Measured on jwm1 (T8103, 2026-09-25, ane-linux-experiments + * receipt jwm1-ane-dvfs-boost): with a mostly-sleeping submitter at + * normal priority, 75 of 489 Qwen decode steps take 100-175 ms instead + * of 70, every program in a slow step ~2x its normal wall, the + * submitter's CPU time and run delay unchanged; both clusters pinned + * at their top p-state (performance governor, or SCHED_FIFO for the + * submitter, which schedutil boosts the same way) -> 0 of 489. The + * whole-encoder Parakeet program (140 ms) moves too once the QoS lapses + * mid-submit: timed from submit start, the hold dropped 100 ms into + * every encoder run, median 171 ms (140-238); held to completion, 138.0 + * (137.7-138.7), jwm1 2026-09-25. The likely + * mechanism is the second p-state field apple-soc-cpufreq writes with + * the cluster p-state on T8103 only (DVFS_CMD PS2, has_ps2); the + * memory-controller tuning its author named as unimplemented is still + * absent from the tree. Whatever the field drives, the cure is the + * cluster p-state, which cpufreq lets a driver request. + * + * So hold a min-frequency QoS at the top of every cpufreq policy while a + * submit runs, and drop it boost_idle_ms after the last one completes. + * The QoS goes through cpufreq's own constraint object, so it composes + * with any governor and never touches the DVFS registers. + */ +#include +#include +#include +#include +#include +#include + +#include "ane.h" + +static unsigned int boost_idle_ms = 100; +module_param(boost_idle_ms, uint, 0644); +MODULE_PARM_DESC(boost_idle_ms, + "hold every CPU cluster at its top p-state while a submit runs and this long after the last one completes (0 = off)"); + +/* Lock held. */ +static void ane_boost_set(struct ane_device *ane, bool on) +{ + struct ane_boost *b = &ane->boost; + int cpu, n = 0; + + if (on) { + for_each_possible_cpu(cpu) { + struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); + int err; + + if (!policy) + continue; + if (cpu != cpumask_first(policy->related_cpus) || + n >= b->nlegs) { + cpufreq_cpu_put(policy); + continue; + } + err = freq_qos_add_request(&policy->constraints, + &b->legs[n], FREQ_QOS_MIN, + policy->cpuinfo.max_freq); + cpufreq_cpu_put(policy); + if (err >= 0) + n++; + } + b->held = n; + } else { + for (n = 0; n < b->held; n++) + freq_qos_remove_request(&b->legs[n]); + b->held = 0; + } + b->on = on; +} + +static void ane_boost_off_work(struct work_struct *work) +{ + struct ane_device *ane = + container_of(work, struct ane_device, boost.off.work); + struct ane_boost *b = &ane->boost; + + mutex_lock(&b->lock); + /* A submit that began or ended after this expiry was armed keeps + * the boost on; its own completion re-arms the drop. + */ + if (b->on && !b->busy && + time_after_eq(jiffies, b->last_end + msecs_to_jiffies(boost_idle_ms))) + ane_boost_set(ane, false); + mutex_unlock(&b->lock); +} + +/* Engine lock held: submits never overlap, so busy is a flag. */ +void ane_boost_begin(struct ane_device *ane) +{ + struct ane_boost *b = &ane->boost; + + if (!b->legs || !boost_idle_ms) + return; + mutex_lock(&b->lock); + b->busy = true; + if (!b->on) + ane_boost_set(ane, true); + mutex_unlock(&b->lock); +} + +void ane_boost_end(struct ane_device *ane) +{ + struct ane_boost *b = &ane->boost; + + if (!b->legs) + return; + mutex_lock(&b->lock); + b->busy = false; + b->last_end = jiffies; + if (b->on) + mod_delayed_work(system_wq, &b->off, + msecs_to_jiffies(boost_idle_ms)); + mutex_unlock(&b->lock); +} + +int ane_boost_init(struct ane_device *ane) +{ + struct ane_boost *b = &ane->boost; + + mutex_init(&b->lock); + INIT_DELAYED_WORK(&b->off, ane_boost_off_work); + b->nlegs = num_possible_cpus(); + b->legs = kcalloc(b->nlegs, sizeof(*b->legs), GFP_KERNEL); + return b->legs ? 0 : -ENOMEM; +} + +void ane_boost_exit(struct ane_device *ane) +{ + struct ane_boost *b = &ane->boost; + + if (!b->legs) + return; + cancel_delayed_work_sync(&b->off); + mutex_lock(&b->lock); + if (b->on) + ane_boost_set(ane, false); + mutex_unlock(&b->lock); + kfree(b->legs); + b->legs = NULL; +} diff --git a/drivers/accel/ane/ane_dart.c b/drivers/accel/ane/ane_dart.c new file mode 100644 index 0000000000000..aea8cd9eff182 --- /dev/null +++ b/drivers/accel/ane/ane_dart.c @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Joshua Warren */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ane.h" + +/* + * T8103 DART is the T8020 register block (apple_dart_hw_t8103 in + * drivers/iommu/apple-dart.c), pagesized 0x4000. Containment policy: + * during a job the apple-dart fault IRQ is masked so the latch survives + * for this driver to read, the completion poll checks that latch before + * every engine access (engine reads hang the SoC once a fault has + * landed), and a faulted IOVA is handed a throwaway page so the retried + * transaction completes and the engine drains. Mapping itself stays + * provider-owned; the page tables are never rewritten here. + */ +#define DART_STREAM_COMMAND 0x20 +#define DART_STREAM_COMMAND_BUSY BIT(2) +#define DART_STREAM_COMMAND_INVALIDATE BIT(20) +#define DART_STREAM_SELECT 0x34 +#define DART_ERROR 0x40 +#define DART_ERROR_FLAG BIT(31) +#define DART_ERROR_ADDR_LO 0x50 +#define DART_ERROR_ADDR_HI 0x54 +#define DART_STREAMS_ENABLE 0xfc +#define DART_TCR 0x100 +#define DART_TTBR 0x200 +#define DART_TTBR_COUNT 4 + +#define DART_TCR_SID(sid) (DART_TCR + ((sid) << 2)) +#define DART_TTBR_SID(sid, idx) \ + (DART_TTBR + ((((sid) * DART_TTBR_COUNT) + (idx)) << 2)) + +static bool dart_contain = true; +module_param(dart_contain, bool, 0444); +MODULE_PARM_DESC(dart_contain, + "Mask ANE DART fault IRQs during a job and restore the stream after a fault (default: on)"); + +static int ane_dart_invalidate(struct ane_device *ane, struct ane_dart *dart) +{ + void __iomem *regs = dart->regs; + u32 cmd; + int ret; + + if (dart->sid >= 32) + return -EINVAL; + writel(BIT(dart->sid), regs + DART_STREAM_SELECT); + writel(DART_STREAM_COMMAND_INVALIDATE, regs + DART_STREAM_COMMAND); + ret = readl_poll_timeout(regs + DART_STREAM_COMMAND, cmd, + !(cmd & DART_STREAM_COMMAND_BUSY), 1, 100); + if (ret) + dev_err(ane->dev, "DART sid %u TLB invalidate timed out\n", + dart->sid); + return ret; +} + +int ane_dart_init(struct ane_device *ane) +{ + struct device_node *np = ane->dev->of_node; + int count, i; + + ane->dart_count = 0; + if (!dart_contain) + return 0; + count = of_count_phandle_with_args(np, "iommus", "#iommu-cells"); + if (count < 0) + return 0; + for (i = 0; i < count && ane->dart_count < ANE_DART_MAX; i++) { + struct of_phandle_args args; + struct platform_device *pdev; + struct resource *res; + struct ane_dart *dart; + void __iomem *regs; + int irq; + + if (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", + i, &args)) + break; + if (!of_device_is_compatible(args.np, "apple,t8103-dart")) { + of_node_put(args.np); + continue; + } + pdev = of_find_device_by_node(args.np); + if (!pdev) { + of_node_put(args.np); + continue; + } + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + irq = platform_get_irq(pdev, 0); + if (!res || irq < 0) { + put_device(&pdev->dev); + of_node_put(args.np); + continue; + } + regs = devm_ioremap(ane->dev, res->start, resource_size(res)); + if (!regs) { + put_device(&pdev->dev); + of_node_put(args.np); + return -ENOMEM; + } + dart = &ane->darts[ane->dart_count++]; + dart->regs = regs; + dart->irq = irq; + dart->sid = args.args_count ? args.args[0] : 0; + dart->masked = false; + dev_info(ane->dev, "DART containment: %pOFn sid %u irq %d\n", + args.np, dart->sid, irq); + put_device(&pdev->dev); + of_node_put(args.np); + } + dev_info(ane->dev, "DART containment armed: %d\n", ane->dart_count); + return 0; +} + +void ane_dart_mask(struct ane_device *ane) +{ + int i; + + for (i = 0; i < ane->dart_count; i++) { + struct ane_dart *dart = &ane->darts[i]; + + if (dart->masked) + continue; + disable_irq(dart->irq); + dart->masked = true; + } +} + +void ane_dart_unmask(struct ane_device *ane) +{ + int i; + + for (i = 0; i < ane->dart_count; i++) { + struct ane_dart *dart = &ane->darts[i]; + + if (!dart->masked) + continue; + enable_irq(dart->irq); + dart->masked = false; + } +} + +bool ane_dart_faulted(struct ane_device *ane, u64 *iova, u32 *status) +{ + int i; + + for (i = 0; i < ane->dart_count; i++) { + void __iomem *regs = ane->darts[i].regs; + u32 err = readl(regs + DART_ERROR); + + if (!(err & DART_ERROR_FLAG)) + continue; + if (status) + *status = err; + if (iova) + *iova = readl(regs + DART_ERROR_ADDR_LO) | + ((u64)readl(regs + DART_ERROR_ADDR_HI) << 32); + return true; + } + return false; +} + +/* + * A fault is a missing PTE, nothing more: the page tables are provider + * owned and unchanged, so the stream-restore apple-dart uses at reset + * time is the wrong tool mid-job (register surgery while the faulting + * transaction is retried is what killed jwm1 twice). Instead the faulted + * IOVA is handed a throwaway DART page, so the retried transaction + * completes and the engine drains on its own; the scratch pages are + * unmapped once the request has been failed cleanly. + * + * The throwaway page is mapped at the faulting address, which is + * allocator-free by definition of a translation fault - but the drm_mm + * allocator knows nothing about a raw iommu_map. A scratch PTE that + * outlives its release path (any future bug on the wedge side) would + * then sit, untracked, in a range the allocator considers free: the + * next BO_INIT maps onto it and dies in dart_init_pte's "we require an + * unmap first" WARN (m1-test-host 2026-09-25, iommu_map failed at 0x4000). So + * the scratch range is RESERVED in the allocator first: a node-tracked + * mapping can never collide with a later BO_INIT, and a scratch whose + * release is skipped degrades into a bounded, benign leak instead of a + * poisoned address space. Reserve failure means a live BO node already + * owns the faulting page (a wild descriptor, not a hole): refuse and + * let the caller wedge, fail closed. + */ +int ane_dart_drain_fault(struct ane_device *ane, u64 fault_iova, + struct ane_dart_scratch *scratch, int *pages) +{ + struct drm_mm_node *node; + struct page *page; + u64 iova; + int ret, i; + + page = alloc_page(GFP_KERNEL); + if (!page) + return -ENOMEM; + iova = fault_iova & ~((1ULL << ane->shift) - 1); + + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (!node) { + __free_page(page); + return -ENOMEM; + } + node->start = iova; + node->size = BIT(ane->shift); + + mutex_lock(&ane->iommu_lock); + ret = drm_mm_reserve_node(&ane->mm, node); + if (ret) { + mutex_unlock(&ane->iommu_lock); + kfree(node); + __free_page(page); + return ret; + } + + ret = iommu_map(ane->domain, iova, page_to_phys(page), + BIT(ane->shift), IOMMU_READ | IOMMU_WRITE, GFP_KERNEL); + mutex_unlock(&ane->iommu_lock); + if (ret) { + mutex_lock(&ane->iommu_lock); + drm_mm_remove_node(node); + mutex_unlock(&ane->iommu_lock); + kfree(node); + __free_page(page); + return ret; + } + scratch[*pages].iova = iova; + scratch[*pages].page = page; + scratch[*pages].node = node; + (*pages)++; + for (i = 0; i < ane->dart_count; i++) + ane_dart_invalidate(ane, &ane->darts[i]); + return 0; +} + +void ane_dart_release_scratch(struct ane_device *ane, + struct ane_dart_scratch *scratch, int pages) +{ + int i, d; + + mutex_lock(&ane->iommu_lock); + for (i = 0; i < pages; i++) { + size_t unmapped; + + unmapped = iommu_unmap(ane->domain, scratch[i].iova, + BIT(ane->shift)); + if (unmapped != BIT(ane->shift)) + dev_err(ane->dev, + "scratch unmap short at %#llx: %zu\n", + scratch[i].iova, unmapped); + drm_mm_remove_node(scratch[i].node); + kfree(scratch[i].node); + scratch[i].node = NULL; + __free_page(scratch[i].page); + scratch[i].page = NULL; + } + mutex_unlock(&ane->iommu_lock); + if (pages) + for (d = 0; d < ane->dart_count; d++) + ane_dart_invalidate(ane, &ane->darts[d]); +} diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c new file mode 100644 index 0000000000000..db03774c9e25a --- /dev/null +++ b/drivers/accel/ane/ane_drv.c @@ -0,0 +1,1234 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2022 Eileen Yoon */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "ane.h" +#include "ane_tm.h" + +/* + * BO mapping mode, runtime-switchable per device (/sys/module/ane/parameters/ + * map_mode) and read at BO-mmap/iommu-map time: + * bit 0 — mark DART descriptors IOMMU_CACHE (cacheable, inner-shareable) + * bit 1 — map the CPU vma cacheable instead of write-combine + * 0 = today's WC CPU vmas + NC descriptors; 3 = both cacheable (device + * accesses cohere with the CPU caches). Each axis is toggleable alone for + * attribution. + * + * Coherency evidence for mode 3, both SoCs: + * - T6001/t6001-test-host (apple,t6000-dart x3, APPLE_DART2 PTEs): (1) the ANE sits in + * the dma-coherent DART class every SoC DMA-API device uses on + * arm64/Asahi, where dma-iommu itself programs descriptors with + * IOMMU_CACHE; (2) eiln's pre-GEM driver (4566c89^) mapped BOs cacheable + * on the CPU while DMAing them through this same iommu_map() path for + * months with no maintenance; (3) empirically, an alternating-pattern + * 16-rep buffer-reuse battery over 6 programs in both directions is + * byte-identical WC+NC vs cached on every rep — stale cache lines would + * surface as the previous pattern's bytes. + * - T8103/m1-test-host (apple,t8103-dart x3, APPLE_DART1 PTEs): the DART1 PTE + * format has no cacheability field (io-pgtable-dart.c dart_prot_to_pte), + * so bit 0 is a structural no-op there and the lever is the cached CPU + * vma alone. eiln's pre-GEM precedent ran exactly that on M1 for months. + * The same reuse battery is byte-identical (14/14) and the certified + * Parakeet E2E pins hold bit-for-bit with ane_exec 2561.2 -> 2402.0 ms. + * + * Default 3 (cached both sides). 0 is the documented rollback: runtime via + * /sys/module/ane/parameters/map_mode, or by loading the retained prior + * module (ane-96d5a88.ko on both hosts). + */ +static int map_mode = 3; +module_param(map_mode, int, 0644); +MODULE_PARM_DESC(map_mode, + "BO mapping: bit0=IOMMU_CACHE DART descriptors, bit1=cacheable CPU vmas (default 3 = cached; 0 = writecombine + non-cacheable rollback)"); + +#define CMD_BUF_BDX 0 +#define KRN_BUF_BDX 1 + +struct ane_bo { + struct drm_gem_object base; + struct list_head node; + struct drm_mm_node *mm; + u32 npages; + struct page **pages; + dma_addr_t iova; +}; + +#define to_bo(gem) (container_of(gem, struct ane_bo, base)) + +static struct ane_bo *bo_lookup(struct drm_file *file, u32 handle) +{ + struct drm_gem_object *gem = drm_gem_object_lookup(file, handle); + + if (!gem) + return NULL; + return to_bo(gem); +} + +/* + * Buffers are mapped and unmapped only through the kernel-owned IOMMU + * domain the device was attached to by its providers. apple-dart + * programs every DART in the device's "iommus" list (TTBRs, stream + * setup, invalidation and fault IRQs are all provider-owned); + * iommu_unmap() flushes through the provider before returning, so pages + * may be reclaimed once it succeeds. + */ +static int ane_iommu_map_pages(struct ane_device *ane, struct ane_bo *bo) +{ + int err; + + lockdep_assert_held(&ane->engine_lock); + if (bo->mm) + return -EBUSY; + + bo->mm = kzalloc(sizeof(*bo->mm), GFP_KERNEL); + if (!bo->mm) + return -ENOMEM; + + mutex_lock(&ane->iommu_lock); + + /* reserve area from ANE address space */ + err = drm_mm_insert_node_generic(&ane->mm, bo->mm, + bo->npages << ane->shift, + 1UL << ane->shift, 0, 0); + if (err < 0) { + dev_err(ane->dev, "out of ANE space: %d\n", err); + goto unlock; + } + + bo->iova = bo->mm->start; + + /* map into ANE address space */ + for (u32 i = 0; i < bo->npages; i++) { + dma_addr_t iova = bo->iova + (i << ane->shift); + int prot = IOMMU_READ | IOMMU_WRITE; + bool healed = false; + int prot_cache = map_mode & 1; + +retry: + if (prot_cache) + prot |= IOMMU_CACHE; + err = iommu_map(ane->domain, iova, page_to_phys(bo->pages[i]), + 1UL << ane->shift, prot, GFP_KERNEL); + if (err < 0) { + dev_err(ane->dev, "iommu_map failed at 0x%llx", iova); + /* Every PTE inside a reserved drm_mm node must be + * ours-or-absent: scratch drains reserve before + * mapping and wedge-preserved nodes stay inserted. + * A VALID PTE in a just-reserved range is therefore + * a stray from a torn-down session or a lost + * unmap. Clear it once and retry instead of + * poisoning every later BO_INIT until reload + * (m1-test-host 2026-09-25: dart_init_pte -EEXIST at + * 0x4000 for the rest of the session). + */ + if (err == -EEXIST && !healed && + iommu_iova_to_phys(ane->domain, iova)) { + dev_warn(ane->dev, + "clearing stray DART PTE at 0x%llx (no node owns it)\n", + iova); + iommu_unmap(ane->domain, iova, + 1UL << ane->shift); + healed = true; + goto retry; + } + while (i-- > 0) { + size_t unmapped; + + unmapped = iommu_unmap(ane->domain, + bo->iova + + (i << ane->shift), + 1UL << ane->shift); + if (unmapped != 1UL << ane->shift) + dev_err(ane->dev, + "rollback unmap short at %#llx: %zu\n", + bo->iova + (i << ane->shift), + unmapped); + } + drm_mm_remove_node(bo->mm); + bo->iova = 0; + break; + } + } + + mutex_unlock(&ane->iommu_lock); + + if (err < 0) { + kfree(bo->mm); + bo->mm = NULL; + return err; + } + + return 0; + +unlock: + mutex_unlock(&ane->iommu_lock); + kfree(bo->mm); + bo->mm = NULL; + return err; +} + +static void ane_iommu_unmap_pages(struct ane_device *ane, struct ane_bo *bo) +{ + struct drm_mm_node *mm = bo->mm; + + lockdep_assert_held(&ane->engine_lock); + + if (!mm) + return; + + list_del_init(&bo->node); + mutex_lock(&ane->iommu_lock); + for (u32 i = 0; i < bo->npages; i++) { + dma_addr_t iova = bo->iova + (i << ane->shift); + size_t unmapped = iommu_unmap(ane->domain, iova, + 1UL << ane->shift); + + if (unmapped != 1UL << ane->shift) + dev_err(ane->dev, "unmap short at %#llx: %zu\n", + iova, unmapped); + } + drm_mm_remove_node(mm); + bo->mm = NULL; + bo->iova = 0; + mutex_unlock(&ane->iommu_lock); + + kfree(mm); +} + +/* + * A BO freed while the engine is wedged keeps its mapping and its pages + * (DMA may still be reading the IOVA), but the GEM object is gone. The + * range's drm_mm node and the page array outlive it here, still linked + * into the allocator, until DMA is provably quiescent: recovery + * (ane_reclaim_preserved after the quiescing power cycle) or remove. + * The node stays reserved the whole time, so the IOVA can never be + * handed to a later BO_INIT while its PTEs are live. + */ +struct ane_preserved { + struct list_head list; + struct drm_mm_node *mm; + struct page **pages; + u32 npages; +}; + +static void ane_preserve_bo(struct ane_device *ane, struct ane_bo *bo) +{ + struct ane_preserved *p; + + p = kzalloc(sizeof(*p), GFP_KERNEL); + if (!p) { + /* Fail closed the old way: the node stays inserted and the + * pages leak until reboot - safe, just unreclaimable. + */ + dev_err(ane->dev, "wedged: preserve registry full, leaking\n"); + bo->mm = NULL; + bo->pages = NULL; + return; + } + p->mm = bo->mm; + p->pages = bo->pages; + p->npages = bo->npages; + list_add_tail(&p->list, &ane->preserved_list); + bo->mm = NULL; + bo->pages = NULL; +} + +void ane_reclaim_preserved(struct ane_device *ane) +{ + struct ane_preserved *p, *tmp; + unsigned long count = 0; + + lockdep_assert_held(&ane->engine_lock); + + list_for_each_entry_safe(p, tmp, &ane->preserved_list, list) { + u64 iova = p->mm->start; + + mutex_lock(&ane->iommu_lock); + for (u32 i = 0; i < p->npages; i++) { + size_t unmapped; + + unmapped = iommu_unmap(ane->domain, + iova + (i << ane->shift), + 1UL << ane->shift); + if (unmapped != 1UL << ane->shift) + dev_err(ane->dev, + "preserved unmap short at %#llx: %zu\n", + iova + (i << ane->shift), unmapped); + } + drm_mm_remove_node(p->mm); + mutex_unlock(&ane->iommu_lock); + + kfree(p->mm); + for (u32 i = 0; i < p->npages; i++) + put_page(p->pages[i]); + kfree(p->pages); + list_del(&p->list); + kfree(p); + count++; + } + if (count) + dev_info(ane->dev, + "reclaimed %lu wedge-preserved mapping(s) after quiesce\n", + count); +} + +static vm_fault_t ane_gem_vm_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct drm_gem_object *gem = vma->vm_private_data; + struct ane_bo *bo = to_bo(gem); + struct page *page; + pgoff_t offset; + + if (!bo->pages) + return VM_FAULT_SIGBUS; + + offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + if (offset >= bo->npages) + return VM_FAULT_SIGBUS; + page = bo->pages[offset]; + + return vmf_insert_page(vma, vmf->address, page); +} + +static const struct vm_operations_struct drm_gem_ane_vm_ops = { + .open = drm_gem_vm_open, + .close = drm_gem_vm_close, + .fault = ane_gem_vm_fault, +}; + +/* + * Sole destruction path for buffer objects: every reference (handles, + * mmap vmas, lookups) funnels here. Nothing is reclaimed while the + * engine is wedged, since DMA may still be reading the IOVA. + */ +static void ane_gem_free_object(struct drm_gem_object *gem) +{ + struct ane_device *ane = gem->dev->dev_private; + struct ane_bo *bo = to_bo(gem); + + mutex_lock(&ane->engine_lock); + if (bo->mm) { + if (atomic_read(&ane->wedged)) { + /* Fail closed: keep the mapping and the pages so no + * IOVA is torn down or page reused underneath active + * DMA. The range's node stays reserved and tracked + * until recovery quiesces the engine and reclaims + * it, so no later BO_INIT can map over these PTEs + * (a leaked-but-forgotten mapping is what turned + * into dart_init_pte -EEXIST WARNs on m1-test-host). + */ + dev_err(ane->dev, "wedged: preserving bo mapping\n"); + list_del_init(&bo->node); + ane_preserve_bo(ane, bo); + } else { + ane_iommu_unmap_pages(ane, bo); + } + } + mutex_unlock(&ane->engine_lock); + + if (bo->pages) { + drm_gem_put_pages(gem, bo->pages, true, true); + bo->pages = NULL; + } + drm_gem_object_release(gem); + kfree(bo); +} + +static const struct drm_gem_object_funcs ane_gem_object_funcs = { + .free = ane_gem_free_object, + .vm_ops = &drm_gem_ane_vm_ops, +}; + +static int ane_bo_init(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct ane_device *ane = drm->dev_private; + struct drm_ane_bo_init *args = data; + struct drm_gem_object *gem; + struct ane_bo *bo; + int err; + + if (args->pad || !args->size) + return -EINVAL; + + bo = kzalloc(sizeof(struct ane_bo), GFP_KERNEL); + if (!bo) + return -ENOMEM; + INIT_LIST_HEAD(&bo->node); + + gem = &bo->base; + gem->funcs = &ane_gem_object_funcs; + err = drm_gem_object_init(drm, gem, round_up(args->size, PAGE_SIZE)); + if (err < 0) { + drm_gem_private_object_fini(gem); + goto free; + } + + err = drm_gem_create_mmap_offset(gem); + if (err < 0) + goto put; + + args->offset = drm_vma_node_offset_addr(&gem->vma_node); + + bo->npages = gem->size >> PAGE_SHIFT; + bo->pages = drm_gem_get_pages(gem); + if (IS_ERR(bo->pages)) { + err = PTR_ERR(bo->pages); + bo->pages = NULL; + goto put; + } + + mutex_lock(&ane->engine_lock); + if (ane->removed) { + err = -ENODEV; + goto unlock_put; + } + if (atomic_read(&ane->wedged)) { + dev_err_ratelimited(ane->dev, "wedged: refusing bo init\n"); + err = -ENODEV; + goto unlock_put; + } + + err = ane_iommu_map_pages(ane, bo); + if (err < 0) + goto unlock_put; + list_add_tail(&bo->node, &ane->bo_list); + + err = drm_gem_handle_create(file, gem, &args->handle); + mutex_unlock(&ane->engine_lock); + drm_gem_object_put(gem); /* handle (or error path) owns it now */ + return err; + +unlock_put: + mutex_unlock(&ane->engine_lock); +put: + drm_gem_object_put(gem); /* routes through ane_gem_free_object() */ + return err; +free: + kfree(bo); + return err; +} + +static int ane_bo_free(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct ane_device *ane = drm->dev_private; + struct drm_ane_bo_free *args = data; + struct ane_bo *bo; + + if (args->pad) + return -EINVAL; + + mutex_lock(&ane->engine_lock); + if (ane->removed) { + mutex_unlock(&ane->engine_lock); + return -ENODEV; + } + if (atomic_read(&ane->wedged)) { + dev_err_ratelimited(ane->dev, "wedged: refusing bo free\n"); + mutex_unlock(&ane->engine_lock); + return -ENODEV; + } + + bo = bo_lookup(file, args->handle); + mutex_unlock(&ane->engine_lock); + if (!bo) + return -EINVAL; + + /* Keep reference drops outside engine_lock: the final put enters + * ane_gem_free_object(), which owns that lock while unmapping. + */ + drm_gem_handle_delete(file, args->handle); + drm_gem_object_put(&bo->base); + return 0; +} + +static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) +{ + struct ane_device *ane = drm->dev_private; + struct drm_ane_submit *args = data; + struct drm_gem_object *gem[ANE_TILE_COUNT] = { 0 }; + struct drm_gem_object *btsp = NULL; + struct ane_bo *bo; + struct ane_request req; + int err; + + memset(&req, 0, sizeof(req)); + + if (args->pad || !args->tsk_size || !args->td_count || + args->td_count > 0xffff || args->td_size < 4 || + args->td_size > 0x40000 || (args->td_size & 3) || + !args->handles[CMD_BUF_BDX] || args->handles[KRN_BUF_BDX] || + !args->btsp_handle) { + return -EINVAL; + } + + /* Fail closed: never queue work behind a wedged engine. */ + if (atomic_read(&ane->wedged)) { + dev_err_ratelimited(ane->dev, "wedged: refusing submit\n"); + return -ECANCELED; + } + + req.qid = 4; + req.nid = ANE_FIFO_NID; + req.td_size = args->td_size; + req.td_count = args->td_count; + + for (int bdx = 0; bdx < ANE_TILE_COUNT; bdx++) { + if (!args->handles[bdx]) + continue; + bo = bo_lookup(file, args->handles[bdx]); + if (!bo) { + err = -EINVAL; + goto put; + } + gem[bdx] = &bo->base; + if (!bo->iova || + ((bdx == CMD_BUF_BDX) && + (args->tsk_size >= (bo->npages << ane->shift)))) { + err = -EINVAL; + goto put; + } + req.bar[bdx] = lower_32_bits(bo->iova); + } + + /* + * The microcode and weights are packed @ 16 gran for bank aligned + * access. Since this isn't page aligned, we represent the two as one + * buffer and calculate the delimiter (where the weights would start). + */ + req.bar[KRN_BUF_BDX] = + req.bar[CMD_BUF_BDX] + round_up(args->tsk_size, ANE_CMD_GRAN); + + bo = bo_lookup(file, args->btsp_handle); + if (!bo) { + err = -EINVAL; + goto put; + } + btsp = &bo->base; + if (!bo->iova || args->td_size > bo->base.size) { + err = -EINVAL; + goto put; + } + req.btsp_iova = lower_32_bits(bo->iova); + + /* + * Lookup references are held across the synchronous execute and + * dropped only after ane_tm_execute() confirmed completion, so a + * concurrent BO_FREE cannot unmap a buffer the engine reads. + */ + mutex_lock(&ane->engine_lock); + if (ane->removed) { + err = -ENODEV; + goto unlock; + } + if (atomic_read(&ane->wedged)) { + dev_err_ratelimited(ane->dev, "wedged: refusing submit\n"); + err = -ECANCELED; + goto unlock; + } + + err = ane_tm_enqueue(ane, &req); + if (err < 0) + goto unlock; + + ane_boost_begin(ane); + err = ane_tm_execute(ane, &req); + ane_boost_end(ane); + +unlock: + mutex_unlock(&ane->engine_lock); +put: + drm_gem_object_put(btsp); + for (int bdx = 0; bdx < ANE_TILE_COUNT; bdx++) { + if (gem[bdx]) + drm_gem_object_put(gem[bdx]); + } + return err; +} + +static const struct drm_ioctl_desc ane_drm_ioctls[] = { + DRM_IOCTL_DEF_DRV(ANE_BO_INIT, ane_bo_init, 0), + DRM_IOCTL_DEF_DRV(ANE_BO_FREE, ane_bo_free, 0), + DRM_IOCTL_DEF_DRV(ANE_SUBMIT, ane_submit, 0), +}; + +static ssize_t wedged_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct ane_device *ane = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%d\n", atomic_read(&ane->wedged)); +} +static DEVICE_ATTR_RO(wedged); + +/* Operator retry for a wedge whose automatic recovery failed: one more + * bounded power-cycle attempt. No-op when not wedged. + */ +static ssize_t reset_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct ane_device *ane = dev_get_drvdata(dev); + int err = 0; + + mutex_lock(&ane->engine_lock); + if (!ane->removed) + err = ane_tm_recover(ane); + mutex_unlock(&ane->engine_lock); + if (err) + return err; + return count; +} +static DEVICE_ATTR_WO(reset); + +static struct attribute *ane_dev_attrs[] = { + &dev_attr_wedged.attr, + &dev_attr_reset.attr, + NULL, +}; +ATTRIBUTE_GROUPS(ane_dev); + +static int ane_drm_open(struct drm_device *drm, struct drm_file *file) +{ + struct ane_device *ane = drm->dev_private; + int err; + + /* Bring up power while the file context is created. A failed + * resume propagates; nothing is released that was not acquired. + */ + err = pm_runtime_resume_and_get(ane->dev); + if (err < 0) + return err; + + pm_runtime_put(ane->dev); + return 0; +} + +/* + * Per-session teardown: every file context open is a fresh boundary. + * Anything the previous session left on the device (wedged flag, tm + * completion counters, in-flight recovery state) belongs to that + * session and must not survive into the next one. + * + * drm_gem_release runs before us and has dropped every handle this + * file held. BOs leaked by `ane_gem_free_object under wedge` stay on + * ane->bo_list until module teardown (their backing `drm_gem_object`s + * are already gone, and touching them here would be use-after-free). + * We do not try to drain them; we only clear the session-leaving + * state (wedged + tm counters + recovering flag) so the next opener + * sees a clean device. The orphan BOs remain inert: they are not in + * any drm_file's handle table. + */ +static void ane_drm_postclose(struct drm_device *drm, struct drm_file *file) +{ + struct ane_device *ane = drm->dev_private; + + mutex_lock(&ane->engine_lock); + if (ane->removed) { + mutex_unlock(&ane->engine_lock); + return; + } + + /* Drop the wedge pin first: a wedged engine that cannot be + * recovered in software must NOT keep the module pinned, or the + * operator cannot unload the ko for an updated build without a + * reboot. ane_wedge_clear releases both the wedged flag and the + * module_refcount; ane_tm_recover then attempts a power-cycle + * recovery (succeeds only if the engine is actually idle-able). + */ + ane_wedge_clear(ane); + ane_tm_recover(ane); + + mutex_unlock(&ane->engine_lock); +} + +/* + * The driver only owns three ioctls (ANE_BO_INIT, ANE_BO_FREE, + * ANE_SUBMIT). Anything outside that table would otherwise fall through + * the standard drm_ioctl path to legacy primary-node handlers that + * dereference drm_device.unique, which an accel node never sets. Reject + * such callers explicitly so they fail with -ENOTTY instead of oopsing. + * + * The accel pattern uses ane_drm_ioctls[] via drm_ioctl() (legacy pattern + * from before DRM_ACCEL_FOPS landed drm_ioctl defaults), so this is a + * belt-and-suspenders gate on top of the existing ioctls table. + */ +static long ane_drm_unlocked_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct drm_file *filp = file->private_data; + struct drm_device *drm = filp->minor->dev; + struct ane_device *ane = drm->dev_private; + long err; + + /* Reject anything outside the drm type ('d') namespace outright, and + * core ioctls that would read a DRM master the accel node does not + * have (GET_UNIQUE, GET_MAP, ... dereference NULL); clients need + * only VERSION from the core set. + * + * _IOC_TYPEMASK is the low 8 bits, which encode the *nr*, not the + * *type*. The type byte lives at offset _IOC_TYPESHIFT (8). Use + * _IOC_TYPE(cmd) to extract the right field; misuse here returns + * -ENOTTY for every legitimate ane_ioctl, which is exactly the + * regression committed and reverted in the lifecycle-reset series. + */ + if (_IOC_TYPE(cmd) != DRM_IOCTL_BASE || + (_IOC_NR(cmd) < DRM_COMMAND_BASE && + _IOC_NR(cmd) != _IOC_NR(DRM_IOCTL_VERSION)) || + _IOC_NR(cmd) >= DRM_COMMAND_END) + return -ENOTTY; + + err = pm_runtime_resume_and_get(ane->dev); + if (err < 0) + return err; + + err = drm_ioctl(file, cmd, arg); + + pm_runtime_put(ane->dev); + + return err; +} + +static int ane_drm_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct drm_gem_object *gem; + struct ane_bo *bo; + int err; + + err = drm_gem_mmap(file, vma); + if (err < 0) + return err; + + /* + * Set vm_pgoff (used as a fake buffer offset by DRM) to 0 and map the + * whole buffer from the start. + */ + vma->vm_pgoff = 0; + gem = vma->vm_private_data; + bo = to_bo(gem); + + if (vma_pages(vma) == 0) + return -ENXIO; + + /* + * Cacheable CPU mapping when bit 1 of map_mode is set, paired with + * IOMMU_CACHE DART descriptors: device accesses cohere with the CPU + * caches and read-back stops paying uncached-mode DRAM latency. + * Default (bit clear) keeps the write-combine mapping. VM_IO stays + * set either way (no struct-page identity is promised). + */ + vm_flags_mod(vma, VM_IO | VM_DONTEXPAND | VM_DONTDUMP, VM_PFNMAP); + + if (map_mode & 2) + vma->vm_page_prot = + pgprot_decrypted(vm_get_page_prot(vma->vm_flags)); + else { + vma->vm_page_prot = + pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); + vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); + } + + return vm_map_pages(vma, bo->pages, bo->npages); +} + +static const struct file_operations ane_drm_fops = { + .owner = THIS_MODULE, + .fop_flags = FOP_UNSIGNED_OFFSET, + .open = accel_open, + .release = drm_release, + .unlocked_ioctl = ane_drm_unlocked_ioctl, + .compat_ioctl = drm_compat_ioctl, + .poll = drm_poll, + .read = drm_read, + .llseek = noop_llseek, + .mmap = ane_drm_mmap, +}; + +static const struct drm_driver ane_drm_driver = { + .driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL, + .open = ane_drm_open, + .postclose = ane_drm_postclose, + .ioctls = ane_drm_ioctls, + .num_ioctls = ARRAY_SIZE(ane_drm_ioctls), + .fops = &ane_drm_fops, + .major = ANE_ABI_MAJOR, + .name = "ane", + .desc = "Apple Neural Engine driver", +}; + +static int ane_iommu_domain_init(struct ane_device *ane) +{ + u64 min_iova, limit; + + struct iommu_domain *domain = iommu_get_domain_for_dev(ane->dev); + + if (!domain) + return -EPROBE_DEFER; + + if (!(domain->pgsize_bitmap & PAGE_SIZE) || + !domain->geometry.force_aperture) + return -EINVAL; + min_iova = max_t(u64, domain->geometry.aperture_start, PAGE_SIZE); + limit = round_down(min_t(u64, domain->geometry.aperture_end, U32_MAX) + 1, + PAGE_SIZE); + if (min_iova >= limit) + return -EINVAL; + min_iova = round_up(min_iova, PAGE_SIZE); + if (min_iova >= limit) + return -EINVAL; + ane->domain = domain; + ane->shift = PAGE_SHIFT; + drm_mm_init(&ane->mm, min_iova, limit - min_iova); + + return 0; +} + +/* + * The DART page table belongs to the IOMMU domain, which outlives this + * module: a mapping a previous instance never unmapped (a wedge preserves + * them) survives a reload, and the first BO_INIT onto it fails with + * -EEXIST. Clear every mapping inside the still-empty allocator range. + * The unmap flushes the DART TLBs, so this runs only after runtime + * resume has powered the engine and its DARTs, and before registration + * lets a client map anything. + */ +static void ane_iommu_purge_stale(struct ane_device *ane) +{ + struct drm_mm_node *hole; + u64 start, end, iova; + unsigned long stale = 0; + + drm_mm_for_each_hole(hole, &ane->mm, start, end) { + for (iova = start; iova < end; iova += 1UL << ane->shift) { + if (!iommu_iova_to_phys(ane->domain, iova)) + continue; + iommu_unmap(ane->domain, iova, 1UL << ane->shift); + stale++; + } + } + if (stale) + dev_warn(ane->dev, + "cleared %lu stale DART mappings from a previous instance\n", + stale); +} + +static void ane_detach_genpd(struct ane_device *ane) +{ + if (ane->pd_count <= 1) + return; + + for (int i = ane->pd_count - 1; i >= 0; i--) { + if (ane->pd_link[i]) + device_link_del(ane->pd_link[i]); + if (!IS_ERR_OR_NULL(ane->pd_dev[i])) + dev_pm_domain_detach(ane->pd_dev[i], true); + } +} + +static int ane_attach_genpd(struct ane_device *ane) +{ + struct device *dev = ane->dev; + + ane->pd_count = of_count_phandle_with_args( + dev->of_node, "power-domains", "#power-domain-cells"); + if (ane->pd_count < 1) + return ane->pd_count < 0 ? ane->pd_count : -EINVAL; + if (ane->pd_count == 1) + return dev->pm_domain ? 0 : -EPROBE_DEFER; + + ane->pd_dev = devm_kcalloc(dev, ane->pd_count, sizeof(*ane->pd_dev), + GFP_KERNEL); + if (!ane->pd_dev) + return -ENOMEM; + + ane->pd_link = devm_kcalloc(dev, ane->pd_count, sizeof(*ane->pd_link), + GFP_KERNEL); + if (!ane->pd_link) + return -ENOMEM; + + for (int i = 0; i < ane->pd_count; i++) { + ane->pd_dev[i] = dev_pm_domain_attach_by_id(dev, i); + if (IS_ERR(ane->pd_dev[i])) { + ane_detach_genpd(ane); + return PTR_ERR(ane->pd_dev[i]); + } + + ane->pd_link[i] = + device_link_add(dev, ane->pd_dev[i], + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME | + DL_FLAG_RPM_ACTIVE); + if (!ane->pd_link[i]) { + ane_detach_genpd(ane); + return -EINVAL; + } + } + + return 0; +} + +/* + * Per-SoC support descriptor, one per Linux `compatible`. + * + * Everything the probe needs except the SET block is derived from the + * device tree by design: DART topology from "iommus", power wiring from + * "power-domains", the engine window from "reg", the IRQ by name. Board + * facts belong in the overlay, not in compiled constants. The SET block + * is the one exception: the live overlays carry no range for it, so it + * is compiled in per SoC — and only from a source that has proven the + * address on hardware (m1n1 proxyclient ANE.ps_map, then a bound device). + * A guessed SET base is not a bug but a brick: direct writes to the + * block external-abort the SoC (T6001 named by netconsole 2026-09-16, + * PS_SET0 down at 0x28e08c000; T8103 same mechanism at 0x23b70c000), + * which is why a SoC without proven constants must refuse to bind + * instead of carrying a guess. + * + * Qualification tiers: + * ANE_QUALIFIED — execution proven on this silicon; binds normally. + * ANE_RECOGNIZED — constants present, never run on hardware; binds + * only with ane.allow_unqualified=1, loudly. + * ANE_UNSUPPORTED — no proven constants; refuses and names the data + * needed to advance the port. + */ +enum ane_qual { + ANE_QUALIFIED, + ANE_RECOGNIZED, + ANE_UNSUPPORTED, +}; + +struct ane_soc { + phys_addr_t ps_base; + enum ane_qual qual; + /* True when the tm/tq register file survives a genpd cycle in + * retention and recovery must drain it (see ane_tm_drain_retained). + */ + bool tm_retention; +}; + +static bool allow_unqualified; +module_param(allow_unqualified, bool, 0444); +MODULE_PARM_DESC(allow_unqualified, + "Bind recognized-but-unproven ANE SoCs (unverified SET base: external-abort risk)"); + +static const struct ane_soc ane_soc_t8103 = { + /* M1 (m1-test-host). SET block mapped read-only for the recovery ACTUAL + * log and the powered-on guard. Execution proven in the fleet. + */ + .ps_base = 0x23b70c000ULL, + .qual = ANE_QUALIFIED, +}; + +static const struct ane_soc ane_soc_t6000 = { + /* M1 Pro, M1 Max and M1 Ultra die 0 share this compatible and SET base + * (proven on T6001/t6001-test-host); M1 Pro and M1 Ultra are untested. + */ + .ps_base = 0x28e08c000ULL, + .qual = ANE_QUALIFIED, + .tm_retention = true, +}; + +static const struct ane_soc ane_soc_t6020 = { + /* M2 Pro (t6020). Community device-tree captures exist, but the + * SET base is unproven and the H14 compiler backend is unqualified: + * no constants may enter here yet. + */ + .ps_base = 0, + .qual = ANE_UNSUPPORTED, +}; + +static const struct ane_soc ane_soc_t6021 = { + /* M2 Max (T6021, t6021-test-host). SET window 0x8e08c000 = pmgr base + * 0x8e080000 + 0xc000, present in ane0's own reg on macOS captures + * of both 26.6.2 and 27.0 (2026-09-18, 11/11 derivation-input + * checks each, zero delta); the Linux translation reuses the pmgr + * high bits proven on T6000/T6020 rows (low-32 match) -> + * 0x28e08c000. Recognized, not qualified: constants complete, but + * no Linux execution yet, and the t6021 pwrstate word layout + * (which of set0/base/set1..4 sits at which offset — macOS-side + * capture cannot see it) is unverified; the first probe resume + * logs the ACTUAL nibbles through this window as the Linux-side + * probe (receipt 2026-09-18-t6021-driver-entry-prepared.md). + * Promotion to ANE_QUALIFIED needs an exact run on this silicon. + */ + .ps_base = 0x28e08c000ULL, + .qual = ANE_RECOGNIZED, +}; + +static const struct of_device_id ane_of_match[] = { + { .compatible = "apple,t8103-ane", .data = &ane_soc_t8103 }, + { .compatible = "apple,t6000-ane", .data = &ane_soc_t6000 }, + { .compatible = "apple,t6020-ane", .data = &ane_soc_t6020 }, + { .compatible = "apple,t6021-ane", .data = &ane_soc_t6021 }, + {} +}; + +MODULE_DEVICE_TABLE(of, ane_of_match); + +static int ane_platform_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + const struct of_device_id *id = of_match_device(ane_of_match, dev); + const struct ane_soc *soc = id->data; + struct ane_device *ane; + struct drm_device *drm; + int err; + + /* Tier gate before any power-domain, MMIO or IRQ interaction: an + * unqualified SoC must fail cleanly, never half-probe. + */ + if (soc->qual == ANE_UNSUPPORTED) { + dev_err(dev, + "%s: unsupported ANE: no proven SET-block base (a guessed base external-aborts the SoC); not binding. To advance this port, run the mlx-omarchy quick collector (scripts/collect_quick.py: captures ANE/DART/PMGR/AIC device-tree data, no driver needed) and submit the capture\n", + id->compatible); + return -ENODEV; + } + if (soc->qual == ANE_RECOGNIZED && !allow_unqualified) { + dev_err(dev, + "%s: recognized but unqualified: constants present, execution never proven on this silicon; not binding. Override with ane.allow_unqualified=1, or prove a run and report it\n", + id->compatible); + return -ENODEV; + } + if (soc->qual == ANE_RECOGNIZED) + dev_warn(dev, + "%s: UNQUALIFIED bind forced by allow_unqualified: no execution proven on this silicon — SET-block base and board topology unverified\n", + id->compatible); + + ane = devm_drm_dev_alloc(dev, &ane_drm_driver, struct ane_device, drm); + if (IS_ERR(ane)) + return PTR_ERR(ane); + + platform_set_drvdata(pdev, ane); + ane->dev = dev; + + drm = &ane->drm; + drm->dev_private = ane; + + atomic_set(&ane->wedged, 0); + ane->tm_retention = soc->tm_retention; + + /* Managed power first: genpd links hold the ANE/DART supplier + * topology awake before any register is touched. + */ + err = ane_attach_genpd(ane); + if (err < 0) { + dev_err(dev, "failed to attach power domains\n"); + return err; + } + + ane->irq = platform_get_irq_byname(pdev, "ane"); + if (ane->irq < 0) { + err = ane->irq; + goto detach_genpd; + } + /* Polled completion design: the engine IRQ is validated but never + * requested here. A DART fault IRQ stays owned by apple-dart. During + * a job this driver masks it, and on a fault restores that stream. + */ + + /* Mapping only; no register access happens while unpowered. */ + ane->engine = devm_platform_ioremap_resource_byname(pdev, "engine"); + if (IS_ERR(ane->engine)) { + err = PTR_ERR(ane->engine); + goto detach_genpd; + } + + /* m1n1 keys the SET block by ADT node name (ANE.ps_map); the live + * overlays carry no range for it, so the SoC descriptor picks the + * base (see the tier table above). Mapped read-only, always: + * recovery logs its ACTUAL nibbles beside every engine write and + * refuses engine MMIO unless the islands read powered on. Unmapped + * is tolerated: recovery then just skips the check. + */ + ane->ps_base = soc->ps_base; + if (ane->ps_base) + ane->ps = devm_ioremap(dev, ane->ps_base, 0x38); + + mutex_init(&ane->iommu_lock); + mutex_init(&ane->engine_lock); + INIT_LIST_HEAD(&ane->bo_list); + INIT_LIST_HEAD(&ane->preserved_list); + err = ane_boost_init(ane); + if (err < 0) + goto detach_genpd; + + /* + * Kernel-owned IOMMU domain. Defers until every "iommus" provider + * has attached. There is deliberately no fallback to direct DART + * programming for mappings: without providers this driver must not + * bind. Fault recovery only rewrites the TTBR and TCR the provider + * already installed. + */ + err = ane_iommu_domain_init(ane); + if (err < 0) + goto detach_genpd; + + /* + * Managed runtime PM from here on. This resume is what raises the + * ANE partition: genpd powers the domain when the device resumes + * (through the platform bus for one power-domain entry, through the + * supplier links for several), and .runtime_resume then does the + * first engine MMIO. Marking the device active up front skips that + * resume, and on T6001 the engine window external-aborts while its + * partition is gated. The reference is held until remove: the device + * stays powered for the whole lifetime, autosuspend stays disabled. + */ + pm_runtime_enable(dev); + err = pm_runtime_resume_and_get(dev); + if (err < 0) + goto disable_pm; + + ane_iommu_purge_stale(ane); + err = ane_dart_init(ane); + if (err < 0) + goto put_pm; + + err = drm_dev_register(drm, 0); + if (err < 0) + goto put_pm; + + dev_info(dev, "loaded ane\n"); + + return 0; + +put_pm: + pm_runtime_put_noidle(dev); +disable_pm: + pm_runtime_disable(dev); + drm_mm_takedown(&ane->mm); +detach_genpd: + ane_boost_exit(ane); + ane_detach_genpd(ane); + return err; +} + +static void ane_platform_remove(struct platform_device *pdev) +{ + struct ane_device *ane = platform_get_drvdata(pdev); + struct ane_bo *bo, *tmp; + + mutex_lock(&ane->engine_lock); + ane->removed = true; + drm_dev_unplug(&ane->drm); + + if (atomic_read(&ane->wedged)) { + dev_err(ane->dev, + "forced removal of a wedged device is unsafe; reboot required\n"); + mutex_unlock(&ane->engine_lock); + return; + } + + list_for_each_entry_safe(bo, tmp, &ane->bo_list, node) + ane_iommu_unmap_pages(ane, bo); + ane_reclaim_preserved(ane); + drm_mm_takedown(&ane->mm); + + ane_boost_exit(ane); + ane_detach_genpd(ane); + + pm_runtime_disable(ane->dev); + pm_runtime_put_noidle(ane->dev); + mutex_unlock(&ane->engine_lock); +} + +static int __maybe_unused ane_runtime_suspend(struct device *dev) +{ + struct ane_device *ane = dev_get_drvdata(dev); + + /* Veto gating while the engine may be DMA-active: there is no + * documented abort/reset to establish quiescence first -- except + * when recovery is power-cycling a wedged engine to establish + * exactly that quiescence. + */ + if (atomic_read(&ane->wedged) && !ane->recovering) + return -EBUSY; + + return 0; +} + +static int __maybe_unused ane_runtime_resume(struct device *dev) +{ + struct ane_device *ane = dev_get_drvdata(dev); + bool first = !ane->tm_status_known; + + /* The only path that touches the engine while its partition comes + * up: probe's first resume and every later ungate land here. Every + * translation is owned by the IOMMU providers. + */ + if (first) { + /* Bisect order, each stage named before it runs: genpd + * raise is already complete when this callback runs; the + * SET window reads come first (pmgr class, always safe + * per T6001/T8103 bisect evidence), then the engine. A + * hard reset after the last off-box line names the + * killing access exactly (T6021 console bring-up). + */ + struct resource *eng = platform_get_resource_byname( + to_platform_device(dev), IORESOURCE_MEM, "engine"); + + dev_info(dev, + "ANE-resume: genpd raise complete; SET window probe next\n"); + ane_ps_act_probe(ane); + + dev_info(dev, + "ANE-resume: SET window probed; first engine access next (TM_TQ_EN tm+0x0c @ engine %pr + 0x2000c)\n", + eng); + } + + ane_tm_enable(ane, first); + + /* First enable is the engine's fresh signature; recovery compares + * its post-reset status against it. + */ + if (!ane->tm_status_known) { + dev_info(dev, "ANE-resume: enable writes survived; TM_STATUS read next\n"); + ane->tm_status_fresh = ane_tm_status(ane); + ane->tm_status_known = true; + /* Linux-side pwrstate probe: ACTUAL nibbles read through + * the SoC descriptor's SET window while the partition is + * raised. 0xffffff means the mapped window is the live + * pmgr SET block with every word on; anything else names + * the t6021 word layout to fix before promotion. + */ + dev_info(dev, "ANERD ps probe act=%#x\n", ane_ps_act(ane)); + } + + return 0; +} + +// clang-format off +static const struct dev_pm_ops ane_pm_ops = { + SET_RUNTIME_PM_OPS(ane_runtime_suspend, ane_runtime_resume, NULL) + SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) +}; + +// clang-format on + +static struct platform_driver ane_platform_driver = { + .probe = ane_platform_probe, + .remove = ane_platform_remove, + .driver = { + .name = "ane", + .suppress_bind_attrs = true, + .dev_groups = ane_dev_groups, + .pm = pm_ptr(&ane_pm_ops), + .of_match_table = ane_of_match, + }, +}; + +module_platform_driver(ane_platform_driver); + +MODULE_AUTHOR("Eileen Yoon "); +MODULE_DESCRIPTION("Apple Neural Engine driver"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_fw_validate.h b/drivers/accel/ane/ane_fw_validate.h new file mode 100644 index 0000000000000..00ef2982c67e4 --- /dev/null +++ b/drivers/accel/ane/ane_fw_validate.h @@ -0,0 +1,480 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * ane_fw_validate.h — pinned ANE firmware images, their validator, and + * iBoot's runtime patches. + * + * Single source of truth shared by the kernel loader + * (ane_t6021_fwload.c) and the offline regression + * (tools/h14_boot_regression.c). Strict EXACT-image assertions (no + * generic Mach-O parsing). + * + * The images are the macOS 13.5 (22G74) payloads that iBoot preloads on + * the stub every Omarchy install boots: selene for T6020/T6021/T6022 + * (byte-verified live at SEG0 on T6021, notebook 20260926T230146Z) and + * bia for T8112 (receipts/2026-10-01-t8112-ane). Both have 5 load + * commands (2 segments + LC_SYMTAB + LC_UUID + LC_UNIXTHREAD), entry 0. + * + * Userspace consumers typedef u8/u32/u64/size_t/bool and provide + * get_unaligned_le32/64, memcmp and memcpy (see that tool's source); + * kernel consumers get these from linux/types.h + linux/string.h. + * This header intentionally includes nothing. + */ +#ifndef __ANE_FW_VALIDATE_H__ +#define __ANE_FW_VALIDATE_H__ + +/* FWIM surface size = config+0x138 byte-count = 0x500000 (Main + * 2026-09-20, audit 751caa4, T6021). Covers the selene vmsize 0x4fc000 + * and the bia vmsize 0x4ec000; NOT derivable from the blob length. + */ +#define ANE_FW_BUF_SIZE 0x500000 +#define ANE_FW_ENTRY_PC 0x0 + +#define MH_MAGIC_64 0xfeedfacfU +#define MH_PRELOAD 5 +#define CPU_TYPE_ARM64 0x0100000cU +#define LC_SEGMENT_64 0x19 +#define LC_UNIXTHREAD 0x5 +#define ARM_THREAD_STATE64 6 + +#define ANE_FW_NCMDS 5 +#define ANE_FW_SIZEOF_CMDS 0x960 +#define ANE_FW_FLAGS 0x1 +#define ANE_FW_NSEGS 2 +#define ANE_FW_RTK_PATCHES 5 + +struct ane_fw_seg { + u64 vmaddr; + u64 vmsize; + u64 fileoff; + u64 filesize; + char name[17]; /* 16 + NUL */ +}; + +/* One pinned image. The patch fields are the payload's LC_SYMTAB + * addresses: the __rtk_patch records _rtk_stack_guard, RTK_soc, + * RTK_soc_revision, RTK_cpu_physical_address and + * RTK_cpu_wrapper_physical_address, then the type-1 + * __rtk_platform_asc_tunables_block. + */ +struct ane_fw_image { + const char *name; /* request_firmware() path */ + u8 sha256[32]; + u32 size; + struct ane_fw_seg segs[ANE_FW_NSEGS]; + u32 rtk_vm[ANE_FW_RTK_PATCHES]; + u32 tunables_vm; +}; + +static const struct ane_fw_image ane_fw_selene = { + .name = "apple/ane/t602x_ane0_fw_selene_rc4x.macho", + .sha256 = { + 0xa9, 0xc4, 0xb7, 0x71, 0x29, 0x4a, 0x6b, 0x11, + 0x56, 0x24, 0xd9, 0x48, 0x0a, 0x62, 0x48, 0xd0, + 0x89, 0x9a, 0x16, 0x81, 0xa5, 0x75, 0xe8, 0x65, + 0x07, 0x0b, 0x87, 0xa3, 0x24, 0x84, 0x27, 0xbc, + }, + .size = 0x4c5b28, + .segs = { + { 0x000000, 0x0c4000, 0x004000, 0x0c4000, "__TEXT" }, + { 0x0c4000, 0x438000, 0x0c8000, 0x3e8000, "__DATA" }, + }, + .rtk_vm = { 0xca848, 0xca9b3, 0xca9bf, 0xca9cb, 0xca9db }, + .tunables_vm = 0xdcd78, +}; + +static const struct ane_fw_image ane_fw_bia = { + .name = "apple/ane/h14_ane_fw_bia_j4xx.macho", + .sha256 = { + 0xaf, 0x58, 0x7d, 0xfa, 0x96, 0xb1, 0xe0, 0x1d, + 0x2b, 0x2e, 0x0f, 0x9f, 0x77, 0x6e, 0x5d, 0xbe, + 0xfe, 0xbb, 0x44, 0xc7, 0x96, 0x8b, 0x86, 0xff, + 0xf7, 0xd2, 0xb1, 0x9c, 0x4c, 0xbe, 0xf0, 0xdd, + }, + .size = 0x4b5b38, + .segs = { + { 0x000000, 0x0b4000, 0x004000, 0x0b4000, "__TEXT" }, + { 0x0b4000, 0x438000, 0x0b8000, 0x3e8000, "__DATA" }, + }, + .rtk_vm = { 0xba688, 0xba7f3, 0xba7ff, 0xba80b, 0xba81b }, + .tunables_vm = 0xccbb8, +}; + +/* Validate a candidate payload against image FW. Returns 0 on pass, + * else -1 with *reason set. actual_sha = caller-computed SHA-256 of + * blob. + */ +static inline int +ane_fw_validate_blob(const u8 *blob, size_t size, + const struct ane_fw_image *fw, const u8 actual_sha[32], + struct ane_fw_seg *segs_out, u64 *entry_out, + const char **reason) +{ + u32 magic, cputype, filetype, ncmds, sizeofcmds, flags; + u32 off, i, nsegs = 0; + u64 entry = (u64)~0ULL; + + if (size != fw->size) { + *reason = "size != pinned image"; + return -1; + } + if (memcmp(fw->sha256, actual_sha, 32) != 0) { + *reason = "sha256 != pinned payload hash"; + return -1; + } + + magic = get_unaligned_le32(blob + 0); + cputype = get_unaligned_le32(blob + 4); + filetype = get_unaligned_le32(blob + 12); + ncmds = get_unaligned_le32(blob + 16); + sizeofcmds = get_unaligned_le32(blob + 20); + flags = get_unaligned_le32(blob + 24); + + if (magic != MH_MAGIC_64) { + *reason = "magic"; + return -1; + } + if (cputype != CPU_TYPE_ARM64) { + *reason = "cputype"; + return -1; + } + if (filetype != MH_PRELOAD) { + *reason = "filetype != MH_PRELOAD"; + return -1; + } + if (ncmds != ANE_FW_NCMDS || sizeofcmds != ANE_FW_SIZEOF_CMDS || + flags != ANE_FW_FLAGS) { + *reason = "header fields != pinned image"; + return -1; + } + + off = 32; + for (i = 0; i < ncmds; i++) { + u32 cmd, cmdsize; + + if (off + 8 > 32 + sizeofcmds || off + 8 > size) { + *reason = "load command header out of bounds"; + return -1; + } + cmd = get_unaligned_le32(blob + off); + cmdsize = get_unaligned_le32(blob + off + 4); + if (cmdsize < 8 || off + cmdsize > 32 + sizeofcmds || + off + cmdsize > size) { + *reason = "cmdsize out of bounds"; + return -1; + } + + if (cmd == LC_SEGMENT_64) { + u64 vmaddr = get_unaligned_le64(blob + off + 24); + u64 vmsize = get_unaligned_le64(blob + off + 32); + u64 fileoff = get_unaligned_le64(blob + off + 40); + u64 filesize = get_unaligned_le64(blob + off + 48); + const char *sname = (const char *)blob + off + 8; + unsigned int k; + + if (fileoff > size || filesize > size - fileoff) { + *reason = "segment file range out of bounds"; + return -1; + } + if (filesize > vmsize) { + *reason = "filesize > vmsize"; + return -1; + } + for (k = 0; k < ANE_FW_NSEGS; k++) { + if (vmaddr == fw->segs[k].vmaddr && + vmsize == fw->segs[k].vmsize && + fileoff == fw->segs[k].fileoff && + filesize == fw->segs[k].filesize && + !memcmp(sname, fw->segs[k].name, 16)) { + if (segs_out) + segs_out[nsegs] = fw->segs[k]; + break; + } + } + if (k == ANE_FW_NSEGS) { + *reason = "segment != pinned layout"; + return -1; + } + nsegs++; + } else if (cmd == LC_UNIXTHREAD) { + u32 p = off + 8; + + /* thread commands: flavor u32, count u32 + * (count = number of 32-bit words in state) + */ + while (p + 8 <= off + cmdsize) { + u32 flavor = get_unaligned_le32(blob + p); + u32 count = get_unaligned_le32(blob + p + 4); + + if (count > (cmdsize - (p - off) - 8) / 4) { + *reason = "thread state count out of bounds"; + return -1; + } + if (flavor == ARM_THREAD_STATE64) { + /* 33 u64 registers; PC at index 32: + * requires count*4 >= 33*8 bytes + */ + if (count < 66 || + p + 8 + count * 4 > off + cmdsize) { + *reason = "thread state too small for pc"; + return -1; + } + entry = get_unaligned_le64(blob + p + 8 + + 32 * 8); + } + p += 8 + count * 4; + } + } + off += cmdsize; + } + + if (nsegs != ANE_FW_NSEGS) { + *reason = "segment count != 2"; + return -1; + } + if (entry != ANE_FW_ENTRY_PC) { + *reason = "entry pc != 0"; + return -1; + } + + if (segs_out) { + for (i = 0; i < ANE_FW_NSEGS; i++) + segs_out[i] = fw->segs[i]; + } + if (entry_out) + *entry_out = entry; + return 0; +} + +/* + * iBoot runtime patches. iBoot places the image at its Mach-O vm layout + * (TEXT vm 0, DATA vm segs[1].vmaddr) and then writes the fields below. + * The 17 pre-Linux T6021 captures (receipts/2026-10-01-t602x-independent) + * differ from the archive in these bytes only; only the stack guard + * changes between boots. A driver that runs its own copy writes them + * itself. + * + * Each __rtk_patch_* record is {u32 tag, u32 value length, value}. + */ +#define ANE_FW_DATA_BASE_VM 0x423c /* TEXT u64: IOVA of DATA (both images) */ +#define ANE_FW_TUNABLES_MAX 0x24 /* type-1 block capacity (header byte 2) */ + +/* + * iBoot's ASC tunables for the ANE (coprocessor id 4): one record list + * and the chip-revision keys of the entries that carry it, highest + * first. iBoot fills the firmware's type-1 block from the first entry + * whose key <= the chip revision: header {type 1, flags 3 (20-byte + * records), capacity 0x24, count, u32 key}, then {u32 offset, u64 mask, + * u64 value} per record (iBoot table offsets carry flag bit 30; the + * block does not). Source: iBootStage2 macOS 26.6.2 (25G83), LZFSE, not + * encrypted (receipts/2026-10-01-t8112-optin). The 13.5 iBoot2 that + * preloads the 13.5 image is encrypted; on T6021 its block (the + * captures) equals the 25G83 records exactly. + */ +struct ane_asc_tunable { + u32 off; + u64 mask; + u64 val; +}; + +struct ane_asc_tunables { + u8 keys[4]; + u8 nkeys; + u8 n; + struct ane_asc_tunable r[ANE_FW_TUNABLES_MAX]; +}; + +/* T6021, T6022: j414c/j416c/j180d/j475d 25G83, type-1 key 0x10 (the + * 0x150010 record) then type-3 key 0x10 (the rest), which is the T6021 + * capture block byte for byte (header key 0x10). 25G83 adds key 0x11; + * the capture (RTK_soc_revision 0x11, block key 0x10) shows 13.5 has + * none, so it is left out. + */ +static const struct ane_asc_tunables ane_t602x_asc_tunables = { + .keys = { 0x10, 0x01, 0x00 }, + .nkeys = 3, + .n = 24, + .r = { + { 0x150010, 0x000000000000003f, 0x0000000000000010 }, + { 0x051c08, 0x0000000010000000, 0x0000000010000000 }, + { 0x051c18, 0x0010000000000000, 0x0010000000000000 }, + { 0x140020, 0x0000000000f00000, 0x0000000000700000 }, + { 0x140120, 0x0000000000002eff, 0x00000000000002b1 }, + { 0x140130, 0x0000000000003fff, 0x0000000000000018 }, + { 0x140138, 0x0000000000007fff, 0x0000000000000064 }, + { 0x140140, 0x0000000000003fff, 0x000000000000002f }, + { 0x140148, 0x0000000000007fff, 0x00000000000000c8 }, + { 0x140150, 0x0000000000003fff, 0x000000000000004f }, + { 0x140158, 0x0000000000007fff, 0x000000000000014e }, + { 0x140160, 0x0000000000003fff, 0x000000000000004f }, + { 0x140168, 0x0000000000007fff, 0x000000000000014e }, + { 0x140170, 0x0000000000003fff, 0x0000000000000018 }, + { 0x140178, 0x0000000000007fff, 0x0000000000000064 }, + { 0x140180, 0x0000000000003fff, 0x0000000000000028 }, + { 0x140188, 0x0000000000007fff, 0x00000000000000a7 }, + { 0x140190, 0x0000000000003fff, 0x0000000000000028 }, + { 0x140198, 0x0000000000007fff, 0x00000000000000a7 }, + { 0x1401a0, 0x0000000000003fff, 0x000000000000004f }, + { 0x1401a8, 0x0000000000007fff, 0x000000000000014e }, + { 0x145010, 0x000000000008000c, 0x000000000000000c }, + { 0x14a008, 0x000000000ff0ffff, 0x0000000000002520 }, + { 0x14a010, 0x000000000ff0ffff, 0x0000000002002520 }, + }, +}; + +/* T6020: j414s/j416s 25G83, one type-1 entry, key 0x00. The offsets of + * T6021; 13 values differ. + */ +static const struct ane_asc_tunables ane_t6020_asc_tunables = { + .keys = { 0x00 }, + .nkeys = 1, + .n = 24, + .r = { + { 0x150010, 0x000000000000003f, 0x0000000000000010 }, + { 0x051c08, 0x0000000010000000, 0x0000000010000000 }, + { 0x051c18, 0x0010000000000000, 0x0010000000000000 }, + { 0x140020, 0x0000000000f00000, 0x0000000000700000 }, + { 0x140120, 0x0000000000002eff, 0x00000000000002b1 }, + { 0x140130, 0x0000000000003fff, 0x0000000000000018 }, + { 0x140138, 0x0000000000007fff, 0x0000000000000064 }, + { 0x140140, 0x0000000000003fff, 0x0000000000000018 }, + { 0x140148, 0x0000000000007fff, 0x0000000000000064 }, + { 0x140150, 0x0000000000003fff, 0x0000000000000020 }, + { 0x140158, 0x0000000000007fff, 0x0000000000000086 }, + { 0x140160, 0x0000000000003fff, 0x000000000000004f }, + { 0x140168, 0x0000000000007fff, 0x000000000000014e }, + { 0x140170, 0x0000000000003fff, 0x0000000000000008 }, + { 0x140178, 0x0000000000007fff, 0x0000000000000022 }, + { 0x140180, 0x0000000000003fff, 0x0000000000000008 }, + { 0x140188, 0x0000000000007fff, 0x0000000000000022 }, + { 0x140190, 0x0000000000003fff, 0x0000000000000018 }, + { 0x140198, 0x0000000000007fff, 0x0000000000000064 }, + { 0x1401a0, 0x0000000000003fff, 0x0000000000000018 }, + { 0x1401a8, 0x0000000000007fff, 0x0000000000000064 }, + { 0x145010, 0x000000000008000c, 0x000000000000000c }, + { 0x14a008, 0x000000000ff0ffff, 0x0000000000002520 }, + { 0x14a010, 0x000000000ff0ffff, 0x0000000001002520 }, + }, +}; + +/* T8112: j413/j415/j473/j493 25G83, one type-1 table under keys 0x10 + * and 0x00, no type-3 entry. The T6021 offsets without 0x150010. + */ +static const struct ane_asc_tunables ane_t8112_asc_tunables = { + .keys = { 0x10, 0x00 }, + .nkeys = 2, + .n = 23, + .r = { + { 0x051c08, 0x0000000010000000, 0x0000000010000000 }, + { 0x051c18, 0x0010000000000000, 0x0010000000000000 }, + { 0x140020, 0x4000000000f80000, 0x4000000000780000 }, + { 0x140120, 0x0000000000002eff, 0x00000000000002b1 }, + { 0x140130, 0x0000000000003fff, 0x0000000000000018 }, + { 0x140138, 0x0000000000007fff, 0x0000000000000064 }, + { 0x140140, 0x0000000000003fff, 0x0000000000000018 }, + { 0x140148, 0x0000000000007fff, 0x0000000000000064 }, + { 0x140150, 0x0000000000003fff, 0x0000000000000020 }, + { 0x140158, 0x0000000000007fff, 0x0000000000000086 }, + { 0x140160, 0x0000000000003fff, 0x0000000000000028 }, + { 0x140168, 0x0000000000007fff, 0x00000000000000a7 }, + { 0x140170, 0x0000000000003fff, 0x0000000000000010 }, + { 0x140178, 0x0000000000007fff, 0x0000000000000043 }, + { 0x140180, 0x0000000000003fff, 0x0000000000000010 }, + { 0x140188, 0x0000000000007fff, 0x0000000000000043 }, + { 0x140190, 0x0000000000003fff, 0x0000000000000018 }, + { 0x140198, 0x0000000000007fff, 0x0000000000000064 }, + { 0x1401a0, 0x0000000000003fff, 0x0000000000000028 }, + { 0x1401a8, 0x0000000000007fff, 0x00000000000000a7 }, + { 0x145010, 0x000000000008000c, 0x000000000008000c }, + { 0x14a008, 0x000000000ff0ffff, 0x0000000000002520 }, + { 0x14a010, 0x000000000ff0ffff, 0x0000000003002520 }, + }, +}; + +/* T8112 RTK_soc_revision from its two eFuse words at 0x23d2c8060, as + * iBootStage2 25G83 j413 0x40270 (and j415/j473/j493) computes it: + * bits 2:0 = w0[29:27], bits 6:4 = {w1[0], w0[31:30]}. + */ +static inline u32 ane_t8112_fuse_revision(u32 w0, u32 w1) +{ + return ((w0 >> 27) & 7) | ((((w0 >> 30) | (w1 << 2)) & 7) << 4); +} + +struct ane_fw_boot_patch { + u64 exec_base; /* IOVA the ASC runs vm 0 at */ + u64 stack_guard; + u32 soc; + u32 soc_revision; + u64 cpu_pa; /* ASC core block, engine + 0x1000000 */ + u64 wrapper_pa; /* ASC wrapper, engine + 0x1400000 */ +}; + +static inline void ane_fw_put_le(u8 *p, u64 v, unsigned int n) +{ + while (n--) { + *p++ = (u8)v; + v >>= 8; + } +} + +/* Patch the staged image FW in place (vm-indexed, at least + * ANE_FW_BUF_SIZE bytes). Every precondition is checked before the first + * write, so a refusal leaves the image untouched. Returns 0, or -1 with + * *reason set. + */ +static inline int +ane_fw_apply_boot_patches(u8 *img, const struct ane_fw_image *fw, + const struct ane_asc_tunables *t, + const struct ane_fw_boot_patch *p, const char **reason) +{ + static const u8 tunables_unset[8] = { + 0x01, 0x03, ANE_FW_TUNABLES_MAX, 0x00, 0xff, 0xff, 0xff, 0xff }; + static const struct { u32 tag, len; } rec[ANE_FW_RTK_PATCHES] = { + { 0x53544b47, 8 }, /* _rtk_stack_guard */ + { 0x534f435f, 4 }, /* RTK_soc */ + { 0x534f4352, 4 }, /* RTK_soc_revision */ + { 0x43704164, 8 }, /* RTK_cpu_physical_address */ + { 0x57724164, 8 }, /* ..._wrapper_physical_address */ + }; + const u64 val[ANE_FW_RTK_PATCHES] = { + p->stack_guard, p->soc, p->soc_revision, p->cpu_pa, + p->wrapper_pa, + }; + u8 *blk = img + fw->tunables_vm; + unsigned int i, k; + + for (k = 0; k < t->nkeys && t->keys[k] > p->soc_revision; k++) + ; + if (k == t->nkeys || t->n > ANE_FW_TUNABLES_MAX) { + *reason = "no ASC tunables for this chip revision"; + return -1; + } + if (get_unaligned_le64(img + ANE_FW_DATA_BASE_VM)) { + *reason = "DATA base field is not the archive zero"; + return -1; + } + for (i = 0; i < ANE_FW_RTK_PATCHES; i++) { + if (get_unaligned_le32(img + fw->rtk_vm[i]) != rec[i].tag || + get_unaligned_le32(img + fw->rtk_vm[i] + 4) != rec[i].len) { + *reason = "__rtk_patch record != pinned image"; + return -1; + } + } + if (memcmp(blk, tunables_unset, 8)) { + *reason = "tunables block header != pinned image"; + return -1; + } + + ane_fw_put_le(img + ANE_FW_DATA_BASE_VM, + p->exec_base + fw->segs[1].vmaddr, 8); + for (i = 0; i < ANE_FW_RTK_PATCHES; i++) + ane_fw_put_le(img + fw->rtk_vm[i] + 8, val[i], rec[i].len); + blk[3] = t->n; + ane_fw_put_le(blk + 4, t->keys[k], 4); + for (i = 0; i < t->n; i++) { + ane_fw_put_le(blk + 8 + 20 * i, t->r[i].off, 4); + ane_fw_put_le(blk + 12 + 20 * i, t->r[i].mask, 8); + ane_fw_put_le(blk + 20 + 20 * i, t->r[i].val, 8); + } + return 0; +} +#endif /* __ANE_FW_VALIDATE_H__ */ diff --git a/drivers/accel/ane/ane_t6021.h b/drivers/accel/ane/ane_t6021.h new file mode 100644 index 0000000000000..5e0ecc6d471f7 --- /dev/null +++ b/drivers/accel/ane/ane_t6021.h @@ -0,0 +1,796 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* T6021 (H14 / J414c) Apple Neural Engine — skeleton constants and types. + * + * Every constant below is static-decode provenance, no device writes: + * - reg windows, IRQ, pmgr islands: ADT j414c (DeviceTree.j414cap.im4p, + * receipts/2026-09-18-t6021-engine-layout-mined §2), Linux translation + * +0x200000000 (proven class, pmgr low-32 match). + * - ASC cpu block (+0x1400000 h14g) + MBI transport (SCRATCH, channel + * table, +0x1844000 doorbell): K14 kext disasm, W4-fix receipt + * (initializeANESoCConfig h14g blocks + InitializeRTBuddy + + * doorbell setter 0x…95ebdd0); m1n1 ASCRegs = t8103 cross-ref only + * - RTKit MGMT protocol: Asahi rtkit.c semantics, u64 message halves as + * staged in omarchy-ane rtkit/h14_rtkit_hello.py (commit 6ad26b7). + * - RTBuddy endpoint table + doorbell word: K14 kext cfg table + * __DATA_CONST.__const+0x814e520 and HandleRTBuddyMessage + * (receipts/2026-09-18-h14-w2-protocol-decode §3). + * + * DT binding (driver + packaging/dt/t6021-ane.dts are the two halves): + * compatible = "apple,t6021-ane" (also "apple,t6020-ane", the + * T6022 die-0 "apple,t6022-ane" and "apple,t8112-ane": + * struct ane_t602x_soc). The T6021 values follow; T8112 + * (packaging/dt/t8112-ane.dts) has engine 0x26a000000, + * pmgr 0x23b700000+0x18000, set 0x23b724000+0x4000, + * IRQ 520, seven power states 0xc008..0xc038, and a + * fourth window "fuse" (0x23d2c8060+8, chip revision). + * reg/reg-names = "engine" (whole 32 MiB ADT range0, 0x284000000; + * the H13-style +0x1c04000 engine delta does not exist + * on this SoC — kext never computes it and first touch + * external-aborted, proven 2026-09-18), + * "pmgr" (0x28e080000+0x4034, pmgr1,t6021 island words), + * "set" (0x28e08c000+0x4000, SET window — read-only by + * repo rule: direct SET writes external-abort the SoC) + * interrupts = one AIC2 level-high interrupt named "ane" (raw 884; + * [INFERENCE W2 §3: the ANE MBI doorbell IRQ]. + * dart-ane0 carries raw 885, provider-owned, never + * fetched here) + * iommus = dart-ane0 streams (apple,t6020-dart/apple,t8110-dart + * nodes). The ADT dart-ane0 is one hardware block with a + * four-window quartet (0x85800000/85810000/85820000/ + * 85804000, each 0x4000); the installed overlay's + * three-node split (t6001-proven pattern) is kept. + * power-domains = eight, phase1 raise order (sys_mpm→td→base→set1..4, + * ane_cpu last): ane_sys_mpm@4000, ane_td@4008, + * ane_base@4010, ane_set1..4@4018-4030, ane_cpu@2e0. + * ane_td/ane_base were the W3 chain gap (six consumed, + * block access reset the machine); receipt + * 2026-09-19-h14-init-sequence-kext-trace §3. + */ + +#ifndef __ANE_T6021_H__ +#define __ANE_T6021_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ane_t6021_boot.h" + +/* reg windows (packaging/dt/t6021-ane.dts reg-names order) */ +enum { + ANE_T6021_REG_ENGINE, + ANE_T6021_REG_PMGR, + ANE_T6021_REG_SET, + ANE_T6021_REG_COUNT +}; + +/* pmgr ps-word fields (apple-pmgr-pwrstate layout). */ +#define ANE_PS_ON 0xf +#define ANE_PS_TARGET GENMASK(3, 0) +#define ANE_PS_ACTUAL GENMASK(7, 4) +#define ANE_PS_WAS_GATED GENMASK(9, 8) +#define ANE_PS_BUSY BIT(11) +#define ANE_PS_AUTO_ENABLE BIT(28) + +/* ANE-block-relative ASC/RTBuddy addresses. CPU block = ANE+0x1400000 + * for h14g (K14 initializeANESoCConfig h14g blocks 0x…961365c / + * 0x…9614084 build 0x1400044 with NO 0x200000 orr — the 0x1600044 + * phase-1 §2 quote is the h16g/h17/h18g variant block 0x…9613458; + * runtime h14g-shaped sites 0x…95d2968 write RUN there). CPU_STATUS + * +0x48 keeps the m1n1 ASCRegs +4 shape (config field dev+0x49c = + * 0x1400048; kext poll 0x…95ecfb4). + * + * W10 (2026-09-19) CORRECTS the claim that stood here — that the m1n1 + * t8103 mailbox (INBOX_CTRL +0x8110 / INBOX0 +0x8800) "has NO h14g + * analog". It has one, and it is live: see ANE_ASC_MBOX_* below. The + * earlier live test read +0x1608114, the h16g base, which is simply + * the wrong address on this part; and the absence of 0x1408xxx + * constants from the kext text means only that the mailbox belongs to + * the RTBuddy provider kext, not to AppleH11ANEInterface. + * + * W10 also read the CPU block for the first time, on a Linux boot with + * all eight pmgr domains at ACTUAL=0xf and the W8 grant applied: + * CPU_CONTROL 0x285400044 = 0x00000000 RUN=0 + * CPU_STATUS 0x285400048 = 0x0000002a RUNNING=0 STOPPED=1 IDLE=1 + * RVBAR 0x285050000 = 0x00000001 valid bit set, entry addr 0 + * The IOP has never been started and no firmware image is programmed, + * so nothing on this transport can answer the host until it is. + */ +#define ANE_ASC_CPU_CONTROL 0x1400044 /* RUN = BIT(4) */ +#define ANE_ASC_CPU_STATUS 0x1400048 /* m1n1 R_CPU_STATUS shape */ +#define ANE_ASC_RVBAR 0x1050000 /* fw entry | valid bit0 */ +#define ANE_ASC_EDPRCR 0x1010310 /* phase1 S2 whitelist */ +#define ANE_ASC_VERS 0x1840000 +#define ANE_ASC_RTB_STATUS 0x1840088 /* K14 poll: value < 2 */ +#define ANE_ASC_RTB_STATUS_UNK7C 0x184007c /* phase1 S2 whitelist */ + +/* ASC blocks iBoot names in the firmware's __rtk_patch records + * (RTK_cpu_physical_address 0x285000000 and RTK_cpu_wrapper_physical_address + * 0x285400000 on the T6021 preload), as engine-relative offsets. + */ +#define ANE_ASC_CPU_BASE 0x1000000 +#define ANE_ASC_WRAPPER_BASE 0x1400000 + +/* The real ASC mailbox, mapped live by W10 (read-only) at ASC+0x8000. + * Three independent sources agree on the offsets: the live DT sibling + * mbox@2a2408000 ("apple,t6020-asc-mailbox" / "apple,asc-mailbox-v4", + * size 0x4000, IRQs send-empty/send-not-empty/recv-empty/ + * recv-not-empty) fixes mailbox = asc_base + 0x8000; upstream + * drivers/soc/apple/mailbox.c apple_mbox_asc_hw fixes the register + * offsets; m1n1 proxyclient/m1n1/hw/asc.py ASCRegs gives the same set + * ASC-relative plus the R_MBOX_CTRL field layout. + * + * Observed on jw14m2 (2026-09-19), both directions identical: + * a2i_control 0x285408110 = 0x00020001 + * i2a_control 0x285408114 = 0x00020001 + * -> ENABLE=1 EMPTY=1 FULL=0 OVERFLOW=0 FIFOCNT=0 WPTR=0 RPTR=0 + * Enabled, and never used: both pointer pairs are still at the origin, + * so not one message has crossed either way since reset. 600 polls of + * i2a_control over 30 s produced zero changes. + * + * Receive is POP-ON-READ and 64-bit: upstream does + * while (!(i2a_control & EMPTY)) { readq(RECV0); readq(RECV1); } + * A 32-bit or memcpy-based read would eat messages. + */ +#define ANE_ASC_MBOX 0x1408000 +#define ANE_ASC_MBOX_A2I_CTRL 0x1408110 /* m1n1 INBOX_CTRL */ +#define ANE_ASC_MBOX_I2A_CTRL 0x1408114 /* m1n1 OUTBOX_CTRL */ +#define ANE_ASC_MBOX_A2I_SEND0 0x1408800 /* m1n1 INBOX0, u64 */ +#define ANE_ASC_MBOX_A2I_SEND1 0x1408808 /* m1n1 INBOX1, u64 */ +#define ANE_ASC_MBOX_I2A_RECV0 0x1408830 /* m1n1 OUTBOX0, u64 */ +#define ANE_ASC_MBOX_I2A_RECV1 0x1408838 /* m1n1 OUTBOX1, u64 */ + +/* R_MBOX_CTRL fields (m1n1 hw/asc.py) */ +#define ANE_ASC_MBOX_CTRL_FIFOCNT GENMASK(23, 20) +#define ANE_ASC_MBOX_CTRL_OVERFLOW BIT(18) +#define ANE_ASC_MBOX_CTRL_EMPTY BIT(17) +#define ANE_ASC_MBOX_CTRL_FULL BIT(16) +#define ANE_ASC_MBOX_CTRL_RPTR GENMASK(15, 12) +#define ANE_ASC_MBOX_CTRL_WPTR GENMASK(11, 8) +#define ANE_ASC_MBOX_CTRL_ENABLE BIT(0) + +/* W10 hazard: a plain READ of ANE+0x1854000 is fabric-fatal (watchdog + * reset ~63 s later, netconsole pinned the pre-log and no value line). + * Nothing may touch 0x1854000..0x1c04000; the old engine kill window + * 0x1c04000..0x1c28000 still stands. + */ +#define ANE_FATAL_READ_LO 0x1854000 +#define ANE_FATAL_READ_HI 0x1c04000 + +/* MBI transport (K14 h14g config: socinit stores the SCRATCH register + * offsets as q-blobs @0x…7503a40/0x…7503a50/0x…7503a60 into dev+0x438 + * and the message-register offsets @0x…7503860 into dev+0x488; runtime + * handshake in InitializeRTBuddy 0x…95e942c: cmd buffer base -> + * SCRATCH0/1 (0x…95eaa94), wake 0xf7fbdff9 -> SCRATCH7 (0x…95eab24), + * poll SCRATCH7 == 0x08042006 "channel description table ready" + * (0x…95eab74), table base read back from SCRATCH0/1 (0x…95ead04), + * per-channel {type,bit,size,phys} entries registered with the + * doorbell setter 0x…95ebdd0 writing (1 << bit) to +0x1844000. Host + * ack = SCRATCH3 = 0x08042006 (0x…95eaee4). + * + * PROVIDER DECODE (2026-09-19, kernelcache.release.mac14j): in + * RTBuddy mode the TX gate lives in com.apple.driver.RTBuddy 1.0.0 + * (carved from the KC, 46088 B, __TEXT_EXEC 0x…b696860). RTBuddy + * publishes per-ANE nubs named "%sEndpoint%u" (0x…7cc7d3f) with the + * role string "ANE" -> ANEEndpoint1..ANEEndpoint5, class + * RTBuddyEndpointService (0x98 B) wrapping RTBuddyEndpoint (0xE8 B), + * over the kernel IOSlaveEndpoint family; K14 EnableRTBuddyEndpoints + * (0x…95feb30) formats the name ("%s%d" @0x…74c481f), waits 10 s + * (x1 = 10^10 ns) for the match and takes the gate from [svc+0x88], + * registering its rx callback at gate+0xd0. The gate's per-message + * send is the vtable+0x1e8 slot; its body ends in the AKF mailbox + * write (AKF_AP_MAILBOX_SET = the +0x1844000 doorbell; RTBuddy dump + * fn 0x…b69d9c8 prints "AKF_KIC_INBOX_CTRL / AKF_KIC_MAILBOX_SET / + * AKF_AP_OUTBOX_CTRL / AKF_AP_MAILBOX_SET" and guards + * "INBOX%d not ready"/"Inbox%d overflow"). Endpoint number = + * doorbell bit: EP0 is RTBuddyManagementEndpoint (_handleHello / + * _handleEPRollCall / _handlePowerAck), EP1..5 the ANE data channels. + * [INFERENCE: the fw->host doorbell/IRQ bit numbering mirrors the + * host->fw SET bit per Asahi rtkit semantics; pinned live by W5.] + */ +#define ANE_MBI_SCRATCH0 0x1840048 /* SCRATCH0..7 = +0x48..+0x64 */ +#define ANE_MBI_SCRATCH6 0x1840060 +#define ANE_MBI_SCRATCH7 0x1840064 +/* Wake/ack handshake words live in ane_t6021_boot.h + * (ANE_T6021_BOOT_WAKE_REQ / ANE_T6021_BOOT_ACK) — single source, shared + * with the userspace boot regression. + */ +#define ANE_MBI_DOORBELL 0x1844000 /* write32 (1 << endpoint id) */ +/* NOT a message pair. W10 proved this is a mirror of the 24 MHz + * architectural counter: across 32 samples the absolute difference + * against CNTVCT_EL0 is a constant 13-18 counts (the MRS-to-MMIO + * instruction gap), aggregate drift 0.054 ppm over 55.2M counts. The + * "type-0 heartbeat, ~29 ticks/3 s" that W6/W9 recorded as firmware + * liveness was this clock plus a poll count: the RTKit type field + * GENMASK_ULL(59,52) is structurally zero until the counter passes + * 2^52 (~6 years of uptime). Kept only so the old reads stay + * identifiable in the logs; do not treat as transport. + */ +#define ANE_MBI_TIMEBASE_LO 0x1170000 /* was ANE_MBI_MSG_I2A_LO */ +#define ANE_MBI_TIMEBASE_HI 0x1170004 /* was ANE_MBI_MSG_I2A_HI */ +#define ANE_MBI_MSG_I2A_LO ANE_MBI_TIMEBASE_LO +#define ANE_MBI_MSG_I2A_HI ANE_MBI_TIMEBASE_HI +#define ANE_MBI_MSG_A2I_RD 0x184c000 /* host->fw message read peer */ +#define ANE_MBI_MSG_A2I_WR 0x1850000 /* host->fw message write */ + +/* Per-message MBI word (48-bit ring notification, K14 + * rtbuddyEndpointSendMessage 0x…95f3bf4-c30): offset = ring write + * cursor [23:0], length [47:24]. NOT the 54-bit surface-announce + * word (ane_ep_doorbell_encode below) — that one rides SetupEndpoints + * buffer mapping; this one is what the gate sends per command: the + * gate call is send(&msg48, 0, 1) at vtable+0x1e8, after the command + * bytes are already memcpy'd into the shared ring at ring_base + + * cursor. Length is capped at 0xffffff by the encoder field. + */ +#define ANE_MBI_MSG48_OFF GENMASK_ULL(23, 0) +#define ANE_MBI_MSG48_LEN GENMASK_ULL(47, 24) + +static inline u64 ane_mbi_msg48_encode(u32 cursor, u32 len) +{ + return (cursor & ANE_MBI_MSG48_OFF) | + FIELD_PREP(ANE_MBI_MSG48_LEN, len); +} + +/* Host->fw TX sequence (RTBuddy mode, all static-decode proven): + * 1. fail if len > ring_size (K14 0xe00002c2 @0x…95f3a24) + * 2. cursor = write_cursor; if (cursor + len >= ring_size) cursor = 0 + * (exact fit wraps: csel lo @0x…95f3b04) + * 3. memcpy(ring + cursor, cmd, len) (DMA-coherent ring) + * 4. dma_wmb() (ring visible before bell) + * 5. write32(ANE_MBI_MSG_A2I_WR, lo) + write32(+4, hi) — 32-bit + * halves (the AKF message register is word-shaped; W6 proved the + * W5-live abort was NOT the 64-bit writeq — the halves abort too) + * 6. write32(ANE_MBI_DOORBELL, 1 << ep) + * 7. on send success only: write_cursor = cursor + len (K14 + * 0x…95f3c70-c); the cursors at rec+0x40/rec+0x58 track the last + * issued slot. + * CSNE_CMD_PING = header-only 0x11 on EP1 (INIT) -> doorbell bit + * 1 << 1 = 0x2. Only the a2i message register + doorbell are + * touched, and only behind the mbi_doorbell opt-in (both stay + * host-write-fatal even behind the W8 grant, W9; SCRATCH family is + * host-writable granted, W9 nonzero latch). + */ + +/* MBI channel-table entry (kext stride 0x100, fields at +0x40 family: + * type @+0x40, doorbell bit @+0x44, size @+0x48, phys @+0x50 — + * InitializeRTBuddy 0x…95eade8-0x…95eae58) + */ +#define ANE_MBI_CHAN_STRIDE 0x100 +#define ANE_MBI_CHAN_MAX_DUMP 8 + +/* RTKit MGMT (EP 0); type bits [59:52], u64 message halves */ +#define ANE_RTKIT_TYPE GENMASK_ULL(59, 52) +#define ANE_RTKIT_MGMT_HELLO 1 +#define ANE_RTKIT_MGMT_HELLO_REPLY 2 +#define ANE_RTKIT_MGMT_STARTEP 5 +#define ANE_RTKIT_MGMT_SET_IOP_PWR_STATE 6 +#define ANE_RTKIT_MGMT_SET_IOP_PWR_STATE_ACK 7 +#define ANE_RTKIT_MGMT_EPMAP 8 +#define ANE_RTKIT_MGMT_SET_AP_PWR_STATE 0xb +#define ANE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK 0xb + +#define ANE_RTKIT_HELLO_MINVER GENMASK_ULL(15, 0) +#define ANE_RTKIT_HELLO_MAXVER GENMASK_ULL(31, 16) +#define ANE_RTKIT_EPMAP_LAST BIT_ULL(51) +#define ANE_RTKIT_EPMAP_BASE GENMASK_ULL(34, 32) +#define ANE_RTKIT_EPMAP_BITMAP GENMASK_ULL(31, 0) +#define ANE_RTKIT_EPMAP_REPLY_MORE BIT_ULL(0) +#define ANE_RTKIT_STARTEP_EP GENMASK_ULL(39, 32) +#define ANE_RTKIT_STARTEP_FLAG BIT_ULL(1) +#define ANE_RTKIT_PWR_STATE GENMASK_ULL(15, 0) +#define ANE_RTKIT_PWR_STATE_ON 0x20 + +#define ANE_RTKIT_VER_MIN 11 +#define ANE_RTKIT_VER_MAX 12 + +/* RTKit system endpoints rtkit.c starts when announced */ +#define ANE_RTKIT_EP_CRASHLOG 1 +#define ANE_RTKIT_EP_SYSLOG 2 +#define ANE_RTKIT_EP_DEBUG 3 +#define ANE_RTKIT_EP_IOREPORT 4 +#define ANE_RTKIT_EP_OSLOG 8 +#define ANE_RTKIT_EP_TRACEKIT 0xa + +/* RTBuddy app endpoints — K14 InitializeRTBuddyEndpoints opens ids 1..6; + * ring sizes + fourccs from the per-EP config table (W2 §3). fourcc is + * byte-reversed in the table (0x54324643 = "T2FC"). + */ +enum ane_t6021_eps { + ANE_T6021_EP_INIT = 1, /* INIT — CSNE_CMD controller channel (W4) */ + ANE_T6021_EP_T2FC, /* fw->host commands */ + ANE_T6021_EP_T2FH, /* fw->host commands */ + ANE_T6021_EP_T2HS, + ANE_T6021_EP_T2HC, + ANE_T6021_EP_T2HT, /* polled on the host */ + ANE_T6021_EP_COUNT = ANE_T6021_EP_T2HT + 1 /* arrays index by id */ +}; + +/* CSNE command ids — selene id->name table, vaddr 0xea430, 96 entries + * (full set: receipts/2026-09-18-h14-w2-protocol-decode §4 and + * receipts/2026-09-18-h14-w2-protocol-decode/fw_cmd_table.json). The + * fw parses the id as u16 at wire offset +4 (W4 correction of the W2 + * §4 "offset 0" claim: fw sites 0x4d244/0x4d264/0x5246c); the kext + * controller header for the fw->host direction is a different shape + * (u32 id @ +0x8, 0x24 bytes) and never rides host->fw submission. + */ +enum ane_t6021_csne_cmd { + CSNE_CMD_START = 0x0000, + CSNE_CMD_STOP = 0x0001, + CSNE_CMD_REG_FILE_LOAD = 0x0005, /* fw _rtk_tunables 1456 B */ + CSNE_CMD_BUILDINFO = 0x0006, + CSNE_CMD_BOOT = 0x0010, + CSNE_CMD_PING = 0x0011, + CSNE_CMD_POWER_DEVICE_ON = 0x0013, + CSNE_CMD_IPC_ENDPOINT_SET = 0x0015, + CSNE_CMD_IPC_ENDPOINT_UNSET = 0x0016, + CSNE_CMD_LOAD_PROGRAM = 0x0200, + CSNE_CMD_CREATE_PROCESS = 0x0202, + CSNE_CMD_PROCEDURE_CALL = 0x0204, + CSNE_CMD_INFERENCE_CALL = 0x0404, + CSNE_CMD_BACK_CHANNEL_RPC = 0x7000, +}; + +/* fw buffer word (selene builder 0x982b4/0x98f84, decoder 0x98fc8; the + * K14 SetupEndpoints word is the same shape): addr[43:0] | + * size_code[51:44] | unit[53:52], unit 0 = no size, 1 = code*4K, + * 2 = code*1M, 3 = code*2M. The fw encoder picks unit 1 below 1 MiB + * and unit 2 at or above, size code = CEILING division (cinc on the + * remainder), so the decoded size is the rounded-up value. rtkit.c's + * BUFFER_REQUEST is this word with unit 1. + */ +#define ANE_EP_DOORBELL_OFFSET GENMASK_ULL(43, 0) +#define ANE_EP_DOORBELL_SIZE GENMASK_ULL(51, 44) +#define ANE_EP_DOORBELL_UNIT GENMASK_ULL(53, 52) + +static const u8 ane_ep_doorbell_shift[] = { 0, 12, 20, 21 }; + +static inline u64 ane_ep_doorbell_encode(u64 offset, u32 size) +{ + u64 unit = (size >= SZ_1M) ? 2 : 1; + u32 code = DIV_ROUND_UP(size, 1u << ane_ep_doorbell_shift[unit]); + + return (offset & ANE_EP_DOORBELL_OFFSET) | + FIELD_PREP(ANE_EP_DOORBELL_SIZE, code) | + FIELD_PREP(ANE_EP_DOORBELL_UNIT, unit); +} + +static inline u32 ane_ep_doorbell_size(u64 msg) +{ + u32 unit = FIELD_GET(ANE_EP_DOORBELL_UNIT, msg); + + return FIELD_GET(ANE_EP_DOORBELL_SIZE, msg) << ane_ep_doorbell_shift[unit]; +} + +struct ane_t6021_ep { + u8 id; + const char *name; + u32 fourcc; + u32 ring_size; + u32 write_cursor; /* next ring slot (K14 rec+0x20) */ + void *ring; /* dma_alloc_coherent, ring_size */ + dma_addr_t ring_iova; + bool started; +}; + +struct reset_control; + +struct ane_t6021 { + struct device *dev; + void __iomem *base[ANE_T6021_REG_COUNT]; + int irq; + + struct device **pd_dev; + struct device_link **pd_link; + int pd_count; + + /* Single MBI consumer (threaded IRQ + probe drain serialize + * here) + */ + struct mutex mbox_lock; + + /* Bring-up state machine — honest semantics (W15): + * power_gated: first_resume passed (eight-island gate + + * whitelist; block access proven-safe), + * cpu_started: RVBAR programmed + CPU RUN released (fw_boot=1), + * fw_alive: fw first-alive ack observed on SCRATCH7 — NOT + * the init handshake, + * booted: full init handshake observed. CSNE/MGMT sessions + * gate on this flag only. + */ + bool power_gated; + bool cpu_started; + bool fw_alive; + bool booted; + + /* W10: the mailbox does exist — at +0x1408xxx, not the h16g + * +0x1608xxx that read-aborted in 2026-09-19 — and it reads a + * live, enabled, permanently-empty ASC v4 control pair. It is + * empty because the ASC CPU is STOPPED with RUN clear and RVBAR + * entry 0, i.e. no firmware was ever started. Default off = + * status-only bring-up. Opt-in runs the kext-evidenced MBI + * handshake (SCRATCH wake -> fw channel table), capture-only. + */ + bool transport; + bool doorbell; /* mbi_doorbell=1: EP rings may write the +0x1844000 + * doorbell + a2i message register (decoded 2026-09-19) + */ + bool irq_requested; + + struct ane_t6021_ep ep[ANE_T6021_EP_COUNT]; + + /* W13 fw surface (fw_load=1): coherent, DART-mapped via the + * device's iommu group. NULL unless loaded. + */ + void *fw_buf; + dma_addr_t fw_iova; + u32 fw_size; + /* W16 entry alias: IOVA the fw pages are aliased at (the latched + * RVBAR entry); 0 = no alias mapped. The DMA allocator has no + * runtime reservation API on this kernel, so instead of a prose + * collision bound every DMA allocation site must pass its iova + * through ane_t6021_fw_alias_iova_ok() and refuse overlaps. + */ + u64 fw_alias_iova; + /* Per-window mapped extents (iova, bytes) — teardown unmaps + * exactly these, because the windows are not assumed adjacent + * (live trace: hole between SEG0 and SEGi) and unmapping a page + * that was never mapped trips dart_unmap_pages (io-pgtable-dart + * .c:319 WARN, 2026-09-26 state-report unwind). Set only on + * successful map. + */ +#define ANE_FW_ALIAS_MAX_WIN 3 + u64 fw_alias_ext_iova[ANE_FW_ALIAS_MAX_WIN]; + size_t fw_alias_ext_len[ANE_FW_ALIAS_MAX_WIN]; + int fw_alias_extn; + /* W16 pass-3: ane_cpu reset controller (ps RESET via the pmgr + * pwrstate reset_controller ops); NULL when the DT carries no + * resets property. + */ + struct reset_control *cpu_rst; + + /* W15 boot allocations (gate-gated: never exist until the + * preflight opens and the sequence passes poll A). Ownership per + * the wedged-pin rule: held, never freed, while cpu_started. + */ + void *boot_pool; /* 'DDM ' pool, 0x40000 (Params word0) */ + dma_addr_t boot_pool_iova; + void *boot_ipc; /* 'IPC ' surface, max(0x4000, ord+1) */ + dma_addr_t boot_ipc_iova; + u64 boot_ipc_size; + void *boot_heap; /* fw-requested HEAP surface or NULL */ + dma_addr_t boot_heap_iova; + u64 boot_heap_size; + u32 prev_fw_len; /* [0x30]: previous fw image length + * (0 first boot; updated per reload) + */ + u64 boot_scratch_result; /* SCRATCH1<<32 | SCRATCH0 captured + * at DONE — raw; success semantics + * UNSOURCED (never inferred) + */ + bool response_validated; + bool hybrid_pinned; /* FALSE until the DONE response + * semantics are sourced AND the + * raw address is range/length + * validated against owned + * windows — transport/CSNE + * sessions stay fenced on first + * boot regardless of booted + */ +}; + +/* Enforced alias-window invariant (W16): the DMA allocator has no + * runtime IOVA reservation on this kernel, so every DMA allocation + * site must refuse a mapping that overlaps the fw entry alias. + */ +static inline bool ane_t6021_fw_alias_iova_ok(const struct ane_t6021 *ane, + dma_addr_t iova, size_t size) +{ + u64 lo = ane->fw_alias_iova; + + if (!lo) + return true; + return iova + size <= lo || iova >= lo + ane->fw_size; +} + +/* ane_t6021_rtkit.c */ +int ane_t6021_rtkit_init(struct ane_t6021 *ane); +void ane_t6021_rtkit_shutdown(struct ane_t6021 *ane); +void ane_t6021_rtkit_drain(struct ane_t6021 *ane); +irqreturn_t ane_t6021_rtkit_irq_thread(int irq, void *data); + +/* ane_t6021_boot.c — W15 boot state resolution (fw_boot=1). An + * explicit boot request fails the probe while the prerequisites in + * ane_t6021_boot.c hold (-ENODATA); fw_boot=0 binds status-only. + * ane_t6021_boot_start() is the sequence dispatcher proper (all gates + * must already hold): it owns the wedged-pin module lifetime once the + * CPU is released; stop_after = 0 full run, 1..4 = fw-start-debug + * step bisect (stop after that step, -ECANCELED, clean unwind while + * no CPU started). + */ +int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, + int rtb_mode); + +/* ---- CSNE_CMD wire structs (host->fw on the INIT channel) ---- + * + * W4 static decode, extending W2 §4 (the receipt's "fw parses a u16 id + * at header offset 0" was WRONG — three independent fw parse sites read + * the id as u16 at offset +4: the generic CSNE processor @0x4d264, its + * preload @0x4d244, and the LOAD_PROGRAM/CREATE_PROCESS/PROCEDURE_CALL + * pre-parse @0x5246c, all `ldrh wN, [xM, #4]` in selene). Wire is + * little-endian; this module is arm64-only so host order is wire order. + * + * The generic processor takes commands < 0x1b89 bytes (work-item size + * bound @0x4d134-0x4d158) and writes completion state back into the + * command block (strb @0x4d324 byte +6, str @0x4d3b0 qword +8) — the + * ring slot doubles as the response area. PING/BUILDINFO/BOOT/ + * REG_FILE_LOAD/IPC_ENDPOINT_SET fall through the processor's id tree + * to the default path (0x4e65c): carried without field parsing, so + * their payloads beyond the header are opaque until the W1 live + * exchange pins them. + */ +struct ane_csne_hdr { + u32 rsvd0; /* bytes 0..3: never read by the fw processor; 0 */ + u16 id; /* PROVEN u16 @ +4 (sites above) */ + u8 flags; /* fw-written byte @ +6 (status/scratch) [INFERENCE] */ + u8 rsvd7; +}; + +static_assert(sizeof(struct ane_csne_hdr) == 8); + +static inline void ane_csne_hdr_init(struct ane_csne_hdr *h, u16 id) +{ + memset(h, 0, sizeof(*h)); + h->id = id; +} + +/* Header-only CSNE_CMDs: BOOT (0x10 — boot-arg surfaces ride + * SCRATCH0-7 / SetupFWInitBootArgs, not the command, phase1 §2.5), + * PING (0x11), BUILDINFO (0x06). sizeof(struct ane_csne_hdr) bytes. + */ + +/* REG_FILE_LOAD (0x05) payload: the 1456 B blob is the fw's own + * __DATA._rtk_tunables section (@0x100590, size 0x5b0) — its transport + * (inline vs shared-memory iova) is [INFERENCE], pinned by W1. + */ +struct ane_csne_cmd_reg_file_load { + struct ane_csne_hdr hdr; + u8 blob[]; +}; + +/* IPC_ENDPOINT_SET (0x15) payload: binds a host RTBuddy endpoint to a + * fw channel. No field parsed by the fw processor — layout + * [INFERENCE], pinned by W1. + */ +struct ane_csne_cmd_ipc_endpoint_set { + struct ane_csne_hdr hdr; + u8 payload[]; +}; + +/* PROCEDURE_CALL (0x204) / INFERENCE_CALL (0x404) — the kext + * CANE_SUB_PACKET_CMD_PROCEDURE_CALL family. Field offsets PROVEN from + * the fw parse; field semantics named only where the kext asserts name + * them (programId, procedureId, numIoBuffers in assert order, W2 §4): + * +0x08/+0x0c u32 pair validated together (ldp @0x4d654) + * +0x10 u64 passed to the fw validator (0x4d6d4) + * +0x18 u32 stats/priority, fw-valid [2,7]; 2 is the value + * proven end-to-end (binding.json stats_type; the + * first-inference receipt's bare calls) + * +0x28 u32 element count (loop bound @0x524bc-0x524c4) + * +0x60 count × 0x30-byte io-buffer records (@0x524a8-0x524b4) + * The kext `size % sizeof(sCSneCmdProcedureCall) == 0` assert says + * commands form same-shape arrays in the ring; this driver submits one + * command per slot. INFERENCE_CALL shares the shape [INFERENCE: the + * 0x404 id is not in the decoded id tree — it is the W4 submission + * endpoint per the phase-1 workstream plan]. + */ +struct ane_csne_io_elem { + u8 bytes[0x30]; /* internal layout not decoded */ +}; + +struct ane_csne_cmd_procedure_call { + struct ane_csne_hdr hdr; + u32 program_id; /* +0x08 */ + u32 procedure_id; /* +0x0c */ + u64 field_10; + u32 field_18; /* fw requires 8..15 */ + u32 rsvd_1c; + u64 field_20; + u32 num_io_buffers; /* +0x28 = element count */ + u32 rsvd_2c; + u8 gap_30[0x30]; + struct ane_csne_io_elem io[]; +}; + +static_assert(offsetof(struct ane_csne_cmd_procedure_call, program_id) == 0x08); +static_assert(offsetof(struct ane_csne_cmd_procedure_call, procedure_id) == 0x0c); +static_assert(offsetof(struct ane_csne_cmd_procedure_call, field_10) == 0x10); +static_assert(offsetof(struct ane_csne_cmd_procedure_call, field_18) == 0x18); +static_assert(offsetof(struct ane_csne_cmd_procedure_call, num_io_buffers) == 0x28); +static_assert(offsetof(struct ane_csne_cmd_procedure_call, io) == 0x60); + +static inline size_t +ane_csne_cmd_procedure_call_size(unsigned int num_io_buffers) +{ + return sizeof(struct ane_csne_cmd_procedure_call) + + num_io_buffers * sizeof(struct ane_csne_io_elem); +} + +/* Fw-side bound: the generic CSNE processor rejects work items of + * 0x1b89 bytes and above (@0x4d134 cmp x2, #0x1b89). What x2 names + * beyond "the command's size" is [INFERENCE] — enforced here as a + * fail-fast so an oversized command cannot enter the ring. + */ +#define ANE_CSNE_CMD_MAX_SIZE 0x1b88 + +/* ---- LOAD_PROGRAM (0x200) wire contract (selene 0x4e44c hook + + * 0x4e53c worker + 0x4e640 registration; decode 2026-09-26, entry + * item 23/23b/23c/23e): the payload is NINE 0x30-byte named section + * records at +0x08. Registration publishes ".bin" into the fw + * program-info symbol table (format "%s.bin" @0xa733f, + * CAneProgramInfo.cpp) with value = record+0x18 and aux = + * u32(record+0x20). The fw dereferences record+0x18 (0x5d0c8), so it + * carries a fw-addressable pointer — in this driver, the IOVA of a + * host-authored section object (below). Absent sections (flags bit0 + * clear) are skipped by every decoded parser. ---- + */ +enum ane_t6021_load_section { + ANE_SEC_GENERIC = 0, + ANE_SEC_KERNEL, + ANE_SEC_TEXT, + ANE_SEC_OPERATION, + ANE_SEC_PROCEDURE, + ANE_SEC_KERNELPROP, + ANE_SEC_TEXTPROP, + ANE_SEC_OPDBG, + ANE_SEC_PROCPROP, + ANE_SEC_COUNT /* 9 */ +}; + +#define ANE_T6021_LOAD_SEC_COUNT 9 + +static const char * const +ane_t6021_load_sec_name[ANE_T6021_LOAD_SEC_COUNT] = { + "genericSection", "kernelSection", "textSection", + "operationSection", "procedureSection", "kernelPropSection", + "textPropSection", "opDbgSection", "procPropSection", +}; + +struct ane_csne_cmd_load_program { + struct ane_csne_hdr hdr; /* id 0x200 @ +4 */ + struct ane_csne_io_elem sec[ANE_T6021_LOAD_SEC_COUNT]; +}; + +static_assert(sizeof(struct ane_csne_cmd_load_program) == + 0x08 + 9 * 0x30); + +/* Section-record wire accessors over the 0x30-byte io_elem (the only + * bytes any decoded parser reads; the rest is unread pass-through): + * +0x00 u8 flags — bit0 present; +0x18 u64 obj — fw-addressable + * pointer, validated by 0x5d0c8 to land inside a registered program + * object's entry table; +0x20 u64 key — per-section lookup key. + */ +#define ANE_SEC_F_PRESENT BIT(0) + +static inline void ane_sec_record_init(void *rec, u64 obj, u64 key) +{ + u8 *r = rec; + + memset(r, 0, 0x30); + r[0] |= ANE_SEC_F_PRESENT; + *(u64 *)(r + 0x18) = obj; + *(u64 *)(r + 0x20) = key; +} + +/* Host-authored PROGRAM OBJECT (the thing section records point at; + * 0x5d0c8 validates: magic 1 @+0, count <= 0x10 @+4, table entry + * count in [0x201, 0x400] @+0x204, 0x30-byte entry table @+0x208 with + * the recorded pointer bounded inside it). Minimum object = header + + * 0x201 zeroed entries; entries are the same 0x30-byte shape. + */ +#define ANE_PROGOBJ_MIN_ENTRIES 0x201 +#define ANE_PROGOBJ_MAX_ENTRIES 0x400 +#define ANE_PROGOBJ_TABLE_OFF 0x208 +#define ANE_PROGOBJ_HDR_SZ 0x208 + +static inline size_t ane_progobj_size(u32 entries) +{ + return ANE_PROGOBJ_TABLE_OFF + (size_t)entries * 0x30; +} + +static inline void ane_progobj_init(void *obj, u32 entries) +{ + struct { u32 magic; u32 count; u32 rsvd[2]; u32 entries; } *h = obj; + + h->magic = 1; + h->count = 0; + h->rsvd[0] = 0; + h->rsvd[1] = 0; + h->entries = entries; + /* caller zeroes the tail: entries table starts at +0x208 */ +} + +/* Host-authored OPERATION-SECTION image (0x5d290: u32 op_count @+0, + * <= 0x80; then op_count 0x40c-byte operation records at +4; the + * section KEY is an offset past the array, >= op_count*0x40c + 4). + * Operation record fields decoded: +0x00 u32 type <= 4; +0x04 u16 + * <= 0x10; +0x08 u32 procedure_count <= 0x80 (non-zero); +0x0c.. + * u32 procedure indices, each <= 0x3c. + */ +#define ANE_OPSEC_OP_REC_SIZE 0x40c +#define ANE_OPSEC_MAX_OPS 0x80 + +static inline size_t ane_opsec_size(u32 ops) +{ + return 4 + (size_t)ops * ANE_OPSEC_OP_REC_SIZE; +} + +/* Submit one CSNE command block on the INIT (EP1) ring: K14 + * rtbuddyEndpointSendMessage semantics (W2 §3) — slot alloc with wrap, + * memcpy into the ring, 54-bit doorbell word, cursor + * advanced only on doorbell success. No synchronous response matching: + * fw->host responses arrive on the T2F* channels and are not walked + * yet (W2 §3). Sleeps (mutex) — process context only. + */ +int ane_t6021_csne_submit(struct ane_t6021 *ane, const void *cmd, size_t size); + +/* Probe-time one-shot CSNE_CMD_PING on EP1 (W5-live), behind + * mbi_doorbell=1 only; watches the fw response surfaces for 3 s. + */ +void ane_t6021_csne_ping_attempt(struct ane_t6021 *ane); + +/* W13/W14 firmware loader (ane_t6021_fwload.c): validate + stage + + * dart-ane0-map the 13.5 PRELOAD payload behind fw_load=1. This is + * the staging half of the boot contract: the Params+0x18 producer + * chain and the RVBAR fold are closed (mapper-callchain audit, commits + * 3762aee/12be074); ane_t6021_boot.c consumes the staged surface. + */ +int ane_t6021_fwload_probe(struct ane_t6021 *ane); +void ane_t6021_fwload_remove(struct ane_t6021 *ane); +bool ane_t6021_fwload_options_ok(void); +bool ane_t6021_fwload_placement_ok(struct device *dev); +bool ane_t6021_fwload_requested(void); + +struct ane_fw_image; +struct ane_asc_tunables; + +/* Per-SoC of_match data (defined in ane_t6021_fwload.c). + * soc, soc_revision: what iBoot writes to RTK_soc and RTK_soc_revision. + * revision_fuse: read the revision from the DT "fuse" window as iBoot + * does instead (T8112). + * preload_placement: the iBoot SEG0/SEGi physical placement is recorded + * for this SoC (T6021 only), so fw_alias_reserved=1 may map it; the + * other SoCs always run the firmware from driver-owned memory. + * ps_cpu_off: the ANE CPU ps word in the DT "pmgr" window (probe guard). + * pwgate_off: the kext's PWGATE word in the DT "set" window, checked + * open before any engine read; 0 = not checked. + * pmu_pa: the page of the seven ANE ps words; the firmware's power + * service writes them through its DART at IOVA == PA. ps_off: the + * first of the seven in that page. + * trace_td_off: engine offset of the TM last-committed-TD word read by + * trace_td; 0 = trace_td unsupported. + */ +struct ane_t602x_soc { + u32 soc; + u32 soc_revision; + bool revision_fuse; + bool preload_placement; + const struct ane_fw_image *fw; + const struct ane_asc_tunables *tunables; + u32 ps_cpu_off; + u32 pwgate_off; + u64 pmu_pa; + u32 ps_off; + u32 trace_td_off; +}; + +extern const struct ane_t602x_soc ane_t6020_soc, ane_t6021_soc, + ane_t6022_soc, ane_t8112_soc; + +#endif /* __ANE_T6021_H__ */ diff --git a/drivers/accel/ane/ane_t6021_boot.c b/drivers/accel/ane/ane_t6021_boot.c new file mode 100644 index 0000000000000..066d2a799851a --- /dev/null +++ b/drivers/accel/ane/ane_t6021_boot.c @@ -0,0 +1,557 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * ane_t6021_boot.c — T6021 boot state reporter (W15). + * + * READ-ONLY. Main review 2026-09-20: the unresolved pre-CPU power and + * config-table prerequisites BLOCK every boot write, so this unit + * performs no MMIO write at all — it resolves the boot-input state, + * reports it with per-read whitelist provenance, and names the exact + * missing lifecycle datum. The write path (RVBAR entry fold + CPU RUN + * release + first-alive ack) lands only after the prerequisites below + * are pinned; its pure contract units (entry fold, latch/entry decode, + * SCRATCH u64 split, init-structure fill) live in ane_t6021_boot.h and + * are regression-tested host-side (tools/h14_boot_regression.c). + * + * Blocked boot writes, with the blocking prerequisite each one waits on: + * - RVBAR entry fold (ENTRY_BASE | fw DVA & ADDR_MASK; single u64 + * store, width_proof 0x95e9860→0x9622b0c read / 0x95e9988→0x9622c94 + * write): semantics of the observed RVBAR read 0x1 are unresolved + * (see the rvbar lifecycle fork) AND the pre-CPU block + * below applies regardless. + * - CPU_CONTROL write32 0 then 0x10 (rvbar-width local order): waits + * on the same pre-CPU block; releasing RUN with an unresolved + * entry state is the exact blind start the lane forbids. + * - SCRATCH0/1 publication + SCRATCH7 wake: init-structure fields + * [0x08] and [0x10,0x38) have no pinned selene fn 0x71A4 consumer + * read-set; zero-filling unknown fields and calling the structure + * valid is forbidden (Main 2026-09-20). + * + * Pre-CPU prerequisites, updated for pass4 (receipts 2026-09-20 + * mapper-callchain pass4 + tools/check_power_anchors.py 33/33, commit + * 04630ef): + * RESOLVED (pass4): the CPU-power branch selector — H14g SoCConfig + * zeroes dev+0x3EA0 (0x9613e14, the block that also sets + * dev+0x4A0=0x01400044), so EnableCPUClocksAndPower takes the + * PLAIN branch: write32(PMGR+0x2e0, 0x0000000f), validate 0xff — + * AUTO_ENABLE bit28 never set. On Linux this END STATE is exactly + * what the first_resume stage-2/3 gate verifies (act=0xf, + * AUTO_ENABLE clear) before any of this code runs — the write is + * therefore not replicated (ps-word writes are forbidden by the + * standing rule and froze the box 2026-09-19); the gate IS the + * equivalence check. + * STILL OPEN (each blocks its write below): + * 1. CLOSED (pass5 c364f24): the pre-CPU engine table writels + * eng+0xb38/0xb98/0xbf8 <- 0x01ff01ff are REQUIRED every + * EnableANEClocksAndPower — dev+0x784 bit0 has no writer, so the + * always-open gate runs the loop unconditionally; the + * order-vs-start question is moot. Receiver PROVEN pass4 + * (dev+0x188 accessor = engine reg[0]); count = config+0x150 = + * 3. Armed below; the first live firing must ride netconsole + * behind the seam logs. + * 2. Provider first-enable enableDeviceClock(1,dev+0x8F0)/ + * enableDevicePower(1,&out,dev+0x8F0) (0x95d1d48/0x95d1d90): + * internal gate-ID arrays are runtime-populated (open). Linux + * substitutes the genpd raise + supplier links (islands verified + * ACTUAL=0xf by the gate); the equivalence datum is the open + * half. + * 3. RVBAR lifecycle — RESOLVED (M2ResetLifecycle, + * rvbar-lifecycle-evidence.json, anchors 38/38 + * check_rvbar_lifecycle_anchors.py): live bit0-set + entry 0 is + * the STALE state; the kext-lawful transition is power_off + * (ANE_deInit when dev+0x3FB, then PMGR ps 0x2e0 ← 0x00000000 via + * the dev+0x190 PMGR accessor) → power_on → re-init, NEVER an + * RVBAR write while bit0 is set. RVBAR (engine accessor, + * 0x1050000, u64) is written only after a read64 with bit0 == 0; + * CPU_CONTROL write32 0 then 0x10 runs on BOTH paths; success = + * SCRATCH7 (dev+0x454 selector) read32 == 0x08042006. Open edge: + * whether ps-off clears bit0 — one live read-only read64 after a + * domain-off answers it. IMPLEMENTATION CAVEAT: the ps-word write + * is the class that froze this host from kernel context + * (2026-09-19 receipt) and the W3 gate demands act=0xf + + * AUTO_ENABLE clear afterwards — the reset vehicle (userspace + * stage vs in-kernel) is a Main decision, so the fold+start path + * stays behind the itemized preflight gates. + * 3. Ack model — TWO-PHASE, SINGLE REGISTER, TRANSITION-BASED + * (rvbar-lifecycle receipt 6288b0b, 57/57 anchors; supersedes the + * pass6 SCRATCH0-beacon reading): pre-CPU, SCRATCH cells are + * cleared and SCRATCH7 is PULSED 1 -> 0 (stale READY/wake + * cleared); after CPU_CONTROL 0 -> 0x10, poll A demands a FRESH + * SCRATCH7 READY 0x08042006 (fw alive; fw_alive). Publication + * (init suballoc DVA -> SCRATCH0/1, dsb st) happens ONLY after + * poll A; then wake 0xf7fbdff9 -> SCRATCH7 releases the fw; poll + * B awaits SCRATCH7 DONE 0x08042006 (booted) and reads back + * SCRATCH0/1. Because of the pulse, a post-CPU READY is + * unambiguous — no stale-ack hazard. The SCRATCH0 marker (fn + * 0x86EC) is not a gate. + * Selene raw anchors: READY write idx7 @0x7360-0x7388 (movz + * 0x2006 @0x7364, mov w1,#7 @0x7368, movk 0x804 @0x736c, slot + * byte48 @0x7380, salt 0x5bdd @0x7384, blraa @0x7388); poll loop + * READ32 idx7 @0x73c4-0x73e4, cmp wake 0xf7fbdff9 @0x73e8, + * sleep(1000) @0x73f8-0x73fc; DONE ack @0x77cc-0x77f0. + * 4. Linux allocation map (legacy branch sizes): FWIM surface = + * config+0x138 byte-count = 0x500000 (Main, audit 751caa4); + * Linux fw_buf is ANE_FW_BUF_SIZE = 0x500000 — the semantic is + * honored directly, covering the image vmsize 0x36c000. 'IPC ' + * surface = min(config+4, dev+0x3A70 + * cap) — the numeric cap needs the h14g config blob field map + * (open prerequisite). Init suballoc = 0x174 from a Linux-owned + * pool standing in for the kext dev+0x968/dev+0x980 pool (pool + * total size open). RTBuddy FW_INIT sizes (64K…) are the OTHER + * branch — never used here. + * 5. Init publication: the fw consumes [0x08]..[0x68] (pass5) and + * the Linux sources of those fields are not pinned ([0x08] = + * *(dev+0x988+0x18) = the 'IPC ' surface DVA — the surface + * itself is CLOSED (pass5c/5d), its Linux allocation size open + * per item 4; [0x10]/[0x18] = config-size terms with + * config+0x138 = image byte-count closed; [0x30] = dev+0x990 + * load-progress word) — publication cannot fire until every + * fw-read field has a pinned Linux source. + * + * Read whitelist — every MMIO read below, individually: + * - RVBAR eng+0x01050000: read64 width-proven (rvbar-width + * width_proof); W8 and W10 live Linux reads, value 0x1 each. + * - CPU_STATUS eng+0x1400048: phase-1 S2 whitelist; kext poll site + * 0x…95ecfb4 (config field dev+0x49c = 0x1400048); W10 live read + * 0x2a. + * - ASC mailbox controls eng+0x1408110/0x1408114: W10 live reads + * (both 0x00020001; 600 polls / 30 s, zero changes, zero aborts). + * The FATAL mailbox-class reads were the +0x1608xxx h16g analog — + * a different address never touched here; the 0x1854000..0x1c04000 + * fabric-fatal window is likewise never touched. + * - SCRATCH eng+0x1840048..0x1840064: phase-1 S2 whitelist + * (all-zero pre-attach), W5 pre-read log clean, first_resume + * re-reads all eight on every load. + * - TICK eng+0x1160008 (PROGRESS dump only): 24 MHz domain tick, + * proven readable post-release (2026-09-25 handshake receipt); + * two reads 20 ms apart for a delta. engine+0x1140008 is + * deliberately NOT read (readability/width/reset semantics + * unproven, no pre-release baseline — Main review 2026-09-26). + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ane_t6021.h" + +#include "ane_t6021_boot.h" + +/* Retain the firmware nap-prevention counter via the init resource bit + * (lab boot_prevent_nap=1, the proven add-path value). + */ +static bool boot_prevent_nap = true; +module_param(boot_prevent_nap, bool, 0444); +MODULE_PARM_DESC(boot_prevent_nap, + "Retain firmware nap-prevention counter via init resource bit (default on: proven add-path value)"); + +/* This object links into both ane_t6021.ko and ane_t6021_rtclient.ko; + * per-object metadata keeps modpost happy for either composition. + */ +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("T6021 ANE contract-pinned boot sequence core"); + +/* Boot-write gates — ITEMIZED, each a HARD gate: the ENTIRE write + * sequence (preboot engine table, scratch clear + pulse, RVBAR + * resolution, CPU release, publication, wake) runs start-to-finish + * only when EVERY gate below is true, checked BEFORE any write (Main + * 2026-09-20: no partial boot; remaining semantics stay hard gates, + * not comments). Each flips only with cited proof in a Main-reviewed + * commit that also populates the sources and adds the kernel io + * backend — never a runtime knob. + */ +/* Provider equivalence — CLOSED (Main provider review 2026-09-20): + * Linux genpd attachment + supplier links accepted INSTEAD of + * reproducing Apple runtime index arrays, CONDITIONAL on the live + * first_resume verifying all 8 islands ACTUAL=0xf, BUSY=0, CPU + * AUTO_ENABLE clear before the sequence — that verification runs on + * every probe and the driver performs no direct kernel PMGR writes. + */ +static const bool pf_provider_genpd_strategy = true; +static const bool pf_pool_word0_proven = true; /* CONFIRMED (Main raw + + * Reset 183fd50, chain + * 0x67dc/0x6824/0x68d8/ + * 0x70dc/0x736c): DDM Params + * word0 = requested bytes = + * 0x40000 — sourced, not + * synthesized + */ +static const bool pf_heap_floor_pinned = true; /* CLOSED: floor = + * getPageSize() return = + * DART page size 0x4000 + * (store chain 0x9602e68-90 + * + dart-ane0 page-size + * 0x4000 node, pass6h); + * sourced in boot_sources + */ +static const bool pf_dart_page_floor = true; /* CLOSED (Main raw, + * pass6h + exact-node + * linkage): dtree-j414c.txt + * sha256 dd2955…d504, ane0 + * iommu-parent 363 (line + * 11499), dart-ane0 + * page-size 0x4000 (line + * 11510), child mapper-ane0 + * phandle 363 (line 11527). + * Runnable linkage check: + * ane-linux-experiments + * tools/ + * check_dart_ane_pagesize_linkage.py + */ +static const bool pf_rvbar_lifecycle = true; /* 6288b0b, 57/57 anchors */ +static const bool pf_pass6_init_contract = true; /* cd25b46, 87/87 */ + +/* FINAL authorization gate — CLOSED (Main provider review + pass6h + * numeric closure + 183fd50 word0 confirmation + netconsole Wi-Fi + * marker m2-wifi-check-1789937086 @ 15:44:47.121595Z). Source + * contract closure complete; live attempt authorized by user + * override (autonomous boot/recovery loop). + */ +/* Table-block mode selection (2026-09-20 16:23:07 wedge): + * mode 0 = ABORT before any write (accidental-repeat prevention), + * mode 1 = write the table (kext-faithful; selected per-run via the + * rtclient fw_start_table_mode param), + * mode 2 = SKIP the table (default: the shipped diagnostic). + * W8 write-grant tunables are mode-independent (proven no-abort + * class, w8-run.out: APERTURE_UNLOCKED) and run in every armed mode. + */ + +static const bool pf_main_lifetime_review = true; + +static bool ane_t6021_boot_preflight_complete(void) +{ + /* pf_preboot_table_safe is INTENTIONALLY NOT in this product: + * mode 2 (table skip) is the authorized diagnostic state — + * the preflight covers the source contract, the mode selector + * covers the live-fault table gating. + */ + return pf_provider_genpd_strategy && pf_pool_word0_proven && + pf_heap_floor_pinned && pf_dart_page_floor && + pf_rvbar_lifecycle && pf_pass6_init_contract && + pf_main_lifetime_review; +} + +/* STATIC boot sources — populated per the pinned contract (Main + * authorization 2026-09-20). Object layout (Main raw correction, + * superseding audit 018abdb): the OSValueObject at +0x10 POINTS to + * the Params; Params+0x00 = size, +0x18 = DVA, +0x38 = hostVA. + * dev+0x980 is the 'DDM ' Params pool (size 0x40000); dev+0x968 is a + * SEPARATE DMM manager constructed over the pool hostVA — never + * conflate the two. Runtime fields (fw_dva, ipc_dva, pool_dma) are + * filled by ane_t6021_boot_prepare() at sequence time (after poll A, + * per "dynamic allocations occur after READY"); heap_floor stays 0 + * here and is supplied together with pf_heap_floor_pinned — NEVER + * used unpinned (the gate above keeps the whole sequence unrun until + * then). + */ +static const struct ane_t6021_init_sources boot_sources = { + .cfg_size = 0x500000, /* config+0x138 (0x9613da8/ dac) */ + .prev_fw_len = 0, /* first boot; static per reload */ + .heap_floor = 0x4000, /* DART page size: dev+0x3A90 = + * getPageSize() return; node + * authority dart-ane0 page-size + * 0x4000 (exact linkage check + * passing) + */ + .pool_word0 = 0x40000, /* DDM Params word0 = requested + * bytes (CONFIRMED, 183fd50) + */ +}; + +/* Poll A/B bound: the kext polls <=1000 x sleep(1ms) (selene poll + * loop 0x73c4-0x73fc analog; rvbar-lifecycle step 6/10). + */ +#define ANE_BOOT_ACK_POLL_US 1000 +#define ANE_BOOT_POLL_MS 1000 + +/* The COMPLETE resolved legacy-boot sequence is implemented ONCE in + * ane_t6021_boot.h (ane_t6021_boot_run) against an io backend, so the + * fake-MMIO trace check exercises the exact code the device will run + * (Main: runnable trace check; implement the device wiring only after + * ALL sources close — until then this file performs no boot write). + * The kernel backend (readl/writeq/udelay wrappers + prepare hook + * doing the pool/IPC allocations and the sourced fill) is the small + * reviewed increment that lands with the gate flip. + */ + +/* ---- kernel io backend for ane_t6021_boot_run() ---- */ + +struct ane_t6021_boot_mmio { + struct ane_t6021 *ane; +}; + +static u32 ane_boot_rd32(void *ctx, unsigned int off) +{ + struct ane_t6021_boot_mmio *mm = ctx; + + return readl(mm->ane->base[ANE_T6021_REG_ENGINE] + off); +} + +static u64 ane_boot_rd64(void *ctx, unsigned int off) +{ + struct ane_t6021_boot_mmio *mm = ctx; + + return readq(mm->ane->base[ANE_T6021_REG_ENGINE] + off); +} + +static void ane_boot_wr32(void *ctx, unsigned int off, u32 v) +{ + struct ane_t6021_boot_mmio *mm = ctx; + + writel(v, mm->ane->base[ANE_T6021_REG_ENGINE] + off); +} + +static void ane_boot_wr64(void *ctx, unsigned int off, u64 v) +{ + struct ane_t6021_boot_mmio *mm = ctx; + + writeq(v, mm->ane->base[ANE_T6021_REG_ENGINE] + off); +} + +static void ane_boot_publish_barrier(void *ctx) +{ + (void)ctx; + /* dma_wmb() = dmb oshst on arm64: orders the coherent pool fill + * before the device publish. ORDERING guarantee — not the kext's + * dsb st (completion). Sufficient for the Linux coherent-DMA + + * writel doorbell contract (writel orders prior accesses before + * the MMIO store). + */ + dma_wmb(); +} + +static void ane_boot_wait(void *ctx) +{ + (void)ctx; + usleep_range(1000, 1500); /* kext poll: sleep(1000us) */ +} + +static void ane_boot_phase(void *ctx, const char *what) +{ + struct ane_t6021_boot_mmio *mm = ctx; + + /* bounded phase marker: one line per block boundary, survives + * netconsole for crash attribution (never per-poll). KERN_EMERG + * + a short drain so the line reaches tty0/netconsole/ssh + * BEFORE the risky write it announces (fw-start-debug + * 2026-09-22: fwstart#2 died with zero capture — the marker + * must beat the write). + */ + dev_emerg(mm->ane->dev, "BOOT-PHASE %s\n", what); + msleep(30); +} + +/* S5 prepare — runs strictly AFTER poll A (fw alive), BEFORE the + * SCRATCH0/1 publish. Dynamic allocations occur here per "dynamic + * allocations occur after READY" (Main): the 'DDM ' pool, the 'IPC ' + * surface, and the trust-bounded fw-requested HEAP surface. Every + * allocation is wedged-pin owned from here on (held while + * cpu_started; reboot reclaims). Publishes the pool DVA (suballoc at + * offset 0) as the SCRATCH0/1 halves. + */ +static void *ane_boot_alloc(void *ctx, u64 size, u64 *iova) +{ + struct ane_t6021 *ane = ctx; + dma_addr_t d = 0; + void *p = dma_alloc_coherent(ane->dev, size, &d, GFP_KERNEL); + + if (p && !ane_t6021_fw_alias_iova_ok(ane, d, size)) { + dev_err(ane->dev, + "boot alloc %#llx+%#llx overlaps fw alias — refusing\n", + (u64)d, size); + dma_free_coherent(ane->dev, size, p, d); + return NULL; + } + + *iova = p ? d : 0; + return p; +} + +/* Kernel prepare: live SCRATCH3/SCRATCH1 reads feed the SHARED + * assembly (ane_t6021_boot_prepare_publish) with the dma_alloc hook — + * the same path the fake-MMIO trace test exercises. Allocations are + * wedged-pin owned from creation (held while cpu_started). + */ +static int ane_t6021_boot_prepare(void *ctx, u32 *lo, u32 *hi) +{ + struct ane_t6021_boot_mmio *mm = ctx; + struct ane_t6021 *ane = mm->ane; + void __iomem *eng = ane->base[ANE_T6021_REG_ENGINE]; + struct ane_t6021_init_sources src = boot_sources; + struct ane_t6021_boot_allocs a; + u32 request, scratch0, scratch1; + int err; + + if (!ane_t6021_boot_preflight_complete()) + return -ENODATA; /* belt: run() already gated */ + + /* dynamic reads (post-READY, live cells — never hardcoded): + * read order SCRATCH0 then SCRATCH1 (0x95ea0d8/0x95ea100); + * SCRATCH0 >= 0x21 refuses before allocations/publication; + * SCRATCH3 = fw extra-heap request; SCRATCH1+1 = ordinal. + */ + scratch0 = readl(eng + ANE_MBI_SCRATCH0); + scratch1 = readl(eng + ANE_MBI_SCRATCH0 + 4); + request = readl(eng + ANE_MBI_SCRATCH0 + 4 * 3); + + src.fw_dva = ane->fw_iova; + err = ane_t6021_boot_prepare_publish(&src, request, scratch0, + scratch1, 0x4000, + ANE_T6021_BOOT_IPC_CEILING, + ANE_T6021_BOOT_HEAP_CEILING, + ane, ane_boot_alloc, &a, + lo, hi); + if (!err && boot_prevent_nap) { + ((u8 *)a.pool)[ANE_T6021_INIT_TEMPLATE_OFF + 0x84] |= 1; + dev_info(ane->dev, "LAB init resource[0x84] bit0=1: prevent nap\n"); + } + /* OWNERSHIP TRANSFER on EVERY path (Main lifetime review): an + * error after the pool/ipc allocations must NOT lose them — a + * started CPU may already be fetching, so partial allocations + * are RETAINED (wedged-pin holds them; reboot reclaims), never + * freed on this path. + */ + ane->boot_pool = a.pool; + ane->boot_pool_iova = a.pool_dva; + ane->boot_ipc = a.ipc; + ane->boot_ipc_iova = a.ipc_dva; + ane->boot_ipc_size = a.ipc_size; + ane->boot_heap = a.heap; + ane->boot_heap_iova = a.heap_dva; + ane->boot_heap_size = a.heap_size; + return err; +} + +/* Dispatch the resolved sequence against the kernel io backend. All + * gates were checked by the caller; the run() core re-checks. After + * the CPU release there is NO ordinary unwind: failures HOLD state + * (wedged-pin cleanup refuses to free under a started CPU) and the + * probe binds fenced. + */ +/* Poll-A-timeout progress dump — PROVEN-readable observables ONLY + * (Main review 2026-09-26): all eight SCRATCH cells (pulse-cleared to 0 + * pre-release, so any nonzero word with SCRATCH7 != READY/WAKE is a + * candidate firmware store), RVBAR, CPU_STATUS, and the 24 MHz domain + * tick at engine+0x1160008 (watched advancing post-release in the + * 2026-09-25 handshake receipt; a static tick separates "ASC domain + * unclocked" from "clocked but silent"). + * + * Deliberately NOT read: engine+0x1140008 (the 13.5 stub's VM 0x30c + * store target). Its readability, width and reset value are unproven + * and no pre-release baseline exists, so an after-value would not + * locate PC (could be preexisting iBoot state, write-only, or + * normalized). + */ +static void ane_t6021_boot_progress_dump(struct ane_t6021 *ane) +{ + void __iomem *eng = ane->base[ANE_T6021_REG_ENGINE]; + u32 tick0 = readl(eng + ANE_T6021_BOOT_REG_TICK); + u32 tick1; + unsigned int i; + + msleep(20); + tick1 = readl(eng + ANE_T6021_BOOT_REG_TICK); + + for (i = 0; i < 8; i++) + dev_emerg(ane->dev, "PROGRESS SCRATCH%u=%08x\n", i, + readl(eng + ANE_T6021_BOOT_REG_SCRATCH0 + 4 * i)); + dev_emerg(ane->dev, + "PROGRESS rvbar=%016llx cpu_status=%08x tick %08x->%08x (%s)\n", + readq(eng + ANE_ASC_RVBAR), + readl(eng + ANE_ASC_CPU_STATUS), + tick0, tick1, + tick1 != tick0 ? "24MHz domain clocked" : "tick STATIC"); +} + +int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, int rtb_mode) +{ + struct ane_t6021_boot_mmio mm = { .ane = ane }; + struct ane_t6021_boot_io io = { + .ctx = &mm, + .rd32 = ane_boot_rd32, .rd64 = ane_boot_rd64, + .wr32 = ane_boot_wr32, .wr64 = ane_boot_wr64, + .publish_barrier = ane_boot_publish_barrier, .poll_wait = ane_boot_wait, + .phase = ane_boot_phase, + .prepare = ane_t6021_boot_prepare, + }; + struct ane_t6021_boot_cfg cfg = { + .preflight_ok = ane_t6021_boot_preflight_complete(), + .preboot_table_mode = table_mode, + .fw_dva = ane->fw_iova, + .stop_after = stop_after, + .rtb_mode = rtb_mode, + }; + int cs = 0, fa = 0, bo = 0; + u64 sres = 0; + int r; + + /* Wedged-pin module lifetime (Main lifetime review): the ref is + * acquired BEFORE the first write — a started CPU can never + * outlive the pin, and a dying module refuses the boot before + * any write happens. Retained on CPU start (never released: + * intentional); released only if no CPU start occurred. + */ + if (!try_module_get(THIS_MODULE)) { + dev_err(ane->dev, + "boot: REFUSED before any write — module ref unavailable (dying); no CPU start possible from a dying module\n"); + return -EBUSY; + } + + dev_emerg(ane->dev, + "BOOT-PHASE dispatch (stop_after=%d%s)\n", stop_after, + stop_after ? " BISECT STOP ARMED" : ""); + + r = ane_t6021_boot_run(&io, &cfg, &cs, &fa, &bo, &sres); + + ane->cpu_started = cs; + ane->fw_alive = fa; + ane->booted = bo; + ane->boot_scratch_result = sres; + + /* Released CPU without READY (poll-A timeout, either transport + * mode): capture the 13.5 execution-progress observables NOW, + * while the islands are powered — this is the exact-progress + * datum the next pre-Linux capture cross-checks. + */ + if (cs && !fa) + ane_t6021_boot_progress_dump(ane); + + if (!cs) { + /* no CPU start: full release path, normal ownership */ + module_put(THIS_MODULE); + dev_emerg(ane->dev, + "BOOT-PHASE done r=%d cpu_started=0 (no CPU release: state clean, module unpinned)\n", + r); + return r; + } + + /* started CPU: retain the pin for the whole wedged lifetime. + * Residual: DT hotplug unbind cannot be fully prevented; devm + * release order frees irq before ioremap (probe-order reverse); + * DMA surfaces are wedge-held, never freed. + */ + dev_warn(ane->dev, + "boot: module PINNED until reboot (started CPU; wedged-pin)\n"); + + if (r == -ENODATA) + return r; /* unreachable: the caller gated */ + if (r && cs) { + dev_err(ane->dev, + "boot: sequence error %d AFTER CPU start (cpu_started=%u fw_alive=%u booted=%u scratch_result=%016llx) — WEDGED-PIN HOLD: all surfaces/rings/IRQ/links preserved; reboot is the only reclamation; no retry. HANDSHAKE state only: the result word has NO sourced success semantics\n", + r, cs, fa, bo, sres); + return 0; /* bind fenced, state held */ + } + if (!r) + dev_info(ane->dev, + "boot: run returned 0 (booted=%u fw_alive=%u) — %s; scratch_result=%016llx (raw device address, semantics UNSOURCED — not a success claim; transport stays fenced until response validation)\n", + bo, fa, + bo ? "DONE observed — handshake complete" + : "rc 0 WITHOUT DONE (rtb_mode: HELLO-gated; DONE not part of this mode)", + sres); + return r; +} + diff --git a/drivers/accel/ane/ane_t6021_boot.h b/drivers/accel/ane/ane_t6021_boot.h new file mode 100644 index 0000000000000..ced0c997439dd --- /dev/null +++ b/drivers/accel/ane/ane_t6021_boot.h @@ -0,0 +1,832 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * ane_t6021_boot.h — T6021 legacy-boot contract units (pure, shared by + * the kernel sequencer and the userspace regression). + * + * Byte anchors (KC kernelcache.release.mac14j sha256 8304156f…, selene + * sha256 9f7915c4…; receipts under ane-linux-experiments/receipts/, + * commits 3762aee + 12be074): + * - RVBAR fold: ANEHWDevice::ANE_Init 0x95e9850–0x95e9988 (K13 + * 0x93203b0–0x93204e0 identical). Read64 of eng+0x01050000; bit0 + * set skips the write (live jw14m2 read 0x1). Otherwise write64 + * ENTRY_BASE | (fw DVA & ADDR_MASK) — single u64 store + * (2026-09-20-h14-rvbar-width.json width_proof). + * - fw DVA = ANESharedMemorySurfaceParams("FWIM")+0x18, produced by + * dartMapMemoryDescriptor (vtable slot 0x8a8, genIOVMSegments) for + * the out-param installed at start+0x336c (mapper-callchain audit + * pass1–3). Linux correspondence: dma_alloc_coherent on the ane + * platform device is dart-ane0-translated (live iommu group 6, + * three apple,t6020-dart streams, DMA domain). + * - SCRATCH0/1 u64 split: low32 → 0x01840048 first, high32 → + * 0x0184004c, dsb st before (legacy-init-publication receipt, + * accessor write32 width-proven at 0x9622d4c). + * - 0x174 init suballocation: header[0x00] u64 = fw DVA (legacy + * branch, dev+0x780 bit0 clear = observed AppleARMIODevice + * provider), [0x68] u32 = 64 count, template 256 B at + * [0x6C,0x16C) with template+0xC0 = 4 RESOLVED (Main raw anchors + * 0x9612b78/7c/80 on the dev+0x998 template; conditional |=0x10 + * separate, unqualified, not set). + * + * This header intentionally includes nothing. Kernel consumers have + * already; userspace consumers typedef u8/u32/u64 and + * include first (tools/h14_boot_regression.c in this + * repository is the shipped runnable check). + */ +#ifndef __ANE_T6021_BOOT_H__ +#define __ANE_T6021_BOOT_H__ + +/* RVBAR entry fold (ANE_Init 0x95e9868–0x95e988c). The mask clears + * bits 0–10, 48 and 55 — bit 11 IS retained (0xf800 keeps 0x800); the + * base supplies bit0 and the fixed upper pattern. + */ +#define ANE_T6021_RVBAR_ENTRY_BASE 0x0081000000000001ULL +#define ANE_T6021_RVBAR_ADDR_MASK 0xff7efffffffff800ULL + +/* CPU_CONTROL release (rvbar-width local order: write32 0 then 0x10; + * h14g config selects eng+0x01400044). RUN = BIT(4). + */ +#define ANE_T6021_CPU_RUN_RELEASE 0x10 + +/* SCRATCH7 handshake words (InitializeRTBuddy / legacy ANE_Init + * publication; selene ack writer 0x77c8). + */ +/* SCRATCH0 request band (pass9-corrected, Main raw trace): post-READY + * SCRATCH0 >= 0x21 REFUSES the boot preparation; below 0x21 proceeds + * to the MAX/alloc path. + */ +#define ANE_T6021_BOOT_IPC_REQ_THRESHOLD 0x21U + +#define ANE_T6021_BOOT_WAKE_REQ 0xf7fbdff9U +#define ANE_T6021_BOOT_ACK 0x08042006U + +/* Init suballocation geometry (ANE_Init 0x95ea710–0x95ea97c; + * legacy-init-publication receipt). + */ +#define ANE_T6021_INIT_STRUCT_SIZE 0x174 +#define ANE_T6021_INIT_FW_DVA_OFF 0x00 /* u64, legacy branch */ +#define ANE_T6021_INIT_COUNT_OFF 0x68 /* u32 = 64 */ +#define ANE_T6021_INIT_COUNT 0x40 +#define ANE_T6021_INIT_TEMPLATE_OFF 0x6c /* 256 B at [0x6C,0x16C) */ +#define ANE_T6021_INIT_TEMPLATE_SIZE 0x100 +#define ANE_T6021_INIT_TBIT_OFF 0xc0 /* template-relative */ +#define ANE_T6021_INIT_TBIT_VAL 0x4 /* RESOLVED (Main raw + * anchors 0x9612b78/7c/80: + * ldr/orr#4/str on the dev+0x998 + * template); the conditional + * |=0x10 is a separate, still + * unqualified mutation and is + * NOT set + */ + +static inline u64 ane_t6021_rvbar_compose(u64 iova) +{ + return ANE_T6021_RVBAR_ENTRY_BASE | (iova & ANE_T6021_RVBAR_ADDR_MASK); +} + +/* The kext skip condition: bit0 of the RVBAR read (tbnz w0,#0 at + * ANE_Init 0x95e9878; live jw14m2 reads have been 0x1). What a set + * bit names in boot-mode terms is the open lifecycle fork — see + * ane_t6021_boot.c; this predicate only decodes the bit. + */ +static inline bool ane_t6021_rvbar_latched(u64 rd) +{ + return rd & 1; +} + +/* Entry bits of an RVBAR read (what the fold would have named). */ +static inline u64 ane_t6021_rvbar_entry_bits(u64 rd) +{ + return rd & ANE_T6021_RVBAR_ADDR_MASK; +} + +/* Boot acceptance predicate — the EXACT check the boot path applies + * to a staged fw DVA: the fold must lose nothing. Bit 11 is retained; + * bits 0-10, 48 and 55 are not, so an iova with any of them set is + * rejected rather than silently truncated. + */ +static inline bool ane_t6021_rvbar_entry_ok(u64 iova) +{ + return (iova & (u64)~ANE_T6021_RVBAR_ADDR_MASK) == 0; +} + +/* Publication convention: low32 → SCRATCH0 (0x01840048) first, then + * high32 → SCRATCH1, after dsb st. + */ +static inline void ane_t6021_scratch64_split(u64 v, u32 *lo, u32 *hi) +{ + *lo = (u32)(v & 0xffffffffU); + *hi = (u32)(v >> 32); +} + +static inline u64 ane_t6021_scratch64_join(u32 lo, u32 hi) +{ + return ((u64)hi << 32) | lo; +} + +/* Linux sources for the init-suballocation fields (pass5b/5c/5d + * b8b7c35 + pass6c 75a017b; validator tools/check_init_contract.py + * 68/68). EVERY fw-read field is explicitly sourced from the struct + * below — nothing is silently zero: [0x00] FWIM DVA, [0x08] 'IPC ' + * surface DVA (Linux alloc, size = ane_t6021_ipc_size), [0x10] + * config+0x138 = 0x500000, [0x18] = 0x10000000 - size, [0x20] obj2 + * DVA, [0x28] w22, [0x30] fw-load progress, [0x50] zext [x23+4], + * [0x58] pool DMA base, [0x60] pool word0, [0x68] count 64, + * template[0x00] = 0, template+0xC0 = 4. The struct values pin BEFORE + * the preflight gate can open (Main: no unresolved field silently + * zero; no partial boot). The caller MUST zero the full 0x174 block + * first (coherent alloc): gaps [0x34..37]/[0x64..67] get no kext + * store and sit inside fw-read u64 units. + */ +/* Init-suballocation contract — pass6 producer semantics + * (rvbar-lifecycle-evidence.json pass6_init_header_producers, commit + * cd25b46, anchors G1-G26; 87/87 total): + * + * STATIC sources (pinned before the preflight can open): + * fw_dva -> [0x00] staged selene ('FWIM') surface DVA, + * ipc_dva -> [0x08] 'IPC ' surface DVA (its size calc remains + * the audit lane's open item), + * cfg_size -> [0x10] config+0x138 = 0x500000; [0x18] = + * 0x10000000 - cfg_size = 0x0fb00000, + * prev_fw_len -> [0x30] previous fw image length (0 on first boot; + * driver-static, updated on each reload), + * heap_floor -> [0x28] floor = config field dev+0x3A90 (value + * pinned by the audit lane at preflight close), + * pool_dma -> [0x58] the 'DDM ' pool surface DVA (Linux allocates + * its own 0x40000-byte DDM pool — the kext size), + * pool_word0 -> [0x60] first qword of the Linux pool-descriptor + * struct. NOT CLOSED: the fw-side consumer of + * pool+0x00 is untraced and the kext never stores it + * (free-list head at manager+0 untouched by + * Allocate). This value is a HARD-GATED source — the + * sequence cannot run until a producer/consumer proof + * pins it; no hypothesis value is shipped. + * PRODUCER OPEN (Main, superseding b21e23f): the + * "pool+0x00 = manager free-list head" claim is + * REJECTED — manager dev+0x968 != Params pool + * dev+0x980 (wrong-object alias likely), and + * Reset6cb048d shows the consumer stores word0 + * VERBATIM at image ops 68 (possible extent use). + * Wait for the exact same-Params producer. + * + * DYNAMIC inputs (read AFTER poll A — the fw publishes its extra-heap + * request into SCRATCH3 (cell idx3) and its boot ordinal is + * SCRATCH1+1; both read from the live cells, never hardcoded): + * scratch3_req -> the fw extra-heap request; 0 = no heap surface, + * scratch1_read -> SCRATCH1 read; [0x50] = (u32)read + 1 (boot + * ordinal). + * + * TRUST BOUNDARY: scratch3_req is firmware-supplied. The heap size + * must pass ane_t6021_heap_size() bounds BEFORE any allocation; the + * heap DVA and size are written to [0x20]/[0x28] only on success. + */ +struct ane_t6021_init_sources { + u64 fw_dva; + u64 ipc_dva; + u32 cfg_size; + u32 prev_fw_len; + u64 heap_floor; + u64 pool_dma; + u64 pool_word0; +}; + +/* 'IPC ' surface size (CURRENT_CONTRACT f9602e0; operands proven: + * u64[dev+0x3A90] = IODARTMapper::getPageSize() — 0x4000, one 16K + * DART page, typical; u32[x23+4] = captured SCRATCH1 + 1, the boot + * ordinal): size = max(page_size, ordinal + 1). Used by the kernel + * prepare hook to size the 'IPC ' allocation. + */ +/* SCRATCH0 read-order note (pass9 phase-1: SCRATCH0 then SCRATCH1 + * after READY): Main raw branch proof SUPERSEDES the audit's + * "SCRATCH0 >= 0x21 arms IPC" reading — 0x95ea1dc b.cc(low) branches + * TO the MAX+IPC-alloc path and >=0x21 goes to logging/error, so + * scratch0 < 33 is likely a VALIDITY gate, not an optional-IPC + * condition. Header [0x08] unconditionally dereferences + * dev+0x988+0x18. Allocation stays UNCONDITIONAL until Reset corrects + * the source; the threshold is NOT implemented. + */ +static inline u64 ane_t6021_ipc_size(u64 page_size, u32 captured) +{ + /* EXACT u32 semantics: the fw-visible ordinal is the captured + * SCRATCH1 value + 1 in 32-bit arithmetic (kext add w8,w0,#1) — + * it wraps. Callers pass the CAPTURED value, never pre-incremented. + * The result is then bounded by ANE_T6021_BOOT_IPC_CEILING at the + * allocation site: captured is fw-supplied and untrusted. + */ + u32 ord = captured + 1; + + return page_size > (u64)ord ? page_size : (u64)ord; +} + +/* Operational ceiling for the fw-influenced 'IPC ' allocation + * (captured is untrusted; the max() can otherwise be forced to an + * arbitrary size). Distinct from ABI width — documented budget. + */ +#define ANE_T6021_BOOT_IPC_CEILING 0x100000ULL /* 1 MiB */ + +/* Trust boundary for the firmware-supplied heap request: 0 disables + * the heap surface; otherwise the size is MAX(request, floor) and is + * refused (negative) when the request exceeds the ceiling. + * + * ANE_T6021_BOOT_HEAP_CEILING is an OPERATIONAL RESOURCE BUDGET, kept + * distinct from ABI width: header [0x28] is a u64 field and can + * express far larger values — the u64-ness says nothing about what is + * safe to allocate. The dart-ane0 vm window only bounds ADDRESSABILITY, + * not allocation safety, so it is deliberately not used as the + * ceiling. 32 MiB is the documented budget bounding a firmware-supplied + * allocation to a size the host can honor without starving; re-anchor + * to a config field when the h14g blob map lands. + */ +#define ANE_T6021_BOOT_HEAP_CEILING 0x02000000ULL /* 32 MiB budget */ + +static inline long long +ane_t6021_heap_size(u32 scratch3_req, u64 floor, u64 max_size) +{ + u64 req = scratch3_req; + + if (!req) + return 0; + if (req > max_size) + return -E2BIG; + return (long long)(req > floor ? req : floor); +} + +/* Fill the COMPLETE init suballocation from the static sources plus + * the post-READY dynamic values. The caller MUST zero the full 0x174 + * block first (coherent alloc): the gap bytes [0x34..37]/[0x64..67] + * get no kext store and sit inside fw-read u64 units. + */ +static inline void +ane_t6021_init_struct_fill(u8 *buf, const struct ane_t6021_init_sources *s, + long long heap_size, u64 heap_dva, + u32 boot_ordinal) +{ + u64 size = s->cfg_size; + int i; + + for (i = 0; i < 8; i++) { + buf[ANE_T6021_INIT_FW_DVA_OFF + i] = + (u8)(s->fw_dva >> (8 * i)); + buf[0x08 + i] = (u8)(s->ipc_dva >> (8 * i)); + buf[0x10 + i] = (u8)(size >> (8 * i)); + buf[0x18 + i] = (u8)((0x10000000ULL - size) >> (8 * i)); + buf[0x20 + i] = (u8)(heap_dva >> (8 * i)); + buf[0x50 + i] = (u8)((u64)boot_ordinal >> (8 * i)); + buf[0x58 + i] = (u8)(s->pool_dma >> (8 * i)); + buf[0x60 + i] = (u8)(s->pool_word0 >> (8 * i)); + } + + /* [0x28] u64 heap size (0 when the fw requested none) */ + for (i = 0; i < 8; i++) + buf[0x28 + i] = (u8)((u64)heap_size >> (8 * i)); + + /* [0x30] u32 FULL width: previous fw image length (0 first + * boot). Main review: low-byte-only serialization dropped the + * upper length bits. + */ + buf[0x30] = (u8)s->prev_fw_len; + buf[0x31] = (u8)(s->prev_fw_len >> 8); + buf[0x32] = (u8)(s->prev_fw_len >> 16); + buf[0x33] = (u8)(s->prev_fw_len >> 24); + + buf[ANE_T6021_INIT_COUNT_OFF + 0] = (u8)ANE_T6021_INIT_COUNT; + buf[ANE_T6021_INIT_COUNT_OFF + 1] = 0; + buf[ANE_T6021_INIT_COUNT_OFF + 2] = 0; + buf[ANE_T6021_INIT_COUNT_OFF + 3] = 0; + + /* template[0x00] = config word [dev+0x1D8], pinned value 0 + * (pass5d); [0x04..0x0B] zero via the caller's zeroing. + */ + buf[ANE_T6021_INIT_TEMPLATE_OFF + 0] = 0; + buf[ANE_T6021_INIT_TEMPLATE_OFF + 1] = 0; + buf[ANE_T6021_INIT_TEMPLATE_OFF + 2] = 0; + buf[ANE_T6021_INIT_TEMPLATE_OFF + 3] = 0; + + /* template+0xC0 = 4: RESOLVED initial state (Main raw anchors + * 0x9612b78/7c/80). The conditional |=0x10 remains unset. + */ + buf[ANE_T6021_INIT_TEMPLATE_OFF + ANE_T6021_INIT_TBIT_OFF + 0] = + ANE_T6021_INIT_TBIT_VAL; +} + +/* ---- Shared prepare-source assembly ---- + * + * The ALLOCATION-SIZING + HEADER-FILL + PUBLISH-VALUE path used by the + * kernel prepare hook AND the fake-MMIO trace test — one + * implementation, so wrong-DVA / double-increment / unbounded-size + * classes fail in the host trace test instead of only on hardware. + * + * Allocation contract: alloc(ctx, size, &iova) is called in the order + * pool (0x40000), 'IPC ' (max(DART page 0x4000, ordinal+1)), HEAP + * (0 when the fw requested none); each must return NULL on failure + * and set *iova to the device-visible address of the returned memory + * (the DMA address, NOT a recorded constant). Refusal classes: the + * fw-supplied request and captured value are UNTRUSTED — over-ceiling + * heap requests and over-budget IPC sizes are refused BEFORE any + * allocation. + */ +struct ane_t6021_boot_allocs { + u64 pool_dva, ipc_dva, heap_dva; + u64 ipc_size, heap_size; + void *pool, *ipc, *heap; +}; + +static inline int +ane_t6021_boot_prepare_publish(const struct ane_t6021_init_sources *s, + u32 scratch3_req, u32 scratch0_read, + u32 scratch1_read, + u64 page_size, u64 ipc_ceiling, + u64 heap_ceiling, + void *alloc_ctx, + void *(*alloc)(void *ctx, u64 size, + u64 *iova), + struct ane_t6021_boot_allocs *a, + u32 *lo, u32 *hi) +{ + int i; + + for (i = 0; i < (int)(sizeof(*a) / sizeof(u64)); i++) + ((u64 *)a)[i] = 0; + + /* SCRATCH0 band check — REFUSAL before allocations/publication + * (Main raw trace: >= 0x21 -> 0x95ea2a0 log -> 0x2c4 -> 0x3330 + * error, w21 = 0xE00002DA; < 0x21 -> 0x214 MAX -> 0x25c alloc -> + * 0x260 -> 0x2c8). Boundary: 0x20 accepted, 0x21 refused. + * Read order: SCRATCH0 first, then SCRATCH1 (0x95ea0d8/0x95ea100). + * Untrusted-input sizing after the band check; heap_size() + * returns SIGNED (negative = refused) — kept local so a refusal + * cannot wrap into the u64 record. + */ + if (scratch0_read >= ANE_T6021_BOOT_IPC_REQ_THRESHOLD) + return -EPROTO; + a->ipc_size = ane_t6021_ipc_size(page_size, scratch1_read); + if (a->ipc_size > ipc_ceiling) + return -E2BIG; + { + long long heap = ane_t6021_heap_size(scratch3_req, + s->heap_floor, + heap_ceiling); + + if (heap < 0) + return (int)heap; + a->heap_size = (u64)heap; + } + + a->pool = alloc(alloc_ctx, 0x40000, &a->pool_dva); + if (!a->pool) + return -ENOMEM; + a->ipc = alloc(alloc_ctx, a->ipc_size, &a->ipc_dva); + if (!a->ipc) + return -ENOMEM; + if (a->heap_size) { + a->heap = alloc(alloc_ctx, a->heap_size, &a->heap_dva); + if (!a->heap) + return -ENOMEM; + } + + /* ordinal: exact u32 +1 on the READ SCRATCH1 value (kext add + * w8,w0,#1; callers pass the read value, never pre-incremented). + * HEADER SOURCES come from the ALLOCATIONS: s->ipc_dva/pool_dma + * may be zero in the caller's static sources — fill from the + * allocator outputs (a->ipc_dva/a->pool_dva), never from s + * (Main review: zero-DVA header bug). + */ + { + struct ane_t6021_init_sources filled = *s; + + filled.ipc_dva = a->ipc_dva; + filled.pool_dma = a->pool_dva; + ane_t6021_init_struct_fill(a->pool, &filled, a->heap_size, + a->heap_dva, + (u32)(scratch1_read + 1)); + } + *lo = (u32)(a->pool_dva & 0xffffffffU); + *hi = (u32)(a->pool_dva >> 32); + return 0; +} + +/* Post-DONE SCRATCH1:SCRATCH0 (read SCRATCH1 first, then SCRATCH0) + * is a fw-produced DEVICE ADDRESS (pass9 phase-2), NOT a return code — + * never dereference it raw. WHITELIST (Main 2026-09-20): only the + * Linux-owned FWIM and 'IPC ' windows may ever be matched against it; + * the 'DDM ' pool is NOT whitelisted. No consumption until range AND + * length checks are proven; until then the raw u64 is stored and + * logged only. + */ + +/* ---- Fake-MMIO-testable sequence core ---- + * + * Register offsets (ENGINE-relative; the Linux driver maps the engine + * window, so kernel offsets == trace offsets) and the io backend. + * ane_t6021_boot_run() executes the resolved total order against ANY + * backend: the kernel wraps readl/writeq/udelay; the host regression + * wraps a recording fake (trace asserts: full order, no writes when + * the preflight gate is closed, publish strictly after poll A). + */ + +#define ANE_T6021_BOOT_REG_TABLE0 0x00000b38 +#define ANE_T6021_BOOT_REG_TABLE1 0x00000b98 +#define ANE_T6021_BOOT_REG_TABLE2 0x00000bf8 +#define ANE_T6021_BOOT_REG_RVBAR 0x01050000 +#define ANE_T6021_BOOT_REG_CPUCTRL 0x01400044 +#define ANE_T6021_BOOT_REG_SCRATCH0 0x01840048 +#define ANE_T6021_BOOT_REG_SCRATCH1 0x0184004c +#define ANE_T6021_BOOT_REG_SCRATCH6 0x01840060 +#define ANE_T6021_BOOT_REG_SCRATCH7 0x01840064 +/* 13.5 stub execution-progress observables (M2StartupRecovery 2026-09-26 + * decode of a9c4b771294a6b11…). Only PROVEN-readable engine-window + * registers: the 24 MHz domain tick at VM 0x1160008 (watched advancing + * ~24 MHz post-release, 2026-09-25 handshake receipt) plus SCRATCH, + * RVBAR and CPU_STATUS already on the whitelist. The stub's VM 0x30c + * store to engine+0x1140008 is real, but the register's readability, + * width and reset state are UNPROVEN and no pre-release baseline exists + * — it is deliberately NOT read here (Main review 2026-09-26). + */ +#define ANE_T6021_BOOT_REG_TICK 0x01160008 +#define ANE_T6021_BOOT_TABLE_VALUE 0x01ff01ffU +#define ANE_T6021_BOOT_TABLE_POLLS 1000 + +struct ane_t6021_boot_io { + void *ctx; + u32 (*rd32)(void *ctx, unsigned int off); + u64 (*rd64)(void *ctx, unsigned int off); + void (*wr32)(void *ctx, unsigned int off, u32 v); + void (*wr64)(void *ctx, unsigned int off, u64 v); + void (*publish_barrier)(void *ctx); /* orders the coherent fill + * before the device publish: + * dma_wmb() = dmb oshst on + * arm64; NOT claimed as dsb st + * (completion) + */ + void (*phase)(void *ctx, const char *what); /* bounded phase log: + * emitted ONCE before each + * write/poll block so a crash + * source survives netconsole + * (never per-poll) + */ + void (*poll_wait)(void *ctx); /* 1 ms poll delay */ + /* prepare(): called once, strictly AFTER poll A and BEFORE the + * SCRATCH0/1 publish; returns the suballoc DVA halves. Kernel + * backend: allocate pool/IPC + fill from pinned sources. + */ + int (*prepare)(void *ctx, u32 *lo, u32 *hi); +}; + +struct ane_t6021_boot_cfg { + int preflight_ok; /* EVERY prerequisite closed (Main: no + * partial boot — the whole sequence or + * nothing) + */ + int preboot_table_mode; /* LIVE-FAULT gating of the pre-CPU table + * block (2026-09-20 wedge): + * 0 = ABORT: -EAGAIN, zero writes + * (accidental-repeat prevention), + * 1 = WRITE the table block (re-armed + * after the table-base analysis), + * 2 = SKIP the table block, run the rest + * (diagnostic: tests fw-alive without + * the kext pre-CPU config). + */ + u64 fw_dva; /* staged surface DVA (fold input) */ + int rtb_mode; /* 1 = S1 writes SCRATCH6=0 (RTBuddy/RTKit-app- + * endpoint select, fw 0x42c8) instead of 1 + * (legacy ChMan/MBI). In RTBuddy mode the fw + * may skip READY/DONE; listen on mailbox. + * Run contract: P4 READY is recorded, not + * required; no legacy publish/wake runs; + * aliveness is gated by the RTKit HELLO. + */ + int stop_after; /* fw-start-debug step bisect (2026-09-22): + * 0 = full run; N in 1..4 = stop AFTER + * step N completes, return -ECANCELED + * (1 grant tunables, 2 scratch, 3 rvbar, + * 4 cpu release + poll A). A poll-A + * timeout inside step 4 still returns + * -ETIMEDOUT — the timeout is the + * answer. Steps complete = writes done; + * the stop never splits a step. + */ +}; + +/* Ownership: a started CPU may be fetching from the staged surfaces — + * the DMA memory is NOT reclaimable on failure/remove while + * cpu_started; reclaimable only via the domain-off reset (reboot). + */ +static inline int ane_t6021_boot_dma_reclaimable(int cpu_started) +{ + return !cpu_started; +} + +/* The cleanup decision the predicate implements: while cpu_started, + * remove() is HELD (H13 wedged-pin pattern — surfaces, rings, IRQ and + * power-domain links preserved; reboot reclaims). + */ +static inline int ane_t6021_boot_remove_held(int cpu_started) +{ + return cpu_started; +} + +/* Returns 0 (DONE observed, booted), -ENODATA (preflight closed: ZERO + * io writes), or -ETIMEDOUT (poll A/B: cpu_started holds, DMA stays + * unreclaimable, no publish/wake happened on poll A timeout). + */ +static inline int +ane_t6021_boot_run(const struct ane_t6021_boot_io *io, + const struct ane_t6021_boot_cfg *cfg, + int *cpu_started, int *fw_alive, int *booted, + u64 *scratch_result) +{ + unsigned int i; + u32 v; + u64 rvbar; + + *cpu_started = 0; + *fw_alive = 0; + *booted = 0; + *scratch_result = 0; + + /* Main: all gates checked BEFORE any boot write — no partial + * sequence when the preflight is closed. + */ + if (!cfg->preflight_ok) + return -ENODATA; + + /* pre-CPU engine table (pass4 receiver, pass5 gate: REQUIRED + * every EnableANEClocksAndPower; residual alias/indirect writer + * risk documented in the preflight list). + * LIVE-FAULT 2026-09-20: this block wedged the machine (P0 then + * silence — stall could be the FIRST write; P1 never reached; + * hardware reset 16:24:08) — pf_preboot_table_safe gates it + * OFF; nothing below runs until re-armed. + */ + switch (cfg->preboot_table_mode) { + case 0: + return -EAGAIN; /* BEFORE any write: accidental-repeat + * prevention (Main 2026-09-20) + */ + case 1: + io->phase(io->ctx, "P0 preboot-table"); + io->phase(io->ctx, "P0-1 eng+0xb38"); + io->wr32(io->ctx, ANE_T6021_BOOT_REG_TABLE0, + ANE_T6021_BOOT_TABLE_VALUE); + io->phase(io->ctx, "P0-2 eng+0xb98"); + io->wr32(io->ctx, ANE_T6021_BOOT_REG_TABLE1, + ANE_T6021_BOOT_TABLE_VALUE); + io->phase(io->ctx, "P0-3 eng+0xbf8"); + io->wr32(io->ctx, ANE_T6021_BOOT_REG_TABLE2, + ANE_T6021_BOOT_TABLE_VALUE); + break; + case 2: + default: + /* diagnostic skip: NO table writes — tests fw-alive + * without the kext pre-CPU config (user override + * 2026-09-20). + */ + io->phase(io->ctx, "P0 table SKIPPED (diagnostic)"); + break; + } + + /* P-1: W8 write-grant tunables (proven no-abort class, + * w8-run.out: APERTURE_UNLOCKED — 12 engine-aperture writes + * preparing the aperture; replayed verbatim). + * Per-write before/after discrimination (Main: attempt 3 stalled + * LAST phase = P-1, possibly FIRST tunable write). + */ + { + static const struct { u32 off; u32 val; } tun[] = { + { 0x000, 0x00000010 }, { 0x038, 0x00050020 }, + { 0x03c, 0x000a0030 }, { 0x400, 0x40010001 }, + { 0x600, 0x01ffffff }, { 0x738, 0x00200020 }, + { 0x798, 0x00100030 }, { 0x7f8, 0x0100000a }, + { 0x900, 0x00000101 }, { 0x410, 0x00001100 }, + { 0x420, 0x00001100 }, { 0x430, 0x00001100 }, + }; + unsigned int ti; + + io->phase(io->ctx, "P-1 grant-tunables begin"); + for (ti = 0; ti < sizeof(tun) / sizeof(tun[0]); ti++) { + io->phase(io->ctx, tun[ti].off == 0x000 ? + "P-1a eng+0x000" : + tun[ti].off == 0x038 ? "P-1b eng+0x038" : + tun[ti].off == 0x03c ? "P-1c eng+0x03c" : + tun[ti].off == 0x400 ? "P-1d eng+0x400" : + tun[ti].off == 0x600 ? "P-1e eng+0x600" : + tun[ti].off == 0x738 ? "P-1f eng+0x738" : + tun[ti].off == 0x798 ? "P-1g eng+0x798" : + tun[ti].off == 0x7f8 ? "P-1h eng+0x7f8" : + tun[ti].off == 0x900 ? "P-1i eng+0x900" : + tun[ti].off == 0x410 ? "P-1j eng+0x410" : + tun[ti].off == 0x420 ? "P-1k eng+0x420" : + "P-1l eng+0x430"); + io->wr32(io->ctx, tun[ti].off, tun[ti].val); + io->phase(io->ctx, "P-1 write done"); + } + io->phase(io->ctx, "P-1 grant-tunables end"); + } + if (cfg->stop_after == 1) + return -ECANCELED; /* tunables done, nothing else fired */ + + io->phase(io->ctx, "P1 scratch-clear+pulse"); + /* S1: InitANEScratchRegisters — clear ALL cells, SCRATCH6 = rtb_mode ? 0 (RTBuddy) : 1 + * (legacy ChMan/MBI); pulse SCRATCH7 1 -> 0 (stale cleared). + */ + for (i = 0; i < 8; i++) + io->wr32(io->ctx, + ANE_T6021_BOOT_REG_SCRATCH0 + 4 * i, 0); + io->wr32(io->ctx, ANE_T6021_BOOT_REG_SCRATCH6, + cfg->rtb_mode ? 0 : 1); + io->phase(io->ctx, cfg->rtb_mode ? + "P1 S1 SCRATCH6=0 (RTBuddy select)" : + "P1 S1 SCRATCH6=1 (legacy ChMan/MBI select)"); + io->wr32(io->ctx, ANE_T6021_BOOT_REG_SCRATCH7, 1); + io->wr32(io->ctx, ANE_T6021_BOOT_REG_SCRATCH7, 0); + if (cfg->stop_after == 2) + return -ECANCELED; /* scratch programmed, CPU untouched */ + + io->phase(io->ctx, "P2 rvbar"); + /* S2: RVBAR skip-or-fold (bit0 set = lawful skip branch; no + * latch override, no reset before first attempt). + */ + rvbar = io->rd64(io->ctx, ANE_T6021_BOOT_REG_RVBAR); + if (!ane_t6021_rvbar_latched(rvbar)) + io->wr64(io->ctx, ANE_T6021_BOOT_REG_RVBAR, + ane_t6021_rvbar_compose(cfg->fw_dva)); + if (cfg->stop_after == 3) + return -ECANCELED; /* entry decision recorded, no RUN */ + + io->phase(io->ctx, "P3 cpu-release"); + /* S3: CPU release — both paths, strictly 0 then 0x10. */ + io->wr32(io->ctx, ANE_T6021_BOOT_REG_CPUCTRL, 0); + io->phase(io->ctx, "P3b cpu-release RUN=0x10"); + io->wr32(io->ctx, ANE_T6021_BOOT_REG_CPUCTRL, + ANE_T6021_CPU_RUN_RELEASE); + *cpu_started = 1; + + io->phase(io->ctx, "P4 pollA-READY"); + /* S4: poll A — FRESH READY (the pulse made it unambiguous). + * RTBuddy mode (SCRATCH6=0, kext fw 0x42c8: rtbuddyFW when + * SCRATCH6 == 0): the fw does not publish BOOT_ACK to SCRATCH7 + * and no legacy publication follows — aliveness is gated by the + * RTKit HELLO after boot_run returns (selene InitializeRTBuddy + * 0x95ead04). A missing READY is recorded, not fatal, and + * *fw_alive stays 0 — never fabricated. + */ + for (i = 0; i < ANE_T6021_BOOT_TABLE_POLLS; i++) { + v = io->rd32(io->ctx, ANE_T6021_BOOT_REG_SCRATCH7); + if (v == ANE_T6021_BOOT_ACK) + break; + io->poll_wait(io->ctx); + } + if (v == ANE_T6021_BOOT_ACK) { + *fw_alive = 1; + io->phase(io->ctx, "P4 pollA READY observed"); + } else if (cfg->rtb_mode) { + io->phase(io->ctx, + "P4 pollA no READY (RTBuddy: HELLO-gated)"); + return 0; /* RTBuddy: no legacy publish/wake */ + } else { + return -ETIMEDOUT; + } + if (cfg->stop_after == 4) + return -ECANCELED; /* fw alive, publish/wake withheld */ + if (cfg->rtb_mode) + return 0; /* RTBuddy: no legacy publish/wake */ + + io->phase(io->ctx, "P5 prepare+publish"); + /* S5-S6: prepare (alloc/fill; kernel backend owns DMA), then the + * publish barrier, then publish suballoc DVA low32/high32, then + * wake. Barrier semantics (Main review 2026-09-20): the kernel + * backend's dma_wmb() is dmb oshst — it ORDERS the coherent fill + * before the device publish; it is NOT the kext's dsb st (which + * additionally waits for completion). Ordering suffices for the + * Linux coherent-DMA + writel doorbell contract (writel itself + * orders prior accesses before the MMIO store); the kext dsb st + * choice is a completion guarantee this driver does not claim. + */ + { + u32 lo = 0, hi = 0; + int err = io->prepare(io->ctx, &lo, &hi); + + if (err) + return err; + io->publish_barrier(io->ctx); + io->wr32(io->ctx, ANE_T6021_BOOT_REG_SCRATCH0, lo); + io->wr32(io->ctx, ANE_T6021_BOOT_REG_SCRATCH1, hi); + } + io->phase(io->ctx, "P6 wake"); + io->wr32(io->ctx, ANE_T6021_BOOT_REG_SCRATCH7, + ANE_T6021_BOOT_WAKE_REQ); + + io->phase(io->ctx, "P7 pollB-DONE"); + /* S7: poll B — DONE; read back the SCRATCH0/1 result u64. */ + for (i = 0; i < ANE_T6021_BOOT_TABLE_POLLS; i++) { + v = io->rd32(io->ctx, ANE_T6021_BOOT_REG_SCRATCH7); + if (v == ANE_T6021_BOOT_ACK) + break; + io->poll_wait(io->ctx); + } + if (v != ANE_T6021_BOOT_ACK) + return -ETIMEDOUT; + *booted = 1; + io->phase(io->ctx, "P7 pollB DONE observed"); + + /* HANDSHAKE COMPLETE != firmware result success: the DONE ack is + * the fw's handshake word; the SCRATCH0/1 result u64 it leaves + * behind has UNSOURCED success semantics (pass6: the fw writes a + * "result" at 0x77a0 whose meaning is not decoded). Expose it + * raw; never infer success from booted alone. + */ + *scratch_result = + ((u64)io->rd32(io->ctx, ANE_T6021_BOOT_REG_SCRATCH1) << 32) | + io->rd32(io->ctx, ANE_T6021_BOOT_REG_SCRATCH0); + return 0; +} + +/* Pinned 13.5 firmware's eight channel descriptors, captured before ACK. + * Type 0 is host-to-firmware; type 1 is firmware-to-host. + * Each ring slot is 64 bytes. The descriptor table uses 256-byte entries. + */ +#define ANE_T6021_CHMAN_ENTRY_SIZE 0x100 +#define ANE_T6021_CHMAN_NAME_LEN 0x40 +#define ANE_T6021_CHMAN_COUNT 8 +#define ANE_T6021_CHMAN_TOTAL 0x10440 + +struct ane_t6021_chman_desc { + char name[ANE_T6021_CHMAN_NAME_LEN]; + u32 type; + u32 bit; + u64 size; + u64 ring; + u8 pad[ANE_T6021_CHMAN_ENTRY_SIZE - ANE_T6021_CHMAN_NAME_LEN - 0x18]; +}; + +struct ane_t6021_chman_static { + const char *name; + u32 type; + u32 bit; + u64 size; + u32 off; /* ring offset from the IPC DVA */ +}; + +static const struct ane_t6021_chman_static +ane_t6021_chman_layout[ANE_T6021_CHMAN_COUNT] = { + { "TERMINAL", 2, 0, 0x300, 0x0800 }, + { "IO", 0, 1, 0x010, 0xc800 }, + { "DEBUG", 0, 2, 0x008, 0xcc00 }, + { "BUF_H2T", 0, 3, 0x040, 0xce00 }, + { "BUF_T2H", 1, 4, 0x040, 0xde00 }, + { "SHAREDMALLOC", 1, 5, 0x008, 0xee00 }, + { "IO_T2H", 1, 6, 0x040, 0xf000 }, + { "DATA_CHAIN_H2T", 0, 7, 0x010, 0x10000 }, +}; + +/* True when entry i of a table read from the IPC surface matches the + * static layout with the rings based at ipc_dva. + */ +static inline bool +ane_t6021_chman_entry_ok(const struct ane_t6021_chman_desc *d, + unsigned int i, u64 ipc_dva) +{ + const struct ane_t6021_chman_static *s = &ane_t6021_chman_layout[i]; + + return !strncmp(d->name, s->name, ANE_T6021_CHMAN_NAME_LEN) && + d->type == s->type && d->bit == s->bit && d->size == s->size && + d->ring == ipc_dva + s->off; +} + +/* Bitmask of mismatching entries (0 = the whole table validated). */ +static inline unsigned int +ane_t6021_chman_check(const struct ane_t6021_chman_desc *t, u64 ipc_dva) +{ + unsigned int i, bad = 0; + + for (i = 0; i < ANE_T6021_CHMAN_COUNT; i++) + if (!ane_t6021_chman_entry_ok(&t[i], i, ipc_dva)) + bad |= 1U << i; + return bad; +} + +/* Before ACK, ownership value 1 prevents firmware from consuming empty H2T slots. */ +static inline bool ane_t6021_chman_host_init(void *ipc, u64 size, u64 dva) +{ + unsigned int i, slot; + + if (!ipc || size < ANE_T6021_CHMAN_TOTAL || + dva > ~(u64)0 - ANE_T6021_CHMAN_TOTAL || + ane_t6021_chman_check(ipc, dva)) + return false; + for (i = 0; i < ANE_T6021_CHMAN_COUNT; i++) { + const struct ane_t6021_chman_static *s = &ane_t6021_chman_layout[i]; + + if (s->type != 0) + continue; + for (slot = 0; slot < s->size; slot++) { + u8 *entry = (u8 *)ipc + s->off + slot * 0x40; + + memset(entry, 0, 24); + entry[0] = 1; + } + } + return true; +} + +#endif /* __ANE_T6021_BOOT_H__ */ diff --git a/drivers/accel/ane/ane_t6021_fwload.c b/drivers/accel/ane/ane_t6021_fwload.c new file mode 100644 index 0000000000000..1e44f0355b485 --- /dev/null +++ b/drivers/accel/ane/ane_t6021_fwload.c @@ -0,0 +1,645 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * T602x/T8112 ANE firmware loader — installed-module port of the proven lab + * staging unit (receipts 2026-09-19-h14-w13-boot-contract.md, + * -w14-staging-proven.md; add-path proof + * 2026-09-28-h14-fsm-secure-park-decode/first-inference.md). + * + * Implements, behind fw_load=1 (default on): + * 1. request_firmware() of the SoC's pinned 13.5 image + * (ane_t602x_soc.fw: selene on T602x, bia on T8112) + * 2. Validation via ane_fw_validate.h (shared with the offline + * regression tools/h14_boot_regression.c): sha256 pin, strict + * exact-image assertions (5 load commands, 2 pinned segments, + * entry 0, bounded LC walk). + * 3. dma_alloc_coherent on the ANE platform device: the buffer is + * DART-mapped through the device's iommu group (stream 0 of the + * three bound instances, H14DartAudit). Segment-wise copy + * (__TEXT fileoff 0x4000 -> vm 0, __DATA -> its vmaddr; vmsize + * tail zero from the coherent alloc). Coherent memory needs no + * explicit cache clean. + * 4. Which copy runs. By default the staged copy (own memory). The lab + * option fw_alias_reserved=1 (T6021 only) maps the copy iBoot + * preloaded at SEG0/SEGi; probe refuses it unless no-map + * /reserved-memory nodes cover both windows (the lab m1n1 adds them, + * the packaged m1n1 1.6.1 does not). The staged copy + * runs: ane_fw_apply_boot_patches() first writes iBoot's runtime + * patches into it (DATA base, RTK_soc, revision, ASC addresses, + * stack guard, ASC tunables), and then it equals the preload byte + * for byte except the random guard (17 captures, + * receipts/2026-10-01-t602x-independent). That needs no reserved + * memory and no preload address, so it is the only mode on SoCs + * without a recorded placement (T6020, T6022, T8112). + * 5. W16 entry alias: the staged fw pages are ALIASED at the latched + * RVBAR entry on the device's default DMA domain. Placement + * contract, receipt receipts/2026-09-20-t6021-entry-alias.md: + * - live RVBAR read64 = 0x10000000001 (bit0 latched, entry + * bits 0x10000000000 = dart-ane vm base); kext law + * (rvbar-lifecycle 6288b0b): bit0 set => the RVBAR write + * branch is skipped, so the ASC fetches AT the latched entry; + * - netconsole 2026-09-20T17:48:19: apple-dart 285800000.iommu + * "translation fault ... code:0x2 (NO PGD FOR IOVA) at + * 0x100000dca10" from the pre-module userspace CPU release — + * the ASC fetched the entry region while nothing was mapped; + * - current-state table walk (kcore, 2026-09-20): entry + * pgd[16] = 0 while the fw leaves are valid -> the region the + * hardware fetches is unmapped and the fw surface is + * physically NON-contiguous (consecutive leaf PAs differ by + * -0x4000). The alias therefore copies each page's PA from + * the live fw translation (iommu_iova_to_phys), never from + * the kernel VA, and verifies every target page unmapped + * before mapping it. + * Driver-managed only: iommu_map() on the attached domain — + * apple_dart implements .map_pages + .iotlb_sync_map, so the + * driver flushes the DART TLB; no raw TTBR/TCR writes. + * Collision ceiling: the iovad allocates top-down from ~4 TiB + * and total mappable RAM is 94.9 GiB (/proc/iomem), so the + * allocator floor 4 TiB - 96 GiB = 0x3e800000000 is far above + * the alias ceiling 0x10005000000; the per-page zero-check + * additionally guards each map. + * + * NOT implemented: the boot step (CPU release + SCRATCH). The userspace + * hybrid stage owns the W8 grant writes; with the alias in place a + * release no longer faults the entry fetch. + * + * Error ownership: every failure path releases the firmware and, once + * allocated, the coherent buffer, and clears ane->fw_buf — remove() + * frees only what fw_buf still names. Load is non-fatal to probe. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ane_t6021.h" +#include "ane_t6021_boot.h" +#include "ane_fw_validate.h" + +/* This object links into both ane_t6021.ko and ane_t6021_rtclient.ko; + * per-object metadata keeps modpost happy for either composition. + */ +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("T6021 ANE firmware staging + entry alias"); + +static bool fw_load = true; +module_param(fw_load, bool, 0444); +MODULE_PARM_DESC(fw_load, + "Validate + DART-map the 13.5 PRELOAD payload (default on: " + "the proven add-path configuration)."); + +static unsigned int fw_extra_ram = 0x200000; +module_param(fw_extra_ram, uint, 0444); +MODULE_PARM_DESC(fw_extra_ram, + "Page-aligned owned RAM after the 5 MiB firmware allocation " + "(default 0x200000, the proven add-path grant; maximum 16 MiB)."); + +/* Preloaded-placement alias, lab only: map the iBoot-reserved SEG0/SEGi + * phys at the entry IOVAs (the preload, with iBoot's patches in place). + * Default off: the own-memory copy needs no reserved RAM, so it runs with + * the packaged m1n1 (T6021 own-memory boot: receipts/2026-10-01-t602x- + * independent, "Boot B"). Only SoCs whose placement is recorded honor it + * (ane_t602x_soc.preload_placement), and only over reserved RAM + * (ane_t6021_fwload_placement_ok). + */ +static bool fw_alias_reserved; +module_param(fw_alias_reserved, bool, 0444); +MODULE_PARM_DESC(fw_alias_reserved, + "Lab, T6021: map reserved SEG0 0x10000848000+0xc4000 at entry and " + "SEG1 0x10001400000+0x438000 after it (probe refuses unless no-map " + "/reserved-memory nodes cover both); 0 (default) = run the staged " + "copy with iBoot's patches replayed (own memory, the only mode on " + "T6020/T6022)."); + +static bool ane_t6021_fw_alias_is_reserved(struct device *dev) +{ + const struct ane_t602x_soc *soc = of_device_get_match_data(dev); + + return fw_alias_reserved && soc->preload_placement; +} + +/* iBoot's T6021 SEG0/SEGi placement (pinned ADT segment-ranges record): + * the physical pages fw_alias_reserved=1 hands to the ASC. + */ +static const struct { u64 phys, len; } ane_t6021_fw_preload[] = { + { 0x10000848000ull, 0xc4000ull }, + { 0x10001400000ull, 0x438000ull }, +}; + +/* Only a no-map /reserved-memory node that the kernel took at boot keeps + * it off those pages. The lab m1n1 adds ane-firmware@ nodes; the packaged + * m1n1 1.6.1 adds none (receipts/2026-10-01-t6021-disk-boot, "Limits"). + */ +static bool ane_t6021_fw_preload_reserved(void) +{ + struct device_node *parent = of_find_node_by_path("/reserved-memory"); + unsigned int i, covered = 0; + + for (i = 0; parent && i < ARRAY_SIZE(ane_t6021_fw_preload); i++) { + u64 phys = ane_t6021_fw_preload[i].phys; + u64 end = phys + ane_t6021_fw_preload[i].len; + struct device_node *np; + + for_each_available_child_of_node(parent, np) { + struct reserved_mem *rmem = of_reserved_mem_lookup(np); + + if (rmem && of_property_read_bool(np, "no-map") && + rmem->base <= phys && end <= rmem->base + rmem->size) { + covered++; + of_node_put(np); + break; + } + } + } + of_node_put(parent); + return covered == ARRAY_SIZE(ane_t6021_fw_preload); +} + +bool ane_t6021_fwload_placement_ok(struct device *dev) +{ + /* BINDING probe-top predicate, as ane_t6021_fwload_options_ok(): + * reserved mode never maps RAM the kernel may own. + */ + return !fw_load || !ane_t6021_fw_alias_is_reserved(dev) || + ane_t6021_fw_preload_reserved(); +} + +bool ane_t6021_fwload_requested(void) +{ + return fw_load; +} + +/* DART page size on t6021 (apple_dart probe line: "pagesize 4000"); + * fw_size is a multiple. Shared with the probe-top predicate below. + */ +#define ANE_T6021_FW_ALIAS_PAGE 0x4000 + +bool ane_t6021_fwload_options_ok(void) +{ + /* BINDING probe-top predicate. MUST be called before + * devm_kzalloc / power / CPU release at every probe site. + * The alloc-time check below runs as defense in depth. + * The lab envelope rule: 16 KiB-aligned, <= 16 MiB. Both alias + * modes map the whole allocation, so the grant needs no mode. + */ + return fw_extra_ram <= SZ_16M && + IS_ALIGNED(fw_extra_ram, ANE_T6021_FW_ALIAS_PAGE); +} + +/* Per-SoC data. RTK_soc_revision on T602x is a measured constant: m1n1 + * gives Linux no chip-revision. T6021: the 17 preload captures (0x11); + * T6020: ANEMinorVersion 1 on community rows 4e8224a39f40, dc53badc436a, + * ea8c35bd20bf, e6069fe959ad; T6022: 17 on row 90c6b9bf3e03 + * (receipts/2026-10-01-community-rows). T6020 and T6022 have the T6021 + * die-0 addresses (13.5 ADTs, t602x-ane.dtsi). + * + * pmu_pa: the firmware's power service programs the seven ANE ps words + * through its DART at IOVA == PA (SetPMUBaseAddress stores the ANE_TD + * word: selene 0x28e084008, bia 0x23b70c010, receipts/2026-10-01-t8112-kit). + * macOS maps the page first; without it the first access faults (NO PMD + * FOR IOVA 0x28e084008, 2026-09-29) and the firmware halts. + * + * T8112 (receipts/2026-10-01-t8112-optin): the revision comes from the + * eFuse words iBoot reads, the ANE_SYS_CPU word is pmgr +0xc008, and the + * kext (type 0x70) opens PWGATE "set" +0x8b8 before the ps words. The TM + * TD word is not known there, so trace_td is off. + */ +const struct ane_t602x_soc ane_t6020_soc = { + .soc = 0x6020, .soc_revision = 0x01, + .fw = &ane_fw_selene, .tunables = &ane_t6020_asc_tunables, + .ps_cpu_off = 0x2e0, .pmu_pa = 0x28e084000ull, + .trace_td_off = 0x1c20458, +}; + +const struct ane_t602x_soc ane_t6021_soc = { + .soc = 0x6021, .soc_revision = 0x11, .preload_placement = true, + .fw = &ane_fw_selene, .tunables = &ane_t602x_asc_tunables, + .ps_cpu_off = 0x2e0, .pmu_pa = 0x28e084000ull, + .trace_td_off = 0x1c20458, +}; + +const struct ane_t602x_soc ane_t6022_soc = { + .soc = 0x6022, .soc_revision = 0x11, + .fw = &ane_fw_selene, .tunables = &ane_t602x_asc_tunables, + .ps_cpu_off = 0x2e0, .pmu_pa = 0x28e084000ull, + .trace_td_off = 0x1c20458, +}; + +const struct ane_t602x_soc ane_t8112_soc = { + .soc = 0x8112, .revision_fuse = true, + .fw = &ane_fw_bia, .tunables = &ane_t8112_asc_tunables, + .ps_cpu_off = 0xc008, .pwgate_off = 0x8b8, + .pmu_pa = 0x23b70c000ull, .ps_off = 0x8, +}; + +static int ane_t6021_pmu_map(struct ane_t6021 *ane, struct iommu_domain *dom) +{ + const struct ane_t602x_soc *soc = of_device_get_match_data(ane->dev); + int prot = IOMMU_READ | IOMMU_WRITE; + int ret; + + if (dev_is_dma_coherent(ane->dev)) + prot |= IOMMU_CACHE; + ret = iommu_map(dom, soc->pmu_pa, soc->pmu_pa, ANE_T6021_FW_ALIAS_PAGE, + prot, GFP_KERNEL); + if (!ret && iommu_iova_to_phys(dom, soc->pmu_pa) != soc->pmu_pa) + ret = -EIO; + dev_info(ane->dev, "pmu: DART map %#llx (IOVA == PA, %#x bytes): %d\n", + soc->pmu_pa, ANE_T6021_FW_ALIAS_PAGE, ret); + return ret; +} + +static int ane_t6021_fw_alias_map(struct ane_t6021 *ane, bool reserved) +{ + struct iommu_domain *dom = iommu_get_domain_for_dev(ane->dev); + void __iomem *eng = ane->base[ANE_T6021_REG_ENGINE]; + u64 rvbar = readq(eng + ANE_ASC_RVBAR); + u64 entry = ane_t6021_rvbar_entry_bits(rvbar); + phys_addr_t pa0 = 0; + int prot = IOMMU_READ | IOMMU_WRITE; + u64 off; + int ret; + + if (!dom) + return -ENODEV; + if (!ane_t6021_rvbar_latched(rvbar) || !entry) { + /* Unlatched branch: the boot path programs RVBAR to the + * fw DVA itself (ane_t6021_rvbar_compose), no alias. + */ + dev_info(ane->dev, + "fwalias: rvbar %016llx not latched/entry 0 — skip (boot reprograms RVBAR)\n", + rvbar); + return 0; + } + if (!ane_t6021_rvbar_entry_ok(entry) || + entry & (ANE_T6021_FW_ALIAS_PAGE - 1)) { + dev_err(ane->dev, + "fwalias: entry %#llx not %#x-aligned in fold\n", + entry, ANE_T6021_FW_ALIAS_PAGE); + return -EINVAL; + } + if (entry + ane->fw_size - 1 > dom->geometry.aperture_end) { + dev_err(ane->dev, + "fwalias: entry %#llx+%#x outside aperture %#llx\n", + entry, ane->fw_size, + (unsigned long long)dom->geometry.aperture_end); + return -ERANGE; + } + if (dev_is_dma_coherent(ane->dev)) + prot |= IOMMU_CACHE; + + if (reserved) { + /* Preloaded placement: the two reserved windows mapped at + * the entry IOVAs. SEG0 0xc4000 is TEXT and SEG1 0x438000 + * is DATA (the 13.5 layout, ane_fw_selene.segs). + * The windows are adjacent, so the mapping is one + * contiguous run of `mapped` bytes starting at entry — + * tracked exactly, because teardown must never unmap a + * page that was not mapped (dart_unmap_pages WARNs on + * holes and the 2026-09-26 state-report unwind hit it). + * SEG1 IOVA = entry + 0xc4000 = 0x100000c4000 per the + * pinned ADT segment-ranges record (0x10000000000 -> + * 0x10000848000 0xc4000; 0x100000c4000 -> 0x10001400000 + * 0x438000). Commit e6612e9 shipped 0x1000000c4000 — one + * hex digit off, mapping DATA at 16 TiB instead: every + * fw_alias_reserved boot since 2026-09-24 left the fw + * DATA section unmapped past the SEG0 head. + */ + struct { u64 iova, phys, len; } win[] = { + { 0x10000000000ull, ane_t6021_fw_preload[0].phys, ane_t6021_fw_preload[0].len }, + { 0, ane_t6021_fw_preload[1].phys, ane_t6021_fw_preload[1].len }, + { entry + 0x4fc000, 0, fw_extra_ram ? ane->fw_size - 0x4fc000 : 0 }, + }; + unsigned int w, windows = fw_extra_ram ? ARRAY_SIZE(win) : 2; + + /* The remap is contiguous: SEG1 base = SEG0 base + SEG0 + * len (segment-ranges order). Derived, never hand-written + * — the standalone literal 0x1000000c4000 that shipped in + * e6612e9 was one digit off and mapped DATA at 16 TiB. + * The remap base must also equal the latched entry, or + * the fetch head is not where we mapped. + */ + win[1].iova = win[0].iova + win[0].len; + if (win[0].iova != entry) { + dev_err(ane->dev, + "fwalias: remap base %#llx != latched entry %#llx\n", + win[0].iova, entry); + return -EINVAL; + } + + for (w = 0; w < windows; w++) { + u64 o; + + for (o = 0; o < win[w].len; o += ANE_T6021_FW_ALIAS_PAGE) { + phys_addr_t pa = w < 2 ? win[w].phys + o : + iommu_iova_to_phys(dom, ane->fw_iova + win[w].iova + o - entry); + if (!pa || !IS_ALIGNED(pa, ANE_T6021_FW_ALIAS_PAGE)) { + ret = -EFAULT; + goto err_unmap_mapped; + } + if (iommu_iova_to_phys(dom, win[w].iova + o)) { + dev_err(ane->dev, + "fwalias: reserved entry +%#llx mapped — refusing\n", + win[w].iova + o - entry); + ret = -EEXIST; + goto err_unmap_mapped; + } + ret = iommu_map(dom, win[w].iova + o, + pa, + ANE_T6021_FW_ALIAS_PAGE, prot, + GFP_KERNEL); + if (ret) { + dev_err(ane->dev, + "fwalias: reserved map +%#llx: %d\n", + win[w].iova + o - entry, ret); + goto err_unmap_mapped; + } + /* per-window successfully mapped bytes */ + ane->fw_alias_ext_len[w] = o + ANE_T6021_FW_ALIAS_PAGE; + ane->fw_alias_ext_iova[w] = win[w].iova; + if (iommu_iova_to_phys(dom, win[w].iova + o) != pa) { + ret = -EIO; + goto err_unmap_mapped; + } + } + } + ane->fw_alias_extn = windows; + ane->fw_alias_iova = entry; + dev_info(ane->dev, + "fwalias: reserved SEG0/SEGi at entry %#llx (%llx+%zx %llx+%zx, preloaded placement)\n", + entry, + ane->fw_alias_ext_iova[0], ane->fw_alias_ext_len[0], + ane->fw_alias_ext_iova[1], ane->fw_alias_ext_len[1]); + if (fw_extra_ram) + dev_info(ane->dev, "fwalias: owned heap [%#llx,%#llx) roundtrip verified\n", + win[2].iova, win[2].iova + win[2].len); + return ane_t6021_pmu_map(ane, dom); + +err_unmap_mapped: + /* Cleanup exactly the per-window bytes we mapped; windows + * are not assumed adjacent (live trace: a hole between + * SEG0 and SEGi), and foreign collision mappings are + * never touched. + */ + for (w = 0; w < windows; w++) + if (ane->fw_alias_ext_len[w]) + iommu_unmap(dom, ane->fw_alias_ext_iova[w], + ane->fw_alias_ext_len[w]); + memset(ane->fw_alias_ext_len, 0, sizeof(ane->fw_alias_ext_len)); + ane->fw_alias_extn = 0; + return ret; + } + + for (off = 0; off < ane->fw_size; off += ANE_T6021_FW_ALIAS_PAGE) { + phys_addr_t pa = iommu_iova_to_phys(dom, ane->fw_iova + off); + + if (!pa || pa & (ANE_T6021_FW_ALIAS_PAGE - 1)) { + dev_err(ane->dev, + "fwalias: fw +%#llx untranslated (%pa)\n", + off, &pa); + ret = -EFAULT; + goto err_unmap; + } + if (iommu_iova_to_phys(dom, entry + off)) { + dev_err(ane->dev, + "fwalias: entry +%#llx already mapped — refusing\n", + off); + ret = -EEXIST; + goto err_unmap; + } + ret = iommu_map(dom, entry + off, pa, + ANE_T6021_FW_ALIAS_PAGE, prot, GFP_KERNEL); + if (ret) { + dev_err(ane->dev, "fwalias: iommu_map +%#llx: %d\n", + off, ret); + goto err_unmap; + } + if (!off) + pa0 = pa; + } + + /* full per-page roundtrip: every alias page must resolve to the + * same PA as its fw source page (not just page 0) + */ + for (off = 0; off < ane->fw_size; off += ANE_T6021_FW_ALIAS_PAGE) { + if (iommu_iova_to_phys(dom, entry + off) != + iommu_iova_to_phys(dom, ane->fw_iova + off)) { + dev_err(ane->dev, + "fwalias: roundtrip mismatch at +%#llx\n", off); + ret = -EIO; + /* Mapping finished: unwind every page, not only the + * prefix already checked by this verification loop. + */ + off = ane->fw_size; + goto err_unmap; + } + } + + ane->fw_alias_iova = entry; + ane->fw_alias_ext_iova[0] = entry; + ane->fw_alias_ext_len[0] = ane->fw_size; + ane->fw_alias_extn = 1; + dev_info(ane->dev, + "fwalias: entry %#llx <- %u dart pages aliased from fw %pad (first %pa, roundtrip OK)\n", + entry, ane->fw_size / ANE_T6021_FW_ALIAS_PAGE, + &ane->fw_iova, &pa0); + /* The firmware's power service needs the pmgr sub-block mapped + * IOVA == PA in every vehicle (NO PMD FOR IOVA 0x28e084008, + * 2026-09-29); the staged-DMA alias branch skipped it and left + * a bisect run booting a halting fw. + */ + return ane_t6021_pmu_map(ane, dom); + +err_unmap: + if (off) + iommu_unmap(dom, entry, off); + return ret; +} + +/* RTK_soc_revision. T8112 reads it as iBoot does: the two eFuse words + * of the DT "fuse" window (0x23d2c8060, 8 bytes), decoded by + * ane_t8112_fuse_revision(). Two non-posted 32-bit reads and nothing + * else: the eFuse block (ADT pmgr reg[36]) is always on, and m1n1 reads + * its ATC and GPU fuses at 0x23d2c8484 and 0x23d2c84dc on every T8112 + * boot. Without the window the revision is unknown, so the load refuses. + */ +static int ane_t6021_soc_revision(struct ane_t6021 *ane, + const struct ane_t602x_soc *soc, u32 *rev) +{ + struct resource *res; + void __iomem *fuse; + u32 w0, w1; + + if (!soc->revision_fuse) { + *rev = soc->soc_revision; + return 0; + } + res = platform_get_resource_byname(to_platform_device(ane->dev), + IORESOURCE_MEM, "fuse"); + if (!res || resource_size(res) != 8) { + dev_err(ane->dev, + "fwload: no 8-byte \"fuse\" window: chip revision unknown, refusing\n"); + return -ENODEV; + } + fuse = ioremap_np(res->start, 8); + if (!fuse) + return -ENOMEM; + w0 = readl(fuse); + w1 = readl(fuse + 4); + iounmap(fuse); + *rev = ane_t8112_fuse_revision(w0, w1); + dev_info(ane->dev, "fwload: chip revision %#x (fuse %#llx: %08x %08x)\n", + *rev, (u64)res->start, w0, w1); + return 0; +} + +/* Own memory (header item 4): iBoot's runtime patches, with values from + * the running system: the latched entry (or the staged DVA that the boot + * path programs when RVBAR is not latched), the DT engine window (probe + * already refused a node without it), the compatible, a fresh guard. + */ +static int ane_t6021_fw_patch(struct ane_t6021 *ane, u8 *img) +{ + const struct ane_t602x_soc *soc = of_device_get_match_data(ane->dev); + struct resource *res = platform_get_resource(to_platform_device(ane->dev), + IORESOURCE_MEM, 0); + u64 rvbar = readq(ane->base[ANE_T6021_REG_ENGINE] + ANE_ASC_RVBAR); + u64 entry = ane_t6021_rvbar_latched(rvbar) ? + ane_t6021_rvbar_entry_bits(rvbar) : 0; + u64 guard = get_random_u64(); + struct ane_fw_boot_patch p = { + .exec_base = entry ?: ane->fw_iova, + /* iBoot's guards have one zero byte at a random position */ + .stack_guard = guard & ~(0xffull << (8 * (guard >> 61))), + .soc = soc->soc, + .cpu_pa = res->start + ANE_ASC_CPU_BASE, + .wrapper_pa = res->start + ANE_ASC_WRAPPER_BASE, + }; + const char *reason = NULL; + int ret; + + ret = ane_t6021_soc_revision(ane, soc, &p.soc_revision); + if (ret) + return ret; + if (ane_fw_apply_boot_patches(img, soc->fw, soc->tunables, &p, &reason)) { + dev_err(ane->dev, "fwload: own memory: %s (rev %#x)\n", reason, + p.soc_revision); + return -EINVAL; + } + dev_info(ane->dev, + "fwload: own memory: iBoot patches replayed (soc %#x rev %#x DATA %#llx cpu %#llx wrapper %#llx)\n", + p.soc, p.soc_revision, + p.exec_base + soc->fw->segs[1].vmaddr, p.cpu_pa, + p.wrapper_pa); + return 0; +} + +int ane_t6021_fwload_probe(struct ane_t6021 *ane) +{ + const struct ane_t602x_soc *soc = of_device_get_match_data(ane->dev); + const struct ane_fw_image *img = soc->fw; + const struct firmware *fw = NULL; + struct ane_fw_seg segs[ANE_FW_NSEGS]; + u64 entry = 0; + u8 actual_sha[32]; + void *buf; + dma_addr_t iova; + unsigned int i; + const char *reason = NULL; + u32 alloc_size = ANE_FW_BUF_SIZE + fw_extra_ram; + int ret; + bool reserved = ane_t6021_fw_alias_is_reserved(ane->dev); + + if (!fw_load) + return 0; + if (fw_extra_ram > SZ_16M || !IS_ALIGNED(fw_extra_ram, ANE_T6021_FW_ALIAS_PAGE)) + return -EINVAL; + /* The 64-bit coherent mask is set once in ane_t6021_probe, + * BEFORE rtkit_init allocates the rings (W15 review). + */ + + ret = request_firmware(&fw, img->name, ane->dev); + if (ret) { + dev_err(ane->dev, + "fwload: request_firmware(%s): %d — stage the payload " + "with omarchy-ane-firmware-fetch\n", img->name, ret); + return ret; + } + + sha256(fw->data, fw->size, actual_sha); + ret = ane_fw_validate_blob(fw->data, fw->size, img, actual_sha, + segs, &entry, &reason); + if (ret) { + dev_err(ane->dev, "fwload: validation failed: %s\n", + reason ? reason : "?"); + release_firmware(fw); + return ret; + } + + buf = dma_alloc_coherent(ane->dev, alloc_size, &iova, GFP_KERNEL); + if (!buf) { + dev_err(ane->dev, "fwload: coherent alloc %#x failed\n", + alloc_size); + release_firmware(fw); + return -ENOMEM; + } + + for (i = 0; i < ANE_FW_NSEGS; i++) { + if (segs[i].filesize) + memcpy(buf + segs[i].vmaddr, + fw->data + segs[i].fileoff, segs[i].filesize); + } + + ane->fw_buf = buf; + ane->fw_iova = iova; + ane->fw_size = alloc_size; + + dev_info(ane->dev, + "fwload: %s PRELOAD validated + DART-mapped: entry %#llx, iova %pad size %#x\n", + img->name, entry, &iova, alloc_size); + + ret = reserved ? 0 : ane_t6021_fw_patch(ane, buf); + if (!ret) + ret = ane_t6021_fw_alias_map(ane, reserved); + if (ret) { + ane_t6021_fwload_remove(ane); + release_firmware(fw); + return ret; + } + + release_firmware(fw); + return 0; +} + +void ane_t6021_fwload_remove(struct ane_t6021 *ane) +{ + if (ane->fw_alias_extn) { + struct iommu_domain *dom = iommu_get_domain_for_dev(ane->dev); + int i; + + /* Unmap exactly the per-window recorded extents: the + * reserved-alias windows are not assumed adjacent (live + * trace hole between SEG0 and SEGi) and unmapping bytes + * that were never mapped trips dart_unmap_pages + * (io-pgtable-dart.c:319 WARN, 2026-09-26). + */ + if (dom) + for (i = 0; i < ane->fw_alias_extn; i++) + iommu_unmap(dom, ane->fw_alias_ext_iova[i], + ane->fw_alias_ext_len[i]); + ane->fw_alias_extn = 0; + } + if (!ane->fw_buf) + return; + dma_free_coherent(ane->dev, ane->fw_size, ane->fw_buf, ane->fw_iova); + ane->fw_buf = NULL; + ane->fw_size = 0; +} diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c new file mode 100644 index 0000000000000..9d1e43ecc508b --- /dev/null +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -0,0 +1,2363 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * ane_t6021_rtclient.c — T6021 (H14 / M2) installed ANE driver. + * + * One module, two surfaces: + * - boot/transport: fenced Linux-context firmware start (fw_start=1 + * default — the proven add-path config), shared boot/fwload + * contract units, then the legacy 13.5 ChMan transport for the + * three load-time commands (LOAD_PROGRAM 0x200, CREATE_PROCESS + * 0x202, PROCEDURE_CALL 0x204) driven through the ioctls in this + * file; + * - DRM accel: BO_INIT/BO_FREE plus PROG_LOAD/PROC_CREATE/EXEC per + * include/uapi/drm/ane_accel.h (ABI 2). Sections ride BOs the + * user supplies; one global mutex serializes every firmware + * command; completion = the legacy exchange's reply PLUS the + * firmware's finish event on the IO_T2H ring. EXEC also returns the + * fw's target-to-host slots (the sequencer's per-step drain). + * + * Compiled defaults are the load-run.sh parameter list — a bare + * `insmod ane_t6021.ko` is the proven add-path configuration on + * boot 3ab812a3 (fw_load=1 fw_start=1 fw_start_dapf=0 legacy_only=1 + * legacy_query=1 scratch3_ack=1 poll_rx=1 ...) — except hello_wait_ms, + * 1000 in that list and 0 here (see its definition). + * Remaining parameters are overridable from sysfs for bisection only. + * + * Division of labor (receipts/2026-09-22-t6021-rtkit-port): + * - CPU start: kernel-context start is the proven-safe route on this + * box — m1n1 RTBuddy path is dormant (W5/W6 MGMT session wall) and + * the 13.5 (22G74) selene image brings up the legacy ChMan + * transport after DONE without RTKit. Boot is fenced in + * ane_t6021_boot.c (entry fold, RVBAR skip-or-fold, scratch, + * RUN, poll A/B, publish/wake) and audited in + * tools/h14_boot_regression.c. + * - genpd/pmgr: Runtime PM + the DT power-domains binding owns the + * raise; probe verifies ACTUAL on ane_cpu (pmgr window) before + * the first engine read (G1 gate). The pre-CPU engine table block + * is intentionally NOT written (preboot_table_mode = 2 = SKIP, the + * 2026-09-20-era live-fault discriminator). macOS-form mpm_off is + * skipped: with fw_start_mpm_off=0 (the proven config) the + * boot-raised ane_sys_mpm stays on; the boot writes do not depend + * on it. + * - mailbox: ane_rtclient_validate_chman sees the 'IPC ' surface the + * fw published during boot; the three load-time commands travel + * the IO ring (channel 1) over ane_rtclient_legacy_exchange. Each + * command advances the cursor on the next 64-byte slot — resending + * to slot 0 wedges the ring (sequencer rule). + * - non-posted MMIO (DT "nonposted-mmio" -> IORESOURCE_MEM_NONPOSTED) + * is mandatory throughout the ANE aperture; posted writel froze the + * box. + * - Firmware alias: the staged copy (own memory, default) or, with the + * lab option fw_alias_reserved=1, the iBoot-reserved SEG0/SEGi phys + * is aliased at the latched RVBAR entry; the subsequent kernel writes + * into the entry region translate through dart-ane0 instead of + * faulting. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ane_t6021.h" +#include "ane_t6021_boot.h" + +#include + +/* Doorbell (IPI) block, engine + 0x1844000: set +0, pending +0x8000, + * ack +0xc000 (kext aneInterruptHandler reads +0x184c000 and writes + * +0x1850000 with no SoC branch, receipts/2026-10-01-t8112-ane). + */ +#define ANE_IPI_OFF 0x1844000 + +/* CPU_STATUS bits (m1n1 ASCRegs shape) */ +#define ANE_ASC_CPU_STATUS_RUNNING BIT(0) +#define ANE_ASC_CPU_STATUS_STOPPED BIT(1) + +/* Boot-time module parameters — defaults are the proven add-path + * parameter list (boot 3ab812a3 / load-run.sh), except fw_alias_reserved, + * which defaults to own memory (receipts/2026-10-01-t602x-independent, + * "Boot B"). fw_load, fw_extra_ram and fw_alias_reserved live in + * ane_t6021_fwload.c and boot_prevent_nap in ane_t6021_boot.c (single + * registration each). Remaining knobs are overridable from sysfs for + * bisection only. + */ +static bool fw_start = true; +module_param(fw_start, bool, 0444); +MODULE_PARM_DESC(fw_start, + "Fenced Linux-context firmware start: stage + alias, then the boot sequence; default on (proven add-path config)."); + +static int fw_start_table_mode = 2; +module_param(fw_start_table_mode, int, 0444); +MODULE_PARM_DESC(fw_start_table_mode, + "Pre-CPU engine table block: 0 abort, 1 write, 2 skip (default, proven config)."); + +static bool fw_start_rtb_mode; +module_param(fw_start_rtb_mode, bool, 0444); +MODULE_PARM_DESC(fw_start_rtb_mode, + "RTBuddy/RTKit-app-endpoint select; default off (legacy ChMan transport)."); + +/* Drive RX by apple_rtkit_poll from a workqueue even though a recv + * IRQ exists (lab poll_rx=1, the proven add-path value: the raw recv + * line is unproven, so the worker is what services HELLO/EPMAP). + */ +static bool poll_rx = true; +module_param(poll_rx, bool, 0444); +MODULE_PARM_DESC(poll_rx, + "Drive RX by apple_rtkit_poll from a workqueue (default on, proven config)."); + +/* STARTEP every fw-announced app endpoint (>= 0x20) after a successful + * RTKit handshake (lab start_app_eps=1 default). + */ +static bool start_app_eps = true; +module_param(start_app_eps, bool, 0444); +MODULE_PARM_DESC(start_app_eps, + "STARTEP every fw-announced app endpoint after the handshake (default on)."); + +static bool scratch3_ack = true; +module_param(scratch3_ack, bool, 0444); +MODULE_PARM_DESC(scratch3_ack, + "After DONE, write SCRATCH3 = 0x08042006 (selene 0x7edc, kext 0x95eaee4). Default on; 0 withholds."); + +static bool legacy_only = true; +module_param(legacy_only, bool, 0444); +MODULE_PARM_DESC(legacy_only, + "Use the pinned 13.5 legacy ChMan transport (default on, proven add-path config)."); + +static bool legacy_query = true; +module_param(legacy_query, bool, 0444); +MODULE_PARM_DESC(legacy_query, + "Service bounded startup allocations and CONFIG_GET; default on (proven config)."); + +/* 0 (default) skips RTKit in legacy mode, so the ANE mailbox never + * starts. Measured 2026-09-30 (receipts/2026-09-30-t6021-stock-mailbox): + * the 13.5 firmware sent no HELLO on any recorded boot (-ETIME after + * 1000 ms), and starting the mailbox enables AIC2 884, a level line + * that then fired ~700,000 times/s (one CPU of hardirq time). A + * firmware that speaks RTKit needs hello_wait_ms=1000. + */ +static unsigned int hello_wait_ms; +module_param(hello_wait_ms, uint, 0444); +MODULE_PARM_DESC(hello_wait_ms, + "RTKit HELLO wait in legacy mode; 0 (default) skips RTKit and leaves the mailbox stopped. A firmware that speaks RTKit needs 1000."); + +#define ANE_LEGACY_ALLOCS 8192 +#define ANE_LEGACY_BYTES SZ_512M + +struct ane_legacy_buffer { + void *cpu; + dma_addr_t dma; + size_t size; +}; + +struct ane_rtclient { + struct device *dev; + void __iomem *engine; + void __iomem *pmgr; + const struct ane_t602x_soc *soc; + struct apple_rtkit *rtk; + struct reset_control *cpu_rst; + struct delayed_work poll_work; + + /* fw_start=1 view over the shared boot/fwload contract units. */ + struct ane_t6021 *fw; + + /* A CPU we released is (or may be) running: state HELD — surfaces, + * rings, IRQ, power links preserved; no unwind; reboot reclaims + * (wedged-pin rule). + */ + bool held; + + bool boot_done; + + /* ChMan descriptor table validated against the 'IPC ' surface. */ + bool chman_ok; + struct ane_legacy_buffer *legacy_buffers; + u32 legacy_allocated; + size_t legacy_bytes; + u32 legacy_malloc_cursor; + u32 legacy_cmd_cursor[ANE_T6021_CHMAN_COUNT]; + /* One reusable 16 KiB command buffer for every host command + * (CONFIG_GET + the three ioctls). Protocol-legal to reuse: an + * exchange returns 0 only after the IO slot flipped back to + * host-owned with a zero status (the lab's own completion + * predicate), so the firmware has fully consumed the previous + * command. The legacy_buffers table stays for fw MALLOC replies, + * which the firmware may reference forever (held until reboot). + */ + struct ane_legacy_buffer *cmd_buf; +}; + +/* Per-open BO ownership (drm_file->driver_priv). Handles live in the + * fd's list; closing the fd drops its handles (postclose). The + * coherent buffers themselves live as long as something references + * them: a user mapping, or the firmware (see ane_t6021_bo_release). + */ +struct ane_t6021_fd { + struct list_head bos; +}; + +/* Per-file handle counter and every per-fd list against same-fd concurrent + * ioctls. A BO whose IOVA reached the firmware (fw_ref) never goes back to + * the kernel: a program section is held until reboot, an io BO is parked + * for reuse (see ane_t6021_bo_release). Every other BO frees at its last + * reference. + * + * Per-BO cap raised 2026-09-30: the 256 KiB Qwen-class matvec weights seen + * by the H14 compiler are bounded by `reduction * columns * 2`, which + * reaches 20 MiB at (K,N)=(2048,5120). The same cap serves the + * Qwen4-attention (K,N)=(4096,4096) constant at 32 MiB and any H14 + * softmax/reduction with an 8 MiB table. Total BO bytes across all fds are + * capped separately (bo_total_max_mb) by an atomic counter, enforced at + * alloc and released when the memory is really freed. Program sections stay + * held until reboot, and the 38 Qwen programs alone hold about 2.6 GiB. + * A cap of 0 refuses every BO_INIT with -ENOSPC; there is no unlimited + * value. Every BO also needs IOVA below 4 GiB (32-bit DMA mask), so + * allocations fail with -ENOMEM near that bound whatever the cap. + * The 16 KiB alignment check is unchanged: every DMA site assumes it. + */ +#define ANE_T6021_BO_MAX SZ_1G +#define ANE_T6021_BO_HASH_CHUNK SZ_1M + +struct ane_t6021_bo { + struct list_head node; + struct ane_t6021_fd *owner; + u32 handle; + struct kref refcount; /* handle + user mappings */ + void *cpu; /* coherent mapping */ + dma_addr_t dma; + size_t size; + bool fw_ref; /* the firmware received this IOVA */ + bool fw_program; /* a loaded program keeps this IOVA */ + struct device *dev; +}; + +static DEFINE_MUTEX(ane_t6021_bo_lock); +static u32 ane_t6021_next_handle = 1; +static atomic64_t ane_t6021_bo_total_bytes = ATOMIC64_INIT(0); + +static unsigned int bo_total_max_mb = 12288; +module_param(bo_total_max_mb, uint, 0444); +MODULE_PARM_DESC(bo_total_max_mb, + "Cap on the BO bytes held at one time, in MiB (default 12288)"); + +static int ane_t6021_bo_total_get(char *buf, const struct kernel_param *kp) +{ + return sysfs_emit(buf, "%lld\n", + (long long)atomic64_read(&ane_t6021_bo_total_bytes)); +} + +static int ane_t6021_bo_total_set(const char *val, + const struct kernel_param *kp) +{ + return -EPERM; +} + +static const struct kernel_param_ops ane_t6021_bo_total_ops = { + .set = ane_t6021_bo_total_set, + .get = ane_t6021_bo_total_get, +}; +module_param_cb(bo_total_bytes, &ane_t6021_bo_total_ops, NULL, 0444); +MODULE_PARM_DESC(bo_total_bytes, + "Read only: the BO bytes counted against bo_total_max_mb now"); + +/* Mark the device quarantined: a timed-out command left the firmware + * queue state unknown. The only safe next step is to refuse further + * ioctls until a reboot reclaims the surfaces (wedged-pin rule). + */ +static atomic_t ane_t6021_quarantined = ATOMIC_INIT(0); + +/* Parked io BOs: their last user is gone, the IOVA stays mapped and the + * bytes stay counted. BO_INIT of the same page-aligned size takes one, + * so held memory stays at the peak of concurrent io BOs instead of + * growing with every process until the BO cap refuses BO_INIT. + */ +static LIST_HEAD(ane_t6021_bo_pool); +static DEFINE_SPINLOCK(ane_t6021_bo_pool_lock); + +/* Final put: the last handle or mapping is gone. The firmware never sees + * a freed IOVA (lab rule), so a fw_ref BO is never freed: a program + * section stays held, because a cached firmware program keeps reading + * it; an io BO goes to the pool, unless a quarantined firmware may still + * write it. Every other BO frees here, so the BO cap bounds only the + * memory the firmware may touch. + */ +static void ane_t6021_bo_release(struct kref *ref) +{ + struct ane_t6021_bo *bo = container_of(ref, struct ane_t6021_bo, + refcount); + + if (!bo->fw_ref) { + atomic64_sub(PAGE_ALIGN(bo->size), &ane_t6021_bo_total_bytes); + dma_free_coherent(bo->dev, bo->size, bo->cpu, bo->dma); + kfree(bo); + } else if (bo->fw_program || atomic_read(&ane_t6021_quarantined)) { + kfree(bo); + } else { + spin_lock(&ane_t6021_bo_pool_lock); + list_add(&bo->node, &ane_t6021_bo_pool); + spin_unlock(&ane_t6021_bo_pool_lock); + } +} + +/* A parked io BO of SIZE's page-aligned size, or NULL. */ +static struct ane_t6021_bo *ane_t6021_bo_pool_take(size_t size) +{ + struct ane_t6021_bo *bo; + + if (atomic_read(&ane_t6021_quarantined)) + return NULL; + spin_lock(&ane_t6021_bo_pool_lock); + list_for_each_entry(bo, &ane_t6021_bo_pool, node) { + if (PAGE_ALIGN(bo->size) == PAGE_ALIGN(size)) { + list_del(&bo->node); + spin_unlock(&ane_t6021_bo_pool_lock); + return bo; + } + } + spin_unlock(&ane_t6021_bo_pool_lock); + return NULL; +} + +/* A user mapping holds its BO's memory until it is torn down: open + * (fork, mremap split) takes a reference, close drops it. A BO the + * user freed while mapped stays allocated until the last mapping + * goes away. + */ +static void ane_t6021_vm_open(struct vm_area_struct *vma) +{ + kref_get(&((struct ane_t6021_bo *)vma->vm_private_data)->refcount); +} + +static void ane_t6021_vm_close(struct vm_area_struct *vma) +{ + kref_put(&((struct ane_t6021_bo *)vma->vm_private_data)->refcount, + ane_t6021_bo_release); +} + +static const struct vm_operations_struct ane_t6021_vm_ops = { + .open = ane_t6021_vm_open, + .close = ane_t6021_vm_close, +}; + +/* Serializes every firmware command (LOAD/CREATE/CALL) so the + * cursor-on-next-64-byte-slot rule and the PMGR/TM gate cannot race. + */ +static DEFINE_MUTEX(ane_t6021_fw_lock); + +/* Forward declaration (defined below). */ +static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, + struct ane_legacy_buffer *command, + size_t length, u16 opcode, + unsigned int channel, + unsigned int timeout_ms); + +/* ---- boot-contract helpers (shared with boot.c / fwload.c) ---- */ + +static int ane_rtclient_legacy_alloc(struct ane_rtclient *ane, u64 size) +{ + struct ane_legacy_buffer *buffer; + + if (!size || !IS_ALIGNED(size, SZ_16K) || size > SZ_2M || + ane->legacy_allocated == ANE_LEGACY_ALLOCS || + size > ANE_LEGACY_BYTES - ane->legacy_bytes) + return -E2BIG; + buffer = &ane->legacy_buffers[ane->legacy_allocated]; + buffer->cpu = dma_alloc_coherent(ane->dev, size, &buffer->dma, + GFP_KERNEL); + if (!buffer->cpu) + return -ENOMEM; + if (!IS_ALIGNED(buffer->dma, SZ_16K) || + (ane->fw && !ane_t6021_fw_alias_iova_ok(ane->fw, buffer->dma, size))) { + dma_free_coherent(ane->dev, size, buffer->cpu, buffer->dma); + buffer->cpu = NULL; + return -ERANGE; + } + memset(buffer->cpu, 0, size); + buffer->size = size; + ane->legacy_bytes += size; + ane->legacy_allocated++; + return 0; +} + +/* Validate the chman table the fw published in the 'IPC ' surface. */ +static void ane_rtclient_validate_chman(struct ane_rtclient *ane) +{ + struct ane_t6021 *a = ane->fw; + const struct ane_t6021_chman_desc *t; + unsigned int i, bad; + + if (!a || !a->boot_ipc) { + dev_info(ane->dev, + "chman: no host IPC surface — table not validated\n"); + return; + } + if (a->boot_ipc_size < ANE_T6021_CHMAN_TOTAL) { + dev_warn(ane->dev, + "chman: IPC surface %#llx bytes < fw layout %#x — table not validated\n", + a->boot_ipc_size, ANE_T6021_CHMAN_TOTAL); + return; + } + + dma_rmb(); + t = a->boot_ipc; + bad = ane_t6021_chman_check(t, a->boot_ipc_iova); + for (i = 0; i < ANE_T6021_CHMAN_COUNT; i++) { + const struct ane_t6021_chman_desc *d = &t[i]; + const struct ane_t6021_chman_static *s = &ane_t6021_chman_layout[i]; + + dev_info(ane->dev, + "chman[%u]: name=\"%.*s\" type=%u bit=%u size=%#llx %s (static: %s/%u/%u/%#llx/ipc+%#x)\n", + i, ANE_T6021_CHMAN_NAME_LEN, d->name, d->type, d->bit, + d->size, + (bad & BIT(i)) ? "MISMATCH" : "OK", + s->name, s->type, s->bit, s->size, s->off); + } + + ane->chman_ok = !bad; + dev_info(ane->dev, "chman: table %s (mismatch mask %#x)\n", + bad ? "NOT VALIDATED" : "VALIDATED", bad); +} + +/* Legacy ChMan exchange (post-DONE 13.5 transport). Resends to the same + * 64-byte IO slot would wedge the ring; cursor advances after each + * completed command (selene fw decodes bit0 = host-owned). + */ +static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, + struct ane_legacy_buffer *command, + size_t length, u16 opcode, + unsigned int channel, + unsigned int timeout_ms) +{ + struct ane_t6021 *a = ane->fw; + u64 *io, *malloc_ring, header; + u64 *t2h_buf, *t2h_ioq; + void __iomem *ipi = ane->engine + ANE_IPI_OFF; + unsigned int cursor = ane->legacy_malloc_cursor; + unsigned long deadline; + u32 *reply; + int result = -ETIMEDOUT; + + if (!ane->held || !ane->chman_ok || + ane_t6021_chman_check(a->boot_ipc, a->boot_ipc_iova)) + return -EPROTO; + if (length < 8 || length > command->size || + channel >= ANE_T6021_CHMAN_COUNT) { + result = -EINVAL; + goto out; + } + io = a->boot_ipc + ane_t6021_chman_layout[channel].off + + (size_t)ane->legacy_cmd_cursor[channel] * 64; + malloc_ring = a->boot_ipc + ane_t6021_chman_layout[5].off; + t2h_buf = a->boot_ipc + ane_t6021_chman_layout[4].off; + t2h_ioq = a->boot_ipc + ane_t6021_chman_layout[6].off; + + if (!(READ_ONCE(io[0]) & 1)) { + result = -EBUSY; + goto out; + } + reply = command->cpu; + ((u16 *)reply)[2] = opcode; + WRITE_ONCE(io[1], length); + WRITE_ONCE(io[2], length); + dma_wmb(); + WRITE_ONCE(io[0], command->dma); + dma_wmb(); + writel(BIT(ane_t6021_chman_layout[channel].bit), ipi); + deadline = jiffies + msecs_to_jiffies(timeout_ms); + while (time_before(jiffies, deadline)) { + u32 pending = readl(ipi + 0x8000); + + if (pending) { + writel(pending, ipi + 0xc000); + mb(); + } + header = READ_ONCE(io[0]); + if (!(header & 1)) { + /* malloc-ring service: the fw may ask for an + * allocation while we wait; service those until + * the command slot flips back. + */ + u64 *slot = malloc_ring + cursor * 8; + u64 alloc_hdr = READ_ONCE(slot[0]); + + if (!(alloc_hdr & 1)) { + u64 size, tag; + struct ane_legacy_buffer *buf; + + dma_rmb(); + size = READ_ONCE(slot[1]); + tag = READ_ONCE(slot[2]); + /* Lab rule: a valid MALLOC slot carries + * header 0 and a u32 tag; anything else + * is a malformed ring and we stop. + */ + if (alloc_hdr || tag > U32_MAX) { + result = -EOPNOTSUPP; + goto out; + } + if (ane_rtclient_legacy_alloc(ane, size)) { + result = -ENOMEM; + goto out; + } + buf = &ane->legacy_buffers[ane->legacy_allocated - 1]; + WRITE_ONCE(slot[1], 0); + WRITE_ONCE(slot[2], ane->legacy_allocated); + dma_wmb(); + WRITE_ONCE(slot[0], buf->dma | 1); + dma_wmb(); + writel(BIT(5), ipi); + cursor = (cursor + 1) % + ane_t6021_chman_layout[5].size; + } + } + header = READ_ONCE(io[0]); + if (header & 1) { + dma_rmb(); + result = (header == (command->dma | 1) && + READ_ONCE(io[1]) == length && + READ_ONCE(io[2]) == 0 && + ((u16 *)reply)[2] == opcode && + ((u16 *)reply)[3] == 0) ? 0 : -EPROTO; + ane->legacy_cmd_cursor[channel] = + (ane->legacy_cmd_cursor[channel] + 1) % + ane_t6021_chman_layout[channel].size; + goto out; + } + /* legacy_fast_poll=1 (the proven add-path cadence): poll + * the rings every 50 us instead of sleeping 1-2 ms. + */ + udelay(50); + } + dev_info(ane->dev, "LEGACY timeout ch=%u io=%016llx\n", + channel, READ_ONCE(io[0])); +out: + ane->legacy_malloc_cursor = cursor; + return result; +} + +/* Time to let the output writes land after the completion signal. + * Measured 2026-09-29 on some boots: the fw ack, the TD counter and the + * TQ words all report done ~0.13 ms before the output reaches DRAM (the + * output read as zeros in about 1 of 5 calls). The finish event arrives + * at the same time for short programs. No signal for "output landed" is + * known, so the wait is a fixed margin of about 8x the lag. + */ +static unsigned int call_settle_us = 1000; +module_param(call_settle_us, uint, 0644); +MODULE_PARM_DESC(call_settle_us, + "Microseconds to wait after a CALL completes so its output lands (default 1000, 0 = none)"); + +/* trace_td: a read-only timeline of each CALL for performance work. Off + * by default; switch it at runtime with + * /sys/module/ane_t6021/parameters/trace_td (no device needed). Off, the + * CALL path is unchanged. On, the completion wait polls every 20-40 us + * and also reads the last-committed-TD word. It reads that word only + * while the seven ANE pmgr PS words read 0x3ff. No register is written. + * + * Provenance (omarchy-ane commits): + * - The TD word: engine + ane_t602x_soc.trace_td_off; on T602x TM + * 0x285c00000 + TD window 0x20400 + 0x58, the word the CALL wait polled + * from be2cf130d761ed675ad86f12b21ac1d53d906e1d until + * 3a942d6cf6278526fbc02bf0c4743c5c1b276cdb. 3a942d6 measured its + * layout: the call's nid in bits 23:16 (+1 per call), the index of the + * last task taken in bits 15:0 (receipts/2026-09-30-t6021-call-wait). + * Not known on T8112, so trace_td records nothing there. + * - The seven ANE power-state words at ane_t602x_soc.pmu_pa + ps_off + * (T602x: DT power-domains ane_sys_mpm 0x4000 .. ane_set4 0x4030 of + * the pmgr at 0x28e080000). The guard "PS words 0x3ff before any TM + * read; a TM read while the compute domains are off hangs the SoC" is + * ane_rtclient_pm_pwrstate_ok in 27e996a6de544a803a71d7a5c4ed11d828d4d049, + * and the CALL wait applied it until 3a942d6. + * + * Each record has a ktime_get_ns() stamp: CALL before the exchange, ACK + * when the firmware acked it, TD for each new TD word value (word = the + * value), EVENT for each IO_T2H event of the CALL (word = its state), + * GATE when a PS word reads other than 0x3ff (word = its offset), DONE + * when the wait ends (word = the TD samples taken). The buffer is + * allocated at the first switch-on and kept until unload; each switch-on + * empties it, and records past its end are counted in `dropped`. Read + * it, while no CALL runs, from debugfs ane_t6021/trace_td (0400). All + * trace state is protected by ane_t6021_fw_lock. + */ +#define ANE_TRACE_MAGIC 0x31445441 /* "ATD1" */ +#define ANE_TRACE_RECS (1U << 18) +#define ANE_PMGR_PS_LAST_OFF 0x30 + +enum { + ANE_TR_CALL = 1, + ANE_TR_ACK, + ANE_TR_TD, + ANE_TR_EVENT, + ANE_TR_GATE, + ANE_TR_DONE, +}; + +struct ane_t6021_trace_rec { + u64 t_ns; + u32 word; + u16 kind; + u16 call; +}; + +struct ane_t6021_trace { + u32 magic; + u32 rec_size; + u32 capacity; + u32 n; + u32 dropped; + u32 calls; + u64 reserved; + struct ane_t6021_trace_rec r[]; +}; + +static bool trace_td; +static bool ane_t6021_tracing; /* the running CALL is traced */ +static struct ane_t6021_trace *ane_t6021_trace; +static struct debugfs_blob_wrapper ane_t6021_trace_blob; +static struct dentry *ane_t6021_trace_dir; + +static void ane_t6021_trace_add(u16 kind, u32 word) +{ + struct ane_t6021_trace *t = ane_t6021_trace; + + if (t->n == t->capacity) { + t->dropped++; + return; + } + t->r[t->n++] = (struct ane_t6021_trace_rec){ + .t_ns = ktime_get_ns(), .word = word, .kind = kind, + .call = t->calls, + }; +} + +static int ane_t6021_trace_set(const char *val, const struct kernel_param *kp) +{ + size_t size = struct_size_t(struct ane_t6021_trace, r, ANE_TRACE_RECS); + bool on; + int ret; + + ret = kstrtobool(val, &on); + if (ret) + return ret; + mutex_lock(&ane_t6021_fw_lock); + if (on && !ane_t6021_trace) { + ane_t6021_trace = vzalloc(size); + if (!ane_t6021_trace) { + ret = -ENOMEM; + goto out; + } + ane_t6021_trace->magic = ANE_TRACE_MAGIC; + ane_t6021_trace->rec_size = sizeof(struct ane_t6021_trace_rec); + ane_t6021_trace->capacity = ANE_TRACE_RECS; + ane_t6021_trace_blob.data = ane_t6021_trace; + ane_t6021_trace_blob.size = size; + ane_t6021_trace_dir = debugfs_create_dir("ane_t6021", NULL); + debugfs_create_blob("trace_td", 0400, ane_t6021_trace_dir, + &ane_t6021_trace_blob); + } + if (on && !trace_td) { + ane_t6021_trace->n = 0; + ane_t6021_trace->dropped = 0; + ane_t6021_trace->calls = 0; + } + trace_td = on; +out: + mutex_unlock(&ane_t6021_fw_lock); + return ret; +} + +static const struct kernel_param_ops ane_t6021_trace_ops = { + .set = ane_t6021_trace_set, + .get = param_get_bool, +}; +module_param_cb(trace_td, &ane_t6021_trace_ops, &trace_td, 0644); +MODULE_PARM_DESC(trace_td, + "Record a read-only per-CALL TD-word timeline in debugfs ane_t6021/trace_td (default 0; T602x only)"); + +static void ane_t6021_trace_free(void) +{ + debugfs_remove_recursive(ane_t6021_trace_dir); + vfree(ane_t6021_trace); +} + +/* The CALL cookie (CALL +0x20); the firmware returns it in the call's + * IO_T2H events. + */ +#define ANE_CALL_COOKIE 0xADD0 + +/* IO_T2H event of a PROCEDURE_CALL, 0x28 bytes (measured 2026-09-30, + * receipts/2026-09-30-t6021-call-wait): u32 sequence, u32 0x300, u64 + * cookie, u32 program id, u32 process id, u32 0, u32 state. The firmware + * posts two per call: state 0 about 0.2-0.5 ms after the ack whatever the + * program length (a 3,597-task program takes its last task 252 ms later; + * receipts/2026-10-01-t6021-trace-td), and state 1 when the procedure has + * finished. + */ +#define ANE_T2H_CALL_COOKIE_OFF 0x08 +#define ANE_T2H_CALL_STATE_OFF 0x1c +#define ANE_T2H_CALL_FINISHED 1 + +/* The CPU address of LEN bytes at the firmware IOVA, or NULL. The firmware + * places T2H payloads in memory the host gave it: a SHAREDMALLOC buffer or + * the 'IPC ' surface. + */ +static const void *ane_rtclient_fw_cpu(struct ane_rtclient *ane, u64 iova, + size_t len) +{ + u32 i; + + for (i = 0; i < ane->legacy_allocated; i++) { + const struct ane_legacy_buffer *b = &ane->legacy_buffers[i]; + + if (iova >= b->dma && iova - b->dma + len <= b->size) + return b->cpu + (iova - b->dma); + } + if (iova >= ane->fw->boot_ipc_iova && + iova - ane->fw->boot_ipc_iova + len <= ane->fw->boot_ipc_size) + return ane->fw->boot_ipc + (iova - ane->fw->boot_ipc_iova); + return NULL; +} + +/* Return every firmware-owned slot on a target-to-host ring to the fw + * (legacy_t2h_ack=1, the lab default): the fw publishes by writing the + * DMA address with bit0 clear; the host returns the slot by setting + * bit0 and ringing the channel's doorbell bit, as the allocation ring + * does. The sequencer drained channels 4/6 after every step; the + * ioctls drain them after every completed exchange. Returns true when + * an IO_T2H slot held the finish event of a CALL. + */ +static bool ane_rtclient_drain_t2h(struct ane_rtclient *ane, + unsigned int channel) +{ + const struct ane_t6021_chman_static *c = &ane_t6021_chman_layout[channel]; + unsigned int n = 0, slot_i = ane->legacy_cmd_cursor[channel]; + void __iomem *ipi = ane->engine + ANE_IPI_OFF; + bool finished = false; + + if (!ane->fw || !ane->fw->boot_ipc) + return false; + while (n < c->size) { + u64 *slot = ane->fw->boot_ipc + c->off + (size_t)slot_i * 64; + u64 hdr = READ_ONCE(slot[0]), len = READ_ONCE(slot[1]); + + if (hdr & 1) + break; + dma_rmb(); + dev_dbg(ane->dev, "T2H ch=%s slot=%u hdr=%016llx len=%#llx\n", + c->name, slot_i, hdr, len); + if (channel == 6 && len >= ANE_T2H_CALL_STATE_OFF + 4) { + const u8 *ev = ane_rtclient_fw_cpu(ane, hdr, + ANE_T2H_CALL_STATE_OFF + 4); + + if (ev && + get_unaligned_le64(ev + ANE_T2H_CALL_COOKIE_OFF) == + ANE_CALL_COOKIE) { + u32 state = get_unaligned_le32(ev + + ANE_T2H_CALL_STATE_OFF); + + if (ane_t6021_tracing) + ane_t6021_trace_add(ANE_TR_EVENT, state); + if (state == ANE_T2H_CALL_FINISHED) + finished = true; + } + } + n++; + WRITE_ONCE(slot[0], hdr | 1); + dma_wmb(); + writel(BIT(c->bit), ipi); + slot_i = (slot_i + 1) % c->size; + ane->legacy_cmd_cursor[channel] = slot_i; + } + return finished; +} + +/* The completion wait with trace_td on: the same finish-event test with a + * 20-40 us poll, and one TD-word sample per poll under the PS-word guard + * (see trace_td). + */ +static int ane_rtclient_call_wait_traced(struct ane_rtclient *ane, + unsigned long deadline) +{ + void __iomem *td = ane->engine + ane->soc->trace_td_off; + void __iomem *ps = ioremap_np(ane->soc->pmu_pa + ane->soc->ps_off, + ANE_PMGR_PS_LAST_OFF + 4); + unsigned int i, gate = ANE_PMGR_PS_LAST_OFF + 8; + u32 last = U32_MAX, samples = 0; + int ret = -ETIMEDOUT; + + do { + if (ane_rtclient_drain_t2h(ane, 6)) { + ret = 0; + break; + } + if (ps) { + for (i = 0; i <= ANE_PMGR_PS_LAST_OFF; i += 8) + if ((readl(ps + i) & 0x3ff) != 0x3ff) + break; + if (i > ANE_PMGR_PS_LAST_OFF) { + u32 w = readl(td); + + samples++; + if (w != last) + ane_t6021_trace_add(ANE_TR_TD, w); + last = w; + } else if (i != gate) { + ane_t6021_trace_add(ANE_TR_GATE, i); + } + gate = i; + } + usleep_range(20, 40); + } while (time_before(jiffies, deadline)); + if (ret && ane_rtclient_drain_t2h(ane, 6)) + ret = 0; + ane_t6021_trace_add(ANE_TR_DONE, samples); + if (ps) + iounmap(ps); + return ret; +} + +/* Completion wait for one PROCEDURE_CALL: the finish event on IO_T2H + * (channel 6). The ack, the eight TQ status words and the last-committed + * TD word (TM +0x20458: the call's nid in bits 23:16 and the index of the + * last task taken in bits 15:0) all report done once the last task has + * been dispatched, not when it has run: measured 2026-09-30, Qwen program + * 20 (20 tasks) showed its last task index 0.22 ms after the ack, posted + * its finish event 3.5 ms after the ack, and a caller that returned at + * the first signal read an all-zero output. Returns 0 when the event + * arrived, -ETIMEDOUT else. + */ +static int ane_rtclient_call_wait(struct ane_rtclient *ane, + unsigned int timeout_ms) +{ + unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms); + + if (ane_t6021_tracing) + return ane_rtclient_call_wait_traced(ane, deadline); + + do { + if (ane_rtclient_drain_t2h(ane, 6)) + return 0; + usleep_range(50, 100); + } while (time_before(jiffies, deadline)); + return ane_rtclient_drain_t2h(ane, 6) ? 0 : -ETIMEDOUT; +} + +static int ane_rtclient_command(struct ane_rtclient *ane, + struct ane_legacy_buffer *command, + size_t length, u16 opcode, + unsigned int channel, + unsigned int timeout_ms) +{ + int ret; + + ane_t6021_tracing = opcode == CSNE_CMD_PROCEDURE_CALL && trace_td && + ane->soc->trace_td_off; + if (ane_t6021_tracing) { + ane_t6021_trace->calls++; + ane_t6021_trace_add(ANE_TR_CALL, 0); + } + ret = ane_rtclient_legacy_exchange(ane, command, length, opcode, + channel, timeout_ms); + if (ret) { + ane_t6021_tracing = false; + dev_info(ane->dev, "EXCH op=%#x failed %d (fw allocs %u, %zu bytes)\n", + opcode, ret, ane->legacy_allocated, ane->legacy_bytes); + atomic_set(&ane_t6021_quarantined, 1); + return ret; + } + if (opcode == CSNE_CMD_PROCEDURE_CALL) { + if (ane_t6021_tracing) + ane_t6021_trace_add(ANE_TR_ACK, 0); + ret = ane_rtclient_call_wait(ane, timeout_ms); + ane_t6021_tracing = false; + if (!ret && call_settle_us) + usleep_range(call_settle_us, call_settle_us + 100); + if (ret) { + dev_info(ane->dev, "call completion wait failed %d\n", + ret); + atomic_set(&ane_t6021_quarantined, 1); + return ret; + } + } + /* The fw talks back on the target-to-host rings (fwlog, perf); + * hand those slots back so the rings never fill (the sequencer + * did this per step; same ack, channels 4 and 6). + */ + ane_rtclient_drain_t2h(ane, 4); + ane_rtclient_drain_t2h(ane, 6); + /* BOs are dma_alloc_coherent memory mapped write-combined for the + * CPU, so no cache maintenance is needed on either side. + */ + return 0; +} + +/* ---- LOAD / CREATE / CALL wrappers ---- */ + +/* The firmware never frees a program or a process, and its program table + * holds 256 entries: measured 2026-09-29, the 257th LOAD_PROGRAM on one + * boot answered with a protocol error and quarantined the device. Two + * loads of byte-identical sections therefore share one firmware program + * and one process. The key is SHA-256 over every section's id, size and + * bytes, so a client can only reach a program whose bytes it also + * supplied. Protected by ane_t6021_fw_lock. + */ +#define ANE_T6021_MAX_PROGRAMS 250 + +struct ane_t6021_prog { + u8 digest[SHA256_DIGEST_SIZE]; + u32 prog_id; + u32 proc_id; /* U32_MAX until a process exists */ +}; + +static struct ane_t6021_prog ane_t6021_progs[ANE_T6021_MAX_PROGRAMS]; +static unsigned int ane_t6021_nprogs; + +static struct ane_t6021_prog *ane_t6021_prog_find(const u8 *digest) +{ + unsigned int i; + + for (i = 0; i < ane_t6021_nprogs; i++) + if (!memcmp(ane_t6021_progs[i].digest, digest, + SHA256_DIGEST_SIZE)) + return &ane_t6021_progs[i]; + return NULL; +} + +static struct ane_t6021_prog *ane_t6021_prog_by_id(u32 prog_id) +{ + unsigned int i; + + for (i = 0; i < ane_t6021_nprogs; i++) + if (ane_t6021_progs[i].prog_id == prog_id) + return &ane_t6021_progs[i]; + return NULL; +} + +/* Build a LOAD_PROGRAM (0x200) message in a kernel-owned buffer. The + * section bytes live in the BOs the user supplied (section_ptr), the + * generic binds (bufferId, bo_handle, size) patch the generic section + * at iova +0x20 / +0x28 once the BO is copied. Returns 0 with + * *prog_id set on success. + */ +static int ane_rtclient_load_program(struct ane_rtclient *ane, + struct drm_file *file, + const struct drm_ane_prog_load *user, + __u32 *prog_id) +{ + struct drm_ane_section *sections; + struct drm_ane_generic_bind *binds; + struct ane_legacy_buffer *command; + struct ane_t6021_fd *fd = file->driver_priv; + struct ane_t6021_prog *cached; + u8 digest[SHA256_DIGEST_SIZE]; + size_t binds_size; + int ret, i, j; + + if (user->section_count < 1 || + user->section_count > ANE_T6021_LOAD_SEC_COUNT || + user->generic_count > ANE_M2_MAX_BINDS || !fd) + return -EINVAL; + + sections = kmalloc_array(user->section_count, sizeof(*sections), + GFP_KERNEL); + binds = kmalloc_array(user->generic_count, sizeof(*binds), + GFP_KERNEL); + if (!sections || !binds) { + ret = -ENOMEM; + goto out; + } + if (copy_from_user(sections, + u64_to_user_ptr(user->sections_ptr), + user->section_count * sizeof(*sections))) { + ret = -EFAULT; + goto out; + } + binds_size = user->generic_count * sizeof(*binds); + if (binds_size && copy_from_user(binds, + u64_to_user_ptr(user->generic_ptr), + binds_size)) { + ret = -EFAULT; + goto out; + } + + /* Lab contract: the record slot IS the section identity (slot = + * id - 1; tdprop id 7 lands at slot 6 and slot 5 stays empty, + * exactly the proven add-path image). Reject out-of-range and + * duplicate ids before building the wire image. + */ + for (i = 0; i < user->section_count; i++) { + if (sections[i].id < 1 || + sections[i].id > ANE_T6021_LOAD_SEC_COUNT) { + ret = -EINVAL; + goto out; + } + for (j = i + 1; j < user->section_count; j++) { + if (sections[i].id == sections[j].id) { + ret = -EINVAL; + goto out; + } + } + } + + /* Identical sections share one firmware program (see the table). + * A constant section can reach 20 MiB (the H14 (K,N)=(2048,5120) + * matvec), so feed the hash in ANE_T6021_BO_HASH_CHUNK-sized slices + * under bo_lock: a single sha256_update of a 20 MiB buffer would + * rely on a 20 MiB stack argument list and has no upper bound on + * the chunk that the BO could supply. + */ + { + struct sha256_ctx sha; + void *scratch; + + sha256_init(&sha); + scratch = kvmalloc(ANE_T6021_BO_HASH_CHUNK, GFP_KERNEL); + if (!scratch) { + ret = -ENOMEM; + goto out; + } + for (i = 0; i < user->section_count; i++) { + struct ane_t6021_bo *bo = NULL, *b; + u64 hdr[2] = { sections[i].id, sections[i].size }; + size_t left; + + mutex_lock(&ane_t6021_bo_lock); + list_for_each_entry(b, &fd->bos, node) { + if (b->handle == sections[i].bo_handle) { + bo = b; + break; + } + } + if (!bo || sections[i].size > bo->size || + sections[i].offset > bo->size - sections[i].size) { + mutex_unlock(&ane_t6021_bo_lock); + kvfree(scratch); + ret = -EINVAL; + goto out; + } + sha256_update(&sha, (const u8 *)hdr, sizeof(hdr)); + left = sections[i].size; + while (left) { + size_t n = min(left, (size_t)ANE_T6021_BO_HASH_CHUNK); + + memcpy(scratch, (const u8 *)bo->cpu + + sections[i].offset + + (sections[i].size - left), n); + sha256_update(&sha, scratch, n); + left -= n; + } + mutex_unlock(&ane_t6021_bo_lock); + } + kvfree(scratch); + sha256_final(&sha, digest); + } + cached = ane_t6021_prog_find(digest); + if (cached) { + *prog_id = cached->prog_id; + ret = 0; + goto out; + } + if (ane_t6021_nprogs == ANE_T6021_MAX_PROGRAMS) { + ret = -ENOSPC; + goto out; + } + + command = ane->cmd_buf; + if (!command) { + ret = -ENODEV; + goto out; + } + memset(command->cpu, 0, SZ_16K); + /* One 0x30-byte section record per supplied section, placed at + * its IDENTITY slot: slot = id - 1 (tdprop id 7 lands at slot + * 6, slot 5 stays empty — exactly the proven add-path image: + * six populated records, ids 1,2,3,4,5 then 7). Unsupplied + * slots stay zero-filled (flags = 0). flags bit0 = 1, id at + * +0x04, iova at +0x18, size at +0x20. + */ + for (i = 0; i < user->section_count; i++) { + struct ane_t6021_bo *bo = NULL, *b; + u64 slot_base, iova; + u8 *cmd = command->cpu; + + mutex_lock(&ane_t6021_bo_lock); + list_for_each_entry(b, &fd->bos, node) { + if (b->handle == sections[i].bo_handle) { + bo = b; + break; + } + } + if (bo && sections[i].size <= bo->size && + sections[i].offset <= bo->size - sections[i].size) { + /* The IOVA below is about to be published to the + * firmware, so mark the BO before the exchange: + * the fw may read the section any time after the + * doorbell rings, including during a timeout. + * Under bo_lock the handle reference cannot go + * away, so no kref is needed here. + */ + bo->fw_ref = true; + bo->fw_program = true; + iova = bo->dma + sections[i].offset; + } else { + bo = NULL; + } + mutex_unlock(&ane_t6021_bo_lock); + if (!bo) { + ret = -EINVAL; + goto out; + } + slot_base = 0x8 + (u64)(sections[i].id - 1) * 0x30; + *(u32 *)(cmd + slot_base + 0x00) = cpu_to_le32(1); + *(u32 *)(cmd + slot_base + 0x04) = + cpu_to_le32(sections[i].id); + *(u64 *)(cmd + slot_base + 0x18) = cpu_to_le64(iova); + *(u64 *)(cmd + slot_base + 0x20) = + cpu_to_le64(sections[i].size); + } + /* Generic binds are accepted for ABI compatibility and unused: + * the LOAD record already carries iova + size, and the generic + * section image is prepared by userspace (the proven add path + * ships it inside the section bytes, generic.bin). + */ + + /* ProgramId placeholder; the firmware writes the assigned id + * back at +0x1b8. + */ + { + u8 *cmd = command->cpu; + + *(u32 *)(cmd + 0x1b8) = cpu_to_le32(U32_MAX); + } + + /* The proven add-path LOAD command is 0x1C0 bytes: the nine + * 0x30 records end at 0x1b8, ProgramId at +0x1b8, 8 bytes of + * zero tail. sizeof(struct ane_csne_cmd_load_program) is only + * 0x1b8, so the length is pinned here (h14_seq_first_add.py + * load_step). + */ + BUILD_BUG_ON(sizeof(struct ane_csne_cmd_load_program) != 0x1b8); + ret = ane_rtclient_command(ane, command, + 0x1c0, + CSNE_CMD_LOAD_PROGRAM, + 1, 5000); + if (!ret) { + u8 *cmd = command->cpu; + + *prog_id = le32_to_cpu(*(u32 *)(cmd + 0x1b8)); + if (*prog_id == U32_MAX) { + ret = -EPROTO; + goto out; + } + memcpy(ane_t6021_progs[ane_t6021_nprogs].digest, digest, + SHA256_DIGEST_SIZE); + ane_t6021_progs[ane_t6021_nprogs].prog_id = *prog_id; + ane_t6021_progs[ane_t6021_nprogs].proc_id = U32_MAX; + ane_t6021_nprogs++; + } + +out: + kfree(sections); + kfree(binds); + return ret; +} + +static int ane_rtclient_create_process(struct ane_rtclient *ane, + __u32 prog_id, __u32 *proc_id) +{ + struct ane_legacy_buffer *command; + struct ane_t6021_prog *prog = ane_t6021_prog_by_id(prog_id); + int ret; + + if (!prog) + return -ENOENT; + if (prog->proc_id != U32_MAX) { + *proc_id = prog->proc_id; + return 0; + } + command = ane->cmd_buf; + if (!command) + return -ENODEV; + memset(command->cpu, 0, SZ_16K); + { + u8 *cmd = command->cpu; + + *(u32 *)(cmd + 0x08) = cpu_to_le32(prog_id); + *(u32 *)(cmd + 0x0c) = cpu_to_le32(U32_MAX); + } + ret = ane_rtclient_command(ane, command, 0x10, + CSNE_CMD_CREATE_PROCESS, + 1, 3000); + if (!ret) { + u8 *cmd = command->cpu; + + *proc_id = le32_to_cpu(*(u32 *)(cmd + 0x0c)); + if (*proc_id == U32_MAX) + ret = -EPROTO; + else + prog->proc_id = *proc_id; + } + return ret; +} + +static int ane_rtclient_procedure_call(struct ane_rtclient *ane, + struct drm_file *file, + const struct drm_ane_exec *user) +{ + struct drm_ane_exec_io *ios; + struct ane_legacy_buffer *command; + struct ane_t6021_fd *fd = file->driver_priv; + size_t ios_size; + size_t cmd_size; + int ret, i; + + if (user->count < 1 || user->count > ANE_M2_MAX_BINDS || + user->priority < 2 || user->priority > 7 || !fd) + return -EINVAL; + + ios_size = (size_t)user->count * sizeof(*ios); + ios = kmalloc(ios_size, GFP_KERNEL); + if (!ios) + return -ENOMEM; + if (copy_from_user(ios, u64_to_user_ptr(user->io_ptr), ios_size)) { + kfree(ios); + return -EFAULT; + } + + cmd_size = sizeof(struct ane_csne_cmd_procedure_call) + + (size_t)user->count * sizeof(struct ane_csne_io_elem); + if (cmd_size > SZ_16K) { + kfree(ios); + return -E2BIG; + } + + mutex_lock(&ane_t6021_fw_lock); + if (atomic_read(&ane_t6021_quarantined)) { + mutex_unlock(&ane_t6021_fw_lock); + kfree(ios); + return -ETIMEDOUT; + } + command = ane->cmd_buf; + if (!command) { + mutex_unlock(&ane_t6021_fw_lock); + kfree(ios); + return -ENODEV; + } + memset(command->cpu, 0, SZ_16K); + { + u8 *cmd = command->cpu; + + *(u32 *)(cmd + 0x08) = cpu_to_le32(user->prog_id); + *(u32 *)(cmd + 0x0c) = cpu_to_le32(user->proc_id); + *(u64 *)(cmd + 0x10) = cpu_to_le64(0); + *(u32 *)(cmd + 0x18) = cpu_to_le32(user->priority); + *(u64 *)(cmd + 0x20) = cpu_to_le64(ANE_CALL_COOKIE); + *(u32 *)(cmd + 0x28) = cpu_to_le32(user->count); + for (i = 0; i < user->count; i++) { + struct ane_t6021_bo *bo = NULL, *b; + u64 slot_base = 0x60 + (u64)i * 0x30; + u64 iova; + + mutex_lock(&ane_t6021_bo_lock); + list_for_each_entry(b, &fd->bos, node) { + if (b->handle == ios[i].bo_handle) { + bo = b; + break; + } + } + if (!bo || ios[i].size > bo->size) { + mutex_unlock(&ane_t6021_bo_lock); + ret = -EINVAL; + goto unlock; + } + /* The IOVA below is about to be published to the + * firmware, so mark the BO before the exchange. + * Under bo_lock the handle reference cannot go + * away, so no kref is needed here. + */ + bo->fw_ref = true; + iova = bo->dma; + mutex_unlock(&ane_t6021_bo_lock); + *(u32 *)(cmd + slot_base + 0x00) = cpu_to_le32(1); + *(u32 *)(cmd + slot_base + 0x04) = + cpu_to_le32(ios[i].buffer_id); + *(u32 *)(cmd + slot_base + 0x08) = + cpu_to_le32(ios[i].type); + *(u64 *)(cmd + slot_base + 0x18) = + cpu_to_le64(iova); + *(u64 *)(cmd + slot_base + 0x20) = + cpu_to_le64(ios[i].size); + } + /* Drain the CPU write buffers so the input BOs the user + * filled through its uncached mapping are in DRAM before + * the fw starts reading them. + */ + wmb(); + ret = ane_rtclient_command(ane, command, + cmd_size, + CSNE_CMD_PROCEDURE_CALL, + 1, + user->timeout_ms ? + user->timeout_ms : 5000); + } +unlock: + mutex_unlock(&ane_t6021_fw_lock); + kfree(ios); + return ret; +} + +/* ---- DRM accel glue (minimal BO + ioctl table) ---- */ + +struct ane_t6021_drm { + struct drm_device drm; + struct device *dev; + struct ane_rtclient *ane; +}; + +static struct ane_t6021_drm *to_ane_t6021_drm(struct drm_device *drm) +{ + return container_of(drm, struct ane_t6021_drm, drm); +} + +static int ane_t6021_bo_init_ioctl(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct drm_ane_bo_init *args = data; + struct ane_t6021_fd *fd = file->driver_priv; + struct ane_t6021_bo *bo; + struct ane_rtclient *ane; + + if (args->size == 0 || args->size > ANE_T6021_BO_MAX || !fd) + return -EINVAL; + /* A parked io BO is already mapped and counted; its old contents + * belong to another process, so it is zeroed like a new one. + */ + bo = ane_t6021_bo_pool_take(args->size); + if (bo) { + memset(bo->cpu, 0, PAGE_ALIGN(args->size)); + goto publish; + } + /* Global coherent-memory accounting. Each BO is 16 KiB-aligned; + * a BO whose IOVA reaches the firmware is never freed (held or + * pooled), so this bound caps the memory that outlives its + * users. + */ + if (atomic64_add_return(PAGE_ALIGN(args->size), &ane_t6021_bo_total_bytes) > + (s64)bo_total_max_mb << 20) { + atomic64_sub(PAGE_ALIGN(args->size), &ane_t6021_bo_total_bytes); + return -ENOSPC; + } + ane = to_ane_t6021_drm(drm)->ane; + bo = kzalloc(sizeof(*bo), GFP_KERNEL); + if (!bo) { + atomic64_sub(PAGE_ALIGN(args->size), &ane_t6021_bo_total_bytes); + return -ENOMEM; + } + bo->cpu = dma_alloc_coherent(drm->dev, args->size, &bo->dma, + GFP_KERNEL); + if (!bo->cpu) { + atomic64_sub(PAGE_ALIGN(args->size), &ane_t6021_bo_total_bytes); + kfree(bo); + return -ENOMEM; + } + /* Every fw-visible DMA surface must be 16 KiB aligned and clear + * of the firmware entry alias (receipt 2026-09-20-t6021-entry-alias: + * the same invariant ane_rtclient_legacy_alloc enforces). + */ + if (!IS_ALIGNED(bo->dma, SZ_16K) || + (ane->fw && !ane_t6021_fw_alias_iova_ok(ane->fw, bo->dma, + args->size))) { + dma_free_coherent(drm->dev, args->size, bo->cpu, bo->dma); + atomic64_sub(PAGE_ALIGN(args->size), &ane_t6021_bo_total_bytes); + kfree(bo); + return -ERANGE; + } +publish: + bo->size = args->size; + bo->owner = fd; + bo->dev = drm->dev; + kref_init(&bo->refcount); /* the handle holds this reference */ + mutex_lock(&ane_t6021_bo_lock); + bo->handle = ane_t6021_next_handle++; + if (bo->handle == 0) + bo->handle = ane_t6021_next_handle++; + list_add_tail(&bo->node, &fd->bos); + mutex_unlock(&ane_t6021_bo_lock); + args->handle = bo->handle; + /* mmap offset = the handle; libane mmaps the fd at exactly this + * offset and ane_t6021_mmap resolves the BO from vm_pgoff. + */ + args->offset = (u64)bo->handle << PAGE_SHIFT; + return 0; +} + +/* Drop one handle owned by this fd. The kref may keep the memory + * alive past this call — a user mapping still holds a reference — so + * the final ane_t6021_bo_release makes the free-or-hold decision. + */ +static void ane_t6021_bo_drop(struct ane_t6021_bo *bo) +{ + list_del(&bo->node); + kref_put(&bo->refcount, ane_t6021_bo_release); +} + +static int ane_t6021_bo_free_ioctl(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct drm_ane_bo_free *args = data; + struct ane_t6021_fd *fd = file->driver_priv; + struct ane_t6021_bo *bo = NULL, *b; + + if (!fd) + return -ENODEV; + mutex_lock(&ane_t6021_bo_lock); + list_for_each_entry(b, &fd->bos, node) { + if (b->handle == args->handle) { + bo = b; + break; + } + } + if (bo) + ane_t6021_bo_drop(bo); + mutex_unlock(&ane_t6021_bo_lock); + return bo ? 0 : -ENOENT; +} + +/* Resolve the BO a user mmap names (BO_INIT returned offset = handle + * << PAGE_SHIFT) and map the coherent buffer cacheably — the device + * half coheres through the DART (IOMMU_CACHE), so reads after EXEC + * observe the firmware's writes without extra sync. The mapping takes + * one BO reference: it survives BO_FREE until the vma is gone. + */ +static int ane_t6021_mmap(struct file *filp, struct vm_area_struct *vma) +{ + struct drm_file *file = filp->private_data; + struct ane_t6021_fd *fd = file->driver_priv; + struct drm_device *drm = file->minor->dev; + struct ane_t6021_bo *bo = NULL, *b; + size_t size = vma->vm_end - vma->vm_start; + u32 handle; + int ret; + + if (!fd) + return -ENODEV; + /* vm_pgoff is the BO_INIT offset in pages: handle << PAGE_SHIFT + * >> PAGE_SHIFT == handle. + */ + handle = (u32)vma->vm_pgoff; + if (!handle) + return -EINVAL; + mutex_lock(&ane_t6021_bo_lock); + list_for_each_entry(b, &fd->bos, node) { + if (b->handle == handle && b->owner == fd) { + kref_get(&b->refcount); + bo = b; + break; + } + } + if (!bo || size > PAGE_ALIGN(bo->size)) { + mutex_unlock(&ane_t6021_bo_lock); + if (bo) + kref_put(&bo->refcount, ane_t6021_bo_release); + return -ENOENT; + } + vma->vm_pgoff = 0; + ret = dma_mmap_coherent(drm->dev, vma, bo->cpu, bo->dma, size); + mutex_unlock(&ane_t6021_bo_lock); + if (ret) { + kref_put(&bo->refcount, ane_t6021_bo_release); + return ret; + } + /* The vma owns the lookup reference: close (and open on fork) + * go through ane_t6021_vm_ops. + */ + vma->vm_private_data = bo; + vma->vm_ops = &ane_t6021_vm_ops; + return 0; +} + +static int ane_t6021_open(struct drm_device *drm, struct drm_file *file) +{ + struct ane_t6021_fd *fd; + + fd = kzalloc(sizeof(*fd), GFP_KERNEL); + if (!fd) + return -ENOMEM; + INIT_LIST_HEAD(&fd->bos); + file->driver_priv = fd; + return 0; +} + +static void ane_t6021_postclose(struct drm_device *drm, struct drm_file *file) +{ + struct ane_t6021_fd *fd = file->driver_priv; + struct ane_t6021_bo *bo, *tmp; + + if (!fd) + return; + mutex_lock(&ane_t6021_bo_lock); + list_for_each_entry_safe(bo, tmp, &fd->bos, node) + ane_t6021_bo_drop(bo); + mutex_unlock(&ane_t6021_bo_lock); + kfree(fd); + file->driver_priv = NULL; +} + +static int ane_t6021_submit_ioctl(struct drm_device *drm, void *data, + struct drm_file *file) +{ + return -ENOTTY; /* ABI 1 SUBMIT is rejected on T6021/M2 */ +} + +static int ane_t6021_prog_load_ioctl(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct ane_t6021_drm *adrm = to_ane_t6021_drm(drm); + struct drm_ane_prog_load *args = data; + int ret; + + mutex_lock(&ane_t6021_fw_lock); + if (atomic_read(&ane_t6021_quarantined)) { + mutex_unlock(&ane_t6021_fw_lock); + return -ETIMEDOUT; + } + ret = ane_rtclient_load_program(adrm->ane, file, args, + &args->prog_id_out); + mutex_unlock(&ane_t6021_fw_lock); + return ret; +} + +/* Firmware perf mode: CSNE_CMD_CH_PROPERTY_WRITE (0x1f), channel 0, + * property 0x10aa, value 1. macOS sends it once during power-on (H13 + * kext evidence in the omarchy-ane perf-mode receipt); the selene 13.5 + * fw routes it to CAneEngineExeLoop::setPerfMode. It is a runtime + * switch (write 1 to /sys/module/ane_t6021/parameters/fw_perf_mode) so + * its effect on call time can be measured on one boot. Only 1 is + * accepted: no other value is known to be safe. + */ +static struct ane_rtclient *ane_t6021_perf_ane; +static bool fw_perf_mode; + +static int ane_t6021_perf_mode_set(const char *val, + const struct kernel_param *kp) +{ + struct ane_rtclient *ane = READ_ONCE(ane_t6021_perf_ane); + struct ane_legacy_buffer *command; + bool on; + int ret; + + ret = kstrtobool(val, &on); + if (ret) + return ret; + if (!on || fw_perf_mode) + return on ? 0 : -EINVAL; + if (!ane) + return -ENODEV; + mutex_lock(&ane_t6021_fw_lock); + if (atomic_read(&ane_t6021_quarantined)) { + mutex_unlock(&ane_t6021_fw_lock); + return -ETIMEDOUT; + } + command = ane->cmd_buf; + if (!command) { + mutex_unlock(&ane_t6021_fw_lock); + return -ENODEV; + } + memset(command->cpu, 0, SZ_16K); + *(u32 *)((u8 *)command->cpu + 0x08) = cpu_to_le32(0); + *(u32 *)((u8 *)command->cpu + 0x0c) = cpu_to_le32(0x10aa); + *(u32 *)((u8 *)command->cpu + 0x10) = cpu_to_le32(1); + ret = ane_rtclient_command(ane, command, 0x14, 0x001f, 1, 3000); + if (!ret) { + fw_perf_mode = true; + dev_info(ane->dev, "fw perf mode set (property 0x10aa = 1)\n"); + } + mutex_unlock(&ane_t6021_fw_lock); + return ret; +} + +static const struct kernel_param_ops ane_t6021_perf_mode_ops = { + .set = ane_t6021_perf_mode_set, + .get = param_get_bool, +}; +module_param_cb(fw_perf_mode, &ane_t6021_perf_mode_ops, &fw_perf_mode, 0644); +MODULE_PARM_DESC(fw_perf_mode, + "Write 1 once to send CH_PROPERTY_WRITE 0x10aa = 1 (fw perf mode); reads back whether it was sent"); + +static int ane_t6021_proc_create_ioctl(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct ane_t6021_drm *adrm = to_ane_t6021_drm(drm); + struct drm_ane_proc_create *args = data; + int ret; + + mutex_lock(&ane_t6021_fw_lock); + if (atomic_read(&ane_t6021_quarantined)) { + mutex_unlock(&ane_t6021_fw_lock); + return -ETIMEDOUT; + } + args->proc_id_out = 0; + ret = ane_rtclient_create_process(adrm->ane, args->prog_id, + &args->proc_id_out); + mutex_unlock(&ane_t6021_fw_lock); + return ret; +} + +static int ane_t6021_exec_ioctl(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct ane_t6021_drm *adrm = to_ane_t6021_drm(drm); + struct drm_ane_exec *args = data; + + return ane_rtclient_procedure_call(adrm->ane, file, args); +} + +static const struct drm_ioctl_desc ane_t6021_ioctls[] = { + DRM_IOCTL_DEF_DRV(ANE_BO_INIT, ane_t6021_bo_init_ioctl, 0), + DRM_IOCTL_DEF_DRV(ANE_BO_FREE, ane_t6021_bo_free_ioctl, 0), + DRM_IOCTL_DEF_DRV(ANE_SUBMIT, ane_t6021_submit_ioctl, 0), + DRM_IOCTL_DEF_DRV(ANE_PROG_LOAD, ane_t6021_prog_load_ioctl, 0), + DRM_IOCTL_DEF_DRV(ANE_PROC_CREATE, ane_t6021_proc_create_ioctl, 0), + DRM_IOCTL_DEF_DRV(ANE_EXEC, ane_t6021_exec_ioctl, 0), +}; + +/* Driver fops: the accel-core entry points plus our BO mmap (the core + * default maps only GEM objects; this driver keeps its own BO table, + * so .mmap resolves the handle from BO_INIT's returned offset). + */ +static const struct file_operations ane_t6021_fops = { + .owner = THIS_MODULE, + .fop_flags = FOP_UNSIGNED_OFFSET, + .open = accel_open, + .release = drm_release, + .unlocked_ioctl = drm_ioctl, + .compat_ioctl = drm_compat_ioctl, + .poll = drm_poll, + .read = drm_read, + .llseek = noop_llseek, + .mmap = ane_t6021_mmap, +}; + +/* Version reported through DRM_IOCTL_VERSION: ABI 2 (T6021). + * DRIVER_COMPUTE_ACCEL puts the node at /dev/accel/accelN. + */ +static const struct drm_driver ane_t6021_drm_driver = { + .driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL, + .open = ane_t6021_open, + .postclose = ane_t6021_postclose, + .ioctls = ane_t6021_ioctls, + .num_ioctls = ARRAY_SIZE(ane_t6021_ioctls), + .fops = &ane_t6021_fops, + .major = ANE_ABI_M2_MAJOR, + .minor = 0, + .name = "ane", + .desc = "Apple Neural Engine (T6021/M2)", +}; + +/* ---- RTKit ops (kept on the boot path; the legacy ChMan transport + * is used by the ioctls once probe confirms legacy_only / chman_ok). + */ + +static void ane_rtclient_recv(void *cookie, u8 ep, u64 message) +{ + struct ane_rtclient *ane = cookie; + + dev_info(ane->dev, + "rtkit app msg: ep=%#x msg=%016llx\n", ep, message); +} + +static void ane_rtclient_crashed(void *cookie, const void *crashlog, + size_t size) +{ + struct ane_rtclient *ane = cookie; + + dev_err(ane->dev, "rtkit: coprocessor crashed (crashlog %zu bytes)\n", + size); + print_hex_dump(KERN_ERR, "ANE crashlog: ", DUMP_PREFIX_OFFSET, 16, 1, + crashlog, min_t(size_t, size, 256), false); +} + +static int ane_rtclient_shmem_setup(void *cookie, + struct apple_rtkit_shmem *bfr) +{ + struct ane_rtclient *ane = cookie; + + if (bfr->iova) { + dev_warn(ane->dev, + "rtkit: fw-provided shmem iova=%pad size=%#zx — refused\n", + &bfr->iova, bfr->size); + return -EINVAL; + } + bfr->buffer = dma_alloc_coherent(ane->dev, bfr->size, &bfr->iova, + GFP_KERNEL); + if (!bfr->buffer) + return -ENOMEM; + if (ane->fw && !ane_t6021_fw_alias_iova_ok(ane->fw, bfr->iova, + bfr->size)) { + dev_err(ane->dev, + "rtkit: shmem grant %pad+%#zx overlaps the fw alias — refusing\n", + &bfr->iova, bfr->size); + dma_free_coherent(ane->dev, bfr->size, bfr->buffer, bfr->iova); + bfr->buffer = NULL; + return -EBUSY; + } + return 0; +} + +static void ane_rtclient_shmem_destroy(void *cookie, + struct apple_rtkit_shmem *bfr) +{ + struct ane_rtclient *ane = cookie; + + if (!bfr->buffer) + return; + if (ane->held) { + dev_warn(ane->dev, + "rtkit: shmem %pad HELD (CPU started) — not freed\n", + &bfr->iova); + return; + } + dma_free_coherent(ane->dev, bfr->size, bfr->buffer, bfr->iova); +} + +static const struct apple_rtkit_ops ane_rtclient_rtkit_ops = { + .crashed = ane_rtclient_crashed, + .recv_message = ane_rtclient_recv, + .shmem_setup = ane_rtclient_shmem_setup, + .shmem_destroy = ane_rtclient_shmem_destroy, +}; + +/* ---- poll worker: RX fallback while the recv line is unproven ---- + * (lab poll_rx=1). Runs every 10 ms until the handshake completes, + * then every second while poll_rx stays on. Armed BEFORE the host ack + * so the fw's HELLO is never missed. + */ +static void ane_rtclient_post_boot(struct work_struct *w) +{ + struct ane_rtclient *ane = + container_of(to_delayed_work(w), struct ane_rtclient, + poll_work); + + apple_rtkit_poll(ane->rtk); + + if (!ane->boot_done) { + schedule_delayed_work(&ane->poll_work, msecs_to_jiffies(10)); + return; + } + + if (poll_rx) + schedule_delayed_work(&ane->poll_work, HZ); +} + +/* STARTEP every fw-announced app endpoint (>= 0x20; the fw mgmt + * dispatcher starts an endpoint on flag bit 1). Same call every Asahi + * RTKit client makes; the lab ran it with start_app_eps=1. + */ +static void ane_rtclient_start_app_eps(struct ane_rtclient *ane) +{ + int ep; + + for (ep = 0x20; ep < 0x100; ep++) { + int ret; + + if (!apple_rtkit_has_endpoint(ane->rtk, ep)) + continue; + ret = apple_rtkit_start_ep(ane->rtk, ep); + dev_info(ane->dev, "rtkit: STARTEP app ep %#x -> %pe\n", + ep, ERR_PTR(ret)); + } +} + +/* Multi-domain genpd attach, ownership-correct and idempotent. */ +struct ane_rtclient_pd { + struct list_head list; + struct device *dev; + struct device **pd_dev; + struct device_link **pd_link; + int count; +}; + +static LIST_HEAD(ane_rtclient_pd_list); +static DEFINE_MUTEX(ane_rtclient_pd_lock); +static bool ane_rtclient_pinned; + +static void ane_rtclient_pd_free(struct ane_rtclient_pd *pd) +{ + list_del(&pd->list); + kfree(pd->pd_dev); + kfree(pd->pd_link); + kfree(pd); +} + +static int ane_rtclient_attach_genpd(struct ane_rtclient *ane) +{ + struct device *dev = ane->dev; + struct ane_rtclient_pd *pd = NULL; + int count, i, err = 0; + + count = of_count_phandle_with_args(dev->of_node, "power-domains", + "#power-domain-cells"); + if (count == -ENOENT) + return 0; + if (count < 0) + return count; + if (count <= 1) + return 0; + + mutex_lock(&ane_rtclient_pd_lock); + list_for_each_entry(pd, &ane_rtclient_pd_list, list) + if (pd->dev == dev) + goto found; + + pd = kzalloc(sizeof(*pd), GFP_KERNEL); + if (!pd) { + err = -ENOMEM; + goto out; + } + INIT_LIST_HEAD(&pd->list); + pd->dev = dev; + pd->pd_dev = kcalloc(count, sizeof(*pd->pd_dev), GFP_KERNEL); + pd->pd_link = kcalloc(count, sizeof(*pd->pd_link), GFP_KERNEL); + if (!pd->pd_dev || !pd->pd_link) { + ane_rtclient_pd_free(pd); + pd = NULL; + err = -ENOMEM; + goto out; + } + list_add_tail(&pd->list, &ane_rtclient_pd_list); + +found: + for (i = pd->count; i < count; i++) { + if (!pd->pd_dev[i]) { + pd->pd_dev[i] = dev_pm_domain_attach_by_id(dev, i); + if (IS_ERR_OR_NULL(pd->pd_dev[i])) { + err = IS_ERR(pd->pd_dev[i]) ? + PTR_ERR(pd->pd_dev[i]) : -ENODEV; + pd->pd_dev[i] = NULL; + goto out; + } + } + if (!pd->pd_link[i]) { + if (!ane_rtclient_pinned) { + if (!try_module_get(THIS_MODULE)) { + err = -ENODEV; + goto out; + } + ane_rtclient_pinned = true; + } + pd->pd_link[i] = + device_link_add(dev, pd->pd_dev[i], + DL_FLAG_STATELESS | + DL_FLAG_PM_RUNTIME | + DL_FLAG_RPM_ACTIVE); + if (!pd->pd_link[i]) { + err = -EINVAL; + goto out; + } + } + pd->count = i + 1; + } + + dev_emerg(dev, "BOOT-PHASE genpd domains attached: %d\n", count); +out: + mutex_unlock(&ane_rtclient_pd_lock); + return err; +} + +static int ane_rtclient_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct resource *res; + struct ane_rtclient *ane; + struct ane_t6021 *a; + u32 cpu_status, ps_cpu; + u64 rvbar; + int ret; + + if (!ane_t6021_fwload_options_ok()) { + dev_err(dev, + "invalid firmware RAM-grant options; refusing before power access\n"); + return -EINVAL; + } + + if (!ane_t6021_fwload_placement_ok(dev)) { + dev_err(dev, + "fw_alias_reserved=1, but no no-map /reserved-memory node covers the iBoot firmware windows (this m1n1 does not reserve them); refusing before power access, ANE off\n"); + return -ENODEV; + } + + /* Early, BEFORE any allocation/power: legacy_only must never be + * rejected after the CPU release, where an unwind could drop + * domains under a running ASC (lab probe order). + */ + if (legacy_only && (!fw_start || fw_start_rtb_mode)) { + dev_err(dev, + "legacy_only=1 requires fw_start=1 and fw_start_rtb_mode=0\n"); + return -EINVAL; + } + if (legacy_query && !legacy_only) { + dev_err(dev, "legacy_query requires legacy_only=1\n"); + return -EINVAL; + } + + ane = devm_kzalloc(dev, sizeof(*ane), GFP_KERNEL); + if (!ane) + return -ENOMEM; + ane->legacy_buffers = devm_kcalloc(dev, ANE_LEGACY_ALLOCS, + sizeof(*ane->legacy_buffers), + GFP_KERNEL); + if (!ane->legacy_buffers) + return -ENOMEM; + ane->dev = dev; + ane->soc = of_device_get_match_data(dev); + platform_set_drvdata(pdev, ane); + INIT_DELAYED_WORK(&ane->poll_work, ane_rtclient_post_boot); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) + return -ENODEV; + if (!(res->flags & IORESOURCE_MEM_NONPOSTED)) + dev_warn(dev, "engine window is not flagged non-posted\n"); + ane->engine = ioremap_np(res->start, resource_size(res)); + if (!ane->engine) + return -ENOMEM; + ane->cpu_rst = devm_reset_control_get_optional_exclusive(dev, NULL); + if (IS_ERR(ane->cpu_rst)) + return dev_err_probe(dev, PTR_ERR(ane->cpu_rst), + "ane_cpu reset control\n"); + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + if (ret) + return ret; + + ret = ane_rtclient_attach_genpd(ane); + if (ret) + return dev_err_probe(dev, ret, "extra genpd attach\n"); + pm_runtime_enable(dev); + ret = pm_runtime_resume_and_get(dev); + if (ret) + return dev_err_probe(dev, ret, "genpd raise failed\n"); + + ane->pmgr = devm_of_iomap(dev, dev->of_node, 1, NULL); + if (IS_ERR(ane->pmgr)) { + ret = PTR_ERR(ane->pmgr); + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + return dev_err_probe(dev, ret, + "pmgr window map failed; G1 gate cannot run\n"); + } + ps_cpu = readl(ane->pmgr + ane->soc->ps_cpu_off); + dev_emerg(dev, "ane_cpu ACTUAL = 0x%x\n", ps_cpu); + if (FIELD_GET(ANE_PS_ACTUAL, ps_cpu) != ANE_PS_ON) { + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + return -EPROBE_DEFER; + } + /* T8112: the kext (type 0x70) opens PWGATE (bits 29:28 = 0) before + * the ps words. This driver only reads it: an engine read behind a + * closed gate is untested, so it refuses first. + */ + if (ane->soc->pwgate_off) { + void __iomem *set = devm_of_iomap(dev, dev->of_node, 2, NULL); + u32 gate = IS_ERR(set) ? U32_MAX : + readl(set + ane->soc->pwgate_off); + + dev_emerg(dev, "PWGATE = 0x%x\n", gate); + if (gate & GENMASK(29, 28)) { + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + return dev_err_probe(dev, -ENODEV, + "PWGATE closed; refusing before any engine read\n"); + } + } + + cpu_status = readl(ane->engine + ANE_ASC_CPU_STATUS); + rvbar = readq(ane->engine + ANE_ASC_RVBAR); + dev_emerg(dev, + "BOOT-PHASE engine reads ok: CPU_STATUS = 0x%x, RVBAR = %016llx (bit0=%u)\n", + cpu_status, rvbar, (u32)(rvbar & 1)); + + if (!(cpu_status & ANE_ASC_CPU_STATUS_RUNNING)) { + if (!fw_start) { + dev_err(dev, + "ANE firmware not alive (CPU_STATUS 0x%x) — start it from a quiesce context, or retry with fw_start=1\n", + cpu_status); + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + return -EPROBE_DEFER; + } + + /* Lab fw_start order: staging requires fw_load and an + * IOMMU-mapped device BEFORE any allocation/staging. + */ + if (!ane_t6021_fwload_requested()) { + dev_err(dev, + "fw_start: requires fw_load=1 (no staged firmware)\n"); + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + return -EINVAL; + } + if (!device_iommu_mapped(dev)) { + dev_err(dev, + "fw_start: device not IOMMU-mapped — a staged DVA/entry alias would be untranslated; refusing\n"); + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + return -EINVAL; + } + + a = devm_kzalloc(dev, sizeof(*a), GFP_KERNEL); + if (!a) { + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + return -ENOMEM; + } + a->dev = dev; + a->base[ANE_T6021_REG_ENGINE] = ane->engine; + a->irq = -1; + a->power_gated = true; + ane->fw = a; + + dev_emerg(dev, "BOOT-PHASE fwload stage+alias begin\n"); + ret = ane_t6021_fwload_probe(a); + if (ret) { + dev_err_probe(dev, ret, "fw_start: staging failed\n"); + ane_t6021_fwload_remove(a); + ane->fw = NULL; + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + return ret; + } + if (!ane_t6021_rvbar_entry_ok(a->fw_iova)) { + dev_err(dev, "fw_start: staged iova %pad invalid\n", + &a->fw_iova); + ane_t6021_fwload_remove(a); + ane->fw = NULL; + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + return -EINVAL; + } + + dev_emerg(dev, "BOOT-PHASE dispatch (table_mode=%d)\n", + fw_start_table_mode); + ret = ane_t6021_boot_start(a, 0, fw_start_table_mode, + fw_start_rtb_mode); + if (ret == -ENODATA || ret == -EAGAIN || + ret == -EBUSY || ret == -ECANCELED) { + ane_t6021_fwload_remove(a); + ane->fw = NULL; + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + return ret; + } + + ane->held = true; + cpu_status = readl(ane->engine + ANE_ASC_CPU_STATUS); + dev_emerg(dev, + "BOOT-PHASE sequence returned %pe (cpu_started=%u fw_alive=%u booted=%u) CPU_STATUS=0x%x\n", + ERR_PTR(ret), a->cpu_started, a->fw_alive, a->booted, + cpu_status); + if (!a->fw_alive && !fw_start_rtb_mode) { + dev_err(dev, + "fw_start: poll A timeout, no READY — HELD until reboot, RTKit handshake skipped\n"); + return 0; + } + } + + if (legacy_only) { + int cfg_err = 0; + + ane_rtclient_validate_chman(ane); + if (ane->fw && ane->fw->booted && scratch3_ack && + ane->chman_ok && + ane_t6021_chman_host_init(ane->fw->boot_ipc, + ane->fw->boot_ipc_size, + ane->fw->boot_ipc_iova)) { + int hello_ret = 0; + + dma_wmb(); + dev_emerg(dev, + "LEGACY P8 host ack: SCRATCH3 <- %08x\n", + ANE_T6021_BOOT_ACK); + /* hello_wait_ms > 0 only: init rtkit and arm the RX + * poll worker BEFORE writing the ack, so a HELLO + * after the ack is not missed. The 13.5 fw sent none + * on any recorded boot. + */ + if (hello_wait_ms && !ane->rtk) { + ane->rtk = devm_apple_rtkit_init(dev, ane, + NULL, 0, + &ane_rtclient_rtkit_ops); + if (IS_ERR(ane->rtk)) { + dev_err(dev, + "LEGACY hello: rtkit init %pe\n", + ane->rtk); + ane->rtk = NULL; + } else { + schedule_delayed_work(&ane->poll_work, + msecs_to_jiffies(10)); + } + } + writel(ANE_T6021_BOOT_ACK, + ane->engine + ANE_MBI_SCRATCH0 + 4 * 3); + if (ane->rtk) { + unsigned long hello_deadline = + jiffies + msecs_to_jiffies(hello_wait_ms); + + dev_emerg(dev, "LEGACY hello: boot begin (%u ms)\n", + hello_wait_ms); + do { + hello_ret = apple_rtkit_boot(ane->rtk); + } while (hello_ret == -ETIME && + time_before(jiffies, hello_deadline)); + dev_emerg(dev, + "LEGACY hello: boot %pe running=%d crashed=%d\n", + ERR_PTR(hello_ret), + apple_rtkit_is_running(ane->rtk), + apple_rtkit_is_crashed(ane->rtk)); + if (!hello_ret) { + ane->boot_done = true; + if (start_app_eps) + ane_rtclient_start_app_eps(ane); + } else { + cancel_delayed_work_sync(&ane->poll_work); + } + } + } else { + dev_warn(dev, + "LEGACY ack withheld (booted=%u scratch3_ack=%u chman_ok=%u)\n", + ane->fw ? ane->fw->booted : 0, + scratch3_ack, ane->chman_ok); + } + if (ane->chman_ok) { + /* One reusable command buffer for the transport + * (CONFIG_GET + every ioctl); reuse is legal + * because each exchange completes with the slot + * host-owned again. The 128-entry table stays + * for fw MALLOC replies only. + */ + if (ane_rtclient_legacy_alloc(ane, SZ_16K) == 0) + ane->cmd_buf = + &ane->legacy_buffers[ane->legacy_allocated - 1]; + else + cfg_err = -ENOMEM; + } + if (!cfg_err && legacy_query && ane->chman_ok) { + /* CONFIG_GET (opcode 0x03) keeps the legacy ChMan + * transport armed for the ioctls; the reply's + * word +0x08 must be nonzero (lab rule). The + * boot heap + 'IPC ' allocations stay HELD until + * reboot. + */ + struct ane_legacy_buffer *command = ane->cmd_buf; + int qret; + + if (command) { + memset(command->cpu, 0, SZ_16K); + qret = ane_rtclient_legacy_exchange(ane, + command, + 16, 0x03, + 1, 3000); + dev_info(dev, + "LEGACY CONFIG_GET words %08x %08x result=%d (DMA remains held)\n", + READ_ONCE(((u32 *)command->cpu)[1]), + READ_ONCE(((u32 *)command->cpu)[2]), + qret); + if (qret) + cfg_err = qret; + else if (!READ_ONCE(((u32 *)command->cpu)[2])) { + dev_err(dev, + "LEGACY CONFIG_GET reply word +0x08 zero\n"); + cfg_err = -EPROTO; + } + } else { + cfg_err = -ENOMEM; + } + } + ane->boot_done = true; + if (cfg_err) { + dev_err(dev, + "install: CONFIG_GET failed (%d) — refusing to register DRM device (ioctls would run on an unproven ring)\n", + cfg_err); + if (!ane->held) { + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + } + return cfg_err; + } + } else { + unsigned long deadline; + int boot_ret; + + ane->rtk = devm_apple_rtkit_init(dev, ane, NULL, 0, + &ane_rtclient_rtkit_ops); + if (IS_ERR(ane->rtk)) { + ret = PTR_ERR(ane->rtk); + ane->rtk = NULL; + dev_err_probe(dev, ret, "apple_rtkit_init failed\n"); + goto err_pm_or_hold; + } + /* RX fallback worker first: the fw's HELLO must not be + * missed while the recv line is unproven. + */ + schedule_delayed_work(&ane->poll_work, msecs_to_jiffies(10)); + deadline = jiffies + msecs_to_jiffies(hello_wait_ms); + do { + boot_ret = apple_rtkit_boot(ane->rtk); + } while (boot_ret == -ETIME && time_before(jiffies, deadline)); + if (boot_ret) { + dev_err(dev, "rtkit boot handshake failed: %pe\n", + ERR_PTR(boot_ret)); + cancel_delayed_work_sync(&ane->poll_work); + ret = boot_ret; + goto err_pm_or_hold; + } + ane->boot_done = true; + if (start_app_eps) + ane_rtclient_start_app_eps(ane); + } + + if (!ane->chman_ok) { + dev_err(dev, + "install: ChMan table not validated — refusing to register DRM device (ioctls would stall the ring)\n"); + cancel_delayed_work_sync(&ane->poll_work); + if (!ane->held) { + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + } + return -EPROTO; + } + + { + struct ane_t6021_drm *adrm; + int drmret; + + /* This kernel dropped drm_dev_init; the resource-managed + * alloc registers the same ABI-2 device. + */ + adrm = devm_drm_dev_alloc(dev, &ane_t6021_drm_driver, + struct ane_t6021_drm, drm); + if (IS_ERR(adrm)) { + ret = dev_err_probe(dev, PTR_ERR(adrm), + "drm device alloc\n"); + goto err_pm_or_hold; + } + adrm->dev = dev; + adrm->ane = ane; + WRITE_ONCE(ane_t6021_perf_ane, ane); + drmret = drm_dev_register(&adrm->drm, 0); + if (drmret) { + dev_err_probe(dev, drmret, "drm_dev_register\n"); + ret = drmret; + goto err_pm_or_hold; + } + dev_info(dev, + "loaded ane_t6021 (DRM major %d minor %d; ABI 2; legacy_only=%u chman_ok=%u booted=%u; state %s; BO cap %u MiB)\n", + ANE_ABI_M2_MAJOR, 0, + legacy_only, ane->chman_ok, + ane->fw ? ane->fw->booted : 0, + ane->held ? "HELD" : "ready", bo_total_max_mb); + } + + return 0; + +err_pm_or_hold: + if (ane->held) + dev_warn(dev, + "probe failed after CPU start (%pe) — binding fenced; power/rings/IRQ HELD until reboot\n", + ERR_PTR(ret)); + else { + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + } + return ret; +} + +static void ane_rtclient_remove(struct platform_device *pdev) +{ + struct ane_rtclient *ane = platform_get_drvdata(pdev); + + if (READ_ONCE(ane_t6021_perf_ane) == ane) + WRITE_ONCE(ane_t6021_perf_ane, NULL); + + cancel_delayed_work_sync(&ane->poll_work); + + if (ane->held) + dev_warn(&pdev->dev, + "remove HELD: no teardown — reboot reclaims\n"); +} + +static const struct of_device_id ane_rtclient_of_match[] = { + { .compatible = "apple,t6020-ane", .data = &ane_t6020_soc }, + { .compatible = "apple,t6021-ane", .data = &ane_t6021_soc }, + { .compatible = "apple,t6022-ane", .data = &ane_t6022_soc }, + { .compatible = "apple,t8112-ane", .data = &ane_t8112_soc }, + { } +}; +MODULE_DEVICE_TABLE(of, ane_rtclient_of_match); + +static struct platform_driver ane_rtclient_driver = { + .driver = { + .name = "ane_t6021", + .of_match_table = ane_rtclient_of_match, + .suppress_bind_attrs = true, + }, + .probe = ane_rtclient_probe, + .remove = ane_rtclient_remove, +}; + +static int __init ane_rtclient_init(void) +{ + int ret = platform_driver_register(&ane_rtclient_driver); + + if (ret) + ane_t6021_trace_free(); + return ret; +} +module_init(ane_rtclient_init); + +static void __exit ane_rtclient_exit(void) +{ + platform_driver_unregister(&ane_rtclient_driver); + ane_t6021_trace_free(); +} +module_exit(ane_rtclient_exit); + +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple Neural Engine (T6021/M2) installed module"); diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c new file mode 100644 index 0000000000000..0e98c48d2aff1 --- /dev/null +++ b/drivers/accel/ane/ane_tm.c @@ -0,0 +1,652 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2022 Eileen Yoon */ + +#include +#include +#include +#include +#include "ane_tm.h" + +#define ANE_TQ_COUNT 8 +static const int TQ_PRTY_TABLE[ANE_TQ_COUNT] = { 0x1, 0x2, 0x3, 0x4, + 0x5, 0x6, 0x1e, 0x1f }; + +#define ANE_TM_BASE 0x20000 +#define ANE_TQ_BASE 0x21000 + +#define TM_ADDR 0x0 +#define TM_INFO 0x4 +#define TM_PUSH 0x8 +#define TM_TQ_EN 0xc + +#define TM_IRQ_EVTC(line) (0x14 + (line * 0x14)) +#define TM_IRQ_INFO(line) (0x18 + (line * 0x14)) +#define TM_IRQ_UNK1(line) (0x1c + (line * 0x14)) +#define TM_IRQ_TMST(line) (0x20 + (line * 0x14)) +#define TM_IRQ_UNK2(line) (0x24 + (line * 0x14)) + +#define TM_COMMITTED 0x44 +#define TM_STATUS 0x54 +#define TM_ERROR1 0x58 +#define TM_ERROR2 0x5c +#define TM_ERROR3 0x60 +#define TM_IRQ_EN1 0x68 +#define TM_IRQ_ACK 0x6c +#define TM_IRQ_EN2 0x70 + +#define TQ_STATUS(qid) (0x000 + (qid * 0x148)) +#define TQ_PRTY(qid) (0x010 + (qid * 0x148)) +#define TQ_VACANT(qid) (0x014 + (qid * 0x148)) +#define TQ_INFO(qid) (0x01c + (qid * 0x148)) + +#define TQ_BAR1(qid, bdx) (0x020 + (qid * 0x148) + (bdx * 0x4)) +#define TQ_NID1(qid) (0x0a0 + (qid * 0x148)) +#define TQ_SIZE2(qid) (0x0a4 + (qid * 0x148)) +#define TQ_ADDR2(qid) (0x0a8 + (qid * 0x148)) + +#define TQ_BAR2(qid, bdx) (0x0ac + (qid * 0x148) + (bdx * 0x4)) +#define TQ_NID2(qid) (0x12c + (qid * 0x148)) +#define TQ_SIZE1(qid) (0x130 + (qid * 0x148)) +#define TQ_ADDR1(qid) (0x134 + (qid * 0x148)) + +#define TM_IS_IDLE 0x1 +#define TM_IS_FINE 0x22222222 + +#define tm_read32(ane, off) (readl(ane->engine + ANE_TM_BASE + off)) +#define tq_read32(ane, off) (readl(ane->engine + ANE_TQ_BASE + off)) +#define tm_write32(ane, off, val) (writel(val, ane->engine + ANE_TM_BASE + off)) +#define tq_write32(ane, off, val) (writel(val, ane->engine + ANE_TQ_BASE + off)) + +/* The ane SET block (m1n1 ANE.ps_map) maps the pmgr power-state words + * for this engine (set0, base, set1..4). pmgr reads are always safe and + * give recovery its ACTUAL power evidence; the SET words themselves are + * firmware-locked: a direct write external-aborts the SoC. Bisect + * evidence, 2026-09-16: the T6001 kill is netconsole-named at + * 0x28e08c000 (PS_SET0 down, all sets gated); T8103 hard-reset the same + * way at 0x23b70c000 with the 95dbcf3-era gate armed. No code here ever + * writes the SET block. + */ +#define ANE_PS_ACTUAL_MASK 0xf0 +#define ANE_PS_WORDS 6 /* set0, base, set1..4 */ + +/* ACTUAL nibble of each SET word, word 0 in the low nibble; 0 when the + * SET block is unmapped. pmgr registers only: engine MMIO is never + * read for power state (a readl through a warm gate external-aborts + * and hard-resets the machine). + */ +u32 ane_ps_act(struct ane_device *ane) +{ + u32 v = 0; + int i; + + if (!ane->ps) + return 0; + for (i = 0; i < ANE_PS_WORDS; i++) + v |= ((readl(ane->ps + i * 8) & ANE_PS_ACTUAL_MASK) >> 4) + << (i * 4); + return v; +} + +/* Bisect probe: per-word named reads of the SET window. Each word logs + * before its readl, so a window whose address does not decode on this + * SoC is named by the last off-box line (word index + byte offset) + * instead of a silent hard reset between two other prints. + */ +u32 ane_ps_act_probe(struct ane_device *ane) +{ + u32 v = 0; + int i; + + if (!ane->ps) { + dev_info(ane->dev, "ps probe: SET window unmapped\n"); + return 0; + } + dev_info(ane->dev, "ps probe: SET window at %pap, %d words\n", + &ane->ps_base, ANE_PS_WORDS); + for (i = 0; i < ANE_PS_WORDS; i++) { + u32 w; + + dev_info(ane->dev, "ps probe: word %d @ ps+0x%02x reading\n", + i, i * 8); + w = readl(ane->ps + i * 8); + v |= ((w & ANE_PS_ACTUAL_MASK) >> 4) << (i * 4); + dev_info(ane->dev, "ps probe: word %d -> %#x\n", i, w); + } + return v; +} + +/* Recovery-path MMIO logging. Every write prints before and after, and + * every engine read prints its value, each line carrying the pmgr + * ACTUAL of the owning partitions. A write that external-aborts the + * SoC is then named by the last line the off-box netconsole carried, + * and the bisect starts from that register instead of a guess. + */ +static void ane_rec_writel(struct ane_device *ane, const char *reg, + void __iomem *addr, u32 val) +{ + dev_info(ane->dev, "ANEWR %s <- %#x (ps act %#x)\n", reg, val, + ane_ps_act(ane)); + writel(val, addr); + dev_info(ane->dev, "ANEWR %s wrote (ps act %#x)\n", reg, + ane_ps_act(ane)); +} + +static u32 ane_rec_read32(struct ane_device *ane, const char *reg, + void __iomem *addr) +{ + u32 val = readl(addr); + + dev_info(ane->dev, "ANERD %s -> %#x (ps act %#x)\n", reg, val, + ane_ps_act(ane)); + return val; +} + +u32 ane_tm_ps_act(struct ane_device *ane) +{ + return ane_ps_act(ane); +} + +void ane_tm_enable(struct ane_device *ane, bool rec) +{ + void __iomem *tq_en = ane->engine + ANE_TM_BASE + TM_TQ_EN; + u32 val = rec ? ane_rec_read32(ane, "TM_TQ_EN tm+0x0c", tq_en) + : readl(tq_en); + char reg[24]; + + val |= 0x1000; + if (rec) + ane_rec_writel(ane, "TM_TQ_EN tm+0x0c", tq_en, val); + else + writel(val, tq_en); + + for (int qid = 0; qid < ANE_TQ_COUNT; qid++) { + void __iomem *prty = + ane->engine + ANE_TQ_BASE + TQ_PRTY(qid); + + if (rec) { + snprintf(reg, sizeof(reg), "TQ_PRTY[%d] tq+%#x", qid, + TQ_PRTY(qid)); + ane_rec_writel(ane, reg, prty, TQ_PRTY_TABLE[qid]); + } else { + writel(TQ_PRTY_TABLE[qid], prty); + } + } + + if (rec) { + ane_rec_writel(ane, "TM_IRQ_EN1 tm+0x68", + ane->engine + ANE_TM_BASE + TM_IRQ_EN1, + 0x4000000); + ane_rec_writel(ane, "TM_IRQ_EN2 tm+0x70", + ane->engine + ANE_TM_BASE + TM_IRQ_EN2, 0x6); + } else { + tm_write32(ane, TM_IRQ_EN1, 0x4000000); + tm_write32(ane, TM_IRQ_EN2, 0x6); + } +} + +u32 ane_tm_status(struct ane_device *ane) +{ + return tm_read32(ane, TM_STATUS); +} + +int ane_tm_enqueue(struct ane_device *ane, struct ane_request *req) +{ + int qid = req->qid; + + tq_write32(ane, TQ_STATUS(qid), 0x1); + + for (int bdx = 0; bdx < ANE_TILE_COUNT; bdx++) { + tq_write32(ane, TQ_BAR1(qid, bdx), req->bar[bdx]); + } + + tq_write32(ane, TQ_SIZE1(qid), ((req->td_size >> 2) - 1) << 0x10); + tq_write32(ane, TQ_ADDR1(qid), req->btsp_iova); + tq_write32(ane, TQ_NID1(qid), (req->nid & 0xff) << 8 | 1); + + return 0; +} + +static void ane_tm_push_tq(struct ane_device *ane, struct ane_request *req) +{ + int qid = req->qid; + + tm_write32(ane, TM_ADDR, tq_read32(ane, TQ_ADDR1(qid))); + tm_write32(ane, TM_INFO, tq_read32(ane, TQ_SIZE1(qid)) | req->td_count); + tm_write32(ane, TM_PUSH, TQ_PRTY_TABLE[qid] | (qid & 7) << 8); // magic +} + +static int ane_tm_collect_events(struct ane_device *ane, + struct ane_request *req, u32 *finished) +{ + for (unsigned int line = 0; line < 2; line++) { + u32 count = tm_read32(ane, TM_IRQ_EVTC(line)); + + if (count > 64) + return -EIO; + for (u32 n = 0; n < count; n++) { + u32 info = tm_read32(ane, TM_IRQ_INFO(line)); + + tm_read32(ane, TM_IRQ_UNK1(line)); + tm_read32(ane, TM_IRQ_TMST(line)); + tm_read32(ane, TM_IRQ_UNK2(line)); + if (req && info == (0x05000000 | (req->nid << 16) | + (req->td_count - 1))) + *finished |= BIT(line); + } + } + tm_write32(ane, TM_IRQ_ACK, tm_read32(ane, TM_IRQ_ACK) | 2); + return *finished == 3 && (tm_read32(ane, TM_STATUS) & TM_IS_IDLE); +} + +/* read_poll_timeout op: consult the DART latch before any engine access. + * Returns 1 to end the poll when a fault is latched; the caller then + * handles the fault without ever reading the wedged engine. + */ +static int ane_tm_poll_step(struct ane_device *ane, struct ane_request *req, + u32 *finished) +{ + if (ane_dart_faulted(ane, NULL, NULL)) + return 1; + return ane_tm_collect_events(ane, req, finished); +} + +int ane_tm_execute(struct ane_device *ane, struct ane_request *req) +{ + u32 finished = 0; + u32 fault_status = 0; + u64 fault_iova = 0; + struct ane_dart_scratch scratch[ANE_DART_SCRATCH_MAX]; + int scratch_pages = 0; + bool faulted = false; + int err, status; + + lockdep_assert_held(&ane->engine_lock); + ane_dart_mask(ane); + + err = ane_tm_collect_events(ane, NULL, &finished); + if (err < 0) + goto wedge; + wmb(); + ane_tm_push_tq(ane, req); + + for (;;) { + /* Latch-first completion poll. Engine register reads hang + * jwm1 once a DART fault has landed (lockups 1 and 3: the + * journal ends inside the 1-microsecond completion poll, + * before any fault handling ran), so the poll op consults + * the DART error latch first and stops the poll before the + * engine is touched. Residual hazard: a fault landing + * between one iteration's latch read and engine read. + */ + err = read_poll_timeout(ane_tm_poll_step, status, + status != 0, 1, 1000000, false, + ane, req, &finished); + if (!err && status < 0) + err = status; + if (ane_dart_faulted(ane, &fault_iova, &fault_status)) { + faulted = true; + if (scratch_pages >= ANE_DART_SCRATCH_MAX) + break; + if (ane_dart_drain_fault(ane, fault_iova, scratch, + &scratch_pages)) + break; + /* The retried transaction now completes; poll on. */ + continue; + } + break; + } + + if (faulted) { + /* The faulting program drained through the scratch pages + * and the engine is idle again (the latch is clear or the + * poll would still be running). Gate the queue with the + * normal completion writes, hand the scratch pages back, + * and fail this request. No partition power cycle, no + * stream disable: the page tables were never changed by + * the fault. + */ + if (err || ane_dart_faulted(ane, NULL, NULL)) { + /* Drain failed or the drained program never reached + * completion: the engine is not idle and access is + * off the table. wedge releases the scratch. + */ + err = err ?: -EIO; + goto wedge; + } + status = tq_read32(ane, TQ_NID1(req->qid)); + tq_write32(ane, TQ_NID1(req->qid), status & ~1U); + rmb(); + tq_write32(ane, TQ_STATUS(req->qid), 0x0); + ane_dart_release_scratch(ane, scratch, scratch_pages); + ane_dart_unmask(ane); + dev_err(ane->dev, + "DART fault contained: status=%#x iova=%#llx scratch=%d\n", + fault_status, fault_iova, scratch_pages); + return -EIO; + } + if (err) + goto wedge; + status = tq_read32(ane, TQ_NID1(req->qid)); + if (((status >> 8) & 0xff) != req->nid) { + err = -EIO; + goto wedge; + } + tq_write32(ane, TQ_NID1(req->qid), status & ~1U); + rmb(); + tq_write32(ane, TQ_STATUS(req->qid), 0x0); + ane_dart_unmask(ane); + return 0; + +wedge: + ane_dart_release_scratch(ane, scratch, scratch_pages); + ane_dart_unmask(ane); + if (atomic_xchg(&ane->wedged, 1) == 0) { + __module_get(THIS_MODULE); + dev_err(ane->dev, + "tm completion failed: %d, finish lines=%x (q%d nid=%#x)\n", + err, finished, req->qid, req->nid); + } + + /* One bounded recovery attempt: stop the tm, power-cycle the engine + * and return to accepting work. Only a failed reset preserves + * resources until reboot. + */ + if (ane_tm_recover(ane) < 0) { + if (ane->tm_retention) + dev_err(ane->dev, + "recovery failed; preserving resources until module reload or reboot (tm/tq file retained through the cycle: the latched task error parks the TM in place; rmmod+insmod restores service)\n"); + else + dev_err(ane->dev, + "recovery failed; preserving resources until reboot\n"); + } + return err; +} + +/* + * Bounded recovery after a failed task. The only reset this driver knows + * is the partition power cycle system sleep already performs on this + * hardware. With the several ANE power partitions attached as genpd + * devices (T8103 five, T6001 similar), gating happens on those devices; + * with a single domain attached directly to the device, on the device + * itself. Gating the partitions stops any DMA still fetching the dead + * task and clears the tm register file; ungating brings the engine back + * through the probe resume path (ane_tm_enable), then the task manager + * must report idle. BO mappings deliberately stay in place: they remain + * coherent through the cycle and valid for the next submit, and once the + * wedge clears, BO_FREE unmaps them through the normal path again. + * Callers hold engine_lock. + * + * T6001 note, bisect evidence 2026-09-16: its set0/base islands are + * firmware-locked (direct pmgr writes external-abort; see the SET block + * comment above) and hold the tm/tq file in retention through any + * genpd cycle - TQ_EN still reads 0x3000 after a 20 ms and a 2 s gate, + * and tasks dispatched afterwards complete but compute nondeterministic + * garbage. The idle-or-fresh poll below therefore fails on T6001 and + * recovery preserves until reboot instead of serving wrong data. + */ + +/* No engine MMIO unless the owning partitions read powered on (ACTUAL + * nibble per word): a readl through a warm gate external-aborts and + * hard resets the machine. Covers set0, base and set1..4 from the SET + * block map; sys_cpu rides its own genpd resume and has no cell in + * this block on T6001. + */ +static int ane_ps_verify_on(struct ane_device *ane) +{ + u32 act = 0; + int err = -ETIMEDOUT; + int i; + + if (!ane->ps) + return 0; + for (i = 0; i < 100; i++) { + act = ane_ps_act(ane); + if (act == ANE_PS_ALL_ON) { + err = 0; + break; + } + usleep_range(1000, 2000); + } + dev_info(ane->dev, "ANERD ps verify act=%#x err=%d\n", act, err); + return err; +} + +static int ane_pd_cycle(struct ane_device *ane) +{ + int err = 0; + + ane->recovering = true; + if (ane->pd_count > 1) { + int gated = 0; + + for (int i = 0; i < ane->pd_count; i++) { + pm_runtime_get_noresume(ane->pd_dev[i]); + gated = i + 1; + dev_info(ane->dev, + "ANERD pd[%d] %s force_suspend begin (ps act %#x)\n", + i, dev_name(ane->pd_dev[i]), ane_ps_act(ane)); + err = pm_runtime_force_suspend(ane->pd_dev[i]); + dev_info(ane->dev, + "ANERD pd[%d] force_suspend -> %d (ps act %#x)\n", + i, err, ane_ps_act(ane)); + if (err) + break; + } + if (!err) + for (int i = 0; i < ane->pd_count; i++) { + dev_info(ane->dev, + "ANERD pd[%d] %s force_resume begin (ps act %#x)\n", + i, dev_name(ane->pd_dev[i]), + ane_ps_act(ane)); + err = pm_runtime_force_resume(ane->pd_dev[i]); + dev_info(ane->dev, + "ANERD pd[%d] force_resume -> %d (ps act %#x)\n", + i, err, ane_ps_act(ane)); + if (err) + break; + } + while (gated--) + pm_runtime_put_noidle(ane->pd_dev[gated]); + } else { + /* Pin a second usage ref for the cycle: force_suspend only + * marks needs_force_resume when a ref beyond the probe one + * is held, and without that mark force_resume would leave + * the partition gated. + */ + pm_runtime_get_noresume(ane->dev); + dev_info(ane->dev, "ANERD dev force_suspend begin\n"); + err = pm_runtime_force_suspend(ane->dev); + dev_info(ane->dev, "ANERD dev force_suspend -> %d\n", err); + if (!err) { + dev_info(ane->dev, "ANERD dev force_resume begin\n"); + err = pm_runtime_force_resume(ane->dev); + dev_info(ane->dev, "ANERD dev force_resume -> %d\n", + err); + } + pm_runtime_put_noidle(ane->dev); + } + ane->recovering = false; + + return err; +} + +/* + * Drain the task-manager state a timed-out task leaves behind when the + * tm/tq file survives the power cycle in retention (T6001; descriptor + * tm_retention). T8103 never runs this: its cycle is a full POR of the + * file, and the re-arm below starts it clean. + * + * Two things must go, or the retained TM never reads idle again: + * - pending events on both IRQ lines. Each event pops by reading + * INFO/UNK1/TMST/UNK2, exactly as ane_tm_collect_events pops them; + * a stale backlog (or one over the 64-entry collect cap) would make + * the next submit's completion poll miss or fail. + * - the latched queue entry of the dead task. TQ_STATUS/NID1 get the + * same clear the normal completion path gives a finished queue. + */ +static void ane_tm_drain_retained(struct ane_device *ane) +{ + int line, qid; + + /* Stop the TM first: the cycle leaves the enable|halt latch set + * (TQ_EN 0x3000, 2026-09-16 evidence). Clear both bits so the + * re-arm below starts the TM from a defined stopped state. + */ + ane_rec_writel(ane, "TM_TQ_EN stop tm+0x0c", + ane->engine + ANE_TM_BASE + TM_TQ_EN, + tm_read32(ane, TM_TQ_EN) & ~0x3000U); + + for (line = 0; line < 2; line++) { + u32 count = tm_read32(ane, TM_IRQ_EVTC(line)); + u32 n; + + if (count > 64) + count = 64; + for (n = 0; n < count; n++) { + tm_read32(ane, TM_IRQ_INFO(line)); + tm_read32(ane, TM_IRQ_UNK1(line)); + tm_read32(ane, TM_IRQ_TMST(line)); + tm_read32(ane, TM_IRQ_UNK2(line)); + } + } + tm_write32(ane, TM_IRQ_ACK, tm_read32(ane, TM_IRQ_ACK) | 3); + + /* The dead task leaves per-queue error codes latched here (seen + * 0x22222222 / 0x2222: four-bit code 2 per TQ slot). While any + * latch is set the TM stays halted and TM_STATUS reads 0. Try the + * W1C convention first (write the read value back), fall back to + * plain zero-clear, and log what this silicon answered to. + */ + { + static const u16 err_reg[3] = { TM_ERROR1, TM_ERROR2, + TM_ERROR3 }; + char reg[24]; + int ei; + + for (ei = 0; ei < 3; ei++) { + void __iomem *addr = + ane->engine + ANE_TM_BASE + err_reg[ei]; + u32 val = readl(addr); + + if (!val) + continue; + writel(val, addr); + if (readl(addr)) + writel(0x0, addr); + snprintf(reg, sizeof(reg), "TM_ERROR%d clear tm+%#x", + ei + 1, err_reg[ei]); + ane_rec_read32(ane, reg, addr); + } + } + + for (qid = 0; qid < ANE_TQ_COUNT; qid++) { + u32 nid = tq_read32(ane, TQ_NID1(qid)); + + tq_write32(ane, TQ_STATUS(qid), 0x0); + if (nid & 1) + tq_write32(ane, TQ_NID1(qid), nid & ~1U); + } + + /* Retention evidence for the off-box console: what the dead task + * left in the committed counter and the error latches. + */ + ane_rec_read32(ane, "TM_COMMITTED tm+0x44 (drain)", + ane->engine + ANE_TM_BASE + TM_COMMITTED); + ane_rec_read32(ane, "TM_ERROR1 tm+0x58 (drain)", + ane->engine + ANE_TM_BASE + TM_ERROR1); + ane_rec_read32(ane, "TM_ERROR2 tm+0x5c (drain)", + ane->engine + ANE_TM_BASE + TM_ERROR2); + ane_rec_read32(ane, "TM_ERROR3 tm+0x60 (drain)", + ane->engine + ANE_TM_BASE + TM_ERROR3); +} + +int ane_tm_recover(struct ane_device *ane) +{ + u32 status; + int err; + + lockdep_assert_held(&ane->engine_lock); + if (!atomic_read(&ane->wedged)) + return 0; + + dev_err(ane->dev, "recovering: power-cycling engine partitions\n"); + + err = ane_pd_cycle(ane); + if (err) { + dev_err(ane->dev, "recovery: engine power cycle failed: %d\n", + err); + return err; + } + + /* No engine MMIO before the islands read powered on. */ + err = ane_ps_verify_on(ane); + if (err) { + dev_err(ane->dev, + "recovery: ane set islands not powered on: %d\n", err); + return err; + } + + /* T6001 (tm_retention): the cycle did NOT clear the tm/tq file — + * the timed-out task's events and queue entry are still latched. + * Drain them, or the TM below never reads idle and recovery + * degrades to preserve-until-reboot. + */ + if (ane->tm_retention) + ane_tm_drain_retained(ane); + + /* Power-on reset cleared the tm register file; re-arm it exactly + * like the probe resume path does. + */ + ane_tm_enable(ane, true); + + err = readl_poll_timeout(ane->engine + ANE_TM_BASE + TM_STATUS, + status, + (status & TM_IS_IDLE) || + status == ane->tm_status_fresh, + 100, 1000000); + status = ane_rec_read32(ane, "TM_STATUS tm+0x54", + ane->engine + ANE_TM_BASE + TM_STATUS); + if (err) { + dev_err(ane->dev, "recovery: tm not idle after reset: %#x\n", + status); + return err; + } + + if (atomic_xchg(&ane->wedged, 0)) { + module_put(THIS_MODULE); /* drop the wedge pin */ + /* The power cycle above quiesced every transaction, so the + * wedge-preserved mappings (dead BOs kept alive only to + * stop IOVA teardown under active DMA) can now be unmapped + * and their ranges handed back to the allocator. Without + * this reclaim the ranges would stay pinned for the whole + * module lifetime. + */ + ane_reclaim_preserved(ane); + dev_info(ane->dev, "tm recovered: idle, accepting work again\n"); + } + return 0; +} + +/* + * Drop the wedge pin on demand. The recovery path in ane_tm_recover + * does an atomic_xchg + module_put, but only on its success branch - + * a real -110 with the engine stuck leaves ane->wedged set and the + * module refcount one too high, so rmmod fails forever and the bug + * becomes a kernel-brick-without-reboot. ane_drm_postclose calls + * this unconditionally so every session close releases the pin, + * letting the operator unload the ko for an updated build even when + * the engine itself cannot be cleared in software. + * + * Pair count: must follow an ane_wedge_set() that called + * __module_get(THIS_MODULE). Today the only such set is in the wedge: + * branch of ane_tm_execute(); future sets must call this in their + * clean-up path too. + */ +void ane_wedge_clear(struct ane_device *ane) +{ + if (atomic_xchg(&ane->wedged, 0)) { + module_put(THIS_MODULE); + dev_info(ane->dev, + "wedge pin released (engine state unknown)\n"); + } +} diff --git a/drivers/accel/ane/ane_tm.h b/drivers/accel/ane/ane_tm.h new file mode 100644 index 0000000000000..94d425ab8d8b7 --- /dev/null +++ b/drivers/accel/ane/ane_tm.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2022 Eileen Yoon */ + +#ifndef __ANE_TM_H__ +#define __ANE_TM_H__ + +#include "ane.h" + +void ane_tm_enable(struct ane_device *ane, bool rec); +u32 ane_tm_status(struct ane_device *ane); +u32 ane_ps_act(struct ane_device *ane); +u32 ane_ps_act_probe(struct ane_device *ane); +bool ane_tm_islands_on(struct ane_device *ane, u32 *act); +int ane_tm_enqueue(struct ane_device *ane, struct ane_request *req); +int ane_tm_execute(struct ane_device *ane, struct ane_request *req); +int ane_tm_recover(struct ane_device *ane); +#define ANE_PS_ALL_ON ((1U << (4 * 6)) - 1) +u32 ane_tm_ps_act(struct ane_device *ane); + +#endif /* __ANE_TM_H__ */ diff --git a/include/uapi/drm/ane_accel.h b/include/uapi/drm/ane_accel.h new file mode 100644 index 0000000000000..e5fd9d12d1d63 --- /dev/null +++ b/include/uapi/drm/ane_accel.h @@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: (GPL-2.0-only WITH Linux-syscall-note) OR MIT */ +/* Copyright 2022 Eileen Yoon */ + +#ifndef __ANE_ACCEL_H__ +#define __ANE_ACCEL_H__ + +#include "drm.h" + +#if defined(__cplusplus) +extern "C" { +#endif + +#define ANE_TILE_COUNT 0x20 +#define ANE_FIFO_NID 0x40 +#define ANE_CMD_GRAN 0x10 + +/* ABI 1: successful SUBMIT guarantees terminal completion and CPU visibility. */ +#define ANE_ABI_MAJOR 1 + +/* + * ABI 2 (T6021 / M2, H14 firmware programs): the driver reports + * DRM_IOCTL_VERSION major 2 and rejects DRM_ANE_SUBMIT. The M1 libane + * requires major 1, so it cannot submit M1 frames to an M2 device. + * A successful DRM_ANE_EXEC guarantees terminal completion (firmware ack + * and task-queue idle) and CPU visibility of the output buffers. + */ +#define ANE_ABI_M2_MAJOR 2 +#define ANE_M2_MAX_SECTIONS 16 +#define ANE_M2_MAX_BINDS 64 + +#define DRM_ANE_BO_INIT 0x1 +#define DRM_ANE_BO_FREE 0x2 +#define DRM_ANE_SUBMIT 0x3 +#define DRM_ANE_PROG_LOAD 0x4 +#define DRM_ANE_PROC_CREATE 0x5 +#define DRM_ANE_EXEC 0x6 + +struct drm_ane_bo_init { + __u32 handle; + __u32 pad; + __u64 size; + __u64 offset; +}; + +struct drm_ane_bo_free { + __u32 handle; + __u32 pad; +}; + +struct drm_ane_submit { + __u64 tsk_size; + __u32 td_count; + __u32 td_size; + __u32 handles[ANE_TILE_COUNT]; + __u32 btsp_handle; + __u32 pad; +}; + +/* One firmware program section, already built by userspace, held in a BO. */ +struct drm_ane_section { + __u32 id; /* firmware section id */ + __u32 bo_handle; /* BO holding the bytes */ + __u64 size; + __u64 offset; /* byte offset inside the BO */ +}; + +/* A buffer whose IOVA and size the driver patches into a generic entry. */ +struct drm_ane_generic_bind { + __u32 buffer_id; + __u32 bo_handle; + __u32 type; + __u32 pad; + __u64 size; +}; + +struct drm_ane_prog_load { + __u64 sections_ptr; /* struct drm_ane_section[section_count] */ + __u64 generic_ptr; /* struct drm_ane_generic_bind[generic_count] */ + __u32 section_count; + __u32 generic_count; + __u32 prog_id_out; + __u32 pad; +}; + +struct drm_ane_proc_create { + __u32 prog_id; + __u32 proc_id_out; +}; + +/* Per-call buffer binding: an input or output for one procedure call. */ +struct drm_ane_exec_io { + __u32 buffer_id; + __u32 bo_handle; + __u32 type; + __u32 flags; + __u64 dma; /* in only, reserved: the driver binds the + * IOVA from bo_handle + */ + __u64 size; +}; + +struct drm_ane_exec { + __u32 prog_id; + __u32 proc_id; + __u32 priority; /* 2..7 */ + __u32 timeout_ms; + __u32 count; /* 1..ANE_M2_MAX_BINDS */ + __u32 pad; + __u64 io_ptr; /* struct drm_ane_exec_io[count] */ +}; + +#define DRM_IOCTL_ANE_BO_INIT \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_BO_INIT, struct drm_ane_bo_init) +#define DRM_IOCTL_ANE_BO_FREE \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_BO_FREE, struct drm_ane_bo_free) +#define DRM_IOCTL_ANE_SUBMIT \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_SUBMIT, struct drm_ane_submit) + +#define DRM_IOCTL_ANE_PROG_LOAD \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_PROG_LOAD, struct drm_ane_prog_load) +#define DRM_IOCTL_ANE_PROC_CREATE \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_PROC_CREATE, struct drm_ane_proc_create) +#define DRM_IOCTL_ANE_EXEC \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_EXEC, struct drm_ane_exec) + +#if defined(__cplusplus) +} +#endif + +#endif /* __ANE_ACCEL_H__ */ From 9cd8d4fa3fc5a73aaa152d6ef35a67d4150a336c Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:23:13 -0500 Subject: [PATCH 0589/2680] accel/ane: Bind only the M1 family in the ane module The ane module matched apple,t6020-ane and apple,t6021-ane, but it refused to bind both: T6020 had no proven SET-block base, and T6021 bound only with the allow_unqualified module parameter. The M2 family does not use the host-driven task manager of this module. Its coprocessor runs the ANE firmware, and the ane_t6021 module drives it for T6020, T6021, T6022 and T8112. The two entries made udev load both modules for every T602x ANE node, and the ane probe, when it ran first, failed with -ENODEV and logged an error. Remove the two entries and their SoC descriptors. Every remaining SoC has proven constants, so also remove the qualification tiers and the allow_unqualified parameter, and get the descriptor with of_device_get_match_data(). Each compatible of the apple,ane binding, and each ANE node in the apple device trees, now matches exactly one of the two match tables. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/accel/ane/ane_drv.c | 94 +++++-------------------------------- 1 file changed, 11 insertions(+), 83 deletions(-) diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index db03774c9e25a..5b269b0f8729f 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -880,54 +880,33 @@ static int ane_attach_genpd(struct ane_device *ane) } /* - * Per-SoC support descriptor, one per Linux `compatible`. + * Per-SoC descriptor, one per Linux `compatible`. * * Everything the probe needs except the SET block is derived from the * device tree by design: DART topology from "iommus", power wiring from - * "power-domains", the engine window from "reg", the IRQ by name. Board - * facts belong in the overlay, not in compiled constants. The SET block - * is the one exception: the live overlays carry no range for it, so it - * is compiled in per SoC — and only from a source that has proven the + * "power-domains", the engine window from "reg", the IRQ by name. The SET + * block is the one exception: the device tree carries no range for it, so + * it is compiled in per SoC, and only from a source that has proven the * address on hardware (m1n1 proxyclient ANE.ps_map, then a bound device). * A guessed SET base is not a bug but a brick: direct writes to the * block external-abort the SoC (T6001 named by netconsole 2026-09-16, - * PS_SET0 down at 0x28e08c000; T8103 same mechanism at 0x23b70c000), - * which is why a SoC without proven constants must refuse to bind - * instead of carrying a guess. - * - * Qualification tiers: - * ANE_QUALIFIED — execution proven on this silicon; binds normally. - * ANE_RECOGNIZED — constants present, never run on hardware; binds - * only with ane.allow_unqualified=1, loudly. - * ANE_UNSUPPORTED — no proven constants; refuses and names the data - * needed to advance the port. + * PS_SET0 down at 0x28e08c000; T8103 same mechanism at 0x23b70c000). + * The M2 family (T602x, T8112) runs the ANE firmware and binds to + * ane_t6021 instead. */ -enum ane_qual { - ANE_QUALIFIED, - ANE_RECOGNIZED, - ANE_UNSUPPORTED, -}; - struct ane_soc { phys_addr_t ps_base; - enum ane_qual qual; /* True when the tm/tq register file survives a genpd cycle in * retention and recovery must drain it (see ane_tm_drain_retained). */ bool tm_retention; }; -static bool allow_unqualified; -module_param(allow_unqualified, bool, 0444); -MODULE_PARM_DESC(allow_unqualified, - "Bind recognized-but-unproven ANE SoCs (unverified SET base: external-abort risk)"); - static const struct ane_soc ane_soc_t8103 = { /* M1 (m1-test-host). SET block mapped read-only for the recovery ACTUAL * log and the powered-on guard. Execution proven in the fleet. */ .ps_base = 0x23b70c000ULL, - .qual = ANE_QUALIFIED, }; static const struct ane_soc ane_soc_t6000 = { @@ -935,42 +914,12 @@ static const struct ane_soc ane_soc_t6000 = { * (proven on T6001/t6001-test-host); M1 Pro and M1 Ultra are untested. */ .ps_base = 0x28e08c000ULL, - .qual = ANE_QUALIFIED, .tm_retention = true, }; -static const struct ane_soc ane_soc_t6020 = { - /* M2 Pro (t6020). Community device-tree captures exist, but the - * SET base is unproven and the H14 compiler backend is unqualified: - * no constants may enter here yet. - */ - .ps_base = 0, - .qual = ANE_UNSUPPORTED, -}; - -static const struct ane_soc ane_soc_t6021 = { - /* M2 Max (T6021, t6021-test-host). SET window 0x8e08c000 = pmgr base - * 0x8e080000 + 0xc000, present in ane0's own reg on macOS captures - * of both 26.6.2 and 27.0 (2026-09-18, 11/11 derivation-input - * checks each, zero delta); the Linux translation reuses the pmgr - * high bits proven on T6000/T6020 rows (low-32 match) -> - * 0x28e08c000. Recognized, not qualified: constants complete, but - * no Linux execution yet, and the t6021 pwrstate word layout - * (which of set0/base/set1..4 sits at which offset — macOS-side - * capture cannot see it) is unverified; the first probe resume - * logs the ACTUAL nibbles through this window as the Linux-side - * probe (receipt 2026-09-18-t6021-driver-entry-prepared.md). - * Promotion to ANE_QUALIFIED needs an exact run on this silicon. - */ - .ps_base = 0x28e08c000ULL, - .qual = ANE_RECOGNIZED, -}; - static const struct of_device_id ane_of_match[] = { { .compatible = "apple,t8103-ane", .data = &ane_soc_t8103 }, { .compatible = "apple,t6000-ane", .data = &ane_soc_t6000 }, - { .compatible = "apple,t6020-ane", .data = &ane_soc_t6020 }, - { .compatible = "apple,t6021-ane", .data = &ane_soc_t6021 }, {} }; @@ -979,32 +928,11 @@ MODULE_DEVICE_TABLE(of, ane_of_match); static int ane_platform_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - const struct of_device_id *id = of_match_device(ane_of_match, dev); - const struct ane_soc *soc = id->data; + const struct ane_soc *soc = of_device_get_match_data(dev); struct ane_device *ane; struct drm_device *drm; int err; - /* Tier gate before any power-domain, MMIO or IRQ interaction: an - * unqualified SoC must fail cleanly, never half-probe. - */ - if (soc->qual == ANE_UNSUPPORTED) { - dev_err(dev, - "%s: unsupported ANE: no proven SET-block base (a guessed base external-aborts the SoC); not binding. To advance this port, run the mlx-omarchy quick collector (scripts/collect_quick.py: captures ANE/DART/PMGR/AIC device-tree data, no driver needed) and submit the capture\n", - id->compatible); - return -ENODEV; - } - if (soc->qual == ANE_RECOGNIZED && !allow_unqualified) { - dev_err(dev, - "%s: recognized but unqualified: constants present, execution never proven on this silicon; not binding. Override with ane.allow_unqualified=1, or prove a run and report it\n", - id->compatible); - return -ENODEV; - } - if (soc->qual == ANE_RECOGNIZED) - dev_warn(dev, - "%s: UNQUALIFIED bind forced by allow_unqualified: no execution proven on this silicon — SET-block base and board topology unverified\n", - id->compatible); - ane = devm_drm_dev_alloc(dev, &ane_drm_driver, struct ane_device, drm); if (IS_ERR(ane)) return PTR_ERR(ane); @@ -1172,7 +1100,7 @@ static int __maybe_unused ane_runtime_resume(struct device *dev) * SET window reads come first (pmgr class, always safe * per T6001/T8103 bisect evidence), then the engine. A * hard reset after the last off-box line names the - * killing access exactly (T6021 console bring-up). + * killing access exactly. */ struct resource *eng = platform_get_resource_byname( to_platform_device(dev), IORESOURCE_MEM, "engine"); @@ -1198,8 +1126,8 @@ static int __maybe_unused ane_runtime_resume(struct device *dev) /* Linux-side pwrstate probe: ACTUAL nibbles read through * the SoC descriptor's SET window while the partition is * raised. 0xffffff means the mapped window is the live - * pmgr SET block with every word on; anything else names - * the t6021 word layout to fix before promotion. + * pmgr SET block with every word on; anything else means + * the window does not match the expected word layout. */ dev_info(dev, "ANERD ps probe act=%#x\n", ane_ps_act(ane)); } From 62bbd70dd50ca7e14bc1b97098bdb49cf2118ded Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:29:44 -0500 Subject: [PATCH 0590/2680] accel/ane: Clean up the UAPI and add DRM_IOCTL_ANE_GET_CAPS Prepare include/uapi/drm/ane_accel.h for a stable in-tree ABI. Prefix every macro of the header with DRM_ANE_, as other DRM UAPI headers do: ANE_TILE_COUNT, ANE_FIFO_NID and ANE_CMD_GRAN become DRM_ANE_TILE_COUNT, DRM_ANE_FIFO_NID and DRM_ANE_CMD_GRAN, ANE_ABI_MAJOR and ANE_ABI_M2_MAJOR become DRM_ANE_ABI_V1 and DRM_ANE_ABI_V2, and ANE_M2_MAX_BINDS becomes DRM_ANE_MAX_BINDS. ANE_M2_MAX_SECTIONS was 16, but the driver accepts at most 9 sections (ids 1 to 9). It becomes DRM_ANE_MAX_SECTIONS with the value 9, and a BUILD_BUG_ON() keeps it equal to the firmware LOAD record count. Add DRM_IOCTL_ANE_GET_CAPS (number 0x00, free until now) to both drivers. It reports the ABI version, the chip family (H13 or H14), and the size of each array entry that PROG_LOAD and EXEC read through a user pointer. Its flags field has no defined flag and must be zero. Its size field gives the number of bytes that the driver filled in, so the structure can grow at its end. Reject a nonzero pad or reserved field with -EINVAL in every ioctl: BO_INIT and BO_FREE on ABI version 2 (ABI version 1 already did), the pad of PROG_LOAD, of each generic bind and of EXEC, and the flags and reserved fields of each EXEC buffer. The field drm_ane_exec_io.dma was already reserved: the driver gives the firmware the address of bo_handle. Rename it to reserved. Add kernel-doc for every structure. Keep one header with an explicit ABI version, not one header per ABI. The two drivers share the DRM driver name, the accel node, the BO_INIT/BO_FREE structures and the ioctl number space. One header per ABI would either duplicate the shared structures under two names or need a third header for them, and it would let the two headers give one ioctl number to two structures. Userspace already supports both ABI versions from one binary and selects by the version major. The version is explicit: DRM_IOCTL_VERSION major and drm_ane_get_caps.abi_version both report DRM_ANE_ABI_V1 or DRM_ANE_ABI_V2, an ioctl number is never reused, and a driver rejects the ioctls of the other version. The ioctl numbers and the layout of every existing structure do not change: an aarch64 offsetof()/sizeof() check of the old and new header gives identical values for all existing structures, fields and ioctl command values. Userspace that left a pad field of BO_FREE or of a generic bind uninitialised can now get -EINVAL on ABI version 2. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/accel/ane/ane.h | 2 +- drivers/accel/ane/ane_drv.c | 37 +++- drivers/accel/ane/ane_t6021_rtclient_main.c | 48 ++++- drivers/accel/ane/ane_tm.c | 3 +- include/uapi/drm/ane_accel.h | 207 ++++++++++++++++---- 5 files changed, 238 insertions(+), 59 deletions(-) diff --git a/drivers/accel/ane/ane.h b/drivers/accel/ane/ane.h index 10fee2864728c..c644ec45e67d4 100644 --- a/drivers/accel/ane/ane.h +++ b/drivers/accel/ane/ane.h @@ -115,7 +115,7 @@ struct ane_request { u32 td_size; u32 td_count; u32 btsp_iova; - u32 bar[ANE_TILE_COUNT]; + u32 bar[DRM_ANE_TILE_COUNT]; }; /* diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 5b269b0f8729f..8b38416c20b36 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -352,6 +352,22 @@ static const struct drm_gem_object_funcs ane_gem_object_funcs = { .vm_ops = &drm_gem_ane_vm_ops, }; +static int ane_get_caps(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct drm_ane_get_caps *args = data; + + if (args->flags || args->pad) + return -EINVAL; + + *args = (struct drm_ane_get_caps) { + .size = sizeof(*args), + .abi_version = DRM_ANE_ABI_V1, + .chip_family = DRM_ANE_CHIP_H13, + }; + return 0; +} + static int ane_bo_init(struct drm_device *drm, void *data, struct drm_file *file) { @@ -460,7 +476,7 @@ static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) { struct ane_device *ane = drm->dev_private; struct drm_ane_submit *args = data; - struct drm_gem_object *gem[ANE_TILE_COUNT] = { 0 }; + struct drm_gem_object *gem[DRM_ANE_TILE_COUNT] = { 0 }; struct drm_gem_object *btsp = NULL; struct ane_bo *bo; struct ane_request req; @@ -483,11 +499,11 @@ static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) } req.qid = 4; - req.nid = ANE_FIFO_NID; + req.nid = DRM_ANE_FIFO_NID; req.td_size = args->td_size; req.td_count = args->td_count; - for (int bdx = 0; bdx < ANE_TILE_COUNT; bdx++) { + for (int bdx = 0; bdx < DRM_ANE_TILE_COUNT; bdx++) { if (!args->handles[bdx]) continue; bo = bo_lookup(file, args->handles[bdx]); @@ -510,8 +526,8 @@ static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) * access. Since this isn't page aligned, we represent the two as one * buffer and calculate the delimiter (where the weights would start). */ - req.bar[KRN_BUF_BDX] = - req.bar[CMD_BUF_BDX] + round_up(args->tsk_size, ANE_CMD_GRAN); + req.bar[KRN_BUF_BDX] = req.bar[CMD_BUF_BDX] + + round_up(args->tsk_size, DRM_ANE_CMD_GRAN); bo = bo_lookup(file, args->btsp_handle); if (!bo) { @@ -553,7 +569,7 @@ static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) mutex_unlock(&ane->engine_lock); put: drm_gem_object_put(btsp); - for (int bdx = 0; bdx < ANE_TILE_COUNT; bdx++) { + for (int bdx = 0; bdx < DRM_ANE_TILE_COUNT; bdx++) { if (gem[bdx]) drm_gem_object_put(gem[bdx]); } @@ -561,6 +577,7 @@ static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) } static const struct drm_ioctl_desc ane_drm_ioctls[] = { + DRM_IOCTL_DEF_DRV(ANE_GET_CAPS, ane_get_caps, 0), DRM_IOCTL_DEF_DRV(ANE_BO_INIT, ane_bo_init, 0), DRM_IOCTL_DEF_DRV(ANE_BO_FREE, ane_bo_free, 0), DRM_IOCTL_DEF_DRV(ANE_SUBMIT, ane_submit, 0), @@ -656,9 +673,9 @@ static void ane_drm_postclose(struct drm_device *drm, struct drm_file *file) } /* - * The driver only owns three ioctls (ANE_BO_INIT, ANE_BO_FREE, - * ANE_SUBMIT). Anything outside that table would otherwise fall through - * the standard drm_ioctl path to legacy primary-node handlers that + * The driver only owns four ioctls (ANE_GET_CAPS, ANE_BO_INIT, + * ANE_BO_FREE, ANE_SUBMIT). The standard drm_ioctl path would otherwise + * pass anything outside that table to legacy primary-node handlers that * dereference drm_device.unique, which an accel node never sets. Reject * such callers explicitly so they fail with -ENOTTY instead of oopsing. * @@ -764,7 +781,7 @@ static const struct drm_driver ane_drm_driver = { .ioctls = ane_drm_ioctls, .num_ioctls = ARRAY_SIZE(ane_drm_ioctls), .fops = &ane_drm_fops, - .major = ANE_ABI_MAJOR, + .major = DRM_ANE_ABI_V1, .name = "ane", .desc = "Apple Neural Engine driver", }; diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index 9d1e43ecc508b..e5cda3d046ed6 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -990,9 +990,10 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, size_t binds_size; int ret, i, j; - if (user->section_count < 1 || + BUILD_BUG_ON(DRM_ANE_MAX_SECTIONS != ANE_T6021_LOAD_SEC_COUNT); + if (user->pad || user->section_count < 1 || user->section_count > ANE_T6021_LOAD_SEC_COUNT || - user->generic_count > ANE_M2_MAX_BINDS || !fd) + user->generic_count > DRM_ANE_MAX_BINDS || !fd) return -EINVAL; sections = kmalloc_array(user->section_count, sizeof(*sections), @@ -1016,6 +1017,12 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, ret = -EFAULT; goto out; } + for (i = 0; i < user->generic_count; i++) { + if (binds[i].pad) { + ret = -EINVAL; + goto out; + } + } /* Lab contract: the record slot IS the section identity (slot = * id - 1; tdprop id 7 lands at slot 6 and slot 5 stays empty, @@ -1248,7 +1255,7 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, size_t cmd_size; int ret, i; - if (user->count < 1 || user->count > ANE_M2_MAX_BINDS || + if (user->pad || user->count < 1 || user->count > DRM_ANE_MAX_BINDS || user->priority < 2 || user->priority > 7 || !fd) return -EINVAL; @@ -1260,6 +1267,12 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, kfree(ios); return -EFAULT; } + for (i = 0; i < user->count; i++) { + if (ios[i].flags || ios[i].reserved) { + kfree(ios); + return -EINVAL; + } + } cmd_size = sizeof(struct ane_csne_cmd_procedure_call) + (size_t)user->count * sizeof(struct ane_csne_io_elem); @@ -1364,7 +1377,8 @@ static int ane_t6021_bo_init_ioctl(struct drm_device *drm, void *data, struct ane_t6021_bo *bo; struct ane_rtclient *ane; - if (args->size == 0 || args->size > ANE_T6021_BO_MAX || !fd) + if (args->pad || args->size == 0 || args->size > ANE_T6021_BO_MAX || + !fd) return -EINVAL; /* A parked io BO is already mapped and counted; its old contents * belong to another process, so it is zeroed like a new one. @@ -1447,6 +1461,8 @@ static int ane_t6021_bo_free_ioctl(struct drm_device *drm, void *data, if (!fd) return -ENODEV; + if (args->pad) + return -EINVAL; mutex_lock(&ane_t6021_bo_lock); list_for_each_entry(b, &fd->bos, node) { if (b->handle == args->handle) { @@ -1540,6 +1556,25 @@ static void ane_t6021_postclose(struct drm_device *drm, struct drm_file *file) file->driver_priv = NULL; } +static int ane_t6021_get_caps_ioctl(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct drm_ane_get_caps *args = data; + + if (args->flags || args->pad) + return -EINVAL; + + *args = (struct drm_ane_get_caps) { + .size = sizeof(*args), + .abi_version = DRM_ANE_ABI_V2, + .chip_family = DRM_ANE_CHIP_H14, + .section_size = sizeof(struct drm_ane_section), + .bind_size = sizeof(struct drm_ane_generic_bind), + .exec_io_size = sizeof(struct drm_ane_exec_io), + }; + return 0; +} + static int ane_t6021_submit_ioctl(struct drm_device *drm, void *data, struct drm_file *file) { @@ -1650,6 +1685,7 @@ static int ane_t6021_exec_ioctl(struct drm_device *drm, void *data, } static const struct drm_ioctl_desc ane_t6021_ioctls[] = { + DRM_IOCTL_DEF_DRV(ANE_GET_CAPS, ane_t6021_get_caps_ioctl, 0), DRM_IOCTL_DEF_DRV(ANE_BO_INIT, ane_t6021_bo_init_ioctl, 0), DRM_IOCTL_DEF_DRV(ANE_BO_FREE, ane_t6021_bo_free_ioctl, 0), DRM_IOCTL_DEF_DRV(ANE_SUBMIT, ane_t6021_submit_ioctl, 0), @@ -1685,7 +1721,7 @@ static const struct drm_driver ane_t6021_drm_driver = { .ioctls = ane_t6021_ioctls, .num_ioctls = ARRAY_SIZE(ane_t6021_ioctls), .fops = &ane_t6021_fops, - .major = ANE_ABI_M2_MAJOR, + .major = DRM_ANE_ABI_V2, .minor = 0, .name = "ane", .desc = "Apple Neural Engine (T6021/M2)", @@ -2289,7 +2325,7 @@ static int ane_rtclient_probe(struct platform_device *pdev) } dev_info(dev, "loaded ane_t6021 (DRM major %d minor %d; ABI 2; legacy_only=%u chman_ok=%u booted=%u; state %s; BO cap %u MiB)\n", - ANE_ABI_M2_MAJOR, 0, + DRM_ANE_ABI_V2, 0, legacy_only, ane->chman_ok, ane->fw ? ane->fw->booted : 0, ane->held ? "HELD" : "ready", bo_total_max_mb); diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index 0e98c48d2aff1..e81af0a7b96db 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -195,9 +195,8 @@ int ane_tm_enqueue(struct ane_device *ane, struct ane_request *req) tq_write32(ane, TQ_STATUS(qid), 0x1); - for (int bdx = 0; bdx < ANE_TILE_COUNT; bdx++) { + for (int bdx = 0; bdx < DRM_ANE_TILE_COUNT; bdx++) tq_write32(ane, TQ_BAR1(qid, bdx), req->bar[bdx]); - } tq_write32(ane, TQ_SIZE1(qid), ((req->td_size >> 2) - 1) << 0x10); tq_write32(ane, TQ_ADDR1(qid), req->btsp_iova); diff --git a/include/uapi/drm/ane_accel.h b/include/uapi/drm/ane_accel.h index e5fd9d12d1d63..76577277b70eb 100644 --- a/include/uapi/drm/ane_accel.h +++ b/include/uapi/drm/ane_accel.h @@ -1,8 +1,8 @@ /* SPDX-License-Identifier: (GPL-2.0-only WITH Linux-syscall-note) OR MIT */ /* Copyright 2022 Eileen Yoon */ -#ifndef __ANE_ACCEL_H__ -#define __ANE_ACCEL_H__ +#ifndef __DRM_UAPI_ANE_ACCEL_H__ +#define __DRM_UAPI_ANE_ACCEL_H__ #include "drm.h" @@ -10,31 +10,85 @@ extern "C" { #endif -#define ANE_TILE_COUNT 0x20 -#define ANE_FIFO_NID 0x40 -#define ANE_CMD_GRAN 0x10 +/** + * DOC: ABI versions + * + * Two drivers implement this interface on an accel node, both with the DRM + * driver name "ane": + * + * - ane (H13 family: T8103, T6000, T6001, T6002) implements ABI version + * %DRM_ANE_ABI_V1: GET_CAPS, BO_INIT, BO_FREE and SUBMIT. + * - ane_t6021 (H14 family: T6020, T6021, T6022, T8112) implements ABI + * version %DRM_ANE_ABI_V2: GET_CAPS, BO_INIT, BO_FREE, PROG_LOAD, + * PROC_CREATE and EXEC. + * + * The DRM_IOCTL_VERSION major and &drm_ane_get_caps.abi_version both give + * the ABI version. The two versions share the ioctl numbers and the + * structures, and a number is never given to a second ioctl. A driver + * rejects the ioctls of the other version. A structure grows only at its + * end; &drm_ane_get_caps.size tells which fields the driver filled in. + * + * Every pad and reserved field, and every flags field with no flags + * defined, must be zero. The drivers reject a nonzero value with -EINVAL. + */ +#define DRM_ANE_ABI_V1 1 +#define DRM_ANE_ABI_V2 2 -/* ABI 1: successful SUBMIT guarantees terminal completion and CPU visibility. */ -#define ANE_ABI_MAJOR 1 +#define DRM_ANE_CHIP_H13 13 +#define DRM_ANE_CHIP_H14 14 -/* - * ABI 2 (T6021 / M2, H14 firmware programs): the driver reports - * DRM_IOCTL_VERSION major 2 and rejects DRM_ANE_SUBMIT. The M1 libane - * requires major 1, so it cannot submit M1 frames to an M2 device. - * A successful DRM_ANE_EXEC guarantees terminal completion (firmware ack - * and task-queue idle) and CPU visibility of the output buffers. - */ -#define ANE_ABI_M2_MAJOR 2 -#define ANE_M2_MAX_SECTIONS 16 -#define ANE_M2_MAX_BINDS 64 +/* ABI version 1: entries of &drm_ane_submit.handles */ +#define DRM_ANE_TILE_COUNT 0x20 +/* ABI version 1: network id of the queue that SUBMIT uses */ +#define DRM_ANE_FIFO_NID 0x40 +/* ABI version 1: the weights start at round_up(tsk_size, DRM_ANE_CMD_GRAN) */ +#define DRM_ANE_CMD_GRAN 0x10 + +/* ABI version 2: highest section id, and most sections per PROG_LOAD */ +#define DRM_ANE_MAX_SECTIONS 9 +/* ABI version 2: most generic binds per PROG_LOAD and buffers per EXEC */ +#define DRM_ANE_MAX_BINDS 64 -#define DRM_ANE_BO_INIT 0x1 -#define DRM_ANE_BO_FREE 0x2 -#define DRM_ANE_SUBMIT 0x3 -#define DRM_ANE_PROG_LOAD 0x4 -#define DRM_ANE_PROC_CREATE 0x5 -#define DRM_ANE_EXEC 0x6 +#define DRM_ANE_GET_CAPS 0x00 +#define DRM_ANE_BO_INIT 0x01 +#define DRM_ANE_BO_FREE 0x02 +#define DRM_ANE_SUBMIT 0x03 +#define DRM_ANE_PROG_LOAD 0x04 +#define DRM_ANE_PROC_CREATE 0x05 +#define DRM_ANE_EXEC 0x06 + +/** + * struct drm_ane_get_caps - DRM_IOCTL_ANE_GET_CAPS argument + * @flags: In. No flags are defined: must be zero. + * @size: Out. Bytes of this structure that the driver filled in. + * @abi_version: Out. %DRM_ANE_ABI_V1 or %DRM_ANE_ABI_V2. + * @chip_family: Out. %DRM_ANE_CHIP_H13 or %DRM_ANE_CHIP_H14. + * @section_size: Out. Size of the &drm_ane_section entries that PROG_LOAD + * reads, or 0 if the driver has no PROG_LOAD. + * @bind_size: Out. Size of the &drm_ane_generic_bind entries that PROG_LOAD + * reads, or 0 if the driver has no PROG_LOAD. + * @exec_io_size: Out. Size of the &drm_ane_exec_io entries that EXEC reads, + * or 0 if the driver has no EXEC. + * @pad: Must be zero. + */ +struct drm_ane_get_caps { + __u32 flags; + __u32 size; + __u32 abi_version; + __u32 chip_family; + __u32 section_size; + __u32 bind_size; + __u32 exec_io_size; + __u32 pad; +}; +/** + * struct drm_ane_bo_init - DRM_IOCTL_ANE_BO_INIT argument + * @handle: Out. Handle of the new buffer. + * @pad: Must be zero. + * @size: In. Size of the buffer in bytes, not zero. + * @offset: Out. Offset to give to mmap() on the accel node. + */ struct drm_ane_bo_init { __u32 handle; __u32 pad; @@ -42,29 +96,65 @@ struct drm_ane_bo_init { __u64 offset; }; +/** + * struct drm_ane_bo_free - DRM_IOCTL_ANE_BO_FREE argument + * @handle: In. Handle from BO_INIT. + * @pad: Must be zero. + */ struct drm_ane_bo_free { __u32 handle; __u32 pad; }; +/** + * struct drm_ane_submit - DRM_IOCTL_ANE_SUBMIT argument (ABI version 1) + * @tsk_size: In. Size of the task microcode at the start of handles[0], not + * zero. + * @td_count: In. Number of task descriptors, 1 to 0xffff. + * @td_size: In. Size of a task descriptor: a multiple of 4, 4 to 0x40000. + * @handles: In. Buffer per tile: handles[0] holds the microcode and the + * weights, handles[1] must be 0 (the driver sets it from handles[0]), + * and 0 marks an unused tile. + * @btsp_handle: In. Buffer with the bootstrap task descriptor. + * @pad: Must be zero. + * + * A successful SUBMIT returns after the engine completed the work, with the + * output buffers visible to the CPU. + */ struct drm_ane_submit { __u64 tsk_size; __u32 td_count; __u32 td_size; - __u32 handles[ANE_TILE_COUNT]; + __u32 handles[DRM_ANE_TILE_COUNT]; __u32 btsp_handle; __u32 pad; }; -/* One firmware program section, already built by userspace, held in a BO. */ +/** + * struct drm_ane_section - one firmware program section (ABI version 2) + * @id: Section id, 1 to %DRM_ANE_MAX_SECTIONS, unique in one PROG_LOAD. + * @bo_handle: Buffer that holds the section bytes. + * @size: Size of the section in bytes. + * @offset: Offset of the section in the buffer. + */ struct drm_ane_section { - __u32 id; /* firmware section id */ - __u32 bo_handle; /* BO holding the bytes */ + __u32 id; + __u32 bo_handle; __u64 size; - __u64 offset; /* byte offset inside the BO */ + __u64 offset; }; -/* A buffer whose IOVA and size the driver patches into a generic entry. */ +/** + * struct drm_ane_generic_bind - buffer of a program (ABI version 2) + * @buffer_id: Firmware buffer id. + * @bo_handle: Buffer handle. + * @type: Buffer type. + * @pad: Must be zero. + * @size: Size of the buffer in bytes. + * + * The driver checks @pad and ignores the other fields: the sections that + * userspace builds already hold this information. + */ struct drm_ane_generic_bind { __u32 buffer_id; __u32 bo_handle; @@ -73,49 +163,86 @@ struct drm_ane_generic_bind { __u64 size; }; +/** + * struct drm_ane_prog_load - DRM_IOCTL_ANE_PROG_LOAD argument (ABI version 2) + * @sections_ptr: In. Pointer to struct drm_ane_section[@section_count]. + * @generic_ptr: In. Pointer to struct drm_ane_generic_bind[@generic_count]. + * @section_count: In. 1 to %DRM_ANE_MAX_SECTIONS. + * @generic_count: In. 0 to %DRM_ANE_MAX_BINDS. + * @prog_id_out: Out. Firmware program id. Identical sections give the same + * program id. + * @pad: Must be zero. + */ struct drm_ane_prog_load { - __u64 sections_ptr; /* struct drm_ane_section[section_count] */ - __u64 generic_ptr; /* struct drm_ane_generic_bind[generic_count] */ + __u64 sections_ptr; + __u64 generic_ptr; __u32 section_count; __u32 generic_count; __u32 prog_id_out; __u32 pad; }; +/** + * struct drm_ane_proc_create - DRM_IOCTL_ANE_PROC_CREATE argument + * (ABI version 2) + * @prog_id: In. Program id from PROG_LOAD. + * @proc_id_out: Out. Firmware process id. + */ struct drm_ane_proc_create { __u32 prog_id; __u32 proc_id_out; }; -/* Per-call buffer binding: an input or output for one procedure call. */ +/** + * struct drm_ane_exec_io - input or output buffer of one EXEC + * (ABI version 2) + * @buffer_id: Firmware buffer id. + * @bo_handle: Buffer handle. The driver gives its address to the firmware. + * @type: Buffer type. + * @flags: No flags are defined: must be zero. + * @reserved: Must be zero. + * @size: Size of the buffer in bytes. + */ struct drm_ane_exec_io { __u32 buffer_id; __u32 bo_handle; __u32 type; __u32 flags; - __u64 dma; /* in only, reserved: the driver binds the - * IOVA from bo_handle - */ + __u64 reserved; __u64 size; }; +/** + * struct drm_ane_exec - DRM_IOCTL_ANE_EXEC argument (ABI version 2) + * @prog_id: In. Program id from PROG_LOAD. + * @proc_id: In. Process id from PROC_CREATE. + * @priority: In. 2 to 7. + * @timeout_ms: In. Firmware completion timeout, 0 for 5000 ms. + * @count: In. 1 to %DRM_ANE_MAX_BINDS. + * @pad: Must be zero. + * @io_ptr: In. Pointer to struct drm_ane_exec_io[@count]. + * + * A successful EXEC returns after the firmware acknowledged completion and + * the task queue is idle, with the output buffers visible to the CPU. + */ struct drm_ane_exec { __u32 prog_id; __u32 proc_id; - __u32 priority; /* 2..7 */ + __u32 priority; __u32 timeout_ms; - __u32 count; /* 1..ANE_M2_MAX_BINDS */ + __u32 count; __u32 pad; - __u64 io_ptr; /* struct drm_ane_exec_io[count] */ + __u64 io_ptr; }; +#define DRM_IOCTL_ANE_GET_CAPS \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_GET_CAPS, struct drm_ane_get_caps) #define DRM_IOCTL_ANE_BO_INIT \ DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_BO_INIT, struct drm_ane_bo_init) #define DRM_IOCTL_ANE_BO_FREE \ DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_BO_FREE, struct drm_ane_bo_free) #define DRM_IOCTL_ANE_SUBMIT \ DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_SUBMIT, struct drm_ane_submit) - #define DRM_IOCTL_ANE_PROG_LOAD \ DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_PROG_LOAD, struct drm_ane_prog_load) #define DRM_IOCTL_ANE_PROC_CREATE \ @@ -127,4 +254,4 @@ struct drm_ane_exec { } #endif -#endif /* __ANE_ACCEL_H__ */ +#endif /* __DRM_UAPI_ANE_ACCEL_H__ */ From 4907f471b9b1ab969a2d3f8e970dec0c655cfd8a Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:34:03 -0500 Subject: [PATCH 0591/2680] accel/ane: Parenthesize the register macro arguments checkpatch --strict reports 27 MACRO_ARG_PRECEDENCE checks for the task manager and task queue register macros. Put each macro argument in parentheses. No functional change: all callers pass simple expressions, and the non-debug sections of all seven objects are byte-identical before and after this change (arm64, gcc 12.2.0). Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/accel/ane/ane_tm.c | 44 ++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index e81af0a7b96db..7898c1dd93aaa 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -19,11 +19,11 @@ static const int TQ_PRTY_TABLE[ANE_TQ_COUNT] = { 0x1, 0x2, 0x3, 0x4, #define TM_PUSH 0x8 #define TM_TQ_EN 0xc -#define TM_IRQ_EVTC(line) (0x14 + (line * 0x14)) -#define TM_IRQ_INFO(line) (0x18 + (line * 0x14)) -#define TM_IRQ_UNK1(line) (0x1c + (line * 0x14)) -#define TM_IRQ_TMST(line) (0x20 + (line * 0x14)) -#define TM_IRQ_UNK2(line) (0x24 + (line * 0x14)) +#define TM_IRQ_EVTC(line) (0x14 + ((line) * 0x14)) +#define TM_IRQ_INFO(line) (0x18 + ((line) * 0x14)) +#define TM_IRQ_UNK1(line) (0x1c + ((line) * 0x14)) +#define TM_IRQ_TMST(line) (0x20 + ((line) * 0x14)) +#define TM_IRQ_UNK2(line) (0x24 + ((line) * 0x14)) #define TM_COMMITTED 0x44 #define TM_STATUS 0x54 @@ -34,28 +34,30 @@ static const int TQ_PRTY_TABLE[ANE_TQ_COUNT] = { 0x1, 0x2, 0x3, 0x4, #define TM_IRQ_ACK 0x6c #define TM_IRQ_EN2 0x70 -#define TQ_STATUS(qid) (0x000 + (qid * 0x148)) -#define TQ_PRTY(qid) (0x010 + (qid * 0x148)) -#define TQ_VACANT(qid) (0x014 + (qid * 0x148)) -#define TQ_INFO(qid) (0x01c + (qid * 0x148)) +#define TQ_STATUS(qid) (0x000 + ((qid) * 0x148)) +#define TQ_PRTY(qid) (0x010 + ((qid) * 0x148)) +#define TQ_VACANT(qid) (0x014 + ((qid) * 0x148)) +#define TQ_INFO(qid) (0x01c + ((qid) * 0x148)) -#define TQ_BAR1(qid, bdx) (0x020 + (qid * 0x148) + (bdx * 0x4)) -#define TQ_NID1(qid) (0x0a0 + (qid * 0x148)) -#define TQ_SIZE2(qid) (0x0a4 + (qid * 0x148)) -#define TQ_ADDR2(qid) (0x0a8 + (qid * 0x148)) +#define TQ_BAR1(qid, bdx) (0x020 + ((qid) * 0x148) + ((bdx) * 0x4)) +#define TQ_NID1(qid) (0x0a0 + ((qid) * 0x148)) +#define TQ_SIZE2(qid) (0x0a4 + ((qid) * 0x148)) +#define TQ_ADDR2(qid) (0x0a8 + ((qid) * 0x148)) -#define TQ_BAR2(qid, bdx) (0x0ac + (qid * 0x148) + (bdx * 0x4)) -#define TQ_NID2(qid) (0x12c + (qid * 0x148)) -#define TQ_SIZE1(qid) (0x130 + (qid * 0x148)) -#define TQ_ADDR1(qid) (0x134 + (qid * 0x148)) +#define TQ_BAR2(qid, bdx) (0x0ac + ((qid) * 0x148) + ((bdx) * 0x4)) +#define TQ_NID2(qid) (0x12c + ((qid) * 0x148)) +#define TQ_SIZE1(qid) (0x130 + ((qid) * 0x148)) +#define TQ_ADDR1(qid) (0x134 + ((qid) * 0x148)) #define TM_IS_IDLE 0x1 #define TM_IS_FINE 0x22222222 -#define tm_read32(ane, off) (readl(ane->engine + ANE_TM_BASE + off)) -#define tq_read32(ane, off) (readl(ane->engine + ANE_TQ_BASE + off)) -#define tm_write32(ane, off, val) (writel(val, ane->engine + ANE_TM_BASE + off)) -#define tq_write32(ane, off, val) (writel(val, ane->engine + ANE_TQ_BASE + off)) +#define tm_read32(ane, off) (readl((ane)->engine + ANE_TM_BASE + (off))) +#define tq_read32(ane, off) (readl((ane)->engine + ANE_TQ_BASE + (off))) +#define tm_write32(ane, off, val) \ + (writel(val, (ane)->engine + ANE_TM_BASE + (off))) +#define tq_write32(ane, off, val) \ + (writel(val, (ane)->engine + ANE_TQ_BASE + (off))) /* The ane SET block (m1n1 ANE.ps_map) maps the pmgr power-state words * for this engine (set0, base, set1..4). pmgr reads are always safe and From 9af2ca256e6a1dd9ae92e3aa78733f1198f7fe9e Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:37:14 -0500 Subject: [PATCH 0592/2680] accel/ane: Comment the memory barriers and the mutexes checkpatch reports a MEMORY_BARRIER warning for the four barriers without a comment, and an UNCOMMENTED_DEFINITION check for the three mutexes. Say what each barrier orders and what each mutex protects. Comments only. The non-debug sections of all seven objects are byte-identical before and after this change, except the line number in the one __dyndbg entry of ane_t6021_rtclient_main.o (arm64, gcc 12.2.0). Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/accel/ane/ane.h | 7 ++++++- drivers/accel/ane/ane_t6021_rtclient_main.c | 3 +++ drivers/accel/ane/ane_tm.c | 8 ++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/drivers/accel/ane/ane.h b/drivers/accel/ane/ane.h index c644ec45e67d4..db30a3e103fff 100644 --- a/drivers/accel/ane/ane.h +++ b/drivers/accel/ane/ane.h @@ -46,7 +46,12 @@ struct ane_device { int dart_count; int irq; + /* Protects mm and the iommu_map()/iommu_unmap() calls on domain. */ struct mutex iommu_lock; + /* + * Serializes engine access (submit, BO mapping, recovery, removal) + * and protects bo_list, preserved_list and removed. + */ struct mutex engine_lock; struct list_head bo_list; @@ -98,7 +103,7 @@ struct ane_device { * run ~2x slower while schedutil parks the idle clusters low. */ struct ane_boost { - struct mutex lock; + struct mutex lock; /* protects the fields below */ struct delayed_work off; struct freq_qos_request *legs; int nlegs; diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index e5cda3d046ed6..abe477879d27d 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -506,6 +506,9 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, if (pending) { writel(pending, ipi + 0xc000); + /* Complete the IPI acknowledgment before the + * next reads of the command and malloc rings. + */ mb(); } header = READ_ONCE(io[0]); diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index 7898c1dd93aaa..9364ae1469b9d 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -267,6 +267,9 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) err = ane_tm_collect_events(ane, NULL, &finished); if (err < 0) goto wedge; + /* Order the CPU writes to the request buffers and the event + * acknowledgments before the queue writes that start the engine. + */ wmb(); ane_tm_push_tq(ane, req); @@ -316,6 +319,10 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) } status = tq_read32(ane, TQ_NID1(req->qid)); tq_write32(ane, TQ_NID1(req->qid), status & ~1U); + /* Kept from the qualified completion sequence (clear the + * NID valid bit, then the queue status). writel() already + * orders the two writes to the engine. + */ rmb(); tq_write32(ane, TQ_STATUS(req->qid), 0x0); ane_dart_release_scratch(ane, scratch, scratch_pages); @@ -333,6 +340,7 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) goto wedge; } tq_write32(ane, TQ_NID1(req->qid), status & ~1U); + /* Same completion sequence as the contained-fault path above. */ rmb(); tq_write32(ane, TQ_STATUS(req->qid), 0x0); ane_dart_unmask(ane); From de0d05e7694adbdec78c8755af4aa2e85405de55 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:38:44 -0500 Subject: [PATCH 0593/2680] accel/ane: Use kzalloc_obj() checkpatch reports seven ALLOC_WITH_SIZEOF warnings and one ALLOC_SIZEOF_STRUCT check. Allocate the single objects with kzalloc_obj(), which uses GFP_KERNEL when no flags are given. No functional change: kzalloc_obj() expands to kzalloc() of the same size and flags, and the non-debug sections of all seven objects are byte-identical before and after this change (arm64, gcc 12.2.0). Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/accel/ane/ane_dart.c | 2 +- drivers/accel/ane/ane_drv.c | 6 +++--- drivers/accel/ane/ane_t6021_rtclient_main.c | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/accel/ane/ane_dart.c b/drivers/accel/ane/ane_dart.c index aea8cd9eff182..f66bfbe15af55 100644 --- a/drivers/accel/ane/ane_dart.c +++ b/drivers/accel/ane/ane_dart.c @@ -206,7 +206,7 @@ int ane_dart_drain_fault(struct ane_device *ane, u64 fault_iova, return -ENOMEM; iova = fault_iova & ~((1ULL << ane->shift) - 1); - node = kzalloc(sizeof(*node), GFP_KERNEL); + node = kzalloc_obj(*node); if (!node) { __free_page(page); return -ENOMEM; diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 8b38416c20b36..054c69c3764d6 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -92,7 +92,7 @@ static int ane_iommu_map_pages(struct ane_device *ane, struct ane_bo *bo) if (bo->mm) return -EBUSY; - bo->mm = kzalloc(sizeof(*bo->mm), GFP_KERNEL); + bo->mm = kzalloc_obj(*bo->mm); if (!bo->mm) return -ENOMEM; @@ -227,7 +227,7 @@ static void ane_preserve_bo(struct ane_device *ane, struct ane_bo *bo) { struct ane_preserved *p; - p = kzalloc(sizeof(*p), GFP_KERNEL); + p = kzalloc_obj(*p); if (!p) { /* Fail closed the old way: the node stays inserted and the * pages leak until reboot - safe, just unreclaimable. @@ -380,7 +380,7 @@ static int ane_bo_init(struct drm_device *drm, void *data, if (args->pad || !args->size) return -EINVAL; - bo = kzalloc(sizeof(struct ane_bo), GFP_KERNEL); + bo = kzalloc_obj(*bo); if (!bo) return -ENOMEM; INIT_LIST_HEAD(&bo->node); diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index abe477879d27d..eb72dbce48814 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -1402,7 +1402,7 @@ static int ane_t6021_bo_init_ioctl(struct drm_device *drm, void *data, return -ENOSPC; } ane = to_ane_t6021_drm(drm)->ane; - bo = kzalloc(sizeof(*bo), GFP_KERNEL); + bo = kzalloc_obj(*bo); if (!bo) { atomic64_sub(PAGE_ALIGN(args->size), &ane_t6021_bo_total_bytes); return -ENOMEM; @@ -1536,7 +1536,7 @@ static int ane_t6021_open(struct drm_device *drm, struct drm_file *file) { struct ane_t6021_fd *fd; - fd = kzalloc(sizeof(*fd), GFP_KERNEL); + fd = kzalloc_obj(*fd); if (!fd) return -ENOMEM; INIT_LIST_HEAD(&fd->bos); @@ -1885,7 +1885,7 @@ static int ane_rtclient_attach_genpd(struct ane_rtclient *ane) if (pd->dev == dev) goto found; - pd = kzalloc(sizeof(*pd), GFP_KERNEL); + pd = kzalloc_obj(*pd); if (!pd) { err = -ENOMEM; goto out; From b4336803b96232d32a38b986b74e227028fb8a6a Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:41:16 -0500 Subject: [PATCH 0594/2680] accel/ane: Fix the remaining coding style issues Fix the style issues that checkpatch --strict reports and that need no change to the generated code: - join the split MODULE_PARM_DESC() strings of ane_t6021_fwload.c (SPLIT_STRING), - wrap three lines longer than 100 columns (LONG_LINE, LONG_LINE_STRING), - move two multi-line trailing comments in ane_t6021_boot.h and one after a #define above the code they describe (BLOCK_COMMENT_STYLE), - use ARRAY_SIZE(), min_t() and BIT() (ARRAY_SIZE, MINMAX, BIT_MACRO), - add braces to the unbalanced if/else statements (BRACES), - remove a pair of unnecessary parentheses (UNNECESSARY_PARENTHESES), - do not end a line with an open parenthesis (OPEN_ENDED_LINE), - reword a comment that checkpatch reads as a fall-through annotation (PREFER_FALLTHROUGH). No functional change: the non-debug sections of all seven objects are byte-identical before and after this change (arm64, gcc 12.2.0). Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/accel/ane/ane_drv.c | 20 +++++++----- drivers/accel/ane/ane_t6021.h | 6 ++-- drivers/accel/ane/ane_t6021_boot.c | 4 +-- drivers/accel/ane/ane_t6021_boot.h | 35 +++++++++------------ drivers/accel/ane/ane_t6021_fwload.c | 18 +++++------ drivers/accel/ane/ane_t6021_rtclient_main.c | 13 ++++---- 6 files changed, 45 insertions(+), 51 deletions(-) diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 054c69c3764d6..738fa5d17b1fa 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -513,8 +513,8 @@ static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) } gem[bdx] = &bo->base; if (!bo->iova || - ((bdx == CMD_BUF_BDX) && - (args->tsk_size >= (bo->npages << ane->shift)))) { + (bdx == CMD_BUF_BDX && + args->tsk_size >= (bo->npages << ane->shift))) { err = -EINVAL; goto put; } @@ -749,10 +749,10 @@ static int ane_drm_mmap(struct file *file, struct vm_area_struct *vma) */ vm_flags_mod(vma, VM_IO | VM_DONTEXPAND | VM_DONTDUMP, VM_PFNMAP); - if (map_mode & 2) + if (map_mode & 2) { vma->vm_page_prot = pgprot_decrypted(vm_get_page_prot(vma->vm_flags)); - else { + } else { vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); @@ -859,8 +859,9 @@ static int ane_attach_genpd(struct ane_device *ane) { struct device *dev = ane->dev; - ane->pd_count = of_count_phandle_with_args( - dev->of_node, "power-domains", "#power-domain-cells"); + ane->pd_count = of_count_phandle_with_args(dev->of_node, + "power-domains", + "#power-domain-cells"); if (ane->pd_count < 1) return ane->pd_count < 0 ? ane->pd_count : -EINVAL; if (ane->pd_count == 1) @@ -1119,8 +1120,11 @@ static int __maybe_unused ane_runtime_resume(struct device *dev) * hard reset after the last off-box line names the * killing access exactly. */ - struct resource *eng = platform_get_resource_byname( - to_platform_device(dev), IORESOURCE_MEM, "engine"); + struct platform_device *pdev = to_platform_device(dev); + struct resource *eng; + + eng = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "engine"); dev_info(dev, "ANE-resume: genpd raise complete; SET window probe next\n"); diff --git a/drivers/accel/ane/ane_t6021.h b/drivers/accel/ane/ane_t6021.h index 5e0ecc6d471f7..03d9ee35cdd96 100644 --- a/drivers/accel/ane/ane_t6021.h +++ b/drivers/accel/ane/ane_t6021.h @@ -529,9 +529,9 @@ int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, * The generic processor takes commands < 0x1b89 bytes (work-item size * bound @0x4d134-0x4d158) and writes completion state back into the * command block (strb @0x4d324 byte +6, str @0x4d3b0 qword +8) — the - * ring slot doubles as the response area. PING/BUILDINFO/BOOT/ - * REG_FILE_LOAD/IPC_ENDPOINT_SET fall through the processor's id tree - * to the default path (0x4e65c): carried without field parsing, so + * ring slot doubles as the response area. The processor's id tree + * sends PING/BUILDINFO/BOOT/REG_FILE_LOAD/IPC_ENDPOINT_SET on to the + * default path (0x4e65c): carried without field parsing, so * their payloads beyond the header are opaque until the W1 live * exchange pins them. */ diff --git a/drivers/accel/ane/ane_t6021_boot.c b/drivers/accel/ane/ane_t6021_boot.c index 066d2a799851a..de11bb8f095fa 100644 --- a/drivers/accel/ane/ane_t6021_boot.c +++ b/drivers/accel/ane/ane_t6021_boot.c @@ -549,8 +549,8 @@ int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, dev_info(ane->dev, "boot: run returned 0 (booted=%u fw_alive=%u) — %s; scratch_result=%016llx (raw device address, semantics UNSOURCED — not a success claim; transport stays fenced until response validation)\n", bo, fa, - bo ? "DONE observed — handshake complete" - : "rc 0 WITHOUT DONE (rtb_mode: HELLO-gated; DONE not part of this mode)", + bo ? "DONE observed — handshake complete" : + "rc 0 WITHOUT DONE (rtb_mode: HELLO-gated; DONE not part of this mode)", sres); return r; } diff --git a/drivers/accel/ane/ane_t6021_boot.h b/drivers/accel/ane/ane_t6021_boot.h index ced0c997439dd..aba267e453f13 100644 --- a/drivers/accel/ane/ane_t6021_boot.h +++ b/drivers/accel/ane/ane_t6021_boot.h @@ -69,14 +69,11 @@ #define ANE_T6021_INIT_TEMPLATE_OFF 0x6c /* 256 B at [0x6C,0x16C) */ #define ANE_T6021_INIT_TEMPLATE_SIZE 0x100 #define ANE_T6021_INIT_TBIT_OFF 0xc0 /* template-relative */ -#define ANE_T6021_INIT_TBIT_VAL 0x4 /* RESOLVED (Main raw - * anchors 0x9612b78/7c/80: - * ldr/orr#4/str on the dev+0x998 - * template); the conditional - * |=0x10 is a separate, still - * unqualified mutation and is - * NOT set - */ +/* RESOLVED (Main raw anchors 0x9612b78/7c/80: ldr/orr#4/str on the + * dev+0x998 template); the conditional |=0x10 is a separate, still + * unqualified mutation and is NOT set. + */ +#define ANE_T6021_INIT_TBIT_VAL 0x4 static inline u64 ane_t6021_rvbar_compose(u64 iova) { @@ -451,18 +448,14 @@ struct ane_t6021_boot_io { u64 (*rd64)(void *ctx, unsigned int off); void (*wr32)(void *ctx, unsigned int off, u32 v); void (*wr64)(void *ctx, unsigned int off, u64 v); - void (*publish_barrier)(void *ctx); /* orders the coherent fill - * before the device publish: - * dma_wmb() = dmb oshst on - * arm64; NOT claimed as dsb st - * (completion) - */ - void (*phase)(void *ctx, const char *what); /* bounded phase log: - * emitted ONCE before each - * write/poll block so a crash - * source survives netconsole - * (never per-poll) - */ + /* Orders the coherent fill before the device publish: dma_wmb() = + * dmb oshst on arm64; NOT claimed as dsb st (completion). + */ + void (*publish_barrier)(void *ctx); + /* Bounded phase log: emitted ONCE before each write/poll block so a + * crash source survives netconsole (never per-poll). + */ + void (*phase)(void *ctx, const char *what); void (*poll_wait)(void *ctx); /* 1 ms poll delay */ /* prepare(): called once, strictly AFTER poll A and BEFORE the * SCRATCH0/1 publish; returns the suballoc DVA halves. Kernel @@ -603,7 +596,7 @@ ane_t6021_boot_run(const struct ane_t6021_boot_io *io, unsigned int ti; io->phase(io->ctx, "P-1 grant-tunables begin"); - for (ti = 0; ti < sizeof(tun) / sizeof(tun[0]); ti++) { + for (ti = 0; ti < ARRAY_SIZE(tun); ti++) { io->phase(io->ctx, tun[ti].off == 0x000 ? "P-1a eng+0x000" : tun[ti].off == 0x038 ? "P-1b eng+0x038" : diff --git a/drivers/accel/ane/ane_t6021_fwload.c b/drivers/accel/ane/ane_t6021_fwload.c index 1e44f0355b485..d677962a828b7 100644 --- a/drivers/accel/ane/ane_t6021_fwload.c +++ b/drivers/accel/ane/ane_t6021_fwload.c @@ -93,14 +93,12 @@ MODULE_DESCRIPTION("T6021 ANE firmware staging + entry alias"); static bool fw_load = true; module_param(fw_load, bool, 0444); MODULE_PARM_DESC(fw_load, - "Validate + DART-map the 13.5 PRELOAD payload (default on: " - "the proven add-path configuration)."); + "Validate + DART-map the 13.5 PRELOAD payload (default on: the proven add-path configuration)."); static unsigned int fw_extra_ram = 0x200000; module_param(fw_extra_ram, uint, 0444); MODULE_PARM_DESC(fw_extra_ram, - "Page-aligned owned RAM after the 5 MiB firmware allocation " - "(default 0x200000, the proven add-path grant; maximum 16 MiB)."); + "Page-aligned owned RAM after the 5 MiB firmware allocation (default 0x200000, the proven add-path grant; maximum 16 MiB)."); /* Preloaded-placement alias, lab only: map the iBoot-reserved SEG0/SEGi * phys at the entry IOVAs (the preload, with iBoot's patches in place). @@ -113,11 +111,7 @@ MODULE_PARM_DESC(fw_extra_ram, static bool fw_alias_reserved; module_param(fw_alias_reserved, bool, 0444); MODULE_PARM_DESC(fw_alias_reserved, - "Lab, T6021: map reserved SEG0 0x10000848000+0xc4000 at entry and " - "SEG1 0x10001400000+0x438000 after it (probe refuses unless no-map " - "/reserved-memory nodes cover both); 0 (default) = run the staged " - "copy with iBoot's patches replayed (own memory, the only mode on " - "T6020/T6022)."); + "Lab, T6021: map reserved SEG0 0x10000848000+0xc4000 at entry and SEG1 0x10001400000+0x438000 after it (probe refuses unless no-map /reserved-memory nodes cover both); 0 (default) = run the staged copy with iBoot's patches replayed (own memory, the only mode on T6020/T6022)."); static bool ane_t6021_fw_alias_is_reserved(struct device *dev) { @@ -314,7 +308,8 @@ static int ane_t6021_fw_alias_map(struct ane_t6021 *ane, bool reserved) * DATA section unmapped past the SEG0 head. */ struct { u64 iova, phys, len; } win[] = { - { 0x10000000000ull, ane_t6021_fw_preload[0].phys, ane_t6021_fw_preload[0].len }, + { 0x10000000000ull, ane_t6021_fw_preload[0].phys, + ane_t6021_fw_preload[0].len }, { 0, ane_t6021_fw_preload[1].phys, ane_t6021_fw_preload[1].len }, { entry + 0x4fc000, 0, fw_extra_ram ? ane->fw_size - 0x4fc000 : 0 }, }; @@ -340,7 +335,8 @@ static int ane_t6021_fw_alias_map(struct ane_t6021 *ane, bool reserved) for (o = 0; o < win[w].len; o += ANE_T6021_FW_ALIAS_PAGE) { phys_addr_t pa = w < 2 ? win[w].phys + o : - iommu_iova_to_phys(dom, ane->fw_iova + win[w].iova + o - entry); + iommu_iova_to_phys(dom, ane->fw_iova + + win[w].iova + o - entry); if (!pa || !IS_ALIGNED(pa, ANE_T6021_FW_ALIAS_PAGE)) { ret = -EFAULT; goto err_unmap_mapped; diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index eb72dbce48814..676029a9e7b27 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -620,7 +620,7 @@ MODULE_PARM_DESC(call_settle_us, * trace state is protected by ane_t6021_fw_lock. */ #define ANE_TRACE_MAGIC 0x31445441 /* "ATD1" */ -#define ANE_TRACE_RECS (1U << 18) +#define ANE_TRACE_RECS BIT(18) #define ANE_PMGR_PS_LAST_OFF 0x30 enum { @@ -1085,7 +1085,8 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, sha256_update(&sha, (const u8 *)hdr, sizeof(hdr)); left = sections[i].size; while (left) { - size_t n = min(left, (size_t)ANE_T6021_BO_HASH_CHUNK); + size_t n = min_t(size_t, left, + ANE_T6021_BO_HASH_CHUNK); memcpy(scratch, (const u8 *)bo->cpu + sections[i].offset + @@ -2238,9 +2239,9 @@ static int ane_rtclient_probe(struct platform_device *pdev) READ_ONCE(((u32 *)command->cpu)[1]), READ_ONCE(((u32 *)command->cpu)[2]), qret); - if (qret) + if (qret) { cfg_err = qret; - else if (!READ_ONCE(((u32 *)command->cpu)[2])) { + } else if (!READ_ONCE(((u32 *)command->cpu)[2])) { dev_err(dev, "LEGACY CONFIG_GET reply word +0x08 zero\n"); cfg_err = -EPROTO; @@ -2337,11 +2338,11 @@ static int ane_rtclient_probe(struct platform_device *pdev) return 0; err_pm_or_hold: - if (ane->held) + if (ane->held) { dev_warn(dev, "probe failed after CPU start (%pe) — binding fenced; power/rings/IRQ HELD until reboot\n", ERR_PTR(ret)); - else { + } else { pm_runtime_put_sync_suspend(dev); pm_runtime_disable(dev); } From 09ccd7e1dfc07211681a22d9a3b80e4910a2a681 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:29:18 -0500 Subject: [PATCH 0595/2680] accel/ane: Declare the M2 ANE firmware files ane_t6021 loads apple/ane/t602x_ane0_fw_selene_rc4x.macho on T602x and apple/ane/h14_ane_fw_bia_j4xx.macho on T8112. Declare both with MODULE_FIRMWARE() and keep each name in one macro. The firmware is a distribution concern: omarchy-ane-firmware-fetch installs these files under /lib/firmware/apple/ane. When request_firmware() fails, name the file and how to install it. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/accel/ane/ane_fw_validate.h | 8 ++++++-- drivers/accel/ane/ane_t6021_fwload.c | 6 ++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/drivers/accel/ane/ane_fw_validate.h b/drivers/accel/ane/ane_fw_validate.h index 00ef2982c67e4..f4f723a3f33df 100644 --- a/drivers/accel/ane/ane_fw_validate.h +++ b/drivers/accel/ane/ane_fw_validate.h @@ -50,6 +50,10 @@ struct ane_fw_seg { char name[17]; /* 16 + NUL */ }; +/* request_firmware() names */ +#define ANE_FW_SELENE_NAME "apple/ane/t602x_ane0_fw_selene_rc4x.macho" +#define ANE_FW_BIA_NAME "apple/ane/h14_ane_fw_bia_j4xx.macho" + /* One pinned image. The patch fields are the payload's LC_SYMTAB * addresses: the __rtk_patch records _rtk_stack_guard, RTK_soc, * RTK_soc_revision, RTK_cpu_physical_address and @@ -66,7 +70,7 @@ struct ane_fw_image { }; static const struct ane_fw_image ane_fw_selene = { - .name = "apple/ane/t602x_ane0_fw_selene_rc4x.macho", + .name = ANE_FW_SELENE_NAME, .sha256 = { 0xa9, 0xc4, 0xb7, 0x71, 0x29, 0x4a, 0x6b, 0x11, 0x56, 0x24, 0xd9, 0x48, 0x0a, 0x62, 0x48, 0xd0, @@ -83,7 +87,7 @@ static const struct ane_fw_image ane_fw_selene = { }; static const struct ane_fw_image ane_fw_bia = { - .name = "apple/ane/h14_ane_fw_bia_j4xx.macho", + .name = ANE_FW_BIA_NAME, .sha256 = { 0xaf, 0x58, 0x7d, 0xfa, 0x96, 0xb1, 0xe0, 0x1d, 0x2b, 0x2e, 0x0f, 0x9f, 0x77, 0x6e, 0x5d, 0xbe, diff --git a/drivers/accel/ane/ane_t6021_fwload.c b/drivers/accel/ane/ane_t6021_fwload.c index d677962a828b7..c8a7ad8d7bee8 100644 --- a/drivers/accel/ane/ane_t6021_fwload.c +++ b/drivers/accel/ane/ane_t6021_fwload.c @@ -89,6 +89,8 @@ */ MODULE_LICENSE("Dual MIT/GPL"); MODULE_DESCRIPTION("T6021 ANE firmware staging + entry alias"); +MODULE_FIRMWARE(ANE_FW_SELENE_NAME); +MODULE_FIRMWARE(ANE_FW_BIA_NAME); static bool fw_load = true; module_param(fw_load, bool, 0444); @@ -565,8 +567,8 @@ int ane_t6021_fwload_probe(struct ane_t6021 *ane) ret = request_firmware(&fw, img->name, ane->dev); if (ret) { dev_err(ane->dev, - "fwload: request_firmware(%s): %d — stage the payload " - "with omarchy-ane-firmware-fetch\n", img->name, ret); + "cannot load firmware %s: %d; install it with omarchy-ane-firmware-fetch (or equivalent for your distribution)\n", + img->name, ret); return ret; } From ca99b9b6e58f51a7ce3481ac6f4cf8503645b59e Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 15:51:13 -0500 Subject: [PATCH 0596/2680] accel/ane: Add the ane_stats counters and ane_timeline Add producer-side statistics to both ANE drivers. /sys/class/accel/accel/device/ane_stats is a read-only (0444) file with two cumulative counters, one "key value" pair per line: - busy_ns: the nanoseconds the engine was busy since bind. A busy period opens when the first submission arrives on an idle engine and closes when the last one completes. Overlapping submissions share a period, so busy_ns is the union of the submit-to-completion windows, not their sum. ane serializes submissions behind engine_lock, so there the union is the sum. A read adds the time of the open period, so the value also increases while work is in flight. - jobs: the completed engine submissions since bind. ane counts each ANE_SUBMIT. ane_t6021 counts only CSNE_CMD_PROCEDURE_CALL. The control-plane commands on the same path and the boot transport are not engine work and do not count. A submission that fails still completes its begin/complete pair, so the counters stay balanced. Documentation/ABI/testing/sysfs-class-accel-ane describes the file. The ane_timeline debugfs file shows the latest submissions from a ring that the driver allocates at probe: seq, submit_ns, start_ns, end_ns, tasks, rc and tmst. Each slot has a sequence number, so a reader skips a slot that is in flight or that a later submission reuses. ane records the raw TM tick as tmst. ane_t6021 has no host TM path and records 0. Both drivers create the file with drm_debugfs_add_file(). The producer uses atomics only. It does not allocate, lock or format. A transition sentinel in the inflight count keeps the fold of one busy period and the start of the next apart. The next period starts at or after the fold end of the previous one. Both modules have a read-only stats parameter, default 1, that the driver reads at probe. With stats=0, or when the ring allocation fails, the driver creates no stats files and the submission path skips the counters. Testing: in the tree, this code is compile-verified only (arm64 W=1, =m and =y, and checkpatch --strict). A host test of the accounting through a kernel API shim passes. It includes a model check of all orders of three jobs, with a preemption at each atomic operation. The out-of-tree omarchy-ane driver with the same accounting passed the hardware gates on T8103 (M1) and T6001 (M1 Max). That version read max_end for the period start before it took the transition sentinel; this commit reads it after. The T6021 (M2 Max) path is in test. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- .../ABI/testing/sysfs-class-accel-ane | 50 ++ MAINTAINERS | 1 + drivers/accel/ane/ane.h | 12 + drivers/accel/ane/ane_drv.c | 84 +++- drivers/accel/ane/ane_stats.h | 431 ++++++++++++++++++ drivers/accel/ane/ane_t6021_rtclient_main.c | 125 +++++ drivers/accel/ane/ane_tm.c | 9 +- drivers/accel/ane/ane_tm.h | 3 + 8 files changed, 713 insertions(+), 2 deletions(-) create mode 100644 Documentation/ABI/testing/sysfs-class-accel-ane create mode 100644 drivers/accel/ane/ane_stats.h diff --git a/Documentation/ABI/testing/sysfs-class-accel-ane b/Documentation/ABI/testing/sysfs-class-accel-ane new file mode 100644 index 0000000000000..668bed9c7cb38 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-class-accel-ane @@ -0,0 +1,50 @@ +What: /sys/class/accel/accel/device/ane_stats +Date: October 2026 +KernelVersion: 7.1 +Contact: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> +Description: + Cumulative Apple Neural Engine (ANE) producer counters for + this device, one ASCII "key value" pair per line, mode + 0444 (readable without root): + + busy_ns: nanoseconds the engine was busy since the + device was bound, sampled with a consistent + close/open snapshot. A busy period opens + when the first submission lands on an idle + engine and closes when the last one + completes; its whole span folds once at + close, and overlapping submissions share + the period, so a serialized engine + (ane.ko, behind engine_lock) reports + sum(end - start) and a parallel engine + (ane_t6021) reports the union, never the + sum. The show callback adds the live tail + of the still-open period, so the value + advances while engine work is in flight. + + jobs: completed engine submissions since the + device was bound. On ane.ko every ANE_SUBMIT + counts (one per libane ane_exec, N per + ane_exec_loop). On ane_t6021 only + CSNE_CMD_PROCEDURE_CALL counts (one per + engine PROCEDURE_CALL); the control-plane + exchanges that ride the same path + (LOAD_PROGRAM, CREATE_PROCESS, + CH_PROPERTY_WRITE, install-time CONFIG_GET) + and the boot transport are not engine + work and are not counted. A submission + that completes with an error still + completes its begin/complete pair, so the + counter stays balanced. + + Neither counter resets while the device is bound. + + The file exists only when the module parameter + ane.stats / ane_t6021.stats (default 1) is set at + probe. With stats=0 the driver creates no stats files + and the submission hot path skips the counters. + +Users: performance samplers that compute engine utilization + and job rate from the two counters. + + diff --git a/MAINTAINERS b/MAINTAINERS index f478d6dc548ed..dc6b7c0f8c1f6 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7950,6 +7950,7 @@ M: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> L: asahi@lists.linux.dev L: dri-devel@lists.freedesktop.org S: Maintained +F: Documentation/ABI/testing/sysfs-class-accel-ane F: Documentation/devicetree/bindings/npu/apple,ane.yaml F: drivers/accel/ane/ F: include/uapi/drm/ane_accel.h diff --git a/drivers/accel/ane/ane.h b/drivers/accel/ane/ane.h index db30a3e103fff..eac81bdab9f74 100644 --- a/drivers/accel/ane/ane.h +++ b/drivers/accel/ane/ane.h @@ -15,6 +15,8 @@ #include +#include "ane_stats.h" + struct ane_device { struct drm_device drm; struct device *dev; @@ -112,6 +114,16 @@ struct ane_device { bool busy; bool on; } boost; + + /* + * Producer-side stats (sysfs ane_stats, debugfs ane_timeline; + * see ane_stats.h). stats=0 or a failed ring allocation leaves + * stats_slots NULL, which is the one hot-path gate: no files, + * no counter updates. + */ + struct ane_stats_counters stats_ctrs; + struct ane_stats_ring stats_ring; + struct ane_stats_ring_entry *stats_slots; }; struct ane_request { diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 738fa5d17b1fa..20e32a76f190a 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -2,7 +2,9 @@ /* Copyright 2022 Eileen Yoon */ #include +#include #include +#include #include #include #include @@ -11,6 +13,7 @@ #include #include +#include #include #include #include @@ -53,6 +56,16 @@ module_param(map_mode, int, 0644); MODULE_PARM_DESC(map_mode, "BO mapping: bit0=IOMMU_CACHE DART descriptors, bit1=cacheable CPU vmas (default 3 = cached; 0 = writecombine + non-cacheable rollback)"); +/* + * Producer-side stats (ane_stats sysfs, ane_timeline debugfs), read + * once at probe. stats=0 keeps the hot path at its single predictable + * branch (the NULL stats_slots check) and no files are created. + */ +static bool stats = true; +module_param(stats, bool, 0444); +MODULE_PARM_DESC(stats, + "Expose ane_stats sysfs and ane_timeline debugfs (default 1; 0 = no files and the hot path skips the counters)"); + #define CMD_BUF_BDX 0 #define KRN_BUF_BDX 1 @@ -480,6 +493,7 @@ static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) struct drm_gem_object *btsp = NULL; struct ane_bo *bo; struct ane_request req; + u64 stats_idx = 0; int err; memset(&req, 0, sizeof(req)); @@ -561,10 +575,19 @@ static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) if (err < 0) goto unlock; + if (ane->stats_slots) + stats_idx = ane_stats_begin(&ane->stats_ctrs, &ane->stats_ring, + ktime_get_ns(), req.td_count); + ane_boost_begin(ane); err = ane_tm_execute(ane, &req); ane_boost_end(ane); + if (ane->stats_slots) + ane_stats_complete(&ane->stats_ctrs, &ane->stats_ring, + stats_idx, ktime_get_ns(), + err ? (u32)err : 0, ane_last_tmst); + unlock: mutex_unlock(&ane->engine_lock); put: @@ -611,12 +634,47 @@ static ssize_t reset_store(struct device *dev, struct device_attribute *attr, } static DEVICE_ATTR_WO(reset); +/* + * ane_stats: cumulative busy_ns/jobs for this device (mode 0444, no + * root needed). Formatting lives in ane_stats_emit() (ane_stats.h), + * shared with ane_t6021.ko. + */ +static ssize_t ane_stats_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ane_device *ane = dev_get_drvdata(dev); + + return ane_stats_emit(buf, &ane->stats_ctrs); +} +static DEVICE_ATTR_RO(ane_stats); + static struct attribute *ane_dev_attrs[] = { &dev_attr_wedged.attr, &dev_attr_reset.attr, + &dev_attr_ane_stats.attr, + NULL, +}; + +/* ane_stats appears only when stats=1 gave the device a ring. */ +static umode_t ane_dev_attr_is_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + struct ane_device *ane = dev_get_drvdata(kobj_to_dev(kobj)); + + if (attr == &dev_attr_ane_stats.attr && !ane->stats_slots) + return 0; + return attr->mode; +} + +static const struct attribute_group ane_dev_group = { + .attrs = ane_dev_attrs, + .is_visible = ane_dev_attr_is_visible, +}; + +static const struct attribute_group *ane_dev_groups[] = { + &ane_dev_group, NULL, }; -ATTRIBUTE_GROUPS(ane_dev); static int ane_drm_open(struct drm_device *drm, struct drm_file *file) { @@ -964,6 +1022,25 @@ static int ane_platform_probe(struct platform_device *pdev) atomic_set(&ane->wedged, 0); ane->tm_retention = soc->tm_retention; + /* Producer-side stats: preallocate the ring at probe; the files + * and the hot-path branch key off stats_slots. + */ + if (stats) { + ane->stats_slots = devm_kcalloc(dev, + 1u << ANE_STATS_RING_ORDER_DEFAULT, + sizeof(*ane->stats_slots), + GFP_KERNEL); + if (ane->stats_slots) { + ane_stats_counters_init(&ane->stats_ctrs, + &ane->stats_ring, + ANE_STATS_RING_ORDER_DEFAULT); + ane->stats_ring.slots = ane->stats_slots; + } else { + dev_warn(dev, + "ane_stats ring allocation failed; stats disabled\n"); + } + } + /* Managed power first: genpd links hold the ANE/DART supplier * topology awake before any register is touched. */ @@ -1040,6 +1117,11 @@ static int ane_platform_probe(struct platform_device *pdev) if (err < 0) goto put_pm; + if (ane->stats_slots) + drm_debugfs_add_file(drm, "ane_timeline", + ane_timeline_show, + &ane->stats_ring); + err = drm_dev_register(drm, 0); if (err < 0) goto put_pm; diff --git a/drivers/accel/ane/ane_stats.h b/drivers/accel/ane/ane_stats.h new file mode 100644 index 0000000000000..768fe97846edc --- /dev/null +++ b/drivers/accel/ane/ane_stats.h @@ -0,0 +1,431 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Producer-side ANE counters and submission ring, shared by ane.ko and + * ane_t6021.ko. + * + * Contract: + * - sysfs under /sys/class/accel/accel/device/ane_stats, + * mode 0444, ASCII "key value" lines, integers only: busy_ns + * (cumulative u64), jobs (cumulative u64). busy_ns counts only + * nanoseconds the engine was busy (union of submit-to-completion + * windows) since the device was bound, and advances while work is + * in flight: the show callback adds the open period's live tail. + * - debugfs ane_timeline: preallocated ring of the last N submissions + * with seq, submit_ns, start_ns, end_ns, tasks, rc (and tmst raw + * when known). Head counter is atomic; per-slot seqlock so the + * reader never blocks the producer and torn reads are detected. + * - `stats` module parameter bool, default 1; stats=0 makes the hot + * path one predictable branch and skips sysfs/debugfs creation. + * + * No allocation, locking, or formatting in the hot path. The sysfs + * show callback formats in process context; the ring writer commits + * the slot with a release store, and the reader does an acquire load + * on begin/end. Counters update with atomics only. busy_ns is the + * union of busy intervals: a busy period opens when the first + * submission lands on an idle engine and closes when the last one + * completes, and its whole span folds once at close; overlapping + * submissions share the period, so a serialized engine (ane.ko, + * behind engine_lock) reports sum(end - start) and a parallel engine + * (ane_t6021) reports the union, never the sum. + */ + +#ifndef __ANE_STATS_H__ +#define __ANE_STATS_H__ + +#include +#include +#include +#include +#include +#include +#include + +/* + * Per-device counters. busy_ns holds the busy time folded from closed + * busy periods: a period opens when the first submission arrives on an + * idle engine and closes when the last one completes, and its whole + * span folds once at close. Overlapping submissions share the period, + * so busy_ns is the union of the submit-to-completion intervals, not + * their sum. The show callback adds the live tail of the still-open + * period, so the reported value advances while engine work is in + * flight. jobs counts completed submissions. last_busy_end is the open + * period's start timestamp; inflight is the number of in-flight + * submissions, with ANE_STATS_INFLIGHT_TRANS as the transient close/ + * open sentinel that keeps a fold and a period start from interleaving. + * + * Module-level `stats` parameter governs whether these counters and + * the ring are created and whether the hot path branches out. + */ +#define ANE_STATS_INFLIGHT_TRANS 0xFFFFFFFFu + +struct ane_stats_counters { + atomic64_t busy_ns; + atomic64_t jobs; + atomic64_t last_busy_end; + atomic64_t max_end; + atomic_t inflight; +}; + +/* + * max_end tracks the latest completion sample fed to complete(); the + * draining completion folds against max(end_ns, max_end) so a stale + * sample on the last completer cannot truncate the period. + */ +static inline void ane_stats_atomic64_max(u64 i, atomic64_t *v) +{ + u64 old = atomic64_read(v); + + while (i > old && !atomic64_try_cmpxchg(v, &old, i)) + ; +} + +/* + * Ring slot: seqlock-style per-slot sequence. Writer bumps seq at + * entry (odd) and again on commit (even); reader loads begin/end + * inside acquire load and the order relative to begin/end is preserved + * by release. begin/end 0 means the slot is empty. submit_ns, end_ns, + * tasks, rc, tmst are the diagnostic fields; tmst is the raw TM tick + * on ane.ko (unknown unit) and 0 on ane_t6021 (unavailable, marked so + * in the file header line). + */ +struct ane_stats_ring_entry { + atomic64_t seq; /* even = committed; odd = writing */ + atomic64_t submit_ns; + atomic64_t start_ns; + atomic64_t end_ns; + atomic_t tasks; + atomic_t rc; + atomic64_t tmst; /* raw TM tick on ane.ko; 0 = unavailable */ +}; + +/* + * Ring container. head is the next slot to write (increments mod N). + * N is a power of two preallocated at probe. slots is the array. + * Reader takes begin/end under acquire and the seqlock protects + * against torn writes. + */ +struct ane_stats_ring { + atomic64_t head; + u32 n; /* power of two */ + u32 mask; /* n - 1 */ + struct ane_stats_ring_entry *slots; +}; + +/* + * Initialize counters and ring from zero. The ring slots array must + * be preallocated (probe-time) and zeroed. Named + * ane_stats_counters_init (not ane_stats_init) so the per-driver + * glue function (ane_stats_init(struct ane_device *) etc.) can name + * its own static initializer without conflicting with the prototype. + */ +static inline void ane_stats_counters_init(struct ane_stats_counters *ctrs, + struct ane_stats_ring *ring, + u32 n_shift) +{ + u32 n_total = (1u << n_shift); + + memset(ctrs, 0, sizeof(*ctrs)); + memset(ring, 0, sizeof(*ring)); + ring->n = n_total; + ring->mask = n_total - 1u; + atomic64_set_release(&ring->head, 0ull); +} + +/* + * Hot-path submission start. Caller holds the device's submission + * serialization if the engine is single-producer (ane.ko); concurrent + * drivers (ane_t6021) pass ring/ctrs and rely on the cmpxchg loops. + * Returns the submission ticket (head + 1, starting at 1) for the + * caller to later call ane_stats_complete() with. The ticket, not a + * slot index, identifies the submission: slots are shared after wrap, + * tickets are not. + * + * Counter side: the first submission onto an idle engine opens a busy + * period by latching last_busy_end = submit_ns under the transition + * sentinel; later overlapping submissions only increment inflight. + * The sentinel window is a handful of instructions, so the bounded + * cmpxchg retries never observe a half-open period. + * + * A period never starts before the previous one folded: a caller + * sample can be stale (taken before the transition), so the latch is + * max(submit_ns, max_end), with max_end read under the transition + * sentinel. + */ +static inline u64 ane_stats_begin(struct ane_stats_counters *ctrs, + struct ane_stats_ring *ring, + u64 submit_ns, u32 tasks) +{ + u64 ticket = atomic64_fetch_add(1ull, &ring->head) + 1ull; + struct ane_stats_ring_entry *e = + &ring->slots[(size_t)(ticket - 1ull) & ring->mask]; + u64 latch; + u32 cur = atomic_read(&ctrs->inflight); + + for (;;) { + if (cur == ANE_STATS_INFLIGHT_TRANS) { + cur = atomic_read(&ctrs->inflight); + } else if (!cur) { + if (atomic_cmpxchg(&ctrs->inflight, 0u, + ANE_STATS_INFLIGHT_TRANS) != 0u) { + cur = atomic_read(&ctrs->inflight); + continue; + } + /* + * Read max_end only now. The drainer of the previous + * period fed its end sample into max_end before its + * release store of inflight = 0, and every other + * member fed before its decrement. The fully ordered + * cmpxchg above read that 0, so this read sees every + * feed and the latch is never below the previous + * fold end: periods cannot overlap. + */ + latch = atomic64_read(&ctrs->max_end); + if (submit_ns > latch) + latch = submit_ns; + atomic64_set_release(&ctrs->last_busy_end, latch); + atomic_set_release(&ctrs->inflight, 1u); + break; + } else if (atomic_try_cmpxchg(&ctrs->inflight, &cur, + cur + 1u)) { + break; + } + } + + /* + * Order the counter stores above (last_busy_end, inflight) before + * the slot stores below. + */ + smp_wmb(); + (void)atomic64_read_acquire(&e->seq); /* pair with reader */ + /* + * In flight: seq stays odd (2*ticket - 1) until complete() + * commits the even final value 2*ticket. The reader only prints + * even seqs it can match, so an unfinished submission never + * prints and a torn write is never visible. + */ + atomic64_set_release(&e->seq, 2ull * ticket - 1ull); + atomic64_set_release(&e->submit_ns, submit_ns); + /* + * start_ns is the busy-period start this submission is counted + * from (the latch), not the caller's sample: a sample taken + * before the transition can predate period entry, and the + * union reference must use consumed values. + */ + atomic64_set_release(&e->start_ns, + atomic64_read(&ctrs->last_busy_end)); + atomic64_set_release(&e->end_ns, submit_ns); + atomic_set(&e->tasks, tasks); + atomic_set(&e->rc, (u32)0xFFFFFFFFu); /* sentinel: not done */ + atomic64_set_release(&e->tmst, 0ull); + return ticket; +} + +/* + * Hot-path submission completion. The last submission out of a busy + * period closes it: under the transition sentinel it folds the whole + * period span [last_busy_end, end_ns] into busy_ns before reopening + * the counter, so no reader can see a period both live and folded. + * Overlapping submissions share the period, so busy_ns is the union + * of the submit-to-completion intervals, not their sum. Updates the + * slot's end_ns/rc/tmst and increments jobs. + */ +static inline void ane_stats_complete(struct ane_stats_counters *ctrs, + struct ane_stats_ring *ring, + u64 ticket, u64 end_ns, + u32 rc, u64 tmst) +{ + struct ane_stats_ring_entry *e = + &ring->slots[(size_t)(ticket - 1ull) & ring->mask]; + u32 cur = atomic_read(&ctrs->inflight); + u64 fold_end = 0ull; + + /* + * Feed this completion's end sample before the transition: a + * drainer can only observe inflight == 1 after every other + * completer has fed, so the fold below sees the whole period. + */ + ane_stats_atomic64_max(end_ns, &ctrs->max_end); + + for (;;) { + if (cur == ANE_STATS_INFLIGHT_TRANS) { + cur = atomic_read(&ctrs->inflight); + } else if (cur > 1u) { + if (atomic_try_cmpxchg(&ctrs->inflight, &cur, + cur - 1u)) + break; + } else if (!cur) { + /* + * Unbalanced complete (cannot happen with the + * documented one-begin-per-complete pairing): + * count the job, fold nothing, never hang. + */ + break; + } else if (atomic_cmpxchg(&ctrs->inflight, 1u, + ANE_STATS_INFLIGHT_TRANS) != 1u) { + cur = atomic_read(&ctrs->inflight); + } else { + u64 s = atomic64_read(&ctrs->last_busy_end); + + /* + * The period ends at the latest completion sample + * in it, not at this caller's (possibly stale) + * sample: every member fed max_end before the + * drain could observe inflight == 1. + */ + fold_end = atomic64_read(&ctrs->max_end); + if (fold_end < end_ns) + fold_end = end_ns; + if (fold_end > s) + atomic64_add(fold_end - s, &ctrs->busy_ns); + atomic_set_release(&ctrs->inflight, 0u); + break; + } + } + atomic64_add(1ull, &ctrs->jobs); + + /* Order the counter stores above before the slot stores below. */ + smp_wmb(); + /* + * The drainer records the consumed fold end, not its own sample, + * so the slot union equals busy_ns exactly. + */ + if (fold_end) + atomic64_set_release(&e->end_ns, fold_end); + else + atomic64_set_release(&e->end_ns, end_ns); + atomic_set(&e->rc, rc); + atomic64_set_release(&e->tmst, tmst); + /* + * Order the slot field stores before the even seq store that + * publishes them; pairs with the acquire load of e->seq in + * ane_timeline_show(). + */ + smp_wmb(); + /* + * Commit the slot with the even final seq 2*ticket. The value + * must not depend on the current ring->head: concurrent + * submissions (ane_t6021) advance it, and a head-derived seq + * would mislabel the slot. + */ + atomic64_set_release(&e->seq, 2ull * ticket); +} + +/* + * The timeline formatter is header-only so each module gets its own + * copy: kbuild rejects one object linked into two modules, and with + * both drivers built-in a shared object would duplicate symbols at + * vmlinux link. Both drivers register it with drm_debugfs_add_file(), + * so m->private is the struct drm_debugfs_entry and the ring is the + * data pointer given at registration. + */ +static inline int ane_timeline_show(struct seq_file *m, void *v) +{ + struct drm_debugfs_entry *entry = m->private; + struct ane_stats_ring *ring = entry->file.data; + struct ane_stats_ring_entry *ring_slots = ring->slots; + u32 mask = ring->mask; + u64 head = atomic64_read(&ring->head); + u64 live = head < (u64)mask + 1ull ? head : (u64)mask + 1ull; + u64 i; + + seq_puts(m, + "# ane_timeline: seq submit_ns start_ns end_ns tasks rc tmst (tmst raw tick on ane.ko, 0 = unavailable on ane_t6021)\n"); + /* + * Newest first. Submission ticket t lives in slot (t - 1) & mask + * and prints once complete() commits seq = 2*t; anything else + * (odd in-flight seq, stale slot) is skipped. + */ + for (i = 0; i < live; i++) { + u64 ticket = head - i; + struct ane_stats_ring_entry *e = + &ring_slots[(size_t)(ticket - 1ull) & mask]; + u64 seq = atomic64_read_acquire(&e->seq); + u64 submit = atomic64_read(&e->submit_ns); + u64 st = atomic64_read(&e->start_ns); + u64 en = atomic64_read(&e->end_ns); + u32 tasks = atomic_read(&e->tasks); + u32 rc = atomic_read(&e->rc); + u64 tmst = atomic64_read(&e->tmst); + + if (seq != 2ull * ticket) + continue; /* in flight or stale; never printed */ + seq_printf(m, "%llu %llu %llu %llu %u %u %llu\n", + 2ull * ticket, submit, st, en, tasks, rc, tmst); + } + return 0; +} + +#define ANE_STATS_RING_ORDER_DEFAULT 8 /* 256 slots */ + +/* + * Accounting rule (coreglass producer contract): jobs counts completed + * submissions, one begin/complete pair per engine submission — one per + * ANE_SUBMIT on ane.ko (so libane ane_exec is 1 job and ane_exec_loop + * with N iterations is N jobs) and one per firmware PROCEDURE_CALL + * (CSNE_CMD_PROCEDURE_CALL) on ane_t6021.ko: that opcode is the only + * engine work on the shared ane_rtclient_command path. The control- + * plane exchanges that ride the same function — LOAD_PROGRAM, + * CREATE_PROCESS, CH_PROPERTY_WRITE, install-time CONFIG_GET — and the + * boot transport are not engine submissions and are not counted; jobs + * must match the number of engine calls the workload made (+-1 at the + * sampler boundary). A submission that completes with an error still + * completes its begin/complete pair, so the counter stays balanced. + * + * Typed sysfs formatter for the ane_stats attribute. The per-driver + * show callbacks fetch the counters from their real drvdata type + * (struct ane_device * on ane.ko, struct ane_rtclient * on + * ane_t6021.ko) and pass &...->stats_ctrs here. The formatter never + * sees the device pointer, so the drvdata type confusion that shipped + * in the first round (reading the head of ane_device as counters) + * cannot compile again: there is no cast to remove. + */ +/* + * Consistent counter snapshot across the tiny close/open transition: + * the transition sentinel is never accepted, and (inflight, busy_ns) + * must read back unchanged, so a value is only reported between + * transitions. The value is busy_ns plus the live tail of the + * still-open period, so it advances while engine work is in flight. + * Over the timeline the reported value is non-decreasing: folds only + * add, and at a close the live tail equals the fold. + */ +static inline u64 ane_stats_snapshot(const struct ane_stats_counters *ctrs) +{ + u64 raw, busy = 0, now, s = 0; + u32 inflight; + + for (;;) { + inflight = atomic_read_acquire(&ctrs->inflight); + if (inflight == ANE_STATS_INFLIGHT_TRANS) + continue; /* close/open in progress: spin it out */ + raw = atomic64_read(&ctrs->busy_ns); + if (inflight) + s = atomic64_read(&ctrs->last_busy_end); + now = ktime_get_ns(); + /* + * Recheck after the timestamp: a fold that landed before + * `now` changes busy_ns and retries; one that lands + * after is genuinely later than `now`, so the value is + * exact, never an overshoot. + */ + if (atomic64_read(&ctrs->busy_ns) != raw || + atomic_read(&ctrs->inflight) != inflight || + (inflight && + atomic64_read(&ctrs->last_busy_end) != s)) + continue; + busy = raw; + if (inflight && now > s) + busy += now - s; + break; + } + return busy; +} + +static inline ssize_t ane_stats_emit(char *buf, + const struct ane_stats_counters *ctrs) +{ + return sysfs_emit(buf, "busy_ns %llu\njobs %llu\n", + (unsigned long long)ane_stats_snapshot(ctrs), + (unsigned long long)atomic64_read(&ctrs->jobs)); +} + +#endif /* __ANE_STATS_H__ */ diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index 676029a9e7b27..18d0ecf6b2f6e 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -87,6 +87,7 @@ #include #include +#include #include #include #include @@ -95,6 +96,7 @@ #include #include +#include "ane_stats.h" #include "ane_t6021.h" #include "ane_t6021_boot.h" @@ -177,6 +179,17 @@ module_param(hello_wait_ms, uint, 0444); MODULE_PARM_DESC(hello_wait_ms, "RTKit HELLO wait in legacy mode; 0 (default) skips RTKit and leaves the mailbox stopped. A firmware that speaks RTKit needs 1000."); +/* + * Producer-side stats (ane_stats sysfs, ane_timeline debugfs), read + * once at probe. stats=0 keeps the hot path at its single predictable + * branch (the NULL stats_slots check) and no files are created. Same + * name and meaning as ane.ko's parameter. + */ +static bool stats = true; +module_param(stats, bool, 0444); +MODULE_PARM_DESC(stats, + "Expose ane_stats sysfs and ane_timeline debugfs (default 1; 0 = no files and the hot path skips the counters)"); + #define ANE_LEGACY_ALLOCS 8192 #define ANE_LEGACY_BYTES SZ_512M @@ -222,6 +235,15 @@ struct ane_rtclient { * which the firmware may reference forever (held until reboot). */ struct ane_legacy_buffer *cmd_buf; + + /* + * Producer-side stats (see ane_stats.h). stats=0 or a failed + * ring allocation leaves stats_slots NULL, which is the one + * hot-path gate: no files, no counter updates. + */ + struct ane_stats_counters stats_ctrs; + struct ane_stats_ring stats_ring; + struct ane_stats_ring_entry *stats_slots; }; /* Per-open BO ownership (drm_file->driver_priv). Handles live in the @@ -889,6 +911,8 @@ static int ane_rtclient_command(struct ane_rtclient *ane, unsigned int timeout_ms) { int ret; + u64 stats_ticket = 0; + u64 stats_submit_ns = 0; ane_t6021_tracing = opcode == CSNE_CMD_PROCEDURE_CALL && trace_td && ane->soc->trace_td_off; @@ -896,10 +920,38 @@ static int ane_rtclient_command(struct ane_rtclient *ane, ane_t6021_trace->calls++; ane_t6021_trace_add(ANE_TR_CALL, 0); } + /* Producer contract hot path (ane_stats.h): record submit at + * command enqueue, completion after the call returns, one + * begin/complete pair per engine submission. Only + * CSNE_CMD_PROCEDURE_CALL is engine work; the control-plane + * exchanges that ride this function (LOAD_PROGRAM, + * CREATE_PROCESS, CH_PROPERTY_WRITE, CONFIG_GET) are not + * counted, so jobs matches the engine calls the workload made. + * The T6021 path can run concurrently (MBI per-channel rings), + * so overlapping calls share a busy period and busy_ns is the + * union of the submit-to-completion windows. tmst is 0 (no + * host TM on T6021; documented in the file header line). + * tasks = 1 (one call per submission); rc = ret. The gate is the + * ring, as on ane.ko: stats=0 or a failed probe allocation leaves + * stats_slots NULL, and the hooks must not run on a NULL ring. + */ + bool stats_call = ane->stats_slots && + opcode == CSNE_CMD_PROCEDURE_CALL; + + if (stats_call) { + stats_submit_ns = ktime_get_ns(); + stats_ticket = ane_stats_begin(&ane->stats_ctrs, + &ane->stats_ring, + stats_submit_ns, 1); + } ret = ane_rtclient_legacy_exchange(ane, command, length, opcode, channel, timeout_ms); if (ret) { ane_t6021_tracing = false; + if (stats_call) + ane_stats_complete(&ane->stats_ctrs, + &ane->stats_ring, stats_ticket, + ktime_get_ns(), (u32)ret, 0); dev_info(ane->dev, "EXCH op=%#x failed %d (fw allocs %u, %zu bytes)\n", opcode, ret, ane->legacy_allocated, ane->legacy_bytes); atomic_set(&ane_t6021_quarantined, 1); @@ -913,12 +965,21 @@ static int ane_rtclient_command(struct ane_rtclient *ane, if (!ret && call_settle_us) usleep_range(call_settle_us, call_settle_us + 100); if (ret) { + if (stats_call) + ane_stats_complete(&ane->stats_ctrs, + &ane->stats_ring, + stats_ticket, + ktime_get_ns(), + (u32)ret, 0); dev_info(ane->dev, "call completion wait failed %d\n", ret); atomic_set(&ane_t6021_quarantined, 1); return ret; } } + if (stats_call) + ane_stats_complete(&ane->stats_ctrs, &ane->stats_ring, + stats_ticket, ktime_get_ns(), 0, 0); /* The fw talks back on the target-to-host rings (fwlog, perf); * hand those slots back so the rings never fill (the sequencer * did this per step; same ack, channels 4 and 6). @@ -2304,6 +2365,25 @@ static int ane_rtclient_probe(struct platform_device *pdev) return -EPROTO; } + /* Producer-side stats: preallocate the ring at probe; the files + * and the hot-path branch key off stats_slots. + */ + if (stats) { + ane->stats_slots = devm_kcalloc(dev, + 1u << ANE_STATS_RING_ORDER_DEFAULT, + sizeof(*ane->stats_slots), + GFP_KERNEL); + if (ane->stats_slots) { + ane_stats_counters_init(&ane->stats_ctrs, + &ane->stats_ring, + ANE_STATS_RING_ORDER_DEFAULT); + ane->stats_ring.slots = ane->stats_slots; + } else { + dev_warn(dev, + "ane_stats ring allocation failed; stats disabled\n"); + } + } + { struct ane_t6021_drm *adrm; int drmret; @@ -2320,6 +2400,10 @@ static int ane_rtclient_probe(struct platform_device *pdev) } adrm->dev = dev; adrm->ane = ane; + if (ane->stats_slots) + drm_debugfs_add_file(&adrm->drm, "ane_timeline", + ane_timeline_show, + &ane->stats_ring); WRITE_ONCE(ane_t6021_perf_ane, ane); drmret = drm_dev_register(&adrm->drm, 0); if (drmret) { @@ -2372,10 +2456,51 @@ static const struct of_device_id ane_rtclient_of_match[] = { }; MODULE_DEVICE_TABLE(of, ane_rtclient_of_match); +/* + * ane_stats: cumulative busy_ns/jobs for this device (mode 0444, no + * root needed). Formatting lives in ane_stats_emit() (ane_stats.h), + * shared with ane.ko. + */ +static ssize_t ane_stats_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ane_rtclient *ane = dev_get_drvdata(dev); + + return ane_stats_emit(buf, &ane->stats_ctrs); +} +static DEVICE_ATTR_RO(ane_stats); + +static struct attribute *ane_t6021_stats_attrs[] = { + &dev_attr_ane_stats.attr, + NULL, +}; + +/* ane_stats appears only when stats=1 gave the device a ring. */ +static umode_t ane_t6021_stats_is_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + struct ane_rtclient *ane = dev_get_drvdata(kobj_to_dev(kobj)); + + if (attr == &dev_attr_ane_stats.attr && !ane->stats_slots) + return 0; + return attr->mode; +} + +static const struct attribute_group ane_t6021_stats_group = { + .attrs = ane_t6021_stats_attrs, + .is_visible = ane_t6021_stats_is_visible, +}; + +static const struct attribute_group *ane_t6021_stats_groups[] = { + &ane_t6021_stats_group, + NULL, +}; + static struct platform_driver ane_rtclient_driver = { .driver = { .name = "ane_t6021", .of_match_table = ane_rtclient_of_match, + .dev_groups = ane_t6021_stats_groups, .suppress_bind_attrs = true, }, .probe = ane_rtclient_probe, diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index 9364ae1469b9d..7a0ade69c395b 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -216,6 +216,13 @@ static void ane_tm_push_tq(struct ane_device *ane, struct ane_request *req) tm_write32(ane, TM_PUSH, TQ_PRTY_TABLE[qid] | (qid & 7) << 8); // magic } +/* + * Last raw TM tick read from TM_IRQ_TMST in the latest event drain. + * Diagnostic only (unit unknown; the timeline file labels it as a + * raw tick); unsynchronized by design. + */ +u32 ane_last_tmst; + static int ane_tm_collect_events(struct ane_device *ane, struct ane_request *req, u32 *finished) { @@ -228,7 +235,7 @@ static int ane_tm_collect_events(struct ane_device *ane, u32 info = tm_read32(ane, TM_IRQ_INFO(line)); tm_read32(ane, TM_IRQ_UNK1(line)); - tm_read32(ane, TM_IRQ_TMST(line)); + ane_last_tmst = tm_read32(ane, TM_IRQ_TMST(line)); tm_read32(ane, TM_IRQ_UNK2(line)); if (req && info == (0x05000000 | (req->nid << 16) | (req->td_count - 1))) diff --git a/drivers/accel/ane/ane_tm.h b/drivers/accel/ane/ane_tm.h index 94d425ab8d8b7..84520c149bd52 100644 --- a/drivers/accel/ane/ane_tm.h +++ b/drivers/accel/ane/ane_tm.h @@ -17,4 +17,7 @@ int ane_tm_recover(struct ane_device *ane); #define ANE_PS_ALL_ON ((1U << (4 * 6)) - 1) u32 ane_tm_ps_act(struct ane_device *ane); +/* Last raw TM tick captured by ane_tm_collect_events (diagnostic). */ +extern u32 ane_last_tmst; + #endif /* __ANE_TM_H__ */ From 73911a8bb278a71d116acae09fdc7cfa23e2b576 Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 14:24:57 -0700 Subject: [PATCH 0597/2680] accel/ane: t6021: stop logging boot progress at emergency level The M2 ANE module from Joshua Warren's aurora-silicon/linux#155 logs its probe and firmware boot progress with dev_emerg(). These lines came from bring-up, where each one had to reach the console before the write it announced. With fw_start on by default, an M2 Max now prints about twenty emergency-level lines on every boot, which go to every console and terminal regardless of loglevel and bury real emergencies. Use the level each message deserves: - the per-phase BOOT-PHASE markers and the probe register readouts (ane_cpu ACTUAL, PWGATE, the engine reads, staging and dispatch, the legacy ack and hello start) become dev_dbg(); dynamic debug still gets them for crash attribution; - the outcome lines (sequence returned, done without a CPU start, the legacy hello result) become dev_info(); - the SCRATCH/RVBAR/tick dump, printed only when the ANE CPU was released but never reported READY, becomes dev_err(). The 30 ms drain after each phase marker is left as it is, and the comment above it now says how to see the marker. No other drivers/accel/ane code logs at emergency, alert or critical level. Found in a review of #155. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit d7ec38757843b95b2414d385965358d615647f01) --- drivers/accel/ane/ane_t6021_boot.c | 43 +++++++++---------- drivers/accel/ane/ane_t6021_rtclient_main.c | 46 ++++++++++----------- 2 files changed, 45 insertions(+), 44 deletions(-) diff --git a/drivers/accel/ane/ane_t6021_boot.c b/drivers/accel/ane/ane_t6021_boot.c index de11bb8f095fa..1ae368ae3e55f 100644 --- a/drivers/accel/ane/ane_t6021_boot.c +++ b/drivers/accel/ane/ane_t6021_boot.c @@ -334,14 +334,15 @@ static void ane_boot_phase(void *ctx, const char *what) { struct ane_t6021_boot_mmio *mm = ctx; - /* bounded phase marker: one line per block boundary, survives - * netconsole for crash attribution (never per-poll). KERN_EMERG - * + a short drain so the line reaches tty0/netconsole/ssh - * BEFORE the risky write it announces (fw-start-debug - * 2026-09-22: fwstart#2 died with zero capture — the marker - * must beat the write). + /* bounded phase marker: one line per block boundary (never + * per-poll), for crash attribution. It is a debug message: turn + * it on with dynamic debug, and raise the console loglevel to + * see it there. The short drain lets the line reach + * tty0/netconsole/ssh BEFORE the risky write it announces + * (fw-start-debug 2026-09-22: fwstart#2 died with zero capture — + * the marker must beat the write). */ - dev_emerg(mm->ane->dev, "BOOT-PHASE %s\n", what); + dev_dbg(mm->ane->dev, "BOOT-PHASE %s\n", what); msleep(30); } @@ -457,14 +458,14 @@ static void ane_t6021_boot_progress_dump(struct ane_t6021 *ane) tick1 = readl(eng + ANE_T6021_BOOT_REG_TICK); for (i = 0; i < 8; i++) - dev_emerg(ane->dev, "PROGRESS SCRATCH%u=%08x\n", i, - readl(eng + ANE_T6021_BOOT_REG_SCRATCH0 + 4 * i)); - dev_emerg(ane->dev, - "PROGRESS rvbar=%016llx cpu_status=%08x tick %08x->%08x (%s)\n", - readq(eng + ANE_ASC_RVBAR), - readl(eng + ANE_ASC_CPU_STATUS), - tick0, tick1, - tick1 != tick0 ? "24MHz domain clocked" : "tick STATIC"); + dev_err(ane->dev, "PROGRESS SCRATCH%u=%08x\n", i, + readl(eng + ANE_T6021_BOOT_REG_SCRATCH0 + 4 * i)); + dev_err(ane->dev, + "PROGRESS rvbar=%016llx cpu_status=%08x tick %08x->%08x (%s)\n", + readq(eng + ANE_ASC_RVBAR), + readl(eng + ANE_ASC_CPU_STATUS), + tick0, tick1, + tick1 != tick0 ? "24MHz domain clocked" : "tick STATIC"); } int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, int rtb_mode) @@ -501,9 +502,9 @@ int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, return -EBUSY; } - dev_emerg(ane->dev, - "BOOT-PHASE dispatch (stop_after=%d%s)\n", stop_after, - stop_after ? " BISECT STOP ARMED" : ""); + dev_dbg(ane->dev, + "BOOT-PHASE dispatch (stop_after=%d%s)\n", stop_after, + stop_after ? " BISECT STOP ARMED" : ""); r = ane_t6021_boot_run(&io, &cfg, &cs, &fa, &bo, &sres); @@ -523,9 +524,9 @@ int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, if (!cs) { /* no CPU start: full release path, normal ownership */ module_put(THIS_MODULE); - dev_emerg(ane->dev, - "BOOT-PHASE done r=%d cpu_started=0 (no CPU release: state clean, module unpinned)\n", - r); + dev_info(ane->dev, + "BOOT-PHASE done r=%d cpu_started=0 (no CPU release: state clean, module unpinned)\n", + r); return r; } diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index 18d0ecf6b2f6e..f9eb131a4f30e 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -1996,7 +1996,7 @@ static int ane_rtclient_attach_genpd(struct ane_rtclient *ane) pd->count = i + 1; } - dev_emerg(dev, "BOOT-PHASE genpd domains attached: %d\n", count); + dev_dbg(dev, "BOOT-PHASE genpd domains attached: %d\n", count); out: mutex_unlock(&ane_rtclient_pd_lock); return err; @@ -2085,7 +2085,7 @@ static int ane_rtclient_probe(struct platform_device *pdev) "pmgr window map failed; G1 gate cannot run\n"); } ps_cpu = readl(ane->pmgr + ane->soc->ps_cpu_off); - dev_emerg(dev, "ane_cpu ACTUAL = 0x%x\n", ps_cpu); + dev_dbg(dev, "ane_cpu ACTUAL = 0x%x\n", ps_cpu); if (FIELD_GET(ANE_PS_ACTUAL, ps_cpu) != ANE_PS_ON) { pm_runtime_put_sync_suspend(dev); pm_runtime_disable(dev); @@ -2100,7 +2100,7 @@ static int ane_rtclient_probe(struct platform_device *pdev) u32 gate = IS_ERR(set) ? U32_MAX : readl(set + ane->soc->pwgate_off); - dev_emerg(dev, "PWGATE = 0x%x\n", gate); + dev_dbg(dev, "PWGATE = 0x%x\n", gate); if (gate & GENMASK(29, 28)) { pm_runtime_put_sync_suspend(dev); pm_runtime_disable(dev); @@ -2111,9 +2111,9 @@ static int ane_rtclient_probe(struct platform_device *pdev) cpu_status = readl(ane->engine + ANE_ASC_CPU_STATUS); rvbar = readq(ane->engine + ANE_ASC_RVBAR); - dev_emerg(dev, - "BOOT-PHASE engine reads ok: CPU_STATUS = 0x%x, RVBAR = %016llx (bit0=%u)\n", - cpu_status, rvbar, (u32)(rvbar & 1)); + dev_dbg(dev, + "BOOT-PHASE engine reads ok: CPU_STATUS = 0x%x, RVBAR = %016llx (bit0=%u)\n", + cpu_status, rvbar, (u32)(rvbar & 1)); if (!(cpu_status & ANE_ASC_CPU_STATUS_RUNNING)) { if (!fw_start) { @@ -2155,7 +2155,7 @@ static int ane_rtclient_probe(struct platform_device *pdev) a->power_gated = true; ane->fw = a; - dev_emerg(dev, "BOOT-PHASE fwload stage+alias begin\n"); + dev_dbg(dev, "BOOT-PHASE fwload stage+alias begin\n"); ret = ane_t6021_fwload_probe(a); if (ret) { dev_err_probe(dev, ret, "fw_start: staging failed\n"); @@ -2175,8 +2175,8 @@ static int ane_rtclient_probe(struct platform_device *pdev) return -EINVAL; } - dev_emerg(dev, "BOOT-PHASE dispatch (table_mode=%d)\n", - fw_start_table_mode); + dev_dbg(dev, "BOOT-PHASE dispatch (table_mode=%d)\n", + fw_start_table_mode); ret = ane_t6021_boot_start(a, 0, fw_start_table_mode, fw_start_rtb_mode); if (ret == -ENODATA || ret == -EAGAIN || @@ -2190,10 +2190,10 @@ static int ane_rtclient_probe(struct platform_device *pdev) ane->held = true; cpu_status = readl(ane->engine + ANE_ASC_CPU_STATUS); - dev_emerg(dev, - "BOOT-PHASE sequence returned %pe (cpu_started=%u fw_alive=%u booted=%u) CPU_STATUS=0x%x\n", - ERR_PTR(ret), a->cpu_started, a->fw_alive, a->booted, - cpu_status); + dev_info(dev, + "BOOT-PHASE sequence returned %pe (cpu_started=%u fw_alive=%u booted=%u) CPU_STATUS=0x%x\n", + ERR_PTR(ret), a->cpu_started, a->fw_alive, a->booted, + cpu_status); if (!a->fw_alive && !fw_start_rtb_mode) { dev_err(dev, "fw_start: poll A timeout, no READY — HELD until reboot, RTKit handshake skipped\n"); @@ -2213,9 +2213,9 @@ static int ane_rtclient_probe(struct platform_device *pdev) int hello_ret = 0; dma_wmb(); - dev_emerg(dev, - "LEGACY P8 host ack: SCRATCH3 <- %08x\n", - ANE_T6021_BOOT_ACK); + dev_dbg(dev, + "LEGACY P8 host ack: SCRATCH3 <- %08x\n", + ANE_T6021_BOOT_ACK); /* hello_wait_ms > 0 only: init rtkit and arm the RX * poll worker BEFORE writing the ack, so a HELLO * after the ack is not missed. The 13.5 fw sent none @@ -2241,17 +2241,17 @@ static int ane_rtclient_probe(struct platform_device *pdev) unsigned long hello_deadline = jiffies + msecs_to_jiffies(hello_wait_ms); - dev_emerg(dev, "LEGACY hello: boot begin (%u ms)\n", - hello_wait_ms); + dev_dbg(dev, "LEGACY hello: boot begin (%u ms)\n", + hello_wait_ms); do { hello_ret = apple_rtkit_boot(ane->rtk); } while (hello_ret == -ETIME && time_before(jiffies, hello_deadline)); - dev_emerg(dev, - "LEGACY hello: boot %pe running=%d crashed=%d\n", - ERR_PTR(hello_ret), - apple_rtkit_is_running(ane->rtk), - apple_rtkit_is_crashed(ane->rtk)); + dev_info(dev, + "LEGACY hello: boot %pe running=%d crashed=%d\n", + ERR_PTR(hello_ret), + apple_rtkit_is_running(ane->rtk), + apple_rtkit_is_crashed(ane->rtk)); if (!hello_ret) { ane->boot_done = true; if (start_app_eps) From 5699b3f4b33db8e3cfa32cf03a129cab48e3ed58 Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 14:24:57 -0700 Subject: [PATCH 0598/2680] accel/ane: t6021: keep one MODULE_LICENSE and MODULE_DESCRIPTION ane_t6021.ko is built from ane_t6021_rtclient_main.o, ane_t6021_fwload.o and ane_t6021_boot.o, and each of the three carries its own MODULE_LICENSE() and MODULE_DESCRIPTION(), so modinfo reports three licenses and three descriptions for the one module. The comment on the extra copies says the objects also link into an ane_t6021_rtclient.ko, but the Makefile from Joshua Warren's aurora-silicon/linux#155 builds no such module. Keep the set next to module_init() in ane_t6021_rtclient_main.c and drop the copies, and their comment, from the fwload and boot objects. The MODULE_FIRMWARE() lines stay with the firmware loader. Found in a review of #155. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit 9b8b60c4c8a11c969a547f3f8805664ae176ed72) --- drivers/accel/ane/ane_t6021_boot.c | 6 ------ drivers/accel/ane/ane_t6021_fwload.c | 5 ----- 2 files changed, 11 deletions(-) diff --git a/drivers/accel/ane/ane_t6021_boot.c b/drivers/accel/ane/ane_t6021_boot.c index 1ae368ae3e55f..ec2bb466133c4 100644 --- a/drivers/accel/ane/ane_t6021_boot.c +++ b/drivers/accel/ane/ane_t6021_boot.c @@ -149,12 +149,6 @@ module_param(boot_prevent_nap, bool, 0444); MODULE_PARM_DESC(boot_prevent_nap, "Retain firmware nap-prevention counter via init resource bit (default on: proven add-path value)"); -/* This object links into both ane_t6021.ko and ane_t6021_rtclient.ko; - * per-object metadata keeps modpost happy for either composition. - */ -MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("T6021 ANE contract-pinned boot sequence core"); - /* Boot-write gates — ITEMIZED, each a HARD gate: the ENTIRE write * sequence (preboot engine table, scratch clear + pulse, RVBAR * resolution, CPU release, publication, wake) runs start-to-finish diff --git a/drivers/accel/ane/ane_t6021_fwload.c b/drivers/accel/ane/ane_t6021_fwload.c index c8a7ad8d7bee8..04153a2a4d576 100644 --- a/drivers/accel/ane/ane_t6021_fwload.c +++ b/drivers/accel/ane/ane_t6021_fwload.c @@ -84,11 +84,6 @@ #include "ane_t6021_boot.h" #include "ane_fw_validate.h" -/* This object links into both ane_t6021.ko and ane_t6021_rtclient.ko; - * per-object metadata keeps modpost happy for either composition. - */ -MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("T6021 ANE firmware staging + entry alias"); MODULE_FIRMWARE(ANE_FW_SELENE_NAME); MODULE_FIRMWARE(ANE_FW_BIA_NAME); From f70d339182cf69c255372ed07e27cdc06e83a9a1 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 18:03:39 -0500 Subject: [PATCH 0599/2680] accel/ane: Log register and address tracing at debug level Both modules log bring-up tracing at info level on every probe: the ANERD/ANEWR recovery trace and the ps probe SET window reads in ane_tm.c, the ANE-resume progress lines and the first ps probe in ane_drv.c, one line for each DART in ane_dart.c, and in ane_t6021 each ChMan table entry, each RTKit endpoint start and app message, the CONFIG_GET words, and the firmware alias, PMU map and staging addresses. Make them dev_dbg(). Dynamic debug still shows them for crash attribution. The outcome lines stay at info level: DART containment armed, loaded ane, tm recovered, the ChMan table result, the chip revision, the boot result lines and loaded ane_t6021. Faults stay at error and warning level. No other code changes. After this change, the only info lines on the ane probe path are "DART containment armed" and "loaded ane". Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/accel/ane/ane_dart.c | 4 +- drivers/accel/ane/ane_drv.c | 14 ++--- drivers/accel/ane/ane_t6021_fwload.c | 48 +++++++-------- drivers/accel/ane/ane_t6021_rtclient_main.c | 30 ++++----- drivers/accel/ane/ane_tm.c | 68 +++++++++++---------- 5 files changed, 83 insertions(+), 81 deletions(-) diff --git a/drivers/accel/ane/ane_dart.c b/drivers/accel/ane/ane_dart.c index f66bfbe15af55..0dc0e955182da 100644 --- a/drivers/accel/ane/ane_dart.c +++ b/drivers/accel/ane/ane_dart.c @@ -113,8 +113,8 @@ int ane_dart_init(struct ane_device *ane) dart->irq = irq; dart->sid = args.args_count ? args.args[0] : 0; dart->masked = false; - dev_info(ane->dev, "DART containment: %pOFn sid %u irq %d\n", - args.np, dart->sid, irq); + dev_dbg(ane->dev, "DART containment: %pOFn sid %u irq %d\n", + args.np, dart->sid, irq); put_device(&pdev->dev); of_node_put(args.np); } diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 20e32a76f190a..52e720aad356c 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -1208,13 +1208,13 @@ static int __maybe_unused ane_runtime_resume(struct device *dev) eng = platform_get_resource_byname(pdev, IORESOURCE_MEM, "engine"); - dev_info(dev, - "ANE-resume: genpd raise complete; SET window probe next\n"); + dev_dbg(dev, + "ANE-resume: genpd raise complete; SET window probe next\n"); ane_ps_act_probe(ane); - dev_info(dev, - "ANE-resume: SET window probed; first engine access next (TM_TQ_EN tm+0x0c @ engine %pr + 0x2000c)\n", - eng); + dev_dbg(dev, + "ANE-resume: SET window probed; first engine access next (TM_TQ_EN tm+0x0c @ engine %pr + 0x2000c)\n", + eng); } ane_tm_enable(ane, first); @@ -1223,7 +1223,7 @@ static int __maybe_unused ane_runtime_resume(struct device *dev) * its post-reset status against it. */ if (!ane->tm_status_known) { - dev_info(dev, "ANE-resume: enable writes survived; TM_STATUS read next\n"); + dev_dbg(dev, "ANE-resume: enable writes survived; TM_STATUS read next\n"); ane->tm_status_fresh = ane_tm_status(ane); ane->tm_status_known = true; /* Linux-side pwrstate probe: ACTUAL nibbles read through @@ -1232,7 +1232,7 @@ static int __maybe_unused ane_runtime_resume(struct device *dev) * pmgr SET block with every word on; anything else means * the window does not match the expected word layout. */ - dev_info(dev, "ANERD ps probe act=%#x\n", ane_ps_act(ane)); + dev_dbg(dev, "ANERD ps probe act=%#x\n", ane_ps_act(ane)); } return 0; diff --git a/drivers/accel/ane/ane_t6021_fwload.c b/drivers/accel/ane/ane_t6021_fwload.c index 04153a2a4d576..44be0d3e72ecc 100644 --- a/drivers/accel/ane/ane_t6021_fwload.c +++ b/drivers/accel/ane/ane_t6021_fwload.c @@ -243,8 +243,8 @@ static int ane_t6021_pmu_map(struct ane_t6021 *ane, struct iommu_domain *dom) prot, GFP_KERNEL); if (!ret && iommu_iova_to_phys(dom, soc->pmu_pa) != soc->pmu_pa) ret = -EIO; - dev_info(ane->dev, "pmu: DART map %#llx (IOVA == PA, %#x bytes): %d\n", - soc->pmu_pa, ANE_T6021_FW_ALIAS_PAGE, ret); + dev_dbg(ane->dev, "pmu: DART map %#llx (IOVA == PA, %#x bytes): %d\n", + soc->pmu_pa, ANE_T6021_FW_ALIAS_PAGE, ret); return ret; } @@ -265,9 +265,9 @@ static int ane_t6021_fw_alias_map(struct ane_t6021 *ane, bool reserved) /* Unlatched branch: the boot path programs RVBAR to the * fw DVA itself (ane_t6021_rvbar_compose), no alias. */ - dev_info(ane->dev, - "fwalias: rvbar %016llx not latched/entry 0 — skip (boot reprograms RVBAR)\n", - rvbar); + dev_dbg(ane->dev, + "fwalias: rvbar %016llx not latched/entry 0 — skip (boot reprograms RVBAR)\n", + rvbar); return 0; } if (!ane_t6021_rvbar_entry_ok(entry) || @@ -366,14 +366,14 @@ static int ane_t6021_fw_alias_map(struct ane_t6021 *ane, bool reserved) } ane->fw_alias_extn = windows; ane->fw_alias_iova = entry; - dev_info(ane->dev, - "fwalias: reserved SEG0/SEGi at entry %#llx (%llx+%zx %llx+%zx, preloaded placement)\n", - entry, - ane->fw_alias_ext_iova[0], ane->fw_alias_ext_len[0], - ane->fw_alias_ext_iova[1], ane->fw_alias_ext_len[1]); + dev_dbg(ane->dev, + "fwalias: reserved SEG0/SEGi at entry %#llx (%llx+%zx %llx+%zx, preloaded placement)\n", + entry, + ane->fw_alias_ext_iova[0], ane->fw_alias_ext_len[0], + ane->fw_alias_ext_iova[1], ane->fw_alias_ext_len[1]); if (fw_extra_ram) - dev_info(ane->dev, "fwalias: owned heap [%#llx,%#llx) roundtrip verified\n", - win[2].iova, win[2].iova + win[2].len); + dev_dbg(ane->dev, "fwalias: owned heap [%#llx,%#llx) roundtrip verified\n", + win[2].iova, win[2].iova + win[2].len); return ane_t6021_pmu_map(ane, dom); err_unmap_mapped: @@ -440,10 +440,10 @@ static int ane_t6021_fw_alias_map(struct ane_t6021 *ane, bool reserved) ane->fw_alias_ext_iova[0] = entry; ane->fw_alias_ext_len[0] = ane->fw_size; ane->fw_alias_extn = 1; - dev_info(ane->dev, - "fwalias: entry %#llx <- %u dart pages aliased from fw %pad (first %pa, roundtrip OK)\n", - entry, ane->fw_size / ANE_T6021_FW_ALIAS_PAGE, - &ane->fw_iova, &pa0); + dev_dbg(ane->dev, + "fwalias: entry %#llx <- %u dart pages aliased from fw %pad (first %pa, roundtrip OK)\n", + entry, ane->fw_size / ANE_T6021_FW_ALIAS_PAGE, + &ane->fw_iova, &pa0); /* The firmware's power service needs the pmgr sub-block mapped * IOVA == PA in every vehicle (NO PMD FOR IOVA 0x28e084008, * 2026-09-29); the staged-DMA alias branch skipped it and left @@ -527,11 +527,11 @@ static int ane_t6021_fw_patch(struct ane_t6021 *ane, u8 *img) p.soc_revision); return -EINVAL; } - dev_info(ane->dev, - "fwload: own memory: iBoot patches replayed (soc %#x rev %#x DATA %#llx cpu %#llx wrapper %#llx)\n", - p.soc, p.soc_revision, - p.exec_base + soc->fw->segs[1].vmaddr, p.cpu_pa, - p.wrapper_pa); + dev_dbg(ane->dev, + "fwload: own memory: iBoot patches replayed (soc %#x rev %#x DATA %#llx cpu %#llx wrapper %#llx)\n", + p.soc, p.soc_revision, + p.exec_base + soc->fw->segs[1].vmaddr, p.cpu_pa, + p.wrapper_pa); return 0; } @@ -595,9 +595,9 @@ int ane_t6021_fwload_probe(struct ane_t6021 *ane) ane->fw_iova = iova; ane->fw_size = alloc_size; - dev_info(ane->dev, - "fwload: %s PRELOAD validated + DART-mapped: entry %#llx, iova %pad size %#x\n", - img->name, entry, &iova, alloc_size); + dev_dbg(ane->dev, + "fwload: %s PRELOAD validated + DART-mapped: entry %#llx, iova %pad size %#x\n", + img->name, entry, &iova, alloc_size); ret = reserved ? 0 : ane_t6021_fw_patch(ane, buf); if (!ret) diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index f9eb131a4f30e..be21e4c1b0eb1 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -464,12 +464,12 @@ static void ane_rtclient_validate_chman(struct ane_rtclient *ane) const struct ane_t6021_chman_desc *d = &t[i]; const struct ane_t6021_chman_static *s = &ane_t6021_chman_layout[i]; - dev_info(ane->dev, - "chman[%u]: name=\"%.*s\" type=%u bit=%u size=%#llx %s (static: %s/%u/%u/%#llx/ipc+%#x)\n", - i, ANE_T6021_CHMAN_NAME_LEN, d->name, d->type, d->bit, - d->size, - (bad & BIT(i)) ? "MISMATCH" : "OK", - s->name, s->type, s->bit, s->size, s->off); + dev_dbg(ane->dev, + "chman[%u]: name=\"%.*s\" type=%u bit=%u size=%#llx %s (static: %s/%u/%u/%#llx/ipc+%#x)\n", + i, ANE_T6021_CHMAN_NAME_LEN, d->name, d->type, d->bit, + d->size, + (bad & BIT(i)) ? "MISMATCH" : "OK", + s->name, s->type, s->bit, s->size, s->off); } ane->chman_ok = !bad; @@ -1800,8 +1800,8 @@ static void ane_rtclient_recv(void *cookie, u8 ep, u64 message) { struct ane_rtclient *ane = cookie; - dev_info(ane->dev, - "rtkit app msg: ep=%#x msg=%016llx\n", ep, message); + dev_dbg(ane->dev, + "rtkit app msg: ep=%#x msg=%016llx\n", ep, message); } static void ane_rtclient_crashed(void *cookie, const void *crashlog, @@ -1901,8 +1901,8 @@ static void ane_rtclient_start_app_eps(struct ane_rtclient *ane) if (!apple_rtkit_has_endpoint(ane->rtk, ep)) continue; ret = apple_rtkit_start_ep(ane->rtk, ep); - dev_info(ane->dev, "rtkit: STARTEP app ep %#x -> %pe\n", - ep, ERR_PTR(ret)); + dev_dbg(ane->dev, "rtkit: STARTEP app ep %#x -> %pe\n", + ep, ERR_PTR(ret)); } } @@ -2295,11 +2295,11 @@ static int ane_rtclient_probe(struct platform_device *pdev) command, 16, 0x03, 1, 3000); - dev_info(dev, - "LEGACY CONFIG_GET words %08x %08x result=%d (DMA remains held)\n", - READ_ONCE(((u32 *)command->cpu)[1]), - READ_ONCE(((u32 *)command->cpu)[2]), - qret); + dev_dbg(dev, + "LEGACY CONFIG_GET words %08x %08x result=%d (DMA remains held)\n", + READ_ONCE(((u32 *)command->cpu)[1]), + READ_ONCE(((u32 *)command->cpu)[2]), + qret); if (qret) { cfg_err = qret; } else if (!READ_ONCE(((u32 *)command->cpu)[2])) { diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index 7a0ade69c395b..52f3781b8cc15 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -92,7 +92,8 @@ u32 ane_ps_act(struct ane_device *ane) /* Bisect probe: per-word named reads of the SET window. Each word logs * before its readl, so a window whose address does not decode on this * SoC is named by the last off-box line (word index + byte offset) - * instead of a silent hard reset between two other prints. + * instead of a silent hard reset between two other prints. The lines + * are debug messages: dynamic debug turns them on. */ u32 ane_ps_act_probe(struct ane_device *ane) { @@ -100,19 +101,19 @@ u32 ane_ps_act_probe(struct ane_device *ane) int i; if (!ane->ps) { - dev_info(ane->dev, "ps probe: SET window unmapped\n"); + dev_dbg(ane->dev, "ps probe: SET window unmapped\n"); return 0; } - dev_info(ane->dev, "ps probe: SET window at %pap, %d words\n", - &ane->ps_base, ANE_PS_WORDS); + dev_dbg(ane->dev, "ps probe: SET window at %pap, %d words\n", + &ane->ps_base, ANE_PS_WORDS); for (i = 0; i < ANE_PS_WORDS; i++) { u32 w; - dev_info(ane->dev, "ps probe: word %d @ ps+0x%02x reading\n", - i, i * 8); + dev_dbg(ane->dev, "ps probe: word %d @ ps+0x%02x reading\n", + i, i * 8); w = readl(ane->ps + i * 8); v |= ((w & ANE_PS_ACTUAL_MASK) >> 4) << (i * 4); - dev_info(ane->dev, "ps probe: word %d -> %#x\n", i, w); + dev_dbg(ane->dev, "ps probe: word %d -> %#x\n", i, w); } return v; } @@ -121,16 +122,17 @@ u32 ane_ps_act_probe(struct ane_device *ane) * every engine read prints its value, each line carrying the pmgr * ACTUAL of the owning partitions. A write that external-aborts the * SoC is then named by the last line the off-box netconsole carried, - * and the bisect starts from that register instead of a guess. + * and the bisect starts from that register instead of a guess. The + * lines are debug messages: dynamic debug turns them on. */ static void ane_rec_writel(struct ane_device *ane, const char *reg, void __iomem *addr, u32 val) { - dev_info(ane->dev, "ANEWR %s <- %#x (ps act %#x)\n", reg, val, - ane_ps_act(ane)); + dev_dbg(ane->dev, "ANEWR %s <- %#x (ps act %#x)\n", reg, val, + ane_ps_act(ane)); writel(val, addr); - dev_info(ane->dev, "ANEWR %s wrote (ps act %#x)\n", reg, - ane_ps_act(ane)); + dev_dbg(ane->dev, "ANEWR %s wrote (ps act %#x)\n", reg, + ane_ps_act(ane)); } static u32 ane_rec_read32(struct ane_device *ane, const char *reg, @@ -138,8 +140,8 @@ static u32 ane_rec_read32(struct ane_device *ane, const char *reg, { u32 val = readl(addr); - dev_info(ane->dev, "ANERD %s -> %#x (ps act %#x)\n", reg, val, - ane_ps_act(ane)); + dev_dbg(ane->dev, "ANERD %s -> %#x (ps act %#x)\n", reg, val, + ane_ps_act(ane)); return val; } @@ -423,7 +425,7 @@ static int ane_ps_verify_on(struct ane_device *ane) } usleep_range(1000, 2000); } - dev_info(ane->dev, "ANERD ps verify act=%#x err=%d\n", act, err); + dev_dbg(ane->dev, "ANERD ps verify act=%#x err=%d\n", act, err); return err; } @@ -438,26 +440,26 @@ static int ane_pd_cycle(struct ane_device *ane) for (int i = 0; i < ane->pd_count; i++) { pm_runtime_get_noresume(ane->pd_dev[i]); gated = i + 1; - dev_info(ane->dev, - "ANERD pd[%d] %s force_suspend begin (ps act %#x)\n", - i, dev_name(ane->pd_dev[i]), ane_ps_act(ane)); + dev_dbg(ane->dev, + "ANERD pd[%d] %s force_suspend begin (ps act %#x)\n", + i, dev_name(ane->pd_dev[i]), ane_ps_act(ane)); err = pm_runtime_force_suspend(ane->pd_dev[i]); - dev_info(ane->dev, - "ANERD pd[%d] force_suspend -> %d (ps act %#x)\n", - i, err, ane_ps_act(ane)); + dev_dbg(ane->dev, + "ANERD pd[%d] force_suspend -> %d (ps act %#x)\n", + i, err, ane_ps_act(ane)); if (err) break; } if (!err) for (int i = 0; i < ane->pd_count; i++) { - dev_info(ane->dev, - "ANERD pd[%d] %s force_resume begin (ps act %#x)\n", - i, dev_name(ane->pd_dev[i]), - ane_ps_act(ane)); + dev_dbg(ane->dev, + "ANERD pd[%d] %s force_resume begin (ps act %#x)\n", + i, dev_name(ane->pd_dev[i]), + ane_ps_act(ane)); err = pm_runtime_force_resume(ane->pd_dev[i]); - dev_info(ane->dev, - "ANERD pd[%d] force_resume -> %d (ps act %#x)\n", - i, err, ane_ps_act(ane)); + dev_dbg(ane->dev, + "ANERD pd[%d] force_resume -> %d (ps act %#x)\n", + i, err, ane_ps_act(ane)); if (err) break; } @@ -470,14 +472,14 @@ static int ane_pd_cycle(struct ane_device *ane) * the partition gated. */ pm_runtime_get_noresume(ane->dev); - dev_info(ane->dev, "ANERD dev force_suspend begin\n"); + dev_dbg(ane->dev, "ANERD dev force_suspend begin\n"); err = pm_runtime_force_suspend(ane->dev); - dev_info(ane->dev, "ANERD dev force_suspend -> %d\n", err); + dev_dbg(ane->dev, "ANERD dev force_suspend -> %d\n", err); if (!err) { - dev_info(ane->dev, "ANERD dev force_resume begin\n"); + dev_dbg(ane->dev, "ANERD dev force_resume begin\n"); err = pm_runtime_force_resume(ane->dev); - dev_info(ane->dev, "ANERD dev force_resume -> %d\n", - err); + dev_dbg(ane->dev, "ANERD dev force_resume -> %d\n", + err); } pm_runtime_put_noidle(ane->dev); } From 92984bbefdafed12ab04aca480abbd4e61fdef00 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:10:14 -0500 Subject: [PATCH 0600/2680] accel/ane: Power the ANE off when it is idle The ane module takes a runtime PM reference in probe and holds it until remove, so the ANE power domains and its three DARTs stay powered for as long as the driver is bound, also when no process uses the ANE. Drop the probe reference at the end of probe and use runtime PM autosuspend. The device suspends autosuspend_ms (default 1500) after the last file open, ioctl or close, and the next one resumes it through ane_runtime_resume(), which enables the task manager again. The new module parameter autosuspend_ms=0 keeps the device powered while the driver is bound. power/autosuspend_delay_ms and power/control change the behavior at run time. Every path that touches the engine or the ANE's own DART registers now holds a reference: - the ioctls, through one wrapper for the native and the 32-bit compat entry points, so both get the command filter and the reference; - the reset sysfs attribute, which power-cycles the partitions and then writes the engine; - the file release, which frees the BOs of the file (each unmap flushes the DART TLBs) and can run the recovery in postclose. It keeps a reference to the struct device, because drm_release() can free the ane_device; - remove, which unmaps the remaining BOs and detaches the power domains. apple-dart saves and restores the stream TCR and TTBR registers in its own runtime PM callbacks, and the ANE is its consumer through a DL_FLAG_PM_RUNTIME device link, so the IOMMU state survives the suspend. Measurements on hardware are pending. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/accel/ane/ane_drv.c | 90 +++++++++++++++++++++++++++++++++---- drivers/accel/ane/ane_tm.c | 6 +-- 2 files changed, 84 insertions(+), 12 deletions(-) diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 52e720aad356c..2147d6a5e7d83 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -66,6 +66,19 @@ module_param(stats, bool, 0444); MODULE_PARM_DESC(stats, "Expose ane_stats sysfs and ane_timeline debugfs (default 1; 0 = no files and the hot path skips the counters)"); +/* + * Runtime PM autosuspend delay. The ANE power domains and its DARTs go + * off this long after the last file open, ioctl or close, and the next + * one powers them up again through ane_runtime_resume(). 0 keeps the + * device powered while the driver is bound. The live value is + * power/autosuspend_delay_ms, and "on" in power/control keeps the + * device powered without a reload. + */ +static int autosuspend_ms = 1500; +module_param(autosuspend_ms, int, 0444); +MODULE_PARM_DESC(autosuspend_ms, + "Power the ANE off this many ms after its last use (default 1500; 0 = keep it powered while the driver is bound)"); + #define CMD_BUF_BDX 0 #define KRN_BUF_BDX 1 @@ -624,10 +637,18 @@ static ssize_t reset_store(struct device *dev, struct device_attribute *attr, struct ane_device *ane = dev_get_drvdata(dev); int err = 0; + /* Recovery power-cycles the partitions and then writes the engine: + * hold a reference so that autosuspend cannot gate it in between. + */ + err = pm_runtime_resume_and_get(dev); + if (err < 0) + return err; + mutex_lock(&ane->engine_lock); if (!ane->removed) err = ane_tm_recover(ane); mutex_unlock(&ane->engine_lock); + pm_runtime_put_autosuspend(dev); if (err) return err; return count; @@ -688,7 +709,7 @@ static int ane_drm_open(struct drm_device *drm, struct drm_file *file) if (err < 0) return err; - pm_runtime_put(ane->dev); + pm_runtime_put_autosuspend(ane->dev); return 0; } @@ -741,8 +762,10 @@ static void ane_drm_postclose(struct drm_device *drm, struct drm_file *file) * from before DRM_ACCEL_FOPS landed drm_ioctl defaults), so this is a * belt-and-suspenders gate on top of the existing ioctls table. */ -static long ane_drm_unlocked_ioctl(struct file *file, unsigned int cmd, - unsigned long arg) +static long ane_drm_ioctl(struct file *file, unsigned int cmd, + unsigned long arg, + long (*dispatch)(struct file *, unsigned int, + unsigned long)) { struct drm_file *filp = file->private_data; struct drm_device *drm = filp->minor->dev; @@ -770,10 +793,47 @@ static long ane_drm_unlocked_ioctl(struct file *file, unsigned int cmd, if (err < 0) return err; - err = drm_ioctl(file, cmd, arg); + err = dispatch(file, cmd, arg); + + pm_runtime_put_autosuspend(ane->dev); + + return err; +} + +static long ane_drm_unlocked_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + return ane_drm_ioctl(file, cmd, arg, drm_ioctl); +} + +/* 32-bit callers get the same command filter and PM reference. */ +#ifdef CONFIG_COMPAT +static long ane_drm_compat_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + return ane_drm_ioctl(file, cmd, arg, drm_compat_ioctl); +} +#else +#define ane_drm_compat_ioctl NULL +#endif - pm_runtime_put(ane->dev); +/* + * Hold power across the close: drm_release() frees the BOs of the file, + * and each unmap flushes the DART TLBs. With the device suspended, + * apple-dart would power the DARTs up and down again for every page. + * The device reference outlives drm_release(), which can drop the last + * drm_device reference and free ane. + */ +static int ane_drm_release(struct inode *inode, struct file *file) +{ + struct drm_file *priv = file->private_data; + struct device *dev = get_device(priv->minor->dev->dev); + int err; + pm_runtime_get_sync(dev); + err = drm_release(inode, file); + pm_runtime_put_autosuspend(dev); + put_device(dev); return err; } @@ -823,9 +883,9 @@ static const struct file_operations ane_drm_fops = { .owner = THIS_MODULE, .fop_flags = FOP_UNSIGNED_OFFSET, .open = accel_open, - .release = drm_release, + .release = ane_drm_release, .unlocked_ioctl = ane_drm_unlocked_ioctl, - .compat_ioctl = drm_compat_ioctl, + .compat_ioctl = ane_drm_compat_ioctl, .poll = drm_poll, .read = drm_read, .llseek = noop_llseek, @@ -1104,9 +1164,13 @@ static int ane_platform_probe(struct platform_device *pdev) * supplier links for several), and .runtime_resume then does the * first engine MMIO. Marking the device active up front skips that * resume, and on T6001 the engine window external-aborts while its - * partition is gated. The reference is held until remove: the device - * stays powered for the whole lifetime, autosuspend stays disabled. + * partition is gated. Probe drops its reference at the end, and the + * device autosuspends autosuspend_ms after its last use. A negative + * delay (autosuspend_ms=0) makes the PM core keep it powered. */ + pm_runtime_set_autosuspend_delay(dev, + autosuspend_ms > 0 ? autosuspend_ms : -1); + pm_runtime_use_autosuspend(dev); pm_runtime_enable(dev); err = pm_runtime_resume_and_get(dev); if (err < 0) @@ -1128,12 +1192,14 @@ static int ane_platform_probe(struct platform_device *pdev) dev_info(dev, "loaded ane\n"); + pm_runtime_put_autosuspend(dev); return 0; put_pm: pm_runtime_put_noidle(dev); disable_pm: pm_runtime_disable(dev); + pm_runtime_dont_use_autosuspend(dev); drm_mm_takedown(&ane->mm); detach_genpd: ane_boost_exit(ane); @@ -1146,6 +1212,11 @@ static void ane_platform_remove(struct platform_device *pdev) struct ane_device *ane = platform_get_drvdata(pdev); struct ane_bo *bo, *tmp; + /* Powered and pinned for the teardown: the unmaps below flush the + * DART TLBs, and no autosuspend may race the genpd detach. + */ + pm_runtime_get_sync(ane->dev); + mutex_lock(&ane->engine_lock); ane->removed = true; drm_dev_unplug(&ane->drm); @@ -1166,6 +1237,7 @@ static void ane_platform_remove(struct platform_device *pdev) ane_detach_genpd(ane); pm_runtime_disable(ane->dev); + pm_runtime_dont_use_autosuspend(ane->dev); pm_runtime_put_noidle(ane->dev); mutex_unlock(&ane->engine_lock); } diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index 52f3781b8cc15..703d595b612a9 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -467,9 +467,9 @@ static int ane_pd_cycle(struct ane_device *ane) pm_runtime_put_noidle(ane->pd_dev[gated]); } else { /* Pin a second usage ref for the cycle: force_suspend only - * marks needs_force_resume when a ref beyond the probe one - * is held, and without that mark force_resume would leave - * the partition gated. + * marks needs_force_resume when a ref beyond the caller's + * one (submit ioctl or reset write) is held, and without + * that mark force_resume would leave the partition gated. */ pm_runtime_get_noresume(ane->dev); dev_dbg(ane->dev, "ANERD dev force_suspend begin\n"); From b42e634bc3e4df87423e6f4f3fb8db046aea9215 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:06:51 -0500 Subject: [PATCH 0601/2680] accel/ane: Map a buffer object with one DART TLB sync BO_INIT maps a buffer object one page at a time with iommu_map(). Each call syncs the TLB of every DART in the domain, so a buffer of N pages costs 3 * N TLB invalidates on the three ANE DARTs. A 458 MB buffer has 27,955 pages and needs 83,865 invalidates. Map the buffer with one iommu_map_sgtable() instead. iommu_map_sg() writes the PTEs of each physically contiguous run and syncs the TLBs once for the whole range. Keep the old per-page path behind the new module parameter map_batch (default 1; 0 selects the old path). BOs mapped either way unmap the same way. Keep the stray PTE recovery on both paths. iommu_map_sg() unmaps what it mapped before it returns an error, so after -EEXIST every valid PTE left in the reserved range is a stray: clear the range page by page and try once more. A short return would give a BO whose tail has no PTEs, so clear the range page by page and fail BO_INIT with -EIO. The page by page clear also serves the stale-mapping purge in probe. Measured with map_batch=1 against map_batch=0, with bit-exact program outputs and exact job counts: M1 (T8103) laptop: BO_INIT 179.85 ms -> 22.77 ms, cold open 165 to 173 ms faster M1 Max (T6001) laptop: BO_INIT 232.7 ms -> 24.7 ms, open 787.9 ms -> 574.3 ms A stress run was clean on both. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/accel/ane/ane_drv.c | 197 +++++++++++++++++++++++++----------- 1 file changed, 137 insertions(+), 60 deletions(-) diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 2147d6a5e7d83..70f0d9bd9b365 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -79,6 +80,18 @@ module_param(autosuspend_ms, int, 0444); MODULE_PARM_DESC(autosuspend_ms, "Power the ANE off this many ms after its last use (default 1500; 0 = keep it powered while the driver is bound)"); +/* + * 1 (default): BO_INIT maps a buffer object with one iommu_map_sg(), so + * apple-dart invalidates the TLB of each DART once for each BO. 0: one + * iommu_map() and one TLB invalidate on each DART for each page (a + * 458 MB buffer: 27,955 pages x 3 DARTs = 83,865 invalidates). Read + * once for each BO_INIT; BOs mapped either way unmap the same way. + */ +static bool map_batch = true; +module_param(map_batch, bool, 0644); +MODULE_PARM_DESC(map_batch, + "BO_INIT DART mapping: 1 = one TLB sync for each buffer object (default); 0 = one TLB sync for each page"); + #define CMD_BUF_BDX 0 #define KRN_BUF_BDX 1 @@ -102,63 +115,44 @@ static struct ane_bo *bo_lookup(struct drm_file *file, u32 handle) return to_bo(gem); } -/* - * Buffers are mapped and unmapped only through the kernel-owned IOMMU - * domain the device was attached to by its providers. apple-dart - * programs every DART in the device's "iommus" list (TTBRs, stream - * setup, invalidation and fault IRQs are all provider-owned); - * iommu_unmap() flushes through the provider before returning, so pages - * may be reclaimed once it succeeds. - */ -static int ane_iommu_map_pages(struct ane_device *ane, struct ane_bo *bo) +/* Unmap every valid PTE in [start, end) and return how many there were. */ +static unsigned long ane_iommu_clear_range(struct ane_device *ane, u64 start, + u64 end) { - int err; + unsigned long cleared = 0; - lockdep_assert_held(&ane->engine_lock); - if (bo->mm) - return -EBUSY; - - bo->mm = kzalloc_obj(*bo->mm); - if (!bo->mm) - return -ENOMEM; - - mutex_lock(&ane->iommu_lock); - - /* reserve area from ANE address space */ - err = drm_mm_insert_node_generic(&ane->mm, bo->mm, - bo->npages << ane->shift, - 1UL << ane->shift, 0, 0); - if (err < 0) { - dev_err(ane->dev, "out of ANE space: %d\n", err); - goto unlock; + for (u64 iova = start; iova < end; iova += 1UL << ane->shift) { + if (!iommu_iova_to_phys(ane->domain, iova)) + continue; + iommu_unmap(ane->domain, iova, 1UL << ane->shift); + cleared++; } + return cleared; +} - bo->iova = bo->mm->start; +/* + * Every PTE inside a reserved drm_mm node must be ours-or-absent: + * scratch drains reserve before mapping and wedge-preserved nodes stay + * inserted. A VALID PTE in a just-reserved range is therefore a stray + * from a torn-down session or a lost unmap. Both map paths clear it once + * and retry instead of poisoning every later BO_INIT until reload + * (m1-test-host 2026-09-25: dart_init_pte -EEXIST at 0x4000 for the rest + * of the session). + */ +static int ane_iommu_map_each(struct ane_device *ane, struct ane_bo *bo, + int prot) +{ + int err = 0; - /* map into ANE address space */ for (u32 i = 0; i < bo->npages; i++) { dma_addr_t iova = bo->iova + (i << ane->shift); - int prot = IOMMU_READ | IOMMU_WRITE; bool healed = false; - int prot_cache = map_mode & 1; retry: - if (prot_cache) - prot |= IOMMU_CACHE; err = iommu_map(ane->domain, iova, page_to_phys(bo->pages[i]), 1UL << ane->shift, prot, GFP_KERNEL); if (err < 0) { dev_err(ane->dev, "iommu_map failed at 0x%llx", iova); - /* Every PTE inside a reserved drm_mm node must be - * ours-or-absent: scratch drains reserve before - * mapping and wedge-preserved nodes stay inserted. - * A VALID PTE in a just-reserved range is therefore - * a stray from a torn-down session or a lost - * unmap. Clear it once and retry instead of - * poisoning every later BO_INIT until reload - * (m1-test-host 2026-09-25: dart_init_pte -EEXIST at - * 0x4000 for the rest of the session). - */ if (err == -EEXIST && !healed && iommu_iova_to_phys(ane->domain, iova)) { dev_warn(ane->dev, @@ -182,26 +176,115 @@ static int ane_iommu_map_pages(struct ane_device *ane, struct ane_bo *bo) bo->iova + (i << ane->shift), unmapped); } - drm_mm_remove_node(bo->mm); - bo->iova = 0; break; } } + return err; +} - mutex_unlock(&ane->iommu_lock); +/* + * iommu_map_sg() writes the PTEs of each physically contiguous run and + * syncs the DART TLBs once for the whole range. On failure it has + * already unmapped everything it mapped, so after an -EEXIST every valid + * PTE left in the range is a stray. + */ +static int ane_iommu_map_batch(struct ane_device *ane, struct ane_bo *bo, + int prot) +{ + size_t size = (size_t)bo->npages << ane->shift; + struct sg_table sgt; + unsigned long strays; + ssize_t mapped; + int err; - if (err < 0) { - kfree(bo->mm); - bo->mm = NULL; + err = sg_alloc_table_from_pages(&sgt, bo->pages, bo->npages, 0, size, + GFP_KERNEL); + if (err) return err; + + mapped = iommu_map_sgtable(ane->domain, bo->iova, &sgt, prot); + if (mapped == -EEXIST) { + strays = ane_iommu_clear_range(ane, bo->iova, bo->iova + size); + if (strays) { + dev_warn(ane->dev, + "cleared %lu stray DART PTE(s) in %#llx+%#zx (no node owns them)\n", + strays, bo->iova, size); + mapped = iommu_map_sgtable(ane->domain, bo->iova, &sgt, + prot); + } } + sg_free_table(&sgt); + /* + * A short map would hand out a BO whose tail has no PTEs. The + * mapped part can have holes, so clear page by page, not with one + * range unmap that stops at the first hole. + */ + if (mapped >= 0 && (size_t)mapped != size) { + ane_iommu_clear_range(ane, bo->iova, bo->iova + size); + mapped = -EIO; + } + + if (mapped < 0) { + dev_err(ane->dev, "iommu_map_sg failed at %#llx+%#zx: %zd\n", + bo->iova, size, mapped); + return mapped; + } return 0; +} + +/* + * Buffers are mapped and unmapped only through the kernel-owned IOMMU + * domain the device was attached to by its providers. apple-dart + * programs every DART in the device's "iommus" list (TTBRs, stream + * setup, invalidation and fault IRQs are all provider-owned); + * iommu_unmap() flushes through the provider before returning, so pages + * may be reclaimed once it succeeds. + */ +static int ane_iommu_map_pages(struct ane_device *ane, struct ane_bo *bo) +{ + int prot = IOMMU_READ | IOMMU_WRITE; + int err; + + lockdep_assert_held(&ane->engine_lock); + if (bo->mm) + return -EBUSY; + + if (map_mode & 1) + prot |= IOMMU_CACHE; + + bo->mm = kzalloc_obj(*bo->mm); + if (!bo->mm) + return -ENOMEM; + + mutex_lock(&ane->iommu_lock); + + /* reserve area from ANE address space */ + err = drm_mm_insert_node_generic(&ane->mm, bo->mm, + bo->npages << ane->shift, + 1UL << ane->shift, 0, 0); + if (err < 0) { + dev_err(ane->dev, "out of ANE space: %d\n", err); + goto unlock; + } + + bo->iova = bo->mm->start; + + if (READ_ONCE(map_batch)) + err = ane_iommu_map_batch(ane, bo, prot); + else + err = ane_iommu_map_each(ane, bo, prot); + if (err < 0) { + drm_mm_remove_node(bo->mm); + bo->iova = 0; + } unlock: mutex_unlock(&ane->iommu_lock); - kfree(bo->mm); - bo->mm = NULL; + if (err < 0) { + kfree(bo->mm); + bo->mm = NULL; + } return err; } @@ -943,17 +1026,11 @@ static int ane_iommu_domain_init(struct ane_device *ane) static void ane_iommu_purge_stale(struct ane_device *ane) { struct drm_mm_node *hole; - u64 start, end, iova; + u64 start, end; unsigned long stale = 0; - drm_mm_for_each_hole(hole, &ane->mm, start, end) { - for (iova = start; iova < end; iova += 1UL << ane->shift) { - if (!iommu_iova_to_phys(ane->domain, iova)) - continue; - iommu_unmap(ane->domain, iova, 1UL << ane->shift); - stale++; - } - } + drm_mm_for_each_hole(hole, &ane->mm, start, end) + stale += ane_iommu_clear_range(ane, start, end); if (stale) dev_warn(ane->dev, "cleared %lu stale DART mappings from a previous instance\n", From 1f85820d1beca4eb813f4f96e34e60e064c3a7f4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:44:55 +1100 Subject: [PATCH 0602/2680] accel/ane: keep the wedge until recovery proves DMA quiescent Closing any ANE file cleared the global wedge before recovery checked it, so the reset was skipped and live DMA backing could be freed or reused. Retry recovery with the wedge and module pin intact. Only the existing successful reset path may clear them; a failed reset requires a reboot and must preserve resources across unrelated file closes. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/accel/ane/ane.h | 9 --------- drivers/accel/ane/ane_drv.c | 24 +----------------------- drivers/accel/ane/ane_tm.c | 23 ----------------------- 3 files changed, 1 insertion(+), 55 deletions(-) diff --git a/drivers/accel/ane/ane.h b/drivers/accel/ane/ane.h index eac81bdab9f74..9a512e7568e60 100644 --- a/drivers/accel/ane/ane.h +++ b/drivers/accel/ane/ane.h @@ -135,15 +135,6 @@ struct ane_request { u32 bar[DRM_ANE_TILE_COUNT]; }; -/* - * Drop the wedge pin (module refcount) without requiring a successful - * recovery. Used by ane_drm_postclose so a wedged engine cannot leave - * the module un-unloadable after a clean session close. Recovery stays - * safe to call afterwards; if it returns error, the postclose path has - * already unlocked the module count so the operator can swap the ko. - */ -void ane_wedge_clear(struct ane_device *ane); - /* * Reclaim wedge-preserved BO ranges (unmap, release nodes and pages). * Only legal once DMA is provably quiescent: after a successful diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 70f0d9bd9b365..7fd0b9e717a91 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -796,21 +796,7 @@ static int ane_drm_open(struct drm_device *drm, struct drm_file *file) return 0; } -/* - * Per-session teardown: every file context open is a fresh boundary. - * Anything the previous session left on the device (wedged flag, tm - * completion counters, in-flight recovery state) belongs to that - * session and must not survive into the next one. - * - * drm_gem_release runs before us and has dropped every handle this - * file held. BOs leaked by `ane_gem_free_object under wedge` stay on - * ane->bo_list until module teardown (their backing `drm_gem_object`s - * are already gone, and touching them here would be use-after-free). - * We do not try to drain them; we only clear the session-leaving - * state (wedged + tm counters + recovering flag) so the next opener - * sees a clean device. The orphan BOs remain inert: they are not in - * any drm_file's handle table. - */ +/* Retry recovery on close, retaining quarantine if DMA cannot be quiesced. */ static void ane_drm_postclose(struct drm_device *drm, struct drm_file *file) { struct ane_device *ane = drm->dev_private; @@ -821,14 +807,6 @@ static void ane_drm_postclose(struct drm_device *drm, struct drm_file *file) return; } - /* Drop the wedge pin first: a wedged engine that cannot be - * recovered in software must NOT keep the module pinned, or the - * operator cannot unload the ko for an updated build without a - * reboot. ane_wedge_clear releases both the wedged flag and the - * module_refcount; ane_tm_recover then attempts a power-cycle - * recovery (succeeds only if the engine is actually idle-able). - */ - ane_wedge_clear(ane); ane_tm_recover(ane); mutex_unlock(&ane->engine_lock); diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index 703d595b612a9..34f5144cea2a9 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -645,26 +645,3 @@ int ane_tm_recover(struct ane_device *ane) return 0; } -/* - * Drop the wedge pin on demand. The recovery path in ane_tm_recover - * does an atomic_xchg + module_put, but only on its success branch - - * a real -110 with the engine stuck leaves ane->wedged set and the - * module refcount one too high, so rmmod fails forever and the bug - * becomes a kernel-brick-without-reboot. ane_drm_postclose calls - * this unconditionally so every session close releases the pin, - * letting the operator unload the ko for an updated build even when - * the engine itself cannot be cleared in software. - * - * Pair count: must follow an ane_wedge_set() that called - * __module_get(THIS_MODULE). Today the only such set is in the wedge: - * branch of ane_tm_execute(); future sets must call this in their - * clean-up path too. - */ -void ane_wedge_clear(struct ane_device *ane) -{ - if (atomic_xchg(&ane->wedged, 0)) { - module_put(THIS_MODULE); - dev_info(ane->dev, - "wedge pin released (engine state unknown)\n"); - } -} From 70f4a6710a1fbe80b3e8f9f92fa7aceda401f793 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:44:57 +1100 Subject: [PATCH 0603/2680] accel/ane: pin H14 EXEC IO buffers through firmware completion The firmware reference flag prevented physical deallocation but did not stop BO_FREE or the last VMA close returning an in-flight IO BO to the global pool. Take one kref per IO element under bo_lock and release all acquired references only after the command completes. Validation errors unwind the same reference budget, and firmware failures latch quarantine before any final put can recycle storage. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/accel/ane/ane_t6021_rtclient_main.c | 31 +++++++++++++-------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index be21e4c1b0eb1..90bb89339486b 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -1314,11 +1314,12 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, const struct drm_ane_exec *user) { struct drm_ane_exec_io *ios; + struct ane_t6021_bo **exec_bos; struct ane_legacy_buffer *command; struct ane_t6021_fd *fd = file->driver_priv; size_t ios_size; size_t cmd_size; - int ret, i; + int ret, i, pinned = 0; if (user->pad || user->count < 1 || user->count > DRM_ANE_MAX_BINDS || user->priority < 2 || user->priority > 7 || !fd) @@ -1346,17 +1347,21 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, return -E2BIG; } + exec_bos = kcalloc(user->count, sizeof(*exec_bos), GFP_KERNEL); + if (!exec_bos) { + kfree(ios); + return -ENOMEM; + } + mutex_lock(&ane_t6021_fw_lock); if (atomic_read(&ane_t6021_quarantined)) { - mutex_unlock(&ane_t6021_fw_lock); - kfree(ios); - return -ETIMEDOUT; + ret = -ETIMEDOUT; + goto unlock; } command = ane->cmd_buf; if (!command) { - mutex_unlock(&ane_t6021_fw_lock); - kfree(ios); - return -ENODEV; + ret = -ENODEV; + goto unlock; } memset(command->cpu, 0, SZ_16K); { @@ -1385,11 +1390,9 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, ret = -EINVAL; goto unlock; } - /* The IOVA below is about to be published to the - * firmware, so mark the BO before the exchange. - * Under bo_lock the handle reference cannot go - * away, so no kref is needed here. - */ + /* Keep this IO element out of the reuse pool until completion. */ + kref_get(&bo->refcount); + exec_bos[pinned++] = bo; bo->fw_ref = true; iova = bo->dma; mutex_unlock(&ane_t6021_bo_lock); @@ -1416,6 +1419,10 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, user->timeout_ms : 5000); } unlock: + /* Command failures latch quarantine before these final puts. */ + while (pinned) + kref_put(&exec_bos[--pinned]->refcount, ane_t6021_bo_release); + kfree(exec_bos); mutex_unlock(&ane_t6021_fw_lock); kfree(ios); return ret; From dd46b05a04db722aa49271d2332a65935be7a318 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:44:58 +1100 Subject: [PATCH 0604/2680] accel/ane: retain fault-drain scratch mappings until quiescence A failed drain freed scratch pages and IOVA reservations before attempting to stop the DMA-active engine. Store the bounded scratch registry on the device, protected by engine_lock, so it survives the request's return. Reclaim it with the existing preserved BO registry only after successful recovery or a safe removal. Normal confirmed-idle completion still releases scratch immediately. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/accel/ane/ane.h | 21 ++++++++++++--------- drivers/accel/ane/ane_drv.c | 2 ++ drivers/accel/ane/ane_tm.c | 16 +++++++--------- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/drivers/accel/ane/ane.h b/drivers/accel/ane/ane.h index 9a512e7568e60..c9d18fb13f521 100644 --- a/drivers/accel/ane/ane.h +++ b/drivers/accel/ane/ane.h @@ -17,6 +17,15 @@ #include "ane_stats.h" +#define ANE_DART_MAX 3 +#define ANE_DART_SCRATCH_MAX 16 + +struct ane_dart_scratch { + u64 iova; + struct page *page; + struct drm_mm_node *node; +}; + struct ane_device { struct drm_device drm; struct device *dev; @@ -62,6 +71,9 @@ struct ane_device { * nodes (see ane_reclaim_preserved). Registered under engine_lock. */ struct list_head preserved_list; + /* Fault-drain DMA backing, retained until completion or a safe reset. */ + struct ane_dart_scratch scratch[ANE_DART_SCRATCH_MAX]; + int scratch_pages; bool removed; /* @@ -147,15 +159,6 @@ void ane_boost_exit(struct ane_device *ane); void ane_boost_begin(struct ane_device *ane); void ane_boost_end(struct ane_device *ane); -#define ANE_DART_MAX 3 -#define ANE_DART_SCRATCH_MAX 16 - -struct ane_dart_scratch { - u64 iova; - struct page *page; - struct drm_mm_node *node; -}; - int ane_dart_init(struct ane_device *ane); void ane_dart_mask(struct ane_device *ane); void ane_dart_unmask(struct ane_device *ane); diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 7fd0b9e717a91..c98f2348e82f1 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -360,6 +360,8 @@ void ane_reclaim_preserved(struct ane_device *ane) unsigned long count = 0; lockdep_assert_held(&ane->engine_lock); + ane_dart_release_scratch(ane, ane->scratch, ane->scratch_pages); + ane->scratch_pages = 0; list_for_each_entry_safe(p, tmp, &ane->preserved_list, list) { u64 iova = p->mm->start; diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index 34f5144cea2a9..ba234d57cdf74 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -265,8 +265,7 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) u32 finished = 0; u32 fault_status = 0; u64 fault_iova = 0; - struct ane_dart_scratch scratch[ANE_DART_SCRATCH_MAX]; - int scratch_pages = 0; + struct ane_dart_scratch *scratch = ane->scratch; bool faulted = false; int err, status; @@ -298,10 +297,10 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) err = status; if (ane_dart_faulted(ane, &fault_iova, &fault_status)) { faulted = true; - if (scratch_pages >= ANE_DART_SCRATCH_MAX) + if (ane->scratch_pages >= ANE_DART_SCRATCH_MAX) break; if (ane_dart_drain_fault(ane, fault_iova, scratch, - &scratch_pages)) + &ane->scratch_pages)) break; /* The retried transaction now completes; poll on. */ continue; @@ -321,7 +320,7 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) if (err || ane_dart_faulted(ane, NULL, NULL)) { /* Drain failed or the drained program never reached * completion: the engine is not idle and access is - * off the table. wedge releases the scratch. + * off the table. Retain scratch backing until reset. */ err = err ?: -EIO; goto wedge; @@ -334,11 +333,12 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) */ rmb(); tq_write32(ane, TQ_STATUS(req->qid), 0x0); - ane_dart_release_scratch(ane, scratch, scratch_pages); + ane_dart_release_scratch(ane, scratch, ane->scratch_pages); ane_dart_unmask(ane); dev_err(ane->dev, "DART fault contained: status=%#x iova=%#llx scratch=%d\n", - fault_status, fault_iova, scratch_pages); + fault_status, fault_iova, ane->scratch_pages); + ane->scratch_pages = 0; return -EIO; } if (err) @@ -356,7 +356,6 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) return 0; wedge: - ane_dart_release_scratch(ane, scratch, scratch_pages); ane_dart_unmask(ane); if (atomic_xchg(&ane->wedged, 1) == 0) { __module_get(THIS_MODULE); @@ -644,4 +643,3 @@ int ane_tm_recover(struct ane_device *ane) } return 0; } - From 85a347dd4ef26dadddffeadae4b1cde80d35a8c0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 17:56:15 +1100 Subject: [PATCH 0605/2680] dt-bindings: npu: apple,ane: describe the T8140 topology T8140 describes two ANE interrupts, three DART clients and a single controllable system power domain. Describe that topology separately from the M1 and M2 requirements without selecting an older firmware interface. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/npu/apple,ane.yaml | 63 +++++++++++++++++-- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/Documentation/devicetree/bindings/npu/apple,ane.yaml b/Documentation/devicetree/bindings/npu/apple,ane.yaml index ebca460a81bc1..79cf942654f7b 100644 --- a/Documentation/devicetree/bindings/npu/apple,ane.yaml +++ b/Documentation/devicetree/bindings/npu/apple,ane.yaml @@ -14,7 +14,8 @@ description: SoCs. It accesses memory through three DARTs. On T8103 and T6000 the host drives the engine through its register window. On T602x and T8112 a coprocessor runs the ANE firmware and the host talks to it through an ASC - mailbox. + mailbox. T8140 has two interrupts and one system power domain; its + coprocessor uses a different firmware interface. properties: compatible: @@ -25,6 +26,7 @@ properties: - apple,t6021-ane - apple,t6022-ane - apple,t8112-ane + - apple,t8140-ane reg: minItems: 1 @@ -45,7 +47,8 @@ properties: - const: fuse interrupts: - maxItems: 1 + minItems: 1 + maxItems: 2 interrupt-names: items: @@ -57,8 +60,8 @@ properties: power-domains: description: The power states the engine needs, five on T8103 and T6000, eight on - T602x and seven on T8112. - minItems: 5 + T602x, seven on T8112 and one system domain on T8140. + minItems: 1 maxItems: 8 resets: @@ -80,11 +83,40 @@ required: - reg - reg-names - interrupts - - interrupt-names - iommus - power-domains allOf: + - if: + properties: + compatible: + contains: + const: apple,t8140-ane + then: + properties: + reg: + maxItems: 1 + reg-names: + maxItems: 1 + interrupts: + minItems: 2 + interrupt-names: false + iommus: + minItems: 3 + power-domains: + maxItems: 1 + resets: false + mboxes: false + memory-region: false + else: + properties: + interrupts: + maxItems: 1 + power-domains: + minItems: 5 + required: + - interrupt-names + - if: properties: compatible: @@ -149,6 +181,27 @@ allOf: additionalProperties: false examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + ane@400000000 { + compatible = "apple,t8140-ane"; + reg = <0x4 0x0 0x0 0x2000000>; + reg-names = "engine"; + interrupt-parent = <&aic>; + interrupts = , + ; + iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + }; + - | #include #include From 0761ae12461eadb8f1d274d57b46a53d67e858b2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 22:12:20 +1100 Subject: [PATCH 0606/2680] accel/ane: Describe the hardware behaviour in comments Rewrite the register, handshake, ring and command-format notes for the T602x/T8112 Neural Engine, and the scratch3_ack module parameter text, so they state the hardware and firmware behaviour directly. Every edit keeps its line count, so line-dependent data (panic locations, WARN/BUG and dynamic-debug records) is unchanged. Apart from the scratch3_ack module parameter description, the change is comments only; there is no functional change. [ryan: split from commit 9266541b8d35 ("treewide: describe hardware behaviour in comments") for the 7.2.10 topics: the accel/ane hunks only; the apple-ave.rst hunk travels with 240-isp, the mailbox hunk with 010-soc, and the SEP, G17, mt7932, reset and other documentation hunks with their own topics] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/accel/ane/ane_fw_validate.h | 12 +- drivers/accel/ane/ane_t6021.h | 230 ++++++++++---------- drivers/accel/ane/ane_t6021_boot.c | 128 +++++------ drivers/accel/ane/ane_t6021_boot.h | 132 +++++------ drivers/accel/ane/ane_t6021_fwload.c | 6 +- drivers/accel/ane/ane_t6021_rtclient_main.c | 16 +- 6 files changed, 262 insertions(+), 262 deletions(-) diff --git a/drivers/accel/ane/ane_fw_validate.h b/drivers/accel/ane/ane_fw_validate.h index f4f723a3f33df..3f8e82978b4de 100644 --- a/drivers/accel/ane/ane_fw_validate.h +++ b/drivers/accel/ane/ane_fw_validate.h @@ -22,7 +22,7 @@ #ifndef __ANE_FW_VALIDATE_H__ #define __ANE_FW_VALIDATE_H__ -/* FWIM surface size = config+0x138 byte-count = 0x500000 (Main +/* FWIM surface size = 0x500000 bytes (Main * 2026-09-20, audit 751caa4, T6021). Covers the selene vmsize 0x4fc000 * and the bia vmsize 0x4ec000; NOT derivable from the blob length. */ @@ -269,10 +269,10 @@ ane_fw_validate_blob(const u8 *blob, size_t size, * whose key <= the chip revision: header {type 1, flags 3 (20-byte * records), capacity 0x24, count, u32 key}, then {u32 offset, u64 mask, * u64 value} per record (iBoot table offsets carry flag bit 30; the - * block does not). Source: iBootStage2 macOS 26.6.2 (25G83), LZFSE, not - * encrypted (receipts/2026-10-01-t8112-optin). The 13.5 iBoot2 that - * preloads the 13.5 image is encrypted; on T6021 its block (the - * captures) equals the 25G83 records exactly. + * block does not). Source: the 25G83 iBoot's records + * (receipts/2026-10-01-t8112-optin). On T6021 the block that the 13.5 + * iBoot leaves in the preloaded image (the captures) equals the 25G83 + * records exactly. */ struct ane_asc_tunable { u32 off; @@ -395,7 +395,7 @@ static const struct ane_asc_tunables ane_t8112_asc_tunables = { }; /* T8112 RTK_soc_revision from its two eFuse words at 0x23d2c8060, as - * iBootStage2 25G83 j413 0x40270 (and j415/j473/j493) computes it: + * iBoot (25G83, on j413 and j415/j473/j493) computes it: * bits 2:0 = w0[29:27], bits 6:4 = {w1[0], w0[31:30]}. */ static inline u32 ane_t8112_fuse_revision(u32 w0, u32 w1) diff --git a/drivers/accel/ane/ane_t6021.h b/drivers/accel/ane/ane_t6021.h index 03d9ee35cdd96..4de6c0dfe6b53 100644 --- a/drivers/accel/ane/ane_t6021.h +++ b/drivers/accel/ane/ane_t6021.h @@ -1,18 +1,18 @@ /* SPDX-License-Identifier: GPL-2.0-only OR MIT */ /* T6021 (H14 / J414c) Apple Neural Engine — skeleton constants and types. * - * Every constant below is static-decode provenance, no device writes: + * Where the constants below come from (none of them needs a device write): * - reg windows, IRQ, pmgr islands: ADT j414c (DeviceTree.j414cap.im4p, * receipts/2026-09-18-t6021-engine-layout-mined §2), Linux translation * +0x200000000 (proven class, pmgr low-32 match). * - ASC cpu block (+0x1400000 h14g) + MBI transport (SCRATCH, channel - * table, +0x1844000 doorbell): K14 kext disasm, W4-fix receipt - * (initializeANESoCConfig h14g blocks + InitializeRTBuddy + - * doorbell setter 0x…95ebdd0); m1n1 ASCRegs = t8103 cross-ref only + * table, +0x1844000 doorbell): the W4-fix receipt plus the live reads + * and the handshake described below; m1n1 ASCRegs = t8103 cross-ref + * only. * - RTKit MGMT protocol: Asahi rtkit.c semantics, u64 message halves as * staged in omarchy-ane rtkit/h14_rtkit_hello.py (commit 6ad26b7). - * - RTBuddy endpoint table + doorbell word: K14 kext cfg table - * __DATA_CONST.__const+0x814e520 and HandleRTBuddyMessage + * - RTBuddy endpoint table + doorbell word: the per-endpoint ring sizes, + * fourccs and doorbell bits listed with ane_t6021_eps below * (receipts/2026-09-18-h14-w2-protocol-decode §3). * * DT binding (driver + packaging/dt/t6021-ane.dts are the two halves): @@ -25,7 +25,7 @@ * fourth window "fuse" (0x23d2c8060+8, chip revision). * reg/reg-names = "engine" (whole 32 MiB ADT range0, 0x284000000; * the H13-style +0x1c04000 engine delta does not exist - * on this SoC — kext never computes it and first touch + * on this SoC — it is never applied here; a first touch * external-aborted, proven 2026-09-18), * "pmgr" (0x28e080000+0x4034, pmgr1,t6021 island words), * "set" (0x28e08c000+0x4000, SET window — read-only by @@ -43,8 +43,8 @@ * ane_cpu last): ane_sys_mpm@4000, ane_td@4008, * ane_base@4010, ane_set1..4@4018-4030, ane_cpu@2e0. * ane_td/ane_base were the W3 chain gap (six consumed, - * block access reset the machine); receipt - * 2026-09-19-h14-init-sequence-kext-trace §3. + * block access reset the machine); see the + * 2026-09-19 init-sequence receipt §3. */ #ifndef __ANE_T6021_H__ @@ -80,20 +80,20 @@ enum { #define ANE_PS_AUTO_ENABLE BIT(28) /* ANE-block-relative ASC/RTBuddy addresses. CPU block = ANE+0x1400000 - * for h14g (K14 initializeANESoCConfig h14g blocks 0x…961365c / - * 0x…9614084 build 0x1400044 with NO 0x200000 orr — the 0x1600044 - * phase-1 §2 quote is the h16g/h17/h18g variant block 0x…9613458; - * runtime h14g-shaped sites 0x…95d2968 write RUN there). CPU_STATUS - * +0x48 keeps the m1n1 ASCRegs +4 shape (config field dev+0x49c = - * 0x1400048; kext poll 0x…95ecfb4). + * for h14g: CPU_CONTROL is 0x1400044, without the 0x200000 offset of + * the h16g/h17/h18g parts (whose block is the 0x1600044 one that the + * phase-1 §2 note quoted). + * RUN is set at 0x1400044 on this part. + * CPU_STATUS +0x48 keeps the m1n1 ASCRegs +4 shape (0x1400048, polled + * while the ASC CPU starts). * * W10 (2026-09-19) CORRECTS the claim that stood here — that the m1n1 * t8103 mailbox (INBOX_CTRL +0x8110 / INBOX0 +0x8800) "has NO h14g * analog". It has one, and it is live: see ANE_ASC_MBOX_* below. The * earlier live test read +0x1608114, the h16g base, which is simply - * the wrong address on this part; and the absence of 0x1408xxx - * constants from the kext text means only that the mailbox belongs to - * the RTBuddy provider kext, not to AppleH11ANEInterface. + * the wrong address on this part. The mailbox belongs to the RTBuddy + * transport rather than to the ANE engine interface, which is why the + * engine-side constants never refer to 0x1408xxx. * * W10 also read the CPU block for the first time, on a Linux boot with * all eight pmgr domains at ACTUAL=0xf and the W8 grant applied: @@ -108,7 +108,7 @@ enum { #define ANE_ASC_RVBAR 0x1050000 /* fw entry | valid bit0 */ #define ANE_ASC_EDPRCR 0x1010310 /* phase1 S2 whitelist */ #define ANE_ASC_VERS 0x1840000 -#define ANE_ASC_RTB_STATUS 0x1840088 /* K14 poll: value < 2 */ +#define ANE_ASC_RTB_STATUS 0x1840088 /* polled: value < 2 */ #define ANE_ASC_RTB_STATUS_UNK7C 0x184007c /* phase1 S2 whitelist */ /* ASC blocks iBoot names in the firmware's __rtk_patch records @@ -164,36 +164,36 @@ enum { #define ANE_FATAL_READ_LO 0x1854000 #define ANE_FATAL_READ_HI 0x1c04000 -/* MBI transport (K14 h14g config: socinit stores the SCRATCH register - * offsets as q-blobs @0x…7503a40/0x…7503a50/0x…7503a60 into dev+0x438 - * and the message-register offsets @0x…7503860 into dev+0x488; runtime - * handshake in InitializeRTBuddy 0x…95e942c: cmd buffer base -> - * SCRATCH0/1 (0x…95eaa94), wake 0xf7fbdff9 -> SCRATCH7 (0x…95eab24), - * poll SCRATCH7 == 0x08042006 "channel description table ready" - * (0x…95eab74), table base read back from SCRATCH0/1 (0x…95ead04), - * per-channel {type,bit,size,phys} entries registered with the - * doorbell setter 0x…95ebdd0 writing (1 << bit) to +0x1844000. Host - * ack = SCRATCH3 = 0x08042006 (0x…95eaee4). +/* MBI transport (h14g): the SCRATCH registers and the message + * registers below are the host side of the handshake. The host writes + * the command buffer base to SCRATCH0/1, then the wake word 0xf7fbdff9 + * to SCRATCH7, and polls until SCRATCH7 == 0x08042006 ("channel + * description table ready"). It then reads the table base back from + * SCRATCH0/1; the table holds per-channel {type,bit,size,phys} + * entries, and a channel is signalled by writing (1 << bit) to the + * +0x1844000 doorbell. + * Host ack = SCRATCH3 = 0x08042006. This driver runs the handshake + * only behind the transport opt-in (struct ane_t6021, transport). + * + * RTBuddy mode: each ANE channel is an RTBuddy endpoint. A message to + * an endpoint is sent by copying it into that endpoint's shared ring + * and then ringing the AP mailbox doorbell (+0x1844000) with the + * endpoint's bit; the firmware's mailbox block has inbox and outbox + * control registers and refuses a message while its inbox is not + * ready or would overflow. + * + * Endpoint number = doorbell bit: EP0 is RTBuddy management (hello, + * endpoint roll call, power acknowledgement), EP1..5 are the ANE data + * channels. The host side of each endpoint is a ring allocated by + * this driver (struct ane_t6021_ep below), with the write cursor + * advanced as described further down. + * + * Responses travel the other way on the fw->host channels (T2F*, + * see ane_t6021_eps), which this driver does not walk yet. * - * PROVIDER DECODE (2026-09-19, kernelcache.release.mac14j): in - * RTBuddy mode the TX gate lives in com.apple.driver.RTBuddy 1.0.0 - * (carved from the KC, 46088 B, __TEXT_EXEC 0x…b696860). RTBuddy - * publishes per-ANE nubs named "%sEndpoint%u" (0x…7cc7d3f) with the - * role string "ANE" -> ANEEndpoint1..ANEEndpoint5, class - * RTBuddyEndpointService (0x98 B) wrapping RTBuddyEndpoint (0xE8 B), - * over the kernel IOSlaveEndpoint family; K14 EnableRTBuddyEndpoints - * (0x…95feb30) formats the name ("%s%d" @0x…74c481f), waits 10 s - * (x1 = 10^10 ns) for the match and takes the gate from [svc+0x88], - * registering its rx callback at gate+0xd0. The gate's per-message - * send is the vtable+0x1e8 slot; its body ends in the AKF mailbox - * write (AKF_AP_MAILBOX_SET = the +0x1844000 doorbell; RTBuddy dump - * fn 0x…b69d9c8 prints "AKF_KIC_INBOX_CTRL / AKF_KIC_MAILBOX_SET / - * AKF_AP_OUTBOX_CTRL / AKF_AP_MAILBOX_SET" and guards - * "INBOX%d not ready"/"Inbox%d overflow"). Endpoint number = - * doorbell bit: EP0 is RTBuddyManagementEndpoint (_handleHello / - * _handleEPRollCall / _handlePowerAck), EP1..5 the ANE data channels. * [INFERENCE: the fw->host doorbell/IRQ bit numbering mirrors the * host->fw SET bit per Asahi rtkit semantics; pinned live by W5.] + * */ #define ANE_MBI_SCRATCH0 0x1840048 /* SCRATCH0..7 = +0x48..+0x64 */ #define ANE_MBI_SCRATCH6 0x1840060 @@ -220,14 +220,14 @@ enum { #define ANE_MBI_MSG_A2I_RD 0x184c000 /* host->fw message read peer */ #define ANE_MBI_MSG_A2I_WR 0x1850000 /* host->fw message write */ -/* Per-message MBI word (48-bit ring notification, K14 - * rtbuddyEndpointSendMessage 0x…95f3bf4-c30): offset = ring write +/* Per-message MBI word (48-bit ring notification, sent once for every + * endpoint message): offset = ring write * cursor [23:0], length [47:24]. NOT the 54-bit surface-announce - * word (ane_ep_doorbell_encode below) — that one rides SetupEndpoints - * buffer mapping; this one is what the gate sends per command: the - * gate call is send(&msg48, 0, 1) at vtable+0x1e8, after the command - * bytes are already memcpy'd into the shared ring at ring_base + - * cursor. Length is capped at 0xffffff by the encoder field. + * word (ane_ep_doorbell_encode below) — that one rides the endpoint + * buffer setup; this one is sent once per command, after the command + * bytes are already copied into the shared ring at ring_base + + * cursor. + * Length is capped at 0xffffff by the encoder field. */ #define ANE_MBI_MSG48_OFF GENMASK_ULL(23, 0) #define ANE_MBI_MSG48_LEN GENMASK_ULL(47, 24) @@ -238,18 +238,18 @@ static inline u64 ane_mbi_msg48_encode(u32 cursor, u32 len) FIELD_PREP(ANE_MBI_MSG48_LEN, len); } -/* Host->fw TX sequence (RTBuddy mode, all static-decode proven): - * 1. fail if len > ring_size (K14 0xe00002c2 @0x…95f3a24) +/* Host->fw TX sequence (RTBuddy mode): + * 1. fail if len > ring_size * 2. cursor = write_cursor; if (cursor + len >= ring_size) cursor = 0 - * (exact fit wraps: csel lo @0x…95f3b04) + * (an exact fit wraps too) * 3. memcpy(ring + cursor, cmd, len) (DMA-coherent ring) * 4. dma_wmb() (ring visible before bell) * 5. write32(ANE_MBI_MSG_A2I_WR, lo) + write32(+4, hi) — 32-bit * halves (the AKF message register is word-shaped; W6 proved the * W5-live abort was NOT the 64-bit writeq — the halves abort too) * 6. write32(ANE_MBI_DOORBELL, 1 << ep) - * 7. on send success only: write_cursor = cursor + len (K14 - * 0x…95f3c70-c); the cursors at rec+0x40/rec+0x58 track the last + * 7. on send success only: write_cursor = cursor + len, so a + * failed send leaves the cursor at the last * issued slot. * CSNE_CMD_PING = header-only 0x11 on EP1 (INIT) -> doorbell bit * 1 << 1 = 0x2. Only the a2i message register + doorbell are @@ -258,9 +258,9 @@ static inline u64 ane_mbi_msg48_encode(u32 cursor, u32 len) * host-writable granted, W9 nonzero latch). */ -/* MBI channel-table entry (kext stride 0x100, fields at +0x40 family: - * type @+0x40, doorbell bit @+0x44, size @+0x48, phys @+0x50 — - * InitializeRTBuddy 0x…95eade8-0x…95eae58) +/* MBI channel-table entry as the firmware publishes it (stride 0x100): + * type @+0x40, doorbell bit @+0x44, size @+0x48, phys @+0x50 within + * each entry. */ #define ANE_MBI_CHAN_STRIDE 0x100 #define ANE_MBI_CHAN_MAX_DUMP 8 @@ -298,7 +298,7 @@ static inline u64 ane_mbi_msg48_encode(u32 cursor, u32 len) #define ANE_RTKIT_EP_OSLOG 8 #define ANE_RTKIT_EP_TRACEKIT 0xa -/* RTBuddy app endpoints — K14 InitializeRTBuddyEndpoints opens ids 1..6; +/* RTBuddy app endpoints — the host opens ids 1..6; * ring sizes + fourccs from the per-EP config table (W2 §3). fourcc is * byte-reversed in the table (0x54324643 = "T2FC"). */ @@ -312,11 +312,11 @@ enum ane_t6021_eps { ANE_T6021_EP_COUNT = ANE_T6021_EP_T2HT + 1 /* arrays index by id */ }; -/* CSNE command ids — selene id->name table, vaddr 0xea430, 96 entries +/* CSNE command ids — the firmware knows 96 of them * (full set: receipts/2026-09-18-h14-w2-protocol-decode §4 and * receipts/2026-09-18-h14-w2-protocol-decode/fw_cmd_table.json). The * fw parses the id as u16 at wire offset +4 (W4 correction of the W2 - * §4 "offset 0" claim: fw sites 0x4d244/0x4d264/0x5246c); the kext + * §4 "offset 0" claim); the * controller header for the fw->host direction is a different shape * (u32 id @ +0x8, 0x24 bytes) and never rides host->fw submission. */ @@ -337,12 +337,12 @@ enum ane_t6021_csne_cmd { CSNE_CMD_BACK_CHANNEL_RPC = 0x7000, }; -/* fw buffer word (selene builder 0x982b4/0x98f84, decoder 0x98fc8; the - * K14 SetupEndpoints word is the same shape): addr[43:0] | +/* fw buffer word (the firmware encodes and decodes buffer words this + * way, and the endpoint setup word has the same shape): addr[43:0] | * size_code[51:44] | unit[53:52], unit 0 = no size, 1 = code*4K, * 2 = code*1M, 3 = code*2M. The fw encoder picks unit 1 below 1 MiB - * and unit 2 at or above, size code = CEILING division (cinc on the - * remainder), so the decoded size is the rounded-up value. rtkit.c's + * and unit 2 at or above, size code = CEILING division (rounded up, + * never down), so the decoded size is the rounded-up value. rtkit.c's * BUFFER_REQUEST is this word with unit 1. */ #define ANE_EP_DOORBELL_OFFSET GENMASK_ULL(43, 0) @@ -373,7 +373,7 @@ struct ane_t6021_ep { const char *name; u32 fourcc; u32 ring_size; - u32 write_cursor; /* next ring slot (K14 rec+0x20) */ + u32 write_cursor; /* next ring slot */ void *ring; /* dma_alloc_coherent, ring_size */ dma_addr_t ring_iova; bool started; @@ -414,7 +414,7 @@ struct ane_t6021 { * live, enabled, permanently-empty ASC v4 control pair. It is * empty because the ASC CPU is STOPPED with RUN clear and RVBAR * entry 0, i.e. no firmware was ever started. Default off = - * status-only bring-up. Opt-in runs the kext-evidenced MBI + * status-only bring-up. Opt-in runs the documented MBI * handshake (SCRATCH wake -> fw channel table), capture-only. */ bool transport; @@ -519,25 +519,25 @@ int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, /* ---- CSNE_CMD wire structs (host->fw on the INIT channel) ---- * - * W4 static decode, extending W2 §4 (the receipt's "fw parses a u16 id - * at header offset 0" was WRONG — three independent fw parse sites read - * the id as u16 at offset +4: the generic CSNE processor @0x4d264, its - * preload @0x4d244, and the LOAD_PROGRAM/CREATE_PROCESS/PROCEDURE_CALL - * pre-parse @0x5246c, all `ldrh wN, [xM, #4]` in selene). Wire is + * W4, extending W2 §4 (the receipt's "fw parses a u16 id + * at header offset 0" was WRONG — the firmware reads + * the id as u16 at offset +4, for generic commands as well as for + * LOAD_PROGRAM/CREATE_PROCESS/PROCEDURE_CALL). + * Wire is * little-endian; this module is arm64-only so host order is wire order. * - * The generic processor takes commands < 0x1b89 bytes (work-item size - * bound @0x4d134-0x4d158) and writes completion state back into the - * command block (strb @0x4d324 byte +6, str @0x4d3b0 qword +8) — the - * ring slot doubles as the response area. The processor's id tree - * sends PING/BUILDINFO/BOOT/REG_FILE_LOAD/IPC_ENDPOINT_SET on to the - * default path (0x4e65c): carried without field parsing, so + * The firmware takes commands < 0x1b89 bytes and writes completion + * state back into the + * command block (byte +6 and the qword at +8) — the + * ring slot doubles as the response area. The firmware handles + * PING/BUILDINFO/BOOT/REG_FILE_LOAD/IPC_ENDPOINT_SET on its + * default path, without parsing their fields, so * their payloads beyond the header are opaque until the W1 live * exchange pins them. */ struct ane_csne_hdr { u32 rsvd0; /* bytes 0..3: never read by the fw processor; 0 */ - u16 id; /* PROVEN u16 @ +4 (sites above) */ + u16 id; /* PROVEN u16 @ +4 (see above) */ u8 flags; /* fw-written byte @ +6 (status/scratch) [INFERENCE] */ u8 rsvd7; }; @@ -551,12 +551,12 @@ static inline void ane_csne_hdr_init(struct ane_csne_hdr *h, u16 id) } /* Header-only CSNE_CMDs: BOOT (0x10 — boot-arg surfaces ride - * SCRATCH0-7 / SetupFWInitBootArgs, not the command, phase1 §2.5), + * SCRATCH0-7 / the firmware init boot args, not the command, phase1 §2.5), * PING (0x11), BUILDINFO (0x06). sizeof(struct ane_csne_hdr) bytes. */ /* REG_FILE_LOAD (0x05) payload: the 1456 B blob is the fw's own - * __DATA._rtk_tunables section (@0x100590, size 0x5b0) — its transport + * _rtk_tunables section (0x5b0 bytes) — its transport * (inline vs shared-memory iova) is [INFERENCE], pinned by W1. */ struct ane_csne_cmd_reg_file_load { @@ -573,21 +573,21 @@ struct ane_csne_cmd_ipc_endpoint_set { u8 payload[]; }; -/* PROCEDURE_CALL (0x204) / INFERENCE_CALL (0x404) — the kext - * CANE_SUB_PACKET_CMD_PROCEDURE_CALL family. Field offsets PROVEN from - * the fw parse; field semantics named only where the kext asserts name - * them (programId, procedureId, numIoBuffers in assert order, W2 §4): - * +0x08/+0x0c u32 pair validated together (ldp @0x4d654) - * +0x10 u64 passed to the fw validator (0x4d6d4) +/* PROCEDURE_CALL (0x204) / INFERENCE_CALL (0x404) — the procedure-call + * command family. Field offsets follow what the firmware accepts; + * field names (programId, procedureId, numIoBuffers, W2 §4) are given + * only where the field's role is known: + * +0x08/+0x0c u32 pair validated together + * +0x10 u64 passed to the fw validator * +0x18 u32 stats/priority, fw-valid [2,7]; 2 is the value * proven end-to-end (binding.json stats_type; the * first-inference receipt's bare calls) - * +0x28 u32 element count (loop bound @0x524bc-0x524c4) - * +0x60 count × 0x30-byte io-buffer records (@0x524a8-0x524b4) - * The kext `size % sizeof(sCSneCmdProcedureCall) == 0` assert says - * commands form same-shape arrays in the ring; this driver submits one + * +0x28 u32 element count + * +0x60 count × 0x30-byte io-buffer records + * Commands are whole multiples of the procedure-call size, so they + * form same-shape arrays in the ring; this driver submits one * command per slot. INFERENCE_CALL shares the shape [INFERENCE: the - * 0x404 id is not in the decoded id tree — it is the W4 submission + * 0x404 id is not among the decoded ids — it is the W4 submission * endpoint per the phase-1 workstream plan]. */ struct ane_csne_io_elem { @@ -622,23 +622,23 @@ ane_csne_cmd_procedure_call_size(unsigned int num_io_buffers) num_io_buffers * sizeof(struct ane_csne_io_elem); } -/* Fw-side bound: the generic CSNE processor rejects work items of - * 0x1b89 bytes and above (@0x4d134 cmp x2, #0x1b89). What x2 names - * beyond "the command's size" is [INFERENCE] — enforced here as a +/* Fw-side bound: the firmware rejects work items of + * 0x1b89 bytes and above. That the bound applies to the whole command + * and not to some other size is [INFERENCE] — enforced here as a * fail-fast so an oversized command cannot enter the ring. */ #define ANE_CSNE_CMD_MAX_SIZE 0x1b88 -/* ---- LOAD_PROGRAM (0x200) wire contract (selene 0x4e44c hook + - * 0x4e53c worker + 0x4e640 registration; decode 2026-09-26, entry - * item 23/23b/23c/23e): the payload is NINE 0x30-byte named section - * records at +0x08. Registration publishes ".bin" into the fw - * program-info symbol table (format "%s.bin" @0xa733f, - * CAneProgramInfo.cpp) with value = record+0x18 and aux = - * u32(record+0x20). The fw dereferences record+0x18 (0x5d0c8), so it +/* ---- LOAD_PROGRAM (0x200) wire contract (2026-09-26, entry item + * 23/23b/23c/23e): the payload is NINE 0x30-byte named section + * records at +0x08. Loading publishes ".bin" for each section + * into the fw program-info symbol table, with value = record+0x18 + * and aux = u32(record+0x20). + * + * The fw dereferences record+0x18, so it * carries a fw-addressable pointer — in this driver, the IOVA of a * host-authored section object (below). Absent sections (flags bit0 - * clear) are skipped by every decoded parser. ---- + * clear) are skipped by the firmware. ---- */ enum ane_t6021_load_section { ANE_SEC_GENERIC = 0, @@ -671,9 +671,9 @@ static_assert(sizeof(struct ane_csne_cmd_load_program) == 0x08 + 9 * 0x30); /* Section-record wire accessors over the 0x30-byte io_elem (the only - * bytes any decoded parser reads; the rest is unread pass-through): + * bytes the firmware reads; the rest is unread pass-through): * +0x00 u8 flags — bit0 present; +0x18 u64 obj — fw-addressable - * pointer, validated by 0x5d0c8 to land inside a registered program + * pointer, which the firmware checks lands inside a registered program * object's entry table; +0x20 u64 key — per-section lookup key. */ #define ANE_SEC_F_PRESENT BIT(0) @@ -689,7 +689,7 @@ static inline void ane_sec_record_init(void *rec, u64 obj, u64 key) } /* Host-authored PROGRAM OBJECT (the thing section records point at; - * 0x5d0c8 validates: magic 1 @+0, count <= 0x10 @+4, table entry + * the firmware validates: magic 1 @+0, count <= 0x10 @+4, table entry * count in [0x201, 0x400] @+0x204, 0x30-byte entry table @+0x208 with * the recorded pointer bounded inside it). Minimum object = header + * 0x201 zeroed entries; entries are the same 0x30-byte shape. @@ -716,7 +716,7 @@ static inline void ane_progobj_init(void *obj, u32 entries) /* caller zeroes the tail: entries table starts at +0x208 */ } -/* Host-authored OPERATION-SECTION image (0x5d290: u32 op_count @+0, +/* Host-authored OPERATION-SECTION image (u32 op_count @+0, * <= 0x80; then op_count 0x40c-byte operation records at +4; the * section KEY is an offset past the array, >= op_count*0x40c + 4). * Operation record fields decoded: +0x00 u32 type <= 4; +0x04 u16 @@ -731,8 +731,8 @@ static inline size_t ane_opsec_size(u32 ops) return 4 + (size_t)ops * ANE_OPSEC_OP_REC_SIZE; } -/* Submit one CSNE command block on the INIT (EP1) ring: K14 - * rtbuddyEndpointSendMessage semantics (W2 §3) — slot alloc with wrap, +/* Submit one CSNE command block on the INIT (EP1) ring: RTBuddy + * endpoint send semantics (W2 §3) — slot alloc with wrap, * memcpy into the ring, 54-bit doorbell word, cursor * advanced only on doorbell success. No synchronous response matching: * fw->host responses arrive on the T2F* channels and are not walked @@ -747,8 +747,8 @@ void ane_t6021_csne_ping_attempt(struct ane_t6021 *ane); /* W13/W14 firmware loader (ane_t6021_fwload.c): validate + stage + * dart-ane0-map the 13.5 PRELOAD payload behind fw_load=1. This is - * the staging half of the boot contract: the Params+0x18 producer - * chain and the RVBAR fold are closed (mapper-callchain audit, commits + * the staging half of the boot contract: the fw DVA source + * and the RVBAR fold are closed (audit commits * 3762aee/12be074); ane_t6021_boot.c consumes the staged surface. */ int ane_t6021_fwload_probe(struct ane_t6021 *ane); @@ -768,7 +768,7 @@ struct ane_asc_tunables; * for this SoC (T6021 only), so fw_alias_reserved=1 may map it; the * other SoCs always run the firmware from driver-owned memory. * ps_cpu_off: the ANE CPU ps word in the DT "pmgr" window (probe guard). - * pwgate_off: the kext's PWGATE word in the DT "set" window, checked + * pwgate_off: the PWGATE word in the DT "set" window, checked * open before any engine read; 0 = not checked. * pmu_pa: the page of the seven ANE ps words; the firmware's power * service writes them through its DART at IOVA == PA. ps_off: the diff --git a/drivers/accel/ane/ane_t6021_boot.c b/drivers/accel/ane/ane_t6021_boot.c index ec2bb466133c4..ccb87ec65b2ff 100644 --- a/drivers/accel/ane/ane_t6021_boot.c +++ b/drivers/accel/ane/ane_t6021_boot.c @@ -14,25 +14,25 @@ * * Blocked boot writes, with the blocking prerequisite each one waits on: * - RVBAR entry fold (ENTRY_BASE | fw DVA & ADDR_MASK; single u64 - * store, width_proof 0x95e9860→0x9622b0c read / 0x95e9988→0x9622c94 - * write): semantics of the observed RVBAR read 0x1 are unresolved + * store; the read and the write are both 64-bit, per width_proof): + * semantics of the observed RVBAR read 0x1 are unresolved * (see the rvbar lifecycle fork) AND the pre-CPU block * below applies regardless. * - CPU_CONTROL write32 0 then 0x10 (rvbar-width local order): waits * on the same pre-CPU block; releasing RUN with an unresolved * entry state is the exact blind start the lane forbids. * - SCRATCH0/1 publication + SCRATCH7 wake: init-structure fields - * [0x08] and [0x10,0x38) have no pinned selene fn 0x71A4 consumer + * [0x08] and [0x10,0x38) have no pinned firmware consumer * read-set; zero-filling unknown fields and calling the structure * valid is forbidden (Main 2026-09-20). * * Pre-CPU prerequisites, updated for pass4 (receipts 2026-09-20 - * mapper-callchain pass4 + tools/check_power_anchors.py 33/33, commit + * pass4 + tools/check_power_anchors.py 33/33, commit * 04630ef): - * RESOLVED (pass4): the CPU-power branch selector — H14g SoCConfig - * zeroes dev+0x3EA0 (0x9613e14, the block that also sets - * dev+0x4A0=0x01400044), so EnableCPUClocksAndPower takes the - * PLAIN branch: write32(PMGR+0x2e0, 0x0000000f), validate 0xff — + * RESOLVED (pass4): the CPU-power branch — the h14g configuration + * selects the PLAIN branch (the same configuration that puts + * CPU_CONTROL at 0x01400044), so the CPU island is powered with + * write32(PMGR+0x2e0, 0x0000000f), validate 0xff — * AUTO_ENABLE bit28 never set. On Linux this END STATE is exactly * what the first_resume stage-2/3 gate verifies (act=0xf, * AUTO_ENABLE clear) before any of this code runs — the write is @@ -41,29 +41,29 @@ * equivalence check. * STILL OPEN (each blocks its write below): * 1. CLOSED (pass5 c364f24): the pre-CPU engine table writels - * eng+0xb38/0xb98/0xbf8 <- 0x01ff01ff are REQUIRED every - * EnableANEClocksAndPower — dev+0x784 bit0 has no writer, so the - * always-open gate runs the loop unconditionally; the + * eng+0xb38/0xb98/0xbf8 <- 0x01ff01ff are REQUIRED every time + * the ANE clocks and power are enabled — they are written + * unconditionally, before the CPU start; the * order-vs-start question is moot. Receiver PROVEN pass4 - * (dev+0x188 accessor = engine reg[0]); count = config+0x150 = + * (the engine register window, reg[0]); count = * 3. Armed below; the first live firing must ride netconsole * behind the seam logs. - * 2. Provider first-enable enableDeviceClock(1,dev+0x8F0)/ - * enableDevicePower(1,&out,dev+0x8F0) (0x95d1d48/0x95d1d90): - * internal gate-ID arrays are runtime-populated (open). Linux + * 2. Provider first-enable of the device clock and power: the + * set of clock and power gates it enables is only known at run + * time (open). Linux * substitutes the genpd raise + supplier links (islands verified * ACTUAL=0xf by the gate); the equivalence datum is the open * half. * 3. RVBAR lifecycle — RESOLVED (M2ResetLifecycle, * rvbar-lifecycle-evidence.json, anchors 38/38 * check_rvbar_lifecycle_anchors.py): live bit0-set + entry 0 is - * the STALE state; the kext-lawful transition is power_off - * (ANE_deInit when dev+0x3FB, then PMGR ps 0x2e0 ← 0x00000000 via - * the dev+0x190 PMGR accessor) → power_on → re-init, NEVER an - * RVBAR write while bit0 is set. RVBAR (engine accessor, + * the STALE state; the valid transition is power_off + * (de-init, then PMGR ps 0x2e0 ← 0x00000000 through the PMGR + * window) → power_on → re-init, NEVER an + * RVBAR write while bit0 is set. RVBAR (engine window, * 0x1050000, u64) is written only after a read64 with bit0 == 0; * CPU_CONTROL write32 0 then 0x10 runs on BOTH paths; success = - * SCRATCH7 (dev+0x454 selector) read32 == 0x08042006. Open edge: + * SCRATCH7 read32 == 0x08042006. Open edge: * whether ps-off clears bit0 — one live read-only read64 after a * domain-off answers it. IMPLEMENTATION CAVEAT: the ps-word write * is the class that froze this host from kernel context @@ -81,37 +81,37 @@ * poll A; then wake 0xf7fbdff9 -> SCRATCH7 releases the fw; poll * B awaits SCRATCH7 DONE 0x08042006 (booted) and reads back * SCRATCH0/1. Because of the pulse, a post-CPU READY is - * unambiguous — no stale-ack hazard. The SCRATCH0 marker (fn - * 0x86EC) is not a gate. - * Selene raw anchors: READY write idx7 @0x7360-0x7388 (movz - * 0x2006 @0x7364, mov w1,#7 @0x7368, movk 0x804 @0x736c, slot - * byte48 @0x7380, salt 0x5bdd @0x7384, blraa @0x7388); poll loop - * READ32 idx7 @0x73c4-0x73e4, cmp wake 0xf7fbdff9 @0x73e8, - * sleep(1000) @0x73f8-0x73fc; DONE ack @0x77cc-0x77f0. + * unambiguous — no stale-ack hazard. The SCRATCH0 marker + * is not a gate. + * On the firmware side: it writes READY (0x08042006) to + * SCRATCH7, then polls SCRATCH7 every 1 ms until it reads the + * wake word 0xf7fbdff9, and writes DONE (0x08042006 again) once + * it has booted; the host's poll B waits for that second + * 0x08042006. * 4. Linux allocation map (legacy branch sizes): FWIM surface = - * config+0x138 byte-count = 0x500000 (Main, audit 751caa4); + * 0x500000 bytes (Main, audit 751caa4); * Linux fw_buf is ANE_FW_BUF_SIZE = 0x500000 — the semantic is * honored directly, covering the image vmsize 0x36c000. 'IPC ' - * surface = min(config+4, dev+0x3A70 - * cap) — the numeric cap needs the h14g config blob field map + * surface = a configured size, capped + * — the numeric cap needs the h14g config field map * (open prerequisite). Init suballoc = 0x174 from a Linux-owned - * pool standing in for the kext dev+0x968/dev+0x980 pool (pool + * pool (pool * total size open). RTBuddy FW_INIT sizes (64K…) are the OTHER * branch — never used here. * 5. Init publication: the fw consumes [0x08]..[0x68] (pass5) and * the Linux sources of those fields are not pinned ([0x08] = - * *(dev+0x988+0x18) = the 'IPC ' surface DVA — the surface + * the 'IPC ' surface DVA — the surface * itself is CLOSED (pass5c/5d), its Linux allocation size open * per item 4; [0x10]/[0x18] = config-size terms with - * config+0x138 = image byte-count closed; [0x30] = dev+0x990 + * the image byte-count closed; [0x30] = the * load-progress word) — publication cannot fire until every * fw-read field has a pinned Linux source. * * Read whitelist — every MMIO read below, individually: * - RVBAR eng+0x01050000: read64 width-proven (rvbar-width * width_proof); W8 and W10 live Linux reads, value 0x1 each. - * - CPU_STATUS eng+0x1400048: phase-1 S2 whitelist; kext poll site - * 0x…95ecfb4 (config field dev+0x49c = 0x1400048); W10 live read + * - CPU_STATUS eng+0x1400048: phase-1 S2 whitelist; polled while + * the CPU starts; W10 live read * 0x2a. * - ASC mailbox controls eng+0x1408110/0x1408114: W10 live reads * (both 0x00020001; 600 polls / 30 s, zero changes, zero aborts). @@ -166,23 +166,23 @@ MODULE_PARM_DESC(boot_prevent_nap, * every probe and the driver performs no direct kernel PMGR writes. */ static const bool pf_provider_genpd_strategy = true; -static const bool pf_pool_word0_proven = true; /* CONFIRMED (Main raw + - * Reset 183fd50, chain - * 0x67dc/0x6824/0x68d8/ - * 0x70dc/0x736c): DDM Params +static const bool pf_pool_word0_proven = true; /* CONFIRMED (Reset + * 183fd50, traced through + * the firmware's pool + * setup): DDM Params * word0 = requested bytes = * 0x40000 — sourced, not * synthesized */ static const bool pf_heap_floor_pinned = true; /* CLOSED: floor = - * getPageSize() return = - * DART page size 0x4000 - * (store chain 0x9602e68-90 - * + dart-ane0 page-size - * 0x4000 node, pass6h); + * the DART page size = + * 0x4000 (the dart-ane0 + * node's page-size + * property is 0x4000, + * pass6h); * sourced in boot_sources */ -static const bool pf_dart_page_floor = true; /* CLOSED (Main raw, +static const bool pf_dart_page_floor = true; /* CLOSED ( * pass6h + exact-node * linkage): dtree-j414c.txt * sha256 dd2955…d504, ane0 @@ -207,7 +207,7 @@ static const bool pf_pass6_init_contract = true; /* cd25b46, 87/87 */ */ /* Table-block mode selection (2026-09-20 16:23:07 wedge): * mode 0 = ABORT before any write (accidental-repeat prevention), - * mode 1 = write the table (kext-faithful; selected per-run via the + * mode 1 = write the table (as on macOS; selected per-run via the * rtclient fw_start_table_mode param), * mode 2 = SKIP the table (default: the shipped diagnostic). * W8 write-grant tunables are mode-independent (proven no-abort @@ -230,12 +230,12 @@ static bool ane_t6021_boot_preflight_complete(void) } /* STATIC boot sources — populated per the pinned contract (Main - * authorization 2026-09-20). Object layout (Main raw correction, - * superseding audit 018abdb): the OSValueObject at +0x10 POINTS to - * the Params; Params+0x00 = size, +0x18 = DVA, +0x38 = hostVA. - * dev+0x980 is the 'DDM ' Params pool (size 0x40000); dev+0x968 is a - * SEPARATE DMM manager constructed over the pool hostVA — never - * conflate the two. Runtime fields (fw_dva, ipc_dva, pool_dma) are + * authorization 2026-09-20). The 'DDM ' pool (size 0x40000) and the + * memory manager built on top of it are separate objects, which + * audit 018abdb had confused; the pool's first word is its + * requested size (see pool_word0 below). + * + * Runtime fields (fw_dva, ipc_dva, pool_dma) are * filled by ane_t6021_boot_prepare() at sequence time (after poll A, * per "dynamic allocations occur after READY"); heap_floor stays 0 * here and is supplied together with pf_heap_floor_pinned — NEVER @@ -243,12 +243,12 @@ static bool ane_t6021_boot_preflight_complete(void) * then). */ static const struct ane_t6021_init_sources boot_sources = { - .cfg_size = 0x500000, /* config+0x138 (0x9613da8/ dac) */ + .cfg_size = 0x500000, /* FWIM surface size */ .prev_fw_len = 0, /* first boot; static per reload */ - .heap_floor = 0x4000, /* DART page size: dev+0x3A90 = - * getPageSize() return; node - * authority dart-ane0 page-size - * 0x4000 (exact linkage check + .heap_floor = 0x4000, /* DART page size; node + * authority: the dart-ane0 + * page-size property, 0x4000 + * (exact linkage check * passing) */ .pool_word0 = 0x40000, /* DDM Params word0 = requested @@ -256,8 +256,8 @@ static const struct ane_t6021_init_sources boot_sources = { */ }; -/* Poll A/B bound: the kext polls <=1000 x sleep(1ms) (selene poll - * loop 0x73c4-0x73fc analog; rvbar-lifecycle step 6/10). +/* Poll A/B bound: up to 1000 polls with a 1 ms sleep, the same bound + * the firmware's own handshake loop uses (rvbar-lifecycle step 6/10). */ #define ANE_BOOT_ACK_POLL_US 1000 #define ANE_BOOT_POLL_MS 1000 @@ -310,7 +310,7 @@ static void ane_boot_publish_barrier(void *ctx) { (void)ctx; /* dma_wmb() = dmb oshst on arm64: orders the coherent pool fill - * before the device publish. ORDERING guarantee — not the kext's + * before the device publish. ORDERING guarantee — not a * dsb st (completion). Sufficient for the Linux coherent-DMA + * writel doorbell contract (writel orders prior accesses before * the MMIO store). @@ -321,7 +321,7 @@ static void ane_boot_publish_barrier(void *ctx) static void ane_boot_wait(void *ctx) { (void)ctx; - usleep_range(1000, 1500); /* kext poll: sleep(1000us) */ + usleep_range(1000, 1500); /* 1 ms between polls */ } static void ane_boot_phase(void *ctx, const char *what) @@ -385,7 +385,7 @@ static int ane_t6021_boot_prepare(void *ctx, u32 *lo, u32 *hi) return -ENODATA; /* belt: run() already gated */ /* dynamic reads (post-READY, live cells — never hardcoded): - * read order SCRATCH0 then SCRATCH1 (0x95ea0d8/0x95ea100); + * read order SCRATCH0 then SCRATCH1; * SCRATCH0 >= 0x21 refuses before allocations/publication; * SCRATCH3 = fw extra-heap request; SCRATCH1+1 = ordinal. */ @@ -435,8 +435,8 @@ static int ane_t6021_boot_prepare(void *ctx, u32 *lo, u32 *hi) * 2026-09-25 handshake receipt; a static tick separates "ASC domain * unclocked" from "clocked but silent"). * - * Deliberately NOT read: engine+0x1140008 (the 13.5 stub's VM 0x30c - * store target). Its readability, width and reset value are unproven + * Deliberately NOT read: engine+0x1140008 (a word the 13.5 stub + * writes). Its readability, width and reset value are unproven * and no pre-release baseline exists, so an after-value would not * locate PC (could be preexisting iBoot state, write-only, or * normalized). diff --git a/drivers/accel/ane/ane_t6021_boot.h b/drivers/accel/ane/ane_t6021_boot.h index aba267e453f13..0677dd046172e 100644 --- a/drivers/accel/ane/ane_t6021_boot.h +++ b/drivers/accel/ane/ane_t6021_boot.h @@ -3,29 +3,29 @@ * ane_t6021_boot.h — T6021 legacy-boot contract units (pure, shared by * the kernel sequencer and the userspace regression). * - * Byte anchors (KC kernelcache.release.mac14j sha256 8304156f…, selene - * sha256 9f7915c4…; receipts under ane-linux-experiments/receipts/, - * commits 3762aee + 12be074): - * - RVBAR fold: ANEHWDevice::ANE_Init 0x95e9850–0x95e9988 (K13 - * 0x93203b0–0x93204e0 identical). Read64 of eng+0x01050000; bit0 + * Evidence: live reads and traces under Linux, plus the receipts in + * ane-linux-experiments/receipts/ (commits 3762aee + 12be074), for + * the firmware image with sha256 9f7915c4…. + * + * - RVBAR fold: read64 of eng+0x01050000; bit0 * set skips the write (live jw14m2 read 0x1). Otherwise write64 * ENTRY_BASE | (fw DVA & ADDR_MASK) — single u64 store * (2026-09-20-h14-rvbar-width.json width_proof). - * - fw DVA = ANESharedMemorySurfaceParams("FWIM")+0x18, produced by - * dartMapMemoryDescriptor (vtable slot 0x8a8, genIOVMSegments) for - * the out-param installed at start+0x336c (mapper-callchain audit - * pass1–3). Linux correspondence: dma_alloc_coherent on the ane + * - fw DVA = the device address of the staged firmware ("FWIM") + * surface, mapped through the ANE DART; the RVBAR fold above starts + * the ASC at this address. + * Linux correspondence: dma_alloc_coherent on the ane * platform device is dart-ane0-translated (live iommu group 6, * three apple,t6020-dart streams, DMA domain). * - SCRATCH0/1 u64 split: low32 → 0x01840048 first, high32 → - * 0x0184004c, dsb st before (legacy-init-publication receipt, - * accessor write32 width-proven at 0x9622d4c). + * 0x0184004c, ordered after the writes that fill the published + * data (legacy-init-publication receipt; each half is a write32). * - 0x174 init suballocation: header[0x00] u64 = fw DVA (legacy - * branch, dev+0x780 bit0 clear = observed AppleARMIODevice - * provider), [0x68] u32 = 64 count, template 256 B at - * [0x6C,0x16C) with template+0xC0 = 4 RESOLVED (Main raw anchors - * 0x9612b78/7c/80 on the dev+0x998 template; conditional |=0x10 - * separate, unqualified, not set). + * branch, the one taken on this provider), [0x68] u32 = 64 count, + * template 256 B at [0x6C,0x16C) with template+0xC0 = 4 RESOLVED + * (the template's initial state); the conditional |=0x10 is + * separate, unqualified and not set. The rest of the block is + * zeroed by the caller (see ane_t6021_init_struct_fill). * * This header intentionally includes nothing. Kernel consumers have * already; userspace consumers typedef u8/u32/u64 and @@ -35,7 +35,7 @@ #ifndef __ANE_T6021_BOOT_H__ #define __ANE_T6021_BOOT_H__ -/* RVBAR entry fold (ANE_Init 0x95e9868–0x95e988c). The mask clears +/* RVBAR entry fold (single u64 store, see above). The mask clears * bits 0–10, 48 and 55 — bit 11 IS retained (0xf800 keeps 0x800); the * base supplies bit0 and the fixed upper pattern. */ @@ -47,10 +47,10 @@ */ #define ANE_T6021_CPU_RUN_RELEASE 0x10 -/* SCRATCH7 handshake words (InitializeRTBuddy / legacy ANE_Init - * publication; selene ack writer 0x77c8). +/* SCRATCH7 handshake words (RTBuddy and legacy init publication; the + * firmware writes the ack). */ -/* SCRATCH0 request band (pass9-corrected, Main raw trace): post-READY +/* SCRATCH0 request band (pass9-corrected): post-READY * SCRATCH0 >= 0x21 REFUSES the boot preparation; below 0x21 proceeds * to the MAX/alloc path. */ @@ -59,7 +59,7 @@ #define ANE_T6021_BOOT_WAKE_REQ 0xf7fbdff9U #define ANE_T6021_BOOT_ACK 0x08042006U -/* Init suballocation geometry (ANE_Init 0x95ea710–0x95ea97c; +/* Init suballocation geometry (legacy init path; * legacy-init-publication receipt). */ #define ANE_T6021_INIT_STRUCT_SIZE 0x174 @@ -69,8 +69,8 @@ #define ANE_T6021_INIT_TEMPLATE_OFF 0x6c /* 256 B at [0x6C,0x16C) */ #define ANE_T6021_INIT_TEMPLATE_SIZE 0x100 #define ANE_T6021_INIT_TBIT_OFF 0xc0 /* template-relative */ -/* RESOLVED (Main raw anchors 0x9612b78/7c/80: ldr/orr#4/str on the - * dev+0x998 template); the conditional |=0x10 is a separate, still +/* RESOLVED (the init path ORs 4 into this template word, whose + * initial value is zero); the conditional |=0x10 is a separate, still * unqualified mutation and is NOT set. */ #define ANE_T6021_INIT_TBIT_VAL 0x4 @@ -80,8 +80,8 @@ static inline u64 ane_t6021_rvbar_compose(u64 iova) return ANE_T6021_RVBAR_ENTRY_BASE | (iova & ANE_T6021_RVBAR_ADDR_MASK); } -/* The kext skip condition: bit0 of the RVBAR read (tbnz w0,#0 at - * ANE_Init 0x95e9878; live jw14m2 reads have been 0x1). What a set +/* The skip condition: bit0 of the RVBAR read (a set bit skips the + * fold; live jw14m2 reads have been 0x1). What a set * bit names in boot-mode terms is the open lifecycle fork — see * ane_t6021_boot.c; this predicate only decodes the bit. */ @@ -125,14 +125,14 @@ static inline u64 ane_t6021_scratch64_join(u32 lo, u32 hi) * 68/68). EVERY fw-read field is explicitly sourced from the struct * below — nothing is silently zero: [0x00] FWIM DVA, [0x08] 'IPC ' * surface DVA (Linux alloc, size = ane_t6021_ipc_size), [0x10] - * config+0x138 = 0x500000, [0x18] = 0x10000000 - size, [0x20] obj2 - * DVA, [0x28] w22, [0x30] fw-load progress, [0x50] zext [x23+4], + * cfg_size = 0x500000, [0x18] = 0x10000000 - size, [0x20] heap + * DVA, [0x28] heap size, [0x30] fw-load progress, [0x50] boot ordinal, * [0x58] pool DMA base, [0x60] pool word0, [0x68] count 64, * template[0x00] = 0, template+0xC0 = 4. The struct values pin BEFORE * the preflight gate can open (Main: no unresolved field silently * zero; no partial boot). The caller MUST zero the full 0x174 block - * first (coherent alloc): gaps [0x34..37]/[0x64..67] get no kext - * store and sit inside fw-read u64 units. + * first (coherent alloc): gaps [0x34..37]/[0x64..67] are never + * written and sit inside fw-read u64 units. */ /* Init-suballocation contract — pass6 producer semantics * (rvbar-lifecycle-evidence.json pass6_init_header_producers, commit @@ -142,25 +142,25 @@ static inline u64 ane_t6021_scratch64_join(u32 lo, u32 hi) * fw_dva -> [0x00] staged selene ('FWIM') surface DVA, * ipc_dva -> [0x08] 'IPC ' surface DVA (its size calc remains * the audit lane's open item), - * cfg_size -> [0x10] config+0x138 = 0x500000; [0x18] = + * cfg_size -> [0x10] = 0x500000; [0x18] = * 0x10000000 - cfg_size = 0x0fb00000, * prev_fw_len -> [0x30] previous fw image length (0 on first boot; * driver-static, updated on each reload), - * heap_floor -> [0x28] floor = config field dev+0x3A90 (value + * heap_floor -> [0x28] floor = the DART page size (value * pinned by the audit lane at preflight close), * pool_dma -> [0x58] the 'DDM ' pool surface DVA (Linux allocates - * its own 0x40000-byte DDM pool — the kext size), + * its own 0x40000-byte DDM pool — the size macOS uses), * pool_word0 -> [0x60] first qword of the Linux pool-descriptor * struct. NOT CLOSED: the fw-side consumer of - * pool+0x00 is untraced and the kext never stores it + * pool+0x00 is untraced and the host never stores it * (free-list head at manager+0 untouched by * Allocate). This value is a HARD-GATED source — the * sequence cannot run until a producer/consumer proof * pins it; no hypothesis value is shipped. * PRODUCER OPEN (Main, superseding b21e23f): the * "pool+0x00 = manager free-list head" claim is - * REJECTED — manager dev+0x968 != Params pool - * dev+0x980 (wrong-object alias likely), and + * REJECTED — the pool manager and the Params pool + * are different objects (wrong-object alias likely), and * Reset6cb048d shows the consumer stores word0 * VERBATIM at image ops 68 (possible extent use). * Wait for the exact same-Params producer. @@ -187,24 +187,24 @@ struct ane_t6021_init_sources { }; /* 'IPC ' surface size (CURRENT_CONTRACT f9602e0; operands proven: - * u64[dev+0x3A90] = IODARTMapper::getPageSize() — 0x4000, one 16K - * DART page, typical; u32[x23+4] = captured SCRATCH1 + 1, the boot + * page_size = the DART page size — 0x4000, one 16K + * DART page, typical; ordinal = captured SCRATCH1 + 1, the boot * ordinal): size = max(page_size, ordinal + 1). Used by the kernel * prepare hook to size the 'IPC ' allocation. */ /* SCRATCH0 read-order note (pass9 phase-1: SCRATCH0 then SCRATCH1 - * after READY): Main raw branch proof SUPERSEDES the audit's - * "SCRATCH0 >= 0x21 arms IPC" reading — 0x95ea1dc b.cc(low) branches - * TO the MAX+IPC-alloc path and >=0x21 goes to logging/error, so + * after READY): later evidence SUPERSEDES the audit's + * "SCRATCH0 >= 0x21 arms IPC" reading — a value below 0x21 leads + * to the MAX+IPC-alloc path and >=0x21 goes to logging/error, so * scratch0 < 33 is likely a VALIDITY gate, not an optional-IPC - * condition. Header [0x08] unconditionally dereferences - * dev+0x988+0x18. Allocation stays UNCONDITIONAL until Reset corrects + * condition. Header [0x08] is always filled with the IPC surface + * DVA. Allocation stays UNCONDITIONAL until Reset corrects * the source; the threshold is NOT implemented. */ static inline u64 ane_t6021_ipc_size(u64 page_size, u32 captured) { /* EXACT u32 semantics: the fw-visible ordinal is the captured - * SCRATCH1 value + 1 in 32-bit arithmetic (kext add w8,w0,#1) — + * SCRATCH1 value + 1 in 32-bit arithmetic (as the firmware sees it) — * it wraps. Callers pass the CAPTURED value, never pre-incremented. * The result is then bounded by ANE_T6021_BOOT_IPC_CEILING at the * allocation site: captured is fw-supplied and untrusted. @@ -250,7 +250,7 @@ ane_t6021_heap_size(u32 scratch3_req, u64 floor, u64 max_size) /* Fill the COMPLETE init suballocation from the static sources plus * the post-READY dynamic values. The caller MUST zero the full 0x174 * block first (coherent alloc): the gap bytes [0x34..37]/[0x64..67] - * get no kext store and sit inside fw-read u64 units. + * are never written and sit inside fw-read u64 units. */ static inline void ane_t6021_init_struct_fill(u8 *buf, const struct ane_t6021_init_sources *s, @@ -290,7 +290,7 @@ ane_t6021_init_struct_fill(u8 *buf, const struct ane_t6021_init_sources *s, buf[ANE_T6021_INIT_COUNT_OFF + 2] = 0; buf[ANE_T6021_INIT_COUNT_OFF + 3] = 0; - /* template[0x00] = config word [dev+0x1D8], pinned value 0 + /* template[0x00] = a config word, pinned value 0 * (pass5d); [0x04..0x0B] zero via the caller's zeroing. */ buf[ANE_T6021_INIT_TEMPLATE_OFF + 0] = 0; @@ -298,8 +298,8 @@ ane_t6021_init_struct_fill(u8 *buf, const struct ane_t6021_init_sources *s, buf[ANE_T6021_INIT_TEMPLATE_OFF + 2] = 0; buf[ANE_T6021_INIT_TEMPLATE_OFF + 3] = 0; - /* template+0xC0 = 4: RESOLVED initial state (Main raw anchors - * 0x9612b78/7c/80). The conditional |=0x10 remains unset. + /* template+0xC0 = 4: RESOLVED initial state (see + * ANE_T6021_INIT_TBIT_VAL). The conditional |=0x10 remains unset. */ buf[ANE_T6021_INIT_TEMPLATE_OFF + ANE_T6021_INIT_TBIT_OFF + 0] = ANE_T6021_INIT_TBIT_VAL; @@ -345,10 +345,10 @@ ane_t6021_boot_prepare_publish(const struct ane_t6021_init_sources *s, ((u64 *)a)[i] = 0; /* SCRATCH0 band check — REFUSAL before allocations/publication - * (Main raw trace: >= 0x21 -> 0x95ea2a0 log -> 0x2c4 -> 0x3330 - * error, w21 = 0xE00002DA; < 0x21 -> 0x214 MAX -> 0x25c alloc -> - * 0x260 -> 0x2c8). Boundary: 0x20 accepted, 0x21 refused. - * Read order: SCRATCH0 first, then SCRATCH1 (0x95ea0d8/0x95ea100). + * (>= 0x21 is logged and fails the boot with an error; < 0x21 + * goes on to the MAX sizing and the allocations). + * Boundary: 0x20 accepted, 0x21 refused. + * Read order: SCRATCH0 first, then SCRATCH1. * Untrusted-input sizing after the band check; heap_size() * returns SIGNED (negative = refused) — kept local so a refusal * cannot wrap into the u64 record. @@ -380,8 +380,8 @@ ane_t6021_boot_prepare_publish(const struct ane_t6021_init_sources *s, return -ENOMEM; } - /* ordinal: exact u32 +1 on the READ SCRATCH1 value (kext add - * w8,w0,#1; callers pass the read value, never pre-incremented). + /* ordinal: exact u32 +1 on the READ SCRATCH1 value (wrapping; + * callers pass the read value, never pre-incremented). * HEADER SOURCES come from the ALLOCATIONS: s->ipc_dva/pool_dma * may be zero in the caller's static sources — fill from the * allocator outputs (a->ipc_dva/a->pool_dva), never from s @@ -477,11 +477,11 @@ struct ane_t6021_boot_cfg { * after the table-base analysis), * 2 = SKIP the table block, run the rest * (diagnostic: tests fw-alive without - * the kext pre-CPU config). + * the pre-CPU table config). */ u64 fw_dva; /* staged surface DVA (fold input) */ int rtb_mode; /* 1 = S1 writes SCRATCH6=0 (RTBuddy/RTKit-app- - * endpoint select, fw 0x42c8) instead of 1 + * endpoint select) instead of 1 * (legacy ChMan/MBI). In RTBuddy mode the fw * may skip READY/DONE; listen on mailbox. * Run contract: P4 READY is recorded, not @@ -544,7 +544,7 @@ ane_t6021_boot_run(const struct ane_t6021_boot_io *io, return -ENODATA; /* pre-CPU engine table (pass4 receiver, pass5 gate: REQUIRED - * every EnableANEClocksAndPower; residual alias/indirect writer + * every clock/power enable; residual alias/indirect writer * risk documented in the preflight list). * LIVE-FAULT 2026-09-20: this block wedged the machine (P0 then * silence — stall could be the FIRST write; P1 never reached; @@ -571,7 +571,7 @@ ane_t6021_boot_run(const struct ane_t6021_boot_io *io, case 2: default: /* diagnostic skip: NO table writes — tests fw-alive - * without the kext pre-CPU config (user override + * without the pre-CPU table config (user override * 2026-09-20). */ io->phase(io->ctx, "P0 table SKIPPED (diagnostic)"); @@ -619,7 +619,7 @@ ane_t6021_boot_run(const struct ane_t6021_boot_io *io, return -ECANCELED; /* tunables done, nothing else fired */ io->phase(io->ctx, "P1 scratch-clear+pulse"); - /* S1: InitANEScratchRegisters — clear ALL cells, SCRATCH6 = rtb_mode ? 0 (RTBuddy) : 1 + /* S1: scratch init — clear ALL cells, SCRATCH6 = rtb_mode ? 0 (RTBuddy) : 1 * (legacy ChMan/MBI); pulse SCRATCH7 1 -> 0 (stale cleared). */ for (i = 0; i < 8; i++) @@ -656,11 +656,11 @@ ane_t6021_boot_run(const struct ane_t6021_boot_io *io, io->phase(io->ctx, "P4 pollA-READY"); /* S4: poll A — FRESH READY (the pulse made it unambiguous). - * RTBuddy mode (SCRATCH6=0, kext fw 0x42c8: rtbuddyFW when - * SCRATCH6 == 0): the fw does not publish BOOT_ACK to SCRATCH7 + * RTBuddy mode (SCRATCH6=0; the firmware runs in RTBuddy mode + * when SCRATCH6 == 0): the fw does not publish BOOT_ACK to SCRATCH7 * and no legacy publication follows — aliveness is gated by the - * RTKit HELLO after boot_run returns (selene InitializeRTBuddy - * 0x95ead04). A missing READY is recorded, not fatal, and + * RTKit HELLO after boot_run returns. + * A missing READY is recorded, not fatal, and * *fw_alive stays 0 — never fabricated. */ for (i = 0; i < ANE_T6021_BOOT_TABLE_POLLS; i++) { @@ -689,11 +689,11 @@ ane_t6021_boot_run(const struct ane_t6021_boot_io *io, * publish barrier, then publish suballoc DVA low32/high32, then * wake. Barrier semantics (Main review 2026-09-20): the kernel * backend's dma_wmb() is dmb oshst — it ORDERS the coherent fill - * before the device publish; it is NOT the kext's dsb st (which + * before the device publish; it is NOT a dsb st (which * additionally waits for completion). Ordering suffices for the * Linux coherent-DMA + writel doorbell contract (writel itself - * orders prior accesses before the MMIO store); the kext dsb st - * choice is a completion guarantee this driver does not claim. + * orders prior accesses before the MMIO store); a dsb st would add + * a completion guarantee this driver does not claim. */ { u32 lo = 0, hi = 0; @@ -725,7 +725,7 @@ ane_t6021_boot_run(const struct ane_t6021_boot_io *io, /* HANDSHAKE COMPLETE != firmware result success: the DONE ack is * the fw's handshake word; the SCRATCH0/1 result u64 it leaves * behind has UNSOURCED success semantics (pass6: the fw writes a - * "result" at 0x77a0 whose meaning is not decoded). Expose it + * "result" whose meaning is not decoded). Expose it * raw; never infer success from booted alone. */ *scratch_result = diff --git a/drivers/accel/ane/ane_t6021_fwload.c b/drivers/accel/ane/ane_t6021_fwload.c index 44be0d3e72ecc..59688070ada62 100644 --- a/drivers/accel/ane/ane_t6021_fwload.c +++ b/drivers/accel/ane/ane_t6021_fwload.c @@ -34,7 +34,7 @@ * RVBAR entry on the device's default DMA domain. Placement * contract, receipt receipts/2026-09-20-t6021-entry-alias.md: * - live RVBAR read64 = 0x10000000001 (bit0 latched, entry - * bits 0x10000000000 = dart-ane vm base); kext law + * bits 0x10000000000 = dart-ane vm base); lifecycle rule * (rvbar-lifecycle 6288b0b): bit0 set => the RVBAR write * branch is skipped, so the ASC fetches AT the latched entry; * - netconsole 2026-09-20T17:48:19: apple-dart 285800000.iommu @@ -193,14 +193,14 @@ bool ane_t6021_fwload_options_ok(void) * die-0 addresses (13.5 ADTs, t602x-ane.dtsi). * * pmu_pa: the firmware's power service programs the seven ANE ps words - * through its DART at IOVA == PA (SetPMUBaseAddress stores the ANE_TD + * through its DART at IOVA == PA (the PMU base it is given is the ANE_TD * word: selene 0x28e084008, bia 0x23b70c010, receipts/2026-10-01-t8112-kit). * macOS maps the page first; without it the first access faults (NO PMD * FOR IOVA 0x28e084008, 2026-09-29) and the firmware halts. * * T8112 (receipts/2026-10-01-t8112-optin): the revision comes from the * eFuse words iBoot reads, the ANE_SYS_CPU word is pmgr +0xc008, and the - * kext (type 0x70) opens PWGATE "set" +0x8b8 before the ps words. The TM + * macOS (type 0x70) opens PWGATE "set" +0x8b8 before the ps words. The TM * TD word is not known there, so trace_td is off. */ const struct ane_t602x_soc ane_t6020_soc = { diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index 90bb89339486b..eb4ae9e383e23 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -103,8 +103,8 @@ #include /* Doorbell (IPI) block, engine + 0x1844000: set +0, pending +0x8000, - * ack +0xc000 (kext aneInterruptHandler reads +0x184c000 and writes - * +0x1850000 with no SoC branch, receipts/2026-10-01-t8112-ane). + * ack +0xc000 (an interrupt is handled by reading +0x184c000 and writing + * +0x1850000, with no SoC difference, receipts/2026-10-01-t8112-ane). */ #define ANE_IPI_OFF 0x1844000 @@ -155,7 +155,7 @@ MODULE_PARM_DESC(start_app_eps, static bool scratch3_ack = true; module_param(scratch3_ack, bool, 0444); MODULE_PARM_DESC(scratch3_ack, - "After DONE, write SCRATCH3 = 0x08042006 (selene 0x7edc, kext 0x95eaee4). Default on; 0 withholds."); + "After DONE, write SCRATCH3 = 0x08042006 (host ack). Default on; 0 withholds."); static bool legacy_only = true; module_param(legacy_only, bool, 0444); @@ -479,7 +479,7 @@ static void ane_rtclient_validate_chman(struct ane_rtclient *ane) /* Legacy ChMan exchange (post-DONE 13.5 transport). Resends to the same * 64-byte IO slot would wedge the ring; cursor advances after each - * completed command (selene fw decodes bit0 = host-owned). + * completed command (the fw treats bit0 as host-owned). */ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, struct ane_legacy_buffer *command, @@ -1672,9 +1672,9 @@ static int ane_t6021_prog_load_ioctl(struct drm_device *drm, void *data, } /* Firmware perf mode: CSNE_CMD_CH_PROPERTY_WRITE (0x1f), channel 0, - * property 0x10aa, value 1. macOS sends it once during power-on (H13 - * kext evidence in the omarchy-ane perf-mode receipt); the selene 13.5 - * fw routes it to CAneEngineExeLoop::setPerfMode. It is a runtime + * property 0x10aa, value 1. macOS sends it once during power-on (see + * the omarchy-ane perf-mode receipt); the 13.5 firmware treats it as + * its engine perf-mode switch. It is a runtime * switch (write 1 to /sys/module/ane_t6021/parameters/fw_perf_mode) so * its effect on call time can be measured on one boot. Only 1 is * accepted: no other value is known to be safe. @@ -2098,7 +2098,7 @@ static int ane_rtclient_probe(struct platform_device *pdev) pm_runtime_disable(dev); return -EPROBE_DEFER; } - /* T8112: the kext (type 0x70) opens PWGATE (bits 29:28 = 0) before + /* T8112: macOS (type 0x70) opens PWGATE (bits 29:28 = 0) before * the ps words. This driver only reads it: an engine read behind a * closed gate is untested, so it refuses first. */ From 96cb1ae92e84de90fa2ca08007db947a358b2bb9 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 2 Oct 2026 23:55:09 -0400 Subject: [PATCH 0607/2680] drm/asahi: use const float bit conversion supported by Rust minimum The kernel requires Rust 1.85, which supports const f32::to_bits. Replace the equivalent transmute and its obsolete lint allowances, including the unnecessary_transmutes lint unknown to Rust 1.85. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 201afd84ae50008095068d028af22ecf4c1356c5) --- drivers/gpu/drm/asahi/float.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs index d58a3d284da12..46c05699d7722 100644 --- a/drivers/gpu/drm/asahi/float.rs +++ b/drivers/gpu/drm/asahi/float.rs @@ -54,11 +54,7 @@ impl F32 { // This must ONLY be used in const context. Use the `f32!{}` macro to do it safely. #[doc(hidden)] pub(crate) const fn from_f32(v: f32) -> F32 { - // Replace with to_bits() after kernel Rust minreq is >= 1.83.0 - #[allow(clippy::transmute_float_to_int)] - #[allow(unnecessary_transmutes)] - // SAFETY: Transmuting f32 to u32 is always safe - F32(unsafe { core::mem::transmute::(v) }) + F32(v.to_bits()) } // Convert an F32 into a `f32` value From d0ecc1864145148743535f54976bc744eb8803d1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:47:26 -0400 Subject: [PATCH 0608/2680] rust: drm: sched: Fix job accounting when cancelling jobs drm_sched_fini() passes each job left on the pending list to cancel_job and frees it right after. Such a job is in one of three states: its hardware fence callback is still armed, the callback has already completed the job, or drm_sched_stop() detached the job from its hardware fence, returning its credits but not its score. cancel_job_cb() treated every job as unfinished and signalled its finished fence with -ECANCELED. For a completed job, that sets an error on a signalled fence. For a job whose callback is still armed, the callback stays armed on a job that is about to be freed, and the job's credits and score are never returned. For a detached job, the score is never returned. drm_sched_fini() stops the scheduler workers, but not the hardware fence, so remove the callback first. Fence callbacks run under the fence lock, so dma_fence_remove_callback() either disarms the callback or waits for it to complete. Then, unless the callback completed the job, return the job's score and any credits drm_sched_stop() has not already returned, as drm_sched_job_done() would have, and signal the finished fence with -ECANCELED. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 12f0a07a61666305a1efe1a1f4c4c88a013abafb) --- rust/kernel/drm/sched.rs | 42 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index ac8f18282ef1b..5d1342c36264f 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -112,13 +112,49 @@ unsafe extern "C" fn free_job_cb(sched_job: *mut bindings::drm_sched unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sched_job) { // SAFETY: All of our jobs are Job. let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; + // SAFETY: A job on the pending list has a valid scheduler fence. + let s_fence = unsafe { (*sched_job).s_fence }; + // SAFETY: `s_fence` is valid and holds a reference to its parent, if any. + let parent = unsafe { (*s_fence).parent }; + // SAFETY: `s_fence` is valid. + let finished = unsafe { addr_of_mut!((*s_fence).finished) }; + + // drm_sched_fini() has stopped the scheduler workers, but the hardware fence can still + // signal concurrently. Fence callbacks run under the fence lock, so removing ours either + // disarms it or waits until it has completed the job. + let callback_removed = !parent.is_null() + // SAFETY: run_job() armed `cb` on `parent` before the submission worker was stopped. + && unsafe { bindings::dma_fence_remove_callback(parent, addr_of_mut!((*sched_job).cb)) }; + + // A job without a parent either completed when run_job() returned, or had its parent + // detached by drm_sched_stop(), which also returned its credits. + let complete_here = callback_removed + // SAFETY: `finished` is a valid, initialized fence. + || (parent.is_null() && unsafe { bindings::dma_fence_get_status(finished) } == 0); // SAFETY: All of our jobs are Job. T::cancel(unsafe { &mut *p }); - let fence = unsafe { Fence::get_raw(&mut (*(*sched_job).s_fence).finished) }; - fence.set_error(ECANCELED); - let _ = fence.signal(); + if complete_here { + // Account for the job exactly as drm_sched_job_done() would have. + // + // SAFETY: The job belongs to the scheduler being torn down, which is still valid. + unsafe { + let sched = (*s_fence).sched; + if callback_removed { + bindings::atomic_sub( + (*sched_job).credits as i32, + addr_of_mut!((*sched).credit_count), + ); + } + bindings::atomic_dec((*sched).score); + } + + // SAFETY: `finished` is a valid fence. + let fence = unsafe { Fence::get_raw(finished) }; + fence.set_error(ECANCELED); + fence.signal(); + } } /// A DRM scheduler job. From b3703fd53d77df5a28104377f17ee8485dc99fe1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:48:27 -0400 Subject: [PATCH 0609/2680] rust: drm: sched: Keep the scheduler alive while jobs reference it drm_sched_entity_destroy() does not necessarily free the jobs still queued on the entity: drm_sched_entity_kill() hands each of them to a work item on the system workqueue, possibly only once the job's last dependency has signalled, and that work calls job->sched->ops->free_job(). The scheduler is embedded in SchedulerInner, which is freed as soon as the last Scheduler or Entity handle is dropped, so dropping a scheduler right after its last entity can leave that work dereferencing freed memory. Let every job hold a reference to the scheduler allocation. Finalizing the scheduler must not wait for those references, since drm_sched_fini() is what cancels and frees the pending jobs, so move it to a separate SchedulerOwner that only the Scheduler and Entity handles reference. The owner is allocated before drm_sched_init(), so that nothing can fail once the scheduler is initialized. The scheduler is now only reachable through shared references while C code keeps mutating it, so store it in an Opaque. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit f8102be460b66b12af865d6250e73ffc6f99c73a) --- rust/kernel/drm/sched.rs | 66 ++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index 5d1342c36264f..56572a5b07ccd 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -11,6 +11,7 @@ use crate::{ prelude::*, sync::{Arc, UniqueArc}, time::{self, msecs_to_jiffies}, + types::Opaque, }; use core::marker::PhantomData; use core::mem::MaybeUninit; @@ -161,6 +162,10 @@ unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sch pub struct Job { job: bindings::drm_sched_job, inner: T, + // Destroying an entity can leave its queued jobs to deferred work that still dereferences + // `job->sched` after the scheduler has been finalized. Keep the scheduler allocation alive + // for as long as any job references it; this does not delay finalization. + _scheduler: Arc>, } impl Deref for Job { @@ -274,14 +279,15 @@ impl<'a> JobFences<'a> { struct EntityInner { entity: bindings::drm_sched_entity, // TODO: Allow users to share guilty flag between entities - sched: Arc>, + sched: Arc>, guilty: bindings::atomic_t, _p: PhantomData, } impl Drop for EntityInner { fn drop(&mut self) { - // SAFETY: The EntityInner is initialized. This will cancel/free all jobs. + // SAFETY: The EntityInner is initialized. This will cancel/free all jobs, possibly from + // deferred work. Each job keeps the scheduler allocation alive until it is freed. unsafe { bindings::drm_sched_entity_destroy(&mut self.entity) }; } } @@ -300,7 +306,7 @@ impl Entity { let mut entity: KBox>> = KBox::new_uninit(GFP_KERNEL | __GFP_ZERO)?; - let mut sched_ptr = &sched.0.sched as *const _ as *mut _; + let mut sched_ptr = sched.0.allocation.sched.get(); // SAFETY: The Box is allocated above and valid. unsafe { @@ -344,33 +350,48 @@ impl Entity { // SAFETY: The Box pointer is valid, and this initializes the inner member. unsafe { addr_of_mut!((*job.as_mut_ptr()).inner).write(inner) }; + // SAFETY: The Box pointer is valid, and this initializes the scheduler reference. + unsafe { + addr_of_mut!((*job.as_mut_ptr())._scheduler).write(self.0.sched.allocation.clone()) + }; + // SAFETY: All fields of the Job are now initialized. Ok(PendingJob(unsafe { job.assume_init() }, PhantomData)) } } -/// DRM scheduler inner data +/// DRM scheduler allocation, kept alive by the scheduler owner and by every job. pub struct SchedulerInner { - sched: bindings::drm_gpu_scheduler, + sched: Opaque, _p: PhantomData, } -impl Drop for SchedulerInner { +// SAFETY: TODO +unsafe impl Sync for SchedulerInner {} +// SAFETY: TODO +unsafe impl Send for SchedulerInner {} + +/// Owner of an initialized DRM scheduler, shared by the scheduler handle and its entities. +/// +/// Jobs only reference the allocation, so that pending jobs cannot keep the scheduler from +/// being finalized. +struct SchedulerOwner { + allocation: Arc>, +} + +impl Drop for SchedulerOwner { fn drop(&mut self) { + let sched = self.allocation.sched.get(); + // SAFETY: The scheduler is valid. This assumes drm_sched_fini() will take care of // freeing all in-progress jobs. - unsafe { bindings::drm_sched_stop(&mut self.sched, core::ptr::null_mut()) }; - unsafe { bindings::drm_sched_fini(&mut self.sched) }; + unsafe { bindings::drm_sched_stop(sched, core::ptr::null_mut()) }; + unsafe { bindings::drm_sched_fini(sched) }; } } -// SAFETY: TODO -unsafe impl Sync for SchedulerInner {} -// SAFETY: TODO -unsafe impl Send for SchedulerInner {} - /// A DRM Scheduler -pub struct Scheduler(Arc>); +pub struct Scheduler(Arc>); impl Scheduler { const OPS: bindings::drm_sched_backend_ops = bindings::drm_sched_backend_ops { @@ -392,11 +413,16 @@ impl Scheduler { ) -> Result> { let mut sched: UniqueArc>> = UniqueArc::new_uninit(GFP_KERNEL)?; + // Allocate the owner up front: nothing may fail once the scheduler is initialized. + let owner: UniqueArc>> = UniqueArc::new_uninit(GFP_KERNEL)?; + + // SAFETY: `sched` was just allocated and is valid for writes. + let sched_ptr = Opaque::cast_into(unsafe { addr_of!((*sched.as_mut_ptr()).sched) }); // SAFETY: zero sched->sched_rq as drm_sched_init() uses it to exit early withoput initialisation // TODO: allocate sched zzeroed instead unsafe { - (*sched.as_mut_ptr()).sched.sched_rq = core::ptr::null_mut(); + (*sched_ptr).sched_rq = core::ptr::null_mut(); }; let init_ops = bindings::drm_sched_init_args { @@ -414,14 +440,10 @@ impl Scheduler { // SAFETY: The drm_sched pointer is valid and pinned as it was just allocated above. // `device` is valid by its type invarants - to_result(unsafe { - bindings::drm_sched_init( - addr_of_mut!((*sched.as_mut_ptr()).sched), - addr_of!(init_ops), - ) - })?; + to_result(unsafe { bindings::drm_sched_init(sched_ptr, addr_of!(init_ops)) })?; // SAFETY: All fields of SchedulerInner are now initialized. - Ok(Scheduler(unsafe { sched.assume_init() }.into())) + let allocation = unsafe { sched.assume_init() }.into(); + Ok(Scheduler(owner.write(SchedulerOwner { allocation }).into())) } } From bb863f71b2e01c6fbd3b694eafcd25277db82010 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:49:13 -0400 Subject: [PATCH 0610/2680] rust: drm: sched: Do not stop the scheduler before tearing it down drm_sched_stop() belongs to the reset recovery protocol, and its documentation says not to call it before drm_sched_fini(). At teardown it detaches every in-flight job from its hardware fence and frees the jobs that have already completed. That is redundant: drm_sched_fini() stops the scheduler workers itself and hands every remaining job to cancel_job, which now handles jobs with an armed, completed or detached hardware fence callback. Call drm_sched_fini() alone, so that cancel_job sees each job's actual state and accounts for it exactly once. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit f1f985d576810c7183ea9bcd1b701cb284927ac6) --- rust/kernel/drm/sched.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index 56572a5b07ccd..7f23f86fd4040 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -381,12 +381,9 @@ struct SchedulerOwner { impl Drop for SchedulerOwner { fn drop(&mut self) { - let sched = self.allocation.sched.get(); - - // SAFETY: The scheduler is valid. This assumes drm_sched_fini() will take care of - // freeing all in-progress jobs. - unsafe { bindings::drm_sched_stop(sched, core::ptr::null_mut()) }; - unsafe { bindings::drm_sched_fini(sched) }; + // SAFETY: The scheduler is valid. drm_sched_fini() stops the scheduler workers, then + // cancels and frees all remaining jobs through `cancel_job_cb()`. + unsafe { bindings::drm_sched_fini(self.allocation.sched.get()) }; } } From d59a502b8c2f3f5aaae816b9c81941889338c7fa Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 01:22:17 -0400 Subject: [PATCH 0611/2680] rust: drm: sched: let drivers handle timeouts without a reset Drivers that recover individual jobs without the scheduler reset protocol need to keep unfinished jobs pending while retaining their original fences. Expose DRM_GPU_SCHED_STAT_NO_HANG as Status::NoHang: the scheduler reinserts the job, rearms its timeout, and frees it when completion is observed. Add Job::is_finished() to detect completion between timeout expiry and its handler. The finished fence is initialized only when armed, so an unarmed job reports false. Let a driver also reconsider a timeout before entering recovery, for work whose execution window has just been renewed. The false_timeout callback defaults to false and must not stop or restart the scheduler. Returning true follows the same NoHang reinsertion path. Existing drivers retain their timeout behavior unless they opt into the new callback or return Status::NoHang. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 14d7b3c379da4c764581d4c34aff495d7346af72) --- rust/kernel/drm/sched.rs | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index 7f23f86fd4040..78f0451f5c5f7 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -25,6 +25,11 @@ pub enum Status { Nominal = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_RESET, /// Device is no longer available NoDevice = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_ENODEV, + /// The job is not hung, or the driver dealt with the timeout without a scheduler reset. + /// + /// The job goes back to the pending list and the timeout is rearmed. The job is freed once + /// it has finished, as usual. + NoHang = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG, } /// Scheduler priorities @@ -42,6 +47,13 @@ pub enum Priority { /// Trait to be implemented by driver job objects. pub trait JobImpl: Sized { + /// Reconsider a timeout before the backend starts recovery. Returning true + /// reinserts this job and rearms its timeout without changing its fences. + /// The callback must not stop or restart the scheduler. + fn false_timeout(_job: &mut Job) -> bool { + false + } + /// Called when the scheduler is considering scheduling this job next, to get another Fence /// for this job to block on. Once it returns None, run() may be called. fn prepare(_job: &mut Job) -> Option { @@ -55,6 +67,9 @@ pub trait JobImpl: Sized { /// Called when a job has taken too long to execute, to trigger GPU recovery. /// + /// A driver that handles timeouts without a scheduler reset returns [`Status::NoHang`]. + /// [`Job::is_finished`] tells whether the job completed after the timeout fired. + /// /// This method is called in a workqueue context. fn timed_out(job: &mut Job) -> Status; @@ -97,7 +112,12 @@ unsafe extern "C" fn timedout_job_cb( // SAFETY: All of our jobs are Job. let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; - // SAFETY: All of our jobs are Job. + // SAFETY: The timeout worker retains this detached Job until this + // callback returns, including when its parent fence signals concurrently. + if T::false_timeout(unsafe { &mut *p }) { + return bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG; + } + // SAFETY: The same retained job remains valid after the first callback. T::timed_out(unsafe { &mut *p }) as bindings::drm_gpu_sched_stat } @@ -168,6 +188,20 @@ pub struct Job { _scheduler: Arc>, } +impl Job { + /// Returns whether the job has finished, successfully or not. + pub fn is_finished(&self) -> bool { + // The finished fence is initialized when the job is armed, which also assigns its + // scheduler. + if self.job.sched.is_null() { + return false; + } + + // SAFETY: The job is armed, so its scheduler fence is valid and initialized. + unsafe { bindings::dma_fence_get_status(addr_of_mut!((*self.job.s_fence).finished)) != 0 } + } +} + impl Deref for Job { type Target = T; From 1be12a566807e2e52cecc038cbebb171cd49e0b2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:53:05 -0400 Subject: [PATCH 0612/2680] dt-bindings: gpu: apple,agx: Add apple,agx-t8140 The T8140 (A18 Pro) GPU is run by two firmware co-processors, GFX and GFX1, each with its own control registers and mailbox. Its node therefore has three register ranges (asc, asc1, sgx) and two mailboxes. The firmware shares the GPU MMU translation tables, page tables and handoff structure through the same reserved memory regions as on earlier GPUs, but needs neither calibration blobs nor a global variable region, and it does not use apple,firmware-abi. The GPU power domain and the performance states are required. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c49ab96deb3c378015d1db27bf2122c837fd3bc0) --- .../devicetree/bindings/gpu/apple,agx.yaml | 67 ++++++++++++++++--- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/Documentation/devicetree/bindings/gpu/apple,agx.yaml b/Documentation/devicetree/bindings/gpu/apple,agx.yaml index 59989d8bd1cbc..cc9c78a7624f8 100644 --- a/Documentation/devicetree/bindings/gpu/apple,agx.yaml +++ b/Documentation/devicetree/bindings/gpu/apple,agx.yaml @@ -17,6 +17,7 @@ properties: - apple,agx-g13s - apple,agx-g14g - apple,agx-g14s + - apple,agx-t8140 - items: - enum: - apple,agx-g13c @@ -29,22 +30,22 @@ properties: - const: apple,agx-g14s reg: - items: - - description: GPU coprocessor control registers - - description: GPU block MMIO registers + minItems: 2 + maxItems: 3 reg-names: - items: - - const: asc - - const: sgx + minItems: 2 + maxItems: 3 power-domains: maxItems: 1 mboxes: - maxItems: 1 + minItems: 1 + maxItems: 2 memory-region: + minItems: 3 items: - description: Region containing GPU MMU TTBs - description: Region containing GPU MMU page tables @@ -56,6 +57,7 @@ properties: - description: Shared global variables with GPU firmware memory-region-names: + minItems: 3 items: - const: ttbs - const: pagetables @@ -64,6 +66,8 @@ properties: - const: hw-cal-b - const: globals + operating-points-v2: true + apple,firmware-abi: $ref: /schemas/types.yaml#/definitions/uint32-array minItems: 3 @@ -77,7 +81,54 @@ required: - reg - mboxes - memory-region - - apple,firmware-abi + +allOf: + - if: + properties: + compatible: + contains: + const: apple,agx-t8140 + then: + properties: + reg: + items: + - description: GFX coprocessor control registers + - description: GFX1 coprocessor control registers + - description: GPU block MMIO registers + reg-names: + items: + - const: asc + - const: asc1 + - const: sgx + mboxes: + items: + - description: GFX coprocessor mailbox + - description: GFX1 coprocessor mailbox + memory-region: + maxItems: 3 + memory-region-names: + maxItems: 3 + required: + - power-domains + - operating-points-v2 + else: + properties: + reg: + items: + - description: GPU coprocessor control registers + - description: GPU block MMIO registers + reg-names: + items: + - const: asc + - const: sgx + mboxes: + maxItems: 1 + memory-region: + minItems: 6 + memory-region-names: + minItems: 6 + required: + - apple,firmware-abi additionalProperties: false From e26a797314f5fed59e45bc718bc48fc66991b637 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:55:29 -0400 Subject: [PATCH 0613/2680] drm/asahi: uapi: Add the G17 additions Extend the UAPI for the G17 generation of GPUs: - drm_asahi_params_global gains the shader instruction set generation and the command stream generation (enum drm_asahi_gpu_hal_generation), which userspace needs to select its code generation and the layout of its control streams. - DRM_ASAHI_FEATURE_VM_STATUS and DRM_ASAHI_PARAM_GROUP_VM_STATUS report the sticky error of a VM, so that userspace can tell whether work behind a signalled fence executed successfully. - DRM_ASAHI_FEATURE_VM_STATUS_MIRROR and DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR let userspace register a word that the kernel keeps equal to that error. - DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES and DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS report how many queues the device supports and how many jobs of a queue may run at once. - DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS promises that every command of a submission waits for all of its input syncs. - The background and end-of-tile resource specifiers of G17 are 64 bits wide. Their high halves are appended to drm_asahi_cmd_render and flagged with DRM_ASAHI_RENDER_RSRC_SPEC_HI, which older kernels reject. The existing driver reports the generations of the G13 and G14 GPUs it supports and none of the new features. It still accepts render commands of the previous size, zero-filling the new fields, and rejects the new flag. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e0a5c5ad84cbc6e48c0194882b8c1c67d72538a8) --- drivers/gpu/drm/asahi/file.rs | 5 + include/uapi/drm/asahi_drm.h | 203 +++++++++++++++++++++++++++++++++- 2 files changed, 206 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 568efbb9f620b..886710637f69d 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -331,6 +331,11 @@ impl File { max_frequency_khz: gpu.get_dyncfg().pwr.max_frequency_khz(), command_timestamp_frequency_hz: 1_000_000_000, // User timestamps always in nanoseconds + + // G13 and G14 share the second shader ISA generation and predate the numbered + // command stream generations. + usc_generation: 2, + gpu_hal_generation: uapi::drm_asahi_gpu_hal_generation_DRM_ASAHI_GPU_HAL_LEGACY, }; for (i, mask) in gpu.get_dyncfg().id.core_masks.iter().enumerate() { diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index de67f1c603afd..8dae3950b4a02 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -191,6 +191,35 @@ struct drm_asahi_params_global { * seconds, rather than hardcoding a particular firmware's rate. */ __u64 command_timestamp_frequency_hz; + + /** + * @usc_generation: Instruction set generation of the unified shader + * cores: 2 for G13 and G14, 3 for G15 and newer. + */ + __u32 usc_generation; + + /** + * @gpu_hal_generation: Command stream generation, from + * enum drm_asahi_gpu_hal_generation. + */ + __u32 gpu_hal_generation; +}; + +/** + * enum drm_asahi_gpu_hal_generation - GPU command stream generation + * + * The command stream generation determines the layout of the control streams + * and descriptors that userspace builds for the GPU. + */ +enum drm_asahi_gpu_hal_generation { + /** @DRM_ASAHI_GPU_HAL_LEGACY: Unnumbered generation of G13 to G15. */ + DRM_ASAHI_GPU_HAL_LEGACY = 0, + + /** @DRM_ASAHI_GPU_HAL_200: Generation 200, used by G17P. */ + DRM_ASAHI_GPU_HAL_200 = 200, + + /** @DRM_ASAHI_GPU_HAL_300: Generation 300. */ + DRM_ASAHI_GPU_HAL_300 = 300, }; /** @@ -213,19 +242,168 @@ enum drm_asahi_feature { * userspace can speculate memory accesses more aggressively. */ DRM_ASAHI_FEATURE_SOFT_FAULTS = (1UL) << 0, + + /** + * @DRM_ASAHI_FEATURE_VM_STATUS: GET_PARAMS supports + * DRM_ASAHI_PARAM_GROUP_VM_STATUS. + */ + DRM_ASAHI_FEATURE_VM_STATUS = (1UL) << 1, + + /** + * @DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES: Each queue reserves its own + * hardware execution state when it is created, and GET_PARAMS + * supports DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS. + */ + DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES = (1UL) << 2, + + /** + * @DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS: Every hardware command of + * a submission waits for all of the submission's input syncs, including + * their errors, so independent commands need no extra dependency on the + * first command. Userspace that splits a batch across several + * submissions and relies on this must pass its input syncs to each of + * them. + */ + DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS = (1UL) << 3, + + /** + * @DRM_ASAHI_FEATURE_VM_STATUS_MIRROR: GET_PARAMS supports + * DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR, which registers a userspace + * word that always holds the VM status that + * DRM_ASAHI_PARAM_GROUP_VM_STATUS would return. The kernel stores a + * failure there before it signals any fence of the failed job, so after + * a successful wait a load of the word replaces the VM_STATUS query. + * Registration may still be refused. + */ + DRM_ASAHI_FEATURE_VM_STATUS_MIRROR = (1UL) << 4, +}; + +/** + * define DRM_ASAHI_PARAM_GROUP_VM_STATUS - GET_PARAMS group for + * struct drm_asahi_vm_status + */ +#define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 + +/** + * define DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS - GET_PARAMS group for + * struct drm_asahi_queue_limits + */ +#define DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS 2 + +/** + * define DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR - GET_PARAMS group for + * struct drm_asahi_vm_status_mirror + */ +#define DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR 3 + +/** + * struct drm_asahi_vm_status - VM status, GET_PARAMS group + * DRM_ASAHI_PARAM_GROUP_VM_STATUS + * + * This is an in/out query on a VM of the calling file. Userspace sets + * &drm_asahi_get_params.pointer to this structure and + * &drm_asahi_get_params.size to its size. + * + * The kernel records the failure of an accepted job before it signals the + * job's fences, so userspace must query the status after a successful syncobj + * wait before it treats the work as successfully executed. A signalled + * syncobj alone does not indicate successful execution. An execution failure + * also marks other VMs whose work can no longer make progress. A query never + * resets or recovers the GPU. + * + * Invalid requests fail with EINVAL (non-zero @flags or @pad, wrong size), + * ENOENT (@vm_id unknown to the calling file) or EFAULT, and write nothing. + */ +struct drm_asahi_vm_status { + /** @vm_id: VM to query */ + __u32 vm_id; + + /** @flags: MBZ */ + __u32 flags; + + /** + * @error: Output: 0 if no submission error was observed, otherwise the + * first error as a negative errno. The error is sticky for the lifetime + * of the VM: neither creating queues nor waiting on other fences clears + * it. + */ + __s32 error; + + /** @pad: MBZ */ + __u32 pad; +}; + +/** + * struct drm_asahi_queue_limits - Queue limits, GET_PARAMS group + * DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS + * + * The query neither allocates nor resets anything. + */ +struct drm_asahi_queue_limits { + /** + * @max_queues: Maximum number of queues on the device, shared by all + * files and VMs. This is a hardware bound, not a promise that as many + * are currently available: QUEUE_CREATE reserves a queue's hardware + * state and fails once it runs out. + */ + __u32 max_queues; + + /** + * @max_in_flight_per_queue: Number of jobs of a queue that the + * hardware may run at once. Further jobs wait behind the preceding work + * and their dependencies. + */ + __u32 max_in_flight_per_queue; + + /** @flags: Reserved, zero */ + __u32 flags; + + /** @pad: Reserved, zero */ + __u32 pad; +}; + +/** + * struct drm_asahi_vm_status_mirror - VM status mirror registration, GET_PARAMS + * group DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR + * + * Registers a 32-bit word in a GEM object of the calling file. The kernel + * keeps its own reference to the object for the lifetime of the VM status, + * zeroes the word, and from then on keeps it equal to the VM status error + * (see &drm_asahi_vm_status.error). Each VM accepts one registration. + * EOPNOTSUPP means that userspace must keep using + * DRM_ASAHI_PARAM_GROUP_VM_STATUS. + */ +struct drm_asahi_vm_status_mirror { + /** @vm_id: VM whose status is mirrored */ + __u32 vm_id; + + /** @handle: GEM object holding the word, at most 64 KiB in size */ + __u32 handle; + + /** @offset: Byte offset of the word in the object, 4-byte aligned */ + __u32 offset; + + /** @flags: MBZ */ + __u32 flags; }; /** * struct drm_asahi_get_params - Arguments passed to DRM_IOCTL_ASAHI_GET_PARAMS */ struct drm_asahi_get_params { - /** @param_group: Parameter group to fetch (MBZ) */ + /** + * @param_group: Parameter group: 0 for struct drm_asahi_params_global, + * otherwise one of DRM_ASAHI_PARAM_GROUP_*. + */ __u32 param_group; /** @pad: MBZ */ __u32 pad; - /** @pointer: User pointer to write parameter struct */ + /** + * @pointer: User pointer to the parameter struct. The VM status groups + * also read their input from it. + */ __u64 pointer; /** @@ -789,6 +967,15 @@ enum drm_asahi_render_flags { */ DRM_ASAHI_RENDER_NO_VERTEX_CLUSTERING = (1U << 2), + /** + * @DRM_ASAHI_RENDER_RSRC_SPEC_HI: The command carries the high halves + * of the background and end-of-tile resource specifiers + * (&drm_asahi_cmd_render.bg_rsrc_spec_hi and the following fields). + * Kernels that do not know the flag reject the command instead of + * silently truncating the specifiers. + */ + DRM_ASAHI_RENDER_RSRC_SPEC_HI = (1U << 4), + /** * @DRM_ASAHI_RENDER_DBIAS_IS_INT: Use integer depth bias formula. * @@ -1107,6 +1294,18 @@ struct drm_asahi_cmd_render { /** @ts_frag: Timestamps for the fragment portion of the render */ struct drm_asahi_timestamps ts_frag; + + /** @bg_eot_rsrc_spec_hi: High half of @eot.rsrc_spec */ + __u32 bg_eot_rsrc_spec_hi; + + /** @bg_eot_partial_rsrc_spec_hi: High half of @partial_eot.rsrc_spec */ + __u32 bg_eot_partial_rsrc_spec_hi; + + /** @bg_rsrc_spec_hi: High half of @bg.rsrc_spec */ + __u32 bg_rsrc_spec_hi; + + /** @bg_partial_rsrc_spec_hi: High half of @partial_bg.rsrc_spec */ + __u32 bg_partial_rsrc_spec_hi; }; /** From df210cce03ddd02b71ccdcb26d89d8b01b0c3f01 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:01:20 +1000 Subject: [PATCH 0614/2680] arm64: cputype: Add T8140 core MIDRs Add the MIDR part numbers of the two core types in the Apple T8140 (A18 Pro) SoC, as read from MIDR_EL1 on a J700: 0x060 for the efficiency cores and 0x061 for the performance cores. The T8140 WFI workaround in the idle code matches on them. These cores have no generational codename, so name them after the SoC codename and the core type, as the "apple,tahiti-e" and "apple,tahiti-p" CPU compatibles for this SoC do. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/include/asm/cputype.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 1b9f0cda1336d..d44795ef73e4c 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -153,6 +153,8 @@ #define APPLE_CPU_PART_M2_AVALANCHE_PRO 0x035 #define APPLE_CPU_PART_M2_BLIZZARD_MAX 0x038 #define APPLE_CPU_PART_M2_AVALANCHE_MAX 0x039 +#define APPLE_CPU_PART_TAHITI_E 0x060 +#define APPLE_CPU_PART_TAHITI_P 0x061 #define AMPERE_CPU_PART_AMPERE1 0xAC3 #define AMPERE_CPU_PART_AMPERE1A 0xAC4 @@ -246,6 +248,8 @@ #define MIDR_APPLE_M2_AVALANCHE_PRO MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M2_AVALANCHE_PRO) #define MIDR_APPLE_M2_BLIZZARD_MAX MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M2_BLIZZARD_MAX) #define MIDR_APPLE_M2_AVALANCHE_MAX MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M2_AVALANCHE_MAX) +#define MIDR_APPLE_TAHITI_E MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_TAHITI_E) +#define MIDR_APPLE_TAHITI_P MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_TAHITI_P) #define MIDR_AMPERE1 MIDR_CPU_MODEL(ARM_CPU_IMP_AMPERE, AMPERE_CPU_PART_AMPERE1) #define MIDR_AMPERE1A MIDR_CPU_MODEL(ARM_CPU_IMP_AMPERE, AMPERE_CPU_PART_AMPERE1A) #define MIDR_MICROSOFT_AZURE_COBALT_100 MIDR_CPU_MODEL(ARM_CPU_IMP_MICROSOFT, MICROSOFT_CPU_PART_AZURE_COBALT_100) From e994fc36a4f6b5bea4a152503254950890dcefe2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:01:44 +1000 Subject: [PATCH 0615/2680] arm64: idle: Preserve architectural state across WFI on T8140 On the Apple T8140 (A18 Pro) in the J700, the secondary CPUs can come back from WFI with their general-purpose registers cleared and their FP/SIMD registers lost; SP and memory survive. cpu_do_idle() is C code that expects its callee-saved registers back, so the default idle=wfi corrupts whatever the idle path keeps in them. Until now the only stable setting on J700 was idle=nop, which leaves every core spinning at full clock: measured on J700 at the same load average, the die idled at 85-92 C (peaks of 106.7 C) with idle=nop and at 43.4 C with WFI and this change. On those CPUs, cpu_do_idle() now enters WFI through apple_cpu_wfi(), which stores x18-x30 (the AAPCS64 callee-saved registers, the frame pointer, the link register, and x18, which may hold the shadow call stack pointer) on the stack, issues DSB SY and WFI, and restores them after an ISB. Being an ordinary function call, it already tells the compiler that x0-x17 and the flags are clobbered. Beforehand, fpsimd_save_and_flush_cpu_state() saves any live user FP/SIMD state and drops this CPU's ownership of it, so the next return to userspace reloads the registers even when the same task resumes here. The workaround is chosen once at boot: an early initcall matches the boot CPU's MIDR against the T8140 E and P core IDs and enables a static key. Every other CPU pays one patched-out branch per idle entry, and kernels without CONFIG_ARCH_APPLE compile the path out. The key is set before the secondaries are brought up; the boot CPU idles with plain WFI until then, and only the secondaries are known to be affected. apple_cpu_wfi() lives in .cpuidle.text like the rest of the idle path. The root cause is not known. Other Apple SoCs keep this state across WFI unless the OS asks for a deep state: cpuidle-apple sets bits 25:24 of s3_5_c15_c5_0 (CYC_OVRD) for its power-down state and brackets that state with cpu_pm_enter()/cpu_pm_exit(). The T8140 secondaries may be left by the boot chain with a WFI mode that does not retain state. Until that is understood, this stays a downstream workaround. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/include/asm/apple-idle.h | 8 +++++ arch/arm64/kernel/Makefile | 2 ++ arch/arm64/kernel/idle-apple.S | 45 ++++++++++++++++++++++++ arch/arm64/kernel/idle.c | 54 +++++++++++++++++++++++++++-- 4 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/include/asm/apple-idle.h create mode 100644 arch/arm64/kernel/idle-apple.S diff --git a/arch/arm64/include/asm/apple-idle.h b/arch/arm64/include/asm/apple-idle.h new file mode 100644 index 0000000000000..3a0a2660877f5 --- /dev/null +++ b/arch/arm64/include/asm/apple-idle.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_APPLE_IDLE_H +#define __ASM_APPLE_IDLE_H + +/* WFI that preserves x18-x30 on cores that lose them (Apple T8140). */ +void apple_cpu_wfi(void); + +#endif diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index e3f10fa71c033..4b9ff74af3091 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -40,6 +40,8 @@ obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ cpufeature_impdef.o \ rsi.o jump_label.o +obj-$(CONFIG_ARCH_APPLE) += idle-apple.o + obj-$(CONFIG_COMPAT) += sys32.o signal32.o \ sys_compat.o obj-$(CONFIG_COMPAT) += sigreturn32.o diff --git a/arch/arm64/kernel/idle-apple.S b/arch/arm64/kernel/idle-apple.S new file mode 100644 index 0000000000000..02c3e4f6402ca --- /dev/null +++ b/arch/arm64/kernel/idle-apple.S @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include +#include + +/* + * T8140 secondaries can resume from WFI with cleared general registers. + * SP and stack memory survive. Preserve the AAPCS64 callee-saved registers + * and x18 (which may hold the shadow call stack), including the return PC. + * This follows the stateless-wait approach used during m1n1 M5 Windows + * bring-up. It does not configure an Apple deep-sleep state or DVFS policy. + * + * IRQ masking/context management remains with cpu_do_idle(). The direct + * function call tells its compiler that x0-x17 and flags are clobbered. + */ + .macro apple_wait_save + stp x29, x30, [sp, #-112]! + mov x29, sp + stp x19, x20, [sp, #16] + stp x21, x22, [sp, #32] + stp x23, x24, [sp, #48] + stp x25, x26, [sp, #64] + stp x27, x28, [sp, #80] + str x18, [sp, #96] + dsb sy + .endm + + .macro apple_wait_restore + isb + ldr x18, [sp, #96] + ldp x19, x20, [sp, #16] + ldp x21, x22, [sp, #32] + ldp x23, x24, [sp, #48] + ldp x25, x26, [sp, #64] + ldp x27, x28, [sp, #80] + ldp x29, x30, [sp], #112 + ret + .endm + + .pushsection .cpuidle.text, "ax" +SYM_FUNC_START(apple_cpu_wfi) + apple_wait_save + wfi + apple_wait_restore +SYM_FUNC_END(apple_cpu_wfi) + .popsection diff --git a/arch/arm64/kernel/idle.c b/arch/arm64/kernel/idle.c index 42c5543f9589c..45bc79f128543 100644 --- a/arch/arm64/kernel/idle.c +++ b/arch/arm64/kernel/idle.c @@ -4,11 +4,16 @@ */ #include +#include #include +#include +#include #include #include #include +#include +#include #include enum { @@ -34,6 +39,41 @@ static int __init setup_idle(char *arg) } early_param("idle", setup_idle); +#ifdef CONFIG_ARCH_APPLE +/* + * Apple T8140 (A18 Pro) secondary CPUs can return from WFI with their + * general-purpose and FP/SIMD registers cleared. The key is enabled once + * at boot on those parts; everywhere else the check is a patched-out branch. + */ +static DEFINE_STATIC_KEY_FALSE(apple_wfi_loses_regs); + +static const struct midr_range apple_wfi_loses_regs_cpus[] __initconst = { + MIDR_ALL_VERSIONS(MIDR_APPLE_TAHITI_E), + MIDR_ALL_VERSIONS(MIDR_APPLE_TAHITI_P), + {} +}; + +static int __init apple_wfi_quirk_init(void) +{ + if (is_midr_in_range_list(apple_wfi_loses_regs_cpus)) { + static_branch_enable(&apple_wfi_loses_regs); + pr_info("Apple T8140: preserving registers across WFI\n"); + } + return 0; +} +early_initcall(apple_wfi_quirk_init); + +static __always_inline bool apple_wfi_needs_save(void) +{ + return static_branch_unlikely(&apple_wfi_loses_regs); +} +#else +static __always_inline bool apple_wfi_needs_save(void) +{ + return false; +} +#endif + /* * cpu_do_idle() * @@ -50,8 +90,18 @@ void __cpuidle cpu_do_idle(void) arm_cpuidle_save_irq_context(&context); if (likely(idle == ARM64_IDLE_WFI)) { - dsb(sy); - wfi(); + if (apple_wfi_needs_save()) { + /* + * The FP/SIMD registers do not survive either: drop the + * lazy ownership so the next return to userspace reloads + * them, even when the same task resumes on this CPU. + */ + fpsimd_save_and_flush_cpu_state(); + apple_cpu_wfi(); + } else { + dsb(sy); + wfi(); + } } else if (idle == ARM64_IDLE_YIELD) { dsb(sy); asm volatile("yield" ::: "memory"); From ddf4af3ee5c93bbd25dc9f71edb48af59a87db09 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 10:25:30 -0400 Subject: [PATCH 0616/2680] arm64: apple: Preserve registers across WFI on T8152 The first J873g boot of aurora-wip reaches /init but faults at PC zero when the boot CPU enters idle, with its general-purpose registers cleared. Retained J873 hardware work identifies the same wait-state register loss on the T8152 CPU parts 0x072 and 0x073. Add those MIDRs to the existing Apple WFI preservation quirk. Reuse its stack-based register save and conservative FP/SIMD save-and-flush path. Other CPU matches and the wait implementation are unchanged. Signed-off-by: Acelogic --- arch/arm64/include/asm/apple-idle.h | 2 +- arch/arm64/include/asm/cputype.h | 4 ++++ arch/arm64/kernel/idle.c | 9 ++++++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/arch/arm64/include/asm/apple-idle.h b/arch/arm64/include/asm/apple-idle.h index 3a0a2660877f5..56f5d32e3d41b 100644 --- a/arch/arm64/include/asm/apple-idle.h +++ b/arch/arm64/include/asm/apple-idle.h @@ -2,7 +2,7 @@ #ifndef __ASM_APPLE_IDLE_H #define __ASM_APPLE_IDLE_H -/* WFI that preserves x18-x30 on cores that lose them (Apple T8140). */ +/* WFI that preserves x18-x30 on affected Apple cores (T8140 and T8152). */ void apple_cpu_wfi(void); #endif diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index d44795ef73e4c..632f980ec55e5 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -155,6 +155,8 @@ #define APPLE_CPU_PART_M2_AVALANCHE_MAX 0x039 #define APPLE_CPU_PART_TAHITI_E 0x060 #define APPLE_CPU_PART_TAHITI_P 0x061 +#define APPLE_CPU_PART_T8152_E 0x072 +#define APPLE_CPU_PART_T8152_P 0x073 #define AMPERE_CPU_PART_AMPERE1 0xAC3 #define AMPERE_CPU_PART_AMPERE1A 0xAC4 @@ -250,6 +252,8 @@ #define MIDR_APPLE_M2_AVALANCHE_MAX MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M2_AVALANCHE_MAX) #define MIDR_APPLE_TAHITI_E MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_TAHITI_E) #define MIDR_APPLE_TAHITI_P MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_TAHITI_P) +#define MIDR_APPLE_T8152_E MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_T8152_E) +#define MIDR_APPLE_T8152_P MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_T8152_P) #define MIDR_AMPERE1 MIDR_CPU_MODEL(ARM_CPU_IMP_AMPERE, AMPERE_CPU_PART_AMPERE1) #define MIDR_AMPERE1A MIDR_CPU_MODEL(ARM_CPU_IMP_AMPERE, AMPERE_CPU_PART_AMPERE1A) #define MIDR_MICROSOFT_AZURE_COBALT_100 MIDR_CPU_MODEL(ARM_CPU_IMP_MICROSOFT, MICROSOFT_CPU_PART_AZURE_COBALT_100) diff --git a/arch/arm64/kernel/idle.c b/arch/arm64/kernel/idle.c index 45bc79f128543..972164e78874c 100644 --- a/arch/arm64/kernel/idle.c +++ b/arch/arm64/kernel/idle.c @@ -41,8 +41,9 @@ early_param("idle", setup_idle); #ifdef CONFIG_ARCH_APPLE /* - * Apple T8140 (A18 Pro) secondary CPUs can return from WFI with their - * general-purpose and FP/SIMD registers cleared. The key is enabled once + * Apple T8140 (A18 Pro) secondary CPUs and T8152 (M6) CPUs can return from + * WFI with general-purpose registers cleared. T8140 also loses FP/SIMD + * state; conservatively preserve it on both parts. The key is enabled once * at boot on those parts; everywhere else the check is a patched-out branch. */ static DEFINE_STATIC_KEY_FALSE(apple_wfi_loses_regs); @@ -50,6 +51,8 @@ static DEFINE_STATIC_KEY_FALSE(apple_wfi_loses_regs); static const struct midr_range apple_wfi_loses_regs_cpus[] __initconst = { MIDR_ALL_VERSIONS(MIDR_APPLE_TAHITI_E), MIDR_ALL_VERSIONS(MIDR_APPLE_TAHITI_P), + MIDR_ALL_VERSIONS(MIDR_APPLE_T8152_E), + MIDR_ALL_VERSIONS(MIDR_APPLE_T8152_P), {} }; @@ -57,7 +60,7 @@ static int __init apple_wfi_quirk_init(void) { if (is_midr_in_range_list(apple_wfi_loses_regs_cpus)) { static_branch_enable(&apple_wfi_loses_regs); - pr_info("Apple T8140: preserving registers across WFI\n"); + pr_info("Apple: preserving registers across WFI\n"); } return 0; } From 6b3866441a323058923b570b23ed81935a95afad Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 11:57:48 -0400 Subject: [PATCH 0617/2680] arm64: apple: preserve registers across WFI on T8142 [ryan: ported to 7.2.10 without the Documentation/arch/arm64/apple-j813.rst hunk; that file is created by 80661974aa5b, which is routed to 000-devicetree, and carries this hunk there] --- arch/arm64/include/asm/cputype.h | 4 ++++ arch/arm64/kernel/idle.c | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 632f980ec55e5..809e39962fe23 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -155,6 +155,8 @@ #define APPLE_CPU_PART_M2_AVALANCHE_MAX 0x039 #define APPLE_CPU_PART_TAHITI_E 0x060 #define APPLE_CPU_PART_TAHITI_P 0x061 +#define APPLE_CPU_PART_T8142_E 0x062 +#define APPLE_CPU_PART_T8142_P 0x063 #define APPLE_CPU_PART_T8152_E 0x072 #define APPLE_CPU_PART_T8152_P 0x073 @@ -252,6 +254,8 @@ #define MIDR_APPLE_M2_AVALANCHE_MAX MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M2_AVALANCHE_MAX) #define MIDR_APPLE_TAHITI_E MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_TAHITI_E) #define MIDR_APPLE_TAHITI_P MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_TAHITI_P) +#define MIDR_APPLE_T8142_E MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_T8142_E) +#define MIDR_APPLE_T8142_P MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_T8142_P) #define MIDR_APPLE_T8152_E MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_T8152_E) #define MIDR_APPLE_T8152_P MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_T8152_P) #define MIDR_AMPERE1 MIDR_CPU_MODEL(ARM_CPU_IMP_AMPERE, AMPERE_CPU_PART_AMPERE1) diff --git a/arch/arm64/kernel/idle.c b/arch/arm64/kernel/idle.c index 972164e78874c..14cde8592c86e 100644 --- a/arch/arm64/kernel/idle.c +++ b/arch/arm64/kernel/idle.c @@ -41,9 +41,9 @@ early_param("idle", setup_idle); #ifdef CONFIG_ARCH_APPLE /* - * Apple T8140 (A18 Pro) secondary CPUs and T8152 (M6) CPUs can return from + * Apple T8140 (A18 Pro), T8142 (M5), and T8152 (M6) CPUs can return from * WFI with general-purpose registers cleared. T8140 also loses FP/SIMD - * state; conservatively preserve it on both parts. The key is enabled once + * state; conservatively preserve it on all three parts. The key is enabled once * at boot on those parts; everywhere else the check is a patched-out branch. */ static DEFINE_STATIC_KEY_FALSE(apple_wfi_loses_regs); @@ -51,6 +51,8 @@ static DEFINE_STATIC_KEY_FALSE(apple_wfi_loses_regs); static const struct midr_range apple_wfi_loses_regs_cpus[] __initconst = { MIDR_ALL_VERSIONS(MIDR_APPLE_TAHITI_E), MIDR_ALL_VERSIONS(MIDR_APPLE_TAHITI_P), + MIDR_ALL_VERSIONS(MIDR_APPLE_T8142_E), + MIDR_ALL_VERSIONS(MIDR_APPLE_T8142_P), MIDR_ALL_VERSIONS(MIDR_APPLE_T8152_E), MIDR_ALL_VERSIONS(MIDR_APPLE_T8152_P), {} From 2e78279358bd692e9d15cc3c4adbc0d35628bda0 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 12:37:37 -0400 Subject: [PATCH 0618/2680] arm64: delay: avoid register-losing WFIT on Apple M5 T8142 can clear registers when returning from WFIT, just as it can with WFI in the idle task. The second Linux cluster-frequency transition on J813 reached __const_udelay while polling the command register, then returned with a cleared callee-saved pointer and faulted in cpufreq. Use the existing counter delay loop on both T8142 core types. The idle task's register/FP flush path cannot be used by arbitrary kernel delays which may have live SIMD state. With this change, repeated E/P state sweeps and pinned integer workloads complete on J813, followed by timer progress and SHA-256 jobs on all ten CPUs. Other CPU models retain their existing delay paths. Signed-off-by: Acelogic --- arch/arm64/lib/delay.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/arch/arm64/lib/delay.c b/arch/arm64/lib/delay.c index e278e060e78a9..e0c5e615b5e5d 100644 --- a/arch/arm64/lib/delay.c +++ b/arch/arm64/lib/delay.c @@ -14,6 +14,7 @@ #include #include +#include #define USECS_TO_CYCLES(time_usecs) \ xloops_to_cycles((time_usecs) * 0x10C7UL) @@ -42,6 +43,18 @@ void __delay(unsigned long cycles) { cycles_t start = __delay_cycles(); + /* + * T8142 can clear registers on WFIT just as it does on WFI. A delay + * may run with live kernel SIMD state, so the idle task's save/flush + * path is not applicable here. Keep the core awake for short delays. + */ + if (IS_ENABLED(CONFIG_ARCH_APPLE)) { + u32 midr = read_cpuid_id() & MIDR_CPU_MODEL_MASK; + + if (midr == MIDR_APPLE_T8142_E || midr == MIDR_APPLE_T8142_P) + goto counter_delay; + } + if (alternative_has_cap_unlikely(ARM64_HAS_WFXT)) { u64 end = start + cycles; @@ -60,6 +73,7 @@ void __delay(unsigned long cycles) wfe(); } +counter_delay: while ((__delay_cycles() - start) < cycles) cpu_relax(); } From ca4e7fb2b4e71cdc03df735b350eeb7b5b153790 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:07:41 -0400 Subject: [PATCH 0619/2680] drm/asahi: separate DRM-facing GPU operations from the firmware manager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The file and ioctl layer uses a manager trait that also exposes G13/G14 firmware allocators, VM slots, event handling and control channels. A GPU with a different firmware interface cannot implement those internals. Separate the DRM-facing operations into Gpu: VM and queue creation, timestamp mapping, sequence IDs, crash state, timer frequency and hardware parameters. Existing managers implement it directly and expose their firmware-specific interface through Gpu::manager(). Keep the legacy firmware-global update at ioctl entry after flag validation, before queue lookup and userspace copies, outside the queue mutex. Invalid queue submissions still refresh the firmware idle policy. Legacy operation ordering and reported hardware values are unchanged. [ryan: adapted to 7.2.10 — minimal; for DJ: asahi-wip keeps the GPU in the DRM RegistrationData, not in device data, so the Gpu trait object lives there and is reached through registration_guard()/reg_data; new_queue() drops its reg_data argument; Asahi's _as_any() name is kept] Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit aae9f7d3525f93a7a5b5ae0529f8e85219735d08) --- drivers/gpu/drm/asahi/driver.rs | 14 +- drivers/gpu/drm/asahi/file.rs | 27 +--- drivers/gpu/drm/asahi/gpu.rs | 204 +++++++++++++++++------------ drivers/gpu/drm/asahi/mmu.rs | 6 +- drivers/gpu/drm/asahi/queue/mod.rs | 12 +- drivers/gpu/drm/asahi/workqueue.rs | 6 +- 6 files changed, 152 insertions(+), 117 deletions(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 5bac8117bb6b8..5a58690cfdf46 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -46,11 +46,11 @@ pub(crate) struct AsahiPlatformDriverData<'bound> { _reg: drm::Registration<'bound, AsahiDrmDriver>, } -/// Holds a reference to the top-level `GpuManager` object. +/// Holds a reference to the top-level GPU object. #[pin_data] pub(crate) struct AsahiDrmRegistrationData<'drm> { #[pin] - pub(crate) gpu: Arc, + pub(crate) gpu: Arc, pub(crate) resources: regs::Resources<'drm>, } @@ -181,19 +181,19 @@ impl platform::Driver for AsahiPlatformDriver { let gpu = match (cfg.gpu_gen, cfg.gpu_variant, compat.as_slice()) { (hw::GpuGen::G13, _, &[12, 3, 0]) => { - gpu::GpuManagerG13V12_3::new(&drm, &resources, cfg)? as Arc + gpu::GpuManagerG13V12_3::new(&drm, &resources, cfg)? as Arc } (hw::GpuGen::G14, hw::GpuVariant::G, &[12, 4, 0]) => { - gpu::GpuManagerG14V12_4::new(&drm, &resources, cfg)? as Arc + gpu::GpuManagerG14V12_4::new(&drm, &resources, cfg)? as Arc } (hw::GpuGen::G13, _, &[13, 5, 0]) => { - gpu::GpuManagerG13V13_5::new(&drm, &resources, cfg)? as Arc + gpu::GpuManagerG13V13_5::new(&drm, &resources, cfg)? as Arc } (hw::GpuGen::G14, hw::GpuVariant::G, &[13, 5, 0]) => { - gpu::GpuManagerG14V13_5::new(&drm, &resources, cfg)? as Arc + gpu::GpuManagerG14V13_5::new(&drm, &resources, cfg)? as Arc } (hw::GpuGen::G14, _, &[13, 5, 0]) => { - gpu::GpuManagerG14XV13_5::new(&drm, &resources, cfg)? as Arc + gpu::GpuManagerG14XV13_5::new(&drm, &resources, cfg)? as Arc } _ => { dev_info!( diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 886710637f69d..1a64ed9f419e9 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -312,35 +312,19 @@ impl File { let mut params = uapi::drm_asahi_params_global { features: 0, - gpu_generation: gpu.get_dyncfg().id.gpu_gen as u32, - gpu_variant: gpu.get_dyncfg().id.gpu_variant as u32, - gpu_revision: gpu.get_dyncfg().id.gpu_rev as u32, - chip_id: gpu.get_cfg().chip_id, - - num_dies: gpu.get_cfg().num_dies, - num_clusters_total: gpu.get_dyncfg().id.num_clusters, - num_cores_per_cluster: gpu.get_dyncfg().id.num_cores, - core_masks: [0; uapi::DRM_ASAHI_MAX_CLUSTERS as usize], - vm_start: VM_USER_RANGE.start, vm_end: VM_USER_RANGE.end, vm_kernel_min_size: VM_KERNEL_MIN_SIZE, max_commands_per_submission: MAX_COMMANDS_PER_SUBMISSION, max_attachments: crate::microseq::MAX_ATTACHMENTS as u32, - max_frequency_khz: gpu.get_dyncfg().pwr.max_frequency_khz(), command_timestamp_frequency_hz: 1_000_000_000, // User timestamps always in nanoseconds - // G13 and G14 share the second shader ISA generation and predate the numbered - // command stream generations. - usc_generation: 2, - gpu_hal_generation: uapi::drm_asahi_gpu_hal_generation_DRM_ASAHI_GPU_HAL_LEGACY, + ..pin_init::zeroed() }; - for (i, mask) in gpu.get_dyncfg().id.core_masks.iter().enumerate() { - *(params.core_masks.get_mut(i).ok_or(EIO)?) = (*mask).into(); - } + gpu.get_params(&mut params)?; if *module_parameters::fault_control.value() == 0xb { params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; @@ -973,7 +957,6 @@ impl File { core::mem::drop(guard); let queue = reg_data.gpu.new_queue( - reg_data, vm, ualloc, ualloc_priv, @@ -1019,7 +1002,9 @@ impl File { } let gpu = ®_data.gpu; - gpu.update_globals(); + if let Some(manager) = gpu.manager() { + manager.update_globals(); + } // Upgrade to Arc to drop the XArray lock early let queue: Arc>> = file @@ -1110,7 +1095,7 @@ impl File { // TODO: Do this on device-init for perf. let gpu = ®_data.gpu; - let frequency_hz = gpu.get_cfg().base_clock_hz as u64; + let frequency_hz = gpu.base_clock_hz() as u64; let ts_gcd = gcd(frequency_hz, NSEC_PER_SEC as u64); let num = (NSEC_PER_SEC as u64) / ts_gcd; diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 28d3562fc9095..ca6e25c1df8c3 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -51,7 +51,8 @@ use kernel::{ Instant, Monotonic, // }, - types::ForeignOwnable, // + types::ForeignOwnable, + uapi, // }; use crate::alloc::Allocator; @@ -59,8 +60,7 @@ use crate::debug::*; use crate::driver::{ AsahiDevRef, AsahiDevice, - AsahiDrmDevice, - AsahiRegData, // + AsahiDrmDevice, // }; use crate::fw::channels::{ ChannelErrorType, @@ -263,36 +263,53 @@ pub(crate) struct GpuManager { garbage_contexts: Mutex>>>, } -/// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. -pub(crate) trait GpuManager: Send + Sync { +/// Operations the DRM driver performs on a GPU, independent of its firmware interface. +pub(crate) trait Gpu: Send + Sync { /// Cast as an Any type. fn _as_any(&self) -> &dyn Any; /// Cast Arc as an Any type. fn arc_as_any(self: Arc) -> Arc; + /// Return the G13/G14 firmware manager, if this GPU uses that firmware interface. + fn manager(&self) -> Option<&dyn GpuManager>; /// Initialize the GPU. fn init(&self) -> Result; - /// Update the GPU globals from global info - /// - /// TODO: Unclear what can and cannot be updated like this. - fn update_globals(&self); - /// Get a reference to the KernelAllocators. - fn alloc(&self) -> Guard<'_, KernelAllocators, MutexBackend>; - /// Create a new `Vm` given a unique `File` ID. + /// Return a reference to the global `SequenceIDs` instance. + fn ids(&self) -> &SequenceIDs; + /// Check whether the GPU is crashed + fn is_crashed(&self) -> bool; + /// Fill in the hardware description reported to userspace. + fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; + /// Frequency of the CPU timer that GPU timestamps are derived from. + fn base_clock_hz(&self) -> u32; + /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; - /// Bind a `Vm` to an available slot and return the `VmBind`. - fn bind_vm(&self, vm: &mmu::Vm) -> Result; /// Create a new user command queue. fn new_queue( &self, - reg_data: &AsahiRegData<'_>, vm: mmu::Vm, ualloc: Arc>, ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, ) -> Result>; - /// Return a reference to the global `SequenceIDs` instance. - fn ids(&self) -> &SequenceIDs; + /// Map a BO as a timestamp buffer + fn map_timestamp_buffer( + &self, + bo: gem::ObjectRef, + range: Range, + ) -> Result; +} + +/// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. +pub(crate) trait GpuManager: Send + Sync { + /// Update the GPU globals from global info + /// + /// TODO: Unclear what can and cannot be updated like this. + fn update_globals(&self); + /// Get a reference to the KernelAllocators. + fn alloc(&self) -> Guard<'_, KernelAllocators, MutexBackend>; + /// Bind a `Vm` to an available slot and return the `VmBind`. + fn bind_vm(&self, vm: &mmu::Vm) -> Result; /// Kick the firmware (wake it up if asleep). /// /// This should be useful to reduce latency on work submission, so we can ask the firmware to @@ -324,14 +341,6 @@ pub(crate) trait GpuManager: Send + Sync { fn get_dyncfg(&self) -> &hw::DynConfig; /// Register an unused context as garbage fn free_context(&self, data: KBox>); - /// Check whether the GPU is crashed - fn is_crashed(&self) -> bool; - /// Map a BO as a timestamp buffer - fn map_timestamp_buffer( - &self, - bo: gem::ObjectRef, - range: Range, - ) -> Result; } /// Private generic trait for functions that don't need to escape this module. @@ -1221,7 +1230,7 @@ impl GpuManager::ver { } #[versions(AGX)] -impl GpuManager for GpuManager::ver { +impl Gpu for GpuManager::ver { fn _as_any(&self) -> &dyn Any { self } @@ -1230,6 +1239,10 @@ impl GpuManager for GpuManager::ver { self as Arc } + fn manager(&self) -> Option<&dyn GpuManager> { + Some(self) + } + fn init(&self) -> Result { self.tx_channels.lock().device_control.send( &fw::channels::DeviceControlMsg::ver::Initialize(Default::default()), @@ -1252,6 +1265,89 @@ impl GpuManager for GpuManager::ver { Ok(()) } + fn ids(&self) -> &SequenceIDs { + &self.ids + } + + fn is_crashed(&self) -> bool { + self.crashed.load(Ordering::Relaxed) + } + + fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { + params.gpu_generation = self.dyncfg.id.gpu_gen as u32; + params.gpu_variant = self.dyncfg.id.gpu_variant as u32; + params.gpu_revision = self.dyncfg.id.gpu_rev as u32; + params.chip_id = self.cfg.chip_id; + params.num_dies = self.cfg.num_dies; + params.num_clusters_total = self.dyncfg.id.num_clusters; + params.num_cores_per_cluster = self.dyncfg.id.num_cores; + params.max_frequency_khz = self.dyncfg.pwr.max_frequency_khz(); + // G13 and G14 share the second shader ISA generation and predate the numbered command + // stream generations. + params.usc_generation = 2; + params.gpu_hal_generation = uapi::drm_asahi_gpu_hal_generation_DRM_ASAHI_GPU_HAL_LEGACY; + + for (i, mask) in self.dyncfg.id.core_masks.iter().enumerate() { + *(params.core_masks.get_mut(i).ok_or(EIO)?) = (*mask).into(); + } + + Ok(()) + } + + fn base_clock_hz(&self) -> u32 { + self.cfg.base_clock_hz + } + + fn new_vm(&self, kernel_range: Range) -> Result { + self.uat.new_vm(self.ids.vm.next(), kernel_range) + } + + fn new_queue( + &self, + vm: mmu::Vm, + ualloc: Arc>, + ualloc_priv: Arc>, + priority: u32, + usc_exec_base: u64, + ) -> Result> { + let mut kalloc = self.alloc(); + let id = self.ids.queue.next(); + Ok(KBox::new( + queue::Queue::ver::new( + self.dev.clone(), + self, + vm, + &mut kalloc, + ualloc, + ualloc_priv, + self.event_manager.clone(), + &self.buffer_mgr, + id, + priority, + usc_exec_base, + )?, + GFP_KERNEL, + )?) + } + + fn map_timestamp_buffer( + &self, + mut bo: gem::ObjectRef, + range: Range, + ) -> Result { + bo.map_range_into_range( + self.uat.kernel_vm(), + range, + IOVA_KERN_TIMESTAMP_RANGE, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + false, + ) + } +} + +#[versions(AGX)] +impl GpuManager for GpuManager::ver { fn update_globals(&self) { let mut timeout: u32 = 2; if debug_enabled(DebugFlags::WaitForPowerOff) { @@ -1310,43 +1406,10 @@ impl GpuManager for GpuManager::ver { guard } - fn new_vm(&self, kernel_range: Range) -> Result { - self.uat.new_vm(self.ids.vm.next(), kernel_range) - } - fn bind_vm(&self, vm: &mmu::Vm) -> Result { self.uat.bind(vm) } - fn new_queue( - &self, - reg_data: &AsahiRegData<'_>, - vm: mmu::Vm, - ualloc: Arc>, - ualloc_priv: Arc>, - priority: u32, - usc_exec_base: u64, - ) -> Result> { - let mut kalloc = self.alloc(); - let id = self.ids.queue.next(); - Ok(KBox::new( - queue::Queue::ver::new( - self.dev.clone(), - reg_data, - vm, - &mut kalloc, - ualloc, - ualloc_priv, - self.event_manager.clone(), - &self.buffer_mgr, - id, - priority, - usc_exec_base, - )?, - GFP_KERNEL, - )?) - } - fn kick_firmware(&self) -> Result { if self.is_crashed() { return Err(ENODEV); @@ -1402,10 +1465,6 @@ impl GpuManager for GpuManager::ver { Ok(()) } - fn ids(&self) -> &SequenceIDs { - &self.ids - } - fn handle_timeout(&self, counter: u32, event_slot: i32, unk: u32) { dev_err!(self.dev.as_ref(), " (\\________/) \n"); dev_err!(self.dev.as_ref(), " | | \n"); @@ -1595,25 +1654,6 @@ impl GpuManager for GpuManager::ver { ); } } - - fn is_crashed(&self) -> bool { - self.crashed.load(Ordering::Relaxed) - } - - fn map_timestamp_buffer( - &self, - mut bo: gem::ObjectRef, - range: Range, - ) -> Result { - bo.map_range_into_range( - self.uat.kernel_vm(), - range, - IOVA_KERN_TIMESTAMP_RANGE, - mmu::UAT_PGSZ as u64, - mmu::PROT_FW_SHARED_RW, - false, - ) - } } #[versions(AGX)] diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index b095098320fe8..3c6080ada6c8f 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -768,7 +768,11 @@ impl KernelMapping { // Tell the firmware to do a cache flush guard.registration_data_with(|reg_data| { - if let Err(e) = reg_data.gpu.fwctl(cmd) { + let ret = match reg_data.gpu.manager() { + Some(gpu) => gpu.fwctl(cmd), + None => Err(ENODEV), + }; + if let Err(e) = ret { dev_err!( owner.dev.as_ref(), "MMU: ASC cache flush {:#x}:{:#x} failed (err: {:?})\n", diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 594b2710d3440..38e1edd23bc31 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -26,7 +26,10 @@ use crate::debug::*; use crate::driver::{AsahiDevRef, AsahiDevice}; use crate::file::MAX_COMMANDS_PER_SUBMISSION; use crate::fw::types::*; -use crate::gpu::GpuManager; +use crate::gpu::{ + Gpu, + GpuManager, // +}; use crate::inner_weak_ptr; use crate::microseq; use crate::module_parameters; @@ -442,9 +445,9 @@ static QUEUE_CLASS_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); impl Queue::ver { /// Create a new user queue. #[allow(clippy::too_many_arguments)] - pub(crate) fn new<'drm>( + pub(crate) fn new( dev: AsahiDevice, - reg_data: &AsahiRegData<'drm>, + gpu: &dyn GpuManager, vm: mmu::Vm, alloc: &mut gpu::KernelAllocators, ualloc: Arc>, @@ -495,8 +498,7 @@ impl Queue::ver { )?; let entity = sched::Entity::new(&sched, sched::Priority::Kernel)?; - let buffer = - buffer::Buffer::ver::new(&*reg_data.gpu, alloc, ualloc.clone(), ualloc_priv, mgr)?; + let buffer = buffer::Buffer::ver::new(gpu, alloc, ualloc.clone(), ualloc_priv, mgr)?; let mut ret = Queue::ver { dev: dev.clone(), diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs index a88fe6d4a6a94..f763f12f340e7 100644 --- a/drivers/gpu/drm/asahi/workqueue.rs +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -134,7 +134,11 @@ impl Drop for GpuContext { guard.registration_data_with(|reg_data| { let data = self.data.take().unwrap(); - reg_data.gpu.free_context(data); + // A context only exists for the manager whose allocators created it, so the manager is + // always present. + if let Some(gpu) = reg_data.gpu.manager() { + gpu.free_context(data); + } }) } } From 05c78ea4acf1b696d681cbe1401b7ab443990150 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 2 Oct 2026 02:57:46 -0400 Subject: [PATCH 0620/2680] drm/asahi: mmu: support per-SoC UAT geometry and user VA range MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit G13 and G14 GPUs translate 39 input address bits per UAT root, and the driver hardcodes that width in the page table walker, the user and kernel VA ranges and the user range reported to userspace. Newer GPUs use the same 16 KiB granule and three-level walk, but translate 42 bits per root. Describe the address width of a UAT instance with a UatGeometry (input and output address bits) and derive the ranges from it: the user range spans the lower root, the dummy page sits two pages below its top, and the driver-managed kernel range is the third top-level entry of the upper root. The input width becomes per-SoC configuration data next to the output width, Uat::new takes the geometry instead of the firmware configuration, the page table walker takes the input width per table, and the file layer gets the user range from the device instead of a constant. All G13/G14 configurations describe a 39-bit input width, for which every derived value equals the constant it replaces: user top 1 << 39, dummy page 0x7f_ffff_8000, kernel range 0xffff_ffa0_0000_0000 to 0xffff_ffb0_0000_0000, upper range from 0xffff_ff80_0000_0000, and the walker's input mask. No functional change on those SoCs. [ryan: adapted to 7.2.10 — minimal; for DJ: asahi-wip removed the GPU dev coredump (crashdump.rs, dump_pages, dump_kernel_pages), so the UatGeometry hunks for it and upper_range() are dropped; Uat/Vm take Asahi's owned AsahiDevice; File::open builds the user range inside registration_data_with()] Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 7d3ddb6c35e1c478f68c78ac8bb8b3ee9d314e58) --- drivers/gpu/drm/asahi/file.rs | 36 ++++++---- drivers/gpu/drm/asahi/gpu.rs | 15 ++++- drivers/gpu/drm/asahi/hw/mod.rs | 2 + drivers/gpu/drm/asahi/hw/t600x.rs | 1 + drivers/gpu/drm/asahi/hw/t602x.rs | 1 + drivers/gpu/drm/asahi/hw/t8103.rs | 1 + drivers/gpu/drm/asahi/hw/t8112.rs | 1 + drivers/gpu/drm/asahi/initdata.rs | 6 +- drivers/gpu/drm/asahi/mmu.rs | 106 ++++++++++++++++++++---------- drivers/gpu/drm/asahi/pgtable.rs | 28 +++++--- 10 files changed, 136 insertions(+), 61 deletions(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 1a64ed9f419e9..cde855fec5326 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -65,6 +65,7 @@ struct Vm { ualloc: Arc>, ualloc_priv: Arc>, vm: mmu::Vm, + user_range: Range, kernel_range: Range, _dummy_mapping: mmu::KernelMapping, } @@ -72,8 +73,8 @@ struct Vm { impl Drop for Vm { fn drop(&mut self) { // When the user Vm is dropped, verify that no user mappings exists. - let left_range = VM_USER_RANGE.start..self.kernel_range.start; - let right_range = self.kernel_range.end..VM_USER_RANGE.end; + let left_range = self.user_range.start..self.kernel_range.start; + let right_range = self.kernel_range.end..self.user_range.end; if !left_range.is_empty() && self @@ -205,6 +206,8 @@ pub(crate) enum Object { // #[pin_data] pub(crate) struct File { id: u64, + /// Available VM range for the user + user_range: Range, // #[pin] vms: xarray::XArray>, // #[pin] @@ -216,9 +219,6 @@ pub(crate) struct File { /// Convenience type alias for our DRM `File` type. pub(crate) type DrmFile = drm::File; -/// Available VM range for the user -const VM_USER_RANGE: Range = mmu::IOVA_USER_USABLE_RANGE; - /// Minimum reserved AS for kernel mappings const VM_KERNEL_MIN_SIZE: u64 = 0x20000000; @@ -232,9 +232,10 @@ impl drm::file::DriverFile for File { device.registration_data_with(|reg_data| { let gpu = ®_data.gpu; let id = gpu.ids().file.next(); + let user_range = gpu.uat_geometry().user_usable_range(); mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); - Ok(KBox::pin_init(File::new(id), GFP_KERNEL)?) + Ok(KBox::pin_init(File::new(id, user_range), GFP_KERNEL)?) }) } @@ -247,7 +248,7 @@ impl drm::file::DriverFile for File { unsafe impl AnyBitPattern for uapi::drm_asahi_gem_bind_op {} impl File { - fn new(id: u64) -> impl PinInit { + fn new(id: u64, user_range: Range) -> impl PinInit { unsafe { pin_init::pin_init_from_closure(move |slot: *mut Self| { let raw_vms = addr_of_mut!((*slot).vms); @@ -265,6 +266,7 @@ impl File { .__pinned_init(raw_objects)?; (*slot).id = id; + (*slot).user_range = user_range; Ok(()) }) } @@ -312,8 +314,8 @@ impl File { let mut params = uapi::drm_asahi_params_global { features: 0, - vm_start: VM_USER_RANGE.start, - vm_end: VM_USER_RANGE.end, + vm_start: file.inner().user_range.start, + vm_end: file.inner().user_range.end, vm_kernel_min_size: VM_KERNEL_MIN_SIZE, max_commands_per_submission: MAX_COMMANDS_PER_SUBMISSION, @@ -352,9 +354,10 @@ impl File { file: &DrmFile, ) -> Result { let kernel_range = data.kernel_start..data.kernel_end; + let user_range = file.inner().user_range.clone(); // Validate requested kernel range - if !VM_USER_RANGE.is_superset(kernel_range.clone()) + if !user_range.is_superset(kernel_range.clone()) || kernel_range.range() < VM_KERNEL_MIN_SIZE || kernel_range.start & (mmu::UAT_PGMSK as u64) != 0 || kernel_range.end & (mmu::UAT_PGMSK as u64) != 0 @@ -428,8 +431,12 @@ impl File { for off in 0..obj_size { vmap.try_write8(0x0, off)?; } - let dummy_mapping = - dummy_obj.map_at(&vm, mmu::IOVA_UNK_PAGE, mmu::PROT_GPU_SHARED_RW, true)?; + let dummy_mapping = dummy_obj.map_at( + &vm, + gpu.uat_geometry().unk_page(), + mmu::PROT_GPU_SHARED_RW, + true, + )?; mod_dev_dbg!(device, "[File {} VM {}]: VM created\n", file_id, id); resv.fill(KBox::new( @@ -437,6 +444,7 @@ impl File { ualloc, ualloc_priv, vm, + user_range, kernel_range, _dummy_mapping: dummy_mapping, }, @@ -635,7 +643,7 @@ impl File { return Err(EINVAL); } - if !VM_USER_RANGE.is_superset(range.clone()) { + if !file.inner().user_range.is_superset(range.clone()) { cls_pr_debug!( Errors, "gem_bind: Invalid map range {:#x}..{:#x} (not contained in user range)\n", @@ -725,7 +733,7 @@ impl File { let end = data.addr.checked_add(data.range).ok_or(EINVAL)?; let range = start..end; - if !VM_USER_RANGE.is_superset(range.clone()) { + if !file.inner().user_range.is_superset(range.clone()) { cls_pr_debug!( Errors, "gem_bind: Invalid unmap range {:#x}..{:#x} (not contained in user range)\n", diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index ca6e25c1df8c3..e2cb5a6a87198 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -281,6 +281,8 @@ pub(crate) trait Gpu: Send + Sync { fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; /// Frequency of the CPU timer that GPU timestamps are derived from. fn base_clock_hz(&self) -> u32; + /// Return the GPU address space geometry. + fn uat_geometry(&self) -> mmu::UatGeometry; /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; /// Create a new user command queue. @@ -677,7 +679,14 @@ impl GpuManager::ver { let map_kernel_to_user = false; Ok(KBox::new( - mmu::Uat::new(dev.into(), cfg, map_kernel_to_user)?, + mmu::Uat::new( + dev.into(), + mmu::UatGeometry { + ias: cfg.uat_ias, + oas: cfg.uat_oas, + }, + map_kernel_to_user, + )?, GFP_KERNEL, )?) } @@ -1298,6 +1307,10 @@ impl Gpu for GpuManager::ver { self.cfg.base_clock_hz } + fn uat_geometry(&self) -> mmu::UatGeometry { + self.uat.geometry() + } + fn new_vm(&self, kernel_range: Range) -> Result { self.uat.new_vm(self.ids.vm.next(), kernel_range) } diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index c51eb4b7f93e5..e221d02ff37eb 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -209,6 +209,8 @@ pub(crate) struct HwConfig { /// Base clock used used for timekeeping. pub(crate) base_clock_hz: u32, + /// Input address bits translated by each UAT page table root on this SoC. + pub(crate) uat_ias: u32, /// Output address space for the UAT on this SoC. pub(crate) uat_oas: u32, /// Number of dies on this SoC. diff --git a/drivers/gpu/drm/asahi/hw/t600x.rs b/drivers/gpu/drm/asahi/hw/t600x.rs index 58665f985ec38..85cf88054e44c 100644 --- a/drivers/gpu/drm/asahi/hw/t600x.rs +++ b/drivers/gpu/drm/asahi/hw/t600x.rs @@ -50,6 +50,7 @@ pub(crate) const HWCONFIG_T6002: super::HwConfig = HwConfig { gpu_core: GpuCore::G13C, base_clock_hz: 24_000_000, + uat_ias: 39, uat_oas: 42, num_dies: 2, max_num_clusters: 8, diff --git a/drivers/gpu/drm/asahi/hw/t602x.rs b/drivers/gpu/drm/asahi/hw/t602x.rs index 98a7ac2b76e57..6e2033ecb50dd 100644 --- a/drivers/gpu/drm/asahi/hw/t602x.rs +++ b/drivers/gpu/drm/asahi/hw/t602x.rs @@ -53,6 +53,7 @@ pub(crate) const HWCONFIG_T6022: super::HwConfig = HwConfig { gpu_core: GpuCore::G14D, base_clock_hz: 24_000_000, + uat_ias: 39, uat_oas: 42, num_dies: 2, max_num_clusters: 8, diff --git a/drivers/gpu/drm/asahi/hw/t8103.rs b/drivers/gpu/drm/asahi/hw/t8103.rs index 484bf6c3414f2..a2a76600c0a3b 100644 --- a/drivers/gpu/drm/asahi/hw/t8103.rs +++ b/drivers/gpu/drm/asahi/hw/t8103.rs @@ -13,6 +13,7 @@ pub(crate) const HWCONFIG: super::HwConfig = HwConfig { gpu_core: GpuCore::G13G, base_clock_hz: 24_000_000, + uat_ias: 39, uat_oas: 40, num_dies: 1, max_num_clusters: 1, diff --git a/drivers/gpu/drm/asahi/hw/t8112.rs b/drivers/gpu/drm/asahi/hw/t8112.rs index 3eba0457d76ac..bac2757175f2c 100644 --- a/drivers/gpu/drm/asahi/hw/t8112.rs +++ b/drivers/gpu/drm/asahi/hw/t8112.rs @@ -13,6 +13,7 @@ pub(crate) const HWCONFIG: super::HwConfig = HwConfig { gpu_core: GpuCore::G14G, base_clock_hz: 24_000_000, + uat_ias: 39, uat_oas: 40, num_dies: 1, max_num_clusters: 1, diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs index 2792797e49900..4c8a4e0fcc370 100644 --- a/drivers/gpu/drm/asahi/initdata.rs +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -540,7 +540,11 @@ impl<'a> InitDataBuilder::ver<'a> { unk_28: U64(0), // U64(0x11_00000000), // Unknown page //unk_30: U64(0x6f_ffff8000), - unk_30: U64(mmu::IOVA_UNK_PAGE), + unk_30: U64(mmu::UatGeometry { + ias: cfg.uat_ias, + oas: cfg.uat_oas, + } + .unk_page()), timestamp_area_base: U64(gpu::IOVA_KERN_TIMESTAMP_RANGE.start), // TODO: yuv matrices chip_id: cfg.chip_id, diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 3c6080ada6c8f..e87755157839f 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -87,7 +87,6 @@ use crate::{ }, fw, gem, - hw, mem, pgtable, slotalloc, @@ -101,11 +100,10 @@ pub(crate) use pgtable::{ UatPageTable, UAT_PGBIT, UAT_PGMSK, - UAT_PGSZ, // + UAT_PGSZ, + UAT_TOP_LEVEL_SHIFT, // }; -use pgtable::UAT_IAS; - use pin_init; const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; @@ -122,28 +120,54 @@ const UAT_USER_CTX: usize = UAT_NUM_CTX - UAT_USER_CTX_START; /// Lower/user base VA pub(crate) const IOVA_USER_BASE: u64 = UAT_PGSZ as u64; -/// Lower/user top VA -pub(crate) const IOVA_USER_TOP: u64 = 1 << (UAT_IAS as u64); -/// Lower/user VA range -pub(crate) const IOVA_USER_RANGE: Range = IOVA_USER_BASE..IOVA_USER_TOP; - -/// Upper/kernel base VA -// const IOVA_TTBR1_BASE: usize = 0xffffff8000000000; -/// Driver-managed kernel base VA -const IOVA_KERN_BASE: u64 = 0xffffffa000000000; -/// Driver-managed kernel top VA -const IOVA_KERN_TOP: u64 = 0xffffffb000000000; -/// Lower/user VA range -const IOVA_KERN_RANGE: Range = IOVA_KERN_BASE..IOVA_KERN_TOP; + +/// Top-level page table entry of the upper half that holds the driver-managed kernel VA range. +const IOVA_KERN_TOP_LEVEL_INDEX: u64 = 2; const TTBR_VALID: u64 = 0x1; // BIT(0) const TTBR_ASID_SHIFT: usize = 48; -/// Address of a special dummy page? -//const IOVA_UNK_PAGE: u64 = 0x6f_ffff8000; -pub(crate) const IOVA_UNK_PAGE: u64 = IOVA_USER_TOP - 2 * UAT_PGSZ as u64; -/// User VA range excluding the unk page -pub(crate) const IOVA_USER_USABLE_RANGE: Range = IOVA_USER_BASE..IOVA_UNK_PAGE; +/// UAT address space geometry of a given SoC. +#[derive(Debug, Copy, Clone)] +pub(crate) struct UatGeometry { + /// Input address bits translated by each page table root. + pub(crate) ias: u32, + /// Output (physical) address bits. + pub(crate) oas: u32, +} + +impl UatGeometry { + /// Lower/user top VA + const fn user_top(&self) -> u64 { + 1 << self.ias + } + + /// Lower/user VA range + pub(crate) const fn user_range(&self) -> Range { + IOVA_USER_BASE..self.user_top() + } + + /// Address of a special dummy page? + pub(crate) const fn unk_page(&self) -> u64 { + self.user_top() - 2 * UAT_PGSZ as u64 + } + + /// User VA range excluding the unk page + pub(crate) const fn user_usable_range(&self) -> Range { + IOVA_USER_BASE..self.unk_page() + } + + /// Upper/kernel base VA + const fn upper_base(&self) -> u64 { + !(self.user_top() - 1) + } + + /// Driver-managed kernel VA range + pub(crate) const fn kernel_range(&self) -> Range { + let start = self.upper_base() + (IOVA_KERN_TOP_LEVEL_INDEX << UAT_TOP_LEVEL_SHIFT); + start..(start + (1 << UAT_TOP_LEVEL_SHIFT)) + } +} /// A pre-allocated memory region for UAT management struct UatRegion { @@ -910,7 +934,7 @@ impl UatInner { /// Top-level UAT manager object pub(crate) struct Uat { dev: driver::AsahiDevRef, - cfg: &'static hw::HwConfig, + geometry: UatGeometry, inner: Arc, slots: slotalloc::SlotAllocator, @@ -1048,7 +1072,7 @@ impl Vm { dev: driver::AsahiDevice, uat_inner: Arc, kernel_range: Range, - cfg: &'static hw::HwConfig, + geometry: UatGeometry, ttb: Option, id: u64, ) -> Result { @@ -1056,15 +1080,15 @@ impl Vm { let is_kernel = ttb.is_some(); let page_table = if let Some(ttb) = ttb { - UatPageTable::new_with_ttb(ttb, IOVA_KERN_RANGE, cfg.uat_oas)? + UatPageTable::new_with_ttb(ttb, geometry.kernel_range(), geometry.ias, geometry.oas)? } else { - UatPageTable::new(cfg.uat_oas)? + UatPageTable::new(geometry.ias, geometry.oas)? }; let (va_range, gpuvm_range) = if is_kernel { - (IOVA_KERN_RANGE, kernel_range.clone()) + (geometry.kernel_range(), kernel_range.clone()) } else { - (IOVA_USER_RANGE, IOVA_USER_USABLE_RANGE) + (geometry.user_range(), geometry.user_usable_range()) }; let mm = mm::Allocator::new(va_range.start, va_range.range(), ())?; @@ -1609,12 +1633,17 @@ impl Uat { self.dev.clone(), self.inner.clone(), kernel_range, - self.cfg, + self.geometry, None, id, ) } + /// Returns the address space geometry of this UAT. + pub(crate) fn geometry(&self) -> UatGeometry { + self.geometry + } + /// Creates the reference-counted inner data for a new `Uat` instance. #[inline(never)] fn make_inner(dev: driver::AsahiDevice) -> Result> { @@ -1646,11 +1675,11 @@ impl Uat { ) } - /// Creates a new `Uat` instance given the relevant hardware config. + /// Creates a new `Uat` instance for the given address space geometry. #[inline(never)] pub(crate) fn new( dev: driver::AsahiDevice, - cfg: &'static hw::HwConfig, + geometry: UatGeometry, map_kernel_to_user: bool, ) -> Result { dev_info!(dev.as_ref(), "MMU: Initializing...\n"); @@ -1668,12 +1697,19 @@ impl Uat { } dev_info!(dev.as_ref(), "MMU: Creating kernel page tables\n"); - let kernel_lower_vm = Vm::new(dev.clone(), inner.clone(), IOVA_USER_RANGE, cfg, None, 1)?; + let kernel_lower_vm = Vm::new( + dev.clone(), + inner.clone(), + geometry.user_range(), + geometry, + None, + 1, + )?; let kernel_vm = Vm::new( dev.clone(), inner.clone(), - IOVA_KERN_RANGE, - cfg, + geometry.kernel_range(), + geometry, Some(ttb1), 0, )?; @@ -1684,7 +1720,7 @@ impl Uat { let uat = Self { dev: dev.clone(), - cfg, + geometry, kernel_vm, kernel_lower_vm, inner, diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 5332d1f310be1..269ed6690f532 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -50,9 +50,8 @@ const UAT_LVMSK: u64 = (UAT_LVSZ - 1) as u64; const UAT_LEVELS: usize = 3; -/// UAT input address space -pub(crate) const UAT_IAS: usize = 39; -const UAT_IASMSK: u64 = (1u64 << UAT_IAS) - 1; +/// Number of address bits below the index of a top-level page table entry. +pub(crate) const UAT_TOP_LEVEL_SHIFT: usize = UAT_PGBIT + (UAT_LEVELS - 1) * UAT_LVBIT; const PTE_TYPE_BITS: u64 = 3; const PTE_TYPE_LEAF_TABLE: u64 = 3; @@ -177,27 +176,35 @@ pub(crate) struct UatPageTable { ttb: PhysicalAddr, ttb_owned: bool, va_range: Range, + ias_mask: u64, oas_mask: u64, } impl UatPageTable { - pub(crate) fn new(oas: u32) -> Result { - mod_pr_debug!("UATPageTable::new: oas={}\n", oas); + pub(crate) fn new(ias: u32, oas: u32) -> Result { + mod_pr_debug!("UATPageTable::new: ias={} oas={}\n", ias, oas); let ttb_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; let ttb = Page::into_phys(ttb_page); Ok(UatPageTable { ttb, ttb_owned: true, - va_range: 0..(1u64 << UAT_IAS), + va_range: 0..(1u64 << ias), + ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, }) } - pub(crate) fn new_with_ttb(ttb: PhysicalAddr, va_range: Range, oas: u32) -> Result { + pub(crate) fn new_with_ttb( + ttb: PhysicalAddr, + va_range: Range, + ias: u32, + oas: u32, + ) -> Result { mod_pr_debug!( - "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} oas={}\n", + "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} ias={} oas={}\n", ttb, va_range, + ias, oas ); if ttb & (UAT_PGMSK as PhysicalAddr) != 0 { @@ -219,6 +226,7 @@ impl UatPageTable { ttb, ttb_owned: false, va_range, + ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, }) } @@ -255,10 +263,10 @@ impl UatPageTable { return Ok(()); } - let mut iova = iova_range.start & UAT_IASMSK; + let mut iova = iova_range.start & self.ias_mask; let mut last_iova = iova; // Handle the case where iova_range.end is just at the top boundary of the IAS - let end = ((iova_range.end - 1) & UAT_IASMSK) + 1; + let end = ((iova_range.end - 1) & self.ias_mask) + 1; let mut pt_addr: [Option; UAT_LEVELS] = Default::default(); pt_addr[UAT_LEVELS - 1] = Some(self.ttb); From 586e9dbfe2cf2f4d0259eef0e913c33b60659060 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:12:48 -0400 Subject: [PATCH 0621/2680] drm/asahi: pgtable: track ownership below firmware-shared roots A borrowed root does not determine ownership of its child tables. The handoff firmware interface leaves the driver owning all child tables in its managed range; another interface may publish its own child tables into that root. Make shared child-table ownership explicit when borrowing a root. In that mode, record each driver allocation before publishing its pointer and free only recorded tables. Keep the borrowed root itself unowned. The existing handoff caller selects non-shared children, preserving its allocation and teardown behavior without an extra ownership allocation. Driver-allocated roots also retain their existing ownership behavior. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5e21536704b56c38b478516d1f65bdcfef748a17) --- drivers/gpu/drm/asahi/mmu.rs | 8 +++++++- drivers/gpu/drm/asahi/pgtable.rs | 35 +++++++++++++++++++++++++++----- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index e87755157839f..b30e31ba53c1a 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1080,7 +1080,13 @@ impl Vm { let is_kernel = ttb.is_some(); let page_table = if let Some(ttb) = ttb { - UatPageTable::new_with_ttb(ttb, geometry.kernel_range(), geometry.ias, geometry.oas)? + UatPageTable::new_with_ttb( + ttb, + geometry.kernel_range(), + geometry.ias, + geometry.oas, + false, + )? } else { UatPageTable::new(geometry.ias, geometry.oas)? }; diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 269ed6690f532..0d814d6d100a8 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -175,6 +175,10 @@ impl Default for Prot { pub(crate) struct UatPageTable { ttb: PhysicalAddr, ttb_owned: bool, + /// Tables below a root the driver did not allocate may belong to the bootloader or the + /// firmware, so only the tables recorded here are ever freed. `None` if the driver owns the + /// tables below the root, as with the handoff firmware interface. + driver_tables: Option>, va_range: Range, ias_mask: u64, oas_mask: u64, @@ -188,17 +192,21 @@ impl UatPageTable { Ok(UatPageTable { ttb, ttb_owned: true, + driver_tables: None, va_range: 0..(1u64 << ias), ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, }) } + /// Borrow a root; track individual child ownership only when other agents + /// can install tables below it. Handoff roots have driver-owned children. pub(crate) fn new_with_ttb( ttb: PhysicalAddr, va_range: Range, ias: u32, oas: u32, + shared_tables: bool, ) -> Result { mod_pr_debug!( "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} ias={} oas={}\n", @@ -213,7 +221,7 @@ impl UatPageTable { if (va_range.start | va_range.end) & (UAT_PGMSK as u64) != 0 { return Err(EINVAL); } - // SAFETY: The TTB is should remain valid (if properly mapped), as it is bootloader-managed. + // SAFETY: The bootloader-managed root stays valid while this page table exists. if unsafe { Page::borrow_phys(&ttb) }.is_none() { pr_err!( "UATPageTable::new_with_ttb: ttb at {:#x} is not mapped (DT using no-map?)\n", @@ -225,6 +233,7 @@ impl UatPageTable { Ok(UatPageTable { ttb, ttb_owned: false, + driver_tables: shared_tables.then(KVec::new), va_range, ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, @@ -284,8 +293,7 @@ impl UatPageTable { level, phys ); - // SAFETY: Page tables for our VA ranges always come from Page::into_phys(). - unsafe { Page::from_phys(phys) }; + self.free_table(phys); } mod_pr_debug!("UATPageTable::with_pages: invalidate level {}\n", level); } @@ -318,6 +326,9 @@ impl UatPageTable { if upte_val == 0 && alloc { let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + if let Some(tables) = self.driver_tables.as_mut() { + tables.push(pt_page.phys(), GFP_KERNEL)?; + } let pt_paddr = Page::into_phys(pt_page); upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; upte.store(upte_val, Ordering::Relaxed); @@ -389,8 +400,7 @@ impl UatPageTable { level, phys ); - // SAFETY: Page tables for our VA ranges always come from Page::into_phys(). - unsafe { Page::from_phys(phys) }; + self.free_table(phys); } } } @@ -398,6 +408,21 @@ impl UatPageTable { Ok(()) } + /// Free a page table, unless it was not allocated by the driver. + fn free_table(&mut self, phys: PhysicalAddr) { + if let Some(tables) = self.driver_tables.as_mut() { + match tables.iter().position(|&table| table == phys) { + Some(index) => { + tables.swap_remove(index); + } + None => return, + } + } + // SAFETY: Page tables allocated by the driver always come from Page::into_phys(). Without + // `driver_tables`, every child table belongs to the driver. + unsafe { Page::from_phys(phys) }; + } + pub(crate) fn alloc_pages(&mut self, iova_range: Range) -> Result { mod_pr_debug!("UATPageTable::alloc_pages: {:#x?}\n", iova_range); self.with_pages(iova_range, true, false, |_, _| Ok(())) From 4d3e85121dbcb56b7c2cdc0db7554c1ecacb71c2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 01:47:16 -0400 Subject: [PATCH 0622/2680] drm/asahi: add the T8140 firmware interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provide boot, address translation and retained firmware resources for T8140, with independent render and compute queues behind the existing GPU interface. Keep visible command storage and VM pins through exact retirement, firmware drain or processor stop, and finish fences outside the device lock. Implement the G17 command ABI, per-VM status, logical ordering, asynchronous resource growth and recovery isolation together so every exposed operation has its firmware and ownership consumers. Validate shared recovery views and use bounded halted-slot identity reads to resolve ambiguous render blame. Retain unexecuted compute work for bounded replay when no firmware consumer has already depended on it. [ryan: adapted to 7.2.10 — minimal; for DJ: (1) DRM registration model: the G17 probe builds an UnregisteredDevice and a drm::Registration with the GPU in RegistrationData; the coprocessors are stopped from platform unbind() and from Drop of the platform driver data, which keeps an Arc of the G17 GPU. (2) MMIO: asahi-wip maps regs::Resources with the bound lifetime, so the G17 register access moved to a new regs::G17Resources and CpuControl holding device-managed IoMem::into_devres() mappings; the core-mask and revision helpers became free functions. (3) GPUVM: the G17 mapping epoch, context TLB invalidation, deferred BO teardown and prepared bind batches are re-applied on Asahi's 7.2 mmu.rs (sm_step_* callbacks, OpMapRequest, VmOpResources, field gpuvm); lock_vm() wraps GpuVm::exec_lock(); prepared maps prefetch the DMA segments and lock without the GEM object like Vm::bind_object(); VmInner.va_range is back; MappingBo relies on the 7.2 deferred GpuVmBo put plus deferred_cleanup(). (4) asahi-wip removed the GPU dev coredump: the G17 crash log is no longer published and Prot::elf_flags() is dropped. (5) shmem VMap: 7.2 VMapOwned/IoSysMap views replace the typed VMap. (6) g17/ uses AsahiDrmDevice where 7.1 had AsahiDevice; Queue::submit() and Vm::map_io() take Asahi's extra arguments; Gpu::as_any() keeps its name] Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c0e36ff2a57f2a461a96b80cebb741f5b76de7ad) --- drivers/gpu/drm/asahi/asahi.rs | 1 + drivers/gpu/drm/asahi/driver.rs | 97 +- drivers/gpu/drm/asahi/file.rs | 252 +- drivers/gpu/drm/asahi/g17/buffer.rs | 889 +++++ drivers/gpu/drm/asahi/g17/buffer/target.rs | 297 ++ drivers/gpu/drm/asahi/g17/channel.rs | 220 ++ drivers/gpu/drm/asahi/g17/command.rs | 428 +++ drivers/gpu/drm/asahi/g17/completion.rs | 390 +++ drivers/gpu/drm/asahi/g17/context.rs | 361 ++ drivers/gpu/drm/asahi/g17/context/identity.rs | 71 + drivers/gpu/drm/asahi/g17/dependency.rs | 83 + drivers/gpu/drm/asahi/g17/event.rs | 121 + drivers/gpu/drm/asahi/g17/fence.rs | 197 ++ drivers/gpu/drm/asahi/g17/freelist.rs | 575 ++++ drivers/gpu/drm/asahi/g17/fw/buffer.rs | 248 ++ drivers/gpu/drm/asahi/g17/fw/channels.rs | 522 +++ drivers/gpu/drm/asahi/g17/fw/compute.rs | 379 ++ drivers/gpu/drm/asahi/g17/fw/context.rs | 72 + drivers/gpu/drm/asahi/g17/fw/freelist.rs | 124 + drivers/gpu/drm/asahi/g17/fw/initdata.rs | 3063 +++++++++++++++++ drivers/gpu/drm/asahi/g17/fw/kick.rs | 539 +++ drivers/gpu/drm/asahi/g17/fw/mod.rs | 109 + drivers/gpu/drm/asahi/g17/fw/qos.rs | 34 + drivers/gpu/drm/asahi/g17/fw/queue.rs | 540 +++ drivers/gpu/drm/asahi/g17/fw/render.rs | 1309 +++++++ .../gpu/drm/asahi/g17/fw/render/storage.rs | 178 + drivers/gpu/drm/asahi/g17/hazard.rs | 120 + drivers/gpu/drm/asahi/g17/ids.rs | 60 + drivers/gpu/drm/asahi/g17/initdata.rs | 1311 +++++++ drivers/gpu/drm/asahi/g17/job.rs | 334 ++ drivers/gpu/drm/asahi/g17/kick.rs | 335 ++ drivers/gpu/drm/asahi/g17/mod.rs | 1129 ++++++ drivers/gpu/drm/asahi/g17/object.rs | 579 ++++ drivers/gpu/drm/asahi/g17/power.rs | 125 + drivers/gpu/drm/asahi/g17/preparation.rs | 186 + drivers/gpu/drm/asahi/g17/qos.rs | 244 ++ drivers/gpu/drm/asahi/g17/queue/admission.rs | 541 +++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 1507 ++++++++ .../gpu/drm/asahi/g17/queue/compute/exit.rs | 207 ++ .../drm/asahi/g17/queue/compute/retirement.rs | 214 ++ drivers/gpu/drm/asahi/g17/queue/mod.rs | 319 ++ drivers/gpu/drm/asahi/g17/queue/render.rs | 1459 ++++++++ .../asahi/g17/queue/render/dependencies.rs | 109 + .../gpu/drm/asahi/g17/queue/render/memory.rs | 708 ++++ .../drm/asahi/g17/queue/render/observation.rs | 85 + .../gpu/drm/asahi/g17/queue/render/parked.rs | 243 ++ .../drm/asahi/g17/queue/render/publication.rs | 164 + .../drm/asahi/g17/queue/render/retirement.rs | 348 ++ .../gpu/drm/asahi/g17/queue/render/warm.rs | 49 + drivers/gpu/drm/asahi/g17/recovery.rs | 286 ++ .../gpu/drm/asahi/g17/recovery/attribution.rs | 100 + drivers/gpu/drm/asahi/g17/recovery/slot.rs | 74 + drivers/gpu/drm/asahi/g17/runtime.rs | 869 +++++ drivers/gpu/drm/asahi/g17/runtime/backend.rs | 186 + .../drm/asahi/g17/runtime/compute_control.rs | 56 + .../gpu/drm/asahi/g17/runtime/compute_exit.rs | 166 + .../drm/asahi/g17/runtime/compute_replay.rs | 82 + drivers/gpu/drm/asahi/g17/runtime/events.rs | 419 +++ drivers/gpu/drm/asahi/g17/runtime/render.rs | 683 ++++ .../asahi/g17/runtime/render/dependencies.rs | 34 + .../gpu/drm/asahi/g17/runtime/render/exit.rs | 39 + .../gpu/drm/asahi/g17/runtime/render/grow.rs | 309 ++ .../drm/asahi/g17/runtime/render/observe.rs | 235 ++ .../gpu/drm/asahi/g17/runtime/render/pools.rs | 224 ++ .../drm/asahi/g17/runtime/render/prepare.rs | 597 ++++ .../drm/asahi/g17/runtime/render/recovery.rs | 27 + .../drm/asahi/g17/runtime/render/settle.rs | 235 ++ .../drm/asahi/g17/runtime/render/submit.rs | 306 ++ .../drm/asahi/g17/runtime/render/teardown.rs | 164 + drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 199 ++ drivers/gpu/drm/asahi/g17/status.rs | 102 + drivers/gpu/drm/asahi/g17/teardown.rs | 222 ++ drivers/gpu/drm/asahi/g17/validation.rs | 145 + drivers/gpu/drm/asahi/gem.rs | 14 +- drivers/gpu/drm/asahi/gpu.rs | 14 +- drivers/gpu/drm/asahi/hw/mod.rs | 2 + drivers/gpu/drm/asahi/hw/t8140.rs | 392 +++ drivers/gpu/drm/asahi/mem.rs | 62 + drivers/gpu/drm/asahi/mmu.rs | 933 ++++- drivers/gpu/drm/asahi/mmu/bind.rs | 202 ++ drivers/gpu/drm/asahi/mmu/context.rs | 231 ++ drivers/gpu/drm/asahi/mmu/lifetime.rs | 605 ++++ drivers/gpu/drm/asahi/mmu/shared.rs | 134 + drivers/gpu/drm/asahi/pgtable.rs | 100 + drivers/gpu/drm/asahi/regs.rs | 339 +- 85 files changed, 30167 insertions(+), 95 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/buffer.rs create mode 100644 drivers/gpu/drm/asahi/g17/buffer/target.rs create mode 100644 drivers/gpu/drm/asahi/g17/channel.rs create mode 100644 drivers/gpu/drm/asahi/g17/command.rs create mode 100644 drivers/gpu/drm/asahi/g17/completion.rs create mode 100644 drivers/gpu/drm/asahi/g17/context.rs create mode 100644 drivers/gpu/drm/asahi/g17/context/identity.rs create mode 100644 drivers/gpu/drm/asahi/g17/dependency.rs create mode 100644 drivers/gpu/drm/asahi/g17/event.rs create mode 100644 drivers/gpu/drm/asahi/g17/fence.rs create mode 100644 drivers/gpu/drm/asahi/g17/freelist.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/buffer.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/channels.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/compute.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/context.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/freelist.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/initdata.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/kick.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/mod.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/qos.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/queue.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/render.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/render/storage.rs create mode 100644 drivers/gpu/drm/asahi/g17/hazard.rs create mode 100644 drivers/gpu/drm/asahi/g17/ids.rs create mode 100644 drivers/gpu/drm/asahi/g17/initdata.rs create mode 100644 drivers/gpu/drm/asahi/g17/job.rs create mode 100644 drivers/gpu/drm/asahi/g17/kick.rs create mode 100644 drivers/gpu/drm/asahi/g17/mod.rs create mode 100644 drivers/gpu/drm/asahi/g17/object.rs create mode 100644 drivers/gpu/drm/asahi/g17/power.rs create mode 100644 drivers/gpu/drm/asahi/g17/preparation.rs create mode 100644 drivers/gpu/drm/asahi/g17/qos.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/admission.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/compute.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/compute/exit.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/mod.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/memory.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/observation.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/parked.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/publication.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/retirement.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/warm.rs create mode 100644 drivers/gpu/drm/asahi/g17/recovery.rs create mode 100644 drivers/gpu/drm/asahi/g17/recovery/attribution.rs create mode 100644 drivers/gpu/drm/asahi/g17/recovery/slot.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/backend.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/compute_control.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/events.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/exit.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/grow.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/observe.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/pools.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/settle.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/submit.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/teardown.rs create mode 100644 drivers/gpu/drm/asahi/g17/status.rs create mode 100644 drivers/gpu/drm/asahi/g17/teardown.rs create mode 100644 drivers/gpu/drm/asahi/g17/validation.rs create mode 100644 drivers/gpu/drm/asahi/hw/t8140.rs create mode 100644 drivers/gpu/drm/asahi/mmu/bind.rs create mode 100644 drivers/gpu/drm/asahi/mmu/context.rs create mode 100644 drivers/gpu/drm/asahi/mmu/lifetime.rs create mode 100644 drivers/gpu/drm/asahi/mmu/shared.rs diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi.rs index 2d0a78b3c7ea2..1e9bd0b3b3be6 100644 --- a/drivers/gpu/drm/asahi/asahi.rs +++ b/drivers/gpu/drm/asahi/asahi.rs @@ -13,6 +13,7 @@ mod file; mod float; mod fw; mod gem; +mod g17; mod gpu; mod hw; mod initdata; diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 5a58690cfdf46..ed588548b6c97 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -25,6 +25,7 @@ use kernel::{ use crate::{ debug, file, + g17, gem::AsahiObject, gpu, hw, @@ -44,6 +45,8 @@ pub(crate) struct AsahiPlatformDriver; pub(crate) struct AsahiPlatformDriverData<'bound> { _device: ARef, _reg: drm::Registration<'bound, AsahiDrmDriver>, + /// The G17 GPU, whose coprocessors must be stopped when the driver is unbound. + g17: Option>, } /// Holds a reference to the top-level GPU object. @@ -110,47 +113,73 @@ impl drm::driver::Driver for AsahiDrmDriver { } } -// OF Device ID table.s +/// Firmware interface selected by the hardware compatible. +pub(crate) enum ProbeConfig { + Legacy(&'static hw::HwConfig), + G17(&'static g17::Config), +} + +// OF Device ID table. kernel::of_device_table!( OF_TABLE, MODULE_OF_TABLE, ::IdInfo, [ - (of::DeviceId::new(c"apple,agx-t8103"), &hw::t8103::HWCONFIG), - (of::DeviceId::new(c"apple,agx-t8112"), &hw::t8112::HWCONFIG), + ( + of::DeviceId::new(c"apple,agx-t8140"), + ProbeConfig::G17(&hw::t8140::CONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t8103"), + ProbeConfig::Legacy(&hw::t8103::HWCONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t8112"), + ProbeConfig::Legacy(&hw::t8112::HWCONFIG) + ), ( of::DeviceId::new(c"apple,agx-t6000"), - &hw::t600x::HWCONFIG_T6000 + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6000) ), ( of::DeviceId::new(c"apple,agx-t6001"), - &hw::t600x::HWCONFIG_T6001 + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6001) ), ( of::DeviceId::new(c"apple,agx-t6002"), - &hw::t600x::HWCONFIG_T6002 + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6002) ), ( of::DeviceId::new(c"apple,agx-t6020"), - &hw::t602x::HWCONFIG_T6020 + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6020) ), ( of::DeviceId::new(c"apple,agx-t6021"), - &hw::t602x::HWCONFIG_T6021 + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6021) ), ( of::DeviceId::new(c"apple,agx-t6022"), - &hw::t602x::HWCONFIG_T6022 + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6022) ), ] ); /// Platform Driver implementation for `AsahiDriver`. impl platform::Driver for AsahiPlatformDriver { - type IdInfo = &'static hw::HwConfig; + type IdInfo = ProbeConfig; const OF_ID_TABLE: Option> = Some(&OF_TABLE); type Data<'bound> = AsahiPlatformDriverData<'bound>; + fn unbind<'bound>( + _pdev: &'bound platform::Device>, + this: Pin<&Self::Data<'bound>>, + ) { + if let Some(gpu) = this.g17.as_ref() { + // Driver data is dropped after devres, so CPU control must happen in unbind. + gpu.shutdown(); + } + } + /// Device probe function. fn probe<'bound>( pdev: &'bound platform::Device>, @@ -160,7 +189,10 @@ impl platform::Driver for AsahiPlatformDriver { dev_info!(pdev.as_ref(), "Probing...\n"); - let cfg = info.ok_or(ENODEV)?; + let cfg = match info.ok_or(ENODEV)? { + ProbeConfig::Legacy(cfg) => *cfg, + ProbeConfig::G17(cfg) => return AsahiPlatformDriver::probe_g17(pdev, cfg), + }; unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; @@ -217,8 +249,51 @@ impl platform::Driver for AsahiPlatformDriver { let driver = AsahiPlatformDriverData { _device: reg.device().into(), _reg: reg, + g17: None, }; Ok(driver) } } + +impl AsahiPlatformDriver { + fn probe_g17<'bound>( + pdev: &'bound platform::Device>, + cfg: &'static g17::Config, + ) -> Result> { + // SAFETY: The GPU performs DMA through a UAT whose output width is part of the SoC config. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; + let resources = regs::Resources::new(pdev)?; + let res = regs::G17Resources::new(pdev)?; + let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + let gpu = g17::Gpu::new(pdev, &drm, cfg, res)?; + let data_gpu = gpu.clone() as Arc; + let data = try_pin_init!(AsahiDrmRegistrationData { + gpu: data_gpu, + resources, + }); + // SAFETY: `reg` is stored in `AsahiPlatformDriverData` and dropped when the driver is + // unbound; it is never forgotten. + let reg = match unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0) } { + Ok(reg) => reg, + Err(error) => { + gpu.shutdown(); + return Err(error); + } + }; + Ok(AsahiPlatformDriverData { + _device: reg.device().into(), + _reg: reg, + g17: Some(gpu), + }) + } +} + +impl Drop for AsahiPlatformDriverData<'_> { + fn drop(&mut self) { + if let Some(gpu) = self.g17.as_ref() { + // Also covers failure to allocate platform driver data after the probe body returned. + gpu.shutdown(); + } + } +} diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index cde855fec5326..27ee72c0a060a 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -40,6 +40,7 @@ use kernel::error::code::*; use kernel::new_mutex; use kernel::prelude::*; use kernel::sync::{ + aref::ARef, Arc, Mutex, // }; @@ -60,6 +61,17 @@ const DEBUG_CLASS: DebugFlags = DebugFlags::File; pub(crate) const MAX_COMMANDS_PER_SUBMISSION: u32 = 64; +/// Context-view registry updates applied after a prepared VM batch commits. +enum MappingUpdate { + Map { + object: ARef, + address: u64, + size: u64, + offset: u64, + }, + Unmap(Range), +} + /// A client instance of an `mmu::Vm` address space. struct Vm { ualloc: Arc>, @@ -72,6 +84,18 @@ struct Vm { impl Drop for Vm { fn drop(&mut self) { + if self.vm.status().is_some() { + // Tracked jobs can outlive their file's VM handle. Their last guard performs + // these unmaps once no accepted work can still dereference the mappings. + if let Err(error) = self + .vm + .unmap_user_ranges(self.user_range.clone(), self.kernel_range.clone()) + { + pr_err!("Vm::Drop: deferred user unmap failed: {:?}\n", error); + } + self.vm.bo_deferred_cleanup(); + return; + } // When the user Vm is dropped, verify that no user mappings exists. let left_range = self.user_range.start..self.kernel_range.start; let right_range = self.kernel_range.end..self.user_range.end; @@ -302,6 +326,32 @@ impl File { let gpu = ®_data.gpu; + match data.param_group { + uapi::DRM_ASAHI_PARAM_GROUP_VM_STATUS if gpu.supports_vm_status() => { + return Self::get_vm_status(data, file); + } + uapi::DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR if gpu.supports_vm_status() => { + return Self::register_vm_status_mirror(data, file); + } + uapi::DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS => { + if data.pad != 0 + || data.size < core::mem::size_of::() as u64 + { + return Err(EINVAL); + } + let limits = gpu.queue_limits().ok_or(EINVAL)?; + let words = [limits.max_queues, limits.max_in_flight_per_queue, 0, 0]; + UserSlice::new( + UserPtr::from_addr(data.pointer as _), + core::mem::size_of_val(&words), + ) + .writer() + .write(&words)?; + return Ok(0); + } + _ => {} + } + if data.param_group != 0 || data.pad != 0 { cls_pr_debug!(Errors, "get_params: Invalid arguments\n"); return Err(EINVAL); @@ -328,6 +378,14 @@ impl File { gpu.get_params(&mut params)?; + if gpu.supports_vm_status() { + params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR as u64; + } + if gpu.queue_limits().is_some() { + params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES as u64; + } + if *module_parameters::fault_control.value() == 0xb { params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; } @@ -346,6 +404,55 @@ impl File { Ok(0) } + /// Query only file-owned VM state; this does not start recovery or wait for work. + fn get_vm_status(data: &uapi::drm_asahi_get_params, file: &DrmFile) -> Result { + let size = core::mem::size_of::(); + if data.pad != 0 || data.size < size as u64 { + return Err(EINVAL); + } + let user = UserPtr::from_addr(data.pointer as _); + let [vm_id, flags, _, pad] = UserSlice::new(user, size).reader().read::<[u32; 4]>()?; + if flags != 0 || pad != 0 { + return Err(EINVAL); + } + let status = Self::vm_status(file, vm_id)?; + UserSlice::new(user, size) + .writer() + .write(&[vm_id, 0, status.get() as u32, 0])?; + Ok(0) + } + + /// Register a file-owned GEM word as the mirror of a file-owned VM's status. + fn register_vm_status_mirror(data: &uapi::drm_asahi_get_params, file: &DrmFile) -> Result { + let size = core::mem::size_of::(); + if data.pad != 0 || data.size < size as u64 { + return Err(EINVAL); + } + let [vm_id, handle, offset, flags] = + UserSlice::new(UserPtr::from_addr(data.pointer as _), size) + .reader() + .read::<[u32; 4]>()?; + if flags != 0 || handle == 0 || offset % core::mem::align_of::() as u32 != 0 { + return Err(EINVAL); + } + let status = Self::vm_status(file, vm_id)?; + let owner = gem::ObjectRef::new(gem::Object::lookup_handle(file, handle)?); + status.attach_mirror(owner, offset as usize)?; + Ok(0) + } + + fn vm_status(file: &DrmFile, vm_id: u32) -> Result> { + let vms = file.inner().vms(); + let guard = vms.lock(); + Ok(guard + .get(vm_id as usize) + .ok_or(ENOENT)? + .vm + .status() + .ok_or(EINVAL)? + .clone()) + } + /// IOCTL: vm_create: Create a new `Vm`. pub(crate) fn vm_create( device: &AsahiDrmDevice, @@ -554,7 +661,7 @@ impl File { /// IOCTL: vm_bind: Map or unmap memory into a Vm. pub(crate) fn vm_bind( device: &AsahiDrmDevice, - _reg_data: &driver::AsahiDrmRegistrationData<'_>, + reg_data: &driver::AsahiDrmRegistrationData<'_>, data: &mut uapi::drm_asahi_vm_bind, file: &DrmFile, ) -> Result { @@ -570,6 +677,10 @@ impl File { return Err(EINVAL); } + if reg_data.gpu.as_any().is::() { + return Self::vm_bind_g17(data, file); + } + let vm_id = data.vm_id.try_into()?; let mut vec = KVec::new(); @@ -586,6 +697,118 @@ impl File { Ok(0) } + /// Validate all operations and retain their objects before changing a tracked VM. + fn vm_bind_g17(data: &uapi::drm_asahi_vm_bind, file: &DrmFile) -> Result { + let count = usize::try_from(data.num_binds)?; + let stride = usize::try_from(data.stride)?; + let size = stride.checked_mul(count).ok_or(EOVERFLOW)?; + let mut bytes = KVec::new(); + UserSlice::new(UserPtr::from_addr(data.userptr as _), size) + .reader() + .read_all(&mut bytes, GFP_KERNEL)?; + let (vm, user_range, kernel_range) = { + let vms = file.inner().vms(); + let guard = vms.lock(); + let entry = guard.get(data.vm_id as usize).ok_or(ENOENT)?; + ( + entry.vm.clone(), + entry.user_range.clone(), + entry.kernel_range.clone(), + ) + }; + let mut batch = mmu::PreparedUserBindBatch::new(count)?; + let mut updates = KVec::with_capacity(count, GFP_KERNEL)?; + let mut unmaps = 0; + // A fresh reader per declared stride also skips extension bytes in longer records. + for bytes in bytes.chunks_exact(stride) { + let op: uapi::drm_asahi_gem_bind_op = Reader::new(bytes).read_up_to(stride)?; + if op.range == 0 || (op.addr | op.range | op.offset) & mmu::UAT_PGMSK as u64 != 0 { + return Err(EINVAL); + } + let range = op.addr..op.addr.checked_add(op.range).ok_or(EINVAL)?; + if !user_range.is_superset(range.clone()) + || kernel_range.overlaps(range.clone()) + || vm.driver_range_overlaps(range.clone()) + { + return Err(EINVAL); + } + if op.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_UNBIND != 0 { + if op.offset != 0 + || op.flags != uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_UNBIND + || op.handle != 0 + { + return Err(EINVAL); + } + batch.push_unmap(vm.prepare_user_unmap(op.addr, op.range)?)?; + updates.push(MappingUpdate::Unmap(range), GFP_KERNEL)?; + unmaps += 1; + continue; + } + vm.wait_for_user_map_admission()?; + if op.flags + & !(uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_READ + | uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE + | uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SINGLE_PAGE) + != 0 + { + return Err(EINVAL); + } + let read = op.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_READ != 0; + let write = op.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE != 0; + let prot = match (read, write) { + (true, true) => mmu::PROT_GPU_SHARED_RW, + (true, false) => mmu::PROT_GPU_SHARED_RO, + (false, true) => mmu::PROT_GPU_SHARED_WO, + (false, false) => return Err(EINVAL), + }; + let single_page = op.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SINGLE_PAGE != 0; + let object = gem::Object::lookup_handle(file, op.handle)?; + let accessed = if single_page { + mmu::UAT_PGSZ as u64 + } else { + op.range + }; + if op.offset.checked_add(accessed).ok_or(EINVAL)? > u64::try_from(object.size())? { + return Err(EINVAL); + } + vm.validate_context_binding(op.addr, op.range, single_page)?; + batch.push_map(vm.prepare_bind_object( + &object, + op.addr, + op.range, + op.offset, + prot, + single_page, + )?)?; + updates.push( + MappingUpdate::Map { + object, + address: op.addr, + size: op.range, + offset: op.offset, + }, + GFP_KERNEL, + )?; + } + vm.reserve_deferred_user_unmaps(unmaps)?; + vm.commit_prepared_user_bind_batch(&mut batch)?; + for update in updates { + match update { + MappingUpdate::Map { + object, + address, + size, + offset, + } => { + vm.track_context_binding(object, address, size, offset)?; + } + MappingUpdate::Unmap(range) => vm.untrack_context_range(range), + } + } + vm.bo_deferred_cleanup(); + Ok(0) + } + pub(crate) fn do_gem_bind_unbind( device: &AsahiDrmDevice, vm_id: usize, @@ -783,6 +1006,7 @@ impl File { let vm = file_vm.borrow().vm.clone(); core::mem::drop(file_vm); vm.drop_mappings(bo)?; + vm.untrack_context_object(bo); if idx == usize::MAX { break; } @@ -1039,6 +1263,21 @@ impl File { data.queue_id, id ); + let mut vec = KVec::new(); + let copy_commands = |bytes: &mut KVec| { + UserSlice::new( + UserPtr::from_addr(data.cmdbuf as _), + data.cmdbuf_size as usize, + ) + .reader() + .read_all(bytes, GFP_KERNEL) + }; + let guarded = gpu.as_any().is::(); + // Once G17 output syncobjs are resolved, its queue must install a fence + // on every failure. Take the fallible userspace snapshot before that point. + if guarded { + copy_commands(&mut vec)?; + } let syncs = SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count)?; @@ -1050,17 +1289,12 @@ impl File { id ); - let mut vec = KVec::new(); - // Copy the command buffer into the kernel. Because we need to iterate // the command buffer twice, we do this in one big copy_from_user to // avoid TOCTOU issues. - let reader = UserSlice::new( - UserPtr::from_addr(data.cmdbuf as _), - data.cmdbuf_size as usize, - ) - .reader(); - reader.read_all(&mut vec, GFP_KERNEL)?; + if !guarded { + copy_commands(&mut vec)?; + } let objects = file.inner().objects(); let ret = queue.lock().submit( diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs new file mode 100644 index 0000000000000..3daa693fe7dbc --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -0,0 +1,889 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Parameter-buffer IDs, scene ownership and sparse tiler heap growth. +//! +//! The device mutex serializes the shared ID allocator. A render graph owns +//! its buffer ID independently of the submission references using that graph; +//! a published command keeps its reference until firmware retirement is witnessed. + +use super::fw::buffer as fw; +use crate::hw::t8140::resources as cfg; +pub(crate) mod target; + +use kernel::prelude::*; + +const BUFFER_IDS: usize = 127; + +#[derive(Copy, Clone)] +struct BufferId { + references: u8, + owner: bool, + token: u64, +} +impl BufferId { + fn free(self) -> bool { + self.references == 0 && !self.owner + } +} + +/// Device-wide identities for retained render parameter buffers. +pub(crate) struct BufferIds { + entries: [BufferId; BUFFER_IDS], + returned: [u8; BUFFER_IDS], + returned_count: usize, + next_token: u64, +} +impl BufferIds { + pub(crate) fn new() -> Self { + let mut returned = [0; BUFFER_IDS]; + for (index, entry) in returned.iter_mut().enumerate() { + *entry = (BUFFER_IDS - index - 1) as u8; + } + Self { + entries: [BufferId { + references: 0, + owner: false, + token: 0, + }; BUFFER_IDS], + returned, + returned_count: BUFFER_IDS, + next_token: 0, + } + } + fn remove_returned(&mut self, id: u8) { + if let Some(at) = self.returned[..self.returned_count] + .iter() + .position(|entry| *entry == id) + { + self.returned.copy_within(at + 1..self.returned_count, at); + self.returned_count -= 1; + } + } + fn return_id(&mut self, id: u8) { + if !self.returned[..self.returned_count].contains(&id) && self.returned_count < BUFFER_IDS { + self.returned[self.returned_count] = id; + self.returned_count += 1; + } + } + fn fresh_token(&mut self) -> u64 { + let token = if self.next_token == u64::MAX { + 0 + } else { + self.next_token + }; + self.next_token = token + 1; + token + } + pub(crate) fn allocate_render(&mut self) -> Result<(u8, u64)> { + let id = if self.returned_count != 0 { + self.returned_count -= 1; + self.returned[self.returned_count] + } else { + self.entries + .iter() + .position(|entry| entry.free()) + .ok_or(ENOSPC)? as u8 + }; + if !self.entries[usize::from(id)].free() { + return Err(EIO); + } + let token = self.fresh_token(); + self.entries[usize::from(id)] = BufferId { + references: 1, + owner: false, + token, + }; + Ok((id, token)) + } + + /// Retains the existing identity for a graph independently of its jobs. + pub(crate) fn reserve_owner(&mut self, id: u8) -> Result { + let entry = self.entries.get(usize::from(id)).ok_or(EINVAL)?; + if entry.owner { + return Ok(()); + } + if entry.free() { + let token = self.fresh_token(); + self.entries[usize::from(id)].token = token; + } + self.entries[usize::from(id)].owner = true; + self.remove_returned(id); + Ok(()) + } + pub(crate) fn acquire_owner(&mut self, id: u8) -> Result<(u8, u64)> { + let entry = self.entries.get_mut(usize::from(id)).ok_or(EINVAL)?; + if !entry.owner { + return Err(EINVAL); + } + entry.references = entry.references.checked_add(1).ok_or(ENOSPC)?; + Ok((id, entry.token)) + } + /// Preflight shared completion bookkeeping before any of its counters change. + pub(crate) fn can_release(&self, id: u8) -> Result { + let entry = self.entries.get(usize::from(id)).ok_or(EINVAL)?; + if entry.references == 0 { + return Err(EINVAL); + } + Ok(()) + } + pub(crate) fn release(&mut self, id: u8) -> Result { + let entry = self.entries.get_mut(usize::from(id)).ok_or(EINVAL)?; + entry.references = entry.references.checked_sub(1).ok_or(EINVAL)?; + if entry.free() { + self.return_id(id); + } + Ok(()) + } + pub(crate) fn unreserve_owner(&mut self, id: u8) -> Result { + let entry = self.entries.get_mut(usize::from(id)).ok_or(EINVAL)?; + if !entry.owner { + return Ok(()); + } + entry.owner = false; + if entry.free() { + self.return_id(id); + } + Ok(()) + } +} + +/// Each scene and its shorter scratch slot are one lifetime. A completed +/// paired render releases both; merely publishing a later scene does not. +pub(crate) struct Scenes { + owners: [Option<(u64, u8)>; fw::SCRATCH_SLOTS], + next: usize, +} +impl Scenes { + pub(crate) const fn new() -> Self { + Self { + owners: [None; fw::SCRATCH_SLOTS], + next: fw::FIRST_SCENE, + } + } + pub(crate) fn select(&self, ordinal: u64) -> Result { + if let Some((_, scene)) = self + .owners + .iter() + .flatten() + .find(|(owner, _)| *owner == ordinal) + { + return Ok(usize::from(*scene)); + } + (0..fw::SCENES) + .map(|offset| (self.next + offset) % fw::SCENES) + .find(|scene| self.owners[scene % fw::SCRATCH_SLOTS].is_none()) + .ok_or(ENOSPC) + } + pub(crate) fn commit(&mut self, ordinal: u64, scene: usize) -> Result { + if self.select(ordinal)? != scene { + return Err(EINVAL); + } + let slot = &mut self.owners[scene % fw::SCRATCH_SLOTS]; + if slot.is_none() { + *slot = Some((ordinal, scene as u8)); + self.next = (scene + 1) % fw::SCENES; + } + Ok(()) + } + pub(crate) fn release(&mut self, ordinal: u64) -> Result { + let slot = self + .owners + .iter_mut() + .find(|entry| entry.is_some_and(|(owner, _)| owner == ordinal)) + .ok_or(EINVAL)?; + *slot = None; + Ok(()) + } +} + +/// A snapshot of the block ring; all values are modulo its fixed capacity. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Cursors { + pub(crate) total: u32, + pub(crate) committed: u32, + pub(crate) read: u32, +} +impl Cursors { + fn valid(self) -> bool { + [self.total, self.committed, self.read] + .into_iter() + .all(|v| (v as usize) < cfg::TVB_BLOCK_SLOTS) + } + /// Firmware-paused growth permits consumption of the already committed + /// prefix. Retired-host growth requires the complete snapshot unchanged. + pub(crate) fn permits(self, after: Self, firmware_request: bool) -> bool { + if !self.valid() + || !after.valid() + || self.total != after.total + || self.committed != after.committed + { + return false; + } + if !firmware_request { + return self == after; + } + let capacity = cfg::TVB_BLOCK_SLOTS as u32; + (after.read + capacity - self.read) % capacity + <= (self.committed + capacity - self.read) % capacity + } +} + +/// Checked block-table indices for one bulk allocation. One ring slot stays +/// empty even when all physical backing has been retained by the owner. +pub(crate) struct Growth { + before: Cursors, + added: usize, + retained: usize, + target: usize, +} +impl Growth { + pub(crate) fn new(before: Cursors, added: usize) -> Result { + if !before.valid() || before.total != before.committed || added == 0 { + return Err(EINVAL); + } + let retained = (before.total as usize + cfg::TVB_BLOCK_SLOTS - before.read as usize) + % cfg::TVB_BLOCK_SLOTS; + if retained.checked_add(added).ok_or(EOVERFLOW)? >= cfg::TVB_BLOCK_SLOTS { + return Err(ENOSPC); + } + let target = (before.total as usize + added) % cfg::TVB_BLOCK_SLOTS; + Ok(Self { + before, + added, + retained, + target, + }) + } + pub(crate) fn target(&self) -> u32 { + self.target as u32 + } + pub(crate) fn retained(&self) -> usize { + self.retained + } + pub(crate) fn available(&self) -> usize { + self.retained + self.added + } + pub(crate) fn added_slot(&self, index: usize) -> Result { + if index >= self.added { + return Err(EINVAL); + } + Ok((self.before.total as usize + index) % cfg::TVB_BLOCK_SLOTS) + } + pub(crate) fn available_slot(&self, index: usize) -> Result { + if index >= self.available() { + return Err(EINVAL); + } + Ok((self.before.read as usize + index) % cfg::TVB_BLOCK_SLOTS) + } +} + +pub(crate) fn firmware_growth_target(current: usize) -> usize { + current + .saturating_add(fw::GROW_BLOCKS) + .min(cfg::TVB_MAX_BLOCKS) +} + +/// The high-water sample estimates capacity, not current occupancy. A growth +/// reserves three times the demand only once twice that demand no longer fits. +pub(crate) fn automatic_growth_target(used_pages: usize, current: usize) -> Option { + let maximum = fw::NORMAL_MAX_BLOCKS; + if current >= maximum { + return None; + } + let need = used_pages + .saturating_mul(2) + .div_ceil(cfg::TVB_PAGES_PER_BLOCK) + .min(maximum); + if need <= current { + return None; + } + Some( + used_pages + .saturating_mul(3) + .div_ceil(cfg::TVB_PAGES_PER_BLOCK) + .min(maximum), + ) +} + +use super::object::{CpuMap, KernelObject}; +use crate::{driver::AsahiDrmDevice, mmu}; +use core::ops::Range; + +pub(crate) fn compact_alias( + object: &KernelObject, + vm: &mmu::Vm, + source: Range, + align: u64, + prot: mmu::Prot, + guard: usize, +) -> Result { + for range in cfg::compact_ranges() { + match object.map_range(vm, source.clone(), range, align, prot, guard) { + Ok(mapping) => return Ok(mapping), + Err(error) if error == ENOSPC => {} + Err(error) => return Err(error), + } + } + Err(ENOSPC) +} + +/// Allocated aliases remain private until the owner installs them in its VM +/// cache. On rollback mappings are destroyed before backing memory. +pub(crate) struct TvbGrowth { + mappings: KVec, + addresses: KVec, + extension: KernelObject, +} + +/// Sparse 128-KiB data blocks at 160-KiB backing strides. Firmware owns their +/// contents; CPU zeroing occurs only on fresh backing, never on pool reuse. +pub(crate) struct Tvb { + addresses: KVec, + // Retain sparse-block backing until the pool and its aliases retire. + _primary: KernelObject, + extensions: KVec, +} +impl Tvb { + fn bytes(blocks: usize) -> Result { + blocks + .checked_sub(1) + .and_then(|v| v.checked_mul(cfg::TVB_BLOCK_STRIDE as usize)) + .and_then(|v| v.checked_add(cfg::TVB_BLOCK_SIZE)) + .ok_or(EOVERFLOW) + } + fn map_block( + object: &KernelObject, + vm: &mmu::Vm, + relative: usize, + ) -> Result { + let start = relative + .checked_mul(cfg::TVB_BLOCK_STRIDE as usize) + .ok_or(EOVERFLOW)?; + let source = start..start.checked_add(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW)?; + let mapping = compact_alias( + object, + vm, + source, + cfg::TVB_PAGE, + mmu::PROT_GPU_SHARED_RW, + cfg::TVB_BLOCK_STRIDE as usize - cfg::TVB_BLOCK_SIZE, + )?; + fw::compact_page(mapping.iova())?; + Ok(mapping) + } + pub(crate) fn new( + dev: &AsahiDrmDevice, + vm: &mmu::Vm, + blocks: usize, + ) -> Result<(Self, KVec)> { + if blocks == 0 || blocks > cfg::TVB_MAX_BLOCKS { + return Err(ERANGE); + } + let backing = KernelObject::backing(dev, Self::bytes(blocks)?, CpuMap::WriteCombined)?; + let mut mappings = KVec::with_capacity(blocks, GFP_KERNEL)?; + let mut addresses = KVec::with_capacity(blocks, GFP_KERNEL)?; + for index in 0..blocks { + let mapping = Self::map_block(&backing, vm, index)?; + addresses.push(mapping.iova(), GFP_KERNEL)?; + mappings.push(mapping, GFP_KERNEL)?; + } + Ok(( + Self { + addresses, + _primary: backing, + extensions: KVec::new(), + }, + mappings, + )) + } + pub(crate) fn blocks(&self) -> usize { + self.addresses.len() + } + pub(crate) fn addresses(&self) -> &[u64] { + &self.addresses + } + + pub(crate) fn prepare_growth( + &mut self, + dev: &AsahiDrmDevice, + vm: &mmu::Vm, + target: usize, + ) -> Result> { + if target <= self.blocks() { + return Ok(None); + } + if target > cfg::TVB_MAX_BLOCKS { + return Err(ERANGE); + } + let added = target - self.blocks(); + self.extensions.reserve(1, GFP_KERNEL)?; + self.addresses.reserve(added, GFP_KERNEL)?; + let backing = KernelObject::backing(dev, Self::bytes(added)?, CpuMap::WriteCombined)?; + let mut mappings = KVec::with_capacity(added, GFP_KERNEL)?; + let mut addresses = KVec::with_capacity(added, GFP_KERNEL)?; + for relative in 0..added { + let mapping = Self::map_block(&backing, vm, relative)?; + addresses.push(mapping.iova(), GFP_KERNEL)?; + mappings.push(mapping, GFP_KERNEL)?; + } + Ok(Some(TvbGrowth { + mappings, + addresses, + extension: backing, + })) + } +} + +use super::object::Allocator; +use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; +use kernel::sync::Arc; + +/// First-free metrics entries are shared by all parameter-buffer graphs. +/// An entry returns only when the graph and its firmware references retire. +pub(crate) struct MetricsIds { + used: [AtomicU64; 2], +} +impl MetricsIds { + pub(crate) fn new() -> Self { + Self { + used: [ + AtomicU64::new(0), + AtomicU64::new(!((1u64 << (fw::METRICS_SLOTS - 64)) - 1)), + ], + } + } +} + +pub(crate) struct MetricsLease { + _backing: Arc, + allocator: Arc, + slot: usize, + addresses: [u64; 2], +} +impl MetricsLease { + /// Clears only the selected retired metrics entry before reusing it. + pub(crate) fn new( + ids: &Arc, + object: &Arc, + bases: [u64; 2], + ) -> Result { + let mut selected = None; + for (bank, used) in ids.used.iter().enumerate() { + let mut value = used.load(Ordering::Acquire); + loop { + let bit = value.trailing_ones(); + if bit == 64 { + break; + } + match used.compare_exchange_weak( + value, + value | (1u64 << bit), + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => { + selected = Some(bank * 64 + bit as usize); + break; + } + Err(current) => value = current, + } + } + if selected.is_some() { + break; + } + } + let slot = selected.ok_or(ENOSPC)?; + // Construct the lease before any fallible address/view validation so + // every constructor failure returns the selected bit. + let mut lease = Self { + _backing: object.clone(), + allocator: ids.clone(), + slot, + addresses: [0; 2], + }; + let offset = slot * fw::METRICS_STRIDE; + lease.addresses = [ + bases[0].checked_add(offset as u64).ok_or(EOVERFLOW)?, + bases[1].checked_add(offset as u64).ok_or(EOVERFLOW)?, + ]; + let words = object.pointer(offset, fw::METRICS_STRIDE)?.cast::(); + object.word(offset)?; + for index in 0..fw::SCENES { + // SAFETY: The complete naturally aligned metrics entry is inside + // the retained object; the exclusive lease excludes other owners. + unsafe { words.add(index).write_volatile(0) }; + } + Ok(lease) + } +} +impl Drop for MetricsLease { + fn drop(&mut self) { + self.allocator.used[self.slot / 64] + .fetch_and(!(1u64 << (self.slot % 64)), Ordering::Release); + } +} + +pub(crate) struct ManagerArgs { + pub(crate) scratch: u64, + pub(crate) discard: u64, + pub(crate) buffer_id: u8, + pub(crate) generation: u64, + pub(crate) first_ordinal: u64, +} + +/// A retained parameter-buffer graph and all of its sparse backing. Client +/// mappings live in the VM's owner cache; graph replacement never returns a +/// published backing early merely because an error fence was signalled. +pub(crate) struct Manager { + _page_list: mmu::KernelMapping, + graph: KernelObject, + tvb: Tvb, + metrics: MetricsLease, + scenes: Scenes, + page_list_client: u64, + scratch: u64, + buffer_id: u8, + generation: u64, +} + +pub(crate) struct Submitted<'a> { + word: &'a AtomicU32, + next: u32, +} +impl Submitted<'_> { + pub(crate) fn commit(self) { + self.word.store(self.next, Ordering::Relaxed); + fence(Ordering::SeqCst); + } +} + +impl Manager { + pub(crate) fn new( + alloc: &Allocator<'_>, + vm: &mmu::Vm, + tvb: Tvb, + metrics: MetricsLease, + args: ManagerArgs, + ) -> Result { + let mut graph = alloc.kernel( + fw::SIZE, + cfg::TVB_PAGE, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let source = fw::PAGE_LIST..fw::PAGE_LIST + fw::PAGE_LIST_BYTES; + let page_list = compact_alias( + &graph, + vm, + source, + cfg::TVB_PAGE, + mmu::PROT_GPU_FW_SHARED_RW, + mmu::UAT_PGSZ, + )?; + let address = graph.gpu_va(); + graph.initialize::(0, |view| { + view.init(&fw::GraphArgs { + fw_va: address, + page_list_client: page_list.iova(), + metrics: metrics.addresses, + blocks: tvb.addresses(), + scratch: args.scratch, + discard: args.discard, + buffer_id: args.buffer_id.into(), + generation: args.generation, + }) + })?; + fence(Ordering::SeqCst); + let mut scenes = Scenes::new(); + scenes.commit(args.first_ordinal, fw::FIRST_SCENE)?; + let page_list_client = page_list.iova(); + Ok(Self { + _page_list: page_list, + graph, + tvb, + metrics, + scenes, + page_list_client, + scratch: args.scratch, + buffer_id: args.buffer_id, + generation: args.generation, + }) + } + pub(crate) fn buffer_id(&self) -> u8 { + self.buffer_id + } + pub(crate) fn blocks(&self) -> usize { + self.tvb.blocks() + } + pub(crate) fn state_va(&self) -> u64 { + self.graph.gpu_va() + } + pub(crate) fn page_list_va(&self) -> u64 { + self.page_list_client + } + pub(crate) fn trailer_va(&self) -> u64 { + self.graph.gpu_va() + fw::SCENE_TABLE as u64 + (fw::SCENES * size_of::()) as u64 + } + pub(crate) fn scene_va(&self, index: usize) -> Result { + if index >= fw::SCENES { + return Err(EINVAL); + } + Ok(self.graph.gpu_va() + fw::SCENE_TABLE as u64 + (index * size_of::()) as u64) + } + pub(crate) fn scene_registers(&self, index: usize) -> Result<(u64, u64)> { + fw::scene_registers(index, self.scratch, self.metrics.addresses[1]) + } + pub(crate) fn prepare_scene(&mut self, ordinal: u64) -> Result { + let scene = self.scenes.select(ordinal)?; + let offset = fw::SCENE_TABLE + scene * size_of::(); + let used = self + .graph + .word(offset + core::mem::offset_of!(fw::Scene, used))?; + let generation = self + .graph + .pointer(offset + core::mem::offset_of!(fw::Scene, generation), 8)?; + let id = self + .graph + .word(core::mem::offset_of!(fw::State, buffer_id))?; + used.store(0, Ordering::Relaxed); + // SAFETY: The complete packed field was checked before any write. + // This scene is exclusively retained or still unclaimed. + unsafe { generation.cast::().write_unaligned(self.generation) }; + id.store(u32::from(self.buffer_id), Ordering::Relaxed); + fence(Ordering::SeqCst); + Ok(scene) + } + pub(crate) fn claim_scene(&mut self, ordinal: u64, scene: usize) -> Result { + self.scenes.commit(ordinal, scene) + } + pub(crate) fn retire_scene(&mut self, ordinal: u64) -> Result { + self.scenes.release(ordinal) + } + pub(crate) fn prepare_submission(&mut self) -> Result> { + let word = self.graph.word(fw::COUNTER)?; + Ok(Submitted { + word, + next: word.load(Ordering::Relaxed).wrapping_add(1), + }) + } + pub(crate) fn automatic_target(&self) -> Result> { + let max_pages = self + .graph + .word(fw::CONTROL + core::mem::offset_of!(fw::Control, max_pages))?; + let reset = self + .graph + .word(fw::CONTROL + core::mem::offset_of!(fw::Control, reset))?; + fence(Ordering::Acquire); + let pages = max_pages.load(Ordering::Relaxed); + fence(Ordering::Release); + reset.store(1, Ordering::Relaxed); + Ok(automatic_growth_target(pages as usize, self.blocks())) + } + fn cursors(&self) -> Result { + Ok(Cursors { + total: self.graph.word(fw::CONTROL)?.load(Ordering::Relaxed), + committed: self.graph.word(fw::CONTROL + 4)?.load(Ordering::Relaxed), + read: self.graph.word(fw::CONTROL + 8)?.load(Ordering::Relaxed), + }) + } +} + +/// All allocations and inventory words prepared without publishing shared state. +/// The pair remains exclusively borrowed until commit or discard. +pub(crate) struct PreparedGrowth { + owner: u64, + previous: usize, + target: usize, + before: Cursors, + plan: Growth, + growth: TvbGrowth, + added: KVec<(usize, u32)>, + rebuilt: KVec, + pages: u32, + descriptor: Option<[u32; 4]>, +} +impl Manager { + /// Validate every retained sparse block and its reserved trailing guard in + /// one VM snapshot before constructing more aliases. + pub(crate) fn validate_vm(&self, vm: &mmu::Vm) -> Result { + if self.blocks() < cfg::TVB_INITIAL_PAGES.len() + || self.blocks() > cfg::TVB_MAX_BLOCKS + || !vm.covers_sparse_blocks_with_guards( + self.tvb.addresses(), + cfg::TVB_BLOCK_SIZE as u64, + cfg::TVB_BLOCK_STRIDE, + ) + { + return Err(EFAULT); + } + Ok(()) + } + + /// Allocate growth and precompute its words while the pair is exclusively + /// borrowed. A descriptor snapshot requests idle-host growth; None leaves + /// the firmware-owned page list and PB state untouched. + pub(crate) fn prepare_growth( + &mut self, + dev: &AsahiDrmDevice, + vm: &mmu::Vm, + target: usize, + descriptor: Option<[u32; 4]>, + ) -> Result> { + let previous = self.blocks(); + if target <= previous { + return Ok(None); + } + self.validate_vm(vm)?; + let Some(growth) = self.tvb.prepare_growth(dev, vm, target)? else { + return Ok(None); + }; + let before = self.cursors()?; + let plan = Growth::new(before, target - previous)?; + let firmware_request = descriptor.is_none(); + let pages = target + .checked_mul(cfg::TVB_PAGES_PER_BLOCK) + .ok_or(EOVERFLOW)? as u32; + let mut added = KVec::with_capacity(growth.addresses.len(), GFP_KERNEL)?; + for (index, &address) in growth.addresses.iter().enumerate() { + let first = fw::compact_page(address)?; + first + .checked_add(cfg::TVB_PAGES_PER_BLOCK as u32 - 1) + .ok_or(EOVERFLOW)?; + added.push((plan.added_slot(index)?, first), GFP_KERNEL)?; + } + let rebuilt_count = if firmware_request { + 0 + } else { + plan.available() * cfg::TVB_PAGES_PER_BLOCK + }; + let mut rebuilt = KVec::with_capacity(rebuilt_count, GFP_KERNEL)?; + if !firmware_request { + if self.cursors()? != before { + return Err(EAGAIN); + } + for ordinal in 0..plan.available() { + let first = if ordinal < plan.retained() { + let slot = plan.available_slot(ordinal)?; + self.graph + .word(fw::BLOCK_TABLE + slot * size_of::())? + .load(Ordering::Relaxed) + } else { + added[ordinal - plan.retained()].1 + }; + first + .checked_add(cfg::TVB_PAGES_PER_BLOCK as u32 - 1) + .ok_or(EOVERFLOW)?; + for page in 0..cfg::TVB_PAGES_PER_BLOCK { + rebuilt.push(first + page as u32, GFP_KERNEL)?; + } + } + } + let descriptor = + descriptor.map(|old| fw::reload_descriptor(old, self.page_list_client, pages)); + Ok(Some(PreparedGrowth { + owner: self.graph.gpu_va(), + previous, + target, + before, + plan, + growth, + added, + rebuilt, + pages, + descriptor, + })) + } + + /// The caller holds the device mutex after revalidating its preparation + /// epoch. The final alias installation and all firmware writes occur under + /// that same exclusion. The returned PB row is committed there as well. + pub(crate) fn commit_growth( + &mut self, + prepared: &mut Option, + install: impl FnOnce(usize, usize, &mut KVec) -> Result, + ) -> Result> { + let candidate = prepared.as_mut().ok_or(EINVAL)?; + if self.graph.gpu_va() != candidate.owner || self.blocks() != candidate.previous { + return Err(EBUSY); + } + let firmware_request = candidate.descriptor.is_none(); + let base = self.graph.pointer(0, fw::SIZE)?.cast::(); + // Bounds, actual alignment and every reserved host entry are checked + // before transferring aliases or consuming the caller-owned plan. + self.graph.word(0)?; + if self.tvb.extensions.len() == self.tvb.extensions.capacity() + || candidate.growth.addresses.len() + > self.tvb.addresses.capacity() - self.tvb.addresses.len() + { + return Err(EIO); + } + if !candidate.before.permits(self.cursors()?, firmware_request) { + return Err(EAGAIN); + } + // The callback retains its input on error. The caller can therefore + // drop every rejected mapping/backing after releasing the device lock. + install( + candidate.previous, + candidate.target, + &mut candidate.growth.mappings, + )?; + if !candidate.growth.mappings.is_empty() { + return Err(EIO); + } + let PreparedGrowth { + before, + plan, + growth, + added, + rebuilt, + pages, + descriptor, + .. + } = prepared.take().ok_or(EIO)?; + let TvbGrowth { + addresses, + extension, + .. + } = growth; + + // From here all vector capacity, indices, addresses and CPU views have + // been checked. Retain backing before publishing its block entries. + for address in addresses.into_iter() { + append_reserved(&mut self.tvb.addresses, address); + } + let write = |offset: usize, value: u32| { + // SAFETY: The full graph and base alignment were checked above. + // Every offset is an aligned field or a checked array index of + // Graph. No mutable reference aliases firmware-owned fields. + unsafe { AtomicU32::from_ptr(base.add(offset / 4)).store(value, Ordering::Relaxed) }; + }; + for (slot, first) in added.into_iter() { + let at = fw::BLOCK_TABLE + slot * size_of::(); + write(at, first); + write(at + 4, 0); + } + if descriptor.is_some() { + for (entry, value) in rebuilt.into_iter().enumerate() { + write(fw::PAGE_LIST + entry * 4, value); + } + write(core::mem::offset_of!(fw::State, pages), pages); + write(core::mem::offset_of!(fw::State, producer), plan.target()); + write(core::mem::offset_of!(fw::State, consumer), before.read); + write(core::mem::offset_of!(fw::State, last_page), pages - 1); + } + append_reserved(&mut self.tvb.extensions, extension); + write(fw::CONTROL, plan.target()); + fence(Ordering::SeqCst); + write(fw::CONTROL + 4, plan.target()); + fence(Ordering::SeqCst); + Ok(descriptor) + } +} + +/// Caller reserves all needed entries before any firmware-visible mutation. +fn append_reserved(values: &mut KVec, value: T) { + let length = values.len(); + values.spare_capacity_mut()[0].write(value); + // SAFETY: All callers preflighted this spare entry and initialized it once. + unsafe { values.set_len(length + 1) }; +} diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs new file mode 100644 index 0000000000000..ed0f3c71a4c93 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Rotating tile-map/heap metadata and the retained tail-pointer cache. +//! Larger targets replace only the objects that need more capacity. Every old +//! backing keeps the exact commands that still reference it until retirement. + +use super::compact_alias; +use crate::g17::{ + fw::render::RenderPass, + object::{CpuMap, KernelObject}, +}; +use crate::{ + driver::AsahiDrmDevice, + hw::t8140::{queues, resources as cfg}, + mmu, +}; +use kernel::prelude::*; + +const DEPTH: usize = queues::RENDER_DEPTH as usize; + +fn aligned(value: usize, alignment: usize) -> Result { + value + .checked_add(alignment - 1) + .map(|v| v & !(alignment - 1)) + .ok_or(EOVERFLOW) +} + +/// Per-command geometry, independent of the capacity already allocated. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Layout { + pub(crate) layer_offset: usize, + pub(crate) heap_offset: usize, + pub(crate) used: usize, + pub(crate) tpc_size: usize, + pub(crate) tvb_blocks: usize, +} +impl Layout { + pub(crate) fn new(pass: &RenderPass, clusters: u32) -> Result { + let width = pass.width as usize; + let height = pass.height as usize; + let layers = pass.layers as usize; + let (uw, uh) = (pass.utile_width as usize, pass.utile_height as usize); + if width == 0 + || height == 0 + || layers == 0 + || clusters == 0 + || uw == 0 + || uh == 0 + || 32 % uw != 0 + || 32 % uh != 0 + { + return Err(EINVAL); + } + let utiles = (32 / uw) * (32 / uh); + let tiles_x = width.div_ceil(32); + let tiles_y = height.div_ceil(32); + let macro_x = aligned(tiles_x.div_ceil(4), 4)?; + let macro_y = aligned(tiles_y.div_ceil(4), 4)?; + let macro_tiles = macro_x.checked_mul(macro_y).ok_or(EOVERFLOW)?; + let tilemap_size = aligned( + 5usize + .checked_mul(macro_tiles) + .and_then(|v| v.checked_mul(utiles)) + .ok_or(EOVERFLOW)?, + 4, + )? + .checked_mul(16) + .and_then(|v| v.checked_mul(layers)) + .ok_or(EOVERFLOW)?; + let tpc_size = 8usize + .checked_mul(utiles) + .and_then(|v| v.checked_mul(macro_tiles)) + .map(|v| v / 4 * 4) + .and_then(|v| v.checked_mul(16)) + .and_then(|v| v.checked_mul(layers)) + .and_then(|v| v.checked_mul(clusters as usize)) + .ok_or(EOVERFLOW)?; + let layer_offset = aligned(tilemap_size, 0x1000)?; + let heap_offset = layer_offset + .checked_add(if layers > 1 { 0x100 } else { 0 }) + .ok_or(EOVERFLOW)?; + let used = heap_offset.checked_add(0x200).ok_or(EOVERFLOW)?; + let tiles = tiles_x.checked_mul(tiles_y).ok_or(EOVERFLOW)?; + let mut tvb_blocks = aligned(tiles.div_ceil(128), 8)?.max(8); + if clusters > 1 { + tvb_blocks = tvb_blocks.max( + 7usize + .checked_add(2usize.checked_mul(layers).ok_or(EOVERFLOW)?) + .ok_or(EOVERFLOW)?, + ); + } + tvb_blocks = tvb_blocks.max(cfg::TVB_INITIAL_PAGES.len()); + Ok(Self { + layer_offset, + heap_offset, + used, + tpc_size, + tvb_blocks, + }) + } + pub(crate) fn tpc_capacity(&self) -> Result { + Ok(aligned(self.tpc_size, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) + } + pub(crate) fn record_stride(&self) -> Result { + Ok(aligned(self.used, mmu::UAT_PGSZ)?.max(cfg::TARGET_RECORD_MIN)) + } +} + +struct Object { + address: u64, + _mapping: mmu::KernelMapping, + backing: KernelObject, +} +impl Object { + fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm, size: usize) -> Result { + let backing = KernelObject::backing(dev, size, CpuMap::WriteCombined)?; + let prot = mmu::PROT_GPU_FW_SHARED_RW; + let mapping = compact_alias(&backing, vm, 0..size, cfg::TVB_PAGE, prot, mmu::UAT_PGSZ)?; + Ok(Self { + address: mapping.iova(), + _mapping: mapping, + backing, + }) + } +} +struct Retired { + _object: Object, + owners: [Option; DEPTH], +} + +#[derive(Copy, Clone, Debug)] +pub(crate) struct Addresses { + pub(crate) tilemap: u64, + pub(crate) layer: u64, + pub(crate) heap: u64, +} + +pub(crate) struct Target { + records: Object, + tpc: Object, + stride: usize, + slots: [Option; DEPTH], + retired: KVec, +} +impl Target { + /// Keep this allocation frame separate from the rest of the render graph. + #[inline(never)] + pub(crate) fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm, layout: &Layout) -> Result> { + let tpc = layout.tpc_capacity()?; + let stride = layout.record_stride()?; + if tpc > cfg::TPC_MAX || stride > cfg::TARGET_RECORD_MAX { + return Err(ERANGE); + } + let records = Object::new(dev, vm, stride * DEPTH)?; + let tpc = Object::new(dev, vm, tpc)?; + KBox::new( + Self { + records, + tpc, + stride, + slots: [None; DEPTH], + retired: KVec::new(), + }, + GFP_KERNEL, + ) + .map_err(Into::into) + } + pub(crate) fn tpc_va(&self) -> u64 { + self.tpc.address + } + + /// Only the span consumed by this render is cleared. A small render does + /// not clear a previous large target's complete retained record capacity. + pub(crate) fn prepare(&mut self, ordinal: u64, layout: &Layout) -> Result { + let slot = ordinal as usize % DEPTH; + if self.slots[slot].is_some_and(|owner| owner != ordinal) { + return Err(EBUSY); + } + if layout.used > self.stride { + return Err(ERANGE); + } + let offset = slot * self.stride; + let span = aligned(layout.used, mmu::UAT_PGSZ)?.min(self.stride); + self.records + .backing + .bytes_mut() + .get_mut(offset..offset + span) + .ok_or(ERANGE)? + .fill(0); + self.slots[slot] = Some(ordinal); + let tilemap = self + .records + .address + .checked_add(offset as u64) + .ok_or(EOVERFLOW)?; + Ok(Addresses { + tilemap, + layer: tilemap + .checked_add(layout.layer_offset as u64) + .ok_or(EOVERFLOW)?, + heap: tilemap + .checked_add(layout.heap_offset as u64) + .ok_or(EOVERFLOW)?, + }) + } + + /// Growth never shrinks a live allocation. A successfully replaced TPC + /// stays enlarged even if allocating the following record object fails. + pub(crate) fn ensure_capacity( + &mut self, + dev: &AsahiDrmDevice, + vm: &mmu::Vm, + layout: &Layout, + owners: &[Option; DEPTH], + ) -> Result { + let tpc = layout.tpc_capacity()?; + let stride = layout.record_stride()?; + let mut grew = false; + if tpc > self.tpc.backing.size() { + if tpc > cfg::TPC_MAX { + return Err(ERANGE); + } + Self::replace(&mut self.retired, &mut self.tpc, dev, vm, tpc, owners)?; + grew = true; + } + if stride > self.stride { + if stride > cfg::TARGET_RECORD_MAX { + return Err(ERANGE); + } + Self::replace( + &mut self.retired, + &mut self.records, + dev, + vm, + stride * DEPTH, + owners, + )?; + self.stride = stride; + self.slots = [None; DEPTH]; + grew = true; + } + Ok(grew) + } + fn replace( + retired: &mut KVec, + old: &mut Object, + dev: &AsahiDrmDevice, + vm: &mmu::Vm, + size: usize, + owners: &[Option; DEPTH], + ) -> Result { + let retain = owners.iter().any(Option::is_some); + if retain { + retired.reserve(1, GFP_KERNEL)?; + } + let new = Object::new(dev, vm, size)?; + let previous = core::mem::replace(old, new); + if retain { + super::append_reserved( + retired, + Retired { + _object: previous, + owners: *owners, + }, + ); + } + Ok(()) + } + pub(crate) fn retire(&mut self, ordinal: u64) -> Result { + let slot = &mut self.slots[ordinal as usize % DEPTH]; + if *slot == Some(ordinal) { + *slot = None; + } + let mut index = 0; + while index < self.retired.len() { + for owner in &mut self.retired[index].owners { + if *owner == Some(ordinal) { + *owner = None; + } + } + if self.retired[index].owners.iter().all(Option::is_none) { + drop(self.retired.remove(index)?); + } else { + index += 1; + } + } + Ok(()) + } + /// Cancels only an unpublished record; older referenced backing remains + /// owned until normal paired retirement or the firmware-stop boundary. + pub(crate) fn cancel(&mut self, ordinal: u64) { + let slot = &mut self.slots[ordinal as usize % DEPTH]; + if *slot == Some(ordinal) { + *slot = None; + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs new file mode 100644 index 0000000000000..3a74e889c5ec7 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Firmware-produced rings and the device-control producer. +//! +//! The device mutex serializes every caller. Entries are copied before a +//! single aligned consumer store returns their slots to the firmware. Trace +//! and statistics transport errors quarantine only that ring; event errors +//! propagate to the device owner. Device-control publication reserves one +//! empty slot and orders the record, power tally and producer together. + +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +use crate::driver::AsahiDrmDevice; + +use super::fw::{channels::*, initdata::status}; +use super::initdata::InitData; +use super::Role; + +/// Firmware rings whose contents do not affect host scheduling. +#[derive(Copy, Clone, Debug)] +enum ReportRing { + Trace = 0, + Statistics = 1, +} + +impl ReportRing { + fn geometry(self) -> (usize, usize, u32, u32) { + match self { + Self::Trace => ( + status::TRACE_STATE as usize, + status::TRACE_RING as usize, + TRACE_SLOTS, + TRACE_TYPES, + ), + Self::Statistics => ( + status::STATS_STATE as usize, + status::STATS_RING as usize, + STATISTICS_SLOTS, + STATISTICS_TYPES, + ), + } + } +} + +/// Host-side receive-ring state. A quarantined ring stays stopped until the +/// device is destroyed; firmware restart does not clear it. +#[derive(Debug)] +pub(crate) struct Rings { + quarantined: [[bool; 2]; 2], +} + +impl Rings { + pub(crate) const fn new() -> Self { + Self { + quarantined: [[false; 2]; 2], + } + } + + pub(crate) fn drain_trace(&mut self, dev: &AsahiDrmDevice, init: &InitData, role: Role) { + self.drain_report(dev, init, role, ReportRing::Trace); + } + + pub(crate) fn drain_statistics(&mut self, dev: &AsahiDrmDevice, init: &InitData, role: Role) { + self.drain_report(dev, init, role, ReportRing::Statistics); + } + + fn drain_report(&mut self, dev: &AsahiDrmDevice, init: &InitData, role: Role, ring: ReportRing) { + let quarantined = &mut self.quarantined[role as usize][ring as usize]; + if *quarantined { + return; + } + if let Err(error) = Self::consume_report(init, role, ring) { + *quarantined = true; + dev_err!( + dev.as_ref(), + "G17 {:?} {:?} ring stopped: {:?}\n", + role, + ring, + error + ); + } + } + + fn consume_report(init: &InitData, role: Role, ring: ReportRing) -> Result { + let (state, entries, capacity, allowed_types) = ring.geometry(); + let consumer_word = init.status_word(role, state)?; + let producer_word = init.status_word(role, state + RX_PRODUCER_OFFSET)?; + fence(Ordering::Acquire); + let mut consumer = consumer_word.load(Ordering::Relaxed); + let producer = producer_word.load(Ordering::Relaxed); + // The producer makes the entries it covers available to the host. + fence(Ordering::Acquire); + if consumer >= capacity || producer >= capacity { + return Err(EIO); + } + while consumer != producer { + let offset = entries + consumer as usize * EVENT_RECORD_SIZE; + let tag = match ring { + // Copy the entire trace entry before returning its slot. + ReportRing::Trace => init.status_read_words::(role, offset)?[0], + ReportRing::Statistics => init.status_read_words::<1>(role, offset)?[0], + }; + if tag >= u32::BITS || allowed_types & (1 << tag) == 0 { + return Err(EIO); + } + consumer = (consumer + 1) & (capacity - 1); + fence(Ordering::SeqCst); + consumer_word.store(consumer, Ordering::Relaxed); + if matches!(ring, ReportRing::Trace) { + fence(Ordering::SeqCst); + } + } + if matches!(ring, ReportRing::Statistics) { + fence(Ordering::SeqCst); + } + Ok(()) + } + + /// Publishes one primary device-control record. On EAGAIN the caller + /// retains the request and retries before consuming more dependent events. + /// The returned cursor is the value the consumer must reach to retire it. + /// The caller announces the publication through the coprocessor mailbox. + pub(crate) fn publish_control(init: &InitData, record: &ControlRecord) -> Result { + fence(Ordering::Acquire); + let consumer = init.control_consumer()?.load(Ordering::Relaxed); + let producer = init.control_producer()?.load(Ordering::Relaxed); + if consumer >= DEVICE_CONTROL_SLOTS || producer >= DEVICE_CONTROL_SLOTS { + return Err(EIO); + } + let next = (producer + 1) & (DEVICE_CONTROL_SLOTS - 1); + if next == consumer { + return Err(EAGAIN); + } + // InitData validates every mapping before the first write, then writes + // record, packed tally, barrier, producer, barrier without failure. + init.publish_control( + producer, + record.bytes(), + record.opcode().asserts_gpu_power(), + next, + )?; + Ok(next) + } +} + +/// One bounded event-ring pass. No shared-memory borrow survives `next`, so +/// the caller can service the copied event using the complete device owner. +pub(crate) struct Events { + role: Role, + remaining: u32, + unknown_reported: bool, +} + +impl Events { + /// A bounded pass can leave records written while its earlier slots were + /// consumed. Check both cursors again before the worker goes idle. + pub(crate) fn pending(init: &InitData, role: Role) -> Result { + let state = status::EVENT_STATE as usize; + fence(Ordering::Acquire); + let consumer = init.status_word(role, state)?.load(Ordering::Relaxed); + let producer = init + .status_word(role, state + RX_PRODUCER_OFFSET)? + .load(Ordering::Relaxed); + if consumer >= EVENT_SLOTS || producer >= EVENT_SLOTS { + return Err(EIO); + } + Ok(consumer != producer) + } + + pub(crate) fn new(role: Role) -> Self { + Self { + role, + remaining: EVENT_SLOTS, + unknown_reported: false, + } + } + + pub(crate) fn next(&mut self, dev: &AsahiDrmDevice, init: &InitData) -> Result> { + let role = self.role; + let state = status::EVENT_STATE as usize; + let consumer_word = init.status_word(role, state)?; + let producer_word = init.status_word(role, state + RX_PRODUCER_OFFSET)?; + while self.remaining != 0 { + init.validate_event_ring(role)?; + self.remaining -= 1; + fence(Ordering::Acquire); + let consumer = consumer_word.load(Ordering::Relaxed); + let producer = producer_word.load(Ordering::Relaxed); + if consumer >= EVENT_SLOTS || producer >= EVENT_SLOTS { + return Err(EIO); + } + if consumer == producer { + self.remaining = 0; + break; + } + let offset = status::EVENT_RING as usize + consumer as usize * EVENT_RECORD_SIZE; + let record = EventRecord::from_words(init.status_read_words(role, offset)?); + let event = record.decode(); + fence(Ordering::SeqCst); + consumer_word.store((consumer + 1) & (EVENT_SLOTS - 1), Ordering::Relaxed); + fence(Ordering::SeqCst); + match event? { + Event::Unknown(tag) => { + if !self.unknown_reported { + dev_warn!( + dev.as_ref(), + "G17 {:?} unknown firmware event {}\n", + role, + tag + ); + self.unknown_reported = true; + } + } + event => return Ok(Some(event)), + } + } + Ok(None) + } +} diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs new file mode 100644 index 0000000000000..fef478310c23b --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Decode a submission and validate its user resources before publishing any work. + +use super::fw::{ + kick::WriteRange, + render::{RenderPass, UscProgram, ZlsSurface}, +}; +use crate::util::{AnyBitPattern, Reader}; +use core::ops::Range; +use kernel::{prelude::*, uapi}; + +const MAX_ATTACHMENTS: usize = crate::microseq::MAX_ATTACHMENTS; +const USC_WINDOW_SIZE: u64 = 1 << 32; +const MAX_SAMPLERS: u32 = 1024; +const SAMPLER_SIZE: u64 = 8; +const PAGE_SIZE: u64 = 0x4000; +const ZLS_METADATA_STRIDE: u64 = 0x80; +const MAX_DIMENSION: u16 = 16384; +const MAX_LAYERS: u16 = 2048; +const MAX_TILE_BUFFER: u64 = 32768; +const PROCESS_EMPTY: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES; +const RSRC_SPEC_HI: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_RSRC_SPEC_HI; +const DBIAS_INT: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT; +const RENDER_FLAGS: u32 = PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT; + +/// Attachment settings persist until replaced, including by an empty list. +#[derive(Clone, Copy)] +pub(crate) struct Attachments { + ranges: [WriteRange; MAX_ATTACHMENTS], + count: usize, +} + +impl Attachments { + const EMPTY: Self = Self { + ranges: [WriteRange { va: 0, size: 0 }; MAX_ATTACHMENTS], + count: 0, + }; + + pub(crate) fn as_slice(&self) -> &[WriteRange] { + &self.ranges[..self.count] + } + + fn parse(bytes: &[u8]) -> Result { + let size = size_of::(); + if bytes.len() % size != 0 || bytes.len() / size > MAX_ATTACHMENTS { + return Err(EINVAL); + } + let mut list = Self::EMPTY; + let mut reader = Reader::new(bytes); + for range in &mut list.ranges[..bytes.len() / size] { + let item: uapi::drm_asahi_attachment = reader.read()?; + if item.flags != 0 || item.pad != 0 { + return Err(EINVAL); + } + *range = WriteRange { + va: item.pointer, + size: item.size, + }; + list.count += 1; + } + Ok(list) + } + + fn validate(&self, space: &impl AddressSpace) -> Result { + for range in self.as_slice() { + require_range(space, range.va, range.size, Access::Write)?; + } + Ok(()) + } +} + +// Commands stay inline in the submission plan, avoiding an allocation per render pass. +#[allow(clippy::large_enum_variant)] +pub(crate) enum Payload { + Render(uapi::drm_asahi_cmd_render, [Attachments; 2]), + Compute(uapi::drm_asahi_cmd_compute, Attachments), +} + +pub(crate) struct Command { + /// Render and compute prefix indices, before this command. + pub(crate) barriers: [u16; 2], + pub(crate) payload: Payload, +} + +/// Parser over the single userspace snapshot taken by the submit ioctl. +/// Call `finish` after consuming it to reject submissions with no hardware work. +pub(crate) struct Parser<'a> { + bytes: &'a [u8], + counts: [u16; 2], + attachments: [Attachments; 3], +} + +impl<'a> Parser<'a> { + pub(crate) fn new(bytes: &'a [u8]) -> Self { + Self { + bytes, + counts: [0; 2], + attachments: [Attachments::EMPTY; 3], + } + } + + pub(crate) fn next(&mut self) -> Result> { + while !self.bytes.is_empty() { + let mut reader = Reader::new(self.bytes); + let header: uapi::drm_asahi_cmd_header = reader.read()?; + let start = size_of::(); + let end = start + header.size as usize; + let bytes = self.bytes.get(start..end).ok_or(EINVAL)?; + self.bytes = &self.bytes[end..]; + let barriers = [header.vdm_barrier, header.cdm_barrier]; + match header.cmd_type as u32 { + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER + | uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { + if barriers.iter().zip(self.counts).any(|(&barrier, count)| { + barrier != uapi::DRM_ASAHI_BARRIER_NONE as u16 && barrier > count + }) || u32::from(self.counts[0]) + u32::from(self.counts[1]) + >= crate::file::MAX_COMMANDS_PER_SUBMISSION + { + return Err(EINVAL); + } + let payload = if header.cmd_type as u32 + == uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER + { + self.counts[0] += 1; + Payload::Render(payload(bytes)?, [self.attachments[0], self.attachments[1]]) + } else { + self.counts[1] += 1; + Payload::Compute(payload(bytes)?, self.attachments[2]) + }; + return Ok(Some(Command { barriers, payload })); + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_VERTEX_ATTACHMENTS + | uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_FRAGMENT_ATTACHMENTS + | uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_COMPUTE_ATTACHMENTS => { + if barriers != [uapi::DRM_ASAHI_BARRIER_NONE as u16; 2] { + return Err(EINVAL); + } + let slot = header.cmd_type as usize + - uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_VERTEX_ATTACHMENTS as usize; + self.attachments[slot] = Attachments::parse(bytes)?; + } + _ => return Err(EINVAL), + } + } + Ok(None) + } + + pub(crate) fn finish(&self) -> Result { + if !self.bytes.is_empty() || self.counts == [0; 2] { + return Err(EINVAL); + } + Ok(()) + } +} + +fn payload(bytes: &[u8]) -> Result { + if bytes + .get(size_of::()..) + .is_some_and(|tail| tail.iter().any(|&b| b != 0)) + { + return Err(EINVAL); + } + Reader::new(bytes).read_up_to(bytes.len()) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Access { + Read, + Write, + ReadWrite, +} + +/// The caller retains the VM mappings while validating and executing the command. +pub(crate) trait AddressSpace { + fn covers(&self, address: u64, size: u64, access: Access) -> bool; +} + +fn require_range(space: &impl AddressSpace, address: u64, size: u64, access: Access) -> Result { + if size == 0 || address.checked_add(size).is_none() || !space.covers(address, size, access) { + return Err(EINVAL); + } + Ok(()) +} + +/// A validated four-GiB USC window owned by one queue. +pub(crate) struct UscWindow { + base: u64, +} +impl UscWindow { + pub(crate) fn new(base: u64, range: Range) -> Result { + if base & (USC_WINDOW_SIZE - 1) != 0 + || base < range.start + || base + .checked_add(USC_WINDOW_SIZE) + .is_none_or(|end| end > range.end) + { + return Err(EINVAL); + } + Ok(Self { base }) + } + + fn program(&self, space: &impl AddressSpace, tagged_offset: u32) -> Result { + if tagged_offset == 0 { + return Err(EINVAL); + } + require_range( + space, + self.base + (tagged_offset & !7) as u64, + 4, + Access::Read, + ) + } +} + +fn sampler(space: &impl AddressSpace, heap: u64, count: u32) -> Result { + if count > MAX_SAMPLERS || (heap == 0) != (count == 0) || heap & (SAMPLER_SIZE - 1) != 0 { + return Err(EINVAL); + } + if count != 0 { + require_range(space, heap, u64::from(count) * SAMPLER_SIZE, Access::Read)?; + } + Ok(()) +} + +fn helper_is_empty(helper: &uapi::drm_asahi_helper_program) -> bool { + helper.binary == 0 && helper.cfg == 0 && helper.data == 0 +} + +fn surface( + space: &impl AddressSpace, + zls: &uapi::drm_asahi_zls_buffer, + layers: u16, +) -> Result { + if (zls.base == 0 && (zls.comp_base != 0 || zls.stride != 0 || zls.comp_stride != 0)) + || (zls.comp_base == 0 && zls.comp_stride != 0) + || (layers > 1 && zls.base != 0 && zls.stride == 0) + || (zls.stride != 0 && u64::from(zls.stride) & (PAGE_SIZE - 1) != 1) + || u64::from(zls.comp_stride) & (PAGE_SIZE - 1) != 0 + { + return Err(EINVAL); + } + if zls.base != 0 { + let stride = if zls.stride == 0 { + 0 + } else { + (u64::from(zls.stride) / PAGE_SIZE + 1) * PAGE_SIZE + }; + require_range( + space, + zls.base, + stride * u64::from(layers - 1) + 1, + Access::ReadWrite, + )?; + } + if zls.comp_base != 0 { + let stride = (u64::from(zls.comp_stride) / PAGE_SIZE + 1) * ZLS_METADATA_STRIDE; + require_range( + space, + zls.comp_base, + stride * u64::from(layers - 1) + 1, + Access::ReadWrite, + )?; + } + Ok(ZlsSurface { + va: zls.base, + compression_va: zls.comp_base, + stride: zls.stride.into(), + compression_stride: zls.comp_stride.into(), + }) +} + +// As with Payload, retain the bounded attachment lists inside the submission plan. +#[allow(clippy::large_enum_variant)] +pub(crate) enum Validated { + Render { + pass: RenderPass, + attachments: [Attachments; 2], + timestamps: [uapi::drm_asahi_timestamps; 2], + }, + Compute { + usc_base: u64, + cdm_va: u64, + cdm_end_va: u64, + sampler_heap_va: u64, + sampler_count: u32, + timestamps: uapi::drm_asahi_timestamps, + }, +} + +impl Payload { + pub(crate) fn validate( + &self, + window: &UscWindow, + space: &impl AddressSpace, + ) -> Result { + match self { + Self::Render(cmd, attachments) => { + if cmd.flags & !RENDER_FLAGS != 0 + || !(1..=MAX_DIMENSION).contains(&cmd.width_px) + || !(1..=MAX_DIMENSION).contains(&cmd.height_px) + || !(1..=MAX_LAYERS).contains(&cmd.layers) + || !matches!( + (cmd.utile_width_px, cmd.utile_height_px), + (16, 16) | (32, 16) | (32, 32) + ) + { + return Err(EINVAL); + } + let samples_log2 = match cmd.samples { + 1 => 0, + 2 => 1, + 4 => 2, + _ => return Err(EINVAL), + }; + if u64::from(cmd.sample_size_B) + * u64::from(cmd.utile_width_px) + * u64::from(cmd.utile_height_px) + * u64::from(cmd.samples) + > MAX_TILE_BUFFER + || cmd.vdm_ctrl_stream_base == 0 + || cmd.vdm_ctrl_stream_base & 3 != 0 + || cmd.isp_scissor_base == 0 + || cmd.isp_scissor_base & 7 != 0 + { + return Err(EINVAL); + } + require_range(space, cmd.vdm_ctrl_stream_base, 4, Access::Read)?; + require_range(space, cmd.isp_scissor_base, 8, Access::Read)?; + for (va, access) in [ + (cmd.isp_dbias_base, Access::Read), + (cmd.isp_oclqry_base, Access::Write), + ] { + if va != 0 { + if va & 7 != 0 { + return Err(EINVAL); + } + require_range(space, va, 8, access)?; + } + } + let depth = surface(space, &cmd.depth, cmd.layers)?; + let stencil = surface(space, &cmd.stencil, cmd.layers)?; + sampler(space, cmd.sampler_heap, cmd.sampler_count.into())?; + if !helper_is_empty(&cmd.vertex_helper) || !helper_is_empty(&cmd.fragment_helper) { + return Err(EINVAL); + } + for program in [&cmd.bg, &cmd.eot, &cmd.partial_bg, &cmd.partial_eot] { + window.program(space, program.usc)?; + } + for list in attachments { + list.validate(space)?; + } + let pass = RenderPass { + width: cmd.width_px.into(), + height: cmd.height_px.into(), + layers: cmd.layers, + utile_width: cmd.utile_width_px, + utile_height: cmd.utile_height_px, + samples_log2, + sample_size: cmd.sample_size_B, + process_empty_tiles: cmd.flags & PROCESS_EMPTY != 0, + msaa_control: cmd.ppp_multisamplectl, + ppp_control: cmd.ppp_ctrl.into(), + vdm_va: cmd.vdm_ctrl_stream_base, + scissor_va: cmd.isp_scissor_base, + depth_bias_va: cmd.isp_dbias_base, + depth_bias_is_int: cmd.flags & DBIAS_INT != 0, + occlusion_query_va: cmd.isp_oclqry_base, + sampler_heap_va: cmd.sampler_heap, + sampler_count: cmd.sampler_count.into(), + zls_control: cmd.zls_ctrl, + depth_dimensions: cmd.isp_zls_pixels.into(), + depth, + stencil, + depth_clear: cmd.isp_bgobjdepth, + stencil_clear: cmd.isp_bgobjvals, + merge_upper_x: cmd.isp_merge_upper_x, + merge_upper_y: cmd.isp_merge_upper_y, + usc_base: window.base, + bg: UscProgram { + offset: cmd.bg.usc, + rsrc_spec: u64::from(cmd.bg.rsrc_spec) + | (u64::from(cmd.bg_rsrc_spec_hi) << 32), + }, + partial_bg: UscProgram { + offset: cmd.partial_bg.usc, + rsrc_spec: u64::from(cmd.partial_bg.rsrc_spec) + | (u64::from(cmd.bg_partial_rsrc_spec_hi) << 32), + }, + eot_offset: cmd.eot.usc, + partial_eot_offset: cmd.partial_eot.usc, + rsrc_spec_hi: cmd.flags & RSRC_SPEC_HI != 0, + }; + Ok(Validated::Render { + pass, + attachments: *attachments, + timestamps: [cmd.ts_vtx, cmd.ts_frag], + }) + } + Self::Compute(cmd, attachments) => { + if cmd.flags != 0 + || cmd.cdm_ctrl_stream_base & 3 != 0 + || cmd.cdm_ctrl_stream_end & 3 != 0 + || cmd.cdm_ctrl_stream_end <= cmd.cdm_ctrl_stream_base + || !helper_is_empty(&cmd.helper) + { + return Err(EINVAL); + } + require_range( + space, + cmd.cdm_ctrl_stream_base, + cmd.cdm_ctrl_stream_end - cmd.cdm_ctrl_stream_base, + Access::Read, + )?; + sampler(space, cmd.sampler_heap, cmd.sampler_count)?; + attachments.validate(space)?; + Ok(Validated::Compute { + usc_base: window.base, + cdm_va: cmd.cdm_ctrl_stream_base, + cdm_end_va: cmd.cdm_ctrl_stream_end - 4, + sampler_heap_va: cmd.sampler_heap, + sampler_count: cmd.sampler_count, + timestamps: cmd.ts, + }) + } + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs new file mode 100644 index 0000000000000..d66c21480711b --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -0,0 +1,390 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Per-command fences, timestamp destinations and VM pins retained through retirement. +//! +//! Runtime queues own completions before making work visible. They defer fence signalling +//! until after the device mutex is released, and retain failed visible work until a retirement +//! witness or processor stop. An error fence never substitutes for that ownership boundary. + +use super::{ + context::{Context, WorkStateLease}, + fence::{self, CompletionFence, Member}, + object::{CpuMap, KernelObject}, + recovery::Class, + status::VmStatus, +}; +use crate::{driver::AsahiDrmDevice, file, hw::t8140::resources as cfg, mmu}; +use core::sync::atomic::{fence as barrier, AtomicBool, AtomicI32, AtomicU8, Ordering}; +use kernel::{ + dma_fence::{Fence, FenceContexts, RawDmaFence, UserFence}, + new_mutex, + prelude::*, + sync::{Arc, Mutex}, + uapi, xarray, +}; + +struct Destination { + mapping: Arc, + offset: usize, +} + +impl Destination { + fn resolve( + objects: Pin<&xarray::XArray>>, + timestamp: uapi::drm_asahi_timestamp, + ) -> Result> { + if timestamp.handle == 0 { + return Ok(None); + } + let object = objects + .lock() + .get(timestamp.handle.try_into()?) + .ok_or(ENOENT)? + .clone(); + let file::Object::TimestampBuffer(mapping) = object; + let offset: usize = timestamp.offset.try_into()?; + if offset.checked_add(size_of::()).ok_or(EINVAL)? > mapping.size() { + return Err(ERANGE); + } + Ok(Some(Self { mapping, offset })) + } + + fn write(&self, ticks: u64) -> Result { + const NSEC_PER_SEC: u64 = 1_000_000_000; + let ns = (ticks / cfg::COMMAND_TIMESTAMP_HZ) + .wrapping_mul(NSEC_PER_SEC) + .wrapping_add( + ticks % cfg::COMMAND_TIMESTAMP_HZ * NSEC_PER_SEC / cfg::COMMAND_TIMESTAMP_HZ, + ); + self.mapping.write_timestamp(self.offset, ns)?; + barrier(Ordering::Release); + Ok(()) + } +} + +/// Firmware writes two aligned words in a fresh write-combined page per compute command. +struct ComputeTimestamps { + // Remove the GPU alias before releasing its backing. + mapping: mmu::KernelMapping, + object: KernelObject, +} + +impl ComputeTimestamps { + fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { + let object = KernelObject::backing(dev, mmu::UAT_PGSZ, CpuMap::WriteCombined)?; + let mapping = object.alias_in( + vm, + cfg::TIMESTAMPS, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + )?; + Ok(Self { mapping, object }) + } +} + +#[pin_data] +pub(crate) struct Completion { + status: Arc, + fence: UserFence, + member: Member, + destinations: [Option; 4], + compute: Option, + #[pin] + vm_job: Mutex>, + #[pin] + render: Mutex>, + signalled: AtomicBool, + timeout_error: AtomicI32, + replays: AtomicU8, + replay_timeout: AtomicBool, + firmware_consumer: AtomicBool, + class: AtomicU8, +} + +impl Completion { + pub(crate) fn new( + dev: &AsahiDrmDevice, + contexts: &FenceContexts, + context: &Arc, + member: Member, + vm_job: mmu::VmJobGuard, + objects: Pin<&xarray::XArray>>, + timestamps: &[uapi::drm_asahi_timestamps], + compute: bool, + ) -> Result> { + if timestamps.len() != if compute { 1 } else { 2 } { + return Err(EINVAL); + } + let mut destinations = core::array::from_fn(|_| None); + for (index, timestamp) in timestamps.iter().enumerate() { + destinations[2 * index] = Destination::resolve(objects, timestamp.start)?; + destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end)?; + } + let fence = fence::independent(contexts)?; + let compute = if compute { + Some(ComputeTimestamps::new(dev, context.vm())?) + } else { + None + }; + Arc::pin_init( + pin_init!(Self { + status: context.status().clone(), + fence, + member, + destinations, + compute, + vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), + render <- new_mutex!(None, "G17 command render lease"), + signalled: AtomicBool::new(false), + timeout_error: AtomicI32::new(0), + replays: AtomicU8::new(0), + replay_timeout: AtomicBool::new(false), + firmware_consumer: AtomicBool::new(false), + class: AtomicU8::new(0), + }), + GFP_KERNEL, + ) + } + + pub(crate) fn fence(&self) -> Fence { + Fence::from_fence(&self.fence) + } + pub(crate) fn status(&self) -> &Arc { + &self.status + } + pub(crate) fn work_state(&self) -> Arc { + self.member.work_state() + } + pub(crate) fn has_timestamps(&self) -> bool { + self.destinations.iter().any(Option::is_some) + } + + pub(crate) fn compute_timestamp_va(&self) -> Result { + Ok(self.compute.as_ref().ok_or(EINVAL)?.mapping.iova()) + } + + /// The caller has proved that firmware retired this command, or has + /// received its scheduler kill acknowledgement with recovery closed. + /// These are the command's public timestamp words, not queue-private state. + pub(crate) fn compute_words(&self) -> Result<[u64; 2]> { + let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let words = [ + object.dword(0)?.load(Ordering::Relaxed), + object.dword(8)?.load(Ordering::Relaxed), + ]; + barrier(Ordering::Acquire); + Ok(words) + } + + pub(crate) fn replay_witness_words(&self) -> Option<[u64; 2]> { + if self.signalled.load(Ordering::Acquire) { + return None; + } + self.compute_words().ok() + } + + pub(crate) fn begin_replay(&self) -> bool { + let accepted = self + .replays + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| { + (count < 3).then_some(count + 1) + }) + .is_ok(); + if accepted { + self.replay_timeout.store(true, Ordering::Release); + } + accepted + } + + pub(crate) fn replay_count(&self) -> u8 { + self.replays.load(Ordering::Acquire) + } + + pub(crate) fn note_firmware_consumer(&self) { + self.firmware_consumer.store(true, Ordering::Release); + } + + pub(crate) fn has_firmware_consumer(&self) -> bool { + self.firmware_consumer.load(Ordering::Acquire) + } + + pub(crate) fn take_replay_timeout(&self) -> bool { + !self.signalled.load(Ordering::Acquire) && self.replay_timeout.swap(false, Ordering::AcqRel) + } + + /// Only a witnessed, retained command can be reset before republishing it. + pub(crate) fn reset_for_replay(&self) -> Result { + if self.signalled.load(Ordering::Acquire) { + return Err(ECANCELED); + } + let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let start = object.dword(0)?; + let end = object.dword(8)?; + start.store(0, Ordering::Relaxed); + end.store(0, Ordering::Relaxed); + barrier(Ordering::SeqCst); + Ok(()) + } + + /// The first timeout request retains its errno while the physical graph is + /// borrowed. Duplicate watchdog or cancellation callbacks cannot replace it. + pub(crate) fn defer_timeout(&self, error: Error) { + let _ = self.timeout_error.compare_exchange( + 0, + error.to_errno(), + Ordering::AcqRel, + Ordering::Acquire, + ); + } + + pub(crate) fn take_deferred_timeout(&self) -> Option { + let error = self.timeout_error.swap(0, Ordering::AcqRel); + (error != 0).then(|| Error::from_errno(error)) + } + + pub(crate) fn classify(&self, class: Class) { + self.class.store( + match class { + Class::Spared => 1, + Class::Guilty => 2, + }, + Ordering::Release, + ); + } + + pub(crate) fn spared(&self) -> bool { + self.class.load(Ordering::Acquire) == 1 + } + + /// Installed before the first firmware-visible prefix. Cancellation takes + /// the same lock after claiming the signal, so it cannot miss a late lease. + pub(crate) fn attach_render_lease( + &self, + lease: super::queue::admission::RenderLease, + ) -> Result { + let mut render = self.render.lock(); + if self.signalled.load(Ordering::Acquire) { + return Err(ECANCELED); + } + if render.is_some() { + return Err(EIO); + } + *render = Some(lease); + Ok(()) + } + + pub(crate) fn render_slot(&self) -> Option { + self.render + .lock() + .as_ref() + .map(super::queue::admission::RenderLease::slot) + } + + /// Called under the device mutex before deferring the error signal. + pub(crate) fn quarantine_render(&self) { + if let Some(lease) = self.render.lock().as_ref() { + lease.quarantine(); + } + } + + /// An exact whole-pair drain or processor stop allows rehabilitation. The + /// VM pin goes first; the still-held lease excludes reuse until both settle. + pub(crate) fn release_after_retirement(&self) { + drop(self.vm_job.lock().take()); + let lease = self.render.lock().take(); + if let Some(lease) = lease.as_ref() { + lease.rehabilitate_after_firmware_drain(); + } + drop(lease); + } + + fn release_healthy(&self) { + drop(self.render.lock().take()); + drop(self.vm_job.lock().take()); + } + + /// Exact retirement permits the physical lease and VM pin to be released + /// before waking userspace. A failure fence alone never establishes this. + pub(crate) fn complete(&self, result: Result<[u64; 4]>) { + if self.signalled.swap(true, Ordering::AcqRel) { + return; + } + let result = result.and_then(|ticks| { + for (destination, ticks) in self.destinations.iter().zip(ticks) { + if let Some(destination) = destination { + destination.write(ticks)?; + } + } + Ok(()) + }); + if let Err(error) = result { + if error != ECANCELED && !self.spared() { + self.status.record(error); + } + self.fence.set_error(error); + } + self.release_healthy(); + self.fence.signal(); + self.member.complete(result); + } + + /// A rejected, never-published command cannot poison the VM for contention. + pub(crate) fn fail_unpublished(&self, error: Error) { + if self.signalled.swap(true, Ordering::AcqRel) { + return; + } + self.status.record_if_device_loss(error); + self.fence.set_error(error); + self.release_healthy(); + self.fence.signal(); + self.member.complete(Err(error)); + } + + /// The physical queue must already be quarantined. Keep mappings pinned while + /// waking waiters, even if queue destruction requested cancellation. + pub(crate) fn fail_while_owned(&self, error: Error) { + if self.signalled.swap(true, Ordering::AcqRel) { + return; + } + if let Some(guard) = self.vm_job.lock().as_ref() { + guard.quarantine(); + } + if error != ECANCELED && !self.spared() { + self.status.record(error); + } + self.fence.set_error(error); + self.fence.signal(); + self.member.complete(Err(error)); + } +} + +/// Collected under the device mutex and executed only after releasing it. +/// Runtime queues establish the ownership witness before constructing `Retired` +/// or `Release`; `FailedOwned` leaves their physical and VM ownership intact. +pub(crate) enum Deferred { + Retired(Arc, Result<[u64; 4]>), + Drained(Arc, Result<[u64; 4]>), + FailedOwned(Arc, Error), + Unpublished(Arc, Error), + Release(Arc), +} + +impl Deferred { + pub(crate) fn finish(self) { + match self { + Self::Retired(completion, result) => { + completion.complete(result); + // An earlier failure may already have signalled this fence while + // retaining the pin. Retirement is a separate release witness. + completion.release_after_retirement(); + } + Self::Drained(completion, result) => { + completion.release_after_retirement(); + completion.complete(result); + } + Self::FailedOwned(completion, error) => completion.fail_while_owned(error), + Self::Unpublished(completion, error) => completion.fail_unpublished(error), + Self::Release(completion) => completion.release_after_retirement(), + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs new file mode 100644 index 0000000000000..c3a3c6e87a2f6 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -0,0 +1,361 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Logical queue ownership of scheduler, QoS and work-state identities. +//! +//! Installed bindings retain this context until replacement is acknowledged. +//! A job keeps its work-state lease through exact retirement, even if its error +//! fence has already been signalled. Context-local accounting never waits for GPU work. + +mod identity; +use super::{ + freelist::{RenderIds, RenderPool}, + fw::context as fw, + object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, + status::VmStatus, +}; +use crate::mmu::{self, ExecutionContext, Vm, UAT_PGSZ}; +use core::sync::atomic::{fence, AtomicBool, AtomicU32, Ordering}; +use identity::{RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; +use kernel::{ + new_mutex, + prelude::*, + sync::{Arc, Mutex}, +}; + +/// QoS IDs belong to one device; the two lowest values are reserved. +#[pin_data] +pub(crate) struct QosIds { + #[pin] + used: Mutex, +} +impl QosIds { + pub(crate) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { used <- new_mutex!(3, "G17 QoS IDs") }), + GFP_KERNEL, + ) + } + fn acquire(self: &Arc) -> Result { + let mut used = self.used.lock(); + let id = used.trailing_ones(); + if id == 128 { + return Err(ENOSPC); + } + *used |= 1u128 << id; + Ok(QosLease { + ids: self.clone(), + id: id as u8, + }) + } +} +struct QosLease { + ids: Arc, + id: u8, +} +impl Drop for QosLease { + fn drop(&mut self) { + *self.ids.used.lock() &= !(1u128 << self.id); + } +} + +struct WorkStorage { + object: PooledObject, + words: [Option; STATE_COUNT], + counts: [u32; STATE_COUNT], + render: RenderSlots, +} +impl WorkStorage { + fn initialize(&mut self, word: u32) -> Result { + let slot = (word & 63) as usize; + if slot >= STATE_COUNT { + return Err(EINVAL); + } + let counter = UAT_PGSZ + slot * 4; + let node = fw::WorkNode::new(self.object.gpu_va() + counter as u64, word).ok_or(EINVAL)?; + // The exact work-state lease excludes a previous firmware owner. Only + // the node's 0xc8 initialized bytes are replaced; the stride tail is retained. + self.object.write(slot * NODE_STRIDE, node)?; + self.object.word(counter)?.store(0, Ordering::Relaxed); + self.words[slot] = Some(word); + self.counts[slot] = 0; + fence(Ordering::Release); + Ok(()) + } + fn change_count(&mut self, word: u32, kicks: u32, add: bool) -> Result { + let slot = (word & 63) as usize; + if self.words.get(slot) != Some(&Some(word)) { + return Err(EINVAL); + } + let value = if add { + self.counts[slot].checked_add(kicks) + } else { + self.counts[slot].checked_sub(kicks) + } + .ok_or(EOVERFLOW)?; + self.object + .word(UAT_PGSZ + slot * 4)? + .store(value, Ordering::Relaxed); + self.counts[slot] = value; + fence(Ordering::SeqCst); + Ok(value) + } +} + +pub(crate) struct Context { + execution: Arc, + status: Arc, + scheduler: Arc, + scheduler_generation: u32, + published: AtomicBool, + qos: QosLease, + owner_pid: u32, + states: WorkStates, + work: Arc>, + work_base: u64, + render: Arc>>>, +} +impl Context { + pub(crate) fn new( + alloc: &Allocator<'_>, + pool: &Arc, + qos: &Arc, + execution: Arc, + owner_pid: u32, + ) -> Result { + let status = execution.vm().status().ok_or(EINVAL)?.clone(); + let qos = qos.acquire()?; + let mut scheduler = alloc.kernel( + UAT_PGSZ, + UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + scheduler.initialize::(0, |value| { + value.init(); + Ok(()) + })?; + fence(Ordering::SeqCst); + let scheduler_generation = super::ids::OBJECTS.object()?; + let scheduler = Arc::new(scheduler, GFP_KERNEL)?; + let mut object = pool.allocate( + alloc, + Family::WorkState, + 2 * UAT_PGSZ, + UAT_PGSZ, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let work_base = object.gpu_va(); + object.write( + UAT_PGSZ + 0x400, + fw::WorkHead { + next: 0, + previous: work_base + (UAT_PGSZ + 0x400) as u64, + unk10: 0, + unk14: 0, + }, + )?; + fence(Ordering::Release); + let work = Arc::pin_init( + new_mutex!( + WorkStorage { + object, + words: [None; STATE_COUNT], + counts: [0; STATE_COUNT], + render: RenderSlots::new() + }, + "G17 work states" + ), + GFP_KERNEL, + )?; + let render = Arc::pin_init(new_mutex!(None, "G17 context private memory"), GFP_KERNEL)?; + Ok(Self { + execution, + status, + scheduler, + scheduler_generation, + published: AtomicBool::new(false), + qos, + owner_pid, + states: WorkStates::new(), + work, + work_base, + render, + }) + } + pub(crate) fn vm(&self) -> &Vm { + self.execution.vm() + } + pub(crate) fn status(&self) -> &Arc { + &self.status + } + pub(crate) fn id(&self) -> u32 { + self.execution.id() + } + + pub(crate) fn generation(&self) -> u8 { + self.execution.generation() + } + pub(crate) fn is_current(&self) -> bool { + self.execution.is_current() + } + pub(crate) fn owner_pid(&self) -> u32 { + self.owner_pid + } + pub(crate) fn qos_id(&self) -> u8 { + self.qos.id + } + pub(crate) fn scheduler_va(&self) -> u64 { + self.scheduler.gpu_va() + } + /// Installed physical channels retain this page after the logical execution + /// context and its VM lease have completed their scheduler release. + pub(crate) fn scheduler_owner(&self) -> Arc { + self.scheduler.clone() + } + pub(crate) fn scheduler_generation(&self) -> u32 { + self.scheduler_generation + } + pub(crate) fn work_head_va(&self) -> u64 { + self.work_base + (UAT_PGSZ + 0x400) as u64 + } + /// Shared scheduler list head, sampled under the context's storage lock. + pub(crate) fn work_head(&self) -> Result<[u64; 2]> { + let work = self.work.lock(); + let head = [ + work.object.dword(UAT_PGSZ + 0x400)?.load(Ordering::Relaxed), + work.object.dword(UAT_PGSZ + 0x408)?.load(Ordering::Relaxed), + ]; + fence(Ordering::Acquire); + Ok(head) + } + + pub(crate) fn is_published(&self) -> bool { + self.published.load(Ordering::Acquire) + } + pub(crate) fn mark_published(&self) { + self.published.store(true, Ordering::Release); + } + + pub(crate) fn release_identity(&self) -> Result<[u8; 4]> { + let mut result = [0; 4]; + fence(Ordering::Acquire); + for (value, offset) in result.iter_mut().zip([0, 1, 4, 0x26]) { + let pointer = self.scheduler.pointer(offset, 1)?; + // SAFETY: pointer checked against retained scheduler allocation; + // a byte read observes firmware state without borrowing its fields. + *value = unsafe { core::ptr::read_volatile(pointer) }; + } + Ok(result) + } + pub(crate) fn render_pool( + &self, + alloc: &Allocator<'_>, + ids: &Arc, + render_global: &Vm, + ) -> Result> { + if !self.is_current() { + return Err(EFAULT); + } + let mut render = self.render.lock(); + if let Some(pool) = &*render { + return Ok(pool.clone()); + } + let pool = Arc::new( + RenderPool::new(alloc, ids, render_global, self.vm())?, + GFP_KERNEL, + )?; + *render = Some(pool.clone()); + Ok(pool) + } + pub(crate) fn render_pools_released(&self, released: impl Fn(u32, u64) -> bool) -> bool { + self.render + .lock() + .as_ref() + .is_none_or(|pool| released(u32::from(pool.id()), pool.control_va())) + } +} + +/// Shared by all engines of one ioctl. Queued work initially consumes no node. +pub(crate) struct WorkStateLease { + context: Arc, + word: AtomicU32, +} +impl WorkStateLease { + pub(crate) fn deferred(context: Arc) -> Self { + Self { + context, + word: AtomicU32::new(FREE), + } + } + pub(crate) fn acquire(&self) -> Result { + let current = self.word(); + if current != FREE { + return Ok(current); + } + let Some(candidate) = self.context.states.allocate() else { + let current = self.word(); + return if current != FREE { + Ok(current) + } else { + Err(EBUSY) + }; + }; + if let Err(error) = self.context.work.lock().initialize(candidate) { + self.context.states.release(candidate); + return Err(error); + } + match self + .word + .compare_exchange(FREE, candidate, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => Ok(candidate), + Err(installed) => { + self.context.states.release(candidate); + Ok(installed) + } + } + } + pub(crate) fn word(&self) -> u32 { + self.word.load(Ordering::Acquire) + } + pub(crate) fn node_va(&self) -> Result { + let word = self.word(); + if word == FREE { + return Err(EINVAL); + } + Ok(self.context.work_base + u64::from(word & 63) * NODE_STRIDE as u64) + } + pub(crate) fn add_submitted_kicks(&self, kicks: u32) -> Result { + self.context + .work + .lock() + .change_count(self.word(), kicks, true) + } + pub(crate) fn cancel_unpublished_kicks(&self, kicks: u32) -> Result { + self.context + .work + .lock() + .change_count(self.word(), kicks, false) + } + + pub(crate) fn render_slot(&self, pair: u8) -> Result { + self.context.work.lock().render.get(pair).ok_or(ENOSPC) + } + pub(crate) fn channel_payload(&self, pair: u8, ordinal: u64) -> Result { + self.context + .work + .lock() + .render + .payload(pair, ordinal) + .ok_or(EINVAL) + } +} +impl Drop for WorkStateLease { + fn drop(&mut self) { + let word = self.word(); + if word != FREE { + self.context.states.release(word); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/context/identity.rs b/drivers/gpu/drm/asahi/g17/context/identity.rs new file mode 100644 index 0000000000000..6c48367e2d686 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/context/identity.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Context-local work-state identities. The low six bits address a node; +//! the complete sequence distinguishes its owner until exact retirement. + +use core::sync::atomic::{AtomicU32, Ordering}; + +pub(super) const FREE: u32 = u32::MAX; +pub(super) const STATE_COUNT: usize = 63; +pub(super) const NODE_STRIDE: usize = 0x100; + +pub(super) struct WorkStates { + next: AtomicU32, + live: [AtomicU32; 64], +} +impl WorkStates { + pub(super) const fn new() -> Self { + Self { + next: AtomicU32::new(0), + live: [const { AtomicU32::new(FREE) }; 64], + } + } + pub(super) fn allocate(&self) -> Option { + for _ in 0..64 { + let word = self.next.fetch_add(1, Ordering::Relaxed); + let index = (word & 63) as usize; + if index != STATE_COUNT + && self.live[index] + .compare_exchange(FREE, word, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + return Some(word); + } + } + None + } + pub(super) fn release(&self, word: u32) -> bool { + let index = (word & 63) as usize; + index != STATE_COUNT + && self.live[index] + .compare_exchange(word, FREE, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } +} + +/// Two stable logical ordinals, independent of physical pair numbering. +pub(super) struct RenderSlots { + pairs: [Option; 2], + origins: [Option; 2], +} +impl RenderSlots { + pub(super) const fn new() -> Self { + Self { + pairs: [None; 2], + origins: [None; 2], + } + } + pub(super) fn get(&mut self, pair: u8) -> Option { + if let Some(index) = self.pairs.iter().position(|value| *value == Some(pair)) { + return Some(index as u32); + } + let index = self.pairs.iter().position(Option::is_none)?; + self.pairs[index] = Some(pair); + Some(index as u32) + } + pub(super) fn payload(&mut self, pair: u8, ordinal: u64) -> Option { + let index = self.get(pair)? as usize; + let origin = *self.origins[index].get_or_insert(ordinal); + Some(ordinal.checked_sub(origin)?.wrapping_add(1) as u8) + } +} diff --git a/drivers/gpu/drm/asahi/g17/dependency.rs b/drivers/gpu/drm/asahi/g17/dependency.rs new file mode 100644 index 0000000000000..5abd494eeb1f5 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/dependency.rs @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Collapse a logical prefix to its newest still-live firmware kick per QID. + +use super::{ + fw::kick::{KickDependency, KICK_BARRIERS_MAX}, + job::Order, +}; +use kernel::prelude::*; + +/// A failed producer does not establish a fault in its consumer's VM. +pub(crate) fn inherited_error(error: Error) -> Error { + if error == EIO || error == ENODEV || error == ETIMEDOUT { + ECANCELED + } else { + error + } +} + +pub(crate) struct Frontier { + dependencies: [KickDependency; KICK_BARRIERS_MAX], + sequences: [u64; KICK_BARRIERS_MAX], + count: usize, +} + +impl Frontier { + pub(crate) const fn new() -> Self { + Self { + dependencies: [KickDependency::ZERO; KICK_BARRIERS_MAX], + sequences: [0; KICK_BARRIERS_MAX], + count: 0, + } + } + + pub(crate) fn as_slice(&self) -> &[KickDependency] { + &self.dependencies[..self.count] + } + + /// Each QID's implicit parent supplies transitive ordering. Choose its newest + /// logical sequence rather than comparing wrapped firmware timestamps. + pub(crate) fn insert(&mut self, sequence: u64, dependency: KickDependency) -> Result { + if let Some(index) = self + .as_slice() + .iter() + .position(|prior| prior.qid() == dependency.qid()) + { + if sequence > self.sequences[index] { + self.dependencies[index] = dependency; + self.sequences[index] = sequence; + } + return Ok(()); + } + if self.count == KICK_BARRIERS_MAX { + return Err(EIO); + } + self.dependencies[self.count] = dependency; + self.sequences[self.count] = sequence; + self.count += 1; + Ok(()) + } + + /// The caller has matched the logical owner and VM. Check every covered + /// predecessor for failure, including one a newer kick would subsume. + pub(crate) fn include( + &mut self, + prefix: Option, + sequence: u64, + status: i32, + word: u64, + ) -> Result { + if !Order::contains(prefix, sequence) { + return Ok(()); + } + if status < 0 { + return Err(ECANCELED); + } + if status > 0 { + return Ok(()); + } + let dependency = KickDependency::from_word(word).ok_or(EIO)?; + self.insert(sequence, dependency) + } +} diff --git a/drivers/gpu/drm/asahi/g17/event.rs b/drivers/gpu/drm/asahi/g17/event.rs new file mode 100644 index 0000000000000..9e4e0253b7c81 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/event.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Event service ordering under the device mutex. A restart ends its role's batch; a worker +//! joins preparation leases and services it on a separate turn before continuing the rings. + +use super::{ + fw::channels::{Event, FreeListCompletion, TvbGrowRequest}, + recovery, Role, +}; +use kernel::prelude::*; + +/// The bounded shared-ring transport; recognized malformed entries return errors after consume. +pub(super) trait Transport { + fn trace(&mut self, role: Role); + fn statistics(&mut self, role: Role); + fn drain( + &mut self, + role: Role, + consume: impl FnMut(&mut Self, Event) -> Result, + ) -> Result; + fn recovery_state(&self) -> Result; +} + +/// Runtime callbacks with the device mutex held. None signals fences or allocates grow backing. +pub(super) trait Queues { + /// Flush the oldest deferred reply/release. False means ring backpressure; each pending + /// transaction retains its own bounded worker-attempt count across service turns. + fn flush_tvb_reply(&mut self) -> Result; + fn flush_freelist_release(&mut self) -> Result; + fn observe_render_releases(&mut self) -> Result; + fn control_backpressured(&self) -> bool; + fn queue_tvb_grow(&mut self, request: TvbGrowRequest) -> Result; + fn complete_freelist_grow(&mut self, completion: FreeListCompletion) -> Result; + fn retain_render_completions(&mut self, masks: [u64; 2]); + fn complete_compute(&mut self, masks: [u64; 2]) -> Result; + fn context_killed(&mut self, cookie: u64); +} + +/// Work requested after releasing the device mutex. +#[derive(Default)] +pub(super) struct Pending { + pub(super) recovery: bool, +} + +/// Access to the single recovery owner while event handlers borrow the device. +pub(super) trait Host: Transport + Queues { + fn recovery(&self) -> &recovery::State; + fn recovery_mut(&mut self) -> &mut recovery::State; +} + +fn batch(host: &mut H, role: Role, masks: &mut [u64; 2]) -> Result { + host.drain(role, |host, event| { + match event { + Event::Stamp(bits) => { + masks[0] |= bits[0]; + masks[1] |= bits[1]; + } + Event::Restart { generation, stamp } => { + if host.recovery_mut().request(generation, stamp)? { + return Ok(false); + } + } + Event::TvbGrow(request) => { + if request.halt_count == host.recovery().generation() { + host.queue_tvb_grow(request)?; + } + if host.control_backpressured() { + return Ok(false); + } + } + Event::FreeListGrow(completion) => host.complete_freelist_grow(completion)?, + Event::ContextKilled(cookie) => host.context_killed(cookie), + Event::Unknown(_) => {} + } + Ok(true) + }) +} + +pub(super) fn service(host: &mut H) -> Result { + if !host.flush_tvb_reply()? || !host.flush_freelist_release()? { + return Ok(Pending::default()); + } + host.observe_render_releases()?; + for role in [Role::Primary, Role::Secondary] { + host.trace(role); + } + for role in [Role::Primary, Role::Secondary] { + host.statistics(role); + } + let mut masks = [0; 2]; + if !host.recovery().pending() { + let mut settled = false; + // The firmware publishes restart before state 1, but its producer can race the snapshot. + for _ in 0..256 { + batch(host, Role::Primary, &mut masks)?; + if host.recovery().pending() || host.control_backpressured() { + settled = true; + break; + } + match host.recovery_state()? { + 0 => { + settled = true; + break; + } + 1 => {} + _ => return Err(EIO), + } + } + if !settled { + return Err(EIO); + } + } + if !host.control_backpressured() { + batch(host, Role::Secondary, &mut masks)?; + } + host.retain_render_completions(masks); + host.complete_compute(masks)?; + Ok(Pending { + recovery: host.recovery().pending(), + }) +} diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs new file mode 100644 index 0000000000000..95a13dbdd54ff --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware completion fences and the aggregate fence of one submit ioctl. +//! +//! These fences can complete out of order, so each has an independent timeline. +//! Signalling belongs outside the device lock, after timestamp and VM-status writes. + +use super::context::{Context, WorkStateLease}; +use crate::file::SyncItem; +use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering}; +use kernel::{ + bindings, c_str, + dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UserFence}, + prelude::*, + str::CStr, + sync::Arc, +}; + +pub(crate) struct CompletionFence; + +#[vtable] +impl FenceOps for CompletionFence { + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("asahi") + } + + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("g17-submit") + } +} + +/// Give independently retired work a timeline that fence merging cannot subsume. +pub(crate) fn independent(contexts: &FenceContexts) -> Result> { + let fence = contexts.new_fence(0, CompletionFence)?; + // SAFETY: The fence is unique and unpublished: there are no waiters, callbacks or + // shared references. The DMA-fence allocator supplies a globally unique context. + unsafe { (*fence.raw()).context = bindings::dma_fence_context_alloc(1) }; + Ok(fence.into()) +} + +/// All command members of an ioctl share its work-state node and aggregate fence. +/// The initial member keeps the aggregate unsignalled until enqueue finishes. +pub(crate) struct Submission { + fence: UserFence, + work_state: Arc, + remaining: AtomicU32, + error: AtomicI32, +} + +impl Submission { + pub(crate) fn new(contexts: &FenceContexts, context: Arc) -> Result> { + Ok(Arc::new( + Self { + fence: independent(contexts)?, + work_state: Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?, + remaining: AtomicU32::new(1), + error: AtomicI32::new(0), + }, + GFP_KERNEL, + )?) + } + + pub(crate) fn fence(&self) -> Fence { + Fence::from_fence(&self.fence) + } + + /// Add one command before enqueue ends. The parser bounds the count to 64. + pub(crate) fn member(self: &Arc) -> Member { + self.remaining.fetch_add(1, Ordering::Relaxed); + Member { + submission: self.clone(), + complete: AtomicBool::new(false), + } + } + + fn member_done(&self, result: Result) { + if let Err(error) = result { + let _ = self.error.compare_exchange( + 0, + error.to_errno(), + Ordering::AcqRel, + Ordering::Acquire, + ); + } + if self.remaining.fetch_sub(1, Ordering::AcqRel) == 1 { + let error = self.error.load(Ordering::Acquire); + if error != 0 { + self.fence.set_error(Error::from_errno(error)); + } + self.fence.signal(); + } + } + + fn finish_enqueue(&self, result: Result) { + // A dropped, never-enqueued member reports cancellation. The ioctl's actual + // failure takes precedence, while published members still retain their work. + if let Err(error) = result { + self.error.store(error.to_errno(), Ordering::Release); + } + self.member_done(result); + } +} + +/// One command's contribution to its submission, including construction failure. +pub(crate) struct Member { + submission: Arc, + complete: AtomicBool, +} + +impl Member { + pub(crate) fn work_state(&self) -> Arc { + self.submission.work_state.clone() + } + + pub(crate) fn complete(&self, result: Result) { + if !self.complete.swap(true, Ordering::AcqRel) { + self.submission.member_done(result); + } + } +} + +impl Drop for Member { + fn drop(&mut self) { + self.complete(Err(ECANCELED)); + } +} + +fn install(outputs: impl Iterator, fence: &Fence) { + for mut sync in outputs { + if let Some(chain) = sync.chain_fence.take() { + sync.syncobj.add_point(chain, fence, sync.timeline_value); + } else { + sync.syncobj.replace_fence(Some(fence)); + } + } +} + +/// Keeps resolved output syncobjs reachable on every fallible submission path. +/// Input syncobjs stay in the same vector, avoiding a separate output allocation. +pub(crate) struct Outputs { + fallback: UserFence, + syncs: KVec, + input_count: usize, + submission: Option>, + result: Result, +} + +impl Outputs { + pub(crate) fn new( + contexts: &FenceContexts, + syncs: KVec, + input_count: usize, + ) -> Result { + if input_count > syncs.len() { + return Err(EINVAL); + } + Ok(Self { + fallback: independent(contexts)?, + syncs, + input_count, + submission: None, + result: Err(ECANCELED), + }) + } + + pub(crate) fn inputs(&self) -> &[SyncItem] { + &self.syncs[..self.input_count] + } + + /// Publish the actual aggregate before enqueueing any command. + pub(crate) fn publish(&mut self, submission: Arc) { + install(self.syncs.drain(self.input_count..), &submission.fence()); + self.submission = Some(submission); + } + + /// The caller first records terminal VM errors and settles prior failed work. + pub(crate) fn finish(mut self, result: Result) -> Result { + self.result = result; + result + } +} + +impl Drop for Outputs { + fn drop(&mut self) { + if let Some(submission) = self.submission.take() { + submission.finish_enqueue(self.result); + } + if let Err(error) = self.result { + self.fallback.set_error(error); + } + install( + self.syncs.drain(self.input_count..), + &Fence::from_fence(&self.fallback), + ); + self.fallback.signal(); + } +} diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs new file mode 100644 index 0000000000000..e1d42944a38fe --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -0,0 +1,575 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Retained USC private-memory pools and their submission counters. +//! +//! A logical context owns one render pool, shared by its render pairs. A +//! physical compute queue owns a separate pool. All mutation and publication +//! runs under the device mutex. Pool aliases must outlive every firmware +//! reference and are released only after the owner's teardown is acknowledged. + +use super::fw::freelist::*; +use super::fw::initdata::{FreeListArgs, FreeListClass, FreeListControl, FREE_LIST_BLOCK_SIZE}; +use super::object::{Allocator, CpuMap, KernelObject}; +use crate::mmu; +use core::sync::atomic::{fence, AtomicBool, AtomicU64, Ordering}; +use kernel::{prelude::*, sync::Arc}; + +/// Render pools occupy 129..255; compute uses its QID plus one, in 1..128. +/// First-free allocation keeps the shared firmware namespace deterministic. +pub(crate) struct RenderIds { + used: [AtomicU64; 2], +} +impl RenderIds { + pub(crate) const fn new() -> Self { + Self { + used: [AtomicU64::new(0), AtomicU64::new(1 << 63)], + } + } +} +struct RenderId { + slot: u16, + allocator: Arc, +} +impl RenderId { + fn new(allocator: &Arc) -> Result { + for (bank, used) in allocator.used.iter().enumerate() { + let mut value = used.load(Ordering::Acquire); + loop { + let bit = value.trailing_ones(); + if bit == 64 { + break; + } + match used.compare_exchange_weak( + value, + value | (1u64 << bit), + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => { + return Ok(Self { + slot: 129 + bank as u16 * 64 + bit as u16, + allocator: allocator.clone(), + }) + } + Err(current) => value = current, + } + } + } + Err(ENOSPC) + } +} +impl Drop for RenderId { + fn drop(&mut self) { + let bit = self.slot as usize - 129; + self.allocator.used[bit / 64].fetch_and(!(1u64 << (bit % 64)), Ordering::Release); + } +} + +struct Pages { + backing: KernelObject, + blocks: usize, + layout: RunLayout, +} +impl Pages { + fn new(alloc: &Allocator<'_>, blocks: usize, layout: RunLayout) -> Result { + let mut backing = alloc.lower( + TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + )?; + let base = backing.gpu_va(); + backing.initialize::(0, |tables| tables.init(base, blocks, layout))?; + Ok(Self { + backing, + blocks, + layout, + }) + } + fn page_list_va(&self) -> u64 { + self.backing.gpu_va() + } + fn run_list_va(&self) -> u64 { + self.page_list_va() + PAGE_LIST_SIZE as u64 + } + fn repopulate(&self) -> Result { + let table = self.backing.pointer(0, TABLE_SIZE)?.cast::(); + PoolTables::for_each_word( + self.page_list_va(), + self.blocks, + self.layout, + |index, value| { + // SAFETY: The page-aligned backing was validated for the complete + // tables. The enumerator bounds every index inside those tables; + // the owner has proven firmware release before this update. + unsafe { table.add(index).write_volatile(value) }; + }, + ) + } +} + +/// Pool-object identities are independent of recycled firmware IDs and VAs. +static RENDER_POOL_SERIAL: AtomicU64 = AtomicU64::new(0); + +/// A render pool retained by a logical context and every graph referencing it. +pub(crate) struct RenderPool { + // Remove all aliases before dropping the backing or returning the ID. + _client: mmu::KernelMapping, + _global: mmu::KernelMapping, + pages: Pages, + control: KernelObject, + state: KernelObject, + served: AtomicBool, + serial: u64, + id: RenderId, +} + +impl RenderPool { + pub(crate) fn new( + alloc: &Allocator<'_>, + ids: &Arc, + render_global: &mmu::Vm, + client: &mmu::Vm, + ) -> Result { + let id = RenderId::new(ids)?; + let pages = Pages::new(alloc, RENDER_BLOCKS, RunLayout::Render)?; + let address = pages.page_list_va(); + let global = pages + .backing + .map_alias(render_global, address, mmu::PROT_GPU_SHARED_RW)?; + let client = pages + .backing + .map_alias(client, address, mmu::PROT_GPU_SHARED_RW)?; + let state = alloc.kernel( + CONTROL_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let mut control = alloc.kernel( + CONTROL_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + control.write( + 0, + FreeListControl::render( + &FreeListArgs { + buffer_slot: u32::from(id.slot), + class: FreeListClass::Two, + page_list_va: address, + run_list_va: pages.run_list_va(), + blocks: RENDER_BLOCKS as u32, + state_va: state.gpu_va(), + }, + 0, + ), + )?; + fence(Ordering::SeqCst); + let serial = RENDER_POOL_SERIAL + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| value.checked_add(1)) + .map_err(|_| EOVERFLOW)? + 1; + Ok(Self { + _client: client, + _global: global, + pages, + control, + state, + served: AtomicBool::new(false), + serial, + id, + }) + } + + pub(crate) fn serial(&self) -> u64 { + self.serial + } + pub(crate) fn id(&self) -> u16 { + self.id.slot + } + pub(crate) fn control_va(&self) -> u64 { + self.control.gpu_va() + } + pub(crate) fn page_list_va(&self) -> u64 { + self.pages.page_list_va() + } + pub(crate) fn run_list_va(&self) -> u64 { + self.pages.run_list_va() + } + pub(crate) fn end_va(&self) -> u64 { + self.run_list_va() + + (RENDER_BLOCKS * size_of::()) as u64 + } + + /// Fresh pools are already populated. Reuse requires the previous release + /// to have been consumed; command completion alone does not establish it. + pub(crate) fn repopulate(&self) -> Result { + if !self.served.swap(true, Ordering::AcqRel) { + return Ok(()); + } + let submitted = self.state.dword(0)?.load(Ordering::Relaxed); + let header = FreeListControl::render( + &FreeListArgs { + buffer_slot: u32::from(self.id()), + class: FreeListClass::Two, + page_list_va: self.page_list_va(), + run_list_va: self.run_list_va(), + blocks: RENDER_BLOCKS as u32, + state_va: self.state.gpu_va(), + }, + submitted, + ); + let mut image = KVVec::from_elem(0u64, CONTROL_SIZE / 8, GFP_KERNEL)?; + // SAFETY: `image` owns a zeroed control page, is large enough for the + // header, and every header byte is initialized integer data. + unsafe { + image + .as_mut_ptr() + .cast::() + .write_unaligned(header) + }; + let control = self.control.pointer(0, CONTROL_SIZE)?.cast::(); + self.pages.repopulate()?; + for (index, value) in image.iter().enumerate() { + // SAFETY: The entire retained page was checked before any store; + // the allocation is page-aligned and release excludes the firmware. + unsafe { control.add(index).write_volatile(*value) }; + } + fence(Ordering::SeqCst); + Ok(()) + } + + /// Submitted is host-owned and aligned. Completed is the packed firmware + /// counter, so retain the byte accesses required by that interface. + pub(crate) fn counters(&self) -> Result<[u64; 2]> { + let submitted = self.state.dword(0)?.load(Ordering::Relaxed); + let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; + let mut bytes = [0; 8]; + for (index, byte) in bytes.iter_mut().enumerate() { + // SAFETY: The checked eight-byte range stays mapped; byte accesses + // permit the unaligned counter without widening shared accesses. + *byte = unsafe { completed.add(index).read_volatile() }; + } + Ok([submitted, u64::from_le_bytes(bytes)]) + } + + /// Snapshot the unaligned shared fields before loading their descriptor. + pub(crate) fn page_pool_state(&self) -> Result { + let raw = self.control.pointer(0x14, 0x1c)?; + fence(Ordering::Acquire); + let read = |offset: usize, count: usize| -> u64 { + let mut bytes = [0; 8]; + for (index, byte) in bytes.iter_mut().take(count).enumerate() { + // SAFETY: The complete 0x14..0x30 span was checked. These fixed + // field ranges lie inside it; byte loads permit unaligned words. + *byte = unsafe { raw.add(offset + index).read_volatile() }; + } + u64::from_le_bytes(bytes) + }; + Ok(PagePoolState { + page_list_va: read(0, 8), + unk_1c: read(8, 4) as u32, + unk_20: read(12, 4) as u32, + unk_24: read(16, 4) as u32, + unk_28: read(20, 4) as u32, + page_count: read(24, 4) as u32, + }) + } + + /// Save mutable descriptor state only after both engines and free-list + /// references retire, before the physical slot changes owners. + pub(crate) fn store_page_pool_state(&self, state: PagePoolState) -> Result { + let raw = self.control.pointer(0x20, 0x10)?; + for (word, value) in [state.unk_20, state.unk_24, state.unk_28, state.page_count] + .into_iter() + .enumerate() + { + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + // SAFETY: The entire mutable range was checked, and each + // enumerated byte lies inside it. Caller owns the retired pool. + unsafe { raw.add(word * 4 + index).write_volatile(byte) }; + } + } + fence(Ordering::SeqCst); + Ok(()) + } + + /// Prepares the counter before any producer is written. Commit belongs to + /// the publication transaction; dropping this plan does not reserve work. + pub(crate) fn prepare_submission(&self, commands: u32) -> Result> { + if commands > 2 { + return Err(EINVAL); + } + let counter = self.state.dword(0)?; + Ok(Submitted { + counter, + next: counter + .load(Ordering::Relaxed) + .wrapping_add(u64::from(commands)), + }) + } +} + +/// One checked, infallible counter update in a submission transaction. +pub(crate) struct Submitted<'a> { + counter: &'a AtomicU64, + next: u64, +} +impl Submitted<'_> { + pub(crate) fn commit(self) { + self.counter.store(self.next, Ordering::Relaxed); + fence(Ordering::SeqCst); + } +} + +/// A pool owned by one physical compute queue, with fixed QID-derived slot. +pub(crate) struct ComputePool { + pages: Pages, + control: KernelObject, + state: KernelObject, + id: u16, +} +impl ComputePool { + pub(crate) fn new(alloc: &Allocator<'_>, qid: u8) -> Result { + if qid > super::fw::kick::QID_MAX { + return Err(EINVAL); + } + let id = u16::from(qid) + 1; + let pages = Pages::new(alloc, COMPUTE_BLOCKS, RunLayout::Compute)?; + let mut control = alloc.kernel( + CONTROL_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let state = alloc.kernel( + CONTROL_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + control.write( + 0, + FreeListControl::compute(&FreeListArgs { + buffer_slot: u32::from(id), + class: FreeListClass::One, + page_list_va: pages.page_list_va(), + run_list_va: pages.run_list_va(), + blocks: COMPUTE_BLOCKS as u32, + state_va: state.gpu_va(), + }), + )?; + fence(Ordering::SeqCst); + Ok(Self { + pages, + control, + state, + id, + }) + } + pub(crate) fn id(&self) -> u16 { + self.id + } + pub(crate) fn page_list_va(&self) -> u64 { + self.pages.page_list_va() + } + pub(crate) fn control_va(&self) -> u64 { + self.control.gpu_va() + } + pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { + let address = self.pages.page_list_va(); + self.pages + .backing + .map_alias(vm, address, mmu::PROT_GPU_SHARED_RW) + } + pub(crate) fn reserve_submission(&self) -> Result { + let counter = self.state.dword(0)?; + let next = counter + .load(Ordering::Relaxed) + .checked_add(1) + .ok_or(EOVERFLOW)?; + counter.store(next, Ordering::Relaxed); + fence(Ordering::SeqCst); + Ok(next) + } + /// The caller has proved no pointer from this submission was published. + pub(crate) fn cancel_unpublished(&self, target: u64) -> Result { + let counter = self.state.dword(0)?; + if target == 0 || counter.load(Ordering::Acquire) != target { + return Err(EINVAL); + } + counter.store(target - 1, Ordering::Relaxed); + fence(Ordering::SeqCst); + Ok(()) + } +} + +/// Registration identity held by a free-list lease. The cookie is the end of +/// the run list; it remains stable while overlapping command pairs share it. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Owner { + /// Identifies the host pool object even when every firmware address recycles. + pub(crate) serial: u64, + pub(crate) buffer_slot: u32, + pub(crate) control_va: u64, + pub(crate) ring_va: u64, + pub(crate) cookie: u64, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +enum LeasePhase { + Preparing, + Ready, + ReleaseSent(u32), + Released, +} + +/// References to one prepopulated owner: preparation and two engine commands +/// per render. Published releases do not gate reacquisition; their consumption +/// witnesses are carried into the successor. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Lease { + owner: Owner, + generation: u64, + references: u32, + commands: u32, + phase: LeasePhase, + previous_release: Option, +} + +impl Lease { + /// Acquires an unchanged overlapping owner, or an owner whose previous + /// release was sent. A changed owner additionally requires caller-side + /// proof of old-pool release before its backing can be repopulated. + pub(crate) fn acquire(previous: Option<&Self>, owner: Owner) -> Result { + let Some(previous) = previous else { + return Ok(Self { + owner, + generation: 0, + references: 3, + commands: 2, + phase: LeasePhase::Preparing, + previous_release: None, + }); + }; + if previous.owner.buffer_slot != owner.buffer_slot { + return Err(EINVAL); + } + let generation = previous.generation.checked_add(1).ok_or(EOVERFLOW)?; + let same_owner = previous.owner == owner; + let overlapping = same_owner + && previous.phase == LeasePhase::Ready + && previous.references >= 2 + && previous.commands >= 2; + let reacquire = previous.reacquirable(); + if !overlapping && !reacquire { + return Err(EBUSY); + } + let references = previous.references.checked_add(3).ok_or(EOVERFLOW)?; + let commands = previous.commands.checked_add(2).ok_or(EOVERFLOW)?; + Ok(Self { + owner, + generation: if overlapping { + previous.generation + } else { + generation + }, + references, + commands, + phase: LeasePhase::Preparing, + previous_release: previous.release_witness(), + }) + } + + pub(crate) fn owner(&self) -> Owner { + self.owner + } + pub(crate) fn generation(&self) -> u64 { + self.generation + } + pub(crate) fn reacquirable(&self) -> bool { + matches!( + self.phase, + LeasePhase::ReleaseSent(_) | LeasePhase::Released + ) && self.references == 0 + && self.commands == 0 + } + pub(crate) fn released(&self) -> bool { + self.phase == LeasePhase::Released && self.previous_release.is_none() + } + pub(crate) fn release_witness(&self) -> Option { + match self.phase { + LeasePhase::ReleaseSent(cursor) => Some(cursor), + _ => self.previous_release, + } + } + + /// Publication retires preparation independently of the command references. + pub(crate) fn retire_prepare(&mut self) -> Result { + if self.phase != LeasePhase::Preparing || self.references < 3 { + return Err(EINVAL); + } + self.references -= 1; + self.phase = LeasePhase::Ready; + Ok(()) + } + /// Fixed prepopulated pools never submit an asynchronous growth request. + /// An incoming growth completion therefore cannot name a live request. + pub(crate) fn complete_grow( + &mut self, + _event: super::fw::channels::FreeListCompletion, + ) -> Result { + Err(EIO) + } + /// Both engines have retired this render's exact retained submission. + pub(crate) fn complete_render( + &mut self, + ) -> Result> { + if self.phase != LeasePhase::Ready || self.references < 2 || self.commands < 2 { + return Err(EINVAL); + } + self.references -= 2; + self.commands -= 2; + Ok(self.pending_release()) + } + pub(crate) fn pending_release(&self) -> Option { + (self.phase == LeasePhase::Ready && self.references == 0).then(|| { + super::fw::channels::FreeListRelease::new(self.generation, self.owner.buffer_slot) + }) + } + /// Records the producer-after cursor once the final release was published. + pub(crate) fn mark_release_sent(&mut self, cursor: u32) -> Result { + if self.pending_release().is_none() || cursor >= super::fw::channels::DEVICE_CONTROL_SLOTS { + return Err(EINVAL); + } + self.phase = LeasePhase::ReleaseSent(cursor); + Ok(()) + } + /// Observes all older release records passed by one consumer snapshot. + pub(crate) fn observe_releases(&mut self, consumer: u32, producer: u32) -> Result { + if let Some(cursor) = self.previous_release { + if control_consumed(consumer, producer, cursor)? { + self.previous_release = None; + } + } + if let LeasePhase::ReleaseSent(cursor) = self.phase { + if control_consumed(consumer, producer, cursor)? { + self.phase = LeasePhase::Released; + } + } + Ok(()) + } +} + +/// A publication is retired once its producer-after cursor leaves the +/// current `(consumer, producer]` interval, including wrapped intervals. +pub(crate) fn control_consumed(consumer: u32, producer: u32, cursor: u32) -> Result { + let slots = super::fw::channels::DEVICE_CONTROL_SLOTS; + if consumer >= slots || producer >= slots || cursor >= slots { + return Err(EIO); + } + let pending = producer.wrapping_sub(consumer) & (slots - 1); + let ahead = cursor.wrapping_sub(consumer) & (slots - 1); + Ok(ahead == 0 || ahead > pending) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/buffer.rs b/drivers/gpu/drm/asahi/g17/fw/buffer.rs new file mode 100644 index 0000000000000..3ed336dbadaad --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/buffer.rs @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Parameter-buffer inventory, scene and control records. These objects are +//! initialized before publication; live access uses the host buffer manager's +//! individual aligned words and packed byte fields. + +use super::{clear, offset_va}; +use crate::hw::t8140::resources as cfg; +use kernel::prelude::*; + +pub(crate) const SCENES: usize = 0x50; +pub(crate) const SCRATCH_SLOTS: usize = 0x24; +pub(crate) const SCRATCH_STRIDE: u64 = 0x20; +pub(crate) const FIRST_SCENE: usize = 1; +pub(crate) const PAGE_LIST: usize = 0x8000; +pub(crate) const PAGE_LIST_BYTES: usize = 0x10000; +pub(crate) const CONTROL: usize = 0x20000; +pub(crate) const COUNTER: usize = 0x28000; +pub(crate) const SCENE_TABLE: usize = 0x38000; +pub(crate) const BLOCK_TABLE: usize = 0x40000; +pub(crate) const SIZE: usize = 0x48000; +pub(crate) const METRICS_STRIDE: usize = SCENES * size_of::(); +pub(crate) const METRICS_SLOTS: usize = cfg::METRICS_SIZE as usize / METRICS_STRIDE; +pub(crate) const NORMAL_MAX_BLOCKS: usize = 0x45e; +pub(crate) const GROW_BLOCKS: usize = 10; + +/// Firmware's active parameter-buffer state. Pointers at +0x44, +0x4c, +/// +0x64 and +0x84 are intentionally unaligned. +#[repr(C, packed)] +pub(crate) struct State { + pub(crate) unk_0: [u8; 0xc], + pub(crate) buffer_id: u32, + pub(crate) unk_10: [u8; 0x10], + pub(crate) page_list_fw: u64, + pub(crate) page_list_client: u64, + pub(crate) page_list_size: u32, + pub(crate) pages: u32, + pub(crate) block_capacity: u32, + pub(crate) producer: u32, + pub(crate) consumer: u32, + pub(crate) block_table: u64, + pub(crate) control: u64, + pub(crate) last_page: u32, + pub(crate) block_size: u32, + pub(crate) unk_5c: [u8; 8], + pub(crate) counter: u64, + pub(crate) unk_6c: [u8; 0x10], + pub(crate) maximum_size: u32, + pub(crate) normal_maximum_size: u32, + pub(crate) discard: u64, + pub(crate) unk_8c: [u8; 0x34], +} + +/// One scene's accounting and scratch identity. +#[repr(C, packed)] +pub(crate) struct Scene { + pub(crate) metric_client: u64, + pub(crate) metric_fw: u64, + pub(crate) unk_10: [u8; 0x18], + pub(crate) scratch: u64, + pub(crate) unk_30: [u8; 0x10], + pub(crate) statistics: u64, + pub(crate) used: u32, + pub(crate) generation: u64, + pub(crate) unk_54: [u8; 0x2c], +} + +/// Circular inventory cursors followed by firmware statistics. Firmware owns +/// `read` and the statistics; the host requests reset by release-storing1. +#[repr(C)] +pub(crate) struct Control { + pub(crate) total: u32, + pub(crate) committed: u32, + pub(crate) read: u32, + pub(crate) unk_c: [u32; 13], + pub(crate) max_pages: u32, + pub(crate) unk_44: [u32; 7], + pub(crate) reset: u32, + pub(crate) unk_64: [u32; 7], +} + +#[repr(C)] +pub(crate) struct Block { + pub(crate) first_page: u32, + pub(crate) unk_4: u32, +} + +/// The complete parameter-buffer graph, including reserved spaces between +/// records. The page-list subrange also has a client mapping. +#[repr(C)] +pub(crate) struct Graph { + pub(crate) state: State, + pub(crate) pad_c0: [u8; PAGE_LIST - size_of::()], + pub(crate) pages: [u32; cfg::TVB_BLOCK_SLOTS * cfg::TVB_PAGES_PER_BLOCK], + pub(crate) pad_151a0: [u8; CONTROL - PAGE_LIST - cfg::TVB_PAGE_LIST_SIZE], + pub(crate) control: Control, + pub(crate) pad_20080: [u8; COUNTER - CONTROL - size_of::()], + pub(crate) counter: u32, + pub(crate) pad_28004: [u8; SCENE_TABLE - COUNTER - 4], + pub(crate) scenes: [Scene; SCENES], + pub(crate) trailer: [u8; 0x40], + pub(crate) pad_3a840: [u8; BLOCK_TABLE - SCENE_TABLE - SCENES * size_of::() - 0x40], + pub(crate) blocks: [Block; cfg::TVB_BLOCK_SLOTS], + pub(crate) pad_468d0: [u8; SIZE - BLOCK_TABLE - cfg::TVB_BLOCK_SLOTS * size_of::()], +} + +// SAFETY: Every field of the complete graph consists solely of integers. +unsafe impl Zeroable for Graph {} + +static_assert!(size_of::() == 0xc0); +static_assert!(size_of::() == 0x80); +static_assert!(size_of::() == 0x80); +static_assert!(size_of::() == SIZE); +static_assert!(core::mem::offset_of!(Graph, pages) == PAGE_LIST); +static_assert!(core::mem::offset_of!(Graph, control) == CONTROL); +static_assert!(core::mem::offset_of!(Graph, counter) == COUNTER); +static_assert!(core::mem::offset_of!(Graph, scenes) == SCENE_TABLE); +static_assert!(core::mem::offset_of!(Graph, blocks) == BLOCK_TABLE); +static_assert!(core::mem::offset_of!(State, block_table) == 0x44); +static_assert!(core::mem::offset_of!(State, discard) == 0x84); +static_assert!(core::mem::offset_of!(Scene, generation) == 0x4c); + +pub(crate) struct GraphArgs<'a> { + pub(crate) fw_va: u64, + pub(crate) page_list_client: u64, + /// Firmware and client views, respectively, of this graph's metrics lease. + pub(crate) metrics: [u64; 2], + pub(crate) blocks: &'a [u64], + pub(crate) scratch: u64, + pub(crate) discard: u64, + pub(crate) buffer_id: u32, + pub(crate) generation: u64, +} + +pub(crate) fn compact_page(address: u64) -> Result { + let relative = address.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; + if relative % cfg::TVB_PAGE != 0 + || address + .checked_add(cfg::TVB_BLOCK_SIZE as u64) + .ok_or(EOVERFLOW)? + > cfg::USER_END + { + return Err(EINVAL); + } + Ok((relative / cfg::TVB_PAGE).try_into()?) +} + +impl Graph { + pub(crate) fn init(&mut self, args: &GraphArgs<'_>) -> Result { + if args.generation == 0 + || args.blocks.len() < cfg::TVB_INITIAL_PAGES.len() + || args.blocks.len() > cfg::TVB_MAX_BLOCKS + { + return Err(EINVAL); + } + let scratch = args.scratch.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; + let discard = args.discard.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; + if scratch > u32::MAX as u64 || discard > u32::MAX as u64 { + return Err(EINVAL); + } + let stats = offset_va(args.fw_va, (CONTROL + 0x40) as u64)?; + clear(self); + for (index, &address) in args.blocks.iter().enumerate() { + let first = compact_page(address)?; + self.blocks[index].first_page = first; + for page in 0..cfg::TVB_PAGES_PER_BLOCK { + self.pages[index * cfg::TVB_PAGES_PER_BLOCK + page] = first + page as u32; + } + } + for (index, scene) in self.scenes.iter_mut().enumerate() { + scene.metric_client = offset_va(args.metrics[1], (index * 4) as u64)?; + scene.metric_fw = offset_va(args.metrics[0], (index * 4) as u64)?; + scene.scratch = scratch + (index % SCRATCH_SLOTS) as u64 * SCRATCH_STRIDE; + scene.statistics = stats; + } + self.scenes[FIRST_SCENE].generation = args.generation; + self.control.total = args.blocks.len() as u32; + self.control.committed = self.control.total; + self.control.reset = 1; + let pages = (args.blocks.len() * cfg::TVB_PAGES_PER_BLOCK) as u32; + self.state.buffer_id = args.buffer_id; + self.state.page_list_fw = offset_va(args.fw_va, PAGE_LIST as u64)?; + self.state.page_list_client = args.page_list_client; + self.state.page_list_size = PAGE_LIST_BYTES as u32; + self.state.pages = pages; + self.state.block_capacity = cfg::TVB_BLOCK_SLOTS as u32; + self.state.producer = args.blocks.len() as u32; + self.state.block_table = offset_va(args.fw_va, BLOCK_TABLE as u64)?; + self.state.control = offset_va(args.fw_va, CONTROL as u64)?; + self.state.last_page = pages - 1; + self.state.block_size = cfg::TVB_BLOCK_SIZE as u32; + self.state.counter = offset_va(args.fw_va, COUNTER as u64)?; + self.state.maximum_size = (cfg::TVB_BLOCK_SLOTS * 4) as u32; + self.state.normal_maximum_size = (NORMAL_MAX_BLOCKS * 4) as u32; + self.state.discard = discard; + Ok(()) + } +} + +/// Rebuilds an idle parameter-buffer descriptor, preserving firmware-owned +/// bits and resetting both ring counters and flags for its rebuilt page list. +pub(crate) fn reload_descriptor(previous: [u32; 4], address: u64, pages: u32) -> [u32; 4] { + const COUNT: u32 = 0x003f_ffff; + let bias = if address & (1u64 << 42) == 0 { + 0x70_0000_0000 + } else { + 0 + }; + let segment = (address.wrapping_add(bias) & 0x70_0000_0000) | ((address >> 3) & 0x80_0000_0000); + [ + ((address >> 4) as u32 & !7) | (previous[0] & 6), + ((segment >> 8) as u32 & 0xf000_0000) | (previous[1] & 0x0fc0_0000) | (pages & COUNT), + previous[2] & !COUNT, + previous[3] & !(COUNT | 0x8000_0000), + ] +} + +/// Compact scratch and tagged metrics addresses carried by scene registers. +pub(crate) fn scene_registers(index: usize, scratch: u64, metrics: u64) -> Result<(u64, u64)> { + const SEGMENT_SELECT: u64 = 1 << 42; + const SEGMENT_BIAS: u64 = 0x70_0000_0000; + const ADDRESS_MASK: u64 = 0x7f_ffff_fffe; + const HIGH_BIT: u64 = 0x80_0000_0000; + if index >= SCENES || metrics & 3 != 0 { + return Err(EINVAL); + } + let scratch = scratch + .checked_sub(cfg::USER_BASE) + .ok_or(EINVAL)? + .checked_add((index % SCRATCH_SLOTS) as u64 * SCRATCH_STRIDE) + .ok_or(EOVERFLOW)?; + if scratch > u32::MAX as u64 || scratch & 0xf != 0 { + return Err(EINVAL); + } + let metric = offset_va(metrics, (index * 4) as u64)?; + let adjusted = offset_va( + metric, + if metric & SEGMENT_SELECT == 0 { + SEGMENT_BIAS + } else { + 0 + }, + )?; + Ok(( + scratch, + (adjusted & ADDRESS_MASK) | ((metric >> 3) & HIGH_BIT) | 1, + )) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/channels.rs b/drivers/gpu/drm/asahi/g17/fw/channels.rs new file mode 100644 index 0000000000000..c2c3b3eb7a80d --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/channels.rs @@ -0,0 +1,522 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Device-control records and work-ring slots. +//! +//! The host sends out-of-band requests and replies through the device-control +//! ring, whose 0x40-byte records start with a [`DeviceControlOp`]. It hands +//! hardware queues to the firmware through work rings, one per priority and +//! engine, each holding [`WORK_RING_SLOTS`] [`WorkSlot`]s. +//! +//! Producers and consumers of both rings are live words in the primary state +//! grid; they are not part of these structures. Before publishing a +//! device-control record whose opcode [`DeviceControlOp::asserts_gpu_power`], +//! the host also increments its power-assert tally. + +use kernel::prelude::*; + +use super::kick::QID_MAX; +use super::queue::{DataMaster, QUEUE_PRIORITY}; + +/// Size of a device-control record. +pub(crate) const DEVICE_CONTROL_RECORD_SIZE: usize = 0x40; +/// Number of records in the device-control ring. +pub(crate) const DEVICE_CONTROL_SLOTS: u32 = 256; + +/// Entries in either coprocessor's event ring. +pub(crate) const EVENT_SLOTS: u32 = 256; +/// Entries in either coprocessor's trace ring. +pub(crate) const TRACE_SLOTS: u32 = 512; +/// Entries in either coprocessor's statistics ring. +pub(crate) const STATISTICS_SLOTS: u32 = 256; +/// Words in an event, trace or statistics entry. +pub(crate) const EVENT_WORDS: usize = 18; +/// Byte stride of an event, trace or statistics entry. +pub(crate) const EVENT_RECORD_SIZE: usize = EVENT_WORDS * size_of::(); +/// Producer offset within a firmware-produced ring's state block. +pub(crate) const RX_PRODUCER_OFFSET: usize = 0x20; +/// Trace records use type 5. +pub(crate) const TRACE_TYPES: u32 = 1 << 5; +/// Statistics record types accepted by the host. +pub(crate) const STATISTICS_TYPES: u32 = 0x5fff_0000; + +/// Completion stamp indices signalled by the firmware. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +struct StampEventRecord { + tag: u32, + masks: [u64; 2], + diagnostic_pairs: u16, + unk_16: [u8; 0x32], +} + +/// Restart generation and the affected stamp, or -1 without attribution. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +struct RestartEventRecord { + tag: u32, + generation: u64, + stamp: i32, + unk_10: [u8; 0x38], +} + +/// Parameter-buffer growth request. The reply moves `halt_count` to +0x18. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +struct TvbGrowEventRecord { + tag: u32, + vm_slot: i32, + buffer_slot: u32, + counter: u32, + subpipe: u32, + halt_count: u64, + unk_1c: [u8; 0x2c], +} + +/// Completed asynchronous free-list growth. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +struct FreeListEventRecord { + tag: u32, + stamp: i32, + buffer_slot: u32, + control_va: u64, + ring_va: u64, + cookie: u64, + result: u32, + unk_28: [u8; 0x20], +} + +/// A snapshot of one firmware event. Every variant occupies a complete slot; +/// callers copy the entry before publishing its consumer cursor. +#[repr(C)] +#[derive(Copy, Clone)] +pub(crate) union EventRecord { + words: [u32; EVENT_WORDS], + stamp: StampEventRecord, + restart: RestartEventRecord, + tvb: TvbGrowEventRecord, + free_list: FreeListEventRecord, +} + +static_assert!(size_of::() == EVENT_RECORD_SIZE); +static_assert!(size_of::() == EVENT_RECORD_SIZE); +static_assert!(size_of::() == EVENT_RECORD_SIZE); +static_assert!(size_of::() == EVENT_RECORD_SIZE); +static_assert!(size_of::() == EVENT_RECORD_SIZE); +static_assert!(core::mem::offset_of!(StampEventRecord, masks) == 4); +static_assert!(core::mem::offset_of!(TvbGrowEventRecord, halt_count) == 0x14); +static_assert!(core::mem::offset_of!(FreeListEventRecord, cookie) == 0x1c); + +impl core::fmt::Debug for EventRecord { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("EventRecord") + .field("tag", &self.tag()) + .finish() + } +} + +/// A decoded asynchronous free-list growth completion. Resource ownership and +/// the expected cookie/result are checked by the free-list owner. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct FreeListCompletion { + pub(crate) stamp: i32, + pub(crate) buffer_slot: u32, + pub(crate) control_va: u64, + pub(crate) ring_va: u64, + pub(crate) cookie: u64, + pub(crate) result: u32, +} + +/// Host actions requested by a firmware event. Unknown tags are consumed so +/// one unrecognized record cannot block later completions. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Event { + Stamp([u64; 2]), + Restart { generation: u64, stamp: i32 }, + TvbGrow(TvbGrowRequest), + FreeListGrow(FreeListCompletion), + ContextKilled(u64), + Unknown(u32), +} + +impl EventRecord { + const STAMP: u32 = 1; + const RESTART: u32 = 4; + const TVB_GROW: u32 = 6; + const FREE_LIST_GROW: u32 = 13; + const CONTEXT_KILLED: u32 = 12; + const DIAGNOSTIC_PAIRS_MAX: u16 = 5; + const TVB_SLOTS: u32 = 127; + const SUBPIPES: u32 = 64; + const FREE_LIST_SLOTS: u32 = 256; + + pub(crate) const fn from_words(words: [u32; EVENT_WORDS]) -> Self { + Self { words } + } + + pub(crate) fn tag(&self) -> u32 { + // SAFETY: All constructors initialize the entire union. Every bit + // pattern is valid for its u32 words, and word zero is the tag. + unsafe { self.words[0] } + } + + /// Validates the record's fields without interpreting resource ownership. + /// A malformed known event is an error, unlike an unknown tag. + pub(crate) fn decode(&self) -> Result { + match self.tag() { + Self::STAMP => { + // SAFETY: The entire slot is initialized, the tag selects + // this layout, and its integer fields have no invalid values. + let record = unsafe { self.stamp }; + if record.diagnostic_pairs > Self::DIAGNOSTIC_PAIRS_MAX { + return Err(EIO); + } + Ok(Event::Stamp(record.masks)) + } + Self::RESTART => { + // SAFETY: As above, for the restart layout. + let record = unsafe { self.restart }; + Ok(Event::Restart { + generation: record.generation, + stamp: record.stamp, + }) + } + Self::TVB_GROW => { + // SAFETY: As above, for the parameter-buffer growth layout. + let record = unsafe { self.tvb }; + if record.vm_slot < -1 + || record.buffer_slot >= Self::TVB_SLOTS + || record.subpipe >= Self::SUBPIPES + { + return Err(EIO); + } + Ok(Event::TvbGrow(TvbGrowRequest { + buffer_slot: record.buffer_slot, + vm_slot: record.vm_slot, + counter: record.counter, + subpipe: record.subpipe, + halt_count: record.halt_count, + })) + } + Self::CONTEXT_KILLED => { + // SAFETY: Every constructor initializes all integer words. + let words = unsafe { self.words }; + Ok(Event::ContextKilled( + u64::from(words[1]) | (u64::from(words[2]) << 32), + )) + } + Self::FREE_LIST_GROW => { + // SAFETY: As above, for the free-list growth layout. + let record = unsafe { self.free_list }; + if record.stamp < -1 + || record.buffer_slot >= Self::FREE_LIST_SLOTS + || record.control_va == 0 + || record.ring_va == 0 + { + return Err(EIO); + } + Ok(Event::FreeListGrow(FreeListCompletion { + stamp: record.stamp, + buffer_slot: record.buffer_slot, + control_va: record.control_va, + ring_va: record.ring_va, + cookie: record.cookie, + result: record.result, + })) + } + tag => Ok(Event::Unknown(tag)), + } + } +} + +/// A complete device-control request. Its variant supplies the power policy; +/// callers construct the contained record with its typed builder. +#[derive(Debug)] +pub(crate) enum ControlRecord { + TvbGrow(TvbGrowReply), + IdlePowerOff(IdlePowerOff), + SchedulerStateRelease(SchedulerStateRelease), + FreeListRelease(FreeListRelease), + ContextKill(ContextKill), +} + +impl ControlRecord { + pub(crate) const fn opcode(&self) -> DeviceControlOp { + match self { + Self::TvbGrow(_) => DeviceControlOp::TvbGrowReply, + Self::IdlePowerOff(_) => DeviceControlOp::IdlePowerOff, + Self::SchedulerStateRelease(_) => DeviceControlOp::SchedulerStateRelease, + Self::FreeListRelease(_) => DeviceControlOp::FreeListRelease, + Self::ContextKill(_) => DeviceControlOp::ContextKill, + } + } + + /// Byte view of the initialized record, excluding the host-only enum tag. + pub(crate) fn bytes(&self) -> &[u8; DEVICE_CONTROL_RECORD_SIZE] { + let record = match self { + Self::TvbGrow(record) => core::ptr::from_ref(record).cast(), + Self::IdlePowerOff(record) => core::ptr::from_ref(record).cast(), + Self::SchedulerStateRelease(record) => core::ptr::from_ref(record).cast(), + Self::FreeListRelease(record) => core::ptr::from_ref(record).cast(), + Self::ContextKill(record) => core::ptr::from_ref(record).cast(), + }; + // SAFETY: Each variant has a statically checked size of 0x40 bytes, + // contains only initialized integer fields with no implicit padding, + // and is borrowed for the lifetime of `self`. The byte view has align 1. + unsafe { &*record } + } +} + +/// Device-control opcodes the driver sends. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum DeviceControlOp { + /// Reply to a parameter-buffer grow request ([`TvbGrowReply`]). + TvbGrowReply = 0x08, + /// Set the idle power-off policy ([`IdlePowerOff`]). + IdlePowerOff = 0x0a, + /// Release a logical queue's scheduler state ([`SchedulerStateRelease`]). + SchedulerStateRelease = 0x14, + /// Release a free list's buffer slot ([`FreeListRelease`]). + FreeListRelease = 0x2e, + /// Stop the queues named by a closed logical scheduler. + ContextKill = 0x29, +} + +impl DeviceControlOp { + /// Tells whether records with this opcode count towards the host + /// power-assert tally. The firmware compares the tally with the counted + /// records it has drained and keeps the GPU cores powered while they + /// differ. + pub(crate) const fn asserts_gpu_power(self) -> bool { + !matches!(self, Self::FreeListRelease) + } +} + +/// The fields of a parameter-buffer grow request event that its reply +/// echoes. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct TvbGrowRequest { + /// Buffer slot of the parameter buffer. + pub(crate) buffer_slot: u32, + /// VM slot of the request, or -1. + pub(crate) vm_slot: i32, + /// Request counter. + pub(crate) counter: u32, + /// Sub-pipe of the request. + pub(crate) subpipe: u32, + /// Firmware halt count the request was made under. + pub(crate) halt_count: u64, +} + +/// Reply to a parameter-buffer grow request. +#[repr(C)] +#[derive(Debug)] +pub(crate) struct TvbGrowReply { + /// [`DeviceControlOp::TvbGrowReply`]. + pub(crate) opcode: u32, + /// Non-zero when the parameter buffer grew. + pub(crate) grew: u32, + /// Buffer slot of the request. + pub(crate) buffer_slot: u32, + /// VM slot of the request. + pub(crate) vm_slot: i32, + /// Request counter. + pub(crate) counter: u32, + /// Sub-pipe of the request. + pub(crate) subpipe: u32, + /// Halt count of the request. + pub(crate) halt_count: u64, + pub(crate) unk_20: [u8; 0x20], +} + +static_assert!(size_of::() == DEVICE_CONTROL_RECORD_SIZE); + +impl TvbGrowReply { + /// Replies to `request`; `grew` tells whether the buffer was grown. + pub(crate) const fn new(request: &TvbGrowRequest, grew: bool) -> Self { + Self { + opcode: DeviceControlOp::TvbGrowReply as u32, + grew: grew as u32, + buffer_slot: request.buffer_slot, + vm_slot: request.vm_slot, + counter: request.counter, + subpipe: request.subpipe, + halt_count: request.halt_count, + unk_20: [0; 0x20], + } + } +} + +/// Idle power-off policy argument of [`IdlePowerOff`]. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum IdlePolicy { + /// Keep the GPU cores powered and power them up now. + KeepPowered = 0, + /// Let the firmware power the GPU cores off when idle. + AllowPowerOff = 1, +} + +/// Sets the firmware's idle power-off policy. +#[repr(C)] +#[derive(Debug)] +pub(crate) struct IdlePowerOff { + /// [`DeviceControlOp::IdlePowerOff`]. + pub(crate) opcode: u32, + /// [`IdlePolicy`]. + pub(crate) policy: u32, + pub(crate) unk_08: [u8; 0x38], +} + +static_assert!(size_of::() == DEVICE_CONTROL_RECORD_SIZE); + +impl IdlePowerOff { + /// Requests `policy`. + pub(crate) const fn new(policy: IdlePolicy) -> Self { + Self { + opcode: DeviceControlOp::IdlePowerOff as u32, + policy: policy as u32, + unk_08: [0; 0x38], + } + } +} + +/// Releases the scheduler state of a logical queue whose work has drained. +/// +/// The page may be freed once the firmware has consumed the record. +#[repr(C, packed)] +#[derive(Debug)] +pub(crate) struct SchedulerStateRelease { + /// [`DeviceControlOp::SchedulerStateRelease`]. + pub(crate) opcode: u32, + pub(crate) unk_04: u32, + /// The scheduler-page bytes at offsets 0x26, 0, 1 and 4, in record order. + /// Read when the release record is built. + pub(crate) id: [u8; 4], + /// GPU address of the scheduler-state page. + pub(crate) page_va: u64, + pub(crate) unk_14: [u8; 0x2c], +} + +static_assert!(size_of::() == DEVICE_CONTROL_RECORD_SIZE); +static_assert!(core::mem::offset_of!(SchedulerStateRelease, page_va) == 0x0c); + +impl SchedulerStateRelease { + /// Releases the scheduler page with its identity bytes in record order. + pub(crate) const fn new(page_va: u64, id: [u8; 4]) -> Self { + Self { + opcode: DeviceControlOp::SchedulerStateRelease as u32, + unk_04: 0, + id, + page_va, + unk_14: [0; 0x2c], + } + } +} + +/// Releases the buffer slot of a free list after its last user. +#[repr(C, packed)] +#[derive(Debug)] +pub(crate) struct FreeListRelease { + /// [`DeviceControlOp::FreeListRelease`]. + pub(crate) opcode: u32, + /// Free-list generation of the registration being released. + pub(crate) generation: u64, + /// Buffer slot of the free list. + pub(crate) buffer_slot: u32, + pub(crate) unk_10: [u8; 0x30], +} + +static_assert!(size_of::() == DEVICE_CONTROL_RECORD_SIZE); +static_assert!(core::mem::offset_of!(FreeListRelease, buffer_slot) == 0x0c); + +impl FreeListRelease { + /// Releases `buffer_slot`, registered at `generation`. + pub(crate) const fn new(generation: u64, buffer_slot: u32) -> Self { + Self { + opcode: DeviceControlOp::FreeListRelease as u32, + generation, + buffer_slot, + unk_10: [0; 0x30], + } + } +} + +/// Number of slots in a work ring. +pub(crate) const WORK_RING_SLOTS: u32 = 256; +/// Number of engines with a work ring per priority. +const WORK_RINGS_PER_PRIORITY: usize = 3; + +/// Index in the channel table of the work ring for `data_master` at the +/// priority of every driver-created queue. +pub(crate) const fn work_ring_index(data_master: DataMaster) -> usize { + QUEUE_PRIORITY as usize * WORK_RINGS_PER_PRIORITY + data_master as usize +} + +/// One work-ring slot: a request to run a queue's newly published commands. +#[repr(C)] +#[derive(Debug)] +pub(crate) struct WorkSlot { + pub(crate) unk_00: u64, + /// GPU address of the queue record. + pub(crate) queue_va: u64, + /// [`DataMaster`] of the queue. + pub(crate) data_master: u32, + /// Item-ring producer after the published commands. + pub(crate) items_end: u16, + /// Queue to run. + pub(crate) qid: u8, + /// One in the first slot naming this queue on this ring. + pub(crate) first: u8, +} + +static_assert!(size_of::() == 0x18); + +impl WorkSlot { + /// Builds the slot asking the firmware to run queue `qid` up to item-ring + /// position `items_end`. `first` tells whether this is the first slot + /// naming the queue on this ring. + pub(crate) fn new( + data_master: DataMaster, + queue_va: u64, + qid: u8, + items_end: u16, + first: bool, + ) -> Result { + if qid > QID_MAX { + return Err(EINVAL); + } + Ok(Self { + unk_00: 0, + queue_va, + data_master: data_master as u32, + items_end, + qid, + first: first as u8, + }) + } +} + +/// Stop a closed context's scheduler page and echo its cookie through event 12. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct ContextKill { + opcode: u32, + page_va: u64, + cookie: u64, + reserved: [u8; 0x2c], +} +static_assert!(size_of::() == DEVICE_CONTROL_RECORD_SIZE); +static_assert!(core::mem::offset_of!(ContextKill, page_va) == 4); +static_assert!(core::mem::offset_of!(ContextKill, cookie) == 0x0c); +impl ContextKill { + pub(crate) fn new(page_va: u64, cookie: u64) -> Self { + Self { + opcode: DeviceControlOp::ContextKill as u32, + page_va, + cookie, + reserved: [0; 0x2c], + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs new file mode 100644 index 0000000000000..22f8259508ff6 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -0,0 +1,379 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Compute work descriptor. +//! +//! A compute kick launches one [`ComputeDescriptor`]. The descriptor embeds +//! the two register arrays its kick entry binds: the main array and a +//! four-entry array that writes the operand-state registers with their +//! regions exchanged. + +use kernel::prelude::*; + +use super::kick::{KickTimestamp, RegisterArray, RegisterArrayBinding, RegisterWrite}; +use super::queue::CommandTag; +use super::{ + clear, offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, + UNK_0D411, UNK_101D9, UNK_10791, UNK_1A0E9, UNK_1C8F8, +}; + +/// Size of a compute descriptor. +pub(crate) const COMPUTE_DESCRIPTOR_SIZE: usize = 0x4000; +/// QoS class of compute kicks. +pub(crate) const COMPUTE_QOS_CLASS: u8 = 0x08; +/// Event mask of compute kicks. +pub(crate) const COMPUTE_KICK_EVENT_MASK: [u64; 4] = [0, 0, 4, 0]; + +const COMPUTE_REGISTERS: usize = 40; +const ALT_REGISTERS: usize = 4; + +/// Object IDs of every compute kick. +const OBJECT_IDS: ObjectIds = ObjectIds { + predecessor: 0x0100_01d7, + current: 0x0200_01dc, +}; +/// Values of the first and second write of register 0x1a440. +const UNK_1A440: u64 = 0x0000_0001_5402_4201; +const UNK_1A440_FINAL: u64 = 0x0000_0001_5402_4209; +/// Value of register 0x1a458: [`UNK_1C8F8`] below bits 31:20 = 0x10c. +const UNK_1A458: u64 = 0x10c0_0000 | UNK_1C8F8; + +/// Offsets in the preempt buffer of the four state words the program names. +const PREEMPT_STATE: [u64; 4] = [0x1480, 0x1488, 0x1490, 0x1498]; +/// Offsets in the operand-state buffer of the regions the program names. +const OPERAND_STATE_10229: u64 = 0x12800; +const OPERAND_STATE_140A8: u64 = 0x13000; +const OPERAND_STATE_10099: u64 = 0x9405; +const OPERAND_STATE_10091: u64 = 0x12400; +const OPERAND_STATE_0A5C1: u64 = 0x0005; +const OPERAND_STATE_0A5C9: u64 = 0x9000; + +/// Checkpoints into the main register array, as entry counts: after the +/// first 0x0a5c9 write, after the final 0x1a440 write (twice), and the full +/// count. +const CHECKPOINTS: [u16; 4] = [38, 39, 39, COMPUTE_REGISTERS as u16]; + +/// Inputs of one compute descriptor. +pub(crate) struct ComputeArgs { + /// Base of the queue's USC window. + pub(crate) usc_base: u64, + /// GPU address of the control stream. + pub(crate) cdm_va: u64, + /// GPU address of the control-stream terminator. + pub(crate) cdm_end_va: u64, + /// GPU address of the sampler heap, or zero. + pub(crate) sampler_heap_va: u64, + /// Number of samplers in the heap. + pub(crate) sampler_count: u32, + /// GPU address of this descriptor, as seen by the work's VM. + pub(crate) descriptor_va: u64, + /// UAT context the work runs in. + pub(crate) context_id: u16, + /// Allocation generation of that context. + pub(crate) context_generation: u8, + /// State word of the logical queue's work-state node. + pub(crate) work_state: u32, + /// GPU address of the logical queue's work-state node. + pub(crate) work_state_va: u64, + /// Queue the work is kicked on. + pub(crate) qid: u8, + /// Number of earlier kicks of the queue. + pub(crate) kick_count: u32, + /// Timestamp the work is kicked at. + pub(crate) kick: KickTimestamp, + /// GPU address of the preempt buffer, as seen by the work's VM. + pub(crate) preempt_va: u64, + /// GPU address of the operand-state buffer, as seen by the work's VM. + pub(crate) operand_state_va: u64, + /// GPU address of the private-memory usage record, as seen by the work's + /// VM. + pub(crate) usage_va: u64, + /// Firmware alias of the private-memory usage record. + pub(crate) usage_fw_va: u64, + /// Buffer slot of the USC private-memory free list. + pub(crate) free_list_slot: u32, + /// GPU address of the free list's shared control object. + pub(crate) free_list_control_va: u64, + /// GPU address the completion stamp value is stored to. + pub(crate) stamp_va: u64, + /// GPU address of the word following the completion stamp. + pub(crate) aux_stamp_va: u64, + /// GPU addresses of the two status words. + pub(crate) status_va: [u64; 2], + /// GPU address of the start/end timestamp pair. + pub(crate) timestamp_va: u64, +} + +fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGISTERS]> { + let preempt = args.preempt_va; + let operand = args.operand_state_va; + let key = work_key(args.context_id, args.work_state); + let ids = OBJECT_IDS.word(); + + Ok([ + RegisterWrite::new(0x1a510, preempt), + RegisterWrite::new(0x1a420, args.cdm_va), + RegisterWrite::new(0x1a4d0, offset_va(preempt, PREEMPT_STATE[0])?), + RegisterWrite::new(0x1a4d8, offset_va(preempt, PREEMPT_STATE[1])?), + RegisterWrite::new(0x1a4e0, offset_va(preempt, PREEMPT_STATE[2])?), + RegisterWrite::new(0x1a4e8, offset_va(preempt, PREEMPT_STATE[3])?), + RegisterWrite::new(0x10071, args.usc_base), + // Helper program binary, data and configuration: none. + RegisterWrite::new(0x11841, 0), + RegisterWrite::new(0x11849, 0), + RegisterWrite::new(0x11f81, 0), + RegisterWrite::new(0x1a440, UNK_1A440), + RegisterWrite::new(0x1a458, UNK_1A458), + RegisterWrite::new(0x101d9, UNK_101D9), + RegisterWrite::new(0x1a089, 0), + RegisterWrite::new(0x1a091, 0), + RegisterWrite::new(0x1a059, 0), + RegisterWrite::new(0x1a061, 0), + RegisterWrite::new(0x1a0b9, 0), + RegisterWrite::new(0x1a0c1, 0), + RegisterWrite::new(0x101d1, 0), + RegisterWrite::new(0x0d479, 0), + RegisterWrite::new(0x1a0e9, UNK_1A0E9), + RegisterWrite::new(0x107a1, UNK_10791), + RegisterWrite::new(0x0a599, UNK_0A599), + RegisterWrite::new(0x0d411, UNK_0D411), + RegisterWrite::new(0x1a540, ids), + RegisterWrite::new(0x014a9, ids), + RegisterWrite::new(0x0a351, ids), + RegisterWrite::new(0x10201, key), + RegisterWrite::new(0x10428, key), + RegisterWrite::new(0x14028, args.free_list_slot as u64), + RegisterWrite::new(0x14070, args.usage_va | 1), + RegisterWrite::new(0x10229, offset_va(operand, OPERAND_STATE_10229)?), + RegisterWrite::new(0x140a8, offset_va(operand, OPERAND_STATE_140A8)?), + RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_10099)?), + RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_10091)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_0A5C1)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_0A5C9)?), + RegisterWrite::new(0x1a440, UNK_1A440_FINAL), + RegisterWrite::new(0x0a599, UNK_0A599), + ]) +} + +/// The second array exchanges the operand-state regions of registers +/// 0x10099/0x0a5c1 and 0x10091/0x0a5c9. +fn alt_registers(args: &ComputeArgs) -> Result<[RegisterWrite; ALT_REGISTERS]> { + let operand = args.operand_state_va; + Ok([ + RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_0A5C1)?), + RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_0A5C9)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_10099)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_10091)?), + ]) +} + +/// Compute work descriptor (tag 3). +#[repr(C, packed)] +#[derive(Debug)] +pub(crate) struct ComputeDescriptor { + /// [`CommandTag::Compute`]. + pub(crate) tag: u32, + pub(crate) unk_04: u64, + /// UAT context of the work. + pub(crate) context_id: u32, + /// GPU address of the logical queue's work-state node. + pub(crate) work_state_va: u64, + /// Checkpoints into the main register array. + pub(crate) checkpoints: [u16; 4], + pub(crate) unk_20: [u8; 0x20], + /// Main register array. + pub(crate) registers: RegisterArray, + /// Second register array. + pub(crate) alt_registers: RegisterArray, + pub(crate) unk_e80: [u8; 0x58], + /// GPU address of the preempt buffer. + pub(crate) preempt_va: u64, + /// GPU address of the control-stream terminator. + pub(crate) cdm_end_va: u64, + pub(crate) unk_ee8: [u8; 0x20], + /// First value of register 0x1a440. + pub(crate) unk_f08: u64, + pub(crate) unk_f10: [u8; 0x10], + /// Predecessor object ID. + pub(crate) predecessor_id: u32, + pub(crate) unk_f24: u32, + pub(crate) unk_f28: u32, + /// GPU address of the sampler heap. + pub(crate) sampler_heap_va: u64, + /// Number of samplers. + pub(crate) sampler_count: u32, + /// Number of samplers plus one, or zero without samplers. + pub(crate) sampler_max: u32, + pub(crate) unk_f3c: u32, + /// GPU address the firmware stores `stamp_value` to on completion. + pub(crate) stamp_va: u64, + /// GPU address of the word following the completion stamp. + pub(crate) aux_stamp_va: u64, + /// Completion stamp value of the kick timestamp. + pub(crate) stamp_value: u32, + /// QID of the compute queue. + pub(crate) qid: u32, + pub(crate) unk_f58: u32, + pub(crate) unk_f5c: u32, + /// Kick timestamp. + pub(crate) kick_timestamp: u64, + /// Current object ID. + pub(crate) current_id: u64, + /// Number of earlier kicks of the queue. + pub(crate) kick_count: u32, + pub(crate) unk_f74: [u8; 8], + /// GPU addresses of the two status words. + pub(crate) status_va: [u64; 2], + /// GPU address the start timestamp is written to. + pub(crate) timestamp_start_va: u64, + /// GPU address the end timestamp is written to. + pub(crate) timestamp_end_va: u64, + pub(crate) unk_f9c: [u8; 0x14], + pub(crate) unk_fb0: u16, + /// GPU address of the USC free list's shared control object. + pub(crate) free_list_control_va: u64, + pub(crate) unk_fba: u32, + pub(crate) unk_fbe: [u8; 7], + pub(crate) unk_fc5: u8, + pub(crate) unk_fc6: [u8; 5], + /// Firmware alias of the private-memory usage record. + pub(crate) usage_va: u64, + /// Allocation generation of the UAT context. + pub(crate) context_generation: u8, + pub(crate) unk_fd4: [u8; 0x302c], +} + +static_assert!(size_of::() == COMPUTE_DESCRIPTOR_SIZE); +static_assert!(core::mem::offset_of!(ComputeDescriptor, registers) == 0x40); +static_assert!(core::mem::offset_of!(ComputeDescriptor, alt_registers) == 0x760); +static_assert!(core::mem::offset_of!(ComputeDescriptor, preempt_va) == 0xed8); +static_assert!(core::mem::offset_of!(ComputeDescriptor, unk_f08) == 0xf08); +static_assert!(core::mem::offset_of!(ComputeDescriptor, predecessor_id) == 0xf20); +static_assert!(core::mem::offset_of!(ComputeDescriptor, sampler_heap_va) == 0xf2c); +static_assert!(core::mem::offset_of!(ComputeDescriptor, stamp_va) == 0xf40); +static_assert!(core::mem::offset_of!(ComputeDescriptor, kick_timestamp) == 0xf60); +static_assert!(core::mem::offset_of!(ComputeDescriptor, current_id) == 0xf68); +static_assert!(core::mem::offset_of!(ComputeDescriptor, status_va) == 0xf7c); +static_assert!(core::mem::offset_of!(ComputeDescriptor, free_list_control_va) == 0xfb2); +static_assert!(core::mem::offset_of!(ComputeDescriptor, usage_va) == 0xfcb); +static_assert!(core::mem::offset_of!(ComputeDescriptor, context_generation) == 0xfd3); + +// SAFETY: `ComputeDescriptor` consists of integers and arrays of integers only. +unsafe impl Zeroable for ComputeDescriptor {} + +impl ComputeDescriptor { + const REGISTERS_OFFSET: u64 = core::mem::offset_of!(Self, registers) as u64; + const ALT_REGISTERS_OFFSET: u64 = core::mem::offset_of!(Self, alt_registers) as u64; + const UNK_F28: u32 = u32::MAX; + const UNK_FB0: u16 = 0x1a; + const UNK_FBA: u32 = 0xe0a0_0001; + const UNK_FC5: u8 = 0x9f; + + /// Writes the complete descriptor. + pub(crate) fn write(&mut self, args: &ComputeArgs) -> Result { + if args.cdm_end_va < args.cdm_va + || (args.sampler_heap_va == 0) != (args.sampler_count == 0) + || args.sampler_heap_va & 7 != 0 + { + return Err(EINVAL); + } + let registers = compute_registers(args)?; + let alt_registers = alt_registers(args)?; + let registers_va = offset_va(args.descriptor_va, Self::REGISTERS_OFFSET)?; + let alt_registers_va = offset_va(args.descriptor_va, Self::ALT_REGISTERS_OFFSET)?; + let timestamp_end_va = timestamp_end_va(args.timestamp_va)?; + let sampler_max = sampler_max(args.sampler_count)?; + + clear(self); + self.tag = CommandTag::Compute as u32; + self.context_id = args.context_id.into(); + self.work_state_va = args.work_state_va; + self.checkpoints = CHECKPOINTS; + self.registers.set(registers_va, ®isters)?; + self.alt_registers.set(alt_registers_va, &alt_registers)?; + self.preempt_va = args.preempt_va; + self.cdm_end_va = args.cdm_end_va; + self.unk_f08 = UNK_1A440; + self.predecessor_id = OBJECT_IDS.predecessor; + self.unk_f28 = Self::UNK_F28; + self.sampler_heap_va = args.sampler_heap_va; + self.sampler_count = args.sampler_count; + self.sampler_max = sampler_max; + self.stamp_va = args.stamp_va; + self.aux_stamp_va = args.aux_stamp_va; + self.stamp_value = stamp_value(args.kick.get() as u32); + self.qid = args.qid.into(); + self.kick_timestamp = args.kick.get(); + self.current_id = OBJECT_IDS.current.into(); + self.kick_count = args.kick_count; + self.status_va = args.status_va; + self.timestamp_start_va = args.timestamp_va; + self.timestamp_end_va = timestamp_end_va; + self.unk_fb0 = Self::UNK_FB0; + self.free_list_control_va = args.free_list_control_va; + self.unk_fba = Self::UNK_FBA; + self.unk_fc5 = Self::UNK_FC5; + self.usage_va = args.usage_fw_va; + self.context_generation = args.context_generation; + Ok(()) + } + + /// Returns the register arrays a compute kick binds; `descriptor_va` is + /// the descriptor's address as seen by the work's VM. + pub(crate) fn register_bindings( + descriptor_va: u64, + ) -> Result<[Option; 4]> { + let registers = RegisterArrayBinding::new( + offset_va(descriptor_va, Self::REGISTERS_OFFSET)?, + CHECKPOINTS[0] as u8, + )?; + let alt_registers = RegisterArrayBinding::new( + offset_va(descriptor_va, Self::ALT_REGISTERS_OFFSET)?, + ALT_REGISTERS as u8, + )?; + Ok([Some(registers), Some(alt_registers), None, None]) + } +} + +/// One retained queue-context record. The descriptor pointer is refreshed for each compute +/// publication; the remaining fields are installed once with the queue. +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct QueueContextItem { + unk_00: u64, + unk_08: u64, + descriptor_va: u64, + queue_va: u64, + unk_20: u64, + qid: u64, + unk_30: [u8; 0x100], + unk_130: u64, + unk_138: u64, + unk_140: [u8; 0x10], + unk_150: u64, + unk_158: u64, + unk_160: [u8; 0x18], + unk_178: u64, + unk_180: [u8; 0x80], +} +impl QueueContextItem { + pub(crate) fn new(descriptor_va: u64, queue_va: u64, qid: u8) -> Self { + Self { + unk_00: 0x1000_1000_0000_0004, + unk_08: 0, + descriptor_va, + queue_va, + unk_20: 0xffff_0801_0000_0001, + qid: u64::from(qid) << 40, + unk_30: [0; 0x100], + unk_130: 0, + unk_138: 4, + unk_140: [0; 0x10], + unk_150: 0x0001_1003_8001_a002, + unk_158: 0x0000_2003_8001_a03b, + unk_160: [0; 0x18], + unk_178: 0x003f_ffff_ffff_ffff, + unk_180: [0; 0x80], + } + } +} +static_assert!(size_of::() == 0x200); diff --git a/drivers/gpu/drm/asahi/g17/fw/context.rs b/drivers/gpu/drm/asahi/g17/fw/context.rs new file mode 100644 index 0000000000000..51dbde231e238 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/context.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Scheduler and timestamp-queue state shared with firmware. Only a freshly +//! leased node may be initialized. After publication, the host writes its +//! separate submitted counter and reads the node completion word individually. + +use kernel::prelude::*; + +#[repr(C)] +pub(crate) struct Scheduler { + unk0: u64, + unk8: [u64; 3], + unk20: u64, + unk28: u64, + unk30: u64, + unk38: [u8; 0x3fc8], +} +impl Scheduler { + /// Called on zeroed storage before its first publication. + pub(crate) fn init(&mut self) { + self.unk0 = 0x0000_0100_0000_ffff; + self.unk20 = 0x0002_0000_0000_0000; + self.unk30 = 0x0000_0000_ff00_0000; + } +} + +#[derive(Clone, Copy)] +#[repr(C)] +pub(crate) struct WorkNode { + pub(crate) submitted: u64, + pub(crate) word: u32, + pub(crate) completed: u32, + pub(crate) threshold: u32, + unk14: [u32; 45], +} +impl WorkNode { + pub(crate) fn new(submitted: u64, word: u32) -> Option { + if submitted == 0 || submitted & 3 != 0 || word & 63 == 63 { + return None; + } + Some(Self { + submitted, + word, + completed: 0, + threshold: 0x50, + unk14: [0; 45], + }) + } +} + +#[derive(Clone, Copy)] +#[repr(C)] +pub(crate) struct WorkHead { + pub(crate) next: u64, + pub(crate) previous: u64, + pub(crate) unk10: u32, + pub(crate) unk14: u32, +} + +const _: () = { + assert!(core::mem::size_of::() == 0x4000); + assert!(core::mem::size_of::() == 0xc8); + assert!(core::mem::offset_of!(WorkNode, completed) == 0xc); + assert!(core::mem::size_of::() == 0x18); +}; + +// SAFETY: These shared records contain only integer fields and arrays. +unsafe impl Zeroable for Scheduler {} +// SAFETY: WorkNode contains only integers; zero is valid for every field. +unsafe impl Zeroable for WorkNode {} +// SAFETY: WorkHead contains only integer addresses and words. +unsafe impl Zeroable for WorkHead {} diff --git a/drivers/gpu/drm/asahi/g17/fw/freelist.rs b/drivers/gpu/drm/asahi/g17/fw/freelist.rs new file mode 100644 index 0000000000000..2e5708dddd455 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/freelist.rs @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! USC free-list page addresses and block run tables. + +use super::initdata::{FreeListRunSlot, FREE_LIST_BLOCK_PAGES, FREE_LIST_BLOCK_SIZE}; +use kernel::prelude::*; + +pub(crate) const PAGE_LIST_SIZE: usize = 0x20_0000; +pub(crate) const RUN_LIST_SIZE: usize = 0x1_0000; +pub(crate) const TABLE_SIZE: usize = PAGE_LIST_SIZE + RUN_LIST_SIZE; +pub(crate) const CONTROL_SIZE: usize = 0x4000; +pub(crate) const RENDER_BLOCKS: usize = 22; +pub(crate) const COMPUTE_BLOCKS: usize = 21; + +/// Render reserves eight run words per block; compute packs runs consecutively. +#[derive(Copy, Clone, Debug)] +pub(crate) enum RunLayout { + Render, + Compute, +} + +impl RunLayout { + const fn words(self) -> usize { + match self { + Self::Render => size_of::() / 8, + Self::Compute => 1, + } + } +} + +#[repr(C)] +pub(crate) struct PoolTables { + pages: [u64; PAGE_LIST_SIZE / 8], + runs: [u64; RUN_LIST_SIZE / 8], +} + +// SAFETY: The table consists entirely of integer arrays. +unsafe impl Zeroable for PoolTables {} +static_assert!(size_of::() == TABLE_SIZE); +static_assert!(core::mem::offset_of!(PoolTables, runs) == PAGE_LIST_SIZE); + +impl PoolTables { + /// Enumerates only the initialized run words and page addresses, in their + /// publication order. Reserved run words and unused table space stay zero. + pub(crate) fn for_each_word( + base: u64, + blocks: usize, + layout: RunLayout, + mut put: impl FnMut(usize, u64), + ) -> Result { + if blocks > RUN_LIST_SIZE / (layout.words() * 8) + || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) + { + return Err(EINVAL); + } + let buffers = base.checked_add(TABLE_SIZE as u64).ok_or(EOVERFLOW)?; + buffers + .checked_add((blocks * FREE_LIST_BLOCK_SIZE) as u64) + .ok_or(EOVERFLOW)?; + for block in 0..blocks { + let address = buffers + (block * FREE_LIST_BLOCK_SIZE) as u64; + put( + PAGE_LIST_SIZE / 8 + block * layout.words(), + FreeListRunSlot::new(address).run, + ); + for page in 0..FREE_LIST_BLOCK_PAGES as usize { + put( + block * FREE_LIST_BLOCK_PAGES as usize + page, + address + + (page * (FREE_LIST_BLOCK_SIZE / FREE_LIST_BLOCK_PAGES as usize)) as u64, + ); + } + } + Ok(()) + } + + /// Initializes a zeroed table before its first publication. + pub(crate) fn init(&mut self, base: u64, blocks: usize, layout: RunLayout) -> Result { + Self::for_each_word(base, blocks, layout, |index, value| { + if index < self.pages.len() { + self.pages[index] = value; + } else { + self.runs[index - self.pages.len()] = value; + } + }) + } +} + +/// Mutable page-pool state carried between the retained control record and +/// its global descriptor. Offsets name the source control-record words. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct PagePoolState { + pub(crate) page_list_va: u64, + pub(crate) unk_1c: u32, + pub(crate) unk_20: u32, + pub(crate) unk_24: u32, + pub(crate) unk_28: u32, + pub(crate) page_count: u32, +} + +impl PagePoolState { + /// The fourth descriptor word belongs to firmware and survives every load. + pub(crate) const fn descriptor(self, unk_18: u64) -> super::initdata::PagePoolDescriptor { + super::initdata::PagePoolDescriptor { + page_list: ((self.unk_1c as u64) << 41) | (self.page_list_va >> 7), + counters: ((self.unk_24 as u64) << 33) + | ((self.unk_20 as u64) << 5) + | ((self.unk_28 as u64) << 61), + page_count: self.page_count as u64, + unk_18, + } + } + + /// Address and allocation unit remain properties of the retained pool. + pub(crate) const fn updated(self, descriptor: &super::initdata::PagePoolDescriptor) -> Self { + Self { + unk_20: ((descriptor.counters >> 5) & 0x0fff_ffff) as u32, + unk_24: ((descriptor.counters >> 33) & 0x0fff_ffff) as u32, + unk_28: (descriptor.counters >> 61) as u32, + page_count: descriptor.page_count as u32, + ..self + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs new file mode 100644 index 0000000000000..c25b843a913e7 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -0,0 +1,3063 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G17 firmware initialization data. +//! +//! Each of the two firmware instances (primary and secondary) is handed one +//! root at boot. The roots name the objects the firmware needs before any +//! work is submitted: +//! +//! - region A and region C, shared by both instances; +//! - one main configuration per instance, holding its channel table; +//! - the hardware-data bundle, one allocation holding the hardware data, the +//! power configuration, the work rings and both main configurations with +//! their device-control rings; +//! - per-instance status blocks inside the private cluster, the fixed-address +//! arena the firmware also writes at fixed offsets. +//! +//! This module defines those layouts and the pure functions that fill them. +//! Allocation, mapping and publication belong to the builder. Initialization +//! requires exclusive access before either firmware instance can access the +//! object. These constructors must not be used to update a published object: +//! its live fields require individual accesses of the widths documented below. +//! +//! All objects start out zeroed. An `unk_` field documents the value +//! the firmware requires there; an `unk_` field without a doc comment is zero. + +use kernel::prelude::*; + +use super::clear; + +/// Interface version at root `+0x00`; the firmware rejects any other. +pub(crate) const INTERFACE_VERSION: [u16; 4] = [0x04c0, 0x0396, 0xa322, 0x0c8a]; + +/// UAT page size, as a shift. +pub(crate) const PAGE_BITS: u32 = 14; +/// UAT page size. +pub(crate) const PAGE_SIZE: usize = 1 << PAGE_BITS; +/// Page-table levels below the root. +const UAT_LEVELS: usize = 3; +/// Address bits resolved by one page-table level. +const BITS_PER_LEVEL: u32 = PAGE_BITS - 3; + +/// Firmware instance, encoded in its root. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum Role { + /// Runs the scheduler, the device-control ring and all work channels. + Primary = 0, + /// Control-only peer of the primary. + Secondary = 1, +} + +// --------------------------------------------------------------------------- +// Roots +// --------------------------------------------------------------------------- + +/// One UAT page-table level, as the firmware walks it. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct LevelDescriptor { + /// Required: 8. + pub(crate) unk_0: u8, + /// Page size shift of the tables at this level. + pub(crate) table_page_bits: u8, + /// Page size shift of the pages they map. + pub(crate) page_bits: u8, + /// Lowest input-address bit indexing this level. + pub(crate) index_shift: u8, + /// Entries per table. + pub(crate) entries: u16, + /// Table size in bytes. + pub(crate) table_size: u16, + /// Required: 1. + pub(crate) unk_8: u64, + /// Output-address bits of an entry. + pub(crate) phys_mask: u64, + /// Input-address bits indexing this level. + pub(crate) index_mask: u64, +} + +// SAFETY: `LevelDescriptor` consists of integers and arrays of integers only. +unsafe impl Zeroable for LevelDescriptor {} + +static_assert!(size_of::() == 0x20); + +impl LevelDescriptor { + /// Returns the descriptor of `level` (0 = top) for a UAT with `ias` + /// input-address and `oas` output-address bits. + const fn new(level: u32, ias: u32, oas: u32) -> Self { + let index_shift = PAGE_BITS + (UAT_LEVELS as u32 - 1 - level) * BITS_PER_LEVEL; + let index_bits = if level == 0 { + ias - index_shift + } else { + BITS_PER_LEVEL + }; + let entries = 1u64 << index_bits; + Self { + unk_0: 8, + table_page_bits: PAGE_BITS as u8, + page_bits: PAGE_BITS as u8, + index_shift: index_shift as u8, + entries: entries as u16, + table_size: PAGE_SIZE as u16, + unk_8: 1, + phys_mask: ((1u64 << oas) - 1) & !(PAGE_SIZE as u64 - 1), + index_mask: (entries - 1) << index_shift, + } + } +} + +/// Addresses and geometry common to both roots. +pub(crate) struct RootArgs { + /// Region A. + pub(crate) region_a_va: u64, + /// Region C. + pub(crate) region_c_va: u64, + /// This instance's main configuration. + pub(crate) main_config_va: u64, + /// This instance's status A block. + pub(crate) status_a_va: u64, + /// UAT input-address bits of the firmware's context. + pub(crate) ias: u32, + /// UAT output-address bits. + pub(crate) oas: u32, +} + +/// Root of the primary instance; the secondary extends it ([`SecondaryRoot`]). +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct Root { + /// [`INTERFACE_VERSION`]. + pub(crate) version: [u16; 4], + /// Region A. + pub(crate) region_a_va: u64, + pub(crate) unk_10: u64, + /// Main configuration of this instance. + pub(crate) main_config_va: u64, + /// Region C. + pub(crate) region_c_va: u64, + /// [`Role`] of this instance. + pub(crate) role: u32, + /// Required: 1. + pub(crate) unk_2c: u32, + /// UAT page size. + pub(crate) page_size: u16, + /// UAT page size shift. + pub(crate) page_bits: u8, + /// Number of page-table levels. + pub(crate) levels: u8, + /// Page-table levels, top first. + pub(crate) level: [LevelDescriptor; UAT_LEVELS], + pub(crate) unk_94: [u8; 0x14], + /// Status A block of this instance. + pub(crate) status_a_va: u64, + /// Primary status B ([`PrimaryStatusB`]); zero for the secondary. + pub(crate) status_b_va: u64, +} + +// SAFETY: `Root` consists of integers and arrays of integers only. +unsafe impl Zeroable for Root {} + +static_assert!(core::mem::offset_of!(Root, role) == 0x28); +static_assert!(core::mem::offset_of!(Root, level) == 0x34); +static_assert!(core::mem::offset_of!(Root, status_a_va) == 0xa8); +static_assert!(size_of::() == 0xb8); + +impl Root { + fn new(role: Role, args: &RootArgs, status_b_va: u64) -> Self { + Self { + version: INTERFACE_VERSION, + region_a_va: args.region_a_va, + unk_10: 0, + main_config_va: args.main_config_va, + region_c_va: args.region_c_va, + role: role as u32, + unk_2c: 1, + page_size: PAGE_SIZE as u16, + page_bits: PAGE_BITS as u8, + levels: UAT_LEVELS as u8, + level: [ + LevelDescriptor::new(0, args.ias, args.oas), + LevelDescriptor::new(1, args.ias, args.oas), + LevelDescriptor::new(2, args.ias, args.oas), + ], + unk_94: [0; 0x14], + status_a_va: args.status_a_va, + status_b_va, + } + } + + /// Returns the primary root. + pub(crate) fn primary(args: &RootArgs, status_b_va: u64) -> Self { + Self::new(Role::Primary, args, status_b_va) + } +} + +/// Root of the secondary instance. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct SecondaryRoot { + /// Common part; the secondary has no status B. + pub(crate) root: Root, + /// The primary's [`PowerState`]. + pub(crate) power_state_va: u64, + /// The secondary's [`SecondaryConfig`]. + pub(crate) config_va: u64, +} + +// SAFETY: `SecondaryRoot` consists of integers and arrays of integers only. +unsafe impl Zeroable for SecondaryRoot {} + +static_assert!(size_of::() == 0xc8); + +impl SecondaryRoot { + /// Returns the secondary root. + pub(crate) fn new(args: &RootArgs, power_state_va: u64, config_va: u64) -> Self { + Self { + root: Root::new(Role::Secondary, args, 0), + power_state_va, + config_va, + } + } +} + +/// Offset of the secondary root from the primary root. Both live in one +/// allocation, and the firmware relies on this distance. +pub(crate) const SECONDARY_ROOT_OFFSET: usize = 0x8000; +/// Size of the allocation holding both roots. +pub(crate) const ROOTS_SIZE: usize = 0xc000; + +// --------------------------------------------------------------------------- +// Channels +// --------------------------------------------------------------------------- + +/// Work channels: TA, 3D and compute at each of four priorities. +pub(crate) const WORK_CHANNELS: usize = 12; +/// Engines with a work channel at each priority (TA, 3D, compute). +const WORK_ENGINES: usize = 3; +/// Work priorities. +const WORK_PRIORITIES: usize = WORK_CHANNELS / WORK_ENGINES; +/// Channel-table index of the device-control channel. +pub(crate) const CONTROL_CHANNEL: usize = 12; +/// Channel-table index of the event and firmware-log rings. +pub(crate) const EVENT_CHANNEL: usize = 13; +/// Channel-table index of the trace and statistics rings. +pub(crate) const TRACE_CHANNEL: usize = 14; +/// Channel-table index of the channel with a single state address. +const CHANNEL_15: usize = 15; +/// Entries in the channel table. +pub(crate) const CHANNELS: usize = 17; + +/// One channel-table entry: four GPU addresses whose meaning depends on the +/// channel. Work and control channels name three counters of their +/// [`ChannelState`] and their ring; the event and trace channels name two +/// state/ring pairs. +#[repr(C)] +pub(crate) struct ChannelEntry(pub(crate) [u64; 4]); + +// SAFETY: `ChannelEntry` is an array of integers. +unsafe impl Zeroable for ChannelEntry {} + +static_assert!(size_of::() == 0x20); + +impl ChannelEntry { + /// Returns the entry of a work or control channel. + const fn ring(state_va: u64, ring_va: u64) -> Self { + Self([ + state_va + ChannelState::CONSUMER as u64, + state_va + ChannelState::UNK_10 as u64, + state_va + ChannelState::PRODUCER as u64, + ring_va, + ]) + } +} + +/// State of a work or control channel in an instance's state grid. +/// +/// All three counters are live words. The firmware advances `consumer` and +/// `unk_10`; the host writes `producer` with single aligned stores, after the +/// records it publishes are complete. At boot the host stores +/// zero to the control channel's consumer, then zero to `unk_10`, then +/// [`OPENING_PRODUCER`] to its producer, primary instance before secondary, +/// after the rest of the initialization data is written. A full barrier follows +/// both instances' stores. These creation-time writes occur before publication; the firmware moves +/// all three to that value once it has drained the opening record. +#[repr(C)] +pub(crate) struct ChannelState { + /// Records the firmware has consumed. + pub(crate) consumer: u32, + pub(crate) unk_4: [u8; 0xc], + /// Firmware-owned. + pub(crate) unk_10: u32, + pub(crate) unk_14: [u8; 0xc], + /// Records the host has published. + pub(crate) producer: u32, + pub(crate) unk_24: [u8; 0x1c], +} + +// SAFETY: `ChannelState` consists of integers and arrays of integers only. +unsafe impl Zeroable for ChannelState {} + +static_assert!(core::mem::offset_of!(ChannelState, consumer) == 0); +static_assert!(core::mem::offset_of!(ChannelState, unk_10) == 0x10); +static_assert!(core::mem::offset_of!(ChannelState, producer) == 0x20); +static_assert!(size_of::() == 0x40); + +impl ChannelState { + pub(crate) const CONSUMER: usize = core::mem::offset_of!(Self, consumer); + pub(crate) const UNK_10: usize = core::mem::offset_of!(Self, unk_10); + /// Offset of [`Self::producer`] in the block. + pub(crate) const PRODUCER: usize = core::mem::offset_of!(Self, producer); +} + +/// Offset of the device-control channel's state in an instance's state grid. +pub(crate) const CONTROL_STATE_OFFSET: usize = WORK_CHANNELS * size_of::(); + +/// Producer value of the device-control ring of either instance at boot: one +/// opening record. +pub(crate) const OPENING_PRODUCER: u32 = 1; + +/// Position of work channel `channel` in the state grid and in the work-ring +/// array: grouped by engine, priorities in descending order. +const fn work_slot(channel: usize) -> usize { + let engine = channel % WORK_ENGINES; + let priority = channel / WORK_ENGINES; + engine * WORK_PRIORITIES + (WORK_PRIORITIES - 1 - priority) +} + +/// Offset of work channel `channel`'s state in the state grid. +pub(crate) const fn work_state_offset(channel: usize) -> usize { + work_slot(channel) * size_of::() +} + +/// Bundle offset of work channel `channel`'s ring. +pub(crate) const fn work_ring_offset(channel: usize) -> usize { + bundle::WORK_RINGS + work_slot(channel) * bundle::WORK_RING_SIZE +} + +/// Offsets in a status area, the region starting at an instance's status A +/// block, of the rings the firmware fills. +pub(crate) mod status { + /// Event ring state. + pub(crate) const EVENT_STATE: u64 = 0x40; + /// Firmware-log state. + pub(crate) const LOG_STATE: u64 = 0x80; + /// Trace ring state. + pub(crate) const TRACE_STATE: u64 = 0x240; + /// Statistics ring state. + pub(crate) const STATS_STATE: u64 = 0x280; + /// Event ring. + pub(crate) const EVENT_RING: u64 = 0x2c0; + /// Firmware-log ring. + pub(crate) const LOG_RING: u64 = 0x4ac0; + /// The single address in channel 15. + pub(crate) const UNK_2D2C0: u64 = 0x2d2c0; + /// Trace ring. + pub(crate) const TRACE_RING: u64 = 0xa6ac0; + /// Statistics ring. + pub(crate) const STATS_RING: u64 = 0xafac0; +} + +// --------------------------------------------------------------------------- +// Hardware-data bundle layout +// --------------------------------------------------------------------------- + +/// Offsets in the hardware-data bundle. +pub(crate) mod bundle { + /// Size of the bundle allocation. + pub(crate) const SIZE: usize = 0x28000; + /// The [`super::HwData`]. + pub(crate) const HW_DATA: usize = 0; + /// The five views named by the primary main configuration. + pub(crate) const VIEWS: [usize; 5] = [0x2740, 0x3380, 0x4400, 0xbc80, 0xbd00]; + /// The [`super::PowerConfig`], inside view 2. + pub(crate) const POWER_CONFIG: usize = 0x5240; + /// The [`super::HwDataAux`]. + pub(crate) const AUX: usize = 0x8000; + /// A second copy of the [`super::PowerConfigHead`], inside views 3 and 4. + pub(crate) const POWER_CONFIG_COPY: usize = 0xd240; + /// Region the firmware writes, named twice by each main configuration. + pub(crate) const FIRMWARE_AREA: usize = 0xc500; + /// The work rings, see [`super::work_ring_offset`]. + pub(crate) const WORK_RINGS: usize = 0xc5c0; + /// Size of one work ring (256 slots of 0x18 bytes). + pub(crate) const WORK_RING_SIZE: usize = 0x1800; + /// Main configuration of each instance, each followed by its + /// device-control ring. + pub(crate) const MAIN_CONFIG: [usize; 2] = [0x1e5c0, 0x22a80]; + /// Separately allocated pages the firmware finds at fixed offsets from + /// the bundle: a zeroed page named by the primary main configuration. + pub(crate) const ZERO_PAGE: usize = 0x40000; + /// The two completion rings ([`super::CompletionRing`]). + pub(crate) const COMPLETION_RINGS: [usize; 2] = [0x48000, 0x70000]; +} + +// --------------------------------------------------------------------------- +// Main configuration +// --------------------------------------------------------------------------- + +/// Record the host stages in slot 0 of each instance's device-control ring +/// before boot. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct OpeningRecord { + /// Opcode: [`Self::PRIMARY`] or [`Self::SECONDARY`]. + pub(crate) opcode: u32, + pub(crate) unk_4: [u8; 0x3c], +} + +// SAFETY: `OpeningRecord` consists of integers and arrays of integers only. +unsafe impl Zeroable for OpeningRecord {} + +static_assert!(size_of::() == 0x40); + +impl OpeningRecord { + const PRIMARY: u32 = 0x16; + const SECONDARY: u32 = 0x2a; + + /// Returns the opening record of `role`. + pub(crate) const fn new(role: Role) -> Self { + Self { + opcode: match role { + Role::Primary => Self::PRIMARY, + Role::Secondary => Self::SECONDARY, + }, + unk_4: [0; 0x3c], + } + } +} + +/// Addresses both main configurations need. +pub(crate) struct MainConfigArgs { + /// Hardware-data bundle. + pub(crate) bundle_va: u64, + /// This main configuration. + pub(crate) main_config_va: u64, + /// This instance's state grid. + pub(crate) state_grid_va: u64, + /// This instance's status A block. + pub(crate) status_a_va: u64, + /// Context-0 alias of the parameter-buffer descriptor table. + pub(crate) pb_descriptors_low_va: u64, + /// Context-0 alias of the page-pool descriptor table. + pub(crate) page_pool_descriptors_low_va: u64, +} + +/// Addresses only the primary main configuration names. +pub(crate) struct PrimaryViews { + /// The zeroed page at [`bundle::ZERO_PAGE`]. + pub(crate) zero_page_va: u64, + /// Parameter-buffer descriptor table. + pub(crate) pb_descriptors_va: u64, + /// page-pool descriptor table. + pub(crate) page_pool_descriptors_va: u64, +} + +/// Main configuration of one instance. Its device-control ring +/// (256 records) follows it directly. +#[repr(C, packed)] +pub(crate) struct MainConfig { + /// Hardware-data bundle ([`HwData`]). + pub(crate) hw_data_va: u64, + /// [`bundle::FIRMWARE_AREA`]. + pub(crate) firmware_area_va: [u64; 2], + pub(crate) unk_18: [u8; 8], + /// Channel table. + pub(crate) channels: [ChannelEntry; CHANNELS], + pub(crate) unk_240: [u8; 0x10], + /// Non-zero enables firmware log output; the driver leaves it off. + pub(crate) log_enable: u32, + /// The bundle views ([`bundle::VIEWS`]); primary only. + pub(crate) view_va: [u64; 5], + pub(crate) unk_27c: [u8; 0x54], + /// [`PrimaryViews::zero_page_va`]; primary only. + pub(crate) zero_page_va: u64, + /// Context-0 alias of the parameter-buffer descriptor table. + pub(crate) pb_descriptors_low_va: u64, + /// Parameter-buffer descriptor table; primary only. + pub(crate) pb_descriptors_va: u64, + /// Context-0 alias of the page-pool descriptor table. + pub(crate) page_pool_descriptors_low_va: u64, + /// page-pool descriptor table; primary only. + pub(crate) page_pool_descriptors_va: u64, + pub(crate) unk_2f8: [u8; 8], + /// Required: 4 for the secondary. + pub(crate) unk_300: u32, + pub(crate) unk_304: [u8; 0x28], + /// Required: 1 for the primary. + pub(crate) unk_32c: u32, + pub(crate) unk_330: [u8; 0x14], + /// Required: 0xabcd_abcd for the primary. + pub(crate) unk_344: u32, + pub(crate) unk_348: [u8; 0x98], + /// Required: 0xff for the primary. + pub(crate) unk_3e0: u32, + pub(crate) unk_3e4: [u8; 0x8d], + /// [`PowerConfig`]; secondary only. + pub(crate) power_config_va: u64, + pub(crate) unk_479: [u8; 0x47], +} + +static_assert!(core::mem::offset_of!(MainConfig, channels) == 0x20); +static_assert!(core::mem::offset_of!(MainConfig, log_enable) == 0x250); +static_assert!(core::mem::offset_of!(MainConfig, view_va) == 0x254); +static_assert!(core::mem::offset_of!(MainConfig, zero_page_va) == 0x2d0); +static_assert!(core::mem::offset_of!(MainConfig, unk_300) == 0x300); +static_assert!(core::mem::offset_of!(MainConfig, unk_3e0) == 0x3e0); +static_assert!(core::mem::offset_of!(MainConfig, power_config_va) == 0x471); +static_assert!(size_of::() == MainConfig::CONTROL_RING); +static_assert!( + bundle::MAIN_CONFIG[0] + MainConfig::CONTROL_RING + 256 * size_of::() + == bundle::MAIN_CONFIG[1] +); + +// SAFETY: `MainConfig` consists of integers and arrays of integers only. +unsafe impl Zeroable for MainConfig {} + +impl MainConfig { + /// Offset of the device-control ring from the main configuration. + pub(crate) const CONTROL_RING: usize = 0x4c0; + + fn init(&mut self, role: Role, args: &MainConfigArgs) { + clear(self); + let bundle = args.bundle_va; + let grid = args.state_grid_va; + let status_a = args.status_a_va; + self.hw_data_va = bundle + bundle::HW_DATA as u64; + self.firmware_area_va = [bundle + bundle::FIRMWARE_AREA as u64; 2]; + if role == Role::Primary { + for channel in 0..WORK_CHANNELS { + self.channels[channel] = ChannelEntry::ring( + grid + work_state_offset(channel) as u64, + bundle + work_ring_offset(channel) as u64, + ); + } + } + self.channels[CONTROL_CHANNEL] = ChannelEntry::ring( + grid + CONTROL_STATE_OFFSET as u64, + args.main_config_va + Self::CONTROL_RING as u64, + ); + self.channels[EVENT_CHANNEL] = ChannelEntry([ + status_a + status::EVENT_STATE, + status_a + status::EVENT_RING, + status_a + status::LOG_STATE, + status_a + status::LOG_RING, + ]); + self.channels[TRACE_CHANNEL] = ChannelEntry([ + status_a + status::TRACE_STATE, + status_a + status::TRACE_RING, + status_a + status::STATS_STATE, + status_a + status::STATS_RING, + ]); + self.channels[CHANNEL_15] = ChannelEntry([status_a + status::UNK_2D2C0, 0, 0, 0]); + self.pb_descriptors_low_va = args.pb_descriptors_low_va; + self.page_pool_descriptors_low_va = args.page_pool_descriptors_low_va; + } + + /// Initializes the primary main configuration. + pub(crate) fn init_primary(&mut self, args: &MainConfigArgs, views: &PrimaryViews) { + self.init(Role::Primary, args); + self.view_va = bundle::VIEWS.map(|offset| args.bundle_va + offset as u64); + self.zero_page_va = views.zero_page_va; + self.pb_descriptors_va = views.pb_descriptors_va; + self.page_pool_descriptors_va = views.page_pool_descriptors_va; + self.unk_32c = 1; + self.unk_344 = 0xabcd_abcd; + self.unk_3e0 = 0xff; + } + + /// Initializes the secondary main configuration. + pub(crate) fn init_secondary(&mut self, args: &MainConfigArgs) { + self.init(Role::Secondary, args); + self.unk_300 = 4; + self.power_config_va = args.bundle_va + bundle::POWER_CONFIG as u64; + } +} + +// --------------------------------------------------------------------------- +// Hardware data +// --------------------------------------------------------------------------- + +/// Slots in the hardware-data register map. +pub(crate) const REGISTER_MAP_SLOTS: usize = 53; + +/// One register window the firmware maps through the hardware data +/// (per-SoC configuration). +#[derive(Copy, Clone, Debug)] +pub(crate) struct RegisterWindow { + /// Register-map slot. + pub(crate) slot: usize, + /// Physical address of the window. + pub(crate) phys: u64, + /// GPU address of the window in the firmware MMIO aperture. + pub(crate) va: u64, + /// Size in bytes; need not be a multiple of the page size. + pub(crate) size: u32, + /// Value of [`RegisterMapEntry::unk_18`]. + pub(crate) unk_18: u64, + /// Value of [`RegisterMapEntry::unk_20`]. + pub(crate) unk_20: u32, +} + +impl RegisterWindow { + /// Returns the page-aligned physical address, GPU address and length the + /// window must be mapped with, or `None` if its physical and GPU + /// addresses differ within a page or the mapped span overflows. + pub(crate) fn mapping(&self) -> Option<(u64, u64, u64)> { + let mask = PAGE_SIZE as u64 - 1; + if self.phys & mask != self.va & mask { + return None; + } + let start = self.va & !mask; + let span = self + .va + .checked_add(self.size as u64)? + .checked_sub(start)? + .checked_add(mask)? + & !mask; + Some((self.phys & !mask, start, span)) + } +} + +/// Value of [`RegisterMapEntry::unk_20`] in a reserved register-map slot. +const RESERVED_SLOT_UNK_20: u32 = 2; + +/// One register-map slot of the hardware data. +#[repr(C)] +pub(crate) struct RegisterMapEntry { + /// Physical address of the window. + pub(crate) phys: u64, + /// GPU address of the window. + pub(crate) va: u64, + /// Window size in bytes. + pub(crate) total_size: u32, + /// Window size in bytes, again. + pub(crate) element_size: u32, + /// Per-window value, see [`RegisterWindow::unk_18`]. + pub(crate) unk_18: u64, + /// Per-window value; [`RESERVED_SLOT_UNK_20`] in a reserved slot. + pub(crate) unk_20: u32, + pub(crate) unk_24: u32, +} + +// SAFETY: `RegisterMapEntry` consists of integers and arrays of integers only. +unsafe impl Zeroable for RegisterMapEntry {} + +static_assert!(size_of::() == 0x28); + +impl RegisterMapEntry { + const fn window(window: &RegisterWindow) -> Self { + Self { + phys: window.phys, + va: window.va, + total_size: window.size, + element_size: window.size, + unk_18: window.unk_18, + unk_20: window.unk_20, + unk_24: 0, + } + } + + const fn reserved() -> Self { + Self { + phys: 0, + va: 0, + total_size: 0, + element_size: 0, + unk_18: 0, + unk_20: RESERVED_SLOT_UNK_20, + unk_24: 0, + } + } +} + +/// Performance states in each performance table. +pub(crate) const PERF_STATES: usize = 11; +/// Voltage lanes of a performance table. +const PERF_LANES: usize = 16; + +/// One GPU operating point, from the device tree, in performance-state +/// order. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Opp { + /// Frequency in Hz. + pub(crate) freq_hz: u64, + /// GPU core voltage in µV. + pub(crate) volt_uv: u32, +} + +/// Per-SoC parts of the performance tables that the operating points do not +/// give. +pub(crate) struct PerfConfig { + /// Operating point of each state of table A. + pub(crate) states_a: [u8; PERF_STATES], + /// Operating point of each state of table B. + pub(crate) states_b: [u8; PERF_STATES], + /// Relative performance of each state of table A, in percent. + pub(crate) relative_a: [u32; PERF_STATES], + /// Relative performance of each state of table B, in percent. + pub(crate) relative_b: [u32; PERF_STATES], + /// Lowest SRAM voltage in mV; a state's SRAM voltage is its core voltage + /// raised to this floor. + pub(crate) sram_min_mv: u32, +} + +/// One performance table of the hardware data. +#[repr(C, packed)] +pub(crate) struct PerfGroup { + /// Required: 10. + pub(crate) unk_0: u32, + /// Frequency of each state in MHz. + pub(crate) freq_mhz: [u32; PERF_STATES], + pub(crate) unk_30: [u8; 0x14], + /// GPU core voltage of each state in mV, per lane. + pub(crate) core_mv: [[u32; PERF_LANES]; PERF_STATES], + pub(crate) unk_304: [u8; 0x140], + /// SRAM voltage of each state in mV, per lane. + pub(crate) sram_mv: [[u32; PERF_LANES]; PERF_STATES], +} + +// SAFETY: `PerfGroup` consists of integers and arrays of integers only. +unsafe impl Zeroable for PerfGroup {} + +static_assert!(core::mem::offset_of!(PerfGroup, core_mv) == 0x44); +static_assert!(core::mem::offset_of!(PerfGroup, sram_mv) == 0x444); +static_assert!(size_of::() == 0x704); + +/// Frequency and voltages of one performance state. +#[derive(Copy, Clone)] +struct PerfState { + freq_mhz: u32, + core_mv: u32, + sram_mv: u32, +} + +impl PerfGroup { + /// Fills the table. `lanes` is the number of voltage lanes holding each + /// state's voltages; the remaining lanes stay zero. + fn fill(&mut self, states: &[PerfState; PERF_STATES], lanes: usize) { + self.unk_0 = 10; + for (index, state) in states.iter().enumerate() { + self.freq_mhz[index] = state.freq_mhz; + for lane in 0..lanes { + self.core_mv[index][lane] = state.core_mv; + self.sram_mv[index][lane] = state.sram_mv; + } + } + } +} + +/// Returns the performance states selected by `map` from `opps`. +fn perf_states( + opps: &[Opp], + map: &[u8; PERF_STATES], + sram_min_mv: u32, +) -> Result<[PerfState; PERF_STATES]> { + let mut states = [PerfState { + freq_mhz: 0, + core_mv: 0, + sram_mv: 0, + }; PERF_STATES]; + for (state, &index) in states.iter_mut().zip(map) { + let opp = opps.get(index as usize).ok_or(EINVAL)?; + let core_mv = opp.volt_uv / 1000; + *state = PerfState { + freq_mhz: u32::try_from(opp.freq_hz / 1_000_000)?, + core_mv, + sram_mv: core_mv.max(sram_min_mv), + }; + } + Ok(states) +} + +/// One completion ring the firmware fills. +#[repr(C)] +pub(crate) struct CompletionRingDescriptor { + /// Context-0 alias of the ring. + pub(crate) low_va: u64, + /// Firmware alias of the ring. + pub(crate) high_va: u64, + /// Entries in the ring. + pub(crate) entries: u32, + /// Size of one entry. + pub(crate) entry_size: u32, + /// Index of this descriptor in the table. + pub(crate) index: u32, + /// Firmware-owned. + pub(crate) unk_1c: u32, +} + +// SAFETY: `CompletionRingDescriptor` consists of integers and arrays of integers only. +unsafe impl Zeroable for CompletionRingDescriptor {} + +static_assert!(size_of::() == 0x20); + +/// Completion-ring descriptors in the hardware data; only +/// [`COMPLETION_RING_INDEX`] are used. +const COMPLETION_RING_DESCRIPTORS: usize = 4; +/// Descriptor index of each completion ring. +pub(crate) const COMPLETION_RING_INDEX: [u32; 2] = [0, 2]; +/// Entries in a completion ring. +const COMPLETION_RING_ENTRIES: u32 = 0x800; +/// Size of a completion-ring entry. +const COMPLETION_RING_ENTRY_SIZE: u32 = 0x40; +/// Size of a completion ring. +pub(crate) const COMPLETION_RING_SIZE: usize = + (COMPLETION_RING_ENTRIES * COMPLETION_RING_ENTRY_SIZE) as usize; + +/// The two aliases of one completion ring. +#[derive(Copy, Clone, Debug)] +pub(crate) struct CompletionRing { + /// Context-0 alias. + pub(crate) low_va: u64, + /// Firmware alias. + pub(crate) high_va: u64, +} + +/// Inputs of the hardware data. +pub(crate) struct HwDataArgs<'a> { + /// Chip identifier. + pub(crate) chip_id: u32, + /// Mapped register windows. + pub(crate) windows: &'a [RegisterWindow], + /// Register-map slots that are reserved but unmapped. + pub(crate) reserved_slots: &'a [usize], + /// GPU operating points, in performance-state order. + pub(crate) opps: &'a [Opp], + /// Per-SoC parts of the performance tables. + pub(crate) perf: &'a PerfConfig, + /// The completion rings, in [`COMPLETION_RING_INDEX`] order. + pub(crate) completion_rings: [CompletionRing; 2], + /// The zeroed QoS object ([`QOS_SIZE`] bytes). + pub(crate) qos_va: u64, +} + +/// Size of the QoS object named by the hardware data. +pub(crate) const QOS_SIZE: usize = 0xe40; + +/// The hardware data, at the base of the bundle and shared by both +/// instances. +#[repr(C, packed)] +pub(crate) struct HwData { + /// Required: 0x6f_0000_0000. + pub(crate) unk_0: u64, + /// Required: 0xffc0_0000. + pub(crate) unk_8: u64, + /// Required: 0x10_0000_0000. + pub(crate) unk_10: u64, + /// Required: 0x10_0000_0000. + pub(crate) unk_18: u64, + /// Required: 0x2ff_ffff_8000. + pub(crate) unk_20: u64, + /// Required: 0xffff_fc21_8140_0000, a GPU address in the firmware MMIO + /// aperture. + pub(crate) unk_28: u64, + pub(crate) unk_30: [u8; 0xb0], + /// Required: see [`HwData::UNK_E0`]. + pub(crate) unk_e0: [u16; 24], + pub(crate) unk_110: [u8; 0x1c8], + /// Required: see [`HwData::UNK_2D8`]. + pub(crate) unk_2d8: [u16; 12], + pub(crate) unk_2f0: [u8; 0xf0], + /// Required: see [`HwData::UNK_3E0`]. + pub(crate) unk_3e0: [u16; 24], + pub(crate) unk_410: [u8; 0x1c8], + /// Required: see [`HwData::UNK_5D8`]. + pub(crate) unk_5d8: [u16; 12], + pub(crate) unk_5f0: [u8; 0x50], + /// Register windows the firmware maps. + pub(crate) register_map: [RegisterMapEntry; REGISTER_MAP_SLOTS], + pub(crate) unk_e88: [u8; 8], + /// Chip identifier. + pub(crate) chip_id: u32, + /// Required: 1. + pub(crate) unk_e94: u32, + /// Required: 1. + pub(crate) unk_e98: u32, + pub(crate) unk_e9c: [u8; 4], + /// Required: 1. + pub(crate) unk_ea0: u32, + /// Required: 1. + pub(crate) unk_ea4: u32, + pub(crate) unk_ea8: [u8; 0x10], + /// Required: 1. + pub(crate) unk_eb8: u32, + pub(crate) unk_ebc: [u8; 4], + /// Required: 1. + pub(crate) unk_ec0: u32, + pub(crate) unk_ec4: [u8; 4], + /// Required: 1. + pub(crate) unk_ec8: u32, + pub(crate) unk_ecc: [u8; 4], + /// Required: 0x5dc0 (24000). + pub(crate) unk_ed0: u32, + /// Required: 1. + pub(crate) unk_ed4: u32, + /// Required: 0x10 (16). + pub(crate) unk_ed8: u32, + pub(crate) unk_edc: [u8; 4], + /// Required: 1 in every entry. + pub(crate) unk_ee0: [u32; 3], + pub(crate) unk_eec: [u8; 0x18], + /// Required: 0x1f (31). + pub(crate) unk_f04: u32, + pub(crate) unk_f08: [u8; 0x1c], + /// Required: 4. + pub(crate) unk_f24: u32, + pub(crate) unk_f28: [u8; 0xc], + /// Required: 1. + pub(crate) unk_f34: u32, + /// Required: 1. + pub(crate) unk_f38: u32, + pub(crate) unk_f3c: [u8; 0x10], + /// Required: 0x31 (49). + pub(crate) unk_f4c: u32, + pub(crate) unk_f50: [u8; 0x1c], + /// Required: 1. + pub(crate) unk_f6c: u32, + pub(crate) unk_f70: [u8; 0x18], + /// Required: 6. + pub(crate) unk_f88: u32, + /// Required: 1. + pub(crate) unk_f8c: u32, + pub(crate) unk_f90: [u8; 0x1c], + /// Required: 1. + pub(crate) unk_fac: u32, + pub(crate) unk_fb0: [u8; 8], + /// Required: 0x1e (30). + pub(crate) unk_fb8: u32, + /// Required: 4. + pub(crate) unk_fbc: u32, + /// Required: 6. + pub(crate) unk_fc0: u32, + /// Performance table A. + pub(crate) perf_a: PerfGroup, + pub(crate) unk_16c8: [u8; 0x140], + /// Frequency of each state of table B in MHz. + pub(crate) perf_b_freq_mhz: [u32; PERF_STATES], + pub(crate) unk_1834: [u8; 0x14], + /// Required: 0x3f82_8f5c (1.02 as an IEEE-754 single) in every entry. + pub(crate) unk_1848: [u32; 11], + pub(crate) unk_1874: [u8; 0x54], + /// Relative performance of each state of table A, in percent. + pub(crate) perf_a_relative: [u32; PERF_STATES], + pub(crate) unk_18f4: [u8; 0x14], + /// Relative performance of each state of table B, in percent. + pub(crate) perf_b_relative: [u32; PERF_STATES], + pub(crate) unk_1934: [u8; 0x94], + /// Operating point of each state of table A. + pub(crate) perf_a_states: [u32; PERF_STATES], + pub(crate) unk_19f4: [u8; 0x14], + /// Operating point of each state of table B. + pub(crate) perf_b_states: [u32; PERF_STATES], + pub(crate) unk_1a34: [u8; 0x2a4], + /// Performance table B. Each voltage is stored in lane 0 only. + pub(crate) perf_b: PerfGroup, + pub(crate) unk_23dc: [u8; 0x168], + /// Required: 4. + pub(crate) unk_2544: u32, + /// Required: 6. + pub(crate) unk_2548: u32, + /// Required: 3. + pub(crate) unk_254c: u32, + /// Required: 7. + pub(crate) unk_2550: u32, + /// Required: 7. + pub(crate) unk_2554: u32, + pub(crate) unk_2558: [u8; 0x10], + /// Required: 5. + pub(crate) unk_2568: u32, + pub(crate) unk_256c: [u8; 4], + /// Required: 1. + pub(crate) unk_2570: u32, + pub(crate) unk_2574: [u8; 0x1a], + /// Required: [`WORK_CHANNELS`]. + pub(crate) unk_258e: u16, + /// Required: 0, 1, 1, 1. + pub(crate) unk_2590: [u32; 4], + /// Required: 0, 1, 1, 1. + pub(crate) unk_25a0: [u32; 4], + pub(crate) unk_25b0: [u8; 4], + /// Required: 0xffff_ffff in every entry, one per work channel. + pub(crate) unk_25b4: [u32; WORK_CHANNELS], + pub(crate) unk_25e4: [u8; 0x10], + /// Required: 1. + pub(crate) unk_25f4: u32, + pub(crate) unk_25f8: [u8; 8], + /// Required: 1. + pub(crate) unk_2600: u32, + pub(crate) unk_2604: [u8; 4], + /// Required: 1. + pub(crate) unk_2608: u32, + pub(crate) unk_260c: [u8; 4], + /// Required: 0x100. + pub(crate) unk_2610: u32, + /// Completion-ring descriptors. + pub(crate) completion_rings: [CompletionRingDescriptor; COMPLETION_RING_DESCRIPTORS], + pub(crate) unk_2694: [u8; 0x18], + /// QoS object ([`QOS_SIZE`] bytes, zeroed). + pub(crate) qos_va: u64, + /// Required: 1. + pub(crate) unk_26b4: u32, + /// Required: 1. + pub(crate) unk_26b8: u32, + pub(crate) unk_26bc: [u8; 0x2c], + /// Required: 1. + pub(crate) unk_26e8: u32, + pub(crate) unk_26ec: [u8; 0x16ac], + /// Required: 0xffff_ffff. + pub(crate) unk_3d98: u32, + pub(crate) unk_3d9c: [u8; 0x14], + /// Required: 0xffff_ffff. + pub(crate) unk_3db0: u32, +} + +static_assert!(core::mem::offset_of!(HwData, register_map) == 0x640); +static_assert!(core::mem::offset_of!(HwData, chip_id) == 0xe90); +static_assert!(core::mem::offset_of!(HwData, perf_a) == 0xfc4); +static_assert!(core::mem::offset_of!(HwData, perf_b_freq_mhz) == 0x1808); +static_assert!(core::mem::offset_of!(HwData, perf_b) == 0x1cd8); +static_assert!(core::mem::offset_of!(HwData, unk_258e) == 0x258e); +static_assert!(core::mem::offset_of!(HwData, completion_rings) == 0x2614); +static_assert!(core::mem::offset_of!(HwData, qos_va) == 0x26ac); +static_assert!(size_of::() == 0x3db4); +static_assert!(bundle::HW_DATA + size_of::() <= bundle::VIEWS[2]); + +// SAFETY: `HwData` consists of integers and arrays of integers only. +unsafe impl Zeroable for HwData {} + +impl HwData { + const UNK_E0: [u16; 24] = [ + 0x2008, 0, 0, 0, 0, 0x2008, 0, 0, 0, 0, 0x2008, 0, 0x24cb, 0, 0x2cfa, 0xc98a, 0x24cb, + 0xf4f6, 0xe917, 0x1877, 0x24cb, 0x38d9, 0, 0xbdab, + ]; + const UNK_2D8: [u16; 12] = [ + 0x2000, 0xfff8, 0x2cdb, 0xd32d, 0x2000, 0xf500, 0xe926, 0x21da, 0x2000, 0x38b6, 8, 0xc742, + ]; + const UNK_3E0: [u16; 24] = [ + 0x7fe0, 0, 0, 0, 0, 0x7fe0, 0, 0, 0, 0, 0x7fe0, 0, 0x8000, 0, 0, 0, 0, 0x8000, 0, 0, 0, 0, + 0x8000, 0, + ]; + const UNK_5D8: [u16; 12] = [ + 0x2645, 0x4b23, 0xe98, 0, 0xea5f, 0xd5a2, 0x3fff, 0x4000, 0x4000, 0xca5e, 0xf5a2, 0x4000, + ]; + + /// Initializes the hardware data. + pub(crate) fn init(&mut self, args: &HwDataArgs<'_>) -> Result { + clear(self); + self.unk_0 = 0x6f_0000_0000; + self.unk_8 = 0xffc0_0000; + self.unk_10 = 0x10_0000_0000; + self.unk_18 = 0x10_0000_0000; + self.unk_20 = 0x2ff_ffff_8000; + self.unk_28 = 0xffff_fc21_8140_0000; + self.unk_e0 = Self::UNK_E0; + self.unk_2d8 = Self::UNK_2D8; + self.unk_3e0 = Self::UNK_3E0; + self.unk_5d8 = Self::UNK_5D8; + + let mut used = [false; REGISTER_MAP_SLOTS]; + let mut claim = |slot: usize| -> Result { + match used.get_mut(slot) { + Some(used) if !*used => { + *used = true; + Ok(()) + } + _ => Err(EINVAL), + } + }; + for window in args.windows { + claim(window.slot)?; + self.register_map[window.slot] = RegisterMapEntry::window(window); + } + for &slot in args.reserved_slots { + claim(slot)?; + self.register_map[slot] = RegisterMapEntry::reserved(); + } + + self.chip_id = args.chip_id; + self.unk_e94 = 1; + self.unk_e98 = 1; + self.unk_ea0 = 1; + self.unk_ea4 = 1; + self.unk_eb8 = 1; + self.unk_ec0 = 1; + self.unk_ec8 = 1; + self.unk_ed0 = 0x5dc0; + self.unk_ed4 = 1; + self.unk_ed8 = 0x10; + self.unk_ee0 = [1; 3]; + self.unk_f04 = 0x1f; + self.unk_f24 = 4; + self.unk_f34 = 1; + self.unk_f38 = 1; + self.unk_f4c = 0x31; + self.unk_f6c = 1; + self.unk_f88 = 6; + self.unk_f8c = 1; + self.unk_fac = 1; + self.unk_fb8 = 0x1e; + self.unk_fbc = 4; + self.unk_fc0 = 6; + + let perf = args.perf; + let states_a = perf_states(args.opps, &perf.states_a, perf.sram_min_mv)?; + let states_b = perf_states(args.opps, &perf.states_b, perf.sram_min_mv)?; + self.perf_a.fill(&states_a, PERF_LANES); + self.perf_b.fill(&states_b, 1); + for (index, state) in states_b.iter().enumerate() { + self.perf_b_freq_mhz[index] = state.freq_mhz; + self.perf_a_states[index] = perf.states_a[index].into(); + self.perf_b_states[index] = perf.states_b[index].into(); + } + self.perf_a_relative = perf.relative_a; + self.perf_b_relative = perf.relative_b; + self.unk_1848 = [0x3f82_8f5c; 11]; + self.unk_2544 = 4; + self.unk_2548 = 6; + self.unk_254c = 3; + self.unk_2550 = 7; + self.unk_2554 = 7; + self.unk_2568 = 5; + self.unk_2570 = 1; + + self.unk_258e = WORK_CHANNELS as u16; + self.unk_2590 = [0, 1, 1, 1]; + self.unk_25a0 = [0, 1, 1, 1]; + self.unk_25b4 = [u32::MAX; WORK_CHANNELS]; + self.unk_25f4 = 1; + self.unk_2600 = 1; + self.unk_2608 = 1; + self.unk_2610 = 0x100; + for (ring, index) in args.completion_rings.iter().zip(COMPLETION_RING_INDEX) { + self.completion_rings[index as usize] = CompletionRingDescriptor { + low_va: ring.low_va, + high_va: ring.high_va, + entries: COMPLETION_RING_ENTRIES, + entry_size: COMPLETION_RING_ENTRY_SIZE, + index, + unk_1c: 0, + }; + } + self.qos_va = args.qos_va; + self.unk_26b4 = 1; + self.unk_26b8 = 1; + self.unk_26e8 = 1; + self.unk_3d98 = 0xffff_ffff; + self.unk_3db0 = 0xffff_ffff; + Ok(()) + } +} + +// --------------------------------------------------------------------------- +// Regions A and C +// --------------------------------------------------------------------------- + +/// Region A: a zeroed page both instances read. +#[repr(C)] +pub(crate) struct RegionA { + pub(crate) unk_0: [u8; 0x4000], +} + +// SAFETY: `RegionA` is an array of integers. +unsafe impl Zeroable for RegionA {} + +/// Engines whose work the firmware's context-switch timer may interrupt +/// (bit 0 TA, bit 1 3D, bit 2 compute). No engine is periodically resumed. +pub(crate) const CONTEXT_SWITCH_TIMER_MASK: u32 = 0; +/// Period of the context-switch timer, in units of the firmware's 1 ms +/// scheduling pass. +pub(crate) const CONTEXT_SWITCH_TIMER_MULTIPLIER: u8 = 1; + +/// Region C: firmware policy settings shared by both instances. +/// +/// Three fields are live. The host updates [`Self::effort`] while it submits +/// work: it stores zero to [`Self::unk_e8`], then the effort (0x10000, 1.0 +/// in 16.16 fixed point, while work is streaming; zero once the GPU has been +/// idle for 16 ms), each with one aligned 32-bit store, and then cleans the +/// cache line to the point of coherency. [`Self::power_assert_tally`] is +/// unaligned: for every device-control record whose opcode asserts GPU +/// power, the host adds one to it after writing the record and before +/// advancing the ring producer (with a barrier before the producer store). +/// The firmware powers the GPU cores up while the tally differs from its +/// own count of drained records. +#[repr(C, packed)] +pub(crate) struct RegionC { + /// Engines the context-switch timer applies to ([`CONTEXT_SWITCH_TIMER_MASK`]). + pub(crate) context_switch_timer_mask: u32, + pub(crate) unk_4: [u8; 0x20], + /// Required: 0xbb8 (3000). + pub(crate) unk_24: u32, + pub(crate) unk_28: [u8; 8], + /// Non-zero lets the firmware power the GPU cores off when idle. + pub(crate) idle_power_off: u32, + /// Required: 1. + pub(crate) unk_34: u32, + pub(crate) unk_38: [u8; 4], + /// Required: 0x78 (120). + pub(crate) unk_3c: u32, + pub(crate) unk_40: [u8; 0x14], + /// Required: 0xffff. + pub(crate) unk_54: u16, + /// Required: 0x28 (40). + pub(crate) unk_56: u16, + /// Required: 0xffff. + pub(crate) unk_58: u16, + pub(crate) unk_5a: [u8; 4], + /// Required: 1. + pub(crate) unk_5e: u32, + /// Required: 1. + pub(crate) unk_62: u32, + /// Required: 3. + pub(crate) unk_66: u32, + /// Period of the context-switch timer ([`CONTEXT_SWITCH_TIMER_MULTIPLIER`]). + pub(crate) context_switch_timer_multiplier: u8, + pub(crate) unk_6b: [u8; 0xd], + /// Required: 1 in every entry. + pub(crate) unk_78: [u32; 3], + /// Required: 0x254e (9550). + pub(crate) unk_84: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_88: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_8c: u32, + pub(crate) unk_90: [u8; 8], + /// Required: 0x3e8 (1000). + pub(crate) unk_98: u32, + /// Required: 0x3000_001d. + pub(crate) unk_9c: u32, + /// Required: 0x3100_0002. + pub(crate) unk_a0: u32, + pub(crate) unk_a4: [u8; 0x40], + /// Performance-floor request (live, see the type documentation). + pub(crate) effort: u32, + /// Live; the host stores zero here whenever it stores [`Self::effort`]. + pub(crate) unk_e8: u32, + pub(crate) unk_ec: [u8; 0x6d8], + /// Required: 0x7d0 (2000). + pub(crate) unk_7c4: u32, + /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). + pub(crate) unk_7c8: u32, + /// Required: 0x3f4c_cccd (0.8 as an IEEE-754 single). + pub(crate) unk_7cc: u32, + /// Required: 0x3e4c_cccd (0.2 as an IEEE-754 single). + pub(crate) unk_7d0: u32, + /// Required: 0x3f66_6666 (0.9 as an IEEE-754 single). + pub(crate) unk_7d4: u32, + /// Required: 0x3dcc_cccd (0.1 as an IEEE-754 single). + pub(crate) unk_7d8: u32, + /// Required: 0x3e80_0000 (0.25 as an IEEE-754 single). + pub(crate) unk_7dc: u32, + /// Required: 0x3f19_999a (0.6 as an IEEE-754 single). + pub(crate) unk_7e0: u32, + /// Required: 0x3f66_6666 (0.9 as an IEEE-754 single). + pub(crate) unk_7e4: u32, + /// Required: 6. + pub(crate) unk_7e8: u32, + /// Required: 1. + pub(crate) unk_7ec: u32, + /// Required: 1. + pub(crate) unk_7f0: u32, + /// Required: 0x64 (100). + pub(crate) unk_7f4: u32, + /// Required: 0x64 (100). + pub(crate) unk_7f8: u32, + pub(crate) unk_7fc: [u8; 0x19c], + /// Required: 0x28 (40). + pub(crate) unk_998: u32, + /// Required: 0xa (10). + pub(crate) unk_99c: u32, + /// Required: 0xfa (250). + pub(crate) unk_9a0: u32, + /// Required: 1. + pub(crate) unk_9a4: u32, + pub(crate) unk_9a8: [u8; 0x10], + /// Required: 2. + pub(crate) unk_9b8: u32, + /// Idle time before the firmware powers the GPU cores off. + pub(crate) idle_power_off_delay: u32, + /// Required: 5. + pub(crate) unk_9c0: u32, + pub(crate) unk_9c4: [u8; 4], + /// Required: 0x28 (40). + pub(crate) unk_9c8: u32, + /// Compute progress watchdog threshold (zero disables the watchdog). + pub(crate) compute_watchdog: u32, + /// Required: 1. + pub(crate) unk_9d0: u32, + pub(crate) unk_9d4: [u8; 0x449], + /// Required: 1. + pub(crate) unk_e1d: u32, + pub(crate) unk_e21: [u8; 0x18], + /// Required: 3. + pub(crate) unk_e39: u32, + /// Required: 1. + pub(crate) unk_e3d: u32, + pub(crate) unk_e41: [u8; 0x10], + /// Host power-assert tally (live, unaligned; see the type documentation). + pub(crate) power_assert_tally: u32, + pub(crate) unk_e55: [u8; 0x1ab], +} + +static_assert!(core::mem::offset_of!(RegionC, idle_power_off) == 0x30); +static_assert!(core::mem::offset_of!(RegionC, context_switch_timer_multiplier) == 0x6a); +static_assert!(core::mem::offset_of!(RegionC, effort) == 0xe4); +static_assert!(core::mem::offset_of!(RegionC, idle_power_off_delay) == 0x9bc); +static_assert!(core::mem::offset_of!(RegionC, compute_watchdog) == 0x9cc); +static_assert!(core::mem::offset_of!(RegionC, power_assert_tally) == 0xe51); +static_assert!(size_of::() == 0x1000); + +// SAFETY: `RegionC` consists of integers and arrays of integers only. +unsafe impl Zeroable for RegionC {} + +impl RegionC { + /// Offset of [`Self::effort`]. + pub(crate) const EFFORT: usize = core::mem::offset_of!(Self, effort); + /// Offset of [`Self::unk_e8`]. + pub(crate) const UNK_E8: usize = core::mem::offset_of!(Self, unk_e8); + /// Offset of [`Self::power_assert_tally`]. + pub(crate) const POWER_ASSERT_TALLY: usize = core::mem::offset_of!(Self, power_assert_tally); + + /// Initializes region C. + pub(crate) fn init(&mut self) { + clear(self); + self.context_switch_timer_mask = CONTEXT_SWITCH_TIMER_MASK; + self.unk_24 = 0xbb8; + self.idle_power_off = 1; + self.unk_34 = 1; + self.unk_3c = 0x78; + self.unk_54 = 0xffff; + self.unk_56 = 0x28; + self.unk_58 = 0xffff; + self.unk_5e = 1; + self.unk_62 = 1; + self.unk_66 = 3; + self.context_switch_timer_multiplier = CONTEXT_SWITCH_TIMER_MULTIPLIER; + self.unk_78 = [1; 3]; + self.unk_84 = 0x254e; + self.unk_88 = 0x3e8; + self.unk_8c = 0x3e8; + self.unk_98 = 0x3e8; + self.unk_9c = 0x3000_001d; + self.unk_a0 = 0x3100_0002; + self.unk_7c4 = 0x7d0; + self.unk_7c8 = 0x3f80_0000; + self.unk_7cc = 0x3f4c_cccd; + self.unk_7d0 = 0x3e4c_cccd; + self.unk_7d4 = 0x3f66_6666; + self.unk_7d8 = 0x3dcc_cccd; + self.unk_7dc = 0x3e80_0000; + self.unk_7e0 = 0x3f19_999a; + self.unk_7e4 = 0x3f66_6666; + self.unk_7e8 = 6; + self.unk_7ec = 1; + self.unk_7f0 = 1; + self.unk_7f4 = 0x64; + self.unk_7f8 = 0x64; + self.unk_998 = 0x28; + self.unk_99c = 0xa; + self.unk_9a0 = 0xfa; + self.unk_9a4 = 1; + self.unk_9b8 = 2; + self.idle_power_off_delay = 40; + self.unk_9c0 = 5; + self.unk_9c8 = 0x28; + self.compute_watchdog = 50; + self.unk_9d0 = 1; + self.unk_e1d = 1; + self.unk_e39 = 3; + self.unk_e3d = 1; + } +} + +// --------------------------------------------------------------------------- +// Status objects +// --------------------------------------------------------------------------- + +/// Status block of an instance (its status A, and the head of the primary +/// status B). +/// +/// At creation only `+0x04` is one. Firmware later owns the status words at +/// `+0x10` and `+0x14`; the host observes each with an aligned 32-bit read. +/// The event-ring state begins at status A `+0x40`, inside this boot view: +/// clearing or copying the whole status block after publication is forbidden. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct StatusBlock { + pub(crate) unk_0: u32, + /// Required: 1. + pub(crate) unk_4: u32, + pub(crate) unk_8: [u8; 0x78], +} + +// SAFETY: `StatusBlock` consists of integers and arrays of integers only. +unsafe impl Zeroable for StatusBlock {} + +static_assert!(size_of::() == 0x80); + +impl StatusBlock { + /// Returns a status block in its boot state. + pub(crate) const fn new() -> Self { + Self { + unk_0: 0, + unk_4: 1, + unk_8: [0; 0x78], + } + } +} + +/// Power state the primary status B carries for the secondary instance, +/// named by [`SecondaryRoot::power_state_va`]. +#[repr(C, packed)] +pub(crate) struct PowerState { + pub(crate) unk_0: [u8; 0x10], + /// Required: 0xf (15). + pub(crate) unk_10: u32, + /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). + pub(crate) unk_14: u32, + /// Required: 0x4088_0000 (4.25 as an IEEE-754 single). + pub(crate) unk_18: u32, + /// Required: 0x28 (40). + pub(crate) unk_1c: u32, + /// Required: 1. + pub(crate) unk_20: u32, + pub(crate) unk_24: [u8; 0xa0], + /// Non-zero asks the secondary to reload the limiter settings that + /// follow into the power configuration; zero at boot. + pub(crate) temp_update: u32, + /// Target temperature of the limiter ([`TEMP_TARGET`]). + pub(crate) temp_target: u32, + /// Release temperature of the limiter ([`TEMP_RELEASE`]). + pub(crate) temp_release: u32, + pub(crate) unk_d0: [u8; 8], + /// Proportional gain of the limiter ([`TEMP_KP`]). + pub(crate) temp_kp: u32, + /// Integral gain of the limiter ([`TEMP_KI`]). + pub(crate) temp_ki: u32, + pub(crate) unk_e0: [u8; 0x10], + /// Required: 0x26ac (9900). + pub(crate) unk_f0: u32, + /// Required: 0xc8 (200). + pub(crate) unk_f4: u32, + pub(crate) unk_f8: [u8; 0x10], + /// Required: 0x42c8_0000 (100.0 as an IEEE-754 single). + pub(crate) unk_108: u32, + /// Required: 0x43c8_0000 (400.0 as an IEEE-754 single). + pub(crate) unk_10c: u32, + /// Required: 0xc8 (200). + pub(crate) unk_110: u32, + pub(crate) unk_114: [u8; 0x14], + /// Required: 1. + pub(crate) unk_128: u32, + /// Required: 0x2616 (9750). + pub(crate) unk_12c: u32, + /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). + pub(crate) unk_130: u32, + /// Required: 0x40cc_cccd (6.4 as an IEEE-754 single). + pub(crate) unk_134: u32, + pub(crate) unk_138: [u8; 0x14], + /// Required: 1. + pub(crate) unk_14c: u32, + pub(crate) unk_150: [u8; 0x70], + /// Required: 1. + pub(crate) unk_1c0: u32, + /// Required: 1. + pub(crate) unk_1c4: u32, + /// Required: 4. + pub(crate) unk_1c8: u32, + /// Required: 1 in every entry. + pub(crate) unk_1cc: [u32; 4], + pub(crate) unk_1dc: [u8; 0x308], + /// Required: 1. + pub(crate) unk_4e4: u32, + /// Required: 0x1f4 (500). + pub(crate) unk_4e8: u32, + /// Required: 6. + pub(crate) unk_4ec: u32, + /// Required: 0x30d4 (12500) in every entry. + pub(crate) unk_4f0: [u32; 6], + pub(crate) unk_508: [u8; 0x14], + /// Required: 0x30d4 (12500) in every entry. + pub(crate) unk_51c: [u32; 5], + /// Required: 1. + pub(crate) unk_530: u32, + pub(crate) unk_534: [u8; 0x58], + /// Required: 6. + pub(crate) unk_58c: u32, + /// Required: 0xdac (3500). + pub(crate) unk_590: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_594: u32, + /// Required: 0xbb8 (3000). + pub(crate) unk_598: u32, + /// Required: 0x64 (100). + pub(crate) unk_59c: u32, + pub(crate) unk_5a0: [u8; 0x2c], + /// Required: 1. + pub(crate) unk_5cc: u32, + /// Required: 4. + pub(crate) unk_5d0: u32, + pub(crate) unk_5d4: [u8; 8], + /// Required: 0x1f40 (8000). + pub(crate) unk_5dc: u32, + /// Required: 0xc8 (200). + pub(crate) unk_5e0: u32, + /// Required: 0xfa0 (4000). + pub(crate) unk_5e4: u32, + /// Required: 0xc8 (200). + pub(crate) unk_5e8: u32, + /// Required: 0x7d0 (2000). + pub(crate) unk_5ec: u32, + /// Required: 0xc8 (200). + pub(crate) unk_5f0: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_5f4: u32, + /// Required: 0xc8 (200). + pub(crate) unk_5f8: u32, + pub(crate) unk_5fc: [u8; 8], + /// Required: 0x4170_0000 (15.0 as an IEEE-754 single). + pub(crate) unk_604: u32, + /// Required: 0x40a0_0000 (5.0 as an IEEE-754 single). + pub(crate) unk_608: u32, + /// Required: 0x20 (32). + pub(crate) unk_60c: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_610: u32, + pub(crate) unk_614: [u8; 0xc], + /// Required: 0x384 (900). + pub(crate) unk_620: u32, + pub(crate) unk_624: [u8; 0x9c], +} + +// SAFETY: `PowerState` consists of integers and arrays of integers only. +unsafe impl Zeroable for PowerState {} + +static_assert!(size_of::() == 0x6c0); + +impl PowerState { + fn init(&mut self) { + self.unk_10 = 0xf; + self.unk_14 = 0x3f00_0000; + self.unk_18 = 0x4088_0000; + self.unk_1c = 0x28; + self.unk_20 = 1; + self.temp_target = TEMP_TARGET; + self.temp_release = TEMP_RELEASE; + self.temp_kp = TEMP_KP; + self.temp_ki = TEMP_KI; + self.unk_f0 = 0x26ac; + self.unk_f4 = 0xc8; + self.unk_108 = 0x42c8_0000; + self.unk_10c = 0x43c8_0000; + self.unk_110 = 0xc8; + self.unk_128 = 1; + self.unk_12c = 0x2616; + self.unk_130 = 0x3f00_0000; + self.unk_134 = 0x40cc_cccd; + self.unk_14c = 1; + self.unk_1c0 = 1; + self.unk_1c4 = 1; + self.unk_1c8 = 4; + self.unk_1cc = [1; 4]; + self.unk_4e4 = 1; + self.unk_4e8 = 0x1f4; + self.unk_4ec = 6; + self.unk_4f0 = [0x30d4; 6]; + self.unk_51c = [0x30d4; 5]; + self.unk_530 = 1; + self.unk_58c = 6; + self.unk_590 = 0xdac; + self.unk_594 = 0x3e8; + self.unk_598 = 0xbb8; + self.unk_59c = 0x64; + self.unk_5cc = 1; + self.unk_5d0 = 4; + self.unk_5dc = 0x1f40; + self.unk_5e0 = 0xc8; + self.unk_5e4 = 0xfa0; + self.unk_5e8 = 0xc8; + self.unk_5ec = 0x7d0; + self.unk_5f0 = 0xc8; + self.unk_5f4 = 0x3e8; + self.unk_5f8 = 0xc8; + self.unk_604 = 0x4170_0000; + self.unk_608 = 0x40a0_0000; + self.unk_60c = 0x20; + self.unk_610 = 0x3e8; + self.unk_620 = 0x384; + } +} + +/// Size of the firmware-control page named by the primary status B. +pub(crate) const FWCTL_SIZE: usize = 0x4000; +/// Offset of the firmware-control ring in its page. +const FWCTL_RING: u64 = 0x40; + +/// One firmware-written recovery-info entry. The host reads each word separately after the +/// recovery handshake reaches state 3; it never copies the live table as a structure. +#[repr(C)] +pub(crate) struct RecoveryInfoEntry { + pub(crate) flags: u32, + pub(crate) stamp: u32, +} +// SAFETY: Both fields are integers. +unsafe impl Zeroable for RecoveryInfoEntry {} +static_assert!(size_of::() == 8); + +/// Status B of the primary instance, directly after its state grid. +#[repr(C, packed)] +pub(crate) struct PrimaryStatusB { + /// Status block. + pub(crate) status: StatusBlock, + pub(crate) unk_80: [u8; 0x4038], + pub(crate) recovery_info: [RecoveryInfoEntry; 256], + pub(crate) unk_48b8: [u8; 0x20], + pub(crate) recovery_slot_mask: u64, + /// Firmware-control state, at the start of the firmware-control page. + pub(crate) fwctl_state_va: u64, + /// Firmware-control ring, in the firmware-control page. + pub(crate) fwctl_ring_va: u64, + /// Firmware-written epoch, read using an aligned 64-bit load. + pub(crate) recovery_epoch: u64, + pub(crate) unk_48f8: [u8; 8], + /// Firmware writes 1 and 3; host writes 2 and 0 with full barriers around each 32-bit store. + pub(crate) recovery_state: u32, + pub(crate) unk_4904: [u8; 0xc], + /// Must be zero for an event-backed restart; read using an aligned 32-bit load. + pub(crate) host_recovery: u32, + pub(crate) unk_4914: [u8; 0x9b20], + /// Required: 1. + pub(crate) unk_e434: u32, + pub(crate) unk_e438: [u8; 8], + /// Power state, see [`PowerState`]. + pub(crate) power: PowerState, +} + +static_assert!(core::mem::offset_of!(PrimaryStatusB, recovery_info) == 0x40b8); +static_assert!(core::mem::offset_of!(PrimaryStatusB, recovery_slot_mask) == 0x48d8); +static_assert!(core::mem::offset_of!(PrimaryStatusB, recovery_epoch) == 0x48f0); +static_assert!(core::mem::offset_of!(PrimaryStatusB, recovery_state) == 0x4900); +static_assert!(core::mem::offset_of!(PrimaryStatusB, host_recovery) == 0x4910); +static_assert!(core::mem::offset_of!(PrimaryStatusB, fwctl_state_va) == 0x48e0); +static_assert!(core::mem::offset_of!(PrimaryStatusB, unk_e434) == 0xe434); +static_assert!(core::mem::offset_of!(PrimaryStatusB, power) == PrimaryStatusB::POWER_STATE); +static_assert!(size_of::() == 0xeb00); + +// SAFETY: `PrimaryStatusB` consists of integers and arrays of integers only. +unsafe impl Zeroable for PrimaryStatusB {} + +impl PrimaryStatusB { + pub(crate) const RECOVERY_SLOT_MASK: usize = core::mem::offset_of!(Self, recovery_slot_mask); + pub(crate) const RECOVERY_INFO: usize = core::mem::offset_of!(Self, recovery_info); + pub(crate) const RECOVERY_STATE: usize = core::mem::offset_of!(Self, recovery_state); + pub(crate) const HOST_RECOVERY: usize = core::mem::offset_of!(Self, host_recovery); + + /// Offset of the [`PowerState`]. + pub(crate) const POWER_STATE: usize = 0xe440; + + /// Initializes the primary status B; `fwctl_va` is the zeroed + /// firmware-control page ([`FWCTL_SIZE`] bytes). + pub(crate) fn init(&mut self, fwctl_va: u64) { + clear(self); + self.status = StatusBlock::new(); + self.fwctl_state_va = fwctl_va; + self.fwctl_ring_va = fwctl_va + FWCTL_RING; + self.unk_e434 = 1; + self.power.init(); + } +} + +/// Configuration of the secondary instance, named by +/// [`SecondaryRoot::config_va`]. +#[repr(C)] +pub(crate) struct SecondaryConfig { + pub(crate) unk_0: [u8; 0x14], + /// Required: 1. + pub(crate) unk_14: u32, + pub(crate) unk_18: [u8; 0x14], + /// Required: 1. + pub(crate) unk_2c: u32, + /// Required: 1. + pub(crate) unk_30: u32, + pub(crate) unk_34: [u8; 0xc], + /// Required: 1. + pub(crate) unk_40: u32, + pub(crate) unk_44: [u8; 8], + /// Required: 1. + pub(crate) unk_4c: u32, + /// Required: 0x186a (6250). + pub(crate) unk_50: u32, + pub(crate) unk_54: [u8; 0x2c], +} + +static_assert!(size_of::() == 0x80); + +// SAFETY: `SecondaryConfig` consists of integers and arrays of integers only. +unsafe impl Zeroable for SecondaryConfig {} + +impl SecondaryConfig { + /// Initializes the secondary configuration. + pub(crate) fn init(&mut self) { + clear(self); + self.unk_14 = 1; + self.unk_2c = 1; + self.unk_30 = 1; + self.unk_40 = 1; + self.unk_4c = 1; + self.unk_50 = 0x186a; + } +} + +/// Offsets in the private cluster. +pub(crate) mod private { + /// Size of the private cluster. + pub(crate) const SIZE: usize = 0x178000; + /// The primary state grid. + pub(crate) const PRIMARY_STATE: usize = 0; + /// The primary status A block. + pub(crate) const PRIMARY_STATUS_A: usize = 0xee40; + /// The secondary status A block. + pub(crate) const SECONDARY_STATUS_A: usize = 0xc3100; + /// The secondary state grid. + pub(crate) const SECONDARY_STATE: usize = 0x1770c0; + /// [`super::PrimaryStatusB`] or [`super::SecondaryConfig`], from an + /// instance's state grid. + pub(crate) const STATUS_B: usize = 0x340; + /// Secondary state named by [`super::HwDataAux`], from the secondary + /// state grid. + pub(crate) const SECONDARY_STATUS: usize = 0x3c0; +} + +static_assert!(private::STATUS_B == CONTROL_STATE_OFFSET + size_of::()); +static_assert!( + private::PRIMARY_STATE + private::STATUS_B + size_of::() + == private::PRIMARY_STATUS_A +); +static_assert!(private::STATUS_B + size_of::() == private::SECONDARY_STATUS); + +// --------------------------------------------------------------------------- +// Power configuration and auxiliary hardware data (bundle) +// --------------------------------------------------------------------------- + +/// Die temperature sensors read by the die temperature limiter. +const TEMP_SENSOR_MASK: u64 = 0x82a; +/// Die temperature the limiter holds the GPU below, in 0.01 °C. +const TEMP_TARGET: u32 = 11_000; +/// Die temperature below which the limiter releases, in 0.01 °C. +const TEMP_RELEASE: u32 = 8_000; +/// Proportional gain of the limiter: 20.34 as an IEEE-754 single. +const TEMP_KP: u32 = 0x41a2_b852; +/// Integral gain of the limiter, per control period: 9.376 as an IEEE-754 +/// single. +const TEMP_KI: u32 = 0x4116_0419; + +/// Leading part of the [`PowerConfig`]. The host writes it twice: in the +/// power configuration and at [`bundle::POWER_CONFIG_COPY`], where the copy +/// ends with the views it lies in. +#[repr(C, packed)] +pub(crate) struct PowerConfigHead { + pub(crate) unk_0: [u8; 4], + /// Required: 0x5_dc00 (384000). + pub(crate) unk_4: u32, + /// Required: 0x5_dc00 (384000). + pub(crate) unk_8: u32, + pub(crate) unk_c: [u8; 4], + /// Required: 4. + pub(crate) unk_10: u32, + /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). + pub(crate) unk_14: u32, + pub(crate) unk_18: [u8; 0x14], + /// Required: 1. + pub(crate) unk_2c: u32, + /// Required: 1. + pub(crate) unk_30: u32, + pub(crate) unk_34: [u8; 0xc], + /// Required: 0x64 (100). + pub(crate) unk_40: u32, + /// Required: 1. + pub(crate) unk_44: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_48: u32, + pub(crate) unk_4c: [u8; 4], + /// Required: 0x64 (100). + pub(crate) unk_50: u32, + pub(crate) unk_54: [u8; 0xc], + /// Required: 4. + pub(crate) unk_60: u32, + /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). + pub(crate) unk_64: u32, + pub(crate) unk_68: [u8; 4], + /// Required: 1. + pub(crate) unk_6c: u32, + /// Required: 1. + pub(crate) unk_70: u32, + pub(crate) unk_74: [u8; 0xc], + /// Required: 0x64 (100). + pub(crate) unk_80: u32, + /// Required: 1. + pub(crate) unk_84: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_88: u32, + pub(crate) unk_8c: [u8; 4], + /// Required: 0x64 (100). + pub(crate) unk_90: u32, + pub(crate) unk_94: [u8; 0x2c], + /// Required: 0x3f82_8f5c (1.02 as an IEEE-754 single) in every entry. + pub(crate) unk_c0: [u32; 11], + pub(crate) unk_ec: [u8; 0x8cc], + /// Required: 0x271 (625). + pub(crate) unk_9b8: u32, + pub(crate) unk_9bc: [u8; 8], + /// Required: 0x3f7f_2e9f. + pub(crate) unk_9c4: u32, + pub(crate) unk_9c8: [u8; 4], + /// Required: 0x3b51_6154. + pub(crate) unk_9cc: u32, + pub(crate) unk_9d0: [u8; 4], + /// Required: 0x3ca5_9586 (0.020213 as an IEEE-754 single). + pub(crate) unk_9d4: u32, + pub(crate) unk_9d8: [u8; 8], + /// Required: 0x4615_3800 (9550.0 as an IEEE-754 single). + pub(crate) unk_9e0: u32, + /// Required: 0x40a9_0fdb. + pub(crate) unk_9e4: u32, + pub(crate) unk_9e8: [u8; 4], + /// Required: 0xbdd6_7344. + pub(crate) unk_9ec: u32, + /// Required: 0x28 (40). + pub(crate) unk_9f0: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_9f4: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_9f8: u32, + pub(crate) unk_9fc: [u8; 8], + /// Required: 0x254e (9550). + pub(crate) unk_a04: u32, + pub(crate) unk_a08: [u8; 0x30], + /// Required: 0x3e8 (1000). + pub(crate) unk_a38: u32, + pub(crate) unk_a3c: [u8; 0x34], + /// Required: 4. + pub(crate) unk_a70: u32, + pub(crate) unk_a74: [u8; 0x10], + /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). + pub(crate) unk_a84: u32, + pub(crate) unk_a88: [u8; 4], + /// Required: 0x40cc_cccd (6.4 as an IEEE-754 single). + pub(crate) unk_a8c: u32, + pub(crate) unk_a90: [u8; 8], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_a98: u32, + /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). + pub(crate) unk_a9c: u32, + pub(crate) unk_aa0: [u8; 8], + /// Required: 0x28 (40). + pub(crate) unk_aa8: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_aac: u32, + pub(crate) unk_ab0: [u8; 0xc], + /// Required: 0x254e (9550). + pub(crate) unk_abc: u32, + pub(crate) unk_ac0: [u8; 8], + /// Required: 0x10 (16). + pub(crate) unk_ac8: u32, + pub(crate) unk_acc: [u8; 4], + /// Required: 0x5_dc00 (384000). + pub(crate) unk_ad0: u32, + pub(crate) unk_ad4: [u8; 0x54], + /// Required: 0x5c (92). + pub(crate) unk_b28: u32, + pub(crate) unk_b2c: [u8; 4], + /// Required: 0x64 (100). + pub(crate) unk_b30: u32, + /// Required: 0x22 (34). + pub(crate) unk_b34: u32, + /// Required: 6. + pub(crate) unk_b38: u32, + pub(crate) unk_b3c: [u8; 4], + /// Required: 6. + pub(crate) unk_b40: u32, + /// Required: 1. + pub(crate) unk_b44: u32, + pub(crate) unk_b48: [u8; 4], + /// Required: 0x3f4c_cccd (0.8 as an IEEE-754 single). + pub(crate) unk_b4c: u32, + /// Required: 0x3f7d_f3b6 (0.992 as an IEEE-754 single). + pub(crate) unk_b50: u32, + /// Required: 0x3e4c_cccd (0.2 as an IEEE-754 single). + pub(crate) unk_b54: u32, + /// Required: 0x3c03_126f (0.008 as an IEEE-754 single). + pub(crate) unk_b58: u32, + /// Required: 0x3f69_d4d8 (0.913404 as an IEEE-754 single). + pub(crate) unk_b5c: u32, + /// Required: 0x3f69_d4d8 (0.913404 as an IEEE-754 single). + pub(crate) unk_b60: u32, + pub(crate) unk_b64: [u8; 4], + /// Required: 0x42be_0000 (95.0 as an IEEE-754 single). + pub(crate) unk_b68: u32, + /// Required: 0x4064_f5c3 (3.5775 as an IEEE-754 single). + pub(crate) unk_b6c: u32, + /// Required: 0x4064_f5c3 (3.5775 as an IEEE-754 single). + pub(crate) unk_b70: u32, + /// Required: 0x4117_9436. + pub(crate) unk_b74: u32, + /// Required: 0x64 (100). + pub(crate) unk_b78: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_b7c: u32, + /// Required: 0x64 (100). + pub(crate) unk_b80: u32, + pub(crate) unk_b84: [u8; 8], + /// Required: 0x5c (92). + pub(crate) unk_b8c: u32, + pub(crate) unk_b90: [u8; 0x30], + /// Required: 0x64 (100). + pub(crate) unk_bc0: u32, + pub(crate) unk_bc4: [u8; 0x1c], + /// Required: 1. + pub(crate) unk_be0: u8, + /// Required: 4. + pub(crate) unk_be1: u32, + /// Required: 0x64 (100). + pub(crate) unk_be5: u32, + pub(crate) unk_be9: [u8; 4], + /// Required: 0x1f40 (8000). + pub(crate) unk_bed: u32, + /// Required: 0xc8 (200). + pub(crate) unk_bf1: u32, + /// Required: 0xfa0 (4000). + pub(crate) unk_bf5: u32, + /// Required: 0xc8 (200). + pub(crate) unk_bf9: u32, + /// Required: 0x7d0 (2000). + pub(crate) unk_bfd: u32, + /// Required: 0xc8 (200). + pub(crate) unk_c01: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_c05: u32, + /// Required: 0xc8 (200). + pub(crate) unk_c09: u32, + /// Required: 1. + pub(crate) unk_c0d: u32, + pub(crate) unk_c11: [u8; 0x53], + /// Required: 1. + pub(crate) unk_c64: u32, + pub(crate) unk_c68: [u8; 0x28], + /// Required: 1. + pub(crate) unk_c90: u32, + pub(crate) unk_c94: [u8; 0x1003], + /// Required: 0x7fff in every entry; the table continues in + /// [`PowerConfigTail`]. + pub(crate) unk_1c97: [u16; 148], + /// Required: 0xff (low byte of the next table entry). + pub(crate) unk_1dbf: u8, +} + +static_assert!(bundle::POWER_CONFIG_COPY + size_of::() == 0xf000); + +// SAFETY: `PowerConfigHead` consists of integers and arrays of integers only. +unsafe impl Zeroable for PowerConfigHead {} + +impl PowerConfigHead { + /// Initializes the leading part of the power configuration. + pub(crate) fn init(&mut self) { + clear(self); + self.unk_4 = 0x5_dc00; + self.unk_8 = 0x5_dc00; + self.unk_10 = 4; + self.unk_14 = 0x3f80_0000; + self.unk_2c = 1; + self.unk_30 = 1; + self.unk_40 = 0x64; + self.unk_44 = 1; + self.unk_48 = 0x3e8; + self.unk_50 = 0x64; + self.unk_60 = 4; + self.unk_64 = 0x3f80_0000; + self.unk_6c = 1; + self.unk_70 = 1; + self.unk_80 = 0x64; + self.unk_84 = 1; + self.unk_88 = 0x3e8; + self.unk_90 = 0x64; + self.unk_c0 = [0x3f82_8f5c; 11]; + self.unk_9b8 = 0x271; + self.unk_9c4 = 0x3f7f_2e9f; + self.unk_9cc = 0x3b51_6154; + self.unk_9d4 = 0x3ca5_9586; + self.unk_9e0 = 0x4615_3800; + self.unk_9e4 = 0x40a9_0fdb; + self.unk_9ec = 0xbdd6_7344; + self.unk_9f0 = 0x28; + self.unk_9f4 = 0x3e8; + self.unk_9f8 = 0x3e8; + self.unk_a04 = 0x254e; + self.unk_a38 = 0x3e8; + self.unk_a70 = 4; + self.unk_a84 = 0x3f80_0000; + self.unk_a8c = 0x40cc_cccd; + self.unk_a98 = 0x4780_0000; + self.unk_a9c = 0x3f00_0000; + self.unk_aa8 = 0x28; + self.unk_aac = 0x3e8; + self.unk_abc = 0x254e; + self.unk_ac8 = 0x10; + self.unk_ad0 = 0x5_dc00; + self.unk_b28 = 0x5c; + self.unk_b30 = 0x64; + self.unk_b34 = 0x22; + self.unk_b38 = 6; + self.unk_b40 = 6; + self.unk_b44 = 1; + self.unk_b4c = 0x3f4c_cccd; + self.unk_b50 = 0x3f7d_f3b6; + self.unk_b54 = 0x3e4c_cccd; + self.unk_b58 = 0x3c03_126f; + self.unk_b5c = 0x3f69_d4d8; + self.unk_b60 = 0x3f69_d4d8; + self.unk_b68 = 0x42be_0000; + self.unk_b6c = 0x4064_f5c3; + self.unk_b70 = 0x4064_f5c3; + self.unk_b74 = 0x4117_9436; + self.unk_b78 = 0x64; + self.unk_b7c = 0x3e8; + self.unk_b80 = 0x64; + self.unk_b8c = 0x5c; + self.unk_bc0 = 0x64; + self.unk_be0 = 1; + self.unk_be1 = 4; + self.unk_be5 = 0x64; + self.unk_bed = 0x1f40; + self.unk_bf1 = 0xc8; + self.unk_bf5 = 0xfa0; + self.unk_bf9 = 0xc8; + self.unk_bfd = 0x7d0; + self.unk_c01 = 0xc8; + self.unk_c05 = 0x3e8; + self.unk_c09 = 0xc8; + self.unk_c0d = 1; + self.unk_c64 = 1; + self.unk_c90 = 1; + self.unk_1c97 = [0x7fff; 148]; + self.unk_1dbf = 0xff; + } +} + +/// Trailing part of the [`PowerConfig`]. Field names give the offset in the +/// whole power configuration. It carries the die temperature limiter: target +/// and release temperatures in 0.01 °C, proportional and integral gains as +/// IEEE-754 singles. +#[repr(C, packed)] +pub(crate) struct PowerConfigTail { + /// Required: 0x7f (high byte of the table entry started in the head). + pub(crate) unk_1dc0: u8, + /// Required: 0x7fff in every entry. + pub(crate) unk_1dc1: [u16; 363], + pub(crate) unk_2097: [u8; 0x201], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_2298: u32, + /// Required: 0x4220_0000 (40.0 as an IEEE-754 single). + pub(crate) unk_229c: u32, + /// Required: 0x447a_0000 (1000.0 as an IEEE-754 single). + pub(crate) unk_22a0: u32, + /// Required: 0x5be (1470). + pub(crate) unk_22a4: u32, + pub(crate) unk_22a8: [u8; 0x10], + /// Required: 0x28 (40). + pub(crate) unk_22b8: u32, + pub(crate) unk_22bc: [u8; 4], + /// Required: 0x42c8_0000 (100.0 as an IEEE-754 single). + pub(crate) unk_22c0: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_22c4: u32, + pub(crate) unk_22c8: [u8; 4], + /// Required: 0x3f4c_cccd (0.8 as an IEEE-754 single). + pub(crate) unk_22cc: u32, + /// Required: 0x3e4c_cccd (0.2 as an IEEE-754 single). + pub(crate) unk_22d0: u32, + pub(crate) unk_22d4: [u8; 0x3c], + /// Die temperature sensors the limiter reads ([`TEMP_SENSOR_MASK`]). + pub(crate) temp_sensor_mask: u64, + /// Release temperature of the limiter ([`TEMP_RELEASE`]). + pub(crate) temp_release: u32, + pub(crate) unk_231c: [u8; 4], + /// Required: 4. + pub(crate) unk_2320: u32, + pub(crate) unk_2324: [u8; 0x10], + /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). + pub(crate) unk_2334: u32, + pub(crate) unk_2338: [u8; 4], + /// Integral gain of the limiter ([`TEMP_KI`]). + pub(crate) temp_ki: u32, + pub(crate) unk_2340: [u8; 8], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_2348: u32, + /// Proportional gain of the limiter ([`TEMP_KP`]). + pub(crate) temp_kp: u32, + pub(crate) unk_2350: [u8; 8], + /// Required: 0x28 (40). + pub(crate) unk_2358: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_235c: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_2360: u32, + /// Margin below the target at which the limiter acts, in 0.01 °C. + pub(crate) temp_margin: u32, + pub(crate) unk_2368: [u8; 4], + /// Target temperature of the limiter ([`TEMP_TARGET`]). + pub(crate) temp_target: u32, + pub(crate) unk_2370: [u8; 0x30], + /// Required: 0x3e8 (1000). + pub(crate) unk_23a0: u32, + pub(crate) unk_23a4: [u8; 0x218], + /// Required: 0x4200_0006. + pub(crate) unk_25bc: u32, + pub(crate) unk_25c0: [u8; 0x212], + /// Required: 0x3f7e_b852 (0.995 as an IEEE-754 single). + pub(crate) unk_27d2: u32, + pub(crate) unk_27d6: [u8; 4], + /// Required: 0x3ba3_d70a (0.005 as an IEEE-754 single). + pub(crate) unk_27da: u32, + pub(crate) unk_27de: [u8; 4], + /// Required: 0x43c8_0000 (400.0 as an IEEE-754 single). + pub(crate) unk_27e2: u32, + pub(crate) unk_27e6: [u8; 8], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_27ee: u32, + /// Required: 0x42c8_0000 (100.0 as an IEEE-754 single). + pub(crate) unk_27f2: u32, + pub(crate) unk_27f6: [u8; 4], + /// Required: 0xbac8_0000. + pub(crate) unk_27fa: u32, + /// Required: 0x384 (900). + pub(crate) unk_27fe: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_2802: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_2806: u32, + pub(crate) unk_280a: [u8; 4], + /// Required: 0x453b_8000 (3000.0 as an IEEE-754 single). + pub(crate) unk_280e: u32, + /// Required: 0x26ac (9900). + pub(crate) unk_2812: u32, + pub(crate) unk_2816: [u8; 0x30], + /// Required: 0x3e8 (1000). + pub(crate) unk_2846: u32, + pub(crate) unk_284a: [u8; 0x18], + /// Required: 0xc8 (200). + pub(crate) unk_2862: u32, + pub(crate) unk_2866: [u8; 0xb2], + /// Required: 0x3e8 (1000). + pub(crate) unk_2918: u32, + pub(crate) unk_291c: [u8; 0x10], + /// Required: 1. + pub(crate) unk_292c: u32, + pub(crate) unk_2930: [u8; 0x30], + /// Required: 1. + pub(crate) unk_2960: u8, + /// Required: 1. + pub(crate) unk_2961: u32, + /// Required: 4. + pub(crate) unk_2965: u32, + pub(crate) unk_2969: [u8; 0x14], + /// Required: 0x23f (575). + pub(crate) unk_297d: u32, + /// Required: 1. + pub(crate) unk_2981: u32, + /// Required: 1. + pub(crate) unk_2985: u32, + pub(crate) unk_2989: [u8; 0x14], + /// Required: 0x240 (576). + pub(crate) unk_299d: u32, + /// Required: 1. + pub(crate) unk_29a1: u32, + /// Required: 1. + pub(crate) unk_29a5: u32, + pub(crate) unk_29a9: [u8; 0x14], + /// Required: 0x241 (577). + pub(crate) unk_29bd: u32, + pub(crate) unk_29c1: [u8; 0x47], + /// Required: 1. + pub(crate) unk_2a08: u32, + pub(crate) unk_2a0c: [u8; 4], + /// Required: 0x254e (9550) in every entry. + pub(crate) unk_2a10: [u32; 3], + pub(crate) unk_2a1c: [u8; 0xc], + /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). + pub(crate) unk_2a28: u32, + /// Required: 4. + pub(crate) unk_2a2c: u32, + /// Required: 0x10 (16). + pub(crate) unk_2a30: u32, + /// Required: 0x5_dc00 (384000). + pub(crate) unk_2a34: u32, + pub(crate) unk_2a38: [u8; 0xcc], + /// Required: 0x3c (60). + pub(crate) unk_2b04: u32, + pub(crate) unk_2b08: [u8; 8], + /// Required: 0x3f6e_eeef. + pub(crate) unk_2b10: u32, + pub(crate) unk_2b14: [u8; 4], + /// Required: 0x3d88_8889. + pub(crate) unk_2b18: u32, + pub(crate) unk_2b1c: [u8; 4], + /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). + pub(crate) unk_2b20: u32, + pub(crate) unk_2b24: [u8; 8], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_2b2c: u32, + /// Required: 0x4088_0000 (4.25 as an IEEE-754 single). + pub(crate) unk_2b30: u32, + pub(crate) unk_2b34: [u8; 8], + /// Required: 0x28 (40). + pub(crate) unk_2b3c: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_2b40: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_2b44: u32, + pub(crate) unk_2b48: [u8; 4], + /// Required: 0x4615_3800 (9550.0 as an IEEE-754 single). + pub(crate) unk_2b4c: u32, + /// Required: 0x254e (9550). + pub(crate) unk_2b50: u32, + pub(crate) unk_2b54: [u8; 8], + /// Required: 0xf0 (240). + pub(crate) unk_2b5c: u32, + pub(crate) unk_2b60: [u8; 4], + /// Required: 0x57_e400. + pub(crate) unk_2b64: u32, + pub(crate) unk_2b68: [u8; 0x1c], + /// Required: 0x3e8 (1000). + pub(crate) unk_2b84: u32, + pub(crate) unk_2b88: [u8; 0x3c], + /// Required: 1. + pub(crate) unk_2bc4: u32, + pub(crate) unk_2bc8: [u8; 0xcc], + /// Required: 0x32 (50). + pub(crate) unk_2c94: u32, + /// Required: 1. + pub(crate) unk_2c98: u32, + pub(crate) unk_2c9c: [u8; 4], + /// Required: 0x3f63_8e39. + pub(crate) unk_2ca0: u32, + /// Required: 0x3f2a_aaab. + pub(crate) unk_2ca4: u32, + /// Required: 0x3de3_8e39. + pub(crate) unk_2ca8: u32, + /// Required: 0x3eaa_aaab. + pub(crate) unk_2cac: u32, + /// Required: 0xbf4c_cccd (-0.8 as an IEEE-754 single). + pub(crate) unk_2cb0: u32, + /// Required: 0xbf4c_cccd (-0.8 as an IEEE-754 single). + pub(crate) unk_2cb4: u32, + pub(crate) unk_2cb8: [u8; 4], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_2cbc: u32, + /// Required: 0xc0a0_0000 (-5.0 as an IEEE-754 single). + pub(crate) unk_2cc0: u32, + /// Required: 0xc0a0_0000 (-5.0 as an IEEE-754 single). + pub(crate) unk_2cc4: u32, + pub(crate) unk_2cc8: [u8; 4], + /// Required: 0x64 (100). + pub(crate) unk_2ccc: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_2cd0: u32, + /// Required: 0x64 (100). + pub(crate) unk_2cd4: u32, + pub(crate) unk_2cd8: [u8; 4], + /// Required: 0x467a_0000 (16000.0 as an IEEE-754 single). + pub(crate) unk_2cdc: u32, + /// Required: 0x578 (1400). + pub(crate) unk_2ce0: u32, + pub(crate) unk_2ce4: [u8; 8], + /// Required: 0x90 (144). + pub(crate) unk_2cec: u32, + /// Required: 0x30 (48). + pub(crate) unk_2cf0: u32, + /// Required: 0x34_bc00. + pub(crate) unk_2cf4: u32, + pub(crate) unk_2cf8: [u8; 4], + /// Required: 0x11_9400. + pub(crate) unk_2cfc: u32, + pub(crate) unk_2d00: [u8; 8], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_2d08: u32, + pub(crate) unk_2d0c: [u8; 0x24], + /// Required: 0x3e80 (16000). + pub(crate) unk_2d30: u32, + /// Required: 2. + pub(crate) unk_2d34: u32, + /// Required: 0x9c4 (2500). + pub(crate) unk_2d38: u32, + /// Required: 0x20d (525). + pub(crate) unk_2d3c: u32, + /// Required: 2. + pub(crate) unk_2d40: u32, + /// Required: 4. + pub(crate) unk_2d44: u32, + /// Required: 0x32 (50). + pub(crate) unk_2d48: u32, + pub(crate) unk_2d4c: [u8; 0x2c], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_2d78: u32, + pub(crate) unk_2d7c: [u8; 0xc], + /// Required: 0x28 (40). + pub(crate) unk_2d88: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_2d8c: u32, + pub(crate) unk_2d90: [u8; 0x30], +} + +static_assert!(core::mem::offset_of!(PowerConfigTail, temp_sensor_mask) == 0x2310 - 0x1dc0); +static_assert!(core::mem::offset_of!(PowerConfigTail, temp_target) == 0x236c - 0x1dc0); +static_assert!(size_of::() == 0x1000); + +// SAFETY: `PowerConfigTail` consists of integers and arrays of integers only. +unsafe impl Zeroable for PowerConfigTail {} + +impl PowerConfigTail { + fn init(&mut self) { + clear(self); + self.unk_1dc0 = 0x7f; + self.unk_1dc1 = [0x7fff; 363]; + self.unk_2298 = 0x4780_0000; + self.unk_229c = 0x4220_0000; + self.unk_22a0 = 0x447a_0000; + self.unk_22a4 = 0x5be; + self.unk_22b8 = 0x28; + self.unk_22c0 = 0x42c8_0000; + self.unk_22c4 = 0x3e8; + self.unk_22cc = 0x3f4c_cccd; + self.unk_22d0 = 0x3e4c_cccd; + self.temp_sensor_mask = TEMP_SENSOR_MASK; + self.temp_release = TEMP_RELEASE; + self.unk_2320 = 4; + self.unk_2334 = 0x3f80_0000; + self.temp_ki = TEMP_KI; + self.unk_2348 = 0x4780_0000; + self.temp_kp = TEMP_KP; + self.unk_2358 = 0x28; + self.unk_235c = 0x3e8; + self.unk_2360 = 0x3e8; + self.temp_target = TEMP_TARGET; + self.unk_23a0 = 0x3e8; + self.unk_25bc = 0x4200_0006; + self.unk_27d2 = 0x3f7e_b852; + self.unk_27da = 0x3ba3_d70a; + self.unk_27e2 = 0x43c8_0000; + self.unk_27ee = 0x4780_0000; + self.unk_27f2 = 0x42c8_0000; + self.unk_27fa = 0xbac8_0000; + self.unk_27fe = 0x384; + self.unk_2802 = 0x3e8; + self.unk_2806 = 0x3e8; + self.unk_280e = 0x453b_8000; + self.unk_2812 = 0x26ac; + self.unk_2846 = 0x3e8; + self.unk_2862 = 0xc8; + self.unk_2918 = 0x3e8; + self.unk_292c = 1; + self.unk_2960 = 1; + self.unk_2961 = 1; + self.unk_2965 = 4; + self.unk_297d = 0x23f; + self.unk_2981 = 1; + self.unk_2985 = 1; + self.unk_299d = 0x240; + self.unk_29a1 = 1; + self.unk_29a5 = 1; + self.unk_29bd = 0x241; + self.unk_2a08 = 1; + self.unk_2a10 = [0x254e; 3]; + self.unk_2a28 = 0x3f80_0000; + self.unk_2a2c = 4; + self.unk_2a30 = 0x10; + self.unk_2a34 = 0x5_dc00; + self.unk_2b04 = 0x3c; + self.unk_2b10 = 0x3f6e_eeef; + self.unk_2b18 = 0x3d88_8889; + self.unk_2b20 = 0x3f00_0000; + self.unk_2b2c = 0x4780_0000; + self.unk_2b30 = 0x4088_0000; + self.unk_2b3c = 0x28; + self.unk_2b40 = 0x3e8; + self.unk_2b44 = 0x3e8; + self.unk_2b4c = 0x4615_3800; + self.unk_2b50 = 0x254e; + self.unk_2b5c = 0xf0; + self.unk_2b64 = 0x57_e400; + self.unk_2b84 = 0x3e8; + self.unk_2bc4 = 1; + self.unk_2c94 = 0x32; + self.unk_2c98 = 1; + self.unk_2ca0 = 0x3f63_8e39; + self.unk_2ca4 = 0x3f2a_aaab; + self.unk_2ca8 = 0x3de3_8e39; + self.unk_2cac = 0x3eaa_aaab; + self.unk_2cb0 = 0xbf4c_cccd; + self.unk_2cb4 = 0xbf4c_cccd; + self.unk_2cbc = 0x4780_0000; + self.unk_2cc0 = 0xc0a0_0000; + self.unk_2cc4 = 0xc0a0_0000; + self.unk_2ccc = 0x64; + self.unk_2cd0 = 0x3e8; + self.unk_2cd4 = 0x64; + self.unk_2cdc = 0x467a_0000; + self.unk_2ce0 = 0x578; + self.unk_2cec = 0x90; + self.unk_2cf0 = 0x30; + self.unk_2cf4 = 0x34_bc00; + self.unk_2cfc = 0x11_9400; + self.unk_2d08 = 0x4780_0000; + self.unk_2d30 = 0x3e80; + self.unk_2d34 = 2; + self.unk_2d38 = 0x9c4; + self.unk_2d3c = 0x20d; + self.unk_2d40 = 2; + self.unk_2d44 = 4; + self.unk_2d48 = 0x32; + self.unk_2d78 = 0x4780_0000; + self.unk_2d88 = 0x28; + self.unk_2d8c = 0x3e8; + } +} + +/// Power configuration, in view 2 of the bundle. The secondary instance +/// finds it through [`MainConfig::power_config_va`]. +#[repr(C, packed)] +pub(crate) struct PowerConfig { + /// Leading part. + pub(crate) head: PowerConfigHead, + /// Trailing part. + pub(crate) tail: PowerConfigTail, +} + +static_assert!(bundle::POWER_CONFIG + size_of::() == bundle::AUX); +static_assert!(bundle::VIEWS[2] + 0xe40 == bundle::POWER_CONFIG); + +// SAFETY: `PowerConfig` consists of integers and arrays of integers only. +unsafe impl Zeroable for PowerConfig {} + +impl PowerConfig { + /// Initializes the power configuration. + pub(crate) fn init(&mut self) { + self.head.init(); + self.tail.init(); + } +} + +/// Auxiliary hardware data the firmware reads at [`bundle::AUX`], up to the +/// start of view 3. No pointer names it. +#[repr(C, packed)] +pub(crate) struct HwDataAux { + pub(crate) unk_0: [u8; 0xac], + /// Required: 6. + pub(crate) unk_ac: u32, + pub(crate) unk_b0: [u8; 8], + /// Required: 1. + pub(crate) unk_b8: u32, + pub(crate) unk_bc: [u8; 0xbb], + /// Required: 0xdac (3500). + pub(crate) unk_177: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_17b: u32, + /// Required: 0xbb8 (3000). + pub(crate) unk_17f: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_183: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_187: u32, + /// Required: 0x64 (100). + pub(crate) unk_18b: u32, + /// Required: 0x20 (32). + pub(crate) unk_18f: u32, + pub(crate) unk_193: [u8; 0x21], + /// Required: 4. + pub(crate) unk_1b4: u32, + pub(crate) unk_1b8: [u8; 8], + /// Required: 0x3f78_0000 (0.96875 as an IEEE-754 single). + pub(crate) unk_1c0: u32, + pub(crate) unk_1c4: [u8; 4], + /// Required: 0x3d00_0000 (0.03125 as an IEEE-754 single). + pub(crate) unk_1c8: u32, + pub(crate) unk_1cc: [u8; 4], + /// Required: 0x40a0_0000 (5.0 as an IEEE-754 single). + pub(crate) unk_1d0: u32, + pub(crate) unk_1d4: [u8; 8], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_1dc: u32, + /// Required: 0x4170_0000 (15.0 as an IEEE-754 single). + pub(crate) unk_1e0: u32, + pub(crate) unk_1e4: [u8; 8], + /// Required: 0x384 (900). + pub(crate) unk_1ec: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_1f0: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_1f4: u32, + pub(crate) unk_1f8: [u8; 8], + /// Required: 0x3e8 (1000). + pub(crate) unk_200: u32, + pub(crate) unk_204: [u8; 0x30], + /// Required: 0x3e8 (1000). + pub(crate) unk_234: u32, + pub(crate) unk_238: [u8; 0x3c], + /// Required: 0x64 (100). + pub(crate) unk_274: u32, + pub(crate) unk_278: [u8; 0x18], + /// Required: 0x82a (2090). + pub(crate) unk_290: u32, + pub(crate) unk_294: [u8; 4], + /// Required: 0x7d (125). + pub(crate) unk_298: u32, + /// Required: 1. + pub(crate) unk_29c: u32, + pub(crate) unk_2a0: [u8; 0xc38], + /// GPU address of the secondary state, [`private::SECONDARY_STATUS`] bytes + /// into the secondary state grid. + pub(crate) secondary_status_va: u64, + /// Required: 0x18ae (6318). + pub(crate) unk_ee0: u32, + pub(crate) unk_ee4: [u8; 4], + /// Required: 1. + pub(crate) unk_ee8: u32, + pub(crate) unk_eec: [u8; 0x2000], + /// Required: 0x41c8_cccd (25.1 as an IEEE-754 single). + pub(crate) unk_2eec: u32, + /// Required: 0x41c8_cccd (25.1 as an IEEE-754 single). + pub(crate) unk_2ef0: u32, + pub(crate) unk_2ef4: [u8; 0x6f0], + /// Required: 1. + pub(crate) unk_35e4: u32, + /// Required: 0x1f4 (500). + pub(crate) unk_35e8: u32, + /// Required: 6. + pub(crate) unk_35ec: u32, + /// Required: 0x30d4 (12500) in every entry. + pub(crate) unk_35f0: [u32; 6], + /// Required: 0xfa0 (4000). + pub(crate) unk_3608: u32, + pub(crate) unk_360c: [u8; 0x10], + /// Required: 0x30d4 (12500) in every entry. + pub(crate) unk_361c: [u32; 5], + /// Required: 1. + pub(crate) unk_3630: u32, + pub(crate) unk_3634: [u8; 0x174], + /// Required: 1. + pub(crate) unk_37a8: u32, + pub(crate) unk_37ac: [u8; 8], + /// Required: 0x424c_cccd (51.2 as an IEEE-754 single). + pub(crate) unk_37b4: u32, + pub(crate) unk_37b8: [u8; 0x3c], + /// Required: 0x4406_8000 (538.0 as an IEEE-754 single). + pub(crate) unk_37f4: u32, + pub(crate) unk_37f8: [u8; 0x3c], + /// Required: 0x41c9_999a (25.2 as an IEEE-754 single). + pub(crate) unk_3834: u32, + pub(crate) unk_3838: [u8; 0xa8], + /// Required: 0x186a (6250). + pub(crate) unk_38e0: u32, + pub(crate) unk_38e4: [u8; 0x39c], +} + +static_assert!(core::mem::offset_of!(HwDataAux, secondary_status_va) == 0xed8); +static_assert!(bundle::AUX + size_of::() == bundle::VIEWS[3]); + +// SAFETY: `HwDataAux` consists of integers and arrays of integers only. +unsafe impl Zeroable for HwDataAux {} + +impl HwDataAux { + /// Initializes the auxiliary hardware data. `secondary_status_va` is + /// the secondary state grid plus [`private::SECONDARY_STATUS`]. + pub(crate) fn init(&mut self, secondary_status_va: u64) { + clear(self); + self.unk_ac = 6; + self.unk_b8 = 1; + self.unk_177 = 0xdac; + self.unk_17b = 0x3e8; + self.unk_17f = 0xbb8; + self.unk_183 = 0x3e8; + self.unk_187 = 0x3e8; + self.unk_18b = 0x64; + self.unk_18f = 0x20; + self.unk_1b4 = 4; + self.unk_1c0 = 0x3f78_0000; + self.unk_1c8 = 0x3d00_0000; + self.unk_1d0 = 0x40a0_0000; + self.unk_1dc = 0x4780_0000; + self.unk_1e0 = 0x4170_0000; + self.unk_1ec = 0x384; + self.unk_1f0 = 0x3e8; + self.unk_1f4 = 0x3e8; + self.unk_200 = 0x3e8; + self.unk_234 = 0x3e8; + self.unk_274 = 0x64; + self.unk_290 = 0x82a; + self.unk_298 = 0x7d; + self.unk_29c = 1; + self.unk_ee0 = 0x18ae; + self.unk_ee8 = 1; + self.unk_2eec = 0x41c8_cccd; + self.unk_2ef0 = 0x41c8_cccd; + self.unk_35e4 = 1; + self.unk_35e8 = 0x1f4; + self.unk_35ec = 6; + self.unk_35f0 = [0x30d4; 6]; + self.unk_3608 = 0xfa0; + self.unk_361c = [0x30d4; 5]; + self.unk_3630 = 1; + self.unk_37a8 = 1; + self.unk_37b4 = 0x424c_cccd; + self.unk_37f4 = 0x4406_8000; + self.unk_3834 = 0x41c9_999a; + self.unk_38e0 = 0x186a; + self.secondary_status_va = secondary_status_va; + } +} +// --------------------------------------------------------------------------- +// Free lists +// --------------------------------------------------------------------------- + +/// Size of a free-list page. +const FREE_LIST_PAGE_SIZE: u64 = 0x1000; +/// Pages in one free-list block. +pub(crate) const FREE_LIST_BLOCK_PAGES: u32 = 0x100; +/// Size of one free-list block. +pub(crate) const FREE_LIST_BLOCK_SIZE: usize = + FREE_LIST_BLOCK_PAGES as usize * FREE_LIST_PAGE_SIZE as usize; +/// Run-list qwords reserved for each block. +const RUN_SLOT_QWORDS: u32 = 8; +/// Bit position of the page count in a run descriptor. +const RUN_PAGES_SHIFT: u32 = 52; +/// Address bits of a run descriptor. +const RUN_ADDRESS_MASK: u64 = 0x0000_ffff_ffff_f000; + +/// Class of a free list, selecting [`FreeListControl::unk_10`] and +/// [`FreeListControl::unk_60`]. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum FreeListClass { + /// `unk_10` 1, `unk_60` 2: the boot render and compute free lists. + One = 1, + /// `unk_10` 2, `unk_60` 3: the auxiliary boot and runtime render pools. + Two = 2, +} + +impl FreeListClass { + const fn unk_60(self) -> u32 { + match self { + Self::One => 2, + Self::Two => 3, + } + } +} + +/// Description of a free list whose blocks are all populated. +pub(crate) struct FreeListArgs { + /// Buffer slot of the free list. + pub(crate) buffer_slot: u32, + /// Class of the free list. + pub(crate) class: FreeListClass, + /// Page list: one 64-bit address per page of the blocks. + pub(crate) page_list_va: u64, + /// Run list: one [`FreeListRunSlot`] per block. + pub(crate) run_list_va: u64, + /// Blocks in the free list. + pub(crate) blocks: u32, + /// [`FreeListState`]. + pub(crate) state_va: u64, +} + +/// Control object of a USC private-memory free list, at the start of its +/// page. +/// +/// This constructor describes the boot objects. Once a pool is published, its +/// page counters (`+0x24`, `+0x2c`) and run cursor (`+0x48`) are shared state. +/// The firmware-completed operation counter occupies the eight bytes at +/// `+0x54`; it is unaligned and the host reads its bytes individually. +/// A released render pool can be repopulated only after its release is +/// consumed: write page-list/run-list words first, then the complete control +/// page in ascending aligned 64-bit words, then a full barrier. The completed +/// count in that image must equal the retained submitted count, not zero. +/// Publication and reuse belong to the host free-list implementation. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct FreeListControl { + /// Boot value 1, compute value 2, or the render submitted-counter address. + pub(crate) unk_0: u64, + /// Buffer slot of the free list. + pub(crate) buffer_slot: u32, + pub(crate) unk_c: u32, + /// Required: see [`FreeListClass`]. + pub(crate) unk_10: u32, + /// Page list. + pub(crate) page_list_va: u64, + /// Required: 0x40000. + pub(crate) unk_1c: u32, + pub(crate) unk_20: u32, + /// Required: [`Self::page_count`]. + pub(crate) unk_24: u32, + pub(crate) unk_28: u32, + /// Pages in the free list's blocks. + pub(crate) page_count: u32, + /// Run list. + pub(crate) run_list_va: u64, + pub(crate) unk_38: [u8; 8], + /// Required: 4. + pub(crate) unk_40: u32, + pub(crate) unk_44: u32, + /// Run-list qwords in use. + pub(crate) run_cursor: u32, + /// [`FreeListState`]. + pub(crate) state_va: u64, + pub(crate) unk_54: [u8; 0xc], + /// Required: see [`FreeListClass`]. + pub(crate) unk_60: u32, + pub(crate) unk_64: u32, +} + +// SAFETY: `FreeListControl` consists of integers and arrays of integers only. +unsafe impl Zeroable for FreeListControl {} + +static_assert!(core::mem::offset_of!(FreeListControl, page_list_va) == 0x14); +static_assert!(core::mem::offset_of!(FreeListControl, run_cursor) == 0x48); +static_assert!(core::mem::offset_of!(FreeListControl, state_va) == 0x4c); +static_assert!(size_of::() == 0x68); + +impl FreeListControl { + /// Runtime render control; submitted and completed work are retained + /// across release/repopulation of the same pool. + pub(crate) fn render(args: &FreeListArgs, completed: u64) -> Self { + let mut control = Self::new(args); + control.unk_0 = args.state_va; + control.unk_54[..8].copy_from_slice(&completed.to_le_bytes()); + control + } + + /// Runtime compute control, with a compact run list and shared counters. + pub(crate) fn compute(args: &FreeListArgs) -> Self { + let mut control = Self::new(args); + control.unk_0 = 2; + control.unk_10 = 2; + control.unk_54[8..12].copy_from_slice(&1u32.to_le_bytes()); + control + } + + /// Offset of the firmware-completed counter. It must be read bytewise. + pub(crate) const COMPLETED: usize = 0x54; + + /// Returns the control object of the free list `args` describes. + pub(crate) const fn new(args: &FreeListArgs) -> Self { + let pages = args.blocks * FREE_LIST_BLOCK_PAGES; + Self { + unk_0: 1, + buffer_slot: args.buffer_slot, + unk_c: 0, + unk_10: args.class as u32, + page_list_va: args.page_list_va, + unk_1c: 0x4_0000, + unk_20: 0, + unk_24: pages, + unk_28: 0, + page_count: pages, + run_list_va: args.run_list_va, + unk_38: [0; 8], + unk_40: 4, + unk_44: 0, + run_cursor: args.blocks * RUN_SLOT_QWORDS, + state_va: args.state_va, + unk_54: [0; 0xc], + unk_60: args.class.unk_60(), + unk_64: 0, + } + } +} + +/// Shared state of a free list, at the start of its page. +/// +/// Compute creation writes a 32-bit one at `+0x00` in a zeroed page. The +/// render boot object starts with a 64-bit two and the boot pool with zero. +/// In a live render pool this becomes the submitted-operation counter: the +/// host uses one aligned 64-bit store before making work visible. It is +/// retained across release/repopulation, never reset by a struct copy. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct FreeListState { + /// [`Self::RENDER`], [`Self::BOOT`] or [`Self::COMPUTE`] at boot. + pub(crate) unk_0: u64, +} + +// SAFETY: `FreeListState` consists of integers and arrays of integers only. +unsafe impl Zeroable for FreeListState {} + +impl FreeListState { + /// Boot state of the render free list. + pub(crate) const RENDER: Self = Self { unk_0: 2 }; + /// Boot state of the boot free list. + pub(crate) const BOOT: Self = Self { unk_0: 0 }; + /// Boot state of a compute free list. + pub(crate) const COMPUTE: Self = Self { unk_0: 1 }; +} + +/// One run-list slot: the run descriptor of one block. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct FreeListRunSlot { + /// Block address, with its page count in the top bits. + pub(crate) run: u64, + pub(crate) unk_8: [u8; 0x38], +} + +// SAFETY: `FreeListRunSlot` consists of integers and arrays of integers only. +unsafe impl Zeroable for FreeListRunSlot {} + +static_assert!(size_of::() == RUN_SLOT_QWORDS as usize * size_of::()); + +impl FreeListRunSlot { + /// Returns the slot describing the block at `block_va`. + pub(crate) const fn new(block_va: u64) -> Self { + Self { + run: (block_va & RUN_ADDRESS_MASK) + | ((FREE_LIST_BLOCK_PAGES as u64) << RUN_PAGES_SHIFT), + unk_8: [0; 0x38], + } + } +} + +/// Writes the run descriptors of `blocks` to the start of `slots`. +pub(crate) fn fill_run_list( + slots: &mut [FreeListRunSlot], + blocks: impl IntoIterator, +) -> Result { + let mut slots = slots.iter_mut(); + for block in blocks { + *slots.next().ok_or(EINVAL)? = FreeListRunSlot::new(block); + } + Ok(()) +} + +/// Writes the address of every page of `blocks` to the start of `list`. +pub(crate) fn fill_page_list(list: &mut [u64], blocks: impl IntoIterator) -> Result { + let mut entries = list.iter_mut(); + for block in blocks { + for page in 0..u64::from(FREE_LIST_BLOCK_PAGES) { + *entries.next().ok_or(EINVAL)? = block + .checked_add(page * FREE_LIST_PAGE_SIZE) + .ok_or(EOVERFLOW)?; + } + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// Parameter-buffer and page-pool descriptor tables +// --------------------------------------------------------------------------- + +/// One parameter-buffer descriptor, indexed by buffer slot. +#[repr(C)] +pub(crate) struct PbDescriptor { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) unk_8: u32, + pub(crate) unk_c: u32, +} + +// SAFETY: `PbDescriptor` consists of integers and arrays of integers only. +unsafe impl Zeroable for PbDescriptor {} + +static_assert!(size_of::() == 0x10); + +/// Parameter-buffer descriptor table, one page. +#[repr(C)] +pub(crate) struct PbDescriptorTable { + /// Descriptors. + pub(crate) entries: [PbDescriptor; PAGE_SIZE / size_of::()], +} + +// SAFETY: `PbDescriptorTable` consists of integers only. +unsafe impl Zeroable for PbDescriptorTable {} + +impl PbDescriptorTable { + /// Required boot contents of slots 0 and 1. + const BOOT: [PbDescriptor; 2] = [ + PbDescriptor { + unk_0: 0x1_9000, + unk_4: 0x20, + unk_8: 0, + unk_c: 0, + }, + PbDescriptor { + unk_0: 0x7_7000, + unk_4: 0x80, + unk_8: 0x17, + unk_c: 0x17, + }, + ]; + + /// Initializes the table. + pub(crate) fn init(&mut self) { + clear(self); + let [slot0, slot1] = Self::BOOT; + self.entries[0] = slot0; + self.entries[1] = slot1; + } +} + +/// Bit position of the unit size in [`PagePoolDescriptor::page_list`]. +const PAGE_POOL_UNIT_SHIFT: u32 = 41; +/// Required unit size of a page-pool descriptor. +const PAGE_POOL_UNIT: u64 = 0x4_0000; +/// Alignment shift of the page-list address in [`PagePoolDescriptor::page_list`]. +const PAGE_POOL_LIST_SHIFT: u32 = 7; +/// Bit position of the free-page count in [`PagePoolDescriptor::counters`]. +const PAGE_POOL_FREE_PAGES_SHIFT: u32 = 33; + +/// One Page-pool descriptor, indexed by buffer slot. +#[repr(C)] +pub(crate) struct PagePoolDescriptor { + /// Unit size and page-list address. + pub(crate) page_list: u64, + /// Pool counters. + pub(crate) counters: u64, + /// Pages in the pool. + pub(crate) page_count: u64, + /// Firmware-owned. + pub(crate) unk_18: u64, +} + +// SAFETY: `PagePoolDescriptor` consists of integers and arrays of integers only. +unsafe impl Zeroable for PagePoolDescriptor {} + +static_assert!(size_of::() == 0x20); + +impl PagePoolDescriptor { + /// Complete row for a physical compute queue's independent 21-block pool. + pub(crate) const fn compute(page_list_va: u64) -> Self { + let pages = 21 * FREE_LIST_BLOCK_PAGES as u64; + Self::new(page_list_va, pages << PAGE_POOL_FREE_PAGES_SHIFT, pages) + } + + const fn new(page_list_va: u64, counters: u64, page_count: u64) -> Self { + Self { + page_list: (PAGE_POOL_UNIT << PAGE_POOL_UNIT_SHIFT) + | (page_list_va >> PAGE_POOL_LIST_SHIFT), + counters, + page_count, + unk_18: 0, + } + } +} + +/// page-pool descriptor table, one page. +#[repr(C)] +pub(crate) struct PagePoolDescriptorTable { + /// Descriptors. + pub(crate) entries: [PagePoolDescriptor; PAGE_SIZE / size_of::()], +} + +// SAFETY: `PagePoolDescriptorTable` consists of integers only. +unsafe impl Zeroable for PagePoolDescriptorTable {} + +impl PagePoolDescriptorTable { + /// Required counters and page count of slot 0 at boot. + const RENDER_COUNTERS: u64 = 0x3d40_0000_3400; + const RENDER_PAGES: u64 = 0x1d00; + + /// Initializes the table: slot 0 describes the render free list's page + /// list, slot 1 the compute page list. + pub(crate) fn init(&mut self, render_page_list_va: u64, compute_page_list_va: u64) { + clear(self); + self.entries[0] = PagePoolDescriptor::new( + render_page_list_va, + Self::RENDER_COUNTERS, + Self::RENDER_PAGES, + ); + self.entries[1] = PagePoolDescriptor::compute(compute_page_list_va); + } +} + +// --------------------------------------------------------------------------- +// Index table +// --------------------------------------------------------------------------- + +/// Groups of the index table: first index and number of groups. +const INDEX_GROUPS: [(u32, usize); 2] = [(0x11, 6), (0x3c, 2)]; +/// Indices per group. +const INDEX_GROUP_SIZE: usize = 4; +/// Distance between the first indices of consecutive groups. +const INDEX_GROUP_STRIDE: u32 = 5; +/// Entries of the index table. +const INDEX_ENTRIES: usize = (INDEX_GROUPS[0].1 + INDEX_GROUPS[1].1) * INDEX_GROUP_SIZE; + +/// Index table at the start of the index object. +#[repr(C)] +pub(crate) struct IndexTable { + /// Groups of consecutive indices. + pub(crate) index: [u32; INDEX_ENTRIES], + pub(crate) unk_80: [u8; PAGE_SIZE - INDEX_ENTRIES * size_of::()], +} + +static_assert!(size_of::() == PAGE_SIZE); + +// SAFETY: `IndexTable` consists of integers only. +unsafe impl Zeroable for IndexTable {} + +impl IndexTable { + /// Initializes the table. + pub(crate) fn init(&mut self) { + clear(self); + let mut entries = self.index.iter_mut(); + for (first, groups) in INDEX_GROUPS { + for group in 0..groups as u32 { + for member in 0..INDEX_GROUP_SIZE as u32 { + if let Some(entry) = entries.next() { + *entry = first + group * INDEX_GROUP_STRIDE + member; + } + } + } + } + } +} + +/// Device-global compute state. Activation writes only the named fields, once, before the +/// first compute queue is exposed. The padding may contain firmware-owned state. +#[repr(C)] +pub(crate) struct ComputeGlobalState { + pub(crate) unk_0: u8, + unk_1: [u8; 7], + pub(crate) unk_8: u8, + unk_9: [u8; 7], + pub(crate) unk_10: u16, + unk_12: [u8; 6], + pub(crate) unk_18: u16, + unk_1a: [u8; 0x3ee], + pub(crate) unk_408: u32, + pub(crate) unk_40c: u32, + unk_410: [u8; 0x1f4], + pub(crate) unk_604: u32, + unk_608: [u8; 0x200], + pub(crate) scheduler_va: u64, + unk_810: [u8; 0x3f4], + pub(crate) unk_c04: u32, + unk_c08: [u8; 0x1f8], + pub(crate) unk_e00: u64, + pub(crate) unk_e08: u64, + unk_e10: [u8; 0x10], + pub(crate) unk_e20: u64, + pub(crate) unk_e28: u64, +} +static_assert!(core::mem::offset_of!(ComputeGlobalState, unk_408) == 0x408); +static_assert!(core::mem::offset_of!(ComputeGlobalState, scheduler_va) == 0x808); +static_assert!(core::mem::offset_of!(ComputeGlobalState, unk_c04) == 0xc04); +static_assert!(core::mem::offset_of!(ComputeGlobalState, unk_e00) == 0xe00); +static_assert!(size_of::() == 0xe30); + +/// Host-owned recovery report and progress records within the bundle views. +pub(crate) mod recovery { + pub(crate) const REPORT_VIEW: usize = 3; + pub(crate) const REASON: usize = 0x4c; + pub(crate) const PROGRESS_VIEW: usize = 1; + pub(crate) const PROGRESS_START: usize = 0xa40; + pub(crate) const PROGRESS_STRIDE: usize = 0x18; + pub(crate) const VIEW_EXTENTS: [usize; 5] = [0x18c0, 0x0c80, 0x3c00, 0x3380, 0x3300]; +} diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs new file mode 100644 index 0000000000000..c8b7d3f0dce3a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -0,0 +1,539 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Kick-queue entries and the objects they reference. +//! +//! Every hardware queue (QID) owns a kick ring of [`KICK_SLOTS`] entries spaced +//! [`KICK_ENTRY_STRIDE`] bytes apart. A [`KickTimestamp`] names one entry: its +//! low eight bits select the ring slot. To kick, the host writes the complete +//! slot selected by the queue's current timestamp, issues a barrier, advances +//! the timestamp and then announces the entry with a +//! [`super::queue::KickAnnounce`] record. +//! +//! An entry points at a work descriptor and its queue record, lists the kicks +//! it depends on, binds up to four [`RegisterArray`]s embedded in the +//! descriptor and may name a table of [`McacheRange`]s. +//! +//! Constructors of plain values ([`KickTimestamp::new`], +//! [`KickDependency::new`], [`KickDependency::from_word`]) return `None` for +//! values outside their domain; builders of firmware records and of encoded +//! words return `EINVAL`. + +use kernel::prelude::*; + +use super::queue::QUEUE_PRIORITY; + +/// Number of entries in a kick ring. +pub(crate) const KICK_SLOTS: usize = 256; +/// Distance between consecutive kick ring entries. +pub(crate) const KICK_ENTRY_STRIDE: usize = 0x200; +/// Size of a kick ring. +pub(crate) const KICK_RING_SIZE: usize = KICK_SLOTS * KICK_ENTRY_STRIDE; +/// Dependencies one entry can carry: explicit barriers plus the implicit parent. +pub(crate) const KICK_DEPENDENCIES: usize = 32; +/// Explicit barriers one entry can carry. +pub(crate) const KICK_BARRIERS_MAX: usize = KICK_DEPENDENCIES - 1; +/// Highest hardware queue ID. +pub(crate) const QID_MAX: u8 = 0x7f; +/// Highest UAT context ID. +pub(crate) const CONTEXT_MAX: u8 = 0x3f; + +const KICK_TIMESTAMP_BITS: u32 = 40; +const KICK_TIMESTAMP_MASK: u64 = (1 << KICK_TIMESTAMP_BITS) - 1; +const KICK_SLOT_MASK: u64 = KICK_SLOTS as u64 - 1; +/// GPU addresses packed into kick entries are 32-byte aligned and 43 bits wide. +const PACKED_VA_SHIFT: u32 = 5; +const PACKED_VA_LIMIT: u64 = 1 << (38 + PACKED_VA_SHIFT); + +/// Packs a 32-byte aligned GPU address into the low 38 bits of a word. +fn packed_va(va: u64) -> Result { + if va & ((1 << PACKED_VA_SHIFT) - 1) != 0 || va >= PACKED_VA_LIMIT { + return Err(EINVAL); + } + Ok(va >> PACKED_VA_SHIFT) +} + +/// A kick timestamp. +/// +/// The value is a 40-bit counter that advances by one per kick of a queue; +/// bits 7:0 select the kick ring slot. A queue's first kick uses +/// [`KickTimestamp::FIRST`]. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct KickTimestamp(u64); + +impl KickTimestamp { + /// The timestamp preceding a queue's first kick. + pub(crate) const ZERO: Self = Self(0); + /// The timestamp of a queue's first kick. + pub(crate) const FIRST: Self = Self(1); + + /// Returns the timestamp with raw value `value`, if it fits 40 bits. + pub(crate) const fn new(value: u64) -> Option { + if value & !KICK_TIMESTAMP_MASK != 0 { + None + } else { + Some(Self(value)) + } + } + + /// Returns the raw 40-bit value. + pub(crate) const fn get(self) -> u64 { + self.0 + } + + /// Returns the kick ring slot this timestamp selects. + pub(crate) const fn slot(self) -> usize { + (self.0 & KICK_SLOT_MASK) as usize + } + + /// Returns the byte offset of the selected entry within the kick ring. + pub(crate) const fn entry_offset(self) -> usize { + self.slot() * KICK_ENTRY_STRIDE + } + + /// Returns the timestamp of the following kick. + pub(crate) const fn next(self) -> Self { + Self(self.0.wrapping_add(1) & KICK_TIMESTAMP_MASK) + } + + /// Returns the timestamp of the preceding kick. + pub(crate) const fn prev(self) -> Self { + Self(self.0.wrapping_sub(1) & KICK_TIMESTAMP_MASK) + } + + /// Returns the completion value of this kick: the timestamp in bits 39:0 + /// and `pass_seq`, the low byte of the logical queue's render-pass count, + /// in bits 47:40. + pub(crate) const fn completion_value(self, pass_seq: u8) -> u64 { + self.0 | (pass_seq as u64) << KICK_TIMESTAMP_BITS + } +} + +/// A dependency on the completion of one kick. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct KickDependency { + qid: u8, + timestamp: KickTimestamp, +} + +impl KickDependency { + const QID_SHIFT: u32 = KICK_TIMESTAMP_BITS; + + /// Initial value for unused entries in a host-side dependency array. + pub(crate) const ZERO: Self = Self { + qid: 0, + timestamp: KickTimestamp::ZERO, + }; + + /// Returns the dependency on kick `timestamp` of queue `qid`. + pub(crate) const fn new(qid: u8, timestamp: KickTimestamp) -> Option { + if qid > QID_MAX { + None + } else { + Some(Self { qid, timestamp }) + } + } + + /// Returns the queue that was kicked. + pub(crate) const fn qid(self) -> u8 { + self.qid + } + + /// Returns the dependency word: the timestamp in bits 39:0 and the QID in + /// bits 46:40. + pub(crate) const fn word(self) -> u64 { + self.timestamp.0 | (self.qid as u64) << Self::QID_SHIFT + } + + /// Decodes a dependency word; bits 63:47 must be clear. + pub(crate) const fn from_word(word: u64) -> Option { + if word >> (Self::QID_SHIFT + 7) != 0 { + return None; + } + Some(Self { + qid: (word >> Self::QID_SHIFT) as u8, + timestamp: KickTimestamp(word & KICK_TIMESTAMP_MASK), + }) + } +} + +/// One register write of a [`RegisterArray`]. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct RegisterWrite { + /// Register number. + pub(crate) reg: u32, + /// Value written to the register. + pub(crate) value: u64, +} + +impl RegisterWrite { + /// Returns the write of `value` to register `reg`. + pub(crate) const fn new(reg: u32, value: u64) -> Self { + Self { reg, value } + } +} + +/// A buffer that work writes. +#[derive(Copy, Clone, Debug)] +pub(crate) struct WriteRange { + /// GPU address of the buffer. + pub(crate) va: u64, + /// Size of the buffer in bytes. + pub(crate) size: u64, +} + +/// Class of an [`McacheRange`]. +/// +/// The auxiliary framebuffer range takes class 2 and attachment ranges take +/// class 1. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum McacheClass { + /// A render-target attachment. + Attachment = 1, + /// The auxiliary framebuffer. + AuxFramebuffer = 2, +} + +/// One MCache range: a GPU address range that work writes, in 128-byte cache +/// lines. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct McacheRange { + /// First cache line in bits 40:0, UAT context in bits 46:41. + pub(crate) start: u64, + /// End cache line (exclusive) in bits 40:0 and the class above. + pub(crate) end: u64, +} + +impl McacheRange { + const LINE_SIZE: u64 = 0x80; + /// Ranges cover fewer than 2^32 cache lines (512 GiB). + const LINES_MAX: u64 = u32::MAX as u64; + const LINE_LIMIT: u64 = 1 << 41; + const CONTEXT_SHIFT: u32 = 41; + /// Bits 55:47 of the start word hold 0x118 in every range. + const UNK_START: u64 = 0x118 << 47; + /// Bit 41 of the end word is set in every range. + const UNK_END: u64 = 1 << 41; + const CLASS_SHIFT: u32 = 42; + + /// Encodes the cache lines `range` touches, translated in UAT context + /// `context`. + pub(crate) fn new(range: &WriteRange, class: McacheClass, context: u8) -> Result { + if range.size == 0 || context > CONTEXT_MAX { + return Err(EINVAL); + } + let start = range.va / Self::LINE_SIZE; + let end = range + .va + .checked_add(range.size) + .ok_or(EINVAL)? + .div_ceil(Self::LINE_SIZE); + if end >= Self::LINE_LIMIT || end - start > Self::LINES_MAX { + return Err(EINVAL); + } + Ok(Self { + start: start | (context as u64) << Self::CONTEXT_SHIFT | Self::UNK_START, + end: end | Self::UNK_END | 1 << (Self::CLASS_SHIFT + class as u32), + }) + } +} + +/// Register-write entries a [`RegisterArray`] holds. +pub(crate) const REGISTER_ARRAY_ENTRIES: usize = 128; +/// MCache ranges that fit in the tail of a [`RegisterArray`]. +pub(crate) const MCACHE_INLINE_RANGES: usize = 16; + +/// A register array embedded in a work descriptor and bound to its kick entry +/// by a [`RegisterArrayBinding`]. +/// +/// The array ends in a trailer that points back at the array and holds its +/// size. It is only written as part of a cleared descriptor: [`Self::set`] +/// leaves the unused entries and the MCache space untouched. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct RegisterArray { + /// Register writes, in order. + pub(crate) entries: [RegisterWrite; REGISTER_ARRAY_ENTRIES], + /// Space after the entries. The first fragment array keeps its MCache + /// table here when the table fits; it is zero everywhere else. + pub(crate) mcache: [McacheRange; MCACHE_INLINE_RANGES], + /// GPU address of this array, as seen by the work's VM. + pub(crate) self_va: u64, + /// Size of the used entries in bytes (bits 31:16) and their count (15:0). + pub(crate) size: u32, + pub(crate) unk_70c: [u8; 0x14], +} + +impl RegisterArray { + /// Stores `writes` and the trailer into a zeroed array. + pub(crate) fn set(&mut self, self_va: u64, writes: &[RegisterWrite]) -> Result { + let count = writes.len(); + if count > REGISTER_ARRAY_ENTRIES { + return Err(EINVAL); + } + self.entries[..count].copy_from_slice(writes); + self.self_va = self_va; + self.size = (size_of_val(writes) as u32) << 16 | count as u32; + Ok(()) + } +} + +/// Binding of one [`RegisterArray`] to a kick entry. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct RegisterArrayBinding { + word: u64, +} + +impl RegisterArrayBinding { + const CHECKPOINT_SHIFT: u32 = 43; + const CHECKPOINT_MAX: u8 = 0x7f; + + /// Binds the array at `array_va`, as seen by the work's VM, whose first + /// checkpoint is after `checkpoint` entries. + pub(crate) fn new(array_va: u64, checkpoint: u8) -> Result { + if checkpoint > Self::CHECKPOINT_MAX { + return Err(EINVAL); + } + Ok(Self { + word: packed_va(array_va)? | (checkpoint as u64) << Self::CHECKPOINT_SHIFT, + }) + } + + fn word(binding: Option) -> u64 { + binding.map_or(0, |binding| binding.word) + } +} + +/// Location of the MCache table a kick entry names. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct McacheTable { + va: u64, + count: u8, +} + +impl McacheTable { + const VA_SHIFT: u32 = 6; + const COUNT_SHIFT: u32 = 54; + const COUNT_MAX: usize = 64; + + /// Names the `count` ranges at `va`, as seen by the work's VM. + pub(crate) fn new(va: u64, count: usize) -> Result { + if count == 0 || count > Self::COUNT_MAX { + return Err(EINVAL); + } + packed_va(va)?; + Ok(Self { + va, + count: count as u8, + }) + } + + fn word(self) -> u64 { + (self.va >> PACKED_VA_SHIFT) << Self::VA_SHIFT + | ((self.count - 1) as u64) << Self::COUNT_SHIFT + } +} + +/// Quality-of-service identity of the kicked queue. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct KickQos { + /// QoS buffer slot of the logical queue. + pub(crate) slot: u8, + /// Scheduling class of the engine, at most 0x1f. + pub(crate) class: u8, +} + +/// Inputs of one kick entry. +pub(crate) struct KickArgs<'a> { + /// Kicked queue. + pub(crate) qid: u8, + /// Timestamp of this kick; it selects the ring slot. + pub(crate) timestamp: KickTimestamp, + /// Kick this kick implicitly depends on: the previous kick of the same + /// queue. A queue's first kick names `timestamp.prev()` on render queues + /// and [`KickTimestamp::ZERO`] on compute queues. + pub(crate) parent: KickTimestamp, + /// Kicks of other queues this kick must wait for. + pub(crate) barriers: &'a [KickDependency], + /// GPU address of the work descriptor (firmware alias). + pub(crate) descriptor_va: u64, + /// GPU address of the queue record. + pub(crate) queue_va: u64, + /// Queue QoS identity. + pub(crate) qos: KickQos, + /// MCache table of the work, if any. + pub(crate) mcache: Option, + /// Engine-specific event mask. + pub(crate) event_mask: [u64; 4], + /// Register arrays bound to the kick. + pub(crate) register_arrays: [Option; 4], +} + +/// One kick ring entry. +/// +/// Live state: none. The host writes the whole entry before announcing it and +/// does not touch it again until the slot is reused. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct KickEntry { + /// Timestamp in bits 41:2, QID in bits 48:42, MCache table present in bit + /// 58 and queue priority in bits 60:59. + pub(crate) header: u64, + /// MCache table: address >> 5 in bits 43:6, count - 1 in bits 59:54. + pub(crate) mcache: u64, + /// GPU address of the work descriptor. + pub(crate) descriptor_va: u64, + /// GPU address of the queue record. + pub(crate) queue_va: u64, + /// Valid-dependency mask in bits 31:0, QoS slot in bits 39:32, class in + /// bits 47:40; bits 63:48 are all ones. + pub(crate) dependency_control: u64, + /// Explicit barriers followed by the implicit parent. + pub(crate) dependencies: [u64; KICK_DEPENDENCIES], + /// Engine-specific event mask. + pub(crate) event_mask: [u64; 4], + pub(crate) unk_148: u64, + /// Register-array bindings: address >> 5 in bits 37:0, checkpoint in bits + /// 49:43. + pub(crate) register_arrays: [u64; 4], + pub(crate) unk_170: u64, + pub(crate) unk_178: u64, + pub(crate) pad_180: [u8; KICK_ENTRY_STRIDE - 0x180], +} + +static_assert!(size_of::() == KICK_ENTRY_STRIDE); +static_assert!(core::mem::offset_of!(KickEntry, dependency_control) == 0x20); +static_assert!(core::mem::offset_of!(KickEntry, event_mask) == 0x128); +static_assert!(core::mem::offset_of!(KickEntry, register_arrays) == 0x150); +static_assert!(core::mem::offset_of!(KickEntry, unk_178) == 0x178); +static_assert!(size_of::() == 0xc); +static_assert!(size_of::() == 0x10); +static_assert!(size_of::() == 0x720); +static_assert!(core::mem::offset_of!(RegisterArray, mcache) == 0x600); +static_assert!(core::mem::offset_of!(RegisterArray, self_va) == 0x700); + +impl KickEntry { + const TIMESTAMP_SHIFT: u32 = 2; + const QID_SHIFT: u32 = 42; + const MCACHE_PRESENT: u64 = 1 << 58; + const PRIORITY_SHIFT: u32 = 59; + const QOS_SLOT_SHIFT: u32 = 32; + const QOS_CLASS_SHIFT: u32 = 40; + const QOS_CLASS_MAX: u8 = 0x1f; + /// Bits 63:48 of the dependency control word. + const UNK_DEPENDENCY_CONTROL: u64 = 0xffff << 48; + /// Required value of the last word of the entry. + const UNK_178: u64 = 0x003f_ffff_ffff_ffff; + + /// Builds the entry described by `args`. + pub(crate) fn new(args: &KickArgs<'_>) -> Result { + let barriers = args.barriers.len(); + if args.qid > QID_MAX + || barriers > KICK_BARRIERS_MAX + || args.qos.class > Self::QOS_CLASS_MAX + { + return Err(EINVAL); + } + + let mut header = args.timestamp.get() << Self::TIMESTAMP_SHIFT + | (args.qid as u64) << Self::QID_SHIFT + | (QUEUE_PRIORITY as u64) << Self::PRIORITY_SHIFT; + let mcache = match args.mcache { + Some(table) => { + header |= Self::MCACHE_PRESENT; + table.word() + } + None => 0, + }; + + let mut dependencies = [0; KICK_DEPENDENCIES]; + for (slot, barrier) in dependencies.iter_mut().zip(args.barriers) { + *slot = barrier.word(); + } + dependencies[barriers] = KickDependency { + qid: args.qid, + timestamp: args.parent, + } + .word(); + let count = barriers + 1; + let mask = if count == KICK_DEPENDENCIES { + u32::MAX + } else { + (1u32 << count) - 1 + }; + + Ok(Self { + header, + mcache, + descriptor_va: args.descriptor_va, + queue_va: args.queue_va, + dependency_control: mask as u64 + | (args.qos.slot as u64) << Self::QOS_SLOT_SHIFT + | (args.qos.class as u64) << Self::QOS_CLASS_SHIFT + | Self::UNK_DEPENDENCY_CONTROL, + dependencies, + event_mask: args.event_mask, + unk_148: 0, + register_arrays: args.register_arrays.map(RegisterArrayBinding::word), + unk_170: 0, + unk_178: Self::UNK_178, + pad_180: [0; KICK_ENTRY_STRIDE - 0x180], + }) + } +} + +/// The complete write-only installation of a fragment/tiling queue pair. +/// Firmware must see both queues configured before either outer command. +pub(crate) struct RenderRegistration { + writes: [(usize, u64); 8], +} + +impl RenderRegistration { + const TAG_PORT: usize = 0x4090; + const DATA_PORT: usize = 0x4098; + const VALID_LOW: usize = 0x21068; + const VALID_HIGH: usize = 0x21070; + const CONFIG_TAG: u64 = 0x0000_0400_0002_1008; + const ADDRESS_MASK: u64 = 0x0000_07ff_ffff_ffe0; + + /// Validates both queues without touching MMIO. The resulting transaction + /// writes fragment configuration and validity before tiling configuration + /// and validity. Each validity mask is stored high before low. + pub(crate) fn new(fragment: (u8, u64), tiling: (u8, u64)) -> Result { + if fragment.0 == tiling.0 { + return Err(EINVAL); + } + let mut writes = [(0, 0); 8]; + for (slot, ((qid, va), engine)) in [ + (fragment, super::queue::DataMaster::Fragment), + (tiling, super::queue::DataMaster::Tiling), + ] + .into_iter() + .enumerate() + { + if qid > QID_MAX || va & !Self::ADDRESS_MASK != 0 { + return Err(EINVAL); + } + let completion = (3 * (4 - u64::from(QUEUE_PRIORITY)) + 0x3fe).wrapping_shl(54); + let payload = ((engine as u64 + 1) << 60) | va | completion; + let (low, high) = if qid < 64 { + (1u64 << qid, 0) + } else { + (0, 1u64 << (qid - 64)) + }; + writes[slot * 4..slot * 4 + 4].copy_from_slice(&[ + (Self::TAG_PORT, Self::CONFIG_TAG | (u64::from(qid) << 43)), + (Self::DATA_PORT, payload), + (Self::VALID_HIGH, high), + (Self::VALID_LOW, low), + ]); + } + Ok(Self { writes }) + } + + pub(crate) fn writes(&self) -> &[(usize, u64); 8] { + &self.writes + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/mod.rs b/drivers/gpu/drm/asahi/g17/fw/mod.rs new file mode 100644 index 0000000000000..4f0ae8aa253ed --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/mod.rs @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G17 firmware structures. +//! +//! Layouts of the records and descriptors the host shares with the G17 +//! firmware, and the pure builders that fill them. Nothing here allocates, +//! locks or publishes: the owners of the rings and queues do that. +//! +//! This module also holds the few rules shared by the render and compute +//! work descriptors. + +use kernel::prelude::*; + +pub(crate) mod buffer; +pub(crate) mod channels; +pub(crate) mod compute; +pub(crate) mod context; +pub(crate) mod freelist; +pub(crate) mod initdata; +pub(crate) mod kick; +pub(crate) mod queue; +pub(crate) mod render; +pub(crate) mod qos; + +/// Clears a firmware structure in place. +/// +/// Work descriptors are several KiB large and must not be built on the stack. +pub(crate) fn clear(object: &mut T) { + // SAFETY: `T: Zeroable` guarantees that all-zero bytes are a valid `T`, + // and `object` is a valid, exclusive reference to one `T`. + unsafe { core::ptr::write_bytes(object as *mut T, 0, 1) }; +} + +/// Returns `va + offset`, or `EINVAL` if the sum overflows. +pub(crate) fn offset_va(va: u64, offset: u64) -> Result { + va.checked_add(offset).ok_or(EINVAL) +} + +/// Size of one slot of a start/end timestamp pair. +const TIMESTAMP_SIZE: u64 = 8; + +/// Returns the address of the end slot of the timestamp pair at `start_va`. +/// +/// A work item's start and end timestamps are two consecutive 64-bit slots. +pub(crate) fn timestamp_end_va(start_va: u64) -> Result { + offset_va(start_va, TIMESTAMP_SIZE) +} + +/// Returns the work key registers take: the UAT context in bits 15:8 and the +/// low six bits of the work-state word below. +pub(crate) const fn work_key(context_id: u16, state_word: u32) -> u64 { + (context_id as u64) << 8 | (state_word & 0x3f) as u64 +} + +/// Returns the sampler bound descriptors store next to the sampler count: the +/// count plus one, or zero without samplers. +pub(crate) fn sampler_max(count: u32) -> Result { + match count { + 0 => Ok(0), + count => count.checked_add(1).ok_or(EINVAL), + } +} + +/// Returns the completion stamp value of sequence number `sequence`: the +/// value the firmware stores to a queue's completion stamp word when the work +/// completes. +pub(crate) const fn stamp_value(sequence: u32) -> u32 { + sequence.wrapping_shl(8) +} + +/// Object IDs of a work item, written to three registers of its engine and +/// to its descriptor. +/// +/// A render pass takes `predecessor` from its queue pair and a newly +/// allocated `current`; compute work uses fixed IDs. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct ObjectIds { + /// Predecessor object ID. + pub(crate) predecessor: u32, + /// Current object ID. + pub(crate) current: u32, +} + +impl ObjectIds { + /// Returns the register value: `predecessor` in bits 63:32 and `current` + /// in bits 31:0. + pub(crate) const fn word(self) -> u64 { + (self.predecessor as u64) << 32 | self.current as u64 + } +} + +/// Required value of the per-engine registers 0x0a599 (compute), 0x0a5a1 +/// (tiling) and 0x0a5a9 (fragment). +pub(crate) const UNK_0A599: u64 = 0x0000_0060_0040_0020; +/// Required value of the per-engine registers 0x0d411 (compute), 0x0d419 +/// (tiling) and 0x0d429 (fragment). +pub(crate) const UNK_0D411: u64 = 0x0000_0002_0000_0001; +/// Required value of the per-engine registers 0x101d9 (compute), 0x101e1 +/// (tiling) and 0x101e9 (fragment). +pub(crate) const UNK_101D9: u64 = 0x1c; +/// Required value of the per-engine registers 0x1a0e9 (compute), 0x1a0f1 +/// (tiling) and 0x1a0f9 (fragment). +pub(crate) const UNK_1A0E9: u64 = 8; +/// Required value of the per-engine registers 0x107a1 (compute) and 0x10799 +/// (tiling). The fragment register 0x10791 also sets bit 9. +pub(crate) const UNK_10791: u64 = 0xff_0000; +/// Required value of the tiling register 0x1c8f8 and the fragment register +/// 0x100b8; the compute register 0x1a458 holds it in its low bits. +pub(crate) const UNK_1C8F8: u64 = 0x8860; diff --git a/drivers/gpu/drm/asahi/g17/fw/qos.rs b/drivers/gpu/drm/asahi/g17/fw/qos.rs new file mode 100644 index 0000000000000..d939e012714ee --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/qos.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! QoS accounting object. Queue rows and submission counts use distinct queue +//! and logical-owner indices. Firmware may observe each live word separately. + +use kernel::prelude::*; + +pub(crate) const QUEUES: usize = 128; +pub(crate) const DATA_MASTERS: usize = 4; + +#[repr(C)] +pub(crate) struct Qos { + pub(crate) rows: [[u8; 8]; QUEUES], + pub(crate) queue_submitted: [u32; QUEUES], + pub(crate) owner_submitted: [u32; QUEUES], + pub(crate) scheduler: [u64; QUEUES], + pub(crate) share: [u32; QUEUES], + pub(crate) intervals: u64, + pub(crate) busy_start: [u64; 3], + pub(crate) dm_submitted: [u64; DATA_MASTERS], +} +// SAFETY: Qos consists entirely of integer fields and arrays. +unsafe impl Zeroable for Qos {} + +const _: () = { + assert!(core::mem::size_of::() == 0xe40); + assert!(core::mem::offset_of!(Qos, queue_submitted) == 0x400); + assert!(core::mem::offset_of!(Qos, owner_submitted) == 0x600); + assert!(core::mem::offset_of!(Qos, scheduler) == 0x800); + assert!(core::mem::offset_of!(Qos, share) == 0xc00); + assert!(core::mem::offset_of!(Qos, intervals) == 0xe00); + assert!(core::mem::offset_of!(Qos, busy_start) == 0xe08); + assert!(core::mem::offset_of!(Qos, dm_submitted) == 0xe20); +}; diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs new file mode 100644 index 0000000000000..db95ba99e0189 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -0,0 +1,540 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Queue records and the commands of a queue's item ring. +//! +//! A hardware queue is described by a [`QueueRecord`]. Its [`PointerBlock`] +//! holds the cursors of the queue's item ring, an array of [`ITEM_RING_ENTRIES`] +//! work pointers: 64-bit GPU addresses of commands. Every command starts with +//! a 32-bit [`CommandTag`]: work descriptors carry the tiling, fragment and +//! compute tags (see `render.rs` and `compute.rs`), and the per-kick records +//! defined here carry the others. +//! +//! One kick appends, in order, the work descriptor, a [`QueueConfig`], a +//! [`KickAnnounce`] record and, when a compute queue is first installed or +//! explicitly waits on another queue, a [`KickPredecessor`]. +//! Compute kicks always carry the [`QueueConfig`]. A render kick carries it +//! only when it installs the queue, updates the context or QoS identity, +//! carries a completion seed, or configures the queue differently from the +//! last [`QueueConfig`] published on it. + +use core::sync::atomic::{AtomicU32, Ordering}; +use kernel::prelude::*; + +use super::kick::{KickTimestamp, QID_MAX}; + +/// Priority of every queue the driver creates. +/// +/// The same value appears in the queue record, [`QueueConfig`], +/// [`KickAnnounce`] and the kick entry header; the work rings it selects are +/// listed in `channels.rs`. +pub(crate) const QUEUE_PRIORITY: u8 = 2; +/// Scheduling policy of every queue the driver creates. +pub(crate) const QUEUE_POLICY: u16 = 2; +/// Number of work pointers in a queue's item ring. +pub(crate) const ITEM_RING_ENTRIES: u32 = 0x500; + +/// Tag in the first word of every command. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum CommandTag { + /// Tiling (TA) work descriptor. + Tiling = 0, + /// Fragment (3D) work descriptor. + Fragment = 1, + /// Compute work descriptor. + Compute = 3, + /// [`KickAnnounce`]. + KickAnnounce = 0x0e, + /// [`QueueConfig`]. + QueueConfig = 0x0f, + /// [`KickPredecessor`]. + KickPredecessor = 0x10, +} + +/// The engine that executes a queue's work. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum DataMaster { + /// Tiling (TA). + Tiling = 0, + /// Fragment (3D). + Fragment = 1, + /// Compute (CL). + Compute = 2, +} + +/// Cursors of a queue's item ring. +/// +/// The firmware writes `done` and `unk_30`. The host writes the block on +/// three paths, never with a structure copy once the queue was published: +/// +/// - creation: the whole block, [`PointerBlock::new`], before the queue is +/// first published; +/// - publication: `producer`, with one aligned 32-bit store per publication +/// step, after a barrier that orders the work pointers and commands before +/// it; +/// - reuse of a render queue for a new owner: [`PointerBlock::rebase`]. +#[repr(C)] +#[derive(Debug)] +pub(crate) struct PointerBlock { + /// Item-ring position up to which the firmware has finished commands. + /// Free ring space is measured from it, so it bounds `producer`. + pub(crate) done: AtomicU32, + pub(crate) unk_04: [u32; 11], + /// Written by the firmware. The host treats the queue as idle only when + /// this cursor, `done` and `producer` are equal. + pub(crate) unk_30: AtomicU32, + pub(crate) unk_34: [u32; 3], + /// Item-ring position after the last work pointer the host published. + pub(crate) producer: AtomicU32, + pub(crate) unk_44: [u32; 3], + /// On recovery, the host copies producer here before acknowledging resume. + pub(crate) recovery_target: u32, + pub(crate) unk_54: [u32; 3], + /// Number of entries in the item ring. + pub(crate) entries: u32, + pub(crate) unk_64: [u32; 7], +} + +static_assert!(size_of::() == 0x80); +static_assert!(core::mem::offset_of!(PointerBlock, unk_30) == 0x30); +static_assert!(core::mem::offset_of!(PointerBlock, producer) == 0x40); +static_assert!(core::mem::offset_of!(PointerBlock, entries) == 0x60); + +// SAFETY: PointerBlock contains only integers and integer atomics. +unsafe impl Zeroable for PointerBlock {} + +impl PointerBlock { + pub(crate) const PRODUCER: usize = core::mem::offset_of!(Self, producer); + pub(crate) const RECOVERY_TARGET: usize = core::mem::offset_of!(Self, recovery_target); + + /// Returns the cursors of an empty item ring. + pub(crate) const fn new() -> Self { + Self { + done: AtomicU32::new(0), + unk_04: [0; 11], + unk_30: AtomicU32::new(0), + unk_34: [0; 3], + producer: AtomicU32::new(0), + unk_44: [0; 3], + recovery_target: u32::MAX, + unk_54: [0; 3], + entries: ITEM_RING_ENTRIES, + unk_64: [0; 7], + } + } + + /// Moves the cursors of an empty queue to item-ring position `base`. + /// + /// Used when a render queue pair that was kicked before is handed to a new + /// owner, with `base` the position the host tracks for the new owner's + /// queues. Only valid while the firmware does not run the queue, with all + /// three cursors zero. Writes `done`, `unk_30` and `producer`, in this + /// order, with single stores; the caller issues a full barrier afterwards. + pub(crate) fn rebase(&self, base: u32) { + self.done.store(base, Ordering::Relaxed); + self.unk_30.store(base, Ordering::Relaxed); + self.producer.store(base, Ordering::Relaxed); + } +} + +/// Head of a queue's job list. +/// +/// Live state: the firmware links and unlinks jobs; the host writes the head +/// only at queue creation. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct JobListHead { + /// GPU address of the first job, or zero. + pub(crate) first_va: u64, + /// GPU address of the last link: the head itself when the list is empty. + pub(crate) last_va: u64, +} + +impl JobListHead { + /// Returns an empty list whose head is at `self_va`. + pub(crate) const fn new(self_va: u64) -> Self { + Self { + first_va: 0, + last_va: self_va, + } + } +} + +/// Addresses and owner a [`QueueRecord`] names. +pub(crate) struct QueueRecordArgs { + /// GPU address of the queue's [`PointerBlock`]. + pub(crate) pointers_va: u64, + /// GPU address of the queue's item ring. + pub(crate) items_va: u64, + /// GPU address of the job list the queue's work is linked into. + pub(crate) job_list_va: u64, + /// GPU address of the owning logical queue's scheduler-state page. + pub(crate) scheduler_va: u64, + /// Thread-group ID of the owning process (0 while unowned). + pub(crate) owner_pid: u32, +} + +/// Description of one hardware queue, named by its work-ring slots. +/// +/// The host writes it whole at queue creation and changes the owner fields of +/// an idle queue with [`QueueRecord::set_owner`] or +/// [`QueueRecord::set_compute_owner`]. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct QueueRecord { + /// GPU address of the [`PointerBlock`]. + pub(crate) pointers_va: u64, + /// GPU address of the item ring. + pub(crate) items_va: u64, + /// GPU address of the job list. + pub(crate) job_list_va: u64, + pub(crate) unk_18: [u8; 0xc], + pub(crate) unk_24: u32, + /// Queue priority, stored twice. + pub(crate) priority: [u32; 2], + pub(crate) unk_30: [u8; 6], + pub(crate) unk_36: u16, + /// Zero at creation; cleared again when a compute queue changes owner. + pub(crate) unk_38: u32, + pub(crate) unk_3c: u32, + /// Scheduling policy. + pub(crate) policy: u32, + pub(crate) unk_44: u32, + /// Thread-group ID of the owning process. + pub(crate) owner_pid: u32, + pub(crate) unk_4c: [u8; 0x50], + /// GPU address of the owner's scheduler-state page. + pub(crate) scheduler_va: u64, + pub(crate) unk_a4: [u8; 0x1c], +} + +static_assert!(size_of::() == 0xc0); +static_assert!(core::mem::offset_of!(QueueRecord, priority) == 0x28); +static_assert!(core::mem::offset_of!(QueueRecord, unk_38) == 0x38); +static_assert!(core::mem::offset_of!(QueueRecord, policy) == 0x40); +static_assert!(core::mem::offset_of!(QueueRecord, scheduler_va) == 0x9c); + +impl QueueRecord { + /// Builds the record of a newly created queue. + pub(crate) fn new(args: &QueueRecordArgs) -> Self { + Self { + pointers_va: args.pointers_va, + items_va: args.items_va, + job_list_va: args.job_list_va, + unk_18: [0; 0xc], + unk_24: u32::MAX, + priority: [QUEUE_PRIORITY as u32; 2], + unk_30: [0; 6], + unk_36: u16::MAX, + unk_38: 0, + unk_3c: 0, + policy: QUEUE_POLICY as u32, + unk_44: u32::MAX, + owner_pid: args.owner_pid, + unk_4c: [0; 0x50], + scheduler_va: args.scheduler_va, + unk_a4: [0; 0x1c], + } + } + + /// Hands an idle compute queue to a new owner. + /// + /// Replaces the job head, process identity and scheduler page, and clears `unk_38`. + pub(crate) fn set_compute_owner( + &mut self, + job_list_va: u64, + scheduler_va: u64, + owner_pid: u32, + ) { + self.job_list_va = job_list_va; + self.unk_38 = 0; + self.owner_pid = owner_pid; + self.scheduler_va = scheduler_va; + } +} + +/// Free list a queue's USC private memory is allocated from. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct FreeListBinding { + /// GPU address of the free list's shared control object. + pub(crate) va: u64, + /// Generation of the free list's registration. + pub(crate) generation: u64, + /// Buffer slot of the free list. + pub(crate) slot: u32, +} + +/// Parameter-buffer binding of a tiling queue. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct TilingBinding { + /// GPU address of the parameter-buffer state object. + pub(crate) pb_va: u64, + /// Allocation token of the parameter buffer's buffer slot. + pub(crate) pb_token: u64, + /// Buffer slot of the parameter buffer. + pub(crate) pb_slot: u32, + /// QID of the fragment queue paired with this tiling queue. + pub(crate) fragment_qid: u8, +} + +/// Queue whose configuration a [`QueueConfig`] carries. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum QueueConfigTarget { + /// A tiling queue and its parameter-buffer binding. + Tiling(TilingBinding), + /// A fragment queue. + Fragment, + /// A compute queue. + Compute, +} + +/// Inputs of one [`QueueConfig`]. +pub(crate) struct QueueConfigArgs { + /// Kind of queue. + pub(crate) target: QueueConfigTarget, + /// GPU address of the queue's kick ring. + pub(crate) kick_ring_va: u64, + /// Firmware alias of the queue's kick ring. + pub(crate) kick_ring_fw_va: u64, + /// Configured queue. + pub(crate) qid: u8, + /// The firmware has not yet installed this QID. + pub(crate) install: bool, + /// Completion value the firmware loads for the queue, or `None` to keep + /// the value it holds. + pub(crate) completion_seed: Option, + /// UAT context the queue's work runs in. + pub(crate) context_id: u16, + /// USC private-memory free list. + pub(crate) free_list: FreeListBinding, + /// GPU address of the owner's scheduler-state page. + pub(crate) scheduler_va: u64, + /// QoS buffer slot of the logical queue. + pub(crate) qos_slot: u8, + /// The QoS identity or queue membership changed since the last update. + pub(crate) qos_update: bool, + /// Thread-group ID of the owning process. + pub(crate) owner_pid: u32, + /// The queue's UAT context changed since the last update. + pub(crate) context_update: bool, +} + +/// Configuration of one hardware queue (tag 15). +/// +/// The firmware applies it when it consumes the command, which may be after +/// later kicks were published, so a record must not be rewritten while a +/// work pointer names it. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct QueueConfig { + /// [`CommandTag::QueueConfig`]. + pub(crate) tag: u32, + pub(crate) unk_04: u32, + /// GPU address of the queue's kick ring. + pub(crate) kick_ring_va: u64, + /// Firmware alias of the queue's kick ring. + pub(crate) kick_ring_fw_va: u64, + /// Configured queue. + pub(crate) qid: u16, + /// Non-zero to install the QID. + pub(crate) install: u16, + pub(crate) unk_1c: u16, + /// Queue priority. + pub(crate) priority: u16, + pub(crate) unk_20: u16, + /// [`DataMaster`] of the queue. + pub(crate) data_master: u16, + pub(crate) unk_24: u16, + /// Non-zero when `completion_seed` is valid. + pub(crate) completion_seed_valid: u32, + /// Completion value: kick timestamp in bits 39:0, render-pass sequence in + /// bits 47:40 (see [`KickTimestamp::completion_value`]). + pub(crate) completion_seed: u64, + /// UAT context of the queue's work. + pub(crate) context_id: u16, + pub(crate) unk_34: u16, + /// GPU address of the USC free list's shared control object. + pub(crate) free_list_va: u64, + /// Generation of the USC free list. + pub(crate) free_list_generation: u64, + /// Buffer slot of the USC free list. + pub(crate) free_list_slot: u32, + /// GPU address of the owner's scheduler-state page. + pub(crate) scheduler_va: u64, + /// Non-zero when the QoS identity or membership changed. + pub(crate) qos_update: u32, + /// QoS buffer slot. + pub(crate) qos_slot: u16, + pub(crate) unk_58: u16, + /// Thread-group ID of the owning process. + pub(crate) owner_pid: u32, + /// Scheduling policy. + pub(crate) policy: u16, + pub(crate) unk_60: u16, + /// Non-zero when the UAT context changed. + pub(crate) context_update: u16, + pub(crate) unk_64: u16, + pub(crate) unk_66: u16, + pub(crate) unk_68: [u8; 6], + /// Tiling queues: GPU address of the parameter-buffer state; else zero. + pub(crate) pb_va: u64, + /// Tiling queues: parameter-buffer slot allocation token; else all ones. + pub(crate) pb_token: u64, + /// Tiling queues: parameter-buffer slot; else all ones. + pub(crate) pb_slot: u32, + /// Tiling queues: QID of the paired fragment queue; else all ones. + pub(crate) fragment_qid: u32, + pub(crate) unk_86: [u8; 0x3a], +} + +static_assert!(size_of::() == 0xc0); +static_assert!(core::mem::offset_of!(QueueConfig, completion_seed_valid) == 0x26); +static_assert!(core::mem::offset_of!(QueueConfig, free_list_va) == 0x36); +static_assert!(core::mem::offset_of!(QueueConfig, scheduler_va) == 0x4a); +static_assert!(core::mem::offset_of!(QueueConfig, qos_slot) == 0x56); +static_assert!(core::mem::offset_of!(QueueConfig, owner_pid) == 0x5a); +static_assert!(core::mem::offset_of!(QueueConfig, pb_va) == 0x6e); +static_assert!(core::mem::offset_of!(QueueConfig, fragment_qid) == 0x82); + +impl QueueConfig { + const UNK_66: u16 = 1; + + /// Builds the record described by `args`. + pub(crate) fn new(args: &QueueConfigArgs) -> Result { + if args.qid > QID_MAX { + return Err(EINVAL); + } + let (data_master, pb_va, pb_token, pb_slot, fragment_qid) = match args.target { + QueueConfigTarget::Tiling(binding) => { + if binding.fragment_qid > QID_MAX { + return Err(EINVAL); + } + ( + DataMaster::Tiling, + binding.pb_va, + binding.pb_token, + binding.pb_slot, + binding.fragment_qid as u32, + ) + } + QueueConfigTarget::Fragment => (DataMaster::Fragment, 0, u64::MAX, u32::MAX, u32::MAX), + QueueConfigTarget::Compute => (DataMaster::Compute, 0, u64::MAX, u32::MAX, u32::MAX), + }; + Ok(Self { + tag: CommandTag::QueueConfig as u32, + unk_04: 0, + kick_ring_va: args.kick_ring_va, + kick_ring_fw_va: args.kick_ring_fw_va, + qid: args.qid as u16, + install: args.install as u16, + unk_1c: 0, + priority: QUEUE_PRIORITY as u16, + unk_20: 0, + data_master: data_master as u16, + unk_24: 0, + completion_seed_valid: args.completion_seed.is_some() as u32, + completion_seed: args.completion_seed.unwrap_or(0), + context_id: args.context_id, + unk_34: 0, + free_list_va: args.free_list.va, + free_list_generation: args.free_list.generation, + free_list_slot: args.free_list.slot, + scheduler_va: args.scheduler_va, + qos_update: args.qos_update as u32, + qos_slot: args.qos_slot as u16, + unk_58: 0, + owner_pid: args.owner_pid, + policy: QUEUE_POLICY, + unk_60: 0, + context_update: args.context_update as u16, + unk_64: 0, + unk_66: Self::UNK_66, + unk_68: [0; 6], + pb_va, + pb_token, + pb_slot, + fragment_qid, + unk_86: [0; 0x3a], + }) + } +} + +/// Announcement of a newly written kick entry of a queue (tag 14). +/// +/// A record slot larger than this structure must be zero beyond it. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct KickAnnounce { + /// [`CommandTag::KickAnnounce`]. + pub(crate) tag: u32, + /// Kicked queue. + pub(crate) qid: u16, + /// Number of entries announced. + pub(crate) count: u16, + /// Queue priority. + pub(crate) priority: u8, + /// Timestamp of the announced entry. + pub(crate) timestamp: u64, + /// [`DataMaster`] of the queue. + pub(crate) data_master: u32, + pub(crate) unk_15: [u8; 3], +} + +static_assert!(size_of::() == 0x18); +static_assert!(core::mem::offset_of!(KickAnnounce, timestamp) == 0x09); +static_assert!(core::mem::offset_of!(KickAnnounce, data_master) == 0x11); + +impl KickAnnounce { + /// Announces the entry of queue `qid` written at `timestamp`. + pub(crate) fn new(qid: u8, data_master: DataMaster, timestamp: KickTimestamp) -> Result { + if qid > QID_MAX { + return Err(EINVAL); + } + Ok(Self { + tag: CommandTag::KickAnnounce as u32, + qid: qid as u16, + count: 1, + priority: QUEUE_PRIORITY, + timestamp: timestamp.get(), + data_master: data_master as u32, + unk_15: [0; 3], + }) + } +} + +/// Names the kick of a compute queue that precedes the announced one +/// (tag 16). +/// +/// A record slot larger than this structure must be zero beyond it. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct KickPredecessor { + /// [`CommandTag::KickPredecessor`]. + pub(crate) tag: u32, + /// Kicked queue. + pub(crate) qid: u16, + /// Timestamp of the queue's kick preceding the announced one. + pub(crate) previous: u64, + pub(crate) unk_0e: u32, +} + +static_assert!(size_of::() == 0x12); + +impl KickPredecessor { + const UNK_0E: u32 = 0xfe; + + /// Names `previous` as the kick of queue `qid` preceding the announced + /// one. + pub(crate) fn new(qid: u8, previous: KickTimestamp) -> Result { + if qid > QID_MAX { + return Err(EINVAL); + } + Ok(Self { + tag: CommandTag::KickPredecessor as u32, + qid: qid as u16, + previous: previous.get(), + unk_0e: Self::UNK_0E, + }) + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs new file mode 100644 index 0000000000000..349d03ed14d71 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -0,0 +1,1309 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Render work descriptors. +//! +//! A render pass is kicked as a tiling (TA) descriptor on one queue and a +//! fragment (3D) descriptor on the queue paired with it. Each descriptor +//! embeds the register arrays its kick entry binds: one for tiling, and for +//! fragment the main array followed by the partial-render store, resume and +//! load arrays. +//! +//! Descriptors are built in place with [`TaDescriptor::write`] and +//! [`FragmentSlot::write`]; the completion link and the fragment MCache table +//! are added just before publication. + +pub(crate) mod storage; + +use kernel::prelude::*; + +use super::kick::{ + KickTimestamp, McacheClass, McacheRange, McacheTable, RegisterArray, RegisterArrayBinding, + RegisterWrite, WriteRange, MCACHE_INLINE_RANGES, +}; +use super::queue::CommandTag; +use super::{ + clear, offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, + UNK_0D411, UNK_101D9, UNK_10791, UNK_1A0E9, UNK_1C8F8, +}; + +/// Size of a tiling descriptor. +pub(crate) const TA_DESCRIPTOR_SIZE: usize = 0x9c0; +/// Size of a [`FragmentSlot`]: the descriptor and its MCache spill table. +pub(crate) const FRAGMENT_SLOT_SIZE: usize = 0x2380; +/// Write ranges of one fragment pass: its attachments and the auxiliary +/// framebuffer. +pub(crate) const MCACHE_RANGES_MAX: usize = 17; +/// Size of the auxiliary framebuffer of a pass. +pub(crate) const AUX_FB_SIZE: u64 = 0x8000; + +/// QoS class of tiling kicks. +pub(crate) const TA_QOS_CLASS: u8 = 0x0c; +/// QoS class of fragment kicks. +pub(crate) const FRAGMENT_QOS_CLASS: u8 = 0x18; +/// Event mask of render kicks. +pub(crate) const RENDER_KICK_EVENT_MASK: [u64; 4] = [0; 4]; + +/// Base of the user address window. Tiling resource registers hold offsets +/// from it rather than addresses. +const USER_BASE: u64 = 0x10_0000_0000; +/// The VDM control-stream register holds a 39-bit offset from [`USER_BASE`]. +const VDM_OFFSET_LIMIT: u64 = 1 << 39; +/// The deflake buffer holds three sub-buffers: at its start and at these +/// offsets. +const DEFLAKE_20: u64 = 0x20; +const DEFLAKE_2A0: u64 = 0x2a0; + +const TA_REGISTERS: usize = 73; +const FRAGMENT_REGISTERS: usize = 89; +const PARTIAL_STORE_REGISTERS: usize = 16; +const PARTIAL_RESUME_REGISTERS: usize = 23; +const PARTIAL_LOAD_REGISTERS: usize = 10; + +/// Checkpoints into the tiling array, as entry counts: before the final two +/// entries, then after the last one (twice). +const TA_CHECKPOINTS: [u16; 3] = [ + TA_REGISTERS as u16 - 2, + TA_REGISTERS as u16, + TA_REGISTERS as u16, +]; +/// Checkpoints into the main fragment array, as entry counts: after the first +/// of the four closing entries, then after the second (twice). +const FRAGMENT_CHECKPOINTS: [u16; 3] = [ + FRAGMENT_REGISTERS as u16 - 3, + FRAGMENT_REGISTERS as u16 - 2, + FRAGMENT_REGISTERS as u16 - 2, +]; + +const TILE_SIZE: u64 = 32; +const MACRO_TILES: u64 = 4; +const REGION_ENTRY_SIZE: u64 = 5; +/// Highest number of samples per pixel, as a power of two. +const SAMPLES_LOG2_MAX: u8 = 2; +/// Bytes of tile buffer per block. +const TIB_BLOCK_SIZE: u64 = 2048; +const TILE_CONFIG_BASE: u64 = 0x280; +const TILE_CONFIG_LAYERED: u64 = 1 << 0; +const TILE_CONFIG_PROCESS_EMPTY_TILES: u64 = 1 << 16; +const AUX_FB_FLAGS_BASE: u64 = 0xc000; +const AUX_FB_FLAGS_DBIAS_INT: u64 = 1 << 18; +/// Tiling layer mode: layer count less one, bit 15, and for layered targets +/// bit 14 plus the empty-tile policy. The second copy keeps only the count and +/// bit 14. +const LAYER_MODE_BASE: u64 = 1 << 15; +const LAYER_MODE_LAYERED: u64 = 1 << 14; +const LAYER_MODE_PROCESS_EMPTY: u64 = 1 << 13; +const LAYER_MODE_SKIP_EMPTY: u64 = 1 << 12; +const LAYER_MODE_PIPE_MASK: u64 = 0x47ff; +/// Fragment tile state: required bits 0x3717f and 37, layered targets in bit +/// 32, the utile size bits of the tile-buffer configuration in bits 43:40, +/// tiles in X less one from bit 44 and tiles in Y less one from bit 53. +const TILE_STATE_BASE: u64 = 0x3717f | 1 << 37; +const TILE_STATE_LAYERED: u64 = 1 << 32; +const TILE_STATE_UTILE_SHIFT: u32 = 28; +const TILE_STATE_TILES_X_SHIFT: u32 = 44; +const TILE_STATE_TILES_Y_SHIFT: u32 = 53; +/// Tile-buffer configuration: utile width / 16 in bits 13:12, utile height / +/// 16 in bits 15:14, samples (log2) in bits 1:0. +const UTILE_CONFIG_WIDTH_SHIFT: u32 = 12; +const UTILE_CONFIG_HEIGHT_SHIFT: u32 = 14; +const UTILE_CONFIG_SIZE_MASK: u64 = 0xf000; +/// The partial-render phases load and store every attached depth and stencil +/// surface. +const ZLS_PARTIAL_DEPTH: u64 = 1 << 15 | 1 << 19; +const ZLS_PARTIAL_STENCIL: u64 = 1 << 14 | 1 << 18; +const STENCIL_CLEAR_BASE: u32 = 0x300; +/// Value of the ZLS swizzle registers and descriptor field. +const ZLS_SWIZZLE: u32 = 0x0404_0404; +/// Value of register 0x15359 in the partial store and resume arrays; the +/// main array writes zero. +const UNK_15359_PARTIAL: u64 = 0x20; +/// Required value of register 0x15049 and of fragment descriptor field +/// +0x1f50. +const UNK_15049: u64 = 0x10_0000; +/// Value of the fragment register 0x10791. +const UNK_10791_FRAGMENT: u64 = UNK_10791 | 1 << 9; +/// USC program offsets carry tag bits that are not part of the address. +const USC_OFFSET_TAG_MASK: u32 = 7; + +/// A depth or stencil surface. +#[derive(Copy, Clone, Debug)] +pub(crate) struct ZlsSurface { + /// GPU address of the surface, or zero when it is not attached. + pub(crate) va: u64, + /// GPU address of its compression metadata. + pub(crate) compression_va: u64, + /// Stride of the surface. + pub(crate) stride: u64, + /// Stride of the compression metadata. + pub(crate) compression_stride: u64, +} + +/// A USC program and the resource specifier it is launched with. +#[derive(Copy, Clone, Debug)] +pub(crate) struct UscProgram { + /// Tagged offset of the program in the queue's USC window. + pub(crate) offset: u32, + /// Resource specifier. + pub(crate) rsrc_spec: u64, +} + +/// Render-pass state from the userspace command. +#[derive(Copy, Clone, Debug)] +pub(crate) struct RenderPass { + /// Width of the render target in pixels. + pub(crate) width: u32, + /// Height of the render target in pixels. + pub(crate) height: u32, + /// Number of layers, 1 to 2048. + pub(crate) layers: u16, + /// Utile width: 16 or 32. + pub(crate) utile_width: u8, + /// Utile height: 16 or 32, at most the width. + pub(crate) utile_height: u8, + /// Samples per pixel, as a power of two: 0 to 2. + pub(crate) samples_log2: u8, + /// Bytes of tile buffer per sample. + pub(crate) sample_size: u8, + /// Process tiles no primitive touches. + pub(crate) process_empty_tiles: bool, + /// Multisample control. + pub(crate) msaa_control: u64, + /// PPP control. + pub(crate) ppp_control: u64, + /// GPU address of the VDM control stream. + pub(crate) vdm_va: u64, + /// GPU address of the scissor array. + pub(crate) scissor_va: u64, + /// GPU address of the depth-bias array, or zero. + pub(crate) depth_bias_va: u64, + /// Depth-bias values are integers. + pub(crate) depth_bias_is_int: bool, + /// GPU address of the occlusion-query results, or zero. + pub(crate) occlusion_query_va: u64, + /// GPU address of the sampler heap, or zero. + pub(crate) sampler_heap_va: u64, + /// Number of samplers in the heap. + pub(crate) sampler_count: u32, + /// ZLS control at the pass boundaries. + pub(crate) zls_control: u64, + /// Depth surface dimensions. + pub(crate) depth_dimensions: u64, + /// Depth surface. + pub(crate) depth: ZlsSurface, + /// Stencil surface. + pub(crate) stencil: ZlsSurface, + /// Depth clear value (IEEE-754 bits). + pub(crate) depth_clear: u32, + /// Stencil clear value (background object register value). + pub(crate) stencil_clear: u32, + /// Merge upper bound in X (IEEE-754 bits). + pub(crate) merge_upper_x: u32, + /// Merge upper bound in Y (IEEE-754 bits). + pub(crate) merge_upper_y: u32, + /// Base of the queue's USC window. + pub(crate) usc_base: u64, + /// Background program. + pub(crate) bg: UscProgram, + /// Partial-render background program. + pub(crate) partial_bg: UscProgram, + /// Tagged USC offset of the end-of-tile program. Its resource specifier + /// is always written as zero. + pub(crate) eot_offset: u32, + /// Tagged USC offset of the partial-render end-of-tile program. + pub(crate) partial_eot_offset: u32, + /// The resource specifiers have their high 32 bits. + pub(crate) rsrc_spec_hi: bool, +} + +/// Kernel-owned objects and identities a render pass uses. +#[derive(Copy, Clone, Debug)] +pub(crate) struct RenderResources { + /// UAT context the pass runs in. + pub(crate) context_id: u16, + /// Allocation generation of that context. + pub(crate) context_generation: u8, + /// Buffer slot of the parameter buffer. + pub(crate) pb_slot: u32, + /// Buffer slot of the USC private-memory free list. + pub(crate) free_list_slot: u32, + /// GPU address of the free list's shared control object. + pub(crate) free_list_control_va: u64, + /// GPU address of the pass's kick record, in which the firmware counts + /// the pass's completed kicks. + pub(crate) kick_record_va: u64, + /// GPU address of the parameter-buffer state object. + pub(crate) pb_va: u64, + /// GPU address of the parameter-buffer scene entry of the pass. + pub(crate) scene_va: u64, + /// GPU address following the last scene entry. + pub(crate) scene_end_va: u64, + /// Parameter-buffer scratch offset register value. + pub(crate) pm_scratch: u64, + /// Parameter-buffer metric pointer register value. + pub(crate) pm_metric: u64, + /// GPU address of the tile map. + pub(crate) tilemap_va: u64, + /// GPU address of the layer metadata. + pub(crate) layermeta_va: u64, + /// GPU address of the heap metadata. + pub(crate) heapmeta_va: u64, + /// GPU address of the tail-pointer cache. + pub(crate) tpc_va: u64, + /// Size of the tail-pointer cache the target needs. + pub(crate) tpc_size: u64, + /// GPU address of the deflake buffer. + pub(crate) deflake_va: u64, + /// GPU address of the tiling status word. + pub(crate) ta_status_va: u64, + /// GPU address of the fragment status word, as seen by the pass's VM. + pub(crate) fragment_status_va: u64, + /// Firmware alias of the fragment status word. + pub(crate) fragment_status_fw_va: u64, + /// GPU address of the auxiliary framebuffer. + pub(crate) aux_fb_va: u64, + /// GPU address of the pass's start/end timestamp pair. + pub(crate) timestamp_va: u64, + /// Object IDs of the tiling stage. + pub(crate) ta_ids: ObjectIds, + /// Object IDs of the fragment stage. + pub(crate) fragment_ids: ObjectIds, + /// Value of fragment descriptor field +0x2104: bit 0 of GPU register + /// 0xe0141c, read at probe. + pub(crate) unk_2104: u32, + /// Value of fragment descriptor field +0x210c: bits 3:0 of GPU register + /// 0xe01480, read at probe. + pub(crate) unk_210c: u32, +} + +/// Per-stage inputs of a render pass. +#[derive(Copy, Clone, Debug)] +pub(crate) struct RenderStage { + /// Queue the stage is kicked on. + pub(crate) qid: u8, + /// GPU address of the stage's descriptor, as seen by the pass's VM. + pub(crate) descriptor_va: u64, + /// Command-buffer state word of the stage. + pub(crate) state_word: u32, + /// GPU address of the queue's completion stamp word. + pub(crate) stamp_va: u64, + /// GPU address of the queue's word in the auxiliary stamp page. + pub(crate) aux_stamp_va: u64, +} + +/// Inputs of both render descriptors. +pub(crate) struct RenderArgs<'a> { + /// Render-pass state. + pub(crate) pass: &'a RenderPass, + /// Kernel-owned objects. + pub(crate) resources: &'a RenderResources, + /// Submission ordinal of the render queue pair, from zero. + pub(crate) ordinal: u64, + /// Tiling stage. + pub(crate) ta: RenderStage, + /// Fragment stage. + pub(crate) fragment: RenderStage, +} + +impl RenderArgs<'_> { + /// Completion stamp value of this pass: the pass's sequence number is the + /// pair's submission ordinal plus one. + fn stamp_value(&self) -> u32 { + stamp_value((self.ordinal as u32).wrapping_add(1)) + } +} + +/// Tile-grid geometry of a render target, in the units its registers take. +struct Geometry { + tiles_x: u64, + tiles_y: u64, + /// Size of the region array. + region_size: u64, + /// Number of macro tiles. + macro_tiles: u64, + /// Two utile slots per utile of every macro tile. + utile_slots: u64, + /// Macro-tile columns times 3, 2 and 1, in 9-bit fields from bit 0. + x_blocks: u64, + /// Macro-tile rows times 3, 2 and 1, in 9-bit fields from bit 0. + y_blocks: u64, + /// Tiles in Y less one from bit 12, tiles in X less one below. + screen: u64, + /// Height less one from bit 16, width less one below. + pixels: u64, + /// Macro-tile width in utiles from bit 16, height in utiles below. + macro_size: u64, +} + +impl Geometry { + fn new(pass: &RenderPass) -> Result { + if pass.width == 0 || pass.height == 0 || pass.layers == 0 || pass.layers > 2048 { + return Err(EINVAL); + } + if !matches!( + (pass.utile_width, pass.utile_height), + (32, 32) | (32, 16) | (16, 16) + ) { + return Err(EINVAL); + } + let width = pass.width as u64; + let height = pass.height as u64; + let tiles_x = width.div_ceil(TILE_SIZE); + let tiles_y = height.div_ceil(TILE_SIZE); + let utiles_x = TILE_SIZE / pass.utile_width as u64; + let utiles_y = TILE_SIZE / pass.utile_height as u64; + let utiles = utiles_x * utiles_y; + let mtile_x = tiles_x.div_ceil(MACRO_TILES).next_multiple_of(4); + let mtile_y = tiles_y.div_ceil(MACRO_TILES).next_multiple_of(4); + let macro_tiles = mtile_x * mtile_y; + Ok(Self { + tiles_x, + tiles_y, + region_size: (REGION_ENTRY_SIZE * macro_tiles * utiles).div_ceil(4), + macro_tiles, + utile_slots: 2 * macro_tiles * utiles, + x_blocks: (3 * mtile_x) | ((2 * mtile_x) << 9) | (mtile_x << 18), + y_blocks: (3 * mtile_y) | ((2 * mtile_y) << 9) | (mtile_y << 18), + screen: (tiles_y - 1) << 12 | (tiles_x - 1), + pixels: (width - 1) | (height - 1) << 16, + macro_size: (mtile_y * utiles_y) | ((mtile_x * utiles_x) << 16), + }) + } +} + +/// Register and descriptor words derived from the [`RenderPass`]. +struct PassWords { + utile_config: u64, + tile_config: u64, + layer_mode: u64, + tib_blocks: u64, + partial_zls_control: u64, + aux_fb_flags: u64, + bg_rsrc_spec: u64, + bg_va: u64, + partial_bg_rsrc_spec: u64, + partial_bg_va: u64, + eot_va: u64, + partial_eot_va: u64, +} + +impl PassWords { + fn new(pass: &RenderPass) -> Result { + if pass.samples_log2 > SAMPLES_LOG2_MAX { + return Err(EINVAL); + } + let samples = 1u64 << pass.samples_log2; + let layered = pass.layers > 1; + + // The tile configuration carries the empty-tile request for every + // target; the tiling layer mode only for layered ones. + let mut tile_config = TILE_CONFIG_BASE; + let mut layer_mode = LAYER_MODE_BASE | (pass.layers as u64 - 1); + if layered { + tile_config |= TILE_CONFIG_LAYERED; + layer_mode |= LAYER_MODE_LAYERED; + layer_mode |= match pass.process_empty_tiles { + true => LAYER_MODE_PROCESS_EMPTY, + false => LAYER_MODE_SKIP_EMPTY, + }; + } + if pass.process_empty_tiles { + tile_config |= TILE_CONFIG_PROCESS_EMPTY_TILES; + } + + let mut partial_zls_control = pass.zls_control; + if pass.depth.va != 0 { + partial_zls_control |= ZLS_PARTIAL_DEPTH; + } + if pass.stencil.va != 0 { + partial_zls_control |= ZLS_PARTIAL_STENCIL; + } + + let aux_fb_flags = match pass.depth_bias_is_int { + true => AUX_FB_FLAGS_BASE | AUX_FB_FLAGS_DBIAS_INT, + false => AUX_FB_FLAGS_BASE, + }; + let rsrc_spec = |spec: u64| match pass.rsrc_spec_hi { + true => spec, + false => spec & u32::MAX as u64, + }; + let usc = |offset: u32| offset_va(pass.usc_base, (offset & !USC_OFFSET_TAG_MASK) as u64); + + Ok(Self { + utile_config: (pass.utile_width as u64 / 16) << UTILE_CONFIG_WIDTH_SHIFT + | (pass.utile_height as u64 / 16) << UTILE_CONFIG_HEIGHT_SHIFT + | pass.samples_log2 as u64, + tile_config, + layer_mode, + tib_blocks: (pass.sample_size as u64 + * pass.utile_width as u64 + * pass.utile_height as u64 + * samples) + .div_ceil(TIB_BLOCK_SIZE), + partial_zls_control, + aux_fb_flags, + bg_rsrc_spec: rsrc_spec(pass.bg.rsrc_spec), + bg_va: usc(pass.bg.offset)?, + partial_bg_rsrc_spec: rsrc_spec(pass.partial_bg.rsrc_spec), + partial_bg_va: usc(pass.partial_bg.offset)?, + eot_va: usc(pass.eot_offset)?, + partial_eot_va: usc(pass.partial_eot_offset)?, + }) + } +} + +/// Offset of an address from the user window base. +fn user_offset(va: u64) -> Result { + va.checked_sub(USER_BASE).ok_or(EINVAL) +} + +/// Offset of an address from the user window base that fits 32 bits. +fn user_offset32(va: u64) -> Result { + u32::try_from(user_offset(va)?).map_err(|_| EINVAL) +} + +fn ta_registers( + args: &RenderArgs<'_>, + g: &Geometry, + w: &PassWords, +) -> Result<[RegisterWrite; TA_REGISTERS]> { + let p = args.pass; + let r = args.resources; + let tilemap = user_offset(r.tilemap_va)?; + let layermeta = user_offset(r.layermeta_va)?; + let heapmeta = user_offset(r.heapmeta_va)? | 1 << 63; + let tpc = user_offset(r.tpc_va)?; + let deflake = user_offset32(r.deflake_va)? as u64; + let deflake_20 = user_offset32(offset_va(r.deflake_va, DEFLAKE_20)?)? as u64; + let deflake_2a0 = user_offset32(offset_va(r.deflake_va, DEFLAKE_2A0)?)? as u64; + let vdm = user_offset(p.vdm_va)?; + if vdm >= VDM_OFFSET_LIMIT { + return Err(EINVAL); + } + let key = work_key(r.context_id, args.ta.state_word); + let ids = r.ta_ids.word(); + + Ok([ + RegisterWrite::new(0x01748, 1), + RegisterWrite::new(0x10141, 0x200), + RegisterWrite::new(0x1c039, tilemap), + RegisterWrite::new(0x1c9c8, tilemap), + RegisterWrite::new(0x1c0a1, tpc), + RegisterWrite::new(0x1c031, heapmeta), + RegisterWrite::new(0x1c9c0, heapmeta), + RegisterWrite::new(0x1c051, 0x003a_0012_006b_0003), + RegisterWrite::new(0x1c061, 1), + RegisterWrite::new(0x10149, w.utile_config), + RegisterWrite::new(0x10139, p.msaa_control), + RegisterWrite::new(0x10111, deflake_2a0), + RegisterWrite::new(0x1c9b0, deflake_2a0), + RegisterWrite::new(0x10119, deflake_20), + RegisterWrite::new(0x1c9b8, deflake_20), + RegisterWrite::new(0x1c958, 1), + RegisterWrite::new(0x1c950, deflake | 1 << 50), + RegisterWrite::new(0x1c930, 0), + RegisterWrite::new(0x1c880, vdm), + RegisterWrite::new(0x1c079, layermeta), + RegisterWrite::new(0x1c9d8, layermeta), + RegisterWrite::new(0x10151, 0), + RegisterWrite::new(0x1c199, 0), + RegisterWrite::new(0x1c1a1, 0), + RegisterWrite::new(0x1c1a9, 0), + RegisterWrite::new(0x1c1b1, 0), + RegisterWrite::new(0x1c1b9, 0), + RegisterWrite::new(0x1c8f8, UNK_1C8F8), + RegisterWrite::new(0x1c0b1, g.region_size), + RegisterWrite::new(0x1c850, g.region_size), + RegisterWrite::new(0x10131, p.msaa_control), + RegisterWrite::new(0x10121, p.ppp_control), + RegisterWrite::new(0x10129, g.pixels), + RegisterWrite::new(0x101b9, g.screen), + RegisterWrite::new(0x1c069, g.x_blocks), + RegisterWrite::new(0x1c071, g.y_blocks), + RegisterWrite::new(0x1c081, g.macro_tiles), + RegisterWrite::new(0x1c0a9, g.utile_slots), + RegisterWrite::new(0x10171, 0x100), + RegisterWrite::new(0x10169, w.layer_mode), + RegisterWrite::new(0x0a309, 0), + RegisterWrite::new(0x1c8e0, u64::MAX), + RegisterWrite::new(0x1c8e8, u64::MAX), + RegisterWrite::new(0x1c898, 0), + RegisterWrite::new(0x101e1, UNK_101D9), + RegisterWrite::new(0x1c9e8, w.layer_mode & LAYER_MODE_PIPE_MASK), + RegisterWrite::new(0x1a099, 0), + RegisterWrite::new(0x1a0a1, 0), + RegisterWrite::new(0x1a069, 0), + RegisterWrite::new(0x1a071, 0), + RegisterWrite::new(0x1a0c9, 0), + RegisterWrite::new(0x1a0d1, 0), + RegisterWrite::new(0x101c9, 0), + RegisterWrite::new(0x0d471, 0), + RegisterWrite::new(0x1a0f1, UNK_1A0E9), + RegisterWrite::new(0x10799, UNK_10791), + RegisterWrite::new(0x1c830, r.pb_slot as u64), + RegisterWrite::new(0x1ca30, r.pm_scratch), + RegisterWrite::new(0x16c39, r.pm_scratch), + RegisterWrite::new(0x1c910, r.pm_metric), + RegisterWrite::new(0x0a5a1, UNK_0A599), + RegisterWrite::new(0x0d419, UNK_0D411), + RegisterWrite::new(0x1ca10, ids), + RegisterWrite::new(0x014a1, ids), + RegisterWrite::new(0x0a349, ids), + RegisterWrite::new(0x10209, key), + RegisterWrite::new(0x1c9f0, key), + RegisterWrite::new(0x14320, key), + RegisterWrite::new(0x14308, r.free_list_slot as u64), + RegisterWrite::new(0x14318, r.ta_status_va | 1), + RegisterWrite::new(0x01740, 1), + RegisterWrite::new(0x1c880, deflake_2a0), + RegisterWrite::new(0x1c898, 1), + ]) +} + +fn fragment_registers( + args: &RenderArgs<'_>, + g: &Geometry, + w: &PassWords, +) -> [RegisterWrite; FRAGMENT_REGISTERS] { + let p = args.pass; + let r = args.resources; + let mut tile_state = TILE_STATE_BASE + | (w.utile_config & UTILE_CONFIG_SIZE_MASK) << TILE_STATE_UTILE_SHIFT + | (g.tiles_x - 1) << TILE_STATE_TILES_X_SHIFT + | (g.tiles_y - 1) << TILE_STATE_TILES_Y_SHIFT; + if p.layers > 1 { + tile_state |= TILE_STATE_LAYERED; + } + let key = work_key(r.context_id, args.fragment.state_word); + let ids = r.fragment_ids.word(); + + [ + RegisterWrite::new(0x01739, 1), + RegisterWrite::new(0x10009, w.utile_config), + // End-of-tile resource specifier. + RegisterWrite::new(0x15379, 0), + RegisterWrite::new(0x15381, w.eot_va), + RegisterWrite::new(0x15369, w.bg_rsrc_spec), + RegisterWrite::new(0x15371, w.bg_va), + RegisterWrite::new(0x15131, p.merge_upper_x as u64), + RegisterWrite::new(0x15139, p.merge_upper_y as u64), + RegisterWrite::new(0x100a1, 0), + RegisterWrite::new(0x15069, 0), + RegisterWrite::new(0x15071, 0), + RegisterWrite::new(0x16058, 0), + RegisterWrite::new(0x10019, p.msaa_control), + RegisterWrite::new(0x100b1, g.macro_size), + RegisterWrite::new(0x16030, g.macro_size), + RegisterWrite::new(0x100d9, g.screen), + RegisterWrite::new(0x0a301, 0), + RegisterWrite::new(0x10791, UNK_10791_FRAGMENT), + RegisterWrite::new(0x16098, r.heapmeta_va), + RegisterWrite::new(0x15109, p.scissor_va), + RegisterWrite::new(0x15101, p.depth_bias_va), + RegisterWrite::new(0x15021, w.aux_fb_flags), + RegisterWrite::new(0x15211, (p.height as u64) << 32 | p.width as u64), + RegisterWrite::new(0x15049, UNK_15049), + RegisterWrite::new(0x10051, w.tib_blocks), + RegisterWrite::new(0x15321, p.depth_dimensions), + RegisterWrite::new(0x15301, p.depth_clear as u64), + RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), + RegisterWrite::new(0x15311, p.occlusion_query_va), + RegisterWrite::new(0x15319, p.zls_control), + RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + RegisterWrite::new(0x15351, 0), + RegisterWrite::new(0x15329, p.depth.va), + RegisterWrite::new(0x15331, p.depth.va), + RegisterWrite::new(0x15339, p.stencil.va), + RegisterWrite::new(0x15341, p.stencil.va), + RegisterWrite::new(0x15231, 0), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15239, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15401, p.depth.stride), + RegisterWrite::new(0x15421, p.depth.stride), + RegisterWrite::new(0x15409, p.stencil.stride), + RegisterWrite::new(0x15429, p.stencil.stride), + RegisterWrite::new(0x153c1, p.depth.compression_va), + RegisterWrite::new(0x15411, p.depth.compression_stride), + RegisterWrite::new(0x153c9, p.depth.compression_va), + RegisterWrite::new(0x15431, p.depth.compression_stride), + RegisterWrite::new(0x153d1, p.stencil.compression_va), + RegisterWrite::new(0x15419, p.stencil.compression_stride), + RegisterWrite::new(0x153d9, p.stencil.compression_va), + RegisterWrite::new(0x15439, p.stencil.compression_stride), + RegisterWrite::new(0x16429, r.tilemap_va), + RegisterWrite::new(0x16060, r.layermeta_va), + RegisterWrite::new(0x16431, (4 * g.region_size) << 24), + RegisterWrite::new(0x10039, w.tile_config), + RegisterWrite::new(0x16020, 0), + RegisterWrite::new(0x16451, 0), + RegisterWrite::new(0x15359, 0), + RegisterWrite::new(0x100b8, UNK_1C8F8), + RegisterWrite::new(0x16461, r.aux_fb_va), + RegisterWrite::new(0x16090, r.aux_fb_va), + RegisterWrite::new(0x101e9, UNK_101D9), + RegisterWrite::new(0x160a8, 0), + RegisterWrite::new(0x16068, tile_state), + RegisterWrite::new(0x1a0a9, 0), + RegisterWrite::new(0x1a0b1, 0), + RegisterWrite::new(0x1a079, 0), + RegisterWrite::new(0x1a081, 0), + RegisterWrite::new(0x1a0d9, 0), + RegisterWrite::new(0x1a0e1, 0), + RegisterWrite::new(0x101c1, 0), + RegisterWrite::new(0x0d469, 0), + RegisterWrite::new(0x1a0f9, UNK_1A0E9), + RegisterWrite::new(0x0a5a9, UNK_0A599), + RegisterWrite::new(0x0d429, UNK_0D411), + RegisterWrite::new(0x160e0, ids), + RegisterWrite::new(0x01499, ids), + RegisterWrite::new(0x0a341, ids), + RegisterWrite::new(0x1c838, r.pb_slot as u64), + RegisterWrite::new(0x1ca28, r.pm_scratch), + RegisterWrite::new(0x10211, key), + RegisterWrite::new(0x10420, key), + RegisterWrite::new(0x14048, r.free_list_slot as u64), + RegisterWrite::new(0x14080, r.fragment_status_va | 1), + RegisterWrite::new(0x01731, 1), + RegisterWrite::new(0x16020, 1), + RegisterWrite::new(0x16020, 0), + RegisterWrite::new(0x16068, 0x4_0000), + ] +} + +fn partial_store_registers( + p: &RenderPass, + w: &PassWords, +) -> [RegisterWrite; PARTIAL_STORE_REGISTERS] { + [ + // End-of-tile resource specifier. + RegisterWrite::new(0x15379, 0), + RegisterWrite::new(0x15381, w.partial_eot_va), + RegisterWrite::new(0x10039, w.tile_config), + RegisterWrite::new(0x15359, UNK_15359_PARTIAL), + RegisterWrite::new(0x15331, p.depth.va), + RegisterWrite::new(0x153c9, p.depth.compression_va), + RegisterWrite::new(0x15341, p.stencil.va), + RegisterWrite::new(0x153d9, p.stencil.compression_va), + RegisterWrite::new(0x15421, p.depth.stride), + RegisterWrite::new(0x15431, p.depth.compression_stride), + RegisterWrite::new(0x15429, p.stencil.stride), + RegisterWrite::new(0x15439, p.stencil.compression_stride), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15319, w.partial_zls_control), + RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + ] +} + +fn partial_resume_registers( + p: &RenderPass, + w: &PassWords, +) -> [RegisterWrite; PARTIAL_RESUME_REGISTERS] { + [ + // End-of-tile resource specifier. + RegisterWrite::new(0x15379, 0), + RegisterWrite::new(0x15381, w.partial_eot_va), + RegisterWrite::new(0x15369, w.partial_bg_rsrc_spec), + RegisterWrite::new(0x15371, w.partial_bg_va), + RegisterWrite::new(0x10039, w.tile_config & !TILE_CONFIG_PROCESS_EMPTY_TILES), + RegisterWrite::new(0x15359, UNK_15359_PARTIAL), + RegisterWrite::new(0x15331, p.depth.va), + RegisterWrite::new(0x153c9, p.depth.compression_va), + RegisterWrite::new(0x15341, p.stencil.va), + RegisterWrite::new(0x153d9, p.stencil.compression_va), + RegisterWrite::new(0x15421, p.depth.stride), + RegisterWrite::new(0x15431, p.depth.compression_stride), + RegisterWrite::new(0x15429, p.stencil.stride), + RegisterWrite::new(0x15439, p.stencil.compression_stride), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), + RegisterWrite::new(0x15329, p.depth.va), + RegisterWrite::new(0x153c1, p.depth.compression_va), + RegisterWrite::new(0x15339, p.stencil.va), + RegisterWrite::new(0x153d1, p.stencil.compression_va), + RegisterWrite::new(0x15319, w.partial_zls_control), + RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + ] +} + +fn partial_load_registers( + p: &RenderPass, + w: &PassWords, +) -> [RegisterWrite; PARTIAL_LOAD_REGISTERS] { + [ + RegisterWrite::new(0x15369, w.partial_bg_rsrc_spec), + RegisterWrite::new(0x15371, w.partial_bg_va), + RegisterWrite::new(0x10039, w.tile_config & !TILE_CONFIG_PROCESS_EMPTY_TILES), + RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), + RegisterWrite::new(0x15329, p.depth.va), + RegisterWrite::new(0x153c1, p.depth.compression_va), + RegisterWrite::new(0x15339, p.stencil.va), + RegisterWrite::new(0x153d1, p.stencil.compression_va), + RegisterWrite::new(0x15319, w.partial_zls_control), + RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + ] +} + +/// Tiling (TA) work descriptor (tag 0). +#[repr(C, packed)] +#[derive(Debug)] +pub(crate) struct TaDescriptor { + /// [`CommandTag::Tiling`]. + pub(crate) tag: u32, + /// Twice the pair's submission ordinal, plus one. + pub(crate) submit_sequence: u64, + /// UAT context of the pass. + pub(crate) context_id: u32, + /// GPU address of the pass's kick record. + pub(crate) kick_record_va: u64, + /// Buffer slot of the parameter buffer. + pub(crate) pb_slot: u32, + pub(crate) unk_1c: u32, + /// GPU address of the parameter-buffer state object. + pub(crate) pb_va: u64, + /// GPU address of the parameter-buffer scene entry. + pub(crate) scene_va: u64, + /// GPU address following the last scene entry. + pub(crate) scene_end_va: u64, + /// Checkpoints into the register array. + pub(crate) checkpoints: [u16; 3], + pub(crate) unk_3e: [u8; 0xa], + /// The pair's submission ordinal plus half of it. + pub(crate) unk_48: u32, + pub(crate) unk_4c: [u8; 0x14], + /// Register array. + pub(crate) registers: RegisterArray, + /// GPU address of the tail-pointer cache. + pub(crate) tpc_va: u64, + /// Size of the tail-pointer cache the target needs. + pub(crate) tpc_size: u64, + pub(crate) unk_790: [u8; 0xc], + /// QID of the fragment queue that consumes this pass. + pub(crate) fragment_qid: u32, + /// Completion stamp value of the pass. + pub(crate) stamp_value: u32, + /// Non-zero when the fragment queue had earlier kicks outstanding. + pub(crate) fragment_busy: u32, + /// Timestamp of the fragment kick of this pass. + pub(crate) fragment_kick: u64, + /// Render-pass sequence of the pass, stored twice. + pub(crate) pass_seq: [u8; 2], + pub(crate) unk_7b2: [u8; 0x24], + /// Offset from the user base of the deflake sub-buffer at +0x2a0. + pub(crate) deflake_2a0_offset: u32, + pub(crate) unk_7da: [u8; 0x94], + /// Predecessor object ID of the tiling stage. + pub(crate) predecessor_id: u32, + pub(crate) unk_872: u32, + pub(crate) unk_876: u32, + /// GPU address of the sampler heap. + pub(crate) sampler_heap_va: u64, + /// Number of samplers. + pub(crate) sampler_count: u32, + /// Number of samplers plus one, or zero without samplers. + pub(crate) sampler_max: u32, + pub(crate) unk_88a: [u8; 8], + pub(crate) unk_892: u32, + pub(crate) unk_896: [u8; 0x10], + /// GPU address of the tiling queue's completion stamp word. + pub(crate) stamp_va: u64, + /// GPU address of the tiling queue's auxiliary stamp word. + pub(crate) aux_stamp_va: u64, + /// Completion stamp value of the pass. + pub(crate) stamp_value_2: u32, + /// QID of the tiling queue. + pub(crate) qid: u32, + pub(crate) unk_8be: u32, + pub(crate) unk_8c2: u32, + /// Completion value of the tiling kick (see + /// [`KickTimestamp::completion_value`]). + pub(crate) completion: u64, + /// Current object ID of the tiling stage. + pub(crate) current_id: u64, + /// Submission ordinal of the pair. + pub(crate) ordinal: u32, + pub(crate) unk_8da: [u8; 0x24], + /// GPU address the pass start timestamp is written to. + pub(crate) timestamp_start_va: u64, + /// Zero: the tiling stage writes no end timestamp. + pub(crate) timestamp_end_va: u64, + pub(crate) unk_90e: [u8; 0x24], + pub(crate) unk_932: u8, + pub(crate) unk_933: u8, + /// GPU address of the USC free list's shared control object. + pub(crate) free_list_control_va: u64, + pub(crate) unk_93c: u8, + pub(crate) unk_93d: u8, + /// Set on the first kick of a queue pair. + pub(crate) unk_93e: [u8; 2], + pub(crate) unk_940: [u8; 5], + /// GPU address of the tiling status word. + pub(crate) status_va: u64, + /// Allocation generation of the UAT context. + pub(crate) context_generation: u8, + pub(crate) unk_94e: [u8; 0x72], +} + +static_assert!(size_of::() == TA_DESCRIPTOR_SIZE); +static_assert!(core::mem::offset_of!(TaDescriptor, unk_48) == 0x48); +static_assert!(core::mem::offset_of!(TaDescriptor, registers) == 0x60); +static_assert!(core::mem::offset_of!(TaDescriptor, tpc_va) == 0x780); +static_assert!(core::mem::offset_of!(TaDescriptor, fragment_qid) == 0x79c); +static_assert!(core::mem::offset_of!(TaDescriptor, deflake_2a0_offset) == 0x7d6); +static_assert!(core::mem::offset_of!(TaDescriptor, predecessor_id) == 0x86e); +static_assert!(core::mem::offset_of!(TaDescriptor, stamp_va) == 0x8a6); +static_assert!(core::mem::offset_of!(TaDescriptor, completion) == 0x8c6); +static_assert!(core::mem::offset_of!(TaDescriptor, timestamp_start_va) == 0x8fe); +static_assert!(core::mem::offset_of!(TaDescriptor, free_list_control_va) == 0x934); +static_assert!(core::mem::offset_of!(TaDescriptor, status_va) == 0x945); + +// SAFETY: `TaDescriptor` consists of integers and arrays of integers only. +unsafe impl Zeroable for TaDescriptor {} + +impl TaDescriptor { + const ARRAY_OFFSET: u64 = core::mem::offset_of!(Self, registers) as u64; + const UNK_876: u32 = u32::MAX; + const UNK_892: u32 = 1; + const UNK_8C2: u32 = 1; + const UNK_932: u8 = 0x44; + const UNK_93C: u8 = 1; + const UNK_93E_FIRST_KICK: [u8; 2] = [0xd0, 0x91]; + + /// Writes the complete descriptor except the completion link. + pub(crate) fn write(&mut self, args: &RenderArgs<'_>) -> Result { + let pass = args.pass; + let resources = args.resources; + let geometry = Geometry::new(pass)?; + let words = PassWords::new(pass)?; + let registers = ta_registers(args, &geometry, &words)?; + let deflake_2a0_offset = user_offset32(offset_va(resources.deflake_va, DEFLAKE_2A0)?)?; + let array_va = offset_va(args.ta.descriptor_va, Self::ARRAY_OFFSET)?; + let sampler_max = sampler_max(pass.sampler_count)?; + let stamp_value = args.stamp_value(); + + clear(self); + self.tag = CommandTag::Tiling as u32; + self.submit_sequence = args.ordinal.wrapping_mul(2).wrapping_add(1); + self.context_id = resources.context_id.into(); + self.kick_record_va = resources.kick_record_va; + self.pb_slot = resources.pb_slot; + self.pb_va = resources.pb_va; + self.scene_va = resources.scene_va; + self.scene_end_va = resources.scene_end_va; + self.checkpoints = TA_CHECKPOINTS; + self.unk_48 = (args.ordinal as u32).wrapping_add((args.ordinal / 2) as u32); + self.registers.set(array_va, ®isters)?; + self.tpc_va = resources.tpc_va; + self.tpc_size = resources.tpc_size; + self.fragment_qid = args.fragment.qid.into(); + self.stamp_value = stamp_value; + self.deflake_2a0_offset = deflake_2a0_offset; + self.predecessor_id = resources.ta_ids.predecessor; + self.unk_876 = Self::UNK_876; + self.sampler_heap_va = pass.sampler_heap_va; + self.sampler_count = pass.sampler_count; + self.sampler_max = sampler_max; + self.unk_892 = Self::UNK_892; + self.stamp_va = args.ta.stamp_va; + self.aux_stamp_va = args.ta.aux_stamp_va; + self.stamp_value_2 = stamp_value; + self.qid = args.ta.qid.into(); + self.unk_8c2 = Self::UNK_8C2; + self.current_id = resources.ta_ids.current.into(); + self.ordinal = args.ordinal as u32; + self.timestamp_start_va = resources.timestamp_va; + self.unk_932 = Self::UNK_932; + self.free_list_control_va = resources.free_list_control_va; + self.unk_93c = Self::UNK_93C; + if args.ordinal == 0 { + self.unk_93e = Self::UNK_93E_FIRST_KICK; + } + self.status_va = resources.ta_status_va; + self.context_generation = resources.context_generation; + Ok(()) + } + + /// Links the pass to the timestamps its two kicks are published at. + /// + /// Must be called before the descriptor is published. `pass_seq` is the + /// low byte of the logical queue's render-pass count (one for its first + /// pass); `fragment_busy` tells whether the fragment queue had not yet + /// completed every earlier kick. + pub(crate) fn set_completion_link( + &mut self, + ta: KickTimestamp, + fragment: KickTimestamp, + pass_seq: u8, + fragment_busy: bool, + ) { + self.fragment_busy = fragment_busy as u32; + self.fragment_kick = fragment.get(); + self.pass_seq = [pass_seq; 2]; + self.completion = ta.completion_value(pass_seq); + } + + /// Returns the register arrays a tiling kick binds; `descriptor_va` is the + /// descriptor's address as seen by the pass's VM. + pub(crate) fn register_bindings( + descriptor_va: u64, + ) -> Result<[Option; 4]> { + let array = RegisterArrayBinding::new( + offset_va(descriptor_va, Self::ARRAY_OFFSET)?, + TA_CHECKPOINTS[0] as u8, + )?; + Ok([Some(array), None, None, None]) + } +} + +/// Fragment (3D) work descriptor (tag 1). +#[repr(C, packed)] +#[derive(Debug)] +pub(crate) struct FragmentDescriptor { + /// [`CommandTag::Fragment`]. + pub(crate) tag: u32, + /// Twice the pair's submission ordinal. + pub(crate) submit_sequence: u64, + /// UAT context of the pass. + pub(crate) context_id: u32, + pub(crate) unk_10: [u8; 0x10], + /// GPU address of the pass's kick record. + pub(crate) kick_record_va: u64, + /// GPU address of the parameter-buffer state object. + pub(crate) pb_va: u64, + /// GPU address of the parameter-buffer scene entry. + pub(crate) scene_va: u64, + /// GPU address following the last scene entry. + pub(crate) scene_end_va: u64, + /// GPU address of the tile map. + pub(crate) tilemap_va: u64, + /// Multisample control. + pub(crate) msaa_control: u64, + /// Samples per pixel. + pub(crate) samples: u32, + /// Macro-tile dimensions in utiles: height in bits 15:0, width above. + pub(crate) macro_size: u32, + pub(crate) unk_58: [u8; 0x10], + /// Merge upper bound in X. + pub(crate) merge_upper_x: u32, + /// Merge upper bound in Y. + pub(crate) merge_upper_y: u32, + pub(crate) unk_70: u64, + /// Number of tiles. + pub(crate) tiles: u64, + /// Checkpoints into the main register array. + pub(crate) checkpoints: [u16; 3], + pub(crate) unk_86: u16, + /// Entry count of the main register array. + pub(crate) entries: u16, + pub(crate) unk_8a: [u8; 6], + /// Render-pass sequence of the pass. + pub(crate) pass_seq: u8, + pub(crate) unk_91: [u8; 0xf], + /// Main, partial-store, partial-resume and partial-load register arrays. + pub(crate) arrays: [RegisterArray; 4], + /// GPU address of the depth-bias array. + pub(crate) depth_bias_va: u64, + pub(crate) unk_1d28: u64, + /// GPU address of the scissor array. + pub(crate) scissor_va: u64, + pub(crate) unk_1d38: u64, + /// GPU address of the occlusion-query results. + pub(crate) occlusion_query_va: u64, + pub(crate) unk_1d48: [u8; 0x130], + /// Background resource specifier. + pub(crate) bg_rsrc_spec: u64, + /// GPU address of the background program. + pub(crate) bg_va: u64, + pub(crate) unk_1e88: [u8; 0x20], + /// Partial-render background resource specifier. + pub(crate) partial_bg_rsrc_spec: u64, + /// GPU address of the partial-render background program. + pub(crate) partial_bg_va: u64, + pub(crate) unk_1eb8: u64, + /// ZLS swizzle. + pub(crate) zls_swizzle: u32, + pub(crate) unk_1ec4: [u8; 0x74], + /// Tile-buffer blocks per utile. + pub(crate) tib_blocks: u64, + /// Auxiliary framebuffer flags. + pub(crate) aux_fb_flags: u64, + /// Width of the render target. + pub(crate) width: u32, + /// Height of the render target. + pub(crate) height: u32, + /// [`UNK_15049`]. + pub(crate) unk_1f50: u64, + /// Tile configuration. + pub(crate) tile_config: u64, + /// Ends with the end-of-tile resource specifier, which is zero. + pub(crate) unk_1f60: [u8; 0x1c], + /// GPU address of the end-of-tile program. + pub(crate) eot_va: u64, + /// Ends with the partial end-of-tile resource specifier, which is zero. + pub(crate) unk_1f84: [u8; 0x18], + /// GPU address of the partial-render end-of-tile program. + pub(crate) partial_eot_va: u64, + pub(crate) unk_1fa4: u32, + /// Depth clear value. + pub(crate) depth_clear: u32, + /// Tile-buffer blocks in bits 63:33 above a constant. + pub(crate) unk_1fac: u64, + pub(crate) unk_1fb4: [u8; 0x150], + /// See [`RenderResources::unk_2104`]. + pub(crate) unk_2104: u32, + /// Predecessor object ID of the fragment stage. + pub(crate) predecessor_id: u32, + /// See [`RenderResources::unk_210c`]. + pub(crate) unk_210c: u32, + pub(crate) unk_2110: u32, + pub(crate) unk_2114: [u8; 0x10], + pub(crate) unk_2124: [u32; 2], + pub(crate) unk_212c: [u8; 0x14], + /// GPU address of the fragment queue's completion stamp word. + pub(crate) stamp_va: u64, + /// GPU address of the fragment queue's auxiliary stamp word. + pub(crate) aux_stamp_va: u64, + /// Completion stamp value of the pass. + pub(crate) stamp_value: u32, + /// QID of the fragment queue. + pub(crate) qid: u32, + pub(crate) unk_2158: u32, + /// Non-zero on the first kick of a queue pair. + pub(crate) first_kick: u32, + /// Completion value of the fragment kick (see + /// [`KickTimestamp::completion_value`]). + pub(crate) completion: u64, + /// Current object ID of the fragment stage. + pub(crate) current_id: u64, + /// Submission ordinal of the pair. + pub(crate) ordinal: u32, + pub(crate) unk_2174: [u8; 0x24], + /// GPU address the pass start timestamp is written to. + pub(crate) timestamp_start_va: u64, + /// GPU address the pass end timestamp is written to. + pub(crate) timestamp_end_va: u64, + pub(crate) unk_21a8: [u8; 0x24], + pub(crate) unk_21cc: u8, + pub(crate) unk_21cd: u8, + /// GPU address of the USC free list's shared control object. + pub(crate) free_list_control_va: u64, + pub(crate) unk_21d6: u8, + /// GPU address of the fragment status word, as seen by the pass's VM. + pub(crate) status_va: u64, + /// Firmware alias of the fragment status word. + pub(crate) status_fw_va: u64, + /// Allocation generation of the UAT context. + pub(crate) context_generation: u8, + pub(crate) unk_21e8: [u8; 0x20], + pub(crate) unk_2208: u8, + pub(crate) unk_2209: [u8; 3], + pub(crate) unk_220c: u8, + pub(crate) unk_220d: [u8; 0x20], + pub(crate) unk_222d: u8, + pub(crate) unk_222e: [u8; 0x12], +} + +static_assert!(size_of::() == 0x2240); +static_assert!(core::mem::offset_of!(FragmentDescriptor, checkpoints) == 0x80); +static_assert!(core::mem::offset_of!(FragmentDescriptor, pass_seq) == 0x90); +static_assert!(core::mem::offset_of!(FragmentDescriptor, arrays) == 0xa0); +static_assert!(core::mem::offset_of!(FragmentDescriptor, depth_bias_va) == 0x1d20); +static_assert!(core::mem::offset_of!(FragmentDescriptor, bg_rsrc_spec) == 0x1e78); +static_assert!(core::mem::offset_of!(FragmentDescriptor, tib_blocks) == 0x1f38); +static_assert!(core::mem::offset_of!(FragmentDescriptor, unk_1f50) == 0x1f50); +static_assert!(core::mem::offset_of!(FragmentDescriptor, eot_va) == 0x1f7c); +static_assert!(core::mem::offset_of!(FragmentDescriptor, depth_clear) == 0x1fa8); +static_assert!(core::mem::offset_of!(FragmentDescriptor, unk_2104) == 0x2104); +static_assert!(core::mem::offset_of!(FragmentDescriptor, stamp_va) == 0x2140); +static_assert!(core::mem::offset_of!(FragmentDescriptor, completion) == 0x2160); +static_assert!(core::mem::offset_of!(FragmentDescriptor, timestamp_start_va) == 0x2198); +static_assert!(core::mem::offset_of!(FragmentDescriptor, free_list_control_va) == 0x21ce); +static_assert!(core::mem::offset_of!(FragmentDescriptor, status_va) == 0x21d7); +static_assert!(core::mem::offset_of!(FragmentDescriptor, unk_222d) == 0x222d); + +// SAFETY: `FragmentDescriptor` consists of integers and arrays of integers only. +unsafe impl Zeroable for FragmentDescriptor {} + +impl FragmentDescriptor { + /// Offset of register array `index` within the descriptor. + const fn array_offset(index: usize) -> u64 { + (core::mem::offset_of!(Self, arrays) + index * size_of::()) as u64 + } + + /// Returns the register arrays a fragment kick binds; `descriptor_va` is + /// the descriptor's address as seen by the pass's VM. + pub(crate) fn register_bindings( + descriptor_va: u64, + ) -> Result<[Option; 4]> { + let array = |index, checkpoint: usize| -> Result> { + let va = offset_va(descriptor_va, Self::array_offset(index))?; + Ok(Some(RegisterArrayBinding::new(va, checkpoint as u8)?)) + }; + Ok([ + array(0, FRAGMENT_CHECKPOINTS[0] as usize)?, + array(1, PARTIAL_STORE_REGISTERS)?, + array(2, PARTIAL_RESUME_REGISTERS)?, + array(3, PARTIAL_LOAD_REGISTERS)?, + ]) + } +} + +/// A fragment descriptor followed by room for an MCache table that does not +/// fit its first register array. +#[repr(C)] +#[derive(Debug)] +pub(crate) struct FragmentSlot { + /// The descriptor. + pub(crate) descriptor: FragmentDescriptor, + /// MCache table of more than [`MCACHE_INLINE_RANGES`] ranges. + pub(crate) mcache: [McacheRange; MCACHE_RANGES_MAX], + pub(crate) pad_2350: [u8; 0x30], +} + +static_assert!(size_of::() == FRAGMENT_SLOT_SIZE); + +impl FragmentSlot { + const INLINE_MCACHE_OFFSET: u64 = + FragmentDescriptor::array_offset(0) + core::mem::offset_of!(RegisterArray, mcache) as u64; + const SPILL_MCACHE_OFFSET: u64 = core::mem::offset_of!(Self, mcache) as u64; + const UNK_2110: u32 = u32::MAX; + const UNK_2124: [u32; 2] = [1, 1]; + const UNK_21CC: u8 = 0x53; + const UNK_21D6: u8 = 1; + const UNK_2208: u8 = 1; + const UNK_220C: u8 = 1; + const UNK_222D: u8 = 1; + const UNK_1FAC_BASE: u64 = 0x300; + const UNK_1FAC_TIB_SHIFT: u32 = 33; + + /// Writes the complete descriptor except the completion link and the + /// MCache table. + /// + /// The spill table is left as it is: a kick entry names it only after + /// [`Self::set_mcache`] rewrote it, so stale bytes there are never read. + pub(crate) fn write(&mut self, args: &RenderArgs<'_>) -> Result { + let pass = args.pass; + let resources = args.resources; + let geometry = Geometry::new(pass)?; + let words = PassWords::new(pass)?; + let base = args.fragment.descriptor_va; + let array_va = |index| offset_va(base, FragmentDescriptor::array_offset(index)); + let arrays = [array_va(0)?, array_va(1)?, array_va(2)?, array_va(3)?]; + let timestamp_end_va = timestamp_end_va(resources.timestamp_va)?; + let desc = &mut self.descriptor; + + clear(desc); + desc.tag = CommandTag::Fragment as u32; + desc.submit_sequence = args.ordinal.wrapping_mul(2); + desc.context_id = resources.context_id.into(); + desc.kick_record_va = resources.kick_record_va; + desc.pb_va = resources.pb_va; + desc.scene_va = resources.scene_va; + desc.scene_end_va = resources.scene_end_va; + desc.tilemap_va = resources.tilemap_va; + desc.msaa_control = pass.msaa_control; + desc.samples = 1 << pass.samples_log2; + desc.macro_size = geometry.macro_size as u32; + desc.merge_upper_x = pass.merge_upper_x; + desc.merge_upper_y = pass.merge_upper_y; + desc.tiles = geometry.tiles_x * geometry.tiles_y; + desc.checkpoints = FRAGMENT_CHECKPOINTS; + desc.entries = FRAGMENT_REGISTERS as u16; + desc.arrays[0].set(arrays[0], &fragment_registers(args, &geometry, &words))?; + desc.arrays[1].set(arrays[1], &partial_store_registers(pass, &words))?; + desc.arrays[2].set(arrays[2], &partial_resume_registers(pass, &words))?; + desc.arrays[3].set(arrays[3], &partial_load_registers(pass, &words))?; + desc.depth_bias_va = pass.depth_bias_va; + desc.scissor_va = pass.scissor_va; + desc.occlusion_query_va = pass.occlusion_query_va; + desc.bg_rsrc_spec = words.bg_rsrc_spec; + desc.bg_va = words.bg_va; + desc.partial_bg_rsrc_spec = words.partial_bg_rsrc_spec; + desc.partial_bg_va = words.partial_bg_va; + desc.zls_swizzle = ZLS_SWIZZLE; + desc.tib_blocks = words.tib_blocks; + desc.aux_fb_flags = words.aux_fb_flags; + desc.width = pass.width; + desc.height = pass.height; + desc.unk_1f50 = UNK_15049; + desc.tile_config = words.tile_config; + desc.eot_va = words.eot_va; + desc.partial_eot_va = words.partial_eot_va; + desc.depth_clear = pass.depth_clear; + desc.unk_1fac = words.tib_blocks << Self::UNK_1FAC_TIB_SHIFT | Self::UNK_1FAC_BASE; + desc.unk_2104 = resources.unk_2104; + desc.predecessor_id = resources.fragment_ids.predecessor; + desc.unk_210c = resources.unk_210c; + desc.unk_2110 = Self::UNK_2110; + desc.unk_2124 = Self::UNK_2124; + desc.stamp_va = args.fragment.stamp_va; + desc.aux_stamp_va = args.fragment.aux_stamp_va; + desc.stamp_value = args.stamp_value(); + desc.qid = args.fragment.qid.into(); + desc.first_kick = (args.ordinal == 0) as u32; + desc.current_id = resources.fragment_ids.current.into(); + desc.ordinal = args.ordinal as u32; + desc.timestamp_start_va = resources.timestamp_va; + desc.timestamp_end_va = timestamp_end_va; + desc.unk_21cc = Self::UNK_21CC; + desc.free_list_control_va = resources.free_list_control_va; + desc.unk_21d6 = Self::UNK_21D6; + desc.status_va = resources.fragment_status_va; + desc.status_fw_va = resources.fragment_status_fw_va; + desc.context_generation = resources.context_generation; + desc.unk_2208 = Self::UNK_2208; + desc.unk_220c = Self::UNK_220C; + desc.unk_222d = Self::UNK_222D; + Ok(()) + } + + /// Links the pass to the timestamp of its fragment kick. + /// + /// Must be called before the descriptor is published; see + /// [`TaDescriptor::set_completion_link`]. + pub(crate) fn set_completion_link(&mut self, fragment: KickTimestamp, pass_seq: u8) { + self.descriptor.pass_seq = pass_seq; + self.descriptor.completion = fragment.completion_value(pass_seq); + } + + /// Writes the MCache table of the pass: the auxiliary framebuffer at + /// `aux_fb_va`, then `attachments`, translated in UAT context `context`. + /// `slot_va` is the GPU address of this slot as seen by the pass's VM. + /// + /// Returns the table for the fragment kick entry, or `None` when the pass + /// has no attachments. + pub(crate) fn set_mcache( + &mut self, + slot_va: u64, + aux_fb_va: u64, + attachments: &[WriteRange], + context: u8, + ) -> Result> { + if attachments.is_empty() { + return Ok(None); + } + let count = attachments.len() + 1; + if count > MCACHE_RANGES_MAX { + return Err(EINVAL); + } + let (offset, ranges) = if count <= MCACHE_INLINE_RANGES { + ( + Self::INLINE_MCACHE_OFFSET, + &mut self.descriptor.arrays[0].mcache[..count], + ) + } else { + (Self::SPILL_MCACHE_OFFSET, &mut self.mcache[..count]) + }; + let table = McacheTable::new(offset_va(slot_va, offset)?, count)?; + let aux_fb = WriteRange { + va: aux_fb_va, + size: AUX_FB_SIZE, + }; + ranges[0] = McacheRange::new(&aux_fb, McacheClass::AuxFramebuffer, context)?; + for (range, attachment) in ranges[1..].iter_mut().zip(attachments) { + *range = McacheRange::new(attachment, McacheClass::Attachment, context)?; + } + Ok(Some(table)) + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/render/storage.rs b/drivers/gpu/drm/asahi/g17/fw/render/storage.rs new file mode 100644 index 0000000000000..e1e6a10949fb4 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/render/storage.rs @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Auxiliary render records and their initial images. Live event counters +//! are accessed individually by the queue owner after publication. + +use super::super::{buffer, clear, context, initdata}; +use kernel::prelude::*; + +pub(crate) const SUPPORT_SIZE: usize = 0x40000; +pub(crate) const EVENT_SLOTS: usize = 0x4000; +pub(crate) const EVENT_COUNT: usize = 35; +pub(crate) const STATE_SIZE: usize = 0x18000; +pub(crate) const DEFLAKE: usize = 0x4000; +pub(crate) const TA_STATUS: usize = 0x8000; +pub(crate) const AUXILIARY: usize = 0x10000; +pub(crate) const AUXILIARY_SIZE: usize = 0x8000; + +#[repr(C)] +pub(crate) struct Event { + pub(crate) node: context::WorkNode, + unk_c8: [u8; 0x38], +} + +#[repr(C)] +pub(crate) struct Support { + secondary_index: [u64; 8], + unk_40: [u8; 0x3fc0], + pub(crate) event_slots: [u32; 0x1000], + scene_slots: [u32; 0x1000], + shared_slots: [u32; 0x1000], + flag: u32, + unk_10004: [u8; 0x3ffc], + pub(crate) events: [Event; EVENT_COUNT], + unk_16300: [u8; 0x1d00], + scenes: [buffer::Scene; 79], + unk_1a780: [u8; 0x1880], + shared: buffer::State, + unk_1c0c0: [u8; 0x7f40], + control: initdata::FreeListControl, + unk_24068: [u8; 0x3f98], + submitted: u32, + unk_28004: [u8; 0x3ffc], + scheduler: context::Scheduler, + unk_30000: [u8; 0x10000], +} +// SAFETY: The entire graph is integer storage; all nested types are Zeroable. +unsafe impl Zeroable for Support {} +impl Support { + pub(crate) fn init( + &mut self, + address: u64, + primary_index: u64, + operand_table: u64, + page_list: u64, + word: u32, + ) -> Result { + clear(self); + address.checked_add(SUPPORT_SIZE as u64).ok_or(EOVERFLOW)?; + self.secondary_index = [0x11, 0x16, 0x1b, 0x20, 0x25, 0x2a, 0x3c, 0x41]; + self.event_slots[1] = 1; + self.shared_slots[0] = 8; + self.shared_slots[1] = 8; + self.shared_slots[0x60 / 4] = 1; + self.flag = 1; + for (index, event) in self.events.iter_mut().enumerate() { + event.node.submitted = address + EVENT_SLOTS as u64 + index as u64 * 4; + if index == 1 { + event.node.threshold = 0x50; + event.node.word = word; + } + } + for (index, scene) in self.scenes.iter_mut().enumerate() { + scene.metric_client = 0x10_0008_0004 + index as u64 * 4; + scene.metric_fw = address + 0x8004 + index as u64 * 4; + let phase = index % buffer::SCRATCH_SLOTS; + scene.scratch = if phase == buffer::SCRATCH_SLOTS - 1 { + 0x178000 + } else { + 0x178020 + phase as u64 * buffer::SCRATCH_STRIDE + }; + scene.statistics = address + 0xc040; + if index == 0 { + scene.generation = 1; + } + } + let shared = &mut self.shared; + shared.page_list_fw = primary_index; + shared.page_list_client = 0x10_0019_0000; + shared.page_list_size = 0x10000; + shared.pages = 0x20; + shared.block_capacity = 0xc18; + shared.producer = 8; + shared.block_table = address; + shared.control = address + 0xc000; + shared.last_page = 0x1f; + shared.block_size = 0x20000; + shared.counter = address + 0x10000; + shared.maximum_size = 0x3060; + shared.normal_maximum_size = 0x1020; + shared.discard = 0x180000; + self.control = initdata::FreeListControl::new(&initdata::FreeListArgs { + buffer_slot: 0, + class: initdata::FreeListClass::Two, + page_list_va: page_list, + run_list_va: operand_table, + blocks: 25, + state_va: address + 0x28000, + }); + self.control.run_cursor = 0xe0; + self.submitted = 1; + self.scheduler.init(); + Ok(()) + } +} + +/// The viewport words consumed from the deflake page at +0x900. +#[repr(C)] +#[derive(Copy, Clone)] +pub(crate) struct Viewport { + unk_0: u32, + tiles_x: u32, + tiles_y: u32, + unk_c: u32, + scale_x: u32, + translate_x: u32, + scale_y: u32, + translate_y: u32, + unk_20: u32, + scale_z: u32, +} +impl Viewport { + /// Refresh only defined words; firmware-owned gaps retain their contents. + pub(crate) fn apply(&self, destination: &mut Self) { + destination.unk_0 = self.unk_0; + destination.tiles_x = self.tiles_x; + destination.tiles_y = self.tiles_y; + destination.scale_x = self.scale_x; + destination.translate_x = self.translate_x; + destination.scale_y = self.scale_y; + destination.translate_y = self.translate_y; + destination.scale_z = self.scale_z; + } + pub(crate) fn new(width: u32, height: u32) -> Result { + fn half(value: u32) -> Result { + if value == 0 { + return Err(EINVAL); + } + let highest = 31 - value.leading_zeros(); + let shift = 23u32.checked_sub(highest).ok_or(EINVAL)?; + Ok(((highest + 126) << 23) | ((value << shift) & 0x7f_ffff)) + } + let x = half(width)?; + let y = half(height)?; + Ok(Self { + unk_0: 0xc00, + tiles_x: 0x8000_0000 | (width.checked_add(31).ok_or(EOVERFLOW)? / 32 - 1), + tiles_y: height.checked_add(31).ok_or(EOVERFLOW)? / 32 - 1, + unk_c: 0, + scale_x: x, + translate_x: x, + scale_y: y, + translate_y: y | 0x8000_0000, + unk_20: 0, + scale_z: 0x3f80_0000, + }) + } +} + +const _: () = { + assert!(core::mem::size_of::() == 0x100); + assert!(core::mem::size_of::() == SUPPORT_SIZE); + assert!(core::mem::offset_of!(Support, events) == 0x14000); + assert!(core::mem::offset_of!(Support, scenes) == 0x18000); + assert!(core::mem::offset_of!(Support, shared) == 0x1c000); + assert!(core::mem::offset_of!(Support, control) == 0x24000); + assert!(core::mem::offset_of!(Support, scheduler) == 0x2c000); + assert!(core::mem::size_of::() == 0x28); +}; diff --git a/drivers/gpu/drm/asahi/g17/hazard.rs b/drivers/gpu/drm/asahi/g17/hazard.rs new file mode 100644 index 0000000000000..20daaab85b832 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/hazard.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Attachment overlap ordering across physical render pairs and shared images. +//! Tile stores can overlap even when the application's pixel regions do not. +//! Keep independent attachments concurrent; order only conflicting live spans. + +use super::job::Order; + +const MAX_SPANS: usize = 24; +pub(crate) const WAIT_MS: u32 = 1000; + +#[derive(Clone, Copy)] +struct Span { + key: u64, + start: u64, + end: u64, +} + +/// Up to sixteen attachment ranges and eight shared-object ranges. +pub(crate) struct Footprint { + vm: u64, + spans: [Span; MAX_SPANS], + count: usize, + shared: bool, +} + +impl Footprint { + pub(crate) fn new(vm: u64) -> Self { + Self { + vm, + spans: [Span { + key: 0, + start: 0, + end: 0, + }; MAX_SPANS], + count: 0, + shared: false, + } + } + + fn push(&mut self, key: u64, start: u64, end: u64) { + if start < end && self.count < MAX_SPANS { + self.spans[self.count] = Span { key, start, end }; + self.count += 1; + } + } + + pub(crate) fn push_vm(&mut self, start: u64, end: u64) { + self.push(self.vm << 1, start, end); + } + + /// An aligned GEM object address is an identity, never dereferenced here. + pub(crate) fn push_object(&mut self, object: usize, start: u64, end: u64) { + let before = self.count; + self.push(object as u64 | 1, start, end); + self.shared |= self.count != before; + } + + fn overlaps(&self, other: &Self) -> bool { + if self.vm != other.vm && !(self.shared && other.shared) { + return false; + } + self.spans[..self.count].iter().any(|a| { + other.spans[..other.count] + .iter() + .any(|b| a.key == b.key && a.start < b.end && b.start < a.end) + }) + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Plan { + Any, + Pair(u8), + Wait, +} + +pub(crate) struct Live<'a> { + pub(crate) owner: u64, + pub(crate) sequence: u64, + pub(crate) slot: Option, + pub(crate) ticket: u64, + pub(crate) footprint: &'a Footprint, + pub(crate) quarantined: bool, +} + +/// A queue prefix already orders covered predecessors. Pending arrivals order +/// different owners fairly without imposing order on unrelated attachments. +pub(crate) fn plan<'a>( + owner: u64, + prefix: Option, + ticket: Option, + footprint: &Footprint, + live: impl IntoIterator>, +) -> Plan { + let mut selected = None; + for entry in live { + if entry.quarantined || !footprint.overlaps(entry.footprint) { + continue; + } + let Some(slot) = entry.slot else { + if entry.owner != owner && ticket.is_none_or(|mine| entry.ticket < mine) { + return Plan::Wait; + } + continue; + }; + if entry.owner != owner { + return Plan::Wait; + } + if Order::contains(prefix, entry.sequence) { + continue; + } + match selected { + None => selected = Some(slot), + Some(prior) if prior == slot => {} + Some(_) => return Plan::Wait, + } + } + selected.map_or(Plan::Any, Plan::Pair) +} diff --git a/drivers/gpu/drm/asahi/g17/ids.rs b/drivers/gpu/drm/asahi/g17/ids.rs new file mode 100644 index 0000000000000..6912c6501527a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/ids.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Firmware-visible object generations shared by contexts, pools and render work. + +use super::fw::ObjectIds; +use core::sync::atomic::{AtomicU32, Ordering}; +use kernel::prelude::*; + +/// Module-lifetime identities remain unique across retained graphs and device rebinds. +pub(crate) static OBJECTS: Counter = Counter::new(); + +pub(crate) struct Counter(AtomicU32); + +impl Counter { + const fn new() -> Self { + Self(AtomicU32::new(1)) + } + + fn reserve(&self, count: u32) -> Result { + let mut old = self.0.load(Ordering::Acquire); + loop { + let next = old.checked_add(count).ok_or(EOVERFLOW)?; + match self + .0 + .compare_exchange_weak(old, next, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => return Ok(old), + Err(current) => old = current, + } + } + } + + pub(crate) fn object(&self) -> Result { + let id = self.reserve(1)?; + if id == 0 { + Err(EOVERFLOW) + } else { + Ok(id) + } + } + + /// Allocate consecutive current IDs atomically, fragment first and tiling + /// second. Both name the same retained predecessor of their render graph. + pub(crate) fn render_pair(&self, predecessor: u32) -> Result<(ObjectIds, ObjectIds)> { + let fragment = self.reserve(2)?; + if predecessor == 0 || fragment == 0 { + return Err(EOVERFLOW); + } + Ok(( + ObjectIds { + predecessor, + current: fragment, + }, + ObjectIds { + predecessor, + current: fragment + 1, + }, + )) + } +} diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs new file mode 100644 index 0000000000000..1e17e8c537d97 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -0,0 +1,1311 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Allocates, initializes and retains the graph shared by the two firmware instances. +//! +//! The owner is installed in the firmware session before construction starts. Even if an +//! allocation fails, the session stops both coprocessors before releasing any published memory. + +use super::{ + fw::initdata as abi, + object::{CpuMap, KernelObject, Placement}, + Config, Role, +}; +use crate::{driver::AsahiDrmDevice, hw::t8140, mmu}; +use core::sync::atomic::{fence, AtomicU32, Ordering}; +use kernel::{c_str, prelude::*}; +use t8140::{kernel_window as high, low}; + +const EVENT_RING_RECHECK: u32 = 256; + +const INDEX_SIZE: usize = 0x10000; +const CONTEXT_SIZE: usize = 0x20000; +const REGION_A_SIZE: usize = core::mem::size_of::(); +const REGION_C_SIZE: usize = 0x1000; + +const COMPUTE_BLOCKS_ALLOCATED: usize = 23; +const COMPUTE_BLOCKS_MAPPED: usize = 22; +const COMPUTE_BLOCKS_POPULATED: usize = 21; +const COMPUTE_CONTROL_BLOCKS: u32 = 25; +const COMPUTE_CONTROL_CURSOR: u32 = 0x18; +const COMPUTE_RUN_ADDRESS_MASK: u64 = 0x0000_ffff_ffff_f000; + +/// Unpublished device-wide compute backing, prepared without the device mutex. The cache +/// identity belongs to this allocation, not to any one physical queue or client VM. +pub(super) struct ComputeShared { + id: u64, + pages: KernelObject, + runs: KernelObject, + blocks: KVec, +} + +impl ComputeShared { + pub(super) fn new(alloc: &super::object::Allocator<'_>) -> Result { + let pages = + KernelObject::backing(alloc.dev, low::RENDER_PAGE_LIST_SIZE, CpuMap::WriteCombined)?; + let runs = + KernelObject::backing(alloc.dev, low::RENDER_RUN_LIST_SIZE, CpuMap::WriteCombined)?; + let mut blocks = KVec::with_capacity(COMPUTE_BLOCKS_ALLOCATED, GFP_KERNEL)?; + for _ in 0..COMPUTE_BLOCKS_ALLOCATED { + blocks.push( + KernelObject::backing(alloc.dev, abi::FREE_LIST_BLOCK_SIZE, CpuMap::WriteCombined)?, + GFP_KERNEL, + )?; + } + let id = u64::from(super::ids::OBJECTS.object()?); + Ok(Self { + id, + pages, + runs, + blocks, + }) + } +} + +/// A device-global scheduler, retained independently of whichever physical queue first +/// needed it. Its generation reservation precedes later graph allocation failures. +pub(super) struct ComputeScheduler { + object: KernelObject, +} + +impl ComputeScheduler { + pub(super) fn new(alloc: &super::object::Allocator<'_>) -> Result { + let mut object = alloc.kernel( + mmu::UAT_PGSZ, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + object.initialize::(0, |state| { + state.init(); + Ok(()) + })?; + fence(Ordering::SeqCst); + // Scheduler creation reserves a generation even when only its address is used by work. + super::ids::OBJECTS.object()?; + Ok(Self { object }) + } +} + +/// Checked views used only within one publication call. Every fallible operation completes +/// before either member of a paired render publication is copied into shared memory. +struct WorkPublication<'a> { + record: &'a super::fw::channels::WorkSlot, + destination: *mut u8, + producer: &'a AtomicU32, + next: u32, +} + +impl WorkPublication<'_> { + fn copy_record(&self) { + // SAFETY: work_publication checked the complete destination range. The record is a + // fully initialized padding-free ABI value, and the device mutex owns publication. + unsafe { + core::ptr::copy_nonoverlapping( + self.record as *const _ as *const u8, + self.destination, + core::mem::size_of_val(self.record), + ); + } + } + + fn commit(self) -> u32 { + self.copy_record(); + fence(Ordering::SeqCst); + self.producer.store(self.next, Ordering::Relaxed); + fence(Ordering::SeqCst); + self.next + } +} + +/// Session-long storage and mappings. Aliases are released before their backing objects. +pub(super) struct InitData { + // Live publication uses interior writes under the device mutex. This owner is Send but not + // Sync, so safe host code cannot issue those writes concurrently through shared references. + _single_publisher: core::marker::PhantomData>, + aliases: KVec, + objects: KVec, + roots: usize, + cluster: usize, + bundle: usize, + region_c: usize, + qos: usize, + pb: usize, + page_pool: usize, + compute_global: usize, + boot_control: usize, + boot_state: usize, + metrics_fw_va: u64, + compute_shared: Option, + compute_scheduler: Option, + compute_active: core::cell::Cell, + event_ring_checked: [core::cell::Cell<(u64, u64, u32)>; 2], + completion_rings: [usize; 2], + stamps: [usize; 2], + render_pages: usize, + render_runs: usize, + render_blocks: core::ops::Range, +} + +impl InitData { + /// Empty construction state, retained even if graph construction fails. + pub(super) fn new() -> Self { + Self { + _single_publisher: core::marker::PhantomData, + aliases: KVec::new(), + objects: KVec::new(), + roots: 0, + cluster: 0, + bundle: 0, + region_c: 0, + qos: 0, + pb: 0, + page_pool: 0, + compute_global: 0, + boot_control: 0, + boot_state: 0, + metrics_fw_va: 0, + compute_shared: None, + compute_scheduler: None, + compute_active: core::cell::Cell::new(false), + event_ring_checked: [const { core::cell::Cell::new((0, 0, 0)) }; 2], + completion_rings: [0; 2], + stamps: [0; 2], + render_pages: 0, + render_runs: 0, + render_blocks: 0..0, + } + } + + pub(super) fn metrics_fw_va(&self) -> Result { + if self.metrics_fw_va == 0 { + return Err(EIO); + } + Ok(self.metrics_fw_va) + } + + pub(super) fn pb_descriptor_table(&self) -> Result<&KernelObject> { + self.object(self.pb) + } + + pub(super) fn page_pool_descriptor_table(&self) -> Result<&KernelObject> { + self.object(self.page_pool) + } + + pub(super) fn stamp_addresses(&self, qid: u8) -> Result<[u64; 2]> { + if qid > super::fw::kick::QID_MAX { + return Err(EINVAL); + } + let offset = u64::from(qid) * core::mem::size_of::() as u64; + Ok([ + self.object(self.stamps[0])?.gpu_va() + offset, + self.object(self.stamps[1])?.gpu_va() + offset, + ]) + } + + /// The caller reads both engine lanes before its acquire fence. Page A is the completed + /// GPU-maintenance witness; page B stays under firmware ownership. + pub(super) fn stamp(&self, qid: u8) -> Result<&AtomicU32> { + if qid > super::fw::kick::QID_MAX { + return Err(EINVAL); + } + self.object(self.stamps[0])? + .word(usize::from(qid) * core::mem::size_of::()) + } + + /// Initialize a fresh physical compute QID's row before exposing its queue. An installed + /// or rebound queue keeps the row's firmware-owned counters and must never call this again. + pub(super) fn initialize_compute_pool(&self, slot: u16, page_list_va: u64) -> Result { + if slot == 0 || slot > u16::from(super::fw::kick::QID_MAX) + 1 { + return Err(EINVAL); + } + let entry = abi::PagePoolDescriptor::compute(page_list_va); + let size = core::mem::size_of::(); + let offset = usize::from(slot) * size; + let object = self.object(self.page_pool)?; + let pointer = object.pointer(offset, size)?; + let page_list = object.dword(offset)?; + let counters = + object.dword(offset + core::mem::offset_of!(abi::PagePoolDescriptor, counters))?; + let count = + object.dword(offset + core::mem::offset_of!(abi::PagePoolDescriptor, page_count))?; + // SAFETY: the whole fresh row was bounds checked, the QID has never been published, + // and the device mutex excludes another host publisher. No other row is touched. + unsafe { + pointer.write_bytes(0, size); + } + page_list.store(entry.page_list, Ordering::Relaxed); + counters.store(entry.counters, Ordering::Relaxed); + count.store(entry.page_count, Ordering::Relaxed); + fence(Ordering::SeqCst); + Ok(()) + } + + pub(super) fn qos(&self) -> Result<&KernelObject> { + self.object(self.qos) + } + + pub(super) fn has_compute_scheduler(&self) -> bool { + self.compute_scheduler.is_some() + } + + /// Only the reserved singleton builder may install a scheduler. Retain it before building + /// the first descriptor graph so later construction failure does not allocate it again. + pub(super) fn install_compute_scheduler(&mut self, scheduler: ComputeScheduler) -> Result { + if self.compute_scheduler.is_some() { + return Err(EBUSY); + } + self.compute_scheduler = Some(scheduler); + Ok(()) + } + + pub(super) fn has_compute_shared(&self) -> bool { + self.compute_shared.is_some() + } + + /// Install a prepared aperture once while holding the device mutex. A competing preparer + /// may have installed its allocation first; the redundant unpublished backing then drops. + pub(super) fn install_compute_shared(&mut self, mut shared: ComputeShared) -> Result { + if self.compute_shared.is_some() { + return Ok(()); + } + let state_va = self.object(self.boot_state)?.gpu_va(); + let mut control = abi::FreeListControl::new(&abi::FreeListArgs { + buffer_slot: u32::MAX, + class: abi::FreeListClass::Two, + page_list_va: low::COMPUTE_POOL_PAGE_LIST, + run_list_va: low::COMPUTE_POOL_RUN_LIST, + blocks: COMPUTE_CONTROL_BLOCKS, + state_va, + }); + control.run_cursor = COMPUTE_CONTROL_CURSOR; + self.object_mut(self.boot_control)?.bytes_mut().fill(0); + self.object_mut(self.boot_control)?.write(0, control)?; + self.object_mut(self.boot_state)?.bytes_mut().fill(0); + shared + .pages + .initialize::<[u64; low::RENDER_PAGE_LIST_SIZE / 8]>(0, |pages| { + abi::fill_page_list(pages, (0..COMPUTE_BLOCKS_POPULATED).map(low::compute_block)) + })?; + shared + .runs + .initialize::<[u64; low::RENDER_RUN_LIST_SIZE / 8]>(0, |runs| { + for (index, run) in runs.iter_mut().take(COMPUTE_BLOCKS_POPULATED).enumerate() { + *run = (low::compute_block(index) & COMPUTE_RUN_ADDRESS_MASK) + | ((abi::FREE_LIST_BLOCK_PAGES as u64) << 52); + } + Ok(()) + })?; + fence(Ordering::SeqCst); + self.compute_shared = Some(shared); + Ok(()) + } + + pub(super) fn cache_compute_vm(&self, vm: &mmu::Vm) -> Result { + let shared = self.compute_shared.as_ref().ok_or(EIO)?; + if vm.compute_pool_mappings_match(shared.id) { + return Ok(()); + } + vm.clear_compute_pool_mappings(); + let mut mappings = KVec::with_capacity(26, GFP_KERNEL)?; + mappings.push( + shared + .pages + .map_alias(vm, low::COMPUTE_POOL_PAGE_LIST, mmu::PROT_GPU_SHARED_RW)?, + GFP_KERNEL, + )?; + mappings.push( + shared + .runs + .map_alias(vm, low::COMPUTE_POOL_RUN_LIST, mmu::PROT_GPU_SHARED_RW)?, + GFP_KERNEL, + )?; + for (index, block) in shared.blocks.iter().take(COMPUTE_BLOCKS_MAPPED).enumerate() { + mappings.push( + block.map_alias(vm, low::compute_block(index), mmu::PROT_GPU_SHARED_RW)?, + GFP_KERNEL, + )?; + } + for slot in 0..2 { + mappings.push( + self.object(self.completion_rings[slot])?.map_alias( + vm, + low::COMPLETION_RINGS[slot], + mmu::PROT_GPU_SHARED_RW, + )?, + GFP_KERNEL, + )?; + } + vm.install_compute_pool_mappings(shared.id, mappings) + } + + /// The descriptor-table views remain immutable for the firmware session. + pub(super) fn validate_descriptor_views(&self) -> Result { + let values = [ + low::PB_DESCRIPTORS, + self.object(self.pb)?.gpu_va(), + low::PAGE_POOL_DESCRIPTORS, + self.object(self.page_pool)?.gpu_va(), + ]; + let base = abi::bundle::MAIN_CONFIG[Role::Primary as usize] + + core::mem::offset_of!(abi::MainConfig, pb_descriptors_low_va); + let bundle = self.object(self.bundle)?; + let mut changed = false; + for (index, value) in values.into_iter().enumerate() { + changed |= bundle.dword(base + index * 8)?.load(Ordering::Relaxed) != value; + } + fence(Ordering::SeqCst); + if changed { + Err(EIO) + } else { + Ok(()) + } + } + + pub(super) fn activate_compute(&self) -> Result { + self.validate_descriptor_views()?; + if self.compute_active.get() { + return Ok(()); + } + let scheduler = self.compute_scheduler.as_ref().ok_or(EIO)?.object.gpu_va(); + let object = self.object(self.compute_global)?; + let state = object + .pointer(0, core::mem::size_of::())? + .cast::(); + // SAFETY: The complete typed view is in the retained page-aligned object. InitData is + // not Sync and its caller holds the sole publication mutex. Each named field is + // naturally aligned and written individually; no reference aliases firmware state. + unsafe { + core::ptr::addr_of_mut!((*state).unk_0).write_volatile(0xff); + core::ptr::addr_of_mut!((*state).unk_8).write_volatile(0xff); + core::ptr::addr_of_mut!((*state).unk_10).write_volatile(0x0f01); + core::ptr::addr_of_mut!((*state).unk_18).write_volatile(0x0f01); + core::ptr::addr_of_mut!((*state).unk_408).write_volatile(2); + core::ptr::addr_of_mut!((*state).unk_40c).write_volatile(2); + core::ptr::addr_of_mut!((*state).unk_604).write_volatile(4); + core::ptr::addr_of_mut!((*state).scheduler_va).write_volatile(scheduler); + core::ptr::addr_of_mut!((*state).unk_c04).write_volatile(0x1000); + core::ptr::addr_of_mut!((*state).unk_e00).write_volatile(7); + core::ptr::addr_of_mut!((*state).unk_e08).write_volatile(0x0000_0001_0e22_618a); + core::ptr::addr_of_mut!((*state).unk_e20).write_volatile(4); + core::ptr::addr_of_mut!((*state).unk_e28).write_volatile(4); + } + self.compute_active.set(true); + Ok(()) + } + + fn work_publication<'a>( + &'a self, + record: &'a super::fw::channels::WorkSlot, + ) -> Result> { + use super::fw::{channels, queue::DataMaster}; + let engine = match record.data_master { + 0 => DataMaster::Tiling, + 1 => DataMaster::Fragment, + 2 => DataMaster::Compute, + _ => return Err(EINVAL), + }; + let channel = channels::work_ring_index(engine); + let [consumer, consumer_b, producer] = self.work_cursors(engine)?; + let next = (producer + 1) % channels::WORK_RING_SLOTS; + if next == consumer || next == consumer_b { + return Err(EINVAL); + } + let destination = self.object(self.bundle)?.pointer( + abi::work_ring_offset(channel) + producer as usize * core::mem::size_of_val(record), + core::mem::size_of_val(record), + )?; + let producer = self.object(self.cluster)?.word( + Self::state_offset(Role::Primary) + + abi::work_state_offset(channel) + + abi::ChannelState::PRODUCER, + )?; + Ok(WorkPublication { + record, + destination, + producer, + next, + }) + } + + /// Two firmware consumers and the host producer, sampled in that order. + pub(super) fn work_cursors(&self, engine: super::fw::queue::DataMaster) -> Result<[u32; 3]> { + use super::fw::channels; + let channel = channels::work_ring_index(engine); + let base = Self::state_offset(Role::Primary) + abi::work_state_offset(channel); + let state = self.object(self.cluster)?; + let mut values = [0; 3]; + for (value, offset) in values.iter_mut().zip([ + abi::ChannelState::CONSUMER, + abi::ChannelState::UNK_10, + abi::ChannelState::PRODUCER, + ]) { + *value = state.word(base + offset)?.load(Ordering::Relaxed); + } + if values + .iter() + .any(|&value| value >= channels::WORK_RING_SLOTS) + { + return Err(EINVAL); + } + Ok(values) + } + + /// Inner queue records must be visible before this outer publication. The returned + /// producer is the retained consumer witness for this slot. + pub(super) fn publish_work(&self, record: &super::fw::channels::WorkSlot) -> Result { + Ok(self.work_publication(record)?.commit()) + } + + /// Validate both destinations before exposing fragment, then tiling. The accounting token + /// is committed after copying fragment; late tiling state is published before its slot. + /// Deferred pairs retain both slot bodies without exposing either producer. + pub(super) fn publish_work_pair( + &self, + fragment: &super::fw::channels::WorkSlot, + tiling: &super::fw::channels::WorkSlot, + deferred: bool, + commit_accounting: impl FnOnce() -> Result, + publish_late_tiling: impl FnOnce(), + ) -> Result<[u32; 2]> { + use super::fw::queue::DataMaster; + if fragment.data_master != DataMaster::Fragment as u32 + || tiling.data_master != DataMaster::Tiling as u32 + { + return Err(EINVAL); + } + let fragment = self.work_publication(fragment)?; + let tiling = self.work_publication(tiling)?; + fragment.copy_record(); + commit_accounting()?; + fence(Ordering::SeqCst); + if !deferred { + fragment.producer.store(fragment.next, Ordering::Relaxed); + } + publish_late_tiling(); + tiling.copy_record(); + fence(Ordering::SeqCst); + if !deferred { + tiling.producer.store(tiling.next, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok([fragment.next, tiling.next]) + } + + fn retained_work_publication<'a>( + &'a self, + record: &'a super::fw::channels::WorkSlot, + next: u32, + ) -> Result> { + use super::fw::{channels, queue::DataMaster}; + let engine = match record.data_master { + 0 => DataMaster::Tiling, + 1 => DataMaster::Fragment, + _ => return Err(EINVAL), + }; + let channel = channels::work_ring_index(engine); + let producer = self.object(self.cluster)?.word( + Self::state_offset(Role::Primary) + + abi::work_state_offset(channel) + + abi::ChannelState::PRODUCER, + )?; + let current = producer.load(Ordering::Relaxed); + if current >= channels::WORK_RING_SLOTS || next != (current + 1) % channels::WORK_RING_SLOTS + { + return Err(EBUSY); + } + let destination = self.object(self.bundle)?.pointer( + abi::work_ring_offset(channel) + current as usize * core::mem::size_of_val(record), + core::mem::size_of_val(record), + )?; + Ok(WorkPublication { + record, + destination, + producer, + next, + }) + } + + /// Release a retained pair after its control transaction. Revalidate the original producer + /// witnesses before either copy; both bodies precede the fragment and tiling publications. + pub(super) fn publish_retained_work_pair( + &self, + fragment: &super::fw::channels::WorkSlot, + tiling: &super::fw::channels::WorkSlot, + next: [u32; 2], + ) -> Result { + use super::fw::queue::DataMaster; + if fragment.data_master != DataMaster::Fragment as u32 + || tiling.data_master != DataMaster::Tiling as u32 + { + return Err(EINVAL); + } + let fragment = self.retained_work_publication(fragment, next[0])?; + let tiling = self.retained_work_publication(tiling, next[1])?; + fragment.copy_record(); + tiling.copy_record(); + fence(Ordering::SeqCst); + fragment.producer.store(fragment.next, Ordering::Relaxed); + fence(Ordering::SeqCst); + tiling.producer.store(tiling.next, Ordering::Relaxed); + fence(Ordering::SeqCst); + Ok(()) + } + + /// Snapshot the shared compute completion ring once per service transaction. The queue + /// scanner latches exact identities; firmware retains sole ownership of these slots. + pub(super) fn scan_compute_completions( + &self, + mut record: impl FnMut([u64; 8]) -> Result, + ) -> Result { + let object = self.object(self.completion_rings[0])?; + fence(Ordering::Acquire); + for slot in 0..(abi::COMPLETION_RING_SIZE / 0x40).min(0x800) { + let mut words = [0; 8]; + for (index, word) in words.iter_mut().enumerate() { + *word = object + .dword(slot * 0x40 + index * 8)? + .load(Ordering::Relaxed); + } + record(words)?; + } + Ok(()) + } + + fn object(&self, index: usize) -> Result<&KernelObject> { + self.objects.get(index).ok_or(EINVAL) + } + + fn object_mut(&mut self, index: usize) -> Result<&mut KernelObject> { + self.objects.get_mut(index).ok_or(EINVAL) + } + + fn add(&mut self, object: KernelObject) -> Result { + let index = self.objects.len(); + self.objects.push(object, GFP_KERNEL)?; + Ok(index) + } + + fn alias(&mut self, index: usize, vm: &mmu::Vm, va: u64, prot: mmu::Prot) -> Result { + let alias = self.object_mut(index)?.map_alias(vm, va, prot)?; + self.aliases.push(alias, GFP_KERNEL)?; + Ok(()) + } + + /// Constructs the graph after the firmware has published the high-root entries. + pub(super) fn build( + &mut self, + dev: &AsahiDrmDevice, + cfg: &Config, + uat: &mmu::Uat, + render: &mmu::Vm, + metrics: &KernelObject, + ) -> Result { + let vm = uat.kernel_vm(); + let lower = uat.kernel_lower_vm(); + let range = uat.geometry().kernel_range(); + let base = range.start; + let dynamic = base + t8140::dynamic::KERNEL_OFFSET..range.end; + let fixed = |offset, size, prot, cpu| { + KernelObject::new(dev, vm, Placement::At(base + offset), size, prot, cpu) + }; + let dynamic_object = |size, align, prot, cpu| { + KernelObject::new( + dev, + vm, + Placement::In(dynamic.clone(), align), + size, + prot, + cpu, + ) + }; + use mmu::{ + PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_PRIV_RW as GFP, + PROT_GPU_FW_SHARED_RW as GFS, PROT_GPU_SHARED_RW as GS, + }; + use CpuMap::{WriteBack as WB, WriteCombined as WC}; + + for window in &t8140::REGISTER_WINDOWS { + let (phys, va, size) = window.mapping().ok_or(EINVAL)?; + self.aliases.push( + vm.map_io( + dev.as_ref().as_ref(), + va, + phys.try_into()?, + size.try_into()?, + mmu::PROT_FW_MMIO_RW, + )?, + GFP_KERNEL, + )?; + } + self.cluster = self.add(fixed(high::PRIVATE_CLUSTER, abi::private::SIZE, FS, WC)?)?; + self.compute_global = self.add(fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?; + let pb = self.add(fixed(high::PB_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; + self.object_mut(pb)? + .initialize::(0, |table| { + table.init(); + Ok(()) + })?; + let page_pool = self.add(fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; + self.object_mut(page_pool)? + .initialize::(0, |table| { + table.init(low::RENDER_PAGE_LIST, low::COMPUTE_POOL_PAGE_LIST); + Ok(()) + })?; + self.pb = pb; + self.page_pool = page_pool; + self.alias(pb, lower, low::PB_DESCRIPTORS, GS)?; + self.alias(page_pool, lower, low::PAGE_POOL_DESCRIPTORS, GS)?; + let index = self.add(fixed(high::INDEX, INDEX_SIZE, GFS, WB)?)?; + self.object_mut(index)? + .initialize::(0, |table| { + table.init(); + Ok(()) + })?; + self.alias(index, lower, low::INDEX, GFS)?; + let fwctl = self.add(fixed(high::FWCTL, abi::FWCTL_SIZE, FS, WB)?)?; + + for (offset, state, class, prot, initial) in [ + ( + high::RENDER_FREE_LIST, + high::RENDER_FREE_LIST_STATE, + abi::FreeListClass::One, + FP, + abi::FreeListState::RENDER, + ), + ( + high::BOOT_FREE_LIST, + high::BOOT_FREE_LIST_STATE, + abi::FreeListClass::Two, + GFP, + abi::FreeListState::BOOT, + ), + ] { + let control = self.add(fixed(offset, mmu::UAT_PGSZ, prot, WC)?)?; + self.object_mut(control)?.write( + 0, + abi::FreeListControl::new(&abi::FreeListArgs { + buffer_slot: 0, + class, + page_list_va: low::RENDER_PAGE_LIST, + run_list_va: low::RENDER_RUN_LIST, + blocks: low::RENDER_BLOCKS_INITIAL, + state_va: base + state, + }), + )?; + let state_prot = if class == abi::FreeListClass::One { + FS + } else { + GFS + }; + let state = self.add(fixed(state, mmu::UAT_PGSZ, state_prot, WC)?)?; + self.object_mut(state)?.write(0, initial)?; + if class == abi::FreeListClass::Two { + self.boot_control = control; + self.boot_state = state; + } + } + self.stamps[0] = self.add(fixed(high::DESCRIPTOR_PAGE_A, mmu::UAT_PGSZ, GFS, WC)?)?; + self.stamps[1] = self.add(fixed(high::DESCRIPTOR_PAGE_B, mmu::UAT_PGSZ, GFP, WC)?)?; + self.add(fixed(high::TA_CONTEXT, CONTEXT_SIZE, FS, WC)?)?; + self.add(fixed(high::FRAGMENT_CONTEXT, CONTEXT_SIZE, FS, WC)?)?; + + let pages = self.add(KernelObject::backing(dev, low::RENDER_PAGE_LIST_SIZE, WB)?)?; + self.object_mut(pages)? + .initialize::<[u64; low::RENDER_PAGE_LIST_SIZE / 8]>(0, |list| { + abi::fill_page_list( + list, + (0..low::RENDER_BLOCKS_INITIAL as usize).map(low::render_block), + ) + })?; + let runs = self.add(KernelObject::backing(dev, low::RENDER_RUN_LIST_SIZE, WB)?)?; + let first_block = self.objects.len(); + for _ in 0..low::RENDER_BLOCKS_ALLOCATED { + self.add(KernelObject::backing(dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; + } + self.render_pages = pages; + self.render_runs = runs; + self.render_blocks = first_block..self.objects.len(); + self.alias(pages, render, low::RENDER_PAGE_LIST, GS)?; + self.alias(runs, render, low::RENDER_RUN_LIST, GS)?; + for block in 0..low::RENDER_BLOCKS_MAPPED { + self.alias(first_block + block, render, low::render_block(block), GS)?; + } + + let compute = self.add(fixed(high::COMPUTE_FREE_LIST, mmu::UAT_PGSZ, GFS, WC)?)?; + self.object_mut(compute)?.write( + 0, + abi::FreeListControl::new(&abi::FreeListArgs { + buffer_slot: 0, + class: abi::FreeListClass::One, + page_list_va: low::COMPUTE_PAGE_LIST, + run_list_va: low::COMPUTE_RUN_LIST, + blocks: low::COMPUTE_BLOCKS, + state_va: base + high::COMPUTE_FREE_LIST_STATE, + }), + )?; + let compute_state = self.add(KernelObject::backing(dev, mmu::UAT_PGSZ, WB)?)?; + self.object_mut(compute_state)? + .write(0, abi::FreeListState::COMPUTE)?; + for _ in &low::COMPUTE_ZERO_BLOCKS { + self.add(KernelObject::backing(dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; + } + let compute_runs = self.add(KernelObject::new( + dev, + lower, + Placement::At(low::COMPUTE_RUN_LIST), + mmu::UAT_PGSZ, + GFS, + WC, + )?)?; + self.object_mut(compute_runs)? + .initialize::<[abi::FreeListRunSlot; mmu::UAT_PGSZ / 0x40]>(0, |list| { + abi::fill_run_list(list, low::COMPUTE_FREE_LIST_BLOCKS) + })?; + let ready = self.add(fixed( + high::COMPUTE_READY_FREE_LIST, + mmu::UAT_PGSZ, + GFS, + WC, + )?)?; + self.object_mut(ready)?.write( + 0, + abi::FreeListControl::new(&abi::FreeListArgs { + buffer_slot: 1, + class: abi::FreeListClass::One, + page_list_va: low::COMPUTE_POOL_PAGE_LIST, + run_list_va: low::COMPUTE_POOL_RUN_LIST, + blocks: low::COMPUTE_READY_BLOCKS, + state_va: base + high::COMPUTE_READY_FREE_LIST_STATE, + }), + )?; + for state in [high::COMPUTE_READY_FREE_LIST_STATE, high::COMPUTE_STATE] { + let obj = self.add(fixed(state, mmu::UAT_PGSZ, GFS, WC)?)?; + self.object_mut(obj)? + .write(0, abi::FreeListState::COMPUTE)?; + } + self.object_mut(first_block + low::COMPUTE_PAGE_LIST_BLOCK)? + .initialize::<[u64; abi::FREE_LIST_BLOCK_SIZE / 8]>(0, |list| { + abi::fill_page_list(list, low::COMPUTE_FREE_LIST_BLOCKS) + })?; + fence(Ordering::SeqCst); + + self.bundle = self.add(fixed(high::BUNDLE, abi::bundle::SIZE, FP, WC)?)?; + let zero = self.add(fixed( + high::BUNDLE + abi::bundle::ZERO_PAGE as u64, + mmu::UAT_PGSZ, + GFS, + WC, + )?)?; + let mut completion_rings = [abi::CompletionRing { + low_va: 0, + high_va: 0, + }; 2]; + for slot in 0..2 { + let offset = high::BUNDLE + abi::bundle::COMPLETION_RINGS[slot] as u64; + let ring = self.add(fixed(offset, abi::COMPLETION_RING_SIZE, FP, WB)?)?; + self.completion_rings[slot] = ring; + self.alias(ring, lower, low::COMPLETION_RINGS[slot], GS)?; + completion_rings[slot] = abi::CompletionRing { + low_va: low::COMPLETION_RINGS[slot], + high_va: base + offset, + }; + } + let qos = self.add(dynamic_object(abi::QOS_SIZE, mmu::UAT_PGSZ as u64, FS, WC)?)?; + self.qos = qos; + fence(Ordering::SeqCst); + let metrics_alias = + metrics.alias_in(vm, dynamic.clone(), cfg.pm_metrics.size as u64, GFS)?; + self.metrics_fw_va = metrics_alias.iova(); + self.aliases.push(metrics_alias, GFP_KERNEL)?; + fence(Ordering::SeqCst); + self.roots = self.add(dynamic_object( + abi::ROOTS_SIZE, + abi::SECONDARY_ROOT_OFFSET as u64, + mmu::PROT_FW_SHARED_RO, + WB, + )?)?; + let region_a = self.add(dynamic_object( + REGION_A_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RO, + WB, + )?)?; + self.region_c = self.add(dynamic_object(REGION_C_SIZE, mmu::UAT_PGSZ as u64, FS, WB)?)?; + + let opps = Self::opps(dev)?; + let hw_args = abi::HwDataArgs { + chip_id: cfg.chip_id, + windows: &t8140::REGISTER_WINDOWS, + reserved_slots: &t8140::RESERVED_REGISTER_SLOTS, + opps: &opps, + perf: &t8140::PERF, + completion_rings, + qos_va: self.object(qos)?.gpu_va(), + }; + self.object_mut(self.bundle)? + .initialize::(abi::bundle::HW_DATA, |hw| hw.init(&hw_args))?; + self.object_mut(self.region_c)? + .initialize::(0, |region| { + region.init(); + Ok(()) + })?; + self.object_mut(self.cluster)? + .write(abi::private::PRIMARY_STATUS_A, abi::StatusBlock::new())?; + self.object_mut(self.cluster)? + .write(abi::private::SECONDARY_STATUS_A, abi::StatusBlock::new())?; + let fwctl_va = self.object(fwctl)?.gpu_va(); + self.object_mut(self.cluster)? + .initialize::(abi::private::STATUS_B, |status| { + status.init(fwctl_va); + Ok(()) + })?; + self.object_mut(self.cluster)? + .initialize::( + abi::private::SECONDARY_STATE + abi::private::STATUS_B, + |config| { + config.init(); + Ok(()) + }, + )?; + self.object_mut(self.bundle)? + .initialize::(abi::bundle::POWER_CONFIG, |power| { + power.init(); + Ok(()) + })?; + let secondary_status = base + + high::PRIVATE_CLUSTER + + (abi::private::SECONDARY_STATE + abi::private::SECONDARY_STATUS) as u64; + self.object_mut(self.bundle)? + .initialize::(abi::bundle::AUX, |aux| { + aux.init(secondary_status); + Ok(()) + })?; + self.object_mut(self.bundle)? + .initialize::(abi::bundle::POWER_CONFIG_COPY, |power| { + power.init(); + Ok(()) + })?; + let views = abi::PrimaryViews { + zero_page_va: self.object(zero)?.gpu_va(), + pb_descriptors_va: self.object(pb)?.gpu_va(), + page_pool_descriptors_va: self.object(page_pool)?.gpu_va(), + }; + for role in [Role::Primary, Role::Secondary] { + let slot = role as usize; + let main_offset = abi::bundle::MAIN_CONFIG[slot]; + let main_va = base + high::BUNDLE + main_offset as u64; + let state_va = base + high::PRIVATE_CLUSTER + Self::state_offset(role) as u64; + let status_va = base + high::PRIVATE_CLUSTER + Self::status_offset(role) as u64; + let args = abi::MainConfigArgs { + bundle_va: base + high::BUNDLE, + main_config_va: main_va, + state_grid_va: state_va, + status_a_va: status_va, + pb_descriptors_low_va: low::PB_DESCRIPTORS, + page_pool_descriptors_low_va: low::PAGE_POOL_DESCRIPTORS, + }; + self.object_mut(self.bundle)? + .initialize::(main_offset, |main| { + match role { + Role::Primary => main.init_primary(&args, &views), + Role::Secondary => main.init_secondary(&args), + }; + Ok(()) + })?; + let abi_role = match role { + Role::Primary => abi::Role::Primary, + Role::Secondary => abi::Role::Secondary, + }; + self.object_mut(self.bundle)?.write( + main_offset + abi::MainConfig::CONTROL_RING, + abi::OpeningRecord::new(abi_role), + )?; + let root = abi::RootArgs { + region_a_va: self.object(region_a)?.gpu_va(), + region_c_va: self.object(self.region_c)?.gpu_va(), + main_config_va: main_va, + status_a_va: status_va, + ias: cfg.uat_ias, + oas: cfg.uat_oas, + }; + match role { + Role::Primary => self.object_mut(self.roots)?.write( + 0, + abi::Root::primary(&root, state_va + abi::private::STATUS_B as u64), + )?, + Role::Secondary => self.object_mut(self.roots)?.write( + abi::SECONDARY_ROOT_OFFSET, + abi::SecondaryRoot::new( + &root, + base + high::PRIVATE_CLUSTER + + (abi::private::STATUS_B + abi::PrimaryStatusB::POWER_STATE) as u64, + state_va + abi::private::STATUS_B as u64, + ), + )?, + } + } + for role in [Role::Primary, Role::Secondary] { + for (offset, value) in [ + (0, 0), + (0x10, 0), + (abi::ChannelState::PRODUCER, abi::OPENING_PRODUCER), + ] { + self.object(self.cluster)? + .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .store(value, Ordering::Relaxed); + } + } + fence(Ordering::SeqCst); + Ok(()) + } + + /// Retained fixed aliases for an ordinary client VM. The final block of the render-global + /// pool overlaps completion mappings in a client root and is deliberately not installed. + pub(super) fn user_aliases( + &mut self, + vm: &mmu::Vm, + metrics: &KernelObject, + ) -> Result> { + let mut mappings = KVec::with_capacity(24, GFP_KERNEL)?; + mappings.push( + self.object_mut(self.render_pages)?.map_alias( + vm, + low::RENDER_PAGE_LIST, + mmu::PROT_GPU_SHARED_RW, + )?, + GFP_KERNEL, + )?; + mappings.push( + self.object_mut(self.render_runs)?.map_alias( + vm, + low::RENDER_RUN_LIST, + mmu::PROT_GPU_SHARED_RW, + )?, + GFP_KERNEL, + )?; + for (block, index) in self.render_blocks.clone().take(21).enumerate() { + mappings.push( + self.object_mut(index)?.map_alias( + vm, + low::render_block(block), + mmu::PROT_GPU_SHARED_RW, + )?, + GFP_KERNEL, + )?; + } + mappings.push( + metrics.map_alias(vm, t8140::CONFIG.pm_metrics.va, mmu::PROT_GPU_FW_SHARED_RW)?, + GFP_KERNEL, + )?; + Ok(mappings) + } + + /// Writes the effort input in firmware order, then cleans its cache line to coherency. + pub(super) fn publish_effort(&self, effort: u32) -> Result { + let region = self.object(self.region_c)?; + let override_word = region.word(abi::RegionC::UNK_E8)?; + let effort_word = region.word(abi::RegionC::EFFORT)?; + let line = region.pointer(abi::RegionC::EFFORT, 4)?; + override_word.store(0, Ordering::Relaxed); + effort_word.store(effort, Ordering::Relaxed); + // SAFETY: line points into the retained kernel mapping; cleaning this host-owned cache + // line and the following system barrier publish the two naturally aligned stores. + unsafe { + core::arch::asm!("dc cvac, {line}", "dsb sy", line = in(reg) line, + options(nostack, preserves_flags)) + }; + Ok(()) + } + + pub(super) fn effort(&self) -> Result { + Ok(self + .object(self.region_c)? + .word(abi::RegionC::EFFORT)? + .load(Ordering::Relaxed)) + } + + fn opps(dev: &AsahiDrmDevice) -> Result> { + let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; + let table = node + .parse_phandle(c_str!("operating-points-v2"), 0) + .ok_or(EIO)?; + let mut opps = KVec::new(); + for opp in table.children() { + opps.push( + abi::Opp { + freq_hz: opp.get_property(c_str!("opp-hz"))?, + volt_uv: opp.get_property(c_str!("opp-microvolt"))?, + }, + GFP_KERNEL, + )?; + } + if opps.len() != t8140::OPPS { + return Err(EINVAL); + } + Ok(opps) + } + + fn state_offset(role: Role) -> usize { + match role { + Role::Primary => abi::private::PRIMARY_STATE, + Role::Secondary => abi::private::SECONDARY_STATE, + } + } + + fn status_offset(role: Role) -> usize { + match role { + Role::Primary => abi::private::PRIMARY_STATUS_A, + Role::Secondary => abi::private::SECONDARY_STATUS_A, + } + } + + /// Recheck the host-authored event pointer pair after a bounded number of drains. + /// The device mutex serializes the cache and all event consumption. + pub(super) fn validate_event_ring(&self, role: Role) -> Result { + let status = self.object(self.cluster)?.gpu_va() + Self::status_offset(role) as u64; + let expected = ( + status + abi::status::EVENT_STATE, + status + abi::status::EVENT_RING, + ); + let cache = &self.event_ring_checked[role as usize]; + let checked = cache.get(); + if checked.2 != 0 && (checked.0, checked.1) == expected { + cache.set((checked.0, checked.1, checked.2 - 1)); + return Ok(()); + } + let offset = abi::bundle::MAIN_CONFIG[role as usize] + + core::mem::offset_of!(abi::MainConfig, channels) + + abi::EVENT_CHANNEL * size_of::(); + let bundle = self.object(self.bundle)?; + let actual = ( + bundle.dword(offset)?.load(Ordering::Relaxed), + bundle + .dword(offset + size_of::())? + .load(Ordering::Relaxed), + ); + if actual != expected { + cache.set((0, 0, 0)); + return Err(EINVAL); + } + cache.set((actual.0, actual.1, EVENT_RING_RECHECK)); + Ok(()) + } + + pub(super) fn status_word(&self, role: Role, offset: usize) -> Result<&AtomicU32> { + self.object(self.cluster)?.word( + Self::status_offset(role) + .checked_add(offset) + .ok_or(EINVAL)?, + ) + } + + pub(super) fn status_read_words( + &self, + role: Role, + offset: usize, + ) -> Result<[u32; N]> { + self.object(self.cluster)?.read_words( + Self::status_offset(role) + .checked_add(offset) + .ok_or(EINVAL)?, + ) + } + + pub(super) fn control_counters(&self) -> Result<[u32; 3]> { + let state = self.object(self.cluster)?; + let mut counters = [0; 3]; + fence(Ordering::Acquire); + for (counter, offset) in counters.iter_mut().zip([ + abi::ChannelState::CONSUMER, + abi::ChannelState::UNK_10, + abi::ChannelState::PRODUCER, + ]) { + *counter = state + .word(abi::CONTROL_STATE_OFFSET + offset)? + .load(Ordering::Relaxed); + } + Ok(counters) + } + + pub(super) fn control_consumer(&self) -> Result<&AtomicU32> { + self.object(self.cluster)? + .word(abi::CONTROL_STATE_OFFSET + abi::ChannelState::CONSUMER) + } + + pub(super) fn control_producer(&self) -> Result<&AtomicU32> { + self.object(self.cluster)? + .word(abi::CONTROL_STATE_OFFSET + abi::ChannelState::PRODUCER) + } + + /// Publishes a control record under the device lock. All checked views are acquired before + /// the first write; record and packed power tally precede the aligned producer store. + pub(super) fn publish_control( + &self, + slot: u32, + record: &[u8; 64], + asserts_power: bool, + next: u32, + ) -> Result { + let offset = abi::bundle::MAIN_CONFIG[Role::Primary as usize] + + abi::MainConfig::CONTROL_RING + + (slot as usize).checked_mul(record.len()).ok_or(EINVAL)?; + let ring = self.object(self.bundle)?.pointer(offset, record.len())?; + let power = self + .object(self.region_c)? + .pointer(abi::RegionC::POWER_ASSERT_TALLY, 4)?; + let producer = self.control_producer()?; + // SAFETY: The checked pointers remain valid for the lifetime of this non-Sync owner. + // Firmware only reads records after the producer store and the device mutex serializes + // host publishers. The unaligned power word uses the interface's four-byte copy access. + unsafe { + core::ptr::copy_nonoverlapping(record.as_ptr(), ring, record.len()); + if asserts_power { + let mut tally = [0u8; 4]; + core::ptr::copy_nonoverlapping(power, tally.as_mut_ptr(), 4); + let value = u32::from_le_bytes(tally).wrapping_add(1).to_le_bytes(); + core::ptr::copy_nonoverlapping(value.as_ptr(), power, 4); + } + } + fence(Ordering::SeqCst); + producer.store(next, Ordering::Relaxed); + fence(Ordering::SeqCst); + Ok(()) + } + + pub(super) fn root_va(&self, role: Role) -> Result { + Ok(self.object(self.roots)?.gpu_va() + role as u64 * abi::SECONDARY_ROOT_OFFSET as u64) + } + + pub(super) fn opening_retired(&self, role: Role) -> Result { + for offset in [0, 0x10, abi::ChannelState::PRODUCER] { + if self + .object(self.cluster)? + .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .load(Ordering::SeqCst) + != abi::OPENING_PRODUCER + { + return Ok(false); + } + } + Ok(true) + } +} + +impl InitData { + /// Validate the serialized view before reading its retained host mapping. + fn recovery_view_words(&self, view: usize, offset: usize) -> Result<[u32; N]> { + let bundle = self.object(self.bundle)?; + let base = *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?; + let extent = *abi::recovery::VIEW_EXTENTS.get(view).ok_or(EINVAL)?; + let end = offset.checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) + .ok_or(EOVERFLOW)?; + if end > extent { + return Err(EINVAL); + } + let pointer_at = abi::bundle::MAIN_CONFIG[Role::Primary as usize] + + core::mem::offset_of!(abi::MainConfig, view_va) + view * size_of::(); + fence(Ordering::Acquire); + let pointer = bundle.read_words::<2>(pointer_at)?; + let actual = u64::from(pointer[0]) | u64::from(pointer[1]) << 32; + if actual != bundle.gpu_va().checked_add(base as u64).ok_or(EOVERFLOW)? { + return Err(EINVAL); + } + bundle.read_words(base + offset) + } +} + +impl super::recovery::Memory for InitData { + fn fault_sources(&self) -> Result { + let mask = self.object(self.cluster)? + .dword(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? + .load(Ordering::Relaxed); + let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { + let key = super::recovery::slot_key(slot).unwrap_or(0); + let monitor = self.recovery_view_words::<6>(abi::recovery::PROGRESS_VIEW, + abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE) + .map(|words| words[0]).unwrap_or(0); + (key, monitor) + }); + sources.reason = self.recovery_view_words::<1>(abi::recovery::REPORT_VIEW, abi::recovery::REASON) + .map(|words| words[0]).unwrap_or(0); + Ok(sources) + } + + fn recovery_state(&self) -> Result { + fence(Ordering::Acquire); + Ok(self + .object(self.cluster)? + .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)? + .load(Ordering::Relaxed)) + } + + fn host_recovery(&self) -> Result { + Ok(self + .object(self.cluster)? + .word(abi::private::STATUS_B + abi::PrimaryStatusB::HOST_RECOVERY)? + .load(Ordering::Relaxed)) + } + + fn transition(&self, expected: u32, next: u32) -> Result { + let state = self + .object(self.cluster)? + .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?; + fence(Ordering::Acquire); + if state.load(Ordering::Relaxed) != expected { + return Err(EBUSY); + } + fence(Ordering::SeqCst); + state.store(next, Ordering::Relaxed); + fence(Ordering::SeqCst); + Ok(()) + } + + fn clear_timestamps(&self) -> Result { + let bundle = self.object(self.bundle)?; + // Validate the complete table before publishing its first live word. + const ENTRIES: usize = 128; + const STRIDE: usize = 0x10; + let offset = abi::bundle::VIEWS[4]; + bundle.pointer(offset, ENTRIES * STRIDE)?; + for qid in 0..ENTRIES { + bundle + .word(offset + qid * STRIDE)? + .store(0, Ordering::Relaxed); + bundle + .dword(offset + qid * STRIDE + 8)? + .store(0, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } + + fn entries( + &self, + out: &mut [super::recovery::Entry; super::recovery::MAX_ENTRIES], + ) -> Result { + let cluster = self.object(self.cluster)?; + let base = abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_INFO; + fence(Ordering::Acquire); + let mut used = 0; + for index in 0..256 { + if used == out.len() { + break; + } + let offset = base + index * core::mem::size_of::(); + let flags = cluster.word(offset)?.load(Ordering::Relaxed); + if flags & 1 == 0 { + continue; + } + let stamp = cluster.word(offset + 4)?.load(Ordering::Relaxed); + if let Some(entry) = super::recovery::Entry::decode(flags, stamp) { + out[used] = entry; + used += 1; + } + } + Ok(used) + } + + fn wait_tick(&self) { + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs new file mode 100644 index 0000000000000..60cc344fd5116 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Immutable submissions and the DRM scheduler boundary for independent engines. + +use super::{command::Validated, completion::Completion, context::Context}; +use crate::driver::AsahiDrmDevice; +use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU8, Ordering}; +use kernel::{ + bindings, c_str, + dma_fence::{Fence, RawDmaFence}, + drm::sched, + prelude::*, + sync::Arc, +}; + +const TIMEOUT_MS: kernel::time::Msecs = 2000; +const RENDER_CREDITS: u32 = 32; +const COMPUTE_CREDITS: u32 = 16; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Engine { + Render, + Compute, +} + +impl Engine { + pub(crate) fn index(self) -> usize { + match self { + Self::Render => 0, + Self::Compute => 1, + } + } +} + +/// Logical sequence numbers do not wrap with firmware timestamps. A zero barrier +/// includes preceding ioctls; the sentinel means independent of that engine. +#[derive(Clone, Copy)] +pub(crate) struct Order { + pub(crate) sequence: u64, + pub(crate) wait_through: [Option; 2], +} + +impl Order { + pub(crate) fn new( + base: [u64; 2], + preceding: [u16; 2], + barriers: [u16; 2], + engine: Engine, + ) -> Result { + let mut sequences = [0; 2]; + let mut wait_through = [None; 2]; + for index in 0..2 { + sequences[index] = base[index] + .checked_add(u64::from(preceding[index])) + .and_then(|value| value.checked_add(1)) + .ok_or(EOVERFLOW)?; + if barriers[index] != u16::MAX { + if barriers[index] > preceding[index] { + return Err(EINVAL); + } + wait_through[index] = Some( + base[index] + .checked_add(u64::from(barriers[index])) + .ok_or(EOVERFLOW)?, + ); + } + } + Ok(Self { + sequence: sequences[engine.index()], + wait_through, + }) + } + + pub(crate) fn contains(prefix: Option, sequence: u64) -> bool { + sequence != 0 && prefix.is_some_and(|last| sequence <= last) + } +} + +pub(crate) struct Packet { + pub(crate) id: u64, + pub(crate) context: Arc, + pub(crate) command: Validated, + pub(crate) order: Order, + pub(crate) completion: Arc, + pub(crate) footprint: Option>, + inputs: KVec, + firmware: KVec, + publication: AtomicU8, + cancelled: AtomicBool, + early: AtomicI32, + backlog: AtomicBool, +} + +impl Packet { + pub(crate) fn new( + id: u64, + context: Arc, + command: Validated, + order: Order, + completion: Arc, + inputs: KVec, + firmware: KVec, + ) -> Result> { + let footprint = match &command { + Validated::Render { attachments, .. } => { + context.vm().render_footprint(&attachments[1])? + } + Validated::Compute { .. } => None, + }; + Ok(Arc::new( + Self { + id, + context, + command, + order, + completion, + footprint, + inputs, + firmware, + publication: AtomicU8::new(0), + cancelled: AtomicBool::new(false), + early: AtomicI32::new(0), + backlog: AtomicBool::new(false), + }, + GFP_KERNEL, + )?) + } + + pub(crate) fn engine(&self) -> Engine { + match &self.command { + Validated::Render { .. } => Engine::Render, + Validated::Compute { .. } => Engine::Compute, + } + } + + /// Scheduler dependency readiness includes failed fences. Consume the actual + /// status before publication, and retain the references through that check. + pub(crate) fn check_dependencies(&self) -> Result { + for fence in &self.inputs { + // SAFETY: The retained fence owns the pointer throughout the wait and read. + let waited = unsafe { + bindings::dma_fence_wait_timeout( + fence.raw(), + true, + kernel::task::MAX_SCHEDULE_TIMEOUT, + ) + }; + if waited < 0 { + return Err(super::dependency::inherited_error(Error::from_errno(waited as i32))); + } + if waited == 0 { + return Err(ECANCELED); + } + check_status(fence)?; + } + for fence in &self.firmware { + check_status(fence)?; + } + Ok(()) + } + + /// Dependency-bearing packets remain on the scheduler publication path. + pub(crate) fn has_dependencies(&self) -> bool { + !self.inputs.is_empty() || !self.firmware.is_empty() + } + + pub(crate) fn begin_publication(&self) -> bool { + self.publication + .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } + pub(crate) fn mark_published(&self) { + self.publication.store(2, Ordering::Release); + } + pub(crate) fn is_published(&self) -> bool { + self.publication.load(Ordering::Acquire) == 2 + } + pub(crate) fn cancel_requested(&self) -> bool { + self.cancelled.load(Ordering::Acquire) + } + pub(crate) fn request_cancel(&self) -> bool { + self.cancelled.store(true, Ordering::Release); + self.publication + .compare_exchange(0, 3, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } + pub(crate) fn set_early_result(&self, result: Result) { + self.early.store( + result.map_or_else(|error| error.to_errno(), |_| 1), + Ordering::Release, + ); + } + pub(crate) fn early_result(&self) -> Option { + match self.early.load(Ordering::Acquire) { + 0 => None, + 1 => Some(Ok(())), + error => Some(Err(Error::from_errno(error))), + } + } + pub(crate) fn count_backlog(&self) { + self.backlog.store(true, Ordering::Release); + } + pub(crate) fn take_backlog(&self) -> bool { + self.backlog.swap(false, Ordering::AcqRel) + } +} + +pub(crate) fn fence_status(fence: &Fence) -> i32 { + // SAFETY: The owned fence keeps the raw DMA fence alive for this read. + unsafe { bindings::dma_fence_get_status(fence.raw()) } +} + +fn check_status(fence: &Fence) -> Result { + let status = fence_status(fence); + if status < 0 { + Err(super::dependency::inherited_error(Error::from_errno(status))) + } else { + Ok(()) + } +} + +/// The backend retains visible packets through exact retirement or processor stop. +/// It performs all completion signalling after releasing the device mutex. +pub(crate) trait Backend: Send + Sync { + /// Lazily create or rebind this logical queue's physical compute graph. + fn ensure_compute(&self) -> Result; + /// Settle outstanding work of a terminal VM before an ioctl error is published. + fn fail_vm(&self, error: Error); + /// Called after both scheduler entities have cancelled their queued jobs. + fn release_owner(&self); + fn prepare(&self, packet: &Packet) -> Option; + fn publish(&self, packet: Arc) -> Result; + fn timed_out(&self, packet: Arc) -> sched::Status; + fn cancel(&self, packet: Arc); + /// Transfer a timeout to an outstanding off-lock graph transaction, if any. + fn defer_timeout(&self, packet: &Arc) -> bool; + /// Return true only after taking publication ownership and recording its result. + fn try_early(&self, packet: &Arc) -> bool; + fn count_backlog(&self, packet: &Packet); +} + +struct Job { + backend: Arc, + packet: Arc, +} + +impl sched::JobImpl for Job { + fn false_timeout(job: &mut sched::Job) -> bool { + job.is_finished() + || job.packet.completion.take_replay_timeout() + || job.backend.defer_timeout(&job.packet) + } + + fn prepare(job: &mut sched::Job) -> Option { + job.backend.prepare(&job.packet) + } + fn run(job: &mut sched::Job) -> Result> { + job.backend.publish(job.packet.clone()).map(Some) + } + fn timed_out(job: &mut sched::Job) -> sched::Status { + // The scheduler retains the detached job throughout this callback. Keep + // its parent callback intact while the driver settles the exact owner; + // reinsertion also handles completion racing with timeout observation. + if !job.is_finished() { + job.backend.timed_out(job.packet.clone()); + } + sched::Status::NoHang + } + fn cancel(job: &mut sched::Job) { + job.backend.cancel(job.packet.clone()); + } +} + +pub(crate) struct Scheduler { + // Cancel entity jobs before dropping the scheduler's owning reference. + entity: sched::Entity>, + _scheduler: sched::Scheduler>, + backend: Arc, +} + +#[derive(Clone)] +pub(crate) struct Fences { + pub(crate) ready: Fence, + pub(crate) completed: Fence, +} + +impl Scheduler { + pub(crate) fn new(dev: &AsahiDrmDevice, backend: Arc, engine: Engine) -> Result { + let (credits, name) = match engine { + Engine::Render => (RENDER_CREDITS, c_str!("asahi_g17_render")), + Engine::Compute => (COMPUTE_CREDITS, c_str!("asahi_g17_compute")), + }; + let scheduler = sched::Scheduler::new(dev.as_ref().as_ref(), 1, credits, 0, TIMEOUT_MS, name)?; + let entity = sched::Entity::new(&scheduler, sched::Priority::Kernel)?; + Ok(Self { + entity, + _scheduler: scheduler, + backend, + }) + } + + pub(crate) fn enqueue( + &mut self, + packet: Arc, + dependencies: KVec, + early: bool, + ) -> Result { + let completed = packet.completion.fence(); + let host_timestamps = packet.completion.has_timestamps(); + let mut job = self.entity.new_job( + 1, + Job { + backend: self.backend.clone(), + packet: packet.clone(), + }, + )?; + for dependency in dependencies { + job.add_dependency(dependency)?; + } + let mut job = job.arm(); + let ready = if host_timestamps { + completed.clone() + } else { + job.fences().scheduled() + }; + // All fallible scheduler setup precedes early hardware publication. The + // scheduler still owns watchdog, credits and cancellation after push. + if packet.engine() == Engine::Render && !(early && self.backend.try_early(&packet)) { + self.backend.count_backlog(&packet); + } + job.push(); + Ok(Fences { ready, completed }) + } +} diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs new file mode 100644 index 0000000000000..9d89262d8a3b6 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Physical kick-ring backing, queue identities and entry publication. +//! +//! The device mutex serializes identities and publication. Installed identities +//! and their ring backing stay owned until both firmware processors stop; +//! command completion alone does not detach a hardware queue. + +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +use super::{ + fw::{kick::*, queue::DataMaster}, + object::{Allocator, CpuMap, KernelObject}, +}; +use crate::{hw::t8140, mem, mmu}; + +const QID_COUNT: usize = QID_MAX as usize + 1; + +/// Stable physical-channel identity. The owner must not be reused within a +/// device lifetime, including after an unpublished allocation is cancelled. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Id { + qid: u8, + owner: u64, + engine: DataMaster, +} + +impl Id { + pub(crate) fn qid(self) -> u8 { + self.qid + } + pub(crate) fn engine(self) -> DataMaster { + self.engine + } +} + +#[derive(Copy, Clone)] +struct Identity { + id: Id, + published: bool, +} + +/// One shared namespace for tiling, fragment and compute queues. +pub(crate) struct QueueIds { + entries: [Option; QID_COUNT], +} + +impl QueueIds { + pub(crate) const fn new() -> Self { + Self { + entries: [None; QID_COUNT], + } + } + + /// Reuses only an exact owner/engine match. Preferred placement is for + /// bootstrapping; it does not reserve a partition for an engine. + pub(crate) fn reserve( + &mut self, + owner: u64, + engine: DataMaster, + preferred: Option, + ) -> Result { + if owner == 0 { + return Err(EINVAL); + } + if let Some(entry) = self + .entries + .iter() + .flatten() + .find(|entry| entry.id.owner == owner) + { + return if entry.id.engine == engine { + Ok(entry.id) + } else { + Err(EINVAL) + }; + } + let index = preferred + .map(usize::from) + .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) + .or_else(|| self.entries.iter().position(Option::is_none)) + .ok_or(ENOSPC)?; + let id = Id { + qid: index as u8, + owner, + engine, + }; + self.entries[index] = Some(Identity { + id, + published: false, + }); + Ok(id) + } + + /// Call after preflighting the complete installation, before any address + /// becomes firmware-visible. A failed or partial installation retains it. + pub(crate) fn publish(&mut self, id: Id) -> Result { + let entry = self + .entries + .get_mut(usize::from(id.qid)) + .and_then(Option::as_mut) + .ok_or(EINVAL)?; + if entry.id != id { + return Err(EINVAL); + } + entry.published = true; + Ok(()) + } + + pub(crate) fn cancel_unpublished(&mut self, id: Id) -> Result { + let slot = self.entries.get_mut(usize::from(id.qid)).ok_or(EINVAL)?; + match slot { + Some(entry) if entry.id == id && !entry.published => { + *slot = None; + Ok(()) + } + _ => Err(EINVAL), + } + } +} + +// Aliases drop before their backing. A compute queue is allocated low first; +// a render queue is allocated high first. +enum Backing { + Compute { + high: mmu::KernelMapping, + low: KernelObject, + }, + Render { + low: mmu::KernelMapping, + high: KernelObject, + }, +} + +/// An installed queue's permanent ring. Client aliases are held by the bound +/// context graph and must be dropped before this object. +pub(crate) struct Queue { + backing: Backing, + id: Id, + timestamp: KickTimestamp, + previous: Option, +} + +/// A checked entry and destination held without changing the shared ring. +/// Discarding it leaves both the slot and the next timestamp unchanged. +pub(crate) struct PreparedKick<'a> { + queue: &'a mut Queue, + timestamp: KickTimestamp, + entry: KickEntry, + destination: *mut KickEntry, +} + +impl PreparedKick<'_> { + /// Publish translation updates after the queue configuration pointer and before its entry. + pub(crate) fn prepare_compute_install(&self) -> Result { + self.queue.prepare_compute_install() + } + + /// Writes and fences the complete entry, then advances host state. The + /// caller invokes this between the queue configuration and announcement. + pub(crate) fn commit(self) -> KickTimestamp { + // SAFETY: prepare_entry checked the complete destination range and + // this token exclusively borrows the queue and retains its mapping. + // The caller only prepares retired ring slots; no firmware access to + // the new entry starts before the outer command is published. + unsafe { self.destination.write_unaligned(self.entry) }; + fence(Ordering::SeqCst); + self.queue.previous = Some(self.timestamp); + self.queue.timestamp = self.timestamp.next(); + self.timestamp + } +} + +impl Queue { + pub(crate) fn compute(alloc: &Allocator<'_>, id: Id) -> Result { + if id.engine != DataMaster::Compute { + return Err(EINVAL); + } + let low = alloc.lower( + KICK_RING_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + )?; + let mut range = alloc.uat.geometry().kernel_range(); + range.start += t8140::dynamic::KERNEL_OFFSET; + let high = low.alias_in( + alloc.uat.kernel_vm(), + range, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + )?; + fence(Ordering::SeqCst); + Ok(Self { + backing: Backing::Compute { high, low }, + id, + timestamp: KickTimestamp::FIRST, + previous: None, + }) + } + + pub(crate) fn render( + alloc: &Allocator<'_>, + id: Id, + client: &mmu::Vm, + ) -> Result<(Self, mmu::KernelMapping)> { + if id.engine == DataMaster::Compute { + return Err(EINVAL); + } + let high = alloc.kernel( + KICK_RING_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let mut range = t8140::dynamic::LOWER; + let (low, client) = loop { + let low = high.alias_in( + alloc.uat.kernel_lower_vm(), + range.clone(), + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + )?; + let end = low + .iova() + .checked_add(KICK_RING_SIZE as u64) + .ok_or(EOVERFLOW)?; + match high.map_alias(client, low.iova(), mmu::PROT_GPU_SHARED_RW) { + Ok(client) => break (low, client), + Err(error) if error == ENOSPC && end < range.end => { + drop(low); + range.start = end; + } + Err(error) => return Err(error), + } + }; + Ok(( + Self { + backing: Backing::Render { low, high }, + id, + timestamp: KickTimestamp::FIRST, + previous: None, + }, + client, + )) + } + + pub(crate) fn id(&self) -> Id { + self.id + } + pub(crate) fn low_va(&self) -> u64 { + match &self.backing { + Backing::Compute { low, .. } => low.gpu_va(), + Backing::Render { low, .. } => low.iova(), + } + } + pub(crate) fn firmware_va(&self) -> u64 { + match &self.backing { + Backing::Compute { high, .. } => high.iova(), + Backing::Render { high, .. } => high.gpu_va(), + } + } + fn object(&mut self) -> &mut KernelObject { + match &mut self.backing { + Backing::Compute { low, .. } => low, + Backing::Render { high, .. } => high, + } + } + pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { + let address = self.low_va(); + self.object() + .map_alias(vm, address, mmu::PROT_GPU_SHARED_RW) + } + + /// Reuse a permanent client view only when every page names this ring. + pub(crate) fn prepare_client_alias(&self, vm: &mmu::Vm) -> Result> { + let object = match &self.backing { + Backing::Compute { low, .. } => low, + Backing::Render { high, .. } => high, + }; + object.prepare_alias(vm, self.low_va(), mmu::PROT_GPU_SHARED_RW, true) + } + + /// Both aliases are owned mappings of the same full ring. Publish their + /// page-table updates to each live translation regime before installation. + pub(crate) fn prepare_compute_install(&self) -> Result { + if self.id.engine != DataMaster::Compute { + return Err(EINVAL); + } + fence(Ordering::SeqCst); + for asid in 0..4 { + mem::tlbi_range_or_asid(asid, self.low_va() as usize, KICK_RING_SIZE); + mem::tlbi_range_or_asid(asid, self.firmware_va() as usize, KICK_RING_SIZE); + } + mem::sync(); + Ok(()) + } + + /// Detaches the implicit dependency after firmware recovery, retaining + /// the monotonically advancing 40-bit producer. + pub(crate) fn clear_parent_after_recovery(&mut self) { + self.previous = None; + } + + pub(crate) fn timestamp(&self) -> KickTimestamp { + self.timestamp + } + pub(crate) fn parent(&self) -> KickTimestamp { + self.previous.unwrap_or_else(|| match self.id.engine { + DataMaster::Compute => KickTimestamp::ZERO, + _ => self.timestamp.prev(), + }) + } + pub(crate) fn prepare_entry(&mut self, args: &KickArgs<'_>) -> Result> { + if args.qid != self.id.qid + || args.timestamp != self.timestamp + || args.parent != self.parent() + { + return Err(EINVAL); + } + let entry = KickEntry::new(args)?; + let destination = self + .object() + .pointer(args.timestamp.entry_offset(), size_of::())? + .cast(); + let timestamp = self.timestamp; + Ok(PreparedKick { + queue: self, + timestamp, + entry, + destination, + }) + } +} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs new file mode 100644 index 0000000000000..66ce58cd0cc7c --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -0,0 +1,1129 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G17 GPU support. +//! +//! G17 GPUs run their firmware on two coprocessor instances, the primary and the secondary +//! [`Role`]. Each has its own CPU control registers, RTKit mailbox, crash log buffer and kernel +//! page table root, and both share the UAT, the TTBAT and the firmware data structures that the +//! driver allocates at boot (see [`initdata`]). +//! +//! The firmware notifies the host of new records in its event and trace rings with a message on +//! either coprocessor; the event worker drains them. +//! +//! # Locking +//! +//! `Shared::state` is the device lock. It serializes all firmware-visible publication and all +//! firmware-shared state, including the bounded waits for the firmware that publication needs. +//! Completions are signalled after dropping it. Lock order: `Shared::state` -> UAT shared state +//! -> VM exec lock -> GEM reservation. `RoleState::ack` nests inside nothing. + +mod buffer; +mod channel; +mod command; +pub(crate) use command::Attachments; +mod completion; +pub(crate) mod context; +mod dependency; +mod event; +mod fence; +mod freelist; +pub(crate) mod fw; +mod ids; +pub(crate) mod hazard; +mod initdata; +mod job; +mod kick; +mod object; +mod power; +mod preparation; +mod qos; +mod queue; +mod recovery; +mod runtime; +pub(crate) mod status; +mod teardown; +mod validation; + +use core::any::Any; +use core::ops::Range; +use core::sync::atomic::{ + AtomicBool, + Ordering, // +}; + +use kernel::{ + c_str, + device::Core, + drm::gem::shmem, + io::{ + IoBase, + IoSysMap, + Region, // + }, + impl_has_delayed_work, impl_has_work, + new_condvar, new_delayed_work, new_mutex, new_work, platform, + prelude::*, + soc::apple::rtkit, + sync::{ + Arc, + CondVar, + Mutex, // + }, + time::{ + delay::fsleep, + msecs_to_jiffies, + Delta, // + }, + types::{ForeignOwnable, Opaque}, + uapi, + workqueue::{ + self, + DelayedWork, + Work, + WorkItem, // + }, +}; + +use crate::driver::{ + AsahiDevRef, + AsahiDrmDevice, // +}; +use crate::gem::AsahiObject; +use crate::{ + alloc, + gem, + gpu, + hw, + mmu, + regs, // +}; + +use object::{ + CpuMap, + KernelObject, + Placement, // +}; + +/// Expected identification of a G17 GPU. +pub(crate) struct Identity { + /// GPU family field of the ID register. + pub(crate) family: u8, + /// GPU variant field of the ID register. + pub(crate) variant: u8, + /// Number of GPU dies. + pub(crate) num_dies: u32, + /// GPU variant reported to userspace. + pub(crate) gpu_variant: hw::GpuVariant, + /// Shader core generation reported to userspace. + pub(crate) usc_generation: u32, + /// Hardware abstraction generation reported to userspace. + pub(crate) hal_generation: u32, +} + +/// A buffer that the firmware expects at a fixed address of the kernel lower root. +pub(crate) struct FixedBuffer { + /// GPU address. + pub(crate) va: u64, + /// Size in bytes. + pub(crate) size: usize, +} + +/// Static configuration of a SoC with a G17 GPU. +pub(crate) struct Config { + /// SoC identifier. + pub(crate) chip_id: u32, + /// Expected GPU identification. + pub(crate) identity: Identity, + /// Input address bits translated by each UAT root. + pub(crate) uat_ias: u32, + /// Output (physical) address bits of the UAT. + pub(crate) uat_oas: u32, + /// Offset of the secondary coprocessor's kernel root in the `pagetables` region. + pub(crate) secondary_root_offset: u64, + /// Crash log buffer of each [`Role`]. + pub(crate) crash_buffers: [FixedBuffer; 2], + /// Parameter buffer page metrics shared with the firmware. + pub(crate) pm_metrics: FixedBuffer, +} + +/// A firmware coprocessor instance. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum Role { + /// The coprocessor that owns the device-control ring. + Primary = 0, + /// The coprocessor that only runs work the primary one hands to it. + Secondary = 1, +} + +/// RTKit endpoint of firmware messages: initialization, its acknowledgement and event +/// notifications. +const EP_FIRMWARE: u8 = 0x20; +/// RTKit endpoint of host notifications to the firmware. +const EP_DOORBELL: u8 = 0x21; + +/// Bit position of the type field of firmware messages. +const MSG_TYPE_SHIFT: u32 = 48; +/// Host message handing a coprocessor the address of its initialization root. +const MSG_INITDATA: u64 = 0x81 << MSG_TYPE_SHIFT; +/// Address bits of an initialization root that [`MSG_INITDATA`] carries. +const MSG_INITDATA_ADDR_MASK: u64 = (1 << 44) - 1; +/// Firmware message acknowledging [`MSG_INITDATA`]. +const MSG_INITDATA_ACK: u64 = 0x09; +/// Firmware message announcing new records in the event and trace rings. +const MSG_EVENTS: u64 = 0x42 << MSG_TYPE_SHIFT; +/// Primary processor notification for newly published device-control records. +const MSG_CONTROL_NOTIFY: u64 = (0x84 << MSG_TYPE_SHIFT) | 0x11; +/// Host message, sent to both coprocessors once both are initialized, ending the power-on +/// transition. The firmware then retires the opening device-control records. +const MSG_POWER_ON_DONE: u64 = 0x89 << MSG_TYPE_SHIFT; + +/// Time the firmware takes at most to acknowledge its initialization. +const INITDATA_ACK_TIMEOUT_MS: u32 = 2000; +/// Bounded mailbox-provider readiness retries, before either coprocessor starts. +const RECEIVER_ATTEMPTS: usize = 3; +const RECEIVER_RETRY_MS: i64 = 20; +/// The firmware does not notify the host when it retires device-control records, so their +/// counters are polled at this interval, at most [`OPENING_RETIRE_POLLS`] times. +const OPENING_RETIRE_POLL_MS: i64 = 1; +const OPENING_RETIRE_POLLS: usize = 100; + +/// Address bits of a buffer the coprocessor places itself; the coprocessor sets higher bits in +/// its buffer requests. +const RTKIT_BUFFER_ADDR_MASK: u64 = (1 << 40) - 1; +/// Initial contents of crash log buffers, so that a crash log shows what the firmware wrote. +const CRASH_BUFFER_FILL: [u8; 2] = [0xa5, 0x5a]; + +/// A crash log buffer handed to RTKit. +pub(crate) struct CrashLog { + iova: usize, + map: shmem::VMapOwned, +} + +impl rtkit::Buffer for CrashLog { + fn iova(&self) -> Result { + Ok(self.iova) + } + + fn buf(&mut self) -> Result> { + Ok(IoSysMap::from((&self.map).as_view())) + } +} + +/// Callback state; initialization ACKs can arrive in hard interrupt context. +#[pin_data] +struct RoleState { + role: Role, + shared: Arc, + crash_log: FixedBuffer, + crash_map: shmem::VMapOwned, + crashed: AtomicBool, + #[pin] + ack: Opaque, +} + +// SAFETY: Completion access uses the kernel completion API; the remaining fields are immutable +// shared references or atomics. The completion is pinned before callbacks can observe it. +unsafe impl Send for RoleState {} +// SAFETY: As above; complete_all is safe concurrently with the single boot waiter. +unsafe impl Sync for RoleState {} + +impl RoleState { + fn signal(&self, crashed: bool) { + if crashed { + self.crashed.store(true, Ordering::Release); + } + // SAFETY: ack is initialized and pinned for the lifetime of the callback state. + unsafe { kernel::bindings::complete_all(self.ack.get()) }; + } + + fn wait_for_ack(&self) -> Result { + // SAFETY: ack is initialized and remains pinned while this reference exists. + let completed = unsafe { + kernel::bindings::wait_for_completion_timeout( + self.ack.get(), + msecs_to_jiffies(INITDATA_ACK_TIMEOUT_MS), + ) + }; + if self.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + if completed == 0 { + return Err(ETIMEDOUT); + } + Ok(()) + } +} + +fn is_events_message(endpoint: u8, message: u64) -> bool { + endpoint == EP_FIRMWARE && message == MSG_EVENTS +} + +/// RTKit callbacks of a coprocessor. +struct RoleOps; + +#[vtable] +impl rtkit::Operations for RoleOps { + type Data = Arc; + type Buffer = CrashLog; + + const COPROC_PLACES_BUFFERS: bool = true; + + fn recv_message_early( + data: ::Borrowed<'_>, + ep: u8, + msg: u64, + ) -> bool { + if ep == EP_FIRMWARE && msg >> MSG_TYPE_SHIFT == MSG_INITDATA_ACK { + data.signal(false); + return true; + } + if !is_events_message(ep, msg) { + return false; + } + data.shared.queue_events(); + true + } + + fn crashed(data: ::Borrowed<'_>, crashlog: Option<&[u8]>) { + data.signal(true); + data.shared.lose_device(data.role, crashlog); + } + + fn shmem_map( + data: ::Borrowed<'_>, + iova: usize, + size: usize, + ) -> Result { + if (iova as u64) & RTKIT_BUFFER_ADDR_MASK != data.crash_log.va || size > data.crash_log.size + { + return Err(EINVAL); + } + Ok(CrashLog { + iova, + map: data.crash_map.owner().owned_vmap()?, + }) + } +} + +/// A running firmware coprocessor. +/// +/// Dropping it stops the CPU before RTKit and the crash log buffer go away. +struct Coprocessor { + cpu: regs::CpuControl, + running: bool, + rtkit: rtkit::RtKit, + state: Arc, + _crash_buffer: KernelObject, +} + +impl Coprocessor { + /// Sets up a coprocessor's RTKit and crash log buffer, without starting it. + fn new( + pdev: &platform::Device>, + dev: &AsahiDrmDevice, + cfg: &Config, + uat: &mmu::Uat, + shared: &Arc, + role: Role, + ) -> Result { + let index = role as usize; + let buffer = &cfg.crash_buffers[index]; + let mut crash_buffer = KernelObject::new( + dev, + uat.kernel_lower_vm(), + Placement::At(buffer.va), + buffer.size, + mmu::PROT_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + crash_buffer.bytes_mut().fill(CRASH_BUFFER_FILL[index]); + + let crash_map = crash_buffer.kernel_map()?; + let state = Arc::pin_init( + pin_init!(RoleState { + role, + shared: shared.clone(), + crash_log: FixedBuffer { + va: buffer.va, + size: buffer.size, + }, + crash_map, + crashed: AtomicBool::new(false), + ack <- Opaque::ffi_init(|ptr| { + // SAFETY: ptr is valid uninitialized completion storage that is pinned + // before any RTKit callback can use it. + unsafe { kernel::bindings::init_completion(ptr) }; + }), + }), + GFP_KERNEL, + )?; + + let name = match role { + Role::Primary => c_str!("asc"), + Role::Secondary => c_str!("asc1"), + }; + Ok(Coprocessor { + cpu: regs::CpuControl::new(pdev, name)?, + running: false, + rtkit: Self::receiver(dev, index, &state)?, + state, + _crash_buffer: crash_buffer, + }) + } + + fn receiver( + dev: &AsahiDrmDevice, + index: usize, + state: &Arc, + ) -> Result> { + for attempt in 0..RECEIVER_ATTEMPTS { + match rtkit::RtKit::::new(dev.as_ref().as_ref(), None, index, state.clone()) { + Ok(receiver) => return Ok(receiver), + Err(error) if error == EINVAL || error == ETIMEDOUT => { + if attempt + 1 < RECEIVER_ATTEMPTS { + fsleep(Delta::from_millis(RECEIVER_RETRY_MS)); + } + } + Err(error) => return Err(error), + } + } + Err(EPROBE_DEFER) + } + + /// Starts the CPU and waits for RTKit to come up. + fn boot(&mut self) -> Result { + self.cpu.start()?; + self.running = true; + Pin::new(&mut self.rtkit).boot() + } + + /// Starts the firmware endpoints and hands the firmware its initialization root. + fn init(&mut self, root_va: u64) -> Result { + let mut rtkit = Pin::new(&mut self.rtkit); + rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; + rtkit.as_mut().start_endpoint(EP_DOORBELL)?; + rtkit.send_message( + EP_FIRMWARE, + MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), + )?; + self.state.wait_for_ack() + } + + /// Sends a host notification to the firmware. + fn notify(&mut self, msg: u64) -> Result { + Pin::new(&mut self.rtkit).send_message(EP_DOORBELL, msg) + } + + /// Stops the CPU. The firmware keeps its state, but no longer runs. + fn stop(&mut self) { + if self.running { + self.cpu.stop(); + self.running = false; + } + } +} + +impl Drop for Coprocessor { + fn drop(&mut self) { + self.stop(); + } +} + +/// The GPU-to-fabric bridge, enabled for as long as this exists. +struct Bridge { + regs: regs::G17Resources, + enabled: bool, +} + +impl Bridge { + fn enable(regs: regs::G17Resources) -> Result { + let bridge = Bridge { + regs, + enabled: true, + }; + bridge.regs.enable_g17_bridge()?; + Ok(bridge) + } + + fn disable(&mut self) { + if self.enabled { + self.regs.disable_g17_bridge(); + self.enabled = false; + } + } +} + +impl Drop for Bridge { + fn drop(&mut self) { + self.disable(); + } +} + +/// Everything that exists while the firmware runs. +/// +/// Field order is teardown order: both coprocessors stop (in [`Drop`]) before their RTKit is +/// dropped, the bridge is disabled once they are stopped, and the firmware objects and their +/// mappings go before the UAT. Page tables the firmware or the bootloader own are never freed. +struct Firmware { + secondary: Coprocessor, + primary: Coprocessor, + bridge: Bridge, + rings: channel::Rings, + recovery: recovery::State, + effort: power::Effort, + queues: runtime::Registry, + pool: Arc, + render_ids: Arc, + init: initdata::InitData, + _render_global: mmu::Vm, + _pm_metrics: Arc, + uat: Arc, +} + +impl Drop for Firmware { + fn drop(&mut self) { + self.secondary.stop(); + self.primary.stop(); + self.pool.stop_after_firmware(); + // RTKit destruction releases the mailbox power references. Clear the bridge while + // those references still keep the register block powered. + self.bridge.disable(); + } +} + +impl Firmware { + /// Publishes controls only while the restart handshake is closed. The device mutex remains + /// held through publication and the following mailbox notification. + fn publish_control(&mut self, record: &fw::channels::ControlRecord) -> Result { + use recovery::Memory; + if self.recovery.pending() || self.init.recovery_state()? != 0 { + return Err(EAGAIN); + } + let target = channel::Rings::publish_control(&self.init, record)?; + core::sync::atomic::fence(Ordering::SeqCst); + self.primary.notify(MSG_CONTROL_NOTIFY)?; + Ok(target) + } + + /// Holds idle descent while a fresh physical render pair is registered. + fn acquire_registration_power(&mut self) -> Result { + power::acquire(self) + } + + fn release_registration_power(&mut self) -> Result { + power::release(self) + } + + /// Called at actual submission publication, not while a queued job is merely being built. + fn note_submission(&mut self) -> Result { + self.effort.submit(); + self.apply_effort(power::FULL_EFFORT) + } + + fn apply_effort(&mut self, effort: u32) -> Result { + Self::apply_shared_effort(&self.init, &mut self.effort, effort) + } + + fn apply_shared_effort( + init: &initdata::InitData, + state: &mut power::Effort, + effort: u32, + ) -> Result { + if state.applied() == effort && init.effort()? == effort { + return Ok(()); + } + init.publish_effort(effort)?; + state.did_apply(effort); + Ok(()) + } + + /// Returns the delay in ns for a follow-up idle check. Queue owners report only submitted, + /// unretired work; quarantined work retaining mappings is excluded. + fn idle_check(&mut self, now_ns: u64, in_flight: bool) -> Result> { + match self.effort.idle(now_ns, in_flight) { + power::IdleAction::None => Ok(None), + power::IdleAction::Clear => { + self.apply_effort(0)?; + Ok(None) + } + power::IdleAction::Recheck(delay) => Ok(Some(delay)), + } + } +} + +impl power::Control for Firmware { + fn crashed(&self) -> bool { + self.primary.state.shared.crashed.load(Ordering::Acquire) + } + fn powered(&self) -> bool { + self.bridge.regs.g17_configuration_powered() + } + fn counters(&self) -> Result<[u32; 3]> { + self.init.control_counters() + } + fn set_idle(&mut self, enabled: bool) -> Result { + use fw::channels::{ControlRecord, IdlePolicy, IdlePowerOff}; + self.publish_control(&ControlRecord::IdlePowerOff(IdlePowerOff::new( + if enabled { + IdlePolicy::AllowPowerOff + } else { + IdlePolicy::KeepPowered + }, + ))) + } + fn wait_tick(&self) { + fsleep(Delta::from_millis(1)); + } +} + +/// Event work item ID. +const EVENT_WORK: u64 = 0; +/// Short control/retirement polls do not wait behind the longer idle deadline. +const POLL_WORK: u64 = 1; +const IDLE_WORK: u64 = 2; +const GROW_WORK: u64 = 3; +const RECLAIM_WORK: u64 = 4; + +/// Device state shared with the coprocessor callbacks and the event worker. +#[pin_data] +struct Shared { + dev: AsahiDevRef, + clusters: u32, + descriptor_flags: [u32; 2], + crashed: AtomicBool, + preparations: Arc, + #[pin] + work: Work, + #[pin] + poll: DelayedWork, + #[pin] + idle: DelayedWork, + #[pin] + grow: Work, + #[pin] + reclaim_work: Work, + /// Nests inside the device mutex; detached mappings are dropped after both unlock. + #[pin] + reclaim: Mutex, + /// The device lock. `None` before boot completes and after teardown. + #[pin] + state: Mutex>>, + #[pin] + changed: CondVar, +} + +impl_has_work! { + impl HasWork for Shared { self.work } +} + +impl_has_delayed_work! { + impl HasDelayedWork for Shared { self.poll } +} +impl_has_delayed_work! { + impl HasDelayedWork for Shared { self.idle } +} +impl_has_work! { + impl HasWork for Shared { self.grow } +} +impl_has_work! { + impl HasWork for Shared { self.reclaim_work } +} +impl WorkItem for Shared { + type Pointer = Arc; + fn run(this: Arc) { + if let Err(error) = this.grow_render() { + if this.state.lock().is_some() { + dev_err!( + this.dev.as_ref(), + "Render parameter-buffer growth failed: {:?}\n", + error + ); + this.lose_device_quietly(); + } + } + this.queue_events(); + } +} +impl WorkItem for Shared { + type Pointer = Arc; + fn run(this: Arc) { + this.drain_reclaims(); + this.queue_events(); + } +} +impl WorkItem for Shared { + type Pointer = Arc; + fn run(this: Arc) { + let result = { + let mut state = this.state.lock(); + match (*state).as_deref_mut() { + Some(firmware) if firmware.render_control_backpressured() => { + firmware.note_render_control_backpressure() + } + _ => Ok(()), + } + }; + // Only an elapsed one-millisecond timer consumes the retry budget. + // Concurrent event IRQs can request a pass without aging the control. + if result.is_err() { + this.lose_device_quietly(); + } else { + this.queue_events(); + } + } +} +impl WorkItem for Shared { + type Pointer = Arc; + fn run(this: Arc) { + this.queue_events(); + } +} + +impl WorkItem for Shared { + type Pointer = Arc; + + fn run(this: Arc) { + this.run_runtime_events(); + } +} + +impl Shared { + /// Queues the event worker. Callable from any context. + fn queue_events(self: &Arc) { + // Already queued if this fails, which is just as good. + let _ = workqueue::system_highpri().enqueue::, EVENT_WORK>(self.clone()); + } + + fn queue_grow(self: &Arc) { + let _ = workqueue::system_unbound().enqueue::, GROW_WORK>(self.clone()); + } + + fn queue_reclaims(self: &Arc) { + let _ = workqueue::system_unbound().enqueue::, RECLAIM_WORK>(self.clone()); + } + + fn drain_reclaims(self: &Arc) { + loop { + let next = self.reclaim.lock().take_one(); + let Some(next) = next else { + break; + }; + next.finish(); + } + } + + fn disable_work(self: &Arc, work: &Work) { + // SAFETY: Callers pass this Shared's initialized pinned work field. + // Disabling joins a running callback and prevents subsequent enqueue. + let cancelled = unsafe { kernel::bindings::disable_work_sync(Work::raw_get(work)) }; + if cancelled { + // SAFETY: Cancelling the pending callback leaves its one enqueue- + // transferred Arc to reclaim; the callback can no longer consume it. + drop(unsafe { Arc::from_raw(Arc::as_ptr(self)) }); + } + } + + fn queue_poll(self: &Arc) { + let _ = workqueue::system_highpri() + .enqueue_delayed::, POLL_WORK>(self.clone(), msecs_to_jiffies(1)); + } + + fn queue_idle(self: &Arc, delay_ns: u64) { + // Effort::idle bounds this by the 16 ms idle interval. Round up so a + // timer never changes the firmware effort word before that deadline. + let delay_ms = delay_ns.div_ceil(1_000_000).min(u64::from(u32::MAX)) as u32; + let _ = workqueue::system_highpri() + .enqueue_delayed::, IDLE_WORK>(self.clone(), msecs_to_jiffies(delay_ms)); + } + + fn disable_timer(self: &Arc, timer: &DelayedWork) { + // SAFETY: Every caller passes one of this Shared's initialized pinned + // timer fields. raw_as_work points into its delayed_work; the C call + // disables future enqueues, cancels pending work and joins execution. + let cancelled = unsafe { + let work = Work::raw_get(DelayedWork::raw_as_work(timer)); + let delayed = kernel::container_of!(work, kernel::bindings::delayed_work, work); + kernel::bindings::disable_delayed_work_sync(delayed) + }; + if cancelled { + // SAFETY: The cancelled pending callback owned exactly one Arc + // transferred by enqueue_delayed. It cannot run to reclaim it. + drop(unsafe { Arc::from_raw(Arc::as_ptr(self)) }); + } + } + + /// Marks the device lost without a crash log. + fn lose_device_quietly(self: &Arc) { + self.preparations.fail(); + if !self.crashed.swap(true, Ordering::AcqRel) { + dev_err!(self.dev.as_ref(), "GPU runtime stopped, device lost\n"); + } + self.queue_events(); + } + + /// Marks the device lost after a coprocessor crash and publishes a crash dump. + /// + /// The firmware cannot be restarted: outstanding and future work fails. + fn lose_device(self: &Arc, role: Role, crashlog: Option<&[u8]>) { + self.preparations.fail(); + self.crashed.store(true, Ordering::Release); + dev_err!( + self.dev.as_ref(), + "GPU firmware ({:?}) crashed, device lost\n", + role + ); + + // The GPU dev coredump support this used is gone; the crash log is not published. + let _ = crashlog; + self.queue_events(); + } +} + +/// A G17 GPU. +pub(crate) struct Gpu { + cfg: &'static Config, + id: regs::G17Id, + max_frequency_khz: u32, + geometry: mmu::UatGeometry, + ids: gpu::SequenceIDs, + shared: Arc, +} + +impl Gpu { + /// Boots the firmware of a G17 GPU. + pub(crate) fn new( + pdev: &platform::Device>, + dev: &AsahiDrmDevice, + cfg: &'static Config, + regs: regs::G17Resources, + ) -> Result> { + let id = regs.get_g17_id()?; + if id.family != cfg.identity.family + || id.variant != cfg.identity.variant + || id.num_dies != cfg.identity.num_dies + { + dev_err!( + dev.as_ref(), + "Unexpected GPU family {:#x} variant {:#x} with {} dies\n", + id.family, + id.variant, + id.num_dies + ); + return Err(ENODEV); + } + + let geometry = mmu::UatGeometry { + ias: cfg.uat_ias, + oas: cfg.uat_oas, + }; + let max_frequency_khz = Self::max_frequency_khz(dev)?; + let preparations = preparation::Gate::new()?; + let reclaim = runtime::teardown::ReclaimBatch::new()?; + let shared = Arc::pin_init( + pin_init!(Shared { + dev: dev.into(), + clusters: id.num_clusters, + descriptor_flags: [id.perf_control, id.perf_map[0]], + crashed: AtomicBool::new(false), + preparations, + work <- new_work!("g17::Shared::work"), + poll <- new_delayed_work!("g17::Shared::poll"), + idle <- new_delayed_work!("g17::Shared::idle"), + grow <- new_work!("g17::Shared::grow"), + reclaim_work <- new_work!("g17::Shared::reclaim_work"), + reclaim <- new_mutex!(reclaim, "g17::Shared::reclaim"), + state <- new_mutex!(None, "g17::Shared::state"), + changed <- new_condvar!("g17::Shared::changed"), + }), + GFP_KERNEL, + )?; + + let gpu = Arc::new( + Gpu { + cfg, + id, + max_frequency_khz, + geometry, + ids: Default::default(), + shared, + }, + GFP_KERNEL, + )?; + let firmware = Self::boot(pdev, dev, cfg, regs, geometry, &gpu.shared)?; + *gpu.shared.state.lock() = Some(firmware); + gpu.shared.queue_events(); + Ok(gpu) + } + + /// Returns the highest advertised operating-point frequency without changing table order. + fn max_frequency_khz(dev: &AsahiDrmDevice) -> Result { + let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; + let opps = node + .parse_phandle(c_str!("operating-points-v2"), 0) + .ok_or(EIO)?; + let mut freq_hz: Option = None; + for opp in opps.children() { + let frequency: u64 = opp.get_property(c_str!("opp-hz"))?; + freq_hz = Some(freq_hz.map_or(frequency, |highest| highest.max(frequency))); + } + Ok((freq_hz.ok_or(EINVAL)? / 1000).try_into()?) + } + + /// Starts both coprocessors and initializes their firmware. + fn boot( + pdev: &platform::Device>, + dev: &AsahiDrmDevice, + cfg: &'static Config, + regs: regs::G17Resources, + geometry: mmu::UatGeometry, + shared: &Arc, + ) -> Result> { + let uat = Arc::new( + mmu::Uat::new( + dev.into(), + geometry, + true, + mmu::UatFirmware::PublishedRoots { + secondary_root_offset: cfg.secondary_root_offset, + }, + )?, + GFP_KERNEL, + )?; + + // The firmware only samples the kernel lower root's top-level entries when it starts, so + // everything it uses there must be mapped first. + let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary)?; + let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary)?; + let pm_metrics = KernelObject::new( + dev, + uat.kernel_lower_vm(), + Placement::At(cfg.pm_metrics.va), + cfg.pm_metrics.size, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let render_global = uat.new_vm(0, geometry.user_range())?; + + let bridge = Bridge::enable(regs)?; + + // Install all session ownership before either CPU starts. Error unwinding then runs + // Firmware::drop, stopping both CPUs before releasing shared objects or translations. + let mut firmware = KBox::new( + Firmware { + secondary, + primary, + bridge, + rings: channel::Rings::new(), + recovery: recovery::State::new(), + effort: power::Effort::new(), + queues: runtime::Registry::new(shared.clusters, shared.descriptor_flags)?, + pool: object::Pool::new()?, + render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, + init: initdata::InitData::new(), + _render_global: render_global, + _pm_metrics: Arc::new(pm_metrics, GFP_KERNEL)?, + uat, + }, + GFP_KERNEL, + )?; + let session = &mut *firmware; + session.primary.boot()?; + session.secondary.boot()?; + let root_entries = session.uat.adopt_firmware_root_entries()?; + session.init.build( + dev, + cfg, + &session.uat, + &session._render_global, + &session._pm_metrics, + )?; + session.uat.confirm_firmware_root_entries(root_entries)?; + session.uat.mirror_secondary_root()?; + session.bridge.regs.enable_g17_gating()?; + session.primary.init(session.init.root_va(Role::Primary)?)?; + session + .secondary + .init(session.init.root_va(Role::Secondary)?)?; + session.primary.notify(MSG_POWER_ON_DONE)?; + session.secondary.notify(MSG_POWER_ON_DONE)?; + Self::wait_for_opening(dev, &session.init, &session.primary, &session.secondary)?; + session + .uat + .activate_render_context(&session._render_global)?; + Ok(firmware) + } + + /// Waits for both coprocessors to retire their opening device-control records. + fn wait_for_opening( + dev: &AsahiDrmDevice, + init: &initdata::InitData, + primary: &Coprocessor, + secondary: &Coprocessor, + ) -> Result { + for _ in 0..OPENING_RETIRE_POLLS { + if primary.state.crashed.load(Ordering::Acquire) + || secondary.state.crashed.load(Ordering::Acquire) + { + return Err(EIO); + } + if init.opening_retired(Role::Primary)? && init.opening_retired(Role::Secondary)? { + return Ok(()); + } + fsleep(Delta::from_millis(OPENING_RETIRE_POLL_MS)); + } + dev_err!( + dev.as_ref(), + "Firmware did not retire its opening records\n" + ); + Err(ETIMEDOUT) + } + + /// Stops the firmware and releases everything it used. + pub(crate) fn shutdown(&self) { + self.shared.preparations.remove(); + self.shared.stop_queues(); + self.shared.disable_timer(&self.shared.poll); + self.shared.disable_timer(&self.shared.idle); + self.shared.disable_work(&self.shared.grow); + self.shared.disable_work(&self.shared.reclaim_work); + self.shared.disable_work(&self.shared.work); + self.shared.drain_reclaims(); + } +} + +impl gpu::Gpu for Gpu { + fn as_any(&self) -> &dyn Any { + self + } + + fn arc_as_any(self: Arc) -> Arc { + self as Arc + } + + fn manager(&self) -> Option<&dyn gpu::GpuManager> { + None + } + + fn init(&self) -> Result { + Ok(()) + } + + fn ids(&self) -> &gpu::SequenceIDs { + &self.ids + } + + fn is_crashed(&self) -> bool { + self.shared.crashed.load(Ordering::Acquire) + } + + fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { + params.gpu_generation = hw::GpuGen::G17 as u32; + params.gpu_variant = self.cfg.identity.gpu_variant as u32; + params.gpu_revision = self.id.gpu_rev as u32; + params.chip_id = self.cfg.chip_id; + params.num_dies = self.id.num_dies; + params.num_clusters_total = self.id.num_clusters; + params.num_cores_per_cluster = self.id.num_cores; + params.max_frequency_khz = self.max_frequency_khz; + params.usc_generation = self.cfg.identity.usc_generation; + params.gpu_hal_generation = self.cfg.identity.hal_generation; + + for (i, mask) in self.id.core_masks.iter().enumerate() { + *(params.core_masks.get_mut(i).ok_or(EIO)?) = (*mask).into(); + } + + Ok(()) + } + + fn base_clock_hz(&self) -> u32 { + let frequency: u64; + // SAFETY: CNTFRQ_EL0 is a read-only register containing the architectural counter rate. + unsafe { core::arch::asm!("mrs {frequency}, CNTFRQ_EL0", frequency = out(reg) frequency) }; + frequency as u32 + } + + fn supports_vm_status(&self) -> bool { + true + } + + fn uat_geometry(&self) -> mmu::UatGeometry { + self.geometry + } + + fn new_vm(&self, kernel_range: Range) -> Result { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_mut().ok_or(ENODEV)?; + let firmware = &mut **firmware; + let vm = firmware + .uat + .new_vm(self.ids.vm.next(), kernel_range)? + .with_status()?; + let mappings = firmware.init.user_aliases(&vm, &firmware._pm_metrics)?; + let ranges = + hw::t8140::resources::client_reserved_ranges(self.cfg.pm_metrics.va).ok_or(EINVAL)?; + let mut reserved = KVec::with_capacity(ranges.len(), GFP_KERNEL)?; + for range in ranges { + reserved.push(range, GFP_KERNEL)?; + } + vm.install_driver_mappings(mappings, reserved)?; + Ok(vm) + } + + fn new_queue( + &self, + vm: mmu::Vm, + _ualloc: Arc>, + _ualloc_priv: Arc>, + priority: u32, + usc_exec_base: u64, + ) -> Result> { + let owner = self.ids.queue.next(); + let frontend = + queue::Frontend::new(priority, usc_exec_base, self.geometry.user_usable_range())?; + let (backend, slot) = loop { + match runtime::Backend::new(&self.shared, &vm, owner) { + Err(error) if error == EAGAIN => continue, + result => break result?, + } + }; + let frontend = match queue::Queue::new( + &self.shared.dev, + backend.context.clone(), + backend.clone(), + frontend, + slot, + ) { + Ok(frontend) => frontend, + Err(error) => { + job::Backend::release_owner(&*backend); + return Err(error); + } + }; + Ok(KBox::new(frontend, GFP_KERNEL)?) + } + + fn map_timestamp_buffer( + &self, + mut bo: gem::ObjectRef, + range: Range, + ) -> Result { + let state = self.shared.state.lock(); + let firmware = (*state).as_ref().ok_or(ENODEV)?; + let mut addresses = firmware.uat.geometry().kernel_range(); + addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; + bo.map_range_into_range( + firmware.uat.kernel_vm(), + range, + addresses, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + false, + ) + } +} + +impl Drop for Gpu { + fn drop(&mut self) { + self.shutdown(); + } +} diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs new file mode 100644 index 0000000000000..46f4e0c56b6ae --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -0,0 +1,579 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Kernel objects shared with the G17 firmware. +//! +//! Each firmware object is backed by its own GEM object, mapped into a kernel address space +//! either at an address the firmware interface fixes or anywhere in a given range, and kept +//! mapped into the kernel CPU address space for its whole lifetime. The host builds an object's +//! contents through [`KernelObject::bytes_mut`] and [`KernelObject::write`] before it publishes +//! its address; after that, words the firmware accesses concurrently are only touched through +//! the single-access helpers. + +use core::ops::Range; +use core::ptr::NonNull; +use core::sync::atomic::{ + AtomicU32, + AtomicU64, // +}; + +use kernel::{ + drm::gem::shmem, + prelude::*, // +}; + +use crate::driver::AsahiDrmDevice; +use crate::{ + gem, + mmu, // +}; + +/// CPU caching of the kernel mapping of a [`KernelObject`]. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum CpuMap { + /// Write-back, for memory the firmware only reads through a cached mapping or that the host + /// only reads. + WriteBack, + /// Write-combined, for memory the firmware reads uncached as soon as the host published it. + WriteCombined, +} + +/// Where a [`KernelObject`] is mapped. +pub(crate) enum Placement { + /// At this address, which the firmware interface fixes. + At(u64), + /// Anywhere in this range, at this alignment, followed by an unmapped guard page. + In(Range, u64), +} + +/// A zero-initialized kernel object mapped into a GPU address space and the kernel. +/// +/// # Invariants +/// +/// `ptr` points to the start of the kernel mapping of `obj`, which is `size` bytes long and stays +/// valid as long as `obj` exists, since `obj` holds an owned kernel mapping. +pub(crate) struct KernelObject { + // Unmap from the GPU before dropping the backing object. + mapping: Option, + obj: gem::ObjectRef, + ptr: NonNull, + size: usize, +} + +// SAFETY: The object only refers to kernel memory; concurrent access goes through `&self` +// methods that use atomic or volatile accesses. +unsafe impl Send for KernelObject {} +// SAFETY: See `Send`. +unsafe impl Sync for KernelObject {} + +impl KernelObject { + /// Allocates a zeroed object of `size` bytes and maps it into `vm`. + pub(crate) fn new( + dev: &AsahiDrmDevice, + vm: &mmu::Vm, + placement: Placement, + size: usize, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { + let mut this = Self::backing(dev, size, cpu)?; + let mapping = match placement { + Placement::At(addr) => this.obj.map_at(vm, addr, prot, false)?, + Placement::In(range, align) => this.obj.map_into_range(vm, range, align, prot, true)?, + }; + this.mapping = Some(mapping); + Ok(this) + } + + /// Allocates CPU-accessible backing whose GPU mappings will be installed later. + pub(crate) fn backing(dev: &AsahiDrmDevice, size: usize, cpu: CpuMap) -> Result { + let mut obj = match cpu { + CpuMap::WriteBack => gem::new_kernel_object(dev, size)?, + CpuMap::WriteCombined => gem::new_kernel_object_wc(dev, size)?, + }; + let vmap = obj.vmap()?; + let ptr = NonNull::new(vmap.addr.cast::()).ok_or(ENOMEM)?; + // SAFETY: The mapping covers the whole object. + unsafe { ptr.as_ptr().write_bytes(0, kernel::drm::gem::BaseObject::size(vmap.owner())) }; + + // INVARIANT: `vmap()` created the owned kernel mapping of `obj` that `ptr` points into, + // and `obj` keeps it until it is dropped. + Ok(Self { + mapping: None, + obj, + ptr, + size, + }) + } + + /// Maps this object into another address space at `addr`, as an alias of the same memory. + pub(crate) fn map_alias( + &self, + vm: &mmu::Vm, + addr: u64, + prot: mmu::Prot, + ) -> Result { + gem::ObjectRef::new(self.obj.gem.clone()).map_at(vm, addr, prot, false) + } + + /// Reuses only a complete alias of this exact retained backing. Permanent + /// queue aliases stay owned by the client VM after a physical pair moves; + /// returning to that VM must not replace or duplicate those mappings. + pub(crate) fn prepare_alias( + &self, + vm: &mmu::Vm, + address: u64, + prot: mmu::Prot, + requires_write: bool, + ) -> Result> { + if self.size == 0 || self.size % mmu::UAT_PGSZ != 0 || address & mmu::UAT_PGMSK as u64 != 0 + { + return Err(EINVAL); + } + address.checked_add(self.size as u64).ok_or(EOVERFLOW)?; + let source = self.mapping.as_ref().ok_or(EINVAL)?; + let mut existing = 0; + for offset in (0..self.size).step_by(mmu::UAT_PGSZ) { + let client = address + offset as u64; + if vm.covers_range(client, mmu::UAT_PGSZ as u64, true, requires_write) { + if vm.translate_iova(client)? != source.translate_offset(offset)? { + return Err(EFAULT); + } + existing += 1; + } + } + if existing != 0 { + if existing != self.size / mmu::UAT_PGSZ { + return Err(EFAULT); + } + return Ok(None); + } + self.map_alias(vm, address, prot).map(Some) + } + + /// Maps this backing into a dynamically placed kernel alias. + pub(crate) fn alias_in( + &self, + vm: &mmu::Vm, + range: Range, + align: u64, + prot: mmu::Prot, + ) -> Result { + gem::ObjectRef::new(self.obj.gem.clone()).map_into_range(vm, range, align, prot, false) + } + + /// Maps an exact backing subrange, retaining an explicit unmapped guard. + pub(crate) fn map_range( + &self, + vm: &mmu::Vm, + source: Range, + range: Range, + align: u64, + prot: mmu::Prot, + guard_size: usize, + ) -> Result { + vm.map_in_range_with_guard_size(&self.obj.gem, source, align, range, prot, guard_size) + } + + /// Initializes a typed view before publishing the object to the firmware. + /// + /// The view is cleared first, making every `Zeroable` representation valid. + pub(crate) fn initialize( + &mut self, + offset: usize, + init: impl FnOnce(&mut T) -> Result, + ) -> Result { + let ptr = self.aligned::(offset)?; + // SAFETY: The checked view lies within the exclusively borrowed object. Zeroable makes + // the cleared bytes a valid T, and the closure cannot retain the reference. + unsafe { + ptr.write_bytes(0, 1); + init(&mut *ptr) + } + } + + /// Returns a reference-counted kernel mapping of the object. + pub(crate) fn kernel_map(&self) -> Result> { + self.obj.gem.owned_vmap() + } + + /// Transfers an unpublished object's GPU mapping to a longer-lived owner. + /// The CPU view remains valid; gpu_va returns zero after this transfer. + pub(crate) fn take_mapping(&mut self) -> Result { + self.mapping.take().ok_or(EINVAL) + } + + /// Returns the GPU address of the object. + pub(crate) fn gpu_va(&self) -> u64 { + self.mapping.as_ref().map_or(0, |mapping| mapping.iova()) + } + + /// Returns the size of the object in bytes. + pub(crate) fn size(&self) -> usize { + self.size + } + + /// Returns the contents of the object. + /// + /// Only for building an object the firmware does not access concurrently, before its address + /// is published or while the firmware is known not to use it. + pub(crate) fn bytes_mut(&mut self) -> &mut [u8] { + // SAFETY: Per the type invariant `ptr` is valid for `size` bytes, and the exclusive borrow + // excludes other host accesses. + unsafe { core::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.size) } + } + + /// Writes `value` at `offset`, with the same restrictions as [`Self::bytes_mut`]. + pub(crate) fn write(&mut self, offset: usize, value: T) -> Result { + let end = offset + .checked_add(core::mem::size_of::()) + .ok_or(EINVAL)?; + if end > self.size { + return Err(EINVAL); + } + // SAFETY: The range was checked to be within the object, and the exclusive borrow excludes + // other host accesses. The destination may be unaligned. + unsafe { + self.ptr + .as_ptr() + .add(offset) + .cast::() + .write_unaligned(value) + }; + Ok(()) + } + + /// Returns a pointer to the naturally aligned `T` at `offset`. + fn aligned(&self, offset: usize) -> Result<*mut T> { + let end = offset + .checked_add(core::mem::size_of::()) + .ok_or(EINVAL)?; + if end > self.size { + return Err(EINVAL); + } + // SAFETY: The checked offset lies within the retained CPU mapping. + let address = unsafe { self.ptr.as_ptr().add(offset) }; + if address as usize % core::mem::align_of::() != 0 { + return Err(EINVAL); + } + // The actual address, not only the offset, satisfies T's alignment. + Ok(address.cast()) + } + + /// Returns a checked raw view; dereferencing it requires external serialization. + pub(super) fn pointer(&self, offset: usize, length: usize) -> Result<*mut u8> { + if offset.checked_add(length).ok_or(EINVAL)? > self.size { + return Err(EINVAL); + } + // SAFETY: The offset was checked against the live object mapping. + Ok(unsafe { self.ptr.as_ptr().add(offset) }) + } + + /// Returns the live 32-bit word at `offset`, which must be naturally aligned. + pub(crate) fn word(&self, offset: usize) -> Result<&AtomicU32> { + // SAFETY: `aligned()` returns a valid, aligned pointer into the object, which lives as long + // as `self`. Other agents only access the word with single 32-bit accesses. + Ok(unsafe { AtomicU32::from_ptr(self.aligned(offset)?) }) + } + + /// Returns the live 64-bit word at `offset`, which must be naturally aligned. + pub(crate) fn dword(&self, offset: usize) -> Result<&AtomicU64> { + // SAFETY: As for `word()`, with 64-bit accesses. + Ok(unsafe { AtomicU64::from_ptr(self.aligned(offset)?) }) + } + + /// Copies `N` 32-bit words that start at `offset` out of the object, one volatile access each. + /// + /// Used for records another agent has published, which it no longer modifies. + pub(crate) fn read_words(&self, offset: usize) -> Result<[u32; N]> { + let base = self.aligned::<[u32; N]>(offset)?.cast::(); + let mut words = [0u32; N]; + for (i, word) in words.iter_mut().enumerate() { + // SAFETY: `aligned()` checked that all `N` words are within the object and aligned. + *word = unsafe { base.add(i).read_volatile() }; + } + Ok(words) + } +} + +/// Allocation domains shared by the G17 resource builders. Dynamic objects +/// retain a guard page; aliases keep the existing backing and its exact VA. +pub(crate) struct Allocator<'a> { + pub(crate) dev: &'a AsahiDrmDevice, + pub(crate) uat: &'a mmu::Uat, +} + +impl Allocator<'_> { + /// Allocates in the dynamic part of the firmware's upper address space. + pub(crate) fn kernel( + &self, + size: usize, + align: u64, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { + let mut range = self.uat.geometry().kernel_range(); + range.start += crate::hw::t8140::dynamic::KERNEL_OFFSET; + KernelObject::new( + self.dev, + self.uat.kernel_vm(), + Placement::In(range, align), + size, + prot, + cpu, + ) + } + + /// Allocates a canonical low mapping, which can be aliased at the same + /// address into an execution context without overlapping client objects. + pub(crate) fn lower( + &self, + size: usize, + align: u64, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { + KernelObject::new( + self.dev, + self.uat.kernel_lower_vm(), + Placement::In(crate::hw::t8140::dynamic::LOWER, align), + size, + prot, + cpu, + ) + } +} + +/// Families whose upper mappings remain cached and installed until firmware +/// stops, even after the logical owner releases the object. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Family { + Descriptor = 0, + WorkState = 1, + QueueState = 2, + QueueConfig = 3, +} + +const POOL_FAMILIES: usize = 4; +const POOL_SLOTS: usize = 64; +const POOL_BYTES: usize = 128 * 1024 * 1024; + +struct Retained { + object: KernelObject, + prefix: usize, + cpu: CpuMap, + poisoned: bool, +} + +struct PoolInner { + returned: KVec>, + reserved: [usize; POOL_FAMILIES], + bytes: usize, + stopped: bool, +} + +/// Per-device cache-safe object recycling. Every converted mapping reserves +/// one return slot before its page-table attributes change. Active leases and +/// poisoned mappings count against the same capacity as reusable objects. +#[pin_data] +pub(crate) struct Pool { + #[pin] + inner: kernel::sync::Mutex, +} + +struct CacheLease { + pool: kernel::sync::Arc, + family: Family, + prefix: usize, + cpu: CpuMap, + poisoned: bool, +} + +/// A firmware object whose cached mapping is returned to its device pool +/// instead of being unmapped while either firmware processor can reference it. +pub(crate) struct PooledObject { + object: core::mem::ManuallyDrop, + lease: Option, +} + +impl core::ops::Deref for PooledObject { + type Target = KernelObject; + fn deref(&self) -> &Self::Target { + &self.object + } +} +impl core::ops::DerefMut for PooledObject { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.object + } +} +impl Drop for PooledObject { + fn drop(&mut self) { + // SAFETY: This destructor is the only operation that moves `object` + // out; ManuallyDrop prevents a second implicit drop of the field. + let object = unsafe { core::mem::ManuallyDrop::take(&mut self.object) }; + match self.lease.take() { + Some(lease) => lease.pool.return_object(object, &lease), + None => drop(object), + } + } +} + +impl Pool { + pub(crate) fn new() -> Result> { + let mut returned = KVec::with_capacity(POOL_FAMILIES * POOL_SLOTS, GFP_KERNEL)?; + for _ in 0..POOL_FAMILIES * POOL_SLOTS { + returned.push(None, GFP_KERNEL)?; + } + kernel::sync::Arc::pin_init( + pin_init!(Self { + inner <- kernel::new_mutex!(PoolInner { returned, reserved: [0; POOL_FAMILIES], bytes: 0, stopped: false }, "g17::object::Pool"), + }), + GFP_KERNEL, + ) + } + + /// Allocates or reuses a matching object. Capacity pressure leaves a fresh + /// mapping uncached, preserving allocation and submission concurrency. + pub(crate) fn allocate( + self: &kernel::sync::Arc, + alloc: &Allocator<'_>, + family: Family, + size: usize, + prefix: usize, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { + let family_index = family as usize; + let begin = family_index * POOL_SLOTS; + let reused = { + let mut inner = self.inner.lock(); + if inner.stopped { + return Err(ENODEV); + } + let slot = inner.returned[begin..begin + POOL_SLOTS] + .iter_mut() + .find(|entry| { + entry.as_ref().is_some_and(|entry| { + !entry.poisoned + && entry.object.size() == size + && entry.prefix == prefix + && entry.cpu == cpu + }) + }); + slot.and_then(Option::take) + }; + let lease = || CacheLease { + pool: self.clone(), + family, + prefix, + cpu, + poisoned: false, + }; + if let Some(mut entry) = reused { + entry.object.bytes_mut().fill(0); + core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); + return Ok(PooledObject { + object: core::mem::ManuallyDrop::new(entry.object), + lease: Some(lease()), + }); + } + let object = alloc.kernel(size, mmu::UAT_PGSZ as u64, prot, cpu)?; + let reserved = { + let mut inner = self.inner.lock(); + if inner.stopped { + return Err(ENODEV); + } + if inner.reserved[family_index] == POOL_SLOTS + || inner + .bytes + .checked_add(size) + .is_none_or(|bytes| bytes > POOL_BYTES) + { + false + } else { + inner.reserved[family_index] += 1; + inner.bytes += size; + true + } + }; + let mut result = PooledObject { + object: core::mem::ManuallyDrop::new(object), + lease: reserved.then(lease), + }; + if !reserved { + return Ok(result); + } + let (proof, mutated) = match result.mapping.as_ref() { + // SAFETY: The allocator just created the private mapping. No + // pointer or alias has been published; its reserved lease retains + // it until processor stop if any attribute change was attempted. + Some(mapping) => unsafe { mapping.set_fw_cached_prefix(prefix) }, + None => (Err(EINVAL), false), + }; + match proof { + Ok(_) => Ok(result), + Err(error) if mutated => { + if let Some(lease) = &mut result.lease { + lease.poisoned = true; + } + // Drop retains the unproven mapping but never makes it reusable. + Err(error) + } + Err(_) => { + result.lease = None; + let mut inner = self.inner.lock(); + inner.reserved[family_index] -= 1; + inner.bytes -= size; + Ok(result) + } + } + } + + fn return_object(&self, object: KernelObject, lease: &CacheLease) { + let mut inner = self.inner.lock(); + let family = lease.family as usize; + if inner.stopped { + inner.reserved[family] -= 1; + inner.bytes -= object.size(); + drop(inner); + drop(object); + return; + } + let begin = family * POOL_SLOTS; + for slot in &mut inner.returned[begin..begin + POOL_SLOTS] { + if slot.is_none() { + *slot = Some(Retained { + object, + prefix: lease.prefix, + cpu: lease.cpu, + poisoned: lease.poisoned, + }); + return; + } + } + // A private CacheLease reserves capacity before conversion, so a + // missing slot means host accounting corruption. Never unmap a cached + // object the firmware may still hold in that failure case. + pr_err!("G17 cached-object return capacity was lost\n"); + core::mem::forget(object); + } + + /// Releases retained mappings after both firmware processors have stopped. + /// Active objects returned later observe the same stop witness. + pub(crate) fn stop_after_firmware(&self) { + self.inner.lock().stopped = true; + for index in 0..POOL_FAMILIES * POOL_SLOTS { + let entry = { + let mut inner = self.inner.lock(); + let entry = inner.returned[index].take(); + if let Some(entry) = &entry { + inner.reserved[index / POOL_SLOTS] -= 1; + inner.bytes -= entry.object.size(); + } + entry + }; + // Mapping teardown must not run while the pool mutex is held. + drop(entry); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs new file mode 100644 index 0000000000000..8806f438234e9 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Work-based firmware performance effort. Idle power gating is managed separately by the +//! device-control ring; this input never keeps an otherwise idle GPU powered. + +pub(super) const FULL_EFFORT: u32 = 0x10000; +const IDLE_HOLD_NS: u64 = 16_000_000; + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum IdleAction { + None, + Clear, + Recheck(u64), +} + +pub(super) struct Effort { + streaming: bool, + idle_since: Option, + applied: u32, +} + +impl Effort { + pub(super) const fn new() -> Self { + Self { + streaming: false, + idle_since: None, + applied: 0, + } + } + + pub(super) fn submit(&mut self) { + self.streaming = true; + self.idle_since = None; + } + + pub(super) fn applied(&self) -> u32 { + self.applied + } + pub(super) fn did_apply(&mut self, value: u32) { + self.applied = value; + } + + /// Quarantined jobs do not count as work in flight; they retain mappings but cannot justify + /// retaining the effort input. New submissions restart the continuous-idle interval. + pub(super) fn idle(&mut self, now_ns: u64, in_flight: bool) -> IdleAction { + if !self.streaming { + return IdleAction::None; + } + if in_flight { + self.idle_since = None; + return IdleAction::None; + } + let since = *self.idle_since.get_or_insert(now_ns.max(1)); + let idle = now_ns.saturating_sub(since); + if idle >= IDLE_HOLD_NS { + self.streaming = false; + self.idle_since = None; + IdleAction::Clear + } else { + IdleAction::Recheck(IDLE_HOLD_NS - idle) + } + } +} + +/// Single-producer control path under the device mutex. Event draining must not publish another +/// control between `set_idle` and the exact three-counter retirement witness. +pub(super) trait Control { + fn crashed(&self) -> bool; + fn powered(&self) -> bool; + fn counters(&self) -> kernel::error::Result<[u32; 3]>; + fn set_idle(&mut self, enabled: bool) -> kernel::error::Result; + fn wait_tick(&self); +} + +fn wait(control: &C, target: u32, require_powered: bool) -> kernel::error::Result { + use kernel::prelude::*; + const RETIRE_POLLS: usize = 20; + const POWER_POLLS: usize = 250; + let mut retired = false; + for elapsed in 0..RETIRE_POLLS + POWER_POLLS { + if control.crashed() { + return Err(EIO); + } + if !retired { + retired = control.counters()? == [target; 3]; + if !retired && elapsed + 1 >= RETIRE_POLLS { + break; + } + } + if retired && (!require_powered || control.powered()) { + return Ok(()); + } + control.wait_tick(); + } + Err(ETIMEDOUT) +} + +/// Disable idle descent through fresh pair registration, including a post-wake reassertion. +pub(super) fn acquire(control: &mut C) -> kernel::error::Result { + use kernel::prelude::*; + let result = (|| { + for _ in 0..3 { + let powered = control.powered(); + let target = control.set_idle(false)?; + wait(control, target, true)?; + if !powered { + let target = control.set_idle(false)?; + wait(control, target, true)?; + } + if control.powered() { + return Ok(()); + } + } + Err(EAGAIN) + })(); + if result.is_err() { + let _ = release(control); + } + result +} + +pub(super) fn release(control: &mut C) -> kernel::error::Result { + let target = control.set_idle(true)?; + wait(control, target, false) +} diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs new file mode 100644 index 0000000000000..8563beda979e6 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Admission and lifetime of off-lock graph construction. +//! +//! Entering and leaving an open epoch use only atomics. Recovery closes the epoch while holding +//! the device mutex, then drops that mutex before waiting for builders. A builder's lease is +//! stored after its tentative mappings so those mappings are destroyed before the join completes. + +use core::sync::atomic::{AtomicU64, Ordering}; +use kernel::{ + new_condvar, new_mutex, + prelude::*, + sync::{Arc, CondVar, Mutex}, +}; + +const COUNT_BITS: u32 = 20; +const COUNT_MASK: u64 = (1 << COUNT_BITS) - 1; +const CLOSED: u64 = 1 << COUNT_BITS; +const REMOVED: u64 = CLOSED << 1; +const FAILED: u64 = CLOSED << 2; +const EPOCH: u64 = CLOSED << 3; + +#[pin_data] +pub(super) struct Gate { + state: AtomicU64, + #[pin] + wait: Mutex<()>, + #[pin] + changed: CondVar, +} + +impl Gate { + pub(super) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + state: AtomicU64::new(EPOCH), + wait <- new_mutex!((), "g17::Gate::wait"), + changed <- new_condvar!("g17::Gate::changed"), + }), + GFP_KERNEL, + ) + } + + pub(super) fn try_enter(self: &Arc) -> Result { + let mut previous = self.state.load(Ordering::Acquire); + loop { + if previous & (REMOVED | FAILED) != 0 { + return Err(ENODEV); + } + if previous & CLOSED != 0 { + return Err(EAGAIN); + } + if previous & COUNT_MASK == COUNT_MASK { + return Err(ENOSPC); + } + match self.state.compare_exchange_weak( + previous, + previous + 1, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => { + return Ok(Lease { + gate: self.clone(), + epoch: previous & !COUNT_MASK, + }) + } + Err(current) => previous = current, + } + } + } + + /// Wait only for a temporary recovery closure. Called with no device lock or existing lease. + pub(super) fn enter(self: &Arc, interruptible: bool) -> Result { + match self.try_enter() { + Err(error) if error == EAGAIN => {} + result => return result, + } + let mut wait = self.wait.lock(); + loop { + match self.try_enter() { + Err(error) if error == EAGAIN => {} + result => return result, + } + if interruptible { + if self.changed.wait_interruptible(&mut wait) { + return Err(ERESTARTSYS); + } + } else { + self.changed.wait(&mut wait); + } + } + } + + fn invalidate(&self, flags: u64) { + let mut previous = self.state.load(Ordering::Acquire); + loop { + let mut next = previous | CLOSED | flags; + if previous & CLOSED == 0 { + next = next + .checked_add(EPOCH) + .unwrap_or(previous | CLOSED | REMOVED | flags); + } + match self.state.compare_exchange_weak( + previous, + next, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return, + Err(current) => previous = current, + } + } + } + + pub(super) fn close(&self) { + self.invalidate(0); + } + + pub(super) fn remove(&self) { + self.invalidate(REMOVED); + let _wait = self.wait.lock(); + self.changed.notify_all(); + } + + pub(super) fn fail(&self) { + self.invalidate(FAILED); + let _wait = self.wait.lock(); + self.changed.notify_all(); + } + + /// Join builders with no device, queue or mapping lock held. + pub(super) fn wait_drained(&self) { + let mut wait = self.wait.lock(); + while self.state.load(Ordering::Acquire) & COUNT_MASK != 0 { + self.changed.wait(&mut wait); + } + } + + /// Reopen only an empty, temporary closure. Failed or removed devices remain closed. + pub(super) fn reopen(&self) -> bool { + let _wait = self.wait.lock(); + let mut previous = self.state.load(Ordering::Acquire); + loop { + if previous & (REMOVED | FAILED | COUNT_MASK) != 0 { + return false; + } + match self.state.compare_exchange_weak( + previous, + previous & !CLOSED, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => { + self.changed.notify_all(); + return true; + } + Err(current) => previous = current, + } + } + } +} + +/// An off-lock builder's admission epoch. Keep after all of its tentative mappings. +pub(super) struct Lease { + gate: Arc, + epoch: u64, +} + +impl Lease { + /// Must be checked under the device mutex immediately before publishing the built graph. + pub(super) fn is_current(&self) -> bool { + self.gate.state.load(Ordering::Acquire) & !COUNT_MASK == self.epoch + } +} + +impl Drop for Lease { + fn drop(&mut self) { + let previous = self.gate.state.fetch_sub(1, Ordering::AcqRel); + if previous & COUNT_MASK == 1 && previous & CLOSED != 0 { + // Pair the final decrement with the joiner's check/sleep lock to avoid a lost wakeup. + let _wait = self.gate.wait.lock(); + self.gate.changed.notify_all(); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/qos.rs b/drivers/gpu/drm/asahi/g17/qos.rs new file mode 100644 index 0000000000000..8a925c23552a7 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/qos.rs @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Shared QoS publication and exact-completion accounting for every engine. +//! +//! The device publication lock serializes this host state. Publishing identity +//! precedes submitted counters; retiring a final owner clears only its two row +//! bytes. Cancellation balances host references without rewinding shared counts. + +use super::{fw::qos as fw, object::KernelObject}; +use core::{ + mem::{offset_of, size_of}, + sync::atomic::{fence, AtomicU32, AtomicU64, AtomicU8, Ordering}, +}; +use kernel::prelude::*; +use crate::hw::t8140::qos as cfg; + +/// Atomic views retain the allocation borrow and never borrow firmware words +/// as ordinary mutable memory. The arrays are disjoint apart from the explicit +/// data-master time/count indexing in `times_and_counts`. +pub(crate) struct View<'a> { + rows: &'a [[AtomicU8; 8]; fw::QUEUES], + queue_submitted: &'a [AtomicU32; fw::QUEUES], + owner_submitted: &'a [AtomicU32; fw::QUEUES], + scheduler: &'a [AtomicU64; fw::QUEUES], + share: &'a [AtomicU32; fw::QUEUES], + intervals: &'a AtomicU64, + times_and_counts: &'a [AtomicU64; 7], +} +impl<'a> View<'a> { + pub(crate) fn new(object: &'a KernelObject) -> Result { + let ptr = object.pointer(0, size_of::())?; + if ptr as usize & 7 != 0 { + return Err(EINVAL); + } + // SAFETY: pointer spans the complete layout, whose asserted offsets + // and size bound every array below. Integer atomics have their integer + // representation and alignment. The retained object borrow keeps every + // shared reference alive; accesses use each field's fixed width. + Ok(unsafe { + Self { + rows: &*ptr.add(offset_of!(fw::Qos, rows)).cast(), + queue_submitted: &*ptr.add(offset_of!(fw::Qos, queue_submitted)).cast(), + owner_submitted: &*ptr.add(offset_of!(fw::Qos, owner_submitted)).cast(), + scheduler: &*ptr.add(offset_of!(fw::Qos, scheduler)).cast(), + share: &*ptr.add(offset_of!(fw::Qos, share)).cast(), + intervals: &*ptr.add(offset_of!(fw::Qos, intervals)).cast(), + times_and_counts: &*ptr.add(offset_of!(fw::Qos, busy_start)).cast(), + } + }) + } + fn row(&self, qid: usize) -> [u8; 8] { + core::array::from_fn(|index| self.rows[qid][index].load(Ordering::Relaxed)) + } +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Owner { + pub(crate) qid: u8, + pub(crate) qos: u8, + pub(crate) data_master: u8, +} +impl Owner { + fn indices(self) -> Result<(usize, usize, usize)> { + let (qid, qos, dm) = ( + usize::from(self.qid), + usize::from(self.qos), + usize::from(self.data_master), + ); + if qid >= fw::QUEUES || qos >= fw::QUEUES || dm >= fw::DATA_MASTERS { + return Err(EINVAL); + } + Ok((qid, qos, dm)) + } +} + +/// Retained until completion, or cancellation before the producer is exposed. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Publication { + pub(crate) record: [u8; 8], + owner: Owner, + dm_submitted: u64, +} + +pub(crate) struct Accounting { + in_flight: [u32; fw::QUEUES], + dm_in_flight: [u64; fw::DATA_MASTERS], + dm_by_qid: [u8; fw::QUEUES], +} +impl Accounting { + pub(crate) const fn new() -> Self { + Self { + in_flight: [0; fw::QUEUES], + dm_in_flight: [0; fw::DATA_MASTERS], + dm_by_qid: [0; fw::QUEUES], + } + } + + /// Called under the device publication lock immediately before exposing + /// the work record. The clock is sampled only when its engine becomes busy. + pub(crate) fn publish( + &mut self, + view: &View<'_>, + owner: Owner, + scheduler: u64, + clock: impl FnOnce() -> (u64, u64), + ) -> Result { + let (qid, qos, dm) = owner.indices()?; + if scheduler == 0 { + return Err(EINVAL); + } + let next_qid = self.in_flight[qid].checked_add(1).ok_or(EOVERFLOW)?; + let next_dm = self.dm_in_flight[dm].checked_add(1).ok_or(EOVERFLOW)?; + let starts_busy = self.dm_in_flight[dm] == 0; + let time = if starts_busy { + let (counter, frequency) = clock(); + time_from_host(counter, frequency)? + } else { + 0 + }; + if self.in_flight[qid] != 0 + && (view.rows[qid][0].load(Ordering::Relaxed) != owner.qos + || self.dm_by_qid[qid] != owner.data_master) + { + return Err(EBUSY); + } + view.rows[qid][0].store(owner.qos, Ordering::Relaxed); + view.rows[qid][1].store(cfg::CLASS, Ordering::Relaxed); + let record = view.row(qid); + view.scheduler[qos].store(scheduler, Ordering::Relaxed); + view.share[qos].store(cfg::SHARE, Ordering::Relaxed); + if self.in_flight[qid] == 0 { + fence(Ordering::SeqCst); + } + view.queue_submitted[qid].store( + view.queue_submitted[qid] + .load(Ordering::Relaxed) + .wrapping_add(1), + Ordering::Relaxed, + ); + view.owner_submitted[qos].store( + view.owner_submitted[qos] + .load(Ordering::Relaxed) + .wrapping_add(1), + Ordering::Relaxed, + ); + if starts_busy { + view.times_and_counts[dm].store(time, Ordering::Relaxed); + } + let submitted = &view.times_and_counts[3 + dm]; + let dm_submitted = submitted.load(Ordering::Relaxed).wrapping_add(1); + submitted.store(dm_submitted, Ordering::Relaxed); + if starts_busy { + view.intervals.store( + view.intervals.load(Ordering::Relaxed).wrapping_add(1), + Ordering::Relaxed, + ); + } + self.in_flight[qid] = next_qid; + self.dm_in_flight[dm] = next_dm; + self.dm_by_qid[qid] = owner.data_master; + fence(Ordering::SeqCst); + Ok(Publication { + record, + owner, + dm_submitted, + }) + } + + /// A possibly exposed producer must retain its reference until completion. + /// A newer submission on the same engine makes an old ticket uncancellable. + pub(crate) fn cancel(&mut self, view: &View<'_>, ticket: Publication) -> Result { + let (qid, _, dm) = ticket.owner.indices()?; + if self.in_flight[qid] == 0 + || self.dm_in_flight[dm] == 0 + || view.row(qid) != ticket.record + || view.times_and_counts[3 + dm].load(Ordering::Relaxed) != ticket.dm_submitted + { + return Err(EIO); + } + self.complete(view, &[ticket.owner]) + } + + /// Preflights the entire set before clearing any final row. QID membership, + /// scheduler mappings and monotonically increasing counts remain intact. + pub(crate) fn complete(&mut self, view: &View<'_>, owners: &[Owner]) -> Result { + let mut completed = [0u64; fw::DATA_MASTERS]; + for (index, owner) in owners.iter().copied().enumerate() { + let (qid, _, dm) = owner.indices()?; + if owners[..index].iter().any(|other| other.qid == owner.qid) { + return Err(EINVAL); + } + if self.in_flight[qid] == 0 + || view.rows[qid][0].load(Ordering::Relaxed) != owner.qos + || self.dm_by_qid[qid] != owner.data_master + { + return Err(EIO); + } + completed[dm] += 1; + } + for (dm, count) in completed.iter().enumerate() { + if *count > self.dm_in_flight[dm] { + return Err(EIO); + } + } + for owner in owners { + let qid = usize::from(owner.qid); + if self.in_flight[qid] == 1 { + view.rows[qid][0].store(0xff, Ordering::Relaxed); + view.rows[qid][1].store(0, Ordering::Relaxed); + } + } + for owner in owners { + self.in_flight[usize::from(owner.qid)] -= 1; + } + for (dm, count) in completed.iter().enumerate() { + self.dm_in_flight[dm] -= count; + } + fence(Ordering::SeqCst); + Ok(()) + } +} + +/// Firmware accounting uses a 24 MHz domain independent of host timer speed. +pub(crate) fn time_from_host(counter: u64, frequency: u64) -> Result { + if frequency == 0 || frequency > u64::from(u32::MAX) { + return Err(EINVAL); + } + let whole = (counter / frequency) + .checked_mul(cfg::CLOCK_HZ) + .ok_or(EOVERFLOW)?; + let fraction = (counter % frequency) * cfg::CLOCK_HZ / frequency; + whole.checked_add(fraction).ok_or(EOVERFLOW) +} + +pub(crate) fn clock() -> (u64, u64) { + let counter: u64; + let frequency: u64; + // SAFETY: Both timer registers are readable at EL1 and have no memory effects. + unsafe { + core::arch::asm!("mrs {counter}, CNTPCT_EL0", "mrs {frequency}, CNTFRQ_EL0", + counter=out(reg)counter, frequency=out(reg)frequency, options(nomem,nostack,preserves_flags)); + } + (counter, frequency) +} diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs new file mode 100644 index 0000000000000..1c0d6822c395f --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -0,0 +1,541 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Logical queue and physical render-pair admission. A retained pair belongs +//! to one logical owner even while idle. Closing that owner makes it reusable +//! only after every active lease retires. Quarantine retains these leases until +//! a firmware drain witness or both processors stopping makes reuse safe. +//! +//! The pool lock protects host accounting only and is never held while building +//! a graph or waiting for GPU work. Capacity waiters sleep on the release event. + +use crate::g17::{ + hazard::{self, Footprint, Plan}, + job::Packet, +}; +use crate::{hw::t8140::queues as cfg, mmu::MAX_EXECUTION_CONTEXTS}; +use core::sync::atomic::{AtomicU32, Ordering}; +use kernel::{ + new_condvar, new_mutex, + prelude::*, + sync::{Arc, CondVar, CondVarTimeoutResult, Mutex}, +}; + +#[derive(Clone, Copy)] +pub(crate) struct Request<'a> { + footprint: &'a Arc, + sequence: u64, + prefix: Option, +} + +impl<'a> Request<'a> { + pub(crate) fn of(packet: &'a Packet) -> Option { + packet.footprint.as_ref().map(|footprint| Self { + footprint, + sequence: packet.order.sequence, + prefix: packet.order.wait_through[0], + }) + } +} + +struct InFlight { + owner: u64, + sequence: u64, + slot: Option, + ticket: u64, + footprint: Arc, +} + +const QUARANTINE_WAIT_MS: u32 = 1000; + +struct State { + active: u64, + refs: [u8; cfg::RENDER_SLOTS], + owners: [u64; cfg::RENDER_SLOTS], + epochs: [u64; cfg::RENDER_SLOTS], + quarantined: u64, + kill_hold: u64, + released: u64, + epoch: u64, + footprints: KVec, + next_ticket: u64, +} +impl State { + const MASK: u64 = (u64::MAX >> (64 - cfg::RENDER_SLOTS)) & !1; + const fn new() -> Self { + Self { + active: 0, + refs: [0; cfg::RENDER_SLOTS], + owners: [0; cfg::RENDER_SLOTS], + epochs: [0; cfg::RENDER_SLOTS], + quarantined: 0, + kill_hold: 0, + released: 0, + epoch: 0, + footprints: KVec::new(), + next_ticket: 1, + } + } + fn exhausted(&self) -> bool { + self.quarantined & Self::MASK == Self::MASK + } + fn refusal(&self, owner: u64) -> Result { + if self.exhausted() { + return Err(EIO); + } + match self.owners.iter().position(|id| *id == owner) { + Some(slot) if self.quarantined & (1u64 << slot) != 0 => Err(EIO), + None if !self.owners[1..].contains(&0) + && self.released + & !self.active + & !self.quarantined + & !self.kill_hold + & Self::MASK + == 0 => + { + Err(ENOSPC) + } + _ => Ok(()), + } + } + fn advance(&mut self, slot: usize) { + self.epoch = self.epoch.wrapping_add(1); + self.epochs[slot] = self.epoch; + } + fn take_idle(&mut self, slot: usize, owner: u64) -> u8 { + self.refs[slot] += 1; + self.active |= 1u64 << slot; + self.released &= !(1u64 << slot); + self.owners[slot] = owner; + self.advance(slot); + slot as u8 + } + fn acquire(&mut self, owner: u64) -> Option { + if owner == 0 { + return None; + } + // Prefer the owner's idle pair. A missing sibling can be claimed + // only while this owner has no live kick, before constructing it. + for slot in 0..cfg::RENDER_SLOTS { + if self.owners[slot] == owner + && self.refs[slot] == 0 + && self.quarantined & (1u64 << slot) == 0 + { + if !(0..cfg::RENDER_SLOTS) + .any(|other| self.owners[other] == owner && self.refs[other] != 0) + { + self.claim_sibling(owner); + } + return Some(self.take_idle(slot, owner)); + } + } + if self.owners.contains(&owner) { + // Consecutive kicks are ordered by firmware dependencies. Stack + // onto the least-loaded retained pair without a retirement wait. + let slot = (0..cfg::RENDER_SLOTS) + .filter(|slot| { + self.owners[*slot] == owner + && self.quarantined & (1u64 << slot) == 0 + && self.active & (1u64 << slot) != 0 + && self.refs[*slot] < cfg::RENDER_DEPTH + }) + .min_by_key(|slot| (self.refs[*slot], u64::MAX - self.epochs[*slot]))?; + self.refs[slot] += 1; + self.advance(slot); + return Some(slot as u8); + } + let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; + let eligible = |slot: usize| free & (1u64 << slot) != 0; + // Reuse a closed, exactly retired owner before consuming a new graph. + let slot = (0..cfg::RENDER_SLOTS) + .filter(|slot| eligible(*slot) && self.released & (1u64 << slot) != 0) + .min_by_key(|slot| self.epochs[*slot]) + .or_else(|| { + (0..cfg::RENDER_SLOTS).find(|slot| eligible(*slot) && self.owners[*slot] == 0) + })?; + let first = self.take_idle(slot, owner); + self.claim_sibling(owner); + Some(first) + } + fn claim_sibling(&mut self, owner: u64) { + if self.owners.iter().filter(|id| **id == owner).count() >= cfg::RENDER_PAIRS_PER_OWNER { + return; + } + let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; + let eligible = |slot: usize| free & (1u64 << slot) != 0; + let sibling = (0..cfg::RENDER_SLOTS) + .filter(|slot| { + eligible(*slot) + && self.owners[*slot] != owner + && self.released & (1u64 << slot) != 0 + }) + .min_by_key(|slot| self.epochs[*slot]) + .or_else(|| { + (0..cfg::RENDER_SLOTS).find(|slot| eligible(*slot) && self.owners[*slot] == 0) + }); + if let Some(slot) = sibling { + self.released &= !(1u64 << slot); + self.owners[slot] = owner; + self.advance(slot); + } + } + fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { + if self.refs.get(usize::from(publishing)) != Some(&1) { + return None; + } + (0..cfg::RENDER_SLOTS) + .find(|slot| { + *slot != usize::from(publishing) + && self.owners[*slot] == owner + && self.refs[*slot] == 0 + && self.quarantined & (1u64 << slot) == 0 + }) + .map(|slot| slot as u8) + } + fn plan(&self, owner: u64, request: Option>, ticket: Option) -> Plan { + let Some(request) = request else { + return Plan::Any; + }; + hazard::plan( + owner, + request.prefix, + ticket, + request.footprint, + self.footprints.iter().map(|entry| hazard::Live { + owner: entry.owner, + sequence: entry.sequence, + slot: entry.slot, + ticket: entry.ticket, + footprint: &entry.footprint, + quarantined: entry + .slot + .is_some_and(|slot| self.quarantined & (1u64 << slot) != 0), + }), + ) + } + + fn pick(&mut self, owner: u64, plan: Plan) -> Option { + match plan { + Plan::Any => self.acquire(owner), + Plan::Wait => None, + Plan::Pair(slot) => { + let index = usize::from(slot); + if self.owners.get(index) != Some(&owner) + || (self.quarantined | self.kill_hold) & (1u64 << slot) != 0 + || self.refs[index] >= cfg::RENDER_DEPTH + { + return None; + } + if self.refs[index] == 0 { + Some(self.take_idle(index, owner)) + } else { + self.refs[index] += 1; + self.advance(index); + Some(slot) + } + } + } + } + + fn record( + &mut self, + owner: u64, + slot: Option, + ticket: u64, + request: Request<'_>, + ) -> Option<(u64, u64)> { + self.footprints + .push( + InFlight { + owner, + sequence: request.sequence, + slot, + ticket, + footprint: request.footprint.clone(), + }, + GFP_KERNEL, + ) + .ok()?; + Some((owner, request.sequence)) + } + + fn forget_waiter(&mut self, ticket: Option) { + if let Some(ticket) = ticket { + self.footprints + .retain(|entry| entry.slot.is_some() || entry.ticket != ticket); + } + } + + fn release(&mut self, slot: u8) { + let slot = usize::from(slot); + if self.refs[slot] != 0 { + self.refs[slot] -= 1; + } + if self.refs[slot] == 0 { + self.active &= !(1u64 << slot); + } + } +} + +#[pin_data] +pub(crate) struct Pool { + #[pin] + state: Mutex, + #[pin] + released: CondVar, + queues: AtomicU32, +} +impl Pool { + pub(crate) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + state <- new_mutex!(State::new(), "G17 render admission"), + released <- new_condvar!("G17 render pair release"), + queues: AtomicU32::new(0), + }), + GFP_KERNEL, + ) + } + pub(crate) fn reserve_queue(self: &Arc) -> Result { + let mut count = self.queues.load(Ordering::Acquire); + loop { + if count >= MAX_EXECUTION_CONTEXTS { + return Err(ENOSPC); + } + match self.queues.compare_exchange( + count, + count + 1, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return Ok(QueueSlot { pool: self.clone() }), + Err(current) => count = current, + } + } + } + pub(crate) fn acquire_render( + self: &Arc, + owner: u64, + request: Option>, + ) -> Result { + if owner == 0 { + return Err(EINVAL); + } + let mut state = self.state.lock(); + let mut conflict_budget = kernel::time::msecs_to_jiffies(hazard::WAIT_MS); + let mut quarantine_budget = kernel::time::msecs_to_jiffies(QUARANTINE_WAIT_MS); + let mut expired = false; + let mut ticket = None; + let result = (|| loop { + if state.exhausted() { + return Err(EIO); + } + // The first retained pair's quarantine is not bypassed through its + // healthy sibling. Reclamation wakes this bounded admission wait. + if state + .owners + .iter() + .position(|id| *id == owner) + .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) + { + if quarantine_budget == 0 { + return Err(EIO); + } + quarantine_budget = match self + .released + .wait_interruptible_timeout(&mut state, quarantine_budget) + { + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + CondVarTimeoutResult::Timeout => 0, + CondVarTimeoutResult::Woken { jiffies } => jiffies, + }; + continue; + } + state.refusal(owner)?; + let plan = if expired { + Plan::Any + } else { + state.plan(owner, request, ticket) + }; + if let Some(slot) = state.pick(owner, plan) { + state.forget_waiter(ticket.take()); + let record = + request.and_then(|request| state.record(owner, Some(slot), 0, request)); + return Ok(RenderLease { + pool: self.clone(), + slot, + record, + }); + } + if plan != Plan::Any { + if ticket.is_none() { + if let Some(request) = request { + let next = state.next_ticket; + state.next_ticket = next.wrapping_add(1); + if state.record(owner, None, next, request).is_some() { + ticket = Some(next); + } + } + } + match self + .released + .wait_interruptible_timeout(&mut state, conflict_budget) + { + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + CondVarTimeoutResult::Timeout => { + conflict_budget = 0; + expired = true; + } + CondVarTimeoutResult::Woken { jiffies } => conflict_budget = jiffies, + } + } else if self.released.wait_interruptible(&mut state) { + return Err(ERESTARTSYS); + } + })(); + state.forget_waiter(ticket); + result + } + + /// Early publication never waits for overlap; the scheduler retries with + /// the same footprint after its dependencies have become ready. + pub(crate) fn try_acquire_render( + self: &Arc, + owner: u64, + request: Option>, + ) -> Result> { + if owner == 0 { + return Ok(None); + } + let mut state = self.state.lock(); + if state.refusal(owner).is_err() || !state.owners.contains(&owner) { + return Ok(None); + } + let plan = state.plan(owner, request, None); + let Some(slot) = state.pick(owner, plan) else { + return Ok(None); + }; + let record = request.and_then(|request| state.record(owner, Some(slot), 0, request)); + Ok(Some(RenderLease { + pool: self.clone(), + slot, + record, + })) + } + pub(crate) fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { + self.state.lock().warmable_sibling(owner, publishing) + } + pub(crate) fn release_owner(&self, owner: u64) { + let mut state = self.state.lock(); + for slot in 0..cfg::RENDER_SLOTS { + if state.owners[slot] == owner { + state.released |= 1u64 << slot; + } + } + drop(state); + self.released.notify_all(); + } + /// Active leases survive this transition. Their later drop is still the + /// event which releases capacity, so a stopped owner is never double-freed. + pub(crate) fn release_retained_owners_after_processor_stop(&self) { + let mut state = self.state.lock(); + let released = state.released & !state.active; + for slot in 1..cfg::RENDER_SLOTS { + if released & (1u64 << slot) != 0 { + state.owners[slot] = 0; + state.epochs[slot] = 0; + } + } + state.released &= !released; + state.quarantined &= !released; + state.kill_hold = 0; + drop(state); + self.released.notify_all(); + } +} + +/// Counted once per userspace logical queue, independently of installed graphs. +pub(crate) struct QueueSlot { + pool: Arc, +} +impl Drop for QueueSlot { + fn drop(&mut self) { + self.pool.queues.fetch_sub(1, Ordering::AcqRel); + } +} + +pub(crate) struct RenderLease { + pool: Arc, + slot: u8, + record: Option<(u64, u64)>, +} +impl RenderLease { + pub(crate) fn slot(&self) -> u8 { + self.slot + } + pub(crate) fn quarantine(&self) { + self.pool.state.lock().quarantined |= 1u64 << self.slot; + self.pool.released.notify_all(); + } + /// The VM pin must be retired first. Keep this lease until after clearing + /// quarantine; its drop is what permits waiting publications to proceed. + pub(crate) fn rehabilitate_after_processor_stop(&self) { + self.pool.state.lock().quarantined &= !(1u64 << self.slot); + } + /// Requires the runtime's closed-recovery, exact pair-drain witness. + pub(crate) fn rehabilitate_after_firmware_drain(&self) { + self.rehabilitate_after_processor_stop(); + } +} +impl Drop for RenderLease { + fn drop(&mut self) { + let mut state = self.pool.state.lock(); + if let Some((owner, sequence)) = self.record { + if let Some(index) = state.footprints.iter().position(|entry| { + entry.slot == Some(self.slot) && entry.owner == owner && entry.sequence == sequence + }) { + state.footprints.swap_remove(index); + } + } + state.release(self.slot); + drop(state); + self.pool.released.notify_all(); + } +} + +impl State { + fn hold_for_exit_kill(&mut self, owner: u64, slots: u64) -> bool { + if owner == 0 || slots & !Self::MASK != 0 { + return false; + } + for slot in 0..cfg::RENDER_SLOTS { + let bit = 1u64 << slot; + if slots & bit == 0 { + continue; + } + if self.owners[slot] != owner + || self.refs[slot] != 0 + || self.active & bit != 0 + || self.released & bit == 0 + { + return false; + } + } + self.kill_hold |= slots; + true + } +} +impl Pool { + /// Hold the departed owner's idle pairs before publishing its exit record. + /// Successful publication retains these bits until both processors stop. + pub(crate) fn hold_render_slots_for_exit_kill(&self, owner: u64, slots: u64) -> bool { + self.state.lock().hold_for_exit_kill(owner, slots) + } + + /// Undo only a hold whose control record was not published. + pub(crate) fn release_exit_kill_hold(&self, slots: u64) { + if slots == 0 { + return; + } + self.state.lock().kill_hold &= !slots; + self.released.notify_all(); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs new file mode 100644 index 0000000000000..f8297f0462ff8 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -0,0 +1,1507 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Retained physical compute queues. Logical ownership and command retirement never free an +//! installed graph; the device retains it until both firmware processors have stopped. + +pub(crate) mod exit; +pub(crate) mod retirement; + +use crate::g17::{ + context::Context, + fw::{compute::COMPUTE_DESCRIPTOR_SIZE, queue::*}, + object::{Allocator, CpuMap, KernelObject}, +}; +use crate::{hw::t8140::dynamic, mmu}; +use core::sync::atomic::{fence, Ordering}; +use kernel::{prelude::*, sync::Arc}; +use retirement::{Cursors, RECORD_SLOTS}; + +const DESCRIPTORS: usize = 256; +const GRAPH_SIZE: usize = 0x8000; +const CONTEXT_SIZE: usize = 0x20000; +const RECORD_STRIDE: usize = 0x1000; +const CONFIG: usize = 0; +const ANNOUNCE: usize = 0x800; +const PREDECESSOR: usize = 0x400; +const POINTERS: usize = 0x100; +const ITEMS: usize = 0x4000; +const STAMP: usize = 0x300; + +#[repr(C)] +struct GraphLayout { + queue: QueueRecord, + unk_c0: [u8; 0x40], + pointers: PointerBlock, + unk_180: [u8; 0x80], + jobs: JobListHead, + unk_210: [u8; 0x3df0], + items: [u64; ITEM_RING_ENTRIES as usize], + unk_6800: [u8; 0x1800], +} +// SAFETY: The layout contains only integer storage and integer atomics, all valid when zeroed. +unsafe impl Zeroable for GraphLayout {} +static_assert!(core::mem::size_of::() == GRAPH_SIZE); +static_assert!(core::mem::offset_of!(GraphLayout, pointers) == POINTERS); +static_assert!(core::mem::offset_of!(GraphLayout, items) == ITEMS); + +/// Aliases are destroyed before the logical root they name. Superseded bindings remain owned +/// until the first exact successful publication under their replacement has retired. +struct Binding { + _kick_alias: mmu::KernelMapping, + _pool_alias: mmu::KernelMapping, + _descriptors: mmu::KernelMapping, + preempt: mmu::KernelMapping, + usage: mmu::KernelMapping, + operand: mmu::KernelMapping, + _queue_context: mmu::KernelMapping, + context: Arc, +} + +/// Firmware-shared storage of one compute QID, separate from its common kick ring. +struct Graph { + descriptor_high: mmu::KernelMapping, + descriptors: KernelObject, + preempt: KernelObject, + usage: KernelObject, + operand: KernelObject, + support: KernelObject, + graph: KernelObject, + context_low: KernelObject, + context_high: KernelObject, + records: KernelObject, + record_ends: [Option; RECORD_SLOTS], + reclaimed: u32, +} + +impl Graph { + fn new(alloc: &Allocator<'_>, context: &Context) -> Result { + let page = mmu::UAT_PGSZ as u64; + let prot = mmu::PROT_GPU_FW_SHARED_RW; + let cpu = CpuMap::WriteCombined; + let descriptors = alloc.lower(DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, page, prot, cpu)?; + let mut high = alloc.uat.geometry().kernel_range(); + high.start += dynamic::KERNEL_OFFSET; + let descriptor_high = descriptors.alias_in(alloc.uat.kernel_vm(), high, page, prot)?; + let preempt = alloc.kernel(0x4000, page, prot, cpu)?; + let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; + let operand = alloc.lower(0x14000, page, prot, cpu)?; + let mut support = alloc.kernel(0xc000, page, prot, cpu)?; + let support_va = support.gpu_va(); + support.write(0, support_va + page)?; + support.write(0x100, support_va + page + 4)?; + support.write(0x110, 0x50u32)?; + support.write(mmu::UAT_PGSZ + 4, 1u32)?; + let mut graph = alloc.kernel(GRAPH_SIZE, page, prot, cpu)?; + let graph_va = graph.gpu_va(); + graph.initialize::(0, |view| { + view.queue = QueueRecord::new(&QueueRecordArgs { + pointers_va: graph_va + POINTERS as u64, + items_va: graph_va + ITEMS as u64, + job_list_va: context.work_head_va(), + scheduler_va: context.scheduler_va(), + owner_pid: context.owner_pid(), + }); + view.pointers = PointerBlock::new(); + view.jobs = JobListHead::new(graph_va + 0x200); + Ok(()) + })?; + let context_low = alloc.lower(CONTEXT_SIZE, page, prot, cpu)?; + let context_high = alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?; + let records = alloc.kernel(RECORD_SLOTS * RECORD_STRIDE, page, prot, cpu)?; + Ok(Self { + descriptor_high, + descriptors, + preempt, + usage, + operand, + support, + graph, + context_low, + context_high, + records, + record_ends: [None; RECORD_SLOTS], + reclaimed: 0, + }) + } + + /// All fallible mapping work finishes before replacing a physical queue's logical owner. + fn bind( + &mut self, + context: Arc, + kick_alias: mmu::KernelMapping, + pool_alias: mmu::KernelMapping, + ) -> Result { + if !context.is_current() { + return Err(EFAULT); + } + let vm = context.vm(); + vm.install_job_context_aliases()?; + let descriptors = + self.descriptors + .map_alias(vm, self.descriptors.gpu_va(), mmu::PROT_GPU_SHARED_RO)?; + let prot = mmu::PROT_GPU_FW_SHARED_RW; + let page = mmu::UAT_PGSZ as u64; + let preempt = self + .preempt + .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; + let usage = self.usage.alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; + let operand = self + .operand + .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; + let queue_context = self + .context_low + .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; + Ok(Binding { + _kick_alias: kick_alias, + _pool_alias: pool_alias, + _descriptors: descriptors, + preempt, + usage, + operand, + _queue_context: queue_context, + context, + }) + } + + /// Caller has excluded publication and proved the retained graph idle. + fn set_owner(&mut self, context: &Context) -> Result { + let pointer = self + .graph + .pointer(0, core::mem::size_of::())? + .cast::(); + // SAFETY: The checked record is in the retained mapping, has byte alignment and consists + // only of integer storage. The device mutex and idle witness exclude simultaneous access. + unsafe { + (*pointer).set_compute_owner( + context.work_head_va(), + context.scheduler_va(), + context.owner_pid(), + ) + }; + Ok(()) + } + + fn cursors(&self) -> Result { + Ok(Cursors { + producer: self.graph.word(POINTERS + 0x40)?.load(Ordering::Relaxed), + consumer: self.graph.word(POINTERS)?.load(Ordering::Relaxed), + count: self.graph.word(POINTERS + 0x60)?.load(Ordering::Relaxed), + }) + } + fn stamp(&self) -> Result { + fence(Ordering::Acquire); + Ok(self.support.word(STAMP)?.load(Ordering::Relaxed)) + } + fn descriptor_vas(&self, slot: u8) -> (u64, u64) { + let offset = u64::from(slot) * COMPUTE_DESCRIPTOR_SIZE as u64; + ( + self.descriptor_high.iova() + offset, + self.descriptors.gpu_va() + offset, + ) + } + fn ensure_record_slot(&self, slot: usize) -> Result { + if let Some(end) = *self.record_ends.get(slot).ok_or(EINVAL)? { + let cursors = self.cursors()?; + if !cursors.consumed(end, cursors.count)? { + return Err(EBUSY); + } + } + Ok(()) + } + + /// Reclaim only consumed item pointers, with individual aligned stores and a release fence. + /// This leaves records still named by firmware untouched across arbitrarily many wraps. + fn recycle(&mut self, cursors: Cursors) -> Result { + if cursors.count != ITEM_RING_ENTRIES { + return Err(EIO); + } + if cursors.consumer >= cursors.count || self.reclaimed >= cursors.count { + return Err(ERANGE); + } + let done = cursors.consumer; + while self.reclaimed != done { + self.graph + .dword(ITEMS + self.reclaimed as usize * 8)? + .store(0, Ordering::Relaxed); + self.reclaimed = (self.reclaimed + 1) % cursors.count; + } + fence(Ordering::SeqCst); + Ok(()) + } +} + +use crate::g17::{ + completion::Deferred, context::WorkStateLease, freelist::ComputePool, job::Packet, kick, qos, + recovery, status::VmStatus, +}; +use retirement::{Batch, Observation, Recordless, Ticket, IN_FLIGHT}; + +struct Fifo { + entries: [Option; IN_FLIGHT], + head: usize, + len: usize, +} +impl Fifo { + fn new() -> Self { + Self { + entries: core::array::from_fn(|_| None), + head: 0, + len: 0, + } + } + fn front(&self) -> Option<&T> { + self.entries[self.head].as_ref() + } + fn back(&self) -> Option<&T> { + if self.len == 0 { + None + } else { + self.entries[(self.head + self.len - 1) % IN_FLIGHT].as_ref() + } + } + fn back_mut(&mut self) -> Option<&mut T> { + if self.len == 0 { + None + } else { + self.entries[(self.head + self.len - 1) % IN_FLIGHT].as_mut() + } + } + fn push(&mut self, value: T) -> core::result::Result<(), T> { + if self.len == IN_FLIGHT { + return Err(value); + } + self.entries[(self.head + self.len) % IN_FLIGHT] = Some(value); + self.len += 1; + Ok(()) + } + fn pop(&mut self) -> Option { + if self.len == 0 { + return None; + } + let value = self.entries[self.head].take(); + self.head = (self.head + 1) % IN_FLIGHT; + self.len -= 1; + value + } + fn pop_back(&mut self) -> Option { + if self.len == 0 { + return None; + } + self.len -= 1; + self.entries[(self.head + self.len) % IN_FLIGHT].take() + } + fn iter(&self) -> impl Iterator { + Iterator::chain( + self.entries[self.head..].iter(), + self.entries[..self.head].iter(), + ) + .take(self.len) + .filter_map(Option::as_ref) + } + fn iter_mut(&mut self) -> impl Iterator { + let (before, after) = self.entries.split_at_mut(self.head); + Iterator::chain(after.iter_mut(), before.iter_mut()) + .take(self.len) + .filter_map(Option::as_mut) + } +} + +struct Active { + packet: Arc, + work: Arc, + ticket: Option, + observed: Option, + failure_deferred: bool, + qos_completed: bool, +} +struct Previous { + binding: Option, + context: Arc, + publication: Option<(u64, u64)>, +} +#[derive(Copy, Clone, PartialEq, Eq)] +struct ConfigIdentity { + scheduler: u64, + qos: u8, + context: u32, +} + +/// A private physical pool whose descriptor row is installed before allocating its kick ring. +/// The QID remains unpublished throughout construction. +pub(crate) struct PoolBacking { + pool: ComputePool, + id: kick::Id, +} + +impl PoolBacking { + pub(crate) fn new(alloc: &Allocator<'_>, id: kick::Id) -> Result { + let pool = ComputePool::new(alloc, id.qid())?; + Ok(Self { pool, id }) + } + + /// Fresh page-pool row data, installed once while the QID is still unpublished. + pub(crate) fn pool_descriptor(&self) -> (u16, u64) { + (self.pool.id(), self.pool.page_list_va()) + } +} + +/// A physical pool and kick ring prepared before the device-global compute scheduler and +/// descriptor graph. No logical VM aliases or firmware publication exist at this stage. +pub(crate) struct Backing { + pool: ComputePool, + kick: kick::Queue, +} + +impl Backing { + pub(crate) fn new(alloc: &Allocator<'_>, backing: PoolBacking) -> Result { + let PoolBacking { pool, id } = backing; + let kick = kick::Queue::compute(alloc, id)?; + Ok(Self { pool, kick }) + } +} + +/// Installed graph and its exact logical owner. A released idle graph may change owners only +/// after a retirement witness. Failed work retains its pointers independently of fence state. +pub(crate) struct Queue { + owner: Option, + binding: Binding, + previous: Option, + kick: kick::Queue, + pool: ComputePool, + graph: Graph, + active: Fifo, + replays: Fifo>, + replay_retirement: bool, + replay_error: Option, + replay_prepared: bool, + exit: Option, + installed: bool, + outer_started: bool, + published_config: Option, + ordinal: u32, + submitted: u32, + last_end: u64, + released: bool, + retirement_ready: bool, + quarantined: bool, + retired_by_teardown: bool, + retire_pending: bool, + awaiting_witness: bool, + spared_quarantine: bool, + spared_deferred: bool, + terminal: Option<(Arc, Error)>, + quarantine_error: Option, + failure_status_pending: bool, + retirement_proved: bool, +} + +impl Queue { + /// The caller reserves a QID first, cancels it on construction failure, and retains every + /// successfully installed queue until processor stop. Shared VM apertures precede this call. + pub(crate) fn new( + alloc: &Allocator<'_>, + backing: Backing, + owner: u64, + context: Arc, + ) -> Result { + let Backing { mut pool, mut kick } = backing; + let mut graph = Graph::new(alloc, &context)?; + let kick_alias = kick.map_client(context.vm())?; + let pool_alias = pool.map_client(context.vm())?; + let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; + context.mark_published(); + Ok(Self { + owner: Some(owner), + binding, + previous: None, + kick, + pool, + graph, + active: Fifo::new(), + replays: Fifo::new(), + replay_retirement: false, + replay_error: None, + replay_prepared: false, + exit: None, + installed: false, + outer_started: false, + published_config: None, + ordinal: 0, + submitted: 0, + last_end: 0, + released: false, + retirement_ready: false, + quarantined: false, + retired_by_teardown: false, + retire_pending: false, + awaiting_witness: false, + spared_quarantine: false, + spared_deferred: false, + terminal: None, + quarantine_error: None, + failure_status_pending: false, + retirement_proved: false, + }) + } + pub(crate) fn qid(&self) -> u8 { + self.kick.id().qid() + } + pub(crate) fn owner(&self) -> Option { + self.owner + } + pub(crate) fn context(&self) -> &Arc { + &self.binding.context + } + /// Quarantined commands still retain the scheduler until their retirement witness. + pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { + !Arc::ptr_eq(self.context(), context) || (self.active.len == 0 && self.replays.len == 0) + } + pub(crate) fn in_flight(&self) -> bool { + self.active.len != 0 && !self.quarantined + } + pub(crate) fn room(&self) -> bool { + !self.quarantined && !self.released && self.replays.len == 0 && self.active.len < IN_FLIGHT + } + pub(crate) fn oldest_spared(&self) -> Option { + self.active + .front() + .map(|a| a.packet.completion.spared() && a.packet.completion.replay_count() == 0) + } + pub(crate) fn visit_owners(&self, visit: &mut dyn FnMut(&Arc)) { + for active in self.active.iter() { + visit(active.packet.completion.status()); + } + } + pub(crate) fn classify(&self, class: &dyn Fn(&Arc) -> recovery::Class) { + for active in self.active.iter() { + active + .packet + .completion + .classify(class(active.packet.completion.status())); + } + } + fn qos_owner(&self) -> qos::Owner { + qos::Owner { + qid: self.qid(), + qos: self.binding.context.qos_id(), + data_master: DataMaster::Compute as u8, + } + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { + self.retired_by_teardown + || self.retire_pending + || masks.is_none_or(|m| m[self.qid() as usize / 64] & (1 << (self.qid() % 64)) != 0) + } + /// Stage every unlatched exact identity before the device's single shared-ring scan. + pub(crate) fn request_observations( + &self, + batch: &mut Batch, + masks: Option<[u64; 2]>, + ) -> Result { + if self.quarantined || !self.selected(masks) { + return Ok(false); + } + let mut requested = false; + for active in self.active.iter() { + let Some(ticket) = active.ticket else { + break; + }; + if active.observed.is_none() { + batch.request(&ticket)?; + requested = true; + } + } + Ok(requested) + } + pub(crate) fn latch_observations( + &mut self, + batch: &mut Batch, + masks: Option<[u64; 2]>, + ) -> Result { + if self.quarantined || !self.selected(masks) { + return Ok(()); + } + for active in self.active.iter_mut() { + let Some(ticket) = active.ticket else { + break; + }; + if active.observed.is_none() { + active.observed = batch.take(&ticket)?; + } + } + Ok(()) + } + fn front_finished(&self) -> bool { + self.active.front().is_some_and(|active| { + active.observed.is_some() + || active.ticket.is_some_and(|ticket| { + self.graph + .cursors() + .and_then(|c| c.consumed(ticket.item_producer, ticket.item_count)) + .unwrap_or(false) + }) + }) + } + fn valid_ticket(&self, active: &Active, ticket: &Ticket) -> Result { + if ticket.qid != self.qid() + || ticket.descriptor == 0 + || ticket.item_count == 0 + || ticket.work_node != active.work.node_va()? + || self.submitted == 0 + || self.owner != Some(ticket.owner) + || active.packet.order.sequence != ticket.submission + || active.packet.context.id() != ticket.context_id + || active.packet.context.id() != self.binding.context.id() + { + return Err(EIO); + } + Ok(()) + } + + /// Return VM identity for queue-local failure fanout. Deferred callbacks are collected under + /// the mutex and invoked afterwards; failed packets stay retained until their witness. + pub(crate) fn quarantine( + &mut self, + error: Error, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result>> { + if !self.quarantined { + self.quarantined = true; + self.quarantine_error = Some(error); + self.failure_status_pending = true; + self.retirement_proved = false; + self.retired_by_teardown = false; + let all_spared = + self.active.len != 0 && self.active.iter().all(|a| a.packet.completion.spared()); + self.spared_quarantine = all_spared && error == ENODATA; + self.spared_deferred = self.spared_quarantine; + } + let error = self.quarantine_error.ok_or(EIO)?; + let mut terminal = None; + for active in self.active.iter_mut() { + if !self.spared_deferred && !active.failure_deferred { + defer(Deferred::FailedOwned( + active.packet.completion.clone(), + error, + ))?; + active.failure_deferred = true; + } + if self.failure_status_pending + && !active.packet.completion.spared() + && terminal.is_none() + { + terminal = Some(active.packet.completion.status().clone()); + } + } + self.failure_status_pending = false; + self.awaiting_witness = + self.active.len != 0 && self.active.iter().all(|a| a.ticket.is_some()); + Ok(terminal) + } + + /// Poll only the ordered prefix; the next pass may recheck a finished front on another QID's + /// event. A final identity-less scan allows recordless retirement without the one-pass grace. + pub(crate) fn retire( + &mut self, + masks: Option<[u64; 2]>, + epoch: u64, + recovery_state: u32, + mut complete_qos: impl FnMut(qos::Owner) -> Result, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + if self.quarantined || !self.selected(masks) { + return Ok(()); + } + let finished_before = self.retire_pending; + self.retire_pending = false; + let mut oldest = true; + while let Some(front) = self.active.front() { + let result = (|| { + let ticket = front.ticket.ok_or(EIO)?; + self.valid_ticket(front, &ticket)?; + if recovery_state != 0 { + return Err(EAGAIN); + } + let recordless = if epoch != ticket.epoch { + Recordless::CrossedRecovery + } else if masks.is_none() || (finished_before && oldest) { + Recordless::Allowed + } else { + Recordless::NotYet + }; + let next_end = self + .active + .iter() + .skip(1) + .find_map(|a| a.observed.map(|o| o.timestamps[1])); + let timestamps = retirement::poll( + &ticket, + front.observed, + self.graph.cursors()?, + self.graph.stamp()?, + recordless, + self.last_end, + next_end, + )?; + if timestamps.is_some() + && self.previous.as_ref().is_some_and(|p| { + p.publication != Some((front.packet.order.sequence, ticket.epoch)) + }) + { + return Err(EIO); + } + Ok(timestamps) + })(); + oldest = false; + match result { + Ok(Some(timestamps)) => { + let submitted = self.submitted.checked_sub(1).ok_or(EIO)?; + if !front.qos_completed { + complete_qos(self.qos_owner())?; + self.active.iter_mut().next().ok_or(EIO)?.qos_completed = true; + } + let front = self.active.front().ok_or(EIO)?; + self.retire_pending = true; + defer(Deferred::Retired( + front.packet.completion.clone(), + Ok([timestamps[0], timestamps[1], 0, 0]), + ))?; + self.retire_pending = false; + self.submitted = submitted; + let active = self.active.pop().ok_or(EIO)?; + if active.observed.is_some() { + self.last_end = self.last_end.max(timestamps[1]); + } + drop(self.previous.take()); + self.retirement_ready = self.active.len == 0; + if self.released && self.retirement_ready { + self.owner = None; + } + } + Ok(None) | Err(EAGAIN) => { + self.retire_pending = self.front_finished(); + break; + } + Err(error) => { + self.terminal = self.quarantine(error, defer)?.map(|status| (status, error)); + return Ok(()); + } + } + } + Ok(()) + } + + /// Exact terminal witness releases command pins, not installed graph ownership. Spared + /// zero-duration work can return its queue to service after settling its accounting. + pub(crate) fn release_witnessed( + &mut self, + recovery_state: u32, + mut complete_qos: impl FnMut(qos::Owner) -> Result, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + self.release_witnessed_inner(recovery_state, false, &mut complete_qos, defer) + } + + pub(crate) fn prepare_replays( + &mut self, + recovery_state: u32, + complete_qos: impl FnMut(qos::Owner) -> Result, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + self.release_witnessed_inner(recovery_state, true, complete_qos, defer) + } + + fn release_witnessed_inner( + &mut self, + recovery_state: u32, + replay: bool, + mut complete_qos: impl FnMut(qos::Owner) -> Result, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + if !self.awaiting_witness || recovery_state != 0 { + return Ok(()); + } + if !self.retirement_proved { + for active in self.active.iter() { + self.valid_ticket(active, &active.ticket.ok_or(EIO)?)?; + } + let newest = self.active.back().and_then(|a| a.ticket).ok_or(EIO)?; + if !retirement::quarantined_retired( + &newest, + self.graph.cursors()?, + self.graph.stamp()?, + self.active.len as u32, + self.submitted, + )? { + return Ok(()); + } + let candidate = self.spared_deferred + && self.spared_quarantine + && !self.released + && self.owner == Some(newest.owner) + && !self + .active + .iter() + .any(|active| active.packet.completion.has_firmware_consumer()) + && self.active.iter().all(|active| { + active + .packet + .completion + .replay_witness_words() + .is_some_and(|words| words[0] == words[1]) + }); + if candidate && !replay { + return Ok(()); + } + self.replay_retirement = candidate; + self.retirement_proved = true; + } + while let Some(front) = self.active.front() { + if self.spared_quarantine && !front.qos_completed { + complete_qos(self.qos_owner())?; + self.active.iter_mut().next().ok_or(EIO)?.qos_completed = true; + } + let front = self.active.front().ok_or(EIO)?; + let completion = front.packet.completion.clone(); + if self.replay_retirement { + self.replays.push(front.packet.clone()).map_err(|_| EIO)?; + } else if self.spared_deferred { + defer(Deferred::Retired(completion, Err(ENODATA)))?; + } else { + defer(Deferred::Release(completion))?; + } + drop(self.active.pop()); + } + self.awaiting_witness = false; + self.retirement_proved = false; + self.replay_retirement = false; + self.spared_deferred = false; + if self.spared_quarantine { + self.spared_quarantine = false; + self.quarantine_error = None; + self.submitted = 0; + self.kick.clear_parent_after_recovery(); + self.quarantined = false; + self.retire_pending = false; + self.retirement_ready = true; + if self + .previous + .as_ref() + .is_some_and(|p| p.publication.is_some()) + { + drop(self.previous.take()); + } + if self.released { + self.owner = None; + } + } + Ok(()) + } + + pub(crate) fn replay_front(&self) -> Option> { + self.replays.front().cloned() + } + + /// Drop the witnessed owner only after either publication retained it or + /// the off-lock completion collector accepted its terminal result. + pub(crate) fn finish_replay(&mut self, packet: &Arc) -> Result { + if !self + .replays + .front() + .is_some_and(|front| Arc::ptr_eq(front, packet)) + { + return Err(EIO); + } + drop(self.replays.pop()); + self.replay_prepared = false; + if self.replays.len == 0 { + self.replay_error = None; + } + Ok(()) + } + + pub(crate) fn prepare_replay(&mut self, packet: &Arc) -> Result { + if !self + .replays + .front() + .is_some_and(|front| Arc::ptr_eq(front, packet)) + { + return Err(EIO); + } + if let Some(error) = self.replay_error { + return Err(error); + } + if !self.replay_prepared { + if !packet.completion.begin_replay() { + return Err(EAGAIN); + } + packet.completion.reset_for_replay()?; + self.replay_prepared = true; + } + Ok(()) + } + + pub(crate) fn fail_replays(&mut self, error: Error) { + if self.replay_error.is_none() { + self.replay_error = Some(error); + } + } + + /// Caller already stopped both processors. No firmware owner can access retained mappings. + pub(crate) fn stopped(&mut self, defer: &mut impl FnMut(Deferred) -> Result) -> Result { + while let Some(packet) = self.replays.front() { + defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; + drop(self.replays.pop()); + } + while let Some(active) = self.active.front() { + defer(Deferred::Retired( + active.packet.completion.clone(), + Err(EIO), + ))?; + drop(self.active.pop()); + } + Ok(()) + } +} + +impl Queue { + pub(crate) fn matches(&self, owner: u64, qid: u8, context: &Arc) -> Result { + if self.owner != Some(owner) { + return Ok(false); + } + if self.released + || self.quarantined + || self.qid() != qid + || !Arc::ptr_eq(&self.binding.context, context) + || !context.is_current() + { + return Err(EIO); + } + Ok(true) + } + pub(crate) fn reusable(&self) -> bool { + self.installed + // An issued kill still names this graph even if its commands retire + // before the acknowledgement. It cannot acquire a successor owner. + && self.exit.is_none() + && self.owner.is_none() + && self.released + && self.retirement_ready + && !self.quarantined + && self.active.len == 0 + && self.replays.len == 0 + && self.previous.is_none() + } + pub(crate) fn bind_owner(&mut self, owner: u64, context: Arc) -> Result { + if !self.reusable() { + return Err(EBUSY); + } + if !context.is_current() { + return Err(EFAULT); + } + let old = self.binding.context.clone(); + if Arc::ptr_eq(old.status(), context.status()) && old.is_current() { + self.graph.set_owner(&context)?; + self.binding.context = context.clone(); + self.previous = Some(Previous { + binding: None, + context: old, + publication: None, + }); + } else { + let kick_alias = self.kick.map_client(context.vm())?; + let pool_alias = self.pool.map_client(context.vm())?; + let binding = self.graph.bind(context.clone(), kick_alias, pool_alias)?; + self.graph.set_owner(&context)?; + let old_binding = core::mem::replace(&mut self.binding, binding); + self.previous = Some(Previous { + binding: Some(old_binding), + context: old, + publication: None, + }); + } + context.mark_published(); + self.owner = Some(owner); + self.released = false; + self.retirement_ready = false; + self.retired_by_teardown = false; + Ok(()) + } + /// True allows the registry to cancel the never-published QID and drop this graph. An + /// installed graph remains retained even after the logical owner exits without work. + pub(crate) fn release_owner(&mut self) -> Result { + if !self.installed && self.active.len == 0 && !self.quarantined && self.previous.is_none() { + return Ok(true); + } + self.released = true; + if self.active.len == 0 && !self.quarantined { + if let Some(previous) = self.previous.as_ref().filter(|p| p.publication.is_none()) { + if let Err(error) = self.graph.set_owner(&previous.context) { + self.quarantined = true; + self.retired_by_teardown = true; + return Err(error); + } + let mut previous = self.previous.take().ok_or(EIO)?; + if let Some(binding) = previous.binding.take() { + self.binding = binding; + } else { + self.binding.context = previous.context; + } + self.retirement_ready = true; + } + if self.retirement_ready { + self.owner = None; + } + } + Ok(false) + } + /// Exact host identity remains valid even when several commands share an ioctl ID. + pub(crate) fn owns_packet(&self, packet: &Arc) -> bool { + self.active + .iter() + .any(|active| Arc::ptr_eq(&active.packet, packet)) + } + + /// A watchdog settles spared, deferred waiters without changing their VM status. + /// Fully ticketed spared work can still return this queue after its drain witness. + pub(crate) fn timeout( + &mut self, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result>> { + let fresh = !self.quarantined; + let return_spared = self.active.len != 0 + && self + .active + .iter() + .all(|active| active.packet.completion.spared() && active.ticket.is_some()); + if self.spared_deferred { + for active in self.active.iter_mut() { + if !active.failure_deferred { + defer(Deferred::FailedOwned( + active.packet.completion.clone(), + ETIMEDOUT, + ))?; + active.failure_deferred = true; + } + } + self.spared_deferred = false; + } + let result = self.quarantine(ETIMEDOUT, defer); + if fresh { + self.spared_quarantine = return_spared; + } + result + } + + pub(crate) fn cancel( + &mut self, + packet: &Arc, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + if !self.owns_packet(packet) { + return Ok(false); + } + if self.quarantined { + if self.spared_deferred { + for active in self.active.iter_mut() { + if !active.failure_deferred { + defer(Deferred::FailedOwned( + active.packet.completion.clone(), + ECANCELED, + ))?; + active.failure_deferred = true; + } + } + self.spared_deferred = false; + } + return Ok(true); + } + self.released = true; + self.retired_by_teardown = true; + if let Some(active) = self + .active + .iter_mut() + .find(|a| Arc::ptr_eq(&a.packet, packet)) + { + if !active.failure_deferred { + defer(Deferred::FailedOwned( + active.packet.completion.clone(), + ECANCELED, + ))?; + active.failure_deferred = true; + } + } + Ok(true) + } + /// Firmware is halted and publication is closed while its target is copied. + pub(crate) fn publish_recovery_target(&self) -> Result { + let producer = self + .graph + .graph + .word(POINTERS + PointerBlock::PRODUCER)? + .load(Ordering::Relaxed); + self.graph + .graph + .word(POINTERS + PointerBlock::RECOVERY_TARGET)? + .store(producer, Ordering::Relaxed); + Ok(()) + } + + /// The newest covered kick of a QID subsumes its earlier prefix; later independent work + /// must not become a dependency. Errors are observed before returning any firmware token. + pub(crate) fn dependency( + &self, + owner: u64, + status: &Arc, + through: Option, + recovery_state: u32, + ) -> Result> { + if through.is_none() || self.owner != Some(owner) { + return Ok(None); + } + let Some(active) = self + .active + .iter() + .filter(|a| crate::g17::job::Order::contains(through, a.packet.order.sequence)) + .last() + else { + return Ok(None); + }; + if self.quarantined && self.spared_quarantine { + return Err(ECANCELED); + } + if !Arc::ptr_eq(active.packet.completion.status(), status) + || self.quarantined + || recovery_state != 0 + { + return Err(EIO); + } + let ticket = active.ticket.ok_or(EIO)?; + let state = crate::g17::job::fence_status(&active.packet.completion.fence()); + if state < 0 { + return Err(crate::g17::dependency::inherited_error(Error::from_errno( + state, + ))); + } + if state > 0 { + return Ok(None); + } + active.packet.completion.note_firmware_consumer(); + let kick = crate::g17::fw::kick::KickTimestamp::new(ticket.kick).ok_or(EIO)?; + Ok(Some( + crate::g17::fw::kick::KickDependency::new(self.qid(), kick).ok_or(EIO)?, + )) + } +} + +impl Graph { + fn record( + &mut self, + slot: Option, + offset: usize, + window: usize, + value: T, + ) -> Result { + let (object, start) = if let Some(slot) = slot { + if slot >= RECORD_SLOTS { + return Err(EINVAL); + } + (&mut self.records, slot * RECORD_STRIDE + offset) + } else { + (&mut self.graph, 0x6800 + offset) + }; + if core::mem::size_of::() > window { + return Err(EINVAL); + } + let pointer = object.pointer(start, window)?; + // SAFETY: The checked record window is not named by any unconsumed item pointer. The + // device mutex excludes other host publishers, and no Rust reference to it is formed. + unsafe { pointer.write_bytes(0, window) }; + object.write(start, value)?; + fence(Ordering::SeqCst); + Ok(object.gpu_va() + start as u64) + } + fn append(&self, expected: u32, address: u64, before_publish: impl FnOnce()) -> Result { + let cursors = self.cursors()?; + if cursors.producer != expected + || cursors.count != ITEM_RING_ENTRIES + || expected >= ITEM_RING_ENTRIES + { + return Err(EBUSY); + } + let next = (expected + 1) % ITEM_RING_ENTRIES; + if next == cursors.consumer { + return Err(EBUSY); + } + let slot = self.graph.dword(ITEMS + expected as usize * 8)?; + let producer = self.graph.word(POINTERS + 0x40)?; + if slot.load(Ordering::Relaxed) != 0 { + return Err(EBUSY); + } + before_publish(); + slot.store(address, Ordering::Relaxed); + fence(Ordering::SeqCst); + producer.store(next, Ordering::Relaxed); + Ok(next) + } +} + +/// Borrowed device operations used during one publication. Implementations borrow the initdata, +/// coprocessor and accounting fields separately from the Registry that owns physical queues. +pub(crate) trait Host { + fn epoch(&self) -> Result<(u64, u32)>; + fn prepare_compute_shared(&mut self) -> Result; + fn qos_publish(&mut self, owner: qos::Owner, scheduler: u64) -> Result; + fn qos_cancel(&mut self, publication: qos::Publication) -> Result; + fn publish_qid(&mut self, id: kick::Id) -> Result; + fn publish_outer(&mut self, slot: &crate::g17::fw::channels::WorkSlot) -> Result; + fn notify(&mut self, message: u64) -> Result; + fn note_submission(&mut self) -> Result; +} + +impl Queue { + pub(crate) fn spared_pending(&self) -> bool { + self.quarantined && self.spared_quarantine && self.awaiting_witness + } + + /// Retains the packet before any potentially visible store. An error before publication + /// rolls back its work count; every ambiguous error quarantines the graph and packet. + pub(crate) fn publish( + &mut self, + packet: Arc, + dependencies: &[crate::g17::fw::kick::KickDependency], + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + use crate::g17::fw::{ + channels::WorkSlot, + compute::{ComputeArgs, ComputeDescriptor, COMPUTE_KICK_EVENT_MASK, COMPUTE_QOS_CLASS}, + kick::{KickArgs, KickQos}, + }; + let (epoch, state) = host.epoch()?; + if state != 0 { + return Err(EBUSY); + } + let replay_front = self + .replays + .front() + .is_some_and(|front| Arc::ptr_eq(front, &packet)); + if !self.room() + && !(replay_front && !self.quarantined && !self.released && self.active.len < IN_FLIGHT) + { + return Err(EBUSY); + } + if !packet.context.is_current() || !Arc::ptr_eq(&packet.context, &self.binding.context) { + return Err(EFAULT); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let crate::g17::command::Validated::Compute { + usc_base, + cdm_va, + cdm_end_va, + sampler_heap_va, + sampler_count, + .. + } = &packet.command + else { + return Err(EINVAL); + }; + let work = packet.completion.work_state(); + let work_word = work.word(); + if work_word == u32::MAX { + return Err(EINVAL); + } + let work_node = work.node_va()?; + let timestamp_va = packet.completion.compute_timestamp_va()?; + let kick_timestamp = self.kick.timestamp(); + let kick_parent = self.kick.parent(); + let qid = self.qid(); + let ordinal = self.ordinal.checked_add(1).ok_or(EOVERFLOW)?; + let (descriptor_high, descriptor_low) = + self.graph.descriptor_vas(kick_timestamp.slot() as u8); + let first = !self.installed && self.ordinal == 0; + let entry_signal = first + || dependencies + .iter() + .any(|dependency| dependency.qid() != qid); + let record_slot = (!first).then_some(self.ordinal as usize % RECORD_SLOTS); + if let Some(slot) = record_slot { + self.graph.ensure_record_slot(slot)?; + } + let args = ComputeArgs { + usc_base: *usc_base, + cdm_va: *cdm_va, + cdm_end_va: *cdm_end_va, + sampler_heap_va: *sampler_heap_va, + sampler_count: *sampler_count, + descriptor_va: descriptor_low, + context_id: packet.context.id().try_into()?, + context_generation: packet.context.generation(), + work_state: work_word, + work_state_va: work_node, + qid, + kick_count: self.ordinal, + kick: kick_timestamp, + preempt_va: self.binding.preempt.iova(), + operand_state_va: self.binding.operand.iova(), + usage_va: self.binding.usage.iova(), + usage_fw_va: self.graph.usage.gpu_va(), + free_list_slot: u32::from(self.pool.id()), + free_list_control_va: self.pool.control_va(), + stamp_va: self.graph.support.gpu_va() + STAMP as u64, + aux_stamp_va: self.graph.support.gpu_va() + STAMP as u64 + 8, + status_va: [ + self.graph.support.gpu_va() + 0x310, + self.graph.support.gpu_va() + 0x318, + ], + timestamp_va, + }; + let identity = ConfigIdentity { + scheduler: packet.context.scheduler_va(), + qos: packet.context.qos_id(), + context: packet.context.id(), + }; + let config = QueueConfig::new(&QueueConfigArgs { + target: QueueConfigTarget::Compute, + kick_ring_va: self.kick.low_va(), + kick_ring_fw_va: self.kick.firmware_va(), + qid, + install: first, + completion_seed: None, + context_id: args.context_id, + free_list: FreeListBinding { + va: self.pool.control_va(), + generation: u64::from(packet.context.scheduler_generation()), + slot: u32::from(self.pool.id()), + }, + scheduler_va: identity.scheduler, + qos_slot: identity.qos, + qos_update: self + .published_config + .is_none_or(|p| p.scheduler != identity.scheduler || p.qos != identity.qos), + owner_pid: packet.context.owner_pid(), + context_update: self + .published_config + .is_none_or(|p| p.scheduler != identity.scheduler || p.context != identity.context), + })?; + let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp)?; + let predecessor = KickPredecessor::new(qid, kick_parent)?; + let kick_args = KickArgs { + qid, + timestamp: kick_timestamp, + parent: kick_parent, + barriers: dependencies, + descriptor_va: descriptor_high, + queue_va: self.graph.graph.gpu_va(), + qos: KickQos { + slot: identity.qos, + class: COMPUTE_QOS_CLASS, + }, + mcache: None, + event_mask: COMPUTE_KICK_EVENT_MASK, + register_arrays: ComputeDescriptor::register_bindings(descriptor_low)?, + }; + self.active + .push(Active { + packet: packet.clone(), + work: work.clone(), + ticket: None, + observed: None, + failure_deferred: false, + qos_completed: false, + }) + .map_err(|_| EBUSY)?; + self.retirement_ready = false; + let mut visible = false; + let mut accounted = false; + let result = (|| -> Result { + work.add_submitted_kicks(1)?; + accounted = true; + self.graph.descriptors.initialize::( + kick_timestamp.slot() * COMPUTE_DESCRIPTOR_SIZE, + |descriptor| descriptor.write(&args), + )?; + if first { + self.graph.context_high.write( + 0x200, + crate::g17::fw::compute::QueueContextItem::new( + descriptor_high, + self.graph.graph.gpu_va(), + qid, + ), + )?; + } else { + // The retained context item remains at index one; only its descriptor changes. + self.graph.context_high.write(0x210, descriptor_high)?; + let cursors = self.graph.cursors()?; + let _ = self.graph.recycle(cursors); + } + let config_va = self.graph.record(record_slot, CONFIG, 0xc0, config)?; + let cursors = self.graph.cursors()?; + if cursors.count != ITEM_RING_ENTRIES { + return Err(EIO); + } + if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { + return Err(ERANGE); + } + let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; + if free < if entry_signal { 4 } else { 3 } { + return Err(ENOSPC); + } + let prepared_kick = self.kick.prepare_entry(&kick_args)?; + let mut producer; + if first { + producer = self.graph.first_descriptor(descriptor_high)?; + host.prepare_compute_shared()?; + packet.mark_published(); + visible = true; + let qos = host.qos_publish( + qos::Owner { + qid, + qos: identity.qos, + data_master: 2, + }, + identity.scheduler, + )?; + let target = self.pool.reserve_submission().or_else(|error| { + host.qos_cancel(qos)?; + Err(error) + })?; + producer = self + .graph + .append(producer, config_va, || {}) + .or_else(|error| { + self.pool.cancel_unpublished(target)?; + host.qos_cancel(qos)?; + Err(error) + })?; + } else { + let target = self.pool.reserve_submission()?; + producer = self + .graph + .append(cursors.producer, descriptor_high, || { + packet.mark_published() + }) + .or_else(|error| { + self.pool.cancel_unpublished(target)?; + Err(error) + })?; + visible = true; + let qos = host.qos_publish( + qos::Owner { + qid, + qos: identity.qos, + data_master: 2, + }, + identity.scheduler, + )?; + producer = self + .graph + .append(producer, config_va, || {}) + .or_else(|error| { + host.qos_cancel(qos)?; + Err(error) + })?; + } + if first { + prepared_kick.prepare_compute_install()?; + } + prepared_kick.commit(); + self.submitted = self.submitted.checked_add(1).ok_or(EOVERFLOW)?; + let announce_va = self.graph.record(record_slot, ANNOUNCE, 0x40, announce)?; + producer = self.graph.append(producer, announce_va, || {})?; + fence(Ordering::SeqCst); + if entry_signal { + let predecessor_va = + self.graph + .record(record_slot, PREDECESSOR, 0x400, predecessor)?; + producer = self.graph.append(producer, predecessor_va, || {})?; + } + if let Some(slot) = record_slot { + self.graph.record_ends[slot] = Some(producer); + } + host.publish_qid(self.kick.id())?; + host.publish_outer(&WorkSlot::new( + DataMaster::Compute, + self.graph.graph.gpu_va(), + qid, + producer.try_into()?, + !self.outer_started, + )?)?; + let notify_activation = !self.outer_started; + self.outer_started = true; + self.installed = true; + self.published_config = Some(identity); + self.ordinal = ordinal; + let (accepted, state) = host.epoch()?; + if state != 0 || accepted != epoch { + return Err(EIO); + } + let ticket = Ticket { + owner: self.owner.ok_or(EIO)?, + submission: packet.order.sequence, + context_id: packet.context.id(), + qid, + descriptor: descriptor_high, + work_node, + kick: kick_timestamp.get(), + ordinal, + item_producer: producer, + item_count: ITEM_RING_ENTRIES, + epoch, + }; + self.active.back_mut().ok_or(EIO)?.ticket = Some(ticket); + if let Some(previous) = self.previous.as_mut() { + if previous.publication.is_none() { + previous.publication = Some((packet.order.sequence, epoch)); + } + } + if notify_activation { + fence(Ordering::SeqCst); + host.notify((0x83 << 48) | 0x0a)?; + } + fence(Ordering::SeqCst); + host.notify((0x83 << 48) | 0x10)?; + host.note_submission() + })(); + if let Err(error) = result { + if !visible { + if accounted { + if let Err(error) = work.cancel_unpublished_kicks(1) { + self.terminal = + self.quarantine(error, defer)?.map(|status| (status, error)); + return Err(error); + } + } + drop(self.active.pop_back()); + return Err(error); + } + self.terminal = self.quarantine(error, defer)?.map(|status| (status, error)); + return Err(error); + } + Ok(()) + } +} + +impl Graph { + /// The queue has never been installed. Retry staging replaces its initial three item slots; + /// no firmware reader can name this graph until its first complete outer publication. + fn first_descriptor(&self, address: u64) -> Result { + let first = self.graph.dword(ITEMS)?; + let second = self.graph.dword(ITEMS + 8)?; + let third = self.graph.dword(ITEMS + 16)?; + let producer = self.graph.word(POINTERS + 0x40)?; + first.store(address, Ordering::Relaxed); + second.store(0, Ordering::Relaxed); + third.store(0, Ordering::Relaxed); + producer.store(1, Ordering::Relaxed); + Ok(1) + } +} +impl Queue { + pub(crate) fn take_terminal(&mut self) -> Option<(Arc, Error)> { + self.terminal.take() + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs new file mode 100644 index 0000000000000..1ddadae2dba69 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Closed-context kill requests and the command-pin release witness. + +use super::{Context, Deferred, Queue}; +use core::sync::atomic::{fence, AtomicU32, Ordering}; +use kernel::{prelude::*, sync::Arc}; + +pub(crate) const POLL_NS: u64 = 2_500_000_000; +const QUARANTINED_DELAY_NS: u64 = 250_000_000; +const CANCELLED_DELAY_NS: u64 = 2_000_000_000; +const ACK_SLOTS: usize = 16; + +// Firmware-visible cookie namespace persists for the loaded module's lifetime. +static NEXT_COOKIE: AtomicU32 = AtomicU32::new(1); + +pub(crate) fn next_cookie() -> u64 { + loop { + let cookie = NEXT_COOKIE.fetch_add(1, Ordering::Relaxed); + if cookie != 0 { + return cookie.into(); + } + } +} + +pub(crate) struct Acks([u64; ACK_SLOTS]); +impl Acks { + pub(crate) fn new() -> Self { + Self([0; ACK_SLOTS]) + } + pub(crate) fn note(&mut self, cookie: u64) { + if cookie == 0 || self.contains(cookie) { + return; + } + if let Some(slot) = self.0.iter_mut().find(|slot| **slot == 0) { + *slot = cookie; + } + } + pub(crate) fn contains(&self, cookie: u64) -> bool { + cookie != 0 && self.0.contains(&cookie) + } + pub(crate) fn forget(&mut self, cookie: u64) { + for slot in &mut self.0 { + if *slot == cookie { + *slot = 0; + } + } + } + pub(crate) fn snapshot(&self) -> [u64; ACK_SLOTS] { + self.0 + } +} + +pub(crate) struct State { + pub(crate) slot: u8, + pub(crate) qids: u128, + pub(crate) killed: u32, +} +impl State { + pub(crate) fn read(context: &Context) -> Result { + let words = context.scheduler_owner().read_words::<14>(0)?; + let mut bytes = [0u8; 0x38]; + for (destination, word) in bytes.chunks_exact_mut(4).zip(words) { + destination.copy_from_slice(&word.to_le_bytes()); + } + fence(Ordering::Acquire); + Ok(Self::decode(&bytes)) + } + fn decode(bytes: &[u8; 0x38]) -> Self { + let mut low = [0; 8]; + let mut high = [0; 8]; + let mut killed = [0; 4]; + low.copy_from_slice(&bytes[0x12..0x1a]); + high.copy_from_slice(&bytes[0x1a..0x22]); + killed.copy_from_slice(&bytes[0x34..0x38]); + Self { + slot: bytes[1], + qids: u128::from(u64::from_le_bytes(low)) + | (u128::from(u64::from_le_bytes(high)) << 64), + killed: u32::from_le_bytes(killed), + } + } + pub(crate) fn scope(&self, target: u8, own: u128, foreign: u128) -> Option { + if self.slot == 0xff || target >= 128 || self.qids & (1u128 << target) == 0 { + return None; + } + let allowed = (own | (1u128 << target)) & !foreign & !0b11; + (self.qids & !allowed == 0).then_some(self.qids) + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Phase { + Published, + Acknowledged, + Unproven, + Released, +} + +pub(super) struct Transaction { + cookie: u64, + published: u64, + phase: Phase, +} + +impl Queue { + pub(crate) fn exit_candidate(&self, context: &Arc, age: u64) -> bool { + Arc::ptr_eq(self.context(), context) + && self.released + && self.active.len != 0 + && self.exit.is_none() + && (self.quarantined || self.retired_by_teardown) + && age + >= if self.quarantined { + QUARANTINED_DELAY_NS + } else { + CANCELLED_DELAY_NS + } + } + pub(crate) fn exit_owner(&self, context: &Arc) -> Option { + if !self + .active + .iter() + .all(|active| Arc::ptr_eq(&active.packet.context, context)) + { + return None; + } + self.owner.or_else(|| { + self.active + .back() + .and_then(|active| active.ticket.map(|ticket| ticket.owner)) + }) + } + pub(crate) fn record_exit(&mut self, cookie: u64, now: u64) { + self.exit = Some(Transaction { + cookie, + published: now, + phase: Phase::Published, + }); + } + pub(crate) fn exit_polling(&self, now: u64) -> bool { + self.exit.as_ref().is_some_and(|exit| { + exit.phase == Phase::Published && now.saturating_sub(exit.published) < POLL_NS + }) + } + pub(crate) fn exit_cookie(&self) -> Option { + self.exit + .as_ref() + .filter(|exit| matches!(exit.phase, Phase::Published | Phase::Acknowledged)) + .map(|exit| exit.cookie) + } + pub(crate) fn settle_exit( + &mut self, + acks: &Acks, + recovery_closed: bool, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + let Some(cookie) = self.exit_cookie() else { + return Ok(()); + }; + if !acks.contains(cookie) { + return Ok(()); + } + self.exit.as_mut().ok_or(EIO)?.phase = Phase::Acknowledged; + if self.active.len != 0 { + if !recovery_closed { + return Ok(()); + } + let state = match State::read(self.context()) { + Ok(state) => state, + Err(_) => return Ok(()), + }; + if state.killed != 1 { + self.exit.as_mut().ok_or(EIO)?.phase = Phase::Unproven; + return Ok(()); + } + if !self.active.iter().all(|active| { + active + .packet + .completion + .compute_words() + .is_ok_and(|words| words == [0, 0] || words[1] != 0) + }) { + return Ok(()); + } + } + while let Some(active) = self.active.front() { + defer(Deferred::Retired( + active.packet.completion.clone(), + Err(if self.spared_deferred { + ENODATA + } else { + ECANCELED + }), + ))?; + drop(self.active.pop()); + } + self.exit.as_mut().ok_or(EIO)?.phase = Phase::Released; + self.quarantined = true; + self.awaiting_witness = false; + self.spared_deferred = false; + self.spared_quarantine = false; + self.retire_pending = false; + self.retirement_ready = false; + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs new file mode 100644 index 0000000000000..7428f99ab1d7f --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Exact identities and visibility witnesses for a physical compute queue. +//! A shared work-state counter can include other engines and cannot prove this queue retired. + +use kernel::prelude::*; + +pub(crate) const IN_FLIGHT: usize = 16; +pub(crate) const RECORD_SLOTS: usize = 32; +const QIDS: usize = 128; +const TIME_MASK: u64 = 0x003f_ffff_ffff_ffff; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Ticket { + pub(crate) owner: u64, + pub(crate) submission: u64, + pub(crate) context_id: u32, + pub(crate) qid: u8, + pub(crate) descriptor: u64, + pub(crate) work_node: u64, + pub(crate) kick: u64, + pub(crate) ordinal: u32, + pub(crate) item_producer: u32, + pub(crate) item_count: u32, + pub(crate) epoch: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Observation { + pub(crate) timestamps: [u64; 2], +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Cursors { + pub(crate) producer: u32, + pub(crate) consumer: u32, + pub(crate) count: u32, +} + +impl Cursors { + /// Whether firmware consumed every record of the given publication, allowing later work. + pub(crate) fn consumed(&self, end: u32, count: u32) -> Result { + if self.count == 0 || self.count != count { + return Err(EIO); + } + let producer = self.producer % count; + let consumer = self.consumer % count; + let unconsumed = (producer + count - consumer) % count; + let following = (producer + count - end % count) % count; + Ok(unconsumed <= following) + } +} + +/// Outstanding kicks span fewer than half the 24-bit stamp sequence. Zero is a valid wrap. +pub(crate) const fn stamp_covers(word: u32, kick: u64) -> bool { + let delta = word.wrapping_sub((kick as u32).wrapping_shl(8)); + delta & 0xff == 0 && delta < 0x8000_0000 +} + +#[derive(Clone, Copy)] +pub(crate) enum Recordless { + Allowed, + NotYet, + CrossedRecovery, +} + +/// A zero-duration record fails before visibility polling. It proves execution did not occur; +/// ownership still requires a separate retirement witness before mappings can be released. +pub(crate) fn poll( + ticket: &Ticket, + observed: Option, + cursors: Cursors, + stamp: u32, + recordless: Recordless, + floor: u64, + next_end: Option, +) -> Result> { + if observed.is_some_and(|record| record.timestamps[0] == record.timestamps[1]) { + return Err(ENODATA); + } + if !stamp_covers(stamp, ticket.kick) + || !cursors.consumed(ticket.item_producer, ticket.item_count)? + { + return Ok(None); + } + Ok(Some(match observed { + Some(record) => record.timestamps, + None => match recordless { + Recordless::Allowed => [floor, next_end.unwrap_or(floor).max(floor)], + Recordless::NotYet => return Ok(None), + Recordless::CrossedRecovery => return Err(ENODATA), + }, + })) +} + +/// The newest exact publication covers all preceding kicks, but only a fully consumed ring +/// permits releasing quarantined command pins. The installed graph remains retained. +pub(crate) fn quarantined_retired( + ticket: &Ticket, + cursors: Cursors, + stamp: u32, + pending: u32, + submitted: u32, +) -> Result { + if ticket.item_count == 0 || ticket.descriptor == 0 || pending == 0 { + return Err(EINVAL); + } + if cursors.count == 0 || cursors.count != ticket.item_count { + return Err(EIO); + } + let end = ticket.item_producer % cursors.count; + Ok(stamp_covers(stamp, ticket.kick) + && cursors.producer % cursors.count == end + && cursors.consumer % cursors.count == end + && submitted == pending) +} + +#[derive(Clone, Copy)] +struct Request { + descriptor: u64, + kick: u64, + observed: Option, +} + +/// Reusable heap scratch for one shared-ring scan. QID requests are contiguous and bounded; +/// staging and folding never allocate under the device mutex. +pub(crate) struct Batch { + requests: KVVec, + slots: [(usize, usize); QIDS], + open: Option, +} + +impl Batch { + pub(crate) fn new() -> Result { + Ok(Self { + requests: KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?, + slots: [(0, 0); QIDS], + open: None, + }) + } + pub(crate) fn clear(&mut self) { + self.requests.clear(); + self.slots.fill((0, 0)); + self.open = None; + } + pub(crate) fn request(&mut self, ticket: &Ticket) -> Result { + if ticket.descriptor == 0 || ticket.kick == 0 { + return Err(EINVAL); + } + let slot = self.slots.get_mut(ticket.qid as usize).ok_or(EINVAL)?; + if slot.1 != 0 && self.open != Some(ticket.qid) { + return Err(EBUSY); + } + if slot.1 >= IN_FLIGHT { + return Err(ENOSPC); + } + let start = if slot.1 == 0 { + self.requests.len() + } else { + slot.0 + }; + self.requests + .push_within_capacity(Request { + descriptor: ticket.descriptor, + kick: ticket.kick, + observed: None, + }) + .map_err(|_| ENOSPC)?; + *slot = (start, slot.1 + 1); + self.open = Some(ticket.qid); + Ok(()) + } + /// The transport reads eight aligned words from a 0x40-byte firmware record. Records can be + /// overwritten during a scan: incomplete or older identities are skipped, never consumed. + pub(crate) fn fold(&mut self, words: [u64; 8]) -> Result { + let qid = ((words[0] >> 9) & 0x7f) as usize; + let (start, count) = self.slots[qid]; + if count == 0 || words[6] & !1 == 0 || words[7] == 0 { + return Ok(()); + } + let descriptor = words[6] & !1; + let kick = + (((words[0] >> 8) & 0xffff_ffff_00) | ((words[0] >> 48) & 0xff)) & 0xff_ffff_ffff; + let requests = self.requests.get_mut(start..start + count).ok_or(EIO)?; + let Some(request) = requests + .iter_mut() + .find(|r| r.descriptor == descriptor && r.kick == kick) + else { + return Ok(()); + }; + let timestamps = [words[2] & TIME_MASK, words[5] & TIME_MASK]; + if timestamps[0] == 0 + || timestamps[1] == 0 + || request + .observed + .is_some_and(|old| old.timestamps[1] >= timestamps[1]) + { + return Ok(()); + } + request.observed = Some(Observation { timestamps }); + Ok(()) + } + pub(crate) fn take(&mut self, ticket: &Ticket) -> Result> { + let &(start, count) = self.slots.get(ticket.qid as usize).ok_or(EINVAL)?; + let request = self + .requests + .get_mut(start..start + count) + .ok_or(EIO)? + .iter_mut() + .find(|r| r.descriptor == ticket.descriptor && r.kick == ticket.kick) + .ok_or(EIO)?; + Ok(request.observed.take()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs new file mode 100644 index 0000000000000..b821ad5849bef --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -0,0 +1,319 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Userspace submission queues: validate whole ioctls, retain mappings, and enqueue +//! independent engines behind user fences and explicit logical prefixes. + +pub(crate) mod compute; +pub(crate) mod render; + +use super::{ + command::{Parser, UscWindow, Validated}, + completion::Completion, + context::Context, + fence::{Outputs, Submission}, + job::{self, Backend, Engine, Fences, Order, Packet, Scheduler}, + validation::Proof, +}; +use crate::{ + driver::{AsahiDevRef, AsahiDrmDevice}, + file, mmu, +}; +use core::ops::Range; +use kernel::{ + c_str, + dma_fence::FenceContexts, + prelude::*, + sync::{Arc, LockClassKey}, + xarray, +}; + +const EARLY_DEPTH: usize = 2; +static FENCE_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); + +pub(crate) mod admission; + +/// Host identities for retained physical channels, separate from firmware object generations. +/// The device mutex protects allocation; cancelled preparations never reuse IDs. +pub(crate) struct Identities(u64); + +impl Identities { + pub(crate) const fn new() -> Self { + Self(3) + } + + /// The first two identities remain reserved with QIDs 0 and 1. + pub(crate) fn reserve(&mut self, count: u64) -> Result> { + let first = self.0; + let end = first.checked_add(count).ok_or(EOVERFLOW)?; + self.0 = end; + Ok(first..end) + } +} + +/// Validated frontend state prepared before allocating a logical execution context. +pub(crate) struct Frontend { + fences: FenceContexts, + range: Range, + window: UscWindow, +} + +impl Frontend { + pub(crate) fn new(priority: u32, usc_base: u64, range: Range) -> Result { + if priority > 3 { + return Err(EINVAL); + } + let window = UscWindow::new(usc_base, range.clone())?; + let fences = FenceContexts::new(1, c_str!("asahi_g17_queue"), FENCE_KEY)?; + Ok(Self { + fences, + range, + window, + }) + } +} + +pub(crate) struct Queue { + dev: AsahiDevRef, + context: Arc, + backend: Arc, + render: Option>, + compute: Option>, + fences: FenceContexts, + range: Range, + window: UscWindow, + proof: Proof, + sequence: [u64; 2], + frontiers: [KVec<(u64, Fences)>; 2], + // Closing the userspace queue releases admission even if firmware still + // retains one of its execution contexts or installed physical graphs. + _slot: admission::QueueSlot, +} + +impl Queue { + pub(crate) fn new( + dev: &AsahiDrmDevice, + context: Arc, + backend: Arc, + frontend: Frontend, + slot: admission::QueueSlot, + ) -> Result { + let Frontend { + fences, + range, + window, + } = frontend; + let render = Some(Scheduler::new(dev, backend.clone(), Engine::Render)?); + Ok(Self { + dev: dev.into(), + context, + backend, + render, + compute: None, + fences, + range, + window, + proof: Proof::new(), + sequence: [0; 2], + frontiers: core::array::from_fn(|_| KVec::new()), + _slot: slot, + }) + } + + fn ensure_compute(&mut self) -> Result { + // Admission is checked even when this queue already owns a scheduler. + self.backend.ensure_compute()?; + if self.compute.is_none() { + self.compute = Some(Scheduler::new( + &self.dev, + self.backend.clone(), + Engine::Compute, + )?); + } + Ok(()) + } + + fn enqueue( + &mut self, + id: u64, + command: Validated, + order: Order, + guard: mmu::VmJobGuard, + aggregate: &Arc, + inputs: &[file::SyncItem], + objects: Pin<&xarray::XArray>>, + ) -> Result { + let engine = match &command { + Validated::Render { .. } => Engine::Render, + Validated::Compute { .. } => Engine::Compute, + }; + let index = engine.index(); + self.frontiers[index].retain(|(_, fences)| job::fence_status(&fences.completed) == 0); + self.frontiers[index].reserve(1, GFP_KERNEL)?; + if engine == Engine::Compute { + self.ensure_compute()?; + } + let completion = match &command { + Validated::Render { timestamps, .. } => Completion::new( + &self.dev, + &self.fences, + &self.context, + aggregate.member(), + guard, + objects, + timestamps, + false, + )?, + Validated::Compute { timestamps, .. } => Completion::new( + &self.dev, + &self.fences, + &self.context, + aggregate.member(), + guard, + objects, + core::slice::from_ref(timestamps), + true, + )?, + }; + if engine == Engine::Compute { + self.frontiers[Engine::Render.index()] + .retain(|(_, fences)| job::fence_status(&fences.completed) == 0); + } + let other = 1 - index; + let prefix = order.wait_through[other]; + let count = self.frontiers[other] + .iter() + .filter(|(sequence, _)| Order::contains(prefix, *sequence)) + .count(); + let mut dependencies = KVec::with_capacity(inputs.len() + count, GFP_KERNEL)?; + let mut checked_inputs = KVec::with_capacity(inputs.len(), GFP_KERNEL)?; + let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; + for (sequence, fences) in &self.frontiers[other] { + if Order::contains(prefix, *sequence) { + dependencies.push(fences.ready.clone(), GFP_KERNEL)?; + firmware.push(fences.completed.clone(), GFP_KERNEL)?; + } + } + // Each command owns every input fence; independent engines may publish in + // either order and must each consume an input's error status before running. + for sync in inputs { + if let Some(fence) = &sync.fence { + dependencies.push(fence.clone(), GFP_KERNEL)?; + checked_inputs.push(fence.clone(), GFP_KERNEL)?; + } + } + let early = engine == Engine::Render + && dependencies.is_empty() + && self.frontiers[index].len() < EARLY_DEPTH; + let packet = Packet::new( + id, + self.context.clone(), + command, + order, + completion, + checked_inputs, + firmware, + )?; + let scheduler = match engine { + Engine::Render => &mut self.render, + Engine::Compute => &mut self.compute, + }; + let fences = scheduler + .as_mut() + .ok_or(ENODEV)? + .enqueue(packet, dependencies, early)?; + self.sequence[index] = order.sequence; + self.frontiers[index] + .push_within_capacity((order.sequence, fences)) + .map_err(|_| ENOMEM)?; + Ok(()) + } + /// Finish parsing, validation and VM pinning before entering publication. + /// Keeping this frame separate bounds the scheduler/ioctl stack depth. + #[inline(never)] + fn prepare_submission( + &mut self, + bytes: &[u8], + ) -> Result<(KVec<(Validated, Order)>, KVec)> { + let parse_guard = self.context.vm().retain_first_job()?; + let mut parser = Parser::new(bytes); + let mut preceding = [0; 2]; + let mut pending = KVec::with_capacity(1, GFP_KERNEL)?; + while let Some(command) = parser.next()? { + let payload = self.proof.validate( + self.context.vm(), + self.range.clone(), + &self.window, + &command.payload, + )?; + let engine = match &payload { + Validated::Render { .. } => Engine::Render, + Validated::Compute { .. } => Engine::Compute, + }; + let order = Order::new(self.sequence, preceding, command.barriers, engine)?; + preceding[engine.index()] += 1; + pending.push((payload, order), GFP_KERNEL)?; + } + parser.finish()?; + // Every accepted command pins the VM before any command is enqueued. + let mut guards = KVec::with_capacity(pending.len(), GFP_KERNEL)?; + guards + .push_within_capacity(parse_guard) + .map_err(|_| ENOMEM)?; + while guards.len() < pending.len() { + guards + .push_within_capacity(self.context.vm().retain_job()?) + .map_err(|_| ENOMEM)?; + } + Ok((pending, guards)) + } +} + +impl crate::queue::Queue for Queue { + fn submit( + &mut self, + _reg_data: &crate::driver::AsahiRegData<'_>, + id: u64, + syncs: KVec, + input_count: usize, + bytes: &[u8], + objects: Pin<&xarray::XArray>>, + ) -> Result { + let mut outputs = Outputs::new(&self.fences, syncs, input_count)?; + let result = (|| { + if self.context.status().get() != 0 { + return Err(EIO); + } + let (pending, mut guards) = self.prepare_submission(bytes)?; + let aggregate = Submission::new(&self.fences, self.context.clone())?; + outputs.publish(aggregate.clone()); + for (command, order) in pending { + let guard = guards.remove(0).map_err(|_| EIO)?; + self.enqueue( + id, + command, + order, + guard, + &aggregate, + outputs.inputs(), + objects, + )?; + } + Ok(()) + })(); + if let Err(error) = result { + self.context.status().record_if_device_loss(error); + if [EIO, ETIMEDOUT, ENODEV].contains(&error) || self.context.status().get() != 0 { + self.backend.fail_vm(error); + } + } + outputs.finish(result) + } +} + +impl Drop for Queue { + fn drop(&mut self) { + drop(self.render.take()); + drop(self.compute.take()); + self.backend.release_owner(); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs new file mode 100644 index 0000000000000..3a7852be1dab0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -0,0 +1,1459 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Physical paired tiling and fragment queues and their retained command storage. + +mod observation; +pub(crate) use observation::Observed; +mod dependencies; +mod memory; +pub(crate) mod parked; +mod publication; +pub(crate) mod retirement; +mod warm; + +use crate::g17::{ + buffer::{self, target}, + command::Validated, + completion::Deferred, + context::Context, + freelist::RenderPool, + fw::{ + channels::WorkSlot, + kick::{KickDependency, KickTimestamp, McacheTable, RenderRegistration}, + queue::DataMaster, + render as abi, + }, + job::Packet, + kick, + object::{Allocator, Pool}, + qos, recovery, + status::VmStatus, +}; +use crate::{hw::t8140::queues, mmu}; +pub(crate) use dependencies::Dependencies; +use kernel::{prelude::*, sync::Arc}; +use retirement::{Ticket, Tracker}; + +#[derive(Copy, Clone)] +pub(crate) struct RenderAddresses { + /// Tiling then fragment; each pair contains the primary and auxiliary stamp. + pub(crate) stamps: [[u64; 2]; 2], +} + +#[derive(Copy, Clone)] +pub(crate) struct PublicationLease { + pub(crate) free_list_generation: u64, + pub(crate) buffer_token: u64, +} + +/// Host services borrow the device-locked firmware state. Resource construction +/// and command geometry preparation do not borrow this interface. +pub(crate) trait Host { + fn epoch(&self) -> Result<(u64, u32)>; + fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]>; + fn qos_publish(&mut self, owner: qos::Owner, scheduler: u64) -> Result; + fn qos_cancel(&mut self, publication: qos::Publication) -> Result; + fn install_pair(&mut self, registration: &RenderRegistration) -> Result; + fn publish_qid(&mut self, id: kick::Id) -> Result; + fn publish_outer_pair( + &mut self, + fragment: &crate::g17::fw::channels::WorkSlot, + tiling: &crate::g17::fw::channels::WorkSlot, + commit: impl FnOnce() -> Result, + late_tiling: impl FnOnce(), + ) -> Result<[u32; 2]>; + fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result; + fn publish_retained_pair( + &mut self, + fragment: &WorkSlot, + tiling: &WorkSlot, + next: [u32; 2], + ) -> Result; + fn notify_render(&mut self) -> Result; + fn note_submission(&mut self) -> Result; + fn complete_render( + &mut self, + owners: Option<[qos::Owner; 2]>, + pool_id: u16, + generation: u64, + buffer_id: u8, + ) -> Result; +} + +struct Prepared { + packet: Arc, + ordinal: u64, + mcache: Option, +} +struct Active { + qos_completed: bool, + flist_completed: bool, + outer_published: bool, + generation: u64, + packet: Arc, + ticket: Ticket, + kicks: [KickTimestamp; 2], + context_update: bool, + failed: bool, + drained_result: Option>, +} + +/// Installed kick backing remains owned even when this pair has no logical +/// owner. Commands retain their exact descriptor and allocation generations. +pub(crate) struct Pair { + kick_aliases: Option<[Option; 2]>, + memory: KBox, + kicks: [kick::Queue; 2], + free_list: Arc, + pending_aliases: Option>, + prepared: Option, + active: Tracker, + slot: u8, + owner: Option, + ordinal: u64, + initial_packet: u64, + clusters: u32, + descriptor_flags: [u32; 2], + installed: [bool; 2], + configs: [Option; 2], + cursors: [u32; 2], + recovery_generation: u64, + registered: bool, + outer_started: bool, + quarantined: bool, + terminal: Option<(Arc, Error)>, + previous: Option, + displaced: Option, + context_update: bool, +} +impl Pair { + #[inline(never)] + pub(crate) fn new( + alloc: &Allocator<'_>, + pool: &Arc, + ids: [kick::Id; 2], + slot: u8, + owner: u64, + context: Arc, + free_list: Arc, + metrics: buffer::MetricsLease, + buffer_id: (u8, u64), + pm_generation: u64, + first: &Arc, + clusters: u32, + descriptor_flags: [u32; 2], + ) -> Result> { + if slot == 0 + || usize::from(slot) >= queues::RENDER_SLOTS + || owner == 0 + || ids[0].engine() != DataMaster::Tiling + || ids[1].engine() != DataMaster::Fragment + || !Arc::ptr_eq(&context, &first.context) + || !context.is_current() + { + return Err(EINVAL); + } + if let Err(error) = alloc.uat.vm_context_mask(context.vm()) { + if error == ENOENT { + context.status().record(error); + } + return Err(error); + } + let Validated::Render { pass, .. } = &first.command else { + return Err(EINVAL); + }; + let word = first.completion.work_state().word(); + if word == u32::MAX { + return Err(EINVAL); + } + let layout = target::Layout::new(pass, clusters)?; + let storage = KBox::::new_uninit(GFP_KERNEL)?; + let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm())?; + let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm())?; + let (memory, aliases) = memory::Memory::new( + alloc, + pool, + context, + [ids[0].qid(), ids[1].qid()], + &layout, + metrics, + buffer_id, + pm_generation, + 0, + word, + )?; + storage.write_init(kernel::try_init!(Self { + kick_aliases: Some([Some(ta_alias), Some(fragment_alias)]), + memory, + kicks: [tiling, fragment], + free_list, + pending_aliases: Some(aliases), + prepared: None, + active: Tracker::new(), + slot, + owner: Some(owner), + ordinal: 0, + initial_packet: first.id, + clusters, + descriptor_flags, + installed: [false; 2], + configs: [None; 2], + cursors: [0; 2], + recovery_generation: 0, + registered: false, + outer_started: false, + quarantined: false, + terminal: None, + previous: None, + displaced: None, + context_update: true, + })) + } + pub(crate) fn qids(&self) -> [u8; 2] { + self.kicks.each_ref().map(|kick| kick.id().qid()) + } + pub(crate) fn slot(&self) -> u8 { + self.slot + } + pub(crate) fn owner(&self) -> Option { + self.owner + } + pub(crate) fn context(&self) -> &Arc { + &self.memory.context + } + pub(crate) fn pool(&self) -> &Arc { + &self.free_list + } + pub(crate) fn pool_id(&self) -> u64 { + u64::from(self.memory.lifecycle[0].predecessor) + } + pub(crate) fn buffer_id(&self) -> u8 { + self.memory.manager.buffer_id() + } + pub(crate) fn manager(&mut self) -> &mut buffer::Manager { + &mut self.memory.manager + } + pub(crate) fn in_flight(&self) -> bool { + self.active.len() != 0 + } + pub(crate) fn published_in_flight(&self) -> bool { + !self.quarantined && self.active.iter().any(|active| active.outer_published) + } + pub(crate) fn room(&self) -> bool { + !self.quarantined && self.owner.is_some() && self.prepared.is_none() && self.active.room() + } + pub(crate) fn take_terminal(&mut self) -> Option<(Arc, Error)> { + self.terminal.take() + } + + /// The registry owns this pair exclusively during potentially allocating + /// preparation. No queue producer or shared submission counter changes. + pub(crate) fn prepare( + &mut self, + alloc: &Allocator<'_>, + packet: &Arc, + addresses: RenderAddresses, + ) -> Result { + if !self.room() + || !Arc::ptr_eq(&self.memory.context, &packet.context) + || !packet.context.is_current() + { + return Err(EBUSY); + } + let Validated::Render { + pass, attachments, .. + } = &packet.command + else { + return Err(EINVAL); + }; + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let ordinal = self.ordinal; + ordinal.checked_add(1).ok_or(EOVERFLOW)?; + let work = packet.completion.work_state(); + let node = work.node_va()?; + let word = work.word(); + let layout = target::Layout::new(pass, self.clusters)?; + let owners = self.memory.owners(); + self.memory + .target + .ensure_capacity(alloc.dev, packet.context.vm(), &layout, &owners)?; + let target = self.memory.target.prepare(ordinal, &layout)?; + let scene = self.memory.manager.prepare_scene(ordinal)?; + self.memory.generation(ordinal)?.prepare(ordinal, pass)?; + let [deflake, ta_status, fragment_status, fragment_fw, auxiliary] = + self.memory.generation(ordinal)?.addresses(); + let lifecycle = if ordinal != 0 || packet.id != self.initial_packet { + let (fragment, tiling) = + crate::g17::ids::OBJECTS.render_pair(self.memory.lifecycle[0].predecessor)?; + [tiling, fragment] + } else { + self.memory.lifecycle + }; + let (scratch, metric) = self.memory.manager.scene_registers(scene)?; + let timestamp_offset = ordinal as usize % memory::RECORDS * 16; + let timestamp = self.memory.timestamps.gpu_va() + timestamp_offset as u64; + let resources = abi::RenderResources { + context_id: packet.context.id().try_into()?, + context_generation: packet.context.generation(), + pb_slot: self.buffer_id().into(), + free_list_slot: self.free_list.id().into(), + free_list_control_va: self.free_list.control_va(), + kick_record_va: node, + pb_va: self.memory.manager.state_va(), + scene_va: self.memory.manager.scene_va(scene)?, + scene_end_va: self.memory.manager.trailer_va(), + pm_scratch: scratch, + pm_metric: metric, + tilemap_va: target.tilemap, + layermeta_va: target.layer, + heapmeta_va: target.heap, + tpc_va: self.memory.target.tpc_va(), + tpc_size: layout.tpc_size as u64, + deflake_va: deflake, + ta_status_va: ta_status, + fragment_status_va: fragment_status, + fragment_status_fw_va: fragment_fw, + aux_fb_va: auxiliary, + timestamp_va: timestamp, + ta_ids: lifecycle[0], + fragment_ids: lifecycle[1], + unk_2104: self.descriptor_flags[0], + unk_210c: self.descriptor_flags[1], + }; + let qids = self.qids(); + let args = abi::RenderArgs { + pass, + resources: &resources, + ordinal, + ta: abi::RenderStage { + qid: qids[0], + descriptor_va: self.memory.graph.descriptor_client(0, ordinal), + state_word: word, + stamp_va: addresses.stamps[0][0], + aux_stamp_va: addresses.stamps[0][1], + }, + fragment: abi::RenderStage { + qid: qids[1], + descriptor_va: self.memory.graph.descriptor_client(1, ordinal), + state_word: word, + stamp_va: addresses.stamps[1][0], + aux_stamp_va: addresses.stamps[1][1], + }, + }; + self.memory.graph.claim(ordinal)?; + let result = self.memory.write_descriptors( + &args, + attachments[1].as_slice(), + packet.context.id().try_into()?, + ); + let mcache = match result { + Ok(mcache) => mcache, + Err(error) => { + self.memory.graph.release(ordinal)?; + self.memory.generation(ordinal)?.ordinal = None; + self.memory.target.cancel(ordinal); + return Err(error); + } + }; + self.memory.timestamps.write(timestamp_offset, [0u64; 2])?; + self.memory.manager.claim_scene(ordinal, scene)?; + self.prepared = Some(Prepared { + packet: packet.clone(), + ordinal, + mcache, + }); + Ok(()) + } +} + +impl Pair { + /// All descriptor bytes and retained allocations are ready before entry. + /// The first inner producer establishes firmware ownership, including on + /// later errors. Such failures retain the active packet for recovery. + pub(crate) fn publish( + &mut self, + packet: Arc, + dependencies: &[KickDependency], + lease: PublicationLease, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + use crate::g17::fw::{ + channels::WorkSlot, + kick::{KickArgs, KickQos}, + queue::{ + self, FreeListBinding, QueueConfig, QueueConfigArgs, QueueConfigTarget, + TilingBinding, + }, + }; + use core::sync::atomic::{fence, Ordering}; + if self.pending_aliases.is_some() || self.kick_aliases.is_some() { + return Err(EBUSY); + } + let (epoch, recovery_state) = host.epoch()?; + if recovery_state != 0 + || self.quarantined + || !self.active.room() + || packet.completion.render_slot() != Some(self.slot) + { + return Err(EBUSY); + } + let prepared = self.prepared.as_ref().ok_or(EINVAL)?; + if !Arc::ptr_eq(&prepared.packet, &packet) + || prepared.ordinal != self.ordinal + || !self.memory.owns(self.ordinal) + || !packet.context.is_current() + { + return Err(EIO); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let ordinal = self.ordinal; + let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; + let mcache = prepared.mcache; + let qids = self.qids(); + let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); + let parents = self.kicks.each_ref().map(kick::Queue::parent); + let work = packet.completion.work_state(); + let work_slot = work.render_slot(self.slot)?; + self.memory.bind_work_slot(ordinal, qids, work_slot)?; + let payload = work.channel_payload(self.slot, ordinal)?; + let stamps = host.stamps(qids)?; + fence(Ordering::Acquire); + let covered = stamps.map(|stamp| retirement::stamp_covers(stamp, ordinal)); + self.memory + .completion_links(ordinal, kicks, payload, !covered[1])?; + let context = &packet.context; + let qos = context.qos_id(); + let qos_owners = [ + qos::Owner { + qid: qids[0], + qos, + data_master: DataMaster::Tiling as u8, + }, + qos::Owner { + qid: qids[1], + qos, + data_master: DataMaster::Fragment as u8, + }, + ]; + let mut configs = [None; 2]; + for stage in 0..2 { + configs[stage] = Some(QueueConfig::new(&QueueConfigArgs { + target: if stage == 0 { + QueueConfigTarget::Tiling(TilingBinding { + pb_va: self.memory.manager.state_va(), + pb_token: lease.buffer_token, + pb_slot: self.buffer_id().into(), + fragment_qid: qids[1], + }) + } else { + QueueConfigTarget::Fragment + }, + kick_ring_va: self.kicks[stage].low_va(), + kick_ring_fw_va: self.kicks[stage].firmware_va(), + qid: qids[stage], + install: !self.installed[stage], + completion_seed: covered[stage].then(|| { + kicks[stage] + .prev() + .completion_value(payload.wrapping_sub(1)) + }), + context_id: context.id().try_into()?, + free_list: FreeListBinding { + va: self.free_list.control_va(), + generation: lease.free_list_generation, + slot: self.free_list.id().into(), + }, + scheduler_va: context.scheduler_va(), + qos_slot: qos, + qos_update: self.context_update, + owner_pid: context.owner_pid(), + context_update: self.context_update, + })?); + } + let configs = [configs[0].ok_or(EIO)?, configs[1].ok_or(EIO)?]; + let cache = if self.recovery_generation == epoch { + self.configs + } else { + [None; 2] + }; + let present = [ + publication::required(cache[0], configs[0]), + publication::required(cache[1], configs[1]), + ]; + let cursors = self.memory.graph.cursors()?; + if cursors[0].write != self.cursors[0] || cursors[1].write != self.cursors[1] { + return Err(EBUSY); + } + let queues = qids.map(|qid| { + self.memory.graph.queues.gpu_va() + + u64::from(qid) * size_of::() as u64 + }); + let fragment_barriers = [KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?]; + let args = |stage: usize, barriers, arrays, mcache| KickArgs { + qid: qids[stage], + timestamp: kicks[stage], + parent: parents[stage], + barriers, + descriptor_va: self.memory.graph.descriptor_va(stage, ordinal), + queue_va: queues[stage], + qos: KickQos { + slot: qos, + class: if stage == 0 { + abi::TA_QOS_CLASS + } else { + abi::FRAGMENT_QOS_CLASS + }, + }, + mcache, + event_mask: abi::RENDER_KICK_EVENT_MASK, + register_arrays: arrays, + }; + let ta_args = args( + 0, + dependencies, + abi::TaDescriptor::register_bindings(self.memory.graph.descriptor_client(0, ordinal))?, + None, + ); + let fragment_args = args( + 1, + &fragment_barriers, + abi::FragmentDescriptor::register_bindings( + self.memory.graph.descriptor_client(1, ordinal), + )?, + mcache, + ); + let registration = RenderRegistration::new( + (qids[1], self.kicks[1].low_va()), + (qids[0], self.kicks[0].low_va()), + )?; + let [tiling_kick, fragment_kick] = &mut self.kicks; + let tiling_kick = tiling_kick.prepare_entry(&ta_args)?; + let fragment_kick = fragment_kick.prepare_entry(&fragment_args)?; + let memory = &mut *self.memory; + let lanes = [ + publication::Lane::new( + &memory.graph, + 0, + ordinal, + self.cursors[0], + configs[0], + present[0], + kicks[0], + )?, + publication::Lane::new( + &memory.graph, + 1, + ordinal, + self.cursors[1], + configs[1], + present[1], + kicks[1], + )?, + ]; + let windows = [lanes[0].window, lanes[1].window]; + let previous = self.active.front().map_or(cursors, |active| { + active.ticket.windows.map(|window| { + let target = window.target(); + retirement::Cursors { + done: target, + read: target, + write: target, + } + }) + }); + let fragment_slot = WorkSlot::new( + DataMaster::Fragment, + queues[1], + qids[1], + windows[1].target().try_into()?, + !self.outer_started, + )?; + let tiling_slot = WorkSlot::new( + DataMaster::Tiling, + queues[0], + qids[0], + windows[0].target().try_into()?, + !self.outer_started, + )?; + let event_slot = memory + .graph + .support + .word(abi::storage::EVENT_SLOTS + 4)?; + let next_event = event_slot.load(Ordering::Relaxed).wrapping_add(1); + let inner = memory.graph.support.word(0x28000)?; + let next_inner = (next_ordinal as u32).wrapping_mul(2); + let flist_submission = self.free_list.prepare_submission(2)?; + let pm_submission = memory.manager.prepare_submission()?; + self.active + .push(Active { + qos_completed: false, + flist_completed: false, + outer_published: false, + generation: lease.free_list_generation, + packet: packet.clone(), + ticket: Ticket { + ordinal, + windows, + previous, + }, + kicks, + context_update: self.context_update, + failed: false, + drained_result: None, + }) + .map_err(|_| EBUSY)?; + self.prepared = None; + packet.mark_published(); + lanes[1].publish_prefix(); + fragment_kick.commit(); + lanes[1].publish_kick(); + tiling_kick.commit(); + let mut qos_publications = [None; 2]; + let result = (|| { + qos_publications[1] = Some(host.qos_publish(qos_owners[1], context.scheduler_va())?); + qos_publications[0] = Some(host.qos_publish(qos_owners[0], context.scheduler_va())?); + if !self.registered { + host.install_pair(®istration)?; + self.registered = true; + } + for kick in &self.kicks { + host.publish_qid(kick.id())?; + } + host.publish_outer_pair( + &fragment_slot, + &tiling_slot, + || { + work.add_submitted_kicks(2)?; + flist_submission.commit(); + pm_submission.commit(); + Ok(()) + }, + || lanes[0].publish_tiling(event_slot, next_event, inner, next_inner), + ) + })(); + let outer_next = match result { + Ok(next) => next, + Err(error) => { + self.quarantined = true; + if !packet.completion.spared() && self.terminal.is_none() { + self.terminal = Some((packet.completion.status().clone(), error)); + } + self.active + .back() + .ok_or(EIO)? + .packet + .completion + .quarantine_render(); + for publication in qos_publications.into_iter().flatten() { + host.qos_cancel(publication)?; + } + self.active.back_mut().ok_or(EIO)?.qos_completed = true; + self.quarantined = true; + if !packet.completion.spared() && self.terminal.is_none() { + self.terminal = Some((packet.completion.status().clone(), error)); + } + defer(Deferred::FailedOwned(packet.completion.clone(), error))?; + self.active.back_mut().ok_or(EIO)?.failed = true; + return Err(error); + } + }; + self.ordinal = next_ordinal; + self.cursors = windows.map(|window| window.target()); + for stage in 0..2 { + if present[stage] { + self.configs[stage] = Some(configs[stage]); + } + } + self.recovery_generation = epoch; + self.installed = [true; 2]; + self.outer_started = true; + if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { + previous.publication = Some(ordinal); + } + packet.context.mark_published(); + let publish = (|| { + host.activate_render_pool(self.free_list.id(), lease.free_list_generation)?; + host.publish_retained_pair(&fragment_slot, &tiling_slot, outer_next)?; + self.active.back_mut().ok_or(EIO)?.outer_published = true; + host.notify_render()?; + host.note_submission() + })(); + if let Err(error) = publish { + self.quarantined = true; + self.active + .back() + .ok_or(EIO)? + .packet + .completion + .quarantine_render(); + if !packet.completion.spared() && self.terminal.is_none() { + self.terminal = Some((packet.completion.status().clone(), error)); + } + defer(Deferred::FailedOwned(packet.completion.clone(), error))?; + self.active.back_mut().ok_or(EIO)?.failed = true; + return Err(error); + } + Ok(()) + } +} + +struct Observer<'a, H> { + memory: &'a memory::Memory, + qids: [u8; 2], + host: &'a mut H, +} +impl retirement::Memory for Observer<'_, H> { + fn snapshot(&mut self, ordinal: u64) -> Result { + observation::snapshot(self.memory, ordinal) + } + + fn stamps(&mut self) -> Result<[u32; 2]> { + self.host.stamps(self.qids) + } + fn pass(&mut self, ordinal: u64) -> Result<[u64; 2]> { + self.memory.pass(ordinal) + } +} + +impl Pair { + fn qos_owners(&self) -> [qos::Owner; 2] { + let qids = self.qids(); + let qos = self.context().qos_id(); + [ + qos::Owner { + qid: qids[0], + qos, + data_master: DataMaster::Tiling as u8, + }, + qos::Owner { + qid: qids[1], + qos, + data_master: DataMaster::Fragment as u8, + }, + ] + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { + masks.is_none_or(|masks| { + self.qids() + .into_iter() + .all(|qid| masks[usize::from(qid) / 64] & (1u64 << (qid % 64)) != 0) + }) + } + pub(crate) fn contains_packet(&self, packet: &Arc) -> bool { + self.active + .iter() + .any(|active| Arc::ptr_eq(&active.packet, packet)) + || self + .prepared + .as_ref() + .is_some_and(|prepared| Arc::ptr_eq(&prepared.packet, packet)) + } + pub(crate) fn take_deferred_timeout(&self) -> Option<(Arc, Error)> { + self.active.iter().find_map(|active| { + active + .packet + .completion + .take_deferred_timeout() + .map(|error| (active.packet.clone(), error)) + }) + } + + pub(crate) fn oldest_packet(&self) -> Option> { + self.active.front().map(|active| active.packet.clone()) + } + /// Sample PM feedback only after this pair's exact retirement. Busy pairs + /// keep their current capacity and let firmware request growth as needed. + pub(crate) fn host_growth_target(&mut self, packet: &Packet) -> Result> { + let Validated::Render { pass, .. } = &packet.command else { + return Err(EINVAL); + }; + let required = target::Layout::new(pass, self.clusters)?.tvb_blocks; + if self.in_flight() { + return Ok(None); + } + if !self.idle_for_context(self.context())? { + return Err(EBUSY); + } + let feedback = if self.ordinal != 0 { + self.memory.manager.automatic_target()?.unwrap_or(0) + } else { + 0 + }; + let required = required.max(feedback); + Ok((required > self.memory.manager.blocks()).then_some(required)) + } + + pub(crate) fn needs_drain(&self) -> bool { + self.quarantined && self.in_flight() + } + + pub(crate) fn oldest_spared(&self) -> Option { + self.active + .front() + .map(|active| active.packet.completion.spared()) + } + pub(crate) fn visit_owners(&self, visit: &mut dyn FnMut(&Arc)) { + for active in self.active.iter() { + visit(active.packet.completion.status()); + } + } + pub(crate) fn classify(&self, classify: &dyn Fn(&Arc) -> recovery::Class) { + for active in self.active.iter() { + active + .packet + .completion + .classify(classify(active.packet.completion.status())); + } + } + /// A slot stamp must name retained work before its execution markers are meaningful. + pub(crate) fn pass_started(&self, stamp: u64) -> Option { + const MASK: u64 = (1 << 40) - 1; + let ordinal = self.active.iter().map(|active| active.ticket.ordinal) + .find(|ordinal| ordinal.wrapping_add(1) & MASK == stamp & MASK)?; + Some(observation::snapshot(&self.memory, ordinal).ok()?.pass[0] != 0) + } + + /// Firmware is halted and all graph borrowers have joined before this copy. + pub(crate) fn publish_recovery_targets(&self) -> Result { + self.memory.graph.publish_recovery_targets() + } + + pub(crate) fn resync(&mut self) { + self.configs = [None; 2]; + for kick in &mut self.kicks { + kick.clear_parent_after_recovery(); + } + } + pub(crate) fn matches(&self, owner: u64, context: &Arc) -> Result { + Ok(self.owner == Some(owner) + && Arc::ptr_eq(self.context(), context) + && context.is_current()) + } + /// Classification controls VM poisoning. Every failed render fence settles, + /// while command memory remains owned until paired retirement or stop. + pub(crate) fn quarantine( + &mut self, + error: Error, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + self.quarantined = true; + // Successors published after recovery can still fail behind its spared + // head. Classify the whole failing group before any fence can settle. + if self.active.iter().any(|active| active.packet.completion.spared()) { + for active in self.active.iter() { + active.packet.completion.classify(recovery::Class::Spared); + } + } + for active in self.active.iter_mut() { + active.packet.completion.quarantine_render(); + if !active.failed { + defer(Deferred::FailedOwned( + active.packet.completion.clone(), + error, + ))?; + active.failed = true; + if !active.packet.completion.spared() && self.terminal.is_none() { + self.terminal = Some((active.packet.completion.status().clone(), error)); + } + } + } + Ok(()) + } + pub(crate) fn retire( + &mut self, + masks: Option<[u64; 2]>, + _epoch: u64, + state: u32, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + self.retire_front(masks, state, None, host, defer) + } + + pub(crate) fn retire_observed( + &mut self, + observed: Observed, + state: u32, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + self.retire_front(None, state, Some(observed), host, defer) + } + + fn retire_front( + &mut self, + masks: Option<[u64; 2]>, + state: u32, + observed: Option, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + if self.quarantined || state != 0 || !self.selected(masks) { + return Ok(false); + } + let qids = self.qids(); + let qos = self.qos_owners(); + let pool = self.free_list.id(); + let buffer = self.buffer_id(); + loop { + let Some(front) = self.active.front_mut() else { + return Ok(false); + }; + if !front.outer_published { + return Ok(false); + } + if !Arc::ptr_eq(&front.packet.context, &self.memory.context) + || !self.memory.owns(front.ticket.ordinal) + { + return Err(EIO); + } + let mut observer = Observer { + memory: &self.memory, + qids, + host, + }; + let result = if let Some(observed) = observed { + if observed.ticket != front.ticket { + return Err(EIO); + } + retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) + } else { + retirement::poll(&mut front.ticket, &mut observer) + }; + let timestamps = match result { + Ok(retirement::Poll::Complete(timestamps)) => timestamps, + Ok(retirement::Poll::Retry | retirement::Poll::Pending) => return Ok(false), + Err(error) => { + self.quarantine(error, defer)?; + return Ok(false); + } + }; + let ordinal = front.ticket.ordinal; + let context_update = front.context_update; + self.note_retirement(ordinal, context_update); + let front = self.active.front_mut().ok_or(EIO)?; + if !front.flist_completed { + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.qos_completed = true; + front.flist_completed = true; + } + defer(Deferred::Retired( + front.packet.completion.clone(), + Ok(timestamps), + ))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + return Ok(true); + } + } + + /// A closed preparation epoch invalidates only this unpublished descriptor. + /// Its completion remains available for a retry under the next epoch. + pub(crate) fn discard_prepared(&mut self, packet: &Arc) -> Result { + let Some(prepared) = self.prepared.as_ref() else { + return Ok(()); + }; + if !Arc::ptr_eq(&prepared.packet, packet) { + return Err(EBUSY); + } + self.memory.retire(prepared.ordinal)?; + self.prepared = None; + Ok(()) + } +} + +impl Pair { + /// Call before and again after servicing firmware events. FList completion + /// counters are full-width, including beyond the low-word wrap boundary. + pub(crate) fn firmware_drained(&self, host: &impl Host) -> Result { + if self.prepared.is_some() || self.active.iter().any(|a| !a.outer_published) { + return Ok(false); + } + let counters = self.free_list.counters()?; + let stamps = host.stamps(self.qids())?; + let (_, state) = host.epoch()?; + Ok(retirement::drained(self.ordinal, stamps, counters, state)) + } + + /// Account one witnessed tracker, preserving all completion ownership. + /// The caller services its pending FList release before accounting another + /// tracker and invokes release_witnessed only after that control stage. + pub(crate) fn account_drained(&mut self, host: &mut impl Host) -> Result { + use retirement::Memory; + if !self.firmware_drained(host)? { + return Ok(false); + } + let qids = self.qids(); + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let Some(active) = self.active.iter_mut().find(|a| !a.flist_completed) else { + return Ok(false); + }; + let mut observer = Observer { + memory: &self.memory, + qids, + host, + }; + let result = match retirement::poll(&mut active.ticket, &mut observer)? { + retirement::Poll::Complete(ticks) => Ok(ticks), + retirement::Poll::Pending | retirement::Poll::Retry => { + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(active.ticket.ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + if !retirement::abandoned(&active.ticket, snapshot, stamps, state, true)? { + return Ok(false); + } + Err(EIO) + } + }; + let ordinal = active.ticket.ordinal; + let context_update = result.is_ok() && active.context_update; + self.note_retirement(ordinal, context_update); + let active = self + .active + .iter_mut() + .find(|active| active.ticket.ordinal == ordinal) + .ok_or(EIO)?; + host.complete_render( + (!active.qos_completed).then_some(qos), + self.free_list.id(), + active.generation, + buffer, + )?; + active.qos_completed = true; + active.flist_completed = true; + active.drained_result = Some(result); + Ok(true) + } + + /// Fresh proof after asynchronous control publication is the ownership + /// boundary. A refused deferred item leaves the precise owner available. + pub(crate) fn release_witnessed( + &mut self, + host: &impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + if self.active.iter().any(|a| !a.flist_completed) || !self.firmware_drained(host)? { + return Ok(()); + } + while let Some(active) = self.active.front() { + let result = active.drained_result.ok_or(EIO)?; + defer(Deferred::Drained(active.packet.completion.clone(), result))?; + let ordinal = active.ticket.ordinal; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + } + self.quarantined = false; + self.resync(); + Ok(()) + } + + /// The registry first polls the completed prefix through this exact Arc. + /// With successors, source policy quarantines every remaining group. A + /// single unexecuted group may use the strict ownership witness instead. + pub(crate) fn fail_after_observation( + &mut self, + packet: &Arc, + error: Error, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + use retirement::Memory; + if !self.active.iter().any(|a| Arc::ptr_eq(&a.packet, packet)) { + return Ok(false); + } + if !self.quarantined && self.active.len() == 1 { + let active = self.active.front().ok_or(EIO)?; + let mut observer = Observer { + memory: &self.memory, + qids: self.qids(), + host, + }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(active.ticket.ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + if active.outer_published + && retirement::abandoned(&active.ticket, snapshot, stamps, state, false)? + { + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let active = self.active.front_mut().ok_or(EIO)?; + if !active.flist_completed { + host.complete_render( + (!active.qos_completed).then_some(qos), + self.free_list.id(), + active.generation, + buffer, + )?; + active.flist_completed = true; + active.qos_completed = true; + } + if !active.failed && !active.packet.completion.spared() { + active.packet.completion.status().record(EIO); + if self.terminal.is_none() { + self.terminal = Some((active.packet.completion.status().clone(), error)); + } + } + defer(Deferred::Retired( + active.packet.completion.clone(), + Err(EIO), + ))?; + let ordinal = active.ticket.ordinal; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + return Ok(true); + } + } + self.quarantine(error, defer)?; + Ok(true) + } + + /// Both processors have stopped before the caller reaches this method. + /// VM ownership settles before quarantined admission is rehabilitated. + pub(crate) fn stopped(&mut self, defer: &mut impl FnMut(Deferred) -> Result) -> Result { + if let Some(prepared) = self.prepared.as_ref() { + defer(Deferred::Unpublished( + prepared.packet.completion.clone(), + EIO, + ))?; + self.memory.retire(prepared.ordinal)?; + self.prepared = None; + } + while let Some(active) = self.active.front() { + defer(Deferred::Drained( + active.packet.completion.clone(), + Err(EIO), + ))?; + let ordinal = active.ticket.ordinal; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + } + if let Some(previous) = self.previous.as_mut() { + previous.ready = true; + } + Ok(()) + } +} + +impl Pair { + /// Commit a cold graph only after the registry revalidates its preparation + /// lease. The original PB row is restored on every later failure. + pub(crate) fn finish_install( + &mut self, + alloc: &Allocator<'_>, + descriptors: &crate::g17::object::KernelObject, + ) -> Result { + install_memory( + alloc, + descriptors, + self.slot, + &mut self.memory, + &mut self.pending_aliases, + &mut self.kick_aliases, + ) + } +} + +fn install_memory( + alloc: &Allocator<'_>, + descriptors: &crate::g17::object::KernelObject, + slot: u8, + memory: &mut memory::Memory, + pending_aliases: &mut Option>, + kick_aliases: &mut Option<[Option; 2]>, +) -> Result { + use core::sync::atomic::{fence, Ordering}; + if pending_aliases.is_none() && kick_aliases.is_none() { + return Ok(()); + } + if !memory.context.is_current() || kick_aliases.is_none() { + return Err(EIO); + } + let mut driver = KVec::new(); + driver.reserve(2, GFP_KERNEL)?; + let aliases = memory.take_persistent(pending_aliases.as_mut().ok_or(EIO)?)?; + let context = memory.context.clone(); + let vm = context.vm(); + let pool_id = u64::from(memory.lifecycle[0].predecessor); + let mut staged: Option<[u32; 4]> = None; + let offset = usize::from(memory.manager.buffer_id()) * 16; + let result: Result = (|| { + vm.install_render_pool_mappings(slot, pool_id, memory.manager.blocks(), aliases)?; + memory.installed_pool = Some(slot); + alloc.uat.flush_vm_contexts(vm)?; + let words = [ + descriptors.word(offset)?, + descriptors.word(offset + 4)?, + descriptors.word(offset + 8)?, + descriptors.word(offset + 12)?, + ]; + let previous = words.map(|word| word.load(Ordering::Relaxed)); + let pages: u32 = (memory.manager.blocks() + * crate::hw::t8140::resources::TVB_PAGES_PER_BLOCK) + .try_into()?; + let next = crate::g17::fw::buffer::reload_descriptor( + previous, + memory.manager.page_list_va(), + pages, + ); + for (word, value) in words.into_iter().zip(next) { + word.store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + staged = Some(previous); + for mapping in kick_aliases.take().ok_or(EIO)?.into_iter().flatten() { + driver.push(mapping, GFP_KERNEL)?; + } + vm.append_driver_mappings(driver)?; + Ok(()) + })(); + if let Err(error) = result { + if error == ENOENT { + context.status().record(error); + } + if let Some(previous) = staged { + for (index, value) in previous.into_iter().enumerate() { + descriptors + .word(offset + index * 4)? + .store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + } + vm.clear_render_pool_mappings_if_owner(slot, pool_id); + memory.installed_pool = None; + return Err(error); + } + *pending_aliases = None; + Ok(()) +} + +/// A binding prepared while the registry exclusively lends the idle pair. +/// Its frontier prevents a stale off-lock result from replacing another owner. +pub(crate) struct PreparedBinding { + previous_context: Arc, + previous_owner: Option, + ordinal: u64, + owner: u64, + context: Arc, + pool: Arc, + memory: Option>, + aliases: Option>, + kicks: Option<[Option; 2]>, + initial_packet: u64, +} + +/// The registry retains displaced backing until this exact context's release +/// is acknowledged. A warm binding retains the old context without its graph. +pub(crate) struct RetiredBinding { + memory: Option>, + context: Arc, + pool: Arc, + buffer_id: u8, + publication: Option, + ready: bool, +} +impl RetiredBinding { + pub(crate) fn context(&self) -> &Arc { + &self.context + } + pub(crate) fn pool(&self) -> &Arc { + &self.pool + } + pub(crate) fn buffer_id(&self) -> Option { + self.memory.as_ref().map(|_| self.buffer_id) + } +} + +impl Pair { + fn note_retirement(&mut self, ordinal: u64, context_update: bool) { + if !context_update { + return; + } + self.context_update = false; + if let Some(previous) = self.previous.as_mut() { + if previous + .publication + .is_some_and(|published| published <= ordinal) + { + previous.ready = true; + } + } + } + pub(crate) fn take_retired_binding(&mut self) -> Option { + if self.displaced.is_some() { + return self.displaced.take(); + } + if self.previous.as_ref().is_some_and(|p| p.ready) { + self.previous.take() + } else { + None + } + } + pub(crate) fn idle_for_context(&self, context: &Arc) -> Result { + if !Arc::ptr_eq(context, self.context()) { + return Ok(true); + } + if self.prepared.is_some() || self.in_flight() || self.quarantined { + return Ok(false); + } + if self.ordinal == 0 { + return Ok(true); + } + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + Ok(self + .memory + .graph + .cursors()? + .into_iter() + .all(|c| c.done == c.read && c.read == c.write)) + } + pub(crate) fn reusable(&self) -> Result { + let previous_ready = self + .previous + .as_ref() + .is_none_or(|p| p.memory.is_none() && self.context_update && !p.ready); + Ok(self.owner.is_none() + && self.displaced.is_none() + && previous_ready + && self.idle_for_context(self.context())?) + } + /// Logical close leaves the physical channel identity and backing retained. + pub(crate) fn release_owner(&mut self) { + self.owner = None; + } + #[inline(never)] + pub(crate) fn prepare_bind( + &mut self, + alloc: &Allocator<'_>, + pool: &Arc, + owner: u64, + context: Arc, + free_list: Arc, + metrics: Option, + buffer_id: Option<(u8, u64)>, + pm_generation: Option, + first: &Arc, + ) -> Result { + if !self.reusable()? + || owner == 0 + || !Arc::ptr_eq(&context, &first.context) + || !context.is_current() + { + return Err(EBUSY); + } + alloc.uat.vm_context_mask(context.vm())?; + let same_vm = Arc::ptr_eq(self.context().status(), context.status()); + let (memory, aliases, kicks) = if same_vm { + if metrics.is_some() || buffer_id.is_some() || pm_generation.is_some() { + return Err(EINVAL); + } + (None, None, None) + } else { + let metrics = metrics.ok_or(EINVAL)?; + let buffer_id = buffer_id.ok_or(EINVAL)?; + let pm_generation = pm_generation.ok_or(EINVAL)?; + let Validated::Render { pass, .. } = &first.command else { + return Err(EINVAL); + }; + let layout = target::Layout::new(pass, self.clusters)?; + let aliases = [ + self.kicks[0].prepare_client_alias(context.vm())?, + self.kicks[1].prepare_client_alias(context.vm())?, + ]; + let (memory, persistent) = memory::Memory::new( + alloc, + pool, + context.clone(), + self.qids(), + &layout, + metrics, + buffer_id, + pm_generation, + self.ordinal, + first.completion.work_state().word(), + )?; + (Some(memory), Some(persistent), Some(aliases)) + }; + Ok(PreparedBinding { + previous_context: self.context().clone(), + previous_owner: self.owner, + ordinal: self.ordinal, + owner, + context, + pool: free_list, + memory, + aliases, + kicks, + initial_packet: first.id, + }) + } + pub(crate) fn bind_owner( + &mut self, + binding: &mut PreparedBinding, + alloc: &Allocator<'_>, + descriptors: &crate::g17::object::KernelObject, + ) -> Result { + if !self.reusable()? + || binding.ordinal != self.ordinal + || binding.previous_owner != self.owner + || !Arc::ptr_eq(&binding.previous_context, self.context()) + || !binding.context.is_current() + { + return Err(EBUSY); + } + let foreign = binding.memory.is_some(); + if let Some(memory) = binding.memory.as_mut() { + install_memory( + alloc, + descriptors, + self.slot, + memory, + &mut binding.aliases, + &mut binding.kicks, + )?; + } else { + self.memory.set_owner(self.qids(), &binding.context)?; + } + let old_context = self.context().clone(); + let old_buffer = self.buffer_id(); + let old_pool = core::mem::replace(&mut self.free_list, binding.pool.clone()); + let old_memory = binding + .memory + .take() + .map(|memory| core::mem::replace(&mut self.memory, memory)); + let retired = RetiredBinding { + memory: old_memory, + context: old_context, + pool: old_pool, + buffer_id: old_buffer, + publication: None, + ready: foreign, + }; + if foreign { + self.displaced = Some(retired); + } else if self.previous.is_none() { + self.previous = Some(retired); + } + self.memory.context = binding.context.clone(); + self.owner = Some(binding.owner); + self.initial_packet = binding.initial_packet; + self.context_update = true; + if foreign { + self.cursors = [0; 2]; + self.configs = [None; 2]; + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs new file mode 100644 index 0000000000000..d1ec0eb76c7e1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Render predecessors retained while a physical pair is prepared off-lock. + +use super::Pair; +use crate::g17::{ + dependency::Frontier, + fw::kick::KickDependency, + job::{self, Packet}, + status::VmStatus, +}; +use crate::hw::t8140::queues::RENDER_DEPTH; +use kernel::{prelude::*, sync::Arc}; + +struct Predecessor { + packet: Arc, + word: u64, +} + +/// A loan of the graph cannot hide work already published on that pair. Fence +/// status remains live through the retained Packet, even while its storage is away. +pub(crate) struct Dependencies { + owner: Option, + entries: [Option; RENDER_DEPTH as usize], +} + +fn include( + frontier: &mut Frontier, + status: &Arc, + prefix: Option, + packet: &Arc, + word: u64, +) -> Result { + if !Arc::ptr_eq(packet.completion.status(), status) { + return Ok(()); + } + if !job::Order::contains(prefix, packet.order.sequence) { + return Ok(()); + } + frontier.include( + prefix, + packet.order.sequence, + job::fence_status(&packet.completion.fence()), + word, + ) +} + +impl Dependencies { + pub(crate) fn contains_packet(&self, packet: &Arc) -> bool { + self.entries + .iter() + .flatten() + .any(|entry| Arc::ptr_eq(&entry.packet, packet)) + } + + pub(crate) fn collect( + &self, + owner: u64, + status: &Arc, + prefix: Option, + frontier: &mut Frontier, + ) -> Result { + if self.owner != Some(owner) { + return Ok(()); + } + for entry in self.entries.iter().flatten() { + include(frontier, status, prefix, &entry.packet, entry.word)?; + } + Ok(()) + } +} + +impl Pair { + /// The fixed tracker bounds this snapshot; no allocation or packet can be + /// lost between capture and the registry moving the pair under the same lock. + pub(crate) fn dependency_snapshot(&self) -> Dependencies { + let qid = self.qids()[1]; + let mut active = self.active.iter(); + Dependencies { + owner: self.owner, + entries: core::array::from_fn(|_| { + active.next().map(|entry| Predecessor { + packet: entry.packet.clone(), + word: KickDependency::new(qid, entry.kicks[1]) + .map_or(u64::MAX, KickDependency::word), + }) + }), + } + } + + pub(crate) fn collect_dependencies( + &self, + owner: u64, + status: &Arc, + prefix: Option, + frontier: &mut Frontier, + ) -> Result { + if self.owner != Some(owner) { + return Ok(()); + } + let qid = self.qids()[1]; + for entry in self.active.iter() { + let word = + KickDependency::new(qid, entry.kicks[1]).map_or(u64::MAX, KickDependency::word); + include(frontier, status, prefix, &entry.packet, word)?; + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs new file mode 100644 index 0000000000000..f614494e1216a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -0,0 +1,708 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Storage of one render pair and its per-command generations. Client aliases +//! precede their backing owners so teardown cannot leave a dangling GPU view. + +use super::retirement::Cursors; +use crate::g17::{ + buffer::{self, target}, + context::Context, + fw::{ + self, queue, + render::{self as abi, storage}, + }, + object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, +}; +use crate::{ + hw::t8140::{dynamic, kernel_window, low, queues, resources as cfg}, + mmu, +}; +use core::sync::atomic::Ordering; +use kernel::{prelude::*, sync::Arc}; + +pub(super) const RECORDS: usize = 32; +pub(super) const TA_BYTES: usize = 0x14000; +pub(super) const FRAGMENT_BYTES: usize = 0x48000; +pub(super) const GRAPH_SIZE: usize = 0x1c000; +pub(super) const POINTERS: [usize; 2] = [0x9000, 0x9080]; +pub(super) const ITEMS: [usize; 2] = [0xc000, 0x10000]; +pub(super) const ANNOUNCE_STRIDE: usize = 0x80; +pub(super) const ANNOUNCE: [usize; 2] = [0x17000, 0x1b000]; +pub(super) const JOB_LIST: usize = 0xa000; +pub(super) const CONFIG: [usize; 2] = [0, 0x4000]; +pub(super) const CONFIG_STRIDE: usize = 0x180; +const DEPTH: usize = queues::RENDER_DEPTH as usize; + +struct Aliases { + client: mmu::KernelMapping, + _global: mmu::KernelMapping, +} +impl Aliases { + fn new( + object: &KernelObject, + alloc: &Allocator<'_>, + vm: &mmu::Vm, + source: core::ops::Range, + ) -> Result { + let mut range = dynamic::LOWER; + let size = (source.end - source.start) as u64; + loop { + let global = object.map_range( + alloc.uat.kernel_lower_vm(), + source.clone(), + range.clone(), + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + 0, + )?; + let address = global.iova(); + let end = address.checked_add(size).ok_or(EOVERFLOW)?; + match object.map_range( + vm, + source.clone(), + address..end, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + 0, + ) { + Ok(client) => { + return Ok(Self { + client, + _global: global, + }) + } + Err(error) if error == ENOSPC && end < range.end => { + drop(global); + range.start = end; + } + Err(error) => return Err(error), + } + } + } +} + +/// Stable queue records and rotating descriptors/configuration records. +pub(super) struct Graph { + aliases: [Aliases; 2], + pub(super) descriptors: PooledObject, + pub(super) queues: PooledObject, + pub(super) config: PooledObject, + pub(super) support: KernelObject, + pub(super) owners: [Option; RECORDS], +} +impl Graph { + pub(super) fn new( + alloc: &Allocator<'_>, + pool: &Arc, + context: &Context, + qids: [u8; 2], + ) -> Result { + let prot = mmu::PROT_GPU_FW_SHARED_RW; + let descriptors = pool.allocate( + alloc, + Family::Descriptor, + TA_BYTES + FRAGMENT_BYTES, + TA_BYTES + FRAGMENT_BYTES, + prot, + CpuMap::WriteBack, + )?; + let ta = Aliases::new(&descriptors, alloc, context.vm(), 0..TA_BYTES)?; + let fragment = Aliases::new( + &descriptors, + alloc, + context.vm(), + TA_BYTES..TA_BYTES + FRAGMENT_BYTES, + )?; + let mut queues = pool.allocate( + alloc, + Family::QueueState, + GRAPH_SIZE, + 0x8000, + prot, + CpuMap::WriteCombined, + )?; + let config = pool.allocate( + alloc, + Family::QueueConfig, + 0x8000, + 0x8000, + prot, + CpuMap::WriteBack, + )?; + let support = alloc.kernel( + storage::SUPPORT_SIZE, + mmu::UAT_PGSZ as u64, + prot, + CpuMap::WriteCombined, + )?; + let address = queues.gpu_va(); + for stage in 0..2 { + queues.write( + qids[stage] as usize * size_of::(), + queue::QueueRecord::new(&queue::QueueRecordArgs { + pointers_va: address + POINTERS[stage] as u64, + items_va: address + ITEMS[stage] as u64, + job_list_va: context.work_head_va(), + scheduler_va: context.scheduler_va(), + owner_pid: context.owner_pid(), + }), + )?; + queues.initialize::(POINTERS[stage], |view| { + *view = queue::PointerBlock::new(); + Ok(()) + })?; + } + queues.write(JOB_LIST, queue::JobListHead::new(address + JOB_LIST as u64))?; + Ok(Self { + aliases: [ta, fragment], + descriptors, + queues, + config, + support, + owners: [None; RECORDS], + }) + } + pub(super) fn offset(stage: usize, ordinal: u64) -> usize { + let slot = ordinal as usize % RECORDS; + if stage == 0 { + slot * abi::TA_DESCRIPTOR_SIZE + } else { + TA_BYTES + slot * abi::FRAGMENT_SLOT_SIZE + } + } + pub(super) fn descriptor_va(&self, stage: usize, ordinal: u64) -> u64 { + self.descriptors.gpu_va() + Self::offset(stage, ordinal) as u64 + } + pub(super) fn descriptor_client(&self, stage: usize, ordinal: u64) -> u64 { + let stride = if stage == 0 { + abi::TA_DESCRIPTOR_SIZE + } else { + abi::FRAGMENT_SLOT_SIZE + }; + self.aliases[stage].client.iova() + (ordinal as usize % RECORDS * stride) as u64 + } + pub(super) fn publish_recovery_targets(&self) -> Result { + for at in POINTERS { + let producer = self.queues.word(at + queue::PointerBlock::PRODUCER)? + .load(Ordering::Relaxed); + self.queues.word(at + queue::PointerBlock::RECOVERY_TARGET)? + .store(producer, Ordering::Relaxed); + } + Ok(()) + } + + pub(super) fn cursors(&self) -> Result<[Cursors; 2]> { + let mut values = [Cursors { + done: 0, + read: 0, + write: 0, + }; 2]; + for stage in 0..2 { + let at = POINTERS[stage]; + values[stage] = Cursors { + done: self.queues.word(at)?.load(Ordering::Relaxed), + read: self.queues.word(at + 0x30)?.load(Ordering::Relaxed), + write: self.queues.word(at + 0x40)?.load(Ordering::Relaxed), + }; + } + Ok(values) + } + pub(super) fn claim(&mut self, ordinal: u64) -> Result { + let owner = &mut self.owners[ordinal as usize % RECORDS]; + if owner.is_some_and(|old| old != ordinal) { + return Err(EBUSY); + } + *owner = Some(ordinal); + Ok(()) + } + pub(super) fn owns(&self, ordinal: u64) -> bool { + self.owners[ordinal as usize % RECORDS] == Some(ordinal) + } + pub(super) fn release(&mut self, ordinal: u64) -> Result { + if !self.owns(ordinal) { + return Err(EIO); + } + self.owners[ordinal as usize % RECORDS] = None; + Ok(()) + } +} + +struct Compact { + mapping: Option, + address: u64, + _backing: KernelObject, +} +impl Compact { + fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, size: usize) -> Result { + let backing = KernelObject::backing(alloc.dev, size, CpuMap::WriteCombined)?; + let mapping = buffer::compact_alias( + &backing, + vm, + 0..size, + cfg::TVB_PAGE, + mmu::PROT_GPU_SHARED_RW, + mmu::UAT_PGSZ, + )?; + Ok(Self { + address: mapping.iova(), + mapping: Some(mapping), + _backing: backing, + }) + } +} + +pub(super) struct Generation { + deflake: mmu::KernelMapping, + auxiliary: mmu::KernelMapping, + fragment_client: mmu::KernelMapping, + state: KernelObject, + fragment: KernelObject, + pub(super) ordinal: Option, +} +impl Generation { + fn allocate_state(alloc: &Allocator<'_>, vm: &mmu::Vm) -> Result { + KernelObject::new( + alloc.dev, + vm, + crate::g17::object::Placement::In(dynamic::CLIENT_LOWER, cfg::TVB_PAGE), + storage::STATE_SIZE, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + ) + } + fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, state: KernelObject) -> Result> { + let deflake = buffer::compact_alias( + &state, + vm, + storage::DEFLAKE..storage::DEFLAKE + mmu::UAT_PGSZ, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + mmu::UAT_PGSZ, + )?; + let auxiliary_range = cfg::AUXILIARY; + let auxiliary = state.map_range( + vm, + storage::AUXILIARY..storage::AUXILIARY + storage::AUXILIARY_SIZE, + auxiliary_range, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + 0, + )?; + let fragment = alloc.kernel( + mmu::UAT_PGSZ, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let fragment_client = buffer::compact_alias( + &fragment, + vm, + 0..mmu::UAT_PGSZ, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + mmu::UAT_PGSZ, + )?; + KBox::new( + Self { + deflake, + auxiliary, + fragment_client, + state, + fragment, + ordinal: None, + }, + GFP_KERNEL, + ) + .map_err(Into::into) + } + pub(super) fn prepare(&mut self, ordinal: u64, pass: &abi::RenderPass) -> Result { + if self.ordinal.is_some_and(|old| old != ordinal) { + return Err(EBUSY); + } + let viewport = storage::Viewport::new(pass.width, pass.height)?; + self.state.bytes_mut()[storage::AUXILIARY..storage::AUXILIARY + storage::AUXILIARY_SIZE] + .fill(0); + let destination = self + .state + .pointer(storage::DEFLAKE + 0x900, size_of::())? + .cast::(); + if destination.align_offset(align_of::()) != 0 { + return Err(EINVAL); + } + // SAFETY: The complete typed integer view is in the retained CPU mapping + // and naturally aligned. This command generation is exclusively owned + // and unpublished; no prior command can access it until reassigned. + unsafe { viewport.apply(&mut *destination) }; + self.fragment.bytes_mut().fill(0); + self.ordinal = Some(ordinal); + Ok(()) + } + pub(super) fn addresses(&self) -> [u64; 5] { + [ + self.deflake.iova(), + self.state.gpu_va() + storage::TA_STATUS as u64, + self.fragment_client.iova(), + self.fragment.gpu_va(), + self.auxiliary.iova(), + ] + } +} + +pub(super) struct Memory { + _fragment_view: mmu::KernelMapping, + pub(super) graph: Graph, + scratch: Compact, + discard: Compact, + pub(super) target: KBox, + generations: [Option>; DEPTH], + generation_count: usize, + pub(super) timestamps: KernelObject, + pub(super) manager: buffer::Manager, + pub(super) lifecycle: [fw::ObjectIds; 2], + // The extra client views retain their allocation positions even though + // kick entries consume the register arrays embedded in the descriptors. + registers: KernelObject, + pub(super) context: Arc, + pub(super) installed_pool: Option, +} +impl Memory { + #[inline(never)] + pub(super) fn new( + alloc: &Allocator<'_>, + pool: &Arc, + context: Arc, + qids: [u8; 2], + layout: &target::Layout, + metrics: buffer::MetricsLease, + buffer_id: (u8, u64), + pm_generation: u64, + ordinal: u64, + word: u32, + ) -> Result<(KBox, KVec)> { + let storage = KBox::::new_uninit(GFP_KERNEL)?; + let vm = context.vm(); + let mut graph = Graph::new(alloc, pool, &context, qids)?; + let mut target = target::Target::new(alloc.dev, vm, layout)?; + target.prepare(ordinal, layout)?; + let state = Generation::allocate_state(alloc, vm)?; + let scratch = Compact::new(alloc, vm, cfg::SCENE_SIZE as usize)?; + let discard = Compact::new(alloc, vm, cfg::DISCARD_SIZE as usize)?; + let (tvb, mappings) = buffer::Tvb::new(alloc.dev, vm, layout.tvb_blocks)?; + let count = DEPTH; + let initial = if ordinal == 0 { + 0 + } else { + (ordinal ^ 1) as usize % count + }; + let mut generations = core::array::from_fn(|_| None); + generations[initial] = Some(Generation::new(alloc, vm, state)?); + for offset in 1..count { + let index = (initial + offset) % count; + let state = Generation::allocate_state(alloc, vm)?; + generations[index] = Some(Generation::new(alloc, vm, state)?); + } + let timestamps = alloc.kernel( + mmu::UAT_PGSZ, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let manager = buffer::Manager::new( + alloc, + vm, + tvb, + metrics, + buffer::ManagerArgs { + scratch: scratch.address, + discard: discard.address, + buffer_id: buffer_id.0, + generation: pm_generation, + first_ordinal: ordinal, + }, + )?; + let predecessor = crate::g17::ids::OBJECTS.object()?; + let (fragment, tiling) = crate::g17::ids::OBJECTS.render_pair(predecessor)?; + let support_va = graph.support.gpu_va(); + graph.support.initialize::(0, |view| { + view.init( + support_va, + alloc.uat.geometry().kernel_range().start + kernel_window::INDEX, + low::RENDER_RUN_LIST, + low::RENDER_PAGE_LIST, + word, + ) + })?; + let registers = KernelObject::new( + alloc.dev, + vm, + crate::g17::object::Placement::In(dynamic::CLIENT_LOWER, mmu::UAT_PGSZ as u64), + mmu::UAT_PGSZ, + mmu::PROT_GPU_SHARED_RO, + CpuMap::WriteCombined, + )?; + let fragment_view = graph.descriptors.map_range( + vm, + TA_BYTES..TA_BYTES + mmu::UAT_PGSZ, + dynamic::CLIENT_LOWER, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RO, + 0, + )?; + Ok(( + storage.write_init(kernel::try_init!(Self { + graph, + scratch, + discard: discard, + target, + generations, + generation_count: count, + timestamps, + manager, + lifecycle: [tiling, fragment], + _fragment_view: fragment_view, + registers: registers, + context, + installed_pool: None, + }))?, + mappings, + )) + } + pub(super) fn generation(&mut self, ordinal: u64) -> Result<&mut Generation> { + self.generations[ordinal as usize % self.generation_count] + .as_deref_mut() + .ok_or(EIO) + } + pub(super) fn owns(&self, ordinal: u64) -> bool { + self.graph.owns(ordinal) + && self.generations[ordinal as usize % self.generation_count] + .as_ref() + .is_some_and(|generation| generation.ordinal == Some(ordinal)) + } + pub(super) fn pass(&self, ordinal: u64) -> Result<[u64; 2]> { + if !self.owns(ordinal) { + return Err(EIO); + } + let offset = ordinal as usize % RECORDS * 16; + Ok([ + self.timestamps.dword(offset)?.load(Ordering::Relaxed), + self.timestamps.dword(offset + 8)?.load(Ordering::Relaxed), + ]) + } + pub(super) fn owners(&self) -> [Option; DEPTH] { + core::array::from_fn(|index| self.generations[index].as_ref().and_then(|g| g.ordinal)) + } + pub(super) fn retire(&mut self, ordinal: u64) -> Result { + if !self.owns(ordinal) { + return Err(EIO); + } + self.target.retire(ordinal)?; + self.manager.retire_scene(ordinal)?; + self.graph.release(ordinal)?; + self.generation(ordinal)?.ordinal = None; + Ok(()) + } +} + +const _: () = { + assert!(abi::TA_DESCRIPTOR_SIZE * RECORDS <= TA_BYTES); + assert!(abi::FRAGMENT_SLOT_SIZE * RECORDS <= FRAGMENT_BYTES); + assert!(RECORDS * CONFIG_STRIDE <= 0x4000); +}; + +impl Memory { + pub(super) fn write_descriptors( + &mut self, + args: &abi::RenderArgs<'_>, + attachments: &[fw::kick::WriteRange], + context: u8, + ) -> Result> { + let tiling = self + .graph + .descriptors + .pointer( + Graph::offset(0, args.ordinal), + size_of::(), + )? + .cast::(); + let fragment = self + .graph + .descriptors + .pointer( + Graph::offset(1, args.ordinal), + size_of::(), + )? + .cast::(); + if tiling.align_offset(align_of::()) != 0 + || fragment.align_offset(align_of::()) != 0 + { + return Err(EINVAL); + } + // SAFETY: Both complete integer-only views lie in the retained mapping, + // are aligned and disjoint. The full ordinal has an exclusive slot + // claim and no producer refers to this command yet. + unsafe { + (&mut *tiling).write(args)?; + let fragment = &mut *fragment; + fragment.write(args)?; + self.registers.bytes_mut().fill(0); + for (index, array) in fragment.descriptor.arrays.iter().enumerate() { + for (entry, value) in array + .entries + .get(..(array.size & 0xffff) as usize) + .ok_or(EINVAL)? + .iter() + .enumerate() + { + self.registers.write( + index * size_of::() + + entry * size_of::(), + *value, + )?; + } + } + fragment.set_mcache( + args.fragment.descriptor_va, + args.resources.aux_fb_va, + attachments, + context, + ) + } + } +} + +impl Memory { + pub(super) fn completion_links( + &mut self, + ordinal: u64, + kicks: [fw::kick::KickTimestamp; 2], + payload: u8, + fragment_busy: bool, + ) -> Result { + if !self.owns(ordinal) { + return Err(EIO); + } + let tiling = self + .graph + .descriptors + .pointer(Graph::offset(0, ordinal), size_of::())? + .cast::(); + let fragment = self + .graph + .descriptors + .pointer(Graph::offset(1, ordinal), size_of::())? + .cast::(); + // SAFETY: The complete views were bounds/alignment checked during + // preparation. The unchanged full ordinal still exclusively owns both + // slots and neither descriptor has been published. + unsafe { + (&mut *tiling).set_completion_link(kicks[0], kicks[1], payload, fragment_busy); + (&mut *fragment).set_completion_link(kicks[1], payload); + } + Ok(()) + } +} + +impl Memory { + /// Preserve the initial mapping order while moving VM-lifetime mappings + /// out of the graph. Reserve all host storage before moving any mapping. + pub(super) fn take_persistent( + &mut self, + tvb: &mut KVec, + ) -> Result> { + if self.scratch.mapping.is_none() + || self.discard.mapping.is_none() + || self.registers.gpu_va() == 0 + { + return Err(EIO); + } + let mut aliases = KVec::new(); + aliases.reserve(tvb.len().checked_add(3).ok_or(EOVERFLOW)?, GFP_KERNEL)?; + aliases.push(self.scratch.mapping.take().ok_or(EIO)?, GFP_KERNEL)?; + aliases.push(self.discard.mapping.take().ok_or(EIO)?, GFP_KERNEL)?; + aliases.push(self.registers.take_mapping()?, GFP_KERNEL)?; + for mapping in core::mem::take(tvb) { + aliases.push(mapping, GFP_KERNEL)?; + } + Ok(aliases) + } +} + +impl Memory { + pub(super) fn bind_work_slot(&mut self, ordinal: u64, qids: [u8; 2], slot: u32) -> Result { + if slot >= 4 || !self.owns(ordinal) { + return Err(EINVAL); + } + let queues = [ + self.graph + .queues + .word(usize::from(qids[1]) * size_of::() + 0x38)?, + self.graph + .queues + .word(usize::from(qids[0]) * size_of::() + 0x38)?, + ]; + let ta = self + .graph + .descriptors + .pointer(Graph::offset(0, ordinal), size_of::())? + .cast::(); + let fragment = self + .graph + .descriptors + .pointer(Graph::offset(1, ordinal), size_of::())? + .cast::(); + for word in queues { + word.store(slot, Ordering::Relaxed); + } + // SAFETY: The generation owns these prepared, unpublished POD views. + // Packed fields are assigned by value without an unaligned reference. + unsafe { + (*ta).unk_8be = slot; + (*fragment).descriptor.unk_2158 = slot; + } + core::sync::atomic::fence(Ordering::Release); + Ok(()) + } + + pub(super) fn set_owner(&self, qids: [u8; 2], context: &Context) -> Result { + let offsets = qids.map(|qid| usize::from(qid) * size_of::()); + let base = self.graph.queues.pointer(0, GRAPH_SIZE)?; + // Check every field before the first update, retaining the required + // job-head pair, PID pair, then packed scheduler-pointer pair order. + for at in offsets { + self.graph.queues.pointer(at + 0x10, 8)?; + self.graph.queues.pointer(at + 0x48, 4)?; + self.graph.queues.pointer(at + 0x9c, 8)?; + } + // SAFETY: Both queues are retired before retargeting. Complete byte + // views were checked above; packed scheduler words use unaligned stores. + unsafe { + for at in offsets { + base.add(at + 0x10) + .cast::() + .write_unaligned(context.work_head_va()); + } + for at in offsets { + base.add(at + 0x48) + .cast::() + .write_unaligned(context.owner_pid()); + } + for at in offsets { + base.add(at + 0x9c) + .cast::() + .write_unaligned(context.scheduler_va()); + } + } + Ok(()) + } +} + +impl Drop for Memory { + fn drop(&mut self) { + if let Some(slot) = self.installed_pool { + self.context.vm().clear_render_pool_mappings_if_owner( + slot, + u64::from(self.lifecycle[0].predecessor), + ); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/observation.rs b/drivers/gpu/drm/asahi/g17/queue/render/observation.rs new file mode 100644 index 0000000000000..3e853115e4f75 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/observation.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Read-only cursor and timestamp snapshots while a physical pair is borrowed. +//! Final stamps, timestamp refresh and ownership changes belong to commit. + +use super::{memory, retirement, Pair}; +use crate::g17::job::Packet; +use core::sync::atomic::{fence, Ordering}; +use kernel::{prelude::*, sync::Arc}; + +pub(crate) struct Observed { + pub(super) ticket: retirement::Ticket, + pub(super) snapshot: retirement::Snapshot, +} + +impl Pair { + /// Reject incomplete event/stamp prefixes before lending the graph. This + /// is only a prefilter; commit repeats the final stamp and timestamp reads. + pub(crate) fn observation_packet( + &self, + masks: [u64; 2], + stamps: [u32; 2], + state: u32, + ) -> Result>> { + if self.quarantined || self.prepared.is_some() || state != 0 || !self.selected(Some(masks)) + { + return Ok(None); + } + let Some(front) = self.active.front() else { + return Ok(None); + }; + if !front.outer_published { + return Ok(None); + } + let published = front.ticket.ordinal.checked_add(1).ok_or(EOVERFLOW)?; + if !stamps + .into_iter() + .all(|stamp| retirement::stamp_covers(stamp, published)) + { + return Ok(None); + } + Ok(Some(front.packet.clone())) + } + + pub(crate) fn observe_front(&self) -> Result { + let front = self.active.front().ok_or(EIO)?; + if !Arc::ptr_eq(&front.packet.context, &self.memory.context) { + return Err(EIO); + } + let ticket = front.ticket; + for _ in 0..2 { + fence(Ordering::Acquire); + let snapshot = snapshot(&self.memory, ticket.ordinal)?; + if retirement::observe(&ticket, snapshot).is_ok() { + return Ok(Observed { ticket, snapshot }); + } + } + Err(EAGAIN) + } +} + +pub(super) fn snapshot(memory: &memory::Memory, ordinal: u64) -> Result { + if !memory.owns(ordinal) { + return Err(EIO); + } + let cursors = memory.graph.cursors()?; + // Sample the private head before the logical shared head, whose membership + // may include other pairs. Individual completion does not require it empty. + let _first = memory + .graph + .queues + .dword(memory::JOB_LIST)? + .load(Ordering::Relaxed); + let _last = memory + .graph + .queues + .dword(memory::JOB_LIST + 8)? + .load(Ordering::Relaxed); + let head = memory.context.work_head()?; + Ok(retirement::Snapshot { + cursors, + job_list_empty: head == [0, memory.context.work_head_va()], + pass: memory.pass(ordinal)?, + }) +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs new file mode 100644 index 0000000000000..35955810a5af1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Installed kick storage retained after a logical scheduler release is acknowledged. + +use super::{install_memory, memory, target, Pair, RetiredBinding, Tracker}; +use crate::g17::{ + buffer, + command::Validated, + context::Context, + freelist::RenderPool, + job::Packet, + kick, + object::{Allocator, KernelObject, Pool}, +}; +use crate::mmu; +use kernel::{prelude::*, sync::Arc}; + +/// The empty physical pair retains installed addresses and the last scheduler page, +/// without retaining that scheduler's execution context or client root. A prior +/// context awaiting an executed replacement keeps its separate retirement owner. +pub(crate) struct ParkedPair { + kicks: Option<[kick::Queue; 2]>, + _scheduler: Arc, + ids: [kick::Id; 2], + slot: u8, + ordinal: u64, + clusters: u32, + descriptor_flags: [u32; 2], + installed: [bool; 2], + recovery_generation: u64, + registered: bool, + outer_started: bool, + previous: Option, +} + +/// All new graph ownership stays separate from the installed kick storage until commit. +pub(crate) struct Prepared { + memory: Option>, + pool: Arc, + aliases: Option>, + kicks: Option<[Option; 2]>, + ids: [kick::Id; 2], + slot: u8, + ordinal: u64, + owner: u64, + initial_packet: u64, + installed: bool, +} + +impl Pair { + /// The registry also checks the shared free-list release and scheduler-release witness. + /// Drain displaced owners before this preflight; they have their own logical releases. + pub(crate) fn can_park(&self, context: &Arc) -> Result { + Ok(Arc::ptr_eq(context, self.context()) + && self.owner.is_none() + && self + .previous + .as_ref() + .is_none_or(|previous| previous.memory.is_none() && !previous.ready) + && self.displaced.is_none() + && self.pending_aliases.is_none() + && self.kick_aliases.is_none() + && self.idle_for_context(context)?) + } + + /// Consumes the pair only after can_park and all collector capacity checks succeeded + /// under the same device mutex. Moving these owners performs no firmware operation. + pub(crate) fn park(self) -> (ParkedPair, RetiredBinding) { + let context = self.memory.context.clone(); + let scheduler = context.scheduler_owner(); + let buffer_id = self.buffer_id(); + let parked = ParkedPair { + ids: self.kicks.each_ref().map(|queue| queue.id()), + kicks: Some(self.kicks), + _scheduler: scheduler, + slot: self.slot, + ordinal: self.ordinal, + clusters: self.clusters, + descriptor_flags: self.descriptor_flags, + installed: self.installed, + recovery_generation: self.recovery_generation, + registered: self.registered, + outer_started: self.outer_started, + previous: self.previous, + }; + let retired = RetiredBinding { + memory: Some(self.memory), + context, + pool: self.free_list, + buffer_id, + publication: None, + ready: true, + }; + (parked, retired) + } +} + +impl ParkedPair { + pub(crate) fn qids(&self) -> [u8; 2] { + self.ids.map(kick::Id::qid) + } + + #[inline(never)] + pub(crate) fn prepare( + &self, + alloc: &Allocator<'_>, + pool: &Arc, + owner: u64, + context: Arc, + free_list: Arc, + metrics: buffer::MetricsLease, + buffer_id: (u8, u64), + pm_generation: u64, + first: &Arc, + ) -> Result { + if owner == 0 || !Arc::ptr_eq(&context, &first.context) || !context.is_current() { + return Err(EINVAL); + } + let kicks = self.kicks.as_ref().ok_or(EBUSY)?; + let Validated::Render { pass, .. } = &first.command else { + return Err(EINVAL); + }; + let word = first.completion.work_state().word(); + if word == u32::MAX { + return Err(EINVAL); + } + alloc.uat.vm_context_mask(context.vm())?; + let layout = target::Layout::new(pass, self.clusters)?; + let aliases = [ + kicks[0].prepare_client_alias(context.vm())?, + kicks[1].prepare_client_alias(context.vm())?, + ]; + let (memory, persistent) = memory::Memory::new( + alloc, + pool, + context, + self.qids(), + &layout, + metrics, + buffer_id, + pm_generation, + self.ordinal, + word, + )?; + Ok(Prepared { + memory: Some(memory), + pool: free_list, + aliases: Some(persistent), + kicks: Some(aliases), + ids: self.ids, + slot: self.slot, + ordinal: self.ordinal, + owner, + initial_packet: first.id, + installed: false, + }) + } + + fn matches(&self, prepared: &Prepared) -> bool { + self.kicks.is_some() + && self.ids == prepared.ids + && self.slot == prepared.slot + && self.ordinal == prepared.ordinal + && prepared + .memory + .as_ref() + .is_some_and(|memory| memory.context.is_current()) + } + + /// All fallible graph installation precedes taking the installed kicks. Keep the empty + /// parked owner until the caller drops it off-lock, including its old scheduler page. + pub(crate) fn commit( + &mut self, + prepared: &mut Option, + storage: KBox>, + ) -> Result> { + let candidate = prepared.as_ref().ok_or(EINVAL)?; + if !candidate.installed || !self.matches(candidate) { + return Err(EBUSY); + } + let kicks = self.kicks.take().ok_or(EIO)?; + // The candidate was validated above and cannot change through the exclusive borrow. + let Some(prepared) = prepared.take() else { + self.kicks = Some(kicks); + return Err(EIO); + }; + let mut next = prepared; + let Some(memory) = next.memory.take() else { + self.kicks = Some(kicks); + return Err(EIO); + }; + storage.write_init(kernel::try_init!(Pair { + kick_aliases: None, + memory, + kicks, + free_list: next.pool.clone(), + pending_aliases: None, + prepared: None, + active: Tracker::new(), + slot: self.slot, + owner: Some(next.owner), + ordinal: self.ordinal, + initial_packet: next.initial_packet, + clusters: self.clusters, + descriptor_flags: self.descriptor_flags, + installed: self.installed, + configs: [None; 2], + cursors: [0; 2], + recovery_generation: self.recovery_generation, + registered: self.registered, + outer_started: self.outer_started, + quarantined: false, + terminal: None, + previous: self.previous.take(), + displaced: None, + context_update: true, + })) + } +} + +impl Prepared { + /// Called after the registry revalidates its preparation epoch and parked pair identity. + pub(crate) fn finish_install( + &mut self, + parked: &ParkedPair, + alloc: &Allocator<'_>, + descriptors: &KernelObject, + ) -> Result { + if !parked.matches(self) { + return Err(EBUSY); + } + install_memory( + alloc, + descriptors, + self.slot, + self.memory.as_mut().ok_or(EIO)?, + &mut self.aliases, + &mut self.kicks, + )?; + self.installed = true; + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs new file mode 100644 index 0000000000000..3df92062735e3 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Preflighted item-ring publications. Each lane retains an independent cursor +//! and omits its configuration record only when its durable identity matches. + +use super::{memory, retirement::Window}; +use crate::g17::fw::{ + kick::KickTimestamp, + queue::{DataMaster, KickAnnounce, QueueConfig, ITEM_RING_ENTRIES}, +}; +use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; +use kernel::prelude::*; + +fn identity(mut config: QueueConfig) -> QueueConfig { + config.install = 0; + config.context_update = 0; + config.completion_seed_valid = 0; + config.completion_seed = 0; + config.qos_update = 0; + config +} +pub(super) fn required(previous: Option, config: QueueConfig) -> bool { + config.install != 0 + || config.context_update != 0 + || config.completion_seed_valid != 0 + || config.qos_update != 0 + || previous.map(identity) != Some(identity(config)) +} + +pub(super) struct Lane<'a> { + items: [&'a AtomicU64; 3], + producer: &'a AtomicU32, + values: [u64; 3], + config: QueueConfig, + config_destination: *mut u64, + config_present: bool, + announce: KickAnnounce, + announce_destination: *mut KickAnnounce, + prefix: usize, + prefix_end: u32, + pub(super) window: Window, +} +impl<'a> Lane<'a> { + pub(super) fn new( + graph: &'a memory::Graph, + stage: usize, + ordinal: u64, + base: u32, + config: QueueConfig, + present: bool, + timestamp: KickTimestamp, + ) -> Result { + let count = 2 + u32::from(present); + let window = Window::new(base, count, ITEM_RING_ENTRIES)?; + let offset = ordinal as usize % memory::RECORDS; + let config_offset = memory::CONFIG[stage] + offset * memory::CONFIG_STRIDE; + let announce_offset = memory::ANNOUNCE[stage] + offset * memory::ANNOUNCE_STRIDE; + let config_destination = graph + .config + .pointer(config_offset, size_of::())? + .cast::(); + if config_destination.align_offset(align_of::()) != 0 { + return Err(EINVAL); + } + let announce_destination = graph + .queues + .pointer(announce_offset, size_of::())? + .cast::(); + let index = + |delta: u32| memory::ITEMS[stage] + ((base + delta) % ITEM_RING_ENTRIES) as usize * 8; + let items = [ + graph.queues.dword(index(0))?, + graph.queues.dword(index(1))?, + graph.queues.dword(index(2))?, + ]; + let announce_va = graph.queues.gpu_va() + announce_offset as u64; + let descriptor = graph.descriptor_va(stage, ordinal); + let values = if present { + [ + descriptor, + graph.config.gpu_va() + config_offset as u64, + announce_va, + ] + } else { + [descriptor, announce_va, 0] + }; + Ok(Self { + items, + producer: graph.queues.word(memory::POINTERS[stage] + 0x40)?, + values, + config, + config_destination, + config_present: present, + announce: KickAnnounce::new( + config.qid.try_into()?, + if stage == 0 { + DataMaster::Tiling + } else { + DataMaster::Fragment + }, + timestamp, + )?, + announce_destination, + prefix: count as usize - 1, + prefix_end: (base + count - 1) % ITEM_RING_ENTRIES, + window, + }) + } + fn items(&self, range: core::ops::Range) { + for index in range { + self.items[index].store(self.values[index], Ordering::Relaxed); + } + } + fn config(&self) { + if !self.config_present { + return; + } + let source = (&self.config as *const QueueConfig).cast::(); + for index in 0..size_of::() / 8 { + // SAFETY: QueueConfig is packed integer storage with no padding; + // every source qword is read unaligned. The destination is a + // checked, aligned record owned by this unpublished ordinal. + unsafe { + self.config_destination + .add(index) + .write(source.add(index).read_unaligned()); + } + } + } + fn announce(&self) { + // SAFETY: The full record is inside the retained queue graph and is + // exclusively owned by this ordinal. Publication follows its copy. + unsafe { + self.announce_destination.write_unaligned(self.announce); + } + } + pub(super) fn publish_prefix(&self) { + self.items(0..self.prefix); + self.config(); + fence(Ordering::SeqCst); + self.producer.store(self.prefix_end, Ordering::Relaxed); + } + pub(super) fn publish_kick(&self) { + self.announce(); + self.items(self.prefix..self.prefix + 1); + fence(Ordering::SeqCst); + self.producer.store(self.window.target(), Ordering::Relaxed); + } + pub(super) fn publish_tiling( + &self, + event_slot: &AtomicU32, + next_slot: u32, + inner: &AtomicU32, + next_inner: u32, + ) { + self.items(0..self.prefix + 1); + self.config(); + self.announce(); + fence(Ordering::SeqCst); + self.producer.store(self.window.target(), Ordering::Relaxed); + event_slot.store(next_slot, Ordering::Relaxed); + inner.store(next_inner, Ordering::Relaxed); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs new file mode 100644 index 0000000000000..d0a2d566bdc8d --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -0,0 +1,348 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Paired render retirement evidence and bounded ownership of in-flight work. +//! Queue progress and execution timestamps precede the completion stamps which +//! authorize releasing command memory. A signalled error is not that witness. + +use crate::hw::t8140::queues; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +const DEPTH: usize = queues::RENDER_DEPTH as usize; +const MAX_GROUP_ITEMS: u32 = 4; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Cursors { + pub(crate) done: u32, + pub(crate) read: u32, + pub(crate) write: u32, +} + +impl Cursors { + fn valid(self) -> bool { + self.done <= self.write && self.read <= self.write + } + + fn idle(self) -> bool { + self.done == self.read && self.read == self.write + } +} + +/// Each engine may omit its configuration update independently. Preserve both +/// exact windows; neither target can be reconstructed from a render ordinal. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Window { + base: u32, + count: u32, + capacity: u32, +} + +impl Window { + pub(crate) fn new(base: u32, count: u32, capacity: u32) -> Result { + if base >= capacity || !(2..=MAX_GROUP_ITEMS).contains(&count) || count >= capacity { + return Err(EINVAL); + } + Ok(Self { + base, + count, + capacity, + }) + } + + pub(crate) fn target(self) -> u32 { + ((u64::from(self.base) + u64::from(self.count)) % u64::from(self.capacity)) as u32 + } + + fn normalize(self, cursors: Cursors) -> Result { + // Earlier and later groups can coexist in a snapshot. Anchor behind + // this group's base so a backwards observation cannot look like wrap. + let back = MAX_GROUP_ITEMS * (DEPTH as u32 - 1); + let limit = MAX_GROUP_ITEMS * DEPTH as u32 + back; + if limit >= self.capacity { + return Err(EINVAL); + } + let base = (u64::from(self.base) + u64::from(self.capacity) - u64::from(back)) + % u64::from(self.capacity); + let distance = |value: u32| -> Result { + if value >= self.capacity { + return Err(EIO); + } + let distance = + (u64::from(value) + u64::from(self.capacity) - base) % u64::from(self.capacity); + if distance > u64::from(limit) { + return Err(EIO); + } + Ok(distance as u32) + }; + Ok(Cursors { + done: distance(cursors.done)?, + read: distance(cursors.read)?, + write: distance(cursors.write)?, + }) + } + + fn progress(self, previous: Cursors, current: Cursors) -> Result<(bool, bool, bool)> { + let previous = self.normalize(previous)?; + let current = self.normalize(current)?; + let prefix = MAX_GROUP_ITEMS * (DEPTH as u32 - 1) + self.count; + if !previous.valid() + || !current.valid() + || current.done < previous.done + || current.read < previous.read + || current.write < previous.write + || prefix > current.write + { + return Err(EIO); + } + Ok(( + current.read >= prefix, + current.done >= prefix, + current.idle(), + )) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Ticket { + /// Full physical-pair ordinal, retained through owner switches and slot wrap. + pub(crate) ordinal: u64, + pub(crate) windows: [Window; 2], + pub(crate) previous: [Cursors; 2], +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Snapshot { + pub(crate) cursors: [Cursors; 2], + pub(crate) job_list_empty: bool, + /// The tiler writes the start and fragment processing writes the end. + pub(crate) pass: [u64; 2], +} + +impl Snapshot { + fn executed(self) -> bool { + self.pass[0] != 0 && self.pass[1] != 0 + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Observation { + pub(crate) accepted: bool, + pub(crate) retired: bool, + pub(crate) complete: bool, + pub(crate) idle: bool, +} + +pub(crate) fn observe(ticket: &Ticket, snapshot: Snapshot) -> Result { + let ta = ticket.windows[0].progress(ticket.previous[0], snapshot.cursors[0])?; + let fragment = ticket.windows[1].progress(ticket.previous[1], snapshot.cursors[1])?; + let retired = ta.1 && fragment.1; + Ok(Observation { + accepted: ta.0 && fragment.0, + retired, + complete: retired && snapshot.executed(), + idle: ta.2 && fragment.2 && snapshot.job_list_empty, + }) +} + +/// Outstanding publications span less than half the 24-bit stamp sequence. +/// A zero stamp is valid when that sequence wraps. +pub(crate) const fn stamp_covers(stamp: u32, published: u64) -> bool { + let delta = stamp.wrapping_sub((published as u32).wrapping_shl(8)); + delta & 0xff == 0 && delta < 0x8000_0000 +} + +fn visible(ticket: &Ticket, stamps: [u32; 2]) -> Result { + let published = ticket.ordinal.checked_add(1).ok_or(EOVERFLOW)?; + Ok(stamps + .into_iter() + .all(|stamp| stamp_covers(stamp, published))) +} + +/// Reads the graph owned by one physical pair. The owner validates the full +/// ordinal before reading reused descriptor/timestamp slots. Word accessors +/// perform aligned atomic loads; the observer supplies the acquire barriers. +pub(crate) trait Memory { + fn snapshot(&mut self, ordinal: u64) -> Result; + fn stamps(&mut self) -> Result<[u32; 2]>; + fn pass(&mut self, ordinal: u64) -> Result<[u64; 2]>; +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Poll { + Retry, + Pending, + Complete([u64; 4]), +} + +/// Observe the ordered front. Individually sampled cursor words can straddle +/// a firmware update, so an invalid snapshot receives one immediate reread. +/// Shared work-state membership can include other engines and cannot gate a +/// successful individual member's completion. +pub(crate) fn poll(ticket: &mut Ticket, memory: &mut impl Memory) -> Result { + fence(Ordering::Acquire); + let mut snapshot = memory.snapshot(ticket.ordinal)?; + let observation = match observe(ticket, snapshot) { + Ok(observation) => observation, + Err(_) => { + fence(Ordering::Acquire); + snapshot = memory.snapshot(ticket.ordinal)?; + match observe(ticket, snapshot) { + Ok(observation) => observation, + Err(_) => return Ok(Poll::Retry), + } + } + }; + complete_poll(ticket, memory, snapshot, observation) +} + +/// Finish an observation sampled while its exact pair was exclusively borrowed. +/// The caller restores that pair and validates its epoch and front ticket first. +/// Completion stamps and the final timestamp refresh remain commit-time reads. +pub(crate) fn poll_observed( + ticket: &mut Ticket, + memory: &mut impl Memory, + snapshot: Snapshot, +) -> Result { + let observation = observe(ticket, snapshot)?; + complete_poll(ticket, memory, snapshot, observation) +} + +fn complete_poll( + ticket: &mut Ticket, + memory: &mut impl Memory, + snapshot: Snapshot, + observation: Observation, +) -> Result { + if observation.complete { + if !visible(ticket, memory.stamps()?)? { + return Ok(Poll::Pending); + } + // Refresh this generation's pass after observing the post-maintenance + // stamps. The pre-stamp snapshot alone cannot authorize export. + fence(Ordering::Acquire); + let [start, end] = memory.pass(ticket.ordinal)?; + return Ok(Poll::Complete([start, start, start, end])); + } + ticket.previous = snapshot.cursors; + Ok(Poll::Pending) +} + +/// Strict timeout settlement accepts only a nonexecuted, retired and unlinked +/// group. Drain reclamation may also settle executed groups, but retains both +/// ownership proofs and the exact post-maintenance stamps in either case. +pub(crate) fn abandoned( + ticket: &Ticket, + snapshot: Snapshot, + stamps: [u32; 2], + recovery_state: u32, + accept_executed: bool, +) -> Result { + let completion = observe(ticket, snapshot)?; + if !completion.retired || !snapshot.job_list_empty || (snapshot.executed() && !accept_executed) + { + return Ok(false); + } + Ok(visible(ticket, stamps)? && recovery_state == 0) +} + +/// Pair-wide proof, checked before and after servicing free-list events and +/// again after retiring all trackers. A proof from before event service must +/// not authorize releasing guards after a new recovery began. +pub(crate) fn drained( + published: u64, + stamps: [u32; 2], + free_list: [u64; 2], + recovery_state: u32, +) -> bool { + published == 0 + || (stamps + .into_iter() + .all(|stamp| stamp_covers(stamp, published)) + && free_list[0] == free_list[1] + && recovery_state == 0) +} + +/// Fixed storage keeps admission, rollback and retirement allocation-free. +/// Each element retains its packet and resource leases independently of fences. +pub(crate) struct Tracker { + entries: [Option; DEPTH], + head: usize, + len: usize, +} + +impl Tracker { + pub(crate) const fn new() -> Self { + Self { + entries: [const { None }; DEPTH], + head: 0, + len: 0, + } + } + + pub(crate) fn len(&self) -> usize { + self.len + } + pub(crate) fn room(&self) -> bool { + self.len < DEPTH + } + pub(crate) fn front(&self) -> Option<&T> { + if self.len == 0 { + None + } else { + self.entries[self.head].as_ref() + } + } + pub(crate) fn front_mut(&mut self) -> Option<&mut T> { + if self.len == 0 { + None + } else { + self.entries[self.head].as_mut() + } + } + pub(crate) fn back(&self) -> Option<&T> { + if self.len == 0 { + None + } else { + self.entries[(self.head + self.len - 1) % DEPTH].as_ref() + } + } + pub(crate) fn back_mut(&mut self) -> Option<&mut T> { + if self.len == 0 { + None + } else { + self.entries[(self.head + self.len - 1) % DEPTH].as_mut() + } + } + pub(crate) fn push(&mut self, value: T) -> core::result::Result<(), T> { + if !self.room() { + return Err(value); + } + self.entries[(self.head + self.len) % DEPTH] = Some(value); + self.len += 1; + Ok(()) + } + pub(crate) fn pop(&mut self) -> Option { + if self.len == 0 { + return None; + } + let value = self.entries[self.head].take(); + self.head = (self.head + 1) % DEPTH; + self.len -= 1; + value + } + + pub(crate) fn iter(&self) -> impl Iterator { + Iterator::chain( + self.entries[self.head..].iter(), + self.entries[..self.head].iter(), + ) + .take(self.len) + .filter_map(Option::as_ref) + } + pub(crate) fn iter_mut(&mut self) -> impl Iterator { + let (before, after) = self.entries.split_at_mut(self.head); + Iterator::chain(after.iter_mut(), before.iter_mut()) + .take(self.len) + .filter_map(Option::as_mut) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/warm.rs b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs new file mode 100644 index 0000000000000..4db471f254ec6 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Fresh render graph cursor initialization at a retained physical ordinal. + +use super::{memory, Pair}; +use crate::g17::fw::queue::PointerBlock; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +impl Pair { + /// A new graph at a retained ordinal starts its variable item rings at zero. + /// The old graph has retired; firmware has not received this graph's queues. + pub(crate) fn rebase_fresh_graph(&mut self) -> Result { + if self.ordinal == 0 { + return Ok(()); + } + fence(Ordering::Acquire); + if self + .memory + .graph + .cursors()? + .into_iter() + .any(|cursor| cursor.done != 0 || cursor.read != 0 || cursor.write != 0) + { + return Err(EBUSY); + } + let mut blocks = [core::ptr::null::(); 2]; + for (block, offset) in blocks.iter_mut().zip(memory::POINTERS) { + let pointer = self + .memory + .graph + .queues + .pointer(offset, size_of::())?; + if pointer.align_offset(align_of::()) != 0 { + return Err(EINVAL); + } + *block = pointer.cast(); + } + for block in blocks { + // SAFETY: Both complete pointer blocks were checked above. This fresh + // graph is unpublished; only the live atomic cursors are accessed. + unsafe { &*block }.rebase(0); + } + fence(Ordering::SeqCst); + self.cursors = [0; 2]; + Ok(()) + } + +} diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs new file mode 100644 index 0000000000000..7e3db2f6dae78 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Firmware restart handshake and per-VM failure classification. +//! +//! The device mutex serializes this state with publication. Preparations are closed and joined +//! without that mutex before `service`; no queue callback may signal a fence here. Queue +//! owners retain failed work until its retirement witness, a complete pair-drain witness, or +//! processor stop. A failed fence alone never releases mappings. + +mod attribution; +mod slot; +pub(crate) use attribution::Sources; +use attribution::Attribution; + +pub(super) fn slot_key(slot: usize) -> Option { + slot::key(slot).ok() +} + +use super::status::VmStatus; +use kernel::{prelude::*, sync::Arc}; + +const QIDS: u8 = 128; +const GUILTY: u8 = 1 << 1; +const CANCELLED: u8 = 1 << 2; +/// Retained valid entries, in firmware table order. +pub(super) const MAX_ENTRIES: usize = 64; +const MAX_GUILTY_VMS: usize = 8; +const RECOVERY_POLLS: usize = 500; + +/// One valid entry from the firmware's 256-entry recovery table. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(super) struct Entry { + pub(super) qid: u8, + pub(super) flags: u8, + pub(super) stamp: u32, +} + +impl Entry { + pub(super) const fn decode(flags: u32, stamp: u32) -> Option { + if flags & 1 == 0 { + return None; + } + Some(Self { + qid: ((flags >> 4) & 0x7f) as u8, + flags: (flags & 0xf) as u8, + stamp: stamp >> 8, + }) + } +} + +/// How an error completion affects its VM's sticky status and other jobs. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum Class { + /// Only this work's fence fails; VM status and unrelated work remain usable. + Spared, + /// This VM owns guilty work. + Guilty, +} + +/// Queue operations under the device mutex. Completion signalling belongs to the caller after +/// that mutex is dropped. `oldest_spared` returns None if the QID has no in-flight owner. +pub(super) trait Queues { + fn render_scheduler_active(&self) -> bool; + fn render_pass_started(&self, qid: u8, stamp: u64) -> Option; + /// Copy every retained queue's live producer to its recovery target, then + /// issue a full barrier. A failed graph must not skip subsequent queues. + fn publish_recovery_targets(&self) -> Option; + fn oldest_spared(&self, qid: u8) -> Option; + fn visit_owners(&self, qid: u8, visit: &mut dyn FnMut(&Arc)); + /// Classify every in-flight completion; return QIDs of render slots with spared work. + fn classify_work(&mut self, class: &dyn Fn(&Arc) -> Class) -> u128; + fn settle_blamed_render(&mut self, qid: u8, error: Error); + fn settle_cancelled_renders(&mut self, qids: u128); + fn rescan_compute(&mut self) -> Result; + fn release_retired_compute(&mut self); +} + +/// Live shared-memory operations. The handshake word uses aligned 32-bit accesses; transitions +/// check the expected value, issue a full barrier, store the next value, then another barrier. +pub(super) trait Memory { + fn recovery_state(&self) -> Result; + fn host_recovery(&self) -> Result; + fn fault_sources(&self) -> Result; + fn transition(&self, expected: u32, next: u32) -> Result; + fn clear_timestamps(&self) -> Result; + fn entries(&self, out: &mut [Entry; MAX_ENTRIES]) -> Result; + /// One millisecond wait; this is polled because the firmware sends no state-3 notification. + fn wait_tick(&self); +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +struct Request { + generation: u64, + blamed: Option, +} + +/// Generation accepted by the host and the coalesced request from either coprocessor. +pub(super) struct State { + accepted: u64, + pending: Option, +} + +/// Non-fatal post-recovery failures. The firmware handshake has already closed successfully. +pub(super) struct ServiceResult { + pub(super) scan_error: Option, + pub(super) target_error: Option, +} + +impl State { + pub(super) const fn new() -> Self { + Self { + accepted: 0, + pending: None, + } + } + pub(super) fn generation(&self) -> u64 { + self.accepted + } + pub(super) fn pending(&self) -> bool { + self.pending.is_some() + } + + /// Returns false for a consumed duplicate from an older generation, true when the event + /// drain must stop until the current request is serviced. The first blamed QID wins. + pub(super) fn request(&mut self, generation: u64, stamp: i32) -> Result { + if generation < self.accepted { + return Ok(false); + } + if generation != self.accepted { + return Err(EIO); + } + if stamp != -1 && !(0..i32::from(QIDS)).contains(&stamp) { + return Err(EIO); + } + let blamed = if (0..i32::from(QIDS)).contains(&stamp) { + Some(stamp as u8) + } else { + None + }; + if let Some(pending) = self.pending.as_mut() { + if pending.generation != generation { + return Err(EBUSY); + } + pending.blamed = pending.blamed.or(blamed); + } else { + self.pending = Some(Request { generation, blamed }); + } + Ok(true) + } +} + +/// The firmware owner implements memory and queue access without splitting its +/// mutable borrow or duplicating the authoritative restart generation. +pub(super) trait Host: Memory + Queues { + fn recovery(&self) -> &State; + fn recovery_mut(&mut self) -> &mut State; +} + +/// Performs one pending restart. The caller has closed preparation admission and joined all +/// builders off-lock. Work remains mapped throughout every callback, including on error. +pub(super) fn service( + host: &mut H, + crashed: impl Fn() -> bool, +) -> Result { + let request = host.recovery().pending.ok_or(EINVAL)?; + if host.recovery_state()? != 1 + || request.generation != host.recovery().accepted + || host.host_recovery()? != 0 + { + return Err(EIO); + } + host.recovery_mut().accepted = host.recovery().accepted.wrapping_add(1); + if request.generation == 0 && !host.render_scheduler_active() { + host.clear_timestamps()?; + } + let mut sources = host.fault_sources()?; + let target_error = host.publish_recovery_targets(); + host.transition(1, 2)?; + let mut recovered = false; + for _ in 0..RECOVERY_POLLS { + if host.recovery_state()? == 3 { + recovered = true; + break; + } + if crashed() { + return Err(EIO); + } + host.wait_tick(); + } + if !recovered { + return Err(ETIMEDOUT); + } + let mut entries = [Entry::default(); MAX_ENTRIES]; + // A failed table read leaves only the explicit blamed QID as evidence of guilt. + let count = host.entries(&mut entries).unwrap_or(0).min(MAX_ENTRIES); + host.transition(3, 0)?; + if host.recovery_state()? != 0 { + return Err(EIO); + } + host.recovery_mut().pending = None; + + let cancelled = classify(host, &entries[..count], request.blamed, &mut sources); + if let Some(qid) = request.blamed { + host.settle_blamed_render(qid, EIO); + } + if cancelled != 0 { + host.settle_cancelled_renders(cancelled); + } + let scan_error = host.rescan_compute().err(); + host.release_retired_compute(); + Ok(ServiceResult { scan_error, target_error }) +} + +/// Resolves guilty QIDs before changing any completion's classification. Unknown ownership or +/// too many distinct guilty VMs leaves all prior classifications intact. The oldest spared work +/// of a QID makes later blame collateral to its previous recovery, rather than new VM guilt. +fn classify(queues: &mut Q, entries: &[Entry], blamed: Option, sources: &mut Sources) -> u128 { + let mut guilty_qids = blamed.map_or(0, |qid| 1u128 << qid); + let mut cancelled = 0u128; + for entry in entries { + if entry.flags & GUILTY != 0 { + guilty_qids |= 1u128 << entry.qid; + } + if entry.flags & CANCELLED != 0 { + cancelled |= 1u128 << entry.qid; + } + } + let attribution = sources.attribute(blamed, |qid, stamp| queues.render_pass_started(qid, stamp)); + match attribution { + Attribution::Firmware => {} + Attribution::Queues(qids) => guilty_qids = qids, + Attribution::NoCulprit => guilty_qids = 0, + } + let no_culprit = attribution == Attribution::NoCulprit; + if guilty_qids == 0 && !no_culprit { + return 0; + } + let mut guilty: [Option>; MAX_GUILTY_VMS] = Default::default(); + let mut count = 0; + let mut collateral = 0u128; + let mut unresolved = false; + for qid in 0..QIDS { + if guilty_qids & (1u128 << qid) == 0 { + continue; + } + match queues.oldest_spared(qid) { + None => unresolved = true, + Some(true) => collateral |= 1u128 << qid, + Some(false) => { + let mut found = false; + queues.visit_owners(qid, &mut |status| { + found = true; + if guilty[..count] + .iter() + .flatten() + .any(|known| Arc::ptr_eq(known, status)) + { + return; + } + if let Some(slot) = guilty.get_mut(count) { + *slot = Some(status.clone()); + count += 1; + } else { + unresolved = true; + } + }); + unresolved |= !found; + } + } + } + if unresolved || (count == 0 && collateral == 0 && !no_culprit) { + return 0; + } + let spared = queues.classify_work(&|status| { + guilty[..count] + .iter() + .find_map(|known| { + known + .as_ref() + .filter(|known| Arc::ptr_eq(known, status)) + .map(|_| Class::Guilty) + }) + .unwrap_or(Class::Spared) + }); + cancelled & !(guilty_qids & !collateral) & spared +} diff --git a/drivers/gpu/drm/asahi/g17/recovery/attribution.rs b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs new file mode 100644 index 0000000000000..b0c42d4e3f7c1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Resolve ambiguous recovery blame using slot identity and render execution evidence. + +pub(super) const SLOTS: usize = 16; +const STAMP_MASK: u64 = (1 << 40) - 1; +const PAGE_FAULT: u32 = 3; + +#[derive(Clone, Copy, Default)] +pub(super) struct Source { + slot: u8, + qid: Option, + stamp: u64, + monitor: u32, + started: Option, +} + +pub(crate) struct Sources { + pub(crate) reason: u32, + mask: u32, + records: [Source; SLOTS], + count: usize, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Attribution { + Firmware, + Queues(u128), + NoCulprit, +} + +impl Sources { + /// Firmware is halted for every selected key and progress-record read. + /// Missing keys stay unknown; they cannot establish execution by an owner. + pub(crate) fn sample(mask: u32, mut read: impl FnMut(usize) -> (u64, u32)) -> Self { + let mut sources = Self { + reason: 0, + mask, + records: [Source::default(); SLOTS], + count: 0, + }; + for slot in 0..SLOTS { + if mask & (1 << slot) == 0 { + continue; + } + let (key, monitor) = read(slot); + let qid = (key >> 40) & 0xff; + sources.records[sources.count] = Source { + slot: slot as u8, + qid: (key != 0 && qid < 128).then_some(qid as u8), + stamp: key & STAMP_MASK, + monitor, + started: None, + }; + sources.count += 1; + } + sources + } + + pub(super) fn attribute( + &mut self, + blamed: Option, + mut started: impl FnMut(u8, u64) -> Option, + ) -> Attribution { + for source in &mut self.records[..self.count] { + if let Some(qid) = source.qid { + source.started = started(qid, source.stamp); + } + } + if self.reason == PAGE_FAULT || self.mask.count_ones() < 2 { + return Attribution::Firmware; + } + let Some(blamed) = blamed else { + return Attribution::Firmware; + }; + let Some(source) = self.records[..self.count] + .iter() + .find(|source| source.qid == Some(blamed)) + else { + return Attribution::Firmware; + }; + if source.started != Some(false) || source.monitor != 0 { + return Attribution::Firmware; + } + let slot = source.slot; + let mut guilty = 0; + for source in &self.records[..self.count] { + if source.slot != slot && source.started == Some(true) { + if let Some(qid) = source.qid { + guilty |= 1u128 << qid; + } + } + } + if guilty == 0 { + Attribution::NoCulprit + } else { + Attribution::Queues(guilty) + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/recovery/slot.rs b/drivers/gpu/drm/asahi/g17/recovery/slot.rs new file mode 100644 index 0000000000000..307534708e922 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/recovery/slot.rs @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounded read-only slot identity sampling while firmware is halted. + +use crate::{hw::t8140, mmu}; +use core::{ + ffi::c_void, + ptr::NonNull, + sync::atomic::{AtomicU64, Ordering}, +}; +use kernel::{bindings, prelude::*}; + +struct Mapping(NonNull); + +impl Drop for Mapping { + fn drop(&mut self) { + // SAFETY: This is the successful memremap result owned exclusively by this guard. + unsafe { bindings::memunmap(self.0.as_ptr()) }; + } +} + +pub(super) fn key(slot: usize) -> Result { + if slot >= super::attribution::SLOTS { + return Err(EINVAL); + } + let physical = t8140::recovery::SLOT_KEYS + slot as u64 * size_of::() as u64; + let offset = physical as usize & (mmu::UAT_PGSZ - 1); + // SAFETY: The fixed recovery-key range is firmware RAM, page aligned here + // for mapping. Only the selected aligned eight-byte word is read below. + let raw = unsafe { + bindings::memremap( + physical - offset as u64, + mmu::UAT_PGSZ, + bindings::MEMREMAP_WB as _, + ) + }; + let mapping = Mapping(NonNull::new(raw).ok_or(ENOMEM)?); + let ctr: u64; + // SAFETY: CTR_EL0 is readable at EL1. DminLine specifies the cache maintenance granule. + unsafe { + core::arch::asm!("mrs {ctr}, ctr_el0", ctr = out(reg) ctr, + options(nomem, nostack, preserves_flags)); + } + let line = 4usize << ((ctr >> 16) & 0xf); + if line > mmu::UAT_PGSZ { + return Err(EINVAL); + } + // SAFETY: The range and offset are eight-byte aligned and lie within the mapped page. + let pointer = unsafe { + mapping + .0 + .as_ptr() + .cast::() + .add(offset) + .cast::() + }; + let mut address = pointer as usize & !(line - 1); + let end = (pointer as usize) + .checked_add(size_of::()) + .ok_or(EOVERFLOW)?; + // SAFETY: The mapping stays live and is never written by the host. The + // invalidated cache line lies within its page; barriers complete the + // invalidation before the aligned load of the selected firmware-owned word. + unsafe { + core::arch::asm!("dsb osh", options(nostack, preserves_flags)); + while address < end { + core::arch::asm!("dc ivac, {address}", address = in(reg) address, + options(nostack, preserves_flags)); + address = address.checked_add(line).ok_or(EOVERFLOW)?; + } + core::arch::asm!("dsb osh", options(nostack, preserves_flags)); + Ok((*pointer).load(Ordering::Acquire)) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs new file mode 100644 index 0000000000000..e908ca3e08e23 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -0,0 +1,869 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Device ownership of physical queues and off-lock construction reservations. +//! +//! The device mutex protects this registry. Logical queues may disappear while +//! their published graphs remain here; only exact retirement permits rebinding, +//! and only processor stop permits dropping installed firmware addresses. +//! A preparation lease keeps Shared::state populated: shutdown and loss join +//! every lease before taking the session, including builders asleep on changed. + +mod backend; +mod compute_control; +mod compute_replay; +mod compute_exit; +mod events; +mod render; +pub(super) mod teardown; + +use super::{ + buffer::{BufferIds, MetricsIds}, + context::{Context, QosIds}, + fw::queue::DataMaster, + kick, qos, + queue::{self, admission, compute}, +}; +use kernel::{prelude::*, sync::Arc}; + +const QIDS: usize = 128; +const FIRST_COMPUTE_QID: u8 = 4; +const RECOVERY_SETTLE_POLLS: usize = 200; + +/// A reservation remains identifiable while its backing is built without the +/// device mutex. Failed construction returns the slot, never its host identity. +#[derive(Clone, Copy)] +pub(super) struct ComputeReservation { + slot: usize, + pub(super) id: kick::Id, + owner: u64, +} + +struct ComputeEntry { + reservation: ComputeReservation, + queue: Option>, +} + +pub(super) struct Registry { + pub(super) qids: kick::QueueIds, + pub(super) buffers: BufferIds, + pub(super) accounting: qos::Accounting, + pub(super) metrics: Arc, + pub(super) qos_ids: Arc, + pub(super) admission: Arc, + render: render::RenderState, + identities: queue::Identities, + compute: KVec>, + pub(super) compute_building: bool, + compute_scheduler_building: bool, + pub(super) observations: compute::retirement::Batch, + worker_deferred: Option, + teardown: super::teardown::Pending, + exit_acks: compute::exit::Acks, +} + +impl Registry { + pub(super) fn new(clusters: u32, descriptor_flags: [u32; 2]) -> Result { + let mut compute = KVec::with_capacity(QIDS, GFP_KERNEL)?; + for _ in 0..QIDS { + compute.push(None, GFP_KERNEL)?; + } + let mut qids = kick::QueueIds::new(); + qids.reserve(1, DataMaster::Tiling, Some(0))?; + qids.reserve(2, DataMaster::Fragment, Some(1))?; + Ok(Self { + qids, + buffers: BufferIds::new(), + accounting: qos::Accounting::new(), + metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, + qos_ids: QosIds::new()?, + admission: admission::Pool::new()?, + render: render::RenderState::new(clusters, descriptor_flags)?, + identities: queue::Identities::new(), + compute, + compute_building: false, + compute_scheduler_building: false, + observations: compute::retirement::Batch::new()?, + worker_deferred: Some(DeferredBatch::worker()?), + teardown: super::teardown::Pending::new()?, + exit_acks: compute::exit::Acks::new(), + }) + } + + pub(super) fn compute(&self, owner: u64) -> Option<&compute::Queue> { + self.compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .find(|queue| queue.owner() == Some(owner)) + } + + pub(super) fn compute_mut(&mut self, owner: u64) -> Option<&mut compute::Queue> { + self.compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.owner() == Some(owner)) + } + + /// An installed idle queue keeps its original QID and backing. Mapping a + /// replacement context must succeed before its old binding is displaced. + pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc) -> Result { + if let Some(queue) = self.compute(owner) { + return queue.matches(owner, queue.qid(), context); + } + let Some(queue) = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.reusable()) + else { + return Ok(false); + }; + queue.bind_owner(owner, context.clone())?; + Ok(true) + } + + pub(super) fn reserve_compute(&mut self, owner: u64) -> Result { + if owner == 0 + || self.compute.iter().flatten().any(|entry| { + entry + .queue + .as_ref() + .map_or(entry.reservation.owner == owner, |queue| { + queue.owner() == Some(owner) + }) + }) + { + return Err(EINVAL); + } + let slot = self + .compute + .iter() + .position(Option::is_none) + .ok_or(ENOSPC)?; + let identity = self.identities.reserve(1)?.start; + let id = self + .qids + .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID))?; + let reservation = ComputeReservation { slot, id, owner }; + self.compute[slot] = Some(ComputeEntry { + reservation, + queue: None, + }); + Ok(reservation) + } + + fn reserved(&mut self, reservation: ComputeReservation) -> Result<&mut ComputeEntry> { + let entry = self + .compute + .get_mut(reservation.slot) + .and_then(Option::as_mut) + .ok_or(EIO)?; + if entry.reservation.id != reservation.id + || entry.reservation.owner != reservation.owner + || entry.queue.is_some() + { + return Err(EIO); + } + Ok(entry) + } + + /// The caller checked its preparation epoch and retained the same device + /// session before placing this graph where publication can find it. + pub(super) fn install_compute( + &mut self, + reservation: ComputeReservation, + queue: &mut Option>, + ) -> Result { + let built = queue.as_ref().ok_or(EINVAL)?; + if built.qid() != reservation.id.qid() || built.owner() != Some(reservation.owner) { + return Err(EIO); + } + let entry = self.reserved(reservation)?; + entry.queue = queue.take(); + Ok(()) + } + + pub(super) fn cancel_compute(&mut self, reservation: ComputeReservation) -> Result { + self.reserved(reservation)?; + self.qids.cancel_unpublished(reservation.id)?; + self.compute[reservation.slot] = None; + Ok(()) + } + + pub(super) fn release_compute(&mut self, owner: u64) -> Result { + let Some(slot) = self.compute.iter().position(|entry| { + entry + .as_ref() + .and_then(|entry| entry.queue.as_ref()) + .is_some_and(|queue| queue.owner() == Some(owner)) + }) else { + return Ok(()); + }; + let entry = self.compute[slot].as_mut().ok_or(EIO)?; + if entry.queue.as_mut().ok_or(EIO)?.release_owner()? { + self.qids.cancel_unpublished(entry.reservation.id)?; + self.compute[slot] = None; + } + Ok(()) + } +} + +/// An immutable logical owner shared by its two scheduler entities. Installed +/// physical graphs retain the context independently of this frontend lifetime. +pub(super) struct Backend { + shared: Arc, + pub(super) context: Arc, + owner: u64, + backlog: core::sync::atomic::AtomicU32, +} + +impl Backend { + pub(super) fn new( + shared: &Arc, + vm: &crate::mmu::Vm, + owner: u64, + ) -> Result<(Arc, admission::QueueSlot)> { + let lease = shared.preparations.enter(true)?; + let (uat, pool, qos, execution, slot) = { + let mut state = shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if shared.crashed.load(core::sync::atomic::Ordering::Acquire) { + return Err(EIO); + } + let slot = firmware.queues.admission.reserve_queue()?; + let execution = firmware.uat.new_execution_context(vm)?; + ( + firmware.uat.clone(), + firmware.pool.clone(), + firmware.queues.qos_ids.clone(), + execution, + slot, + ) + }; + let alloc = super::object::Allocator { + dev: &shared.dev, + uat: &uat, + }; + let owner_pid = + u32::try_from(kernel::current!().group_leader().pid()).map_err(|_| EOVERFLOW)?; + let context = Arc::new( + Context::new(&alloc, &pool, &qos, execution, owner_pid)?, + GFP_KERNEL, + )?; + let state = shared.state.lock(); + if !lease.is_current() || state.is_none() { + return Err(EAGAIN); + } + drop(state); + Ok(( + Arc::new( + Self { + shared: shared.clone(), + context, + owner, + backlog: core::sync::atomic::AtomicU32::new(0), + }, + GFP_KERNEL, + )?, + slot, + )) + } + + /// The shared aperture has exactly one builder. Other queues sleep without + /// holding the device mutex; recovery joins its lease before closing roots. + fn shared_compute(&self) -> Result { + loop { + let lease = self.shared.preparations.enter(false)?; + let uat = { + let mut state = self.shared.state.lock(); + loop { + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if self + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Err(EIO); + } + if !lease.is_current() { + return Err(EAGAIN); + } + if firmware.init.has_compute_shared() { + return firmware.init.cache_compute_vm(self.context.vm()); + } + if !firmware.queues.compute_building { + firmware.queues.compute_building = true; + break firmware.uat.clone(); + } + self.shared.changed.wait(&mut state); + } + }; + let alloc = super::object::Allocator { + dev: &self.shared.dev, + uat: &uat, + }; + let mut built = Some(super::initdata::ComputeShared::new(&alloc)); + let result = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + Err(EAGAIN) + } else { + built + .take() + .ok_or(EIO)? + .and_then(|built| firmware.init.install_compute_shared(built)) + .and_then(|()| firmware.init.cache_compute_vm(self.context.vm())) + } + }; + drop(built); + { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .compute_building = false; + } + self.shared.changed.notify_all(); + drop(lease); + if result != Err(EAGAIN) { + return result; + } + } + } + + /// Uses the queue builder's existing lease. Entering another closed epoch + /// here would make recovery wait for the very builder blocked on reopening. + fn compute_scheduler( + &self, + alloc: &super::object::Allocator<'_>, + lease: &super::preparation::Lease, + ) -> Result { + { + let mut state = self.shared.state.lock(); + loop { + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + if firmware.init.has_compute_scheduler() { + return Ok(()); + } + if !firmware.queues.compute_scheduler_building { + firmware.queues.compute_scheduler_building = true; + break; + } + self.shared.changed.wait(&mut state); + } + } + let mut built = Some(super::initdata::ComputeScheduler::new(alloc)); + let result = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + Err(EAGAIN) + } else { + built + .take() + .ok_or(EIO)? + .and_then(|built| firmware.init.install_compute_scheduler(built)) + } + }; + drop(built); + { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .compute_scheduler_building = false; + } + self.shared.changed.notify_all(); + result + } + + fn build_compute(&self) -> Result { + loop { + match self.shared_compute() { + Err(error) if error == EAGAIN => continue, + result => { + result?; + break; + } + } + } + loop { + let lease = self.shared.preparations.enter(false)?; + let (uat, reservation) = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if self + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Err(EIO); + } + if firmware.queues.rebind_compute(self.owner, &self.context)? { + return Ok(()); + } + let reservation = firmware.queues.reserve_compute(self.owner)?; + (firmware.uat.clone(), reservation) + }; + let alloc = super::object::Allocator { + dev: &self.shared.dev, + uat: &uat, + }; + let built = (|| { + let pool = compute::PoolBacking::new(&alloc, reservation.id)?; + { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + firmware.queues.reserved(reservation)?; + let (slot, page_list) = pool.pool_descriptor(); + firmware.init.initialize_compute_pool(slot, page_list)?; + } + let backing = compute::Backing::new(&alloc, pool)?; + // The global scheduler is created after pool/kick backing and + // retained even if this queue's later descriptor graph fails. + self.compute_scheduler(&alloc, &lease)?; + let queue = compute::Queue::new(&alloc, backing, self.owner, self.context.clone())?; + Ok(KBox::new(queue, GFP_KERNEL)?) + })(); + let (mut built, build_error) = match built { + Ok(queue) => (Some(queue), None), + Err(error) => (None, Some(error)), + }; + let result = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + Err(EAGAIN) + } else if let Some(error) = build_error { + Err(error) + } else { + firmware.queues.install_compute(reservation, &mut built) + } + }; + // The old aliases must disappear before a waiting constructor can + // reuse the QID and its queue-local free-list identity. Keep the + // preparation lease until both mappings and reservation are gone. + drop(built); + if result.is_err() { + let mut state = self.shared.state.lock(); + (*state) + .as_mut() + .ok_or(ENODEV)? + .queues + .cancel_compute(reservation)?; + } + drop(lease); + if result != Err(EAGAIN) { + return result; + } + } + } +} + +/// One device transaction can fail each retained kick once and then retire it. +/// The physical namespace bounds the batch independently of open logical queues. +/// Collection failure leaves the engine's Active owner intact for a stopped retry. +pub(super) struct DeferredBatch { + values: KVVec, +} +impl DeferredBatch { + const DEPTH: usize = if crate::hw::t8140::queues::RENDER_DEPTH as usize + > crate::hw::t8140::queues::COMPUTE_DEPTH + { + crate::hw::t8140::queues::RENDER_DEPTH as usize + } else { + crate::hw::t8140::queues::COMPUTE_DEPTH + }; + + /// Reused by the sole event work item; no allocation on a completion pass. + fn worker() -> Result { + Self::with_capacity(2 * QIDS * Self::DEPTH) + } + /// One publication can quarantine only its selected physical queue/pair. + /// VM-wide settlement is a separate transaction before publishing its error. + fn publication() -> Result { + Self::with_capacity(2 * Self::DEPTH) + } + fn with_capacity(capacity: usize) -> Result { + Ok(Self { + values: KVVec::with_capacity(capacity, GFP_KERNEL)?, + }) + } + pub(super) fn push(&mut self, value: super::completion::Deferred) -> Result { + use super::completion::Deferred; + // Publishers may acquire the device mutex as soon as this pass drops + // it. Expose terminal VM status now, before off-lock fence callbacks. + // Recovery-spared and cancelled commands never poison their VM. + if let Deferred::FailedOwned(completion, error) = &value { + if *error != ECANCELED && !completion.spared() { + completion.status().record(*error); + } + } + self.values.push_within_capacity(value).map_err(|_| EIO) + } + /// Called only after releasing the device mutex. + pub(super) fn finish(&mut self) { + for value in self.values.drain_all() { + value.finish(); + } + } +} + +struct ComputeHost<'a> { + init: &'a super::initdata::InitData, + primary: &'a mut super::Coprocessor, + effort: &'a mut super::power::Effort, + qids: &'a mut kick::QueueIds, + accounting: &'a mut qos::Accounting, + generation: u64, +} +impl compute::Host for ComputeHost<'_> { + fn epoch(&self) -> Result<(u64, u32)> { + Ok(( + self.generation, + super::recovery::Memory::recovery_state(self.init)?, + )) + } + fn prepare_compute_shared(&mut self) -> Result { + self.init.activate_compute() + } + fn qos_publish(&mut self, owner: qos::Owner, scheduler: u64) -> Result { + self.accounting.publish( + &qos::View::new(self.init.qos()?)?, + owner, + scheduler, + qos::clock, + ) + } + fn qos_cancel(&mut self, publication: qos::Publication) -> Result { + self.accounting + .cancel(&qos::View::new(self.init.qos()?)?, publication) + } + fn publish_qid(&mut self, id: kick::Id) -> Result { + self.qids.publish(id) + } + fn publish_outer(&mut self, slot: &super::fw::channels::WorkSlot) -> Result { + self.init.publish_work(slot).map(|_| ()) + } + fn notify(&mut self, message: u64) -> Result { + self.primary.notify(message) + } + fn note_submission(&mut self) -> Result { + self.effort.submit(); + super::Firmware::apply_shared_effort(self.init, self.effort, super::power::FULL_EFFORT) + } +} + +impl super::Firmware { + fn publish_compute( + &mut self, + owner: u64, + packet: Arc, + dependencies: &[super::fw::kick::KickDependency], + deferred: &mut DeferredBatch, + ) -> Result { + if self.recovery.pending() { + return Err(EAGAIN); + } + let queues = &mut self.queues; + let queue = queues + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.owner() == Some(owner)) + .ok_or(EIO)?; + let mut host = ComputeHost { + init: &self.init, + primary: &mut self.primary, + effort: &mut self.effort, + qids: &mut queues.qids, + accounting: &mut queues.accounting, + generation: self.recovery.generation(), + }; + queue.publish(packet, dependencies, &mut host, &mut |event| { + deferred.push(event) + }) + } +} + +impl Registry { + fn release_compute_witness( + &mut self, + init: &super::initdata::InitData, + owner: u64, + deferred: &mut DeferredBatch, + ) -> Result { + let Some(queue) = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.owner() == Some(owner)) + else { + return Ok(false); + }; + if !queue.spared_pending() { + return Ok(false); + } + let view = qos::View::new(init.qos()?)?; + queue.release_witnessed( + super::recovery::Memory::recovery_state(init)?, + |owner| self.accounting.complete(&view, &[owner]), + &mut |event| deferred.push(event), + )?; + Ok(queue.spared_pending()) + } + + fn fail_compute_vm( + &mut self, + status: &Arc, + error: Error, + deferred: &mut DeferredBatch, + ) -> Result { + for queue in self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + if queue.in_flight() && Arc::ptr_eq(queue.context().status(), status) { + queue.quarantine(error, &mut |event| deferred.push(event))?; + } + } + Ok(()) + } +} + +impl super::Firmware { + /// Latch one shared snapshot for all selected queues before settling any VM. + /// A later queue's exact success in this batch wins over a peer's failure. + fn scan_compute(&mut self, masks: Option<[u64; 2]>, deferred: &mut DeferredBatch) -> Result { + let state = super::recovery::Memory::recovery_state(&self.init)?; + let generation = self.recovery.generation(); + let registry = &mut self.queues; + registry.observations.clear(); + let mut requested = false; + for queue in registry + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + { + requested |= queue.request_observations(&mut registry.observations, masks)?; + } + if requested { + self.init + .scan_compute_completions(|record| registry.observations.fold(record))?; + } + let view = qos::View::new(self.init.qos()?)?; + for queue in registry + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + queue.latch_observations(&mut registry.observations, masks)?; + queue.retire( + masks, + generation, + state, + |owner| registry.accounting.complete(&view, &[owner]), + &mut |event| deferred.push(event), + )?; + } + // All compute observations in this batch are latched before a VM + // failure can settle its render peers. Exact successes win first. + for index in 0..self.queues.compute.len() { + let terminal = self.queues.compute[index] + .as_mut() + .and_then(|entry| entry.queue.as_deref_mut()) + .and_then(compute::Queue::take_terminal); + if let Some((status, error)) = terminal { + status.record(error); + self.settle_render_vm(&status, error, deferred)?; + self.queues.fail_compute_vm(&status, error, deferred)?; + } + } + let view = qos::View::new(self.init.qos()?)?; + let registry = &mut self.queues; + for queue in registry + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + queue.release_witnessed( + state, + |owner| registry.accounting.complete(&view, &[owner]), + &mut |event| deferred.push(event), + )?; + } + Ok(()) + } +} + +impl Backend { + fn ensure_compute(&self) -> Result { + use kernel::time::{delay::fsleep, Delta}; + const SPARED_WAIT_MS: usize = 200; + let mut deferred = DeferredBatch::publication()?; + for _ in 0..SPARED_WAIT_MS { + let pending = (|| { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + firmware.replay_compute(&mut deferred)?; + firmware + .queues + .release_compute_witness(&firmware.init, self.owner, &mut deferred) + })(); + deferred.finish(); + let pending = pending?; + if !pending { + break; + } + // Other owners continue submitting while this owner gives its + // queue-local drain witness time to arrive. + fsleep(Delta::from_millis(1)); + } + self.build_compute() + } + + /// Called after taking this packet's publication claim. All waits and work + /// state allocation precede the device mutex; only the checked epoch commits. + fn compute_packet( + &self, + packet: &Arc, + deferred: &mut DeferredBatch, + ) -> Result { + packet.check_dependencies()?; + let mut recovery_polls = 0; + loop { + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let lease = self.shared.preparations.enter(false)?; + packet.completion.work_state().acquire()?; + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + drop(state); + continue; + } + if packet.cancel_requested() { + return Err(ECANCELED); + } + if self + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Err(EIO); + } + let recovery_state = super::recovery::Memory::recovery_state(&firmware.init)?; + if firmware.recovery.pending() || recovery_state != 0 { + if recovery_polls == RECOVERY_SETTLE_POLLS { + return Err(EBUSY); + } + recovery_polls += 1; + drop(state); + drop(lease); + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + continue; + } + firmware.replay_compute(deferred)?; + let dependencies = firmware.queues.render_prefix(self.owner, packet)?; + let result = firmware.publish_compute( + self.owner, + packet.clone(), + dependencies.as_slice(), + deferred, + ); + if matches!(result, Err(EBUSY) | Err(EAGAIN)) + && !packet.is_published() + && (firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0) + { + if recovery_polls == RECOVERY_SETTLE_POLLS { + return Err(EBUSY); + } + recovery_polls += 1; + drop(state); + drop(lease); + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + continue; + } + return result; + } + } +} + +impl super::Firmware { + /// Both processors are stopped and their owner detached before this call. + /// Error fences release their command mappings without the device mutex + /// or an allocation-dependent collector. + fn finish_stopped_queues(&mut self) { + self.stop_render(); + for queue in self + .queues + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + let result = queue.stopped(&mut |event| { + event.finish(); + Ok(()) + }); + if let Err(error) = result { + dev_err!( + self.primary.state.shared.dev.as_ref(), + "Could not settle stopped compute queue: {:?}\n", + error + ); + } + } + self.queues + .admission + .release_retained_owners_after_processor_stop(); + } +} + +impl super::Shared { + /// Joins construction and serializes the stop writes with detachment. + /// After installation, seeing no owner proves that an earlier stop caller + /// has already stopped both processors. Callbacks and destruction follow + /// outside the device mutex. + pub(super) fn stop_queues(&self) { + self.preparations.wait_drained(); + let firmware = { + let mut state = self.state.lock(); + if let Some(firmware) = (*state).as_deref_mut() { + firmware.secondary.stop(); + firmware.primary.stop(); + } + state.take() + }; + if let Some(mut firmware) = firmware { + firmware.finish_stopped_queues(); + } + } +} + +/// One monotonic domain for recovery grace, idle effort and control retirement. +fn now_ns() -> u64 { + ::ktime_get() as u64 +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs new file mode 100644 index 0000000000000..f1ae1ffe80783 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Scheduler callbacks and the off-lock fence boundary of a logical queue. + +use super::{Backend, DeferredBatch}; +use crate::g17::{ + completion::Deferred, + job::{self, Engine, Packet}, +}; +use core::sync::atomic::Ordering; +use kernel::{dma_fence::Fence, drm::sched, prelude::*, sync::Arc}; + +impl Backend { + pub(super) fn release_backlog(&self, packet: &Packet) { + if packet.take_backlog() { + self.backlog.fetch_sub(1, Ordering::AcqRel); + } + } + + fn stop_after_error(&self) { + self.shared.lose_device_quietly(); + self.shared.stop_queues(); + } + + /// Complete all failure fanout before signalling this packet's unpublished + /// error. Visible failures keep the engine's retained owner through a witness. + pub(super) fn finish_publication( + &self, + packet: &Arc, + result: Result, + deferred: &mut DeferredBatch, + ) -> Result { + if let Err(error) = result { + if [EIO, ETIMEDOUT, ENODEV].contains(&error) || packet.completion.status().get() != 0 { + ::fail_vm(self, error); + } + deferred.finish(); + if !packet.is_published() { + packet.completion.fail_unpublished(error); + } + Err(error) + } else { + deferred.finish(); + Ok(packet.completion.fence()) + } + } + + fn release_queue(&self) -> Result { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + // Retain a published scheduler before a never-kicked physical graph is + // removed. The teardown entry owns its root until control consumption. + firmware + .queues + .teardown + .close(&self.context, super::now_ns())?; + firmware.queues.release_compute(self.owner)?; + firmware.queues.release_render(self.owner)?; + firmware.queues.admission.release_owner(self.owner); + drop(state); + self.shared.queue_events(); + Ok(()) + } +} + +impl job::Backend for Backend { + fn ensure_compute(&self) -> Result { + Backend::ensure_compute(self) + } + + fn fail_vm(&self, error: Error) { + self.context.status().record(error); + let mut deferred = match DeferredBatch::worker() { + Ok(batch) => batch, + Err(_) => { + self.stop_after_error(); + return; + } + }; + let result = (|| { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + // Final observations win over this VM's terminal fanout. + firmware.scan_compute(None, &mut deferred)?; + firmware.settle_render_vm(self.context.status(), error, &mut deferred)?; + firmware + .queues + .fail_compute_vm(self.context.status(), error, &mut deferred) + })(); + deferred.finish(); + if result.is_err() { + self.stop_after_error(); + } + } + + fn release_owner(&self) { + if self.release_queue().is_err() { + self.stop_after_error(); + } + } + + /// Firmware kick parents order the same engine. Explicit cross-engine and + /// input dependencies were already attached to the DRM scheduled job. + fn prepare(&self, _packet: &Packet) -> Option { + None + } + + fn publish(&self, packet: Arc) -> Result { + if let Some(result) = packet.early_result() { + return result.map(|()| packet.completion.fence()); + } + let mut deferred = match DeferredBatch::publication() { + Ok(batch) => batch, + Err(error) => { + self.release_backlog(&packet); + if packet.completion.status().get() != 0 { + ::fail_vm(self, error); + } + packet.completion.fail_unpublished(error); + return Err(error); + } + }; + let result = if !packet.begin_publication() { + Err(ECANCELED) + } else if packet.completion.status().get() != 0 { + Err(EIO) + } else { + match packet.engine() { + Engine::Compute => self.compute_packet(&packet, &mut deferred), + Engine::Render => self.render_packet(&packet, &mut deferred), + } + }; + self.release_backlog(&packet); + self.finish_publication(&packet, result, &mut deferred) + } + + fn timed_out(&self, packet: Arc) -> sched::Status { + self.release_backlog(&packet); + let result = match packet.engine() { + Engine::Compute => self.timeout_compute(&packet), + Engine::Render => self.timeout_render(&packet), + }; + if result.is_err() { + self.stop_after_error(); + } + sched::Status::NoHang + } + + fn cancel(&self, packet: Arc) { + self.release_backlog(&packet); + if packet.request_cancel() { + Deferred::Unpublished(packet.completion.clone(), ECANCELED).finish(); + return; + } + let result = match packet.engine() { + Engine::Compute => self.cancel_compute(&packet), + Engine::Render => self.cancel_render(&packet), + }; + if result.is_err() { + self.stop_after_error(); + } + } + + fn defer_timeout(&self, packet: &Arc) -> bool { + if packet.engine() != Engine::Render { + return false; + } + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .is_some_and(|firmware| firmware.queues.defer_render_timeout(packet, ETIMEDOUT)) + } + + fn try_early(&self, packet: &Arc) -> bool { + self.try_render_early(packet) + } + + fn count_backlog(&self, packet: &Packet) { + packet.count_backlog(); + self.backlog.fetch_add(1, Ordering::AcqRel); + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs new file mode 100644 index 0000000000000..610643ef6e379 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Scheduler cancellation and final retirement checks for retained compute work. + +use super::{Backend, DeferredBatch}; +use crate::g17::job::{self, Packet}; +use kernel::{prelude::*, sync::Arc}; + +impl Backend { + /// The caller owns cancellation of a packet that never claimed publication. + /// A missing retained packet may already have a healthy callback waiting off-lock. + pub(super) fn timeout_compute(&self, packet: &Arc) -> Result { + let mut deferred = DeferredBatch::worker()?; + let result = (|| { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + // Teardown owns the detached queues and will settle their callbacks. + return Ok(None); + }; + firmware.scan_compute(None, &mut deferred)?; + let Some(queue) = firmware.queues.compute_mut(self.owner) else { + return Ok(None); + }; + if !queue.owns_packet(packet) { + return Ok(None); + } + queue.timeout(&mut |event| deferred.push(event)) + })(); + if result.as_ref().is_ok_and(|status| status.is_some()) { + // VM status and peer settlement precede signalling this timeout. The backend + // owns the same logical VM as the retained packet matched under the mutex. + ::fail_vm(self, ETIMEDOUT); + } + deferred.finish(); + result.map(|_| ()) + } + + /// Cancel only a scheduler waiter. Visible mappings stay pinned until their witness; + /// cancellation neither quarantines hardware admission nor fans out a VM failure. + pub(super) fn cancel_compute(&self, packet: &Arc) -> Result { + let mut deferred = DeferredBatch::worker()?; + let result = (|| { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + firmware.scan_compute(None, &mut deferred)?; + if let Some(queue) = firmware.queues.compute_mut(self.owner) { + queue.cancel(packet, &mut |event| deferred.push(event))?; + } + Ok(()) + })(); + deferred.finish(); + result + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs new file mode 100644 index 0000000000000..0643da70d72ca --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Process-exit controls for closed contexts whose compute commands remain owned. + +use super::{now_ns, DeferredBatch}; +use crate::g17::{ + channel::Rings, + context::Context, + fw::channels::{ContextKill, ControlRecord}, + queue::compute::exit, + recovery::Memory as _, + Firmware, MSG_CONTROL_NOTIFY, +}; +use core::sync::atomic::{fence, Ordering}; +use kernel::{prelude::*, sync::Arc}; + +impl Firmware { + pub(in crate::g17) fn service_compute_exit(&mut self, deferred: &mut DeferredBatch) -> Result { + let recovery_closed = self.init.recovery_state()? == 0; + for entry in self.queues.compute.iter_mut().flatten() { + if let Some(queue) = entry.queue.as_deref_mut() { + queue.settle_exit(&self.queues.exit_acks, recovery_closed, &mut |event| { + deferred.push(event) + })?; + } + } + for cookie in self.queues.exit_acks.snapshot() { + if cookie != 0 + && !self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.exit_cookie() == Some(cookie)) + { + self.queues.exit_acks.forget(cookie); + } + } + if self.recovery.pending() || !recovery_closed { + return Ok(()); + } + let now = now_ns(); + for index in 0..self.queues.teardown.len() { + let Some((context, age)) = self.queues.teardown.closed(index, now) else { + continue; + }; + if !self.queues.render.teardown_idle(&context)? + || !self.queues.render.teardown_pools_released(&context) + { + continue; + } + if let Err(error) = self.publish_compute_exit(&context, age, now) { + dev_err!( + self.primary.state.shared.dev.as_ref(), + "Scheduler {} exit request failed: {:?}\n", + context.id(), + error + ); + } + } + Ok(()) + } + + fn publish_compute_exit(&mut self, context: &Arc, age: u64, now: u64) -> Result { + let Some(index) = self.queues.compute.iter().position(|entry| { + entry + .as_ref() + .and_then(|entry| entry.queue.as_deref()) + .is_some_and(|queue| queue.exit_candidate(context, age)) + }) else { + return Ok(()); + }; + if self.render_control_backpressured() { + return Ok(()); + } + let queue = self.queues.compute[index] + .as_ref() + .and_then(|entry| entry.queue.as_deref()) + .ok_or(EIO)?; + let Some(owner) = queue.exit_owner(context) else { + return Ok(()); + }; + let target = queue.qid(); + let (mut own, mut foreign, slots) = self.queues.render.exit_scope(context); + for entry in self.queues.compute.iter().flatten() { + if let Some(queue) = entry.queue.as_deref() { + let bit = 1u128 << queue.qid(); + if Arc::ptr_eq(queue.context(), context) { + own |= bit; + } else { + foreign |= bit; + } + } + } + let state = exit::State::read(context)?; + let Some(qids) = state.scope(target, own, foreign) else { + return Ok(()); + }; + let mut hold = 0u64; + for slot in 0..crate::hw::t8140::queues::RENDER_SLOTS as u8 { + if slots & (1u64 << slot) != 0 && self.queues.render.qids_for_slot(slot) & qids != 0 { + hold |= 1u64 << slot; + } + } + if hold != 0 + && !self + .queues + .admission + .hold_render_slots_for_exit_kill(owner, hold) + { + return Ok(()); + } + let cookie = exit::next_cookie(); + let record = ControlRecord::ContextKill(ContextKill::new(context.scheduler_va(), cookie)); + if let Err(error) = Rings::publish_control(&self.init, &record) { + self.queues.admission.release_exit_kill_hold(hold); + return if error == EAGAIN { Ok(()) } else { Err(error) }; + } + // Publication is irrevocable even if its doorbell fails. Keep both + // the killed graph and the held render identities until processor stop. + self.queues.compute[index] + .as_mut() + .and_then(|entry| entry.queue.as_deref_mut()) + .ok_or(EIO)? + .record_exit(cookie, now); + fence(Ordering::SeqCst); + if let Err(error) = self.primary.notify(MSG_CONTROL_NOTIFY) { + dev_warn!( + self.primary.state.shared.dev.as_ref(), + "Scheduler {} exit announcement failed: {:?}\n", + context.id(), + error + ); + } + Ok(()) + } + + pub(in crate::g17) fn compute_exit_polling(&self, now: u64) -> bool { + if self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.exit_polling(now)) + { + return true; + } + (0..self.queues.teardown.len()).any(|index| { + self.queues + .teardown + .closed(index, now) + .is_some_and(|(context, age)| { + age < exit::POLL_NS + && self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.exit_candidate(&context, u64::MAX)) + }) + }) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs new file mode 100644 index 0000000000000..17f228b9d1f04 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Ordered republication of witnessed, unexecuted compute commands. + +use super::DeferredBatch; +use crate::g17::{completion::Deferred, qos, recovery, Firmware}; +use kernel::prelude::*; + +impl Firmware { + /// Called only from an outer worker or client submission, never from event + /// dispatch or a recovery callback nested within publication. All candidate + /// queues are observed before any retained packet is published again. + pub(in crate::g17) fn replay_compute(&mut self, deferred: &mut DeferredBatch) -> Result { + if self.recovery.pending() { + return Ok(()); + } + let state = recovery::Memory::recovery_state(&self.init)?; + if state != 0 { + return Ok(()); + } + let view = qos::View::new(self.init.qos()?)?; + for entry in self.queues.compute.iter_mut().flatten() { + if let Some(queue) = entry.queue.as_deref_mut() { + queue.prepare_replays( + state, + |owner| self.queues.accounting.complete(&view, &[owner]), + &mut |event| deferred.push(event), + )?; + } + } + for index in 0..self.queues.compute.len() { + loop { + if self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0 { + return Ok(()); + } + let Some(queue) = self.queues.compute[index] + .as_ref() + .and_then(|entry| entry.queue.as_deref()) + else { + break; + }; + let Some(packet) = queue.replay_front() else { + break; + }; + let owner = queue.owner(); + + let result = (|| { + self.queues.compute[index] + .as_mut() + .and_then(|entry| entry.queue.as_deref_mut()) + .ok_or(EIO)? + .prepare_replay(&packet)?; + let owner = owner.ok_or(EIO)?; + packet.check_dependencies()?; + let dependencies = self.queues.render_prefix(owner, &packet)?; + self.publish_compute(owner, packet.clone(), dependencies.as_slice(), deferred) + })(); + let recovery_wait = matches!(result, Err(EAGAIN) | Err(EBUSY)) + && (self.recovery.pending() + || recovery::Memory::recovery_state(&self.init)? != 0); + let queue = self.queues.compute[index] + .as_mut() + .and_then(|entry| entry.queue.as_deref_mut()) + .ok_or(EIO)?; + if let Err(error) = result { + if recovery_wait && !queue.owns_packet(&packet) { + // Keep this prepared attempt and its one timeout credit + // while the outer worker closes the new recovery. + return Ok(()); + } + queue.fail_replays(error); + if !queue.owns_packet(&packet) { + deferred + .push(Deferred::Retired(packet.completion.clone(), Err(ENODATA)))?; + } + } + queue.finish_replay(&packet)?; + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs new file mode 100644 index 0000000000000..e9fc977d623b8 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -0,0 +1,419 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Event and restart callbacks borrow the complete firmware owner. The reusable +//! completion collector belongs to the worker and is drained outside its mutex. + +use super::DeferredBatch; +use crate::g17::{ + channel, event, + fw::channels::{Event, FreeListCompletion, TvbGrowRequest}, + recovery, + status::VmStatus, + Firmware, Role, +}; +use kernel::{prelude::*, sync::Arc}; + +struct Service<'a> { + firmware: &'a mut Firmware, + deferred: &'a mut DeferredBatch, + render_masks: [u64; 2], + settlement_error: Option, +} + +impl event::Transport for Service<'_> { + fn trace(&mut self, role: Role) { + let firmware = &mut self.firmware; + firmware + .rings + .drain_trace(&firmware.primary.state.shared.dev, &firmware.init, role); + } + fn statistics(&mut self, role: Role) { + let firmware = &mut self.firmware; + firmware + .rings + .drain_statistics(&firmware.primary.state.shared.dev, &firmware.init, role); + } + fn drain( + &mut self, + role: Role, + mut consume: impl FnMut(&mut Self, Event) -> Result, + ) -> Result { + let mut events = channel::Events::new(role); + while let Some(event) = + events.next(&self.firmware.primary.state.shared.dev, &self.firmware.init)? + { + if !consume(self, event)? { + break; + } + } + Ok(()) + } + fn recovery_state(&self) -> Result { + recovery::Memory::recovery_state(&self.firmware.init) + } +} + +impl event::Host for Service<'_> { + fn recovery(&self) -> &recovery::State { + &self.firmware.recovery + } + fn recovery_mut(&mut self) -> &mut recovery::State { + &mut self.firmware.recovery + } +} + +impl event::Queues for Service<'_> { + fn context_killed(&mut self, cookie: u64) { + self.firmware.queues.exit_acks.note(cookie); + } + fn flush_tvb_reply(&mut self) -> Result { + self.firmware.flush_tvb_reply() + } + fn flush_freelist_release(&mut self) -> Result { + self.firmware.flush_freelist_release() + } + fn observe_render_releases(&mut self) -> Result { + self.firmware.observe_render_releases() + } + fn control_backpressured(&self) -> bool { + self.firmware.render_control_backpressured() + } + fn queue_tvb_grow(&mut self, request: TvbGrowRequest) -> Result { + self.firmware.queue_tvb_grow(request) + } + fn complete_freelist_grow(&mut self, completion: FreeListCompletion) -> Result { + self.firmware.complete_freelist_grow(completion) + } + fn retain_render_completions(&mut self, masks: [u64; 2]) { + // Inline timeout and drain callers may discard the worker hints. The + // queue owner retains stamps until its borrowed pair returns to scan. + self.firmware.queues.retain_render_completions(masks); + self.render_masks[0] |= masks[0]; + self.render_masks[1] |= masks[1]; + } + fn complete_compute(&mut self, masks: [u64; 2]) -> Result { + self.firmware.scan_compute(Some(masks), self.deferred) + } +} + +impl recovery::Host for Service<'_> { + fn recovery(&self) -> &recovery::State { + &self.firmware.recovery + } + fn recovery_mut(&mut self) -> &mut recovery::State { + &mut self.firmware.recovery + } +} + +impl recovery::Memory for Service<'_> { + fn recovery_state(&self) -> Result { + self.firmware.init.recovery_state() + } + fn host_recovery(&self) -> Result { + self.firmware.init.host_recovery() + } + fn fault_sources(&self) -> Result { + self.firmware.init.fault_sources() + } + + fn transition(&self, expected: u32, next: u32) -> Result { + self.firmware.init.transition(expected, next) + } + fn clear_timestamps(&self) -> Result { + self.firmware.init.clear_timestamps() + } + fn entries(&self, out: &mut [recovery::Entry; recovery::MAX_ENTRIES]) -> Result { + self.firmware.init.entries(out) + } + fn wait_tick(&self) { + self.firmware.init.wait_tick(); + } +} + +impl recovery::Queues for Service<'_> { + fn render_pass_started(&self, qid: u8, stamp: u64) -> Option { + self.firmware.queues.render_pass_started(qid, stamp) + } + fn publish_recovery_targets(&self) -> Option { + let mut failure = self.firmware.queues.publish_render_recovery_targets(); + for queue in self + .firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + { + if let Err(error) = queue.publish_recovery_target() { + failure = failure.or(Some(error)); + } + } + core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); + failure + } + fn render_scheduler_active(&self) -> bool { + self.firmware.queues.render_scheduler_active() || !self.firmware.queues.teardown.is_empty() + } + fn oldest_spared(&self, qid: u8) -> Option { + self.firmware.queues.render_oldest_spared(qid).or_else(|| { + self.firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .find(|queue| queue.qid() == qid) + .and_then(|queue| queue.oldest_spared()) + }) + } + fn visit_owners(&self, qid: u8, visit: &mut dyn FnMut(&Arc)) { + self.firmware.queues.visit_render_owners(qid, visit); + for queue in self + .firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + { + if queue.qid() == qid { + queue.visit_owners(visit); + } + } + } + fn classify_work(&mut self, class: &dyn Fn(&Arc) -> recovery::Class) -> u128 { + for queue in self + .firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + { + queue.classify(class); + } + self.firmware.queues.classify_render(class) + } + fn settle_blamed_render(&mut self, qid: u8, error: Error) { + let result = self + .firmware + .settle_blamed_render(qid, error, self.deferred); + self.settlement_error = self.settlement_error.or(result.err()); + } + fn settle_cancelled_renders(&mut self, qids: u128) { + let result = self.firmware.settle_cancelled_renders(qids, self.deferred); + self.settlement_error = self.settlement_error.or(result.err()); + } + fn rescan_compute(&mut self) -> Result { + self.firmware.scan_compute(None, self.deferred) + } + fn release_retired_compute(&mut self) { + let result = self.firmware.release_compute_witnesses(self.deferred); + self.settlement_error = self.settlement_error.or(result.err()); + } +} + +impl Firmware { + fn release_compute_witnesses(&mut self, deferred: &mut DeferredBatch) -> Result { + let state = recovery::Memory::recovery_state(&self.init)?; + let view = crate::g17::qos::View::new(self.init.qos()?)?; + let registry = &mut self.queues; + for queue in registry + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + queue.release_witnessed( + state, + |owner| registry.accounting.complete(&view, &[owner]), + &mut |event| deferred.push(event), + )?; + } + Ok(()) + } + + pub(super) fn service_runtime_events( + &mut self, + deferred: &mut DeferredBatch, + ) -> Result<(event::Pending, [u64; 2])> { + let mut service = Service { + firmware: self, + deferred, + render_masks: [0; 2], + settlement_error: None, + }; + let pending = event::service(&mut service)?; + if pending.recovery { + // Inline timeout/drain service can consume the restart record too. + // Close publication in this same critical section and ensure the + // event worker will join builders before the restart handshake. + let shared = &service.firmware.primary.state.shared; + shared.preparations.close(); + shared.queue_events(); + } + Ok((pending, service.render_masks)) + } + + /// Preparation admission is already closed and all builders have joined. + pub(super) fn service_runtime_recovery(&mut self, deferred: &mut DeferredBatch) -> Result { + let shared = self.primary.state.shared.clone(); + let mut service = Service { + firmware: self, + deferred, + render_masks: [0; 2], + settlement_error: None, + }; + let result = recovery::service(&mut service, || { + shared.crashed.load(core::sync::atomic::Ordering::Acquire) + })?; + if let Some(error) = result.target_error { + dev_warn!( + shared.dev.as_ref(), + "Recovery target copy failed: {:?}\n", + error + ); + } + if let Some(error) = result.scan_error { + dev_warn!( + shared.dev.as_ref(), + "Post-recovery compute scan failed: {:?}\n", + error + ); + } + if let Some(error) = service.settlement_error { + return Err(error); + } + Ok(()) + } +} + +impl crate::g17::Shared { + /// The single event work item owns this preallocated collector until all + /// callbacks are finished. Shutdown can detach Firmware during an off-lock + /// boundary; the collector itself still owns every deferred completion. + pub(in crate::g17) fn run_runtime_events(self: &Arc) { + if self.crashed.load(core::sync::atomic::Ordering::Acquire) { + self.stop_queues(); + return; + } + let mut deferred = { + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return; + }; + let Some(deferred) = firmware.queues.worker_deferred.take() else { + drop(state); + self.lose_device_quietly(); + return; + }; + deferred + }; + let result = self.runtime_event_turn(&mut deferred); + deferred.finish(); + if let Some(firmware) = (*self.state.lock()).as_deref_mut() { + firmware.queues.worker_deferred = Some(deferred); + } + if let Err(error) = result { + dev_err!(self.dev.as_ref(), "GPU event service failed: {:?}\n", error); + self.lose_device_quietly(); + self.stop_queues(); + } + } + + fn runtime_event_turn(self: &Arc, deferred: &mut DeferredBatch) -> Result { + let (recover, masks) = { + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + let (pending, masks) = firmware.service_runtime_events(deferred)?; + let recover = pending.recovery || firmware.recovery.pending(); + if recover { + // Epoch closure and event consumption are serialized with every + // publisher. Joining builders must happen after unlocking. + self.preparations.close(); + } + (recover, masks) + }; + deferred.finish(); + if recover { + self.preparations.wait_drained(); + } + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + if self.crashed.load(core::sync::atomic::Ordering::Acquire) { + return Err(EIO); + } + if recover { + firmware.service_runtime_recovery(deferred)?; + if !firmware.recovery.pending() { + if !self.preparations.reopen() { + return Ok(()); + } + // Restart stopped consumption before its following record. + // Resume builders before observing pairs outside this lock. + self.queue_events(); + } + } + // Replay only at this outer boundary, after recovery has settled. + // Publication may service events but must never recurse into replay. + firmware.replay_compute(deferred)?; + drop(state); + self.observe_renders(masks, deferred)?; + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + if self.crashed.load(core::sync::atomic::Ordering::Acquire) { + return Err(EIO); + } + let now = super::now_ns(); + let compute = firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.in_flight()); + let in_flight = compute || firmware.queues.render_in_flight(); + let idle = firmware.idle_check(now, in_flight)?; + firmware.reclaim_drained_renders(deferred)?; + firmware.service_compute_exit(deferred)?; + { + let mut reclaim = self.reclaim.lock(); + firmware.service_teardown(&mut reclaim)?; + if !reclaim.is_empty() { + self.queue_reclaims(); + } + } + let backpressured = firmware.render_control_backpressured(); + let polling = backpressured + || firmware.queues.teardown.polling(now) + || firmware.compute_exit_polling(now); + if polling { + self.queue_poll(); + } + if let Some(delay) = idle { + self.queue_idle(delay); + } + if firmware.recovery.pending() { + self.preparations.close(); + self.queue_events(); + } else if !recover + && !polling + && (channel::Events::pending(&firmware.init, Role::Primary)? + || channel::Events::pending(&firmware.init, Role::Secondary)?) + { + // A full bounded pass must not depend on a fresh notification for + // records that arrived while it was consuming the same ring. + self.queue_events(); + } + if firmware.render_grow_pending() && !firmware.recovery.pending() { + self.queue_grow(); + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs new file mode 100644 index 0000000000000..a6aa740ecf4e3 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -0,0 +1,683 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Device ownership of retained render pairs and their shared free-list leases. +//! The device mutex protects identities and publication. A preparation gate +//! lease permits graph construction outside that mutex without session teardown. + +mod exit; +mod dependencies; +mod grow; +mod observe; +mod pools; +mod prepare; +mod recovery; +mod settle; +mod submit; +pub(super) mod teardown; + +use super::{DeferredBatch, Registry}; +use crate::g17::{context::Context, fw::queue::DataMaster, job::Packet, kick, queue::render::Pair}; +use crate::hw::t8140::queues::RENDER_SLOTS; +use kernel::{prelude::*, sync::Arc}; + +#[derive(Copy, Clone)] +struct Config { + clusters: u32, + descriptor_flags: [u32; 2], +} + +#[derive(Copy, Clone)] +struct Reservation { + slot: u8, + ids: [kick::Id; 2], + owner: u64, + buffer: (u8, u64), +} + +struct Entry { + reservation: Reservation, + pair: Option>, + parked: Option, + preparing: Option>, + context: Option>, + pool: Option>, + pending_owner: Option, + pending_owner_closed: bool, + borrowed_active: bool, + borrowed_dependencies: Option, + event_masks: [u64; 2], + closed: bool, + pending_error: Option<(Arc, Error)>, + pending_timeout: bool, + constructing: bool, + growing: bool, + published: bool, +} + +pub(super) struct RenderState { + config: Config, + entries: KVec>, + pools: pools::Pools, + grow: grow::Grow, + retired: KVec, + next_pm_generation: u64, +} + +impl RenderState { + pub(super) fn new(clusters: u32, descriptor_flags: [u32; 2]) -> Result { + if clusters == 0 { + return Err(EINVAL); + } + let mut entries = KVec::with_capacity(RENDER_SLOTS, GFP_KERNEL)?; + for _ in 0..RENDER_SLOTS { + entries.push(None, GFP_KERNEL)?; + } + Ok(Self { + config: Config { + clusters, + descriptor_flags, + }, + entries, + pools: pools::Pools::new()?, + grow: grow::Grow::new()?, + retired: KVec::with_capacity(128, GFP_KERNEL)?, + next_pm_generation: 1, + }) + } + + fn reserve_pm_generation(&mut self) -> Result { + let generation = self.next_pm_generation; + self.next_pm_generation = generation.checked_add(1).ok_or(EOVERFLOW)?; + Ok(generation) + } + + fn collect_retired(&mut self, slot: usize) -> Result { + loop { + if self.retired.len() == self.retired.capacity() { + return Ok(()); + } + let retired = self.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + .and_then(|pair| pair.take_retired_binding()); + let Some(retired) = retired else { + return Ok(()); + }; + self.retired + .push_within_capacity(retired) + .map_err(|_| EIO)?; + } + } + + fn entry(&self, slot: u8) -> Result<&Entry> { + self.entries + .get(usize::from(slot)) + .and_then(Option::as_ref) + .ok_or(EIO) + } + + fn entry_mut(&mut self, slot: u8) -> Result<&mut Entry> { + self.entries + .get_mut(usize::from(slot)) + .and_then(Option::as_mut) + .ok_or(EIO) + } + + fn reserved(&mut self, reservation: Reservation) -> Result<&mut Entry> { + let entry = self.entry_mut(reservation.slot)?; + if entry.reservation.ids != reservation.ids + || entry.reservation.owner != reservation.owner + || entry.reservation.buffer != reservation.buffer + || entry.pair.is_some() + || entry.preparing.is_some() + || entry.context.is_some() + || !entry.constructing + { + return Err(EIO); + } + Ok(entry) + } + + /// Keep metadata while graph preparation owns the pair outside the mutex. + /// Completion scans skip it; cancellation is recorded and applied on return. + fn take_pair(&mut self, slot: u8, owner: u64, packet: &Arc) -> Result> { + let entry = self.entry_mut(slot)?; + if entry.closed + || entry.reservation.owner != owner + || entry.preparing.is_some() + || entry.pending_error.is_some() + || entry.pending_timeout + { + return Err(EBUSY); + } + let pair = entry.pair.as_ref().ok_or(EIO)?; + if !pair.matches(owner, &packet.context)? { + return Err(EIO); + } + entry.borrowed_active = pair.published_in_flight(); + entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.preparing = Some(packet.clone()); + entry.pair.take().ok_or(EIO) + } + + /// Validate before taking the owner from the caller. Even rejected returns + /// keep their backing alive until the caller resolves the reservation. + fn return_pair(&mut self, slot: u8, pair: &mut Option>) -> Result { + let returning = pair.as_ref().ok_or(EINVAL)?; + let entry = self.entry_mut(slot)?; + if entry.pair.is_some() + || entry.preparing.is_none() + || returning.slot() != slot + || returning.qids() != entry.reservation.ids.map(kick::Id::qid) + || returning.owner() != Some(entry.reservation.owner) + { + return Err(EIO); + } + entry.pair = pair.take(); + entry.borrowed_dependencies = None; + entry.preparing = None; + Ok(()) + } + + fn save_detaching_pool(&self, init: &crate::g17::initdata::InitData, slot: u8) -> Result { + let pool = self.entry(slot)?.pool.as_ref().ok_or(EIO)?; + let shared = self.entries.iter().enumerate().any(|(index, entry)| { + index != usize::from(slot) + && entry.as_ref().is_some_and(|entry| { + entry.pool.as_ref().is_some_and(|other| { + other.id() == pool.id() && other.control_va() == pool.control_va() + }) + }) + }); + if !shared { + self.pools.save(init, pool)?; + } + Ok(()) + } +} + +impl Registry { + fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { + if owner == 0 || slot == 0 || usize::from(slot) >= RENDER_SLOTS { + return Err(EINVAL); + } + if self.render.entries[usize::from(slot)].is_none() { + let identity = self.identities.reserve(2)?.start; + let tiling = self.qids.reserve(identity, DataMaster::Tiling, None)?; + let fragment = match self.qids.reserve(identity + 1, DataMaster::Fragment, None) { + Ok(id) => id, + Err(error) => { + self.qids.cancel_unpublished(tiling)?; + return Err(error); + } + }; + self.render.entries[usize::from(slot)] = Some(Entry { + reservation: Reservation { + slot, + ids: [tiling, fragment], + owner, + buffer: (0, 0), + }, + pair: None, + parked: None, + preparing: None, + context: None, + pool: None, + pending_owner: None, + pending_owner_closed: false, + borrowed_active: false, + borrowed_dependencies: None, + event_masks: [0; 2], + closed: false, + pending_error: None, + pending_timeout: false, + constructing: false, + growing: false, + published: false, + }); + } + Ok(()) + } + + fn reserve_render(&mut self, slot: u8, owner: u64) -> Result { + self.ensure_render_ids(slot, owner)?; + let entry = self.render.entry(slot)?; + if entry.constructing + || entry.pair.is_some() + || entry.parked.is_some() + || entry.preparing.is_some() + || entry.growing + { + return Err(EBUSY); + } + let ids = self.render.entry(slot)?.reservation.ids; + // Physical queue identities survive failed graph construction. Only + // failure to acquire the second identity can return the first one. + for id in ids { + self.qids.publish(id)?; + } + let buffer = self.buffers.allocate_render()?; + let entry = self.render.entry_mut(slot)?; + entry.reservation.owner = owner; + entry.reservation.buffer = buffer; + entry.constructing = true; + entry.closed = false; + Ok(entry.reservation) + } + + fn install_render( + &mut self, + reservation: Reservation, + pair: &mut Option>, + ) -> Result { + let built = pair.as_ref().ok_or(EINVAL)?; + if built.slot() != reservation.slot + || built.qids() != reservation.ids.map(kick::Id::qid) + || built.owner() != Some(reservation.owner) + || built.buffer_id() != reservation.buffer.0 + { + return Err(EIO); + } + if self.render.reserved(reservation)?.closed { + return Err(ECANCELED); + } + // Construction holds one PB reference; retain the owner independently + // and return that reference before the first command acquires its own. + self.buffers.reserve_owner(reservation.buffer.0)?; + self.buffers.release(reservation.buffer.0)?; + let entry = self.render.reserved(reservation)?; + entry.context = Some(built.context().clone()); + entry.pool = Some(built.pool().clone()); + entry.pair = pair.take(); + entry.constructing = false; + Ok(()) + } + + /// Drop the rejected graph outside the device mutex before returning its + /// temporary PB reference. The physical queue identities remain reserved. + fn cancel_render_reservation(&mut self, reservation: Reservation) -> Result { + self.render.reserved(reservation)?; + self.buffers.release(reservation.buffer.0)?; + self.render.entry_mut(reservation.slot)?.constructing = false; + Ok(()) + } + + pub(super) fn fail_render_vm( + &mut self, + status: &Arc, + error: Error, + deferred: &mut DeferredBatch, + ) -> Result { + for entry in self.render.entries.iter_mut().flatten() { + if !entry + .context + .as_ref() + .is_some_and(|context| Arc::ptr_eq(context.status(), status)) + { + continue; + } + if let Some(pair) = entry.pair.as_mut() { + pair.quarantine(error, &mut |event| deferred.push(event))?; + } else if entry.preparing.is_some() { + entry.pending_error = Some((status.clone(), error)); + } + } + Ok(()) + } +} + +struct Host<'a> { + firmware: &'a mut crate::g17::Firmware, + registration_powered: bool, +} +impl Host<'_> { + fn release_registration(&mut self) { + if !core::mem::take(&mut self.registration_powered) { + return; + } + if let Err(error) = self.firmware.release_registration_power() { + dev_warn!( + self.firmware.primary.state.shared.dev.as_ref(), + "Could not restore render idle policy: {:?}\n", + error + ); + } + } +} +impl crate::g17::queue::render::Host for Host<'_> { + fn epoch(&self) -> Result<(u64, u32)> { + Ok(( + self.firmware.recovery.generation(), + crate::g17::recovery::Memory::recovery_state(&self.firmware.init)?, + )) + } + fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]> { + use core::sync::atomic::Ordering; + let stamps = [ + self.firmware.init.stamp(qids[0])?.load(Ordering::Relaxed), + self.firmware.init.stamp(qids[1])?.load(Ordering::Relaxed), + ]; + core::sync::atomic::fence(Ordering::Acquire); + Ok(stamps) + } + fn qos_publish( + &mut self, + owner: crate::g17::qos::Owner, + scheduler: u64, + ) -> Result { + use crate::g17::qos; + self.firmware.queues.accounting.publish( + &qos::View::new(self.firmware.init.qos()?)?, + owner, + scheduler, + qos::clock, + ) + } + fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { + use crate::g17::qos; + self.firmware + .queues + .accounting + .cancel(&qos::View::new(self.firmware.init.qos()?)?, publication) + } + fn install_pair(&mut self, registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + for _ in 0..3 { + if !self.firmware.bridge.regs.g17_configuration_powered() { + self.firmware.acquire_registration_power()?; + self.registration_powered = true; + } + if self + .firmware + .bridge + .regs + .install_render_pair(registration)? + { + return Ok(()); + } + } + Err(EAGAIN) + } + fn publish_qid(&mut self, id: kick::Id) -> Result { + self.firmware.queues.qids.publish(id) + } + fn publish_outer_pair( + &mut self, + fragment: &crate::g17::fw::channels::WorkSlot, + tiling: &crate::g17::fw::channels::WorkSlot, + commit: impl FnOnce() -> Result, + late_tiling: impl FnOnce(), + ) -> Result<[u32; 2]> { + self.firmware + .init + .publish_work_pair(fragment, tiling, true, commit, late_tiling) + } + fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result { + self.release_registration(); + let result = self.firmware.queues.render.pools.activate(id, generation); + if result.is_err() { + self.firmware.primary.state.shared.lose_device_quietly(); + } + result + } + fn publish_retained_pair( + &mut self, + fragment: &crate::g17::fw::channels::WorkSlot, + tiling: &crate::g17::fw::channels::WorkSlot, + next: [u32; 2], + ) -> Result { + let result = self + .firmware + .init + .publish_retained_work_pair(fragment, tiling, next); + if result.is_err() { + self.firmware.primary.state.shared.lose_device_quietly(); + } + result + } + fn notify_render(&mut self) -> Result { + self.release_registration(); + let result = self + .firmware + .primary + .notify((0x83 << crate::g17::MSG_TYPE_SHIFT) | 0x08); + if result.is_err() { + self.firmware.primary.state.shared.lose_device_quietly(); + } + result + } + fn note_submission(&mut self) -> Result { + self.firmware.note_submission() + } + fn complete_render( + &mut self, + owners: Option<[crate::g17::qos::Owner; 2]>, + pool_id: u16, + generation: u64, + buffer_id: u8, + ) -> Result { + use crate::g17::qos; + let queues = &mut self.firmware.queues; + let next = queues.render.pools.completed(pool_id, generation)?; + queues.buffers.can_release(buffer_id)?; + let view = qos::View::new(self.firmware.init.qos()?)?; + if let Some(owners) = owners { + queues.accounting.complete(&view, &owners)?; + } + queues.render.pools.commit_completed(pool_id, next); + queues.buffers.release(buffer_id) + } +} + +impl crate::g17::Firmware { + /// The pair is returned even when transport, accounting or collection fails. + /// No off-lock operation or callback which can re-enter the device runs here. + fn with_render( + &mut self, + slot: usize, + operation: impl FnOnce(&mut Pair, &mut Host<'_>) -> Result, + ) -> Result { + let mut pair = self + .queues + .render + .entries + .get_mut(slot) + .and_then(Option::as_mut) + .and_then(|entry| entry.pair.take()) + .ok_or(EBUSY)?; + let mut host = Host { + firmware: self, + registration_powered: false, + }; + let result = operation(&mut pair, &mut host); + host.release_registration(); + self.queues.render.entries[slot].as_mut().ok_or(EIO)?.pair = Some(pair); + result + } + + fn release_render_lists(&mut self) -> Result { + if self.queues.render.grow.reply.is_some() { + return Err(EAGAIN); + } + let primary = &mut self.primary; + self.queues.render.pools.publish_releases(&self.init, || { + primary.notify(crate::g17::MSG_CONTROL_NOTIFY) + }) + } + + /// Events have already been consumed by the caller. Each healthy completion + /// services its final pool release before examining the next retained job. + fn retire_render_slot( + &mut self, + slot: usize, + masks: Option<[u64; 2]>, + deferred: &mut DeferredBatch, + ) -> Result { + let generation = self.recovery.generation(); + let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; + loop { + let progressed = self.with_render(slot, |pair, host| { + pair.retire(masks, generation, state, host, &mut |event| { + deferred.push(event) + }) + })?; + if !progressed { + break; + } + match self.release_render_lists() { + Err(error) if error == EAGAIN => break, + result => result?, + } + } + Ok(()) + } + + pub(in crate::g17) fn settle_render_vm( + &mut self, + status: &Arc, + error: Error, + deferred: &mut DeferredBatch, + ) -> Result { + for slot in 1..self.queues.render.entries.len() { + if self.queues.render.entries[slot] + .as_ref() + .is_some_and(|entry| { + entry.pair.is_some() + && entry + .context + .as_ref() + .is_some_and(|context| Arc::ptr_eq(context.status(), status)) + }) + { + self.retire_render_slot(slot, None, deferred)?; + } + } + self.queues.fail_render_vm(status, error, deferred) + } + + pub(in crate::g17) fn stop_render(&mut self) { + for entry in self.queues.render.entries.iter_mut().flatten() { + if let Some(pair) = entry.pair.as_mut() { + if let Err(error) = pair.stopped(&mut |event| { + event.finish(); + Ok(()) + }) { + dev_err!( + self.primary.state.shared.dev.as_ref(), + "Could not settle stopped render pair: {:?}\n", + error + ); + } + } + } + } +} + +impl Registry { + pub(in crate::g17) fn render_oldest_spared(&self, qid: u8) -> Option { + self.render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .find(|pair| pair.qids().contains(&qid)) + .and_then(|pair| pair.oldest_spared()) + } + pub(in crate::g17) fn visit_render_owners( + &self, + qid: u8, + visit: &mut dyn FnMut(&Arc), + ) { + for pair in self + .render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + { + if pair.qids().contains(&qid) { + pair.visit_owners(visit); + } + } + } + pub(in crate::g17) fn classify_render( + &self, + classify: &dyn Fn(&Arc) -> crate::g17::recovery::Class, + ) -> u128 { + let mut spared = 0; + for pair in self + .render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + { + pair.classify(classify); + if pair.oldest_spared() == Some(true) { + for qid in pair.qids() { + spared |= 1u128 << qid; + } + } + } + spared + } + + pub(in crate::g17) fn render_scheduler_active(&self) -> bool { + self.render.entries.iter().any(Option::is_some) + || !self.render.retired.is_empty() + || !self.teardown.is_empty() + } + pub(in crate::g17) fn render_in_flight(&self) -> bool { + self.render.entries.iter().flatten().any(|entry| { + entry.pair.as_ref().map_or( + (entry.preparing.is_some() || entry.growing) && entry.borrowed_active, + |pair| pair.published_in_flight(), + ) + }) + } + pub(in crate::g17) fn retain_render_completions(&mut self, masks: [u64; 2]) { + for entry in self.render.entries.iter_mut().flatten() { + for id in entry.reservation.ids { + let qid = usize::from(id.qid()); + entry.event_masks[qid / 64] |= masks[qid / 64] & (1u64 << (qid % 64)); + } + } + } + pub(in crate::g17) fn render_needs_rescan(&self) -> bool { + self.render.entries.iter().flatten().any(|entry| { + let active = entry + .pair + .as_ref() + .map_or(entry.borrowed_active, |pair| pair.published_in_flight()); + active + && entry.reservation.ids.into_iter().all(|id| { + let qid = usize::from(id.qid()); + entry.event_masks[qid / 64] & (1u64 << (qid % 64)) != 0 + }) + }) + } + pub(in crate::g17) fn defer_render_timeout( + &mut self, + packet: &Arc, + error: Error, + ) -> bool { + let Some(slot) = packet.completion.render_slot() else { + return false; + }; + let Ok(entry) = self.render.entry_mut(slot) else { + return false; + }; + if entry.preparing.is_none() && !entry.growing { + return false; + } + if !entry + .borrowed_dependencies + .as_ref() + .is_some_and(|dependencies| dependencies.contains_packet(packet)) + { + return false; + } + packet.completion.defer_timeout(error); + entry.pending_timeout = true; + true + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs new file mode 100644 index 0000000000000..09e4322bf42ce --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Both engines preserve the covered render prefix while physical graphs are borrowed. + +use crate::g17::{dependency::Frontier, job::Packet, runtime::Registry}; +use kernel::{prelude::*, sync::Arc}; + +impl Registry { + pub(in crate::g17::runtime) fn render_prefix( + &self, + owner: u64, + packet: &Arc, + ) -> Result { + let mut frontier = Frontier::new(); + for entry in self.render.entries.iter().flatten() { + if let Some(pair) = entry.pair.as_ref() { + pair.collect_dependencies( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut frontier, + )?; + } else if let Some(dependencies) = entry.borrowed_dependencies.as_ref() { + dependencies.collect( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut frontier, + )?; + } + } + Ok(frontier) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/exit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/exit.rs new file mode 100644 index 0000000000000..c47110cb76fad --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/exit.rs @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Physical render queue scope for an exiting logical context. A graph loan +//! remains foreign to the transaction until its exact owner is restored. + +use super::RenderState; +use crate::g17::context::Context; +use kernel::sync::Arc; + +impl RenderState { + pub(in crate::g17::runtime) fn exit_scope(&self, context: &Arc) -> (u128, u128, u64) { + let (mut own, mut foreign, mut slots) = (0, 0, 0); + for (slot, entry) in self.entries.iter().enumerate() { + let Some(entry) = entry else { + continue; + }; + let qids = self.qids_for_slot(slot as u8); + let preparing = entry.preparing.is_some() || entry.constructing || entry.growing; + if !preparing && entry.context.as_ref().is_some_and(|owner| Arc::ptr_eq(owner, context)) { + own |= qids; + slots |= 1u64 << slot; + } else if preparing || entry.context.is_some() { + foreign |= qids; + } + } + (own, foreign, slots) + } + + pub(in crate::g17::runtime) fn qids_for_slot(&self, slot: u8) -> u128 { + self.entries + .get(usize::from(slot)) + .and_then(Option::as_ref) + .map_or(0, |entry| { + entry.reservation.ids.into_iter().fold(0, |mask, id| { + mask | (1u128 << id.qid()) + }) + }) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs new file mode 100644 index 0000000000000..a9636cb7ceda0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Deferred parameter-buffer growth. Event ingress retains the entire request; +//! allocation borrows one pair under a preparation lease and runs off-lock. + +use super::RenderState; +use crate::g17::{ + channel::Rings, + fw::channels::{ControlRecord, FreeListCompletion, TvbGrowReply, TvbGrowRequest, EVENT_SLOTS}, +}; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +const RETRY_LIMIT: u16 = 1000; + +pub(super) struct Grow { + requests: KVec, + active: Option, + pub(super) reply: Option<(TvbGrowRequest, bool, u16)>, + release_attempts: u16, +} +impl Grow { + pub(super) fn new() -> Result { + Ok(Self { + requests: KVec::with_capacity(2 * EVENT_SLOTS as usize, GFP_KERNEL)?, + active: None, + reply: None, + release_attempts: 0, + }) + } +} + +impl RenderState { + pub(in crate::g17::runtime) fn control_backpressured(&self) -> bool { + self.grow.reply.is_some() || self.pools.pending() + } + fn grow_owner(&self, request: TvbGrowRequest) -> Option { + self.entries.iter().position(|entry| { + entry.as_ref().is_some_and(|entry| { + entry.published + && u32::from(entry.reservation.buffer.0) == request.buffer_slot + && entry.context.as_ref().is_some_and(|context| { + request.vm_slot == -1 || request.vm_slot as u32 == u32::from(context.id()) + }) + }) + }) + } + fn grow_owner_busy(&self, request: TvbGrowRequest) -> bool { + self.grow_owner(request).is_some_and(|slot| { + self.entries[slot] + .as_ref() + .is_some_and(|entry| entry.preparing.is_some() || entry.growing) + }) + } +} + +impl crate::g17::Firmware { + pub(in crate::g17) fn queue_tvb_grow(&mut self, request: TvbGrowRequest) -> Result { + let render = &mut self.queues.render; + if request.halt_count != self.recovery.generation() + || render.grow.active == Some(request) + || render.grow.requests.contains(&request) + { + return Ok(()); + } + render + .grow + .requests + .push_within_capacity(request) + .map_err(|_| EOVERFLOW) + } + pub(in crate::g17) fn render_grow_pending(&self) -> bool { + !self.queues.render.grow.requests.is_empty() + } + pub(in crate::g17) fn render_control_backpressured(&self) -> bool { + self.queues.render.grow.reply.is_some() || self.queues.render.pools.pending() + } + pub(in crate::g17) fn complete_freelist_grow(&mut self, event: FreeListCompletion) -> Result { + self.queues.render.pools.complete_grow(event) + } + pub(in crate::g17) fn observe_render_releases(&mut self) -> Result { + self.queues.render.pools.observe(&self.init) + } + pub(in crate::g17) fn flush_tvb_reply(&mut self) -> Result { + let Some((request, grew, _)) = self.queues.render.grow.reply else { + return Ok(true); + }; + if request.halt_count != self.recovery.generation() { + return Err(EIO); + } + let record = ControlRecord::TvbGrow(TvbGrowReply::new(&request, grew)); + match Rings::publish_control(&self.init, &record) { + Err(error) if error == EAGAIN => return Ok(false), + result => { + result?; + } + } + // Producer ownership is irreversible, even if announcement fails. + self.queues.render.grow.reply = None; + fence(Ordering::SeqCst); + self.primary + .notify((0x84 << crate::g17::MSG_TYPE_SHIFT) | 0x11)?; + Ok(true) + } + pub(in crate::g17) fn flush_freelist_release(&mut self) -> Result { + if self.queues.render.grow.reply.is_some() { + return Ok(false); + } + match self.release_render_lists() { + Err(error) if error == EAGAIN => Ok(false), + result => { + result?; + self.queues.render.grow.release_attempts = 0; + Ok(true) + } + } + } + /// Only the sleeping event worker consumes this budget. Inline submission + /// probes cannot turn a transient full control ring into a rapid timeout. + pub(in crate::g17) fn note_render_control_backpressure(&mut self) -> Result { + let render = &mut self.queues.render; + let attempts = if let Some((_, _, attempts)) = render.grow.reply.as_mut() { + attempts + } else if render.pools.pending() { + &mut render.grow.release_attempts + } else { + return Err(EINVAL); + }; + *attempts = attempts.checked_add(1).ok_or(ETIMEDOUT)?; + if *attempts > RETRY_LIMIT { + return Err(ETIMEDOUT); + } + Ok(()) + } +} + +impl crate::g17::Firmware { + pub(super) fn commit_render_growth( + &mut self, + slot: u8, + prepared: &mut Option, + ) -> Result { + if prepared.is_none() { + return Ok(()); + } + let pair = self + .queues + .render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)?; + let vm = pair.context().vm().clone(); + let pool = pair.pool_id(); + let buffer_id = pair.buffer_id(); + // Preflight every destination before the graph starts publishing. + let table = self.init.pb_descriptor_table()?; + let offset = usize::from(buffer_id) * 16; + let words = [ + table.word(offset)?, + table.word(offset + 4)?, + table.word(offset + 8)?, + table.word(offset + 12)?, + ]; + let descriptor = pair + .manager() + .commit_growth(prepared, |previous, blocks, mappings| { + vm.append_render_pool_mappings_prepared(slot, pool, previous, blocks, mappings) + })?; + if let Some(descriptor) = descriptor { + for (word, value) in words.into_iter().zip(descriptor) { + word.store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + } + Ok(()) + } +} + +impl crate::g17::Shared { + pub(in crate::g17) fn grow_render(&self) -> Result { + loop { + let lease = self.preparations.enter(false)?; + let (request, slot, mut pair) = { + let mut state = self.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if firmware.render_control_backpressured() { + return Ok(()); + } + let generation = firmware.recovery.generation(); + let render = &mut firmware.queues.render; + if render.grow.active.is_some() { + return Ok(()); + } + let Some(index) = render.grow.requests.iter().position(|request| { + request.halt_count != generation || !render.grow_owner_busy(*request) + }) else { + return Ok(()); + }; + let request = render.grow.requests.remove(index).map_err(|_| EIO)?; + if request.halt_count != generation { + continue; + } + let slot = render.grow_owner(request); + let pair = match slot { + Some(slot) => { + let entry = render.entries[slot].as_mut().ok_or(EIO)?; + if entry.preparing.is_some() || entry.growing { + return Err(EIO); + } + entry.borrowed_active = entry + .pair + .as_ref() + .is_some_and(|pair| pair.published_in_flight()); + entry.borrowed_dependencies = + entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + entry.growing = true; + Some(entry.pair.take().ok_or(EIO)?) + } + None => None, + }; + render.grow.active = Some(request); + (request, slot, pair) + }; + let prepared = match pair.as_mut() { + Some(pair) => { + let vm = pair.context().vm().clone(); + let pool = pair.pool_id(); + let slot = pair.slot(); + let manager = pair.manager(); + let before = manager.blocks(); + if vm.render_pool_tvb_blocks(slot, pool) != Some(before) { + Err(EFAULT) + } else { + manager.prepare_growth( + &self.dev, + &vm, + crate::g17::buffer::firmware_growth_target(before), + None, + ) + } + } + None => Err(ENOENT), + }; + let (mut prepared, prepare_error) = match prepared { + Ok(plan) => (plan, None), + Err(error) => (None, Some(error)), + }; + let mut state = self.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if let Some(slot) = slot { + let entry = firmware.queues.render.entries[slot].as_mut().ok_or(EIO)?; + if entry.pair.is_some() || !entry.growing { + return Err(EIO); + } + entry.pair = pair.take(); + entry.borrowed_dependencies = None; + entry.growing = false; + } + if firmware.queues.render.grow.active != Some(request) { + return Err(EIO); + } + firmware.queues.render.grow.active = None; + self.changed.notify_all(); + if !lease.is_current() || request.halt_count != firmware.recovery.generation() { + continue; + } + let result = if let Some(error) = prepare_error { + Err(error) + } else if prepared.is_none() { + Ok(false) + } else if let Some(slot) = slot { + let pair = firmware.queues.render.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + .ok_or(EIO)?; + let vm = pair.context().vm().clone(); + let pool = pair.pool_id(); + let slot = pair.slot(); + let grew = prepared.is_some(); + pair.manager() + .commit_growth(&mut prepared, |previous, blocks, mappings| { + vm.append_render_pool_mappings_prepared( + slot, pool, previous, blocks, mappings, + ) + }) + .map(|_| grew) + } else { + Err(ENOENT) + }; + let grew = match result { + Ok(grew) => grew, + Err(error) => { + dev_warn!( + self.dev.as_ref(), + "Could not grow render parameter buffer {}: {:?}\n", + request.buffer_slot, + error + ); + false + } + }; + firmware.queues.render.grow.reply = Some((request, grew, 0)); + if !firmware.flush_tvb_reply()? { + return Ok(()); + } + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs new file mode 100644 index 0000000000000..31bcb688c8001 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Event-driven render snapshots outside the device mutex. The preparation +//! epoch and exact pair owner survive every read, cancellation and retry. + +use super::super::DeferredBatch; +use crate::g17::queue::render::Pair; +use core::sync::atomic::Ordering; +use kernel::{prelude::*, sync::Arc}; + +impl crate::g17::Firmware { + fn take_render_observation(&mut self, start: usize) -> Result)>> { + if self.recovery.pending() { + return Ok(None); + } + let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; + for slot in start..self.queues.render.entries.len() { + let Some(entry) = self.queues.render.entries[slot].as_mut() else { + continue; + }; + if entry.preparing.is_some() || entry.growing { + continue; + } + let Some(pair) = entry.pair.as_ref() else { + continue; + }; + let qids = pair.qids(); + if !pair.published_in_flight() + || !qids + .into_iter() + .all(|qid| entry.event_masks[usize::from(qid) / 64] & (1u64 << (qid % 64)) != 0) + { + continue; + } + let stamps = [ + self.init.stamp(qids[0])?.load(Ordering::Relaxed), + self.init.stamp(qids[1])?.load(Ordering::Relaxed), + ]; + core::sync::atomic::fence(Ordering::Acquire); + let Some(packet) = pair.observation_packet(entry.event_masks, stamps, state)? else { + continue; + }; + entry.borrowed_active = pair.published_in_flight(); + entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.preparing = Some(packet); + return Ok(Some((slot, entry.pair.take().ok_or(EIO)?))); + } + Ok(None) + } + + fn return_render_observation(&mut self, slot: usize, pair: &mut Option>) -> Result { + let returning = pair.as_ref().ok_or(EINVAL)?; + let entry = self.queues.render.entry_mut(slot as u8)?; + if entry.pair.is_some() + || entry.preparing.is_none() + || returning.slot() as usize != slot + || returning.qids() != entry.reservation.ids.map(crate::g17::kick::Id::qid) + { + return Err(EIO); + } + entry.pair = pair.take(); + entry.borrowed_dependencies = None; + entry.preparing = None; + Ok(()) + } + + pub(super) fn settle_render_terminal( + &mut self, + slot: usize, + deferred: &mut DeferredBatch, + ) -> Result { + let terminal = self.queues.render.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + .and_then(|pair| pair.take_terminal()); + if let Some((status, error)) = terminal { + status.record(error); + self.scan_compute(None, deferred)?; + self.settle_render_vm(&status, error, deferred)?; + self.queues.fail_compute_vm(&status, error, deferred)?; + } + Ok(()) + } +} + +impl crate::g17::Shared { + pub(in crate::g17) fn observe_renders( + self: &Arc, + masks: [u64; 2], + deferred: &mut DeferredBatch, + ) -> Result { + { + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + firmware.queues.retain_render_completions(masks); + // Returned preparers can have deferred a VM failure or timeout. + // Only this full worker collector runs their final observations. + for slot in 1..firmware.queues.render.entries.len() { + if firmware.queues.render.entries[slot] + .as_ref() + .is_some_and(|entry| { + entry.pair.is_some() + && (entry.pending_error.is_some() + || entry.pending_timeout + || entry.closed) + }) + { + firmware.apply_render_deferred(slot as u8, deferred)?; + } + } + } + deferred.finish(); + let mut cursor = 1; + let mut remaining = crate::hw::t8140::queues::RENDER_DEPTH; + let mut retries = 0; + loop { + let epoch = match self.preparations.try_enter() { + Err(error) if error == EAGAIN || error == ENODEV => break, + result => result?, + }; + let loan = { + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + if !epoch.is_current() || self.crashed.load(Ordering::Acquire) { + break; + } + firmware.take_render_observation(cursor)? + }; + let Some((slot, pair)) = loan else { + break; + }; + if slot != cursor { + remaining = crate::hw::t8140::queues::RENDER_DEPTH; + retries = 0; + } + let observed = pair.observe_front(); + let mut pair = Some(pair); + let mut state = self.state.lock(); + // Return installed ownership before examining read errors or epoch. + let restored = (*state) + .as_deref_mut() + .ok_or(ENODEV) + .and_then(|firmware| firmware.return_render_observation(slot, &mut pair)); + if let Err(error) = restored { + // A failed invariant must not drop firmware-owned storage. Join + // the other preparers only after releasing our own epoch, and + // keep this detached owner until both processors have stopped. + drop(state); + drop(epoch); + self.lose_device_quietly(); + self.stop_queues(); + if let Some(mut pair) = pair { + pair.stopped(&mut |event| { + event.finish(); + Ok(()) + })?; + } + return Err(error); + } + let firmware = (*state).as_deref_mut().ok_or(EIO)?; + self.changed.notify_all(); + if firmware.render_grow_pending() { + self.queue_events(); + } + if !epoch.is_current() + || firmware.recovery.pending() + || self.crashed.load(Ordering::Acquire) + { + break; + } + let result = (|| { + let state = crate::g17::recovery::Memory::recovery_state(&firmware.init)?; + let progressed = match observed { + Ok(observed) => firmware.with_render(slot, |pair, host| { + pair.retire_observed(observed, state, host, &mut |event| { + deferred.push(event) + }) + })?, + Err(error) if error == EAGAIN => false, + Err(error) => return Err(error), + }; + if progressed { + match firmware.release_render_lists() { + Err(error) if error == EAGAIN => {} + result => result?, + } + } + firmware.queues.render.collect_retired(slot)?; + // An observation of the old front is committed before a queued + // exact timeout is permitted to advance that same tracker. + firmware.apply_render_deferred(slot as u8, deferred)?; + firmware.settle_render_terminal(slot, deferred)?; + if firmware + .queues + .render + .entry(slot as u8)? + .pair + .as_ref() + .is_some_and(|pair| !pair.in_flight()) + { + firmware.queues.render.entry_mut(slot as u8)?.event_masks = [0; 2]; + } + Ok(progressed) + })(); + drop(state); + drop(epoch); + deferred.finish(); + if result? { + remaining -= 1; + retries = 0; + cursor = if remaining == 0 { slot + 1 } else { slot }; + } else { + retries += 1; + cursor = if retries >= 2 { slot + 1 } else { slot }; + } + if cursor != slot { + remaining = crate::hw::t8140::queues::RENDER_DEPTH; + retries = 0; + } + } + { + let state = self.state.lock(); + if (*state).as_deref().is_some_and(|firmware| { + !firmware.recovery.pending() && firmware.queues.render_needs_rescan() + }) { + self.queue_poll(); + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs new file mode 100644 index 0000000000000..7047992754b37 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Device-wide references to render free lists. Two physical pairs bound to +//! one logical context share one registration and one final release witness. +//! The device mutex serializes every transition and descriptor-table access. + +use crate::g17::{ + channel::Rings, + freelist::{Lease, Owner, RenderPool}, + fw::{ + channels::{ControlRecord, FreeListCompletion}, + initdata::PagePoolDescriptor, + }, + initdata::InitData, + object::KernelObject, +}; +use core::sync::atomic::{fence, AtomicU64, Ordering}; +use kernel::prelude::*; + +const POOLS: usize = 256; +const FIRST_RENDER_POOL: usize = 129; + +pub(super) struct Pools { + leases: KVec>, +} + +/// Undo state cannot leave the device-locked publication transaction. Rollback +/// is permitted only before the packet exposes its first firmware reference. +pub(super) struct Preparation { + previous: Option, + slot: usize, + pub(super) generation: u64, +} + +impl Pools { + pub(super) fn new() -> Result { + let mut leases = KVec::with_capacity(POOLS, GFP_KERNEL)?; + for _ in 0..POOLS { + leases.push(None, GFP_KERNEL)?; + } + Ok(Self { leases }) + } + + fn slot(id: u16) -> Result { + let slot = usize::from(id); + if !(FIRST_RENDER_POOL..POOLS).contains(&slot) { + return Err(EINVAL); + } + Ok(slot) + } + + fn owner(pool: &RenderPool) -> Owner { + Owner { + serial: pool.serial(), + buffer_slot: u32::from(pool.id()), + control_va: pool.control_va(), + ring_va: pool.run_list_va(), + cookie: pool.end_va(), + } + } + + pub(super) fn released(&self, id: u32, _control: u64) -> bool { + self.leases + .get(id as usize) + .is_none_or(|lease| lease.as_ref().is_none_or(|lease| lease.released())) + } + + /// Install both engine references before either inner queue becomes + /// visible. Warm overlap keeps the existing generation and pool contents. + pub(super) fn prepare(&mut self, init: &InitData, pool: &RenderPool) -> Result { + self.observe(init)?; + let slot = Self::slot(pool.id())?; + let owner = Self::owner(pool); + let previous = self.leases[slot]; + if previous.is_some_and(|lease| lease.owner() != owner && !lease.released()) { + return Err(EBUSY); + } + // Pools arrive fully populated. There is no asynchronous initial grow. + let next = Lease::acquire(previous.as_ref(), owner)?; + if previous.is_none_or(|lease| lease.owner() != owner) { + self.stage(init, pool)?; + } + let generation = next.generation(); + self.leases[slot] = Some(next); + Ok(Preparation { + previous, + slot, + generation, + }) + } + + /// Warming validates the incoming owner and stages its descriptor without + /// acquiring command references or publishing an asynchronous request. + pub(super) fn warm(&self, init: &InitData, pool: &RenderPool) -> Result { + let previous = self.leases[Self::slot(pool.id())?]; + let owner = Self::owner(pool); + if previous.is_some_and(|lease| lease.owner() == owner) { + return Ok(()); + } + if previous.is_some_and(|lease| !lease.released()) { + return Err(EBUSY); + } + Lease::acquire(previous.as_ref(), owner)?; + self.stage(init, pool) + } + + fn stage(&self, init: &InitData, pool: &RenderPool) -> Result { + let words = descriptor_words(init.page_pool_descriptor_table()?, pool.id())?; + pool.repopulate()?; + let descriptor = pool + .page_pool_state()? + .descriptor(words[3].load(Ordering::Relaxed)); + for (word, value) in words.into_iter().zip([ + descriptor.page_list, + descriptor.counters, + descriptor.page_count, + descriptor.unk_18, + ]) { + word.store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } + + /// Retire preparation only after both deferred outer bodies and their + /// inner commands exist, before publishing either outer producer. + pub(super) fn activate(&mut self, id: u16, generation: u64) -> Result { + let lease = self.leases[Self::slot(id)?].as_mut().ok_or(EIO)?; + if lease.generation() != generation { + return Err(EIO); + } + lease.retire_prepare() + } + + pub(super) fn rollback(&mut self, preparation: Preparation) { + self.leases[preparation.slot] = preparation.previous; + } + + /// Preflight a paired retirement before QoS or PB reference accounting. + pub(super) fn completed(&self, id: u16, generation: u64) -> Result { + let mut next = self.leases[Self::slot(id)?].ok_or(EIO)?; + if next.generation() != generation { + return Err(EIO); + } + next.complete_render()?; + Ok(next) + } + + /// The matching completed() result and other accounting preflights were + /// accepted under the same device mutex, so this assignment cannot fail. + pub(super) fn commit_completed(&mut self, id: u16, next: Lease) { + self.leases[usize::from(id)] = Some(next); + } + + pub(super) fn pending(&self) -> bool { + self.leases + .iter() + .flatten() + .any(|lease| lease.pending_release().is_some()) + } + + pub(super) fn complete_grow(&mut self, event: FreeListCompletion) -> Result { + let id = u16::try_from(event.buffer_slot).map_err(|_| EINVAL)?; + let lease = self.leases[Self::slot(id)?].as_mut().ok_or(EIO)?; + lease.complete_grow(event) + } + + pub(super) fn observe(&mut self, init: &InitData) -> Result { + fence(Ordering::Acquire); + let consumer = init.control_consumer()?.load(Ordering::Relaxed); + let producer = init.control_producer()?.load(Ordering::Relaxed); + for lease in self.leases.iter_mut().flatten() { + lease.observe_releases(consumer, producer)?; + } + Ok(()) + } + + /// Store the release witness immediately after the ring producer. A failed + /// mailbox announcement must never cause the same release to be replayed. + pub(super) fn publish_releases( + &mut self, + init: &InitData, + mut announce: impl FnMut() -> Result, + ) -> Result { + for lease in self.leases.iter_mut().flatten() { + let Some(release) = lease.pending_release() else { + continue; + }; + let cursor = Rings::publish_control(init, &ControlRecord::FreeListRelease(release))?; + lease.mark_release_sent(cursor)?; + fence(Ordering::SeqCst); + announce()?; + } + self.observe(init) + } + + /// A detached pool may capture firmware counters only after its last pair + /// stopped using this registration. Callers check all retained pair owners. + pub(super) fn save(&self, init: &InitData, pool: &RenderPool) -> Result { + let state = pool.page_pool_state()?; + let words = descriptor_words(init.page_pool_descriptor_table()?, pool.id())?; + fence(Ordering::Acquire); + let descriptor = PagePoolDescriptor { + page_list: words[0].load(Ordering::Relaxed), + counters: words[1].load(Ordering::Relaxed), + page_count: words[2].load(Ordering::Relaxed), + unk_18: words[3].load(Ordering::Relaxed), + }; + if descriptor.page_list != state.descriptor(descriptor.unk_18).page_list { + return Err(EIO); + } + pool.store_page_pool_state(state.updated(&descriptor)) + } +} + +fn descriptor_words(table: &KernelObject, id: u16) -> Result<[&AtomicU64; 4]> { + let offset = Pools::slot(id)? * size_of::(); + Ok([ + table.dword(offset)?, + table.dword(offset + 8)?, + table.dword(offset + 16)?, + table.dword(offset + 24)?, + ]) +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs new file mode 100644 index 0000000000000..30ea174f611b1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -0,0 +1,597 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Off-lock render graph construction and command preparation. Every borrowed +//! installed pair returns to its device owner before its preparation lease ends. + +use super::{Config, Reservation}; +use crate::g17::{ + buffer::MetricsLease, + job::Packet, + object::{Allocator, KernelObject, Pool}, + queue::render::{Pair, RenderAddresses}, +}; +use kernel::{prelude::*, sync::Arc}; + +struct BuildResources { + uat: Arc, + objects: Arc, + ids: Arc, + global: crate::mmu::Vm, + metrics: Arc, + metrics_ids: Arc, + metrics_bases: [u64; 2], + config: Config, +} +impl BuildResources { + fn new(firmware: &crate::g17::Firmware) -> Result { + Ok(Self { + uat: firmware.uat.clone(), + objects: firmware.pool.clone(), + ids: firmware.render_ids.clone(), + global: firmware._render_global.clone(), + metrics: firmware._pm_metrics.clone(), + metrics_ids: firmware.queues.metrics.clone(), + metrics_bases: [ + crate::hw::t8140::CONFIG.pm_metrics.va, + firmware.init.metrics_fw_va()?, + ], + config: firmware.queues.render.config, + }) + } + fn metrics(&self) -> Result { + MetricsLease::new(&self.metrics_ids, &self.metrics, self.metrics_bases) + } +} + +enum Build { + Fresh(Reservation, u64, MetricsLease), + Parked { + parked: crate::g17::queue::render::parked::ParkedPair, + buffer: (u8, u64), + pm_generation: u64, + metrics: MetricsLease, + }, + Rebind { + pair: KBox, + buffer: Option<(u8, u64)>, + pm_generation: Option, + metrics: Option, + }, +} + +impl super::super::Backend { + fn build_render_pair(&self, slot: u8, first: &Arc) -> Result { + let lease = self.shared.preparations.enter(false)?; + let (resources, build) = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + firmware.queues.ensure_render_ids(slot, self.owner)?; + let resources = BuildResources::new(firmware)?; + let build = + if let Some(entry) = firmware.queues.render.entries[usize::from(slot)].as_ref() { + if entry.pending_error.is_some() || entry.pending_timeout { + self.shared.queue_events(); + return Err(EBUSY); + } + if entry.constructing || entry.preparing.is_some() || entry.growing { + return Err(EBUSY); + } + if entry.parked.is_some() { + let metrics = resources.metrics()?; + let pm_generation = firmware.queues.render.reserve_pm_generation()?; + let buffer = firmware.queues.buffers.allocate_render()?; + let entry = firmware.queues.render.entry_mut(slot)?; + entry.preparing = Some(first.clone()); + entry.pending_owner = Some(self.owner); + entry.borrowed_active = false; + Build::Parked { + parked: entry.parked.take().ok_or(EIO)?, + buffer, + pm_generation, + metrics, + } + } else if entry.pair.is_none() { + let metrics = resources.metrics()?; + let pm_generation = firmware.queues.render.reserve_pm_generation()?; + Build::Fresh( + firmware.queues.reserve_render(slot, self.owner)?, + pm_generation, + metrics, + ) + } else { + let pair = entry.pair.as_ref().ok_or(EBUSY)?; + if pair.matches(self.owner, &self.context)? { + return Ok(()); + } + if !pair.reusable()? { + return Err(EBUSY); + } + let foreign = !Arc::ptr_eq(pair.context().status(), self.context.status()); + if foreign { + firmware + .queues + .render + .save_detaching_pool(&firmware.init, slot)?; + } + let metrics = if foreign { + Some(resources.metrics()?) + } else { + None + }; + let pm_generation = if foreign { + Some(firmware.queues.render.reserve_pm_generation()?) + } else { + None + }; + let buffer = if foreign { + Some(firmware.queues.buffers.allocate_render()?) + } else { + None + }; + let entry = firmware.queues.render.entry_mut(slot)?; + entry.preparing = Some(first.clone()); + entry.pending_owner = Some(self.owner); + entry.borrowed_active = entry + .pair + .as_ref() + .is_some_and(|pair| pair.published_in_flight()); + entry.borrowed_dependencies = + entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + Build::Rebind { + pair: entry.pair.take().ok_or(EIO)?, + buffer, + pm_generation, + metrics, + } + } + } else { + let metrics = resources.metrics()?; + let pm_generation = firmware.queues.render.reserve_pm_generation()?; + Build::Fresh( + firmware.queues.reserve_render(slot, self.owner)?, + pm_generation, + metrics, + ) + }; + (resources, build) + }; + let alloc = Allocator { + dev: &self.shared.dev, + uat: &resources.uat, + }; + let fresh_graph = match &build { + Build::Fresh(..) | Build::Parked { .. } => true, + Build::Rebind { buffer, .. } => buffer.is_some(), + }; + let result = match build { + Build::Fresh(reservation, pm_generation, metrics) => { + let built = (|| -> Result> { + let free_list = + self.context + .render_pool(&alloc, &resources.ids, &resources.global)?; + Pair::new( + &alloc, + &resources.objects, + reservation.ids, + slot, + self.owner, + self.context.clone(), + free_list, + metrics, + reservation.buffer, + pm_generation, + first, + resources.config.clusters, + resources.config.descriptor_flags, + ) + })(); + let (mut built, error) = match built { + Ok(pair) => (Some(pair), None), + Err(error) => (None, Some(error)), + }; + let result = (|| { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + if let Some(error) = error { + return Err(error); + } + if !self.context.is_current() || self.context.status().get() != 0 { + return Err(EFAULT); + } + if firmware.queues.render.reserved(reservation)?.closed { + return Err(ECANCELED); + } + built + .as_mut() + .ok_or(EIO)? + .finish_install(&alloc, firmware.init.pb_descriptor_table()?)?; + firmware.queues.install_render(reservation, &mut built) + })(); + // Fresh backing disappears before its temporary PB reference returns. + drop(built); + if result.is_err() { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .cancel_render_reservation(reservation)?; + } + result + } + Build::Parked { + mut parked, + buffer, + pm_generation, + metrics, + } => { + let built = (|| -> Result<_> { + let free_list = + self.context + .render_pool(&alloc, &resources.ids, &resources.global)?; + let prepared = parked.prepare( + &alloc, + &resources.objects, + self.owner, + self.context.clone(), + free_list, + metrics, + buffer, + pm_generation, + first, + )?; + let storage = KBox::::new_uninit(GFP_KERNEL)?; + Ok((prepared, storage)) + })(); + let (mut prepared, mut storage, error) = match built { + Ok((prepared, storage)) => (Some(prepared), Some(storage), None), + Err(error) => (None, None, Some(error)), + }; + let result = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + (|| { + if !lease.is_current() { + return Err(EAGAIN); + } + if let Some(error) = error { + return Err(error); + } + if !self.context.is_current() || self.context.status().get() != 0 { + return Err(EFAULT); + } + let entry = firmware.queues.render.entry(slot)?; + if entry.pending_owner != Some(self.owner) + || entry.pending_owner_closed + || entry.pair.is_some() + || entry.parked.is_some() + { + return Err(ECANCELED); + } + let storage = storage.take().ok_or(EIO)?; + firmware.queues.buffers.can_release(buffer.0)?; + prepared.as_mut().ok_or(EIO)?.finish_install( + &parked, + &alloc, + firmware.init.pb_descriptor_table()?, + )?; + let pair = parked.commit(&mut prepared, storage)?; + firmware.queues.buffers.reserve_owner(buffer.0)?; + firmware.queues.buffers.release(buffer.0)?; + let entry = firmware.queues.render.entry_mut(slot)?; + entry.reservation.owner = self.owner; + entry.reservation.buffer = buffer; + entry.context = Some(self.context.clone()); + entry.pool = Some(pair.pool().clone()); + entry.pair = Some(pair); + entry.closed = false; + entry.published = false; + Ok(()) + })() + }; + // Rejected candidate mappings disappear before the PB identity + // returns, while the original installed kick owner stays live. + drop(prepared); + drop(storage); + { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if result.is_err() { + firmware.queues.render.entry_mut(slot)?.parked = Some(parked); + firmware.queues.buffers.release(buffer.0)?; + } else { + drop(state); + drop(parked); + state = self.shared.state.lock(); + } + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + let entry = firmware.queues.render.entry_mut(slot)?; + entry.preparing = None; + entry.pending_owner = None; + entry.pending_owner_closed = false; + } + self.shared.changed.notify_all(); + result + } + Build::Rebind { + mut pair, + buffer, + pm_generation, + metrics, + } => { + let prepared = (|| { + let free_list = + self.context + .render_pool(&alloc, &resources.ids, &resources.global)?; + pair.prepare_bind( + &alloc, + &resources.objects, + self.owner, + self.context.clone(), + free_list, + metrics, + buffer, + pm_generation, + first, + ) + })(); + let (mut prepared, error) = match prepared { + Ok(binding) => (Some(binding), None), + Err(error) => (None, Some(error)), + }; + let result = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + let result = (|| { + if !lease.is_current() { + return Err(EAGAIN); + } + if let Some(error) = error { + return Err(error); + } + if !self.context.is_current() || self.context.status().get() != 0 { + return Err(EFAULT); + } + let entry = firmware.queues.render.entry(slot)?; + if entry.pending_owner != Some(self.owner) || entry.pending_owner_closed { + return Err(ECANCELED); + } + if firmware.queues.render.retired.len() + == firmware.queues.render.retired.capacity() + { + return Err(ENOSPC); + } + firmware.queues.teardown.displace( + pair.context(), + true, + super::super::now_ns(), + )?; + pair.bind_owner( + prepared.as_mut().ok_or(EIO)?, + &alloc, + firmware.init.pb_descriptor_table()?, + )?; + if let Some((id, _)) = buffer { + firmware.queues.buffers.reserve_owner(id)?; + firmware.queues.buffers.release(id)?; + } + let entry = firmware.queues.render.entry_mut(slot)?; + entry.reservation.owner = self.owner; + if let Some(buffer) = buffer { + entry.reservation.buffer = buffer; + } + entry.context = Some(self.context.clone()); + entry.pool = Some(pair.pool().clone()); + entry.closed = false; + Ok(()) + })(); + // The original installed pair remains owned on every error. + let entry = firmware.queues.render.entry_mut(slot)?; + entry.preparing = None; + entry.pending_owner = None; + entry.pending_owner_closed = false; + entry.pair = Some(pair); + entry.borrowed_dependencies = None; + result + }; + drop(prepared); + if result.is_err() { + if let Some((id, _)) = buffer { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .buffers + .release(id)?; + } + } + self.shared.changed.notify_all(); + result + } + }; + result?; + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + if firmware.queues.render.entry(slot)?.closed { + return Err(ECANCELED); + } + if !self.context.is_current() || self.context.status().get() != 0 { + return Err(EFAULT); + } + // From this point every error leaves the complete graph retained for + // retry. Physical queue identities and installed mappings stay owned. + if fresh_graph { + firmware + .queues + .render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)? + .rebase_fresh_graph()?; + } + let pool = firmware + .queues + .render + .entry(slot)? + .pool + .clone() + .ok_or(EIO)?; + firmware.queues.render.pools.warm(&firmware.init, &pool) + } + + pub(super) fn ensure_render(&self, packet: &Arc, slot: u8) -> Result { + self.build_render_pair(slot, packet)?; + let sibling = { + let state = self.shared.state.lock(); + (*state) + .as_deref() + .ok_or(ENODEV)? + .queues + .admission + .warmable_sibling(self.owner, slot) + }; + if let Some(sibling) = sibling { + self.build_render_pair(sibling, packet)?; + } + Ok(()) + } + + pub(super) fn prepare_render( + &self, + slot: u8, + packet: &Arc, + deferred: &mut super::super::DeferredBatch, + ) -> Result { + let lease = self.shared.preparations.enter(false)?; + let (uat, mut pair, addresses, growth_target) = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + let qids = firmware + .queues + .render + .entry(slot)? + .pair + .as_ref() + .ok_or(EBUSY)? + .qids(); + let addresses = RenderAddresses { + stamps: [ + firmware.init.stamp_addresses(qids[0])?, + firmware.init.stamp_addresses(qids[1])?, + ], + }; + let pair = firmware + .queues + .render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)?; + if !pair.matches(self.owner, &packet.context)? { + return Err(EIO); + } + // The selected scheduler is retained through release acknowledgement + // even if later private command allocation fails. Sibling warming + // does not cross this publication boundary. + packet.context.mark_published(); + let target = pair.host_growth_target(packet)?; + let buffer_id = pair.buffer_id(); + let growth_target = if let Some(target) = target { + let table = firmware.init.pb_descriptor_table()?; + let offset = usize::from(buffer_id) * 16; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let mut descriptor = [0; 4]; + for (index, word) in descriptor.iter_mut().enumerate() { + *word = table + .word(offset + index * 4)? + .load(core::sync::atomic::Ordering::Relaxed); + } + Some((target, descriptor)) + } else { + None + }; + let pair = firmware.queues.render.take_pair(slot, self.owner, packet)?; + (firmware.uat.clone(), pair, addresses, growth_target) + }; + let alloc = Allocator { + dev: &self.shared.dev, + uat: &uat, + }; + let mut growth = None; + let result = (|| { + if let Some((target, descriptor)) = growth_target { + let vm = pair.context().vm().clone(); + growth = pair.manager().prepare_growth( + &self.shared.dev, + &vm, + target, + Some(descriptor), + )?; + } + pair.prepare(&alloc, packet, addresses) + })(); + let mut returned = Some(pair); + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + firmware.queues.render.return_pair(slot, &mut returned)?; + if firmware.queues.render.entry(slot)?.closed { + firmware + .queues + .render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)? + .release_owner(); + } + self.shared.changed.notify_all(); + let published = (|| { + let entry = firmware.queues.render.entry(slot)?; + if entry.pending_error.is_some() || entry.pending_timeout { + // Final timeout observations can settle unrelated queues. The + // event worker owns the full collector needed for that sweep. + self.shared.queue_events(); + return Err(EBUSY); + } + if !lease.is_current() + || firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + return Err(EAGAIN); + } + // Private restaging can fail after preparing a larger inventory. + // Finish its descriptor before discarding the command preparation. + firmware.commit_render_growth(slot, &mut growth)?; + result?; + if packet.cancel_requested() || firmware.queues.render.entry(slot)?.closed { + return Err(ECANCELED); + } + firmware.publish_render(self.owner, slot, packet, deferred) + })(); + if published.is_err() && !packet.is_published() { + firmware + .queues + .render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)? + .discard_prepared(packet)?; + } + published + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs b/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs new file mode 100644 index 0000000000000..3a5b042fff683 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Recovery targets for retained render graphs while firmware is halted. + +use crate::g17::runtime::Registry; +use kernel::prelude::*; + +impl Registry { + pub(in crate::g17::runtime) fn render_pass_started(&self, qid: u8, stamp: u64) -> Option { + self.render.entries.iter().flatten() + .filter_map(|entry| entry.pair.as_ref()) + .find(|pair| pair.qids().contains(&qid))? + .pass_started(stamp) + } + + pub(in crate::g17::runtime) fn publish_render_recovery_targets(&self) -> Option { + let mut failure = None; + for pair in self.render.entries.iter().flatten() + .filter_map(|entry| entry.pair.as_ref()) + { + if let Err(error) = pair.publish_recovery_targets() { + failure = failure.or(Some(error)); + } + } + failure + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs new file mode 100644 index 0000000000000..a6d5821561d24 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Exact packet timeout and paired firmware-drain settlement. Every fence and +//! admission release is collected under the device lock and finished outside it. + +use crate::g17::{ + job::Packet, + runtime::{Backend, DeferredBatch}, +}; +use kernel::{dma_fence::RawDmaFence, prelude::*, sync::Arc}; + +impl Backend { + pub(in crate::g17) fn timeout_render(&self, packet: &Arc) -> Result { + let mut deferred = DeferredBatch::worker()?; + let result = { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + if firmware.queues.defer_render_timeout(packet, ETIMEDOUT) { + return Ok(()); + } + firmware.fail_render_packet(packet, ETIMEDOUT, &mut deferred) + }; + deferred.finish(); + result + } + pub(in crate::g17) fn cancel_render(&self, packet: &Arc) -> Result { + let fence = packet.completion.fence(); + // SAFETY: The retained fence remains alive across this bounded, + // uninterruptible queue-teardown wait, without holding a device lock. + let completed = unsafe { + kernel::bindings::dma_fence_wait_timeout( + fence.raw(), + false, + kernel::time::msecs_to_jiffies(2000) as _, + ) + }; + if completed > 0 { + return Ok(()); + } + if packet.is_published() { + self.timeout_render(packet) + } else { + // The outer callback already completed a successful 0 -> 3 + // cancellation. A claimed publisher still owns its VM pin and + // observes cancel_requested before committing or finishing. + Ok(()) + } + } +} + +impl crate::g17::Firmware { + pub(super) fn fail_render_packet( + &mut self, + packet: &Arc, + error: Error, + deferred: &mut DeferredBatch, + ) -> Result { + let Some(slot) = packet.completion.render_slot() else { + return Ok(()); + }; + let slot = usize::from(slot); + loop { + let pair = self + .queues + .render + .entries + .get(slot) + .and_then(Option::as_ref) + .and_then(|entry| entry.pair.as_ref()) + .ok_or(EIO)?; + if !pair.contains_packet(packet) { + return Ok(()); + } + let generation = self.recovery.generation(); + let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; + let progressed = self.with_render(slot, |pair, host| { + pair.retire(None, generation, state, host, &mut |event| { + deferred.push(event) + }) + })?; + if !progressed { + break; + } + match self.release_render_lists() { + Err(error) if error == EAGAIN => {} + result => result?, + } + } + self.with_render(slot, |pair, host| { + pair.fail_after_observation(packet, error, host, &mut |event| deferred.push(event)) + })?; + self.queues.render.collect_retired(slot)?; + let terminal = self.queues.render.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + .and_then(|pair| pair.take_terminal()); + if let Some((status, error)) = terminal { + status.record(error); + self.scan_compute(None, deferred)?; + self.settle_render_vm(&status, error, deferred)?; + self.queues.fail_compute_vm(&status, error, deferred)?; + } + Ok(()) + } + + pub(in crate::g17) fn settle_blamed_render( + &mut self, + qid: u8, + error: Error, + deferred: &mut DeferredBatch, + ) -> Result { + let packet = self + .queues + .render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .find(|pair| pair.qids().contains(&qid)) + .and_then(|pair| pair.oldest_packet()); + if let Some(packet) = packet { + self.fail_render_packet(&packet, error, deferred)?; + } + Ok(()) + } + + pub(in crate::g17) fn settle_cancelled_renders( + &mut self, + mask: u128, + deferred: &mut DeferredBatch, + ) -> Result { + for slot in 1..self.queues.render.entries.len() { + let packet = self.queues.render.entries[slot] + .as_ref() + .and_then(|entry| entry.pair.as_ref()) + .filter(|pair| { + pair.qids() + .into_iter() + .any(|qid| mask & (1u128 << qid) != 0) + && pair.oldest_spared() == Some(true) + }) + .and_then(|pair| pair.oldest_packet()); + if let Some(packet) = packet { + self.fail_render_packet(&packet, ECANCELED, deferred)?; + } + } + Ok(()) + } + + pub(in crate::g17) fn reclaim_drained_renders( + &mut self, + deferred: &mut DeferredBatch, + ) -> Result { + if self.recovery.pending() + || self + .primary + .state + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Ok(()); + } + let mut serviced = false; + for slot in 1..self.queues.render.entries.len() { + if !self.queues.render.entries[slot] + .as_ref() + .and_then(|entry| entry.pair.as_ref()) + .is_some_and(|pair| pair.needs_drain()) + { + continue; + } + if let Err(error) = self.drain_render_slot(slot, &mut serviced, deferred) { + dev_warn!( + self.primary.state.shared.dev.as_ref(), + "Could not settle drained render pair {}: {:?}\n", + slot, + error + ); + } + } + Ok(()) + } + + fn drain_render_slot( + &mut self, + slot: usize, + serviced: &mut bool, + deferred: &mut DeferredBatch, + ) -> Result { + if !self.with_render(slot, |pair, host| pair.firmware_drained(host))? { + return Ok(()); + } + if !*serviced { + self.service_runtime_events(deferred)?; + *serviced = true; + } + if self.recovery.pending() + || self + .primary + .state + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Ok(()); + } + loop { + let accounted = self.with_render(slot, |pair, host| pair.account_drained(host))?; + if !accounted { + break; + } + match self.release_render_lists() { + Err(error) if error == EAGAIN => {} + result => result?, + } + } + if self.recovery.pending() + || self + .primary + .state + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Ok(()); + } + self.with_render(slot, |pair, host| { + pair.release_witnessed(host, &mut |event| deferred.push(event)) + })?; + self.queues.render.collect_retired(slot) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs new file mode 100644 index 0000000000000..86a00e7bd741a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Submission admission, preparation and the firmware visibility boundary. + +use super::super::{Backend, DeferredBatch, Registry}; +use crate::g17::{ + job::Packet, + queue::admission::{RenderLease, Request}, +}; +use core::sync::atomic::Ordering; +use kernel::{ + prelude::*, + sync::Arc, + time::{delay::fsleep, Delta}, +}; + +impl Backend { + pub(in crate::g17) fn render_packet( + &self, + packet: &Arc, + deferred: &mut DeferredBatch, + ) -> Result { + self.render_with_lease(packet, None, deferred) + } + + fn render_with_lease( + &self, + packet: &Arc, + early: Option, + deferred: &mut DeferredBatch, + ) -> Result { + packet.check_dependencies()?; + if packet.cancel_requested() { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let interruptible = early.is_none(); + let admission = match early { + Some(lease) => lease, + None => { + let pool = { + let state = self.shared.state.lock(); + (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() + }; + pool.acquire_render(self.owner, Request::of(packet))? + } + }; + let slot = admission.slot(); + packet.completion.attach_render_lease(admission)?; + let mut state_polls = 0; + let mut state_delay = 20i64; + let mut takeover_polls = 0; + let mut recovery_polls = 0; + loop { + if packet.cancel_requested() { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 || self.shared.crashed.load(Ordering::Acquire) + { + return Err(EIO); + } + if !self.context.is_current() { + return Err(EFAULT); + } + let work_state = { + let preparation = self.shared.preparations.enter(interruptible)?; + let result = packet.completion.work_state().acquire(); + if preparation.is_current() { + result + } else { + Err(EAGAIN) + } + }; + match work_state { + Err(error) if error == EAGAIN => continue, + Err(error) if error == EBUSY => { + if state_polls == 1000 { + return Err(ETIMEDOUT); + } + state_polls += 1; + fsleep(Delta::from_micros(state_delay)); + state_delay = (state_delay * 2).min(1000); + continue; + } + result => { + result?; + } + } + match self.ensure_render(packet, slot) { + Err(error) if error == EAGAIN => continue, + Err(error) if error == EBUSY => { + if takeover_polls == 5000 { + return Err(ETIMEDOUT); + } + takeover_polls += 1; + fsleep(Delta::from_micros(200)); + continue; + } + result => result?, + } + match self.prepare_render(slot, packet, deferred) { + Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { + let recovering = { + let state = self.shared.state.lock(); + let firmware = (*state).as_deref().ok_or(ENODEV)?; + firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + }; + if recovering { + if recovery_polls == super::super::RECOVERY_SETTLE_POLLS { + return Err(ETIMEDOUT); + } + recovery_polls += 1; + self.shared.queue_events(); + fsleep(Delta::from_millis(1)); + } else if error == EBUSY { + if takeover_polls == 5000 { + return Err(ETIMEDOUT); + } + takeover_polls += 1; + fsleep(Delta::from_micros(200)); + } + } + result => return result, + } + } + } + + pub(in crate::g17) fn try_render_early(&self, packet: &Arc) -> bool { + if self.backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { + return false; + } + let admission = { + let state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref() else { + return false; + }; + match firmware + .queues + .admission + .try_acquire_render(self.owner, Request::of(packet)) + { + Ok(Some(lease)) => lease, + _ => return false, + } + }; + if !packet.begin_publication() { + packet.completion.fail_unpublished(ECANCELED); + packet.set_early_result(Err(ECANCELED)); + return true; + } + let mut deferred = match DeferredBatch::publication() { + Ok(deferred) => deferred, + Err(error) => { + if packet.completion.status().get() != 0 { + ::fail_vm(self, error); + } + packet.completion.fail_unpublished(error); + packet.set_early_result(Err(error)); + return true; + } + }; + let result = self.render_with_lease(packet, Some(admission), &mut deferred); + let result = self + .finish_publication(packet, result, &mut deferred) + .map(|_| ()); + packet.set_early_result(result); + true + } +} + +impl Registry { + fn render_dependencies( + &self, + owner: u64, + packet: &Arc, + state: u32, + dependencies: &mut [crate::g17::fw::kick::KickDependency; 3], + ) -> Result { + let mut count = 0; + let frontier = self.render_prefix(owner, packet)?; + for dependency in frontier.as_slice() { + *dependencies.get_mut(count).ok_or(EOVERFLOW)? = *dependency; + count += 1; + } + if let Some(queue) = self.compute(owner) { + if let Some(dependency) = queue.dependency( + owner, + packet.completion.status(), + packet.order.wait_through[1], + state, + )? { + *dependencies.get_mut(count).ok_or(EOVERFLOW)? = dependency; + count += 1; + } + } + Ok(count) + } +} + +impl crate::g17::Firmware { + pub(super) fn apply_render_deferred( + &mut self, + slot: u8, + deferred: &mut DeferredBatch, + ) -> Result { + let entry = self.queues.render.entry_mut(slot)?; + let pair = entry.pair.as_mut().ok_or(EIO)?; + if let Some((status, error)) = entry.pending_error.take() { + if Arc::ptr_eq(pair.context().status(), &status) { + pair.quarantine(error, &mut |event| deferred.push(event))?; + } + } + if entry.closed { + pair.release_owner(); + } + if !entry.pending_timeout { + return Ok(()); + } + for _ in 0..crate::hw::t8140::queues::RENDER_DEPTH { + if self.recovery.pending() || self.primary.state.shared.crashed.load(Ordering::Acquire) + { + return Ok(()); + } + let Some((packet, error)) = self + .queues + .render + .entry(slot)? + .pair + .as_ref() + .ok_or(EIO)? + .take_deferred_timeout() + else { + self.queues.render.entry_mut(slot)?.pending_timeout = false; + return Ok(()); + }; + if let Err(failure) = self.fail_render_packet(&packet, error, deferred) { + packet.completion.defer_timeout(error); + return Err(failure); + } + } + self.queues.render.entry_mut(slot)?.pending_timeout = false; + Ok(()) + } + + pub(super) fn publish_render( + &mut self, + owner: u64, + slot: u8, + packet: &Arc, + deferred: &mut DeferredBatch, + ) -> Result { + use crate::g17::{fw::kick::KickDependency, queue::render::PublicationLease}; + let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; + if self.recovery.pending() || state != 0 { + return Err(EAGAIN); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let mut dependencies = [KickDependency::ZERO; 3]; + let count = self + .queues + .render_dependencies(owner, packet, state, &mut dependencies)?; + let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; + let pool = pair.pool().clone(); + let buffer_id = pair.buffer_id(); + let (buffer_id, buffer_token) = self.queues.buffers.acquire_owner(buffer_id)?; + let preparation = match self.queues.render.pools.prepare(&self.init, &pool) { + Ok(preparation) => preparation, + Err(error) => { + self.queues.buffers.release(buffer_id)?; + return Err(error); + } + }; + let lease = PublicationLease { + free_list_generation: preparation.generation, + buffer_token, + }; + let result = self.with_render(usize::from(slot), |pair, host| { + pair.publish( + packet.clone(), + &dependencies[..count], + lease, + host, + &mut |event| deferred.push(event), + ) + }); + if packet.is_published() { + self.queues.render.entry_mut(slot)?.published = true; + } else if result.is_err() { + self.queues.render.pools.rollback(preparation); + self.queues.buffers.release(buffer_id)?; + self.queues + .render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)? + .discard_prepared(packet)?; + } + result + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs new file mode 100644 index 0000000000000..759c0c2e43dc0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Exact logical ownership gates for scheduler release and physical-pair parking. + +use super::RenderState; +use crate::g17::{ + buffer::BufferIds, + context::Context, + initdata::InitData, + runtime::{teardown::ReclaimBatch, Registry}, +}; +use kernel::{prelude::*, sync::Arc}; + +impl RenderState { + pub(in crate::g17::runtime) fn observe_teardown_releases(&mut self, init: &InitData) -> Result { + self.pools.observe(init) + } + + pub(in crate::g17::runtime) fn teardown_idle(&self, context: &Arc) -> Result { + for entry in self.entries.iter().flatten() { + if !entry + .context + .as_ref() + .is_some_and(|owner| Arc::ptr_eq(owner, context)) + { + continue; + } + if entry.preparing.is_some() || entry.growing { + return Ok(false); + } + if let Some(pair) = entry.pair.as_ref() { + if !pair.idle_for_context(context)? { + return Ok(false); + } + } + } + Ok(true) + } + + pub(in crate::g17::runtime) fn teardown_pools_released(&self, context: &Arc) -> bool { + context.render_pools_released(|id, control| self.pools.released(id, control)) + && self + .retired + .iter() + .filter(|old| Arc::ptr_eq(old.context(), context)) + .all(|old| { + self.pools + .released(u32::from(old.pool().id()), old.pool().control_va()) + }) + } + + /// Warm superseded roots need only their exact executed replacement witness. + /// Graphs with their own memory remain until that Context's scheduler ack. + pub(in crate::g17::runtime) fn collect_reclaimable( + &mut self, + out: &mut ReclaimBatch, + ) -> Result { + for slot in 0..self.entries.len() { + self.collect_retired(slot)?; + } + let mut index = 0; + while index < self.retired.len() { + if self.retired[index].buffer_id().is_some() { + index += 1; + continue; + } + let Ok(target) = out.binding_slot() else { + break; + }; + *target = Some(self.retired.remove(index).map_err(|_| EIO)?); + } + Ok(()) + } + + /// Preflight the entire Context before detaching its first graph. Successor + /// graphs on the same VM have a different Arc and remain installed. + pub(in crate::g17::runtime) fn detach_context( + &mut self, + context: &Arc, + buffers: &mut BufferIds, + out: &mut ReclaimBatch, + ) -> Result { + let mut count = self + .retired + .iter() + .filter(|old| Arc::ptr_eq(old.context(), context)) + .count(); + for entry in self.entries.iter().flatten() { + if !entry + .context + .as_ref() + .is_some_and(|owner| Arc::ptr_eq(owner, context)) + { + continue; + } + if entry.preparing.is_some() || entry.growing || entry.parked.is_some() { + return Err(EBUSY); + } + if let Some(pair) = entry.pair.as_ref() { + if !pair.can_park(context)? { + return Err(EBUSY); + } + count += 1; + } + } + if !out.has_room(count, 1) { + return Err(EBUSY); + } + for entry in self.entries.iter_mut().flatten() { + if !entry + .context + .as_ref() + .is_some_and(|owner| Arc::ptr_eq(owner, context)) + { + continue; + } + let Some(pair) = entry.pair.as_ref() else { + continue; + }; + buffers.unreserve_owner(pair.buffer_id())?; + let target = out.binding_slot()?; + let pair = entry.pair.take().ok_or(EIO)?; + let (parked, retired) = KBox::into_inner(pair).park(); + *target = Some(retired); + entry.parked = Some(parked); + // Detached backing retains these references until the worker runs. + entry.context = None; + entry.pool = None; + } + let mut index = 0; + while index < self.retired.len() { + if !Arc::ptr_eq(self.retired[index].context(), context) { + index += 1; + continue; + } + if let Some(id) = self.retired[index].buffer_id() { + buffers.unreserve_owner(id)?; + } + let target = out.binding_slot()?; + *target = Some(self.retired.remove(index).map_err(|_| EIO)?); + } + Ok(()) + } +} + +impl Registry { + /// Close while a builder owns a pair records intent. Its return path applies + /// the release before admitting further commands from either logical owner. + pub(in crate::g17::runtime) fn release_render(&mut self, owner: u64) -> Result { + for entry in self.render.entries.iter_mut().flatten() { + if entry.pending_owner == Some(owner) { + entry.pending_owner_closed = true; + } + if entry.reservation.owner != owner { + continue; + } + entry.closed = true; + if let Some(pair) = entry.pair.as_mut() { + pair.release_owner(); + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs new file mode 100644 index 0000000000000..10f334e4fe561 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Scheduler release and off-lock destruction of acknowledged logical queue resources. + +use super::{ComputeEntry, Registry}; +use crate::g17::{ + buffer::BufferIds, + channel::Rings, + context::Context, + fw::channels::{ControlRecord, SchedulerStateRelease}, + initdata::InitData, + queue::render::RetiredBinding, + recovery::Memory as _, + teardown::{Failure, Host}, + Coprocessor, Firmware, MSG_CONTROL_NOTIFY, +}; +use crate::hw::t8140::queues::RENDER_SLOTS; +use core::sync::atomic::Ordering; +use kernel::{prelude::*, sync::Arc}; + +const CONTEXTS: usize = 128; + +/// Pop under the reclaim mutex, then finish after dropping every device lock. +pub(in crate::g17) enum Reclaim { + Binding(RetiredBinding), + Context(Arc), +} + +impl Reclaim { + pub(in crate::g17) fn finish(self) { + match self { + Self::Binding(binding) => drop(binding), + Self::Context(context) => drop(context), + } + } +} + +/// All slots are allocated at probe. A full collector postpones detach. Empty +/// slots let a validated transfer move ownership without a fallible vector push. +pub(in crate::g17) struct ReclaimBatch { + bindings: KVVec>, + contexts: KVec>>, +} + +impl ReclaimBatch { + pub(in crate::g17) fn new() -> Result { + let mut bindings = KVVec::with_capacity(CONTEXTS + RENDER_SLOTS, GFP_KERNEL)?; + for _ in 0..CONTEXTS + RENDER_SLOTS { + bindings.push(None, GFP_KERNEL)?; + } + let mut contexts = KVec::with_capacity(CONTEXTS, GFP_KERNEL)?; + for _ in 0..CONTEXTS { + contexts.push(None, GFP_KERNEL)?; + } + Ok(Self { bindings, contexts }) + } + + pub(in crate::g17) fn is_empty(&self) -> bool { + self.bindings.iter().all(Option::is_none) && self.contexts.iter().all(Option::is_none) + } + + /// Graphs go before the Context owners that armed their scheduler releases. + pub(in crate::g17) fn take_one(&mut self) -> Option { + if let Some(slot) = self.bindings.iter_mut().find(|slot| slot.is_some()) { + return slot.take().map(Reclaim::Binding); + } + self.contexts + .iter_mut() + .find(|slot| slot.is_some()) + .and_then(Option::take) + .map(Reclaim::Context) + } + + pub(in crate::g17::runtime) fn has_room(&self, bindings: usize, contexts: usize) -> bool { + self.bindings.iter().filter(|slot| slot.is_none()).count() >= bindings + && self.contexts.iter().filter(|slot| slot.is_none()).count() >= contexts + } + + pub(in crate::g17::runtime) fn binding_slot(&mut self) -> Result<&mut Option> { + self.bindings + .iter_mut() + .find(|slot| slot.is_none()) + .ok_or(EBUSY) + } + + fn context(&mut self, context: &Arc) -> Result { + let slot = self + .contexts + .iter_mut() + .find(|slot| slot.is_none()) + .ok_or(EBUSY)?; + *slot = Some(context.clone()); + Ok(()) + } +} + +struct Service<'a> { + render: &'a mut super::render::RenderState, + compute: &'a [Option], + buffers: &'a mut BufferIds, + init: &'a InitData, + primary: &'a mut Coprocessor, + reclaim: &'a mut ReclaimBatch, +} + +impl Host for Service<'_> { + fn render_idle(&mut self, context: &Arc) -> Result { + self.render.teardown_idle(context) + } + fn compute_idle(&self, context: &Arc) -> bool { + self.compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_ref()) + .all(|queue| queue.idle_for_context(context)) + } + fn flists_released(&self, context: &Arc) -> bool { + self.render.teardown_pools_released(context) + } + fn control_backpressured(&self) -> bool { + self.render.control_backpressured() + } + fn publish(&mut self, record: &SchedulerStateRelease) -> Result { + Rings::publish_control( + self.init, + &ControlRecord::SchedulerStateRelease(SchedulerStateRelease::new( + record.page_va, + record.id, + )), + ) + } + fn notify(&mut self) -> Result { + self.primary.notify(MSG_CONTROL_NOTIFY) + } + fn detach(&mut self, context: &Arc) -> Result { + self.render + .detach_context(context, self.buffers, self.reclaim)?; + // detach_context preflights this slot together with every graph slot. + // Retain it before Pending removes its reference under the device lock. + self.reclaim.context(context) + } + fn report(&mut self, context: &Arc, failure: Failure) { + let dev = self.primary.state.shared.dev.as_ref(); + match failure { + Failure::Publication(error) => dev_err!( + dev, + "Scheduler {} release failed: {:?}\n", + context.id(), + error + ), + Failure::Announcement(error) => dev_warn!( + dev, + "Scheduler {} release announcement failed: {:?}\n", + context.id(), + error + ), + Failure::Stalled => { + dev_warn!(dev, "Scheduler {} release is still pending\n", context.id()) + } + } + } +} + +impl Firmware { + /// The event worker has already reclaimed drained pairs and reannounced + /// pending controls. Recovery blocks new releases, never an observed ack. + pub(in crate::g17) fn service_teardown(&mut self, reclaim: &mut ReclaimBatch) -> Result { + self.queues.render.collect_reclaimable(reclaim)?; + if self.queues.teardown.is_empty() { + return Ok(()); + } + self.queues.render.observe_teardown_releases(&self.init)?; + let recovery = self.recovery.pending() || self.init.recovery_state()? != 0; + let Registry { + render, + compute, + buffers, + teardown, + .. + } = &mut self.queues; + let mut service = Service { + render, + compute, + buffers, + init: &self.init, + primary: &mut self.primary, + reclaim, + }; + teardown.publish_ready(&mut service, recovery)?; + let consumer = self.init.control_consumer()?.load(Ordering::Relaxed); + let producer = self.init.control_producer()?.load(Ordering::Relaxed); + while teardown + .take_consumed(consumer, producer, &mut service)? + .is_some() + {} + teardown.report_stalled(super::now_ns(), &mut service); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/status.rs b/drivers/gpu/drm/asahi/g17/status.rs new file mode 100644 index 0000000000000..831c6813b7084 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/status.rs @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Sticky VM errors and their optional userspace mirror. No firmware state is accessed here. + +use crate::gem; +use core::cell::UnsafeCell; +use core::sync::atomic::{AtomicBool, AtomicI32, AtomicPtr, Ordering}; +use kernel::prelude::*; + +const MIRROR_MAX_SIZE: usize = 64 * 1024; + +/// A VM and its accepted jobs share this status, including after queue destruction. +/// The first error is permanent. Callers record it before signalling the job's fence. +pub(crate) struct VmStatus { + error: AtomicI32, + mirror: AtomicPtr, + mirror_claimed: AtomicBool, + mirror_owner: UnsafeCell>, +} + +// SAFETY: Only the winner of mirror_claimed writes mirror_owner. The owner is never +// read or replaced while the status lives, and its mapping is retained until drop. +// Other shared accesses use atomics, including accesses through the mirror pointer. +unsafe impl Sync for VmStatus {} +// SAFETY: The retained GEM reference and mapping can move between threads. Moving +// the status does not move that mapping, and its remaining fields are atomic. +unsafe impl Send for VmStatus {} + +impl VmStatus { + pub(crate) fn new() -> Self { + Self { + error: AtomicI32::new(0), + mirror: AtomicPtr::new(core::ptr::null_mut()), + mirror_claimed: AtomicBool::new(false), + mirror_owner: UnsafeCell::new(None), + } + } + + pub(crate) fn get(&self) -> i32 { + self.error.load(Ordering::Acquire) + } + + /// Store the first error in both the status and any registered mirror. + pub(crate) fn record(&self, error: Error) { + let error = match self.error.compare_exchange( + 0, + error.to_errno(), + Ordering::SeqCst, + Ordering::SeqCst, + ) { + Ok(_) => error.to_errno(), + Err(first) => first, + }; + let word = self.mirror.load(Ordering::SeqCst); + if !word.is_null() { + // SAFETY: attach_mirror publishes only an aligned word within a mapping + // owned by this status. Every kernel access to this word is atomic. + unsafe { AtomicI32::from_ptr(word) }.store(error, Ordering::SeqCst); + } + } + + /// Rejected submissions and contention leave the VM usable. + pub(crate) fn record_if_device_loss(&self, error: Error) { + if error == EIO || error == ETIMEDOUT || error == ENODEV { + self.record(error); + } + } + + /// Retain one mapped GEM word until the last reference to this status is dropped. + pub(crate) fn attach_mirror(&self, mut owner: gem::ObjectRef, offset: usize) -> Result { + if owner.size() > MIRROR_MAX_SIZE + || offset % core::mem::align_of::() != 0 + || offset + .checked_add(core::mem::size_of::()) + .is_none_or(|end| end > owner.size()) + { + return Err(EINVAL); + } + let base = owner.vmap()?.addr.cast::(); + // SAFETY: The checked range is inside the retained, page-aligned mapping. + let word = unsafe { base.add(offset) }.cast::(); + if self.mirror_claimed.swap(true, Ordering::AcqRel) { + return Err(EBUSY); + } + // SAFETY: This is the only thread that may write mirror_owner. It is not + // read again until the status is dropped, after all shared references end. + unsafe { *self.mirror_owner.get() = Some(owner) }; + // SAFETY: The retained owner keeps this checked, aligned word mapped. + unsafe { AtomicI32::from_ptr(word) }.store(0, Ordering::SeqCst); + + // The pointer publication and error update form a store/load handshake: + // either this load sees record's error or record sees our pointer. Stores + // after publishing the pointer write only the same first error, never zero. + self.mirror.store(word, Ordering::SeqCst); + let error = self.error.load(Ordering::SeqCst); + if error != 0 { + // SAFETY: The mapping remains owned by this status as above. + unsafe { AtomicI32::from_ptr(word) }.store(error, Ordering::SeqCst); + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs new file mode 100644 index 0000000000000..9acc01fbc2125 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Retained scheduler-state releases. Logical close arms a release; render and compute drain, +//! private-memory release and control-ring consumption are separate ownership witnesses. +//! The device mutex serializes this state with the Registry's displaced graph ownership. + +use super::{context::Context, freelist::control_consumed, fw::channels::SchedulerStateRelease}; +use core::sync::atomic::{fence, Ordering}; +use kernel::{prelude::*, sync::Arc}; + +const CAPACITY: usize = 128; +const POLL_NS: u64 = 2_000_000_000; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Phase { + Deferred, + Draining, + Released(u32), +} + +struct Entry { + context: Arc, + phase: Phase, + armed: u64, + warned: bool, +} + +/// Publication and announcement errors retain the entry. A stalled release is reported once; +/// its timeout never authorizes freeing memory still owned by firmware. +#[derive(Clone, Copy)] +pub(crate) enum Failure { + Publication(Error), + Announcement(Error), + Stalled, +} + +/// Borrowed device operations. Readiness covers every matching installed and displaced graph, +/// including pools kept by the Context after a same-VM successor took over its physical pair. +/// `detach` must retain all detached backing for off-lock destruction before returning success. +/// It compares logical Arc identity so a successor sharing an execution ID remains installed. +pub(crate) trait Host { + fn render_idle(&mut self, context: &Arc) -> Result; + fn compute_idle(&self, context: &Arc) -> bool; + fn flists_released(&self, context: &Arc) -> bool; + fn control_backpressured(&self) -> bool; + fn publish(&mut self, record: &SchedulerStateRelease) -> Result; + fn notify(&mut self) -> Result; + fn detach(&mut self, context: &Arc) -> Result; + fn report(&mut self, context: &Arc, failure: Failure); +} + +pub(crate) struct Pending { + entries: KVec, +} + +impl Pending { + pub(crate) fn new() -> Result { + Ok(Self { + entries: KVec::with_capacity(CAPACITY, GFP_KERNEL)?, + }) + } + + pub(crate) fn closed(&self, index: usize, now: u64) -> Option<(Arc, u64)> { + self.entries + .get(index) + .filter(|entry| entry.phase == Phase::Draining) + .map(|entry| (entry.context.clone(), now.saturating_sub(entry.armed))) + } + + pub(crate) fn len(&self) -> usize { + self.entries.len() + } + + pub(crate) fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + fn find(&self, context: &Arc) -> Option { + self.entries + .iter() + .position(|entry| Arc::ptr_eq(&entry.context, context)) + } + + fn insert(&mut self, context: &Arc, phase: Phase, now: u64) -> Result { + self.entries + .push_within_capacity(Entry { + context: context.clone(), + phase, + armed: now, + warned: false, + }) + .map_err(|_| ENOSPC) + } + + /// Call before replacing a graph's logical owner. Failure leaves that graph with its + /// original owner. The Registry, rather than this state machine, retains displaced backing. + pub(crate) fn displace( + &mut self, + context: &Arc, + owner_closed: bool, + now: u64, + ) -> Result { + if self.find(context).is_some() { + return Ok(()); + } + let phase = if owner_closed { + Phase::Draining + } else { + Phase::Deferred + }; + self.insert(context, phase, now) + } + + /// No allocation is needed at close. A scheduler page never exposed to firmware requires + /// no release, unless the Registry already retained one of its displaced graphs. + pub(crate) fn close(&mut self, context: &Arc, now: u64) -> Result { + if let Some(index) = self.find(context) { + let entry = &mut self.entries[index]; + if entry.phase == Phase::Deferred { + entry.phase = Phase::Draining; + entry.armed = now; + } + return Ok(()); + } + if context.is_published() { + self.insert(context, Phase::Draining, now)?; + } + Ok(()) + } + + /// Publish in insertion order. Mandatory control replies and a full control ring stop this + /// pass; an error for one page leaves it retained while other ready releases may progress. + pub(crate) fn publish_ready(&mut self, host: &mut impl Host, recovery: bool) -> Result { + if !recovery { + for entry in self.entries.iter_mut() { + if entry.phase != Phase::Draining + || !host.render_idle(&entry.context)? + || !host.flists_released(&entry.context) + || !host.compute_idle(&entry.context) + { + continue; + } + if host.control_backpressured() { + break; + } + let result = (|| { + let id = entry.context.release_identity()?; + let record = SchedulerStateRelease::new( + entry.context.scheduler_va(), + [id[3], id[0], id[1], id[2]], + ); + host.publish(&record) + })(); + let next = match result { + Ok(next) => next, + Err(EAGAIN) => break, + Err(error) => { + host.report(&entry.context, Failure::Publication(error)); + continue; + } + }; + entry.phase = Phase::Released(next); + fence(Ordering::SeqCst); + if let Err(error) = host.notify() { + host.report(&entry.context, Failure::Announcement(error)); + } + } + } + fence(Ordering::SeqCst); + Ok(()) + } + + /// The caller snapshots both control cursors once for this settlement pass and repeatedly + /// takes consumed entries. It retains each returned Context until its detached graphs and + /// VM aliases can be destroyed outside the device mutex. + pub(crate) fn take_consumed( + &mut self, + consumer: u32, + producer: u32, + host: &mut impl Host, + ) -> Result>> { + for index in 0..self.entries.len() { + let entry = &self.entries[index]; + let Phase::Released(target) = entry.phase else { + continue; + }; + if !control_consumed(consumer, producer, target)? + || !host.flists_released(&entry.context) + { + continue; + } + match host.detach(&entry.context) { + Ok(()) => {} + Err(EBUSY) => continue, + Err(error) => return Err(error), + } + let entry = self.entries.remove(index).map_err(|_| EIO)?; + return Ok(Some(entry.context)); + } + Ok(None) + } + + /// Poll quickly only during the initial release window; late arrivals still settle when + /// serviced. Time passage never substitutes for a firmware-consumption witness. + pub(crate) fn polling(&self, now: u64) -> bool { + self.entries.iter().any(|entry| { + entry.phase != Phase::Deferred && now.saturating_sub(entry.armed) < POLL_NS + }) + } + + pub(crate) fn report_stalled(&mut self, now: u64, host: &mut impl Host) { + for entry in self.entries.iter_mut() { + if entry.phase != Phase::Deferred + && !entry.warned + && now.saturating_sub(entry.armed) >= POLL_NS + { + entry.warned = true; + host.report(&entry.context, Failure::Stalled); + } + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/validation.rs b/drivers/gpu/drm/asahi/g17/validation.rs new file mode 100644 index 0000000000000..da43c10cd3bef --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/validation.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Reuse address-coverage proofs only while the VM's page tables are unchanged. + +use super::command::{Access, AddressSpace, Payload, UscWindow, Validated}; +use crate::mmu; +use core::{cell::RefCell, ops::Range}; +use kernel::prelude::*; + +const MAX_RANGES: usize = 48; +type Request = (u64, u64, u8); + +#[derive(Clone, Copy)] +struct Requests { + ranges: [Request; MAX_RANGES], + count: usize, + overflow: bool, +} + +impl Requests { + const fn new() -> Self { + Self { + ranges: [(0, 0, 0); MAX_RANGES], + count: 0, + overflow: false, + } + } + + fn as_slice(&self) -> &[Request] { + &self.ranges[..self.count] + } + + fn insert(&mut self, request: Request) { + if self.as_slice().contains(&request) { + return; + } + if self.count == MAX_RANGES { + self.overflow = true; + return; + } + self.ranges[self.count] = request; + self.count += 1; + } +} + +/// One logical queue's bounded cache of exact address, size and permission tuples. +/// Mapping mutations make the epoch unavailable until their final page-table write. +pub(crate) struct Proof { + epoch: Option, + requests: Requests, +} + +impl Proof { + pub(crate) const fn new() -> Self { + Self { + epoch: None, + requests: Requests::new(), + } + } + + fn covers(&self, epoch: Option, requests: &Requests) -> bool { + self.epoch.is_some() + && self.epoch == epoch + && requests + .as_slice() + .iter() + .all(|request| self.requests.as_slice().contains(request)) + } + + fn record(&mut self, epoch: Option, requests: Requests) { + // Alternating in-flight buffers must not evict each other's proofs. + if epoch.is_some() && self.epoch == epoch && !requests.overflow { + for &request in requests.as_slice() { + self.requests.insert(request); + } + if !self.requests.overflow { + return; + } + } + // An incomplete union cannot be reused; retain the latest complete set. + self.epoch = if requests.overflow { None } else { epoch }; + self.requests = requests; + } + + /// The VM-job guard must already pin the mappings for this command. Keep the + /// range arrays off the caller's frame before descending into publication. + #[inline(never)] + pub(crate) fn validate( + &mut self, + vm: &mmu::Vm, + range: Range, + window: &UscWindow, + payload: &Payload, + ) -> Result { + let requests = RefCell::new(Requests::new()); + let optimistic = payload.validate(window, &Space::Record(&range, &requests)); + let requests = requests.into_inner(); + if optimistic.is_ok() && !requests.overflow { + if self.covers(vm.mapping_validation_epoch(), &requests) { + return optimistic; + } + let (covered, epoch) = vm.covers_ranges_batch(requests.as_slice()); + if covered { + self.record(epoch, requests); + return optimistic; + } + self.epoch = None; + } + // Preserve per-range validation's failure and concurrent-mutation behavior. + payload.validate(window, &Space::Checked(&range, vm)) + } +} + +enum Space<'a> { + Record(&'a Range, &'a RefCell), + Checked(&'a Range, &'a mmu::Vm), +} + +impl AddressSpace for Space<'_> { + fn covers(&self, address: u64, size: u64, access: Access) -> bool { + let range = match self { + Self::Record(range, _) | Self::Checked(range, _) => range, + }; + let Some(end) = address.checked_add(size) else { + return false; + }; + if size == 0 || address < range.start || end > range.end { + return false; + } + let permissions = match access { + Access::Read => 1, + Access::Write => 2, + Access::ReadWrite => 3, + }; + match self { + Self::Record(_, requests) => { + requests.borrow_mut().insert((address, size, permissions)); + true + } + Self::Checked(_, vm) => { + vm.covers_range(address, size, permissions & 1 != 0, permissions & 2 != 0) + } + } + } +} diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 027d9b052fa5e..db49d418b0382 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -159,11 +159,23 @@ pub(crate) struct AsahiObjConfig { /// Create a new kernel-owned GEM object. pub(crate) fn new_kernel_object(dev: &AsahiDrmDevice, size: usize) -> Result { + new_kernel_object_mapped(dev, size, false) +} + +/// Create a new kernel-owned GEM object whose kernel mapping is write-combined. +/// +/// Used for memory that a non-coherent agent reads as soon as the host has stored to it and +/// issued a barrier, without cache maintenance. +pub(crate) fn new_kernel_object_wc(dev: &AsahiDrmDevice, size: usize) -> Result { + new_kernel_object_mapped(dev, size, true) +} + +fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> Result { let gem = shmem::Object::::new( dev, align(size, mmu::UAT_PGSZ), shmem::ObjectConfig:: { - map_wc: false, + map_wc, parent_resv_obj: None, }, AsahiObjConfig { diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index e2cb5a6a87198..90db79eaeac64 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -266,7 +266,7 @@ pub(crate) struct GpuManager { /// Operations the DRM driver performs on a GPU, independent of its firmware interface. pub(crate) trait Gpu: Send + Sync { /// Cast as an Any type. - fn _as_any(&self) -> &dyn Any; + fn as_any(&self) -> &dyn Any; /// Cast Arc as an Any type. fn arc_as_any(self: Arc) -> Arc; /// Return the G13/G14 firmware manager, if this GPU uses that firmware interface. @@ -279,6 +279,15 @@ pub(crate) trait Gpu: Send + Sync { fn is_crashed(&self) -> bool; /// Fill in the hardware description reported to userspace. fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; + /// Whether user VMs retain a sticky error and support a userspace mirror. + fn supports_vm_status(&self) -> bool { + false + } + /// Limits for independently executing queues, if supported by this GPU. + fn queue_limits(&self) -> Option { + None + } + /// Frequency of the CPU timer that GPU timestamps are derived from. fn base_clock_hz(&self) -> u32; /// Return the GPU address space geometry. @@ -686,6 +695,7 @@ impl GpuManager::ver { oas: cfg.uat_oas, }, map_kernel_to_user, + mmu::UatFirmware::Handoff, )?, GFP_KERNEL, )?) @@ -1240,7 +1250,7 @@ impl GpuManager::ver { #[versions(AGX)] impl Gpu for GpuManager::ver { - fn _as_any(&self) -> &dyn Any { + fn as_any(&self) -> &dyn Any { self } diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index e221d02ff37eb..5303a386c13dc 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -15,6 +15,7 @@ pub(crate) mod t600x; pub(crate) mod t602x; pub(crate) mod t8103; pub(crate) mod t8112; +pub(crate) mod t8140; /// GPU generation enumeration. Note: Part of the UABI. #[derive(Debug, PartialEq, Copy, Clone)] @@ -22,6 +23,7 @@ pub(crate) mod t8112; pub(crate) enum GpuGen { G13 = 13, G14 = 14, + G17 = 17, } /// GPU variant enumeration. Note: Part of the UABI. diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs new file mode 100644 index 0000000000000..4a97217a00705 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -0,0 +1,392 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t8140 platforms (A18 Pro). + +use crate::g17::{ + Config, + FixedBuffer, + Identity, // +}; +use crate::hw::GpuVariant; + +/// Static configuration of the T8140 GPU. +pub(crate) const CONFIG: Config = Config { + chip_id: CHIP_ID, + identity: Identity { + family: 0xa, + variant: 0, + num_dies: 1, + gpu_variant: GpuVariant::P, + usc_generation: 3, + hal_generation: 200, + }, + uat_ias: 42, + uat_oas: 42, + // The secondary coprocessor's kernel root follows the primary one's 256 KiB of page tables. + secondary_root_offset: 0x40000, + // The coprocessors request their crash log buffers at these addresses, in this order, and + // expect them to be mapped before they start. + crash_buffers: [ + FixedBuffer { + va: 0x01e1_c000, + size: 0x8000, + }, + FixedBuffer { + va: 0x01f5_0000, + size: 0x8000, + }, + ], + // Read by the firmware through the kernel lower root, whose top-level entries it only + // samples when it starts. + pm_metrics: FixedBuffer { + va: 0x10_0008_0000, + size: 0x8000, + }, +}; + +use crate::g17::fw::initdata::{PerfConfig, RegisterWindow, PERF_STATES}; + +/// Chip identifier written to the hardware data. +pub(crate) const CHIP_ID: u32 = 0x8140; + +/// Base of the firmware MMIO aperture in the kernel window. +const MMIO_APERTURE: u64 = 0xffff_fc21_8000_0000; + +const fn window( + slot: usize, + phys: u64, + offset: u64, + size: u32, + unk_18: u64, + unk_20: u32, +) -> RegisterWindow { + RegisterWindow { + slot, + phys, + va: MMIO_APERTURE + offset, + size, + unk_18, + unk_20, + } +} + +/// Register windows the firmware maps, by register-map slot. +pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 18] = [ + window(0, 0x3_0101_4000, 0x0_0000, 0x4000, 0, 2), + window(3, 0x2_2010_4000, 0x0_8000, 0x1_8000, 0, 2), + window(9, 0x3_003d_0000, 0x2_8000, 0x1000, 0, 2), + window(10, 0x3_003c_0000, 0x3_0000, 0x2000, 0, 0), + window(12, 0x4_0165_c000, 0x3_8000, 0x4000, 0, 2), + window(14, 0x3_0028_0000, 0x4_0000, 0x8000, 0, 0), + window(17, 0x4_8000_0000, 0x5_0000, 0x2_1400, 0, 2), + window(22, 0x4_8100_0000, 0x7_8000, 0x8000, 0, 2), + window(26, 0x4_80d0_4000, 0x8_8000, 0x8000, 0xd0_4000, 2), + window(27, 0x4_80d0_d000, 0x9_9000, 0x1000, 0xd0_d000, 2), + window(28, 0x4_80d5_8000, 0xa_0000, 0x8000, 0xd5_8000, 2), + window(29, 0x4_80d1_0000, 0xb_0000, 0x4000, 0xd1_0000, 2), + window(31, 0x4_80d4_0000, 0xb_8000, 0x4000, 0xd4_0000, 2), + window(32, 0x4_80d6_0000, 0xc_0000, 0x4000, 0xd6_0000, 2), + window(35, 0x4_80e0_0000, 0xc_8000, 0x4000, 0xe0_0000, 2), + window(39, 0x4_80e0_8000, 0xd_0000, 0x8000, 0, 2), + window(40, 0x4_80e1_c000, 0xe_0000, 0x4000, 0xe1_c000, 2), + window(41, 0x4_80e1_f800, 0xe_b800, 0x4000, 0, 2), +]; + +/// Register-map slots that are reserved but have no window. +pub(crate) const RESERVED_REGISTER_SLOTS: [usize; 17] = [ + 2, 5, 6, 7, 8, 30, 33, 34, 37, 38, 42, 43, 46, 47, 48, 49, 52, +]; + +/// Performance-table configuration. The device tree lists the operating +/// points in performance-state order; table A uses the odd states above +/// state 5, table B the even ones. +pub(crate) const PERF: PerfConfig = PerfConfig { + states_a: [0, 1, 2, 3, 4, 5, 7, 9, 11, 13, 15], + states_b: [0, 1, 2, 3, 4, 5, 6, 8, 10, 12, 14], + relative_a: [0, 15, 22, 29, 42, 56, 74, 86, 90, 92, 100], + relative_b: [0, 0, 13, 24, 40, 52, 63, 73, 83, 88, 100], + sram_min_mv: 765, +}; + +/// Operating points the performance tables reference. +pub(crate) const OPPS: usize = 16; + +const _: () = { + let mut index = 0; + while index < PERF_STATES { + assert!((PERF.states_a[index] as usize) < OPPS); + assert!((PERF.states_b[index] as usize) < OPPS); + index += 1; + } +}; + +/// Offsets of the fixed objects in the kernel window, in the order the +/// builder creates them. +pub(crate) mod kernel_window { + /// Private cluster (`initdata::private`). + pub(crate) const PRIVATE_CLUSTER: u64 = 0x2_0000; + /// Zeroed firmware-private page. + pub(crate) const ZERO_PAGE: u64 = 0x15d_8000; + /// Parameter-buffer descriptor table. + pub(crate) const PB_DESCRIPTORS: u64 = 0x15e_0000; + /// page-pool descriptor table. + pub(crate) const PAGE_POOL_DESCRIPTORS: u64 = 0x15e_8000; + /// Index object (64 KiB, `initdata::IndexTable` in its first page). + pub(crate) const INDEX: u64 = 0xc084_8000; + /// Firmware-control page. + pub(crate) const FWCTL: u64 = 0x1c_0000; + /// Render free-list control and state pages. + pub(crate) const RENDER_FREE_LIST: u64 = 0xc083_0000; + pub(crate) const RENDER_FREE_LIST_STATE: u64 = 0x160_8000; + /// Boot free-list control and state pages. + pub(crate) const BOOT_FREE_LIST: u64 = 0xc082_8000; + pub(crate) const BOOT_FREE_LIST_STATE: u64 = 0x160_0000; + /// Zeroed descriptor pages. + pub(crate) const DESCRIPTOR_PAGE_A: u64 = 0x1c_8000; + pub(crate) const DESCRIPTOR_PAGE_B: u64 = 0xc07c_0000; + /// Zeroed TA and 3D context pages (128 KiB each). + pub(crate) const TA_CONTEXT: u64 = 0x1d_8000; + pub(crate) const FRAGMENT_CONTEXT: u64 = 0x20_0000; + /// Compute free-list control page and the state page it names (the + /// state page is not mapped at boot). + pub(crate) const COMPUTE_FREE_LIST: u64 = 0xc086_8000; + pub(crate) const COMPUTE_FREE_LIST_STATE: u64 = 0x163_0000; + /// Second compute free-list control page and its state page. + pub(crate) const COMPUTE_READY_FREE_LIST: u64 = 0xc087_8000; + pub(crate) const COMPUTE_READY_FREE_LIST_STATE: u64 = 0x164_8000; + /// Further compute free-list state page. + pub(crate) const COMPUTE_STATE: u64 = 0x165_0000; + /// Hardware-data bundle (`initdata::bundle`). + pub(crate) const BUNDLE: u64 = 0xc078_8000; +} + +/// Arenas used by dynamically allocated firmware objects. +pub(crate) mod dynamic { + /// First dynamic byte, relative to the upper kernel window. + pub(crate) const KERNEL_OFFSET: u64 = 0x168_4000; + /// Client-private compute and render backing after the shared queue arena. + pub(crate) const CLIENT_LOWER: core::ops::Range = 0x70_8000_0000..0x71_0000_0000; + /// Canonical low mappings shared by hardware queues and their clients. + /// Client-private allocations start after this arena. + pub(crate) const LOWER: core::ops::Range = 0x70_0400_0000..0x70_8000_0000; +} + +/// Fixed addresses in the firmware's context-0 and context-1 address spaces. +pub(crate) mod low { + /// Context-0 alias of the parameter-buffer descriptor table. + pub(crate) const PB_DESCRIPTORS: u64 = 0x70_0183_8000; + /// Context-0 alias of the page-pool descriptor table. + pub(crate) const PAGE_POOL_DESCRIPTORS: u64 = 0x70_0184_0000; + /// Context-0 alias of the index object. + pub(crate) const INDEX: u64 = 0x10_0019_0000; + /// Context-0 aliases of the completion rings. + pub(crate) const COMPLETION_RINGS: [u64; 2] = [0x70_03f0_0000, 0x70_03f2_8000]; + /// Context-0 run list of the compute free list. + pub(crate) const COMPUTE_RUN_LIST: u64 = 0x70_03ad_8000; + + /// Render free list (context 1): page list. + pub(crate) const RENDER_PAGE_LIST: u64 = 0x70_0000_0000; + /// Size of the render page list. + pub(crate) const RENDER_PAGE_LIST_SIZE: usize = 0x20_0000; + /// Render free list: run list (64 KiB, zero at boot). + pub(crate) const RENDER_RUN_LIST: u64 = 0x70_0020_8000; + /// Size of the render run list. + pub(crate) const RENDER_RUN_LIST_SIZE: usize = 0x1_0000; + /// Render free list: first block. + pub(crate) const RENDER_BLOCKS: u64 = 0x70_0022_0000; + /// Distance between consecutive blocks. + pub(crate) const BLOCK_STRIDE: u64 = 0x10_8000; + /// Render blocks allocated at boot. + pub(crate) const RENDER_BLOCKS_ALLOCATED: usize = 28; + /// Render blocks mapped at boot. + pub(crate) const RENDER_BLOCKS_MAPPED: usize = 22; + /// Render blocks in the free list at boot. + pub(crate) const RENDER_BLOCKS_INITIAL: u32 = 17; + /// Render block holding the compute page list. + pub(crate) const COMPUTE_PAGE_LIST_BLOCK: usize = 22; + + /// Distance of the compute pool from the render pool. + const COMPUTE: u64 = 0x200_0000; + /// Compute pool: page list. + pub(crate) const COMPUTE_POOL_PAGE_LIST: u64 = RENDER_PAGE_LIST + COMPUTE; + /// Compute pool: run list. + pub(crate) const COMPUTE_POOL_RUN_LIST: u64 = RENDER_RUN_LIST + COMPUTE; + /// Compute pool: first block. + pub(crate) const COMPUTE_POOL_BLOCKS: u64 = RENDER_BLOCKS + COMPUTE; + /// Blocks the compute free list declares. + pub(crate) const COMPUTE_BLOCKS: u32 = 18; + /// Compute pool blocks in the second compute free list. + pub(crate) const COMPUTE_READY_BLOCKS: u32 = 17; + /// Page list of the compute free list (inside the block + /// [`COMPUTE_PAGE_LIST_BLOCK`] holds, relocated to the compute pool). + pub(crate) const COMPUTE_PAGE_LIST: u64 = + RENDER_BLOCKS + COMPUTE_PAGE_LIST_BLOCK as u64 * BLOCK_STRIDE + COMPUTE; + /// Two zeroed compute blocks. + pub(crate) const COMPUTE_ZERO_BLOCKS: [u64; 2] = [ + RENDER_PAGE_LIST + COMPUTE, + RENDER_PAGE_LIST + BLOCK_STRIDE + COMPUTE, + ]; + + /// Returns block `index` of the render pool. + pub(crate) const fn render_block(index: usize) -> u64 { + RENDER_BLOCKS + index as u64 * BLOCK_STRIDE + } + + /// Returns block `index` of the compute pool. + pub(crate) const fn compute_block(index: usize) -> u64 { + COMPUTE_POOL_BLOCKS + index as u64 * BLOCK_STRIDE + } + + /// Blocks of the compute free list, in run-list order: pool blocks 0 to + /// 17, the two zeroed blocks, then pool block 18. + pub(crate) const COMPUTE_FREE_LIST_BLOCKS: [u64; 21] = { + let mut blocks = [0; 21]; + let mut index = 0; + while index < 18 { + blocks[index] = compute_block(index); + index += 1; + } + blocks[18] = COMPUTE_ZERO_BLOCKS[0]; + blocks[19] = COMPUTE_ZERO_BLOCKS[1]; + blocks[20] = compute_block(18); + blocks + }; +} + +/// Client-root reservations and parameter-buffer geometry. +pub(crate) mod resources { + use core::ops::Range; + + pub(crate) const UAT_PAGE: u64 = 0x4000; + pub(crate) const USER_BASE: u64 = 0x10_0000_0000; + pub(crate) const TIMESTAMPS: Range = 0x70_0000_0000..0x71_0000_0000; + pub(crate) const COMMAND_TIMESTAMP_HZ: u64 = 24_000_000; + pub(crate) const USER_END: u64 = USER_BASE + (1 << 32); + pub(crate) const FIXED_PAIRS: usize = 2; + pub(crate) const FIXED_PAIR_STRIDE: u64 = 0x8_0000; + pub(crate) const SCENE: u64 = USER_BASE + 0x17_8000; + pub(crate) const SCENE_SIZE: u64 = 0x8000; + pub(crate) const DISCARD: u64 = USER_BASE + 0x18_0000; + pub(crate) const DISCARD_SIZE: u64 = 0x8000; + pub(crate) const PAGE_LIST: u64 = USER_BASE + 0x19_0000; + pub(crate) const FRAGMENT_STATUS: u64 = USER_BASE + 0x1a_8000; + pub(crate) const RENDER_ALIASES: Range = 0x10_0200_0000..0x10_0220_0000; + pub(crate) const TARGETS: Range = 0x10_5000_0000..0x10_6400_0000; + pub(crate) const TARGET_RECORD_MIN: usize = 0x4_0000; + pub(crate) const TARGET_RECORD_MAX: usize = 0x80_0000; + pub(crate) const TPC_MIN: usize = 0x3_0000; + pub(crate) const TPC_MAX: usize = 0x200_0000; + pub(crate) const AUXILIARY: Range = 0x100_0200_0000..0x100_0240_0000; + pub(crate) const METRICS_SIZE: u64 = 0x8000; + pub(crate) const TVB_PAGE: u64 = 0x8000; + pub(crate) const TVB_BLOCK_SIZE: usize = 0x2_0000; + pub(crate) const TVB_BLOCK_STRIDE: u64 = 0x2_8000; + pub(crate) const TVB_BLOCK_SLOTS: usize = 0xd1a; + pub(crate) const TVB_MAX_BLOCKS: usize = TVB_BLOCK_SLOTS - 1; + pub(crate) const TVB_PAGES_PER_BLOCK: usize = 4; + pub(crate) const TVB_PAGE_LIST_SIZE: usize = TVB_BLOCK_SLOTS * TVB_PAGES_PER_BLOCK * 4; + pub(crate) const TVB_INITIAL_PAGES: [u32; 32] = [ + 0x11, 0x16, 0x1b, 0x20, 0x25, 0x2a, 0x4a, 0x4f, 0x54, 0x59, 0x5e, 0x63, 0x68, 0x6d, 0x72, + 0x77, 0x7c, 0x81, 0x86, 0x8b, 0x90, 0x95, 0x9a, 0x9f, 0xa4, 0xa9, 0xae, 0xb3, 0xb8, 0xbd, + 0xc2, 0xc7, + ]; + pub(crate) const TVB_GROWTH: u64 = USER_BASE + 0x0400_0000; + pub(crate) const TVB_PAIR_STRIDE: u64 = 0x2800_0000; + pub(crate) const RESERVED_RANGES: usize = 48; + + /// Compact allocations search these shared client arenas in this order. + pub(crate) fn compact_ranges() -> [Range; 4] { + let second = TVB_GROWTH + TVB_PAIR_STRIDE; + [ + RENDER_ALIASES, + TVB_GROWTH + ..TVB_GROWTH + (TVB_MAX_BLOCKS - TVB_INITIAL_PAGES.len()) as u64 * TVB_BLOCK_STRIDE, + second..second + TVB_MAX_BLOCKS as u64 * TVB_BLOCK_STRIDE, + TARGETS, + ] + } + + /// All driver-owned windows reserved before the first userspace bind. + /// Reserving them does not install pages or allocate backing memory. + pub(crate) fn client_reserved_ranges(metrics_va: u64) -> Option<[Range; RESERVED_RANGES]> { + let mut ranges = core::array::from_fn(|_| 0..0); + let mut index = 0; + let mut add = |range| { + ranges[index] = range; + index += 1; + }; + add(0..0x1_0000); + add(TIMESTAMPS); + let page_list_size = (TVB_PAGE_LIST_SIZE as u64 + UAT_PAGE - 1) & !(UAT_PAGE - 1); + for pair in 0..FIXED_PAIRS { + let offset = pair as u64 * FIXED_PAIR_STRIDE; + add(SCENE + offset..SCENE + offset + SCENE_SIZE); + add(DISCARD + offset..DISCARD + offset + DISCARD_SIZE); + add(PAGE_LIST + offset..PAGE_LIST + offset + page_list_size); + add(FRAGMENT_STATUS + offset..FRAGMENT_STATUS + offset + UAT_PAGE); + } + add(RENDER_ALIASES); + add(TARGETS); + for page in TVB_INITIAL_PAGES { + let start = USER_BASE + u64::from(page) * TVB_PAGE; + add(start..start + TVB_BLOCK_STRIDE); + } + add(TVB_GROWTH + ..TVB_GROWTH + (TVB_MAX_BLOCKS - TVB_INITIAL_PAGES.len()) as u64 * TVB_BLOCK_STRIDE); + for pair in 1..FIXED_PAIRS { + let start = TVB_GROWTH + pair as u64 * TVB_PAIR_STRIDE; + add(start..start + TVB_MAX_BLOCKS as u64 * TVB_BLOCK_STRIDE); + } + add(AUXILIARY); + add(metrics_va..metrics_va.checked_add(METRICS_SIZE)?); + Some(ranges) + } +} + +/// Canonical userspace resource and its presentation in an execution context. +#[derive(Clone, Copy)] +pub(crate) struct ContextView { + pub(crate) source: u64, + pub(crate) low: u64, + pub(crate) size: usize, + pub(crate) write: bool, +} + +const fn context_view(low: u64, size: usize, write: bool) -> ContextView { + ContextView { + source: 0x10_0000_0000 + low, + low, + size, + write, + } +} + +pub(crate) const CONTEXT_VIEWS: [ContextView; 7] = [ + context_view(0, 0x10000, false), + context_view(0x18000, 0x8000, false), + context_view(0x28000, 0x8000, false), + context_view(0x38000, 0x8000, false), + context_view(0x48000, 0x8000, false), + context_view(0x58000, 0x8000, true), + context_view(0x68000, 0xc000, true), +]; + +/// Scheduler accounting policy and its firmware clock domain. +pub(crate) mod qos { + pub(crate) const CLASS: u8 = 0x0f; + pub(crate) const SHARE: u32 = 0x2000; + pub(crate) const CLOCK_HZ: u64 = 24_000_000; +} + +/// Physical workqueue capacity and logical-owner admission limits. +pub(crate) mod queues { + pub(crate) const RENDER_SLOTS: usize = 64; + pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 2; + pub(crate) const RENDER_DEPTH: u8 = 16; + pub(crate) const COMPUTE_DEPTH: usize = 16; +} + +/// Read-only recovery evidence, sampled only for selected halted slots. +pub(crate) mod recovery { + pub(crate) const SLOT_KEYS: u64 = 0x0000_0100_01ea_74b8; +} diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs index 60a64e23a161c..f354e4bb1f71a 100644 --- a/drivers/gpu/drm/asahi/mem.rs +++ b/drivers/gpu/drm/asahi/mem.rs @@ -11,6 +11,8 @@ use core::arch::asm; use core::cmp::min; +use kernel::page::PAGE_SIZE; + use crate::debug::*; use crate::mmu; @@ -134,6 +136,66 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { } } +/// Range length from which [`tlbi_range_or_asid`] invalidates the whole ASID. +const TLBI_RANGE_MAX: usize = 0x200_0000; +/// Range length from which [`tlbi_range_or_asid`] counts in 1 MiB units (scale 1) instead of +/// 32 KiB units (scale 0). +const TLBI_RANGE_SCALE1_MIN: usize = 0x10_0000; +/// Translation granule field value for 16 KiB pages. +const TLBI_TG_16K: u64 = 2; +/// Width of the base address field of a range TLBI operand. +const TLBI_RANGE_BADDR_MASK: u64 = (1 << 38) - 1; + +/// Invalidate a range of pages for a given ASID with a single range operation. +/// +/// Unlike [`tlbi_range`], which issues as many range operations as needed, this invalidates +/// ranges that do not fit one operation (and ranges starting at VA 0) by ASID. Single pages use a +/// page operation, at the CPU page size. +#[inline(always)] +pub(crate) fn tlbi_range_or_asid(asid: Asid, va: usize, len: usize) { + if debug_enabled(DebugFlags::ConservativeTlbi) { + tlbi_all(); + sync(); + return; + } + + if len == 0 { + return; + } + + if va == 0 || len >= TLBI_RANGE_MAX { + tlbi_asid(asid); + return; + } + if len <= PAGE_SIZE { + tlbi_page(asid, va); + return; + } + + // The range covers (NUM + 1) << (5 * SCALE + 1) granules of 16 KiB. + let (scale, unit_shift) = if len >= TLBI_RANGE_SCALE1_MIN { + (1u64, 20) + } else { + (0u64, 15) + }; + let num = ((len - 1) >> unit_shift) as u64; + + let val = ((asid as u64) << 48) + | (TLBI_TG_16K << 46) + | (scale << 44) + | (num << 39) + | (((va >> mmu::UAT_PGBIT) as u64) & TLBI_RANGE_BADDR_MASK); + + // SAFETY: tlbi is always safe by definition + unsafe { + asm!( + ".arch armv8.4-a", + "tlbi rvae1os, {x}", + x = in(reg) val + ); + } +} + /// Issue a memory barrier (`dsb sy`). #[inline(always)] pub(crate) fn sync() { diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index b30e31ba53c1a..2dfe30ae3617d 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -9,6 +9,10 @@ //! unmap buffer objects into a single user or kernel address space. //! //! The actual page table management is in the `pt` module. +//! +//! G17 firmware does not take part in the handoff protocol. Instead, each of its two coprocessor +//! instances publishes the top-level entries it owns into its own kernel root while it boots, and +//! TTBAT contexts other than a VM's bind slot may name a VM's root. See [`UatFirmware`]. use core::fmt::Debug; use core::mem::size_of; @@ -108,11 +112,24 @@ use pin_init; const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; +mod bind; +mod context; +mod shared; +mod lifetime; +pub(crate) use bind::PreparedUserBindBatch; +pub(crate) use lifetime::VmJobGuard; + /// PPL magic number for the handoff region const PPL_MAGIC: u64 = 0x4b1d000000000002; /// Number of supported context entries in the TTBAT const UAT_NUM_CTX: usize = 64; + +/// Application contexts owned by independent logical queues. Contexts 0..4 +/// and 63 remain reserved for the firmware and device-global resources. +const EXECUTION_CONTEXT_MASK: u64 = ((1u64 << 63) - 1) & !((1u64 << 5) - 1); +/// Maximum simultaneously retained logical queue contexts. +pub(crate) const MAX_EXECUTION_CONTEXTS: u32 = EXECUTION_CONTEXT_MASK.count_ones(); /// First context available for users const UAT_USER_CTX_START: usize = 1; /// Number of available user contexts @@ -124,6 +141,30 @@ pub(crate) const IOVA_USER_BASE: u64 = UAT_PGSZ as u64; /// Top-level page table entry of the upper half that holds the driver-managed kernel VA range. const IOVA_KERN_TOP_LEVEL_INDEX: u64 = 2; +/// TTBAT context that starts out as an alias of the kernel context (with its own ASID) with +/// [`UatFirmware::PublishedRoots`]. +const KERNEL_ALIAS_CTX: usize = 1; + +/// How the GPU firmware shares UAT management with the driver. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum UatFirmware { + /// One coprocessor that publishes its magic number in the handoff region, arbitrates the + /// handoff lock with the driver, and acknowledges cache flush requests (G13/G14). + Handoff, + /// Two coprocessor instances that never touch the handoff region (G17). + /// + /// Each instance has its own kernel root and writes the top-level entries below and including + /// the driver-managed kernel window into it while it boots: the lower ones are private to the + /// instance, and the kernel window entry points at a table that the bootloader reserved. The + /// primary instance uses the `pagetables` root, the secondary one the root at + /// `secondary_root_offset` in the same region. TTBAT context 1 starts as an alias of the kernel + /// context. + PublishedRoots { + /// Offset of the secondary instance's kernel root in the `pagetables` region. + secondary_root_offset: u64, + }, +} + const TTBR_VALID: u64 = 0x1; // BIT(0) const TTBR_ASID_SHIFT: usize = 48; @@ -221,17 +262,89 @@ const SLOTS_SIZE: usize = UAT_NUM_CTX * size_of::(); // We need at least page 0 (ttb0) const PAGETABLES_SIZE: usize = UAT_PGSZ; +/// Lock-free visibility of the mapping epoch. Writers run under the GPUVM +/// exec lock; nested mutations keep cached validation unavailable until the +/// outermost mutation has finished its leaf writes and invalidations. +struct MappingEpoch { + visible: AtomicU64, + pending: AtomicU64, + writers: AtomicU32, +} + +impl MappingEpoch { + fn new() -> Self { + Self { + visible: AtomicU64::new(0), + pending: AtomicU64::new(0), + writers: AtomicU32::new(0), + } + } + fn snapshot(&self) -> Option { + let value = self.visible.load(Ordering::Acquire); + (value != u64::MAX).then_some(value) + } +} + +#[must_use = "retain until the mapping mutation and its invalidations finish"] +struct MappingMutation(Option>); + +impl Drop for MappingMutation { + fn drop(&mut self) { + if let Some(epoch) = self.0.as_ref() { + if epoch.writers.fetch_sub(1, Ordering::Relaxed) == 1 { + epoch + .visible + .store(epoch.pending.load(Ordering::Relaxed), Ordering::Release); + } + } + } +} + /// Inner data for a Vm instance. struct VmInner { dev: driver::AsahiDevRef, is_kernel: bool, + va_range: Range, page_table: UatPageTable, + mapping_epoch: Option>, + epoch: Option, mm: mm::Allocator<(), KernelMappingInner>, uat_inner: Arc, binding: Arc>, id: u64, } +/// Exec-locked access to the [`VmInner`] of a GPUVM. +/// +/// `GpuVm::exec_lock()` returns a `GpuVmExec`; this guard gives direct access to the driver data +/// it protects. +struct VmExecGuard(gpuvm::GpuVmExec); + +impl core::ops::Deref for VmExecGuard { + type Target = VmInner; + + fn deref(&self) -> &VmInner { + self.0.unique_gpuvm.data_ref() + } +} + +impl core::ops::DerefMut for VmExecGuard { + fn deref_mut(&mut self) -> &mut VmInner { + self.0.unique_gpuvm.data() + } +} + +/// Take the exec lock of a VM's GPUVM. +fn lock_vm( + gpuvm: &gpuvm::GpuVm, + obj: Option<&gem::Object>, + interruptible: bool, +) -> Result { + // SAFETY: As in the rest of this file, the `UniqueRefGpuVm` returned by `GpuVm::new()` is not + // retained, so the exec lock is the only way to reach the VM's data. + Ok(VmExecGuard(unsafe { gpuvm.exec_lock(obj, interruptible)? })) +} + /// Slot binding-related inner data for a Vm instance. struct VmBinding { active_users: usize, @@ -322,6 +435,8 @@ impl gpuvm::DriverGpuVm for VmInner { op: OpMap<'op, Self>, ctx: &mut Self::SmContext<'ctx>, ) -> Result, Error> { + let _mutation = self.mapping_mutation(); + let (map_addr, map_length) = (op.addr(), op.length() as usize); let mut iova = op.addr(); let mut left = op.length() as usize; let mut offset = op.gem_offset() as usize; @@ -390,6 +505,7 @@ impl gpuvm::DriverGpuVm for VmInner { let gpuva = ctx.preallocated_gpuva()?; let op = op.insert(gpuva, pin_init::init_zeroed()); + self.tlbi_contexts_naming_root(map_addr, map_length); Ok(op) } @@ -404,11 +520,12 @@ impl gpuvm::DriverGpuVm for VmInner { mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", iova, length); + let _mutation = self.mapping_mutation(); self.page_table.unmap_pages(iova..(iova + length))?; if let Some(asid) = self.slot() { fence(Ordering::SeqCst); - mem::tlbi_range(asid as u8, iova as usize, length as usize); + self.tlbi_range(asid as u8, iova, length as usize); mod_dev_dbg!( self.dev, "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", @@ -419,6 +536,8 @@ impl gpuvm::DriverGpuVm for VmInner { mem::sync(); } + self.tlbi_contexts_naming_root(iova, length as usize); + let (op_unmapped, _va_removed) = op.remove(); Ok(op_unmapped) @@ -454,11 +573,12 @@ impl gpuvm::DriverGpuVm for VmInner { let unmap_range = unmap_end - unmap_start; + let _mutation = self.mapping_mutation(); self.page_table.unmap_pages(unmap_start..unmap_end)?; if let Some(asid) = self.slot() { fence(Ordering::SeqCst); - mem::tlbi_range(asid as u8, unmap_start as usize, unmap_range as usize); + self.tlbi_range(asid as u8, unmap_start, unmap_range as usize); mod_dev_dbg!( self.dev, "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", @@ -469,6 +589,8 @@ impl gpuvm::DriverGpuVm for VmInner { mem::sync(); } + self.tlbi_contexts_naming_root(unmap_start, unmap_range as usize); + let prev_gpuva = ctx.preallocated_gpuva()?; let next_gpuva = ctx.preallocated_gpuva()?; @@ -487,6 +609,19 @@ impl gpuvm::DriverGpuVm for VmInner { } impl VmInner { + fn mapping_mutation(&mut self) -> MappingMutation { + let state = self.mapping_epoch.clone(); + if let Some(state) = state.as_ref() { + state.writers.fetch_add(1, Ordering::Relaxed); + state.visible.swap(u64::MAX, Ordering::AcqRel); + self.epoch = self.epoch.and_then(|epoch| epoch.checked_add(1)); + state + .pending + .store(self.epoch.unwrap_or(u64::MAX), Ordering::Relaxed); + } + MappingMutation(state) + } + /// Returns the slot index, if this VM is bound. fn slot(&self) -> Option { if self.is_kernel { @@ -514,10 +649,46 @@ impl VmInner { self.page_table.ttb() } + /// Invalidate a VA range of this Vm's address space in the TLB of one ASID. + fn tlbi_range(&self, asid: u8, iova: u64, size: usize) { + match self.uat_inner.firmware { + UatFirmware::Handoff => mem::tlbi_range(asid, iova as usize, size), + UatFirmware::PublishedRoots { .. } => { + mem::tlbi_range_or_asid(asid, iova as usize, size) + } + } + } + + /// Invalidate a VA range in every TTBAT context that names this Vm's root as its lower half. + /// + /// With [`UatFirmware::PublishedRoots`], contexts other than a Vm's bind slot name its root + /// (the kernel lower root sits in contexts 0 and 1 and has no slot at all), so a mapping + /// change must reach all of them. Other firmware only uses the bind slot, which the callers + /// already invalidate. + fn tlbi_contexts_naming_root(&self, iova: u64, size: usize) { + if self.uat_inner.firmware == UatFirmware::Handoff { + return; + } + + let contexts = self.uat_inner.lock().contexts_naming_root(self.ttb()); + if contexts == 0 { + return; + } + + fence(Ordering::SeqCst); + for ctx in 0..UAT_NUM_CTX { + if contexts & (1 << ctx) != 0 { + mem::tlbi_range_or_asid(ctx as u8, iova as usize, size); + } + } + mem::sync(); + } + /// Map an `mm::Node` representing an mapping in VA space. fn map_node(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { let mut iova = node.start(); - let segments = node.bo.as_ref().ok_or(EINVAL)?.data().segments.lock(); + let segments = node.bo.as_ref().ok_or(EINVAL)?.get()?.data().segments.lock(); + let _mutation = self.mapping_mutation(); let mut offset = node.offset; let mut left = node.mapped_size; @@ -574,6 +745,9 @@ impl VmInner { left -= len; } } + + // A context naming this root may be live and hold a negative translation for the range. + self.tlbi_contexts_naming_root(node.start(), node.mapped_size); Ok(()) } } @@ -583,12 +757,84 @@ impl VmInner { pub(crate) struct Vm { dev: ARef, id: u64, + mapping_epoch: Option>, + lifetime: Option>, + context_bindings: Option>, + shared_bindings: Option>, + status: Option>, gpuvm: ARef>, dummy_obj: ARef, binding: Arc>, } no_debug!(Vm); +/// An application TTBAT identity retained by a logical queue and its jobs. +/// The root lease is dropped before the VM, so no page table can be freed +/// while a context still names it. Installed firmware queues must retain this +/// object through replacement retirement or processor stop. +pub(crate) struct ExecutionContext { + root: ExecutionRoot, + vm: Vm, +} + +struct ExecutionRoot { + inner: Arc, + id: u8, + generation: u8, + low: u64, + high: u64, +} + +impl ExecutionContext { + pub(crate) fn id(&self) -> u32 { + self.root.id.into() + } + pub(crate) fn generation(&self) -> u8 { + self.root.generation + } + pub(crate) fn vm(&self) -> &Vm { + &self.vm + } + + pub(crate) fn is_current(&self) -> bool { + let inner = self.root.inner.lock(); + let roots = &inner.ttbs()[usize::from(self.root.id)]; + roots.ttb0.load(Ordering::Acquire) == self.root.low + && roots.ttb1.load(Ordering::Acquire) == self.root.high + } +} + +impl Drop for ExecutionRoot { + fn drop(&mut self) { + let id = usize::from(self.id); + let release = { + let inner = self.inner.lock(); + inner.handoff().lock(); + let roots = &inner.ttbs()[id]; + let low = roots.ttb0.load(Ordering::Acquire); + let high = roots.ttb1.load(Ordering::Acquire); + let owned = low == self.low && high == self.high; + if owned { + roots.ttb0.store(0, Ordering::Release); + roots.ttb1.store(0, Ordering::Release); + } + inner.handoff().unlock(); + owned || (low == 0 && high == 0) + }; + if release { + fence(Ordering::SeqCst); + mem::tlbi_asid(self.id); + mem::sync(); + self.inner.lock().execution_contexts &= !(1u64 << id); + } else { + pr_err!( + "MMU: application context {} changed owners while leased\n", + id + ); + } + } +} + /// Slot data for a [`Vm`] slot (nothing, we only care about the indices). pub(crate) struct SlotInner(); @@ -640,13 +886,60 @@ impl Clone for VmBind { } } +/// A mapping BO reference selecting the teardown protocol of its firmware. +/// Retain the owner until deferred cleanup completes, including its final BO. +struct MappingBo { + deferred: bool, + bo: Option>>, + owner: ARef>, +} +impl MappingBo { + fn new( + bo: ARef>, + owner: ARef>, + deferred: bool, + ) -> Self { + Self { + deferred, + bo: Some(bo), + owner, + } + } + fn get(&self) -> Result<&gpuvm::GpuVmBo> { + self.bo.as_deref().ok_or(EINVAL) + } +} +impl Drop for MappingBo { + fn drop(&mut self) { + let Some(bo) = self.bo.take() else { + return; + }; + if !self.deferred { + // Handoff VMs retain the legacy ARef decrement and reservation + // locking behavior. Only published-root VMs use deferred cleanup. + drop(bo); + return; + } + let raw = ARef::into_raw(bo); + // SAFETY: GpuVmBo is repr(C) with drm_gpuvm_bo first. into_raw + // transfers one live reference. Every Vm here uses IMMEDIATE_MODE; + // deferred put selects its GPUVA lock and retains cleanup ownership. + unsafe { + kernel::bindings::drm_gpuvm_bo_put_deferred( + raw.as_ptr().cast::(), + ) + }; + self.owner.deferred_cleanup(); + } +} + /// Inner data required for an object mapping into a [`Vm`]. pub(crate) struct KernelMappingInner { // Drop order matters: - // - Drop the GpuVmBo first, which resv locks its BO and drops a GpuVm reference + // - Deferred-put the GpuVmBo under its immediate-mode GPUVA lock // - Drop the GEM BO next, since BO free can take the resv lock itself // - Drop the owner GpuVm last, since that again can take resv locks when the refcount drops to 0 - bo: Option>>, + bo: Option, _gem: Option>, owner: ARef>, uat_inner: Arc, @@ -674,6 +967,108 @@ impl KernelMapping { self.0.start()..(self.0.start() + self.0.mapped_size as u64) } + /// Resolves a byte in this retained mapping without creating page tables. + pub(crate) fn translate_offset(&self, offset: usize) -> Result { + if offset >= self.size() { + return Err(EINVAL); + } + let address = self.iova().checked_add(offset as u64).ok_or(EOVERFLOW)?; + lock_vm(&self.0.owner, None, false)?.page_table.translate_iova(address) + } + + /// Writes a userspace timestamp through the retained GEM mapping. Byte + /// stores support unaligned destinations and concurrent completion writes + /// without creating mutable aliases. The complete timestamp is not atomic. + pub(crate) fn write_timestamp(&self, offset: usize, value: u64) -> Result { + use core::sync::atomic::AtomicU8; + use kernel::drm::gem::BaseObject; + let gem = self.0._gem.as_ref().ok_or(EINVAL)?; + if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { + return Err(ERANGE); + } + let start = self.0.offset.checked_add(offset).ok_or(EOVERFLOW)?; + let end = start.checked_add(size_of::()).ok_or(EOVERFLOW)?; + if end > gem.size() { + return Err(ERANGE); + } + let vmap = gem.vmap::<0>()?; + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + // SAFETY: The retained GEM/vmap covers each checked byte. AtomicU8 + // has byte alignment and all concurrent completion writers use + // the same access width, without borrowing a mutable byte slice. + let destination = unsafe { &*vmap.addr.cast::().add(start + index).cast::() }; + destination.store(byte, Ordering::Relaxed); + } + Ok(()) + } + + /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. + /// The result includes whether any page-table entry may have changed; after + /// mutation the caller must retain the mapping until processor stop even if + /// the readback fails. The original protection is retained for teardown. + /// + /// # Safety + /// The mapping must be unpublished. After mutation it must not be unmapped + /// while a firmware instance can retain a cached translation or contents. + pub(crate) unsafe fn set_fw_cached_prefix(&self, prefix: usize) -> (Result<[u64; 2]>, bool) { + const MAX_PAGES: usize = 32; + const ATTRIBUTE_SHIFT: u32 = 2; + const ATTRIBUTE_MASK: u64 = 7; + const UNCACHED: u64 = 2; + const CACHE_ATTRIBUTE_CHANGE: u64 = UNCACHED << ATTRIBUTE_SHIFT; + let mut mutated = false; + let result = (|| { + if self.0.uat_inner.firmware == UatFirmware::Handoff + || prefix == 0 + || prefix > self.size() + || (prefix | self.size()) & UAT_PGMSK != 0 + || self.size() > MAX_PAGES * UAT_PGSZ + { + return Err(EINVAL); + } + let mut owner = lock_vm(&self.0.owner, None, false)?; + if !owner.is_kernel { + return Err(EINVAL); + } + let pages = self.size() / UAT_PGSZ; + let cached_pages = prefix / UAT_PGSZ; + let mut before = [0; MAX_PAGES]; + for (index, value) in before[..pages].iter_mut().enumerate() { + *value = owner + .page_table + .leaf_entry(self.iova() + (index * UAT_PGSZ) as u64)?; + if (*value >> ATTRIBUTE_SHIFT) & ATTRIBUTE_MASK != UNCACHED { + return Err(EINVAL); + } + } + let _mutation = owner.mapping_mutation(); + mutated = true; + owner.page_table.reprot_pages( + self.iova()..self.iova() + prefix as u64, + PROT_GPU_FW_PRIV_RW, + )?; + fence(Ordering::SeqCst); + mem::tlbi_all(); + mem::sync(); + let mut after = [0; MAX_PAGES]; + for (index, value) in after[..pages].iter_mut().enumerate() { + *value = owner + .page_table + .leaf_entry(self.iova() + (index * UAT_PGSZ) as u64)?; + let change = if index < cached_pages { + CACHE_ATTRIBUTE_CHANGE + } else { + 0 + }; + if before[index] ^ *value != change { + return Err(EFAULT); + } + } + Ok([after[0], after[cached_pages.min(pages - 1)]]) + })(); + (result, mutated) + } + /// Remap a cached mapping as uncached, then synchronously flush that range of VAs from the /// coprocessor cache. This is required to safely unmap cached/private mappings. fn remap_uncached_and_flush(&mut self) { @@ -695,6 +1090,7 @@ impl KernelMapping { // Remap in-place as uncached. // Do not try to unmap the guard page (-1) let prot = self.0.prot.as_uncached(); + let _mutation = owner.mapping_mutation(); if owner .page_table .reprot_pages(self.iova_range(), prot) @@ -837,7 +1233,10 @@ impl Drop for KernelMapping { // 4. Unmap // 5. Flush the TLB range again - if self.0.prot.is_cached_noncoherent() { + // Firmware that does not use the handoff protocol cannot be asked to flush its cache. + // Such firmware only caches kernel mappings that live until its coprocessors stop. + if self.0.prot.is_cached_noncoherent() && self.0.uat_inner.firmware == UatFirmware::Handoff + { mod_pr_debug!( "MMU: remap as uncached {:#x}:{:#x}\n", self.iova(), @@ -861,6 +1260,7 @@ impl Drop for KernelMapping { self.size() ); + let _mutation = owner.mapping_mutation(); if owner.page_table.unmap_pages(self.iova_range()).is_err() { dev_err!( owner.dev.as_ref(), @@ -872,7 +1272,7 @@ impl Drop for KernelMapping { if let Some(asid) = owner.slot() { fence(Ordering::SeqCst); - mem::tlbi_range(asid as u8, self.iova() as usize, self.size()); + owner.tlbi_range(asid as u8, self.iova(), self.size()); mod_dev_dbg!( owner.dev, "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", @@ -882,12 +1282,20 @@ impl Drop for KernelMapping { ); mem::sync(); } + + // The range returns to the VA allocator once this mapping is gone, so no context that + // names this root may keep a stale translation for it. + owner.tlbi_contexts_naming_root(self.iova(), self.size()); } } /// Shared UAT global data structures struct UatShared { kernel_ttb1: u64, + execution_contexts: u64, + context_generations: [u8; UAT_NUM_CTX], + /// Kernel root of the secondary firmware instance, if there is one. + secondary_ttb1: Option, map_kernel_to_user: bool, handoff_rgn: UatRegion, ttbs_rgn: UatRegion, @@ -905,6 +1313,20 @@ impl UatShared { // SAFETY: pointer is non-null per the type invariant unsafe { (self.ttbs_rgn.map.ptr() as *mut [SlotTTBS; UAT_NUM_CTX]).as_ref() }.unwrap() } + + /// Returns the mask of TTBAT contexts whose lower half is the root `ttb`. + fn contexts_naming_root(&self, ttb: u64) -> u64 { + self.ttbs() + .iter() + .enumerate() + .filter(|(ctx, slot)| slot.ttb0.load(Ordering::Acquire) == tagged_root(ttb, *ctx)) + .fold(0, |mask, (ctx, _)| mask | (1 << ctx)) + } +} + +/// Returns the TTBAT value naming root `ttb` in context `ctx`. +const fn tagged_root(ttb: u64, ctx: usize) -> u64 { + ttb | TTBR_VALID | (ctx as u64) << TTBR_ASID_SHIFT } // SAFETY: Nothing here is unsafe to send across threads. @@ -913,6 +1335,7 @@ unsafe impl Send for UatShared {} /// Inner data for the top-level UAT instance. #[pin_data] struct UatInner { + firmware: UatFirmware, #[pin] shared: Mutex, #[pin] @@ -977,12 +1400,27 @@ impl Handoff { } /// Initialize the handoff region - fn init(&self) -> Result { + fn init(&self, firmware: UatFirmware) -> Result { self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); self.cur_slot.store(0, Ordering::Relaxed); self.unk3.store(0, Ordering::Relaxed); fence(Ordering::SeqCst); + if firmware == UatFirmware::Handoff { + self.wait_for_firmware()?; + } + + for i in 0..=UAT_NUM_CTX { + self.flush[i].state.store(0, Ordering::Relaxed); + self.flush[i].addr.store(0, Ordering::Relaxed); + self.flush[i].size.store(0, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } + + /// Wait for the firmware to publish its half of the handoff region. + fn wait_for_firmware(&self) -> Result { let start = Instant::::now(); const TIMEOUT: Delta = Delta::from_millis(1000); @@ -1004,13 +1442,6 @@ impl Handoff { } self.unlock(); - - for i in 0..=UAT_NUM_CTX { - self.flush[i].state.store(0, Ordering::Relaxed); - self.flush[i].addr.store(0, Ordering::Relaxed); - self.flush[i].size.store(0, Ordering::Relaxed); - } - fence(Ordering::SeqCst); Ok(()) } } @@ -1085,7 +1516,7 @@ impl Vm { geometry.kernel_range(), geometry.ias, geometry.oas, - false, + uat_inner.firmware != UatFirmware::Handoff, )? } else { UatPageTable::new(geometry.ias, geometry.oas)? @@ -1094,7 +1525,11 @@ impl Vm { let (va_range, gpuvm_range) = if is_kernel { (geometry.kernel_range(), kernel_range.clone()) } else { - (geometry.user_range(), geometry.user_usable_range()) + let mut range = geometry.user_range(); + if uat_inner.firmware != UatFirmware::Handoff { + range.start = 0; + } + (range, geometry.user_usable_range()) }; let mm = mm::Allocator::new(va_range.start, va_range.range(), ())?; @@ -1112,6 +1547,11 @@ impl Vm { GFP_KERNEL, )?; + let mapping_epoch = if uat_inner.firmware != UatFirmware::Handoff { + Some(Arc::new(MappingEpoch::new(), GFP_KERNEL)?) + } else { + None + }; let unique_gpuvm = gpuvm::GpuVm::new::( c"Asahi::GpuVm", &dev, @@ -1121,7 +1561,10 @@ impl Vm { VmInner { dev: dev.clone(), is_kernel, + va_range: va_range.clone(), page_table, + mapping_epoch: mapping_epoch.clone(), + epoch: Some(0), mm, uat_inner, binding: binding.clone(), @@ -1134,11 +1577,112 @@ impl Vm { dev: dev.as_ref().as_ref().into(), id, gpuvm, + mapping_epoch, + lifetime: None, + context_bindings: None, + shared_bindings: None, + status: None, dummy_obj: dummy_obj.gem.clone(), binding, }) } + /// Attach a status before sharing a newly created user VM with queues. + pub(crate) fn with_status(mut self) -> Result { + self.status = Some(Arc::new(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); + self.lifetime = Some(lifetime::VmLifetime::new()?); + self.context_bindings = Some(context::ContextBindings::new()?); + self.shared_bindings = Some(shared::SharedBindings::new()?); + Ok(self) + } + + /// Status shared by every queue using this VM, if enabled by its GPU. + pub(crate) fn status(&self) -> Option<&Arc> { + self.status.as_ref() + } + + fn covers_range_locked( + inner: &mut VmInner, + address: u64, + size: u64, + read: bool, + write: bool, + ) -> bool { + if size == 0 { + return false; + } + let Some(end) = address.checked_add(size) else { + return false; + }; + let mask = UAT_PGMSK as u64; + let start = address & !mask; + let end = if end & mask == 0 { + end + } else { + let Some(end) = (end | mask).checked_add(1) else { + return false; + }; + end + }; + if start < inner.va_range.start || end > inner.va_range.end { + return false; + } + inner + .page_table + .covers_range(start..end, read, write) + .unwrap_or(false) + } + + pub(crate) fn covers_range(&self, address: u64, size: u64, read: bool, write: bool) -> bool { + let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { + return false; + }; + Self::covers_range_locked(&mut inner, address, size, read, write) + } + + /// Checks all writable sparse blocks and their guard coverage under one + /// GPUVM lock. A fully mapped guard is not a valid sparse allocation. + pub(crate) fn covers_sparse_blocks_with_guards( + &self, + addresses: &[u64], + block_size: u64, + stride: u64, + ) -> bool { + if stride <= block_size { + return false; + } + let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { + return false; + }; + addresses.iter().all(|&address| { + let Some(guard) = address.checked_add(block_size) else { + return false; + }; + Self::covers_range_locked(&mut inner, address, block_size, true, true) + && !Self::covers_range_locked( + &mut inner, guard, stride - block_size, false, false, + ) + }) + } + + /// Checks all exact request tuples under one GPUVM exec lock. Access bit0 + /// requests reading and bit1 writing. The epoch names this same snapshot. + pub(crate) fn covers_ranges_batch(&self, ranges: &[(u64, u64, u8)]) -> (bool, Option) { + let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { + return (false, None); + }; + let epoch = inner.epoch; + let covered = ranges.iter().all(|&(address, size, access)| { + Self::covers_range_locked(&mut inner, address, size, access & 1 != 0, access & 2 != 0) + }); + (covered, epoch) + } + + /// Returns None while a leaf mutation is in progress or after epoch overflow. + pub(crate) fn mapping_validation_epoch(&self) -> Option { + self.mapping_epoch.as_ref()?.snapshot() + } + /// Get the translation table base for this Vm fn ttb(&self) -> u64 { self.binding.lock().ttb @@ -1155,6 +1699,39 @@ impl Vm { prot: Prot, guard: bool, ) -> Result { + self.map_in_range_with_guard_size( + gem, + object_range, + alignment, + range, + prot, + if guard { UAT_PGSZ } else { 0 }, + ) + } + + /// Reserves a whole-page unmapped guard without adding it to the mapped backing. + #[allow(clippy::too_many_arguments)] + pub(crate) fn map_in_range_with_guard_size( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_size: usize, + ) -> Result { + use kernel::drm::gem::BaseObject; + if guard_size % UAT_PGSZ != 0 { + return Err(EINVAL); + } + let size = object_range + .end + .checked_sub(object_range.start) + .ok_or(EINVAL)?; + if size == 0 || object_range.end > gem.size() { + return Err(EINVAL); + } + let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; // TODO: hack, populate the GEM object sg table first { let dev = unsafe { self.dev.as_bound() }; @@ -1178,24 +1755,41 @@ impl Vm { } core::mem::drop(vm_bo_guard); + let deferred = inner.uat_inner.firmware != UatFirmware::Handoff; + let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone(), deferred)); let uat_inner = inner.uat_inner.clone(); - let node = inner.mm.insert_node_in_range( + // A failed reservation drops its payload while exec still owns GEM + // reservations. Published-root VMs attach BO/GEM only after success; + // handoff VMs retain their existing payload and decrement behavior. + let result = inner.mm.insert_node_in_range( KernelMappingInner { owner: self.gpuvm.clone(), uat_inner, prot, - bo: Some(vm_bo), - _gem: Some(gem.into()), + bo: if deferred { None } else { vm_bo.take() }, + _gem: if deferred { None } else { Some(gem.into()) }, offset: object_range.start, mapped_size: size, }, - (size + if guard { UAT_PGSZ } else { 0 }) as u64, // Add guard page + reserved_size as u64, alignment, 0, range.start, range.end, mm::InsertMode::Best, - )?; + ); + let mut node = match result { + Ok(node) => node, + Err(error) => { + drop(vm_exec); + return Err(error); + } + }; + if deferred { + let payload = node.as_mut().inner_mut(); + payload.bo = vm_bo.take(); + payload._gem = Some(gem.into()); + } let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the @@ -1216,6 +1810,9 @@ impl Vm { prot: Prot, guard: bool, ) -> Result { + let reserved_size = size + .checked_add(if guard { UAT_PGSZ } else { 0 }) + .ok_or(EOVERFLOW)?; // TODO: hack, populate the GEM object sg table first { let dev = unsafe { self.dev.as_bound() }; @@ -1243,21 +1840,36 @@ impl Vm { } core::mem::drop(vm_bo_guard); + let deferred = inner.uat_inner.firmware != UatFirmware::Handoff; + let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone(), deferred)); let uat_inner = inner.uat_inner.clone(); - let node = inner.mm.reserve_node( + // Published-root VMs defer BO/GEM attachment until reservation succeeds. + let result = inner.mm.reserve_node( KernelMappingInner { owner: self.gpuvm.clone(), uat_inner, prot, - bo: Some(vm_bo), - _gem: Some(gem.into()), + bo: if deferred { None } else { vm_bo.take() }, + _gem: if deferred { None } else { Some(gem.into()) }, offset: 0, mapped_size: size, }, addr, - (size + if guard { UAT_PGSZ } else { 0 }) as u64, // Add guard page + reserved_size as u64, 0, - )?; + ); + let mut node = match result { + Ok(node) => node, + Err(error) => { + drop(vm_exec); + return Err(error); + } + }; + if deferred { + let payload = node.as_mut().inner_mut(); + payload.bo = vm_bo.take(); + payload._gem = Some(gem.into()); + } let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the @@ -1279,6 +1891,7 @@ impl Vm { prot: Prot, single_page: bool, ) -> Result { + self.wait_for_user_map_admission()?; // TODO: hack, populate the GEM object sg table first { let dev = unsafe { self.dev.as_bound() }; @@ -1340,6 +1953,7 @@ impl Vm { size, addr ); + let _commit = self.mapping_commit()?; let req = OpMapRequest { addr, @@ -1394,6 +2008,7 @@ impl Vm { 0, )?; + let mutation = inner.mapping_mutation(); let ret = inner.page_table.map_pages( iova..(iova + size as u64), phys as PhysicalAddr, @@ -1402,6 +2017,7 @@ impl Vm { ); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. + drop(mutation); let _ = inner; ret?; Ok(KernelMapping(node)) @@ -1413,6 +2029,23 @@ impl Vm { dev: Option<&'drm device::Device>, iova: u64, size: u64, + ) -> Result { + if self.defer_unmap(iova, size)? { + return Ok(()); + } + self.unmap_range_commit(dev, iova, size, true) + } + + fn unmap_range_now(&self, iova: u64, size: u64) -> Result { + self.unmap_range_commit(None, iova, size, false) + } + + fn unmap_range_commit<'drm>( + &self, + dev: Option<&'drm device::Device>, + iova: u64, + size: u64, + user: bool, ) -> Result { // Unmapping a range can only do a single split, so just preallocate // two GPUVAs. @@ -1428,11 +2061,25 @@ impl Vm { let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; mod_dev_dbg!(self.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size); - vm_exec.unique_gpuvm.sm_unmap(iova, size, &mut ctx) + let _commit = if user { + match self.mapping_commit() { + Ok(guard) => guard, + Err(error) => { + if self.defer_unmap(iova, size)? { + return Ok(()); + } + return Err(error); + } + } + } else { + None + }; + vm_exec.unique_gpuvm.sm_unmap(iova, size, &mut ctx)?; + self.untrack_shared_range(iova, size); + Ok(()) } - /// Drop mappings for a given bo. - pub(crate) fn drop_mappings(&self, gem: &gem::Object) -> Result { + fn drop_mappings_now(&self, gem: &gem::Object) -> Result { // Removing whole mappings only does unmaps, so no preallocated VAs let mut resources = VmOpResources { preallocated_gpuvas: [None, None, None], @@ -1444,6 +2091,7 @@ impl Vm { if let Some(bo) = vm_exec.unique_gpuvm.find_bo(gem) { mod_dev_dbg!(self.dev, "MMU: bo_unmap\n"); vm_exec.unique_gpuvm.bo_unmap(&mut ctx, &bo)?; + self.untrack_shared_object(gem); mod_dev_dbg!(self.dev, "MMU: bo_unmap done\n"); // We need to drop the exec_lock first, then the GpuVmBo since that will take the lock itself. core::mem::drop(vm_exec); @@ -1568,6 +2216,39 @@ impl Uat { &self.kernel_lower_vm } + /// Installed descriptor and bind contexts rooted in this user VM. A graph + /// constructor checks this before changing a retained pool's mappings. + pub(crate) fn vm_context_mask(&self, vm: &Vm) -> Result { + if !matches!(self.inner.firmware, UatFirmware::PublishedRoots { .. }) { + return Err(EINVAL); + } + let root = { + let owner = lock_vm(&vm.gpuvm, None, false)?; + if owner.is_kernel || !Arc::ptr_eq(&owner.uat_inner, &self.inner) { + return Err(EINVAL); + } + owner.ttb() + }; + Ok(self.inner.lock().contexts_naming_root(root)) + } + + /// Publish a completed alias set before its descriptors become visible. + /// Match actual roots, including independently allocated execution contexts. + pub(crate) fn flush_vm_contexts(&self, vm: &Vm) -> Result { + let contexts = self.vm_context_mask(vm)?; + if contexts == 0 { + return Err(ENOENT); + } + fence(Ordering::SeqCst); + for context in 0..UAT_NUM_CTX { + if contexts & (1u64 << context) != 0 { + mem::tlbi_asid(context as u8); + } + } + mem::sync(); + Ok(contexts) + } + /// Returns the base physical address of the TTBAT region. pub(crate) fn ttb_base(&self) -> u64 { let inner = self.inner.lock(); @@ -1633,6 +2314,63 @@ impl Uat { Ok(VmBind(vm.clone(), slot)) } + /// Allocates a stable application context without changing the VM bind slot. + /// Both roots and the generation remain unchanged for the lease's lifetime. + pub(crate) fn new_execution_context(&self, vm: &Vm) -> Result> { + if self.inner.firmware == UatFirmware::Handoff { + return Err(EINVAL); + } + { + let owner = lock_vm(&vm.gpuvm, None, false)?; + if owner.is_kernel + || !core::ptr::eq(Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner)) + { + return Err(EINVAL); + } + } + let (id, generation) = { + let mut inner = self.inner.lock(); + let free = !inner.execution_contexts & EXECUTION_CONTEXT_MASK; + if free == 0 { + return Err(ENOSPC); + } + let id = free.trailing_zeros() as usize; + inner.execution_contexts |= 1u64 << id; + inner.context_generations[id] = inner.context_generations[id].wrapping_add(1); + (id, inner.context_generations[id]) + }; + let low = tagged_root(vm.ttb(), id); + let high = tagged_root(self.kernel_vm.ttb(), id); + // Allocate the complete host owner before either TTBAT half is published. + // ExecutionRoot releases an empty claimed slot if allocation fails. + let context = Arc::new( + ExecutionContext { + root: ExecutionRoot { + inner: self.inner.clone(), + id: id as u8, + generation, + low, + high, + }, + vm: vm.clone(), + }, + GFP_KERNEL, + )?; + { + let inner = self.inner.lock(); + let roots = &inner.ttbs()[id]; + if roots.ttb0.load(Ordering::Acquire) != 0 || roots.ttb1.load(Ordering::Acquire) != 0 { + return Err(EBUSY); + } + roots.ttb0.store(low, Ordering::Release); + roots.ttb1.store(high, Ordering::Release); + } + fence(Ordering::SeqCst); + mem::tlbi_asid(id as u8); + mem::sync(); + Ok(context) + } + /// Creates a new `Vm` linked to this UAT. pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { Vm::new( @@ -1650,9 +2388,109 @@ impl Uat { self.geometry } + /// Adopts the top-level entries that the primary firmware instance published in the kernel + /// root while booting ([`UatFirmware::PublishedRoots`]). + /// + /// The entries below the kernel window belong to the firmware, and the kernel window entry + /// points at a table that the bootloader reserved, so the driver maps kernel objects beneath + /// it and never frees it. Entries above the kernel window do not belong to anyone and are + /// cleared. Must run after the primary instance booted and before anything is mapped into the + /// kernel `Vm`. + pub(crate) fn adopt_firmware_root_entries(&self) -> Result<[u64; 3]> { + let owned = IOVA_KERN_TOP_LEVEL_INDEX as usize + 1; + let mut snapshot = [0; 3]; + // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives + // this `Uat`. + unsafe { + pgtable::with_root_entries(self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { + for (value, pte) in snapshot.iter_mut().zip(ptes) { + *value = pte.load(Ordering::Relaxed); + } + for pte in &ptes[owned..] { + pte.store(0, Ordering::Relaxed); + } + }) + }?; + + fence(Ordering::SeqCst); + mem::tlbi_all(); + mem::sync(); + Ok(snapshot) + } + + /// Checks that allocation under the shared subtree did not replace a firmware-owned entry. + pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { + // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. + let matches = unsafe { + pgtable::with_root_entries(self.kernel_vm.ttb(), snapshot.len(), |ptes| { + ptes.iter() + .zip(snapshot) + .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) + }) + }?; + if !matches { + return Err(EIO); + } + Ok(()) + } + + /// Copies the kernel root's top-level entries from the kernel window up into the root of the + /// secondary firmware instance ([`UatFirmware::PublishedRoots`]), whose lower entries are + /// private to that instance. + pub(crate) fn mirror_secondary_root(&self) -> Result { + let secondary = self.inner.lock().secondary_ttb1.ok_or(EINVAL)?; + let count = UAT_PGSZ / size_of::(); + let first = IOVA_KERN_TOP_LEVEL_INDEX as usize; + + // SAFETY: Both roots are in the bootloader-reserved `pagetables` region, which outlives + // this `Uat`. + unsafe { + pgtable::with_root_entries(self.kernel_vm.ttb(), count, |primary| { + pgtable::with_root_entries(secondary, count, |secondary| { + for (src, dst) in primary.iter().zip(secondary).skip(first) { + let pte = src.load(Ordering::Relaxed); + if pte != 0 && dst.load(Ordering::Relaxed) != pte { + dst.store(pte, Ordering::Relaxed); + } + } + }) + }) + }??; + + fence(Ordering::SeqCst); + mem::tlbi_all(); + mem::sync(); + Ok(()) + } + + /// Activates the device-global render root in context 1 after the boot records retire. + /// Both contexts must still contain their original kernel roots. + pub(crate) fn activate_render_context(&self, vm: &Vm) -> Result { + let inner = self.inner.lock(); + inner.handoff().lock(); + let ttbs = inner.ttbs(); + let ready = [0, KERNEL_ALIAS_CTX].into_iter().all(|ctx| { + ttbs[ctx].ttb0.load(Ordering::Acquire) == tagged_root(self.kernel_lower_vm.ttb(), ctx) + && ttbs[ctx].ttb1.load(Ordering::Acquire) == tagged_root(self.kernel_vm.ttb(), ctx) + }); + if !ready { + inner.handoff().unlock(); + return Err(EIO); + } + ttbs[KERNEL_ALIAS_CTX] + .ttb0 + .store(tagged_root(vm.ttb(), KERNEL_ALIAS_CTX), Ordering::Release); + inner.handoff().unlock(); + core::mem::drop(inner); + fence(Ordering::SeqCst); + mem::tlbi_asid(KERNEL_ALIAS_CTX as u8); + mem::sync(); + Ok(()) + } + /// Creates the reference-counted inner data for a new `Uat` instance. #[inline(never)] - fn make_inner(dev: driver::AsahiDevice) -> Result> { + fn make_inner(dev: driver::AsahiDevice, firmware: UatFirmware) -> Result> { let handoff_rgn = Self::map_region(dev.as_ref().as_ref(), c"handoff", HANDOFF_SIZE, true)?; let ttbs_rgn = Self::map_region(dev.as_ref().as_ref(), c"ttbs", SLOTS_SIZE, true)?; @@ -1664,12 +2502,16 @@ impl Uat { Arc::pin_init( try_pin_init!(UatInner { + firmware, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), shared <- new_mutex!( UatShared { kernel_ttb1: 0, + execution_contexts: !EXECUTION_CONTEXT_MASK, + context_generations: [0; UAT_NUM_CTX], + secondary_ttb1: None, map_kernel_to_user: false, handoff_rgn, ttbs_rgn, @@ -1681,23 +2523,33 @@ impl Uat { ) } - /// Creates a new `Uat` instance for the given address space geometry. + /// Creates a new `Uat` instance for the given address space geometry and firmware. #[inline(never)] pub(crate) fn new( dev: driver::AsahiDevice, geometry: UatGeometry, map_kernel_to_user: bool, + firmware: UatFirmware, ) -> Result { dev_info!(dev.as_ref(), "MMU: Initializing...\n"); - let inner = Self::make_inner(dev.clone())?; + let inner = Self::make_inner(dev.clone(), firmware)?; let of_node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; let res = of_node.reserved_mem_region_to_resource_byname(c_str!("pagetables"))?; let ttb1 = res.start(); let ttb1size: usize = res.size().try_into()?; - if ttb1size < PAGETABLES_SIZE { + let (pagetables_size, secondary_ttb1) = match firmware { + UatFirmware::Handoff => (PAGETABLES_SIZE, None), + UatFirmware::PublishedRoots { + secondary_root_offset, + } => ( + secondary_root_offset as usize + UAT_PGSZ, + Some(ttb1 + secondary_root_offset), + ), + }; + if ttb1size < pagetables_size { dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); return Err(ENOMEM); } @@ -1744,8 +2596,9 @@ impl Uat { inner.map_kernel_to_user = map_kernel_to_user; inner.kernel_ttb1 = ttb1; + inner.secondary_ttb1 = secondary_ttb1; - inner.handoff().init()?; + inner.handoff().init(firmware)?; dev_info!(dev.as_ref(), "MMU: Initializing TTBs\n"); @@ -1761,6 +2614,16 @@ impl Uat { ctx.ttb1.store(0, Ordering::Relaxed); } + if secondary_ttb1.is_some() { + let alias = &ttbs[KERNEL_ALIAS_CTX]; + alias + .ttb0 + .store(tagged_root(ttb0, KERNEL_ALIAS_CTX), Ordering::SeqCst); + alias + .ttb1 + .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); + } + inner.handoff().unlock(); core::mem::drop(inner); diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs new file mode 100644 index 0000000000000..9bd9a50331860 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Prepared userspace mapping transactions. Host callback resources are reserved for the +//! complete ioctl before its first visible mapping changes; leaf commits retain their +//! individual interruptible GPUVM reservation acquisition and quarantine admission check. + +use super::*; + +pub(crate) struct PreparedUserMap { + gem: ARef, + vm_bo: ARef>, + resources: VmOpResources, + prot: Prot, + addr: u64, + size: u64, + offset: u64, + single_page: bool, +} + +pub(crate) struct PreparedUserUnmap { + pub(super) iova: u64, + pub(super) size: u64, + resources: VmOpResources, +} + +pub(super) enum PreparedUserBindOp { + Map(PreparedUserMap), + Unmap(PreparedUserUnmap), +} + +pub(crate) struct PreparedUserBindBatch { + pub(super) ops: KVec, +} + +impl PreparedUserBindBatch { + pub(crate) fn new(capacity: usize) -> Result { + Ok(Self { + ops: KVec::with_capacity(capacity, GFP_KERNEL)?, + }) + } + pub(crate) fn push_map(&mut self, map: PreparedUserMap) -> Result { + Ok(self.ops.push(PreparedUserBindOp::Map(map), GFP_KERNEL)?) + } + pub(crate) fn push_unmap(&mut self, unmap: PreparedUserUnmap) -> Result { + Ok(self + .ops + .push(PreparedUserBindOp::Unmap(unmap), GFP_KERNEL)?) + } +} + +impl Vm { + /// Prepares host state without taking DMA reservations or changing any leaf. + #[allow(clippy::too_many_arguments)] + pub(crate) fn prepare_bind_object( + &self, + gem: &gem::Object, + addr: u64, + size: u64, + offset: u64, + prot: Prot, + single_page: bool, + ) -> Result { + let resources = VmOpResources { + preallocated_gpuvas: [ + Some(GpuVaAlloc::::new(GFP_KERNEL)?), + Some(GpuVaAlloc::::new(GFP_KERNEL)?), + Some(GpuVaAlloc::::new(GFP_KERNEL)?), + ], + }; + // SAFETY: As in `Vm::bind_object()`, the device is bound while its file can issue binds. + let dev = unsafe { self.dev.as_bound() }; + // Populate the scatter-gather table before any GPUVM lock is taken. + gem.sg_table(dev)?; + let vm_bo = self + .gpuvm + .obtain(gem, pin_init!(VmBoData { segments <- new_mutex!(None) }))?; + { + let mut segments = vm_bo.data().segments.lock(); + if segments.is_none() { + segments.replace(prefetch_sgt_segments(dev, gem)?); + } + } + if (addr | size | offset) & UAT_PGMSK as u64 != 0 { + return Err(EINVAL); + } + Ok(PreparedUserMap { + gem: gem.into(), + vm_bo, + resources, + prot, + addr, + size, + offset, + single_page, + }) + } + + pub(crate) fn prepare_user_unmap(&self, iova: u64, size: u64) -> Result { + Ok(PreparedUserUnmap { + iova, + size, + resources: VmOpResources { + preallocated_gpuvas: [ + Some(GpuVaAlloc::::new(GFP_KERNEL)?), + Some(GpuVaAlloc::::new(GFP_KERNEL)?), + None, + ], + }, + }) + } + + /// Commits in ioctl order. Quarantine may defer an all-unmap batch, but can + /// never allow a map to replace a mapping still owned by failed work. + pub(crate) fn commit_prepared_user_bind_batch( + &self, + batch: &mut PreparedUserBindBatch, + ) -> Result { + if self.defer_bind_batch(batch)? { + return Ok(()); + } + let mut index = 0; + while index < batch.ops.len() { + let result = match &mut batch.ops[index] { + PreparedUserBindOp::Map(map) => { + // SAFETY: See `lock_vm()`. As in `Vm::bind_object()`, the object itself is + // not locked: the map step may still populate its pages. + let mut inner = unsafe { self.gpuvm.exec_lock(None, true)? }; + match self.mapping_commit() { + Ok(commit) => { + let (flags, repeat) = if map.single_page { + (GpuVaFlag::REPEAT.into(), UAT_PGSZ as u32) + } else { + (GpuVaFlags::empty(), 0) + }; + // SAFETY: As in `Vm::bind_object()`. + let dev = unsafe { self.dev.as_bound() }; + let mut ctx = StepContext::new(Some(dev), map.prot, &mut map.resources)?; + let result = inner.unique_gpuvm.sm_map(OpMapRequest { + addr: map.addr, + range: map.size, + gem_offset: map.offset, + gem_range: repeat, + vm_bo: &map.vm_bo, + context: &mut ctx, + flags, + }); + if result.is_ok() { + self.track_shared_binding( + &map.gem, + map.addr, + map.size, + map.offset, + map.single_page, + ); + } + drop(inner); + drop(commit); + Some(result) + } + Err(error) if error == EIO => { + drop(inner); + None + } + Err(error) => return Err(error), + } + } + PreparedUserBindOp::Unmap(unmap) => { + // SAFETY: See `lock_vm()`. + let mut inner = unsafe { self.gpuvm.exec_lock(None, true)? }; + match self.mapping_commit() { + Ok(commit) => { + let mut ctx = + StepContext::new(None, Prot::default(), &mut unmap.resources)?; + let result = + inner + .unique_gpuvm + .sm_unmap(unmap.iova, unmap.size, &mut ctx); + if result.is_ok() { + self.untrack_shared_range(unmap.iova, unmap.size); + } + drop(inner); + drop(commit); + Some(result) + } + Err(error) if error == EIO => { + drop(inner); + None + } + Err(error) => return Err(error), + } + } + }; + if let Some(result) = result { + result?; + index += 1; + } else if self.defer_bind_batch(batch)? { + return Ok(()); + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/mmu/context.rs b/drivers/gpu/drm/asahi/mmu/context.rs new file mode 100644 index 0000000000000..43244c2ce1730 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/context.rs @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Tracks canonical context resources and retains their fixed low presentation. +//! +//! Userspace owns low views 1 through 6. Only the reserved null view is created +//! here. Its installation is serialized per VM and retained through VM teardown. + +use super::*; +use crate::hw::t8140::CONTEXT_VIEWS; +use kernel::drm::gem::BaseObject; + +struct Binding { + gem: ARef, + offset: usize, +} +struct Bindings { + generation: u64, + slots: [Option; CONTEXT_VIEWS.len()], +} +#[pin_data] +pub(super) struct ContextBindings { + #[pin] + bindings: Mutex, + #[pin] + install: Mutex<()>, +} +impl ContextBindings { + pub(super) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + bindings <- new_mutex!(Bindings { generation: 0, + slots: core::array::from_fn(|_| None) }, "VM context resources"), + install <- new_mutex!((), "VM context alias installation"), + }), + GFP_KERNEL, + ) + } +} + +impl Vm { + pub(crate) fn validate_context_binding( + &self, + address: u64, + size: u64, + single_page: bool, + ) -> Result { + if self.context_bindings.is_none() || !single_page { + return Ok(()); + } + let end = address.checked_add(size).ok_or(EOVERFLOW)?; + if CONTEXT_VIEWS + .iter() + .any(|view| address < view.source + view.size as u64 && view.source < end) + { + return Err(EINVAL); + } + Ok(()) + } + + /// Arithmetic is preflighted before invalidating any retained candidate. + pub(crate) fn track_context_binding( + &self, + gem: ARef, + address: u64, + size: u64, + object_offset: u64, + ) -> Result { + let Some(context) = self.context_bindings.as_ref() else { + return Ok(()); + }; + let end = address.checked_add(size).ok_or(EOVERFLOW)?; + let mut offsets = [None; CONTEXT_VIEWS.len()]; + for (index, view) in CONTEXT_VIEWS.iter().enumerate() { + if address <= view.source && view.source + view.size as u64 <= end { + offsets[index] = Some(usize::try_from( + object_offset + .checked_add(view.source - address) + .ok_or(EOVERFLOW)?, + )?); + } + } + let mut bindings = context.bindings.lock(); + let mut touched = false; + for (index, view) in CONTEXT_VIEWS.iter().enumerate() { + if address < view.source + view.size as u64 && view.source < end { + bindings.slots[index] = None; + touched = true; + } + if let Some(offset) = offsets[index] { + bindings.slots[index] = Some(Binding { + gem: gem.clone(), + offset, + }); + touched = true; + } + } + if touched { + bindings.generation = bindings.generation.wrapping_add(1); + } + Ok(()) + } + + pub(crate) fn untrack_context_range(&self, range: Range) { + let Some(context) = self.context_bindings.as_ref() else { + return; + }; + let mut bindings = context.bindings.lock(); + let mut touched = false; + for (index, view) in CONTEXT_VIEWS.iter().enumerate() { + if range.start < view.source + view.size as u64 && view.source < range.end { + bindings.slots[index] = None; + touched = true; + } + } + if touched { + bindings.generation = bindings.generation.wrapping_add(1); + } + } + + pub(crate) fn untrack_context_object(&self, gem: &gem::Object) { + let Some(context) = self.context_bindings.as_ref() else { + return; + }; + let mut bindings = context.bindings.lock(); + let mut touched = false; + for slot in &mut bindings.slots { + if slot + .as_ref() + .is_some_and(|binding| core::ptr::eq(&*binding.gem, gem)) + { + *slot = None; + touched = true; + } + } + if touched { + bindings.generation = bindings.generation.wrapping_add(1); + } + } + + pub(crate) fn translate_iova(&self, address: u64) -> Result { + lock_vm(&self.gpuvm, None, false)? + .page_table + .translate_iova(address) + } + + /// Requires a job mapping guard before validating or installing these views. + pub(crate) fn install_job_context_aliases(&self) -> Result { + let context = self.context_bindings.as_ref().ok_or(EINVAL)?; + let (generation, snapshot) = { + let bindings = context.bindings.lock(); + let mut snapshot = KVec::with_capacity(CONTEXT_VIEWS.len(), GFP_KERNEL)?; + for slot in &bindings.slots { + let binding = slot.as_ref().ok_or(ENOENT)?; + snapshot.push((binding.gem.clone(), binding.offset), GFP_KERNEL)?; + } + (bindings.generation, snapshot) + }; + for (index, (gem, offset)) in snapshot.iter().enumerate().skip(1) { + let view = CONTEXT_VIEWS[index]; + if !self.covers_range(view.source, view.size as u64, true, view.write) + || !self.covers_range(view.low, view.size as u64, true, view.write) + { + return Err(EFAULT); + } + if offset.checked_add(view.size).ok_or(EOVERFLOW)? > gem.size() { + return Err(ERANGE); + } + for page in (0..view.size).step_by(UAT_PGSZ) { + if self.translate_iova(view.source + page as u64)? + != self.translate_iova(view.low + page as u64)? + { + return Err(EFAULT); + } + } + } + let _install = context.install.lock(); + let view = CONTEXT_VIEWS[0]; + let (gem, offset) = &snapshot[0]; + // Null presentation requires writes in addition to the canonical view's + // ordinary read requirement. + if !self.covers_range(view.source, view.size as u64, true, true) { + return Err(EFAULT); + } + let end = offset.checked_add(view.size).ok_or(EOVERFLOW)?; + if end > gem.size() { + return Err(ERANGE); + } + let mut existing = 0; + for page in (0..view.size).step_by(UAT_PGSZ) { + if self.covers_range(view.low + page as u64, UAT_PGSZ as u64, true, true) { + existing += 1; + if self.translate_iova(view.source + page as u64)? + != self.translate_iova(view.low + page as u64)? + { + return Err(EFAULT); + } + } + } + if existing != 0 { + if existing != view.size / UAT_PGSZ { + return Err(EFAULT); + } + return if context.bindings.lock().generation == generation { + Ok(()) + } else { + Err(EAGAIN) + }; + } + let mut mappings = KVec::with_capacity(1, GFP_KERNEL)?; + let mapping = self.map_in_range( + gem, + *offset..end, + UAT_PGSZ as u64, + view.low..view.low + view.size as u64, + PROT_GPU_SHARED_RW, + false, + )?; + for page in (0..view.size).step_by(UAT_PGSZ) { + if self.translate_iova(view.source + page as u64)? + != self.translate_iova(view.low + page as u64)? + { + return Err(EFAULT); + } + } + mappings.push(mapping, GFP_KERNEL)?; + if context.bindings.lock().generation != generation { + return Err(EAGAIN); + } + self.append_driver_mappings(mappings) + } +} diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs new file mode 100644 index 0000000000000..3b8a5e5a40aeb --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -0,0 +1,605 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Keeps user mappings alive until accepted work has relinquished firmware ownership. +//! +//! The state lock is taken after GPUVM's exec lock when committing user mappings. Waiting for +//! that lock's commits never holds a GPUVM reservation. Draining takes no state lock while it +//! unmaps memory. Driver aliases retain only the inner GPUVM, so they create no cycle with Vm. + +use super::*; +use kernel::{ + new_condvar, + sync::{CondVar, CondVarTimeoutResult}, + time::msecs_to_jiffies, +}; + +const ADMISSION_WAIT_MS: u32 = 1000; + +pub(super) struct DriverMappings { + mappings: KVec, + reserved: KVec>, + compute: Option, + render: [Option; crate::hw::t8140::queues::RENDER_SLOTS], +} + +struct PoolMappings { + id: u64, + blocks: usize, + mappings: KVec, +} + +impl DriverMappings { + fn ranges(&self) -> impl Iterator> + '_ { + Iterator::chain( + Iterator::chain( + self.mappings.iter(), + Iterator::chain( + self.compute.iter().flat_map(|pool| pool.mappings.iter()), + self.render + .iter() + .flatten() + .flat_map(|pool| pool.mappings.iter()), + ), + ) + .map(KernelMapping::iova_range), + self.reserved.iter().cloned(), + ) + } +} + +struct State { + active: usize, + commits: usize, + blocked: bool, + quarantined: bool, + draining: bool, + closed: bool, + close_ranges: Option<(Range, Range)>, + unmaps: KVec<(u64, u64)>, + objects: KVec>, +} + +#[pin_data] +pub(super) struct VmLifetime { + #[pin] + state: Mutex, + #[pin] + changed: CondVar, + #[pin] + driver: Mutex>, +} + +impl VmLifetime { + pub(super) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + state <- new_mutex!(State { active:0, commits:0, blocked:false, + quarantined:false, draining:false, closed:false, close_ranges:None, + unmaps:KVec::new(), objects:KVec::new() }, "VM job lifetime"), + changed <- new_condvar!("VM mapping admission"), + driver <- new_mutex!(None, "VM driver mappings"), + }), + GFP_KERNEL, + ) + } +} + +/// Retains a VM's mappings through a command's exact retirement. An error fence is not +/// retirement: potentially visible work must retain its guard through quarantine or stop. +pub(crate) struct VmJobGuard { + vm: Vm, +} + +impl VmJobGuard { + /// Blocks mapping commits before the error fence can become visible to userspace. + pub(crate) fn quarantine(&self) { + if let Some(lifetime) = self.vm.lifetime.as_ref() { + let mut state = lifetime.state.lock(); + state.blocked = true; + while state.commits != 0 { + lifetime.changed.wait(&mut state); + } + state.quarantined = true; + } + } +} + +pub(super) struct MappingCommit { + lifetime: Arc, +} +impl Drop for MappingCommit { + fn drop(&mut self) { + let mut state = self.lifetime.state.lock(); + state.commits -= 1; + if state.commits == 0 { + self.lifetime.changed.notify_all(); + } + } +} + +impl Vm { + /// Attaches aliases while constructing this VM, before exposing it to userspace. + pub(crate) fn install_driver_mappings( + &self, + mappings: KVec, + reserved: KVec>, + ) -> Result { + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut driver = lifetime.driver.lock(); + if driver.is_some() { + return Err(EBUSY); + } + *driver = Some(DriverMappings { + mappings, + reserved, + compute: None, + render: core::array::from_fn(|_| None), + }); + Ok(()) + } + + /// Reserves the complete append before moving aliases into VM ownership. + pub(crate) fn append_driver_mappings(&self, mappings: KVec) -> Result { + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut driver = lifetime.driver.lock(); + let driver = Option::as_mut(&mut *driver).ok_or(EINVAL)?; + driver.mappings.reserve(mappings.len(), GFP_KERNEL)?; + for mapping in mappings { + driver.mappings.push(mapping, GFP_KERNEL)?; + } + Ok(()) + } + + pub(crate) fn render_pool_tvb_blocks(&self, slot: u8, id: u64) -> Option { + if id == 0 { + return None; + } + self.lifetime + .as_ref()? + .driver + .lock() + .as_ref()? + .render + .get(usize::from(slot))? + .as_ref() + .filter(|pool| pool.id == id) + .map(|pool| pool.blocks) + } + + /// Retains a new generation after all references to a replaced pool retire. + pub(crate) fn install_render_pool_mappings( + &self, + slot: u8, + id: u64, + blocks: usize, + mappings: KVec, + ) -> Result { + if id == 0 { + return Err(EINVAL); + } + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut driver = lifetime.driver.lock(); + let driver = Option::as_mut(&mut *driver).ok_or(EINVAL)?; + let pool = driver.render.get_mut(usize::from(slot)).ok_or(EINVAL)?; + if pool.is_some() { + return Err(EBUSY); + } + *pool = Some(PoolMappings { + id, + blocks, + mappings, + }); + Ok(()) + } + + /// Consumes aliases only after the exact cached owner and complete append + /// capacity have been checked. Failed preparation retains its mappings for + /// destruction after the caller releases the device mutex. + pub(crate) fn append_render_pool_mappings_prepared( + &self, + slot: u8, + id: u64, + previous: usize, + blocks: usize, + mappings: &mut KVec, + ) -> Result { + if id == 0 || blocks <= previous || mappings.len() != blocks - previous { + return Err(EINVAL); + } + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut driver = lifetime.driver.lock(); + let pool = Option::as_mut(&mut *driver) + .and_then(|driver| driver.render.get_mut(usize::from(slot))) + .and_then(Option::as_mut) + .filter(|pool| pool.id == id) + .ok_or(EINVAL)?; + if pool.blocks != previous { + return Err(EBUSY); + } + pool.mappings + .len() + .checked_add(mappings.len()) + .ok_or(EOVERFLOW)?; + pool.mappings.reserve(mappings.len(), GFP_KERNEL)?; + for mapping in mappings.drain_all() { + pool.mappings.push(mapping, GFP_KERNEL)?; + } + pool.blocks = blocks; + Ok(()) + } + + /// A delayed teardown must not remove a successor generation. Remove + /// under the cache lock, then unmap after dropping that lock. + pub(crate) fn clear_render_pool_mappings_if_owner(&self, slot: u8, id: u64) { + if id == 0 { + return; + } + let previous = self.lifetime.as_ref().and_then(|lifetime| { + let mut driver = lifetime.driver.lock(); + Option::as_mut(&mut *driver) + .and_then(|driver| driver.render.get_mut(usize::from(slot))) + .filter(|pool| pool.as_ref().is_some_and(|pool| pool.id == id)) + .and_then(Option::take) + }); + drop(previous); + } + + pub(crate) fn compute_pool_mappings_match(&self, id: u64) -> bool { + id != 0 + && self.lifetime.as_ref().is_some_and(|lifetime| { + lifetime + .driver + .lock() + .as_ref() + .is_some_and(|driver| driver.compute.as_ref().is_some_and(|pool| pool.id == id)) + }) + } + + /// The device publication lock serializes first-use alias construction. + /// Never replace a live generation; its jobs must retire before clearing. + pub(crate) fn install_compute_pool_mappings( + &self, + id: u64, + mappings: KVec, + ) -> Result { + if id == 0 { + return Err(EINVAL); + } + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut driver = lifetime.driver.lock(); + let driver = Option::as_mut(&mut *driver).ok_or(EINVAL)?; + if driver.compute.is_some() { + return Err(EBUSY); + } + driver.compute = Some(PoolMappings { + id, + blocks: 0, + mappings, + }); + Ok(()) + } + + /// Called under the publication lock after all obsolete pool work retires. + /// Drop outside the driver lock because mapping teardown acquires GPUVM locks. + pub(crate) fn clear_compute_pool_mappings(&self) { + let previous = self.lifetime.as_ref().and_then(|lifetime| { + let mut driver = lifetime.driver.lock(); + Option::as_mut(&mut *driver).and_then(|driver| driver.compute.take()) + }); + drop(previous); + } + + /// Tests mappings and reservations which userspace must never replace. + pub(crate) fn driver_range_overlaps(&self, range: Range) -> bool { + let Some(lifetime) = self.lifetime.as_ref() else { + return false; + }; + let driver = lifetime.driver.lock(); + driver.as_ref().is_some_and(|driver| { + driver + .ranges() + .any(|reserved| range.start < reserved.end && reserved.start < range.end) + }) + } + + /// Waits before reservation acquisition, allowing healthy retirement to drain. + pub(crate) fn wait_for_user_map_admission(&self) -> Result { + let Some(lifetime) = self.lifetime.as_ref() else { + return Ok(()); + }; + let mut state = lifetime.state.lock(); + let mut remaining = msecs_to_jiffies(ADMISSION_WAIT_MS); + loop { + let failed = self.status.as_ref().is_some_and(|status| status.get() != 0); + if !failed + && !state.closed + && (state.blocked || (state.active != 0 && state.quarantined)) + && remaining != 0 + { + remaining = match lifetime + .changed + .wait_interruptible_timeout(&mut state, remaining) + { + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + CondVarTimeoutResult::Timeout => 0, + CondVarTimeoutResult::Woken { jiffies } => jiffies, + }; + continue; + } + if state.closed { + return Err(ENOENT); + } + if failed || state.blocked || (state.active != 0 && state.quarantined) { + return Err(EIO); + } + if !state.draining { + return Ok(()); + } + if lifetime.changed.wait_interruptible(&mut state) { + return Err(ERESTARTSYS); + } + } + } + + /// Acquires a job reference without imposing a VM-wide limit on healthy work. + pub(crate) fn retain_job(&self) -> Result { + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut state = lifetime.state.lock(); + loop { + if state.closed { + return Err(ENOENT); + } + if state.blocked { + return Err(EIO); + } + if !state.draining { + state.active = state.active.checked_add(1).ok_or(EOVERFLOW)?; + return Ok(VmJobGuard { vm: self.clone() }); + } + lifetime.changed.wait(&mut state); + } + } + + /// The first job of a submission may wait for a clean VM's quarantine drain. + pub(crate) fn retain_first_job(&self) -> Result { + match self.retain_job() { + Err(error) + if error == EIO && self.status.as_ref().is_some_and(|status| status.get() == 0) => { + } + result => return result, + } + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut remaining = msecs_to_jiffies(ADMISSION_WAIT_MS); + let mut state = lifetime.state.lock(); + while state.blocked + && remaining != 0 + && self.status.as_ref().is_some_and(|status| status.get() == 0) + { + remaining = match lifetime + .changed + .wait_interruptible_timeout(&mut state, remaining) + { + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + CondVarTimeoutResult::Timeout => 0, + CondVarTimeoutResult::Woken { jiffies } => jiffies, + }; + } + drop(state); + self.retain_job() + } + + /// Called only after the GPUVM exec/reservation locks have been acquired. + pub(super) fn mapping_commit(&self) -> Result> { + let Some(lifetime) = self.lifetime.as_ref() else { + return Ok(None); + }; + let mut state = lifetime.state.lock(); + if state.closed { + return Err(ENOENT); + } + if state.blocked { + return Err(EIO); + } + state.commits = state.commits.checked_add(1).ok_or(EOVERFLOW)?; + Ok(Some(MappingCommit { + lifetime: lifetime.clone(), + })) + } + + /// Reserves cleanup bookkeeping before a batch starts committing leaves. + pub(crate) fn reserve_deferred_user_unmaps(&self, count: usize) -> Result { + if let Some(lifetime) = self.lifetime.as_ref() { + lifetime.state.lock().unmaps.reserve(count, GFP_KERNEL)?; + } + Ok(()) + } + + pub(super) fn defer_bind_batch(&self, batch: &PreparedUserBindBatch) -> Result { + use super::bind::PreparedUserBindOp; + let Some(lifetime) = self.lifetime.as_ref() else { + return Ok(false); + }; + let mut state = lifetime.state.lock(); + if !state.blocked { + return Ok(false); + } + if state.closed { + return Err(ENOENT); + } + if batch + .ops + .iter() + .any(|op| matches!(op, PreparedUserBindOp::Map(_))) + { + return Err(EIO); + } + if !state.quarantined { + return Err(EBUSY); + } + for op in &batch.ops { + if let PreparedUserBindOp::Unmap(unmap) = op { + let range = (unmap.iova, unmap.size); + if !state.unmaps.contains(&range) { + state.unmaps.push(range, GFP_KERNEL)?; + } + } + } + Ok(true) + } + + /// Returns true if a quarantined owner requires this unmap to be deferred. + pub(super) fn defer_unmap(&self, iova: u64, size: u64) -> Result { + let Some(lifetime) = self.lifetime.as_ref() else { + return Ok(false); + }; + let mut state = lifetime.state.lock(); + if state.closed { + return Err(ENOENT); + } + if !state.blocked { + return Ok(false); + } + if !state.quarantined { + return Err(EBUSY); + } + if !state.unmaps.contains(&(iova, size)) { + state.unmaps.push((iova, size), GFP_KERNEL)?; + } + Ok(true) + } + + /// Defers handle-close cleanup while any accepted job can still use the BO. + pub(crate) fn drop_mappings(&self, gem: &gem::Object) -> Result { + let Some(lifetime) = self.lifetime.as_ref() else { + return self.drop_mappings_now(gem); + }; + let mut state = lifetime.state.lock(); + if !state + .objects + .iter() + .any(|object| core::ptr::eq(&**object, gem)) + { + state.objects.push(gem.into(), GFP_KERNEL)?; + } + if state.active != 0 || state.draining { + return Ok(()); + } + state.draining = true; + drop(state); + self.drain_mappings(lifetime); + Ok(()) + } + + /// Closes admission permanently and preserves mappings referenced by accepted jobs. + pub(crate) fn unmap_user_ranges(&self, user: Range, kernel: Range) -> Result { + if let Some(lifetime) = self.lifetime.as_ref() { + let mut state = lifetime.state.lock(); + state.blocked = true; + while state.commits != 0 { + lifetime.changed.wait(&mut state); + } + state.closed = true; + if state.active != 0 { + state.close_ranges = Some((user, kernel)); + return Ok(()); + } + } + self.unmap_user_ranges_now(user, kernel) + } + + fn unmap_user_ranges_now(&self, user: Range, kernel: Range) -> Result { + let driver = self + .lifetime + .as_ref() + .map(|lifetime| lifetime.driver.lock()); + let mut cursor = user.start; + while cursor < user.end { + let mut next = user.end; + let mut covered = cursor; + for range in Iterator::chain( + core::iter::once(kernel.clone()), + driver + .as_ref() + .and_then(|guard| guard.as_ref()) + .into_iter() + .flat_map(DriverMappings::ranges), + ) { + if range.contains(&cursor) { + covered = covered.max(range.end); + } else if cursor < range.start { + next = next.min(range.start); + } + } + if covered != cursor { + cursor = covered.min(user.end); + continue; + } + if next <= cursor { + return Err(EIO); + } + self.unmap_range_now(cursor, next - cursor)?; + cursor = next; + } + Ok(()) + } +} + +impl Drop for VmJobGuard { + fn drop(&mut self) { + let Some(lifetime) = self.vm.lifetime.as_ref() else { + return; + }; + let mut state = lifetime.state.lock(); + state.active -= 1; + if state.active != 0 { + return; + } + state.draining = true; + drop(state); + self.vm.drain_mappings(lifetime); + } +} + +impl Vm { + /// Admission stays closed to new jobs while the exec lock is acquired for cleanup. + /// Recheck queued cleanup and reopen under the same lock used to append it. + fn drain_mappings(&self, lifetime: &VmLifetime) { + let mut state = lifetime.state.lock(); + loop { + let close = state.close_ranges.take(); + let unmaps = core::mem::replace(&mut state.unmaps, KVec::new()); + let objects = core::mem::replace(&mut state.objects, KVec::new()); + drop(state); + if let Some((user, kernel)) = close { + if self.unmap_user_ranges_now(user, kernel).is_err() { + pr_err!("MMU: deferred VM close failed\n"); + } + } else { + for (address, size) in unmaps { + if self.unmap_range_now(address, size).is_err() { + pr_err!("MMU: deferred user unmap failed\n"); + } + } + } + for object in objects { + if self.drop_mappings_now(&object).is_err() { + pr_err!("MMU: deferred object unmap failed\n"); + } + } + state = lifetime.state.lock(); + if state.close_ranges.is_some() || !state.unmaps.is_empty() || !state.objects.is_empty() + { + continue; + } + state.quarantined = false; + state.draining = false; + if !state.closed { + state.blocked = false; + } + drop(state); + lifetime.changed.notify_all(); + break; + } + self.bo_deferred_cleanup(); + } +} diff --git a/drivers/gpu/drm/asahi/mmu/shared.rs b/drivers/gpu/drm/asahi/mmu/shared.rs new file mode 100644 index 0000000000000..cf29e53fc9cc9 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/shared.rs @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Shared-object identities for attachment ordering across user address spaces. +//! These records do not own mappings. The VM's job guard protects the mappings +//! while submission captures the intersecting object offsets. Mapping commits +//! hold the GPUVM exec lock through metadata updates, then release it. The lock +//! order is GPUVM exec followed by bindings; footprint readers take only bindings. + +use super::*; +use crate::g17::{hazard::Footprint, Attachments}; + +struct Binding { + start: u64, + end: u64, + object: usize, + offset: u64, +} + +#[pin_data] +pub(super) struct SharedBindings { + #[pin] + bindings: Mutex>, + count: AtomicU32, +} + +impl SharedBindings { + pub(super) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + bindings <- new_mutex!(KVec::new(), "Shared attachment bindings"), + count: AtomicU32::new(0), + }), + GFP_KERNEL, + ) + } +} + +impl Vm { + pub(super) fn track_shared_binding( + &self, + gem: &gem::Object, + address: u64, + size: u64, + offset: u64, + single_page: bool, + ) { + let Some(shared) = &self.shared_bindings else { + return; + }; + let Some(end) = address.checked_add(size) else { + return; + }; + let external = !single_page && self.is_extobj(gem); + if !external && shared.count.load(Ordering::Relaxed) == 0 { + return; + } + let mut bindings = shared.bindings.lock(); + bindings.retain(|entry| entry.end <= address || end <= entry.start); + if external { + let _ = bindings.push( + Binding { + start: address, + end, + object: gem as *const gem::Object as usize, + offset, + }, + GFP_KERNEL, + ); + } + shared.count.store(bindings.len() as u32, Ordering::Relaxed); + } + + pub(super) fn untrack_shared_range(&self, address: u64, size: u64) { + let Some(shared) = &self.shared_bindings else { + return; + }; + if shared.count.load(Ordering::Relaxed) == 0 { + return; + } + let Some(end) = address.checked_add(size) else { + return; + }; + let mut bindings = shared.bindings.lock(); + bindings.retain(|entry| entry.end <= address || end <= entry.start); + shared.count.store(bindings.len() as u32, Ordering::Relaxed); + } + + pub(crate) fn untrack_shared_object(&self, gem: &gem::Object) { + let Some(shared) = &self.shared_bindings else { + return; + }; + if shared.count.load(Ordering::Relaxed) == 0 { + return; + } + let object = gem as *const gem::Object as usize; + let mut bindings = shared.bindings.lock(); + bindings.retain(|entry| entry.object != object); + shared.count.store(bindings.len() as u32, Ordering::Relaxed); + } + + pub(crate) fn render_footprint( + &self, + attachments: &Attachments, + ) -> Result>> { + if attachments.as_slice().is_empty() { + return Ok(None); + } + let mut footprint = Footprint::new(self.id); + for attachment in attachments.as_slice() { + let end = attachment.va.saturating_add(attachment.size); + footprint.push_vm(attachment.va, end); + let Some(shared) = &self.shared_bindings else { + continue; + }; + if shared.count.load(Ordering::Relaxed) == 0 { + continue; + } + let bindings = shared.bindings.lock(); + for entry in bindings.iter() { + let first = entry.start.max(attachment.va); + let last = entry.end.min(end); + if first < last { + let offset = entry.offset.saturating_add(first - entry.start); + footprint.push_object( + entry.object, + offset, + offset.saturating_add(last - first), + ); + } + } + } + Ok(Some(Arc::new(footprint, GFP_KERNEL)?)) + } +} diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 0d814d6d100a8..6a8dba48174f0 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -53,6 +53,32 @@ const UAT_LEVELS: usize = 3; /// Number of address bits below the index of a top-level page table entry. pub(crate) const UAT_TOP_LEVEL_SHIFT: usize = UAT_PGBIT + (UAT_LEVELS - 1) * UAT_LVBIT; +/// Runs `f` on the first `count` top-level entries of the root page table at `ttb`. +/// +/// # Safety +/// +/// `ttb` must be the physical address of a page table root in RAM that stays allocated while `f` +/// runs. Other agents (the GPU firmware) may access the entries concurrently, which is why they +/// are only exposed as atomics. +pub(crate) unsafe fn with_root_entries( + ttb: PhysicalAddr, + count: usize, + f: impl FnOnce(&[AtomicU64]) -> R, +) -> Result { + if count > UAT_NPTE { + return Err(EINVAL); + } + // SAFETY: The caller guarantees that `ttb` is an allocated page in RAM. + let page = unsafe { Page::borrow_phys(&ttb) }.ok_or(EIO)?; + page.with_pointer_into_page(0, count * PTE_SIZE, |p| { + // SAFETY: with_pointer_into_page() ensures the pointer is valid for `count` PTEs, and page + // table entries are naturally aligned. + Ok(f(unsafe { + core::slice::from_raw_parts(p as *const AtomicU64, count) + })) + }) +} + const PTE_TYPE_BITS: u64 = 3; const PTE_TYPE_LEAF_TABLE: u64 = 3; @@ -146,6 +172,14 @@ impl Prot { } } + pub(crate) const fn from_pte(pte: u64) -> Self { + Prot { + high_bits: (pte >> UAT_HIGH_BITS_SHIFT) as u16, + ap: ((pte & UAT_AP_BITS) >> UAT_AP_SHIFT) as u8, + memattr: ((pte & UAT_MEMATTR_BITS) >> UAT_MEMATTR_SHIFT) as u8, + } + } + const fn memattr(&self, memattr: u8) -> Self { Self { memattr, ..*self } } @@ -157,6 +191,19 @@ impl Prot { | UAT_AF } + /// Whether this leaf allows the requested GPU accesses. + pub(crate) const fn allows_gpu(&self, need_read: bool, need_write: bool) -> bool { + let uxn = self.high_bits & HIGH_BITS_UXN != 0; + let pxn = self.high_bits & HIGH_BITS_PXN != 0; + let (readable, writable) = match self.ap { + AP_FW_GPU => (pxn, uxn), + AP_FW => (uxn && pxn, false), + AP_GPU => (!pxn, uxn != pxn), + _ => (false, false), + }; + (!need_read || readable) && (!need_write || writable) + } + pub(crate) const fn is_cached_noncoherent(&self) -> bool { self.ap != AP_GPU && self.memattr == MEMATTR_CACHED } @@ -423,6 +470,59 @@ impl UatPageTable { unsafe { Page::from_phys(phys) }; } + /// Checks every page, including holes skipped by the page-table walker. + pub(crate) fn covers_range( + &mut self, + range: Range, + read: bool, + write: bool, + ) -> Result { + if range.is_empty() || (range.start | range.end) & UAT_PGMSK as u64 != 0 { + return Ok(false); + } + let expected = (range.end - range.start) >> UAT_PGBIT; + let mut visited = 0u64; + let mut permitted = true; + self.with_pages(range, false, false, |_, ptes| { + visited = visited.checked_add(ptes.len() as u64).ok_or(EOVERFLOW)?; + for pte in ptes { + let value = pte.load(Ordering::Acquire); + if value & PTE_TYPE_BITS != PTE_TYPE_LEAF_TABLE + || !Prot::from_pte(value).allows_gpu(read, write) + { + permitted = false; + } + } + Ok(()) + })?; + Ok(expected == visited && permitted) + } + + /// Resolves an existing leaf without creating page tables. + pub(crate) fn translate_iova(&mut self, iova: u64) -> Result { + let entry = self.leaf_entry(iova & !(UAT_PGMSK as u64))?; + Ok((entry & self.oas_mask & !(UAT_PGMSK as u64)) | (iova & UAT_PGMSK as u64)) + } + + /// Reads one mapped leaf entry without allocating missing tables. + pub(crate) fn leaf_entry(&mut self, iova: u64) -> Result { + if iova & UAT_PGMSK as u64 != 0 { + return Err(EINVAL); + } + let end = iova.checked_add(UAT_PGSZ as u64).ok_or(EINVAL)?; + let mut value = 0; + self.with_pages(iova..end, false, false, |_, ptes| { + if let Some(pte) = ptes.first() { + value = pte.load(Ordering::Relaxed); + } + Ok(()) + })?; + if value & PTE_TYPE_BITS != PTE_TYPE_LEAF_TABLE { + return Err(EFAULT); + } + Ok(value) + } + pub(crate) fn alloc_pages(&mut self, iova_range: Range) -> Result { mod_pr_debug!("UATPageTable::alloc_pages: {:#x?}\n", iova_range); self.with_pages(iova_range, true, false, |_, _| Ok(())) diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index 4541709fb7523..7eecb26a8b0ff 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -6,11 +6,15 @@ //! there is very little need to interact directly with GPU MMIO register. This module abstracts //! the few operations that require that, mainly reading the MMU fault status, reading GPU ID //! information, and starting the GPU firmware coprocessor. +//! +//! On G17, the driver also enables the bridge between the GPU and the fabric before the +//! coprocessors start, and enables dynamic power gating before the firmware is initialized. use crate::hw; use kernel::{ c_str, device::Core, + devres::Devres, io::{ mem::IoMem, Io, @@ -19,6 +23,7 @@ use kernel::{ platform, prelude::*, soc::apple::rtkit::ASC_CPU_CONTROL, + str::CStr, sync::{ aref::ARef, Arc, // @@ -33,6 +38,10 @@ pub(crate) const SGX_SIZE: usize = 0x1000000; const FAULT_INFO: usize = 0x17030; +/// A nonzero low nibble permits writes through the G17 queue configuration port. +const G17_HOST_IRQ_SUMMARY: usize = 0xe01000; +const G17_HOST_POWERED: u32 = 0xf; + const ID_VERSION: usize = 0xd04000; const ID_UNK08: usize = 0xd04008; const ID_COUNTS_1: usize = 0xd04010; @@ -47,6 +56,26 @@ const CORE_MASKS_G14X: usize = 0xe01500; const FAULT_INFO_G14X: usize = 0xd8c0; const FAULT_ADDR_G14X: usize = 0xd8c8; +/// ASC CPU control register and its run bit, for the G17 coprocessors. +const CPU_CONTROL: usize = 0x44; +const CPU_RUN: u32 = 0x1 << 4; // BIT(4) + +/// Cleared after `CPU_RUN` when stopping a G17 coprocessor. +const CPU_STOP_ACK: u32 = 0x1 << 5; // BIT(5) + +/// G17 GPU-to-fabric bridge control words. Bit 0 enables the bridge; the other bits are status. +const G17_BRIDGE_CONTROL: [usize; 2] = [0x1000104, 0x1000108]; +const G17_BRIDGE_ENABLE: u32 = 0x1; // BIT(0) + +/// G17 dynamic power gating control. Clearing these bits enables gating of idle GPU cores. +const G17_GATING_CONTROL: usize = 0xd06030; +const G17_GATING_DISABLE: u32 = 0x6; // BIT(1) | BIT(2) + +/// These performance selectors must be sampled before firmware can gate the core registers. +const G17_PERF_CONTROL: usize = 0xe0141c; +const G17_PERF_MAP: usize = 0xe01480; +const G17_CORE_POWER: usize = 0xe01000; + /// Enum representing the unit that caused an MMU fault. #[allow(non_camel_case_types)] #[allow(clippy::upper_case_acronyms)] @@ -152,6 +181,63 @@ pub(crate) struct FaultInfo { pub(crate) reason: FaultReason, } +/// Identification of a G17 GPU. +pub(crate) struct G17Id { + /// GPU family field of the ID register. + pub(crate) family: u8, + /// GPU variant field of the ID register. + pub(crate) variant: u8, + /// Silicon revision. + pub(crate) gpu_rev: hw::GpuRevision, + /// Number of dies. + pub(crate) num_dies: u32, + /// Number of clusters, over all dies. + pub(crate) num_clusters: u32, + /// Number of cores per cluster. + pub(crate) num_cores: u32, + /// Enabled cores of each cluster. + pub(crate) core_masks: KVec, + /// Performance selector control, sampled before core power gating starts. + pub(crate) perf_control: u32, + /// Performance selector table indices, low and high nibbles. + pub(crate) perf_map: [u32; 2], +} + +/// CPU control registers of one GPU coprocessor. +pub(crate) struct CpuControl { + asc: Devres>, +} + +impl CpuControl { + /// Map the CPU control registers in the `name` register region. + pub(crate) fn new(pdev: &platform::Device>, name: &CStr) -> Result { + let req = pdev.io_request_by_name(name).ok_or(EINVAL)?; + Ok(CpuControl { + asc: req.iomap_sized::()?.into_devres()?, + }) + } + + /// Start the coprocessor CPU. + pub(crate) fn start(&self) -> Result { + let asc = self.asc.try_access().ok_or(ENODEV)?; + let asc = (&*asc).as_view(); + let val = asc.read32(CPU_CONTROL); + asc.write32(val | CPU_RUN, CPU_CONTROL); + Ok(()) + } + + /// Stop the coprocessor CPU. This only gates it: its firmware keeps its state. + pub(crate) fn stop(&self) { + if let Some(asc) = self.asc.try_access() { + let asc = (&*asc).as_view(); + let val = asc.read32(CPU_CONTROL); + asc.write32(val & !CPU_RUN, CPU_CONTROL); + let val = asc.read32(CPU_CONTROL); + asc.write32(val & !CPU_STOP_ACK, CPU_CONTROL); + } + } +} + /// Device resources for this GPU instance. pub(crate) struct Resources<'drm> { dev: ARef, @@ -276,43 +362,9 @@ impl Resources<'_> { return Err(ENODEV); } - let mut core_masks = KVec::new(); - let mut total_active_cores: u32 = 0; - - let max_core_mask = ((1u64 << num_cores) - 1) as u32; - for _ in 0..num_clusters { - let mask = core_mask_regs[0] & max_core_mask; - core_masks.push(mask, GFP_KERNEL)?; - for i in 0..core_mask_regs.len() { - core_mask_regs[i] >>= num_cores; - if i < (core_mask_regs.len() - 1) { - core_mask_regs[i] |= core_mask_regs[i + 1] << (32 - num_cores); - } - } - total_active_cores += mask.count_ones(); - } - - if core_mask_regs.iter().any(|a| *a != 0) { - dev_err!( - self.dev.as_ref(), - "Leftover core mask: {:#x?}\n", - core_mask_regs - ); - return Err(EIO); - } - - let (gpu_rev, gpu_rev_id) = match (id_version >> 8) & 0xff { - 0x00 => (hw::GpuRevision::A0, hw::GpuRevisionID::A0), - 0x01 => (hw::GpuRevision::A1, hw::GpuRevisionID::A1), - 0x10 => (hw::GpuRevision::B0, hw::GpuRevisionID::B0), - 0x11 => (hw::GpuRevision::B1, hw::GpuRevisionID::B1), - 0x20 => (hw::GpuRevision::C0, hw::GpuRevisionID::C0), - 0x21 => (hw::GpuRevision::C1, hw::GpuRevisionID::C1), - a => { - dev_err!(self.dev.as_ref(), "Unknown GPU revision {}\n", a); - return Err(ENODEV); - } - }; + let (core_masks, total_active_cores) = + split_core_masks(&self.dev, &mut core_mask_regs, num_clusters, num_cores)?; + let (gpu_rev, gpu_rev_id) = gpu_revision(&self.dev, id_version)?; Ok(hw::GpuIdConfig { gpu_gen: match (id_version >> 24) & 0xff { @@ -474,3 +526,216 @@ impl Resources<'_> { }) } } + +/// Device resources of a G17 GPU. +/// +/// The G17 GPU object outlives the probe call, so its register mapping is device-managed and +/// revoked when the device is unbound. +#[derive(Clone)] +pub(crate) struct G17Resources { + dev: ARef, + sgx: Arc>>, +} + +impl G17Resources { + /// Map the required resources given our platform device. + pub(crate) fn new(pdev: &platform::Device>) -> Result { + let sgx_req = pdev.io_request_by_name(c"sgx").ok_or(EINVAL)?; + let sgx = Arc::new(sgx_req.iomap_sized::()?.into_devres()?, GFP_KERNEL)?; + + Ok(G17Resources { + dev: pdev.into(), + sgx, + }) + } + + /// Installs both render queues while the caller holds the device mutex. + /// No other port publisher may interleave between this power witness and + /// the final validity strobe. False means no write was attempted. + pub(crate) fn install_render_pair( + &self, + registration: &crate::g17::fw::kick::RenderRegistration, + ) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + if sgx.relaxed().read32(G17_HOST_IRQ_SUMMARY) & G17_HOST_POWERED == 0 { + return Ok(false); + } + for &(offset, value) in registration.writes() { + sgx.try_write64(value, offset)?; + } + Ok(true) + } + + fn sgx_read32(&self) -> u32 { + if let Some(sgx) = self.sgx.try_access() { + (&*sgx).as_view().relaxed().read32(OFF) + } else { + 0 + } + } + + /// Get the identification of a G17 GPU from its registers. + pub(crate) fn get_g17_id(&self) -> Result { + let id_version = self.sgx_read32::(); + let id_unk08 = self.sgx_read32::(); + let id_counts_1 = self.sgx_read32::(); + let id_counts_2 = self.sgx_read32::(); + let id_unk18 = self.sgx_read32::(); + let id_clusters = self.sgx_read32::(); + + dev_info!( + self.dev.as_ref(), + "GPU ID registers: {:#x} {:#x} {:#x} {:#x} {:#x} {:#x}\n", + id_version, + id_unk08, + id_counts_1, + id_counts_2, + id_unk18, + id_clusters + ); + + // Later reads can fault while the firmware has gated the cores. Submission uses only + // these saved values and never re-reads the registers. + let perf_control = self.sgx_read32::() & 1; + let perf_map = self.sgx_read32::(); + let mut core_mask_regs = [ + self.sgx_read32::(), + self.sgx_read32::<{ CORE_MASKS_G14X + 4 }>(), + self.sgx_read32::<{ CORE_MASKS_G14X + 8 }>(), + ]; + dev_info!(self.dev.as_ref(), "Core masks: {:#x?}\n", core_mask_regs); + + let num_dies = (id_counts_1 >> 16) & 0xf; + let num_clusters = ((id_counts_1 >> 8) & 0xff) * num_dies; + let num_cores = id_counts_1 & 0xff; + if num_cores == 0 || num_cores > 32 || num_cores * num_clusters > 3 * 32 { + dev_err!( + self.dev.as_ref(), + "Invalid core counts ({} x {})\n", + num_clusters, + num_cores + ); + return Err(ENODEV); + } + let (core_masks, _) = + split_core_masks(&self.dev, &mut core_mask_regs, num_clusters, num_cores)?; + let (gpu_rev, _) = gpu_revision(&self.dev, id_version)?; + + Ok(G17Id { + family: ((id_version >> 24) & 0xff) as u8, + variant: ((id_version >> 16) & 0xff) as u8, + gpu_rev, + num_dies, + num_clusters, + num_cores, + core_masks, + perf_control, + perf_map: [perf_map & 0xf, (perf_map >> 16) & 0xf], + }) + } + + /// Enable the G17 GPU-to-fabric bridge and check that it took effect. + /// + /// The GPU power domain must be on. The coprocessors cannot reach memory until this is done. + pub(crate) fn enable_g17_bridge(&self) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + for offset in G17_BRIDGE_CONTROL { + let val = sgx.try_read32(offset)?; + sgx.try_write32(val | G17_BRIDGE_ENABLE, offset)?; + if sgx.try_read32(offset)? & G17_BRIDGE_ENABLE == 0 { + dev_err!( + self.dev.as_ref(), + "GPU bridge {:#x} did not enable\n", + offset + ); + return Err(EIO); + } + } + Ok(()) + } + + /// Disable the G17 GPU-to-fabric bridge. Both coprocessors must be stopped. + pub(crate) fn disable_g17_bridge(&self) { + if let Some(sgx) = self.sgx.try_access() { + let sgx = (&*sgx).as_view(); + for offset in G17_BRIDGE_CONTROL { + if let Ok(val) = sgx.try_read32(offset) { + // Cannot fail: the offset was read just before. + let _ = sgx.try_write32(val & !G17_BRIDGE_ENABLE, offset); + } + } + } + } + + /// Power witness for the queue-configuration port, accessible while the cores are gated. + pub(crate) fn g17_configuration_powered(&self) -> bool { + self.sgx_read32::() & 0xf != 0 + } + + /// Enable dynamic power gating of idle G17 GPU cores. The firmware expects this before it is + /// initialized. + pub(crate) fn enable_g17_gating(&self) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + let val = sgx.try_read32(G17_GATING_CONTROL)?; + sgx.try_write32(val & !G17_GATING_DISABLE, G17_GATING_CONTROL) + } + +} + +/// Split the packed per-cluster core masks into one mask per cluster. +/// +/// Returns the masks and the total number of enabled cores. +fn split_core_masks( + dev: &platform::Device, + core_mask_regs: &mut [u32], + num_clusters: u32, + num_cores: u32, +) -> Result<(KVec, u32)> { + let mut core_masks = KVec::new(); + let mut total_active_cores: u32 = 0; + + let max_core_mask = ((1u64 << num_cores) - 1) as u32; + for _ in 0..num_clusters { + let mask = core_mask_regs[0] & max_core_mask; + core_masks.push(mask, GFP_KERNEL)?; + for i in 0..core_mask_regs.len() { + core_mask_regs[i] = core_mask_regs[i].checked_shr(num_cores).unwrap_or(0); + if i < (core_mask_regs.len() - 1) { + core_mask_regs[i] |= core_mask_regs[i + 1] + .checked_shl(32 - num_cores) + .unwrap_or(0); + } + } + total_active_cores += mask.count_ones(); + } + + if core_mask_regs.iter().any(|a| *a != 0) { + dev_err!( + dev.as_ref(), + "Leftover core mask: {:#x?}\n", + core_mask_regs + ); + return Err(EIO); + } + + Ok((core_masks, total_active_cores)) +} + +/// Decode the silicon revision from the ID version register. +fn gpu_revision(dev: &platform::Device, id_version: u32) -> Result<(hw::GpuRevision, hw::GpuRevisionID)> { + Ok(match (id_version >> 8) & 0xff { + 0x00 => (hw::GpuRevision::A0, hw::GpuRevisionID::A0), + 0x01 => (hw::GpuRevision::A1, hw::GpuRevisionID::A1), + 0x10 => (hw::GpuRevision::B0, hw::GpuRevisionID::B0), + 0x11 => (hw::GpuRevision::B1, hw::GpuRevisionID::B1), + 0x20 => (hw::GpuRevision::C0, hw::GpuRevisionID::C0), + 0x21 => (hw::GpuRevision::C1, hw::GpuRevisionID::C1), + a => { + dev_err!(dev.as_ref(), "Unknown GPU revision {}\n", a); + return Err(ENODEV); + } + }) +} From bf4ef659bcf71ac2da35e785b105cb4bb9bc61df Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 01:47:16 -0400 Subject: [PATCH 0623/2680] sched/core: add scoped utilization floors for kernel waiters Allow a kernel waiter to request a temporary minimum utilization while preserving the task's requested clamp and aggregate runqueue accounting. Balance nested scopes in LIFO order and restore the previous floor on exit. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 29a2e2acec42f8361d72bfd9506e3a0527afaf40) --- include/linux/sched.h | 10 +++++++ kernel/sched/core.c | 68 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/include/linux/sched.h b/include/linux/sched.h index 5b36a3364ca09..f80d6780a5e7e 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -913,6 +913,8 @@ struct task_struct { * Must be updated with task_rq_lock() held. */ struct uclamp_se uclamp[UCLAMP_CNT]; + /* Current-task kernel scope; separate from userspace clamp requests. */ + unsigned int uclamp_scope_min; #endif struct sched_statistics stats; @@ -1960,6 +1962,14 @@ extern void sched_set_fifo_low(struct task_struct *p); extern void sched_set_fifo_secondary(struct task_struct *p); extern void sched_set_normal(struct task_struct *p, int nice); extern int sched_setattr(struct task_struct *, const struct sched_attr *); +#ifdef CONFIG_UCLAMP_TASK +unsigned int sched_util_min_scope_enter(unsigned int value); +void sched_util_min_scope_exit(unsigned int previous); +#else +static inline unsigned int sched_util_min_scope_enter(unsigned int value) { return 0; } +static inline void sched_util_min_scope_exit(unsigned int previous) { } +#endif + extern int sched_setattr_nocheck(struct task_struct *, const struct sched_attr *); extern struct task_struct *idle_task(int cpu); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index c104562638e33..0c570f46f1646 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1696,7 +1696,20 @@ uclamp_tg_restrict(struct task_struct *p, enum uclamp_id clamp_id) struct uclamp_se uc_req = p->uclamp_req[clamp_id]; #ifdef CONFIG_UCLAMP_TASK_GROUP unsigned int tg_min, tg_max, value; +#endif + + /* + * Kernel scopes never replace user attributes or exceed their cap. + * The existing group and system restrictions are applied below. + */ + if (clamp_id == UCLAMP_MIN && p->uclamp_scope_min) { + unsigned int hint = min(p->uclamp_scope_min, + (unsigned int)p->uclamp_req[UCLAMP_MAX].value); + if (hint > uc_req.value) + uclamp_se_set(&uc_req, hint, false); + } +#ifdef CONFIG_UCLAMP_TASK_GROUP /* * Tasks in autogroups or root task group will be * restricted by system defaults. @@ -1932,6 +1945,58 @@ static inline void uclamp_rq_reinc_id(struct rq *rq, struct task_struct *p, rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE; } +/* + * Only the current task may enter/leave its own balanced, nestable scope. + * pi/rq locking serializes the effective clamp with wakeup, migration and + * concurrent userspace attribute changes; user requests remain untouched. + */ +static unsigned int uclamp_scope_set(unsigned int value, bool raise) +{ + struct task_struct *p = current; + struct rq_flags rf; + struct rq *rq; + unsigned int previous; + + might_sleep(); + value = min(value, (unsigned int)SCHED_CAPACITY_SCALE); + if (value) + sched_uclamp_enable(); + rq = task_rq_lock(p, &rf); + previous = p->uclamp_scope_min; + if (raise) + value = max(value, previous); + if (value != previous) { + p->uclamp_scope_min = value; + uclamp_rq_reinc_id(rq, p, UCLAMP_MIN); + } + task_rq_unlock(rq, p, &rf); + return previous; +} + +/** + * sched_util_min_scope_enter - request a temporary current-task capacity floor + * @value: capacity hint, 0..1024, subject to user, cgroup and system caps + * + * May sleep. The caller must balance this scope in LIFO order on the same + * task with sched_util_min_scope_exit(), including error and signal paths. + * Return: opaque previous scope value for the matching exit. + */ +unsigned int sched_util_min_scope_enter(unsigned int value) +{ + return uclamp_scope_set(value, true); +} +EXPORT_SYMBOL_GPL(sched_util_min_scope_enter); + +/** + * sched_util_min_scope_exit - restore a balanced current-task capacity scope + * @previous: value returned by the matching sched_util_min_scope_enter() + */ +void sched_util_min_scope_exit(unsigned int previous) +{ + uclamp_scope_set(previous, false); +} +EXPORT_SYMBOL_GPL(sched_util_min_scope_exit); + static inline void uclamp_update_active(struct task_struct *p) { @@ -2088,6 +2153,9 @@ static void uclamp_fork(struct task_struct *p) { enum uclamp_id clamp_id; + /* A child never inherits its parent's temporary kernel scope. */ + p->uclamp_scope_min = 0; + /* * We don't need to hold task_rq_lock() when updating p->uclamp_* here * as the task is still at its early fork stages. From fea28d36204aa0b3f6d32235e80057c80819c668 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 01:47:16 -0400 Subject: [PATCH 0624/2680] drm/syncobj: apply expiring capacity hints during blocking waits Associate a bounded utilization hint with a DRM file after successful submission. Apply it only around a blocking syncobj wait while it remains valid, and release the scheduler scope on every wait exit. Expose the hint setter to Rust DRM drivers. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 7c99d8debcc54ebf3f76ede14ced9b845db18215) --- drivers/gpu/drm/drm_syncobj.c | 55 +++++++++++++++++++++++++++++++++-- include/drm/drm_file.h | 4 +++ include/drm/drm_syncobj.h | 3 ++ rust/kernel/drm/file.rs | 8 +++++ 4 files changed, 67 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/drm_syncobj.c b/drivers/gpu/drm/drm_syncobj.c index c9dbf64c0c9f2..2ac67e51a5b15 100644 --- a/drivers/gpu/drm/drm_syncobj.c +++ b/drivers/gpu/drm/drm_syncobj.c @@ -1032,14 +1032,52 @@ static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj, list_del_init(&wait->node); } +/** + * drm_syncobj_set_wait_hint - tag recent GPU-feeder work on an open DRM file + * @file: submitting client + * @util_min: scheduler capacity hint, bounded to 1024 + * @duration_ms: validity for subsequent waits, bounded to one second + * + * This changes no scheduling attribute or affinity. A blocking syncobj wait + * may use the hint in a balanced kernel scope while this recent-work tag is + * valid. Sleeping tasks consume no CPU; the scope is removed before return. + */ +void drm_syncobj_set_wait_hint(struct drm_file *file, unsigned int util_min, + unsigned int duration_ms) +{ + unsigned long flags; + + spin_lock_irqsave(&file->table_lock, flags); + file->syncobj_wait_util_min = min(util_min, 1024u); + file->syncobj_wait_hint_until = jiffies + + msecs_to_jiffies(min(duration_ms, 1000u)); + spin_unlock_irqrestore(&file->table_lock, flags); +} +EXPORT_SYMBOL_GPL(drm_syncobj_set_wait_hint); + +static unsigned int drm_syncobj_wait_hint(struct drm_file *file) +{ + unsigned long flags; + unsigned int value = 0; + + spin_lock_irqsave(&file->table_lock, flags); + if (time_before(jiffies, file->syncobj_wait_hint_until)) + value = file->syncobj_wait_util_min; + spin_unlock_irqrestore(&file->table_lock, flags); + return value; +} + static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs, void __user *user_points, uint32_t count, uint32_t flags, signed long timeout, uint32_t *idx, - ktime_t *deadline) + ktime_t *deadline, + struct drm_file *file_private) { + unsigned int scope_previous = 0, util_min; + bool scope_active = false; struct syncobj_wait_entry *entries; struct dma_fence *fence; uint64_t *points; @@ -1132,6 +1170,15 @@ static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs, } } + /* Recheck expiry after allocations, user copies and deadline callbacks. */ + util_min = drm_syncobj_wait_hint(file_private); + /* Polls, availability waits and already-complete fences need no boost. */ + if (util_min && timeout > 0 && + !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) { + scope_previous = sched_util_min_scope_enter(util_min); + scope_active = true; + } + do { set_current_state(TASK_INTERRUPTIBLE); @@ -1176,6 +1223,8 @@ static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs, done_waiting: __set_current_state(TASK_RUNNING); + if (scope_active) + sched_util_min_scope_exit(scope_previous); cleanup_entries: for (i = 0; i < count; ++i) { @@ -1243,7 +1292,7 @@ static int drm_syncobj_array_wait(struct drm_device *dev, wait->count_handles, wait->flags, timeout, &first, - deadline); + deadline, file_private); if (timeout < 0) return timeout; wait->first_signaled = first; @@ -1254,7 +1303,7 @@ static int drm_syncobj_array_wait(struct drm_device *dev, timeline_wait->count_handles, timeline_wait->flags, timeout, &first, - deadline); + deadline, file_private); if (timeout < 0) return timeout; timeline_wait->first_signaled = first; diff --git a/include/drm/drm_file.h b/include/drm/drm_file.h index 6ee70ad65e1fd..b766e194561d9 100644 --- a/include/drm/drm_file.h +++ b/include/drm/drm_file.h @@ -320,6 +320,10 @@ struct drm_file { /** @syncobj_xa: Mapping of sync object handles to object pointers. */ struct xarray syncobj_xa; + /* Driver-provided recent-feeder hint, protected by table_lock. */ + unsigned int syncobj_wait_util_min; + unsigned long syncobj_wait_hint_until; + /** @filp: Pointer to the core file structure. */ struct file *filp; diff --git a/include/drm/drm_syncobj.h b/include/drm/drm_syncobj.h index b40052132e523..ef53cc2557b1b 100644 --- a/include/drm/drm_syncobj.h +++ b/include/drm/drm_syncobj.h @@ -133,4 +133,7 @@ int drm_syncobj_get_handle(struct drm_file *file_private, struct drm_syncobj *syncobj, u32 *handle); int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd); +void drm_syncobj_set_wait_hint(struct drm_file *file, unsigned int util_min, + unsigned int duration_ms); + #endif diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index f978061afc5f8..8a56f00203f56 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -48,6 +48,14 @@ impl File { self.0.get() } + /// Tag recent feeder work for temporary capacity hints during syncobj waits. + /// User scheduling attributes and CPU affinity remain unchanged. + pub fn set_syncobj_wait_hint(&self, util_min: u32, duration_ms: u32) { + // SAFETY: this File keeps the opened drm_file alive; the C helper + // bounds both arguments and serializes the per-file hint internally. + unsafe { bindings::drm_syncobj_set_wait_hint(self.as_raw(), util_min, duration_ms) }; + } + fn driver_priv(&self) -> *mut T { // SAFETY: By the type invariants of `Self`, `self.as_raw()` is always valid. unsafe { (*self.as_raw()).driver_priv }.cast() From dcd8b7668b0aa659cfbd6f5c05e64f84ec698c06 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 01:47:17 -0400 Subject: [PATCH 0625/2680] drm/asahi: request temporary CPU capacity for sustained GPU feeds Detect sustained render completion intervals and request temporary CPU capacity while submissions remain latency-sensitive. Retain detector ownership through completion, service the actuator on one work item, and remove its frequency requests after joining producers during shutdown. Provide an expiring syncobj wait hint only while the detector and applied capacity request agree. Keep this policy separate from firmware execution, queue depth and resource retirement. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit bc4c50abf8ede16fec7f2a79546a310134b208df) --- drivers/gpu/drm/asahi/file.rs | 6 + drivers/gpu/drm/asahi/g17/completion.rs | 6 + drivers/gpu/drm/asahi/g17/feed.rs | 330 +++++++++++++++++++ drivers/gpu/drm/asahi/g17/job.rs | 5 + drivers/gpu/drm/asahi/g17/mod.rs | 15 + drivers/gpu/drm/asahi/g17/queue/mod.rs | 2 + drivers/gpu/drm/asahi/g17/runtime.rs | 1 + drivers/gpu/drm/asahi/g17/runtime/backend.rs | 4 + drivers/gpu/drm/asahi/g17/runtime/events.rs | 1 + drivers/gpu/drm/asahi/g17/runtime/feed.rs | 33 ++ drivers/gpu/drm/asahi/gpu.rs | 5 +- 11 files changed, 407 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/asahi/g17/feed.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/feed.rs diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 27ee72c0a060a..6da9e6825e661 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -1306,6 +1306,12 @@ impl File { objects, ); + if ret.is_ok() { + if let Some((util_min, duration_ms)) = gpu.syncobj_wait_hint() { + file.set_syncobj_wait_hint(util_min, duration_ms); + } + } + match ret { Err(ERESTARTSYS) => Err(ERESTARTSYS), Err(e) => { diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index d66c21480711b..21ead1107fe27 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -89,6 +89,7 @@ pub(crate) struct Completion { member: Member, destinations: [Option; 4], compute: Option, + feed: Option>, #[pin] vm_job: Mutex>, #[pin] @@ -111,6 +112,7 @@ impl Completion { objects: Pin<&xarray::XArray>>, timestamps: &[uapi::drm_asahi_timestamps], compute: bool, + feed: Option>, ) -> Result> { if timestamps.len() != if compute { 1 } else { 2 } { return Err(EINVAL); @@ -133,6 +135,7 @@ impl Completion { member, destinations, compute, + feed, vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), render <- new_mutex!(None, "G17 command render lease"), signalled: AtomicBool::new(false), @@ -309,6 +312,9 @@ impl Completion { if self.signalled.swap(true, Ordering::AcqRel) { return; } + if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { + feed.note_render_pass(ticks[2], ticks[3]); + } let result = result.and_then(|ticks| { for (destination, ticks) in self.destinations.iter().zip(ticks) { if let Some(destination) = destination { diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs new file mode 100644 index 0000000000000..ca3ecc85ee25a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Detects sustained host refill gaps and temporarily requests CPU capacity. +//! Detector updates are lock-free. The device's single feed worker owns the +//! frequency requests; its lock never nests inside a driver publication lock. +//! Shutdown first closes the detector, joins its producers and the worker, +//! then removes requests. All request storage and policy references are owned +//! by this device, independently of other GPU instances. + +use crate::hw::t8140::qos::CLOCK_HZ; +use core::{ + pin::Pin, + ptr, + sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, +}; +use kernel::{ + bindings, new_mutex, + prelude::*, + sync::{Arc, Mutex}, + time::{delay::fsleep, ClockSource, Delta, Monotonic}, + types::Opaque, +}; + +const MAX_REFILL_TICKS: u64 = CLOCK_HZ / 500; +const STREAK_ON: u32 = 8; +const STREAK_MAX: u32 = 64; +const LINGER_NS: u64 = 8_000_000; +const POLL_MS: i64 = 2; +const FLOOR_PERCENT: u64 = 50; +const WAIT_UTIL_MIN: u32 = 512; +const WAIT_HINT_MS: u32 = 8; +const MAX_POLICIES: usize = 8; + +struct Detector { + last_end: AtomicU64, + idle: AtomicU64, + busy: AtomicU64, + streak: AtomicU32, + until: AtomicU64, + active: AtomicBool, + pending: AtomicBool, + stopping: AtomicBool, +} + +impl Detector { + const fn new() -> Self { + Self { + last_end: AtomicU64::new(0), + idle: AtomicU64::new(0), + busy: AtomicU64::new(0), + streak: AtomicU32::new(0), + until: AtomicU64::new(0), + active: AtomicBool::new(false), + pending: AtomicBool::new(false), + stopping: AtomicBool::new(false), + } + } + + fn decay_add(sum: &AtomicU64, add: u64) -> u64 { + let mut next = 0; + let _ = sum.fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| { + next = value - value / 8 + add; + Some(next) + }); + next + } + + fn note(&self, start: u64, end: u64, now: u64) { + if end <= start { + return; + } + let previous = self.last_end.fetch_max(end, Ordering::AcqRel); + let (idle, busy, paced) = if previous == 0 { + (0, end - start, true) + } else if start >= previous { + let gap = start - previous; + if gap < MAX_REFILL_TICKS { + (gap, end - start, false) + } else { + (0, end - start, true) + } + } else if end > previous { + (0, end - previous, false) + } else { + (0, 0, false) + }; + let idle_sum = Self::decay_add(&self.idle, idle); + let busy_sum = Self::decay_add(&self.busy, busy); + let streak = if paced { + self.streak.store(0, Ordering::Release); + 0 + } else { + let previous = self + .streak + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| { + Some((value + 1).min(STREAK_MAX)) + }); + // The closure always returns Some; fetch_update retries contention. + match previous { + Ok(value) => value + 1, + Err(_) => 0, + } + }; + if streak >= STREAK_ON && idle_sum.saturating_mul(4) >= busy_sum { + self.until.store(now + LINGER_NS, Ordering::Release); + if !self.active.swap(true, Ordering::AcqRel) { + self.pending.store(true, Ordering::Release); + } + } + } + + fn wanted(&self, now: u64) -> bool { + !self.stopping.load(Ordering::Acquire) && now < self.until.load(Ordering::Acquire) + } +} + +#[pin_data] +struct Requests { + #[pin] + requests: Opaque<[bindings::freq_qos_request; MAX_POLICIES]>, + policies: [*mut bindings::cpufreq_policy; MAX_POLICIES], + maxima: [u32; MAX_POLICIES], + applied: [u32; MAX_POLICIES], + count: usize, +} + +// SAFETY: Each nonnull policy retains cpufreq_cpu_get's reference. Access to +// the pinned request array and metadata is exclusive under Feed::requests. +unsafe impl Send for Requests {} + +impl Requests { + fn new() -> Self { + Self { + requests: Opaque::zeroed(), + policies: [ptr::null_mut(); MAX_POLICIES], + maxima: [0; MAX_POLICIES], + applied: [0; MAX_POLICIES], + count: 0, + } + } + + fn register(self: Pin<&mut Self>) { + let this = self.project(); + if *this.count != 0 { + return; + } + // SAFETY: nr_cpu_ids is initialized before driver probe. + let cpus = unsafe { bindings::nr_cpu_ids }; + for cpu in 0..cpus { + // SAFETY: Accepts any CPU number; a nonnull result owns one reference. + let policy = unsafe { bindings::cpufreq_cpu_get(cpu) }; + if policy.is_null() { + continue; + } + let count = *this.count; + if count < MAX_POLICIES && !this.policies[..count].contains(&policy) { + // SAFETY: The policy is referenced. This zeroed, unregistered + // slot is pinned and remains alive until remove_request. + let result = unsafe { + bindings::freq_qos_add_request( + ptr::addr_of_mut!((*policy).constraints), + this.requests + .get() + .cast::() + .add(count), + bindings::freq_qos_req_type_FREQ_QOS_MIN, + bindings::FREQ_QOS_MIN_DEFAULT_VALUE as i32, + ) + }; + if result >= 0 { + this.policies[count] = policy; + // SAFETY: cpuinfo is immutable while this policy is referenced. + this.maxima[count] = unsafe { (*policy).cpuinfo.max_freq }; + this.applied[count] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; + *this.count += 1; + continue; + } + } + // SAFETY: Balances the reference acquired above for a duplicate, + // unavailable slot or failed registration. Successful slots retain it. + unsafe { bindings::cpufreq_cpu_put(policy) }; + } + } + + fn apply(mut self: Pin<&mut Self>, enabled: bool) -> bool { + if enabled { + self.as_mut().register(); + } + let this = self.project(); + for index in 0..*this.count { + let floor = if enabled { + (u64::from(this.maxima[index]) * FLOOR_PERCENT / 100) as u32 + } else { + 0 + }; + let value = if floor == 0 { + bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32 + } else { + floor + }; + if this.applied[index] == value { + continue; + } + this.applied[index] = value; + // SAFETY: This slot is registered, pinned and exclusively locked; + // its referenced policy keeps the constraints alive. + unsafe { + bindings::freq_qos_update_request( + this.requests + .get() + .cast::() + .add(index), + value as i32, + ) + }; + } + *this.count != 0 && enabled + } + + fn clear(self: Pin<&mut Self>) { + let this = self.project(); + for index in 0..*this.count { + // SAFETY: Registered pinned slot, exclusively locked. Remove the + // constraint link before releasing its policy reference. + unsafe { + bindings::freq_qos_remove_request( + this.requests + .get() + .cast::() + .add(index), + ); + bindings::cpufreq_cpu_put(this.policies[index]); + } + this.policies[index] = ptr::null_mut(); + } + *this.count = 0; + } +} + +#[pin_data(PinnedDrop)] +pub(crate) struct Feed { + detector: Detector, + #[pin] + requests: Mutex, + applied: AtomicBool, +} + +impl Feed { + pub(crate) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + detector: Detector::new(), + requests <- new_mutex!(Requests::new(), "G17 CPU feed"), + applied: AtomicBool::new(false), + }), + GFP_KERNEL, + ) + } + + fn now() -> u64 { + Monotonic::ktime_get() as u64 + } + + pub(crate) fn note_render_pass(&self, start: u64, end: u64) { + self.detector.note(start, end, Self::now()); + } + + /// Called after a firmware event pass, before queuing the single feed work item. + pub(crate) fn take_pending(&self) -> bool { + self.detector.pending.swap(false, Ordering::AcqRel) + } + + fn apply(&self, enabled: bool) { + let mut requests = self.requests.lock(); + let enabled = enabled && !self.detector.stopping.load(Ordering::Acquire); + let applied = requests.as_mut().apply(enabled); + self.applied.store(applied, Ordering::Relaxed); + } + + /// Worker context; the linger deadline has no firmware event of its own. + pub(crate) fn run(&self) { + loop { + if self.detector.wanted(Self::now()) { + self.apply(true); + fsleep(Delta::from_millis(POLL_MS)); + continue; + } + self.apply(false); + self.detector.active.store(false, Ordering::SeqCst); + // A detector can extend the deadline after observing the old active + // bit. Reclaim that work unless it already queued another owner. + if self.detector.wanted(Self::now()) + && !self.detector.active.swap(true, Ordering::AcqRel) + { + continue; + } + return; + } + } + + pub(crate) fn wait_hint(&self) -> Option<(u32, u32)> { + if cfg!(CONFIG_UCLAMP_TASK) + && self.detector.wanted(Self::now()) + && self.applied.load(Ordering::Relaxed) + { + Some((WAIT_UTIL_MIN, WAIT_HINT_MS)) + } else { + None + } + } + + /// Call before disabling/joining the event producer and actuator work items. + pub(crate) fn begin_shutdown(&self) { + self.detector.stopping.store(true, Ordering::Release); + } + + /// Call after work items are joined. Removing requests withdraws their floor. + pub(crate) fn shutdown(&self) { + self.begin_shutdown(); + self.requests.lock().as_mut().clear(); + self.applied.store(false, Ordering::Relaxed); + } +} + +#[pinned_drop] +impl PinnedDrop for Feed { + fn drop(self: Pin<&mut Self>) { + self.shutdown(); + } +} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 60cc344fd5116..f9d04e9e85f7e 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -222,6 +222,11 @@ fn check_status(fence: &Fence) -> Result { /// The backend retains visible packets through exact retirement or processor stop. /// It performs all completion signalling after releasing the device mutex. pub(crate) trait Backend: Send + Sync { + /// Device detector retained by render completions after queue teardown. + fn feed(&self) -> Option> { + None + } + /// Lazily create or rebind this logical queue's physical compute graph. fn ensure_compute(&self) -> Result; /// Settle outstanding work of a terminal VM before an ioctl error is published. diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 66ce58cd0cc7c..9a3b6b47f29aa 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -25,6 +25,7 @@ mod completion; pub(crate) mod context; mod dependency; mod event; +mod feed; mod fence; mod freelist; pub(crate) mod fw; @@ -583,6 +584,7 @@ const POLL_WORK: u64 = 1; const IDLE_WORK: u64 = 2; const GROW_WORK: u64 = 3; const RECLAIM_WORK: u64 = 4; +const FEED_WORK: u64 = 5; /// Device state shared with the coprocessor callbacks and the event worker. #[pin_data] @@ -591,6 +593,9 @@ struct Shared { clusters: u32, descriptor_flags: [u32; 2], crashed: AtomicBool, + feed: Arc, + #[pin] + feed_work: Work, preparations: Arc, #[pin] work: Work, @@ -819,12 +824,15 @@ impl Gpu { let max_frequency_khz = Self::max_frequency_khz(dev)?; let preparations = preparation::Gate::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; + let feed = feed::Feed::new()?; let shared = Arc::pin_init( pin_init!(Shared { dev: dev.into(), clusters: id.num_clusters, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), + feed, + feed_work <- new_work!("g17::Shared::feed_work"), preparations, work <- new_work!("g17::Shared::work"), poll <- new_delayed_work!("g17::Shared::poll"), @@ -980,6 +988,7 @@ impl Gpu { /// Stops the firmware and releases everything it used. pub(crate) fn shutdown(&self) { + self.shared.feed.begin_shutdown(); self.shared.preparations.remove(); self.shared.stop_queues(); self.shared.disable_timer(&self.shared.poll); @@ -987,11 +996,17 @@ impl Gpu { self.shared.disable_work(&self.shared.grow); self.shared.disable_work(&self.shared.reclaim_work); self.shared.disable_work(&self.shared.work); + self.shared.disable_work(&self.shared.feed_work); + self.shared.feed.shutdown(); self.shared.drain_reclaims(); } } impl gpu::Gpu for Gpu { + fn syncobj_wait_hint(&self) -> Option<(u32, u32)> { + self.shared.feed.wait_hint() + } + fn as_any(&self) -> &dyn Any { self } diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index b821ad5849bef..51284dc378bf4 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -162,6 +162,7 @@ impl Queue { objects, timestamps, false, + self.backend.feed(), )?, Validated::Compute { timestamps, .. } => Completion::new( &self.dev, @@ -172,6 +173,7 @@ impl Queue { objects, core::slice::from_ref(timestamps), true, + None, )?, }; if engine == Engine::Compute { diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index e908ca3e08e23..d7a71a2c4c899 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -13,6 +13,7 @@ mod compute_control; mod compute_replay; mod compute_exit; mod events; +mod feed; mod render; pub(super) mod teardown; diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index f1ae1ffe80783..085655beb8194 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -66,6 +66,10 @@ impl Backend { } impl job::Backend for Backend { + fn feed(&self) -> Option> { + Some(self.shared.feed.clone()) + } + fn ensure_compute(&self) -> Result { Backend::ensure_compute(self) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index e9fc977d623b8..43a27b272ac88 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -311,6 +311,7 @@ impl crate::g17::Shared { }; let result = self.runtime_event_turn(&mut deferred); deferred.finish(); + self.queue_feed(); if let Some(firmware) = (*self.state.lock()).as_deref_mut() { firmware.queues.worker_deferred = Some(deferred); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/feed.rs b/drivers/gpu/drm/asahi/g17/runtime/feed.rs new file mode 100644 index 0000000000000..f3faffb89df76 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/feed.rs @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Schedules CPU capacity updates after completed firmware event passes. The +//! worker owns frequency requests independently of the device publication lock. + +use crate::g17::{Shared, FEED_WORK}; +use kernel::{ + impl_has_work, + sync::Arc, + workqueue::{self, WorkItem}, +}; + +impl_has_work! { + impl HasWork for Shared { self.feed_work } +} + +impl WorkItem for Shared { + type Pointer = Arc; + + fn run(this: Arc) { + this.feed.run(); + } +} + +impl Shared { + /// The completion callbacks have finished outside the device mutex before + /// the event worker hands the detector's request to the actuator. + pub(in crate::g17) fn queue_feed(self: &Arc) { + if self.feed.take_pending() { + let _ = workqueue::system_unbound().enqueue::, FEED_WORK>(self.clone()); + } + } +} diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 90db79eaeac64..eba89548c1d12 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -287,7 +287,10 @@ pub(crate) trait Gpu: Send + Sync { fn queue_limits(&self) -> Option { None } - + /// Temporary capacity hint and validity in milliseconds for subsequent syncobj waits. + fn syncobj_wait_hint(&self) -> Option<(u32, u32)> { + None + } /// Frequency of the CPU timer that GPU timestamps are derived from. fn base_clock_hz(&self) -> u32; /// Return the GPU address space geometry. From ca1896503525858c8d9e7902b38d5afa288c5ae1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:34:52 -0400 Subject: [PATCH 0626/2680] drm/asahi: observe compute completions without a queue mask An event batch without a usable queue identity must still observe completed compute commands. Use the full scan for zero masks while retaining QID filtering for identified batches, so coalesced notifications cannot strand an otherwise retired command. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e1e0c5c90faf0307f95374c83efe3009d58f0bdb) --- drivers/gpu/drm/asahi/g17/runtime/events.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 43a27b272ac88..6776707fd944c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -92,7 +92,10 @@ impl event::Queues for Service<'_> { self.render_masks[1] |= masks[1]; } fn complete_compute(&mut self, masks: [u64; 2]) -> Result { - self.firmware.scan_compute(Some(masks), self.deferred) + // A batch without a queue identity still observes completions whose + // notification was coalesced; identified batches retain QID filtering. + let selected = (masks != [0; 2]).then_some(masks); + self.firmware.scan_compute(selected, self.deferred) } } From 3110d999c95dd34174cb0c7b2590c614d76d712f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:34:52 -0400 Subject: [PATCH 0627/2680] drm/asahi: avoid repeated submission bookkeeping Consume prepared VM guards in order without shifting the remaining vector for every command. Iterator destruction still releases unused pins on partial enqueue failure. Capture shared attachment metadata under one binding-lock hold per footprint, preserving range insertion order and releasing the lock before allocation. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 0c69741cdc26f2dbb26754e87f1b53db1f4bed38) --- drivers/gpu/drm/asahi/g17/queue/mod.rs | 5 +++-- drivers/gpu/drm/asahi/mmu/shared.rs | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 51284dc378bf4..e6b95193c97db 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -285,11 +285,12 @@ impl crate::queue::Queue for Queue { if self.context.status().get() != 0 { return Err(EIO); } - let (pending, mut guards) = self.prepare_submission(bytes)?; + let (pending, guards) = self.prepare_submission(bytes)?; + let mut guards = guards.into_iter(); let aggregate = Submission::new(&self.fences, self.context.clone())?; outputs.publish(aggregate.clone()); for (command, order) in pending { - let guard = guards.remove(0).map_err(|_| EIO)?; + let guard = guards.next().ok_or(EIO)?; self.enqueue( id, command, diff --git a/drivers/gpu/drm/asahi/mmu/shared.rs b/drivers/gpu/drm/asahi/mmu/shared.rs index cf29e53fc9cc9..5b87c55a06b72 100644 --- a/drivers/gpu/drm/asahi/mmu/shared.rs +++ b/drivers/gpu/drm/asahi/mmu/shared.rs @@ -106,16 +106,15 @@ impl Vm { return Ok(None); } let mut footprint = Footprint::new(self.id); + let bindings = self.shared_bindings.as_ref().and_then(|shared| { + (shared.count.load(Ordering::Relaxed) != 0).then(|| shared.bindings.lock()) + }); for attachment in attachments.as_slice() { let end = attachment.va.saturating_add(attachment.size); footprint.push_vm(attachment.va, end); - let Some(shared) = &self.shared_bindings else { + let Some(bindings) = &bindings else { continue; }; - if shared.count.load(Ordering::Relaxed) == 0 { - continue; - } - let bindings = shared.bindings.lock(); for entry in bindings.iter() { let first = entry.start.max(attachment.va); let last = entry.end.min(end); @@ -129,6 +128,7 @@ impl Vm { } } } + drop(bindings); Ok(Some(Arc::new(footprint, GFP_KERNEL)?)) } } From 8cf2e413b3924c7506cb689d4e7b1770a0e84b35 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:25:07 -0400 Subject: [PATCH 0628/2680] drm/asahi: settle unpublished jobs discarded by entity teardown Entity teardown can free a queued job without invoking its cancellation callback. Signal its independent completion fence through the existing unpublished cancellation path so cross-engine timestamp dependencies can retire. Leave jobs whose publication has already been claimed with their existing owner. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e0d2761bd854136076d642b614c9b6078699243e) --- drivers/gpu/drm/asahi/g17/job.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index f9d04e9e85f7e..76393a1802b82 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -249,6 +249,17 @@ struct Job { packet: Arc, } +impl Drop for Job { + fn drop(&mut self) { + // Entity teardown can free a queued job without invoking cancel. + // Its independent fence may still order another engine. A claimed + // publisher keeps responsibility for visible work and its VM pin. + if self.packet.publication.load(Ordering::Acquire) == 0 { + self.backend.cancel(self.packet.clone()); + } + } +} + impl sched::JobImpl for Job { fn false_timeout(job: &mut sched::Job) -> bool { job.is_finished() From df452242dd4fc9d61fcaa789bf217af7c923f323 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:25:08 -0400 Subject: [PATCH 0629/2680] drm/asahi: reserve completion capacity for global compute replay Submission collectors are sized for one physical queue, but recovery can settle retained commands from every compute queue. Reserve for the complete replay pass before changing any queue, including the publication that follows it. Skip reservation when no queue needs recovery work, preserving the small collector used by healthy submissions. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit d41b760af0bcc10744b4145cfdbc19dab7625e8e) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 10 ++++++++++ drivers/gpu/drm/asahi/g17/runtime.rs | 8 ++++++++ drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs | 12 ++++++++++++ 3 files changed, 30 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index f8297f0462ff8..832d5073ecc7b 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -798,6 +798,16 @@ impl Queue { Ok(()) } + /// Both FIFOs are bounded by IN_FLIGHT. Count all retained commands of + /// queues which this pass can release, replay or quarantine on failure. + pub(crate) fn replay_commands(&self) -> usize { + if self.awaiting_witness || self.replays.len != 0 { + self.active.len + self.replays.len + } else { + 0 + } + } + pub(crate) fn replay_front(&self) -> Option> { self.replays.front().cloned() } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index d7a71a2c4c899..6fe67478a777f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -501,6 +501,14 @@ impl DeferredBatch { values: KVVec::with_capacity(capacity, GFP_KERNEL)?, }) } + /// A recovery pass can fail and retire each retained command at most once. + /// Reserve before changing queues, plus a full publication for its caller. + /// Healthy submissions keep the small publication collector unchanged. + fn reserve_replays(&mut self, commands: usize) -> Result { + self.values + .reserve(2 * commands + 2 * Self::DEPTH, GFP_KERNEL)?; + Ok(()) + } pub(super) fn push(&mut self, value: super::completion::Deferred) -> Result { use super::completion::Deferred; // Publishers may acquire the device mutex as soon as this pass drops diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index 17f228b9d1f04..c872a7c8dacc7 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -18,6 +18,18 @@ impl Firmware { if state != 0 { return Ok(()); } + let commands = self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .map(|queue| queue.replay_commands()) + .sum::(); + if commands == 0 { + return Ok(()); + } + deferred.reserve_replays(commands)?; let view = qos::View::new(self.init.qos()?)?; for entry in self.queues.compute.iter_mut().flatten() { if let Some(queue) = entry.queue.as_deref_mut() { From 8b793f762a4d39a08181aade799db1ddd3c00dcb Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:30:36 -0400 Subject: [PATCH 0630/2680] drm/asahi: filter compute completion identities before payload reads A shared completion scan needs only the control word of unrelated QIDs. Retain a bounded record view, capture control once, and read descriptor and queue identity before the two timestamps used for retirement. This avoids unused payload loads while preserving the single Acquire-ordered scan and exact kick matching. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 868c80c921527154df5e73a271583a4b0b63fdeb) --- drivers/gpu/drm/asahi/g17/initdata.rs | 33 ++++++++++++++----- .../drm/asahi/g17/queue/compute/retirement.rs | 22 ++++++++----- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 1e17e8c537d97..3183c4e94e4a8 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -29,6 +29,26 @@ const COMPUTE_CONTROL_BLOCKS: u32 = 25; const COMPUTE_CONTROL_CURSOR: u32 = 0x18; const COMPUTE_RUN_ADDRESS_MASK: u64 = 0x0000_ffff_ffff_f000; +/// A borrowed completion slot. Read the identity before requesting its timestamps; +/// unrelated QIDs need only the control word. Each access is an aligned live load. +pub(crate) struct ComputeCompletion<'a> { + object: &'a KernelObject, + offset: usize, +} + +impl ComputeCompletion<'_> { + pub(crate) fn read(&self, word: usize) -> Result { + if word >= 8 { + return Err(EINVAL); + } + // The scanner bounds offset to a 0x40-byte slot in the completion ring. + Ok(self + .object + .dword(self.offset + word * 8)? + .load(Ordering::Relaxed)) + } +} + /// Unpublished device-wide compute backing, prepared without the device mutex. The cache /// identity belongs to this allocation, not to any one physical queue or client VM. pub(super) struct ComputeShared { @@ -555,18 +575,15 @@ impl InitData { /// scanner latches exact identities; firmware retains sole ownership of these slots. pub(super) fn scan_compute_completions( &self, - mut record: impl FnMut([u64; 8]) -> Result, + mut record: impl FnMut(ComputeCompletion<'_>) -> Result, ) -> Result { let object = self.object(self.completion_rings[0])?; fence(Ordering::Acquire); for slot in 0..(abi::COMPLETION_RING_SIZE / 0x40).min(0x800) { - let mut words = [0; 8]; - for (index, word) in words.iter_mut().enumerate() { - *word = object - .dword(slot * 0x40 + index * 8)? - .load(Ordering::Relaxed); - } - record(words)?; + record(ComputeCompletion { + object, + offset: slot * 0x40, + })?; } Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index 7428f99ab1d7f..766177ebc32c9 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -3,6 +3,7 @@ //! Exact identities and visibility witnesses for a physical compute queue. //! A shared work-state counter can include other engines and cannot prove this queue retired. +use crate::g17::initdata::ComputeCompletion; use kernel::prelude::*; pub(crate) const IN_FLIGHT: usize = 16; @@ -170,17 +171,20 @@ impl Batch { self.open = Some(ticket.qid); Ok(()) } - /// The transport reads eight aligned words from a 0x40-byte firmware record. Records can be - /// overwritten during a scan: incomplete or older identities are skipped, never consumed. - pub(crate) fn fold(&mut self, words: [u64; 8]) -> Result { - let qid = ((words[0] >> 9) & 0x7f) as usize; + /// Capture the control word once, then read only a requested identity's payload. + /// Records can be overwritten during a scan: incomplete or older identities are skipped. + pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { + let control = record.read(0)?; + let qid = ((control >> 9) & 0x7f) as usize; let (start, count) = self.slots[qid]; - if count == 0 || words[6] & !1 == 0 || words[7] == 0 { + if count == 0 { return Ok(()); } - let descriptor = words[6] & !1; - let kick = - (((words[0] >> 8) & 0xffff_ffff_00) | ((words[0] >> 48) & 0xff)) & 0xff_ffff_ffff; + let descriptor = record.read(6)? & !1; + if descriptor == 0 || record.read(7)? == 0 { + return Ok(()); + } + let kick = (((control >> 8) & 0xffff_ffff_00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; let requests = self.requests.get_mut(start..start + count).ok_or(EIO)?; let Some(request) = requests .iter_mut() @@ -188,7 +192,7 @@ impl Batch { else { return Ok(()); }; - let timestamps = [words[2] & TIME_MASK, words[5] & TIME_MASK]; + let timestamps = [record.read(2)? & TIME_MASK, record.read(5)? & TIME_MASK]; if timestamps[0] == 0 || timestamps[1] == 0 || request From 26a1a7caa4fe8c4a4405bf66203c9ed565e1bf85 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:31:21 -0400 Subject: [PATCH 0631/2680] drm/asahi: sample opening counters at the polling deadline Read the opening control counters after the final one-millisecond wait, matching the existing firmware initialization polling budget. Firmware that retires the records during that last interval now completes initialization instead of being torn down as timed out. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit f61c86ab28302e46cba8fbdbaa8339aaa4e17b40) --- drivers/gpu/drm/asahi/g17/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 9a3b6b47f29aa..0fbe9a4cde15c 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -968,7 +968,7 @@ impl Gpu { primary: &Coprocessor, secondary: &Coprocessor, ) -> Result { - for _ in 0..OPENING_RETIRE_POLLS { + for poll in 0..=OPENING_RETIRE_POLLS { if primary.state.crashed.load(Ordering::Acquire) || secondary.state.crashed.load(Ordering::Acquire) { @@ -977,7 +977,9 @@ impl Gpu { if init.opening_retired(Role::Primary)? && init.opening_retired(Role::Secondary)? { return Ok(()); } - fsleep(Delta::from_millis(OPENING_RETIRE_POLL_MS)); + if poll < OPENING_RETIRE_POLLS { + fsleep(Delta::from_millis(OPENING_RETIRE_POLL_MS)); + } } dev_err!( dev.as_ref(), From 5cabef11e0ff02799172762917347ac3a74e45b9 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:44:57 -0400 Subject: [PATCH 0632/2680] drm/asahi: clarify the opening polling budget The initial sample precedes the bounded wait intervals; the final interval also ends with a counter sample. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit b4ee9bd8d923949b19cc22484262670948cafc57) --- drivers/gpu/drm/asahi/g17/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 0fbe9a4cde15c..825f20d0cfdf5 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -184,7 +184,7 @@ const INITDATA_ACK_TIMEOUT_MS: u32 = 2000; const RECEIVER_ATTEMPTS: usize = 3; const RECEIVER_RETRY_MS: i64 = 20; /// The firmware does not notify the host when it retires device-control records, so their -/// counters are polled at this interval, at most [`OPENING_RETIRE_POLLS`] times. +/// counters are sampled initially and after at most [`OPENING_RETIRE_POLLS`] waits. const OPENING_RETIRE_POLL_MS: i64 = 1; const OPENING_RETIRE_POLLS: usize = 100; From 61391e31a0598e423c5d1fd3d30e78ef6c276601 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 00:04:18 -0400 Subject: [PATCH 0633/2680] thunderbolt: apple: activate PCIe-C after the USB4 tunnel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — mainline 7.2 added its own tb_pci_pre_activate() (69a7b98770b7, LTSSM in Detect before setup); the host hook becomes tb_pci_host_pre_activate(), runs after those checks for new tunnels and alone for discovered ones, as before] [ryan: adapted to 7.2.10 — Asahi apple.c now documents struct apple_nhi; the helpers go above that comment] Signed-off-by: DjDeveloperr --- drivers/thunderbolt/apple.c | 423 +++++++++++++++++++++++++++++++++-- drivers/thunderbolt/nhi.h | 6 + drivers/thunderbolt/tunnel.c | 32 ++- 3 files changed, 442 insertions(+), 19 deletions(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index d8b20e49a9a27..adbb3493d43ab 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -79,15 +79,18 @@ #include #include +#include #include #include #include #include #include #include +#include #include #include #include +#include #include #include #include @@ -96,6 +99,7 @@ #include #include #include +#include #include "nhi.h" #include "tb.h" @@ -123,6 +127,10 @@ #define APPLE_CIO_SRAM_IOVA_BASE 0x10000000 #define APPLE_CIO_NHI_BOOT_TIMEOUT_MS 10000 +#define APPLE_CIO_PCIEC_READY_DELAY_MS 300 + +#define APPLE_CIO_PCIEC_INTR2AXI_CTRL 0x80 +#define APPLE_CIO_PCIEC_INTR2AXI_ENABLE BIT(0) /* * The Apple vendor-specific extended capability contains a cable-information @@ -163,6 +171,8 @@ struct apple_cio { struct device *dev; struct device_node *np; struct device_node *pcie_tunnel_np; + bool pcie_tunnel_preinitialized; + void __iomem *pcie_intr2axi_base; struct apple_rtkit *rtk; void __iomem *rc_base; @@ -184,13 +194,115 @@ struct apple_cio { int nhi_boot_status; struct typec_thunderbolt_switch_dev *tbt_switch; + /* Serializes PCIe-C population with cable teardown. */ + struct mutex pcie_tunnel_lock; + struct delayed_work pcie_tunnel_work; + bool pcie_tunnel_requested; + bool pcie_tunnel_populated; }; +/* + * Temporary hardware bring-up gate. A non-zero value stops ACIO startup after + * the corresponding phase so each power/reset/MMIO transition can be tested + * independently on new SoCs without advancing into the next phase. + */ +static unsigned int apple_cio_debug_stage; +module_param_named(debug_stage, apple_cio_debug_stage, uint, 0644); +MODULE_PARM_DESC(debug_stage, "Stop ACIO startup after the selected debug phase"); + +static unsigned int apple_cio_pcie_tunnel_trace_delay_ms; +module_param_named(pcie_tunnel_trace_delay_ms, + apple_cio_pcie_tunnel_trace_delay_ms, uint, 0644); +MODULE_PARM_DESC(pcie_tunnel_trace_delay_ms, + "Delay before PCIe-C child population so its trace reaches persistent storage"); + +static bool apple_cio_pcie_intr2axi_prearmed; +module_param_named(pcie_intr2axi_prearmed, + apple_cio_pcie_intr2axi_prearmed, bool, 0644); +MODULE_PARM_DESC(pcie_intr2axi_prearmed, + "Diagnostic: consume a live userspace Intr2AXI pulse instead of pulsing again"); + +static bool apple_cio_defer_start; +module_param_named(defer_start, apple_cio_defer_start, bool, 0644); +MODULE_PARM_DESC(defer_start, + "Hold a detected USB4/TBT cable until ACIO is started through sysfs"); + +static int apple_cio_power_domain_limit = -1; +module_param_named(power_domain_limit, apple_cio_power_domain_limit, int, 0644); +MODULE_PARM_DESC(power_domain_limit, + "Enable this many traced ACIO power domains before returning (-1 = all)"); + static void apple_cio_of_node_put(void *data) { of_node_put(data); } +static bool +apple_cio_pcie_tunnel_is_preinitialized(struct device_node *parent) +{ + for_each_available_child_of_node_scoped(parent, child) { + u32 status; + + if (!of_device_is_compatible(child, "apple,t8103-pciec") && + !of_device_is_compatible(child, "apple,t6000-pciec")) + continue; + + return !of_property_read_u32(child, + "apple,pciec-preinit-status", + &status) && status == 1; + } + + return false; +} + +static void apple_cio_iounmap(void *data) +{ + iounmap((__force void __iomem *)data); +} + +static int apple_cio_map_pcie_intr2axi(struct apple_cio *acio) +{ + struct resource res; + int index, ret; + + for_each_available_child_of_node_scoped(acio->pcie_tunnel_np, child) { + if (!of_device_is_compatible(child, "apple,t8103-pciec") && + !of_device_is_compatible(child, "apple,t6000-pciec")) + continue; + + index = of_property_match_string(child, "reg-names", "intr2axi"); + if (index < 0) + return index; + + ret = of_address_to_resource(child, index, &res); + if (ret) + return ret; + + /* + * Intr2AXI controls PCIe configuration transaction forwarding and + * must therefore use non-posted Device-nGnRnE accesses. A regular + * ioremap() is Device-nGnRE on arm64 and can leave this pulse pending + * until the following DART access, which raises an asynchronous SError. + * + * Do not reserve the resource here: the PCIe-C child owns the same DT + * aperture and will claim it when the tunnel children are populated. + */ + acio->pcie_intr2axi_base = + ioremap_np(res.start, resource_size(&res)); + if (!acio->pcie_intr2axi_base) + return -ENOMEM; + + ret = devm_add_action_or_reset(acio->dev, apple_cio_iounmap, + (__force void *)acio->pcie_intr2axi_base); + if (ret) + return ret; + + return 0; + } + + return -ENODEV; +} + /** * struct apple_nhi - Apple Native Host Interface * @dev: NHI device @@ -477,12 +589,203 @@ static void apple_nhi_ring_configure(struct tb_ring *ring, u32 flags, u32 e2e_fl writel(flags | e2e_flags, options); } +static int apple_cio_populate_pcie_tunnel(struct apple_cio *acio) +{ + /* + * PCIe-C was cold-initialized by m1n1 before the ACIO M3 started, but + * enabling ACIO's cable-powered PCIe domain closes the Intr2AXI bridge + * again. The port enable sequence pulses this register immediately + * before forcing its DART active. Do the same after ACIO has + * enabled both tunnel adapters and before either child can touch DART MMIO. + * The bit self-clears after opening the bridge. + */ + if (READ_ONCE(apple_cio_pcie_intr2axi_prearmed)) { + WRITE_ONCE(apple_cio_pcie_intr2axi_prearmed, false); + dev_info(acio->dev, + "PCIe-C Intr2AXI bridge accepted from live diagnostic handoff\n"); + } else { + writel(APPLE_CIO_PCIEC_INTR2AXI_ENABLE, + acio->pcie_intr2axi_base + APPLE_CIO_PCIEC_INTR2AXI_CTRL); + /* Complete the pulse without reading the transient register. */ + mb(); + dev_info(acio->dev, "PCIe-C Intr2AXI bridge enabled\n"); + } + if (apple_cio_debug_stage == 12) { + dev_info(acio->dev, + "ACIO debug stage 12: Intr2AXI pulsed; tunnel children held\n"); + return -EAGAIN; + } + + dev_info(acio->dev, + "PCIe-C live handoff accepted; populating tunnel children\n"); + if (READ_ONCE(apple_cio_pcie_tunnel_trace_delay_ms)) + msleep(READ_ONCE(apple_cio_pcie_tunnel_trace_delay_ms)); + + return of_platform_populate(acio->pcie_tunnel_np, NULL, NULL, + acio->dev); +} + +static int apple_cio_activate_pcie_tunnel_locked(struct apple_cio *acio) +{ + int ret; + + lockdep_assert_held(&acio->pcie_tunnel_lock); + + if (!acio->pcie_tunnel_np || acio->pcie_tunnel_populated) + return 0; + if (!acio->pcie_tunnel_preinitialized) + return dev_err_probe(acio->dev, -ENODEV, + "PCIe-C requires a successful m1n1 preinit handoff\n"); + + if (acio->pd_list->num_pds < 4 || !acio->pd_list->pd_links[3]) + return dev_err_probe(acio->dev, -ENODEV, + "pre-ACIO PCIe-C initialization is not active\n"); + + if (apple_cio_debug_stage == 11) { + dev_info(acio->dev, + "ACIO debug stage 11: PCIe tunnel adapters held for manual population\n"); + return 0; + } + + ret = apple_cio_populate_pcie_tunnel(acio); + if (ret) + return dev_err_probe(acio->dev, ret, + "failed to populate tunneled PCIe controller\n"); + + acio->pcie_tunnel_populated = true; + dev_info(acio->dev, + "PCIe-C populated after USB4 PCIe tunnel adapter enable\n"); + return 0; +} + +static void apple_cio_pcie_tunnel_work(struct work_struct *work) +{ + struct apple_cio *acio = + container_of(to_delayed_work(work), struct apple_cio, + pcie_tunnel_work); + int ret; + + mutex_lock(&acio->pcie_tunnel_lock); + if (!READ_ONCE(acio->pcie_tunnel_requested)) { + mutex_unlock(&acio->pcie_tunnel_lock); + return; + } + + ret = apple_cio_activate_pcie_tunnel_locked(acio); + mutex_unlock(&acio->pcie_tunnel_lock); + if (ret && ret != -EAGAIN) + dev_err(acio->dev, "deferred PCIe-C activation failed: %d\n", ret); +} + +static int apple_cio_start(struct apple_cio *acio); + +static ssize_t acio_start_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct apple_cio *acio = dev_get_drvdata(dev); + int ret; + + if (!sysfs_streq(buf, "1")) + return -EINVAL; + + ret = mutex_lock_interruptible(&acio->lock); + if (ret) + return ret; + + if (acio->current_cable_info) { + ret = -EALREADY; + goto out_unlock; + } + if (!acio->target_cable_info) { + ret = -ENODEV; + goto out_unlock; + } + + dev_info(acio->dev, + "starting deferred ACIO cable transition %#x\n", + acio->target_cable_info); + ret = apple_cio_start(acio); + +out_unlock: + mutex_unlock(&acio->lock); + return ret ?: count; +} +static DEVICE_ATTR_WO(acio_start); + +static ssize_t pcie_tunnel_populate_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct apple_cio *acio = dev_get_drvdata(dev); + int ret; + + if (!sysfs_streq(buf, "1")) + return -EINVAL; + + ret = mutex_lock_interruptible(&acio->pcie_tunnel_lock); + if (ret) + return ret; + + ret = apple_cio_activate_pcie_tunnel_locked(acio); + mutex_unlock(&acio->pcie_tunnel_lock); + + return ret ?: count; +} +static DEVICE_ATTR_WO(pcie_tunnel_populate); + +static struct attribute *apple_cio_attrs[] = { + &dev_attr_acio_start.attr, + &dev_attr_pcie_tunnel_populate.attr, + NULL, +}; + +static const struct attribute_group apple_cio_group = { + .attrs = apple_cio_attrs, +}; + +static int apple_nhi_pci_tunnel_pre_activate(struct tb_nhi *nhi) +{ + struct apple_nhi *anhi = nhi_to_anhi(nhi); + struct apple_cio *acio = anhi->acio; + + /* + * Remember that the PCIe tunnel reached activation. PCIe-C must not be + * populated here: the tunnel paths and both adapters are still disabled. + */ + WRITE_ONCE(acio->pcie_tunnel_requested, true); + return 0; +} + +static int apple_nhi_pci_tunnel_post_activate(struct tb_nhi *nhi) +{ + struct apple_nhi *anhi = nhi_to_anhi(nhi); + struct apple_cio *acio = anhi->acio; + int ret; + + /* + * This is the Linux equivalent of Apple's startUsingTunnel() boundary. + * Only now can releaseResetFlow() clear the tunneled PERST state before + * the root port enables LTSSM. + */ + mutex_lock(&acio->pcie_tunnel_lock); + ret = apple_cio_activate_pcie_tunnel_locked(acio); + mutex_unlock(&acio->pcie_tunnel_lock); + + /* Stage 12 intentionally leaves the active tunnel paths in place. */ + if (ret == -EAGAIN && apple_cio_debug_stage == 12) + return 0; + return ret; +} + static const struct tb_nhi_ops apple_nhi_ops = { .request_ring_irq = apple_nhi_request_irq, .release_ring_irq = apple_nhi_release_irq, .ring_interrupt_active = apple_nhi_ring_interrupt_active, .ring_interrupt_mask = apple_nhi_ring_interrupt_mask, .ring_configure = apple_nhi_ring_configure, + .pci_tunnel_pre_activate = apple_nhi_pci_tunnel_pre_activate, + .pci_tunnel_post_activate = apple_nhi_pci_tunnel_post_activate, }; static const struct tb_nhi_ring_layout apple_nhi_ring_layout = { @@ -685,7 +988,12 @@ static void apple_cio_stop(struct apple_cio *acio) * After we shut down the ACIO co-processor we will no longer be able to access * the MMIO space of these so make sure nothing tries to do just that. */ + WRITE_ONCE(acio->pcie_tunnel_requested, false); + cancel_delayed_work_sync(&acio->pcie_tunnel_work); + mutex_lock(&acio->pcie_tunnel_lock); of_platform_depopulate(acio->dev); + acio->pcie_tunnel_populated = false; + mutex_unlock(&acio->pcie_tunnel_lock); /* Try to shut down and power off the co-processor gracefully */ ret = apple_rtkit_poweroff(acio->rtk); @@ -725,6 +1033,17 @@ static int apple_cio_start(struct apple_cio *acio) /* Create device links to the power domains in order to power them on */ for (i = 0; i < acio->pd_list->num_pds; i++) { + if (READ_ONCE(apple_cio_power_domain_limit) >= 0) { + dev_emerg(acio->dev, + "ACIO power-domain access %d armed: enable %s\n", + i + 1, dev_name(acio->pd_list->pd_devs[i])); + if (i >= READ_ONCE(apple_cio_power_domain_limit)) { + ret = -EAGAIN; + goto remove_links; + } + dev_emerg(acio->dev, + "ACIO power-domain access %d executing\n", i + 1); + } link = device_link_add(acio->dev, acio->pd_list->pd_devs[i], DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE); if (!link) { @@ -732,6 +1051,20 @@ static int apple_cio_start(struct apple_cio *acio) goto remove_links; } acio->pd_list->pd_links[i] = link; + if (READ_ONCE(apple_cio_power_domain_limit) >= 0) + dev_emerg(acio->dev, + "ACIO power-domain access %d completed\n", i + 1); + dev_info(acio->dev, + "ACIO power domain %d (%s): runtime %s, usage %d\n", + i, dev_name(acio->pd_list->pd_devs[i]), + pm_runtime_active(acio->pd_list->pd_devs[i]) ? + "active" : "inactive", + atomic_read(&acio->pd_list->pd_devs[i]->power.usage_count)); + } + if (apple_cio_debug_stage == 1) { + dev_info(acio->dev, "ACIO debug stage 1: power domains enabled\n"); + ret = -EAGAIN; + goto remove_links; } /* @@ -743,6 +1076,11 @@ static int apple_cio_start(struct apple_cio *acio) dev_err(acio->dev, "ACIO block failed to start: %d\n", ret); goto remove_links; } + if (apple_cio_debug_stage == 2) { + dev_info(acio->dev, "ACIO debug stage 2: reset deasserted\n"); + ret = -EAGAIN; + goto remove_links; + } /* * Start and wait for the co-processor to boot. @@ -753,19 +1091,39 @@ static int apple_cio_start(struct apple_cio *acio) * them with a bare store. */ val = readl(acio->rc_base + APPLE_CIO_M3_CTRL); + if (apple_cio_debug_stage == 3) { + dev_info(acio->dev, "ACIO debug stage 3: M3 control read as %#x\n", val); + ret = -EAGAIN; + goto remove_links; + } writel(val | APPLE_CIO_M3_CTRL_START, acio->rc_base + APPLE_CIO_M3_CTRL); + if (apple_cio_debug_stage == 4) { + dev_info(acio->dev, "ACIO debug stage 4: M3 start requested\n"); + ret = -EAGAIN; + goto remove_links; + } acio->rtk = apple_rtkit_init(acio->dev, acio, NULL, 0, &apple_cio_rtkit_ops); if (IS_ERR(acio->rtk)) { ret = PTR_ERR(acio->rtk); dev_err(acio->dev, "Failed to initialize RTKit: %d\n", ret); goto remove_links; } + if (apple_cio_debug_stage == 5) { + dev_info(acio->dev, "ACIO debug stage 5: RTKit initialized\n"); + ret = -EAGAIN; + goto err_free_rtkit; + } ret = apple_rtkit_boot(acio->rtk); if (ret) { dev_err(acio->dev, "M3 RTKit failed to boot: %d\n", ret); goto err_free_rtkit; } + if (apple_cio_debug_stage == 6) { + dev_info(acio->dev, "ACIO debug stage 6: RTKit booted\n"); + ret = -EAGAIN; + goto err_shutdown_rtkit; + } ret = readl_poll_timeout(acio->rc_base + APPLE_CIO_M3_STAT, state, state & APPLE_CIO_M3_STAT_STATE, 100, 500000); @@ -773,8 +1131,18 @@ static int apple_cio_start(struct apple_cio *acio) dev_err(acio->dev, "M3 firmware failed to get ready: %d\n", ret); goto err_shutdown_rtkit; } + if (apple_cio_debug_stage == 7) { + dev_info(acio->dev, "ACIO debug stage 7: M3 ready state %#x\n", state); + ret = -EAGAIN; + goto err_shutdown_rtkit; + } apple_tunable_apply(acio->rc_base, acio->rc_tunable); + if (apple_cio_debug_stage == 8) { + dev_info(acio->dev, "ACIO debug stage 8: RC tunables applied\n"); + ret = -EAGAIN; + goto err_shutdown_rtkit; + } /* * Bring up devices which are part of ACIO and are now accessible by the main SoC @@ -786,6 +1154,11 @@ static int apple_cio_start(struct apple_cio *acio) dev_err(acio->dev, "failed to populate children: %d\n", ret); goto err_depopulate; } + if (apple_cio_debug_stage == 9) { + dev_info(acio->dev, "ACIO debug stage 9: child devices populated\n"); + ret = -EAGAIN; + goto err_depopulate; + } if (!wait_for_completion_timeout(&acio->nhi_boot_completion, msecs_to_jiffies(APPLE_CIO_NHI_BOOT_TIMEOUT_MS))) { @@ -797,22 +1170,10 @@ static int apple_cio_start(struct apple_cio *acio) ret = acio->nhi_boot_status; goto err_depopulate; } - - /* - * The PCIe-C root complex is outside the ACIO MMIO window but shares - * its cable-controlled power lifetime. Instantiate it only after the - * router and NHI are live; parenting it to ACIO makes depopulation tear - * it down before the co-processor and power domains disappear. - */ - if (acio->pcie_tunnel_np) { - ret = of_platform_populate(acio->pcie_tunnel_np, NULL, NULL, - acio->dev); - if (ret) { - dev_err(acio->dev, - "failed to populate tunneled PCIe controller: %d\n", - ret); - goto err_depopulate; - } + if (apple_cio_debug_stage == 10) { + dev_info(acio->dev, "ACIO debug stage 10: NHI is ready\n"); + ret = -EAGAIN; + goto err_depopulate; } acio->current_cable_info = acio->target_cable_info; @@ -915,6 +1276,13 @@ static int apple_cio_tbt_switch_set(struct typec_thunderbolt_switch_dev *sw, * Bring up or power down the ACIO complex * current_cable_info will be updated in the start/stop functions */ + if (acio->target_cable_info && READ_ONCE(apple_cio_defer_start)) { + dev_info(acio->dev, + "USB4/TBT cable transition %#x held for controlled ACIO start\n", + acio->target_cable_info); + return 0; + } + if (acio->target_cable_info) return apple_cio_start(acio); @@ -936,6 +1304,11 @@ static int apple_cio_probe(struct platform_device *pdev) ret = devm_mutex_init(dev, &acio->lock); if (ret) return ret; + ret = devm_mutex_init(dev, &acio->pcie_tunnel_lock); + if (ret) + return ret; + INIT_DELAYED_WORK(&acio->pcie_tunnel_work, + apple_cio_pcie_tunnel_work); init_completion(&acio->nhi_boot_completion); acio->dev = &pdev->dev; acio->np = dev->of_node; @@ -946,6 +1319,19 @@ static int apple_cio_probe(struct platform_device *pdev) acio->pcie_tunnel_np); if (ret) return ret; + + acio->pcie_tunnel_preinitialized = + apple_cio_pcie_tunnel_is_preinitialized( + acio->pcie_tunnel_np); + if (!acio->pcie_tunnel_preinitialized) { + dev_warn(dev, + "PCIe-C tunnel disabled: m1n1 handoff is not initialized\n"); + } else { + ret = apple_cio_map_pcie_intr2axi(acio); + if (ret) + return dev_err_probe(dev, ret, + "failed to map PCIe-C Intr2AXI registers\n"); + } } acio->sram_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "sram"); @@ -992,6 +1378,10 @@ static int apple_cio_probe(struct platform_device *pdev) else if (ret < 3) return dev_err_probe(dev, -EINVAL, "Not enough PM domains\n"); + ret = devm_device_add_group(dev, &apple_cio_group); + if (ret) + return ret; + /* And finally register the OOB notification for Thunderbolt/USB4 cables */ struct typec_thunderbolt_switch_desc desc = { .fwnode = pdev->dev.fwnode, @@ -1011,6 +1401,7 @@ static void apple_cio_remove(struct platform_device *pdev) struct apple_cio *acio = platform_get_drvdata(pdev); typec_thunderbolt_switch_unregister(acio->tbt_switch); + cancel_delayed_work_sync(&acio->pcie_tunnel_work); guard(mutex)(&acio->lock); if (acio->current_cable_info) diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index afd9a11489a93..95c8ec724678a 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -79,6 +79,10 @@ struct tb_nhi_ring_layout { * @ring_configure: NHI specific hook to program the ring options registers * and enable the ring with the given flags. If not set * the standard USB4 NHI registers are used. + * @pci_tunnel_pre_activate: NHI specific hook run after a PCIe tunnel has + * been allocated but before its paths and adapters are enabled + * @pci_tunnel_post_activate: NHI specific hook run after a PCIe tunnel's + * paths and adapters have been enabled * @is_present: Whether the device is currently present on the parent bus * @init_interrupts: NHI specific interrupt initialization hook * @reset_interface: Resets the host interface @@ -97,6 +101,8 @@ struct tb_nhi_ops { void (*ring_interrupt_active)(struct tb_ring *ring, bool active); void (*ring_interrupt_mask)(struct tb_ring *ring, bool mask); void (*ring_configure)(struct tb_ring *ring, u32 flags, u32 e2e_flags); + int (*pci_tunnel_pre_activate)(struct tb_nhi *nhi); + int (*pci_tunnel_post_activate)(struct tb_nhi *nhi); bool (*is_present)(struct tb_nhi *nhi); int (*init_interrupts)(struct tb_nhi *nhi); void (*reset_interface)(struct tb_nhi *nhi); diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index e4c2001b74512..09d6b1d3f2637 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -309,6 +309,8 @@ static int tb_pci_port_ltssm_state_detect(struct tb_port *port) return -ETIMEDOUT; } +static int tb_pci_host_pre_activate(struct tb_tunnel *tunnel); + static int tb_pci_pre_activate(struct tb_tunnel *tunnel) { struct tb_port *down = tunnel->src_port; @@ -320,8 +322,12 @@ static int tb_pci_pre_activate(struct tb_tunnel *tunnel) if (ret) return ret; - return tb_switch_is_usb4(up->sw) ? + ret = tb_switch_is_usb4(up->sw) ? tb_pci_port_ltssm_state_detect(up) : 0; + if (ret) + return ret; + + return tb_pci_host_pre_activate(tunnel); } static int tb_pci_set_ext_encapsulation(struct tb_tunnel *tunnel, bool enable) @@ -358,6 +364,24 @@ static int tb_pci_set_ext_encapsulation(struct tb_tunnel *tunnel, bool enable) return 0; } +static int tb_pci_host_pre_activate(struct tb_tunnel *tunnel) +{ + const struct tb_nhi_ops *ops = tunnel->tb->nhi->ops; + + if (ops && ops->pci_tunnel_pre_activate) + return ops->pci_tunnel_pre_activate(tunnel->tb->nhi); + return 0; +} + +static int tb_pci_post_activate(struct tb_tunnel *tunnel) +{ + const struct tb_nhi_ops *ops = tunnel->tb->nhi->ops; + + if (ops && ops->pci_tunnel_post_activate) + return ops->pci_tunnel_post_activate(tunnel->tb->nhi); + return 0; +} + static int tb_pci_activate(struct tb_tunnel *tunnel, bool activate) { int res; @@ -375,17 +399,18 @@ static int tb_pci_activate(struct tb_tunnel *tunnel, bool activate) if (res) return res; - if (activate) { res = tb_pci_port_enable(tunnel->src_port, activate); if (res) return res; + + return tb_pci_post_activate(tunnel); } else { /* Downstream router could be unplugged */ tb_pci_port_enable(tunnel->dst_port, activate); } - return activate ? 0 : tb_pci_set_ext_encapsulation(tunnel, activate); + return tb_pci_set_ext_encapsulation(tunnel, activate); } static int tb_pci_init_credits(struct tb_path_hop *hop) @@ -463,6 +488,7 @@ struct tb_tunnel *tb_tunnel_discover_pci(struct tb *tb, struct tb_port *down, return NULL; tunnel->activate = tb_pci_activate; + tunnel->pre_activate = tb_pci_host_pre_activate; tunnel->src_port = down; /* From 1d84f3d167cc28074ba9a3ed8b0bc0d882d34052 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 00:07:39 -0400 Subject: [PATCH 0634/2680] thunderbolt: apple: complete PCIe-C build dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — split by topic: the PCIe controller hunks are carried by 080-radio] [ryan: adapted to 7.2.10 — mainline 7.2 added ADP_PCIE_CS_0_LTSSM_MASK next to the new bit; both kept in bit order] Signed-off-by: DjDeveloperr --- drivers/thunderbolt/Kconfig | 1 + drivers/thunderbolt/tb_regs.h | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/thunderbolt/Kconfig b/drivers/thunderbolt/Kconfig index d9fbe3a0dc3fd..ba82f5c57959f 100644 --- a/drivers/thunderbolt/Kconfig +++ b/drivers/thunderbolt/Kconfig @@ -80,6 +80,7 @@ config USB4_APPLE_SOC depends on ARCH_APPLE || COMPILE_TEST depends on OF && APPLE_RTKIT depends on TYPEC + depends on PCIE_APPLE select APPLE_TUNABLE help Say Y here to add support for the USB4 and Thunderbolt host diff --git a/drivers/thunderbolt/tb_regs.h b/drivers/thunderbolt/tb_regs.h index 92d893634d2bc..47a1a1e909b87 100644 --- a/drivers/thunderbolt/tb_regs.h +++ b/drivers/thunderbolt/tb_regs.h @@ -473,6 +473,7 @@ struct tb_regs_port_header { /* PCIe adapter registers */ #define ADP_PCIE_CS_0 0x00 +#define ADP_PCIE_CS_0_LA BIT(16) #define ADP_PCIE_CS_0_LTSSM_MASK GENMASK(28, 25) #define ADP_PCIE_CS_0_PE BIT(31) #define ADP_PCIE_CS_1 0x01 From 8063a17185e983da58ed5e18489cfbfba4b92c89 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:49:09 -0400 Subject: [PATCH 0635/2680] drm/asahi: commit canonical binding metadata with each user mapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prepared mapping batches can fail after earlier leaves have committed. Update canonical context metadata with each successful leaf so context setup observes mappings already visible in the VM. Preflight arithmetic in a compact map plan and retain displaced GEM references until reservation locks are released. Preserve canonical invalidation when quarantined unmaps enter deferred cleanup. Remove the duplicate ioctl update vector, extra object references and post-batch metadata pass. [ryan: adapted to 7.2.10 — minimal; for DJ: the bind batch hunks are applied to the 7.2 GPUVM form of mmu/bind.rs (sm_map/sm_unmap through OpMapRequest and VmOpResources)] Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 99dd8deec12fe28abb844854059bce6a87764199) --- drivers/gpu/drm/asahi/file.rs | 36 -------- drivers/gpu/drm/asahi/mmu/bind.rs | 19 ++++- drivers/gpu/drm/asahi/mmu/context.rs | 117 +++++++++++++++++++------- drivers/gpu/drm/asahi/mmu/lifetime.rs | 9 ++ 4 files changed, 110 insertions(+), 71 deletions(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 6da9e6825e661..fd392be710d33 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -40,7 +40,6 @@ use kernel::error::code::*; use kernel::new_mutex; use kernel::prelude::*; use kernel::sync::{ - aref::ARef, Arc, Mutex, // }; @@ -61,17 +60,6 @@ const DEBUG_CLASS: DebugFlags = DebugFlags::File; pub(crate) const MAX_COMMANDS_PER_SUBMISSION: u32 = 64; -/// Context-view registry updates applied after a prepared VM batch commits. -enum MappingUpdate { - Map { - object: ARef, - address: u64, - size: u64, - offset: u64, - }, - Unmap(Range), -} - /// A client instance of an `mmu::Vm` address space. struct Vm { ualloc: Arc>, @@ -717,7 +705,6 @@ impl File { ) }; let mut batch = mmu::PreparedUserBindBatch::new(count)?; - let mut updates = KVec::with_capacity(count, GFP_KERNEL)?; let mut unmaps = 0; // A fresh reader per declared stride also skips extension bytes in longer records. for bytes in bytes.chunks_exact(stride) { @@ -740,7 +727,6 @@ impl File { return Err(EINVAL); } batch.push_unmap(vm.prepare_user_unmap(op.addr, op.range)?)?; - updates.push(MappingUpdate::Unmap(range), GFP_KERNEL)?; unmaps += 1; continue; } @@ -780,31 +766,9 @@ impl File { prot, single_page, )?)?; - updates.push( - MappingUpdate::Map { - object, - address: op.addr, - size: op.range, - offset: op.offset, - }, - GFP_KERNEL, - )?; } vm.reserve_deferred_user_unmaps(unmaps)?; vm.commit_prepared_user_bind_batch(&mut batch)?; - for update in updates { - match update { - MappingUpdate::Map { - object, - address, - size, - offset, - } => { - vm.track_context_binding(object, address, size, offset)?; - } - MappingUpdate::Unmap(range) => vm.untrack_context_range(range), - } - } vm.bo_deferred_cleanup(); Ok(0) } diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index 9bd9a50331860..e8efd4847e02f 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -15,6 +15,7 @@ pub(crate) struct PreparedUserMap { size: u64, offset: u64, single_page: bool, + context: context::PreparedBinding, } pub(crate) struct PreparedUserUnmap { @@ -60,6 +61,7 @@ impl Vm { prot: Prot, single_page: bool, ) -> Result { + let context = self.prepare_context_binding(addr, size, offset)?; let resources = VmOpResources { preallocated_gpuvas: [ Some(GpuVaAlloc::::new(GFP_KERNEL)?), @@ -92,6 +94,7 @@ impl Vm { size, offset, single_page, + context, }) } @@ -144,7 +147,8 @@ impl Vm { context: &mut ctx, flags, }); - if result.is_ok() { + let retired = result.is_ok().then(|| { + let retired = self.commit_context_binding(&map.gem, &map.context); self.track_shared_binding( &map.gem, map.addr, @@ -152,9 +156,11 @@ impl Vm { map.offset, map.single_page, ); - } + retired + }); drop(inner); drop(commit); + drop(retired); Some(result) } Err(error) if error == EIO => { @@ -175,11 +181,16 @@ impl Vm { inner .unique_gpuvm .sm_unmap(unmap.iova, unmap.size, &mut ctx); - if result.is_ok() { + let retired = result.is_ok().then(|| { + let retired = self.untrack_context_ranges(core::iter::once( + unmap.iova..unmap.iova + unmap.size, + )); self.untrack_shared_range(unmap.iova, unmap.size); - } + retired + }); drop(inner); drop(commit); + drop(retired); Some(result) } Err(error) if error == EIO => { diff --git a/drivers/gpu/drm/asahi/mmu/context.rs b/drivers/gpu/drm/asahi/mmu/context.rs index 43244c2ce1730..d459a3a91afce 100644 --- a/drivers/gpu/drm/asahi/mmu/context.rs +++ b/drivers/gpu/drm/asahi/mmu/context.rs @@ -4,6 +4,8 @@ //! //! Userspace owns low views 1 through 6. Only the reserved null view is created //! here. Its installation is serialized per VM and retained through VM teardown. +//! Commits take bindings after the GPUVM exec lock; deferred unmaps take it +//! after the VM lifetime lock. Readers release bindings before taking either. use super::*; use crate::hw::t8140::CONTEXT_VIEWS; @@ -17,6 +19,24 @@ struct Bindings { generation: u64, slots: [Option; CONTEXT_VIEWS.len()], } +/// All arithmetic for a canonical map is checked before its page-table commit. +pub(super) struct PreparedBinding { + address: u64, + object_offset: u64, + touched: u8, + covered: u8, +} + +/// Final GEM release can acquire DMA reservations. Keep replaced references +/// until the caller has released its GPUVM and mapping-admission locks. +pub(super) struct RetiredBindings([Option; CONTEXT_VIEWS.len()]); + +impl RetiredBindings { + fn new() -> Self { + Self(core::array::from_fn(|_| None)) + } +} + #[pin_data] pub(super) struct ContextBindings { #[pin] @@ -57,64 +77,99 @@ impl Vm { Ok(()) } - /// Arithmetic is preflighted before invalidating any retained candidate. - pub(crate) fn track_context_binding( + pub(super) fn prepare_context_binding( &self, - gem: ARef, address: u64, size: u64, object_offset: u64, - ) -> Result { - let Some(context) = self.context_bindings.as_ref() else { - return Ok(()); + ) -> Result { + let mut update = PreparedBinding { + address, + object_offset, + touched: 0, + covered: 0, }; + if self.context_bindings.is_none() { + return Ok(update); + } let end = address.checked_add(size).ok_or(EOVERFLOW)?; - let mut offsets = [None; CONTEXT_VIEWS.len()]; for (index, view) in CONTEXT_VIEWS.iter().enumerate() { + if address < view.source + view.size as u64 && view.source < end { + update.touched |= 1 << index; + } if address <= view.source && view.source + view.size as u64 <= end { - offsets[index] = Some(usize::try_from( + update.covered |= 1 << index; + usize::try_from( object_offset .checked_add(view.source - address) .ok_or(EOVERFLOW)?, - )?); + )?; } } + Ok(update) + } + + /// Called after a successful leaf commit, while its exec lock still orders + /// mapping changes. The prepared update cannot fail after visibility. + pub(super) fn commit_context_binding( + &self, + gem: &ARef, + update: &PreparedBinding, + ) -> RetiredBindings { + let mut retired = RetiredBindings::new(); + if update.touched == 0 { + return retired; + } + let Some(context) = self.context_bindings.as_ref() else { + return retired; + }; let mut bindings = context.bindings.lock(); - let mut touched = false; - for (index, view) in CONTEXT_VIEWS.iter().enumerate() { - if address < view.source + view.size as u64 && view.source < end { - bindings.slots[index] = None; - touched = true; + for (index, slot) in bindings.slots.iter_mut().enumerate() { + if update.touched & (1 << index) != 0 { + retired.0[index] = slot.take(); } - if let Some(offset) = offsets[index] { - bindings.slots[index] = Some(Binding { + if update.covered & (1 << index) != 0 { + // Coverage proves source >= address. Preparation checked this + // exact sum and its usize conversion against the same view. + let offset = (update.object_offset + + (CONTEXT_VIEWS[index].source - update.address)) as usize; + *slot = Some(Binding { gem: gem.clone(), offset, }); - touched = true; } } - if touched { - bindings.generation = bindings.generation.wrapping_add(1); - } - Ok(()) + bindings.generation = bindings.generation.wrapping_add(1); + retired } - pub(crate) fn untrack_context_range(&self, range: Range) { + /// Successful user unmaps and accepted deferred unmaps invalidate the + /// matching candidates. Returning their owners avoids destruction under + /// the caller's exec or lifetime lock. + pub(super) fn untrack_context_ranges( + &self, + ranges: impl Iterator>, + ) -> RetiredBindings { + let mut retired = RetiredBindings::new(); let Some(context) = self.context_bindings.as_ref() else { - return; + return retired; }; let mut bindings = context.bindings.lock(); - let mut touched = false; - for (index, view) in CONTEXT_VIEWS.iter().enumerate() { - if range.start < view.source + view.size as u64 && view.source < range.end { - bindings.slots[index] = None; - touched = true; + for range in ranges { + let mut touched = false; + for (index, view) in CONTEXT_VIEWS.iter().enumerate() { + if range.start < view.source + view.size as u64 && view.source < range.end { + if let Some(binding) = bindings.slots[index].take() { + retired.0[index] = Some(binding); + } + touched = true; + } + } + if touched { + bindings.generation = bindings.generation.wrapping_add(1); } } - if touched { - bindings.generation = bindings.generation.wrapping_add(1); - } + retired } pub(crate) fn untrack_context_object(&self, gem: &gem::Object) { diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 3b8a5e5a40aeb..bff43fbb3795e 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -436,6 +436,9 @@ impl Vm { if !state.quarantined { return Err(EBUSY); } + // Reserve before accepting any range; a partial deferred batch must + // not leave canonical metadata ahead of the accepted cleanup. + state.unmaps.reserve(batch.ops.len(), GFP_KERNEL)?; for op in &batch.ops { if let PreparedUserBindOp::Unmap(unmap) = op { let range = (unmap.iova, unmap.size); @@ -444,6 +447,12 @@ impl Vm { } } } + let retired = self.untrack_context_ranges(batch.ops.iter().filter_map(|op| match op { + PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), + PreparedUserBindOp::Map(_) => None, + })); + drop(state); + drop(retired); Ok(true) } From ec8e92767c77008341897d2e1ae66540365794bd Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 03:04:01 -0400 Subject: [PATCH 0636/2680] drm/asahi: order canonical object invalidation with mapping removal Remove canonical candidates under the existing GPUVM exec guard after successful object unmap. Keep deferred acceptance invalidation under the lifetime lock, with the queued GEM reference retaining the object. This prevents a trailing close callback from erasing a newer committed binding without adding locking or allocation. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 11b50b74597c2c3e1f5687d5db50ce0eaa103c21) --- drivers/gpu/drm/asahi/file.rs | 1 - drivers/gpu/drm/asahi/mmu.rs | 3 +++ drivers/gpu/drm/asahi/mmu/lifetime.rs | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index fd392be710d33..a772374bb4f98 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -970,7 +970,6 @@ impl File { let vm = file_vm.borrow().vm.clone(); core::mem::drop(file_vm); vm.drop_mappings(bo)?; - vm.untrack_context_object(bo); if idx == usize::MAX { break; } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 2dfe30ae3617d..48221de9953b2 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -2091,6 +2091,9 @@ impl Vm { if let Some(bo) = vm_exec.unique_gpuvm.find_bo(gem) { mod_dev_dbg!(self.dev, "MMU: bo_unmap\n"); vm_exec.unique_gpuvm.bo_unmap(&mut ctx, &bo)?; + // The close callback or deferred object list retains this GEM, so + // clearing its candidate cannot release the final reference here. + self.untrack_context_object(gem); self.untrack_shared_object(gem); mod_dev_dbg!(self.dev, "MMU: bo_unmap done\n"); // We need to drop the exec_lock first, then the GpuVmBo since that will take the lock itself. diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index bff43fbb3795e..78f906ff9c46c 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -491,6 +491,10 @@ impl Vm { state.objects.push(gem.into(), GFP_KERNEL)?; } if state.active != 0 || state.draining { + // Stop new jobs from selecting this accepted cleanup candidate. + // The queued object retains its GEM while the candidate is removed. + // Actual unmap repeats this under exec to order any intervening bind. + self.untrack_context_object(gem); return Ok(()); } state.draining = true; From e0b2f0eb6fe3e137dbe9e04cc5bfa7b79e048b8e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:22:22 -0400 Subject: [PATCH 0637/2680] drm/asahi: advertise G17 queue and input-sync capabilities Report the independently reserved logical queue capacity and per-command input synchronization already provided by G17. Userspace can then expose multiple queues and preserve independent command roots while forwarding input syncs to each submission. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit f594e2be7f492d0cb4fdbdf5197122418f330036) --- drivers/gpu/drm/asahi/g17/mod.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 825f20d0cfdf5..c4293633a0715 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1034,6 +1034,8 @@ impl gpu::Gpu for Gpu { } fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { + params.features |= + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64; params.gpu_generation = hw::GpuGen::G17 as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; @@ -1063,6 +1065,14 @@ impl gpu::Gpu for Gpu { true } + fn queue_limits(&self) -> Option { + Some(uapi::drm_asahi_queue_limits { + max_queues: mmu::MAX_EXECUTION_CONTEXTS, + max_in_flight_per_queue: hw::t8140::queues::RENDER_PAIRS_PER_OWNER as u32, + ..Default::default() + }) + } + fn uat_geometry(&self) -> mmu::UatGeometry { self.geometry } From cd65f17cb97608275056a3c80863efe27c14ec35 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:27:10 -0400 Subject: [PATCH 0638/2680] drm/asahi: acquire event records after observing their producer Order firmware event payload reads after the producer cursor establishes that the record is available. Move the existing acquire fence to this boundary, preserving the record-copy and consumer-store ordering without adding a barrier to nonempty drains. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 98783a0fc38dbac9eecb4164ed6c090e3adf591d) --- drivers/gpu/drm/asahi/g17/channel.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index 3a74e889c5ec7..59e8e95d4bbd9 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -184,7 +184,6 @@ impl Events { while self.remaining != 0 { init.validate_event_ring(role)?; self.remaining -= 1; - fence(Ordering::Acquire); let consumer = consumer_word.load(Ordering::Relaxed); let producer = producer_word.load(Ordering::Relaxed); if consumer >= EVENT_SLOTS || producer >= EVENT_SLOTS { @@ -194,6 +193,8 @@ impl Events { self.remaining = 0; break; } + // Observe the producer before reading the record it publishes. + fence(Ordering::Acquire); let offset = status::EVENT_RING as usize + consumer as usize * EVENT_RECORD_SIZE; let record = EventRecord::from_words(init.status_read_words(role, offset)?); let event = record.decode(); From 65b4446cc8a2f35d6d1ee76998c4c5cc4aca4983 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:27:47 -0400 Subject: [PATCH 0639/2680] drm/asahi: discard partial G17 mappings when binding fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A G17 map can fail allocating a later page table after earlier leaves have been written, before a GPUVA retains the backing. Clear the full failed operation range and invalidate every context naming its root before returning the mapping error. Apply the same cleanup if the final GPUVA insertion fails. Use a nonallocating partial-map discard that tolerates absent leaves without normal unmap diagnostics. Preserve Handoff error behavior and avoid additional work on successful mappings. [ryan: adapted to 7.2.10 — minimal; for DJ: 7.2 OpMap::insert() cannot fail and there is no map_and_link_va(), so the rollback runs when map_pages() fails and when no preallocated GPUVA is left; the coredump-only context of pgtable.rs is gone] Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 267dc9ba9ea8e068a3d9d0683a58b72a51291ece) --- drivers/gpu/drm/asahi/mmu.rs | 34 +++++++++++++++++++++++++++++--- drivers/gpu/drm/asahi/pgtable.rs | 11 +++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 48221de9953b2..23d695bee8af9 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -491,18 +491,27 @@ impl gpuvm::DriverGpuVm for VmInner { one_page ); - self.page_table.map_pages( + if let Err(error) = self.page_table.map_pages( iova..(iova + len as u64), addr as PhysicalAddr, ctx.prot, one_page, - )?; + ) { + self.rollback_failed_map(map_addr, map_length as u64); + return Err(error); + } left -= len; iova += len as u64; } - let gpuva = ctx.preallocated_gpuva()?; + let gpuva = match ctx.preallocated_gpuva() { + Ok(gpuva) => gpuva, + Err(error) => { + self.rollback_failed_map(map_addr, map_length as u64); + return Err(error); + } + }; let op = op.insert(gpuva, pin_init::init_zeroed()); self.tlbi_contexts_naming_root(map_addr, map_length); @@ -609,6 +618,25 @@ impl gpuvm::DriverGpuVm for VmInner { } impl VmInner { + /// A failed map has no GPUVA owner to retain its backing or remove its leaves. + /// GPUVM has already split or removed overlaps throughout this operation's range. + fn rollback_failed_map(&mut self, addr: u64, size: u64) { + if self.uat_inner.firmware == UatFirmware::Handoff { + return; + } + // Include the current segment: map_pages can fail after writing some of its leaves. + // The checked GPUVM operation supplies an aligned range without overflow. + let result = self.page_table.discard_partial_map(addr..addr + size); + self.tlbi_contexts_naming_root(addr, size as usize); + if let Err(error) = result { + dev_err!( + self.dev.as_ref(), + "Could not discard failed mapping: {:?}\n", + error + ); + } + } + fn mapping_mutation(&mut self) -> MappingMutation { let state = self.mapping_epoch.clone(); if let Some(state) = state.as_ref() { diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 6a8dba48174f0..db91e456ab8b2 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -615,6 +615,17 @@ impl UatPageTable { Ok(()) }) } + + /// Discards an aligned map operation that failed before acquiring a GPUVA owner. + /// Missing tables and leaves are expected; existing table storage stays allocated. + pub(crate) fn discard_partial_map(&mut self, iova_range: Range) -> Result { + self.with_pages(iova_range, false, false, |_, ptes| { + for pte in ptes { + pte.store(0, Ordering::Relaxed); + } + Ok(()) + }) + } } impl Drop for UatPageTable { From 9ba24b3f0b3144ed2b0c10afb658fee1f68f689c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:34:59 -0400 Subject: [PATCH 0640/2680] drm/asahi: reuse the reserved output fence for submitted work Reserve one fence before submission validation and reuse it as the aggregate after preparation succeeds. Keep the same fence available for every pre-publication failure, including aggregate allocation failure, without allocating a second unused fence on successful submits. Once published, only the aggregate enqueue sentinel and command members may signal it. Leave output installation and command completion ordering unchanged. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 9f204371cd6118ecaa4635b76a763e6c4d53b728) --- drivers/gpu/drm/asahi/g17/fence.rs | 11 +++++++---- drivers/gpu/drm/asahi/g17/queue/mod.rs | 3 +-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 95a13dbdd54ff..32d25b3b608cb 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -48,10 +48,10 @@ pub(crate) struct Submission { } impl Submission { - pub(crate) fn new(contexts: &FenceContexts, context: Arc) -> Result> { + fn new(fence: UserFence, context: Arc) -> Result> { Ok(Arc::new( Self { - fence: independent(contexts)?, + fence, work_state: Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?, remaining: AtomicU32::new(1), error: AtomicI32::new(0), @@ -168,9 +168,11 @@ impl Outputs { } /// Publish the actual aggregate before enqueueing any command. - pub(crate) fn publish(&mut self, submission: Arc) { + pub(crate) fn publish(&mut self, context: Arc) -> Result> { + let submission = Submission::new(self.fallback.clone(), context)?; install(self.syncs.drain(self.input_count..), &submission.fence()); - self.submission = Some(submission); + self.submission = Some(submission.clone()); + Ok(submission) } /// The caller first records terminal VM errors and settles prior failed work. @@ -184,6 +186,7 @@ impl Drop for Outputs { fn drop(&mut self) { if let Some(submission) = self.submission.take() { submission.finish_enqueue(self.result); + return; } if let Err(error) = self.result { self.fallback.set_error(error); diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index e6b95193c97db..3c117ce88182a 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -287,8 +287,7 @@ impl crate::queue::Queue for Queue { } let (pending, guards) = self.prepare_submission(bytes)?; let mut guards = guards.into_iter(); - let aggregate = Submission::new(&self.fences, self.context.clone())?; - outputs.publish(aggregate.clone()); + let aggregate = outputs.publish(self.context.clone())?; for (command, order) in pending { let guard = guards.next().ok_or(EIO)?; self.enqueue( From 98e90f61f47005b4eacf4a6df90ac682d8753100 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:34:59 -0400 Subject: [PATCH 0641/2680] drm/syncobj: skip wait hint lookup for ineligible waits Check timeout and availability-wait eligibility before reading the optional utilization hint under the file spinlock. Preserve expiry checking immediately before entering the wait scope and avoid the extra lock on polls and availability waits. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 7dce231ad73d98e102b4b9e0b362cbab21750854) --- drivers/gpu/drm/drm_syncobj.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/drm_syncobj.c b/drivers/gpu/drm/drm_syncobj.c index 2ac67e51a5b15..d9a6445c63353 100644 --- a/drivers/gpu/drm/drm_syncobj.c +++ b/drivers/gpu/drm/drm_syncobj.c @@ -1170,13 +1170,14 @@ static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs, } } - /* Recheck expiry after allocations, user copies and deadline callbacks. */ - util_min = drm_syncobj_wait_hint(file_private); /* Polls, availability waits and already-complete fences need no boost. */ - if (util_min && timeout > 0 && - !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) { - scope_previous = sched_util_min_scope_enter(util_min); - scope_active = true; + if (timeout > 0 && !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) { + /* Recheck expiry after allocations, user copies and deadline callbacks. */ + util_min = drm_syncobj_wait_hint(file_private); + if (util_min) { + scope_previous = sched_util_min_scope_enter(util_min); + scope_active = true; + } } do { From 8c722be7fde8b16e0bdc4192cd63aa57c121e1d5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:29:06 -0400 Subject: [PATCH 0642/2680] drm/asahi: wake render growth when its prerequisites become ready A retained TVB request can wait for a borrowed render pair or control-ring publication. Requeuing the growth worker after every event pass makes it bounce between growth and event work while those prerequisites are unchanged. Schedule growth only when a request can run, including stale requests that need discarding. Wake parked requests after render preparation restores its pair, including failed and cancelled preparation. Preserve the independent growth worker and existing control and recovery wakeups. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit fa771fbd7629800795268b560e9e2ff1b0cb806c) --- drivers/gpu/drm/asahi/g17/runtime/events.rs | 4 +- .../gpu/drm/asahi/g17/runtime/render/grow.rs | 13 ++++++ .../drm/asahi/g17/runtime/render/prepare.rs | 43 +++++++++++++------ 3 files changed, 45 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 6776707fd944c..f249e3520e0dd 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -415,7 +415,9 @@ impl crate::g17::Shared { // records that arrived while it was consuming the same ring. self.queue_events(); } - if firmware.render_grow_pending() && !firmware.recovery.pending() { + let grow_ready = firmware.render_grow_ready(); + drop(state); + if grow_ready { self.queue_grow(); } Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index a9636cb7ceda0..db5820cfb6663 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -72,6 +72,19 @@ impl crate::g17::Firmware { pub(in crate::g17) fn render_grow_pending(&self) -> bool { !self.queues.render.grow.requests.is_empty() } + /// Borrowed owners and control publication each have their own retry wake. + /// A stale request is runnable so the worker can discard it immediately. + pub(in crate::g17) fn render_grow_ready(&self) -> bool { + let render = &self.queues.render; + !render.grow.requests.is_empty() + && !self.recovery.pending() + && !self.render_control_backpressured() + && render.grow.active.is_none() + && render.grow.requests.iter().any(|request| { + request.halt_count != self.recovery.generation() + || !render.grow_owner_busy(*request) + }) + } pub(in crate::g17) fn render_control_backpressured(&self) -> bool { self.queues.render.grow.reply.is_some() || self.queues.render.pools.pending() } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 30ea174f611b1..1aa6cb67a90eb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -299,7 +299,7 @@ impl super::super::Backend { // returns, while the original installed kick owner stays live. drop(prepared); drop(storage); - { + let grow_ready = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if result.is_err() { @@ -315,8 +315,12 @@ impl super::super::Backend { entry.preparing = None; entry.pending_owner = None; entry.pending_owner_closed = false; - } + firmware.render_grow_ready() + }; self.shared.changed.notify_all(); + if grow_ready { + self.shared.queue_grow(); + } result } Build::Rebind { @@ -345,7 +349,7 @@ impl super::super::Backend { Ok(binding) => (Some(binding), None), Err(error) => (None, Some(error)), }; - let result = { + let (result, grow_ready) = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; let result = (|| { @@ -398,7 +402,7 @@ impl super::super::Backend { entry.pending_owner_closed = false; entry.pair = Some(pair); entry.borrowed_dependencies = None; - result + (result, firmware.render_grow_ready()) }; drop(prepared); if result.is_err() { @@ -413,6 +417,9 @@ impl super::super::Backend { } } self.shared.changed.notify_all(); + if grow_ready { + self.shared.queue_grow(); + } result } }; @@ -582,16 +589,24 @@ impl super::super::Backend { } firmware.publish_render(self.owner, slot, packet, deferred) })(); - if published.is_err() && !packet.is_published() { - firmware - .queues - .render - .entry_mut(slot)? - .pair - .as_mut() - .ok_or(EIO)? - .discard_prepared(packet)?; + let result = (|| { + if published.is_err() && !packet.is_published() { + firmware + .queues + .render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)? + .discard_prepared(packet)?; + } + published + })(); + let grow_ready = firmware.render_grow_ready(); + drop(state); + if grow_ready { + self.shared.queue_grow(); } - published + result } } From 609c603add8b827f7b761bc1e666e50cd45d1845 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:36:44 -0400 Subject: [PATCH 0643/2680] drm/asahi: skip frontier scans without a cross-engine prefix Select an empty borrowed frontier when a command has no opposite-engine prefix. The existing count and fill paths then do no predecessor work for that command, while explicit prefixes retain every covered fence in its original order. Keep both same-engine and compute-triggered render pruning unchanged, along with per-command input fences and early-publication eligibility. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 956a8e2fd2ddcb7e9dc596796b79f62b4e017c55) --- drivers/gpu/drm/asahi/g17/queue/mod.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 3c117ce88182a..dd594b2710d3d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -182,14 +182,19 @@ impl Queue { } let other = 1 - index; let prefix = order.wait_through[other]; - let count = self.frontiers[other] + let frontier = if prefix.is_some() { + self.frontiers[other].as_slice() + } else { + &[] + }; + let count = frontier .iter() .filter(|(sequence, _)| Order::contains(prefix, *sequence)) .count(); let mut dependencies = KVec::with_capacity(inputs.len() + count, GFP_KERNEL)?; let mut checked_inputs = KVec::with_capacity(inputs.len(), GFP_KERNEL)?; let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; - for (sequence, fences) in &self.frontiers[other] { + for (sequence, fences) in frontier { if Order::contains(prefix, *sequence) { dependencies.push(fences.ready.clone(), GFP_KERNEL)?; firmware.push(fences.completed.clone(), GFP_KERNEL)?; From 58ae0b1cd228c97240eafd16b051257d571886dc Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:40:54 -0400 Subject: [PATCH 0644/2680] drm/asahi: track CPU feed requests through policy lifetime Register a device-owned policy notifier before a single existing-policy snapshot. Match creation with removal of the exact pinned frequency request and policy reference, so CPUFreq teardown does not wait for GPU removal. Revalidate snapshot policy identity under the request lock after policy initialization, and keep notifier unregistration outside that lock. Preserve feed thresholds, the frequency floor and worker cadence. Handle later policy creation without periodic CPU scans, and leave the detector inactive when CPUFreq is disabled. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5026a187d811c441374d11c71a3fa0df58e118fd) --- drivers/gpu/drm/asahi/g17/feed.rs | 331 ++++++++++++++++++++++-------- 1 file changed, 245 insertions(+), 86 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs index ca3ecc85ee25a..4cd05fde16581 100644 --- a/drivers/gpu/drm/asahi/g17/feed.rs +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -1,34 +1,41 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! Detects sustained host refill gaps and temporarily requests CPU capacity. -//! Detector updates are lock-free. The device's single feed worker owns the -//! frequency requests; its lock never nests inside a driver publication lock. +//! Detector updates are lock-free. The worker and policy callbacks serialize +//! frequency requests without nesting their mutex inside a publication lock. //! Shutdown first closes the detector, joins its producers and the worker, //! then removes requests. All request storage and policy references are owned //! by this device, independently of other GPU instances. +//! Policy creation and the initial snapshot take the policy rwsem before the +//! requests mutex. Removal callbacks and the worker take only requests; +//! notifier unregistration joins callbacks without holding that mutex. use crate::hw::t8140::qos::CLOCK_HZ; use core::{ pin::Pin, - ptr, sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, }; use kernel::{ - bindings, new_mutex, prelude::*, - sync::{Arc, Mutex}, + sync::Arc, time::{delay::fsleep, ClockSource, Delta, Monotonic}, - types::Opaque, }; +#[cfg(CONFIG_CPU_FREQ)] +use core::ptr; +#[cfg(CONFIG_CPU_FREQ)] +use kernel::{bindings, new_mutex, sync::Mutex, types::Opaque}; + const MAX_REFILL_TICKS: u64 = CLOCK_HZ / 500; const STREAK_ON: u32 = 8; const STREAK_MAX: u32 = 64; const LINGER_NS: u64 = 8_000_000; const POLL_MS: i64 = 2; +#[cfg(CONFIG_CPU_FREQ)] const FLOOR_PERCENT: u64 = 50; const WAIT_UTIL_MIN: u32 = 512; const WAIT_HINT_MS: u32 = 8; +#[cfg(CONFIG_CPU_FREQ)] const MAX_POLICIES: usize = 8; struct Detector { @@ -114,6 +121,7 @@ impl Detector { } } +#[cfg(CONFIG_CPU_FREQ)] #[pin_data] struct Requests { #[pin] @@ -121,13 +129,14 @@ struct Requests { policies: [*mut bindings::cpufreq_policy; MAX_POLICIES], maxima: [u32; MAX_POLICIES], applied: [u32; MAX_POLICIES], - count: usize, } -// SAFETY: Each nonnull policy retains cpufreq_cpu_get's reference. Access to -// the pinned request array and metadata is exclusive under Feed::requests. +// SAFETY: Each nonnull policy owns a kobject reference. The device requests +// mutex serializes the pinned slots with the worker and policy callbacks. +#[cfg(CONFIG_CPU_FREQ)] unsafe impl Send for Requests {} +#[cfg(CONFIG_CPU_FREQ)] impl Requests { fn new() -> Self { Self { @@ -135,126 +144,255 @@ impl Requests { policies: [ptr::null_mut(); MAX_POLICIES], maxima: [0; MAX_POLICIES], applied: [0; MAX_POLICIES], - count: 0, } } - fn register(self: Pin<&mut Self>) { + /// The caller holds the policy's read or write lock and keeps it alive. + /// Policy removal must subsequently pass through this device's notifier. + unsafe fn add(self: Pin<&mut Self>, policy: *mut bindings::cpufreq_policy) { let this = self.project(); - if *this.count != 0 { + if this.policies.contains(&policy) { return; } - // SAFETY: nr_cpu_ids is initialized before driver probe. - let cpus = unsafe { bindings::nr_cpu_ids }; - for cpu in 0..cpus { - // SAFETY: Accepts any CPU number; a nonnull result owns one reference. - let policy = unsafe { bindings::cpufreq_cpu_get(cpu) }; - if policy.is_null() { + let Some(index) = this.policies.iter().position(|entry| entry.is_null()) else { + return; + }; + // SAFETY: The caller keeps the initialized policy alive. The vacant + // request slot is pinned and exclusively owned under the requests mutex. + let result = unsafe { + bindings::freq_qos_add_request( + ptr::addr_of_mut!((*policy).constraints), + this.requests + .get() + .cast::() + .add(index), + bindings::freq_qos_req_type_FREQ_QOS_MIN, + bindings::FREQ_QOS_MIN_DEFAULT_VALUE as i32, + ) + }; + if result >= 0 { + // SAFETY: The caller's policy reference or CREATE callback keeps the + // kobject alive. Retain it until REMOVE or device shutdown removes + // this request; cpuinfo is protected by the caller's policy lock. + unsafe { + bindings::kobject_get(ptr::addr_of_mut!((*policy).kobj)); + this.maxima[index] = (*policy).cpuinfo.max_freq; + } + this.policies[index] = policy; + this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; + } + } + + fn apply(self: Pin<&mut Self>, enabled: bool) -> bool { + let this = self.project(); + let mut applied = false; + for index in 0..MAX_POLICIES { + if this.policies[index].is_null() { continue; } - let count = *this.count; - if count < MAX_POLICIES && !this.policies[..count].contains(&policy) { - // SAFETY: The policy is referenced. This zeroed, unregistered - // slot is pinned and remains alive until remove_request. + let value = if enabled { + (u64::from(this.maxima[index]) * FLOOR_PERCENT / 100) as u32 + } else { + bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32 + }; + if this.applied[index] != value { + // SAFETY: This slot is registered, pinned and exclusively locked; + // the policy reference keeps its constraints alive through REMOVE. let result = unsafe { - bindings::freq_qos_add_request( - ptr::addr_of_mut!((*policy).constraints), + bindings::freq_qos_update_request( this.requests .get() .cast::() - .add(count), - bindings::freq_qos_req_type_FREQ_QOS_MIN, - bindings::FREQ_QOS_MIN_DEFAULT_VALUE as i32, + .add(index), + value as i32, ) }; if result >= 0 { - this.policies[count] = policy; - // SAFETY: cpuinfo is immutable while this policy is referenced. - this.maxima[count] = unsafe { (*policy).cpuinfo.max_freq }; - this.applied[count] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; - *this.count += 1; - continue; + this.applied[index] = value; } } - // SAFETY: Balances the reference acquired above for a duplicate, - // unavailable slot or failed registration. Successful slots retain it. - unsafe { bindings::cpufreq_cpu_put(policy) }; + applied |= this.applied[index] != bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; } + applied } - fn apply(mut self: Pin<&mut Self>, enabled: bool) -> bool { - if enabled { - self.as_mut().register(); - } + fn remove(self: Pin<&mut Self>, policy: *mut bindings::cpufreq_policy) { let this = self.project(); - for index in 0..*this.count { - let floor = if enabled { - (u64::from(this.maxima[index]) * FLOOR_PERCENT / 100) as u32 - } else { - 0 - }; - let value = if floor == 0 { - bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32 - } else { - floor - }; - if this.applied[index] == value { - continue; - } - this.applied[index] = value; - // SAFETY: This slot is registered, pinned and exclusively locked; - // its referenced policy keeps the constraints alive. - unsafe { - bindings::freq_qos_update_request( - this.requests - .get() - .cast::() - .add(index), - value as i32, - ) - }; + let Some(index) = this.policies.iter().position(|entry| *entry == policy) else { + return; + }; + // SAFETY: This registered pinned slot is exclusively locked. The policy + // cannot finish removal until this reference is dropped. Keep all other + // request slots at their original addresses when releasing this slot. + unsafe { + bindings::freq_qos_remove_request( + this.requests + .get() + .cast::() + .add(index), + ); + bindings::cpufreq_cpu_put(policy); } - *this.count != 0 && enabled + this.policies[index] = ptr::null_mut(); + this.maxima[index] = 0; + this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; } - fn clear(self: Pin<&mut Self>) { - let this = self.project(); - for index in 0..*this.count { - // SAFETY: Registered pinned slot, exclusively locked. Remove the - // constraint link before releasing its policy reference. - unsafe { - bindings::freq_qos_remove_request( - this.requests - .get() - .cast::() - .add(index), - ); - bindings::cpufreq_cpu_put(this.policies[index]); + fn clear(mut self: Pin<&mut Self>) { + for index in 0..MAX_POLICIES { + let policy = self.policies[index]; + if !policy.is_null() { + self.as_mut().remove(policy); } - this.policies[index] = ptr::null_mut(); } - *this.count = 0; } } +/// Only the blocking notifier core accesses this block after registration. +#[cfg(CONFIG_CPU_FREQ)] +#[pin_data] +struct PolicyNotifier { + #[pin] + block: Opaque, +} + +// SAFETY: The pinned notifier is registered once before publishing Feed and +// unregistered at shutdown. The blocking notifier core serializes its links; +// callbacks use only Feed's atomic state and request mutex. +#[cfg(CONFIG_CPU_FREQ)] +unsafe impl Send for PolicyNotifier {} +// SAFETY: As above; no Rust access mutates the registered block. +#[cfg(CONFIG_CPU_FREQ)] +unsafe impl Sync for PolicyNotifier {} + #[pin_data(PinnedDrop)] pub(crate) struct Feed { detector: Detector, + #[cfg(CONFIG_CPU_FREQ)] #[pin] requests: Mutex, + #[cfg(CONFIG_CPU_FREQ)] + #[pin] + notifier: PolicyNotifier, + #[cfg(CONFIG_CPU_FREQ)] + registered: AtomicBool, applied: AtomicBool, } impl Feed { pub(crate) fn new() -> Result> { - Arc::pin_init( + let feed = Arc::pin_init( pin_init!(Self { detector: Detector::new(), + #[cfg(CONFIG_CPU_FREQ)] requests <- new_mutex!(Requests::new(), "G17 CPU feed"), + #[cfg(CONFIG_CPU_FREQ)] + notifier <- pin_init!(PolicyNotifier { + block: Opaque::new(bindings::notifier_block { + notifier_call: Some(Self::policy_event), + next: ptr::null_mut(), + priority: 0, + }), + }), + #[cfg(CONFIG_CPU_FREQ)] + registered: AtomicBool::new(false), applied: AtomicBool::new(false), }), GFP_KERNEL, - ) + )?; + #[cfg(CONFIG_CPU_FREQ)] + feed.register_policies(); + Ok(feed) + } + + /// Seed existing policies once; later creation/removal is event driven. + #[cfg(CONFIG_CPU_FREQ)] + fn register_policies(&self) { + // SAFETY: Feed is pinned in its Arc and has not been published. The + // notifier block lives until shutdown unregisters and joins callbacks. + let result = unsafe { + bindings::cpufreq_register_notifier( + self.notifier.block.get(), + bindings::CPUFREQ_POLICY_NOTIFIER, + ) + }; + if result < 0 { + // CPUFreq can be disabled at boot independently of this driver. + return; + } + self.registered.store(true, Ordering::Release); + // SAFETY: nr_cpu_ids is initialized before driver probe. + let cpus = unsafe { bindings::nr_cpu_ids }; + for cpu in 0..cpus { + // SAFETY: The bounded CPU index is valid; a nonnull result owns a + // reference even if removal begins before acquiring the policy lock. + let policy = unsafe { bindings::cpufreq_cpu_get(cpu) }; + if policy.is_null() { + continue; + } + // CREATE holds the policy write lock before taking requests. Use + // the same lock order and wait out partial policy initialization. + // SAFETY: The retained reference keeps the policy and rwsem alive. + unsafe { bindings::down_read(ptr::addr_of_mut!((*policy).rwsem)) }; + { + let mut requests = self.requests.lock(); + // Removal unpublishes the CPU pointer before its notifier takes + // requests. Rechecking under this mutex makes either the seed + // skip the policy or the subsequent REMOVE release its slot. + // SAFETY: The CPU index is valid; balance this extra reference below. + let current = unsafe { bindings::cpufreq_cpu_get(cpu) }; + // SAFETY: The policy read lock excludes unfinished CREATE. If + // this policy is still published, REMOVE cannot yet have removed + // the core max request without passing our held requests mutex. + let initialized = + current == policy && unsafe { !(*policy).max_freq_req.qos.is_null() }; + if initialized { + // SAFETY: The reference and read lock keep the initialized + // policy alive; registered REMOVE callbacks cover its lifetime. + unsafe { requests.as_mut().add(policy) }; + } + if !current.is_null() { + // SAFETY: Balances the second lookup, independently of its identity. + unsafe { bindings::cpufreq_cpu_put(current) }; + } + } + // SAFETY: Balance the read lock and first lookup after releasing requests. + unsafe { + bindings::up_read(ptr::addr_of_mut!((*policy).rwsem)); + bindings::cpufreq_cpu_put(policy); + } + } + } + + #[cfg(CONFIG_CPU_FREQ)] + unsafe extern "C" fn policy_event( + block: *mut bindings::notifier_block, + event: c_ulong, + data: *mut core::ffi::c_void, + ) -> core::ffi::c_int { + // SAFETY: This callback is installed only on Feed::notifier.block. Its + // owner remains pinned until unregister joins every callback. + let notifier = + unsafe { kernel::container_of!(Opaque::cast_from(block), PolicyNotifier, block) }; + // SAFETY: The notifier is the embedded, pinned field of this live Feed. + let feed = unsafe { &*kernel::container_of!(notifier, Feed, notifier) }; + let policy = data.cast::(); + let mut requests = feed.requests.lock(); + if event == bindings::CPUFREQ_REMOVE_POLICY as c_ulong { + requests.as_mut().remove(policy); + } else if event == bindings::CPUFREQ_CREATE_POLICY as c_ulong + && !feed.detector.stopping.load(Ordering::Acquire) + { + // SAFETY: CREATE supplies an initialized policy with its write lock + // held and guarantees REMOVE before the core releases its storage. + unsafe { requests.as_mut().add(policy) }; + } else { + return bindings::NOTIFY_DONE as _; + } + let enabled = feed.detector.wanted(Self::now()); + let applied = requests.as_mut().apply(enabled); + feed.applied.store(applied, Ordering::Relaxed); + bindings::NOTIFY_OK as _ } fn now() -> u64 { @@ -262,7 +400,9 @@ impl Feed { } pub(crate) fn note_render_pass(&self, start: u64, end: u64) { - self.detector.note(start, end, Self::now()); + if cfg!(CONFIG_CPU_FREQ) { + self.detector.note(start, end, Self::now()); + } } /// Called after a firmware event pass, before queuing the single feed work item. @@ -270,6 +410,7 @@ impl Feed { self.detector.pending.swap(false, Ordering::AcqRel) } + #[cfg(CONFIG_CPU_FREQ)] fn apply(&self, enabled: bool) { let mut requests = self.requests.lock(); let enabled = enabled && !self.detector.stopping.load(Ordering::Acquire); @@ -277,6 +418,9 @@ impl Feed { self.applied.store(applied, Ordering::Relaxed); } + #[cfg(not(CONFIG_CPU_FREQ))] + fn apply(&self, _enabled: bool) {} + /// Worker context; the linger deadline has no firmware event of its own. pub(crate) fn run(&self) { loop { @@ -317,7 +461,22 @@ impl Feed { /// Call after work items are joined. Removing requests withdraws their floor. pub(crate) fn shutdown(&self) { self.begin_shutdown(); - self.requests.lock().as_mut().clear(); + #[cfg(CONFIG_CPU_FREQ)] + { + // Unregister joins callbacks, which take requests: never hold that + // mutex here. Device shutdown has already joined the feed worker. + if self.registered.swap(false, Ordering::AcqRel) { + // SAFETY: The block was registered successfully and remains + // pinned and alive until this synchronous removal returns. + unsafe { + bindings::cpufreq_unregister_notifier( + self.notifier.block.get(), + bindings::CPUFREQ_POLICY_NOTIFIER, + ); + } + } + self.requests.lock().as_mut().clear(); + } self.applied.store(false, Ordering::Relaxed); } } From d1cce54a213303f891f814934c7d360b8fb8b0f3 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 05:15:19 -0400 Subject: [PATCH 0645/2680] drm/asahi: bound dependency scans to the ordered command prefix Accepted queue sequences are nonzero and increasing, and frontier pruning preserves their order. Locate the end of an explicit cross-engine prefix with partition_point and clone that slice directly. Keep the empty slice for absent prefixes and leave completion pruning, input synchronization and early-publication admission unchanged. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c42fd8c45c3a20ed2ec94c40bf5fbcf782220e7c) --- drivers/gpu/drm/asahi/g17/queue/mod.rs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index dd594b2710d3d..9ae1c7811ac1c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -183,22 +183,21 @@ impl Queue { let other = 1 - index; let prefix = order.wait_through[other]; let frontier = if prefix.is_some() { - self.frontiers[other].as_slice() + let frontier = self.frontiers[other].as_slice(); + // Accepted sequences increase from one, and retain preserves their + // order, so all members of a logical prefix form one slice. + let end = frontier.partition_point(|(sequence, _)| Order::contains(prefix, *sequence)); + &frontier[..end] } else { &[] }; - let count = frontier - .iter() - .filter(|(sequence, _)| Order::contains(prefix, *sequence)) - .count(); + let count = frontier.len(); let mut dependencies = KVec::with_capacity(inputs.len() + count, GFP_KERNEL)?; let mut checked_inputs = KVec::with_capacity(inputs.len(), GFP_KERNEL)?; let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; - for (sequence, fences) in frontier { - if Order::contains(prefix, *sequence) { - dependencies.push(fences.ready.clone(), GFP_KERNEL)?; - firmware.push(fences.completed.clone(), GFP_KERNEL)?; - } + for (_, fences) in frontier { + dependencies.push(fences.ready.clone(), GFP_KERNEL)?; + firmware.push(fences.completed.clone(), GFP_KERNEL)?; } // Each command owns every input fence; independent engines may publish in // either order and must each consume an input's error status before running. From bc901aab0dd3d5c00f79370ae4894206780c006d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 4 Oct 2026 02:13:13 -0400 Subject: [PATCH 0646/2680] rust: check generated layouts and provide corrected private bindgen Keep compile-time size, alignment and field-offset assertions in generated bindings so incompatible layouts fail during the ordinary bindings build. Provide a pinned private generator recipe that preserves alignment on defined zero-sized structs and actual anonymous record members. Include explicit-target C and Rust compiler checks and generator provenance without changing C storage, scheduler configuration or the system toolchain. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 1f7383d5a2811b99a9f2950581ec392837bfc469) --- Documentation/rust/bindgen-layout-checks.rst | 49 +++++++++++ Documentation/rust/index.rst | 1 + rust/Makefile | 5 +- tools/aurora-bindgen/.gitattributes | 1 + tools/aurora-bindgen/README.rst | 72 ++++++++++++++++ tools/aurora-bindgen/build.py | 22 +++++ .../patches/0001-empty-alignment.patch | 20 +++++ .../patches/0002-anonymous-tag-members.patch | 85 +++++++++++++++++++ tools/aurora-bindgen/sources.json | 19 +++++ tools/aurora-bindgen/test.py | 44 ++++++++++ tools/aurora-bindgen/tests/aligned-empty.h | 13 +++ tools/aurora-bindgen/tests/aligned-empty.hpp | 13 +++ .../aurora-bindgen/tests/anonymous-members.h | 9 ++ tools/aurora-bindgen/tests/tag-only.h | 4 + 14 files changed, 356 insertions(+), 1 deletion(-) create mode 100644 Documentation/rust/bindgen-layout-checks.rst create mode 100644 tools/aurora-bindgen/.gitattributes create mode 100644 tools/aurora-bindgen/README.rst create mode 100644 tools/aurora-bindgen/build.py create mode 100644 tools/aurora-bindgen/patches/0001-empty-alignment.patch create mode 100644 tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch create mode 100644 tools/aurora-bindgen/sources.json create mode 100644 tools/aurora-bindgen/test.py create mode 100644 tools/aurora-bindgen/tests/aligned-empty.h create mode 100644 tools/aurora-bindgen/tests/aligned-empty.hpp create mode 100644 tools/aurora-bindgen/tests/anonymous-members.h create mode 100644 tools/aurora-bindgen/tests/tag-only.h diff --git a/Documentation/rust/bindgen-layout-checks.rst b/Documentation/rust/bindgen-layout-checks.rst new file mode 100644 index 0000000000000..8ad8c24526b67 --- /dev/null +++ b/Documentation/rust/bindgen-layout-checks.rst @@ -0,0 +1,49 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Checking generated binding layouts +================================== + +Kernel bindings retain bindgen's layout checks. With the configured Rust target, +these are unconditional constant assertions, checked while the bindings crate +is compiled. They compare Rust sizes, alignments and field offsets against the +values obtained from Clang, without executing kernel code. + +Do not disable these checks to make an incompatible binding generator compile. +A type can have the right size and still have incorrect member offsets. For +example, an empty GNU C struct may have size zero and alignment 64; losing that +alignment moves later fields in an enclosing struct without necessarily +changing the enclosing size. + +Known generator requirements +---------------------------- + +Unmodified bindgen 0.73.2 loses explicit alignment on defined zero-sized C +structs. It also omits the named anonymous record member accepted by Clang's +``-fms-extensions`` in a declaration such as:: + + struct payload { unsigned long words[20]; }; + union container { + struct payload; + unsigned long other; + }; + +The generator used for this tree must preserve these layouts. Use a corrected +bindgen build through the normal ``BINDGEN=/path/to/bindgen`` make variable. +``tools/aurora-bindgen/`` contains pinned patches, a private build recipe and +compile-only valid-input tests. It never installs a system-wide tool. The +associated build provenance must retain the generator source revision, +patches, binary digest, and valid-input C/Rust layout checks. A version string +alone does not identify a locally corrected tool. Do not add dummy kernel +storage, hardcode task offsets, or change scheduler configuration to conceal a +binding generator error. + +After a binding layout correction, rebuild the kernel Image and all matching +modules. The generated bindings are consumed by the Rust kernel crate and by +multiple drivers; rebuilding just one module can leave incompatible layouts in +the same kernel. Keep the old Image and its own modules together as a fallback. + +The minimum accepted bindgen version is 0.71.1. Its Rust 1.77 ``offset_of`` +feature enables unconditional const layout assertions; the configured Rust +target 1.85 selects that path rather than ``#[test]``-only functions. The pinned +0.73.2 tool has the same behavior. See the source links and explicit target +accounting in ``tools/aurora-bindgen/README.rst``. diff --git a/Documentation/rust/index.rst b/Documentation/rust/index.rst index b78ed0efa7846..c03031b241086 100644 --- a/Documentation/rust/index.rst +++ b/Documentation/rust/index.rst @@ -37,6 +37,7 @@ more details. coding-guidelines arch-support testing + bindgen-layout-checks You can also find learning materials for Rust in its section in :doc:`../process/kernel-docs`. diff --git a/rust/Makefile b/rust/Makefile index 9211d82ac96da..6af6d6e58aa9d 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -502,10 +502,13 @@ endif bindgen_c_flags_final = $(bindgen_c_flags_lto) -fno-builtin -D__BINDGEN__ # `--rust-target` points to our minimum supported Rust version. +# Keep bindgen layout assertions enabled: a lost alignment can change field +# offsets without changing the containing type size. See Documentation/rust/ +# bindgen-layout-checks.rst for generator requirements. quiet_cmd_bindgen = BINDGEN $@ cmd_bindgen = \ $(BINDGEN) $< $(bindgen_target_flags) --rust-target 1.85 \ - --use-core --with-derive-default --ctypes-prefix ffi --no-layout-tests \ + --use-core --with-derive-default --ctypes-prefix ffi \ --no-debug '.*' --enable-function-attribute-detection \ -o $@ -- $(bindgen_c_flags_final) -DMODULE \ $(bindgen_target_cflags) $(bindgen_target_extra) diff --git a/tools/aurora-bindgen/.gitattributes b/tools/aurora-bindgen/.gitattributes new file mode 100644 index 0000000000000..813c74962d8c7 --- /dev/null +++ b/tools/aurora-bindgen/.gitattributes @@ -0,0 +1 @@ +patches/*.patch whitespace=-blank-at-eol diff --git a/tools/aurora-bindgen/README.rst b/tools/aurora-bindgen/README.rst new file mode 100644 index 0000000000000..bd05375d92ddd --- /dev/null +++ b/tools/aurora-bindgen/README.rst @@ -0,0 +1,72 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Private bindgen with checked C/Rust layouts +========================================== + +This package builds bindgen 0.73.2 with two generic binding-generation repairs: +preserve explicit alignment on defined zero-sized structs, and retain actual +unnamed named-tag fields exposed by Clang's type-field visitor. The latter is +needed for the Microsoft anonymous-member extension used by kernel namespaces. +Neither repair changes C structure storage or kernel configuration. + +``sources.json`` pins the official source archive, Cargo.lock, both patches and +the patched source files. It also records the qualified private tool's digest; +that digest is evidence for that build, not a promise that a different host, +compiler or build path produces identical bytes. + +Build in a fresh private directory:: + + python3 tools/aurora-bindgen/build.py --output-dir /absolute/new/tool-build + +Python 3.12 or newer, Cargo/Rust, Clang/libclang and patch are required. Cargo's +locked dependency graph is used, with at most two build jobs. ``--archive`` can +supply an already downloaded source archive; its digest is still checked. The +script builds and runs compile-only valid-input checks, then prints the exact +``BINDGEN=/absolute/path/to/bindgen`` make argument. It installs nothing globally. +Use that argument for the complete kernel Image and matching modules build. + +Tests and target accounting +--------------------------- + +``test.py`` independently compiles the C/C++ fixtures and generated Rust. It +covers empty alignments 1/2/8/64/128, nesting, packed empty records, anonymous +struct/union members, field access and ordinary tag-only declarations. It +requires unconditional const layout assertions in the generated output. +Nothing is executed on a GPU, and the generated object files are not run. + +Example using an existing private tool and the installed unpatched baseline:: + + python3 tools/aurora-bindgen/test.py \ + --bindgen /absolute/fixed/bindgen --baseline-bindgen /usr/bin/bindgen \ + --output-dir /absolute/new/check-results + +The optional baseline must be an affected unpatched tool: alignment/member +fixtures must fail at compile time, while tag-only controls must pass unchanged. +Without that option only the repaired generator is checked. The Rust target is +always passed explicitly, defaulting to the Rust compiler's reported host; the +Clang target defaults to the same triple. ``--rust-target`` and ``--clang-target`` +allow explicit kernel/cross targets, with matching architecture required. +``--rust-sysroot`` and ``--rust-libdir`` support a separately prepared Rust core. +An unavailable Rust target/core emits a machine-readable SKIP and exit 77; +that target has not passed. Other compiler or layout errors fail normally. + +Repository guard and supported tool versions +-------------------------------------------- + +The kernel keeps bindgen layout checks enabled. Its minimum bindgen is 0.71.1, +and its fixed ``--rust-target 1.85`` enables the ``offset_of`` feature (available +from Rust 1.77). In both official 0.71.1 and the pinned 0.73.2 generator, that +feature selects unconditional const size/alignment/offset assertions rather +than ``#[test]`` functions. A normal bindings crate compile therefore checks +layouts without requiring a Rust unit-test configuration. This is source-backed +for the accepted minimum and pinned tool; future generator changes must retain +that capability. Never remove the assertions to accept an incompatible tool. + +Primary sources: + +* https://github.com/rust-lang/rust-bindgen/blob/v0.71.1/bindgen/features.rs +* https://github.com/rust-lang/rust-bindgen/blob/v0.71.1/bindgen/codegen/mod.rs +* https://github.com/rust-lang/rust-bindgen/tree/v0.73.2 + +The complete kernel build remains the configuration-specific layout gate; these +small generator fixtures do not replace it or establish runtime GPU stability. diff --git a/tools/aurora-bindgen/build.py b/tools/aurora-bindgen/build.py new file mode 100644 index 0000000000000..cf3afbc4a53a5 --- /dev/null +++ b/tools/aurora-bindgen/build.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Fetch pinned source, apply generic fixes, and build/test a private bindgen.""" +import argparse,hashlib,json,subprocess,tarfile,urllib.request,shutil,os,sys +from pathlib import Path +P=Path(__file__).resolve().parent;ap=argparse.ArgumentParser();ap.add_argument('--output-dir',required=True,type=Path);ap.add_argument('--archive',type=Path);ap.add_argument('--jobs',type=int,default=2);a=ap.parse_args();assert 1<=a.jobs<=2 +O=a.output_dir.resolve();assert not O.exists(),'refusing to overwrite an existing source or tool';O.mkdir(parents=True) +m=json.loads((P/'sources.json').read_text());sha=lambda p:hashlib.sha256(p.read_bytes()).hexdigest();archive=O/'source.tar.gz' +if a.archive:shutil.copyfile(a.archive,archive) +else: + with urllib.request.urlopen(m['source_url'],timeout=60) as response, archive.open('wb') as f:shutil.copyfileobj(response,f) +assert sha(archive)==m['source_sha256'],'source digest mismatch' +with tarfile.open(archive) as t:t.extractall(O,filter='data') +S=O/m['source_directory'] +for name,h in m['patches'].items(): + p=P/'patches'/name;assert sha(p)==h,name;subprocess.run(['patch','--batch','--fuzz=0','-p1','-i',str(p)],cwd=S,check=True) +assert sha(S/'Cargo.lock')==m['cargo_lock_sha256'] +for f,h in m['patched_files'].items():assert sha(S/f)==h,f +cmd=['cargo','build','--locked','--release','-p','bindgen-cli','-j'+str(a.jobs)] +with (O/'build.log').open('w') as log:subprocess.run(cmd,cwd=S,stdout=log,stderr=subprocess.STDOUT,check=True) +binary=S/'target/release/bindgen';subprocess.run([sys.executable,str(P/'test.py'),'--bindgen',str(binary),'--output-dir',str(O/'tests')],check=True) +r={'status':'BUILT_AND_COMPILE_TESTED','binary':str(binary),'binary_sha256':sha(binary),'source_manifest_sha256':sha(P/'sources.json'),'build_command':cmd,'rustc_version':subprocess.check_output(['rustc','-vV'],text=True),'cargo_version':subprocess.check_output(['cargo','-vV'],text=True),'tests_manifest_sha256':sha(O/'tests/result.json'),'installed_globally':False};(O/'result.json').write_text(json.dumps(r,indent=2)+'\n');print(json.dumps(r,indent=2));print('Use make BINDGEN='+str(binary)+' ...; keep full kernel layout assertions enabled.') diff --git a/tools/aurora-bindgen/patches/0001-empty-alignment.patch b/tools/aurora-bindgen/patches/0001-empty-alignment.patch new file mode 100644 index 0000000000000..f595b480c6b8d --- /dev/null +++ b/tools/aurora-bindgen/patches/0001-empty-alignment.patch @@ -0,0 +1,20 @@ +--- a/bindgen/codegen/mod.rs ++++ b/bindgen/codegen/mod.rs +@@ -2468,6 +2468,17 @@ + } + } + } ++ } else if !is_union && !forward_decl { ++ // A zero-sized C struct can still require explicit alignment. ++ // Its alignment also determines the offsets of subsequent fields ++ // when it is embedded in another type. Do not add storage here. ++ if let Some(layout) = layout { ++ if struct_layout.requires_explicit_align(layout) && ++ layout.align > 1 ++ { ++ explicit_align = Some(layout.align); ++ } ++ } + } else if is_union && !forward_decl { + if let Some(layout) = layout { + // TODO(emilio): It'd be nice to unify this with the struct path above somehow. diff --git a/tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch b/tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch new file mode 100644 index 0000000000000..395b537796a45 --- /dev/null +++ b/tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch @@ -0,0 +1,85 @@ +--- a/bindgen/clang.rs ++++ b/bindgen/clang.rs +@@ -1204,6 +1204,29 @@ + } + + impl Type { ++ /// Get actual record fields, including implicit anonymous fields omitted ++ /// from `clang_visitChildren` for Microsoft anonymous tag members. ++ pub(crate) fn fields(&self) -> Vec { ++ extern "C" fn collect_field( ++ cur: CXCursor, ++ data: CXClientData, ++ ) -> CXVisitorResult { ++ let fields = unsafe { &mut *data.cast::>() }; ++ fields.push(Cursor { x: cur }); ++ CXVisit_Continue ++ } ++ ++ let mut fields = Vec::::new(); ++ unsafe { ++ clang_Type_visitFields( ++ self.x, ++ collect_field, ++ (&mut fields as *mut Vec).cast(), ++ ); ++ } ++ fields ++ } ++ + /// Get this type's kind. + pub(crate) fn kind(&self) -> CXTypeKind { + self.x.kind +--- a/bindgen/ir/comp.rs ++++ b/bindgen/ir/comp.rs +@@ -1340,6 +1340,16 @@ + _ => false, + }); + ++ // Microsoft anonymous named-tag members have real implicit FieldDecls ++ // in the type's field list, but only a non-definition record cursor in ++ // child traversal. Keep positive field evidence: a tag declaration by ++ // itself is not necessarily storage in the containing record. ++ let mut implicit_fields: Vec<_> = ty ++ .fields() ++ .into_iter() ++ .filter(|field| field.spelling().is_empty() && !field.is_bit_field()) ++ .collect(); ++ + let mut maybe_anonymous_struct_field = None; + cursor.visit(|cur| { + if cur.kind() != CXCursor_FieldDecl { +@@ -1463,6 +1473,33 @@ + // definition, it's a valid inner type. + // + // [1]: https://github.com/rust-lang/rust-bindgen/issues/482 ++ if matches!(cur.kind(), CXCursor_StructDecl | CXCursor_UnionDecl) && ++ !cur.is_definition() ++ { ++ if let Some(index) = implicit_fields.iter().position(|field| { ++ field.cur_type().canonical_type() == ++ cur.cur_type().canonical_type() ++ }) { ++ let field = implicit_fields.remove(index); ++ let field_type = Item::from_ty_or_ref( ++ field.cur_type(), ++ field, ++ Some(potential_id), ++ ctx, ++ ); ++ ci.fields.append_raw_field(RawField::new( ++ None, ++ field_type, ++ field.raw_comment(), ++ Annotations::new(&field), ++ None, ++ field.public_accessible(), ++ field.offset_of_field().ok(), ++ )); ++ return CXChildVisit_Continue; ++ } ++ } ++ + let is_inner_struct = + cur.semantic_parent() == cursor || cur.is_definition(); + if !is_inner_struct { diff --git a/tools/aurora-bindgen/sources.json b/tools/aurora-bindgen/sources.json new file mode 100644 index 0000000000000..ca75ddc018d28 --- /dev/null +++ b/tools/aurora-bindgen/sources.json @@ -0,0 +1,19 @@ +{ + "version": "0.73.2", + "source_url": "https://codeload.github.com/rust-lang/rust-bindgen/tar.gz/refs/tags/v0.73.2", + "source_sha256": "5fc3277dd334e4bbbd7bdb8cad5d4fb87a667b2e95a2bae350a8d5424c280629", + "source_directory": "rust-bindgen-0.73.2", + "cargo_lock_sha256": "85edb9aab2d5649f39b805dc52e3f27ebf8af4d94b049601aee1c55fcf450393", + "patches": { + "0001-empty-alignment.patch": "eb71efca5c60adc9db6837c933ff377e8aefad53655057406ff9f1464dd3147f", + "0002-anonymous-tag-members.patch": "9072ec2fd9d0f0160615cbfd0d9cfe16d23cd0ae90747a2aa984b8694bdee9ec" + }, + "patched_files": { + "bindgen/codegen/mod.rs": "4c100fa631bdd655b509e1797411e341f9a954207d64f8f2efc285201f5f14d9", + "bindgen/clang.rs": "bd65a80b028045dec2ff1ca9ee347301726b30f94a0ee743a3041d4f598e45e4", + "bindgen/ir/comp.rs": "7d53770db84376699421c8057fbecbfc00e8a6f62ca38919bd4c718e56e40438" + }, + "qualified_private_tool_sha256": "dc794acc7b0d7e3cbfa1533b4b59f2518cd0cc1bff1dbb7fd104229c23c0d146", + "qualified_host_toolchain": "rustc/cargo1.98.1, LLVM/libclang22.1.8, nativeAArch64; kernel Rust1.85.0", + "runtime_tests_performed": false +} diff --git a/tools/aurora-bindgen/test.py b/tools/aurora-bindgen/test.py new file mode 100644 index 0000000000000..91b6d18d7fd4a --- /dev/null +++ b/tools/aurora-bindgen/test.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Compile valid generated bindings for explicitly matching C/Rust targets.""" +import argparse,hashlib,json,re,subprocess,sys +from pathlib import Path +P=Path(__file__).resolve().parent +ap=argparse.ArgumentParser();ap.add_argument('--bindgen',required=True);ap.add_argument('--output-dir',required=True,type=Path);ap.add_argument('--rustc',default='rustc');ap.add_argument('--clang',default='clang');ap.add_argument('--rust-target');ap.add_argument('--clang-target');ap.add_argument('--rust-sysroot');ap.add_argument('--rust-libdir');ap.add_argument('--baseline-bindgen');a=ap.parse_args() +O=a.output_dir.resolve();assert not O.exists(),'use a fresh evidence directory';O.mkdir(parents=True) +def run(cmd):return subprocess.run(cmd,capture_output=True,text=True) +host=next(x.split(': ',1)[1] for x in subprocess.check_output([a.rustc,'-vV'],text=True).splitlines() if x.startswith('host: '));rt=a.rust_target or host;ct=a.clang_target or rt +arch=lambda t:{'arm64':'aarch64','amd64':'x86_64'}.get(t.split('-')[0],t.split('-')[0]) +assert arch(rt)==arch(ct),('C/Rust architecture mismatch',ct,rt) +rust=[a.rustc,'--edition=2021','--crate-type=rlib','--crate-name','layout_check','--emit=obj','-Cpanic=abort','--target='+rt] +if a.rust_sysroot:rust+=['--sysroot='+a.rust_sysroot] +if a.rust_libdir:rust+=['-L',str(Path(a.rust_libdir).resolve())] +probe=O/'target.rs';probe.write_text('#![no_std]\npub fn available() -> usize { core::mem::size_of::() }\n');r=run(rust+[str(probe),'-o',str(O/'target.o')]);(O/'target.log').write_text(r.stdout+r.stderr) +if r.returncode: + status='SKIP' if 'E0463' in r.stderr and 'core' in r.stderr else 'FAIL' + result={'status':status,'reason':'matching Rust target/core unavailable' if status=='SKIP' else 'target preflight compile failed','clang_target':ct,'rust_target':rt,'rust_argv':rust,'log':'target.log'};(O/'result.json').write_text(json.dumps(result,indent=2)+'\n');print(json.dumps(result));sys.exit(77 if status=='SKIP' else 1) +access=''' +unsafe fn check_fields(a: &mut outer, b: &mut wrapper, c: &mut union_member, d: &mut trailing_member) { + a.__bindgen_anon_1.__bindgen_anon_1.words[19] = 1; + b.__bindgen_anon_1.words[19] = 2; + c.__bindgen_anon_1.bytes[23] = 3; + d.__bindgen_anon_1.words[0] = 4; +} +''' +cases=[];equal={};toolset=[('fixed',a.bindgen)]+([('baseline',a.baseline_bindgen)] if a.baseline_bindgen else []) +for fixture,language,extra in [('aligned-empty.h','c',[]),('aligned-empty.hpp','c++',[]),('anonymous-members.h','c',['-fms-extensions']),('tag-only.h','c',[])]: + header=P/'tests'/fixture + # Also ask the C compiler to accept the ordinary fixture independently. + ccmd=[a.clang,'--target='+ct,'-x',language,*extra,'-Wno-microsoft-anon-tag','-c',str(header),'-o',str(O/(fixture+'.c.o'))];cr=run(ccmd);(O/(fixture+'.c.log')).write_text(cr.stdout+cr.stderr);assert cr.returncode==0,(fixture,cr.stderr) + for label,tool in toolset: + cmd=[tool,str(header),'--rust-target','1.85','--no-doc-comments','--use-core','--no-derive-debug','--ctypes-prefix','core::ffi','--','-target',ct,'-x',language,*extra] + g=run(cmd);assert g.returncode==0,(fixture,g.stderr) + assert 'const _: ()' in g.stdout and '#[test]' not in g.stdout,'generator did not emit unconditional const layout checks' + rs=O/(fixture+'.'+label+'.rs');rs.write_text('#![no_std]\n'+g.stdout+(access if fixture=='anonymous-members.h' and label=='fixed' else '')) + rcmd=rust+[str(rs),'-o',str(O/(fixture+'.'+label+'.o'))];r=run(rcmd);(O/(fixture+'.'+label+'.log')).write_text(r.stdout+r.stderr) + expected=label=='fixed' or fixture=='tag-only.h';assert (r.returncode==0)==expected,(fixture,label,r.stderr) + if not expected:assert 'E0080' in r.stderr,'baseline failure must be a const layout mismatch' + cases.append({'fixture':fixture,'tool':label,'clang_argv':ccmd,'generator_argv':cmd,'rust_argv':rcmd,'compile_exit':r.returncode,'expected':'layout compile success' if expected else 'const layout rejection','generated_sha256':hashlib.sha256(rs.read_bytes()).hexdigest()}) + if fixture=='tag-only.h':equal[label]=g.stdout +if a.baseline_bindgen:assert equal['baseline']==equal['fixed'],'ordinary tag-only binding changed' +result={'status':'PASS','clang_target':ct,'rust_target':rt,'rust_host':host,'explicit_rust_target':True,'cases':cases,'baseline_checked':bool(a.baseline_bindgen),'hardware_executed':False};(O/'result.json').write_text(json.dumps(result,indent=2)+'\n');print('PASS matching C/Rust targets:',ct,rt) diff --git a/tools/aurora-bindgen/tests/aligned-empty.h b/tools/aurora-bindgen/tests/aligned-empty.h new file mode 100644 index 0000000000000..9ac1f0546a623 --- /dev/null +++ b/tools/aurora-bindgen/tests/aligned-empty.h @@ -0,0 +1,13 @@ +struct incomplete; +struct empty1 {} __attribute__((aligned(1))); +struct outer1 { unsigned char prefix[12]; struct empty1 value; unsigned long after; }; +struct empty2 {} __attribute__((aligned(2))); +struct outer2 { unsigned char prefix[12]; struct empty2 value; unsigned long after; }; +struct empty8 {} __attribute__((aligned(8))); +struct outer8 { unsigned char prefix[12]; struct empty8 value; unsigned long after; }; +struct empty64 {} __attribute__((aligned(64))); +struct outer64 { unsigned char prefix[12]; struct empty64 value; unsigned long after; }; +struct empty128 {} __attribute__((aligned(128))); +struct outer128 { unsigned char prefix[12]; struct empty128 value; unsigned long after; }; +struct packed_empty {} __attribute__((packed,aligned(64))); +struct packed_container { char prefix; struct packed_empty value; unsigned long after; }; diff --git a/tools/aurora-bindgen/tests/aligned-empty.hpp b/tools/aurora-bindgen/tests/aligned-empty.hpp new file mode 100644 index 0000000000000..9ac1f0546a623 --- /dev/null +++ b/tools/aurora-bindgen/tests/aligned-empty.hpp @@ -0,0 +1,13 @@ +struct incomplete; +struct empty1 {} __attribute__((aligned(1))); +struct outer1 { unsigned char prefix[12]; struct empty1 value; unsigned long after; }; +struct empty2 {} __attribute__((aligned(2))); +struct outer2 { unsigned char prefix[12]; struct empty2 value; unsigned long after; }; +struct empty8 {} __attribute__((aligned(8))); +struct outer8 { unsigned char prefix[12]; struct empty8 value; unsigned long after; }; +struct empty64 {} __attribute__((aligned(64))); +struct outer64 { unsigned char prefix[12]; struct empty64 value; unsigned long after; }; +struct empty128 {} __attribute__((aligned(128))); +struct outer128 { unsigned char prefix[12]; struct empty128 value; unsigned long after; }; +struct packed_empty {} __attribute__((packed,aligned(64))); +struct packed_container { char prefix; struct packed_empty value; unsigned long after; }; diff --git a/tools/aurora-bindgen/tests/anonymous-members.h b/tools/aurora-bindgen/tests/anonymous-members.h new file mode 100644 index 0000000000000..56b129a6aab14 --- /dev/null +++ b/tools/aurora-bindgen/tests/anonymous-members.h @@ -0,0 +1,9 @@ +struct payload { unsigned long words[20]; }; +union tag_union { unsigned long word; unsigned char bytes[24]; }; +struct outer { unsigned before; union { struct payload; void *ptr; }; unsigned after; }; +struct wrapper { unsigned prefix; struct payload; unsigned suffix; }; +struct union_member { char prefix; union tag_union; unsigned suffix; }; +struct normal_named { char prefix; struct payload named; unsigned suffix; }; +struct anonymous_definition { char prefix; struct { unsigned long inner; }; unsigned suffix; }; +struct forward_only { struct never_defined *ptr; unsigned suffix; }; +struct trailing_member { unsigned prefix; struct payload; }; diff --git a/tools/aurora-bindgen/tests/tag-only.h b/tools/aurora-bindgen/tests/tag-only.h new file mode 100644 index 0000000000000..525bc24774be0 --- /dev/null +++ b/tools/aurora-bindgen/tests/tag-only.h @@ -0,0 +1,4 @@ +struct payload { unsigned long words[20]; }; +struct harmless_tag { unsigned before; struct payload; unsigned after; }; +struct ordinary_forward { struct forward_tag; void *ptr; unsigned after; }; +struct normal_named { unsigned before; struct payload named; unsigned after; }; From 888a6334ee10723c76732cc73e73b263231dcf41 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 4 Oct 2026 02:19:36 -0400 Subject: [PATCH 0647/2680] drm/asahi: wait for host timestamp writes in compute prefixes Explicit compute prefixes must cover timestamp copies performed by the host after GPU retirement. Add completion dependencies and retained status checks for timestamp publishers while preserving firmware ordering for ordinary compute work. Keep the earliest failed compute timestamp as a bounded error witness so cancellation or recovery-spared failures cannot become valid query results after pruning. Reuse ordered prefix lookup and retain only independent fences, without extending packet or mapping lifetimes. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e4d41aaa9c0cdd145d5d07fe0b3d0eb209bf0879) --- drivers/gpu/drm/asahi/g17/queue/mod.rs | 48 +++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 9ae1c7811ac1c..dcd6f8696f3a2 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -21,7 +21,7 @@ use crate::{ use core::ops::Range; use kernel::{ c_str, - dma_fence::FenceContexts, + dma_fence::{FenceContexts, RawDmaFence}, prelude::*, sync::{Arc, LockClassKey}, xarray, @@ -147,7 +147,24 @@ impl Queue { Validated::Compute { .. } => Engine::Compute, }; let index = engine.index(); - self.frontiers[index].retain(|(_, fences)| job::fence_status(&fences.completed) == 0); + // One earliest failed timestamp covers every prefix containing a later + // failure, without retaining the producer's packet or mappings. + let mut retained_timestamp_failure = false; + self.frontiers[index].retain(|(_, fences)| { + let status = job::fence_status(&fences.completed); + if status == 0 { + return true; + } + if engine == Engine::Compute + && status < 0 + && fences.ready.raw() == fences.completed.raw() + && !retained_timestamp_failure + { + retained_timestamp_failure = true; + return true; + } + false + }); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { self.ensure_compute()?; @@ -192,9 +209,32 @@ impl Queue { &[] }; let count = frontier.len(); - let mut dependencies = KVec::with_capacity(inputs.len() + count, GFP_KERNEL)?; - let mut checked_inputs = KVec::with_capacity(inputs.len(), GFP_KERNEL)?; + let timestamp_prefix = order.wait_through[index]; + let timestamps = if engine == Engine::Compute && timestamp_prefix.is_some() { + let frontier = self.frontiers[index].as_slice(); + let end = frontier + .partition_point(|(sequence, _)| Order::contains(timestamp_prefix, *sequence)); + &frontier[..end] + } else { + &[] + }; + let timestamp_count = timestamps + .iter() + .filter(|(_, fences)| fences.ready.raw() == fences.completed.raw()) + .count(); + let mut dependencies = + KVec::with_capacity(inputs.len() + count + timestamp_count, GFP_KERNEL)?; + let mut checked_inputs = KVec::with_capacity(inputs.len() + timestamp_count, GFP_KERNEL)?; let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; + // Same-engine parents order GPU execution, not the host's timestamp + // writes. Only timestamp publishers need these completion waits and + // status checks; ordinary compute keeps its existing queue ordering. + for (_, fences) in timestamps { + if fences.ready.raw() == fences.completed.raw() { + checked_inputs.push(fences.completed.clone(), GFP_KERNEL)?; + dependencies.push(fences.ready.clone(), GFP_KERNEL)?; + } + } for (_, fences) in frontier { dependencies.push(fences.ready.clone(), GFP_KERNEL)?; firmware.push(fences.completed.clone(), GFP_KERNEL)?; From b88ec75ff4b7d2532b14257aa629f7c42106a2f2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 4 Oct 2026 02:20:09 -0400 Subject: [PATCH 0648/2680] tools: preserve bindgen recipe validation across Python modes Reject optimized Python before creating output so archive and layout assertions cannot be disabled. Exercise bitfield generation with unknown Rust lints denied, and document explicit compiler selection separately from the generator language target. Document the synchronous Clang visitor safety requirements and refresh the affected pinned patch and source digests. Retain the previously qualified binary digest as historical build evidence without claiming a rebuilt binary is identical. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit bc7394673cbed4ef22971a49926be6070a7312be) --- tools/aurora-bindgen/README.rst | 14 ++++++++++++++ tools/aurora-bindgen/build.py | 3 +++ .../patches/0002-anonymous-tag-members.patch | 7 ++++++- tools/aurora-bindgen/sources.json | 7 ++++--- tools/aurora-bindgen/test.py | 10 ++++++++-- tools/aurora-bindgen/tests/bitfields.h | 11 +++++++++++ 6 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 tools/aurora-bindgen/tests/bitfields.h diff --git a/tools/aurora-bindgen/README.rst b/tools/aurora-bindgen/README.rst index bd05375d92ddd..2c560aa13f96e 100644 --- a/tools/aurora-bindgen/README.rst +++ b/tools/aurora-bindgen/README.rst @@ -13,6 +13,8 @@ Neither repair changes C structure storage or kernel configuration. the patched source files. It also records the qualified private tool's digest; that digest is evidence for that build, not a promise that a different host, compiler or build path produces identical bytes. +The manifest identifies when that digest predates comment-only patch changes; +each rebuild must record its own binary digest. Build in a fresh private directory:: @@ -24,6 +26,8 @@ supply an already downloaded source archive; its digest is still checked. The script builds and runs compile-only valid-input checks, then prints the exact ``BINDGEN=/absolute/path/to/bindgen`` make argument. It installs nothing globally. Use that argument for the complete kernel Image and matching modules build. +Both scripts reject optimized Python (``-O``, ``-OO`` or ``PYTHONOPTIMIZE``) +before creating output, since their validation assertions must remain enabled. Tests and target accounting --------------------------- @@ -32,6 +36,9 @@ Tests and target accounting covers empty alignments 1/2/8/64/128, nesting, packed empty records, anonymous struct/union members, field access and ordinary tag-only declarations. It requires unconditional const layout assertions in the generated output. +An additional bitfield fixture rejects unknown Rust lints in the fixed tool's +output. This fixture checks compiler compatibility, not the layout repairs, +and is excluded from the optional unpatched-baseline comparison. Nothing is executed on a GPU, and the generated object files are not run. Example using an existing private tool and the installed unpatched baseline:: @@ -50,6 +57,13 @@ allow explicit kernel/cross targets, with matching architecture required. An unavailable Rust target/core emits a machine-readable SKIP and exit 77; that target has not passed. Other compiler or layout errors fail normally. +Select ``--rustc`` explicitly to check the compiler used for the kernel. The +generator's ``--rust-target 1.85`` selects emitted language features; it does +not select the Rust compiler used by the tests. A compatibility adapter may +remove an exact generated lint attribute unsupported by that compiler, while +retaining all layout assertions. Its underlying binary must be the corrected +generator, and its changes and both tool digests belong in the build record. + Repository guard and supported tool versions -------------------------------------------- diff --git a/tools/aurora-bindgen/build.py b/tools/aurora-bindgen/build.py index cf3afbc4a53a5..5895a84a7b5c4 100644 --- a/tools/aurora-bindgen/build.py +++ b/tools/aurora-bindgen/build.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only """Fetch pinned source, apply generic fixes, and build/test a private bindgen.""" +if not __debug__: + raise SystemExit("optimized Python is unsupported: validation assertions must remain enabled") + import argparse,hashlib,json,subprocess,tarfile,urllib.request,shutil,os,sys from pathlib import Path P=Path(__file__).resolve().parent;ap=argparse.ArgumentParser();ap.add_argument('--output-dir',required=True,type=Path);ap.add_argument('--archive',type=Path);ap.add_argument('--jobs',type=int,default=2);a=ap.parse_args();assert 1<=a.jobs<=2 diff --git a/tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch b/tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch index 395b537796a45..7881d88237235 100644 --- a/tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch +++ b/tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch @@ -1,6 +1,6 @@ --- a/bindgen/clang.rs +++ b/bindgen/clang.rs -@@ -1204,6 +1204,29 @@ +@@ -1204,6 +1204,34 @@ } impl Type { @@ -11,12 +11,17 @@ + cur: CXCursor, + data: CXClientData, + ) -> CXVisitorResult { ++ // SAFETY: clang_Type_visitFields invokes this callback synchronously ++ // with the live Vec pointer supplied below; callbacks do not overlap. + let fields = unsafe { &mut *data.cast::>() }; + fields.push(Cursor { x: cur }); + CXVisit_Continue + } + + let mut fields = Vec::::new(); ++ // SAFETY: self.x is the live Clang type owned by the parsing context. ++ // The visitor only borrows fields during this call, and Clang does not ++ // retain the callback data after returning. + unsafe { + clang_Type_visitFields( + self.x, diff --git a/tools/aurora-bindgen/sources.json b/tools/aurora-bindgen/sources.json index ca75ddc018d28..cf97c6753218e 100644 --- a/tools/aurora-bindgen/sources.json +++ b/tools/aurora-bindgen/sources.json @@ -6,14 +6,15 @@ "cargo_lock_sha256": "85edb9aab2d5649f39b805dc52e3f27ebf8af4d94b049601aee1c55fcf450393", "patches": { "0001-empty-alignment.patch": "eb71efca5c60adc9db6837c933ff377e8aefad53655057406ff9f1464dd3147f", - "0002-anonymous-tag-members.patch": "9072ec2fd9d0f0160615cbfd0d9cfe16d23cd0ae90747a2aa984b8694bdee9ec" + "0002-anonymous-tag-members.patch": "3c1f6a9fdf02b7cd6f11b1d0787eec7d8a14cf6f140f414cc3069eeb75524562" }, "patched_files": { "bindgen/codegen/mod.rs": "4c100fa631bdd655b509e1797411e341f9a954207d64f8f2efc285201f5f14d9", - "bindgen/clang.rs": "bd65a80b028045dec2ff1ca9ee347301726b30f94a0ee743a3041d4f598e45e4", + "bindgen/clang.rs": "4271adf1fd24f9a8e58e2a69c48f3de6574318461203b513d34308e38852dcd1", "bindgen/ir/comp.rs": "7d53770db84376699421c8057fbecbfc00e8a6f62ca38919bd4c718e56e40438" }, "qualified_private_tool_sha256": "dc794acc7b0d7e3cbfa1533b4b59f2518cd0cc1bff1dbb7fd104229c23c0d146", "qualified_host_toolchain": "rustc/cargo1.98.1, LLVM/libclang22.1.8, nativeAArch64; kernel Rust1.85.0", - "runtime_tests_performed": false + "runtime_tests_performed": false, + "qualified_private_tool_note": "Digest identifies the previously qualified binary before safety-comment-only patch changes; a rebuild from this manifest must record its own binary digest." } diff --git a/tools/aurora-bindgen/test.py b/tools/aurora-bindgen/test.py index 91b6d18d7fd4a..6d93f8b35b9d8 100644 --- a/tools/aurora-bindgen/test.py +++ b/tools/aurora-bindgen/test.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only """Compile valid generated bindings for explicitly matching C/Rust targets.""" +if not __debug__: + raise SystemExit("optimized Python is unsupported: validation assertions must remain enabled") + import argparse,hashlib,json,re,subprocess,sys from pathlib import Path P=Path(__file__).resolve().parent @@ -10,7 +13,7 @@ def run(cmd):return subprocess.run(cmd,capture_output=True,text=True) host=next(x.split(': ',1)[1] for x in subprocess.check_output([a.rustc,'-vV'],text=True).splitlines() if x.startswith('host: '));rt=a.rust_target or host;ct=a.clang_target or rt arch=lambda t:{'arm64':'aarch64','amd64':'x86_64'}.get(t.split('-')[0],t.split('-')[0]) assert arch(rt)==arch(ct),('C/Rust architecture mismatch',ct,rt) -rust=[a.rustc,'--edition=2021','--crate-type=rlib','--crate-name','layout_check','--emit=obj','-Cpanic=abort','--target='+rt] +rust=[a.rustc,'--edition=2021','--crate-type=rlib','--crate-name','layout_check','--deny=unknown-lints','--emit=obj','-Cpanic=abort','--target='+rt] if a.rust_sysroot:rust+=['--sysroot='+a.rust_sysroot] if a.rust_libdir:rust+=['-L',str(Path(a.rust_libdir).resolve())] probe=O/'target.rs';probe.write_text('#![no_std]\npub fn available() -> usize { core::mem::size_of::() }\n');r=run(rust+[str(probe),'-o',str(O/'target.o')]);(O/'target.log').write_text(r.stdout+r.stderr) @@ -26,11 +29,14 @@ def run(cmd):return subprocess.run(cmd,capture_output=True,text=True) } ''' cases=[];equal={};toolset=[('fixed',a.bindgen)]+([('baseline',a.baseline_bindgen)] if a.baseline_bindgen else []) -for fixture,language,extra in [('aligned-empty.h','c',[]),('aligned-empty.hpp','c++',[]),('anonymous-members.h','c',['-fms-extensions']),('tag-only.h','c',[])]: +for fixture,language,extra in [('aligned-empty.h','c',[]),('aligned-empty.hpp','c++',[]),('anonymous-members.h','c',['-fms-extensions']),('tag-only.h','c',[]),('bitfields.h','c',[])]: header=P/'tests'/fixture # Also ask the C compiler to accept the ordinary fixture independently. ccmd=[a.clang,'--target='+ct,'-x',language,*extra,'-Wno-microsoft-anon-tag','-c',str(header),'-o',str(O/(fixture+'.c.o'))];cr=run(ccmd);(O/(fixture+'.c.log')).write_text(cr.stdout+cr.stderr);assert cr.returncode==0,(fixture,cr.stderr) for label,tool in toolset: + # The baseline comparison covers layout repairs; this fixture checks the fixed tool + # against the selected compiler, including compatibility adapters when needed. + if fixture == 'bitfields.h' and label == 'baseline':continue cmd=[tool,str(header),'--rust-target','1.85','--no-doc-comments','--use-core','--no-derive-debug','--ctypes-prefix','core::ffi','--','-target',ct,'-x',language,*extra] g=run(cmd);assert g.returncode==0,(fixture,g.stderr) assert 'const _: ()' in g.stdout and '#[test]' not in g.stdout,'generator did not emit unconditional const layout checks' diff --git a/tools/aurora-bindgen/tests/bitfields.h b/tools/aurora-bindgen/tests/bitfields.h new file mode 100644 index 0000000000000..47a4332ccf4cf --- /dev/null +++ b/tools/aurora-bindgen/tests/bitfields.h @@ -0,0 +1,11 @@ +struct bitfields { + unsigned int enabled : 1; + unsigned int count : 7; + signed int delta : 8; + unsigned int reserved : 16; +}; +struct bitfield_container { + unsigned long before; + struct bitfields value; + unsigned long after; +}; From ce4145542949b026e1b3a2d325fc3f55910c5ac9 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 01:53:22 -0400 Subject: [PATCH 0649/2680] drm/asahi: release successful submission work-state ownership Drop the aggregate bookkeeping reference once every command and the enqueue gate have succeeded, before signalling its fence. Scheduler packets can otherwise retain all work-state slots while their cleanup waits behind another submission on the same worker. Preserve failed submissions and independent runtime owners for retirement and replay. Reject work-state access after successful aggregate completion instead of creating a replacement allocation. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 81d324022d0f97346639b06a6abaa8b86d480ead) --- drivers/gpu/drm/asahi/g17/completion.rs | 2 +- drivers/gpu/drm/asahi/g17/fence.rs | 35 ++++++++++++++----- drivers/gpu/drm/asahi/g17/queue/compute.rs | 2 +- drivers/gpu/drm/asahi/g17/queue/render.rs | 8 ++--- .../gpu/drm/asahi/g17/queue/render/parked.rs | 2 +- drivers/gpu/drm/asahi/g17/runtime.rs | 2 +- .../drm/asahi/g17/runtime/render/submit.rs | 2 +- 7 files changed, 35 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 21ead1107fe27..a37c7fb045d69 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -155,7 +155,7 @@ impl Completion { pub(crate) fn status(&self) -> &Arc { &self.status } - pub(crate) fn work_state(&self) -> Arc { + pub(crate) fn work_state(&self) -> Result> { self.member.work_state() } pub(crate) fn has_timestamps(&self) -> bool { diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 32d25b3b608cb..1217d7e7b3f4e 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -11,9 +11,10 @@ use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering}; use kernel::{ bindings, c_str, dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UserFence}, + new_mutex, prelude::*, str::CStr, - sync::Arc, + sync::{Arc, Mutex}, }; pub(crate) struct CompletionFence; @@ -40,24 +41,29 @@ pub(crate) fn independent(contexts: &FenceContexts) -> Result, - work_state: Arc, + /// Successful retirement drops only this bookkeeping owner; runtime owners + /// and failed submissions keep their work-state allocation. + #[pin] + work_state: Mutex>>, remaining: AtomicU32, error: AtomicI32, } impl Submission { fn new(fence: UserFence, context: Arc) -> Result> { - Ok(Arc::new( - Self { + let work_state = Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?; + Arc::pin_init( + pin_init!(Self { fence, - work_state: Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?, + work_state <- new_mutex!(Some(work_state), "G17 submission work state"), remaining: AtomicU32::new(1), error: AtomicI32::new(0), - }, + }), GFP_KERNEL, - )?) + ) } pub(crate) fn fence(&self) -> Fence { @@ -86,6 +92,12 @@ impl Submission { let error = self.error.load(Ordering::Acquire); if error != 0 { self.fence.set_error(Error::from_errno(error)); + } else { + // Scheduler cleanup can wait behind submission on the same worker. + // Once every member and enqueue have succeeded, retained packets + // must not prevent new submissions from acquiring work-state nodes. + let retired = self.work_state.lock().take(); + drop(retired); } self.fence.signal(); } @@ -108,8 +120,13 @@ pub(crate) struct Member { } impl Member { - pub(crate) fn work_state(&self) -> Arc { - self.submission.work_state.clone() + pub(crate) fn work_state(&self) -> Result> { + self.submission + .work_state + .lock() + .as_ref() + .cloned() + .ok_or(ECANCELED) } pub(crate) fn complete(&self, result: Result) { diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 832d5073ecc7b..2a30cd54cecd1 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -1218,7 +1218,7 @@ impl Queue { else { return Err(EINVAL); }; - let work = packet.completion.work_state(); + let work = packet.completion.work_state()?; let work_word = work.word(); if work_word == u32::MAX { return Err(EINVAL); diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 3a7852be1dab0..c3ac091333c27 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -162,7 +162,7 @@ impl Pair { let Validated::Render { pass, .. } = &first.command else { return Err(EINVAL); }; - let word = first.completion.work_state().word(); + let word = first.completion.work_state()?.word(); if word == u32::MAX { return Err(EINVAL); } @@ -271,7 +271,7 @@ impl Pair { } let ordinal = self.ordinal; ordinal.checked_add(1).ok_or(EOVERFLOW)?; - let work = packet.completion.work_state(); + let work = packet.completion.work_state()?; let node = work.node_va()?; let word = work.word(); let layout = target::Layout::new(pass, self.clusters)?; @@ -417,7 +417,7 @@ impl Pair { let qids = self.qids(); let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); let parents = self.kicks.each_ref().map(kick::Queue::parent); - let work = packet.completion.work_state(); + let work = packet.completion.work_state()?; let work_slot = work.render_slot(self.slot)?; self.memory.bind_work_slot(ordinal, qids, work_slot)?; let payload = work.channel_payload(self.slot, ordinal)?; @@ -1382,7 +1382,7 @@ impl Pair { buffer_id, pm_generation, self.ordinal, - first.completion.work_state().word(), + first.completion.work_state()?.word(), )?; (Some(memory), Some(persistent), Some(aliases)) }; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 35955810a5af1..3d67578e2aaf9 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -120,7 +120,7 @@ impl ParkedPair { let Validated::Render { pass, .. } = &first.command else { return Err(EINVAL); }; - let word = first.completion.work_state().word(); + let word = first.completion.work_state()?.word(); if word == u32::MAX { return Err(EINVAL); } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 6fe67478a777f..8978693a789ea 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -763,7 +763,7 @@ impl Backend { return Err(EIO); } let lease = self.shared.preparations.enter(false)?; - packet.completion.work_state().acquire()?; + packet.completion.work_state()?.acquire()?; let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 86a00e7bd741a..0586dfdc0944f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -66,7 +66,7 @@ impl Backend { } let work_state = { let preparation = self.shared.preparations.enter(interruptible)?; - let result = packet.completion.work_state().acquire(); + let result = packet.completion.work_state().and_then(|state| state.acquire()); if preparation.is_current() { result } else { From 74042a535ebcd78a6ff26655aec3266711225c27 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 01:54:32 -0400 Subject: [PATCH 0650/2680] drm/asahi: preserve exact historical command prefixes Accept queue-local render and compute ordinals from extensible payloads. Validate them against frozen ioctl-entry counters before publication and merge them with relative barriers without including unrelated newer work. Keep the reserved compute layout slot before the new fields and require it to remain zero. Existing short payloads retain their zero-filled extension behavior. Advertise the protocol only for G17 queues. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e0f7410fd5b368d06ab17d802720f9d76e12880c) --- drivers/gpu/drm/asahi/g17/command.rs | 8 +++++ drivers/gpu/drm/asahi/g17/job.rs | 12 ++++++- drivers/gpu/drm/asahi/g17/mod.rs | 3 +- drivers/gpu/drm/asahi/g17/queue/mod.rs | 11 +++++- include/uapi/drm/asahi_drm.h | 46 ++++++++++++++++++++++++++ 5 files changed, 77 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index fef478310c23b..47f9c0048e673 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -289,6 +289,13 @@ pub(crate) enum Validated { } impl Payload { + pub(crate) fn prior(&self) -> [u64; 2] { + match self { + Self::Render(cmd, _) => [cmd.prior_vdm, cmd.prior_cdm], + Self::Compute(cmd, _) => [cmd.prior_vdm, cmd.prior_cdm], + } + } + pub(crate) fn validate( &self, window: &UscWindow, @@ -399,6 +406,7 @@ impl Payload { } Self::Compute(cmd, attachments) => { if cmd.flags != 0 + || cmd.scs_layout != 0 || cmd.cdm_ctrl_stream_base & 3 != 0 || cmd.cdm_ctrl_stream_end & 3 != 0 || cmd.cdm_ctrl_stream_end <= cmd.cdm_ctrl_stream_base diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 76393a1802b82..9dee3b0e5bb2f 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -33,7 +33,7 @@ impl Engine { } /// Logical sequence numbers do not wrap with firmware timestamps. A zero barrier -/// includes preceding ioctls; the sentinel means independent of that engine. +/// includes preceding ioctls; the sentinel leaves only an explicit historical prefix. #[derive(Clone, Copy)] pub(crate) struct Order { pub(crate) sequence: u64, @@ -45,11 +45,16 @@ impl Order { base: [u64; 2], preceding: [u16; 2], barriers: [u16; 2], + prior: [u64; 2], engine: Engine, ) -> Result { let mut sequences = [0; 2]; let mut wait_through = [None; 2]; for index in 0..2 { + // Earlier commands in this ioctl are not historical commands. + if prior[index] > base[index] { + return Err(EINVAL); + } sequences[index] = base[index] .checked_add(u64::from(preceding[index])) .and_then(|value| value.checked_add(1)) @@ -64,6 +69,11 @@ impl Order { .ok_or(EOVERFLOW)?, ); } + if prior[index] != 0 { + wait_through[index] = Some( + wait_through[index].map_or(prior[index], |last| last.max(prior[index])), + ); + } } Ok(Self { sequence: sequences[engine.index()], diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index c4293633a0715..4445374b22e49 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1035,7 +1035,8 @@ impl gpu::Gpu for Gpu { fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { params.features |= - uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64; + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64; params.gpu_generation = hw::GpuGen::G17 as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index dcd6f8696f3a2..40fa4c5c57b16 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -282,6 +282,9 @@ impl Queue { ) -> Result<(KVec<(Validated, Order)>, KVec)> { let parse_guard = self.context.vm().retain_first_job()?; let mut parser = Parser::new(bytes); + // All historical ordinals use the same entry counters; publication + // starts only after the entire ioctl has been validated and pinned. + let base = self.sequence; let mut preceding = [0; 2]; let mut pending = KVec::with_capacity(1, GFP_KERNEL)?; while let Some(command) = parser.next()? { @@ -295,7 +298,13 @@ impl Queue { Validated::Render { .. } => Engine::Render, Validated::Compute { .. } => Engine::Compute, }; - let order = Order::new(self.sequence, preceding, command.barriers, engine)?; + let order = Order::new( + base, + preceding, + command.barriers, + command.payload.prior(), + engine, + )?; preceding[engine.index()] += 1; pending.push((payload, order), GFP_KERNEL)?; } diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 8dae3950b4a02..cba36bd5db82e 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -276,6 +276,13 @@ enum drm_asahi_feature { * Registration may still be refused. */ DRM_ASAHI_FEATURE_VM_STATUS_MIRROR = (1UL) << 4, + + /** + * @DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS: Render and compute payloads + * support exact queue-local historical prefixes in prior_vdm/prior_cdm. + * These fields supplement the header barriers. + */ + DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS = (1UL) << 5, }; /** @@ -1306,6 +1313,24 @@ struct drm_asahi_cmd_render { /** @bg_partial_rsrc_spec_hi: High half of @partial_bg.rsrc_spec */ __u32 bg_partial_rsrc_spec_hi; + + /** + * @prior_vdm: Historical render prefix, or zero for no added dependency. + * + * Requires DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS. Counts hardware render + * commands accepted on this queue since creation, starting at one. A + * nonzero value must not exceed the render count at this ioctl's entry; + * it waits for every render command through that ordinal. The effective + * dependency is the maximum of this prefix and the header's resolved + * vdm_barrier. Header NONE does not suppress this field. + */ + __u64 prior_vdm; + + /** + * @prior_cdm: Historical compute prefix; see @prior_vdm, using the + * queue's compute command count and the header's cdm_barrier instead. + */ + __u64 prior_cdm; }; /** @@ -1343,6 +1368,27 @@ struct drm_asahi_cmd_compute { /** @ts: Timestamps for the compute command */ struct drm_asahi_timestamps ts; + + /** @scs_layout: Reserved; must be zero. */ + __u64 scs_layout; + + /** + * @prior_vdm: Historical render prefix, or zero for no added dependency. + * + * Requires DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS. Counts hardware render + * commands accepted on this queue since creation, starting at one. A + * nonzero value must not exceed the render count at this ioctl's entry; + * it waits for every render command through that ordinal. The effective + * dependency is the maximum of this prefix and the header's resolved + * vdm_barrier. Header NONE does not suppress this field. + */ + __u64 prior_vdm; + + /** + * @prior_cdm: Historical compute prefix; see @prior_vdm, using the + * queue's compute command count and the header's cdm_barrier instead. + */ + __u64 prior_cdm; }; /** From e56c0ecc8cb8b04e7a9a290ff9a5c278c2b247d9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 19:43:22 +1100 Subject: [PATCH 0651/2680] media: apple: avd: h264: Grow the job table with its own allocator avd_init_job() allocates the job segment table with kvcalloc(), and it is freed with kvfree(). The growing slice table that asahi-wip added to avd_h264_run() (commit b00e2172cbb5 ("fixup! media: apple: avd: add a dedicated "submit" step")) reallocates that job table with kzalloc() and frees the old one with kfree(). Its error exit frees the table without clearing ctx->job.segments, so the context keeps a stale pointer: later slices of the frame write through it, and it is freed again later. Allocate and free the job table with kvcalloc() and kvfree() in the slice table growth as well, and clear the pointer after the error exit frees it. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-h264.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index 23a213beb21c6..dcb70d771257b 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -700,8 +700,8 @@ static int avd_h264_realloc_slices(struct avd_ctx *ctx) kfree(tmp); tmp = ctx->job.segments; - ctx->job.segments = kzalloc( - sizeof(*ctx->job.segments) * (alloc_slice_num + 1), GFP_KERNEL); + ctx->job.segments = kvcalloc(alloc_slice_num + 1, + sizeof(*ctx->job.segments), GFP_KERNEL); if (!ctx->job.segments) { ctx->job.segments = tmp; return -ENOMEM; @@ -709,7 +709,7 @@ static int avd_h264_realloc_slices(struct avd_ctx *ctx) memcpy(ctx->job.segments, tmp, sizeof(*ctx->job.segments) * (ctx->job.num + 1)); - kfree(tmp); + kvfree(tmp); return 0; } @@ -829,7 +829,8 @@ static int avd_h264_run(struct avd_ctx *ctx) return avd_submit_job(ctx); err_free_jobs: - kfree(ctx->job.segments); + kvfree(ctx->job.segments); + ctx->job.segments = NULL; avd_run_postamble(ctx, &run.base); return ret; } From 61ab577fe322d7eb3de3eba88245f61ad20a32d9 Mon Sep 17 00:00:00 2001 From: iconidentify <303316+iconidentify@users.noreply.github.com> Date: Mon, 21 Sep 2026 12:33:05 -0700 Subject: [PATCH 0652/2680] media: apple: avd: track job state so completions cannot race the run A completion IRQ or the watchdog could finish the current m2m job while avd_device_run() was still building or submitting it. v4l2-mem2mem then ran the next job on a kworker against the same ctx->job. On T8103 this oopsed twice: a NULL dereference in avd_submit_job() (segment table freed under the submit loop) and, with an earlier attempt at a fix, one in avd_hevc_run() right after avd_init_job(). Each oops left the video device lock held, so every later access to /dev/video0 hung until reboot. Track the device's one job in avd_dev under a spinlock: RUNNING while avd_device_run() executes, SUBMITTED once the hardware owns it, FINISHING once the IRQ handler or the watchdog has claimed it. The watchdog is armed only on the move to SUBMITTED, and it resets the hardware only for the job it was armed for. A completion during RUNNING is dropped as stale unless the END command already went out; then it is remembered and avd_device_run() finishes the job on return. Every finish path resets the state in avd_job_finish_no_pm(). avd_submit_job() walks a local copy of the segment table and frees it only at the end, and release cancels the watchdog. Transitions log with dev_dbg("avd-job: ...") for dynamic debug. --- drivers/media/platform/apple/avd/avd-drv.c | 134 ++++++++++++++++++-- drivers/media/platform/apple/avd/avd-v4l2.c | 11 ++ drivers/media/platform/apple/avd/avd.h | 20 +++ 3 files changed, 154 insertions(+), 11 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 4c7bfb97404f6..4fe757bd949f0 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -149,7 +149,9 @@ int avd_submit_job(struct avd_ctx *ctx) { struct avd_dev *avd = ctx->dev; struct avd_job *sub = &ctx->job; - struct avd_segment *seg; + struct avd_segment *segments, *seg; + unsigned long flags; + size_t num; int i, idx = 0, vp = 0; void __iomem *reg; u32 exec_mask = sub->codec == AVD_CODEC_VP9 && @@ -164,7 +166,16 @@ int avd_submit_job(struct avd_ctx *ctx) for (i = 0; i < sub->codec; i++) vp += avd->variant->vp_slots[i]; - schedule_delayed_work(&ctx->watchdog_work, msecs_to_jiffies(2000)); + /* + * avd_device_run() keeps this job from being finished (and the next one + * from starting) until it returns, and arms the watchdog itself once the + * job belongs to the hardware. Still walk a local copy of the table. + */ + segments = sub->segments; + num = sub->num; + if (WARN_ON_ONCE(!segments)) + return -EINVAL; + avd->variant->configure_stream(avd, ctx->inst.addr, ctx->fifo_idx, vp); reg = avd->ctrl + avd->variant->vp_slot_offset + vp * 4; @@ -173,22 +184,29 @@ int avd_submit_job(struct avd_ctx *ctx) writel(AVD_OP_EXEC | exec_mask | exec_rev_flag | AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx), reg); - seg = &sub->segments[idx++]; + seg = &segments[idx++]; for (i = 0; i < seg->num; i++) writel(seg->instructions[i], reg); - for (; idx <= sub->num; idx++) { - seg = &sub->segments[idx]; + for (; idx <= num; idx++) { + seg = &segments[idx]; for (i = 0; i < seg->num; i++) writel(seg->instructions[i], reg); if (avd_wait_submission_queue(ctx, vp)) break; + if (idx == num) { + /* A completion from here on belongs to this job. */ + spin_lock_irqsave(&avd->job_lock, flags); + avd->job_end_sent = true; + spin_unlock_irqrestore(&avd->job_lock, flags); + } writel(AVD_OP_EXEC | exec_mask | - AVD_OP_EXEC_FLAG_END(idx == sub->num), + AVD_OP_EXEC_FLAG_END(idx == num), reg); } - kvfree(sub->segments); - sub->segments = NULL; + if (sub->segments == segments) + sub->segments = NULL; + kvfree(segments); return 0; } @@ -222,6 +240,8 @@ static void avd_watchdog_func(struct work_struct *work) { struct avd_dev *avd; struct avd_ctx *ctx; + unsigned long flags; + bool claimed; int ret; ctx = container_of(to_delayed_work(work), struct avd_ctx, watchdog_work); @@ -230,6 +250,22 @@ static void avd_watchdog_func(struct work_struct *work) avd = ctx->dev; + /* + * Time out only the job this watchdog was armed for, and only while the + * hardware still owns it. Otherwise that job already finished, and + * resetting the hardware would break whichever job is running now. + */ + spin_lock_irqsave(&avd->job_lock, flags); + claimed = avd->job_ctx == ctx && avd->job_seq == ctx->wd_seq && + avd->job_state == AVD_JOB_SUBMITTED; + if (claimed) + avd->job_state = AVD_JOB_FINISHING; + spin_unlock_irqrestore(&avd->job_lock, flags); + dev_dbg(avd->dev, "avd-job: watchdog seq %llu: %s\n", ctx->wd_seq, + claimed ? "timeout" : "stale, ignored"); + if (!claimed) + return; + dev_err(avd->dev, "Frame processing timed out!"); writel(0, avd->mbox + AVD_REG_MBOX_IRQ_ENABLE); @@ -246,6 +282,8 @@ static irqreturn_t avd_irq_handler(int irq, void *data) struct avd_ctx *ctx = v4l2_m2m_get_curr_priv(avd->m2m_dev); enum vb2_buffer_state state; + unsigned long flags; + bool finish = false, stale = false; u32 status; if (!ctx) return IRQ_HANDLED; @@ -274,9 +312,33 @@ static irqreturn_t avd_irq_handler(int irq, void *data) goto done; } - /* if the watchdog_work has run the work has already been submitted */ - if (cancel_delayed_work(&ctx->watchdog_work)) + /* + * Finish only a job the hardware owns. While avd_device_run() is still + * setting up or submitting, a completion is left over from an older job + * unless this job's END command already went out; then remember it and + * let avd_device_run() finish the job when it returns. + */ + spin_lock_irqsave(&avd->job_lock, flags); + if (avd->job_ctx != ctx) { + stale = true; + } else if (avd->job_state == AVD_JOB_SUBMITTED) { + avd->job_state = AVD_JOB_FINISHING; + finish = true; + } else if (avd->job_state == AVD_JOB_RUNNING && avd->job_end_sent) { + avd->job_pending = true; + avd->job_pending_result = state; + } else { + stale = true; + } + spin_unlock_irqrestore(&avd->job_lock, flags); + dev_dbg(avd->dev, "avd-job: irq %#x: %s\n", status, + finish ? "finish" : stale ? "stale, ignored" : "pending"); + if (stale) + dev_warn_ratelimited(avd->dev, "ignoring stale completion"); + if (finish) { + cancel_delayed_work(&ctx->watchdog_work); avd_job_finish(ctx, state); + } done: return IRQ_HANDLED; @@ -287,6 +349,11 @@ static void avd_device_run(void *priv) struct avd_ctx *ctx = priv; struct avd_dev *avd = ctx->dev; const struct avd_coded_fmt_desc *desc = ctx->coded_fmt_desc; + enum vb2_buffer_state result = VB2_BUF_STATE_ERROR; + const char *outcome = "already finished"; + unsigned long flags; + bool finish = false; + u64 seq; int ret; if (WARN_ON(!desc)) @@ -298,9 +365,51 @@ static void avd_device_run(void *priv) return; } + /* + * While the job is RUNNING neither completion IRQs nor the watchdog may + * finish it: v4l2-mem2mem would start the next job on a kworker while + * this one still builds and submits ctx->job. + */ + spin_lock_irqsave(&avd->job_lock, flags); + seq = ++avd->job_seq; + avd->job_ctx = ctx; + avd->job_state = AVD_JOB_RUNNING; + avd->job_end_sent = false; + avd->job_pending = false; + spin_unlock_irqrestore(&avd->job_lock, flags); + dev_dbg(avd->dev, "avd-job: run seq %llu\n", seq); + ret = desc->ops->run(ctx); - if (ret) + if (ret) { + dev_dbg(avd->dev, "avd-job: run seq %llu failed: %d\n", seq, ret); avd_job_finish(ctx, VB2_BUF_STATE_ERROR); + return; + } + + /* + * The run either finished the job itself (held capture buffer) or handed + * it to the hardware: arm the watchdog, or finish right away if the + * completion already arrived. + */ + spin_lock_irqsave(&avd->job_lock, flags); + if (avd->job_seq == seq && avd->job_state == AVD_JOB_RUNNING) { + if (avd->job_pending) { + avd->job_state = AVD_JOB_FINISHING; + result = avd->job_pending_result; + finish = true; + outcome = "completed during submission"; + } else { + avd->job_state = AVD_JOB_SUBMITTED; + ctx->wd_seq = seq; + schedule_delayed_work(&ctx->watchdog_work, msecs_to_jiffies(2000)); + outcome = "submitted"; + } + } + spin_unlock_irqrestore(&avd->job_lock, flags); + dev_dbg(avd->dev, "avd-job: run seq %llu: %s\n", seq, outcome); + + if (finish) + avd_job_finish(ctx, result); } static int avd_queue_init(void *priv, struct vb2_queue *src_vq, @@ -396,6 +505,8 @@ static int avd_release(struct file *filp) v4l2_fh_del(&ctx->fh, filp); v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); + /* The job has finished; its watchdog must not run on the freed ctx. */ + cancel_delayed_work_sync(&ctx->watchdog_work); v4l2_ctrl_handler_free(&ctx->ctrl_hdl); v4l2_fh_exit(&ctx->fh); avd_buf_free(ctx->dev, &ctx->inst); @@ -697,6 +808,7 @@ static int avd_probe(struct platform_device *pdev) avd->pdev = pdev; mutex_init(&avd->vdev_lock); + spin_lock_init(&avd->job_lock); match = of_match_node(avd_of_match, pdev->dev.of_node); avd->variant = match->data; diff --git a/drivers/media/platform/apple/avd/avd-v4l2.c b/drivers/media/platform/apple/avd/avd-v4l2.c index 40be79b1e02e1..987894779ed59 100644 --- a/drivers/media/platform/apple/avd/avd-v4l2.c +++ b/drivers/media/platform/apple/avd/avd-v4l2.c @@ -925,6 +925,17 @@ const struct vb2_ops avd_queue_ops = { void avd_job_finish_no_pm(struct avd_ctx *ctx, enum vb2_buffer_state result) { + struct avd_dev *avd = ctx->dev; + unsigned long flags; + + spin_lock_irqsave(&avd->job_lock, flags); + if (avd->job_ctx == ctx) { + avd->job_state = AVD_JOB_IDLE; + avd->job_ctx = NULL; + avd->job_pending = false; + } + spin_unlock_irqrestore(&avd->job_lock, flags); + if (ctx->coded_fmt_desc->ops->done) { struct vb2_v4l2_buffer *src_buf, *dst_buf; diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index 77eb6a39e1fd2..f026b45913c33 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -186,6 +187,14 @@ struct avd_variant { unsigned int quirks; }; +/* Lifecycle of the one m2m job the device runs at a time. */ +enum avd_job_state { + AVD_JOB_IDLE, + AVD_JOB_RUNNING, /* inside avd_device_run() */ + AVD_JOB_SUBMITTED, /* handed to the hardware, watchdog armed */ + AVD_JOB_FINISHING, /* claimed by the IRQ handler or the watchdog */ +}; + struct avd_dev { struct device *dev; struct v4l2_device v4l2_dev; @@ -205,6 +214,15 @@ struct avd_dev { struct mutex vdev_lock; + /* The current job; see avd_device_run(). Protected by job_lock. */ + spinlock_t job_lock; + enum avd_job_state job_state; + struct avd_ctx *job_ctx; + u64 job_seq; + bool job_end_sent; + bool job_pending; + enum vb2_buffer_state job_pending_result; + struct reset_control *rstc; const struct avd_variant *variant; @@ -241,6 +259,8 @@ struct avd_ctx { bool decomp; struct delayed_work watchdog_work; + /* job_seq of the job watchdog_work was armed for */ + u64 wd_seq; void *priv; From f970997f07e20f60464ca9ef890080cf2d54a082 Mon Sep 17 00:00:00 2001 From: iconidentify <303316+iconidentify@users.noreply.github.com> Date: Tue, 15 Sep 2026 07:40:24 -0700 Subject: [PATCH 0653/2680] media: apple: avd: hevc: read controls after applying the request avd_hevc_run_preamble() read the HEVC controls, including the number of slices and entry point offsets (ctrl->new_elems), before avd_run_preamble() applied the request with v4l2_ctrl_request_setup(). new_elems is scratch space: setting controls on any request writes the count there. Userspace that queues requests ahead (FFmpeg through VA-API does) sets the next picture's controls while the current one is being prepared, so the current picture could be decoded with the next picture's slice count. In WPP_C_ericsson_MAIN_2, picture 12 has 16 slices and picture 13 has one; picture 12 was sent with only its first slice and the firmware rejected it (H0 error), in 11 of 20 decodes. Reading elems at the same place is no better: before request setup it holds the previous picture's count (0 of 20 decodes correct). Apply the request first, then read the controls and their counts (elems). This also stops holding p_cur of a dynamic array across the setup, which can move the array's storage when it grows. --- drivers/media/platform/apple/avd/avd-hevc.c | 22 ++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-hevc.c b/drivers/media/platform/apple/avd/avd-hevc.c index be062bf09b302..d72cba7eb0fbf 100644 --- a/drivers/media/platform/apple/avd/avd-hevc.c +++ b/drivers/media/platform/apple/avd/avd-hevc.c @@ -1250,13 +1250,25 @@ static int avd_hevc_run_preamble(struct avd_ctx *ctx, struct avd_hevc_run *run) struct v4l2_ctrl *ctrl; u32 dst_len, mv_color_len; + if (!v4l2_ctrl_find(&ctx->ctrl_hdl, + V4L2_CID_STATELESS_HEVC_ENTRY_POINT_OFFSETS)) + return -EINVAL; + + /* + * Apply the request's controls before reading them. Until then the + * element counts of the dynamic arrays belong to the previous request + * (elems) or to whoever set controls last (new_elems), and setting a + * larger array can move their storage. + */ + avd_run_preamble(ctx, &run->base); + ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl, V4L2_CID_STATELESS_HEVC_DECODE_PARAMS); run->decode = ctrl ? ctrl->p_cur.p : NULL; ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl, V4L2_CID_STATELESS_HEVC_SLICE_PARAMS); run->sl = ctrl ? ctrl->p_cur.p : NULL; - run->num_slices = ctrl ? ctrl->new_elems : 0; + run->num_slices = ctrl ? ctrl->elems : 0; ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl, V4L2_CID_STATELESS_HEVC_SPS); run->sps = ctrl ? ctrl->p_cur.p : NULL; ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl, V4L2_CID_STATELESS_HEVC_PPS); @@ -1267,12 +1279,8 @@ static int avd_hevc_run_preamble(struct avd_ctx *ctx, struct avd_hevc_run *run) ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl, V4L2_CID_STATELESS_HEVC_ENTRY_POINT_OFFSETS); - run->entry_point_offsets = ctrl ? ctrl->p_cur.p : NULL; - if (!run->entry_point_offsets) - return -EINVAL; - run->num_entry_point_offsets = ctrl ? ctrl->new_elems : 0; - - avd_run_preamble(ctx, &run->base); + run->entry_point_offsets = ctrl->p_cur.p; + run->num_entry_point_offsets = ctrl->elems; dst_len = run->base.bufs.dst->vb2_buf.planes[0].length; From 8ec43996c2a0de45590bb29ad58acbe4571a77e7 Mon Sep 17 00:00:00 2001 From: iconidentify <303316+iconidentify@users.noreply.github.com> Date: Tue, 15 Sep 2026 10:38:22 -0700 Subject: [PATCH 0654/2680] media: apple: avd: size the compressed chroma plane for 4:2:2 fill_comp() sized the compressed chroma plane for half-height chroma whatever the image format. 4:2:2 chroma is full height, so reference chroma did not fit: 4:2:2 intra streams decoded, every 4:2:2 stream with P or B slices decoded wrong. JVT-FR-EXT through GStreamer: FREXT1-3_TANDBERG_A, Hi422FR2-4_SONY_A and Hi422FR11-12_SONY_A now pass (8-bit 4:2:2; 10-bit 4:2:2 still has no P210 capture format). --- drivers/media/platform/apple/avd/avd-drv.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 4fe757bd949f0..9e27895958b8d 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -58,9 +58,11 @@ void fill_comp(struct avd_comp *comp, enum avd_image_fmt image_fmt, u32 width, /* y has 32x32 tiles and 32 bytes of metadata per tile */ calc_tile_meta(width, height, bit_depth, 32, 32, &y, &y_meta); - /* uv has 16x16 tiles and 8 bytes of metadata per tile */ - calc_tile_meta(width / 2, height / 2, bit_depth * 2, 16, 8, &uv, - &uv_meta); + /* uv has 16x16 tiles and 8 bytes of metadata per tile; 4:2:2 chroma is full height */ + calc_tile_meta(width / 2, + (image_fmt == AVD_IMG_FMT_422_8BIT || + image_fmt == AVD_IMG_FMT_422_10BIT) ? height : height / 2, + bit_depth * 2, 16, 8, &uv, &uv_meta); /* output like DCP driver expects */ comp->offsets[0] = y; From 5df4fd6303d31668d805ef88253bd49e60914b9c Mon Sep 17 00:00:00 2001 From: iconidentify <303316+iconidentify@users.noreply.github.com> Date: Tue, 15 Sep 2026 10:38:22 -0700 Subject: [PATCH 0655/2680] media: apple: avd: h264: fill chroma with grey for 4:0:0 pictures The driver accepts monochrome streams and decodes them into 4:2:0 buffers, and the firmware writes zero chroma, so pictures came out green. Fill the chroma plane with 0x80 when a picture completes. HPCAMOLQ_BRCM_B and HPCVMOLQ_BRCM_B now match software decoding, through GStreamer and through VA-API. --- drivers/media/platform/apple/avd/avd-h264.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index dcb70d771257b..67f5e5debb129 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -50,6 +50,7 @@ struct avd_h264_run { /* state */ struct avd_h264_ctx { + bool monochrome; struct avd_h264_reflists { struct v4l2_h264_reference p[V4L2_H264_REF_LIST_LEN]; struct v4l2_h264_reference b0[V4L2_H264_REF_LIST_LEN]; @@ -787,6 +788,7 @@ static int avd_h264_run(struct avd_ctx *ctx) goto err_free_jobs; memcpy(h264_ctx->active_slice->cpu, data, payload_len); h264_ctx->slice_num++; + h264_ctx->monochrome = run.sps->chroma_format_idc == 0; /* Build the P/B{0,1} ref lists. */ v4l2_h264_init_reflist_builder(&reflist_builder, run.decode_params, @@ -847,6 +849,16 @@ static void avd_h264_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, for (i = 0; i < h264_ctx->slice_num; i++) avd_buf_free(avd, &h264_ctx->slices[i]); h264_ctx->slice_num = 0; + + /* 4:0:0 has no chroma to decode; the picture is grey */ + if (h264_ctx->monochrome && dst_buf && result == VB2_BUF_STATE_DONE) { + const struct v4l2_pix_format_mplane *pix = &ctx->decoded_fmt.fmt.pix_mp; + u8 *y = vb2_plane_vaddr(&dst_buf->vb2_buf, 0); + size_t luma = (size_t)pix->plane_fmt[0].bytesperline * pix->height; + + if (y) + memset(y + luma, 0x80, luma / 2); + } } } From bc0121ba454b36ad659670714b8062a72d685a60 Mon Sep 17 00:00:00 2001 From: iconidentify <303316+iconidentify@users.noreply.github.com> Date: Tue, 15 Sep 2026 10:38:22 -0700 Subject: [PATCH 0656/2680] media: apple: avd: hevc: reject unequal bit depths and non-4:2:0 SPS An SPS with different luma and chroma bit depth (TSUNEQBD_A_MAIN10_ Technicolor_2: 10-bit luma, 9-bit chroma) was accepted and ended in a firmware H0 error and a watchdog reset of the VP, which disturbs every other decoding context. Only 4:2:0 has a capture format. H.264 already rejects both; do the same for HEVC so such streams fail at once. --- drivers/media/platform/apple/avd/avd-hevc.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/media/platform/apple/avd/avd-hevc.c b/drivers/media/platform/apple/avd/avd-hevc.c index d72cba7eb0fbf..0acb18e30f349 100644 --- a/drivers/media/platform/apple/avd/avd-hevc.c +++ b/drivers/media/platform/apple/avd/avd-hevc.c @@ -1015,6 +1015,12 @@ static int avd_hevc_validate_sps(struct avd_ctx *ctx, if (sps->pic_width_in_luma_samples > ctx->coded_fmt.fmt.pix_mp.width || sps->pic_height_in_luma_samples > ctx->coded_fmt.fmt.pix_mp.height) return -EINVAL; + if (sps->bit_depth_luma_minus8 != sps->bit_depth_chroma_minus8) + /* Luma and chroma bit depth mismatch: the firmware faults (H0 error) */ + return -EINVAL; + if (sps->chroma_format_idc != 1) + /* Only 4:2:0 has a capture format */ + return -EINVAL; return 0; } From b482c9ccfd075c6bf28bb3407cdcbbd4ee96031a Mon Sep 17 00:00:00 2001 From: iconidentify <303316+iconidentify@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:49:47 -0700 Subject: [PATCH 0657/2680] media: apple: avd: hevc: copy the slice and entry point arrays for the run Setting a dynamic array control on a queued request with more elements than ever before reallocates the control's storage (user_to_new() in the control core) and frees the old array. avd_hevc_run() kept using p_cur of the slice params and entry point offsets without the control lock while building the job, so userspace preparing the next picture could free the arrays under it. Copy both arrays, and their element counts, under the control lock right after the request is applied, and use the copies. --- drivers/media/platform/apple/avd/avd-hevc.c | 37 +++++++++++++++------ 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-hevc.c b/drivers/media/platform/apple/avd/avd-hevc.c index 0acb18e30f349..b0bd8dc84f140 100644 --- a/drivers/media/platform/apple/avd/avd-hevc.c +++ b/drivers/media/platform/apple/avd/avd-hevc.c @@ -1253,10 +1253,11 @@ static void avd_hevc_stop(struct avd_ctx *ctx) static int avd_hevc_run_preamble(struct avd_ctx *ctx, struct avd_hevc_run *run) { - struct v4l2_ctrl *ctrl; + struct v4l2_ctrl *ctrl, *sl, *ep; u32 dst_len, mv_color_len; - if (!v4l2_ctrl_find(&ctx->ctrl_hdl, + if (!v4l2_ctrl_find(&ctx->ctrl_hdl, V4L2_CID_STATELESS_HEVC_SLICE_PARAMS) || + !v4l2_ctrl_find(&ctx->ctrl_hdl, V4L2_CID_STATELESS_HEVC_ENTRY_POINT_OFFSETS)) return -EINVAL; @@ -1271,10 +1272,6 @@ static int avd_hevc_run_preamble(struct avd_ctx *ctx, struct avd_hevc_run *run) ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl, V4L2_CID_STATELESS_HEVC_DECODE_PARAMS); run->decode = ctrl ? ctrl->p_cur.p : NULL; - ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl, - V4L2_CID_STATELESS_HEVC_SLICE_PARAMS); - run->sl = ctrl ? ctrl->p_cur.p : NULL; - run->num_slices = ctrl ? ctrl->elems : 0; ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl, V4L2_CID_STATELESS_HEVC_SPS); run->sps = ctrl ? ctrl->p_cur.p : NULL; ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl, V4L2_CID_STATELESS_HEVC_PPS); @@ -1283,10 +1280,28 @@ static int avd_hevc_run_preamble(struct avd_ctx *ctx, struct avd_hevc_run *run) V4L2_CID_STATELESS_HEVC_SCALING_MATRIX); run->scaling_matrix = ctrl ? ctrl->p_cur.p : NULL; - ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl, - V4L2_CID_STATELESS_HEVC_ENTRY_POINT_OFFSETS); - run->entry_point_offsets = ctrl->p_cur.p; - run->num_entry_point_offsets = ctrl->elems; + sl = v4l2_ctrl_find(&ctx->ctrl_hdl, V4L2_CID_STATELESS_HEVC_SLICE_PARAMS); + ep = v4l2_ctrl_find(&ctx->ctrl_hdl, + V4L2_CID_STATELESS_HEVC_ENTRY_POINT_OFFSETS); + + /* + * Setting a larger dynamic array on a queued request reallocates the + * array's storage, which this run would otherwise keep using. Copy what + * the run needs while holding the control lock. + */ + v4l2_ctrl_lock(sl); + run->num_slices = sl->elems; + run->sl = kmemdup(sl->p_cur.p, sl->elems * sl->elem_size, GFP_KERNEL); + run->num_entry_point_offsets = ep->elems; + run->entry_point_offsets = + kmemdup(ep->p_cur.p, ep->elems * ep->elem_size, GFP_KERNEL); + v4l2_ctrl_unlock(sl); + if (!run->sl || !run->entry_point_offsets) { + kfree(run->sl); + kfree(run->entry_point_offsets); + avd_run_postamble(ctx, &run->base); + return -ENOMEM; + } dst_len = run->base.bufs.dst->vb2_buf.planes[0].length; @@ -1330,6 +1345,8 @@ static int avd_hevc_run(struct avd_ctx *ctx) done: kfree(run.tile_info.ctb_addr_rs_to_ts); kfree(run.tile_info.tile_ids); + kfree(run.sl); + kfree(run.entry_point_offsets); return ret; } From 2e48bfd915e4d9a0393c4387dc8f09f3e6d05dac Mon Sep 17 00:00:00 2001 From: iconidentify <303316+iconidentify@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:49:47 -0700 Subject: [PATCH 0658/2680] media: apple: avd: hevc: bound entry points and complete the request on errors stream_slices() accepted slices claiming one entry point offset more than the request carried (> count + 1). With tiles that reads one offset past the array and pushes one segment past the job table, which is sized num_slices + num_entry_point_offsets + 1. Require the sum to fit, and fail the job instead of submitting a half-built one. avd_hevc_run() returned from compute_tiles, alloc_scratch and init_job failures without avd_run_postamble(), so the applied request was never completed and a client waiting on the request fd waited forever. It also freed tile_info pointers that were uninitialised when compute_tiles failed before allocating them. Zero the run and complete the request on every path. --- drivers/media/platform/apple/avd/avd-hevc.c | 27 +++++++++++++-------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-hevc.c b/drivers/media/platform/apple/avd/avd-hevc.c index b0bd8dc84f140..6580f8a31600f 100644 --- a/drivers/media/platform/apple/avd/avd-hevc.c +++ b/drivers/media/platform/apple/avd/avd-hevc.c @@ -819,7 +819,7 @@ struct sl_ctx { }; /* TODO */ -static void stream_slices(struct avd_ctx *ctx, struct avd_hevc_run *run) +static int stream_slices(struct avd_ctx *ctx, struct avd_hevc_run *run) { const struct v4l2_ctrl_hevc_sps *sps = run->sps; const struct v4l2_ctrl_hevc_pps *pps = run->pps; @@ -860,12 +860,12 @@ static void stream_slices(struct avd_ctx *ctx, struct avd_hevc_run *run) to = tiles_enabled ? sl->num_entry_point_offsets + 1 : 1; if (tiles_enabled && sl->num_entry_point_offsets + entry_point_idx > - run->num_entry_point_offsets + 1) { + run->num_entry_point_offsets) { dev_err(ctx->dev->dev, - "to few entry points! has: %d, needs > %d", + "too few entry points: has %d, needs %d", run->num_entry_point_offsets, sl->num_entry_point_offsets + entry_point_idx); - return; + return -EINVAL; } for (i = 0; i < to; i++) { first_segment = i == 0; @@ -966,6 +966,8 @@ static void stream_slices(struct avd_ctx *ctx, struct avd_hevc_run *run) } hevc_ctx->submit_num = pos; + + return 0; } static void update_dec_buf_info(struct avd_decoded_buffer *buf, @@ -1313,7 +1315,7 @@ static int avd_hevc_run_preamble(struct avd_ctx *ctx, struct avd_hevc_run *run) static int avd_hevc_run(struct avd_ctx *ctx) { - struct avd_hevc_run run; + struct avd_hevc_run run = {}; struct avd_decoded_buffer *dst; int ret; @@ -1326,22 +1328,27 @@ static int avd_hevc_run(struct avd_ctx *ctx) ret = avd_hevc_compute_tiles(ctx, &run); if (ret) - goto done; + goto complete; ret = avd_hevc_alloc_scratch(ctx, &run); if (ret) - goto done; + goto complete; ret = avd_init_job(ctx, AVD_CODEC_HEVC, run.num_slices + run.num_entry_point_offsets + 1); if (ret) - goto done; + goto complete; set_header(ctx, &run); - stream_slices(ctx, &run); + ret = stream_slices(ctx, &run); avd_run_postamble(ctx, &run.base); + if (!ret) + ret = avd_submit_job(ctx); + goto done; - ret = avd_submit_job(ctx); +complete: + /* The request was applied; complete it on every way out. */ + avd_run_postamble(ctx, &run.base); done: kfree(run.tile_info.ctb_addr_rs_to_ts); kfree(run.tile_info.tile_ids); From ba478909271bc10c0e1d4270a3d455ff9adae4b6 Mon Sep 17 00:00:00 2001 From: iconidentify <303316+iconidentify@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:49:47 -0700 Subject: [PATCH 0659/2680] media: apple: avd: free job tables that were never submitted avd_init_job() replaced ctx->job.segments without freeing it, and only avd_submit_job() ever freed a table. A frame that was started but not submitted leaked its table: for H.264 that is 4096 segments, about 8 MiB, each time a slice of a started frame was rejected (invalid references), on stream-off in the middle of a frame, or on close. A table left over across stream-off could also be appended to by the next stream. Free the old table in avd_init_job(), on OUTPUT stream-off and on release. Jobs run one at a time and stream-off and release wait for the running job, so no submission can be walking the table. --- drivers/media/platform/apple/avd/avd-drv.c | 3 +++ drivers/media/platform/apple/avd/avd-v4l2.c | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 9e27895958b8d..8ccaa23e96812 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -141,6 +141,8 @@ int avd_init_job(struct avd_ctx *ctx, enum avd_codec codec, size_t segments) job->codec = codec; job->num = 0; + /* A table that was built but never submitted still belongs to us. */ + kvfree(job->segments); job->segments = kvcalloc(segments, sizeof(*job->segments), GFP_KERNEL); if (!job->segments) ret = -ENOMEM; @@ -512,6 +514,7 @@ static int avd_release(struct file *filp) v4l2_ctrl_handler_free(&ctx->ctrl_hdl); v4l2_fh_exit(&ctx->fh); avd_buf_free(ctx->dev, &ctx->inst); + kvfree(ctx->job.segments); kfree(ctx); return 0; diff --git a/drivers/media/platform/apple/avd/avd-v4l2.c b/drivers/media/platform/apple/avd/avd-v4l2.c index 987894779ed59..e9e6ca9e24913 100644 --- a/drivers/media/platform/apple/avd/avd-v4l2.c +++ b/drivers/media/platform/apple/avd/avd-v4l2.c @@ -908,6 +908,10 @@ static void avd_stop_streaming(struct vb2_queue *q) if (desc->ops->stop) desc->ops->stop(ctx); + + /* Drop a job table left over from a frame that never finished. */ + kvfree(ctx->job.segments); + ctx->job.segments = NULL; } avd_queue_cleanup(q, VB2_BUF_STATE_ERROR); From fcae01cf00b2cc898ef139c178d50f059d894c35 Mon Sep 17 00:00:00 2001 From: iconidentify <303316+iconidentify@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:49:47 -0700 Subject: [PATCH 0660/2680] media: apple: avd: h264: keep slice data and the job table in bounds The job table holds a header segment plus one segment per slice, but was sized for MAX_SLICES segments, so the 4096th slice of a frame wrote one segment past the end. The scan for emulation prevention bytes in stream_slice() walked the slice data until it had counted header_bit_size bits, which comes from userspace, without looking at the slice length. Stop at the end of the slice data. [ryan: adapted to 7.2.10 - asahi-wip's growing slice table (b00e2172cbb5) already sizes the job table for the header segment plus one segment per slice; only the comment and the stream_slice() bound remain] --- drivers/media/platform/apple/avd/avd-h264.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index 67f5e5debb129..d959deb1ff3ac 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -405,7 +405,7 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) u32 off = 2; u32 bytes_read = 2; - while (bytes_read < min_off) { + while (bytes_read < min_off && off < payload_len) { if (data[off - 2] != 0x00 || data[off - 1] != 0x00 || data[off] != 0x03) bytes_read++; @@ -810,6 +810,7 @@ static int avd_h264_run(struct avd_ctx *ctx) } if (is_new_frame(run.slice_params)) { + /* the header segment plus one per slice */ ret = avd_init_job(ctx, AVD_CODEC_H264, h264_ctx->alloc_slice_num + 1); if (ret) From da45c6991ec00b568a24c8fd4854088fe9d83049 Mon Sep 17 00:00:00 2001 From: iconidentify <303316+iconidentify@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:49:47 -0700 Subject: [PATCH 0661/2680] media: apple: avd: h264: use the 10-bit grey value for 10-bit 4:0:0 10-bit 4:0:0 streams decode into P010, where filling chroma with 0x80 bytes gives 0x8080 (514) instead of mid-grey 0x8000 (512). --- drivers/media/platform/apple/avd/avd-h264.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index d959deb1ff3ac..08eaec78f00c1 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -857,8 +857,15 @@ static void avd_h264_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, u8 *y = vb2_plane_vaddr(&dst_buf->vb2_buf, 0); size_t luma = (size_t)pix->plane_fmt[0].bytesperline * pix->height; - if (y) + if (y && ctx->image_fmt == AVD_IMG_FMT_420_10BIT) { + /* P010: 16-bit little-endian samples, 512 << 6 */ + for (size_t i = 0; i + 1 < luma / 2; i += 2) { + y[luma + i] = 0x00; + y[luma + i + 1] = 0x80; + } + } else if (y) { memset(y + luma, 0x80, luma / 2); + } } } } From 26e25f3ec7a6b14f29b2b9874ac29d46248ee2bf Mon Sep 17 00:00:00 2001 From: iconidentify <303316+iconidentify@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:49:47 -0700 Subject: [PATCH 0662/2680] media: apple: avd: accept HEVC 4:2:2 again, reject formats without a capture format The previous HEVC SPS check rejected every chroma format but 4:2:0, although 8-bit 4:2:2 maps to NV16, and it made the zeroed default SPS invalid. Only reject what cannot be decoded into an offered format: bit depths other than 8 and 10, and 4:2:2 at 10 bit, which is mapped to P010 whose buffer is too small for 4:2:2 chroma (there is no P210). Do the same for H.264, which accepted 4:2:2 10-bit into P010 as well. --- drivers/media/platform/apple/avd/avd-h264.c | 6 ++++++ drivers/media/platform/apple/avd/avd-hevc.c | 7 +++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index 08eaec78f00c1..7d89c8ca0e4bf 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -577,6 +577,12 @@ static int avd_h264_validate_sps(struct avd_ctx *ctx, if (sps->bit_depth_luma_minus8 != sps->bit_depth_chroma_minus8) /* Luma and chroma bit depth mismatch */ return -EINVAL; + if (sps->bit_depth_luma_minus8 != 0 && sps->bit_depth_luma_minus8 != 2) + /* Only 8 and 10 bit have capture formats */ + return -EINVAL; + if (sps->chroma_format_idc == 2 && sps->bit_depth_luma_minus8 == 2) + /* 4:2:2 10 bit would need P210; P010 is too small for its chroma */ + return -EINVAL; if (!(sps->flags & V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY)) /* no interlaced support */ return -EINVAL; diff --git a/drivers/media/platform/apple/avd/avd-hevc.c b/drivers/media/platform/apple/avd/avd-hevc.c index 6580f8a31600f..cc71ac58bea34 100644 --- a/drivers/media/platform/apple/avd/avd-hevc.c +++ b/drivers/media/platform/apple/avd/avd-hevc.c @@ -1020,8 +1020,11 @@ static int avd_hevc_validate_sps(struct avd_ctx *ctx, if (sps->bit_depth_luma_minus8 != sps->bit_depth_chroma_minus8) /* Luma and chroma bit depth mismatch: the firmware faults (H0 error) */ return -EINVAL; - if (sps->chroma_format_idc != 1) - /* Only 4:2:0 has a capture format */ + if (sps->bit_depth_luma_minus8 != 0 && sps->bit_depth_luma_minus8 != 2) + /* Only 8 and 10 bit have capture formats */ + return -EINVAL; + if (sps->chroma_format_idc == 2 && sps->bit_depth_luma_minus8 == 2) + /* 4:2:2 10 bit would need P210, which is not offered */ return -EINVAL; return 0; From 22c934675c1e4421f348f37da367f065c87be948 Mon Sep 17 00:00:00 2001 From: iconidentify <303316+iconidentify@users.noreply.github.com> Date: Mon, 21 Sep 2026 12:40:04 -0700 Subject: [PATCH 0663/2680] media: apple: avd: drain the old watchdog before reusing its generation An IRQ can complete a job after its watchdog starts running but before the callback checks wd_seq. cancel_delayed_work does not wait for that callback. Reusing ctx->wd_seq for the next submission lets the old callback claim and reset the new job. Drain watchdog_work synchronously at the start of the sleepable device_run callback, before publishing the next generation and outside job_lock. IRQ completion remains nonblocking. This also waits for a previous timeout callback to finish its completion tail. A finite userspace schedule fixture using the extracted driver functions reproduces the premature reset before this change and preserves the next submission afterward. Kernel workqueue primitives are modeled; this is not hardware validation. --- drivers/media/platform/apple/avd/avd-drv.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 8ccaa23e96812..6c00e4df53774 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -363,6 +363,13 @@ static void avd_device_run(void *priv) if (WARN_ON(!desc)) return; + /* + * IRQ completion cannot wait for an already running watchdog. Drain it + * here, before reusing wd_seq, so an old callback cannot time out this + * context's next job. device_run is sleepable; hold no job_lock here. + */ + cancel_delayed_work_sync(&ctx->watchdog_work); + ret = pm_runtime_resume_and_get(avd->dev); if (ret < 0) { avd_job_finish_no_pm(ctx, VB2_BUF_STATE_ERROR); From ab752ae48cb80467ef2c693aaf7a97363e80c600 Mon Sep 17 00:00:00 2001 From: iconidentify <303316+iconidentify@users.noreply.github.com> Date: Mon, 21 Sep 2026 12:40:04 -0700 Subject: [PATCH 0664/2680] media: apple: avd: free AV1 above_info on stop The upstream buffer rename moved scratch.tile_col[0] to bufs.above_info, but dropped its free from avd_av1_stop. The work-buffer allocator still allocates it for decoding, leaking the final DMA-coherent buffer whenever an AV1 context stops. Restore the matching free. An offline allocation fixture invoking the extracted stop function detects the old leak and verifies that all buffer records are released after this change. --- drivers/media/platform/apple/avd/avd-av1.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/media/platform/apple/avd/avd-av1.c b/drivers/media/platform/apple/avd/avd-av1.c index ddb38a8961c6c..9c1ed0ca8193b 100644 --- a/drivers/media/platform/apple/avd/avd-av1.c +++ b/drivers/media/platform/apple/avd/avd-av1.c @@ -1358,6 +1358,7 @@ static void avd_av1_stop(struct avd_ctx *ctx) struct avd_av1_ctx *av1_ctx = ctx->priv; struct avd_dev *avd = ctx->dev; + avd_buf_free(avd, &av1_ctx->bufs.above_info); avd_buf_free(avd, &av1_ctx->bufs.rf_above_info); avd_buf_free(avd, &av1_ctx->bufs.az_above); avd_buf_free(avd, &av1_ctx->bufs.ip_above); From 4502903f5401baa6cff0fc4cf43bc472240c16a9 Mon Sep 17 00:00:00 2001 From: iconidentify <303316+iconidentify@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:03:30 -0700 Subject: [PATCH 0665/2680] media: apple: avd: clear failed allocations before retrying A failed allocation leaves cpu NULL and size set to the failed request. A subsequent smaller H.264 slice then takes the inverted reuse shortcut and copies its payload through NULL. The failed slot is not included in the slice count, so normal completion does not clear it first. Remove the false-success shortcut and clear bookkeeping on failure. Preserve exact-size allocation: H.264 also uses the buffer size as its coded payload length, so reusing a larger allocation changes semantics. Fixes: 71f0cc967499 ("media: apple: avd: dedicated submit step") Signed-off-by: iconidentify <303316+iconidentify@users.noreply.github.com> --- drivers/media/platform/apple/avd/avd-drv.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 6c00e4df53774..90762f5079143 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -75,10 +75,8 @@ void fill_comp(struct avd_comp *comp, enum avd_image_fmt image_fmt, u32 width, int avd_buf_alloc(struct avd_dev *avd, struct avd_buf *buf, size_t size) { - if (!buf->cpu && size < buf->size) - return 0; - else if (buf->cpu) - avd_buf_free(avd, buf); + /* Keep size equal to the payload length used by the H.264 slice path. */ + avd_buf_free(avd, buf); if (size <= 0) return -ENOMEM; @@ -86,7 +84,11 @@ int avd_buf_alloc(struct avd_dev *avd, struct avd_buf *buf, size_t size) buf->size = size; buf->cpu = dma_alloc_coherent(avd->dev, buf->size, &buf->addr, GFP_KERNEL); - return buf->cpu ? 0 : -ENOMEM; + if (!buf->cpu) { + memset(buf, 0, sizeof(*buf)); + return -ENOMEM; + } + return 0; } void avd_buf_free(struct avd_dev *avd, struct avd_buf *buf) From 9822414869846a5ffc9c2f984abf164d5d43118b Mon Sep 17 00:00:00 2001 From: iconidentify <303316+iconidentify@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:03:30 -0700 Subject: [PATCH 0666/2680] media: apple: avd: complete requests when job allocation fails The AV1 and VP9 preambles apply request controls before allocating the instruction table. If that allocation fails, the source buffer is returned but the control request object is never completed. Clients waiting for the request can remain blocked or unable to reinitialize it. Complete the applied request before returning the allocation error, matching the other failure paths and the HEVC implementation. Fixes: 71f0cc967499 ("media: apple: avd: dedicated submit step") Signed-off-by: iconidentify <303316+iconidentify@users.noreply.github.com> --- drivers/media/platform/apple/avd/avd-av1.c | 4 +++- drivers/media/platform/apple/avd/avd-vp9.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-av1.c b/drivers/media/platform/apple/avd/avd-av1.c index 9c1ed0ca8193b..5bfa4623f372b 100644 --- a/drivers/media/platform/apple/avd/avd-av1.c +++ b/drivers/media/platform/apple/avd/avd-av1.c @@ -1291,8 +1291,10 @@ static int avd_av1_run(struct avd_ctx *ctx) run.frame->tile_info.tile_cols * run.frame->tile_info.tile_rows + 1); - if (ret) + if (ret) { + avd_run_postamble(ctx, &run.base); return ret; + } dst = vb2_to_avd_decoded_buf(&run.base.bufs.dst->vb2_buf); update_dec_buf_info(dst, run.seq, run.frame); diff --git a/drivers/media/platform/apple/avd/avd-vp9.c b/drivers/media/platform/apple/avd/avd-vp9.c index 16c126e9afb34..dc10c39cbe285 100644 --- a/drivers/media/platform/apple/avd/avd-vp9.c +++ b/drivers/media/platform/apple/avd/avd-vp9.c @@ -784,8 +784,10 @@ static int avd_vp9_run(struct avd_ctx *ctx) (1 << run.decode_params->tile_rows_log2) * (1 << run.decode_params->tile_cols_log2) + 1); - if (ret) + if (ret) { + avd_run_postamble(ctx, &run.base); return ret; + } init_probs(ctx, &run); From 96f84ff1cd99cb7a0905cdfe6d3ccc2c8169e3d5 Mon Sep 17 00:00:00 2001 From: iconidentify <303316+iconidentify@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:03:30 -0700 Subject: [PATCH 0667/2680] media: apple: avd: fill monochrome chroma after capture synchronization H.264 completion writes neutral chroma before vb2_buffer_done returns cache ownership to the CPU. With non-coherent MMAP capture, the following DMA synchronization can invalidate those dirty CPU writes. Record the pending fill on the destination buffer and perform it from buf_finish, after the framework has synchronized the buffer. Retain the chroma offset and bit depth per buffer so delayed dequeue cannot pick up another picture's format. Clear the pending fill on prepare and consume it only for successfully completed capture buffers. Fixes: f456f8a26f09 ("media: apple: avd: h264: fill chroma with grey for 4:0:0 pictures") Signed-off-by: iconidentify <303316+iconidentify@users.noreply.github.com> --- drivers/media/platform/apple/avd/avd-h264.c | 20 +++++------- drivers/media/platform/apple/avd/avd-v4l2.c | 36 ++++++++++++++++++++- drivers/media/platform/apple/avd/avd.h | 3 ++ 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index 7d89c8ca0e4bf..b0b51bfb24a9c 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -860,18 +860,14 @@ static void avd_h264_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, /* 4:0:0 has no chroma to decode; the picture is grey */ if (h264_ctx->monochrome && dst_buf && result == VB2_BUF_STATE_DONE) { const struct v4l2_pix_format_mplane *pix = &ctx->decoded_fmt.fmt.pix_mp; - u8 *y = vb2_plane_vaddr(&dst_buf->vb2_buf, 0); - size_t luma = (size_t)pix->plane_fmt[0].bytesperline * pix->height; - - if (y && ctx->image_fmt == AVD_IMG_FMT_420_10BIT) { - /* P010: 16-bit little-endian samples, 512 << 6 */ - for (size_t i = 0; i + 1 < luma / 2; i += 2) { - y[luma + i] = 0x00; - y[luma + i + 1] = 0x80; - } - } else if (y) { - memset(y + luma, 0x80, luma / 2); - } + struct avd_decoded_buffer *dst = + vb2_to_avd_decoded_buf(&dst_buf->vb2_buf); + + /* vb2 must return cache ownership before the CPU writes. */ + dst->grey_chroma_offset = + (size_t)pix->plane_fmt[0].bytesperline * pix->height; + dst->grey_chroma_10bit = + ctx->image_fmt == AVD_IMG_FMT_420_10BIT; } } } diff --git a/drivers/media/platform/apple/avd/avd-v4l2.c b/drivers/media/platform/apple/avd/avd-v4l2.c index e9e6ca9e24913..92258f39d46c2 100644 --- a/drivers/media/platform/apple/avd/avd-v4l2.c +++ b/drivers/media/platform/apple/avd/avd-v4l2.c @@ -823,13 +823,46 @@ static int avd_buf_prepare(struct vb2_buffer *vb) * (for OUTPUT buffers, if userspace passes 0 bytesused, v4l2-core sets * it to buffer length). */ - if (V4L2_TYPE_IS_CAPTURE(vq->type)) + if (V4L2_TYPE_IS_CAPTURE(vq->type)) { + vb2_to_avd_decoded_buf(vb)->grey_chroma_offset = 0; vb2_set_plane_payload(vb, 0, f->fmt.pix_mp.plane_fmt[0].sizeimage); + } return 0; } +static void avd_buf_finish(struct vb2_buffer *vb) +{ + struct avd_decoded_buffer *buf; + size_t luma; + u8 *y; + + if (!V4L2_TYPE_IS_CAPTURE(vb->vb2_queue->type)) + return; + + buf = vb2_to_avd_decoded_buf(vb); + luma = buf->grey_chroma_offset; + buf->grey_chroma_offset = 0; + if (!luma || vb->state != VB2_BUF_STATE_DONE) + return; + + /* VB2 has synchronized the capture buffer for CPU access. */ + y = vb2_plane_vaddr(vb, 0); + if (!y) + return; + + if (buf->grey_chroma_10bit) { + /* P010: 16-bit little-endian samples, 512 << 6. */ + for (size_t i = 0; i + 1 < luma / 2; i += 2) { + y[luma + i] = 0x00; + y[luma + i + 1] = 0x80; + } + } else { + memset(y + luma, 0x80, luma / 2); + } +} + static void avd_buf_queue(struct vb2_buffer *vb) { struct avd_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue); @@ -920,6 +953,7 @@ static void avd_stop_streaming(struct vb2_queue *q) const struct vb2_ops avd_queue_ops = { .queue_setup = avd_queue_setup, .buf_prepare = avd_buf_prepare, + .buf_finish = avd_buf_finish, .buf_queue = avd_buf_queue, .buf_out_validate = avd_buf_out_validate, .buf_request_complete = avd_buf_request_complete, diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index f026b45913c33..1a26ecc0a7518 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -115,6 +115,9 @@ struct avd_decoded_buffer { struct v4l2_m2m_buffer base; struct avd_comp comp; + /* Pending CPU fill, retained with the buffer until dequeue. */ + size_t grey_chroma_offset; + bool grey_chroma_10bit; union { struct avd_vp9_decoded_buffer_info vp9; From 2e106b4d71f4029c0958abe240ecc1331479b47d Mon Sep 17 00:00:00 2001 From: iconidentify <303316+iconidentify@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:24:34 -0700 Subject: [PATCH 0668/2680] media: apple: avd: release codec buffers when startup fails The codec startup error paths free only the private context after a DMA buffer allocation fails. Buffers allocated earlier in the same startup sequence become unreachable, so repeated failed STREAMON calls leak DMA memory for H.264, HEVC, VP9 and AV1. Reuse each codec's stop routine to unwind partially initialized contexts. The contexts are zero-initialized and avd_buf_free() accepts empty buffer descriptors. Clear ctx->priv after freeing the context on both startup failure and normal teardown. [ryan: adapted to 7.2.10 - the moved avd_h264_stop() keeps the kfree() of the slice table that asahi-wip b00e2172cbb5 added to it] Signed-off-by: iconidentify <303316+iconidentify@users.noreply.github.com> --- drivers/media/platform/apple/avd/avd-av1.c | 44 +++++++++--------- drivers/media/platform/apple/avd/avd-h264.c | 51 +++++++++++---------- drivers/media/platform/apple/avd/avd-hevc.c | 48 +++++++++---------- drivers/media/platform/apple/avd/avd-vp9.c | 46 +++++++++---------- 4 files changed, 95 insertions(+), 94 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-av1.c b/drivers/media/platform/apple/avd/avd-av1.c index 5bfa4623f372b..ff346201fef6b 100644 --- a/drivers/media/platform/apple/avd/avd-av1.c +++ b/drivers/media/platform/apple/avd/avd-av1.c @@ -1333,28 +1333,6 @@ static int avd_av1_alloc_bufs(struct avd_ctx *ctx) return 0; } -static int avd_av1_start(struct avd_ctx *ctx) -{ - struct avd_av1_ctx *av1_ctx; - int ret; - - av1_ctx = kzalloc(sizeof(*av1_ctx), GFP_KERNEL); - if (!av1_ctx) - return -ENOMEM; - - ctx->priv = av1_ctx; - ret = avd_av1_alloc_bufs(ctx); - if (ret) - goto err_free_ctx; - - return 0; - -err_free_ctx: - kfree(av1_ctx); - ctx->priv = NULL; - return ret; -} - static void avd_av1_stop(struct avd_ctx *ctx) { struct avd_av1_ctx *av1_ctx = ctx->priv; @@ -1379,6 +1357,28 @@ static void avd_av1_stop(struct avd_ctx *ctx) avd_buf_free(avd, &av1_ctx->bufs.probs); kfree(av1_ctx); + ctx->priv = NULL; +} + +static int avd_av1_start(struct avd_ctx *ctx) +{ + struct avd_av1_ctx *av1_ctx; + int ret; + + av1_ctx = kzalloc(sizeof(*av1_ctx), GFP_KERNEL); + if (!av1_ctx) + return -ENOMEM; + + ctx->priv = av1_ctx; + ret = avd_av1_alloc_bufs(ctx); + if (ret) + goto err_free_ctx; + + return 0; + +err_free_ctx: + avd_av1_stop(ctx); + return ret; } static enum avd_image_fmt avd_av1_get_image_fmt(struct avd_ctx *ctx, diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index b0b51bfb24a9c..1373faf42e5fa 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -590,6 +590,31 @@ static int avd_h264_validate_sps(struct avd_ctx *ctx, return 0; } +static void avd_h264_stop(struct avd_ctx *ctx) +{ + struct avd_h264_ctx *h264_ctx = ctx->priv; + struct avd_dev *dev = ctx->dev; + int i; + + if (!h264_ctx) + return; + + avd_buf_free(dev, &h264_ctx->bufs.pipe_state); + avd_buf_free(dev, &h264_ctx->bufs.inst); + avd_buf_free(dev, &h264_ctx->bufs.above_info); + avd_buf_free(dev, &h264_ctx->bufs.lf_above_info); + avd_buf_free(dev, &h264_ctx->bufs.lf_above); + avd_buf_free(dev, &h264_ctx->bufs.ip_above); + avd_buf_free(dev, &h264_ctx->bufs.mv_above_info); + + for (i = 0; i < h264_ctx->slice_num; i++) + avd_buf_free(dev, &h264_ctx->slices[i]); + kfree(h264_ctx->slices); + + kfree(h264_ctx); + ctx->priv = NULL; +} + static int avd_h264_start(struct avd_ctx *ctx) { struct avd_h264_ctx *h264_ctx; @@ -626,34 +651,10 @@ static int avd_h264_start(struct avd_ctx *ctx) return 0; err_free_ctx: - kfree(h264_ctx); - ctx->priv = NULL; + avd_h264_stop(ctx); return ret; } -static void avd_h264_stop(struct avd_ctx *ctx) -{ - struct avd_h264_ctx *h264_ctx = ctx->priv; - struct avd_dev *dev = ctx->dev; - int i; - - if (!h264_ctx) - return; - - avd_buf_free(dev, &h264_ctx->bufs.pipe_state); - avd_buf_free(dev, &h264_ctx->bufs.inst); - avd_buf_free(dev, &h264_ctx->bufs.above_info); - avd_buf_free(dev, &h264_ctx->bufs.lf_above_info); - avd_buf_free(dev, &h264_ctx->bufs.lf_above); - avd_buf_free(dev, &h264_ctx->bufs.ip_above); - avd_buf_free(dev, &h264_ctx->bufs.mv_above_info); - - for (i = 0; i < h264_ctx->slice_num; i++) - avd_buf_free(dev, &h264_ctx->slices[i]); - kfree(h264_ctx->slices); - kfree(h264_ctx); -} - static void avd_h264_run_preamble(struct avd_ctx *ctx, struct avd_h264_run *run) { struct v4l2_ctrl *ctrl; diff --git a/drivers/media/platform/apple/avd/avd-hevc.c b/drivers/media/platform/apple/avd/avd-hevc.c index cc71ac58bea34..97880fc3016dc 100644 --- a/drivers/media/platform/apple/avd/avd-hevc.c +++ b/drivers/media/platform/apple/avd/avd-hevc.c @@ -1047,6 +1047,29 @@ static int avd_hevc_alloc_bufs(struct avd_ctx *ctx) return 0; } +static void avd_hevc_stop(struct avd_ctx *ctx) +{ + struct avd_hevc_ctx *hevc_ctx = ctx->priv; + struct avd_dev *avd = ctx->dev; + + if (!hevc_ctx) + return; + + avd_buf_free(avd, &hevc_ctx->bufs.pipe_state); + avd_buf_free(avd, &hevc_ctx->bufs.inst); + avd_buf_free(avd, &hevc_ctx->bufs.mv_above_info); + avd_buf_free(avd, &hevc_ctx->bufs.az_above); + avd_buf_free(avd, &hevc_ctx->bufs.ip_above); + avd_buf_free(avd, &hevc_ctx->bufs.lf_above); + avd_buf_free(avd, &hevc_ctx->bufs.lf_above_info); + avd_buf_free(avd, &hevc_ctx->bufs.lf_left); + avd_buf_free(avd, &hevc_ctx->bufs.lf_left_info); + avd_buf_free(avd, &hevc_ctx->bufs.sw_left); + + kfree(hevc_ctx); + ctx->priv = NULL; +} + static int avd_hevc_start(struct avd_ctx *ctx) { struct avd_hevc_ctx *hevc_ctx; @@ -1064,8 +1087,7 @@ static int avd_hevc_start(struct avd_ctx *ctx) return 0; err_free_ctx: - kfree(hevc_ctx); - ctx->priv = NULL; + avd_hevc_stop(ctx); return ret; } @@ -1234,28 +1256,6 @@ static int avd_hevc_compute_tiles(struct avd_ctx *ctx, struct avd_hevc_run *run) return 0; } -static void avd_hevc_stop(struct avd_ctx *ctx) -{ - struct avd_hevc_ctx *hevc_ctx = ctx->priv; - struct avd_dev *avd = ctx->dev; - - if (!hevc_ctx) - return; - - avd_buf_free(avd, &hevc_ctx->bufs.pipe_state); - avd_buf_free(avd, &hevc_ctx->bufs.inst); - avd_buf_free(avd, &hevc_ctx->bufs.mv_above_info); - avd_buf_free(avd, &hevc_ctx->bufs.az_above); - avd_buf_free(avd, &hevc_ctx->bufs.ip_above); - avd_buf_free(avd, &hevc_ctx->bufs.lf_above); - avd_buf_free(avd, &hevc_ctx->bufs.lf_above_info); - avd_buf_free(avd, &hevc_ctx->bufs.lf_left); - avd_buf_free(avd, &hevc_ctx->bufs.lf_left_info); - avd_buf_free(avd, &hevc_ctx->bufs.sw_left); - - kfree(hevc_ctx); -} - static int avd_hevc_run_preamble(struct avd_ctx *ctx, struct avd_hevc_run *run) { struct v4l2_ctrl *ctrl, *sl, *ep; diff --git a/drivers/media/platform/apple/avd/avd-vp9.c b/drivers/media/platform/apple/avd/avd-vp9.c index dc10c39cbe285..4cdcb6594c4d2 100644 --- a/drivers/media/platform/apple/avd/avd-vp9.c +++ b/drivers/media/platform/apple/avd/avd-vp9.c @@ -981,6 +981,28 @@ avd_init_v4l2_vp9_count_tbl(struct avd_ctx *ctx) } } +static void avd_vp9_stop(struct avd_ctx *ctx) +{ + struct avd_vp9_ctx *vp9_ctx = ctx->priv; + struct avd_dev *avd = ctx->dev; + + avd_buf_free(avd, &vp9_ctx->bufs.pipe_state); + avd_buf_free(avd, &vp9_ctx->bufs.inst); + avd_buf_free(avd, &vp9_ctx->bufs.probs); + avd_buf_free(avd, &vp9_ctx->bufs.counts); + avd_buf_free(avd, &vp9_ctx->bufs.seg); + avd_buf_free(avd, &vp9_ctx->bufs.color); + avd_buf_free(avd, &vp9_ctx->bufs.ip_above); + avd_buf_free(avd, &vp9_ctx->bufs.lf_above); + avd_buf_free(avd, &vp9_ctx->bufs.above_info); + avd_buf_free(avd, &vp9_ctx->bufs.az_left); + avd_buf_free(avd, &vp9_ctx->bufs.lf_left_info); + avd_buf_free(avd, &vp9_ctx->bufs.lf_left); + + kfree(vp9_ctx); + ctx->priv = NULL; +} + static int avd_vp9_start(struct avd_ctx *ctx) { struct avd_vp9_ctx *vp9_ctx; @@ -1000,32 +1022,10 @@ static int avd_vp9_start(struct avd_ctx *ctx) return 0; err_free_ctx: - kfree(vp9_ctx); - ctx->priv = NULL; + avd_vp9_stop(ctx); return ret; } -static void avd_vp9_stop(struct avd_ctx *ctx) -{ - struct avd_vp9_ctx *vp9_ctx = ctx->priv; - struct avd_dev *avd = ctx->dev; - - avd_buf_free(avd, &vp9_ctx->bufs.pipe_state); - avd_buf_free(avd, &vp9_ctx->bufs.inst); - avd_buf_free(avd, &vp9_ctx->bufs.probs); - avd_buf_free(avd, &vp9_ctx->bufs.counts); - avd_buf_free(avd, &vp9_ctx->bufs.seg); - avd_buf_free(avd, &vp9_ctx->bufs.color); - avd_buf_free(avd, &vp9_ctx->bufs.ip_above); - avd_buf_free(avd, &vp9_ctx->bufs.lf_above); - avd_buf_free(avd, &vp9_ctx->bufs.above_info); - avd_buf_free(avd, &vp9_ctx->bufs.az_left); - avd_buf_free(avd, &vp9_ctx->bufs.lf_left_info); - avd_buf_free(avd, &vp9_ctx->bufs.lf_left); - - kfree(vp9_ctx); -} - static enum avd_image_fmt avd_vp9_get_image_fmt(struct avd_ctx *ctx, struct v4l2_ctrl *ctrl) { From 9e400bcb62b0d912ddbfc94ebca59ab3d646d89c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 19:44:14 +1100 Subject: [PATCH 0669/2680] media: apple: avd: h264: Fail the start without a slice table avd_h264_start() jumps to its error exit when the slice table allocation fails, but ret still holds the 0 that avd_h264_validate_sps() returned. The context is freed and ctx->priv cleared, yet the start reports success, and the first slice then runs on a NULL context. Return -ENOMEM in that case. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-h264.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index 1373faf42e5fa..cf17ed297d970 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -641,8 +641,10 @@ static int avd_h264_start(struct avd_ctx *ctx) h264_ctx->slices = kzalloc(sizeof(*h264_ctx->slices) * h264_ctx->alloc_slice_num, GFP_KERNEL); - if (!h264_ctx->slices) + if (!h264_ctx->slices) { + ret = -ENOMEM; goto err_free_ctx; + } ret = avd_h264_alloc_bufs(ctx); if (ret) From ceda049b3116f90514ba080de693ba489b38d4ff Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 19:44:45 +1100 Subject: [PATCH 0670/2680] media: apple: avd: h264: Always grow a full slice table avd_h264_run() stores each slice in the slice table before it checks for a job table, and only grows the slice table while a job table exists. Slices that arrive without one (a client that joins a stream in the middle of a frame and holds the capture buffer) are stored and counted, rejected with -EINVAL, and kept until the capture buffer is released, so enough of them write past the end of the slice table. The growth also multiplies the size by 3/2 in integer arithmetic, which leaves the one-entry table of a picture one macroblock row high at one entry, so its second slice writes past the end as well. Grow the slice table whenever it is full, by at least one entry, and grow the job table with it only when there is one. The fixed table the growing one replaced rejected the slice at its limit instead. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-h264.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index cf17ed297d970..8876784820d22 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -695,7 +695,9 @@ static int avd_h264_realloc_slices(struct avd_ctx *ctx) size_t alloc_slice_num; tmp = h264_ctx->slices; - alloc_slice_num = (h264_ctx->alloc_slice_num * 3) / 2; + /* grow by half, and by at least one for a one-row picture */ + alloc_slice_num = h264_ctx->alloc_slice_num + + max_t(size_t, h264_ctx->alloc_slice_num / 2, 1); h264_ctx->slices = kzalloc(sizeof(*h264_ctx->slices) * alloc_slice_num, GFP_KERNEL); if (!h264_ctx->slices) { @@ -709,6 +711,10 @@ static int avd_h264_realloc_slices(struct avd_ctx *ctx) sizeof(*h264_ctx->slices) * h264_ctx->slice_num); kfree(tmp); + /* a slice that arrives before the frame's header has no job table yet */ + if (!ctx->job.segments) + return 0; + tmp = ctx->job.segments; ctx->job.segments = kvcalloc(alloc_slice_num + 1, sizeof(*ctx->job.segments), GFP_KERNEL); @@ -781,8 +787,7 @@ static int avd_h264_run(struct avd_ctx *ctx) avd_h264_run_preamble(ctx, &run); - if (ctx->job.segments && - h264_ctx->slice_num >= h264_ctx->alloc_slice_num) { + if (h264_ctx->slice_num >= h264_ctx->alloc_slice_num) { ret = avd_h264_realloc_slices(ctx); if (ret) goto err_free_jobs; From 64ea945dec1d050ca4e4b50a11eb06a50f7a6560 Mon Sep 17 00:00:00 2001 From: iconidentify <303316+iconidentify@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:24:34 -0700 Subject: [PATCH 0671/2680] media: apple: avd: handle AV1 work buffer allocation failures AV1 ignores the return value from avd_av1_alloc_work_bufs() and continues building and submitting a frame after an allocation fails. The resulting command can reference missing or incomplete working storage. Check the allocation result before updating destination metadata or probabilities. Complete the applied request controls and return the error so avd_device_run() finishes the buffers with an error without submitting the job. Partially allocated work buffers remain owned by the context and are released by retry allocation or codec teardown. Signed-off-by: iconidentify <303316+iconidentify@users.noreply.github.com> --- drivers/media/platform/apple/avd/avd-av1.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/avd/avd-av1.c b/drivers/media/platform/apple/avd/avd-av1.c index ff346201fef6b..f129b7d3e4bef 100644 --- a/drivers/media/platform/apple/avd/avd-av1.c +++ b/drivers/media/platform/apple/avd/avd-av1.c @@ -1296,11 +1296,16 @@ static int avd_av1_run(struct avd_ctx *ctx) return ret; } + ret = avd_av1_alloc_work_bufs(ctx, &run); + if (ret) { + avd_run_postamble(ctx, &run.base); + return ret; + } + dst = vb2_to_avd_decoded_buf(&run.base.bufs.dst->vb2_buf); update_dec_buf_info(dst, run.seq, run.frame); avd_av1_set_prob(ctx, &run); - avd_av1_alloc_work_bufs(ctx, &run); set_header(ctx, &run); set_tiles(ctx, &run); From 421c5595c872013ce13ea1b9d1ee04a7b90eac47 Mon Sep 17 00:00:00 2001 From: iconidentify <303316+iconidentify@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:24:34 -0700 Subject: [PATCH 0672/2680] media: apple: avd: return source buffers when streaming fails to start vb2 requires start_streaming() to return all driver-owned buffers in the QUEUED state on failure. AVD leaves those buffers on the mem2mem source ready list when codec startup fails. vb2's fallback returns their buffer state but cannot remove the driver's ready links, so a direct STREAMON retry enqueues the same links again. Drain the source ready queue on every output startup error and return each buffer with VB2_BUF_STATE_QUEUED. Preserve request controls for the retry; the normal queue cleanup helper completes them and is unsuitable here. Capture startup and successful output startup remain unchanged. Signed-off-by: iconidentify <303316+iconidentify@users.noreply.github.com> --- drivers/media/platform/apple/avd/avd-v4l2.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-v4l2.c b/drivers/media/platform/apple/avd/avd-v4l2.c index 92258f39d46c2..4622e92055486 100644 --- a/drivers/media/platform/apple/avd/avd-v4l2.c +++ b/drivers/media/platform/apple/avd/avd-v4l2.c @@ -890,22 +890,31 @@ static int avd_start_streaming(struct vb2_queue *q, unsigned int count) { struct avd_ctx *ctx = vb2_get_drv_priv(q); const struct avd_coded_fmt_desc *desc; + struct vb2_v4l2_buffer *vbuf; int ret; if (V4L2_TYPE_IS_CAPTURE(q->type)) return 0; desc = ctx->coded_fmt_desc; - if (WARN_ON(!desc)) - return -EINVAL; + if (WARN_ON(!desc)) { + ret = -EINVAL; + goto err_return_buffers; + } if (desc->ops->start) { ret = desc->ops->start(ctx); if (ret) - return ret; + goto err_return_buffers; } return 0; + +err_return_buffers: + /* Keep requests queued for a later STREAMON retry. */ + while ((vbuf = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx))) + v4l2_m2m_buf_done(vbuf, VB2_BUF_STATE_QUEUED); + return ret; } static void avd_queue_cleanup(struct vb2_queue *vq, u32 state) From 7ab91efd668a8cc4ec1dddcb63f0db6df8947e6a Mon Sep 17 00:00:00 2001 From: Ace <7362641+Acelogic@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:10:51 -0400 Subject: [PATCH 0673/2680] dt-bindings: media: apple,avd: document T8140 decoder Add the compatible used by the J700 AVD node. The existing four-register layout and two interrupts cover this generation. --- Documentation/devicetree/bindings/media/apple,avd.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/media/apple,avd.yaml b/Documentation/devicetree/bindings/media/apple,avd.yaml index 46a0cfc226efe..a724e982b461c 100644 --- a/Documentation/devicetree/bindings/media/apple,avd.yaml +++ b/Documentation/devicetree/bindings/media/apple,avd.yaml @@ -26,6 +26,7 @@ properties: - apple,t8112-avd - apple,t6020-avd - apple,t8122-avd + - apple,t8140-avd reg: items: From edabe1d44401190ddd7ff14adf162c5fa4b906dc Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 2 Oct 2026 00:10:51 -0400 Subject: [PATCH 0674/2680] media: apple: avd: enable T8140 codec submission and buffer layout T8140 has sparse codec-specific VP blocks rather than the sequential layout used by earlier variants. Its HEVC stream uses DMA configuration words and an extended scratch-address table. Apply the J700 queue pacing and polling behavior to this variant while preserving earlier chips' behavior. Align the AV1 auxiliary storage and HEVC MV tail to keep their addresses outside the compressed frame data and aligned to the hardware granularity. Adapt DJ's existing J700 bring-up, integrated by Ryan, onto the public decoder series in PR #45. Preserve the original driver notices and authors. The legacy FIFO-mask experiment is not part of this port. Tested on J700/T8140 with Aurora v5-t0 firmware: H.264, HEVC, VP9 and AV1 hardware downloads matched 1740 software-decoded frames in 34 runs, including 8/10-bit cases, vaGetImage, repeated opens and H.264/HEVC/VP9 seek. Firmware source: https://github.com/aurora-silicon/avd-fw Revision: 9cbde5c7d993844bf086318974958b11076600a0 Document the firmware build source and installation path in Kconfig help. Retain the firmware project's license and contributor attribution. Co-authored-by: Ryan Murray Co-authored-by: Ace <7362641+Acelogic@users.noreply.github.com> --- drivers/media/platform/apple/avd/Kconfig | 6 +++ drivers/media/platform/apple/avd/avd-av1.c | 3 ++ drivers/media/platform/apple/avd/avd-drv.c | 21 ++++++-- drivers/media/platform/apple/avd/avd-hevc.c | 57 +++++++++++++++------ drivers/media/platform/apple/avd/avd.h | 3 ++ 5 files changed, 71 insertions(+), 19 deletions(-) diff --git a/drivers/media/platform/apple/avd/Kconfig b/drivers/media/platform/apple/avd/Kconfig index 930b89d546cf5..d01634d938857 100644 --- a/drivers/media/platform/apple/avd/Kconfig +++ b/drivers/media/platform/apple/avd/Kconfig @@ -11,3 +11,9 @@ config VIDEO_APPLE_AVD select VIDEOBUF2_DMA_CONTIG select V4L2_H264 select V4L2_VP9 + help + Support the Apple Silicon stateless video decoder. + + Build the decoder firmware from https://github.com/aurora-silicon/avd-fw + and install it under /lib/firmware/apple/. J700/T8140 uses the + avd-fw-v5-t0.bin image. Firmware is supplied separately from this driver. diff --git a/drivers/media/platform/apple/avd/avd-av1.c b/drivers/media/platform/apple/avd/avd-av1.c index f129b7d3e4bef..cd8e876b4e10a 100644 --- a/drivers/media/platform/apple/avd/avd-av1.c +++ b/drivers/media/platform/apple/avd/avd-av1.c @@ -1430,6 +1430,9 @@ static void avd_av1_submit(struct avd_ctx *ctx) static void avd_av1_adjust_decoded_fmt(struct avd_ctx *ctx, struct v4l2_pix_format_mplane *pix_mp) { + /* Keep both auxiliary regions aligned after compressed storage. */ + pix_mp->plane_fmt[0].sizeimage = + ALIGN(pix_mp->plane_fmt[0].sizeimage, AVD_ALIGN); pix_mp->plane_fmt[0].sizeimage += ALIGN(AVD_CDFS_SIZE, AVD_ALIGN); pix_mp->plane_fmt[0].sizeimage += ALIGN(avd_color_size(pix_mp->width, pix_mp->height), AVD_ALIGN); diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 90762f5079143..9751b8d292ffe 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -130,8 +130,10 @@ static int avd_wait_submission_queue(struct avd_ctx *ctx, int vp) } if (cur >= max / 2) { - /* TODO: to high? low? Has weird side effects??? */ - usleep_range(500, 650); + if (avd->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER) + usleep_range(100, 150); + else + usleep_range(500, 650); } return 0; } @@ -171,6 +173,8 @@ int avd_submit_job(struct avd_ctx *ctx) ctx->fifo_idx = 0; for (i = 0; i < sub->codec; i++) vp += avd->variant->vp_slots[i]; + if (avd->variant->vp_slot_base[sub->codec]) + vp = avd->variant->vp_slot_base[sub->codec]; /* * avd_device_run() keeps this job from being finished (and the next one @@ -197,7 +201,9 @@ int avd_submit_job(struct avd_ctx *ctx) seg = &segments[idx]; for (i = 0; i < seg->num; i++) writel(seg->instructions[i], reg); - if (avd_wait_submission_queue(ctx, vp)) + if ((!(avd->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER) || + sub->codec == AVD_CODEC_HEVC) && + avd_wait_submission_queue(ctx, vp)) break; if (idx == num) { /* A completion from here on belongs to this job. */ @@ -729,13 +735,19 @@ static const struct avd_variant avd_t8122_variant = { }; static const struct avd_variant avd_t8140_variant = { - /* This is filled in using the tunables, what vp/pp to use? */ .vp_slots = { [AVD_CODEC_HEVC] = 2, [AVD_CODEC_H264] = 1, [AVD_CODEC_VP9] = 1, [AVD_CODEC_AV1] = 1, }, + /* Tier 0 has holes between the codec-specific VP blocks. */ + .vp_slot_base = { + [AVD_CODEC_HEVC] = 0, + [AVD_CODEC_H264] = 4, + [AVD_CODEC_VP9] = 8, + [AVD_CODEC_AV1] = 9, + }, .fifo_slots = 7, .configure_stream = t8122_configure_stream, .capabilities = AVD_CAPABILITY_HEVC | @@ -748,6 +760,7 @@ static const struct avd_variant avd_t8140_variant = { .submit_offset = 0x40, .submit_queue_max_offset = 0x44, .submit_queue_status_offset = 0x7c, + .quirks = AVD_QUIRK_HEVC_V5_HEADER, }; static const struct avd_variant avd_t8132_variant = { diff --git a/drivers/media/platform/apple/avd/avd-hevc.c b/drivers/media/platform/apple/avd/avd-hevc.c index 97880fc3016dc..9dfa20ab71913 100644 --- a/drivers/media/platform/apple/avd/avd-hevc.c +++ b/drivers/media/platform/apple/avd/avd-hevc.c @@ -43,6 +43,7 @@ #define HEVC_FLAG_TMVP_EN(v) FIELD_PREP(BIT(21), !!(v)) #define HEVC_SCL_DIMS 0x127ffff +#define HEVC_DMA_CONFIG BIT(26) static inline u32 mv_color_size(u32 w, u32 h) { @@ -107,7 +108,8 @@ static void stream_refs(struct avd_ctx *ctx, struct avd_hevc_run *run) dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); - push(0, ""); + push(ctx->dev->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER ? + HEVC_DMA_CONFIG : 0, "reference_dma_config"); pusha(hevc_ctx->bufs.mv_above_info.addr, "mv_above_info", 7); pusha(run->addresses.mv_color, "mv_color", 0); @@ -289,11 +291,14 @@ static void hevc_set_flags(struct avd_ctx *ctx, struct avd_hevc_run *run) static void set_header(struct avd_ctx *ctx, struct avd_hevc_run *run) { const struct v4l2_ctrl_hevc_sps *sps = run->sps; + const struct v4l2_ctrl_hevc_pps *pps = run->pps; struct avd_dev *avd = ctx->dev; struct avd_hevc_ctx *hevc_ctx = ctx->priv; u32 bytesperline; u32 width = sps->pic_width_in_luma_samples; u32 height = sps->pic_height_in_luma_samples; + u32 dma_config = avd->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER ? + HEVC_DMA_CONFIG : 0; bool is_intra = run->sl[0].slice_type == V4L2_HEVC_SLICE_TYPE_I; @@ -340,27 +345,45 @@ static void set_header(struct avd_ctx *ctx, struct avd_hevc_run *run) !(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)), "hdr_98_const_30"); - push(0, ""); + push(dma_config, "dma_config"); push(0, ""); if (avd->variant->revision == 3) push(0, "zero"); - push(0, ""); - push(0, ""); + push(dma_config, "dma_config"); + push(dma_config, "dma_config"); if (avd->variant->revision == 3) push(0, "zero"); else if (!(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)) pusha(hevc_ctx->bufs.pipe_state.addr, "pipe_state", 0); - pusha(hevc_ctx->bufs.ip_above.addr, "ip_above", 0); - pusha(hevc_ctx->bufs.lf_above.addr, "lf_above", 1); - pusha(hevc_ctx->bufs.lf_above_info.addr, "lf_above_info", 2); - pusha(hevc_ctx->bufs.lf_left.addr, "lf_left", 3); - pusha(hevc_ctx->bufs.lf_left_info.addr, "lf_left_info", 4); - pusha(hevc_ctx->bufs.az_above.addr, "az_above", 8); - pusha(hevc_ctx->bufs.sw_left.addr, "sw_left", 9); + pusha(hevc_ctx->bufs.ip_above.addr, "scratch_addr", 0); + pusha(hevc_ctx->bufs.lf_above.addr, "scratch_addr", 1); + pusha(hevc_ctx->bufs.lf_above_info.addr, "scratch_addr", 2); + + if ((pps->flags & V4L2_HEVC_PPS_FLAG_TILES_ENABLED) || + !(avd->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER)) { + pusha(hevc_ctx->bufs.lf_left.addr, + "scratch_addr", 3); + pusha(hevc_ctx->bufs.lf_left_info.addr, + "scratch_addr", 4); + if (avd->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER) + pusha(0, "reserved_scratch", 0); + pusha(hevc_ctx->bufs.az_above.addr, + "scratch_addr", 8); + pusha(hevc_ctx->bufs.sw_left.addr, + "scratch_addr", 9); + } else { + pusha(0, "", 3); + pusha(0, "", 4); + if (avd->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER) + pusha(0, "reserved_scratch", 0); + pusha(hevc_ctx->bufs.az_above.addr, + "scratch_addr", 8); + pusha(0, "", 9); + } push(0, ""); @@ -621,8 +644,9 @@ static void stream_slice_mv(struct avd_ctx *ctx, struct avd_hevc_run *run, dma_addr_t mv_color_addr = vb2_dma_contig_plane_dma_addr(&ref->base.vb.vb2_buf, 0) + - (ref->base.vb.planes[0].length - - mv_color_size(fmt_width(ctx), fmt_height(ctx))); + round_down(ref->base.vb.planes[0].length - + mv_color_size(fmt_width(ctx), fmt_height(ctx)), + AVD_ALIGN); pusha(mv_color_addr, "slc_bd4_sps_tile_addr2_lsb8", decode->dpb[ref_list[sl->collocated_ref_idx]] .pic_order_cnt_val); @@ -979,7 +1003,9 @@ static void update_dec_buf_info(struct avd_decoded_buffer *buf, static void avd_hevc_adjust_decoded_fmt(struct avd_ctx *ctx, struct v4l2_pix_format_mplane *pix_mp) { - pix_mp->plane_fmt[0].sizeimage += + /* Keep the MV table outside 128-byte-aligned compressed storage. */ + pix_mp->plane_fmt[0].sizeimage = + ALIGN(pix_mp->plane_fmt[0].sizeimage, AVD_ALIGN) + mv_color_size(pix_mp->width, pix_mp->height); } @@ -1312,7 +1338,8 @@ static int avd_hevc_run_preamble(struct avd_ctx *ctx, struct avd_hevc_run *run) mv_color_len = mv_color_size(fmt_width(ctx), fmt_height(ctx)); - run->addresses.mv_color = run->base.y_out + (dst_len - mv_color_len); + run->addresses.mv_color = + run->base.y_out + round_down(dst_len - mv_color_len, AVD_ALIGN); return 0; } diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index 1a26ecc0a7518..e5b991a00fa47 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -32,6 +32,8 @@ /* Shifts addreses right and bytesperline?? */ #define AVD_QUIRK_LSR BIT(0) #define AVD_QUIRK_NO_PIPE_STATE BIT(1) +/* HEVC DMA configuration and extended scratch-address table. */ +#define AVD_QUIRK_HEVC_V5_HEADER BIT(2) #define VP_SLOT_NONE 255 #define INST_FIFO_SLOT_NONE 255 @@ -176,6 +178,7 @@ struct avd_coded_fmt_desc { struct avd_variant { unsigned int vp_slots[4]; + unsigned int vp_slot_base[4]; unsigned int fifo_slots; unsigned int capabilities; void (*configure_stream)(struct avd_dev *avd, dma_addr_t addr, From 3f0d94fc29046e90c92521d2220625bc67ad40e9 Mon Sep 17 00:00:00 2001 From: Acelogic <7362641+Acelogic@users.noreply.github.com> Date: Fri, 2 Oct 2026 02:06:57 -0400 Subject: [PATCH 0675/2680] dt-bindings: media: describe Apple T8140 AVE Describe the encoder register blocks, paired DARTs, five power domains and two bootloader-supplied firmware reservations used by J700. Signed-off-by: Acelogic <7362641+Acelogic@users.noreply.github.com> --- .../bindings/media/apple,t8140-ave.yaml | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 Documentation/devicetree/bindings/media/apple,t8140-ave.yaml diff --git a/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml b/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml new file mode 100644 index 0000000000000..110ebbc3d09e2 --- /dev/null +++ b/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/media/apple,t8140-ave.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 video encoder + +maintainers: + - Miguel Cruz <7362641+Acelogic@users.noreply.github.com> + +description: | + The T8140 AVE coprocessor encodes video using firmware preloaded by the + bootloader. Its two DART instances translate the same stream. The + bootloader supplies two reserved firmware regions and their IOVA mappings. + +properties: + compatible: + const: apple,t8140-ave + + reg: + items: + - description: Codec registers + - description: Mailbox registers + - description: Coprocessor registers + + reg-names: + items: + - const: codec + - const: mailbox + - const: coproc + + interrupts: + maxItems: 5 + + power-domains: + maxItems: 5 + + power-domain-names: + items: + - const: dma + - const: pipe4 + - const: pipe5 + - const: me0 + - const: me1 + + iommus: + maxItems: 2 + + memory-region: + items: + - description: Preloaded firmware text + - description: Preloaded firmware data + +required: + - compatible + - reg + - reg-names + - interrupts + - power-domains + - power-domain-names + - iommus + +additionalProperties: false From 40fec27011700ff20009605a6f1ca36a15d85810 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 05:17:22 -0400 Subject: [PATCH 0676/2680] drm/asahi: bring the cleanroom driver up on Neo hardware MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Acquire both GPU ASC lifecycle handles through the mailbox provider rather than looking for a nonexistent secondary platform resource. Initialize the firmware registry and queue/buffer arrays in place to bound the probe stack. Keep firmware and client metrics addresses in named fields. Their reversed order caused the first render to fault in firmware at the client address; the corrected addresses pass both light and heavy rendering on hardware. Report failed spared work to userspace independently of permanent VM admission failure, and skip already successful internal prefix fences. Validation on local Neo with unchanged Mesa: desktop at 2408x1506, four exact-image game replays, native/4K userspace checks, render/compute depth 1/2/4/8/16, four concurrent clients, and 45 matched benchmark windows. Static probe chain drops from 24784 to 13584 bytes (16384-byte audit limit). Latest named-address form also passes the 15-window hardware matrix. This is hardware bring-up, not a full-parity or leak-free qualification. [ryan: adapted to 7.2.10 — minimal; for DJ: CpuControl drops the 7.2 Devres register mapping from the T8140 interface commit in favour of the mailbox provider, as in the original; probe takes Core<'_>] Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit fbfb5d96f25efbf2b1571ca635b195e0ad4a54be) --- drivers/gpu/drm/asahi/file.rs | 2 +- drivers/gpu/drm/asahi/g17/buffer.rs | 32 ++++---- drivers/gpu/drm/asahi/g17/completion.rs | 8 +- drivers/gpu/drm/asahi/g17/fw/buffer.rs | 14 +++- drivers/gpu/drm/asahi/g17/kick.rs | 8 +- drivers/gpu/drm/asahi/g17/mod.rs | 20 +++-- drivers/gpu/drm/asahi/g17/queue/mod.rs | 6 ++ drivers/gpu/drm/asahi/g17/runtime.rs | 32 +++++--- .../drm/asahi/g17/runtime/render/prepare.rs | 10 +-- drivers/gpu/drm/asahi/g17/status.rs | 34 ++++++-- drivers/gpu/drm/asahi/regs.rs | 79 ++++++++++++------- 11 files changed, 151 insertions(+), 94 deletions(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index a772374bb4f98..76757dae0b182 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -406,7 +406,7 @@ impl File { let status = Self::vm_status(file, vm_id)?; UserSlice::new(user, size) .writer() - .write(&[vm_id, 0, status.get() as u32, 0])?; + .write(&[vm_id, 0, status.reported_error() as u32, 0])?; Ok(0) } diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index 3daa693fe7dbc..10bcf13c68789 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -34,21 +34,17 @@ pub(crate) struct BufferIds { next_token: u64, } impl BufferIds { - pub(crate) fn new() -> Self { - let mut returned = [0; BUFFER_IDS]; - for (index, entry) in returned.iter_mut().enumerate() { - *entry = (BUFFER_IDS - index - 1) as u8; - } - Self { - entries: [BufferId { + pub(crate) fn new() -> impl Init { + kernel::try_init!(Self { + entries <- pin_init::init_array_from_fn(|_| BufferId { references: 0, owner: false, token: 0, - }; BUFFER_IDS], - returned, + }), + returned <- pin_init::init_array_from_fn(|index| (BUFFER_IDS - index - 1) as u8), returned_count: BUFFER_IDS, next_token: 0, - } + }) } fn remove_returned(&mut self, id: u8) { if let Some(at) = self.returned[..self.returned_count] @@ -458,14 +454,14 @@ pub(crate) struct MetricsLease { _backing: Arc, allocator: Arc, slot: usize, - addresses: [u64; 2], + addresses: fw::MetricsAddresses, } impl MetricsLease { /// Clears only the selected retired metrics entry before reusing it. pub(crate) fn new( ids: &Arc, object: &Arc, - bases: [u64; 2], + bases: fw::MetricsAddresses, ) -> Result { let mut selected = None; for (bank, used) in ids.used.iter().enumerate() { @@ -499,13 +495,13 @@ impl MetricsLease { _backing: object.clone(), allocator: ids.clone(), slot, - addresses: [0; 2], + addresses: fw::MetricsAddresses::default(), }; let offset = slot * fw::METRICS_STRIDE; - lease.addresses = [ - bases[0].checked_add(offset as u64).ok_or(EOVERFLOW)?, - bases[1].checked_add(offset as u64).ok_or(EOVERFLOW)?, - ]; + lease.addresses = fw::MetricsAddresses { + firmware: bases.firmware.checked_add(offset as u64).ok_or(EOVERFLOW)?, + client: bases.client.checked_add(offset as u64).ok_or(EOVERFLOW)?, + }; let words = object.pointer(offset, fw::METRICS_STRIDE)?.cast::(); object.word(offset)?; for index in 0..fw::SCENES { @@ -631,7 +627,7 @@ impl Manager { Ok(self.graph.gpu_va() + fw::SCENE_TABLE as u64 + (index * size_of::()) as u64) } pub(crate) fn scene_registers(&self, index: usize) -> Result<(u64, u64)> { - fw::scene_registers(index, self.scratch, self.metrics.addresses[1]) + fw::scene_registers(index, self.scratch, self.metrics.addresses.client) } pub(crate) fn prepare_scene(&mut self, ordinal: u64) -> Result { let scene = self.scenes.select(ordinal)?; diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index a37c7fb045d69..2c66d745af112 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -324,7 +324,9 @@ impl Completion { Ok(()) }); if let Err(error) = result { - if error != ECANCELED && !self.spared() { + if self.spared() { + self.status.report_failure(error); + } else if error != ECANCELED { self.status.record(error); } self.fence.set_error(error); @@ -355,7 +357,9 @@ impl Completion { if let Some(guard) = self.vm_job.lock().as_ref() { guard.quarantine(); } - if error != ECANCELED && !self.spared() { + if self.spared() { + self.status.report_failure(error); + } else if error != ECANCELED { self.status.record(error); } self.fence.set_error(error); diff --git a/drivers/gpu/drm/asahi/g17/fw/buffer.rs b/drivers/gpu/drm/asahi/g17/fw/buffer.rs index 3ed336dbadaad..ec6797637bdbe 100644 --- a/drivers/gpu/drm/asahi/g17/fw/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/fw/buffer.rs @@ -120,11 +120,17 @@ static_assert!(core::mem::offset_of!(State, block_table) == 0x44); static_assert!(core::mem::offset_of!(State, discard) == 0x84); static_assert!(core::mem::offset_of!(Scene, generation) == 0x4c); +/// The same metrics storage in the firmware and application address spaces. +#[derive(Clone, Copy, Default)] +pub(crate) struct MetricsAddresses { + pub(crate) firmware: u64, + pub(crate) client: u64, +} + pub(crate) struct GraphArgs<'a> { pub(crate) fw_va: u64, pub(crate) page_list_client: u64, - /// Firmware and client views, respectively, of this graph's metrics lease. - pub(crate) metrics: [u64; 2], + pub(crate) metrics: MetricsAddresses, pub(crate) blocks: &'a [u64], pub(crate) scratch: u64, pub(crate) discard: u64, @@ -168,8 +174,8 @@ impl Graph { } } for (index, scene) in self.scenes.iter_mut().enumerate() { - scene.metric_client = offset_va(args.metrics[1], (index * 4) as u64)?; - scene.metric_fw = offset_va(args.metrics[0], (index * 4) as u64)?; + scene.metric_client = offset_va(args.metrics.client, (index * 4) as u64)?; + scene.metric_fw = offset_va(args.metrics.firmware, (index * 4) as u64)?; scene.scratch = scratch + (index % SCRATCH_SLOTS) as u64 * SCRATCH_STRIDE; scene.statistics = stats; } diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 9d89262d8a3b6..f36ab9798996a 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -47,10 +47,10 @@ pub(crate) struct QueueIds { } impl QueueIds { - pub(crate) const fn new() -> Self { - Self { - entries: [None; QID_COUNT], - } + pub(crate) fn new() -> impl Init { + kernel::try_init!(Self { + entries <- pin_init::init_array_from_fn(|_| None), + }) } /// Reuses only an exact owner/engine match. Preferred placement is for diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 4445374b22e49..9c3d4ad04e84b 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -359,12 +359,8 @@ impl Coprocessor { GFP_KERNEL, )?; - let name = match role { - Role::Primary => c_str!("asc"), - Role::Secondary => c_str!("asc1"), - }; Ok(Coprocessor { - cpu: regs::CpuControl::new(pdev, name)?, + cpu: regs::CpuControl::new(pdev, index)?, running: false, rtkit: Self::receiver(dev, index, &state)?, state, @@ -900,8 +896,6 @@ impl Gpu { // The firmware only samples the kernel lower root's top-level entries when it starts, so // everything it uses there must be mapped first. - let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary)?; - let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary)?; let pm_metrics = KernelObject::new( dev, uat.kernel_lower_vm(), @@ -910,28 +904,32 @@ impl Gpu { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; + let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary) + .inspect_err(|e| dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e))?; + let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary) + .inspect_err(|e| dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e))?; let render_global = uat.new_vm(0, geometry.user_range())?; let bridge = Bridge::enable(regs)?; // Install all session ownership before either CPU starts. Error unwinding then runs // Firmware::drop, stopping both CPUs before releasing shared objects or translations. - let mut firmware = KBox::new( - Firmware { + let mut firmware = KBox::try_init( + kernel::try_init!(Firmware { secondary, primary, bridge, rings: channel::Rings::new(), recovery: recovery::State::new(), effort: power::Effort::new(), - queues: runtime::Registry::new(shared.clusters, shared.descriptor_flags)?, + queues <- runtime::Registry::new(shared.clusters, shared.descriptor_flags), pool: object::Pool::new()?, render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, init: initdata::InitData::new(), _render_global: render_global, _pm_metrics: Arc::new(pm_metrics, GFP_KERNEL)?, uat, - }, + }), GFP_KERNEL, )?; let session = &mut *firmware; diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 40fa4c5c57b16..1634854f17ea4 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -236,6 +236,12 @@ impl Queue { } } for (_, fences) in frontier { + // Successful completion includes host timestamp publication. Its + // ready fence adds no ordering and would prevent shallow inline + // publication. Failed and pending producers still need checking. + if job::fence_status(&fences.completed) > 0 { + continue; + } dependencies.push(fences.ready.clone(), GFP_KERNEL)?; firmware.push(fences.completed.clone(), GFP_KERNEL)?; } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 8978693a789ea..d100fef631630 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -63,24 +63,32 @@ pub(super) struct Registry { } impl Registry { - pub(super) fn new(clusters: u32, descriptor_flags: [u32; 2]) -> Result { - let mut compute = KVec::with_capacity(QIDS, GFP_KERNEL)?; - for _ in 0..QIDS { - compute.push(None, GFP_KERNEL)?; - } - let mut qids = kick::QueueIds::new(); - qids.reserve(1, DataMaster::Tiling, Some(0))?; - qids.reserve(2, DataMaster::Fragment, Some(1))?; - Ok(Self { - qids, - buffers: BufferIds::new(), + /// Initialize inside the firmware allocation instead of staging the whole + /// registry on the probe stack. No additional allocation is required. + pub(super) fn new( + clusters: u32, + descriptor_flags: [u32; 2], + ) -> impl Init { + kernel::try_init!(Self { + qids <- kick::QueueIds::new().chain(|qids| { + qids.reserve(1, DataMaster::Tiling, Some(0))?; + qids.reserve(2, DataMaster::Fragment, Some(1))?; + Ok(()) + }), + buffers <- BufferIds::new(), accounting: qos::Accounting::new(), metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, qos_ids: QosIds::new()?, admission: admission::Pool::new()?, render: render::RenderState::new(clusters, descriptor_flags)?, identities: queue::Identities::new(), - compute, + compute: { + let mut compute = KVec::with_capacity(QIDS, GFP_KERNEL)?; + for _ in 0..QIDS { + compute.push(None, GFP_KERNEL)?; + } + compute + }, compute_building: false, compute_scheduler_building: false, observations: compute::retirement::Batch::new()?, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 1aa6cb67a90eb..e8fc30560a890 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -19,7 +19,7 @@ struct BuildResources { global: crate::mmu::Vm, metrics: Arc, metrics_ids: Arc, - metrics_bases: [u64; 2], + metrics_bases: crate::g17::fw::buffer::MetricsAddresses, config: Config, } impl BuildResources { @@ -31,10 +31,10 @@ impl BuildResources { global: firmware._render_global.clone(), metrics: firmware._pm_metrics.clone(), metrics_ids: firmware.queues.metrics.clone(), - metrics_bases: [ - crate::hw::t8140::CONFIG.pm_metrics.va, - firmware.init.metrics_fw_va()?, - ], + metrics_bases: crate::g17::fw::buffer::MetricsAddresses { + firmware: firmware.init.metrics_fw_va()?, + client: crate::hw::t8140::CONFIG.pm_metrics.va, + }, config: firmware.queues.render.config, }) } diff --git a/drivers/gpu/drm/asahi/g17/status.rs b/drivers/gpu/drm/asahi/g17/status.rs index 831c6813b7084..2ad06958c1359 100644 --- a/drivers/gpu/drm/asahi/g17/status.rs +++ b/drivers/gpu/drm/asahi/g17/status.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Sticky VM errors and their optional userspace mirror. No firmware state is accessed here. +//! VM admission errors and client failure reporting. No firmware state is accessed here. use crate::gem; use core::cell::UnsafeCell; @@ -10,9 +10,11 @@ use kernel::prelude::*; const MIRROR_MAX_SIZE: usize = 64 * 1024; /// A VM and its accepted jobs share this status, including after queue destruction. -/// The first error is permanent. Callers record it before signalling the job's fence. +/// Client failure is separate from hardware admission: recovery may preserve the +/// VM while failed accepted work must still be reported before signalling fences. pub(crate) struct VmStatus { error: AtomicI32, + reported_error: AtomicI32, mirror: AtomicPtr, mirror_claimed: AtomicBool, mirror_owner: UnsafeCell>, @@ -30,6 +32,7 @@ impl VmStatus { pub(crate) fn new() -> Self { Self { error: AtomicI32::new(0), + reported_error: AtomicI32::new(0), mirror: AtomicPtr::new(core::ptr::null_mut()), mirror_claimed: AtomicBool::new(false), mirror_owner: UnsafeCell::new(None), @@ -40,9 +43,26 @@ impl VmStatus { self.error.load(Ordering::Acquire) } - /// Store the first error in both the status and any registered mirror. + /// Record permanent admission failure and report it to the client. pub(crate) fn record(&self, error: Error) { - let error = match self.error.compare_exchange( + self.report_failure(error); + let _ = self.error.compare_exchange( + 0, + error.to_errno(), + Ordering::SeqCst, + Ordering::SeqCst, + ); + } + + pub(crate) fn reported_error(&self) -> i32 { + self.reported_error.load(Ordering::Acquire) + } + + /// Report failed accepted work without preventing recovery of the VM. + /// Every reporter publishes the first error before returning: a losing + /// reporter may signal its fence before the winning reporter resumes. + pub(crate) fn report_failure(&self, error: Error) { + let error = match self.reported_error.compare_exchange( 0, error.to_errno(), Ordering::SeqCst, @@ -88,11 +108,11 @@ impl VmStatus { // SAFETY: The retained owner keeps this checked, aligned word mapped. unsafe { AtomicI32::from_ptr(word) }.store(0, Ordering::SeqCst); - // The pointer publication and error update form a store/load handshake: - // either this load sees record's error or record sees our pointer. Stores + // The pointer publication and reported error form a store/load handshake: + // either this load sees the error or the reporter sees our pointer. Stores // after publishing the pointer write only the same first error, never zero. self.mirror.store(word, Ordering::SeqCst); - let error = self.error.load(Ordering::SeqCst); + let error = self.reported_error.load(Ordering::SeqCst); if error != 0 { // SAFETY: The mapping remains owned by this status as above. unsafe { AtomicI32::from_ptr(word) }.store(error, Ordering::SeqCst); diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index 7eecb26a8b0ff..fb22569ea5fdf 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -11,10 +11,12 @@ //! coprocessors start, and enables dynamic power gating before the firmware is initialized. use crate::hw; +use core::mem::MaybeUninit; use kernel::{ - c_str, + bindings, c_str, device::Core, devres::Devres, + error::{from_err_ptr, to_result}, io::{ mem::IoMem, Io, @@ -23,7 +25,6 @@ use kernel::{ platform, prelude::*, soc::apple::rtkit::ASC_CPU_CONTROL, - str::CStr, sync::{ aref::ARef, Arc, // @@ -56,13 +57,6 @@ const CORE_MASKS_G14X: usize = 0xe01500; const FAULT_INFO_G14X: usize = 0xd8c0; const FAULT_ADDR_G14X: usize = 0xd8c8; -/// ASC CPU control register and its run bit, for the G17 coprocessors. -const CPU_CONTROL: usize = 0x44; -const CPU_RUN: u32 = 0x1 << 4; // BIT(4) - -/// Cleared after `CPU_RUN` when stopping a G17 coprocessor. -const CPU_STOP_ACK: u32 = 0x1 << 5; // BIT(5) - /// G17 GPU-to-fabric bridge control words. Bit 0 enables the bridge; the other bits are status. const G17_BRIDGE_CONTROL: [usize; 2] = [0x1000104, 0x1000108]; const G17_BRIDGE_ENABLE: u32 = 0x1; // BIT(0) @@ -203,37 +197,62 @@ pub(crate) struct G17Id { pub(crate) perf_map: [u32; 2], } -/// CPU control registers of one GPU coprocessor. +/// Provider-owned lifecycle of one G17 firmware coprocessor. +/// +/// The mailbox provider owns the wrapper mappings and runtime-PM references. +/// The consumer device link keeps it alive until this GPU has stopped using it. pub(crate) struct CpuControl { - asc: Devres>, + dev: ARef, + mbox: *mut bindings::apple_mbox, } +// SAFETY: The live provider serializes CPU transitions with its lifecycle mutex; +// the device link orders provider removal after this consumer's teardown. +unsafe impl Send for CpuControl {} +// SAFETY: All accesses use the same provider-serialized lifecycle API. +unsafe impl Sync for CpuControl {} + impl CpuControl { - /// Map the CPU control registers in the `name` register region. - pub(crate) fn new(pdev: &platform::Device>, name: &CStr) -> Result { - let req = pdev.io_request_by_name(name).ok_or(EINVAL)?; - Ok(CpuControl { - asc: req.iomap_sized::()?.into_devres()?, - }) + /// Resolve the mailbox phandle and validate its firmware role before use. + pub(crate) fn new(pdev: &platform::Device>, index: usize) -> Result { + let expected = match index { + 0 => bindings::apple_mbox_ascwrap_v6_role_APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, + 1 => bindings::apple_mbox_ascwrap_v6_role_APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1, + _ => return Err(EINVAL), + }; + // SAFETY: pdev is live; the provider establishes the consumer device link. + let mbox = unsafe { + from_err_ptr(bindings::apple_mbox_get(pdev.as_ref().as_raw(), index.try_into()?))? + }; + let mut lifecycle = MaybeUninit::uninit(); + // SAFETY: mbox is live, and lifecycle is writable storage for the C result. + to_result(unsafe { + bindings::apple_mbox_ascwrap_v6_get_lifecycle(mbox, lifecycle.as_mut_ptr()) + })?; + // SAFETY: A successful provider query initialized every field. + if unsafe { lifecycle.assume_init() }.role != expected { + return Err(EINVAL); + } + // SAFETY: This only checks the provider's matching start/stop capability. + to_result(unsafe { bindings::apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(mbox) })?; + Ok(Self { dev: pdev.into(), mbox }) } - /// Start the coprocessor CPU. + /// Start through the owner of the CPU registers and runtime-PM reference. pub(crate) fn start(&self) -> Result { - let asc = self.asc.try_access().ok_or(ENODEV)?; - let asc = (&*asc).as_view(); - let val = asc.read32(CPU_CONTROL); - asc.write32(val | CPU_RUN, CPU_CONTROL); - Ok(()) + // SAFETY: The constructor established the live, role-checked provider. + to_result(unsafe { bindings::apple_mbox_ascwrap_v6_start_cpu(self.mbox) }) } - /// Stop the coprocessor CPU. This only gates it: its firmware keeps its state. + /// Stop before releasing the transport or any firmware-visible memory. pub(crate) fn stop(&self) { - if let Some(asc) = self.asc.try_access() { - let asc = (&*asc).as_view(); - let val = asc.read32(CPU_CONTROL); - asc.write32(val & !CPU_RUN, CPU_CONTROL); - let val = asc.read32(CPU_CONTROL); - asc.write32(val & !CPU_STOP_ACK, CPU_CONTROL); + // SAFETY: The provider remains live during consumer teardown. Removal + // stops the CPU before setting the fence that can return ENODEV here. + let result = to_result(unsafe { bindings::apple_mbox_ascwrap_v6_stop_cpu(self.mbox) }); + if let Err(error) = result { + if error != ENODEV { + dev_err!(self.dev.as_ref(), "Could not stop GPU coprocessor: {:?}\n", error); + } } } } From 42a8c9e7edab834b6be25c0342cd3f57b096d313 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 6 Oct 2026 00:26:31 +1100 Subject: [PATCH 0677/2680] drm/asahi: report live GEM object accounting Nothing in the driver says how much memory its GEM objects hold. fdinfo has no drm-* keys, and debugfs only lists flink names. A G17P GEM object is pinned once it is mapped into a GPU VM and stays pinned until its last reference drops, and drm_gem_get_pages() marks the shmem mapping unevictable. So when /proc/meminfo Unevictable grows, nothing shows whether that is live client memory, kernel firmware objects or references the driver kept after the client went away. Count every object from creation until it is freed, split by user and kernel ownership, and expose the totals read-only as /sys/module/asahi/parameters/gem_stats: user_objects user_bytes user_created user_freed kernel_objects kernel_bytes kernel_created kernel_freed The counters are relaxed atomics, updated in the DriverObject constructor and in a new PinnedDrop of the per-object data, which records the object size. If user_bytes stays flat across a client-exit loop, clients are not leaking. If it keeps growing while the per-process mappings do not, the driver is holding objects for clients that have gone. Assisted-by: LLM Signed-off-by: Ryan Murray Adapted to the rewrite GEM constructors and pinned drop. The counter balances construction and destruction; its sysfs output is verified on local Neo. Upstream-commit: 3361051608523c5c31f5dba92f48ff4258126f2d (cherry picked from commit 66af386568f748d2499be3e3587ce1e7ca0ae9cd) --- drivers/gpu/drm/asahi/asahi.rs | 1 + drivers/gpu/drm/asahi/gem.rs | 19 ++++++- drivers/gpu/drm/asahi/gem_stats.rs | 91 ++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 drivers/gpu/drm/asahi/gem_stats.rs diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi.rs index 1e9bd0b3b3be6..ec093aeb009bf 100644 --- a/drivers/gpu/drm/asahi/asahi.rs +++ b/drivers/gpu/drm/asahi/asahi.rs @@ -13,6 +13,7 @@ mod file; mod float; mod fw; mod gem; +mod gem_stats; mod g17; mod gpu; mod hw; diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index db49d418b0382..50a5946ff4a37 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -45,10 +45,12 @@ use crate::{ const DEBUG_CLASS: DebugFlags = DebugFlags::Gem; /// Represents the inner data of a GEM object for this driver. -#[pin_data] +#[pin_data(PinnedDrop)] pub(crate) struct AsahiObject { /// ID for debug id: u64, + /// Object size retained for live-object accounting. + size: usize, /// Object creation flags. flags: u32, /// Whether this object can be exported. @@ -151,6 +153,13 @@ impl ObjectRef { } } +#[pinned_drop] +impl PinnedDrop for AsahiObject { + fn drop(self: Pin<&mut Self>) { + crate::gem_stats::note_free(self.kernel, self.size); + } +} + pub(crate) struct AsahiObjConfig { flags: u32, exportable: bool, @@ -227,11 +236,17 @@ impl DriverObject for AsahiObject { const HAS_EXPORT: bool = true; /// Callback to create the inner data of a GEM object - fn new(_dev: &AsahiDrmDevice, _size: usize, args: Self::Args) -> impl PinInit { + fn new(_dev: &AsahiDrmDevice, size: usize, args: Self::Args) -> impl PinInit { let id = GEM_ID.fetch_add(1, Ordering::Relaxed); mod_pr_debug!("AsahiObject::new id={}\n", id); try_pin_init!(AsahiObject { id, + // All remaining fields are infallible. PinnedDrop balances the + // accounting once this initializer completes. + size: { + crate::gem_stats::note_new(args.kernel, size); + size + }, flags: args.flags, exportable: args.exportable, kernel: args.kernel, diff --git a/drivers/gpu/drm/asahi/gem_stats.rs b/drivers/gpu/drm/asahi/gem_stats.rs new file mode 100644 index 0000000000000..abd01922273bd --- /dev/null +++ b/drivers/gpu/drm/asahi/gem_stats.rs @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Live GEM object accounting (`gem_stats`, root-only, read-only sysfs). +//! +//! Every shmem object this driver creates is counted from creation until its +//! last reference drops, split by user and kernel ownership. Once a GEM object +//! is mapped into a GPU VM its pages stay pinned (and unevictable) until the +//! object is freed, so `user_bytes + kernel_bytes` is an upper bound on the +//! GPU share of /proc/meminfo `Unevictable`, and a value that keeps growing +//! after clients exit identifies a reference leak in the driver. The +//! `*_created` / `*_freed` totals show allocation churn. +use core::fmt::Write; +use core::sync::atomic::{AtomicU64, Ordering}; +use kernel::{bindings, module_param::KernelParam}; + +static USER_OBJECTS: AtomicU64 = AtomicU64::new(0); +static USER_BYTES: AtomicU64 = AtomicU64::new(0); +static USER_CREATED: AtomicU64 = AtomicU64::new(0); +static KERNEL_OBJECTS: AtomicU64 = AtomicU64::new(0); +static KERNEL_BYTES: AtomicU64 = AtomicU64::new(0); +static KERNEL_CREATED: AtomicU64 = AtomicU64::new(0); + +fn counters(kernel: bool) -> (&'static AtomicU64, &'static AtomicU64, &'static AtomicU64) { + if kernel { + (&KERNEL_OBJECTS, &KERNEL_BYTES, &KERNEL_CREATED) + } else { + (&USER_OBJECTS, &USER_BYTES, &USER_CREATED) + } +} + +/// Count one new GEM object. +pub(crate) fn note_new(kernel: bool, size: usize) { + let (objects, bytes, created) = counters(kernel); + objects.fetch_add(1, Ordering::Relaxed); + bytes.fetch_add(size as u64, Ordering::Relaxed); + created.fetch_add(1, Ordering::Relaxed); +} + +/// Count one freed GEM object. +pub(crate) fn note_free(kernel: bool, size: usize) { + let (objects, bytes, _) = counters(kernel); + objects.fetch_sub(1, Ordering::Relaxed); + bytes.fetch_sub(size as u64, Ordering::Relaxed); +} + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> core::fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(core::fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); + Ok(()) + } +} + +unsafe extern "C" fn stats_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1024) }; + let mut out = Output { bytes, used: 0 }; + let r = |c: &AtomicU64| c.load(Ordering::Relaxed); + let uc = r(&USER_CREATED); + let kc = r(&KERNEL_CREATED); + let (uo, ko) = (r(&USER_OBJECTS), r(&KERNEL_OBJECTS)); + let _ = writeln!(out, + "user_objects={} user_bytes={} user_created={} user_freed={} \ + kernel_objects={} kernel_bytes={} kernel_created={} kernel_freed={}", + uo, r(&USER_BYTES), uc, uc.wrapping_sub(uo), + ko, r(&KERNEL_BYTES), kc, kc.wrapping_sub(ko)); + out.used as core::ffi::c_int +} + +unsafe extern "C" fn stats_set(_: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + -1 // EPERM: read-only counters +} + +static STATS_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(stats_set), get: Some(stats_get), free: None, +}; +#[link_section = "__param"] +#[used] +static STATS_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"gem_stats" + } else { c"asahi.gem_stats" }), + #[cfg(MODULE)] + mod_: core::ptr::addr_of_mut!(bindings::__this_module), + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&STATS_OPS), perm: 0o400, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); From 3a7fabcddcea7baf76bef301d49c08689c90295d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 6 Oct 2026 09:40:35 +1100 Subject: [PATCH 0678/2680] drm/asahi: port j700 queue accounting and compute recovery isolation Keep sixteen physical QIDs available for render while allowing existing compute channels to continue unchanged. Account for queue ownership and exhaustion at structural transitions, without adding full-table scans to normal retirement. Classify bounded compute watchdog deferrals using older unfinished work from another VM; do not blame a potential victim VM when that grace period expires. Adapted from Ryan Murray's upstream commits: commit dc66a43fdbd8 ("drm/asahi: report firmware queue-slot accounting") commit dd27c0e2de7b ("drm/asahi: keep QIDs for render pairs and name the exhausted pool") (render QID reserve and exhausted-pool diagnostics) commit ccd546c5cb23 ("drm/asahi: G17P recovery isolation: do not blame a compute job stuck behind another VM") (compute watchdog isolation) The rewrite already reclaims drained quarantined render pairs after a fresh retirement witness; commit f055aaf00ec2 ("drm/asahi: reclaim drained quarantined render pairs by default") requires no default toggle here. Validation: matching module build and stack audit; production allocator, accounting and timeout-policy tests; local Neo desktop and the full light/heavy/compute depth 1/2/4/8/16 matrix; four concurrent clients for 60 seconds with sampled readback checks. Fault-injected recovery remains unqualified by these healthy-path hardware tests. Signed-off-by: Ryan Murray (cherry picked from commit f7fcc50568b9467f241a425b20b92ff1de10f641) --- drivers/gpu/drm/asahi/g17/context.rs | 2 +- drivers/gpu/drm/asahi/g17/job.rs | 15 +- drivers/gpu/drm/asahi/g17/kick.rs | 65 ++++++- drivers/gpu/drm/asahi/g17/mod.rs | 2 + drivers/gpu/drm/asahi/g17/queue/compute.rs | 41 ++++ drivers/gpu/drm/asahi/g17/queue_stats.rs | 177 ++++++++++++++++++ drivers/gpu/drm/asahi/g17/runtime.rs | 3 + drivers/gpu/drm/asahi/g17/runtime/backend.rs | 12 +- .../drm/asahi/g17/runtime/compute_control.rs | 40 ++++ .../drm/asahi/g17/runtime/render/settle.rs | 3 + drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 2 + drivers/gpu/drm/asahi/g17/timeout.rs | 60 ++++++ 12 files changed, 410 insertions(+), 12 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/queue_stats.rs create mode 100644 drivers/gpu/drm/asahi/g17/timeout.rs diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index c3a3c6e87a2f6..1a28386d788d0 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -39,7 +39,7 @@ impl QosIds { let mut used = self.used.lock(); let id = used.trailing_ones(); if id == 128 { - return Err(ENOSPC); + return Err(crate::g17::queue_stats::enospc(crate::g17::queue_stats::Pool::QueueContextQos)); } *used |= 1u128 << id; Ok(QosLease { diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 9dee3b0e5bb2f..f470b1116f0df 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -248,7 +248,7 @@ pub(crate) trait Backend: Send + Sync { fn timed_out(&self, packet: Arc) -> sched::Status; fn cancel(&self, packet: Arc); /// Transfer a timeout to an outstanding off-lock graph transaction, if any. - fn defer_timeout(&self, packet: &Arc) -> bool; + fn defer_timeout(&self, packet: &Arc, renewals: u8) -> bool; /// Return true only after taking publication ownership and recording its result. fn try_early(&self, packet: &Arc) -> bool; fn count_backlog(&self, packet: &Packet); @@ -256,6 +256,7 @@ pub(crate) trait Backend: Send + Sync { struct Job { backend: Arc, + victim_renewals: u8, packet: Arc, } @@ -272,9 +273,14 @@ impl Drop for Job { impl sched::JobImpl for Job { fn false_timeout(job: &mut sched::Job) -> bool { - job.is_finished() - || job.packet.completion.take_replay_timeout() - || job.backend.defer_timeout(&job.packet) + if job.is_finished() || job.packet.completion.take_replay_timeout() { + return true; + } + let deferred = job.backend.defer_timeout(&job.packet, job.victim_renewals); + if deferred && job.packet.engine() == Engine::Compute { + job.victim_renewals += 1; + } + deferred } fn prepare(job: &mut sched::Job) -> Option { @@ -337,6 +343,7 @@ impl Scheduler { 1, Job { backend: self.backend.clone(), + victim_renewals: 0, packet: packet.clone(), }, )?; diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index f36ab9798996a..aa2c054a9bbc7 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -6,7 +6,7 @@ //! and their ring backing stay owned until both firmware processors stop; //! command completion alone does not detach a hardware queue. -use core::sync::atomic::{fence, Ordering}; +use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; use kernel::prelude::*; use super::{ @@ -15,7 +15,30 @@ use super::{ }; use crate::{hw::t8140, mem, mmu}; -const QID_COUNT: usize = QID_MAX as usize + 1; +pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; + +pub(super) const COMPUTE_RENDER_RESERVE: usize = 16; +pub(super) static LEASED: [AtomicU32; 3] = [const { AtomicU32::new(0) }; 3]; +pub(super) static PUBLISHED: [AtomicU32; 3] = [const { AtomicU32::new(0) }; 3]; +pub(super) static HIGH_WATER: AtomicU32 = AtomicU32::new(0); +pub(super) static EXHAUSTED: AtomicU64 = AtomicU64::new(0); +pub(super) static RESERVE_REFUSED: AtomicU64 = AtomicU64::new(0); + +pub(super) fn leased_total() -> u32 { + LEASED.iter().map(|count| count.load(Ordering::Relaxed)).sum() +} + +fn exhausted(engine: DataMaster, reserve: bool) -> Error { + if reserve { + RESERVE_REFUSED.fetch_add(1, Ordering::Relaxed); + } else { + EXHAUSTED.fetch_add(1, Ordering::Relaxed); + } + super::queue_stats::enospc(match engine { + DataMaster::Compute => super::queue_stats::Pool::QidCompute, + _ => super::queue_stats::Pool::QidRenderPair, + }) +} /// Stable physical-channel identity. The owner must not be reused within a /// device lifetime, including after an unpublished allocation is cancelled. @@ -44,12 +67,16 @@ struct Identity { /// One shared namespace for tiling, fragment and compute queues. pub(crate) struct QueueIds { entries: [Option; QID_COUNT], + compute_publication: u64, + leased: usize, } impl QueueIds { pub(crate) fn new() -> impl Init { kernel::try_init!(Self { entries <- pin_init::init_array_from_fn(|_| None), + compute_publication: 0, + leased: 0, }) } @@ -76,11 +103,16 @@ impl QueueIds { Err(EINVAL) }; } + // Healthy existing channels keep their QID. Never recycle a published + // identity to recover capacity after a fault; leave eight render pairs. + if engine == DataMaster::Compute && QID_COUNT - self.leased <= COMPUTE_RENDER_RESERVE { + return Err(exhausted(engine, true)); + } let index = preferred .map(usize::from) .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) .or_else(|| self.entries.iter().position(Option::is_none)) - .ok_or(ENOSPC)?; + .ok_or_else(|| exhausted(engine, false))?; let id = Id { qid: index as u8, owner, @@ -90,9 +122,18 @@ impl QueueIds { id, published: false, }); + self.leased += 1; + LEASED[engine as usize].fetch_add(1, Ordering::Relaxed); + HIGH_WATER.fetch_max(leased_total(), Ordering::Relaxed); Ok(id) } + /// Ordered under the device mutex; no atomic operation on the submission path. + pub(crate) fn next_compute_publication(&mut self) -> Result { + self.compute_publication = self.compute_publication.checked_add(1).ok_or(EOVERFLOW)?; + Ok(self.compute_publication) + } + /// Call after preflighting the complete installation, before any address /// becomes firmware-visible. A failed or partial installation retains it. pub(crate) fn publish(&mut self, id: Id) -> Result { @@ -104,7 +145,10 @@ impl QueueIds { if entry.id != id { return Err(EINVAL); } - entry.published = true; + if !entry.published { + entry.published = true; + PUBLISHED[id.engine as usize].fetch_add(1, Ordering::Relaxed); + } Ok(()) } @@ -113,6 +157,8 @@ impl QueueIds { match slot { Some(entry) if entry.id == id && !entry.published => { *slot = None; + self.leased -= 1; + LEASED[id.engine as usize].fetch_sub(1, Ordering::Relaxed); Ok(()) } _ => Err(EINVAL), @@ -120,6 +166,17 @@ impl QueueIds { } } +impl Drop for QueueIds { + fn drop(&mut self) { + for entry in self.entries.iter().flatten() { + LEASED[entry.id.engine as usize].fetch_sub(1, Ordering::Relaxed); + if entry.published { + PUBLISHED[entry.id.engine as usize].fetch_sub(1, Ordering::Relaxed); + } + } + } +} + // Aliases drop before their backing. A compute queue is allocated low first; // a render queue is allocated high first. enum Backing { diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 9c3d4ad04e84b..ad548d722864b 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -43,6 +43,8 @@ mod recovery; mod runtime; pub(crate) mod status; mod teardown; +mod timeout; +mod queue_stats; mod validation; use core::any::Any; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 2a30cd54cecd1..e0cdc05044362 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -308,6 +308,7 @@ impl Fifo { struct Active { packet: Arc, + published: u64, work: Arc, ticket: Option, observed: Option, @@ -363,6 +364,7 @@ impl Backing { /// Installed graph and its exact logical owner. A released idle graph may change owners only /// after a retirement witness. Failed work retains its pointers independently of fence state. pub(crate) struct Queue { + slot_accounting: crate::g17::queue_stats::ComputeAccount, owner: Option, binding: Binding, previous: Option, @@ -412,6 +414,7 @@ impl Queue { context.mark_published(); Ok(Self { owner: Some(owner), + slot_accounting: crate::g17::queue_stats::ComputeAccount::new(), binding, previous: None, kick, @@ -446,6 +449,11 @@ impl Queue { pub(crate) fn qid(&self) -> u8 { self.kick.id().qid() } + fn update_slot_accounting(&mut self) { + self.slot_accounting.update(self.owner.is_some(), self.released, + self.quarantined, self.retired_by_teardown); + } + pub(crate) fn owner(&self) -> Option { self.owner } @@ -462,6 +470,22 @@ impl Queue { pub(crate) fn room(&self) -> bool { !self.quarantined && !self.released && self.replays.len == 0 && self.active.len < IN_FLIGHT } + /// Quarantines without a pending witness no longer describe live pipe order. + pub(crate) fn timeout_head(&self) -> Option { + if self.quarantined && !self.awaiting_witness { + return None; + } + self.active.front().map(|active| crate::g17::timeout::Head { + vm: Arc::as_ptr(active.packet.completion.status()) as usize, + published: active.published, + finished: active.observed.is_some(), + }) + } + + pub(crate) fn can_classify_timeout(&self, packet: &Arc) -> bool { + !self.quarantined && self.owns_packet(packet) + } + pub(crate) fn oldest_spared(&self) -> Option { self.active .front() @@ -567,10 +591,12 @@ impl Queue { ) -> Result>> { if !self.quarantined { self.quarantined = true; + self.update_slot_accounting(); self.quarantine_error = Some(error); self.failure_status_pending = true; self.retirement_proved = false; self.retired_by_teardown = false; + self.update_slot_accounting(); let all_spared = self.active.len != 0 && self.active.iter().all(|a| a.packet.completion.spared()); self.spared_quarantine = all_spared && error == ENODATA; @@ -676,6 +702,7 @@ impl Queue { self.retirement_ready = self.active.len == 0; if self.released && self.retirement_ready { self.owner = None; + self.update_slot_accounting(); } } Ok(None) | Err(EAGAIN) => { @@ -782,6 +809,7 @@ impl Queue { self.submitted = 0; self.kick.clear_parent_after_recovery(); self.quarantined = false; + self.update_slot_accounting(); self.retire_pending = false; self.retirement_ready = true; if self @@ -793,6 +821,7 @@ impl Queue { } if self.released { self.owner = None; + self.update_slot_accounting(); } } Ok(()) @@ -932,9 +961,12 @@ impl Queue { } context.mark_published(); self.owner = Some(owner); + self.update_slot_accounting(); self.released = false; + self.update_slot_accounting(); self.retirement_ready = false; self.retired_by_teardown = false; + self.update_slot_accounting(); Ok(()) } /// True allows the registry to cancel the never-published QID and drop this graph. An @@ -944,11 +976,14 @@ impl Queue { return Ok(true); } self.released = true; + self.update_slot_accounting(); if self.active.len == 0 && !self.quarantined { if let Some(previous) = self.previous.as_ref().filter(|p| p.publication.is_none()) { if let Err(error) = self.graph.set_owner(&previous.context) { self.quarantined = true; + self.update_slot_accounting(); self.retired_by_teardown = true; + self.update_slot_accounting(); return Err(error); } let mut previous = self.previous.take().ok_or(EIO)?; @@ -961,6 +996,7 @@ impl Queue { } if self.retirement_ready { self.owner = None; + self.update_slot_accounting(); } } Ok(false) @@ -1027,7 +1063,9 @@ impl Queue { return Ok(true); } self.released = true; + self.update_slot_accounting(); self.retired_by_teardown = true; + self.update_slot_accounting(); if let Some(active) = self .active .iter_mut() @@ -1159,6 +1197,7 @@ impl Graph { /// Borrowed device operations used during one publication. Implementations borrow the initdata, /// coprocessor and accounting fields separately from the Registry that owns physical queues. pub(crate) trait Host { + fn next_compute_publication(&mut self) -> Result; fn epoch(&self) -> Result<(u64, u32)>; fn prepare_compute_shared(&mut self) -> Result; fn qos_publish(&mut self, owner: qos::Owner, scheduler: u64) -> Result; @@ -1313,9 +1352,11 @@ impl Queue { event_mask: COMPUTE_KICK_EVENT_MASK, register_arrays: ComputeDescriptor::register_bindings(descriptor_low)?, }; + let published = host.next_compute_publication()?; self.active .push(Active { packet: packet.clone(), + published, work: work.clone(), ticket: None, observed: None, diff --git a/drivers/gpu/drm/asahi/g17/queue_stats.rs b/drivers/gpu/drm/asahi/g17/queue_stats.rs new file mode 100644 index 0000000000000..5a5605e1d19a2 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue_stats.rs @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Firmware queue-slot accounting (`g17p_queue_slots`, root-only, read-only +//! sysfs). Relaxed counters; exhaustion logging is rate-limited. +//! +//! Every render pair and compute queue leases QIDs from one 128-entry table +//! (`kick::QueueIds`). A published QID is kept until processor stop, and a +//! quarantined or exit-killed compute graph is never handed out again, so a +//! boot can run out of QIDs while memory is fine. Queue creation then fails +//! with ENOSPC, and so does a new render pair (`render-warm-pairs`). The +//! `qids` fields are exact (kept by the table itself); compute +//! graph ownership gauges track structural changes. Pending teardown and +//! draining-pair counts are sampled in their existing service passes. +//! +//! Every ENOSPC from one of these pools is counted by pool and named in +//! dmesg (the first one, then each power of two), so a failed QUEUE_CREATE +//! or submit says which pool ran out. +use core::fmt::Write; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use kernel::{bindings, module_param::KernelParam, prelude::*}; + +use super::kick as g17_channel; +use g17_channel::QID_COUNT; + +static COMPUTE_GRAPHS: AtomicU32 = AtomicU32::new(0); +static COMPUTE_OWNED: AtomicU32 = AtomicU32::new(0); +static COMPUTE_RELEASED: AtomicU32 = AtomicU32::new(0); +static COMPUTE_QUARANTINED: AtomicU32 = AtomicU32::new(0); +static COMPUTE_EXIT_KILLED: AtomicU32 = AtomicU32::new(0); +static RETIRED_GRAPHS: AtomicU32 = AtomicU32::new(0); +static TEARDOWNS: AtomicU32 = AtomicU32::new(0); +static RENDER_QUARANTINED: AtomicU32 = AtomicU32::new(0); + +/// Pools whose exhaustion returns ENOSPC to a client. +#[derive(Clone, Copy)] +pub(crate) enum Pool { + /// A render slot's first use leases two QIDs (`render-warm-pairs`). + QidRenderPair, + /// A new compute graph leases one QID (queue creation / first compute). + QidCompute, + /// A new logical queue context (QUEUE_CREATE): its only 128-wide pool + /// is the QoS hardware-buffer IDs, held by logical queues and installed + /// graph bindings. + QueueContextQos, +} + +const POOL_COUNT: usize = 3; +const POOL_NAMES: [&str; POOL_COUNT] = ["qid-render-pair", "qid-compute", "queue-context-qos"]; +static POOL_ENOSPC: [AtomicU64; POOL_COUNT] = + [AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0)]; +/// Submits that failed with ENOSPC, whichever pool. +static SUBMIT_ENOSPC: AtomicU64 = AtomicU64::new(0); + +fn log_enospc(pool: &str, stage: &str, count: u64) { + // First one, then each power of two: a stuck pool cannot flood dmesg. + if count.is_power_of_two() { + pr_warn!( + "G17P ENOSPC pool={} stage={} (#{}): qids leased={}/{} compute={} compute-quarantined={} \ + reserve-refused={}\n", + pool, stage, count, g17_channel::leased_total(), QID_COUNT, + g17_channel::LEASED[2].load(Ordering::Relaxed), + COMPUTE_QUARANTINED.load(Ordering::Relaxed), + g17_channel::RESERVE_REFUSED.load(Ordering::Relaxed), + ); + } +} + +/// Count and name one ENOSPC from `pool`, and return it. +#[cold] +pub(crate) fn enospc(pool: Pool) -> Error { + let count = POOL_ENOSPC[pool as usize].fetch_add(1, Ordering::Relaxed) + 1; + log_enospc(POOL_NAMES[pool as usize], "-", count); + ENOSPC +} + +/// A submit failed with ENOSPC at `stage`. Pools tagged at their source +/// were already counted and named; this names the rest by stage. +#[cold] +pub(crate) fn note_submit_enospc(stage: &str) { + let count = SUBMIT_ENOSPC.fetch_add(1, Ordering::Relaxed) + 1; + // Source-specific QID/QoS failures already counted their actual pool. + // This boundary knows the engine stage, not which nested allocator failed. + log_enospc("submit", stage, count); +} + +/// Incremental graph ownership accounting. Normal submissions do not touch +/// these counters; only graph creation, ownership transitions and drop do. +pub(crate) struct ComputeAccount { flags: u8 } +impl ComputeAccount { + pub(crate) fn new() -> Self { + COMPUTE_GRAPHS.fetch_add(1, Ordering::Relaxed); + COMPUTE_OWNED.fetch_add(1, Ordering::Relaxed); + Self { flags: 1 } + } + pub(crate) fn update(&mut self, owned: bool, released: bool, quarantined: bool, exit_killed: bool) { + let next = u8::from(owned) | (u8::from(released) << 1) + | (u8::from(quarantined) << 2) | (u8::from(exit_killed) << 3); + let changed = next ^ self.flags; + for (bit, counter) in [&COMPUTE_OWNED, &COMPUTE_RELEASED, + &COMPUTE_QUARANTINED, &COMPUTE_EXIT_KILLED].into_iter().enumerate() + { + if changed & (1 << bit) != 0 { + if next & (1 << bit) != 0 { counter.fetch_add(1, Ordering::Relaxed); } + else { counter.fetch_sub(1, Ordering::Relaxed); } + } + } + self.flags = next; + } +} +impl Drop for ComputeAccount { + fn drop(&mut self) { + self.update(false, false, false, false); + COMPUTE_GRAPHS.fetch_sub(1, Ordering::Relaxed); + } +} + +pub(crate) fn note_teardowns(count: usize) { + TEARDOWNS.store(count as u32, Ordering::Relaxed); +} +pub(crate) fn note_draining_pairs(count: usize) { + RENDER_QUARANTINED.store(count as u32, Ordering::Relaxed); +} + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> core::fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(core::fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); + Ok(()) + } +} + +unsafe extern "C" fn slots_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1024) }; + let mut out = Output { bytes, used: 0 }; + let r = |c: &AtomicU32| c.load(Ordering::Relaxed); + let e = |pool: usize| POOL_ENOSPC[pool].load(Ordering::Relaxed); + let leased = g17_channel::leased_total(); + let published: u32 = g17_channel::PUBLISHED.iter().map(r).sum(); + let _ = writeln!(out, + "qids leased={} free={} published={} tiling={} fragment={} compute={} high_water={} exhausted={} \ + reserve={} reserve_refused={} \ + compute graphs={} owned={} released={} quarantined={} exit_killed={} retired_graphs={} \ + teardowns={} render_draining_pairs={} \ + enospc qid-render-pair={} qid-compute={} queue-context-qos={} submits={}", + leased, (QID_COUNT as u32).saturating_sub(leased), published, + r(&g17_channel::LEASED[0]), r(&g17_channel::LEASED[1]), r(&g17_channel::LEASED[2]), + r(&g17_channel::HIGH_WATER), g17_channel::EXHAUSTED.load(Ordering::Relaxed), + g17_channel::COMPUTE_RENDER_RESERVE, g17_channel::RESERVE_REFUSED.load(Ordering::Relaxed), + r(&COMPUTE_GRAPHS), r(&COMPUTE_OWNED), r(&COMPUTE_RELEASED), r(&COMPUTE_QUARANTINED), + r(&COMPUTE_EXIT_KILLED), r(&RETIRED_GRAPHS), r(&TEARDOWNS), r(&RENDER_QUARANTINED), + e(0), e(1), e(2), SUBMIT_ENOSPC.load(Ordering::Relaxed)); + out.used as core::ffi::c_int +} + +unsafe extern "C" fn slots_set(_: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + -1 // EPERM: read-only counters +} + +static SLOTS_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(slots_set), get: Some(slots_get), free: None, +}; +#[link_section = "__param"] +#[used] +static SLOTS_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_queue_slots" + } else { c"asahi.g17p_queue_slots" }), + #[cfg(MODULE)] + mod_: core::ptr::addr_of_mut!(bindings::__this_module), + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&SLOTS_OPS), perm: 0o400, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index d100fef631630..294d66b91f8dc 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -546,6 +546,9 @@ struct ComputeHost<'a> { generation: u64, } impl compute::Host for ComputeHost<'_> { + fn next_compute_publication(&mut self) -> Result { + self.qids.next_compute_publication() + } fn epoch(&self) -> Result<(u64, u32)> { Ok(( self.generation, diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 085655beb8194..3fb40779021e5 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -138,6 +138,12 @@ impl job::Backend for Backend { Engine::Render => self.render_packet(&packet, &mut deferred), } }; + if result.as_ref().is_err_and(|error| *error == ENOSPC) { + crate::g17::queue_stats::note_submit_enospc(match packet.engine() { + Engine::Compute => "compute-publish", + Engine::Render => "render-publish", + }); + } self.release_backlog(&packet); self.finish_publication(&packet, result, &mut deferred) } @@ -169,9 +175,9 @@ impl job::Backend for Backend { } } - fn defer_timeout(&self, packet: &Arc) -> bool { - if packet.engine() != Engine::Render { - return false; + fn defer_timeout(&self, packet: &Arc, renewals: u8) -> bool { + if packet.engine() == Engine::Compute { + return self.defer_compute_timeout(packet, renewals); } let mut state = self.shared.state.lock(); (*state) diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index 610643ef6e379..dac4ca48a34ae 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -7,6 +7,46 @@ use crate::g17::job::{self, Packet}; use kernel::{prelude::*, sync::Arc}; impl Backend { + /// A final retirement pass precedes bounded cross-VM timeout attribution. + /// All completion callbacks run after releasing the device mutex. + pub(super) fn defer_compute_timeout(&self, packet: &Arc, renewals: u8) -> bool { + use crate::g17::{recovery::Class, timeout::{self, Decision}}; + let Ok(mut deferred) = DeferredBatch::worker() else { + return false; + }; + let result = (|| -> Result { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(Decision::Own); + }; + firmware.scan_compute(None, &mut deferred)?; + let Some(queue) = firmware.queues.compute(self.owner) + .filter(|queue| queue.can_classify_timeout(packet)) else { + return Ok(Decision::Own); + }; + let Some(own) = queue.timeout_head() else { + return Ok(Decision::Own); + }; + let others = firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|other| other.owner() != Some(self.owner)) + .filter_map(|other| other.timeout_head()); + let decision = timeout::classify(own, renewals, others); + if decision == Decision::Spare { + firmware.queues.compute(self.owner).ok_or(EIO)? + .classify(&|_| Class::Spared); + } + if decision != Decision::Own && (renewals == 0 || decision == Decision::Spare) { + dev_warn!(self.shared.dev.as_ref(), + "Compute watchdog owner={} publication={} renewals={} decision={:?} behind earlier work of another VM\n", + self.owner, own.published, renewals, decision); + } + Ok(decision) + })(); + deferred.finish(); + matches!(result, Ok(Decision::Defer)) + } + /// The caller owns cancellation of a packet that never claimed publication. /// A missing retained packet may already have a healthy callback waiting off-lock. pub(super) fn timeout_compute(&self, packet: &Arc) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index a6d5821561d24..2cf7a5d77ee5a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -164,6 +164,7 @@ impl crate::g17::Firmware { return Ok(()); } let mut serviced = false; + let mut draining = 0; for slot in 1..self.queues.render.entries.len() { if !self.queues.render.entries[slot] .as_ref() @@ -172,6 +173,7 @@ impl crate::g17::Firmware { { continue; } + draining += 1; if let Err(error) = self.drain_render_slot(slot, &mut serviced, deferred) { dev_warn!( self.primary.state.shared.dev.as_ref(), @@ -181,6 +183,7 @@ impl crate::g17::Firmware { ); } } + crate::g17::queue_stats::note_draining_pairs(draining); Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 10f334e4fe561..06465629eb2be 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -165,6 +165,7 @@ impl Firmware { /// The event worker has already reclaimed drained pairs and reannounced /// pending controls. Recovery blocks new releases, never an observed ack. pub(in crate::g17) fn service_teardown(&mut self, reclaim: &mut ReclaimBatch) -> Result { + crate::g17::queue_stats::note_teardowns(self.queues.teardown.len()); self.queues.render.collect_reclaimable(reclaim)?; if self.queues.teardown.is_empty() { return Ok(()); @@ -194,6 +195,7 @@ impl Firmware { .is_some() {} teardown.report_stalled(super::now_ns(), &mut service); + crate::g17::queue_stats::note_teardowns(teardown.len()); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs new file mode 100644 index 0000000000000..e76dab07f56db --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounded isolation for a compute watchdog blocked behind another VM's older kick. +//! Publication order is evidence of a possible blocker, not proof of a firmware +//! dependency. Renewals are bounded; the oldest unfinished queue is never excused. + +pub(crate) const MAX_RENEWALS: u8 = 8; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Head { + pub(crate) vm: usize, + pub(crate) published: u64, + pub(crate) finished: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Decision { + Own, + Defer, + Spare, +} + +pub(crate) fn classify(own: Head, renewals: u8, others: impl Iterator) -> Decision { + if own.finished || !others.into_iter().any(|other| { + other.vm != own.vm && !other.finished && other.published < own.published + }) { + Decision::Own + } else if renewals < MAX_RENEWALS { + Decision::Defer + } else { + Decision::Spare + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn head(vm: usize, published: u64, finished: bool) -> Head { + Head { vm, published, finished } + } + #[test] + fn earlier_other_vm_gets_bounded_grace() { + let own = head(5, 101, false); + for count in 0..MAX_RENEWALS { + assert_eq!(classify(own, count, [head(7, 100, false)].into_iter()), Decision::Defer); + } + assert_eq!(classify(own, MAX_RENEWALS, [head(7, 100, false)].into_iter()), Decision::Spare); + assert_eq!(classify(own, 2, [].into_iter()), Decision::Own); + } + #[test] + fn oldest_unfinished_queue_is_never_excused() { + assert_eq!(classify(head(7, 100, false), 0, [head(5, 101, false)].into_iter()), Decision::Own); + } + #[test] + fn same_vm_finished_and_later_work_do_not_block() { + let own = head(5, 101, false); + assert_eq!(classify(own, 0, [head(5, 100, false), head(7, 100, true), head(7, 101, false), head(7, 102, false)].into_iter()), Decision::Own); + assert_eq!(classify(head(5, 101, true), 0, [head(7, 100, false)].into_iter()), Decision::Own); + } +} From acef8811a314e3c2120a76fad5a6e97288f41a93 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 06:59:34 -0400 Subject: [PATCH 0679/2680] drm/asahi: account once per compute queue transition Queue quarantine, owner rebinding, release failure and cancellation each updated the accounting state between consecutive field assignments. Keep one update after each complete transition, removing five redundant calls without changing the final gauges or queue policy. Exercise the production state assignments and counter implementation over all 64 transition/flag combinations. The work-count assertion fails on the previous implementation and passes here. The native GPU module builds without warnings against the fully built integration kernel; the static module-frame stack gate passes. No hardware performance gain is claimed. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 3c65418ee698735d33f299fe208aa8c22ff310a4) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index e0cdc05044362..3b259b37b38f4 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -591,7 +591,6 @@ impl Queue { ) -> Result>> { if !self.quarantined { self.quarantined = true; - self.update_slot_accounting(); self.quarantine_error = Some(error); self.failure_status_pending = true; self.retirement_proved = false; @@ -961,9 +960,7 @@ impl Queue { } context.mark_published(); self.owner = Some(owner); - self.update_slot_accounting(); self.released = false; - self.update_slot_accounting(); self.retirement_ready = false; self.retired_by_teardown = false; self.update_slot_accounting(); @@ -981,7 +978,6 @@ impl Queue { if let Some(previous) = self.previous.as_ref().filter(|p| p.publication.is_none()) { if let Err(error) = self.graph.set_owner(&previous.context) { self.quarantined = true; - self.update_slot_accounting(); self.retired_by_teardown = true; self.update_slot_accounting(); return Err(error); @@ -1063,7 +1059,6 @@ impl Queue { return Ok(true); } self.released = true; - self.update_slot_accounting(); self.retired_by_teardown = true; self.update_slot_accounting(); if let Some(active) = self From cd9f2e080b5a60d66f02c95eb97df2f4f3f73b8c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 22:55:07 -0400 Subject: [PATCH 0680/2680] drm: carry shared GPUVM and scheduler interfaces used by the G17 review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Carry the GPUVM array reservation, allocator interval query and scheduler lifetime interfaces from the cleanroom current-kernel integration. Preserve the zero-fence GPUVM locking contract and initialize zero-valid C storage without importing unexported bindgen Default methods into loadable modules. This isolates the four DRM prerequisite files from the wider j700 merge; the public series retains its own platform and RTKit base. The following review fixes are applied against these shared interfaces. [ryan: adapted to 7.2.10 — minimal; for DJ: rust/kernel/drm/gpuvm.rs is deleted in 7.2 (GPUVM moved to rust/kernel/drm/gpuvm/), so its hunk is not carried: exec_lock_array(), sm_map_indexed() and the zeroed C initializers have no user in the driver; the drm_gpuvm.c, mm.rs and sched.rs parts are unchanged] Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 982df2dcc66e2cb2012df476c4325c8fce016dda) --- drivers/gpu/drm/drm_gpuvm.c | 18 ++- rust/kernel/drm/mm.rs | 38 ++++++- rust/kernel/drm/sched.rs | 216 +++++++++++++++++++++--------------- 3 files changed, 181 insertions(+), 91 deletions(-) diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c index 91f7861e36b84..b123f6164206b 100644 --- a/drivers/gpu/drm/drm_gpuvm.c +++ b/drivers/gpu/drm/drm_gpuvm.c @@ -1396,9 +1396,23 @@ fn_lock_array(struct drm_gpuvm_exec *vm_exec) struct drm_gem_object **objs; unsigned int num_objs; } *args = vm_exec->extra.priv; + unsigned int i; + int ret; + + /* + * Keep the zero-fence contract of every other drm_gpuvm_exec helper. + * drm_exec_prepare_array() unconditionally calls + * dma_resv_reserve_fences(), where zero is invalid; exec_prepare_obj() + * deliberately selects drm_exec_lock_obj() for that case instead. + */ + for (i = 0; i < args->num_objs; i++) { + ret = exec_prepare_obj(&vm_exec->exec, args->objs[i], + vm_exec->num_fences); + if (ret) + return ret; + } - return drm_exec_prepare_array(&vm_exec->exec, args->objs, - args->num_objs, vm_exec->num_fences); + return 0; } /** diff --git a/rust/kernel/drm/mm.rs b/rust/kernel/drm/mm.rs index 7b13cfd7d5309..e1ec944d8424f 100644 --- a/rust/kernel/drm/mm.rs +++ b/rust/kernel/drm/mm.rs @@ -7,7 +7,7 @@ use crate::{ alloc::flags::*, bindings, - error::{to_result, Result}, + error::{code::EINVAL, to_result, Result}, sync::{new_mutex, Arc, Mutex, UniqueArc}, types::Opaque, }; @@ -17,7 +17,7 @@ use crate::prelude::KBox; use core::{ marker::{PhantomData, PhantomPinned}, - ops::Deref, + ops::{Deref, Range}, pin::Pin, }; @@ -161,6 +161,40 @@ pub struct Allocator, T> { } impl, T> Allocator { + /// Return the first allocator reservation intersecting an address range. + /// Unlike a page-table walk, this includes unmapped guard space retained + /// inside a node. The interval query and its result are protected by the + /// allocator mutex; no probe node is inserted into the live VA space. + pub fn first_overlap(&self, range: Range) -> Result>> { + if range.start >= range.end { + return Err(EINVAL); + } + let guard = self.mm.lock(); + // The DRM interval query requires both endpoints to lie inside its + // allocator. head_node is the end sentinel; its wrapping size is the + // negated allocator span. + let head = unsafe { &(*guard.0.get()).head_node }; + let allocator_start = head.start.wrapping_add(head.size); + if range.start < allocator_start || range.end > head.start { + return Err(EINVAL); + } + // SAFETY: The allocator is initialized for the lifetime of `self`. + // Its mutex prevents node removal while the returned pointer is read. + // drm_mm returns its end sentinel when no interval intersects. + let node = unsafe { + bindings::__drm_mm_interval_first( + guard.0.get(), range.start, range.end - 1, + ) + }; + // SAFETY: drm_mm_interval_first returns a valid node or the allocator + // end sentinel, both of which remain live while `guard` is held. + let node = unsafe { node.as_ref().ok_or(EINVAL)? }; + if node.start >= range.end { + return Ok(None); + } + Ok(Some(node.start..node.start.saturating_add(node.size))) + } + /// Create a new range allocator for the given start and size range of addresses. /// /// The user may optionally provide an inner object representing allocator state, which will diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index 78f0451f5c5f7..a1f5eb04f575f 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -11,7 +11,6 @@ use crate::{ prelude::*, sync::{Arc, UniqueArc}, time::{self, msecs_to_jiffies}, - types::Opaque, }; use core::marker::PhantomData; use core::mem::MaybeUninit; @@ -25,10 +24,7 @@ pub enum Status { Nominal = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_RESET, /// Device is no longer available NoDevice = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_ENODEV, - /// The job is not hung, or the driver dealt with the timeout without a scheduler reset. - /// - /// The job goes back to the pending list and the timeout is rearmed. The job is freed once - /// it has finished, as usual. + /// Reinsert the job and rearm its timeout without resetting the scheduler. NoHang = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG, } @@ -47,12 +43,20 @@ pub enum Priority { /// Trait to be implemented by driver job objects. pub trait JobImpl: Sized { - /// Reconsider a timeout before the backend starts recovery. Returning true - /// reinserts this job and rearms its timeout without changing its fences. - /// The callback must not stop or restart the scheduler. - fn false_timeout(_job: &mut Job) -> bool { - false - } + /// Handle a scoped timeout without resetting the C scheduler. + /// + /// The backend must settle/quarantine its exact hardware work, or retain + /// that owner until an in-progress host transaction can classify it. It + /// must not invoke drm_sched_stop/start or replay scheduler jobs. The core + /// reinserts the detached guilty job through NO_HANG, preserving every + /// parent callback and the backend's actual success/error fence result. + const MANAGED_TIMEOUT_RECOVERY: bool = false; + + /// Reconsider a timeout before stopping the scheduler or settling fences. + /// Returning true asks the core to reinsert the retained job and rearm. + /// The backend must not stop/start the scheduler from this hook. + fn false_timeout(_job: &mut Job) -> bool { false } + /// Called when the scheduler is considering scheduling this job next, to get another Fence /// for this job to block on. Once it returns None, run() may be called. @@ -67,9 +71,6 @@ pub trait JobImpl: Sized { /// Called when a job has taken too long to execute, to trigger GPU recovery. /// - /// A driver that handles timeouts without a scheduler reset returns [`Status::NoHang`]. - /// [`Job::is_finished`] tells whether the job completed after the timeout fired. - /// /// This method is called in a workqueue context. fn timed_out(job: &mut Job) -> Status; @@ -112,19 +113,49 @@ unsafe extern "C" fn timedout_job_cb( // SAFETY: All of our jobs are Job. let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; - // SAFETY: The timeout worker retains this detached Job until this - // callback returns, including when its parent fence signals concurrently. + // The C timeout worker owns this job outside pending_list until return. + // Free work cannot release it even if its parent signals concurrently. + if T::MANAGED_TIMEOUT_RECOVERY { + let finished = unsafe { addr_of_mut!((*(*sched_job).s_fence).finished) }; + if unsafe { bindings::dma_fence_get_status(finished) } != 0 { + return bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG; + } + } + // SAFETY: the C core retains the detached job across both callbacks. if T::false_timeout(unsafe { &mut *p }) { return bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG; } - // SAFETY: The same retained job remains valid after the first callback. - T::timed_out(unsafe { &mut *p }) as bindings::drm_gpu_sched_stat + let status = T::timed_out(unsafe { &mut *p }); + if T::MANAGED_TIMEOUT_RECOVERY { + // This is scoped fence handling, not C reset recovery. stop would + // detach and discard healthy parents; start would cancel those jobs. + // NO_HANG reinserts the exact job and queues free work if it completed + // during the callback. It does not change the parent's error or replay + // work. A deferred exact owner remains eligible for its real callback. + bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG + } else { + status as bindings::drm_gpu_sched_stat + } } unsafe extern "C" fn free_job_cb(sched_job: *mut bindings::drm_sched_job) { // SAFETY: All of our jobs are Job. let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; + // Fault-only ownership witness: this callback is where the core returns + // ownership of an errored job. A signalled fence alone is not that proof. + // SAFETY: A job reaching free_job has a live scheduler fence; its error + // was fixed before signal and the core's completion callback has finished. + if T::MANAGED_TIMEOUT_RECOVERY + && unsafe { (*(*sched_job).s_fence).finished.error } == ETIMEDOUT.to_errno() + { + // Use numeric identity: the fmt adapter borrows arguments, and + // Pointer formatting of that reference can print a stack address. + pr_info!("DRM Rust scheduler: released timed-out job={:#x} context={} seqno={}\n", + sched_job as usize, + unsafe { (*(*sched_job).s_fence).finished.context }, + unsafe { (*(*sched_job).s_fence).finished.seqno }); + } // Convert the job back to a Box and drop it // SAFETY: All of our Jobs are created inside a box. unsafe { drop(KBox::from_raw(p)) }; @@ -133,71 +164,70 @@ unsafe extern "C" fn free_job_cb(sched_job: *mut bindings::drm_sched unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sched_job) { // SAFETY: All of our jobs are Job. let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; - // SAFETY: A job on the pending list has a valid scheduler fence. + + // fini has stopped/synchronized submission and timeout workers. A parent + // callback can still run on the device completion thread. The scheduler + // fence retains the parent reference while we remove its callback under + // the parent's lock; that also waits for an already-running callback. let s_fence = unsafe { (*sched_job).s_fence }; - // SAFETY: `s_fence` is valid and holds a reference to its parent, if any. let parent = unsafe { (*s_fence).parent }; - // SAFETY: `s_fence` is valid. - let finished = unsafe { addr_of_mut!((*s_fence).finished) }; - - // drm_sched_fini() has stopped the scheduler workers, but the hardware fence can still - // signal concurrently. Fence callbacks run under the fence lock, so removing ours either - // disarms it or waits until it has completed the job. - let callback_removed = !parent.is_null() - // SAFETY: run_job() armed `cb` on `parent` before the submission worker was stopped. - && unsafe { bindings::dma_fence_remove_callback(parent, addr_of_mut!((*sched_job).cb)) }; - - // A job without a parent either completed when run_job() returned, or had its parent - // detached by drm_sched_stop(), which also returned its credits. - let complete_here = callback_removed - // SAFETY: `finished` is a valid, initialized fence. - || (parent.is_null() && unsafe { bindings::dma_fence_get_status(finished) } == 0); - - // SAFETY: All of our jobs are Job. + let removed_parent_callback = if !parent.is_null() { + // SAFETY: run_job installed (or attempted to install on an already + // signaled parent) this callback before fini synchronized submit work. + unsafe { bindings::dma_fence_remove_callback(parent, addr_of_mut!((*sched_job).cb)) } + } else { false }; + let finish_here = removed_parent_callback || (parent.is_null() + && unsafe { bindings::dma_fence_get_status(addr_of_mut!((*s_fence).finished)) == 0 }); + // Null/error run_job calls job_done before submit work returns, so its + // finished fence is already signaled. NULL+unfinished can instead survive + // a legacy backend's prior reset stop: C stop dropped the parent and its + // credits, but not its score. Do not subtract those credits twice. + + // SAFETY: cancellation cannot race a scheduler callback using this job + // after the removal/parent-lock synchronization above. The backend must + // retain any still-hardware-owned resources independently of Job storage. T::cancel(unsafe { &mut *p }); - if complete_here { - // Account for the job exactly as drm_sched_job_done() would have. - // - // SAFETY: The job belongs to the scheduler being torn down, which is still valid. + if finish_here { + // Match drm_sched_job_done's accounting exactly once, only when its + // callback did not do it. Submission/free work is stopped, so fini + // itself unlinks and frees this job; no workqueue rearm is appropriate. + let sched = unsafe { (*sched_job).sched }; unsafe { - let sched = (*s_fence).sched; - if callback_removed { - bindings::atomic_sub( - (*sched_job).credits as i32, - addr_of_mut!((*sched).credit_count), - ); + if removed_parent_callback { + bindings::atomic_sub((*sched_job).credits as i32, + addr_of_mut!((*sched).credit_count)); } - bindings::atomic_dec((*sched).score); + bindings::atomic_sub(1, (*sched).score); } - - // SAFETY: `finished` is a valid fence. - let fence = unsafe { Fence::get_raw(finished) }; + let fence = unsafe { Fence::get_raw(addr_of_mut!((*s_fence).finished)) }; fence.set_error(ECANCELED); - fence.signal(); + let _ = fence.signal(); } + // Do not clear/put parent here: s_fence still owns that reference and its + // normal release path drops it. A completed callback's result is retained. } /// A DRM scheduler job. pub struct Job { job: bindings::drm_sched_job, inner: T, - // Destroying an entity can leave its queued jobs to deferred work that still dereferences - // `job->sched` after the scheduler has been finalized. Keep the scheduler allocation alive - // for as long as any job references it; this does not delay finalization. + // Entity destruction can leave kill_jobs_work queued on the system + // workqueue or waiting for a dependency. That work still dereferences + // job->sched->ops after drm_sched_fini has stopped the scheduler workers. + // Retain only the allocation: retaining SchedulerOwner here would keep + // fini from running while pending jobs still need its cancellation. _scheduler: Arc>, } impl Job { - /// Returns whether the job has finished, successfully or not. + /// Returns whether an armed job has finished, successfully or not. pub fn is_finished(&self) -> bool { - // The finished fence is initialized when the job is armed, which also assigns its - // scheduler. if self.job.sched.is_null() { return false; } - - // SAFETY: The job is armed, so its scheduler fence is valid and initialized. + // SAFETY: An armed job has an initialized scheduler fence, retained + // for its lifetime, including while the timeout worker owns the job. unsafe { bindings::dma_fence_get_status(addr_of_mut!((*self.job.s_fence).finished)) != 0 } } } @@ -320,8 +350,8 @@ struct EntityInner { impl Drop for EntityInner { fn drop(&mut self) { - // SAFETY: The EntityInner is initialized. This will cancel/free all jobs, possibly from - // deferred work. Each job keeps the scheduler allocation alive until it is freed. + // SAFETY: The EntityInner is initialized. Jobs removed from the entity + // may still await kill_jobs_work; each retains the scheduler allocation. unsafe { bindings::drm_sched_entity_destroy(&mut self.entity) }; } } @@ -340,7 +370,7 @@ impl Entity { let mut entity: KBox>> = KBox::new_uninit(GFP_KERNEL | __GFP_ZERO)?; - let mut sched_ptr = sched.0.allocation.sched.get(); + let mut sched_ptr = &sched.0.allocation.sched as *const _ as *mut _; // SAFETY: The Box is allocated above and valid. unsafe { @@ -384,9 +414,13 @@ impl Entity { // SAFETY: The Box pointer is valid, and this initializes the inner member. unsafe { addr_of_mut!((*job.as_mut_ptr()).inner).write(inner) }; - // SAFETY: The Box pointer is valid, and this initializes the scheduler reference. + // SAFETY: The job allocation is valid and this initializes its last + // field. Arc::clone is infallible and the entity still owns the live + // scheduler, so every successfully initialized job retains its C + // callback target before it can be armed or submitted. unsafe { - addr_of_mut!((*job.as_mut_ptr())._scheduler).write(self.0.sched.allocation.clone()) + addr_of_mut!((*job.as_mut_ptr())._scheduler) + .write(self.0.as_ref().get_ref().sched.allocation.clone()) }; // SAFETY: All fields of the Job are now initialized. @@ -394,33 +428,36 @@ impl Entity { } } -/// DRM scheduler allocation, kept alive by the scheduler owner and by every job. +/// DRM scheduler allocation, retained by live owners and deferred job callbacks. pub struct SchedulerInner { - sched: Opaque, + sched: bindings::drm_gpu_scheduler, _p: PhantomData, } -// SAFETY: TODO -unsafe impl Sync for SchedulerInner {} -// SAFETY: TODO -unsafe impl Send for SchedulerInner {} - -/// Owner of an initialized DRM scheduler, shared by the scheduler handle and its entities. -/// -/// Jobs only reference the allocation, so that pending jobs cannot keep the scheduler from -/// being finalized. +// Only public scheduler/entity handles keep this owner alive. Jobs retain +// SchedulerInner instead, so outstanding work cannot prevent finalization. struct SchedulerOwner { allocation: Arc>, } impl Drop for SchedulerOwner { fn drop(&mut self) { - // SAFETY: The scheduler is valid. drm_sched_fini() stops the scheduler workers, then - // cancels and frees all remaining jobs through `cancel_job_cb()`. - unsafe { bindings::drm_sched_fini(self.allocation.sched.get()) }; + // SAFETY: fini stops submission, synchronizes the timeout worker and + // cancels/frees remaining jobs through our synchronized cancel callback. + // The C contract explicitly prohibits drm_sched_stop before fini. + // Deferred entity-kill callbacks are on a different workqueue and may + // survive this call. Their job guards retain allocation through cleanup. + unsafe { + bindings::drm_sched_fini(addr_of!(self.allocation.sched).cast_mut()) + }; } } +// SAFETY: TODO +unsafe impl Sync for SchedulerInner {} +// SAFETY: TODO +unsafe impl Send for SchedulerInner {} + /// A DRM Scheduler pub struct Scheduler(Arc>); @@ -444,16 +481,15 @@ impl Scheduler { ) -> Result> { let mut sched: UniqueArc>> = UniqueArc::new_uninit(GFP_KERNEL)?; - // Allocate the owner up front: nothing may fail once the scheduler is initialized. - let owner: UniqueArc>> = UniqueArc::new_uninit(GFP_KERNEL)?; - - // SAFETY: `sched` was just allocated and is valid for writes. - let sched_ptr = Opaque::cast_into(unsafe { addr_of!((*sched.as_mut_ptr()).sched) }); + // Allocate the owner before starting C workers: no fallible allocation + // may abandon an initialized scheduler without running drm_sched_fini. + let owner: UniqueArc>> = + UniqueArc::new_uninit(GFP_KERNEL)?; // SAFETY: zero sched->sched_rq as drm_sched_init() uses it to exit early withoput initialisation // TODO: allocate sched zzeroed instead unsafe { - (*sched_ptr).sched_rq = core::ptr::null_mut(); + (*sched.as_mut_ptr()).sched.sched_rq = core::ptr::null_mut(); }; let init_ops = bindings::drm_sched_init_args { @@ -471,9 +507,15 @@ impl Scheduler { // SAFETY: The drm_sched pointer is valid and pinned as it was just allocated above. // `device` is valid by its type invarants - to_result(unsafe { bindings::drm_sched_init(sched_ptr, addr_of!(init_ops)) })?; + to_result(unsafe { + bindings::drm_sched_init( + addr_of_mut!((*sched.as_mut_ptr()).sched), + addr_of!(init_ops), + ) + })?; - // SAFETY: All fields of SchedulerInner are now initialized. + // SAFETY: C initialized sched and PhantomData has no stored state. + // No fallible operation follows successful initialization. let allocation = unsafe { sched.assume_init() }.into(); Ok(Scheduler(owner.write(SchedulerOwner { allocation }).into())) } From a94102f68cdf476d5f55822b9311781af08274e7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0681/2680] mm: make VM operations aware of per-process mapping sizes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Separate target-mm virtual geometry from native physical pages across faults, COW, reclaim, swap, GUP, VMAs and accounting. Supply explicit fragment interfaces and keep native defaults until the optional backend is enabled later in the series. [ryan: adapted to 7.2.10 — minimal; for DJ: fs/userfaultfd.c hunks moved to mm/userfaultfd.c (7.2 merged the files); madvise range check rebuilt on 7.2's check_input_range() with the mm granule and the subpage EOPNOTSUPP list; empty_zero_page stays const (7.2) at ARCH_ZERO_PAGE_SIZE; mm_init() lost its user_ns argument; khugepaged/rmap/migrate/swapfile hunks re-applied on 7.2's mTHP collapse, hugetlb pte read and swap-table rework; the MGLRU legacy-memcg writeback hunk is dropped because 7.2 removed that path; limit_gfp_mask() is now thp_shmem_limit_gfp_mask(); the coarse swapin charge looks up the swap-table cgroup id for 7.2's mem_cgroup_swapin_charge_folio()] Signed-off-by: Diljit Singh (cherry picked from commit 81fd8e0b21629d0fdcfd36bed3c684cf278b6113) --- arch/x86/kernel/uprobes.c | 6 +- fs/proc/array.c | 3 +- fs/proc/task_mmu.c | 295 ++++- include/asm-generic/pgalloc.h | 17 +- include/asm-generic/tlb.h | 43 +- include/linux/binfmts.h | 6 + include/linux/coredump.h | 13 +- include/linux/highmem.h | 15 + include/linux/huge_mm.h | 4 + include/linux/khugepaged.h | 8 + include/linux/mm.h | 104 +- include/linux/mm_granule.h | 245 ++++ include/linux/mm_subpage.h | 277 ++++ include/linux/mm_types.h | 31 +- include/linux/mman.h | 11 + include/linux/mmu_context.h | 5 + include/linux/page_table_check.h | 4 + include/linux/pagemap.h | 5 +- include/linux/pgtable.h | 98 +- include/linux/rmap.h | 106 +- include/linux/sched.h | 5 + include/linux/sched/mm.h | 4 + include/linux/sched/signal.h | 2 +- include/linux/shmem_fs.h | 51 + include/linux/swap.h | 26 +- include/linux/syscalls.h | 5 + include/linux/uprobes.h | 1 + include/trace/events/kmem.h | 2 +- include/trace/events/mmap.h | 2 +- include/trace/events/oom.h | 9 +- include/uapi/linux/prctl.h | 18 + ipc/shm.c | 36 +- kernel/events/core.c | 83 +- kernel/events/internal.h | 37 +- kernel/events/ring_buffer.c | 47 +- kernel/events/uprobes.c | 212 ++- kernel/fork.c | 35 +- kernel/futex/core.c | 57 +- kernel/sched/fair.c | 6 +- kernel/sys.c | 52 +- kernel/trace/trace_events_user.c | 36 +- kernel/tsacct.c | 16 +- lib/vdso/datastore.c | 3 +- mm/Kconfig | 30 +- mm/damon/ops-common.c | 10 +- mm/filemap.c | 73 +- mm/gup.c | 590 ++++++++- mm/huge_memory.c | 11 +- mm/internal.h | 87 +- mm/khugepaged.c | 104 +- mm/ksm.c | 9 +- mm/madvise.c | 151 ++- mm/memory-failure.c | 45 + mm/memory.c | 2085 +++++++++++++++++++++++++----- mm/mempolicy.c | 57 +- mm/migrate.c | 85 +- mm/migrate_device.c | 2 +- mm/mincore.c | 43 +- mm/mlock.c | 42 +- mm/mm_init.c | 2 +- mm/mmap.c | 139 +- mm/mmu_gather.c | 67 +- mm/mprotect.c | 143 +- mm/mremap.c | 89 +- mm/mseal.c | 4 +- mm/msync.c | 6 +- mm/oom_kill.c | 25 +- mm/page_idle.c | 7 +- mm/page_io.c | 61 +- mm/page_table_check.c | 142 +- mm/page_vma_mapped.c | 61 +- mm/pagewalk.c | 49 +- mm/pgtable-generic.c | 35 +- mm/process_vm_access.c | 96 +- mm/rmap.c | 518 +++++++- mm/shmem.c | 513 +++++++- mm/swap.h | 22 +- mm/swap_state.c | 6 +- mm/swapfile.c | 388 +++++- mm/truncate.c | 2 + mm/user-subpage-internal.h | 69 + mm/userfaultfd.c | 575 +++++++- mm/util.c | 106 +- mm/vma.c | 180 ++- mm/vma.h | 40 +- mm/vma_exec.c | 6 +- mm/vma_init.c | 5 +- mm/vmscan.c | 109 +- 88 files changed, 7666 insertions(+), 1264 deletions(-) create mode 100644 include/linux/mm_granule.h create mode 100644 include/linux/mm_subpage.h create mode 100644 mm/user-subpage-internal.h diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c index 65a2de82ecd29..746a94f2a1ee3 100644 --- a/arch/x86/kernel/uprobes.c +++ b/arch/x86/kernel/uprobes.c @@ -880,13 +880,15 @@ struct write_opcode_ctx { * Verification callback used by uprobe_write calls to make sure the underlying * instruction is in the expected stage of the INT3 update sequence. */ -static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode, +static int verify_insn(struct page *page, unsigned long vaddr, unsigned int page_offset, + uprobe_opcode_t *new_opcode, int nbytes, void *data) { struct write_opcode_ctx *ctx = data; uprobe_opcode_t old_opcode[OPT_INSN_SIZE]; - uprobe_copy_from_page(page, ctx->base, old_opcode, OPT_INSN_SIZE); + uprobe_copy_from_page(page, page_offset - (vaddr - ctx->base), + old_opcode, OPT_INSN_SIZE); switch (ctx->expect) { case EXPECT_SWBP: diff --git a/fs/proc/array.c b/fs/proc/array.c index 479ea8cb4ef43..5b45506942991 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -610,7 +610,8 @@ static int do_task_stat(struct seq_file *m, struct pid_namespace *ns, seq_put_decimal_ull(m, " ", 0); seq_put_decimal_ull(m, " ", start_time); seq_put_decimal_ull(m, " ", vsize); - seq_put_decimal_ull(m, " ", mm ? get_mm_rss(mm) : 0); + seq_put_decimal_ull(m, " ", mm ? + mm_pages_for_reader(mm, current->mm, get_mm_rss(mm)) : 0); seq_put_decimal_ull(m, " ", rsslim); seq_put_decimal_ull(m, " ", mm ? (permitted ? mm->start_code : 1) : 0); seq_put_decimal_ull(m, " ", mm ? (permitted ? mm->end_code : 1) : 0); diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 229d1fc3d7f1c..985e5beb6e408 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include #include #include #include @@ -33,7 +34,7 @@ #define SENTINEL_VMA_GATE -2 #define SEQ_PUT_DEC(str, val) \ - seq_put_decimal_ull_width(m, str, (val) << (PAGE_SHIFT-10), 8) + seq_put_decimal_ull_width(m, str, mm_pages_to_kb(mm, val), 8) void task_mem(struct seq_file *m, struct mm_struct *mm) { unsigned long text, lib, swap, anon, file, shmem; @@ -58,15 +59,18 @@ void task_mem(struct seq_file *m, struct mm_struct *mm) hiwater_rss = mm->hiwater_rss; /* split executable areas between text and lib */ - text = PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK); - text = min(text, mm->exec_vm << PAGE_SHIFT); - lib = (mm->exec_vm << PAGE_SHIFT) - text; + text = mm_page_align(mm, mm->end_code) - + (mm->start_code & mm_page_mask(mm)); + text = min(text, mm_pages_to_bytes(mm, mm->exec_vm)); + lib = (mm_pages_to_bytes(mm, mm->exec_vm)) - text; swap = get_mm_counter_sum(mm, MM_SWAPENTS); SEQ_PUT_DEC("VmPeak:\t", hiwater_vm); SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm); - SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm); - SEQ_PUT_DEC(" kB\nVmPin:\t", atomic64_read(&mm->pinned_vm)); + seq_put_decimal_ull_width(m, " kB\nVmLck:\t", + mm_locked_vm_bytes(mm) >> 10, 8); + seq_put_decimal_ull_width(m, " kB\nVmPin:\t", + atomic64_read(&mm->pinned_vm) << (PAGE_SHIFT - 10), 8); SEQ_PUT_DEC(" kB\nVmHWM:\t", hiwater_rss); SEQ_PUT_DEC(" kB\nVmRSS:\t", total_rss); SEQ_PUT_DEC(" kB\nRssAnon:\t", anon); @@ -88,20 +92,25 @@ void task_mem(struct seq_file *m, struct mm_struct *mm) unsigned long task_vsize(struct mm_struct *mm) { - return PAGE_SIZE * mm->total_vm; + return mm_pages_to_bytes(mm, mm->total_vm); } unsigned long task_statm(struct mm_struct *mm, unsigned long *shared, unsigned long *text, unsigned long *data, unsigned long *resident) { - *shared = get_mm_counter_sum(mm, MM_FILEPAGES) + + struct mm_struct *reader = current->mm; + unsigned long file = get_mm_counter_sum(mm, MM_FILEPAGES) + get_mm_counter_sum(mm, MM_SHMEMPAGES); - *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) - >> PAGE_SHIFT; - *data = mm->data_vm + mm->stack_vm; - *resident = *shared + get_mm_counter_sum(mm, MM_ANONPAGES); - return mm->total_vm; + unsigned long code = (mm_page_align(mm, mm->end_code) - + (mm->start_code & mm_page_mask(mm))) >> mm_page_shift(mm); + + *shared = mm_pages_for_reader(mm, reader, file); + *text = mm_pages_for_reader(mm, reader, code); + *data = mm_pages_for_reader(mm, reader, mm->data_vm + mm->stack_vm); + *resident = mm_pages_for_reader(mm, reader, + file + get_mm_counter_sum(mm, MM_ANONPAGES)); + return mm_pages_for_reader(mm, reader, mm->total_vm); } #ifdef CONFIG_NUMA @@ -508,7 +517,8 @@ show_map_vma(struct seq_file *m, struct vm_area_struct *vma) dev = inode->i_sb->s_dev; ino = inode->i_ino; - pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT; + pgoff = (((loff_t)vma->vm_pgoff) << PAGE_SHIFT) + + vma_subpage_offset(vma); } start = vma->vm_start; @@ -748,7 +758,8 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg) if (vma->vm_file) { const struct inode *inode = file_user_inode(vma->vm_file); - karg.vma_offset = ((__u64)vma->vm_pgoff) << PAGE_SHIFT; + karg.vma_offset = (((__u64)vma->vm_pgoff) << PAGE_SHIFT) + + vma_subpage_offset(vma); karg.dev_major = MAJOR(inode->i_sb->s_dev); karg.dev_minor = MINOR(inode->i_sb->s_dev); karg.inode = inode->i_ino; @@ -888,6 +899,50 @@ const struct file_operations proc_pid_maps_operations = { */ #define PSS_SHIFT 12 +#if defined(CONFIG_MM_SUBPAGE) && (defined(CONFIG_PROC_PAGE_MONITOR) || defined(CONFIG_NUMA)) +/* Snapshot only: rmap and quarter counters can change in other address spaces. */ +static void file_quarter_mapcounts(struct page *page, struct mm_subpage_mapcounts *counts) +{ + int maps = folio_precise_page_mapcount(page_folio(page), page); + unsigned int small = mm_subpage_file_mapcounts(page, counts); + unsigned int native = maps > small ? maps - small : 0; + unsigned int i; + + for (i = 0; i < MM_SUBPAGES_PER_PAGE; i++) + counts->slots[i] += native; +} + +static unsigned int proc_leaf_mapcount(struct vm_area_struct *vma, pte_t pte, + struct page *page) +{ + struct folio *folio = page_folio(page); + struct mm_subpage_mapcounts counts; + unsigned int i, count = 0; + + if (!folio_test_anon(folio)) { + unsigned int first = mm_page_size(vma->vm_mm) < PAGE_SIZE ? + offset_in_page(pte_phys_mm(vma->vm_mm, pte)) / SZ_4K : 0; + unsigned int nr = mm_page_size(vma->vm_mm) / SZ_4K; + + file_quarter_mapcounts(page, &counts); + for (i = first; i < first + nr; i++) + count = max(count, counts.slots[i]); + return count; + } + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + struct mm_subpage *slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + + if (slot) { + count = mm_subpage_mapcount(slot); + mm_subpage_put(slot); + } + return count; + } + return folio_precise_page_mapcount(folio, page); +} + +#endif + #ifdef CONFIG_PROC_PAGE_MONITOR struct mem_size_stats { unsigned long resident; @@ -944,6 +999,73 @@ static void smaps_page_accumulate(struct mem_size_stats *mss, } } +#ifdef CONFIG_MM_SUBPAGE +static void smaps_account_file_extent(struct mem_size_stats *mss, struct page *page, + unsigned int offset, unsigned long size, bool dirty, bool locked, bool present) +{ + struct folio *folio = page_folio(page); + struct mm_subpage_mapcounts counts = {}; + unsigned int i; + + if (present) + file_quarter_mapcounts(page, &counts); + for (i = offset / SZ_4K; i < (offset + size) / SZ_4K; i++) { + unsigned long pss = SZ_4K << PSS_SHIFT; + + if (counts.slots[i] > 1) + pss /= counts.slots[i]; + smaps_page_accumulate(mss, folio, SZ_4K, pss, dirty, locked, + present && counts.slots[i] < 2); + } +} + +static void smaps_account_small(struct mem_size_stats *mss, struct vm_area_struct *vma, + struct page *page, pte_t pte, bool young, + bool dirty, bool locked, bool present) +{ + struct folio *folio = page_folio(page); + unsigned long size = mm_page_size(vma->vm_mm), pss = size << PSS_SHIFT; + unsigned int count = present && folio_test_anon(folio) ? + proc_leaf_mapcount(vma, pte, page) : 0; + + mss->resident += size; + if (folio_test_anon(folio)) { + mss->anonymous += size; + if (present && !dirty) { + struct mm_subpage *slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + + if (slot) { + if (mm_subpage_is_lazyfree(slot)) + mss->lazyfree += size; + mm_subpage_put(slot); + } + } + } + if (folio_test_ksm(folio)) + mss->ksm += size; + if (young || folio_test_young(folio) || folio_test_referenced(folio)) + mss->referenced += size; + if (present && !folio_test_anon(folio)) { + smaps_account_file_extent(mss, page, offset_in_page(pte_phys_mm(vma->vm_mm, pte)), + size, dirty, locked, present); + return; + } + if (count > 1) + pss /= count; + smaps_page_accumulate(mss, folio, size, pss, dirty, locked, present && count < 2); +} + +static bool smaps_account_file_quarters(struct mem_size_stats *mss, struct page *page, + bool dirty, bool locked) +{ + if (folio_test_anon(page_folio(page))) + return false; + smaps_account_file_extent(mss, page, 0, PAGE_SIZE, dirty, locked, true); + return true; +} + +#endif + static void smaps_account(struct mem_size_stats *mss, struct page *page, bool compound, bool young, bool dirty, bool locked, bool present) @@ -1007,6 +1129,11 @@ static void smaps_account(struct mem_size_stats *mss, struct page *page, for (i = 0; i < nr; i++, page++) { unsigned long pss = PAGE_SIZE << PSS_SHIFT; +#ifdef CONFIG_MM_SUBPAGE + if (smaps_account_file_quarters(mss, page, dirty, locked)) + continue; +#endif + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) { mapcount = folio_precise_page_mapcount(folio, page); exclusive = mapcount < 2; @@ -1026,6 +1153,14 @@ static int smaps_pte_hole(unsigned long addr, unsigned long end, struct mem_size_stats *mss = walk->private; struct vm_area_struct *vma = walk->vma; + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + struct vm_page_offset pos = vma_page_offset_at(vma, addr); + u64 start = ((u64)pos.index << PAGE_SHIFT) + pos.offset; + + mss->swap += shmem_partial_swap_usage_bytes(vma->vm_file->f_mapping, + start, start + end - addr); + return 0; + } mss->swap += shmem_partial_swap_usage(walk->vma->vm_file->f_mapping, linear_page_index(vma, addr), linear_page_index(vma, end)); @@ -1041,7 +1176,7 @@ static void smaps_pte_hole_lookup(unsigned long addr, struct mm_walk *walk) #ifdef CONFIG_SHMEM if (walk->ops->pte_hole) { /* depth is not used */ - smaps_pte_hole(addr, addr + PAGE_SIZE, 0, walk); + smaps_pte_hole(addr, addr + mm_page_size(walk->mm), 0, walk); } #endif } @@ -1069,15 +1204,20 @@ static void smaps_pte_entry(pte_t *pte, unsigned long addr, if (softleaf_is_swap(entry)) { int mapcount; - mss->swap += PAGE_SIZE; - mapcount = swp_swapcount(entry); + mss->swap += mm_page_size(vma->vm_mm); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + mapcount = swap_subpage_count(entry, pte_swp_subpage_offset(ptent)); + else +#endif + mapcount = swp_swapcount(entry); if (mapcount >= 2) { - u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT; + u64 pss_delta = (u64)mm_page_size(vma->vm_mm) << PSS_SHIFT; do_div(pss_delta, mapcount); mss->swap_pss += pss_delta; } else { - mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT; + mss->swap_pss += (u64)mm_page_size(vma->vm_mm) << PSS_SHIFT; } } else if (softleaf_has_pfn(entry)) { if (softleaf_is_device_private(entry)) @@ -1089,6 +1229,12 @@ static void smaps_pte_entry(pte_t *pte, unsigned long addr, if (!page) return; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + smaps_account_small(mss, vma, page, ptent, young, dirty, locked, present); + return; + } +#endif smaps_account(mss, page, false, young, dirty, locked, present); } @@ -1155,7 +1301,7 @@ static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; pte++, addr += PAGE_SIZE) + for (; addr != end; pte++, addr += mm_page_size(vma->vm_mm)) smaps_pte_entry(pte, addr, walk); pte_unmap_unlock(pte - 1, ptl); out: @@ -1799,7 +1945,7 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; pte++, addr += PAGE_SIZE) { + for (; addr != end; pte++, addr += mm_page_size(vma->vm_mm)) { ptent = ptep_get(pte); if (cp->type == CLEAR_REFS_SOFT_DIRTY) { @@ -1942,9 +2088,6 @@ struct pagemapread { bool show_pfn; }; -#define PAGEMAP_WALK_SIZE (PMD_SIZE) -#define PAGEMAP_WALK_MASK (PMD_MASK) - #define PM_ENTRY_BYTES sizeof(pagemap_entry_t) #define PM_PFRAME_BITS 55 #define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0) @@ -1971,7 +2114,7 @@ static int add_to_pagemap(pagemap_entry_t *pme, struct pagemapread *pm) return 0; } -static bool __folio_page_mapped_exclusively(struct folio *folio, struct page *page) +static inline bool __folio_page_mapped_exclusively(struct folio *folio, struct page *page) { if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) return folio_precise_page_mapcount(folio, page) == 1; @@ -1996,7 +2139,7 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end, else hole_end = end; - for (; addr < hole_end; addr += PAGE_SIZE) { + for (; addr < hole_end; addr += mm_page_size(walk->mm)) { err = add_to_pagemap(&pme, pm); if (err) goto out; @@ -2008,7 +2151,7 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end, /* Addresses in the VMA. */ if (vma->vm_flags & VM_SOFTDIRTY) pme = make_pme(0, PM_SOFT_DIRTY); - for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) { + for (; addr < min(end, vma->vm_end); addr += mm_page_size(walk->mm)) { err = add_to_pagemap(&pme, pm); if (err) goto out; @@ -2018,6 +2161,16 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end, return err; } +static bool pagemap_page_exclusive(struct vm_area_struct *vma, pte_t pte, + struct folio *folio, struct page *page) +{ +#ifdef CONFIG_MM_SUBPAGE + return proc_leaf_mapcount(vma, pte, page) == 1; +#else + return __folio_page_mapped_exclusively(folio, page); +#endif +} + static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, struct vm_area_struct *vma, unsigned long addr, pte_t pte) { @@ -2030,7 +2183,7 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, if (pte_present(pte)) { if (pm->show_pfn) - frame = pte_pfn(pte); + frame = pte_phys_mm(vma->vm_mm, pte) >> mm_page_shift(vma->vm_mm); flags |= PM_PRESENT; page = vm_normal_page(vma, addr, pte); if (pte_soft_dirty(pte)) @@ -2056,6 +2209,12 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, offset = softleaf_to_pfn(entry); else offset = swp_offset(entry); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE && + (softleaf_is_swap(entry) || softleaf_has_pfn(entry))) + offset = (offset << (PAGE_SHIFT - mm_page_shift(vma->vm_mm))) + + (pte_swp_subpage_offset(pte) >> mm_page_shift(vma->vm_mm)); +#endif frame = swp_type(entry) | (offset << MAX_SWAPFILES_SHIFT); } @@ -2072,8 +2231,7 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, folio = page_folio(page); if (!folio_test_anon(folio)) flags |= PM_FILE; - if ((flags & PM_PRESENT) && - __folio_page_mapped_exclusively(folio, page)) + if ((flags & PM_PRESENT) && pagemap_page_exclusive(vma, pte, folio, page)) flags |= PM_MMAP_EXCLUSIVE; } @@ -2189,7 +2347,7 @@ static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, walk->action = ACTION_AGAIN; return err; } - for (; addr < end; pte++, addr += PAGE_SIZE) { + for (; addr < end; pte++, addr += mm_page_size(walk->mm)) { pagemap_entry_t pme; pme = pte_to_pagemap_entry(pm, vma, addr, ptep_get(pte)); @@ -2323,7 +2481,7 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, /* do not disclose physical addresses: attack vector */ pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN); - pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT); + pm.len = (pmd_size_mm(mm) >> mm_page_shift(mm)); pm.buffer = kmalloc_array(pm.len, PM_ENTRY_BYTES, GFP_KERNEL); ret = -ENOMEM; if (!pm.buffer) @@ -2335,16 +2493,16 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, /* watch out for wraparound */ start_vaddr = end_vaddr; - if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) { + if (svpfn <= (ULONG_MAX >> mm_page_shift(mm))) { unsigned long end; ret = mmap_read_lock_killable(mm); if (ret) goto out_free; - start_vaddr = untagged_addr_remote(mm, svpfn << PAGE_SHIFT); + start_vaddr = untagged_addr_remote(mm, svpfn << mm_page_shift(mm)); mmap_read_unlock(mm); - end = start_vaddr + ((count / PM_ENTRY_BYTES) << PAGE_SHIFT); + end = start_vaddr + ((count / PM_ENTRY_BYTES) << mm_page_shift(mm)); if (end >= start_vaddr && end < mm->task_size) end_vaddr = end; } @@ -2359,7 +2517,7 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, unsigned long end; pm.pos = 0; - end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK; + end = (start_vaddr + pmd_size_mm(mm)) & pmd_mask_mm(mm); /* overflow ? */ if (end < start_vaddr || end > end_vaddr) end = end_vaddr; @@ -2420,6 +2578,7 @@ static int pagemap_release(struct inode *inode, struct file *file) struct pagemap_scan_private { struct pm_scan_arg arg; + unsigned long page_size; unsigned long masks_of_interest, cur_vma_category; struct page_region *vec_buf; unsigned long vec_buf_len, vec_buf_index, found_pages; @@ -2646,7 +2805,7 @@ static void pagemap_scan_backout_range(struct pagemap_scan_private *p, else cur_buf->start = cur_buf->end = 0; - p->found_pages -= (end - addr) / PAGE_SIZE; + p->found_pages -= (end - addr) / p->page_size; } #endif @@ -2758,11 +2917,11 @@ static int pagemap_scan_output(unsigned long categories, categories &= p->arg.return_mask; - n_pages = (*end - addr) / PAGE_SIZE; + n_pages = (*end - addr) / p->page_size; if (check_add_overflow(p->found_pages, n_pages, &total_pages) || total_pages > p->arg.max_pages) { size_t n_too_much = total_pages - p->arg.max_pages; - *end -= n_too_much * PAGE_SIZE; + *end -= n_too_much * p->page_size; n_pages -= n_too_much; ret = -ENOSPC; } @@ -2856,7 +3015,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, if ((p->arg.flags & PM_SCAN_WP_MATCHING) && !p->vec_out) { /* Fast path for performing exclusive WP */ - for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { + for (addr = start; addr != end; pte++, addr += p->page_size) { pte_t ptent = ptep_get(pte); if ((pte_present(ptent) && pte_uffd_wp(ptent)) || @@ -2865,7 +3024,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, make_uffd_wp_pte(vma, addr, pte, ptent); if (!flush_end) start = addr; - flush_end = addr + PAGE_SIZE; + flush_end = addr + p->page_size; } goto flush_and_return; } @@ -2873,8 +3032,8 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, if (!p->arg.category_anyof_mask && !p->arg.category_inverted && p->arg.category_mask == PAGE_IS_WRITTEN && p->arg.return_mask == PAGE_IS_WRITTEN) { - for (addr = start; addr < end; pte++, addr += PAGE_SIZE) { - unsigned long next = addr + PAGE_SIZE; + for (addr = start; addr < end; pte++, addr += p->page_size) { + unsigned long next = addr + p->page_size; pte_t ptent = ptep_get(pte); if ((pte_present(ptent) && pte_uffd_wp(ptent)) || @@ -2894,11 +3053,11 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, goto flush_and_return; } - for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { + for (addr = start; addr != end; pte++, addr += p->page_size) { pte_t ptent = ptep_get(pte); unsigned long categories = p->cur_vma_category | pagemap_page_category(p, vma, addr, ptent); - unsigned long next = addr + PAGE_SIZE; + unsigned long next = addr + p->page_size; if (!pagemap_scan_is_interesting_page(categories, p)) continue; @@ -3094,7 +3253,7 @@ static const struct mm_walk_ops pagemap_scan_ops = { .hugetlb_entry = pagemap_scan_hugetlb_entry, }; -static int pagemap_scan_get_args(struct pm_scan_arg *arg, +static int pagemap_scan_get_args(struct mm_struct *mm, struct pm_scan_arg *arg, unsigned long uarg) { if (copy_from_user(arg, (void __user *)uarg, sizeof(*arg))) @@ -3115,7 +3274,7 @@ static int pagemap_scan_get_args(struct pm_scan_arg *arg, arg->vec = untagged_addr((unsigned long)arg->vec); /* Validate memory pointers */ - if (!IS_ALIGNED(arg->start, PAGE_SIZE)) + if (!IS_ALIGNED(arg->start, mm_page_size(mm))) return -EINVAL; if (!access_ok((void __user *)(long)arg->start, arg->end - arg->start)) return -EFAULT; @@ -3128,7 +3287,7 @@ static int pagemap_scan_get_args(struct pm_scan_arg *arg, return -EFAULT; /* Fixup default values */ - arg->end = ALIGN(arg->end, PAGE_SIZE); + arg->end = ALIGN(arg->end, mm_page_size(mm)); arg->walk_end = 0; if (!arg->max_pages) arg->max_pages = ULONG_MAX; @@ -3147,12 +3306,13 @@ static int pagemap_scan_writeback_args(struct pm_scan_arg *arg, return 0; } -static int pagemap_scan_init_bounce_buffer(struct pagemap_scan_private *p) +static int pagemap_scan_init_bounce_buffer(struct mm_struct *mm, + struct pagemap_scan_private *p) { if (!p->arg.vec_len) return 0; - p->vec_buf_len = min_t(size_t, PAGEMAP_WALK_SIZE >> PAGE_SHIFT, + p->vec_buf_len = min_t(size_t, pmd_size_mm(mm) >> mm_page_shift(mm), p->arg.vec_len); p->vec_buf = kmalloc_objs(*p->vec_buf, p->vec_buf_len); if (!p->vec_buf) @@ -3193,18 +3353,18 @@ static long pagemap_scan_flush_buffer(struct pagemap_scan_private *p) static long do_pagemap_scan(struct mm_struct *mm, unsigned long uarg) { - struct pagemap_scan_private p = {0}; + struct pagemap_scan_private p = { .page_size = mm_page_size(mm) }; unsigned long walk_start; size_t n_ranges_out = 0; int ret; - ret = pagemap_scan_get_args(&p.arg, uarg); + ret = pagemap_scan_get_args(mm, &p.arg, uarg); if (ret) return ret; p.masks_of_interest = p.arg.category_mask | p.arg.category_anyof_mask | p.arg.return_mask; - ret = pagemap_scan_init_bounce_buffer(&p); + ret = pagemap_scan_init_bounce_buffer(mm, &p); if (ret) return ret; @@ -3306,16 +3466,17 @@ struct numa_maps_private { struct numa_maps md; }; +static int numa_page_mapcount(struct page *page) +{ + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + return folio_precise_page_mapcount(page_folio(page), page); + return folio_average_page_mapcount(page_folio(page)); +} + static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty, - unsigned long nr_pages) + unsigned long nr_pages, int count) { struct folio *folio = page_folio(page); - int count; - - if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) - count = folio_precise_page_mapcount(folio, page); - else - count = folio_average_page_mapcount(folio); md->pages += nr_pages; if (pte_dirty || folio_test_dirty(folio)) @@ -3405,7 +3566,7 @@ static int gather_pte_stats(pmd_t *pmd, unsigned long addr, page = can_gather_numa_stats_pmd(*pmd, vma, addr); if (page) gather_stats(page, md, pmd_dirty(*pmd), - HPAGE_PMD_SIZE/PAGE_SIZE); + HPAGE_PMD_SIZE/PAGE_SIZE, numa_page_mapcount(page)); spin_unlock(ptl); return 0; } @@ -3420,9 +3581,13 @@ static int gather_pte_stats(pmd_t *pmd, unsigned long addr, struct page *page = can_gather_numa_stats(ptent, vma, addr); if (!page) continue; - gather_stats(page, md, pte_dirty(ptent), 1); +#ifdef CONFIG_MM_SUBPAGE + gather_stats(page, md, pte_dirty(ptent), 1, proc_leaf_mapcount(vma, ptent, page)); +#else + gather_stats(page, md, pte_dirty(ptent), 1, numa_page_mapcount(page)); +#endif - } while (pte++, addr += PAGE_SIZE, addr != end); + } while (pte++, addr += mm_page_size(vma->vm_mm), addr != end); pte_unmap_unlock(orig_pte, ptl); cond_resched(); return 0; @@ -3444,7 +3609,7 @@ static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask, page = pte_page(huge_pte); md = walk->private; - gather_stats(page, md, pte_dirty(huge_pte), 1); + gather_stats(page, md, pte_dirty(huge_pte), 1, numa_page_mapcount(page)); out: spin_unlock(ptl); return 0; diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h index 051aa1331051c..26360ab2f81ba 100644 --- a/include/asm-generic/pgalloc.h +++ b/include/asm-generic/pgalloc.h @@ -81,7 +81,7 @@ static inline pgtable_t __pte_alloc_one_noprof(struct mm_struct *mm, gfp_t gfp) return NULL; } - return ptdesc_page(ptdesc); + return pgtable_from_page(ptdesc_page(ptdesc)); } #define __pte_alloc_one(...) alloc_hooks(__pte_alloc_one_noprof(__VA_ARGS__)) @@ -111,17 +111,18 @@ static inline pgtable_t pte_alloc_one_noprof(struct mm_struct *mm) * @mm: the mm_struct of the current context * @pte_page: the `struct page` referencing the ptdesc */ -static inline void pte_free(struct mm_struct *mm, struct page *pte_page) +#ifndef __HAVE_ARCH_PTE_FREE +static inline void pte_free(struct mm_struct *mm, pgtable_t pte_page) { - struct ptdesc *ptdesc = page_ptdesc(pte_page); + struct ptdesc *ptdesc = page_ptdesc(pgtable_to_page(pte_page)); pagetable_dtor_free(ptdesc); } +#endif #if CONFIG_PGTABLE_LEVELS > 2 -#ifndef __HAVE_ARCH_PMD_ALLOC_ONE /** * pmd_alloc_one - allocate memory for a PMD-level page table * @mm: the mm_struct of the current context @@ -133,7 +134,7 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte_page) * * Return: pointer to the allocated memory or %NULL on error */ -static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +static inline pmd_t *__pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) { struct ptdesc *ptdesc; gfp_t gfp = GFP_PGTABLE_USER; @@ -153,6 +154,12 @@ static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long ad return ptdesc_address(ptdesc); } + +#ifndef __HAVE_ARCH_PMD_ALLOC_ONE +static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +{ + return __pmd_alloc_one_noprof(mm, addr); +} #define pmd_alloc_one(...) alloc_hooks(pmd_alloc_one_noprof(__VA_ARGS__)) #endif diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index bdcc2778ac64f..900bee13b40cb 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -222,6 +222,15 @@ static inline void __tlb_remove_table(void *table) } #endif +/* Architectures may pass opaque table tokens instead of descriptors. */ +#ifndef tlb_table_is_ptdesc +static inline bool tlb_table_is_ptdesc(void *table) +{ + (void)table; + return true; +} +#endif + extern void tlb_remove_table(struct mmu_gather *tlb, void *table); #else /* !CONFIG_MMU_GATHER_TABLE_FREE */ @@ -295,6 +304,12 @@ extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, in bool __tlb_remove_folio_pages(struct mmu_gather *tlb, struct page *page, unsigned int nr_pages, bool delay_rmap); +#ifdef CONFIG_MM_SUBPAGE +struct mm_subpage; +/* Consumes one slot reference; true requires flushing the full batch. */ +bool __tlb_remove_subpage(struct mmu_gather *tlb, struct mm_subpage *subpage); +#endif + #ifdef CONFIG_SMP /* * This both sets 'delayed_rmap', and returns true. It would be an inline @@ -543,15 +558,15 @@ static inline void tlb_change_page_size(struct mmu_gather *tlb, static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb) { if (tlb->cleared_ptes) - return PAGE_SHIFT; + return mm_page_shift(tlb->mm); if (tlb->cleared_pmds) - return PMD_SHIFT; + return pmd_shift_mm(tlb->mm); if (tlb->cleared_puds) - return PUD_SHIFT; + return pud_shift_mm(tlb->mm); if (tlb->cleared_p4ds) - return P4D_SHIFT; + return p4d_shift_mm(tlb->mm); - return PAGE_SHIFT; + return mm_page_shift(tlb->mm); } static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) @@ -658,7 +673,7 @@ static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, u */ #define tlb_remove_tlb_entry(tlb, ptep, address) \ do { \ - tlb_flush_pte_range(tlb, address, PAGE_SIZE); \ + tlb_flush_pte_range(tlb, address, mm_page_size((tlb)->mm)); \ __tlb_remove_tlb_entry(tlb, ptep, address); \ } while (0) @@ -672,13 +687,15 @@ static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, u static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, pte_t *ptep, unsigned int nr, unsigned long address) { - tlb_flush_pte_range(tlb, address, PAGE_SIZE * nr); + unsigned long page_size = mm_page_size(tlb->mm); + + tlb_flush_pte_range(tlb, address, page_size * nr); for (;;) { __tlb_remove_tlb_entry(tlb, ptep, address); if (--nr == 0) break; ptep++; - address += PAGE_SIZE; + address += page_size; } } @@ -739,13 +756,13 @@ static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, * architecture to do its own odd thing, not cause pain for others * http://lkml.kernel.org/r/CA+55aFzBggoXtNXQeng5d_mRoDnaMBE5Y+URs+PHR67nUpMtaw@mail.gmail.com * - * For now w.r.t page table cache, mark the range_size as PAGE_SIZE + * For now w.r.t page table cache, mark the range_size as one mm granule. */ #ifndef pte_free_tlb #define pte_free_tlb(tlb, ptep, address) \ do { \ - tlb_flush_pmd_range(tlb, address, PAGE_SIZE); \ + tlb_flush_pmd_range(tlb, address, mm_page_size((tlb)->mm)); \ tlb->freed_tables = 1; \ __pte_free_tlb(tlb, ptep, address); \ } while (0) @@ -754,7 +771,7 @@ static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, #ifndef pmd_free_tlb #define pmd_free_tlb(tlb, pmdp, address) \ do { \ - tlb_flush_pud_range(tlb, address, PAGE_SIZE); \ + tlb_flush_pud_range(tlb, address, mm_page_size((tlb)->mm)); \ tlb->freed_tables = 1; \ __pmd_free_tlb(tlb, pmdp, address); \ } while (0) @@ -763,7 +780,7 @@ static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, #ifndef pud_free_tlb #define pud_free_tlb(tlb, pudp, address) \ do { \ - tlb_flush_p4d_range(tlb, address, PAGE_SIZE); \ + tlb_flush_p4d_range(tlb, address, mm_page_size((tlb)->mm)); \ tlb->freed_tables = 1; \ __pud_free_tlb(tlb, pudp, address); \ } while (0) @@ -772,7 +789,7 @@ static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, #ifndef p4d_free_tlb #define p4d_free_tlb(tlb, pudp, address) \ do { \ - __tlb_adjust_range(tlb, address, PAGE_SIZE); \ + __tlb_adjust_range(tlb, address, mm_page_size((tlb)->mm)); \ tlb->freed_tables = 1; \ __p4d_free_tlb(tlb, pudp, address); \ } while (0) diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h index 2c77e383e7375..68946678225e7 100644 --- a/include/linux/binfmts.h +++ b/include/linux/binfmts.h @@ -96,6 +96,8 @@ struct linux_binfmt { #ifdef CONFIG_COREDUMP int (*core_dump)(struct coredump_params *cprm); unsigned long min_coredump; /* minimal dump size */ + /* Optional per-mm replacement for formats with a variable page ABI. */ + unsigned long (*min_coredump_size)(void); #endif } __randomize_layout; @@ -124,6 +126,10 @@ static inline void insert_binfmt(struct linux_binfmt *fmt) extern void unregister_binfmt(struct linux_binfmt *); +#ifdef CONFIG_MM_SUBPAGE +int bprm_set_page_shift(struct linux_binprm *bprm, unsigned int page_shift); +#endif + extern int __must_check remove_arg_zero(struct linux_binprm *); extern int begin_new_exec(struct linux_binprm * bprm); extern void setup_new_exec(struct linux_binprm * bprm); diff --git a/include/linux/coredump.h b/include/linux/coredump.h index 7b38ee2e7913b..1b3697424b1fb 100644 --- a/include/linux/coredump.h +++ b/include/linux/coredump.h @@ -13,10 +13,21 @@ struct core_vma_metadata { unsigned long start, end; vm_flags_t flags; unsigned long dump_size; - unsigned long pgoff; + unsigned long pgoff; /* File offset in the dumped mm's page units. */ struct file *file; }; +/* NT_FILE offsets are expressed in the dumped process's page units. */ +static inline unsigned long core_vma_pgoff(const struct vm_area_struct *vma) +{ + unsigned int shift = mm_page_shift(vma->vm_mm); + + if (shift > PAGE_SHIFT) + return vma->vm_pgoff >> (shift - PAGE_SHIFT); + return (vma->vm_pgoff << (PAGE_SHIFT - shift)) + + (vma_subpage_offset(vma) >> shift); +} + struct coredump_params { const kernel_siginfo_t *siginfo; struct file *file; diff --git a/include/linux/highmem.h b/include/linux/highmem.h index d7aac9de1c8a4..37812575512be 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -330,6 +330,21 @@ struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, } #endif +#ifndef vma_alloc_zeroed_movable_folio_order +static inline struct folio *vma_alloc_zeroed_movable_folio_order( + struct vm_area_struct *vma, unsigned long addr, unsigned int order) +{ + struct folio *folio; + + if (!order) + return vma_alloc_zeroed_movable_folio(vma, addr); + folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, order, vma, addr); + if (folio && user_alloc_needs_zeroing()) + clear_user_highpages(&folio->page, addr, folio_nr_pages(folio)); + return folio; +} +#endif + static inline void clear_highpage(struct page *page) { void *kaddr = kmap_local_page(page); diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index ad20f7f8c1794..dad1a7803bbd5 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -3,6 +3,7 @@ #define _LINUX_HUGE_MM_H #include +#include #include /* only for vma_is_dax() */ #include @@ -361,6 +362,9 @@ struct thpsize { static inline bool vma_thp_disabled(struct vm_area_struct *vma, vm_flags_t vm_flags, bool forced_collapse) { + /* Native THP geometry and rmap cannot describe alternative user leaves. */ + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return true; /* Are THPs disabled for this VMA? */ if (vm_flags & VM_NOHUGEPAGE) return true; diff --git a/include/linux/khugepaged.h b/include/linux/khugepaged.h index d7a9053ff4fed..1510fdb048010 100644 --- a/include/linux/khugepaged.h +++ b/include/linux/khugepaged.h @@ -8,6 +8,8 @@ extern unsigned int khugepaged_max_ptes_none __read_mostly; #ifdef CONFIG_TRANSPARENT_HUGEPAGE extern struct attribute_group khugepaged_attr_group; +int collapse_file_user_page(struct mm_struct *mm, struct file *file, pgoff_t start); + extern int khugepaged_init(void); extern void khugepaged_destroy(void); extern int start_stop_khugepaged(void); @@ -32,6 +34,12 @@ static inline void khugepaged_exit(struct mm_struct *mm) __khugepaged_exit(mm); } #else /* CONFIG_TRANSPARENT_HUGEPAGE */ +static inline int collapse_file_user_page(struct mm_struct *mm, struct file *file, + pgoff_t start) +{ + return -EOPNOTSUPP; +} + static inline void khugepaged_fork(struct mm_struct *mm, struct mm_struct *oldmm) { } diff --git a/include/linux/mm.h b/include/linux/mm.h index 1659cb8f42fd5..749028a1b2c6b 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -750,6 +751,9 @@ struct vm_fault { }; struct page *cow_page; /* Page handler may use for COW fault */ +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage *cow_subpage; /* Owns cow_page's selected fragment */ +#endif struct page *page; /* ->fault handlers should return a * page here, unless VM_FAULT_NOPAGE * is set (which is also implied by @@ -1620,7 +1624,7 @@ static inline unsigned long vma_kernel_pagesize(struct vm_area_struct *vma) { if (unlikely(vma->vm_ops && vma->vm_ops->pagesize)) return vma->vm_ops->pagesize(vma); - return PAGE_SIZE; + return mm_page_size(vma->vm_mm); } unsigned long vma_mmu_pagesize(struct vm_area_struct *vma); @@ -2005,6 +2009,10 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *pa void set_pte_range(struct vm_fault *vmf, struct folio *folio, struct page *page, unsigned int nr, unsigned long addr); +#ifdef CONFIG_MM_SUBPAGE +vm_fault_t map_file_user_pages(struct vm_fault *vmf, struct folio *folio, + pgoff_t first, pgoff_t last, bool fault); +#endif vm_fault_t finish_fault(struct vm_fault *vmf); #endif @@ -3138,10 +3146,13 @@ struct follow_pfnmap_args { * @address: the virtual address to walk * @write: if true, fail with -EFAULT unless the mapping is * writable + * @allow_subpage: caller handles exact byte offsets and user-leaf bounds; + * otherwise a non-native user granule is rejected */ struct vm_area_struct *vma; unsigned long address; bool write; + bool allow_subpage; /** * Internals: * @@ -3152,13 +3163,15 @@ struct follow_pfnmap_args { /** * Outputs: * - * @pfn: the PFN of the address - * @addr_mask: address mask covering pfn + * @pfn: native PFN containing the requested physical byte + * @offset: byte offset of that byte within the native physical page + * @addr_mask: virtual address mask of the leaf mapping * @pgprot: the pgprot_t of the mapping * @writable: whether the mapping is writable * @special: whether the mapping is a special mapping (real PFN maps) */ unsigned long pfn; + unsigned int offset; unsigned long addr_mask; pgprot_t pgprot; bool writable; @@ -3210,6 +3223,13 @@ static inline void unmap_mapping_range(struct address_space *mapping, loff_t const holebegin, loff_t const holelen, int even_cows) { } #endif +#ifdef CONFIG_MM_SUBPAGE +void unmap_mapping_subpage_tail(struct address_space *mapping, loff_t newsize); +void unmap_mapping_subpage_hole(struct address_space *mapping, loff_t start, u64 end); +#else +static inline void unmap_mapping_subpage_tail(struct address_space *mapping, loff_t newsize) { } +#endif + static inline void unmap_shared_mapping_range(struct address_space *mapping, loff_t const holebegin, loff_t const holelen) { @@ -3438,7 +3458,8 @@ static inline void reset_mm_hiwater_rss(struct mm_struct *mm) static inline void setmax_mm_hiwater_rss(unsigned long *maxrss, struct mm_struct *mm) { - unsigned long hiwater_rss = get_mm_hiwater_rss(mm); + /* signal->maxrss/cmaxrss survive exec and child reaping: store kB. */ + unsigned long hiwater_rss = mm_pages_to_kb(mm, get_mm_hiwater_rss(mm)); if (*maxrss < hiwater_rss) *maxrss = hiwater_rss; @@ -3509,14 +3530,14 @@ static inline void mm_inc_nr_puds(struct mm_struct *mm) { if (mm_pud_folded(mm)) return; - atomic_long_add(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes); + atomic_long_add(pud_table_bytes_mm(mm), &mm->pgtables_bytes); } static inline void mm_dec_nr_puds(struct mm_struct *mm) { if (mm_pud_folded(mm)) return; - atomic_long_sub(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes); + atomic_long_sub(pud_table_bytes_mm(mm), &mm->pgtables_bytes); } #endif @@ -3537,14 +3558,14 @@ static inline void mm_inc_nr_pmds(struct mm_struct *mm) { if (mm_pmd_folded(mm)) return; - atomic_long_add(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes); + atomic_long_add(pmd_table_bytes_mm(mm), &mm->pgtables_bytes); } static inline void mm_dec_nr_pmds(struct mm_struct *mm) { if (mm_pmd_folded(mm)) return; - atomic_long_sub(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes); + atomic_long_sub(pmd_table_bytes_mm(mm), &mm->pgtables_bytes); } #endif @@ -3561,12 +3582,12 @@ static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm) static inline void mm_inc_nr_ptes(struct mm_struct *mm) { - atomic_long_add(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes); + atomic_long_add(pte_table_bytes_mm(mm), &mm->pgtables_bytes); } static inline void mm_dec_nr_ptes(struct mm_struct *mm) { - atomic_long_sub(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes); + atomic_long_sub(pte_table_bytes_mm(mm), &mm->pgtables_bytes); } #else @@ -3589,32 +3610,40 @@ static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address) { return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ? - NULL : p4d_offset(pgd, address); + NULL : p4d_offset_mm(mm, pgd, address); } static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address) { - return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ? - NULL : pud_offset(p4d, address); + return (unlikely(p4d_none_mm(mm, *p4d)) && __pud_alloc(mm, p4d, address)) ? + NULL : pud_offset_mm(mm, p4d, address); } static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) { return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))? - NULL: pmd_offset(pud, address); + NULL : pmd_offset_mm(mm, pud, address); } #endif /* CONFIG_MMU */ enum pt_flags { PT_kernel = PG_referenced, PT_reserved = PG_reserved, + PT_fragmented = PG_private, + PT_frag_defer = PG_active, /* High bits are used for zone/node/section */ }; static inline struct ptdesc *virt_to_ptdesc(const void *x) { - return page_ptdesc(virt_to_page(x)); + struct page *page = virt_to_page(x); + +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + /* One hardware table may span multiple native allocator pages. */ + page = compound_head(page); +#endif + return page_ptdesc(page); } /** @@ -3831,12 +3860,25 @@ static inline bool pagetable_pte_ctor(struct mm_struct *mm, } pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp); +#ifdef CONFIG_ARCH_HAS_MM_PGTABLE +pte_t *__pte_offset_map_mm(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr, pmd_t *pmdvalp); +#else +#define __pte_offset_map_mm(mm, pmd, addr, pmdvalp) \ + __pte_offset_map(pmd, addr, pmdvalp) +#endif static inline pte_t *pte_offset_map(pmd_t *pmd, unsigned long addr) { return __pte_offset_map(pmd, addr, NULL); } +static inline pte_t *pte_offset_map_mm(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr) +{ + return __pte_offset_map_mm(mm, pmd, addr, NULL); +} + pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, spinlock_t **ptlp); @@ -3854,7 +3896,7 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, #define pte_alloc(mm, pmd) (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd)) #define pte_alloc_map(mm, pmd, address) \ - (pte_alloc(mm, pmd) ? NULL : pte_offset_map(pmd, address)) + (pte_alloc(mm, pmd) ? NULL : pte_offset_map_mm(mm, pmd, address)) #define pte_alloc_map_lock(mm, pmd, address, ptlp) \ (pte_alloc(mm, pmd) ? \ @@ -3869,7 +3911,7 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, static inline struct page *pmd_pgtable_page(pmd_t *pmd) { unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1); - return virt_to_page((void *)((unsigned long) pmd & mask)); + return ptdesc_page(virt_to_ptdesc((void *)((unsigned long)pmd & mask))); } static inline struct ptdesc *pmd_ptdesc(pmd_t *pmd) @@ -4155,6 +4197,12 @@ extern unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf); +#ifdef CONFIG_MM_SUBPAGE +unsigned long do_mmap_offset(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flags, + vm_flags_t vm_flags, struct vm_page_offset pos, unsigned long *populate, + struct list_head *uf); +#endif extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf, bool unlock); @@ -4283,7 +4331,11 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma) static inline unsigned long vma_last_pgoff(struct vm_area_struct *vma) { +#ifdef CONFIG_MM_SUBPAGE + return vma_page_offset_at(vma, vma->vm_end - 1).index; +#else return vma->vm_pgoff + vma_pages(vma) - 1; +#endif } static inline unsigned long vma_desc_size(const struct vm_area_desc *desc) @@ -4536,6 +4588,9 @@ int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t pgprot); int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *); +int vm_insert_pages_range(struct vm_area_struct *vma, unsigned long addr, + struct page **pages, unsigned long nr_pages, + unsigned long offset, unsigned long length); int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num); int map_kernel_pages_prepare(struct vm_area_desc *desc); @@ -4547,6 +4602,9 @@ int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, unsigned long num); vm_fault_t vmf_insert_page_mkwrite(struct vm_fault *vmf, struct page *page, bool write); +vm_fault_t vmf_insert_pfn_prot_mkwrite_offset(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, unsigned int offset, + pgprot_t pgprot, bool mkwrite); vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t pgprot, bool mkwrite); vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr, @@ -4643,6 +4701,18 @@ static inline vm_fault_t vmf_insert_page(struct vm_area_struct *vma, return VM_FAULT_NOPAGE; } +/* Insert one user granule from an explicit offset within a native page. */ +#ifdef CONFIG_MM_SUBPAGE +vm_fault_t vmf_insert_page_offset(struct vm_area_struct *vma, unsigned long addr, + struct page *page, unsigned int offset); +#else +static inline vm_fault_t vmf_insert_page_offset(struct vm_area_struct *vma, + unsigned long addr, struct page *page, unsigned int offset) +{ + return offset ? VM_FAULT_SIGBUS : vmf_insert_page(vma, addr, page); +} +#endif + #ifndef io_remap_pfn_range_pfn static inline unsigned long io_remap_pfn_range_pfn(unsigned long pfn, unsigned long size) diff --git a/include/linux/mm_granule.h b/include/linux/mm_granule.h new file mode 100644 index 0000000000000..6e73216edcf96 --- /dev/null +++ b/include/linux/mm_granule.h @@ -0,0 +1,245 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_MM_GRANULE_H +#define _LINUX_MM_GRANULE_H + +#include +#include +#include + +/* + * User virtual mapping granule. Never use these for physical PFNs, struct + * page/folio sizes, page-cache indices, or physical memory accounting. + * The granule is immutable once an mm has mappings or is visible to tasks. + */ +#ifndef arch_mm_page_shift +static inline unsigned int arch_mm_page_shift(const struct mm_struct *mm) +{ + return PAGE_SHIFT; +} +#endif + +static inline unsigned int mm_page_shift(const struct mm_struct *mm) +{ + return arch_mm_page_shift(mm); +} + +static inline unsigned long mm_page_size(const struct mm_struct *mm) +{ + return 1UL << mm_page_shift(mm); +} + +/* A byte-range reference must fit both the user leaf and a native page. */ +static inline unsigned long mm_user_fragment_size(const struct mm_struct *mm) +{ + return mm_page_shift(mm) < PAGE_SHIFT ? mm_page_size(mm) : PAGE_SIZE; +} + +/* Native physical pages covered by one base user PTE, including fragments. */ +static inline unsigned long mm_pte_native_pages(const struct mm_struct *mm) +{ + unsigned long pages = mm_page_size(mm) >> PAGE_SHIFT; + + return pages ?: 1; +} + +static inline unsigned long mm_page_mask(const struct mm_struct *mm) +{ + return ~(mm_page_size(mm) - 1); +} + +/* + * ASLR tunables count native-page bits. Retain their byte window when a + * process uses another granule, including the extra low user-page bits. + * native_mask must be a contiguous low-bit mask whose byte span fits ulong. + */ +static inline unsigned long mm_aslr_mask(const struct mm_struct *mm, + unsigned long native_mask) +{ + unsigned int shift = mm_page_shift(mm); + + if (shift < PAGE_SHIFT) { + unsigned int extra = PAGE_SHIFT - shift; + + return (native_mask << extra) | ((1UL << extra) - 1); + } + return native_mask >> (shift - PAGE_SHIFT); +} + +/* Virtual-range accounting. Physical pages and cache indices stay native. */ +static inline unsigned long vma_user_pages(const struct vm_area_struct *vma) +{ + return (vma->vm_end - vma->vm_start) >> mm_page_shift(vma->vm_mm); +} + +static inline unsigned long mm_page_align(const struct mm_struct *mm, unsigned long bytes) +{ + return (bytes + mm_page_size(mm) - 1) & mm_page_mask(mm); +} + +/* + * RSS and virtual counters use the target mm's user granule. Consumers + * comparing different mms or physical memory must convert explicitly. + */ +static inline unsigned long mm_pages_to_bytes(const struct mm_struct *mm, + unsigned long pages) +{ + return pages << mm_page_shift(mm); +} + +static inline unsigned long mm_pages_to_kb(const struct mm_struct *mm, + unsigned long pages) +{ + return pages << (mm_page_shift(mm) - 10); +} + +/* Round up only when reducing precision; do not hide a partial user page. */ +static inline unsigned long mm_pages_to_shift(const struct mm_struct *mm, + unsigned long pages, unsigned int shift) +{ + unsigned int source = mm_page_shift(mm); + + if (source >= shift) + return pages << (source - shift); + return (pages >> (shift - source)) + + !!(pages & ((1UL << (shift - source)) - 1)); +} + +static inline unsigned long mm_pages_to_native(const struct mm_struct *mm, + unsigned long pages) +{ + return mm_pages_to_shift(mm, pages, PAGE_SHIFT); +} + +/* A page-count proc ABI uses the reader's granule, including remote reads. */ +static inline unsigned long mm_pages_for_reader(const struct mm_struct *mm, + const struct mm_struct *reader, unsigned long pages) +{ + return mm_pages_to_shift(mm, pages, + reader ? mm_page_shift(reader) : PAGE_SHIFT); +} + +/* + * Time integrals need one unit across exec. Locked VM also uses this unit: + * it must represent both logical mappings and native physical charges exactly, + * including a single native-page pin in a larger-granule userspace mm. + */ +#ifdef CONFIG_MM_SUBPAGE +#define MM_ACCOUNT_SHIFT 12 +#else +#define MM_ACCOUNT_SHIFT PAGE_SHIFT +#endif + +static inline unsigned long mm_pages_to_account(const struct mm_struct *mm, + unsigned long pages) +{ + return pages << (mm_page_shift(mm) - MM_ACCOUNT_SHIFT); +} + +static inline unsigned long mm_locked_vm_bytes(const struct mm_struct *mm) +{ + return READ_ONCE(mm->locked_vm) << MM_ACCOUNT_SHIFT; +} + +/* Physical-page limits must include any partial native page of locked VM. */ +static inline unsigned long mm_locked_vm_native_pages(const struct mm_struct *mm) +{ + unsigned long units = READ_ONCE(mm->locked_vm); + unsigned int shift = PAGE_SHIFT - MM_ACCOUNT_SHIFT; + + return (units >> shift) + !!(units & ((1UL << shift) - 1)); +} + +/* File-cache indices remain native-page units even for a 4K VMA. */ +struct vm_page_offset { + pgoff_t index; + unsigned int offset; +}; + +static inline struct vm_page_offset vm_page_offset_add(struct vm_page_offset pos, + unsigned long bytes) +{ + unsigned long low = pos.offset + (bytes & ~PAGE_MASK); + + pos.index += (bytes >> PAGE_SHIFT) + (low >> PAGE_SHIFT); + pos.offset = low & ~PAGE_MASK; + return pos; +} + +static inline struct vm_page_offset vm_page_offset_sub(struct vm_page_offset pos, + unsigned long bytes) +{ + unsigned long low = bytes & ~PAGE_MASK; + + pos.index -= (bytes >> PAGE_SHIFT) + (low > pos.offset); + pos.offset = (pos.offset - low) & ~PAGE_MASK; + return pos; +} + +static inline bool vm_page_offset_equal(struct vm_page_offset a, struct vm_page_offset b) +{ + return a.index == b.index && a.offset == b.offset; +} + +static inline unsigned int vma_subpage_offset(const struct vm_area_struct *vma) +{ +#ifdef CONFIG_MM_SUBPAGE + return vma->vm_subpage_offset; +#else + return 0; +#endif +} + +/* address must be >= vm_start; vm_end computes the next contiguous offset. */ +static inline struct vm_page_offset vma_page_offset_at(const struct vm_area_struct *vma, + unsigned long address) +{ + unsigned long delta = address - vma->vm_start; + unsigned long low = (delta & ~PAGE_MASK) + vma_subpage_offset(vma); + + return (struct vm_page_offset) { + .index = vma->vm_pgoff + (delta >> PAGE_SHIFT) + (low >> PAGE_SHIFT), + .offset = low & ~PAGE_MASK, + }; +} + +/* The start byte of a faulting granule must precede EOF. */ +static inline bool vma_file_offset_valid(const struct vm_area_struct *vma, + unsigned long address, loff_t size) +{ + struct vm_page_offset pos = vma_page_offset_at(vma, address); + pgoff_t end = size >> PAGE_SHIFT; + + return pos.index < end || + (pos.index == end && pos.offset < (size & ~PAGE_MASK)); +} + +static inline void vma_set_page_offset(struct vm_area_struct *vma, struct vm_page_offset pos) +{ + vma->vm_pgoff = pos.index; +#ifdef CONFIG_MM_SUBPAGE + vma->vm_subpage_offset = pos.offset; +#endif +} + +/* Inverse lookup of a single byte; an offset outside the VMA is not clamped. */ +static inline unsigned long vma_address_at_offset(const struct vm_area_struct *vma, + struct vm_page_offset pos) +{ + unsigned long len = vma->vm_end - vma->vm_start; + unsigned long index, delta, last; + unsigned int offset = vma_subpage_offset(vma); + + if (!len || pos.index < vma->vm_pgoff || pos.offset >= PAGE_SIZE) + return -EFAULT; + index = pos.index - vma->vm_pgoff; + last = ((len - 1) >> PAGE_SHIFT) + + ((((len - 1) & ~PAGE_MASK) + offset) >> PAGE_SHIFT); + if (index > last) + return -EFAULT; + delta = (index << PAGE_SHIFT) + pos.offset; + if (delta < offset || delta - offset >= len) + return -EFAULT; + return vma->vm_start + delta - offset; +} + +#endif diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h new file mode 100644 index 0000000000000..13344e3a92384 --- /dev/null +++ b/include/linux/mm_subpage.h @@ -0,0 +1,277 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_MM_SUBPAGE_H +#define _LINUX_MM_SUBPAGE_H + +#include +#include +#include + +#define MM_SUBPAGE_SHIFT 12 +#define MM_SUBPAGE_SIZE (1UL << MM_SUBPAGE_SHIFT) +#define MM_SUBPAGES_PER_PAGE (PAGE_SIZE / MM_SUBPAGE_SIZE) + +struct mm_subpage_mapcounts { + unsigned int slots[MM_SUBPAGES_PER_PAGE]; +}; + +struct folio; +struct page; +struct mm_struct; +struct mm_subpage; +struct mm_subpage_pool; +struct page_ext_operations; +struct vm_area_struct; + +extern struct page_ext_operations mm_subpage_ext_ops; + +#ifdef CONFIG_MM_SUBPAGE +/* Per-native-page shmem validity; caller holds the containing folio lock. */ +unsigned long mm_subpage_shmem_missing(struct page *page); +void mm_subpage_shmem_set_missing(struct page *page, unsigned long missing); +#endif + +/* + * A pool is one caller-defined allocation/policy/charge domain. The caller + * supplies fresh, unmapped native folios using its normal NUMA and memcg + * allocation path. No mm, VMA, rmap, swap or folio-private field is repurposed. + * Fault paths transfer slot references to PTEs through the rmap helpers below. + * + * Each caller must hold a pool reference through pool operations. close() + * stops allocation and releases unused backing; it does not consume that + * reference. Existing slot references outlive close() and the originating mm. + */ +struct mm_subpage_pool *mm_subpage_pool_create(gfp_t gfp); +struct mm_subpage_pool *mm_subpage_pool_create_granule(gfp_t gfp, unsigned int shift); +void mm_subpage_pool_get(struct mm_subpage_pool *pool); +void mm_subpage_pool_put(struct mm_subpage_pool *pool); +void mm_subpage_pool_close(struct mm_subpage_pool *pool); + +/* Success consumes the folio reference. On error the caller retains it. */ +int mm_subpage_pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, + gfp_t gfp); +int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *folio, + unsigned int offset, gfp_t gfp); +unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool); + +/* -EAGAIN requires more compatible backing; -ESHUTDOWN means pool closed. */ +struct mm_subpage *mm_subpage_alloc(struct mm_subpage_pool *pool); +struct mm_subpage *mm_subpage_copy(struct mm_subpage_pool *pool, + const struct mm_subpage *source); +/* Exact physical byte offset within a native folio; no other slot is substituted. */ +struct mm_subpage *mm_subpage_alloc_at(struct mm_subpage_pool *pool, unsigned int offset); +struct mm_subpage *mm_subpage_copy_at(struct mm_subpage_pool *pool, + const struct mm_subpage *source, unsigned int offset); + +/* + * Reattach a slot to an uptodate, locked order-0 swap-cache folio, without + * zeroing or copying its bytes. Caller holds a native folio reference and + * retains it. The returned slot owns an additional reference. -EAGAIN means + * ownership changed concurrently; -ENOMEM means metadata allocation failed. + */ +struct mm_subpage *mm_subpage_restore(struct folio *folio, unsigned int offset, gfp_t gfp); +struct mm_subpage *mm_subpage_restore_granule(struct folio *folio, unsigned int offset, + unsigned int shift, gfp_t gfp); + +/* PTE lock held, one file-backed alternative leaf per call. Native mms are no-ops. */ +#ifdef CONFIG_MM_SUBPAGE +void __mm_subpage_file_map_add(phys_addr_t phys, unsigned long size); +void __mm_subpage_file_map_del(phys_addr_t phys, unsigned long size); +static inline void mm_subpage_file_map_add(struct vm_area_struct *vma, phys_addr_t phys) +{ + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + __mm_subpage_file_map_add(phys, mm_page_size(vma->vm_mm)); +} +static inline void mm_subpage_file_map_del(struct vm_area_struct *vma, phys_addr_t phys) +{ + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + __mm_subpage_file_map_del(phys, mm_page_size(vma->vm_mm)); +} +/* Fill 4K coverage counts; return alternative leaf count. Statistics only. */ +unsigned int mm_subpage_file_mapcounts(struct page *page, + struct mm_subpage_mapcounts *counts); +#else +static inline void mm_subpage_file_map_add(struct vm_area_struct *vma, phys_addr_t phys) { } +static inline void mm_subpage_file_map_del(struct vm_area_struct *vma, phys_addr_t phys) { } +#endif + +/* Both folios locked; preallocate destination metadata before removing PTEs. */ +#ifdef CONFIG_MM_SUBPAGE +int mm_subpage_migrate_prepare(struct folio *src, struct folio *dst); +bool mm_subpage_migrating(struct folio *folio); +void mm_subpage_migrate_save_lazyfree(struct mm_subpage *subpage); +/* Infallible metadata reuse after prepare; caller still holds the folio lock. */ +struct mm_subpage *mm_subpage_migrate_restore(struct folio *folio, unsigned int offset); +void mm_subpage_migrate_finish(struct folio *folio); +#else +static inline int mm_subpage_migrate_prepare(struct folio *src, struct folio *dst) +{ + return 0; +} +static inline bool mm_subpage_migrating(struct folio *folio) +{ + return false; +} +static inline void mm_subpage_migrate_finish(struct folio *folio) { } +#endif + +/* + * get() requires a live reference. A mapping reference must not be put until + * its PTE is removed and TLB invalidation completes. Every pin/alias holds + * its own reference: the slot becomes reusable only after the final put. + * Every live slot reference also holds one reference to the native folio. + * copy() requires the caller to stabilise the source contents (as for COW). + * A failed data copy returns -EHWPOISON without publishing the destination. + */ +void mm_subpage_get(struct mm_subpage *subpage); +void mm_subpage_put(struct mm_subpage *subpage); +/* + * Acquire the live identity containing a 4K-aligned physical address, or NULL. + * Interior quarters of a larger identity return that same identity. + * The caller must stabilise the physical allocation, e.g. with a live PTE + * under its lock or a folio reference. A lockless walker additionally needs + * GUP's TLB/free ordering; RCU protects metadata, not arbitrary physical PFNs. + * This is not a mapping identity check: a lockless PTE reader must revalidate + * its PTE after acquisition, just as for speculative folio references. + */ +struct mm_subpage *mm_subpage_get_from_phys(phys_addr_t phys); +/* + * Optional shared sets for asynchronous consumers of native byte spans. + * add() requires the source slot identities to remain live for the call. + * The pointer being modified is exclusively owned by the caller; get() shares + * a read-only set, and mutation uses copy-on-write. Duplicate ranges add no + * references. prune() requires a stable, side-effect-free membership predicate. + * Allocation failure leaves the original set intact. put() accepts NULL. + */ +struct mm_subpage_refs; +#ifdef CONFIG_MM_SUBPAGE +struct mm_subpage_refs *mm_subpage_refs_get(struct mm_subpage_refs *refs); +void mm_subpage_refs_put(struct mm_subpage_refs *refs); +int mm_subpage_refs_add(struct mm_subpage_refs **refs, struct page *page, + unsigned int offset, unsigned int len, gfp_t gfp); +int mm_subpage_refs_prune(struct mm_subpage_refs **refs, + bool (*keep)(const struct mm_subpage *, void *), + void *context, gfp_t gfp); +#else +static inline struct mm_subpage_refs *mm_subpage_refs_get(struct mm_subpage_refs *refs) +{ + return refs; +} +static inline void mm_subpage_refs_put(struct mm_subpage_refs *refs) { } +static inline int mm_subpage_refs_add(struct mm_subpage_refs **refs, struct page *page, + unsigned int offset, unsigned int len, gfp_t gfp) +{ + return 0; +} +static inline int mm_subpage_refs_prune(struct mm_subpage_refs **refs, + bool (*keep)(const struct mm_subpage *, void *), void *context, gfp_t gfp) +{ + return 0; +} +#endif + +/* + * Anonymous rmap attachment: caller holds the folio lock and PTE lock, has + * prepared vma->anon_vma, and transfers an existing slot reference to its PTE. + * A folio may cover only one native linear index and anon_vma root; its slot + * offset must match the VMA's linear byte offset. Removal needs the PTE lock. + * These helpers do not install PTEs, update RSS, enqueue on LRU, or drop refs. + */ +int mm_subpage_add_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address); +int mm_subpage_add_new_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address); +void mm_subpage_remove_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma); +/* Both PTE locks and source write_protect_seq held; both mappings become R/O. */ +int mm_subpage_dup_anon_rmap(struct mm_subpage *subpage, + struct vm_area_struct *dst, struct vm_area_struct *src, unsigned long address); +/* + * Per-slot anonymous exclusivity. Setting requires the folio and PTE locks, + * and proof of a fresh private slot or an exclusive migration/swap entry. + * Sharing requires the PTE lock and fails while a typed pin exists. + */ +void mm_subpage_set_exclusive(struct mm_subpage *subpage); +bool mm_subpage_is_exclusive(const struct mm_subpage *subpage); +/* + * Folio and PTE locked; caller proves one remaining swap PTE for this slot. + * Rmap has been attached, but its PTE is not published yet. Reject old GET + * references or present aliases before recovering lost exclusive ownership. + */ +bool mm_subpage_try_reuse_swap(struct mm_subpage *subpage); +/* Present PTE plus caller lookup ref; caller proves no swapped aliases. */ +bool mm_subpage_try_reuse_anon(struct mm_subpage *subpage); +/* PTE locked, caller holds one GET; retire the PTE/TLB before dropping its ref. */ +bool mm_subpage_can_zap(struct mm_subpage *subpage); +bool mm_subpage_try_share(struct mm_subpage *subpage); +/* Folio/PTE locked, caller owns a lookup reference. */ +bool mm_subpage_mark_lazyfree(struct mm_subpage *subpage); +bool mm_subpage_all_lazyfree(struct mm_subpage *subpage); +bool mm_subpage_discardable(struct mm_subpage *subpage); +bool mm_subpage_is_lazyfree(const struct mm_subpage *subpage); +void mm_subpage_cancel_lazyfree(struct mm_subpage *subpage); +/* + * Caller holds a live slot reference and its PTE lock. Success takes an + * additional slot reference plus a native FOLL_PIN reference. The pin keeps + * the exact fragment occupied after unmap; release only with subpage_unpin. + * This is not a GUP walker and does not validate user addresses/permissions. + */ +int mm_subpage_pin(struct mm_subpage *subpage); +void mm_subpage_unpin(struct mm_subpage *subpage); +unsigned int mm_subpage_pincount(const struct mm_subpage *subpage); +#ifdef CONFIG_MM_SUBPAGE +/* A referenced/pinned folio range; never infer offset from the user VA. */ +struct user_page_fragment { + struct folio *folio; + struct mm_subpage *subpage; + unsigned long offset; + unsigned int length; + bool pinned; +}; + +/* + * Takes/releases mmap_lock internally. Returns the number of records filled, + * or -errno if none. A positive partial result is valid; sum record lengths + * to advance the user address. Each record stops at the target mm's granule + * or a native physical-page boundary, whichever comes first. + * Caller owns the target mm lifetime. Release either kind of record with + * release_user_fragments(); copying a record does not acquire a reference. + * FOLL_LONGTERM pins migrate unpinnable backing before returning success. + * Device/PFN mappings are not accepted by this interface. + */ +long pin_user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity); +long get_user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity); +/* + * Single-record GET with a stable VMA. Caller holds mmap_lock throughout; + * this function never releases it. Returns 1 or -errno (0 for zero length). + * FOLL_NOWAIT is not supported. The returned VMA is valid until unlock. + */ +int get_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragment, struct vm_area_struct **vma); +/* Single-record PIN under the caller's mmap_lock, retained throughout. + * Same stable-VMA/partial-record contract as GET above. FOLL_LONGTERM and + * FOLL_NOWAIT are rejected: long-term migration can require dropping locks. + */ +int pin_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragment, struct vm_area_struct **vma); +/* Current mm, one aligned fragment; sparse zero pages return a negative errno. */ +int get_dump_fragment(unsigned long addr, struct user_page_fragment *fragment); +/* Dirty a retained reference without releasing its allocation identity. */ +void mark_user_fragment_dirty(struct user_page_fragment *fragment); +void release_user_fragments(struct user_page_fragment *fragments, + unsigned long count, bool dirty); +#endif + +int mm_subpage_mapcount(const struct mm_subpage *subpage); +/* Borrowed folio: a raw folio reference alone does not keep a slot occupied. */ +struct folio *mm_subpage_folio(const struct mm_subpage *subpage); +unsigned int mm_subpage_shift(const struct mm_subpage *subpage); +unsigned int mm_subpage_size(const struct mm_subpage *subpage); +unsigned int mm_subpage_offset(const struct mm_subpage *subpage); +phys_addr_t mm_subpage_phys(const struct mm_subpage *subpage); + +#endif diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 2b109cdaaad86..a8cc91f8966db 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -559,7 +559,8 @@ FOLIO_MATCH(compound_info, _head_3); * @__page_mapping: Aliases with page->mapping. Unused for page tables. * @pt_index: Used for s390 gmap. * @pt_mm: Used for x86 pgds. - * @pt_frag_refcount: For fragmented page table tracking. Powerpc only. + * @pt_frag_refcount: For fragmented page table tracking (powerpc, arm64). + * @pt_frag_shift: Immutable arm64 fragment granule, valid through RCU release. * @pt_share_count: Used for HugeTLB PMD page table share count. * @_pt_pad_2: Padding to ensure proper alignment. * @ptl: Lock for the page table. @@ -586,7 +587,15 @@ struct ptdesc { union { pgoff_t pt_index; struct mm_struct *pt_mm; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + /* arm64 is 64-bit: use the spare half of this existing word. */ + struct { + atomic_t pt_frag_refcount; + u8 pt_frag_shift; + }; +#else atomic_t pt_frag_refcount; +#endif #ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING atomic_t pt_share_count; #endif @@ -893,6 +902,9 @@ struct vm_area_desc { /* Mutable fields. Populated with initial state. */ pgoff_t pgoff; +#ifdef CONFIG_MM_SUBPAGE + unsigned int subpage_offset; +#endif struct file *vm_file; vma_flags_t vma_flags; pgprot_t page_prot; @@ -980,6 +992,9 @@ struct vm_area_struct { /* Information about our backing store: */ unsigned long vm_pgoff; /* Offset (within vm_file) in PAGE_SIZE units */ +#ifdef CONFIG_MM_SUBPAGE + unsigned int vm_subpage_offset; /* Byte offset within vm_pgoff. */ +#endif struct file * vm_file; /* File we map to (can be NULL). */ void * vm_private_data; /* was vm_pte (shared mem) */ @@ -1262,12 +1277,16 @@ struct mm_struct { #endif struct futex_mm_data futex; - unsigned long hiwater_rss; /* High-watermark of RSS usage */ - unsigned long hiwater_vm; /* High-water virtual memory usage */ + unsigned long hiwater_rss; /* Peak RSS in user granules */ + unsigned long hiwater_vm; /* Peak VM in user granules */ - unsigned long total_vm; /* Total pages mapped */ - unsigned long locked_vm; /* Pages that have PG_mlocked set */ - atomic64_t pinned_vm; /* Refcount permanently increased */ + unsigned long total_vm; /* Mapped user granules */ +#ifdef CONFIG_MM_SUBPAGE + /* VM_ACCOUNT granules; global commitment rounds this sum once. */ + unsigned long committed_user_pages; +#endif + unsigned long locked_vm; /* Locked bytes >> MM_ACCOUNT_SHIFT */ + atomic64_t pinned_vm; /* Pinned native physical pages */ unsigned long data_vm; /* VM_WRITE & ~VM_SHARED & ~VM_STACK */ unsigned long exec_vm; /* VM_EXEC & ~VM_WRITE & ~VM_STACK */ unsigned long stack_vm; /* VM_STACK */ diff --git a/include/linux/mman.h b/include/linux/mman.h index 389521594c69c..7b80c19f6aa5d 100644 --- a/include/linux/mman.h +++ b/include/linux/mman.h @@ -83,6 +83,17 @@ static inline void vm_unacct_memory(long pages) vm_acct_memory(-pages); } +/* VMA reservations use user granules; shmem/physical reservations use pages. */ +#ifdef CONFIG_MM_SUBPAGE +int mm_account_memory(struct mm_struct *mm, struct mm_struct *policy_mm, long pages); +void mm_acct_memory(struct mm_struct *mm, long pages); +void mm_unacct_memory(struct mm_struct *mm, unsigned long pages); +#else +#define mm_account_memory(mm, policy_mm, pages) security_vm_enough_memory_mm(policy_mm, pages) +#define mm_acct_memory(mm, pages) vm_acct_memory(pages) +#define mm_unacct_memory(mm, pages) vm_unacct_memory(pages) +#endif + /* * Allow architectures to handle additional protection and flag bits. The * overriding macros must be defined in the arch-specific asm/mman.h file. diff --git a/include/linux/mmu_context.h b/include/linux/mmu_context.h index ed3dd0f3fe194..a4260bc898205 100644 --- a/include/linux/mmu_context.h +++ b/include/linux/mmu_context.h @@ -5,6 +5,11 @@ #include #include +/* Select immutable exec geometry before allocating any page tables. */ +#ifndef arch_mm_init_exec +static inline void arch_mm_init_exec(struct mm_struct *mm, unsigned int page_shift) { } +#endif + /* Architectures that care about IRQ state in switch_mm can override this. */ #ifndef switch_mm_irqs_off # define switch_mm_irqs_off switch_mm diff --git a/include/linux/page_table_check.h b/include/linux/page_table_check.h index 12268a32e8be1..3ee1096fb0e53 100644 --- a/include/linux/page_table_check.h +++ b/include/linux/page_table_check.h @@ -12,6 +12,10 @@ extern struct static_key_true page_table_check_disabled; extern struct page_ext_operations page_table_check_ops; +#if IS_ENABLED(CONFIG_KUNIT) +int page_table_check_get_counts(struct page *page, unsigned int offset, + int *anon, int *file); +#endif void __page_table_check_zero(struct page *page, unsigned int order); void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr, diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 2c3718d592d6d..046c665976c2f 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1066,10 +1066,7 @@ static inline pgoff_t folio_pgoff(const struct folio *folio) static inline pgoff_t linear_page_index(const struct vm_area_struct *vma, const unsigned long address) { - pgoff_t pgoff; - pgoff = (address - vma->vm_start) >> PAGE_SHIFT; - pgoff += vma->vm_pgoff; - return pgoff; + return vma_page_offset_at(vma, address).index; } struct wait_page_key { diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 2981e386da7b3..2b6765f9efb2c 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -12,11 +12,18 @@ #ifdef CONFIG_MMU #include +#include #include #include #include #include +/* pgtable_t is an opaque allocation token, not necessarily struct page *. */ +#ifndef pgtable_to_page +#define pgtable_to_page(table) (table) +#define pgtable_from_page(page) (page) +#endif + #if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \ defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS #error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED @@ -155,6 +162,59 @@ static inline pgd_t *pgd_offset_pgd(pgd_t *pgd, unsigned long address) */ #define pgd_offset_k(address) pgd_offset(&init_mm, (address)) +/* Explicit target-mm accessors; legacy accessors keep native geometry. */ +#ifndef pte_phys_mm +#define pte_phys_mm(mm, pte) PFN_PHYS(pte_pfn(pte)) +#endif +#ifndef phys_pte_mm +#define phys_pte_mm(mm, phys, prot) pfn_pte((phys) >> PAGE_SHIFT, prot) +#endif +#ifndef pte_pgprot_mm +#define pte_pgprot_mm(mm, pte) pte_pgprot(pte) +#endif +#ifndef pgd_offset_pgd_mm +#define pgd_offset_pgd_mm(mm, pgd, addr) pgd_offset_pgd(pgd, addr) +#endif +#ifndef p4d_offset_mm +#define p4d_offset_mm(mm, pgd, addr) p4d_offset(pgd, addr) +#endif +#ifndef pud_offset_mm +#define pud_offset_mm(mm, p4d, addr) pud_offset(p4d, addr) +#endif +#ifndef pmd_offset_mm +#define pmd_offset_mm(mm, pud, addr) pmd_offset(pud, addr) +#endif +#ifndef __pte_map_mm +#define __pte_map_mm(mm, pmd, addr) __pte_map(pmd, addr) +#endif +#ifndef pte_offset_kernel_mm +#define pte_offset_kernel_mm(mm, pmd, addr) pte_offset_kernel(pmd, addr) +#endif + +#ifndef pud_pgtable_mm +#define pud_pgtable_mm(mm, pud) pud_pgtable(pud) +#endif + +#ifndef pmd_size_mm +#define pmd_shift_mm(mm) PMD_SHIFT +#define pud_shift_mm(mm) PUD_SHIFT +#define p4d_shift_mm(mm) P4D_SHIFT +#define pmd_size_mm(mm) PMD_SIZE +#define pud_size_mm(mm) PUD_SIZE +#define p4d_size_mm(mm) P4D_SIZE +#define pgd_size_mm(mm) PGDIR_SIZE +#define pmd_mask_mm(mm) PMD_MASK +#define pud_mask_mm(mm) PUD_MASK +#define p4d_mask_mm(mm) P4D_MASK +#define pgd_mask_mm(mm) PGDIR_MASK +#endif + +#ifndef pte_table_bytes_mm +#define pte_table_bytes_mm(mm) (PTRS_PER_PTE * sizeof(pte_t)) +#define pmd_table_bytes_mm(mm) (PTRS_PER_PMD * sizeof(pmd_t)) +#define pud_table_bytes_mm(mm) (PTRS_PER_PUD * sizeof(pud_t)) +#endif + /* * In many cases it is known that a virtual address is mapped at PMD or PTE * level, so instead of traversing all the page table levels, we can get a @@ -422,6 +482,10 @@ static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) #define pte_next_pfn(pte) pte_advance_pfn(pte, 1) +#ifndef pte_advance_pfn_mm +#define pte_advance_pfn_mm(mm, pte, nr) pte_advance_pfn(pte, nr) +#endif + #ifndef set_ptes /** * set_ptes - Map consecutive pages to a contiguous range of addresses. @@ -1542,6 +1606,13 @@ static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio) * and any p?d_bad entries - reporting the error before resetting to none. * Do the tests inline, but report and clear the bad entry in mm/memory.c. */ +#ifndef pgd_addr_end_mm +#define pgd_addr_end_mm(mm, addr, end) pgd_addr_end(addr, end) +#define p4d_addr_end_mm(mm, addr, end) p4d_addr_end(addr, end) +#define pud_addr_end_mm(mm, addr, end) pud_addr_end(addr, end) +#define pmd_addr_end_mm(mm, addr, end) pmd_addr_end(addr, end) +#endif + void pgd_clear_bad(pgd_t *); #ifndef __PAGETABLE_P4D_FOLDED @@ -1569,6 +1640,25 @@ static inline int pgd_none_or_clear_bad(pgd_t *pgd) return 0; } +#ifndef p4d_none_mm +#define p4d_none_mm(mm, p4d) p4d_none(p4d) +#define p4d_bad_mm(mm, p4d) p4d_bad(p4d) +#define p4d_present_mm(mm, p4d) p4d_present(p4d) +#define p4d_clear_mm(mm, p4d) p4d_clear(p4d) +#endif + +static inline int p4d_none_or_clear_bad_mm(struct mm_struct *mm, p4d_t *p4d) +{ + if (p4d_none_mm(mm, *p4d)) + return 1; + if (unlikely(p4d_bad_mm(mm, *p4d))) { + p4d_ERROR(*p4d); + p4d_clear_mm(mm, p4d); + return 1; + } + return 0; +} + static inline int p4d_none_or_clear_bad(p4d_t *p4d) { if (p4d_none(*p4d)) @@ -2056,11 +2146,15 @@ static inline int is_zero_pfn(unsigned long pfn) #define zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr)) #else +#ifndef ARCH_ZERO_PAGE_SIZE +#define ARCH_ZERO_PAGE_SIZE PAGE_SIZE +#endif + static inline int is_zero_pfn(unsigned long pfn) { extern unsigned long zero_page_pfn; - return pfn == zero_page_pfn; + return pfn - zero_page_pfn < (ARCH_ZERO_PAGE_SIZE >> PAGE_SHIFT); } static inline unsigned long zero_pfn(unsigned long addr) @@ -2070,7 +2164,7 @@ static inline unsigned long zero_pfn(unsigned long addr) return zero_page_pfn; } -extern const uint8_t empty_zero_page[PAGE_SIZE]; +extern const uint8_t empty_zero_page[ARCH_ZERO_PAGE_SIZE]; extern struct page *__zero_page; static inline struct page *_zero_page(unsigned long addr) diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 8dc0871e5f001..2bc2c986eb76e 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -40,6 +40,10 @@ struct anon_vma { * anon_vma if they are the last user on release */ atomic_t refcount; +#ifdef CONFIG_MM_SUBPAGE + /* Immutable base-page requirement shared by this anonymous lineage. */ + unsigned int min_folio_order; +#endif /* * Count of child anon_vmas. Equals to the count of all anon_vmas that @@ -398,29 +402,40 @@ static __always_inline void __folio_rmap_sanity_checks(const struct folio *folio } /* - * rmap interfaces called when adding or removing pte of page + * Single-PTE interfaces cover the complete native-page span of a user leaf. + * Multi-page interfaces below retain explicit native physical-page counts. */ void folio_move_anon_rmap(struct folio *, struct vm_area_struct *); void folio_add_anon_rmap_ptes(struct folio *, struct page *, int nr_pages, struct vm_area_struct *, unsigned long address, rmap_t flags); -#define folio_add_anon_rmap_pte(folio, page, vma, address, flags) \ - folio_add_anon_rmap_ptes(folio, page, 1, vma, address, flags) +static inline void folio_add_anon_rmap_pte(struct folio *folio, struct page *page, + struct vm_area_struct *vma, unsigned long address, rmap_t flags) +{ + folio_add_anon_rmap_ptes(folio, page, mm_pte_native_pages(vma->vm_mm), + vma, address, flags); +} void folio_add_anon_rmap_pmd(struct folio *, struct page *, struct vm_area_struct *, unsigned long address, rmap_t flags); void folio_add_new_anon_rmap(struct folio *, struct vm_area_struct *, unsigned long address, rmap_t flags); void folio_add_file_rmap_ptes(struct folio *, struct page *, int nr_pages, struct vm_area_struct *); -#define folio_add_file_rmap_pte(folio, page, vma) \ - folio_add_file_rmap_ptes(folio, page, 1, vma) +static inline void folio_add_file_rmap_pte(struct folio *folio, struct page *page, + struct vm_area_struct *vma) +{ + folio_add_file_rmap_ptes(folio, page, mm_pte_native_pages(vma->vm_mm), vma); +} void folio_add_file_rmap_pmd(struct folio *, struct page *, struct vm_area_struct *); void folio_add_file_rmap_pud(struct folio *, struct page *, struct vm_area_struct *); void folio_remove_rmap_ptes(struct folio *, struct page *, int nr_pages, struct vm_area_struct *); -#define folio_remove_rmap_pte(folio, page, vma) \ - folio_remove_rmap_ptes(folio, page, 1, vma) +static inline void folio_remove_rmap_pte(struct folio *folio, struct page *page, + struct vm_area_struct *vma) +{ + folio_remove_rmap_ptes(folio, page, mm_pte_native_pages(vma->vm_mm), vma); +} void folio_remove_rmap_pmd(struct folio *, struct page *, struct vm_area_struct *); void folio_remove_rmap_pud(struct folio *, struct page *, @@ -541,7 +556,8 @@ static inline void folio_dup_file_rmap_ptes(struct folio *folio, static __always_inline void folio_dup_file_rmap_pte(struct folio *folio, struct page *page, struct vm_area_struct *dst_vma) { - __folio_dup_file_rmap(folio, page, 1, dst_vma, PGTABLE_LEVEL_PTE); + __folio_dup_file_rmap(folio, page, mm_pte_native_pages(dst_vma->vm_mm), + dst_vma, PGTABLE_LEVEL_PTE); } /** @@ -665,8 +681,8 @@ static __always_inline int folio_try_dup_anon_rmap_pte(struct folio *folio, struct page *page, struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) { - return __folio_try_dup_anon_rmap(folio, page, 1, dst_vma, src_vma, - PGTABLE_LEVEL_PTE); + return __folio_try_dup_anon_rmap(folio, page, mm_pte_native_pages(src_vma->vm_mm), + dst_vma, src_vma, PGTABLE_LEVEL_PTE); } /** @@ -708,13 +724,16 @@ static inline int folio_try_dup_anon_rmap_pmd(struct folio *folio, static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, struct page *page, int nr_pages, enum pgtable_level level) { + int i, exclusive_pages = level == PGTABLE_LEVEL_PTE ? nr_pages : 1; + VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); VM_WARN_ON_FOLIO(!PageAnonExclusive(page), folio); __folio_rmap_sanity_checks(folio, page, nr_pages, level); /* device private folios cannot get pinned via GUP. */ if (unlikely(folio_is_device_private(folio))) { - ClearPageAnonExclusive(page); + for (i = 0; i < exclusive_pages; i++) + ClearPageAnonExclusive(page + i); return 0; } @@ -766,7 +785,8 @@ static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, if (unlikely(folio_maybe_dma_pinned(folio))) return -EBUSY; - ClearPageAnonExclusive(page); + for (i = 0; i < exclusive_pages; i++) + ClearPageAnonExclusive(page + i); /* * This is conceptually a smp_wmb() paired with the smp_rmb() in @@ -782,7 +802,8 @@ static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, * mapped by a PTE possibly shared to prepare * for KSM or temporary unmapping * @folio: The folio to share a mapping of - * @page: The mapped exclusive page + * @page: The first native page covered by the exclusive user PTE + * @vma: The VMA defining the base user PTE granule * * The caller needs to hold the page table lock and has to have the page table * entries cleared/invalidated. @@ -799,9 +820,10 @@ static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, * -EBUSY otherwise. */ static inline int folio_try_share_anon_rmap_pte(struct folio *folio, - struct page *page) + struct page *page, struct vm_area_struct *vma) { - return __folio_try_share_anon_rmap(folio, page, 1, PGTABLE_LEVEL_PTE); + return __folio_try_share_anon_rmap(folio, page, mm_pte_native_pages(vma->vm_mm), + PGTABLE_LEVEL_PTE); } /** @@ -855,12 +877,23 @@ struct page *make_device_exclusive(struct mm_struct *mm, unsigned long addr, #define PVMW_SYNC (1 << 0) /* Look for migration entries rather than present PTEs */ #define PVMW_MIGRATION (1 << 1) +/* Match one present subpage PTE at address, including its physical offset. */ +#define PVMW_SUBPAGE (1 << 2) /* Result flags */ /* The page is mapped across page table boundary */ #define PVMW_PGTABLE_CROSSED (1 << 16) +/* One reverse-map callback, optionally restricted to one physical user leaf. */ +struct rmap_walk_range { + unsigned long address; +#ifdef CONFIG_MM_SUBPAGE + unsigned int subpage_offset; + bool subpage; +#endif +}; + struct page_vma_mapped_walk { unsigned long pfn; unsigned long nr_pages; @@ -871,6 +904,10 @@ struct page_vma_mapped_walk { pte_t *pte; spinlock_t *ptl; unsigned int flags; +#ifdef CONFIG_MM_SUBPAGE + /* PVMW_SUBPAGE: byte offset within pfn; nr_pages must be one. */ + unsigned int subpage_offset; +#endif }; #define DEFINE_FOLIO_VMA_WALK(name, _folio, _vma, _address, _flags) \ @@ -883,6 +920,32 @@ struct page_vma_mapped_walk { .flags = _flags, \ } +static inline struct page_vma_mapped_walk folio_rmap_walk_init( + struct folio *folio, struct vm_area_struct *vma, + struct rmap_walk_range range, unsigned int flags) +{ + struct page_vma_mapped_walk walk = { + .pfn = folio_pfn(folio), + .nr_pages = folio_nr_pages(folio), + .pgoff = folio_pgoff(folio), + .vma = vma, + .address = range.address, + .flags = flags, + }; + +#ifdef CONFIG_MM_SUBPAGE + if (range.subpage) { + walk.flags |= PVMW_SUBPAGE; + walk.subpage_offset = range.subpage_offset; + } +#endif + return walk; +} + +#define DEFINE_FOLIO_RMAP_WALK(name, _folio, _vma, _range, _flags) \ + struct page_vma_mapped_walk name = \ + folio_rmap_walk_init(_folio, _vma, _range, _flags) + static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw) { /* HugeTLB pte is set to the relevant page table entry without pte_mapped. */ @@ -943,7 +1006,7 @@ void remove_migration_ptes(struct folio *src, struct folio *dst, * arg: passed to rmap_one() and invalid_vma() * try_lock: bail out if the rmap lock is contended * contended: indicate the rmap traversal bailed out due to lock contention - * rmap_one: executed on each vma where page is mapped + * rmap_one: executed for each mapped VMA range (possibly one user leaf) * done: for checking traversing termination condition * anon_lock: for getting anon_lock by optimized way rather than default * invalid_vma: for skipping uninterested vma @@ -957,7 +1020,7 @@ struct rmap_walk_control { * Otherwise, return true. */ bool (*rmap_one)(struct folio *folio, struct vm_area_struct *vma, - unsigned long addr, void *arg); + struct rmap_walk_range range, void *arg); int (*done)(struct folio *folio); struct anon_vma *(*anon_lock)(const struct folio *folio, struct rmap_walk_control *rwc); @@ -966,6 +1029,15 @@ struct rmap_walk_control { void rmap_walk(struct folio *folio, struct rmap_walk_control *rwc); void rmap_walk_locked(struct folio *folio, struct rmap_walk_control *rwc); +#ifdef CONFIG_MM_SUBPAGE +unsigned int folio_anon_min_order(struct folio *folio); +#else +static inline unsigned int folio_anon_min_order(struct folio *folio) +{ + return 0; +} +#endif + struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio, struct rmap_walk_control *rwc); diff --git a/include/linux/sched.h b/include/linux/sched.h index 5b36a3364ca09..c6f6bcb5590b8 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -970,6 +970,11 @@ struct task_struct { struct mm_struct *mm; struct mm_struct *active_mm; +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE + /* Per-thread exec policy; fork inherits it, the live mm is immutable. */ + unsigned int exec_page_shift; + unsigned int default_exec_page_shift; +#endif struct task_exec_state __rcu *exec_state; diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 95d0040df5841..38d573026d7fb 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -14,6 +14,10 @@ * Routines for handling mm_structs */ extern struct mm_struct *mm_alloc(void); +extern struct mm_struct *mm_alloc_exec(void); +extern struct mm_struct *mm_alloc_exec_page_shift(unsigned int page_shift); +/* Return zero for native, an alternative shift, or a negative errno. */ +int arch_exec_page_size_shift(unsigned long size); /** * mmgrab() - Pin a &struct mm_struct. diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index d45a5476b97de..08b1c15a0fb9e 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -194,7 +194,7 @@ struct signal_struct { unsigned long nvcsw, nivcsw, cnvcsw, cnivcsw; unsigned long min_flt, maj_flt, cmin_flt, cmaj_flt; unsigned long inblock, oublock, cinblock, coublock; - unsigned long maxrss, cmaxrss; + unsigned long maxrss, cmaxrss; /* kB, independent of each mm granule */ struct task_io_accounting ioac; /* diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index e729b9b0e38d4..abd7786aaa23f 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -32,6 +32,8 @@ struct swap_iocb; * isn't directly visible to userspace. */ #define SHMEM_F_MAPPING_FROZEN BIT(2) +/* This inode has used sub-native shared-page validity. */ +#define SHMEM_F_UFFD_SUBPAGE BIT(3) struct shmem_inode_info { spinlock_t lock; @@ -52,6 +54,10 @@ struct shmem_inode_info { pgoff_t fallocend; /* highest fallocate endindex */ unsigned int fsflags; /* for FS_IOC_[SG]ETFLAGS */ atomic_t stop_eviction; /* hold when working on inode */ +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + /* Swapped missing 4K units, keyed by native cache index. */ + struct xarray uffd_missing; +#endif #ifdef CONFIG_TMPFS_QUOTA struct dquot __rcu *i_dquot[MAXQUOTAS]; #endif @@ -113,13 +119,56 @@ extern unsigned long shmem_get_unmapped_area(struct file *, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); extern int shmem_lock(struct file *file, int lock, struct ucounts *ucounts); #ifdef CONFIG_SHMEM +struct folio *shmem_alloc_collapse_folio(struct mm_struct *mm, struct inode *inode, + pgoff_t index, unsigned int order); bool shmem_mapping(const struct address_space *mapping); #else +static inline struct folio *shmem_alloc_collapse_folio(struct mm_struct *mm, + struct inode *inode, pgoff_t index, unsigned int order) +{ + return ERR_PTR(-EOPNOTSUPP); +} static inline bool shmem_mapping(const struct address_space *mapping) { return false; } #endif /* CONFIG_SHMEM */ +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +/* Return a locked cache folio and caller reference; new backing is all missing. */ +struct folio *shmem_uffd_prepare_folio(struct vm_area_struct *vma, + unsigned long address, bool *new); +void shmem_uffd_abort_folio(struct folio *folio, struct vm_area_struct *vma, bool new); +bool shmem_uffd_has_missing_native_page(struct folio *folio); +bool shmem_uffd_range_missing(struct folio *folio, pgoff_t index, + unsigned int offset, unsigned long length); +/* Resident masks require no allocation; inode metadata persists only in swap. */ +void shmem_uffd_copy_page(struct page *dst, struct page *src); +void shmem_uffd_migrate(struct folio *dst, struct folio *src); +/* Locked, uptodate cache folio. */ +unsigned long shmem_uffd_missing_mask(struct folio *folio, pgoff_t index); +int shmem_uffd_set_missing(struct folio *folio, pgoff_t index, + unsigned long missing); +/* Clear populated units; no allocation. Caller holds the folio lock. */ +void shmem_uffd_mark_present(struct folio *folio, pgoff_t index, + unsigned long present); +/* Mark exactly the byte range exposed by a mapping; folio lock held. */ +void shmem_uffd_mark_mapped(struct folio *folio, unsigned long offset, + unsigned long length); +/* i_pages lock held; deletion, not replacement by swap or migration. */ +void shmem_uffd_forget(struct address_space *mapping, pgoff_t start, + unsigned long nr); +#else +static inline void shmem_uffd_copy_page(struct page *dst, struct page *src) { } +static inline void shmem_uffd_migrate(struct folio *dst, struct folio *src) { } +static inline bool shmem_uffd_has_missing_native_page(struct folio *folio) { return false; } +static inline bool shmem_uffd_range_missing(struct folio *folio, pgoff_t index, + unsigned int offset, unsigned long length) { return false; } +static inline void shmem_uffd_mark_mapped(struct folio *folio, + unsigned long offset, unsigned long length) { } +static inline void shmem_uffd_forget(struct address_space *mapping, + pgoff_t start, unsigned long nr) { } +#endif + void shmem_unlock_mapping(struct address_space *mapping); struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, pgoff_t index, gfp_t gfp_mask); @@ -160,6 +209,8 @@ static inline void shmem_uncharge(struct inode *inode, long pages) { } #endif +unsigned long shmem_partial_swap_usage_bytes(struct address_space *mapping, + u64 start, u64 end); extern unsigned long shmem_partial_swap_usage(struct address_space *mapping, pgoff_t start, pgoff_t end); diff --git a/include/linux/swap.h b/include/linux/swap.h index 8f0f68e245baa..d06880bd9e499 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -229,7 +229,7 @@ enum { */ #define SWAP_ENTRY_INVALID 0 -#ifdef CONFIG_THP_SWAP +#if defined(CONFIG_THP_SWAP) || defined(CONFIG_MM_SUBPAGE) #define SWAP_NR_ORDERS (PMD_ORDER + 1) #else #define SWAP_NR_ORDERS 1 @@ -445,6 +445,7 @@ extern int swp_swapcount(swp_entry_t entry); struct backing_dev_info; extern struct swap_info_struct *get_swap_device(swp_entry_t entry); sector_t swap_folio_sector(struct folio *folio); +sector_t swap_extent_sector(swp_entry_t entry, unsigned long *nr_pages); /* * If there is an existing swap slot reference (swap entry) and the caller @@ -457,6 +458,13 @@ sector_t swap_folio_sector(struct folio *folio); * a swap count > 1. See comments of folio_*_swap helpers for more info. */ int swap_dup_entry_direct(swp_entry_t entry); +int swap_dup_entries_direct(swp_entry_t entry, unsigned int nr); +#ifdef CONFIG_MM_SUBPAGE +/* Offset is bytes within one native swap entry; caller owns a PTE/cache ref. */ +int swap_subpage_dup(swp_entry_t entry, unsigned int offset); +void swap_subpage_put(swp_entry_t entry, unsigned int offset, bool reclaim_cache); +unsigned int swap_subpage_count(swp_entry_t entry, unsigned int offset); +#endif void swap_put_entries_direct(swp_entry_t entry, int nr); /* @@ -500,11 +508,27 @@ static inline void free_swap_cache(struct folio *folio) { } +#ifdef CONFIG_MM_SUBPAGE +static inline int swap_subpage_dup(swp_entry_t entry, unsigned int offset) +{ + return -EINVAL; +} +static inline void swap_subpage_put(swp_entry_t entry, unsigned int offset, bool reclaim_cache) { } +static inline unsigned int swap_subpage_count(swp_entry_t entry, unsigned int offset) +{ + return 0; +} +#endif static inline int swap_dup_entry_direct(swp_entry_t ent) { return 0; } +static inline int swap_dup_entries_direct(swp_entry_t ent, unsigned int nr) +{ + return 0; +} + static inline void swap_put_entries_direct(swp_entry_t ent, int nr) { } diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 874d9067a43b5..d77a8aff5b842 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -1262,6 +1262,11 @@ static inline int ksys_fadvise64_64(int fd, loff_t offset, loff_t len, unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, unsigned long fd, unsigned long pgoff); +#ifdef CONFIG_MM_SUBPAGE +unsigned long ksys_mmap_user(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, unsigned long fd, unsigned long offset); +#endif + ssize_t ksys_readahead(int fd, loff_t offset, size_t count); int ksys_ipc(unsigned int call, int first, unsigned long second, unsigned long third, void __user * ptr, long fifth); diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h index 18be159bbc341..fa87f82605cc9 100644 --- a/include/linux/uprobes.h +++ b/include/linux/uprobes.h @@ -189,6 +189,7 @@ struct uprobes_state { }; typedef int (*uprobe_write_verify_t)(struct page *page, unsigned long vaddr, + unsigned int page_offset, uprobe_opcode_t *insn, int nbytes, void *data); extern void __init uprobes_init(void); diff --git a/include/trace/events/kmem.h b/include/trace/events/kmem.h index cd7920c81f85b..fefbf65d61441 100644 --- a/include/trace/events/kmem.h +++ b/include/trace/events/kmem.h @@ -449,7 +449,7 @@ TRACE_EVENT(rss_stat, __entry->curr = current->mm == mm && !(current->flags & PF_KTHREAD); __entry->member = member; __entry->size = (percpu_counter_sum_positive(&mm->rss_stat[member]) - << PAGE_SHIFT); + << mm_page_shift(mm)); ), TP_printk("mm_id=%u curr=%d type=%s size=%ldB", diff --git a/include/trace/events/mmap.h b/include/trace/events/mmap.h index ee2843a5daef1..46df97560969d 100644 --- a/include/trace/events/mmap.h +++ b/include/trace/events/mmap.h @@ -26,7 +26,7 @@ TRACE_EVENT(vm_unmapped_area, TP_fast_assign( __entry->addr = addr; - __entry->total_vm = current->mm->total_vm; + __entry->total_vm = mm_pages_to_native(current->mm, current->mm->total_vm); __entry->flags = info->flags; __entry->length = info->length; __entry->low_limit = info->low_limit; diff --git a/include/trace/events/oom.h b/include/trace/events/oom.h index 9f0a5d1482c44..d5321b6f01ffa 100644 --- a/include/trace/events/oom.h +++ b/include/trace/events/oom.h @@ -7,7 +7,6 @@ #include #include -#define PG_COUNT_TO_KB(x) ((x) << (PAGE_SHIFT - 10)) TRACE_EVENT(oom_score_adj_update, @@ -93,10 +92,10 @@ TRACE_EVENT(mark_victim, TP_fast_assign( __entry->pid = task->pid; __assign_str(comm); - __entry->total_vm = PG_COUNT_TO_KB(task->mm->total_vm); - __entry->anon_rss = PG_COUNT_TO_KB(get_mm_counter(task->mm, MM_ANONPAGES)); - __entry->file_rss = PG_COUNT_TO_KB(get_mm_counter(task->mm, MM_FILEPAGES)); - __entry->shmem_rss = PG_COUNT_TO_KB(get_mm_counter(task->mm, MM_SHMEMPAGES)); + __entry->total_vm = mm_pages_to_kb(task->mm, task->mm->total_vm); + __entry->anon_rss = mm_pages_to_kb(task->mm, get_mm_counter(task->mm, MM_ANONPAGES)); + __entry->file_rss = mm_pages_to_kb(task->mm, get_mm_counter(task->mm, MM_FILEPAGES)); + __entry->shmem_rss = mm_pages_to_kb(task->mm, get_mm_counter(task->mm, MM_SHMEMPAGES)); __entry->uid = uid; __entry->pgtables = mm_pgtables_bytes(task->mm) >> 10; __entry->oom_score_adj = task->signal->oom_score_adj; diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h index 1a06e7feb7dc9..6e08471fbe822 100644 --- a/include/uapi/linux/prctl.h +++ b/include/uapi/linux/prctl.h @@ -421,4 +421,22 @@ struct prctl_mm_map { # define PR_CFI_DISABLE _BITUL(1) # define PR_CFI_LOCK _BITUL(2) +/* + * Aurora experimental ABI: private numbers, not upstream assignments. + * SET_EXEC selects the calling thread's next successful exec granule. + * Zero requests native pages; with an alternative DEFAULT in effect this + * installs an explicit one-shot native override. GET_EXEC returns zero when + * no override remains, or its size in bytes. Successful exec consumes it. + * SET_DEFAULT selects the fallback for subsequent execs; zero restores native. + * GET_DEFAULT returns the alternative size in bytes, or zero for native. + * Fork inherits both policies; exec preserves DEFAULT. Failed exec preserves + * both. Neither setter changes the live address space or other threads. + * AArch32 always selects 4K when supported, otherwise the native granule, + * independently of these preferences. Sizes are capability-checked. + */ +#define PR_AURORA_SET_EXEC_PAGE_SIZE 0x41555001 +#define PR_AURORA_GET_EXEC_PAGE_SIZE 0x41555002 +#define PR_AURORA_SET_DEFAULT_PAGE_SIZE 0x41555003 +#define PR_AURORA_GET_DEFAULT_PAGE_SIZE 0x41555004 + #endif /* _LINUX_PRCTL_H */ diff --git a/ipc/shm.c b/ipc/shm.c index b3e8a58e177d3..2929b9243f8e8 100644 --- a/ipc/shm.c +++ b/ipc/shm.c @@ -51,6 +51,14 @@ #include "util.h" +#ifndef arch_shm_attach_align +static unsigned long arch_shm_attach_align(const struct mm_struct *mm, + unsigned long align) +{ + return align; +} +#endif + struct shmid_kernel /* private to the kernel */ { struct kern_ipc_perm shm_perm; @@ -1538,6 +1546,7 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, if (shmid < 0) goto out; + shmlba = arch_shm_attach_align(current->mm, shmlba); if (addr) { if (addr & (shmlba - 1)) { if (shmflg & SHM_RND) { @@ -1552,7 +1561,7 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, goto out; } else #ifndef __ARCH_FORCE_SHMLBA - if (addr & ~PAGE_MASK) + if (addr & ~mm_page_mask(current->mm)) #endif goto out; } @@ -1723,6 +1732,20 @@ COMPAT_SYSCALL_DEFINE3(shmat, int, shmid, compat_uptr_t, shmaddr, int, shmflg) } #endif +/* VMA splits retain native file indices plus a subpage byte offset. */ +#ifdef CONFIG_MMU +static bool shm_vma_matches_addr(struct vm_area_struct *vma, unsigned long addr) +{ + unsigned long offset; + + if (vma->vm_ops != &shm_vm_ops || vma->vm_start < addr) + return false; + offset = vma->vm_start - addr; + return (offset >> PAGE_SHIFT) == vma->vm_pgoff && + (offset & ~PAGE_MASK) == vma_subpage_offset(vma); +} +#endif + /* * detach and kill segment if marked destroyed. * The work is done in shm_close. @@ -1739,7 +1762,7 @@ long ksys_shmdt(char __user *shmaddr) VMA_ITERATOR(vmi, mm, addr); #endif - if (addr & ~PAGE_MASK) + if (addr & ~mm_page_mask(mm)) return retval; if (mmap_write_lock_killable(mm)) @@ -1774,8 +1797,7 @@ long ksys_shmdt(char __user *shmaddr) * a fragment created by mprotect() and/or munmap(), or it * otherwise it starts at this address with no hassles. */ - if ((vma->vm_ops == &shm_vm_ops) && - (vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) { + if (shm_vma_matches_addr(vma, addr)) { /* * Record the file of the shm segment being @@ -1804,12 +1826,10 @@ long ksys_shmdt(char __user *shmaddr) * could possibly have landed at. Also cast things to loff_t to * prevent overflows and make comparisons vs. equal-width types. */ - size = PAGE_ALIGN(size); + size = ALIGN(size, mm_page_size(mm)); while (vma && (loff_t)(vma->vm_end - addr) <= size) { /* finding a matching vma now does not alter retval */ - if ((vma->vm_ops == &shm_vm_ops) && - ((vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) && - (vma->vm_file == file)) { + if (shm_vma_matches_addr(vma, addr) && vma->vm_file == file) { do_vmi_align_munmap(&vmi, vma, mm, vma->vm_start, vma->vm_end, NULL, false); } diff --git a/kernel/events/core.c b/kernel/events/core.c index 4959b287f589d..235fc75b967ae 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -6822,7 +6822,7 @@ static void perf_event_init_userpage(struct perf_event *event) /* Allow new userspace to detect that bit 0 is deprecated */ userpg->cap_bit0_is_deprecated = 1; userpg->size = offsetof(struct perf_event_mmap_page, __reserved); - userpg->data_offset = PAGE_SIZE; + userpg->data_offset = perf_metadata_size(rb); userpg->data_size = perf_data_size(rb); unlock: @@ -7007,6 +7007,11 @@ typedef void (*mapped_f)(struct perf_event *event, struct mm_struct *mm); f; \ }) +static u64 perf_mmap_offset(struct vm_area_struct *vma) +{ + return ((u64)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); +} + static void perf_mmap_open(struct vm_area_struct *vma) { struct perf_event *event = vma->vm_file->private_data; @@ -7015,7 +7020,7 @@ static void perf_mmap_open(struct vm_area_struct *vma) refcount_inc(&event->mmap_count); refcount_inc(&event->rb->mmap_count); - if (vma->vm_pgoff) + if (perf_mmap_offset(vma)) refcount_inc(&event->rb->aux_mmap_count); if (mapped) @@ -7048,7 +7053,7 @@ static void perf_mmap_close(struct vm_area_struct *vma) * The AUX buffer is strictly a sub-buffer, serialize using aux_mutex * to avoid complications. */ - if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff && + if (rb_has_aux(rb) && perf_mmap_offset(vma) == rb->aux_offset && refcount_dec_and_mutex_lock(&rb->aux_mmap_count, &rb->aux_mutex)) { /* * Stop all AUX events that are writing to this buffer, @@ -7142,7 +7147,9 @@ static void perf_mmap_close(struct vm_area_struct *vma) static vm_fault_t perf_mmap_pfn_mkwrite(struct vm_fault *vmf) { /* The first page is the user control page, others are read-only. */ - return vmf->pgoff == 0 ? 0 : VM_FAULT_SIGBUS; + return !perf_mmap_offset(vmf->vma) && + vmf->address - vmf->vma->vm_start < mm_page_size(vmf->vma->vm_mm) ? + 0 : VM_FAULT_SIGBUS; } static int perf_mmap_may_split(struct vm_area_struct *vma, unsigned long addr) @@ -7163,7 +7170,8 @@ static const struct vm_operations_struct perf_mmap_vmops = { static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma) { - unsigned long nr_pages = vma_pages(vma); + unsigned long va = vma->vm_start; + bool aux = perf_mmap_offset(vma) != 0; int err = 0; unsigned long pagenum; @@ -7207,9 +7215,21 @@ static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma) * perf_mmap_open() and perf_mmap_close()) so we ensure the lifetime of * this mapping is maintained correctly. */ - for (pagenum = 0; pagenum < nr_pages; pagenum++) { - unsigned long va = vma->vm_start + PAGE_SIZE * pagenum; - struct page *page = perf_mmap_to_page(rb, vma->vm_pgoff + pagenum); + for (pagenum = 0; va < vma->vm_end; pagenum++) { + unsigned long size = min(PAGE_SIZE, vma->vm_end - va); + struct page *page; + + if (aux) { + if (pagenum >= rb->aux_nr_pages) { + err = -EINVAL; + break; + } + page = virt_to_page(rb->aux_pages[pagenum]); + } else { + page = perf_mmap_to_page(rb, pagenum); + if (!pagenum) + size = perf_metadata_size(rb); + } if (page == NULL) { err = -EINVAL; @@ -7217,16 +7237,17 @@ static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma) } /* Map readonly, perf_mmap_pfn_mkwrite() called on write fault. */ - err = remap_pfn_range(vma, va, page_to_pfn(page), PAGE_SIZE, + err = remap_pfn_range(vma, va, page_to_pfn(page), size, vm_get_page_prot(vma->vm_flags & ~VM_SHARED)); if (err) break; + va += size; } #ifdef CONFIG_MMU /* Clear any partial mappings on error. */ if (err) - zap_vma_range(vma, vma->vm_start, nr_pages * PAGE_SIZE); + zap_vma_range(vma, vma->vm_start, vma->vm_end - vma->vm_start); #endif return err; @@ -7279,7 +7300,7 @@ static void perf_mmap_unaccount(struct vm_area_struct *vma, struct perf_buffer * { struct user_struct *user = rb->mmap_user; - atomic_long_sub((perf_data_size(rb) >> PAGE_SHIFT) + 1 - rb->mmap_locked, + atomic_long_sub(data_page_nr(rb) + 1 - rb->mmap_locked, &user->locked_vm); atomic64_sub(rb->mmap_locked, &vma->vm_mm->pinned_vm); } @@ -7287,7 +7308,9 @@ static void perf_mmap_unaccount(struct vm_area_struct *vma, struct perf_buffer * static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, unsigned long nr_pages) { - long extra = 0, user_extra = nr_pages; + unsigned int page_shift = mm_page_shift(vma->vm_mm); + unsigned long data_size; + long extra = 0, user_extra; struct perf_buffer *rb; int rb_flags = 0; @@ -7299,11 +7322,14 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, */ if (nr_pages != 0 && !is_power_of_2(nr_pages)) return -EINVAL; + data_size = nr_pages << page_shift; + /* Physical allocation and locked-memory accounting stay in native pages. */ + user_extra = DIV_ROUND_UP(data_size, PAGE_SIZE) + 1; WARN_ON_ONCE(event->ctx->parent_ctx); if (event->rb) { - if (data_page_nr(event->rb) != nr_pages) + if (perf_data_size(event->rb) != data_size) return -EINVAL; /* @@ -7339,7 +7365,7 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, if (vma->vm_flags & VM_WRITE) rb_flags |= RING_BUFFER_WRITABLE; - rb = rb_alloc(nr_pages, + rb = rb_alloc(data_size, page_shift, event->attr.watermark ? event->attr.wakeup_watermark : 0, event->cpu, rb_flags); @@ -7360,9 +7386,10 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, return 0; } -static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, - unsigned long nr_pages) +static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event) { + unsigned long size = vma->vm_end - vma->vm_start; + unsigned long nr_pages = size >> PAGE_SHIFT; long extra = 0, user_extra = nr_pages; u64 aux_offset, aux_size; struct perf_buffer *rb; @@ -7371,6 +7398,9 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, rb = event->rb; if (!rb) return -EINVAL; + /* PMU setup_aux() consumes whole native pages, independent of mmap layout. */ + if (!IS_ALIGNED(size, PAGE_SIZE)) + return -EINVAL; guard(mutex)(&rb->aux_mutex); @@ -7382,14 +7412,14 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, aux_offset = READ_ONCE(rb->user_page->aux_offset); aux_size = READ_ONCE(rb->user_page->aux_size); - if (aux_offset < perf_data_size(rb) + PAGE_SIZE) + if (aux_offset < perf_data_size(rb) + perf_metadata_size(rb)) return -EINVAL; - if (aux_offset != vma->vm_pgoff << PAGE_SHIFT) + if (aux_offset != perf_mmap_offset(vma)) return -EINVAL; /* already mapped with a different offset */ - if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff) + if (rb_has_aux(rb) && rb->aux_offset != aux_offset) return -EINVAL; if (aux_size != nr_pages * PAGE_SIZE) @@ -7419,7 +7449,7 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, if (vma->vm_flags & VM_WRITE) rb_flags |= RING_BUFFER_WRITABLE; - ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages, + ret = rb_alloc_aux(rb, event, aux_offset, nr_pages, event->attr.aux_watermark, rb_flags); if (ret) { refcount_dec(&rb->mmap_count); @@ -7459,12 +7489,13 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) return ret; vma_size = vma->vm_end - vma->vm_start; - nr_pages = vma_size / PAGE_SIZE; + nr_pages = vma_size >> mm_page_shift(vma->vm_mm); if (nr_pages > INT_MAX) return -ENOMEM; - if (vma_size != PAGE_SIZE * nr_pages) + if (vma_size != mm_page_size(vma->vm_mm) * nr_pages || + mm_page_shift(vma->vm_mm) > PAGE_SHIFT) return -EINVAL; scoped_guard (mutex, &event->mmap_mutex) { @@ -7476,10 +7507,14 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) if (event->state <= PERF_EVENT_STATE_REVOKED) return -ENODEV; - if (vma->vm_pgoff == 0) + /* One ring has one immutable metadata/data boundary for all aliases. */ + if (event->rb && event->rb->mmap_page_shift != mm_page_shift(vma->vm_mm)) + return -EINVAL; + + if (!perf_mmap_offset(vma)) ret = perf_mmap_rb(vma, event, nr_pages); else - ret = perf_mmap_aux(vma, event, nr_pages); + ret = perf_mmap_aux(vma, event); if (ret) return ret; diff --git a/kernel/events/internal.h b/kernel/events/internal.h index c03c4f2eea571..5238cfa3399b1 100644 --- a/kernel/events/internal.h +++ b/kernel/events/internal.h @@ -15,9 +15,10 @@ struct perf_buffer { struct rcu_head rcu_head; #ifdef CONFIG_PERF_USE_VMALLOC struct work_struct work; - int page_order; /* allocation order */ #endif int nr_pages; /* nr of data pages */ + unsigned int data_page_shift; /* bytes per output chunk */ + unsigned int mmap_page_shift; /* userspace metadata size */ int overwrite; /* can overwrite itself */ int paused; /* can write into ring buffer */ @@ -44,7 +45,7 @@ struct perf_buffer { long aux_head; unsigned int aux_nest; long aux_wakeup; /* last aux_watermark boundary crossed by aux_head */ - unsigned long aux_pgoff; + u64 aux_offset; int aux_nr_pages; int aux_overwrite; refcount_t aux_mmap_count; @@ -80,10 +81,11 @@ static inline void rb_toggle_paused(struct perf_buffer *rb, bool pause) } extern struct perf_buffer * -rb_alloc(int nr_pages, long watermark, int cpu, int flags); +rb_alloc(unsigned long data_size, unsigned int mmap_page_shift, + long watermark, int cpu, int flags); extern void perf_event_wakeup(struct perf_event *event); extern int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, - pgoff_t pgoff, int nr_pages, long watermark, int flags); + u64 offset, int nr_pages, long watermark, int flags); extern void rb_free_aux(struct perf_buffer *rb); extern struct perf_buffer *ring_buffer_get(struct perf_event *event); extern void ring_buffer_put(struct perf_buffer *rb); @@ -99,34 +101,19 @@ void perf_event_aux_event(struct perf_event *event, unsigned long head, extern struct page * perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff); -#ifdef CONFIG_PERF_USE_VMALLOC -/* - * Back perf_mmap() with vmalloc memory. - * - * Required for architectures that have d-cache aliasing issues. - */ - -static inline int page_order(struct perf_buffer *rb) -{ - return rb->page_order; -} - -#else - -static inline int page_order(struct perf_buffer *rb) +static inline unsigned long perf_data_size(struct perf_buffer *rb) { - return 0; + return (unsigned long)rb->nr_pages << rb->data_page_shift; } -#endif static inline int data_page_nr(struct perf_buffer *rb) { - return rb->nr_pages << page_order(rb); + return DIV_ROUND_UP(perf_data_size(rb), PAGE_SIZE); } -static inline unsigned long perf_data_size(struct perf_buffer *rb) +static inline unsigned long perf_metadata_size(struct perf_buffer *rb) { - return rb->nr_pages << (PAGE_SHIFT + page_order(rb)); + return 1UL << rb->mmap_page_shift; } static inline unsigned long perf_aux_size(struct perf_buffer *rb) @@ -154,7 +141,7 @@ static inline unsigned long perf_aux_size(struct perf_buffer *rb) handle->page++; \ handle->page &= rb->nr_pages - 1; \ handle->addr = rb->data_pages[handle->page]; \ - handle->size = PAGE_SIZE << page_order(rb); \ + handle->size = 1UL << rb->data_page_shift; \ } \ } while (len && written == size); \ \ diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index 13bd42e5493d5..5c172c186ccd2 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c @@ -238,7 +238,7 @@ __perf_output_begin(struct perf_output_handle *handle, if (unlikely(head - local_read(&rb->wakeup) > rb->watermark)) local_add(rb->watermark, &rb->wakeup); - page_shift = PAGE_SHIFT + page_order(rb); + page_shift = rb->data_page_shift; handle->page = (offset >> page_shift) & (rb->nr_pages - 1); offset &= (1UL << page_shift) - 1; @@ -681,7 +681,7 @@ static void __rb_free_aux(struct perf_buffer *rb) } int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, - pgoff_t pgoff, int nr_pages, long watermark, int flags) + u64 offset, int nr_pages, long watermark, int flags) { bool overwrite = !(flags & RING_BUFFER_WRITABLE); int node = (event->cpu == -1) ? -1 : cpu_to_node(event->cpu); @@ -788,7 +788,7 @@ int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, out: if (!ret) - rb->aux_pgoff = pgoff; + rb->aux_offset = offset; else __rb_free_aux(rb); @@ -807,8 +807,8 @@ void rb_free_aux(struct perf_buffer *rb) * Back perf_mmap() with regular GFP_KERNEL-0 pages. */ -static struct page * -__perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) +struct page * +perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) { if (pgoff > rb->nr_pages) return NULL; @@ -839,10 +839,12 @@ static void perf_mmap_free_page(void *addr) __free_page(page); } -struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) +struct perf_buffer *rb_alloc(unsigned long data_size, unsigned int mmap_page_shift, + long watermark, int cpu, int flags) { struct perf_buffer *rb; unsigned long size; + int nr_pages = DIV_ROUND_UP(data_size, PAGE_SIZE); int i, node; size = sizeof(struct perf_buffer); @@ -867,6 +869,10 @@ struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) } rb->nr_pages = nr_pages; + rb->mmap_page_shift = mmap_page_shift; + /* A ring smaller than a native page still wraps at its requested size. */ + rb->data_page_shift = data_size && data_size < PAGE_SIZE ? + ilog2(data_size) : PAGE_SHIFT; ring_buffer_init(rb, watermark, flags); @@ -896,8 +902,8 @@ void rb_free(struct perf_buffer *rb) } #else -static struct page * -__perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) +struct page * +perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) { /* The '>' counts in the user page. */ if (pgoff > data_page_nr(rb)) @@ -921,10 +927,12 @@ void rb_free(struct perf_buffer *rb) schedule_work(&rb->work); } -struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) +struct perf_buffer *rb_alloc(unsigned long data_size, unsigned int mmap_page_shift, + long watermark, int cpu, int flags) { struct perf_buffer *rb; unsigned long size; + int nr_pages = DIV_ROUND_UP(data_size, PAGE_SIZE); void *all_buf; int node; @@ -944,9 +952,10 @@ struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) rb->user_page = all_buf; rb->data_pages[0] = all_buf + PAGE_SIZE; + rb->mmap_page_shift = mmap_page_shift; if (nr_pages) { rb->nr_pages = 1; - rb->page_order = ilog2(nr_pages); + rb->data_page_shift = ilog2(data_size); } ring_buffer_init(rb, watermark, flags); @@ -961,21 +970,3 @@ struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) } #endif - -struct page * -perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) -{ - if (rb->aux_nr_pages) { - /* above AUX space */ - if (pgoff > rb->aux_pgoff + rb->aux_nr_pages) - return NULL; - - /* AUX space */ - if (pgoff >= rb->aux_pgoff) { - int aux_pgoff = array_index_nospec(pgoff - rb->aux_pgoff, rb->aux_nr_pages); - return virt_to_page(rb->aux_pages[aux_pgoff]); - } - } - - return __perf_mmap_to_page(rb, pgoff); -} diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index e4f526c9bfbf0..23dcd90acdc0c 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -30,6 +30,7 @@ #include #include /* check_stable_address_space */ #include +#include #include @@ -144,12 +145,14 @@ static bool valid_vma(struct vm_area_struct *vma, bool is_register) static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset) { - return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT) - + vma_subpage_offset(vma); } static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr) { - return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start); + return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma) + + (vaddr - vma->vm_start); } /** @@ -191,7 +194,8 @@ static void copy_to_page(struct page *page, unsigned long vaddr, const void *src kunmap_local(kaddr); } -static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *insn, +static int verify_opcode(struct page *page, unsigned long vaddr, unsigned int page_offset, + uprobe_opcode_t *insn, int nbytes, void *data) { uprobe_opcode_t old_opcode; @@ -206,7 +210,7 @@ static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t * is a trap variant; uprobes always wins over any other (gdb) * breakpoint. */ - uprobe_copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE); + uprobe_copy_from_page(page, page_offset, &old_opcode, UPROBE_SWBP_INSN_SIZE); is_swbp = is_swbp_insn(&old_opcode); if (is_swbp_insn(insn)) { @@ -307,14 +311,24 @@ __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d) { void *kaddr; struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment; + struct vm_area_struct *vma; + bool alternative = mm_page_size(mm) != PAGE_SIZE; +#endif int ret; short *ptr; if (!vaddr || !d) return -EINVAL; - ret = get_user_pages_remote(mm, vaddr, 1, - FOLL_WRITE, &page, NULL); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) + ret = get_user_fragment_vma_remote(mm, vaddr, sizeof(short), + FOLL_WRITE, &fragment, &vma); + else +#endif + ret = get_user_pages_remote(mm, vaddr, 1, FOLL_WRITE, &page, NULL); if (unlikely(ret <= 0)) { /* * We are asking for 1 page. If get_user_pages_remote() fails, @@ -323,8 +337,16 @@ __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d) return ret == 0 ? -EBUSY : ret; } - kaddr = kmap_local_page(page); - ptr = kaddr + (vaddr & ~PAGE_MASK); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) { + kaddr = kmap_local_folio(fragment.folio, fragment.offset); + ptr = kaddr; + } else +#endif + { + kaddr = kmap_local_page(page); + ptr = kaddr + (vaddr & ~PAGE_MASK); + } if (unlikely(*ptr + d < 0)) { pr_warn("ref_ctr going negative. vaddr: 0x%lx, " @@ -337,7 +359,12 @@ __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d) ret = 0; out: kunmap_local(kaddr); - put_page(page); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) + release_user_fragments(&fragment, 1, !ret); + else +#endif + put_page(page); return ret; } @@ -381,7 +408,8 @@ static int update_ref_ctr(struct uprobe *uprobe, struct mm_struct *mm, } static bool orig_page_is_identical(struct vm_area_struct *vma, - unsigned long vaddr, struct page *page, bool *pmd_mappable) + unsigned long vaddr, struct page *page, unsigned int page_offset, + bool *pmd_mappable) { const pgoff_t index = vaddr_to_offset(vma, vaddr) >> PAGE_SHIFT; struct folio *orig_folio = filemap_get_folio(vma->vm_file->f_mapping, @@ -393,19 +421,30 @@ static bool orig_page_is_identical(struct vm_area_struct *vma, return false; orig_page = folio_file_page(orig_folio, index); - *pmd_mappable = folio_test_pmd_mappable(orig_folio); - identical = folio_test_uptodate(orig_folio) && - pages_identical(page, orig_page); + *pmd_mappable = mm_page_size(vma->vm_mm) == PAGE_SIZE && + folio_test_pmd_mappable(orig_folio); + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + unsigned int offset = vma_page_offset_at(vma, vaddr).offset; + void *a = kmap_local_page(page), *b = kmap_local_page(orig_page); + + identical = folio_test_uptodate(orig_folio) && + !memcmp(a + page_offset, b + offset, mm_page_size(vma->vm_mm)); + kunmap_local(b); + kunmap_local(a); + } else { + identical = folio_test_uptodate(orig_folio) && + pages_identical(page, orig_page); + } folio_put(orig_folio); return identical; } static int __uprobe_write(struct vm_area_struct *vma, - struct folio_walk *fw, struct folio *folio, - unsigned long insn_vaddr, uprobe_opcode_t *insn, int nbytes, + struct folio_walk *fw, struct folio *folio, struct mm_subpage *slot, + unsigned int page_offset, unsigned long insn_vaddr, uprobe_opcode_t *insn, int nbytes, bool is_register) { - const unsigned long vaddr = insn_vaddr & PAGE_MASK; + const unsigned long vaddr = insn_vaddr & mm_page_mask(vma->vm_mm); bool pmd_mappable; /* For now, we'll only handle PTE-mapped folios. */ @@ -417,6 +456,12 @@ static int __uprobe_write(struct vm_area_struct *vma, * but the VMA might not be writable. */ if (!pte_write(fw->pte)) { +#ifdef CONFIG_MM_SUBPAGE + if (slot) { + if (!mm_subpage_is_exclusive(slot)) + return -EFAULT; + } else +#endif if (!PageAnonExclusive(fw->page)) return -EFAULT; if (unlikely(userfaultfd_pte_wp(vma, fw->pte))) @@ -430,31 +475,49 @@ static int __uprobe_write(struct vm_area_struct *vma, */ flush_cache_page(vma, vaddr, pte_pfn(fw->pte)); fw->pte = ptep_clear_flush(vma, vaddr, fw->ptep); - copy_to_page(fw->page, insn_vaddr, insn, nbytes); + copy_to_page(fw->page, page_offset, insn, nbytes); /* * When unregistering, we may only zap a PTE if uffd is disabled and * there are no unexpected folio references ... */ - if (is_register || userfaultfd_missing(vma) || - (folio_ref_count(folio) != folio_expected_ref_count(folio) + 1)) + if (is_register || userfaultfd_missing(vma)) + goto remap; +#ifdef CONFIG_MM_SUBPAGE + if (slot) { + if (!mm_subpage_can_zap(slot)) + goto remap; + } else +#endif + if (folio_ref_count(folio) != folio_expected_ref_count(folio) + 1) goto remap; /* * ... and the mapped page is identical to the original page that * would get faulted in on next access. */ - if (!orig_page_is_identical(vma, vaddr, fw->page, &pmd_mappable)) + if (!orig_page_is_identical(vma, vaddr, fw->page, + page_offset - (insn_vaddr - vaddr), &pmd_mappable)) goto remap; dec_mm_counter(vma->vm_mm, MM_ANONPAGES); - folio_remove_rmap_pte(folio, fw->page, vma); +#ifdef CONFIG_MM_SUBPAGE + if (slot) + mm_subpage_remove_anon_rmap(slot, vma); + else +#endif + folio_remove_rmap_pte(folio, fw->page, vma); if (!folio_mapped(folio) && folio_test_swapcache(folio) && folio_trylock(folio)) { folio_free_swap(folio); folio_unlock(folio); } - folio_put(folio); +#ifdef CONFIG_MM_SUBPAGE + if (slot) + mm_subpage_put(slot); /* Retired PTE reference; caller still has its GET. */ + else +#endif + folio_put(folio); return pmd_mappable; remap: @@ -494,13 +557,24 @@ int uprobe_write_opcode(struct arch_uprobe *auprobe, struct vm_area_struct *vma, verify_opcode, is_register, true /* do_update_ref_ctr */, NULL); } +static void uprobe_put_folio(struct folio *folio, struct mm_subpage *slot) +{ +#ifdef CONFIG_MM_SUBPAGE + if (slot) { + mm_subpage_put(slot); + return; + } +#endif + folio_put(folio); +} + int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, const unsigned long insn_vaddr, uprobe_opcode_t *insn, int nbytes, uprobe_write_verify_t verify, bool is_register, bool do_update_ref_ctr, void *data) { - const unsigned long vaddr = insn_vaddr & PAGE_MASK; struct mm_struct *mm = vma->vm_mm; + const unsigned long vaddr = insn_vaddr & mm_page_mask(mm); struct uprobe *uprobe; int ret, ref_ctr_updated = 0; unsigned int gup_flags = FOLL_FORCE; @@ -508,7 +582,16 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, struct folio_walk fw; struct folio *folio; struct page *page; - + struct mm_subpage *slot = NULL; + unsigned int page_offset; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment; + struct vm_area_struct *found; +#endif + + /* Larger-than-native public mms are not enabled by the mixed-page ABI yet. */ + if (mm_page_size(mm) > PAGE_SIZE) + return -EOPNOTSUPP; uprobe = container_of(auprobe, struct uprobe, arch); if (WARN_ON_ONCE(!is_cow_mapping(vma->vm_flags))) @@ -526,14 +609,29 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, gup_flags |= FOLL_WRITE | FOLL_SPLIT_PMD; retry: - ret = get_user_pages_remote(mm, vaddr, 1, gup_flags, &page, NULL); - if (ret <= 0) - goto out; - folio = page_folio(page); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + ret = get_user_fragment_vma_remote(mm, vaddr, mm_page_size(mm), + gup_flags & ~FOLL_SPLIT_PMD, &fragment, &found); + if (ret <= 0) + goto out; + folio = fragment.folio; + slot = fragment.subpage; + page = folio_page(folio, fragment.offset >> PAGE_SHIFT); + page_offset = offset_in_page(fragment.offset) + insn_vaddr - vaddr; + } else +#endif + { + ret = get_user_pages_remote(mm, vaddr, 1, gup_flags, &page, NULL); + if (ret <= 0) + goto out; + folio = page_folio(page); + page_offset = offset_in_page(insn_vaddr); + } - ret = verify(page, insn_vaddr, insn, nbytes, data); + ret = verify(page, insn_vaddr, page_offset, insn, nbytes, data); if (ret <= 0) { - folio_put(folio); + uprobe_put_folio(folio, slot); goto out; } @@ -541,7 +639,7 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, if (do_update_ref_ctr && !ref_ctr_updated && uprobe->ref_ctr_offset) { ret = update_ref_ctr(uprobe, mm, is_register ? 1 : -1); if (ret) { - folio_put(folio); + uprobe_put_folio(folio, slot); goto out; } @@ -551,7 +649,7 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, ret = 0; if (unlikely(!folio_test_anon(folio) || folio_is_zone_device(folio))) { VM_WARN_ON_ONCE(is_register); - folio_put(folio); + uprobe_put_folio(folio, slot); goto out; } @@ -562,22 +660,29 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, * be able to do it under PTL. */ mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, - vaddr, vaddr + PAGE_SIZE); + vaddr, vaddr + mm_page_size(mm)); mmu_notifier_invalidate_range_start(&range); } ret = -EAGAIN; /* Walk the page tables again, to perform the actual update. */ if (folio_walk_start(&fw, vma, vaddr, 0)) { - if (fw.page == page) - ret = __uprobe_write(vma, &fw, folio, insn_vaddr, insn, nbytes, is_register); + bool same = fw.page == page; +#ifdef CONFIG_MM_SUBPAGE + if (slot) + same &= fw.level == FW_LEVEL_PTE && + pte_phys_mm(mm, fw.pte) == mm_subpage_phys(slot); +#endif + if (same) + ret = __uprobe_write(vma, &fw, folio, slot, page_offset, + insn_vaddr, insn, nbytes, is_register); folio_walk_end(&fw, vma); } if (!is_register) mmu_notifier_invalidate_range_end(&range); - folio_put(folio); + uprobe_put_folio(folio, slot); switch (ret) { case -EFAULT: gup_flags |= FOLL_WRITE | FOLL_SPLIT_PMD; @@ -1211,7 +1316,9 @@ build_map_info(struct address_space *mapping, loff_t offset, bool is_register) again: i_mmap_lock_read(mapping); vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { - if (!valid_vma(vma, is_register)) + if (!valid_vma(vma, is_register) || + offset < vaddr_to_offset(vma, vma->vm_start) || + offset >= vaddr_to_offset(vma, vma->vm_end)) continue; if (!prev && !more) { @@ -1482,7 +1589,7 @@ static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) file_inode(vma->vm_file) != uprobe->inode) continue; - offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT; + offset = vaddr_to_offset(vma, vma->vm_start); if (uprobe->offset < offset || uprobe->offset >= offset + vma->vm_end - vma->vm_start) continue; @@ -1696,8 +1803,14 @@ static const struct vm_special_mapping xol_mapping = { unsigned long __weak arch_uprobe_get_xol_area(void) { - /* Try to map as high as possible, this is only a hint. */ - return get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, PAGE_SIZE, 0, 0); + unsigned long size = PAGE_SIZE; + unsigned long addr; + + /* XOL arch copy hooks use native offsets into the one backing page. */ + if (mm_page_size(current->mm) < PAGE_SIZE) + size += PAGE_SIZE - mm_page_size(current->mm); + addr = get_unmapped_area(NULL, TASK_SIZE - size, size, 0, 0); + return IS_ERR_VALUE(addr) ? addr : ALIGN(addr, PAGE_SIZE); } /* Slot allocation for XOL */ @@ -2403,6 +2516,23 @@ static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr) if (likely(result == 0)) goto out; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + struct user_page_fragment fragment; + struct vm_area_struct *vma; + void *kaddr; + + result = get_user_fragment_vma_remote(mm, vaddr, UPROBE_SWBP_INSN_SIZE, + FOLL_FORCE, &fragment, &vma); + if (result != 1) + return result < 0 ? result : -EFAULT; + kaddr = kmap_local_folio(fragment.folio, fragment.offset); + memcpy(&opcode, kaddr, UPROBE_SWBP_INSN_SIZE); + kunmap_local(kaddr); + release_user_fragments(&fragment, 1, false); + goto out; + } +#endif result = get_user_pages(vaddr, 1, FOLL_FORCE, &page); if (result < 0) return result; @@ -2443,7 +2573,7 @@ static struct uprobe *find_active_uprobe_speculative(unsigned long bp_vaddr) if (!vm_file) return NULL; - offset = (loff_t)(vma->vm_pgoff << PAGE_SHIFT) + (bp_vaddr - vma->vm_start); + offset = vaddr_to_offset(vma, bp_vaddr); uprobe = find_uprobe_rcu(vm_file->f_inode, offset); if (!uprobe) return NULL; diff --git a/kernel/fork.c b/kernel/fork.c index 854448152696b..0c9c9a9c12e3c 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -113,7 +113,7 @@ #include #include -#include +#include #include #include @@ -642,6 +642,12 @@ static void check_mm(struct mm_struct *mm) } } +#ifdef CONFIG_MM_SUBPAGE + if (mm->committed_user_pages) + pr_alert("BUG: non-zero user commitment on freeing mm: %lu\n", + mm->committed_user_pages); +#endif + if (mm_pgtables_bytes(mm)) pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n", mm_pgtables_bytes(mm)); @@ -1092,6 +1098,9 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p) INIT_LIST_HEAD(&mm->mmlist); mm_pgtables_bytes_init(mm); mm->map_count = 0; +#ifdef CONFIG_MM_SUBPAGE + mm->committed_user_pages = 0; +#endif mm->locked_vm = 0; atomic64_set(&mm->pinned_vm, 0); memset(&mm->rss_stat, 0, sizeof(mm->rss_stat)); @@ -1162,7 +1171,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p) /* * Allocate and initialize an mm_struct. */ -struct mm_struct *mm_alloc(void) +static struct mm_struct *__mm_alloc(bool for_exec, unsigned int page_shift) { struct mm_struct *mm; @@ -1171,10 +1180,32 @@ struct mm_struct *mm_alloc(void) return NULL; memset(mm, 0, sizeof(*mm)); + if (for_exec) + arch_mm_init_exec(mm, page_shift); return mm_init(mm, current); } + +struct mm_struct *mm_alloc(void) +{ + return __mm_alloc(false, 0); +} EXPORT_SYMBOL_IF_KUNIT(mm_alloc); +struct mm_struct *mm_alloc_exec(void) +{ +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE + return __mm_alloc(true, current->exec_page_shift); +#else + return __mm_alloc(true, 0); +#endif +} + +/* The caller has validated the requested exec granule. */ +struct mm_struct *mm_alloc_exec_page_shift(unsigned int page_shift) +{ + return __mm_alloc(true, page_shift); +} + static inline void __mmput(struct mm_struct *mm) { VM_BUG_ON(atomic_read(&mm->mm_users)); diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 6a1971f9588e4..c5f76bd72eaa8 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -481,6 +482,44 @@ static u64 get_inode_sequence_number(struct inode *inode) } } +/* Keep a typed slot reference alive until the mapping identity is sampled. */ +struct futex_page_ref { + struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment; +#endif +}; + +static int futex_get_page_ref(struct mm_struct *mm, unsigned long address, + unsigned long uaddr, unsigned int size, unsigned int flags, + struct futex_page_ref *ref) +{ +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) != PAGE_SIZE) { + int ret = get_user_fragments_remote(mm, uaddr, size, flags, + &ref->fragment, 1); + + if (ret != 1) + return ret ?: -EFAULT; + ref->page = folio_page(ref->fragment.folio, + ref->fragment.offset >> PAGE_SHIFT); + return 1; + } +#endif + return get_user_pages_fast(address, 1, flags, &ref->page); +} + +static void futex_put_page_ref(struct futex_page_ref *ref) +{ +#ifdef CONFIG_MM_SUBPAGE + if (ref->fragment.folio) { + release_user_fragments(&ref->fragment, 1, false); + return; + } +#endif + put_page(ref->page); +} + /** * get_futex_key() - Get parameters which are the keys for a futex * @uaddr: virtual address of the futex @@ -514,6 +553,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, unsigned long address = (unsigned long)uaddr; struct mm_struct *mm = current->mm; struct page *page; + struct futex_page_ref ref = {}; struct folio *folio; struct address_space *mapping; int node, err, size, ro = 0; @@ -598,13 +638,13 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, if (unlikely(should_fail_futex(true))) return -EFAULT; - err = get_user_pages_fast(address, 1, FOLL_WRITE, &page); + err = futex_get_page_ref(mm, address, (unsigned long)uaddr, size, FOLL_WRITE, &ref); /* * If write access is not required (eg. FUTEX_WAIT), try * and get read-only access. */ if (err == -EFAULT && rw == FUTEX_READ) { - err = get_user_pages_fast(address, 1, 0, &page); + err = futex_get_page_ref(mm, address, (unsigned long)uaddr, size, 0, &ref); ro = 1; } if (err < 0) @@ -629,6 +669,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, * filesystem-backed pages, the precise page is required as the * index of the page determines the key. */ + page = ref.page; folio = page_folio(page); mapping = READ_ONCE(folio->mapping); @@ -658,7 +699,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, folio_lock(folio); shmem_swizzled = folio_test_swapcache(folio) || folio->mapping; folio_unlock(folio); - folio_put(folio); + futex_put_page_ref(&ref); if (shmem_swizzled) goto again; @@ -708,7 +749,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, if (READ_ONCE(folio->mapping) != mapping) { rcu_read_unlock(); - folio_put(folio); + futex_put_page_ref(&ref); goto again; } @@ -716,11 +757,15 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, inode = READ_ONCE(mapping->host); if (!inode) { rcu_read_unlock(); - folio_put(folio); + futex_put_page_ref(&ref); goto again; } +#ifdef CONFIG_MM_SUBPAGE + if (ref.fragment.folio) + key->both.offset = offset_in_page(ref.fragment.offset); +#endif key->both.offset |= FUT_OFF_INODE; /* inode-based key */ key->shared.i_seq = get_inode_sequence_number(inode); key->shared.pgoff = page_pgoff(folio, page); @@ -728,7 +773,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, } out: - folio_put(folio); + futex_put_page_ref(&ref); return err; } diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index b6cace1d0e322..41a14d7da1118 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -2466,7 +2466,7 @@ static unsigned int task_nr_scan_windows(struct task_struct *p) * on resident pages */ nr_scan_pages = MB_TO_PAGES(sysctl_numa_balancing_scan_size); - rss = get_mm_rss(p->mm); + rss = mm_pages_to_native(p->mm, get_mm_rss(p->mm)); if (!rss) rss = nr_scan_pages; @@ -4479,8 +4479,8 @@ static void task_numa_work(struct callback_head *work) * areas faster. */ if (nr_pte_updates) - pages -= (end - start) >> PAGE_SHIFT; - virtpages -= (end - start) >> PAGE_SHIFT; + pages -= DIV_ROUND_UP(end - start, PAGE_SIZE); + virtpages -= DIV_ROUND_UP(end - start, PAGE_SIZE); start = end; if (pages <= 0 || virtpages <= 0) diff --git a/kernel/sys.c b/kernel/sys.c index feed7a0bfa887..59f5bb6d4825f 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -1927,7 +1927,7 @@ void getrusage(struct task_struct *p, int who, struct rusage *r) } out_children: - r->ru_maxrss = maxrss * (PAGE_SIZE / 1024); /* convert pages to KBs */ + r->ru_maxrss = maxrss; /* signal and mm high-water maxima are in kB */ r->ru_utime = ns_to_kernel_old_timeval(utime); r->ru_stime = ns_to_kernel_old_timeval(stime); } @@ -2532,6 +2532,40 @@ static int prctl_set_thp_disable(bool thp_disable, unsigned long flags, return 0; } +static int prctl_page_size_get(struct task_struct *task, bool persistent) +{ +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE + unsigned int shift = persistent ? task->default_exec_page_shift : + task->exec_page_shift; + + return shift ? 1U << shift : 0; +#else + return -EINVAL; +#endif +} + +static int prctl_page_size_set(struct task_struct *task, unsigned long size, + bool persistent) +{ +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE + int shift = arch_exec_page_size_shift(size); + + if (shift < 0) + return shift; + if (persistent) { + task->default_exec_page_shift = shift; + } else { + /* A native request overrides an inherited default for one exec. */ + if (!shift && task->default_exec_page_shift) + shift = PAGE_SHIFT; + task->exec_page_shift = shift; + } + return 0; +#else + return -EINVAL; +#endif +} + int __weak arch_prctl_mem_model_get(struct task_struct *t) { return -EINVAL; @@ -2555,6 +2589,22 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, error = 0; switch (option) { + case PR_AURORA_GET_EXEC_PAGE_SIZE: + if (arg2 || arg3 || arg4 || arg5) + return -EINVAL; + return prctl_page_size_get(me, false); + case PR_AURORA_SET_EXEC_PAGE_SIZE: + if (arg3 || arg4 || arg5) + return -EINVAL; + return prctl_page_size_set(me, arg2, false); + case PR_AURORA_GET_DEFAULT_PAGE_SIZE: + if (arg2 || arg3 || arg4 || arg5) + return -EINVAL; + return prctl_page_size_get(me, true); + case PR_AURORA_SET_DEFAULT_PAGE_SIZE: + if (arg3 || arg4 || arg5) + return -EINVAL; + return prctl_page_size_set(me, arg2, true); case PR_GET_MEM_MODEL: if (arg2 || arg3 || arg4 || arg5) return -EINVAL; diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index e190dad1eb914..21e4e12d70007 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include "trace_dynevent.h" @@ -554,6 +555,11 @@ static int user_event_enabler_write(struct user_event_mm *mm, unsigned long uaddr = enabler->addr; unsigned long *ptr; struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment; + struct vm_area_struct *vma; + bool alternative = mm_page_size(mm->mm) != PAGE_SIZE; +#endif void *kaddr; int bit = ENABLE_BIT(enabler); int ret; @@ -573,8 +579,14 @@ static int user_event_enabler_write(struct user_event_mm *mm, align_addr_bit(&uaddr, &bit, ENABLE_BITOPS(enabler)); - ret = pin_user_pages_remote(mm->mm, uaddr, 1, FOLL_WRITE | FOLL_NOFAULT, - &page, NULL); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) + ret = pin_user_fragment_vma_remote(mm->mm, uaddr, sizeof(long), + FOLL_WRITE | FOLL_NOFAULT, &fragment, &vma); + else +#endif + ret = pin_user_pages_remote(mm->mm, uaddr, 1, FOLL_WRITE | FOLL_NOFAULT, + &page, NULL); if (unlikely(ret <= 0)) { if (!fixup_fault) @@ -586,8 +598,17 @@ static int user_event_enabler_write(struct user_event_mm *mm, return -EFAULT; } - kaddr = kmap_local_page(page); - ptr = kaddr + (uaddr & ~PAGE_MASK); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) { + /* align_addr_bit() keeps the complete atomic word in this fragment. */ + kaddr = kmap_local_folio(fragment.folio, fragment.offset); + ptr = kaddr; + } else +#endif + { + kaddr = kmap_local_page(page); + ptr = kaddr + (uaddr & ~PAGE_MASK); + } /* Update bit atomically, user tracers must be atomic as well */ if (enabler->event && enabler->event->status) @@ -596,7 +617,12 @@ static int user_event_enabler_write(struct user_event_mm *mm, clear_bit(bit, ptr); kunmap_local(kaddr); - unpin_user_pages_dirty_lock(&page, 1, true); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) + release_user_fragments(&fragment, 1, true); + else +#endif + unpin_user_pages_dirty_lock(&page, 1, true); return 0; } diff --git a/kernel/tsacct.c b/kernel/tsacct.c index 5c153106e6427..5f59d071ab565 100644 --- a/kernel/tsacct.c +++ b/kernel/tsacct.c @@ -91,16 +91,16 @@ void xacct_add_tsk(struct taskstats *stats, struct task_struct *p) { struct mm_struct *mm; - /* convert pages-nsec/1024 to Mbyte-usec, see __acct_update_integrals */ - stats->coremem = p->acct_rss_mem1 * PAGE_SIZE; + /* convert account-units-nsec/1024 to Mbyte-usec, see __acct_update_integrals */ + stats->coremem = p->acct_rss_mem1 * (1UL << MM_ACCOUNT_SHIFT); do_div(stats->coremem, 1000 * KB); - stats->virtmem = p->acct_vm_mem1 * PAGE_SIZE; + stats->virtmem = p->acct_vm_mem1 * (1UL << MM_ACCOUNT_SHIFT); do_div(stats->virtmem, 1000 * KB); mm = get_task_mm(p); if (mm) { /* adjust to KB unit */ - stats->hiwater_rss = get_mm_hiwater_rss(mm) * PAGE_SIZE / KB; - stats->hiwater_vm = get_mm_hiwater_vm(mm) * PAGE_SIZE / KB; + stats->hiwater_rss = mm_pages_to_kb(mm, get_mm_hiwater_rss(mm)); + stats->hiwater_vm = mm_pages_to_kb(mm, get_mm_hiwater_vm(mm)); mmput(mm); } stats->read_char = p->ioac.rchar & KB_MASK; @@ -140,8 +140,10 @@ static void __acct_update_integrals(struct task_struct *tsk, * The final unit reported to userspace is Mbyte-usecs, * the rest of the math is done in xacct_add_tsk. */ - tsk->acct_rss_mem1 += delta * get_mm_rss(tsk->mm) >> 10; - tsk->acct_vm_mem1 += delta * READ_ONCE(tsk->mm->total_vm) >> 10; + tsk->acct_rss_mem1 += delta * mm_pages_to_shift(tsk->mm, get_mm_rss(tsk->mm), + MM_ACCOUNT_SHIFT) >> 10; + tsk->acct_vm_mem1 += delta * mm_pages_to_shift(tsk->mm, READ_ONCE(tsk->mm->total_vm), + MM_ACCOUNT_SHIFT) >> 10; } /** diff --git a/lib/vdso/datastore.c b/lib/vdso/datastore.c index 17d37b82ebc68..d808ddcb94b01 100644 --- a/lib/vdso/datastore.c +++ b/lib/vdso/datastore.c @@ -83,7 +83,8 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, vm_fault_t err; addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE; - err = vmf_insert_page(vma, addr, page); + err = vmf_insert_page_offset(vma, addr, page, + vma_page_offset_at(vma, vmf->address).offset); if (unlikely(err & VM_FAULT_ERROR)) return err; page = timens_page; diff --git a/mm/Kconfig b/mm/Kconfig index 9e0ca48249054..7243e2810fb6e 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1203,6 +1203,28 @@ config IDLE_PAGE_TRACKING See Documentation/admin-guide/mm/idle_page_tracking.rst for more details. +config ARCH_HAS_MM_PGTABLE + bool + help + Architecture supplies target-mm page-table geometry and accessors. + This does not change the native physical allocation granule. + +config ARCH_HAS_USER_PAGE_SIZE + bool + help + Architecture supports selecting the user mapping granule at exec. + It supplies arch_exec_page_size_shift() to validate requests, and + integrates the selected granule into mm allocation and binary loading. + +config MM_SUBPAGE + bool + select PAGE_EXTENSION + select XARRAY_MULTI + help + Reference-counted user data slots smaller than a native folio. + Architectures must integrate rmap, faults, reclaim and pins before + exposing these backing primitives to userspace. + # Architectures which implement cpu_dcache_is_aliasing() to query # whether the data caches are aliased (VIVT or VIPT with dcache # aliasing) need to select this. @@ -1478,9 +1500,15 @@ config ARCH_HAS_USER_SHADOW_STACK config HAVE_ARCH_TLB_REMOVE_TABLE def_bool n +config ARCH_SUPPORTS_PT_RECLAIM + bool + help + The architecture's custom table-free callback supports PTE reclamation. + config PT_RECLAIM def_bool y - depends on MMU_GATHER_RCU_TABLE_FREE && !HAVE_ARCH_TLB_REMOVE_TABLE + depends on MMU_GATHER_RCU_TABLE_FREE + depends on !HAVE_ARCH_TLB_REMOVE_TABLE || ARCH_SUPPORTS_PT_RECLAIM help Try to reclaim empty user page table pages in paths other than munmap and exit_mmap path. diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index 58ccb14f80b35..71ab7b1bf3d9a 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -162,9 +162,10 @@ int damon_cold_score(struct damon_ctx *c, struct damon_region *r, } static bool damon_folio_mkold_one(struct folio *folio, - struct vm_area_struct *vma, unsigned long addr, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); + unsigned long addr = rmap.address; + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); while (page_vma_mapped_walk(&pvmw)) { addr = pvmw.address; @@ -197,10 +198,11 @@ void damon_folio_mkold(struct folio *folio) } static bool damon_folio_young_one(struct folio *folio, - struct vm_area_struct *vma, unsigned long addr, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { + unsigned long addr = rmap.address; bool *accessed = arg; - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); pte_t pte; *accessed = false; diff --git a/mm/filemap.c b/mm/filemap.c index ebc51fabfcd8d..501ebe3419f68 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -224,6 +224,7 @@ void __filemap_remove_folio(struct folio *folio, void *shadow) trace_mm_filemap_delete_from_page_cache(folio); filemap_unaccount_folio(mapping, folio); + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); page_cache_delete(mapping, folio, shadow); } @@ -307,6 +308,7 @@ static void page_cache_delete_batch(struct address_space *mapping, WARN_ON_ONCE(!folio_test_locked(folio)); + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); folio->mapping = NULL; /* Leave folio->index set: truncation lookup relies on it */ @@ -826,6 +828,7 @@ void replace_page_cache_folio(struct folio *old, struct folio *new) mem_cgroup_replace_folio(old, new); xas_lock_irq(&xas); + shmem_uffd_migrate(new, old); xas_store(&xas, new); old->mapping = NULL; @@ -3409,7 +3412,7 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) */ struct vm_area_struct *vma = vmf->vma; unsigned long start = vma->vm_pgoff; - unsigned long end = start + vma_pages(vma); + unsigned long end = vma_last_pgoff(vma) + 1; unsigned long ra_end; ra->order = exec_folio_order(); @@ -3556,7 +3559,12 @@ vm_fault_t filemap_fault(struct vm_fault *vmf) bool mapping_locked = false; max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); - if (unlikely(index >= max_idx)) + if (unlikely(index >= max_idx +#ifdef CONFIG_MM_SUBPAGE + || (mm_page_size(vmf->vma->vm_mm) < PAGE_SIZE && + !vma_file_offset_valid(vmf->vma, vmf->address, i_size_read(inode))) +#endif + )) return VM_FAULT_SIGBUS; trace_mm_filemap_fault(mapping, index); @@ -3660,7 +3668,12 @@ vm_fault_t filemap_fault(struct vm_fault *vmf) * We must recheck i_size under page lock. */ max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); - if (unlikely(index >= max_idx)) { + if (unlikely(index >= max_idx +#ifdef CONFIG_MM_SUBPAGE + || (mm_page_size(vmf->vma->vm_mm) < PAGE_SIZE && + !vma_file_offset_valid(vmf->vma, vmf->address, i_size_read(inode))) +#endif + )) { folio_unlock(folio); folio_put(folio); return VM_FAULT_SIGBUS; @@ -3809,6 +3822,10 @@ static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf, } do { + if (userfaultfd_missing(vmf->vma) && + shmem_uffd_range_missing(folio, folio->index + + folio_page_idx(folio, page + count), 0, PAGE_SIZE)) + goto skip; if (PageHWPoison(page + count)) goto skip; @@ -3863,6 +3880,9 @@ static vm_fault_t filemap_map_order0_folio(struct vm_fault *vmf, vm_fault_t ret = 0; struct page *page = &folio->page; + if (userfaultfd_missing(vmf->vma) && + shmem_uffd_range_missing(folio, folio->index, 0, PAGE_SIZE)) + goto out; if (PageHWPoison(page)) goto out; @@ -3887,6 +3907,48 @@ static vm_fault_t filemap_map_order0_folio(struct vm_fault *vmf, return ret; } +#ifdef CONFIG_MM_SUBPAGE +static vm_fault_t filemap_map_user_pages(struct vm_fault *vmf, + pgoff_t first, pgoff_t last) +{ + struct vm_area_struct *vma = vmf->vma; + struct address_space *mapping = vma->vm_file->f_mapping; + XA_STATE(xas, &mapping->i_pages, first); + struct folio *folio; + vm_fault_t ret = 0; + unsigned short miss = 0, saved; + + rcu_read_lock(); + folio = next_uptodate_folio(&xas, mapping, last); + if (!folio) + goto out; + if (pmd_none(*vmf->pmd) && vmf->prealloc_pte) + pmd_install(vma->vm_mm, vmf->pmd, &vmf->prealloc_pte); + vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, + vmf->address, &vmf->ptl); + if (!vmf->pte) { + folio_unlock(folio); + folio_put(folio); + goto out; + } + do { + ret |= map_file_user_pages(vmf, folio, first, last, false); + if (!folio_test_workingset(folio)) + miss += min(folio_next_index(folio) - 1, last) - + max(folio->index, first) + 1; + folio_unlock(folio); + folio_put(folio); + } while ((folio = next_uptodate_folio(&xas, mapping, last)) != NULL); + pte_unmap_unlock(vmf->pte, vmf->ptl); + trace_mm_filemap_map_pages(mapping, first, last); +out: + rcu_read_unlock(); + saved = READ_ONCE(vma->vm_file->f_ra.mmap_miss); + WRITE_ONCE(vma->vm_file->f_ra.mmap_miss, saved > miss ? saved - miss : 0); + return ret; +} +#endif + vm_fault_t filemap_map_pages(struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff) { @@ -3901,6 +3963,11 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, unsigned long rss = 0; unsigned int nr_pages = 0, folio_type; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return filemap_map_user_pages(vmf, start_pgoff, end_pgoff); +#endif + /* * Recalculate end_pgoff based on file_end before calling * next_uptodate_folio() to avoid races with concurrent diff --git a/mm/gup.c b/mm/gup.c index bde05664fe9a9..001e804f4e38c 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -1297,6 +1298,11 @@ static struct vm_area_struct *gup_vma_lookup(struct mm_struct *mm, #endif } +#ifdef CONFIG_MM_SUBPAGE +static long prefault_subpage_range(struct mm_struct *mm, unsigned long start, + unsigned long nr_pages, unsigned int flags, int *locked); +#endif + /** * __get_user_pages() - pin user pages in memory * @mm: mm_struct of target mm @@ -1363,6 +1369,15 @@ static long __get_user_pages(struct mm_struct *mm, if (!nr_pages) return 0; +#ifdef CONFIG_MM_SUBPAGE + /* A prefault has no page-array ownership or physical-offset ambiguity. */ + if (mm_page_size(mm) != PAGE_SIZE) { + if (pages || (gup_flags & (FOLL_GET | FOLL_PIN))) + return -EOPNOTSUPP; + return prefault_subpage_range(mm, start, nr_pages, gup_flags, locked); + } +#endif + start = untagged_addr_remote(mm, start); VM_WARN_ON_ONCE(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN))); @@ -1721,7 +1736,7 @@ static __always_inline long __get_user_pages_locked(struct mm_struct *mm, */ if (likely(pages)) pages += ret; - start += ret << PAGE_SHIFT; + start += ret << mm_page_shift(mm); /* The lock was temporarily dropped, so we must unlock later */ must_unlock = true; @@ -1768,7 +1783,7 @@ static __always_inline long __get_user_pages_locked(struct mm_struct *mm, break; if (likely(pages)) pages++; - start += PAGE_SIZE; + start += mm_page_size(mm); } if (must_unlock && *locked) { /* @@ -1814,13 +1829,13 @@ long populate_vma_page_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, int *locked) { struct mm_struct *mm = vma->vm_mm; - unsigned long nr_pages = (end - start) / PAGE_SIZE; + unsigned long nr_pages = (end - start) / mm_page_size(mm); int local_locked = 1; int gup_flags; long ret; - VM_WARN_ON_ONCE(!PAGE_ALIGNED(start)); - VM_WARN_ON_ONCE(!PAGE_ALIGNED(end)); + VM_WARN_ON_ONCE(!IS_ALIGNED(start, mm_page_size(mm))); + VM_WARN_ON_ONCE(!IS_ALIGNED(end, mm_page_size(mm))); VM_WARN_ON_ONCE_VMA(start < vma->vm_start, vma); VM_WARN_ON_ONCE_VMA(end > vma->vm_end, vma); mmap_assert_locked(mm); @@ -1887,12 +1902,12 @@ long populate_vma_page_range(struct vm_area_struct *vma, long faultin_page_range(struct mm_struct *mm, unsigned long start, unsigned long end, bool write, int *locked) { - unsigned long nr_pages = (end - start) / PAGE_SIZE; + unsigned long nr_pages = (end - start) / mm_page_size(mm); int gup_flags; long ret; - VM_WARN_ON_ONCE(!PAGE_ALIGNED(start)); - VM_WARN_ON_ONCE(!PAGE_ALIGNED(end)); + VM_WARN_ON_ONCE(!IS_ALIGNED(start, mm_page_size(mm))); + VM_WARN_ON_ONCE(!IS_ALIGNED(end, mm_page_size(mm))); mmap_assert_locked(mm); /* @@ -1968,7 +1983,7 @@ int __mm_populate(unsigned long start, unsigned long len, int ignore_errors) } break; } - nend = nstart + ret * PAGE_SIZE; + nend = nstart + ret * mm_page_size(mm); ret = 0; } if (locked) @@ -2047,6 +2062,7 @@ size_t fault_in_writeable(char __user *uaddr, size_t size) { const unsigned long start = (unsigned long)uaddr; const unsigned long end = start + size; + const unsigned long page_size = current->mm ? mm_page_size(current->mm) : PAGE_SIZE; unsigned long cur; if (unlikely(size == 0)) @@ -2055,7 +2071,7 @@ size_t fault_in_writeable(char __user *uaddr, size_t size) return size; /* Stop once we overflow to 0. */ - for (cur = start; cur && cur < end; cur = PAGE_ALIGN_DOWN(cur + PAGE_SIZE)) + for (cur = start; cur && cur < end; cur = round_down(cur + page_size, page_size)) unsafe_put_user(0, (char __user *)cur, out); out: user_write_access_end(); @@ -2116,6 +2132,7 @@ size_t fault_in_safe_writeable(const char __user *uaddr, size_t size) { const unsigned long start = (unsigned long)uaddr; const unsigned long end = start + size; + const unsigned long page_size = current->mm ? mm_page_size(current->mm) : PAGE_SIZE; unsigned long cur; struct mm_struct *mm = current->mm; bool unlocked = false; @@ -2125,7 +2142,7 @@ size_t fault_in_safe_writeable(const char __user *uaddr, size_t size) mmap_read_lock(mm); /* Stop once we overflow to 0. */ - for (cur = start; cur && cur < end; cur = PAGE_ALIGN_DOWN(cur + PAGE_SIZE)) + for (cur = start; cur && cur < end; cur = round_down(cur + page_size, page_size)) if (fixup_user_fault(mm, cur, FAULT_FLAG_WRITE, &unlocked)) break; mmap_read_unlock(mm); @@ -2148,6 +2165,7 @@ size_t fault_in_readable(const char __user *uaddr, size_t size) { const unsigned long start = (unsigned long)uaddr; const unsigned long end = start + size; + const unsigned long page_size = current->mm ? mm_page_size(current->mm) : PAGE_SIZE; unsigned long cur; volatile char c; @@ -2157,7 +2175,7 @@ size_t fault_in_readable(const char __user *uaddr, size_t size) return size; /* Stop once we overflow to 0. */ - for (cur = start; cur && cur < end; cur = PAGE_ALIGN_DOWN(cur + PAGE_SIZE)) + for (cur = start; cur && cur < end; cur = round_down(cur + page_size, page_size)) unsafe_get_user(c, (const char __user *)cur, out); out: user_read_access_end(); @@ -2211,11 +2229,18 @@ struct pages_or_folios { void **entries; }; bool has_folios; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment *fragments; +#endif long nr_entries; }; static struct folio *pofs_get_folio(struct pages_or_folios *pofs, long i) { +#ifdef CONFIG_MM_SUBPAGE + if (pofs->fragments) + return pofs->fragments[i].folio; +#endif if (pofs->has_folios) return pofs->folios[i]; return page_folio(pofs->pages[i]); @@ -2223,22 +2248,56 @@ static struct folio *pofs_get_folio(struct pages_or_folios *pofs, long i) static void pofs_clear_entry(struct pages_or_folios *pofs, long i) { +#ifdef CONFIG_MM_SUBPAGE + if (pofs->fragments) { + pofs->fragments[i] = (struct user_page_fragment) {}; + return; + } +#endif pofs->entries[i] = NULL; } static void pofs_unpin(struct pages_or_folios *pofs) { +#ifdef CONFIG_MM_SUBPAGE + if (pofs->fragments) { + release_user_fragments(pofs->fragments, pofs->nr_entries, false); + return; + } +#endif if (pofs->has_folios) unpin_folios(pofs->folios, pofs->nr_entries); else unpin_user_pages(pofs->pages, pofs->nr_entries); } +static void pofs_unpin_entry(struct pages_or_folios *pofs, long i) +{ +#ifdef CONFIG_MM_SUBPAGE + if (pofs->fragments) { + release_user_fragments(&pofs->fragments[i], 1, false); + return; + } +#endif + unpin_folio(pofs_get_folio(pofs, i)); +} + static struct folio *pofs_next_folio(struct folio *folio, struct pages_or_folios *pofs, long *index_ptr) { long i = *index_ptr + 1; +#ifdef CONFIG_MM_SUBPAGE + /* Several adjacent user quarters can pin the same native folio. */ + if (pofs->fragments) { + while (i < pofs->nr_entries && pofs->fragments[i].folio == folio) + i++; + if (i == pofs->nr_entries) + return NULL; + *index_ptr = i; + return pofs->fragments[i].folio; + } +#endif if (!pofs->has_folios && folio_test_large(folio)) { const unsigned long start_pfn = folio_pfn(folio); const unsigned long end_pfn = start_pfn + folio_nr_pages(folio); @@ -2339,9 +2398,9 @@ migrate_longterm_unpinnable_folios(struct list_head *movable_folio_list, * convert the pin on the source folio to a normal * reference. */ - pofs_clear_entry(pofs, i); folio_get(folio); - gup_put_folio(folio, 1, FOLL_PIN); + pofs_unpin_entry(pofs, i); + pofs_clear_entry(pofs, i); if (migrate_device_coherent_folio(folio)) { ret = -EBUSY; @@ -2358,7 +2417,7 @@ migrate_longterm_unpinnable_folios(struct list_head *movable_folio_list, * calling folio_isolate_lru() which takes a reference so the * folio won't be freed if it's migrating. */ - unpin_folio(folio); + pofs_unpin_entry(pofs, i); pofs_clear_entry(pofs, i); } @@ -2464,6 +2523,23 @@ static long check_and_migrate_movable_folios(unsigned long nr_folios, } #endif /* CONFIG_MIGRATION */ +#ifdef CONFIG_MM_SUBPAGE +static long check_and_migrate_movable_fragments(unsigned long count, + struct user_page_fragment *fragments) +{ +#ifdef CONFIG_MIGRATION + struct pages_or_folios pofs = { + .fragments = fragments, + .nr_entries = count, + }; + + return check_and_migrate_movable_pages_or_folios(&pofs); +#else + return 0; +#endif +} +#endif + /* * __gup_longterm_locked() is a wrapper for __get_user_pages_locked which * allows us to process the FOLL_LONGTERM flag. @@ -3139,6 +3215,11 @@ static unsigned long gup_fast(unsigned long start, unsigned long end, int nr_pinned = 0; unsigned seq; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return 0; +#endif + if (!IS_ENABLED(CONFIG_HAVE_GUP_FAST) || !gup_fast_permitted(start, end)) return 0; @@ -3361,6 +3442,464 @@ long pin_user_pages_remote(struct mm_struct *mm, } EXPORT_SYMBOL(pin_user_pages_remote); +#ifdef CONFIG_MM_SUBPAGE +/* 1: acquired, 0: fault required, -EMLINK: read pin needs private backing. */ +static int follow_subpage_fragment(struct vm_area_struct *vma, unsigned long addr, + unsigned int flags, struct user_page_fragment *fragment) +{ + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *slot = NULL; + struct folio *folio; + struct page *page; + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *ptep, pte; + spinlock_t *ptl; + phys_addr_t phys; + int ret = 0; + bool file_ref = false; + + if (pgd_none(*pgd) || pgd_bad(*pgd)) + return PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + p4d = p4d_offset_mm(mm, pgd, addr); + if (p4d_none_mm(mm, *p4d) || p4d_bad_mm(mm, *p4d)) + return PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + pud = pud_offset_mm(mm, p4d, addr); + if (pud_none(*pud)) + return PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + if (pud_leaf(*pud) || pud_bad(*pud)) + return -EOPNOTSUPP; + pmd = pmd_offset_mm(mm, pud, addr); + if (pmd_none(*pmd)) + return PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + if (pmd_leaf(*pmd) || pmd_bad(*pmd)) + return -EOPNOTSUPP; + ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!ptep) + return 0; + pte = ptep_get(ptep); + if (pte_none(pte)) { + ret = PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + goto out; + } + if (!pte_present(pte) || (pte_protnone(pte) && !gup_can_follow_protnone(vma, flags))) + goto out; + phys = pte_phys_mm(mm, pte); + page = vm_normal_page(vma, addr, pte); + if (mm_page_size(mm) > PAGE_SIZE) { + unsigned long offset = (addr & (mm_page_size(mm) - 1)) & PAGE_MASK; + + phys += offset; + if (page) + page += offset >> PAGE_SHIFT; + } + if (!page) { + if (flags & FOLL_DUMP) { + ret = -EFAULT; + goto out; + } + if (!is_zero_pfn(pte_pfn(pte))) { + ret = -EOPNOTSUPP; + goto out; + } + if (flags & FOLL_WRITE) + goto out; + folio = page_folio(pfn_to_page(PHYS_PFN(phys))); + if (flags & FOLL_GET) + folio_get(folio); + } else { + folio = page_folio(page); + /* Coarse leaves use ordinary native-page ownership, not slots. */ + if (mm_page_size(mm) > PAGE_SIZE) { + if ((flags & FOLL_WRITE) && + !can_follow_write_pte(pte, page, vma, flags)) + goto out; + if (!pte_write(pte) && gup_must_unshare(vma, flags, page)) { + ret = -EMLINK; + goto out; + } + ret = try_grab_folio(folio, 1, flags); + if (ret) + goto out; + file_ref = true; + goto acquired; + } + if (!folio_test_anon(folio)) { + if (vma->vm_file && folio_mapping(folio) != vma->vm_file->f_mapping) { + ret = -EOPNOTSUPP; + goto out; + } + if ((flags & FOLL_WRITE) && !pte_write(pte)) + goto out; + /* Private non-anonymous read pins must first acquire COW ownership. */ + if ((flags & FOLL_PIN) && is_cow_mapping(vma->vm_flags)) { + ret = -EMLINK; + goto out; + } + ret = try_grab_folio(folio, 1, flags); + if (ret) + goto out; + file_ref = true; + goto acquired; + } + if (folio_test_ksm(folio)) { + ret = -EOPNOTSUPP; + goto out; + } + slot = mm_subpage_get_from_phys(phys); + if (!slot) { + ret = -EFAULT; + goto out; + } + if ((flags & FOLL_WRITE) && !pte_write(pte)) { + bool force = (flags & FOLL_FORCE) && + !(vma->vm_flags & (VM_SHARED | VM_MAYSHARE | VM_WRITE)) && + (vma->vm_flags & VM_MAYWRITE) && mm_subpage_is_exclusive(slot) && + !pte_needs_soft_dirty_wp(vma, pte) && !userfaultfd_pte_wp(vma, pte); + + if (!force) + goto out; + } + if ((flags & FOLL_PIN) && !mm_subpage_is_exclusive(slot)) { + ret = flags & FOLL_WRITE ? 0 : -EMLINK; + goto out; + } + if (flags & FOLL_PIN) { + ret = mm_subpage_pin(slot); + if (ret) { + if (ret == -EAGAIN) + ret = 0; + goto out; + } + } + } +acquired: + if (slot && (flags & FOLL_WRITE)) + mm_subpage_cancel_lazyfree(slot); + ret = flags & FOLL_PIN ? arch_make_folio_accessible(folio) : 0; + if (ret) { + if (slot) + mm_subpage_unpin(slot); + else if (file_ref) + gup_put_folio(folio, 1, flags); + goto out; + } + if ((flags & FOLL_TOUCH) && (flags & FOLL_WRITE) && + !pte_dirty(pte) && !folio_test_dirty(folio)) + folio_mark_dirty(folio); + folio_mark_accessed(folio); + flush_anon_page(vma, &folio->page, addr); + flush_dcache_folio(folio); + fragment->folio = folio; + fragment->subpage = slot; + fragment->pinned = flags & FOLL_PIN; + if (flags & FOLL_GET) + slot = NULL; /* The returned record consumes the lookup reference. */ + fragment->offset = phys - PFN_PHYS(folio_pfn(folio)) + + (addr & (mm_user_fragment_size(mm) - 1)); + ret = 1; +out: + if (slot) + mm_subpage_put(slot); /* Lookup reference; success retains a separate pin. */ + pte_unmap_unlock(ptep, ptl); + return ret; +} + +/* No references escape: only ensure each user leaf can be followed. */ +static long prefault_subpage_range(struct mm_struct *mm, unsigned long start, + unsigned long nr_pages, unsigned int flags, int *locked) +{ + unsigned long done = 0; + long ret = 0; + + mmap_assert_locked(mm); + start = untagged_addr_remote(mm, start); + while (done < nr_pages) { + struct user_page_fragment unused; + struct vm_area_struct *vma = vma_lookup(mm, start); + + if (!vma) { + ret = flags & FOLL_MADV_POPULATE ? -ENOMEM : -EFAULT; + break; + } + ret = check_vma_flags(vma, flags); + if (ret) { + if (flags & FOLL_MADV_POPULATE) + ret = -EINVAL; + break; + } + if (is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { + ret = flags & FOLL_MADV_POPULATE ? -EINVAL : -EOPNOTSUPP; + break; + } + if (gup_signal_pending(flags)) { + ret = -EINTR; + break; + } + cond_resched(); + ret = follow_subpage_fragment(vma, start, flags, &unused); + if (!ret) { + ret = faultin_page(vma, start, flags, false, locked); + if (!ret) + continue; + if (ret == -EBUSY || ret == -EAGAIN) + ret = 0; /* Preserve GUP's caller-managed retry contract. */ + break; + } + if (ret < 0) + break; + done++; + start += mm_page_size(mm); + } + if (ret == -EOPNOTSUPP && (flags & FOLL_MADV_POPULATE)) + ret = -EINVAL; + return done ? (long)done : ret; +} + +static long user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity, bool pin, + struct vm_area_struct **vma_out, bool dump) +{ + const unsigned int allowed = FOLL_WRITE | FOLL_FORCE | FOLL_NOFAULT | + FOLL_NOWAIT | FOLL_INTERRUPTIBLE | FOLL_LONGTERM | FOLL_ANON; + unsigned int internal = flags | FOLL_TOUCH | FOLL_REMOTE | + (pin ? FOLL_PIN : FOLL_GET) | (dump ? FOLL_DUMP : 0); + unsigned long end, count = 0; + int locked = 0; + long ret; + + if (!length) + return 0; + if ((flags & FOLL_LONGTERM) && !pin) + return -EINVAL; + if (!fragments || !capacity || capacity > LONG_MAX || (flags & ~allowed)) + return -EINVAL; + if (vma_out) { + mmap_assert_locked(mm); + *vma_out = NULL; + } else { + internal |= FOLL_UNLOCKABLE; + ret = mmap_read_lock_killable(mm); + if (ret) + return ret; + } + locked = 1; + start = untagged_addr_remote(mm, start); + if (check_add_overflow(start, length, &end)) { + ret = -EFAULT; + goto out; + } + if (pin) + mm_set_has_pinned_flag(mm); + while (start < end && count < capacity) { + unsigned long bytes = min(end - start, + mm_user_fragment_size(mm) - (start & (mm_user_fragment_size(mm) - 1))); + struct vm_area_struct *vma; + + if (!locked) { + ret = mmap_read_lock_killable(mm); + if (ret) + break; + locked = 1; + } + if (gup_signal_pending(internal)) { + ret = -EINTR; + break; + } + cond_resched(); + if (mm_page_size(mm) == PAGE_SIZE) { + struct page *page; + + if (dump) + ret = __get_user_pages_locked(mm, start, 1, &page, &locked, internal); + else if (pin) + ret = pin_user_pages_remote(mm, start, 1, flags, &page, + vma_out ? NULL : &locked); + else + ret = get_user_pages_remote(mm, start, 1, flags, &page, + vma_out ? NULL : &locked); + if (ret != 1) { + if (!ret) + ret = -EAGAIN; + break; + } + fragments[count].folio = page_folio(page); + fragments[count].subpage = NULL; + fragments[count].pinned = pin; + fragments[count].offset = folio_page_idx(page_folio(page), page) * PAGE_SIZE + + offset_in_page(start); + } else { + vma = gup_vma_lookup(mm, start); + ret = vma ? check_vma_flags(vma, internal) : -EFAULT; + if (ret) + break; + /* Huge and special/device mappings need separate integration. */ + if (is_vm_hugetlb_page(vma) || + (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP | VM_IO))) { + ret = -EOPNOTSUPP; + break; + } + ret = follow_subpage_fragment(vma, start, internal, &fragments[count]); + if (!ret || ret == -EMLINK) { + ret = faultin_page(vma, start, internal, ret == -EMLINK, &locked); + if (!ret) + continue; + if (!locked && (ret == -EBUSY || ret == -EAGAIN)) { + internal |= FOLL_TRIED; + continue; + } + if (ret == -EBUSY) + ret = -EAGAIN; + break; + } + if (ret < 0) + break; + } + if (vma_out) + *vma_out = vma_lookup(mm, start); + fragments[count++].length = bytes; + start += bytes; + internal &= ~FOLL_TRIED; + } +out: + if (locked && !vma_out) + mmap_read_unlock(mm); + return count ? (long)count : ret; +} + +long pin_user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity) +{ + unsigned int saved; + long ret, count; + + if (!(flags & FOLL_LONGTERM) || mm_page_size(mm) == PAGE_SIZE) + return user_fragments_remote(mm, start, length, flags, fragments, + capacity, true, NULL, false); + /* Match native long-term GUP: fault outside movable zones, migrate any + * existing unpinnable backing, then retry with fresh typed references. + */ + saved = memalloc_pin_save(); + do { + count = user_fragments_remote(mm, start, length, flags, fragments, + capacity, true, NULL, false); + if (count <= 0) { + ret = count; + break; + } + ret = check_and_migrate_movable_fragments(count, fragments); + } while (ret == -EAGAIN); + memalloc_pin_restore(saved); + return ret ? ret : count; +} +EXPORT_SYMBOL_GPL(pin_user_fragments_remote); + +long get_user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity) +{ + return user_fragments_remote(mm, start, length, flags, fragments, capacity, + false, NULL, false); +} +EXPORT_SYMBOL_GPL(get_user_fragments_remote); + +int get_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragment, struct vm_area_struct **vma) +{ + int ret; + + /* As with get_user_page_vma_remote(), preserve the caller's VMA lock. */ + if (!vma) + return -EINVAL; + *vma = NULL; + if (flags & FOLL_NOWAIT) + return -EINVAL; + ret = user_fragments_remote(mm, start, length, flags, fragment, 1, false, vma, false); + if (ret > 0 && !*vma) { + release_user_fragments(fragment, 1, false); + return -EFAULT; + } + return ret; +} + +int pin_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragment, struct vm_area_struct **vma) +{ + int ret; + + if (!vma) + return -EINVAL; + *vma = NULL; + if (flags & (FOLL_NOWAIT | FOLL_LONGTERM)) + return -EINVAL; + ret = user_fragments_remote(mm, start, length, flags, fragment, 1, true, vma, false); + if (ret > 0 && !*vma) { + release_user_fragments(fragment, 1, false); + return -EFAULT; + } + return ret; +} + +#ifdef CONFIG_ELF_CORE +/* Like get_dump_page(), but preserves the selected user-page ownership. */ +int get_dump_fragment(unsigned long addr, struct user_page_fragment *fragment) +{ + return user_fragments_remote(current->mm, addr, mm_user_fragment_size(current->mm), + FOLL_FORCE, fragment, 1, false, NULL, true); +} +#endif + +void mark_user_fragment_dirty(struct user_page_fragment *fragment) +{ + struct folio *folio = fragment->folio; + + if (!folio) + return; + if (!is_zero_folio(folio) && !folio_test_dirty(folio)) { + folio_lock(folio); + folio_mark_dirty(folio); + folio_unlock(folio); + } + if (fragment->subpage) + mm_subpage_cancel_lazyfree(fragment->subpage); +} +EXPORT_SYMBOL_GPL(mark_user_fragment_dirty); + +void release_user_fragments(struct user_page_fragment *fragments, + unsigned long count, bool dirty) +{ + unsigned long i; + + for (i = 0; i < count; i++) { + struct user_page_fragment *fragment = &fragments[i]; + struct folio *folio = fragment->folio; + + if (!folio) + continue; + if (dirty) + mark_user_fragment_dirty(fragment); + if (fragment->subpage) { + if (fragment->pinned) + mm_subpage_unpin(fragment->subpage); + else + mm_subpage_put(fragment->subpage); + } else if (fragment->pinned) { + unpin_folio(folio); + } else { + folio_put(folio); + } + *fragment = (struct user_page_fragment) {}; + } +} +EXPORT_SYMBOL_GPL(release_user_fragments); +#endif + /** * pin_user_pages() - pin user pages in memory for use by other devices * @@ -3413,6 +3952,25 @@ long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages, } EXPORT_SYMBOL(pin_user_pages_unlocked); +/* Direct cache pins have no PTE fault to publish fine shmem exposure. */ +static int memfd_pin_folio(struct file *memfd, struct folio *folio) +{ + bool lock = IS_ENABLED(CONFIG_MM_SUBPAGE) && + IS_ENABLED(CONFIG_USERFAULTFD) && shmem_file(memfd); + int ret; + + /* Serialize both acquisition and exposure against partial hole punch. */ + if (lock) + folio_lock(folio); + ret = try_grab_folio(folio, 1, FOLL_PIN); + if (lock) { + if (!ret) + shmem_uffd_mark_mapped(folio, 0, folio_size(folio)); + folio_unlock(folio); + } + return ret; +} + /** * memfd_pin_folios() - pin folios associated with a memfd * @memfd: the memfd whose folios are to be pinned @@ -3498,7 +4056,7 @@ long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end, for (i = 0; i < nr_found; i++) { folio = fbatch.folios[i]; - if (try_grab_folio(folio, 1, FOLL_PIN)) { + if (memfd_pin_folio(memfd, folio)) { folio_batch_release(&fbatch); ret = -EINVAL; goto err; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index c1b5092eb2a3c..e52239fa0aa22 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2725,7 +2725,8 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, goto unlock; if (!folio_can_map_prot_numa(pmd_folio(*pmd), vma, - vma_is_single_threaded_private(vma))) + vma_is_single_threaded_private(vma), + folio_maybe_mapped_shared(pmd_folio(*pmd)))) goto unlock; } /* @@ -4123,6 +4124,12 @@ static int __folio_split(struct folio *folio, unsigned int new_order, } anon_vma_lock_write(anon_vma); mapping = NULL; +#ifdef CONFIG_MM_SUBPAGE + if (new_order < anon_vma->min_folio_order) { + ret = -EBUSY; + goto out_unlock; + } +#endif } else { unsigned int min_order; gfp_t gfp; @@ -4402,7 +4409,7 @@ int folio_split(struct folio *folio, unsigned int new_order, unsigned int min_order_for_split(struct folio *folio) { if (folio_test_anon(folio)) - return 0; + return folio_anon_min_order(folio); /* * If the folio got truncated, we don't know the previous mapping and diff --git a/mm/internal.h b/mm/internal.h index 5acba3470659d..7e4c6d7bc9484 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -304,6 +304,50 @@ static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags) return pte_mkold(pte); } +#ifdef CONFIG_MM_SUBPAGE +/* Batch contiguous user leaves without confusing their offsets with PFNs. */ +static inline unsigned int folio_subpage_pte_batch_flags(struct vm_area_struct *vma, + struct folio *folio, pte_t *ptep, pte_t *ptentp, unsigned int max_nr, fpb_t flags) +{ + struct mm_struct *mm = vma->vm_mm; + pte_t pte = *ptentp; + phys_addr_t offset = pte_phys_mm(mm, pte) - PFN_PHYS(folio_pfn(folio)); + pte_t expected; + unsigned int nr; + + VM_BUG_ON_FOLIO(offset >= folio_size(folio), folio); + max_nr = min_t(unsigned long, max_nr, (folio_size(folio) - offset) >> mm_page_shift(mm)); + VM_BUG_ON_FOLIO(!max_nr, folio); + nr = min(max_nr, pte_batch_hint(ptep, pte)); + expected = pte_advance_pfn_mm(mm, __pte_batch_clear_ignored(pte, flags), nr); + while (nr < max_nr) { + unsigned int batch; + + pte = ptep_get(ptep + nr); + if (!pte_same(__pte_batch_clear_ignored(pte, flags), expected)) + break; + if ((flags & FPB_MERGE_WRITE) && pte_write(pte)) + *ptentp = pte_mkwrite(*ptentp, vma); + if (flags & FPB_MERGE_YOUNG_DIRTY) { + if (pte_young(pte)) + *ptentp = pte_mkyoung(*ptentp); + if (pte_dirty(pte)) + *ptentp = pte_mkdirty(*ptentp); + } + batch = min(max_nr - nr, pte_batch_hint(ptep + nr, pte)); + nr += batch; + expected = pte_advance_pfn_mm(mm, expected, batch); + } + return nr; +} + +static inline unsigned int folio_subpage_pte_batch(struct vm_area_struct *vma, + struct folio *folio, pte_t *ptep, pte_t pte, unsigned int max_nr, fpb_t flags) +{ + return folio_subpage_pte_batch_flags(vma, folio, ptep, &pte, max_nr, flags); +} +#endif + /** * folio_pte_batch_flags - detect a PTE batch for a large folio * @folio: The large folio to detect a PTE batch for. @@ -415,6 +459,10 @@ static inline pte_t pte_move_swp_offset(pte_t pte, long delta) if (pte_swp_uffd_wp(pte)) new = pte_swp_mkuffd_wp(new); +#ifdef CONFIG_MM_SUBPAGE + new = pte_swp_set_subpage_offset(new, pte_swp_subpage_offset(pte)); +#endif + return new; } @@ -1165,6 +1213,11 @@ folio_within_range(struct folio *folio, struct vm_area_struct *vma, end = vma->vm_end; pgoff = folio_pgoff(folio); + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) { + addr = vma_address_at_offset(vma, (struct vm_page_offset) { .index = pgoff }); + return !IS_ERR_VALUE(addr) && addr >= start && addr <= end && + end - addr >= folio_size(folio); + } /* if folio start address is not in vma range */ if (!in_range(pgoff, vma->vm_pgoff, vma_pglen)) @@ -1247,6 +1300,8 @@ static inline unsigned long vma_address(const struct vm_area_struct *vma, if (pgoff >= vma->vm_pgoff) { address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + if (pgoff != vma->vm_pgoff) + address -= vma_subpage_offset(vma); /* Check for address beyond vma (or wrapped through 0?) */ if (address < vma->vm_start || address >= vma->vm_end) address = -EFAULT; @@ -1269,15 +1324,24 @@ static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw) pgoff_t pgoff; unsigned long address; +#ifdef CONFIG_MM_SUBPAGE + if (pvmw->flags & PVMW_SUBPAGE) + return min(pvmw->address + mm_page_size(vma->vm_mm), vma->vm_end); +#endif + /* Common case, plus ->pgoff is invalid for KSM */ - if (pvmw->nr_pages == 1) - return pvmw->address + PAGE_SIZE; + if (pvmw->nr_pages == 1 && mm_page_size(vma->vm_mm) >= PAGE_SIZE) + return min((pvmw->address & mm_page_mask(vma->vm_mm)) + + mm_page_size(vma->vm_mm), vma->vm_end); pgoff = pvmw->pgoff + pvmw->nr_pages; address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + address -= vma_subpage_offset(vma); /* Check for address beyond vma (or wrapped through 0?) */ if (address < vma->vm_start || address > vma->vm_end) address = vma->vm_end; + if (mm_page_size(vma->vm_mm) > PAGE_SIZE) + address = min(mm_page_align(vma->vm_mm, address), vma->vm_end); return address; } @@ -1575,11 +1639,11 @@ static inline bool vma_is_single_threaded_private(struct vm_area_struct *vma) #ifdef CONFIG_NUMA_BALANCING bool folio_can_map_prot_numa(struct folio *folio, struct vm_area_struct *vma, - bool is_private_single_threaded); + bool is_private_single_threaded, bool shared); #else static inline bool folio_can_map_prot_numa(struct folio *folio, - struct vm_area_struct *vma, bool is_private_single_threaded) + struct vm_area_struct *vma, bool is_private_single_threaded, bool shared) { return false; } @@ -1907,7 +1971,7 @@ static inline bool clear_flush_young_ptes_notify(struct vm_area_struct *vma, young = clear_flush_young_ptes(vma, addr, ptep, nr); young |= mmu_notifier_clear_flush_young(vma->vm_mm, addr, - addr + nr * PAGE_SIZE); + addr + nr * mm_page_size(vma->vm_mm)); return young; } @@ -1927,7 +1991,8 @@ static inline bool test_and_clear_young_ptes_notify(struct vm_area_struct *vma, bool young; young = test_and_clear_young_ptes(vma, addr, ptep, nr); - young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + nr * PAGE_SIZE); + young |= mmu_notifier_clear_young(vma->vm_mm, addr, + addr + nr * mm_page_size(vma->vm_mm)); return young; } @@ -1959,4 +2024,14 @@ static inline int get_sysctl_max_map_count(void) bool may_expand_vm(struct mm_struct *mm, const vma_flags_t *vma_flags, unsigned long npages); +#ifdef CONFIG_MM_SUBPAGE +struct iov_iter; +int process_vm_rw_fragments(struct mm_struct *mm, unsigned long addr, + unsigned long len, struct iov_iter *iter, bool write); +#endif + +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SWAP) +vm_fault_t do_swap_coarse_page(struct vm_fault *vmf); +#endif + #endif /* __MM_INTERNAL_H */ diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 2d0aee0312f24..06100c59677bb 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -2234,14 +2234,16 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) * + unlock and free huge page; */ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, - struct file *file, pgoff_t start, struct collapse_control *cc) + struct file *file, pgoff_t start, struct collapse_control *cc, + unsigned int order) { struct address_space *mapping = file->f_mapping; struct page *dst; struct folio *folio, *tmp, *new_folio; - pgoff_t index = 0, end = start + HPAGE_PMD_NR; + unsigned long nr_pages = 1UL << order; + pgoff_t index = 0, end = start + nr_pages; LIST_HEAD(pagelist); - XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); + XA_STATE_ORDER(xas, &mapping->i_pages, start, order); enum scan_result result = SCAN_SUCCEED; int nr_none = 0; bool is_shmem = shmem_file(file); @@ -2253,9 +2255,18 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * mapping, the shmem check can be removed. */ VM_WARN_ON_ONCE(!is_shmem && !mapping_pmd_folio_support(mapping)); - VM_WARN_ON_ONCE(start & (HPAGE_PMD_NR - 1)); + VM_WARN_ON_ONCE(start & (nr_pages - 1)); - result = alloc_charge_folio(&new_folio, mm, cc, HPAGE_PMD_ORDER); + if (is_pmd_order(order)) { + result = alloc_charge_folio(&new_folio, mm, cc, order); + } else { + /* Sub-PMD user leaves must respect the shmem inode's allocation policy. */ + new_folio = shmem_alloc_collapse_folio(mm, mapping->host, start, order); + if (IS_ERR(new_folio)) { + new_folio = NULL; + result = SCAN_ALLOC_HUGE_PAGE_FAIL; + } + } if (result != SCAN_SUCCEED) goto out; @@ -2386,7 +2397,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * we locked the first folio, then a THP might be there already. * This will be discovered on the first iteration. */ - if (is_pmd_order(folio_order(folio))) { + if (folio_order(folio) >= order) { result = SCAN_PTE_MAPPED_HUGEPAGE; goto out_unlock; } @@ -2506,6 +2517,8 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, while (index < folio->index) { clear_highpage(dst); + if (is_shmem) + shmem_uffd_copy_page(dst, NULL); index++; dst++; } @@ -2515,12 +2528,16 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, result = SCAN_COPY_MC; goto rollback; } + if (is_shmem) + shmem_uffd_copy_page(dst, folio_page(folio, i)); index++; dst++; } } while (index < end) { clear_highpage(dst); + if (is_shmem) + shmem_uffd_copy_page(dst, NULL); index++; dst++; } @@ -2562,8 +2579,13 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * not be able to observe any missing pages due to the * previously inserted retry entries. */ + /* + * A required user leaf populates its whole granule, as a native + * fault does. Only optional PMD collapse must preserve cache holes + * merely because another VMA has missing-page registration. + */ vma_interval_tree_foreach(vma, &mapping->i_mmap, start, end) { - if (userfaultfd_missing(vma)) { + if (is_pmd_order(order) && userfaultfd_missing(vma)) { result = SCAN_EXCEED_NONE_PTE; goto immap_locked; } @@ -2586,12 +2608,13 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, } if (is_shmem) { - lruvec_stat_mod_folio(new_folio, NR_SHMEM, HPAGE_PMD_NR); - lruvec_stat_mod_folio(new_folio, NR_SHMEM_THPS, HPAGE_PMD_NR); - } else { - lruvec_stat_mod_folio(new_folio, NR_FILE_THPS, HPAGE_PMD_NR); + lruvec_stat_mod_folio(new_folio, NR_SHMEM, nr_pages); + if (is_pmd_order(order)) + lruvec_stat_mod_folio(new_folio, NR_SHMEM_THPS, nr_pages); + } else if (is_pmd_order(order)) { + lruvec_stat_mod_folio(new_folio, NR_FILE_THPS, nr_pages); } - lruvec_stat_mod_folio(new_folio, NR_FILE_PAGES, HPAGE_PMD_NR); + lruvec_stat_mod_folio(new_folio, NR_FILE_PAGES, nr_pages); /* * Mark new_folio as uptodate before inserting it into the @@ -2599,14 +2622,14 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * unwritten page. */ folio_mark_uptodate(new_folio); - folio_ref_add(new_folio, HPAGE_PMD_NR - 1); + folio_ref_add(new_folio, nr_pages - 1); if (is_shmem) folio_mark_dirty(new_folio); folio_add_lru(new_folio); /* Join all the small entries into a single multi-index entry. */ - xas_set_order(&xas, start, HPAGE_PMD_ORDER); + xas_set_order(&xas, start, order); xas_store(&xas, new_folio); WARN_ON_ONCE(xas_error(&xas)); xas_unlock_irq(&xas); @@ -2615,9 +2638,11 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * Remove pte page tables, so we can re-fault the page as huge. * If MADV_COLLAPSE, adjust result to call try_collapse_pte_mapped_thp(). */ - retract_page_tables(mapping, start); - if (cc && !cc->is_khugepaged) - result = SCAN_PTE_MAPPED_HUGEPAGE; + if (is_pmd_order(order)) { + retract_page_tables(mapping, start); + if (cc && !cc->is_khugepaged) + result = SCAN_PTE_MAPPED_HUGEPAGE; + } folio_unlock(new_folio); /* @@ -2661,7 +2686,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, folio_put(new_folio); out: VM_BUG_ON(!list_empty(&pagelist)); - trace_mm_khugepaged_collapse_file(mm, new_folio, index, addr, is_shmem, file, HPAGE_PMD_NR, result); + trace_mm_khugepaged_collapse_file(mm, new_folio, index, addr, is_shmem, file, nr_pages, result); return result; } @@ -2764,7 +2789,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm, result = SCAN_EXCEED_NONE_PTE; count_vm_event(THP_SCAN_EXCEED_NONE_PTE); } else { - result = collapse_file(mm, addr, file, start, cc); + result = collapse_file(mm, addr, file, start, cc, HPAGE_PMD_ORDER); } } @@ -3176,6 +3201,47 @@ static int madvise_collapse_errno(enum scan_result r) } } +/* + * Promote the cache backing for one larger-than-native user leaf. The caller + * owns mm/file references; no folio or page-table lock may be held. Start is + * a native file index. This sub-PMD shmem path may run under the fault lock: + * it neither retracts page tables (which requires mmap write locking) nor + * invokes filesystem writeback. Source folios are trylocked and rolled back + * on contention. Ordinary PMD collapse retains its existing lock policy. + */ +int collapse_file_user_page(struct mm_struct *mm, struct file *file, pgoff_t start) +{ + struct collapse_control *cc; + unsigned int order; + enum scan_result result; + + if (mm_page_shift(mm) <= PAGE_SHIFT || !shmem_file(file)) + return -EOPNOTSUPP; + order = mm_page_shift(mm) - PAGE_SHIFT; + /* Shmem's order limit reflects THP policy, not an architectural leaf limit. */ + if (order >= HPAGE_PMD_ORDER || order > MAX_PAGECACHE_ORDER || + !IS_ALIGNED(start, 1UL << order) || start > ULONG_MAX - (1UL << order)) + return -EINVAL; + cc = kzalloc(sizeof(*cc), GFP_KERNEL); + if (!cc) + return -ENOMEM; + lru_add_drain_all(); + result = collapse_file(mm, 0, file, start, cc, order); + kfree(cc); + if (result == SCAN_SUCCEED || result == SCAN_PTE_MAPPED_HUGEPAGE) + return 0; + /* Retained references cannot be displaced by replacing their cache folio. */ + if (result == SCAN_PAGE_COUNT) + return -EBUSY; + if (result == SCAN_CGROUP_CHARGE_FAIL) + return -ENOMEM; + if (result == SCAN_COPY_MC) + return -EHWPOISON; + if (result == SCAN_TRUNCATED) + return -EAGAIN; + return madvise_collapse_errno(result); +} + int madvise_collapse(struct vm_area_struct *vma, unsigned long start, unsigned long end, bool *lock_dropped) { diff --git a/mm/ksm.c b/mm/ksm.c index 7d5b76478f0b6..cd0398e972941 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -804,6 +804,10 @@ static struct page *get_mergeable_page(struct ksm_rmap_item *rmap_item) struct folio_walk fw; struct folio *folio; + /* The KSM comparison and replacement paths still require native leaves. */ + if (mm_page_size(mm) != PAGE_SIZE) + return NULL; + mmap_read_lock(mm); vma = find_mergeable_vma(mm, addr); if (!vma) @@ -1338,7 +1342,7 @@ static int write_protect_page(struct vm_area_struct *vma, struct folio *folio, /* See folio_try_share_anon_rmap_pte(): clear PTE first. */ if (anon_exclusive && - folio_try_share_anon_rmap_pte(folio, &folio->page)) { + folio_try_share_anon_rmap_pte(folio, &folio->page, vma)) { set_pte_at(mm, pvmw.address, pvmw.pte, entry); goto out_unlock; } @@ -3206,7 +3210,8 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc) if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) continue; - if (!rwc->rmap_one(folio, vma, addr, rwc->arg)) { + if (!rwc->rmap_one(folio, vma, + (struct rmap_walk_range) { .address = addr }, rwc->arg)) { anon_vma_unlock_read(anon_vma); return; } diff --git a/mm/madvise.c b/mm/madvise.c index 65e40fc8699db..86210ebeb9fea 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -201,7 +202,7 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start, spinlock_t *ptl; unsigned long addr; - for (addr = start; addr < end; addr += PAGE_SIZE) { + for (addr = start; addr < end; addr += mm_page_size(vma->vm_mm)) { pte_t pte; softleaf_t entry; struct folio *folio; @@ -244,7 +245,7 @@ static void shmem_swapin_range(struct vm_area_struct *vma, struct address_space *mapping) { XA_STATE(xas, &mapping->i_pages, linear_page_index(vma, start)); - pgoff_t end_index = linear_page_index(vma, end) - 1; + pgoff_t end_index = linear_page_index(vma, end - 1); struct folio *folio; struct swap_iocb *splug = NULL; @@ -260,8 +261,11 @@ static void shmem_swapin_range(struct vm_area_struct *vma, if (!softleaf_is_swap(entry)) continue; - addr = vma->vm_start + - ((xas.xa_index - vma->vm_pgoff) << PAGE_SHIFT); + /* The first cache page may start before this VMA's byte offset. */ + addr = xas.xa_index == linear_page_index(vma, start) ? start : + vma_address_at_offset(vma, (struct vm_page_offset) { + .index = xas.xa_index, + }); xas_pause(&xas); rcu_read_unlock(); @@ -326,7 +330,7 @@ static long madvise_willneed(struct madvise_behavior *madv_behavior) mark_mmap_lock_dropped(madv_behavior); get_file(file); offset = (loff_t)(start - vma->vm_start) - + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + + ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); mmap_read_unlock(mm); vfs_fadvise(file, offset, end - start, POSIX_FADV_WILLNEED); fput(file); @@ -348,12 +352,18 @@ static inline bool can_do_file_pageout(struct vm_area_struct *vma) file_permission(vma->vm_file, MAY_WRITE) == 0; } -static inline int madvise_folio_pte_batch(unsigned long addr, unsigned long end, +static inline int madvise_folio_pte_batch(struct vm_area_struct *vma, + unsigned long addr, unsigned long end, struct folio *folio, pte_t *ptep, pte_t *ptentp) { - int max_nr = (end - addr) / PAGE_SIZE; + int max_nr = (end - addr) >> mm_page_shift(vma->vm_mm); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return folio_subpage_pte_batch_flags(vma, folio, ptep, ptentp, + max_nr, FPB_MERGE_YOUNG_DIRTY); +#endif return folio_pte_batch_flags(folio, NULL, ptep, ptentp, max_nr, FPB_MERGE_YOUNG_DIRTY); } @@ -384,7 +394,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, #ifdef CONFIG_TRANSPARENT_HUGEPAGE if (pmd_trans_huge(*pmd)) { pmd_t orig_pmd; - unsigned long next = pmd_addr_end(addr, end); + unsigned long next = pmd_addr_end_mm(mm, addr, end); tlb_change_page_size(tlb, HPAGE_PMD_SIZE); ptl = pmd_trans_huge_lock(pmd, vma); @@ -454,14 +464,14 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, regular_folio: #endif - tlb_change_page_size(tlb, PAGE_SIZE); + tlb_change_page_size(tlb, mm_page_size(mm)); restart: start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); if (!start_pte) return 0; flush_tlb_batched_pending(mm); lazy_mmu_mode_enable(); - for (; addr < end; pte += nr, addr += nr * PAGE_SIZE) { + for (; addr < end; pte += nr, addr += nr * mm_page_size(mm)) { nr = 1; ptent = ptep_get(pte); @@ -492,9 +502,10 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, * fail to split a folio, leave it in place and advance to the * next pte in the range. */ - if (folio_test_large(folio)) { - nr = madvise_folio_pte_batch(addr, end, folio, pte, &ptent); - if (nr < folio_nr_pages(folio)) { + if (folio_test_large(folio) || mm_page_size(mm) != PAGE_SIZE) { + nr = madvise_folio_pte_batch(vma, addr, end, folio, pte, &ptent); + if (folio_test_large(folio) && + nr < (folio_size(folio) >> mm_page_shift(mm))) { int err; if (folio_maybe_mapped_shared(folio)) @@ -529,7 +540,8 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, * number of pages, it must be exclusive. */ if (!folio_test_lru(folio) || - folio_mapcount(folio) != folio_nr_pages(folio)) + folio_mapcount(folio) != + (mm_page_size(mm) != PAGE_SIZE ? nr : folio_nr_pages(folio))) continue; if (pageout_anon_only_filter && !folio_test_anon(folio)) @@ -657,6 +669,72 @@ static long madvise_pageout(struct madvise_behavior *madv_behavior) return 0; } +#ifdef CONFIG_MM_SUBPAGE +static int madvise_free_subpage_range(pmd_t *pmd, unsigned long addr, + unsigned long end, struct mm_walk *walk) +{ + struct vm_area_struct *vma = walk->vma; + struct mm_struct *mm = vma->vm_mm; + struct mmu_gather *tlb = walk->private; + spinlock_t *ptl; + pte_t *base, *ptep; + long swapped = 0; + + base = ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!base) + return 0; + flush_tlb_batched_pending(mm); + for (; addr < end; addr += mm_page_size(mm), ptep++) { + pte_t pte = ptep_get(ptep); + struct mm_subpage *slot; + struct folio *folio; + struct page *page; + + if (!pte_present(pte)) { + softleaf_t entry = softleaf_from_pte(pte); + + if (softleaf_is_swap(entry)) { + swap_subpage_put(entry, + pte_swp_subpage_offset(pte), true); + pte_clear_not_present_full(mm, addr, ptep, tlb->fullmm); + swapped--; + } + continue; + } + page = vm_normal_page(vma, addr, pte); + if (!page) + continue; + folio = page_folio(page); + if (!folio_test_anon(folio) || !folio_trylock(folio)) + continue; + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + if (!slot) + goto unlock; + if (folio_test_swapcache(folio) && !folio_free_swap(folio)) + goto put; + if (!mm_subpage_mark_lazyfree(slot)) + goto put; + clear_young_dirty_ptes(vma, addr, ptep, 1, + CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY); + tlb_remove_tlb_entry(tlb, ptep, addr); + /* Partial hints must never clear a neighbor's dirty state. */ + if (mm_subpage_all_lazyfree(slot)) { + folio_clear_dirty(folio); + folio_mark_lazyfree(folio); + } +put: + mm_subpage_put(slot); +unlock: + folio_unlock(folio); + } + if (swapped) + add_mm_counter(mm, MM_SWAPENTS, swapped); + pte_unmap_unlock(base, ptl); + cond_resched(); + return 0; +} +#endif + static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, struct mm_walk *walk) @@ -672,6 +750,10 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, unsigned long next; int nr, max_nr; + #ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) != PAGE_SIZE) + return madvise_free_subpage_range(pmd, addr, end, walk); +#endif next = pmd_addr_end(addr, end); if (pmd_trans_huge(*pmd)) if (madvise_free_huge_pmd(tlb, vma, pmd, addr, next)) @@ -722,7 +804,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, * next pte in the range. */ if (folio_test_large(folio)) { - nr = madvise_folio_pte_batch(addr, end, folio, pte, &ptent); + nr = madvise_folio_pte_batch(vma, addr, end, folio, pte, &ptent); if (nr < folio_nr_pages(folio)) { int err; @@ -996,7 +1078,7 @@ static long madvise_populate(struct madvise_behavior *madv_behavior) return -ENOMEM; } } - start += pages * PAGE_SIZE; + start += pages * mm_page_size(mm); } return 0; } @@ -1030,7 +1112,7 @@ static long madvise_remove(struct madvise_behavior *madv_behavior) return -EACCES; offset = (loff_t)(start - vma->vm_start) - + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + + ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); /* * Filesystem's fallocate may need to take i_rwsem. We need to @@ -1190,7 +1272,7 @@ static long madvise_guard_install(struct madvise_behavior *madv_behavior) if (err == 0) { unsigned long nr_expected_pages = - PHYS_PFN(range->end - range->start); + (range->end - range->start) >> mm_page_shift(vma->vm_mm); VM_WARN_ON(nr_pages != nr_expected_pages); return 0; @@ -1843,18 +1925,21 @@ static void madvise_finish_tlb(struct madvise_behavior *madv_behavior) /** * check_input_range() - Check if the requested range is valid. + * @mm: Target mm; its user page size sets the alignment. * @start: Start address of madvise-requested address range. * @len_in: Length of madvise-requested address range. + * @behavior: Requested madvise behavior. * * Returns: 0 if the input range is valid, otherwise an error code. */ -static int check_input_range(unsigned long start, size_t len_in) +static int check_input_range(struct mm_struct *mm, unsigned long start, + size_t len_in, int behavior) { size_t len; - if (!PAGE_ALIGNED(start)) + if (!IS_ALIGNED(start, mm_page_size(mm))) return -EINVAL; - len = PAGE_ALIGN(len_in); + len = mm_page_align(mm, len_in); /* Check to see whether len was rounded up from small -ve to zero */ if (len_in && !len) @@ -1863,6 +1948,20 @@ static int check_input_range(unsigned long start, size_t len_in) if (start + len < start) return -EINVAL; +#ifdef CONFIG_MM_SUBPAGE + /* These engines still assume whole native folios/tables. */ + if (mm_page_size(mm) != PAGE_SIZE) { + switch (behavior) { + case MADV_MERGEABLE: + case MADV_UNMERGEABLE: + case MADV_HUGEPAGE: + case MADV_COLLAPSE: + case MADV_HWPOISON: + case MADV_SOFT_OFFLINE: + return -EOPNOTSUPP; + } + } +#endif return 0; } @@ -1906,7 +2005,7 @@ static int madvise_do_behavior(unsigned long start, size_t len_in, } range->start = get_untagged_addr(madv_behavior->mm, start); - range->end = range->start + PAGE_ALIGN(len_in); + range->end = range->start + mm_page_align(madv_behavior->mm, len_in); blk_start_plug(&plug); if (is_madvise_populate(madv_behavior)) @@ -2002,7 +2101,7 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh if (!madvise_behavior_valid(behavior)) return -EINVAL; - error = check_input_range(start, len_in); + error = check_input_range(mm, start, len_in, behavior); if (error || !len_in) return error; @@ -2047,7 +2146,7 @@ static ssize_t vector_madvise(struct mm_struct *mm, struct iov_iter *iter, size_t len_in = iter_iov_len(iter); int error; - error = check_input_range(start, len_in); + error = check_input_range(mm, start, len_in, behavior); if (error || !len_in) ret = error; else @@ -2183,9 +2282,9 @@ static int madvise_set_anon_name(struct mm_struct *mm, unsigned long start, .anon_name = anon_name, }; - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(mm)) return -EINVAL; - len = (len_in + ~PAGE_MASK) & PAGE_MASK; + len = mm_page_align(mm, len_in); /* Check to see whether len was rounded up from small -ve to zero */ if (len_in && !len) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 51508a55c4055..f138b6dfe6b03 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -67,6 +67,9 @@ #include "swap.h" #include "internal.h" +#ifdef CONFIG_MM_SUBPAGE +#include "user-subpage-internal.h" +#endif static int sysctl_memory_failure_early_kill __read_mostly; @@ -527,6 +530,37 @@ struct task_struct *task_early_kill(struct task_struct *tsk, int force_early) return find_early_kill_thread(tsk); } +#ifdef CONFIG_MM_SUBPAGE +struct subpage_collect_procs { + const struct page *page; + struct list_head *to_kill; + int force_early; +}; + +static bool collect_procs_subpage(struct folio *folio, struct vm_area_struct *vma, + struct rmap_walk_range range, void *arg) +{ + struct subpage_collect_procs *args = arg; + DEFINE_FOLIO_RMAP_WALK(walk, folio, vma, range, PVMW_SYNC); + struct task_struct *tsk; + unsigned long address; + + if (!page_vma_mapped_walk(&walk)) + return true; + address = walk.address; + page_vma_mapped_walk_done(&walk); + rcu_read_lock(); + for_each_process(tsk) { + struct task_struct *t = task_early_kill(tsk, args->force_early); + + if (t && t->mm == vma->vm_mm) + add_to_kill_anon_file(t, args->page, vma, args->to_kill, address); + } + rcu_read_unlock(); + return true; +} +#endif + /* * Collect processes when the error hit an anonymous page. */ @@ -537,6 +571,17 @@ static void collect_procs_anon(const struct folio *folio, struct task_struct *tsk; struct anon_vma *av; pgoff_t pgoff; +#ifdef CONFIG_MM_SUBPAGE + struct subpage_collect_procs args = { page, to_kill, force_early }; + struct rmap_walk_control rwc = { + .rmap_one = collect_procs_subpage, + .arg = &args, + }; + + if (!folio_test_large(folio) && + rmap_walk_subpages((struct folio *)folio, &rwc, false)) + return; +#endif av = folio_lock_anon_vma_read(folio, NULL); if (av == NULL) /* Not actually mapped anymore */ diff --git a/mm/memory.c b/mm/memory.c index a14e63e63e1bb..dd87a48f42397 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -87,8 +87,11 @@ #include #include "pgalloc-track.h" +#include + #include "internal.h" #include "swap.h" +#include "swap_table.h" #if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST) #warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid. @@ -191,26 +194,26 @@ static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud, unsigned long start; start = addr; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(tlb->mm, pud, addr); do { - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(tlb->mm, addr, end); if (pmd_none_or_clear_bad(pmd)) continue; free_pte_range(tlb, pmd, addr); } while (pmd++, addr = next, addr != end); - start &= PUD_MASK; + start &= pud_mask_mm(tlb->mm); if (start < floor) return; if (ceiling) { - ceiling &= PUD_MASK; + ceiling &= pud_mask_mm(tlb->mm); if (!ceiling) return; } if (end - 1 > ceiling - 1) return; - pmd = pmd_offset(pud, start); + pmd = pmd_offset_mm(tlb->mm, pud, start); pud_clear(pud); pmd_free_tlb(tlb, pmd, start); mm_dec_nr_pmds(tlb->mm); @@ -225,27 +228,27 @@ static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d, unsigned long start; start = addr; - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(tlb->mm, p4d, addr); do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(tlb->mm, addr, end); if (pud_none_or_clear_bad(pud)) continue; free_pmd_range(tlb, pud, addr, next, floor, ceiling); } while (pud++, addr = next, addr != end); - start &= P4D_MASK; + start &= p4d_mask_mm(tlb->mm); if (start < floor) return; if (ceiling) { - ceiling &= P4D_MASK; + ceiling &= p4d_mask_mm(tlb->mm); if (!ceiling) return; } if (end - 1 > ceiling - 1) return; - pud = pud_offset(p4d, start); - p4d_clear(p4d); + pud = pud_offset_mm(tlb->mm, p4d, start); + p4d_clear_mm(tlb->mm, p4d); pud_free_tlb(tlb, pud, start); mm_dec_nr_puds(tlb->mm); } @@ -259,26 +262,26 @@ static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd, unsigned long start; start = addr; - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(tlb->mm, pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(p4d)) + next = p4d_addr_end_mm(tlb->mm, addr, end); + if (p4d_none_or_clear_bad_mm(tlb->mm, p4d)) continue; free_pud_range(tlb, p4d, addr, next, floor, ceiling); } while (p4d++, addr = next, addr != end); - start &= PGDIR_MASK; + start &= pgd_mask_mm(tlb->mm); if (start < floor) return; if (ceiling) { - ceiling &= PGDIR_MASK; + ceiling &= pgd_mask_mm(tlb->mm); if (!ceiling) return; } if (end - 1 > ceiling - 1) return; - p4d = p4d_offset(pgd, start); + p4d = p4d_offset_mm(tlb->mm, pgd, start); pgd_clear(pgd); p4d_free_tlb(tlb, p4d, start); } @@ -320,7 +323,7 @@ void free_pgd_range(struct mmu_gather *tlb, * Wherever addr is brought up or ceiling brought down, we must * be careful to reject "the opposite 0" before it confuses the * subsequent tests. But what about where end is brought down - * by PMD_SIZE below? no, end can't go down to 0 there. + * by one PMD span below? no, end can't go down to 0 there. * * Whereas we round start (addr) and ceiling down, by different * masks at different levels, in order to test whether a table @@ -328,19 +331,19 @@ void free_pgd_range(struct mmu_gather *tlb, * bother to round floor or end up - the tests don't need that. */ - addr &= PMD_MASK; + addr &= pmd_mask_mm(tlb->mm); if (addr < floor) { - addr += PMD_SIZE; + addr += pmd_size_mm(tlb->mm); if (!addr) return; } if (ceiling) { - ceiling &= PMD_MASK; + ceiling &= pmd_mask_mm(tlb->mm); if (!ceiling) return; } if (end - 1 > ceiling - 1) - end -= PMD_SIZE; + end -= pmd_size_mm(tlb->mm); if (addr > end - 1) return; /* @@ -350,7 +353,7 @@ void free_pgd_range(struct mmu_gather *tlb, tlb_change_page_size(tlb, PAGE_SIZE); pgd = pgd_offset(tlb->mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(tlb->mm, addr, end); if (pgd_none_or_clear_bad(pgd)) continue; free_p4d_range(tlb, pgd, addr, next, floor, ceiling); @@ -539,16 +542,16 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add return; } - p4dp = p4d_offset(pgdp, addr); + p4dp = p4d_offset_mm(mm, pgdp, addr); p4d = p4dp_get(p4dp); p4dv = p4d_val(p4d); - if (!p4d_present(p4d) || p4d_leaf(p4d)) { + if (!p4d_present_mm(mm, p4d) || p4d_leaf(p4d)) { pr_alert("pgd:%08llx p4d:%08llx\n", pgdv, p4dv); return; } - pudp = pud_offset(p4dp, addr); + pudp = pud_offset_mm(mm, p4dp, addr); pud = pudp_get(pudp); pudv = pud_val(pud); @@ -557,7 +560,7 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add return; } - pmdp = pmd_offset(pudp, addr); + pmdp = pmd_offset_mm(mm, pudp, addr); pmd = pmdp_get(pmdp); pmdv = pmd_val(pmd); @@ -950,7 +953,15 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, struct page *page; if (likely(softleaf_is_swap(entry))) { - if (swap_dup_entry_direct(entry) < 0) + int err; + +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_mm) < PAGE_SIZE) + err = swap_subpage_dup(entry, pte_swp_subpage_offset(orig_pte)); + else +#endif + err = swap_dup_entries_direct(entry, mm_pte_native_pages(src_mm)); + if (err < 0) return -EIO; /* make sure dst_mm is on swapoff's mmlist. */ @@ -982,6 +993,10 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, entry = make_readable_migration_entry( swp_offset(entry)); pte = softleaf_to_pte(entry); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_mm) < PAGE_SIZE) + pte = pte_swp_set_subpage_offset(pte, pte_swp_subpage_offset(orig_pte)); +#endif if (pte_swp_soft_dirty(orig_pte)) pte = pte_swp_mksoft_dirty(pte); if (pte_swp_uffd_wp(orig_pte)) @@ -1076,10 +1091,14 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma * over and copy the page & arm it. */ - if (copy_mc_user_highpage(&new_folio->page, page, addr, src_vma)) - return -EHWPOISON; + for (unsigned int i = 0; i < mm_pte_native_pages(src_vma->vm_mm); i++) { + if (copy_mc_user_highpage(folio_page(new_folio, i), page + i, + addr + i * PAGE_SIZE, src_vma)) + return -EHWPOISON; + } *prealloc = NULL; + folio_ref_add(new_folio, mm_pte_native_pages(dst_vma->vm_mm) - 1); __folio_mark_uptodate(new_folio); folio_add_new_anon_rmap(new_folio, dst_vma, addr, RMAP_EXCLUSIVE); folio_add_lru_vma(new_folio, dst_vma); @@ -1095,6 +1114,55 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma return 0; } +#ifdef CONFIG_MM_SUBPAGE +/* One new native folio may publish several independently locked user leaves. */ +static void subpage_add_new_lru(struct folio *folio, struct vm_area_struct *vma, + unsigned int mapped) +{ + folio_add_lru_vma(folio, vma); + if ((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED) + while (--mapped) + mlock_folio(folio); +} + +/* The destination slot and its charge/metadata were allocated outside PTLs. */ +static int copy_pinned_subpage(struct vm_area_struct *dst_vma, pte_t *dst_pte, + pte_t source_pte, unsigned long addr, int *rss, + struct mm_subpage *source, struct mm_subpage **prealloc) +{ + struct mm_subpage *dest = *prealloc; + struct folio *new; + pte_t pte; + int ret; + + if (!dest) + return -EAGAIN; + new = mm_subpage_folio(dest); + ret = copy_user_subpage_range(&new->page, mm_subpage_offset(dest), + &mm_subpage_folio(source)->page, mm_subpage_offset(source), + mm_subpage_size(source)); + if (ret) + return ret; + /* Fresh, unmapped backing is private to this preallocation. */ + if (WARN_ON_ONCE(!folio_trylock(new))) + return -EBUSY; + mm_subpage_set_exclusive(dest); + ret = mm_subpage_add_new_anon_rmap(dest, dst_vma, addr); + if (!ret) { + subpage_add_new_lru(new, dst_vma, 1); + rss[MM_ANONPAGES]++; + pte = phys_pte_mm(dst_vma->vm_mm, mm_subpage_phys(dest), dst_vma->vm_page_prot); + pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma); + if (userfaultfd_pte_wp(dst_vma, source_pte)) + pte = pte_mkuffd_wp(pte); + set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); + *prealloc = NULL; + } + folio_unlock(new); + return ret; +} +#endif + static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte, pte_t pte, unsigned long addr, int nr) @@ -1128,18 +1196,46 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma, static inline int copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte, pte_t pte, unsigned long addr, - int max_nr, int *rss, struct folio **prealloc) + int max_nr, int *rss, struct folio **prealloc, + struct mm_subpage **subprealloc) { fpb_t flags = FPB_MERGE_WRITE; struct page *page; struct folio *folio; - int err, nr; + int err, nr, nr_native; + unsigned int span = mm_pte_native_pages(src_vma->vm_mm); page = vm_normal_page(src_vma, addr, pte); if (unlikely(!page)) goto copy_pte; folio = page_folio(page); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_vma->vm_mm) < PAGE_SIZE) { + if (folio_test_anon(folio)) { + struct mm_subpage *slot; + + slot = mm_subpage_get_from_phys(pte_phys_mm(src_vma->vm_mm, pte)); + if (WARN_ON_ONCE(!slot)) + return -EINVAL; + err = mm_subpage_dup_anon_rmap(slot, dst_vma, src_vma, addr); + if (err == -EBUSY) { + err = copy_pinned_subpage(dst_vma, dst_pte, pte, addr, + rss, slot, subprealloc); + mm_subpage_put(slot); + return err ? err : 1; + } + if (err) { + mm_subpage_put(slot); + return err; + } + /* The lookup reference becomes the child's PTE reference. */ + rss[MM_ANONPAGES]++; + goto copy_pte; + } + max_nr = 1; + } +#endif /* * If we likely have to copy, just don't bother with batching. Make @@ -1152,18 +1248,25 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma if (vma_soft_dirty_enabled(src_vma)) flags |= FPB_RESPECT_SOFT_DIRTY; - nr = folio_pte_batch_flags(folio, src_vma, src_pte, &pte, max_nr, flags); - folio_ref_add(folio, nr); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_vma->vm_mm) > PAGE_SIZE) + nr = folio_subpage_pte_batch_flags(src_vma, folio, src_pte, + &pte, max_nr, flags); + else +#endif + nr = folio_pte_batch_flags(folio, src_vma, src_pte, &pte, max_nr, flags); + nr_native = nr * span; + folio_ref_add(folio, nr_native); if (folio_test_anon(folio)) { if (unlikely(folio_try_dup_anon_rmap_ptes(folio, page, - nr, dst_vma, src_vma))) { - folio_ref_sub(folio, nr); + nr_native, dst_vma, src_vma))) { + folio_ref_sub(folio, nr_native); return -EAGAIN; } rss[MM_ANONPAGES] += nr; VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio); } else { - folio_dup_file_rmap_ptes(folio, page, nr, dst_vma); + folio_dup_file_rmap_ptes(folio, page, nr_native, dst_vma); rss[mm_counter_file(folio)] += nr; } __copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte, @@ -1171,7 +1274,7 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma return nr; } - folio_get(folio); + folio_ref_add(folio, span); if (folio_test_anon(folio)) { /* * If this page may have been pinned by the parent process, @@ -1181,7 +1284,7 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma */ if (unlikely(folio_try_dup_anon_rmap_pte(folio, page, dst_vma, src_vma))) { /* Page may be pinned, we have to copy. */ - folio_put(folio); + folio_put_refs(folio, span); err = copy_present_page(dst_vma, src_vma, dst_pte, src_pte, addr, rss, prealloc, page); return err ? err : 1; @@ -1189,6 +1292,7 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma rss[MM_ANONPAGES]++; VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio); } else { + mm_subpage_file_map_add(dst_vma, pte_phys_mm(src_vma->vm_mm, pte)); folio_dup_file_rmap_pte(folio, page, dst_vma); rss[mm_counter_file(folio)]++; } @@ -1202,11 +1306,12 @@ static inline struct folio *folio_prealloc(struct mm_struct *src_mm, struct vm_area_struct *vma, unsigned long addr, bool need_zero) { struct folio *new_folio; + unsigned int order = max_t(unsigned int, mm_page_shift(src_mm), PAGE_SHIFT) - PAGE_SHIFT; if (need_zero) - new_folio = vma_alloc_zeroed_movable_folio(vma, addr); + new_folio = vma_alloc_zeroed_movable_folio_order(vma, addr, order); else - new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, addr); + new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, order, vma, addr); if (!new_folio) return NULL; @@ -1220,6 +1325,34 @@ static inline struct folio *folio_prealloc(struct mm_struct *src_mm, return new_folio; } +#ifdef CONFIG_MM_SUBPAGE +static struct mm_subpage *anon_subpage_prealloc(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr) +{ + struct mm_subpage_pool *pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + struct mm_subpage *slot = ERR_PTR(-ENOMEM); + struct folio *folio; + int ret; + + if (!pool) + return slot; + folio = folio_prealloc(mm, vma, addr, false); + if (!folio) + goto out; + ret = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (ret) { + folio_put(folio); + slot = ERR_PTR(ret); + goto out; + } + slot = mm_subpage_alloc_at(pool, vma_page_offset_at(vma, addr).offset); +out: + mm_subpage_pool_close(pool); + mm_subpage_pool_put(pool); + return slot; +} +#endif + static int copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, @@ -1236,6 +1369,7 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, int rss[NR_MM_COUNTERS]; softleaf_t entry = softleaf_mk_none(); struct folio *prealloc = NULL; + struct mm_subpage *subprealloc = NULL; int nr; again: @@ -1316,16 +1450,23 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, WARN_ON_ONCE(ret != -ENOENT); } /* copy_present_ptes() will clear `*prealloc' if consumed */ - max_nr = (end - addr) / PAGE_SIZE; + max_nr = (end - addr) >> mm_page_shift(src_mm); ret = copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, - ptent, addr, max_nr, rss, &prealloc); + ptent, addr, max_nr, rss, &prealloc, &subprealloc); /* * If we need a pre-allocated page for this pte, drop the * locks, allocate, and try again. * If copy failed due to hwpoison in source page, break out. */ - if (unlikely(ret == -EAGAIN || ret == -EHWPOISON)) + if (unlikely(ret == -EAGAIN || ret == -EHWPOISON || + (IS_ENABLED(CONFIG_MM_SUBPAGE) && ret < 0))) break; +#ifdef CONFIG_MM_SUBPAGE + if (subprealloc) { + mm_subpage_put(subprealloc); + subprealloc = NULL; + } +#endif if (unlikely(prealloc)) { /* * pre-alloc page cannot be reused by next time so as @@ -1338,7 +1479,7 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, } nr = ret; progress += 8 * nr; - } while (dst_pte += nr, src_pte += nr, addr += PAGE_SIZE * nr, + } while (dst_pte += nr, src_pte += nr, addr += mm_page_size(src_mm) * nr, addr != end); lazy_mmu_mode_disable(); @@ -1357,10 +1498,22 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, } else if (ret == -EBUSY || unlikely(ret == -EHWPOISON)) { goto out; } else if (ret == -EAGAIN) { +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_mm) < PAGE_SIZE) { + subprealloc = anon_subpage_prealloc(src_mm, src_vma, addr); + if (IS_ERR(subprealloc)) + return PTR_ERR(subprealloc); + ret = 0; + goto again; + } +#endif prealloc = folio_prealloc(src_mm, src_vma, addr, false); if (!prealloc) return -ENOMEM; } else if (ret < 0) { + /* Subpage ownership checks can return additional fatal errors. */ + if (IS_ENABLED(CONFIG_MM_SUBPAGE)) + goto out; VM_WARN_ON_ONCE(1); } @@ -1370,6 +1523,10 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, if (addr != end) goto again; out: +#ifdef CONFIG_MM_SUBPAGE + if (subprealloc) + mm_subpage_put(subprealloc); +#endif if (unlikely(prealloc)) folio_put(prealloc); return ret; @@ -1388,9 +1545,9 @@ copy_pmd_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, dst_pmd = pmd_alloc(dst_mm, dst_pud, addr); if (!dst_pmd) return -ENOMEM; - src_pmd = pmd_offset(src_pud, addr); + src_pmd = pmd_offset_mm(src_vma->vm_mm, src_pud, addr); do { - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(src_vma->vm_mm, addr, end); if (pmd_is_huge(*src_pmd)) { int err; @@ -1425,9 +1582,9 @@ copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, dst_pud = pud_alloc(dst_mm, dst_p4d, addr); if (!dst_pud) return -ENOMEM; - src_pud = pud_offset(src_p4d, addr); + src_pud = pud_offset_mm(src_vma->vm_mm, src_p4d, addr); do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(src_vma->vm_mm, addr, end); if (pud_trans_huge(*src_pud)) { int err; @@ -1461,10 +1618,10 @@ copy_p4d_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr); if (!dst_p4d) return -ENOMEM; - src_p4d = p4d_offset(src_pgd, addr); + src_p4d = p4d_offset_mm(src_vma->vm_mm, src_pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(src_p4d)) + next = p4d_addr_end_mm(src_vma->vm_mm, addr, end); + if (p4d_none_or_clear_bad_mm(src_vma->vm_mm, src_p4d)) continue; if (copy_pud_range(dst_vma, src_vma, dst_p4d, src_p4d, addr, next)) @@ -1516,6 +1673,9 @@ copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) bool is_cow; int ret; + if (mm_page_shift(dst_mm) != mm_page_shift(src_mm)) + return -EINVAL; + if (!vma_needs_copy(dst_vma, src_vma)) return 0; @@ -1549,7 +1709,7 @@ copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) dst_pgd = pgd_offset(dst_mm, addr); src_pgd = pgd_offset(src_mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(src_vma->vm_mm, addr, end); if (pgd_none_or_clear_bad(src_pgd)) continue; if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd, @@ -1629,7 +1789,7 @@ zap_install_uffd_wp_if_needed(struct vm_area_struct *vma, if (--nr == 0) break; pte++; - addr += PAGE_SIZE; + addr += mm_page_size(vma->vm_mm); } return was_installed; @@ -1642,10 +1802,12 @@ static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb, bool *force_flush, bool *force_break, bool *any_skipped) { struct mm_struct *mm = tlb->mm; + unsigned int nr_native = nr * mm_pte_native_pages(mm); bool delay_rmap = false; if (!folio_test_anon(folio)) { ptent = get_and_clear_full_ptes(mm, addr, pte, nr, tlb->fullmm); + mm_subpage_file_map_del(vma, pte_phys_mm(mm, ptent)); if (pte_dirty(ptent)) { folio_mark_dirty(folio); if (tlb_delay_rmap(tlb)) { @@ -1669,16 +1831,44 @@ static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb, nr, details, ptent); if (!delay_rmap) { - folio_remove_rmap_ptes(folio, page, nr, vma); + folio_remove_rmap_ptes(folio, page, nr_native, vma); if (unlikely(folio_mapcount(folio) < 0)) print_bad_pte(vma, addr, ptent, page); } - if (unlikely(__tlb_remove_folio_pages(tlb, page, nr, delay_rmap))) { + /* Gather payloads count native references, not logical PTEs. */ + if (unlikely(__tlb_remove_folio_pages(tlb, page, nr_native, delay_rmap))) { + *force_flush = true; + *force_break = true; + } +} + +#ifdef CONFIG_MM_SUBPAGE +static void zap_present_subpage(struct mmu_gather *tlb, + struct vm_area_struct *vma, pte_t *pte, pte_t ptent, + unsigned long addr, int *rss, bool *force_flush, + bool *force_break, bool *any_skipped) +{ + struct mm_subpage *slot = mm_subpage_get_from_phys(pte_phys_mm(tlb->mm, ptent)); + + /* All small-granule anonymous PTEs must carry slot ownership. */ + if (WARN_ON_ONCE(!slot)) { + *any_skipped = true; + return; + } + ptep_get_and_clear_full(tlb->mm, addr, pte, tlb->fullmm); + arch_check_zapped_pte(vma, ptent); + tlb_remove_tlb_entry(tlb, pte, addr); + mm_subpage_remove_anon_rmap(slot, vma); + rss[MM_ANONPAGES]--; + /* Drop the lookup ref, then transfer the PTE ref to the TLB batch. */ + mm_subpage_put(slot); + if (__tlb_remove_subpage(tlb, slot)) { *force_flush = true; *force_break = true; } } +#endif /* * Zap or skip at least one present PTE, trying to batch-process subsequent @@ -1716,12 +1906,30 @@ static inline int zap_present_ptes(struct mmu_gather *tlb, return 1; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + if (folio_test_anon(folio)) { + zap_present_subpage(tlb, vma, pte, ptent, addr, rss, + force_flush, force_break, any_skipped); + return 1; + } + /* Native folio batching still expresses physical PAGE_SIZE steps. */ + max_nr = 1; + } +#endif + /* * Make sure that the common "small folio" case is as fast as possible * by keeping the batching logic separate. */ if (unlikely(folio_test_large(folio) && max_nr != 1)) { - nr = folio_pte_batch(folio, pte, ptent, max_nr); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) > PAGE_SIZE) + nr = folio_subpage_pte_batch(vma, folio, pte, ptent, + max_nr, 0); + else +#endif + nr = folio_pte_batch(folio, pte, ptent, max_nr); zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr, addr, details, rss, force_flush, force_break, any_skipped); @@ -1764,9 +1972,20 @@ static inline int zap_nonpresent_ptes(struct mmu_gather *tlb, if (!should_zap_cows(details)) return 1; - nr = swap_pte_batch(pte, max_nr, ptent); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + swap_subpage_put(entry, pte_swp_subpage_offset(ptent), true); + } else +#endif + { + unsigned int span = mm_pte_native_pages(vma->vm_mm); + + /* Adjacent coarse PTEs advance by a full group of swap slots. */ + if (span == 1) + nr = swap_pte_batch(pte, max_nr, ptent); + swap_put_entries_direct(entry, nr * span); + } rss[MM_SWAPENTS] -= nr; - swap_put_entries_direct(entry, nr); } else if (softleaf_is_migration(entry)) { struct folio *folio = softleaf_to_folio(entry); @@ -1811,7 +2030,7 @@ static inline int do_zap_pte_range(struct mmu_gather *tlb, bool *any_skipped) { pte_t ptent = ptep_get(pte); - int max_nr = (end - addr) / PAGE_SIZE; + int max_nr = (end - addr) >> mm_page_shift(vma->vm_mm); int nr = 0; /* Skip all consecutive none ptes */ @@ -1825,7 +2044,7 @@ static inline int do_zap_pte_range(struct mmu_gather *tlb, if (!max_nr) return nr; pte += nr; - addr += nr * PAGE_SIZE; + addr += nr * mm_page_size(vma->vm_mm); } if (pte_present(ptent)) @@ -1839,13 +2058,14 @@ static inline int do_zap_pte_range(struct mmu_gather *tlb, return nr; } -static bool pte_table_reclaim_possible(unsigned long start, unsigned long end, +static bool pte_table_reclaim_possible(struct mm_struct *mm, + unsigned long start, unsigned long end, struct zap_details *details) { if (!IS_ENABLED(CONFIG_PT_RECLAIM)) return false; /* Only zap if we are allowed to and cover the full page table. */ - return details && details->reclaim_pt && (end - start >= PMD_SIZE); + return details && details->reclaim_pt && (end - start >= pmd_size_mm(mm)); } static bool zap_empty_pte_table(struct mm_struct *mm, pmd_t *pmd, @@ -1877,7 +2097,7 @@ static bool zap_pte_table_if_empty(struct mm_struct *mm, pmd_t *pmd, if (ptl != pml) spin_lock_nested(ptl, SINGLE_DEPTH_NESTING); - for (i = 0, pte = start_pte; i < PTRS_PER_PTE; i++, pte++) { + for (i = 0, pte = start_pte; i < pte_table_bytes_mm(mm) / sizeof(pte_t); i++, pte++) { if (!pte_none(ptep_get(pte))) goto out_ptl; } @@ -1902,7 +2122,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, unsigned long addr, unsigned long end, struct zap_details *details) { - bool can_reclaim_pt = pte_table_reclaim_possible(addr, end, details); + bool can_reclaim_pt = pte_table_reclaim_possible(tlb->mm, addr, end, details); bool force_flush = false, force_break = false; struct mm_struct *mm = tlb->mm; int rss[NR_MM_COUNTERS]; @@ -1915,7 +2135,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, int nr; retry: - tlb_change_page_size(tlb, PAGE_SIZE); + tlb_change_page_size(tlb, mm_page_size(mm)); init_rss_vec(rss); start_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl); if (!pte) @@ -1936,11 +2156,11 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, if (any_skipped) can_reclaim_pt = false; if (unlikely(force_break)) { - addr += nr * PAGE_SIZE; + addr += nr * mm_page_size(vma->vm_mm); direct_reclaim = false; break; } - } while (pte += nr, addr += PAGE_SIZE * nr, addr != end); + } while (pte += nr, addr += mm_page_size(mm) * nr, addr != end); /* * Fast path: try to hold the pmd lock and unmap the PTE page. @@ -1997,9 +2217,9 @@ static inline unsigned long zap_pmd_range(struct mmu_gather *tlb, pmd_t *pmd; unsigned long next; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(tlb->mm, pud, addr); do { - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(tlb->mm, addr, end); if (pmd_is_huge(*pmd)) { if (next - addr != HPAGE_PMD_SIZE) __split_huge_pmd(vma, pmd, addr, false); @@ -2033,9 +2253,9 @@ static inline unsigned long zap_pud_range(struct mmu_gather *tlb, pud_t *pud; unsigned long next; - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(tlb->mm, p4d, addr); do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(tlb->mm, addr, end); if (pud_trans_huge(*pud)) { if (next - addr != HPAGE_PUD_SIZE) split_huge_pud(vma, pud, addr); @@ -2061,10 +2281,10 @@ static inline unsigned long zap_p4d_range(struct mmu_gather *tlb, p4d_t *p4d; unsigned long next; - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(tlb->mm, pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(p4d)) + next = p4d_addr_end_mm(tlb->mm, addr, end); + if (p4d_none_or_clear_bad_mm(tlb->mm, p4d)) continue; next = zap_pud_range(tlb, vma, p4d, addr, next, details); } while (p4d++, addr = next, addr != end); @@ -2103,7 +2323,7 @@ static void __zap_vma_range(struct mmu_gather *tlb, struct vm_area_struct *vma, tlb_start_vma(tlb, vma); pgd = pgd_offset(vma->vm_mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(tlb->mm, addr, end); if (pgd_none_or_clear_bad(pgd)) continue; next = zap_p4d_range(tlb, vma, pgd, addr, next, details); @@ -2350,7 +2570,7 @@ static int validate_page_before_insert(struct vm_area_struct *vma, static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, unsigned long addr, struct page *page, - pgprot_t prot, bool mkwrite) + pgprot_t prot, bool mkwrite, unsigned int offset) { struct folio *folio = page_folio(page); pte_t pteval = ptep_get(pte); @@ -2360,7 +2580,7 @@ static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, return -EBUSY; /* see insert_pfn(). */ - if (pte_pfn(pteval) != page_to_pfn(page)) { + if (pte_phys_mm(vma->vm_mm, pteval) != page_to_phys(page) + offset) { WARN_ON_ONCE(!is_zero_pfn(pte_pfn(pteval))); return -EFAULT; } @@ -2372,17 +2592,18 @@ static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, } /* Ok, finally just insert the thing.. */ - pteval = mk_pte(page, prot); + pteval = phys_pte_mm(vma->vm_mm, page_to_phys(page) + offset, prot); if (unlikely(is_zero_folio(folio))) { pteval = pte_mkspecial(pteval); } else { folio_get(folio); - pteval = mk_pte(page, prot); + pteval = phys_pte_mm(vma->vm_mm, page_to_phys(page) + offset, prot); if (mkwrite) { pteval = pte_mkyoung(pteval); pteval = maybe_mkwrite(pte_mkdirty(pteval), vma); } inc_mm_counter(vma->vm_mm, mm_counter_file(folio)); + mm_subpage_file_map_add(vma, pte_phys_mm(vma->vm_mm, pteval)); folio_add_file_rmap_pte(folio, page, vma); } set_pte_at(vma->vm_mm, addr, pte, pteval); @@ -2390,7 +2611,7 @@ static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, } static int insert_page(struct vm_area_struct *vma, unsigned long addr, - struct page *page, pgprot_t prot, bool mkwrite) + struct page *page, pgprot_t prot, bool mkwrite, unsigned int offset) { int retval; pte_t *pte; @@ -2404,83 +2625,120 @@ static int insert_page(struct vm_area_struct *vma, unsigned long addr, if (!pte) goto out; retval = insert_page_into_pte_locked(vma, pte, addr, page, prot, - mkwrite); + mkwrite, offset); pte_unmap_unlock(pte, ptl); out: return retval; } -static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte, - unsigned long addr, struct page *page, pgprot_t prot) +/* Keep lock batching in units of hardware leaves, backing in native pages. */ +static int insert_page_range(struct vm_area_struct *vma, unsigned long addr, + struct page **pages, unsigned long offset, + unsigned long *length, pgprot_t prot) { - int err; + struct mm_struct *mm = vma->vm_mm; + unsigned long remaining = *length; + unsigned long size = mm_page_size(mm); + int ret = 0; - err = validate_page_before_insert(vma, page); - if (err) - return err; - return insert_page_into_pte_locked(vma, pte, addr, page, prot, false); + while (remaining) { + unsigned long in_pmd = min(remaining, + pmd_size_mm(mm) - (addr & ~pmd_mask_mm(mm))); + pmd_t *pmd = walk_to_pmd(mm, addr); + + if (!pmd) { + ret = -EFAULT; + break; + } + if (pte_alloc(mm, pmd)) { + ret = -ENOMEM; + break; + } + while (in_pmd) { + unsigned int batch = min_t(unsigned long, in_pmd / size, 8); + unsigned int i; + spinlock_t *ptl; + pte_t *start = pte_offset_map_lock(mm, pmd, addr, &ptl); + + if (!start) { + ret = -EFAULT; + goto out; + } + for (i = 0; i < batch; i++) { + struct page *page = pages[offset >> PAGE_SHIFT]; + + ret = validate_page_before_insert(vma, page); + if (!ret) + ret = insert_page_into_pte_locked(vma, start + i, + addr, page, prot, false, offset_in_page(offset)); + if (ret) + break; + addr += size; + offset += size; + remaining -= size; + in_pmd -= size; + } + pte_unmap_unlock(start, ptl); + if (ret) + goto out; + } + } +out: + *length = remaining; + return ret; } -/* insert_pages() amortizes the cost of spinlock operations - * when inserting pages in a loop. - */ static int insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num, pgprot_t prot) { - pmd_t *pmd = NULL; - pte_t *start_pte, *pte; - spinlock_t *pte_lock; - struct mm_struct *const mm = vma->vm_mm; - unsigned long curr_page_idx = 0; - unsigned long remaining_pages_total = *num; - unsigned long pages_to_write_in_pmd; + unsigned long length; int ret; -more: - ret = -EFAULT; - pmd = walk_to_pmd(mm, addr); - if (!pmd) - goto out; - pages_to_write_in_pmd = min_t(unsigned long, - remaining_pages_total, PTRS_PER_PTE - pte_index(addr)); - - /* Allocate the PTE if necessary; takes PMD lock once only. */ - ret = -ENOMEM; - if (pte_alloc(mm, pmd)) - goto out; + if (check_mul_overflow(*num, PAGE_SIZE, &length)) + return -EOVERFLOW; + ret = insert_page_range(vma, addr, pages, 0, &length, prot); + /* A partially inserted native page is still an incomplete input page. */ + *num = DIV_ROUND_UP(length, PAGE_SIZE); + return ret; +} - while (pages_to_write_in_pmd) { - int pte_idx = 0; - const int batch_size = min_t(int, pages_to_write_in_pmd, 8); +/** + * vm_insert_pages_range - map a byte range from a native-page array + * @vma: target VMA, held under the mmap write lock + * @addr: first user address + * @pages: native backing pages + * @nr_pages: size of @pages + * @offset: byte offset within the backing array + * @length: bytes to map + * + * Address, offset and length must match the target mm's granule. A failed + * insertion can leave a prefix mapped, as with vm_insert_pages(); the caller + * must unmap that prefix when abandoning the mapping. Physical ownership + * remains in native pages. Each installed user leaf acquires its own ref. + */ +int vm_insert_pages_range(struct vm_area_struct *vma, unsigned long addr, + struct page **pages, unsigned long nr_pages, + unsigned long offset, unsigned long length) +{ + unsigned long bytes, end, size = mm_page_size(vma->vm_mm); - start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock); - if (!start_pte) { - ret = -EFAULT; - goto out; - } - for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) { - int err = insert_page_in_batch_locked(vma, pte, - addr, pages[curr_page_idx], prot); - if (unlikely(err)) { - pte_unmap_unlock(start_pte, pte_lock); - ret = err; - remaining_pages_total -= pte_idx; - goto out; - } - addr += PAGE_SIZE; - ++curr_page_idx; - } - pte_unmap_unlock(start_pte, pte_lock); - pages_to_write_in_pmd -= batch_size; - remaining_pages_total -= batch_size; + if (!IS_ALIGNED(addr | offset | length, size) || + check_mul_overflow(nr_pages, PAGE_SIZE, &bytes) || + offset > bytes || length > bytes - offset || + check_add_overflow(addr, length, &end) || + !range_in_vma(vma, addr, end)) + return -EINVAL; + if (!length) + return 0; + if (vma->vm_flags & VM_PFNMAP) + return -EINVAL; + if (!(vma->vm_flags & VM_MIXEDMAP)) { + mmap_assert_write_locked(vma->vm_mm); + vm_flags_set(vma, VM_MIXEDMAP); } - if (remaining_pages_total) - goto more; - ret = 0; -out: - *num = remaining_pages_total; - return ret; + return insert_page_range(vma, addr, pages, offset, &length, vma->vm_page_prot); } +EXPORT_SYMBOL_GPL(vm_insert_pages_range); /** * vm_insert_pages - insert multiple pages into user vma, batching the pmd lock. @@ -2501,9 +2759,12 @@ int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num) { const unsigned long nr_pages = *num; - const unsigned long end = addr + PAGE_SIZE * nr_pages; + unsigned long length, end; - if (!range_in_vma(vma, addr, end)) + if (!IS_ALIGNED(addr, mm_page_size(vma->vm_mm)) || + check_mul_overflow(nr_pages, PAGE_SIZE, &length) || + check_add_overflow(addr, length, &end) || + !range_in_vma(vma, addr, end)) return -EFAULT; if (!(vma->vm_flags & VM_MIXEDMAP)) { VM_WARN_ON_ONCE(mmap_read_trylock(vma->vm_mm)); @@ -2528,8 +2789,10 @@ int map_kernel_pages_prepare(struct vm_area_desc *desc) } nr_pages = action->map_kernel.nr_pages; - end = addr + PAGE_SIZE * nr_pages; - if (!range_in_vma_desc(desc, addr, end)) + if (!IS_ALIGNED(addr, mm_page_size(desc->mm)) || + check_mul_overflow(nr_pages, PAGE_SIZE, &end) || + check_add_overflow(addr, end, &end) || + !range_in_vma_desc(desc, addr, end)) return -EFAULT; return 0; @@ -2588,10 +2851,35 @@ int vm_insert_page(struct vm_area_struct *vma, unsigned long addr, BUG_ON(vma->vm_flags & VM_PFNMAP); vm_flags_set(vma, VM_MIXEDMAP); } - return insert_page(vma, addr, page, vma->vm_page_prot, false); + return insert_page(vma, addr, page, vma->vm_page_prot, false, 0); } EXPORT_SYMBOL(vm_insert_page); +#ifdef CONFIG_MM_SUBPAGE +/* Explicit physical offset; the VMA and supplied page retain native ownership. */ +vm_fault_t vmf_insert_page_offset(struct vm_area_struct *vma, unsigned long addr, + struct page *page, unsigned int offset) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long size = mm_page_size(mm); + int err; + + if (!(vma->vm_flags & VM_MIXEDMAP) || (vma->vm_flags & VM_PFNMAP) || + (addr & (size - 1)) || (offset & (size - 1)) || + offset > PAGE_SIZE - size || addr < vma->vm_start || + addr >= vma->vm_end || vma->vm_end - addr < size) + return VM_FAULT_SIGBUS; + err = insert_page(vma, addr, page, vma->vm_page_prot, false, offset); + if (err == -ENOMEM) + return VM_FAULT_OOM; + if (err < 0 && err != -EBUSY) + return VM_FAULT_SIGBUS; + return VM_FAULT_NOPAGE; +} +EXPORT_SYMBOL_GPL(vmf_insert_page_offset); +#endif + + /* * __vm_map_pages - maps range of kernel pages into user vma * @vma: user vma to map to @@ -2606,20 +2894,22 @@ EXPORT_SYMBOL(vm_insert_page); * Return: 0 on success and error code otherwise. */ static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages, - unsigned long num, unsigned long offset) + unsigned long num, unsigned long offset, + unsigned int subpage_offset) { - unsigned long count = vma_pages(vma); - unsigned long uaddr = vma->vm_start; + unsigned long bytes, available; - /* Fail if the user requested offset is beyond the end of the object */ - if (offset >= num) + if (offset >= num || + check_mul_overflow(num - offset, PAGE_SIZE, &available) || + subpage_offset >= PAGE_SIZE || + check_mul_overflow(offset, PAGE_SIZE, &bytes) || + check_add_overflow(bytes, (unsigned long)subpage_offset, &bytes)) return -ENXIO; - - /* Fail if the user requested size exceeds available object size */ - if (count > num - offset) + if (vma->vm_end - vma->vm_start > + available - subpage_offset) return -ENXIO; - - return vm_insert_pages(vma, uaddr, pages + offset, &count); + return vm_insert_pages_range(vma, vma->vm_start, pages, num, bytes, + vma->vm_end - vma->vm_start); } /** @@ -2643,7 +2933,7 @@ static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages, int vm_map_pages(struct vm_area_struct *vma, struct page **pages, unsigned long num) { - return __vm_map_pages(vma, pages, num, vma->vm_pgoff); + return __vm_map_pages(vma, pages, num, vma->vm_pgoff, vma_subpage_offset(vma)); } EXPORT_SYMBOL(vm_map_pages); @@ -2663,12 +2953,12 @@ EXPORT_SYMBOL(vm_map_pages); int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, unsigned long num) { - return __vm_map_pages(vma, pages, num, 0); + return __vm_map_pages(vma, pages, num, 0, 0); } EXPORT_SYMBOL(vm_map_pages_zero); static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, pgprot_t prot, bool mkwrite) + unsigned long pfn, pgprot_t prot, bool mkwrite, unsigned int offset) { struct mm_struct *mm = vma->vm_mm; pte_t *pte, entry; @@ -2690,7 +2980,7 @@ static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, * allocation and mapping invalidation so just skip the * update. */ - if (pte_pfn(entry) != pfn) { + if (pte_phys_mm(mm, entry) != PFN_PHYS(pfn) + offset) { WARN_ON_ONCE(!is_zero_pfn(pte_pfn(entry))); goto out_unlock; } @@ -2703,7 +2993,7 @@ static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, } /* Ok, finally just insert the thing.. */ - entry = pte_mkspecial(pfn_pte(pfn, prot)); + entry = pte_mkspecial(phys_pte_mm(mm, PFN_PHYS(pfn) + offset, prot)); if (mkwrite) { entry = pte_mkyoung(entry); @@ -2765,9 +3055,9 @@ static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, * Context: Process context. May allocate using %GFP_KERNEL. * Return: vm_fault_t value. */ -vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, +static vm_fault_t __vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t pgprot, - bool mkwrite) + bool mkwrite, unsigned int offset) { /* * Technically, architectures with pte_special can avoid all these @@ -2789,10 +3079,31 @@ vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, pfnmap_setup_cachemode_pfn(pfn, &pgprot); - return insert_pfn(vma, addr, pfn, pgprot, mkwrite); + return insert_pfn(vma, addr, pfn, pgprot, mkwrite, offset); +} +vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, pgprot_t pgprot, + bool mkwrite) +{ + return __vmf_insert_pfn_prot_mkwrite(vma, addr, pfn, pgprot, mkwrite, 0); } EXPORT_SYMBOL(vmf_insert_pfn_prot_mkwrite); +/* Explicit quarter offset; a native PFN alone cannot encode a 4K leaf. */ +vm_fault_t vmf_insert_pfn_prot_mkwrite_offset(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, unsigned int offset, + pgprot_t pgprot, bool mkwrite) +{ + unsigned long size = mm_page_size(vma->vm_mm); + + if (!IS_ALIGNED(addr | offset, size) || offset > PAGE_SIZE - size || + addr < vma->vm_start || addr >= vma->vm_end || + vma->vm_end - addr < size) + return VM_FAULT_SIGBUS; + return __vmf_insert_pfn_prot_mkwrite(vma, addr, pfn, pgprot, mkwrite, offset); +} +EXPORT_SYMBOL_GPL(vmf_insert_pfn_prot_mkwrite_offset); + static bool vm_mixed_ok(struct vm_area_struct *vma, unsigned long pfn, bool mkwrite) { @@ -2840,9 +3151,9 @@ static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma, * result in pfn_t_has_page() == false. */ page = pfn_to_page(pfn); - err = insert_page(vma, addr, page, pgprot, mkwrite); + err = insert_page(vma, addr, page, pgprot, mkwrite, 0); } else { - return insert_pfn(vma, addr, pfn, pgprot, mkwrite); + return insert_pfn(vma, addr, pfn, pgprot, mkwrite, 0); } if (err == -ENOMEM) @@ -2863,7 +3174,7 @@ vm_fault_t vmf_insert_page_mkwrite(struct vm_fault *vmf, struct page *page, if (addr < vmf->vma->vm_start || addr >= vmf->vma->vm_end) return VM_FAULT_SIGBUS; - err = insert_page(vmf->vma, addr, page, pgprot, write); + err = insert_page(vmf->vma, addr, page, pgprot, write, 0); if (err == -ENOMEM) return VM_FAULT_OOM; if (err < 0 && err != -EBUSY) @@ -2898,7 +3209,7 @@ vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma, */ static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, unsigned long end, - unsigned long pfn, pgprot_t prot) + phys_addr_t phys, pgprot_t prot) { pte_t *pte, *mapped_pte; spinlock_t *ptl; @@ -2910,13 +3221,13 @@ static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, lazy_mmu_mode_enable(); do { BUG_ON(!pte_none(ptep_get(pte))); - if (!pfn_modify_allowed(pfn, prot)) { + if (!pfn_modify_allowed(PHYS_PFN(phys), prot)) { err = -EACCES; break; } - set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot))); - pfn++; - } while (pte++, addr += PAGE_SIZE, addr != end); + set_pte_at(mm, addr, pte, pte_mkspecial(phys_pte_mm(mm, phys, prot))); + phys += mm_page_size(mm); + } while (pte++, addr += mm_page_size(mm), addr != end); lazy_mmu_mode_disable(); pte_unmap_unlock(mapped_pte, ptl); return err; @@ -2924,67 +3235,67 @@ static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud, unsigned long addr, unsigned long end, - unsigned long pfn, pgprot_t prot) + phys_addr_t phys, pgprot_t prot) { pmd_t *pmd; unsigned long next; int err; - pfn -= addr >> PAGE_SHIFT; pmd = pmd_alloc(mm, pud, addr); if (!pmd) return -ENOMEM; VM_BUG_ON(pmd_trans_huge(*pmd)); do { - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(mm, addr, end); err = remap_pte_range(mm, pmd, addr, next, - pfn + (addr >> PAGE_SHIFT), prot); + phys, prot); if (err) return err; + phys += next - addr; } while (pmd++, addr = next, addr != end); return 0; } static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d, unsigned long addr, unsigned long end, - unsigned long pfn, pgprot_t prot) + phys_addr_t phys, pgprot_t prot) { pud_t *pud; unsigned long next; int err; - pfn -= addr >> PAGE_SHIFT; pud = pud_alloc(mm, p4d, addr); if (!pud) return -ENOMEM; do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(mm, addr, end); err = remap_pmd_range(mm, pud, addr, next, - pfn + (addr >> PAGE_SHIFT), prot); + phys, prot); if (err) return err; + phys += next - addr; } while (pud++, addr = next, addr != end); return 0; } static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd, unsigned long addr, unsigned long end, - unsigned long pfn, pgprot_t prot) + phys_addr_t phys, pgprot_t prot) { p4d_t *p4d; unsigned long next; int err; - pfn -= addr >> PAGE_SHIFT; p4d = p4d_alloc(mm, pgd, addr); if (!p4d) return -ENOMEM; do { - next = p4d_addr_end(addr, end); + next = p4d_addr_end_mm(mm, addr, end); err = remap_pud_range(mm, p4d, addr, next, - pfn + (addr >> PAGE_SHIFT), prot); + phys, prot); if (err) return err; + phys += next - addr; } while (p4d++, addr = next, addr != end); return 0; } @@ -3013,25 +3324,26 @@ static int remap_pfn_range_internal(struct vm_area_struct *vma, unsigned long ad { pgd_t *pgd; unsigned long next; - unsigned long end = addr + PAGE_ALIGN(size); struct mm_struct *mm = vma->vm_mm; + unsigned long end = addr + mm_page_align(mm, size); + phys_addr_t phys = PFN_PHYS(pfn); int err; - if (WARN_ON_ONCE(!PAGE_ALIGNED(addr))) + if (WARN_ON_ONCE(!IS_ALIGNED(addr, mm_page_size(mm)))) return -EINVAL; VM_WARN_ON_ONCE(!vma_test_all_mask(vma, VMA_REMAP_FLAGS)); BUG_ON(addr >= end); - pfn -= addr >> PAGE_SHIFT; pgd = pgd_offset(mm, addr); flush_cache_range(vma, addr, end); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(mm, addr, end); err = remap_p4d_range(mm, pgd, addr, next, - pfn + (addr >> PAGE_SHIFT), prot); + phys, prot); if (err) return err; + phys += next - addr; } while (pgd++, addr = next, addr != end); return 0; @@ -3093,7 +3405,7 @@ static int remap_pfn_range_track(struct vm_area_struct *vma, unsigned long addr, struct pfnmap_track_ctx *ctx = NULL; int err; - size = PAGE_ALIGN(size); + size = mm_page_align(vma->vm_mm, size); /* * If we cover the full VMA, we'll perform actual tracking, and @@ -3162,7 +3474,7 @@ static int remap_pfn_range_prepare_vma(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size) { - const unsigned long end = addr + PAGE_ALIGN(size); + const unsigned long end = addr + mm_page_align(vma->vm_mm, size); const bool is_cow = is_cow_mapping(vma->vm_flags); int err; @@ -3540,15 +3852,14 @@ static inline int pte_unmap_same(struct vm_fault *vmf) * -EHWPOISON: copy failed due to hwpoison in source page * -EAGAIN: copied failed (some other reason) */ -static inline int __wp_page_copy_user(struct page *dst, struct page *src, - struct vm_fault *vmf) +static inline int __wp_native_page_copy_user(struct page *dst, struct page *src, + struct vm_fault *vmf, unsigned long addr) { int ret; void *kaddr; void __user *uaddr; struct vm_area_struct *vma = vmf->vma; struct mm_struct *mm = vma->vm_mm; - unsigned long addr = vmf->address; if (likely(src)) { if (copy_mc_user_highpage(dst, src, addr, vma)) @@ -3638,6 +3949,22 @@ static inline int __wp_page_copy_user(struct page *dst, struct page *src, return ret; } +static int __wp_page_copy_user(struct page *dst, struct page *src, + struct vm_fault *vmf) +{ + unsigned int i, nr = mm_pte_native_pages(vmf->vma->vm_mm); + + for (i = 0; i < nr; i++) { + int ret = __wp_native_page_copy_user(dst + i, src ? src + i : NULL, + vmf, vmf->address + i * PAGE_SIZE); + + if (ret) + return ret; + kmsan_copy_page_meta(dst + i, src ? src + i : NULL); + } + return 0; +} + static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma) { struct file *vm_file = vma->vm_file; @@ -3883,14 +4210,13 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) delayacct_wpcopy_end(); return err == -EHWPOISON ? VM_FAULT_HWPOISON : 0; } - kmsan_copy_page_meta(&new_folio->page, vmf->page); } __folio_mark_uptodate(new_folio); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, - vmf->address & PAGE_MASK, - (vmf->address & PAGE_MASK) + PAGE_SIZE); + vmf->address & mm_page_mask(mm), + (vmf->address & mm_page_mask(mm)) + mm_page_size(mm)); mmu_notifier_invalidate_range_start(&range); /* @@ -3927,6 +4253,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) * some TLBs while the old PTE remains in others. */ ptep_clear_flush(vma, vmf->address, vmf->pte); + folio_ref_add(new_folio, mm_pte_native_pages(mm) - 1); folio_add_new_anon_rmap(new_folio, vma, vmf->address, RMAP_EXCLUSIVE); folio_add_lru_vma(new_folio, vma); BUG_ON(unshare && pte_write(entry)); @@ -3970,7 +4297,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) mmu_notifier_invalidate_range_end(&range); if (new_folio) - folio_put(new_folio); + folio_put_refs(new_folio, page_copied ? mm_pte_native_pages(mm) : 1); if (old_folio) { if (page_copied) free_swap_cache(old_folio); @@ -4092,7 +4419,7 @@ static vm_fault_t wp_page_shared(struct vm_fault *vmf, struct folio *folio) return ret; } -#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#ifdef CONFIG_MM_ID static bool __wp_can_reuse_large_anon_folio(struct folio *folio, struct vm_area_struct *vma) { @@ -4158,18 +4485,18 @@ static bool __wp_can_reuse_large_anon_folio(struct folio *folio, folio_unlock_large_mapcount(folio); return exclusive; } -#else /* !CONFIG_TRANSPARENT_HUGEPAGE */ +#else /* !CONFIG_MM_ID */ static bool __wp_can_reuse_large_anon_folio(struct folio *folio, struct vm_area_struct *vma) { BUILD_BUG(); } -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +#endif /* CONFIG_MM_ID */ static bool wp_can_reuse_anon_folio(struct folio *folio, struct vm_area_struct *vma) { - if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && folio_test_large(folio)) + if (IS_ENABLED(CONFIG_MM_ID) && folio_test_large(folio)) return __wp_can_reuse_large_anon_folio(folio, vma); /* @@ -4207,67 +4534,238 @@ static bool wp_can_reuse_anon_folio(struct folio *folio, return true; } -/* - * This routine handles present pages, when - * * users try to write to a shared page (FAULT_FLAG_WRITE) - * * GUP wants to take a R/O pin on a possibly shared anonymous page - * (FAULT_FLAG_UNSHARE) - * - * It is done by copying the page to a new address and decrementing the - * shared-page counter for the old page. - * - * Note that this routine assumes that the protection checks have been - * done by the caller (the low-level page fault routine in most cases). - * Thus, with FAULT_FLAG_WRITE, we can safely just mark it writable once we've - * done any necessary COW. - * - * In case of FAULT_FLAG_WRITE, we also mark the page dirty at this point even - * though the page will change only once the write actually happens. This - * avoids a few races, and potentially makes it more efficient. - * - * We enter with either the VMA lock or the mmap_lock held (see - * FAULT_FLAG_VMA_LOCK) and pte both mapped and locked. We return with - * the same lock still held, but pte unmapped and unlocked. - */ -static vm_fault_t do_wp_page(struct vm_fault *vmf) +#ifdef CONFIG_MM_SUBPAGE +/* Enter with the original PTE locked; a slot, not its native folio, is copied. */ +static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) __releases(vmf->ptl) { - const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; struct vm_area_struct *vma = vmf->vma; - struct folio *folio = NULL; - pte_t pte; + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *source = NULL, *dest = NULL; + struct mm_subpage_pool *pool = NULL; + struct folio *old = vmf->page ? page_folio(vmf->page) : NULL; + struct folio *new = NULL; + struct mmu_notifier_range range; + bool copied = false, unshare = vmf->flags & FAULT_FLAG_UNSHARE; + unsigned int offset; + vm_fault_t ret = VM_FAULT_SIGBUS; + pte_t entry; + int err; - if (likely(!unshare)) { - if (userfaultfd_pte_wp(vma, ptep_get(vmf->pte))) { - if (!userfaultfd_wp_async(vma)) { - pte_unmap_unlock(vmf->pte, vmf->ptl); - return handle_userfault(vmf, VM_UFFD_WP); - } + if (old && folio_test_anon(old)) { + source = mm_subpage_get_from_phys(pte_phys_mm(mm, vmf->orig_pte)); + if (WARN_ON_ONCE(!source)) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + return VM_FAULT_SIGBUS; + } + if (!mm_subpage_is_exclusive(source) && mm_subpage_mapcount(source) == 1 && + folio_trylock(old)) { + bool reusable = true; - /* - * Nothing needed (cache flush, TLB invalidations, - * etc.) because we're only removing the uffd-wp bit, - * which is completely invisible to the user. - */ - pte = pte_clear_uffd_wp(ptep_get(vmf->pte)); + if (folio_test_swapcache(old)) { + swp_entry_t entry = page_swap_entry(&old->page); - set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); - /* - * Update this to be prepared for following up CoW - * handling - */ - vmf->orig_pte = pte; + reusable = !swap_subpage_count(entry, mm_subpage_offset(source)) && + !(folio_test_writeback(old) && + (folio_swap_flags(old) & SWP_STABLE_WRITES)); + } + if (reusable) + mm_subpage_try_reuse_anon(source); + folio_unlock(old); } - - /* - * Userfaultfd write-protect can defer flushes. Ensure the TLB - * is flushed in this case before copying. - */ - if (unlikely(userfaultfd_wp(vmf->vma) && - mm_tlb_flush_pending(vmf->vma->vm_mm))) - flush_tlb_page(vmf->vma, vmf->address); + if (mm_subpage_is_exclusive(source)) { + if (unshare) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + } else { + folio_xchg_last_cpupid(old, (1 << LAST_CPUPID_SHIFT) - 1); + /* Exclusivity belongs to the slot, not PageAnonExclusive. */ + wp_page_reuse(vmf, NULL); + } + mm_subpage_put(source); + return 0; + } + } else if (old) { + folio_get(old); + } else if (!is_zero_pfn(pte_pfn(vmf->orig_pte))) { + /* Private device/PFN mappings need their own subpage contract. */ + pte_unmap_unlock(vmf->pte, vmf->ptl); + return VM_FAULT_SIGBUS; } - + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; + delayacct_wpcopy_start(); + ret = vmf_anon_prepare(vmf); + if (ret) + goto out; + offset = vma_page_offset_at(vma, vmf->address).offset; + ret = VM_FAULT_OOM; + pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!pool) + goto out; + new = folio_prealloc(mm, vma, vmf->address, false); + if (!new) + goto out; + err = mm_subpage_pool_add_folio(pool, new, GFP_KERNEL); + if (err) { + folio_put(new); + goto out; + } + dest = source ? mm_subpage_copy_at(pool, source, offset) : + mm_subpage_alloc_at(pool, offset); + if (IS_ERR(dest)) { + if (PTR_ERR(dest) == -EHWPOISON) + ret = VM_FAULT_HWPOISON; + dest = NULL; + goto out; + } + if (old && !source) { + err = copy_user_subpage_range(&new->page, offset, vmf->page, + pte_phys_mm(mm, vmf->orig_pte) & ~PAGE_MASK, + mm_page_size(mm)); + if (err) { + ret = VM_FAULT_HWPOISON; + goto out; + } + } + mm_subpage_pool_close(pool); + mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, + vmf->address, vmf->address + mm_page_size(mm)); + mmu_notifier_invalidate_range_start(&range); + /* Keep the folio alive after a concurrent zap can consume its PTE ref. */ + folio_get(new); + folio_lock(new); + ret = 0; + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (!vmf->pte) + goto unlock_folio; + if (!pte_same(ptep_get(vmf->pte), vmf->orig_pte)) { + update_mmu_tlb(vma, vmf->address, vmf->pte); + goto unlock_pte; + } + mm_subpage_set_exclusive(dest); + err = mm_subpage_add_new_anon_rmap(dest, vma, vmf->address); + if (WARN_ON_ONCE(err)) { + ret = VM_FAULT_SIGBUS; + goto unlock_pte; + } + entry = pte_sw_mkyoung(phys_pte_mm(mm, mm_subpage_phys(dest), vma->vm_page_prot)); + if (unshare) { + if (pte_soft_dirty(vmf->orig_pte)) + entry = pte_mksoft_dirty(entry); + if (pte_uffd_wp(vmf->orig_pte)) + entry = pte_mkuffd_wp(entry); + } else { + entry = maybe_mkwrite(pte_mkdirty(entry), vma); + } + BUG_ON(unshare && pte_write(entry)); + flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte)); + /* Break-before-make also orders removal of the old slot's rmap. */ + ptep_clear_flush(vma, vmf->address, vmf->pte); + subpage_add_new_lru(new, vma, 1); + set_pte_at(mm, vmf->address, vmf->pte, entry); + update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + if (source) { + mm_subpage_remove_anon_rmap(source, vma); + } else if (old) { + mm_subpage_file_map_del(vma, pte_phys_mm(mm, vmf->orig_pte)); + folio_remove_rmap_pte(old, vmf->page, vma); + dec_mm_counter(mm, mm_counter_file(old)); + inc_mm_counter(mm, MM_ANONPAGES); + } else { + ksm_might_unmap_zero_page(mm, vmf->orig_pte); + inc_mm_counter(mm, MM_ANONPAGES); + } + dest = NULL; /* The new PTE owns the reference. */ + copied = true; +unlock_pte: + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; +unlock_folio: + folio_unlock(new); + folio_put(new); + mmu_notifier_invalidate_range_end(&range); +out: + if (pool) { + mm_subpage_pool_close(pool); + if (dest) + mm_subpage_put(dest); + mm_subpage_pool_put(pool); + } + if (source) { + if (copied) + mm_subpage_put(source); /* Retired PTE reference, after TLBI. */ + mm_subpage_put(source); /* Source stabilisation reference. */ + } else if (old) { + if (copied) + folio_put(old); + folio_put(old); + } + delayacct_wpcopy_end(); + return ret; +} +#endif + +/* + * This routine handles present pages, when + * * users try to write to a shared page (FAULT_FLAG_WRITE) + * * GUP wants to take a R/O pin on a possibly shared anonymous page + * (FAULT_FLAG_UNSHARE) + * + * It is done by copying the page to a new address and decrementing the + * shared-page counter for the old page. + * + * Note that this routine assumes that the protection checks have been + * done by the caller (the low-level page fault routine in most cases). + * Thus, with FAULT_FLAG_WRITE, we can safely just mark it writable once we've + * done any necessary COW. + * + * In case of FAULT_FLAG_WRITE, we also mark the page dirty at this point even + * though the page will change only once the write actually happens. This + * avoids a few races, and potentially makes it more efficient. + * + * We enter with either the VMA lock or the mmap_lock held (see + * FAULT_FLAG_VMA_LOCK) and pte both mapped and locked. We return with + * the same lock still held, but pte unmapped and unlocked. + */ +static vm_fault_t do_wp_page(struct vm_fault *vmf) + __releases(vmf->ptl) +{ + const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; + struct vm_area_struct *vma = vmf->vma; + struct folio *folio = NULL; + pte_t pte; + + if (likely(!unshare)) { + if (userfaultfd_pte_wp(vma, ptep_get(vmf->pte))) { + if (!userfaultfd_wp_async(vma)) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + return handle_userfault(vmf, VM_UFFD_WP); + } + + /* + * Nothing needed (cache flush, TLB invalidations, + * etc.) because we're only removing the uffd-wp bit, + * which is completely invisible to the user. + */ + pte = pte_clear_uffd_wp(ptep_get(vmf->pte)); + + set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); + /* + * Update this to be prepared for following up CoW + * handling + */ + vmf->orig_pte = pte; + } + + /* + * Userfaultfd write-protect can defer flushes. Ensure the TLB + * is flushed in this case before copying. + */ + if (unlikely(userfaultfd_wp(vmf->vma) && + mm_tlb_flush_pending(vmf->vma->vm_mm))) + flush_tlb_page(vmf->vma, vmf->address); + } + vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte); if (vmf->page) @@ -4292,6 +4790,11 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) return wp_page_shared(vmf, folio); } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return wp_subpage_copy(vmf); +#endif + /* * Private mapping: create an exclusive anonymous page copy if reuse * is impossible. We might miss VM_WRITE for FOLL_FORCE handling. @@ -4301,8 +4804,12 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) */ if (folio && folio_test_anon(folio) && (PageAnonExclusive(vmf->page) || wp_can_reuse_anon_folio(folio, vma))) { - if (!PageAnonExclusive(vmf->page)) - SetPageAnonExclusive(vmf->page); + if (!PageAnonExclusive(vmf->page)) { + unsigned int i, nr = mm_pte_native_pages(vma->vm_mm); + + for (i = 0; i < nr; i++) + SetPageAnonExclusive(vmf->page + i); + } if (unlikely(unshare)) { pte_unmap_unlock(vmf->pte, vmf->ptl); return 0; @@ -4340,12 +4847,116 @@ static inline void unmap_mapping_range_tree(struct rb_root_cached *root, start = vma->vm_start + ((start_idx - vma->vm_pgoff) << PAGE_SHIFT); size = (end_idx - start_idx) << PAGE_SHIFT; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + unsigned long end = vma->vm_end; + + start = vma_address(vma, first_index, last_index - first_index + 1); + /* The interval tree guarantees a nonempty intersection. */ + VM_BUG_ON_VMA(IS_ERR_VALUE(start), vma); + if (last_index < vma_last_pgoff(vma)) + end = vma_address_at_offset(vma, (struct vm_page_offset) { + .index = last_index + 1, + }); + size = end - start; + } else if (mm_page_size(vma->vm_mm) > PAGE_SIZE) { + unsigned long granule = mm_page_size(vma->vm_mm); + unsigned long end = min(start + size, vma->vm_end); + + /* A native-page invalidation removes every overlapping leaf. */ + start = max(round_down(start, granule), vma->vm_start); + end = min(round_up(end, granule), vma->vm_end); + size = end - start; + } +#endif + tlb_gather_mmu(&tlb, vma->vm_mm); zap_vma_range_batched(&tlb, vma, start, size, details); tlb_finish_mmu(&tlb); } } +#ifdef CONFIG_MM_SUBPAGE +/* Unmap affected user leaves, including coarse aliases; preserve private COW. */ +void unmap_mapping_subpage_hole(struct address_space *mapping, loff_t first, u64 last) +{ + struct vm_area_struct *vma; + struct zap_details details = { .skip_cows = true }; + + i_mmap_lock_read(mapping); + vma_interval_tree_foreach(vma, &mapping->i_mmap, + first >> PAGE_SHIFT, (last - 1) >> PAGE_SHIFT) { + unsigned long start, end, granule = mm_page_size(vma->vm_mm); + u64 low = round_down((u64)first, granule), high = round_up(last, granule); + struct vm_page_offset begin = { low >> PAGE_SHIFT, low & ~PAGE_MASK }; + struct vm_page_offset finish = { high >> PAGE_SHIFT, high & ~PAGE_MASK }; + struct vm_page_offset vbegin = vma_page_offset_at(vma, vma->vm_start); + struct vm_page_offset vend = vma_page_offset_at(vma, vma->vm_end); + struct mmu_gather tlb; + + if (begin.index < vbegin.index || + (begin.index == vbegin.index && begin.offset <= vbegin.offset)) + start = vma->vm_start; + else + start = vma_address_at_offset(vma, begin); + if (finish.index > vend.index || + (finish.index == vend.index && finish.offset >= vend.offset)) + end = vma->vm_end; + else + end = vma_address_at_offset(vma, finish); + if (IS_ERR_VALUE(start) || IS_ERR_VALUE(end) || start >= end) + continue; + tlb_gather_mmu(&tlb, vma->vm_mm); + zap_vma_range_batched(&tlb, vma, start, end - start, &details); + tlb_finish_mmu(&tlb); + } + i_mmap_unlock_read(mapping); +} + +/* + * Native truncation keeps its final partial page. Smaller-granule mms must + * additionally discard complete granules in that native page, including + * private COW copies. Call on both sides of cache truncation, as for the + * ordinary unmap_mapping_range() calls. + */ +void unmap_mapping_subpage_tail(struct address_space *mapping, loff_t newsize) +{ + struct vm_area_struct *vma; + struct zap_details details = { }; + pgoff_t index = newsize >> PAGE_SHIFT; + unsigned int offset = newsize & ~PAGE_MASK; + + if (!offset) + return; + i_mmap_lock_read(mapping); + vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) { + unsigned long start, end, granule = mm_page_size(vma->vm_mm); + struct vm_page_offset pos = { .index = index, .offset = round_up(offset, granule) }; + struct vm_page_offset vpos = vma_page_offset_at(vma, vma->vm_start); + struct mmu_gather tlb; + + if (pos.offset >= PAGE_SIZE) + continue; + if (pos.index < vpos.index || + (pos.index == vpos.index && pos.offset < vpos.offset)) + start = vma->vm_start; + else + start = vma_address_at_offset(vma, pos); + if (IS_ERR_VALUE(start)) + continue; + end = vma->vm_end; + if (index < vma_last_pgoff(vma)) + end = vma_address_at_offset(vma, (struct vm_page_offset) { + .index = index + 1, + }); + tlb_gather_mmu(&tlb, vma->vm_mm); + zap_vma_range_batched(&tlb, vma, start, end - start, &details); + tlb_finish_mmu(&tlb); + } + i_mmap_unlock_read(mapping); +} +#endif + /** * unmap_mapping_folio() - Unmap single folio from processes. * @folio: The locked folio to be unmapped. @@ -4724,6 +5335,134 @@ static void check_swap_exclusive(struct folio *folio, swp_entry_t entry, } while (--nr_pages); } +#ifdef CONFIG_MM_SUBPAGE +/* The swap-cache index remains native-sized; the PTE retains its slot offset. */ +static vm_fault_t do_swap_subpage(struct vm_fault *vmf, swp_entry_t entry) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + struct swap_info_struct *si; + struct mm_subpage *slot = NULL; + struct folio *folio; + unsigned int offset = pte_swp_subpage_offset(vmf->orig_pte); + vm_fault_t ret; + pte_t pte; + int err; + + ret = vmf_anon_prepare(vmf); + if (ret) + return ret; + /* Swap encodes the physical slot, independent of its virtual position. */ + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, mm_page_size(mm))) + return VM_FAULT_SIGBUS; + si = get_swap_device(entry); + if (!si) + return 0; + folio = swap_cache_get_folio(entry); + if (folio) { + swap_update_readahead(folio, vma, vmf->address); + } else { + folio = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE, vmf); + if (!folio) { + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (vmf->pte) { + if (pte_same(ptep_get(vmf->pte), vmf->orig_pte)) + ret = VM_FAULT_OOM; + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; + } + goto out_device; + } + ret = VM_FAULT_MAJOR; + count_vm_event(PGMAJFAULT); + count_memcg_event_mm(mm, PGMAJFAULT); + } + ret |= folio_lock_or_retry(folio, vmf); + if (ret & VM_FAULT_RETRY) + goto out_folio; + if (!folio_matches_swap_entry(folio, entry)) + goto unlock_folio; + if (folio_order(folio) || !folio_test_uptodate(folio) || folio_test_ksm(folio)) { + ret = VM_FAULT_SIGBUS; + goto unlock_folio; + } + if (PageHWPoison(&folio->page)) { + ret = VM_FAULT_HWPOISON; + goto unlock_folio; + } + arch_swap_restore(folio_swap(entry, folio), folio); + slot = mm_subpage_restore_granule(folio, offset, mm_page_shift(mm), GFP_KERNEL); + if (IS_ERR(slot)) { + err = PTR_ERR(slot); + slot = NULL; + if (err != -EAGAIN) + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + goto unlock_folio; + } + folio_throttle_swaprate(folio, GFP_KERNEL); + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (!vmf->pte) + goto unlock_folio; + if (!pte_same(ptep_get(vmf->pte), vmf->orig_pte)) + goto unlock_pte; + err = mm_subpage_add_anon_rmap(slot, vma, vmf->address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + goto unlock_pte; + } + /* + * A carried marker proves exclusivity directly. After the last fork + * sharer exits, recover it only with one swap reference to this exact + * quarter and no other mapping or escaped slot reference. Dropping a + * nonpresent PTE in zap flushes pending reclaim TLB invalidations first. + * Stable-write backends must still finish writeback before reuse. + */ + if (!(folio_test_writeback(folio) && data_race(si->flags & SWP_STABLE_WRITES))) { + if (pte_swp_exclusive(vmf->orig_pte)) + mm_subpage_set_exclusive(slot); + else if (swap_subpage_count(entry, offset) == 1) + mm_subpage_try_reuse_swap(slot); + } + pte = pte_sw_mkyoung(phys_pte_mm(mm, mm_subpage_phys(slot), vma->vm_page_prot)); + if (pte_swp_soft_dirty(vmf->orig_pte)) + pte = pte_mksoft_dirty(pte); + if (pte_swp_uffd_wp(vmf->orig_pte)) + pte = pte_mkuffd_wp(pte); + inc_mm_counter(mm, MM_ANONPAGES); + dec_mm_counter(mm, MM_SWAPENTS); + swap_subpage_put(entry, offset, false); + set_pte_at(mm, vmf->address, vmf->pte, pte); + arch_do_swap_page_nr(mm, vma, vmf->address, pte, pte, 1); + slot = NULL; /* The restored PTE owns the slot reference. */ + vmf->orig_pte = pte; + if (should_try_to_free_swap(si, folio, vma, 2, vmf->flags)) + folio_free_swap(folio); + folio_unlock(folio); + folio_put(folio); /* Drop the cache lookup, not the PTE reference. */ + put_swap_device(si); + if (vmf->flags & FAULT_FLAG_WRITE) { + ret |= do_wp_page(vmf); + return ret & VM_FAULT_ERROR ? ret & VM_FAULT_ERROR : ret; + } + update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; + return ret; +unlock_pte: + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; +unlock_folio: + if (slot) + mm_subpage_put(slot); + folio_unlock(folio); +out_folio: + folio_put(folio); +out_device: + put_swap_device(si); + return ret; +} +#endif + /* * We enter with either the VMA lock or the mmap_lock held (see * FAULT_FLAG_VMA_LOCK), and pte mapped but not yet locked. @@ -4732,6 +5471,222 @@ static void check_swap_exclusive(struct folio *folio, swp_entry_t entry, * When returning, the lock may have been released in the same cases * as done by filemap_fault(). */ +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SWAP) +/* 7.2 charges swapin against the cgroup id recorded in the swap table. */ +static unsigned short swap_coarse_memcg_id(swp_entry_t entry) +{ + struct swap_cluster_info *ci; + unsigned short id; + + ci = swap_cluster_lock(__swap_entry_to_info(entry), swp_offset(entry)); + id = __swap_cgroup_get(ci, swp_cluster_offset(entry)); + swap_cluster_unlock(ci); + return id; +} + +/* One coarse PTE owns a naturally aligned group of native swap entries. */ +vm_fault_t do_swap_coarse_page(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + /* Arm64's largest base user page is 64K, its smallest native page 4K. */ + struct folio *cache[SZ_64K / PAGE_SIZE]; + unsigned int nr = mm_pte_native_pages(mm), covered = 0, count = 0, i; + swp_entry_t entry = softleaf_from_pte(vmf->orig_pte), next; + struct swap_info_struct *si; + struct folio *backing = NULL; + struct page *page; + unsigned int start; + bool copied = false, exclusive, mapped = false; + vm_fault_t ret = 0; + pte_t pte; + + vmf->pte = NULL; + if (WARN_ON_ONCE(nr <= 1 || nr > ARRAY_SIZE(cache) || + !IS_ALIGNED(swp_offset(entry), nr))) + return VM_FAULT_SIGBUS; + si = get_swap_device(entry); + if (!si) + return 0; + + while (covered < nr) { + struct folio *folio; + unsigned int take; + vm_fault_t wait; + + next = entry; + next.val += covered; + folio = swap_cache_get_folio(next); + if (!folio) { + folio = read_swap_cache_async(next, GFP_HIGHUSER_MOVABLE, vma, + vmf->address + covered * PAGE_SIZE, NULL); + if (!(ret & VM_FAULT_MAJOR)) { + ret |= VM_FAULT_MAJOR; + count_vm_event(PGMAJFAULT); + count_memcg_event_mm(mm, PGMAJFAULT); + } + } + if (!folio) + goto oom; + if (!count) { + wait = folio_lock_or_retry(folio, vmf); + if (wait & VM_FAULT_RETRY) { + folio_put(folio); + ret |= wait; + goto out; + } + } else if (!folio_trylock(folio)) { + /* Drop earlier cache locks before waiting for this I/O or owner. */ + for (i = 0; i < count; i++) { + folio_unlock(cache[i]); + folio_put(cache[i]); + } + count = 0; + covered = 0; + wait = folio_lock_or_retry(folio, vmf); + if (wait & VM_FAULT_RETRY) { + folio_put(folio); + ret |= wait; + goto out; + } + folio_unlock(folio); + folio_put(folio); + cond_resched(); + continue; + } + cache[count++] = folio; + if (!folio_matches_swap_entry(folio, next)) + goto out; + if (!folio_test_uptodate(folio)) { + ret = VM_FAULT_SIGBUS; + goto out; + } + start = swp_offset(next) - swp_offset(folio->swap); + take = min(nr - covered, folio_nr_pages(folio) - start); + for (i = 0; i < take; i++) { + if (PageHWPoison(folio_page(folio, start + i))) { + ret = VM_FAULT_HWPOISON; + goto out; + } + } + arch_swap_restore(folio_swap(next, folio), folio); + covered += take; + } + + start = swp_offset(entry) - swp_offset(cache[0]->swap); + if (count == 1 && IS_ALIGNED(start, nr) && + (folio_test_anon(cache[0]) || folio_nr_pages(cache[0]) == nr)) { + backing = cache[0]; + page = folio_page(backing, start); + } else { + backing = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, ilog2(nr), vma, vmf->address); + if (!backing) + goto oom; + copied = true; + if (mem_cgroup_swapin_charge_folio(backing, swap_coarse_memcg_id(entry), + mm, GFP_KERNEL)) + goto oom; + covered = 0; + for (i = 0; i < count; i++) { + unsigned int j, take; + + start = swp_offset(entry) + covered - swp_offset(cache[i]->swap); + take = min(nr - covered, folio_nr_pages(cache[i]) - start); + for (j = 0; j < take; j++) { + struct page *dst = folio_page(backing, covered + j); + struct page *src = folio_page(cache[i], start + j); + + if (copy_mc_user_highpage(dst, src, + vmf->address + (covered + j) * PAGE_SIZE, vma)) { + ret = VM_FAULT_HWPOISON; + goto out; + } + kmsan_copy_page_meta(dst, src); + } + covered += take; + } + folio_set_dirty(backing); + __folio_mark_uptodate(backing); + page = &backing->page; + } + + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte)) + goto out; + exclusive = copied || pte_swp_exclusive(vmf->orig_pte); + if (!copied && exclusive) { + check_swap_exclusive(backing, entry, nr); + if (folio_test_writeback(backing) && data_race(si->flags & SWP_STABLE_WRITES)) + exclusive = false; + } + pte = mk_pte(page, vma->vm_page_prot); + if (pte_swp_soft_dirty(vmf->orig_pte)) + pte = pte_mksoft_dirty(pte); + if (pte_swp_uffd_wp(vmf->orig_pte)) + pte = pte_mkuffd_wp(pte); + if (exclusive && (vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) && + !pte_needs_soft_dirty_wp(vma, pte)) { + pte = pte_mkwrite(pte, vma); + if (vmf->flags & FAULT_FLAG_WRITE) { + pte = pte_mkdirty(pte); + vmf->flags &= ~FAULT_FLAG_WRITE; + } + } + folio_ref_add(backing, nr - copied); + if (!folio_test_anon(backing)) + folio_add_new_anon_rmap(backing, vma, vmf->address, + exclusive ? RMAP_EXCLUSIVE : RMAP_NONE); + else + folio_add_anon_rmap_pte(backing, page, vma, vmf->address, + exclusive ? RMAP_EXCLUSIVE : RMAP_NONE); + if (copied) + folio_add_lru_vma(backing, vma); + inc_mm_counter(mm, MM_ANONPAGES); + dec_mm_counter(mm, MM_SWAPENTS); + flush_icache_pages(vma, page, nr); + vmf->orig_pte = pte; + set_pte_at(mm, vmf->address, vmf->pte, pte); + arch_do_swap_page_nr(mm, vma, vmf->address, pte, pte, 1); + mapped = true; + covered = 0; + for (i = 0; i < count; i++) { + unsigned int take; + + start = swp_offset(entry) + covered - swp_offset(cache[i]->swap); + take = min(nr - covered, folio_nr_pages(cache[i]) - start); + folio_put_swap_range(cache[i], folio_page(cache[i], start), take); + covered += take; + } + goto out; + +oom: + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte)) + ret |= VM_FAULT_OOM; +out: + if (copied && !mapped) + folio_put(backing); + for (i = 0; i < count; i++) { + if (mapped) + folio_free_swap(cache[i]); + folio_unlock(cache[i]); + folio_put(cache[i]); + } + if (mapped && (vmf->flags & FAULT_FLAG_WRITE)) { + ret |= do_wp_page(vmf); + vmf->pte = NULL; /* do_wp_page drops the PT lock. */ + } else if (mapped) { + update_mmu_cache(vma, vmf->address, vmf->pte); + } + if (vmf->pte) + pte_unmap_unlock(vmf->pte, vmf->ptl); + put_swap_device(si); + if (!mapped && !(ret & (VM_FAULT_ERROR | VM_FAULT_RETRY))) + cond_resched(); + return ret; +} +#endif + vm_fault_t do_swap_page(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; @@ -4806,6 +5761,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) goto out; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return do_swap_subpage(vmf, entry); +#ifdef CONFIG_SWAP + if (mm_page_size(vma->vm_mm) > PAGE_SIZE) + return do_swap_coarse_page(vmf); +#endif +#endif + /* Prevent swapoff from happening to us. */ si = get_swap_device(entry); if (unlikely(!si)) @@ -5240,6 +6204,7 @@ void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte, bool uffd_wp) { const unsigned int nr_pages = folio_nr_pages(folio); + const unsigned int nr_ptes = folio_size(folio) >> mm_page_shift(vma->vm_mm); pte_t entry = folio_mk_pte(folio, vma->vm_page_prot); entry = pte_sw_mkyoung(entry); @@ -5252,8 +6217,8 @@ void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte, folio_ref_add(folio, nr_pages - 1); folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE); folio_add_lru_vma(folio, vma); - set_ptes(vma->vm_mm, addr, pte, entry, nr_pages); - update_mmu_cache_range(NULL, vma, addr, pte, nr_pages); + set_ptes(vma->vm_mm, addr, pte, entry, nr_ptes); + update_mmu_cache_range(NULL, vma, addr, pte, nr_ptes); } static void map_anon_folio_pte_pf(struct folio *folio, pte_t *pte, @@ -5262,10 +6227,123 @@ static void map_anon_folio_pte_pf(struct folio *folio, pte_t *pte, const unsigned int order = folio_order(folio); map_anon_folio_pte_nopf(folio, pte, vma, addr, uffd_wp); - add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1L << order); + add_mm_counter(vma->vm_mm, MM_ANONPAGES, folio_size(folio) >> mm_page_shift(vma->vm_mm)); count_mthp_stat(order, MTHP_STAT_ANON_FAULT_ALLOC); } +#ifdef CONFIG_MM_SUBPAGE +static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *slots[PAGE_SIZE / MM_SUBPAGE_SIZE] = {}; + struct vm_page_offset pos = vma_page_offset_at(vma, vmf->address); + struct mm_subpage_pool *pool; + struct folio *folio; + unsigned long start, end, lower; + unsigned int first_offset, nr, target, i, mapped = 0; + vm_fault_t ret = VM_FAULT_OOM; + pte_t *first; + int err; + + /* One native linear index, clipped to the VMA and this PTE table. */ + lower = max(vma->vm_start, vmf->address & pmd_mask_mm(mm)); + start = vmf->address - min_t(unsigned long, pos.offset, vmf->address - lower); + end = vmf->address + min(PAGE_SIZE - pos.offset, + pmd_addr_end_mm(mm, vmf->address, vma->vm_end) - vmf->address); + first_offset = pos.offset - (vmf->address - start); + nr = (end - start) >> mm_page_shift(mm); + target = (vmf->address - start) >> mm_page_shift(mm); + + pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!pool) + return VM_FAULT_OOM; + /* Supply initialises every byte once, including the unissued slots. */ + folio = folio_prealloc(mm, vma, vmf->address, false); + if (!folio) + goto release; + err = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (err) { + folio_put(folio); + goto release; + } + for (i = 0; i < nr; i++) { + slots[i] = mm_subpage_alloc_at(pool, first_offset + i * mm_page_size(mm)); + if (IS_ERR(slots[i])) { + slots[i] = NULL; + goto release; + } + } + + /* PTE references may disappear as soon as we drop the PTL. */ + folio_get(folio); + folio_lock(folio); + ret = 0; + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (!vmf->pte) + goto unlock_folio; + if (vmf_pte_changed(vmf)) { + update_mmu_tlb(vma, vmf->address, vmf->pte); + goto unlock_pte; + } + ret = check_stable_address_space(mm); + if (ret) + goto unlock_pte; + if (userfaultfd_missing(vma)) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; + folio_unlock(folio); + folio_put(folio); + ret = handle_userfault(vmf, VM_UFFD_MISSING); + goto release; + } + + first = vmf->pte - target; + for (i = 0; i < nr; i++) { + unsigned long address = start + i * mm_page_size(mm); + pte_t entry; + + /* Do not replace existing mappings or userfaultfd markers nearby. */ + if (i != target && !pte_none(ptep_get(first + i))) + continue; + mm_subpage_set_exclusive(slots[i]); + err = mm_subpage_add_new_anon_rmap(slots[i], vma, address); + if (WARN_ON_ONCE(err)) { + ret = VM_FAULT_SIGBUS; + break; + } + entry = phys_pte_mm(mm, mm_subpage_phys(slots[i]), vma->vm_page_prot); + entry = pte_sw_mkyoung(entry); + if (vma->vm_flags & VM_WRITE) + entry = pte_mkwrite(pte_mkdirty(entry), vma); + if (i == target && vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + set_pte_at(mm, address, first + i, entry); + update_mmu_cache(vma, address, first + i); + slots[i] = NULL; /* The PTE now owns the reference. */ + mapped++; + } + if (mapped) { + subpage_add_new_lru(folio, vma, mapped); + add_mm_counter(mm, MM_ANONPAGES, mapped); + count_mthp_stat(0, MTHP_STAT_ANON_FAULT_ALLOC); + } +unlock_pte: + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; +unlock_folio: + folio_unlock(folio); + folio_put(folio); +release: + mm_subpage_pool_close(pool); + for (i = 0; i < ARRAY_SIZE(slots); i++) + if (slots[i]) + mm_subpage_put(slots[i]); + mm_subpage_pool_put(pool); + return ret; +} +#endif + /* * We enter with either the VMA lock or the mmap_lock held (see * FAULT_FLAG_VMA_LOCK), and pte unmapped and unlocked. @@ -5326,6 +6404,10 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) ret = vmf_anon_prepare(vmf); if (ret) return ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return do_anonymous_subpage(vmf); +#endif /* Returns NULL on OOM or ERR_PTR(-EAGAIN) if we must retry the fault */ folio = alloc_anon_folio(vmf); if (IS_ERR(folio)) @@ -5333,8 +6415,8 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) if (!folio) goto oom; - nr_pages = folio_nr_pages(folio); - addr = ALIGN_DOWN(vmf->address, nr_pages * PAGE_SIZE); + nr_pages = folio_size(folio) >> mm_page_shift(vma->vm_mm); + addr = ALIGN_DOWN(vmf->address, folio_size(folio)); /* * The memory barrier inside __folio_mark_uptodate makes sure that @@ -5468,6 +6550,10 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *pa * PMD mappings if THPs are disabled. As we already have a THP, * behave as if we are forcing a collapse. */ + /* Match the native PTE fault policy: partial data exposes its zero subunits. */ + if (userfaultfd_missing(vma) && shmem_uffd_has_missing_native_page(folio)) + return ret; + if (thp_disabled_by_hw() || vma_thp_disabled(vma, vma->vm_flags, /* forced_collapse=*/ true)) return ret; @@ -5509,6 +6595,8 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *pa entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); add_mm_counter(vma->vm_mm, mm_counter_file(folio), HPAGE_PMD_NR); + shmem_uffd_mark_mapped(folio, folio_page_idx(folio, page) << PAGE_SHIFT, + HPAGE_PMD_SIZE); folio_add_file_rmap_pmd(folio, page, vma); /* @@ -5571,6 +6659,8 @@ void set_pte_range(struct vm_fault *vmf, struct folio *folio, folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE); folio_add_lru_vma(folio, vma); } else { + shmem_uffd_mark_mapped(folio, folio_page_idx(folio, page) << PAGE_SHIFT, + (unsigned long)nr << PAGE_SHIFT); folio_add_file_rmap_ptes(folio, page, nr, vma); } set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr); @@ -5587,6 +6677,252 @@ static bool vmf_pte_changed(struct vm_fault *vmf) return !pte_none(ptep_get(vmf->pte)); } +#ifdef CONFIG_MM_SUBPAGE +static bool folio_user_range_valid(struct folio *folio, unsigned long offset, + unsigned long size) +{ + return offset <= folio_size(folio) && size <= folio_size(folio) - offset && + IS_ALIGNED(page_to_phys(&folio->page) + offset, size); +} + +/* + * Map the intersection of a locked cache folio, file-index range, VMA and + * the fault's PTE table. vmf->pte points at vmf->address and its lock is held. + * Each PTE owns a reference per covered native page (one for a subpage). + * The caller's reference is untouched, including when no PTE is installed. + */ +vm_fault_t map_file_user_pages(struct vm_fault *vmf, struct folio *folio, + pgoff_t first, pgoff_t last, bool fault) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + unsigned long addr, end, table_start = vmf->address & pmd_mask_mm(mm); + unsigned long size = mm_page_size(mm); + loff_t file_size = i_size_read(vma->vm_file->f_mapping->host); + vm_fault_t ret = 0; + unsigned int mapped = 0; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + first = max(first, folio->index); + last = min(last, folio_next_index(folio) - 1); + if (first > last) + return 0; + addr = vma_address(vma, first, last - first + 1); + if (IS_ERR_VALUE(addr)) + return 0; + addr = ALIGN(max(addr, table_start), size); + end = pmd_addr_end_mm(mm, vmf->address, vma->vm_end); + if (userfaultfd_armed(vma)) { + addr = max(addr, vmf->address); + end = min(end, vmf->address + size); + } + for (; addr < end; addr += size) { + struct vm_page_offset pos = vma_page_offset_at(vma, addr); + struct page *page; + unsigned long offset; + unsigned int i, nr = mm_pte_native_pages(mm); + bool poisoned = false; + pte_t entry, *ptep; + + if (pos.index > last || !vma_file_offset_valid(vma, addr, file_size)) + break; + offset = ((pos.index - folio->index) << PAGE_SHIFT) + pos.offset; + /* + * A coarse leaf cannot span separate cache folios or a partial + * requested index range, even if neighboring PFNs happen to match. + */ + if (nr - 1 > last - pos.index || + !folio_user_range_valid(folio, offset, size)) + continue; + /* A native cache folio can contain independently missing leaves. */ + if (userfaultfd_missing(vma) && + shmem_uffd_range_missing(folio, pos.index, pos.offset, size)) + continue; + page = folio_file_page(folio, pos.index); + ptep = vmf->pte + ((long)(addr - vmf->address) >> mm_page_shift(mm)); + for (i = 0; i < nr; i++) + poisoned |= PageHWPoison(page + i); + if (poisoned) + continue; + if (!pte_none(ptep_get(ptep)) && + !(fault && addr == vmf->address && !vmf_pte_changed(vmf))) + continue; + flush_icache_pages(vma, page, nr); + entry = phys_pte_mm(mm, page_to_phys(page) + pos.offset, vma->vm_page_prot); + if (addr != vmf->address && arch_wants_old_prefaulted_pte()) + entry = pte_mkold(entry); + else + entry = pte_sw_mkyoung(entry); + if (vmf->flags & FAULT_FLAG_WRITE) + entry = maybe_mkwrite(pte_mkdirty(entry), vma); + else if (pte_write(entry) && folio_test_dirty(folio)) + entry = pte_mkdirty(entry); + if (fault && addr == vmf->address && vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + folio_ref_add(folio, nr); + mm_subpage_file_map_add(vma, pte_phys_mm(vma->vm_mm, entry)); + shmem_uffd_mark_mapped(folio, + ((pos.index - folio->index) << PAGE_SHIFT) + pos.offset, size); + folio_add_file_rmap_pte(folio, page, vma); + set_pte_at(mm, addr, ptep, entry); + update_mmu_cache_range(vmf, vma, addr, ptep, 1); + mapped++; + if (addr == vmf->address) + ret = VM_FAULT_NOPAGE; + } + add_mm_counter(mm, mm_counter_file(folio), mapped); + return ret; +} + +/* + * A non-file fault callback supplies one native page, not a cache-indexed + * folio. In particular, VVAR callbacks may reorder native pages. Only map + * quarters of this supplied page; never infer adjacent pages from its folio. + */ +static vm_fault_t map_special_subpages(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + struct page *page = vmf->page; + struct folio *folio = page_folio(page); + unsigned int offset = vma_page_offset_at(vma, vmf->address).offset; + unsigned long size = mm_page_size(mm), addr, end; + vm_fault_t ret = 0; + unsigned int mapped = 0; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (PageHWPoison(page)) + return 0; + addr = offset > vmf->address ? vma->vm_start : + max(vma->vm_start, vmf->address - offset); + addr = max(addr, vmf->address & pmd_mask_mm(mm)); + end = min(vmf->address + PAGE_SIZE - offset, + pmd_addr_end_mm(mm, vmf->address, vma->vm_end)); + if (userfaultfd_armed(vma)) { + addr = vmf->address; + end = vmf->address + size; + } + flush_icache_pages(vma, page, 1); + for (; addr < end; addr += size) { + unsigned int physical_offset = addr < vmf->address ? + offset - (vmf->address - addr) : offset + (addr - vmf->address); + pte_t *ptep = vmf->pte + ((long)(addr - vmf->address) >> mm_page_shift(mm)); + pte_t entry; + + if (!pte_none(ptep_get(ptep)) && + !(addr == vmf->address && !vmf_pte_changed(vmf))) + continue; + entry = phys_pte_mm(mm, page_to_phys(page) + physical_offset, vma->vm_page_prot); + entry = pte_sw_mkyoung(entry); + if (vmf->flags & FAULT_FLAG_WRITE) + entry = maybe_mkwrite(pte_mkdirty(entry), vma); + if (addr == vmf->address && vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + folio_get(folio); + mm_subpage_file_map_add(vma, pte_phys_mm(vma->vm_mm, entry)); + folio_add_file_rmap_pte(folio, page, vma); + set_pte_at(mm, addr, ptep, entry); + update_mmu_cache_range(vmf, vma, addr, ptep, 1); + mapped++; + if (addr == vmf->address) + ret = VM_FAULT_NOPAGE; + } + add_mm_counter(mm, mm_counter_file(folio), mapped); + return ret; +} + +static vm_fault_t finish_user_page_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *slot = vmf->cow_subpage; + bool is_cow = (vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED); + struct folio *folio = page_folio(is_cow ? vmf->cow_page : vmf->page); + vm_fault_t ret; + pte_t entry; + + if (!(vma->vm_flags & VM_SHARED)) { + ret = check_stable_address_space(mm); + if (ret) + return ret; + } + if (pmd_none(*vmf->pmd)) { + if (vmf->prealloc_pte) + pmd_install(mm, vmf->pmd, &vmf->prealloc_pte); + else if (pte_alloc(mm, vmf->pmd)) + return VM_FAULT_OOM; + } + /* The private destination is not yet visible through a PTE. */ + if (is_cow) { + folio_get(folio); + folio_lock(folio); + } + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + ret = VM_FAULT_NOPAGE; + if (!vmf->pte) + goto unlock_folio; + if (vmf_pte_changed(vmf)) { + update_mmu_tlb(vma, vmf->address, vmf->pte); + goto unlock; + } + if (slot) { + mm_subpage_set_exclusive(slot); + if (WARN_ON_ONCE(mm_subpage_add_new_anon_rmap(slot, vma, vmf->address))) { + ret = VM_FAULT_SIGBUS; + goto unlock; + } + entry = phys_pte_mm(mm, mm_subpage_phys(slot), vma->vm_page_prot); + entry = maybe_mkwrite(pte_mkdirty(pte_sw_mkyoung(entry)), vma); + if (vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + subpage_add_new_lru(folio, vma, 1); + set_pte_at(mm, vmf->address, vmf->pte, entry); + update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + inc_mm_counter(mm, MM_ANONPAGES); + vmf->cow_subpage = NULL; + ret = 0; + } else if (is_cow) { + if (!folio_user_range_valid(folio, 0, mm_page_size(mm))) { + ret = VM_FAULT_SIGBUS; + goto unlock; + } + entry = phys_pte_mm(mm, page_to_phys(&folio->page), vma->vm_page_prot); + entry = maybe_mkwrite(pte_mkdirty(pte_sw_mkyoung(entry)), vma); + if (vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + folio_ref_add(folio, mm_pte_native_pages(mm) - 1); + folio_add_new_anon_rmap(folio, vma, vmf->address, RMAP_EXCLUSIVE); + folio_add_lru_vma(folio, vma); + set_pte_at(mm, vmf->address, vmf->pte, entry); + update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + inc_mm_counter(mm, MM_ANONPAGES); + ret = 0; + } else { + if (vma->vm_file) + ret = map_file_user_pages(vmf, folio, folio->index, + folio_next_index(folio) - 1, true); + else if (mm_page_size(mm) < PAGE_SIZE) + ret = map_special_subpages(vmf); + else + ret = VM_FAULT_SIGBUS; + if (ret & VM_FAULT_NOPAGE) { + folio_put(folio); /* Consume the fault callback's reference. */ + ret = 0; + } else { + ret = VM_FAULT_SIGBUS; + } + } +unlock: + pte_unmap_unlock(vmf->pte, vmf->ptl); +unlock_folio: + if (is_cow) { + folio_unlock(folio); + folio_put(folio); + } + return ret; +} +#endif + /** * finish_fault - finish page fault once we have prepared the page to fault * @@ -5614,6 +6950,11 @@ vm_fault_t finish_fault(struct vm_fault *vmf) unsigned long addr; bool needs_fallback = false; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return finish_user_page_fault(vmf); +#endif + fallback: addr = vmf->address; @@ -5719,13 +7060,12 @@ vm_fault_t finish_fault(struct vm_fault *vmf) return ret; } -static unsigned long fault_around_pages __read_mostly = - 65536 >> PAGE_SHIFT; +static unsigned long fault_around_bytes __read_mostly = SZ_64K; #ifdef CONFIG_DEBUG_FS static int fault_around_bytes_get(void *data, u64 *val) { - *val = fault_around_pages << PAGE_SHIFT; + *val = READ_ONCE(fault_around_bytes); return 0; } @@ -5739,11 +7079,13 @@ static int fault_around_bytes_set(void *data, u64 val) return -EINVAL; /* - * The minimum value is 1 page, however this results in no fault-around - * at all. See should_fault_around(). + * Keep the policy in bytes: one native page can still contain several + * user leaves. The smallest supported leaf disables fault-around for + * every mm; larger values are compared with each mm's own granule. */ - val = max(val, PAGE_SIZE); - fault_around_pages = rounddown_pow_of_two(val) >> PAGE_SHIFT; + val = max_t(u64, val, IS_ENABLED(CONFIG_MM_SUBPAGE) ? + MM_SUBPAGE_SIZE : PAGE_SIZE); + WRITE_ONCE(fault_around_bytes, rounddown_pow_of_two(val)); return 0; } @@ -5770,24 +7112,42 @@ late_initcall(fault_around_debugfs); * This function doesn't cross VMA or page table boundaries, in order to call * map_pages() and acquire a PTE lock only once. * - * fault_around_pages defines how many pages we'll try to map. - * do_fault_around() expects it to be set to a power of two less than or equal - * to PTRS_PER_PTE. - * - * The virtual address of the area that we map is naturally aligned to - * fault_around_pages * PAGE_SIZE rounded down to the machine page size - * (and therefore to page order). This way it's easier to guarantee - * that we don't cross page table boundaries. + * fault_around_bytes is a power-of-two byte policy, independent of either + * physical or user granules. The native path converts it to native pages; + * the alternative path clamps it to the target mm's PTE-table coverage. + * Natural alignment keeps the window inside one page table. */ static vm_fault_t do_fault_around(struct vm_fault *vmf) { - pgoff_t nr_pages = READ_ONCE(fault_around_pages); + unsigned long configured_bytes = READ_ONCE(fault_around_bytes); + pgoff_t nr_pages = max(1UL, configured_bytes >> PAGE_SHIFT); pgoff_t pte_off = pte_index(vmf->address); /* The page offset of vmf->address within the VMA. */ pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff; pgoff_t from_pte, to_pte; vm_fault_t ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vmf->vma->vm_mm) != PAGE_SIZE) { + struct vm_area_struct *vma = vmf->vma; + unsigned long bytes = min(configured_bytes, + pmd_size_mm(vma->vm_mm)); + unsigned long start = max(ALIGN_DOWN(vmf->address, bytes), vma->vm_start); + unsigned long end = min(ALIGN_DOWN(vmf->address, bytes) + bytes, vma->vm_end); + + if (pmd_none(*vmf->pmd)) { + vmf->prealloc_pte = pte_alloc_one(vma->vm_mm); + if (!vmf->prealloc_pte) + return VM_FAULT_OOM; + } + rcu_read_lock(); + ret = vma->vm_ops->map_pages(vmf, vma_page_offset_at(vma, start).index, + vma_page_offset_at(vma, end - 1).index); + rcu_read_unlock(); + return ret; + } +#endif + /* The PTE offset of the start address, clamped to the VMA. */ from_pte = max(ALIGN_DOWN(pte_off, nr_pages), pte_off - min(pte_off, vma_off)); @@ -5822,7 +7182,7 @@ static inline bool should_fault_around(struct vm_fault *vmf) return false; /* A single page implies no faulting 'around' at all. */ - return fault_around_pages > 1; + return READ_ONCE(fault_around_bytes) > mm_page_size(vmf->vma->vm_mm); } static vm_fault_t do_read_fault(struct vm_fault *vmf) @@ -5869,10 +7229,26 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) if (ret) return ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + vmf->cow_subpage = anon_subpage_prealloc(vma->vm_mm, vma, vmf->address); + if (IS_ERR(vmf->cow_subpage)) { + vmf->cow_subpage = NULL; + return VM_FAULT_OOM; + } + folio = mm_subpage_folio(vmf->cow_subpage); + goto allocated; + } +#endif + folio = folio_prealloc(vma->vm_mm, vma, vmf->address, false); if (!folio) return VM_FAULT_OOM; +#ifdef CONFIG_MM_SUBPAGE +allocated: +#endif + vmf->cow_page = &folio->page; ret = __do_fault(vmf); @@ -5881,10 +7257,38 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) if (ret & VM_FAULT_DONE_COW) return ret; - if (copy_mc_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma)) { - ret = VM_FAULT_HWPOISON; +#ifdef CONFIG_MM_SUBPAGE + if (vmf->cow_subpage) { + if (copy_user_subpage_range(vmf->cow_page, mm_subpage_offset(vmf->cow_subpage), + vmf->page, vma_page_offset_at(vma, vmf->address).offset, + mm_page_size(vma->vm_mm))) { + ret = VM_FAULT_HWPOISON; + goto unlock; + } + goto copied; + } +#endif + +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) > PAGE_SIZE && + !folio_user_range_valid(page_folio(vmf->page), + folio_page_idx(page_folio(vmf->page), vmf->page) << PAGE_SHIFT, + mm_page_size(vma->vm_mm))) { + ret = VM_FAULT_SIGBUS; goto unlock; } +#endif + for (unsigned int i = 0; i < mm_pte_native_pages(vma->vm_mm); i++) { + if (copy_mc_user_highpage(vmf->cow_page + i, vmf->page + i, + vmf->address + i * PAGE_SIZE, vma)) { + ret = VM_FAULT_HWPOISON; + goto unlock; + } + } +#ifdef CONFIG_MM_SUBPAGE +copied: +#endif + __folio_mark_uptodate(folio); ret |= finish_fault(vmf); @@ -5895,6 +7299,13 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) goto uncharge_out; return ret; uncharge_out: +#ifdef CONFIG_MM_SUBPAGE + if (vmf->cow_subpage) { + mm_subpage_put(vmf->cow_subpage); + vmf->cow_subpage = NULL; + return ret; + } +#endif folio_put(folio); return ret; } @@ -6055,31 +7466,33 @@ static void numa_rebuild_single_mapping(struct vm_fault *vmf, struct vm_area_str update_mmu_cache_range(vmf, vma, fault_addr, fault_pte, 1); } -static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_struct *vma, +static void numa_rebuild_folio_mapping(struct vm_fault *vmf, struct vm_area_struct *vma, struct folio *folio, pte_t fault_pte, bool ignore_writable, bool pte_write_upgrade) { - int nr = pte_pfn(fault_pte) - folio_pfn(folio); + struct mm_struct *mm = vma->vm_mm; + unsigned int page_shift = mm_page_shift(mm); + unsigned long page_size = 1UL << page_shift; + unsigned long offset = pte_phys_mm(mm, fault_pte) - PFN_PHYS(folio_pfn(folio)); unsigned long start, end, addr = vmf->address; - unsigned long addr_start = addr - (nr << PAGE_SHIFT); - unsigned long pt_start = ALIGN_DOWN(addr, PMD_SIZE); + unsigned long lower = max(addr & pmd_mask_mm(mm), vma->vm_start); + unsigned long upper = pmd_addr_end_mm(mm, addr, vma->vm_end); pte_t *start_ptep; - /* Stay within the VMA and within the page table. */ - start = max3(addr_start, pt_start, vma->vm_start); - end = min3(addr_start + folio_size(folio), pt_start + PMD_SIZE, - vma->vm_end); - start_ptep = vmf->pte - ((addr - start) >> PAGE_SHIFT); + /* Stay within the VMA, page table and physical owner without wrapping. */ + start = addr - min(offset, addr - lower); + end = addr + min(folio_size(folio) - offset, upper - addr); + start_ptep = vmf->pte - ((addr - start) >> page_shift); - /* Restore all PTEs' mapping of the large folio */ - for (addr = start; addr != end; start_ptep++, addr += PAGE_SIZE) { + /* Restore the owner's PTEs, including smaller leaves in a native folio. */ + for (addr = start; addr != end; start_ptep++, addr += page_size) { pte_t ptent = ptep_get(start_ptep); bool writable = false; if (!pte_present(ptent) || !pte_protnone(ptent)) continue; - if (pfn_folio(pte_pfn(ptent)) != folio) + if (pfn_folio(PHYS_PFN(pte_phys_mm(mm, ptent))) != folio) continue; if (!ignore_writable) { @@ -6172,8 +7585,8 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf) * Make it present again, depending on how arch implements * non-accessible ptes, some can allow access by kernel mode. */ - if (folio && folio_test_large(folio)) - numa_rebuild_large_mapping(vmf, vma, folio, pte, ignore_writable, + if (folio && (folio_test_large(folio) || mm_page_shift(vma->vm_mm) < PAGE_SHIFT)) + numa_rebuild_folio_mapping(vmf, vma, folio, pte, ignore_writable, pte_write_upgrade); else numa_rebuild_single_mapping(vmf, vma, vmf->address, vmf->pte, @@ -6407,7 +7820,7 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, { struct vm_fault vmf = { .vma = vma, - .address = address & PAGE_MASK, + .address = address & mm_page_mask(vma->vm_mm), .real_address = address, .flags = flags, .pgoff = linear_page_index(vma, address), @@ -6738,7 +8151,7 @@ int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address) return -ENOMEM; spin_lock(&mm->page_table_lock); - if (!p4d_present(*p4d)) { + if (!p4d_present_mm(mm, *p4d)) { mm_inc_nr_puds(mm); smp_wmb(); /* See comment in pmd_install() */ p4d_populate(mm, p4d, new); @@ -6776,16 +8189,19 @@ int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) static inline int pfnmap_args_setup(struct follow_pfnmap_args *args, spinlock_t *lock, pte_t *ptep, - pgprot_t pgprot, unsigned long pfn_base, + pgprot_t pgprot, phys_addr_t phys_base, unsigned long addr_mask, bool writable, bool special) { + phys_addr_t phys = phys_base + (args->address & ~addr_mask); + if (!writable && args->write) return -EFAULT; args->lock = lock; args->ptep = ptep; - args->pfn = pfn_base + ((args->address & ~addr_mask) >> PAGE_SHIFT); + args->pfn = PHYS_PFN(phys); + args->offset = offset_in_page(phys); args->addr_mask = addr_mask; args->pgprot = pgprot; args->writable = writable; @@ -6837,8 +8253,14 @@ static inline void pfnmap_lockdep_assert(struct vm_area_struct *vma) * * This function must not be used to modify PTE content. * + * Legacy callers retain the native-page contract. A caller setting + * allow_subpage must use pfn plus offset to identify the requested physical + * byte and must limit access to the leaf described by addr_mask. This does + * not pin the native page or grant access to neighbouring user mappings. + * * Return: zero on success, -EFAULT if @args->write was set but the - * mapping is not writable, -EINVAL if there is no mapping at all. + * mapping is not writable, -EINVAL if there is no mapping at all, or + * -EOPNOTSUPP for a non-native granule without allow_subpage. */ int follow_pfnmap_start(struct follow_pfnmap_args *args) { @@ -6860,17 +8282,20 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) goto out; + /* A legacy PFN consumer cannot represent partial native-page ownership. */ + if (mm_page_size(mm) != PAGE_SIZE && !args->allow_subpage) + return -EOPNOTSUPP; retry: pgdp = pgd_offset(mm, address); if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp))) goto out; - p4dp = p4d_offset(pgdp, address); + p4dp = p4d_offset_mm(mm, pgdp, address); p4d = p4dp_get(p4dp); - if (p4d_none(p4d) || unlikely(p4d_bad(p4d))) + if (p4d_none_mm(mm, p4d) || unlikely(p4d_bad_mm(mm, p4d))) goto out; - pudp = pud_offset(p4dp, address); + pudp = pud_offset_mm(mm, p4dp, address); pud = pudp_get(pudp); if (!pud_present(pud)) goto out; @@ -6886,14 +8311,14 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) goto retry; } r = pfnmap_args_setup(args, lock, NULL, pud_pgprot(pud), - pud_pfn(pud), PUD_MASK, pud_write(pud), + PFN_PHYS(pud_pfn(pud)), pud_mask_mm(mm), pud_write(pud), pud_special(pud)); if (r) spin_unlock(lock); return r; } - pmdp = pmd_offset(pudp, address); + pmdp = pmd_offset_mm(mm, pudp, address); pmd = pmdp_get_lockless(pmdp); if (!pmd_present(pmd)) goto out; @@ -6909,7 +8334,7 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) goto retry; } r = pfnmap_args_setup(args, lock, NULL, pmd_pgprot(pmd), - pmd_pfn(pmd), PMD_MASK, pmd_write(pmd), + PFN_PHYS(pmd_pfn(pmd)), pmd_mask_mm(mm), pmd_write(pmd), pmd_special(pmd)); if (r) spin_unlock(lock); @@ -6922,8 +8347,8 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) pte = ptep_get(ptep); if (!pte_present(pte)) goto unlock; - r = pfnmap_args_setup(args, lock, ptep, pte_pgprot(pte), - pte_pfn(pte), PAGE_MASK, pte_write(pte), + r = pfnmap_args_setup(args, lock, ptep, pte_pgprot_mm(mm, pte), + pte_phys_mm(mm, pte), mm_page_mask(mm), pte_write(pte), pte_special(pte)); if (r) goto unlock; @@ -6970,20 +8395,30 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, resource_size_t phys_addr; pgprot_t prot = __pgprot(0); void __iomem *maddr; - int offset = offset_in_page(addr); + int offset; int ret = -EINVAL; bool writable; struct follow_pfnmap_args args = { .vma = vma, .address = addr, - .write = !!(write & FOLL_WRITE) + .write = !!(write & FOLL_WRITE), + .allow_subpage = true }; + if (len <= 0) + return 0; + if (addr < vma->vm_start || addr >= vma->vm_end) + return -EINVAL; + /* Each following leaf needs its own translation and permission check. */ + len = min_t(unsigned long, len, vma->vm_end - addr); + len = min_t(unsigned long, len, mm_page_size(vma->vm_mm) - + (addr & ~mm_page_mask(vma->vm_mm))); retry: if (follow_pfnmap_start(&args)) return -EINVAL; prot = args.pgprot; phys_addr = (resource_size_t)args.pfn << PAGE_SHIFT; + offset = args.offset; writable = args.writable; follow_pfnmap_end(&args); @@ -6995,7 +8430,8 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, goto out_unmap; if ((pgprot_val(prot) != pgprot_val(args.pgprot)) || - (phys_addr != (args.pfn << PAGE_SHIFT)) || + (phys_addr != ((resource_size_t)args.pfn << PAGE_SHIFT)) || + (offset != args.offset) || (writable != args.writable)) { follow_pfnmap_end(&args); iounmap(maddr); @@ -7041,8 +8477,19 @@ static int __access_remote_vm(struct mm_struct *mm, unsigned long addr, void *maddr; struct folio *folio; struct vm_area_struct *vma = NULL; - struct page *page = get_user_page_vma_remote(mm, addr, - gup_flags, &vma); + struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment = {}; + + if (mm_page_size(mm) != PAGE_SIZE) { + int got = get_user_fragment_vma_remote(mm, addr, len, + gup_flags, &fragment, &vma); + + page = got > 0 ? folio_page(fragment.folio, + fragment.offset >> PAGE_SHIFT) : ERR_PTR(got ?: -EFAULT); + } else +#endif + page = get_user_page_vma_remote(mm, addr, gup_flags, &vma); if (IS_ERR(page)) { /* We might need to expand the stack to access it */ @@ -7076,6 +8523,12 @@ static int __access_remote_vm(struct mm_struct *mm, unsigned long addr, offset = addr & (PAGE_SIZE-1); if (bytes > PAGE_SIZE-offset) bytes = PAGE_SIZE-offset; +#ifdef CONFIG_MM_SUBPAGE + if (fragment.folio) { + offset = offset_in_page(fragment.offset); + bytes = fragment.length; + } +#endif maddr = kmap_local_folio(folio, folio_page_idx(folio, page) * PAGE_SIZE); if (write) { @@ -7086,7 +8539,13 @@ static int __access_remote_vm(struct mm_struct *mm, unsigned long addr, copy_from_user_page(vma, page, addr, buf, maddr + offset, bytes); } - folio_release_kmap(folio, maddr); +#ifdef CONFIG_MM_SUBPAGE + if (fragment.folio) { + kunmap_local(maddr); + release_user_fragments(&fragment, 1, false); + } else +#endif + folio_release_kmap(folio, maddr); } len -= bytes; buf += bytes; diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 9d7cb63d32732..f94b1599fc14b 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -78,6 +78,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include #include #include #include @@ -690,6 +691,8 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, struct folio *folio; struct queue_pages *qp = walk->private; unsigned long flags = qp->flags; + unsigned int page_shift = mm_page_shift(walk->mm); + unsigned long page_size = 1UL << page_shift; pte_t *pte, *mapped_pte; pte_t ptent; spinlock_t *ptl; @@ -707,8 +710,8 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; pte += nr, addr += nr * PAGE_SIZE) { - max_nr = (end - addr) >> PAGE_SHIFT; + for (; addr != end; pte += nr, addr += nr * page_size) { + max_nr = (end - addr) >> page_shift; nr = 1; ptent = ptep_get(pte); if (pte_none(ptent)) @@ -723,7 +726,11 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, folio = vm_normal_folio(vma, addr, ptent); if (!folio || folio_is_zone_device(folio)) continue; - if (folio_test_large(folio) && max_nr != 1) +#ifdef CONFIG_MM_SUBPAGE + if (page_shift < PAGE_SHIFT) + nr = folio_subpage_pte_batch(vma, folio, pte, ptent, max_nr, 0); +#endif + if (page_shift >= PAGE_SHIFT && folio_test_large(folio) && max_nr != 1) nr = folio_pte_batch(folio, pte, ptent, max_nr); /* * vm_normal_folio() filters out zero pages, but there might @@ -733,9 +740,10 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, continue; if (!queue_folio_required(folio, qp)) continue; - if (folio_test_large(folio)) { + if (folio_test_large(folio) || page_shift < PAGE_SHIFT) { /* - * A large folio can only be isolated from LRU once, + * A native folio backing smaller leaves, like a large + * folio, can only be isolated from LRU once, * but may be mapped by many PTEs (and Copy-On-Write may * intersperse PTEs of other, order 0, folios). This is * a common case, so don't mistake it for failure (but @@ -757,7 +765,7 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, if (!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) || !vma_migratable(vma) || !migrate_folio_add(folio, qp->pagelist, flags)) { - qp->nr_failed += nr; + qp->nr_failed += page_shift < PAGE_SHIFT ? 1 : nr; if (strictly_unmovable(flags)) break; } @@ -826,6 +834,7 @@ static int queue_folios_hugetlb(pte_t *pte, unsigned long hmask, * @folio: The folio whose mapping considered for being made NUMA hintable * @vma: The VMA that the folio belongs to. * @is_private_single_threaded: Is this a single-threaded private VMA or not + * @shared: Is the mapped memory potentially shared between address spaces * * This function checks to see if the folio actually indicates that * we need to make the mapping one which causes a NUMA hinting fault, @@ -835,7 +844,7 @@ static int queue_folios_hugetlb(pte_t *pte, unsigned long hmask, * Return: True if the mapping of the folio needs to be changed, false otherwise. */ bool folio_can_map_prot_numa(struct folio *folio, struct vm_area_struct *vma, - bool is_private_single_threaded) + bool is_private_single_threaded, bool shared) { int nid; @@ -843,7 +852,7 @@ bool folio_can_map_prot_numa(struct folio *folio, struct vm_area_struct *vma, return false; /* Also skip shared copy-on-write folios */ - if (is_cow_mapping(vma->vm_flags) && folio_maybe_mapped_shared(folio)) + if (is_cow_mapping(vma->vm_flags) && shared) return false; /* Folios are pinned and can't be migrated */ @@ -1052,6 +1061,12 @@ static int mbind_range(struct vma_iterator *vmi, struct vm_area_struct *vma, vmstart = vma->vm_start; } + /* Shared policy is indexed by native page-cache allocation units. */ + if (vma_is_shmem(vma) && + (vma_page_offset_at(vma, vmstart).offset || + ((vmend - vmstart) & ~PAGE_MASK))) + return -EINVAL; + if (mpol_equal(vma->vm_policy, new_pol)) { *prev = vma; return 0; @@ -1137,6 +1152,18 @@ static int lookup_node(struct mm_struct *mm, unsigned long addr) struct page *p = NULL; int ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_shift(mm) != PAGE_SHIFT) { + struct user_page_fragment fragment; + + ret = get_user_fragments_remote(mm, addr, 1, 0, &fragment, 1); + if (ret > 0) { + ret = folio_nid(fragment.folio); + release_user_fragments(&fragment, 1, false); + } + return ret; + } +#endif ret = get_user_pages_fast(addr & PAGE_MASK, 1, 0, &p); if (ret > 0) { ret = page_to_nid(p); @@ -1504,13 +1531,13 @@ static long do_mbind(unsigned long start, unsigned long len, if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE)) return -EPERM; - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(mm)) return -EINVAL; if (mode == MPOL_DEFAULT) flags &= ~MPOL_MF_STRICT; - len = PAGE_ALIGN(len); + len = mm_page_align(mm, len); end = start + len; if (end < start) @@ -1770,7 +1797,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le VMA_ITERATOR(vmi, mm, start); start = untagged_addr(start); - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(mm)) return -EINVAL; /* * flags is used for future extension if any. @@ -1785,7 +1812,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le if (home_node >= MAX_NUMNODES || !node_online(home_node)) return -EINVAL; - len = PAGE_ALIGN(len); + len = mm_page_align(mm, len); end = start + len; if (end < start) @@ -2051,7 +2078,7 @@ struct mempolicy *get_vma_policy(struct vm_area_struct *vma, if (pol->mode == MPOL_INTERLEAVE || pol->mode == MPOL_WEIGHTED_INTERLEAVE) { *ilx += vma->vm_pgoff >> order; - *ilx += (addr - vma->vm_start) >> (PAGE_SHIFT + order); + *ilx += (vma_page_offset_at(vma, addr).index - vma->vm_pgoff) >> order; } return pol; } @@ -3247,6 +3274,10 @@ int mpol_set_shared_policy(struct shared_policy *sp, struct sp_node *new = NULL; unsigned long sz = vma_pages(vma); + /* Never truncate a partial cache-page policy update to zero or a neighbour. */ + if (vma_subpage_offset(vma) || ((vma->vm_end - vma->vm_start) & ~PAGE_MASK)) + return -EINVAL; + if (pol) { new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, pol); if (!new) diff --git a/mm/migrate.c b/mm/migrate.c index 35381ce36a8f5..68b697558edde 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -14,6 +14,8 @@ */ #include +#include +#include #include #include #include @@ -340,14 +342,50 @@ struct rmap_walk_arg { bool map_unused_to_zeropage; }; +#ifdef CONFIG_MM_SUBPAGE +static void remove_migration_subpage(struct folio *folio, + struct page_vma_mapped_walk *pvmw, pte_t old) +{ + struct vm_area_struct *vma = pvmw->vma; + softleaf_t entry = softleaf_from_pte(old); + struct mm_subpage *slot; + pte_t pte; + int ret; + + slot = mm_subpage_migrate_restore(folio, pte_swp_subpage_offset(old)); + pte = phys_pte_mm(vma->vm_mm, mm_subpage_phys(slot), READ_ONCE(vma->vm_page_prot)); + if (!softleaf_is_migration_young(entry)) + pte = pte_mkold(pte); + if (folio_test_dirty(folio) && softleaf_is_migration_dirty(entry)) + pte = pte_mkdirty(pte); + if (pte_swp_soft_dirty(old)) + pte = pte_mksoft_dirty(pte); + else + pte = pte_clear_soft_dirty(pte); + if (softleaf_is_migration_write(entry)) + pte = pte_mkwrite(pte, vma); + else if (pte_swp_uffd_wp(old)) + pte = pte_mkuffd_wp(pte); + if (!softleaf_is_migration_read(entry)) + mm_subpage_set_exclusive(slot); + ret = mm_subpage_add_anon_rmap(slot, vma, pvmw->address); + VM_BUG_ON_FOLIO(ret, folio); + set_pte_at(vma->vm_mm, pvmw->address, pvmw->pte, pte); + if (READ_ONCE(vma->vm_flags) & VM_LOCKED) + mlock_drain_local(); + trace_remove_migration_pte(pvmw->address, pte_val(pte), 0); + update_mmu_cache(vma, pvmw->address, pvmw->pte); +} +#endif + /* * Restore a potential migration pte to a working pte entry */ static bool remove_migration_pte(struct folio *folio, - struct vm_area_struct *vma, unsigned long addr, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { struct rmap_walk_arg *rmap_walk_arg = arg; - DEFINE_FOLIO_VMA_WALK(pvmw, rmap_walk_arg->folio, vma, addr, PVMW_SYNC | PVMW_MIGRATION); + DEFINE_FOLIO_RMAP_WALK(pvmw, rmap_walk_arg->folio, vma, rmap, PVMW_SYNC | PVMW_MIGRATION); while (page_vma_mapped_walk(&pvmw)) { rmap_t rmap_flags = RMAP_NONE; @@ -376,13 +414,29 @@ static bool remove_migration_pte(struct folio *folio, pvmw.pte); else old_pte = ptep_get(pvmw.pte); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE && folio_test_anon(folio)) { + remove_migration_subpage(folio, &pvmw, old_pte); + continue; + } +#endif if (rmap_walk_arg->map_unused_to_zeropage && try_to_map_unused_to_zeropage(&pvmw, folio, old_pte, idx)) continue; - folio_get(folio); + /* Match the native-page references dropped by try_to_migrate_one(). */ + if (!folio_test_hugetlb(folio)) + folio_ref_add(folio, mm_pte_native_pages(vma->vm_mm)); + else + folio_get(folio); pte = mk_pte(new, READ_ONCE(vma->vm_page_prot)); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + pte = phys_pte_mm(vma->vm_mm, page_to_phys(new) + + pte_swp_subpage_offset(old_pte), READ_ONCE(vma->vm_page_prot)); +#endif + entry = softleaf_from_pte(old_pte); if (!softleaf_is_migration_young(entry)) pte = pte_mkold(pte); @@ -435,8 +489,10 @@ static bool remove_migration_pte(struct folio *folio, if (folio_test_anon(folio)) folio_add_anon_rmap_pte(folio, new, vma, pvmw.address, rmap_flags); - else + else { + mm_subpage_file_map_add(vma, pte_phys_mm(vma->vm_mm, pte)); folio_add_file_rmap_pte(folio, new, vma); + } set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); } if (READ_ONCE(vma->vm_flags) & VM_LOCKED) @@ -645,10 +701,12 @@ static int __folio_migrate_mapping(struct address_space *mapping, folio_set_dirty(newfolio); } - if (folio_test_swapcache(folio)) + if (folio_test_swapcache(folio)) { __swap_cache_replace_folio(ci, folio, newfolio); - else + } else { + shmem_uffd_migrate(newfolio, folio); xas_store(&xas, newfolio); + } /* * Drop cache reference from old folio by unfreezing @@ -1173,8 +1231,10 @@ static void migrate_folio_undo_src(struct folio *src, /* Drop an anon_vma reference if we took one */ if (anon_vma) put_anon_vma(anon_vma); - if (locked) + if (locked) { + mm_subpage_migrate_finish(src); folio_unlock(src); + } if (ret) list_move_tail(&src->lru, ret); } @@ -1183,8 +1243,10 @@ static void migrate_folio_undo_src(struct folio *src, static void migrate_folio_undo_dst(struct folio *dst, bool locked, free_folio_t put_new_folio, unsigned long private) { - if (locked) + if (locked) { + mm_subpage_migrate_finish(dst); folio_unlock(dst); + } if (put_new_folio) put_new_folio(dst, private); else @@ -1309,6 +1371,11 @@ static int migrate_folio_unmap(new_folio_t get_new_folio, return 0; } + rc = mm_subpage_migrate_prepare(src, dst); + if (rc) + goto out; + rc = -EAGAIN; + /* * Corner case handling: * 1. When a new swap-cache page is read into, it is added to the LRU @@ -1414,6 +1481,8 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private, remove_migration_ptes(src, dst, 0); out_unlock_both: + mm_subpage_migrate_finish(src); + mm_subpage_migrate_finish(dst); folio_unlock(dst); folio_set_owner_migrate_reason(dst, reason); /* diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 52ba68f2c9151..aa9407397e5a8 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -408,7 +408,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp, if (anon_exclusive) { pte = ptep_clear_flush(vma, addr, ptep); - if (folio_try_share_anon_rmap_pte(folio, page)) { + if (folio_try_share_anon_rmap_pte(folio, page, vma)) { set_pte_at(mm, addr, ptep, pte); if (fault_folio != folio) folio_unlock(folio); diff --git a/mm/mincore.c b/mm/mincore.c index c8757c5085bf9..66458b6bb2091 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -48,7 +48,7 @@ static int mincore_hugetlb(pte_t *pte, unsigned long hmask, unsigned long addr, present = 1; } - for (; addr != end; vec++, addr += PAGE_SIZE) + for (; addr != end; vec++, addr += mm_page_size(walk->mm)) *vec = present; walk->private = vec; spin_unlock(ptl); @@ -135,15 +135,22 @@ static unsigned char mincore_page(struct address_space *mapping, pgoff_t index) static int __mincore_unmapped_range(unsigned long addr, unsigned long end, struct vm_area_struct *vma, unsigned char *vec) { - unsigned long nr = (end - addr) >> PAGE_SHIFT; + unsigned long nr = (end - addr) >> mm_page_shift(vma->vm_mm); int i; if (vma->vm_file) { - pgoff_t pgoff; + unsigned int shift = mm_page_shift(vma->vm_mm); - pgoff = linear_page_index(vma, addr); - for (i = 0; i < nr; i++, pgoff++) - vec[i] = mincore_page(vma->vm_file->f_mapping, pgoff); + /* Cache indices are native; several user leaves can share one. */ + for (i = 0; i < nr; ) { + struct vm_page_offset pos = vma_page_offset_at(vma, addr); + unsigned long batch = min(nr - i, (PAGE_SIZE - pos.offset) >> shift); + unsigned char present = mincore_page(vma->vm_file->f_mapping, pos.index); + + memset(vec + i, present, batch); + i += batch; + addr += batch << shift; + } } else { for (i = 0; i < nr; i++) vec[i] = 0; @@ -167,7 +174,7 @@ static int mincore_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, struct vm_area_struct *vma = walk->vma; pte_t *ptep; unsigned char *vec = walk->private; - int nr = (end - addr) >> PAGE_SHIFT; + int nr = (end - addr) >> mm_page_shift(walk->mm); int step, i; ptl = pmd_trans_huge_lock(pmd, vma); @@ -182,19 +189,19 @@ static int mincore_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; ptep += step, addr += step * PAGE_SIZE) { + for (; addr != end; ptep += step, addr += step * mm_page_size(walk->mm)) { pte_t pte = ptep_get(ptep); step = 1; /* We need to do cache lookup too for markers */ if (pte_none(pte) || pte_is_marker(pte)) - __mincore_unmapped_range(addr, addr + PAGE_SIZE, + __mincore_unmapped_range(addr, addr + mm_page_size(walk->mm), vma, vec); else if (pte_present(pte)) { unsigned int batch = pte_batch_hint(ptep, pte); if (batch > 1) { - unsigned int max_nr = (end - addr) >> PAGE_SHIFT; + unsigned int max_nr = (end - addr) >> mm_page_shift(walk->mm); step = min_t(unsigned int, batch, max_nr); } @@ -252,16 +259,16 @@ static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *v vma = vma_lookup(current->mm, addr); if (!vma) return -ENOMEM; - end = min(vma->vm_end, addr + (pages << PAGE_SHIFT)); + end = min(vma->vm_end, addr + (pages << mm_page_shift(vma->vm_mm))); if (!can_do_mincore(vma)) { - unsigned long pages = DIV_ROUND_UP(end - addr, PAGE_SIZE); + unsigned long pages = DIV_ROUND_UP(end - addr, mm_page_size(vma->vm_mm)); memset(vec, 1, pages); return pages; } err = walk_page_range(vma->vm_mm, addr, end, &mincore_walk_ops, vec); if (err < 0) return err; - return (end - addr) >> PAGE_SHIFT; + return (end - addr) >> mm_page_shift(vma->vm_mm); } /* @@ -281,7 +288,7 @@ static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *v * return values: * zero - success * -EFAULT - vec points to an illegal address - * -EINVAL - addr is not a multiple of PAGE_SIZE + * -EINVAL - addr is not aligned to the process page size * -ENOMEM - Addresses in the range [addr, addr + len] are * invalid for the address space of this process, or * specify one or more pages which are not currently @@ -298,7 +305,7 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, start = untagged_addr(start); /* Check the start address: needs to be page-aligned.. */ - if (unlikely(start & ~PAGE_MASK)) + if (unlikely(start & ~mm_page_mask(current->mm))) return -EINVAL; /* ..and we need to be passed a valid user-space range */ @@ -306,8 +313,8 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, return -ENOMEM; /* This also avoids any overflows on PAGE_ALIGN */ - pages = len >> PAGE_SHIFT; - pages += (offset_in_page(len)) != 0; + pages = len >> mm_page_shift(current->mm); + pages += (len & ~mm_page_mask(current->mm)) != 0; if (!access_ok(vec, pages)) return -EFAULT; @@ -334,7 +341,7 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, } pages -= retval; vec += retval; - start += retval << PAGE_SHIFT; + start += retval << mm_page_shift(current->mm); retval = 0; } free_page((unsigned long) tmp); diff --git a/mm/mlock.c b/mm/mlock.c index 2d8f26a1cf263..e3d20947535b0 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -304,15 +304,19 @@ void munlock_folio(struct folio *folio) local_unlock(&mlock_fbatch.lock); } -static inline unsigned int folio_mlock_step(struct folio *folio, +static inline unsigned int folio_mlock_step(struct vm_area_struct *vma, struct folio *folio, pte_t *pte, unsigned long addr, unsigned long end) { - unsigned int count = (end - addr) >> PAGE_SHIFT; + unsigned int count = (end - addr) >> mm_page_shift(vma->vm_mm); pte_t ptent = ptep_get(pte); if (!folio_test_large(folio)) return 1; + #ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return folio_subpage_pte_batch(vma, folio, pte, ptent, count, 0); +#endif return folio_pte_batch(folio, pte, ptent, count); } @@ -341,7 +345,7 @@ static inline bool allow_mlock_munlock(struct folio *folio, return false; /* folio is not fully mapped, skip mlock */ - if (step != folio_nr_pages(folio)) + if (step != (folio_size(folio) >> mm_page_shift(vma->vm_mm))) return false; return true; @@ -381,7 +385,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, return 0; } - for (pte = start_pte; addr != end; pte++, addr += PAGE_SIZE) { + for (pte = start_pte; addr != end; pte++, addr += mm_page_size(vma->vm_mm)) { ptent = ptep_get(pte); if (!pte_present(ptent)) continue; @@ -389,7 +393,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, if (!folio || folio_is_zone_device(folio)) continue; - step = folio_mlock_step(folio, pte, addr, end); + step = folio_mlock_step(vma, folio, pte, addr, end); if (!allow_mlock_munlock(folio, vma, start, end, step)) goto next_entry; @@ -400,7 +404,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, next_entry: pte += step - 1; - addr += (step - 1) << PAGE_SHIFT; + addr += (step - 1) << mm_page_shift(vma->vm_mm); } pte_unmap(start_pte); out: @@ -471,7 +475,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_flags_t new_vma_flags = legacy_to_vma_flags(newflags); const vma_flags_t old_vma_flags = vma->flags; struct mm_struct *mm = vma->vm_mm; - int nr_pages; + long nr_pages; int ret = 0; if (vma_flags_same_pair(&old_vma_flags, &new_vma_flags) || @@ -492,7 +496,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, /* * Keep track of amount of locked VM. */ - nr_pages = (end - start) >> PAGE_SHIFT; + nr_pages = (end - start) >> MM_ACCOUNT_SHIFT; if (!vma_flags_test(&new_vma_flags, VMA_LOCKED_BIT)) nr_pages = -nr_pages; else if (vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT)) @@ -524,8 +528,8 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, struct vm_area_struct *vma, *prev; VMA_ITERATOR(vmi, current->mm, start); - VM_BUG_ON(offset_in_page(start)); - VM_BUG_ON(len != PAGE_ALIGN(len)); + VM_BUG_ON((start & ~mm_page_mask(current->mm))); + VM_BUG_ON(len != mm_page_align(current->mm, len)); end = start + len; if (end < start) return -EINVAL; @@ -572,7 +576,7 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, * vma pages, as return value. * Note deferred memory locking case(mlock2(,,MLOCK_ONFAULT) * is also counted. - * Return value: previously mlocked page counts + * Return value: previously mlocked MM_ACCOUNT_SHIFT units */ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm, unsigned long start, size_t len) @@ -600,7 +604,7 @@ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm, } } - return count >> PAGE_SHIFT; + return count >> MM_ACCOUNT_SHIFT; } /* @@ -626,12 +630,12 @@ static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t fla if (!can_do_mlock()) return -EPERM; - len = PAGE_ALIGN(len + (offset_in_page(start))); - start &= PAGE_MASK; + len = mm_page_align(current->mm, len + (start & ~mm_page_mask(current->mm))); + start &= mm_page_mask(current->mm); lock_limit = rlimit(RLIMIT_MEMLOCK); - lock_limit >>= PAGE_SHIFT; - locked = len >> PAGE_SHIFT; + lock_limit >>= MM_ACCOUNT_SHIFT; + locked = len >> MM_ACCOUNT_SHIFT; if (mmap_write_lock_killable(current->mm)) return -EINTR; @@ -686,8 +690,8 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) start = untagged_addr(start); - len = PAGE_ALIGN(len + (offset_in_page(start))); - start &= PAGE_MASK; + len = mm_page_align(current->mm, len + (start & ~mm_page_mask(current->mm))); + start &= mm_page_mask(current->mm); if (mmap_write_lock_killable(current->mm)) return -EINTR; @@ -761,7 +765,7 @@ SYSCALL_DEFINE1(mlockall, int, flags) return -EPERM; lock_limit = rlimit(RLIMIT_MEMLOCK); - lock_limit >>= PAGE_SHIFT; + lock_limit >>= mm_page_shift(current->mm); if (mmap_write_lock_killable(current->mm)) return -EINTR; diff --git a/mm/mm_init.c b/mm/mm_init.c index d50a54dbb1064..640a8457e282e 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -57,7 +57,7 @@ unsigned long zero_page_pfn __ro_after_init; EXPORT_SYMBOL(zero_page_pfn); #ifndef __HAVE_COLOR_ZERO_PAGE -const uint8_t empty_zero_page[PAGE_SIZE] __aligned(PAGE_SIZE); +const uint8_t empty_zero_page[ARCH_ZERO_PAGE_SIZE] __aligned(ARCH_ZERO_PAGE_SIZE); EXPORT_SYMBOL(empty_zero_page); struct page *__zero_page __ro_after_init; diff --git a/mm/mmap.c b/mm/mmap.c index 2311ae7c2ff45..097161e3e61ef 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -151,8 +151,8 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) mm->end_data, mm->start_data)) goto out; - newbrk = PAGE_ALIGN(brk); - oldbrk = PAGE_ALIGN(mm->brk); + newbrk = mm_page_align(mm, brk); + oldbrk = mm_page_align(mm, mm->brk); if (oldbrk == newbrk) { mm->brk = brk; goto success; @@ -186,8 +186,8 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) * expansion area */ vma_iter_init(&vmi, mm, oldbrk); - next = vma_find(&vmi, newbrk + PAGE_SIZE + stack_guard_gap); - if (next && newbrk + PAGE_SIZE > vm_start_gap(next)) + next = vma_find(&vmi, newbrk + mm_page_size(mm) + stack_guard_gap); + if (next && newbrk + mm_page_size(mm) > vm_start_gap(next)) goto out; brkvma = vma_prev_limit(&vmi, mm->start_brk); @@ -218,12 +218,12 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) * If a hint addr is less than mmap_min_addr change hint to be as * low as possible but still greater than mmap_min_addr */ -static inline unsigned long round_hint_to_min(unsigned long hint) +static inline unsigned long round_hint_to_min(struct mm_struct *mm, unsigned long hint) { - hint &= PAGE_MASK; + hint &= mm_page_mask(mm); if (((void *)hint != NULL) && (hint < mmap_min_addr)) - return PAGE_ALIGN(mmap_min_addr); + return mm_page_align(mm, mmap_min_addr); return hint; } @@ -235,11 +235,11 @@ bool mlock_future_ok(const struct mm_struct *mm, bool is_vma_locked, if (!is_vma_locked || capable(CAP_IPC_LOCK)) return true; - locked_pages = bytes >> PAGE_SHIFT; + locked_pages = bytes >> MM_ACCOUNT_SHIFT; locked_pages += mm->locked_vm; limit_pages = rlimit(RLIMIT_MEMLOCK); - limit_pages >>= PAGE_SHIFT; + limit_pages >>= MM_ACCOUNT_SHIFT; return locked_pages <= limit_pages; } @@ -264,24 +264,25 @@ static inline u64 file_mmap_size_max(struct file *file, struct inode *inode) } static inline bool file_mmap_ok(struct file *file, struct inode *inode, - unsigned long pgoff, unsigned long len) + struct vm_page_offset pos, unsigned long len) { u64 maxsize = file_mmap_size_max(file, inode); if (maxsize && len > maxsize) return false; maxsize -= len; - if (pgoff > maxsize >> PAGE_SHIFT) + if (pos.index > maxsize >> PAGE_SHIFT || + (pos.index == maxsize >> PAGE_SHIFT && pos.offset > (maxsize & ~PAGE_MASK))) return false; return true; } /** - * do_mmap() - Perform a userland memory mapping into the current process + * do_mmap_internal() - Perform a userland memory mapping into the current process * address space of length @len with protection bits @prot, mmap flags @flags * (from which VMA flags will be inferred), and any additional VMA flags to * apply @vm_flags. If this is a file-backed mapping then the file is specified - * in @file and page offset into the file via @pgoff. + * in @file and its byte position via @pos. * * This function does not perform security checks on the file and assumes, if * @uf is non-NULL, the caller has provided a list head to track unmap events @@ -321,7 +322,7 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, * @flags: Flags specifying how the mapping should be performed, see mmap (2) * for details. * @vm_flags: VMA flags which should be set by default, or 0 otherwise. - * @pgoff: Page offset into the @file if file-backed, should be 0 otherwise. + * @pos: Native cache index and byte offset; ignored for anonymous mappings. * @populate: A pointer to a value which will be set to 0 if no population of * the range is required, or the number of bytes to populate if it is. Must be * non-NULL. See mmap (2) for details as to under what circumstances population @@ -333,20 +334,22 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, * Returns: Either an error, or the address at which the requested mapping has * been performed. */ -unsigned long do_mmap(struct file *file, unsigned long addr, +static unsigned long do_mmap_internal(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, vm_flags_t vm_flags, - unsigned long pgoff, unsigned long *populate, + struct vm_page_offset pos, unsigned long *populate, struct list_head *uf) { struct mm_struct *mm = current->mm; + unsigned long pgoff = pos.index; int pkey = 0; *populate = 0; mmap_assert_write_locked(mm); - if (!len) + if (!len || pos.offset >= PAGE_SIZE || + !IS_ALIGNED(pos.offset, mm_page_size(mm))) return -EINVAL; /* @@ -364,15 +367,15 @@ unsigned long do_mmap(struct file *file, unsigned long addr, flags |= MAP_FIXED; if (!(flags & MAP_FIXED)) - addr = round_hint_to_min(addr); + addr = round_hint_to_min(mm, addr); /* Careful about overflows.. */ - len = PAGE_ALIGN(len); + len = mm_page_align(mm, len); if (!len) return -ENOMEM; /* offset overflow? */ - if ((pgoff + (len >> PAGE_SHIFT)) < pgoff) + if (vm_page_offset_add(pos, len).index < pgoff) return -EOVERFLOW; /* Too many mappings? */ @@ -426,7 +429,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long flags_mask; int err; - if (!file_mmap_ok(file, inode, pgoff, len)) + if (!file_mmap_ok(file, inode, pos, len)) return -EOVERFLOW; flags_mask = LEGACY_MAP_MASK; @@ -500,6 +503,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, * Ignore pgoff. */ pgoff = 0; + pos.offset = 0; vm_flags |= VM_SHARED | VM_MAYSHARE; break; case MAP_DROPPABLE: @@ -537,6 +541,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, * Set pgoff according to addr for anon_vma. */ pgoff = addr >> PAGE_SHIFT; + pos.offset = addr & ~PAGE_MASK; break; default: return -EINVAL; @@ -557,7 +562,11 @@ unsigned long do_mmap(struct file *file, unsigned long addr, vm_flags |= VM_NORESERVE; } - addr = mmap_region(file, addr, len, vm_flags, pgoff, uf); + addr = mmap_region(file, addr, len, vm_flags, + (struct vm_page_offset) { + .index = pgoff, + .offset = pos.offset, + }, uf); if (!IS_ERR_VALUE(addr) && ((vm_flags & VM_LOCKED) || (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE)) @@ -565,9 +574,28 @@ unsigned long do_mmap(struct file *file, unsigned long addr, return addr; } -unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, +unsigned long do_mmap(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flags, + vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, + struct list_head *uf) +{ + return do_mmap_internal(file, addr, len, prot, flags, vm_flags, + (struct vm_page_offset) { pgoff, 0 }, populate, uf); +} + +#ifdef CONFIG_MM_SUBPAGE +unsigned long do_mmap_offset(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flags, + vm_flags_t vm_flags, struct vm_page_offset pos, unsigned long *populate, + struct list_head *uf) +{ + return do_mmap_internal(file, addr, len, prot, flags, vm_flags, pos, populate, uf); +} +#endif + +static unsigned long ksys_mmap_offset(unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, - unsigned long fd, unsigned long pgoff) + unsigned long fd, unsigned long pgoff, unsigned int suboffset) { struct file *file = NULL; unsigned long retval; @@ -603,13 +631,34 @@ unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, return PTR_ERR(file); } - retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff); + if (suboffset) + retval = vm_mmap(file, addr, len, prot, flags, + (pgoff << PAGE_SHIFT) | suboffset); + else + retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff); out_fput: if (file) fput(file); return retval; } +unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, unsigned long fd, unsigned long pgoff) +{ + return ksys_mmap_offset(addr, len, prot, flags, fd, pgoff, 0); +} + +#ifdef CONFIG_MM_SUBPAGE +unsigned long ksys_mmap_user(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, unsigned long fd, unsigned long offset) +{ + if (!IS_ALIGNED(offset, mm_page_size(current->mm))) + return -EINVAL; + return ksys_mmap_offset(addr, len, prot, flags, fd, + offset >> PAGE_SHIFT, offset & ~PAGE_MASK); +} +#endif + SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len, unsigned long, prot, unsigned long, flags, unsigned long, fd, unsigned long, pgoff) @@ -703,7 +752,7 @@ generic_get_unmapped_area(struct file *filp, unsigned long addr, return addr; if (addr) { - addr = PAGE_ALIGN(addr); + addr = mm_page_align(mm, addr); vma = find_vma_prev(mm, addr, &prev); if (mmap_end - len >= addr && addr >= mmap_min_addr && (!vma || addr + len <= vm_start_gap(vma)) && @@ -754,7 +803,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, /* requesting a specific address */ if (addr) { - addr = PAGE_ALIGN(addr); + addr = mm_page_align(mm, addr); vma = find_vma_prev(mm, addr, &prev); if (mmap_end - len >= addr && addr >= mmap_min_addr && (!vma || addr + len <= vm_start_gap(vma)) && @@ -764,7 +813,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, info.flags = VM_UNMAPPED_AREA_TOPDOWN; info.length = len; - info.low_limit = PAGE_SIZE; + info.low_limit = mm_page_size(mm); info.high_limit = arch_get_mmap_base(addr, mm->mmap_base); info.start_gap = stack_guard_placement(vm_flags); if (filp && is_file_hugepages(filp)) @@ -777,7 +826,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, * can happen with large stack limits and large mmap() * allocations. */ - if (offset_in_page(addr)) { + if (addr & ~mm_page_mask(mm)) { VM_BUG_ON(addr != -ENOMEM); info.flags = 0; info.low_limit = TASK_UNMAPPED_BASE; @@ -857,7 +906,7 @@ __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, if (addr > TASK_SIZE - len) return -ENOMEM; - if (offset_in_page(addr)) + if (addr & ~mm_page_mask(current->mm)) return -EINVAL; error = security_mmap_addr(addr); @@ -962,7 +1011,7 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned lon { struct vm_area_struct *vma, *prev; - addr &= PAGE_MASK; + addr &= mm_page_mask(mm); vma = find_vma_prev(mm, addr, &prev); if (vma && (vma->vm_start <= addr)) return vma; @@ -985,7 +1034,7 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned lon struct vm_area_struct *vma; unsigned long start; - addr &= PAGE_MASK; + addr &= mm_page_mask(mm); vma = find_vma(mm, addr); if (!vma) return NULL; @@ -1214,7 +1263,7 @@ int vm_brk_flags(unsigned long addr, unsigned long request, bool is_exec) LIST_HEAD(uf); VMA_ITERATOR(vmi, mm, addr); - len = PAGE_ALIGN(request); + len = mm_page_align(mm, request); if (len < request) return -ENOMEM; if (!len) @@ -1257,7 +1306,7 @@ unsigned long tear_down_vmas(struct mm_struct *mm, struct vma_iterator *vmi, vma_iter_set(vmi, vma->vm_end); do { if (vma->vm_flags & VM_ACCOUNT) - nr_accounted += vma_pages(vma); + nr_accounted += vma_user_pages(vma); vma_mark_detached(vma); remove_vma(vma); count++; @@ -1322,8 +1371,8 @@ void exit_mmap(struct mm_struct *mm) destroy: __mt_destroy(&mm->mm_mt); trace_exit_mmap(mm); + mm_unacct_memory(mm, nr_accounted); mmap_write_unlock(mm); - vm_unacct_memory(nr_accounted); } /* @@ -1333,19 +1382,19 @@ void exit_mmap(struct mm_struct *mm) bool may_expand_vm(struct mm_struct *mm, const vma_flags_t *vma_flags, unsigned long npages) { - if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> PAGE_SHIFT) + if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> mm_page_shift(mm)) return false; if (is_data_mapping_vma_flags(vma_flags) && - mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) { + mm->data_vm + npages > rlimit(RLIMIT_DATA) >> mm_page_shift(mm)) { /* Workaround for Valgrind */ if (rlimit(RLIMIT_DATA) == 0 && - mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> PAGE_SHIFT) + mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> mm_page_shift(mm)) return true; pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Update limits%s.\n", current->comm, current->pid, - (mm->data_vm + npages) << PAGE_SHIFT, + (mm->data_vm + npages) << mm_page_shift(mm), rlimit(RLIMIT_DATA), ignore_rlimit_data ? "" : " or use boot option ignore_rlimit_data"); @@ -1474,7 +1523,7 @@ static struct vm_area_struct *__install_special_mapping( if (ret) goto out; - vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT); + vm_stat_account(mm, vma->vm_flags, len >> mm_page_shift(mm)); perf_event_mmap(vma); @@ -1771,14 +1820,14 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) if (retval) goto loop_out; - vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt)); + vm_stat_account(mm, mpnt->vm_flags, -vma_user_pages(mpnt)); continue; } charge = 0; if (mpnt->vm_flags & VM_ACCOUNT) { - unsigned long len = vma_pages(mpnt); + unsigned long len = vma_user_pages(mpnt); - if (security_vm_enough_memory_mm(oldmm, len)) /* sic */ + if (mm_account_memory(mm, oldmm, len)) /* sic */ goto fail_nomem; charge = len; } @@ -1890,7 +1939,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) flush_cache_mm(mm); unmap_region(&unmap); charge = tear_down_vmas(mm, &vmi, tmp, end); - vm_unacct_memory(charge); + mm_unacct_memory(mm, charge); } __mt_destroy(&mm->mm_mt); /* @@ -1916,6 +1965,6 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) vm_area_free(tmp); fail_nomem: retval = -ENOMEM; - vm_unacct_memory(charge); + mm_unacct_memory(mm, charge); goto loop_out; } diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index 3985d856de7f9..42419a47a1880 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -12,6 +12,9 @@ #include #include #include +#include + +#include "user-subpage-internal.h" #include @@ -100,6 +103,49 @@ void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma) */ #define MAX_NR_FOLIOS_PER_FREE 512 +#ifdef CONFIG_MM_SUBPAGE +/* Private to gather batches: never pass this token to encoded_page_ptr(). */ +#define TLB_SUBPAGE_TAG BIT(2) + +static void tlb_free_pages_and_subpages(struct encoded_page **pages, unsigned int nr) +{ + unsigned int start = 0, i = 0; + + while (i < nr) { + unsigned long token = (unsigned long)pages[i]; + + if (token & TLB_SUBPAGE_TAG) { + if (i != start) + free_pages_and_swap_cache(pages + start, i - start); + mm_subpage_put((struct mm_subpage *)(token & ~TLB_SUBPAGE_TAG)); + start = ++i; + } else { + /* The count payload is not a pointer and may have bit 2 set. */ + i += encoded_page_flags(pages[i]) & ENCODED_PAGE_BIT_NR_PAGES_NEXT ? 2 : 1; + } + } + if (i != start) + free_pages_and_swap_cache(pages + start, i - start); +} + +bool __tlb_remove_subpage(struct mmu_gather *tlb, struct mm_subpage *subpage) +{ + struct mmu_gather_batch *batch = tlb->active; + + BUILD_BUG_ON(__alignof__(struct page) < 8); + BUILD_BUG_ON(__alignof__(struct mm_subpage) < 8); + BUILD_BUG_ON(TLB_SUBPAGE_TAG & ENCODED_PAGE_BITS); + VM_BUG_ON(!tlb->end); + VM_BUG_ON(mm_page_shift(tlb->mm) != mm_subpage_shift(subpage)); + batch->encoded_pages[batch->nr++] = + (struct encoded_page *)((unsigned long)subpage | TLB_SUBPAGE_TAG); + /* Keep room for a following native page/count pair. */ + return batch->nr >= batch->max - 1 && !tlb_next_batch(tlb); +} +#else +#define tlb_free_pages_and_subpages free_pages_and_swap_cache +#endif + static void __tlb_batch_free_encoded_pages(struct mmu_gather_batch *batch) { struct encoded_page **pages = batch->encoded_pages; @@ -135,7 +181,7 @@ static void __tlb_batch_free_encoded_pages(struct mmu_gather_batch *batch) } } - free_pages_and_swap_cache(pages, nr); + tlb_free_pages_and_subpages(pages, nr); pages += nr; batch->nr -= nr; @@ -174,7 +220,15 @@ static bool __tlb_remove_folio_pages_size(struct mmu_gather *tlb, #ifdef CONFIG_MMU_GATHER_PAGE_SIZE VM_WARN_ON(tlb->page_size != page_size); - VM_WARN_ON_ONCE(nr_pages != 1 && page_size != PAGE_SIZE); + if (page_size > PAGE_SIZE && page_size == mm_page_size(tlb->mm)) { + /* A coarse base PTE owns one reference per covered native page. */ + unsigned long span = mm_pte_native_pages(tlb->mm); + + VM_WARN_ON_ONCE(!nr_pages || nr_pages % span || + !IS_ALIGNED(page_to_pfn(page), span)); + } else { + VM_WARN_ON_ONCE(nr_pages != 1 && page_size != PAGE_SIZE); + } VM_WARN_ON_ONCE(page_folio(page) != page_folio(page + nr_pages - 1)); #endif @@ -208,7 +262,7 @@ bool __tlb_remove_folio_pages(struct mmu_gather *tlb, struct page *page, unsigned int nr_pages, bool delay_rmap) { return __tlb_remove_folio_pages_size(tlb, page, nr_pages, delay_rmap, - PAGE_SIZE); + mm_page_size(tlb->mm)); } bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size) @@ -352,6 +406,13 @@ static inline void __tlb_remove_table_one(void *table) { struct ptdesc *ptdesc; + /* Preserve opaque architecture tokens through the RCU grace period. */ + if (!tlb_table_is_ptdesc(table)) { + tlb_remove_table_sync_rcu(); + __tlb_remove_table(table); + return; + } + ptdesc = table; call_rcu(&ptdesc->pt_rcu_head, __tlb_remove_table_one_rcu); } diff --git a/mm/mprotect.c b/mm/mprotect.c index 9cbf932b028cf..0f5ea22bf4c90 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -58,6 +59,31 @@ static bool maybe_change_pte_writable(struct vm_area_struct *vma, pte_t pte) return true; } +#ifdef CONFIG_MM_SUBPAGE +static bool subpage_pte_exclusive(struct vm_area_struct *vma, pte_t pte) +{ + struct mm_subpage *slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + bool exclusive; + + if (!slot) + return false; + exclusive = mm_subpage_is_exclusive(slot); + mm_subpage_put(slot); + return exclusive; +} + +static int subpage_exclusive_pte_batch(struct vm_area_struct *vma, pte_t *ptep, + int nr_ptes, bool exclusive) +{ + int nr; + + for (nr = 1; nr < nr_ptes; nr++) + if (subpage_pte_exclusive(vma, ptep_get(ptep + nr)) != exclusive) + break; + return nr; +} +#endif + static bool can_change_private_pte_writable(struct vm_area_struct *vma, unsigned long addr, pte_t pte) { @@ -73,6 +99,10 @@ static bool can_change_private_pte_writable(struct vm_area_struct *vma, * any additional checks while holding the PT lock. */ page = vm_normal_page(vma, addr, pte); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return page && PageAnon(page) && subpage_pte_exclusive(vma, pte); +#endif return page && PageAnon(page) && PageAnonExclusive(page); } @@ -103,13 +133,20 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, return can_change_shared_pte_writable(vma, pte); } -static int mprotect_folio_pte_batch(struct folio *folio, pte_t *ptep, - pte_t pte, int max_nr_ptes, fpb_t flags) + +static __always_inline int mprotect_folio_pte_batch(struct vm_area_struct *vma, + struct folio *folio, pte_t *ptep, pte_t pte, int max_nr_ptes, + fpb_t flags, unsigned int page_shift) { /* No underlying folio, so cannot batch */ if (!folio) return 1; +#ifdef CONFIG_MM_SUBPAGE + if (page_shift != PAGE_SHIFT) + return folio_subpage_pte_batch(vma, folio, ptep, pte, max_nr_ptes, flags); +#endif + if (!folio_test_large(folio)) return 1; @@ -125,17 +162,17 @@ static __always_inline void prot_commit_flush_ptes(struct vm_area_struct *vma, * Advance the position in the batch by idx; note that if idx > 0, * then the nr_ptes passed here is <= batch size - idx. */ - addr += idx * PAGE_SIZE; + addr += idx * mm_page_size(vma->vm_mm); ptep += idx; - oldpte = pte_advance_pfn(oldpte, idx); - ptent = pte_advance_pfn(ptent, idx); + oldpte = pte_advance_pfn_mm(vma->vm_mm, oldpte, idx); + ptent = pte_advance_pfn_mm(vma->vm_mm, ptent, idx); if (set_write) ptent = pte_mkwrite(ptent, vma); modify_prot_commit_ptes(vma, addr, ptep, oldpte, ptent, nr_ptes); if (pte_needs_flush(oldpte, ptent)) - tlb_flush_pte_range(tlb, addr, nr_ptes * PAGE_SIZE); + tlb_flush_pte_range(tlb, addr, nr_ptes * mm_page_size(vma->vm_mm)); } /* @@ -177,6 +214,25 @@ static __always_inline void commit_anon_folio_batch(struct vm_area_struct *vma, int sub_batch_idx = 0; int len; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + while (nr_ptes) { + expected_anon_exclusive = subpage_pte_exclusive(vma, + pte_advance_pfn_mm(vma->vm_mm, oldpte, sub_batch_idx)); + for (len = 1; len < nr_ptes; len++) + if (subpage_pte_exclusive(vma, + pte_advance_pfn_mm(vma->vm_mm, oldpte, sub_batch_idx + len)) != + expected_anon_exclusive) + break; + prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, len, + sub_batch_idx, expected_anon_exclusive, tlb); + sub_batch_idx += len; + nr_ptes -= len; + } + return; + } +#endif + while (nr_ptes) { expected_anon_exclusive = PageAnonExclusive(first_page + sub_batch_idx); len = page_anon_exclusive_sub_batch(sub_batch_idx, nr_ptes, @@ -265,6 +321,11 @@ static long change_softleaf_pte(struct vm_area_struct *vma, newpte = oldpte; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + newpte = pte_swp_set_subpage_offset(newpte, pte_swp_subpage_offset(oldpte)); +#endif + if (uffd_wp) newpte = pte_swp_mkuffd_wp(newpte); else if (uffd_wp_resolve) @@ -320,6 +381,9 @@ static long change_pte_range(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, unsigned long end, pgprot_t newprot, unsigned long cp_flags) { + /* The mm's mapping granule cannot change during the walk. */ + const unsigned int page_shift = mm_page_shift(vma->vm_mm); + const unsigned long page_size = 1UL << page_shift; pte_t *pte, oldpte; spinlock_t *ptl; long pages = 0; @@ -328,7 +392,7 @@ static long change_pte_range(struct mmu_gather *tlb, bool uffd_wp = cp_flags & MM_CP_UFFD_WP; int nr_ptes; - tlb_change_page_size(tlb, PAGE_SIZE); + tlb_change_page_size(tlb, page_size); pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); if (!pte) return -EAGAIN; @@ -343,7 +407,7 @@ static long change_pte_range(struct mmu_gather *tlb, oldpte = ptep_get(pte); if (pte_present(oldpte)) { const fpb_t flags = FPB_RESPECT_SOFT_DIRTY | FPB_RESPECT_WRITE; - int max_nr_ptes = (end - addr) >> PAGE_SHIFT; + int max_nr_ptes = (end - addr) >> page_shift; struct folio *folio = NULL; struct page *page; @@ -359,17 +423,32 @@ static long change_pte_range(struct mmu_gather *tlb, * Avoid trapping faults against the zero or KSM * pages. See similar comment in change_huge_pmd. */ - if (prot_numa && - !folio_can_map_prot_numa(folio, vma, - is_private_single_threaded)) { - - /* determine batch to skip */ - nr_ptes = mprotect_folio_pte_batch(folio, - pte, oldpte, max_nr_ptes, /* flags = */ 0); - continue; + if (prot_numa) { + bool shared = folio && folio_maybe_mapped_shared(folio); + +#ifdef CONFIG_MM_SUBPAGE + /* Multiple private leaves do not imply shared COW data. */ + if (folio && folio_test_anon(folio) && page_shift < PAGE_SHIFT) { + shared = !subpage_pte_exclusive(vma, oldpte); + max_nr_ptes = folio_subpage_pte_batch(vma, folio, + pte, oldpte, max_nr_ptes, flags); + /* COW sharing can differ within one physical owner. */ + max_nr_ptes = subpage_exclusive_pte_batch(vma, pte, + max_nr_ptes, !shared); + } +#endif + if (!folio_can_map_prot_numa(folio, vma, + is_private_single_threaded, shared)) { + /* determine batch to skip */ + nr_ptes = mprotect_folio_pte_batch(vma, folio, + pte, oldpte, max_nr_ptes, /* flags = */ 0, + page_shift); + continue; + } } - nr_ptes = mprotect_folio_pte_batch(folio, pte, oldpte, max_nr_ptes, flags); + nr_ptes = mprotect_folio_pte_batch(vma, folio, pte, oldpte, + max_nr_ptes, flags, page_shift); /* * Optimize for the small-folio common case by @@ -408,7 +487,7 @@ static long change_pte_range(struct mmu_gather *tlb, } else { pages += change_softleaf_pte(vma, addr, pte, oldpte, cp_flags); } - } while (pte += nr_ptes, addr += nr_ptes * PAGE_SIZE, addr != end); + } while (pte += nr_ptes, addr += nr_ptes * page_size, addr != end); lazy_mmu_mode_disable(); pte_unmap_unlock(pte - 1, ptl); @@ -487,12 +566,12 @@ static inline long change_pmd_range(struct mmu_gather *tlb, long pages = 0; unsigned long nr_huge_updates = 0; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(vma->vm_mm, pud, addr); do { long ret; pmd_t _pmd; again: - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(vma->vm_mm, addr, end); ret = change_pmd_prepare(vma, pmd, cp_flags); if (ret) { @@ -559,10 +638,10 @@ static inline long change_pud_range(struct mmu_gather *tlb, range.start = 0; - pudp = pud_offset(p4d, addr); + pudp = pud_offset_mm(vma->vm_mm, p4d, addr); do { again: - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(vma->vm_mm, addr, end); ret = change_prepare(vma, pudp, pmd, addr, cp_flags); if (ret) { pages = ret; @@ -615,13 +694,13 @@ static inline long change_p4d_range(struct mmu_gather *tlb, unsigned long next; long pages = 0, ret; - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(vma->vm_mm, pgd, addr); do { - next = p4d_addr_end(addr, end); + next = p4d_addr_end_mm(vma->vm_mm, addr, end); ret = change_prepare(vma, p4d, pud, addr, cp_flags); if (ret) return ret; - if (p4d_none_or_clear_bad(p4d)) + if (p4d_none_or_clear_bad_mm(vma->vm_mm, p4d)) continue; pages += change_pud_range(tlb, vma, p4d, addr, next, newprot, cp_flags); @@ -643,7 +722,7 @@ static long change_protection_range(struct mmu_gather *tlb, pgd = pgd_offset(mm, addr); tlb_start_vma(tlb, vma); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(vma->vm_mm, addr, end); ret = change_prepare(vma, pgd, p4d, addr, cp_flags); if (ret) { pages = ret; @@ -729,7 +808,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, struct mm_struct *mm = vma->vm_mm; const vma_flags_t old_vma_flags = READ_ONCE(vma->flags); vma_flags_t new_vma_flags = legacy_to_vma_flags(newflags); - long nrpages = (end - start) >> PAGE_SHIFT; + long nrpages = (end - start) >> mm_page_shift(mm); unsigned int mm_cp_flags = 0; unsigned long charged = 0; int error; @@ -777,7 +856,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, VMA_ACCOUNT_BIT, VMA_WRITE_BIT, VMA_HUGETLB_BIT, VMA_SHARED_BIT, VMA_NORESERVE_BIT)) { charged = nrpages; - if (security_vm_enough_memory_mm(mm, charged)) + if (mm_account_memory(mm, mm, charged)) return -ENOMEM; vma_flags_set(&new_vma_flags, VMA_ACCOUNT_BIT); } @@ -808,7 +887,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, if (vma_flags_test(&old_vma_flags, VMA_ACCOUNT_BIT) && !vma_flags_test(&new_vma_flags, VMA_ACCOUNT_BIT)) - vm_unacct_memory(nrpages); + mm_unacct_memory(mm, nrpages); /* * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major @@ -826,7 +905,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, return 0; fail: - vm_unacct_memory(charged); + mm_unacct_memory(mm, charged); return error; } @@ -851,11 +930,11 @@ static int do_mprotect_pkey(unsigned long start, size_t len, if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */ return -EINVAL; - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(current->mm)) return -EINVAL; if (!len) return 0; - len = PAGE_ALIGN(len); + len = mm_page_align(current->mm, len); end = start + len; if (end <= start) return -ENOMEM; diff --git a/mm/mremap.c b/mm/mremap.c index ff1cf59b26c71..c0756102bd97f 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -82,11 +82,11 @@ static pud_t *get_old_pud(struct mm_struct *mm, unsigned long addr) if (pgd_none_or_clear_bad(pgd)) return NULL; - p4d = p4d_offset(pgd, addr); - if (p4d_none_or_clear_bad(p4d)) + p4d = p4d_offset_mm(mm, pgd, addr); + if (p4d_none_or_clear_bad_mm(mm, p4d)) return NULL; - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(mm, p4d, addr); if (pud_none_or_clear_bad(pud)) return NULL; @@ -102,7 +102,7 @@ static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr) if (!pud) return NULL; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(mm, pud, addr); if (pmd_none(*pmd)) return NULL; @@ -183,6 +183,14 @@ static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr if (max_nr == 1) return 1; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) { + folio = vm_normal_folio(vma, addr, pte); + return folio ? folio_subpage_pte_batch(vma, folio, ptep, pte, + max_nr, FPB_RESPECT_WRITE) : 1; + } +#endif + /* Avoid expensive folio lookup if we stand no chance of benefit. */ if (pte_batch_hint(ptep, pte) == 1) return 1; @@ -262,12 +270,12 @@ static int move_ptes(struct pagetable_move_control *pmc, flush_tlb_batched_pending(vma->vm_mm); lazy_mmu_mode_enable(); - for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * PAGE_SIZE, - new_ptep += nr_ptes, new_addr += nr_ptes * PAGE_SIZE) { + for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * mm_page_size(mm), + new_ptep += nr_ptes, new_addr += nr_ptes * mm_page_size(mm)) { VM_WARN_ON_ONCE(!pte_none(*new_ptep)); nr_ptes = 1; - max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT; + max_nr_ptes = (old_end - old_addr) >> mm_page_shift(mm); old_pte = ptep_get(old_ptep); if (pte_none(old_pte)) continue; @@ -407,7 +415,7 @@ static bool move_normal_pmd(struct pagetable_move_control *pmc, VM_BUG_ON(!pmd_none(*new_pmd)); pmd_populate(mm, new_pmd, pmd_pgtable(pmd)); - flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + PMD_SIZE); + flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + pmd_size_mm(mm)); out_unlock: if (new_ptl != old_ptl) spin_unlock(new_ptl); @@ -458,8 +466,8 @@ static bool move_normal_pud(struct pagetable_move_control *pmc, VM_BUG_ON(!pud_none(*new_pud)); - pud_populate(mm, new_pud, pud_pgtable(pud)); - flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + PUD_SIZE); + pud_populate(mm, new_pud, pud_pgtable_mm(mm, pud)); + flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + pud_size_mm(mm)); if (new_ptl != old_ptl) spin_unlock(new_ptl); spin_unlock(old_ptl); @@ -549,13 +557,13 @@ static __always_inline unsigned long get_extent(enum pgt_entry entry, switch (entry) { case HPAGE_PMD: case NORMAL_PMD: - mask = PMD_MASK; - size = PMD_SIZE; + mask = pmd_mask_mm(pmc->old->vm_mm); + size = pmd_size_mm(pmc->old->vm_mm); break; case HPAGE_PUD: case NORMAL_PUD: - mask = PUD_MASK; - size = PUD_SIZE; + mask = pud_mask_mm(pmc->old->vm_mm); + size = pud_size_mm(pmc->old->vm_mm); break; default: BUILD_BUG(); @@ -811,7 +819,7 @@ unsigned long move_page_tables(struct pagetable_move_control *pmc) * If possible, realign addresses to PMD boundary for faster copy. * Only realign if the mremap copying hits a PMD boundary. */ - try_realign_addr(pmc, PMD_MASK); + try_realign_addr(pmc, pmd_mask_mm(mm)); flush_cache_range(pmc->old, pmc->old_addr, pmc->old_end); mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, mm, @@ -838,7 +846,7 @@ unsigned long move_page_tables(struct pagetable_move_control *pmc) /* We ignore and continue on error? */ continue; } - } else if (IS_ENABLED(CONFIG_HAVE_MOVE_PUD) && extent == PUD_SIZE) { + } else if (IS_ENABLED(CONFIG_HAVE_MOVE_PUD) && extent == pud_size_mm(mm)) { if (move_pgt_entry(pmc, NORMAL_PUD, old_pud, new_pud)) continue; } @@ -857,7 +865,7 @@ unsigned long move_page_tables(struct pagetable_move_control *pmc) continue; split_huge_pmd(pmc->old, old_pmd, pmc->old_addr); } else if (IS_ENABLED(CONFIG_HAVE_MOVE_PMD) && - extent == PMD_SIZE) { + extent == pmd_size_mm(mm)) { /* * If the extent is PMD-sized, try to speed the move by * moving at the PMD level if possible. @@ -947,9 +955,7 @@ static unsigned long vrm_set_new_addr(struct vma_remap_struct *vrm) { struct vm_area_struct *vma = vrm->vma; unsigned long map_flags = 0; - /* Page Offset _into_ the VMA. */ - pgoff_t internal_pgoff = (vrm->addr - vma->vm_start) >> PAGE_SHIFT; - pgoff_t pgoff = vma->vm_pgoff + internal_pgoff; + pgoff_t pgoff = vma_page_offset_at(vma, vrm->addr).index; unsigned long new_addr = vrm_implies_new_addr(vrm) ? vrm->new_addr : 0; unsigned long res; @@ -985,13 +991,13 @@ static bool vrm_calc_charge(struct vma_remap_struct *vrm) * the length of the new one. Otherwise it's just the delta in size. */ if (vrm->flags & MREMAP_DONTUNMAP) - charged = vrm->new_len >> PAGE_SHIFT; + charged = vrm->new_len >> mm_page_shift(current->mm); else - charged = vrm->delta >> PAGE_SHIFT; + charged = vrm->delta >> mm_page_shift(current->mm); /* This accounts 'charged' pages of memory. */ - if (security_vm_enough_memory_mm(current->mm, charged)) + if (mm_account_memory(current->mm, current->mm, charged)) return false; vrm->charged = charged; @@ -1007,7 +1013,7 @@ static void vrm_uncharge(struct vma_remap_struct *vrm) if (!(vrm->vma->vm_flags & VM_ACCOUNT)) return; - vm_unacct_memory(vrm->charged); + mm_unacct_memory(current->mm, vrm->charged); vrm->charged = 0; } @@ -1019,13 +1025,13 @@ static void vrm_uncharge(struct vma_remap_struct *vrm) static void vrm_stat_account(struct vma_remap_struct *vrm, unsigned long bytes) { - unsigned long pages = bytes >> PAGE_SHIFT; + unsigned long pages = bytes >> mm_page_shift(current->mm); struct mm_struct *mm = current->mm; struct vm_area_struct *vma = vrm->vma; vm_stat_account(mm, vma->vm_flags, pages); if (vma->vm_flags & VM_LOCKED) - mm->locked_vm += pages; + mm->locked_vm += bytes >> MM_ACCOUNT_SHIFT; } static bool __check_map_count_against_split(struct mm_struct *mm, @@ -1228,7 +1234,7 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) vrm->vmi_needs_invalidate = true; if (err) { /* OOM: unable to split vma, just get accounts right */ - vm_acct_memory(len >> PAGE_SHIFT); + mm_acct_memory(current->mm, len >> mm_page_shift(current->mm)); return; } @@ -1285,16 +1291,14 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) static int copy_vma_and_data(struct vma_remap_struct *vrm, struct vm_area_struct **new_vma_ptr) { - unsigned long internal_offset = vrm->addr - vrm->vma->vm_start; - unsigned long internal_pgoff = internal_offset >> PAGE_SHIFT; - unsigned long new_pgoff = vrm->vma->vm_pgoff + internal_pgoff; + struct vm_page_offset pos = vma_page_offset_at(vrm->vma, vrm->addr); unsigned long moved_len; struct vm_area_struct *vma = vrm->vma; struct vm_area_struct *new_vma; int err = 0; PAGETABLE_MOVE(pmc, NULL, NULL, vrm->addr, vrm->new_addr, vrm->old_len); - new_vma = copy_vma(&vma, vrm->new_addr, vrm->new_len, new_pgoff, + new_vma = copy_vma(&vma, vrm->new_addr, vrm->new_len, pos, &pmc.need_rmap_locks, vrm->flags & MREMAP_DONTUNMAP); if (!new_vma) { @@ -1517,7 +1521,7 @@ static unsigned long mremap_to(struct vma_remap_struct *vrm) /* MREMAP_DONTUNMAP expands by old_len since old_len == new_len */ if (vrm->flags & MREMAP_DONTUNMAP) { vma_flags_t vma_flags = vrm->vma->flags; - unsigned long pages = vrm->old_len >> PAGE_SHIFT; + unsigned long pages = vrm->old_len >> mm_page_shift(current->mm); if (!may_expand_vm(mm, &vma_flags, pages)) return -ENOMEM; @@ -1539,7 +1543,7 @@ static int vma_expandable(struct vm_area_struct *vma, unsigned long delta) if (find_vma_intersection(vma->vm_mm, vma->vm_end, end)) return 0; if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start, - 0, MAP_FIXED) & ~PAGE_MASK) + 0, MAP_FIXED) & ~mm_page_mask(vma->vm_mm)) return 0; return 1; } @@ -1771,7 +1775,8 @@ static int check_prep_vma(struct vma_remap_struct *vrm) struct vm_area_struct *vma = vrm->vma; struct mm_struct *mm = current->mm; unsigned long addr = vrm->addr; - unsigned long old_len, new_len, pgoff; + unsigned long old_len, new_len; + struct vm_page_offset pos, end; if (!vma) return -EFAULT; @@ -1846,9 +1851,9 @@ static int check_prep_vma(struct vma_remap_struct *vrm) vrm->populate_expand = true; /* Need to be careful about a growing mapping */ - pgoff = (addr - vma->vm_start) >> PAGE_SHIFT; - pgoff += vma->vm_pgoff; - if (pgoff + (new_len >> PAGE_SHIFT) < pgoff) + pos = vma_page_offset_at(vma, addr); + end = vm_page_offset_add(pos, new_len); + if (end.index < pos.index) return -EINVAL; if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)) @@ -1857,7 +1862,7 @@ static int check_prep_vma(struct vma_remap_struct *vrm) if (!mlock_future_ok(mm, vma->vm_flags & VM_LOCKED, vrm->delta)) return -EAGAIN; - if (!may_expand_vm(mm, &vma->flags, vrm->delta >> PAGE_SHIFT)) + if (!may_expand_vm(mm, &vma->flags, vrm->delta >> mm_page_shift(current->mm))) return -ENOMEM; return 0; @@ -1878,7 +1883,7 @@ static unsigned long check_mremap_params(struct vma_remap_struct *vrm) return -EINVAL; /* Start address must be page-aligned. */ - if (offset_in_page(addr)) + if (addr & ~mm_page_mask(current->mm)) return -EINVAL; /* @@ -1902,7 +1907,7 @@ static unsigned long check_mremap_params(struct vma_remap_struct *vrm) return -EINVAL; /* The new address must be page-aligned. */ - if (offset_in_page(vrm->new_addr)) + if (vrm->new_addr & ~mm_page_mask(current->mm)) return -EINVAL; /* A fixed address implies a move. */ @@ -2015,8 +2020,8 @@ static unsigned long do_mremap(struct vma_remap_struct *vrm) unsigned long res; bool failed; - vrm->old_len = PAGE_ALIGN(vrm->old_len); - vrm->new_len = PAGE_ALIGN(vrm->new_len); + vrm->old_len = mm_page_align(mm, vrm->old_len); + vrm->new_len = mm_page_align(mm, vrm->new_len); res = check_mremap_params(vrm); if (res) diff --git a/mm/mseal.c b/mm/mseal.c index 9781647483d19..24803b047fc0d 100644 --- a/mm/mseal.c +++ b/mm/mseal.c @@ -152,10 +152,10 @@ int do_mseal(unsigned long start, size_t len_in, unsigned long flags) return -EINVAL; start = untagged_addr(start); - if (!PAGE_ALIGNED(start)) + if (start & ~mm_page_mask(mm)) return -EINVAL; - len = PAGE_ALIGN(len_in); + len = mm_page_align(mm, len_in); /* Check to see whether len was rounded up from small -ve to zero. */ if (len_in && !len) return -EINVAL; diff --git a/mm/msync.c b/mm/msync.c index ac4c9bfea2e7f..6d9027ee47246 100644 --- a/mm/msync.c +++ b/mm/msync.c @@ -41,12 +41,12 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags) if (flags & ~(MS_ASYNC | MS_INVALIDATE | MS_SYNC)) goto out; - if (offset_in_page(start)) + if (start & ~mm_page_mask(mm)) goto out; if ((flags & MS_ASYNC) && (flags & MS_SYNC)) goto out; error = -ENOMEM; - len = (len + ~PAGE_MASK) & PAGE_MASK; + len = mm_page_align(mm, len); end = start + len; if (end < start) goto out; @@ -86,7 +86,7 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags) } file = vma->vm_file; fstart = (start - vma->vm_start) + - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); fend = fstart + (min(end, vma->vm_end) - start) - 1; start = vma->vm_end; if ((flags & MS_SYNC) && file && diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 5f372f6e26fa3..5345f43c10b6f 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -225,7 +225,8 @@ long oom_badness(struct task_struct *p, unsigned long totalpages) * The baseline for the badness score is the proportion of RAM that each * task's rss, pagetable and swap space use. */ - points = get_mm_rss_sum(p->mm) + get_mm_counter_sum(p->mm, MM_SWAPENTS) + + points = mm_pages_to_native(p->mm, get_mm_rss_sum(p->mm) + + get_mm_counter_sum(p->mm, MM_SWAPENTS)) + mm_pgtables_bytes(p->mm) / PAGE_SIZE; task_unlock(p); @@ -399,10 +400,10 @@ static int dump_task(struct task_struct *p, void *arg) pr_info("[%7d] %5d %5d %8lu %8lu %8lu %8lu %9lu %8ld %8lu %5hd %s\n", task->pid, from_kuid(&init_user_ns, task_uid(task)), - task->tgid, task->mm->total_vm, get_mm_rss_sum(task->mm), - get_mm_counter_sum(task->mm, MM_ANONPAGES), get_mm_counter_sum(task->mm, MM_FILEPAGES), - get_mm_counter_sum(task->mm, MM_SHMEMPAGES), mm_pgtables_bytes(task->mm), - get_mm_counter_sum(task->mm, MM_SWAPENTS), + task->tgid, mm_pages_to_native(task->mm, task->mm->total_vm), mm_pages_to_native(task->mm, get_mm_rss_sum(task->mm)), + mm_pages_to_native(task->mm, get_mm_counter_sum(task->mm, MM_ANONPAGES)), mm_pages_to_native(task->mm, get_mm_counter_sum(task->mm, MM_FILEPAGES)), + mm_pages_to_native(task->mm, get_mm_counter_sum(task->mm, MM_SHMEMPAGES)), mm_pgtables_bytes(task->mm), + mm_pages_to_native(task->mm, get_mm_counter_sum(task->mm, MM_SWAPENTS)), task->signal->oom_score_adj, task->comm); task_unlock(task); @@ -588,9 +589,9 @@ static bool oom_reap_task_mm(struct task_struct *tsk, struct mm_struct *mm) pr_info("oom_reaper: reaped process %d (%s), now anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB\n", task_pid_nr(tsk), tsk->comm, - K(get_mm_counter_sum(mm, MM_ANONPAGES)), - K(get_mm_counter_sum(mm, MM_FILEPAGES)), - K(get_mm_counter_sum(mm, MM_SHMEMPAGES))); + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_ANONPAGES)), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_FILEPAGES)), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_SHMEMPAGES))); out_finish: trace_finish_task_reaping(tsk->pid); out_unlock: @@ -943,10 +944,10 @@ static void __oom_kill_process(struct task_struct *victim, const char *message) do_send_sig_info(SIGKILL, SEND_SIG_PRIV, victim, PIDTYPE_TGID); mark_oom_victim(victim); pr_err("%s: Killed process %d (%s) total-vm:%lukB, anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB, UID:%u pgtables:%lukB oom_score_adj:%d\n", - message, task_pid_nr(victim), victim->comm, K(mm->total_vm), - K(get_mm_counter_sum(mm, MM_ANONPAGES)), - K(get_mm_counter_sum(mm, MM_FILEPAGES)), - K(get_mm_counter_sum(mm, MM_SHMEMPAGES)), + message, task_pid_nr(victim), victim->comm, mm_pages_to_kb(mm, mm->total_vm), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_ANONPAGES)), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_FILEPAGES)), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_SHMEMPAGES)), from_kuid(&init_user_ns, task_uid(victim)), mm_pgtables_bytes(mm) >> 10, victim->signal->oom_score_adj); task_unlock(victim); diff --git a/mm/page_idle.c b/mm/page_idle.c index 9c67cbac29659..8af46b0d8f746 100644 --- a/mm/page_idle.c +++ b/mm/page_idle.c @@ -51,9 +51,10 @@ static struct folio *page_idle_get_folio(unsigned long pfn) static bool page_idle_clear_pte_refs_one(struct folio *folio, struct vm_area_struct *vma, - unsigned long addr, void *arg) + struct rmap_walk_range rmap, void *arg) { - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); + unsigned long addr = rmap.address; + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); bool referenced = false; while (page_vma_mapped_walk(&pvmw)) { @@ -69,7 +70,7 @@ static bool page_idle_clear_pte_refs_one(struct folio *folio, */ if (likely(pte_present(ptep_get(pvmw.pte)))) referenced |= ptep_test_and_clear_young(vma, addr, pvmw.pte); - referenced |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + PAGE_SIZE); + referenced |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + mm_page_size(vma->vm_mm)); } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { pmd_t pmdval = pmdp_get(pvmw.pmd); diff --git a/mm/page_io.c b/mm/page_io.c index b23f494fcc83d..041c21b85d7ee 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -25,6 +25,7 @@ #include #include #include +#include #include "swap.h" #include "swap_table.h" @@ -426,6 +427,58 @@ static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug) *swap_plug = sio; } +static DEFINE_MUTEX(swap_extent_io_mutex); +static struct bio_set swap_extent_bioset; + +int swap_extent_io_init(void) +{ + int ret = 0; + + /* Swapon prepares the reserve before any reclaim can use this file. */ + mutex_lock(&swap_extent_io_mutex); + if (!bioset_initialized(&swap_extent_bioset)) + ret = bioset_init(&swap_extent_bioset, SWAP_CLUSTER_MAX, 0, + BIOSET_NEED_RESCUER); + mutex_unlock(&swap_extent_io_mutex); + return ret; +} + +/* + * A mandatory large base page cannot split to fit a swap-file extent. Chain + * each physical run to the original bio: its folio completion must wait for + * every run, including on partial I/O or allocation failure. + */ +static void swap_submit_bio(struct bio *bio, void *private) +{ + struct folio *folio = private; + struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); + swp_entry_t entry = folio->swap; + + if (!(READ_ONCE(sis->flags) & SWP_BLKDEV) && folio_test_large(folio)) { + while (true) { + unsigned long nr = bio->bi_iter.bi_size >> PAGE_SHIFT; + unsigned int bytes; + struct bio *split; + + bio->bi_iter.bi_sector = swap_extent_sector(entry, &nr); + bytes = nr << PAGE_SHIFT; + if (bytes == bio->bi_iter.bi_size) + break; + /* The unsubmitted parent may own a bio from fs_bio_set. */ + split = bio_split(bio, bytes >> 9, GFP_NOIO, &swap_extent_bioset); + if (IS_ERR(split)) { + bio->bi_status = BLK_STS_RESOURCE; + bio_endio(bio); + return; + } + bio_chain(split, bio); + submit_bio(split); + entry.val += nr; + } + } + submit_bio(bio); +} + static void swap_writepage_bdev_sync(struct folio *folio, struct swap_info_struct *sis) { @@ -442,7 +495,7 @@ static void swap_writepage_bdev_sync(struct folio *folio, folio_start_writeback(folio); folio_unlock(folio); - submit_bio_wait(&bio); + bio_await(&bio, folio, swap_submit_bio); __end_swap_bio_write(&bio); } @@ -460,7 +513,7 @@ static void swap_writepage_bdev_async(struct folio *folio, count_swpout_vm_event(folio); folio_start_writeback(folio); folio_unlock(folio); - submit_bio(bio); + swap_submit_bio(bio, folio); } void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug) @@ -638,7 +691,7 @@ static void swap_read_folio_bdev_sync(struct folio *folio, count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio)); count_vm_events(PSWPIN, folio_nr_pages(folio)); - submit_bio_wait(&bio); + bio_await(&bio, folio, swap_submit_bio); __end_swap_bio_read(&bio); put_task_struct(current); } @@ -655,7 +708,7 @@ static void swap_read_folio_bdev_async(struct folio *folio, count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio)); count_vm_events(PSWPIN, folio_nr_pages(folio)); - submit_bio(bio); + swap_submit_bio(bio, folio); } void swap_read_folio(struct folio *folio, struct swap_iocb **plug) diff --git a/mm/page_table_check.c b/mm/page_table_check.c index 2403f5a114105..dc6797a41daaa 100644 --- a/mm/page_table_check.c +++ b/mm/page_table_check.c @@ -4,6 +4,7 @@ * Copyright (c) 2021, Google LLC. * Pasha Tatashin */ +#include #include #include #include @@ -13,11 +14,21 @@ #undef pr_fmt #define pr_fmt(fmt) "page_table_check: " fmt -struct page_table_check { +#ifdef CONFIG_MM_SUBPAGE +#define PAGE_TABLE_CHECK_SLOTS (PAGE_SIZE / SZ_4K) +#else +#define PAGE_TABLE_CHECK_SLOTS 1 +#endif + +struct page_table_check_slot { atomic_t anon_map_count; atomic_t file_map_count; }; +struct page_table_check { + struct page_table_check_slot slots[PAGE_TABLE_CHECK_SLOTS]; +}; + static bool __page_table_check_enabled __initdata = IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED); @@ -56,6 +67,74 @@ static struct page_table_check *get_page_table_check(struct page_ext *page_ext) return page_ext_data(page_ext, &page_table_check_ops); } +#if IS_ENABLED(CONFIG_KUNIT) +/* Diagnostic snapshot for private test allocations, not a synchronization API. */ +int page_table_check_get_counts(struct page *page, unsigned int offset, + int *anon, int *file) +{ + struct page_ext *ext; + struct page_table_check *ptc; + unsigned int slot = 0; + + if (static_branch_likely(&page_table_check_disabled)) + return -EOPNOTSUPP; + if (offset >= PAGE_SIZE) + return -EINVAL; +#ifdef CONFIG_MM_SUBPAGE + slot = offset / SZ_4K; +#endif + ext = page_ext_get(page); + if (!ext) + return -ENOENT; + ptc = get_page_table_check(ext); + *anon = atomic_read(&ptc->slots[slot].anon_map_count); + *file = atomic_read(&ptc->slots[slot].file_map_count); + page_ext_put(ext); + return 0; +} +EXPORT_SYMBOL_IF_KUNIT(page_table_check_get_counts); +#endif + +static void page_table_check_update(struct page_table_check_slot *slot, + bool anon, bool clear, bool rw) +{ + if (anon) { + BUG_ON(atomic_read(&slot->file_map_count)); + if (clear) + BUG_ON(atomic_dec_return(&slot->anon_map_count) < 0); + else + BUG_ON(atomic_inc_return(&slot->anon_map_count) > 1 && rw); + } else { + BUG_ON(atomic_read(&slot->anon_map_count)); + if (clear) + BUG_ON(atomic_dec_return(&slot->file_map_count) < 0); + else + BUG_ON(atomic_inc_return(&slot->file_map_count) < 0); + } +} + +#ifdef CONFIG_MM_SUBPAGE +static void page_table_check_subpage(phys_addr_t phys, unsigned long size, + bool clear, bool rw) +{ + unsigned long pfn = PHYS_PFN(phys); + struct page_table_check *ptc; + struct page_ext *ext; + struct page *page; + unsigned int first = offset_in_page(phys) / SZ_4K, i; + + if (!pfn_valid(pfn)) + return; + page = pfn_to_page(pfn); + BUG_ON(PageSlab(page)); + ext = page_ext_get(page); + ptc = get_page_table_check(ext); + for (i = first; i < first + size / SZ_4K; i++) + page_table_check_update(&ptc->slots[i], PageAnon(page), clear, rw); + page_ext_put(ext); +} +#endif + /* * An entry is removed from the page table, decrement the counters for that page * verify that it is of correct type and counters do not become negative. @@ -78,13 +157,8 @@ static void page_table_check_clear(unsigned long pfn, unsigned long pgcnt) for_each_page_ext(page, pgcnt, page_ext, iter) { struct page_table_check *ptc = get_page_table_check(page_ext); - if (anon) { - BUG_ON(atomic_read(&ptc->file_map_count)); - BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0); - } else { - BUG_ON(atomic_read(&ptc->anon_map_count)); - BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0); - } + for (unsigned int i = 0; i < PAGE_TABLE_CHECK_SLOTS; i++) + page_table_check_update(&ptc->slots[i], anon, true, false); } rcu_read_unlock(); } @@ -113,13 +187,8 @@ static void page_table_check_set(unsigned long pfn, unsigned long pgcnt, for_each_page_ext(page, pgcnt, page_ext, iter) { struct page_table_check *ptc = get_page_table_check(page_ext); - if (anon) { - BUG_ON(atomic_read(&ptc->file_map_count)); - BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw); - } else { - BUG_ON(atomic_read(&ptc->anon_map_count)); - BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0); - } + for (unsigned int i = 0; i < PAGE_TABLE_CHECK_SLOTS; i++) + page_table_check_update(&ptc->slots[i], anon, false, rw); } rcu_read_unlock(); } @@ -139,8 +208,10 @@ void __page_table_check_zero(struct page *page, unsigned int order) for_each_page_ext(page, 1 << order, page_ext, iter) { struct page_table_check *ptc = get_page_table_check(page_ext); - BUG_ON(atomic_read(&ptc->anon_map_count)); - BUG_ON(atomic_read(&ptc->file_map_count)); + for (unsigned int i = 0; i < PAGE_TABLE_CHECK_SLOTS; i++) { + BUG_ON(atomic_read(&ptc->slots[i].anon_map_count)); + BUG_ON(atomic_read(&ptc->slots[i].file_map_count)); + } } rcu_read_unlock(); } @@ -151,8 +222,15 @@ void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr, if (&init_mm == mm) return; - if (pte_user_accessible_page(mm, addr, pte) && !pte_special(pte)) - page_table_check_clear(pte_pfn(pte), PAGE_SIZE >> PAGE_SHIFT); + if (!pte_user_accessible_page(mm, addr, pte) || pte_special(pte)) + return; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + page_table_check_subpage(pte_phys_mm(mm, pte), mm_page_size(mm), true, false); + return; + } +#endif + page_table_check_clear(pte_pfn(pte), mm_pte_native_pages(mm)); } EXPORT_SYMBOL(__page_table_check_pte_clear); @@ -218,9 +296,20 @@ void __page_table_check_ptes_set(struct mm_struct *mm, unsigned long addr, page_table_check_pte_flags(pte); for (i = 0; i < nr; i++) - __page_table_check_pte_clear(mm, addr + PAGE_SIZE * i, ptep_get(ptep + i)); - if (pte_user_accessible_page(mm, addr, pte) && !pte_special(pte)) - page_table_check_set(pte_pfn(pte), nr, pte_write(pte)); + __page_table_check_pte_clear(mm, addr + mm_page_size(mm) * i, ptep_get(ptep + i)); + if (!pte_user_accessible_page(mm, addr, pte) || pte_special(pte)) + return; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + phys_addr_t phys = pte_phys_mm(mm, pte); + + for (i = 0; i < nr; i++) + page_table_check_subpage(phys + i * mm_page_size(mm), mm_page_size(mm), false, + pte_write(pte)); + return; + } +#endif + page_table_check_set(pte_pfn(pte), nr * mm_pte_native_pages(mm), pte_write(pte)); } EXPORT_SYMBOL(__page_table_check_ptes_set); @@ -279,16 +368,17 @@ void __page_table_check_pte_clear_range(struct mm_struct *mm, return; if (!pmd_bad(pmd) && !pmd_leaf(pmd)) { - pte_t *ptep = pte_offset_map(&pmd, addr); + pte_t *ptep = pte_offset_map_mm(mm, &pmd, addr); + unsigned long nr = pte_table_bytes_mm(mm) / sizeof(pte_t); unsigned long i; if (WARN_ON(!ptep)) return; - for (i = 0; i < PTRS_PER_PTE; i++) { + for (i = 0; i < nr; i++) { __page_table_check_pte_clear(mm, addr, ptep_get(ptep)); - addr += PAGE_SIZE; + addr += mm_page_size(mm); ptep++; } - pte_unmap(ptep - PTRS_PER_PTE); + pte_unmap(ptep - nr); } } diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index d7670ba4147bf..272e7f98d1d2a 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -87,7 +87,7 @@ static bool map_pte(struct page_vma_mapped_walk *pvmw, pmd_t *pmdvalp, * mapped at the @pvmw->pte * @pvmw: page_vma_mapped_walk struct, includes a pair pte and pfn range * for checking - * @pte_nr: the number of small pages described by @pvmw->pte. + * @pte_nr: the number of native physical pages covered by @pvmw->pte. * * page_vma_mapped_walk() found a place where pfn range is *potentially* * mapped. check_pte() has to validate this. @@ -115,6 +115,18 @@ static bool check_pte(struct page_vma_mapped_walk *pvmw, unsigned long pte_nr) else ptent = ptep_get(pvmw->pte); +#ifdef CONFIG_MM_SUBPAGE + if (pvmw->flags & PVMW_SUBPAGE) { + if (!(pvmw->flags & PVMW_MIGRATION)) + return pte_present(ptent) && + pte_phys_mm(pvmw->vma->vm_mm, ptent) == + PFN_PHYS(pvmw->pfn) + pvmw->subpage_offset; + if (pte_swp_subpage_offset(ptent) != pvmw->subpage_offset) + return false; + /* The common migration check below validates the native PFN. */ + } +#endif + if (pvmw->flags & PVMW_MIGRATION) { const softleaf_t entry = softleaf_from_pte(ptent); @@ -195,6 +207,18 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) pud_t *pud; pmd_t pmde; +#ifdef CONFIG_MM_SUBPAGE + if (pvmw->flags & PVMW_SUBPAGE) { + if (WARN_ON_ONCE(pvmw->nr_pages != 1 || + mm_page_size(mm) >= PAGE_SIZE || + !IS_ALIGNED(pvmw->address, mm_page_size(mm)) || + pvmw->subpage_offset >= PAGE_SIZE || + !IS_ALIGNED(pvmw->subpage_offset, mm_page_size(mm)) || + is_vm_hugetlb_page(vma))) + return not_found(pvmw); + } +#endif + /* The only possible pmd mapping has been handled on last iteration */ if (pvmw->pmd && !pvmw->pte) return not_found(pvmw); @@ -220,6 +244,9 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) return true; } + /* A query for an interior native page still visits one whole user leaf. */ + if (mm_page_size(mm) > PAGE_SIZE) + pvmw->address &= mm_page_mask(mm); end = vma_address_end(pvmw); if (pvmw->pte) goto next_pte; @@ -227,21 +254,21 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) do { pgd = pgd_offset(mm, pvmw->address); if (!pgd_present(*pgd)) { - step_forward(pvmw, PGDIR_SIZE); + step_forward(pvmw, pgd_size_mm(mm)); continue; } - p4d = p4d_offset(pgd, pvmw->address); - if (!p4d_present(*p4d)) { - step_forward(pvmw, P4D_SIZE); + p4d = p4d_offset_mm(mm, pgd, pvmw->address); + if (!p4d_present_mm(mm, *p4d)) { + step_forward(pvmw, p4d_size_mm(mm)); continue; } - pud = pud_offset(p4d, pvmw->address); + pud = pud_offset_mm(mm, p4d, pvmw->address); if (!pud_present(*pud)) { - step_forward(pvmw, PUD_SIZE); + step_forward(pvmw, pud_size_mm(mm)); continue; } - pvmw->pmd = pmd_offset(pud, pvmw->address); + pvmw->pmd = pmd_offset_mm(mm, pud, pvmw->address); /* * Make sure the pmd value isn't cached in a register by the * compiler and used as a stale value after we've observed a @@ -252,6 +279,9 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && (pmd_trans_huge(pmde) || pmd_is_migration_entry(pmde) || pmd_is_device_private_entry(pmde))) { + /* Subpage PMD mappings need a distinct huge-page implementation. */ + if (mm_page_size(mm) != PAGE_SIZE) + return not_found(pvmw); pvmw->ptl = pmd_lock(mm, pvmw->pmd); pmde = *pvmw->pmd; if (pmd_is_migration_entry(pmde)) { @@ -292,7 +322,7 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) (pvmw->nr_pages >= HPAGE_PMD_NR)) sync_with_folio_pmd_zap(mm, pvmw->pmd); - step_forward(pvmw, PMD_SIZE); + step_forward(pvmw, pmd_size_mm(mm)); continue; } if (!map_pte(pvmw, &pmde, &ptl)) { @@ -301,15 +331,15 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) goto next_pte; } this_pte: - if (check_pte(pvmw, 1)) + if (check_pte(pvmw, mm_pte_native_pages(mm))) return true; next_pte: do { - pvmw->address += PAGE_SIZE; + pvmw->address += mm_page_size(mm); if (pvmw->address >= end) return not_found(pvmw); /* Did we cross page table boundary? */ - if ((pvmw->address & (PMD_SIZE - PAGE_SIZE)) == 0) { + if ((pvmw->address & (pmd_size_mm(mm) - mm_page_size(mm))) == 0) { if (pvmw->ptl) { spin_unlock(pvmw->ptl); pvmw->ptl = NULL; @@ -356,6 +386,7 @@ unsigned long page_mapped_in_vma(const struct page *page, struct vm_area_struct *vma) { const struct folio *folio = page_folio(page); + unsigned long address; struct page_vma_mapped_walk pvmw = { .pfn = page_to_pfn(page), .nr_pages = 1, @@ -363,13 +394,15 @@ unsigned long page_mapped_in_vma(const struct page *page, .flags = PVMW_SYNC, }; - pvmw.address = vma_address(vma, page_pgoff(folio, page), 1); + address = vma_address(vma, page_pgoff(folio, page), 1); + pvmw.address = address; if (pvmw.address == -EFAULT) goto out; if (!page_vma_mapped_walk(&pvmw)) return -EFAULT; page_vma_mapped_walk_done(&pvmw); out: - return pvmw.address; + /* Report this native page's byte address, not the covering leaf start. */ + return address; } #endif diff --git a/mm/pagewalk.c b/mm/pagewalk.c index d3bfece319336..8d569e5c45cae 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -30,13 +30,14 @@ static int walk_pte_range_inner(pte_t *pte, unsigned long addr, unsigned long end, struct mm_walk *walk) { const struct mm_walk_ops *ops = walk->ops; + const unsigned long page_size = mm_page_size(walk->mm); int err = 0; for (;;) { if (ops->install_pte && pte_none(ptep_get(pte))) { pte_t new_pte; - err = ops->install_pte(addr, addr + PAGE_SIZE, &new_pte, + err = ops->install_pte(addr, addr + page_size, &new_pte, walk); if (err) break; @@ -46,13 +47,13 @@ static int walk_pte_range_inner(pte_t *pte, unsigned long addr, if (!WARN_ON_ONCE(walk->no_vma)) update_mmu_cache(walk->vma, addr, pte); } else { - err = ops->pte_entry(pte, addr, addr + PAGE_SIZE, walk); + err = ops->pte_entry(pte, addr, addr + page_size, walk); if (err) break; } - if (addr >= end - PAGE_SIZE) + if (addr >= end - page_size) break; - addr += PAGE_SIZE; + addr += page_size; pte++; } return err; @@ -67,15 +68,15 @@ static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, if (walk->no_vma) { /* - * pte_offset_map() might apply user-specific validation. + * pte_offset_map_mm() might apply user-specific validation. * Indeed, on x86_64 the pmd entries set up by init_espfix_ap() * fit its pmd_bad() check (_PAGE_NX set and _PAGE_RW clear), * and CONFIG_EFI_PGT_DUMP efi_mm goes so far as to walk them. */ if (walk->mm == &init_mm || addr >= TASK_SIZE) - pte = pte_offset_kernel(pmd, addr); + pte = pte_offset_kernel_mm(walk->mm, pmd, addr); else - pte = pte_offset_map(pmd, addr); + pte = pte_offset_map_mm(walk->mm, pmd, addr); if (pte) { err = walk_pte_range_inner(pte, addr, end, walk); if (walk->mm != &init_mm && addr < TASK_SIZE) @@ -123,11 +124,11 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, return 0; } - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(walk->mm, pud, addr); do { again: walk->action = ACTION_SUBTREE; - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(walk->mm, addr, end); if (pmd_none(*pmd)) { if (has_install) err = __pte_alloc(walk->mm, pmd); @@ -192,11 +193,11 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, int err = 0; int depth = real_depth(2); - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(walk->mm, p4d, addr); do { again: walk->action = ACTION_SUBTREE; - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(walk->mm, addr, end); if (pud_none(*pud)) { if (has_install) err = __pmd_alloc(walk->mm, pud, addr); @@ -256,10 +257,10 @@ static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, int err = 0; int depth = real_depth(1); - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(walk->mm, pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(p4d)) { + next = p4d_addr_end_mm(walk->mm, addr, end); + if (p4d_none_or_clear_bad_mm(walk->mm, p4d)) { if (has_install) err = __pud_alloc(walk->mm, p4d, addr); else if (ops->pte_hole) @@ -295,11 +296,11 @@ static int walk_pgd_range(unsigned long addr, unsigned long end, int err = 0; if (walk->pgd) - pgd = walk->pgd + pgd_index(addr); + pgd = pgd_offset_pgd_mm(walk->mm, walk->pgd, addr); else pgd = pgd_offset(walk->mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(walk->mm, addr, end); if (pgd_none_or_clear_bad(pgd)) { if (has_install) err = __p4d_alloc(walk->mm, pgd, addr); @@ -922,11 +923,11 @@ struct folio *folio_walk_start(struct folio_walk *fw, if (pgd_none_or_clear_bad(pgdp)) goto not_found; - p4dp = p4d_offset(pgdp, addr); - if (p4d_none_or_clear_bad(p4dp)) + p4dp = p4d_offset_mm(vma->vm_mm, pgdp, addr); + if (p4d_none_or_clear_bad_mm(vma->vm_mm, p4dp)) goto not_found; - pudp = pud_offset(p4dp, addr); + pudp = pud_offset_mm(vma->vm_mm, p4dp, addr); pud = pudp_get(pudp); if (pud_none(pud)) goto not_found; @@ -935,7 +936,7 @@ struct folio *folio_walk_start(struct folio_walk *fw, ptl = pud_lock(vma->vm_mm, pudp); pud = pudp_get(pudp); - entry_size = PUD_SIZE; + entry_size = pud_size_mm(vma->vm_mm); fw->level = FW_LEVEL_PUD; fw->pudp = pudp; fw->pud = pud; @@ -957,7 +958,7 @@ struct folio *folio_walk_start(struct folio_walk *fw, pmd_table: VM_WARN_ON_ONCE(!pud_present(pud) || pud_leaf(pud)); - pmdp = pmd_offset(pudp, addr); + pmdp = pmd_offset_mm(vma->vm_mm, pudp, addr); pmd = pmdp_get_lockless(pmdp); if (pmd_none(pmd)) goto not_found; @@ -966,7 +967,7 @@ struct folio *folio_walk_start(struct folio_walk *fw, ptl = pmd_lock(vma->vm_mm, pmdp); pmd = pmdp_get(pmdp); - entry_size = PMD_SIZE; + entry_size = pmd_size_mm(vma->vm_mm); fw->level = FW_LEVEL_PMD; fw->pmdp = pmdp; fw->pmd = pmd; @@ -999,7 +1000,7 @@ struct folio *folio_walk_start(struct folio_walk *fw, goto not_found; pte = ptep_get(ptep); - entry_size = PAGE_SIZE; + entry_size = mm_page_size(vma->vm_mm); fw->level = FW_LEVEL_PTE; fw->ptep = ptep; fw->pte = pte; @@ -1026,5 +1027,5 @@ struct folio *folio_walk_start(struct folio_walk *fw, else fw->page = NULL; fw->ptl = ptl; - return page_folio(page); + return page_folio(zeropage ? page : fw->page); } diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index cd227fc05d2d8..70f1cd37f007b 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -168,13 +168,15 @@ pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address, void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable) { + struct page *page = pgtable_to_page(pgtable); + assert_spin_locked(pmd_lockptr(mm, pmdp)); /* FIFO */ if (!pmd_huge_pte(mm, pmdp)) - INIT_LIST_HEAD(&pgtable->lru); + INIT_LIST_HEAD(&page->lru); else - list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru); + list_add(&page->lru, &pgtable_to_page(pmd_huge_pte(mm, pmdp))->lru); pmd_huge_pte(mm, pmdp) = pgtable; } #endif @@ -184,15 +186,17 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) { pgtable_t pgtable; + struct page *page, *next; assert_spin_locked(pmd_lockptr(mm, pmdp)); /* FIFO */ pgtable = pmd_huge_pte(mm, pmdp); - pmd_huge_pte(mm, pmdp) = list_first_entry_or_null(&pgtable->lru, - struct page, lru); + page = pgtable_to_page(pgtable); + next = list_first_entry_or_null(&page->lru, struct page, lru); + pmd_huge_pte(mm, pmdp) = pgtable_from_page(next); if (pmd_huge_pte(mm, pmdp)) - list_del(&pgtable->lru); + list_del(&page->lru); return pgtable; } #endif @@ -282,7 +286,12 @@ static unsigned long pmdp_get_lockless_start(void) { return 0; } static void pmdp_get_lockless_end(unsigned long irqflags) { } #endif +#ifdef CONFIG_ARCH_HAS_MM_PGTABLE +pte_t *__pte_offset_map_mm(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr, pmd_t *pmdvalp) +#else pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp) +#endif { unsigned long irqflags; pmd_t pmdval; @@ -302,19 +311,27 @@ pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp) pmd_clear_bad(pmd); goto nomap; } - return __pte_map(&pmdval, addr); + return __pte_map_mm(mm, &pmdval, addr); nomap: rcu_read_unlock(); return NULL; } +#ifdef CONFIG_ARCH_HAS_MM_PGTABLE +pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp) +{ + /* NULL retains native geometry for callers not yet converted. */ + return __pte_offset_map_mm(NULL, pmd, addr, pmdvalp); +} +#endif + pte_t *pte_offset_map_ro_nolock(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, spinlock_t **ptlp) { pmd_t pmdval; pte_t *pte; - pte = __pte_offset_map(pmd, addr, &pmdval); + pte = __pte_offset_map_mm(mm, pmd, addr, &pmdval); if (likely(pte)) *ptlp = pte_lockptr(mm, &pmdval); return pte; @@ -327,7 +344,7 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, pte_t *pte; VM_WARN_ON_ONCE(!pmdvalp); - pte = __pte_offset_map(pmd, addr, pmdvalp); + pte = __pte_offset_map_mm(mm, pmd, addr, pmdvalp); if (likely(pte)) *ptlp = pte_lockptr(mm, pmdvalp); return pte; @@ -397,7 +414,7 @@ pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, pmd_t pmdval; pte_t *pte; again: - pte = __pte_offset_map(pmd, addr, &pmdval); + pte = __pte_offset_map_mm(mm, pmd, addr, &pmdval); if (unlikely(!pte)) return pte; ptl = pte_lockptr(mm, &pmdval); diff --git a/mm/process_vm_access.c b/mm/process_vm_access.c index 656d3e88755b6..f178b5fdb6589 100644 --- a/mm/process_vm_access.c +++ b/mm/process_vm_access.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -15,6 +16,8 @@ #include #include +#include "internal.h" + /** * process_vm_rw_pages - read/write pages from task specified * @pages: array of pointers to pages we want to copy @@ -135,6 +138,63 @@ static int process_vm_rw_single_vec(unsigned long addr, which lives on stack */ #define PVM_MAX_PP_ARRAY_COUNT 16 +#ifdef CONFIG_MM_SUBPAGE +/* Same pin/copy/dirty lifetime as the page-array path, with explicit offsets. */ +int process_vm_rw_fragments(struct mm_struct *mm, unsigned long addr, + unsigned long len, struct iov_iter *iter, bool write) +{ + struct user_page_fragment stack[PVM_MAX_PP_ARRAY_COUNT]; + struct user_page_fragment *fragments = stack; + unsigned long capacity, nr; + int ret = 0; + + if (!len || !iov_iter_count(iter)) + return 0; + capacity = 1 + ((len - 1) >> MM_SUBPAGE_SHIFT) + + ((((len - 1) & (MM_SUBPAGE_SIZE - 1)) + + (addr & (MM_SUBPAGE_SIZE - 1))) >> MM_SUBPAGE_SHIFT); + capacity = min_t(unsigned long, capacity, + PVM_MAX_KMALLOC_PAGES * PAGE_SIZE / sizeof(*fragments)); + if (capacity > ARRAY_SIZE(stack)) { + fragments = kmalloc_array(capacity, sizeof(*fragments), GFP_KERNEL); + if (!fragments) + return -ENOMEM; + } + while (len && iov_iter_count(iter)) { + long got = pin_user_fragments_remote(mm, addr, len, + write ? FOLL_WRITE : 0, fragments, capacity); + + if (got <= 0) { + ret = -EFAULT; + break; + } + for (nr = 0; nr < got && iov_iter_count(iter); nr++) { + struct user_page_fragment *f = &fragments[nr]; + struct page *page = folio_page(f->folio, f->offset >> PAGE_SHIFT); + unsigned int offset = offset_in_page(f->offset); + size_t copied; + + if (write) + copied = copy_page_from_iter(page, offset, f->length, iter); + else + copied = copy_page_to_iter(page, offset, f->length, iter); + addr += copied; + len -= copied; + if (copied < f->length && iov_iter_count(iter)) { + ret = -EFAULT; + break; + } + } + release_user_fragments(fragments, got, write); + if (ret) + break; + } + if (fragments != stack) + kfree(fragments); + return ret; +} +#endif + /** * process_vm_rw_core - core of reading/writing pages from task specified * @pid: PID of process to read/write from/to @@ -182,17 +242,6 @@ static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter, if (nr_pages == 0) return 0; - if (nr_pages > PVM_MAX_PP_ARRAY_COUNT) { - /* For reliability don't try to kmalloc more than - 2 pages worth */ - process_pages = kmalloc(min_t(size_t, PVM_MAX_KMALLOC_PAGES * PAGE_SIZE, - sizeof(struct page *)*nr_pages), - GFP_KERNEL); - - if (!process_pages) - return -ENOMEM; - } - /* Get process information */ task = find_get_task_by_vpid(pid); if (!task) { @@ -212,11 +261,35 @@ static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter, goto put_task_struct; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) != PAGE_SIZE) { + for (i = 0; i < riovcnt && iov_iter_count(iter) && !rc; i++) + rc = process_vm_rw_fragments(mm, (unsigned long)rvec[i].iov_base, + rvec[i].iov_len, iter, vm_write); + goto copied; + } +#endif + if (nr_pages > PVM_MAX_PP_ARRAY_COUNT) { + /* For reliability don't try to kmalloc more than + 2 pages worth */ + process_pages = kmalloc(min_t(size_t, PVM_MAX_KMALLOC_PAGES * PAGE_SIZE, + sizeof(struct page *)*nr_pages), + GFP_KERNEL); + + if (!process_pages) { + rc = -ENOMEM; + goto put_mm; + } + } + for (i = 0; i < riovcnt && iov_iter_count(iter) && !rc; i++) rc = process_vm_rw_single_vec( (unsigned long)rvec[i].iov_base, rvec[i].iov_len, iter, process_pages, mm, task, vm_write); +#ifdef CONFIG_MM_SUBPAGE +copied: +#endif /* copied = space before - space after */ total_len -= iov_iter_count(iter); @@ -226,6 +299,7 @@ static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter, if (total_len) rc = total_len; +put_mm: mmput(mm); put_task_struct: diff --git a/mm/rmap.c b/mm/rmap.c index acdadb470279a..cda21c4fe5742 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -75,6 +75,7 @@ #include #include #include +#include #include @@ -83,17 +84,22 @@ #include "internal.h" #include "swap.h" +#include "user-subpage-internal.h" static struct kmem_cache *anon_vma_cachep; static struct kmem_cache *anon_vma_chain_cachep; -static inline struct anon_vma *anon_vma_alloc(void) +static inline struct anon_vma *anon_vma_alloc(struct vm_area_struct *vma) { struct anon_vma *anon_vma; anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL); if (anon_vma) { atomic_set(&anon_vma->refcount, 1); +#ifdef CONFIG_MM_SUBPAGE + anon_vma->min_folio_order = + max_t(unsigned int, mm_page_shift(vma->vm_mm), PAGE_SHIFT) - PAGE_SHIFT; +#endif anon_vma->num_children = 0; anon_vma->num_active_vmas = 0; anon_vma->parent = anon_vma; @@ -198,7 +204,7 @@ int __anon_vma_prepare(struct vm_area_struct *vma) anon_vma = find_mergeable_anon_vma(vma); allocated = NULL; if (!anon_vma) { - anon_vma = anon_vma_alloc(); + anon_vma = anon_vma_alloc(vma); if (unlikely(!anon_vma)) goto out_enomem_free_avc; anon_vma->num_children++; /* self-parent link for new root */ @@ -392,7 +398,7 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) /* Drop inherited anon_vma, we'll reuse existing or allocate new. */ vma->anon_vma = NULL; - anon_vma = anon_vma_alloc(); + anon_vma = anon_vma_alloc(vma); if (!anon_vma) return -ENOMEM; avc = anon_vma_chain_alloc(GFP_KERNEL); @@ -626,6 +632,35 @@ struct anon_vma *folio_get_anon_vma(const struct folio *folio) return anon_vma; } +#ifdef CONFIG_MM_SUBPAGE +/* The caller retains the folio; splitting rechecks the order under its lock. */ +unsigned int folio_anon_min_order(struct folio *folio) +{ + struct anon_vma *anon_vma = NULL; + unsigned long mapping; + unsigned int order = 0; + + /* Also used by memory-failure callers before they lock the folio. */ + rcu_read_lock(); + mapping = (unsigned long)READ_ONCE(folio->mapping); + if ((mapping & FOLIO_MAPPING_FLAGS) != FOLIO_MAPPING_ANON || !folio_mapped(folio)) + goto out; + anon_vma = (struct anon_vma *)(mapping - FOLIO_MAPPING_ANON); + if (!atomic_inc_not_zero(&anon_vma->refcount)) { + anon_vma = NULL; + goto out; + } + /* Revalidate against unmap/reuse of the RCU-typesafe anon_vma slab. */ + if (folio_mapped(folio) && mapping == (unsigned long)READ_ONCE(folio->mapping)) + order = anon_vma->min_folio_order; +out: + rcu_read_unlock(); + if (anon_vma) + put_anon_vma(anon_vma); + return order; +} +#endif + /* * Similar to folio_get_anon_vma() except it locks the anon_vma. * @@ -860,6 +895,13 @@ static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) unsigned long page_address_in_vma(const struct folio *folio, const struct page *page, const struct vm_area_struct *vma) { +#ifdef CONFIG_MM_SUBPAGE + unsigned long address; + + if (folio_test_anon(folio) && !folio_test_large(folio) && + mm_subpage_address_in_vma(folio, vma, &address)) + return address; +#endif if (folio_test_anon(folio)) { struct anon_vma *anon_vma = folio_anon_vma(folio); /* @@ -895,15 +937,15 @@ pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) if (!pgd_present(*pgd)) goto out; - p4d = p4d_offset(pgd, address); - if (!p4d_present(*p4d)) + p4d = p4d_offset_mm(mm, pgd, address); + if (!p4d_present_mm(mm, *p4d)) goto out; - pud = pud_offset(p4d, address); + pud = pud_offset_mm(mm, p4d, address); if (!pud_present(*pud)) goto out; - pmd = pmd_offset(pud, address); + pmd = pmd_offset_mm(mm, pud, address); out: return pmd; } @@ -919,10 +961,11 @@ struct folio_referenced_arg { * arg: folio_referenced_arg will be passed */ static bool folio_referenced_one(struct folio *folio, - struct vm_area_struct *vma, unsigned long address, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { + unsigned long address = rmap.address; struct folio_referenced_arg *pra = arg; - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); int ptes = 0, referenced = 0; unsigned int nr; @@ -932,10 +975,11 @@ static bool folio_referenced_one(struct folio *folio, if (vma->vm_flags & VM_LOCKED) { ptes++; - pra->mapcount--; + pra->mapcount -= pvmw.pte ? mm_pte_native_pages(vma->vm_mm) : 1; /* Only mlock fully mapped pages */ - if (pvmw.pte && ptes != pvmw.nr_pages) + if (pvmw.pte && ptes != (folio_test_large(folio) ? + ((unsigned long)pvmw.nr_pages << PAGE_SHIFT) >> mm_page_shift(vma->vm_mm) : 1)) continue; /* @@ -970,11 +1014,18 @@ static bool folio_referenced_one(struct folio *folio, } if (pvmw.pte && folio_test_large(folio)) { - const unsigned long end_addr = pmd_addr_end(address, vma->vm_end); - const unsigned int max_nr = (end_addr - address) >> PAGE_SHIFT; + const unsigned long end_addr = pmd_addr_end_mm(vma->vm_mm, + address, vma->vm_end); + const unsigned int max_nr = (end_addr - address) >> mm_page_shift(vma->vm_mm); pte_t pteval = ptep_get(pvmw.pte); - nr = folio_pte_batch(folio, pvmw.pte, pteval, max_nr); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + nr = folio_subpage_pte_batch(vma, folio, pvmw.pte, + pteval, max_nr, 0); + else +#endif + nr = folio_pte_batch(folio, pvmw.pte, pteval, max_nr); } /* @@ -998,19 +1049,19 @@ static bool folio_referenced_one(struct folio *folio, } ptes += nr; - pra->mapcount -= nr; + pra->mapcount -= nr * (pvmw.pte ? mm_pte_native_pages(vma->vm_mm) : 1); /* * If we are sure that we batched the entire folio, * we can just optimize and stop right here. */ - if (ptes == pvmw.nr_pages) { + if (ptes == (((unsigned long)pvmw.nr_pages << PAGE_SHIFT) >> mm_page_shift(vma->vm_mm))) { page_vma_mapped_walk_done(&pvmw); break; } /* Skip the batched PTEs */ pvmw.pte += nr - 1; - pvmw.address += (nr - 1) * PAGE_SIZE; + pvmw.address += (nr - 1) * mm_page_size(vma->vm_mm); } if (referenced) @@ -1182,9 +1233,9 @@ static int page_vma_mkclean_one(struct page_vma_mapped_walk *pvmw) } static bool page_mkclean_one(struct folio *folio, struct vm_area_struct *vma, - unsigned long address, void *arg) + struct rmap_walk_range rmap, void *arg) { - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, PVMW_SYNC); int *cleaned = arg; *cleaned += page_vma_mkclean_one(&pvmw); @@ -1233,8 +1284,9 @@ struct wrprotect_file_state { }; static bool mapping_wrprotect_range_one(struct folio *folio, - struct vm_area_struct *vma, unsigned long address, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { + unsigned long address = rmap.address; struct wrprotect_file_state *state = (struct wrprotect_file_state *)arg; struct page_vma_mapped_walk pvmw = { .pfn = state->pfn, @@ -1519,7 +1571,7 @@ static void __page_check_anon_rmap(const struct folio *folio, static __always_inline void __folio_add_anon_rmap(struct folio *folio, struct page *page, int nr_pages, struct vm_area_struct *vma, - unsigned long address, rmap_t flags, enum pgtable_level level) + unsigned long address, rmap_t flags, enum pgtable_level level, bool new) { int i; @@ -1576,7 +1628,7 @@ static __always_inline void __folio_add_anon_rmap(struct folio *folio, * Partially mapped folios can be split on reclaim and part outside * of mlocked VMA can be evicted or freed. */ - if (folio_nr_pages(folio) == nr_pages) + if (!new && folio_nr_pages(folio) == nr_pages) mlock_vma_folio(folio, vma); } @@ -1601,7 +1653,7 @@ void folio_add_anon_rmap_ptes(struct folio *folio, struct page *page, rmap_t flags) { __folio_add_anon_rmap(folio, page, nr_pages, vma, address, flags, - PGTABLE_LEVEL_PTE); + PGTABLE_LEVEL_PTE, false); } /** @@ -1622,7 +1674,7 @@ void folio_add_anon_rmap_pmd(struct folio *folio, struct page *page, { #ifdef CONFIG_TRANSPARENT_HUGEPAGE __folio_add_anon_rmap(folio, page, HPAGE_PMD_NR, vma, address, flags, - PGTABLE_LEVEL_PMD); + PGTABLE_LEVEL_PMD, false); #else WARN_ON_ONCE(true); #endif @@ -1701,6 +1753,112 @@ void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma, mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, 1); } +#ifdef CONFIG_MM_SUBPAGE +static int __mm_subpage_add_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address, bool new) +{ + struct folio *folio = mm_subpage_folio(subpage); + int ret; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (mm_page_shift(vma->vm_mm) != mm_subpage_shift(subpage) || !vma->anon_vma || + address < vma->vm_start || address >= vma->vm_end || + !IS_ALIGNED(address, mm_subpage_size(subpage))) + return -EINVAL; + if (mm_subpage_anon_root(subpage)) { + if (mm_subpage_is_droppable(subpage) != !!(vma->vm_flags & VM_DROPPABLE)) + return -EINVAL; + if (!folio_test_anon(folio) || folio_test_ksm(folio) || + PageAnonExclusive(&folio->page)) + return -EINVAL; + } else { + bool was_anon = folio_test_anon(folio); + + /* Swap-cache folios can retain a stale, unmapped anon pointer. */ + if (folio_mapped(folio) || folio_test_ksm(folio) || + (folio->mapping && !was_anon) || + (was_anon && PageAnonExclusive(&folio->page))) + return -EINVAL; + /* Slot references can survive the last VMA in this root. */ + mm_subpage_hold_anon_root(subpage, vma->anon_vma->root, + vma->vm_flags & VM_DROPPABLE); + /* + * A restored folio is already visible in the swap cache. Other + * faults may set PG_waiters while we hold its lock; preserve those + * concurrent flag updates even when PG_swapbacked is already set. + */ + if (!(vma->vm_flags & VM_DROPPABLE)) + folio_set_swapbacked(folio); + /* Neighboring slots may acquire different fork/COW relationships. */ + __folio_set_anon(folio, vma, address, false); + if (!was_anon) + mod_mthp_stat(0, MTHP_STAT_NR_ANON, 1); + } + ret = mm_subpage_bind_rmap(subpage, vma, address); + if (ret) + return ret; + atomic_inc(&subpage->mapcount); + /* Identity was checked per slot, not against the folio's linear anchor. */ + __folio_add_rmap(folio, &folio->page, 1, vma, PGTABLE_LEVEL_PTE); + if (!new) + mlock_vma_folio(folio, vma); + return 0; +} + +int mm_subpage_add_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address) +{ + return __mm_subpage_add_anon_rmap(subpage, vma, address, false); +} + +/* Caller adds this fresh folio to LRU and accounts its locked leaves together. */ +int mm_subpage_add_new_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address) +{ + return __mm_subpage_add_anon_rmap(subpage, vma, address, true); +} + +void mm_subpage_remove_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma) +{ + struct folio *folio = mm_subpage_folio(subpage); + + VM_BUG_ON_FOLIO(atomic_read(&subpage->mapcount) <= 0, folio); + atomic_dec(&subpage->mapcount); + folio_remove_rmap_pte(folio, &folio->page, vma); +} + +int mm_subpage_dup_anon_rmap(struct mm_subpage *subpage, + struct vm_area_struct *dst, struct vm_area_struct *src, unsigned long address) +{ + struct folio *folio = mm_subpage_folio(subpage); + struct vm_page_offset pos; + int ret; + + mmap_assert_write_locked(src->vm_mm); + mmap_assert_write_locked(dst->vm_mm); + if (mm_page_shift(src->vm_mm) != mm_subpage_shift(subpage) || + mm_page_shift(dst->vm_mm) != mm_subpage_shift(subpage) || + !src->anon_vma || !dst->anon_vma || !folio_test_anon(folio) || + folio_test_ksm(folio) || PageAnonExclusive(&folio->page) || + address < dst->vm_start || address >= dst->vm_end || + address < src->vm_start || address >= src->vm_end) + return -EINVAL; + pos = vma_page_offset_at(dst, address); + /* Both mmap locks and the source PTE stabilize this slot's identity. */ + if (!mm_subpage_match_rmap(subpage, dst, address) || + src->anon_vma->root != dst->anon_vma->root || + !vm_page_offset_equal(pos, vma_page_offset_at(src, address))) + return -EINVAL; + /* A pin in a neighboring physical slot does not share this slot. */ + if (!mm_subpage_try_share(subpage)) + return -EBUSY; + ret = folio_try_dup_anon_rmap_pte(folio, &folio->page, dst, src); + if (!ret) + atomic_inc(&subpage->mapcount); + return ret; +} +#endif + static __always_inline void __folio_add_file_rmap(struct folio *folio, struct page *page, int nr_pages, struct vm_area_struct *vma, enum pgtable_level level) @@ -1953,14 +2111,20 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio, struct vm_area_struct *vma = pvmw->vma; unsigned int max_nr; +#ifdef CONFIG_MM_SUBPAGE + /* Native PFN batching advances physical pages, not 4K file quarters. */ + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return 1; +#endif + if (flags & TTU_HWPOISON) return 1; if (!folio_test_large(folio)) return 1; /* We may only batch within a single VMA and a single page table. */ - end_addr = pmd_addr_end(addr, vma->vm_end); - max_nr = (end_addr - addr) >> PAGE_SHIFT; + end_addr = pmd_addr_end_mm(vma->vm_mm, addr, vma->vm_end); + max_nr = (end_addr - addr) >> mm_page_shift(vma->vm_mm); /* We only support lazyfree or file folios batching for now ... */ if (folio_test_anon(folio) && folio_test_swapbacked(folio)) @@ -1982,20 +2146,116 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio, FPB_RESPECT_WRITE | FPB_RESPECT_SOFT_DIRTY); } +#ifdef CONFIG_MM_SUBPAGE +/* Folio and PTE locks held. Each PTE takes one reference to a native swap entry. */ +static bool try_to_unmap_subpage(struct folio *folio, struct vm_area_struct *vma, + struct page_vma_mapped_walk *pvmw, enum ttu_flags flags) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long address = pvmw->address, end = address + mm_page_size(mm); + struct mm_subpage *slot; + pte_t pte = ptep_get(pvmw->pte), swapped; + swp_entry_t entry; + bool exclusive, ret = false; + + if (!pte_present(pte) || (flags & TTU_HWPOISON)) + return false; + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + if (WARN_ON_ONCE(!slot)) + return false; + if ((vma->vm_flags & VM_DROPPABLE) || mm_subpage_is_lazyfree(slot)) { + flush_cache_range(vma, address, end); + pte = ptep_get_and_clear(mm, address, pvmw->pte); + /* A released quarter can be reused before its native folio is freed. */ + flush_tlb_range(vma, address, end); + smp_mb(); + if (((vma->vm_flags & VM_DROPPABLE) || !pte_dirty(pte)) && + mm_subpage_discardable(slot) && + arch_unmap_one(mm, vma, address, pte) >= 0) { + update_hiwater_rss(mm); + dec_mm_counter(mm, MM_ANONPAGES); + mm_subpage_remove_anon_rmap(slot, vma); + mm_subpage_put(slot); /* Removed PTE reference. */ + ret = true; + goto out; + } + set_pte_at(mm, address, pvmw->pte, pte); + /* Retained references must not turn a droppable folio into swap data. */ + if (vma->vm_flags & VM_DROPPABLE) + goto out; + if (pte_dirty(pte)) { + mm_subpage_cancel_lazyfree(slot); + folio_mark_dirty(folio); + } + } + if (!folio_test_swapbacked(folio)) { + /* A write or retained reference cancelled whole-folio lazy reclaim. */ + folio_set_swapbacked(folio); + goto out; + } + if (!folio_test_swapcache(folio)) + goto out; + entry = page_swap_entry(&folio->page); + if (swap_subpage_dup(entry, mm_subpage_offset(slot)) < 0) + goto out; + flush_cache_range(vma, address, end); + pte = ptep_get_and_clear(mm, address, pvmw->pte); + if (should_defer_flush(mm, flags)) + set_tlb_ubc_flush_pending(mm, pte, address, end); + else + flush_tlb_range(vma, address, end); + if (pte_dirty(pte)) + folio_mark_dirty(folio); + /* The PTE lock serializes fork and this slot's ownership transfer. */ + exclusive = mm_subpage_is_exclusive(slot); + if (arch_unmap_one(mm, vma, address, pte) < 0 || !mm_subpage_try_share(slot)) { + swap_subpage_put(entry, mm_subpage_offset(slot), false); + set_pte_at(mm, address, pvmw->pte, pte); + goto out; + } + if (list_empty(&mm->mmlist)) { + spin_lock(&mmlist_lock); + if (list_empty(&mm->mmlist)) + list_add(&mm->mmlist, &init_mm.mmlist); + spin_unlock(&mmlist_lock); + } + swapped = pte_swp_set_subpage_offset(swp_entry_to_pte(entry), mm_subpage_offset(slot)); + if (exclusive) + swapped = pte_swp_mkexclusive(swapped); + if (pte_soft_dirty(pte)) + swapped = pte_swp_mksoft_dirty(swapped); + if (pte_uffd_wp(pte)) + swapped = pte_swp_mkuffd_wp(swapped); + update_hiwater_rss(mm); + dec_mm_counter(mm, MM_ANONPAGES); + inc_mm_counter(mm, MM_SWAPENTS); + set_pte_at(mm, address, pvmw->pte, swapped); + mm_subpage_remove_anon_rmap(slot, vma); + /* Swap-cache backing cannot be recycled while a deferred TLBI is pending. */ + mm_subpage_put(slot); + ret = true; +out: + mm_subpage_put(slot); + return ret; +} +#endif + /* * @arg: enum ttu_flags will be passed to this argument */ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, - unsigned long address, void *arg) + struct rmap_walk_range rmap, void *arg) { + unsigned long address = rmap.address; struct mm_struct *mm = vma->vm_mm; - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); bool anon_exclusive, ret = true; pte_t pteval; + phys_addr_t file_phys = 0; struct page *subpage; struct mmu_notifier_range range; enum ttu_flags flags = (enum ttu_flags)(long)arg; - unsigned long nr_pages = 1, end_addr; + unsigned long nr_pages = 1, end_addr, nr_refs; unsigned long pfn; unsigned long hsz = 0; int ptes = 0; @@ -2007,7 +2267,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * if page table locking is skipped: use TTU_SYNC to wait for that. */ if (flags & TTU_SYNC) - pvmw.flags = PVMW_SYNC; + pvmw.flags |= PVMW_SYNC; /* * For THP, we have to assume the worse case ie pmd for invalidation. @@ -2053,7 +2313,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, ret = false; /* Only mlock fully mapped pages */ - if (pvmw.pte && ptes != pvmw.nr_pages) + if (pvmw.pte && ptes != (folio_test_large(folio) ? + ((unsigned long)pvmw.nr_pages << PAGE_SHIFT) >> mm_page_shift(vma->vm_mm) : 1)) continue; /* @@ -2105,6 +2366,15 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, } else { pteval = ptep_get(pvmw.pte); } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE && folio_test_anon(folio)) { + if (!try_to_unmap_subpage(folio, vma, &pvmw, flags)) + goto walk_abort; + /* One native folio may cover four independently mapped slots. */ + continue; + } +#endif + if (likely(pte_present(pteval))) { pfn = pte_pfn(pteval); } else { @@ -2119,6 +2389,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, } subpage = folio_page(folio, pfn - folio_pfn(folio)); + if (!folio_test_anon(folio)) + file_phys = pte_phys_mm(mm, ptep_get(pvmw.pte)); anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(subpage); @@ -2175,7 +2447,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, folio_mark_dirty(folio); } else if (likely(pte_present(pteval))) { nr_pages = folio_unmap_pte_batch(folio, &pvmw, flags, pteval); - end_addr = address + nr_pages * PAGE_SIZE; + end_addr = address + nr_pages * mm_page_size(mm); flush_cache_range(vma, address, end_addr); /* Nuke the page table entry. */ @@ -2289,7 +2561,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, goto discard; } - if (folio_dup_swap(folio, subpage) < 0) { + if (folio_dup_swap_range(folio, subpage, mm_pte_native_pages(mm)) < 0) { set_pte_at(mm, address, pvmw.pte, pteval); goto walk_abort; } @@ -2300,15 +2572,15 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * so we'll not check/care. */ if (arch_unmap_one(mm, vma, address, pteval) < 0) { - folio_put_swap(folio, subpage); + folio_put_swap_range(folio, subpage, mm_pte_native_pages(mm)); set_pte_at(mm, address, pvmw.pte, pteval); goto walk_abort; } /* See folio_try_share_anon_rmap(): clear PTE first. */ if (anon_exclusive && - folio_try_share_anon_rmap_pte(folio, subpage)) { - folio_put_swap(folio, subpage); + folio_try_share_anon_rmap_pte(folio, subpage, vma)) { + folio_put_swap_range(folio, subpage, mm_pte_native_pages(mm)); set_pte_at(mm, address, pvmw.pte, pteval); goto walk_abort; } @@ -2350,20 +2622,26 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, add_mm_counter(mm, mm_counter_file(folio), -nr_pages); } discard: + nr_refs = nr_pages; + /* A base user leaf can own references to several native pages. */ + if (!folio_test_hugetlb(folio)) + nr_refs *= mm_pte_native_pages(mm); + if (!folio_test_anon(folio)) + mm_subpage_file_map_del(vma, file_phys); if (unlikely(folio_test_hugetlb(folio))) { hugetlb_remove_rmap(folio); } else { - folio_remove_rmap_ptes(folio, subpage, nr_pages, vma); + folio_remove_rmap_ptes(folio, subpage, nr_refs, vma); } if (vma->vm_flags & VM_LOCKED) mlock_drain_local(); - folio_put_refs(folio, nr_pages); + folio_put_refs(folio, nr_refs); /* * If we are sure that we batched the entire folio and cleared * all PTEs, we can just optimize and stop right here. */ - if (nr_pages == folio_nr_pages(folio)) + if (nr_pages == folio_nr_pages(folio) && mm_page_size(mm) == PAGE_SIZE) goto walk_done; continue; walk_abort: @@ -2420,11 +2698,72 @@ void try_to_unmap(struct folio *folio, enum ttu_flags flags) * If TTU_SPLIT_HUGE_PMD is specified any PMD mappings will be split into PTEs * containing migration entries. */ +#ifdef CONFIG_MM_SUBPAGE +static bool try_to_migrate_subpage(struct folio *folio, struct vm_area_struct *vma, + struct page_vma_mapped_walk *pvmw, enum ttu_flags flags) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long address = pvmw->address, end = address + mm_page_size(mm); + struct mm_subpage *slot; + pte_t pte = ptep_get(pvmw->pte), migration; + swp_entry_t entry; + bool ret = false; + + /* Other migration drivers must prepare ownership before removing PTEs. */ + if (!pte_present(pte) || !mm_subpage_migrating(folio) || + folio_test_hwpoison(folio) || folio_is_zone_device(folio)) + return false; + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + if (WARN_ON_ONCE(!slot)) + return false; + flush_cache_range(vma, address, end); + pte = ptep_get_and_clear(mm, address, pvmw->pte); + if (should_defer_flush(mm, flags)) + set_tlb_ubc_flush_pending(mm, pte, address, end); + else + flush_tlb_range(vma, address, end); + if (pte_dirty(pte)) + folio_mark_dirty(folio); + if (arch_unmap_one(mm, vma, address, pte) < 0) { + set_pte_at(mm, address, pvmw->pte, pte); + goto out; + } + if (pte_write(pte)) + entry = make_writable_migration_entry(folio_pfn(folio)); + else if (mm_subpage_is_exclusive(slot)) + entry = make_readable_exclusive_migration_entry(folio_pfn(folio)); + else + entry = make_readable_migration_entry(folio_pfn(folio)); + if (pte_young(pte)) + entry = make_migration_entry_young(entry); + if (pte_dirty(pte)) + entry = make_migration_entry_dirty(entry); + migration = pte_swp_set_subpage_offset(swp_entry_to_pte(entry), mm_subpage_offset(slot)); + if (pte_soft_dirty(pte)) + migration = pte_swp_mksoft_dirty(migration); + if (pte_uffd_wp(pte)) + migration = pte_swp_mkuffd_wp(migration); + set_pte_at(mm, address, pvmw->pte, migration); + trace_set_migration_pte(address, pte_val(migration), 0); + mm_subpage_migrate_save_lazyfree(slot); + mm_subpage_remove_anon_rmap(slot, vma); + if (vma->vm_flags & VM_LOCKED) + mlock_drain_local(); + /* The driver holds backing until its batched invalidation completes. */ + mm_subpage_put(slot); + ret = true; +out: + mm_subpage_put(slot); + return ret; +} +#endif + static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, - unsigned long address, void *arg) + struct rmap_walk_range rmap, void *arg) { + unsigned long address = rmap.address; struct mm_struct *mm = vma->vm_mm; - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); bool anon_exclusive, writable, ret = true; pte_t pteval; struct page *subpage; @@ -2432,6 +2771,11 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, enum ttu_flags flags = (enum ttu_flags)(long)arg; unsigned long pfn; unsigned long hsz = 0; + unsigned int nr_refs = 1; + + /* A base PTE owns one reference for each covered native page. */ + if (!folio_test_hugetlb(folio)) + nr_refs = mm_pte_native_pages(mm); /* * When racing against e.g. zap_pte_range() on another cpu, @@ -2440,7 +2784,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, * if page table locking is skipped: use TTU_SYNC to wait for that. */ if (flags & TTU_SYNC) - pvmw.flags = PVMW_SYNC; + pvmw.flags |= PVMW_SYNC; /* * For THP, we have to assume the worse case ie pmd for invalidation. @@ -2510,6 +2854,16 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, VM_BUG_ON_FOLIO(!pvmw.pte, folio); address = pvmw.address; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE && folio_test_anon(folio)) { + if (!try_to_migrate_subpage(folio, vma, &pvmw, flags)) { + ret = false; + page_vma_mapped_walk_done(&pvmw); + break; + } + continue; + } +#endif if (folio_test_hugetlb(folio)) pteval = huge_ptep_get(mm, address, pvmw.pte); else @@ -2597,7 +2951,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, */ pteval = ptep_get_and_clear(mm, address, pvmw.pte); - set_tlb_ubc_flush_pending(mm, pteval, address, address + PAGE_SIZE); + set_tlb_ubc_flush_pending(mm, pteval, address, address + mm_page_size(mm)); } else { pteval = ptep_clear_flush(vma, address, pvmw.pte); } @@ -2674,7 +3028,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, break; } } else if (anon_exclusive && - folio_try_share_anon_rmap_pte(folio, subpage)) { + folio_try_share_anon_rmap_pte(folio, subpage, vma)) { set_pte_at(mm, address, pvmw.pte, pteval); ret = false; page_vma_mapped_walk_done(&pvmw); @@ -2712,6 +3066,13 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, if (pte_swp_uffd_wp(pteval)) swp_pte = pte_swp_mkuffd_wp(swp_pte); } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) + swp_pte = pte_swp_set_subpage_offset(swp_pte, + pte_present(pteval) ? pte_phys_mm(mm, pteval) & ~PAGE_MASK : + pte_swp_subpage_offset(pteval)); +#endif + if (folio_test_hugetlb(folio)) set_huge_pte_at(mm, address, pvmw.pte, swp_pte, hsz); @@ -2725,13 +3086,15 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, */ } + if (!folio_test_anon(folio)) + mm_subpage_file_map_del(vma, pte_phys_mm(mm, pteval)); if (unlikely(folio_test_hugetlb(folio))) hugetlb_remove_rmap(folio); else folio_remove_rmap_pte(folio, subpage, vma); if (vma->vm_flags & VM_LOCKED) mlock_drain_local(); - folio_put(folio); + folio_put_refs(folio, nr_refs); } mmu_notifier_invalidate_range_end(&range); @@ -2962,6 +3325,61 @@ static struct anon_vma *rmap_walk_anon_lock(const struct folio *folio, return anon_vma; } +#ifdef CONFIG_MM_SUBPAGE +noinline bool rmap_walk_subpages(struct folio *folio, + struct rmap_walk_control *rwc, bool locked) +{ + struct mm_subpage_rmap maps[MM_SUBPAGES_PER_PAGE]; + int i, nr = mm_subpage_snapshot_rmap(folio, maps); + + if (nr < 0) + return false; + /* Existing locked anonymous walks operate on large folios. */ + VM_BUG_ON_FOLIO(locked, folio); + for (i = 0; i < nr; i++) { + struct mm_subpage_rmap *map = &maps[i]; + struct anon_vma_chain *avc; + bool stop = false; + + if (!anon_vma_trylock_read(map->root)) { + if (rwc->try_lock) { + rwc->contended = true; + break; + } + anon_vma_lock_read(map->root); + } + anon_vma_interval_tree_foreach(avc, &map->root->rb_root, + map->pos.index, map->pos.index) { + struct vm_area_struct *vma = avc->vma; + unsigned long address = vma_address_at_offset(vma, map->pos); + struct rmap_walk_range range = { + .address = address, + .subpage = true, + .subpage_offset = map->offset, + }; + + /* Native-index overlap need not include this exact leaf. */ + if (address == -EFAULT) + continue; + cond_resched(); + if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) + continue; + if (!rwc->rmap_one(folio, vma, range, rwc->arg) || + (rwc->done && rwc->done(folio))) { + stop = true; + break; + } + } + anon_vma_unlock_read(map->root); + if (stop) + break; + } + for (i = 0; i < nr; i++) + put_anon_vma(maps[i].root); + return true; +} +#endif + /* * rmap_walk_anon - do something to anonymous page using the object-based * rmap method @@ -2984,6 +3402,10 @@ static void rmap_walk_anon(struct folio *folio, * to an anon_vma with different root. */ VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); +#ifdef CONFIG_MM_SUBPAGE + if (!folio_test_large(folio) && rmap_walk_subpages(folio, rwc, locked)) + return; +#endif if (locked) { anon_vma = folio_anon_vma(folio); @@ -3009,7 +3431,8 @@ static void rmap_walk_anon(struct folio *folio, if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) continue; - if (!rwc->rmap_one(folio, vma, address, rwc->arg)) + if (!rwc->rmap_one(folio, vma, + (struct rmap_walk_range) { .address = address }, rwc->arg)) break; if (rwc->done && rwc->done(folio)) break; @@ -3072,7 +3495,8 @@ static void __rmap_walk_file(struct folio *folio, struct address_space *mapping, if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) continue; - if (!rwc->rmap_one(folio, vma, address, rwc->arg)) + if (!rwc->rmap_one(folio, vma, + (struct rmap_walk_range) { .address = address }, rwc->arg)) goto done; if (rwc->done && rwc->done(folio)) goto done; diff --git a/mm/shmem.c b/mm/shmem.c index cfb5467852951..a1d9a648916ab 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -30,6 +30,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -885,6 +888,266 @@ static void shmem_update_stats(struct folio *folio, int nr_pages) lruvec_stat_mod_folio(folio, NR_SHMEM, nr_pages); } +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +#define SHMEM_UFFD_ALL_UNITS GENMASK(PAGE_SIZE / SZ_4K - 1, 0) + +/* + * Resident masks live in page_ext, allowing partial truncate to update them + * without allocation. All cache insertion/replacement paths initialize or + * transfer them under the folio lock. The inode XArray persists masks only + * for swapped indices. Its publication/removal additionally holds i_pages, + * before taking uffd_missing's XArray lock. + */ +static void shmem_uffd_enable(struct inode *inode) +{ + struct shmem_inode_info *info = SHMEM_I(inode); + + spin_lock(&info->lock); + info->flags |= SHMEM_F_UFFD_SUBPAGE; + spin_unlock(&info->lock); +} + +static unsigned long shmem_uffd_mask(unsigned int offset, unsigned long length) +{ + return GENMASK((offset + length - 1) / SZ_4K, offset / SZ_4K); +} + +unsigned long shmem_uffd_missing_mask(struct folio *folio, pgoff_t index) +{ + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio); + VM_BUG_ON_FOLIO(!folio_mapping(folio) || !shmem_mapping(folio_mapping(folio)), folio); + return mm_subpage_shmem_missing(folio_page(folio, index - folio->index)); +} + +int shmem_uffd_set_missing(struct folio *folio, pgoff_t index, + unsigned long missing) +{ + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!folio_mapping(folio) || !shmem_mapping(folio_mapping(folio)) || + !folio_contains(folio, index) || + !folio_test_uptodate(folio) || (missing & ~SHMEM_UFFD_ALL_UNITS)) + return -EINVAL; + if (missing) + shmem_uffd_enable(folio->mapping->host); + mm_subpage_shmem_set_missing(folio_page(folio, index - folio->index), missing); + return 0; +} + +bool shmem_uffd_has_missing_native_page(struct folio *folio) +{ + struct address_space *mapping = folio_mapping(folio); + unsigned long i; + + if (!mapping || !shmem_mapping(mapping) || + !(READ_ONCE(SHMEM_I(mapping->host)->flags) & SHMEM_F_UFFD_SUBPAGE)) + return false; + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + /* Native PMD exposure is allowed when every native leaf has some data. */ + for (i = 0; i < folio_nr_pages(folio); i++) + if (mm_subpage_shmem_missing(folio_page(folio, i)) == SHMEM_UFFD_ALL_UNITS) + return true; + return false; +} + +bool shmem_uffd_range_missing(struct folio *folio, pgoff_t index, + unsigned int offset, unsigned long length) +{ + struct address_space *mapping = folio_mapping(folio); + unsigned long mask; + + if (!mapping || !shmem_mapping(mapping)) + return false; + mask = shmem_uffd_mask(offset, length); + return (shmem_uffd_missing_mask(folio, index) & mask) == mask; +} + +/* A coarse alias with any populated units preserves them and exposes zero holes. */ +static unsigned long shmem_uffd_reason(struct vm_fault *vmf, unsigned long missing) +{ + struct vm_area_struct *vma = vmf->vma; + struct vm_page_offset pos = vma_page_offset_at(vma, vmf->address); + unsigned long mask = shmem_uffd_mask(pos.offset, mm_page_size(vma->vm_mm)); + + if ((missing & mask) == mask) + return userfaultfd_missing(vma) ? VM_UFFD_MISSING : 0; + return userfaultfd_minor(vma) ? VM_UFFD_MINOR : 0; +} + +void shmem_uffd_mark_present(struct folio *folio, pgoff_t index, + unsigned long present) +{ + unsigned long missing = shmem_uffd_missing_mask(folio, index); + + if (missing & present) + mm_subpage_shmem_set_missing(folio_page(folio, index - folio->index), + missing & ~present); +} + +void shmem_uffd_mark_mapped(struct folio *folio, unsigned long offset, + unsigned long length) +{ + struct address_space *mapping = folio_mapping(folio); + + if (!mapping || !shmem_mapping(mapping) || + !(READ_ONCE(SHMEM_I(mapping->host)->flags) & SHMEM_F_UFFD_SUBPAGE)) + return; + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + VM_BUG_ON_FOLIO(offset + length > folio_size(folio), folio); + while (length) { + unsigned int low = offset & ~PAGE_MASK; + unsigned long size = min(length, PAGE_SIZE - low); + unsigned long mask = GENMASK((low + size - 1) / SZ_4K, low / SZ_4K); + + shmem_uffd_mark_present(folio, folio->index + (offset >> PAGE_SHIFT), mask); + offset += size; + length -= size; + } +} + +void shmem_uffd_forget(struct address_space *mapping, pgoff_t start, + unsigned long nr) +{ + struct xarray *xa; + unsigned long index; + void *entry; + + if (!shmem_mapping(mapping)) + return; + lockdep_assert_held(&mapping->i_pages.xa_lock); + xa = &SHMEM_I(mapping->host)->uffd_missing; + if (xa_empty(xa)) + return; + xa_lock(xa); + xa_for_each_range(xa, index, entry, start, start + nr - 1) + __xa_erase(xa, index); + xa_unlock(xa); +} + +/* New cache identity or swap-in: initialize every native page before unlock. */ +static void shmem_uffd_cache_init(struct folio *folio, bool from_swap) +{ + struct address_space *mapping = folio->mapping; + struct xarray *xa = &SHMEM_I(mapping->host)->uffd_missing; + unsigned long i; + + lockdep_assert_held(&mapping->i_pages.xa_lock); + for (i = 0; i < folio_nr_pages(folio); i++) { + void *entry = from_swap ? xa_load(xa, folio->index + i) : NULL; + + mm_subpage_shmem_set_missing(folio_page(folio, i), + entry ? xa_to_value(entry) : 0); + } + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); +} + +/* + * Reserve persistence before committing swap. On failure the cache folio and + * its resident mask stay intact; reclaim may retry once memory is available. + */ +static int shmem_uffd_swap_prepare(struct folio *folio) +{ + struct address_space *mapping = folio->mapping; + struct xarray *xa = &SHMEM_I(mapping->host)->uffd_missing; + struct mem_cgroup *memcg, *saved; + unsigned long i; + int err = 0; + + if (!(READ_ONCE(SHMEM_I(mapping->host)->flags) & SHMEM_F_UFFD_SUBPAGE)) + return 0; + /* Reclaim may run outside the backing folio's charge domain. */ + rcu_read_lock(); + do { + memcg = folio_memcg(folio) ?: root_mem_cgroup; + } while (memcg && !mem_cgroup_tryget(memcg)); + rcu_read_unlock(); + saved = set_active_memcg(memcg); + for (i = 0; i < folio_nr_pages(folio); i++) { + unsigned long mask = mm_subpage_shmem_missing(folio_page(folio, i)); + + if (!mask) + continue; + err = xa_err(xa_store(xa, folio->index + i, xa_mk_value(mask), + GFP_NOWAIT | __GFP_ACCOUNT | __GFP_NOWARN)); + if (err) + break; + } + set_active_memcg(saved); + mem_cgroup_put(memcg); + if (err) { + xa_lock_irq(&mapping->i_pages); + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); + xa_unlock_irq(&mapping->i_pages); + } + return err; +} + +void shmem_uffd_copy_page(struct page *dst, struct page *src) +{ + mm_subpage_shmem_set_missing(dst, src ? mm_subpage_shmem_missing(src) : 0); +} + +void shmem_uffd_migrate(struct folio *dst, struct folio *src) +{ + unsigned long i; + + if (!shmem_mapping(src->mapping)) + return; + VM_BUG_ON_FOLIO(!folio_test_locked(src), src); + VM_BUG_ON_FOLIO(!folio_test_locked(dst), dst); + for (i = 0; i < folio_nr_pages(src); i++) + shmem_uffd_copy_page(folio_page(dst, i), folio_page(src, i)); +} + +/* Only fully removed 4K units become holes; sub-unit edge bytes stay valid. */ +static void shmem_uffd_partial_hole(struct folio *folio, loff_t start, uoff_t end) +{ + loff_t first = max(start, folio_pos(folio)); + u64 last = min_t(u64, end, folio_next_pos(folio) - 1); + unsigned long offset, length; + bool pinned; + + if (first >= last || + (first == folio_pos(folio) && last == folio_next_pos(folio) - 1)) + return; + first = round_up(first, SZ_4K); + last = round_down(last + 1, SZ_4K); + if (first >= last) + return; + shmem_uffd_enable(folio->mapping->host); + /* Revoke covering coarse aliases before their holes can be refilled. */ + unmap_mapping_subpage_hole(folio->mapping, first, last); + /* Pair with fast GUP's pin-before-PTE-recheck barrier. */ + smp_mb(); + pinned = folio_maybe_dma_pinned(folio); + offset = first - folio_pos(folio); + length = last - first; + while (length) { + unsigned int low = offset & ~PAGE_MASK; + unsigned long nr = min(length, PAGE_SIZE - low); + struct page *page = folio_page(folio, offset >> PAGE_SHIFT); + unsigned long mask = shmem_uffd_mask(low, nr); + unsigned long missing = mm_subpage_shmem_missing(page); + + /* + * A pin survives PTE removal and can modify retained backing. + * Preserve native partial-hole semantics: zero it, but do not + * let a missing-page COPY overwrite a subsequent device write. + * Pins have folio granularity, so exposure is conservative when + * only a neighbouring range is pinned. Full-folio removal still + * detaches old pins from the file's new cache identity. + */ + mm_subpage_shmem_set_missing(page, pinned ? missing & ~mask : missing | mask); + offset += nr; + length -= nr; + } +} +#else +static void shmem_uffd_cache_init(struct folio *folio, bool from_swap) { } +static int shmem_uffd_swap_prepare(struct folio *folio) { return 0; } +static void shmem_uffd_partial_hole(struct folio *folio, loff_t start, uoff_t end) { } +#endif + /* * Somewhat like filemap_add_folio, but error if expected item has gone. */ @@ -932,6 +1195,7 @@ int shmem_add_to_page_cache(struct folio *folio, xas_store(&xas, folio); if (xas_error(&xas)) goto unlock; + shmem_uffd_cache_init(folio, expected != NULL); shmem_update_stats(folio, nr); mapping->nrpages += nr; unlock: @@ -986,8 +1250,10 @@ static long shmem_free_swap(struct address_space *mapping, base = round_down(xas.xa_index, nr_pages); if (base < index || base + nr_pages - 1 > end) nr_pages = 0; - else + else { + shmem_uffd_forget(mapping, base, nr_pages); xas_store(&xas, NULL); + } } xas_unlock_irq(&xas); @@ -1030,6 +1296,39 @@ unsigned long shmem_partial_swap_usage(struct address_space *mapping, return swapped << PAGE_SHIFT; } +/* Byte-clipped variant for mappings smaller than a native cache page. */ +unsigned long shmem_partial_swap_usage_bytes(struct address_space *mapping, + u64 start, u64 end) +{ + XA_STATE(xas, &mapping->i_pages, start >> PAGE_SHIFT); + void *entry; + unsigned long swapped = 0, last; + + if (start >= end) + return 0; + last = (end - 1) >> PAGE_SHIFT; + rcu_read_lock(); + xas_for_each(&xas, entry, last) { + if (xas_retry(&xas, entry)) + continue; + if (xa_is_value(entry)) { + unsigned int order = xas_get_order(&xas); + u64 first = ((u64)xas.xa_index & ~((1UL << order) - 1)) << PAGE_SHIFT; + u64 stop = first + ((u64)PAGE_SIZE << order); + + swapped += min(stop, end) - max(first, start); + } + if (xas.xa_index == last) + break; + if (need_resched()) { + xas_pause(&xas); + cond_resched_rcu(); + } + } + rcu_read_unlock(); + return swapped; +} + /* * Determine (in bytes) how many of the shmem object's pages mapped by the * given vma is swapped out. @@ -1054,6 +1353,12 @@ unsigned long shmem_swap_usage(struct vm_area_struct *vma) */ if (!swapped) return 0; + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + u64 start = ((u64)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); + + return shmem_partial_swap_usage_bytes(mapping, start, + start + vma->vm_end - vma->vm_start); + } if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size) return swapped << PAGE_SHIFT; @@ -1111,6 +1416,12 @@ static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index) return folio; } +static bool shmem_truncate_partial_folio(struct folio *folio, loff_t start, uoff_t end) +{ + shmem_uffd_partial_hole(folio, start, end); + return truncate_inode_partial_folio(folio, start, end); +} + /* * Remove range of pages and swap entries from page cache, and free them. * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate. @@ -1174,7 +1485,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend, if (folio) { same_folio = lend < folio_next_pos(folio); folio_mark_dirty(folio); - if (!truncate_inode_partial_folio(folio, lstart, lend)) { + if (!shmem_truncate_partial_folio(folio, lstart, lend)) { start = folio_next_index(folio); if (same_folio) end = folio->index; @@ -1188,7 +1499,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend, folio = shmem_get_partial_folio(inode, lend >> PAGE_SHIFT); if (folio) { folio_mark_dirty(folio); - if (!truncate_inode_partial_folio(folio, lstart, lend)) + if (!shmem_truncate_partial_folio(folio, lstart, lend)) end = folio->index; folio_unlock(folio); folio_put(folio); @@ -1257,7 +1568,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend, if (!folio_test_large(folio)) { truncate_inode_folio(mapping, folio); - } else if (truncate_inode_partial_folio(folio, lstart, lend)) { + } else if (shmem_truncate_partial_folio(folio, lstart, lend)) { /* * If we split a page, reset the loop so * that we pick up the new sub pages. @@ -1365,6 +1676,7 @@ static int shmem_setattr(struct mnt_idmap *idmap, } if (newsize <= oldsize) { loff_t holebegin = round_up(newsize, PAGE_SIZE); + unmap_mapping_subpage_tail(inode->i_mapping, newsize); if (oldsize > holebegin) unmap_mapping_range(inode->i_mapping, holebegin, 0, 1); @@ -1372,6 +1684,7 @@ static int shmem_setattr(struct mnt_idmap *idmap, shmem_truncate_range(inode, newsize, (loff_t)-1); /* unmap again to remove racily COWed private pages */ + unmap_mapping_subpage_tail(inode->i_mapping, newsize); if (oldsize > holebegin) unmap_mapping_range(inode->i_mapping, holebegin, 0, 1); @@ -1687,6 +2000,8 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug, folio_mark_uptodate(folio); } + if (shmem_uffd_swap_prepare(folio)) + goto redirty; if (!folio_alloc_swap(folio)) { bool first_swapped = shmem_recalc_inode(inode, 0, nr_pages); int error; @@ -1740,6 +2055,10 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug, swap_cache_del_folio(folio); goto redirty; } + /* Allocation failed: no swap entry owns the reserved inode metadata. */ + xa_lock_irq(&mapping->i_pages); + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); + xa_unlock_irq(&mapping->i_pages); if (nr_pages > 1) goto try_split; redirty: @@ -1923,6 +2242,24 @@ static struct folio *shmem_alloc_folio(gfp_t gfp, int order, return folio; } +/* Allocate unexposed promotion backing with the inode's zone/NUMA policy. */ +struct folio *shmem_alloc_collapse_folio(struct mm_struct *mm, struct inode *inode, + pgoff_t index, unsigned int order) +{ + gfp_t gfp = thp_shmem_limit_gfp_mask(GFP_TRANSHUGE, + mapping_gfp_mask(inode->i_mapping)); + struct folio *folio; + + folio = shmem_alloc_folio(gfp, order, SHMEM_I(inode), index); + if (!folio) + return ERR_PTR(-ENOMEM); + if (mem_cgroup_charge(folio, mm, gfp)) { + folio_put(folio); + return ERR_PTR(-ENOMEM); + } + return folio; +} + static struct folio *shmem_alloc_and_add_folio(struct vm_fault *vmf, gfp_t gfp, struct inode *inode, pgoff_t index, struct mm_struct *fault_mm, unsigned long orders) @@ -2437,13 +2774,36 @@ static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index, fault_mm = vma ? vma->vm_mm : NULL; folio = filemap_get_entry(inode->i_mapping, index); - if (folio && vma && userfaultfd_minor(vma)) { + if (folio && vma && userfaultfd_minor(vma) && + !(READ_ONCE(SHMEM_I(inode)->flags) & SHMEM_F_UFFD_SUBPAGE)) { if (!xa_is_value(folio)) folio_put(folio); *fault_type = handle_userfault(vmf, VM_UFFD_MINOR); return 0; } +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + if (xa_is_value(folio) && vma && userfaultfd_armed(vma) && + (READ_ONCE(SHMEM_I(inode)->flags) & SHMEM_F_UFFD_SUBPAGE)) { + unsigned long reason; + void *partial; + + /* Cache identity and its persisted mask must describe the same entry. */ + xa_lock_irq(&inode->i_mapping->i_pages); + if (xa_load(&inode->i_mapping->i_pages, index) != folio) { + xa_unlock_irq(&inode->i_mapping->i_pages); + goto repeat; + } + partial = xa_load(&SHMEM_I(inode)->uffd_missing, index); + reason = shmem_uffd_reason(vmf, partial ? xa_to_value(partial) : 0); + xa_unlock_irq(&inode->i_mapping->i_pages); + if (reason) { + *fault_type = handle_userfault(vmf, reason); + return 0; + } + } +#endif + if (xa_is_value(folio)) { error = shmem_swapin_folio(inode, index, &folio, sgp, gfp, vmf, fault_type); @@ -2463,6 +2823,20 @@ static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index, folio_put(folio); goto repeat; } +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + if (vma && userfaultfd_armed(vma) && + (READ_ONCE(SHMEM_I(inode)->flags) & SHMEM_F_UFFD_SUBPAGE)) { + unsigned long reason = shmem_uffd_reason(vmf, + shmem_uffd_missing_mask(folio, index)); + + if (reason) { + folio_unlock(folio); + folio_put(folio); + *fault_type = handle_userfault(vmf, reason); + return 0; + } + } +#endif if (sgp == SGP_WRITE) folio_mark_accessed(folio); if (folio_test_uptodate(folio)) @@ -2701,15 +3075,33 @@ static vm_fault_t shmem_fault(struct vm_fault *vmf) struct folio *folio = NULL; vm_fault_t ret = 0; int err; +#ifdef CONFIG_MM_SUBPAGE + bool promoted = false; + unsigned long user_size = mm_page_size(vmf->vma->vm_mm); + /* Missing coarse PTEs require a blocking caller to prepare backing. */ + if (user_size > PAGE_SIZE && (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) + return VM_FAULT_RETRY; +#endif + +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vmf->vma->vm_mm) < PAGE_SIZE && + !vma_file_offset_valid(vmf->vma, vmf->address, i_size_read(inode))) + return VM_FAULT_SIGBUS; +#endif + +#ifdef CONFIG_MM_SUBPAGE +retry: +#endif /* * Trinity finds that probing a hole which tmpfs is punching can * prevent the hole-punch from ever completing: noted in i_private. */ if (unlikely(inode->i_private)) { - ret = shmem_falloc_wait(vmf, inode); - if (ret) - return ret; + vm_fault_t wait = shmem_falloc_wait(vmf, inode); + + if (wait) + return ret | wait; } WARN_ON_ONCE(vmf->page != NULL); @@ -2718,6 +3110,45 @@ static vm_fault_t shmem_fault(struct vm_fault *vmf) if (err) return vmf_error(err); if (folio) { +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vmf->vma->vm_mm) < PAGE_SIZE && + !vma_file_offset_valid(vmf->vma, vmf->address, i_size_read(inode))) { + folio_unlock(folio); + folio_put(folio); + return VM_FAULT_SIGBUS; + } + if (user_size > PAGE_SIZE) { + struct vm_page_offset pos = vma_page_offset_at(vmf->vma, vmf->address); + unsigned long nr = user_size >> PAGE_SHIFT; + + if (pos.offset || !IS_ALIGNED(pos.index, nr)) { + folio_unlock(folio); + folio_put(folio); + return VM_FAULT_SIGBUS; + } + if (folio_size(folio) < user_size) { + folio_unlock(folio); + folio_put(folio); + folio = NULL; + /* Bound work if splitting races a successful promotion. */ + if (promoted) { + cond_resched(); + return ret | VM_FAULT_NOPAGE; + } + err = collapse_file_user_page(vmf->vma->vm_mm, + vmf->vma->vm_file, pos.index); + if (err == -EAGAIN) { + cond_resched(); + return ret | VM_FAULT_NOPAGE; + } + if (err) + return vmf_error(err); + promoted = true; + goto retry; + } + } +#endif + vmf->page = folio_file_page(folio, vmf->pgoff); ret |= VM_FAULT_LOCKED; } @@ -2852,7 +3283,7 @@ static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, * by page order, as in shmem_get_pgoff_policy() and get_vma_policy()). */ *ilx = inode->i_ino; - index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; + index = vma_page_offset_at(vma, addr).index; return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); } @@ -3039,6 +3470,9 @@ static struct inode *__shmem_get_inode(struct mnt_idmap *idmap, memset(info, 0, (char *)inode - (char *)info); INIT_LIST_HEAD_RCU(&info->xattrs); spin_lock_init(&info->lock); +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + xa_init_flags(&info->uffd_missing, XA_FLAGS_ACCOUNT); +#endif atomic_set(&info->stop_eviction, 0); info->seals = F_SEAL_SEAL; info->flags = vma_flags_test(&flags, VMA_NORESERVE_BIT) @@ -3195,6 +3629,57 @@ static void shmem_mfill_filemap_remove(struct folio *folio, folio_unlock(folio); } +#ifdef CONFIG_MM_SUBPAGE +struct folio *shmem_uffd_prepare_folio(struct vm_area_struct *vma, + unsigned long address, bool *new) +{ + struct inode *inode = file_inode(vma->vm_file); + pgoff_t index = vma_page_offset_at(vma, address).index; + struct folio *folio; + int err; + + *new = false; + /* Must precede cache insertion: minor faults must wait for initialization. */ + shmem_uffd_enable(inode); +retry: + err = shmem_get_folio(inode, index, 0, &folio, SGP_NOALLOC); + if (!err) + return folio; + if (err != -ENOENT) + return ERR_PTR(err); + folio = shmem_mfill_folio_alloc(vma, address); + if (!folio) + return ERR_PTR(-ENOMEM); + clear_highpage(&folio->page); + flush_dcache_folio(folio); + __folio_mark_uptodate(folio); + err = shmem_mfill_filemap_add(folio, vma, address); + if (err) { + folio_put(folio); + if (err == -EEXIST) + goto retry; + return ERR_PTR(err); + } + err = shmem_uffd_set_missing(folio, index, SHMEM_UFFD_ALL_UNITS); + if (err) { + shmem_mfill_filemap_remove(folio, vma); + folio_put(folio); + return ERR_PTR(err); + } + *new = true; + return folio; +} + +void shmem_uffd_abort_folio(struct folio *folio, struct vm_area_struct *vma, bool new) +{ + if (new) + shmem_mfill_filemap_remove(folio, vma); + else + folio_unlock(folio); + folio_put(folio); +} +#endif + static struct folio *shmem_get_folio_noalloc(struct inode *inode, pgoff_t pgoff) { struct folio *folio; @@ -3281,6 +3766,7 @@ shmem_write_end(const struct kiocb *iocb, struct address_space *mapping, } folio_mark_uptodate(folio); } + shmem_uffd_mark_mapped(folio, offset_in_folio(folio, pos), copied); folio_mark_dirty(folio); folio_unlock(folio); folio_put(folio); @@ -5093,6 +5579,10 @@ static struct inode *shmem_alloc_inode(struct super_block *sb) info = alloc_inode_sb(sb, shmem_inode_cachep, GFP_KERNEL); if (!info) return NULL; +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + /* Also cover inode-init failure before __shmem_get_inode() runs. */ + xa_init_flags(&info->uffd_missing, XA_FLAGS_ACCOUNT); +#endif return &info->vfs_inode; } @@ -5105,6 +5595,9 @@ static void shmem_free_in_core_inode(struct inode *inode) static void shmem_destroy_inode(struct inode *inode) { +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + xa_destroy(&SHMEM_I(inode)->uffd_missing); +#endif if (S_ISREG(inode->i_mode)) mpol_free_shared_policy(&SHMEM_I(inode)->policy); if (S_ISDIR(inode->i_mode)) @@ -5945,6 +6438,8 @@ struct folio *shmem_read_folio_gfp(struct address_space *mapping, if (error) return ERR_PTR(error); + /* A kernel caller receives and may modify the complete native folio. */ + shmem_uffd_mark_mapped(folio, 0, folio_size(folio)); folio_unlock(folio); return folio; #else diff --git a/mm/swap.h b/mm/swap.h index 77d2d14eda421..865c6ba8d3066 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -28,9 +28,12 @@ extern int page_cluster; #ifdef CONFIG_THP_SWAP #define SWAPFILE_CLUSTER HPAGE_PMD_NR -#define swap_entry_order(order) (order) #else #define SWAPFILE_CLUSTER 256 +#endif +#if defined(CONFIG_THP_SWAP) || defined(CONFIG_MM_SUBPAGE) +#define swap_entry_order(order) (order) +#else #define swap_entry_order(order) 0 #endif @@ -61,6 +64,9 @@ struct swap_cluster_info { #endif #if !SWAP_TABLE_HAS_ZEROFLAG unsigned long *zero_bitmap; +#endif +#ifdef CONFIG_MM_SUBPAGE + unsigned int *subpage_counts; /* Per-quarter PTE refs, protected by ci->lock */ #endif struct list_head list; }; @@ -231,6 +237,8 @@ extern int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp); */ int folio_alloc_swap(struct folio *folio); int folio_dup_swap(struct folio *folio, struct page *subpage); +int folio_dup_swap_range(struct folio *folio, struct page *page, unsigned int nr); +void folio_put_swap_range(struct folio *folio, struct page *page, unsigned int nr); void folio_put_swap(struct folio *folio, struct page *subpage); /* For internal use */ @@ -240,6 +248,7 @@ extern void __swap_cluster_free_entries(struct swap_info_struct *si, /* linux/mm/page_io.c */ int sio_pool_init(void); +int swap_extent_io_init(void); struct swap_iocb; void swap_read_folio(struct folio *folio, struct swap_iocb **plug); void __swap_read_unplug(struct swap_iocb *plug); @@ -381,6 +390,17 @@ static inline void folio_put_swap(struct folio *folio, struct page *page) { } +static inline int folio_dup_swap_range(struct folio *folio, struct page *page, + unsigned int nr) +{ + return -EINVAL; +} + +static inline void folio_put_swap_range(struct folio *folio, struct page *page, + unsigned int nr) +{ +} + static inline void swap_read_folio(struct folio *folio, struct swap_iocb **plug) { } diff --git a/mm/swap_state.c b/mm/swap_state.c index 9c3a5cf997786..160fbfca3d282 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -612,6 +612,10 @@ void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma, if (unlikely(folio_test_large(folio))) return; + /* Small-granule PTEs currently use native swap-cluster readahead. */ + if (vma && mm_page_size(vma->vm_mm) != PAGE_SIZE) + vma = NULL; + readahead = folio_test_clear_readahead(folio); if (vma && vma_ra) { unsigned long ra_val; @@ -979,7 +983,7 @@ struct folio *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask, struct folio *folio; mpol = get_vma_policy(vmf->vma, vmf->address, 0, &ilx); - folio = swap_use_vma_readahead() ? + folio = swap_use_vma_readahead() && mm_page_size(vmf->vma->vm_mm) == PAGE_SIZE ? swap_vma_readahead(entry, gfp_mask, mpol, ilx, vmf) : swap_cluster_readahead(entry, gfp_mask, mpol, ilx); mpol_cond_put(mpol); diff --git a/mm/swapfile.c b/mm/swapfile.c index 623b30e9c2dfb..5009971f3c90c 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -328,17 +329,22 @@ offset_to_swap_extent(struct swap_info_struct *sis, unsigned long offset) BUG(); } +/* The extent tree is immutable while the swap entry is owned. */ +sector_t swap_extent_sector(swp_entry_t entry, unsigned long *nr_pages) +{ + struct swap_info_struct *sis = __swap_entry_to_info(entry); + pgoff_t offset = swp_offset(entry); + struct swap_extent *se = offset_to_swap_extent(sis, offset); + + *nr_pages = min(*nr_pages, se->start_page + se->nr_pages - offset); + return (se->start_block + offset - se->start_page) << (PAGE_SHIFT - 9); +} + sector_t swap_folio_sector(struct folio *folio) { - struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); - struct swap_extent *se; - sector_t sector; - pgoff_t offset; + unsigned long nr = folio_nr_pages(folio); - offset = swp_offset(folio->swap); - se = offset_to_swap_extent(sis, offset); - sector = se->start_block + (offset - se->start_page); - return sector << (PAGE_SHIFT - 9); + return swap_extent_sector(folio->swap, &nr); } /* @@ -432,6 +438,17 @@ static void swap_cluster_free_table(struct swap_cluster_info *ci) ci->zero_bitmap = NULL; #endif +#ifdef CONFIG_MM_SUBPAGE + if (ci->subpage_counts) { + unsigned int i; + + for (i = 0; i < SWAPFILE_CLUSTER * (PAGE_SIZE / MM_SUBPAGE_SIZE); i++) + VM_WARN_ON_ONCE(ci->subpage_counts[i]); + kfree(ci->subpage_counts); + ci->subpage_counts = NULL; + } +#endif + table = (struct swap_table *)rcu_access_pointer(ci->table); if (!table) return; @@ -1085,6 +1102,19 @@ static void swap_reclaim_work(struct work_struct *work) swap_reclaim_full_clusters(si, true); } +static bool swap_folio_alloc_allowed(struct swap_info_struct *si, struct folio *folio) +{ + unsigned int flags; + + if (!folio || !folio_order(folio)) + return true; + flags = READ_ONCE(si->flags); + if (flags & SWP_BLKDEV) + return true; + /* Filesystem swap_rw implementations retain their existing contract. */ + return si->bdev && !(flags & SWP_FS_OPS) && folio_anon_min_order(folio); +} + /* * Try to allocate swap entries with specified order and try set a new * cluster for current CPU too. @@ -1096,11 +1126,7 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, unsigned int order = likely(folio) ? folio_order(folio) : 0; unsigned int offset = SWAP_ENTRY_INVALID, found = SWAP_ENTRY_INVALID; - /* - * Swapfile is not block device so unable - * to allocate large entries. - */ - if (order && !(si->flags & SWP_BLKDEV)) + if (!swap_folio_alloc_allowed(si, folio)) return 0; if (!(si->flags & SWP_SOLIDSTATE)) { @@ -1373,6 +1399,10 @@ static bool swap_alloc_fast(struct folio *folio) offset = this_cpu_read(percpu_swap_cluster.offset[order]); if (!si || !offset || !get_swap_device_info(si)) return false; + if (!swap_folio_alloc_allowed(si, folio)) { + put_swap_device(si); + return false; + } ci = swap_cluster_lock(si, offset); if (cluster_is_usable(ci, order)) { @@ -1581,7 +1611,7 @@ static void __swap_cluster_put_entry(struct swap_cluster_info *ci, */ static void swap_put_entries_cluster(struct swap_info_struct *si, pgoff_t offset, int nr, - bool reclaim_cache) + bool reclaim_cache, int subpage) { struct swap_cluster_info *ci; unsigned int ci_off, ci_end; @@ -1594,6 +1624,17 @@ static void swap_put_entries_cluster(struct swap_info_struct *si, ci = swap_cluster_lock(si, offset); ci_off = offset % SWAPFILE_CLUSTER; ci_end = ci_off + nr; +#ifdef CONFIG_MM_SUBPAGE + if (subpage >= 0) { + unsigned int index = ci_off * (PAGE_SIZE / MM_SUBPAGE_SIZE) + subpage; + + if (WARN_ON_ONCE(nr != 1 || !ci->subpage_counts || !ci->subpage_counts[index])) { + swap_cluster_unlock(ci); + return; + } + ci->subpage_counts[index]--; + } +#endif do { swp_tb = __swap_table_get(ci, ci_off); if (swp_tb_get_count(swp_tb) == 1) { @@ -1687,7 +1728,7 @@ static int __swap_cluster_dup_entry(struct swap_cluster_info *ci, * and failed to allocate an extended table, -EINVAL if any entry is bad entry. */ static int swap_dup_entries_cluster(struct swap_info_struct *si, - pgoff_t offset, int nr) + pgoff_t offset, int nr, int subpage) { int err; struct swap_cluster_info *ci; @@ -1697,6 +1738,25 @@ static int swap_dup_entries_cluster(struct swap_info_struct *si, ci_end = ci_start + nr; ci_off = ci_start; ci = swap_cluster_lock(si, offset); +#ifdef CONFIG_MM_SUBPAGE + if (subpage >= 0) { + unsigned int index = ci_start * (PAGE_SIZE / MM_SUBPAGE_SIZE) + subpage; + + VM_BUG_ON(nr != 1); + if (!ci->subpage_counts) + ci->subpage_counts = kcalloc(SWAPFILE_CLUSTER, + (PAGE_SIZE / MM_SUBPAGE_SIZE) * sizeof(*ci->subpage_counts), + GFP_ATOMIC | __GFP_NOWARN); + if (!ci->subpage_counts) { + err = -ENOMEM; + goto failed; + } + if (ci->subpage_counts[index] == UINT_MAX) { + err = -EOVERFLOW; + goto failed; + } + } +#endif restart: do { err = __swap_cluster_dup_entry(ci, ci_off); @@ -1711,6 +1771,10 @@ static int swap_dup_entries_cluster(struct swap_info_struct *si, goto failed; } } while (++ci_off < ci_end); +#ifdef CONFIG_MM_SUBPAGE + if (subpage >= 0) + ci->subpage_counts[ci_start * (PAGE_SIZE / MM_SUBPAGE_SIZE) + subpage]++; +#endif swap_cluster_unlock(ci); return 0; failed: @@ -1721,6 +1785,132 @@ static int swap_dup_entries_cluster(struct swap_info_struct *si, return err; } +#ifdef CONFIG_MM_SUBPAGE +static bool swap_subpage_offset_valid(unsigned int offset) +{ + return offset < PAGE_SIZE && IS_ALIGNED(offset, MM_SUBPAGE_SIZE); +} + +int swap_subpage_dup(swp_entry_t entry, unsigned int offset) +{ + struct swap_info_struct *si = swap_entry_to_info(entry); + + if (WARN_ON_ONCE(!si || !swap_subpage_offset_valid(offset))) + return -EINVAL; + /* A locked swap-cache folio also permits the initial zero-to-one ref. */ + return swap_dup_entries_cluster(si, swp_offset(entry), 1, offset >> MM_SUBPAGE_SHIFT); +} + +void swap_subpage_put(swp_entry_t entry, unsigned int offset, bool reclaim_cache) +{ + struct swap_info_struct *si; + + if (WARN_ON_ONCE(!swap_subpage_offset_valid(offset))) + return; + si = get_swap_device(entry); + if (WARN_ON_ONCE(!si)) + return; + swap_put_entries_cluster(si, swp_offset(entry), 1, reclaim_cache, + offset >> MM_SUBPAGE_SHIFT); + put_swap_device(si); +} + +unsigned int swap_subpage_count(swp_entry_t entry, unsigned int offset) +{ + struct swap_cluster_info *ci; + struct swap_info_struct *si; + unsigned int count = 0; + + if (!swap_subpage_offset_valid(offset)) + return 0; + si = get_swap_device(entry); + if (!si) + return 0; + ci = swap_cluster_lock(si, swp_offset(entry)); + if (ci->subpage_counts) + count = ci->subpage_counts[swp_cluster_offset(entry) * + (PAGE_SIZE / MM_SUBPAGE_SIZE) + (offset >> MM_SUBPAGE_SHIFT)]; + swap_cluster_unlock(ci); + put_swap_device(si); + return count; +} +#endif + +/* + * Optional THPs can split if their order-specific cluster lists have no room. + * A mandatory base user page cannot: find a free aligned group without keeping + * the per-CPU or device allocation lock held over a device-wide scan. + */ +static void swap_alloc_required_from_device(struct swap_info_struct *si, + struct folio *folio) +{ + unsigned int order = folio_order(folio), nr = folio_nr_pages(folio); + unsigned long start; + + if (!swap_folio_alloc_allowed(si, folio)) + return; + for (start = 0; start < si->max; start += SWAPFILE_CLUSTER) { + struct swap_cluster_info *ci = __swap_offset_to_cluster(si, start); + unsigned long offset, end = min(start + SWAPFILE_CLUSTER, si->max); + bool found = false; + + if (!(READ_ONCE(si->flags) & SWP_WRITEOK)) + break; + if (!spin_trylock(&ci->lock)) + goto next; + /* Order zero's usability test permits occupied clusters of any order. */ + if (!cluster_is_usable(ci, 0) || ci->count + nr > SWAPFILE_CLUSTER) + goto unlock; + for (offset = start; offset + nr <= end; offset += nr) { + bool need_reclaim = false; + + /* Only genuinely free slots qualify; retain every existing owner. */ + if (!cluster_scan_range(si, ci, offset, nr, &need_reclaim) || need_reclaim) + continue; + if (!__swap_cluster_alloc_entries(si, ci, folio, offset - start)) + break; + ci->order = order; + if (ci->count < SWAPFILE_CLUSTER && ci->flags == CLUSTER_FLAG_FRAG) { + /* Same flag, different order: relocate the list explicitly. */ + spin_lock(&si->lock); + list_move_tail(&ci->list, &si->frag_clusters[order]); + spin_unlock(&si->lock); + } else { + relocate_cluster(si, ci); + } + found = true; + break; + } +unlock: + swap_cluster_unlock(ci); + if (found) + return; +next: + cond_resched(); + } +} + +static void swap_alloc_required(struct folio *folio) +{ + struct swap_info_struct *devices[MAX_SWAPFILES], *si; + unsigned int count = 0, i; + + if (get_nr_swap_pages() < folio_nr_pages(folio)) + return; + /* Pin a bounded snapshot in the allocator's existing priority order. */ + spin_lock(&swap_avail_lock); + plist_for_each_entry(si, &swap_avail_head, avail_list) { + if (get_swap_device_info(si)) + devices[count++] = si; + } + spin_unlock(&swap_avail_lock); + for (i = 0; i < count; i++) { + if (!folio_test_swapcache(folio)) + swap_alloc_required_from_device(devices[i], folio); + put_swap_device(devices[i]); + } +} + /** * folio_alloc_swap - allocate swap space for a folio * @folio: folio we want to move to swap @@ -1735,16 +1925,17 @@ int folio_alloc_swap(struct folio *folio) { unsigned int order = folio_order(folio); unsigned int size = 1 << order; + bool mandatory = order && folio_anon_min_order(folio); VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); VM_BUG_ON_FOLIO(!folio_test_uptodate(folio), folio); if (order) { /* - * Reject large allocation when THP_SWAP is disabled, - * the caller should split the folio and try again. + * Optional THP backing may split when THP_SWAP is disabled. + * A larger base user page must retain contiguous swap ownership. */ - if (!IS_ENABLED(CONFIG_THP_SWAP)) + if (!IS_ENABLED(CONFIG_THP_SWAP) && !mandatory) return -EAGAIN; /* @@ -1763,6 +1954,9 @@ int folio_alloc_swap(struct folio *folio) swap_alloc_slow(folio); local_unlock(&percpu_swap_cluster.lock); + if (mandatory && unlikely(!folio_test_swapcache(folio))) + swap_alloc_required(folio); + if (!order && unlikely(!folio_test_swapcache(folio))) { if (swap_sync_discard()) goto again; @@ -1795,6 +1989,24 @@ int folio_alloc_swap(struct folio *folio) * swap_put_entries_direct on its swap entry before this helper returns, or * the swap count may underflow. */ +int folio_dup_swap_range(struct folio *folio, struct page *page, unsigned int nr) +{ + swp_entry_t entry = page_swap_entry(page); + + VM_BUG_ON_FOLIO(!folio_test_locked(folio) || !folio_test_swapcache(folio), folio); + VM_BUG_ON_FOLIO(!nr || folio_page_idx(folio, page) + nr > folio_nr_pages(folio), folio); + return swap_dup_entries_cluster(swap_entry_to_info(entry), swp_offset(entry), nr, -1); +} + +void folio_put_swap_range(struct folio *folio, struct page *page, unsigned int nr) +{ + swp_entry_t entry = page_swap_entry(page); + + VM_BUG_ON_FOLIO(!folio_test_locked(folio) || !folio_test_swapcache(folio), folio); + VM_BUG_ON_FOLIO(!nr || folio_page_idx(folio, page) + nr > folio_nr_pages(folio), folio); + swap_put_entries_cluster(__swap_entry_to_info(entry), swp_offset(entry), nr, false, -1); +} + int folio_dup_swap(struct folio *folio, struct page *subpage) { swp_entry_t entry = folio->swap; @@ -1809,7 +2021,7 @@ int folio_dup_swap(struct folio *folio, struct page *subpage) } return swap_dup_entries_cluster(swap_entry_to_info(entry), - swp_offset(entry), nr_pages); + swp_offset(entry), nr_pages, -1); } /** @@ -1835,7 +2047,7 @@ void folio_put_swap(struct folio *folio, struct page *subpage) nr_pages = 1; } - swap_put_entries_cluster(si, swp_offset(entry), nr_pages, false); + swap_put_entries_cluster(si, swp_offset(entry), nr_pages, false, -1); } /* @@ -2130,7 +2342,7 @@ void swap_put_entries_direct(swp_entry_t entry, int nr) offset = start_offset; do { cluster_end = min(round_up(offset + 1, SWAPFILE_CLUSTER), end_offset); - swap_put_entries_cluster(si, offset, cluster_end - offset, true); + swap_put_entries_cluster(si, offset, cluster_end - offset, true, -1); offset = cluster_end; } while (offset < end_offset); out: @@ -2405,14 +2617,80 @@ static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte) return pte_same(pte_swp_clear_flags(pte), swp_pte); } +#ifdef CONFIG_MM_SUBPAGE +/* Caller holds the swap-cache folio lock and the VMA's mmap read lock. */ +static int unuse_subpage(struct vm_area_struct *vma, pmd_t *pmd, + unsigned long addr, swp_entry_t entry, struct folio *folio, + unsigned int offset) +{ + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *slot = NULL; + spinlock_t *ptl; + pte_t *ptep, old, pte; + bool poisoned = PageHWPoison(&folio->page) || !folio_test_uptodate(folio); + int ret = 0; + + if (folio_order(folio) || folio_test_ksm(folio)) + return -EOPNOTSUPP; + if (!poisoned) { + arch_swap_restore(folio_swap(entry, folio), folio); + /* Allocate metadata before taking the PTE spinlock. */ + slot = mm_subpage_restore_granule(folio, offset, mm_page_shift(mm), GFP_KERNEL); + if (IS_ERR(slot)) + return PTR_ERR(slot) == -EAGAIN ? 0 : PTR_ERR(slot); + } + ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!ptep) + goto out; + old = ptep_get(ptep); + if (softleaf_from_pte(old).val != entry.val) + goto unlock; + /* A moved swap PTE retains its backing offset, not its VMA offset. */ + if (pte_swp_subpage_offset(old) != offset) + goto unlock; + if (poisoned) { + pte = swp_entry_to_pte(PageHWPoison(&folio->page) ? + make_hwpoison_entry(&folio->page) : make_poisoned_swp_entry()); + } else { + ret = mm_subpage_add_anon_rmap(slot, vma, addr); + if (ret) + goto unlock; + /* unuse_pte_range() waited for writeback and holds the folio lock. */ + VM_BUG_ON_FOLIO(folio_test_writeback(folio), folio); + if (pte_swp_exclusive(old)) + mm_subpage_set_exclusive(slot); + else if (swap_subpage_count(entry, offset) == 1) + mm_subpage_try_reuse_swap(slot); + pte = pte_mkold(phys_pte_mm(mm, mm_subpage_phys(slot), vma->vm_page_prot)); + if (pte_swp_soft_dirty(old)) + pte = pte_mksoft_dirty(pte); + if (pte_swp_uffd_wp(old)) + pte = pte_mkuffd_wp(pte); + inc_mm_counter(mm, MM_ANONPAGES); + slot = NULL; + ret = 1; + } + dec_mm_counter(mm, MM_SWAPENTS); + set_pte_at(mm, addr, ptep, pte); + swap_subpage_put(entry, offset, false); +unlock: + pte_unmap_unlock(ptep, ptl); +out: + if (slot) + mm_subpage_put(slot); + return ret; +} +#endif + /* * No need to decide whether this PTE shares the swap entry with others, * just let do_wp_page work it out if a write is requested later - to * force COW, vm_page_prot omits write permission from any private vma. */ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, - unsigned long addr, swp_entry_t entry, struct folio *folio) + unsigned long addr, pte_t orig_pte, struct folio *folio) { + swp_entry_t entry = softleaf_from_pte(orig_pte); struct page *page; struct folio *swapcache; spinlock_t *ptl; @@ -2427,6 +2705,12 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, if (!folio_matches_swap_entry(folio, entry)) return 0; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return unuse_subpage(vma, pmd, addr, entry, folio, + pte_swp_subpage_offset(orig_pte)); +#endif + swapcache = folio; folio = ksm_might_need_to_copy(folio, vma, addr); if (unlikely(!folio)) @@ -2532,7 +2816,7 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, pte_t ptent; if (!pte++) { - pte = pte_offset_map(pmd, addr); + pte = pte_offset_map_mm(vma->vm_mm, pmd, addr); if (!pte) break; } @@ -2548,6 +2832,22 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, pte_unmap(pte); pte = NULL; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) > PAGE_SIZE) { + struct vm_fault vmf = { + .vma = vma, .address = addr, .real_address = addr, + .pmd = pmd, .orig_pte = ptent, + }; + vm_fault_t fault = do_swap_coarse_page(&vmf); + + if (fault & VM_FAULT_OOM) + return -ENOMEM; + if (fault & VM_FAULT_ERROR) + return -EIO; + continue; + } +#endif + folio = swap_cache_get_folio(entry); if (!folio) { struct vm_fault vmf = { @@ -2570,7 +2870,7 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, folio_lock(folio); folio_wait_writeback(folio); - ret = unuse_pte(vma, pmd, addr, entry, folio); + ret = unuse_pte(vma, pmd, addr, ptent, folio); if (ret < 0) { folio_unlock(folio); folio_put(folio); @@ -2580,7 +2880,7 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, folio_free_swap(folio); folio_unlock(folio); folio_put(folio); - } while (addr += PAGE_SIZE, addr != end); + } while (addr += mm_page_size(vma->vm_mm), addr != end); if (pte) pte_unmap(pte); @@ -2595,10 +2895,10 @@ static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud, unsigned long next; int ret; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(vma->vm_mm, pud, addr); do { cond_resched(); - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(vma->vm_mm, addr, end); ret = unuse_pte_range(vma, pmd, addr, next, type); if (ret) return ret; @@ -2614,9 +2914,9 @@ static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d, unsigned long next; int ret; - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(vma->vm_mm, p4d, addr); do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(vma->vm_mm, addr, end); if (pud_none_or_clear_bad(pud)) continue; ret = unuse_pmd_range(vma, pud, addr, next, type); @@ -2634,10 +2934,10 @@ static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd, unsigned long next; int ret; - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(vma->vm_mm, pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(p4d)) + next = p4d_addr_end_mm(vma->vm_mm, addr, end); + if (p4d_none_or_clear_bad_mm(vma->vm_mm, p4d)) continue; ret = unuse_pud_range(vma, p4d, addr, next, type); if (ret) @@ -2657,7 +2957,7 @@ static int unuse_vma(struct vm_area_struct *vma, unsigned int type) pgd = pgd_offset(vma->vm_mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(vma->vm_mm, addr, end); if (pgd_none_or_clear_bad(pgd)) continue; ret = unuse_p4d_range(vma, pgd, addr, next, type); @@ -2964,10 +3264,15 @@ static int setup_swap_extents(struct swap_info_struct *sis, destroy_swap_extents(sis, swap_file); return -ENOMEM; } - return ret; + } else { + ret = generic_swapfile_activate(sis, swap_file, span); } - - return generic_swapfile_activate(sis, swap_file, span); + if (ret >= 0 && sis->bdev && + !(sis->flags & SWP_FS_OPS) && swap_extent_io_init()) { + destroy_swap_extents(sis, swap_file); + return -ENOMEM; + } + return ret; } static void _enable_swap_info(struct swap_info_struct *si) @@ -3865,7 +4170,7 @@ void si_swapinfo(struct sysinfo *val) * Also the swap entry must have a count >= 1. Otherwise folio_dup_swap should * be used. */ -int swap_dup_entry_direct(swp_entry_t entry) +int swap_dup_entries_direct(swp_entry_t entry, unsigned int nr) { struct swap_info_struct *si; @@ -3882,7 +4187,14 @@ int swap_dup_entry_direct(swp_entry_t entry) */ VM_WARN_ON_ONCE(!swap_entry_swapped(si, entry)); - return swap_dup_entries_cluster(si, swp_offset(entry), 1); + if (WARN_ON_ONCE(!nr || swp_cluster_offset(entry) + nr > SWAPFILE_CLUSTER)) + return -EINVAL; + return swap_dup_entries_cluster(si, swp_offset(entry), nr, -1); +} + +int swap_dup_entry_direct(swp_entry_t entry) +{ + return swap_dup_entries_direct(entry, 1); } #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) diff --git a/mm/truncate.c b/mm/truncate.c index b58ba940be474..00ba4766e02b8 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -794,8 +794,10 @@ void truncate_pagecache(struct inode *inode, loff_t newsize) * truncate_inode_pages finishes, hence the second * unmap_mapping_range call must be made for correctness. */ + unmap_mapping_subpage_tail(mapping, newsize); unmap_mapping_range(mapping, holebegin, 0, 1); truncate_inode_pages(mapping, newsize); + unmap_mapping_subpage_tail(mapping, newsize); unmap_mapping_range(mapping, holebegin, 0, 1); } EXPORT_SYMBOL(truncate_pagecache); diff --git a/mm/user-subpage-internal.h b/mm/user-subpage-internal.h new file mode 100644 index 0000000000000..bddb5eeb2ca7b --- /dev/null +++ b/mm/user-subpage-internal.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _MM_USER_SUBPAGE_INTERNAL_H +#define _MM_USER_SUBPAGE_INTERNAL_H + +#include +#include +#include +#include + +struct mm_subpage_owner; +struct rmap_walk_control; +struct anon_vma; + +struct mm_subpage { + struct mm_subpage_owner *owner; + refcount_t refs; + atomic_t mapcount; + unsigned int index; + u8 shift; + unsigned int pins; + bool exclusive; + bool lazyfree; + bool rmap_bound; +}; + +/* Logical identity is independent of the physical slot after relocation. */ +struct mm_subpage_rmap { + struct anon_vma *root; + struct vm_page_offset pos; + unsigned int offset; +}; + +struct subpage_retired_root; +struct mm_subpage_move { + struct mm_subpage *slot; + struct mm_subpage_rmap map; + struct subpage_retired_root *retired; + bool swap; + bool root_changed; + bool committed; +}; + +/* Folio and both PTE locks stay held across prepare, commit and finish. */ +int mm_subpage_move_prepare(struct mm_subpage_move *move, struct mm_subpage *slot, + struct vm_area_struct *src, unsigned long src_address, + struct vm_area_struct *dst, unsigned long dst_address, bool swap); +/* Present source PTE must be cleared and its TLB invalidated before commit. */ +int mm_subpage_move_commit(struct mm_subpage_move *move); +void mm_subpage_move_finish(struct mm_subpage_move *move); + +int mm_subpage_bind_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, + unsigned long address); +bool mm_subpage_match_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, + unsigned long address); +/* Return -1 for ordinary linear rmap; otherwise return referenced identities. */ +int mm_subpage_snapshot_rmap(struct folio *folio, struct mm_subpage_rmap *maps); +bool mm_subpage_address_in_vma(const struct folio *folio, + const struct vm_area_struct *vma, unsigned long *address); +bool rmap_walk_subpages(struct folio *folio, struct rmap_walk_control *rwc, + bool locked); + +struct anon_vma *mm_subpage_anon_root(const struct mm_subpage *subpage); + +/* Folio lock held; called once before publishing anonymous folio mapping. */ +void mm_subpage_hold_anon_root(struct mm_subpage *subpage, struct anon_vma *root, + bool droppable); +bool mm_subpage_is_droppable(const struct mm_subpage *subpage); + +#endif diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index ec5d511bd1791..4edaf4dacf345 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -32,10 +32,19 @@ #include #include #include +#include +#include #include #include #include "internal.h" #include "swap.h" +#ifdef CONFIG_MM_SUBPAGE +#include "user-subpage-internal.h" +#endif + +struct mfill_shared_data { + struct folio *source; /* NULL requests a zero-fill. */ +}; struct mfill_state { struct userfaultfd_ctx *ctx; @@ -48,6 +57,10 @@ struct mfill_state { unsigned long src_addr; unsigned long dst_addr; pmd_t *pmd; + struct folio *copy_buffer; +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage *slots[MM_SUBPAGES_PER_PAGE]; +#endif }; static bool anon_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags) @@ -210,8 +223,24 @@ static void uffd_mfill_unlock(struct vm_area_struct *vma) } #endif +/* Reservations keep the uninstalled leaves private to this operation. */ +static void mfill_free_subpages(struct mfill_state *state) +{ +#ifdef CONFIG_MM_SUBPAGE + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(state->slots); i++) { + if (state->slots[i]) { + mm_subpage_put(state->slots[i]); + state->slots[i] = NULL; + } + } +#endif +} + static void mfill_put_vma(struct mfill_state *state) { + mfill_free_subpages(state); if (!state->vma) return; @@ -264,6 +293,7 @@ static int mfill_get_vma(struct mfill_state *state) if ((flags & MFILL_ATOMIC_WP) && !(dst_vma->vm_flags & VM_UFFD_WP)) goto out_unlock; + if (is_vm_hugetlb_page(dst_vma)) return 0; @@ -336,16 +366,11 @@ static int mfill_establish_pmd(struct mfill_state *state) static bool mfill_file_over_size(struct vm_area_struct *dst_vma, unsigned long dst_addr) { - struct inode *inode; - pgoff_t offset, max_off; - if (!dst_vma->vm_file) return false; - inode = dst_vma->vm_file->f_inode; - offset = linear_page_index(dst_vma, dst_addr); - max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); - return offset >= max_off; + return !vma_file_offset_valid(dst_vma, dst_addr, + i_size_read(file_inode(dst_vma->vm_file))); } /* @@ -357,7 +382,8 @@ static bool mfill_file_over_size(struct vm_area_struct *dst_vma, static int mfill_atomic_install_pte(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, struct page *page, - uffd_flags_t flags) + uffd_flags_t flags, + const struct mfill_shared_data *shared_data) { int ret; struct mm_struct *dst_mm = dst_vma->vm_mm; @@ -370,6 +396,11 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, pte_t dst_ptep; _dst_pte = mk_pte(page, dst_vma->vm_page_prot); +#ifdef CONFIG_MM_SUBPAGE + if (page_in_cache && mm_page_size(dst_mm) < PAGE_SIZE) + _dst_pte = phys_pte_mm(dst_mm, page_to_phys(page) + + vma_page_offset_at(dst_vma, dst_addr).offset, dst_vma->vm_page_prot); +#endif _dst_pte = pte_mkdirty(_dst_pte); if (page_in_cache && !vm_shared) writable = false; @@ -400,7 +431,39 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, if (!pte_none(dst_ptep) && !pte_is_uffd_marker(dst_ptep)) goto out_unlock; +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) + if (shared_data) { + struct vm_page_offset pos = vma_page_offset_at(dst_vma, dst_addr); + unsigned long offset = ((pos.index - folio->index) << PAGE_SHIFT) + pos.offset; + unsigned long size = mm_page_size(dst_mm); + void *dst, *src; + + if (PageHWPoison(page)) { + ret = -EIO; + goto out_unlock; + } + if (!shmem_uffd_range_missing(folio, pos.index, pos.offset, size)) + goto out_unlock; + dst = kmap_local_folio(folio, offset); + if (shared_data->source) { + src = kmap_local_folio(shared_data->source, 0); + memcpy(dst, src, size); + kunmap_local(src); + } else { + memset(dst, 0, size); + } + kunmap_local(dst); + flush_dcache_folio(folio); + __folio_mark_uptodate(folio); + folio_mark_dirty(folio); + } +#endif + if (page_in_cache) { + mm_subpage_file_map_add(dst_vma, pte_phys_mm(dst_mm, _dst_pte)); + shmem_uffd_mark_mapped(folio, + (folio_page_idx(folio, page) << PAGE_SHIFT) + + vma_page_offset_at(dst_vma, dst_addr).offset, mm_page_size(dst_mm)); folio_add_file_rmap_pte(folio, page, dst_vma); } else { folio_add_new_anon_rmap(folio, dst_vma, dst_addr, RMAP_EXCLUSIVE); @@ -427,12 +490,13 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, return ret; } -static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) +static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr, + unsigned int offset, unsigned long len) { void *kaddr; int ret; - kaddr = kmap_local_folio(folio, 0); + kaddr = kmap_local_folio(folio, offset); /* * The read mmap_lock is held here. Despite the * mmap_lock being read recursive a deadlock is still @@ -449,8 +513,7 @@ static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) * and retry the copy outside the mmap_lock. */ pagefault_disable(); - ret = copy_from_user(kaddr, (const void __user *) src_addr, - PAGE_SIZE); + ret = copy_from_user(kaddr, (const void __user *)src_addr, len); pagefault_enable(); kunmap_local(kaddr); @@ -523,8 +586,11 @@ DEFINE_FREE(retry_put, struct mfill_retry_state *, if (_T) mfill_retry_state_put(_T)); static int mfill_copy_folio_retry(struct mfill_state *mfill_state, - struct folio *folio) + struct folio *folio, unsigned int offset, + unsigned long len) { + struct vm_page_offset pos = vma_page_offset_at(mfill_state->vma, + mfill_state->dst_addr); struct mfill_retry_state retry_state = { 0 }; struct mfill_retry_state *for_free __free(retry_put) = &retry_state; unsigned long src_addr = mfill_state->src_addr; @@ -536,8 +602,8 @@ static int mfill_copy_folio_retry(struct mfill_state *mfill_state, /* retry copying with mm_lock dropped */ mfill_put_vma(mfill_state); - kaddr = kmap_local_folio(folio, 0); - err = copy_from_user(kaddr, (const void __user *) src_addr, PAGE_SIZE); + kaddr = kmap_local_folio(folio, offset); + err = copy_from_user(kaddr, (const void __user *) src_addr, len); kunmap_local(kaddr); if (unlikely(err)) return -EFAULT; @@ -549,7 +615,10 @@ static int mfill_copy_folio_retry(struct mfill_state *mfill_state, if (err) return err; - if (mfill_retry_state_changed(&retry_state, mfill_state->vma)) + if (mfill_retry_state_changed(&retry_state, mfill_state->vma) || + (mm_page_size(mfill_state->ctx->mm) < PAGE_SIZE && + !vm_page_offset_equal(pos, vma_page_offset_at(mfill_state->vma, + mfill_state->dst_addr)))) return -EAGAIN; err = mfill_establish_pmd(mfill_state); @@ -559,6 +628,28 @@ static int mfill_copy_folio_retry(struct mfill_state *mfill_state, return 0; } +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) +/* Fill one missing leaf in new or retained shmem backing using staged data. */ +static int mfill_atomic_shmem_hole(struct mfill_state *state, + const struct mfill_shared_data *data) +{ + struct vm_page_offset pos; + struct folio *folio; + bool new; + int err; + + folio = shmem_uffd_prepare_folio(state->vma, state->dst_addr, &new); + if (IS_ERR(folio)) + return PTR_ERR(folio); + pos = vma_page_offset_at(state->vma, state->dst_addr); + err = mfill_atomic_install_pte(state->pmd, state->vma, state->dst_addr, + folio_file_page(folio, pos.index), state->flags, data); + if (err) + shmem_uffd_abort_folio(folio, state->vma, new); + return err; +} +#endif + static int __mfill_atomic_pte(struct mfill_state *state, const struct vm_uffd_ops *ops) { @@ -578,7 +669,7 @@ static int __mfill_atomic_pte(struct mfill_state *state, return -ENOMEM; if (uffd_flags_mode_is(flags, MFILL_ATOMIC_COPY)) { - ret = mfill_copy_folio_locked(folio, src_addr); + ret = mfill_copy_folio_locked(folio, src_addr, 0, PAGE_SIZE); /* * Fallback to copy_from_user outside mmap_lock. * If retry is successful, mfill_copy_folio_locked() returns @@ -587,7 +678,7 @@ static int __mfill_atomic_pte(struct mfill_state *state, * will take care of unlocking if needed. */ if (unlikely(ret)) { - ret = mfill_copy_folio_retry(state, folio); + ret = mfill_copy_folio_retry(state, folio, 0, PAGE_SIZE); if (ret) goto err_folio_put; } @@ -606,12 +697,29 @@ static int __mfill_atomic_pte(struct mfill_state *state, if (ops->filemap_add) { ret = ops->filemap_add(folio, state->vma, state->dst_addr); +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) + /* + * A pinned large shmem folio can survive a partial hole punch. + * Its cache entry exists while this native-sized range is missing. + * Reuse the prepared source and preserve all neighbouring bytes. + * Ordinary native insertion and non-shmem paths stay unchanged. + */ + if (ret == -EEXIST && (state->vma->vm_flags & VM_SHARED) && + shmem_file(state->vma->vm_file) && + (READ_ONCE(SHMEM_I(file_inode(state->vma->vm_file))->flags) & + SHMEM_F_UFFD_SUBPAGE)) { + struct mfill_shared_data data = { .source = folio }; + + ret = mfill_atomic_shmem_hole(state, &data); + goto err_folio_put; + } +#endif if (ret) goto err_folio_put; } ret = mfill_atomic_install_pte(state->pmd, state->vma, dst_addr, - &folio->page, flags); + &folio->page, flags, NULL); if (ret) goto err_filemap_remove; @@ -625,10 +733,184 @@ static int __mfill_atomic_pte(struct mfill_state *state, return ret; } +#ifdef CONFIG_MM_SUBPAGE +/* Pack only this request's leaves in one native linear index. */ +static struct mm_subpage *mfill_alloc_subpage(struct mfill_state *state) +{ + struct vm_area_struct *vma = state->vma; + struct mm_struct *mm = vma->vm_mm; + struct vm_page_offset pos = vma_page_offset_at(vma, state->dst_addr); + unsigned long size = mm_page_size(mm); + unsigned int first = pos.offset >> mm_page_shift(mm), i, nr; + struct mm_subpage_pool *pool; + struct mm_subpage *slot; + struct folio *folio; + int err = -ENOMEM; + + if (state->slots[first]) + goto take; + + mfill_free_subpages(state); + pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!pool) + return ERR_PTR(-ENOMEM); + folio = anon_alloc_folio(vma, state->dst_addr); + if (!folio) + goto out; + folio_throttle_swaprate(folio, GFP_KERNEL); + err = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (err) { + folio_put(folio); + goto out; + } + nr = min_t(unsigned long, PAGE_SIZE - pos.offset, + state->dst_start + state->len - state->dst_addr) / size; + for (i = first; i < first + nr; i++) { + slot = mm_subpage_alloc_at(pool, i * size); + if (IS_ERR(slot)) { + err = PTR_ERR(slot); + mfill_free_subpages(state); + goto out; + } + state->slots[i] = slot; + } +out: + mm_subpage_pool_close(pool); + mm_subpage_pool_put(pool); + if (err) + return ERR_PTR(err); +take: + slot = state->slots[first]; + state->slots[first] = NULL; + return slot; +} + +static int mfill_atomic_subpage(struct mfill_state *state) +{ + struct mm_struct *mm = state->ctx->mm; + struct mm_subpage *slot = mfill_alloc_subpage(state); + struct vm_area_struct *vma; + struct folio *folio; + pte_t entry, *pte; + spinlock_t *ptl; + bool fresh; + int err; + + if (IS_ERR(slot)) + return PTR_ERR(slot); + folio = mm_subpage_folio(slot); + folio_lock(folio); + /* A previous leaf may have entered reclaim between iterations. */ + err = -EAGAIN; + if (folio_test_swapcache(folio)) + goto unlock_folio; + if (uffd_flags_mode_is(state->flags, MFILL_ATOMIC_COPY)) { + err = mfill_copy_folio_locked(folio, state->src_addr, + mm_subpage_offset(slot), mm_page_size(mm)); + if (err) { + /* + * The faulting retry must not hold the folio lock, or write + * a folio whose already-visible neighbours can be reclaimed. + * Use fresh backing and discard the remaining reservations + * before dropping the VMA lock. + */ + folio_unlock(folio); + mm_subpage_put(slot); + mfill_free_subpages(state); + slot = mfill_alloc_subpage(state); + if (IS_ERR(slot)) + return PTR_ERR(slot); + folio = mm_subpage_folio(slot); + err = mfill_copy_folio_retry(state, folio, + mm_subpage_offset(slot), mm_page_size(mm)); + if (err) + goto put_slot; + folio_lock(folio); + } + } + /* The allocator zeroed every byte, including unissued neighbours. */ + __folio_mark_uptodate(folio); + vma = state->vma; + pte = pte_offset_map_lock(mm, state->pmd, state->dst_addr, &ptl); + err = -EAGAIN; + if (!pte) + goto unlock_folio; + err = -EFAULT; + if (mfill_file_over_size(vma, state->dst_addr)) + goto unlock_pte; + err = -EEXIST; + if (!pte_none(ptep_get(pte)) && !pte_is_uffd_marker(ptep_get(pte))) + goto unlock_pte; + fresh = !folio_test_anon(folio); + mm_subpage_set_exclusive(slot); + err = mm_subpage_add_new_anon_rmap(slot, vma, state->dst_addr); + if (err) + goto unlock_pte; + if (fresh) + folio_add_lru_vma(folio, vma); + else if ((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED) + mlock_folio(folio); + inc_mm_counter(mm, MM_ANONPAGES); + entry = phys_pte_mm(mm, mm_subpage_phys(slot), vma->vm_page_prot); + entry = pte_mkdirty(entry); + if (vma->vm_flags & VM_WRITE) + entry = pte_mkwrite(entry, vma); + if (state->flags & MFILL_ATOMIC_WP) + entry = pte_mkuffd_wp(entry); + set_pte_at(mm, state->dst_addr, pte, entry); + update_mmu_cache(vma, state->dst_addr, pte); + /* The PTE owns slot now; release the folio lock before dropping PTL. */ + folio_unlock(folio); + pte_unmap_unlock(pte, ptl); + return 0; +unlock_pte: + pte_unmap_unlock(pte, ptl); +unlock_folio: + folio_unlock(folio); +put_slot: + mm_subpage_put(slot); + return err; +} +#endif + +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) +static int mfill_atomic_shared_subpage(struct mfill_state *state) +{ + struct mfill_shared_data data = {}; + int err; + + if (uffd_flags_mode_is(state->flags, MFILL_ATOMIC_COPY)) { + if (!state->copy_buffer) + state->copy_buffer = folio_alloc(GFP_KERNEL_ACCOUNT, 0); + data.source = state->copy_buffer; + if (!data.source) + return -ENOMEM; + err = mfill_copy_folio_locked(data.source, state->src_addr, 0, + mm_page_size(state->ctx->mm)); + if (err) + err = mfill_copy_folio_retry(state, data.source, 0, + mm_page_size(state->ctx->mm)); + if (err) + return err; + } + return mfill_atomic_shmem_hole(state, &data); +} +#endif + static int mfill_atomic_pte_copy(struct mfill_state *state) { const struct vm_uffd_ops *ops = vma_uffd_ops(state->vma); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(state->ctx->mm) < PAGE_SIZE) { +#ifdef CONFIG_SHMEM + if (state->vma->vm_flags & VM_SHARED) + return mfill_atomic_shared_subpage(state); +#endif + return mfill_atomic_subpage(state); + } +#endif + /* * The normal page fault path for a MAP_PRIVATE mapping in a * file-backed VMA will invoke the fault, fill the hole in the file and @@ -648,6 +930,16 @@ static int mfill_atomic_pte_zeroed_folio(struct mfill_state *state) { const struct vm_uffd_ops *ops = vma_uffd_ops(state->vma); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(state->ctx->mm) < PAGE_SIZE) { +#ifdef CONFIG_SHMEM + if (state->vma->vm_flags & VM_SHARED) + return mfill_atomic_shared_subpage(state); +#endif + return mfill_atomic_subpage(state); + } +#endif + return __mfill_atomic_pte(state, ops); } @@ -711,6 +1003,13 @@ static int mfill_atomic_pte_continue(struct mfill_state *state) if (IS_ERR_OR_NULL(folio)) return -EFAULT; +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) + if (vma_is_shmem(dst_vma) && shmem_uffd_range_missing(folio, pgoff, + vma_page_offset_at(dst_vma, dst_addr).offset, mm_page_size(dst_vma->vm_mm))) { + ret = -EFAULT; + goto out_release; + } +#endif page = folio_file_page(folio, pgoff); if (PageHWPoison(page)) { ret = -EIO; @@ -718,7 +1017,7 @@ static int mfill_atomic_pte_continue(struct mfill_state *state) } ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr, - page, flags); + page, flags, NULL); if (ret) goto out_release; @@ -980,8 +1279,8 @@ static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx, /* * Sanitize the command parameters: */ - VM_WARN_ON_ONCE(dst_start & ~PAGE_MASK); - VM_WARN_ON_ONCE(len & ~PAGE_MASK); + VM_WARN_ON_ONCE(dst_start & ~mm_page_mask(ctx->mm)); + VM_WARN_ON_ONCE(len & ~mm_page_mask(ctx->mm)); /* Does the address range wrap, or is the span zero-sized? */ VM_WARN_ON_ONCE(src_start + len <= src_start); @@ -1014,9 +1313,9 @@ static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx, cond_resched(); if (!err) { - state.dst_addr += PAGE_SIZE; - state.src_addr += PAGE_SIZE; - copied += PAGE_SIZE; + state.dst_addr += mm_page_size(ctx->mm); + state.src_addr += mm_page_size(ctx->mm); + copied += mm_page_size(ctx->mm); if (fatal_signal_pending(current)) err = -EINTR; @@ -1027,6 +1326,8 @@ static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx, mfill_put_vma(&state); out: + if (state.copy_buffer) + folio_put(state.copy_buffer); VM_WARN_ON_ONCE(copied < 0); VM_WARN_ON_ONCE(err > 0); VM_WARN_ON_ONCE(!copied && !err); @@ -1115,8 +1416,8 @@ static int mwriteprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, /* * Sanitize the command parameters: */ - VM_WARN_ON_ONCE(start & ~PAGE_MASK); - VM_WARN_ON_ONCE(len & ~PAGE_MASK); + VM_WARN_ON_ONCE(start & ~mm_page_mask(dst_mm)); + VM_WARN_ON_ONCE(len & ~mm_page_mask(dst_mm)); /* Does the address range wrap, or is the span zero-sized? */ VM_WARN_ON_ONCE(start + len <= start); @@ -1319,7 +1620,102 @@ static long move_present_ptes(struct mm_struct *mm, return src_addr > src_start ? src_addr - src_start : err; } +#ifdef CONFIG_MM_SUBPAGE +static long move_present_subpage(struct mm_struct *mm, + struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, + unsigned long dst_addr, unsigned long src_addr, + pte_t *dst_pte, pte_t *src_pte, pte_t orig_dst, pte_t orig_src, + pmd_t *dst_pmd, pmd_t dst_pmdval, + spinlock_t *dst_ptl, spinlock_t *src_ptl, struct folio *folio) +{ + struct mm_subpage_move move; + struct mm_subpage *slot = NULL; + pte_t pte; + long ret = -EAGAIN; + + flush_cache_range(src_vma, src_addr, src_addr + mm_page_size(mm)); + double_pt_lock(dst_ptl, src_ptl); + if (!is_pte_pages_stable(dst_pte, src_pte, orig_dst, orig_src, + dst_pmd, dst_pmdval)) + goto out; + ret = -EBUSY; + if (!folio_test_anon(folio) || folio_test_large(folio)) + goto out; + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, orig_src)); + if (!slot || mm_subpage_folio(slot) != folio) + goto out; + ret = mm_subpage_move_prepare(&move, slot, src_vma, src_addr, + dst_vma, dst_addr, false); + if (ret) + goto out; + orig_src = ptep_clear_flush(src_vma, src_addr, src_pte); + ret = mm_subpage_move_commit(&move); + if (ret) { + set_pte_at(mm, src_addr, src_pte, orig_src); + } else { + pte = phys_pte_mm(mm, mm_subpage_phys(slot), dst_vma->vm_page_prot); + if (pgtable_supports_soft_dirty()) + pte = pte_mksoft_dirty(pte); + if (pte_dirty(orig_src)) + pte = pte_mkdirty(pte); + pte = pte_mkwrite(pte, dst_vma); + set_pte_at(mm, dst_addr, dst_pte, pte); + ret = mm_page_size(mm); + } + mm_subpage_move_finish(&move); +out: + if (slot) + mm_subpage_put(slot); + double_pt_unlock(dst_ptl, src_ptl); + return ret; +} + +/* Both PTLs and, if present, the swap-cache folio lock are held. */ +static int move_swap_subpage(struct mm_struct *mm, struct vm_area_struct *dst_vma, + struct vm_area_struct *src_vma, unsigned long dst_addr, + unsigned long src_addr, pte_t *dst_pte, pte_t *src_pte, + pte_t orig_src, struct folio *folio, swp_entry_t entry) +{ + unsigned int offset = pte_swp_subpage_offset(orig_src); + struct mm_subpage_move move; + struct mm_subpage *slot = NULL; + int ret = -EBUSY; + + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, mm_page_size(mm))) + return -EFAULT; + if (swap_subpage_count(entry, offset) != 1) + return -EBUSY; + if (folio) { + slot = mm_subpage_get_from_phys(PFN_PHYS(folio_pfn(folio)) + offset); + if (slot) { + ret = mm_subpage_move_prepare(&move, slot, src_vma, src_addr, + dst_vma, dst_addr, true); + if (ret) + goto out; + ret = mm_subpage_move_commit(&move); + if (ret) { + mm_subpage_move_finish(&move); + goto out; + } + } + } + /* No mapping count changes: the same-mm exclusive swap PTE is transferred. */ + orig_src = ptep_get_and_clear(mm, src_addr, src_pte); + if (pgtable_supports_soft_dirty()) + orig_src = pte_swp_mksoft_dirty(orig_src); + set_pte_at(mm, dst_addr, dst_pte, orig_src); + if (slot) + mm_subpage_move_finish(&move); + ret = mm_page_size(mm); +out: + if (slot) + mm_subpage_put(slot); + return ret; +} +#endif + static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, + struct vm_area_struct *src_vma, unsigned long dst_addr, unsigned long src_addr, pte_t *dst_pte, pte_t *src_pte, pte_t orig_dst_pte, pte_t orig_src_pte, @@ -1345,6 +1741,21 @@ static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, return -EAGAIN; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + int ret; + + if (!src_folio && swap_cache_has_folio(entry)) { + double_pt_unlock(dst_ptl, src_ptl); + return -EAGAIN; + } + ret = move_swap_subpage(mm, dst_vma, src_vma, dst_addr, src_addr, + dst_pte, src_pte, orig_src_pte, src_folio, entry); + double_pt_unlock(dst_ptl, src_ptl); + return ret; + } +#endif + /* * The src_folio resides in the swapcache, requiring an update to its * index and mapping to align with the dst_vma, where a swap-in may @@ -1397,13 +1808,13 @@ static int move_zeropage_pte(struct mm_struct *mm, return -EAGAIN; } - zero_pte = pte_mkspecial(pfn_pte(zero_pfn(dst_addr), - dst_vma->vm_page_prot)); + zero_pte = pte_mkspecial(phys_pte_mm(mm, PFN_PHYS(zero_pfn(dst_addr)), + dst_vma->vm_page_prot)); ptep_clear_flush(src_vma, src_addr, src_pte); set_pte_at(mm, dst_addr, dst_pte, zero_pte); double_pt_unlock(dst_ptl, src_ptl); - return PAGE_SIZE; + return mm_page_size(mm); } @@ -1490,7 +1901,7 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd if (!(mode & UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES)) ret = -ENOENT; else /* nothing to do to move a hole */ - ret = PAGE_SIZE; + ret = mm_page_size(mm); goto out; } @@ -1530,7 +1941,8 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd } folio = vm_normal_folio(src_vma, src_addr, orig_src_pte); - if (!folio || !PageAnonExclusive(&folio->page)) { + if (!folio || (mm_page_size(mm) >= PAGE_SIZE && + !PageAnonExclusive(&folio->page))) { spin_unlock(src_ptl); ret = -EBUSY; goto out; @@ -1570,6 +1982,15 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd } } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + ret = move_present_subpage(mm, dst_vma, src_vma, dst_addr, src_addr, + dst_pte, src_pte, orig_dst_pte, orig_src_pte, + dst_pmd, dst_pmdval, dst_ptl, src_ptl, src_folio); + goto out; + } +#endif + /* at this point we have src_folio locked */ if (folio_test_large(src_folio)) { /* split_folio() can block */ @@ -1608,7 +2029,8 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd goto out; } - if (!pte_swp_exclusive(orig_src_pte)) { + /* Smaller leaves validate per-slot swap and mapping counts under PTL. */ + if (mm_page_size(mm) >= PAGE_SIZE && !pte_swp_exclusive(orig_src_pte)) { ret = -EBUSY; goto out; } @@ -1651,7 +2073,7 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd goto retry; } } - ret = move_swap_pte(mm, dst_vma, dst_addr, src_addr, dst_pte, src_pte, + ret = move_swap_pte(mm, dst_vma, src_vma, dst_addr, src_addr, dst_pte, src_pte, orig_dst_pte, orig_src_pte, dst_pmd, dst_pmdval, dst_ptl, src_ptl, src_folio, si, entry); } @@ -1710,6 +2132,11 @@ static int validate_move_areas(struct userfaultfd_ctx *ctx, pgprot_val(src_vma->vm_page_prot) != pgprot_val(dst_vma->vm_page_prot)) return -EINVAL; + /* Native backing cannot mix droppable and swap-backed slot policies. */ + if (mm_page_size(src_vma->vm_mm) < PAGE_SIZE && + ((src_vma->vm_flags ^ dst_vma->vm_flags) & VM_DROPPABLE)) + return -EINVAL; + /* Only allow moving if both are mlocked or both aren't */ if ((src_vma->vm_flags & VM_LOCKED) != (dst_vma->vm_flags & VM_LOCKED)) return -EINVAL; @@ -1942,9 +2369,9 @@ static ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, ssize_t moved = 0; /* Sanitize the command parameters. */ - VM_WARN_ON_ONCE(src_start & ~PAGE_MASK); - VM_WARN_ON_ONCE(dst_start & ~PAGE_MASK); - VM_WARN_ON_ONCE(len & ~PAGE_MASK); + VM_WARN_ON_ONCE(src_start & ~mm_page_mask(mm)); + VM_WARN_ON_ONCE(dst_start & ~mm_page_mask(mm)); + VM_WARN_ON_ONCE(len & ~mm_page_mask(mm)); /* Does the address range wrap, or is the span zero-sized? */ VM_WARN_ON_ONCE(src_start + len < src_start); @@ -2069,10 +2496,12 @@ static ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, ret = move_pages_ptes(mm, dst_pmd, src_pmd, dst_vma, src_vma, dst_addr, src_addr, src_end - src_addr, mode); - if (ret < 0) + if (ret < 0) { err = ret; - else + } else { + err = 0; step_size = ret; + } } cond_resched(); @@ -2113,6 +2542,17 @@ static bool vma_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags, { const struct vm_uffd_ops *ops = vma_uffd_ops(vma); + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) { + if (mm_page_size(vma->vm_mm) > PAGE_SIZE) + return false; + if (vma_is_anonymous(vma)) { + if (vm_flags & VM_UFFD_MINOR) + return false; + } else if (!vma_is_shmem(vma)) { + return false; + } + } + if (vma->vm_flags & (VM_DROPPABLE | VM_SHADOW_STACK)) return false; @@ -2595,6 +3035,7 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, pmd_t *pmd, _pmd; pte_t *pte; pte_t ptent; + spinlock_t *ptl; bool ret; assert_fault_locked(vmf); @@ -2602,13 +3043,13 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, pgd = pgd_offset(mm, address); if (!pgd_present(*pgd)) return true; - p4d = p4d_offset(pgd, address); - if (!p4d_present(*p4d)) + p4d = p4d_offset_mm(mm, pgd, address); + if (!p4d_present_mm(mm, *p4d)) return true; - pud = pud_offset(p4d, address); + pud = pud_offset_mm(mm, p4d, address); if (!pud_present(*pud)) return true; - pmd = pmd_offset(pud, address); + pmd = pmd_offset_mm(mm, pud, address); again: _pmd = pmdp_get_lockless(pmd); if (pmd_none(_pmd)) @@ -2625,7 +3066,7 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, if (pmd_trans_huge(_pmd)) return !pmd_write(_pmd) && (reason & VM_UFFD_WP); - pte = pte_offset_map(pmd, address); + pte = pte_offset_map_ro_nolock(mm, pmd, address, &ptl); if (!pte) goto again; @@ -3545,13 +3986,11 @@ static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx, __wake_userfault(ctx, range); } -static __always_inline int validate_unaligned_range( +static __always_inline int validate_byte_range( struct mm_struct *mm, __u64 start, __u64 len) { __u64 task_size = mm->task_size; - if (len & ~PAGE_MASK) - return -EINVAL; if (!len) return -EINVAL; if (start >= task_size) @@ -3563,10 +4002,19 @@ static __always_inline int validate_unaligned_range( return 0; } +static __always_inline int validate_unaligned_range( + struct mm_struct *mm, __u64 start, __u64 len) +{ + if (len & ~mm_page_mask(mm)) + return -EINVAL; + + return validate_byte_range(mm, start, len); +} + static __always_inline int validate_range(struct mm_struct *mm, __u64 start, __u64 len) { - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(mm)) return -EINVAL; return validate_unaligned_range(mm, start, len); @@ -3722,6 +4170,13 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, ioctls_out = basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC : UFFD_API_RANGE_IOCTLS; + if (mm_page_size(mm) != PAGE_SIZE) { + ioctls_out &= (1ULL << _UFFDIO_WAKE) | (1ULL << _UFFDIO_WRITEPROTECT) | + (1ULL << _UFFDIO_CONTINUE) | (1ULL << _UFFDIO_COPY) | + (1ULL << _UFFDIO_ZEROPAGE) | (1ULL << _UFFDIO_POISON); + } + if (mm_page_size(mm) < PAGE_SIZE) + ioctls_out |= 1ULL << _UFFDIO_MOVE; /* * Declare the WP ioctl only if the WP mode is @@ -3939,8 +4394,12 @@ static int userfaultfd_copy(struct userfaultfd_ctx *ctx, sizeof(uffdio_copy)-sizeof(__s64))) goto out; - ret = validate_unaligned_range(ctx->mm, uffdio_copy.src, - uffdio_copy.len); + /* + * COPY reads the pager's address space, which can have a different + * task size and granule from the registered target. Only the target + * range below requires a page-aligned length. + */ + ret = validate_byte_range(current->mm, uffdio_copy.src, uffdio_copy.len); if (ret) goto out; ret = validate_range(ctx->mm, uffdio_copy.dst, uffdio_copy.len); @@ -4309,6 +4768,18 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx, /* report all available features and ioctls to userland */ uffdio_api.features = UFFD_API_FEATURES; + /* Base-page anonymous and shared shmem operations support alternative leaves. */ + if (mm_page_size(ctx->mm) != PAGE_SIZE) + uffdio_api.features &= UFFD_FEATURE_PAGEFAULT_FLAG_WP | + UFFD_FEATURE_MISSING_SHMEM | UFFD_FEATURE_MINOR_SHMEM | + UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_EVENT_REMAP | + UFFD_FEATURE_EVENT_REMOVE | UFFD_FEATURE_EVENT_UNMAP | + UFFD_FEATURE_SIGBUS | UFFD_FEATURE_THREAD_ID | UFFD_FEATURE_EXACT_ADDRESS | + UFFD_FEATURE_WP_UNPOPULATED | UFFD_FEATURE_WP_ASYNC | UFFD_FEATURE_POISON; + + if (mm_page_size(ctx->mm) < PAGE_SIZE) + uffdio_api.features |= UFFD_FEATURE_MOVE; + #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR uffdio_api.features &= ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM); @@ -4356,6 +4827,10 @@ static long userfaultfd_ioctl(struct file *file, unsigned cmd, if (cmd != UFFDIO_API && !userfaultfd_is_initialized(ctx)) return -EINVAL; + /* Larger-than-native MOVE still requires contiguous-owner support. */ + if (cmd == UFFDIO_MOVE && mm_page_size(ctx->mm) > PAGE_SIZE) + return -EOPNOTSUPP; + switch (cmd) { case UFFDIO_API: ret = userfaultfd_api(ctx, arg); diff --git a/mm/util.c b/mm/util.c index 34cb43b3eaa4c..a4c25d097ec61 100644 --- a/mm/util.c +++ b/mm/util.c @@ -344,21 +344,22 @@ unsigned long randomize_stack_top(unsigned long stack_top) if (current->flags & PF_RANDOMIZE) { random_variable = get_random_long(); - random_variable &= STACK_RND_MASK; - random_variable <<= PAGE_SHIFT; + random_variable &= mm_aslr_mask(current->mm, STACK_RND_MASK); + random_variable <<= mm_page_shift(current->mm); } #ifdef CONFIG_STACK_GROWSUP - return PAGE_ALIGN(stack_top) + random_variable; + return mm_page_align(current->mm, stack_top) + random_variable; #else - return PAGE_ALIGN(stack_top) - random_variable; + return mm_page_align(current->mm, stack_top) - random_variable; #endif } /** - * randomize_page - Generate a random, page aligned address + * randomize_page_shift - Generate a random, granule aligned address * @start: The smallest acceptable address the caller will take. * @range: The size of the area, starting at @start, within which the * random address must fall. + * @shift: Log2 of the required address alignment. * * If @start + @range would overflow, @range is capped. * @@ -368,22 +369,31 @@ unsigned long randomize_stack_top(unsigned long stack_top) * Return: A page aligned address within [start, start + range). On error, * @start is returned. */ -unsigned long randomize_page(unsigned long start, unsigned long range) +static unsigned long randomize_page_shift(unsigned long start, + unsigned long range, unsigned int shift) { - if (!PAGE_ALIGNED(start)) { - range -= PAGE_ALIGN(start) - start; - start = PAGE_ALIGN(start); + unsigned long page_size = 1UL << shift; + + if (!IS_ALIGNED(start, page_size)) { + range -= ALIGN(start, page_size) - start; + start = ALIGN(start, page_size); } if (start > ULONG_MAX - range) range = ULONG_MAX - start; - range >>= PAGE_SHIFT; + range >>= shift; if (range == 0) return start; - return start + (get_random_long() % range << PAGE_SHIFT); + return start + (get_random_long() % range << shift); +} + +/* Existing callers retain native-page alignment. */ +unsigned long randomize_page(unsigned long start, unsigned long range) +{ + return randomize_page_shift(start, range, PAGE_SHIFT); } #ifdef CONFIG_ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT @@ -391,23 +401,25 @@ unsigned long __weak arch_randomize_brk(struct mm_struct *mm) { /* Is the current task 32bit ? */ if (!IS_ENABLED(CONFIG_64BIT) || is_compat_task()) - return randomize_page(mm->brk, SZ_32M); + return randomize_page_shift(mm->brk, SZ_32M, mm_page_shift(mm)); - return randomize_page(mm->brk, SZ_1G); + return randomize_page_shift(mm->brk, SZ_1G, mm_page_shift(mm)); } unsigned long arch_mmap_rnd(void) { - unsigned long rnd; + unsigned int bits; + unsigned long mask; #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS if (is_compat_task()) - rnd = get_random_long() & ((1UL << mmap_rnd_compat_bits) - 1); + bits = mmap_rnd_compat_bits; else #endif /* CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS */ - rnd = get_random_long() & ((1UL << mmap_rnd_bits) - 1); + bits = mmap_rnd_bits; - return rnd << PAGE_SHIFT; + mask = mm_aslr_mask(current->mm, (1UL << bits) - 1); + return (get_random_long() & mask) << mm_page_shift(current->mm); } static int mmap_is_legacy(const struct rlimit *rlim_stack) @@ -440,14 +452,15 @@ static unsigned long mmap_base(const unsigned long rnd, const struct rlimit *rli * task. mmap_base starts directly below the stack and grows * downwards. */ - return PAGE_ALIGN_DOWN(mmap_upper_limit(rlim_stack) - rnd); + return (mmap_upper_limit(rlim_stack) - rnd) & mm_page_mask(current->mm); #else unsigned long gap = rlim_stack->rlim_cur; unsigned long pad = stack_guard_gap; /* Account for stack randomization if necessary */ if (current->flags & PF_RANDOMIZE) - pad += (STACK_RND_MASK << PAGE_SHIFT); + pad += mm_aslr_mask(current->mm, STACK_RND_MASK) << + mm_page_shift(current->mm); /* Values close to RLIM_INFINITY can overflow. */ if (gap + pad > gap) @@ -458,7 +471,7 @@ static unsigned long mmap_base(const unsigned long rnd, const struct rlimit *rli else if (gap > MAX_GAP) gap = MAX_GAP; - return PAGE_ALIGN(STACK_TOP - gap - rnd); + return mm_page_align(current->mm, STACK_TOP - gap - rnd); #endif } @@ -491,7 +504,7 @@ EXPORT_SYMBOL_IF_KUNIT(arch_pick_mmap_layout); /** * __account_locked_vm - account locked pages to an mm's locked_vm * @mm: mm to account against - * @pages: number of pages to account + * @pages: number of native physical pages to account * @inc: %true if @pages should be considered positive, %false if not * @task: task used to check RLIMIT_MEMLOCK * @bypass_rlim: %true if checking RLIMIT_MEMLOCK should be skipped @@ -507,27 +520,35 @@ int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, const struct task_struct *task, bool bypass_rlim) { unsigned long locked_vm, limit; + unsigned int shift = PAGE_SHIFT - MM_ACCOUNT_SHIFT; int ret = 0; mmap_assert_write_locked(mm); + /* This exported API receives native physical-page counts. */ + if (pages > ULONG_MAX >> shift) + return -ENOMEM; + pages <<= shift; locked_vm = mm->locked_vm; if (inc) { if (!bypass_rlim) { - limit = task_rlimit(task, RLIMIT_MEMLOCK) >> PAGE_SHIFT; - if (locked_vm + pages > limit) + limit = task_rlimit(task, RLIMIT_MEMLOCK) >> MM_ACCOUNT_SHIFT; + if (pages > limit || locked_vm > limit - pages) ret = -ENOMEM; } + if (!ret && pages > ULONG_MAX - locked_vm) + ret = -ENOMEM; if (!ret) mm->locked_vm = locked_vm + pages; } else { - WARN_ON_ONCE(pages > locked_vm); + if (WARN_ON_ONCE(pages > locked_vm)) + return -EINVAL; mm->locked_vm = locked_vm - pages; } pr_debug("%s: [%d] caller %ps %c%lu %lu/%lu%s\n", __func__, task->pid, - (void *)_RET_IP_, (inc) ? '+' : '-', pages << PAGE_SHIFT, - locked_vm << PAGE_SHIFT, task_rlimit(task, RLIMIT_MEMLOCK), + (void *)_RET_IP_, (inc) ? '+' : '-', pages << MM_ACCOUNT_SHIFT, + locked_vm << MM_ACCOUNT_SHIFT, task_rlimit(task, RLIMIT_MEMLOCK), ret ? " - exceeded" : ""); return ret; @@ -562,11 +583,11 @@ int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc) } EXPORT_SYMBOL_GPL(account_locked_vm); -unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, +static unsigned long vm_mmap_offset_internal(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, - unsigned long flag, unsigned long pgoff) + unsigned long flag, struct vm_page_offset pos) { - loff_t off = (loff_t)pgoff << PAGE_SHIFT; + loff_t off = ((loff_t)pos.index << PAGE_SHIFT) | pos.offset; unsigned long ret; struct mm_struct *mm = current->mm; unsigned long populate; @@ -578,8 +599,11 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, if (!ret) { if (mmap_write_lock_killable(mm)) return -EINTR; - ret = do_mmap(file, addr, len, prot, flag, 0, pgoff, &populate, - &uf); +#ifdef CONFIG_MM_SUBPAGE + ret = do_mmap_offset(file, addr, len, prot, flag, 0, pos, &populate, &uf); +#else + ret = do_mmap(file, addr, len, prot, flag, 0, pos.index, &populate, &uf); +#endif mmap_write_unlock(mm); userfaultfd_unmap_complete(mm, &uf); if (populate) @@ -588,6 +612,13 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, return ret; } +unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flag, unsigned long pgoff) +{ + return vm_mmap_offset_internal(file, addr, len, prot, flag, + (struct vm_page_offset) { pgoff, 0 }); +} + /* * Perform a userland memory mapping into the current process address space. See * the comment for do_mmap() for more details on this operation in general. @@ -609,12 +640,13 @@ unsigned long vm_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flag, unsigned long offset) { - if (unlikely(offset + PAGE_ALIGN(len) < offset)) + if (unlikely(offset + mm_page_align(current->mm, len) < offset)) return -EINVAL; - if (unlikely(offset_in_page(offset))) + if (unlikely(offset & ~mm_page_mask(current->mm))) return -EINVAL; - return vm_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT); + return vm_mmap_offset_internal(file, addr, len, prot, flag, + (struct vm_page_offset) { offset >> PAGE_SHIFT, offset & ~PAGE_MASK }); } EXPORT_SYMBOL(vm_mmap); @@ -988,7 +1020,8 @@ int __vm_enough_memory(const struct mm_struct *mm, long pages, int cap_sys_admin if (mm) { long reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10); - allowed -= min_t(long, mm->total_vm / 32, reserve); + allowed -= min_t(long, + mm_pages_to_bytes(mm, mm->total_vm) >> (PAGE_SHIFT + 5), reserve); } if (percpu_counter_read_positive(&vm_committed_as) < allowed) @@ -1189,6 +1222,9 @@ void compat_set_desc_from_vma(struct vm_area_desc *desc, desc->end = vma->vm_end; desc->pgoff = vma->vm_pgoff; +#ifdef CONFIG_MM_SUBPAGE + desc->subpage_offset = vma->vm_subpage_offset; +#endif desc->vm_file = vma->vm_file; desc->vma_flags = vma->flags; desc->page_prot = vma->vm_page_prot; diff --git a/mm/vma.c b/mm/vma.c index 07881c0987cf9..8646a0a8d91a9 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -13,7 +13,7 @@ struct mmap_state { unsigned long addr; unsigned long end; - pgoff_t pgoff; + struct vm_page_offset offset; unsigned long pglen; union { vm_flags_t vm_flags; @@ -42,19 +42,25 @@ struct mmap_state { bool file_doesnt_need_get :1; }; -#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pgoff_, vma_flags_, file_) \ +#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pos_, vma_flags_, file_) \ struct mmap_state name = { \ .mm = mm_, \ .vmi = vmi_, \ .addr = addr_, \ .end = (addr_) + (len_), \ - .pgoff = pgoff_, \ - .pglen = PHYS_PFN(len_), \ + .offset = pos_, \ + .pglen = (len_) >> mm_page_shift(mm_), \ .vma_flags = vma_flags_, \ .file = file_, \ .page_prot = vma_get_page_prot(vma_flags_), \ } +#ifdef CONFIG_MM_SUBPAGE +#define VMG_MMAP_SUBPAGE_STATE(map_) .subpage_offset = (map_)->offset.offset, +#else +#define VMG_MMAP_SUBPAGE_STATE(map_) +#endif + #define VMG_MMAP_STATE(name, map_, vma_) \ struct vma_merge_struct name = { \ .mm = (map_)->mm, \ @@ -62,7 +68,8 @@ struct mmap_state { .start = (map_)->addr, \ .end = (map_)->end, \ .vma_flags = (map_)->vma_flags, \ - .pgoff = (map_)->pgoff, \ + .pgoff = (map_)->offset.index, \ + VMG_MMAP_SUBPAGE_STATE(map_) \ .file = (map_)->file, \ .prev = (map_)->prev, \ .middle = vma_, \ @@ -197,11 +204,12 @@ static void init_multi_vma_prep(struct vma_prepare *vp, */ static bool can_vma_merge_before(struct vma_merge_struct *vmg) { - pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start); + struct vm_page_offset end = vm_page_offset_add(vmg_page_offset(vmg), + vmg->end - vmg->start); if (is_mergeable_vma(vmg, /* merge_next = */ true) && is_mergeable_anon_vma(vmg, /* merge_next = */ true)) { - if (vmg->next->vm_pgoff == vmg->pgoff + pglen) + if (vm_page_offset_equal(vma_page_offset_at(vmg->next, vmg->next->vm_start), end)) return true; } @@ -221,7 +229,8 @@ static bool can_vma_merge_after(struct vma_merge_struct *vmg) { if (is_mergeable_vma(vmg, /* merge_next = */ false) && is_mergeable_anon_vma(vmg, /* merge_next = */ false)) { - if (vmg->prev->vm_pgoff + vma_pages(vmg->prev) == vmg->pgoff) + if (vm_page_offset_equal(vma_page_offset_at(vmg->prev, vmg->prev->vm_end), + vmg_page_offset(vmg))) return true; } return false; @@ -521,7 +530,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, new->vm_end = addr; } else { new->vm_start = addr; - new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT); + vma_set_page_offset(new, vma_page_offset_at(vma, addr)); } err = -ENOMEM; @@ -560,7 +569,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, if (new_below) { vma->vm_start = addr; - vma->vm_pgoff += (addr - new->vm_start) >> PAGE_SHIFT; + vma_set_page_offset(vma, vma_page_offset_at(new, addr)); } else { vma->vm_end = addr; } @@ -705,19 +714,21 @@ void validate_mm(struct mm_struct *mm) static void vmg_adjust_set_range(struct vma_merge_struct *vmg) { struct vm_area_struct *adjust; - pgoff_t pgoff; + struct vm_page_offset pos; if (vmg->__adjust_middle_start) { adjust = vmg->middle; - pgoff = adjust->vm_pgoff + PHYS_PFN(vmg->end - adjust->vm_start); + pos = vma_page_offset_at(adjust, vmg->end); } else if (vmg->__adjust_next_start) { adjust = vmg->next; - pgoff = adjust->vm_pgoff - PHYS_PFN(adjust->vm_start - vmg->end); + pos = vm_page_offset_sub(vma_page_offset_at(adjust, adjust->vm_start), + adjust->vm_start - vmg->end); } else { return; } - vma_set_range(adjust, vmg->end, adjust->vm_end, pgoff); + vma_set_range(adjust, vmg->end, adjust->vm_end, pos.index); + vma_set_page_offset(adjust, pos); } /* @@ -762,6 +773,7 @@ static int commit_merge(struct vma_merge_struct *vmg) vma_adjust_trans_huge(vma, vmg->start, vmg->end, vmg->__adjust_middle_start ? vmg->middle : NULL); vma_set_range(vma, vmg->start, vmg->end, vmg->pgoff); + vma_set_page_offset(vma, vmg_page_offset(vmg)); vmg_adjust_set_range(vmg); vma_iter_store_overwrite(vmg->vmi, vmg->target); @@ -931,7 +943,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( vmg->start = prev->vm_start; vmg->end = next->vm_end; - vmg->pgoff = prev->vm_pgoff; + vmg_set_page_offset(vmg, vma_page_offset_at(prev, prev->vm_start)); /* * We already ensured anon_vma compatibility above, so now it's @@ -950,7 +962,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( */ vmg->start = prev->vm_start; - vmg->pgoff = prev->vm_pgoff; + vmg_set_page_offset(vmg, vma_page_offset_at(prev, prev->vm_start)); if (!vmg->__remove_middle) vmg->__adjust_middle_start = true; @@ -965,7 +977,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( * shrink/delete extend */ - pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start); + unsigned long len = vmg->end - vmg->start; VM_WARN_ON_VMG(!merge_right, vmg); /* If we are offset into a VMA, then prev must be middle. */ @@ -973,13 +985,14 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( if (vmg->__remove_middle) { vmg->end = next->vm_end; - vmg->pgoff = next->vm_pgoff - pglen; + vmg_set_page_offset(vmg, + vm_page_offset_sub(vma_page_offset_at(next, next->vm_start), len)); } else { /* We shrink middle and expand next. */ vmg->__adjust_next_start = true; vmg->start = middle->vm_start; vmg->end = start; - vmg->pgoff = middle->vm_pgoff; + vmg_set_page_offset(vmg, vma_page_offset_at(middle, middle->vm_start)); } err = dup_anon_vma(next, middle, &anon_dup); @@ -1088,7 +1101,7 @@ struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg) if (can_merge_left) { vmg->start = prev->vm_start; vmg->target = prev; - vmg->pgoff = prev->vm_pgoff; + vmg_set_page_offset(vmg, vma_page_offset_at(prev, prev->vm_start)); /* * If this merge would result in removal of the next VMA but we @@ -1245,7 +1258,7 @@ int vma_expand(struct vma_merge_struct *vmg) * Returns: 0 on success, -ENOMEM otherwise */ int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, - unsigned long start, unsigned long end, pgoff_t pgoff) + unsigned long start, unsigned long end, struct vm_page_offset pos) { struct vma_prepare vp; @@ -1266,7 +1279,8 @@ int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_adjust_trans_huge(vma, start, end, NULL); vma_iter_clear(vmi); - vma_set_range(vma, start, end, pgoff); + vma_set_range(vma, start, end, pos.index); + vma_set_page_offset(vma, pos); vma_complete(&vp, vmi, vma->vm_mm); validate_mm(vma->vm_mm); return 0; @@ -1335,7 +1349,7 @@ static void vms_complete_munmap_vmas(struct vma_munmap_struct *vms, mm = current->mm; mm->map_count -= vms->vma_count; - mm->locked_vm -= vms->locked_vm; + mm->locked_vm -= mm_pages_to_account(mm, vms->locked_vm); if (vms->unlock) mmap_write_downgrade(mm); @@ -1360,7 +1374,7 @@ static void vms_complete_munmap_vmas(struct vma_munmap_struct *vms, mas_for_each(mas_detach, vma, ULONG_MAX) remove_vma(vma); - vm_unacct_memory(vms->nr_accounted); + mm_unacct_memory(mm, vms->nr_accounted); validate_mm(mm); if (vms->unlock) mmap_read_unlock(mm); @@ -1456,7 +1470,7 @@ static int vms_gather_munmap_vmas(struct vma_munmap_struct *vms, goto munmap_gather_failed; vma_mark_detached(next); - nrpages = vma_pages(next); + nrpages = vma_user_pages(next); vms->nr_pages += nrpages; if (vma_test(next, VMA_LOCKED_BIT)) @@ -1634,10 +1648,10 @@ int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, unsigned long end; struct vm_area_struct *vma; - if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start) + if ((start & ~mm_page_mask(mm)) || start > TASK_SIZE || len > TASK_SIZE-start) return -EINVAL; - end = start + PAGE_ALIGN(len); + end = start + mm_page_align(mm, len); if (end == start) return -EINVAL; @@ -1862,7 +1876,7 @@ static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma) * prior to moving page table entries, to effect an mremap move. */ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, - unsigned long addr, unsigned long len, pgoff_t pgoff, + unsigned long addr, unsigned long len, struct vm_page_offset pos, bool *need_rmap_locks, bool keep_source) { struct vm_area_struct *vma = *vmap; @@ -1878,7 +1892,10 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, * to match new location, to increase its chance of merging. */ if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) { - pgoff = addr >> PAGE_SHIFT; + pos = (struct vm_page_offset) { + .index = addr >> PAGE_SHIFT, + .offset = addr & ~PAGE_MASK, + }; faulted_in_anon_vma = false; } @@ -1894,7 +1911,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, if (new_vma && new_vma->vm_start < addr + len) return NULL; /* should never get here */ - vmg.pgoff = pgoff; + vmg_set_page_offset(&vmg, pos); vmg.next = vma_iter_next_rewind(&vmi, NULL); /* @@ -1939,7 +1956,8 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, new_vma = vm_area_dup(vma); if (!new_vma) goto out; - vma_set_range(new_vma, addr, addr + len, pgoff); + vma_set_range(new_vma, addr, addr + len, pos.index); + vma_set_page_offset(new_vma, pos); if (vma_dup_policy(vma, new_vma)) goto out_free_vma; if (anon_vma_clone(new_vma, vma, VMA_OP_REMAP)) @@ -1994,7 +2012,8 @@ static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct * mpol_equal(vma_policy(a), vma_policy(b)) && a->vm_file == b->vm_file && vma_flags_empty(&diff) && - b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT); + vm_page_offset_equal(vma_page_offset_at(a, b->vm_start), + vma_page_offset_at(b, b->vm_start)); } /* @@ -2421,7 +2440,10 @@ static void set_desc_from_map(struct vm_area_desc *desc, desc->start = map->addr; desc->end = map->end; - desc->pgoff = map->pgoff; + desc->pgoff = map->offset.index; +#ifdef CONFIG_MM_SUBPAGE + desc->subpage_offset = map->offset.offset; +#endif desc->vm_file = map->file; desc->vma_flags = map->vma_flags; desc->page_prot = map->page_prot; @@ -2481,7 +2503,7 @@ static int __mmap_setup(struct mmap_state *map, struct vm_area_desc *desc, map->charged = map->pglen; map->charged -= vms->nr_accounted; if (map->charged) { - error = security_vm_enough_memory_mm(map->mm, map->charged); + error = mm_account_memory(map->mm, map->mm, map->charged); if (error) return error; } @@ -2572,7 +2594,8 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, return -ENOMEM; vma_iter_config(vmi, map->addr, map->end); - vma_set_range(vma, map->addr, map->end, map->pgoff); + vma_set_range(vma, map->addr, map->end, map->offset.index); + vma_set_page_offset(vma, map->offset); vma->flags = map->vma_flags; vma->vm_page_prot = map->page_prot; @@ -2644,7 +2667,7 @@ static void __mmap_complete(struct mmap_state *map, struct vm_area_struct *vma) if (!vma_supports_mlock(vma)) vma_clear_flags_mask(vma, VMA_LOCKED_MASK); else - mm->locked_vm += map->pglen; + mm->locked_vm += mm_pages_to_account(mm, map->pglen); } if (vma->vm_file) @@ -2701,7 +2724,10 @@ static int call_mmap_prepare(struct mmap_state *map, return err; /* Update fields permitted to be changed. */ - map->pgoff = desc->pgoff; + map->offset.index = desc->pgoff; +#ifdef CONFIG_MM_SUBPAGE + map->offset.offset = desc->subpage_offset; +#endif if (desc->vm_file != map->file) { map->file_doesnt_need_get = true; map->file = desc->vm_file; @@ -2712,6 +2738,12 @@ static int call_mmap_prepare(struct mmap_state *map, map->vm_ops = desc->vm_ops; map->vm_private_data = desc->private_data; +#ifdef CONFIG_MM_SUBPAGE + if (map->offset.offset >= PAGE_SIZE || + !IS_ALIGNED(map->offset.offset, mm_page_size(map->mm))) + return -EINVAL; +#endif + return 0; } @@ -2756,13 +2788,13 @@ static bool can_set_ksm_flags_early(struct mmap_state *map) static unsigned long __mmap_region(struct file *file, unsigned long addr, unsigned long len, vma_flags_t vma_flags, - unsigned long pgoff, struct list_head *uf) + struct vm_page_offset pos, struct list_head *uf) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma = NULL; const bool have_mmap_prepare = file && file->f_op->mmap_prepare; VMA_ITERATOR(vmi, mm, addr); - MMAP_STATE(map, mm, &vmi, addr, len, pgoff, vma_flags, file); + MMAP_STATE(map, mm, &vmi, addr, len, pos, vma_flags, file); struct vm_area_desc desc = { .mm = mm, .file = file, @@ -2819,7 +2851,7 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, /* Accounting was done by __mmap_setup(). */ unacct_error: if (map.charged) - vm_unacct_memory(map.charged); + mm_unacct_memory(mm, map.charged); abort_munmap: /* * This indicates that .mmap_prepare has set a new file, differing from @@ -2835,7 +2867,7 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, /** * mmap_region() - Actually perform the userland mapping of a VMA into * current->mm with known, aligned and overflow-checked @addr and @len, and - * correctly determined VMA flags @vm_flags and page offset @pgoff. + * correctly determined VMA flags @vm_flags and file byte position @pos. * * This is an internal memory management function, and should not be used * directly. @@ -2847,8 +2879,8 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, * @addr: The page-aligned address at which to perform the mapping. * @len: The page-aligned, non-zero, length of the mapping. * @vm_flags: The VMA flags which should be applied to the mapping. - * @pgoff: If @file is specified, the page offset into the file, if not then - * the virtual page offset in memory of the anonymous mapping. + * @pos: Native page-cache index and byte offset within it. For anonymous + * mappings, the position is derived from the virtual address. * @uf: Optionally, a pointer to a list head used for tracking userfaultfd unmap * events. * @@ -2857,13 +2889,15 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, */ unsigned long mmap_region(struct file *file, unsigned long addr, unsigned long len, vm_flags_t vm_flags, - unsigned long pgoff, struct list_head *uf) + struct vm_page_offset pos, struct list_head *uf) { unsigned long ret; bool writable_file_mapping = false; const vma_flags_t vma_flags = legacy_to_vma_flags(vm_flags); mmap_assert_write_locked(current->mm); + if (pos.offset >= PAGE_SIZE || !IS_ALIGNED(pos.offset, mm_page_size(current->mm))) + return -EINVAL; /* Check to see if MDWE is applicable. */ if (map_deny_write_exec(&vma_flags, &vma_flags)) @@ -2882,7 +2916,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, writable_file_mapping = true; } - ret = __mmap_region(file, addr, len, vma_flags, pgoff, uf); + ret = __mmap_region(file, addr, len, vma_flags, pos, uf); /* Clear our write mapping regardless of error. */ if (writable_file_mapping) @@ -2920,13 +2954,13 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_flags_set_mask(&vma_flags, mm->def_vma_flags); vma_flags = ksm_vma_flags(mm, NULL, vma_flags); - if (!may_expand_vm(mm, &vma_flags, len >> PAGE_SHIFT)) + if (!may_expand_vm(mm, &vma_flags, len >> mm_page_shift(mm))) return -ENOMEM; if (mm->map_count > get_sysctl_max_map_count()) return -ENOMEM; - if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT)) + if (mm_account_memory(mm, mm, len >> mm_page_shift(mm))) return -ENOMEM; /* @@ -2936,6 +2970,10 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, if (vma && vma->vm_end == addr) { VMG_STATE(vmg, mm, vmi, addr, addr + len, vma_flags, PHYS_PFN(addr)); + vmg_set_page_offset(&vmg, (struct vm_page_offset) { + .index = addr >> PAGE_SHIFT, .offset = addr & ~PAGE_MASK, + }); + vmg.prev = vma; /* vmi is positioned at prev, which this mode expects. */ vmg.just_expand = true; @@ -2955,6 +2993,9 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_set_anonymous(vma); vma_set_range(vma, addr, addr + len, addr >> PAGE_SHIFT); + vma_set_page_offset(vma, (struct vm_page_offset) { + .index = addr >> PAGE_SHIFT, .offset = addr & ~PAGE_MASK, + }); vma->flags = vma_flags; vma->vm_page_prot = vm_get_page_prot(vma_flags_to_legacy(vma_flags)); vma_start_write(vma); @@ -2965,10 +3006,10 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, validate_mm(mm); out: perf_event_mmap(vma); - mm->total_vm += len >> PAGE_SHIFT; - mm->data_vm += len >> PAGE_SHIFT; + mm->total_vm += len >> mm_page_shift(mm); + mm->data_vm += len >> mm_page_shift(mm); if (vma_flags_test(&vma_flags, VMA_LOCKED_BIT)) - mm->locked_vm += (len >> PAGE_SHIFT); + mm->locked_vm += len >> MM_ACCOUNT_SHIFT; if (pgtable_supports_soft_dirty()) vma_set_flags(vma, VMA_SOFTDIRTY_BIT); return 0; @@ -2976,7 +3017,7 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, mas_store_fail: vm_area_free(vma); unacct_fail: - vm_unacct_memory(len >> PAGE_SHIFT); + mm_unacct_memory(mm, len >> mm_page_shift(mm)); return -ENOMEM; } @@ -3112,7 +3153,7 @@ static int acct_stack_growth(struct vm_area_struct *vma, /* mlock limit tests */ if (!mlock_future_ok(mm, vma_test(vma, VMA_LOCKED_BIT), - grow << PAGE_SHIFT)) + grow << mm_page_shift(mm))) return -ENOMEM; /* Check to ensure the stack will not grow into a hugetlb-only region */ @@ -3128,7 +3169,7 @@ static int acct_stack_growth(struct vm_area_struct *vma, * Overcommit.. This must be the final test, as it will * update security statistics. */ - if (security_vm_enough_memory_mm(mm, grow)) + if (mm_account_memory(mm, mm, grow)) return -ENOMEM; return 0; @@ -3153,10 +3194,10 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) mmap_assert_write_locked(mm); /* Guard against exceeding limits of the address space. */ - address &= PAGE_MASK; - if (address >= (TASK_SIZE & PAGE_MASK)) + address &= mm_page_mask(mm); + if (address >= (TASK_SIZE & mm_page_mask(mm))) return -ENOMEM; - address += PAGE_SIZE; + address += mm_page_size(mm); /* Enforce stack_guard_gap */ gap_addr = address + stack_guard_gap; @@ -3195,14 +3236,14 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) unsigned long size, grow; size = address - vma->vm_start; - grow = (address - vma->vm_end) >> PAGE_SHIFT; + grow = (address - vma->vm_end) >> mm_page_shift(mm); error = -ENOMEM; if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) { error = acct_stack_growth(vma, size, grow); if (!error) { if (vma_test(vma, VMA_LOCKED_BIT)) - mm->locked_vm += grow; + mm->locked_vm += mm_pages_to_account(mm, grow); vm_stat_account(mm, vma->vm_flags, grow); anon_vma_interval_tree_pre_update_vma(vma); vma->vm_end = address; @@ -3237,7 +3278,7 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address) mmap_assert_write_locked(mm); - address &= PAGE_MASK; + address &= mm_page_mask(mm); if (address < mmap_min_addr || address < FIRST_USER_ADDRESS) return -EPERM; @@ -3272,20 +3313,22 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address) /* Somebody else might have raced and expanded it already */ if (address < vma->vm_start) { unsigned long size, grow; + struct vm_page_offset pos = vm_page_offset_sub( + vma_page_offset_at(vma, vma->vm_start), vma->vm_start - address); size = vma->vm_end - address; - grow = (vma->vm_start - address) >> PAGE_SHIFT; + grow = (vma->vm_start - address) >> mm_page_shift(mm); error = -ENOMEM; - if (grow <= vma->vm_pgoff) { + if (pos.index <= vma->vm_pgoff) { error = acct_stack_growth(vma, size, grow); if (!error) { if (vma_test(vma, VMA_LOCKED_BIT)) - mm->locked_vm += grow; + mm->locked_vm += mm_pages_to_account(mm, grow); vm_stat_account(mm, vma->vm_flags, grow); anon_vma_interval_tree_pre_update_vma(vma); vma->vm_start = address; - vma->vm_pgoff -= grow; + vma_set_page_offset(vma, pos); /* Overwrite old entry in mtree. */ vma_iter_store_overwrite(&vmi, vma); anon_vma_interval_tree_post_update_vma(vma); @@ -3324,13 +3367,13 @@ int __vm_munmap(unsigned long start, size_t len, bool unlock) */ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) { - unsigned long charged = vma_pages(vma); + unsigned long charged = vma_user_pages(vma); if (find_vma_intersection(mm, vma->vm_start, vma->vm_end)) return -ENOMEM; if (vma_test(vma, VMA_ACCOUNT_BIT) && - security_vm_enough_memory_mm(mm, charged)) + mm_account_memory(mm, mm, charged)) return -ENOMEM; /* @@ -3347,12 +3390,15 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) */ if (vma_is_anonymous(vma)) { BUG_ON(vma->anon_vma); - vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT; + vma_set_page_offset(vma, (struct vm_page_offset) { + .index = vma->vm_start >> PAGE_SHIFT, + .offset = vma->vm_start & ~PAGE_MASK, + }); } if (vma_link(mm, vma)) { if (vma_test(vma, VMA_ACCOUNT_BIT)) - vm_unacct_memory(charged); + mm_unacct_memory(mm, charged); return -ENOMEM; } diff --git a/mm/vma.h b/mm/vma.h index 3d2fa4f0d7779..9f9dac01dde4b 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -97,6 +97,9 @@ struct vma_merge_struct { unsigned long start; unsigned long end; pgoff_t pgoff; +#ifdef CONFIG_MM_SUBPAGE + unsigned int subpage_offset; +#endif union { /* Temporary while VMA flags are being converted. */ @@ -234,9 +237,34 @@ static inline bool vmg_nomem(struct vma_merge_struct *vmg) static inline pgoff_t vma_pgoff_offset(struct vm_area_struct *vma, unsigned long addr) { - return vma->vm_pgoff + PHYS_PFN(addr - vma->vm_start); + return vma_page_offset_at(vma, addr).index; +} + +static inline struct vm_page_offset vmg_page_offset(const struct vma_merge_struct *vmg) +{ + return (struct vm_page_offset) { + .index = vmg->pgoff, +#ifdef CONFIG_MM_SUBPAGE + .offset = vmg->subpage_offset, +#endif + }; +} + +static inline void vmg_set_page_offset(struct vma_merge_struct *vmg, struct vm_page_offset pos) +{ + vmg->pgoff = pos.index; +#ifdef CONFIG_MM_SUBPAGE + vmg->subpage_offset = pos.offset; +#endif } +#ifdef CONFIG_MM_SUBPAGE +#define VMG_SUBPAGE_STATE(vma, start) \ + .subpage_offset = vma_page_offset_at(vma, start).offset, +#else +#define VMG_SUBPAGE_STATE(vma, start) +#endif + #define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_) \ struct vma_merge_struct name = { \ .mm = mm_, \ @@ -259,6 +287,7 @@ static inline pgoff_t vma_pgoff_offset(struct vm_area_struct *vma, .end = end_, \ .vm_flags = vma_->vm_flags, \ .pgoff = vma_pgoff_offset(vma_, start_), \ + VMG_SUBPAGE_STATE(vma_, start_) \ .file = vma_->vm_file, \ .anon_vma = vma_->anon_vma, \ .policy = vma_policy(vma_), \ @@ -276,7 +305,7 @@ void validate_mm(struct mm_struct *mm); __must_check int vma_expand(struct vma_merge_struct *vmg); __must_check int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, - unsigned long start, unsigned long end, pgoff_t pgoff); + unsigned long start, unsigned long end, struct vm_page_offset pos); static inline int vma_iter_store_gfp(struct vma_iterator *vmi, struct vm_area_struct *vma, gfp_t gfp) @@ -311,6 +340,9 @@ static inline void compat_set_vma_from_desc(struct vm_area_struct *vma, /* Mutable fields. Populated with initial state. */ vma->vm_pgoff = desc->pgoff; +#ifdef CONFIG_MM_SUBPAGE + vma->vm_subpage_offset = desc->subpage_offset; +#endif if (desc->vm_file != vma->vm_file) vma_set_file(vma, desc->vm_file); vma->flags = desc->vma_flags; @@ -448,7 +480,7 @@ void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb, struct vm_area_struct *vma); struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, - unsigned long addr, unsigned long len, pgoff_t pgoff, + unsigned long addr, unsigned long len, struct vm_page_offset pos, bool *need_rmap_locks, bool keep_source); struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma); @@ -460,7 +492,7 @@ int mm_take_all_locks(struct mm_struct *mm); void mm_drop_all_locks(struct mm_struct *mm); unsigned long mmap_region(struct file *file, unsigned long addr, - unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, + unsigned long len, vm_flags_t vm_flags, struct vm_page_offset pos, struct list_head *uf); int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma, diff --git a/mm/vma_exec.c b/mm/vma_exec.c index 5cee8b7efa0f2..8394f45a91113 100644 --- a/mm/vma_exec.c +++ b/mm/vma_exec.c @@ -42,6 +42,8 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) struct mmu_gather tlb; PAGETABLE_MOVE(pmc, vma, vma, old_start, new_start, length); + vmg_set_page_offset(&vmg, vma_page_offset_at(vma, old_start)); + BUG_ON(new_start > new_end); /* @@ -89,7 +91,7 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) vma_prev(&vmi); /* Shrink the vma to just the new range */ - return vma_shrink(&vmi, vma, new_start, new_end, vma->vm_pgoff); + return vma_shrink(&vmi, vma, new_start, new_end, vma_page_offset_at(vma, vma->vm_start)); } /* @@ -138,7 +140,7 @@ int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap, */ VM_WARN_ON_ONCE(VM_STACK_FLAGS & VM_STACK_INCOMPLETE_SETUP); vma->vm_end = STACK_TOP_MAX; - vma->vm_start = vma->vm_end - PAGE_SIZE; + vma->vm_start = vma->vm_end - mm_page_size(mm); if (pgtable_supports_soft_dirty()) flags |= VM_SOFTDIRTY; vm_flags_init(vma, flags); diff --git a/mm/vma_init.c b/mm/vma_init.c index 3c0b65950510e..a41ba35de7320 100644 --- a/mm/vma_init.c +++ b/mm/vma_init.c @@ -46,7 +46,10 @@ static void vm_area_init_from(const struct vm_area_struct *src, dest->vm_start = src->vm_start; dest->vm_end = src->vm_end; dest->anon_vma = src->anon_vma; - dest->vm_pgoff = src->vm_pgoff; + vma_set_page_offset(dest, (struct vm_page_offset) { + .index = src->vm_pgoff, + .offset = vma_subpage_offset(src), + }); dest->vm_file = src->vm_file; dest->vm_private_data = src->vm_private_data; vm_flags_init(dest, src->vm_flags); diff --git a/mm/vmscan.c b/mm/vmscan.c index 8781dbcdb99a6..590c21b581903 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3454,6 +3454,46 @@ static void walk_update_folio(struct lru_gen_mm_walk *walk, struct folio *folio, } } +static unsigned long lru_pte_index(struct mm_struct *mm, unsigned long addr) +{ + if (mm_page_size(mm) == PAGE_SIZE) + return pte_index(addr); + return (addr >> mm_page_shift(mm)) & (pte_table_bytes_mm(mm) / sizeof(pte_t) - 1); +} + +static unsigned long lru_pmd_index(struct mm_struct *mm, unsigned long addr) +{ + if (mm_page_size(mm) == PAGE_SIZE) + return pmd_index(addr); + return (addr >> pmd_shift_mm(mm)) & (pmd_table_bytes_mm(mm) / sizeof(pmd_t) - 1); +} + +static unsigned long lru_pud_entries(struct mm_struct *mm) +{ + return mm_pud_folded(mm) ? 1 : pud_table_bytes_mm(mm) / sizeof(pud_t); +} + +static unsigned long lru_pud_index(struct mm_struct *mm, unsigned long addr) +{ + if (mm_pud_folded(mm)) + return 0; + if (mm_page_size(mm) == PAGE_SIZE) + return pud_index(addr); + return (addr >> pud_shift_mm(mm)) & (lru_pud_entries(mm) - 1); +} + +static unsigned int lru_folio_pte_batch(struct vm_area_struct *vma, + struct folio *folio, pte_t *ptep, pte_t *entry, unsigned int max_nr) +{ +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return folio_subpage_pte_batch_flags(vma, folio, ptep, entry, + max_nr, FPB_MERGE_YOUNG_DIRTY); +#endif + return folio_pte_batch_flags(folio, NULL, ptep, entry, max_nr, + FPB_MERGE_YOUNG_DIRTY); +} + static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, struct mm_walk *args) { @@ -3473,7 +3513,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, unsigned int nr; pmd_t pmdval; - pte = pte_offset_map_rw_nolock(args->mm, pmd, start & PMD_MASK, &pmdval, &ptl); + pte = pte_offset_map_rw_nolock(args->mm, pmd, start & pmd_mask_mm(args->mm), &pmdval, &ptl); if (!pte) return false; @@ -3489,7 +3529,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, lazy_mmu_mode_enable(); restart: - for (i = pte_index(start), addr = start; addr != end; i += nr, addr += nr * PAGE_SIZE) { + for (i = lru_pte_index(args->mm, start), addr = start; addr != end; i += nr, addr += nr * mm_page_size(args->mm)) { unsigned long pfn; struct folio *folio; pte_t *cur_pte = pte + i; @@ -3508,10 +3548,9 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, continue; if (folio_test_large(folio)) { - const unsigned int max_nr = (end - addr) >> PAGE_SHIFT; + const unsigned int max_nr = (end - addr) >> mm_page_shift(args->mm); - nr = folio_pte_batch_flags(folio, NULL, cur_pte, &ptent, - max_nr, FPB_MERGE_YOUNG_DIRTY); + nr = lru_folio_pte_batch(args->vma, folio, cur_pte, &ptent, max_nr); total += nr - 1; walk->mm_stats[MM_LEAF_TOTAL] += nr - 1; } @@ -3536,7 +3575,8 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, walk_update_folio(walk, last, gen, dirty); last = NULL; - if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) + if (i < pte_table_bytes_mm(args->mm) / sizeof(pte_t) && + get_next_vma(pmd_mask_mm(args->mm), mm_page_size(args->mm), args, &start, &end)) goto restart; lazy_mmu_mode_disable(); @@ -3568,13 +3608,13 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area return; } - i = addr == -1 ? 0 : pmd_index(addr) - pmd_index(*first); + i = addr == -1 ? 0 : lru_pmd_index(args->mm, addr) - lru_pmd_index(args->mm, *first); if (i && i <= MIN_LRU_BATCH) { __set_bit(i - 1, bitmap); return; } - pmd = pmd_offset(pud, *first); + pmd = pmd_offset_mm(args->mm, pud, *first); ptl = pmd_lockptr(args->mm, pmd); if (!spin_trylock(ptl)) @@ -3587,7 +3627,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area struct folio *folio; /* don't round down the first address */ - addr = i ? (*first & PMD_MASK) + i * PMD_SIZE : *first; + addr = i ? (*first & pmd_mask_mm(args->mm)) + i * pmd_size_mm(args->mm) : *first; if (!pmd_present(pmd[i])) goto next; @@ -3653,14 +3693,14 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, * tables to avoid taking the PMD lock; the second, if necessary, takes * the PMD lock to clear the accessed bit in PMD entries. */ - pmd = pmd_offset(pud, start & PUD_MASK); + pmd = pmd_offset_mm(args->mm, pud, start & pud_mask_mm(args->mm)); restart: /* walk_pte_range() may call get_next_vma() */ vma = args->vma; - for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) { + for (i = lru_pmd_index(args->mm, start), addr = start; addr != end; i++, addr = next) { pmd_t val = pmdp_get_lockless(pmd + i); - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(args->mm, addr, end); if (!pmd_present(val) || is_huge_zero_pmd(val)) { walk->mm_stats[MM_LEAF_TOTAL]++; @@ -3702,7 +3742,8 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first); - if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end)) + if (i < pmd_table_bytes_mm(args->mm) / sizeof(pmd_t) && + get_next_vma(pud_mask_mm(args->mm), pmd_size_mm(args->mm), args, &start, &end)) goto restart; } @@ -3717,12 +3758,12 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, VM_WARN_ON_ONCE(p4d_leaf(*p4d)); - pud = pud_offset(p4d, start & P4D_MASK); + pud = pud_offset_mm(args->mm, p4d, start & p4d_mask_mm(args->mm)); restart: - for (i = pud_index(start), addr = start; addr != end; i++, addr = next) { + for (i = lru_pud_index(args->mm, start), addr = start; addr != end; i++, addr = next) { pud_t val = pudp_get(pud + i); - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(args->mm, addr, end); if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) continue; @@ -3730,15 +3771,16 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, walk_pmd_range(&val, addr, next, args); if (need_resched() || walk->batched >= MAX_LRU_BATCH) { - end = (addr | ~PUD_MASK) + 1; + end = (addr | ~pud_mask_mm(args->mm)) + 1; goto done; } } - if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end)) + if (i < lru_pud_entries(args->mm) && + get_next_vma(p4d_mask_mm(args->mm), pud_size_mm(args->mm), args, &start, &end)) goto restart; - end = round_up(end, P4D_SIZE); + end = round_up(end, p4d_size_mm(args->mm)); done: if (!end || !args->vma) return 1; @@ -4208,20 +4250,20 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) /* avoid taking the LRU lock under the PTL when possible */ walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; - start = max(addr & PMD_MASK, vma->vm_start); - end = min(addr | ~PMD_MASK, vma->vm_end - 1) + 1; + start = max(addr & pmd_mask_mm(vma->vm_mm), vma->vm_start); + end = min(addr | ~pmd_mask_mm(vma->vm_mm), vma->vm_end - 1) + 1; - if (end - start == PAGE_SIZE) + if (end - start == mm_page_size(vma->vm_mm)) return true; - if (end - start > MIN_LRU_BATCH * PAGE_SIZE) { - if (addr - start < MIN_LRU_BATCH * PAGE_SIZE / 2) - end = start + MIN_LRU_BATCH * PAGE_SIZE; - else if (end - addr < MIN_LRU_BATCH * PAGE_SIZE / 2) - start = end - MIN_LRU_BATCH * PAGE_SIZE; + if (end - start > MIN_LRU_BATCH * mm_page_size(vma->vm_mm)) { + if (addr - start < MIN_LRU_BATCH * mm_page_size(vma->vm_mm) / 2) + end = start + MIN_LRU_BATCH * mm_page_size(vma->vm_mm); + else if (end - addr < MIN_LRU_BATCH * mm_page_size(vma->vm_mm) / 2) + start = end - MIN_LRU_BATCH * mm_page_size(vma->vm_mm); else { - start = addr - MIN_LRU_BATCH * PAGE_SIZE / 2; - end = addr + MIN_LRU_BATCH * PAGE_SIZE / 2; + start = addr - MIN_LRU_BATCH * mm_page_size(vma->vm_mm) / 2; + end = addr + MIN_LRU_BATCH * mm_page_size(vma->vm_mm) / 2; } } @@ -4233,10 +4275,10 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) lazy_mmu_mode_enable(); - pte -= (addr - start) / PAGE_SIZE; + pte -= (addr - start) / mm_page_size(vma->vm_mm); for (i = 0, addr = start; addr != end; - i += nr, pte += nr, addr += nr * PAGE_SIZE) { + i += nr, pte += nr, addr += nr * mm_page_size(vma->vm_mm)) { unsigned long pfn; pte_t ptent = ptep_get(pte); @@ -4250,10 +4292,9 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) continue; if (folio_test_large(folio)) { - const unsigned int max_nr = (end - addr) >> PAGE_SHIFT; + const unsigned int max_nr = (end - addr) >> mm_page_shift(vma->vm_mm); - nr = folio_pte_batch_flags(folio, NULL, pte, &ptent, - max_nr, FPB_MERGE_YOUNG_DIRTY); + nr = lru_folio_pte_batch(vma, folio, pte, &ptent, max_nr); } if (!test_and_clear_young_ptes_notify(vma, addr, pte, nr)) From c325fe9a2ab4f757ccdd5752b82af2c4305cc33a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0682/2680] mm: add retained subpage backing ownership Implement reference-counted user slots within native folios, including zeroing, mapping and pin ownership, migration and deferred lifetime. The backend remains disabled until its architecture integration. Signed-off-by: Diljit Singh (cherry picked from commit c1a5f4938d74e70de292adee44236d69f7b14a2b) --- mm/Makefile | 1 + mm/page_ext.c | 4 + mm/user-subpage.c | 1673 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1678 insertions(+) create mode 100644 mm/user-subpage.c diff --git a/mm/Makefile b/mm/Makefile index eff9f9e7e061c..5845922bf5f37 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -147,3 +147,4 @@ obj-$(CONFIG_SHRINKER_DEBUG) += shrinker_debug.o obj-$(CONFIG_EXECMEM) += execmem.o obj-$(CONFIG_TMPFS_QUOTA) += shmem_quota.o obj-$(CONFIG_LAZY_MMU_MODE_KUNIT_TEST) += tests/lazy_mmu_mode_kunit.o +obj-$(CONFIG_MM_SUBPAGE) += user-subpage.o diff --git a/mm/page_ext.c b/mm/page_ext.c index e2e92bd27ebdd..a73aab719c040 100644 --- a/mm/page_ext.c +++ b/mm/page_ext.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #include #include @@ -78,6 +79,9 @@ static struct page_ext_operations page_idle_ops __initdata = { #endif static struct page_ext_operations *page_ext_ops[] __initdata = { +#ifdef CONFIG_MM_SUBPAGE + &mm_subpage_ext_ops, +#endif #ifdef CONFIG_PAGE_OWNER &page_owner_ops, #endif diff --git a/mm/user-subpage.c b/mm/user-subpage.c new file mode 100644 index 0000000000000..133d75241485a --- /dev/null +++ b/mm/user-subpage.c @@ -0,0 +1,1673 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Reference-counted user data slots in native folios. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "user-subpage-internal.h" +#include "internal.h" + +#define SUBPAGES_PER_FOLIO (PAGE_SIZE / MM_SUBPAGE_SIZE) +static bool subpage_shift_valid(unsigned int shift) +{ + return shift >= MM_SUBPAGE_SHIFT && shift < PAGE_SHIFT; +} + +static unsigned long subpage_mask(unsigned int shift) +{ + return GENMASK((PAGE_SIZE >> shift) - 1, 0); +} + +static unsigned long offset_mask(unsigned int shift, unsigned int offset) +{ + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, 1U << shift)) + return 0; + return BIT(offset >> shift); +} + +struct mm_subpage_owner { + struct rcu_head rcu; + struct work_struct free_work; + struct mm_subpage_pool *pool; + struct folio *folio; + struct anon_vma *anon_root; + struct mm_subpage_owner *migration_target; + /* Allocated only when slots need independent logical identities. */ + struct mm_subpage_rmap *rmap; + struct list_head all; + struct list_head available; + unsigned long used; + unsigned long zeroed; + u8 shift; + bool detached; + bool migrating; + bool droppable; + struct mm_subpage slots[SUBPAGES_PER_FOLIO]; +}; + +struct mm_subpage_ext { + struct mm_subpage_owner __rcu *owner; + atomic_t file_maps[SUBPAGES_PER_FOLIO]; + atomic_t file_leaves; + /* Folio lock protects validity; fits the 16K/64K layout padding. */ + u16 shmem_missing; +}; +static_assert(SUBPAGES_PER_FOLIO == 1 || sizeof(struct mm_subpage_ext) == + ALIGN(sizeof(void *) + sizeof(atomic_t) * (SUBPAGES_PER_FOLIO + 1), + sizeof(void *))); +static_assert(SUBPAGES_PER_FOLIO <= sizeof(u16) * BITS_PER_BYTE); + +unsigned long mm_subpage_shmem_missing(struct page *page) +{ + struct page_ext *ext = page_ext_get(page); + struct mm_subpage_ext *subext; + unsigned long missing; + + /* A referenced native page keeps its initialized section online. */ + VM_BUG_ON(!ext); + subext = page_ext_data(ext, &mm_subpage_ext_ops); + missing = subext->shmem_missing; + page_ext_put(ext); + return missing; +} + +void mm_subpage_shmem_set_missing(struct page *page, unsigned long missing) +{ + struct page_ext *ext = page_ext_get(page); + struct mm_subpage_ext *subext; + + VM_BUG_ON(!ext); + VM_BUG_ON(missing & ~GENMASK(SUBPAGES_PER_FOLIO - 1, 0)); + subext = page_ext_data(ext, &mm_subpage_ext_ops); + subext->shmem_missing = missing; + page_ext_put(ext); +} + +static bool __init subpage_ext_needed(void) +{ + return true; +} + +struct page_ext_operations mm_subpage_ext_ops = { + .size = sizeof(struct mm_subpage_ext), + .need = subpage_ext_needed, +}; + +/* Only alternative PTE mappings contribute; native rmap counts stay unchanged. */ +static void subpage_file_map_update(phys_addr_t phys, unsigned long size, int delta) +{ + struct page_ext *ext; + struct mm_subpage_ext *subext; + unsigned int index, nr, i; + int count; + + VM_BUG_ON(!is_power_of_2(size) || size < MM_SUBPAGE_SIZE || + size >= PAGE_SIZE || !IS_ALIGNED(phys, size)); + ext = page_ext_from_phys(phys); + if (WARN_ON_ONCE(!ext)) + return; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + index = offset_in_page(phys) >> MM_SUBPAGE_SHIFT; + nr = size >> MM_SUBPAGE_SHIFT; + for (i = index; i < index + nr; i++) { + count = atomic_add_return(delta, &subext->file_maps[i]); + VM_WARN_ON(count < 0); + } + /* Rmap counts one mapping, regardless of its coverage in 4K units. */ + count = atomic_add_return(delta, &subext->file_leaves); + VM_WARN_ON(count < 0); + page_ext_put(ext); +} + +void __mm_subpage_file_map_add(phys_addr_t phys, unsigned long size) +{ + subpage_file_map_update(phys, size, 1); +} + +void __mm_subpage_file_map_del(phys_addr_t phys, unsigned long size) +{ + subpage_file_map_update(phys, size, -1); +} + +/* Snapshot for statistics only, never an ownership or pinning decision. */ +unsigned int mm_subpage_file_mapcounts(struct page *page, struct mm_subpage_mapcounts *counts) +{ + struct page_ext *ext = page_ext_get(page); + struct mm_subpage_ext *subext; + unsigned int i, leaves; + + if (WARN_ON_ONCE(!ext)) { + memset(counts, 0, sizeof(*counts)); + return 0; + } + subext = page_ext_data(ext, &mm_subpage_ext_ops); + for (i = 0; i < SUBPAGES_PER_FOLIO; i++) + counts->slots[i] = max(atomic_read(&subext->file_maps[i]), 0); + leaves = max(atomic_read(&subext->file_leaves), 0); + page_ext_put(ext); + return leaves; +} + +static int owner_bind(struct mm_subpage_owner *owner) +{ + struct page_ext *ext = page_ext_get(&owner->folio->page); + struct mm_subpage_ext *subext; + int ret; + + if (!ext) + return -EOPNOTSUPP; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + ret = cmpxchg(&subext->owner, NULL, owner) ? -EEXIST : 0; + page_ext_put(ext); + return ret; +} + +static void owner_unbind(struct mm_subpage_owner *owner) +{ + struct page_ext *ext = page_ext_get(&owner->folio->page); + struct mm_subpage_ext *subext; + + /* The allocated, referenced folio prevents its section going offline. */ + VM_BUG_ON(!ext); + subext = page_ext_data(ext, &mm_subpage_ext_ops); + VM_BUG_ON(rcu_access_pointer(subext->owner) != owner); + rcu_assign_pointer(subext->owner, NULL); + page_ext_put(ext); +} + +struct mm_subpage *mm_subpage_get_from_phys(phys_addr_t phys) +{ + struct mm_subpage_owner *owner; + struct mm_subpage_ext *subext; + struct mm_subpage *slot = NULL; + struct page_ext *ext; + unsigned int index; + + if (!IS_ALIGNED(phys, MM_SUBPAGE_SIZE)) + return NULL; + ext = page_ext_from_phys(phys); + if (!ext) + return NULL; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + index = owner ? offset_in_page(phys) >> owner->shift : 0; + /* Keep the native allocation alive before publishing another slot ref. */ + if (owner && folio_try_get(owner->folio)) { + if (refcount_inc_not_zero_acquire(&owner->slots[index].refs)) + slot = &owner->slots[index]; + else + folio_put(owner->folio); + } + page_ext_put(ext); + return slot; +} + +struct mm_subpage_pool { + struct rcu_head rcu; + spinlock_t lock; + refcount_t refs; + struct list_head owners; + struct list_head available; + struct mm_subpage_owner *unissued; + unsigned long backing_pages; + unsigned long all_slots; + u8 shift; + bool closed; +}; + +static void owner_init(struct mm_subpage_owner *owner, struct mm_subpage_pool *pool, + struct folio *folio) +{ + unsigned int i; + + owner->pool = pool; + owner->shift = pool->shift; + owner->folio = folio; + INIT_LIST_HEAD(&owner->available); + for (i = 0; i < (PAGE_SIZE >> pool->shift); i++) { + owner->slots[i].owner = owner; + owner->slots[i].index = i; + owner->slots[i].shift = pool->shift; + refcount_set(&owner->slots[i].refs, 0); + atomic_set(&owner->slots[i].mapcount, 0); + } +} + +struct mm_subpage_pool *mm_subpage_pool_create_granule(gfp_t gfp, unsigned int shift) +{ + struct mm_subpage_pool *pool; + + BUILD_BUG_ON(PAGE_SHIFT <= MM_SUBPAGE_SHIFT); + BUILD_BUG_ON(SUBPAGES_PER_FOLIO > BITS_PER_LONG); + if (!subpage_shift_valid(shift)) + return NULL; + pool = kzalloc(sizeof(*pool), gfp | __GFP_ACCOUNT); + if (!pool) + return NULL; + pool->shift = shift; + pool->all_slots = subpage_mask(shift); + spin_lock_init(&pool->lock); + refcount_set(&pool->refs, 1); + INIT_LIST_HEAD(&pool->owners); + INIT_LIST_HEAD(&pool->available); + return pool; +} + +struct mm_subpage_pool *mm_subpage_pool_create(gfp_t gfp) +{ + return mm_subpage_pool_create_granule(gfp, MM_SUBPAGE_SHIFT); +} + +void mm_subpage_pool_get(struct mm_subpage_pool *pool) +{ + refcount_inc(&pool->refs); +} + +void mm_subpage_pool_put(struct mm_subpage_pool *pool) +{ + if (!refcount_dec_and_test(&pool->refs)) + return; + WARN_ON_ONCE(!pool->closed || !list_empty(&pool->owners)); + kfree_rcu(pool, rcu); +} + +static void owner_remove(struct mm_subpage_owner *owner) +{ + lockdep_assert_held(&owner->pool->lock); + owner->detached = true; + list_del(&owner->all); + list_del_init(&owner->available); + if (owner->pool->unissued == owner) + owner->pool->unissued = NULL; + owner->pool->backing_pages--; +} + +static void owner_free_rmap(struct mm_subpage_owner *owner) +{ + unsigned int i; + + if (!owner->rmap) + return; + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) + if (owner->rmap[i].root) + put_anon_vma(owner->rmap[i].root); + kfree(owner->rmap); +} + +static void owner_free_metadata(struct mm_subpage_owner *owner) +{ + struct mm_subpage_pool *pool = owner->pool; + + if (owner->anon_root) + put_anon_vma(owner->anon_root); + owner_free_rmap(owner); + kfree_rcu(owner, rcu); + mm_subpage_pool_put(pool); +} + +static void owner_free_work(struct work_struct *work) +{ + owner_free_metadata(container_of(work, struct mm_subpage_owner, free_work)); +} + +static void owner_free(struct mm_subpage_owner *owner, bool put_folio) +{ + owner_unbind(owner); + if (put_folio) { + if (owner->anon_root) + free_swap_cache(owner->folio); + /* Last slot reference, or wholly unissued supply. */ + folio_put(owner->folio); + } + if (owner->rmap) { + /* Historical roots can have no live VMA; their final put may sleep. */ + INIT_WORK(&owner->free_work, owner_free_work); + queue_work(system_unbound_wq, &owner->free_work); + } else { + owner_free_metadata(owner); + } +} + +struct anon_vma *mm_subpage_anon_root(const struct mm_subpage *subpage) +{ + VM_BUG_ON_FOLIO(!folio_test_locked(subpage->owner->folio), subpage->owner->folio); + return subpage->owner->anon_root; +} + +void mm_subpage_hold_anon_root(struct mm_subpage *subpage, struct anon_vma *root, + bool droppable) +{ + struct mm_subpage_owner *owner = subpage->owner; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + VM_BUG_ON(owner->anon_root || root != root->root); + get_anon_vma(root); + owner->droppable = droppable; + owner->anon_root = root; +} + +static struct mm_subpage_rmap slot_rmap(struct mm_subpage *slot) +{ + struct mm_subpage_owner *owner = slot->owner; + + lockdep_assert_held(&owner->pool->lock); + if (owner->rmap) + return owner->rmap[slot->index]; + return (struct mm_subpage_rmap) { + .root = owner->anon_root, + .pos = { .index = owner->folio->index, + .offset = mm_subpage_offset(slot) }, + .offset = mm_subpage_offset(slot), + }; +} + +static bool rmap_matches(struct mm_subpage_rmap map, struct vm_area_struct *vma, + unsigned long address) +{ + return map.root == vma->anon_vma->root && + vm_page_offset_equal(map.pos, vma_page_offset_at(vma, address)); +} + +bool mm_subpage_match_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, + unsigned long address) +{ + struct mm_subpage_pool *pool = slot->owner->pool; + unsigned long flags; + bool matches; + + spin_lock_irqsave(&pool->lock, flags); + matches = slot->rmap_bound && rmap_matches(slot_rmap(slot), vma, address); + spin_unlock_irqrestore(&pool->lock, flags); + return matches; +} + +static void *owner_kcalloc(struct mm_subpage_owner *owner, size_t nr, size_t size) +{ + struct mem_cgroup *memcg, *saved; + void *maps; + + rcu_read_lock(); + do { + memcg = mem_cgroup_from_virt(owner) ?: root_mem_cgroup; + } while (memcg && !mem_cgroup_tryget(memcg)); + rcu_read_unlock(); + saved = set_active_memcg(memcg); + maps = kcalloc(nr, size, GFP_NOWAIT | __GFP_NOWARN | __GFP_ACCOUNT); + set_active_memcg(saved); + mem_cgroup_put(memcg); + return maps; +} + +struct subpage_retired_root { + struct work_struct work; + struct anon_vma *root; +}; + +static void subpage_put_retired_root(struct work_struct *work) +{ + struct subpage_retired_root *retired = + container_of(work, struct subpage_retired_root, work); + + put_anon_vma(retired->root); + kfree(retired); +} + +/* The folio lock serializes binding and sidecar creation against migration. */ +int mm_subpage_bind_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, + unsigned long address) +{ + struct mm_subpage_owner *owner = slot->owner; + struct mm_subpage_pool *pool = owner->pool; + struct mm_subpage_rmap *maps = NULL; + struct subpage_retired_root *retired = NULL; + unsigned long flags; + unsigned int i; + int ret = 0; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + spin_lock_irqsave(&pool->lock, flags); + if (slot->rmap_bound) { + ret = rmap_matches(slot_rmap(slot), vma, address) ? 0 : -EINVAL; + goto unlock; + } + if (!owner->rmap && !rmap_matches(slot_rmap(slot), vma, address)) { + spin_unlock_irqrestore(&pool->lock, flags); + /* The caller may also hold a PTE lock. Never enter reclaim here. */ + maps = owner_kcalloc(owner, PAGE_SIZE >> owner->shift, sizeof(*maps)); + if (!maps) + return -ENOMEM; + spin_lock_irqsave(&pool->lock, flags); + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) { + maps[i].offset = i << owner->shift; + if (!owner->slots[i].rmap_bound) + continue; + maps[i] = slot_rmap(&owner->slots[i]); + get_anon_vma(maps[i].root); + } + owner->rmap = maps; + } + if (owner->rmap) { + struct mm_subpage_rmap *map = &owner->rmap[slot->index]; + + if (map->root && map->root != vma->anon_vma->root) { + /* A final anon_vma put may sleep; the caller can hold a PTL. */ + spin_unlock_irqrestore(&pool->lock, flags); + retired = owner_kcalloc(owner, 1, sizeof(*retired)); + if (!retired) + return -ENOMEM; + INIT_WORK(&retired->work, subpage_put_retired_root); + spin_lock_irqsave(&pool->lock, flags); + retired->root = map->root; + } + if (map->root != vma->anon_vma->root) { + get_anon_vma(vma->anon_vma->root); + map->root = vma->anon_vma->root; + } + map->pos = vma_page_offset_at(vma, address); + map->offset = mm_subpage_offset(slot); + } + slot->rmap_bound = true; +unlock: + spin_unlock_irqrestore(&pool->lock, flags); + if (retired) + queue_work(system_unbound_wq, &retired->work); + return ret; +} + +static bool subpage_move_allowed(struct mm_subpage *slot, bool swap) +{ + lockdep_assert_held(&slot->owner->pool->lock); + return !slot->owner->migrating && !slot->pins && + atomic_read(&slot->mapcount) == (swap ? 0 : 1) && + (swap || (slot->rmap_bound && slot->exclusive)); +} + +int mm_subpage_move_prepare(struct mm_subpage_move *move, struct mm_subpage *slot, + struct vm_area_struct *src, unsigned long src_address, + struct vm_area_struct *dst, unsigned long dst_address, bool swap) +{ + struct mm_subpage_owner *owner = slot->owner; + struct mm_subpage_rmap *maps, old; + unsigned long flags; + unsigned int i; + int ret = -EBUSY; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + memset(move, 0, sizeof(*move)); + if (!src->anon_vma || !dst->anon_vma || !owner->anon_root || + mm_page_shift(src->vm_mm) != slot->shift || + mm_page_shift(dst->vm_mm) != slot->shift || + owner->droppable != !!(dst->vm_flags & VM_DROPPABLE)) + return -EINVAL; + spin_lock_irqsave(&owner->pool->lock, flags); + if (!subpage_move_allowed(slot, swap) || + (slot->rmap_bound && !rmap_matches(slot_rmap(slot), src, src_address))) + goto unlock; + spin_unlock_irqrestore(&owner->pool->lock, flags); + + /* Allocate before touching either PTE. The folio lock serializes writers. */ + if (!owner->rmap) { + maps = owner_kcalloc(owner, PAGE_SIZE >> owner->shift, sizeof(*maps)); + if (!maps) + return -ENOMEM; + spin_lock_irqsave(&owner->pool->lock, flags); + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) { + maps[i].offset = i << owner->shift; + if (!owner->slots[i].rmap_bound) + continue; + maps[i] = slot_rmap(&owner->slots[i]); + get_anon_vma(maps[i].root); + } + owner->rmap = maps; + spin_unlock_irqrestore(&owner->pool->lock, flags); + } + old = owner->rmap[slot->index]; + move->root_changed = old.root != dst->anon_vma->root; + if (move->root_changed && old.root) { + move->retired = owner_kcalloc(owner, 1, sizeof(*move->retired)); + if (!move->retired) + return -ENOMEM; + INIT_WORK(&move->retired->work, subpage_put_retired_root); + move->retired->root = old.root; + } + if (move->root_changed) + get_anon_vma(dst->anon_vma->root); + move->slot = slot; + move->swap = swap; + move->map = (struct mm_subpage_rmap) { + .root = dst->anon_vma->root, + .pos = vma_page_offset_at(dst, dst_address), + .offset = mm_subpage_offset(slot), + }; + return 0; +unlock: + spin_unlock_irqrestore(&owner->pool->lock, flags); + return ret; +} + +int mm_subpage_move_commit(struct mm_subpage_move *move) +{ + struct mm_subpage *slot = move->slot; + struct mm_subpage_owner *owner = slot->owner; + unsigned long flags; + int ret = -EBUSY; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + spin_lock_irqsave(&owner->pool->lock, flags); + /* Recheck pins after source-PTE clearing and the matching TLB barrier. */ + if (subpage_move_allowed(slot, move->swap)) { + owner->rmap[slot->index] = move->map; + slot->rmap_bound = true; + move->committed = true; + ret = 0; + } + spin_unlock_irqrestore(&owner->pool->lock, flags); + return ret; +} + +void mm_subpage_move_finish(struct mm_subpage_move *move) +{ + if (move->committed) { + if (move->retired) + queue_work(system_unbound_wq, &move->retired->work); + } else { + /* The locked destination VMA still holds this root. */ + if (move->root_changed) + put_anon_vma(move->map.root); + kfree(move->retired); + } +} + +int mm_subpage_snapshot_rmap(struct folio *folio, struct mm_subpage_rmap *maps) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_ext *subext; + struct mm_subpage_owner *owner; + unsigned long flags; + unsigned int i; + int nr = -1; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!ext) + return nr; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) + goto out; + spin_lock_irqsave(&owner->pool->lock, flags); + if (!owner->detached && owner->rmap) { + nr = 0; + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) { + if (!owner->slots[i].rmap_bound) + continue; + maps[nr] = owner->rmap[i]; + get_anon_vma(maps[nr].root); + nr++; + } + } + spin_unlock_irqrestore(&owner->pool->lock, flags); +out: + page_ext_put(ext); + return nr; +} + +/* A referenced folio plus a stable VMA suffices; no folio lock required. */ +bool mm_subpage_address_in_vma(const struct folio *folio, + const struct vm_area_struct *vma, unsigned long *address) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_ext *subext; + struct mm_subpage_owner *owner; + unsigned long flags; + unsigned int i; + bool nonlinear = false; + + if (!ext) + return false; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) + goto out; + spin_lock_irqsave(&owner->pool->lock, flags); + if (!owner->detached && owner->rmap) { + nonlinear = true; + *address = -EFAULT; + for (i = 0; vma->anon_vma && i < (PAGE_SIZE >> owner->shift); i++) { + if (!owner->slots[i].rmap_bound || + owner->rmap[i].root != vma->anon_vma->root) + continue; + *address = vma_address_at_offset(vma, owner->rmap[i].pos); + if (*address != -EFAULT) + break; + } + } + spin_unlock_irqrestore(&owner->pool->lock, flags); +out: + page_ext_put(ext); + return nonlinear; +} + +bool mm_subpage_is_droppable(const struct mm_subpage *subpage) +{ + return subpage->owner->droppable; +} + +void mm_subpage_pool_close(struct mm_subpage_pool *pool) +{ + struct mm_subpage_owner *owner; + unsigned long flags; + + spin_lock_irqsave(&pool->lock, flags); + pool->closed = true; + /* At most one newly supplied owner has no issued slots. Close is O(1). */ + owner = pool->unissued; + if (owner) + owner_remove(owner); + spin_unlock_irqrestore(&pool->lock, flags); + if (owner) + owner_free(owner, true); +} + +static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, + unsigned long eligible) +{ + struct mm_subpage_owner *owner; + + lockdep_assert_held(&pool->lock); + list_for_each_entry(owner, &pool->available, available) + /* A nonpresent PTE may still need any of this folio's slot bytes. */ + if (!owner->migrating && (~owner->used & eligible) && + !folio_test_swapcache(owner->folio)) + return owner; + return NULL; +} + +static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, + unsigned long eligible, gfp_t gfp) +{ + struct mm_subpage_owner *owner; + unsigned long flags; + int ret = 0; + + /* Only an unexposed order-0 allocation can become a slot owner. */ + if (folio_order(folio) || folio_ref_count(folio) != 1 || + folio->mapping || folio_mapped(folio) || folio_test_lru(folio) || + folio_test_private(folio) || folio_test_swapcache(folio)) + return -EINVAL; + owner = kzalloc(sizeof(*owner), gfp | __GFP_ACCOUNT); + if (!owner) + return -ENOMEM; + owner_init(owner, pool, folio); + /* Reject duplicate ownership before touching any existing data. */ + ret = owner_bind(owner); + if (ret) { + kfree(owner); + return ret; + } + /* Initialise even unissued bytes before any future whole-folio operation. */ + clear_user_subpage_range(&folio->page, 0, PAGE_SIZE); + __folio_mark_uptodate(folio); + owner->zeroed = pool->all_slots; + spin_lock_irqsave(&pool->lock, flags); + if (pool->closed) + ret = -ESHUTDOWN; + else if (find_available(pool, eligible)) + /* Another allocator supplied backing while we allocated metadata. */ + ret = -EEXIST; + else { + mm_subpage_pool_get(pool); + list_add_tail(&owner->all, &pool->owners); + list_add_tail(&owner->available, &pool->available); + pool->backing_pages++; + pool->unissued = owner; + } + spin_unlock_irqrestore(&pool->lock, flags); + if (ret) { + owner_unbind(owner); + kfree_rcu(owner, rcu); + } + return ret; +} + +int mm_subpage_pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, + gfp_t gfp) +{ + return pool_add_folio(pool, folio, pool->all_slots, gfp); +} + +int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *folio, + unsigned int offset, gfp_t gfp) +{ + unsigned long eligible = offset_mask(pool->shift, offset); + + if (!eligible) + return -EINVAL; + return pool_add_folio(pool, folio, eligible, gfp); +} + +unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool) +{ + unsigned long pages, flags; + + spin_lock_irqsave(&pool->lock, flags); + pages = pool->backing_pages; + spin_unlock_irqrestore(&pool->lock, flags); + return pages; +} + +static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, + unsigned long eligible, bool *needs_zero) +{ + struct mm_subpage_owner *owner; + struct mm_subpage *subpage; + unsigned long flags; + unsigned int index; + + spin_lock_irqsave(&pool->lock, flags); + if (pool->closed) { + subpage = ERR_PTR(-ESHUTDOWN); + goto out; + } + owner = find_available(pool, eligible); + if (!owner) { + subpage = ERR_PTR(-EAGAIN); + goto out; + } + if (pool->unissued == owner) + /* Transfer the supplied reference to the first reservation. */ + pool->unissued = NULL; + else + folio_get(owner->folio); + index = __ffs(~owner->used & eligible); + __set_bit(index, &owner->used); + if (needs_zero) + *needs_zero = !test_bit(index, &owner->zeroed); + __clear_bit(index, &owner->zeroed); + if (owner->used == pool->all_slots) + list_del_init(&owner->available); + subpage = &owner->slots[index]; + WRITE_ONCE(subpage->lazyfree, false); + /* The used bit keeps backing alive while data is initialised. */ + VM_BUG_ON(refcount_read(&subpage->refs)); + VM_BUG_ON(atomic_read(&subpage->mapcount)); +out: + spin_unlock_irqrestore(&pool->lock, flags); + return subpage; +} + +static struct mm_subpage *subpage_alloc(struct mm_subpage_pool *pool, unsigned long eligible) +{ + bool needs_zero; + struct mm_subpage *subpage = subpage_reserve(pool, eligible, &needs_zero); + + if (IS_ERR(subpage)) + return subpage; + /* Fresh supply was already zeroed as a whole; reused slots must be cleared. */ + if (needs_zero) + clear_user_subpage_range(&subpage->owner->folio->page, + mm_subpage_offset(subpage), mm_subpage_size(subpage)); + /* Data stores must precede publication in a PTE or another CPU's reference. */ + refcount_set_release(&subpage->refs, 1); + return subpage; +} + +struct mm_subpage *mm_subpage_alloc(struct mm_subpage_pool *pool) +{ + return subpage_alloc(pool, pool->all_slots); +} + +struct mm_subpage *mm_subpage_alloc_at(struct mm_subpage_pool *pool, unsigned int offset) +{ + unsigned long eligible = offset_mask(pool->shift, offset); + + if (!eligible) + return ERR_PTR(-EINVAL); + return subpage_alloc(pool, eligible); +} + +static void subpage_release_locked(struct mm_subpage *subpage, unsigned long flags) +{ + struct mm_subpage_owner *owner = subpage->owner; + struct mm_subpage_pool *pool = owner->pool; + struct folio *folio = owner->folio; + bool release; + + lockdep_assert_held(&pool->lock); + VM_BUG_ON(refcount_read(&subpage->refs)); + VM_BUG_ON(atomic_read(&subpage->mapcount)); + VM_BUG_ON(subpage->pins); + WRITE_ONCE(subpage->exclusive, false); + if (!owner->migrating) { + WRITE_ONCE(subpage->lazyfree, false); + subpage->rmap_bound = false; + } + __clear_bit(subpage->index, &owner->used); + release = !owner->used && !owner->migrating; + if (release) + owner_remove(owner); + else if (!pool->closed && !owner->migrating && list_empty(&owner->available)) + list_add(&owner->available, &pool->available); + spin_unlock_irqrestore(&pool->lock, flags); + if (release) + owner_free(owner, true); + else + folio_put(folio); +} + +static void subpage_release(struct mm_subpage *subpage) +{ + unsigned long flags; + + spin_lock_irqsave(&subpage->owner->pool->lock, flags); + subpage_release_locked(subpage, flags); +} + +static struct mm_subpage *subpage_copy(struct mm_subpage_pool *pool, + const struct mm_subpage *source, unsigned long eligible) +{ + struct mm_subpage *dest; + int err; + + if (pool->shift != mm_subpage_shift(source)) + return ERR_PTR(-EINVAL); + dest = subpage_reserve(pool, eligible, NULL); + + if (IS_ERR(dest)) + return dest; + err = copy_user_subpage_range(&dest->owner->folio->page, mm_subpage_offset(dest), + &source->owner->folio->page, mm_subpage_offset(source), + mm_subpage_size(source)); + if (err) { + /* Never publish a reservation whose copy did not finish. */ + subpage_release(dest); + return ERR_PTR(err); + } + refcount_set_release(&dest->refs, 1); + return dest; +} + +struct mm_subpage *mm_subpage_copy(struct mm_subpage_pool *pool, + const struct mm_subpage *source) +{ + return subpage_copy(pool, source, pool->all_slots); +} + +struct mm_subpage *mm_subpage_copy_at(struct mm_subpage_pool *pool, + const struct mm_subpage *source, unsigned int offset) +{ + unsigned long eligible = offset_mask(pool->shift, offset); + + if (!eligible) + return ERR_PTR(-EINVAL); + return subpage_copy(pool, source, eligible); +} + +/* The page-extension RCU section also protects the pool until its lock is held. */ +static struct mm_subpage *restore_bound_slot(struct folio *folio, unsigned int offset, + unsigned int shift) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_owner *owner; + struct mm_subpage_pool *pool; + struct mm_subpage_ext *subext; + struct mm_subpage *slot = NULL; + unsigned int index; + unsigned long flags; + + if (!ext) + return ERR_PTR(-EOPNOTSUPP); + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) + goto out; + if ((shift && shift != owner->shift) || !offset_mask(owner->shift, offset)) { + slot = ERR_PTR(-EINVAL); + goto out; + } + index = offset >> owner->shift; + pool = owner->pool; + spin_lock_irqsave(&pool->lock, flags); + if (owner->detached) { + slot = ERR_PTR(-EAGAIN); + goto unlock; + } + slot = &owner->slots[index]; + if (test_bit(index, &owner->used)) { + folio_get(folio); + if (!refcount_inc_not_zero_acquire(&slot->refs)) { + /* Its final put or initial publication has not finished. */ + folio_put(folio); + slot = ERR_PTR(-EAGAIN); + } + goto unlock; + } + VM_BUG_ON(refcount_read(&slot->refs) || atomic_read(&slot->mapcount)); + if (pool->unissued == owner) + pool->unissued = NULL; + else + folio_get(folio); + __set_bit(index, &owner->used); + __clear_bit(index, &owner->zeroed); + if (owner->used == pool->all_slots) + list_del_init(&owner->available); + /* The folio lock and uptodate state stabilise the restored bytes. */ + refcount_set_release(&slot->refs, 1); +unlock: + spin_unlock_irqrestore(&pool->lock, flags); +out: + page_ext_put(ext); + return slot; +} + +struct mm_subpage *mm_subpage_restore_granule(struct folio *folio, unsigned int offset, + unsigned int shift, gfp_t gfp) +{ + struct mm_subpage_pool *pool; + struct mm_subpage_owner *owner; + struct mm_subpage *slot; + unsigned int index; + int err; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!subpage_shift_valid(shift) || !offset_mask(shift, offset) || folio_order(folio) || !folio_test_swapcache(folio) || + !folio_test_uptodate(folio) || folio_test_ksm(folio) || + (folio->mapping && !folio_test_anon(folio)) || + (folio_test_anon(folio) && PageAnonExclusive(&folio->page))) + return ERR_PTR(-EINVAL); + index = offset >> shift; + slot = restore_bound_slot(folio, offset, shift); + if (slot) + return slot; + + /* A reclaimed folio has no surviving owner; do not initialise its data. */ + pool = mm_subpage_pool_create_granule(gfp, shift); + if (!pool) + return ERR_PTR(-ENOMEM); + pool->closed = true; + owner = kzalloc(sizeof(*owner), gfp | __GFP_ACCOUNT); + if (!owner) { + mm_subpage_pool_put(pool); + return ERR_PTR(-ENOMEM); + } + owner_init(owner, pool, folio); + owner->used = BIT(index); + refcount_set(&owner->slots[index].refs, 1); + list_add(&owner->all, &pool->owners); + pool->backing_pages = 1; + folio_get(folio); + err = owner_bind(owner); + if (err) { + folio_put(folio); + list_del(&owner->all); + pool->backing_pages = 0; + kfree(owner); + mm_subpage_pool_put(pool); + return ERR_PTR(err == -EEXIST ? -EAGAIN : err); + } + /* The private closed pool's initial reference belongs to this owner. */ + return &owner->slots[index]; +} + +struct mm_subpage *mm_subpage_restore(struct folio *folio, unsigned int offset, gfp_t gfp) +{ + return mm_subpage_restore_granule(folio, offset, MM_SUBPAGE_SHIFT, gfp); +} + +/* Preserve the metadata charge when migration runs outside its owning task. */ +static struct mm_subpage_owner *owner_alloc_migration(struct mm_subpage_owner *owner) +{ + struct mem_cgroup *memcg, *saved; + struct mm_subpage_owner *new; + + rcu_read_lock(); + do { + memcg = mem_cgroup_from_virt(owner) ?: root_mem_cgroup; + } while (memcg && !mem_cgroup_tryget(memcg)); + rcu_read_unlock(); + saved = set_active_memcg(memcg); + /* Folio locks are held: do not recurse into reclaim or trigger memcg OOM. */ + new = kzalloc(sizeof(*new), GFP_NOWAIT | __GFP_NOWARN | __GFP_ACCOUNT); + if (new && owner->rmap) { + new->rmap = kcalloc(PAGE_SIZE >> owner->shift, sizeof(*new->rmap), + GFP_NOWAIT | __GFP_NOWARN | __GFP_ACCOUNT); + if (!new->rmap) { + kfree(new); + new = NULL; + } + } + set_active_memcg(saved); + mem_cgroup_put(memcg); + return new; +} + +/* A migration hold pins metadata, not the native folio: the driver pins it. */ +int mm_subpage_migrate_prepare(struct folio *src, struct folio *dst) +{ + struct page_ext *ext; + struct mm_subpage_ext *subext; + struct mm_subpage_owner *owner, *new; + struct mm_subpage_pool *pool; + unsigned long flags; + int ret = 0; + unsigned int i; + + VM_BUG_ON_FOLIO(!folio_test_locked(src), src); + VM_BUG_ON_FOLIO(!folio_test_locked(dst), dst); + ext = page_ext_get(&src->page); + if (!ext) + return 0; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) { + page_ext_put(ext); + return 0; + } + pool = owner->pool; + spin_lock_irqsave(&pool->lock, flags); + if (owner->detached) { + owner = NULL; + } else if (src == dst || folio_order(src) || folio_order(dst) || + owner->migrating || pool->unissued == owner) { + ret = -EBUSY; + } else { + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) + if (owner->slots[i].pins) + ret = -EBUSY; + if (!ret) + owner->migrating = true; + } + spin_unlock_irqrestore(&pool->lock, flags); + page_ext_put(ext); + if (!owner || ret) + return ret; + + /* Async migration must not recurse into reclaim while holding folio locks. */ + new = owner_alloc_migration(owner); + if (!new) { + ret = -ENOMEM; + goto undo; + } + owner_init(new, pool, dst); + new->migrating = true; + new->zeroed = owner->zeroed; + spin_lock_irqsave(&pool->lock, flags); + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) { + new->slots[i].rmap_bound = owner->slots[i].rmap_bound; + if (new->rmap) { + new->rmap[i] = owner->rmap[i]; + if (new->rmap[i].root) + get_anon_vma(new->rmap[i].root); + } + } + spin_unlock_irqrestore(&pool->lock, flags); + + new->droppable = owner->droppable; + new->anon_root = owner->anon_root; + if (new->anon_root) + get_anon_vma(new->anon_root); + mm_subpage_pool_get(pool); + spin_lock_irqsave(&pool->lock, flags); + list_add_tail(&new->all, &pool->owners); + pool->backing_pages++; + ret = owner_bind(new); + if (ret) + owner_remove(new); + else + owner->migration_target = new; + spin_unlock_irqrestore(&pool->lock, flags); + if (!ret) + return 0; + if (new->anon_root) + put_anon_vma(new->anon_root); + owner_free_rmap(new); + kfree(new); + mm_subpage_pool_put(pool); +undo: + mm_subpage_migrate_finish(src); + return ret; +} + +bool mm_subpage_migrating(struct folio *folio) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_owner *owner; + struct mm_subpage_ext *subext; + bool active = false; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!ext) + return false; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (owner) + active = READ_ONCE(owner->migrating); + page_ext_put(ext); + return active; +} + +/* Source folio/PTE locked: writable GETs cannot race this final snapshot. */ +void mm_subpage_migrate_save_lazyfree(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + + VM_BUG_ON_FOLIO(!owner->migrating || !owner->migration_target, owner->folio); + WRITE_ONCE(owner->migration_target->slots[subpage->index].lazyfree, + READ_ONCE(subpage->lazyfree)); +} + +struct mm_subpage *mm_subpage_migrate_restore(struct folio *folio, unsigned int offset) +{ + struct mm_subpage *slot; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + VM_BUG_ON_FOLIO(!offset_mask(MM_SUBPAGE_SHIFT, offset) || !mm_subpage_migrating(folio), folio); + /* No allocation or data initialisation can fail during migration recovery. */ + slot = restore_bound_slot(folio, offset, 0); + VM_BUG_ON_FOLIO(IS_ERR_OR_NULL(slot), folio); + return slot; +} + +void mm_subpage_migrate_finish(struct folio *folio) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_owner *owner; + struct mm_subpage_ext *subext; + struct mm_subpage_pool *pool; + unsigned long flags; + unsigned int i; + bool release = false; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!ext) + return; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) + goto out; + pool = owner->pool; + spin_lock_irqsave(&pool->lock, flags); + if (owner->migrating) { + VM_BUG_ON(owner->detached); + owner->migrating = false; + owner->migration_target = NULL; + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) + if (!(owner->used & BIT(i))) + owner->slots[i].rmap_bound = false; + release = !owner->used; + if (release) + owner_remove(owner); + else if (!pool->closed && owner->used != pool->all_slots && + list_empty(&owner->available)) + list_add(&owner->available, &pool->available); + } + spin_unlock_irqrestore(&pool->lock, flags); +out: + page_ext_put(ext); + if (release) + /* No slot references remain; only the migration driver pins backing. */ + owner_free(owner, false); +} + +void mm_subpage_get(struct mm_subpage *subpage) +{ + folio_get(subpage->owner->folio); + refcount_inc(&subpage->refs); +} + +void mm_subpage_put(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + struct mm_subpage_pool *pool = owner->pool; + struct folio *folio = owner->folio; + unsigned long flags; + + /* + * Snapshot the backing while our slot reference still retains owner. + * A non-final decrement can race the final put and metadata reclamation; + * only our physical folio reference survives that decrement. + */ + if (refcount_dec_and_lock_irqsave(&subpage->refs, &pool->lock, &flags)) + subpage_release_locked(subpage, flags); + else + folio_put(folio); +} + +void mm_subpage_set_exclusive(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + + VM_BUG_ON_FOLIO(!folio_test_locked(subpage->owner->folio), subpage->owner->folio); + spin_lock_irqsave(&pool->lock, flags); + VM_BUG_ON(atomic_read(&subpage->mapcount) > 1 || !refcount_read(&subpage->refs)); + WRITE_ONCE(subpage->exclusive, true); + spin_unlock_irqrestore(&pool->lock, flags); +} + +bool mm_subpage_is_exclusive(const struct mm_subpage *subpage) +{ + return READ_ONCE(subpage->exclusive); +} + +static bool subpage_try_reuse(struct mm_subpage *subpage, unsigned int references) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool reusable; + + VM_BUG_ON_FOLIO(!folio_test_locked(subpage->owner->folio), subpage->owner->folio); + spin_lock_irqsave(&pool->lock, flags); + reusable = atomic_read(&subpage->mapcount) == 1 && + refcount_read(&subpage->refs) == references && !subpage->pins; + if (reusable) + WRITE_ONCE(subpage->exclusive, true); + spin_unlock_irqrestore(&pool->lock, flags); + return reusable; +} + +bool mm_subpage_try_reuse_swap(struct mm_subpage *subpage) +{ + return subpage_try_reuse(subpage, 1); +} + +bool mm_subpage_try_reuse_anon(struct mm_subpage *subpage) +{ + return subpage_try_reuse(subpage, 2); +} + +bool mm_subpage_can_zap(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool eligible; + + spin_lock_irqsave(&pool->lock, flags); + eligible = !subpage->owner->migrating && !subpage->pins && + atomic_read(&subpage->mapcount) == 1 && + refcount_read(&subpage->refs) == 2; + spin_unlock_irqrestore(&pool->lock, flags); + return eligible; +} + +bool mm_subpage_try_share(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool shared; + + spin_lock_irqsave(&pool->lock, flags); + shared = !subpage->pins; + if (shared) + WRITE_ONCE(subpage->exclusive, false); + spin_unlock_irqrestore(&pool->lock, flags); + return shared; +} + +bool mm_subpage_is_lazyfree(const struct mm_subpage *subpage) +{ + return READ_ONCE(subpage->lazyfree); +} + +void mm_subpage_cancel_lazyfree(struct mm_subpage *subpage) +{ + WRITE_ONCE(subpage->lazyfree, false); +} + +bool mm_subpage_mark_lazyfree(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool eligible; + + VM_BUG_ON_FOLIO(!folio_test_locked(subpage->owner->folio), subpage->owner->folio); + spin_lock_irqsave(&pool->lock, flags); + eligible = atomic_read(&subpage->mapcount) == 1 && + refcount_read(&subpage->refs) == 2 && !subpage->pins; + if (eligible) + WRITE_ONCE(subpage->lazyfree, true); + spin_unlock_irqrestore(&pool->lock, flags); + return eligible; +} + +bool mm_subpage_all_lazyfree(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + unsigned long flags; + unsigned int i; + bool all = true; + + spin_lock_irqsave(&owner->pool->lock, flags); + for (i = 0; i < SUBPAGES_PER_FOLIO; i++) { + if (!(owner->used & BIT(i))) + continue; + if (!READ_ONCE(owner->slots[i].lazyfree)) { + all = false; + break; + } + } + spin_unlock_irqrestore(&owner->pool->lock, flags); + return all; +} + +bool mm_subpage_discardable(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool eligible; + + spin_lock_irqsave(&pool->lock, flags); + eligible = (subpage->owner->droppable || READ_ONCE(subpage->lazyfree)) && + !subpage->pins && + refcount_read(&subpage->refs) == atomic_read(&subpage->mapcount) + 1; + spin_unlock_irqrestore(&pool->lock, flags); + return eligible; +} + +int mm_subpage_pin(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + unsigned long flags; + int ret = -EAGAIN; + + spin_lock_irqsave(&owner->pool->lock, flags); + if (!owner->anon_root || owner->migrating || !subpage->exclusive || + !atomic_read(&subpage->mapcount)) + goto out; + ret = try_grab_folio(owner->folio, 1, FOLL_PIN); + if (ret) + goto out; + mm_subpage_get(subpage); + subpage->pins++; +out: + spin_unlock_irqrestore(&owner->pool->lock, flags); + return ret; +} + +void mm_subpage_unpin(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + unsigned long flags; + + spin_lock_irqsave(&owner->pool->lock, flags); + VM_BUG_ON(!subpage->pins || !subpage->exclusive); + subpage->pins--; + spin_unlock_irqrestore(&owner->pool->lock, flags); + unpin_folio(owner->folio); + mm_subpage_put(subpage); +} + +unsigned int mm_subpage_pincount(const struct mm_subpage *subpage) +{ + return READ_ONCE(subpage->pins); +} + +int mm_subpage_mapcount(const struct mm_subpage *subpage) +{ + return atomic_read(&subpage->mapcount); +} + +struct folio *mm_subpage_folio(const struct mm_subpage *subpage) +{ + return subpage->owner->folio; +} + +unsigned int mm_subpage_shift(const struct mm_subpage *subpage) +{ + return subpage->shift; +} + +unsigned int mm_subpage_size(const struct mm_subpage *subpage) +{ + return 1U << subpage->shift; +} + +unsigned int mm_subpage_offset(const struct mm_subpage *subpage) +{ + return subpage->index << subpage->shift; +} + +phys_addr_t mm_subpage_phys(const struct mm_subpage *subpage) +{ + return PFN_PHYS(folio_pfn(subpage->owner->folio)) + mm_subpage_offset(subpage); +} + +/* + * Round the mm's aggregate reservation, never each VMA: splitting a VMA + * must not create extra charges or refunds. mmap writers serialize updates; + * munmap may retain the lock downgraded while finishing its refund. + */ +static int mm_commit_delta(struct mm_struct *mm, long pages, + unsigned long *next, long *delta) +{ + unsigned long old = mm->committed_user_pages; + unsigned int shift = mm_page_shift(mm); + + mmap_assert_locked(mm); + if (pages < 0 && WARN_ON_ONCE(pages < -(long)old)) + return -EINVAL; + if (pages > 0 && (unsigned long)pages > LONG_MAX - old) + return -ENOMEM; + *next = old + pages; + if (shift > PAGE_SHIFT) { + unsigned int extra = shift - PAGE_SHIFT; + + /* A logical reservation may span several native commit units. */ + if (*next > (LONG_MAX >> extra) || old > (LONG_MAX >> extra)) + return -ENOMEM; + *delta = (long)(*next << extra) - (long)(old << extra); + } else { + unsigned long per_native = 1UL << (PAGE_SHIFT - shift); + + *delta = DIV_ROUND_UP(*next, per_native) - DIV_ROUND_UP(old, per_native); + } + return 0; +} + +int mm_account_memory(struct mm_struct *mm, struct mm_struct *policy_mm, long pages) +{ + unsigned long next; + long delta; + int ret; + + if (mm_page_size(mm) == PAGE_SIZE) + return security_vm_enough_memory_mm(policy_mm, pages); + ret = mm_commit_delta(mm, pages, &next, &delta); + if (ret) + return ret; + ret = security_vm_enough_memory_mm(policy_mm, delta); + if (!ret) + mm->committed_user_pages = next; + return ret; +} + +void mm_acct_memory(struct mm_struct *mm, long pages) +{ + unsigned long next; + long delta; + + if (mm_page_size(mm) == PAGE_SIZE) { + vm_acct_memory(pages); + return; + } + if (WARN_ON_ONCE(mm_commit_delta(mm, pages, &next, &delta))) + return; + vm_acct_memory(delta); + mm->committed_user_pages = next; +} + +void mm_unacct_memory(struct mm_struct *mm, unsigned long pages) +{ + mm_acct_memory(mm, -(long)pages); +} + +/* Shared ownership for consumers whose native page spans contain user slots. */ +struct mm_subpage_refs { + refcount_t refs; + unsigned int count; + unsigned int capacity; + struct mm_subpage *slots[]; +}; + +struct mm_subpage_refs *mm_subpage_refs_get(struct mm_subpage_refs *refs) +{ + if (refs) + refcount_inc(&refs->refs); + return refs; +} +EXPORT_SYMBOL_GPL(mm_subpage_refs_get); + +void mm_subpage_refs_put(struct mm_subpage_refs *refs) +{ + unsigned int i; + + if (!refs || !refcount_dec_and_test(&refs->refs)) + return; + for (i = 0; i < refs->count; i++) + mm_subpage_put(refs->slots[i]); + kfree(refs); +} +EXPORT_SYMBOL_GPL(mm_subpage_refs_put); + +static struct mm_subpage_refs *subpage_refs_new(unsigned int capacity, gfp_t gfp) +{ + struct mm_subpage_refs *refs = kmalloc(struct_size(refs, slots, capacity), gfp); + + if (!refs) + return NULL; + refcount_set(&refs->refs, 1); + refs->count = 0; + refs->capacity = capacity; + return refs; +} + +static int subpage_refs_prepare(struct mm_subpage_refs **refsp, + unsigned int needed, gfp_t gfp) +{ + struct mm_subpage_refs *old = *refsp, *refs; + unsigned int i; + + if (old && refcount_read(&old->refs) == 1 && old->capacity >= needed) + return 0; + refs = subpage_refs_new(max(needed, 4U), gfp); + if (!refs) + return -ENOMEM; + if (old) { + for (i = 0; i < old->count; i++) { + mm_subpage_get(old->slots[i]); + refs->slots[refs->count++] = old->slots[i]; + } + } + *refsp = refs; + mm_subpage_refs_put(old); + return 0; +} + +int mm_subpage_refs_add(struct mm_subpage_refs **refsp, struct page *page, + unsigned int offset, unsigned int len, gfp_t gfp) +{ + struct mm_subpage *slots[PAGE_SIZE / MM_SUBPAGE_SIZE]; + struct mm_subpage_refs *refs = *refsp; + unsigned int pos, count = 0, i, j; + int ret = 0; + bool found = false; + + if (!len || folio_order(page_folio(page)) || !folio_test_anon(page_folio(page))) + return 0; + if (offset >= PAGE_SIZE || len > PAGE_SIZE - offset) + return -EINVAL; + for (pos = round_down(offset, MM_SUBPAGE_SIZE); pos < offset + len; + pos += MM_SUBPAGE_SIZE) { + struct mm_subpage *slot = mm_subpage_get_from_phys(page_to_phys(page) + pos); + + if (!slot) { + /* A native anonymous allocation has no quarter owner. */ + ret = found ? -EFAULT : 0; + goto out; + } + found = true; + for (j = 0; refs && j < refs->count; j++) + if (refs->slots[j] == slot) + break; + if (refs && j < refs->count) { + mm_subpage_put(slot); + continue; + } + for (j = 0; j < count; j++) + if (slots[j] == slot) + break; + if (j < count) + mm_subpage_put(slot); + else + slots[count++] = slot; + } + if (!count) + return 0; + if (refs && refs->count > UINT_MAX - count) { + ret = -ENOMEM; + goto out; + } + ret = subpage_refs_prepare(refsp, (refs ? refs->count : 0) + count, gfp); + if (ret) + goto out; + refs = *refsp; + for (i = 0; i < count; i++) + refs->slots[refs->count++] = slots[i]; + return 0; +out: + for (i = 0; i < count; i++) + mm_subpage_put(slots[i]); + return ret; +} +EXPORT_SYMBOL_GPL(mm_subpage_refs_add); + +int mm_subpage_refs_prune(struct mm_subpage_refs **refsp, + bool (*keep)(const struct mm_subpage *, void *), + void *context, gfp_t gfp) +{ + struct mm_subpage_refs *refs = *refsp, *new; + unsigned int i, count = 0; + + if (!refs) + return 0; + for (i = 0; i < refs->count; i++) + count += keep(refs->slots[i], context); + if (count == refs->count) + return 0; + if (!count) { + *refsp = NULL; + mm_subpage_refs_put(refs); + return 0; + } + if (refcount_read(&refs->refs) == 1) { + for (i = 0, count = 0; i < refs->count; i++) { + if (keep(refs->slots[i], context)) + refs->slots[count++] = refs->slots[i]; + else + mm_subpage_put(refs->slots[i]); + } + refs->count = count; + return 0; + } + new = subpage_refs_new(count, gfp); + if (!new) + return -ENOMEM; + for (i = 0; i < refs->count; i++) { + if (!keep(refs->slots[i], context)) + continue; + mm_subpage_get(refs->slots[i]); + new->slots[new->count++] = refs->slots[i]; + } + *refsp = new; + mm_subpage_refs_put(refs); + return 0; +} +EXPORT_SYMBOL_GPL(mm_subpage_refs_prune); From 039cd8c6f32406b8878b76e5482c5187990fbd56 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0683/2680] exec: stage arguments and ELF mappings at the selected user page size Resolve the per-thread exec preference before exposing the new address space. Transfer staged arguments across granules, bound temporary copies by their native mappings, and use the selected ABI for ELF and core dumps. Signed-off-by: Diljit Singh (cherry picked from commit c16faad6c36dd65a3cf47cbde2e89b7ac84e46a5) --- fs/binfmt_elf.c | 42 ++++++-- fs/coredump.c | 81 ++++++++++++--- fs/exec.c | 265 +++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 313 insertions(+), 75 deletions(-) diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index 16a56b6b3f6ca..826235fd977c2 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -79,11 +79,11 @@ static int elf_core_dump(struct coredump_params *cprm); #define elf_core_dump NULL #endif -#if ELF_EXEC_PAGESIZE > PAGE_SIZE -#define ELF_MIN_ALIGN ELF_EXEC_PAGESIZE -#else -#define ELF_MIN_ALIGN PAGE_SIZE +#ifndef ELF_EXEC_PAGESIZE_MM +#define ELF_EXEC_PAGESIZE_MM(mm) ELF_EXEC_PAGESIZE #endif +#define ELF_MIN_ALIGN max_t(unsigned long, mm_page_size(current->mm), \ + ELF_EXEC_PAGESIZE_MM(current->mm)) #ifndef ELF_CORE_EFLAGS #define ELF_CORE_EFLAGS 0 @@ -93,12 +93,20 @@ static int elf_core_dump(struct coredump_params *cprm); #define ELF_PAGEOFFSET(_v) ((_v) & (ELF_MIN_ALIGN-1)) #define ELF_PAGEALIGN(_v) (((_v) + ELF_MIN_ALIGN - 1) & ~(ELF_MIN_ALIGN - 1)) +#ifdef CONFIG_COREDUMP +static unsigned long elf_min_coredump_size(void) +{ + return ELF_EXEC_PAGESIZE_MM(current->mm); +} +#endif + static struct linux_binfmt elf_format = { .module = THIS_MODULE, .load_binary = load_elf_binary, #ifdef CONFIG_COREDUMP .core_dump = elf_core_dump, .min_coredump = ELF_EXEC_PAGESIZE, + .min_coredump_size = elf_min_coredump_size, #endif }; @@ -248,7 +256,7 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec, ARCH_DLINFO; #endif NEW_AUX_ENT(AT_HWCAP, ELF_HWCAP); - NEW_AUX_ENT(AT_PAGESZ, ELF_EXEC_PAGESIZE); + NEW_AUX_ENT(AT_PAGESZ, ELF_EXEC_PAGESIZE_MM(mm)); NEW_AUX_ENT(AT_CLKTCK, CLOCKS_PER_SEC); NEW_AUX_ENT(AT_PHDR, phdr_addr); NEW_AUX_ENT(AT_PHENT, sizeof(struct elf_phdr)); @@ -622,6 +630,13 @@ static inline int arch_check_elf(struct elfhdr *ehdr, bool has_interp, #endif /* !CONFIG_ARCH_BINFMT_ELF_STATE */ +#ifndef arch_bprm_prepare_mm +static inline int arch_bprm_prepare_mm(struct linux_binprm *bprm, bool compat) +{ + return 0; +} +#endif + static inline int make_prot(u32 p_flags, struct arch_elf_state *arch_state, bool has_interp, bool is_interp) { @@ -1006,6 +1021,11 @@ static int load_elf_binary(struct linux_binprm *bprm) if (retval) goto out_free_dentry; + retval = arch_bprm_prepare_mm(bprm, + elf_ex->e_ident[EI_CLASS] == ELFCLASS32); + if (retval) + goto out_free_dentry; + /* Flush all traces of the currently running executable */ retval = begin_new_exec(bprm); if (retval) @@ -1335,7 +1355,7 @@ static int load_elf_binary(struct linux_binprm *bprm) * leave a gap between .bss and brk. */ if (!brk_moved) - mm->brk = mm->start_brk = mm->brk + PAGE_SIZE; + mm->brk = mm->start_brk = mm->brk + mm_page_size(mm); mm->brk = mm->start_brk = arch_randomize_brk(mm); brk_moved = true; @@ -1351,10 +1371,10 @@ static int load_elf_binary(struct linux_binprm *bprm) and some applications "depend" upon this behavior. Since we do not have the power to recompile these, we emulate the SVr4 behavior. Sigh. */ - error = vm_mmap(NULL, 0, PAGE_SIZE, PROT_READ | PROT_EXEC, + error = vm_mmap(NULL, 0, mm_page_size(mm), PROT_READ | PROT_EXEC, MAP_FIXED | MAP_PRIVATE, 0); - retval = do_mseal(0, PAGE_SIZE, 0); + retval = do_mseal(0, mm_page_size(mm), 0); if (retval) pr_warn_ratelimited("pid=%d, couldn't seal address 0, ret=%d.\n", task_pid_nr(current), retval); @@ -1658,7 +1678,7 @@ static int fill_files_note(struct memelfnote *note, struct coredump_params *cprm /* Now we know exact count of files, can store it */ data[0] = count; - data[1] = PAGE_SIZE; + data[1] = mm_page_size(current->mm); /* * Count usually is less than mm->map_count, * we need to move filenames down. @@ -2043,7 +2063,7 @@ static int elf_core_dump(struct coredump_params *cprm) offset += sz; } - dataoff = offset = roundup(offset, ELF_EXEC_PAGESIZE); + dataoff = offset = roundup(offset, ELF_EXEC_PAGESIZE_MM(current->mm)); offset += cprm->vma_data_size; offset += elf_core_extra_data_size(cprm); @@ -2083,7 +2103,7 @@ static int elf_core_dump(struct coredump_params *cprm) phdr.p_flags |= PF_W; if (meta->flags & VM_EXEC) phdr.p_flags |= PF_X; - phdr.p_align = ELF_EXEC_PAGESIZE; + phdr.p_align = ELF_EXEC_PAGESIZE_MM(current->mm); if (!dump_emit(cprm, &phdr, sizeof(phdr))) goto end_coredump; diff --git a/fs/coredump.c b/fs/coredump.c index e68a76ff92a38..c15ac11a43bae 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -883,7 +884,8 @@ static bool coredump_file(struct core_name *cn, struct coredump_params *cprm, struct file *file __free(fput) = NULL; int open_flags = O_CREAT | O_WRONLY | O_NOFOLLOW | O_LARGEFILE | O_EXCL; - if (cprm->limit < binfmt->min_coredump) + if (cprm->limit < (binfmt->min_coredump_size ? + binfmt->min_coredump_size() : binfmt->min_coredump)) return false; if (coredump_force_suid_safe(cprm) && cn->corename[0] != '/') { @@ -1272,7 +1274,8 @@ void dump_skip(struct coredump_params *cprm, size_t nr) EXPORT_SYMBOL(dump_skip); #ifdef CONFIG_ELF_CORE -static int dump_emit_page(struct coredump_params *cprm, struct page *page) +static int dump_emit_page(struct coredump_params *cprm, struct page *page, + unsigned int offset, unsigned int size) { struct bio_vec bvec; struct iov_iter iter; @@ -1288,19 +1291,19 @@ static int dump_emit_page(struct coredump_params *cprm, struct page *page) return 0; cprm->to_skip = 0; } - if (cprm->written + PAGE_SIZE > cprm->limit) + if (cprm->written + size > cprm->limit) return 0; if (dump_interrupted()) return 0; pos = file->f_pos; - bvec_set_page(&bvec, page, PAGE_SIZE, 0); - iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, PAGE_SIZE); + bvec_set_page(&bvec, page, size, offset); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, size); n = __kernel_write_iter(cprm->file, &iter, &pos); - if (n != PAGE_SIZE) + if (n != size) return 0; file->f_pos = pos; - cprm->written += PAGE_SIZE; - cprm->pos += PAGE_SIZE; + cprm->written += size; + cprm->pos += size; return 1; } @@ -1315,10 +1318,11 @@ static int dump_emit_page(struct coredump_params *cprm, struct page *page) #define dump_page_alloc() alloc_page(GFP_KERNEL) #define dump_page_free(x) __free_page(x) -static struct page *dump_page_copy(struct page *src, struct page *dst) +static struct page *dump_page_copy(struct page *src, struct page *dst, + unsigned int offset, unsigned int size) { void *buf = kmap_local_page(src); - size_t left = copy_mc_to_kernel(page_address(dst), buf, PAGE_SIZE); + size_t left = copy_mc_to_kernel(page_address(dst) + offset, buf + offset, size); kunmap_local(buf); return left ? NULL : dst; } @@ -1328,12 +1332,51 @@ static struct page *dump_page_copy(struct page *src, struct page *dst) /* We just want to return non-NULL; it's never used. */ #define dump_page_alloc() ERR_PTR(-EINVAL) #define dump_page_free(x) ((void)(x)) -static inline struct page *dump_page_copy(struct page *src, struct page *dst) +static inline struct page *dump_page_copy(struct page *src, struct page *dst, + unsigned int offset, unsigned int size) { return src; } #endif +#ifdef CONFIG_MM_SUBPAGE +static int dump_subpage_range(struct coredump_params *cprm, unsigned long start, + unsigned long len) +{ + unsigned long size = mm_user_fragment_size(current->mm), addr; + struct page *scratch = dump_page_alloc(); + int ret = 0; + + if (!scratch) + return 0; + for (addr = start; addr < start + len; addr += size) { + struct user_page_fragment fragment; + + if (get_dump_fragment(addr, &fragment) == 1) { + struct page *page = folio_page(fragment.folio, + fragment.offset >> PAGE_SHIFT); + unsigned int offset = offset_in_page(fragment.offset); + int emitted; + + emitted = dump_emit_page(cprm, + dump_page_copy(page, scratch, offset, size), offset, size); + release_user_fragments(&fragment, 1, false); + if (!emitted) + goto out; + } else { + dump_skip(cprm, size); + } + if (dump_interrupted()) + goto out; + cond_resched(); + } + ret = 1; +out: + dump_page_free(scratch); + return ret; +} +#endif + int dump_user_range(struct coredump_params *cprm, unsigned long start, unsigned long len) { @@ -1341,6 +1384,10 @@ int dump_user_range(struct coredump_params *cprm, unsigned long start, struct page *dump_page; int locked, ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return dump_subpage_range(cprm, start, len); +#endif dump_page = dump_page_alloc(); if (!dump_page) return 0; @@ -1369,7 +1416,8 @@ int dump_user_range(struct coredump_params *cprm, unsigned long start, mmap_read_unlock(current->mm); locked = 0; } - int stop = !dump_emit_page(cprm, dump_page_copy(page, dump_page)); + int stop = !dump_emit_page(cprm, + dump_page_copy(page, dump_page, 0, PAGE_SIZE), 0, PAGE_SIZE); put_page(page); if (stop) goto out; @@ -1647,9 +1695,10 @@ static unsigned long vma_dump_size(struct vm_area_struct *vma, * dump the first page to aid in determining what was mapped here. */ if (FILTER(ELF_HEADERS) && - vma->vm_pgoff == 0 && (vma->vm_flags & VM_READ)) { + vma->vm_pgoff == 0 && !vma_subpage_offset(vma) && + (vma->vm_flags & VM_READ)) { if ((READ_ONCE(file_inode(vma->vm_file)->i_mode) & 0111) != 0) - return PAGE_SIZE; + return min(mm_page_size(vma->vm_mm), vma->vm_end - vma->vm_start); /* * ELF libraries aren't always executable. @@ -1749,7 +1798,7 @@ static bool dump_vma_snapshot(struct coredump_params *cprm) m->end = vma->vm_end; m->flags = vma->vm_flags; m->dump_size = vma_dump_size(vma, cprm->mm_flags); - m->pgoff = vma->vm_pgoff; + m->pgoff = core_vma_pgoff(vma); m->file = vma->vm_file; if (m->file) get_file(m->file); @@ -1768,7 +1817,7 @@ static bool dump_vma_snapshot(struct coredump_params *cprm) memcmp(elfmag, ELFMAG, SELFMAG) != 0) { m->dump_size = 0; } else { - m->dump_size = PAGE_SIZE; + m->dump_size = min(mm_page_size(mm), m->end - m->start); } } diff --git a/fs/exec.c b/fs/exec.c index a598d48b5ffcc..b08816ff1f735 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -123,6 +124,54 @@ bool path_noexec(const struct path *path) (path->mnt->mnt_sb->s_iflags & SB_I_NOEXEC); } +/* A temporary argument mapping owns one native page or an exact subpage slot. */ +struct arg_page { + struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage *subpage; + unsigned int offset; +#endif +}; + +static unsigned long bprm_page_size(const struct linux_binprm *bprm) +{ +#ifdef CONFIG_MMU + return mm_page_size(bprm->mm); +#else + return PAGE_SIZE; +#endif +} + +/* kmap_local_page() maps one native page, even for a larger user leaf. */ +static unsigned long bprm_copy_size(const struct linux_binprm *bprm) +{ + return min(bprm_page_size(bprm), PAGE_SIZE); +} + +static unsigned long bprm_copy_mask(const struct linux_binprm *bprm) +{ + return ~(bprm_copy_size(bprm) - 1); +} + +static unsigned long bprm_max_arg_strlen(const struct linux_binprm *bprm) +{ + return (MAX_ARG_STRLEN / PAGE_SIZE) * bprm_page_size(bprm); +} + +static unsigned int arg_page_offset(struct arg_page page) +{ +#ifdef CONFIG_MM_SUBPAGE + return page.offset; +#else + return 0; +#endif +} + +static void *map_arg_page(struct arg_page page) +{ + return kmap_local_page(page.page) + arg_page_offset(page); +} + #ifdef CONFIG_MMU /* * The nascent bprm->mm is not visible until exec_mmap() but it can @@ -142,10 +191,10 @@ static void acct_arg_size(struct linux_binprm *bprm, unsigned long pages) add_mm_counter(mm, MM_ANONPAGES, diff); } -static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, +static struct arg_page get_arg_page(struct linux_binprm *bprm, unsigned long pos, int write) { - struct page *page; + struct arg_page page = {}; struct vm_area_struct *vma = bprm->vma; struct mm_struct *mm = bprm->mm; int ret; @@ -156,28 +205,47 @@ static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, * ahead of time. */ if (!mmap_read_lock_maybe_expand(mm, vma, pos, write)) - return NULL; + return page; /* * We are doing an exec(). 'current' is the process * doing the exec and 'mm' is the new process's mm. */ - ret = get_user_pages_remote(mm, pos, 1, - write ? FOLL_WRITE : 0, - &page, NULL); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) != PAGE_SIZE) { + struct user_page_fragment fragment; + struct vm_area_struct *area; + + ret = get_user_fragment_vma_remote(mm, pos, 1, write ? FOLL_WRITE : 0, + &fragment, &area); + if (ret > 0) { + page.page = folio_page(fragment.folio, fragment.offset >> PAGE_SHIFT); + page.subpage = fragment.subpage; + page.offset = offset_in_page(fragment.offset) & mm_page_mask(mm); + } + } else +#endif + ret = get_user_pages_remote(mm, pos, 1, + write ? FOLL_WRITE : 0, &page.page, NULL); mmap_read_unlock(mm); if (ret <= 0) - return NULL; + return page; if (write) - acct_arg_size(bprm, vma_pages(vma)); + acct_arg_size(bprm, mm_pages_for_reader(mm, current->mm, vma_user_pages(vma))); return page; } -static void put_arg_page(struct page *page) +static void put_arg_page(struct arg_page page) { - put_page(page); +#ifdef CONFIG_MM_SUBPAGE + if (page.subpage) { + mm_subpage_put(page.subpage); + return; + } +#endif + put_page(page.page); } static void free_arg_pages(struct linux_binprm *bprm) @@ -185,14 +253,14 @@ static void free_arg_pages(struct linux_binprm *bprm) } static void flush_arg_page(struct linux_binprm *bprm, unsigned long pos, - struct page *page) + struct arg_page page) { - flush_cache_page(bprm->vma, pos, page_to_pfn(page)); + flush_cache_page(bprm->vma, pos, page_to_pfn(page.page)); } static bool valid_arg_len(struct linux_binprm *bprm, long len) { - return len <= MAX_ARG_STRLEN; + return len <= bprm_max_arg_strlen(bprm); } #else @@ -201,7 +269,7 @@ static inline void acct_arg_size(struct linux_binprm *bprm, unsigned long pages) { } -static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, +static struct arg_page get_arg_page(struct linux_binprm *bprm, unsigned long pos, int write) { struct page *page; @@ -210,14 +278,14 @@ static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, if (!page && write) { page = alloc_page(GFP_HIGHUSER|__GFP_ZERO); if (!page) - return NULL; + return (struct arg_page) {}; bprm->page[pos / PAGE_SIZE] = page; } - return page; + return (struct arg_page) { .page = page }; } -static void put_arg_page(struct page *page) +static void put_arg_page(struct arg_page page) { } @@ -238,7 +306,7 @@ static void free_arg_pages(struct linux_binprm *bprm) } static void flush_arg_page(struct linux_binprm *bprm, unsigned long pos, - struct page *page) + struct arg_page page) { } @@ -260,7 +328,7 @@ static int bprm_mm_init(struct linux_binprm *bprm) int err; struct mm_struct *mm = NULL; - bprm->mm = mm = mm_alloc(); + bprm->mm = mm = mm_alloc_exec(); err = -ENOMEM; if (!mm) goto err; @@ -292,6 +360,102 @@ static int bprm_mm_init(struct linux_binprm *bprm) return err; } +#ifdef CONFIG_MM_SUBPAGE +/* + * Final ELF selection may change the granule after argv, envp and script + * arguments have been staged. Table geometry is immutable: build a second mm + * and transfer bytes at the same temporary stack addresses before committing + * exec. Until success, the original bprm and its argument charge remain intact. + */ +int bprm_set_page_shift(struct linux_binprm *bprm, unsigned int page_shift) +{ + struct linux_binprm replacement = {}; + struct mm_struct *old_mm; + unsigned long pos, end, new_top, string_len = 0; + unsigned long old_size = mm_user_fragment_size(bprm->mm), new_size; + int ret; + + if (mm_page_shift(bprm->mm) == page_shift) + return 0; + if (WARN_ON_ONCE(bprm->point_of_no_return)) + return -EINVAL; + + replacement.mm = mm_alloc_exec_page_shift(page_shift); + if (!replacement.mm) + return -ENOMEM; + ret = -EINVAL; + if (mm_page_shift(replacement.mm) != page_shift) + goto out; + new_size = mm_user_fragment_size(replacement.mm); + ret = create_init_stack_vma(replacement.mm, &replacement.vma, &new_top); + if (ret) + goto out; + end = bprm->vma->vm_end - sizeof(void *); + ret = -EINVAL; + if (new_top != end || bprm->p > end) + goto out; + + for (pos = bprm->p; pos < end;) { + struct arg_page src, dst; + unsigned long src_off = pos & (old_size - 1); + unsigned long dst_off = pos & (new_size - 1); + unsigned long len = min3(end - pos, old_size - src_off, + new_size - dst_off); + const char *from; + char *to; + + ret = -ERESTARTNOHAND; + if (fatal_signal_pending(current)) + goto out; + src = get_arg_page(bprm, pos, 0); + ret = -EFAULT; + if (!src.page) + goto out; + dst = get_arg_page(&replacement, pos, 1); + if (!dst.page) { + put_arg_page(src); + ret = -ENOMEM; + goto out; + } + from = map_arg_page(src); + to = map_arg_page(dst); + /* Recheck per-string limits if the selected granule shrank. */ + ret = 0; + for (unsigned long i = 0; i < len; i++) { + if (++string_len > bprm_max_arg_strlen(&replacement)) { + ret = -E2BIG; + break; + } + if (!from[src_off + i]) + string_len = 0; + } + if (!ret) + memcpy(to + dst_off, from + src_off, len); + kunmap_local(to); + kunmap_local(from); + flush_arg_page(&replacement, pos, dst); + put_arg_page(dst); + put_arg_page(src); + if (ret) + goto out; + pos += len; + cond_resched(); + } + + old_mm = bprm->mm; + acct_arg_size(bprm, 0); + bprm->mm = replacement.mm; + bprm->vma = replacement.vma; + bprm->vma_pages = replacement.vma_pages; + mmput(old_mm); + return 0; +out: + acct_arg_size(&replacement, 0); + mmput(replacement.mm); + return ret; +} +#endif + struct user_arg_ptr { #ifdef CONFIG_COMPAT bool is_compat; @@ -453,7 +617,7 @@ static int bprm_stack_limits(struct linux_binprm *bprm) static int copy_strings(int argc, struct user_arg_ptr argv, struct linux_binprm *bprm) { - struct page *kmapped_page = NULL; + struct arg_page kmapped_page = {}; char *kaddr = NULL; unsigned long kpos = 0; int ret; @@ -468,7 +632,7 @@ static int copy_strings(int argc, struct user_arg_ptr argv, if (IS_ERR(str)) goto out; - len = strnlen_user(str, MAX_ARG_STRLEN); + len = strnlen_user(str, bprm_max_arg_strlen(bprm)); if (!len) goto out; @@ -492,9 +656,9 @@ static int copy_strings(int argc, struct user_arg_ptr argv, } cond_resched(); - offset = pos % PAGE_SIZE; + offset = pos & ~bprm_copy_mask(bprm); if (offset == 0) - offset = PAGE_SIZE; + offset = bprm_copy_size(bprm); bytes_to_copy = offset; if (bytes_to_copy > len) @@ -505,23 +669,23 @@ static int copy_strings(int argc, struct user_arg_ptr argv, str -= bytes_to_copy; len -= bytes_to_copy; - if (!kmapped_page || kpos != (pos & PAGE_MASK)) { - struct page *page; + if (!kmapped_page.page || kpos != (pos & bprm_copy_mask(bprm))) { + struct arg_page page; page = get_arg_page(bprm, pos, 1); - if (!page) { + if (!page.page) { ret = -E2BIG; goto out; } - if (kmapped_page) { - flush_dcache_page(kmapped_page); + if (kmapped_page.page) { + flush_dcache_page(kmapped_page.page); kunmap_local(kaddr); put_arg_page(kmapped_page); } kmapped_page = page; - kaddr = kmap_local_page(kmapped_page); - kpos = pos & PAGE_MASK; + kaddr = map_arg_page(kmapped_page); + kpos = pos & bprm_copy_mask(bprm); flush_arg_page(bprm, kpos, kmapped_page); } if (copy_from_user(kaddr+offset, str, bytes_to_copy)) { @@ -532,8 +696,8 @@ static int copy_strings(int argc, struct user_arg_ptr argv, } ret = 0; out: - if (kmapped_page) { - flush_dcache_page(kmapped_page); + if (kmapped_page.page) { + flush_dcache_page(kmapped_page.page); kunmap_local(kaddr); put_arg_page(kmapped_page); } @@ -545,7 +709,7 @@ static int copy_strings(int argc, struct user_arg_ptr argv, */ int copy_string_kernel(const char *arg, struct linux_binprm *bprm) { - int len = strnlen(arg, MAX_ARG_STRLEN) + 1 /* terminating NUL */; + int len = strnlen(arg, bprm_max_arg_strlen(bprm)) + 1 /* terminating NUL */; unsigned long pos = bprm->p; if (len == 0) @@ -561,18 +725,19 @@ int copy_string_kernel(const char *arg, struct linux_binprm *bprm) while (len > 0) { unsigned int bytes_to_copy = min(len, - min_not_zero(offset_in_page(pos), PAGE_SIZE)); - struct page *page; + min_not_zero(pos & ~bprm_copy_mask(bprm), bprm_copy_size(bprm))); + struct arg_page page; pos -= bytes_to_copy; arg -= bytes_to_copy; len -= bytes_to_copy; page = get_arg_page(bprm, pos, 1); - if (!page) + if (!page.page) return -E2BIG; - flush_arg_page(bprm, pos & PAGE_MASK, page); - memcpy_to_page(page, offset_in_page(pos), arg, bytes_to_copy); + flush_arg_page(bprm, pos & bprm_copy_mask(bprm), page); + memcpy_to_page(page.page, arg_page_offset(page) + + (pos & ~bprm_copy_mask(bprm)), arg, bytes_to_copy); put_arg_page(page); } @@ -625,20 +790,20 @@ int setup_arg_pages(struct linux_binprm *bprm, /* Add space for stack randomization. */ if (current->flags & PF_RANDOMIZE) - stack_base += (STACK_RND_MASK << PAGE_SHIFT); + stack_base += (STACK_RND_MASK << mm_page_shift(mm)); /* Make sure we didn't let the argument array grow too large. */ if (vma->vm_end - vma->vm_start > stack_base) return -ENOMEM; - stack_base = PAGE_ALIGN(stack_top - stack_base); + stack_base = mm_page_align(mm, stack_top - stack_base); stack_shift = vma->vm_start - stack_base; mm->arg_start = bprm->p - stack_shift; bprm->p = vma->vm_end - stack_shift; #else stack_top = arch_align_stack(stack_top); - stack_top = PAGE_ALIGN(stack_top); + stack_top = mm_page_align(mm, stack_top); if (unlikely(stack_top < mmap_min_addr) || unlikely(vma->vm_end - vma->vm_start >= stack_top - mmap_min_addr)) @@ -706,7 +871,7 @@ int setup_arg_pages(struct linux_binprm *bprm, * Align this down to a page boundary as expand_stack * will align it up. */ - rlim_stack = bprm->rlim_stack.rlim_cur & PAGE_MASK; + rlim_stack = bprm->rlim_stack.rlim_cur & mm_page_mask(mm); stack_expand = min(rlim_stack, stack_size + stack_expand); @@ -1361,6 +1526,10 @@ void setup_new_exec(struct linux_binprm * bprm) arch_pick_mmap_layout(me->mm, &bprm->rlim_stack); + /* Consume the one-shot request only after committing to the new image. */ +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE + me->exec_page_shift = 0; +#endif arch_setup_new_exec(); /* Set the new mm task size. We have to do that late because it may @@ -1665,25 +1834,25 @@ int remove_arg_zero(struct linux_binprm *bprm) { unsigned long offset; char *kaddr; - struct page *page; + struct arg_page page; if (!bprm->argc) return 0; do { - offset = bprm->p & ~PAGE_MASK; + offset = bprm->p & ~bprm_copy_mask(bprm); page = get_arg_page(bprm, bprm->p, 0); - if (!page) + if (!page.page) return -EFAULT; - kaddr = kmap_local_page(page); + kaddr = map_arg_page(page); - for (; offset < PAGE_SIZE && kaddr[offset]; + for (; offset < bprm_copy_size(bprm) && kaddr[offset]; offset++, bprm->p++) ; kunmap_local(kaddr); put_arg_page(page); - } while (offset == PAGE_SIZE); + } while (offset == bprm_copy_size(bprm)); bprm->p++; bprm->argc--; From 2cf57b173bdc036f30c7b1e7100e32a8813bbcee Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0684/2680] block: preserve user fragment ownership across I/O MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Carry the backing offset and retained fragment reference through bio, splice and io_uring paths so a smaller user mapping is not treated as an entire native physical page. [ryan: adapted to 7.2.10 — minimal; for DJ: 7.2 dropped imu->acct_pages for per-ring compound-page refs (df0a52537c0f); fragment buffers keep their charge in struct io_fragment_buffer, and the unaccount/clone helpers skip fragment-backed buffers] Signed-off-by: Diljit Singh (cherry picked from commit c5f12989feb7fadc77031ea60829bdc879112864) --- block/bio.c | 150 +++++++++++++++++++++++++++++++++ block/blk-map.c | 2 +- fs/splice.c | 172 ++++++++++++++++++++++++++++++++++++++ include/linux/bio.h | 8 ++ include/linux/blk_types.h | 4 + io_uring/memmap.c | 8 +- io_uring/rsrc.c | 144 +++++++++++++++++++++++++++++++ 7 files changed, 486 insertions(+), 2 deletions(-) diff --git a/block/bio.c b/block/bio.c index bdcb60cb8d88f..e60b37c491333 100644 --- a/block/bio.c +++ b/block/bio.c @@ -3,6 +3,7 @@ * Copyright (C) 2001 Jens Axboe */ #include +#include #include #include #include @@ -246,6 +247,10 @@ void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table, #endif #ifdef CONFIG_BLK_DEV_INTEGRITY bio->bi_integrity = NULL; +#endif +#ifdef CONFIG_MM_SUBPAGE + bio->bi_user_fragments = NULL; + bio->bi_nr_user_fragments = 0; #endif bio->bi_vcnt = 0; @@ -1168,6 +1173,18 @@ void __bio_release_pages(struct bio *bio, bool mark_dirty) { struct folio_iter fi; +#ifdef CONFIG_MM_SUBPAGE + if (bio->bi_user_fragments) { + release_user_fragments(bio->bi_user_fragments, + bio->bi_nr_user_fragments, mark_dirty); + kfree(bio->bi_user_fragments); + bio->bi_user_fragments = NULL; + bio->bi_nr_user_fragments = 0; + bio_clear_flag(bio, BIO_PAGE_PINNED); + return; + } +#endif + bio_for_each_folio_all(fi, bio) { size_t nr_pages; @@ -1255,6 +1272,134 @@ static inline bool bio_iov_bvec_aligned(const struct bio *bio, } #endif +#ifdef CONFIG_MM_SUBPAGE +static bool bio_subpage_iter(const struct iov_iter *iter) +{ + return user_backed_iter(iter) && current->mm && + mm_page_size(current->mm) < PAGE_SIZE; +} + +int bio_iov_user_vecs(const struct iov_iter *iter, int max_vecs) +{ + struct iov_iter walk = *iter; + unsigned int count = 0; + + if (!bio_subpage_iter(iter)) + return iov_iter_npages(iter, max_vecs); + while (iov_iter_count(&walk) && count < max_vecs) { + size_t len, offset, nr; + + iov_iter_advance(&walk, 0); + len = min(iter_iov_len(&walk), iov_iter_count(&walk)); + if (WARN_ON_ONCE(!len)) + return max_vecs; + offset = (unsigned long)iter_iov_addr(&walk) & (MM_SUBPAGE_SIZE - 1); + nr = (len >> MM_SUBPAGE_SHIFT) + + DIV_ROUND_UP(offset + (len & (MM_SUBPAGE_SIZE - 1)), MM_SUBPAGE_SIZE); + if (nr >= max_vecs - count) + return max_vecs; + count += nr; + iov_iter_advance(&walk, len); + } + return count; +} +EXPORT_SYMBOL_GPL(bio_iov_user_vecs); + +/* Pins belong to the submitting bio, just as with ordinary BIO_PAGE_PINNED. + * Split/cloned bios borrow its vectors and complete before their owner does. + */ +static int bio_iov_iter_get_fragments(struct bio *bio, struct iov_iter *iter, + unsigned int align_mask) +{ + unsigned int capacity = bio->bi_max_vecs, total = 0; + unsigned int flags = iov_iter_rw(iter) == READ ? FOLL_WRITE : 0; + int ret = 0; + + if (WARN_ON_ONCE(bio->bi_vcnt || bio->bi_user_fragments || !capacity)) + return -EINVAL; + bio->bi_user_fragments = kmalloc_array(capacity, + sizeof(*bio->bi_user_fragments), GFP_KERNEL); + if (!bio->bi_user_fragments) + return -ENOMEM; + while (iov_iter_count(iter) && bio->bi_nr_user_fragments < capacity && + total < BIO_MAX_SIZE) { + unsigned int i, bytes = 0; + long nr; + + iov_iter_advance(iter, 0); + nr = pin_user_fragments_remote(current->mm, + (unsigned long)iter_iov_addr(iter), + min3(iter_iov_len(iter), iov_iter_count(iter), + (size_t)(BIO_MAX_SIZE - total)), flags, + bio->bi_user_fragments + bio->bi_nr_user_fragments, + capacity - bio->bi_nr_user_fragments); + if (nr <= 0) { + ret = nr ?: -EFAULT; + break; + } + for (i = 0; i < nr; i++) { + struct user_page_fragment *fragment = + &bio->bi_user_fragments[bio->bi_nr_user_fragments++]; + struct page *page = folio_page(fragment->folio, + fragment->offset >> PAGE_SHIFT); + unsigned int offset = offset_in_page(fragment->offset); + struct bio_vec *prev = bio->bi_vcnt ? + &bio->bi_io_vec[bio->bi_vcnt - 1] : NULL; + + /* Coalesce physically adjacent quarters within one folio. */ + if (prev && page_folio(prev->bv_page) == fragment->folio && + page_to_phys(prev->bv_page) + prev->bv_offset + prev->bv_len == + page_to_phys(page) + offset) + prev->bv_len += fragment->length; + else + bvec_set_page(&bio->bi_io_vec[bio->bi_vcnt++], page, + fragment->length, offset); + bytes += fragment->length; + } + iov_iter_advance(iter, bytes); + total += bytes; + } + /* Trim pins separately from vectors: several pins may share one bvec. */ + if (total & align_mask) { + unsigned int trim = total & align_mask, left = trim; + + while (left) { + struct user_page_fragment *fragment = + &bio->bi_user_fragments[bio->bi_nr_user_fragments - 1]; + unsigned int bytes = min(left, fragment->length); + + if (bytes == fragment->length) { + release_user_fragments(fragment, 1, false); + bio->bi_nr_user_fragments--; + } else { + fragment->length -= bytes; + } + left -= bytes; + } + left = trim; + while (left) { + struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; + unsigned int bytes = min(left, bv->bv_len); + + bv->bv_len -= bytes; + if (!bv->bv_len) + bio->bi_vcnt--; + left -= bytes; + } + iov_iter_revert(iter, trim); + total -= trim; + } + bio->bi_iter.bi_size = total; + if (!total) { + kfree(bio->bi_user_fragments); + bio->bi_user_fragments = NULL; + return ret ?: -EFAULT; + } + bio_set_flag(bio, BIO_PAGE_PINNED); + return 0; +} +#endif + /** * bio_iov_iter_get_pages - add user or kernel pages to a bio * @bio: bio to add pages to @@ -1296,6 +1441,11 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, return 0; } +#ifdef CONFIG_MM_SUBPAGE + if (bio_subpage_iter(iter)) + return bio_iov_iter_get_fragments(bio, iter, len_align_mask); +#endif + if (iov_iter_extract_will_pin(iter)) bio_set_flag(bio, BIO_PAGE_PINNED); if (bio->bi_bdev && blk_queue_pci_p2pdma(bio->bi_bdev->bd_disk->queue)) diff --git a/block/blk-map.c b/block/blk-map.c index 615d29bb840eb..8257c69a2ff44 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -260,7 +260,7 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, gfp_t gfp_mask) { - unsigned int nr_vecs = iov_iter_npages(iter, BIO_MAX_VECS); + unsigned int nr_vecs = bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS); struct bio *bio; int ret; diff --git a/fs/splice.c b/fs/splice.c index 9d8f63e2fd1ab..4b707ce00e86d 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include #include #include @@ -183,6 +185,79 @@ static void wakeup_pipe_readers(struct pipe_inode_info *pipe) kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); } +#ifdef CONFIG_MM_SUBPAGE +/* A shared lifetime for all user leaves coalesced into one native pipe page. */ +struct subpage_pipe_refs { + refcount_t refs; + unsigned int count; + struct user_page_fragment fragments[]; +}; + +static void subpage_pipe_buf_release(struct pipe_inode_info *pipe, struct pipe_buffer *buf) +{ + struct subpage_pipe_refs *refs = (void *)buf->private; + + if (refcount_dec_and_test(&refs->refs)) { + release_user_fragments(refs->fragments, refs->count, false); + kfree(refs); + } +} + +static bool subpage_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf) +{ + struct subpage_pipe_refs *refs = (void *)buf->private; + + return refcount_inc_not_zero(&refs->refs); +} + +static const struct pipe_buf_operations subpage_pipe_buf_ops = { + .release = subpage_pipe_buf_release, + .get = subpage_pipe_buf_get, + /* A gift of user leaves is not ownership of the containing native page. */ +}; + +static int subpage_pipe_buf_retain(struct pipe_buffer *buf) +{ + struct subpage_pipe_refs *refs; + unsigned int pos, count = 0; + + if (!buf->len || buf->ops != &nosteal_pipe_buf_ops || + !folio_test_anon(page_folio(buf->page))) + return 0; + refs = kmalloc(struct_size(refs, fragments, PAGE_SIZE / MM_SUBPAGE_SIZE), GFP_KERNEL); + if (!refs) + return -ENOMEM; + for (pos = round_down(buf->offset, MM_SUBPAGE_SIZE); + pos < buf->offset + buf->len; pos += MM_SUBPAGE_SIZE) { + struct mm_subpage *slot = mm_subpage_get_from_phys(page_to_phys(buf->page) + pos); + + if (!slot) { + release_user_fragments(refs->fragments, count, false); + kfree(refs); + return count ? -EFAULT : 0; + } + refs->fragments[count++] = (struct user_page_fragment) { + .folio = page_folio(buf->page), + .subpage = slot, + .offset = pos, + .length = MM_SUBPAGE_SIZE, + }; + } + refcount_set(&refs->refs, 1); + refs->count = count; + /* Replace the incoming native reference with the typed references. */ + put_page(buf->page); + buf->ops = &subpage_pipe_buf_ops; + buf->private = (unsigned long)refs; + return 0; +} +#else +static int subpage_pipe_buf_retain(struct pipe_buffer *buf) +{ + return 0; +} +#endif + /** * splice_to_pipe - fill passed data into a pipe * @pipe: pipe to fill @@ -221,6 +296,16 @@ ssize_t splice_to_pipe(struct pipe_inode_info *pipe, buf->private = spd->partial[page_nr].private; buf->ops = spd->ops; buf->flags = 0; + { + int err = subpage_pipe_buf_retain(buf); + + if (err) { + buf->ops = NULL; + if (!ret) + ret = err; + break; + } + } head++; pipe->head = head; @@ -852,6 +937,16 @@ ssize_t splice_to_socket(struct pipe_inode_info *pipe, struct file *out, continue; } +#ifdef CONFIG_MM_SUBPAGE + /* Other transports still need their downstream lifetime audit. */ + if (buf->ops == &subpage_pipe_buf_ops && + sock->ops->family != AF_UNIX && + sock->ops->family != AF_INET && + sock->ops->family != AF_INET6) { + ret = -EOPNOTSUPP; + break; + } +#endif seg = min_t(size_t, remain, buf->len); ret = pipe_buf_confirm(pipe, buf); @@ -1440,6 +1535,79 @@ static ssize_t __do_splice(struct file *in, loff_t __user *off_in, return ret; } +#ifdef CONFIG_MM_SUBPAGE +static ssize_t subpage_iter_to_pipe(struct iov_iter *from, + struct pipe_inode_info *pipe, unsigned int flags) +{ + struct user_page_fragment fragments[16]; + size_t total = 0; + long ret = 0; + + if (WARN_ON_ONCE(!user_backed_iter(from))) + return -EINVAL; + while (iov_iter_count(from)) { + unsigned int i = 0; + long count; + + /* Normalize empty iovecs before taking the next address. */ + iov_iter_advance(from, 0); + count = get_user_fragments_remote(current->mm, + (unsigned long)iter_iov_addr(from), + min(iov_iter_count(from), iter_iov_len(from)), 0, + fragments, ARRAY_SIZE(fragments)); + if (count <= 0) { + ret = count; + break; + } + while (i < count) { + struct user_page_fragment *first = &fragments[i]; + struct page *page = folio_page(first->folio, first->offset >> PAGE_SHIFT); + unsigned int offset = offset_in_page(first->offset); + unsigned int length = first->length, nr = 1; + struct subpage_pipe_refs *refs; + struct pipe_buffer buf; + + /* Preserve native pipe capacity for contiguous packed quarters. */ + while (i + nr < count && length < PAGE_SIZE - offset) { + struct user_page_fragment *next = &fragments[i + nr]; + + if (folio_page(next->folio, next->offset >> PAGE_SHIFT) != page || + offset_in_page(next->offset) != offset + length) + break; + length += next->length; + nr++; + } + refs = kmalloc(struct_size(refs, fragments, nr), GFP_KERNEL); + if (!refs) { + ret = -ENOMEM; + break; + } + refcount_set(&refs->refs, 1); + refs->count = nr; + memcpy(refs->fragments, first, sizeof(*first) * nr); + buf = (struct pipe_buffer) { + .page = page, + .offset = offset, + .len = length, + .ops = &subpage_pipe_buf_ops, + .flags = flags, + .private = (unsigned long)refs, + }; + i += nr; /* add_to_pipe consumes these references even on failure. */ + ret = add_to_pipe(pipe, &buf); + if (ret < 0) + break; + iov_iter_advance(from, ret); + total += ret; + } + release_user_fragments(fragments + i, count - i, false); + if (ret < 0) + break; + } + return total ? total : ret; +} +#endif + static ssize_t iter_to_pipe(struct iov_iter *from, struct pipe_inode_info *pipe, unsigned int flags) @@ -1451,6 +1619,10 @@ static ssize_t iter_to_pipe(struct iov_iter *from, size_t total = 0; ssize_t ret = 0; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return subpage_iter_to_pipe(from, pipe, flags); +#endif while (iov_iter_count(from)) { struct page *pages[16]; ssize_t left; diff --git a/include/linux/bio.h b/include/linux/bio.h index ce34ea49ef358..0fd444ceb92b6 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -398,11 +398,19 @@ static inline void bio_wouldblock_error(struct bio *bio) * pointed by @iter. If @iter is backed by bvec it's going to be reused * instead of allocating a new one. */ +#ifdef CONFIG_MM_SUBPAGE +int bio_iov_user_vecs(const struct iov_iter *iter, int max_vecs); +#endif + static inline int bio_iov_vecs_to_alloc(struct iov_iter *iter, int max_segs) { if (iov_iter_is_bvec(iter)) return 0; +#ifdef CONFIG_MM_SUBPAGE + return bio_iov_user_vecs(iter, max_segs); +#else return iov_iter_npages(iter, max_segs); +#endif } /** diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 8808ee76e73c0..cc9b1eeddd415 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -267,6 +267,10 @@ struct bio { struct bio_integrity_payload *bi_integrity; /* data integrity */ #endif +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment *bi_user_fragments; + unsigned short bi_nr_user_fragments; +#endif unsigned short bi_vcnt; /* how many bio_vec's */ /* diff --git a/io_uring/memmap.c b/io_uring/memmap.c index d4d7169469714..7e83c48ad8d96 100644 --- a/io_uring/memmap.c +++ b/io_uring/memmap.c @@ -312,7 +312,10 @@ static int io_region_mmap(struct io_ring_ctx *ctx, unsigned long nr_pages = min(mr->nr_pages, max_pages); vm_flags_set(vma, VM_DONTEXPAND); - return vm_insert_pages(vma, vma->vm_start, mr->pages, &nr_pages); + if (mm_page_size(vma->vm_mm) == PAGE_SIZE) + return vm_insert_pages(vma, vma->vm_start, mr->pages, &nr_pages); + return vm_insert_pages_range(vma, vma->vm_start, mr->pages, nr_pages, 0, + min_t(unsigned long, vma->vm_end - vma->vm_start, nr_pages << PAGE_SHIFT)); } __cold int io_uring_mmap(struct file *file, struct vm_area_struct *vma) @@ -324,6 +327,9 @@ __cold int io_uring_mmap(struct file *file, struct vm_area_struct *vma) struct io_mapped_region *region; void *ptr; + if (vma_subpage_offset(vma)) + return -EINVAL; + guard(mutex)(&ctx->mmap_lock); ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff); diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 174f74cbbf601..39d0d06ca38bc 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -174,6 +175,11 @@ static void io_free_imu(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu) kvfree(imu); } +#ifdef CONFIG_MM_SUBPAGE +static void io_release_fragment_buffer(void *priv); +static unsigned long io_fragment_buffer_charge(void *priv); +#endif + static unsigned long io_buffer_unaccount_pages(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu) { @@ -181,6 +187,11 @@ static unsigned long io_buffer_unaccount_pages(struct io_ring_ctx *ctx, unsigned long acct = 0; int i; +#ifdef CONFIG_MM_SUBPAGE + /* Fragment-backed buffers carry their own charge, not compound refs. */ + if (imu->release == io_release_fragment_buffer) + return io_fragment_buffer_charge(imu->priv); +#endif if (imu->flags & IO_REGBUF_F_KBUF || !ctx->user) return 0; @@ -852,6 +863,101 @@ bool io_check_coalesce_buffer(struct page **page_array, int nr_pages, return true; } +#ifdef CONFIG_MM_SUBPAGE +struct io_fragment_buffer { + unsigned int count; + unsigned long charged; + struct user_page_fragment fragments[]; +}; + +static unsigned long io_fragment_buffer_charge(void *priv) +{ + return ((struct io_fragment_buffer *)priv)->charged; +} + +static void io_release_fragment_buffer(void *priv) +{ + struct io_fragment_buffer *buffer = priv; + + release_user_fragments(buffer->fragments, buffer->count, false); + kvfree(buffer); +} + +static struct io_rsrc_node *io_register_fragment_buffer(struct io_ring_ctx *ctx, + struct iovec *iov) +{ + unsigned long size = mm_page_size(current->mm); + unsigned long first = (unsigned long)iov->iov_base & (size - 1); + unsigned int count = DIV_ROUND_UP(first + iov->iov_len, size); + struct io_fragment_buffer *buffer; + struct io_mapped_ubuf *imu = NULL; + struct io_rsrc_node *node; + unsigned long charged = 0; + unsigned int i; + struct xarray seen; + long nr; + int ret = -ENOMEM; + + node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); + if (!node) + return ERR_PTR(-ENOMEM); + buffer = kvmalloc(struct_size(buffer, fragments, count), GFP_KERNEL_ACCOUNT); + if (!buffer) + goto free_node; + buffer->count = 0; + buffer->charged = 0; + nr = pin_user_fragments_remote(current->mm, (unsigned long)iov->iov_base, + iov->iov_len, FOLL_WRITE | FOLL_LONGTERM, buffer->fragments, count); + if (nr > 0) + buffer->count = nr; + if (nr != count) { + ret = nr < 0 ? nr : -EFAULT; + goto free_buffer; + } + imu = io_alloc_imu(ctx, count); + if (!imu) + goto free_buffer; + imu->nr_bvecs = count; + /* Several user leaves can share native backing, including aliases. */ + xa_init(&seen); + for (i = 0; i < count; i++) { + struct user_page_fragment *f = &buffer->fragments[i]; + + ret = xa_insert(&seen, folio_pfn(f->folio), xa_mk_value(1), GFP_KERNEL); + if (!ret) + charged += folio_nr_pages(f->folio); + else if (ret != -EBUSY) + break; + bvec_set_page(&imu->bvec[i], folio_page(f->folio, f->offset >> PAGE_SHIFT), + f->length, offset_in_page(f->offset)); + } + xa_destroy(&seen); + if (ret && ret != -EBUSY) + goto free_imu; + ret = io_account_mem(ctx->user, ctx->mm_account, charged); + if (ret) + goto free_imu; + buffer->charged = charged; + imu->ubuf = (unsigned long)iov->iov_base; + imu->len = iov->iov_len; + imu->folio_shift = mm_page_shift(current->mm); + imu->release = io_release_fragment_buffer; + imu->priv = buffer; + imu->flags = 0; + imu->dir = IO_IMU_DEST | IO_IMU_SOURCE; + refcount_set(&imu->refs, 1); + node->buf = imu; + return node; +free_imu: + io_free_imu(ctx, imu); +free_buffer: + io_release_fragment_buffer(buffer); +free_node: + io_cache_free(&ctx->node_cache, node); + return ERR_PTR(ret); +} +#endif + static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov) { @@ -876,6 +982,11 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, if (ret) return ERR_PTR(ret); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) < PAGE_SIZE) + return io_register_fragment_buffer(ctx, iov); +#endif + node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); if (!node) return ERR_PTR(-ENOMEM); @@ -1182,6 +1293,14 @@ static int io_import_fixed(int ddir, struct iov_iter *iter, bvec += seg_skip; offset &= folio_mask; } +#ifdef CONFIG_MM_SUBPAGE + if (imu->folio_shift < PAGE_SHIFT) { + /* Physical fragment offsets are independent of the user address. */ + size_t first = min_t(size_t, len, bvec->bv_len - offset); + + nr_segs = 1 + DIV_ROUND_UP(len - first, 1UL << imu->folio_shift); + } else +#endif nr_segs = (offset + len + bvec->bv_offset + folio_mask) >> imu->folio_shift; iov_iter_bvec(iter, ddir, bvec, nr_segs, len); iter->iov_offset = offset; @@ -1231,6 +1350,10 @@ static int io_buffer_acct_cloned_hpages(struct io_ring_ctx *ctx, if (imu->flags & IO_REGBUF_F_KBUF || !ctx->user) return 0; +#ifdef CONFIG_MM_SUBPAGE + if (imu->release == io_release_fragment_buffer) + return 0; +#endif for (i = 0; i < imu->nr_bvecs; i++) { struct page *page = imu->bvec[i].bv_page; @@ -1501,6 +1624,27 @@ static int io_vec_fill_bvec(int ddir, struct iov_iter *iter, return -EOVERFLOW; offset = buf_addr - imu->ubuf; +#ifdef CONFIG_MM_SUBPAGE + if (imu->folio_shift < PAGE_SHIFT) { + unsigned int index; + unsigned long first = imu->ubuf & folio_mask; + + offset += first; + index = offset >> imu->folio_shift; + offset &= folio_mask; + if (!index) + offset -= first; + src_bvec = imu->bvec + index; + for (; iov_len; offset = 0, bvec_idx++, src_bvec++) { + size_t bytes = min_t(size_t, iov_len, src_bvec->bv_len - offset); + + bvec_set_page(&res_bvec[bvec_idx], src_bvec->bv_page, + bytes, src_bvec->bv_offset + offset); + iov_len -= bytes; + } + continue; + } +#endif /* * Only the first bvec can have non zero bv_offset, account it * here and work with full folios below. From 425b9fd576056a34c7f7cbb35ccc83101ea22d9a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0685/2680] net: retain user fragments in networking and BPF consumers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Preserve fragment ownership and byte ranges in socket, TLS, BPF and vhost consumers of user memory. [ryan: adapted to 7.2.10 — minimal; for DJ: the TLS split/merge/redirect hunks are dropped because 7.2 removed sockmap handling from the TLS SW path (79511603a65b); skb_splice_csum_page() already takes the batch offset in 7.2, so the progress argument is not needed; 7.2 removed the nouarg zerocopy encoding (2a0c9afefc0e), so its check is dropped] Signed-off-by: Diljit Singh (cherry picked from commit e6bf191ec8db185fcc908e5a5410497aac3a7543) --- drivers/vhost/vhost.c | 21 +++ include/linux/bpf.h | 22 ++++ include/linux/skmsg.h | 40 +++++- include/net/tls.h | 7 + kernel/bpf/bpf_local_storage.c | 23 ++-- kernel/bpf/syscall.c | 128 ++++++++++++++++-- net/core/skbuff.c | 112 ++++++++++++++++ net/core/skmsg.c | 88 ++++++++++++- net/ipv4/tcp_bpf.c | 5 + net/tls/tls_device.c | 22 ++++ net/tls/tls_sw.c | 233 +++++++++++++++++++++++++++++++-- 11 files changed, 664 insertions(+), 37 deletions(-) diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index f9511be1fa43f..0df2fef63f970 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -2491,6 +2492,26 @@ static int set_bit_to_user(int nr, void __user *addr) int bit = nr + (log % PAGE_SIZE) * 8; int r; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + struct user_page_fragment fragment; + unsigned long word = log & ~(sizeof(unsigned long) - 1); + + /* Pin the complete atomic word, which stays inside one leaf. + * A relocated leaf's physical offset need not match its VA. + */ + r = pin_user_fragments_remote(current->mm, word, + sizeof(unsigned long), FOLL_WRITE, &fragment, 1); + if (r != 1) + return r < 0 ? r : -EFAULT; + base = kmap_local_folio(fragment.folio, fragment.offset); + set_bit(nr + (log - word) * 8, base); + kunmap_local(base); + release_user_fragments(&fragment, 1, true); + return 0; + } +#endif + r = pin_user_pages_fast(log, 1, FOLL_WRITE, &page); if (r < 0) return r; diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 3ad3c2038c402..316b1dd24cdc0 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -267,6 +267,11 @@ struct btf_record { int wq_off; int refcount_off; int task_work_off; +#ifdef CONFIG_MM_SUBPAGE + /* Private ownership pointer appended to task-storage values with UPTRs. */ + u32 uptr_pins_off; + u32 uptr_extra_size; +#endif struct btf_field fields[]; }; @@ -596,6 +601,15 @@ static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src bpf_obj_memcpy(map->record, dst, src, round_up(map->value_size, 8), true); } +static inline u32 bpf_obj_uptr_extra_size(const struct btf_record *rec) +{ +#ifdef CONFIG_MM_SUBPAGE + return IS_ERR_OR_NULL(rec) ? 0 : rec->uptr_extra_size; +#else + return 0; +#endif +} + static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src) { unsigned long *src_uptr, *dst_uptr; @@ -613,6 +627,14 @@ static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, v dst_uptr = dst + field->offset; swap(*src_uptr, *dst_uptr); } +#ifdef CONFIG_MM_SUBPAGE + if (rec->uptr_extra_size) { + void **src_pins = src + rec->uptr_pins_off; + void **dst_pins = dst + rec->uptr_pins_off; + + swap(*src_pins, *dst_pins); + } +#endif } static inline void bpf_obj_memzero(struct btf_record *rec, void *dst, u32 size) diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h index d5e35f24738d5..6d7934c645ae2 100644 --- a/include/linux/skmsg.h +++ b/include/linux/skmsg.h @@ -41,6 +41,8 @@ struct sk_msg_sg { }; /* UAPI in filter.c depends on struct sk_msg_sg being first element. */ +struct mm_subpage_refs; + struct sk_msg { struct sk_msg_sg sg; void *data; @@ -49,6 +51,9 @@ struct sk_msg { u32 cork_bytes; u32 flags; struct sk_buff *skb; +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage_refs *subpage_refs; +#endif struct sock *sk_redir; struct sock *sk; struct list_head list; @@ -124,6 +129,33 @@ struct sk_psock { struct rcu_work rwork; }; +#ifdef CONFIG_MM_SUBPAGE +int sk_msg_subpages_add(struct sk_msg *msg, struct page *page, u32 offset, u32 len); +void sk_msg_subpages_get(struct sk_msg *msg); +void sk_msg_subpages_share(struct sk_msg *dst, const struct sk_msg *src); +void sk_msg_subpages_release(struct sk_msg *msg); +static inline void sk_msg_subpages_transfer(struct sk_msg *dst, struct sk_msg *src) +{ + dst->subpage_refs = src->subpage_refs; + src->subpage_refs = NULL; +} +#else +static inline int sk_msg_subpages_add(struct sk_msg *msg, struct page *page, u32 offset, u32 len) +{ + return 0; +} +static inline void sk_msg_subpages_get(struct sk_msg *msg) { } +static inline void sk_msg_subpages_share(struct sk_msg *dst, const struct sk_msg *src) { } +static inline void sk_msg_subpages_release(struct sk_msg *msg) { } +static inline void sk_msg_subpages_transfer(struct sk_msg *dst, struct sk_msg *src) { } +#endif + +static inline void sk_msg_subpages_drop_empty(struct sk_msg *msg) +{ + if (!msg->sg.size) + sk_msg_subpages_release(msg); +} + int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len, int elem_first_coalesce); int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src, @@ -259,10 +291,14 @@ static inline void sk_msg_compute_data_pointers(struct sk_msg *msg) } } -static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page, +static inline int sk_msg_page_add(struct sk_msg *msg, struct page *page, u32 len, u32 offset) { struct scatterlist *sge; + int ret = sk_msg_subpages_add(msg, page, offset, len); + + if (ret) + return ret; get_page(page); sge = sk_msg_elem(msg, msg->sg.end); @@ -272,6 +308,7 @@ static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page, __set_bit(msg->sg.end, msg->sg.copy); msg->sg.size += len; sk_msg_iter_next(msg, end); + return 0; } static inline void sk_msg_sg_copy(struct sk_msg *msg, u32 i, bool copy_state) @@ -420,6 +457,7 @@ static inline bool sk_psock_queue_empty(const struct sk_psock *psock) static inline void kfree_sk_msg(struct sk_msg *msg) { + sk_msg_subpages_release(msg); if (msg->skb) consume_skb(msg->skb); kfree(msg); diff --git a/include/net/tls.h b/include/net/tls.h index e57bef58851ea..555bd949d08e4 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -34,6 +34,7 @@ #ifndef _TLS_OFFLOAD_H #define _TLS_OFFLOAD_H +#include #include #include #include @@ -143,6 +144,9 @@ struct tls_sw_context_rx { struct tls_strparser strp; atomic_t decrypt_pending; +#ifdef CONFIG_MM_SUBPAGE + struct llist_head subpage_cleanup; +#endif struct sk_buff_head async_hold; struct wait_queue_head wq; }; @@ -152,6 +156,9 @@ struct tls_record_info { u32 end_seq; int len; int num_frags; +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage_refs *subpage_refs; +#endif skb_frag_t frags[MAX_SKB_FRAGS]; }; diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c index 6fc6a4b672b55..9abd4972966ff 100644 --- a/kernel/bpf/bpf_local_storage.c +++ b/kernel/bpf/bpf_local_storage.c @@ -24,6 +24,11 @@ select_bucket(struct bpf_local_storage_map *smap, return &smap->buckets[hash_ptr(local_storage, smap->bucket_log)]; } +static u32 bpf_selem_size(const struct bpf_local_storage_map *smap) +{ + return smap->elem_size + bpf_obj_uptr_extra_size(smap->map.record); +} + static int mem_charge(struct bpf_local_storage_map *smap, void *owner, u32 size) { struct bpf_map *map = &smap->map; @@ -72,10 +77,10 @@ bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, { struct bpf_local_storage_elem *selem; - if (mem_charge(smap, owner, smap->elem_size)) + if (mem_charge(smap, owner, bpf_selem_size(smap))) return NULL; - selem = bpf_map_kmalloc_nolock(&smap->map, smap->elem_size, + selem = bpf_map_kmalloc_nolock(&smap->map, bpf_selem_size(smap), __GFP_ZERO, NUMA_NO_NODE); if (selem) { @@ -91,7 +96,7 @@ bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, return selem; } - mem_uncharge(smap, owner, smap->elem_size); + mem_uncharge(smap, owner, bpf_selem_size(smap)); return NULL; } @@ -178,7 +183,7 @@ static void bpf_selem_unlink_storage_nolock_misc(struct bpf_local_storage_elem * bool free_local_storage, bool pin_owner) { void *owner = local_storage->owner; - u32 uncharge = smap->elem_size; + u32 uncharge = bpf_selem_size(smap); if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) == SDATA(selem)) @@ -204,7 +209,7 @@ static void bpf_selem_unlink_storage_nolock_misc(struct bpf_local_storage_elem * /* local_storage->lock must be held and selem->local_storage == local_storage. * The caller must ensure selem->smap is still valid to be - * dereferenced for its smap->elem_size and smap->cache_idx. + * dereferenced for its element size and smap->cache_idx. */ static bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage, struct bpf_local_storage_elem *selem, @@ -234,7 +239,7 @@ void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage, struct bpf_local_storage_map *smap; smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held()); - local_storage->mem_charge += smap->elem_size; + local_storage->mem_charge += bpf_selem_size(smap); RCU_INIT_POINTER(selem->local_storage, local_storage); hlist_add_head_rcu(&selem->snode, &local_storage->list); @@ -577,7 +582,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, err = bpf_local_storage_alloc(owner, smap, selem); if (err) { bpf_selem_free(selem, true); - mem_uncharge(smap, owner, smap->elem_size); + mem_uncharge(smap, owner, bpf_selem_size(smap)); return ERR_PTR(err); } @@ -661,7 +666,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, free_selem: bpf_selem_free_list(&old_selem_free_list, false); if (alloc_selem) { - mem_uncharge(smap, owner, smap->elem_size); + mem_uncharge(smap, owner, bpf_selem_size(smap)); bpf_selem_free(alloc_selem, true); } return err ? ERR_PTR(err) : SDATA(selem); @@ -862,7 +867,7 @@ void bpf_local_storage_map_free(struct bpf_map *map, * exit immediately. * * However, while freeing the storage one still needs to access the - * smap->elem_size to do the uncharging in + * bpf_selem_size(smap) to do the uncharging in * bpf_selem_unlink_storage_nolock(). * * Hence, wait another rcu grace period for the storage to be freed. diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index e92e251e67a9b..351f5dae52949 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include #include #include @@ -165,6 +167,43 @@ static void maybe_wait_bpf_programs(struct bpf_map *map) synchronize_rcu_expedited(); } +#ifdef CONFIG_MM_SUBPAGE +struct bpf_uptr_pins { + struct work_struct work; + u32 count; + struct user_page_fragment fragments[]; +}; + +static void bpf_uptr_release_work(struct work_struct *work) +{ + struct bpf_uptr_pins *pins = container_of(work, struct bpf_uptr_pins, work); + + release_user_fragments(pins->fragments, pins->count, true); + kfree(pins); +} +#endif + +/* May run from an RCU callback. The bundle owns its pins independently of + * the map, task and value, so dirty release can wait for folio locks in work. + */ +static bool bpf_obj_release_uptr_pins(const struct btf_record *rec, void *obj) +{ +#ifdef CONFIG_MM_SUBPAGE + struct bpf_uptr_pins **owner, *pins; + + if (!bpf_obj_uptr_extra_size(rec)) + return false; + owner = obj + rec->uptr_pins_off; + pins = *owner; + if (pins) { + *owner = NULL; + queue_work(system_unbound_wq, &pins->work); + return true; + } +#endif + return false; +} + static void unpin_uptr_kaddr(void *kaddr) { if (kaddr) @@ -177,6 +216,9 @@ static void __bpf_obj_unpin_uptrs(struct btf_record *rec, u32 cnt, void *obj) void **uptr_addr; int i; + if (bpf_obj_release_uptr_pins(rec, obj)) + return; + for (i = 0, field = rec->fields; i < cnt; i++, field++) { if (field->type != BPF_UPTR) continue; @@ -194,8 +236,12 @@ static void bpf_obj_unpin_uptrs(struct btf_record *rec, void *obj) __bpf_obj_unpin_uptrs(rec, rec->cnt, obj); } -static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj) +static int bpf_obj_pin_uptrs(struct bpf_map *map, void *obj) { + struct btf_record *rec = map->record; +#ifdef CONFIG_MM_SUBPAGE + struct bpf_uptr_pins *pins = NULL; +#endif const struct btf_field *field; const struct btf_type *t; unsigned long start, end; @@ -206,6 +252,18 @@ static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj) if (!btf_record_has_field(rec, BPF_UPTR)) return 0; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + pins = bpf_map_kzalloc(map, struct_size(pins, fragments, rec->cnt), + GFP_KERNEL); + if (!pins) + return -ENOMEM; + INIT_WORK(&pins->work, bpf_uptr_release_work); + pins->count = rec->cnt; + *(struct bpf_uptr_pins **)(obj + rec->uptr_pins_off) = pins; + } +#endif + for (i = 0, field = rec->fields; i < rec->cnt; i++, field++) { if (field->type != BPF_UPTR) continue; @@ -222,12 +280,35 @@ static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj) goto unpin_all; } - /* The uptr's struct cannot span across two pages */ - if ((start & PAGE_MASK) != (end & PAGE_MASK)) { + /* The object must fit in one page of the supplying process. */ + if ((start & mm_page_mask(current->mm)) != + (end & mm_page_mask(current->mm))) { err = -EOPNOTSUPP; goto unpin_all; } +#ifdef CONFIG_MM_SUBPAGE + if (pins) { + struct user_page_fragment *fragment = &pins->fragments[i]; + + err = pin_user_fragments_remote(current->mm, start, t->size, + FOLL_LONGTERM | FOLL_WRITE, fragment, 1); + if (err != 1) { + if (!err) + err = -EFAULT; + goto unpin_all; + } + if (fragment->length != t->size || + PageHighMem(folio_page(fragment->folio, + fragment->offset >> PAGE_SHIFT))) { + err = -EOPNOTSUPP; + goto unpin_all; + } + *uptr_addr = folio_address(fragment->folio) + fragment->offset; + continue; + } +#endif + err = pin_user_pages_fast(start, 1, FOLL_LONGTERM | FOLL_WRITE, &page); if (err != 1) goto unpin_all; @@ -292,7 +373,7 @@ static int bpf_map_update_value(struct bpf_map *map, struct file *map_file, map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) { err = map->ops->map_push_elem(map, value, flags); } else { - err = bpf_obj_pin_uptrs(map->record, value); + err = bpf_obj_pin_uptrs(map, value); if (!err) { rcu_read_lock(); err = map->ops->map_update_elem(map, key, value, flags); @@ -816,10 +897,12 @@ void bpf_obj_cancel_fields(struct bpf_map *map, void *obj) void bpf_obj_free_fields(const struct btf_record *rec, void *obj) { const struct btf_field *fields; + bool fragment_pins; int i; if (IS_ERR_OR_NULL(rec)) return; + fragment_pins = bpf_obj_release_uptr_pins(rec, obj); fields = rec->fields; for (i = 0; i < rec->cnt; i++) { struct btf_struct_meta *pointee_struct_meta; @@ -860,8 +943,9 @@ void bpf_obj_free_fields(const struct btf_record *rec, void *obj) } break; case BPF_UPTR: - /* The caller ensured that no one is using the uptr */ - unpin_uptr_kaddr(*(void **)field_ptr); + /* The caller ensured that no one is using the uptr. */ + if (!fragment_pins) + unpin_uptr_kaddr(*(void **)field_ptr); break; case BPF_LIST_HEAD: if (WARN_ON_ONCE(rec->spin_lock_off < 0)) @@ -1354,6 +1438,14 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, if (ret < 0) goto free_map_tab; +#ifdef CONFIG_MM_SUBPAGE + if (btf_record_has_field(map->record, BPF_UPTR)) { + map->record->uptr_pins_off = ALIGN(map->value_size, sizeof(void *)); + map->record->uptr_extra_size = map->record->uptr_pins_off + + sizeof(void *) - map->value_size; + } +#endif + if (map->ops->map_check_btf) { ret = map->ops->map_check_btf(map, btf, key_type, value_type); if (ret < 0) @@ -1841,6 +1933,24 @@ static int map_lookup_elem(union bpf_attr *attr) #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags +static void *bpf_map_memdup_value(struct bpf_map *map, bpfptr_t src, u32 size) +{ + u32 extra = bpf_obj_uptr_extra_size(map->record); + void *value; + + if (!extra) + return kvmemdup_bpfptr(src, size); + value = kvmalloc(size + extra, GFP_USER); + if (!value) + return ERR_PTR(-ENOMEM); + memset(value + size, 0, extra); + if (copy_from_bpfptr(value, src, size)) { + kvfree(value); + return ERR_PTR(-EFAULT); + } + return value; +} + static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) { bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel); @@ -1874,7 +1984,7 @@ static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) } value_size = bpf_map_value_size(map, attr->flags); - value = kvmemdup_bpfptr(uvalue, value_size); + value = bpf_map_memdup_value(map, uvalue, value_size); if (IS_ERR(value)) { err = PTR_ERR(value); goto free_key; @@ -2087,7 +2197,9 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file, if (!key) return -ENOMEM; - value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN); + value = kvmalloc(value_size + bpf_obj_uptr_extra_size(map->record), + GFP_USER | __GFP_NOWARN | + (bpf_obj_uptr_extra_size(map->record) ? __GFP_ZERO : 0)); if (!value) { kvfree(key); return -ENOMEM; diff --git a/net/core/skbuff.c b/net/core/skbuff.c index fa5c1c15d417e..1c1a491e75834 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -7400,6 +7401,111 @@ nodefer: kfree_skb_napi_cache(skb); kick_defer_list_purge(cpu); } +#ifdef CONFIG_MM_SUBPAGE +/* The ordinary frag page ref protects allocation, not an anonymous quarter. */ +struct skb_subpage_refs { + struct ubuf_info ubuf; + struct mm_subpage_refs *owner; +}; + +static void skb_subpage_complete(struct sk_buff *skb, struct ubuf_info *ubuf, + bool success) +{ + struct skb_subpage_refs *refs = container_of(ubuf, struct skb_subpage_refs, ubuf); + + if (!refcount_dec_and_test(&ubuf->refcnt)) + return; + mm_subpage_refs_put(refs->owner); + kfree(refs); +} + +static const struct ubuf_info_ops skb_subpage_ops = { + .complete = skb_subpage_complete, +}; + +static bool skb_contains_subpage(const struct sk_buff *skb, phys_addr_t phys, + unsigned int size, unsigned int depth) +{ + struct sk_buff *child; + unsigned int i; + + /* Match the networking recursion bound, conservatively keeping unknowns. */ + if (depth >= 24) + return true; + if (skb->head_frag && skb_headlen(skb)) { + phys_addr_t start = page_to_phys(virt_to_page(skb->data)) + offset_in_page(skb->data); + + if (start < phys + size && phys < start + skb_headlen(skb)) + return true; + } + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { + const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; + struct page *page = skb_frag_page(frag); + phys_addr_t start; + + if (!page) + continue; + start = page_to_phys(page) + skb_frag_off(frag); + if (start < phys + size && phys < start + skb_frag_size(frag)) + return true; + } + skb_walk_frags(skb, child) + if (skb_contains_subpage(child, phys, size, depth + 1)) + return true; + return false; +} + +static bool skb_keeps_subpage(const struct mm_subpage *slot, void *context) +{ + return skb_contains_subpage(context, mm_subpage_phys(slot), mm_subpage_size(slot), 0); +} + +/* Caller owns the writable skb and keeps the source slot identity alive. */ +static int skb_splice_hold_subpages(struct sk_buff *skb, struct page *page, + unsigned int off, unsigned int len) +{ + struct skb_subpage_refs *refs, *new; + struct ubuf_info *old = skb_zcopy(skb); + struct mm_subpage *probe; + int ret; + + if (!len || folio_order(page_folio(page)) || !folio_test_anon(page_folio(page))) + return 0; + /* Preserve the allocation-free native anonymous path. */ + probe = mm_subpage_get_from_phys(page_to_phys(page) + round_down(off, MM_SUBPAGE_SIZE)); + if (!probe) + return 0; + mm_subpage_put(probe); + if (old && old->ops != &skb_subpage_ops) + return -EOPNOTSUPP; + refs = old ? container_of(old, struct skb_subpage_refs, ubuf) : NULL; + if (!refs || refcount_read(&old->refcnt) != 1) { + new = kzalloc(sizeof(*new), GFP_ATOMIC); + if (!new) + return -ENOMEM; + new->ubuf.ops = &skb_subpage_ops; + new->ubuf.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; + refcount_set(&new->ubuf.refcnt, 1); + if (refs) + new->owner = mm_subpage_refs_get(refs->owner); + skb_zcopy_init(skb, &new->ubuf); + net_zcopy_put(old); + refs = new; + } + /* Repeated trim/append must not accumulate references to discarded data. */ + ret = mm_subpage_refs_prune(&refs->owner, skb_keeps_subpage, skb, GFP_ATOMIC); + if (ret) + return ret; + return mm_subpage_refs_add(&refs->owner, page, off, len, GFP_ATOMIC); +} +#else +static int skb_splice_hold_subpages(struct sk_buff *skb, struct page *page, + unsigned int off, unsigned int len) +{ + return 0; +} +#endif + static void skb_splice_csum_page(struct sk_buff *skb, struct page *page, size_t offset, size_t len, unsigned int csum_offset) @@ -7462,6 +7568,12 @@ ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter, if (WARN_ON_ONCE(!sendpage_ok(page))) goto out; + ret = skb_splice_hold_subpages(skb, page, off, part); + if (ret < 0) { + iov_iter_revert(iter, len); + goto out; + } + ret = skb_append_pagefrags(skb, page, off, part, frag_limit); if (ret < 0) { diff --git a/net/core/skmsg.c b/net/core/skmsg.c index df385a5a961e8..fbf049089776c 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */ +#include +#include #include #include #include @@ -23,6 +25,65 @@ static bool sk_msg_try_coalesce_ok(struct sk_msg *msg, int elem_first_coalesce) return false; } +#ifdef CONFIG_MM_SUBPAGE +static bool sk_msg_keeps_subpage(const struct mm_subpage *slot, void *context) +{ + struct sk_msg *msg = context; + phys_addr_t phys = mm_subpage_phys(slot); + phys_addr_t end = phys + mm_subpage_size(slot); + u32 i = msg->sg.start; + + if (!msg->sg.size) + return false; + while (i != msg->sg.end) { + struct scatterlist *sg = &msg->sg.data[i]; + + if (sg->length) { + phys_addr_t start = page_to_phys(sg_page(sg)) + sg->offset; + + if (start < end && phys < start + sg->length) + return true; + } + sk_msg_iter_var_next(i); + } + return false; +} + +int sk_msg_subpages_add(struct sk_msg *msg, struct page *page, u32 offset, u32 len) +{ + int ret; + + ret = mm_subpage_refs_prune(&msg->subpage_refs, sk_msg_keeps_subpage, msg, GFP_ATOMIC); + if (ret) + return ret; + return mm_subpage_refs_add(&msg->subpage_refs, page, offset, len, GFP_ATOMIC); +} +EXPORT_SYMBOL_GPL(sk_msg_subpages_add); + +/* Called after a borrowed descriptor copy whose source remains live. */ +void sk_msg_subpages_get(struct sk_msg *msg) +{ + mm_subpage_refs_get(msg->subpage_refs); +} +EXPORT_SYMBOL_GPL(sk_msg_subpages_get); + +void sk_msg_subpages_share(struct sk_msg *dst, const struct sk_msg *src) +{ + WARN_ON_ONCE(dst->subpage_refs); + dst->subpage_refs = mm_subpage_refs_get(src->subpage_refs); +} +EXPORT_SYMBOL_GPL(sk_msg_subpages_share); + +void sk_msg_subpages_release(struct sk_msg *msg) +{ + struct mm_subpage_refs *refs = msg->subpage_refs; + + msg->subpage_refs = NULL; + mm_subpage_refs_put(refs); +} +EXPORT_SYMBOL_GPL(sk_msg_subpages_release); +#endif + int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len, int elem_first_coalesce) { @@ -87,7 +148,7 @@ EXPORT_SYMBOL_GPL(sk_msg_alloc); int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src, u32 off, u32 len) { - int i = src->sg.start; + int i = src->sg.start, ret; struct scatterlist *sge = sk_msg_elem(src, i); struct scatterlist *sgd = NULL; u32 sge_len, sge_off; @@ -113,11 +174,16 @@ int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src, if (sgd && (sg_page(sge) == sg_page(sgd)) && (sg_virt(sge) + off == sg_virt(sgd) + sgd->length)) { + ret = sk_msg_subpages_add(dst, sg_page(sge), sge->offset + off, sge_len); + if (ret) + return ret; sgd->length += sge_len; dst->sg.size += sge_len; } else if (!sk_msg_full(dst)) { sge_off = sge->offset + off; - sk_msg_page_add(dst, sg_page(sge), sge_len, sge_off); + ret = sk_msg_page_add(dst, sg_page(sge), sge_len, sge_off); + if (ret) + return ret; } else { return -ENOSPC; } @@ -204,6 +270,7 @@ static int __sk_msg_free(struct sock *sk, struct sk_msg *msg, u32 i, sk_msg_check_to_free(msg, i, msg->sg.size); sge = sk_msg_elem(msg, i); } + sk_msg_subpages_release(msg); consume_skb(msg->skb); sk_msg_init(msg); return freed; @@ -247,6 +314,7 @@ static void __sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, sk_msg_check_to_free(msg, i, bytes); } msg->sg.start = i; + sk_msg_subpages_drop_empty(msg); } void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes) @@ -266,6 +334,7 @@ void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len) int trim = msg->sg.size - len; u32 i = msg->sg.end; + sk_msg_subpages_drop_empty(msg); if (trim <= 0) { WARN_ON(trim < 0); return; @@ -298,6 +367,7 @@ void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len) * does not require an update. */ if (!msg->sg.size) { + sk_msg_subpages_release(msg); msg->sg.curr = msg->sg.start; msg->sg.copybreak = 0; } else if (sk_msg_iter_dist(msg->sg.start, msg->sg.curr) >= @@ -333,11 +403,19 @@ int sk_msg_zerocopy_from_iter(struct sock *sk, struct iov_iter *from, goto out; } - bytes -= copied; - msg->sg.size += copied; - while (copied) { use = min_t(int, copied, PAGE_SIZE - offset); + ret = sk_msg_subpages_add(msg, pages[i], offset, use); + if (ret) { + unsigned int nr = DIV_ROUND_UP(offset + copied, PAGE_SIZE); + + while (nr--) + put_page(pages[i++]); + iov_iter_revert(from, copied); + goto out; + } + bytes -= use; + msg->sg.size += use; sg_set_page(&msg->sg.data[msg->sg.end], pages[i], use, offset); sg_unmark_end(&msg->sg.data[msg->sg.end]); diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c index 2e234d155b5e6..6f50f9125f90c 100644 --- a/net/ipv4/tcp_bpf.c +++ b/net/ipv4/tcp_bpf.c @@ -44,6 +44,7 @@ static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock, return -ENOMEM; lock_sock(sk); + sk_msg_subpages_share(tmp, msg); tmp->sg.start = msg->sg.start; i = msg->sg.start; do { @@ -76,6 +77,7 @@ static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock, if (!ret) { msg->sg.start = i; + sk_msg_subpages_drop_empty(msg); if (!sk_psock_queue_msg(psock, tmp)) atomic_sub(copied, &sk->sk_rmem_alloc); sk_psock_data_ready(sk, psock); @@ -147,6 +149,7 @@ static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes, break; } + sk_msg_subpages_drop_empty(msg); return 0; } @@ -449,6 +452,8 @@ static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock, } } memcpy(psock->cork, msg, sizeof(*msg)); + if (psock->cork != msg) + sk_msg_subpages_transfer(psock->cork, msg); return 0; } diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index f11d0528fc431..7d2727264ed39 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -32,6 +32,8 @@ #include #include #include +#include +#include #include #include #include @@ -144,6 +146,10 @@ static void destroy_record(struct tls_record_info *record) for (i = 0; i < record->num_frags; i++) __skb_frag_unref(&record->frags[i], false); + +#ifdef CONFIG_MM_SUBPAGE + mm_subpage_refs_put(record->subpage_refs); +#endif kfree(record); } @@ -349,6 +355,9 @@ static int tls_create_new_record(struct tls_offload_context_tx *offload_ctx, record = kmalloc_obj(*record); if (!record) return -ENOMEM; +#ifdef CONFIG_MM_SUBPAGE + record->subpage_refs = NULL; +#endif frag = &record->frags[0]; skb_frag_fill_page_desc(frag, pfrag->page, pfrag->offset, @@ -513,6 +522,15 @@ static int tls_push_data(struct sock *sk, goto handle_error; } +#ifdef CONFIG_MM_SUBPAGE + rc = mm_subpage_refs_add(&record->subpage_refs, zc_pfrag.page, + off, copy, GFP_KERNEL); + if (rc) { + iov_iter_revert(iter, copy); + goto handle_error; + } +#endif + zc_pfrag.offset = off; zc_pfrag.size = copy; tls_append_frag(record, &zc_pfrag, copy); @@ -1119,6 +1137,10 @@ int tls_set_device_offload(struct sock *sk) goto release_netdev; } + +#ifdef CONFIG_MM_SUBPAGE + start_marker_record->subpage_refs = NULL; +#endif offload_ctx = alloc_offload_ctx_tx(ctx); if (!offload_ctx) { rc = -ENOMEM; diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index d1ad31986cf2c..d1ddad907413e 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -35,8 +35,11 @@ * SOFTWARE. */ +#include #include #include +#include +#include #include #include #include @@ -65,9 +68,36 @@ struct tls_decrypt_ctx { u8 aad[TLS_MAX_AAD_SIZE]; u8 tail; bool free_sgout; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment *fragments; + unsigned int nr_fragments; + bool fragments_dirty; + struct llist_node cleanup; + void *allocation; +#endif struct scatterlist sg[]; }; +#if IS_ENABLED(CONFIG_KUNIT) && IS_ENABLED(CONFIG_MM_SUBPAGE) +/* Explicit test-only async provider selection; normal cipher selection stays intact. */ +static bool user4k_test_async; +module_param_named(user4k_test_async, user4k_test_async, bool, 0400); +MODULE_PARM_DESC(user4k_test_async, "KUnit: select cryptd for TLS receive testing"); +static atomic_long_t user4k_rx_deferred = ATOMIC_LONG_INIT(0); +static atomic_long_t user4k_rx_released = ATOMIC_LONG_INIT(0); + +static int user4k_read_count(char *buffer, const struct kernel_param *param) +{ + return scnprintf(buffer, PAGE_SIZE, "%ld\n", atomic_long_read(param->arg)); +} + +static const struct kernel_param_ops user4k_count_ops = { + .get = user4k_read_count, +}; +module_param_cb(user4k_rx_deferred, &user4k_count_ops, &user4k_rx_deferred, 0444); +module_param_cb(user4k_rx_released, &user4k_count_ops, &user4k_rx_released, 0444); +#endif + noinline void tls_err_abort(struct sock *sk, int err) { WARN_ON_ONCE(err >= 0); @@ -184,6 +214,35 @@ static int tls_padding_length(struct tls_prot_info *prot, struct sk_buff *skb, return sub; } +#ifdef CONFIG_MM_SUBPAGE +static void tls_release_user_fragments(struct tls_decrypt_ctx *dctx) +{ + release_user_fragments(dctx->fragments, dctx->nr_fragments, dctx->fragments_dirty); + kfree(dctx->fragments); + dctx->fragments = NULL; + dctx->nr_fragments = 0; +} +#endif + +/* Dirty pin release can take folio locks, so it belongs to the waiting task. */ +static void tls_finish_subpage_decrypts(struct tls_sw_context_rx *ctx) +{ +#ifdef CONFIG_MM_SUBPAGE + struct tls_decrypt_ctx *dctx, *next; + struct llist_node *head = llist_del_all(&ctx->subpage_cleanup); + + llist_for_each_entry_safe(dctx, next, head, cleanup) { + void *allocation = dctx->allocation; + + tls_release_user_fragments(dctx); +#if IS_ENABLED(CONFIG_KUNIT) + atomic_long_inc(&user4k_rx_released); +#endif + kfree(allocation); + } +#endif +} + static void tls_decrypt_done(void *data, int err) { struct aead_request *aead_req = data; @@ -196,6 +255,7 @@ static void tls_decrypt_done(void *data, int err) unsigned int pages; struct sock *sk; int aead_size; + bool deferred = false; /* If requests get too backlogged crypto API returns -EBUSY and calls * ->complete(-EINPROGRESS) immediately followed by ->complete(0) @@ -224,17 +284,29 @@ static void tls_decrypt_done(void *data, int err) tls_err_abort(sk, err); } - /* Free the destination pages if skb was not decrypted inplace */ - if (dctx->free_sgout) { - /* Skip the first S/G entry as it points to AAD */ - for_each_sg(sg_next(sgout), sg, UINT_MAX, pages) { - if (!sg) - break; - put_page(sg_page(sg)); +#ifdef CONFIG_MM_SUBPAGE + if (dctx->fragments) { + dctx->allocation = aead_req; +#if IS_ENABLED(CONFIG_KUNIT) + atomic_long_inc(&user4k_rx_deferred); +#endif + llist_add(&dctx->cleanup, &ctx->subpage_cleanup); + deferred = true; + } +#endif + if (!deferred) { + /* Free the destination pages if skb was not decrypted inplace */ + if (dctx->free_sgout) { + /* Skip the first S/G entry as it points to AAD */ + for_each_sg(sg_next(sgout), sg, UINT_MAX, pages) { + if (!sg) + break; + put_page(sg_page(sg)); + } } - } - kfree(aead_req); + kfree(aead_req); + } if (atomic_dec_and_test(&ctx->decrypt_pending)) complete(&ctx->async_wait.completion); @@ -246,6 +318,7 @@ static int tls_decrypt_async_wait(struct tls_sw_context_rx *ctx) crypto_wait_req(-EINPROGRESS, &ctx->async_wait); atomic_inc(&ctx->decrypt_pending); + tls_finish_subpage_decrypts(ctx); __skb_queue_purge(&ctx->async_hold); return ctx->async_wait.err; } @@ -752,7 +825,14 @@ static int tls_sw_sendmsg_splice(struct sock *sk, struct msghdr *msg, return -EIO; } - sk_msg_page_add(msg_pl, page, part, off); + { + int ret = sk_msg_page_add(msg_pl, page, part, off); + + if (ret) { + iov_iter_revert(&msg->msg_iter, part); + return ret; + } + } msg_pl->sg.copybreak = 0; msg_pl->sg.curr = msg_pl->sg.end; sk_mem_charge(sk, part); @@ -1165,10 +1245,96 @@ tls_rx_rec_wait(struct sock *sk, bool nonblock, bool released, bool has_copied) return 1; } +#ifdef CONFIG_MM_SUBPAGE +static bool tls_subpage_iter(const struct iov_iter *iter) +{ + return user_backed_iter(iter) && current->mm && mm_page_size(current->mm) < PAGE_SIZE; +} +#endif + +static int tls_iter_sg_capacity(struct iov_iter *iter, size_t length) +{ +#ifdef CONFIG_MM_SUBPAGE + if (tls_subpage_iter(iter)) { + struct iov_iter walk = *iter; + unsigned int count = 0; + + length = min(length, iov_iter_count(iter)); + while (length) { + size_t bytes, offset; + + iov_iter_advance(&walk, 0); + bytes = min(length, iter_iov_len(&walk)); + if (WARN_ON_ONCE(!bytes)) + return INT_MAX; + offset = (unsigned long)iter_iov_addr(&walk) & (MM_SUBPAGE_SIZE - 1); + count += DIV_ROUND_UP(offset + bytes, MM_SUBPAGE_SIZE); + iov_iter_advance(&walk, bytes); + length -= bytes; + } + return count; + } +#endif + return iov_iter_npages_cap(iter, INT_MAX, length); +} + +#ifdef CONFIG_MM_SUBPAGE +static int tls_setup_subpage_iter(struct iov_iter *from, int length, int *pages_used, + struct scatterlist *to, int capacity, + struct tls_decrypt_ctx *dctx) +{ + unsigned int used = 0, total = 0; + int ret = 0; + + if (!length) + return 0; + if (WARN_ON_ONCE(*pages_used || capacity <= 0)) + return -EINVAL; + dctx->fragments = kmalloc_array(capacity, sizeof(*dctx->fragments), GFP_KERNEL); + if (!dctx->fragments) + return -ENOMEM; + while (length) { + long nr; + unsigned int i, bytes = 0; + + iov_iter_advance(from, 0); + nr = pin_user_fragments_remote(current->mm, + (unsigned long)iter_iov_addr(from), + min_t(size_t, length, iter_iov_len(from)), FOLL_WRITE, + dctx->fragments + used, capacity - used); + if (nr <= 0) { + ret = nr ?: -EFAULT; + goto out; + } + for (i = 0; i < nr; i++) { + struct user_page_fragment *fragment = &dctx->fragments[used]; + + sg_set_page(&to[used], + folio_page(fragment->folio, fragment->offset >> PAGE_SHIFT), + fragment->length, offset_in_page(fragment->offset)); + sg_unmark_end(&to[used]); + bytes += fragment->length; + used++; + } + dctx->nr_fragments = used; + iov_iter_advance(from, bytes); + length -= bytes; + total += bytes; + } + if (used) + sg_mark_end(&to[used - 1]); +out: + if (ret) + iov_iter_revert(from, total); + *pages_used = used; + return ret; +} +#endif + static int tls_setup_from_iter(struct iov_iter *from, int length, int *pages_used, struct scatterlist *to, - int to_max_pages) + int to_max_pages, struct tls_decrypt_ctx *dctx) { int rc = 0, i = 0, num_elem = *pages_used, maxpages; struct page *pages[MAX_SKB_FRAGS]; @@ -1176,6 +1342,10 @@ static int tls_setup_from_iter(struct iov_iter *from, ssize_t copied, use; size_t offset; +#ifdef CONFIG_MM_SUBPAGE + if (tls_subpage_iter(from)) + return tls_setup_subpage_iter(from, length, pages_used, to, to_max_pages, dctx); +#endif while (length > 0) { i = 0; maxpages = to_max_pages - num_elem; @@ -1293,7 +1463,7 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, if (out_iov) n_sgout = 1 + tail_pages + - iov_iter_npages_cap(out_iov, INT_MAX, data_len); + tls_iter_sg_capacity(out_iov, data_len); else n_sgout = sg_nents(out_sg); } else { @@ -1326,6 +1496,11 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, aead_req = (struct aead_request *)mem; dctx = (struct tls_decrypt_ctx *)(mem + aead_size); dctx->sk = sk; +#ifdef CONFIG_MM_SUBPAGE + dctx->fragments = NULL; + dctx->nr_fragments = 0; + dctx->fragments_dirty = false; +#endif sgin = &dctx->sg[0]; sgout = &dctx->sg[n_sgin]; @@ -1383,7 +1558,7 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, sg_set_buf(&sgout[0], dctx->aad, prot->aad_size); err = tls_setup_from_iter(out_iov, data_len, &pages, &sgout[1], - (n_sgout - 1 - tail_pages)); + (n_sgout - 1 - tail_pages), dctx); if (err < 0) goto exit_free_pages; @@ -1398,6 +1573,10 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, } dctx->free_sgout = !!pages; +#ifdef CONFIG_MM_SUBPAGE + /* An error from crypto may still leave writes in the destination. */ + dctx->fragments_dirty = true; +#endif /* Prepare and submit AEAD request */ err = tls_do_decryption(sk, sgin, sgout, dctx->iv, data_len + prot->tail_size, aead_req, darg); @@ -1426,6 +1605,12 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, darg->tail = dctx->tail; exit_free_pages: +#ifdef CONFIG_MM_SUBPAGE + if (dctx->fragments) { + tls_release_user_fragments(dctx); + pages = 0; + } +#endif /* Release the pages in case iov was mapped to pages */ for (; pages > 0; pages--) put_page(sg_page(&sgout[pages])); @@ -2475,6 +2660,9 @@ static struct tls_sw_context_rx *init_ctx_rx(struct tls_context *ctx) crypto_init_wait(&sw_ctx_rx->async_wait); atomic_set(&sw_ctx_rx->decrypt_pending, 1); +#ifdef CONFIG_MM_SUBPAGE + init_llist_head(&sw_ctx_rx->subpage_cleanup); +#endif init_waitqueue_head(&sw_ctx_rx->wq); skb_queue_head_init(&sw_ctx_rx->rx_list); skb_queue_head_init(&sw_ctx_rx->async_hold); @@ -2583,7 +2771,24 @@ int tls_set_sw_offload(struct sock *sk, int tx, rec_seq = crypto_info_rec_seq(src_crypto_info, cipher_desc); if (!*aead) { - *aead = crypto_alloc_aead(cipher_desc->cipher_name, 0, 0); +#if IS_ENABLED(CONFIG_KUNIT) && IS_ENABLED(CONFIG_MM_SUBPAGE) + if (!tx && user4k_test_async) { + struct crypto_aead *sync; + char name[CRYPTO_MAX_ALG_NAME]; + + /* cryptd instances are named for the concrete child driver. */ + sync = crypto_alloc_aead(cipher_desc->cipher_name, 0, CRYPTO_ALG_ASYNC); + if (IS_ERR(sync)) { + *aead = sync; + } else { + snprintf(name, sizeof(name), "cryptd(%s)", + crypto_tfm_alg_driver_name(crypto_aead_tfm(sync))); + *aead = crypto_alloc_aead(name, CRYPTO_ALG_ASYNC, CRYPTO_ALG_ASYNC); + crypto_free_aead(sync); + } + } else +#endif + *aead = crypto_alloc_aead(cipher_desc->cipher_name, 0, 0); if (IS_ERR(*aead)) { rc = PTR_ERR(*aead); *aead = NULL; From 200a4da7e0fee98e5ae40c179b34995525eb763f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0686/2680] iommu: carry exact user fragments through device mappings Convert IOMMUFD, VFIO and GEM/DMA-BUF paths to preserve the backing offset, mapping span and ownership of smaller user pages. Signed-off-by: Diljit Singh (cherry picked from commit 25dc8bcddb0298f46cbe5d335f6b56388e43e198) --- drivers/dma-buf/dma-buf.c | 35 +- drivers/gpu/drm/drm_gem.c | 7 +- drivers/gpu/drm/drm_gem_shmem_helper.c | 32 +- drivers/iommu/iommufd/Makefile | 2 + drivers/iommu/iommufd/device.c | 102 +++- drivers/iommu/iommufd/fragments.c | 421 +++++++++++++++ drivers/iommu/iommufd/io_pagetable.c | 15 +- drivers/iommu/iommufd/io_pagetable.h | 77 ++- drivers/iommu/iommufd/iommufd_private.h | 4 + drivers/iommu/iommufd/iova_bitmap.c | 131 ++++- drivers/iommu/iommufd/pages.c | 99 +++- drivers/iommu/iommufd/selftest.c | 61 ++- drivers/iommu/iommufd/viommu.c | 56 +- drivers/vfio/vfio_iommu_type1.c | 534 +++++++++++++------ drivers/vfio/vfio_iommu_type1_fragments.c | 609 ++++++++++++++++++++++ 15 files changed, 1918 insertions(+), 267 deletions(-) create mode 100644 drivers/iommu/iommufd/fragments.c create mode 100644 drivers/vfio/vfio_iommu_type1_fragments.c diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index 4c9add51f9ef5..aef376fbfae91 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -225,9 +225,23 @@ static struct file_system_type dma_buf_fs_type = { .kill_sb = kill_anon_super, }; +static int dma_buf_mmap_range_check(struct dma_buf *dmabuf, + struct vm_area_struct *vma, unsigned long pgoff, unsigned int suboffset) +{ + unsigned long offset; + + if (check_mul_overflow(pgoff, PAGE_SIZE, &offset) || + check_add_overflow(offset, (unsigned long)suboffset, &offset)) + return -EOVERFLOW; + if (offset > dmabuf->size || vma->vm_end - vma->vm_start > dmabuf->size - offset) + return -EINVAL; + return 0; +} + static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma) { struct dma_buf *dmabuf; + int ret; if (!is_dma_buf_file(file)) return -EINVAL; @@ -239,9 +253,9 @@ static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma) return -EINVAL; /* check for overflowing the buffer's size */ - if (vma->vm_pgoff + vma_pages(vma) > - dmabuf->size >> PAGE_SHIFT) - return -EINVAL; + ret = dma_buf_mmap_range_check(dmabuf, vma, vma->vm_pgoff, vma_subpage_offset(vma)); + if (ret) + return ret; DMA_BUF_TRACE(trace_dma_buf_mmap_internal, dmabuf); @@ -1603,6 +1617,8 @@ EXPORT_SYMBOL_NS_GPL(dma_buf_end_cpu_access, "DMA_BUF"); int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma, unsigned long pgoff) { + int ret; + if (WARN_ON(!dmabuf || !vma)) return -EINVAL; @@ -1610,18 +1626,13 @@ int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma, if (!dmabuf->ops->mmap) return -EINVAL; - /* check for offset overflow */ - if (pgoff + vma_pages(vma) < pgoff) - return -EOVERFLOW; - - /* check for overflowing the buffer's size */ - if (pgoff + vma_pages(vma) > - dmabuf->size >> PAGE_SHIFT) - return -EINVAL; + ret = dma_buf_mmap_range_check(dmabuf, vma, pgoff, 0); + if (ret) + return ret; /* readjust the vma */ vma_set_file(vma, dmabuf->file); - vma->vm_pgoff = pgoff; + vma_set_page_offset(vma, (struct vm_page_offset) { .index = pgoff }); DMA_BUF_TRACE(trace_dma_buf_mmap, dmabuf); diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c index 018df97d590dc..0a700d19c4422 100644 --- a/drivers/gpu/drm/drm_gem.c +++ b/drivers/gpu/drm/drm_gem.c @@ -1342,7 +1342,7 @@ unsigned long drm_gem_get_unmapped_area(struct file *filp, unsigned long uaddr, struct drm_gem_object *obj; unsigned long ret; - obj = drm_gem_object_lookup_at_offset(filp, pgoff, len >> PAGE_SHIFT); + obj = drm_gem_object_lookup_at_offset(filp, pgoff, DIV_ROUND_UP(len, PAGE_SIZE)); if (IS_ERR(obj)) obj = NULL; @@ -1378,8 +1378,11 @@ int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma) struct drm_gem_object *obj; int ret; + /* GEM fake offsets identify an exact object start, not an interior byte. */ + if (vma_subpage_offset(vma)) + return -EINVAL; obj = drm_gem_object_lookup_at_offset(filp, vma->vm_pgoff, - vma_pages(vma)); + DIV_ROUND_UP(vma->vm_end - vma->vm_start, PAGE_SIZE)); if (IS_ERR(obj)) return PTR_ERR(obj); diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index 4b5ec19a2ca9b..c2cb32dd58621 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -574,13 +574,23 @@ int drm_gem_shmem_dumb_create(struct drm_file *file, struct drm_device *dev, } EXPORT_SYMBOL_GPL(drm_gem_shmem_dumb_create); +/* GEM and PRIME add the object's fake offset; VMA splits retain it. + * Direct helper callers may instead supply an object-relative offset. + */ +static pgoff_t drm_gem_shmem_object_offset(struct drm_gem_object *obj, pgoff_t pgoff) +{ + pgoff_t base = drm_vma_node_start(&obj->vma_node); + + return pgoff >= base ? pgoff - base : pgoff; +} + static void drm_gem_shmem_record_mkwrite(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; struct drm_gem_object *obj = vma->vm_private_data; struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); loff_t num_pages = obj->size >> PAGE_SHIFT; - pgoff_t page_offset = vmf->pgoff - vma->vm_pgoff; /* page offset within VMA */ + pgoff_t page_offset = drm_gem_shmem_object_offset(obj, vmf->pgoff); if (drm_WARN_ON(obj->dev, !shmem->pages || page_offset >= num_pages)) return; @@ -607,7 +617,10 @@ static vm_fault_t try_insert_pfn(struct vm_fault *vmf, unsigned int order, vm_fault_t ret = VM_FAULT_FALLBACK; if (!order) { - ret = vmf_insert_pfn_mkwrite(vmf->vma, vmf->address, pfn, write); + struct vm_page_offset pos = vma_page_offset_at(vmf->vma, vmf->address); + + ret = vmf_insert_pfn_prot_mkwrite_offset(vmf->vma, vmf->address, + pfn, pos.offset, vmf->vma->vm_page_prot, write); #ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP } else if (order == PMD_ORDER) { unsigned long paddr = pfn << PAGE_SHIFT; @@ -639,12 +652,12 @@ static vm_fault_t drm_gem_shmem_any_fault(struct vm_fault *vmf, unsigned int ord loff_t num_pages = obj->size >> PAGE_SHIFT; vm_fault_t ret = VM_FAULT_SIGBUS; struct page **pages = shmem->pages; - pgoff_t page_offset = vmf->pgoff - vma->vm_pgoff; /* page offset within VMA */ + pgoff_t page_offset = drm_gem_shmem_object_offset(obj, vmf->pgoff); struct page *page; struct folio *folio; unsigned long pfn; - if (order && order != PMD_ORDER) + if (order && (order != PMD_ORDER || mm_page_size(vma->vm_mm) != PAGE_SIZE)) return VM_FAULT_FALLBACK; dma_resv_lock(obj->resv, NULL); @@ -762,6 +775,17 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct if (is_cow_mapping(vma->vm_flags)) return -EINVAL; + /* Validate bytes, including a partial native-page offset from PRIME. */ + { + pgoff_t index = drm_gem_shmem_object_offset(obj, vma->vm_pgoff); + unsigned long offset, length = vma->vm_end - vma->vm_start; + + if (check_mul_overflow(index, PAGE_SIZE, &offset) || + check_add_overflow(offset, (unsigned long)vma_subpage_offset(vma), &offset) || + offset > obj->size || length > obj->size - offset) + return -EINVAL; + } + dma_resv_lock(shmem->base.resv, NULL); ret = drm_gem_shmem_get_pages_locked(shmem); dma_resv_unlock(shmem->base.resv); diff --git a/drivers/iommu/iommufd/Makefile b/drivers/iommu/iommufd/Makefile index 71d692c9a8f49..a868155e208d5 100644 --- a/drivers/iommu/iommufd/Makefile +++ b/drivers/iommu/iommufd/Makefile @@ -10,6 +10,8 @@ iommufd-y := \ vfio_compat.o \ viommu.o +iommufd-$(CONFIG_MM_SUBPAGE) += fragments.o + iommufd-$(CONFIG_IOMMUFD_TEST) += selftest.o obj-$(CONFIG_IOMMUFD) += iommufd.o diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c index c5d122f33ba2d..7488279cf3bef 100644 --- a/drivers/iommu/iommufd/device.c +++ b/drivers/iommu/iommufd/device.c @@ -1164,6 +1164,18 @@ struct iommufd_access *iommufd_access_create_internal(struct iommufd_ctx *ictx) return access; } +/* Physical byte-span consumers do not impose native-page IOVA alignment. */ +struct iommufd_access *iommufd_access_create_internal_phys(struct iommufd_ctx *ictx) +{ + struct iommufd_access *access = __iommufd_access_create(ictx); + + if (IS_ERR(access)) + return access; + access->iova_alignment = 1; + iommufd_object_finalize(ictx, &access->obj); + return access; +} + /** * iommufd_access_create - Create an iommufd_access * @ictx: iommufd file descriptor @@ -1397,6 +1409,12 @@ static bool check_area_prot(struct iopt_area *area, unsigned int flags) * ioas alignment is >= PAGE_SIZE and the iova is PAGE_SIZE aligned. However * smaller alignments have corner cases where this API can fail on otherwise * aligned iova. + * + * Alternative user granules are accepted only if each native output page has + * an aligned physical base and consecutive backing bytes within the requested + * range. The driver must access only @length bytes: a partial final page does + * not grant access to its remaining bytes. Fragment ownership stays in the IOAS + * until unpin; incompatible physical layouts return -EINVAL. */ int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova, unsigned long length, struct page **out_pages, @@ -1449,7 +1467,7 @@ int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova, flags, internal); if (rc) goto err_remove; - out_pages += last_index - index + 1; + out_pages += (last - iter.cur_iova) / PAGE_SIZE + 1; } if (!iopt_area_contig_done(&iter)) { rc = -ENOENT; @@ -1478,6 +1496,88 @@ int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova, } EXPORT_SYMBOL_NS_GPL(iommufd_access_pin_pages, "IOMMUFD"); +/** + * iommufd_access_pin_phys - Pin an internal contiguous physical byte span + * @access: Internal access created for byte-span pinning + * @iova: First IOVA byte + * @length: Exact byte length + * @out_phys: Physical address of the first byte, set only on success + * + * Retains native or fragment pin ownership and prevents IOAS unmap until the + * matching iommufd_access_unpin_pages(). No surrounding IOVA bytes are required. + * The caller must stop hardware access before unpinning or destroying access. + */ +int iommufd_access_pin_phys(struct iommufd_access *access, unsigned long iova, + unsigned long length, phys_addr_t *out_phys) +{ + struct iopt_area_contig_iter iter; + struct io_pagetable *iopt; + struct iopt_area *area; + unsigned long last_iova; + phys_addr_t base = 0; + int rc; + + if (!iommufd_access_is_internal(access) || !out_phys || !length) + return -EINVAL; + if (check_add_overflow(iova, length - 1, &last_iova)) + return -EOVERFLOW; + + mutex_lock(&access->ioas_lock); + if (!access->ioas) { + rc = -ENOENT; + goto out_unlock; + } + iopt = &access->ioas->iopt; + down_read(&iopt->iova_rwsem); + iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova) { + unsigned long last = min(last_iova, iopt_area_last_iova(area)); + phys_addr_t phys; + + if (area->prevent_access || !(area->iommu_prot & IOMMU_READ)) { + rc = -EPERM; + goto out_remove; + } + rc = iopt_area_add_phys_access(area, + iopt_area_start_byte(area, iter.cur_iova), + last - iter.cur_iova + 1, &phys); + if (rc) + goto out_remove; + if (iter.cur_iova == iova) { + base = phys; + } else if (phys != base + (iter.cur_iova - iova)) { + iopt_area_remove_access(area, + iopt_area_iova_to_index(area, iter.cur_iova), + iopt_area_iova_to_index(area, last), true); + rc = -EFAULT; + goto out_remove; + } + } + if (!iopt_area_contig_done(&iter)) { + rc = -ENOENT; + goto out_remove; + } + *out_phys = base; + up_read(&iopt->iova_rwsem); + rc = 0; + goto out_unlock; + +out_remove: + /* iter.cur_iova is the first byte whose access was not retained. */ + if (iter.cur_iova > iova) { + unsigned long end = iter.cur_iova - 1; + + iopt_for_each_contig_area(&iter, area, iopt, iova, end) + iopt_area_remove_access(area, + iopt_area_iova_to_index(area, iter.cur_iova), + iopt_area_iova_to_index(area, min(end, iopt_area_last_iova(area))), + true); + } + up_read(&iopt->iova_rwsem); +out_unlock: + mutex_unlock(&access->ioas_lock); + return rc; +} + /** * iommufd_access_rw - Read or write data under the iova * @access: IOAS access to act on diff --git a/drivers/iommu/iommufd/fragments.c b/drivers/iommu/iommufd/fragments.c new file mode 100644 index 0000000000000..5229605852133 --- /dev/null +++ b/drivers/iommu/iommufd/fragments.c @@ -0,0 +1,421 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Alternative user granules need an ownership record in addition to a physical + * address. A device translation cannot retain a subpage allocation identity. + * Keep those records until the final domain interval has gone away. The native + * PFN reader remains in pages.c; file and dma-buf suppliers retain its geometry. + */ +#include +#include +#include +#include + +#include "iommufd_private.h" +#include "io_pagetable.h" + +static phys_addr_t fragment_phys(const struct user_page_fragment *fragment) +{ + return PFN_PHYS(folio_pfn(fragment->folio)) + fragment->offset; +} + +/* Charge each native physical page once, even when several slots pin it. */ +static int fragment_store(struct iopt_pages *pages, unsigned long index, + struct user_page_fragment *fragment) +{ + unsigned long pfn = PHYS_PFN(fragment_phys(fragment)); + unsigned long count = xa_to_value(xa_load(&pages->fragment_pages, pfn)); + struct user_page_fragment *record; + int rc; + + record = kmemdup(fragment, sizeof(*record), GFP_KERNEL_ACCOUNT); + if (!record) + return -ENOMEM; + rc = xa_reserve(&pages->fragments, index, GFP_KERNEL_ACCOUNT); + if (rc) + goto out_free; + rc = xa_err(xa_store(&pages->fragment_pages, pfn, xa_mk_value(count + 1), + GFP_KERNEL_ACCOUNT)); + if (rc) + goto out_release; + /* xa_reserve() made this infallible. Ownership transfers here. */ + xa_store(&pages->fragments, index, record, GFP_NOWAIT); + if (!count) + pages->npinned++; + *fragment = (struct user_page_fragment) {}; + return 0; + +out_release: + xa_release(&pages->fragments, index); +out_free: + kfree(record); + return rc; +} + +/* No allocation on teardown, and callers have already unmapped all devices. */ +static void fragments_release_unused(struct iopt_pages *pages, + unsigned long first, unsigned long last) +{ + struct user_page_fragment *record; + unsigned long index = first; + + lockdep_assert_held(&pages->mutex); + xa_for_each_range(&pages->fragments, index, record, first, last) { + unsigned long pfn, count; + + if (interval_tree_iter_first(&pages->domains_itree, index, index) || + interval_tree_iter_first(&pages->access_itree, index, index)) + continue; + pfn = PHYS_PFN(fragment_phys(record)); + count = xa_to_value(xa_load(&pages->fragment_pages, pfn)); + if (WARN_ON(!count)) + continue; + if (count == 1) { + xa_erase(&pages->fragment_pages, pfn); + pages->npinned--; + } else { + xa_store(&pages->fragment_pages, pfn, xa_mk_value(count - 1), + GFP_NOWAIT); + } + xa_erase(&pages->fragments, index); + release_user_fragments(record, 1, pages->writable); + kfree(record); + cond_resched(); + } + if (pages->npinned < pages->last_npinned) + iopt_pages_update_pinned(pages, pages->last_npinned - pages->npinned, + false, NULL); +} + +/* Reuse existing records even if the original VA was unmapped or replaced. */ +static int fragments_acquire(struct iopt_pages *pages, unsigned long first, + unsigned long last) +{ + unsigned long size = 1UL << pages->page_shift, index = first; + unsigned long capacity = PAGE_SIZE / sizeof(struct user_page_fragment); + struct user_page_fragment *batch = NULL; + bool mm_live = false; + int rc = 0; + + lockdep_assert_held(&pages->mutex); + while (index <= last) { + unsigned long next = index, end, i; + long count; + + if (xa_load(&pages->fragments, index)) { + index++; + continue; + } + if (!mm_live) { + if (!mmget_not_zero(pages->source_mm)) { + rc = -EFAULT; + goto out; + } + mm_live = true; + batch = kmalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); + if (!batch) { + rc = -ENOMEM; + goto out; + } + } + /* Stop before any cached pin; never replace its ownership record. */ + end = xa_find(&pages->fragments, &next, last, XA_PRESENT) ? next : last + 1; + end = min(end, index + capacity); + count = pin_user_fragments_remote(pages->source_mm, + (unsigned long)pages->uptr + (index << pages->page_shift), + (end - index) << pages->page_shift, + FOLL_LONGTERM | (pages->writable ? FOLL_WRITE : 0), batch, capacity); + if (count <= 0) { + rc = count ?: -EFAULT; + goto out; + } + for (i = 0; i < count; i++) { + /* Each aligned logical index must have exactly one full pin. */ + if (WARN_ON(batch[i].length != size || + !IS_ALIGNED(fragment_phys(&batch[i]), size))) + rc = -EFAULT; + else + rc = fragment_store(pages, index, &batch[i]); + if (rc) { + release_user_fragments(batch + i, count - i, false); + goto out; + } + index++; + } + cond_resched(); + } + if (pages->npinned != pages->last_npinned) { + if (iommufd_should_fail()) + rc = -ENOMEM; + else + rc = iopt_pages_update_pinned(pages, + pages->npinned - pages->last_npinned, true, NULL); + } +out: + kfree(batch); + if (mm_live) + mmput(pages->source_mm); + if (rc) + fragments_release_unused(pages, first, last); + return rc; +} + +static void fragments_unmap(struct iommu_domain *domain, unsigned long iova, + unsigned long length) +{ + if (length) + WARN_ON(iommu_unmap(domain, iova, length) != length); +} + +/* All pins are present. Merge physically consecutive fragments before mapping. */ +static int fragments_map(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain) +{ + unsigned long size = 1UL << pages->page_shift; + unsigned long index = iopt_area_index(area), last = iopt_area_last_index(area); + unsigned long remaining = iopt_area_length(area), done = 0; + unsigned long offset = area->page_offset; + unsigned long min_size = 1UL << __ffs(domain->pgsize_bitmap); + int rc; + + while (remaining) { + struct user_page_fragment *record = xa_load(&pages->fragments, index); + phys_addr_t phys = fragment_phys(record) + offset; + unsigned long bytes = min(size - offset, remaining); + + index++; + /* Legacy splitting forbids mappings larger than a native page. */ + while (index <= last && bytes < remaining && + (!area->iopt->disable_large_pages || bytes < PAGE_SIZE)) { + record = xa_load(&pages->fragments, index); + if (fragment_phys(record) != phys + bytes) + break; + bytes += min(size, remaining - bytes); + index++; + } + /* Never widen a device mapping to include an unowned neighbour. */ + if (!IS_ALIGNED(phys | (iopt_area_iova(area) + done) | bytes, min_size)) { + rc = -EINVAL; + goto out_unmap; + } + rc = iommu_map(domain, iopt_area_iova(area) + done, phys, bytes, + area->iommu_prot, GFP_KERNEL_ACCOUNT); + if (rc) + goto out_unmap; + done += bytes; + remaining -= bytes; + offset = 0; + cond_resched(); + } + return 0; + +out_unmap: + fragments_unmap(domain, iopt_area_iova(area), done); + return rc; +} + +int iopt_fragments_fill_domain(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain) +{ + int rc; + + lockdep_assert_held(&pages->mutex); + rc = fragments_acquire(pages, iopt_area_index(area), iopt_area_last_index(area)); + if (rc) + return rc; + rc = fragments_map(area, pages, domain); + if (rc) + fragments_release_unused(pages, iopt_area_index(area), iopt_area_last_index(area)); + return rc; +} + +int iopt_fragments_fill_domains(struct iopt_area *area, struct iopt_pages *pages) +{ + struct iommu_domain *domain; + unsigned long index, failed; + int rc; + + guard(mutex)(&pages->mutex); + rc = fragments_acquire(pages, iopt_area_index(area), iopt_area_last_index(area)); + if (rc) + return rc; + xa_for_each(&area->iopt->domains, index, domain) { + rc = fragments_map(area, pages, domain); + if (rc) + goto out_unmap; + } + area->storage_domain = xa_load(&area->iopt->domains, 0); + interval_tree_insert(&area->pages_node, &pages->domains_itree); + return 0; + +out_unmap: + failed = index; + xa_for_each(&area->iopt->domains, index, domain) { + if (index == failed) + break; + fragments_unmap(domain, iopt_area_iova(area), iopt_area_length(area)); + } + fragments_release_unused(pages, iopt_area_index(area), iopt_area_last_index(area)); + return rc; +} + +void iopt_fragments_unfill_domain(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain) +{ + lockdep_assert_held(&pages->mutex); + fragments_unmap(domain, iopt_area_iova(area), iopt_area_length(area)); + fragments_release_unused(pages, iopt_area_index(area), iopt_area_last_index(area)); +} + +int iopt_fragments_rw(struct iopt_pages *pages, unsigned long start_byte, + void *data, unsigned long length, unsigned int flags) +{ + unsigned long size = 1UL << pages->page_shift; + unsigned long first = start_byte >> pages->page_shift; + unsigned long last = (start_byte + length - 1) >> pages->page_shift; + unsigned long index, offset = start_byte & (size - 1); + int rc; + + guard(mutex)(&pages->mutex); + rc = fragments_acquire(pages, first, last); + if (rc) + return rc; + for (index = first; index <= last; index++) { + struct user_page_fragment *record = xa_load(&pages->fragments, index); + unsigned long bytes = min(size - offset, length); + void *addr = kmap_local_folio(record->folio, record->offset + offset); + + if (flags & IOMMUFD_ACCESS_RW_WRITE) + memcpy(addr, data, bytes); + else + memcpy(data, addr, bytes); + kunmap_local(addr); + data += bytes; + length -= bytes; + offset = 0; + } + fragments_release_unused(pages, first, last); + return 0; +} + +/* The retained typed records, not the output page array, own the pins. */ +static int fragments_register_access(struct iopt_area *area, unsigned long first, + unsigned long last, bool lock_area) +{ + struct iopt_pages *pages = area->pages; + struct iopt_pages_access *access; + struct interval_tree_node *node; + + lockdep_assert_held(&pages->mutex); + for (node = interval_tree_iter_first(&pages->access_itree, first, last); + node; node = interval_tree_iter_next(node, first, last)) { + if (node->start == first && node->last == last) { + access = container_of(node, struct iopt_pages_access, node); + access->users++; + goto out_count; + } + } + access = kzalloc_obj(*access, GFP_KERNEL_ACCOUNT); + if (!access) + return -ENOMEM; + access->node.start = first; + access->node.last = last; + access->users = 1; + interval_tree_insert(&access->node, &pages->access_itree); +out_count: + area->num_accesses++; + if (lock_area) + area->num_locks++; + return 0; +} + +/* Register a physical byte span without discarding fragment ownership. */ +int iopt_fragments_add_phys_access(struct iopt_area *area, unsigned long start_byte, + unsigned long length, phys_addr_t *out_phys) +{ + struct iopt_pages *pages = area->pages; + unsigned long size = 1UL << pages->page_shift; + unsigned long first = start_byte >> pages->page_shift; + unsigned long last = (start_byte + length - 1) >> pages->page_shift; + struct user_page_fragment *record; + unsigned long index; + phys_addr_t base; + int rc; + + guard(mutex)(&pages->mutex); + rc = fragments_acquire(pages, first, last); + if (rc) + return rc; + record = xa_load(&pages->fragments, first); + base = fragment_phys(record); + for (index = first + 1; index <= last; index++) { + record = xa_load(&pages->fragments, index); + if (fragment_phys(record) != + base + ((index - first) << pages->page_shift)) { + rc = -EFAULT; + goto out_release; + } + cond_resched(); + } + rc = fragments_register_access(area, first, last, true); + if (rc) + goto out_release; + *out_phys = base + (start_byte & (size - 1)); + return 0; + +out_release: + fragments_release_unused(pages, first, last); + return rc; +} + +/* + * A native page pointer has no fragment offset. Require an aligned physical + * base and consecutive fragments within each output page; different output + * pages may be scattered. Only the requested logical interval is retained. + */ +int iopt_fragments_add_access(struct iopt_area *area, unsigned long first, + unsigned long last, struct page **out_pages, + bool lock_area) +{ + struct iopt_pages *pages = area->pages; + unsigned long per_page = 1UL << (PAGE_SHIFT - pages->page_shift); + unsigned long index; + phys_addr_t base = 0; + int rc; + + if (first & (per_page - 1)) + return -EINVAL; + guard(mutex)(&pages->mutex); + rc = fragments_acquire(pages, first, last); + if (rc) + return rc; + for (index = first; index <= last; index++) { + struct user_page_fragment *record = xa_load(&pages->fragments, index); + unsigned long offset = (index - first) & (per_page - 1); + phys_addr_t phys = fragment_phys(record); + + if (!offset) { + base = phys; + if (!IS_ALIGNED(base, PAGE_SIZE)) { + rc = -EINVAL; + goto out_release; + } + *out_pages++ = pfn_to_page(PHYS_PFN(base)); + } else if (phys != base + (offset << pages->page_shift)) { + rc = -EINVAL; + goto out_release; + } + cond_resched(); + } + rc = fragments_register_access(area, first, last, lock_area); + if (!rc) + return 0; +out_release: + fragments_release_unused(pages, first, last); + return rc; +} + +void iopt_fragments_unfill_access(struct iopt_pages *pages, unsigned long first, + unsigned long last) +{ + fragments_release_unused(pages, first, last); +} diff --git a/drivers/iommu/iommufd/io_pagetable.c b/drivers/iommu/iommufd/io_pagetable.c index 24d4917105d94..bf8cabee53fb6 100644 --- a/drivers/iommu/iommufd/io_pagetable.c +++ b/drivers/iommu/iommufd/io_pagetable.c @@ -217,7 +217,8 @@ static int iopt_insert_area(struct io_pagetable *iopt, struct iopt_area *area, return -EPERM; area->iommu_prot = iommu_prot; - area->page_offset = start_byte % PAGE_SIZE; + area->page_shift = pages->page_shift; + area->page_offset = start_byte & ((1UL << area->page_shift) - 1); if (area->page_offset & (iopt->iova_alignment - 1)) return -EINVAL; @@ -225,10 +226,10 @@ static int iopt_insert_area(struct io_pagetable *iopt, struct iopt_area *area, if (check_add_overflow(iova, length - 1, &area->node.last)) return -EOVERFLOW; - area->pages_node.start = start_byte / PAGE_SIZE; + area->pages_node.start = start_byte >> area->page_shift; if (check_add_overflow(start_byte, length - 1, &area->pages_node.last)) return -EOVERFLOW; - area->pages_node.last = area->pages_node.last / PAGE_SIZE; + area->pages_node.last >>= area->page_shift; if (WARN_ON(area->pages_node.last >= pages->npages)) return -EOVERFLOW; @@ -572,7 +573,7 @@ iommu_read_and_clear_dirty(struct iommu_domain *domain, struct iommu_dirty_bitmap dirty; struct iova_bitmap_fn_arg arg; struct iova_bitmap *iter; - int ret = 0; + int ret; if (!ops || !ops->read_and_clear_dirty) return -EOPNOTSUPP; @@ -581,7 +582,7 @@ iommu_read_and_clear_dirty(struct iommu_domain *domain, bitmap->page_size, u64_to_user_ptr(bitmap->data)); if (IS_ERR(iter)) - return -ENOMEM; + return PTR_ERR(iter); iommu_dirty_bitmap_init(&dirty, iter, &gather); @@ -589,7 +590,7 @@ iommu_read_and_clear_dirty(struct iommu_domain *domain, arg.iopt = iopt; arg.domain = domain; arg.dirty = &dirty; - iova_bitmap_for_each(iter, &arg, __iommu_read_and_clear_dirty); + ret = iova_bitmap_for_each(iter, &arg, __iommu_read_and_clear_dirty); if (!(flags & IOMMU_DIRTY_NO_CLEAR)) iommu_iotlb_sync(domain, &gather); @@ -615,7 +616,7 @@ int iommufd_check_iova_range(struct io_pagetable *iopt, ((last_iova + 1) & (iommu_pgsize - 1))) return -EINVAL; - if (!bitmap->page_size) + if (!is_power_of_2(bitmap->page_size)) return -EINVAL; if ((bitmap->iova & (bitmap->page_size - 1)) || diff --git a/drivers/iommu/iommufd/io_pagetable.h b/drivers/iommu/iommufd/io_pagetable.h index 27e3e311d395b..d15e51fee0c10 100644 --- a/drivers/iommu/iommufd/io_pagetable.h +++ b/drivers/iommu/iommufd/io_pagetable.h @@ -5,6 +5,7 @@ #ifndef __IO_PAGETABLE_H #define __IO_PAGETABLE_H +#include #include #include #include @@ -45,6 +46,7 @@ struct iopt_area { struct iommu_domain *storage_domain; /* How many bytes into the first page the area starts */ unsigned int page_offset; + u8 page_shift; /* IOMMU_READ, IOMMU_WRITE, etc */ int iommu_prot; bool prevent_access : 1; @@ -107,8 +109,8 @@ static inline size_t iopt_area_length(struct iopt_area *area) /* * Number of bytes from the start of the iopt_pages that the iova begins. - * iopt_area_start_byte() / PAGE_SIZE encodes the starting page index - * iopt_area_start_byte() % PAGE_SIZE encodes the offset within that page + * iopt_area_start_byte() >> area->page_shift encodes the starting page index + * iopt_area_start_byte() % (1UL << area->page_shift) encodes the offset within that page */ static inline unsigned long iopt_area_start_byte(struct iopt_area *area, unsigned long iova) @@ -117,13 +119,13 @@ static inline unsigned long iopt_area_start_byte(struct iopt_area *area, WARN_ON(iova < iopt_area_iova(area) || iova > iopt_area_last_iova(area)); return (iova - iopt_area_iova(area)) + area->page_offset + - iopt_area_index(area) * PAGE_SIZE; + (iopt_area_index(area) << area->page_shift); } static inline unsigned long iopt_area_iova_to_index(struct iopt_area *area, unsigned long iova) { - return iopt_area_start_byte(area, iova) / PAGE_SIZE; + return iopt_area_start_byte(area, iova) >> area->page_shift; } #define __make_iopt_iter(name) \ @@ -214,7 +216,7 @@ struct iopt_pages_dmabuf { * io_pagetable's, through their iopt_area's, can share a single iopt_pages * which avoids multi-pinning and double accounting of page consumption. * - * indexes in this structure are measured in PAGE_SIZE units, are 0 based from + * Indexes use (1UL << page_shift) units and are 0 based from * the start of the uptr and extend to npages. pages are pinned dynamically * according to the intervals in the access_itree and domains_itree, npinned * records the current number of pages pinned. @@ -222,6 +224,8 @@ struct iopt_pages_dmabuf { struct iopt_pages { struct kref kref; struct mutex mutex; + /* Logical indices use page_shift; pin accounting remains native pages. */ + u8 page_shift; size_t npages; size_t npinned; size_t last_npinned; @@ -242,12 +246,73 @@ struct iopt_pages { u8 account_mode; struct xarray pinned_pfns; +#ifdef CONFIG_MM_SUBPAGE + /* Exact fragment pins, indexed independently of device translations. */ + struct xarray fragments; + /* Native physical PFN -> fragment count, for pin accounting. */ + struct xarray fragment_pages; +#endif /* Of iopt_pages_access::node */ struct rb_root_cached access_itree; /* Of iopt_area::pages_node */ struct rb_root_cached domains_itree; }; +static inline bool iopt_pages_use_fragments(struct iopt_pages *pages) +{ + return IS_ENABLED(CONFIG_MM_SUBPAGE) && pages->type == IOPT_ADDRESS_USER && + mm_page_size(pages->source_mm) != PAGE_SIZE; +} + +#ifdef CONFIG_MM_SUBPAGE +int iopt_fragments_fill_domain(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain); +int iopt_fragments_fill_domains(struct iopt_area *area, struct iopt_pages *pages); +void iopt_fragments_unfill_domain(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain); +int iopt_fragments_add_access(struct iopt_area *area, unsigned long first, + unsigned long last, struct page **out_pages, + bool lock_area); +int iopt_fragments_add_phys_access(struct iopt_area *area, unsigned long start_byte, + unsigned long length, phys_addr_t *out_phys); +void iopt_fragments_unfill_access(struct iopt_pages *pages, unsigned long first, + unsigned long last); +int iopt_fragments_rw(struct iopt_pages *pages, unsigned long start_byte, + void *data, unsigned long length, unsigned int flags); +#else +static inline int iopt_fragments_fill_domain(struct iopt_area *area, + struct iopt_pages *pages, struct iommu_domain *domain) +{ + return -EOPNOTSUPP; +} +static inline int iopt_fragments_fill_domains(struct iopt_area *area, + struct iopt_pages *pages) +{ + return -EOPNOTSUPP; +} +static inline void iopt_fragments_unfill_domain(struct iopt_area *area, + struct iopt_pages *pages, struct iommu_domain *domain) { } +static inline int iopt_fragments_add_access(struct iopt_area *area, + unsigned long first, unsigned long last, struct page **out_pages, + bool lock_area) +{ + return -EOPNOTSUPP; +} +static inline int iopt_fragments_add_phys_access(struct iopt_area *area, + unsigned long start_byte, unsigned long length, phys_addr_t *out_phys) +{ + return -EOPNOTSUPP; +} +static inline void iopt_fragments_unfill_access(struct iopt_pages *pages, + unsigned long first, unsigned long last) { } +static inline int iopt_fragments_rw(struct iopt_pages *pages, + unsigned long start_byte, void *data, unsigned long length, + unsigned int flags) +{ + return -EOPNOTSUPP; +} +#endif + static inline bool iopt_is_dmabuf(struct iopt_pages *pages) { if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER)) @@ -290,6 +355,8 @@ void iopt_pages_unfill_xarray(struct iopt_pages *pages, unsigned long start, int iopt_area_add_access(struct iopt_area *area, unsigned long start, unsigned long last, struct page **out_pages, unsigned int flags, bool lock_area); +int iopt_area_add_phys_access(struct iopt_area *area, unsigned long start_byte, + unsigned long length, phys_addr_t *out_phys); void iopt_area_remove_access(struct iopt_area *area, unsigned long start, unsigned long last, bool unlock_area); int iopt_pages_rw_access(struct iopt_pages *pages, unsigned long start_byte, diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h index 421d0cc7c1bc3..9b6c41b16d348 100644 --- a/drivers/iommu/iommufd/iommufd_private.h +++ b/drivers/iommu/iommufd/iommufd_private.h @@ -532,6 +532,10 @@ static inline bool iommufd_access_is_internal(struct iommufd_access *access) } struct iommufd_access *iommufd_access_create_internal(struct iommufd_ctx *ictx); +struct iommufd_access *iommufd_access_create_internal_phys(struct iommufd_ctx *ictx); +int iommufd_access_pin_phys(struct iommufd_access *access, unsigned long iova, + unsigned long length, phys_addr_t *out_phys); + static inline void iommufd_access_destroy_internal(struct iommufd_ctx *ictx, diff --git a/drivers/iommu/iommufd/iova_bitmap.c b/drivers/iommu/iommufd/iova_bitmap.c index dac3e657d498d..12e1dd035871a 100644 --- a/drivers/iommu/iommufd/iova_bitmap.c +++ b/drivers/iommu/iommufd/iova_bitmap.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #define BITS_PER_PAGE (PAGE_SIZE * BITS_PER_BYTE) @@ -44,11 +45,16 @@ struct iova_bitmap_map { /* page offset of the first user page pinned */ unsigned long pgoff; - /* number of pages pinned */ + /* number of native pages or typed fragments pinned */ unsigned long npages; /* pinned pages representing the bitmap data */ struct page **pages; +#ifdef CONFIG_MM_SUBPAGE + /* Exact byte ranges, including offsets unrelated to the user VA. */ + struct user_page_fragment *fragments; + unsigned long pinned_bytes; +#endif }; /* @@ -63,7 +69,10 @@ struct iova_bitmap_map { * The bitmap object uses one base page to store all the pinned pages * pointers related to the bitmap. For sizeof(struct page*) == 8 it stores * 512 struct page pointers which, if the base page size is 4K, it means - * 2M of bitmap data is pinned at a time. If the iova_bitmap page size is + * 2M of bitmap data is pinned at a time. Alternative-granule processes use + * one native page of fragment descriptors instead, retaining each physical + * byte offset and bounding the window by the sum of fragment lengths. + * If the iova_bitmap page size is * also 4K then the range window to iterate is 64G. * * For example iterating on a total IOVA range of 4G..128G, it will walk @@ -116,6 +125,9 @@ struct iova_bitmap { /* length of the IOVA range for the whole bitmap */ size_t length; + + /* First lazy pin/write error during an iteration. */ + int error; }; /* @@ -192,6 +204,26 @@ static int iova_bitmap_get(struct iova_bitmap *bitmap) npages = min(npages + !!offset_in_page(addr), PAGE_SIZE / sizeof(struct page *)); +#ifdef CONFIG_MM_SUBPAGE + if (mapped->fragments) { + unsigned long bytes = bitmap->mapped_total_index - bitmap->mapped_base_index; + unsigned long i; + + ret = pin_user_fragments_remote(current->mm, (unsigned long)addr, + bytes, FOLL_WRITE, mapped->fragments, + PAGE_SIZE / sizeof(*mapped->fragments)); + if (ret <= 0) + return -EFAULT; + mapped->npages = ret; + mapped->pinned_bytes = 0; + for (i = 0; i < mapped->npages; i++) + mapped->pinned_bytes += mapped->fragments[i].length; + mapped->pgoff = 0; + mapped->iova = iova_bitmap_mapped_iova(bitmap); + mapped->length = iova_bitmap_mapped_length(bitmap); + return 0; + } +#endif ret = pin_user_pages_fast((unsigned long)addr, npages, FOLL_WRITE, mapped->pages); if (ret <= 0) @@ -221,7 +253,13 @@ static void iova_bitmap_put(struct iova_bitmap *bitmap) struct iova_bitmap_map *mapped = &bitmap->mapped; if (mapped->npages) { - unpin_user_pages(mapped->pages, mapped->npages); +#ifdef CONFIG_MM_SUBPAGE + if (mapped->fragments) { + release_user_fragments(mapped->fragments, mapped->npages, true); + mapped->pinned_bytes = 0; + } else +#endif + unpin_user_pages_dirty_lock(mapped->pages, mapped->npages, true); mapped->npages = 0; } } @@ -259,6 +297,16 @@ struct iova_bitmap *iova_bitmap_alloc(unsigned long iova, size_t length, bitmap->iova = iova; bitmap->length = length; mapped->iova = iova; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + mapped->fragments = (void *)__get_free_page(GFP_KERNEL); + if (!mapped->fragments) { + rc = -ENOMEM; + goto err; + } + return bitmap; + } +#endif mapped->pages = (struct page **)__get_free_page(GFP_KERNEL); if (!mapped->pages) { rc = -ENOMEM; @@ -291,6 +339,10 @@ void iova_bitmap_free(struct iova_bitmap *bitmap) mapped->pages = NULL; } +#ifdef CONFIG_MM_SUBPAGE + if (mapped->fragments) + free_page((unsigned long)mapped->fragments); +#endif kfree(bitmap); } EXPORT_SYMBOL_NS_GPL(iova_bitmap_free, "IOMMUFD"); @@ -303,7 +355,12 @@ static unsigned long iova_bitmap_mapped_remaining(struct iova_bitmap *bitmap) { unsigned long remaining, bytes; - bytes = (bitmap->mapped.npages << PAGE_SHIFT) - bitmap->mapped.pgoff; +#ifdef CONFIG_MM_SUBPAGE + if (bitmap->mapped.fragments) + bytes = bitmap->mapped.pinned_bytes; + else +#endif + bytes = (bitmap->mapped.npages << PAGE_SHIFT) - bitmap->mapped.pgoff; remaining = bitmap->mapped_total_index - bitmap->mapped_base_index; remaining = min_t(unsigned long, remaining, @@ -356,16 +413,21 @@ static int iova_bitmap_advance_to(struct iova_bitmap *bitmap, unsigned long iova) { unsigned long index; + int ret = -EINVAL; index = iova_bitmap_offset_to_index(bitmap, iova - bitmap->iova); if (index >= bitmap->mapped_total_index) - return -EINVAL; + goto out; bitmap->mapped_base_index = index; iova_bitmap_put(bitmap); /* Pin the next set of bitmap pages */ - return iova_bitmap_get(bitmap); + ret = iova_bitmap_get(bitmap); +out: + if (ret && !bitmap->error) + bitmap->error = ret; + return ret; } /** @@ -379,15 +441,58 @@ static int iova_bitmap_advance_to(struct iova_bitmap *bitmap, * mapped bitmap user pages into IOVA ranges to process. * * Return: 0 on success, and an error on failure either upon - * iteration or when the callback returns an error. + * iteration, a lazy bitmap write, or when the callback returns an error. */ int iova_bitmap_for_each(struct iova_bitmap *bitmap, void *opaque, iova_bitmap_fn_t fn) { - return fn(bitmap, bitmap->iova, bitmap->length, opaque); + int ret; + + bitmap->error = 0; + ret = fn(bitmap, bitmap->iova, bitmap->length, opaque); + return ret ?: bitmap->error; } EXPORT_SYMBOL_NS_GPL(iova_bitmap_for_each, "IOMMUFD"); +#ifdef CONFIG_MM_SUBPAGE +static void iova_bitmap_set_fragments(struct iova_bitmap *bitmap, + unsigned long iova, size_t length) +{ + struct iova_bitmap_map *mapped = &bitmap->mapped; + unsigned long bit = (iova - bitmap->iova) >> mapped->pgshift; + unsigned long end = ((iova + length - 1 - bitmap->iova) >> mapped->pgshift) + 1; + + while (bit < end) { + unsigned long base, relative, i; + + iova = bitmap->iova + (bit << mapped->pgshift); + if (!iova_bitmap_mapped_range(mapped, iova, 1) && + iova_bitmap_advance_to(bitmap, iova)) + return; + base = bitmap->mapped_base_index * BITS_PER_BYTE; + relative = bit - base; + for (i = 0; i < mapped->npages && bit < end; i++) { + struct user_page_fragment *fragment = &mapped->fragments[i]; + unsigned long bits = fragment->length * BITS_PER_BYTE; + unsigned long count; + void *addr; + + if (relative >= bits) { + relative -= bits; + continue; + } + count = min(end - bit, bits - relative); + addr = kmap_local_folio(fragment->folio, fragment->offset & PAGE_MASK); + bitmap_set(addr, offset_in_page(fragment->offset) * BITS_PER_BYTE + + relative, count); + kunmap_local(addr); + bit += count; + relative = 0; + } + } +} +#endif + /** * iova_bitmap_set() - Records an IOVA range in bitmap * @bitmap: IOVA bitmap @@ -395,7 +500,8 @@ EXPORT_SYMBOL_NS_GPL(iova_bitmap_for_each, "IOMMUFD"); * @length: IOVA range length * * Set the bits corresponding to the range [iova .. iova+length-1] in - * the user bitmap. + * the user bitmap. A lazy pin failure is retained for the enclosing + * iova_bitmap_for_each() to report; successful prefix writes are not undone. * */ void iova_bitmap_set(struct iova_bitmap *bitmap, @@ -404,6 +510,13 @@ void iova_bitmap_set(struct iova_bitmap *bitmap, struct iova_bitmap_map *mapped = &bitmap->mapped; unsigned long cur_bit, last_bit, last_page_idx; +#ifdef CONFIG_MM_SUBPAGE + if (mapped->fragments) { + iova_bitmap_set_fragments(bitmap, iova, length); + return; + } +#endif + update_indexes: if (unlikely(!iova_bitmap_mapped_range(mapped, iova, length))) { /* diff --git a/drivers/iommu/iommufd/pages.c b/drivers/iommu/iommufd/pages.c index 404f31d8f7291..0839878bfb67c 100644 --- a/drivers/iommu/iommufd/pages.c +++ b/drivers/iommu/iommufd/pages.c @@ -52,6 +52,7 @@ #include #include #include +#include #include #include #include @@ -207,7 +208,7 @@ static unsigned long iopt_area_index_to_iova(struct iopt_area *area, index -= iopt_area_index(area); if (index == 0) return iopt_area_iova(area); - return iopt_area_iova(area) - area->page_offset + index * PAGE_SIZE; + return iopt_area_iova(area) - area->page_offset + (index << area->page_shift); } static unsigned long iopt_area_index_to_iova_last(struct iopt_area *area, @@ -219,7 +220,7 @@ static unsigned long iopt_area_index_to_iova_last(struct iopt_area *area, if (index == iopt_area_last_index(area)) return iopt_area_last_iova(area); return iopt_area_iova(area) - area->page_offset + - (index - iopt_area_index(area) + 1) * PAGE_SIZE - 1; + ((index - iopt_area_index(area) + 1) << area->page_shift) - 1; } static void iommu_unmap_nofail(struct iommu_domain *domain, unsigned long iova, @@ -993,9 +994,11 @@ static int update_mm_locked_vm(struct iopt_pages *pages, unsigned long npages, } else if ((!user || (!user->upages && !user->ufolios)) && pages->source_mm != current->mm) { - if (!mmget_not_zero(pages->source_mm)) + if (mmget_not_zero(pages->source_mm)) + do_put = true; + else if (inc) return -EINVAL; - do_put = true; + /* mmgrab() keeps the lock and counters alive for final uncharge. */ } mmap_write_lock(pages->source_mm); @@ -1378,10 +1381,15 @@ static struct iopt_pages *iopt_alloc_pages(unsigned long start_byte, kref_init(&pages->kref); xa_init_flags(&pages->pinned_pfns, XA_FLAGS_ACCOUNT); +#ifdef CONFIG_MM_SUBPAGE + xa_init_flags(&pages->fragments, XA_FLAGS_ACCOUNT); + xa_init_flags(&pages->fragment_pages, XA_FLAGS_ACCOUNT); +#endif + pages->page_shift = PAGE_SHIFT; mutex_init(&pages->mutex); pages->source_mm = current->mm; mmgrab(pages->source_mm); - pages->npages = DIV_ROUND_UP(length + start_byte, PAGE_SIZE); + pages->npages = ((length - 1 + start_byte) >> PAGE_SHIFT) + 1; pages->access_itree = RB_ROOT_CACHED; pages->domains_itree = RB_ROOT_CACHED; pages->writable = writable; @@ -1400,8 +1408,10 @@ struct iopt_pages *iopt_alloc_user_pages(void __user *uptr, { struct iopt_pages *pages; unsigned long end; + unsigned int shift = min_t(unsigned int, mm_page_shift(current->mm), PAGE_SHIFT); + unsigned long granule = 1UL << shift; void __user *uptr_down = - (void __user *)ALIGN_DOWN((uintptr_t)uptr, PAGE_SIZE); + (void __user *)ALIGN_DOWN((uintptr_t)uptr, granule); if (check_add_overflow((unsigned long)uptr, length, &end)) return ERR_PTR(-EOVERFLOW); @@ -1409,6 +1419,8 @@ struct iopt_pages *iopt_alloc_user_pages(void __user *uptr, pages = iopt_alloc_pages(uptr - uptr_down, length, writable); if (IS_ERR(pages)) return pages; + pages->page_shift = shift; + pages->npages = ((length - 1 + (uptr - uptr_down)) >> shift) + 1; pages->uptr = uptr_down; pages->type = IOPT_ADDRESS_USER; return pages; @@ -1656,6 +1668,12 @@ void iopt_release_pages(struct kref *kref) WARN_ON(!RB_EMPTY_ROOT(&pages->domains_itree.rb_root)); WARN_ON(pages->npinned); WARN_ON(!xa_empty(&pages->pinned_pfns)); +#ifdef CONFIG_MM_SUBPAGE + WARN_ON(!xa_empty(&pages->fragments)); + WARN_ON(!xa_empty(&pages->fragment_pages)); + xa_destroy(&pages->fragments); + xa_destroy(&pages->fragment_pages); +#endif if (iopt_is_dmabuf(pages) && pages->dmabuf.attach) { struct dma_buf *dmabuf = pages->dmabuf.attach->dmabuf; @@ -1832,6 +1850,9 @@ void iopt_area_unmap_domain(struct iopt_area *area, struct iommu_domain *domain) void iopt_area_unfill_domain(struct iopt_area *area, struct iopt_pages *pages, struct iommu_domain *domain) { + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_unfill_domain(area, pages, domain); + if (iopt_dmabuf_revoked(pages)) return; @@ -1855,6 +1876,9 @@ int iopt_area_fill_domain(struct iopt_area *area, struct iommu_domain *domain) lockdep_assert_held(&area->pages->mutex); + if (iopt_pages_use_fragments(area->pages)) + return iopt_fragments_fill_domain(area, area->pages, domain); + if (iopt_dmabuf_revoked(area->pages)) return 0; @@ -1916,6 +1940,9 @@ int iopt_area_fill_domains(struct iopt_area *area, struct iopt_pages *pages) if (xa_empty(&area->iopt->domains)) return 0; + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_fill_domains(area, pages); + mutex_lock(&pages->mutex); if (iopt_is_dmabuf(pages)) { rc = iopt_dmabuf_track_all_domains(area, pages); @@ -2365,6 +2392,10 @@ int iopt_pages_rw_access(struct iopt_pages *pages, unsigned long start_byte, if (iopt_is_dmabuf(pages)) return -EINVAL; + if (iopt_pages_use_fragments(pages) && change_mm && + !(flags & IOMMUFD_ACCESS_RW_KTHREAD)) + return iopt_fragments_rw(pages, start_byte, data, length, flags); + if (pages->type != IOPT_ADDRESS_USER) return iopt_pages_rw_slow(pages, start_index, last_index, start_byte % PAGE_SIZE, data, length, @@ -2385,11 +2416,14 @@ int iopt_pages_rw_access(struct iopt_pages *pages, unsigned long start_byte, * ignore any pinning inconsistencies, unlike a real DMA path. */ if (change_mm) { - if (!mmget_not_zero(pages->source_mm)) + if (!mmget_not_zero(pages->source_mm)) { + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_rw(pages, start_byte, data, length, flags); return iopt_pages_rw_slow(pages, start_index, last_index, start_byte % PAGE_SIZE, data, length, flags); + } kthread_use_mm(pages->source_mm); } @@ -2454,6 +2488,11 @@ int iopt_area_add_access(struct iopt_area *area, unsigned long start_index, if (iopt_is_dmabuf(pages)) return -EINVAL; + /* Preserve typed ownership and validate the native output geometry. */ + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_add_access(area, start_index, last_index, + out_pages, lock_area); + mutex_lock(&pages->mutex); access = iopt_pages_get_exact_access(pages, start_index, last_index); if (access) { @@ -2494,6 +2533,47 @@ int iopt_area_add_access(struct iopt_area *area, unsigned long start_index, return rc; } +/* Internal hardware queues consume a contiguous physical byte range. */ +int iopt_area_add_phys_access(struct iopt_area *area, unsigned long start_byte, + unsigned long length, phys_addr_t *out_phys) +{ + struct iopt_pages *pages = area->pages; + unsigned long first = start_byte >> pages->page_shift; + unsigned long last = (start_byte + length - 1) >> pages->page_shift; + unsigned long count = last - first + 1, i; + struct page **list; + phys_addr_t base; + int rc; + + if (iopt_is_dmabuf(pages)) + return -EINVAL; + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_add_phys_access(area, start_byte, length, out_phys); + + /* Retain the native page supplier and its accounting/batching rules. */ + list = kvcalloc(count, sizeof(*list), GFP_KERNEL_ACCOUNT); + if (!list) + return -ENOMEM; + rc = iopt_area_add_access(area, first, last, list, 0, true); + if (rc) + goto out_free; + base = PFN_PHYS(page_to_pfn(list[0])); + for (i = 1; i < count; i++) { + if (page_to_pfn(list[i]) != page_to_pfn(list[0]) + i) { + rc = -EFAULT; + goto out_unpin; + } + } + *out_phys = base + offset_in_page(start_byte); + goto out_free; + +out_unpin: + iopt_area_remove_access(area, first, last, true); +out_free: + kvfree(list); + return rc; +} + /** * iopt_area_remove_access() - Release an in-kernel access for PFNs * @area: The source of PFNs @@ -2526,7 +2606,10 @@ void iopt_area_remove_access(struct iopt_area *area, unsigned long start_index, goto out_unlock; interval_tree_remove(&access->node, &pages->access_itree); - iopt_pages_unfill_xarray(pages, start_index, last_index); + if (iopt_pages_use_fragments(pages)) + iopt_fragments_unfill_access(pages, start_index, last_index); + else + iopt_pages_unfill_xarray(pages, start_index, last_index); kfree(access); out_unlock: mutex_unlock(&pages->mutex); diff --git a/drivers/iommu/iommufd/selftest.c b/drivers/iommu/iommufd/selftest.c index 55f21c799d8e6..d63143f4ba1ab 100644 --- a/drivers/iommu/iommufd/selftest.c +++ b/drivers/iommu/iommufd/selftest.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -1212,6 +1213,28 @@ static int iommufd_test_md_check_pa(struct iommufd_ucmd *ucmd, unsigned long pfn; long npages; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + struct user_page_fragment fragment; + phys_addr_t user_phys; + + npages = get_user_fragments_remote(current->mm, + (uintptr_t)uptr, page_size, 0, &fragment, 1); + if (npages != 1) { + rc = npages < 0 ? npages : -EFAULT; + goto out_put; + } + user_phys = PFN_PHYS(folio_pfn(fragment.folio)) + fragment.offset; + io_phys = iommu_iova_to_phys(&mock->domain, iova); + rc = fragment.length == page_size && io_phys == user_phys ? 0 : -EINVAL; + release_user_fragments(&fragment, 1, false); + if (rc) + goto out_put; + iova += page_size; + uptr += page_size; + continue; + } +#endif npages = get_user_pages_fast((uintptr_t)uptr & PAGE_MASK, 1, 0, pages); if (npages < 0) { @@ -1517,9 +1540,39 @@ static int iommufd_test_access_replace_ioas(struct iommufd_ucmd *ucmd, /* Check that the pages in a page array match the pages in the user VA */ static int iommufd_test_check_pages(void __user *uptr, struct page **pages, - size_t npages) -{ - for (; npages; npages--) { + size_t length) +{ +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + unsigned long offset = 0; + + while (offset < length) { + struct user_page_fragment fragment; + unsigned long bytes = min_t(unsigned long, length - offset, + mm_user_fragment_size(current->mm) - + (((unsigned long)uptr + offset) & + (mm_user_fragment_size(current->mm) - 1))); + phys_addr_t actual, expected; + long rc; + + rc = get_user_fragments_remote(current->mm, + (unsigned long)uptr + offset, bytes, 0, &fragment, 1); + if (rc <= 0) + return rc ?: -EFAULT; + actual = PFN_PHYS(folio_pfn(fragment.folio)) + fragment.offset; + expected = PFN_PHYS(page_to_pfn(pages[offset / PAGE_SIZE])) + + offset_in_page(offset); + bytes = fragment.length; + release_user_fragments(&fragment, 1, false); + if (actual != expected || !bytes || + bytes > PAGE_SIZE - offset_in_page(offset)) + return -EBADE; + offset += bytes; + } + return 0; + } +#endif + for (size_t npages = DIV_ROUND_UP(length, PAGE_SIZE); npages; npages--) { struct page *tmp_pages[1]; long rc; @@ -1597,7 +1650,7 @@ static int iommufd_test_access_pages(struct iommufd_ucmd *ucmd, if (uptr) { rc = iommufd_test_check_pages( uptr - (iova - ALIGN_DOWN(iova, PAGE_SIZE)), pages, - npages); + length + offset_in_page(iova)); if (rc) goto out_unaccess; } diff --git a/drivers/iommu/iommufd/viommu.c b/drivers/iommu/iommufd/viommu.c index fc13cf4737ea3..fa8296b15ef59 100644 --- a/drivers/iommu/iommufd/viommu.c +++ b/drivers/iommu/iommufd/viommu.c @@ -263,11 +263,7 @@ static void iommufd_hw_queue_destroy_access(struct iommufd_ctx *ictx, struct iommufd_access *access, u64 base_iova, size_t length) { - u64 aligned_iova = PAGE_ALIGN_DOWN(base_iova); - u64 offset = base_iova - aligned_iova; - - iommufd_access_unpin_pages(access, aligned_iova, - PAGE_ALIGN(length + offset)); + iommufd_access_unpin_pages(access, base_iova, length); iommufd_access_detach_internal(access); iommufd_access_destroy_internal(ictx, access); } @@ -300,66 +296,28 @@ static struct iommufd_access * iommufd_hw_queue_alloc_phys(struct iommu_hw_queue_alloc *cmd, struct iommufd_viommu *viommu, phys_addr_t *base_pa) { - u64 aligned_iova = PAGE_ALIGN_DOWN(cmd->nesting_parent_iova); - u64 offset = cmd->nesting_parent_iova - aligned_iova; struct iommufd_access *access; - struct page **pages; - size_t max_npages; - size_t length; - size_t i; int rc; - /* max_npages = DIV_ROUND_UP(offset + cmd->length, PAGE_SIZE) */ - if (check_add_overflow(offset, cmd->length, &length)) - return ERR_PTR(-ERANGE); - if (check_add_overflow(length, PAGE_SIZE - 1, &length)) + if (cmd->nesting_parent_iova > ULONG_MAX || cmd->length > ULONG_MAX) return ERR_PTR(-ERANGE); - max_npages = length / PAGE_SIZE; - /* length needs to be page aligned too */ - length = max_npages * PAGE_SIZE; - - /* - * Use kvcalloc() to avoid memory fragmentation for a large page array. - * Set __GFP_NOWARN to avoid syzkaller blowups - */ - pages = kvzalloc_objs(*pages, max_npages, GFP_KERNEL | __GFP_NOWARN); - if (!pages) - return ERR_PTR(-ENOMEM); - - access = iommufd_access_create_internal(viommu->ictx); - if (IS_ERR(access)) { - rc = PTR_ERR(access); - goto out_free; - } + access = iommufd_access_create_internal_phys(viommu->ictx); + if (IS_ERR(access)) + return access; rc = iommufd_access_attach_internal(access, viommu->hwpt->ioas); if (rc) goto out_destroy; - - rc = iommufd_access_pin_pages(access, aligned_iova, length, pages, 0); + rc = iommufd_access_pin_phys(access, cmd->nesting_parent_iova, + cmd->length, base_pa); if (rc) goto out_detach; - - /* Validate if the underlying physical pages are contiguous */ - for (i = 1; i < max_npages; i++) { - if (page_to_pfn(pages[i]) == page_to_pfn(pages[i - 1]) + 1) - continue; - rc = -EFAULT; - goto out_unpin; - } - - *base_pa = (page_to_pfn(pages[0]) << PAGE_SHIFT) + offset; - kvfree(pages); return access; -out_unpin: - iommufd_access_unpin_pages(access, aligned_iova, length); out_detach: iommufd_access_detach_internal(access); out_destroy: iommufd_access_destroy_internal(viommu->ictx, access); -out_free: - kvfree(pages); return ERR_PTR(rc); } diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index e6d3a2311a99b..df079de9ee5de 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -28,11 +28,14 @@ #include #include #include +#include +#include #include #include #include #include #include +#include #include #include #include @@ -84,6 +87,8 @@ struct vfio_domain { bool enforce_cache_coherency : 1; }; +struct vfio_fragment_provider; + struct vfio_dma { struct rb_node node; dma_addr_t iova; /* Device address */ @@ -99,6 +104,7 @@ struct vfio_dma { unsigned long *bitmap; struct mm_struct *mm; size_t locked_vm; + struct vfio_fragment_provider *fragments; }; struct vfio_batch { @@ -128,7 +134,9 @@ struct vfio_pfn { struct rb_node node; dma_addr_t iova; /* Device address */ unsigned long pfn; /* Host pfn */ + unsigned long length; /* Bytes retained in this DMA */ unsigned int ref_count; + bool native_pin; /* separately adopted legacy external PIN */ }; struct vfio_regions { @@ -152,6 +160,9 @@ struct vfio_regions { #define DIRTY_BITMAP_SIZE_MAX DIRTY_BITMAP_BYTES(DIRTY_BITMAP_PAGES_MAX) static int put_pfn(unsigned long pfn, int prot); +static void vfio_fragments_put_native_pin(struct vfio_dma *dma, unsigned long pfn); +static void vfio_fragments_release_range(struct vfio_dma *dma, unsigned long start, + unsigned long length); static struct vfio_iommu_group* vfio_iommu_find_iommu_group(struct vfio_iommu *iommu, @@ -238,6 +249,12 @@ static void vfio_unlink_dma(struct vfio_iommu *iommu, struct vfio_dma *old) } +/* Software dirty granularity must survive domain attach/detach unchanged. */ +static size_t vfio_dirty_pgsize(struct vfio_iommu *iommu) +{ + return IS_ENABLED(CONFIG_MM_SUBPAGE) ? SZ_4K : 1UL << __ffs(iommu->pgsize_bitmap); +} + static int vfio_dma_bitmap_alloc(struct vfio_dma *dma, size_t pgsize) { uint64_t npages = dma->size / pgsize; @@ -272,14 +289,15 @@ static void vfio_dma_populate_bitmap(struct vfio_dma *dma, size_t pgsize) for (p = rb_first(&dma->pfn_list); p; p = rb_next(p)) { struct vfio_pfn *vpfn = rb_entry(p, struct vfio_pfn, node); - bitmap_set(dma->bitmap, (vpfn->iova - dma->iova) >> pgshift, 1); + bitmap_set(dma->bitmap, (vpfn->iova - dma->iova) >> pgshift, + vpfn->length >> pgshift); } } static void vfio_iommu_populate_bitmap_full(struct vfio_iommu *iommu) { struct rb_node *n; - unsigned long pgshift = __ffs(iommu->pgsize_bitmap); + unsigned long pgshift = __ffs(vfio_dirty_pgsize(iommu)); for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) { struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); @@ -383,7 +401,7 @@ static void vfio_unlink_pfn(struct vfio_dma *dma, struct vfio_pfn *old) } static int vfio_add_to_pfn_list(struct vfio_dma *dma, dma_addr_t iova, - unsigned long pfn) + unsigned long pfn, unsigned long length) { struct vfio_pfn *vpfn; @@ -393,6 +411,7 @@ static int vfio_add_to_pfn_list(struct vfio_dma *dma, dma_addr_t iova, vpfn->iova = iova; vpfn->pfn = pfn; + vpfn->length = length; vpfn->ref_count = 1; vfio_link_pfn(dma, vpfn); return 0; @@ -421,8 +440,21 @@ static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn) vpfn->ref_count--; if (!vpfn->ref_count) { - ret = put_pfn(vpfn->pfn, dma->prot); - vfio_remove_from_pfn_list(dma, vpfn); + if (dma->fragments) { + unsigned long offset = vpfn->iova - dma->iova; + unsigned long length = vpfn->length; + + unsigned long pfn = vpfn->pfn; + bool native_pin = vpfn->native_pin; + + vfio_remove_from_pfn_list(dma, vpfn); + if (native_pin) + vfio_fragments_put_native_pin(dma, pfn); + vfio_fragments_release_range(dma, offset, length); + } else { + ret = put_pfn(vpfn->pfn, dma->prot); + vfio_remove_from_pfn_list(dma, vpfn); + } } return ret; } @@ -430,10 +462,19 @@ static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn) static int mm_lock_acct(struct task_struct *task, struct mm_struct *mm, bool lock_cap, long npage) { - int ret = mmap_write_lock_killable(mm); + int ret; - if (ret) - return ret; + if (!npage) + return 0; + + /* A released pin must be refunded even if the caller is interrupted. */ + if (npage < 0) { + mmap_write_lock(mm); + } else { + ret = mmap_write_lock_killable(mm); + if (ret) + return ret; + } ret = __account_locked_vm(mm, abs(npage), npage > 0, task, lock_cap); mmap_write_unlock(mm); @@ -443,25 +484,28 @@ static int mm_lock_acct(struct task_struct *task, struct mm_struct *mm, static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async) { struct mm_struct *mm; + bool get_mm = async && npage > 0; int ret; if (!npage) return 0; mm = dma->mm; - if (async && !mmget_not_zero(mm)) + if (get_mm && !mmget_not_zero(mm)) return -ESRCH; /* process exited */ ret = mm_lock_acct(dma->task, mm, dma->lock_cap, npage); if (!ret) dma->locked_vm += npage; - if (async) + if (get_mm) mmput(mm); return ret; } +#include "vfio_iommu_type1_fragments.c" + /* * Some mappings aren't backed by a struct page, for example an mmap'd * MMIO range for our own or another device. These use a different @@ -765,7 +809,8 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, */ if (acct_pages) { if (!dma->lock_cap && - mm->locked_vm + lock_acct + acct_pages > limit) { + mm_locked_vm_native_pages(mm) + + lock_acct + acct_pages > limit) { pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__, limit << PAGE_SHIFT); ret = -ENOMEM; @@ -889,6 +934,78 @@ static int vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova, return unlocked; } +/* A native page may contain independently owned fine-grained DMA areas. */ +static unsigned long vfio_unpin_external_range(struct vfio_iommu *iommu, + dma_addr_t start, unsigned long length, + bool do_accounting) +{ + unsigned long done = 0; + + while (done < length) { + dma_addr_t iova = start + done; + struct vfio_dma *dma = vfio_find_dma(iommu, iova, 1); + unsigned long bytes; + + if (!dma || iova < dma->iova) + break; + bytes = min_t(unsigned long, length - done, + PAGE_SIZE - offset_in_page(iova)); + bytes = min(bytes, dma->size - (iova - dma->iova)); + vfio_unpin_page_external(dma, iova, do_accounting); + done += bytes; + } + return done; +} + +static int vfio_pin_external_segment(struct vfio_iommu *iommu, + struct vfio_dma *dma, dma_addr_t iova, unsigned long length, + int prot, bool do_accounting, unsigned long *pfn) +{ + struct vfio_pfn *vpfn; + int ret; + + if ((dma->prot & prot) != prot) + return -EPERM; + vpfn = vfio_iova_get_vfio_pfn(dma, iova); + if (vpfn) { + if (vpfn->length != length) { + vfio_unpin_page_external(dma, iova, do_accounting); + return -EINVAL; + } + *pfn = vpfn->pfn; + } else if (dma->fragments) { + ret = vfio_fragments_external(dma, iova, length, pfn); + if (ret) + return ret; + } else { + /* Ordinary native mappings still use one native GUP per page. */ + if (length != PAGE_SIZE) + return -EINVAL; + ret = vfio_pin_page_external(dma, dma->vaddr + iova - dma->iova, + pfn, do_accounting); + if (ret) + return ret; + if (!pfn_valid(*pfn)) { + ret = -EINVAL; + goto put_native; + } + ret = vfio_add_to_pfn_list(dma, iova, *pfn, length); + if (ret) + goto put_native; + } + if (iommu->dirty_page_tracking) { + unsigned long pgshift = __ffs(vfio_dirty_pgsize(iommu)); + + bitmap_set(dma->bitmap, (iova - dma->iova) >> pgshift, + length >> pgshift); + } + return 0; +put_native: + if (put_pfn(*pfn, dma->prot) && do_accounting) + vfio_lock_acct(dma, -1, true); + return ret; +} + static int vfio_iommu_type1_pin_pages(void *iommu_data, struct iommu_group *iommu_group, dma_addr_t user_iova, @@ -897,119 +1014,72 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, { struct vfio_iommu *iommu = iommu_data; struct vfio_iommu_group *group; - int i, j, ret; - unsigned long remote_vaddr; - struct vfio_dma *dma; + unsigned long pinned = 0; bool do_accounting; dma_addr_t iova_end; size_t iova_size; + int i, ret; if (!iommu || !pages || npage <= 0) return -EINVAL; - - /* Supported for v2 version only */ + /* Callers recover the original byte offset from the returned page. */ + user_iova = ALIGN_DOWN(user_iova, PAGE_SIZE); if (!iommu->v2) return -EACCES; - if (check_mul_overflow(npage, PAGE_SIZE, &iova_size) || check_add_overflow(user_iova, iova_size - 1, &iova_end)) return -EOVERFLOW; mutex_lock(&iommu->lock); - if (WARN_ONCE(iommu->vaddr_invalid_count, "vfio_pin_pages not allowed with VFIO_UPDATE_VADDR\n")) { ret = -EBUSY; goto pin_done; } - - /* Fail if no dma_umap notifier is registered */ if (list_empty(&iommu->device_list)) { ret = -EINVAL; goto pin_done; } - - /* - * If iommu capable domain exist in the container then all pages are - * already pinned and accounted. Accounting should be done if there is no - * iommu capable domain in the container. - */ do_accounting = list_empty(&iommu->domain_list); - for (i = 0; i < npage; i++) { - unsigned long phys_pfn; - dma_addr_t iova; - struct vfio_pfn *vpfn; - - iova = user_iova + PAGE_SIZE * i; - dma = vfio_find_dma(iommu, iova, PAGE_SIZE); - if (!dma) { - ret = -EINVAL; - goto pin_unwind; - } - - if ((dma->prot & prot) != prot) { - ret = -EPERM; - goto pin_unwind; - } - - vpfn = vfio_iova_get_vfio_pfn(dma, iova); - if (vpfn) { - pages[i] = pfn_to_page(vpfn->pfn); - continue; - } - - remote_vaddr = dma->vaddr + (iova - dma->iova); - ret = vfio_pin_page_external(dma, remote_vaddr, &phys_pfn, - do_accounting); - if (ret) - goto pin_unwind; - - if (!pfn_valid(phys_pfn)) { - ret = -EINVAL; - goto pin_unwind; - } - - ret = vfio_add_to_pfn_list(dma, iova, phys_pfn); - if (ret) { - if (put_pfn(phys_pfn, dma->prot) && do_accounting) - vfio_lock_acct(dma, -1, true); - goto pin_unwind; - } + unsigned long done = 0; - pages[i] = pfn_to_page(phys_pfn); + pages[i] = NULL; + while (done < PAGE_SIZE) { + dma_addr_t iova = user_iova + PAGE_SIZE * i + done; + struct vfio_dma *dma = vfio_find_dma(iommu, iova, 1); + unsigned long length, pfn; - if (iommu->dirty_page_tracking) { - unsigned long pgshift = __ffs(iommu->pgsize_bitmap); - - /* - * Bitmap populated with the smallest supported page - * size - */ - bitmap_set(dma->bitmap, - (iova - dma->iova) >> pgshift, 1); + if (!dma || iova < dma->iova) { + ret = -EINVAL; + goto pin_unwind; + } + length = min_t(unsigned long, PAGE_SIZE - done, + dma->size - (iova - dma->iova)); + ret = vfio_pin_external_segment(iommu, dma, iova, length, + prot, do_accounting, &pfn); + if (ret) + goto pin_unwind; + pinned += length; + if (pages[i] && page_to_pfn(pages[i]) != pfn) { + ret = -EINVAL; + goto pin_unwind; + } + pages[i] = pfn_to_page(pfn); + done += length; } } - ret = i; - + ret = npage; group = vfio_iommu_find_iommu_group(iommu, iommu_group); if (!group->pinned_page_dirty_scope) { group->pinned_page_dirty_scope = true; iommu->num_non_pinned_groups--; } - goto pin_done; - pin_unwind: - pages[i] = NULL; - for (j = 0; j < i; j++) { - dma_addr_t iova; - - iova = user_iova + PAGE_SIZE * j; - dma = vfio_find_dma(iommu, iova, PAGE_SIZE); - vfio_unpin_page_external(dma, iova, do_accounting); - pages[j] = NULL; - } + vfio_unpin_external_range(iommu, user_iova, pinned, do_accounting); + while (i >= 0) + pages[i--] = NULL; pin_done: mutex_unlock(&iommu->lock); return ret; @@ -1019,39 +1089,21 @@ static void vfio_iommu_type1_unpin_pages(void *iommu_data, dma_addr_t user_iova, int npage) { struct vfio_iommu *iommu = iommu_data; - bool do_accounting; dma_addr_t iova_end; size_t iova_size; - int i; - - /* Supported for v2 version only */ - if (WARN_ON(!iommu->v2)) - return; + unsigned long unpinned; - if (WARN_ON(npage <= 0)) + if (WARN_ON(!iommu->v2) || WARN_ON(npage <= 0)) return; - + user_iova = ALIGN_DOWN(user_iova, PAGE_SIZE); if (WARN_ON(check_mul_overflow(npage, PAGE_SIZE, &iova_size) || check_add_overflow(user_iova, iova_size - 1, &iova_end))) return; - mutex_lock(&iommu->lock); - - do_accounting = list_empty(&iommu->domain_list); - for (i = 0; i < npage; i++) { - dma_addr_t iova = user_iova + PAGE_SIZE * i; - struct vfio_dma *dma; - - dma = vfio_find_dma(iommu, iova, PAGE_SIZE); - if (!dma) - break; - - vfio_unpin_page_external(dma, iova, do_accounting); - } - + unpinned = vfio_unpin_external_range(iommu, user_iova, iova_size, + list_empty(&iommu->domain_list)); mutex_unlock(&iommu->lock); - - WARN_ON(i != npage); + WARN_ON(unpinned != iova_size); } static long vfio_sync_unpin(struct vfio_dma *dma, struct vfio_domain *domain, @@ -1155,6 +1207,10 @@ static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma, if (!dma->size) return 0; + if (dma->fragments) { + vfio_fragments_unmap(iommu, dma); + return 0; + } if (list_empty(&iommu->domain_list)) return 0; @@ -1233,6 +1289,7 @@ static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma) { WARN_ON(!RB_EMPTY_ROOT(&dma->pfn_list)); vfio_unmap_unpin(iommu, dma, true); + vfio_fragments_destroy(dma); vfio_unlink_dma(iommu, dma); put_task_struct(dma->task); mmdrop(dma->mm); @@ -1246,21 +1303,18 @@ static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma) static void vfio_update_pgsize_bitmap(struct vfio_iommu *iommu) { struct vfio_domain *domain; + unsigned long minimum = SZ_4K; iommu->pgsize_bitmap = ULONG_MAX; - - list_for_each_entry(domain, &iommu->domain_list, next) + list_for_each_entry(domain, &iommu->domain_list, next) { iommu->pgsize_bitmap &= domain->domain->pgsize_bitmap; - - /* - * In case the IOMMU supports page sizes smaller than PAGE_SIZE - * we pretend PAGE_SIZE is supported and hide sub-PAGE_SIZE sizes. - * That way the user will be able to map/unmap buffers whose size/ - * start address is aligned with PAGE_SIZE. Pinning code uses that - * granularity while iommu driver can use the sub-PAGE_SIZE size - * to map the buffer. - */ - if (iommu->pgsize_bitmap & ~PAGE_MASK) { + minimum = max(minimum, 1UL << __ffs(domain->domain->pgsize_bitmap)); + } + if (IS_ENABLED(CONFIG_MM_SUBPAGE)) { + /* Every domain can compose its minimum pages into this common unit. */ + iommu->pgsize_bitmap &= ~(minimum - 1); + iommu->pgsize_bitmap |= minimum; + } else if (iommu->pgsize_bitmap & ~PAGE_MASK) { iommu->pgsize_bitmap &= PAGE_MASK; iommu->pgsize_bitmap |= PAGE_SIZE; } @@ -1367,6 +1421,8 @@ static void vfio_notify_dma_unmap(struct vfio_iommu *iommu, struct vfio_dma *dma) { struct vfio_device *device; + dma_addr_t iova = dma->iova; + size_t size = dma->size; if (list_empty(&iommu->device_list)) return; @@ -1377,11 +1433,12 @@ static void vfio_notify_dma_unmap(struct vfio_iommu *iommu, * call back down to this code and try to obtain the iommu->lock we must * drop it. */ - mutex_lock(&iommu->device_list_lock); + /* The range is a snapshot: another unmap may free dma while unlocked. */ mutex_unlock(&iommu->lock); + mutex_lock(&iommu->device_list_lock); list_for_each_entry(device, &iommu->device_list, iommu_entry) - device->ops->dma_unmap(device, dma->iova, dma->size); + device->ops->dma_unmap(device, iova, size); mutex_unlock(&iommu->device_list_lock); mutex_lock(&iommu->lock); @@ -1437,11 +1494,10 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, /* When dirty tracking is enabled, allow only min supported pgsize */ if ((unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) && - (!iommu->dirty_page_tracking || (bitmap->pgsize != pgsize))) { + (!iommu->dirty_page_tracking || (bitmap->pgsize != vfio_dirty_pgsize(iommu)))) { goto unlock; } - WARN_ON((pgsize - 1) & PAGE_MASK); again: /* * vfio-iommu-type1 (v1) - User mappings were coalesced together to @@ -1530,7 +1586,7 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, if (unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { ret = update_user_bitmap(bitmap->data, iommu, dma, - iova, pgsize); + iova, vfio_dirty_pgsize(iommu)); if (ret) break; } @@ -1587,6 +1643,13 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; int ret = 0; + if (dma->fragments) { + dma->size = map_size; + ret = vfio_fragments_map_all(iommu, dma); + if (ret) + vfio_remove_dma(iommu, dma); + return ret; + } vfio_batch_init(&batch); while (size) { @@ -1644,27 +1707,30 @@ static bool vfio_iommu_iova_dma_valid(struct vfio_iommu *iommu, return list_empty(iova); } -static int vfio_change_dma_owner(struct vfio_dma *dma) +static int vfio_change_dma_mm_pages(struct vfio_dma *dma, struct task_struct *task, + struct mm_struct *mm, bool lock_cap, unsigned long pages) { - struct task_struct *task = current->group_leader; - struct mm_struct *mm = current->mm; - long npage = dma->locked_vm; - bool lock_cap; + long old_pages = dma->locked_vm; int ret; - if (mm == dma->mm) - return 0; - - lock_cap = capable(CAP_IPC_LOCK); - ret = mm_lock_acct(task, mm, lock_cap, npage); + if (mm == dma->mm) { + if (pages == old_pages) + return 0; + ret = mm_lock_acct(task, mm, lock_cap, (long)pages - old_pages); + if (!ret) + dma->locked_vm = pages; + return ret; + } + ret = mm_lock_acct(task, mm, lock_cap, pages); if (ret) return ret; - if (mmget_not_zero(dma->mm)) { - mm_lock_acct(dma->task, dma->mm, dma->lock_cap, -npage); - mmput(dma->mm); + /* Metadata adoption can deduplicate charges, but not original PIN refs. */ + ret = mm_lock_acct(dma->task, dma->mm, dma->lock_cap, -old_pages); + if (ret) { + mm_lock_acct(task, mm, lock_cap, -(long)pages); + return ret; } - if (dma->task != task) { put_task_struct(dma->task); dma->task = get_task_struct(task); @@ -1673,9 +1739,40 @@ static int vfio_change_dma_owner(struct vfio_dma *dma) dma->mm = mm; mmgrab(dma->mm); dma->lock_cap = lock_cap; + dma->locked_vm = pages; return 0; } +static int vfio_change_dma_mm(struct vfio_dma *dma, struct task_struct *task, + struct mm_struct *mm, bool lock_cap) +{ + return vfio_change_dma_mm_pages(dma, task, mm, lock_cap, dma->locked_vm); +} + +static int vfio_change_dma_owner(struct vfio_iommu *iommu, struct vfio_dma *dma, + unsigned long vaddr) +{ + struct vfio_fragment_provider *provider; + int ret; + + if (dma->fragments || (mm_page_size(current->mm) == PAGE_SIZE && + IS_ALIGNED(vaddr, PAGE_SIZE))) + return vfio_change_dma_mm(dma, current->group_leader, current->mm, + capable(CAP_IPC_LOCK)); + + /* Prepare only metadata: failure must preserve every legacy pin/charge. */ + provider = vfio_fragments_prepare_native(iommu, dma); + if (IS_ERR(provider)) + return PTR_ERR(provider); + ret = vfio_change_dma_mm_pages(dma, current->group_leader, current->mm, + capable(CAP_IPC_LOCK), vfio_fragments_native_pages(provider)); + if (ret) + vfio_fragments_abort_native(provider); + else + vfio_fragments_commit_native(dma, provider); + return ret; +} + static int vfio_dma_do_map(struct vfio_iommu *iommu, struct vfio_iommu_type1_dma_map *map) { @@ -1713,8 +1810,6 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap); - WARN_ON((pgsize - 1) & PAGE_MASK); - if ((size | iova | vaddr) & (pgsize - 1)) { ret = -EINVAL; goto out_unlock; @@ -1728,7 +1823,7 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, dma->size != size) { ret = -EINVAL; } else { - ret = vfio_change_dma_owner(dma); + ret = vfio_change_dma_owner(iommu, dma, vaddr); if (ret) goto out_unlock; dma->vaddr = vaddr; @@ -1781,6 +1876,11 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, /* Insert zero-sized and grow as we map chunks of it */ vfio_link_dma(iommu, dma); + ret = vfio_fragments_init(dma, size); + if (ret) { + vfio_remove_dma(iommu, dma); + goto out_unlock; + } /* Don't pin and map if container doesn't contain IOMMU capable domain*/ if (list_empty(&iommu->domain_list)) @@ -1789,7 +1889,7 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, ret = vfio_pin_map_dma(iommu, dma, size); if (!ret && iommu->dirty_page_tracking) { - ret = vfio_dma_bitmap_alloc(dma, pgsize); + ret = vfio_dma_bitmap_alloc(dma, vfio_dirty_pgsize(iommu)); if (ret) vfio_remove_dma(iommu, dma); } @@ -1822,6 +1922,14 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, size_t pos = 0; dma = rb_entry(n, struct vfio_dma, node); + if (dma->fragments) { + ret = vfio_fragments_acquire(dma, 0, dma->size); + if (!ret) + ret = vfio_fragments_map_domain(dma, domain->domain); + if (ret) + goto unwind; + continue; + } while (pos < dma->size) { dma_addr_t iova = dma->iova + pos; @@ -1906,6 +2014,11 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); size_t pos = 0; + if (dma->fragments) { + iommu_unmap(domain->domain, dma->iova, dma->size); + vfio_fragments_release_unused(dma); + continue; + } if (dma->iommu_mapped) { iommu_unmap(domain->domain, dma->iova, dma->size); continue; @@ -2442,6 +2555,10 @@ static void vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu *iommu) long locked = 0, unlocked = 0; dma = rb_entry(n, struct vfio_dma, node); + if (dma->fragments) { + vfio_unmap_unpin(iommu, dma, true); + continue; + } unlocked += vfio_unmap_unpin(iommu, dma, false); p = rb_first(&dma->pfn_list); for (; p; p = rb_next(p)) { @@ -2646,7 +2763,8 @@ static void *vfio_iommu_type1_open(unsigned long arg) mutex_init(&iommu->lock); mutex_init(&iommu->device_list_lock); INIT_LIST_HEAD(&iommu->device_list); - iommu->pgsize_bitmap = PAGE_MASK; + iommu->pgsize_bitmap = IS_ENABLED(CONFIG_MM_SUBPAGE) ? + GENMASK(BITS_PER_LONG - 1, 12) : PAGE_MASK; INIT_LIST_HEAD(&iommu->emulated_iommu_groups); return iommu; @@ -2812,7 +2930,7 @@ static int vfio_iommu_migration_build_caps(struct vfio_iommu *iommu, cap_mig.flags = 0; /* support minimum pgsize */ - cap_mig.pgsize_bitmap = (size_t)1 << __ffs(iommu->pgsize_bitmap); + cap_mig.pgsize_bitmap = vfio_dirty_pgsize(iommu); cap_mig.max_dirty_bitmap_size = DIRTY_BITMAP_SIZE_MAX; return vfio_info_add_capability(caps, &cap_mig.header, sizeof(cap_mig)); @@ -2910,6 +3028,63 @@ static int vfio_iommu_type1_map_dma(struct vfio_iommu *iommu, return vfio_dma_do_map(iommu, &map); } +#ifdef CONFIG_COMPAT +struct compat_vfio_bitmap { + compat_u64 pgsize; + compat_u64 size; + compat_uptr_t data; +}; +#endif + +static size_t vfio_bitmap_size(void) +{ +#ifdef CONFIG_COMPAT + /* IA32 differs from AArch32/x32 in its u64 alignment and tail padding. */ + if (in_compat_syscall() && compat_need_64bit_alignment_fixup()) + return sizeof(struct compat_vfio_bitmap); +#endif + return sizeof(struct vfio_bitmap); +} + +static size_t vfio_dirty_bitmap_get_size(void) +{ + return offsetof(struct vfio_iommu_type1_dirty_bitmap_get, bitmap) + + vfio_bitmap_size(); +} + +static int vfio_bitmap_from_user(struct vfio_bitmap *bitmap, void __user *user) +{ +#ifdef CONFIG_COMPAT + if (in_compat_syscall()) { + compat_uptr_t data; + unsigned long offset = offsetof(struct vfio_bitmap, data); + + /* Read only the pointer field, never its trailing compat padding. */ + if (copy_from_user(bitmap, user, offset) || + get_user(data, (compat_uptr_t __user *)(user + offset))) + return -EFAULT; + bitmap->data = compat_ptr(data); + return 0; + } +#endif + return copy_from_user(bitmap, user, sizeof(*bitmap)) ? -EFAULT : 0; +} + +static int vfio_dirty_bitmap_get_from_user( + struct vfio_iommu_type1_dirty_bitmap_get *range, void __user *user) +{ +#ifdef CONFIG_COMPAT + if (in_compat_syscall()) { + unsigned long offset = offsetof(typeof(*range), bitmap); + + if (copy_from_user(range, user, offset)) + return -EFAULT; + return vfio_bitmap_from_user(&range->bitmap, user + offset); + } +#endif + return copy_from_user(range, user, sizeof(*range)) ? -EFAULT : 0; +} + static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, unsigned long arg) { @@ -2937,13 +3112,12 @@ static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, if (unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { unsigned long pgshift; - if (unmap.argsz < (minsz + sizeof(bitmap))) + if (unmap.argsz < minsz + vfio_bitmap_size()) return -EINVAL; - if (copy_from_user(&bitmap, - (void __user *)(arg + minsz), - sizeof(bitmap))) - return -EFAULT; + ret = vfio_bitmap_from_user(&bitmap, (void __user *)(arg + minsz)); + if (ret) + return ret; if (!access_ok((void __user *)bitmap.data, bitmap.size)) return -EINVAL; @@ -2992,7 +3166,7 @@ static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu, size_t pgsize; mutex_lock(&iommu->lock); - pgsize = 1 << __ffs(iommu->pgsize_bitmap); + pgsize = vfio_dirty_pgsize(iommu); if (!iommu->dirty_page_tracking) { ret = vfio_dma_bitmap_alloc_all(iommu, pgsize); if (!ret) @@ -3015,12 +3189,12 @@ static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu, size_t size, iommu_pgsize; dma_addr_t iova, iova_end; - if (!data_size || data_size < sizeof(range)) + if (data_size < vfio_dirty_bitmap_get_size()) return -EINVAL; - if (copy_from_user(&range, (void __user *)(arg + minsz), - sizeof(range))) - return -EFAULT; + ret = vfio_dirty_bitmap_get_from_user(&range, (void __user *)(arg + minsz)); + if (ret) + return ret; iova = range.iova; size = range.size; @@ -3046,7 +3220,7 @@ static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu, mutex_lock(&iommu->lock); - iommu_pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap); + iommu_pgsize = vfio_dirty_pgsize(iommu); /* allow only smallest supported pgsize */ if (range.bitmap.pgsize != iommu_pgsize) { @@ -3112,11 +3286,12 @@ static void vfio_iommu_type1_register_device(void *iommu_data, * Holding both locks here allows avoiding the device_list_lock in * several fast paths. See vfio_notify_dma_unmap() */ - mutex_lock(&iommu->lock); + /* Unmap callbacks hold device_list_lock while reentering iommu->lock. */ mutex_lock(&iommu->device_list_lock); + mutex_lock(&iommu->lock); list_add(&vdev->iommu_entry, &iommu->device_list); - mutex_unlock(&iommu->device_list_lock); mutex_unlock(&iommu->lock); + mutex_unlock(&iommu->device_list_lock); } static void vfio_iommu_type1_unregister_device(void *iommu_data, @@ -3127,11 +3302,12 @@ static void vfio_iommu_type1_unregister_device(void *iommu_data, if (!vdev->ops->dma_unmap) return; - mutex_lock(&iommu->lock); + /* Unmap callbacks hold device_list_lock while reentering iommu->lock. */ mutex_lock(&iommu->device_list_lock); + mutex_lock(&iommu->lock); list_del(&vdev->iommu_entry); - mutex_unlock(&iommu->device_list_lock); mutex_unlock(&iommu->lock); + mutex_unlock(&iommu->device_list_lock); } static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, @@ -3156,6 +3332,22 @@ static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, return -EPERM; mm = dma->mm; + if (dma->fragments) { + int ret; + + if (!kthread && current->mm != mm) + return -EFAULT; + offset = user_iova - dma->iova; + count = min(count, dma->size - offset); + ret = vfio_fragments_rw(dma, offset, data, count, write, copied); + if (*copied && write && iommu->dirty_page_tracking) { + unsigned long shift = __ffs(vfio_dirty_pgsize(iommu)); + + bitmap_set(dma->bitmap, offset >> shift, + ((offset + *copied - 1) >> shift) - (offset >> shift) + 1); + } + return ret; + } if (!mmget_not_zero(mm)) return -EPERM; @@ -3175,7 +3367,7 @@ static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, *copied = copy_to_user((void __user *)vaddr, data, count) ? 0 : count; if (*copied && iommu->dirty_page_tracking) { - unsigned long pgshift = __ffs(iommu->pgsize_bitmap); + unsigned long pgshift = __ffs(vfio_dirty_pgsize(iommu)); /* * Bitmap populated with the smallest supported page * size @@ -3281,3 +3473,13 @@ MODULE_VERSION(DRIVER_VERSION); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR(DRIVER_AUTHOR); MODULE_DESCRIPTION(DRIVER_DESC); + +#ifdef CONFIG_VFIO_TYPE1_ACCOUNT_KUNIT_TEST +#include "vfio_iommu_type1_account_test.c" +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_ARM64_USER4K_EXPERIMENTAL) +#include "vfio_iommu_type1_fragments_test.c" +#ifdef CONFIG_ARM64_USER4K_VFIO_IOCTL_TEST +#include "vfio_iommu_type1_ioctl_test.c" +#endif +#endif +#endif diff --git a/drivers/vfio/vfio_iommu_type1_fragments.c b/drivers/vfio/vfio_iommu_type1_fragments.c new file mode 100644 index 0000000000000..22d46a1b805a7 --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_fragments.c @@ -0,0 +1,609 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included by type1: retained byte ranges, rather than IOMMU PFNs, own pins. */ +#ifdef CONFIG_MM_SUBPAGE +struct vfio_fragment { + struct rb_node node; + unsigned long start; + struct user_page_fragment pin; +}; + +struct vfio_fragment_provider { + struct rb_root records; + struct xarray physical_pages; + unsigned long npinned; +}; + +static phys_addr_t vfio_fragment_phys(const struct user_page_fragment *pin) +{ + return PFN_PHYS(folio_pfn(pin->folio)) + pin->offset; +} + +/* Returns a containing range, or the first range after the requested byte. */ +static struct vfio_fragment *vfio_fragment_find(struct vfio_dma *dma, + unsigned long offset) +{ + struct rb_node *node = dma->fragments->records.rb_node; + struct vfio_fragment *next = NULL; + + while (node) { + struct vfio_fragment *record = rb_entry(node, struct vfio_fragment, node); + + if (offset < record->start) { + next = record; + node = node->rb_left; + } else if (offset - record->start >= record->pin.length) { + node = node->rb_right; + } else { + return record; + } + } + return next; +} + +static int vfio_fragments_init(struct vfio_dma *dma, size_t length) +{ + struct vfio_fragment_provider *provider; + + if (mm_page_size(dma->mm) == PAGE_SIZE && + IS_ALIGNED(dma->vaddr | dma->iova | length, PAGE_SIZE)) + return 0; + provider = kzalloc_obj(*provider, GFP_KERNEL_ACCOUNT); + if (!provider) + return -ENOMEM; + provider->records = RB_ROOT; + xa_init(&provider->physical_pages); + dma->fragments = provider; + return 0; +} + +static int vfio_fragment_store(struct vfio_dma *dma, unsigned long start, + struct user_page_fragment *pin) +{ + struct vfio_fragment_provider *provider = dma->fragments; + struct rb_node **link = &provider->records.rb_node, *parent = NULL; + unsigned long pfn = PHYS_PFN(vfio_fragment_phys(pin)); + unsigned long count = xa_to_value(xa_load(&provider->physical_pages, pfn)); + struct vfio_fragment *record; + int ret; + + record = kmalloc_obj(*record, GFP_KERNEL_ACCOUNT); + if (!record) + return -ENOMEM; + ret = xa_err(xa_store(&provider->physical_pages, pfn, xa_mk_value(count + 1), + GFP_KERNEL_ACCOUNT)); + if (ret) { + kfree(record); + return ret; + } + while (*link) { + struct vfio_fragment *cur = rb_entry(*link, struct vfio_fragment, node); + + parent = *link; + link = start < cur->start ? &(*link)->rb_left : &(*link)->rb_right; + } + record->start = start; + record->pin = *pin; + rb_link_node(&record->node, parent, link); + rb_insert_color(&record->node, &provider->records); + if (!count) + provider->npinned++; + *pin = (struct user_page_fragment) {}; + return 0; +} + +/* These descriptors borrow existing native GUP pins until owner commit. */ +static int vfio_fragment_native_pin(unsigned long pfn, struct user_page_fragment *pin) +{ + struct page *page; + + if (!pfn_valid(pfn)) + return -EOPNOTSUPP; + page = pfn_to_page(pfn); + if (PageReserved(page)) + return -EOPNOTSUPP; + *pin = (struct user_page_fragment) { + .folio = page_folio(page), + .offset = page_to_phys(page) - PFN_PHYS(folio_pfn(page_folio(page))), + .length = PAGE_SIZE, + .pinned = true, + }; + return 0; +} + +/* No PIN, GET or account reference was acquired while preparing metadata. */ +static void vfio_fragments_abort_native(struct vfio_fragment_provider *provider) +{ + struct rb_node *node; + + while ((node = rb_first(&provider->records))) { + struct vfio_fragment *record = rb_entry(node, struct vfio_fragment, node); + + rb_erase(node, &provider->records); + kfree(record); + } + xa_destroy(&provider->physical_pages); + kfree(provider); +} + +static struct vfio_fragment_provider * +vfio_fragments_prepare_native(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + struct vfio_fragment_provider *provider; + struct vfio_dma stage = {}; + struct vfio_domain *domain = NULL; + struct rb_node *node; + int ret; + + if (mm_page_size(dma->mm) != PAGE_SIZE || dma->has_rsvd) + return ERR_PTR(-EOPNOTSUPP); + if (dma->iommu_mapped) { + if (list_empty(&iommu->domain_list)) + return ERR_PTR(-EFAULT); + domain = list_first_entry(&iommu->domain_list, struct vfio_domain, next); + } + provider = kzalloc_obj(*provider, GFP_KERNEL_ACCOUNT); + if (!provider) + return ERR_PTR(-ENOMEM); + provider->records = RB_ROOT; + xa_init(&provider->physical_pages); + stage.fragments = provider; + /* One descriptor owns each original native GUP pin, including aliases. */ + if (domain) { + for (unsigned long offset = 0; offset < dma->size; offset += PAGE_SIZE) { + phys_addr_t phys = iommu_iova_to_phys(domain->domain, dma->iova + offset); + struct user_page_fragment pin; + + if (!phys || !IS_ALIGNED(phys, PAGE_SIZE)) { + ret = -EFAULT; + goto out; + } + ret = vfio_fragment_native_pin(PHYS_PFN(phys), &pin); + if (!ret) + ret = vfio_fragment_store(&stage, offset, &pin); + if (ret) + goto out; + cond_resched(); + } + } + for (node = rb_first(&dma->pfn_list); node; node = rb_next(node)) { + struct vfio_pfn *vpfn = rb_entry(node, struct vfio_pfn, node); + struct user_page_fragment pin; + + ret = vfio_fragment_native_pin(vpfn->pfn, &pin); + if (ret) + goto out; + if (!domain) { + /* Without a domain, the external pin is the cached buffer. */ + ret = vfio_fragment_store(&stage, vpfn->iova - dma->iova, &pin); + } else { + unsigned long count = xa_to_value(xa_load(&provider->physical_pages, + vpfn->pfn)); + + /* The external GUP reference may identify a different page. */ + ret = xa_err(xa_store(&provider->physical_pages, vpfn->pfn, + xa_mk_value(count + 1), GFP_KERNEL_ACCOUNT)); + if (!ret && !count) + provider->npinned++; + } + if (ret) + goto out; + } + return provider; +out: + vfio_fragments_abort_native(provider); + return ERR_PTR(ret); +} + +static unsigned long vfio_fragments_native_pages(struct vfio_fragment_provider *provider) +{ + return provider->npinned; +} + +static void vfio_fragments_commit_native(struct vfio_dma *dma, + struct vfio_fragment_provider *provider) +{ + struct rb_node *node; + + /* Mapped legacy external consumers own separate native GUP references. */ + if (dma->iommu_mapped) { + for (node = rb_first(&dma->pfn_list); node; node = rb_next(node)) + rb_entry(node, struct vfio_pfn, node)->native_pin = true; + } + dma->fragments = provider; +} + +static void vfio_fragments_put_native_pin(struct vfio_dma *dma, unsigned long pfn) +{ + struct vfio_fragment_provider *provider = dma->fragments; + unsigned long count = xa_to_value(xa_load(&provider->physical_pages, pfn)); + struct user_page_fragment pin; + + if (WARN_ON(!count || vfio_fragment_native_pin(pfn, &pin))) + return; + if (count == 1) { + xa_erase(&provider->physical_pages, pfn); + provider->npinned--; + } else { + xa_store(&provider->physical_pages, pfn, xa_mk_value(count - 1), GFP_NOWAIT); + } + release_user_fragments(&pin, 1, dma->prot & IOMMU_WRITE); + if (provider->npinned < dma->locked_vm) + vfio_lock_acct(dma, -(long)(dma->locked_vm - provider->npinned), true); +} + +static bool vfio_fragments_external_overlap(struct vfio_dma *dma, + dma_addr_t start, unsigned long length) +{ + struct rb_node *node = dma->pfn_list.rb_node; + + while (node) { + struct vfio_pfn *vpfn = rb_entry(node, struct vfio_pfn, node); + + if (start + length <= vpfn->iova) + node = node->rb_left; + else if (start >= vpfn->iova + vpfn->length) + node = node->rb_right; + else + return true; + } + return false; +} + +/* Device unmaps and their IOTLB synchronization must precede this call. */ +static void vfio_fragments_release_range(struct vfio_dma *dma, unsigned long start, + unsigned long length) +{ + struct vfio_fragment_provider *provider = dma->fragments; + struct vfio_fragment *first = vfio_fragment_find(dma, start); + struct rb_node *node = first ? &first->node : NULL; + unsigned long end = start + length; + + if (dma->iommu_mapped) + return; + while (node) { + struct vfio_fragment *record = rb_entry(node, struct vfio_fragment, node); + dma_addr_t iova = dma->iova + record->start; + unsigned long pfn, count; + + if (record->start >= end) + break; + node = rb_next(node); + /* Native-page consumers retain all records overlapping their page. */ + if (vfio_fragments_external_overlap(dma, iova, record->pin.length)) + continue; + pfn = PHYS_PFN(vfio_fragment_phys(&record->pin)); + count = xa_to_value(xa_load(&provider->physical_pages, pfn)); + if (WARN_ON(!count)) + continue; + if (count == 1) { + xa_erase(&provider->physical_pages, pfn); + provider->npinned--; + } else { + xa_store(&provider->physical_pages, pfn, xa_mk_value(count - 1), + GFP_NOWAIT); + } + rb_erase(&record->node, &provider->records); + release_user_fragments(&record->pin, 1, dma->prot & IOMMU_WRITE); + kfree(record); + cond_resched(); + } + if (provider->npinned < dma->locked_vm) + vfio_lock_acct(dma, -(long)(dma->locked_vm - provider->npinned), true); +} + +static void vfio_fragments_release_unused(struct vfio_dma *dma) +{ + vfio_fragments_release_range(dma, 0, dma->size); +} + +/* Cached records survive source VA replacement and the last source mm user. */ +static int vfio_fragments_acquire(struct vfio_dma *dma, unsigned long start, + unsigned long length) +{ + unsigned long end = start + length, original = start; + unsigned long capacity = PAGE_SIZE / sizeof(struct user_page_fragment); + struct user_page_fragment *batch = NULL; + bool live = false; + int ret = 0; + + while (start < end) { + struct vfio_fragment *record = vfio_fragment_find(dma, start); + unsigned long stop = end; + long count; + + if (record && record->start <= start) { + start = record->start + record->pin.length; + continue; + } + if (record) + stop = min(stop, record->start); + if (!live) { + if (!mmget_not_zero(dma->mm)) { + ret = -EFAULT; + goto out; + } + live = true; + batch = kmalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); + if (!batch) { + ret = -ENOMEM; + goto out; + } + } + count = pin_user_fragments_remote(dma->mm, dma->vaddr + start, + stop - start, FOLL_LONGTERM | (dma->prot & IOMMU_WRITE ? FOLL_WRITE : 0), + batch, capacity); + if (count <= 0) { + ret = count ?: -EFAULT; + goto out; + } + for (unsigned long i = 0; i < count; i++) { + unsigned long bytes = batch[i].length; + + if (WARN_ON(!bytes || bytes > stop - start || + offset_in_page(vfio_fragment_phys(&batch[i])) > + PAGE_SIZE - bytes)) + ret = -EFAULT; + else + ret = vfio_fragment_store(dma, start, &batch[i]); + if (ret) { + release_user_fragments(batch + i, count - i, false); + goto out; + } + start += bytes; + } + cond_resched(); + } + if (dma->fragments->npinned > dma->locked_vm) + ret = vfio_lock_acct(dma, dma->fragments->npinned - dma->locked_vm, false); +out: + kfree(batch); + if (live) + mmput(dma->mm); + if (ret) + vfio_fragments_release_range(dma, original, length); + return ret; +} + +/* Merge physical byte runs while retaining each original pin identity. */ +static int vfio_fragments_map_domain(struct vfio_dma *dma, struct iommu_domain *domain) +{ + unsigned long done = 0, min_size = 1UL << __ffs(domain->pgsize_bitmap); + int ret = 0; + + while (done < dma->size) { + struct vfio_fragment *record = vfio_fragment_find(dma, done); + unsigned long bytes, limit = dma->size - done; + phys_addr_t phys; + + if (WARN_ON(!record || record->start > done)) { + ret = -EFAULT; + goto out; + } + if (disable_hugepages) + limit = min(limit, PAGE_SIZE); + phys = vfio_fragment_phys(&record->pin) + done - record->start; + bytes = min(record->pin.length - (done - record->start), limit); + while (bytes < limit) { + record = vfio_fragment_find(dma, done + bytes); + if (!record || record->start != done + bytes || + vfio_fragment_phys(&record->pin) != phys + bytes) + break; + bytes += min_t(unsigned long, record->pin.length, limit - bytes); + } + if (!IS_ALIGNED((dma->iova + done) | phys | bytes, min_size)) { + ret = -EINVAL; + goto out; + } + ret = iommu_map(domain, dma->iova + done, phys, bytes, + dma->prot | IOMMU_CACHE, GFP_KERNEL_ACCOUNT); + if (ret) + goto out; + done += bytes; + cond_resched(); + } +out: + if (ret && done) + iommu_unmap(domain, dma->iova, done); + return ret; +} + +static int vfio_fragments_map_all(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + struct vfio_domain *domain; + int ret; + + ret = vfio_fragments_acquire(dma, 0, dma->size); + if (ret) + return ret; + list_for_each_entry(domain, &iommu->domain_list, next) { + ret = vfio_fragments_map_domain(dma, domain->domain); + if (ret) + goto unwind; + } + dma->iommu_mapped = true; + return 0; +unwind: + list_for_each_entry_continue_reverse(domain, &iommu->domain_list, next) + iommu_unmap(domain->domain, dma->iova, dma->size); + vfio_fragments_release_unused(dma); + return ret; +} + +static void vfio_fragments_unmap(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + struct vfio_domain *domain; + + if (dma->iommu_mapped) { + list_for_each_entry(domain, &iommu->domain_list, next) + WARN_ON(iommu_unmap(domain->domain, dma->iova, dma->size) != dma->size); + dma->iommu_mapped = false; + } + vfio_fragments_release_unused(dma); +} + +static void vfio_fragments_destroy(struct vfio_dma *dma) +{ + if (!dma->fragments) + return; + vfio_fragments_release_unused(dma); + if (WARN_ON(!RB_EMPTY_ROOT(&dma->fragments->records))) + return; + xa_destroy(&dma->fragments->physical_pages); + kfree(dma->fragments); + dma->fragments = NULL; +} + +/* The output page has no offset field: validate every byte and its native offset. */ +static int vfio_fragments_external(struct vfio_dma *dma, dma_addr_t iova, + unsigned long length, unsigned long *pfn) +{ + unsigned long first = iova - dma->iova, done = 0; + phys_addr_t phys = 0; + int ret; + + ret = vfio_fragments_acquire(dma, first, length); + if (ret) + return ret; + while (done < length) { + struct vfio_fragment *record = vfio_fragment_find(dma, first + done); + phys_addr_t next; + unsigned long offset; + + if (WARN_ON(!record || record->start > first + done)) { + ret = -EFAULT; + goto out; + } + offset = first + done - record->start; + next = vfio_fragment_phys(&record->pin) + offset; + if (!done) + phys = next; + if (offset_in_page(phys) != offset_in_page(iova) || next != phys + done) { + ret = -EINVAL; + goto out; + } + done += min_t(unsigned long, record->pin.length - offset, length - done); + } + *pfn = PHYS_PFN(phys); + ret = vfio_add_to_pfn_list(dma, iova, *pfn, length); +out: + if (ret) + vfio_fragments_release_range(dma, first, length); + return ret; +} + +/* CPU-only missing ranges take GET references, not new long-term DMA pins. */ +static int vfio_fragments_rw(struct vfio_dma *dma, unsigned long offset, void *data, + unsigned long length, bool write, size_t *copied) +{ + bool live = false; + int ret = 0; + + while (length) { + struct vfio_fragment *record = vfio_fragment_find(dma, offset); + struct user_page_fragment temp = {}, *pin; + unsigned long in_record = 0, bytes = length; + void *addr; + + if (record && record->start <= offset) { + pin = &record->pin; + in_record = offset - record->start; + } else { + long count; + + if (record) + bytes = min(bytes, record->start - offset); + if (!live) { + if (!mmget_not_zero(dma->mm)) { + ret = -EFAULT; + break; + } + live = true; + } + count = get_user_fragments_remote(dma->mm, dma->vaddr + offset, bytes, + write ? FOLL_WRITE : 0, &temp, 1); + if (count <= 0) { + ret = count ?: -EFAULT; + break; + } + pin = &temp; + } + bytes = min_t(unsigned long, bytes, pin->length - in_record); + addr = kmap_local_folio(pin->folio, pin->offset + in_record); + if (write) + memcpy(addr, data, bytes); + else + memcpy(data, addr, bytes); + kunmap_local(addr); + if (write && record && record->start <= offset) + mark_user_fragment_dirty(pin); + release_user_fragments(&temp, 1, write); + data += bytes; + offset += bytes; + length -= bytes; + *copied += bytes; + } + if (live) + mmput(dma->mm); + return ret; +} +#else +static struct vfio_fragment_provider * +vfio_fragments_prepare_native(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + return ERR_PTR(-EOPNOTSUPP); +} +static void vfio_fragments_abort_native(struct vfio_fragment_provider *provider) +{ +} +static unsigned long vfio_fragments_native_pages(struct vfio_fragment_provider *provider) +{ + return 0; +} +static void vfio_fragments_commit_native(struct vfio_dma *dma, + struct vfio_fragment_provider *provider) +{ +} +static void vfio_fragments_put_native_pin(struct vfio_dma *dma, unsigned long pfn) +{ +} +static int vfio_fragments_init(struct vfio_dma *dma, size_t length) +{ + return 0; +} +static void vfio_fragments_release_range(struct vfio_dma *dma, unsigned long start, + unsigned long length) +{ +} +static void vfio_fragments_release_unused(struct vfio_dma *dma) +{ +} +static int vfio_fragments_acquire(struct vfio_dma *dma, unsigned long start, + unsigned long length) +{ + return -EOPNOTSUPP; +} +static int vfio_fragments_map_domain(struct vfio_dma *dma, struct iommu_domain *domain) +{ + return -EOPNOTSUPP; +} +static int vfio_fragments_map_all(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + return -EOPNOTSUPP; +} +static void vfio_fragments_unmap(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ +} +static void vfio_fragments_destroy(struct vfio_dma *dma) +{ +} +static int vfio_fragments_external(struct vfio_dma *dma, dma_addr_t iova, + unsigned long length, unsigned long *pfn) +{ + return -EOPNOTSUPP; +} +static int vfio_fragments_rw(struct vfio_dma *dma, unsigned long offset, void *data, + unsigned long length, bool write, size_t *copied) +{ + return -EOPNOTSUPP; +} +#endif From e4eb5cae680525df47d642031ef9e39bb459d9e8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0687/2680] arm64: support smaller userspace translation granules per mm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep kernel mappings at their native granule while selecting TTBR0 geometry per address space. Add capability checks, table-fragment lifetime, TLB invalidation and exec selection for 4K on 16K/64K and 16K on 64K. The backend remains opt-in; private launch commands are experimental. [ryan: adapted to 7.2.10 — minimal; for DJ: show_pte() keeps 7.2's lockless walk for native mms and uses the _mm walkers only for alternate-granule mms; the contpte flush range takes this patch's mm_page_size() stride on top of 7.2's end-address fix] Signed-off-by: Diljit Singh (cherry picked from commit d2bfa3d271d01d3204306eb4c8cb6875a1310e35) --- arch/arm64/Kconfig | 1 + arch/arm64/Kconfig.debug | 21 +++ arch/arm64/include/asm/cpucaps.h | 5 + arch/arm64/include/asm/elf.h | 9 ++ arch/arm64/include/asm/kernel-pgtable.h | 2 +- arch/arm64/include/asm/mmu.h | 40 ++++++ arch/arm64/include/asm/mmu_context.h | 61 +++++++- arch/arm64/include/asm/mte.h | 8 ++ arch/arm64/include/asm/page.h | 33 ++++- arch/arm64/include/asm/pgalloc.h | 35 ++++- arch/arm64/include/asm/pgtable-hwdef.h | 12 +- arch/arm64/include/asm/pgtable.h | 51 +++++-- arch/arm64/include/asm/shmparam.h | 6 + arch/arm64/include/asm/tlb.h | 27 +++- arch/arm64/include/asm/tlbflush.h | 124 +++++++++++------ arch/arm64/include/asm/user-pgalloc.h | 59 ++++++++ arch/arm64/include/asm/user-pgtable.h | 164 ++++++++++++++++++++++ arch/arm64/include/asm/user-pte.h | 116 ++++++++++++++++ arch/arm64/kernel/cpufeature.c | 83 ++++++++++- arch/arm64/kernel/pi/idreg-override.c | 32 +++++ arch/arm64/kernel/pi/map_kernel.c | 2 +- arch/arm64/kernel/process.c | 33 ++++- arch/arm64/kernel/sys.c | 5 + arch/arm64/kernel/sys32.c | 6 + arch/arm64/kernel/vmlinux.lds.S | 5 +- arch/arm64/lib/mte.S | 46 +++++++ arch/arm64/mm/Makefile | 1 + arch/arm64/mm/context.c | 43 ++++++ arch/arm64/mm/contpte.c | 143 +++++++++++-------- arch/arm64/mm/copypage.c | 52 +++++++ arch/arm64/mm/fault.c | 34 ++--- arch/arm64/mm/fixmap.c | 2 +- arch/arm64/mm/init.c | 5 +- arch/arm64/mm/mmu.c | 4 +- arch/arm64/mm/pgd.c | 41 +++++- arch/arm64/mm/proc.S | 16 +-- arch/arm64/mm/user4k-pgalloc.c | 176 ++++++++++++++++++++++++ arch/arm64/tools/cpucaps | 2 + 38 files changed, 1353 insertions(+), 152 deletions(-) create mode 100644 arch/arm64/include/asm/user-pgalloc.h create mode 100644 arch/arm64/include/asm/user-pgtable.h create mode 100644 arch/arm64/include/asm/user-pte.h create mode 100644 arch/arm64/mm/user4k-pgalloc.c diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 02b53240ed70a..2f8d978dbd760 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -361,6 +361,7 @@ config FIX_EARLYCON_MEM config PGTABLE_LEVELS int + default 4 if ARM64_USER4K_EXPERIMENTAL default 2 if ARM64_16K_PAGES && ARM64_VA_BITS_36 default 2 if ARM64_64K_PAGES && ARM64_VA_BITS_42 default 3 if ARM64_64K_PAGES && (ARM64_VA_BITS_48 || ARM64_VA_BITS_52) diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index 265c4461031f4..d15ccdeee2125 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -21,3 +21,24 @@ config ARM64_RELOC_TEST tristate "Relocation testing module" source "drivers/hwtracing/coresight/Kconfig" + +config ARM64_USER4K_EXPERIMENTAL + bool "Experimental mixed-granule userspace" + select ARCH_HAS_MM_PGTABLE + select ARCH_HAS_USER_PAGE_SIZE + select MM_SUBPAGE + select MM_ID + select HAVE_ARCH_TLB_REMOVE_TABLE + select ARCH_SUPPORTS_PT_RECLAIM + depends on (ARM64_16K_PAGES || ARM64_64K_PAGES) && ARM64_VA_BITS_48 && ARM64_PA_BITS_48 + depends on !ARM64_LPA2 && !NO_PAGE_MAPCOUNT + default n + help + Build the in-development MM support for alternative 4K user tables + on 16K/64K kernels and 16K user tables on 64K kernels. + Ordinary address spaces and kernel tables retain the native granule. An experimental + one-shot prctl selects a supported granule for the next exec. + This is a development ABI: several MM interfaces and device mappings + still require integration and validation. The private prctl numbers + are not upstream assignments or a stable ABI promise. + Say N for normal kernels. diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index 25c61cda901c5..df3ab19c7697e 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -19,6 +19,11 @@ cpucap_is_possible(const unsigned int cap) "cap must be < ARM64_NCAPS"); switch (cap) { + case ARM64_HAS_USER16K_GRANULE: + return IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) && + IS_ENABLED(CONFIG_ARM64_64K_PAGES); + case ARM64_HAS_USER4K_GRANULE: + return IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL); case ARM64_HAS_EPAN: return IS_ENABLED(CONFIG_ARM64_EPAN); case ARM64_SVE: diff --git a/arch/arm64/include/asm/elf.h b/arch/arm64/include/asm/elf.h index d2779d604c7be..23320b49c3208 100644 --- a/arch/arm64/include/asm/elf.h +++ b/arch/arm64/include/asm/elf.h @@ -121,6 +121,9 @@ #define CORE_DUMP_USE_REGSET #define ELF_EXEC_PAGESIZE PAGE_SIZE +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define ELF_EXEC_PAGESIZE_MM(mm) mm_page_size(mm) +#endif /* * This is the base location for PIE (ET_DYN with INTERP) loads. On @@ -181,6 +184,12 @@ do { \ NEW_AUX_ENT(AT_IGNORE, 0); \ } while (0) +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +struct linux_binprm; +int arm64_bprm_prepare_mm(struct linux_binprm *bprm, bool compat); +#define arch_bprm_prepare_mm arm64_bprm_prepare_mm +#endif + #define ARCH_HAS_SETUP_ADDITIONAL_PAGES struct linux_binprm; extern int arch_setup_additional_pages(struct linux_binprm *bprm, diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h index 229ee7976f693..2bd73e773095c 100644 --- a/arch/arm64/include/asm/kernel-pgtable.h +++ b/arch/arm64/include/asm/kernel-pgtable.h @@ -27,7 +27,7 @@ #endif #define SWAPPER_BLOCK_SIZE (UL(1) << SWAPPER_BLOCK_SHIFT) -#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS - SWAPPER_SKIP_LEVEL) +#define SWAPPER_PGTABLE_LEVELS (ARM64_NATIVE_PGTABLE_LEVELS - SWAPPER_SKIP_LEVEL) #define INIT_IDMAP_PGTABLE_LEVELS (IDMAP_LEVELS - SWAPPER_SKIP_LEVEL) #define IDMAP_VA_BITS 48 diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index 5e1211c540ab2..15943bec53392 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -8,6 +8,10 @@ #include #define MMCF_AARCH32 0x1 /* mm context flag for AArch32 executables */ +#define MMCF_USER_4K 0x2 /* experimental, immutable mm table geometry */ +#define MMCF_USER_16K 0x4 /* experimental, immutable mm table geometry */ +#define MMCF_USER_64K 0x8 /* internal geometry; larger user ABI not enabled */ +#define MMCF_USER_GRANULE (MMCF_USER_4K | MMCF_USER_16K | MMCF_USER_64K) #define USER_ASID_BIT 48 #define USER_ASID_FLAG (UL(1) << USER_ASID_BIT) @@ -16,6 +20,15 @@ #include #include +/* AArch32 falls back to the native ABI when 4K translation is unavailable. */ +static inline unsigned int arm64_exec_page_shift(bool compat, bool has_4k, + unsigned int native_shift, unsigned int requested_shift) +{ + if (compat) + return has_4k ? 12 : native_shift; + return requested_shift ?: native_shift; +} + typedef struct { atomic64_t id; #ifdef CONFIG_COMPAT @@ -25,8 +38,35 @@ typedef struct { void *vdso; unsigned long flags; u8 pkey_allocation_map; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + /* Unissued PTE/PMD/PUD fragments; protected by page_table_lock. */ + void *user4k_pt_frag[3]; +#endif } mm_context_t; +static inline bool arm64_context_user4k(const mm_context_t *context) +{ + return IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) && + (context->flags & MMCF_USER_4K); +} + +static inline unsigned int arm64_context_page_shift(const mm_context_t *context) +{ + if (IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL)) { + if (context->flags & MMCF_USER_4K) + return 12; + if (context->flags & MMCF_USER_16K) + return 14; + if (context->flags & MMCF_USER_64K) + return 16; + } + return PAGE_SHIFT; +} + +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define arch_mm_page_shift(mm) arm64_context_page_shift(&(mm)->context) +#endif + /* * We use atomic64_read() here because the ASID for an 'mm_struct' can * be reallocated when scheduling one of its threads following a diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index 803b687581527..db27ff606555b 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -26,6 +26,17 @@ #include #include +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define arch_mm_init_exec arch_mm_init_exec +static inline void arch_mm_init_exec(struct mm_struct *mm, unsigned int page_shift) +{ + if (page_shift == 12) + mm->context.flags |= MMCF_USER_4K; + else if (page_shift == 14 && PAGE_SHIFT > 14) + mm->context.flags |= MMCF_USER_16K; +} +#endif + extern bool rodata_full; static inline void contextidr_thread_switch(struct task_struct *next) @@ -53,6 +64,29 @@ static inline void cpu_set_reserved_ttbr0(void) isb(); } +/* + * The caller has installed reserved_pg_dir in TTBR0 and excludes preemption. + * On a granule transition, complete that installation before changing TG0, + * then synchronise the new walk geometry before installing a user root. + * ASIDs remain unique across both granules; switching does not flush them. + */ +static inline void cpu_set_mm_tg0(struct mm_struct *mm) +{ +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + unsigned long tcr = read_sysreg(tcr_el1); + unsigned int shift = mm_page_shift(mm); + unsigned long tg0 = shift == 12 ? TCR_TG0_4K : + (shift == 14 ? TCR_TG0_16K : TCR_TG0_64K); + + if ((tcr & TCR_EL1_TG0_MASK) == tg0) + return; + + isb(); + write_sysreg((tcr & ~TCR_EL1_TG0_MASK) | tg0, tcr_el1); + isb(); +#endif +} + void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); static inline void cpu_switch_mm(pgd_t *pgd, struct mm_struct *mm) @@ -62,16 +96,21 @@ static inline void cpu_switch_mm(pgd_t *pgd, struct mm_struct *mm) } /* - * Ensure TCR.T0SZ is set to the provided value. + * Set TCR.T0SZ and restore native TG0 for the idmap/transition tables. */ static inline void __cpu_set_tcr_t0sz(unsigned long t0sz) { unsigned long tcr = read_sysreg(tcr_el1); + unsigned long mask = TCR_EL1_T0SZ_MASK; - if ((tcr & TCR_EL1_T0SZ_MASK) == t0sz) +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + mask |= TCR_EL1_TG0_MASK; + t0sz |= IS_ENABLED(CONFIG_ARM64_64K_PAGES) ? TCR_TG0_64K : TCR_TG0_16K; +#endif + if ((tcr & mask) == t0sz) return; - tcr &= ~TCR_EL1_T0SZ_MASK; + tcr &= ~mask; tcr |= t0sz; write_sysreg(tcr, tcr_el1); isb(); @@ -97,6 +136,8 @@ static inline void cpu_uninstall_idmap(void) local_flush_tlb_all(); __cpu_set_tcr_t0sz(TCR_T0SZ(vabits_actual)); + /* SW PAN will restore TTBR0 later, but already needs the mm's TG0. */ + cpu_set_mm_tg0(mm); if (mm != &init_mm && !system_uses_ttbr0_pan()) cpu_switch_mm(mm->pgd, mm); } @@ -168,6 +209,9 @@ init_new_context(struct task_struct *tsk, struct mm_struct *mm) { atomic64_set(&mm->context.id, 0); refcount_set(&mm->context.pinned, 0); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + memset(mm->context.user4k_pt_frag, 0, sizeof(mm->context.user4k_pt_frag)); +#endif /* pkey 0 is the default, so always reserve it. */ mm->context.pkey_allocation_map = BIT(0); @@ -175,6 +219,16 @@ init_new_context(struct task_struct *tsk, struct mm_struct *mm) return 0; } +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +void arm64_user4k_pt_cache_destroy(struct mm_struct *mm); + +static inline void destroy_context(struct mm_struct *mm) +{ + arm64_user4k_pt_cache_destroy(mm); +} +#define destroy_context destroy_context +#endif + static inline void arch_dup_pkeys(struct mm_struct *oldmm, struct mm_struct *mm) { @@ -241,6 +295,7 @@ static inline void __switch_mm(struct mm_struct *next) */ if (next == &init_mm) { cpu_set_reserved_ttbr0(); + cpu_set_mm_tg0(next); return; } diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h index 7f7b97e099968..890d49a4398b2 100644 --- a/arch/arm64/include/asm/mte.h +++ b/arch/arm64/include/asm/mte.h @@ -98,6 +98,8 @@ static inline bool try_page_mte_tagging(struct page *page) void mte_zero_clear_page_tags(void *addr); void mte_sync_tags(pte_t pte, unsigned int nr_pages); void mte_copy_page_tags(void *kto, const void *kfrom); +void mte_copy_subpage_tags_range(void *kto, const void *kfrom, unsigned long size); +void mte_clear_subpage_tags_range(void *addr, unsigned long size); void mte_thread_init_user(void); void mte_thread_switch(struct task_struct *next); void mte_cpu_setup(void); @@ -134,6 +136,12 @@ static inline void mte_sync_tags(pte_t pte, unsigned int nr_pages) static inline void mte_copy_page_tags(void *kto, const void *kfrom) { } +static inline void mte_copy_subpage_tags_range(void *kto, const void *kfrom, unsigned long size) +{ +} +static inline void mte_clear_subpage_tags_range(void *addr, unsigned long size) +{ +} static inline void mte_thread_init_user(void) { } diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h index 58200de8a221a..96fa9d8ea23c5 100644 --- a/arch/arm64/include/asm/page.h +++ b/arch/arm64/include/asm/page.h @@ -10,6 +10,11 @@ #include +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +/* A shared zero mapping must cover the largest supported user granule. */ +#define ARCH_ZERO_PAGE_SIZE (1UL << 16) +#endif + #ifndef __ASSEMBLER__ #include /* for READ_IMPLIES_EXEC */ @@ -26,11 +31,28 @@ void copy_user_highpage(struct page *to, struct page *from, unsigned long vaddr, struct vm_area_struct *vma); #define __HAVE_ARCH_COPY_USER_HIGHPAGE +#ifdef CONFIG_MM_SUBPAGE +void clear_user_subpage_range(struct page *page, unsigned int offset, + unsigned int size); +int copy_user_subpage_range(struct page *to, unsigned int to_offset, + struct page *from, unsigned int from_offset, + unsigned int size); +void clear_user_subpage(struct page *page, unsigned int offset); +int copy_user_subpage(struct page *to, unsigned int to_offset, + struct page *from, unsigned int from_offset); +#endif + void copy_highpage(struct page *to, struct page *from); #define __HAVE_ARCH_COPY_HIGHPAGE -struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, - unsigned long vaddr); +struct folio *vma_alloc_zeroed_movable_folio_order(struct vm_area_struct *vma, + unsigned long vaddr, unsigned int order); +#define vma_alloc_zeroed_movable_folio_order vma_alloc_zeroed_movable_folio_order +static inline struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, + unsigned long vaddr) +{ + return vma_alloc_zeroed_movable_folio_order(vma, vaddr, 0); +} #define vma_alloc_zeroed_movable_folio vma_alloc_zeroed_movable_folio bool tag_clear_highpages(struct page *to, int numpages, bool clear_pages); @@ -38,7 +60,14 @@ bool tag_clear_highpages(struct page *to, int numpages, bool clear_pages); #define copy_user_page(to, from, vaddr, pg) copy_page(to, from) +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +/* A table may occupy a 4K fragment of a native 16K physical page. */ +typedef pte_t *pgtable_t; +#define pgtable_to_page(table) virt_to_page(table) +#define pgtable_from_page(page) ((page) ? (pgtable_t)page_address(page) : NULL) +#else typedef struct page *pgtable_t; +#endif int pfn_is_map_memory(unsigned long pfn); diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h index 1b4509d3382c6..41b81203e575c 100644 --- a/arch/arm64/include/asm/pgalloc.h +++ b/arch/arm64/include/asm/pgalloc.h @@ -15,7 +15,15 @@ #define __HAVE_ARCH_PGD_FREE #define __HAVE_ARCH_PUD_FREE +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define __HAVE_ARCH_PTE_ALLOC_ONE +#define __HAVE_ARCH_PTE_FREE +#define __HAVE_ARCH_PMD_ALLOC_ONE +#define __HAVE_ARCH_PMD_FREE +#define __HAVE_ARCH_PUD_ALLOC_ONE +#endif #include +#include #define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t)) @@ -31,6 +39,12 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_AF; pudval |= (mm == &init_mm) ? PUD_TABLE_UXN : PUD_TABLE_PXN; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_mm_alt_granule(mm)) { + set_pud(pudp, __pud(__pa(pmdp) | pudval)); + return; + } +#endif __pud_populate(pudp, __pa(pmdp), pudval); } #else @@ -53,14 +67,24 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_AF; p4dval |= (mm == &init_mm) ? P4D_TABLE_UXN : P4D_TABLE_PXN; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_mm_alt_granule(mm)) { + set_p4d(p4dp, __p4d(__pa(pudp) | p4dval)); + return; + } +#endif __p4d_populate(p4dp, __pa(pudp), p4dval); } static inline void pud_free(struct mm_struct *mm, pud_t *pud) { - if (!pgtable_l4_enabled()) + if (mm_pud_folded(mm)) return; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + arm64_pgtable_free(pud); +#else __pud_free(mm, pud); +#endif } #else static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) @@ -117,7 +141,16 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) { VM_BUG_ON(mm == &init_mm); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_mm_alt_granule(mm)) { + set_pmd(pmdp, __pmd(__pa(ptep) | PMD_TYPE_TABLE | + PMD_TABLE_AF | PMD_TABLE_PXN)); + return; + } + __pmd_populate(pmdp, __pa(ptep), +#else __pmd_populate(pmdp, page_to_phys(ptep), +#endif PMD_TYPE_TABLE | PMD_TABLE_AF | PMD_TABLE_PXN); } diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h index 72f31800c7030..cb211cbcdb7fd 100644 --- a/arch/arm64/include/asm/pgtable-hwdef.h +++ b/arch/arm64/include/asm/pgtable-hwdef.h @@ -28,8 +28,10 @@ * * which gets simplified as : */ +#define ARM64_HW_PGTABLE_LEVELS_FOR(va_bits, page_shift) \ + (((va_bits) - PTDESC_ORDER - 1) / ((page_shift) - PTDESC_ORDER)) #define ARM64_HW_PGTABLE_LEVELS(va_bits) \ - (((va_bits) - PTDESC_ORDER - 1) / PTDESC_TABLE_SHIFT) + ARM64_HW_PGTABLE_LEVELS_FOR(va_bits, PAGE_SHIFT) /* * Size mapped by an entry at level n ( -1 <= n <= 3) @@ -44,7 +46,10 @@ * Rearranging it a bit we get : * (4 - n) * PTDESC_TABLE_SHIFT + PTDESC_ORDER */ -#define ARM64_HW_PGTABLE_LEVEL_SHIFT(n) (PTDESC_TABLE_SHIFT * (4 - (n)) + PTDESC_ORDER) +#define ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(n, page_shift) \ + (((page_shift) - PTDESC_ORDER) * (4 - (n)) + PTDESC_ORDER) +#define ARM64_HW_PGTABLE_LEVEL_SHIFT(n) \ + ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(n, PAGE_SHIFT) #define PTRS_PER_PTE (1 << PTDESC_TABLE_SHIFT) @@ -79,7 +84,8 @@ * PGDIR_SHIFT determines the size a top-level page table entry can map * (depending on the configuration, this level can be -1, 0, 1 or 2). */ -#define PGDIR_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(4 - CONFIG_PGTABLE_LEVELS) +#define ARM64_NATIVE_PGTABLE_LEVELS ARM64_HW_PGTABLE_LEVELS(VA_BITS) +#define PGDIR_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(4 - ARM64_NATIVE_PGTABLE_LEVELS) #define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT) #define PGDIR_MASK (~(PGDIR_SIZE-1)) #define PTRS_PER_PGD (1 << (VA_BITS - PGDIR_SHIFT)) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 27689c62bd25d..c349424aa0a33 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -467,6 +467,8 @@ static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) return pfn_pte(pte_pfn(pte) + nr, pte_pgprot(pte)); } +#include + /* * Hugetlb definitions. */ @@ -711,6 +713,23 @@ static inline void __set_ptes_anysz(struct mm_struct *mm, unsigned long addr, static inline void __set_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte, unsigned int nr) { + if (arm64_mm_alt_granule(mm)) { + unsigned int native_pages = DIV_ROUND_UP((pte_phys_mm(mm, pte) & ~PAGE_MASK) + + ((unsigned long)nr << mm_page_shift(mm)), PAGE_SIZE); + + page_table_check_ptes_set(mm, addr, ptep, pte, nr); + __sync_cache_and_tags(pte, native_pages); + for (;;) { + __check_safe_pte_update(mm, ptep, pte); + __set_pte_nosync(ptep, pte); + if (--nr == 0) + break; + ptep++; + pte = pte_advance_pfn_mm(mm, pte, 1); + } + __set_pte_complete(pte); + return; + } __set_ptes_anysz(mm, addr, ptep, pte, nr, PAGE_SIZE); } @@ -946,6 +965,8 @@ static inline pmd_t *pud_pgtable(pud_t pud) static __always_inline bool pgtable_l4_enabled(void) { + if (IS_ENABLED(CONFIG_ARM64_64K_PAGES)) + return false; if (CONFIG_PGTABLE_LEVELS > 4 || !IS_ENABLED(CONFIG_ARM64_LPA2)) return true; if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT)) @@ -955,6 +976,8 @@ static __always_inline bool pgtable_l4_enabled(void) static inline bool mm_pud_folded(const struct mm_struct *mm) { + if (arm64_mm_alt_granule(mm)) + return mm_page_shift(mm) == 16; return !pgtable_l4_enabled(); } #define mm_pud_folded mm_pud_folded @@ -994,6 +1017,8 @@ static inline phys_addr_t p4d_page_paddr(p4d_t p4d) static inline pud_t *p4d_to_folded_pud(p4d_t *p4dp, unsigned long addr) { + if (IS_ENABLED(CONFIG_ARM64_64K_PAGES)) + return (pud_t *)p4dp; /* Ensure that 'p4dp' indexes a page table according to 'addr' */ VM_BUG_ON(((addr >> P4D_SHIFT) ^ ((u64)p4dp >> 3)) % PTRS_PER_P4D); @@ -1216,6 +1241,8 @@ p4d_t *p4d_offset_lockless_folded(pgd_t *pgdp, pgd_t pgd, unsigned long addr) #endif /* CONFIG_PGTABLE_LEVELS > 4 */ +#include + #define pgd_ERROR(e) \ pr_err("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e)) @@ -1261,7 +1288,7 @@ static inline int __ptep_set_access_flags(struct vm_area_struct *vma, pte_t entry, int dirty) { return __ptep_set_access_flags_anysz(vma, address, ptep, entry, dirty, - PAGE_SIZE); + mm_page_size(vma->vm_mm)); } #ifdef CONFIG_TRANSPARENT_HUGEPAGE @@ -1389,7 +1416,7 @@ static inline void __clear_full_ptes(struct mm_struct *mm, unsigned long addr, if (--nr == 0) break; ptep++; - addr += PAGE_SIZE; + addr += mm_page_size(mm); } } @@ -1402,7 +1429,7 @@ static inline pte_t __get_and_clear_full_ptes(struct mm_struct *mm, pte = __ptep_get_and_clear(mm, addr, ptep); while (--nr) { ptep++; - addr += PAGE_SIZE; + addr += mm_page_size(mm); tmp_pte = __ptep_get_and_clear(mm, addr, ptep); if (pte_dirty(tmp_pte)) pte = pte_mkdirty(pte); @@ -1450,7 +1477,7 @@ static inline void __wrprotect_ptes(struct mm_struct *mm, unsigned long address, { unsigned int i; - for (i = 0; i < nr; i++, address += PAGE_SIZE, ptep++) + for (i = 0; i < nr; i++, address += mm_page_size(mm), ptep++) __ptep_set_wrprotect(mm, address, ptep); } @@ -1490,7 +1517,7 @@ static inline void __clear_young_dirty_ptes(struct vm_area_struct *vma, if (--nr == 0) break; ptep++; - addr += PAGE_SIZE; + addr += mm_page_size(vma->vm_mm); } } @@ -1516,9 +1543,11 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, * bits 0-1: present (must be zero) * bits 2: remember PG_anon_exclusive * bit 3: remember uffd-wp state + * bits 4-5: 4K slot offset within a native swap page (MM_SUBPAGE) * bits 6-10: swap type * bit 11: PTE_PRESENT_INVALID (must be zero) * bits 12-61: swap offset + * bits 62-63: high 4K slot offset bits (MM_SUBPAGE, up to 64K) */ #define __SWP_TYPE_SHIFT 6 #define __SWP_TYPE_BITS 5 @@ -1688,11 +1717,11 @@ static __always_inline void contpte_try_fold(struct mm_struct *mm, * We can't fold special mappings, because there is no associated folio. */ - const unsigned long contmask = CONT_PTES - 1; - bool valign = ((addr >> PAGE_SHIFT) & contmask) == contmask; + const unsigned long contmask = cont_ptes_mm(mm) - 1; + bool valign = ((addr >> mm_page_shift(mm)) & contmask) == contmask; if (unlikely(valign)) { - bool palign = (pte_pfn(pte) & contmask) == contmask; + bool palign = ((pte_phys_mm(mm, pte) >> mm_page_shift(mm)) & contmask) == contmask; if (unlikely(palign && pte_valid(pte) && !pte_cont(pte) && !pte_special(pte))) @@ -1713,7 +1742,11 @@ static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte) if (!pte_valid_cont(pte)) return 1; - return CONT_PTES - (((unsigned long)ptep >> 3) & (CONT_PTES - 1)); + { + unsigned int nr = arm64_cont_ptes(arm64_ptep_page_shift(ptep)); + + return nr - (((unsigned long)ptep >> 3) & (nr - 1)); + } } /* diff --git a/arch/arm64/include/asm/shmparam.h b/arch/arm64/include/asm/shmparam.h index f920e22ec677c..75120d3a71889 100644 --- a/arch/arm64/include/asm/shmparam.h +++ b/arch/arm64/include/asm/shmparam.h @@ -12,6 +12,12 @@ */ #define COMPAT_SHMLBA (4 * PAGE_SIZE) +/* SHMLBA also aligns kernel allocations; scale only SysV attachments. */ +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define arch_shm_attach_align(mm, align) \ + (((align) >> PAGE_SHIFT) << mm_page_shift(mm)) +#endif + #include #endif /* __ASM_SHMPARAM_H */ diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h index 751bd57bc3bac..21453f3125e0f 100644 --- a/arch/arm64/include/asm/tlb.h +++ b/arch/arm64/include/asm/tlb.h @@ -10,6 +10,11 @@ #include +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +void __tlb_remove_table(void *token); +/* Fragment tokens do not carry a private descriptor callback head. */ +#define tlb_table_is_ptdesc(token) (!((unsigned long)(token) & 1UL)) +#endif #define tlb_flush tlb_flush static void tlb_flush(struct mmu_gather *tlb); @@ -76,8 +81,14 @@ static inline void tlb_flush(struct mmu_gather *tlb) static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, unsigned long addr) { - struct ptdesc *ptdesc = page_ptdesc(pte); + struct ptdesc *ptdesc = page_ptdesc(pgtable_to_page(pte)); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (test_bit(PT_fragmented, &ptdesc->pt_flags.f)) { + tlb_remove_table(tlb, (void *)((unsigned long)pte | 1)); + return; + } +#endif tlb_remove_ptdesc(tlb, ptdesc); } @@ -87,6 +98,12 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, { struct ptdesc *ptdesc = virt_to_ptdesc(pmdp); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (test_bit(PT_fragmented, &ptdesc->pt_flags.f)) { + tlb_remove_table(tlb, (void *)((unsigned long)pmdp | 1)); + return; + } +#endif tlb_remove_ptdesc(tlb, ptdesc); } #endif @@ -97,9 +114,15 @@ static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp, { struct ptdesc *ptdesc = virt_to_ptdesc(pudp); - if (!pgtable_l4_enabled()) + if (mm_pud_folded(tlb->mm)) return; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (test_bit(PT_fragmented, &ptdesc->pt_flags.f)) { + tlb_remove_table(tlb, (void *)((unsigned long)pudp | 1)); + return; + } +#endif tlb_remove_ptdesc(tlb, ptdesc); } #endif diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index e0e84332f51be..ee6191ac2aca6 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -56,8 +57,8 @@ }) /* - * Get translation granule of the system, which is decided by - * PAGE_SIZE. Used by TTL. + * Encode a translation granule for TTL and range operands. Native kernel + * and stage-2 callers use PAGE_SHIFT; userspace callers use their mm. * - 4KB : 1 * - 16KB : 2 * - 64KB : 3 @@ -66,20 +67,25 @@ #define TLBI_TTL_TG_16K 2 #define TLBI_TTL_TG_64K 3 -static inline unsigned long get_trans_granule(void) +static inline unsigned long get_trans_granule_shift(unsigned int page_shift) { - switch (PAGE_SIZE) { - case SZ_4K: + switch (page_shift) { + case 12: return TLBI_TTL_TG_4K; - case SZ_16K: + case 14: return TLBI_TTL_TG_16K; - case SZ_64K: + case 16: return TLBI_TTL_TG_64K; default: return 0; } } +static inline unsigned long get_trans_granule(void) +{ + return get_trans_granule_shift(PAGE_SHIFT); +} + #ifdef CONFIG_ARM64_ERRATUM_4193714 extern cpumask_t sme_active_cpus; @@ -175,13 +181,14 @@ static __always_inline void ipas2e1is(u64 arg) __tlbi(ipas2e1is, arg); } -static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 level, - u16 asid) +static __always_inline void __tlbi_level_asid_granule(tlbi_op op, u64 addr, + u32 level, u16 asid, + unsigned int page_shift) { u64 arg = __TLBI_VADDR(addr, asid); if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && level <= 3) { - u64 ttl = level | (get_trans_granule() << 2); + u64 ttl = level | (get_trans_granule_shift(page_shift) << 2); FIELD_MODIFY(TLBI_TTL_MASK, &arg, ttl); } @@ -189,6 +196,12 @@ static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 level, op(arg); } +static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 level, + u16 asid) +{ + __tlbi_level_asid_granule(op, addr, level, asid, PAGE_SHIFT); +} + static inline void __tlbi_level(tlbi_op op, u64 addr, u32 level) { __tlbi_level_asid(op, addr, level, 0); @@ -471,36 +484,45 @@ static __always_inline void ripas2e1is(u64 arg) __tlbi(ripas2e1is, arg); } -static __always_inline void __tlbi_range(tlbi_op op, u64 addr, - u16 asid, int scale, int num, - u32 level, bool lpa2) +static __always_inline void __tlbi_range_granule(tlbi_op op, u64 addr, + u16 asid, int scale, int num, + u32 level, bool lpa2, + unsigned int page_shift) { u64 arg = 0; - arg |= FIELD_PREP(TLBIR_BADDR_MASK, addr >> (lpa2 ? 16 : PAGE_SHIFT)); + arg |= FIELD_PREP(TLBIR_BADDR_MASK, addr >> (lpa2 ? 16 : page_shift)); arg |= FIELD_PREP(TLBIR_TTL_MASK, level > 3 ? 0 : level); arg |= FIELD_PREP(TLBIR_NUM_MASK, num); arg |= FIELD_PREP(TLBIR_SCALE_MASK, scale); - arg |= FIELD_PREP(TLBIR_TG_MASK, get_trans_granule()); + arg |= FIELD_PREP(TLBIR_TG_MASK, get_trans_granule_shift(page_shift)); arg |= FIELD_PREP(TLBIR_ASID_MASK, asid); op(arg); } -static __always_inline void __flush_tlb_range_op(tlbi_op lop, tlbi_op rop, +/* + * Production callers supply the constant system_supports_tlb_range function, + * which inlines to the existing alternative branch inside the loop. Do not + * evaluate it into a bool outside the loop: that adds runtime flag handling + * to the native path. Recording tests supply constant true/false predicates. + */ +static __always_inline void __flush_tlb_range_op_granule(tlbi_op lop, tlbi_op rop, u64 start, size_t pages, u64 stride, u16 asid, - u32 level, bool lpa2) + u32 level, bool lpa2, + unsigned int page_shift, + bool (*range_supported)(void)) { - u64 addr = start, end = start + pages * PAGE_SIZE; + u64 addr = start, end = start + (pages << page_shift); int scale = 3; while (addr != end) { int num; - pages = (end - addr) >> PAGE_SHIFT; + pages = (end - addr) >> page_shift; - if (!system_supports_tlb_range() || pages == 1) + if (!range_supported() || pages == 1) goto invalidate_one; if (lpa2 && !IS_ALIGNED(addr, SZ_64K)) @@ -508,26 +530,41 @@ static __always_inline void __flush_tlb_range_op(tlbi_op lop, tlbi_op rop, num = __TLBI_RANGE_NUM(pages, scale); if (num >= 0) { - __tlbi_range(rop, addr, asid, scale, num, level, lpa2); - addr += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; + __tlbi_range_granule(rop, addr, asid, scale, num, level, + lpa2, page_shift); + addr += __TLBI_RANGE_PAGES(num, scale) << page_shift; } scale--; continue; invalidate_one: - __tlbi_level_asid(lop, addr, level, asid); + __tlbi_level_asid_granule(lop, addr, level, asid, page_shift); addr += stride; } } +static __always_inline void __flush_tlb_range_op(tlbi_op lop, tlbi_op rop, + u64 start, size_t pages, + u64 stride, u16 asid, + u32 level, bool lpa2) +{ + __flush_tlb_range_op_granule(lop, rop, start, pages, stride, asid, + level, lpa2, PAGE_SHIFT, system_supports_tlb_range); +} + #define __flush_s1_tlb_range_op(op, start, pages, stride, asid, tlb_level) \ __flush_tlb_range_op(op, r##op, start, pages, stride, asid, tlb_level, lpa2_is_enabled()) #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \ __flush_tlb_range_op(op, r##op, start, pages, stride, 0, tlb_level, kvm_lpa2_is_enabled()) -static inline bool __flush_tlb_range_limit_excess(unsigned long pages, - unsigned long stride) +#define __flush_user_tlb_range_op(op, start, pages, stride, asid, level, shift) \ + __flush_tlb_range_op_granule(op, r##op, start, pages, stride, asid, \ + level, lpa2_is_enabled(), shift, system_supports_tlb_range) + +static inline bool __flush_tlb_range_limit_excess_granule(unsigned long pages, + unsigned long stride, + unsigned int page_shift) { /* * Assume that the worst case number of DVM ops required to flush a @@ -538,7 +575,13 @@ static inline bool __flush_tlb_range_limit_excess(unsigned long pages, if (system_supports_tlb_range()) return pages > MAX_TLBI_RANGE_PAGES; - return pages >= (MAX_DVM_OPS * stride) >> PAGE_SHIFT; + return pages >= (MAX_DVM_OPS * stride) >> page_shift; +} + +static inline bool __flush_tlb_range_limit_excess(unsigned long pages, + unsigned long stride) +{ + return __flush_tlb_range_limit_excess_granule(pages, stride, PAGE_SHIFT); } typedef unsigned __bitwise tlbf_t; @@ -565,10 +608,11 @@ static __always_inline void __do_flush_tlb_range(struct vm_area_struct *vma, { struct mm_struct *mm = vma->vm_mm; unsigned long asid, pages; + unsigned int page_shift = mm_page_shift(mm); - pages = (end - start) >> PAGE_SHIFT; + pages = (end - start) >> page_shift; - if (__flush_tlb_range_limit_excess(pages, stride)) { + if (__flush_tlb_range_limit_excess_granule(pages, stride, page_shift)) { flush_tlb_mm(mm); return; } @@ -582,20 +626,20 @@ static __always_inline void __do_flush_tlb_range(struct vm_area_struct *vma, switch (flags & (TLBF_NOWALKCACHE | TLBF_NOBROADCAST)) { case TLBF_NONE: - __flush_s1_tlb_range_op(vae1is, start, pages, stride, - asid, tlb_level); + __flush_user_tlb_range_op(vae1is, start, pages, stride, + asid, tlb_level, page_shift); break; case TLBF_NOWALKCACHE: - __flush_s1_tlb_range_op(vale1is, start, pages, stride, - asid, tlb_level); + __flush_user_tlb_range_op(vale1is, start, pages, stride, + asid, tlb_level, page_shift); break; case TLBF_NOBROADCAST: /* Combination unused */ BUG(); break; case TLBF_NOWALKCACHE | TLBF_NOBROADCAST: - __flush_s1_tlb_range_op(vale1, start, pages, stride, - asid, tlb_level); + __flush_user_tlb_range_op(vale1, start, pages, stride, + asid, tlb_level, page_shift); break; } @@ -629,16 +673,18 @@ static inline void flush_tlb_range(struct vm_area_struct *vma, * Set the tlb_level to TLBI_TTL_UNKNOWN because we can not get enough * information here. */ - __flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, TLBF_NONE); + __flush_tlb_range(vma, start, end, mm_page_size(vma->vm_mm), + TLBI_TTL_UNKNOWN, TLBF_NONE); } static inline void __flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr, tlbf_t flags) { - unsigned long start = round_down(uaddr, PAGE_SIZE); - unsigned long end = start + PAGE_SIZE; + unsigned long page_size = mm_page_size(vma->vm_mm); + unsigned long start = round_down(uaddr, page_size); + unsigned long end = start + page_size; - __do_flush_tlb_range(vma, start, end, PAGE_SIZE, 3, + __do_flush_tlb_range(vma, start, end, page_size, 3, TLBF_NOWALKCACHE | flags); } @@ -688,7 +734,7 @@ static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *b { struct vm_area_struct vma = { .vm_mm = mm, .vm_flags = 0 }; - __flush_tlb_range(&vma, start, end, PAGE_SIZE, 3, + __flush_tlb_range(&vma, start, end, mm_page_size(mm), 3, TLBF_NOWALKCACHE | TLBF_NOSYNC); } diff --git a/arch/arm64/include/asm/user-pgalloc.h b/arch/arm64/include/asm/user-pgalloc.h new file mode 100644 index 0000000000000..3b4dbde0c7736 --- /dev/null +++ b/arch/arm64/include/asm/user-pgalloc.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_USER_PGALLOC_H +#define __ASM_USER_PGALLOC_H + +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + +enum arm64_user4k_pt_level { + USER4K_PT_PTE, + USER4K_PT_PMD, + USER4K_PT_PUD, +}; + +/* Explicit geometry entry point; callers must keep each mm's granule fixed. */ +void *arm64_user_pt_alloc_granule(struct mm_struct *mm, + enum arm64_user4k_pt_level level, unsigned int shift); +void *arm64_user4k_pt_alloc(struct mm_struct *mm, enum arm64_user4k_pt_level level); +void arm64_user4k_pt_free(void *table); +void arm64_user4k_pt_cache_destroy(struct mm_struct *mm); +void arm64_pgtable_free(void *table); +void arm64_pte_free_defer(struct mm_struct *mm, pgtable_t table); + +static inline pgtable_t pte_alloc_one_noprof(struct mm_struct *mm) +{ + if (arm64_mm_alt_granule(mm)) + return arm64_user_pt_alloc_granule(mm, USER4K_PT_PTE, mm_page_shift(mm)); + return __pte_alloc_one_noprof(mm, GFP_PGTABLE_USER); +} +#define pte_alloc_one(...) alloc_hooks(pte_alloc_one_noprof(__VA_ARGS__)) + +static inline void pte_free(struct mm_struct *mm, pgtable_t table) +{ + arm64_pgtable_free(table); +} + +static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return arm64_user_pt_alloc_granule(mm, USER4K_PT_PMD, mm_page_shift(mm)); + return __pmd_alloc_one_noprof(mm, addr); +} +#define pmd_alloc_one(...) alloc_hooks(pmd_alloc_one_noprof(__VA_ARGS__)) + +static inline void pmd_free(struct mm_struct *mm, pmd_t *table) +{ + arm64_pgtable_free(table); +} + +static inline pud_t *pud_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return arm64_user_pt_alloc_granule(mm, USER4K_PT_PUD, mm_page_shift(mm)); + return __pud_alloc_one_noprof(mm, addr); +} +#define pud_alloc_one(...) alloc_hooks(pud_alloc_one_noprof(__VA_ARGS__)) + +#define pte_free_defer arm64_pte_free_defer + +#endif +#endif diff --git a/arch/arm64/include/asm/user-pgtable.h b/arch/arm64/include/asm/user-pgtable.h new file mode 100644 index 0000000000000..b244f2387a7b4 --- /dev/null +++ b/arch/arm64/include/asm/user-pgtable.h @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_USER_PGTABLE_H +#define __ASM_USER_PGTABLE_H + +/* Included by asm/pgtable.h after the native accessors. */ +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + +/* Alternative tables use 48-bit VA; 64K folds the PUD level. */ +static inline unsigned int arm64_user_level_shift(const struct mm_struct *mm, + unsigned int level) +{ + unsigned int shift = mm_page_shift(mm); + + if (shift == 16 && !level) + level = 1; + return shift + (3 - level) * (shift - 3); +} + +static inline unsigned long arm64_user_table_index(const struct mm_struct *mm, + unsigned long addr, unsigned int level) +{ + unsigned int shift = arm64_user_level_shift(mm, level); + unsigned int bits = level ? mm_page_shift(mm) - 3 : 48 - shift; + + return (addr >> shift) & ((1UL << bits) - 1); +} + +#define p4d_none_mm(mm, p4d) \ + (arm64_mm_alt_granule(mm) ? (!mm_pud_folded(mm) && !p4d_val(p4d)) : p4d_none(p4d)) +#define p4d_bad_mm(mm, p4d) \ + (arm64_mm_alt_granule(mm) ? (!mm_pud_folded(mm) && ((p4d_val(p4d) & P4D_TYPE_MASK) != P4D_TYPE_TABLE)) : p4d_bad(p4d)) +#define p4d_present_mm(mm, p4d) (!p4d_none_mm(mm, p4d)) +static inline void p4d_clear_mm(struct mm_struct *mm, p4d_t *p4d) +{ + if (arm64_mm_alt_granule(mm)) { + if (!mm_pud_folded(mm)) + set_p4d(p4d, __p4d(0)); + } else { + p4d_clear(p4d); + } +} +#define p4d_clear_mm p4d_clear_mm + + +/* pgtable_t preserves the fragment offset; pmd_page() remains its owner. */ +#define pmd_pgtable(pmd) ((pgtable_t)__va(pmd_val(pmd) & ARM64_USER4K_TABLE_ADDR_MASK)) + +static inline void *arm64_user4k_table(u64 descriptor) +{ + /* Unlike native __pXd_to_phys(), retain sub-native table address bits. */ + return __va(descriptor & ARM64_USER4K_TABLE_ADDR_MASK); +} + +static inline pgd_t *pgd_offset_pgd_mm(struct mm_struct *mm, pgd_t *pgd, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return pgd + arm64_user_table_index(mm, addr, 0); + return pgd + ((addr >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)); +} + +#define pgd_offset_pgd_mm pgd_offset_pgd_mm + +static inline pgd_t *pgd_offset_mm(struct mm_struct *mm, unsigned long addr) +{ + return pgd_offset_pgd_mm(mm, mm->pgd, addr); +} + +#define pgd_offset(mm, addr) pgd_offset_mm(mm, addr) + +static inline p4d_t *p4d_offset_mm(struct mm_struct *mm, pgd_t *pgd, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return (p4d_t *)pgd; + return p4d_offset(pgd, addr); +} + +#define p4d_offset_mm p4d_offset_mm + +static inline pud_t *pud_offset_mm(struct mm_struct *mm, p4d_t *p4d, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) { + if (mm_pud_folded(mm)) + return (pud_t *)p4d; + return (pud_t *)arm64_user4k_table(p4d_val(READ_ONCE(*p4d))) + + arm64_user_table_index(mm, addr, 1); + } + return pud_offset(p4d, addr); +} + +#define pud_offset_mm pud_offset_mm + +static inline pmd_t *pud_pgtable_mm(struct mm_struct *mm, pud_t pud) +{ + if (arm64_mm_alt_granule(mm)) + return arm64_user4k_table(pud_val(pud)); + return pud_pgtable(pud); +} + +#define pud_pgtable_mm pud_pgtable_mm + +static inline pmd_t *pmd_offset_mm(struct mm_struct *mm, pud_t *pud, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return (pmd_t *)arm64_user4k_table(pud_val(READ_ONCE(*pud))) + + arm64_user_table_index(mm, addr, 2); + return (pmd_t *)pud_pgtable(READ_ONCE(*pud)) + + ((addr >> PMD_SHIFT) & (PTRS_PER_PMD - 1)); +} + +#define pmd_offset_mm pmd_offset_mm + +static inline pte_t *pte_offset_kernel_mm(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return (pte_t *)arm64_user4k_table(pmd_val(READ_ONCE(*pmd))) + + arm64_user_table_index(mm, addr, 3); + return (pte_t *)pmd_page_vaddr(READ_ONCE(*pmd)) + + ((addr >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)); +} + +#define pte_offset_kernel_mm pte_offset_kernel_mm +#define __pte_map_mm(mm, pmd, addr) pte_offset_kernel_mm(mm, pmd, addr) + +#define pmd_shift_mm(mm) (arm64_mm_alt_granule(mm) ? arm64_user_level_shift(mm, 2) : PMD_SHIFT) +#define pud_shift_mm(mm) (arm64_mm_alt_granule(mm) ? arm64_user_level_shift(mm, 1) : PUD_SHIFT) +#define p4d_shift_mm(mm) (arm64_mm_alt_granule(mm) ? arm64_user_level_shift(mm, 0) : P4D_SHIFT) +#define pmd_size_mm(mm) (1UL << pmd_shift_mm(mm)) +#define pud_size_mm(mm) (1UL << pud_shift_mm(mm)) +#define p4d_size_mm(mm) (1UL << p4d_shift_mm(mm)) +#define pgd_size_mm(mm) (arm64_mm_alt_granule(mm) ? (1UL << arm64_user_level_shift(mm, 0)) : PGDIR_SIZE) +#define pte_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PTE * sizeof(pte_t)) +#define pmd_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PMD * sizeof(pmd_t)) +#define pud_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PUD * sizeof(pud_t)) +#define pmd_mask_mm(mm) (~(pmd_size_mm(mm) - 1)) +#define pud_mask_mm(mm) (~(pud_size_mm(mm) - 1)) +#define p4d_mask_mm(mm) (~(p4d_size_mm(mm) - 1)) +#define pgd_mask_mm(mm) (~(pgd_size_mm(mm) - 1)) + +static inline unsigned long arm64_user4k_addr_end(unsigned long addr, + unsigned long end, unsigned int shift) +{ + unsigned long size = 1UL << shift; + unsigned long boundary = (addr + size) & ~(size - 1); + + return boundary - 1 < end - 1 ? boundary : end; +} + +/* Native fallbacks retain their architecture-specific folding semantics. */ +#define pgd_addr_end_mm(mm, addr, end) \ + (arm64_mm_alt_granule(mm) ? arm64_user4k_addr_end(addr, end, arm64_user_level_shift(mm, 0)) : pgd_addr_end(addr, end)) +#define p4d_addr_end_mm(mm, addr, end) \ + (arm64_mm_alt_granule(mm) ? (end) : p4d_addr_end(addr, end)) +#define pud_addr_end_mm(mm, addr, end) \ + (arm64_mm_alt_granule(mm) ? arm64_user4k_addr_end(addr, end, arm64_user_level_shift(mm, 1)) : pud_addr_end(addr, end)) +#define pmd_addr_end_mm(mm, addr, end) \ + (arm64_mm_alt_granule(mm) ? arm64_user4k_addr_end(addr, end, arm64_user_level_shift(mm, 2)) : pmd_addr_end(addr, end)) + +#endif /* CONFIG_ARM64_USER4K_EXPERIMENTAL */ +#endif diff --git a/arch/arm64/include/asm/user-pte.h b/arch/arm64/include/asm/user-pte.h new file mode 100644 index 0000000000000..aa4c1e3a5aab9 --- /dev/null +++ b/arch/arm64/include/asm/user-pte.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_USER_PTE_H +#define __ASM_USER_PTE_H + +/* Included after native leaf accessors, before their batched setters. */ +#define ARM64_USER4K_TABLE_ADDR_MASK GENMASK_ULL(47, 12) + +static inline bool arm64_mm_user4k(const struct mm_struct *mm) +{ + return mm && arm64_context_user4k(&mm->context); +} + +static inline bool arm64_mm_alt_granule(const struct mm_struct *mm) +{ + return mm && arm64_context_page_shift(&mm->context) != PAGE_SHIFT; +} + +static inline u64 arm64_user_leaf_addr_mask(const struct mm_struct *mm) +{ + return ARM64_USER4K_TABLE_ADDR_MASK & mm_page_mask(mm); +} + +/* PFNs still identify native pages; leaf physical addresses identify slots. */ +static inline phys_addr_t pte_phys_mm(const struct mm_struct *mm, pte_t pte) +{ + if (arm64_mm_alt_granule(mm)) + return pte_val(pte) & arm64_user_leaf_addr_mask(mm); + return __pte_to_phys(pte); +} +#define pte_phys_mm pte_phys_mm + +static inline pte_t phys_pte_mm(const struct mm_struct *mm, phys_addr_t phys, + pgprot_t prot) +{ + if (arm64_mm_alt_granule(mm)) { + VM_BUG_ON(phys & ~arm64_user_leaf_addr_mask(mm)); + return __pte(phys | pgprot_val(prot)); + } + return pfn_pte(phys >> PAGE_SHIFT, prot); +} +#define phys_pte_mm phys_pte_mm + +static inline pgprot_t pte_pgprot_mm(const struct mm_struct *mm, pte_t pte) +{ + if (arm64_mm_alt_granule(mm)) + return __pgprot(pte_val(pte) & ~arm64_user_leaf_addr_mask(mm)); + return pte_pgprot(pte); +} +#define pte_pgprot_mm pte_pgprot_mm + + +#ifdef CONFIG_MM_SUBPAGE +/* Slot bits belong to the PTE, not the native swap-cache index. */ +#define ARM64_SWP_SUBPAGE_MASK (GENMASK_ULL(5, 4) | GENMASK_ULL(63, 62)) + +static inline unsigned int pte_swp_subpage_offset(pte_t pte) +{ + return ((((pte_val(pte) >> 4) & 3) | + ((pte_val(pte) >> 60) & 12)) << 12); +} + +static inline pte_t pte_swp_set_subpage_offset(pte_t pte, unsigned int offset) +{ + VM_BUG_ON(!IS_ALIGNED(offset, SZ_4K) || offset >= PAGE_SIZE); + return __pte((pte_val(pte) & ~ARM64_SWP_SUBPAGE_MASK) | + (((u64)(offset >> 12) & 3) << 4) | + (((u64)(offset >> 12) & 12) << 60)); +} +#endif + +static inline pte_t pte_advance_pfn_mm(const struct mm_struct *mm, pte_t pte, + unsigned long nr) +{ + if (arm64_mm_alt_granule(mm)) + return phys_pte_mm(mm, pte_phys_mm(mm, pte) + (nr << mm_page_shift(mm)), + pte_pgprot_mm(mm, pte)); + return pte_advance_pfn(pte, nr); +} + +#define pte_advance_pfn_mm pte_advance_pfn_mm + +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +unsigned int arm64_ptep_page_shift(pte_t *ptep); +#else +static inline unsigned int arm64_ptep_page_shift(pte_t *ptep) +{ + return PAGE_SHIFT; +} +#endif + +static inline unsigned int arm64_cont_ptes(unsigned int shift) +{ + /* Architectural contiguous group lengths for 4K, 16K and 64K. */ + return shift == 12 ? 16 : shift == 14 ? 128 : 32; +} + +static inline unsigned int cont_ptes_mm(const struct mm_struct *mm) +{ + return arm64_cont_ptes(mm_page_shift(mm)); +} + +static inline unsigned long cont_pte_size_mm(const struct mm_struct *mm) +{ + return (unsigned long)cont_ptes_mm(mm) << mm_page_shift(mm); +} + +static inline unsigned long pte_cont_addr_end_mm(const struct mm_struct *mm, + unsigned long addr, unsigned long end) +{ + unsigned long size = cont_pte_size_mm(mm); + unsigned long boundary = (addr + size) & ~(size - 1); + + return boundary - 1 < end - 1 ? boundary : end; +} + +#endif diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 41035916c755d..c581303f8381c 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1698,6 +1698,62 @@ has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope) return cpufeature_matches(val, entry); } +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST +/* Removal-only model for late-CPU verification in a disposable test VM. */ +static int user4k_deny_late_cpu __ro_after_init = -1; + +static int __init user4k_deny_late_cpu_setup(char *str) +{ + int cpu; + + if (kstrtoint(str, 0, &cpu) || cpu <= 0 || cpu >= NR_CPUS) + return -EINVAL; + user4k_deny_late_cpu = cpu; + return 0; +} +early_param("arm64.user4k_deny_late_cpu", user4k_deny_late_cpu_setup); +#ifdef CONFIG_ARM64_64K_PAGES +static int user16k_deny_late_cpu __ro_after_init = -1; + +static int __init user16k_deny_late_cpu_setup(char *str) +{ + int cpu; + + if (kstrtoint(str, 0, &cpu) || cpu <= 0 || cpu >= NR_CPUS) + return -EINVAL; + user16k_deny_late_cpu = cpu; + return 0; +} +early_param("arm64.user16k_deny_late_cpu", user16k_deny_late_cpu_setup); +#endif +#endif + +static bool has_user4k_granule(const struct arm64_cpu_capabilities *entry, + int scope) +{ +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST + if (scope == SCOPE_LOCAL_CPU && system_capabilities_finalized() && + smp_processor_id() == user4k_deny_late_cpu) + return false; +#endif + return has_cpuid_feature(entry, scope); +} + +#ifdef CONFIG_ARM64_64K_PAGES +static bool has_user16k_granule(const struct arm64_cpu_capabilities *entry, + int scope) +{ +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST + if (scope == SCOPE_LOCAL_CPU && system_capabilities_finalized() && + smp_processor_id() == user16k_deny_late_cpu) + return false; +#endif + return has_cpuid_feature(entry, scope); +} +#endif +#endif + const struct cpumask *system_32bit_el0_cpumask(void) { if (!system_supports_32bit_el0()) @@ -2428,8 +2484,11 @@ static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap) * clear the tags or set PG_mte_tagged. */ if (!cleared_zero_page) { + unsigned long offset; + cleared_zero_page = true; - mte_clear_page_tags(lm_alias(empty_zero_page)); + for (offset = 0; offset < sizeof(empty_zero_page); offset += PAGE_SIZE) + mte_clear_page_tags(lm_alias(empty_zero_page + offset)); } kasan_init_hw_tags_cpu(); @@ -2566,6 +2625,28 @@ test_has_gicv5_legacy(const struct arm64_cpu_capabilities *entry, int scope) } static const struct arm64_cpu_capabilities arm64_features[] = { +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + { + .desc = "4K userspace translation granule", + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .capability = ARM64_HAS_USER4K_GRANULE, + .matches = has_user4k_granule, + __ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, TGRAN4, + ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MIN, + ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MAX) + }, +#ifdef CONFIG_ARM64_64K_PAGES + { + .desc = "16K userspace translation granule", + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .capability = ARM64_HAS_USER16K_GRANULE, + .matches = has_user16k_granule, + __ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, TGRAN16, + ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MIN, + ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MAX) + }, +#endif +#endif { .capability = ARM64_ALWAYS_BOOT, .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, diff --git a/arch/arm64/kernel/pi/idreg-override.c b/arch/arm64/kernel/pi/idreg-override.c index d4e0b2c6f1e94..93119c122625c 100644 --- a/arch/arm64/kernel/pi/idreg-override.c +++ b/arch/arm64/kernel/pi/idreg-override.c @@ -38,11 +38,32 @@ struct ftr_set_desc { #define FIELD(n, s, f) { .name = n, .shift = s, .width = 4, .filter = f } +#if defined(CONFIG_ARM64_USER4K_KUNIT_TEST) && !defined(CONFIG_ARM64_4K_PAGES) +/* VM capability-model test: only remove 4K support, never fabricate it. */ +static bool __init user4k_granule_test_filter(u64 val) +{ + return val == ID_AA64MMFR0_EL1_TGRAN4_NI; +} + +#ifdef CONFIG_ARM64_64K_PAGES +static bool __init user16k_granule_test_filter(u64 val) +{ + return val == ID_AA64MMFR0_EL1_TGRAN16_NI; +} +#endif +#endif + static const struct ftr_set_desc mmfr0 __prel64_initconst = { .name = "id_aa64mmfr0", .override = &id_aa64mmfr0_override, .fields = { FIELD("ecv", ID_AA64MMFR0_EL1_ECV_SHIFT, NULL), +#if defined(CONFIG_ARM64_USER4K_KUNIT_TEST) && !defined(CONFIG_ARM64_4K_PAGES) + FIELD("tgran4", ID_AA64MMFR0_EL1_TGRAN4_SHIFT, user4k_granule_test_filter), +#ifdef CONFIG_ARM64_64K_PAGES + FIELD("tgran16", ID_AA64MMFR0_EL1_TGRAN16_SHIFT, user16k_granule_test_filter), +#endif +#endif {} }, }; @@ -59,11 +80,22 @@ static bool __init mmfr1_vh_filter(u64 val) val == 0); } +#if defined(CONFIG_ARM64_USER4K_KUNIT_TEST) && defined(CONFIG_ARM64_SW_TTBR0_PAN) +/* VM-only coverage of deferred TTBR0 restore on CPUs which implement PAN. */ +static bool __init user4k_pan_test_filter(u64 val) +{ + return val == 0; +} +#endif + static const struct ftr_set_desc mmfr1 __prel64_initconst = { .name = "id_aa64mmfr1", .override = &id_aa64mmfr1_override, .fields = { FIELD("vh", ID_AA64MMFR1_EL1_VH_SHIFT, mmfr1_vh_filter), +#if defined(CONFIG_ARM64_USER4K_KUNIT_TEST) && defined(CONFIG_ARM64_SW_TTBR0_PAN) + FIELD("pan", ID_AA64MMFR1_EL1_PAN_SHIFT, user4k_pan_test_filter), +#endif {} }, }; diff --git a/arch/arm64/kernel/pi/map_kernel.c b/arch/arm64/kernel/pi/map_kernel.c index fb44cbdd2f292..d81b3012de836 100644 --- a/arch/arm64/kernel/pi/map_kernel.c +++ b/arch/arm64/kernel/pi/map_kernel.c @@ -243,7 +243,7 @@ asmlinkage void __init early_map_kernel(u64 boot_status, phys_addr_t fdt) static char const chosen_str[] __initconst = "/chosen"; u64 va_base, pa_base = (u64)&_text; u64 kaslr_offset = pa_base % MIN_KIMG_ALIGN; - int root_level = 4 - CONFIG_PGTABLE_LEVELS; + int root_level = 4 - ARM64_NATIVE_PGTABLE_LEVELS; int va_bits = VA_BITS; int chosen; void *fdt_mapped = map_fdt(fdt); diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index ab1beffa8d7a4..a326135b164d4 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -7,11 +7,13 @@ * Copyright (C) 2012 ARM Ltd. */ #include +#include #include #include #include #include #include +#include #include #include #include @@ -884,6 +886,33 @@ unsigned long arch_align_stack(unsigned long sp) return sp & ~0xf; } +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +int arm64_bprm_prepare_mm(struct linux_binprm *bprm, bool compat) +{ + unsigned int requested = current->exec_page_shift ?: + current->default_exec_page_shift; + unsigned int shift = arm64_exec_page_shift(compat, + cpus_have_final_cap(ARM64_HAS_USER4K_GRANULE), PAGE_SHIFT, + requested); + + return bprm_set_page_shift(bprm, shift); +} + +/* Return zero for the native default, or a validated alternative shift. */ +int arch_exec_page_size_shift(unsigned long size) +{ + if (!size || size == PAGE_SIZE) + return 0; + if (size == SZ_4K) + return cpus_have_final_cap(ARM64_HAS_USER4K_GRANULE) ? + 12 : -EOPNOTSUPP; + if (PAGE_SIZE > SZ_16K && size == SZ_16K) + return cpus_have_final_cap(ARM64_HAS_USER16K_GRANULE) ? + 14 : -EOPNOTSUPP; + return -EINVAL; +} +#endif + #ifdef CONFIG_COMPAT int compat_elf_check_arch(const struct elf32_hdr *hdr) { @@ -911,10 +940,10 @@ int compat_elf_check_arch(const struct elf32_hdr *hdr) */ void arch_setup_new_exec(void) { - unsigned long mmflags = 0; + unsigned long mmflags = current->mm->context.flags & MMCF_USER_GRANULE; if (is_compat_task()) { - mmflags = MMCF_AARCH32; + mmflags |= MMCF_AARCH32; /* * Restrict the CPU affinity mask for a 32-bit task so that diff --git a/arch/arm64/kernel/sys.c b/arch/arm64/kernel/sys.c index f08408b6e8263..bdd2531554f27 100644 --- a/arch/arm64/kernel/sys.c +++ b/arch/arm64/kernel/sys.c @@ -22,6 +22,11 @@ SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, unsigned long, prot, unsigned long, flags, unsigned long, fd, unsigned long, off) { +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return ksys_mmap_user(addr, len, prot, flags, fd, off); +#endif + if (offset_in_page(off) != 0) return -EINVAL; diff --git a/arch/arm64/kernel/sys32.c b/arch/arm64/kernel/sys32.c index 12a948f3a5043..ba66b65f506f4 100644 --- a/arch/arm64/kernel/sys32.c +++ b/arch/arm64/kernel/sys32.c @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -51,6 +52,11 @@ COMPAT_SYSCALL_DEFINE6(aarch32_mmap2, unsigned long, addr, unsigned long, len, unsigned long, prot, unsigned long, flags, unsigned long, fd, unsigned long, off_4k) { +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return ksys_mmap_user(addr, len, prot, flags, fd, off_4k << 12); +#endif + if (off_4k & (~PAGE_MASK >> 12)) return -EINVAL; diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index af1d720209764..6302ef42e277e 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -66,9 +66,10 @@ * We require that __hyp_bss_start and __bss_start are aligned, and enforce it * with an assertion. But the BSS_SECTION macro places an empty .sbss section * between them, which can in some cases cause the linker to misalign them. To - * work around the issue, force a page alignment for __bss_start. + * work around the issue, align __bss_start to the BSS section's alignment, + * which can exceed PAGE_SIZE for the mixed-granule zero page. */ -#define SBSS_ALIGN PAGE_SIZE +#define SBSS_ALIGN MAX(PAGE_SIZE, ALIGNOF(.bss)) #else /* CONFIG_KVM */ #define HYPERVISOR_EXTABLE #define HYPERVISOR_RODATA_SECTIONS diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S index 5018ac03b6bf3..c1ce370068cf4 100644 --- a/arch/arm64/lib/mte.S +++ b/arch/arm64/lib/mte.S @@ -80,6 +80,52 @@ SYM_FUNC_START(mte_copy_page_tags) ret SYM_FUNC_END(mte_copy_page_tags) +#ifdef CONFIG_MM_SUBPAGE +/* Copy one aligned power-of-two extent, preserving neighbouring tags. */ +SYM_FUNC_START(mte_copy_subpage_tags_range) + mov x6, x2 + add x2, x0, x2 + multitag_transfer_size x4, x5 + cmp x4, x6 + b.hi 2f +1: ldgm x3, [x1] + stgm x3, [x0] + add x0, x0, x4 + add x1, x1, x4 + cmp x0, x2 + b.ne 1b + ret +2: mov x3, xzr +3: ldg x3, [x1] + stg x3, [x0] + add x0, x0, #MTE_GRANULE_SIZE + add x1, x1, #MTE_GRANULE_SIZE + cmp x0, x2 + b.ne 3b + ret +SYM_FUNC_END(mte_copy_subpage_tags_range) + +/* Clear an aligned extent without changing data or neighbouring tags. */ +SYM_FUNC_START(mte_clear_subpage_tags_range) + mov x4, x1 + add x1, x0, x1 + multitag_transfer_size x2, x3 + cmp x2, x4 + b.hi 2f +1: stgm xzr, [x0] + add x0, x0, x2 + cmp x0, x1 + b.ne 1b + ret +2: mov x2, xzr +3: stg x2, [x0] + add x0, x0, #MTE_GRANULE_SIZE + cmp x0, x1 + b.ne 3b + ret +SYM_FUNC_END(mte_clear_subpage_tags_range) +#endif + /* * Read tags from a user buffer (one tag per byte) and set the corresponding * tags at the given kernel address. Used by PTRACE_POKEMTETAGS. diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile index c26489cf96cd9..32d18488e527c 100644 --- a/arch/arm64/mm/Makefile +++ b/arch/arm64/mm/Makefile @@ -12,6 +12,7 @@ obj-$(CONFIG_TRANS_TABLE) += trans_pgd-asm.o obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o obj-$(CONFIG_ARM64_MTE) += mteswap.o obj-$(CONFIG_ARM64_GCS) += gcs.o +obj-$(CONFIG_ARM64_USER4K_EXPERIMENTAL) += user4k-pgalloc.o KASAN_SANITIZE_physaddr.o += n obj-$(CONFIG_KASAN) += kasan_init.o diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c index 0f4a28b874692..cd3f56b942708 100644 --- a/arch/arm64/mm/context.c +++ b/arch/arm64/mm/context.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -38,6 +39,34 @@ static unsigned long *pinned_asid_map; #define ctxid2asid(asid) ((asid) & ~ASID_MASK) #define asid2ctxid(asid, genid) ((asid) | (genid)) +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST +/* Test-only capacity reduction; hardware ASID width and normal boots are unchanged. */ +static bool user4k_test_small_asids; + +static int __init user4k_test_asid_bits_setup(char *str) +{ + unsigned int bits; + + if (kstrtouint(str, 0, &bits) || bits != 8) + return -EINVAL; + user4k_test_small_asids = true; + return 0; +} +early_param("arm64.user4k_test_asid_bits", user4k_test_asid_bits_setup); + +/* Read-only evidence that an EL0 stress run actually crossed ASID rollover. */ +static int user4k_asid_generation_get(char *buffer, const struct kernel_param *kp) +{ + return sysfs_emit(buffer, "%llu\n", (unsigned long long)atomic64_read(&asid_generation)); +} + +static const struct kernel_param_ops user4k_asid_generation_ops = { + .get = user4k_asid_generation_get, +}; +module_param_cb(user4k_asid_generation, &user4k_asid_generation_ops, NULL, 0444); +MODULE_PARM_DESC(user4k_asid_generation, "Current ASID generation for mixed-granule VM tests"); +#endif + /* Get the ASIDBits supported by the current CPU */ static u32 get_cpu_asid_bits(void) { @@ -268,6 +297,13 @@ void check_and_switch_context(struct mm_struct *mm) */ if (!system_uses_ttbr0_pan()) cpu_switch_mm(mm->pgd, mm); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + else { + /* uaccess/EL0 return restore only TTBRs, so select TG0 here. */ + cpu_set_reserved_ttbr0_nosync(); + cpu_set_mm_tg0(mm); + } +#endif } unsigned long arm64_mm_context_get(struct mm_struct *mm) @@ -365,6 +401,7 @@ void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm) ttbr1 |= FIELD_PREP(TTBRx_EL1_ASID_MASK, asid); cpu_set_reserved_ttbr0_nosync(); + cpu_set_mm_tg0(mm); write_sysreg(ttbr1, ttbr1_el1); write_sysreg(ttbr0, ttbr0_el1); isb(); @@ -401,6 +438,12 @@ arch_initcall(asids_update_limit); static int asids_init(void) { asid_bits = get_cpu_asid_bits(); +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST + if (user4k_test_small_asids) { + pr_info("user4k test: limiting ASID allocator to 8 bits\n"); + asid_bits = 8; + } +#endif atomic64_set(&asid_generation, ASID_FIRST_VERSION); asid_map = bitmap_zalloc(NUM_USER_ASIDS, GFP_KERNEL); if (!asid_map) diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c index 2de12656b4d8b..f8faf1a99eeca 100644 --- a/arch/arm64/mm/contpte.c +++ b/arch/arm64/mm/contpte.c @@ -8,6 +8,25 @@ #include #include +static inline unsigned int contpte_nr(pte_t *ptep) +{ + return arm64_cont_ptes(arm64_ptep_page_shift(ptep)); +} + +static inline phys_addr_t contpte_phys(pte_t pte, unsigned int shift) +{ + if (IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) && shift != PAGE_SHIFT) + return pte_val(pte) & (ARM64_USER4K_TABLE_ADDR_MASK & ~((1ULL << shift) - 1)); + return __pte_to_phys(pte); +} + +static inline pgprot_t contpte_prot(pte_t pte, unsigned int shift) +{ + if (IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) && shift != PAGE_SHIFT) + return __pgprot(pte_val(pte) & ~(ARM64_USER4K_TABLE_ADDR_MASK & ~((1ULL << shift) - 1))); + return pte_pgprot(pte); +} + static inline bool mm_is_user(struct mm_struct *mm) { /* @@ -23,10 +42,10 @@ static inline bool mm_is_user(struct mm_struct *mm) static inline pte_t *contpte_align_down(pte_t *ptep) { - return PTR_ALIGN_DOWN(ptep, sizeof(*ptep) * CONT_PTES); + return PTR_ALIGN_DOWN(ptep, sizeof(*ptep) * contpte_nr(ptep)); } -static inline pte_t *contpte_align_addr_ptep(unsigned long *start, +static inline pte_t *contpte_align_addr_ptep(struct mm_struct *mm, unsigned long *start, unsigned long *end, pte_t *ptep, unsigned int nr) { @@ -36,10 +55,10 @@ static inline pte_t *contpte_align_addr_ptep(unsigned long *start, * single VMA and a single page table. */ if (pte_cont(__ptep_get(ptep + nr - 1))) - *end = ALIGN(*end, CONT_PTE_SIZE); + *end = ALIGN(*end, cont_pte_size_mm(mm)); if (pte_cont(__ptep_get(ptep))) { - *start = ALIGN_DOWN(*start, CONT_PTE_SIZE); + *start = ALIGN_DOWN(*start, cont_pte_size_mm(mm)); ptep = contpte_align_down(ptep); } @@ -54,11 +73,11 @@ static void contpte_try_unfold_partial(struct mm_struct *mm, unsigned long addr, * of the range. */ - if (ptep != contpte_align_down(ptep) || nr < CONT_PTES) + if (ptep != contpte_align_down(ptep) || nr < cont_ptes_mm(mm)) contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); if (ptep + nr != contpte_align_down(ptep + nr)) { - unsigned long last_addr = addr + PAGE_SIZE * (nr - 1); + unsigned long last_addr = addr + mm_page_size(mm) * (nr - 1); pte_t *last_ptep = ptep + nr - 1; contpte_try_unfold(mm, last_addr, last_ptep, @@ -75,10 +94,11 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, int i; start_ptep = ptep = contpte_align_down(ptep); - start_addr = addr = ALIGN_DOWN(addr, CONT_PTE_SIZE); - pte = pfn_pte(ALIGN_DOWN(pte_pfn(pte), CONT_PTES), pte_pgprot(pte)); + start_addr = addr = ALIGN_DOWN(addr, cont_pte_size_mm(mm)); + pte = phys_pte_mm(mm, ALIGN_DOWN(pte_phys_mm(mm, pte), cont_pte_size_mm(mm)), + pte_pgprot_mm(mm, pte)); - for (i = 0; i < CONT_PTES; i++, ptep++, addr += PAGE_SIZE) { + for (i = 0; i < cont_ptes_mm(mm); i++, ptep++, addr += mm_page_size(mm)) { pte_t ptent = __ptep_get_and_clear(mm, addr, ptep); if (pte_dirty(ptent)) @@ -225,10 +245,10 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, */ if (!system_supports_bbml2_noabort()) - __flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, 3, + __flush_tlb_range(&vma, start_addr, addr, mm_page_size(mm), 3, TLBF_NOWALKCACHE); - __set_ptes(mm, start_addr, start_ptep, pte, CONT_PTES); + __set_ptes(mm, start_addr, start_ptep, pte, cont_ptes_mm(mm)); } void __contpte_try_fold(struct mm_struct *mm, unsigned long addr, @@ -269,24 +289,26 @@ void __contpte_try_fold(struct mm_struct *mm, unsigned long addr, page = pte_page(pte); folio = page_folio(page); folio_start = addr - (page - &folio->page) * PAGE_SIZE; + if (arm64_mm_alt_granule(mm)) + folio_start -= pte_phys_mm(mm, pte) & ~PAGE_MASK; folio_end = folio_start + folio_nr_pages(folio) * PAGE_SIZE; - cont_start = ALIGN_DOWN(addr, CONT_PTE_SIZE); - cont_end = cont_start + CONT_PTE_SIZE; + cont_start = ALIGN_DOWN(addr, cont_pte_size_mm(mm)); + cont_end = cont_start + cont_pte_size_mm(mm); if (folio_start > cont_start || folio_end < cont_end) return; - pfn = ALIGN_DOWN(pte_pfn(pte), CONT_PTES); - prot = pte_pgprot(pte_mkold(pte_mkclean(pte))); - expected_pte = pfn_pte(pfn, prot); + pfn = ALIGN_DOWN(pte_phys_mm(mm, pte) >> mm_page_shift(mm), cont_ptes_mm(mm)); + prot = pte_pgprot_mm(mm, pte_mkold(pte_mkclean(pte))); + expected_pte = phys_pte_mm(mm, pfn << mm_page_shift(mm), prot); orig_ptep = ptep; ptep = contpte_align_down(ptep); - for (i = 0; i < CONT_PTES; i++) { + for (i = 0; i < cont_ptes_mm(mm); i++) { subpte = pte_mkold(pte_mkclean(__ptep_get(ptep))); if (!pte_same(subpte, expected_pte)) return; - expected_pte = pte_advance_pfn(expected_pte, 1); + expected_pte = pte_advance_pfn_mm(mm, expected_pte, 1); ptep++; } @@ -319,17 +341,18 @@ pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte) * feet. */ + unsigned int nr = contpte_nr(ptep); pte_t pte; int i; ptep = contpte_align_down(ptep); - for (i = 0; i < CONT_PTES; i++, ptep++) { + for (i = 0; i < nr; i++, ptep++) { pte = __ptep_get(ptep); if (pte_dirty(pte)) { orig_pte = pte_mkdirty(orig_pte); - for (; i < CONT_PTES; i++, ptep++) { + for (; i < nr; i++, ptep++) { pte = __ptep_get(ptep); if (pte_young(pte)) { orig_pte = pte_mkyoung(orig_pte); @@ -343,7 +366,7 @@ pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte) orig_pte = pte_mkyoung(orig_pte); i++; ptep++; - for (; i < CONT_PTES; i++, ptep++) { + for (; i < nr; i++, ptep++) { pte = __ptep_get(ptep); if (pte_dirty(pte)) { orig_pte = pte_mkdirty(orig_pte); @@ -359,11 +382,12 @@ pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte) EXPORT_SYMBOL_GPL(contpte_ptep_get); static inline bool contpte_is_consistent(pte_t pte, unsigned long pfn, + unsigned int shift, pgprot_t orig_prot) { - pgprot_t prot = pte_pgprot(pte_mkold(pte_mkclean(pte))); + pgprot_t prot = contpte_prot(pte_mkold(pte_mkclean(pte)), shift); - return pte_valid_cont(pte) && pte_pfn(pte) == pfn && + return pte_valid_cont(pte) && (contpte_phys(pte, shift) >> shift) == pfn && pgprot_val(prot) == pgprot_val(orig_prot); } @@ -386,6 +410,8 @@ pte_t contpte_ptep_get_lockless(pte_t *orig_ptep) * because it is not part of a contpte range. */ + unsigned int shift = arm64_ptep_page_shift(orig_ptep); + unsigned int nr = arm64_cont_ptes(shift); pgprot_t orig_prot; unsigned long pfn; pte_t orig_pte; @@ -399,22 +425,22 @@ pte_t contpte_ptep_get_lockless(pte_t *orig_ptep) if (!pte_valid_cont(orig_pte)) return orig_pte; - orig_prot = pte_pgprot(pte_mkold(pte_mkclean(orig_pte))); + orig_prot = contpte_prot(pte_mkold(pte_mkclean(orig_pte)), shift); ptep = contpte_align_down(orig_ptep); - pfn = pte_pfn(orig_pte) - (orig_ptep - ptep); + pfn = (contpte_phys(orig_pte, shift) >> shift) - (orig_ptep - ptep); - for (i = 0; i < CONT_PTES; i++, ptep++, pfn++) { + for (i = 0; i < nr; i++, ptep++, pfn++) { pte = __ptep_get(ptep); - if (!contpte_is_consistent(pte, pfn, orig_prot)) + if (!contpte_is_consistent(pte, pfn, shift, orig_prot)) goto retry; if (pte_dirty(pte)) { orig_pte = pte_mkdirty(orig_pte); - for (; i < CONT_PTES; i++, ptep++, pfn++) { + for (; i < nr; i++, ptep++, pfn++) { pte = __ptep_get(ptep); - if (!contpte_is_consistent(pte, pfn, orig_prot)) + if (!contpte_is_consistent(pte, pfn, shift, orig_prot)) goto retry; if (pte_young(pte)) { @@ -430,10 +456,10 @@ pte_t contpte_ptep_get_lockless(pte_t *orig_ptep) i++; ptep++; pfn++; - for (; i < CONT_PTES; i++, ptep++, pfn++) { + for (; i < nr; i++, ptep++, pfn++) { pte = __ptep_get(ptep); - if (!contpte_is_consistent(pte, pfn, orig_prot)) + if (!contpte_is_consistent(pte, pfn, shift, orig_prot)) goto retry; if (pte_dirty(pte)) { @@ -468,16 +494,16 @@ void contpte_set_ptes(struct mm_struct *mm, unsigned long addr, if (!mm_is_user(mm)) return __set_ptes(mm, addr, ptep, pte, nr); - end = addr + (nr << PAGE_SHIFT); - pfn = pte_pfn(pte); - prot = pte_pgprot(pte); + end = addr + (nr << mm_page_shift(mm)); + pfn = pte_phys_mm(mm, pte) >> mm_page_shift(mm); + prot = pte_pgprot_mm(mm, pte); do { - next = pte_cont_addr_end(addr, end); - nr = (next - addr) >> PAGE_SHIFT; - pte = pfn_pte(pfn, prot); + next = pte_cont_addr_end_mm(mm, addr, end); + nr = (next - addr) >> mm_page_shift(mm); + pte = phys_pte_mm(mm, pfn << mm_page_shift(mm), prot); - if (((addr | next | (pfn << PAGE_SHIFT)) & ~CONT_PTE_MASK) == 0) + if (((addr | next | (pfn << mm_page_shift(mm))) & (cont_pte_size_mm(mm) - 1)) == 0) pte = pte_mkcont(pte); else pte = pte_mknoncont(pte); @@ -512,6 +538,8 @@ EXPORT_SYMBOL_GPL(contpte_get_and_clear_full_ptes); bool contpte_test_and_clear_young_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr) { + struct mm_struct *mm = vma->vm_mm; + /* * ptep_clear_flush_young() technically requires us to clear the access * flag for a _single_ pte. However, the core-mm code actually tracks @@ -524,11 +552,11 @@ bool contpte_test_and_clear_young_ptes(struct vm_area_struct *vma, * of the same large folio in a single VMA and a single page table. */ - unsigned long end = addr + nr * PAGE_SIZE; + unsigned long end = addr + nr * mm_page_size(mm); bool young = false; - ptep = contpte_align_addr_ptep(&addr, &end, ptep, nr); - for (; addr != end; ptep++, addr += PAGE_SIZE) + ptep = contpte_align_addr_ptep(vma->vm_mm, &addr, &end, ptep, nr); + for (; addr != end; ptep++, addr += mm_page_size(mm)) young |= __ptep_test_and_clear_young(vma, addr, ptep); return young; @@ -538,19 +566,21 @@ EXPORT_SYMBOL_GPL(contpte_test_and_clear_young_ptes); bool contpte_clear_flush_young_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr) { + struct mm_struct *mm = vma->vm_mm; + bool young; young = contpte_test_and_clear_young_ptes(vma, addr, ptep, nr); if (young) { - unsigned long end = addr + nr * PAGE_SIZE; + unsigned long end = addr + nr * mm_page_size(mm); - contpte_align_addr_ptep(&addr, &end, ptep, nr); + contpte_align_addr_ptep(vma->vm_mm, &addr, &end, ptep, nr); /* * See comment in __ptep_clear_flush_young(); same rationale for * eliding the trailing DSB applies here. */ - __flush_tlb_range(vma, addr, end, PAGE_SIZE, 3, + __flush_tlb_range(vma, addr, end, mm_page_size(mm), 3, TLBF_NOWALKCACHE | TLBF_NOSYNC); } @@ -580,6 +610,8 @@ void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr, cydp_t flags) { + struct mm_struct *mm = vma->vm_mm; + /* * We can safely clear access/dirty without needing to unfold from * the architectures perspective, even when contpte is set. If the @@ -591,14 +623,15 @@ void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, * clearing access/dirty for the whole block. */ unsigned long start = addr; - unsigned long end = start + nr * PAGE_SIZE; + unsigned long end = start + nr * mm_page_size(mm); - ptep = contpte_align_addr_ptep(&start, &end, ptep, nr); - __clear_young_dirty_ptes(vma, start, ptep, (end - start) / PAGE_SIZE, flags); + ptep = contpte_align_addr_ptep(vma->vm_mm, &start, &end, ptep, nr); + __clear_young_dirty_ptes(vma, start, ptep, (end - start) >> mm_page_shift(mm), flags); } EXPORT_SYMBOL_GPL(contpte_clear_young_dirty_ptes); -static bool contpte_all_subptes_match_access_flags(pte_t *ptep, pte_t entry) +static bool contpte_all_subptes_match_access_flags(struct mm_struct *mm, + pte_t *ptep, pte_t entry) { pte_t *cont_ptep = contpte_align_down(ptep); /* @@ -609,7 +642,7 @@ static bool contpte_all_subptes_match_access_flags(pte_t *ptep, pte_t entry) pteval_t entry_cmp = pte_val(entry) & cmp_mask; int i; - for (i = 0; i < CONT_PTES; i++) { + for (i = 0; i < cont_ptes_mm(mm); i++) { pteval_t pte_cmp = pte_val(__ptep_get(cont_ptep + i)) & cmp_mask; if (pte_cmp != entry_cmp) @@ -623,6 +656,8 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t entry, int dirty) { + struct mm_struct *mm = vma->vm_mm; + unsigned long start_addr; pte_t orig_pte; int i; @@ -651,7 +686,7 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, * no-op, and when any sub-PTE mismatches, proceed to update the whole * range. */ - if (contpte_all_subptes_match_access_flags(ptep, entry)) + if (contpte_all_subptes_match_access_flags(mm, ptep, entry)) return 0; /* @@ -672,7 +707,7 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, * and instead flush the whole range at the end. */ ptep = contpte_align_down(ptep); - start_addr = addr = ALIGN_DOWN(addr, CONT_PTE_SIZE); + start_addr = addr = ALIGN_DOWN(addr, cont_pte_size_mm(mm)); /* * We are not advancing entry because __ptep_set_access_flags() @@ -680,13 +715,13 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, * for the whole contpte block and returned early, pte_same() * within __ptep_set_access_flags() is likely false. */ - for (i = 0; i < CONT_PTES; i++, ptep++, addr += PAGE_SIZE) + for (i = 0; i < cont_ptes_mm(mm); i++, ptep++, addr += mm_page_size(mm)) __ptep_set_access_flags(vma, addr, ptep, entry, 0); if (dirty) __flush_tlb_range(vma, start_addr, - start_addr + CONT_PTE_SIZE, - PAGE_SIZE, 3, + start_addr + cont_pte_size_mm(mm), + mm_page_size(mm), 3, TLBF_NOWALKCACHE | TLBF_NOBROADCAST); } else { __contpte_try_unfold(vma->vm_mm, addr, ptep, orig_pte); diff --git a/arch/arm64/mm/copypage.c b/arch/arm64/mm/copypage.c index cd5912ba617b7..b81ccf42b22d1 100644 --- a/arch/arm64/mm/copypage.c +++ b/arch/arm64/mm/copypage.c @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -72,3 +73,54 @@ void copy_user_highpage(struct page *to, struct page *from, flush_dcache_page(to); } EXPORT_SYMBOL_GPL(copy_user_highpage); + +#ifdef CONFIG_MM_SUBPAGE +void clear_user_subpage_range(struct page *page, unsigned int offset, unsigned int size) +{ + void *addr = page_address(page) + offset; + + VM_BUG_ON(!is_power_of_2(size) || size < SZ_4K || size > PAGE_SIZE); + VM_BUG_ON(!IS_ALIGNED(offset, size) || offset > PAGE_SIZE - size); + memset(addr, 0, size); + if (system_supports_mte() && page_mte_tagged(page)) + mte_clear_subpage_tags_range(addr, size); + flush_dcache_page(page); +} + +int copy_user_subpage_range(struct page *to, unsigned int to_offset, + struct page *from, unsigned int from_offset, unsigned int size) +{ + void *kto = page_address(to), *kfrom = page_address(from); + + VM_BUG_ON(!is_power_of_2(size) || size < SZ_4K || size > PAGE_SIZE); + VM_BUG_ON(!IS_ALIGNED(to_offset, size) || to_offset > PAGE_SIZE - size); + VM_BUG_ON(!IS_ALIGNED(from_offset, size) || from_offset > PAGE_SIZE - size); + if (PageHWPoison(from) || copy_mc_to_kernel(kto + to_offset, kfrom + from_offset, size)) + return -EHWPOISON; + if (kasan_hw_tags_enabled()) + page_kasan_tag_reset(to); + if (system_supports_mte()) { + if (page_mte_tagged(from)) { + if (try_page_mte_tagging(to)) + mte_clear_page_tags(kto); + mte_copy_subpage_tags_range(kto + to_offset, kfrom + from_offset, size); + set_page_mte_tagged(to); + } else if (page_mte_tagged(to)) { + mte_clear_subpage_tags_range(kto + to_offset, size); + } + } + flush_dcache_page(to); + return 0; +} +/* Existing 4K fault and ownership callers keep their fixed-size contract. */ +void clear_user_subpage(struct page *page, unsigned int offset) +{ + clear_user_subpage_range(page, offset, SZ_4K); +} + +int copy_user_subpage(struct page *to, unsigned int to_offset, + struct page *from, unsigned int from_offset) +{ + return copy_user_subpage_range(to, to_offset, from, from_offset, SZ_4K); +} +#endif diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 75c3e463df2ef..2ea95d7490dca 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -153,7 +153,7 @@ static void show_pte(unsigned long addr) } pr_alert("%s pgtable: %luk pages, %llu-bit VAs, pgdp=%016lx\n", - mm == &init_mm ? "swapper" : "user", PAGE_SIZE / SZ_1K, + mm == &init_mm ? "swapper" : "user", mm_page_size(mm) / SZ_1K, vabits_actual, mm_to_pgd_phys(mm)); guard(irqsave)(); @@ -171,25 +171,29 @@ static void show_pte(unsigned long addr) if (pgd_none(pgd) || pgd_bad(pgd)) break; - p4dp = p4d_offset_lockless(pgdp, pgd, addr); + p4dp = arm64_mm_alt_granule(mm) ? p4d_offset_mm(mm, pgdp, addr) : + p4d_offset_lockless(pgdp, pgd, addr); p4d = READ_ONCE(*p4dp); pr_cont(", p4d=%016llx", p4d_val(p4d)); - if (p4d_none(p4d) || p4d_bad(p4d)) + if (p4d_none_mm(mm, p4d) || p4d_bad_mm(mm, p4d)) break; - pudp = pud_offset_lockless(p4dp, p4d, addr); + pudp = arm64_mm_alt_granule(mm) ? pud_offset_mm(mm, p4dp, addr) : + pud_offset_lockless(p4dp, p4d, addr); pud = READ_ONCE(*pudp); pr_cont(", pud=%016llx", pud_val(pud)); if (pud_none(pud) || pud_bad(pud)) break; - pmdp = pmd_offset_lockless(pudp, pud, addr); + pmdp = arm64_mm_alt_granule(mm) ? pmd_offset_mm(mm, pudp, addr) : + pmd_offset_lockless(pudp, pud, addr); pmd = READ_ONCE(*pmdp); pr_cont(", pmd=%016llx", pmd_val(pmd)); if (pmd_none(pmd) || pmd_bad(pmd)) break; - ptep = pte_offset_map(&pmd, addr); + ptep = arm64_mm_alt_granule(mm) ? pte_offset_map_mm(mm, pmdp, addr) : + pte_offset_map(&pmd, addr); if (!ptep) break; @@ -247,19 +251,15 @@ int __ptep_set_access_flags_anysz(struct vm_area_struct *vma, * flush_tlb_fix_spurious_fault(). */ if (dirty) { - switch (pgsize) { - case PAGE_SIZE: + if (pgsize == mm_page_size(vma->vm_mm)) { level = 3; - break; - case PMD_SIZE: + } else if (pgsize == pmd_size_mm(vma->vm_mm)) { level = 2; - break; #ifndef __PAGETABLE_PMD_FOLDED - case PUD_SIZE: + } else if (pgsize == pud_size_mm(vma->vm_mm)) { level = 1; - break; #endif - default: + } else { level = TLBI_TTL_UNKNOWN; WARN_ON(1); } @@ -1011,8 +1011,8 @@ NOKPROBE_SYMBOL(do_sp_pc_abort); /* * Used during anonymous page fault handling. */ -struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, - unsigned long vaddr) +struct folio *vma_alloc_zeroed_movable_folio_order(struct vm_area_struct *vma, + unsigned long vaddr, unsigned int order) { gfp_t flags = GFP_HIGHUSER_MOVABLE | __GFP_ZERO; @@ -1024,7 +1024,7 @@ struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, if (vma->vm_flags & VM_MTE) flags |= __GFP_ZEROTAGS; - return vma_alloc_folio(flags, 0, vma, vaddr); + return vma_alloc_folio(flags, order, vma, vaddr); } bool tag_clear_highpages(struct page *page, int numpages, bool clear_pages) diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c index f66a0016dd02d..1ab82dc11b2b1 100644 --- a/arch/arm64/mm/fixmap.c +++ b/arch/arm64/mm/fixmap.c @@ -77,7 +77,7 @@ static void __init early_fixmap_init_pud(p4d_t *p4dp, unsigned long addr, p4d_t p4d = READ_ONCE(*p4dp); pud_t *pudp; - if (CONFIG_PGTABLE_LEVELS > 3 && !p4d_none(p4d) && + if (pgtable_l4_enabled() && !p4d_none(p4d) && p4d_page_paddr(p4d) != __pa_symbol(bm_pud)) { /* * We only end up here if the kernel mapping and the fixmap diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 97987f850a33c..1e5f314b542d5 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -368,8 +368,9 @@ void __init arch_mm_preinit(void) * Selected page table levels should match when derived from * scratch using the virtual address range and page size. */ - BUILD_BUG_ON(ARM64_HW_PGTABLE_LEVELS(CONFIG_ARM64_VA_BITS) != - CONFIG_PGTABLE_LEVELS); + BUILD_BUG_ON(CONFIG_PGTABLE_LEVELS != + (IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) ? 4 : + ARM64_HW_PGTABLE_LEVELS(CONFIG_ARM64_VA_BITS))); if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) { extern int sysctl_overcommit_memory; diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 1a5cf32a71fce..13507ecf4baca 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -2245,8 +2245,8 @@ pte_t modify_prot_start_ptes(struct vm_area_struct *vma, unsigned long addr, * in cases where cpu is affected with errata #2645198. */ if (pte_accessible(vma->vm_mm, pte) && pte_user_exec(pte)) - __flush_tlb_range(vma, addr, addr + nr * PAGE_SIZE, - PAGE_SIZE, 3, TLBF_NOWALKCACHE); + __flush_tlb_range(vma, addr, addr + nr * mm_page_size(vma->vm_mm), + mm_page_size(vma->vm_mm), 3, TLBF_NOWALKCACHE); } return pte; diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c index bf5110b91e2fa..3b361b95322e1 100644 --- a/arch/arm64/mm/pgd.c +++ b/arch/arm64/mm/pgd.c @@ -16,14 +16,19 @@ #include static struct kmem_cache *pgd_cache __ro_after_init; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +static struct kmem_cache *user4k_pgd_cache __ro_after_init; +static struct kmem_cache *user16k_pgd_cache __ro_after_init; +static struct kmem_cache *user64k_pgd_cache __ro_after_init; +#endif static bool pgdir_is_page_size(void) { if (PGD_SIZE == PAGE_SIZE) return true; - if (CONFIG_PGTABLE_LEVELS == 4) + if (ARM64_NATIVE_PGTABLE_LEVELS == 4) return !pgtable_l4_enabled(); - if (CONFIG_PGTABLE_LEVELS == 5) + if (ARM64_NATIVE_PGTABLE_LEVELS == 5) return !pgtable_l5_enabled(); return false; } @@ -32,6 +37,14 @@ pgd_t *pgd_alloc(struct mm_struct *mm) { gfp_t gfp = GFP_PGTABLE_USER; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_context_user4k(&mm->context)) + return kmem_cache_alloc(user4k_pgd_cache, gfp); + if (arm64_mm_alt_granule(mm) && mm_page_shift(mm) == 14) + return kmem_cache_alloc(user16k_pgd_cache, gfp); + if (arm64_mm_alt_granule(mm) && mm_page_shift(mm) == 16) + return kmem_cache_alloc(user64k_pgd_cache, gfp); +#endif if (pgdir_is_page_size()) return __pgd_alloc(mm, 0); else @@ -40,6 +53,20 @@ pgd_t *pgd_alloc(struct mm_struct *mm) void pgd_free(struct mm_struct *mm, pgd_t *pgd) { +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_context_user4k(&mm->context)) { + kmem_cache_free(user4k_pgd_cache, pgd); + return; + } + if (arm64_mm_alt_granule(mm) && mm_page_shift(mm) == 14) { + kmem_cache_free(user16k_pgd_cache, pgd); + return; + } + if (arm64_mm_alt_granule(mm) && mm_page_shift(mm) == 16) { + kmem_cache_free(user64k_pgd_cache, pgd); + return; + } +#endif if (pgdir_is_page_size()) __pgd_free(mm, pgd); else @@ -48,6 +75,16 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd) void __init pgtable_cache_init(void) { +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (PAGE_SHIFT < 16) + user64k_pgd_cache = kmem_cache_create("user64k_pgd_cache", SZ_64K, SZ_64K, + SLAB_PANIC, NULL); + if (PAGE_SHIFT != 14) + user16k_pgd_cache = kmem_cache_create("user16k_pgd_cache", SZ_16K, SZ_16K, + SLAB_PANIC, NULL); + user4k_pgd_cache = kmem_cache_create("user4k_pgd_cache", SZ_4K, SZ_4K, + SLAB_PANIC, NULL); +#endif if (pgdir_is_page_size()) return; diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index 22866b49be372..7f947589a3b4d 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -291,7 +291,7 @@ SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings) cbnz cpu, __idmap_kpti_secondary -#if CONFIG_PGTABLE_LEVELS > 4 +#if ARM64_NATIVE_PGTABLE_LEVELS > 4 stp x29, x30, [sp, #-32]! mov x29, sp stp x19, x20, [sp, #16] @@ -352,7 +352,7 @@ alternative_else_nop_endif /* Set the flag to zero to indicate that we're all done */ str wzr, [flag_ptr] -#if CONFIG_PGTABLE_LEVELS > 4 +#if ARM64_NATIVE_PGTABLE_LEVELS > 4 ldp x19, x20, [sp, #16] ldp x29, x30, [sp], #32 #endif @@ -360,39 +360,39 @@ alternative_else_nop_endif .Lderef_pgd: /* P4D */ - .if CONFIG_PGTABLE_LEVELS > 4 + .if ARM64_NATIVE_PGTABLE_LEVELS > 4 p4d .req x30 pte_to_phys cur_p4dp, pgd kpti_map_pgtbl p4d, 0 kpti_mk_tbl_ng p4d, PTRS_PER_P4D b .Lnext_pgd - .else /* CONFIG_PGTABLE_LEVELS <= 4 */ + .else /* ARM64_NATIVE_PGTABLE_LEVELS <= 4 */ p4d .req pgd .set .Lnext_p4d, .Lnext_pgd .endif .Lderef_p4d: /* PUD */ - .if CONFIG_PGTABLE_LEVELS > 3 + .if ARM64_NATIVE_PGTABLE_LEVELS > 3 pud .req x10 pte_to_phys cur_pudp, p4d kpti_map_pgtbl pud, 1 kpti_mk_tbl_ng pud, PTRS_PER_PUD b .Lnext_p4d - .else /* CONFIG_PGTABLE_LEVELS <= 3 */ + .else /* ARM64_NATIVE_PGTABLE_LEVELS <= 3 */ pud .req pgd .set .Lnext_pud, .Lnext_pgd .endif .Lderef_pud: /* PMD */ - .if CONFIG_PGTABLE_LEVELS > 2 + .if ARM64_NATIVE_PGTABLE_LEVELS > 2 pmd .req x13 pte_to_phys cur_pmdp, pud kpti_map_pgtbl pmd, 2 kpti_mk_tbl_ng pmd, PTRS_PER_PMD b .Lnext_pud - .else /* CONFIG_PGTABLE_LEVELS <= 2 */ + .else /* ARM64_NATIVE_PGTABLE_LEVELS <= 2 */ pmd .req pgd .set .Lnext_pmd, .Lnext_pgd .endif diff --git a/arch/arm64/mm/user4k-pgalloc.c b/arch/arm64/mm/user4k-pgalloc.c new file mode 100644 index 0000000000000..0bcaab5c3a8d2 --- /dev/null +++ b/arch/arm64/mm/user4k-pgalloc.c @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Hardware tables have immutable granule/owner metadata. Smaller tables share + * a native page; larger tables own a compound allocation. Only unissued slots + * in a shared native page are cached by the mm. + */ +#include +#include +#include +#include +#include + +unsigned int arm64_ptep_page_shift(pte_t *ptep) +{ + struct ptdesc *pt = virt_to_ptdesc(ptep); + + /* Table owners retain geometry through their last RCU reader. */ + return test_bit(PT_fragmented, &pt->pt_flags.f) ? pt->pt_frag_shift : PAGE_SHIFT; +} +EXPORT_SYMBOL_GPL(arm64_ptep_page_shift); + +static void fragment_owner_free(struct ptdesc *pt) +{ + clear_bit(PT_fragmented, &pt->pt_flags.f); + pagetable_dtor_free(pt); +} + +static void fragment_owner_free_rcu(struct rcu_head *rcu) +{ + fragment_owner_free(container_of(rcu, struct ptdesc, pt_rcu_head)); +} + +static void fragment_put(struct ptdesc *pt, unsigned int nr) +{ + if (!atomic_sub_and_test(nr, &pt->pt_frag_refcount)) + return; + if (test_and_clear_bit(PT_frag_defer, &pt->pt_flags.f)) + call_rcu(&pt->pt_rcu_head, fragment_owner_free_rcu); + else + fragment_owner_free(pt); +} + +void *arm64_user_pt_alloc_granule(struct mm_struct *mm, + enum arm64_user4k_pt_level level, unsigned int shift) +{ + struct ptdesc *pt; + void *table, *next; + bool constructed; + unsigned int frag_size, nr_fragments, order; + + if (shift != 12 && shift != 14 && shift != 16) + return NULL; + frag_size = 1U << shift; + order = shift > PAGE_SHIFT ? shift - PAGE_SHIFT : 0; + nr_fragments = (PAGE_SIZE << order) >> shift; + if (WARN_ON_ONCE((unsigned int)level >= ARRAY_SIZE(mm->context.user4k_pt_frag))) + return NULL; + + spin_lock(&mm->page_table_lock); + table = mm->context.user4k_pt_frag[level]; + if (table) { + /* An mm's geometry is immutable; never consume another granule. */ + if (virt_to_ptdesc(table)->pt_frag_shift != shift) { + spin_unlock(&mm->page_table_lock); + return NULL; + } + next = table + frag_size; + mm->context.user4k_pt_frag[level] = + ((unsigned long)next & ~PAGE_MASK) ? next : NULL; + } + spin_unlock(&mm->page_table_lock); + if (table) + return table; + + pt = pagetable_alloc(GFP_PGTABLE_USER, order); + if (!pt) + return NULL; + switch (level) { + case USER4K_PT_PTE: + constructed = pagetable_pte_ctor(mm, pt); + break; + case USER4K_PT_PMD: + constructed = pagetable_pmd_ctor(mm, pt); + break; + case USER4K_PT_PUD: + pagetable_pud_ctor(pt); + constructed = true; + break; + default: + constructed = false; + } + if (!constructed) { + pagetable_free(pt); + return NULL; + } + pt->pt_frag_shift = shift; + set_bit(PT_fragmented, &pt->pt_flags.f); + atomic_set(&pt->pt_frag_refcount, 1); + table = ptdesc_address(pt); + spin_lock(&mm->page_table_lock); + if (nr_fragments > 1 && !mm->context.user4k_pt_frag[level]) { + atomic_set(&pt->pt_frag_refcount, nr_fragments); + mm->context.user4k_pt_frag[level] = table + frag_size; + } + /* A racing allocation may own the cache; this page then has one user. */ + spin_unlock(&mm->page_table_lock); + return table; +} + +void *arm64_user4k_pt_alloc(struct mm_struct *mm, enum arm64_user4k_pt_level level) +{ + return arm64_user_pt_alloc_granule(mm, level, 12); +} + +void arm64_user4k_pt_free(void *table) +{ + struct ptdesc *pt = virt_to_ptdesc(table); + + VM_BUG_ON(!test_bit(PT_fragmented, &pt->pt_flags.f)); + VM_BUG_ON(!IS_ALIGNED((unsigned long)table, 1UL << pt->pt_frag_shift)); + VM_BUG_ON(atomic_read(&pt->pt_frag_refcount) <= 0); + fragment_put(pt, 1); +} + +void arm64_pgtable_free(void *table) +{ + struct ptdesc *pt = virt_to_ptdesc(table); + + if (test_bit(PT_fragmented, &pt->pt_flags.f)) + arm64_user4k_pt_free(table); + else { + VM_BUG_ON(!PAGE_ALIGNED(table)); + pagetable_dtor_free(pt); + } +} + +void arm64_user4k_pt_cache_destroy(struct mm_struct *mm) +{ + unsigned int level; + + /* The mm is private or dying; no allocator may still access its cache. */ + for (level = 0; level < ARRAY_SIZE(mm->context.user4k_pt_frag); level++) { + void *next = mm->context.user4k_pt_frag[level]; + struct ptdesc *pt; + unsigned int issued, shift; + + if (!next) + continue; + mm->context.user4k_pt_frag[level] = NULL; + pt = virt_to_ptdesc(next); + shift = pt->pt_frag_shift; + issued = ((unsigned long)next & ~PAGE_MASK) >> shift; + fragment_put(pt, (PAGE_SIZE >> shift) - issued); + } +} + +void arm64_pte_free_defer(struct mm_struct *mm, pgtable_t table) +{ + struct ptdesc *pt = virt_to_ptdesc(table); + + if (test_bit(PT_fragmented, &pt->pt_flags.f)) { + /* No freed slot is reused. Delay the final owner release for RCU. */ + set_bit(PT_frag_defer, &pt->pt_flags.f); + arm64_user4k_pt_free(table); + } else { + call_rcu(&pt->pt_rcu_head, fragment_owner_free_rcu); + } +} + +void __tlb_remove_table(void *token) +{ + if ((unsigned long)token & 1) + arm64_user4k_pt_free((void *)((unsigned long)token & ~1UL)); + else + pagetable_dtor_free(token); +} diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index c844edd285607..f32f9e9e2bf41 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -69,6 +69,8 @@ HAS_TIDCP1 HAS_TLB_RANGE HAS_TSO_APPLE HAS_TSO_FIXED +HAS_USER4K_GRANULE +HAS_USER16K_GRANULE HAS_VA52 HAS_VIRT_HOST_EXTN HAS_WFXT From bb23ad565c819270b643ed9d9bdfbf502a72bb95 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 12 Sep 2026 13:59:39 -0400 Subject: [PATCH 0688/2680] soc: apple: use shared xART gigalocker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Serve SEP anti-replay records from the existing device-wide xART extent instead of a Linux-private seeded store. Validate the entire slot grid and both root records before SEP registration, preserve the store's copy-on-write ordering, and require both an explicit module parameter and a writable block mapping before any write. Keep Linux-only biometric metadata in a separate namespaced host store. [ryan: adapted to 7.2.10 — Key::new(), Intent::Delete and Store::delete() have no user until the next commit, and the J700 config builds Rust with -Dwarnings (CONFIG_WERROR), so they carry #[allow(dead_code)] here for bisectability; the next commit removes the allowances] Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 5 + drivers/soc/apple/keybag.rs | 2 +- drivers/soc/apple/refkey.rs | 34 ++- drivers/soc/apple/sbio.rs | 10 +- drivers/soc/apple/seed.rs | 113 --------- drivers/soc/apple/sep.rs | 84 ++++--- drivers/soc/apple/shim.h | 1 + drivers/soc/apple/shim.rs | 49 ++++ drivers/soc/apple/store.rs | 38 ++- drivers/soc/apple/store_shim.c | 19 ++ drivers/soc/apple/xarm.rs | 11 +- drivers/soc/apple/xart_store.rs | 402 ++++++++++++++++++++++++++++++++ 12 files changed, 575 insertions(+), 193 deletions(-) delete mode 100644 drivers/soc/apple/seed.rs create mode 100644 drivers/soc/apple/xart_store.rs diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 0386028537565..edd3b87de65c6 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -127,6 +127,11 @@ config APPLE_SEP device (enrol and match), exposes the SEP hardware RNG, and registers a SEP-backed trusted key source so keyctl can seal keys to the enclave. + The driver requires the existing machine-wide xART gigalocker at + /dev/mapper/sep-xart-gigalocker. It validates the complete store and both + root records before registering with SEP. Writes remain disabled unless + the xart_writes module parameter is set and the mapping is writable. + Say Y here if you have an Apple silicon Mac. config APPLE_PMP diff --git a/drivers/soc/apple/keybag.rs b/drivers/soc/apple/keybag.rs index 133a1ad5769db..90bed45619c50 100644 --- a/drivers/soc/apple/keybag.rs +++ b/drivers/soc/apple/keybag.rs @@ -11,7 +11,7 @@ use crate::shim; use crate::store::crc16_ccitt_false; use kernel::prelude::*; -pub(crate) const KEYBAG_PATH: &CStr = c"/var/lib/apple-sep-keybag.bin"; +pub(crate) const KEYBAG_PATH: &CStr = c"/var/lib/aurora-sep-keybag.bin"; #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum Slot { diff --git a/drivers/soc/apple/refkey.rs b/drivers/soc/apple/refkey.rs index c0456a604b38d..d2ffe586594ea 100644 --- a/drivers/soc/apple/refkey.rs +++ b/drivers/soc/apple/refkey.rs @@ -22,7 +22,8 @@ impl SepData { body.put_blob(der_set)?; let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; - let msg = crate::sks::encode_sks_perform_operation(self.sks_next_seq(), len).ok_or(EINVAL)?; + let msg = + crate::sks::encode_sks_perform_operation(self.sks_next_seq(), len).ok_or(EINVAL)?; Ok(SksRequest { name: crate::sks::SKS_PERFORM_OP_NAME, msg, @@ -54,8 +55,12 @@ impl SepData { let create_der = { let mut items: KVec<(&[u8], RV<'_>)> = KVec::new(); if items.push((b"o", RV::Utf8(b"oc")), GFP_KERNEL).is_err() - || items.push((b"bc", RV::Integer(PROTECTION_CLASS)), GFP_KERNEL).is_err() - || items.push((b"kt", RV::Integer(KEY_TYPE)), GFP_KERNEL).is_err() + || items + .push((b"bc", RV::Integer(PROTECTION_CLASS)), GFP_KERNEL) + .is_err() + || items + .push((b"kt", RV::Integer(KEY_TYPE)), GFP_KERNEL) + .is_err() { return None; } @@ -65,8 +70,7 @@ impl SepData { if cout.reply.status != 0 { return None; } - let cbody = - self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &cout)?; + let cbody = self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &cout)?; let mut cf = proto::FieldCursor::new(cbody); let (Some(0), Some(cblob)) = (cf.i32(), cf.blob()) else { return None; @@ -78,13 +82,13 @@ impl SepData { fn refkey_pub(blob: &[u8]) -> Option<&[u8]> { let rk_tlv = crate::der::refkey_find(blob, b"rk")?; - let pub_raw = crate::der::refkey_find(rk_tlv, b"pub") - .and_then(crate::der::octet_string_body)?; + let pub_raw = + crate::der::refkey_find(rk_tlv, b"pub").and_then(crate::der::octet_string_body)?; (pub_raw.len() == refkey_seal::POINT_LEN).then_some(pub_raw) } pub(crate) fn sks_machine_refkey(&self, handle: crate::sks::KeyBagHandle, secret: &[u8]) { - const MACHINE_REFKEY_PATH: &CStr = c"/var/lib/apple-sep-refkey.bin"; + const MACHINE_REFKEY_PATH: &CStr = c"/var/lib/aurora-sep-refkey.bin"; if self.machine_refkey.lock().is_some() { return; @@ -198,7 +202,11 @@ impl SepData { let der = crate::der::encode_refkey_set(&items).ok()?; let out = self.sks_send(self.sks_req_refkey(handle.value(), &der))?; if out.reply.status != 0 { - dev_warn!(self.dev, "sks: ref-key attest sign failed (status {})\n", out.reply.status); + dev_warn!( + self.dev, + "sks: ref-key attest sign failed (status {})\n", + out.reply.status + ); return None; } let body = self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &out)?; @@ -213,7 +221,7 @@ impl SepData { Some(owned) } - /// Attestation of key possession by signing proof; op `oa` (Apple-CA-chained) + /// Attestation of key possession by signing proof; op `oa` (attestation-chained) /// needs the SEP device attestation key, absent on a Linux-attached SEP. pub(crate) fn refkey_attest_sign(&self, challenge: &[u8]) -> Result<(KVec, KVec)> { self.ensure_machine_refkey()?; @@ -258,7 +266,11 @@ impl SepData { let der = crate::der::encode_refkey_set(&items).ok()?; let out = self.sks_send(self.sks_req_refkey(handle.value(), &der))?; if out.reply.status != 0 { - dev_warn!(self.dev, "trusted-keys: ref-key unseal failed (status {})\n", out.reply.status); + dev_warn!( + self.dev, + "trusted-keys: ref-key unseal failed (status {})\n", + out.reply.status + ); return None; } let body = self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &out)?; diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 308f3b76c01b9..fff495601bb85 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -174,7 +174,7 @@ impl SepData { } let persisted = { let index = self.bio_index.lock(); - self.with_store(|store| index.persist(store)) + self.with_host_store(|store| index.persist(store)) }; match persisted { Some(Ok(())) => {}, @@ -218,7 +218,7 @@ impl SepData { // lock ordering: store lock outside the session lock (UAF otherwise) if outcome.is_ok() { let index = self.bio_index.lock(); - let saved = self.with_store(|store| index.persist(store)); + let saved = self.with_host_store(|store| index.persist(store)); match saved { Some(Ok(())) => {}, Some(Err(e)) => dev_err!( @@ -589,7 +589,7 @@ impl SepData { return Some(blob); } } - match self.with_store(|store| store.read(&store::Key::root(kind))) { + match self.with_host_store(|store| store.read(&store::Key::root(kind))) { Some(Ok(Some(blob))) => Some(blob), Some(Ok(None)) => None, Some(Err(_)) => { @@ -709,7 +709,7 @@ impl SepData { if blob.is_empty() { return false; } - match self.with_store(|store| store.write(&store::Key::root(PRIVATE_TYPE_LOCKOUT), &blob)) { + match self.with_host_store(|store| store.write(&store::Key::root(PRIVATE_TYPE_LOCKOUT), &blob)) { Some(Ok(())) => { true } @@ -2219,7 +2219,7 @@ impl SepData { let persisted = { let mut index = self.bio_index.lock(); index.remove(uuid); - self.with_store(|store| index.persist(store)) + self.with_host_store(|store| index.persist(store)) }; match persisted { Some(Ok(())) => {}, diff --git a/drivers/soc/apple/seed.rs b/drivers/soc/apple/seed.rs deleted file mode 100644 index d1793d8ebe361..0000000000000 --- a/drivers/soc/apple/seed.rs +++ /dev/null @@ -1,113 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -// Copyright 2026 Dj - -//! Importing the enclave's existing records. - -use crate::shim; -use crate::store::{Key, Store}; -use kernel::prelude::*; - -pub(crate) const SEED_PATH: &CStr = c"/var/lib/apple-sep-seed.bin"; - -const MAGIC: [u8; 4] = *b"AXRT"; -const HEADER_LEN: usize = 8; -const RECORD_HEADER_LEN: usize = 1 + 16 + 4; - -const MIN_RECORD_LEN: u32 = 1; -const MAX_RECORD_LEN: u32 = 0x8000; - -const MAX_SEED_BYTES: u64 = 1 << 20; - -const MAX_RECORDS: u32 = 64; - -const TYPE_ROOT_LOW: u8 = 1; -const TYPE_ROOT_HIGH: u8 = 2; -const TYPE_SESSION_HIGH: u8 = 4; - -pub(crate) struct Imported { - pub(crate) records: usize, - pub(crate) roots: usize, - pub(crate) sessions: usize, - pub(crate) bytes: usize, -} - -fn le32(buf: &[u8], off: usize) -> u32 { - u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]]) -} - -fn read_seed() -> Result> { - let file = shim::StoreFile::open_readonly(SEED_PATH)?; - let size = file.size()?; - if size < HEADER_LEN as u64 || size > MAX_SEED_BYTES { - return Err(EINVAL); - } - let mut buf = KVec::with_capacity(size as usize, GFP_KERNEL)?; - buf.resize(size as usize, 0, GFP_KERNEL)?; - file.read_exact(0, &mut buf)?; - Ok(buf) -} - -pub(crate) fn import(store: &mut Store) -> Result { - let buf = read_seed()?; - - if buf[..4] != MAGIC { - return Err(EINVAL); - } - let count = le32(&buf, 4); - if count == 0 || count > MAX_RECORDS { - return Err(EINVAL); - } - - let mut offsets: KVec<(u8, [u8; 16], usize, usize)> = KVec::new(); - let mut pos = HEADER_LEN; - for _ in 0..count { - if pos + RECORD_HEADER_LEN > buf.len() { - return Err(EINVAL); - } - let kind = buf[pos]; - let mut uuid = [0u8; 16]; - uuid.copy_from_slice(&buf[pos + 1..pos + 17]); - let len = le32(&buf, pos + 17); - pos += RECORD_HEADER_LEN; - - if !(TYPE_ROOT_LOW..=TYPE_SESSION_HIGH).contains(&kind) { - return Err(EINVAL); - } - if !(MIN_RECORD_LEN..=MAX_RECORD_LEN).contains(&len) { - return Err(EINVAL); - } - // Root records carry an all-zero UUID; anything else would be unreachable. - let is_root = kind == TYPE_ROOT_LOW || kind == TYPE_ROOT_HIGH; - if is_root && uuid != [0u8; 16] { - return Err(EINVAL); - } - let len = len as usize; - if pos + len > buf.len() { - return Err(EINVAL); - } - offsets.push((kind, uuid, pos, len), GFP_KERNEL)?; - pos += len; - } - - // Writes are keyed, so re-running after a crash simply replaces. - let mut result = Imported { - records: 0, - roots: 0, - sessions: 0, - bytes: 0, - }; - for (kind, uuid, at, len) in offsets { - store.write(&Key::new(kind, uuid), &buf[at..at + len])?; - result.records += 1; - result.bytes += len; - if kind == TYPE_ROOT_LOW || kind == TYPE_ROOT_HIGH { - result.roots += 1; - } else { - result.sessions += 1; - } - } - - // Mark consumed last: a crash above leaves the flag clear and the import reruns. - store.mark_seeded()?; - Ok(result) -} diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index ea8f95c5121cc..4f546e66602e3 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -23,7 +23,6 @@ mod refkey_seal; mod rxring; mod sbio; mod scrd; -mod seed; mod sensor; mod shim; mod shmem; @@ -32,6 +31,7 @@ mod store; mod transfer; mod trusted; mod xarm; +mod xart_store; use kernel::{ device, @@ -794,7 +794,10 @@ struct SepData { bio_dev: Mutex>, #[pin] - store: Mutex>, + store: Mutex>, + + #[pin] + host_store: Mutex>, #[pin] xarm: Mutex, @@ -883,36 +886,44 @@ impl SepData { )?; let dma_ring = dma::Coherent::::zeroed_slice(dev, DMA_RING_SIZE, GFP_KERNEL)?; - let mut store = match store::Store::open() { - Ok(mut store) => { - - if !store.seeded() { - match seed::import(&mut store) { - Ok(_imported) => {}, - Err(e) => dev_err!( - dev, - "seed '{}' could not be imported ({:?}); store was never seeded, so the exchange stops after the first ROOT_READ and endpoint count stays at {}\n", - seed::SEED_PATH, - e, - ENDPOINTS_BEFORE_EXCHANGE - ), - } - } - + let xart_writes = *module_parameters::xart_writes.value() != 0; + let store = match xart_store::Store::open(xart_writes) { + Ok(store) => { + let (slots, records, revision, malformed, duplicates, repaired, writable) = + store.summary(); + dev_info!( + dev, + "xART: {} slots, {} live records, max revision {}, {} malformed, {} duplicate, {} repaired; writes {}\n", + slots, + records, + revision, + malformed, + duplicates, + repaired, + if writable { "ENABLED" } else { "disabled" } + ); Some(store) } Err(e) => { dev_err!( dev, - "could not open the backing store '{}': {:?}; persistent-state exchange cannot run\n", - store::STORE_PATH, + "shared xART mapping '{}' is unavailable or invalid: {:?}\n", + xart_store::STORE_PATH, e ); + return Err(e); + } + }; + + let mut host_store = match store::Store::open() { + Ok(store) => Some(store), + Err(e) => { + dev_warn!(dev, "Linux host-state store unavailable: {:?}\n", e); None } }; - let bio_index = match store.as_mut().map(bio::IdentityIndex::load) { + let bio_index = match host_store.as_mut().map(bio::IdentityIndex::load) { Some(Ok(index)) => index, Some(Err(_)) => bio::IdentityIndex::new(), None => bio::IdentityIndex::new(), @@ -968,6 +979,7 @@ impl SepData { bio_index <- new_mutex!(bio_index), bio_dev <- new_mutex!(None), store <- new_mutex!(store), + host_store <- new_mutex!(host_store), xarm <- new_mutex!(XarmState::new()), rng <- new_mutex!(None), machine_refkey <- new_mutex!(None), @@ -1233,17 +1245,23 @@ impl SepData { Ok(()) } - fn with_store(&self, f: impl FnOnce(&mut store::Store) -> R) -> Option { + fn with_store(&self, f: impl FnOnce(&mut xart_store::Store) -> R) -> Option { let mut guard = self.store.lock(); + let store: &mut Option = &mut guard; + store.as_mut().map(f) + } + + fn with_host_store(&self, f: impl FnOnce(&mut store::Store) -> R) -> Option { + let mut guard = self.host_store.lock(); let store: &mut Option = &mut guard; store.as_mut().map(f) } // pre-generate: an entropy draw on the drain path would deadlock on its own reply fn prepare_os_uuid(&self) { - let key = xarm::os_uuid_key(); + let key = store::Key::root(0xf0); - let existing = match self.with_store(|store| store.read(&key)) { + let existing = match self.with_host_store(|store| store.read(&key)) { Some(result) => result, None => return, }; @@ -1258,17 +1276,12 @@ impl SepData { } let mut uuid = [0u8; 16]; - for word in 0..4 { - match self.get_entropy_word() { - Ok(v) => uuid[word * 4..word * 4 + 4].copy_from_slice(&v.to_le_bytes()), - Err(_) => { - return; - } - } + if shim::random_bytes(&mut uuid).is_err() { + return; } xarm::make_uuid_v4(&mut uuid); - match self.with_store(|store| store.write(&key, &uuid)) { + match self.with_host_store(|store| store.write(&key, &uuid)) { Some(Ok(())) => {} Some(Err(_)) => { return; @@ -1971,6 +1984,7 @@ impl SepData { *self.mbox.lock() = None; let _ = self.store.lock().take(); + let _ = self.host_store.lock().take(); for slot in [&self.ool_xarm, &self.ool_sbio, &self.ool_sks] { if let Some(buffers) = slot.lock().take() { @@ -2209,4 +2223,10 @@ module! { name: "apple_sep", description: "Apple SEP coprocessor: warm attach, endpoint discovery and tag-correlated control endpoint", license: "Dual MIT/GPL", + params: { + xart_writes: u8 { + default: 0, + description: "Allow writes to the validated shared xART mapping", + }, + }, } diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index 2abb13765bad7..e39f036142825 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -33,6 +33,7 @@ void sep_hwrng_unregister(void *mem); void *sep_store_open(const char *path); void *sep_store_open_trunc(const char *path); void *sep_store_open_ro(const char *path); +void *sep_store_open_block(const char *path, int writable); void sep_store_close(void *handle); long long sep_store_size(void *handle); long sep_store_read(void *handle, long long off, void *buf, size_t len); diff --git a/drivers/soc/apple/shim.rs b/drivers/soc/apple/shim.rs index 2968a04943e99..a615b3aee0a96 100644 --- a/drivers/soc/apple/shim.rs +++ b/drivers/soc/apple/shim.rs @@ -9,6 +9,7 @@ extern "C" { fn sep_store_open(path: *const c_char) -> *mut c_void; fn sep_store_open_trunc(path: *const c_char) -> *mut c_void; fn sep_store_open_ro(path: *const c_char) -> *mut c_void; + fn sep_store_open_block(path: *const c_char, writable: c_int) -> *mut c_void; fn sep_store_close(handle: *mut c_void); fn sep_store_size(handle: *mut c_void) -> i64; fn sep_store_read(handle: *mut c_void, off: i64, buf: *mut c_void, len: usize) @@ -20,6 +21,12 @@ extern "C" { len: usize, ) -> c_long; fn sep_store_sync(handle: *mut c_void) -> c_int; + fn sep_random_bytes(buf: *mut c_void, len: usize) -> c_int; +} + +pub(crate) fn random_bytes(buf: &mut [u8]) -> Result<()> { + // SAFETY: `buf` is writable for exactly `buf.len()` bytes. + kernel::error::to_result(unsafe { sep_random_bytes(buf.as_mut_ptr().cast(), buf.len()) }) } /// Backing-store file handle. @@ -70,6 +77,16 @@ impl StoreFile { Ok(StoreFile { handle }) } + pub(crate) fn open_block(path: &CStr, writable: bool) -> Result { + // SAFETY: `path` is NUL-terminated; the shim accepts only a block + // device whose global read-only state matches `writable`. + let handle = unsafe { sep_store_open_block(path.as_char_ptr(), c_int::from(writable)) }; + if handle.is_null() { + return Err(ENODEV); + } + Ok(StoreFile { handle }) + } + pub(crate) fn size(&self) -> Result { // SAFETY: `handle` is live per the type invariant. let n = unsafe { sep_store_size(self.handle) }; @@ -100,6 +117,22 @@ impl StoreFile { Ok(()) } + pub(crate) fn read_block_exact(&self, off: u64, buf: &mut [u8]) -> Result<()> { + // SAFETY: `handle` remains live and `buf` is writable for its length. + let n = result_of(unsafe { + sep_store_read( + self.handle, + off as i64, + buf.as_mut_ptr().cast::(), + buf.len(), + ) + })?; + if n != buf.len() { + return Err(EIO); + } + Ok(()) + } + pub(crate) fn write_all(&self, off: u64, buf: &[u8]) -> Result<()> { let mut done = 0usize; while done < buf.len() { @@ -120,6 +153,22 @@ impl StoreFile { Ok(()) } + pub(crate) fn write_block_exact(&self, off: u64, buf: &[u8]) -> Result<()> { + // SAFETY: `handle` remains live and `buf` is readable for its length. + let n = result_of(unsafe { + sep_store_write( + self.handle, + off as i64, + buf.as_ptr().cast::(), + buf.len(), + ) + })?; + if n != buf.len() { + return Err(EIO); + } + Ok(()) + } + pub(crate) fn sync(&self) -> Result<()> { // SAFETY: `handle` is live per the type invariant. kernel::error::to_result(unsafe { sep_store_sync(self.handle) }) diff --git a/drivers/soc/apple/store.rs b/drivers/soc/apple/store.rs index ef4140ec781dc..1bb97fb92ccb8 100644 --- a/drivers/soc/apple/store.rs +++ b/drivers/soc/apple/store.rs @@ -1,14 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj -//! Host-owned slotted backing store for the SEP's persistent records: a -//! compile-time path exclusive to this driver, re-initialised and reseeded on a -//! magic mismatch. Re-initialised is never served empty — an empty store halts the SEP. +//! Linux-only state. SEP anti-replay records live in `xart_store`. use crate::shim; use kernel::prelude::*; -pub(crate) const STORE_PATH: &CStr = c"/var/lib/apple-sep-state.bin"; +pub(crate) const STORE_PATH: &CStr = c"/var/lib/aurora-sep-host-state.bin"; const BLOCK_SIZE: usize = 0x8000; const BLOCK_COUNT: usize = 72; @@ -16,7 +14,7 @@ pub(crate) const STORE_SIZE: usize = BLOCK_SIZE * BLOCK_COUNT; const SLOT_COUNT: usize = BLOCK_COUNT - 1; pub(crate) const MAX_VALUE: usize = BLOCK_SIZE; -const MAGIC: [u8; 16] = *b"APPLE-SEP-STOR01"; +const MAGIC: [u8; 16] = *b"AURORA-SEP-STOR\x01"; const VERSION: u32 = 1; #[derive(Clone, Copy, PartialEq, Eq)] @@ -26,6 +24,7 @@ pub(crate) struct Key { } impl Key { + #[allow(dead_code)] pub(crate) const fn new(kind: u8, uuid: [u8; 16]) -> Key { Key { kind, uuid } } @@ -66,6 +65,7 @@ enum Intent { slot: u16, kind: u8, }, + #[allow(dead_code)] Delete { slot: u16, kind: u8, @@ -80,8 +80,6 @@ const SB_GENERATION: usize = 28; const SB_INTENT_KIND: usize = 36; const SB_INTENT_SLOT: usize = 37; const SB_INTENT_TYPE: usize = 39; -// In previously reserved space, so an older store reads it as zero (unseeded). -const SB_SEEDED: usize = 40; const SB_SLOT_TABLE: usize = 64; const SLOT_ENTRY_SIZE: usize = 24; @@ -96,7 +94,6 @@ pub(crate) struct Store { generation: u64, pub(crate) recovered: bool, pub(crate) fresh: bool, - seeded: bool, } fn le32(buf: &[u8], off: usize) -> u32 { @@ -119,7 +116,6 @@ impl Store { generation: 0, recovered: false, fresh: false, - seeded: false, }; let size = store.file.size()?; @@ -142,8 +138,6 @@ impl Store { } store.generation = le64(&sb, SB_GENERATION); - store.seeded = sb[SB_SEEDED] != 0; - for i in 0..SLOT_COUNT { let off = SB_SLOT_TABLE + i * SLOT_ENTRY_SIZE; let mut uuid = [0u8; 16]; @@ -174,7 +168,6 @@ impl Store { self.slots = [Slot::FREE; SLOT_COUNT]; self.generation = 1; self.fresh = true; - self.seeded = false; let mut zero = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; zero.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; @@ -206,7 +199,6 @@ impl Store { Intent::Write { slot, kind } => (INTENT_WRITE, slot, kind), Intent::Delete { slot, kind } => (INTENT_DELETE, slot, kind), }; - sb[SB_SEEDED] = u8::from(self.seeded); sb[SB_INTENT_KIND] = ikind; sb[SB_INTENT_SLOT..SB_INTENT_SLOT + 2].copy_from_slice(&islot.to_le_bytes()); sb[SB_INTENT_TYPE] = itype; @@ -235,6 +227,9 @@ impl Store { } pub(crate) fn read(&mut self, key: &Key) -> Result>> { + if !(0xf0..=0xf5).contains(&key.kind) { + return Err(EINVAL); + } let Some(idx) = self.find(key) else { return Ok(None); }; @@ -248,6 +243,9 @@ impl Store { } pub(crate) fn write(&mut self, key: &Key, value: &[u8]) -> Result<()> { + if !(0xf0..=0xf5).contains(&key.kind) { + return Err(EINVAL); + } if value.len() > MAX_VALUE { return Err(ENOSPC); } @@ -282,7 +280,11 @@ impl Store { self.commit() } + #[allow(dead_code)] pub(crate) fn delete(&mut self, key: &Key) -> Result { + if !(0xf0..=0xf5).contains(&key.kind) { + return Err(EINVAL); + } let Some(idx) = self.find(key) else { return Ok(false); }; @@ -299,16 +301,6 @@ impl Store { Ok(true) } - // Gates import, not store-emptiness: a store emptied after the SEP moved on must not be reseeded. - pub(crate) fn seeded(&self) -> bool { - self.seeded - } - - pub(crate) fn mark_seeded(&mut self) -> Result<()> { - self.seeded = true; - self.commit() - } - } pub(crate) const fn crc16_ccitt_false(data: &[u8]) -> u16 { diff --git a/drivers/soc/apple/store_shim.c b/drivers/soc/apple/store_shim.c index 33ed552f54798..715c7c9c6f8db 100644 --- a/drivers/soc/apple/store_shim.c +++ b/drivers/soc/apple/store_shim.c @@ -11,6 +11,7 @@ */ #include +#include #include #include #include @@ -65,6 +66,22 @@ void *sep_store_open_ro(const char *path) return IS_ERR(f) ? NULL : f; } +void *sep_store_open_block(const char *path, int writable) +{ + struct file *f = open_as_kernel(path, + writable ? O_RDWR | O_LARGEFILE : O_RDONLY | O_LARGEFILE, + 0); + + if (IS_ERR(f)) + return NULL; + if (!S_ISBLK(file_inode(f)->i_mode) || + bdev_read_only(file_bdev(f)) == !!writable) { + filp_close(f, NULL); + return NULL; + } + return f; +} + void sep_store_close(void *handle) { if (handle) @@ -76,6 +93,8 @@ long long sep_store_size(void *handle) { struct file *f = handle; + if (S_ISBLK(file_inode(f)->i_mode)) + return bdev_nr_bytes(file_bdev(f)); return i_size_read(file_inode(f)); } diff --git a/drivers/soc/apple/xarm.rs b/drivers/soc/apple/xarm.rs index 1c39467149b41..acb912ea5d5e5 100644 --- a/drivers/soc/apple/xarm.rs +++ b/drivers/soc/apple/xarm.rs @@ -7,7 +7,8 @@ use crate::xarm::{XarmReply as Reply, XarmRequest as Request}; use kernel::soc::apple::mailbox::Message; -use crate::store::{crc16_ccitt_false, Key, Store, MAX_VALUE}; +use crate::store::crc16_ccitt_false; +use crate::xart_store::{Key, Store, MAX_VALUE}; use kernel::prelude::*; pub(crate) use crate::proto::EP_XARM; @@ -36,8 +37,6 @@ const ROOT_READ_PREFIX: usize = 17; /// `0x01` rejects when `(length >> 4) >= 0x7ff`. const ROOT_WRITE_MAX: usize = 0x7fef; -pub(crate) const PRIVATE_TYPE_OS_UUID: u8 = 0xF0; - impl Reply { fn ok(req: &Request, length: u16) -> Reply { Reply { @@ -172,7 +171,7 @@ pub(crate) fn service( } OP_SESSION_READ => { - if outbound.len() < 16 { + if outbound.len() != 16 { return Serviced { reply: Reply::fail(req, STATUS_FAILED), reply_bytes: 0, @@ -296,10 +295,6 @@ pub(crate) fn make_uuid_v4(bytes: &mut [u8; 16]) { bytes[8] = (bytes[8] & 0x3f) | 0x80; } -pub(crate) fn os_uuid_key() -> Key { - Key::root(PRIVATE_TYPE_OS_UUID) -} - pub(crate) struct XarmRequest { pub(crate) tag: u8, pub(crate) opcode: u8, diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs new file mode 100644 index 0000000000000..d4a7a3566ec13 --- /dev/null +++ b/drivers/soc/apple/xart_store.rs @@ -0,0 +1,402 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj +// Copyright 2026 Aurora Silicon + +//! Apple's device-wide xART gigalocker record store. +//! +//! The APFS locator exposes the existing `.gl` file as a block device. This +//! module implements the record validation, duplicate repair, lookup and +//! copy-on-write ordering used by AppleSEPManager. It never creates storage +//! and it can be opened read-only for safe inspection and bring-up. + +use crate::shim; +use kernel::prelude::*; + +pub(crate) const STORE_PATH: &CStr = c"/dev/mapper/sep-xart-gigalocker"; + +const BLOCK_SIZE: usize = 0x1000; +const SLOT_SIZE: usize = 0x9000; +const HEADER_SIZE: usize = 0x22; +const DELETE_SIZE: usize = BLOCK_SIZE; +/// Size of the APFS raw extent located on the target machine. +/// +/// Accepting a larger block device would make a bad device-mapper table a +/// corruption hazard. Accepting a smaller one could silently truncate the +/// slot grid. A future locator for a machine with a different extent size must +/// pass that size through an explicit, reviewed interface instead of weakening +/// this check. +const STORE_SIZE: u64 = 0x600000; +const MAX_SLOTS: usize = 4096; + +pub(crate) const MAX_VALUE: usize = 0x8000; + +const KEY_KIND: usize = 0x01; +const KEY_UUID: usize = 0x02; +const LENGTH: usize = 0x12; +const CRC: usize = 0x16; +const REVISION: usize = 0x1a; +const PAYLOAD: usize = HEADER_SIZE; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct Key { + pub(crate) kind: u8, + pub(crate) uuid: [u8; 16], +} + +impl Key { + pub(crate) const fn new(kind: u8, uuid: [u8; 16]) -> Key { + Key { kind, uuid } + } + + pub(crate) const fn root(kind: u8) -> Key { + Key { + kind, + uuid: [0; 16], + } + } +} + +#[derive(Clone, Copy)] +struct Slot { + used: bool, + key: Key, + len: u32, + crc: u32, + revision: u64, +} + +impl Slot { + const FREE: Slot = Slot { + used: false, + key: Key::root(0), + len: 0, + crc: 0, + revision: 0, + }; + + fn matches(&self, key: &Key) -> bool { + self.used && self.key == *key + } +} + +pub(crate) struct Store { + file: shim::StoreFile, + slots: KVec, + revision: u64, + writes_enabled: bool, + valid_records: usize, + malformed_records: usize, + duplicate_records: usize, + repaired_records: usize, +} + +fn le32(bytes: &[u8], off: usize) -> u32 { + u32::from_le_bytes([bytes[off], bytes[off + 1], bytes[off + 2], bytes[off + 3]]) +} + +fn le64(bytes: &[u8], off: usize) -> u64 { + u64::from_le_bytes([ + bytes[off], + bytes[off + 1], + bytes[off + 2], + bytes[off + 3], + bytes[off + 4], + bytes[off + 5], + bytes[off + 6], + bytes[off + 7], + ]) +} + +fn valid_key(key: &Key) -> bool { + (1..=4).contains(&key.kind) && (key.kind > 2 || key.uuid == [0; 16]) +} + +impl Store { + pub(crate) fn open(writes_enabled: bool) -> Result { + Self::open_at(STORE_PATH, writes_enabled) + } + + /// Opens a caller-selected block mapping. + /// + /// Production always uses [`STORE_PATH`]. The separate xART self-test + /// module uses this entry point with its fixed loop-only mapper name, so it + /// can execute the real parser and write ordering without enabling SEP. + pub(crate) fn open_at(path: &CStr, writes_enabled: bool) -> Result { + let file = shim::StoreFile::open_block(path, writes_enabled)?; + let size = file.size()?; + if size != STORE_SIZE || size % BLOCK_SIZE as u64 != 0 { + return Err(EINVAL); + } + let count = (size / SLOT_SIZE as u64) as usize; + if count == 0 || count > MAX_SLOTS { + return Err(EINVAL); + } + + let mut slots = KVec::with_capacity(count, GFP_KERNEL)?; + for _ in 0..count { + slots.push(Slot::FREE, GFP_KERNEL)?; + } + let mut store = Store { + file, + slots, + revision: 0, + // Discovery is always read-only. Do not arm even repair writes + // until the required Apple root records have validated. + writes_enabled: false, + valid_records: 0, + malformed_records: 0, + duplicate_records: 0, + repaired_records: 0, + }; + store.scan()?; + // Serving an empty or unrelated 6 MiB mapping is the failure mode that + // originally desynchronised SEP from macOS. This Linux driver is never + // the authority that provisions a blank device-wide store. Require + // both existing root families before any mailbox registration can run. + if store.find(&Key::root(1)).is_none() || store.find(&Key::root(2)).is_none() { + return Err(ENODATA); + } + store.writes_enabled = writes_enabled; + if writes_enabled { + store.repair_disk()?; + } + Ok(store) + } + + fn slot_offset(slot: usize) -> u64 { + (slot * SLOT_SIZE) as u64 + } + + fn scan(&mut self) -> Result<()> { + let mut raw = KVec::with_capacity(SLOT_SIZE, GFP_KERNEL)?; + raw.resize(SLOT_SIZE, 0, GFP_KERNEL)?; + + for idx in 0..self.slots.len() { + self.file + .read_block_exact(Self::slot_offset(idx), &mut raw)?; + let kind = raw[KEY_KIND]; + if kind == 0 { + continue; + } + + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&raw[KEY_UUID..KEY_UUID + 16]); + let key = Key { kind, uuid }; + let len = le32(&raw, LENGTH) as usize; + let crc = le32(&raw, CRC); + let revision = le64(&raw, REVISION); + let valid = valid_key(&key) + && (1..=MAX_VALUE).contains(&len) + && crc32_ieee(&raw[PAYLOAD..PAYLOAD + len.min(MAX_VALUE)]) == crc; + + if !valid { + self.malformed_records += 1; + continue; + } + + self.revision = self.revision.max(revision); + let candidate = Slot { + used: true, + key, + len: len as u32, + crc, + revision, + }; + + if let Some(old) = self.find(&key) { + self.duplicate_records += 1; + // Equal revisions keep the later physical slot, matching the + // forward scan in AppleSEPManager's fixup pass. + if revision >= self.slots[old].revision { + self.slots[old] = Slot::FREE; + self.slots[idx] = candidate; + } + } else { + self.slots[idx] = candidate; + } + } + + self.valid_records = self.slots.iter().filter(|slot| slot.used).count(); + Ok(()) + } + + /// Removes malformed records and duplicate losers only after the mapping + /// has passed its complete read-only scan and both Apple roots exist. + fn repair_disk(&mut self) -> Result<()> { + let mut header = KVec::with_capacity(DELETE_SIZE, GFP_KERNEL)?; + header.resize(DELETE_SIZE, 0, GFP_KERNEL)?; + + for idx in 0..self.slots.len() { + if self.slots[idx].used { + continue; + } + self.file + .read_block_exact(Self::slot_offset(idx), &mut header)?; + if header[KEY_KIND] != 0 { + self.delete_slot(idx)?; + self.repaired_records += 1; + } + } + Ok(()) + } + + fn find(&self, key: &Key) -> Option { + let mut best: Option = None; + for (idx, slot) in self.slots.iter().enumerate() { + if !slot.matches(key) { + continue; + } + if best.is_none_or(|old| slot.revision >= self.slots[old].revision) { + best = Some(idx); + } + } + best + } + + fn find_free(&self, skip: usize) -> Option { + self.slots + .iter() + .enumerate() + .filter(|(_, slot)| !slot.used) + .nth(skip) + .map(|(idx, _)| idx) + } + + fn delete_slot(&self, slot: usize) -> Result<()> { + let mut zero = KVec::with_capacity(DELETE_SIZE, GFP_KERNEL)?; + zero.resize(DELETE_SIZE, 0, GFP_KERNEL)?; + self.file + .write_block_exact(Self::slot_offset(slot), &zero)?; + self.file.sync() + } + + pub(crate) fn read(&mut self, key: &Key) -> Result>> { + if !valid_key(key) { + return Err(EINVAL); + } + let Some(idx) = self.find(key) else { + return Ok(None); + }; + let slot = self.slots[idx]; + let mut raw = KVec::with_capacity(SLOT_SIZE, GFP_KERNEL)?; + raw.resize(SLOT_SIZE, 0, GFP_KERNEL)?; + self.file + .read_block_exact(Self::slot_offset(idx), &mut raw)?; + + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&raw[KEY_UUID..KEY_UUID + 16]); + let disk_key = Key { + kind: raw[KEY_KIND], + uuid, + }; + let len = le32(&raw, LENGTH) as usize; + if disk_key != *key + || len != slot.len as usize + || le32(&raw, CRC) != slot.crc + || le64(&raw, REVISION) != slot.revision + || crc32_ieee(&raw[PAYLOAD..PAYLOAD + len]) != slot.crc + { + return Err(EIO); + } + + let mut value = KVec::with_capacity(len, GFP_KERNEL)?; + value.extend_from_slice(&raw[PAYLOAD..PAYLOAD + len], GFP_KERNEL)?; + Ok(Some(value)) + } + + pub(crate) fn write(&mut self, key: &Key, value: &[u8]) -> Result<()> { + if !self.writes_enabled { + return Err(EROFS); + } + if !valid_key(key) || value.is_empty() || value.len() > MAX_VALUE { + return Err(EINVAL); + } + + let old = self.find(key); + // Apple skips the first free slot when creating a new key, but uses the + // first free slot for replacement. + let fresh = self.find_free(usize::from(old.is_none())).ok_or(ENOSPC)?; + let revision = self.revision.checked_add(1).ok_or(EINVAL)?; + let crc = crc32_ieee(value); + + let mut raw = KVec::with_capacity(SLOT_SIZE, GFP_KERNEL)?; + raw.resize(SLOT_SIZE, 0, GFP_KERNEL)?; + raw[KEY_KIND] = key.kind; + raw[KEY_UUID..KEY_UUID + 16].copy_from_slice(&key.uuid); + raw[LENGTH..LENGTH + 4].copy_from_slice(&(value.len() as u32).to_le_bytes()); + raw[CRC..CRC + 4].copy_from_slice(&crc.to_le_bytes()); + raw[REVISION..REVISION + 8].copy_from_slice(&revision.to_le_bytes()); + raw[PAYLOAD..PAYLOAD + value.len()].copy_from_slice(value); + + // The new record becomes authoritative only after its complete slot is + // durable. The old record is then removed and flushed separately. + self.file + .write_block_exact(Self::slot_offset(fresh), &raw)?; + self.file.sync()?; + self.slots[fresh] = Slot { + used: true, + key: *key, + len: value.len() as u32, + crc, + revision, + }; + self.revision = revision; + + if let Some(old) = old { + self.delete_slot(old)?; + self.slots[old] = Slot::FREE; + } + self.valid_records = self.slots.iter().filter(|slot| slot.used).count(); + Ok(()) + } + + pub(crate) fn delete(&mut self, key: &Key) -> Result { + if !valid_key(key) { + return Err(EINVAL); + } + let Some(idx) = self.find(key) else { + return Ok(false); + }; + if !self.writes_enabled { + return Err(EROFS); + } + self.delete_slot(idx)?; + self.slots[idx] = Slot::FREE; + self.valid_records -= 1; + Ok(true) + } + + pub(crate) fn summary(&self) -> (usize, usize, u64, usize, usize, usize, bool) { + ( + self.slots.len(), + self.valid_records, + self.revision, + self.malformed_records, + self.duplicate_records, + self.repaired_records, + self.writes_enabled, + ) + } +} + +/// Reflected CRC-32/ISO-HDLC (the IEEE CRC-32 used in gigalocker records). +pub(crate) const fn crc32_ieee(data: &[u8]) -> u32 { + let mut crc = 0xffff_ffffu32; + let mut i = 0; + while i < data.len() { + crc ^= data[i] as u32; + let mut bit = 0; + while bit < 8 { + crc = if crc & 1 != 0 { + (crc >> 1) ^ 0xedb8_8320 + } else { + crc >> 1 + }; + bit += 1; + } + i += 1; + } + crc ^ 0xffff_ffff +} + +static_assert!(crc32_ieee(b"123456789") == 0xcbf4_3926); From 28a64ca3bbe867189b3fe77aa34bfcfc87763029 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 14:57:22 -0400 Subject: [PATCH 0689/2680] soc: apple: add production SEP FileVault integration Unwrap a FileVault volume's key through the enclave and load the volume's class keys into the enclave's volatile state, then drive the storage controller's inline AES-XTS with the wrapped key so an encrypted APFS volume is read and written with no plaintext key material on the application processor. The volume and key-bag records come from plaintext APFS metadata; only wrapped keys cross to the enclave. [ryan: split from commit e7db648a8064 ("soc: apple: add production SEP FileVault integration"): this commit carries its SEP side (drivers/soc/apple, fv_shim and include/linux/apple-sep-fv.h, spi-apple, trusted keys, tools/aurora-sep). Its nvme-apple, nvme core, nvme Kconfig and blk-crypto parts are commit a4415a9a473f ("nvme-apple: Drive the NVMMU inline AES-XTS with wrapped FileVault keys") in 050-nvme and are not duplicated here. Adapted to 7.2.10: the probe keeps the lifetime-bound driver-core form; the dead_code allowances the previous commit needed are gone now that the store items have users.] Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 4 + drivers/soc/apple/Makefile | 4 +- drivers/soc/apple/bio.rs | 9 +- drivers/soc/apple/control.rs | 46 +- drivers/soc/apple/crypto_shim.c | 29 +- drivers/soc/apple/der.rs | 6 +- drivers/soc/apple/dt.rs | 33 +- drivers/soc/apple/fv.rs | 1106 ++++++++++++++-- drivers/soc/apple/fv_shim.c | 315 +++++ drivers/soc/apple/image.rs | 47 +- drivers/soc/apple/keybag.rs | 40 +- drivers/soc/apple/proto.rs | 33 +- drivers/soc/apple/sbio.rs | 279 ++-- drivers/soc/apple/scrd.rs | 8 +- drivers/soc/apple/sensor.rs | 15 +- drivers/soc/apple/sep.rs | 345 ++--- drivers/soc/apple/shim.h | 47 +- drivers/soc/apple/shim.rs | 20 +- drivers/soc/apple/sks.rs | 450 +++---- drivers/soc/apple/store.rs | 39 +- drivers/soc/apple/transfer.rs | 58 +- drivers/soc/apple/work_shim.c | 14 + drivers/soc/apple/xarm.rs | 49 +- drivers/soc/apple/xart_store.rs | 22 +- drivers/spi/spi-apple.c | 87 ++ include/linux/apple-sep-fv.h | 54 + security/keys/trusted-keys/trusted_core.c | 21 +- tools/aurora-sep/README.md | 34 + tools/aurora-sep/aurora-sep.service | 13 + tools/aurora-sep/fprintd-aurora.conf | 2 + tools/aurora-sep/load-driver | 40 + .../libfprint-1.94.100-apple-sep.patch | 1179 +++++++++++++++++ 32 files changed, 3383 insertions(+), 1065 deletions(-) create mode 100644 drivers/soc/apple/fv_shim.c create mode 100644 drivers/soc/apple/work_shim.c create mode 100644 include/linux/apple-sep-fv.h create mode 100644 tools/aurora-sep/README.md create mode 100644 tools/aurora-sep/aurora-sep.service create mode 100644 tools/aurora-sep/fprintd-aurora.conf create mode 100755 tools/aurora-sep/load-driver create mode 100644 tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index edd3b87de65c6..412f3ce64ee4d 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -109,6 +109,9 @@ config APPLE_AOP Say 'y' here if you have an Apple laptop. +config APPLE_SEP_FV_INTERFACE + bool + config APPLE_SEP tristate "Apple SEP (Secure Enclave Processor)" depends on ARCH_APPLE @@ -121,6 +124,7 @@ config APPLE_SEP select CRYPTO_AES select CRYPTO_GCM select CRYPTO_ECDH + select APPLE_SEP_FV_INTERFACE help Driver for the Apple SEP (Secure Enclave Processor) on Apple silicon. It drives the Touch ID fingerprint sensor over a /dev/sep-bio character diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index b8cd56e097439..2920afa1502d2 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -22,9 +22,11 @@ apple-tunable-y = tunable.o obj-$(CONFIG_APPLE_AOP) += aop.o +obj-$(CONFIG_APPLE_SEP_FV_INTERFACE) += fv_shim.o + obj-$(CONFIG_APPLE_SEP) += apple-sep.o apple-sep-y := sep.o hwrng_shim.o store_shim.o bio_shim.o sha_shim.o \ - crypto_shim.o sensor_shim.o p256_shim.o trusted_shim.o + crypto_shim.o sensor_shim.o p256_shim.o trusted_shim.o work_shim.o obj-$(CONFIG_APPLE_PMP) += pmp.o diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index 27d7bd25d1110..e66710b6935cf 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -705,6 +705,7 @@ fn verify_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { nonce: request.nonce, terminal: Some(VerifyOutcome::NoMatch), }; + ctx.session.unseen = true; return Ok(Handled { ret: 0, wake: true, @@ -853,9 +854,7 @@ fn delete(ctx: &mut Context<'_>, user: UserPtr) -> Result { // `0x57` takes one `identity_v1_t` (signed user id + 16-byte UUID). let Some(identity) = ctx.index.identity_v1_for(&request.uuid, ENROL_USER_ID) else { - pr_warn!( - "sep_bio: DELETE of a held identity could not be expressed as an identity_v1_t\n" - ); + pr_warn!("sep_bio: DELETE of a held identity could not be expressed as an identity_v1_t\n"); return Err(ENOENT); }; @@ -873,9 +872,7 @@ fn delete_all(ctx: &mut Context<'_>) -> Result { require_admin()?; if ctx.index.total() == 0 { - pr_info!( - "sep_bio: DELETE_ALL on empty index; reporting success (nothing to delete)\n" - ); + pr_info!("sep_bio: DELETE_ALL on empty index; reporting success (nothing to delete)\n"); return ok(); } diff --git a/drivers/soc/apple/control.rs b/drivers/soc/apple/control.rs index be81aa1de38df..5e395dc6a0b80 100644 --- a/drivers/soc/apple/control.rs +++ b/drivers/soc/apple/control.rs @@ -28,8 +28,6 @@ impl Slot { struct EntropySink { value: Option, - msg1: u32, - unclaimed: u32, busy: bool, } @@ -44,11 +42,7 @@ pub(crate) struct ControlState { next_tag: u8, retired: [u64; 4], retired_count: u32, - // SEP-initiated messages (type != 0x01). - unsolicited: u32, entropy: EntropySink, - // After the persistent-state exchange the SEP parks this endpoint; refuse rather than time out. - closed: bool, } impl ControlState { @@ -58,14 +52,10 @@ impl ControlState { next_tag: proto::TAG_POOL_FIRST, retired: [0; 4], retired_count: 0, - unsolicited: 0, entropy: EntropySink { value: None, - msg1: 0, - unclaimed: 0, busy: false, }, - closed: false, } } @@ -80,14 +70,20 @@ impl ControlState { } } + fn reclaim_retired(&mut self, tag: u8) -> bool { + if !self.is_retired(tag) { + return false; + } + self.retired[(tag >> 6) as usize] &= !(1u64 << (tag & 0x3f)); + self.retired_count -= 1; + true + } + fn tag_in_flight(&self, tag: u8) -> bool { self.slots.iter().any(|s| s.used && s.tag == tag) } pub(crate) fn alloc(&mut self) -> Result<(usize, u8)> { - if self.closed { - return Err(EPIPE); - } let idx = self.slots.iter().position(|s| !s.used).ok_or(EBUSY)?; for _ in 0..POOL_LEN { @@ -132,11 +128,7 @@ impl ControlState { pub(crate) fn deliver(&mut self, reply: proto::ControlReply) -> Delivery { // The reserved entropy tag is claimed first, outstanding request or not. if reply.tag == proto::TAG_ENTROPY { - if self.entropy.value.is_some() { - self.entropy.unclaimed = self.entropy.unclaimed.wrapping_add(1); - } self.entropy.value = Some(reply.data_lo); - self.entropy.msg1 = reply.msg1; return Delivery::Entropy; } @@ -149,31 +141,25 @@ impl ControlState { return Delivery::Matched; } + if self.reclaim_retired(reply.tag) { + return Delivery::Unmatched; + } + // No match: drop it, never hand it to another waiter. Delivery::Unmatched } - pub(crate) fn note_unsolicited(&mut self) -> u32 { - self.unsolicited = self.unsolicited.wrapping_add(1); - self.unsolicited - } - pub(crate) fn entropy_begin(&mut self) -> Result<()> { - if self.closed { - return Err(EPIPE); - } if self.entropy.busy { return Err(EBUSY); } self.entropy.busy = true; - if self.entropy.value.take().is_some() { - self.entropy.unclaimed = self.entropy.unclaimed.wrapping_add(1); - } + self.entropy.value = None; Ok(()) } - pub(crate) fn entropy_take(&mut self) -> Option<(u32, u32)> { - self.entropy.value.take().map(|v| (v, self.entropy.msg1)) + pub(crate) fn entropy_take(&mut self) -> Option { + self.entropy.value.take() } pub(crate) fn entropy_end(&mut self) { diff --git a/drivers/soc/apple/crypto_shim.c b/drivers/soc/apple/crypto_shim.c index e4d0863cce01b..be7336ba00539 100644 --- a/drivers/soc/apple/crypto_shim.c +++ b/drivers/soc/apple/crypto_shim.c @@ -7,17 +7,26 @@ #include #include #include +#include #include #include #include #include -#include -#include #include "shim.h" #define SEP_GCM_TAG_LEN 16 +int sep_random_bytes(void *buf, size_t len) +{ + int ret = wait_for_random_bytes(); + + if (ret) + return ret; + get_random_bytes(buf, len); + return 0; +} + /* * AES-GCM with a 16-byte (non-96-bit) IV, over the raw AES block cipher. The * kernel's gcm(aes) only takes a 12-byte IV; the SEP's ECIES uses a 16-byte IV, @@ -174,21 +183,6 @@ static int sep_gcm16(int encrypt, const void *key, size_t keylen, return rc; } -/* - * Fills `buf` from the kernel CSPRNG, waiting for the seed first so it never - * returns unseeded bytes. Host-side entropy for key-bag secrets the host must - * reproduce; anything measuring the enclave's own entropy stays on SEP. - */ -int sep_random_bytes(void *buf, size_t len) -{ - int ret = wait_for_random_bytes(); - - if (ret) - return ret; - get_random_bytes(buf, len); - return 0; -} - /* HMAC-SHA256 over one message; writes 32 bytes to `out`, untouched on error. */ int sep_hmac_sha256(const void *key, size_t keylen, const void *data, size_t datalen, u8 *out) @@ -242,7 +236,6 @@ int sep_gcm(int encrypt, const void *key, size_t keylen, (ivlen != 12 && ivlen != 16) || ivlen > sizeof(ivcopy)) return -EINVAL; - /* The whole AAD-plus-payload-plus-tag extent must be inside the buffer. */ if (aadlen + datalen + SEP_GCM_TAG_LEN < aadlen || aadlen + datalen + SEP_GCM_TAG_LEN > buflen) return -EINVAL; diff --git a/drivers/soc/apple/der.rs b/drivers/soc/apple/der.rs index bc3db32343eae..fcd198561872f 100644 --- a/drivers/soc/apple/der.rs +++ b/drivers/soc/apple/der.rs @@ -140,8 +140,11 @@ pub(crate) fn refkey_find<'a>(set_blob: &'a [u8], key: &[u8]) -> Option<&'a [u8] if key_tag != TAG_UTF8 { return None; } - let (_val_tag, _value, after_val) = take_tlv(after_key)?; if this_key == key { + let (_val_tag, _value, after_val) = take_tlv(after_key)?; + if !after_val.is_empty() { + return None; + } return Some(&after_key[..after_key.len() - after_val.len()]); } body = next; @@ -155,4 +158,3 @@ pub(crate) fn octet_string_body(tlv: &[u8]) -> Option<&[u8]> { _ => None, } } - diff --git a/drivers/soc/apple/dt.rs b/drivers/soc/apple/dt.rs index 7fc5be540c654..ca3b9445e8d60 100644 --- a/drivers/soc/apple/dt.rs +++ b/drivers/soc/apple/dt.rs @@ -24,6 +24,8 @@ const DMA_RANGE_PROP: &CStr = c"apple,dma-range"; const DMA_RANGE_CELLS: [u32; 4] = [0, 0, 1, 0]; const REGISTERED_PROP: &CStr = c"apple,sep-shmem-registered-iova"; +const CHOSEN_PATH: &CStr = c"/chosen"; +const PREBOOT_UUID_PROP: &CStr = c"apfs-preboot-uuid"; pub(crate) struct DtNode(*mut bindings::device_node); @@ -151,6 +153,35 @@ pub(crate) fn sep_node() -> Option { DtNode::find_compatible(SEP_COMPATIBLE) } +pub(crate) fn preboot_uuid() -> Option<[u8; 16]> { + // SAFETY: the path is NUL-terminated; a non-NULL result owns one node + // reference, which `DtNode` releases. + let raw = unsafe { + bindings::of_find_node_opts_by_path(CHOSEN_PATH.as_char_ptr(), core::ptr::null_mut()) + }; + let chosen = (!raw.is_null()).then_some(DtNode(raw))?; + let mut text = core::ptr::null(); + // SAFETY: `chosen` is live, the property name is NUL-terminated, and + // `text` is a valid output pointer. + if unsafe { + bindings::of_property_read_string( + chosen.as_ptr(), + PREBOOT_UUID_PROP.as_char_ptr(), + &mut text, + ) + } != 0 + { + return None; + } + + let mut uuid = bindings::uuid_t { b: [0; 16] }; + // SAFETY: the property is NUL-terminated and `uuid` is a valid output. + if unsafe { bindings::uuid_parse(text, &mut uuid) } != 0 || uuid.b.iter().all(|&b| b == 0) { + return None; + } + Some(uuid.b) +} + pub(crate) fn enable_sep_and_dart() -> Result<()> { let sep = sep_node().ok_or_else(|| { pr_err!( @@ -393,7 +424,7 @@ pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { } pr_info!( - "apple_sep: creating the sensor node (absent from Linux's tree; Apple has it at /arm-io/spi2/mesa)\n" + "apple_sep: creating the sensor node (absent from Linux's tree; the firmware exposes it at /arm-io/spi2/mesa)\n" ); with_changeset(|cs_handle| { diff --git a/drivers/soc/apple/fv.rs b/drivers/soc/apple/fv.rs index ba05aba7d854f..9ba5156dee386 100644 --- a/drivers/soc/apple/fv.rs +++ b/drivers/soc/apple/fv.rs @@ -1,158 +1,996 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj -//! Native FileVault key hierarchy: device-bound volume-key provisioning, -//! reverse-engineered and kept as capability. Not wired to a caller. -#![allow(dead_code)] -use crate::{image, proto, shim}; -use crate::{LockState, SepData, SksRequest}; +use crate::{image, proto, SepData, SksRequest, PHASE_READY}; use kernel::prelude::*; +use kernel::sync::atomic::Relaxed; + +const WRAPPED_KEY_LEN: usize = 40; +const OPAQUE_KEY_LEN: usize = 64; +const IV_KEY_LEN: usize = 16; +const SYSTEM_CLIENT: u64 = 1; +const NO_KEYBAG_HANDLE: i32 = -1; +const WRAPPED_KEY_FLAG: i32 = 1 << 1; +const VOLUME_KEY_FLAG: i32 = 1; +const SECRET_MAX_LEN: usize = 1024; +const RECORD_MAX_LEN: usize = 528; +const FILE_KEY_MAX_LEN: usize = 168; +const EPHEMERAL_KEY_MAX_LEN: usize = 64; +const FV_UNWRAP_VERSION: u32 = 0; +const FV_UNWRAP_OPTIONS: u32 = 2; +const PFK_UNWRAP_VERSION: u32 = 2; +const PFK_SYSTEM_VOLUME_HANDLE: i32 = -5; +const PFK_UNWRAP_OPTIONS: u32 = 0x102; +const PFK_KEY_FLAG: i32 = 1 << 1; +const PFK_NEW_VERSION: u32 = 2; +const PFK_NEW_OPTIONS: u32 = 0x102; +const PFK_NEW_KEY_FLAG: i32 = 1 << 1; +const PFK_FS_CONTEXT_LEN: usize = 28; +const FV_LOAD_CLASS_KEYS: u32 = 0x12; +const FV_UNLOAD_CLASS_KEYS: u32 = 0x13; +const FV_SYSTEM_VOLUME_OPTION: u64 = 4; +const FV_MAX_VOLUME_MAPS: usize = 32; +const FV_STATE_UUID_KEY: &[u8] = b"kid"; + +pub(crate) struct VolumeMap { + apfs_uuid: [u8; 16], + bag_uuid: [u8; 16], + refs: u32, +} + +#[repr(C)] +struct KernelKey { + opaque: [u8; OPAQUE_KEY_LEN], + iv: [u8; IV_KEY_LEN], +} + +#[repr(C)] +struct KernelNewFileKey { + key: KernelKey, + wrapped_ekwk: [u8; FILE_KEY_MAX_LEN], + wrapped_ek: [u8; FILE_KEY_MAX_LEN], + wrapped_ekwk_len: usize, + wrapped_ek_len: usize, +} + +#[repr(C)] +struct KernelOps { + unwrap_media_key: + unsafe extern "C" fn(*mut c_void, *const u8, usize, u32, *mut KernelKey) -> c_int, + unwrap_volume_key: unsafe extern "C" fn( + *mut c_void, + *const u8, + usize, + *const u8, + usize, + *const u8, + usize, + *mut KernelKey, + ) -> c_int, + load_class_keys: unsafe extern "C" fn( + *mut c_void, + *const u8, + *const u8, + usize, + *const u8, + usize, + *const u8, + usize, + ) -> c_int, + unload_class_keys: unsafe extern "C" fn(*mut c_void, *const u8, *const u8, usize) -> c_int, + unwrap_file_key: unsafe extern "C" fn( + *mut c_void, + *const u8, + u32, + *const u8, + usize, + *const u8, + usize, + *mut KernelKey, + ) -> c_int, + new_file_key: unsafe extern "C" fn( + *mut c_void, + *const u8, + u32, + u64, + u16, + *mut KernelNewFileKey, + ) -> c_int, +} + +extern "C" { + fn sep_fv_register_v2(context: *mut c_void, ops: *const KernelOps) -> c_int; + fn sep_fv_unregister_v2(context: *mut c_void); +} + +static KERNEL_OPS: KernelOps = KernelOps { + unwrap_media_key: kernel_unwrap_media_key, + unwrap_volume_key: kernel_unwrap_volume_key, + load_class_keys: kernel_load_class_keys, + unload_class_keys: kernel_unload_class_keys, + unwrap_file_key: kernel_unwrap_file_key, + new_file_key: kernel_new_file_key, +}; + +const PFK_VOLUME_PARAMS_PREFIX: [u8; 12] = [ + 0x31, 0x1a, 0x30, 0x18, 0x0c, 0x04, b'v', b'u', b'i', b'd', 0x04, 0x10, +]; +const PFK_VOLUME_PARAMS_LEN: usize = PFK_VOLUME_PARAMS_PREFIX.len() + 16; + +const FV_PARAMS_DER: [u8; 35] = [ + 0x31, 0x21, 0x30, 0x0a, 0x0c, 0x02, b'k', b'c', 0x04, 0x04, 0, 0, 0, 0, 0x30, 0x13, 0x0c, 0x07, + b'o', b'p', b't', b'i', b'o', b'n', b's', 0x04, 0x08, 0, 0, 0, 0, 0, 0, 0, 0, +]; + +struct MediaKey { + opaque: [u8; OPAQUE_KEY_LEN], + iv_key: [u8; IV_KEY_LEN], +} + +struct VolumeKey { + opaque: [u8; OPAQUE_KEY_LEN], +} + +struct FileKey { + opaque: [u8; OPAQUE_KEY_LEN], + iv_key: [u8; IV_KEY_LEN], +} + +struct NewFileKey { + key: FileKey, + wrapped_ekwk: [u8; FILE_KEY_MAX_LEN], + wrapped_ek: [u8; FILE_KEY_MAX_LEN], + wrapped_ekwk_len: usize, + wrapped_ek_len: usize, +} + +impl Drop for NewFileKey { + fn drop(&mut self) { + image::wipe(&mut self.wrapped_ekwk); + image::wipe(&mut self.wrapped_ek); + } +} + +impl Drop for VolumeKey { + fn drop(&mut self) { + image::wipe(&mut self.opaque); + } +} + +impl Drop for MediaKey { + fn drop(&mut self) { + image::wipe(&mut self.opaque); + image::wipe(&mut self.iv_key); + } +} + +impl Drop for FileKey { + fn drop(&mut self) { + image::wipe(&mut self.opaque); + image::wipe(&mut self.iv_key); + } +} impl SepData { - const FV_LABEL: &'static [u8; 16] = b"AppleSEPvek00001"; - const FV_PKH: &'static [u8; 16] = b"AppleSEP-KEK-001"; - const FV_PARAM_LEN: usize = 0x130; + fn pfk_params(volume_uuid: &[u8; 16]) -> [u8; PFK_VOLUME_PARAMS_LEN] { + let mut params = [0u8; PFK_VOLUME_PARAMS_LEN]; + params[..PFK_VOLUME_PARAMS_PREFIX.len()].copy_from_slice(&PFK_VOLUME_PARAMS_PREFIX); + params[PFK_VOLUME_PARAMS_PREFIX.len()..].copy_from_slice(volume_uuid); + params + } + + fn pfk_class(protection_class: u32) -> Result { + match protection_class & 0x1f { + 1..=4 => Ok(protection_class), + 6 => Ok((protection_class & !0x1f) | 13), + 7 => Ok((protection_class & !0x1f) | 17), + _ => Err(EINVAL), + } + } + + fn fv_options(&self, volume_uuid: &[u8; 16]) -> u64 { + if self.xarm.lock().os_uuid == Some(*volume_uuid) { + FV_SYSTEM_VOLUME_OPTION + } else { + 0 + } + } + + fn fv_params(options: u64) -> [u8; FV_PARAMS_DER.len()] { + let mut params = FV_PARAMS_DER; + let options_at = params.len() - 8; + params[options_at..].copy_from_slice(&options.to_le_bytes()); + params + } + + fn fv_ready(&self) -> Result<()> { + if self.phase.load(Relaxed) != PHASE_READY { + return Err(EAGAIN); + } + if !self.sks_ready() { + return Err(ENODEV); + } + Ok(()) + } - fn fv_param(&self) -> Result> { - let mut p: KVec = KVec::new(); - p.resize(Self::FV_PARAM_LEN, 0u8, GFP_KERNEL)?; - p[0x10..0x20].copy_from_slice(Self::FV_LABEL); - Ok(p) + fn resolve_fv_uuid(&self, apfs_uuid: &[u8; 16]) -> [u8; 16] { + self.fv_volumes + .lock() + .iter() + .find(|entry| entry.apfs_uuid == *apfs_uuid) + .map_or(*apfs_uuid, |entry| entry.bag_uuid) } - fn sks_req_fv(&self, selector: u8, body: &image::Body) -> Result { - let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), body)?; + fn record_fv_volume(&self, apfs_uuid: &[u8; 16], bag_uuid: &[u8; 16]) -> Result<()> { + let mut volumes = self.fv_volumes.lock(); + if let Some(entry) = volumes + .iter_mut() + .find(|entry| entry.apfs_uuid == *apfs_uuid) + { + if entry.bag_uuid != *bag_uuid { + return Err(EINVAL); + } + entry.refs = entry.refs.checked_add(1).ok_or(EOVERFLOW)?; + return Ok(()); + } + if volumes.len() >= FV_MAX_VOLUME_MAPS { + return Err(ENOSPC); + } + volumes.push( + VolumeMap { + apfs_uuid: *apfs_uuid, + bag_uuid: *bag_uuid, + refs: 1, + }, + GFP_KERNEL, + )?; + Ok(()) + } + + fn unrecord_fv_volume(&self, apfs_uuid: &[u8; 16], bag_uuid: &[u8; 16]) -> Result<()> { + let mut volumes = self.fv_volumes.lock(); + let index = volumes + .iter() + .position(|entry| entry.apfs_uuid == *apfs_uuid) + .ok_or(ENOENT)?; + if volumes[index].bag_uuid != *bag_uuid { + return Err(EINVAL); + } + if volumes[index].refs > 1 { + volumes[index].refs -= 1; + } else { + volumes.swap_remove(index); + } + Ok(()) + } + + fn sks_req_fv_blob_state( + &self, + volume_uuid: &[u8; 16], + volume_key: &[u8], + ) -> Result { + let params = Self::fv_params(self.fv_options(volume_uuid)); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(SYSTEM_CLIENT)?; + body.put_blob(¶ms)?; + body.put_blob(volume_key)?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + Ok(SksRequest { + name: crate::sks::SKS_GET_BLOB_STATE_NAME, + msg: crate::sks::encode_sks_get_blob_state(self.sks_next_seq(), len), + img, + }) + } + + fn fv_blob_uuid(&self, volume_uuid: &[u8; 16], volume_key: &[u8]) -> Result<[u8; 16]> { + let out = self + .sks_send(self.sks_req_fv_blob_state(volume_uuid, volume_key)) + .ok_or(EIO)?; + if out.reply.status != 0 { + let status: i32 = out.reply.status.into(); + dev_err!(self.dev, "fv: GET_BLOB_STATE failed with status {}\n", status); + return Err(EACCES); + } + let body = self + .sks_report_response(crate::sks::SKS_GET_BLOB_STATE_NAME, &out) + .ok_or(EMSGSIZE)?; + let mut fields = proto::FieldCursor::new(body); + let version = fields.i32(); + if version != Some(0) { + dev_err!( + self.dev, + "fv: GET_BLOB_STATE response has version {:?}, {} body bytes\n", + version, + body.len() + ); + return Err(EMSGSIZE); + } + let state = match fields.blob() { + Some(state) => state, + None => { + dev_err!( + self.dev, + "fv: GET_BLOB_STATE has no complete state blob in {} body bytes\n", + body.len() + ); + return Err(EMSGSIZE); + } + }; + let uuid_tlv = match crate::der::refkey_find(state, FV_STATE_UUID_KEY) { + Some(uuid) => uuid, + None => { + let head0 = state + .get(..8) + .and_then(|bytes| bytes.try_into().ok()) + .map(u64::from_be_bytes) + .unwrap_or(0); + let head1 = state + .get(8..16) + .and_then(|bytes| bytes.try_into().ok()) + .map(u64::from_be_bytes) + .unwrap_or(0); + dev_err!( + self.dev, + "fv: GET_BLOB_STATE returned {} state bytes without a UUID (head {:016x}{:016x})\n", + state.len(), + head0, + head1 + ); + return Err(EMSGSIZE); + } + }; + let raw_uuid = crate::der::octet_string_body(uuid_tlv).ok_or(EMSGSIZE)?; + if raw_uuid.len() != 16 { + dev_err!( + self.dev, + "fv: GET_BLOB_STATE returned a {}-byte UUID\n", + raw_uuid.len() + ); + return Err(EMSGSIZE); + } + let mut bag_uuid = [0u8; 16]; + bag_uuid.copy_from_slice(raw_uuid); + Ok(bag_uuid) + } + + pub(crate) fn register_fv_kernel(&self) -> Result<()> { + let context = core::ptr::from_ref(self).cast_mut().cast::(); + // SAFETY: `KERNEL_OPS` is static and `remove()` unregisters this + // pointer before the driver's `Arc` can be dropped. + kernel::error::to_result(unsafe { sep_fv_register_v2(context, &KERNEL_OPS) }) + } + + pub(crate) fn unregister_fv_kernel(&self) { + let context = core::ptr::from_ref(self).cast_mut().cast::(); + // SAFETY: the pointer is the one passed to `sep_fv_register_v2`. + unsafe { sep_fv_unregister_v2(context) }; + } + + fn sks_req_unwrap_media_key_from_class( + &self, + wrapped: &[u8; WRAPPED_KEY_LEN], + protection_class: u32, + ) -> Result { + let mut body = image::Body::new(); + body.put_u32(1)?; + body.put_u64(SYSTEM_CLIENT)?; + body.put_blob(wrapped)?; + body.put_i32(NO_KEYBAG_HANDLE)?; + body.put_u32(protection_class)?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; - let msg = crate::sks::encode_sks_fv(selector, self.sks_next_seq(), len).ok_or(EINVAL)?; + let msg = + crate::sks::encode_sks_unwrap_media_key(self.sks_next_seq(), len).ok_or(EINVAL)?; Ok(SksRequest { - name: crate::sks::SKS_PERFORM_OP_NAME, + name: crate::sks::SKS_UNWRAP_MEDIA_KEY_NAME, msg, img, }) } - fn fv_send( + fn decode_media_key(&self, out: crate::SksOutcome) -> Result { + if out.reply.status != 0 { + let status: i32 = out.reply.status.into(); + dev_err!( + self.dev, + "fv: UNWRAP_MEDIA_KEY failed with status {}\n", + status + ); + return Err(EACCES); + } + let body = self + .sks_report_response(crate::sks::SKS_UNWRAP_MEDIA_KEY_NAME, &out) + .ok_or(EMSGSIZE)?; + let mut fields = proto::FieldCursor::new(body); + if fields.i32() != Some(1) { + return Err(EMSGSIZE); + } + let key = fields.blob().ok_or(EMSGSIZE)?; + let iv_key = fields.blob().ok_or(EMSGSIZE)?; + let flags = fields.i32().ok_or(EMSGSIZE)?; + + if key.len() != OPAQUE_KEY_LEN + || iv_key.len() != IV_KEY_LEN + || flags & WRAPPED_KEY_FLAG == 0 + { + return Err(EMSGSIZE); + } + + let mut opaque = [0; OPAQUE_KEY_LEN]; + opaque.copy_from_slice(key); + let mut iv = [0; IV_KEY_LEN]; + iv.copy_from_slice(iv_key); + Ok(MediaKey { opaque, iv_key: iv }) + } + + fn unwrap_media_key_from_class( &self, - selector: u8, - body: image::Body, - _what: &str, - ) -> Option<(i32, Option, Option>)> { - let out = self.sks_send(self.sks_req_fv(selector, &body))?; - let mailbox: i32 = out.reply.status.into(); - let (mut opst, mut blob) = (None, None); - if mailbox == 0 { - if let Some(rbody) = self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &out) { - let mut f = proto::FieldCursor::new(rbody); - opst = f.i32(); - if let Some(b) = f.blob() { - let mut owned: KVec = KVec::new(); - owned.extend_from_slice(b, GFP_KERNEL).ok()?; - blob = Some(owned); - } + wrapped: &[u8; WRAPPED_KEY_LEN], + protection_class: u32, + ) -> Result { + self.fv_ready()?; + let out = self + .sks_send(self.sks_req_unwrap_media_key_from_class(wrapped, protection_class)) + .ok_or(EIO)?; + self.decode_media_key(out) + } + + fn sks_req_unwrap_vek(&self, secret: &[u8], kek: &[u8], vek: &[u8]) -> Result { + let mut body = image::Body::new(); + body.put_u32(FV_UNWRAP_VERSION)?; + body.put_u64(SYSTEM_CLIENT)?; + body.put_blob(&FV_PARAMS_DER)?; + body.put_u32(FV_UNWRAP_OPTIONS)?; + body.put_blob(secret)?; + body.put_blob(kek)?; + body.put_blob(vek)?; + body.put_blob(&[])?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_unwrap_vek(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_UNWRAP_VEK_NAME, + msg, + img, + }) + } + + fn unwrap_vek(&self, secret: &[u8], kek: &[u8], vek: &[u8]) -> Result { + self.fv_ready()?; + let out = self + .sks_send(self.sks_req_unwrap_vek(secret, kek, vek)) + .ok_or(EIO)?; + if out.reply.status != 0 { + let status: i32 = out.reply.status.into(); + dev_err!(self.dev, "fv: UNWRAP_VEK failed with status {}\n", status); + return Err(EACCES); + } + let body = self + .sks_report_response(crate::sks::SKS_UNWRAP_VEK_NAME, &out) + .ok_or(EMSGSIZE)?; + let mut fields = proto::FieldCursor::new(body); + if fields.i32() != Some(FV_UNWRAP_VERSION as i32) { + return Err(EMSGSIZE); + } + let key = fields.blob().ok_or(EMSGSIZE)?; + let flags = fields.i32().ok_or(EMSGSIZE)?; + if key.len() != OPAQUE_KEY_LEN || flags & VOLUME_KEY_FLAG == 0 { + return Err(EMSGSIZE); + } + + let mut opaque = [0; OPAQUE_KEY_LEN]; + opaque.copy_from_slice(key); + Ok(VolumeKey { opaque }) + } + + fn sks_req_unwrap_file_key( + &self, + volume_uuid: &[u8; 16], + protection_class: u32, + wrapped_ekwk: &[u8], + wrapped_ek: &[u8], + ) -> Result { + let class = Self::pfk_class(protection_class)?; + let bag_uuid = self.resolve_fv_uuid(volume_uuid); + let params = Self::pfk_params(&bag_uuid); + + let mut body = image::Body::new(); + body.put_u32(PFK_UNWRAP_VERSION)?; + body.put_u64(SYSTEM_CLIENT)?; + body.put_i32(PFK_SYSTEM_VOLUME_HANDLE)?; + body.put_blob(¶ms)?; + body.put_u32(class)?; + body.put_blob(wrapped_ek)?; + body.put_blob(wrapped_ekwk)?; + body.put_blob(&[])?; + body.put_u32(PFK_UNWRAP_OPTIONS)?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + Ok(SksRequest { + name: crate::sks::SKS_UNWRAP_PFK_NAME, + msg: crate::sks::encode_sks_unwrap_pfk(self.sks_next_seq(), len), + img, + }) + } + + fn unwrap_file_key( + &self, + volume_uuid: &[u8; 16], + protection_class: u32, + wrapped_ekwk: &[u8], + wrapped_ek: &[u8], + ) -> Result { + self.fv_ready()?; + let out = self + .sks_send(self.sks_req_unwrap_file_key( + volume_uuid, + protection_class, + wrapped_ekwk, + wrapped_ek, + )) + .ok_or(EIO)?; + if out.reply.status != 0 { + let status: i32 = out.reply.status.into(); + dev_err!( + self.dev, + "fv: UNWRAP_PFK v2 class {} ekwk {} ek {} failed with status {}\n", + protection_class, + wrapped_ekwk.len(), + wrapped_ek.len(), + status + ); + return Err(EACCES); + } + let body = self + .sks_report_response(crate::sks::SKS_UNWRAP_PFK_NAME, &out) + .ok_or(EMSGSIZE)?; + let mut fields = proto::FieldCursor::new(body); + if fields.i32() != Some(PFK_UNWRAP_VERSION as i32) { + return Err(EMSGSIZE); + } + let key = fields.blob().ok_or(EMSGSIZE)?; + let iv_key = fields.blob().ok_or(EMSGSIZE)?; + let _ephemeral_key = fields.blob().ok_or(EMSGSIZE)?; + let flags = fields.i32().ok_or(EMSGSIZE)?; + if key.len() != OPAQUE_KEY_LEN || iv_key.len() != IV_KEY_LEN || flags & PFK_KEY_FLAG == 0 { + return Err(EMSGSIZE); + } + let mut opaque = [0; OPAQUE_KEY_LEN]; + opaque.copy_from_slice(key); + let mut iv = [0; IV_KEY_LEN]; + iv.copy_from_slice(iv_key); + Ok(FileKey { opaque, iv_key: iv }) + } + + fn sks_req_new_file_key( + &self, + volume_uuid: &[u8; 16], + protection_class: u32, + ) -> Result { + let class = Self::pfk_class(protection_class)?; + let bag_uuid = self.resolve_fv_uuid(volume_uuid); + let params = Self::pfk_params(&bag_uuid); + let mut context = [0u8; PFK_FS_CONTEXT_LEN]; + context[10..26].copy_from_slice(&bag_uuid); + let mut body = image::Body::new(); + body.put_u32(PFK_NEW_VERSION)?; + body.put_u64(SYSTEM_CLIENT)?; + body.put_i32(PFK_SYSTEM_VOLUME_HANDLE)?; + body.put_blob(¶ms)?; + body.put_u32(class)?; + body.put_u32(PFK_NEW_OPTIONS)?; + body.put_blob(&context)?; + body.put_blob(&[])?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + Ok(SksRequest { + name: crate::sks::SKS_NEW_PFK_NAME, + msg: crate::sks::encode_sks_new_pfk(self.sks_next_seq(), len), + img, + }) + } + + fn new_file_key( + &self, + volume_uuid: &[u8; 16], + protection_class: u32, + ) -> Result { + self.fv_ready()?; + let class = Self::pfk_class(protection_class)?; + let out = self + .sks_send(self.sks_req_new_file_key(volume_uuid, protection_class)) + .ok_or(EIO)?; + if out.reply.status != 0 { + let status: i32 = out.reply.status.into(); + dev_err!( + self.dev, + "fv: NEW_PFK class {} failed with status {}\n", + protection_class, + status + ); + return Err(EACCES); + } + let body = self + .sks_report_response(crate::sks::SKS_NEW_PFK_NAME, &out) + .ok_or(EMSGSIZE)?; + let mut fields = proto::FieldCursor::new(body); + if fields.i32() != Some(PFK_NEW_VERSION as i32) { + return Err(EMSGSIZE); + } + let opaque = fields.blob().ok_or(EMSGSIZE)?; + let iv = fields.blob().ok_or(EMSGSIZE)?; + let ephemeral = fields.blob().ok_or(EMSGSIZE)?; + let wrapped_ek = fields.blob().ok_or(EMSGSIZE)?; + let wrapped_ekwk = fields.blob().ok_or(EMSGSIZE)?; + let key_flags = fields.i32().ok_or(EMSGSIZE)?; + let wrapped_class = fields.i32().ok_or(EMSGSIZE)?; + if opaque.len() != OPAQUE_KEY_LEN + || iv.len() != IV_KEY_LEN + || ephemeral.is_empty() + || ephemeral.len() > EPHEMERAL_KEY_MAX_LEN + || wrapped_ek.is_empty() + || wrapped_ek.len() > FILE_KEY_MAX_LEN + || wrapped_ekwk.is_empty() + || wrapped_ekwk.len() > FILE_KEY_MAX_LEN + || key_flags & PFK_NEW_KEY_FLAG == 0 + || wrapped_class != class as i32 + { + dev_err!( + self.dev, + "fv: NEW_PFK malformed response: key {} iv {} eph {} ek {} ekwk {} flags {:#x} class {} expected {}\n", + opaque.len(), + iv.len(), + ephemeral.len(), + wrapped_ek.len(), + wrapped_ekwk.len(), + key_flags, + wrapped_class, + class + ); + return Err(EMSGSIZE); + } + + let mut key = FileKey { + opaque: [0; OPAQUE_KEY_LEN], + iv_key: [0; IV_KEY_LEN], + }; + key.opaque.copy_from_slice(opaque); + key.iv_key.copy_from_slice(iv); + let mut result = NewFileKey { + key, + wrapped_ekwk: [0; FILE_KEY_MAX_LEN], + wrapped_ek: [0; FILE_KEY_MAX_LEN], + wrapped_ekwk_len: wrapped_ekwk.len(), + wrapped_ek_len: wrapped_ek.len(), + }; + result.wrapped_ekwk[..wrapped_ekwk.len()].copy_from_slice(wrapped_ekwk); + result.wrapped_ek[..wrapped_ek.len()].copy_from_slice(wrapped_ek); + Ok(result) + } + + fn sks_req_load_class_keys( + &self, + volume_uuid: &[u8; 16], + secret: &[u8], + unlock_record: &[u8], + volume_key: &[u8], + ) -> Result { + let options = self.fv_options(volume_uuid); + let params = Self::fv_params(options); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(SYSTEM_CLIENT)?; + body.put_blob(¶ms)?; + body.put_u32(FV_LOAD_CLASS_KEYS)?; + body.put_u64(options)?; + body.put_blob(secret)?; + body.put_blob(unlock_record)?; + body.put_blob(volume_key)?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + Ok(SksRequest { + name: crate::sks::SKS_SET_PROTECTION_NAME, + msg: crate::sks::encode_sks_set_protection(self.sks_next_seq(), len), + img, + }) + } + + fn load_class_keys( + &self, + volume_uuid: &[u8; 16], + secret: &[u8], + unlock_record: &[u8], + volume_key: &[u8], + ) -> Result<()> { + self.fv_ready()?; + let bag_uuid = self.fv_blob_uuid(volume_uuid, volume_key)?; + let request = self.sks_req_load_class_keys( + volume_uuid, + secret, + unlock_record, + volume_key, + )?; + self.record_fv_volume(volume_uuid, &bag_uuid)?; + let out = match self.sks_send(Ok(request)) { + Some(out) => out, + None => { + let _ = self.unrecord_fv_volume(volume_uuid, &bag_uuid); + return Err(EIO); } + }; + if out.reply.status != 0 { + let status: i32 = out.reply.status.into(); + dev_err!( + self.dev, + "fv: LOAD_CLASS_KEYS failed with status {}\n", + status + ); + let _ = self.unrecord_fv_volume(volume_uuid, &bag_uuid); + return Err(EACCES); } - Some((mailbox, opst, blob)) - } - - /// `0x42` mint the device-bound key-encryption key. - fn fv_new_kek(&self, handle: u64, param: &[u8]) -> Option> { - let mut b = image::Body::new(); - b.put_u32(0).ok()?; - b.put_u64(handle).ok()?; - b.put_blob(param).ok()?; - b.put_u32(0).ok()?; - b.put_blob(&[]).ok()?; - b.put_blob(Self::FV_PKH).ok()?; - let (mb, op, blob) = self.fv_send(crate::sks::OP_SKS_FV_NEW_KEK, b, "new_kek")?; - (mb == 0 && op == Some(0)).then_some(())?; - blob - } - - /// `0x40` mint the wrapped, device-bound volume key. - fn fv_new_vek(&self, handle: u64, param: &[u8]) -> Option> { - let mut b = image::Body::new(); - b.put_u32(0).ok()?; - b.put_u64(handle).ok()?; - b.put_blob(param).ok()?; - b.put_blob(&[]).ok()?; - b.put_blob(&[]).ok()?; - b.put_blob(Self::FV_PKH).ok()?; - let (mb, op, blob) = self.fv_send(crate::sks::OP_SKS_FV_NEW_VEK, b, "new_vek")?; - (mb == 0 && op == Some(0)).then_some(())?; - blob - } - - /// `0x41` install the volume key into our collection (empty KEK slot = self-derive). - fn fv_unwrap_vek(&self, handle: u64, param: &[u8], wrapped_vek: &[u8]) -> Option> { - let mut b = image::Body::new(); - b.put_u32(0).ok()?; - b.put_u64(handle).ok()?; - b.put_blob(param).ok()?; - b.put_u32(0).ok()?; - b.put_blob(&[]).ok()?; - b.put_blob(&[]).ok()?; - b.put_blob(wrapped_vek).ok()?; - b.put_blob(&[]).ok()?; - let (mb, op, blob) = self.fv_send(crate::sks::OP_SKS_FV_UNWRAP_VEK, b, "unwrap_vek")?; - (mb == 0 && op == Some(0)).then_some(())?; - blob.or_else(|| Some(KVec::new())) - } - - fn fv_dump(path: &CStr, data: &[u8]) { - if let Ok(f) = shim::StoreFile::open(path) { - let _ = f.write_all(0, data); - let _ = f.sync(); - } - } - - fn fv_load(path: &CStr) -> Option> { - let f = shim::StoreFile::open_readonly(path).ok()?; - let sz = f.size().unwrap_or(0); - if !(1..=8192).contains(&sz) { - return None; - } - let mut buf: KVec = KVec::new(); - buf.resize(sz as usize, 0u8, GFP_KERNEL).ok()?; - f.read_exact(0, &mut buf).ok()?; - Some(buf) - } - - /// Provisions the device-bound FileVault key hierarchy and installs the volume - /// key. The destructive clear (`0x47`) is never emitted; the installed VEK is - /// consumed by the storage inline-AES engine (ANS), so on its own this seals - /// nothing on Linux. - pub(crate) fn fv_provision(&self, handle: crate::sks::KeyBagHandle, secret: &[u8]) -> Option { - const KEK_PATH: &CStr = c"/var/lib/apple-sep-fv-kek.bin"; - const VEK_PATH: &CStr = c"/var/lib/apple-sep-fv-vek.bin"; - - if let Some(healthy) = self.sks_health_check(c"fv provision") { - let _ = self.sks_send(self.sks_req_change_lock_state( - handle, - LockState::Unlocked, - secret, - healthy, - )); - } - let param = self.fv_param().ok()?; - let h = handle.value() as u64; - - let (_kek, vek) = match (Self::fv_load(KEK_PATH), Self::fv_load(VEK_PATH)) { - (Some(kek), Some(vek)) => (kek, vek), - _ => { - let kek = self.fv_new_kek(h, ¶m)?; - let vek = self.fv_new_vek(h, ¶m)?; - Self::fv_dump(KEK_PATH, &kek); - Self::fv_dump(VEK_PATH, &vek); - (kek, vek) + let body = match self.sks_report_response(crate::sks::SKS_SET_PROTECTION_NAME, &out) { + Some(body) => body, + None => { + self.rollback_class_keys(volume_uuid, volume_key); + let _ = self.unrecord_fv_volume(volume_uuid, &bag_uuid); + return Err(EMSGSIZE); } }; + let mut fields = proto::FieldCursor::new(body); + if fields.i32() != Some(0) || fields.blob().is_none() { + self.rollback_class_keys(volume_uuid, volume_key); + let _ = self.unrecord_fv_volume(volume_uuid, &bag_uuid); + return Err(EMSGSIZE); + } + let apfs_hi = u64::from_be_bytes(volume_uuid[..8].try_into().unwrap()); + let apfs_lo = u64::from_be_bytes(volume_uuid[8..].try_into().unwrap()); + let bag_hi = u64::from_be_bytes(bag_uuid[..8].try_into().unwrap()); + let bag_lo = u64::from_be_bytes(bag_uuid[8..].try_into().unwrap()); + dev_info!( + self.dev, + "fv: mapped APFS UUID {:016x}-{:016x} to keybag UUID {:016x}-{:016x}\n", + apfs_hi, + apfs_lo, + bag_hi, + bag_lo + ); + Ok(()) + } - let installed = self.fv_unwrap_vek(h, ¶m, &vek)?; - let vek_handle = if installed.len() >= 4 { - u64::from(u32::from_le_bytes([installed[0], installed[1], installed[2], installed[3]])) - } else { - 0 - }; - Some(vek_handle) + fn rollback_class_keys(&self, volume_uuid: &[u8; 16], volume_key: &[u8]) { + if let Ok(request) = self.sks_req_unload_class_keys(volume_uuid, volume_key) { + let _ = self.sks_send(Ok(request)); + } + } + + fn sks_req_unload_class_keys( + &self, + volume_uuid: &[u8; 16], + volume_key: &[u8], + ) -> Result { + let options = self.fv_options(volume_uuid); + let params = Self::fv_params(options); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(SYSTEM_CLIENT)?; + body.put_blob(¶ms)?; + body.put_u32(FV_UNLOAD_CLASS_KEYS)?; + body.put_u64(options)?; + body.put_blob(&[])?; + body.put_blob(&[])?; + body.put_blob(volume_key)?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + Ok(SksRequest { + name: crate::sks::SKS_SET_PROTECTION_NAME, + msg: crate::sks::encode_sks_set_protection(self.sks_next_seq(), len), + img, + }) + } + + fn unload_class_keys(&self, volume_uuid: &[u8; 16], volume_key: &[u8]) -> Result<()> { + self.fv_ready()?; + let bag_uuid = self.fv_blob_uuid(volume_uuid, volume_key)?; + let out = self + .sks_send(self.sks_req_unload_class_keys(volume_uuid, volume_key)) + .ok_or(EIO)?; + if out.reply.status != 0 { + return Err(EACCES); + } + let body = self + .sks_report_response(crate::sks::SKS_SET_PROTECTION_NAME, &out) + .ok_or(EMSGSIZE)?; + let mut fields = proto::FieldCursor::new(body); + if fields.i32() != Some(0) || fields.blob().is_none() { + return Err(EMSGSIZE); + } + self.unrecord_fv_volume(volume_uuid, &bag_uuid) + } +} + +unsafe fn input<'a>(ptr: *const u8, len: usize, max: usize) -> Result<&'a [u8]> { + if ptr.is_null() || len == 0 || len > max { + return Err(EINVAL); + } + // SAFETY: the C API requires `ptr` to remain readable for `len` bytes for + // the duration of the callback, and the callback does not retain it. + Ok(unsafe { core::slice::from_raw_parts(ptr, len) }) +} + +unsafe fn optional_input<'a>(ptr: *const u8, len: usize, max: usize) -> Result<&'a [u8]> { + if len == 0 { + return Ok(&[]); + } + unsafe { input(ptr, len, max) } +} + +unsafe fn output<'a>(ptr: *mut KernelKey) -> Result<&'a mut KernelKey> { + if ptr.is_null() { + return Err(EINVAL); + } + // SAFETY: the C API provides exclusive writable storage for one key. + Ok(unsafe { &mut *ptr }) +} + +unsafe fn new_file_output<'a>(ptr: *mut KernelNewFileKey) -> Result<&'a mut KernelNewFileKey> { + if ptr.is_null() { + return Err(EINVAL); + } + // SAFETY: the C API provides exclusive writable storage for one result. + Ok(unsafe { &mut *ptr }) +} + +unsafe fn sep<'a>(context: *mut c_void) -> Result<&'a SepData> { + if context.is_null() { + return Err(ENODEV); } + // SAFETY: registration keeps `SepData` alive until all callbacks finish. + Ok(unsafe { &*context.cast::() }) +} + +unsafe extern "C" fn kernel_unwrap_media_key( + context: *mut c_void, + wrapped: *const u8, + wrapped_len: usize, + protection_class: u32, + key: *mut KernelKey, +) -> c_int { + let result: Result<()> = (|| { + let this = unsafe { sep(context)? }; + let wrapped = unsafe { input(wrapped, wrapped_len, WRAPPED_KEY_LEN)? }; + let wrapped: &[u8; WRAPPED_KEY_LEN] = wrapped.try_into().map_err(|_| EINVAL)?; + let key = unsafe { output(key)? }; + let unwrapped = this.unwrap_media_key_from_class(wrapped, protection_class)?; + key.opaque.copy_from_slice(&unwrapped.opaque); + key.iv.copy_from_slice(&unwrapped.iv_key); + Ok(()) + })(); + result.map_or_else(|error| error.to_errno(), |_| 0) +} + +unsafe extern "C" fn kernel_unwrap_volume_key( + context: *mut c_void, + secret: *const u8, + secret_len: usize, + unlock_record: *const u8, + unlock_record_len: usize, + volume_key: *const u8, + volume_key_len: usize, + key: *mut KernelKey, +) -> c_int { + let result: Result<()> = (|| { + let this = unsafe { sep(context)? }; + let secret = unsafe { optional_input(secret, secret_len, SECRET_MAX_LEN)? }; + let unlock_record = + unsafe { optional_input(unlock_record, unlock_record_len, RECORD_MAX_LEN)? }; + let volume_key = unsafe { input(volume_key, volume_key_len, RECORD_MAX_LEN)? }; + let key = unsafe { output(key)? }; + let unwrapped = this.unwrap_vek(secret, unlock_record, volume_key)?; + key.opaque.copy_from_slice(&unwrapped.opaque); + key.iv.fill(0); + Ok(()) + })(); + result.map_or_else(|error| error.to_errno(), |_| 0) +} + +unsafe extern "C" fn kernel_load_class_keys( + context: *mut c_void, + volume_uuid: *const u8, + secret: *const u8, + secret_len: usize, + unlock_record: *const u8, + unlock_record_len: usize, + volume_key: *const u8, + volume_key_len: usize, +) -> c_int { + let result: Result<()> = (|| { + let this = unsafe { sep(context)? }; + let volume_uuid = unsafe { input(volume_uuid, 16, 16)? }; + let volume_uuid: &[u8; 16] = volume_uuid.try_into().map_err(|_| EINVAL)?; + let secret = unsafe { optional_input(secret, secret_len, SECRET_MAX_LEN)? }; + let unlock_record = + unsafe { optional_input(unlock_record, unlock_record_len, RECORD_MAX_LEN)? }; + let volume_key = unsafe { input(volume_key, volume_key_len, RECORD_MAX_LEN)? }; + this.load_class_keys(volume_uuid, secret, unlock_record, volume_key) + })(); + result.map_or_else(|error| error.to_errno(), |_| 0) +} + +unsafe extern "C" fn kernel_unload_class_keys( + context: *mut c_void, + volume_uuid: *const u8, + volume_key: *const u8, + volume_key_len: usize, +) -> c_int { + let result: Result<()> = (|| { + let this = unsafe { sep(context)? }; + let volume_uuid = unsafe { input(volume_uuid, 16, 16)? }; + let volume_uuid: &[u8; 16] = volume_uuid.try_into().map_err(|_| EINVAL)?; + let volume_key = unsafe { input(volume_key, volume_key_len, RECORD_MAX_LEN)? }; + this.unload_class_keys(volume_uuid, volume_key) + })(); + result.map_or_else(|error| error.to_errno(), |_| 0) +} + +unsafe extern "C" fn kernel_unwrap_file_key( + context: *mut c_void, + volume_uuid: *const u8, + protection_class: u32, + wrapped_ekwk: *const u8, + wrapped_ekwk_len: usize, + wrapped_ek: *const u8, + wrapped_ek_len: usize, + key: *mut KernelKey, +) -> c_int { + let result: Result<()> = (|| { + let this = unsafe { sep(context)? }; + let volume_uuid = unsafe { input(volume_uuid, 16, 16)? }; + let volume_uuid: &[u8; 16] = volume_uuid.try_into().map_err(|_| EINVAL)?; + let wrapped_ekwk = unsafe { input(wrapped_ekwk, wrapped_ekwk_len, FILE_KEY_MAX_LEN)? }; + let wrapped_ek = unsafe { input(wrapped_ek, wrapped_ek_len, FILE_KEY_MAX_LEN)? }; + let key = unsafe { output(key)? }; + let unwrapped = + this.unwrap_file_key(volume_uuid, protection_class, wrapped_ekwk, wrapped_ek)?; + key.opaque.copy_from_slice(&unwrapped.opaque); + key.iv.copy_from_slice(&unwrapped.iv_key); + Ok(()) + })(); + result.map_or_else(|error| error.to_errno(), |_| 0) +} + +unsafe extern "C" fn kernel_new_file_key( + context: *mut c_void, + volume_uuid: *const u8, + protection_class: u32, + crypto_id: u64, + key_revision: u16, + key: *mut KernelNewFileKey, +) -> c_int { + let result: Result<()> = (|| { + let this = unsafe { sep(context)? }; + let volume_uuid = unsafe { input(volume_uuid, 16, 16)? }; + let volume_uuid: &[u8; 16] = volume_uuid.try_into().map_err(|_| EINVAL)?; + let key = unsafe { new_file_output(key)? }; + if crypto_id == 0 || key_revision == 0 { + return Err(EINVAL); + } + let generated = this.new_file_key(volume_uuid, protection_class)?; + key.key.opaque.copy_from_slice(&generated.key.opaque); + key.key.iv.copy_from_slice(&generated.key.iv_key); + key.wrapped_ekwk.copy_from_slice(&generated.wrapped_ekwk); + key.wrapped_ek.copy_from_slice(&generated.wrapped_ek); + key.wrapped_ekwk_len = generated.wrapped_ekwk_len; + key.wrapped_ek_len = generated.wrapped_ek_len; + Ok(()) + })(); + result.map_or_else(|error| error.to_errno(), |_| 0) } diff --git a/drivers/soc/apple/fv_shim.c b/drivers/soc/apple/fv_shim.c new file mode 100644 index 0000000000000..566852005c2f8 --- /dev/null +++ b/drivers/soc/apple/fv_shim.c @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include +#include + +#include "shim.h" + +static DECLARE_RWSEM(apple_sep_fv_registration_lock); +static DEFINE_MUTEX(apple_sep_fv_call_lock); +static struct sep_fv_ops apple_sep_fv_ops; +static void *apple_sep_fv_context; + +int sep_fv_register_v2(void *context, const struct sep_fv_ops *ops) +{ + int ret = 0; + + if (!context || !ops || !ops->unwrap_media_key || + !ops->unwrap_volume_key || !ops->load_class_keys || + !ops->unload_class_keys || !ops->unwrap_file_key || + !ops->new_file_key) + return -EINVAL; + + down_write(&apple_sep_fv_registration_lock); + if (apple_sep_fv_context) + ret = -EBUSY; + else { + apple_sep_fv_ops = *ops; + apple_sep_fv_context = context; + } + up_write(&apple_sep_fv_registration_lock); + return ret; +} +EXPORT_SYMBOL_GPL(sep_fv_register_v2); + +void sep_fv_unregister_v2(void *context) +{ + down_write(&apple_sep_fv_registration_lock); + if (apple_sep_fv_context == context) { + apple_sep_fv_context = NULL; + memzero_explicit(&apple_sep_fv_ops, sizeof(apple_sep_fv_ops)); + } + up_write(&apple_sep_fv_registration_lock); +} +EXPORT_SYMBOL_GPL(sep_fv_unregister_v2); + +static int apple_sep_fv_call(int (*call)(const struct sep_fv_ops *, void *), + void *argument) +{ + int ret; + + down_read(&apple_sep_fv_registration_lock); + if (!apple_sep_fv_context) { + ret = -ENODEV; + goto out; + } + mutex_lock(&apple_sep_fv_call_lock); + ret = call(&apple_sep_fv_ops, argument); + mutex_unlock(&apple_sep_fv_call_lock); +out: + up_read(&apple_sep_fv_registration_lock); + return ret; +} + +struct media_key_call { + const u8 *wrapped; + size_t wrapped_len; + u32 protection_class; + struct apple_sep_fv_key *key; +}; + +static int call_unwrap_media_key(const struct sep_fv_ops *ops, void *argument) +{ + struct media_key_call *call = argument; + + return ops->unwrap_media_key(apple_sep_fv_context, call->wrapped, + call->wrapped_len, call->protection_class, + call->key); +} + +int apple_sep_fv_unwrap_media_key(const u8 *wrapped, size_t wrapped_len, + u32 protection_class, + struct apple_sep_fv_key *key) +{ + struct media_key_call call = { + .wrapped = wrapped, + .wrapped_len = wrapped_len, + .protection_class = protection_class, + .key = key, + }; + int ret; + + if (!wrapped || !wrapped_len || !key) + return -EINVAL; + memzero_explicit(key, sizeof(*key)); + ret = apple_sep_fv_call(call_unwrap_media_key, &call); + if (ret) + memzero_explicit(key, sizeof(*key)); + return ret; +} +EXPORT_SYMBOL_GPL(apple_sep_fv_unwrap_media_key); + +struct volume_key_call { + const u8 *volume_uuid; + const u8 *secret; + size_t secret_len; + const u8 *unlock_record; + size_t unlock_record_len; + const u8 *volume_key; + size_t volume_key_len; + struct apple_sep_fv_key *key; +}; + +static int call_unwrap_volume_key(const struct sep_fv_ops *ops, void *argument) +{ + struct volume_key_call *call = argument; + + return ops->unwrap_volume_key(apple_sep_fv_context, call->secret, + call->secret_len, call->unlock_record, + call->unlock_record_len, call->volume_key, + call->volume_key_len, call->key); +} + +int apple_sep_fv_unwrap_volume_key(const u8 *secret, size_t secret_len, + const u8 *unlock_record, + size_t unlock_record_len, + const u8 *volume_key, + size_t volume_key_len, + struct apple_sep_fv_key *key) +{ + struct volume_key_call call = { + .secret = secret, + .secret_len = secret_len, + .unlock_record = unlock_record, + .unlock_record_len = unlock_record_len, + .volume_key = volume_key, + .volume_key_len = volume_key_len, + .key = key, + }; + int ret; + + if ((!secret && secret_len) || (secret && !secret_len) || + (!unlock_record && unlock_record_len) || + (unlock_record && !unlock_record_len) || + !volume_key || !volume_key_len || !key) + return -EINVAL; + memzero_explicit(key, sizeof(*key)); + ret = apple_sep_fv_call(call_unwrap_volume_key, &call); + if (ret) + memzero_explicit(key, sizeof(*key)); + return ret; +} +EXPORT_SYMBOL_GPL(apple_sep_fv_unwrap_volume_key); + +static int call_load_class_keys(const struct sep_fv_ops *ops, void *argument) +{ + struct volume_key_call *call = argument; + + return ops->load_class_keys(apple_sep_fv_context, call->volume_uuid, + call->secret, + call->secret_len, call->unlock_record, + call->unlock_record_len, call->volume_key, + call->volume_key_len); +} + +int apple_sep_fv_load_class_keys_v2(const u8 volume_uuid[16], + const u8 *secret, size_t secret_len, + const u8 *unlock_record, + size_t unlock_record_len, + const u8 *volume_key, + size_t volume_key_len) +{ + struct volume_key_call call = { + .volume_uuid = volume_uuid, + .secret = secret, + .secret_len = secret_len, + .unlock_record = unlock_record, + .unlock_record_len = unlock_record_len, + .volume_key = volume_key, + .volume_key_len = volume_key_len, + }; + + if (!volume_uuid || (!secret && secret_len) || (secret && !secret_len) || + (!unlock_record && unlock_record_len) || + (unlock_record && !unlock_record_len) || + !volume_key || !volume_key_len) + return -EINVAL; + return apple_sep_fv_call(call_load_class_keys, &call); +} +EXPORT_SYMBOL_GPL(apple_sep_fv_load_class_keys_v2); + +struct unload_class_keys_call { + const u8 *volume_uuid; + const u8 *volume_key; + size_t volume_key_len; +}; + +static int call_unload_class_keys(const struct sep_fv_ops *ops, + void *argument) +{ + struct unload_class_keys_call *call = argument; + + return ops->unload_class_keys(apple_sep_fv_context, call->volume_uuid, + call->volume_key, + call->volume_key_len); +} + +int apple_sep_fv_unload_class_keys_v2(const u8 volume_uuid[16], + const u8 *volume_key, + size_t volume_key_len) +{ + struct unload_class_keys_call call = { + .volume_uuid = volume_uuid, + .volume_key = volume_key, + .volume_key_len = volume_key_len, + }; + + if (!volume_uuid || !volume_key || !volume_key_len) + return -EINVAL; + return apple_sep_fv_call(call_unload_class_keys, &call); +} +EXPORT_SYMBOL_GPL(apple_sep_fv_unload_class_keys_v2); + +struct file_key_call { + const u8 *volume_uuid; + u32 protection_class; + const u8 *wrapped_ekwk; + size_t wrapped_ekwk_len; + const u8 *wrapped_ek; + size_t wrapped_ek_len; + struct apple_sep_fv_key *key; +}; + +static int call_unwrap_file_key(const struct sep_fv_ops *ops, void *argument) +{ + struct file_key_call *call = argument; + + return ops->unwrap_file_key(apple_sep_fv_context, call->volume_uuid, + call->protection_class, call->wrapped_ekwk, + call->wrapped_ekwk_len, call->wrapped_ek, + call->wrapped_ek_len, call->key); +} + +int apple_sep_fv_unwrap_file_key(const u8 volume_uuid[16], + u32 protection_class, + const u8 *wrapped_ekwk, + size_t wrapped_ekwk_len, + const u8 *wrapped_ek, + size_t wrapped_ek_len, + struct apple_sep_fv_key *key) +{ + struct file_key_call call = { + .volume_uuid = volume_uuid, + .protection_class = protection_class, + .wrapped_ekwk = wrapped_ekwk, + .wrapped_ekwk_len = wrapped_ekwk_len, + .wrapped_ek = wrapped_ek, + .wrapped_ek_len = wrapped_ek_len, + .key = key, + }; + int ret; + + if (!volume_uuid || !wrapped_ekwk || !wrapped_ekwk_len || !wrapped_ek || + !wrapped_ek_len || !key) + return -EINVAL; + memzero_explicit(key, sizeof(*key)); + ret = apple_sep_fv_call(call_unwrap_file_key, &call); + if (ret) + memzero_explicit(key, sizeof(*key)); + return ret; +} +EXPORT_SYMBOL_GPL(apple_sep_fv_unwrap_file_key); + +struct new_file_key_call { + const u8 *volume_uuid; + u32 protection_class; + u64 crypto_id; + u16 key_revision; + struct apple_sep_fv_new_file_key *key; +}; + +static int call_new_file_key(const struct sep_fv_ops *ops, void *argument) +{ + struct new_file_key_call *call = argument; + + return ops->new_file_key(apple_sep_fv_context, call->volume_uuid, + call->protection_class, call->crypto_id, + call->key_revision, call->key); +} + +int apple_sep_fv_new_file_key_v2(const u8 volume_uuid[16], + u32 protection_class, + u64 crypto_id, u16 key_revision, + struct apple_sep_fv_new_file_key *key) +{ + struct new_file_key_call call = { + .volume_uuid = volume_uuid, + .protection_class = protection_class, + .crypto_id = crypto_id, + .key_revision = key_revision, + .key = key, + }; + int ret; + + if (!volume_uuid || !crypto_id || !key_revision || !key) + return -EINVAL; + memzero_explicit(key, sizeof(*key)); + ret = apple_sep_fv_call(call_new_file_key, &call); + if (ret) + memzero_explicit(key, sizeof(*key)); + return ret; +} +EXPORT_SYMBOL_GPL(apple_sep_fv_new_file_key_v2); diff --git a/drivers/soc/apple/image.rs b/drivers/soc/apple/image.rs index b81f8130fe602..8d4889268179d 100644 --- a/drivers/soc/apple/image.rs +++ b/drivers/soc/apple/image.rs @@ -48,21 +48,11 @@ pub(crate) fn sha256(bytes: &[u8]) -> Result<[u8; SHA256_LEN]> { const OFF_VERSION: usize = 0x10; const OFF_TIMESTAMP: usize = 0x14; -const OFF_FLAGS: usize = 0x1c; -const OFF_RESERVED: usize = 0x20; -const OFF_PROC_ID: usize = 0x28; -const OFF_PID: usize = 0x30; -const OFF_CDHASH: usize = 0x34; const OFF_TRAILER: usize = 0x48; static_assert!(OFF_VERSION == DIGEST_OFF + DIGEST_LEN); static_assert!(OFF_TIMESTAMP == OFF_VERSION + 4); -static_assert!(OFF_FLAGS == OFF_TIMESTAMP + 8); -static_assert!(OFF_RESERVED == OFF_FLAGS + 4); -static_assert!(OFF_PROC_ID == OFF_RESERVED + 8); -static_assert!(OFF_PID == OFF_PROC_ID + 8); -static_assert!(OFF_CDHASH == OFF_PID + 4); -static_assert!(OFF_TRAILER == OFF_CDHASH + 20); +static_assert!(OFF_TRAILER == OFF_TIMESTAMP + 8 + 4 + 8 + 8 + 4 + 20); static_assert!(OFF_TRAILER + 8 == HEADER_SIZE as usize); static_assert!(HEADER_WIRE == 4 + HEADER_SIZE as usize); @@ -89,6 +79,14 @@ pub(crate) struct Body { bytes: KVec, } +pub(crate) fn wipe(bytes: &mut [u8]) { + for byte in bytes { + // SAFETY: `byte` is uniquely borrowed and valid. A volatile write keeps + // the compiler from eliding destruction of request secrets. + unsafe { core::ptr::write_volatile(byte, 0) }; + } +} + impl Body { pub(crate) fn new() -> Body { Body { bytes: KVec::new() } @@ -125,6 +123,12 @@ impl Body { } } +impl Drop for Body { + fn drop(&mut self) { + wipe(&mut self.bytes); + } +} + const fn pad_of(len: usize) -> usize { len.wrapping_neg() % 4 } @@ -148,12 +152,19 @@ impl RequestImage { } } +impl Drop for RequestImage { + fn drop(&mut self) { + wipe(&mut self.bytes); + } +} + pub(crate) fn build_request( version: Version, timestamp_us: u64, body: &Body, ) -> Result { - let mut bytes = KVec::new(); + let mut image = RequestImage { bytes: KVec::new() }; + let bytes = &mut image.bytes; bytes.extend_from_slice(&HEADER_SIZE.to_le_bytes(), GFP_KERNEL)?; let header_at = bytes.len(); @@ -164,8 +175,8 @@ pub(crate) fn build_request( let put = |bytes: &mut KVec, off: usize, src: &[u8]| { bytes[header_at + off..header_at + off + src.len()].copy_from_slice(src); }; - put(&mut bytes, OFF_VERSION, &version.wire().to_le_bytes()); - put(&mut bytes, OFF_TIMESTAMP, ×tamp_us.to_le_bytes()); + put(bytes, OFF_VERSION, &version.wire().to_le_bytes()); + put(bytes, OFF_TIMESTAMP, ×tamp_us.to_le_bytes()); // Flags, reserved, proc_id, pid, cdhash and trailer stay zero; the enclave accepts that. bytes.extend_from_slice(body.as_slice(), GFP_KERNEL)?; @@ -195,7 +206,7 @@ pub(crate) fn build_request( bytes[header_at + DIGEST_OFF..header_at + DIGEST_OFF + DIGEST_LEN] .copy_from_slice(&digest[..DIGEST_LEN]); - Ok(RequestImage { bytes }) + Ok(image) } pub(crate) struct ResponseImage<'a> { @@ -232,7 +243,11 @@ pub(crate) fn read_blob(body: &[u8], off: usize) -> Option<(&[u8], usize)> { if body.len() < end { return None; } - Some((&body[len_end..end], end + pad_of(len))) + let next = end.checked_add(pad_of(len))?; + if body.get(end..next)?.iter().any(|byte| *byte != 0) { + return None; + } + Some((&body[len_end..end], next)) } pub(crate) fn operation_status(body: &[u8]) -> Option { diff --git a/drivers/soc/apple/keybag.rs b/drivers/soc/apple/keybag.rs index 90bed45619c50..515948b28b91f 100644 --- a/drivers/soc/apple/keybag.rs +++ b/drivers/soc/apple/keybag.rs @@ -86,7 +86,15 @@ const MAX_WRAPPED: usize = crate::store::MAX_VALUE; /// Proof the host holds no bag; create takes one by value, so it is unreachable /// without it and unrepeatable with it. -pub(crate) struct NoStoredKeyBag; +pub(crate) struct NoStoredKeyBag { + slot: Slot, +} + +impl NoStoredKeyBag { + pub(crate) fn slot(&self) -> Slot { + self.slot + } +} pub(crate) struct StoredKeyBag { wrapped: KVec, @@ -143,7 +151,7 @@ pub(crate) fn read(slot: Slot) -> Result { let file = match shim::StoreFile::open_readonly(slot.path()) { Ok(f) => f, Err(e) if e == ENOENT => { - return Ok(State::Absent(NoStoredKeyBag)); + return Ok(State::Absent(NoStoredKeyBag { slot })); } Err(e) => return Err(e), }; @@ -174,7 +182,7 @@ pub(crate) fn read(slot: Slot) -> Result { head[OFF_STATE + 3], ]); if state == STATE_REFUSED { - return Ok(State::Absent(NoStoredKeyBag)); + return Ok(State::Absent(NoStoredKeyBag { slot })); } let Some(provenance) = UuidProvenance::from_state(state) else { // Intent or unknown state: a bag may exist unnamed; creating again would @@ -233,6 +241,32 @@ pub(crate) fn read(slot: Slot) -> Result { })) } +pub(crate) fn write_intent(slot: Slot) -> Result<()> { + write_record(slot, 0, &[], &[0; UUID_LEN], &[]) +} + +pub(crate) fn mark_refused(slot: Slot) -> Result<()> { + write_record(slot, STATE_REFUSED, &[], &[0; UUID_LEN], &[]) +} + +pub(crate) fn write_bag_uuid( + slot: Slot, + wrapped: &[u8], + uuid: &[u8; UUID_LEN], + secret: &[u8], +) -> Result<()> { + if wrapped.is_empty() || wrapped.len() > MAX_WRAPPED || secret.len() > MAX_WRAPPED { + return Err(EINVAL); + } + write_record( + slot, + UuidProvenance::ReadBackFromBag.state(), + wrapped, + uuid, + secret, + ) +} + /// Replaces only the wrapped blob. The enclave ratchets bag material while a bag /// is active, so this snapshot must be re-taken at the catacomb-save commit or /// the enclave answers a later restore empty. diff --git a/drivers/soc/apple/proto.rs b/drivers/soc/apple/proto.rs index d084a889109c5..28324172c5e7b 100644 --- a/drivers/soc/apple/proto.rs +++ b/drivers/soc/apple/proto.rs @@ -3,8 +3,6 @@ //! The SEP wire protocol, reverse-engineered: opcode tables, request encoders //! and reply decoders. -#![allow(dead_code)] - use kernel::prelude::*; use kernel::soc::apple::mailbox::Message; @@ -14,7 +12,6 @@ pub(crate) const EP_SHMEM: u8 = 0xFE; pub(crate) const EP_BOOT: u8 = 0xFF; pub(crate) const EP_XARM: u8 = 0x13; pub(crate) const EP_SBIO: u8 = 0x08; -pub(crate) const EP_XARS: u8 = 0x10; pub(crate) const EP_SCRD: u8 = 0x0a; pub(crate) const EP_SKS: u8 = 0x12; @@ -106,8 +103,6 @@ pub(crate) fn fourcc(msg: &Message) -> Fourcc { pub(crate) const CONTROL_REPLY_TYPE: u8 = 0x01; -pub(crate) const TAG_UNSOLICITED: u8 = 0x00; - pub(crate) const TAG_ENTROPY: u8 = 0xE7; pub(crate) const CONTROL_TIMEOUT_MS: u32 = 2000; @@ -115,7 +110,7 @@ pub(crate) const CONTROL_TIMEOUT_MS: u32 = 2000; pub(crate) const TAG_POOL_FIRST: u8 = 0x01; pub(crate) const TAG_POOL_LAST: u8 = 0x7e; -static_assert!(TAG_POOL_FIRST > TAG_UNSOLICITED); +static_assert!(TAG_POOL_FIRST > 0); static_assert!(TAG_POOL_LAST < TAG_ENTROPY); pub(crate) struct ControlOp { @@ -209,24 +204,6 @@ fn op_ool(ty: u8, endpoint: u8, data: u32, name: &'static CStr) -> ControlOp { } } -const OP_DMA_RING_PAGES: u8 = 0x19; -const OP_DMA_RING_ADDR: u8 = 0x1a; - -pub(crate) const DMA_RING_PAGES: u32 = 4; - -pub(crate) fn op_dma_ring_pages(endpoint: u8, pages: u32) -> ControlOp { - op_ool(OP_DMA_RING_PAGES, endpoint, pages, c"RING_PAGES") -} - -pub(crate) fn op_dma_ring_addr(endpoint: u8, iova: u64) -> ControlOp { - op_ool( - OP_DMA_RING_ADDR, - endpoint, - (iova >> IOVA_SHIFT) as u32, - c"RING_ADDR", - ) -} - pub(crate) fn op_ool_inbound_size(endpoint: u8, len: u32) -> ControlOp { op_ool(OP_OOL_INBOUND_SIZE, endpoint, len, c"OOL_IN_SIZE") } @@ -277,7 +254,11 @@ impl<'a> FieldCursor<'a> { let end = len_end.checked_add(len)?; let bytes = self.body.get(len_end..end)?; let pad = len.wrapping_neg() % 4; - self.at = end.checked_add(pad)?; + let next = end.checked_add(pad)?; + if self.body.get(end..next)?.iter().any(|byte| *byte != 0) { + return None; + } + self.at = next; Some(bytes) } } @@ -286,7 +267,6 @@ impl<'a> FieldCursor<'a> { pub(crate) struct ControlReply { pub(crate) tag: u8, pub(crate) data_lo: u32, - pub(crate) msg1: u32, } impl ControlReply { @@ -295,7 +275,6 @@ impl ControlReply { ControlReply { tag: f.tag, data_lo: f.data_lo, - msg1: msg.msg1, } } } diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index fff495601bb85..352609f6b4d08 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -3,8 +3,6 @@ //! Touch ID: the biometric endpoint (SBIO) and its `/dev/sep-bio` interface. //! The enclave matches; no biometric image ever crosses to userspace. -#![allow(dead_code)] - use super::*; use kernel::prelude::*; use kernel::soc::apple::mailbox::Message; @@ -107,12 +105,10 @@ impl SepData { } fn stash_enrol_identity(&self, record: &[u8]) { - let mut found: KVec<([u8; bio::UUID_LEN], u32)> = KVec::new(); - crate::sbio::enrol_identity_candidates(record, SBIO_PROBE_USER_ID, |at, uuid| { - let _ = found.push((uuid, at as u32), GFP_KERNEL); + let mut found: KVec<[u8; bio::UUID_LEN]> = KVec::new(); + crate::sbio::enrol_identity_candidates(record, SBIO_PROBE_USER_ID, |_at, uuid| { + let _ = found.push(uuid, GFP_KERNEL); }); - for (_uuid, _at) in found.iter() { - } *self.enrol_identity_candidates.lock() = found; } @@ -120,27 +116,31 @@ impl SepData { let candidates = core::mem::take(&mut *self.enrol_identity_candidates.lock()); let listed = self.enclave_identities_for(SBIO_PROBE_USER_ID)?; - let mut confirmed: KVec<([u8; bio::UUID_LEN], u32)> = KVec::new(); - for (uuid, at) in candidates.iter() { - if listed.iter().any(|u| u == uuid) && !confirmed.iter().any(|(u, _)| u == uuid) { - if confirmed.push((*uuid, *at), GFP_KERNEL).is_err() { + let mut confirmed: KVec<[u8; bio::UUID_LEN]> = KVec::new(); + for uuid in candidates.iter() { + if listed.iter().any(|u| u == uuid) && !confirmed.iter().any(|u| u == uuid) { + if confirmed.push(*uuid, GFP_KERNEL).is_err() { break; } } } - match confirmed.len() { - 1 => { - let (uuid, _at) = confirmed[0]; - Some(uuid) - } - 0 => { - None + if confirmed.len() == 1 { + return Some(confirmed[0]); + } + + let index = self.bio_index.lock(); + let mut new_identity = None; + for uuid in listed.iter() { + if index.contains_uuid(uuid) { + continue; } - _ => { - None + if new_identity.is_some() { + return None; } + new_identity = Some(*uuid); } + new_identity } fn reconcile_identities(&self) { @@ -166,9 +166,6 @@ impl SepData { let index = self.bio_index.lock(); index.total() }; - for _uuid in listed.iter() { - } - if dropped.is_empty() && held == listed.len() { return; } @@ -475,7 +472,7 @@ impl SepData { crate::sbio::ComponentAction::AlreadyActive => { RestoreOutcome::AlreadyActive } - crate::sbio::ComponentAction::Unsupported(_) => { + crate::sbio::ComponentAction::Unsupported => { RestoreOutcome::Failed } crate::sbio::ComponentAction::Load => { @@ -750,27 +747,78 @@ impl SepData { } } - // ordering: the catacomb restore must complete before the sensor is woken + fn bring_sensor_online(&self) -> bool { + let Some(mut patch) = self.wake_sensor() else { + return false; + }; + + if !self.sensor_calibrated.load(Relaxed) { + if !self.calibrate_sensor() { + return false; + } + self.sensor_calibrated.store(true, Relaxed); + let Some(reloaded_patch) = self.wake_sensor() else { + return false; + }; + patch = reloaded_patch; + } + + self.complete_bringup(patch) + } + pub(crate) fn run_bringup(&self) { if self.bringup_started.xchg(true, Relaxed) { return; } - if let Err(e) = self.enable_sbio() { - dev_err!(self.dev, "bringup: could not enable the biometric transport ({:?}); Touch ID is unavailable this boot\n", e); - return; - } if let Err(e) = self.enable_sks() { dev_warn!(self.dev, "bringup: could not enable the key store ({:?}); keybag and ref-key operations are unavailable\n", e); + return; } - if let Ok(keybag::State::Present(stored)) = keybag::read(keybag::Slot::Identity) { - if let Some((handle, uuid)) = self.sks_recover(&stored) { - self.sks_designate_user_keybag(handle, stored.secret()); - self.sks_machine_refkey(handle, stored.secret()); - let prepared = self.cold_match_continue(handle, uuid); - self.ensure_restored_after(prepared); + if *module_parameters::provision_keybag.value() != 0 { + if *module_parameters::xart_writes.value() == 0 { + dev_err!(self.dev, "bringup: provision_keybag=1 requires xart_writes=1\n"); + return; + } + if !self.sks_provision_identity_keybag() { + dev_err!(self.dev, "bringup: identity-keybag provisioning failed; no retry this boot\n"); + return; } } + } + + fn activate_touchid(&self) -> Result<()> { + if self.touchid_started.load(Relaxed) { + return Ok(()); + } + + let mut store = store::Store::open()?; + let index = bio::IdentityIndex::load(&mut store)?; + *self.host_store.lock() = Some(store); + *self.bio_index.lock() = index; + + self.attach_sensor(); + self.enable_sbio()?; + let keybag::State::Present(stored) = keybag::read(keybag::Slot::Identity)? else { + return Err(ENOENT); + }; + let (handle, uuid) = self.sks_recover(&stored).ok_or(EIO)?; + self.sks_designate_user_keybag(handle, stored.secret()); + self.sks_machine_refkey(handle, stored.secret()); + let prepared = self.cold_match_continue(handle, uuid); + self.ensure_restored_after(prepared); self.attach_bringup(); + self.touchid_started.store(true, Relaxed); + Ok(()) + } + + // Touch ID starts from the caller's real-root namespace. SEP can therefore + // unlock root in initramfs without pinning biometric persistence to tmpfs. + fn prepare_bio_open(&self) -> Result<()> { + if let Err(e) = self.activate_touchid() { + dev_err!(self.dev, "Touch ID activation failed: {:?}\n", e); + return Err(e); + } + Ok(()) } pub(crate) fn run_verify(&self) { @@ -796,31 +844,8 @@ impl SepData { self.settle_before_capture(); - let Some(patch) = self.wake_sensor() else { - self.finish_verify(bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), token_bytes); - let _ = sensor::idle(); - return; - }; - - // enclave refuses a capture from an uncalibrated sensor (0x65 answers 1) - let already_calibrated = self.sensor_calibrated.load(Relaxed); - self.sensor_calibrated.store(true, Relaxed); - let patch = if already_calibrated { - patch - } else { - if !self.calibrate_sensor() { - self.finish_verify(bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), token_bytes); - let _ = sensor::idle(); - return; - } - let Some(patch) = self.wake_sensor() else { - self.finish_verify(bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), token_bytes); - let _ = sensor::idle(); - return; - }; - patch - }; - if !self.complete_bringup(patch) { + // The enclave refuses a capture from an uncalibrated sensor (0x65 answers 1). + if !self.bring_sensor_online() { self.finish_verify(bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), token_bytes); let _ = sensor::idle(); return; @@ -953,31 +978,7 @@ impl SepData { } pub(crate) fn run_enrolment(&self) { - let Some(patch) = self.wake_sensor() else { - self.finish_enrolment(Err(ENROL_STATUS_SENSOR)); - let _ = sensor::idle(); - return; - }; - - let already_calibrated = self.sensor_calibrated.load(Relaxed); - self.sensor_calibrated.store(true, Relaxed); - let patch = if already_calibrated { - patch - } else { - if !self.calibrate_sensor() { - self.finish_enrolment(Err(ENROL_STATUS_SENSOR)); - let _ = sensor::idle(); - return; - } - let Some(patch) = self.wake_sensor() else { - self.finish_enrolment(Err(ENROL_STATUS_SENSOR)); - let _ = sensor::idle(); - return; - }; - patch - }; - - if !self.complete_bringup(patch) { + if !self.bring_sensor_online() { self.finish_enrolment(Err(ENROL_STATUS_SENSOR)); let _ = sensor::idle(); return; @@ -1703,15 +1704,6 @@ impl SepData { } }; - let _repeated_header = { - let mut bytes = [0u8; transfer::HEADER_LEN]; - bytes.copy_from_slice(&header[..transfer::HEADER_LEN]); - let mut last = self.sbio_last_header.lock(); - let same = last.as_ref().is_some_and(|prev| *prev == bytes); - *last = Some(bytes); - same - }; - let chunk = packet.chunk as usize; let payload = if chunk == 0 { KVec::new() @@ -1738,14 +1730,12 @@ impl SepData { // 0xFE requests the peer's next packet and acks the final one; nothing to send self.sbio_wq.notify_all(); } - transfer::Progress::Ignored(_why) => {}, + transfer::Progress::Ignored => {}, transfer::Progress::Grant => { self.sbio_wq.notify_all(); } - transfer::Progress::Notification { tag: _tag, opcode: _opcode } => { - let _ = self.sbio_rx.lock().awaiting(); - } - transfer::Progress::Failed(_why) => { + transfer::Progress::Notification => {}, + transfer::Progress::Failed => { self.sbio_wq.notify_all(); } } @@ -1805,7 +1795,6 @@ impl SepData { let mut guard = self.sbio_rx.lock(); loop { if let Some(done) = guard.take_done() { - let _ = done.opcode; return Ok(done); } if remaining == 0 { @@ -1947,7 +1936,14 @@ impl SepData { } pub(crate) fn bio_open(&self) -> Result<()> { - bio::open(&mut self.bio_session.lock()) + let mut session = self.bio_session.lock(); + bio::open(&mut session)?; + drop(session); + if let Err(e) = self.prepare_bio_open() { + bio::release(&mut self.bio_session.lock()); + return Err(e); + } + Ok(()) } pub(crate) fn bio_release(&self) { @@ -2125,8 +2121,6 @@ impl SepData { return false; } - // unlock/0x18 is the destructive-template path (bag stays designated, locked) - self.cold_prepared.store(true, Relaxed); true } @@ -2273,8 +2267,6 @@ impl SbioOp { pub(crate) const SBIO_PROTOCOL_GENERATION: u32 = 1; -const OP_SBIO_INIT_COMMS: u16 = 0x73; - const OP_SBIO_REGISTER_SENSOR: u16 = 0x80; pub(crate) fn sbio_register_sensor(id: &crate::sensor::Identifier) -> SbioOp { SbioOp { @@ -2285,8 +2277,6 @@ pub(crate) fn sbio_register_sensor(id: &crate::sensor::Identifier) -> SbioOp { } } -const OP_SBIO_SEND_SERIAL: u16 = 0x48; - pub(crate) const SBIO_SIGNAL_QUALITY: u32 = 0; const OP_SBIO_COVERAGE_PARAMS: u16 = 0x5d; @@ -2413,10 +2403,7 @@ pub(crate) fn sbio_protected_config(id: UserId) -> SbioOp { const OP_SBIO_BEGIN_ENROL: u16 = 0x03; -// type 0 (ACM context) survives a reboot; type 1 (SKS token) does not pub(crate) const BE_AUTH_TYPE_ACM_CONTEXT: u32 = 0; -pub(crate) const BE_AUTH_TYPE_SKS_TOKEN: u32 = 1; -static_assert!(BE_AUTH_TYPE_ACM_CONTEXT != BE_AUTH_TYPE_SKS_TOKEN); pub(crate) const SBIO_BEGIN_ENROL_LEN: usize = 0x44; static_assert!(SBIO_BEGIN_ENROL_LEN <= SBIO_MAX_PAYLOAD); @@ -2426,22 +2413,7 @@ const BE_USER_ID: usize = 4; const BE_AUTH_TYPE: usize = 8; const BE_TOKEN_LEN: usize = 12; const BE_TOKEN: usize = 16; -const BE_SELECTOR: usize = 48; -static_assert!(BE_TOKEN + SKS_AUTH_TOKEN_LEN <= BE_SELECTOR); -const BE_SELECTOR_LEN: usize = 20; -static_assert!(BE_SELECTOR + BE_SELECTOR_LEN == SBIO_BEGIN_ENROL_LEN); - -pub(crate) const SBIO_BEGIN_ENROL_COPIED: usize = match SBIO_PROTOCOL_GENERATION { - 1 => 0x30, - 6 => SBIO_BEGIN_ENROL_LEN, - _ => 0, -}; -static_assert!(SBIO_BEGIN_ENROL_COPIED != 0); -static_assert!(SBIO_BEGIN_ENROL_COPIED <= SBIO_BEGIN_ENROL_LEN); - -const BE_SELECTOR_IN_RECORD: bool = SBIO_BEGIN_ENROL_COPIED >= BE_SELECTOR + BE_SELECTOR_LEN; - -static_assert!(!BE_SELECTOR_IN_RECORD); +static_assert!(BE_TOKEN + SKS_AUTH_TOKEN_LEN <= SBIO_BEGIN_ENROL_LEN); pub(crate) fn sbio_begin_enrol( user: UserId, @@ -2540,14 +2512,10 @@ pub(crate) fn sbio_match_policy() -> SbioOp { pub(crate) const SBIO_ENROL_RESULT_LEN: usize = 0xc98; -const ER_STATUS: usize = 0x000; -const ER_ERROR: usize = 0x002; const ER_PROGRESS: usize = 0x004; const ER_HAS_TEMPLATE: usize = 0x006; const ER_COMPLETE: usize = 0xbfe; -static_assert!(ER_STATUS + 2 <= SBIO_ENROL_RESULT_LEN); -static_assert!(ER_ERROR + 2 <= SBIO_ENROL_RESULT_LEN); static_assert!(ER_PROGRESS < SBIO_ENROL_RESULT_LEN); static_assert!(ER_HAS_TEMPLATE + 4 <= SBIO_ENROL_RESULT_LEN); static_assert!(ER_COMPLETE + 4 <= SBIO_ENROL_RESULT_LEN); @@ -2645,9 +2613,6 @@ static_assert!(SBIO_SAVED_USER_ID_AT + 4 <= SBIO_SAVED_MIN); pub(crate) const SBIO_SAVE_SELECTOR_LEN: usize = 24; const SS_USER_ID: usize = 0; -const SS_DEVICE: usize = 4; -const SS_DEVICE_LEN: usize = 20; -static_assert!(SS_DEVICE + SS_DEVICE_LEN == SBIO_SAVE_SELECTOR_LEN); static_assert!(SBIO_SAVE_SELECTOR_LEN <= SBIO_MAX_PAYLOAD); pub(crate) struct SaveSelector([u8; SBIO_SAVE_SELECTOR_LEN]); @@ -2689,10 +2654,6 @@ impl<'a> ComponentStates<'a> { self.0.len() / COMPONENT_PAIR_LEN } - pub(crate) fn trailing(&self) -> usize { - self.0.len() % COMPONENT_PAIR_LEN - } - pub(crate) fn pair(&self, i: usize) -> Option<(i32, u32)> { let at = i.checked_mul(COMPONENT_PAIR_LEN)?; let bytes = self.0.get(at..at + COMPONENT_PAIR_LEN)?; @@ -2713,7 +2674,7 @@ impl<'a> ComponentStates<'a> { pub(crate) enum ComponentAction { AlreadyActive, Load, - Unsupported(u32), + Unsupported, } pub(crate) fn component_action(state: u32) -> ComponentAction { @@ -2722,7 +2683,7 @@ pub(crate) fn component_action(state: u32) -> ComponentAction { } else if state & COMPONENT_STATE_COLD != 0 { ComponentAction::Load } else { - ComponentAction::Unsupported(state) + ComponentAction::Unsupported } } @@ -2825,10 +2786,6 @@ pub(crate) fn sbio_list_identities() -> SbioOp { } } -const OP_SBIO_GROUP_STATE: u16 = 0x79; - -const OP_SBIO_LIST_IDENTITIES_SCOPED: u16 = 0x6e; - pub(crate) struct IdentityRecords<'a>(&'a [u8]); impl<'a> IdentityRecords<'a> { @@ -2921,18 +2878,12 @@ pub(crate) const SBIO_MATCH_RESULT_LEN: usize = 0xca2; const MR_USER_ID: usize = 0x000; const MR_IDENTITY: usize = 0x004; -const MR_CANDIDATES: usize = 0x014; const MR_FLAGS: usize = 0xc8a; -const MR_SECOND_IDENTITY_UNUSED: usize = 0xc8e; -static_assert!(MR_IDENTITY != MR_SECOND_IDENTITY_UNUSED); +const MR_FLAG_MATCH: u32 = 1; static_assert!(MR_USER_ID + 4 <= SBIO_MATCH_RESULT_LEN); static_assert!(MR_IDENTITY + IDENTITY_UUID_LEN <= SBIO_MATCH_RESULT_LEN); -static_assert!(MR_CANDIDATES + 4 <= SBIO_MATCH_RESULT_LEN); static_assert!(MR_FLAGS + 4 <= SBIO_MATCH_RESULT_LEN); -static_assert!(MR_SECOND_IDENTITY_UNUSED + IDENTITY_UUID_LEN <= SBIO_MATCH_RESULT_LEN); static_assert!(MR_IDENTITY == MR_USER_ID + 4); -static_assert!(MR_IDENTITY + IDENTITY_UUID_LEN == MR_CANDIDATES); -static_assert!(MR_FLAGS > MR_USER_ID + 4); static_assert!(SBIO_MATCH_RESULT_LEN != SBIO_ENROL_RESULT_LEN); pub(crate) const IDENTITY_UUID_LEN: usize = 16; @@ -2965,6 +2916,7 @@ impl IdentityV1 { pub(crate) struct MatchResult { user_id: i32, identity: [u8; IDENTITY_UUID_LEN], + flags: u32, } impl MatchResult { @@ -2982,6 +2934,7 @@ impl MatchResult { bytes[MR_USER_ID + 3], ]), identity, + flags: u32::from_le_bytes(bytes[MR_FLAGS..MR_FLAGS + 4].try_into().ok()?), }) } @@ -2990,7 +2943,7 @@ impl MatchResult { } pub(crate) fn matches(&self, user: UserId) -> bool { - self.user_id == user.value() + self.flags & MR_FLAG_MATCH != 0 && self.user_id == user.value() } } @@ -3035,16 +2988,6 @@ pub(crate) fn sbio_register_sensor_serial(serial: &crate::sensor::SensorSerial) } } -const OP_SBIO_ENUMERATE: u16 = 0x7c; -pub(crate) const fn sbio_enumerate() -> SbioOp { - SbioOp { - opcode: OP_SBIO_ENUMERATE, - payload: [0; SBIO_MAX_PAYLOAD], - payload_len: 0, - name: c"ENUMERATE", - } -} - const OP_SBIO_DIAGNOSTICS: u16 = 0x63; pub(crate) const fn sbio_diagnostics() -> SbioOp { SbioOp { @@ -3055,12 +2998,6 @@ pub(crate) const fn sbio_diagnostics() -> SbioOp { } } -const SURVEY_ENUMERATE: SbioOp = sbio_enumerate(); -const SURVEY_DIAGNOSTICS: SbioOp = sbio_diagnostics(); - -static_assert!(SURVEY_ENUMERATE.payload_len == 0); -static_assert!(SURVEY_DIAGNOSTICS.payload_len == 0); - #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum ImagePurpose { Enrolment, @@ -3217,16 +3154,10 @@ pub(crate) fn sbio_enrolment_result() -> SbioOp { } pub(crate) const ASSESS_MIN_LEN: usize = 0x89; -pub(crate) const ASSESS_ERROR: usize = 0x00; pub(crate) const ASSESS_USABLE_MATCH: usize = 0x06; pub(crate) const ASSESS_USABLE_ENROL: usize = 0x07; -pub(crate) const ASSESS_FEEDBACK: usize = 0x0e; -pub(crate) const ASSESS_DIRTY: usize = 0x51; -static_assert!(ASSESS_ERROR + 2 <= ASSESS_MIN_LEN); static_assert!(ASSESS_USABLE_MATCH < ASSESS_USABLE_ENROL); -static_assert!(ASSESS_FEEDBACK + 4 <= ASSESS_MIN_LEN); -static_assert!(ASSESS_DIRTY < ASSESS_MIN_LEN); pub(crate) const SBIO_SESSION_SHARE_LEN: usize = 40; @@ -3301,7 +3232,7 @@ const fn encode_sbio(opcode: u16, marker: u8, seq: u16) -> Message { } } -static_assert!(encode_sbio(OP_SBIO_INIT_COMMS, transfer::MARKER_FIRST, 0).msg0 == 0x0000_0073_fc08); +static_assert!(encode_sbio(0x73, transfer::MARKER_FIRST, 0).msg0 == 0x0000_0073_fc08); pub(crate) fn encode_sbio_raw(opcode: u16, seq: u16) -> Option { Some(encode_sbio(opcode, transfer::MARKER_FIRST, seq)) diff --git a/drivers/soc/apple/scrd.rs b/drivers/soc/apple/scrd.rs index 0326308651ba4..acdcdab12db65 100644 --- a/drivers/soc/apple/scrd.rs +++ b/drivers/soc/apple/scrd.rs @@ -3,12 +3,10 @@ //! SEP credential endpoint (SCRD, EP 0x0a) wire protocol. -#![allow(dead_code)] - -use kernel::prelude::*; -use kernel::soc::apple::mailbox::Message; use crate::proto::*; use crate::sks::SKS_AUTH_TOKEN_LEN; +use kernel::prelude::*; +use kernel::soc::apple::mailbox::Message; pub(crate) const SCRD_ACM_HANDLE_LEN: usize = SKS_AUTH_TOKEN_LEN; @@ -25,8 +23,6 @@ const SCRD_INIT_LOG_LEVEL: u8 = 0x28; const SCRD_POLICY_TOUCHID_ENROLLMENT: &[u8] = b"TouchIdEnrollment"; -pub(crate) const SCRD_CONTEXT_CREATE_REPLY_LEN: usize = SCRD_ACM_HANDLE_LEN + 1 + 4; - const SCRD_MAX_LOGICAL: usize = 64; static_assert!( 8 + SCRD_ACM_HANDLE_LEN + SCRD_POLICY_TOUCHID_ENROLLMENT.len() + 1 + 1 + 8 <= SCRD_MAX_LOGICAL diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index 44eeb912868c7..79d9e3bc7d50c 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -111,14 +111,10 @@ static_assert!(STATUS_AT + STATUS_LEN == STATUS_XFER_LEN); // Offsets index the 16-byte status, not the 23-byte transfer. const STATUS_STATE: usize = STATUS_AT + 7; const STATUS_COUNT: usize = STATUS_AT + 12; -const STATUS_STATE_TRANSFER_REL: usize = 7; -const STATUS_COUNT_TRANSFER_REL: usize = 12; static_assert!(STATUS_STATE == 14); static_assert!(STATUS_COUNT == 19); static_assert!(STATUS_COUNT + 4 == STATUS_XFER_LEN); -static_assert!(STATUS_STATE != STATUS_STATE_TRANSFER_REL); -static_assert!(STATUS_COUNT != STATUS_COUNT_TRANSFER_REL); static_assert!(STATUS_STATE >= STATUS_AT && STATUS_STATE < STATUS_AT + STATUS_LEN); static_assert!(STATUS_COUNT >= STATUS_AT && STATUS_COUNT + 4 <= STATUS_AT + STATUS_LEN); @@ -251,11 +247,10 @@ fn send_framed(frame_type: u8, payload: &[u8]) -> Result<()> { check(unsafe { sep_sensor_xfer_tx(frame.as_ptr().cast(), total) }) } -const ENCRYPTED_DECLARED_MAX: usize = 0x12c; const ENCRYPTED_OVERHEAD: usize = FRAME_HEADER + CRC_LEN; static_assert!(ENCRYPTED_OVERHEAD == 9); const ENCRYPTED_TRANSFER_MAX: usize = 0x12b; -static_assert!(ENCRYPTED_TRANSFER_MAX < ENCRYPTED_DECLARED_MAX); +static_assert!(ENCRYPTED_TRANSFER_MAX < 0x12c); pub(crate) struct Geometry { declared: usize, @@ -293,14 +288,14 @@ impl Geometry { } } -static_assert!(Geometry::MODULE_CHALLENGE.declared < ENCRYPTED_DECLARED_MAX); +static_assert!(Geometry::MODULE_CHALLENGE.declared < 0x12c); static_assert!(Geometry::MODULE_CHALLENGE.declared > ENCRYPTED_OVERHEAD); static_assert!(Geometry::MODULE_CHALLENGE.transfer >= Geometry::MODULE_CHALLENGE.declared); static_assert!(Geometry::MODULE_CHALLENGE.transfer <= ENCRYPTED_TRANSFER_MAX); static_assert!(Geometry::MODULE_CHALLENGE.payload_capacity() == 0x40); -static_assert!(Geometry::COVERAGE.declared < ENCRYPTED_DECLARED_MAX); -static_assert!(Geometry::OPERATION.declared < ENCRYPTED_DECLARED_MAX); -static_assert!(Geometry::TRANSPARENT.declared < ENCRYPTED_DECLARED_MAX); +static_assert!(Geometry::COVERAGE.declared < 0x12c); +static_assert!(Geometry::OPERATION.declared < 0x12c); +static_assert!(Geometry::TRANSPARENT.declared < 0x12c); static_assert!(Geometry::COVERAGE.declared > ENCRYPTED_OVERHEAD); static_assert!(Geometry::OPERATION.declared > ENCRYPTED_OVERHEAD); static_assert!(Geometry::TRANSPARENT.declared > ENCRYPTED_OVERHEAD); diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 4f546e66602e3..77b06dcae705f 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -93,8 +93,6 @@ const ENDPOINTS_BEFORE_EXCHANGE: usize = 7; const OOL_SIZE_XARM: usize = 0x8000; -const OOL_SIZE_XARS: usize = 0x8000; - const OOL_SIZE_SBIO: usize = 0x4000; const OOL_SIZE_SCRD: usize = 0x4000; @@ -103,11 +101,9 @@ const SBIO_TIMEOUT_MS: time::Msecs = 5000; const SKS_ALLOC: usize = 0x8000; - +const SKS_SECRET_LEN: usize = 32; const SKS_MAX_CAPTURE: usize = 8; -const XARS_MAX_CAPTURE: usize = 8; - const SCRD_MAX_CAPTURE: usize = 8; const SCRD_TIMEOUT_MS: time::Msecs = 2000; @@ -153,11 +149,13 @@ const PROTECTED_DATA_AVAILABLE: bool = true; const RNG_MAX_WORDS_PER_READ: usize = 16; -const SECMODE_UNKNOWN: u32 = u32::MAX; +extern "C" { + fn sep_cancel_work_sync(work: *mut c_void); + fn sep_cancel_delayed_work_sync(work: *mut c_void); +} const MAX_ENDPOINTS: usize = 64; - #[derive(Clone, Copy)] struct Endpoint { id: u8, @@ -199,8 +197,6 @@ fn sks_declared_sizes() -> (usize, usize) { (0x8000, 0x4000) } -const SKS_MALFORMED: i8 = crate::sks::SKS_STATUS_MALFORMED; - struct Hex<'a>(&'a [u8]); impl kernel::fmt::Display for Hex<'_> { @@ -218,28 +214,22 @@ impl kernel::fmt::Display for Hex<'_> { } } -const SKS_SECRET_LEN: usize = 32; -static_assert!(SKS_SECRET_LEN % 4 == 0); - const SKS_LOCK_STATE_VARIANT: u32 = 1; #[derive(Clone, Copy)] enum LockState { Unlocked, - Locked, } impl LockState { const fn wire(self) -> i32 { match self { LockState::Unlocked => 0, - LockState::Locked => 1, } } } static_assert!(LockState::Unlocked.wire() == 0); -static_assert!(LockState::Locked.wire() == 1); const SKS_LOCK_STATE_FLAGS: u64 = 0; @@ -403,129 +393,15 @@ const ENROL_IDLE_TIMEOUT_MS: u32 = 2000; const PATCH_POLL_MS: u32 = 20; const PATCH_POLL_ATTEMPTS: u32 = 250; -const SEAL_VECTOR: &[u8] = b"apple-sep seal round trip v1"; - -const SEAL_STATUS_SHAPE: i8 = SKS_MALFORMED; - -const SEAL_STATUS_NAMES_NOTHING: i8 = -11; - -const SEAL_STATUS_WRONG_KIND: i8 = -12; - -const SEAL_STATUS_MISSING_PREREQUISITE: i8 = -3; -static_assert!(SEAL_STATUS_MISSING_PREREQUISITE != SEAL_STATUS_WRONG_KIND); -static_assert!(SEAL_STATUS_MISSING_PREREQUISITE != SEAL_STATUS_NAMES_NOTHING); - -const SEAL_STATUS_BACKUP_WRAP: i8 = -14; - -static_assert!(SEAL_STATUS_WRONG_KIND != SEAL_STATUS_NAMES_NOTHING); -static_assert!(SEAL_STATUS_WRONG_KIND != SEAL_STATUS_SHAPE); -static_assert!(SEAL_STATUS_BACKUP_WRAP != SEAL_STATUS_WRONG_KIND); -static_assert!(SEAL_STATUS_BACKUP_WRAP != SEAL_STATUS_SHAPE); - -const SKS_WRAP_OVERHEAD: usize = 256; - -const SKS_WRAP_PRODUCTION_CAPACITY: u32 = SEAL_VECTOR.len() as u32 + SKS_WRAP_OVERHEAD as u32; - -const SKS_CAPACITY_MAP: [u32; 23] = [ - 128, 192, 255, 256, 257, 258, 300, 320, 384, 512, 513, 640, 768, 1023, 1024, 1025, 1026, 1280, - 1536, 2048, 2049, 3072, 4096, -]; - -const fn capacity_map_ascends() -> bool { - let mut i = 1; - while i < SKS_CAPACITY_MAP.len() { - if SKS_CAPACITY_MAP[i - 1] >= SKS_CAPACITY_MAP[i] { - return false; - } - i += 1; - } - true -} -static_assert!(capacity_map_ascends()); -static_assert!(SKS_CAPACITY_MAP[0] > 0); -static_assert!(SKS_CAPACITY_MAP[5] < SKS_WRAP_PRODUCTION_CAPACITY); -static_assert!(SKS_WRAP_PRODUCTION_CAPACITY < SKS_CAPACITY_MAP[6]); -static_assert!(SKS_CAPACITY_MAP[2] + 1 == SKS_CAPACITY_MAP[3]); -static_assert!(SKS_CAPACITY_MAP[13] + 1 == SKS_CAPACITY_MAP[14]); - const CALIBRATION_FIRMWARE: &CStr = c"apple/mesa_calibration.bin"; -const SKS_CONFIG_MASKS: [u32; 17] = [ - 0xffff_ffff, - 0x1, - 0x3, - 0x5, - 0x9, - 0x11, - 0x21, - 0x41, - 0x81, - 0x101, - 0x201, - 0x401, - 0x801, - 0x1001, - 0x2001, - 0x4001, - 0x8001, -]; - -const fn config_masks_are_single_bit_probes() -> bool { - if SKS_CONFIG_MASKS[0] != u32::MAX || SKS_CONFIG_MASKS[1] != 1 { - return false; - } - let mut i = 2; - while i < SKS_CONFIG_MASKS.len() { - let m = SKS_CONFIG_MASKS[i]; - if m & 1 != 1 || (m & !1u32).count_ones() != 1 { - return false; - } - if i > 2 && SKS_CONFIG_MASKS[i - 1] >= m { - return false; - } - i += 1; - } - true -} -static_assert!(config_masks_are_single_bit_probes()); - -const CONFIG_STAGE_NONE: u32 = 0; const SBIO_PROBE_USER_ID: i32 = 1000; static_assert!(SBIO_PROBE_USER_ID >= crate::sks::SKS_DESIGNATE_USER_MIN); static_assert!(SBIO_PROBE_USER_ID == bio::ENROL_USER_ID); static_assert!(SBIO_PROBE_USER_ID > 0); -static_assert!(SBIO_PROBE_USER_ID == crate::sks::SKS_IDENTITY_USER_ID); const SKS_LOAD_REPLY_LEN: usize = 8; -// StoreType: the u32 at +0x60 of the create body -#[derive(Clone, Copy, PartialEq, Eq)] -struct StoreType(u32); - -const STORE_TYPE_MAX: u32 = 8; - -impl StoreType { - const IDENTITY: StoreType = StoreType(0); - const SEALING: StoreType = StoreType(1); - - const fn new(value: u32) -> Option { - if value <= STORE_TYPE_MAX { - Some(StoreType(value)) - } else { - None - } - } - - const fn wire(self) -> u32 { - self.0 - } -} - -static_assert!(StoreType::IDENTITY.wire() == 0); -static_assert!(StoreType::SEALING.wire() == 1); -static_assert!(StoreType::new(STORE_TYPE_MAX).is_some()); -static_assert!(StoreType::new(STORE_TYPE_MAX + 1).is_none()); - #[derive(Clone, Copy)] #[must_use] @@ -560,12 +436,7 @@ impl core::ops::Deref for Secret { impl Drop for Secret { fn drop(&mut self) { - for b in self.0.iter_mut() { - // SAFETY: `b` is a valid, uniquely borrowed byte for this write, and - // a volatile store is what stops the compiler discarding a wipe of - // memory nothing reads afterwards. - unsafe { core::ptr::write_volatile(b, 0) }; - } + image::wipe(&mut self.0); } } @@ -599,20 +470,6 @@ impl SksProbe { } } -struct XarsProbe { - active: bool, - captured: KVec, -} - -impl XarsProbe { - fn new() -> Self { - XarsProbe { - active: false, - captured: KVec::new(), - } - } -} - struct ScrdProbe { active: bool, captured: KVec, @@ -699,8 +556,6 @@ struct SepData { #[pin] control_wq: CondVar, - security_mode: Atomic, - sks_wedged: Atomic, sks_seq: Atomic, @@ -714,9 +569,6 @@ struct SepData { #[pin] ool_sbio: Mutex>, - #[pin] - sbio_last_header: Mutex>, - #[pin] dma_ring: Mutex>, @@ -730,8 +582,6 @@ struct SepData { templates_restored: Atomic, - restore_attempted: Atomic, - sensor_calibrated: Atomic, enrol_open: Atomic, @@ -740,23 +590,16 @@ struct SepData { enrol_material: Mutex>, #[pin] - enrol_identity_candidates: Mutex>, + enrol_identity_candidates: Mutex>, last_capture_end_ns: Atomic, device_view_synced: Atomic, - backup_bag_other: Atomic, - - borrowed_sealing_handle: Atomic, - sealing_designations: Atomic, - config_stage: Atomic, - config_writes_sent: Atomic, - keybag_designated: Atomic, - cold_prepared: Atomic, bringup_started: Atomic, + touchid_started: Atomic, bringup: Atomic, @@ -768,13 +611,6 @@ struct SepData { #[pin] sks_wq: CondVar, - #[pin] - ool_xars: Mutex>, - #[pin] - xars_probe: Mutex, - #[pin] - xars_wq: CondVar, - #[pin] ool_scrd: Mutex>, #[pin] @@ -808,6 +644,9 @@ struct SepData { #[pin] machine_refkey: Mutex>, + #[pin] + fv_volumes: Mutex>, + rng_shutdown: Atomic, rng_failures: Atomic, @@ -820,6 +659,8 @@ struct SepData { registered: Atomic, + shutting_down: Atomic, + #[pin] rx_work: Work, @@ -872,12 +713,6 @@ impl SepData { )?; let sks_declared = sks_declared_sizes(); let ool_sks = Self::alloc_ool(dev, proto::EP_SKS, SKS_ALLOC, sks_declared)?; - let ool_xars = Self::alloc_ool( - dev, - xarm::EP_XARS, - OOL_SIZE_XARS, - (OOL_SIZE_XARS, OOL_SIZE_XARS), - )?; let ool_scrd = Self::alloc_ool( dev, proto::EP_SCRD, @@ -937,36 +772,25 @@ impl SepData { endpoints <- new_mutex!(EndpointTable::new()), control <- new_mutex!(control::ControlState::new()), control_wq <- new_condvar!("SepData::control_wq"), - security_mode: Atomic::new(SECMODE_UNKNOWN), sks_seq: Atomic::new(0), sks_wedged: Atomic::new(0), bringup: Atomic::new(BRINGUP_FRESH), keybag_designated: Atomic::new(false), - cold_prepared: Atomic::new(false), bringup_started: Atomic::new(false), + touchid_started: Atomic::new(false), enrol_open: Atomic::new(false), sensor_calibrated: Atomic::new(false), templates_restored: Atomic::new(false), - restore_attempted: Atomic::new(false), enrol_material <- new_mutex!(None), enrol_identity_candidates <- new_mutex!(KVec::new()), last_capture_end_ns: Atomic::new(0), device_view_synced: Atomic::new(false), - backup_bag_other: Atomic::new(0), - borrowed_sealing_handle: Atomic::new(0), - sealing_designations: Atomic::new(0), - config_writes_sent: Atomic::new(0), - config_stage: Atomic::new(CONFIG_STAGE_NONE), phase: Atomic::new(PHASE_ATTACH), ool_xarm <- new_mutex!(Some(ool_xarm)), ool_sbio <- new_mutex!(Some(ool_sbio)), - sbio_last_header <- new_mutex!(None::<[u8; transfer::HEADER_LEN]>), ool_sks <- new_mutex!(Some(ool_sks)), sks_probe <- new_mutex!(SksProbe::new()), sks_wq <- new_condvar!("SepData::sks_wq"), - ool_xars <- new_mutex!(Some(ool_xars)), - xars_probe <- new_mutex!(XarsProbe::new()), - xars_wq <- new_condvar!("SepData::xars_wq"), ool_scrd <- new_mutex!(Some(ool_scrd)), scrd_probe <- new_mutex!(ScrdProbe::new()), scrd_wq <- new_condvar!("SepData::scrd_wq"), @@ -983,6 +807,7 @@ impl SepData { xarm <- new_mutex!(XarmState::new()), rng <- new_mutex!(None), machine_refkey <- new_mutex!(None), + fv_volumes <- new_mutex!(KVec::new()), rng_shutdown: Atomic::new(false), rng_failures: Atomic::new(0), rx: rxring::RxRing::new(), @@ -990,6 +815,7 @@ impl SepData { settle_mark: Atomic::new(0), settle_idle_ticks: Atomic::new(0), registered: Atomic::new(false), + shutting_down: Atomic::new(false), rx_work <- new_work!("SepData::rx_work"), enrol_work <- new_work!("SepData::enrol_work"), verify_work <- new_work!("SepData::verify_work"), @@ -1100,7 +926,7 @@ impl SepData { let mut remaining = time::msecs_to_jiffies(op.timeout_ms()); let mut guard = self.control.lock(); let result = loop { - if let Some((value, _msg1)) = guard.entropy_take() { + if let Some(value) = guard.entropy_take() { break Ok(value); } if self.rng_shutdown.load(Relaxed) { @@ -1128,13 +954,7 @@ impl SepData { let _ = self.control_request(&proto::op_nop(param)); } - match self.control_request(&proto::op_security_mode()) { - Ok(Some(reply)) => { - self.security_mode.store(reply.data_lo, Relaxed); - } - Ok(None) => {} - Err(_) => {} - } + let _ = self.control_request(&proto::op_security_mode()); let mut words = [0u32; 4]; let mut got = 0; @@ -1257,40 +1077,30 @@ impl SepData { store.as_mut().map(f) } - // pre-generate: an entropy draw on the drain path would deadlock on its own reply fn prepare_os_uuid(&self) { - let key = store::Key::root(0xf0); - - let existing = match self.with_host_store(|store| store.read(&key)) { - Some(result) => result, - None => return, - }; - - if let Ok(Some(value)) = existing { - if value.len() == 16 { - let mut uuid = [0u8; 16]; - uuid.copy_from_slice(&value); - self.xarm.lock().os_uuid = Some(uuid); - return; - } - } - - let mut uuid = [0u8; 16]; - if shim::random_bytes(&mut uuid).is_err() { + let hi = *module_parameters::os_uuid_hi.value(); + let lo = *module_parameters::os_uuid_lo.value(); + if hi != 0 || lo != 0 { + let mut uuid = [0u8; 16]; + uuid[..8].copy_from_slice(&hi.to_be_bytes()); + uuid[8..].copy_from_slice(&lo.to_be_bytes()); + self.xarm.lock().os_uuid = Some(uuid); + dev_info!( + self.dev, + "xART: using explicit OS UUID {:016x}-{:016x}\n", + hi, + lo + ); return; } - xarm::make_uuid_v4(&mut uuid); - match self.with_host_store(|store| store.write(&key, &uuid)) { - Some(Ok(())) => {} - Some(Err(_)) => { - return; - } - None => { - return; - } + if let Some(uuid) = dt::preboot_uuid() { + self.xarm.lock().os_uuid = Some(uuid); + dev_info!(self.dev, "xART: using /chosen/apfs-preboot-uuid\n"); + } else { + self.xarm.lock().os_uuid = None; + dev_warn!(self.dev, "xART: /chosen/apfs-preboot-uuid is unavailable\n"); } - self.xarm.lock().os_uuid = Some(uuid); } fn on_xarm(&self, msg: Message) { @@ -1456,18 +1266,6 @@ impl SepData { } - fn on_xars(&self, msg: Message) { - let mut probe = self.xars_probe.lock(); - if probe.active { - if probe.captured.len() < XARS_MAX_CAPTURE { - let _ = probe.captured.push(msg, GFP_KERNEL); - } - drop(probe); - self.xars_wq.notify_all(); - } - } - - fn on_scrd(&self, msg: Message) { let mut probe = self.scrd_probe.lock(); if probe.active { @@ -1783,6 +1581,9 @@ impl SepData { fn arm_settle(this: &Arc) { + if this.shutting_down.load(Relaxed) { + return; + } let delay = time::msecs_to_jiffies(SETTLE_MS); let _ = workqueue::system() .enqueue_delayed::, SETTLE_WORK_ID>(this.clone(), delay); @@ -1889,9 +1690,6 @@ impl SepData { proto::EP_SKS => self.on_sks(msg), - // xars: only 0x08 and 0x04 are sent; other opcodes are destructive - xarm::EP_XARS => self.on_xars(msg), - proto::EP_SCRD => self.on_scrd(msg), proto::EP_BOOT => dev_warn!( @@ -1909,7 +1707,6 @@ impl SepData { fn on_control(&self, msg: Message, f: proto::Fields) { if f.ty != proto::CONTROL_REPLY_TYPE { - self.control.lock().note_unsolicited(); return; } @@ -1963,8 +1760,9 @@ impl SepData { fn remove(&self) { - // first: resets the keyring static calls before freeing, so no keyctl op hits freed data + self.shutting_down.store(true, Relaxed); trusted::unregister(); + self.unregister_fv_kernel(); let dev = self.bio_dev.lock().take(); if dev.is_some() { @@ -1976,17 +1774,35 @@ impl SepData { self.rng_shutdown.store(true, Relaxed); self.control_wq.notify_all(); + self.sbio_wq.notify_all(); + self.sks_wq.notify_all(); + self.scrd_wq.notify_all(); if let Some(mut handle) = self.rng.lock().take() { handle.unregister(); } - // stop the mailbox last: no callbacks after this *self.mbox.lock() = None; + // SAFETY: all four pointers refer to pinned work fields in `self`. + // Shutdown blocks requeueing, and the mailbox can no longer add work. + unsafe { + sep_cancel_work_sync(Work::raw_get(core::ptr::addr_of!(self.rx_work)).cast()); + sep_cancel_work_sync(Work::raw_get(core::ptr::addr_of!(self.enrol_work)).cast()); + sep_cancel_work_sync(Work::raw_get(core::ptr::addr_of!(self.verify_work)).cast()); + sep_cancel_delayed_work_sync( + DelayedWork::raw_as_work(core::ptr::addr_of!(self.settle_work)).cast(), + ); + } + let _ = self.store.lock().take(); let _ = self.host_store.lock().take(); - for slot in [&self.ool_xarm, &self.ool_sbio, &self.ool_sks] { + for slot in [ + &self.ool_xarm, + &self.ool_sbio, + &self.ool_sks, + &self.ool_scrd, + ] { if let Some(buffers) = slot.lock().take() { if buffers.registered { core::mem::forget(buffers); @@ -2086,6 +1902,9 @@ impl MailCallback for SepData { type Data = Arc; fn recv_message(data: ::Borrowed<'_>, msg: Message) { + if data.shutting_down.load(Relaxed) { + return; + } if !data.rx.push(msg) && data.rx.dropped() == 1 { dev_err!( data.dev, @@ -2103,6 +1922,9 @@ impl WorkItem for SepData { type Pointer = Arc; fn run(this: Arc) { + if this.shutting_down.load(Relaxed) { + return; + } if this.drain() > 0 { SepData::arm_settle(&this); } @@ -2113,6 +1935,9 @@ impl WorkItem for SepData { type Pointer = Arc; fn run(this: Arc) { + if this.shutting_down.load(Relaxed) { + return; + } SepData::settle_tick(&this); } } @@ -2121,6 +1946,9 @@ impl WorkItem for SepData { type Pointer = Arc; fn run(this: Arc) { + if this.shutting_down.load(Relaxed) { + return; + } this.run_enrolment(); } } @@ -2129,6 +1957,9 @@ impl WorkItem for SepData { type Pointer = Arc; fn run(this: Arc) { + if this.shutting_down.load(Relaxed) { + return; + } this.run_verify(); } } @@ -2150,6 +1981,12 @@ impl platform::Driver for SepDriver { _info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { let dev: &device::Device> = pdev.as_ref(); + if *module_parameters::provision_keybag.value() != 0 + && *module_parameters::xart_writes.value() == 0 + { + dev_err!(dev, "provision_keybag=1 requires xart_writes=1\n"); + return Err(EINVAL); + } let sep_node = dt::DtNode::of_device(dev).ok_or(ENODEV)?; if dt::registration_already_sent(&sep_node) { @@ -2175,7 +2012,9 @@ impl platform::Driver for SepDriver { data.attach(&sep_node)?; - data.attach_sensor(); + if let Err(e) = data.register_fv_kernel() { + dev_err!(dev, "could not register the FileVault kernel API: {:?}\n", e); + } if let Err(e) = trusted::register(data.clone()) { dev_warn!(data.dev, "trusted-keys: registration failed ({:?})\n", e); @@ -2228,5 +2067,17 @@ module! { default: 0, description: "Allow writes to the validated shared xART mapping", }, + provision_keybag: u8 { + default: 0, + description: "Create the Linux identity keybag when none exists; requires xart_writes=1", + }, + os_uuid_hi: u64 { + default: 0, + description: "High 64 bits of an explicit xART OS UUID", + }, + os_uuid_lo: u64 { + default: 0, + description: "Low 64 bits of an explicit xART OS UUID", + }, }, } diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index e39f036142825..34c34f29bc68b 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -18,6 +18,11 @@ #include +struct apple_sep_fv_new_file_key; + +void sep_cancel_work_sync(void *work); +void sep_cancel_delayed_work_sync(void *work); + /* -- hwrng_shim.c ------------------------------------------------------- */ void *sep_hwrng_alloc(void); @@ -48,6 +53,8 @@ int sep_sha256(const void *a, size_t alen, const void *b, size_t blen, /* -- crypto_shim.c ------------------------------------------------------ */ +int sep_random_bytes(void *buf, size_t len); + /* HMAC-SHA256 over one message, 32 bytes out. */ int sep_hmac_sha256(const void *key, size_t keylen, const void *data, size_t datalen, u8 *out); @@ -61,12 +68,40 @@ int sep_gcm(int encrypt, const void *key, size_t keylen, const void *iv, size_t ivlen, size_t aadlen, void *buf, size_t buflen, size_t datalen); -/* - * Fill `buf` from the kernel CSPRNG, seeded, or fail. For key-bag secrets only; - * anything measuring the enclave's own entropy (0x59 device-key probe, - * /dev/hwrng) stays on SEP. - */ -int sep_random_bytes(void *buf, size_t len); +/* -- fv_shim.c -------------------------------------------------------- */ + +struct apple_sep_fv_key; + +struct sep_fv_ops { + int (*unwrap_media_key)(void *context, const u8 *wrapped, + size_t wrapped_len, u32 protection_class, + struct apple_sep_fv_key *key); + int (*unwrap_volume_key)(void *context, const u8 *secret, + size_t secret_len, const u8 *unlock_record, + size_t unlock_record_len, const u8 *volume_key, + size_t volume_key_len, + struct apple_sep_fv_key *key); + int (*load_class_keys)(void *context, const u8 volume_uuid[16], + const u8 *secret, + size_t secret_len, const u8 *unlock_record, + size_t unlock_record_len, const u8 *volume_key, + size_t volume_key_len); + int (*unload_class_keys)(void *context, const u8 volume_uuid[16], + const u8 *volume_key, + size_t volume_key_len); + int (*unwrap_file_key)(void *context, const u8 volume_uuid[16], + u32 protection_class, const u8 *wrapped_ekwk, + size_t wrapped_ekwk_len, const u8 *wrapped_ek, + size_t wrapped_ek_len, + struct apple_sep_fv_key *key); + int (*new_file_key)(void *context, const u8 volume_uuid[16], + u32 protection_class, u64 crypto_id, + u16 key_revision, + struct apple_sep_fv_new_file_key *key); +}; + +int sep_fv_register_v2(void *context, const struct sep_fv_ops *ops); +void sep_fv_unregister_v2(void *context); /* -- p256_shim.c ------------------------------------------------------- */ diff --git a/drivers/soc/apple/shim.rs b/drivers/soc/apple/shim.rs index a615b3aee0a96..f02a9b86e32ef 100644 --- a/drivers/soc/apple/shim.rs +++ b/drivers/soc/apple/shim.rs @@ -12,21 +12,18 @@ extern "C" { fn sep_store_open_block(path: *const c_char, writable: c_int) -> *mut c_void; fn sep_store_close(handle: *mut c_void); fn sep_store_size(handle: *mut c_void) -> i64; - fn sep_store_read(handle: *mut c_void, off: i64, buf: *mut c_void, len: usize) - -> c_long; - fn sep_store_write( - handle: *mut c_void, - off: i64, - buf: *const c_void, - len: usize, - ) -> c_long; + fn sep_store_read(handle: *mut c_void, off: i64, buf: *mut c_void, len: usize) -> c_long; + fn sep_store_write(handle: *mut c_void, off: i64, buf: *const c_void, len: usize) -> c_long; fn sep_store_sync(handle: *mut c_void) -> c_int; fn sep_random_bytes(buf: *mut c_void, len: usize) -> c_int; } pub(crate) fn random_bytes(buf: &mut [u8]) -> Result<()> { - // SAFETY: `buf` is writable for exactly `buf.len()` bytes. - kernel::error::to_result(unsafe { sep_random_bytes(buf.as_mut_ptr().cast(), buf.len()) }) + // SAFETY: `buf.as_mut_ptr()` is valid for writes of `buf.len()` bytes for + // the duration of the call, and `sep_random_bytes` writes exactly that many + // bytes (via `get_random_bytes`) and retains no reference to the buffer. + let ret = unsafe { sep_random_bytes(buf.as_mut_ptr().cast(), buf.len()) }; + kernel::error::to_result(ret) } /// Backing-store file handle. @@ -199,9 +196,6 @@ extern "C" { fn sep_bio_boottime_ns() -> u64; } -extern "C" { -} - pub(crate) fn capable_admin() -> bool { // SAFETY: no preconditions; reads the current task's credentials. unsafe { sep_bio_capable_admin() != 0 } diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index f30f4870db076..4d68f40bab0d7 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -3,13 +3,41 @@ //! SEP key store (SKS, endpoint `0x12`): key-bag and key-management request/reply //! framing, DER-imaged request builders, and lock-state control. -#![allow(dead_code)] - use super::*; use kernel::prelude::*; use kernel::soc::apple::mailbox::Message; use crate::proto::*; +struct KeybagCreateIntent<'a> { + dev: &'a device::Device, + slot: keybag::Slot, + sent: bool, +} + +impl<'a> KeybagCreateIntent<'a> { + fn new(dev: &'a device::Device, slot: keybag::Slot) -> Option { + if let Err(e) = keybag::write_intent(slot) { + dev_err!(dev, "sks: cannot persist keybag-create intent: {:?}\n", e); + return None; + } + Some(Self { dev, slot, sent: false }) + } + + fn sending(&mut self) { + self.sent = true; + } +} + +impl Drop for KeybagCreateIntent<'_> { + fn drop(&mut self) { + if !self.sent { + if let Err(e) = keybag::mark_refused(self.slot) { + dev_err!(self.dev, "sks: cannot mark unsent keybag create retryable: {:?}\n", e); + } + } + } +} + impl SepData { pub(crate) fn on_sks(&self, msg: Message) { let r = crate::sks::decode_sks(&msg); @@ -30,6 +58,7 @@ impl SepData { .copied(); drop(probe); if let Some(a) = matched { + let _ = self.sks_zero_buffers(); self.sks_wedged.store(0, Relaxed); dev_warn!(self.dev, "sks: late answer to {} after {} ms; wedge lifted\n", a.label, a.waited_ms); } @@ -130,6 +159,7 @@ impl SepData { self.sks_arm(label); if self.send(msg).is_err() { self.sks_disarm(); + let _ = self.sks_zero_buffers(); return None; } let started_ns = crate::shim::boottime_ns(); @@ -191,6 +221,10 @@ impl SepData { } } + // The correlated reply means the enclave has finished with both OOL + // buffers. Keep only the private response copy returned to the caller. + let _ = self.sks_zero_buffers(); + Some(SksOutcome { reply, response }) } @@ -325,6 +359,15 @@ impl SepData { self.sks_seal(&op, &body) } + fn sks_req_copy_keybag(&self, handle: crate::sks::KeyBagHandle) -> Result { + let op = crate::sks::sks_copy_keybag(); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(handle.value())?; + self.sks_seal(&op, &body) + } + /// `0x02` copy the designated biometric identity bag. fn sks_req_copy_keybag_special(&self, handle: crate::sks::SpecialHandle) -> Result { let op = crate::sks::sks_copy_keybag(); @@ -352,6 +395,107 @@ impl SepData { }) } + fn sks_req_create_identity_keybag( + &self, + secret: &[u8], + uuid: &[u8; crate::sks::SKS_IDENTITY_UUID_LEN], + proof: keybag::NoStoredKeyBag, + ) -> Result { + if proof.slot() != keybag::Slot::Identity { + return Err(EINVAL); + } + let mut body = image::Body::new(); + body.put_u32(crate::sks::SKS_CREATE_VARIANT_IDENTITY)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_u32(crate::sks::CreateFlags::none().value())?; + body.put_i32(crate::sks::SpecialHandle::first_identity().value())?; + body.put_blob(secret)?; + body.put_blob(&[])?; + body.put_blob(uuid)?; + body.put_blob(&[])?; + body.put_u64(0)?; + body.put_u64(0)?; + body.put_blob(&[])?; + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_create(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { name: crate::sks::SKS_CREATE_NAME, msg, img }) + } + + pub(crate) fn sks_provision_identity_keybag(&self) -> bool { + let proof = match keybag::read(keybag::Slot::Identity) { + Ok(keybag::State::Present(_)) => return true, + Ok(keybag::State::Absent(proof)) => proof, + Err(e) => { + dev_err!(self.dev, "sks: identity keybag state is ambiguous: {:?}\n", e); + return false; + } + }; + let slot = proof.slot(); + let mut intent = match KeybagCreateIntent::new(&self.dev, slot) { + Some(intent) => intent, + None => return false, + }; + let mut secret_bytes = KVec::new(); + if secret_bytes.resize(SKS_SECRET_LEN, 0, GFP_KERNEL).is_err() + || shim::random_bytes(&mut secret_bytes).is_err() + { + return false; + } + let secret = Secret(secret_bytes); + let mut uuid = [0u8; crate::sks::SKS_IDENTITY_UUID_LEN]; + if shim::random_bytes(&mut uuid).is_err() { + return false; + } + uuid[6] = (uuid[6] & 0x0f) | 0x40; + uuid[8] = (uuid[8] & 0x3f) | 0x80; + + let request = match self.sks_req_create_identity_keybag(&secret, &uuid, proof) { + Ok(request) => request, + Err(e) => { + dev_err!(self.dev, "sks: cannot build identity keybag request: {:?}\n", e); + return false; + } + }; + intent.sending(); + let Some(out) = self.sks_exchange(request.name, request.msg, &request.img) else { + return false; + }; + let Some(body) = self.sks_report_response(crate::sks::SKS_CREATE_NAME, &out) else { + return false; + }; + if out.reply.status != 0 || body.len() < 12 { + dev_err!(self.dev, "sks: CREATE_KEYBAG failed: mailbox {}, body {} bytes\n", out.reply.status, body.len()); + return false; + } + let variant = u32::from_le_bytes(body[0..4].try_into().unwrap()); + let raw_handle = i32::from_le_bytes(body[4..8].try_into().unwrap()); + let Some((_fv_data, end)) = image::read_blob(body, 8) else { + return false; + }; + if variant != crate::sks::SKS_CREATE_VARIANT_IDENTITY || raw_handle < 0 || end != body.len() { + dev_err!(self.dev, "sks: invalid CREATE_KEYBAG reply: variant {}, handle {}\n", variant, raw_handle); + return false; + } + let handle = crate::sks::KeyBagHandle::from_create_reply(raw_handle); + let Some(bag_uuid) = self.sks_read_uuid(handle) else { + return false; + }; + let Some(out) = self.sks_send(self.sks_req_copy_keybag(handle)) else { + return false; + }; + let Some(wrapped) = self.wrapped_from_copy_reply(&out, c"new identity keybag") else { + return false; + }; + if let Err(e) = keybag::write_bag_uuid(slot, &wrapped, &bag_uuid, &secret) { + dev_err!(self.dev, "sks: cannot commit identity keybag: {:?}\n", e); + return false; + } + let _ = self.sks_send(self.sks_req_unload_keybag(handle)); + dev_info!(self.dev, "sks: identity keybag provisioned\n"); + true + } + pub(crate) fn sks_send(&self, req: Result) -> Option { match req { Ok(r) => self.sks_exchange(r.name, r.msg, &r.img), @@ -499,16 +643,21 @@ impl SepData { return None; } if body.len() != SKS_LOAD_REPLY_LEN { + dev_warn!(self.dev, "sks: LOAD_KEYBAG returned {} bytes, expected {}\n", body.len(), SKS_LOAD_REPLY_LEN); return None; } let status = i32::from_le_bytes([body[0], body[1], body[2], body[3]]); let handle = i32::from_le_bytes([body[4], body[5], body[6], body[7]]); if status != 0 || handle < 0 { + dev_warn!(self.dev, "sks: LOAD_KEYBAG operation status {}, handle {}\n", status, handle); return None; } let handle = crate::sks::KeyBagHandle::from_load_reply(handle); - let uuid = self.sks_read_uuid(handle)?; + let Some(uuid) = self.sks_read_uuid(handle) else { + dev_warn!(self.dev, "sks: loaded keybag has no readable UUID\n"); + return None; + }; if uuid == *stored.uuid() { Some((handle, uuid)) @@ -636,12 +785,8 @@ impl SepData { } } -const SKS_SELECTOR_MAX: u8 = 0x5f; - pub(crate) const SKS_REPLY_BIT: u8 = 0x80; -const OP_SKS_REWRAP_FORBIDDEN: u8 = 0x0f; - pub(crate) struct SksOp { opcode: u8, name: &'static CStr, @@ -656,47 +801,57 @@ impl SksOp { } } -const OP_SKS_DEVICE_STATE: u8 = 0x19; -pub(crate) fn sks_get_device_state() -> SksOp { - SksOp { - opcode: OP_SKS_DEVICE_STATE, - name: c"GET_DEVICE_STATE", - } -} +const OP_SKS_UNWRAP_PFK: u8 = 0x09; +pub(crate) const SKS_UNWRAP_PFK_NAME: &CStr = c"UNWRAP_PFK"; -const OP_SKS_GET_CONFIGURATION: u8 = 0x23; +const OP_SKS_NEW_PFK: u8 = 0x10; +pub(crate) const SKS_NEW_PFK_NAME: &CStr = c"NEW_PFK"; -const OP_SKS_SET_CONFIGURATION: u8 = 0x24; -static_assert!(OP_SKS_SET_CONFIGURATION != OP_SKS_GET_CONFIGURATION); +const OP_SKS_UNWRAP_MEDIA_KEY: u8 = 0x32; +pub(crate) const SKS_UNWRAP_MEDIA_KEY_NAME: &CStr = c"UNWRAP_MEDIA_KEY"; -const OP_SKS_NEW_PFK: u8 = 0x10; -static_assert!(OP_SKS_NEW_PFK != 0x0f); -static_assert!(OP_SKS_NEW_PFK != 0x09); +const OP_SKS_UNWRAP_VEK: u8 = 0x41; +pub(crate) const SKS_UNWRAP_VEK_NAME: &CStr = c"FV_UNWRAP_VEK"; + +const OP_SKS_SET_PROTECTION: u8 = 0x47; +pub(crate) const SKS_SET_PROTECTION_NAME: &CStr = c"FV_SET_PROTECTION"; -// FileVault seal order: 0x42 KEK, then 0x40 VEK, then 0x41 install; 0x47 clear forbidden -pub(crate) const OP_SKS_FV_NEW_VEK: u8 = 0x40; -pub(crate) const OP_SKS_FV_UNWRAP_VEK: u8 = 0x41; -pub(crate) const OP_SKS_FV_NEW_KEK: u8 = 0x42; +const OP_SKS_GET_BLOB_STATE: u8 = 0x48; +pub(crate) const SKS_GET_BLOB_STATE_NAME: &CStr = c"FV_GET_BLOB_STATE"; -const OP_SKS_GENERIC_OPERATION: u8 = 0x1a; const OP_SKS_PERFORM_OPERATION: u8 = 0x22; -const OP_SKS_IDENTITY_OPERATION: u8 = 0x51; pub(crate) const SKS_PERFORM_OP_NAME: &CStr = c"PERFORM_OPERATION"; -static_assert!(OP_SKS_PERFORM_OPERATION != OP_SKS_GET_CONFIGURATION); -static_assert!(OP_SKS_PERFORM_OPERATION != OP_SKS_SET_CONFIGURATION); pub(crate) fn encode_sks_perform_operation(seq: Sequence, len: ImageLen) -> Option { let msg = encode_sks_raw(OP_SKS_PERFORM_OPERATION, seq.value(), len.value()); Some(msg) } -pub(crate) fn encode_sks_fv(selector: u8, seq: Sequence, len: ImageLen) -> Option { - let msg = encode_sks_raw(selector, seq.value(), len.value()); +pub(crate) fn encode_sks_unwrap_media_key(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_UNWRAP_MEDIA_KEY, seq.value(), len.value()); + Some(msg) +} + +pub(crate) fn encode_sks_unwrap_vek(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_UNWRAP_VEK, seq.value(), len.value()); Some(msg) } -const OP_SKS_LAST_USER_OPERATION: u8 = 0x53; -static_assert!(OP_SKS_LAST_USER_OPERATION != 0x56); +pub(crate) fn encode_sks_unwrap_pfk(seq: Sequence, len: ImageLen) -> Message { + encode_sks_raw(OP_SKS_UNWRAP_PFK, seq.value(), len.value()) +} + +pub(crate) fn encode_sks_new_pfk(seq: Sequence, len: ImageLen) -> Message { + encode_sks_raw(OP_SKS_NEW_PFK, seq.value(), len.value()) +} + +pub(crate) fn encode_sks_set_protection(seq: Sequence, len: ImageLen) -> Message { + encode_sks_raw(OP_SKS_SET_PROTECTION, seq.value(), len.value()) +} + +pub(crate) fn encode_sks_get_blob_state(seq: Sequence, len: ImageLen) -> Message { + encode_sks_raw(OP_SKS_GET_BLOB_STATE, seq.value(), len.value()) +} const OP_SKS_CAPABILITIES: u8 = 0x4d; pub(crate) fn sks_get_capabilities() -> SksOp { SksOp { @@ -722,40 +877,29 @@ pub(crate) fn sks_copy_keybag() -> SksOp { } const OP_SKS_CREATE_KEYBAG: u8 = 0x01; +pub(crate) const SKS_CREATE_NAME: &CStr = c"CREATE_KEYBAG"; +pub(crate) const SKS_CREATE_VARIANT_IDENTITY: u32 = 5; +pub(crate) const SKS_IDENTITY_UUID_LEN: usize = 16; -const OP_SKS_LOAD_KEYBAG: u8 = 0x03; - -pub(crate) const SKS_LOAD_NAME: &CStr = c"LOAD_KEYBAG"; - -const OP_SKS_CHANGE_LOCK_STATE: u8 = 0x04; - -pub(crate) const SKS_LOCK_STATE_NAME: &CStr = c"CHANGE_LOCK_STATE"; - -const OP_SKS_TOKEN_CREATE: u8 = 0x1c; - -pub(crate) const SKS_TOKEN_CREATE_NAME: &CStr = c"AUTH_TOKEN_CREATE"; - -const OP_SKS_TOKEN_VERIFY: u8 = 0x1d; - -static_assert!(OP_SKS_TOKEN_VERIFY == OP_SKS_TOKEN_CREATE + 1); - -#[derive(Clone, Copy)] -pub(crate) struct NewDeviceState(i32); +pub(crate) struct CreateFlags(u32); -impl NewDeviceState { - pub(crate) const UNLOCKED: NewDeviceState = NewDeviceState(0); +impl CreateFlags { + pub(crate) const fn none() -> Self { + Self(0) + } - pub(crate) const fn value(&self) -> i32 { + pub(crate) const fn value(&self) -> u32 { self.0 } } -static_assert!(NewDeviceState::UNLOCKED.value() == 0); -const OP_SKS_DEVICE_STATE_TRANSITION: u8 = 0x18; -static_assert!(OP_SKS_DEVICE_STATE_TRANSITION + 1 == 0x19); +const OP_SKS_LOAD_KEYBAG: u8 = 0x03; -pub(crate) const SKS_DEVICE_STATE_REPLY_LEN: usize = 20; -static_assert!(SKS_DEVICE_STATE_REPLY_LEN == 4 + 8 + 8); +pub(crate) const SKS_LOAD_NAME: &CStr = c"LOAD_KEYBAG"; + +const OP_SKS_CHANGE_LOCK_STATE: u8 = 0x04; + +pub(crate) const SKS_LOCK_STATE_NAME: &CStr = c"CHANGE_LOCK_STATE"; const OP_SKS_VERIFY_SECRET: u8 = 0x21; pub(crate) const SKS_VERIFY_SECRET_NAME: &CStr = c"VERIFY_SECRET"; @@ -766,25 +910,18 @@ pub(crate) fn encode_sks_verify_secret(seq: Sequence, len: ImageLen) -> Option KeyBagHandle { + KeyBagHandle(v) + } + pub(crate) const fn from_load_reply(v: i32) -> KeyBagHandle { KeyBagHandle(v) } @@ -797,20 +934,7 @@ impl KeyBagHandle { const OP_SKS_DESIGNATE_KEYBAG: u8 = 0x0d; pub(crate) const SKS_DESIGNATE_VARIANT: u32 = 1; - -pub(crate) const SKS_DESIGNATE_VARIANT_GENERIC: u32 = 0; - static_assert!(SKS_DESIGNATE_VARIANT == 1); -static_assert!(SKS_DESIGNATE_VARIANT_GENERIC == 0); -static_assert!(SKS_DESIGNATE_VARIANT != SKS_DESIGNATE_VARIANT_GENERIC); - -pub(crate) const SKS_CREATE_VARIANT_IDENTITY: u32 = 5; -static_assert!(SKS_CREATE_VARIANT_IDENTITY != 1); - -pub(crate) const SKS_IDENTITY_UUID_LEN: usize = 16; -static_assert!(SKS_IDENTITY_UUID_LEN == crate::sbio::IDENTITY_UUID_LEN); - -pub(crate) const SKS_IDENTITY_USER_ID: i32 = 1000; pub(crate) const SKS_DESIGNATE_USER_MIN: i32 = 10; static_assert!(SKS_DESIGNATE_USER_MIN > 0); @@ -836,9 +960,10 @@ impl DesignateUser { pub(crate) struct SpecialHandle(i32); impl SpecialHandle { -} + pub(crate) const fn first_identity() -> SpecialHandle { + SpecialHandle(-1) + } -impl SpecialHandle { pub(crate) const fn value(&self) -> i32 { self.0 } @@ -846,37 +971,6 @@ impl SpecialHandle { pub(crate) const SKS_AUTH_TOKEN_LEN: usize = 16; -pub(crate) struct AuthToken([u8; SKS_AUTH_TOKEN_LEN]); - -impl AuthToken { - pub(crate) fn from_reply(body: &[u8]) -> Option { - if body.len() < 8 { - return None; - } - let status = i32::from_le_bytes([body[0], body[1], body[2], body[3]]); - if status != 0 { - return None; - } - let len = u32::from_le_bytes([body[4], body[5], body[6], body[7]]) as usize; - if len != SKS_AUTH_TOKEN_LEN || body.len() < 8 + len { - return None; - } - let mut out = [0u8; SKS_AUTH_TOKEN_LEN]; - out.copy_from_slice(&body[8..8 + len]); - Some(AuthToken(out)) - } - -} - -impl Drop for AuthToken { - fn drop(&mut self) { - for b in self.0.iter_mut() { - // SAFETY: a valid, uniquely borrowed byte; volatile so the wipe is not elided. - unsafe { core::ptr::write_volatile(b, 0) }; - } - } -} - pub(crate) struct Designation { source: KeyBagHandle, user: DesignateUser, @@ -899,43 +993,7 @@ impl Designation { pub(crate) const SKS_DESIGNATE_REPLY_LEN: usize = 4; pub(crate) const SKS_DESIGNATE_FLAGS: u64 = 0; -pub(crate) const SKS_DESIGNATE_FLAGS_DEVICE_KEYBAG: u64 = 0x100; static_assert!(SKS_DESIGNATE_FLAGS == 0); -static_assert!(SKS_DESIGNATE_FLAGS != SKS_DESIGNATE_FLAGS_DEVICE_KEYBAG); - -// keybag_flags sits at +0x64 in a 0x01 body -pub(crate) struct CreateFlags(u32); - -pub(crate) const SKS_KEYBAG_FLAG_MAX: u32 = 0xff; -static_assert!((SKS_KEYBAG_FLAG_MAX as u64) < SKS_DESIGNATE_FLAGS_DEVICE_KEYBAG); - -impl CreateFlags { - pub(crate) const fn new(bits: u32) -> Option { - if bits <= SKS_KEYBAG_FLAG_MAX { - Some(CreateFlags(bits)) - } else { - None - } - } - - pub(crate) const fn none() -> CreateFlags { - CreateFlags(0) - } - - pub(crate) const fn value(&self) -> u32 { - self.0 - } -} - -static_assert!(CreateFlags::new(SKS_DESIGNATE_FLAGS_DEVICE_KEYBAG as u32).is_none()); -static_assert!(CreateFlags::new(0x100).is_none()); -static_assert!(CreateFlags::new(0x101).is_none()); -static_assert!(CreateFlags::new(0x1ff).is_none()); -static_assert!(CreateFlags::new(u32::MAX).is_none()); -static_assert!(CreateFlags::new(0x80).is_some()); -static_assert!(CreateFlags::none().value() == 0); - -// device keybag = 0x0d with a u64 flags of 0x100 at +0x78 pub(crate) fn encode_sks_designate(seq: Sequence, len: ImageLen) -> Option { let msg = encode_sks_raw(OP_SKS_DESIGNATE_KEYBAG, seq.value(), len.value()); @@ -944,55 +1002,6 @@ pub(crate) fn encode_sks_designate(seq: Sequence, len: ImageLen) -> Option Option { - let msg = encode_sks_raw(OP_SKS_SET_BACKUP_BAG, seq.value(), len.value()); - Some(msg) -} - -const OP_SKS_SET_ENV: u8 = 0x2a; - -pub(crate) const SKS_CLIENT_ID_ALT: u64 = u64::from_be_bytes(*b"LINUXSKT"); -static_assert!(SKS_CLIENT_ID_ALT != SKS_CLIENT_ID); -static_assert!(SKS_CLIENT_ID_ALT != SKS_CLIENT_ID_SEALING); -const fn client_id_byte_distance(a: u64, b: u64) -> u32 { - let (x, y) = (a.to_be_bytes(), b.to_be_bytes()); - let mut differing = 0; - let mut i = 0; - while i < 8 { - if x[i] != y[i] { - differing += 1; - } - i += 1; - } - differing -} -static_assert!(client_id_byte_distance(SKS_CLIENT_ID, SKS_CLIENT_ID_ALT) == 1); - const OP_SKS_UNLOAD_KEYBAG: u8 = 0x05; pub(crate) const SKS_UNLOAD_NAME: &CStr = c"UNLOAD_KEYBAG"; @@ -1016,29 +1025,8 @@ impl Sequence { } } -pub(crate) const SEQ_SPACE: usize = 64; - -pub(crate) const SEQ_FIRST_REUSE: u8 = 32; - static_assert!(Sequence::from_counter(0).value() != Sequence::from_counter(1).value()); -static_assert!( - Sequence::from_counter(0).value() == Sequence::from_counter(SEQ_FIRST_REUSE).value() -); -static_assert!(SEQ_SPACE == 64); - -const fn every_sequence_is_above_the_selector_range() -> bool { - let mut n = 0u8; - loop { - if Sequence::from_counter(n).value() <= SKS_SELECTOR_MAX { - return false; - } - if n == 0xff { - return true; - } - n += 1; - } -} -static_assert!(every_sequence_is_above_the_selector_range()); +static_assert!(Sequence::from_counter(0).value() == Sequence::from_counter(32).value()); #[derive(Clone, Copy)] pub(crate) struct ImageLen(u16); @@ -1078,13 +1066,12 @@ pub(crate) fn encode_sks_load(seq: Sequence, len: ImageLen) -> Option { Some(msg) } -pub(crate) fn encode_sks_change_lock_state(seq: Sequence, len: ImageLen) -> Option { - let msg = encode_sks_raw(OP_SKS_CHANGE_LOCK_STATE, seq.value(), len.value()); - Some(msg) +pub(crate) fn encode_sks_create(seq: Sequence, len: ImageLen) -> Option { + Some(encode_sks_raw(OP_SKS_CREATE_KEYBAG, seq.value(), len.value())) } -pub(crate) fn encode_sks_token_create(seq: Sequence, len: ImageLen) -> Option { - let msg = encode_sks_raw(OP_SKS_TOKEN_CREATE, seq.value(), len.value()); +pub(crate) fn encode_sks_change_lock_state(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_CHANGE_LOCK_STATE, seq.value(), len.value()); Some(msg) } @@ -1092,7 +1079,6 @@ pub(crate) struct SksReply { pub(crate) selector: u8, pub(crate) seq: u8, pub(crate) status: i8, - pub(crate) flags: u16, pub(crate) response_size: u16, } @@ -1102,14 +1088,6 @@ pub(crate) fn decode_sks(msg: &Message) -> SksReply { selector: b[1] & !SKS_REPLY_BIT, seq: b[2], status: b[3] as i8, - flags: u16::from_le_bytes([b[4], b[5]]), response_size: u16::from_le_bytes([b[6], b[7]]), } } - -pub(crate) const SKS_STATUS_MALFORMED: i8 = -13; -static_assert!(SKS_STATUS_MALFORMED as u8 == 0xf3); - -pub(crate) const SKS_STATUS_REFUSED: i8 = -19; -static_assert!(SKS_STATUS_REFUSED != SKS_STATUS_MALFORMED); -static_assert!(SKS_STATUS_REFUSED != 0); diff --git a/drivers/soc/apple/store.rs b/drivers/soc/apple/store.rs index 1bb97fb92ccb8..dc37025ce3b8a 100644 --- a/drivers/soc/apple/store.rs +++ b/drivers/soc/apple/store.rs @@ -24,11 +24,6 @@ pub(crate) struct Key { } impl Key { - #[allow(dead_code)] - pub(crate) const fn new(kind: u8, uuid: [u8; 16]) -> Key { - Key { kind, uuid } - } - pub(crate) const fn root(kind: u8) -> Key { Key { kind, @@ -61,15 +56,7 @@ impl Slot { #[derive(Clone, Copy, PartialEq, Eq)] enum Intent { None, - Write { - slot: u16, - kind: u8, - }, - #[allow(dead_code)] - Delete { - slot: u16, - kind: u8, - }, + Write { slot: u16, kind: u8 }, } const SB_MAGIC: usize = 0; @@ -85,7 +72,6 @@ const SLOT_ENTRY_SIZE: usize = 24; const INTENT_NONE: u8 = 0; const INTENT_WRITE: u8 = 1; -const INTENT_DELETE: u8 = 2; // Not internally locked; the caller holds a mutex around all access. pub(crate) struct Store { @@ -197,7 +183,6 @@ impl Store { let (ikind, islot, itype) = match intent { Intent::None => (INTENT_NONE, 0u16, 0u8), Intent::Write { slot, kind } => (INTENT_WRITE, slot, kind), - Intent::Delete { slot, kind } => (INTENT_DELETE, slot, kind), }; sb[SB_INTENT_KIND] = ikind; sb[SB_INTENT_SLOT..SB_INTENT_SLOT + 2].copy_from_slice(&islot.to_le_bytes()); @@ -279,28 +264,6 @@ impl Store { self.generation = self.generation.wrapping_add(1); self.commit() } - - #[allow(dead_code)] - pub(crate) fn delete(&mut self, key: &Key) -> Result { - if !(0xf0..=0xf5).contains(&key.kind) { - return Err(EINVAL); - } - let Some(idx) = self.find(key) else { - return Ok(false); - }; - - self.write_superblock(Intent::Delete { - slot: idx as u16, - kind: key.kind, - })?; - self.file.sync()?; - - self.slots[idx] = Slot::FREE; - self.generation = self.generation.wrapping_add(1); - self.commit()?; - Ok(true) - } - } pub(crate) const fn crc16_ccitt_false(data: &[u8]) -> u16 { diff --git a/drivers/soc/apple/transfer.rs b/drivers/soc/apple/transfer.rs index 67cf7ed13c312..1f77f45162576 100644 --- a/drivers/soc/apple/transfer.rs +++ b/drivers/soc/apple/transfer.rs @@ -1,8 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj -//! The generic transfer layer. - use kernel::prelude::*; pub(crate) const MARKER_FIRST: u8 = 0xFC; @@ -140,7 +138,6 @@ struct Active { } pub(crate) struct Completed { - pub(crate) opcode: u32, pub(crate) status: DeviceStatus, pub(crate) payload: KVec, } @@ -169,10 +166,10 @@ pub(crate) enum Progress { Complete, /// A stray chunk, dropped without touching transfer state — distinct from /// `Failed`, which kills the transfer. - Ignored(&'static CStr), + Ignored, Grant, - Notification { tag: u8, opcode: u32 }, - Failed(&'static CStr), + Notification, + Failed, } pub(crate) struct Reassembly { @@ -206,10 +203,6 @@ impl Reassembly { Ok(()) } - pub(crate) fn awaiting(&self) -> Option { - self.active.as_ref().map(|a| a.opcode) - } - pub(crate) fn begin_send(&mut self, opcode: u32) { self.sending = Some(opcode); self.grants = 0; @@ -237,9 +230,8 @@ impl Reassembly { } pub(crate) fn abort_with(&mut self, status: DeviceStatus) { - if let Some(active) = self.active.take() { + if self.active.take().is_some() { self.done = Some(Completed { - opcode: active.opcode, status, payload: KVec::new(), }); @@ -252,10 +244,7 @@ impl Reassembly { pub(crate) fn on_chunk(&mut self, marker: u8, packet: &Packet, payload: &[u8]) -> Progress { if marker < MARKER_FIRST { - return Progress::Notification { - tag: marker, - opcode: packet.opcode, - }; + return Progress::Notification; } // 0xFE is flow control for the request being sent, answered from `sending`. @@ -266,12 +255,12 @@ impl Reassembly { self.grants = self.grants.saturating_add(1); Progress::Grant } - None => Progress::Ignored(c"a 0xFE arrived with nothing being sent"), + None => Progress::Ignored, }; } let Some(active_opcode) = self.active.as_ref().map(|a| a.opcode) else { - return Progress::Ignored(c"no transfer outstanding"); + return Progress::Ignored; }; // Checked before the opcode test below, deliberately. @@ -287,34 +276,34 @@ impl Reassembly { // The device echoes the opcode on a data chunk. if packet.opcode != active_opcode { - return Progress::Ignored(c"chunk belongs to a different opcode"); + return Progress::Ignored; } if packet.version != VERSION { self.fail(); - return Progress::Failed(c"header version is not 1"); + return Progress::Failed; } if packet.chunk as usize != payload.len() { self.fail(); - return Progress::Failed(c"chunk length disagrees with the payload taken"); + return Progress::Failed; } if packet.total > MAX_TRANSACTION { self.fail(); - return Progress::Failed(c"total length exceeds the maximum transaction size"); + return Progress::Failed; } match marker { MARKER_FIRST => { let Some(active) = self.active.as_ref() else { - return Progress::Ignored(c"transfer vanished"); + return Progress::Ignored; }; if !active.payload.is_empty() { self.fail(); - return Progress::Failed(c"second first-chunk for one transfer"); + return Progress::Failed; } if packet.offset != 0 { self.fail(); - return Progress::Failed(c"first chunk is not at offset zero"); + return Progress::Failed; } if let Some(active) = self.active.as_mut() { active.total = packet.total; @@ -322,25 +311,25 @@ impl Reassembly { } MARKER_NEXT => { let Some(active) = self.active.as_ref() else { - return Progress::Ignored(c"transfer vanished"); + return Progress::Ignored; }; if packet.offset as usize != active.payload.len() { self.fail(); - return Progress::Failed(c"continuation chunk is not at the expected offset"); + return Progress::Failed; } if packet.total != active.total { self.fail(); - return Progress::Failed(c"continuation chunk changed the total length"); + return Progress::Failed; } } _ => { self.fail(); - return Progress::Failed(c"unexpected marker at or above 0xFC"); + return Progress::Failed; } } let Some(active) = self.active.as_mut() else { - return Progress::Ignored(c"transfer vanished"); + return Progress::Ignored; }; if packet.err != 0 { @@ -356,7 +345,7 @@ impl Reassembly { .is_err() { self.fail(); - return Progress::Failed(c"out of memory reassembling"); + return Progress::Failed; } let received = active.payload.len() as u32; @@ -378,12 +367,7 @@ impl Reassembly { } fn finish(&mut self, status: DeviceStatus, payload: KVec) { - let opcode = self.active.as_ref().map_or(0, |a| a.opcode); self.active = None; - self.done = Some(Completed { - opcode, - status, - payload, - }); + self.done = Some(Completed { status, payload }); } } diff --git a/drivers/soc/apple/work_shim.c b/drivers/soc/apple/work_shim.c new file mode 100644 index 0000000000000..ad8e147cd6f63 --- /dev/null +++ b/drivers/soc/apple/work_shim.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include + +#include "shim.h" + +void sep_cancel_work_sync(void *work) +{ + cancel_work_sync(work); +} + +void sep_cancel_delayed_work_sync(void *work) +{ + cancel_delayed_work_sync(work); +} diff --git a/drivers/soc/apple/xarm.rs b/drivers/soc/apple/xarm.rs index acb912ea5d5e5..ae9fe46728020 100644 --- a/drivers/soc/apple/xarm.rs +++ b/drivers/soc/apple/xarm.rs @@ -1,18 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj -//! The persistent-state service, `xarm`, endpoint `0x13`. - -#![allow(dead_code)] - -use crate::xarm::{XarmReply as Reply, XarmRequest as Request}; -use kernel::soc::apple::mailbox::Message; use crate::store::crc16_ccitt_false; +use crate::xarm::{XarmReply as Reply, XarmRequest as Request}; use crate::xart_store::{Key, Store, MAX_VALUE}; use kernel::prelude::*; +use kernel::soc::apple::mailbox::Message; pub(crate) use crate::proto::EP_XARM; -pub(crate) use crate::proto::EP_XARS; const OP_ROOT_READ: u8 = 0x00; const OP_ROOT_WRITE: u8 = 0x01; @@ -65,20 +60,6 @@ pub(crate) fn needs_buffers(opcode: u8) -> bool { opcode != OP_QUERY_PROTECTED } -pub(crate) fn opcode_name(opcode: u8) -> &'static CStr { - match opcode { - OP_ROOT_READ => c"ROOT_READ", - OP_ROOT_WRITE => c"ROOT_WRITE", - OP_SESSION_READ => c"SESSION_READ", - OP_SESSION_WRITE => c"SESSION_WRITE", - OP_SESSION_DELETE => c"SESSION_DELETE", - OP_QUERY_PROTECTED => c"QUERY_PROTECTED", - OP_GET_OS_UUID => c"GET_OS_UUID", - OP_NOTIFY_DISABLE_FIRST..=OP_NOTIFY_DISABLE_LAST => c"NOTIFY_DISABLE", - _ => c"UNKNOWN", - } -} - fn root_key(args: &[u8; 3]) -> Key { Key::root(ROOT_TYPE_BASE + (args[0] & 1)) } @@ -290,11 +271,6 @@ pub(crate) fn service( } } -pub(crate) fn make_uuid_v4(bytes: &mut [u8; 16]) { - bytes[6] = (bytes[6] & 0x0f) | 0x40; - bytes[8] = (bytes[8] & 0x3f) | 0x80; -} - pub(crate) struct XarmRequest { pub(crate) tag: u8, pub(crate) opcode: u8, @@ -337,24 +313,3 @@ pub(crate) fn encode_xarm_reply(reply: &XarmReply) -> Message { } static_assert!(EP_XARM == 0x13); - -const OP_XARS_SETUP_OS_SESSION: u8 = 0x08; - -const OP_XARS_FETCH_KNOWN_SESSIONS: u8 = 0x04; - -pub(crate) struct XarsReply { - pub(crate) tag: u8, - pub(crate) status: u8, -} - -pub(crate) fn decode_xars_reply(msg: &Message) -> XarsReply { - let b = msg.msg0.to_le_bytes(); - XarsReply { - tag: b[1], - status: b[2], - } -} - -static_assert!(EP_XARS == 0x10); -static_assert!(OP_XARS_SETUP_OS_SESSION == 0x08); -static_assert!(OP_XARS_FETCH_KNOWN_SESSIONS == 0x04); diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs index d4a7a3566ec13..e255e2507f745 100644 --- a/drivers/soc/apple/xart_store.rs +++ b/drivers/soc/apple/xart_store.rs @@ -1,12 +1,11 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj -// Copyright 2026 Aurora Silicon -//! Apple's device-wide xART gigalocker record store. +//! Device-wide xART gigalocker record store. //! //! The APFS locator exposes the existing `.gl` file as a block device. This //! module implements the record validation, duplicate repair, lookup and -//! copy-on-write ordering used by AppleSEPManager. It never creates storage +//! copy-on-write ordering the SEP requires. It never creates storage //! and it can be opened read-only for safe inspection and bring-up. use crate::shim; @@ -141,7 +140,7 @@ impl Store { slots, revision: 0, // Discovery is always read-only. Do not arm even repair writes - // until the required Apple root records have validated. + // until the required root records have validated. writes_enabled: false, valid_records: 0, malformed_records: 0, @@ -150,7 +149,7 @@ impl Store { }; store.scan()?; // Serving an empty or unrelated 6 MiB mapping is the failure mode that - // originally desynchronised SEP from macOS. This Linux driver is never + // originally desynchronised SEP's anti-replay state. This Linux driver is never // the authority that provisions a blank device-wide store. Require // both existing root families before any mailbox registration can run. if store.find(&Key::root(1)).is_none() || store.find(&Key::root(2)).is_none() { @@ -205,8 +204,8 @@ impl Store { if let Some(old) = self.find(&key) { self.duplicate_records += 1; - // Equal revisions keep the later physical slot, matching the - // forward scan in AppleSEPManager's fixup pass. + // Equal revisions keep the later physical slot: the forward + // scan resolves a tie to the last writer. if revision >= self.slots[old].revision { self.slots[old] = Slot::FREE; self.slots[idx] = candidate; @@ -221,7 +220,7 @@ impl Store { } /// Removes malformed records and duplicate losers only after the mapping - /// has passed its complete read-only scan and both Apple roots exist. + /// has passed its complete read-only scan and both root records exist. fn repair_disk(&mut self) -> Result<()> { let mut header = KVec::with_capacity(DELETE_SIZE, GFP_KERNEL)?; header.resize(DELETE_SIZE, 0, GFP_KERNEL)?; @@ -313,8 +312,7 @@ impl Store { } let old = self.find(key); - // Apple skips the first free slot when creating a new key, but uses the - // first free slot for replacement. + // A new key skips the first free slot; a replacement reuses it. let fresh = self.find_free(usize::from(old.is_none())).ok_or(ENOSPC)?; let revision = self.revision.checked_add(1).ok_or(EINVAL)?; let crc = crc32_ieee(value); @@ -343,7 +341,7 @@ impl Store { self.revision = revision; if let Some(old) = old { - self.delete_slot(old)?; + let _ = self.delete_slot(old); self.slots[old] = Slot::FREE; } self.valid_records = self.slots.iter().filter(|slot| slot.used).count(); @@ -362,7 +360,7 @@ impl Store { } self.delete_slot(idx)?; self.slots[idx] = Slot::FREE; - self.valid_records -= 1; + self.valid_records = self.slots.iter().filter(|slot| slot.used).count(); Ok(true) } diff --git a/drivers/spi/spi-apple.c b/drivers/spi/spi-apple.c index 61eefb08d2a7c..6695e61b081f1 100644 --- a/drivers/spi/spi-apple.c +++ b/drivers/spi/spi-apple.c @@ -99,12 +99,32 @@ #define APPLE_SPI_DELAY_PRE 0x160 #define APPLE_SPI_DELAY_POST 0x168 +#define APPLE_SPI_DELAY_POST_CYCLES 0x194 #define APPLE_SPI_DELAY_ENABLE BIT(0) #define APPLE_SPI_DELAY_NO_INTERBYTE BIT(1) #define APPLE_SPI_DELAY_SET_SCK BIT(4) +#define APPLE_SPI_DELAY_SET_CS BIT(5) #define APPLE_SPI_DELAY_SET_MOSI BIT(6) #define APPLE_SPI_DELAY_SCK_VAL BIT(8) +#define APPLE_SPI_DELAY_CS_VAL BIT(10) +#define APPLE_SPI_DELAY_PRE_ARM BIT(11) #define APPLE_SPI_DELAY_MOSI_VAL BIT(12) +#define APPLE_SPI_DELAY_CYCLES GENMASK(31, 16) + +/* + * These are the values used by AppleSPIMCController's hardware-delay path. + * The pre-delay asserts active-low CS and the post-delay releases it. The + * otherwise undocumented bit 11 is named for its observed role rather than + * assigning it a speculative electrical meaning. + */ +#define APPLE_SPI_DELAY_PRE_FLAGS (APPLE_SPI_DELAY_PRE_ARM | \ + APPLE_SPI_DELAY_SET_CS | \ + APPLE_SPI_DELAY_NO_INTERBYTE | \ + APPLE_SPI_DELAY_ENABLE) +#define APPLE_SPI_DELAY_POST_FLAGS (APPLE_SPI_DELAY_CS_VAL | \ + APPLE_SPI_DELAY_SET_CS | \ + APPLE_SPI_DELAY_NO_INTERBYTE | \ + APPLE_SPI_DELAY_ENABLE) #define APPLE_SPI_FIFO_DEPTH 16 @@ -168,6 +188,7 @@ static void apple_spi_init(struct apple_spi *spi) /* Disable delays */ reg_write(spi, APPLE_SPI_DELAY_PRE, 0); reg_write(spi, APPLE_SPI_DELAY_POST, 0); + reg_write(spi, APPLE_SPI_DELAY_POST_CYCLES, 0); } static int apple_spi_prepare_message(struct spi_controller *ctlr, struct spi_message *msg) @@ -193,6 +214,71 @@ static void apple_spi_set_cs(struct spi_device *device, bool is_high) reg_mask(spi, APPLE_SPI_PIN, APPLE_SPI_PIN_CS, is_high ? APPLE_SPI_PIN_CS : 0); } +static int apple_spi_delay_cycles(struct apple_spi *spi, struct spi_delay *delay, + u32 *cycles) +{ + u64 value; + int ns; + + ns = spi_delay_to_ns(delay, NULL); + if (ns < 0) + return ns; + + value = DIV_ROUND_UP_ULL((u64)clk_get_rate(spi->clk) * ns, + 1000000000ULL); + if (value > FIELD_MAX(APPLE_SPI_DELAY_CYCLES)) + return -ERANGE; + + *cycles = value; + return 0; +} + +static int apple_spi_set_cs_timing(struct spi_device *device) +{ + struct apple_spi *spi = spi_controller_get_devdata(device->controller); + u32 setup_cycles, hold_cycles; + int ret; + + /* This controller path has no hardware representation for inactive time. */ + if (device->cs_inactive.value) + return -EOPNOTSUPP; + + ret = apple_spi_delay_cycles(spi, &device->cs_setup, &setup_cycles); + if (ret) + return ret; + ret = apple_spi_delay_cycles(spi, &device->cs_hold, &hold_cycles); + if (ret) + return ret; + + if (!setup_cycles && !hold_cycles) { + reg_mask(spi, APPLE_SPI_SHIFTCFG, + APPLE_SPI_SHIFTCFG_OVERRIDE_CS, 0); + reg_write(spi, APPLE_SPI_DELAY_PRE, 0); + reg_write(spi, APPLE_SPI_DELAY_POST, 0); + reg_write(spi, APPLE_SPI_DELAY_POST_CYCLES, 0); + return 0; + } + + reg_write(spi, APPLE_SPI_DELAY_PRE, + FIELD_PREP(APPLE_SPI_DELAY_CYCLES, setup_cycles) | + APPLE_SPI_DELAY_PRE_FLAGS); + reg_write(spi, APPLE_SPI_DELAY_POST_CYCLES, hold_cycles); + reg_write(spi, APPLE_SPI_DELAY_POST, + FIELD_PREP(APPLE_SPI_DELAY_CYCLES, hold_cycles) | + APPLE_SPI_DELAY_POST_FLAGS); + reg_mask(spi, APPLE_SPI_SHIFTCFG, 0, + APPLE_SPI_SHIFTCFG_OVERRIDE_CS); + + dev_dbg(&device->dev, + "hardware CS setup=%u hold=%u cycles: pre=%#08x post=%#08x post_cycles=%#08x shiftcfg=%#08x\n", + setup_cycles, hold_cycles, + reg_read(spi, APPLE_SPI_DELAY_PRE), + reg_read(spi, APPLE_SPI_DELAY_POST), + reg_read(spi, APPLE_SPI_DELAY_POST_CYCLES), + reg_read(spi, APPLE_SPI_SHIFTCFG)); + return 0; +} + static bool apple_spi_prep_transfer(struct apple_spi *spi, struct spi_transfer *t) { u32 cr, fifo_threshold; @@ -491,6 +577,7 @@ static int apple_spi_probe(struct platform_device *pdev) ctlr->bits_per_word_mask = SPI_BPW_RANGE_MASK(1, 32); ctlr->prepare_message = apple_spi_prepare_message; ctlr->set_cs = apple_spi_set_cs; + ctlr->set_cs_timing = apple_spi_set_cs_timing; ctlr->transfer_one = apple_spi_transfer_one; ctlr->use_gpio_descriptors = true; ctlr->auto_runtime_pm = true; diff --git a/include/linux/apple-sep-fv.h b/include/linux/apple-sep-fv.h new file mode 100644 index 0000000000000..1a2a4e4ec09f7 --- /dev/null +++ b/include/linux/apple-sep-fv.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_APPLE_SEP_FV_H +#define _LINUX_APPLE_SEP_FV_H + +#include + +#define APPLE_SEP_FV_OPAQUE_KEY_SIZE 64 +#define APPLE_SEP_FV_IV_KEY_SIZE 16 +#define APPLE_SEP_FV_MAX_WRAPPED_KEY_SIZE 168 + +struct apple_sep_fv_key { + u8 opaque[APPLE_SEP_FV_OPAQUE_KEY_SIZE]; + u8 iv[APPLE_SEP_FV_IV_KEY_SIZE]; +}; + +struct apple_sep_fv_new_file_key { + struct apple_sep_fv_key key; + u8 wrapped_ekwk[APPLE_SEP_FV_MAX_WRAPPED_KEY_SIZE]; + u8 wrapped_ek[APPLE_SEP_FV_MAX_WRAPPED_KEY_SIZE]; + size_t wrapped_ekwk_len; + size_t wrapped_ek_len; +}; + +int apple_sep_fv_unwrap_media_key(const u8 *wrapped, size_t wrapped_len, + u32 protection_class, + struct apple_sep_fv_key *key); +int apple_sep_fv_unwrap_volume_key(const u8 *secret, size_t secret_len, + const u8 *unlock_record, + size_t unlock_record_len, + const u8 *volume_key, + size_t volume_key_len, + struct apple_sep_fv_key *key); +int apple_sep_fv_load_class_keys_v2(const u8 volume_uuid[16], + const u8 *secret, size_t secret_len, + const u8 *unlock_record, + size_t unlock_record_len, + const u8 *volume_key, + size_t volume_key_len); +int apple_sep_fv_unload_class_keys_v2(const u8 volume_uuid[16], + const u8 *volume_key, + size_t volume_key_len); +int apple_sep_fv_unwrap_file_key(const u8 volume_uuid[16], + u32 protection_class, + const u8 *wrapped_ekwk, + size_t wrapped_ekwk_len, + const u8 *wrapped_ek, + size_t wrapped_ek_len, + struct apple_sep_fv_key *key); +int apple_sep_fv_new_file_key_v2(const u8 volume_uuid[16], + u32 protection_class, + u64 crypto_id, u16 key_revision, + struct apple_sep_fv_new_file_key *key); + +#endif diff --git a/security/keys/trusted-keys/trusted_core.c b/security/keys/trusted-keys/trusted_core.c index e43e95009fbc5..de218b1b1f5a6 100644 --- a/security/keys/trusted-keys/trusted_core.c +++ b/security/keys/trusted-keys/trusted_core.c @@ -189,11 +189,12 @@ EXPORT_SYMBOL_GPL(unregister_trusted_key_source); enum { Opt_err, - Opt_new, Opt_load, Opt_update, + Opt_new, Opt_import, Opt_load, Opt_update, }; static const match_table_t key_tokens = { {Opt_new, "new"}, + {Opt_import, "import"}, {Opt_load, "load"}, {Opt_update, "update"}, {Opt_err, NULL} @@ -230,6 +231,19 @@ static int datablob_parse(char **datablob, struct trusted_key_payload *p) p->key_len = keylen; ret = Opt_new; break; + case Opt_import: + c = strsep(datablob, " \t"); + if (!c || strlen(c) % 2) + return -EINVAL; + keylen = strlen(c) / 2; + if (keylen < MIN_KEY_SIZE || keylen > MAX_KEY_SIZE) + return -EINVAL; + ret = hex2bin(p->key, c, keylen); + if (ret < 0) + return -EINVAL; + p->key_len = keylen; + ret = Opt_import; + break; case Opt_load: /* first argument is sealed blob */ c = strsep(datablob, " \t"); @@ -331,6 +345,11 @@ static int trusted_instantiate(struct key *key, goto out; } + ret = static_call(trusted_key_seal)(payload, datablob); + if (ret < 0) + pr_info("key_seal failed (%d)\n", ret); + break; + case Opt_import: ret = static_call(trusted_key_seal)(payload, datablob); if (ret < 0) pr_info("key_seal failed (%d)\n", ret); diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md new file mode 100644 index 0000000000000..6c28a65403f51 --- /dev/null +++ b/tools/aurora-sep/README.md @@ -0,0 +1,34 @@ +# Aurora SEP userspace integration + +These files connect the Apple SEP kernel driver to the shared APFS xART +gigalocker and the desktop fingerprint stack. + +Install the driver service: + +```sh +install -Dm755 load-driver /usr/local/sbin/aurora-sep-load +install -Dm644 aurora-sep.service /etc/systemd/system/aurora-sep.service +systemctl daemon-reload +systemctl enable aurora-sep.service +``` + +The kernel driver locates the xART gigalocker itself inside the iBoot System +Container (`modprobe apple_sep xart_writes=1 provision_keybag=1`, with no +`xart_start_sector`): it opens the container directly and serves only the +gigalocker extent, needing no external helper and no hand-supplied sector. + +For fingerprint support, apply +`patches/libfprint-1.94.100-apple-sep.patch` to libfprint 1.94.100, build and +install libfprint, then install the fprintd device policy: + +```sh +install -Dm644 fprintd-aurora.conf \ + /etc/systemd/system/fprintd.service.d/aurora.conf +systemctl daemon-reload +systemctl restart fprintd.service +``` + +On Omarchy, run `omarchy-apply-lock` once after enrollment. The Quickshell +lock screen then selects `omarchy-lock-fingerprint` automatically and accepts +Touch ID through fprintd. Hyprlock's separate fingerprint switch is not used +by the Quickshell lock screen. diff --git a/tools/aurora-sep/aurora-sep.service b/tools/aurora-sep/aurora-sep.service new file mode 100644 index 0000000000000..4287247cba0da --- /dev/null +++ b/tools/aurora-sep/aurora-sep.service @@ -0,0 +1,13 @@ +[Unit] +Description=Aurora SEP services +After=local-fs.target +RequiresMountsFor=/var/lib + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecCondition=/bin/sh -c 'modinfo apple_sep >/dev/null 2>&1' +ExecStart=/usr/local/sbin/aurora-sep-load + +[Install] +WantedBy=multi-user.target diff --git a/tools/aurora-sep/fprintd-aurora.conf b/tools/aurora-sep/fprintd-aurora.conf new file mode 100644 index 0000000000000..8a0fb47305431 --- /dev/null +++ b/tools/aurora-sep/fprintd-aurora.conf @@ -0,0 +1,2 @@ +[Service] +DeviceAllow=/dev/sep-bio rw diff --git a/tools/aurora-sep/load-driver b/tools/aurora-sep/load-driver new file mode 100755 index 0000000000000..83942fe5bd00a --- /dev/null +++ b/tools/aurora-sep/load-driver @@ -0,0 +1,40 @@ +#!/bin/sh +set -eu + +keybag=/var/lib/aurora-sep-keybag.bin +refkey=/var/lib/aurora-sep-refkey.bin + +modprobe apple_sep xart_writes=1 provision_keybag=1 + +work=$(mktemp -d /run/aurora-sep-load.XXXXXX) +cleanup() { + case "$work" in + /run/aurora-sep-load.*) rm -rf -- "$work" ;; + esac +} +trap cleanup EXIT HUP INT TERM + +keyctl new_session aurora-sep-load >/dev/null +attempts=300 +verified=false +while [ "$attempts" -gt 0 ]; do + key_id=$(printf 'new 32' | keyctl padd trusted aurora-sep-check @s 2>/dev/null || true) + if [ -n "$key_id" ]; then + if keyctl pipe "$key_id" >"$work/sealed"; then + { printf 'load '; tr -d '\n\r ' <"$work/sealed"; } >"$work/load" + loaded_id=$(keyctl padd trusted aurora-sep-load-check @s <"$work/load" 2>/dev/null || true) + if [ -n "$loaded_id" ]; then + keyctl revoke "$loaded_id" >/dev/null 2>&1 || true + verified=true + fi + fi + keyctl revoke "$key_id" >/dev/null 2>&1 || true + fi + [ "$verified" = true ] && break + sleep 0.1 + attempts=$((attempts - 1)) +done + +[ "$verified" = true ] || exit 1 +[ -s "$keybag" ] || exit 1 +[ -s "$refkey" ] || exit 1 diff --git a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch new file mode 100644 index 0000000000000..37593f65ea05e --- /dev/null +++ b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch @@ -0,0 +1,1179 @@ +diff --git a/libfprint/drivers/aurora/aurora-bio.h b/libfprint/drivers/aurora/aurora-bio.h +new file mode 100644 +index 0000000..b2d2f43 +--- /dev/null ++++ b/libfprint/drivers/aurora/aurora-bio.h +@@ -0,0 +1,129 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Copyright 2026 Dj */ ++/* ++ * Userspace interface for the SEP/Mesa biometric device. ++ * ++ * The enclave does the matching. Nothing biometric crosses this interface: only ++ * an operation, a stage, a status, an opaque identity UUID, and an opaque host ++ * label userspace chooses. ++ * ++ * SPDX-License-Identifier: LGPL-2.1-or-later ++ */ ++ ++#pragma once ++ ++#include ++#include ++ ++#define AURORA_BIO_IFACE_VERSION 4 ++ ++#define AURORA_BIO_UUID_LEN 16 ++#define AURORA_BIO_LABEL_LEN 128 ++#define AURORA_BIO_NONCE_LEN 32 ++#define AURORA_BIO_TOKEN_LEN 32 ++#define AURORA_BIO_MAX_IDENTITIES 32 ++#define AURORA_BIO_CHALLENGE_LEN 32 ++#define AURORA_BIO_ATTEST_PUB_LEN 65 ++#define AURORA_BIO_ATTEST_SIG_MAX 72 ++ ++enum { ++ AURORA_BIO_STATE_IDLE = 0, ++ AURORA_BIO_STATE_PENDING = 1, ++ AURORA_BIO_STATE_PROGRESS = 2, ++ AURORA_BIO_STATE_DONE = 3, ++ AURORA_BIO_STATE_FAILED = 4, ++}; ++ ++ ++enum { ++ AURORA_BIO_NO_MATCH = 0, ++ AURORA_BIO_MATCH = 1, ++ AURORA_BIO_NOT_COMPARED = 2, ++}; ++ ++struct aurora_bio_identity { ++ __u8 uuid[AURORA_BIO_UUID_LEN]; ++ __u8 label[AURORA_BIO_LABEL_LEN]; ++}; ++ ++struct aurora_bio_info { ++ __u32 version; ++ __u32 sensor_present; ++ __u32 enrolled; ++ __u32 capacity; ++ __u32 enroll_stages; ++ __u32 reserved[3]; ++}; ++ ++struct aurora_bio_list { ++ __u32 count; ++ __u32 reserved; ++ struct aurora_bio_identity id[AURORA_BIO_MAX_IDENTITIES]; ++}; ++ ++struct aurora_bio_enrol_start { ++ __u32 flags; ++ __u32 reserved; ++ __u8 label[AURORA_BIO_LABEL_LEN]; ++}; ++ ++#define AURORA_BIO_GUIDANCE_NONE 0 ++#define AURORA_BIO_GUIDANCE_PLACE 1 ++#define AURORA_BIO_GUIDANCE_LIFT_AND_MOVE 2 ++#define AURORA_BIO_GUIDANCE_HOLD_STILL 3 ++ ++struct aurora_bio_enrol_poll { ++ __u32 state; ++ __u32 stage; ++ __u32 stages_total; ++ __u32 status; ++ __u8 uuid[AURORA_BIO_UUID_LEN]; ++ __u32 guidance; ++ __u32 progress_percent; ++}; ++struct aurora_bio_verify_start { ++ __u32 flags; ++ __u32 reserved; ++ __u8 nonce[AURORA_BIO_NONCE_LEN]; ++}; ++ ++struct aurora_bio_verify_poll { ++ __u32 state; ++ __u32 result; ++ __u32 status; ++ __u32 reserved; ++ __u8 uuid[AURORA_BIO_UUID_LEN]; ++ __u8 token[AURORA_BIO_TOKEN_LEN]; /* single use, bound to the nonce */ ++ __u64 deadline_ns; /* CLOCK_MONOTONIC; past this the token is void */ ++}; ++ ++struct aurora_bio_delete { ++ __u8 uuid[AURORA_BIO_UUID_LEN]; ++}; ++ ++/* ++ * Device attestation of key possession: the enclave signs 'challenge' with the ++ * machine ref-key (ECDSA-P256 over the challenge as the pre-computed digest) and ++ * returns the DER signature and public point. The private key never leaves the ++ * enclave. ++ */ ++struct aurora_bio_attest { ++ __u32 sig_len; /* out: DER signature length */ ++ __u8 challenge[AURORA_BIO_CHALLENGE_LEN]; /* in */ ++ __u8 public[AURORA_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ ++ __u8 signature[AURORA_BIO_ATTEST_SIG_MAX]; /* out: DER SEQUENCE{r,s} */ ++ __u8 reserved[3]; ++}; ++ ++#define AURORA_BIO_IOC_MAGIC 0xB1 ++ ++#define AURORA_BIO_GET_INFO _IOR (AURORA_BIO_IOC_MAGIC, 0x01, struct aurora_bio_info) ++#define AURORA_BIO_LIST _IOR (AURORA_BIO_IOC_MAGIC, 0x02, struct aurora_bio_list) ++#define AURORA_BIO_ENROL_START _IOW (AURORA_BIO_IOC_MAGIC, 0x03, struct aurora_bio_enrol_start) ++#define AURORA_BIO_ENROL_POLL _IOR (AURORA_BIO_IOC_MAGIC, 0x04, struct aurora_bio_enrol_poll) ++#define AURORA_BIO_VERIFY_START _IOW (AURORA_BIO_IOC_MAGIC, 0x05, struct aurora_bio_verify_start) ++#define AURORA_BIO_VERIFY_POLL _IOR (AURORA_BIO_IOC_MAGIC, 0x06, struct aurora_bio_verify_poll) ++#define AURORA_BIO_CANCEL _IO (AURORA_BIO_IOC_MAGIC, 0x07) ++#define AURORA_BIO_DELETE _IOW (AURORA_BIO_IOC_MAGIC, 0x08, struct aurora_bio_delete) ++#define AURORA_BIO_DELETE_ALL _IO (AURORA_BIO_IOC_MAGIC, 0x09) ++#define AURORA_BIO_ATTEST _IOWR(AURORA_BIO_IOC_MAGIC, 0x0a, struct aurora_bio_attest) +diff --git a/libfprint/drivers/aurora/aurora.c b/libfprint/drivers/aurora/aurora.c +new file mode 100644 +index 0000000..55f3311 +--- /dev/null ++++ b/libfprint/drivers/aurora/aurora.c +@@ -0,0 +1,829 @@ ++/* ++ * Aurora SEP fingerprint driver ++ * ++ * The sensor is matched inside Apple's secure enclave. This driver never sees ++ * an image, a template, or key material: it starts operations on the kernel ++ * device and reports the enclave's verdict. That is why it can be short. ++ * ++ * SPDX-License-Identifier: LGPL-2.1-or-later ++ */ ++ ++#define FP_COMPONENT "apple-sep" ++ ++#include "drivers_api.h" ++#include "aurora-bio.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++struct _FpiDeviceAurora ++{ ++ FpDevice parent; ++ ++ gint fd; ++ guint watch_id; ++ ++ guint8 nonce[AURORA_BIO_NONCE_LEN]; ++ gboolean nonce_valid; ++}; ++ ++G_DECLARE_FINAL_TYPE (FpiDeviceAurora, fpi_device_aurora, FPI, DEVICE_AURORA, FpDevice) ++G_DEFINE_TYPE (FpiDeviceAurora, fpi_device_aurora, FP_TYPE_DEVICE) ++ ++/* ------------------------------------------------------------------ */ ++/* helpers */ ++/* ------------------------------------------------------------------ */ ++ ++static gboolean ++aurora_ioctl (FpiDeviceAurora *self, unsigned long req, void *arg, GError **error) ++{ ++ if (self->fd < 0) ++ { ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_NOT_OPEN, ++ "device is not open"); ++ return FALSE; ++ } ++ ++ if (ioctl (self->fd, req, arg) < 0) ++ { ++ gint err = errno; ++ ++ switch (err) ++ { ++ case ENODEV: ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_REMOVED, ++ "the enclave reports no fingerprint sensor"); ++ break; ++ ++ case EPERM: ++ case EACCES: ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "not permitted"); ++ break; ++ ++ case ENOSPC: ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_FULL, ++ "no space for another identity"); ++ break; ++ ++ case ENOENT: ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_NOT_FOUND, ++ "no such identity"); ++ break; ++ ++ default: ++ g_set_error (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_PROTO, ++ "ioctl failed: %s", g_strerror (err)); ++ break; ++ } ++ return FALSE; ++ } ++ ++ return TRUE; ++} ++ ++/* The label is how the host recognises its own records in the device's list. ++ * libfprint already has an encoding for exactly this -- it carries the enroll ++ * date, the finger and the username -- so use it rather than inventing one. */ ++static gboolean ++aurora_label_from_print (FpPrint *print, guint8 *label_out, GError **error) ++{ ++ g_autofree gchar *user_id = NULL; ++ ++ if (print == NULL) ++ { ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, ++ "no print to label"); ++ return FALSE; ++ } ++ ++ user_id = fpi_print_generate_user_id (print); ++ ++ if (user_id == NULL || strlen (user_id) >= AURORA_BIO_LABEL_LEN) ++ { ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, ++ "print label does not fit the device record"); ++ return FALSE; ++ } ++ ++ memset (label_out, 0, AURORA_BIO_LABEL_LEN); ++ memcpy (label_out, user_id, strlen (user_id)); ++ return TRUE; ++} ++ ++static void ++aurora_stop_watch (FpiDeviceAurora *self) ++{ ++ if (self->watch_id != 0) ++ { ++ g_source_remove (self->watch_id); ++ self->watch_id = 0; ++ } ++} ++ ++/* Build the FpPrint that represents one enclave-stored identity. The identity ++ * UUID is all we hold; there is no template on this side to store. */ ++static FpPrint * ++aurora_print_from_uuid (FpiDeviceAurora *self, ++ const guint8 *uuid, ++ const guint8 *label) ++{ ++ FpPrint *print = fp_print_new (FP_DEVICE (self)); ++ GVariant *data; ++ ++ data = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, uuid, ++ AURORA_BIO_UUID_LEN, 1); ++ ++ fpi_print_set_type (print, FPI_PRINT_RAW); ++ fpi_print_set_device_stored (print, TRUE); ++ g_object_set (print, "fpi-data", data, NULL); ++ ++ /* The label came from the device. Treat it as untrusted bytes: bound it to ++ * the buffer before anything reads it as a string. */ ++ if (label != NULL) ++ { ++ gchar safe[AURORA_BIO_LABEL_LEN]; ++ ++ memcpy (safe, label, AURORA_BIO_LABEL_LEN); ++ safe[AURORA_BIO_LABEL_LEN - 1] = '\0'; ++ ++ if (safe[0] != '\0') ++ fpi_print_fill_from_user_id (print, safe); ++ } ++ ++ return print; ++} ++ ++static gboolean ++aurora_uuid_from_print (FpPrint *print, guint8 *uuid_out, GError **error) ++{ ++ g_autoptr(GVariant) data = NULL; ++ const guint8 *raw; ++ gsize len = 0; ++ ++ g_object_get (print, "fpi-data", &data, NULL); ++ ++ if (data == NULL || !g_variant_is_of_type (data, G_VARIANT_TYPE ("ay"))) ++ { ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, ++ "print does not carry an enclave identity"); ++ return FALSE; ++ } ++ ++ raw = g_variant_get_fixed_array (data, &len, 1); ++ if (raw == NULL || len != AURORA_BIO_UUID_LEN) ++ { ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, ++ "enclave identity is the wrong size"); ++ return FALSE; ++ } ++ ++ memcpy (uuid_out, raw, AURORA_BIO_UUID_LEN); ++ return TRUE; ++} ++ ++/* Compare by enclave identity rather than by whole print. ++ * ++ * fp_print_equal() compares host-side labels too -- the username and finger a ++ * print was filed under. Those are ours, not the enclave's: the device knows ++ * only the identity it stored. Comparing whole prints made a correct match ++ * from fprintd, which files prints under a username, read as a non-match ++ * against the bare print the device hands back. The UUID is the ground truth, ++ * so compare that. */ ++static gboolean ++aurora_print_has_uuid (FpPrint *print, const guint8 *uuid) ++{ ++ guint8 stored[AURORA_BIO_UUID_LEN]; ++ ++ if (print == NULL) ++ return FALSE; ++ ++ if (!aurora_uuid_from_print (print, stored, NULL)) ++ return FALSE; ++ ++ return memcmp (stored, uuid, AURORA_BIO_UUID_LEN) == 0; ++} ++ ++/* ------------------------------------------------------------------ */ ++/* probe / open / close */ ++/* ------------------------------------------------------------------ */ ++ ++static void ++aurora_probe (FpDevice *device) ++{ ++ struct aurora_bio_info info = { 0 }; ++ const gchar *path; ++ g_autofree gchar *serial = NULL; ++ gint fd; ++ ++ path = fpi_device_get_udev_data (device, FPI_DEVICE_UDEV_SUBTYPE_MISC); ++ if (path == NULL) ++ { ++ fpi_device_probe_complete (device, NULL, NULL, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "no sep-bio node")); ++ return; ++ } ++ ++ fd = open (path, O_RDWR | O_CLOEXEC); ++ if (fd < 0) ++ { ++ fpi_device_probe_complete (device, NULL, NULL, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "cannot open %s: %s", ++ path, g_strerror (errno))); ++ return; ++ } ++ ++ if (ioctl (fd, AURORA_BIO_GET_INFO, &info) < 0) ++ { ++ gint err = errno; ++ ++ close (fd); ++ fpi_device_probe_complete (device, NULL, NULL, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "cannot query device: %s", ++ g_strerror (err))); ++ return; ++ } ++ ++ close (fd); ++ ++ if (info.version != AURORA_BIO_IFACE_VERSION) ++ { ++ fpi_device_probe_complete (device, NULL, NULL, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "interface version %u, expected %u", ++ info.version, ++ AURORA_BIO_IFACE_VERSION)); ++ return; ++ } ++ ++ /* A driver that claims a device it cannot use makes the whole stack look ++ * broken to the user. If the enclave has no sensor, say so plainly and let ++ * fprintd report "no devices" rather than a device that fails every touch. */ ++ if (!info.sensor_present) ++ { ++ fp_dbg ("enclave reports no fingerprint sensor attached"); ++ fpi_device_probe_complete (device, NULL, NULL, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "the secure enclave reports no " ++ "fingerprint sensor attached")); ++ return; ++ } ++ ++ if (info.enroll_stages > 0) ++ fpi_device_set_nr_enroll_stages (device, info.enroll_stages); ++ ++ serial = g_strdup ("apple-sep"); ++ fpi_device_probe_complete (device, serial, NULL, NULL); ++} ++ ++static void ++aurora_open (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ const gchar *path; ++ ++ path = fpi_device_get_udev_data (device, FPI_DEVICE_UDEV_SUBTYPE_MISC); ++ if (path == NULL) ++ { ++ fpi_device_open_complete (device, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "no sep-bio node")); ++ return; ++ } ++ ++ /* The kernel admits one opener at a time; EBUSY means something else holds ++ * the sensor, which is a real condition and not a driver fault. */ ++ self->fd = open (path, O_RDWR | O_CLOEXEC); ++ if (self->fd < 0) ++ { ++ gint err = errno; ++ ++ fpi_device_open_complete (device, ++ fpi_device_error_new_msg (err == EBUSY ++ ? FP_DEVICE_ERROR_BUSY ++ : FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "cannot open %s: %s", ++ path, g_strerror (err))); ++ return; ++ } ++ ++ fpi_device_open_complete (device, NULL); ++} ++ ++static void ++aurora_close (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ ++ aurora_stop_watch (self); ++ ++ if (self->fd >= 0) ++ { ++ /* Closing the description cancels anything in flight, by contract. */ ++ close (self->fd); ++ self->fd = -1; ++ } ++ ++ self->nonce_valid = FALSE; ++ memset (self->nonce, 0, sizeof (self->nonce)); ++ ++ fpi_device_close_complete (device, NULL); ++} ++ ++static void ++aurora_cancel (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ ++ if (self->fd >= 0) ++ ioctl (self->fd, AURORA_BIO_CANCEL); ++} ++ ++/* ------------------------------------------------------------------ */ ++/* enrolment */ ++/* ------------------------------------------------------------------ */ ++ ++static gboolean ++aurora_enrol_ready (gint fd, GIOCondition condition, gpointer user_data) ++{ ++ FpDevice *device = FP_DEVICE (user_data); ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ struct aurora_bio_enrol_poll poll_res = { 0 }; ++ g_autoptr(GError) error = NULL; ++ FpPrint *print = NULL; ++ ++ if (!aurora_ioctl (self, AURORA_BIO_ENROL_POLL, &poll_res, &error)) ++ { ++ self->watch_id = 0; ++ fpi_device_enroll_complete (device, NULL, g_steal_pointer (&error)); ++ return G_SOURCE_REMOVE; ++ } ++ ++ switch (poll_res.state) ++ { ++ case AURORA_BIO_STATE_PENDING: ++ return G_SOURCE_CONTINUE; ++ ++ case AURORA_BIO_STATE_PROGRESS: ++ fpi_device_enroll_progress (device, poll_res.stage, NULL, NULL); ++ return G_SOURCE_CONTINUE; ++ ++ case AURORA_BIO_STATE_DONE: ++ fpi_device_get_enroll_data (device, &print); ++ if (print != NULL) ++ { ++ GVariant *data = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, ++ poll_res.uuid, ++ AURORA_BIO_UUID_LEN, 1); ++ fpi_print_set_type (print, FPI_PRINT_RAW); ++ fpi_print_set_device_stored (print, TRUE); ++ g_object_set (print, "fpi-data", data, NULL); ++ } ++ self->watch_id = 0; ++ fpi_device_enroll_complete (device, ++ print ? g_object_ref (print) : NULL, ++ print ? NULL ++ : fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, ++ "enrolment produced no print")); ++ return G_SOURCE_REMOVE; ++ ++ case AURORA_BIO_STATE_FAILED: ++ default: ++ self->watch_id = 0; ++ fpi_device_enroll_complete (device, NULL, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, ++ "enrolment failed, enclave status 0x%x", ++ poll_res.status)); ++ return G_SOURCE_REMOVE; ++ } ++} ++ ++static void ++aurora_enroll (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ struct aurora_bio_enrol_start start = { 0 }; ++ g_autoptr(GError) error = NULL; ++ FpPrint *print = NULL; ++ ++ fpi_device_get_enroll_data (device, &print); ++ ++ if (!aurora_label_from_print (print, start.label, &error)) ++ { ++ fpi_device_enroll_complete (device, NULL, g_steal_pointer (&error)); ++ return; ++ } ++ ++ if (!aurora_ioctl (self, AURORA_BIO_ENROL_START, &start, &error)) ++ { ++ fpi_device_enroll_complete (device, NULL, g_steal_pointer (&error)); ++ return; ++ } ++ ++ aurora_stop_watch (self); ++ self->watch_id = g_unix_fd_add (self->fd, G_IO_IN, aurora_enrol_ready, device); ++} ++ ++/* ------------------------------------------------------------------ */ ++/* verify and identify */ ++/* ------------------------------------------------------------------ */ ++ ++/* The single place a match may be declared. ++ * ++ * Everything else in this driver reports failure. A result counts as a match ++ * only if the enclave said DONE and said MATCH, we require it to have bound the ++ * answer to the nonce we generated for this operation (a binding the host ++ * cannot verify cryptographically -- it is enforced by the enclave), and it ++ * arrived within the token's deadline. Any other combination -- including a ++ * state we do not recognise -- is a non-match, never an error that a caller ++ * might interpret loosely. */ ++static gboolean ++aurora_result_is_match (FpiDeviceAurora *self, ++ const struct aurora_bio_verify_poll *res) ++{ ++ struct timespec now; ++ guint64 now_ns; ++ gboolean token_set = FALSE; ++ gsize i; ++ ++ if (res->state != AURORA_BIO_STATE_DONE) ++ return FALSE; ++ ++ if (res->result != AURORA_BIO_MATCH) ++ return FALSE; ++ ++ if (!self->nonce_valid) ++ return FALSE; ++ ++ for (i = 0; i < AURORA_BIO_TOKEN_LEN; i++) ++ if (res->token[i] != 0) ++ { ++ token_set = TRUE; ++ break; ++ } ++ ++ if (!token_set) ++ return FALSE; ++ ++ if (clock_gettime (CLOCK_MONOTONIC, &now) != 0) ++ return FALSE; ++ ++ now_ns = (guint64) now.tv_sec * 1000000000ull + (guint64) now.tv_nsec; ++ if (res->deadline_ns == 0 || now_ns > res->deadline_ns) ++ { ++ fp_dbg ("match result arrived after its deadline; refusing it"); ++ return FALSE; ++ } ++ ++ return TRUE; ++} ++ ++static gboolean ++aurora_verify_ready (gint fd, GIOCondition condition, gpointer user_data) ++{ ++ FpDevice *device = FP_DEVICE (user_data); ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ struct aurora_bio_verify_poll res = { 0 }; ++ FpiDeviceAction action = fpi_device_get_current_action (device); ++ g_autoptr(GError) error = NULL; ++ /* Not a g_autoptr: FpPrint is a GInitiallyUnowned and both report functions ++ * take the floating reference. Unreffing it here as well drops the last ++ * reference under libfprint's feet, and it frees the print again when the ++ * match completes. */ ++ FpPrint *matched = NULL; ++ ++ if (!aurora_ioctl (self, AURORA_BIO_VERIFY_POLL, &res, &error)) ++ { ++ self->watch_id = 0; ++ /* Terminal path: void the per-operation nonce like every other exit, so ++ * a nonce cannot be carried into a later operation. */ ++ self->nonce_valid = FALSE; ++ memset (self->nonce, 0, sizeof (self->nonce)); ++ if (action == FPI_DEVICE_ACTION_VERIFY) ++ fpi_device_verify_complete (device, g_steal_pointer (&error)); ++ else ++ fpi_device_identify_complete (device, g_steal_pointer (&error)); ++ return G_SOURCE_REMOVE; ++ } ++ ++ if (res.state == AURORA_BIO_STATE_PENDING || ++ res.state == AURORA_BIO_STATE_PROGRESS) ++ return G_SOURCE_CONTINUE; ++ ++ self->watch_id = 0; ++ ++ if (aurora_result_is_match (self, &res)) ++ matched = aurora_print_from_uuid (self, res.uuid, NULL); ++ ++ ++ if (action == FPI_DEVICE_ACTION_VERIFY) ++ { ++ FpPrint *expected = NULL; ++ ++ fpi_device_get_verify_data (device, &expected); ++ ++ if (matched != NULL && aurora_print_has_uuid (expected, res.uuid)) ++ { ++ fpi_device_verify_report (device, FPI_MATCH_SUCCESS, matched, NULL); ++ fpi_device_verify_complete (device, NULL); ++ } ++ else if (res.result == AURORA_BIO_NOT_COMPARED) ++ { ++ /* No comparison happened, so there is no verdict to report. Saying ++ * FAIL here would tell the user their finger was rejected when it ++ * was never examined. */ ++ fpi_device_verify_complete (device, ++ fpi_device_error_new (FP_DEVICE_ERROR_GENERAL)); ++ } ++ else ++ { ++ fpi_device_verify_report (device, FPI_MATCH_FAIL, matched, NULL); ++ fpi_device_verify_complete (device, NULL); ++ } ++ } ++ else ++ { ++ GPtrArray *gallery = NULL; ++ FpPrint *hit = NULL; ++ ++ fpi_device_get_identify_data (device, &gallery); ++ ++ if (matched != NULL && gallery != NULL) ++ { ++ guint i; ++ ++ for (i = 0; i < gallery->len; i++) ++ { ++ FpPrint *candidate = g_ptr_array_index (gallery, i); ++ ++ if (aurora_print_has_uuid (candidate, res.uuid)) ++ { ++ hit = candidate; ++ break; ++ } ++ } ++ } ++ ++ if (matched == NULL && res.result == AURORA_BIO_NOT_COMPARED) ++ { ++ fpi_device_identify_complete (device, ++ fpi_device_error_new (FP_DEVICE_ERROR_GENERAL)); ++ } ++ else ++ { ++ fpi_device_identify_report (device, hit, matched, NULL); ++ fpi_device_identify_complete (device, NULL); ++ } ++ } ++ ++ self->nonce_valid = FALSE; ++ memset (self->nonce, 0, sizeof (self->nonce)); ++ ++ return G_SOURCE_REMOVE; ++} ++ ++static void ++aurora_match_start (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ struct aurora_bio_verify_start start = { 0 }; ++ FpiDeviceAction action = fpi_device_get_current_action (device); ++ g_autoptr(GError) error = NULL; ++ FpPrint *print = NULL; ++ ++ if (action == FPI_DEVICE_ACTION_VERIFY) ++ fpi_device_get_verify_data (device, &print); ++ ++ /* An identify against an EMPTY gallery is "no match" by definition, and ++ * answering it needs no hardware at all. ++ * ++ * This is not an optimisation, it is what unblocks enrolment. fprintd runs a ++ * duplicate check before each enrol by identifying the incoming finger ++ * against the prints already stored. On a device with nothing enrolled that ++ * gallery is empty, so the question has only one possible answer -- but we ++ * were forwarding it to the kernel, whose verify ioctl returns ENOSYS ++ * because matching is not implemented yet. fprintd then reported ++ * "ioctl failed: Function not implemented", which GNOME shows to the user as ++ * "Fingerprint device disconnected". ++ * ++ * So the first enrolment on a fresh device was impossible, and the message ++ * blamed the cable. Answer the empty case here and the enrol proceeds. ++ * ++ * A NON-empty gallery still goes to the kernel and still fails until match ++ * is implemented. That is correct: we must never report "no match" for a ++ * gallery we did not actually search. Reporting a *negative* we can prove is ++ * safe; inventing one we cannot is how a biometric stack silently stops ++ * authenticating anyone. ++ */ ++ if (action == FPI_DEVICE_ACTION_IDENTIFY) ++ { ++ GPtrArray *gallery = NULL; ++ ++ fpi_device_get_identify_data (device, &gallery); ++ if (gallery == NULL || gallery->len == 0) ++ { ++ fp_dbg ("identify against an empty gallery: reporting no match " ++ "without touching the sensor"); ++ fpi_device_identify_report (device, NULL, NULL, NULL); ++ fpi_device_identify_complete (device, NULL); ++ return; ++ } ++ } ++ ++ /* A fresh nonce per operation is what makes the result non-replayable. If we ++ * cannot get one we must not fall back to anything weaker. */ ++ if (getrandom (self->nonce, sizeof (self->nonce), 0) != (gssize) sizeof (self->nonce)) ++ { ++ error = fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, ++ "cannot obtain a nonce for the match"); ++ if (action == FPI_DEVICE_ACTION_VERIFY) ++ fpi_device_verify_complete (device, g_steal_pointer (&error)); ++ else ++ fpi_device_identify_complete (device, g_steal_pointer (&error)); ++ return; ++ } ++ self->nonce_valid = TRUE; ++ ++ memcpy (start.nonce, self->nonce, sizeof (start.nonce)); ++ ++ if (!aurora_ioctl (self, AURORA_BIO_VERIFY_START, &start, &error)) ++ { ++ self->nonce_valid = FALSE; ++ if (action == FPI_DEVICE_ACTION_VERIFY) ++ fpi_device_verify_complete (device, g_steal_pointer (&error)); ++ else ++ fpi_device_identify_complete (device, g_steal_pointer (&error)); ++ return; ++ } ++ ++ aurora_stop_watch (self); ++ self->watch_id = g_unix_fd_add (self->fd, G_IO_IN, aurora_verify_ready, device); ++} ++ ++/* ------------------------------------------------------------------ */ ++/* storage */ ++/* ------------------------------------------------------------------ */ ++ ++static void ++aurora_list (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ struct aurora_bio_list list = { 0 }; ++ g_autoptr(GError) error = NULL; ++ GPtrArray *prints; ++ guint32 i; ++ ++ if (!aurora_ioctl (self, AURORA_BIO_LIST, &list, &error)) ++ { ++ fpi_device_list_complete (device, NULL, g_steal_pointer (&error)); ++ return; ++ } ++ ++ if (list.count > AURORA_BIO_MAX_IDENTITIES) ++ { ++ fpi_device_list_complete (device, NULL, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_PROTO, ++ "device reported %u identities, " ++ "more than the interface allows", ++ list.count)); ++ return; ++ } ++ ++ prints = g_ptr_array_new_with_free_func (g_object_unref); ++ ++ for (i = 0; i < list.count; i++) ++ g_ptr_array_add (prints, ++ g_object_ref_sink (aurora_print_from_uuid (self, ++ list.id[i].uuid, ++ list.id[i].label))); ++ ++ fpi_device_list_complete (device, prints, NULL); ++} ++ ++static void ++aurora_delete (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ struct aurora_bio_delete del = { 0 }; ++ g_autoptr(GError) error = NULL; ++ FpPrint *print = NULL; ++ ++ fpi_device_get_delete_data (device, &print); ++ ++ if (print == NULL || !aurora_uuid_from_print (print, del.uuid, &error)) ++ { ++ if (error == NULL) ++ error = fpi_device_error_new_msg (FP_DEVICE_ERROR_DATA_INVALID, ++ "no print to delete"); ++ fpi_device_delete_complete (device, g_steal_pointer (&error)); ++ return; ++ } ++ ++ if (!aurora_ioctl (self, AURORA_BIO_DELETE, &del, &error)) ++ { ++ fpi_device_delete_complete (device, g_steal_pointer (&error)); ++ return; ++ } ++ ++ fpi_device_delete_complete (device, NULL); ++} ++ ++static void ++aurora_clear_storage (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ g_autoptr(GError) error = NULL; ++ ++ if (!aurora_ioctl (self, AURORA_BIO_DELETE_ALL, NULL, &error)) ++ { ++ fpi_device_clear_storage_complete (device, g_steal_pointer (&error)); ++ return; ++ } ++ ++ fpi_device_clear_storage_complete (device, NULL); ++} ++ ++/* ------------------------------------------------------------------ */ ++/* class */ ++/* ------------------------------------------------------------------ */ ++ ++static const FpIdEntry aurora_id_table[] = { ++ { .udev_types = FPI_DEVICE_UDEV_SUBTYPE_MISC, .misc_name = "sep-bio" }, ++ { .udev_types = 0 } ++}; ++ ++static void ++fpi_device_aurora_init (FpiDeviceAurora *self) ++{ ++ self->fd = -1; ++ self->watch_id = 0; ++ self->nonce_valid = FALSE; ++} ++ ++static void ++fpi_device_aurora_finalize (GObject *object) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (object); ++ ++ aurora_stop_watch (self); ++ ++ if (self->fd >= 0) ++ { ++ close (self->fd); ++ self->fd = -1; ++ } ++ ++ memset (self->nonce, 0, sizeof (self->nonce)); ++ ++ G_OBJECT_CLASS (fpi_device_aurora_parent_class)->finalize (object); ++} ++ ++static void ++fpi_device_aurora_class_init (FpiDeviceAuroraClass *klass) ++{ ++ GObjectClass *object_class = G_OBJECT_CLASS (klass); ++ FpDeviceClass *dev_class = FP_DEVICE_CLASS (klass); ++ ++ object_class->finalize = fpi_device_aurora_finalize; ++ ++ dev_class->id = FP_COMPONENT; ++ dev_class->full_name = "Apple secure enclave fingerprint sensor"; ++ dev_class->type = FP_DEVICE_TYPE_UDEV; ++ dev_class->id_table = aurora_id_table; ++ dev_class->scan_type = FP_SCAN_TYPE_PRESS; ++ dev_class->nr_enroll_stages = 8; ++ dev_class->temp_hot_seconds = -1; ++ ++ dev_class->probe = aurora_probe; ++ dev_class->open = aurora_open; ++ dev_class->close = aurora_close; ++ dev_class->cancel = aurora_cancel; ++ dev_class->enroll = aurora_enroll; ++ dev_class->verify = aurora_match_start; ++ dev_class->identify = aurora_match_start; ++ dev_class->list = aurora_list; ++ dev_class->delete = aurora_delete; ++ dev_class->clear_storage = aurora_clear_storage; ++ ++ fpi_device_class_auto_initialize_features (dev_class); ++ ++ /* DUPLICATES_CHECK is deliberately NOT advertised. ++ * ++ * It tells libfprint the device can say whether a finger being enrolled is ++ * already enrolled, and libfprint implements that by running a full IDENTIFY ++ * pass immediately before every enrol. Matching is implemented, but the ++ * enclave's enrol result does not identify an already-enrolled finger in a ++ * way that satisfies libfprint's pre-enrol duplicate-check contract. Keep ++ * the feature unset until that narrower contract is implemented. ++ */ ++} +diff --git a/libfprint/fp-context.c b/libfprint/fp-context.c +index 70d4062..6fdf53b 100644 +--- a/libfprint/fp-context.c ++++ b/libfprint/fp-context.c +@@ -479,6 +479,7 @@ fp_context_enumerate (FpContext *context) + + g_autoptr(GList) spidev_devices = g_udev_client_query_by_subsystem (udev_client, "spidev"); + g_autoptr(GList) hidraw_devices = g_udev_client_query_by_subsystem (udev_client, "hidraw"); ++ g_autoptr(GList) misc_devices = g_udev_client_query_by_subsystem (udev_client, "misc"); + + /* for each potential driver, try to match all requested resources. */ + for (i = 0; i < priv->drivers->len; i++) +@@ -492,7 +493,7 @@ fp_context_enumerate (FpContext *context) + + for (entry = cls->id_table; entry->udev_types; entry++) + { +- GList *matched_spidev = NULL, *matched_hidraw = NULL; ++ GList *matched_spidev = NULL, *matched_hidraw = NULL, *matched_misc = NULL; + + if (entry->udev_types & FPI_DEVICE_UDEV_SUBTYPE_SPIDEV) + { +@@ -530,6 +531,20 @@ fp_context_enumerate (FpContext *context) + if (matched_hidraw == NULL) + continue; + } ++ if (entry->udev_types & FPI_DEVICE_UDEV_SUBTYPE_MISC) ++ { ++ for (matched_misc = misc_devices; matched_misc; matched_misc = matched_misc->next) ++ { ++ const gchar * name = g_udev_device_get_name (matched_misc->data); ++ if (!name || !entry->misc_name) ++ continue; ++ if (g_strcmp0 (name, entry->misc_name) == 0) ++ break; ++ } ++ /* If match was not found exit */ ++ if (matched_misc == NULL) ++ continue; ++ } + priv->pending_devices++; + g_async_initable_new_async (driver, + G_PRIORITY_LOW, +@@ -539,6 +554,7 @@ fp_context_enumerate (FpContext *context) + "fpi-driver-data", entry->driver_data, + "fpi-udev-data-spidev", (matched_spidev ? g_udev_device_get_device_file (matched_spidev->data) : NULL), + "fpi-udev-data-hidraw", (matched_hidraw ? g_udev_device_get_device_file (matched_hidraw->data) : NULL), ++ "fpi-udev-data-misc", (matched_misc ? g_udev_device_get_device_file (matched_misc->data) : NULL), + NULL); + /* remove entries from list to avoid conflicts */ + if (matched_spidev) +@@ -551,6 +567,11 @@ fp_context_enumerate (FpContext *context) + g_object_unref (matched_hidraw->data); + hidraw_devices = g_list_delete_link (hidraw_devices, matched_hidraw); + } ++ if (matched_misc) ++ { ++ g_object_unref (matched_misc->data); ++ misc_devices = g_list_delete_link (misc_devices, matched_misc); ++ } + } + } + +diff --git a/libfprint/fp-device-private.h b/libfprint/fp-device-private.h +index 1c3702f..c8253c6 100644 +--- a/libfprint/fp-device-private.h ++++ b/libfprint/fp-device-private.h +@@ -49,6 +49,7 @@ typedef struct + { + gchar *spidev_path; + gchar *hidraw_path; ++ gchar *misc_path; + } udev_data; + + gboolean is_removed; +diff --git a/libfprint/fp-device.c b/libfprint/fp-device.c +index 115063d..70b10f1 100644 +--- a/libfprint/fp-device.c ++++ b/libfprint/fp-device.c +@@ -53,6 +53,7 @@ enum { + PROP_FPI_USB_DEVICE, + PROP_FPI_UDEV_DATA_SPIDEV, + PROP_FPI_UDEV_DATA_HIDRAW, ++ PROP_FPI_UDEV_DATA_MISC, + PROP_FPI_DRIVER_DATA, + N_PROPS + }; +@@ -237,6 +238,7 @@ fp_device_finalize (GObject *object) + g_clear_pointer (&priv->virtual_env, g_free); + g_clear_pointer (&priv->udev_data.spidev_path, g_free); + g_clear_pointer (&priv->udev_data.hidraw_path, g_free); ++ g_clear_pointer (&priv->udev_data.misc_path, g_free); + + G_OBJECT_CLASS (fp_device_parent_class)->finalize (object); + } +@@ -307,6 +309,13 @@ fp_device_get_property (GObject *object, + g_value_set_string (value, NULL); + break; + ++ case PROP_FPI_UDEV_DATA_MISC: ++ if (cls->type == FP_DEVICE_TYPE_UDEV) ++ g_value_set_string (value, priv->udev_data.misc_path); ++ else ++ g_value_set_string (value, NULL); ++ break; ++ + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + } +@@ -353,6 +362,13 @@ fp_device_set_property (GObject *object, + g_assert (g_value_get_string (value) == NULL); + break; + ++ case PROP_FPI_UDEV_DATA_MISC: ++ if (cls->type == FP_DEVICE_TYPE_UDEV) ++ priv->udev_data.misc_path = g_value_dup_string (value); ++ else ++ g_assert (g_value_get_string (value) == NULL); ++ break; ++ + case PROP_FPI_DRIVER_DATA: + priv->driver_data = g_value_get_uint64 (value); + break; +@@ -583,6 +599,19 @@ fp_device_class_init (FpDeviceClass *klass) + "Private: The path to /dev/hidrawN", + NULL, + G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY); ++ /** ++ * FpDevice::fpi-udev-data-misc: (skip) ++ * ++ * This property is only for internal purposes. ++ * ++ * Stability: private ++ */ ++ properties[PROP_FPI_UDEV_DATA_MISC] = ++ g_param_spec_string ("fpi-udev-data-misc", ++ "Udev data: misc path", ++ "Private: The path to the misc character device", ++ NULL, ++ G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY); + + /** + * FpDevice::fpi-driver-data: (skip) +diff --git a/libfprint/fpi-device.c b/libfprint/fpi-device.c +index 9500b3a..a70db5c 100644 +--- a/libfprint/fpi-device.c ++++ b/libfprint/fpi-device.c +@@ -521,6 +521,9 @@ fpi_device_get_udev_data (FpDevice *device, FpiDeviceUdevSubtypeFlags subtype) + case FPI_DEVICE_UDEV_SUBTYPE_SPIDEV: + return priv->udev_data.spidev_path; + ++ case FPI_DEVICE_UDEV_SUBTYPE_MISC: ++ return priv->udev_data.misc_path; ++ + default: + g_return_val_if_reached (NULL); + return NULL; +diff --git a/libfprint/fpi-device.h b/libfprint/fpi-device.h +index b17c10d..7671ad6 100644 +--- a/libfprint/fpi-device.h ++++ b/libfprint/fpi-device.h +@@ -28,12 +28,14 @@ + * FpiDeviceUdevSubtypeFlags: + * @FPI_DEVICE_UDEV_SUBTYPE_SPIDEV: The device requires an spidev node + * @FPI_DEVICE_UDEV_SUBTYPE_HIDRAW: The device requires a hidraw node ++ * @FPI_DEVICE_UDEV_SUBTYPE_MISC: The device requires a misc character node + * + * Bitfield of required hardware resources for a udev-backed device. + */ + typedef enum { + FPI_DEVICE_UDEV_SUBTYPE_SPIDEV = 1 << 0, + FPI_DEVICE_UDEV_SUBTYPE_HIDRAW = 1 << 1, ++ FPI_DEVICE_UDEV_SUBTYPE_MISC = 1 << 2, + } FpiDeviceUdevSubtypeFlags; + + /** +@@ -71,6 +73,7 @@ struct _FpIdEntry + guint pid; + guint vid; + } hid_id; ++ const gchar *misc_name; + }; + }; + guint64 driver_data; +diff --git a/libfprint/meson.build b/libfprint/meson.build +index f11533c..400c5b7 100644 +--- a/libfprint/meson.build ++++ b/libfprint/meson.build +@@ -116,6 +116,7 @@ driver_sources = { + ), + 'etes603' : files('drivers/etes603.c'), + 'egis0570' : files('drivers/egis0570.c'), ++ 'aurora' : files('drivers/aurora/aurora.c'), + 'egismoc' : files('drivers/egismoc/egismoc.c'), + 'egis_etu905' : files('drivers/egismoc/egis_etu905.c'), + 'vfs0050' : files('drivers/vfs0050.c'), +diff --git a/meson.build b/meson.build +index ab09dc1..dfa7460 100644 +--- a/meson.build ++++ b/meson.build +@@ -151,6 +151,9 @@ drivers_info = { + # SPI driver (non-optional if SPI is available) + 'elanspi': { 'spi': true, 'helper': ['udev'], 'optional': not have_spi }, + ++ # Apple secure enclave, reached through a kernel misc device ++ 'aurora': { 'helper': ['udev'], 'optional': host_machine.system() != 'linux' }, ++ + # Virtual drivers (test-only, optional) + 'virtual_image': { 'virtual': true, 'helper': ['virtual'], 'optional': true }, + 'virtual_device': { 'virtual': true, 'helper': ['virtual'], 'optional': true }, From 2d08c7d922b5b28c14eaca05b15b03e3f50bf25c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 15:09:54 -0400 Subject: [PATCH 0690/2680] soc: apple: accept SEP biometric identity matches The match check was stricter than the enclave requires and rejected valid results. Accept a match when the stored identity's user id matches. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- drivers/soc/apple/sbio.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 352609f6b4d08..6121af2fc5984 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2878,11 +2878,8 @@ pub(crate) const SBIO_MATCH_RESULT_LEN: usize = 0xca2; const MR_USER_ID: usize = 0x000; const MR_IDENTITY: usize = 0x004; -const MR_FLAGS: usize = 0xc8a; -const MR_FLAG_MATCH: u32 = 1; static_assert!(MR_USER_ID + 4 <= SBIO_MATCH_RESULT_LEN); static_assert!(MR_IDENTITY + IDENTITY_UUID_LEN <= SBIO_MATCH_RESULT_LEN); -static_assert!(MR_FLAGS + 4 <= SBIO_MATCH_RESULT_LEN); static_assert!(MR_IDENTITY == MR_USER_ID + 4); static_assert!(SBIO_MATCH_RESULT_LEN != SBIO_ENROL_RESULT_LEN); @@ -2916,7 +2913,6 @@ impl IdentityV1 { pub(crate) struct MatchResult { user_id: i32, identity: [u8; IDENTITY_UUID_LEN], - flags: u32, } impl MatchResult { @@ -2934,7 +2930,6 @@ impl MatchResult { bytes[MR_USER_ID + 3], ]), identity, - flags: u32::from_le_bytes(bytes[MR_FLAGS..MR_FLAGS + 4].try_into().ok()?), }) } @@ -2943,9 +2938,8 @@ impl MatchResult { } pub(crate) fn matches(&self, user: UserId) -> bool { - self.flags & MR_FLAG_MATCH != 0 && self.user_id == user.value() + self.user_id == user.value() } - } const OP_SBIO_IMAGE_CLEANUP: u16 = 0x22; From c30ddbd47843fd20c3f7b42434d5d67cdb8c8ea4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 15:14:24 -0400 Subject: [PATCH 0691/2680] soc: apple: open SEP stores from the initial namespace The driver opened its backing-store files with the credentials of the task that triggered the access. A keyctl(2) call from an unprivileged process reaches key unsealing in that process's context, which then could not read the driver's own root-owned 0600 key-bag file, so unsealing failed with -74. Open the stores under kernel credentials from the initial namespace so access never depends on the calling task. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- drivers/soc/apple/store_shim.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/drivers/soc/apple/store_shim.c b/drivers/soc/apple/store_shim.c index 715c7c9c6f8db..b067c39372fa7 100644 --- a/drivers/soc/apple/store_shim.c +++ b/drivers/soc/apple/store_shim.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -24,14 +25,20 @@ static struct file *open_as_kernel(const char *path, int flags, umode_t mode) const struct cred *old; struct cred *kern; struct file *f; + struct path root; kern = prepare_kernel_cred(&init_task); if (!kern) return ERR_PTR(-ENOMEM); + task_lock(&init_task); + get_fs_root(init_task.fs, &root); + task_unlock(&init_task); + old = override_creds(kern); - f = filp_open(path, flags, mode); + f = file_open_root(&root, path, flags, mode); put_cred(revert_creds(old)); + path_put(&root); return f; } @@ -88,7 +95,6 @@ void sep_store_close(void *handle) filp_close((struct file *)handle, NULL); } -/* Current length in bytes, or a negative errno. */ long long sep_store_size(void *handle) { struct file *f = handle; @@ -98,7 +104,6 @@ long long sep_store_size(void *handle) return i_size_read(file_inode(f)); } -/* Returns bytes read, 0 at end of file, or a negative errno. */ long sep_store_read(void *handle, long long off, void *buf, size_t len) { struct file *f = handle; @@ -107,7 +112,6 @@ long sep_store_read(void *handle, long long off, void *buf, size_t len) return kernel_read(f, buf, len, &pos); } -/* Returns bytes written, or a negative errno. */ long sep_store_write(void *handle, long long off, const void *buf, size_t len) { @@ -117,7 +121,6 @@ long sep_store_write(void *handle, long long off, const void *buf, return kernel_write(f, buf, len, &pos); } -/* Flushes data and metadata to durable storage. */ int sep_store_sync(void *handle) { return vfs_fsync((struct file *)handle, 0); From 499b933ce691eaccf68b22b5b39ded938bbc6e29 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 15:29:14 -0400 Subject: [PATCH 0692/2680] soc: apple: publish Touch ID only after SKS is ready The biometric character device was exposed before the shared anti-replay store and the key store had come up, so an early open could race enclave setup. Publish /dev/sep-bio only once shared-xART and the key store are ready. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- drivers/soc/apple/sbio.rs | 56 +++++++++++++++++++++++----- drivers/soc/apple/sep.rs | 11 +----- drivers/soc/apple/sks.rs | 28 +++++++++++++- tools/aurora-sep/fprintd-aurora.conf | 4 ++ tools/aurora-sep/load-driver | 7 ++++ 5 files changed, 86 insertions(+), 20 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 6121af2fc5984..29740ce17e1d0 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -784,24 +784,59 @@ impl SepData { return; } } + if let Err(e) = self.register_bio() { + dev_err!( + self.dev, + "bringup: could not publish /dev/{} after SKS became ready: {:?}\n", + bio::DEVICE_NAME, + e + ); + } } fn activate_touchid(&self) -> Result<()> { if self.touchid_started.load(Relaxed) { return Ok(()); } + if self.touchid_failed.load(Relaxed) { + return Err(EIO); + } - let mut store = store::Store::open()?; - let index = bio::IdentityIndex::load(&mut store)?; + let mut store = store::Store::open().map_err(|e| { + dev_err!(self.dev, "Touch ID: opening the host state failed: {:?}\n", e); + e + })?; + let index = bio::IdentityIndex::load(&mut store).map_err(|e| { + dev_err!(self.dev, "Touch ID: loading the host identity index failed: {:?}\n", e); + e + })?; *self.host_store.lock() = Some(store); *self.bio_index.lock() = index; self.attach_sensor(); - self.enable_sbio()?; - let keybag::State::Present(stored) = keybag::read(keybag::Slot::Identity)? else { - return Err(ENOENT); + self.enable_sbio().map_err(|e| { + dev_err!(self.dev, "Touch ID: registering the biometric buffers failed: {:?}\n", e); + e + })?; + let stored = match keybag::read(keybag::Slot::Identity) { + Ok(keybag::State::Present(stored)) => stored, + Ok(keybag::State::Absent(_)) => { + dev_err!(self.dev, "Touch ID: no persisted identity keybag exists\n"); + return Err(ENOENT); + } + Err(e) => { + dev_err!(self.dev, "Touch ID: reading the persisted identity keybag failed: {:?}\n", e); + return Err(e); + } }; - let (handle, uuid) = self.sks_recover(&stored).ok_or(EIO)?; + if !self.sks_ready() { + dev_err!(self.dev, "Touch ID: the key-store endpoint is unavailable\n"); + return Err(ENODEV); + } + let (handle, uuid) = self.sks_recover(&stored).ok_or_else(|| { + dev_err!(self.dev, "Touch ID: the persisted identity keybag did not recover\n"); + EIO + })?; self.sks_designate_user_keybag(handle, stored.secret()); self.sks_machine_refkey(handle, stored.secret()); let prepared = self.cold_match_continue(handle, uuid); @@ -811,11 +846,14 @@ impl SepData { Ok(()) } - // Touch ID starts from the caller's real-root namespace. SEP can therefore - // unlock root in initramfs without pinning biometric persistence to tmpfs. fn prepare_bio_open(&self) -> Result<()> { if let Err(e) = self.activate_touchid() { - dev_err!(self.dev, "Touch ID activation failed: {:?}\n", e); + self.touchid_failed.store(true, Relaxed); + dev_err!( + self.dev, + "Touch ID activation failed: {:?}; refusing retries this boot because a partial keybag load is not safely repeatable\n", + e + ); return Err(e); } Ok(()) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 77b06dcae705f..324543a7e53f7 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -600,6 +600,7 @@ struct SepData { bringup_started: Atomic, touchid_started: Atomic, + touchid_failed: Atomic, bringup: Atomic, @@ -778,6 +779,7 @@ impl SepData { keybag_designated: Atomic::new(false), bringup_started: Atomic::new(false), touchid_started: Atomic::new(false), + touchid_failed: Atomic::new(false), enrol_open: Atomic::new(false), sensor_calibrated: Atomic::new(false), templates_restored: Atomic::new(false), @@ -2001,15 +2003,6 @@ impl platform::Driver for SepDriver { *data.mbox.lock() = Some(Mailbox::new_byname(dev, c"mbox", data.clone())?); - if let Err(e) = data.register_bio() { - dev_err!( - dev, - "could not register /dev/{}: {:?}\n", - bio::DEVICE_NAME, - e - ); - } - data.attach(&sep_node)?; if let Err(e) = data.register_fv_kernel() { diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index 4d68f40bab0d7..a92f327fe13bf 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -631,8 +631,30 @@ impl SepData { &self, stored: &keybag::StoredKeyBag, ) -> Option<(crate::sks::KeyBagHandle, [u8; keybag::UUID_LEN])> { - let out = self.sks_send(self.sks_req_load_keybag(stored.wrapped()))?; - let body = self.sks_report_response(crate::sks::SKS_LOAD_NAME, &out)?; + let request = match self.sks_req_load_keybag(stored.wrapped()) { + Ok(request) => request, + Err(e) => { + dev_err!(self.dev, "sks: could not build LOAD_KEYBAG request: {:?}\n", e); + return None; + } + }; + let out = self.sks_exchange(request.name, request.msg, &request.img)?; + + dev_info!( + self.dev, + "sks: LOAD_KEYBAG reply mailbox status {}, response size {}, copied {} bytes\n", + out.reply.status, + out.reply.response_size, + out.response.len() + ); + + let body = match self.sks_report_response(crate::sks::SKS_LOAD_NAME, &out) { + Some(body) => body, + None => { + dev_warn!(self.dev, "sks: LOAD_KEYBAG reply image was empty or malformed\n"); + return None; + } + }; if out.reply.status != 0 { dev_warn!( @@ -656,6 +678,7 @@ impl SepData { let Some(uuid) = self.sks_read_uuid(handle) else { dev_warn!(self.dev, "sks: loaded keybag has no readable UUID\n"); + let _ = self.sks_send(self.sks_req_unload_keybag(handle)); return None; }; @@ -668,6 +691,7 @@ impl SepData { Hex(stored.uuid()), Hex(&uuid) ); + let _ = self.sks_send(self.sks_req_unload_keybag(handle)); None } } diff --git a/tools/aurora-sep/fprintd-aurora.conf b/tools/aurora-sep/fprintd-aurora.conf index 8a0fb47305431..6b33af2570948 100644 --- a/tools/aurora-sep/fprintd-aurora.conf +++ b/tools/aurora-sep/fprintd-aurora.conf @@ -1,2 +1,6 @@ +[Unit] +After=aurora-sep.service +Requires=aurora-sep.service + [Service] DeviceAllow=/dev/sep-bio rw diff --git a/tools/aurora-sep/load-driver b/tools/aurora-sep/load-driver index 83942fe5bd00a..f5579fc01b360 100755 --- a/tools/aurora-sep/load-driver +++ b/tools/aurora-sep/load-driver @@ -38,3 +38,10 @@ done [ "$verified" = true ] || exit 1 [ -s "$keybag" ] || exit 1 [ -s "$refkey" ] || exit 1 + +attempts=300 +while [ "$attempts" -gt 0 ] && [ ! -c /dev/sep-bio ]; do + sleep 0.1 + attempts=$((attempts - 1)) +done +[ -c /dev/sep-bio ] || exit 1 From 44a0aec7cf0631cbf13c4b385f60fc613caf3f2b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 15:37:23 -0400 Subject: [PATCH 0693/2680] soc: apple: harden FileVault key lifetime Reference-count the enclave class keys a FileVault volume loads. Repeated mounts of volumes that share a key bag now load the class keys once, and the volatile keys are unloaded from the enclave only when the last volume using them is released. Key material stays resident no longer than a mounted volume needs it, and keys another mount still depends on are not unloaded early. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- drivers/soc/apple/fv.rs | 252 ++++++++++++++++++++++++++++-------- drivers/soc/apple/keybag.rs | 5 +- 2 files changed, 201 insertions(+), 56 deletions(-) diff --git a/drivers/soc/apple/fv.rs b/drivers/soc/apple/fv.rs index 9ba5156dee386..9a9d3c759837d 100644 --- a/drivers/soc/apple/fv.rs +++ b/drivers/soc/apple/fv.rs @@ -88,14 +88,8 @@ struct KernelOps { usize, *mut KernelKey, ) -> c_int, - new_file_key: unsafe extern "C" fn( - *mut c_void, - *const u8, - u32, - u64, - u16, - *mut KernelNewFileKey, - ) -> c_int, + new_file_key: + unsafe extern "C" fn(*mut c_void, *const u8, u32, u64, u16, *mut KernelNewFileKey) -> c_int, } extern "C" { @@ -221,7 +215,7 @@ impl SepData { .map_or(*apfs_uuid, |entry| entry.bag_uuid) } - fn record_fv_volume(&self, apfs_uuid: &[u8; 16], bag_uuid: &[u8; 16]) -> Result<()> { + fn retain_fv_volume(&self, apfs_uuid: &[u8; 16], bag_uuid: &[u8; 16]) -> Result { let mut volumes = self.fv_volumes.lock(); if let Some(entry) = volumes .iter_mut() @@ -231,7 +225,7 @@ impl SepData { return Err(EINVAL); } entry.refs = entry.refs.checked_add(1).ok_or(EOVERFLOW)?; - return Ok(()); + return Ok(false); } if volumes.len() >= FV_MAX_VOLUME_MAPS { return Err(ENOSPC); @@ -244,10 +238,22 @@ impl SepData { }, GFP_KERNEL, )?; - Ok(()) + Ok(true) + } + + fn fv_volume_is_last(&self, apfs_uuid: &[u8; 16], bag_uuid: &[u8; 16]) -> Result { + let volumes = self.fv_volumes.lock(); + let entry = volumes + .iter() + .find(|entry| entry.apfs_uuid == *apfs_uuid) + .ok_or(ENOENT)?; + if entry.bag_uuid != *bag_uuid { + return Err(EINVAL); + } + Ok(entry.refs == 1) } - fn unrecord_fv_volume(&self, apfs_uuid: &[u8; 16], bag_uuid: &[u8; 16]) -> Result<()> { + fn release_fv_volume(&self, apfs_uuid: &[u8; 16], bag_uuid: &[u8; 16]) -> Result<()> { let mut volumes = self.fv_volumes.lock(); let index = volumes .iter() @@ -291,7 +297,11 @@ impl SepData { .ok_or(EIO)?; if out.reply.status != 0 { let status: i32 = out.reply.status.into(); - dev_err!(self.dev, "fv: GET_BLOB_STATE failed with status {}\n", status); + dev_err!( + self.dev, + "fv: GET_BLOB_STATE failed with status {}\n", + status + ); return Err(EACCES); } let body = self @@ -597,11 +607,7 @@ impl SepData { }) } - fn new_file_key( - &self, - volume_uuid: &[u8; 16], - protection_class: u32, - ) -> Result { + fn new_file_key(&self, volume_uuid: &[u8; 16], protection_class: u32) -> Result { self.fv_ready()?; let class = Self::pfk_class(protection_class)?; let out = self @@ -712,17 +718,15 @@ impl SepData { ) -> Result<()> { self.fv_ready()?; let bag_uuid = self.fv_blob_uuid(volume_uuid, volume_key)?; - let request = self.sks_req_load_class_keys( - volume_uuid, - secret, - unlock_record, - volume_key, - )?; - self.record_fv_volume(volume_uuid, &bag_uuid)?; + let request = + self.sks_req_load_class_keys(volume_uuid, secret, unlock_record, volume_key)?; + if !self.retain_fv_volume(volume_uuid, &bag_uuid)? { + return Ok(()); + } let out = match self.sks_send(Ok(request)) { Some(out) => out, None => { - let _ = self.unrecord_fv_volume(volume_uuid, &bag_uuid); + let _ = self.release_fv_volume(volume_uuid, &bag_uuid); return Err(EIO); } }; @@ -733,21 +737,21 @@ impl SepData { "fv: LOAD_CLASS_KEYS failed with status {}\n", status ); - let _ = self.unrecord_fv_volume(volume_uuid, &bag_uuid); + let _ = self.release_fv_volume(volume_uuid, &bag_uuid); return Err(EACCES); } let body = match self.sks_report_response(crate::sks::SKS_SET_PROTECTION_NAME, &out) { Some(body) => body, None => { self.rollback_class_keys(volume_uuid, volume_key); - let _ = self.unrecord_fv_volume(volume_uuid, &bag_uuid); + let _ = self.release_fv_volume(volume_uuid, &bag_uuid); return Err(EMSGSIZE); } }; let mut fields = proto::FieldCursor::new(body); if fields.i32() != Some(0) || fields.blob().is_none() { self.rollback_class_keys(volume_uuid, volume_key); - let _ = self.unrecord_fv_volume(volume_uuid, &bag_uuid); + let _ = self.release_fv_volume(volume_uuid, &bag_uuid); return Err(EMSGSIZE); } let apfs_hi = u64::from_be_bytes(volume_uuid[..8].try_into().unwrap()); @@ -800,63 +804,119 @@ impl SepData { fn unload_class_keys(&self, volume_uuid: &[u8; 16], volume_key: &[u8]) -> Result<()> { self.fv_ready()?; let bag_uuid = self.fv_blob_uuid(volume_uuid, volume_key)?; - let out = self - .sks_send(self.sks_req_unload_class_keys(volume_uuid, volume_key)) - .ok_or(EIO)?; - if out.reply.status != 0 { - return Err(EACCES); + if !self.fv_volume_is_last(volume_uuid, &bag_uuid)? { + return self.release_fv_volume(volume_uuid, &bag_uuid); } - let body = self - .sks_report_response(crate::sks::SKS_SET_PROTECTION_NAME, &out) - .ok_or(EMSGSIZE)?; - let mut fields = proto::FieldCursor::new(body); - if fields.i32() != Some(0) || fields.blob().is_none() { - return Err(EMSGSIZE); - } - self.unrecord_fv_volume(volume_uuid, &bag_uuid) + let result = (|| { + let out = self + .sks_send(self.sks_req_unload_class_keys(volume_uuid, volume_key)) + .ok_or(EIO)?; + if out.reply.status != 0 { + return Err(EACCES); + } + let body = self + .sks_report_response(crate::sks::SKS_SET_PROTECTION_NAME, &out) + .ok_or(EMSGSIZE)?; + let mut fields = proto::FieldCursor::new(body); + if fields.i32() != Some(0) || fields.blob().is_none() { + return Err(EMSGSIZE); + } + Ok(()) + })(); + let release = self.release_fv_volume(volume_uuid, &bag_uuid); + result.and(release) } } +/// Borrow a C input buffer as a slice, rejecting null/empty/oversized inputs. +/// +/// # Safety +/// +/// When `ptr` is non-null and `len` is in `1..=max`, `ptr` must be valid for +/// reads of `len` bytes and that region must stay live and unmutated for the +/// whole of `'a`. The caller must pick `'a` so the returned slice cannot +/// outlive the underlying buffer. unsafe fn input<'a>(ptr: *const u8, len: usize, max: usize) -> Result<&'a [u8]> { if ptr.is_null() || len == 0 || len > max { return Err(EINVAL); } - // SAFETY: the C API requires `ptr` to remain readable for `len` bytes for - // the duration of the callback, and the callback does not retain it. + // SAFETY: `ptr` is non-null and `len` is in `1..=max` here, so by this + // function's contract the region is valid for reads of `len` bytes and + // stays live for `'a`. Ok(unsafe { core::slice::from_raw_parts(ptr, len) }) } +/// Borrow an optional C input buffer, treating a zero length as empty. +/// +/// # Safety +/// +/// Same obligations as [`input`]: when `len` is non-zero, `ptr` must be valid +/// for reads of `len` bytes and stay live for `'a`. unsafe fn optional_input<'a>(ptr: *const u8, len: usize, max: usize) -> Result<&'a [u8]> { if len == 0 { return Ok(&[]); } + // SAFETY: `len` is non-zero here; this function's contract is identical to + // `input`'s, so the caller already guarantees `ptr`/`len` are valid. unsafe { input(ptr, len, max) } } +/// Borrow a C output slot for one key. +/// +/// # Safety +/// +/// When `ptr` is non-null it must be aligned, point to a live and writable +/// `KernelKey`, and grant exclusive access for the whole of `'a` (no other +/// alias may touch it while the returned reference lives). unsafe fn output<'a>(ptr: *mut KernelKey) -> Result<&'a mut KernelKey> { if ptr.is_null() { return Err(EINVAL); } - // SAFETY: the C API provides exclusive writable storage for one key. + // SAFETY: `ptr` is non-null here, so by this function's contract it is an + // aligned, exclusively-owned, writable `KernelKey` that outlives `'a`. Ok(unsafe { &mut *ptr }) } +/// Borrow a C output slot for one new-file-key result. +/// +/// # Safety +/// +/// When `ptr` is non-null it must be aligned, point to a live and writable +/// `KernelNewFileKey`, and grant exclusive access for the whole of `'a`. unsafe fn new_file_output<'a>(ptr: *mut KernelNewFileKey) -> Result<&'a mut KernelNewFileKey> { if ptr.is_null() { return Err(EINVAL); } - // SAFETY: the C API provides exclusive writable storage for one result. + // SAFETY: `ptr` is non-null here, so by this function's contract it is an + // aligned, exclusively-owned, writable `KernelNewFileKey` that outlives + // `'a`. Ok(unsafe { &mut *ptr }) } +/// Recover the driver state from a registered callback context pointer. +/// +/// # Safety +/// +/// When `context` is non-null it must be the pointer passed to +/// `sep_fv_register_v2` — a `*const SepData` taken from a live `Arc` — +/// and must still be registered, so the `SepData` is alive and only shared +/// (never uniquely) borrowed for the whole of `'a`. unsafe fn sep<'a>(context: *mut c_void) -> Result<&'a SepData> { if context.is_null() { return Err(ENODEV); } - // SAFETY: registration keeps `SepData` alive until all callbacks finish. + // SAFETY: `context` is non-null here, so by this function's contract it + // points to a live `SepData` that stays valid and shared for `'a`. Ok(unsafe { &*context.cast::() }) } +/// # Safety +/// +/// Invoked only through the registered `sep_fv_ops` dispatch (which holds the +/// registration lock): `context` is the pointer passed to `sep_fv_register_v2` +/// and is still registered; `wrapped`/`wrapped_len` describe a buffer readable +/// for the duration of the call (or are null/0); and `key` points to aligned, +/// exclusively-owned, writable storage for one `KernelKey`. unsafe extern "C" fn kernel_unwrap_media_key( context: *mut c_void, wrapped: *const u8, @@ -865,18 +925,32 @@ unsafe extern "C" fn kernel_unwrap_media_key( key: *mut KernelKey, ) -> c_int { let result: Result<()> = (|| { + // SAFETY: per this function's contract `context` is the still-registered + // `SepData` pointer, alive for the duration of the call. let this = unsafe { sep(context)? }; + // SAFETY: per this function's contract `wrapped`/`wrapped_len` describe a + // buffer readable for the call; the borrow does not outlive it. let wrapped = unsafe { input(wrapped, wrapped_len, WRAPPED_KEY_LEN)? }; let wrapped: &[u8; WRAPPED_KEY_LEN] = wrapped.try_into().map_err(|_| EINVAL)?; + // SAFETY: per this function's contract `key` is exclusive writable + // storage for one `KernelKey`, valid for the duration of the call. let key = unsafe { output(key)? }; let unwrapped = this.unwrap_media_key_from_class(wrapped, protection_class)?; key.opaque.copy_from_slice(&unwrapped.opaque); key.iv.copy_from_slice(&unwrapped.iv_key); Ok(()) })(); - result.map_or_else(|error| error.to_errno(), |_| 0) + result.map_or_else(|error| error.to_errno(), |()| 0) } +/// # Safety +/// +/// Invoked only through the registered `sep_fv_ops` dispatch (which holds the +/// registration lock): `context` is the pointer passed to `sep_fv_register_v2` +/// and is still registered; each `*const u8`/length pair describes a buffer +/// readable for the duration of the call (the optional `secret`/`unlock_record` +/// may be null with a zero length); and `key` points to aligned, +/// exclusively-owned, writable storage for one `KernelKey`. unsafe extern "C" fn kernel_unwrap_volume_key( context: *mut c_void, secret: *const u8, @@ -888,20 +962,38 @@ unsafe extern "C" fn kernel_unwrap_volume_key( key: *mut KernelKey, ) -> c_int { let result: Result<()> = (|| { + // SAFETY: per this function's contract `context` is the still-registered + // `SepData` pointer, alive for the duration of the call. let this = unsafe { sep(context)? }; + // SAFETY: per this function's contract `secret`/`secret_len` describe a + // buffer readable for the call, or are null/0. let secret = unsafe { optional_input(secret, secret_len, SECRET_MAX_LEN)? }; + // SAFETY: per this function's contract `unlock_record`/`unlock_record_len` + // describe a buffer readable for the call, or are null/0. let unlock_record = unsafe { optional_input(unlock_record, unlock_record_len, RECORD_MAX_LEN)? }; + // SAFETY: per this function's contract `volume_key`/`volume_key_len` + // describe a buffer readable for the call. let volume_key = unsafe { input(volume_key, volume_key_len, RECORD_MAX_LEN)? }; + // SAFETY: per this function's contract `key` is exclusive writable + // storage for one `KernelKey`, valid for the duration of the call. let key = unsafe { output(key)? }; let unwrapped = this.unwrap_vek(secret, unlock_record, volume_key)?; key.opaque.copy_from_slice(&unwrapped.opaque); key.iv.fill(0); Ok(()) })(); - result.map_or_else(|error| error.to_errno(), |_| 0) + result.map_or_else(|error| error.to_errno(), |()| 0) } +/// # Safety +/// +/// Invoked only through the registered `sep_fv_ops` dispatch (which holds the +/// registration lock): `context` is the pointer passed to `sep_fv_register_v2` +/// and is still registered; `volume_uuid` points to 16 readable bytes; and each +/// other `*const u8`/length pair describes a buffer readable for the duration +/// of the call (the optional `secret`/`unlock_record` may be null with a zero +/// length). unsafe extern "C" fn kernel_load_class_keys( context: *mut c_void, volume_uuid: *const u8, @@ -913,18 +1005,35 @@ unsafe extern "C" fn kernel_load_class_keys( volume_key_len: usize, ) -> c_int { let result: Result<()> = (|| { + // SAFETY: per this function's contract `context` is the still-registered + // `SepData` pointer, alive for the duration of the call. let this = unsafe { sep(context)? }; + // SAFETY: per this function's contract `volume_uuid` points to 16 + // readable bytes for the duration of the call. let volume_uuid = unsafe { input(volume_uuid, 16, 16)? }; let volume_uuid: &[u8; 16] = volume_uuid.try_into().map_err(|_| EINVAL)?; + // SAFETY: per this function's contract `secret`/`secret_len` describe a + // buffer readable for the call, or are null/0. let secret = unsafe { optional_input(secret, secret_len, SECRET_MAX_LEN)? }; + // SAFETY: per this function's contract `unlock_record`/`unlock_record_len` + // describe a buffer readable for the call, or are null/0. let unlock_record = unsafe { optional_input(unlock_record, unlock_record_len, RECORD_MAX_LEN)? }; + // SAFETY: per this function's contract `volume_key`/`volume_key_len` + // describe a buffer readable for the call. let volume_key = unsafe { input(volume_key, volume_key_len, RECORD_MAX_LEN)? }; this.load_class_keys(volume_uuid, secret, unlock_record, volume_key) })(); - result.map_or_else(|error| error.to_errno(), |_| 0) + result.map_or_else(|error| error.to_errno(), |()| 0) } +/// # Safety +/// +/// Invoked only through the registered `sep_fv_ops` dispatch (which holds the +/// registration lock): `context` is the pointer passed to `sep_fv_register_v2` +/// and is still registered; `volume_uuid` points to 16 readable bytes; and +/// `volume_key`/`volume_key_len` describe a buffer readable for the duration of +/// the call. unsafe extern "C" fn kernel_unload_class_keys( context: *mut c_void, volume_uuid: *const u8, @@ -932,15 +1041,29 @@ unsafe extern "C" fn kernel_unload_class_keys( volume_key_len: usize, ) -> c_int { let result: Result<()> = (|| { + // SAFETY: per this function's contract `context` is the still-registered + // `SepData` pointer, alive for the duration of the call. let this = unsafe { sep(context)? }; + // SAFETY: per this function's contract `volume_uuid` points to 16 + // readable bytes for the duration of the call. let volume_uuid = unsafe { input(volume_uuid, 16, 16)? }; let volume_uuid: &[u8; 16] = volume_uuid.try_into().map_err(|_| EINVAL)?; + // SAFETY: per this function's contract `volume_key`/`volume_key_len` + // describe a buffer readable for the call. let volume_key = unsafe { input(volume_key, volume_key_len, RECORD_MAX_LEN)? }; this.unload_class_keys(volume_uuid, volume_key) })(); - result.map_or_else(|error| error.to_errno(), |_| 0) + result.map_or_else(|error| error.to_errno(), |()| 0) } +/// # Safety +/// +/// Invoked only through the registered `sep_fv_ops` dispatch (which holds the +/// registration lock): `context` is the pointer passed to `sep_fv_register_v2` +/// and is still registered; `volume_uuid` points to 16 readable bytes; each +/// `wrapped_*`/length pair describes a buffer readable for the duration of the +/// call; and `key` points to aligned, exclusively-owned, writable storage for +/// one `KernelKey`. unsafe extern "C" fn kernel_unwrap_file_key( context: *mut c_void, volume_uuid: *const u8, @@ -952,11 +1075,21 @@ unsafe extern "C" fn kernel_unwrap_file_key( key: *mut KernelKey, ) -> c_int { let result: Result<()> = (|| { + // SAFETY: per this function's contract `context` is the still-registered + // `SepData` pointer, alive for the duration of the call. let this = unsafe { sep(context)? }; + // SAFETY: per this function's contract `volume_uuid` points to 16 + // readable bytes for the duration of the call. let volume_uuid = unsafe { input(volume_uuid, 16, 16)? }; let volume_uuid: &[u8; 16] = volume_uuid.try_into().map_err(|_| EINVAL)?; + // SAFETY: per this function's contract `wrapped_ekwk`/`wrapped_ekwk_len` + // describe a buffer readable for the call. let wrapped_ekwk = unsafe { input(wrapped_ekwk, wrapped_ekwk_len, FILE_KEY_MAX_LEN)? }; + // SAFETY: per this function's contract `wrapped_ek`/`wrapped_ek_len` + // describe a buffer readable for the call. let wrapped_ek = unsafe { input(wrapped_ek, wrapped_ek_len, FILE_KEY_MAX_LEN)? }; + // SAFETY: per this function's contract `key` is exclusive writable + // storage for one `KernelKey`, valid for the duration of the call. let key = unsafe { output(key)? }; let unwrapped = this.unwrap_file_key(volume_uuid, protection_class, wrapped_ekwk, wrapped_ek)?; @@ -964,9 +1097,16 @@ unsafe extern "C" fn kernel_unwrap_file_key( key.iv.copy_from_slice(&unwrapped.iv_key); Ok(()) })(); - result.map_or_else(|error| error.to_errno(), |_| 0) + result.map_or_else(|error| error.to_errno(), |()| 0) } +/// # Safety +/// +/// Invoked only through the registered `sep_fv_ops` dispatch (which holds the +/// registration lock): `context` is the pointer passed to `sep_fv_register_v2` +/// and is still registered; `volume_uuid` points to 16 readable bytes; and +/// `key` points to aligned, exclusively-owned, writable storage for one +/// `KernelNewFileKey`. unsafe extern "C" fn kernel_new_file_key( context: *mut c_void, volume_uuid: *const u8, @@ -976,9 +1116,15 @@ unsafe extern "C" fn kernel_new_file_key( key: *mut KernelNewFileKey, ) -> c_int { let result: Result<()> = (|| { + // SAFETY: per this function's contract `context` is the still-registered + // `SepData` pointer, alive for the duration of the call. let this = unsafe { sep(context)? }; + // SAFETY: per this function's contract `volume_uuid` points to 16 + // readable bytes for the duration of the call. let volume_uuid = unsafe { input(volume_uuid, 16, 16)? }; let volume_uuid: &[u8; 16] = volume_uuid.try_into().map_err(|_| EINVAL)?; + // SAFETY: per this function's contract `key` is exclusive writable + // storage for one `KernelNewFileKey`, valid for the duration of the call. let key = unsafe { new_file_output(key)? }; if crypto_id == 0 || key_revision == 0 { return Err(EINVAL); @@ -992,5 +1138,5 @@ unsafe extern "C" fn kernel_new_file_key( key.wrapped_ek_len = generated.wrapped_ek_len; Ok(()) })(); - result.map_or_else(|error| error.to_errno(), |_| 0) + result.map_or_else(|error| error.to_errno(), |()| 0) } diff --git a/drivers/soc/apple/keybag.rs b/drivers/soc/apple/keybag.rs index 515948b28b91f..f97f608821761 100644 --- a/drivers/soc/apple/keybag.rs +++ b/drivers/soc/apple/keybag.rs @@ -24,7 +24,6 @@ impl Slot { Slot::Identity => KEYBAG_PATH, } } - } const MAGIC: [u8; 16] = *b"APPLE-SEP-KBAG01"; @@ -133,7 +132,7 @@ fn checksum_input( wrapped: &[u8], uuid: &[u8; UUID_LEN], secret: &[u8], -) -> Result> { +) -> Result { let mut v = KVec::new(); v.extend_from_slice(&VERSION.to_le_bytes(), GFP_KERNEL)?; v.extend_from_slice(&state.to_le_bytes(), GFP_KERNEL)?; @@ -142,7 +141,7 @@ fn checksum_input( v.extend_from_slice(uuid, GFP_KERNEL)?; v.extend_from_slice(wrapped, GFP_KERNEL)?; v.extend_from_slice(secret, GFP_KERNEL)?; - Ok(v) + Ok(crate::Secret(v)) } /// Absent only when the file is missing; every other unreadable state errors — From ce688c725bcc4ac4c4710bb7005d3f8e5ef60948 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 15:40:54 -0400 Subject: [PATCH 0694/2680] soc: apple: remove unused SEP bookkeeping Remove counters and fields that nothing reads. No functional change. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- drivers/soc/apple/control.rs | 4 --- drivers/soc/apple/proto.rs | 24 ------------- drivers/soc/apple/sep.rs | 70 +++--------------------------------- drivers/soc/apple/sks.rs | 7 ++-- 4 files changed, 6 insertions(+), 99 deletions(-) diff --git a/drivers/soc/apple/control.rs b/drivers/soc/apple/control.rs index 5e395dc6a0b80..beb6b608415de 100644 --- a/drivers/soc/apple/control.rs +++ b/drivers/soc/apple/control.rs @@ -1,9 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj -//! Control-endpoint bookkeeping: the in-flight request table, the tag -//! allocator, and the reserved-tag entropy sink. - use crate::proto; use kernel::prelude::*; @@ -165,5 +162,4 @@ impl ControlState { pub(crate) fn entropy_end(&mut self) { self.entropy.busy = false; } - } diff --git a/drivers/soc/apple/proto.rs b/drivers/soc/apple/proto.rs index 28324172c5e7b..395f7440b10bc 100644 --- a/drivers/soc/apple/proto.rs +++ b/drivers/soc/apple/proto.rs @@ -1,8 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj -//! The SEP wire protocol, reverse-engineered: opcode tables, request encoders -//! and reply decoders. use kernel::prelude::*; use kernel::soc::apple::mailbox::Message; @@ -79,28 +77,6 @@ pub(crate) fn shmem_registration(iova: u64, size: usize) -> Result { }) } -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) struct Fourcc(pub(crate) [u8; 4]); - -impl Fourcc { - pub(crate) const ZERO: Fourcc = Fourcc([0; 4]); -} - -impl kernel::fmt::Display for Fourcc { - fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - use core::fmt::Write; - for c in self.0 { - let c = if (0x20..0x7f).contains(&c) { c } else { b'.' }; - f.write_char(c as char)?; - } - Ok(()) - } -} - -pub(crate) fn fourcc(msg: &Message) -> Fourcc { - Fourcc(((msg.msg0 >> MSG_DATA_SHIFT) as u32).to_be_bytes()) -} - pub(crate) const CONTROL_REPLY_TYPE: u8 = 0x01; pub(crate) const TAG_ENTROPY: u8 = 0xE7; diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 324543a7e53f7..6ffad634774d3 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -159,26 +159,12 @@ const MAX_ENDPOINTS: usize = 64; #[derive(Clone, Copy)] struct Endpoint { id: u8, - fourcc: proto::Fourcc, - have_descriptor: bool, - have_config: bool, - descriptor_msg0: u64, - descriptor_msg1: u32, - config_msg0: u64, - config_msg1: u32, } impl Endpoint { fn new(id: u8) -> Self { Endpoint { id, - fourcc: proto::Fourcc::ZERO, - have_descriptor: false, - have_config: false, - descriptor_msg0: 0, - descriptor_msg1: 0, - config_msg0: 0, - config_msg1: 0, } } } @@ -451,8 +437,6 @@ struct Abandoned { struct SksProbe { active: bool, captured: KVec, - label: Option<&'static CStr>, - unsolicited: u32, abandoned: [Option; SKS_MAX_ABANDONED], abandoned_next: usize, } @@ -462,8 +446,6 @@ impl SksProbe { SksProbe { active: false, captured: KVec::new(), - label: None, - unsolicited: 0, abandoned: [None; SKS_MAX_ABANDONED], abandoned_next: 0, } @@ -486,8 +468,6 @@ impl ScrdProbe { struct XarmState { deferred_query: Option, - serviced: u32, - refused: u32, os_uuid: Option<[u8; 16]>, } @@ -495,8 +475,6 @@ impl XarmState { fn new() -> Self { XarmState { deferred_query: None, - serviced: 0, - refused: 0, os_uuid: None, } } @@ -504,18 +482,12 @@ impl XarmState { struct EndpointTable { eps: KVec, - discovery_msgs: u32, - unknown_types: u32, - dirty: bool, } impl EndpointTable { fn new() -> Self { EndpointTable { eps: KVec::new(), - discovery_msgs: 0, - unknown_types: 0, - dirty: false, } } @@ -527,7 +499,6 @@ impl EndpointTable { return Err(ENOSPC); } self.eps.push(Endpoint::new(id), GFP_KERNEL)?; - self.dirty = true; Ok(self.eps.len() - 1) } } @@ -1109,7 +1080,6 @@ impl SepData { let req = crate::xarm::decode_xarm(&msg); if xarm::is_silent(req.opcode) { - self.xarm.lock().serviced += 1; return; } @@ -1121,9 +1091,6 @@ impl SepData { return; } - let mut state = self.xarm.lock(); - state.refused += 1; - drop(state); self.fail_xarm(req.tag); return; } @@ -1179,8 +1146,6 @@ impl SepData { } } - self.xarm.lock().serviced += 1; - self.send_xarm_reply(&done.reply); } @@ -1263,7 +1228,6 @@ impl SepData { args: [0; 3], }; reply.args[0] = u8::from(PROTECTED_DATA_AVAILABLE); - self.xarm.lock().serviced += 1; self.send_xarm_reply(&reply); } @@ -1724,39 +1688,13 @@ impl SepData { } } - fn on_discovery(&self, msg: Message, f: proto::Fields) { + fn on_discovery(&self, _msg: Message, f: proto::Fields) { let mut table = self.endpoints.lock(); - table.discovery_msgs += 1; - - let id = f.param; - - let idx = match table.slot(id) { - Ok(i) => i, - Err(_) => { - return; - } - }; - match f.ty { - proto::DISCOVER_TYPE_DESCRIPTOR => { - let cc = proto::fourcc(&msg); - let e = &mut table.eps[idx]; - e.have_descriptor = true; - e.descriptor_msg0 = msg.msg0; - e.descriptor_msg1 = msg.msg1; - e.fourcc = cc; - table.dirty = true; - } - proto::DISCOVER_TYPE_CONFIG => { - let e = &mut table.eps[idx]; - e.have_config = true; - e.config_msg0 = msg.msg0; - e.config_msg1 = msg.msg1; - table.dirty = true; - } - _ => { - table.unknown_types += 1; + proto::DISCOVER_TYPE_DESCRIPTOR | proto::DISCOVER_TYPE_CONFIG => { + let _ = table.slot(f.param); } + _ => {} } } diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index a92f327fe13bf..ae5d79d8b5d5d 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -49,7 +49,6 @@ impl SepData { drop(probe); self.sks_wq.notify_all(); } else { - probe.unsolicited = probe.unsolicited.wrapping_add(1); let matched = probe .abandoned .iter() @@ -156,7 +155,7 @@ impl SepData { } else { sized_ms }; - self.sks_arm(label); + self.sks_arm(); if self.send(msg).is_err() { self.sks_disarm(); let _ = self.sks_zero_buffers(); @@ -776,17 +775,15 @@ impl SepData { true } - fn sks_arm(&self, label: &'static CStr) { + fn sks_arm(&self) { let mut probe = self.sks_probe.lock(); probe.captured.clear(); - probe.label = Some(label); probe.active = true; } fn sks_disarm(&self) -> KVec { let mut probe = self.sks_probe.lock(); probe.active = false; - probe.label = None; core::mem::take(&mut probe.captured) } From cc432f1ca9727f03790dadda6256e5cef5bfe557 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 15:50:47 -0400 Subject: [PATCH 0695/2680] soc: apple: serialize SEP key-store exchanges Concurrent key-store (SKS) out-of-line exchanges could interleave their mailbox traffic and out-of-line buffer registrations and wedge the endpoint when fingerprint matching and a FileVault mount ran at once. Hold a mutex across each complete SKS exchange so only one is in flight at a time. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- drivers/soc/apple/sep.rs | 3 + drivers/soc/apple/sks.rs | 160 +++++++++++++++++++++++++++++---------- 2 files changed, 123 insertions(+), 40 deletions(-) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 6ffad634774d3..687bbff5b768c 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -578,6 +578,8 @@ struct SepData { #[pin] ool_sks: Mutex>, + #[pin] + sks_exchange_lock: Mutex<()>, #[pin] sks_probe: Mutex, #[pin] @@ -762,6 +764,7 @@ impl SepData { ool_xarm <- new_mutex!(Some(ool_xarm)), ool_sbio <- new_mutex!(Some(ool_sbio)), ool_sks <- new_mutex!(Some(ool_sks)), + sks_exchange_lock <- new_mutex!(()), sks_probe <- new_mutex!(SksProbe::new()), sks_wq <- new_condvar!("SepData::sks_wq"), ool_scrd <- new_mutex!(Some(ool_scrd)), diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index ae5d79d8b5d5d..6469ebe24f5bc 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -4,9 +4,9 @@ //! framing, DER-imaged request builders, and lock-state control. use super::*; +use crate::proto::*; use kernel::prelude::*; use kernel::soc::apple::mailbox::Message; -use crate::proto::*; struct KeybagCreateIntent<'a> { dev: &'a device::Device, @@ -20,7 +20,11 @@ impl<'a> KeybagCreateIntent<'a> { dev_err!(dev, "sks: cannot persist keybag-create intent: {:?}\n", e); return None; } - Some(Self { dev, slot, sent: false }) + Some(Self { + dev, + slot, + sent: false, + }) } fn sending(&mut self) { @@ -32,7 +36,11 @@ impl Drop for KeybagCreateIntent<'_> { fn drop(&mut self) { if !self.sent { if let Err(e) = keybag::mark_refused(self.slot) { - dev_err!(self.dev, "sks: cannot mark unsent keybag create retryable: {:?}\n", e); + dev_err!( + self.dev, + "sks: cannot mark unsent keybag create retryable: {:?}\n", + e + ); } } } @@ -59,7 +67,12 @@ impl SepData { if let Some(a) = matched { let _ = self.sks_zero_buffers(); self.sks_wedged.store(0, Relaxed); - dev_warn!(self.dev, "sks: late answer to {} after {} ms; wedge lifted\n", a.label, a.waited_ms); + dev_warn!( + self.dev, + "sks: late answer to {} after {} ms; wedge lifted\n", + a.label, + a.waited_ms + ); } } } @@ -134,6 +147,7 @@ impl SepData { floor_ms: time::Msecs, condemn: bool, ) -> Option { + let _exchange = self.sks_exchange_lock.lock(); if !self.ool_registered(&self.ool_sks) { return None; } @@ -206,7 +220,12 @@ impl SepData { if condemn { self.sks_wedged.store(1, Relaxed); } - dev_err!(self.dev, "sks: {} got no reply after {} ms\n", label, waited_ms); + dev_err!( + self.dev, + "sks: {} got no reply after {} ms\n", + label, + waited_ms + ); return None; }; @@ -220,8 +239,7 @@ impl SepData { } } - // The correlated reply means the enclave has finished with both OOL - // buffers. Keep only the private response copy returned to the caller. + // The correlated reply means the enclave has finished with both OOL buffers. let _ = self.sks_zero_buffers(); Some(SksOutcome { reply, response }) @@ -299,7 +317,10 @@ impl SepData { } /// `0x05` unload the source handle. - pub(crate) fn sks_req_unload_keybag(&self, handle: crate::sks::KeyBagHandle) -> Result { + pub(crate) fn sks_req_unload_keybag( + &self, + handle: crate::sks::KeyBagHandle, + ) -> Result { let mut body = image::Body::new(); body.put_u32(0)?; body.put_u64(crate::sks::SKS_CLIENT_ID)?; @@ -315,7 +336,10 @@ impl SepData { } /// `0x06` UUID read, against the special handle. - pub(crate) fn sks_req_copy_uuid_special(&self, special: crate::sks::SpecialHandle) -> Result { + pub(crate) fn sks_req_copy_uuid_special( + &self, + special: crate::sks::SpecialHandle, + ) -> Result { let op = crate::sks::sks_copy_keybag_uuid(); let mut body = image::Body::new(); body.put_u32(0)?; @@ -340,7 +364,8 @@ impl SepData { body.put_u64(SKS_LOCK_STATE_FLAGS)?; let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; - let msg = crate::sks::encode_sks_change_lock_state(self.sks_next_seq(), len).ok_or(EINVAL)?; + let msg = + crate::sks::encode_sks_change_lock_state(self.sks_next_seq(), len).ok_or(EINVAL)?; Ok(SksRequest { name: crate::sks::SKS_LOCK_STATE_NAME, msg, @@ -418,7 +443,11 @@ impl SepData { let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; let msg = crate::sks::encode_sks_create(self.sks_next_seq(), len).ok_or(EINVAL)?; - Ok(SksRequest { name: crate::sks::SKS_CREATE_NAME, msg, img }) + Ok(SksRequest { + name: crate::sks::SKS_CREATE_NAME, + msg, + img, + }) } pub(crate) fn sks_provision_identity_keybag(&self) -> bool { @@ -426,7 +455,11 @@ impl SepData { Ok(keybag::State::Present(_)) => return true, Ok(keybag::State::Absent(proof)) => proof, Err(e) => { - dev_err!(self.dev, "sks: identity keybag state is ambiguous: {:?}\n", e); + dev_err!( + self.dev, + "sks: identity keybag state is ambiguous: {:?}\n", + e + ); return false; } }; @@ -452,7 +485,11 @@ impl SepData { let request = match self.sks_req_create_identity_keybag(&secret, &uuid, proof) { Ok(request) => request, Err(e) => { - dev_err!(self.dev, "sks: cannot build identity keybag request: {:?}\n", e); + dev_err!( + self.dev, + "sks: cannot build identity keybag request: {:?}\n", + e + ); return false; } }; @@ -464,7 +501,12 @@ impl SepData { return false; }; if out.reply.status != 0 || body.len() < 12 { - dev_err!(self.dev, "sks: CREATE_KEYBAG failed: mailbox {}, body {} bytes\n", out.reply.status, body.len()); + dev_err!( + self.dev, + "sks: CREATE_KEYBAG failed: mailbox {}, body {} bytes\n", + out.reply.status, + body.len() + ); return false; } let variant = u32::from_le_bytes(body[0..4].try_into().unwrap()); @@ -472,8 +514,14 @@ impl SepData { let Some((_fv_data, end)) = image::read_blob(body, 8) else { return false; }; - if variant != crate::sks::SKS_CREATE_VARIANT_IDENTITY || raw_handle < 0 || end != body.len() { - dev_err!(self.dev, "sks: invalid CREATE_KEYBAG reply: variant {}, handle {}\n", variant, raw_handle); + if variant != crate::sks::SKS_CREATE_VARIANT_IDENTITY || raw_handle < 0 || end != body.len() + { + dev_err!( + self.dev, + "sks: invalid CREATE_KEYBAG reply: variant {}, handle {}\n", + variant, + raw_handle + ); return false; } let handle = crate::sks::KeyBagHandle::from_create_reply(raw_handle); @@ -502,7 +550,11 @@ impl SepData { } } - pub(crate) fn sks_designate_user_keybag(&self, handle: crate::sks::KeyBagHandle, secret: &[u8]) { + pub(crate) fn sks_designate_user_keybag( + &self, + handle: crate::sks::KeyBagHandle, + secret: &[u8], + ) { let Some(user) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) else { return; }; @@ -510,12 +562,13 @@ impl SepData { let designation = crate::sks::Designation::new(handle, user); let Some(out) = self.sks_send(self.sks_req_designate(&designation, secret)) else { - dev_warn!(self.dev, "sks: DESIGNATE_KEYBAG did not complete; enrolment will refuse\n"); + dev_warn!( + self.dev, + "sks: DESIGNATE_KEYBAG did not complete; enrolment will refuse\n" + ); return; }; - let Some(body) = - self.sks_report_response(crate::sks::SKS_DESIGNATE_NAME, &out) - else { + let Some(body) = self.sks_report_response(crate::sks::SKS_DESIGNATE_NAME, &out) else { return; }; @@ -529,17 +582,10 @@ impl SepData { self.keybag_designated.store(true, Relaxed); - self.sks_remember_enrolment_material( - designation.user().special_handle(), - secret, - ); + self.sks_remember_enrolment_material(designation.user().special_handle(), secret); } - fn sks_remember_enrolment_material( - &self, - special: crate::sks::SpecialHandle, - secret: &[u8], - ) { + fn sks_remember_enrolment_material(&self, special: crate::sks::SpecialHandle, secret: &[u8]) { let mut copy = KVec::new(); if copy.extend_from_slice(secret, GFP_KERNEL).is_err() { return; @@ -569,7 +615,8 @@ impl SepData { body.put_u64(SKS_LOCK_STATE_FLAGS)?; let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; - let msg = crate::sks::encode_sks_change_lock_state(self.sks_next_seq(), len).ok_or(EINVAL)?; + let msg = + crate::sks::encode_sks_change_lock_state(self.sks_next_seq(), len).ok_or(EINVAL)?; Ok(SksRequest { name: crate::sks::SKS_LOCK_STATE_NAME, msg, @@ -616,11 +663,20 @@ impl SepData { pub(crate) fn sks_health_check(&self, why: &CStr) -> Option { let Some(out) = self.sks_send(self.sks_req_get_capabilities()) else { - dev_err!(self.dev, "sks: health check ({}) got no reply; endpoint gone for this boot\n", why); + dev_err!( + self.dev, + "sks: health check ({}) got no reply; endpoint gone for this boot\n", + why + ); return None; }; if out.reply.status != 0 { - dev_err!(self.dev, "sks: health check ({}) returned status {}\n", why, out.reply.status); + dev_err!( + self.dev, + "sks: health check ({}) returned status {}\n", + why, + out.reply.status + ); return None; } Some(Healthy(())) @@ -633,7 +689,11 @@ impl SepData { let request = match self.sks_req_load_keybag(stored.wrapped()) { Ok(request) => request, Err(e) => { - dev_err!(self.dev, "sks: could not build LOAD_KEYBAG request: {:?}\n", e); + dev_err!( + self.dev, + "sks: could not build LOAD_KEYBAG request: {:?}\n", + e + ); return None; } }; @@ -650,7 +710,10 @@ impl SepData { let body = match self.sks_report_response(crate::sks::SKS_LOAD_NAME, &out) { Some(body) => body, None => { - dev_warn!(self.dev, "sks: LOAD_KEYBAG reply image was empty or malformed\n"); + dev_warn!( + self.dev, + "sks: LOAD_KEYBAG reply image was empty or malformed\n" + ); return None; } }; @@ -664,13 +727,23 @@ impl SepData { return None; } if body.len() != SKS_LOAD_REPLY_LEN { - dev_warn!(self.dev, "sks: LOAD_KEYBAG returned {} bytes, expected {}\n", body.len(), SKS_LOAD_REPLY_LEN); + dev_warn!( + self.dev, + "sks: LOAD_KEYBAG returned {} bytes, expected {}\n", + body.len(), + SKS_LOAD_REPLY_LEN + ); return None; } let status = i32::from_le_bytes([body[0], body[1], body[2], body[3]]); let handle = i32::from_le_bytes([body[4], body[5], body[6], body[7]]); if status != 0 || handle < 0 { - dev_warn!(self.dev, "sks: LOAD_KEYBAG operation status {}, handle {}\n", status, handle); + dev_warn!( + self.dev, + "sks: LOAD_KEYBAG operation status {}, handle {}\n", + status, + handle + ); return None; } let handle = crate::sks::KeyBagHandle::from_load_reply(handle); @@ -769,7 +842,11 @@ impl SepData { return false; } if let Err(e) = self.enable_sks() { - dev_err!(self.dev, "sks: could not register out-of-line buffers ({:?})\n", e); + dev_err!( + self.dev, + "sks: could not register out-of-line buffers ({:?})\n", + e + ); return false; } true @@ -938,7 +1015,6 @@ static_assert!(SKS_CLIENT_ID == 0x4c49_4e55_5853_4b53); pub(crate) struct KeyBagHandle(i32); impl KeyBagHandle { - pub(crate) const fn from_create_reply(v: i32) -> KeyBagHandle { KeyBagHandle(v) } @@ -1088,7 +1164,11 @@ pub(crate) fn encode_sks_load(seq: Sequence, len: ImageLen) -> Option { } pub(crate) fn encode_sks_create(seq: Sequence, len: ImageLen) -> Option { - Some(encode_sks_raw(OP_SKS_CREATE_KEYBAG, seq.value(), len.value())) + Some(encode_sks_raw( + OP_SKS_CREATE_KEYBAG, + seq.value(), + len.value(), + )) } pub(crate) fn encode_sks_change_lock_state(seq: Sequence, len: ImageLen) -> Option { From be4b09d9c49c47707488411949646787f04a7952 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 20:41:00 -0400 Subject: [PATCH 0696/2680] soc: apple: restrict the fixed power-GPIO fallback to j414s When the sensor node carries no power GPIO the shim fell back to a fixed pinctrl node and line number specific to j414s. That fallback is machine-blind: on a sibling board whose sensor sits on a different line it would drive the wrong line. Take the fallback only on j414s; every other machine must describe the power GPIO in its device-tree node. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- drivers/soc/apple/sensor_shim.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index a182d6ac04817..924433b094a64 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -70,6 +70,17 @@ static void sep_acquire_power(struct spi_device *spi) } sep_power = NULL; + /* + * The fallback below is the j414s power line; a sibling board's sensor sits + * on a different pin, so only j414s may use it. Every other machine must + * describe the power GPIO in its device node (the gpiod_get_index path). + */ + if (!of_machine_is_compatible("apple,j414s")) { + dev_warn(&spi->dev, + "sep sensor: no power GPIO in the device node; describe gpios in DT\n"); + return; + } + np = of_find_node_by_path(SEP_SENSOR_GPIO_NODE); if (!np) { dev_warn(&spi->dev, From 5aacb54d455ba212c3f048784f0affeeb7bf59b5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 20:41:00 -0400 Subject: [PATCH 0697/2680] soc: apple: find the fingerprint sensor by its device-tree node Sensor bring-up located its SPI controller by a fixed address and so only found the bus on j414s. Look up the apple,mesa-fingerprint node wherever the device tree describes it and enable that node and its parent controller directly, taking the power and interrupt GPIOs from the node. Any machine that describes the sensor in its device tree can then bring it up. When no such node exists the driver still creates one at the caller's fixed controller address, so j414s is unchanged. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- drivers/soc/apple/dt.rs | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/dt.rs b/drivers/soc/apple/dt.rs index ca3b9445e8d60..148e8268b2950 100644 --- a/drivers/soc/apple/dt.rs +++ b/drivers/soc/apple/dt.rs @@ -65,6 +65,13 @@ impl DtNode { (!np.is_null()).then_some(DtNode(np)) } + fn parent(&self) -> Option { + // SAFETY: `self.0` is a valid node per the type invariant; `of_get_parent` + // takes a reference on what it returns and returns NULL at the root. + let np = unsafe { bindings::of_get_parent(self.0) }; + (!np.is_null()).then_some(DtNode(np)) + } + fn as_ptr(&self) -> *mut bindings::device_node { self.0 } @@ -82,7 +89,6 @@ impl DtNode { }; !p.is_null() } - } impl Drop for DtNode { @@ -392,6 +398,33 @@ unsafe fn add_u32( } pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { + // A machine that describes the sensor in its own device tree (compatible + // apple,mesa-fingerprint) is handled machine-agnostically: enable that node + // and its SPI controller wherever they sit, with the power/IRQ GPIOs coming + // from the node itself. Only when no such node exists do we fall back to + // creating one at the caller's fixed controller address. + if let Some(sensor) = DtNode::find_compatible(SENSOR_COMPATIBLE) { + let controller = sensor.parent().ok_or_else(|| { + pr_err!("apple_sep: sensor node has no parent SPI controller\n"); + ENODEV + })?; + pr_info!( + "apple_sep: sensor node present in the device tree; enabling it and its SPI bus\n" + ); + return with_changeset(|cs_handle| { + let mut queued = false; + if !controller.is_available() { + queue_status_okay(cs_handle, &controller)?; + queued = true; + } + if !sensor.is_available() { + queue_status_okay(cs_handle, &sensor)?; + queued = true; + } + Ok(queued) + }); + } + let controller = node_at_address(base).ok_or_else(|| { pr_err!( "apple_sep: no device-tree node with reg base 0x{:x}; the sensor's SPI bus is not in this tree\n", From 6fe75ed650ed3f4b0b913a3a715c784f7f859a68 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 20:41:00 -0400 Subject: [PATCH 0698/2680] soc: apple: accept the newer fingerprint sensor revision The match path rejected any sensor whose reported identifier was not 0x3352, the revision fitted up to M4. Newer machines ship a 0x335e part that is identical over the wire. Accept both revisions so matching is not refused on those machines. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- drivers/soc/apple/sbio.rs | 241 +++++++++++++++++------------------- drivers/soc/apple/sensor.rs | 15 +-- 2 files changed, 118 insertions(+), 138 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 29740ce17e1d0..ed23c237c531d 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -4,10 +4,10 @@ //! The enclave matches; no biometric image ever crosses to userspace. use super::*; -use kernel::prelude::*; -use kernel::soc::apple::mailbox::Message; use crate::proto::*; use crate::sks::SKS_AUTH_TOKEN_LEN; +use kernel::prelude::*; +use kernel::soc::apple::mailbox::Message; impl SepData { fn sbio_expect_ok(&self, op: &crate::sbio::SbioOp) -> Option> { @@ -31,15 +31,9 @@ impl SepData { match err as u16 { crate::sbio::SBIO_STATUS_OK => SbioOutcome::Ok(done.payload), - crate::sbio::SBIO_STATUS_PREREQUISITE => { - SbioOutcome::PrerequisiteMissing - } - crate::sbio::SBIO_STATUS_16 => { - SbioOutcome::Status16 - } - _ => { - SbioOutcome::Other - } + crate::sbio::SBIO_STATUS_PREREQUISITE => SbioOutcome::PrerequisiteMissing, + crate::sbio::SBIO_STATUS_16 => SbioOutcome::Status16, + _ => SbioOutcome::Other, } } @@ -58,15 +52,9 @@ impl SepData { ); let policy_ok = match self.sbio_call(&crate::sbio::sbio_match_policy()) { - SbioOutcome::Ok(policy) if policy.len() == crate::sbio::SBIO_MATCH_POLICY_LEN => { - true - } - SbioOutcome::Ok(_) => { - false - } - _ => { - false - } + SbioOutcome::Ok(policy) if policy.len() == crate::sbio::SBIO_MATCH_POLICY_LEN => true, + SbioOutcome::Ok(_) => false, + _ => false, }; synced && policy_ok @@ -101,7 +89,9 @@ impl SepData { ); } - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(ENROL_REPOSITION_MS))); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from( + ENROL_REPOSITION_MS, + ))); } fn stash_enrol_identity(&self, record: &[u8]) { @@ -290,7 +280,11 @@ impl SepData { true } - fn open_fresh_context(&self, user: crate::sbio::UserId, proof: &crate::sbio::NoExistingCatacomb) -> bool { + fn open_fresh_context( + &self, + user: crate::sbio::UserId, + proof: &crate::sbio::NoExistingCatacomb, + ) -> bool { let system = crate::sbio::sbio_select_context(crate::sbio::ContextScope::SYSTEM, proof); if self.sbio_expect_ok(&system).is_none() { return false; @@ -313,7 +307,8 @@ impl SepData { } } - let per_user = crate::sbio::sbio_select_context(crate::sbio::ContextScope::user(user), proof); + let per_user = + crate::sbio::sbio_select_context(crate::sbio::ContextScope::user(user), proof); if self.sbio_expect_ok(&per_user).is_none() { return false; } @@ -365,7 +360,8 @@ impl SepData { ); return None; }; - let op = crate::sbio::sbio_begin_enrol(user, crate::sbio::BE_AUTH_TYPE_ACM_CONTEXT, &acm_handle); + let op = + crate::sbio::sbio_begin_enrol(user, crate::sbio::BE_AUTH_TYPE_ACM_CONTEXT, &acm_handle); match self.sbio_call(&op) { SbioOutcome::Ok(_payload) => { @@ -375,9 +371,7 @@ impl SepData { armed: true, }) } - _ => { - None - } + _ => None, } } @@ -469,12 +463,8 @@ impl SepData { return RestoreOutcome::Failed; }; match crate::sbio::component_action(state) { - crate::sbio::ComponentAction::AlreadyActive => { - RestoreOutcome::AlreadyActive - } - crate::sbio::ComponentAction::Unsupported => { - RestoreOutcome::Failed - } + crate::sbio::ComponentAction::AlreadyActive => RestoreOutcome::AlreadyActive, + crate::sbio::ComponentAction::Unsupported => RestoreOutcome::Failed, crate::sbio::ComponentAction::Load => { let Some(blob) = self.read_stored(kind, what) else { return RestoreOutcome::NoStoredFile; @@ -499,13 +489,15 @@ impl SepData { self.confirm_active(id, what, LoadAnswer::StatusZero) } Ok(done) - if done.status.answered() == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) + if done.status.answered() + == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) && cold_ok => { self.confirm_active(id, what, LoadAnswer::ColdTransition) } Ok(done) - if done.status.answered() == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) => + if done.status.answered() + == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) => { RestoreOutcome::Failed } @@ -516,9 +508,7 @@ impl SepData { { self.confirm_active(id, what, LoadAnswer::AlreadyActive) } - _ => { - RestoreOutcome::Failed - } + _ => RestoreOutcome::Failed, } } } @@ -540,15 +530,9 @@ impl SepData { Some(state) if state & crate::sbio::COMPONENT_STATE_ACTIVE != 0 => { RestoreOutcome::Restored } - Some(state) if state & crate::sbio::COMPONENT_STATE_COLD != 0 => { - RestoreOutcome::Failed - } - Some(_) => { - RestoreOutcome::Failed - } - None => { - RestoreOutcome::Failed - } + Some(state) if state & crate::sbio::COMPONENT_STATE_COLD != 0 => RestoreOutcome::Failed, + Some(_) => RestoreOutcome::Failed, + None => RestoreOutcome::Failed, } } @@ -560,9 +544,7 @@ impl SepData { return RestoreOutcome::Failed; }; match self.sbio_transfer_raw(request.opcode(), request.name(), request.payload()) { - Ok(done) if done.status.is_ok() => { - RestoreOutcome::Restored - } + Ok(done) if done.status.is_ok() => RestoreOutcome::Restored, Ok(done) if matches!( done.status, @@ -571,12 +553,8 @@ impl SepData { { RestoreOutcome::EmptyTolerated } - Ok(_) => { - RestoreOutcome::Failed - } - Err(_) => { - RestoreOutcome::Failed - } + Ok(_) => RestoreOutcome::Failed, + Err(_) => RestoreOutcome::Failed, } } @@ -589,9 +567,7 @@ impl SepData { match self.with_host_store(|store| store.read(&store::Key::root(kind))) { Some(Ok(Some(blob))) => Some(blob), Some(Ok(None)) => None, - Some(Err(_)) => { - None - } + Some(Err(_)) => None, None => None, } } @@ -626,7 +602,6 @@ impl SepData { } fn save_after_match(&self, user: crate::sbio::UserId) -> bool { - if !self.save_lockout() { dev_err!( self.dev, @@ -706,13 +681,11 @@ impl SepData { if blob.is_empty() { return false; } - match self.with_host_store(|store| store.write(&store::Key::root(PRIVATE_TYPE_LOCKOUT), &blob)) { - Some(Ok(())) => { - true - } - Some(Err(_)) => { - false - } + match self + .with_host_store(|store| store.write(&store::Key::root(PRIVATE_TYPE_LOCKOUT), &blob)) + { + Some(Ok(())) => true, + Some(Err(_)) => false, None => false, } } @@ -776,11 +749,17 @@ impl SepData { } if *module_parameters::provision_keybag.value() != 0 { if *module_parameters::xart_writes.value() == 0 { - dev_err!(self.dev, "bringup: provision_keybag=1 requires xart_writes=1\n"); + dev_err!( + self.dev, + "bringup: provision_keybag=1 requires xart_writes=1\n" + ); return; } if !self.sks_provision_identity_keybag() { - dev_err!(self.dev, "bringup: identity-keybag provisioning failed; no retry this boot\n"); + dev_err!( + self.dev, + "bringup: identity-keybag provisioning failed; no retry this boot\n" + ); return; } } @@ -802,21 +781,30 @@ impl SepData { return Err(EIO); } - let mut store = store::Store::open().map_err(|e| { - dev_err!(self.dev, "Touch ID: opening the host state failed: {:?}\n", e); - e + let mut store = store::Store::open().inspect_err(|e| { + dev_err!( + self.dev, + "Touch ID: opening the host state failed: {:?}\n", + e + ); })?; - let index = bio::IdentityIndex::load(&mut store).map_err(|e| { - dev_err!(self.dev, "Touch ID: loading the host identity index failed: {:?}\n", e); - e + let index = bio::IdentityIndex::load(&mut store).inspect_err(|e| { + dev_err!( + self.dev, + "Touch ID: loading the host identity index failed: {:?}\n", + e + ); })?; *self.host_store.lock() = Some(store); *self.bio_index.lock() = index; self.attach_sensor(); - self.enable_sbio().map_err(|e| { - dev_err!(self.dev, "Touch ID: registering the biometric buffers failed: {:?}\n", e); - e + self.enable_sbio().inspect_err(|e| { + dev_err!( + self.dev, + "Touch ID: registering the biometric buffers failed: {:?}\n", + e + ); })?; let stored = match keybag::read(keybag::Slot::Identity) { Ok(keybag::State::Present(stored)) => stored, @@ -825,16 +813,26 @@ impl SepData { return Err(ENOENT); } Err(e) => { - dev_err!(self.dev, "Touch ID: reading the persisted identity keybag failed: {:?}\n", e); + dev_err!( + self.dev, + "Touch ID: reading the persisted identity keybag failed: {:?}\n", + e + ); return Err(e); } }; if !self.sks_ready() { - dev_err!(self.dev, "Touch ID: the key-store endpoint is unavailable\n"); + dev_err!( + self.dev, + "Touch ID: the key-store endpoint is unavailable\n" + ); return Err(ENODEV); } let (handle, uuid) = self.sks_recover(&stored).ok_or_else(|| { - dev_err!(self.dev, "Touch ID: the persisted identity keybag did not recover\n"); + dev_err!( + self.dev, + "Touch ID: the persisted identity keybag did not recover\n" + ); EIO })?; self.sks_designate_user_keybag(handle, stored.secret()); @@ -1121,7 +1119,10 @@ impl SepData { let stage = self.bringup.load(Relaxed); if stage >= BRINGUP_ESTABLISHED { - if self.sbio_expect_ok(&crate::sbio::sbio_clear_state()).is_none() { + if self + .sbio_expect_ok(&crate::sbio::sbio_clear_state()) + .is_none() + { return false; } } @@ -1151,9 +1152,7 @@ impl SepData { self.bringup.store(BRINGUP_ESTABLISHED, Relaxed); true } - _ => { - false - } + _ => false, } } @@ -1184,7 +1183,7 @@ impl SepData { } if let sensor::Offset12::Identifier(id) = st.offset12() { - if id != 0 && id != sensor::EXPECTED_IDENTIFIER { + if id != 0 && !sensor::KNOWN_IDENTIFIERS.contains(&id) { return None; } } @@ -1283,14 +1282,16 @@ impl SepData { } fn complete_bringup(&self, patch: PatchLoaded) -> bool { - let Some(params) = self.apply_sensor_parameters() else { return false; }; let op = crate::sbio::sbio_complete_init(patch, params); if self.sbio_expect_ok(&op).is_none() { - dev_err!(self.dev, "sensor: 0x01 COMPLETE_INIT failed — status above\n"); + dev_err!( + self.dev, + "sensor: 0x01 COMPLETE_INIT failed — status above\n" + ); return false; } @@ -1368,12 +1369,8 @@ impl SepData { let relayed = blob.len(); match sensor::send_encrypted_parameters(&blob, geom) { - Ok(()) => { - Some(relayed) - } - Err(sensor::ParamsError::Empty) => { - None - } + Ok(()) => Some(relayed), + Err(sensor::ParamsError::Empty) => None, Err(sensor::ParamsError::TooLong(n, capacity)) => { dev_err!( self.dev, @@ -1385,14 +1382,11 @@ impl SepData { ); None } - Err(sensor::ParamsError::Transfer(_e)) => { - None - } + Err(sensor::ParamsError::Transfer(_e)) => None, } } fn establish_session(&self) -> bool { - let share = match self.sbio_call(&crate::sbio::sbio_request_session_share()) { SbioOutcome::Ok(sh) => sh, SbioOutcome::Status16 => { @@ -1426,20 +1420,13 @@ impl SepData { }; match self.sbio_transfer(&crate::sbio::sbio_commit_session_share(&reply)) { - Ok(done) if done.status.is_ok() => { - true - } - Ok(_) => { - false - } - Err(_) => { - false - } + Ok(done) if done.status.is_ok() => true, + Ok(_) => false, + Err(_) => false, } } fn init_sequence_counter(&self) -> bool { - let challenge = match self.sbio_call(&crate::sbio::sbio_request_challenge()) { SbioOutcome::Ok(c) => c, SbioOutcome::PrerequisiteMissing => { @@ -1478,15 +1465,9 @@ impl SepData { }; match self.sbio_transfer(&crate::sbio::sbio_commit_challenge(&reply)) { - Ok(done) if done.status.is_ok() => { - true - } - Ok(_) => { - false - } - Err(_) => { - false - } + Ok(done) if done.status.is_ok() => true, + Ok(_) => false, + Err(_) => false, } } @@ -1538,7 +1519,6 @@ impl SepData { } }; if st.patch_ack() == sensor::PATCH_ACCEPTED { - if let Ok(after) = sensor::status() { if after.state == sensor::STATE_NEEDS_PATCH { return None; @@ -1584,7 +1564,10 @@ impl SepData { let capture = match sensor::read_capture(available) { Ok(c) => c, - Err(sensor::CaptureError::Checksum { advertised: _advertised, computed: _computed }) => { + Err(sensor::CaptureError::Checksum { + advertised: _advertised, + computed: _computed, + }) => { return ImageOutcome::Retry; } Err(sensor::CaptureError::Length(_n)) => { @@ -1768,11 +1751,11 @@ impl SepData { // 0xFE requests the peer's next packet and acks the final one; nothing to send self.sbio_wq.notify_all(); } - transfer::Progress::Ignored => {}, + transfer::Progress::Ignored => {} transfer::Progress::Grant => { self.sbio_wq.notify_all(); } - transfer::Progress::Notification => {}, + transfer::Progress::Notification => {} transfer::Progress::Failed => { self.sbio_wq.notify_all(); } @@ -2070,9 +2053,10 @@ impl SepData { fn bio_attest(&self, arg: usize) -> Result { let user = kernel::uaccess::UserPtr::from_addr(arg); - let req: bio::Attest = kernel::uaccess::UserSlice::new(user, core::mem::size_of::()) - .reader() - .read()?; + let req: bio::Attest = + kernel::uaccess::UserSlice::new(user, core::mem::size_of::()) + .reader() + .read()?; let (sig, pubk) = self.refkey_attest_sign(&req.challenge)?; if sig.len() > bio::ATTEST_SIG_MAX || pubk.len() != bio::ATTEST_PUB_LEN { return Err(EIO); @@ -2146,7 +2130,7 @@ impl SepData { .sks_send(self.sks_req_copy_uuid_special(special)) .and_then(|out| self.sks_uuid_from_reply(&out)); match uuid_ok { - Some(got) if got == uuid => {}, + Some(got) if got == uuid => {} Some(_) => { return false; } @@ -2172,7 +2156,11 @@ impl SepData { Ok(()) } - pub(crate) fn wrapped_from_copy_reply(&self, out: &SksOutcome, _from: &CStr) -> Option> { + pub(crate) fn wrapped_from_copy_reply( + &self, + out: &SksOutcome, + _from: &CStr, + ) -> Option> { let body = self.sks_report_response(c"COPY_KEYBAG", out)?; if out.reply.status != 0 || image::operation_status(body).unwrap_or(-1) != 0 { return None; @@ -3050,7 +3038,6 @@ impl ImagePurpose { ImagePurpose::Matching => 0x09, } } - } static_assert!(ImagePurpose::Enrolment.offset() != ImagePurpose::Matching.offset()); diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index 79d9e3bc7d50c..d94cc00a1cff2 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -18,12 +18,7 @@ extern "C" { fn sep_sensor_power(on: c_int) -> c_int; fn sep_sensor_xfer(tx: *const c_void, rx: *mut c_void, len: usize) -> c_int; fn sep_sensor_xfer_tx(tx: *const c_void, len: usize) -> c_int; - fn sep_sensor_xfer2( - tx: *const c_void, - tx_len: usize, - rx: *mut c_void, - rx_len: usize, - ) -> c_int; + fn sep_sensor_xfer2(tx: *const c_void, tx_len: usize, rx: *mut c_void, rx_len: usize) -> c_int; } // Spi2. @@ -89,7 +84,8 @@ impl PowerSource { pub(crate) const POWER_ON_READ_DELAYS_MS: [u32; 4] = [0, 3, 10, 50]; pub(crate) const STATUS_IDENTIFIER: usize = 12; -pub(crate) const EXPECTED_IDENTIFIER: u16 = 0x3352; +// Known Mesa sensor revisions: 0x3352 through M4, 0x335e on newer parts. +pub(crate) const KNOWN_IDENTIFIERS: [u16; 2] = [0x3352, 0x335e]; static_assert!(STATUS_IDENTIFIER + 2 <= STATUS_LEN); const CMD_LEN: usize = 7; @@ -613,9 +609,7 @@ pub(crate) fn status() -> Result { let mut rx = [0u8; STATUS_XFER_LEN]; // SAFETY: both buffers are `STATUS_XFER_LEN` bytes and live across the call. - check(unsafe { - sep_sensor_xfer(tx.as_ptr().cast(), rx.as_mut_ptr().cast(), STATUS_XFER_LEN) - })?; + check(unsafe { sep_sensor_xfer(tx.as_ptr().cast(), rx.as_mut_ptr().cast(), STATUS_XFER_LEN) })?; let mut raw = [0u8; STATUS_LEN]; raw.copy_from_slice(&rx[STATUS_AT..STATUS_AT + STATUS_LEN]); @@ -638,7 +632,6 @@ impl Capture { pub(crate) fn bytes(&self) -> &[u8] { &self.0 } - } impl Drop for Capture { From e70916265d0b7516be5dac8a0f56af643d8d6a79 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 13:42:47 -0400 Subject: [PATCH 0699/2680] dt-bindings: apple: add mesa-fingerprint and dart dma-range Add a binding for the Mesa fingerprint sensor on the SPI2 bus of Apple laptops, driven by the Secure Enclave through a /dev/sep-bio character device. The sensor uses the SPI controller's native chip select; a board describes its power line (enable-gpios), an optional data-ready line (interrupts), and the per-device calibration firmware name. Also document the apple,dma-range DART property, used to place the SEP firmware mapping on the M1 boot path. Signed-off-by: Chromatischer [dj: adapted for aurora feat/sep, clean-room] Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 --- .../input/apple,mesa-fingerprint.yaml | 77 +++++++++++++++++++ MAINTAINERS | 1 + 2 files changed, 78 insertions(+) create mode 100644 Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml diff --git a/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml b/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml new file mode 100644 index 0000000000000..be9bbcf473e85 --- /dev/null +++ b/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/input/apple,mesa-fingerprint.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple Mesa fingerprint sensor + +maintainers: + - Chromatischer + +description: | + Fingerprint sensor on an SPI bus of an Apple SoC, normally owned by the + Secure Enclave Processor. The chip-select setup and hold delays have to be + produced by the controller, so the device must use the native chip select of + its parent controller and not a GPIO one. + + The SPI mode is not given a default here because the boards that carry this + sensor have not been observed to agree on one, so each board description has + to state the mode it needs. + +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +properties: + compatible: + const: apple,mesa-fingerprint + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + description: Data-ready line from the sensor. + + enable-gpios: + maxItems: 1 + description: Power line to the sensor, asserted to power it up. + + firmware-name: + maxItems: 1 + description: + Calibration blob for this sensor. It is per-device factory data written + at manufacture, so it is neither redistributable nor derivable from the + compatible; the board description has to name it. + + spi-cpha: true + + spi-cpol: true + +required: + - compatible + - reg + - enable-gpios + - firmware-name + +unevaluatedProperties: false + +examples: + - | + #include + + spi { + #address-cells = <1>; + #size-cells = <0>; + + fingerprint@0 { + compatible = "apple,mesa-fingerprint"; + reg = <0>; + spi-max-frequency = <8000000>; + spi-cpha; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; + enable-gpios = <&pinctrl_ap 108 GPIO_ACTIVE_HIGH>; + firmware-name = "apple/mesacal-j313.bin"; + }; + }; diff --git a/MAINTAINERS b/MAINTAINERS index b60cd73cce49d..fc1651b6d8159 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2598,6 +2598,7 @@ F: Documentation/devicetree/bindings/gpio/apple,smc-gpio.yaml F: Documentation/devicetree/bindings/gpu/apple,agx.yaml F: Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml F: Documentation/devicetree/bindings/i2c/apple,i2c.yaml +F: Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml F: Documentation/devicetree/bindings/input/touchscreen/apple,z2-multitouch.yaml F: Documentation/devicetree/bindings/interrupt-controller/apple,* F: Documentation/devicetree/bindings/iommu/apple,dart.yaml From 9383917a5560559ef063b0c94f8c8e2eb7a9adb5 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 14:58:52 -0400 Subject: [PATCH 0700/2680] soc: apple: sep: add typed platform profiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Name every SoC-specific fact the driver needs — the shared-memory capacity and first-item fourcc, the boot handshake, the OS-identity source and the sensor transport — in one typed profile, and refuse an unsupported SoC rather than run a handshake with the wrong geometry. Two targets are modelled: T8103/J313 (M1) boots the SEP with the boot endpoint handshake over a 0x30000 window whose first item is CNIP; T6020/J414s (M2 Pro) does the warm registration over a 0x40000 window whose first item is CINP. The shared-memory build takes the capacity and first-item spelling from the profile; T6020's values are unchanged. Signed-off-by: Chromatischer [dj: adapted for aurora feat/sep - dropped the storage-backend enum (single in-kernel owner), identity is /chosen only, kept the driver modular; clean-room] [ryan: adapted to 7.2.10 — the lines this commit changes keep the 7.2 lifetime-bound driver-core form (device::Core<'_>, Data<'bound>)] Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Ryan Murray --- drivers/soc/apple/profile.rs | 191 +++++++++++++++++++++++++++++++++++ drivers/soc/apple/sep.rs | 8 +- drivers/soc/apple/shmem.rs | 28 ++--- 3 files changed, 214 insertions(+), 13 deletions(-) create mode 100644 drivers/soc/apple/profile.rs diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs new file mode 100644 index 0000000000000..7d7bee3f0208a --- /dev/null +++ b/drivers/soc/apple/profile.rs @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Typed platform profiles for the SEP driver. +//! +//! Every SoC-specific fact the driver needs — the shared-memory capacity, the +//! boot handshake, the OS-identity source and the sensor transport — is named +//! here once. The driver reads addresses from the device tree, never scans for +//! them, and never patches properties at runtime. +//! +//! Two bring-up targets are modelled: +//! +//! * `T8103` / J313 (MacBook Air, M1): the host boots the SEP with the boot +//! endpoint handshake over a 0x30000 shared-memory window. +//! * `T6020` / J414s (MacBook Pro 14", M2 Pro): the driver does the warm +//! single-message registration over a 0x40000 window. + +// The boot handshake, identity source, sensor and DART fields are consumed by +// the boot-endpoint, identity and sensor-transport paths. +#![allow(dead_code)] + +use kernel::of; +use kernel::prelude::*; + +/// How the driver brings the shared-memory table to the SEP. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Bootstrap { + /// Cold boot: send TZ0, map the firmware reserved region and answer the + /// second boot acknowledgement with the firmware address and `SET_SHMEM`. + Boot, + /// Warm attach: send the single shared-memory registration message with the + /// table address and size. + WarmRegister, +} + +/// Where the 16-byte OS identity comes from. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum IdentitySource { + /// `/chosen/apfs-preboot-uuid`, the value the firmware handed to this boot + /// and the boot chain forwards. A malformed or absent property is refused; + /// the identity is never invented. + Chosen, + /// A UUID provisioned in the Linux host-state store on first bring-up. + HostPersisted, +} + +/// SPI mode as the device tree spells it. Mode 1 is CPOL=0/CPHA=1 (`spi-cpha`); +/// mode 2 is CPOL=1/CPHA=0 (`spi-cpol`). +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum SpiMode { + Mode1, + Mode2, +} + +impl SpiMode { + /// The value the SPI shim programs into `spi_setup()`, using the kernel's + /// `SPI_CPHA`/`SPI_CPOL` bits. + pub(crate) const fn wire(self) -> u32 { + match self { + SpiMode::Mode1 => 1, // SPI_CPHA + SpiMode::Mode2 => 2, // SPI_CPOL + } + } + + pub(crate) const fn name(self) -> &'static CStr { + match self { + SpiMode::Mode1 => c"mode 1 (CPOL=0 CPHA=1)", + SpiMode::Mode2 => c"mode 2 (CPOL=1 CPHA=0)", + } + } +} + +pub(crate) struct SensorProfile { + /// SPI controller register base. Diagnostics only: the device tree is + /// authoritative and the driver never enables or creates the bus. + pub(crate) controller_base: u64, + pub(crate) chip_select: u32, + pub(crate) max_hz: u32, + /// Chip-select setup and hold in nanoseconds. + pub(crate) cs_setup_ns: u32, + pub(crate) cs_hold_ns: u32, + pub(crate) mode: SpiMode, + pub(crate) expected_id: u16, + /// Capture traffic stays disabled until a qualified DMA transport exists; + /// the PIO path is status-only. + pub(crate) capture_qualified: bool, +} + +pub(crate) struct PlatformProfile { + pub(crate) name: &'static str, + /// Capacity of the boot shared-memory window. The layout is still checked + /// against the manifests; this only bounds the allocation. + pub(crate) shmem_capacity: usize, + /// Fourcc of the first shared-memory item. The two paths spell it + /// differently, so each profile carries its own spelling. + pub(crate) shmem_first_item: &'static [u8; 4], + pub(crate) bootstrap: Bootstrap, + pub(crate) identity: IdentitySource, + pub(crate) sensor: SensorProfile, + /// Require a static `apple,dma-range` on the SEP DART. The T6020 SEP only + /// accepts IOVAs below 4 GiB; T8103 works with the stock DART aperture. + pub(crate) dart_range_required: bool, + /// Reserved-memory region holding the SEP firmware image (cold-boot path). + pub(crate) firmware_region: &'static CStr, +} + +const T8103: PlatformProfile = PlatformProfile { + name: "T8103/J313", + shmem_capacity: 0x3_0000, + shmem_first_item: b"CNIP", + bootstrap: Bootstrap::Boot, + identity: IdentitySource::Chosen, + sensor: SensorProfile { + controller_base: 0x2_3510_8000, + chip_select: 0, + max_hz: 8_000_000, + cs_setup_ns: 20, + cs_hold_ns: 20, + mode: SpiMode::Mode1, + expected_id: 0x3352, + capture_qualified: false, + }, + dart_range_required: false, + firmware_region: c"sepfw", +}; + +const T6020: PlatformProfile = PlatformProfile { + name: "T6020/J414s", + shmem_capacity: 0x4_0000, + shmem_first_item: b"CINP", + bootstrap: Bootstrap::WarmRegister, + identity: IdentitySource::HostPersisted, + sensor: SensorProfile { + controller_base: 0x3_9b10_8000, + chip_select: 0, + max_hz: 8_000_000, + cs_setup_ns: 20, + cs_hold_ns: 20, + mode: SpiMode::Mode2, + expected_id: 0x3352, + capture_qualified: false, + }, + dart_range_required: true, + firmware_region: c"sepfw", +}; + +static_assert!(T8103.shmem_capacity == 0x30000); +static_assert!(T6020.shmem_capacity == 0x40000); +static_assert!(T8103.sensor.controller_base == 0x235108000); +static_assert!(T6020.sensor.controller_base == 0x39b108000); +static_assert!(T8103.sensor.expected_id == T6020.sensor.expected_id); + +/// Whether the machine root declares `compatible`. +/// +/// The root property is a NUL-separated list, so whole entries are compared +/// instead of substrings: `apple,t8103` cannot match a longer unrelated value. +fn machine_has(compatible: &[u8]) -> bool { + let Some(root) = of::root() else { + return false; + }; + let Ok(list) = root.get_property::>(c"compatible") else { + return false; + }; + + let mut at = 0usize; + while at < list.len() { + let end = match list[at..].iter().position(|&byte| byte == 0) { + Some(offset) => at + offset, + None => list.len(), + }; + if &list[at..end] == compatible { + return true; + } + at = end + 1; + } + + false +} + +/// Select the profile for the running machine. An unsupported SoC is refused +/// rather than guessed at, so the driver cannot run a handshake with the wrong +/// geometry. +pub(crate) fn detect() -> Result<&'static PlatformProfile> { + if machine_has(b"apple,t8103") { + return Ok(&T8103); + } + if machine_has(b"apple,t6020") { + return Ok(&T6020); + } + Err(ENODEV) +} diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 687bbff5b768c..fa3b86988ff2e 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -17,6 +17,7 @@ mod fv; mod hwrng; mod image; mod keybag; +mod profile; mod proto; mod refkey; mod refkey_seal; @@ -513,6 +514,8 @@ struct MachineRefKey { struct SepData { dev: ARef, + profile: &'static profile::PlatformProfile, + #[pin] mbox: Mutex>>, @@ -668,8 +671,10 @@ unsafe impl Sync for SepData {} impl SepData { fn new(pdev: &platform::Device>) -> Result> { - let built = shmem::build(pdev)?; + let profile = profile::detect()?; + let built = shmem::build(pdev, profile.shmem_capacity, profile.shmem_first_item)?; let dev: &device::Device> = pdev.as_ref(); + dev_info!(dev, "SEP platform profile: {}\n", profile.name); let buf = built.buf; @@ -741,6 +746,7 @@ impl SepData { Arc::pin_init( try_pin_init!(SepData { dev: ARef::::from(dev), + profile, mbox <- new_mutex!(None), shmem <- new_mutex!(Some(buf)), endpoints <- new_mutex!(EndpointTable::new()), diff --git a/drivers/soc/apple/shmem.rs b/drivers/soc/apple/shmem.rs index 1d9a7ba724d00..fb4e71962d466 100644 --- a/drivers/soc/apple/shmem.rs +++ b/drivers/soc/apple/shmem.rs @@ -6,8 +6,6 @@ use kernel::dma; use kernel::platform; use kernel::prelude::*; -pub(crate) const SHMEM_SIZE: usize = 0x40000; - const ENTRY_SIZE: usize = 16; const ENTRY_OFF_FOURCC: usize = 0; const ENTRY_OFF_SIZE: usize = 4; @@ -21,8 +19,8 @@ const CINP_MIN_SIZE: usize = 0x8000; const CINP_PAYLOAD: [u8; 1] = [0]; -// Wire byte order, not byte-reversed; llun is the terminator spelling. -const FOURCC_CINP: &[u8; 4] = b"CINP"; +// Wire byte order, not byte-reversed; llun is the terminator spelling. The +// first item's fourcc is per-SoC and comes from the platform profile. const FOURCC_OPLA: &[u8; 4] = b"OPLA"; const FOURCC_IPIS: &[u8; 4] = b"IPIS"; const FOURCC_TERM: &[u8; 4] = b"llun"; @@ -68,7 +66,7 @@ const fn align_up(v: usize, a: usize) -> usize { fn write_at(buf: &mut ShMem, off: usize, src: &[u8]) -> Result<()> { let end = off.checked_add(src.len()).ok_or(EINVAL)?; - if end > SHMEM_SIZE { + if end > buf.len() { return Err(EINVAL); } // SAFETY: runs in probe before the SEP is told the buffer exists, and probe @@ -131,9 +129,11 @@ fn verify_layout( opla: &Region, ipis: &Region, used: usize, + capacity: usize, + first_item: &[u8; 4], ) -> Result<()> { let regions = [ - (FOURCC_CINP, cinp), + (first_item, cinp), (FOURCC_OPLA, opla), (FOURCC_IPIS, ipis), ]; @@ -143,7 +143,7 @@ fn verify_layout( || r.offset % PAYLOAD_ALIGN != 0 || r.size % PAYLOAD_ALIGN != 0 || r.size < r.payload_len + 4 - || r.end() > SHMEM_SIZE; + || r.end() > capacity; if bad { dev_err!( dev, @@ -181,14 +181,18 @@ fn verify_layout( return Err(EINVAL); } - if 4 * ENTRY_SIZE > PAYLOAD_BASE || used > SHMEM_SIZE { + if 4 * ENTRY_SIZE > PAYLOAD_BASE || used > capacity { return Err(ENOSPC); } Ok(()) } -pub(crate) fn build(pdev: &platform::Device>) -> Result { +pub(crate) fn build( + pdev: &platform::Device>, + capacity: usize, + first_item: &[u8; 4], +) -> Result { let dev: &device::Device> = pdev.as_ref(); // Read manifests before allocating: a CINP-only registration would burn the one-shot and fault. @@ -200,9 +204,9 @@ pub(crate) fn build(pdev: &platform::Device>) -> Result let ipis = Region::place(opla.end(), ipis_blob.len(), 0); let used = ipis.end(); - verify_layout(dev, &cinp, &opla, &ipis, used)?; + verify_layout(dev, &cinp, &opla, &ipis, used, capacity, first_item)?; - let mut buf = dma::Coherent::::zeroed_slice(dev, SHMEM_SIZE, GFP_KERNEL)?; + let mut buf = dma::Coherent::::zeroed_slice(dev, capacity, GFP_KERNEL)?; // Payloads before entries: a failure leaves an all-zero table, not a valid-looking one. write_at( @@ -226,7 +230,7 @@ pub(crate) fn build(pdev: &platform::Device>) -> Result )?; write_at(&mut buf, ipis.offset + 4, &ipis_blob)?; - write_entry(&mut buf, 0, FOURCC_CINP, cinp.size, cinp.offset)?; + write_entry(&mut buf, 0, first_item, cinp.size, cinp.offset)?; write_entry(&mut buf, 1, FOURCC_OPLA, opla.size, opla.offset)?; write_entry(&mut buf, 2, FOURCC_IPIS, ipis.size, ipis.offset)?; write_entry(&mut buf, 3, FOURCC_TERM, 0, 0)?; From d435c0105617582883aa926851414d94c86268e7 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 15:08:33 -0400 Subject: [PATCH 0701/2680] soc: apple: sep: boot the SEP from the boot endpoint The T8103 SEP is not brought up by the warm registration message. On that target, send TZ0 on the boot endpoint and let its acknowledgements drive the handoff: map the firmware reserved region, submit the firmware address (IMG4) and the shared-memory table (SET_SHMEM), and mark the registration complete on the IMG4 acknowledgement. The warm target keeps the single registration message; the profile selects which path runs, and only the warm target enables the SEP and its DART with a runtime changeset - the cold-boot target's DART is already enabled before probe and is described statically. The firmware mapping is created once and released exactly once in remove(), gated on the teardown flag so a boot message racing removal cannot map firmware or hand memory to the peer after teardown. Signed-off-by: Chromatischer [dj: ported onto aurora feat/sep - reused the existing teardown flag and warm attach path, profile-gated the runtime DT enable, clean-room] Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 --- drivers/soc/apple/dt.rs | 36 ++++++++ drivers/soc/apple/proto.rs | 9 ++ drivers/soc/apple/sep.rs | 173 +++++++++++++++++++++++++++++++++++-- 3 files changed, 209 insertions(+), 9 deletions(-) diff --git a/drivers/soc/apple/dt.rs b/drivers/soc/apple/dt.rs index 148e8268b2950..46dd70d15d03f 100644 --- a/drivers/soc/apple/dt.rs +++ b/drivers/soc/apple/dt.rs @@ -469,3 +469,39 @@ pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { Ok(true) }) } + +/// Resolve a reserved-memory region by name on the SEP node's device. +/// +/// The cold-boot path needs this: the firmware image is handed over in a +/// bootloader-injected `sepfw` region. +pub(crate) fn reserved_region(dev: &kernel::device::Device, name: &CStr) -> Result<(u64, usize)> { + // SAFETY: `dev` is valid and its `of_node` is either NULL or a live node + // the device holds a reference to. + let np = unsafe { (*dev.as_raw()).of_node }; + if np.is_null() { + return Err(ENODEV); + } + + // SAFETY: `struct resource` is plain integers with no pointer state, so an + // all-zero value is valid. + let mut res: bindings::resource = unsafe { core::mem::zeroed() }; + // SAFETY: `np` is a live node, `name` is NUL-terminated and `res` is a + // valid out-parameter. + let ret = unsafe { + bindings::of_reserved_mem_region_to_resource_byname(np, name.as_char_ptr(), &mut res) + }; + to_result(ret).map_err(|e| { + pr_err!( + "apple_sep: reserved-memory region '{}' is unavailable ({:?})\n", + name, + e + ); + e + })?; + + let size = res.end.wrapping_sub(res.start).wrapping_add(1); + if size == 0 || size > usize::MAX as u64 { + return Err(EINVAL); + } + Ok((res.start, size as usize)) +} diff --git a/drivers/soc/apple/proto.rs b/drivers/soc/apple/proto.rs index 395f7440b10bc..de2bf4580d84a 100644 --- a/drivers/soc/apple/proto.rs +++ b/drivers/soc/apple/proto.rs @@ -21,6 +21,15 @@ pub(crate) const MSG_TYPE_SHIFT: u32 = 16; pub(crate) const MSG_PARAM_SHIFT: u32 = 24; pub(crate) const MSG_DATA_SHIFT: u32 = 32; +// Boot-endpoint (EP_BOOT) message types for the cold-boot handshake: send TZ0, +// then hand over the firmware image (IMG4) and the shared-memory table. +pub(crate) const MSG_BOOT_TZ0: u64 = 0x5; +pub(crate) const MSG_BOOT_IMG4: u64 = 0x6; +pub(crate) const MSG_SET_SHMEM: u64 = 0x18; +pub(crate) const MSG_BOOT_TZ0_ACK1: u64 = 0x69; +pub(crate) const MSG_BOOT_TZ0_ACK2: u64 = 0xd2; +pub(crate) const MSG_BOOT_IMG4_ACK: u64 = 0x6a; + // 4 KiB units even though CPU pages are 16 KiB pub(crate) const IOVA_SHIFT: u32 = 12; diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index fa3b86988ff2e..18be7d8debe4d 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -35,6 +35,7 @@ mod xarm; mod xart_store; use kernel::{ + bindings, device, dma, driver, @@ -510,12 +511,21 @@ struct MachineRefKey { pub_raw: KVec, } +/// One mapped firmware reserved region, unmapped exactly once in `remove()`. +struct FirmwareMap { + size: usize, + iova: u64, +} + #[pin_data] struct SepData { dev: ARef, profile: &'static profile::PlatformProfile, + #[pin] + fw_map: Mutex>, + #[pin] mbox: Mutex>>, @@ -747,6 +757,7 @@ impl SepData { try_pin_init!(SepData { dev: ARef::::from(dev), profile, + fw_map <- new_mutex!(None), mbox <- new_mutex!(None), shmem <- new_mutex!(Some(buf)), endpoints <- new_mutex!(EndpointTable::new()), @@ -807,7 +818,17 @@ impl SepData { ) } + /// Bring the shared-memory table to the SEP using the profile's bootstrap. fn attach(&self, sep_node: &dt::DtNode) -> Result<()> { + match self.profile.bootstrap { + profile::Bootstrap::WarmRegister => self.attach_warm(sep_node), + profile::Bootstrap::Boot => self.attach_m1(), + } + } + + /// Warm attach: the firmware left a running SEP, so one registration message + /// hands it the shared-memory table. + fn attach_warm(&self, sep_node: &dt::DtNode) -> Result<()> { let (iova, size) = { let guard = self.shmem.lock(); let buf = guard.as_ref().ok_or(EINVAL)?; @@ -837,6 +858,137 @@ impl SepData { Ok(()) } + /// Cold boot: send TZ0 and let the boot-endpoint acknowledgements drive the + /// firmware and shared-memory handoff. + fn attach_m1(&self) -> Result<()> { + let msg = Message { + msg0: u64::from(proto::EP_BOOT) | (proto::MSG_BOOT_TZ0 << proto::MSG_TYPE_SHIFT), + msg1: 0, + }; + self.send(msg) + } + + /// Handle a message from the boot endpoint during the cold-boot handshake. + fn on_boot(&self, msg: Message) { + let ty = (msg.msg0 >> proto::MSG_TYPE_SHIFT) & 0xff; + match ty { + proto::MSG_BOOT_TZ0_ACK1 => { + dev_info!(self.dev, "boot: first TZ0 acknowledgement\n"); + } + proto::MSG_BOOT_TZ0_ACK2 => { + dev_info!( + self.dev, + "boot: TZ0 accepted; handing over firmware and shared memory\n" + ); + if let Err(e) = self.load_firmware_and_shmem() { + dev_err!( + self.dev, + "boot: firmware/shared-memory handoff failed ({:?})\n", + e + ); + } + } + proto::MSG_BOOT_IMG4_ACK => { + dev_info!( + self.dev, + "boot: IMG4 acknowledged; the SEP owns the shared-memory table\n" + ); + self.registered.store(true, Relaxed); + } + _ => { + dev_warn!( + self.dev, + "boot: unknown message type {} (msg0 {:#018x})\n", + ty, + msg.msg0 + ); + } + } + } + + /// Map the firmware reserved region once and hand the SEP the firmware + /// address (IMG4) and the shared-memory table (SET_SHMEM). + fn load_firmware_and_shmem(&self) -> Result<()> { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } + + let (phys, size) = dt::reserved_region(&self.dev, self.profile.firmware_region)?; + + // Hold the mapping lock across the teardown re-check, the map and the + // store: `remove()` takes the same lock, so a mapping either exists + // before removal and is unmapped by it, or is refused here. + let mut fw_map = self.fw_map.lock(); + if fw_map.is_some() { + // One-shot per boot: a second acknowledgement cannot remap it. + return Ok(()); + } + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } + // SAFETY: `self.dev` is live; the reserved region is owned by the + // firmware handoff and the mapping is retained until `remove()`. + let iova = unsafe { + let mapped = bindings::dma_map_resource( + self.dev.as_raw(), + phys, + size, + bindings::dma_data_direction_DMA_TO_DEVICE, + 0, + ); + if bindings::dma_mapping_error(self.dev.as_raw(), mapped) != 0 { + return Err(ENOMEM); + } + mapped + }; + *fw_map = Some(FirmwareMap { size, iova }); + drop(fw_map); + + let msg = Message { + msg0: u64::from(proto::EP_BOOT) + | (proto::MSG_BOOT_IMG4 << proto::MSG_TYPE_SHIFT) + | ((iova >> proto::IOVA_SHIFT) << proto::MSG_DATA_SHIFT), + msg1: 0, + }; + self.send(msg)?; + + let shm = { + let guard = self.shmem.lock(); + let buf = guard.as_ref().ok_or(EINVAL)?; + buf.dma_handle() + }; + let msg = Message { + msg0: u64::from(proto::EP_SHMEM) + | (proto::MSG_SET_SHMEM << proto::MSG_TYPE_SHIFT) + | ((shm >> proto::IOVA_SHIFT) << proto::MSG_DATA_SHIFT), + msg1: 0, + }; + // The peer may know the address even if this send fails ambiguously, so + // the buffer stays retained: `remove()` must never free shared memory + // the SEP was handed. + self.registered.store(true, Relaxed); + self.send(msg)?; + Ok(()) + } + + /// Release the firmware mapping if one is live. Called by `remove()` so a + /// mapping created by a racing boot message cannot outlive the driver. + fn release_firmware_mapping(&self) { + if let Some(mapped) = self.fw_map.lock().take() { + // SAFETY: created by `dma_map_resource` with these exact parameters + // and unmapped exactly once. + unsafe { + bindings::dma_unmap_resource( + self.dev.as_raw(), + mapped.iova, + mapped.size, + bindings::dma_data_direction_DMA_TO_DEVICE, + 0, + ); + } + } + } + fn send(&self, msg: Message) -> Result<()> { self.mbox.lock().as_ref().ok_or(ENODEV)?.send(msg, false) } @@ -1667,14 +1819,7 @@ impl SepData { proto::EP_SCRD => self.on_scrd(msg), - proto::EP_BOOT => dev_warn!( - self.dev, - "unexpected message from boot endpoint 0xff: type 0x{:02x} param 0x{:02x} msg0 {:#018x} msg1 {:#010x}\n", - f.ty, - f.param, - msg.msg0, - msg.msg1 - ), + proto::EP_BOOT => self.on_boot(msg), _ep => {}, } @@ -1710,6 +1855,7 @@ impl SepData { fn remove(&self) { self.shutting_down.store(true, Relaxed); + self.release_firmware_mapping(); trusted::unregister(); self.unregister_fv_kernel(); @@ -1987,7 +2133,16 @@ struct SepModule { impl kernel::InPlaceModule for SepModule { fn init(module: &'static ThisModule) -> impl PinInit { try_pin_init!(Self { - _dt: dt::enable_sep_and_dart()?, + // The warm-attach target enables the SEP and its DART with a runtime + // changeset; the cold-boot target describes them statically (its DART + // is enabled before probe), so no changeset runs there. + _dt: { + if let Ok(p) = profile::detect() { + if matches!(p.bootstrap, profile::Bootstrap::WarmRegister) { + dt::enable_sep_and_dart()?; + } + } + }, _driver <- driver::Registration::new( ::NAME, From d81f83243f41d7a0240f2f81d59709c12076f326 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 15:15:44 -0400 Subject: [PATCH 0702/2680] soc: apple: sep: take the calibration blob's name from the device tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Mesa calibration blob is per-device factory data, so its name is board data, not a build constant. Read it from the sensor node's firmware-name property, falling back to the built-in default when the property is absent so the runtime-fabricated node keeps working. [ryan: adapted to 7.2.10 — declares sep_sensor_firmware_name() in shim.h, which aurora-wip only did in commit 1c35a898cec9 ("soc: apple: sep: defer Neo sensor until SMC power is ready"); without it -Wmissing-prototypes fails the J700 (CONFIG_WERROR) build of every commit in between] Signed-off-by: Chromatischer [dj: kept the default fallback so the fabricated j414s node still loads; clean-room] Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 7 +++++-- drivers/soc/apple/sensor.rs | 14 ++++++++++++++ drivers/soc/apple/sensor_shim.c | 16 ++++++++++++++++ drivers/soc/apple/shim.h | 1 + 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index ed23c237c531d..16a8312a56137 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1217,13 +1217,16 @@ impl SepData { } fn calibrate_sensor(&self) -> bool { - let blob = match kernel::firmware::Firmware::request(CALIBRATION_FIRMWARE, &self.dev) { + // The blob is per-device factory data, so its name is board data: take + // it from the sensor node's firmware-name, falling back to the default. + let name = sensor::firmware_name().unwrap_or(CALIBRATION_FIRMWARE); + let blob = match kernel::firmware::Firmware::request(name, &self.dev) { Ok(fw) => CalibrationBlob::new(fw), Err(e) => { dev_err!( self.dev, "sensor: calibration blob {} could not be loaded ({:?}); per-device factory data, cannot be synthesised, stopping\n", - CALIBRATION_FIRMWARE, + name, e ); return false; diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index d94cc00a1cff2..66ae99800a40d 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -19,6 +19,20 @@ extern "C" { fn sep_sensor_xfer(tx: *const c_void, rx: *mut c_void, len: usize) -> c_int; fn sep_sensor_xfer_tx(tx: *const c_void, len: usize) -> c_int; fn sep_sensor_xfer2(tx: *const c_void, tx_len: usize, rx: *mut c_void, rx_len: usize) -> c_int; + fn sep_sensor_firmware_name() -> *const c_char; +} + +/// The calibration blob's name from the board description, or `None` when the +/// sensor node has no `firmware-name` property (the caller then uses a default). +pub(crate) fn firmware_name() -> Option<&'static CStr> { + // SAFETY: the shim returns NULL or a pointer to a NUL-terminated string + // owned by the device property and valid for the device's lifetime. + let ptr = unsafe { sep_sensor_firmware_name() }; + if ptr.is_null() { + return None; + } + // SAFETY: a non-NULL return is a valid NUL-terminated C string. + Some(unsafe { CStr::from_char_ptr(ptr) }) } // Spi2. diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index 924433b094a64..22623213f48c3 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -322,6 +322,22 @@ int sep_sensor_power_line(void) return desc_to_gpio(sep_power); } +/* + * The per-device calibration blob name, as the board description gives it in + * the sensor node's "firmware-name" property. NULL when absent, so the caller + * can fall back to a default. + */ +const char *sep_sensor_firmware_name(void) +{ + const char *name = NULL; + + if (!sep_spi) + return NULL; + if (device_property_read_string(&sep_spi->dev, "firmware-name", &name)) + return NULL; + return name; +} + /* Powers the sensor on/off, holding the hardware settling delay. */ int sep_sensor_power(int on) { diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index 34c34f29bc68b..cc3f4ccb7a19d 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -124,6 +124,7 @@ int sep_sensor_cs_timing_mode(void); int sep_sensor_power_cycle(void); int sep_sensor_power_source(void); int sep_sensor_power(int on); +const char *sep_sensor_firmware_name(void); int sep_sensor_xfer(const void *tx, void *rx, size_t len); int sep_sensor_xfer_tx(const void *tx, size_t len); int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len); From 892380c3c40b7ad36f072852478a8e9cde56d80d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 16 Sep 2026 18:36:41 -0400 Subject: [PATCH 0703/2680] soc: apple: sep: report key-store endpoint readiness Log once when the key-store endpoint (0x12) comes up and its out-of-line buffers are registered, and enrich the "not advertised" diagnostic to show how far the endpoint ladder reached: the live endpoint count and which of control/xarm/sbio/scrd are present. An incomplete xART bring-up is then diagnosable from dmesg alone, without instrumenting the driver. Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Assisted-by: Claude Opus 4.8 --- drivers/soc/apple/sks.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index 6469ebe24f5bc..c6cf39bee5e25 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -836,8 +836,13 @@ impl SepData { if !self.endpoint_present(proto::EP_SKS) { dev_err!( self.dev, - "sks: endpoint 0x{:02x} (key store) was not advertised on this boot; keybag and ref-key operations cannot run\n", - proto::EP_SKS + "sks: endpoint 0x{:02x} (key store) not advertised; {} EPs up (control={} xarm={} sbio={} scrd={}); keybag/ref-key cannot run\n", + proto::EP_SKS, + self.endpoint_count(), + self.endpoint_present(proto::EP_CONTROL) as u8, + self.endpoint_present(proto::EP_XARM) as u8, + self.endpoint_present(proto::EP_SBIO) as u8, + self.endpoint_present(proto::EP_SCRD) as u8, ); return false; } @@ -849,6 +854,12 @@ impl SepData { ); return false; } + dev_info!( + self.dev, + "sks: endpoint 0x{:02x} (key store) up; {} EPs; out-of-line buffers registered\n", + proto::EP_SKS, + self.endpoint_count() + ); true } From 263ef49fe49755d2be31b2db4dcfb3c741e31202 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 17 Sep 2026 00:39:46 -0400 Subject: [PATCH 0704/2680] soc: apple: sep: encode the identity keybag per platform profile The identity keybag CREATE_KEYBAG request carries the bag type in a word the strict T8103 enclave reads as the type and the lenient T6020 enclave does not. T6020 takes the variant in the first word (the proven encoding, hardware-verified for enrol, match and reboot); T8103 rejects that and wants the type in the third word, as macOS sends it. Carry the three field values in the platform profile so each SoC sends what its enclave requires, leaving the T6020 wire encoding byte-for-byte unchanged. Drop the now-unused variant constant, create-flags helper and first-identity handle. Verified on j414s (T6020): the identity keybag still provisions. Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Assisted-by: Claude Opus 4.8 --- drivers/soc/apple/profile.rs | 29 +++++++++++++++++++++++++++++ drivers/soc/apple/sks.rs | 30 +++++++++--------------------- 2 files changed, 38 insertions(+), 21 deletions(-) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 7d7bee3f0208a..61faf6b88736f 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -86,6 +86,21 @@ pub(crate) struct SensorProfile { pub(crate) capture_qualified: bool, } +/// Identity keybag `CREATE_KEYBAG` field encoding. The strict T8103 enclave +/// reads the request's first word as the bag type and rejects the value the +/// lenient T6020 enclave accepts there; macOS carries the type in the third +/// word instead. Kept per-SoC so the strict path is correct without disturbing +/// the proven T6020 encoding (hardware-verified for enrol, match, and reboot). +pub(crate) struct KeybagCreate { + /// First word: request variant, echoed back in the reply. + pub(crate) variant: u32, + /// Third word: bag type. Identity is `0x20000` on the strict path; `0` on + /// T6020, which distinguishes the bag by the variant word instead. + pub(crate) bag_type: u32, + /// Fourth word: create argument / parent handle. + pub(crate) arg: i32, +} + pub(crate) struct PlatformProfile { pub(crate) name: &'static str, /// Capacity of the boot shared-memory window. The layout is still checked @@ -102,6 +117,8 @@ pub(crate) struct PlatformProfile { pub(crate) dart_range_required: bool, /// Reserved-memory region holding the SEP firmware image (cold-boot path). pub(crate) firmware_region: &'static CStr, + /// Identity keybag CREATE_KEYBAG field encoding (per-SoC; see [`KeybagCreate`]). + pub(crate) keybag_create: KeybagCreate, } const T8103: PlatformProfile = PlatformProfile { @@ -122,6 +139,12 @@ const T8103: PlatformProfile = PlatformProfile { }, dart_range_required: false, firmware_region: c"sepfw", + // Strict enclave: the type goes in the third word; the first word is 0. + keybag_create: KeybagCreate { + variant: 0, + bag_type: 0x20000, + arg: 0, + }, }; const T6020: PlatformProfile = PlatformProfile { @@ -142,6 +165,12 @@ const T6020: PlatformProfile = PlatformProfile { }, dart_range_required: true, firmware_region: c"sepfw", + // Proven encoding: the lenient enclave takes the variant in the first word. + keybag_create: KeybagCreate { + variant: 5, + bag_type: 0, + arg: -1, + }, }; static_assert!(T8103.shmem_capacity == 0x30000); diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index c6cf39bee5e25..03caee8f0767b 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -428,11 +428,16 @@ impl SepData { if proof.slot() != keybag::Slot::Identity { return Err(EINVAL); } + // The strict (T8103) and lenient (T6020) enclaves encode the identity + // bag differently; the per-SoC profile carries the correct field values + // so the proven T6020 path is unchanged while T8103 gets the bag type in + // the third word (see profile::KeybagCreate). + let enc = &self.profile.keybag_create; let mut body = image::Body::new(); - body.put_u32(crate::sks::SKS_CREATE_VARIANT_IDENTITY)?; + body.put_u32(enc.variant)?; body.put_u64(crate::sks::SKS_CLIENT_ID)?; - body.put_u32(crate::sks::CreateFlags::none().value())?; - body.put_i32(crate::sks::SpecialHandle::first_identity().value())?; + body.put_u32(enc.bag_type)?; + body.put_i32(enc.arg)?; body.put_blob(secret)?; body.put_blob(&[])?; body.put_blob(uuid)?; @@ -514,7 +519,7 @@ impl SepData { let Some((_fv_data, end)) = image::read_blob(body, 8) else { return false; }; - if variant != crate::sks::SKS_CREATE_VARIANT_IDENTITY || raw_handle < 0 || end != body.len() + if variant != self.profile.keybag_create.variant || raw_handle < 0 || end != body.len() { dev_err!( self.dev, @@ -987,21 +992,8 @@ pub(crate) fn sks_copy_keybag() -> SksOp { const OP_SKS_CREATE_KEYBAG: u8 = 0x01; pub(crate) const SKS_CREATE_NAME: &CStr = c"CREATE_KEYBAG"; -pub(crate) const SKS_CREATE_VARIANT_IDENTITY: u32 = 5; pub(crate) const SKS_IDENTITY_UUID_LEN: usize = 16; -pub(crate) struct CreateFlags(u32); - -impl CreateFlags { - pub(crate) const fn none() -> Self { - Self(0) - } - - pub(crate) const fn value(&self) -> u32 { - self.0 - } -} - const OP_SKS_LOAD_KEYBAG: u8 = 0x03; pub(crate) const SKS_LOAD_NAME: &CStr = c"LOAD_KEYBAG"; @@ -1068,10 +1060,6 @@ impl DesignateUser { pub(crate) struct SpecialHandle(i32); impl SpecialHandle { - pub(crate) const fn first_identity() -> SpecialHandle { - SpecialHandle(-1) - } - pub(crate) const fn value(&self) -> i32 { self.0 } From 0ee02b1d26e12797dffb336650b50b32f80e2b2b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 17 Sep 2026 00:39:47 -0400 Subject: [PATCH 0705/2680] soc: apple: sep: open the xART gigalocker by raw extent in the kernel Add an in-kernel path that opens the iBoot system container directly and serves the STORE_SIZE gigalocker window at a configured sector, so the driver reaches the shared anti-replay store on its own. It is additive and fail-closed: the xart_start_sector module parameter selects the raw extent, the logical store is always exactly STORE_SIZE so a larger backing device serves only its extent, and a wrong window still fails the existing root-record check rather than feeding SEP an unrelated store. Verified on j414s: with xart_start_sector at the gigalocker's first sector the store opens with the expected slot count, record count and revision, and the key store comes up. Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Assisted-by: Claude Opus 4.8 --- drivers/soc/apple/sep.rs | 7 +++- drivers/soc/apple/xart_store.rs | 70 +++++++++++++++++++++++++-------- 2 files changed, 59 insertions(+), 18 deletions(-) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 18be7d8debe4d..4cfcae712d0b4 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -711,7 +711,8 @@ impl SepData { let dma_ring = dma::Coherent::::zeroed_slice(dev, DMA_RING_SIZE, GFP_KERNEL)?; let xart_writes = *module_parameters::xart_writes.value() != 0; - let store = match xart_store::Store::open(xart_writes) { + let xart_start = *module_parameters::xart_start_sector.value(); + let store = match xart_store::Store::open_owner(xart_writes, xart_start) { Ok(store) => { let (slots, records, revision, malformed, duplicates, repaired, writable) = store.summary(); @@ -2162,6 +2163,10 @@ module! { default: 0, description: "Allow writes to the validated shared xART mapping", }, + xart_start_sector: u64 { + default: 0, + description: "First 512-byte sector of the xART gigalocker extent within the iBoot system container. Non-zero selects the in-kernel raw-extent owner (no userspace mapper); zero uses the device-mapper mapping at /dev/mapper/sep-xart-gigalocker", + }, provision_keybag: u8 { default: 0, description: "Create the Linux identity keybag when none exists; requires xart_writes=1", diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs index e255e2507f745..ea70d6dcc508d 100644 --- a/drivers/soc/apple/xart_store.rs +++ b/drivers/soc/apple/xart_store.rs @@ -13,17 +13,24 @@ use kernel::prelude::*; pub(crate) const STORE_PATH: &CStr = c"/dev/mapper/sep-xart-gigalocker"; +/// The raw-extent owner opens this whole partition (the iBoot system container, +/// where the gigalocker lives) and serves the window at the configured sector. +/// It replaces the userspace APFS parser + device-mapper mapping. +pub(crate) const OWNER_PATH: &CStr = c"/dev/disk/by-partlabel/iBootSystemContainer"; + const BLOCK_SIZE: usize = 0x1000; const SLOT_SIZE: usize = 0x9000; const HEADER_SIZE: usize = 0x22; const DELETE_SIZE: usize = BLOCK_SIZE; /// Size of the APFS raw extent located on the target machine. /// -/// Accepting a larger block device would make a bad device-mapper table a -/// corruption hazard. Accepting a smaller one could silently truncate the -/// slot grid. A future locator for a machine with a different extent size must -/// pass that size through an explicit, reviewed interface instead of weakening -/// this check. +/// The logical store is always exactly this many bytes. The device-mapper +/// mapping is exactly this size; the raw-extent owner opens a larger container +/// and serves only the [`STORE_SIZE`] window at its configured base, so a wrong +/// base cannot read past the extent. A wrong window in either mode fails closed: +/// the two root records will not validate and `open_based` returns `ENODATA` +/// rather than serving SEP an unrelated store. A machine with a different extent +/// size must pass that size through an explicit, reviewed interface. const STORE_SIZE: u64 = 0x600000; const MAX_SLOTS: usize = 4096; @@ -80,6 +87,10 @@ impl Slot { pub(crate) struct Store { file: shim::StoreFile, + /// Byte offset of the first slot within the opened block device. Zero for + /// the device-mapper mapping (which starts at the gigalocker); non-zero for + /// the raw-extent owner, which opens the whole container and points here. + base: u64, slots: KVec, revision: u64, writes_enabled: bool, @@ -112,21 +123,45 @@ fn valid_key(key: &Key) -> bool { impl Store { pub(crate) fn open(writes_enabled: bool) -> Result { - Self::open_at(STORE_PATH, writes_enabled) + Self::open_based(STORE_PATH, writes_enabled, 0) + } + + /// Opens the store, selecting the device-mapper mapping (`start_sector == 0`) + /// or the in-kernel raw-extent owner. The owner opens the whole iBoot system + /// container and treats the [`STORE_SIZE`] window at `start_sector` as the + /// gigalocker, reproducing the mapper's bytes without a userspace parser. + pub(crate) fn open_owner(writes_enabled: bool, start_sector: u64) -> Result { + if start_sector == 0 { + return Self::open(writes_enabled); + } + Self::open_based(OWNER_PATH, writes_enabled, start_sector << 9) } - /// Opens a caller-selected block mapping. + /// Opens a caller-selected block mapping at offset zero. /// - /// Production always uses [`STORE_PATH`]. The separate xART self-test - /// module uses this entry point with its fixed loop-only mapper name, so it - /// can execute the real parser and write ordering without enabling SEP. + /// The separate xART self-test module uses this entry point with its fixed + /// loop-only mapper name, so it can execute the real parser and write + /// ordering without enabling SEP. + #[allow(dead_code)] pub(crate) fn open_at(path: &CStr, writes_enabled: bool) -> Result { + Self::open_based(path, writes_enabled, 0) + } + + /// Opens `path` and serves the [`STORE_SIZE`] window starting at byte + /// `base`. The window must be block-aligned and fit within the device; the + /// logical store is always exactly [`STORE_SIZE`], so a larger backing + /// device (the raw container) serves only its gigalocker extent. + fn open_based(path: &CStr, writes_enabled: bool, base: u64) -> Result { let file = shim::StoreFile::open_block(path, writes_enabled)?; let size = file.size()?; - if size != STORE_SIZE || size % BLOCK_SIZE as u64 != 0 { + if base % BLOCK_SIZE as u64 != 0 || STORE_SIZE % BLOCK_SIZE as u64 != 0 { + return Err(EINVAL); + } + let end = base.checked_add(STORE_SIZE).ok_or(EINVAL)?; + if end > size { return Err(EINVAL); } - let count = (size / SLOT_SIZE as u64) as usize; + let count = (STORE_SIZE / SLOT_SIZE as u64) as usize; if count == 0 || count > MAX_SLOTS { return Err(EINVAL); } @@ -137,6 +172,7 @@ impl Store { } let mut store = Store { file, + base, slots, revision: 0, // Discovery is always read-only. Do not arm even repair writes @@ -172,7 +208,7 @@ impl Store { for idx in 0..self.slots.len() { self.file - .read_block_exact(Self::slot_offset(idx), &mut raw)?; + .read_block_exact(self.base + Self::slot_offset(idx), &mut raw)?; let kind = raw[KEY_KIND]; if kind == 0 { continue; @@ -230,7 +266,7 @@ impl Store { continue; } self.file - .read_block_exact(Self::slot_offset(idx), &mut header)?; + .read_block_exact(self.base + Self::slot_offset(idx), &mut header)?; if header[KEY_KIND] != 0 { self.delete_slot(idx)?; self.repaired_records += 1; @@ -265,7 +301,7 @@ impl Store { let mut zero = KVec::with_capacity(DELETE_SIZE, GFP_KERNEL)?; zero.resize(DELETE_SIZE, 0, GFP_KERNEL)?; self.file - .write_block_exact(Self::slot_offset(slot), &zero)?; + .write_block_exact(self.base + Self::slot_offset(slot), &zero)?; self.file.sync() } @@ -280,7 +316,7 @@ impl Store { let mut raw = KVec::with_capacity(SLOT_SIZE, GFP_KERNEL)?; raw.resize(SLOT_SIZE, 0, GFP_KERNEL)?; self.file - .read_block_exact(Self::slot_offset(idx), &mut raw)?; + .read_block_exact(self.base + Self::slot_offset(idx), &mut raw)?; let mut uuid = [0u8; 16]; uuid.copy_from_slice(&raw[KEY_UUID..KEY_UUID + 16]); @@ -329,7 +365,7 @@ impl Store { // The new record becomes authoritative only after its complete slot is // durable. The old record is then removed and flushed separately. self.file - .write_block_exact(Self::slot_offset(fresh), &raw)?; + .write_block_exact(self.base + Self::slot_offset(fresh), &raw)?; self.file.sync()?; self.slots[fresh] = Slot { used: true, From 5b149339fd872651bd25d6ed7806c73ff019778c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 17 Sep 2026 17:58:49 -0400 Subject: [PATCH 0706/2680] soc: apple: sep: locate the xART gigalocker automatically Drop the last manual step from the in-kernel raw-extent owner: instead of a hand-supplied start sector, the driver finds the gigalocker itself. It reads the iBoot system container sequentially and searches, at block-aligned offsets, for a root record's signature (a 1 or 2 key kind with an all-zero UUID, the shape of Key::root). A root signature alone does not pin the base. A window shifted a few slots off the true origin still keeps both root records inside its 6 MiB extent, so it passes the two-root test yet silently drops the live records that fall outside the shift -- and, with writes on, would repair (write) at that wrong origin. The driver therefore probes every candidate base a hit implies read-only: through a writable handle, because the block layer only grants a read-only handle to a read-only device and the container is writable, but with repair disarmed so nothing is written at an unconfirmed origin. It then pins the base that recovers the most live records. Only the true origin captures the whole record set, and container noise never forges a CRC-valid record, so max live records -- ties broken to fewer malformed, then the higher base -- is exact. Only the pinned base is re-opened writable, so repair runs at the confirmed origin alone. open_based gains open_based_ext, which decouples the block-handle writability from whether repair is armed, so discovery of the writable container never mutates it. With automatic location in place, xart_start_sector becomes an explicit override (default 0 = automatic) and the driver has no external dependency for reaching the store. Boot-verified on j414s (T6020): the scan pins the true base 0x4d4000, reads all 10 live records with 0 malformed and 0 repaired (no errant write), and the key store (EP 0x12) reaches 12 endpoints. Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Assisted-by: Claude Opus 4.8 --- drivers/soc/apple/Kconfig | 8 +- drivers/soc/apple/sep.rs | 11 +- drivers/soc/apple/xart_store.rs | 193 ++++++++++++++++++++++++++++---- 3 files changed, 179 insertions(+), 33 deletions(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 412f3ce64ee4d..19131befc5d92 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -131,10 +131,10 @@ config APPLE_SEP device (enrol and match), exposes the SEP hardware RNG, and registers a SEP-backed trusted key source so keyctl can seal keys to the enclave. - The driver requires the existing machine-wide xART gigalocker at - /dev/mapper/sep-xart-gigalocker. It validates the complete store and both - root records before registering with SEP. Writes remain disabled unless - the xart_writes module parameter is set and the mapping is writable. + The driver reads the existing machine-wide xART gigalocker directly from + the iBoot system container, locating it by its root records. It validates + the complete store and both root records before registering with SEP. + Writes remain disabled unless the xart_writes module parameter is set. Say Y here if you have an Apple silicon Mac. diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 4cfcae712d0b4..f284876840a52 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -716,9 +716,11 @@ impl SepData { Ok(store) => { let (slots, records, revision, malformed, duplicates, repaired, writable) = store.summary(); + let base = store.base(); dev_info!( dev, - "xART: {} slots, {} live records, max revision {}, {} malformed, {} duplicate, {} repaired; writes {}\n", + "xART: in-kernel raw-extent owner (base {:#x}); {} slots, {} live records, max revision {}, {} malformed, {} duplicate, {} repaired; writes {}\n", + base, slots, records, revision, @@ -732,8 +734,7 @@ impl SepData { Err(e) => { dev_err!( dev, - "shared xART mapping '{}' is unavailable or invalid: {:?}\n", - xart_store::STORE_PATH, + "xART gigalocker in the iBoot container is unavailable or invalid: {:?}\n", e ); return Err(e); @@ -2161,11 +2162,11 @@ module! { params: { xart_writes: u8 { default: 0, - description: "Allow writes to the validated shared xART mapping", + description: "Allow writes to the validated shared xART store", }, xart_start_sector: u64 { default: 0, - description: "First 512-byte sector of the xART gigalocker extent within the iBoot system container. Non-zero selects the in-kernel raw-extent owner (no userspace mapper); zero uses the device-mapper mapping at /dev/mapper/sep-xart-gigalocker", + description: "Explicit 512-byte start sector of the xART gigalocker extent within the iBoot system container. Zero (default) locates the gigalocker automatically in the container by its root records; a non-zero value overrides the search. Either way the driver opens the container directly", }, provision_keybag: u8 { default: 0, diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs index ea70d6dcc508d..ac9e0bf6c51e2 100644 --- a/drivers/soc/apple/xart_store.rs +++ b/drivers/soc/apple/xart_store.rs @@ -11,11 +11,8 @@ use crate::shim; use kernel::prelude::*; -pub(crate) const STORE_PATH: &CStr = c"/dev/mapper/sep-xart-gigalocker"; - /// The raw-extent owner opens this whole partition (the iBoot system container, -/// where the gigalocker lives) and serves the window at the configured sector. -/// It replaces the userspace APFS parser + device-mapper mapping. +/// where the gigalocker lives) and serves the gigalocker window directly. pub(crate) const OWNER_PATH: &CStr = c"/dev/disk/by-partlabel/iBootSystemContainer"; const BLOCK_SIZE: usize = 0x1000; @@ -24,16 +21,26 @@ const HEADER_SIZE: usize = 0x22; const DELETE_SIZE: usize = BLOCK_SIZE; /// Size of the APFS raw extent located on the target machine. /// -/// The logical store is always exactly this many bytes. The device-mapper -/// mapping is exactly this size; the raw-extent owner opens a larger container -/// and serves only the [`STORE_SIZE`] window at its configured base, so a wrong -/// base cannot read past the extent. A wrong window in either mode fails closed: +/// The logical store is always exactly this many bytes. The raw-extent owner +/// opens a larger container and serves only the [`STORE_SIZE`] window at its +/// located base, so a wrong base cannot read past the extent. A wrong window +/// still fails closed: /// the two root records will not validate and `open_based` returns `ENODATA` /// rather than serving SEP an unrelated store. A machine with a different extent /// size must pass that size through an explicit, reviewed interface. const STORE_SIZE: u64 = 0x600000; const MAX_SLOTS: usize = 4096; +/// Sequential read window for the automatic gigalocker search (64 KiB — small +/// enough for a reliable kernel allocation, large enough to keep the scan of +/// the container to a modest number of reads). +const SCAN_CHUNK: usize = 1 << 16; +/// Upper bound on candidate bases confirmed during the automatic search, so a +/// container full of coincidental root-shaped headers cannot spin the scan. The +/// real store's roots sit within the first slots, so it is found long before +/// this; exceeding it falls back rather than looping. +const MAX_LOCATE_ATTEMPTS: u32 = 64; + pub(crate) const MAX_VALUE: usize = 0x8000; const KEY_KIND: usize = 0x01; @@ -87,9 +94,8 @@ impl Slot { pub(crate) struct Store { file: shim::StoreFile, - /// Byte offset of the first slot within the opened block device. Zero for - /// the device-mapper mapping (which starts at the gigalocker); non-zero for - /// the raw-extent owner, which opens the whole container and points here. + /// Byte offset of the gigalocker window within the opened container: the + /// located extent's offset (an explicit start-sector override sets it). base: u64, slots: KVec, revision: u64, @@ -122,19 +128,131 @@ fn valid_key(key: &Key) -> bool { } impl Store { - pub(crate) fn open(writes_enabled: bool) -> Result { - Self::open_based(STORE_PATH, writes_enabled, 0) + /// Opens the store as the in-kernel raw-extent owner: it opens the iBoot + /// system container directly and serves only the gigalocker extent, with no + /// external helper and no hand-supplied sector. + /// + /// `start_sector == 0` (the default) *locates the gigalocker automatically* + /// inside the container by its CRC-checked root records; a non-zero + /// `start_sector` is an explicit override for the rare case the scan should + /// be skipped. + pub(crate) fn open_owner(writes_enabled: bool, start_sector: u64) -> Result { + if start_sector != 0 { + return Self::open_based(OWNER_PATH, writes_enabled, start_sector << 9); + } + Self::open_located(writes_enabled) } - /// Opens the store, selecting the device-mapper mapping (`start_sector == 0`) - /// or the in-kernel raw-extent owner. The owner opens the whole iBoot system - /// container and treats the [`STORE_SIZE`] window at `start_sector` as the - /// gigalocker, reproducing the mapper's bytes without a userspace parser. - pub(crate) fn open_owner(writes_enabled: bool, start_sector: u64) -> Result { - if start_sector == 0 { - return Self::open(writes_enabled); + /// Locates the gigalocker inside the iBoot system container with no external + /// help. The container is read sequentially and searched, at block-aligned + /// offsets, for a root record's signature — a `1` or `2` key kind with an + /// all-zero UUID, the shape of [`Key::root`]. Each base implied by such a hit + /// is probed read-only with the full CRC-checked [`open_based`]; because a + /// base shifted a few slots off the true origin still keeps both roots in its + /// window (passing the two-root test while dropping records that fall + /// outside), the base is pinned by the candidate that recovers the MOST live + /// records — the exact discriminator, since only the true origin captures the + /// whole set and container noise never forges a CRC-valid record. Repair is + /// never armed during discovery; only the pinned base is re-opened writable. + /// Bounded by [`MAX_LOCATE_ATTEMPTS`] probes; returns `ENODATA` if not found. + fn open_located(writes_enabled: bool) -> Result { + // The container is a writable block device; the block layer only grants + // a read-only handle to a read-only device, so scan it through a writable + // handle. This does not arm any write — discovery never mutates the + // container (see open_based_ext, called below with arm_writes == false). + let probe = shim::StoreFile::open_block(OWNER_PATH, true)?; + let size = probe.size()?; + if size < STORE_SIZE { + return Err(ENODATA); + } + + let mut buf: KVec = KVec::new(); + buf.resize(SCAN_CHUNK, 0, GFP_KERNEL)?; + let mut attempts: u32 = 0; + let mut off: u64 = 0; + + while off + HEADER_SIZE as u64 <= size { + let want = core::cmp::min(SCAN_CHUNK as u64, size - off) as usize; + probe.read_exact(off, &mut buf[..want])?; + + let mut p = 0usize; + while p + HEADER_SIZE <= want { + let kind = buf[p + KEY_KIND]; + let uuid_zero = buf[p + KEY_UUID..p + KEY_UUID + 16].iter().all(|&b| b == 0); + if (kind == 1 || kind == 2) && uuid_zero { + // A root signature anchors the gigalocker: the true base is + // this hit minus a whole number of slots (slots and blocks are + // both 0x1000-aligned, so every candidate stays block-aligned). + // The two-root test alone does NOT pin the base — a base + // shifted off the true origin by a few slots still keeps both + // roots inside its 6 MiB window, so it passes yet silently + // drops the live records that fall outside the shifted window. + // Probe every candidate READ-ONLY (writes_enabled forced false, + // so repair never fires at an unconfirmed base) and pin the one + // that recovers the MOST live records: only the true origin + // captures the whole record set, and container noise never + // forges a CRC-valid record, so max live records is exact. + let hit = off + p as u64; + let mut best: Option<(usize, usize, u64)> = None; // (valid, malformed, base) + let mut k: u64 = 0; + while k as usize <= MAX_SLOTS { + let step = k * SLOT_SIZE as u64; + if step > hit { + break; + } + let base = hit - step; + k += 1; + if base + STORE_SIZE > size { + continue; + } + attempts += 1; + if attempts > MAX_LOCATE_ATTEMPTS { + break; + } + if let Ok(store) = Self::open_based_ext(OWNER_PATH, true, false, base) { + let cand = (store.valid_records, store.malformed_records, base); + let better = match best { + None => true, + // More live records wins; ties break to fewer + // malformed, then to the HIGHER base. An up-shift + // drops the lowest occupied slots (and any root + // there, which fails the two-root test), so it + // never ties on live count; the only bases that + // can tie are down-shifts, which are all lower + // than the true origin — so the highest surviving + // candidate is the true origin. + Some(b) => { + cand.0 > b.0 + || (cand.0 == b.0 && cand.1 < b.1) + || (cand.0 == b.0 && cand.1 == b.1 && cand.2 > b.2) + } + }; + if better { + best = Some(cand); + } + } + } + // The gigalocker is unique, so the first hit that confirms any + // base has pinned it. Re-open the winner through a writable + // handle and arm repair per `writes_enabled` — repair runs + // now, and only at this confirmed origin. + if let Some((_, _, base)) = best { + return Self::open_based_ext(OWNER_PATH, true, writes_enabled, base); + } + if attempts > MAX_LOCATE_ATTEMPTS { + return Err(ENODATA); + } + } + p += BLOCK_SIZE; + } + + if want < SCAN_CHUNK { + break; + } + // Overlap one block so a signature on the boundary is not missed. + off += (SCAN_CHUNK - BLOCK_SIZE) as u64; } - Self::open_based(OWNER_PATH, writes_enabled, start_sector << 9) + Err(ENODATA) } /// Opens a caller-selected block mapping at offset zero. @@ -151,8 +269,29 @@ impl Store { /// `base`. The window must be block-aligned and fit within the device; the /// logical store is always exactly [`STORE_SIZE`], so a larger backing /// device (the raw container) serves only its gigalocker extent. + /// + /// The block handle's writability equals `writes_enabled`, and repair runs + /// when writes are enabled. For the raw-extent owner, discovery instead + /// needs a writable handle (the block layer only grants a read-only handle + /// to a read-only device, and the container is writable) *without* arming + /// repair at an unconfirmed base — see [`open_based_ext`]. fn open_based(path: &CStr, writes_enabled: bool, base: u64) -> Result { - let file = shim::StoreFile::open_block(path, writes_enabled)?; + Self::open_based_ext(path, writes_enabled, writes_enabled, base) + } + + /// As [`open_based`], but with the block-handle writability (`dev_writable`) + /// decoupled from whether repair writes are armed (`arm_writes`). Discovery + /// of the raw-extent owner opens the writable container with + /// `dev_writable == true` yet `arm_writes == false`, so a candidate base is + /// fully scanned and CRC-validated without a single write landing at an + /// origin that has not yet been confirmed as the true gigalocker. + fn open_based_ext( + path: &CStr, + dev_writable: bool, + arm_writes: bool, + base: u64, + ) -> Result { + let file = shim::StoreFile::open_block(path, dev_writable)?; let size = file.size()?; if base % BLOCK_SIZE as u64 != 0 || STORE_SIZE % BLOCK_SIZE as u64 != 0 { return Err(EINVAL); @@ -191,8 +330,8 @@ impl Store { if store.find(&Key::root(1)).is_none() || store.find(&Key::root(2)).is_none() { return Err(ENODATA); } - store.writes_enabled = writes_enabled; - if writes_enabled { + store.writes_enabled = arm_writes; + if arm_writes { store.repair_disk()?; } Ok(store) @@ -400,6 +539,12 @@ impl Store { Ok(true) } + /// Byte offset of the served gigalocker window within the opened container: + /// the located extent's offset (an explicit start-sector override sets it). + pub(crate) fn base(&self) -> u64 { + self.base + } + pub(crate) fn summary(&self) -> (usize, usize, u64, usize, usize, usize, bool) { ( self.slots.len(), From 463d9f8a0364c733af85e4dc1c00c02ac8719ba8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 17 Sep 2026 22:59:40 -0400 Subject: [PATCH 0707/2680] soc: apple: sep: wait for the key-store endpoint before bring-up The SEP endpoint ladder climbs 7 -> 11 (sbio, scrd) -> 12 (the key store, EP 0x12). tick_exchange declared the persistent-state exchange done as soon as the endpoint count passed a hardcoded pre-exchange baseline (7), but the SEP advertises the key store a beat after it falls briefly quiet following scrd. On some boots the driver therefore reached run_bringup at 11 endpoints, found EP 0x12 absent, and stranded the boot with no key store -- enable_sks fails with no retry, so keybag and ref-key operations (and thus Touch ID) are unavailable for the whole boot. Wait through quiescence until the key store itself is advertised, bounded by the same EXCHANGE_TIMEOUT_MS. A boot that already has it exits immediately, so healthy boots are unaffected; the hardcoded endpoint count, which is not portable across SoCs, is dropped in favour of the endpoint that actually matters. Boot-verified on j414s (T6020): the key store (EP 0x12) came up at 12 endpoints on 5 of 5 reboots, up from roughly 2 in 3 before the change. Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Assisted-by: Claude Opus 4.8 --- drivers/soc/apple/sep.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index f284876840a52..951f42a40163b 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -91,8 +91,6 @@ const PHASE_ATTACH: u32 = 0; const PHASE_EXCHANGE: u32 = 1; const PHASE_READY: u32 = 2; -const ENDPOINTS_BEFORE_EXCHANGE: usize = 7; - const OOL_SIZE_XARM: usize = 0x8000; const OOL_SIZE_SBIO: usize = 0x4000; @@ -1781,8 +1779,16 @@ impl SepData { return; } - let endpoints = this.endpoint_count(); - if endpoints <= ENDPOINTS_BEFORE_EXCHANGE { + // The endpoint ladder climbs 7 -> 11 (sbio, scrd) -> 12 (the key + // store, EP 0x12). The SEP can advertise the key store a beat after it + // falls briefly quiet following scrd, so treating any count past the + // pre-exchange baseline as "done" races that last step and strands the + // boot at 11 endpoints with no key store. Wait through quiescence until + // the key store itself is advertised, bounded by the same + // EXCHANGE_TIMEOUT_MS. A boot that already has it exits immediately, so + // healthy boots are unaffected; this also drops the hardcoded endpoint + // count, which is not portable across SoCs. + if !this.endpoint_present(proto::EP_SKS) { let ticks = this.settle_idle_ticks.load(Relaxed).wrapping_add(1); this.settle_idle_ticks.store(ticks, Relaxed); if ticks.saturating_mul(u64::from(SETTLE_MS)) < u64::from(EXCHANGE_TIMEOUT_MS) { From 12a87ca2385bc07bcfe7aeef523cbff3864f6c67 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 18 Sep 2026 02:11:24 -0400 Subject: [PATCH 0708/2680] soc: apple: sep: optional interrupt-driven fingerprint capture The capture loop polled the sensor's status over SPI every 2 ms to spot a ready frame. Add an opt-in data-ready interrupt (capture_irq=1; default 0 keeps polling) that wakes the loop the instant the sensor asserts its data-ready line. The SPI status read still gates every frame, so a missed or spurious interrupt costs at most one poll interval and the poll path is untouched when the option is off. The interrupt is configured once, while the sensor is patched and idle -- the only safe moment, since reconfiguring the line mid-capture is what drops the sensor's firmware patch -- and then left alone for the driver's life. On j414s it is the pin adjacent to the power line (GPIO 121 -> IRQ 144); other boards describe it as the SPI node's interrupt. It is edge-triggered on both edges: the line idles low, so a level trigger would storm. Boot-verified on j414s (T6020): with capture_irq=1 the data-ready IRQ 144 is configured, fingerprint verify captures and matches through the interrupt, the IRQ fires a handful of times per touch (not a level storm), and the sensor keeps its firmware patch (no STATE_NEEDS_PATCH). Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Assisted-by: Claude Opus 4.8 --- drivers/soc/apple/sbio.rs | 33 +++++++- drivers/soc/apple/sensor.rs | 29 +++++++ drivers/soc/apple/sensor_shim.c | 144 ++++++++++++++++++++++++++++++++ drivers/soc/apple/sep.rs | 4 + drivers/soc/apple/shim.h | 11 +++ 5 files changed, 219 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 16a8312a56137..eb98d261124f7 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -736,7 +736,21 @@ impl SepData { patch = reloaded_patch; } - self.complete_bringup(patch) + let ok = self.complete_bringup(patch); + // The sensor is patched and idle here -- the only safe moment to + // configure the data-ready interrupt, which is then left alone for the + // driver's life. Opt-in; the poll path is unchanged when off. + if ok && *module_parameters::capture_irq.value() != 0 && !sensor::irq_available() { + if sensor::irq_setup() { + dev_info!(self.dev, "sensor: interrupt-driven capture enabled\n"); + } else { + dev_warn!( + self.dev, + "sensor: data-ready interrupt unavailable; capture stays on SPI polling\n" + ); + } + } + ok } pub(crate) fn run_bringup(&self) { @@ -1651,6 +1665,14 @@ impl SepData { let mut previous: Option<[u8; sensor::STATUS_LEN]> = None; let mut armed_reported = false; + // Interrupt-driven capture only replaces the fixed inter-poll sleep with + // a data-ready wait; the SPI status read below still gates every frame, + // so a missed or spurious interrupt costs at most one poll interval. + let use_irq = *module_parameters::capture_irq.value() != 0 && sensor::irq_available(); + if use_irq { + sensor::irq_arm(); + } + for attempt in 0..ENROL_POLL_ATTEMPTS { if !bio::capture_is_live(&self.bio_session.lock()) { return CaptureWait::Abandon; @@ -1693,7 +1715,14 @@ impl SepData { return CaptureWait::Ready(count); } - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(ENROL_POLL_MS))); + if use_irq { + // Wake the instant the sensor asserts data-ready, or after the + // same interval on timeout; then re-arm for the next frame. + let _ = sensor::irq_wait(ENROL_POLL_MS); + sensor::irq_arm(); + } else { + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(ENROL_POLL_MS))); + } } let _ = sensor::status(); CaptureWait::Timeout diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index 66ae99800a40d..6b6393dc28c04 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -19,6 +19,10 @@ extern "C" { fn sep_sensor_xfer(tx: *const c_void, rx: *mut c_void, len: usize) -> c_int; fn sep_sensor_xfer_tx(tx: *const c_void, len: usize) -> c_int; fn sep_sensor_xfer2(tx: *const c_void, tx_len: usize, rx: *mut c_void, rx_len: usize) -> c_int; + fn sep_sensor_irq_setup() -> c_int; + fn sep_sensor_irq_available() -> c_int; + fn sep_sensor_irq_arm(); + fn sep_sensor_irq_wait(timeout_ms: c_uint) -> c_int; fn sep_sensor_firmware_name() -> *const c_char; } @@ -222,6 +226,31 @@ pub(crate) fn power(on: bool) -> bool { unsafe { sep_sensor_power(if on { 1 } else { 0 }) == 0 } } +/// Sets up the data-ready interrupt once, while the sensor is idle. Returns +/// true when an interrupt is available for interrupt-driven capture. +pub(crate) fn irq_setup() -> bool { + // SAFETY: no arguments; idempotent, falls back to -errno when unavailable. + unsafe { sep_sensor_irq_setup() == 0 } +} + +pub(crate) fn irq_available() -> bool { + // SAFETY: reads one int. + unsafe { sep_sensor_irq_available() != 0 } +} + +/// Clears any stale data-ready signal before a capture. +pub(crate) fn irq_arm() { + // SAFETY: no arguments; a no-op when no IRQ is configured. + unsafe { sep_sensor_irq_arm() } +} + +/// Waits up to `timeout_ms` for the data-ready line to assert. Returns true if +/// it fired, false on timeout or when no interrupt is configured. +pub(crate) fn irq_wait(timeout_ms: c_uint) -> bool { + // SAFETY: no pointers; the shim owns the completion and the IRQ line. + unsafe { sep_sensor_irq_wait(timeout_ms) == 0 } +} + fn command(cmd: &[u8; CMD_LEN]) -> Result<()> { let mut rx = [0u8; CMD_LEN]; // SAFETY: both buffers are `CMD_LEN` bytes and live across the call. diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index 22623213f48c3..b9db956f1fc13 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -3,11 +3,13 @@ /* Fingerprint sensor SPI shim: moves bytes over the bus and toggles power. */ #include +#include #include #include #include #include #include +#include #include #include #include @@ -51,6 +53,16 @@ static bool sep_registered; static int sep_power_source; +/* + * Data-ready line for interrupt-driven capture. On j414s it is the pin adjacent + * to the power line (122) on the same controller; other boards describe it as + * the SPI node's interrupt, which the SPI core resolves into spi->irq. + */ +#define SEP_SENSOR_DRDY_LINE 121 +static struct gpio_desc *sep_drdy; /* set only when we own the gpiochip line (j414s) */ +static int sep_drdy_irq = -1; +static DECLARE_COMPLETION(sep_drdy_done); + /* * Takes the power line as an output driven low: the power cycle begins with an * off phase, so the line must be actively driven off, not merely read as low. @@ -227,6 +239,7 @@ static int sep_sensor_probe(struct spi_device *spi) static void sep_sensor_remove(struct spi_device *spi) { + sep_sensor_irq_teardown(); sep_release_power(); sep_spi = NULL; } @@ -352,6 +365,137 @@ int sep_sensor_power(int on) return 0; } +/* + * Threaded data-ready handler. IRQF_ONESHOT keeps the line masked while this + * runs, so a level-asserted line cannot storm the CPU; it only wakes the + * capture loop, which still confirms the sensor state over SPI before reading. + */ +static irqreturn_t sep_drdy_isr(int irq, void *dev_id) +{ + complete(&sep_drdy_done); + return IRQ_HANDLED; +} + +/* + * Set up the data-ready interrupt once, while the sensor is idle. It is left + * configured and enabled for the driver's lifetime -- never toggled per capture + * -- because reconfiguring this line mid-capture is what drops the sensor's + * firmware patch. Returns 0 when an interrupt is available, -errno otherwise + * (the caller then falls back to polling). + */ +int sep_sensor_irq_setup(void) +{ + struct device_node *np; + struct gpio_device *gdev; + struct gpio_chip *gc; + int irq, rc; + + if (!sep_spi) + return -ENODEV; + if (sep_drdy_irq >= 0) + return 0; + + if (sep_spi->irq > 0) { + /* A board that describes the data-ready line as the SPI node's + * interrupt gets it resolved by the SPI core. */ + irq = sep_spi->irq; + } else if (of_machine_is_compatible("apple,j414s")) { + np = of_find_node_by_path(SEP_SENSOR_GPIO_NODE); + if (!np) + return -ENODEV; + gdev = gpio_device_find_by_fwnode(of_fwnode_handle(np)); + of_node_put(np); + if (!gdev) + return -ENODEV; + gc = gpio_device_get_chip(gdev); + if (!gc) { + gpio_device_put(gdev); + return -ENODEV; + } + sep_drdy = gpiochip_request_own_desc(gc, SEP_SENSOR_DRDY_LINE, + "apple-mesa-drdy", + GPIO_LOOKUP_FLAGS_DEFAULT, + GPIOD_IN); + gpio_device_put(gdev); + if (IS_ERR(sep_drdy)) { + sep_drdy = NULL; + return -ENODEV; + } + irq = gpiod_to_irq(sep_drdy); + if (irq < 0) { + gpiochip_free_own_desc(sep_drdy); + sep_drdy = NULL; + return irq; + } + } else { + return -ENODEV; + } + + init_completion(&sep_drdy_done); + /* + * Edge-triggered, both edges: the data-ready line idles low, so a level + * trigger would storm continuously. An edge fires once per data-ready + * transition regardless of the line's asserted polarity, so the capture + * loop actually waits; a missed edge only costs one status poll. + */ + rc = request_threaded_irq(irq, NULL, sep_drdy_isr, + IRQF_ONESHOT | IRQF_TRIGGER_RISING | + IRQF_TRIGGER_FALLING, + "apple-mesa-drdy", sep_spi); + if (rc) { + if (sep_drdy) { + gpiochip_free_own_desc(sep_drdy); + sep_drdy = NULL; + } + return rc; + } + sep_drdy_irq = irq; + dev_info(&sep_spi->dev, + "sep sensor: data-ready IRQ %d configured for interrupt capture\n", + irq); + return 0; +} + +void sep_sensor_irq_teardown(void) +{ + if (sep_drdy_irq >= 0) { + free_irq(sep_drdy_irq, sep_spi); + sep_drdy_irq = -1; + } + if (sep_drdy) { + gpiochip_free_own_desc(sep_drdy); + sep_drdy = NULL; + } +} + +int sep_sensor_irq_available(void) +{ + return sep_drdy_irq >= 0; +} + +/* Clear any stale signal before a capture so the next wait reflects a fresh + * data-ready edge, not a leftover from the previous frame. */ +void sep_sensor_irq_arm(void) +{ + if (sep_drdy_irq >= 0) + reinit_completion(&sep_drdy_done); +} + +/* + * Wait for the data-ready line to assert, up to timeout_ms. 0 = fired, + * -ETIMEDOUT = no signal (the caller re-polls status over SPI regardless, so a + * missed interrupt only costs one poll interval), -ENODEV = no interrupt. + */ +int sep_sensor_irq_wait(unsigned int timeout_ms) +{ + if (sep_drdy_irq < 0) + return -ENODEV; + if (wait_for_completion_timeout(&sep_drdy_done, + msecs_to_jiffies(timeout_ms)) == 0) + return -ETIMEDOUT; + return 0; +} + /* * One chip-select assertion, `len` clocks, full duplex. The caller supplies all * `len` transmit bytes (trailing ones as 0xff) rather than relying on the diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 951f42a40163b..fda8742f4d7c8 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -2186,5 +2186,9 @@ module! { default: 0, description: "Low 64 bits of an explicit xART OS UUID", }, + capture_irq: u8 { + default: 0, + description: "Wake the capture loop from the sensor's data-ready interrupt instead of polling SPI status (experimental; 0 = poll, the default). The interrupt only accelerates the loop -- status is still read over SPI to gate each frame -- and is set up once while the sensor is idle, never toggled mid-capture.", + }, }, } diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index cc3f4ccb7a19d..e13ca258d92f8 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -129,6 +129,17 @@ int sep_sensor_xfer(const void *tx, void *rx, size_t len); int sep_sensor_xfer_tx(const void *tx, size_t len); int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len); +/* + * Optional data-ready interrupt for interrupt-driven capture. Set up once while + * the sensor is idle (never toggled per capture), then armed and waited on to + * wake the capture loop the instant a frame is ready instead of polling. + */ +int sep_sensor_irq_setup(void); +void sep_sensor_irq_teardown(void); +int sep_sensor_irq_available(void); +void sep_sensor_irq_arm(void); +int sep_sensor_irq_wait(unsigned int timeout_ms); + /* -- bio_shim.c --------------------------------------------------------- */ void *sep_bio_register(const char *name, unsigned short mode, void *ctx, From 3c22cadbc95f47554242ee6121455d3fa9750162 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 18 Sep 2026 02:11:40 -0400 Subject: [PATCH 0709/2680] soc: apple: sep: surface a retry hint when an enrol capture is rejected An enrol capture the enclave rejects (partial or unusable contact) silently looped for another frame with no feedback, so the person had no idea their touch had not counted. Set the hold-still guidance on that path so the enrol poll reports it and userspace can prompt for a firmer press, matching the guidance already surfaced from the armed and reading sensor states. Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Assisted-by: Claude Opus 4.8 --- drivers/soc/apple/sbio.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index eb98d261124f7..9dd36e88ca3c3 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1080,6 +1080,12 @@ impl SepData { } ImageOutcome::Retry => { counter = counter.saturating_add(1); + // The capture did not take (partial/unusable contact). Nudge + // the person to hold still and press again rather than + // silently re-capturing with no feedback. + if bio::enrol_guide(&mut self.bio_session.lock(), bio::Guidance::HoldStill) { + self.bio_wake(); + } } ImageOutcome::NoFinger => { break Some(Err(ENROL_STATUS_TIMEOUT)); From 02c155fffd012c35227a6f1f5af194746d39a388 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 18 Sep 2026 03:10:37 -0400 Subject: [PATCH 0710/2680] soc: apple: sep: make interrupt-driven capture the default Land the interrupt path as the default from hardware validation: it is no longer behind the capture_irq parameter (removed), and it is set up whenever the sensor exposes a data-ready line, with SPI polling kept only as the fallback for a machine that does not describe one. Two fixes the enrolment stress test surfaced: - Trigger on the rising edge only. The line idles low, so a level trigger storms and both-edges also fires on the deassert -- a spurious wake that, mid-enrolment (the finger lifts and repositions between the eight stages), spun the capture loop. - Bound the capture wait by wall-clock time, not a fixed attempt count. A stray edge returns the wait early, so an attempt budget would be spent in milliseconds and time the capture out before a frame lands; the time bound degrades, at worst, to the poll path's duration. The wait now blocks on the data-ready line for up to ENROL_IRQ_WAIT_MS (250 ms) rather than re-reading SPI status every 2 ms, waking the instant a frame is ready. Boot-verified on j414s (T6020): fprintd verify and a full eight-stage enrol both capture and match through the interrupt, with no STATE_NEEDS_PATCH and no level storm. Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Assisted-by: Claude Opus 4.8 --- drivers/soc/apple/sbio.rs | 42 +++++++++++++++++++++++---------- drivers/soc/apple/sensor_shim.c | 12 +++++----- drivers/soc/apple/sep.rs | 9 +++---- 3 files changed, 41 insertions(+), 22 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 9dd36e88ca3c3..8b885ed98753e 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -739,14 +739,15 @@ impl SepData { let ok = self.complete_bringup(patch); // The sensor is patched and idle here -- the only safe moment to // configure the data-ready interrupt, which is then left alone for the - // driver's life. Opt-in; the poll path is unchanged when off. - if ok && *module_parameters::capture_irq.value() != 0 && !sensor::irq_available() { + // driver's life. Interrupt capture is the default; a machine that does + // not describe a data-ready line falls back to SPI polling. + if ok && !sensor::irq_available() { if sensor::irq_setup() { - dev_info!(self.dev, "sensor: interrupt-driven capture enabled\n"); + dev_info!(self.dev, "sensor: interrupt-driven capture\n"); } else { dev_warn!( self.dev, - "sensor: data-ready interrupt unavailable; capture stays on SPI polling\n" + "sensor: no data-ready interrupt (describe it in DT); capture falls back to SPI polling\n" ); } } @@ -1671,15 +1672,26 @@ impl SepData { let mut previous: Option<[u8; sensor::STATUS_LEN]> = None; let mut armed_reported = false; - // Interrupt-driven capture only replaces the fixed inter-poll sleep with - // a data-ready wait; the SPI status read below still gates every frame, - // so a missed or spurious interrupt costs at most one poll interval. - let use_irq = *module_parameters::capture_irq.value() != 0 && sensor::irq_available(); + // Interrupt-driven capture blocks on the data-ready line instead of + // polling; the SPI status read below still gates every frame, so a + // missed or spurious interrupt costs at most one backstop interval. A + // machine with no data-ready line falls back to SPI polling. + let use_irq = sensor::irq_available(); if use_irq { sensor::irq_arm(); } - for attempt in 0..ENROL_POLL_ATTEMPTS { + // Bound the wait by wall-clock time, not a fixed iteration count. The + // data-ready interrupt can wake early on a stray edge (the sensor + // toggles the line as the finger moves between enrol frames); a fixed + // attempt budget would then be spent in milliseconds and time the + // capture out before the frame lands. Time-bounding degrades, at worst, + // to the same duration as the poll path. + let deadline_ns = crate::shim::monotonic_ns().saturating_add( + u64::from(ENROL_POLL_ATTEMPTS) * u64::from(ENROL_POLL_MS) * 1_000_000, + ); + let mut attempt: u32 = 0; + loop { if !bio::capture_is_live(&self.bio_session.lock()) { return CaptureWait::Abandon; } @@ -1721,14 +1733,20 @@ impl SepData { return CaptureWait::Ready(count); } + if crate::shim::monotonic_ns() >= deadline_ns { + break; + } + if use_irq { - // Wake the instant the sensor asserts data-ready, or after the - // same interval on timeout; then re-arm for the next frame. - let _ = sensor::irq_wait(ENROL_POLL_MS); + // Block on the data-ready line; the interrupt wakes this the + // instant a frame is ready, and the backstop bounds a missed + // edge. Re-arm for the next frame. + let _ = sensor::irq_wait(ENROL_IRQ_WAIT_MS); sensor::irq_arm(); } else { kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(ENROL_POLL_MS))); } + attempt = attempt.saturating_add(1); } let _ = sensor::status(); CaptureWait::Timeout diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index b9db956f1fc13..aff7da78fe49e 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -433,14 +433,14 @@ int sep_sensor_irq_setup(void) init_completion(&sep_drdy_done); /* - * Edge-triggered, both edges: the data-ready line idles low, so a level - * trigger would storm continuously. An edge fires once per data-ready - * transition regardless of the line's asserted polarity, so the capture - * loop actually waits; a missed edge only costs one status poll. + * Edge-triggered on the rising edge only. The line idles low, so a level + * trigger storms and both-edges fires again on the deassert -- a spurious + * wake that, mid-enrolment, spins the capture loop. The data-ready assert + * is the low->high edge; the capture loop's status read gates the frame, + * and a missed edge only costs one status poll. */ rc = request_threaded_irq(irq, NULL, sep_drdy_isr, - IRQF_ONESHOT | IRQF_TRIGGER_RISING | - IRQF_TRIGGER_FALLING, + IRQF_ONESHOT | IRQF_TRIGGER_RISING, "apple-mesa-drdy", sep_spi); if (rc) { if (sep_drdy) { diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index fda8742f4d7c8..e76b959444558 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -366,6 +366,11 @@ const ENROL_MAX_CAPTURES: u32 = 12; const ENROL_POLL_MS: u32 = 2; +// Interrupt-driven capture blocks on the data-ready line rather than polling +// SPI status. This is the maximum a single wait blocks before re-reading status +// as a backstop; the interrupt wakes it far sooner when a frame is ready. +const ENROL_IRQ_WAIT_MS: u32 = 250; + const ENROL_CAPTURE_TIMEOUT_MS: u32 = 60_000; const ENROL_POLL_ATTEMPTS: u32 = ENROL_CAPTURE_TIMEOUT_MS / ENROL_POLL_MS; static_assert!(ENROL_POLL_ATTEMPTS * ENROL_POLL_MS == ENROL_CAPTURE_TIMEOUT_MS); @@ -2186,9 +2191,5 @@ module! { default: 0, description: "Low 64 bits of an explicit xART OS UUID", }, - capture_irq: u8 { - default: 0, - description: "Wake the capture loop from the sensor's data-ready interrupt instead of polling SPI status (experimental; 0 = poll, the default). The interrupt only accelerates the loop -- status is still read over SPI to gate each frame -- and is set up once while the sensor is idle, never toggled mid-capture.", - }, }, } From 717e44989aa9b91246ea4e8996ff5f77c27da2b3 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:04:35 -0400 Subject: [PATCH 0711/2680] soc: apple: sep: add the J700 platform and Mesa revision Port the Neo warm SEP profile and 0x3356 sensor identity from linux-aurora 450f581707. Fold in the later hardware finding from 18165cf1de: this J700 accepted the T6020-style keybag encoding and refused the strict form. Keep the wider DMA capability described in the profile so older boards retain their masks. (cherry picked from commit eec15fc3c4782e9f6785eaa888e724ef9c3cd808) --- drivers/soc/apple/profile.rs | 44 +++++++++++++++++++++++++++++++++++- drivers/soc/apple/sbio.rs | 5 ++++ drivers/soc/apple/sensor.rs | 8 +++++-- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 61faf6b88736f..2c3c8c1e70db5 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -8,12 +8,14 @@ //! here once. The driver reads addresses from the device tree, never scans for //! them, and never patches properties at runtime. //! -//! Two bring-up targets are modelled: +//! Three bring-up targets are modelled: //! //! * `T8103` / J313 (MacBook Air, M1): the host boots the SEP with the boot //! endpoint handshake over a 0x30000 shared-memory window. //! * `T6020` / J414s (MacBook Pro 14", M2 Pro): the driver does the warm //! single-message registration over a 0x40000 window. +//! * `T8140` / J700 (MacBook Neo, A18 Pro): iBoot boots sepOS before the AP +//! OS, so the driver uses the warm registration path. // The boot handshake, identity source, sensor and DART fields are consumed by // the boot-endpoint, identity and sensor-transport paths. @@ -115,6 +117,8 @@ pub(crate) struct PlatformProfile { /// Require a static `apple,dma-range` on the SEP DART. The T6020 SEP only /// accepts IOVAs below 4 GiB; T8103 works with the stock DART aperture. pub(crate) dart_range_required: bool, + /// Whether the SEP DART IOVA window needs a DMA mask wider than 32 bits. + pub(crate) wide_dma_mask: bool, /// Reserved-memory region holding the SEP firmware image (cold-boot path). pub(crate) firmware_region: &'static CStr, /// Identity keybag CREATE_KEYBAG field encoding (per-SoC; see [`KeybagCreate`]). @@ -138,6 +142,7 @@ const T8103: PlatformProfile = PlatformProfile { capture_qualified: false, }, dart_range_required: false, + wide_dma_mask: false, firmware_region: c"sepfw", // Strict enclave: the type goes in the third word; the first word is 0. keybag_create: KeybagCreate { @@ -164,6 +169,7 @@ const T6020: PlatformProfile = PlatformProfile { capture_qualified: false, }, dart_range_required: true, + wide_dma_mask: false, firmware_region: c"sepfw", // Proven encoding: the lenient enclave takes the variant in the first word. keybag_create: KeybagCreate { @@ -173,10 +179,43 @@ const T6020: PlatformProfile = PlatformProfile { }, }; +/// MacBook Neo. The J700 ADT records a pre-booted SEP, a spi2 Mesa sensor +/// (0x3356), and a SEP DART aperture above 4 GiB. These values describe that +/// board; device-tree nodes remain authoritative for addresses and resources. +const T8140: PlatformProfile = PlatformProfile { + name: "T8140/J700", + shmem_capacity: 0x4_0000, + shmem_first_item: b"CINP", + bootstrap: Bootstrap::WarmRegister, + identity: IdentitySource::Chosen, + sensor: SensorProfile { + controller_base: 0x3_8510_8000, + chip_select: 0, + max_hz: 8_000_000, + cs_setup_ns: 20, + cs_hold_ns: 20, + mode: SpiMode::Mode2, + expected_id: 0x3356, + capture_qualified: false, + }, + dart_range_required: true, + wide_dma_mask: true, + firmware_region: c"sepfw", + // J700 hardware refused the strict encoding and accepted this T6020 form + // (linux-aurora commit 18165cf1de, 2026-09-19). + keybag_create: KeybagCreate { + variant: 5, + bag_type: 0, + arg: -1, + }, +}; + static_assert!(T8103.shmem_capacity == 0x30000); static_assert!(T6020.shmem_capacity == 0x40000); +static_assert!(T8140.shmem_capacity == 0x40000); static_assert!(T8103.sensor.controller_base == 0x235108000); static_assert!(T6020.sensor.controller_base == 0x39b108000); +static_assert!(T8140.sensor.controller_base == 0x385108000); static_assert!(T8103.sensor.expected_id == T6020.sensor.expected_id); /// Whether the machine root declares `compatible`. @@ -216,5 +255,8 @@ pub(crate) fn detect() -> Result<&'static PlatformProfile> { if machine_has(b"apple,t6020") { return Ok(&T6020); } + if machine_has(b"apple,t8140") { + return Ok(&T8140); + } Err(ENODEV) } diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 8b885ed98753e..8c5d6b6a0f950 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1205,6 +1205,11 @@ impl SepData { if let sensor::Offset12::Identifier(id) = st.offset12() { if id != 0 && !sensor::KNOWN_IDENTIFIERS.contains(&id) { + dev_err!( + self.dev, + "sensor: unknown Mesa identifier {:#06x} at status offset 12; not registering it with the enclave\n", + id + ); return None; } } diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index 6b6393dc28c04..f5f3ea6307c24 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -79,6 +79,8 @@ pub(crate) enum PowerSource { None, NodeProperty, ChipLine, + /// Rails switched by the SMC on J700. + SmcRails, } impl PowerSource { @@ -86,6 +88,7 @@ impl PowerSource { match v { 1 => PowerSource::NodeProperty, 2 => PowerSource::ChipLine, + 3 => PowerSource::SmcRails, _ => PowerSource::None, } } @@ -95,6 +98,7 @@ impl PowerSource { PowerSource::None => c"none — no power line was taken", PowerSource::NodeProperty => c"the sensor node's own gpios property", PowerSource::ChipLine => c"/soc/pinctrl@39b028000 line 122, taken by device-tree node", + PowerSource::SmcRails => c"the sensor node's SMC-switched rails (apple,smc-power-key)", } } } @@ -102,8 +106,8 @@ impl PowerSource { pub(crate) const POWER_ON_READ_DELAYS_MS: [u32; 4] = [0, 3, 10, 50]; pub(crate) const STATUS_IDENTIFIER: usize = 12; -// Known Mesa sensor revisions: 0x3352 through M4, 0x335e on newer parts. -pub(crate) const KNOWN_IDENTIFIERS: [u16; 2] = [0x3352, 0x335e]; +// Known Mesa sensor revisions, including the J700's 0x3356. +pub(crate) const KNOWN_IDENTIFIERS: [u16; 3] = [0x3352, 0x3356, 0x335e]; static_assert!(STATUS_IDENTIFIER + 2 <= STATUS_LEN); const CMD_LEN: usize = 7; From 47df81cc091fc65a749758206e9488c020e197c0 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:04:55 -0400 Subject: [PATCH 0712/2680] soc: apple: sep: power J700 Mesa through SMC rails Port linux-aurora 29adacc631 with DT-described pcLD rail selectors and symmetric power sequencing. Normalize macsmc write replies so a successful response size is not mistaken for an error; retain a GPIO path for older boards. (cherry picked from commit 889bb3af8e416a065f6c65e8cec574abc67564f4) --- .../input/apple,mesa-fingerprint.yaml | 21 ++- drivers/soc/apple/Kconfig | 1 + drivers/soc/apple/sbio.rs | 4 +- drivers/soc/apple/sensor.rs | 1 + drivers/soc/apple/sensor_shim.c | 147 ++++++++++++++++++ 5 files changed, 172 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml b/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml index be9bbcf473e85..68531cf93ba0b 100644 --- a/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml +++ b/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml @@ -37,6 +37,19 @@ properties: maxItems: 1 description: Power line to the sensor, asserted to power it up. + apple,smc-power-key: + $ref: /schemas/types.yaml#/definitions/string + description: + Four-character SMC key used by boards with SMC-switched sensor rails. + The rail operation is an eight-byte little-endian pair of an on/off + value and a rail selector word. + + apple,smc-power-rails: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 4 + description: Rail selector words, in power-on order. + firmware-name: maxItems: 1 description: @@ -51,9 +64,15 @@ properties: required: - compatible - reg - - enable-gpios - firmware-name +oneOf: + - required: + - enable-gpios + - required: + - apple,smc-power-key + - apple,smc-power-rails + unevaluatedProperties: false examples: diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 19131befc5d92..fa1cc0758e594 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -119,6 +119,7 @@ config APPLE_SEP depends on HW_RANDOM depends on TRUSTED_KEYS depends on CRYPTO + depends on MFD_MACSMC || !MFD_MACSMC select CRYPTO_HMAC select CRYPTO_SHA256 select CRYPTO_AES diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 8c5d6b6a0f950..99665791198b6 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -249,12 +249,14 @@ impl SepData { let bound = sensor::is_bound(); self.sensor_present.store(bound, Relaxed); - if bound && sensor::power_line().is_none() { + if bound && sensor::power_source() == sensor::PowerSource::None { dev_warn!( self.dev, "sensor: no power line ({}); an unpowered sensor answers sixteen zero bytes like a dead bus\n", sensor::power_source().name() ); + } else if bound { + dev_info!(self.dev, "sensor: power via {}\n", sensor::power_source().name()); } } diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index f5f3ea6307c24..8fec6b71c8715 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -200,6 +200,7 @@ pub(crate) fn is_bound() -> bool { unsafe { sep_sensor_bound() != 0 } } +#[allow(dead_code)] pub(crate) fn power_line() -> Option { // SAFETY: no arguments; returns a negative errno when unbound. let n = unsafe { sep_sensor_power_line() }; diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index aff7da78fe49e..ec4d30581da67 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -10,10 +10,14 @@ #include #include #include +#include #include #include +#include +#include #include #include +#include #include "shim.h" @@ -50,9 +54,19 @@ static bool sep_registered; #define SEP_POWER_NONE 0 #define SEP_POWER_NODE_PROPERTY 1 #define SEP_POWER_CHIP_LINE 2 +#define SEP_POWER_SMC_RAILS 3 static int sep_power_source; +/* J700's Mesa power rails are selected by the pcLD SMC pKW8 operation. + * The DT gives the key and selectors; each write is { u32 on/off, u32 rail }. + */ +#define SEP_SMC_MAX_RAILS 4 +static struct apple_smc *sep_smc; +static smc_key sep_smc_key; +static u32 sep_smc_rails[SEP_SMC_MAX_RAILS]; +static int sep_smc_nrails; + /* * Data-ready line for interrupt-driven capture. On j414s it is the pin adjacent * to the power line (122) on the same controller; other boards describe it as @@ -63,6 +77,107 @@ static struct gpio_desc *sep_drdy; /* set only when we own the gpiochip line (j4 static int sep_drdy_irq = -1; static DECLARE_COMPLETION(sep_drdy_done); +#if IS_ENABLED(CONFIG_MFD_MACSMC) +static int sep_smc_write_rail(u32 rail, u16 value) +{ + u8 payload[8]; + int ret; + + put_unaligned_le32(value, payload); + put_unaligned_le32(rail, payload + 4); + ret = apple_smc_write(sep_smc, sep_smc_key, payload, sizeof(payload)); + /* macsmc forwards the SMC response size. Write replies may report no + * payload or echo the request length; neither is a failure status. + */ + if (ret == 0 || ret == (int)sizeof(payload)) + return 0; + return ret < 0 ? ret : -EIO; +} + +static int sep_smc_set_power(int on) +{ + int i, rc = 0; + + if (on) { + for (i = 0; i < sep_smc_nrails && !rc; i++) + rc = sep_smc_write_rail(sep_smc_rails[i], 1); + } else { + for (i = sep_smc_nrails - 1; i >= 0 && !rc; i--) + rc = sep_smc_write_rail(sep_smc_rails[i], 0); + } + if (rc) + pr_err("apple_sep: sensor: SMC rail write failed (%d)\n", rc); + return rc; +} + +/* Nothing is written until the expected eight-byte SMC key is verified. */ +static bool sep_acquire_smc_power(struct spi_device *spi) +{ + struct device_node *np = spi->dev.of_node, *smc_np; + struct platform_device *smc_pdev; + struct apple_smc_key_info info; + const char *key = NULL; + int n, rc; + + if (!np || of_property_read_string(np, "apple,smc-power-key", &key)) + return false; + if (!key || strlen(key) != 4) { + dev_warn(&spi->dev, "sep sensor: apple,smc-power-key must have four characters\n"); + return false; + } + n = of_property_count_u32_elems(np, "apple,smc-power-rails"); + if (n <= 0 || n > SEP_SMC_MAX_RAILS || + of_property_read_u32_array(np, "apple,smc-power-rails", sep_smc_rails, n)) { + dev_warn(&spi->dev, "sep sensor: expected 1..%d SMC rail words\n", + SEP_SMC_MAX_RAILS); + return false; + } + + smc_np = of_find_compatible_node(NULL, NULL, "apple,smc"); + if (!smc_np) + return false; + smc_pdev = of_find_device_by_node(smc_np); + of_node_put(smc_np); + if (!smc_pdev) + return false; + sep_smc = dev_get_drvdata(&smc_pdev->dev); + put_device(&smc_pdev->dev); + if (!sep_smc) + return false; + + sep_smc_key = __SMC_KEY(key[0], key[1], key[2], key[3]); + rc = apple_smc_get_key_info(sep_smc, sep_smc_key, &info); + if (rc || info.size != 8) { + dev_warn(&spi->dev, "sep sensor: SMC key %s absent or not eight bytes\n", key); + sep_smc = NULL; + return false; + } + sep_smc_nrails = n; + sep_power_source = SEP_POWER_SMC_RAILS; + dev_info(&spi->dev, "sep sensor: power from SMC key %s (%d rails)\n", key, n); + if (sep_smc_set_power(0)) { + sep_power_source = SEP_POWER_NONE; + sep_smc_nrails = 0; + sep_smc = NULL; + return false; + } + return true; +} +#else +static int sep_smc_set_power(int on) +{ + return -ENODEV; +} + +static bool sep_acquire_smc_power(struct spi_device *spi) +{ + if (spi->dev.of_node && + of_property_present(spi->dev.of_node, "apple,smc-power-key")) + dev_warn(&spi->dev, "sep sensor: SMC power needs CONFIG_MFD_MACSMC\n"); + return false; +} +#endif + /* * Takes the power line as an output driven low: the power cycle begins with an * off phase, so the line must be actively driven off, not merely read as low. @@ -73,6 +188,9 @@ static void sep_acquire_power(struct spi_device *spi) struct gpio_device *gdev; struct gpio_chip *gc; + if (sep_acquire_smc_power(spi)) + return; + sep_power = gpiod_get_index(&spi->dev, NULL, 0, GPIOD_OUT_LOW); if (!IS_ERR(sep_power)) { sep_power_source = SEP_POWER_NODE_PROPERTY; @@ -137,6 +255,13 @@ static void sep_acquire_power(struct spi_device *spi) static void sep_release_power(void) { + if (sep_power_source == SEP_POWER_SMC_RAILS) { + sep_smc_set_power(0); + sep_smc = NULL; + sep_smc_nrails = 0; + sep_power_source = SEP_POWER_NONE; + return; + } if (!sep_power) return; gpiod_set_value_cansleep(sep_power, 0); @@ -312,6 +437,18 @@ int sep_sensor_cs_timing_mode(void) /* Power cycle: off, 10 ms, on, 7 ms. -ENODEV if there is no power line. */ int sep_sensor_power_cycle(void) { + if (sep_power_source == SEP_POWER_SMC_RAILS) { + int rc = sep_smc_set_power(0); + + if (rc) + return rc; + msleep(SEP_SENSOR_OFF_DELAY_MS); + rc = sep_smc_set_power(1); + if (rc) + return rc; + msleep(SEP_SENSOR_ON_DELAY_MS); + return 0; + } if (!sep_power) return -ENODEV; @@ -330,6 +467,8 @@ int sep_sensor_power_source(void) int sep_sensor_power_line(void) { + if (sep_power_source == SEP_POWER_SMC_RAILS) + return -ENODEV; if (!sep_power) return -ENODEV; return desc_to_gpio(sep_power); @@ -354,6 +493,14 @@ const char *sep_sensor_firmware_name(void) /* Powers the sensor on/off, holding the hardware settling delay. */ int sep_sensor_power(int on) { + if (sep_power_source == SEP_POWER_SMC_RAILS) { + int rc = sep_smc_set_power(on ? 1 : 0); + + if (rc) + return rc; + msleep(on ? SEP_SENSOR_ON_DELAY_MS : SEP_SENSOR_OFF_DELAY_MS); + return 0; + } if (!sep_power) return -ENODEV; From dc21a72da07000f2e69cdbdd4166eef87b5378cf Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:05:18 -0400 Subject: [PATCH 0713/2680] soc: apple: sep: widen the J700 DMA mask before allocation Port linux-aurora 11d4dff9a3. The Neo SEP DART aperture starts above 4 GiB, so select a 42-bit DMA mask for that profile before allocating shared memory. Keep the established M1/M2 DMA masks unchanged for regression testing. (cherry picked from commit b65f17cced0b25bda3bb40ae9a4b4d673c5da778) --- drivers/soc/apple/sep.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index e76b959444558..0dce1b07e3e04 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -38,6 +38,7 @@ use kernel::{ bindings, device, dma, + dma::Device as _, driver, new_mutex, of, @@ -2097,6 +2098,13 @@ impl platform::Driver for SepDriver { } let sep_node = dt::DtNode::of_device(dev).ok_or(ENODEV)?; + // J700's SEP DART window starts above 4 GiB. Leave the proven masks + // on older platforms unchanged for the M1/M2 regression run. + if profile::detect()?.wide_dma_mask { + // SAFETY: probe has not allocated DMA memory for this device yet. + unsafe { pdev.dma_set_mask_and_coherent(dma::DmaMask::new::<42>())? }; + } + if dt::registration_already_sent(&sep_node) { dev_err!( dev, From c38c4fe864134dc178cf2a420f64391341c26637 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:05:32 -0400 Subject: [PATCH 0714/2680] soc: apple: sep: wait for late J700 key-store discovery Port linux-aurora 41017b63c1: the measured J700 application-endpoint burst arrives around 43 seconds after boot. Allow up to 90 seconds, returning as soon as the key store is advertised. (cherry picked from commit f3a5cf1ead57b735ee8428f8895570b004c63388) --- drivers/soc/apple/sep.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 0dce1b07e3e04..df754d5487a2a 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -86,7 +86,10 @@ const VERIFY_WORK_ID: u64 = 3; static_assert!(VERIFY_WORK_ID != ENROL_WORK_ID && VERIFY_WORK_ID != SETTLE_WORK_ID); static_assert!(VERIFY_WORK_ID != 0); -const EXCHANGE_TIMEOUT_MS: time::Msecs = 15000; +// J700 traces show the key-store endpoint arriving around 43 s after boot, +// well after the first infrastructure endpoints. The wait exits promptly on +// other machines when their key store becomes ready. +const EXCHANGE_TIMEOUT_MS: time::Msecs = 90000; const PHASE_ATTACH: u32 = 0; const PHASE_EXCHANGE: u32 = 1; From b44d40db291da93d3142d849b8f52667d9a3b3f3 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 21 Sep 2026 23:05:45 -0400 Subject: [PATCH 0715/2680] soc: apple: sep: refuse unverified raw xART writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The raw-owner locator scores CRC-valid record signatures in a fixed 6 MiB window rather than resolving the current .gl APFS file extent. If both root records have moved past slot zero, multiple shifted windows can pass the root test; a higher-base tie can extend into unrelated APFS blocks. An explicit start sector does not prove ownership. Fail closed on xart_writes=1 before SEP attach, including the explicit-sector path; leave read-only inspection available. Disable the write-enabled loader recommendation. This is a safety interlock, not a macOS-parity repair: the APFS mapping and record update protocol still need hardware validation. (cherry picked from commit fe3e8cf31b016761ecc8c1e25ac5256ea90b682d) [ryan: adapted to 7.2.10 — the lines this commit changes keep the 7.2 lifetime-bound driver-core form (device::Core<'_>, Data<'bound>)] Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 3 +- drivers/soc/apple/sep.rs | 9 +++++- drivers/soc/apple/xart_store.rs | 55 +++++++++++++++++---------------- tools/aurora-sep/README.md | 12 ++++--- 4 files changed, 46 insertions(+), 33 deletions(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index fa1cc0758e594..0a295fe8e29ac 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -135,7 +135,8 @@ config APPLE_SEP The driver reads the existing machine-wide xART gigalocker directly from the iBoot system container, locating it by its root records. It validates the complete store and both root records before registering with SEP. - Writes remain disabled unless the xart_writes module parameter is set. + Raw APFS xART writes are refused until the extent and update protocol + can be verified; the module parameter cannot override this safety gate. Say Y here if you have an Apple silicon Mac. diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index df754d5487a2a..ce0f6fa432feb 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -2093,6 +2093,13 @@ impl platform::Driver for SepDriver { _info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { let dev: &device::Device> = pdev.as_ref(); + if *module_parameters::xart_writes.value() != 0 { + dev_err!( + dev, + "raw APFS xART writes are blocked until the .gl extent and update protocol are validated\n" + ); + return Err(ENOTSUPP); + } if *module_parameters::provision_keybag.value() != 0 && *module_parameters::xart_writes.value() == 0 { @@ -2184,7 +2191,7 @@ module! { params: { xart_writes: u8 { default: 0, - description: "Allow writes to the validated shared xART store", + description: "Request shared xART writes (refused by the raw APFS owner until its extent is verified)", }, xart_start_sector: u64 { default: 0, diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs index ac9e0bf6c51e2..54d289d8a2946 100644 --- a/drivers/soc/apple/xart_store.rs +++ b/drivers/soc/apple/xart_store.rs @@ -5,8 +5,9 @@ //! //! The APFS locator exposes the existing `.gl` file as a block device. This //! module implements the record validation, duplicate repair, lookup and -//! copy-on-write ordering the SEP requires. It never creates storage -//! and it can be opened read-only for safe inspection and bring-up. +//! copy-on-write ordering the SEP requires. It never creates storage. +//! The raw block owner is restricted to read-only until APFS metadata can +//! establish the current physical extent of the `.gl` file. use crate::shim; use kernel::prelude::*; @@ -132,13 +133,22 @@ impl Store { /// system container directly and serves only the gigalocker extent, with no /// external helper and no hand-supplied sector. /// - /// `start_sector == 0` (the default) *locates the gigalocker automatically* - /// inside the container by its CRC-checked root records; a non-zero - /// `start_sector` is an explicit override for the rare case the scan should - /// be skipped. + /// `start_sector == 0` (the default) searches for CRC-checked root records; + /// a non-zero sector skips the search. Neither method proves that the + /// window belongs to the current APFS `.gl` file, so writes are refused. pub(crate) fn open_owner(writes_enabled: bool, start_sector: u64) -> Result { + // A CRC-valid record signature does not establish the APFS file's + // physical extent. Once both roots move away from slot zero, several + // shifted windows pass the two-root test, and a shifted window can + // extend into APFS-owned blocks. Never write through this raw owner + // until the mapping has been checked against APFS file metadata. + // An explicit sector is not proof of ownership either. + if writes_enabled { + return Err(ENOTSUPP); + } if start_sector != 0 { - return Self::open_based(OWNER_PATH, writes_enabled, start_sector << 9); + let base = start_sector.checked_mul(512).ok_or(EINVAL)?; + return Self::open_based(OWNER_PATH, false, base); } Self::open_located(writes_enabled) } @@ -150,10 +160,9 @@ impl Store { /// is probed read-only with the full CRC-checked [`open_based`]; because a /// base shifted a few slots off the true origin still keeps both roots in its /// window (passing the two-root test while dropping records that fall - /// outside), the base is pinned by the candidate that recovers the MOST live - /// records — the exact discriminator, since only the true origin captures the - /// whole set and container noise never forges a CRC-valid record. Repair is - /// never armed during discovery; only the pinned base is re-opened writable. + /// outside), the candidate with the most live records is used for read-only + /// diagnostics. This heuristic cannot establish the APFS file's origin or + /// prove it is safe to write. Repair is never armed during discovery. /// Bounded by [`MAX_LOCATE_ATTEMPTS`] probes; returns `ENODATA` if not found. fn open_located(writes_enabled: bool) -> Result { // The container is a writable block device; the block layer only grants @@ -187,11 +196,9 @@ impl Store { // shifted off the true origin by a few slots still keeps both // roots inside its 6 MiB window, so it passes yet silently // drops the live records that fall outside the shifted window. - // Probe every candidate READ-ONLY (writes_enabled forced false, - // so repair never fires at an unconfirmed base) and pin the one - // that recovers the MOST live records: only the true origin - // captures the whole record set, and container noise never - // forges a CRC-valid record, so max live records is exact. + // Probe candidates without writes and prefer the largest + // recovered record set for diagnostics. An equally valid + // shifted window is possible; this is not an APFS lookup. let hit = off + p as u64; let mut best: Option<(usize, usize, u64)> = None; // (valid, malformed, base) let mut k: u64 = 0; @@ -214,13 +221,9 @@ impl Store { let better = match best { None => true, // More live records wins; ties break to fewer - // malformed, then to the HIGHER base. An up-shift - // drops the lowest occupied slots (and any root - // there, which fails the two-root test), so it - // never ties on live count; the only bases that - // can tie are down-shifts, which are all lower - // than the true origin — so the highest surviving - // candidate is the true origin. + // malformed, then to the higher base. The tie + // rule is a heuristic and may choose an + // up-shifted base when both roots moved. Some(b) => { cand.0 > b.0 || (cand.0 == b.0 && cand.1 < b.1) @@ -232,10 +235,8 @@ impl Store { } } } - // The gigalocker is unique, so the first hit that confirms any - // base has pinned it. Re-open the winner through a writable - // handle and arm repair per `writes_enabled` — repair runs - // now, and only at this confirmed origin. + // Re-open the diagnostic candidate. open_owner refuses + // writes, including automatic repair, on raw APFS extents. if let Some((_, _, base)) = best { return Self::open_based_ext(OWNER_PATH, true, writes_enabled, base); } diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 6c28a65403f51..c3f17ad3c1171 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -12,10 +12,14 @@ systemctl daemon-reload systemctl enable aurora-sep.service ``` -The kernel driver locates the xART gigalocker itself inside the iBoot System -Container (`modprobe apple_sep xart_writes=1 provision_keybag=1`, with no -`xart_start_sector`): it opens the container directly and serves only the -gigalocker extent, needing no external helper and no hand-supplied sector. +The raw xART owner is currently read-only. Do not enable the installed +`aurora-sep.service` or run `load-driver` on a shared macOS/Linux installation: +that script requests `xart_writes=1`, which the kernel now rejects while the +physical `.gl` extent is selected by scanning record signatures rather than +validated against APFS file metadata. The scan is ambiguous when live root +records have moved away from the start of the file. Read-only inspection does +not establish write safety. Re-enable provisioning only after the APFS mapping +and macOS-compatible record update protocol are verified on hardware. For fingerprint support, apply `patches/libfprint-1.94.100-apple-sep.patch` to libfprint 1.94.100, build and From 72d1cfe8c0e1664a3fdfc87fe3280e095744daf0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 21 Sep 2026 23:56:38 -0400 Subject: [PATCH 0716/2680] soc: apple: sep: resolve read-only xART extent through APFS metadata Replace the ambiguous root-record signature locator with a narrow, checksum-validated APFS checkpoint and .gl file-extent lookup. Open the shared container read-only and make an explicit sector an assertion rather than an override. Keep writes refused while the macOS ownership and update protocol are verified. (cherry picked from commit 9c0186e307f425f12c7d114e31dbc439ba55cc31) --- drivers/soc/apple/sep.rs | 9 +- drivers/soc/apple/shim.rs | 4 +- drivers/soc/apple/store_shim.c | 2 +- drivers/soc/apple/xart_apfs.rs | 342 ++++++++++++++++++++ drivers/soc/apple/xart_store.rs | 188 ++--------- tools/aurora-sep/README.md | 27 +- tools/aurora-sep/XART-SAFETY.md | 41 +++ tools/aurora-sep/apfs-xart-inspect.py | 320 ++++++++++++++++++ tools/aurora-sep/tests/test_xart_locator.py | 49 +++ 9 files changed, 805 insertions(+), 177 deletions(-) create mode 100644 drivers/soc/apple/xart_apfs.rs create mode 100644 tools/aurora-sep/XART-SAFETY.md create mode 100644 tools/aurora-sep/apfs-xart-inspect.py create mode 100644 tools/aurora-sep/tests/test_xart_locator.py diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index ce0f6fa432feb..f15dc31f33b0e 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -32,6 +32,7 @@ mod store; mod transfer; mod trusted; mod xarm; +mod xart_apfs; mod xart_store; use kernel::{ @@ -726,7 +727,7 @@ impl SepData { let base = store.base(); dev_info!( dev, - "xART: in-kernel raw-extent owner (base {:#x}); {} slots, {} live records, max revision {}, {} malformed, {} duplicate, {} repaired; writes {}\n", + "xART: APFS-resolved read-only .gl extent (base {:#x}); {} slots, {} live records, max revision {}, {} malformed, {} duplicate, {} repaired; writes {}\n", base, slots, records, @@ -2096,7 +2097,7 @@ impl platform::Driver for SepDriver { if *module_parameters::xart_writes.value() != 0 { dev_err!( dev, - "raw APFS xART writes are blocked until the .gl extent and update protocol are validated\n" + "raw APFS xART writes are blocked until the .gl update protocol and ownership invariants are validated\n" ); return Err(ENOTSUPP); } @@ -2191,11 +2192,11 @@ module! { params: { xart_writes: u8 { default: 0, - description: "Request shared xART writes (refused by the raw APFS owner until its extent is verified)", + description: "Request shared xART writes (refused until the macOS update protocol is verified)", }, xart_start_sector: u64 { default: 0, - description: "Explicit 512-byte start sector of the xART gigalocker extent within the iBoot system container. Zero (default) locates the gigalocker automatically in the container by its root records; a non-zero value overrides the search. Either way the driver opens the container directly", + description: "Expected 512-byte start sector of the APFS .gl extent in the iBoot container. Zero accepts the APFS result; nonzero must match it and cannot override it", }, provision_keybag: u8 { default: 0, diff --git a/drivers/soc/apple/shim.rs b/drivers/soc/apple/shim.rs index f02a9b86e32ef..4e25060d15d25 100644 --- a/drivers/soc/apple/shim.rs +++ b/drivers/soc/apple/shim.rs @@ -75,8 +75,8 @@ impl StoreFile { } pub(crate) fn open_block(path: &CStr, writable: bool) -> Result { - // SAFETY: `path` is NUL-terminated; the shim accepts only a block - // device whose global read-only state matches `writable`. + // SAFETY: `path` is NUL-terminated; the shim opens read-only unless + // explicitly asked to write, and rejects writes to a read-only device. let handle = unsafe { sep_store_open_block(path.as_char_ptr(), c_int::from(writable)) }; if handle.is_null() { return Err(ENODEV); diff --git a/drivers/soc/apple/store_shim.c b/drivers/soc/apple/store_shim.c index b067c39372fa7..bdfd5dfbb6391 100644 --- a/drivers/soc/apple/store_shim.c +++ b/drivers/soc/apple/store_shim.c @@ -82,7 +82,7 @@ void *sep_store_open_block(const char *path, int writable) if (IS_ERR(f)) return NULL; if (!S_ISBLK(file_inode(f)->i_mode) || - bdev_read_only(file_bdev(f)) == !!writable) { + (writable && bdev_read_only(file_bdev(f)))) { filp_close(f, NULL); return NULL; } diff --git a/drivers/soc/apple/xart_apfs.rs b/drivers/soc/apple/xart_apfs.rs new file mode 100644 index 0000000000000..9eb833baca82d --- /dev/null +++ b/drivers/soc/apple/xart_apfs.rs @@ -0,0 +1,342 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Narrow, read-only APFS lookup for the existing xART `.gl` file. +//! +//! This is deliberately not a general APFS implementation. Unsupported tree +//! shapes, encryption, fragmented files, and bad metadata fail closed. In +//! particular, finding an extent does not authorize raw writes to that file. + +use crate::shim; +use kernel::prelude::*; + +const BLOCK: usize = 4096; +const ROOT_INFO_SIZE: usize = 40; +const XART_ROLE: u16 = 0x100; +const FILE_SIZE: u64 = 0x600000; +const FS_UNENCRYPTED: u64 = 1; +const MOD: u64 = 0xffff_ffff; + +fn le16(data: &[u8], off: usize) -> Result { + Ok(u16::from_le_bytes( + data.get(off..off + 2) + .ok_or(EINVAL)? + .try_into() + .map_err(|_| EINVAL)?, + )) +} + +fn le32(data: &[u8], off: usize) -> Result { + Ok(u32::from_le_bytes( + data.get(off..off + 4) + .ok_or(EINVAL)? + .try_into() + .map_err(|_| EINVAL)?, + )) +} + +fn le64(data: &[u8], off: usize) -> Result { + Ok(u64::from_le_bytes( + data.get(off..off + 8) + .ok_or(EINVAL)? + .try_into() + .map_err(|_| EINVAL)?, + )) +} + +/// APFS's Fletcher-64 over the little-endian words after the checksum field. +fn checksum(data: &[u8]) -> Result { + if data.len() != BLOCK { + return Err(EINVAL); + } + let mut s1 = 0u64; + let mut s2 = 0u64; + for word in data[8..].chunks_exact(4) { + let value = u32::from_le_bytes(word.try_into().map_err(|_| EINVAL)?) as u64; + s1 = (s1 + value) % MOD; + s2 = (s2 + s1) % MOD; + } + let c1 = (MOD - (s1 + s2) % MOD) % MOD; + let c2 = (MOD - (s1 + c1) % MOD) % MOD; + Ok((c2 << 32) | c1) +} + +fn read_object(file: &shim::StoreFile, size: u64, paddr: u64) -> Result> { + let start = paddr.checked_mul(BLOCK as u64).ok_or(EINVAL)?; + if start.checked_add(BLOCK as u64).ok_or(EINVAL)? > size { + return Err(EINVAL); + } + let mut data = KVec::new(); + data.resize(BLOCK, 0, GFP_KERNEL)?; + file.read_block_exact(start, &mut data)?; + if le64(&data, 0)? != checksum(&data)? { + return Err(EIO); + } + Ok(data) +} + +/// The descriptor ring holds mapping blocks followed by its superblock. A +/// checksummed superblock alone is not a complete checkpoint: verify every +/// mapping and its ephemeral object before accepting that transaction. +fn valid_checkpoint( + file: &shim::StoreFile, + size: u64, + sb: &[u8], + sb_paddr: u64, + desc_base: u64, + desc_count: u64, +) -> Result<()> { + if le64(sb, 112)? != desc_base || le32(sb, 104)? as u64 != desc_count { + return Err(ENOTSUPP); + } + let index = le32(sb, 136)? as u64; + let len = le32(sb, 140)? as u64; + if index >= desc_count || len < 2 || len > desc_count { + return Err(EINVAL); + } + let last = desc_base + (index + len - 1) % desc_count; + if last != sb_paddr { + return Err(EINVAL); + } + let xid = le64(sb, 16)?; + for j in 0..len - 1 { + let paddr = desc_base + (index + j) % desc_count; + let map = read_object(file, size, paddr)?; + if le32(&map, 24)? & 0xffff != 12 || le64(&map, 16)? != xid { + return Err(EINVAL); + } + let flags = le32(&map, 32)?; + if flags & 1 != u32::from(j == len - 2) { + return Err(EINVAL); + } + let count = le32(&map, 36)? as usize; + if count > (BLOCK - 40) / 40 { + return Err(EINVAL); + } + for i in 0..count { + let at = 40 + i * 40; + if le32(&map, at + 8)? != BLOCK as u32 { + return Err(ENOTSUPP); + } + let object = read_object(file, size, le64(&map, at + 32)?)?; + if le64(&object, 8)? != le64(&map, at + 24)? + || le64(&object, 16)? != xid + || le32(&object, 24)? != le32(&map, at)? + { + return Err(EINVAL); + } + } + } + Ok(()) +} + +fn node_entry(node: &[u8], index: usize, fixed: bool) -> Result<(&[u8], &[u8])> { + if node.len() != BLOCK || le16(node, 34)? != 0 { + return Err(ENOTSUPP); + } + let flags = le16(node, 32)?; + if (flags & 4 != 0) != fixed { + return Err(ENOTSUPP); + } + let count = le32(node, 36)? as usize; + if index >= count || count > 512 { + return Err(EINVAL); + } + let toc = 56usize + .checked_add(le16(node, 40)? as usize) + .ok_or(EINVAL)?; + let table_len = le16(node, 42)? as usize; + let key_start = toc.checked_add(table_len).ok_or(EINVAL)?; + let val_end = BLOCK - if flags & 1 != 0 { ROOT_INFO_SIZE } else { 0 }; + let stride = if fixed { 4 } else { 8 }; + if key_start > val_end || toc.checked_add(count * stride).ok_or(EINVAL)? > key_start { + return Err(EINVAL); + } + let item = toc.checked_add(index * stride).ok_or(EINVAL)?; + let (ko, kl, vo, vl) = if fixed { + ( + le16(node, item)? as usize, + 16, + le16(node, item + 2)? as usize, + 16, + ) + } else { + ( + le16(node, item)? as usize, + le16(node, item + 2)? as usize, + le16(node, item + 4)? as usize, + le16(node, item + 6)? as usize, + ) + }; + let ks = key_start.checked_add(ko).ok_or(EINVAL)?; + let vs = val_end.checked_sub(vo).ok_or(EINVAL)?; + let ke = ks.checked_add(kl).ok_or(EINVAL)?; + let ve = vs.checked_add(vl).ok_or(EINVAL)?; + if ks < key_start || ke > val_end || vs < key_start || ve > val_end { + return Err(EINVAL); + } + Ok((&node[ks..ke], &node[vs..ve])) +} + +fn omap_lookup( + file: &shim::StoreFile, + size: u64, + omap_paddr: u64, + oid: u64, + xid: u64, +) -> Result { + let omap = read_object(file, size, omap_paddr)?; + if le64(&omap, 8)? != omap_paddr { + return Err(EINVAL); + } + let tree_paddr = le64(&omap, 48)?; + let tree = read_object(file, size, tree_paddr)?; + if le64(&tree, 8)? != tree_paddr || le16(&tree, 34)? != 0 || le16(&tree, 32)? & 4 == 0 { + return Err(ENOTSUPP); + } + let mut selected: Option<(u64, u32, u32, u64)> = None; + for i in 0..le32(&tree, 36)? as usize { + let (key, val) = node_entry(&tree, i, true)?; + let key_xid = le64(key, 8)?; + if le64(key, 0)? == oid && key_xid <= xid && selected.is_none_or(|v| key_xid > v.0) { + selected = Some((key_xid, le32(val, 0)?, le32(val, 4)?, le64(val, 8)?)); + } + } + let (_, flags, object_size, paddr) = selected.ok_or(ENODATA)?; + if flags & 1 != 0 || object_size != BLOCK as u32 { + return Err(ENOTSUPP); + } + Ok(paddr) +} + +fn xart_extent(file: &shim::StoreFile, size: u64, vol: &[u8], xid: u64) -> Result { + if le64(vol, 264)? & FS_UNENCRYPTED == 0 { + return Err(ENOTSUPP); + } + let omap = le64(vol, 128)?; + let root_oid = le64(vol, 136)?; + let root_paddr = omap_lookup(file, size, omap, root_oid, xid)?; + let root = read_object(file, size, root_paddr)?; + if le64(&root, 8)? != root_oid || le16(&root, 34)? != 0 || le16(&root, 32)? & 4 != 0 { + return Err(ENOTSUPP); + } + + let count = le32(&root, 36)? as usize; + let mut gl_file: Option = None; + for i in 0..count { + let (key, val) = node_entry(&root, i, false)?; + if key.len() < 12 || val.len() < 8 || le64(key, 0)? != ((9u64 << 60) | 2) { + continue; + } + let name_len = (le32(key, 8)? & 0x3ff) as usize; + if name_len == 0 || 12 + name_len > key.len() || key[12 + name_len - 1] != 0 { + return Err(EINVAL); + } + if &key[12..12 + name_len - 1] == b".gl" { + if gl_file.replace(le64(val, 0)?).is_some() { + return Err(ENOTSUPP); + } + } + } + let gl_file = gl_file.ok_or(ENODATA)?; + let mut private_id: Option = None; + for i in 0..count { + let (key, val) = node_entry(&root, i, false)?; + if key.len() >= 8 && le64(key, 0)? == ((3u64 << 60) | gl_file) { + if val.len() < 92 || private_id.replace(le64(val, 8)?).is_some() { + return Err(EINVAL); + } + } + } + let private_id = private_id.ok_or(ENODATA)?; + let mut extent: Option = None; + for i in 0..count { + let (key, val) = node_entry(&root, i, false)?; + if key.len() < 16 || le64(key, 0)? != ((8u64 << 60) | private_id) { + continue; + } + if val.len() < 24 || le64(key, 8)? != 0 || le64(val, 0)? != FILE_SIZE || le64(val, 16)? != 0 + { + return Err(ENOTSUPP); + } + let paddr = le64(val, 8)?; + let start = paddr.checked_mul(BLOCK as u64).ok_or(EINVAL)?; + if start.checked_add(FILE_SIZE).ok_or(EINVAL)? > size || extent.replace(start).is_some() { + return Err(EINVAL); + } + } + extent.ok_or(ENODATA) +} + +/// Return the byte offset of the sole contiguous 6 MiB `.gl` file extent. +pub(crate) fn locate(file: &shim::StoreFile) -> Result { + let device_size = file.size()?; + let base = read_object(file, device_size, 0)?; + if &base[32..36] != b"NXSB" || le32(&base, 36)? != BLOCK as u32 { + return Err(ENOTSUPP); + } + let blocks = le64(&base, 40)?; + if blocks == 0 + || blocks > device_size / BLOCK as u64 + || le64(&base, 56)? != 0 + || le64(&base, 64)? != 2 + { + return Err(ENOTSUPP); + } + let size = blocks.checked_mul(BLOCK as u64).ok_or(EINVAL)?; + let desc_count_raw = le32(&base, 104)?; + if desc_count_raw & (1 << 31) != 0 { + return Err(ENOTSUPP); + } + let desc_count = desc_count_raw as u64; + let desc_base = le64(&base, 112)?; + if desc_count == 0 + || desc_count > 1024 + || desc_base.checked_add(desc_count).ok_or(EINVAL)? > size / BLOCK as u64 + { + return Err(EINVAL); + } + let mut best: Option> = None; + let mut best_xid = 0u64; + for paddr in desc_base..desc_base + desc_count { + let Ok(candidate) = read_object(file, size, paddr) else { + continue; + }; + let candidate_xid = le64(&candidate, 16)?; + if &candidate[32..36] == b"NXSB" + && le32(&candidate, 36)? == BLOCK as u32 + && le64(&candidate, 40)? == blocks + && le64(&candidate, 56)? == 0 + && le64(&candidate, 64)? == 2 + && valid_checkpoint(file, size, &candidate, paddr, desc_base, desc_count).is_ok() + && candidate_xid > best_xid + { + best_xid = candidate_xid; + best = Some(candidate); + } + } + let best = best.ok_or(ENODATA)?; + let xid = le64(&best, 16)?; + let omap = le64(&best, 160)?; + let mut found: Option = None; + for i in 0..100usize { + let oid = le64(&best, 184 + i * 8)?; + if oid == 0 { + continue; + } + let paddr = omap_lookup(file, size, omap, oid, xid)?; + let vol = read_object(file, size, paddr)?; + if &vol[32..36] != b"APSB" || le64(&vol, 8)? != oid || le64(&vol, 16)? > xid { + return Err(EINVAL); + } + if le16(&vol, 964)? == XART_ROLE { + if found + .replace(xart_extent(file, size, &vol, le64(&vol, 16)?)?) + .is_some() + { + return Err(ENOTSUPP); + } + } + } + found.ok_or(ENODATA) +} diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs index 54d289d8a2946..7c73107bc2ae8 100644 --- a/drivers/soc/apple/xart_store.rs +++ b/drivers/soc/apple/xart_store.rs @@ -3,11 +3,10 @@ //! Device-wide xART gigalocker record store. //! -//! The APFS locator exposes the existing `.gl` file as a block device. This -//! module implements the record validation, duplicate repair, lookup and -//! copy-on-write ordering the SEP requires. It never creates storage. -//! The raw block owner is restricted to read-only until APFS metadata can -//! establish the current physical extent of the `.gl` file. +//! The APFS locator identifies the existing `.gl` file's physical extent. +//! This module validates records and serves SEP reads from that extent. It +//! never creates storage. Raw writes remain blocked until the record update +//! protocol and APFS ownership rules are verified against macOS. use crate::shim; use kernel::prelude::*; @@ -20,28 +19,11 @@ const BLOCK_SIZE: usize = 0x1000; const SLOT_SIZE: usize = 0x9000; const HEADER_SIZE: usize = 0x22; const DELETE_SIZE: usize = BLOCK_SIZE; -/// Size of the APFS raw extent located on the target machine. -/// -/// The logical store is always exactly this many bytes. The raw-extent owner -/// opens a larger container and serves only the [`STORE_SIZE`] window at its -/// located base, so a wrong base cannot read past the extent. A wrong window -/// still fails closed: -/// the two root records will not validate and `open_based` returns `ENODATA` -/// rather than serving SEP an unrelated store. A machine with a different extent -/// size must pass that size through an explicit, reviewed interface. +/// The current `.gl` format occupies one contiguous six-MiB APFS file extent. +/// Unsupported layouts fail closed in `xart_apfs::locate`. const STORE_SIZE: u64 = 0x600000; const MAX_SLOTS: usize = 4096; -/// Sequential read window for the automatic gigalocker search (64 KiB — small -/// enough for a reliable kernel allocation, large enough to keep the scan of -/// the container to a modest number of reads). -const SCAN_CHUNK: usize = 1 << 16; -/// Upper bound on candidate bases confirmed during the automatic search, so a -/// container full of coincidental root-shaped headers cannot spin the scan. The -/// real store's roots sit within the first slots, so it is found long before -/// this; exceeding it falls back rather than looping. -const MAX_LOCATE_ATTEMPTS: u32 = 64; - pub(crate) const MAX_VALUE: usize = 0x8000; const KEY_KIND: usize = 0x01; @@ -95,8 +77,7 @@ impl Slot { pub(crate) struct Store { file: shim::StoreFile, - /// Byte offset of the gigalocker window within the opened container: the - /// located extent's offset (an explicit start-sector override sets it). + /// Byte offset of the APFS-resolved gigalocker extent in the container. base: u64, slots: KVec, revision: u64, @@ -129,131 +110,23 @@ fn valid_key(key: &Key) -> bool { } impl Store { - /// Opens the store as the in-kernel raw-extent owner: it opens the iBoot - /// system container directly and serves only the gigalocker extent, with no - /// external helper and no hand-supplied sector. - /// - /// `start_sector == 0` (the default) searches for CRC-checked root records; - /// a non-zero sector skips the search. Neither method proves that the - /// window belongs to the current APFS `.gl` file, so writes are refused. + /// Opens the iBoot system container read-only, resolves `.gl` through APFS + /// metadata, then validates the record store at exactly that extent. + /// A nonzero start sector is an assertion against the APFS result, never + /// an override of it. Neither lookup nor record validation authorizes + /// raw writes: macOS's update protocol is still unverified. pub(crate) fn open_owner(writes_enabled: bool, start_sector: u64) -> Result { - // A CRC-valid record signature does not establish the APFS file's - // physical extent. Once both roots move away from slot zero, several - // shifted windows pass the two-root test, and a shifted window can - // extend into APFS-owned blocks. Never write through this raw owner - // until the mapping has been checked against APFS file metadata. - // An explicit sector is not proof of ownership either. if writes_enabled { return Err(ENOTSUPP); } + let file = shim::StoreFile::open_block(OWNER_PATH, false)?; + let base = crate::xart_apfs::locate(&file)?; if start_sector != 0 { - let base = start_sector.checked_mul(512).ok_or(EINVAL)?; - return Self::open_based(OWNER_PATH, false, base); - } - Self::open_located(writes_enabled) - } - - /// Locates the gigalocker inside the iBoot system container with no external - /// help. The container is read sequentially and searched, at block-aligned - /// offsets, for a root record's signature — a `1` or `2` key kind with an - /// all-zero UUID, the shape of [`Key::root`]. Each base implied by such a hit - /// is probed read-only with the full CRC-checked [`open_based`]; because a - /// base shifted a few slots off the true origin still keeps both roots in its - /// window (passing the two-root test while dropping records that fall - /// outside), the candidate with the most live records is used for read-only - /// diagnostics. This heuristic cannot establish the APFS file's origin or - /// prove it is safe to write. Repair is never armed during discovery. - /// Bounded by [`MAX_LOCATE_ATTEMPTS`] probes; returns `ENODATA` if not found. - fn open_located(writes_enabled: bool) -> Result { - // The container is a writable block device; the block layer only grants - // a read-only handle to a read-only device, so scan it through a writable - // handle. This does not arm any write — discovery never mutates the - // container (see open_based_ext, called below with arm_writes == false). - let probe = shim::StoreFile::open_block(OWNER_PATH, true)?; - let size = probe.size()?; - if size < STORE_SIZE { - return Err(ENODATA); - } - - let mut buf: KVec = KVec::new(); - buf.resize(SCAN_CHUNK, 0, GFP_KERNEL)?; - let mut attempts: u32 = 0; - let mut off: u64 = 0; - - while off + HEADER_SIZE as u64 <= size { - let want = core::cmp::min(SCAN_CHUNK as u64, size - off) as usize; - probe.read_exact(off, &mut buf[..want])?; - - let mut p = 0usize; - while p + HEADER_SIZE <= want { - let kind = buf[p + KEY_KIND]; - let uuid_zero = buf[p + KEY_UUID..p + KEY_UUID + 16].iter().all(|&b| b == 0); - if (kind == 1 || kind == 2) && uuid_zero { - // A root signature anchors the gigalocker: the true base is - // this hit minus a whole number of slots (slots and blocks are - // both 0x1000-aligned, so every candidate stays block-aligned). - // The two-root test alone does NOT pin the base — a base - // shifted off the true origin by a few slots still keeps both - // roots inside its 6 MiB window, so it passes yet silently - // drops the live records that fall outside the shifted window. - // Probe candidates without writes and prefer the largest - // recovered record set for diagnostics. An equally valid - // shifted window is possible; this is not an APFS lookup. - let hit = off + p as u64; - let mut best: Option<(usize, usize, u64)> = None; // (valid, malformed, base) - let mut k: u64 = 0; - while k as usize <= MAX_SLOTS { - let step = k * SLOT_SIZE as u64; - if step > hit { - break; - } - let base = hit - step; - k += 1; - if base + STORE_SIZE > size { - continue; - } - attempts += 1; - if attempts > MAX_LOCATE_ATTEMPTS { - break; - } - if let Ok(store) = Self::open_based_ext(OWNER_PATH, true, false, base) { - let cand = (store.valid_records, store.malformed_records, base); - let better = match best { - None => true, - // More live records wins; ties break to fewer - // malformed, then to the higher base. The tie - // rule is a heuristic and may choose an - // up-shifted base when both roots moved. - Some(b) => { - cand.0 > b.0 - || (cand.0 == b.0 && cand.1 < b.1) - || (cand.0 == b.0 && cand.1 == b.1 && cand.2 > b.2) - } - }; - if better { - best = Some(cand); - } - } - } - // Re-open the diagnostic candidate. open_owner refuses - // writes, including automatic repair, on raw APFS extents. - if let Some((_, _, base)) = best { - return Self::open_based_ext(OWNER_PATH, true, writes_enabled, base); - } - if attempts > MAX_LOCATE_ATTEMPTS { - return Err(ENODATA); - } - } - p += BLOCK_SIZE; - } - - if want < SCAN_CHUNK { - break; + if start_sector.checked_mul(512).ok_or(EINVAL)? != base { + return Err(EINVAL); } - // Overlap one block so a signature on the boundary is not missed. - off += (SCAN_CHUNK - BLOCK_SIZE) as u64; } - Err(ENODATA) + Self::open_file(file, false, base) } /// Opens a caller-selected block mapping at offset zero. @@ -272,27 +145,15 @@ impl Store { /// device (the raw container) serves only its gigalocker extent. /// /// The block handle's writability equals `writes_enabled`, and repair runs - /// when writes are enabled. For the raw-extent owner, discovery instead - /// needs a writable handle (the block layer only grants a read-only handle - /// to a read-only device, and the container is writable) *without* arming - /// repair at an unconfirmed base — see [`open_based_ext`]. + /// when writes are enabled. The APFS owner is always read-only. fn open_based(path: &CStr, writes_enabled: bool, base: u64) -> Result { - Self::open_based_ext(path, writes_enabled, writes_enabled, base) + let file = shim::StoreFile::open_block(path, writes_enabled)?; + Self::open_file(file, writes_enabled, base) } - /// As [`open_based`], but with the block-handle writability (`dev_writable`) - /// decoupled from whether repair writes are armed (`arm_writes`). Discovery - /// of the raw-extent owner opens the writable container with - /// `dev_writable == true` yet `arm_writes == false`, so a candidate base is - /// fully scanned and CRC-validated without a single write landing at an - /// origin that has not yet been confirmed as the true gigalocker. - fn open_based_ext( - path: &CStr, - dev_writable: bool, - arm_writes: bool, - base: u64, - ) -> Result { - let file = shim::StoreFile::open_block(path, dev_writable)?; + /// Complete record validation using the same read-only handle with which + /// APFS ownership was resolved, avoiding a close/reopen window. + fn open_file(file: shim::StoreFile, arm_writes: bool, base: u64) -> Result { let size = file.size()?; if base % BLOCK_SIZE as u64 != 0 || STORE_SIZE % BLOCK_SIZE as u64 != 0 { return Err(EINVAL); @@ -540,8 +401,7 @@ impl Store { Ok(true) } - /// Byte offset of the served gigalocker window within the opened container: - /// the located extent's offset (an explicit start-sector override sets it). + /// Byte offset of the APFS-resolved gigalocker extent in the container. pub(crate) fn base(&self) -> u64 { self.base } diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index c3f17ad3c1171..767b9043d2ad5 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -14,12 +14,27 @@ systemctl enable aurora-sep.service The raw xART owner is currently read-only. Do not enable the installed `aurora-sep.service` or run `load-driver` on a shared macOS/Linux installation: -that script requests `xart_writes=1`, which the kernel now rejects while the -physical `.gl` extent is selected by scanning record signatures rather than -validated against APFS file metadata. The scan is ambiguous when live root -records have moved away from the start of the file. Read-only inspection does -not establish write safety. Re-enable provisioning only after the APFS mapping -and macOS-compatible record update protocol are verified on hardware. +that script requests `xart_writes=1`, which the kernel rejects. The kernel now +resolves the physical `.gl` extent from checksum-validated APFS metadata for +read-only operation; unsupported layouts fail closed. This removes the raw +record-signature locator, which was ambiguous when roots moved away from the +start of the file. APFS lookup alone does not establish write safety. Re-enable +provisioning only after macOS-compatible record updates and APFS ownership +invariants are verified on hardware. + +For read-only structural diagnostics on Linux, run +`sudo python3 apfs-xart-inspect.py /dev/disk/by-partlabel/iBootSystemContainer`. +The inspector checks APFS metadata checksums, resolves the latest complete +checkpoint's xART `.gl` file extent, and compares it with the legacy root-record +heuristic. It prints physical addresses and record metadata, not payloads. +It currently supports only a single-node object map and file-system tree and +cannot authorize raw writes. Apple's APFS reference is the format authority: +https://developer.apple.com/support/apple-file-system/Apple-File-System-Reference.pdf. +`python3 -m unittest discover -s tests -v` runs a synthetic regression case: +when the first live root moves to slot 10, the legacy heuristic chooses a +window shifted 90 APFS blocks beyond the actual `.gl` extent. This proves the +locator can misidentify the extent; it does not identify the historical cause +of any particular damaged container. For fingerprint support, apply `patches/libfprint-1.94.100-apple-sep.patch` to libfprint 1.94.100, build and diff --git a/tools/aurora-sep/XART-SAFETY.md b/tools/aurora-sep/XART-SAFETY.md new file mode 100644 index 0000000000000..1ea3708dc0a2e --- /dev/null +++ b/tools/aurora-sep/XART-SAFETY.md @@ -0,0 +1,41 @@ +# Shared xART gigalocker safety gate + +The iBoot APFS container and its XART `.gl` file are shared with macOS. The +kernel owner must remain read-only. `apple_sep.xart_writes=1` is rejected even +with an explicit `xart_start_sector`; the latter is now only an assertion +against the APFS-resolved file extent. Do not enable provisioning services on a +dual-boot installation while this gate is in place. + +## Evidence collected on 2026-09-21 + +- A read-only inspection of `m2`'s iBoot partition found a complete, valid + checkpoint at transaction 57, one XART volume (role `0x100`), and one + unencrypted `.gl` extent: 6 MiB beginning at APFS block 1236. All 11 live + gigalocker records in that extent passed their payload CRC checks. The raw + signature heuristic happened to select the same block on this snapshot. +- A synthetic image with the first root moved to slot 10 reproduces a legacy + locator failure: the highest-scoring window begins 90 APFS blocks after the + true file start and extends beyond `.gl`. This proves ambiguity in that + locator, **not** the cause of the previously damaged installation. +- Static inspection of the macOS 27.2 AppleSEPManager kernel collection found + `gl_rec_write` allocating a fresh 0x9000-byte slot, writing it, then clearing + 0x1000 bytes at the former slot. The low-level write path also calls a disk + cache-synchronization ioctl. `gl_fixup_rev_init` validates records and + handles duplicates during open. These observations do not prove that our + APFS ownership and failure ordering match macOS. + +## Before authorizing writes + +1. Confirm the exact macOS vnode/raw-device and extent-locking contract, + including whether any APFS copy-on-write, snapshot, encryption, or remap + state can invalidate the physical mapping during Linux's lifetime. +2. Verify record ordering, duplicate selection, failed-write recovery, + barriers/cache synchronization, and revision behavior against both macOS + code and controlled non-production tests. +3. Compare a pre/post macOS-and-Linux power-cycle trace, with APFS checksums, + `.gl` mapping and records checked before and after. Keep a recoverable image + of the iBoot partition before the first write experiment. + +The current APFS locator supports only a complete checkpoint, single-node +object maps and file-system tree, and a single unencrypted 6 MiB `.gl` extent. +Unsupported layouts fail closed; this is not a general APFS implementation. diff --git a/tools/aurora-sep/apfs-xart-inspect.py b/tools/aurora-sep/apfs-xart-inspect.py new file mode 100644 index 0000000000000..63cc276646f5e --- /dev/null +++ b/tools/aurora-sep/apfs-xart-inspect.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +"""Read-only, metadata-only inspector for the iBoot APFS xART volume. + +This intentionally never prints gigalocker contents or attempts a write. +It rejects metadata that it cannot validate instead of guessing an extent. +Only single-node object maps and a single-node xART file-system tree are +supported; this is a diagnostic, not an APFS mount or write authorization. +""" + +import os +import struct +import sys +import zlib + +BLOCK = 4096 +MOD = 0xffffffff + + +def u16(buf, off): + return struct.unpack_from("= self.count: + raise ValueError("physical address out of partition") + data = os.pread(self.fd, BLOCK, paddr * BLOCK) + if len(data) != BLOCK: + raise ValueError("short block read") + if verify and u64(data, 0) != checksum(data): + raise ValueError("APFS checksum mismatch at block %d" % paddr) + return data + + +def node_entries(buf): + level, flags, count = u16(buf, 34), u16(buf, 32), u32(buf, 36) + fixed = bool(flags & 4) + start = 56 + u16(buf, 40) + key_start = start + u16(buf, 42) + val_end = BLOCK - (40 if flags & 1 else 0) + stride = 4 if fixed else 8 + if count > 512 or start < 56 or start + stride * count > key_start or key_start > val_end: + raise ValueError("invalid B-tree table") + entries = [] + for i in range(count): + at = start + i * stride + if fixed: + koff, voff = u16(buf, at), u16(buf, at + 2) + klen, vlen = 16, 16 + else: + koff, klen, voff, vlen = struct.unpack_from("= desc_count or not 2 <= length <= desc_count: + return False + if desc_base + (index + length - 1) % desc_count != sb_paddr: + return False + xid = u64(sb, 16) + try: + for j in range(length - 1): + mapping = disk.read(desc_base + (index + j) % desc_count) + if u32(mapping, 24) & 0xffff != 12 or u64(mapping, 16) != xid: + return False + if bool(u32(mapping, 32) & 1) != (j == length - 2): + return False + count = u32(mapping, 36) + if count > (BLOCK - 40) // 40: + return False + for i in range(count): + off = 40 + i * 40 + if u32(mapping, off + 8) != BLOCK: + return False + item = disk.read(u64(mapping, off + 32)) + if (u64(item, 8) != u64(mapping, off + 24) or + u64(item, 16) != xid or + u32(item, 24) != u32(mapping, off)): + return False + except ValueError: + return False + return True + + +def omap_lookup(disk, omap_paddr, oid, max_xid): + omap = disk.read(omap_paddr) + if u64(omap, 8) != omap_paddr: + raise ValueError("object-map identity mismatch") + tree = u64(omap, 48) + node = disk.read(tree) + level, flags, entries = node_entries(node) + if level: + raise ValueError("multilevel object map needs further inspection") + if not flags & 4: + raise ValueError("object map is not fixed-size") + matches = [] + for key, val in entries: + if u64(key, 0) == oid and u64(key, 8) <= max_xid: + matches.append((u64(key, 8), u32(val, 0), u64(val, 8), u32(val, 4))) + if not matches: + raise ValueError("object %d not found in map" % oid) + selected = max(matches) + if selected[1] & 1: + raise ValueError("object %d deleted at selected transaction" % oid) + return selected[2:] + + +def inspect(disk): + base = disk.read(0) + if base[32:36] != b"NXSB" or u32(base, 36) != BLOCK: + raise ValueError("unsupported container superblock") + desc_base, desc_count = u64(base, 112), u32(base, 104) + if desc_count < 1 or desc_count > 1024: + raise ValueError("unsupported checkpoint descriptor geometry") + checkpoints = [] + for paddr in range(desc_base, desc_base + desc_count): + try: + block = disk.read(paddr) + except ValueError: + continue + if (block[32:36] == b"NXSB" and u32(block, 36) == BLOCK and + checkpoint_complete(disk, block, paddr, desc_base, desc_count)): + checkpoints.append((u64(block, 16), paddr, block)) + if not checkpoints: + raise ValueError("no valid container superblock") + xid, cpaddr, sb = max(checkpoints) + print("partition_bytes=%d blocks=%d" % (disk.size, disk.count)) + print("block_zero_xid=%d" % u64(base, 16)) + print("valid_superblocks=%s" % [(x, p) for x, p, _ in checkpoints]) + print("selected_checkpoint_xid=%d block=%d" % (xid, cpaddr)) + print("container_omap=%d" % u64(sb, 160)) + fs_oids = [u64(sb, 184 + i * 8) for i in range(100)] + fs_oids = [v for v in fs_oids if v] + print("volume_oids=%s" % fs_oids) + for oid in fs_oids: + paddr, size = omap_lookup(disk, u64(sb, 160), oid, xid) + vol = disk.read(paddr) + if vol[32:36] != b"APSB" or u64(vol, 8) != oid: + raise ValueError("bad volume superblock") + role = u16(vol, 964) + print("volume oid=%d block=%d xid=%d role=0x%x" % + (oid, paddr, u64(vol, 16), role)) + if role != 0x100: + continue + vol_omap = u64(vol, 128) + root_oid = u64(vol, 136) + root_paddr, root_size = omap_lookup(disk, vol_omap, root_oid, xid) + root = disk.read(root_paddr) + level, flags, entries = node_entries(root) + print("xart_root oid=%d block=%d level=%d records=%d" % + (root_oid, root_paddr, level, len(entries))) + if level: + raise ValueError("multilevel xART FS tree needs further inspection") + files = [] + for key, val in entries: + if len(key) < 12 or len(val) < 8: + continue + k = u64(key, 0) + if k == ((9 << 60) | 2): + name_len = u32(key, 8) & 0x3ff + if name_len and 12 + name_len <= len(key): + name = key[12:12 + name_len - 1] + if name == b".gl": + files.append(u64(val, 0)) + print("gigalocker_file_ids=%s" % files) + if len(files) != 1: + raise ValueError("expected exactly one .gl file") + inode_values = [value for key, value in entries + if len(key) >= 8 and u64(key, 0) == ((3 << 60) | files[0])] + if len(inode_values) != 1 or len(inode_values[0]) < 92: + raise ValueError(".gl inode missing or short") + inode = inode_values[0] + private_id, inode_flags = u64(inode, 8), u64(inode, 48) + print("gigalocker_inode_private_id=%d flags=0x%x links=%d write_generation=%d" % + (private_id, inode_flags, u32(inode, 56), u32(inode, 64))) + print("xart_volume_flags=0x%x snapshot_count=%d" % (u64(vol, 264), u32(disk.read(vol_omap), 36))) + extents = [] + for key, val in entries: + if len(key) >= 16 and len(val) >= 24 and u64(key, 0) == ((8 << 60) | private_id): + logical, length_flags, physical = u64(key, 8), u64(val, 0), u64(val, 8) + length = length_flags & ((1 << 56) - 1) + if logical % BLOCK or length % BLOCK or physical + length // BLOCK > disk.count: + raise ValueError("invalid .gl extent") + extents.append((logical, length, physical, length_flags >> 56, u64(val, 16))) + extents.sort() + print("gigalocker_extents_logical_bytes_physical_blocks_flags=%s" % extents) + if not extents: + raise ValueError("no .gl extents") + if len(extents) == 1 and extents[0][0] == 0 and extents[0][1] == 0x600000: + inspect_raw_locator(disk, extents[0][2]) + scan_other_roots(disk, extents[0][2], extents[0][1]) + + +def inspect_raw_locator(disk, gl_block): + """Mirror the removed first-root candidate scoring, without writes.""" + slot_size, store_size = 0x9000, 0x600000 + gl_base = gl_block * BLOCK + gl = os.pread(disk.fd, store_size, gl_base) + if len(gl) != store_size: + raise ValueError("short gigalocker extent read") + roots = [] + records = [] + for idx in range(store_size // slot_size): + off = idx * slot_size + if gl[off + 1] in (1, 2) and gl[off + 2:off + 18] == bytes(16): + roots.append(idx) + kind = gl[off + 1] + if not kind: + continue + length = u32(gl, off + 0x12) + valid = (1 <= kind <= 4 and (kind > 2 or gl[off + 2:off + 18] == bytes(16)) + and 1 <= length <= 0x8000 and + zlib.crc32(gl[off + 0x22:off + 0x22 + length]) == u32(gl, off + 0x16)) + records.append((idx, kind, length, u64(gl, off + 0x1a), valid)) + print("gigalocker_records_slot_kind_length_revision_crc_valid=%s" % records) + print("raw_root_signature_slots=%s" % roots) + if not roots: + return + hit = gl_base + roots[0] * slot_size + candidates = [] + attempts = 0 + for k in range(4097): + step = k * slot_size + if step > hit: + break + base = hit - step + if base + store_size > disk.size: + continue + attempts += 1 + if attempts > 64: + break + raw = os.pread(disk.fd, store_size, base) + live = {} + malformed = 0 + for idx in range(store_size // slot_size): + off = idx * slot_size + kind = raw[off + 1] + if kind == 0: + continue + uid = raw[off + 2:off + 18] + length = u32(raw, off + 0x12) + crc = u32(raw, off + 0x16) + revision = u64(raw, off + 0x1a) + if not (1 <= kind <= 4 and (kind > 2 or uid == bytes(16)) and + 1 <= length <= 0x8000 and + zlib.crc32(raw[off + 0x22:off + 0x22 + length]) == crc): + malformed += 1 + continue + key = (kind, uid) + if key not in live or revision >= live[key]: + live[key] = revision + if (1, bytes(16)) in live and (2, bytes(16)) in live: + candidates.append((len(live), -malformed, base)) + if not candidates: + print("raw_locator=no_valid_candidate") + return + best = max(candidates) + print("raw_locator_candidates=%d attempts=%d" % (len(candidates), attempts)) + print("raw_locator_best_block=%d apfs_extent_block=%d shift_blocks=%d live_records=%d malformed=%d" % + (best[2] // BLOCK, gl_block, (best[2] - gl_base) // BLOCK, best[0], -best[1])) + + +def scan_other_roots(disk, gl_block, gl_size): + """Count CRC-valid root signatures outside APFS's current file extent.""" + outside = [] + for block in range(disk.count): + off = block * BLOCK + header = os.pread(disk.fd, 0x22, off) + if len(header) != 0x22 or header[1] not in (1, 2) or header[2:18] != bytes(16): + continue + length = u32(header, 0x12) + if not 1 <= length <= 0x8000: + continue + payload = os.pread(disk.fd, length, off + 0x22) + if len(payload) != length or zlib.crc32(payload) != u32(header, 0x16): + continue + if not gl_block <= block < gl_block + gl_size // BLOCK: + outside.append((block, header[1], u64(header, 0x1a))) + print("crc_valid_root_signatures_outside_apfs_gl=%s" % outside[:32]) + if len(outside) > 32: + print("crc_valid_root_signatures_outside_apfs_gl_additional=%d" % (len(outside) - 32)) + + +if __name__ == "__main__": + try: + inspect(Disk(sys.argv[1])) + except (ValueError, OSError, IndexError) as exc: + print("INCOMPLETE: %s" % exc, file=sys.stderr) + sys.exit(1) diff --git a/tools/aurora-sep/tests/test_xart_locator.py b/tools/aurora-sep/tests/test_xart_locator.py new file mode 100644 index 0000000000000..167c977ec2bd0 --- /dev/null +++ b/tools/aurora-sep/tests/test_xart_locator.py @@ -0,0 +1,49 @@ +"""Regression proof for the ambiguity of the legacy raw-header locator.""" + +import contextlib +import importlib.util +import io +from pathlib import Path +import struct +import tempfile +import unittest +import zlib + + +SCRIPT = Path(__file__).resolve().parents[1] / "apfs-xart-inspect.py" +SPEC = importlib.util.spec_from_file_location("apfs_xart_inspect", SCRIPT) +INSPECTOR = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(INSPECTOR) + + +def record(kind, revision): + out = bytearray(0x9000) + payload = b"test" + out[1] = kind + struct.pack_into(" Date: Fri, 25 Sep 2026 17:03:59 -0400 Subject: [PATCH 0717/2680] soc: apple: sep: port J414s enrollment and xART persistence fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Carry only the SEP-related part of 95731b2390, including the SEP device-tree alias, calibration tooling, and fingerprint protocol changes. Exclude Thunderbolt and display modifications from this branch. [ryan: adapted to 7.2.10 — split by topic: the t602x-j414-j416.dtsi sep alias is carried by 000-devicetree; the lines this commit changes keep the 7.2 lifetime-bound driver-core form] [ryan: unified assembly: the gigalocker file name in the locator test is replaced with the obviously synthetic UUID 01234567-89AB-5CDE-8F01-23456789ABCD; the test checks only the name format, and the original value could not be shown to be synthetic] Source-Commit: 95731b2390 Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 11 +- drivers/soc/apple/bio.rs | 27 +- drivers/soc/apple/sbio.rs | 550 ++++++++++++++---- drivers/soc/apple/sep-bio.h | 3 + drivers/soc/apple/sep.rs | 143 +++-- drivers/soc/apple/xart_apfs.rs | 99 +++- drivers/soc/apple/xart_store.rs | 26 +- tools/aurora-sep/README.md | 88 ++- tools/aurora-sep/XART-SAFETY.md | 48 +- tools/aurora-sep/apfs-xart-inspect.py | 65 ++- tools/aurora-sep/extract-mesa-calibration.py | 343 +++++++++++ tools/aurora-sep/load-driver | 8 +- .../libfprint-1.94.100-apple-sep.patch | 26 +- .../tests/test_extract_mesa_calibration.py | 72 +++ tools/aurora-sep/tests/test_xart_locator.py | 40 ++ 15 files changed, 1357 insertions(+), 192 deletions(-) create mode 100644 tools/aurora-sep/extract-mesa-calibration.py create mode 100644 tools/aurora-sep/tests/test_extract_mesa_calibration.py diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 0a295fe8e29ac..f9c7498abd593 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -133,10 +133,13 @@ config APPLE_SEP SEP-backed trusted key source so keyctl can seal keys to the enclave. The driver reads the existing machine-wide xART gigalocker directly from - the iBoot system container, locating it by its root records. It validates - the complete store and both root records before registering with SEP. - Raw APFS xART writes are refused until the extent and update protocol - can be verified; the module parameter cannot override this safety gate. + the iBoot system container, locating its .gl extent through read-only, + checksum-validated APFS metadata. It validates the complete store and + both root records before registering with SEP. + Raw APFS xART writes are disabled by default. An explicit xart_writes + opt-in is accepted only when the driver validates narrow physical + extent ownership invariants; this does not prove all macOS-compatible + update behavior. Keep it off until the target machine has been checked. Say Y here if you have an Apple silicon Mac. diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index e66710b6935cf..2bc1a492799c3 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -49,7 +49,9 @@ enum MatchResult { NotCompared = 2, } -pub(crate) const ENROL_STAGES: u32 = 8; +// A stage is an accepted capture, not a bucket of SEP's coverage percentage. +// Leave the final stage for the completed template. +pub(crate) const ENROL_STAGES: u32 = 16; const TOKEN_LIFETIME_NS: u64 = 10 * 1_000_000_000; @@ -180,7 +182,7 @@ ioctl_pod! { const MAGIC: u32 = 0xB1; -const IOC_GET_INFO: u32 = _IOR::(MAGIC, 0x01); +pub(crate) const IOC_GET_INFO: u32 = _IOR::(MAGIC, 0x01); const IOC_LIST: u32 = _IOR::(MAGIC, 0x02); const IOC_ENROL_START: u32 = _IOW::(MAGIC, 0x03); const IOC_ENROL_POLL: u32 = _IOR::(MAGIC, 0x04); @@ -414,6 +416,7 @@ pub(crate) struct Session { open: bool, op: Op, unseen: bool, + enrol_stage_unseen: bool, token: Option, } @@ -423,6 +426,7 @@ impl Session { open: false, op: Op::Idle, unseen: false, + enrol_stage_unseen: false, token: None, } } @@ -431,6 +435,7 @@ impl Session { self.open = false; self.op = Op::Idle; self.unseen = false; + self.enrol_stage_unseen = false; self.token = None; } @@ -443,6 +448,7 @@ pub(crate) struct Context<'a> { pub(crate) session: &'a mut Session, pub(crate) index: &'a IdentityIndex, pub(crate) sensor_present: bool, + pub(crate) live_identity_count: Option, } pub(crate) struct Handled { @@ -505,7 +511,14 @@ fn get_info(ctx: &mut Context<'_>, user: UserPtr) -> Result { enrolled: ctx.index.total() as u32, capacity: MAX_IDENTITIES as u32, enroll_stages: ENROL_STAGES, - reserved: [0; 3], + // reserved[0] is a validity flag. A zero count is only meaningful + // when SEP itself successfully answered LIST_IDENTITIES; older + // kernels and unavailable SEP contexts leave this flag clear. + reserved: [ + u32::from(ctx.live_identity_count.is_some()), + ctx.live_identity_count.unwrap_or(0) as u32, + 0, + ], }; UserSlice::new(user, core::mem::size_of::()) .writer() @@ -557,6 +570,8 @@ fn enrol_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { guidance: Guidance::Place, percent: 0, }; + ctx.session.unseen = false; + ctx.session.enrol_stage_unseen = false; Ok(Handled { ret: 0, wake: false, @@ -586,6 +601,7 @@ pub(crate) fn enrol_advance( *percent = percent_now; *guidance = guide; session.unseen = true; + session.enrol_stage_unseen = true; return true; } } @@ -661,7 +677,9 @@ fn enrol_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { out.progress_percent = *percent; match terminal { None => { - out.state = if ctx.session.unseen { + // Guidance also wakes the poller, but it is not an image + // capture or a completed enrollment stage. + out.state = if ctx.session.enrol_stage_unseen { State::Progress } else { State::Pending @@ -683,6 +701,7 @@ fn enrol_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { .writer() .write(&out)?; ctx.session.unseen = false; + ctx.session.enrol_stage_unseen = false; ok() } diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 99665791198b6..91e5125b55acb 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -10,6 +10,24 @@ use kernel::prelude::*; use kernel::soc::apple::mailbox::Message; impl SepData { + fn sbio_enrol_step(&self, op: &crate::sbio::SbioOp) -> Option> { + match self.sbio_call(op) { + SbioOutcome::Ok(payload) => Some(payload), + SbioOutcome::PrerequisiteMissing => { + dev_err!(self.dev, "enrol: {} prerequisite missing (status 0x01)\n", op.name()); + None + } + SbioOutcome::Status16 => { + dev_err!(self.dev, "enrol: {} refused (status 0x16)\n", op.name()); + None + } + SbioOutcome::Other => { + dev_err!(self.dev, "enrol: {} got no usable answer\n", op.name()); + None + } + } + } + fn sbio_expect_ok(&self, op: &crate::sbio::SbioOp) -> Option> { match self.sbio_call(op) { SbioOutcome::Ok(payload) => Some(payload), @@ -40,8 +58,14 @@ impl SepData { fn sbio_relay(&self, relay: &crate::sbio::SbioRelay<'_>) -> Option> { match self.sbio_transfer_raw(relay.opcode(), relay.name(), relay.payload()) { Ok(done) if done.status.is_ok() => Some(done.payload), - Ok(_) => None, - Err(_) => None, + Ok(done) => { + dev_err!(self.dev, "enrol: {} relay ({} bytes) refused with status {}\n", relay.name(), relay.payload().len(), done.status); + None + } + Err(e) => { + dev_err!(self.dev, "enrol: {} relay ({} bytes) failed ({:?})\n", relay.name(), relay.payload().len(), e); + None + } } } @@ -264,6 +288,7 @@ impl SepData { let listed = match self.enclave_identities_for(SBIO_PROBE_USER_ID) { Some(list) => list.len(), None => { + dev_warn!(self.dev, "enrol: identity enumeration failed; trying the existing context\n"); return self.enrol_into_existing_context(user); } }; @@ -288,10 +313,17 @@ impl SepData { proof: &crate::sbio::NoExistingCatacomb, ) -> bool { let system = crate::sbio::sbio_select_context(crate::sbio::ContextScope::SYSTEM, proof); - if self.sbio_expect_ok(&system).is_none() { + if self.sbio_enrol_step(&system).is_none() { return false; } + let Some(reply) = self.read_component_states() else { + dev_warn!(self.dev, "enrol: could not read component state after system SELECT_CONTEXT\n"); + return false; + }; + let states = crate::sbio::ComponentStates::new(&reply); + let owner_initial_state = states.state_for(crate::sbio::CatacombUser::OWNER.value()); + for (who, kind, what) in [ ( crate::sbio::CatacombUser::MASTER, @@ -304,14 +336,52 @@ impl SepData { c"owner catacomb", ), ] { + let Some(state) = states.state_for(who.value()) else { + dev_err!(self.dev, "enrol: {} is absent after system SELECT_CONTEXT\n", what); + return false; + }; + dev_info!(self.dev, "enrol: after system SELECT_CONTEXT {} state 0x{:x}\n", what, state); + if state & crate::sbio::COMPONENT_STATE_ACTIVE == 0 { + dev_err!(self.dev, "enrol: {} is not active after system SELECT_CONTEXT\n", what); + return false; + } + // On m2, state 0x3 refused SAVE_CATACOMB with status 0x6. The + // 0x04 save-pending bit is also used by the T2 Catacomb protocol; + // only persist a fresh component when SEP reports that bit. + if state & crate::sbio::COMPONENT_STATE_SAVE_PENDING == 0 { + continue; + } if !self.save_catacomb(who, kind, what) { + dev_err!(self.dev, "enrol: could not save {} after selecting the system context\n", what); + return false; + } + } + + // The J414s owner exported successfully from state 0x3 in a bounded + // no-finger probe, although an earlier pre-UUID boot saw the master + // refuse an early save from the same state. Keep this path opt-in + // until a new enrollment and reboot prove that the owner blob belongs + // to the new user context. + if *module_parameters::j414s_persistent_enrol.value() != 0 { + if owner_initial_state == Some(0x3) + && !self.save_catacomb( + crate::sbio::CatacombUser::OWNER, + PRIVATE_TYPE_CATACOMB_OWNER, + c"fresh-context owner catacomb (user 501)", + ) + { + dev_err!(self.dev, "enrol: opt-in fresh-context owner export failed\n"); + return false; + } + if crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_none() { + dev_err!(self.dev, "enrol: opt-in fresh context has no owner file\n"); return false; } } let per_user = crate::sbio::sbio_select_context(crate::sbio::ContextScope::user(user), proof); - if self.sbio_expect_ok(&per_user).is_none() { + if self.sbio_enrol_step(&per_user).is_none() { return false; } @@ -324,10 +394,11 @@ impl SepData { fn activate_protected_config(&self, user: crate::sbio::UserId) -> bool { let op = crate::sbio::sbio_protected_config(user); - let Some(config) = self.sbio_expect_ok(&op) else { + let Some(config) = self.sbio_enrol_step(&op) else { return false; }; if config.len() != crate::sbio::SBIO_PROTECTED_CONFIG_LEN { + dev_err!(self.dev, "enrol: protected configuration has {} bytes, expected {}\n", config.len(), crate::sbio::SBIO_PROTECTED_CONFIG_LEN); return false; } true @@ -339,19 +410,25 @@ impl SepData { let _ = self.sbio_call(&crate::sbio::sbio_cancel_operation()); } - let user = crate::sbio::UserId::new(SBIO_PROBE_USER_ID)?; + let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + dev_err!(self.dev, "enrol: invalid probe user id\n"); + return None; + }; // 0x03 answers -3 until a user key bag is designated if !self.keybag_designated.load(Relaxed) { + dev_err!(self.dev, "enrol: no user keybag was designated\n"); return None; } if self.enrol_material.lock().is_none() { + dev_err!(self.dev, "enrol: no enrolment material is held\n"); return None; } // 0x03 answers 0x1 until an enrolment context exists if !self.open_enrolment_context(user) { + dev_err!(self.dev, "enrol: could not open the enclave enrolment context\n"); return None; } @@ -365,8 +442,8 @@ impl SepData { let op = crate::sbio::sbio_begin_enrol(user, crate::sbio::BE_AUTH_TYPE_ACM_CONTEXT, &acm_handle); - match self.sbio_call(&op) { - SbioOutcome::Ok(_payload) => { + match self.sbio_enrol_step(&op) { + Some(_payload) => { self.enrol_open.store(true, Relaxed); Some(OpenEnrolment { sep: self, @@ -377,6 +454,50 @@ impl SepData { } } + fn probe_owner_export(&self) { + if *module_parameters::probe_owner_export.value() == 0 { + return; + } + if self.templates_restored.load(Relaxed) + || crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_some() + || crate::catacomb::read(PRIVATE_TYPE_CATACOMB_MASTER).is_none() + || crate::catacomb::read(PRIVATE_TYPE_CATACOMB_USER).is_none() + { + dev_warn!(self.dev, "owner-probe: preconditions not met; no context selected\n"); + return; + } + let Some(identities) = self.enclave_identities_for(SBIO_PROBE_USER_ID) else { + dev_warn!(self.dev, "owner-probe: identity count unavailable; no context selected\n"); + return; + }; + let Some(proof) = crate::sbio::NoExistingCatacomb::from_zero_identities(identities.len()) else { + dev_warn!(self.dev, "owner-probe: enclave identity exists; no context selected\n"); + return; + }; + + // This is destructive to the enclave's *volatile* current context. + // It is opt-in and only reached after proving zero live identities. + let system = crate::sbio::sbio_select_context(crate::sbio::ContextScope::SYSTEM, &proof); + if self.sbio_expect_ok(&system).is_none() { + dev_warn!(self.dev, "owner-probe: system context selection refused\n"); + return; + } + let state = self.read_component_states().and_then(|reply| { + crate::sbio::ComponentStates::new(&reply) + .state_for(crate::sbio::CatacombUser::OWNER.value()) + }); + dev_info!(self.dev, "owner-probe: selected owner component state {:?}\n", state); + if !matches!(state, Some(value) if value & crate::sbio::COMPONENT_STATE_ACTIVE != 0) { + return; + } + let saved = self.save_catacomb( + crate::sbio::CatacombUser::OWNER, + PRIVATE_TYPE_CATACOMB_OWNER, + c"owner catacomb (user 501) diagnostic", + ); + dev_warn!(self.dev, "owner-probe: owner SAVE_CATACOMB and CONFIRM_SAVE completed {}\n", saved); + } + fn restore_all_components(&self) -> bool { // 0x6b is re-read before every component, not once up front. let user_id = SBIO_PROBE_USER_ID; @@ -384,27 +505,34 @@ impl SepData { let mut missing_files = false; let mut user_outcome = RestoreOutcome::NoStoredFile; - // 0x8002 (cold transition) is a success only on the owner component. - for (id, kind, what, cold_ok) in [ + // A 0x8002 reply is not sufficient evidence of restoration. Re-read + // the component state for every load, including the user component. + for (id, kind, what) in [ ( crate::sbio::CatacombUser::MASTER.value(), PRIVATE_TYPE_CATACOMB_MASTER, c"master catacomb", - false, ), ( crate::sbio::CatacombUser::OWNER.value(), PRIVATE_TYPE_CATACOMB_OWNER, c"owner catacomb", - true, ), - (user_id, PRIVATE_TYPE_CATACOMB_USER, c"user catacomb", false), + (user_id, PRIVATE_TYPE_CATACOMB_USER, c"user catacomb"), ] { let Some(reply) = self.read_component_states() else { return false; }; let states = crate::sbio::ComponentStates::new(&reply); - let outcome = self.restore_catacomb(states.state_for(id), id, kind, what, cold_ok); + let state = states.state_for(id); + let outcome = self.restore_catacomb(state, id, kind, what); + dev_info!( + self.dev, + "sbio: restore {} state {:?} outcome {:?}\n", + what, + state, + outcome + ); match outcome { RestoreOutcome::Restored | RestoreOutcome::AlreadyActive => any = true, RestoreOutcome::NoStoredFile => missing_files = true, @@ -419,7 +547,9 @@ impl SepData { user_outcome, RestoreOutcome::Restored | RestoreOutcome::AlreadyActive ); - let lockout = match self.restore_lockout() { + let lockout_outcome = self.restore_lockout(); + dev_info!(self.dev, "sbio: restore lockout outcome {:?}\n", lockout_outcome); + let lockout = match lockout_outcome { RestoreOutcome::Restored | RestoreOutcome::AlreadyActive => true, RestoreOutcome::EmptyTolerated => true, RestoreOutcome::NoStoredFile => { @@ -432,7 +562,7 @@ impl SepData { if missing_files { dev_warn!( self.dev, - "sbio: at least one of four artefacts has no file on disk; if enrolled before the four-artefact save, re-enrol once to write all four\n" + "sbio: at least one catacomb/lockout file is absent; match readiness depends on the user restore and device-view proof\n" ); } @@ -459,7 +589,6 @@ impl SepData { id: i32, kind: u8, what: &CStr, - cold_ok: bool, ) -> RestoreOutcome { let Some(state) = state else { return RestoreOutcome::Failed; @@ -490,18 +619,56 @@ impl SepData { Ok(done) if done.status.is_ok() => { self.confirm_active(id, what, LoadAnswer::StatusZero) } - Ok(done) - if done.status.answered() - == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) - && cold_ok => - { - self.confirm_active(id, what, LoadAnswer::ColdTransition) - } Ok(done) if done.status.answered() == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) => { - RestoreOutcome::Failed + let active = self.confirm_active(id, what, LoadAnswer::ColdTransition); + if id != SBIO_PROBE_USER_ID + || !matches!(&active, RestoreOutcome::Restored) + { + return active; + } + match self.log_identity_count(c"after first user cold-transition load") { + Some(n) if n > 0 => return active, + Some(0) => {} + _ => return RestoreOutcome::Failed, + } + + // Diagnostic only: one idempotent retry of the same + // host blob, with no context reset or save. An active + // state by itself did not prove a live identity on J414s. + let second = self.sbio_transfer_raw( + request.opcode(), request.name(), request.payload() + ); + let answer = match second { + Ok(done) if done.status.is_ok() => LoadAnswer::StatusZero, + Ok(done) + if done.status.answered() + == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) => + { + LoadAnswer::ColdTransition + } + Ok(done) + if done.status.answered() + == Some(crate::sbio::SBIO_STATUS_ALREADY_ACTIVE) => + { + LoadAnswer::AlreadyActive + } + Ok(done) => { + dev_warn!(self.dev, "sbio: second user LOAD_CATACOMB refused with status {}\n", done.status); + return RestoreOutcome::Failed; + } + Err(e) => { + dev_warn!(self.dev, "sbio: second user LOAD_CATACOMB transfer failed ({:?})\n", e); + return RestoreOutcome::Failed; + } + }; + let active = self.confirm_active(id, what, answer); + match (active, self.log_identity_count(c"after second user cold-transition load")) { + (RestoreOutcome::Restored, Some(n)) if n > 0 => RestoreOutcome::Restored, + _ => RestoreOutcome::Failed, + } } // 0x101 ALREADY_ACTIVE is a success Ok(done) @@ -510,7 +677,14 @@ impl SepData { { self.confirm_active(id, what, LoadAnswer::AlreadyActive) } - _ => RestoreOutcome::Failed, + Ok(done) => { + dev_warn!(self.dev, "sbio: {} LOAD_CATACOMB refused with status {}\n", what, done.status); + RestoreOutcome::Failed + } + Err(e) => { + dev_warn!(self.dev, "sbio: {} LOAD_CATACOMB transfer failed ({:?})\n", what, e); + RestoreOutcome::Failed + } } } } @@ -528,7 +702,9 @@ impl SepData { }; let states = crate::sbio::ComponentStates::new(&reply); - match states.state_for(id) { + let state = states.state_for(id); + dev_info!(self.dev, "sbio: {} LOAD_CATACOMB answered {} with post-load state {:?}\n", what, answer.describe(), state); + match state { Some(state) if state & crate::sbio::COMPONENT_STATE_ACTIVE != 0 => { RestoreOutcome::Restored } @@ -575,7 +751,14 @@ impl SepData { } fn save_all_components(&self, user: crate::sbio::UserId) -> bool { - let mut all = true; + if *module_parameters::j414s_persistent_enrol.value() != 0 { + return self.save_all_components_user_first(user); + } + // On m2fix11, a completed enrollment marked both master and user 0x7, + // but saving and confirming the user first was followed by master + // SAVE_CATACOMB status 0x6. Try the master before the user at this + // completion gate; post-match persistence keeps its separate order. + // Re-read state before each save and never send an unconditional one. for (who, kind, what) in [ ( crate::sbio::CatacombUser::MASTER, @@ -593,47 +776,144 @@ impl SepData { c"user catacomb", ), ] { - if !self.save_catacomb(who, kind, what) { - all = false; + let Some(reply) = self.read_component_states() else { + dev_err!(self.dev, "enrol: cannot read component state before saving {}\n", what); + return false; + }; + let states = crate::sbio::ComponentStates::new(&reply); + let Some(state) = states.state_for(who.value()) else { + dev_err!(self.dev, "enrol: {} is absent at enrollment completion\n", what); + return false; + }; + dev_info!(self.dev, "enrol: completed {} state 0x{:x}\n", what, state); + if state & crate::sbio::COMPONENT_STATE_ACTIVE == 0 { + dev_err!(self.dev, "enrol: {} is not active at enrollment completion\n", what); + return false; + } + if state & crate::sbio::COMPONENT_STATE_SAVE_PENDING != 0 { + if !self.save_catacomb(who, kind, what) { + return false; + } + } else if who.value() == user.value() { + dev_err!(self.dev, "enrol: completed user catacomb is not marked for saving\n"); + return false; + } + if (who.value() == user.value() + || who.value() == crate::sbio::CatacombUser::MASTER.value()) + && crate::catacomb::read(kind).is_none() + { + dev_err!(self.dev, "enrol: {} has no durable host file\n", what); + return false; } } if !self.save_lockout() { - all = false; + dev_err!(self.dev, "enrol: lockout persistence failed\n"); + return false; } - all + true } - fn save_after_match(&self, user: crate::sbio::UserId) -> bool { + fn save_all_components_user_first(&self, user: crate::sbio::UserId) -> bool { + if crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_none() { + dev_err!(self.dev, "enrol: opt-in completion has no owner Catacomb file\n"); + return false; + } + // Post-match persistence saves the user before the master because the + // user save can change system material. Test that order for a completed + // J414s enrollment, without changing the default path on other Macs. + for (who, kind, what) in [ + ( + crate::sbio::CatacombUser::enrolling(user), + PRIVATE_TYPE_CATACOMB_USER, + c"opt-in completed user catacomb", + ), + ( + crate::sbio::CatacombUser::MASTER, + PRIVATE_TYPE_CATACOMB_MASTER, + c"opt-in post-user master catacomb", + ), + ] { + let Some(reply) = self.read_component_states() else { + dev_err!(self.dev, "enrol: opt-in cannot read state before saving {}\n", what); + return false; + }; + let state = crate::sbio::ComponentStates::new(&reply).state_for(who.value()); + dev_info!(self.dev, "enrol: opt-in {} state {:?}\n", what, state); + let Some(state) = state else { + return false; + }; + if state & crate::sbio::COMPONENT_STATE_ACTIVE == 0 { + return false; + } + if who.value() == user.value() + && state & crate::sbio::COMPONENT_STATE_SAVE_PENDING == 0 + { + dev_err!(self.dev, "enrol: opt-in user catacomb is not save-pending\n"); + return false; + } + if !self.save_catacomb(who, kind, what) { + return false; + } + if crate::catacomb::read(kind).is_none() { + dev_err!(self.dev, "enrol: opt-in {} has no durable host file\n", what); + return false; + } + } if !self.save_lockout() { - dev_err!( - self.dev, - "verify: post-match lockout persistence failed; refusing to claim the AP and SEP anti-replay state are synchronized\n" - ); + dev_err!(self.dev, "enrol: opt-in lockout persistence failed\n"); return false; } + true + } - if !self.save_catacomb( - crate::sbio::CatacombUser::enrolling(user), - PRIVATE_TYPE_CATACOMB_USER, - c"post-match user catacomb", - ) { + fn save_after_match(&self, user: crate::sbio::UserId) -> bool { + if !self.save_lockout() { dev_err!( self.dev, - "verify: post-match user-catacomb persistence failed; the next boot may reject the older AP blob\n" + "verify: post-match lockout persistence failed; refusing to claim the AP and SEP anti-replay state are synchronized\n" ); return false; } - if !self.save_catacomb( - crate::sbio::CatacombUser::MASTER, - PRIVATE_TYPE_CATACOMB_MASTER, - c"post-match system (id -1) material snapshot", - ) { - dev_err!( - self.dev, - "verify: post-match system material snapshot failed; next boot may reinstall pre-match material and the user load could cold-transition (0x8002)\n" - ); - return false; + // Matching can update a template without making every component + // saveable. The user must be considered before the master, and 0x6b + // must be re-read after each save because saving the user can change + // the master's state. Never claim persistence without host copies. + for (who, kind, what) in [ + ( + crate::sbio::CatacombUser::enrolling(user), + PRIVATE_TYPE_CATACOMB_USER, + c"post-match user catacomb", + ), + ( + crate::sbio::CatacombUser::MASTER, + PRIVATE_TYPE_CATACOMB_MASTER, + c"post-match master catacomb", + ), + ] { + let Some(reply) = self.read_component_states() else { + dev_err!(self.dev, "verify: cannot read state before saving {}\n", what); + return false; + }; + let states = crate::sbio::ComponentStates::new(&reply); + let Some(state) = states.state_for(who.value()) else { + dev_err!(self.dev, "verify: {} is absent after matching\n", what); + return false; + }; + if state & crate::sbio::COMPONENT_STATE_ACTIVE == 0 { + dev_err!(self.dev, "verify: {} is not active after matching (state 0x{:x})\n", what, state); + return false; + } + if state & crate::sbio::COMPONENT_STATE_SAVE_PENDING != 0 + && !self.save_catacomb(who, kind, what) + { + dev_err!(self.dev, "verify: {} persistence failed\n", what); + return false; + } + if crate::catacomb::read(kind).is_none() { + dev_err!(self.dev, "verify: {} has no durable host file\n", what); + return false; + } } if !self.resnapshot_identity_keybag() { @@ -647,33 +927,49 @@ impl SepData { true } - fn save_catacomb(&self, who: crate::sbio::CatacombUser, kind: u8, _what: &CStr) -> bool { + fn save_catacomb(&self, who: crate::sbio::CatacombUser, kind: u8, what: &CStr) -> bool { let selector = crate::sbio::SaveSelector::new(who); - let Some(blob) = self.sbio_expect_ok(&crate::sbio::sbio_save_catacomb(&selector)) else { - return false; + let save = match self.sbio_transfer(&crate::sbio::sbio_save_catacomb(&selector)) { + Ok(done) if done.status.is_ok() => done, + Ok(done) => { + dev_err!(self.dev, "enrol: {} SAVE_CATACOMB refused with status {}\n", what, done.status); + return false; + } + Err(e) => { + dev_err!(self.dev, "enrol: {} SAVE_CATACOMB transfer failed ({:?})\n", what, e); + return false; + } }; + let blob = save.payload; if blob.len() < crate::sbio::SBIO_SAVED_MIN || blob.len() > crate::sbio::SBIO_SAVED_MAX { + dev_err!(self.dev, "enrol: {} SAVE_CATACOMB returned invalid length {}\n", what, blob.len()); return false; } let at = crate::sbio::SBIO_SAVED_USER_ID_AT; let carried = i32::from_le_bytes([blob[at], blob[at + 1], blob[at + 2], blob[at + 3]]); if carried != who.value() { + dev_err!(self.dev, "enrol: {} SAVE_CATACOMB returned user id {}, expected {}\n", what, carried, who.value()); return false; } if crate::catacomb::write(kind, &blob).is_err() { + dev_err!(self.dev, "enrol: {} SAVE_CATACOMB host persistence failed\n", what); return false; } - if self - .sbio_expect_ok(&crate::sbio::sbio_confirm_save(&selector)) - .is_none() - { - return false; + match self.sbio_transfer(&crate::sbio::sbio_confirm_save(&selector)) { + Ok(done) if done.status.is_ok() => true, + Ok(done) => { + dev_err!(self.dev, "enrol: {} CONFIRM_SAVE refused with status {}\n", what, done.status); + false + } + Err(e) => { + dev_err!(self.dev, "enrol: {} CONFIRM_SAVE transfer failed ({:?})\n", what, e); + false + } } - true } fn save_lockout(&self) -> bool { @@ -855,8 +1151,9 @@ impl SepData { self.sks_designate_user_keybag(handle, stored.secret()); self.sks_machine_refkey(handle, stored.secret()); let prepared = self.cold_match_continue(handle, uuid); - self.ensure_restored_after(prepared); self.attach_bringup(); + self.ensure_restored_after(prepared); + self.probe_owner_export(); self.touchid_started.store(true, Relaxed); Ok(()) } @@ -878,7 +1175,7 @@ impl SepData { if !self.templates_restored.load(Relaxed) { dev_err!( self.dev, - "verify: refusing before touching the sensor; cold-match prep or restore did not complete, enclave holds no template (every match refused 0x1). A restore failure, not a non-matching finger\n" + "verify: refusing before touching the sensor; cold-match prep, restore, or device-view identity proof did not complete. This is not a non-matching finger\n" ); self.finish_verify( bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE), @@ -1031,6 +1328,8 @@ impl SepData { } pub(crate) fn run_enrolment(&self) { + self.enrol_frames_accepted.store(0, Relaxed); + if !self.bring_sensor_online() { self.finish_enrolment(Err(ENROL_STATUS_SENSOR)); let _ = sensor::idle(); @@ -1038,6 +1337,7 @@ impl SepData { } let Some(open) = self.begin_enrolment_on_enclave() else { + dev_err!(self.dev, "enrol: the enclave would not start an enrolment; see the preceding step\n"); self.finish_enrolment(Err(ENROL_STATUS_ENCLAVE)); let _ = sensor::idle(); return; @@ -1045,6 +1345,7 @@ impl SepData { let mut counter: u32 = 0; let mut enrolment_completed = false; + let mut enrolment_persisted = false; let mut identity: Option<[u8; bio::UUID_LEN]> = None; let outcome = loop { @@ -1104,18 +1405,18 @@ impl SepData { if let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) { if !self.save_all_components(user) { - dev_warn!( + dev_err!( + self.dev, + "enrol: enclave completed but Catacomb persistence failed; not reporting enrollment success\n" + ); + } else if !self.resnapshot_identity_keybag() { + dev_err!( self.dev, - "enrol: enrolment succeeded but at least one of four artefacts was not persisted; works until the next reboot (see which component above)\n" + "enrol: Catacombs saved but identity-bag snapshot failed; not reporting enrollment success\n" ); } else { self.templates_restored.store(true, Relaxed); - if !self.resnapshot_identity_keybag() { - dev_warn!( - self.dev, - "enrol: catacombs saved but post-commit identity-bag snapshot was not; template works this boot but is not reboot-persistent\n" - ); - } + enrolment_persisted = true; } } @@ -1129,10 +1430,10 @@ impl SepData { self.note_capture_end(); if let Some(result) = outcome { - let filed = match (result, identity) { - (Ok(()), Some(uuid)) => Ok(uuid), - (Ok(()), None) => Err(ENROL_STATUS_UNFILED), - (Err(status), _) => Err(status), + let filed = match (result, identity, enrolment_persisted) { + (Ok(()), Some(uuid), true) => Ok(uuid), + (Ok(()), _, _) => Err(ENROL_STATUS_UNFILED), + (Err(status), _, _) => Err(status), }; self.finish_enrolment(filed); } @@ -1140,14 +1441,17 @@ impl SepData { fn register_sensor(&self, id: &sensor::Identifier) -> bool { let stage = self.bringup.load(Relaxed); + dev_info!(self.dev, "sbio: registering sensor at bring-up stage {}\n", stage); if stage >= BRINGUP_ESTABLISHED { + let _ = self.log_identity_count(c"before CLEAR_STATE for sensor re-registration"); if self .sbio_expect_ok(&crate::sbio::sbio_clear_state()) .is_none() { return false; } + let _ = self.log_identity_count(c"after CLEAR_STATE for sensor re-registration"); } if stage == BRINGUP_FRESH { @@ -1173,6 +1477,7 @@ impl SepData { match self.sbio_call(&op) { SbioOutcome::Ok(_payload) => { self.bringup.store(BRINGUP_ESTABLISHED, Relaxed); + let _ = self.log_identity_count(c"after sensor serial registration"); true } _ => false, @@ -1330,6 +1635,7 @@ impl SepData { if listed && !self.device_view_synced.xchg(true, Relaxed) { let count = self.log_identity_count(c"after device-view synchronisation"); + dev_info!(self.dev, "sbio: device-view identity count {:?}, restore-ready before proof {}\n", count, self.templates_restored.load(Relaxed)); match count { Some(0) => { self.templates_restored.store(false, Relaxed); @@ -1348,9 +1654,12 @@ impl SepData { self.templates_restored.store(false, Relaxed); } } + dev_info!(self.dev, "sbio: restore-ready after device-view proof {}\n", self.templates_restored.load(Relaxed)); if self.templates_restored.load(Relaxed) { self.reconcile_identities(); } + } else if !listed { + dev_warn!(self.dev, "sbio: device-view synchronisation or match policy unavailable\n"); } true @@ -1613,10 +1922,11 @@ impl SepData { }; let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + dev_err!(self.dev, "enrol: invalid user id before image processing\n"); return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); }; if self - .sbio_expect_ok(&crate::sbio::sbio_prepare_image_processing()) + .sbio_enrol_step(&crate::sbio::sbio_prepare_image_processing()) .is_none() { return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); @@ -1631,7 +1941,7 @@ impl SepData { user, crate::shim::monotonic_ns(), ); - if self.sbio_expect_ok(&init).is_none() { + if self.sbio_enrol_step(&init).is_none() { return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); } @@ -1641,10 +1951,11 @@ impl SepData { } drop(capture); - let Some(assessment) = self.sbio_expect_ok(&crate::sbio::sbio_image_assessment()) else { + let Some(assessment) = self.sbio_enrol_step(&crate::sbio::sbio_image_assessment()) else { return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); }; if assessment.len() < crate::sbio::ASSESS_MIN_LEN { + dev_err!(self.dev, "enrol: image assessment short ({} bytes)\n", assessment.len()); return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); } let usable = assessment[crate::sbio::ASSESS_USABLE_ENROL] != 0; @@ -1652,10 +1963,11 @@ impl SepData { return ImageOutcome::Retry; } - let Some(result) = self.sbio_expect_ok(&crate::sbio::sbio_enrolment_result()) else { + let Some(result) = self.sbio_enrol_step(&crate::sbio::sbio_enrolment_result()) else { return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); }; let Some(parsed) = crate::sbio::EnrolmentResult::parse(&result) else { + dev_err!(self.dev, "enrol: malformed enrolment result ({} bytes)\n", result.len()); return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); }; @@ -1664,11 +1976,17 @@ impl SepData { } let percent = parsed.progress_percent(); + // Coverage can stop changing while SEP continues accepting frames. + // Count accepted captures so each one remains visible to libfprint. + let accepted = self.enrol_frames_accepted.fetch_add(1, Relaxed) + 1; + let stage = if parsed.complete() { + bio::ENROL_STAGES + } else { + accepted.min(bio::ENROL_STAGES - 1) + }; ImageOutcome::Progress { - stage: (percent * bio::ENROL_STAGES) - .div_ceil(100) - .min(bio::ENROL_STAGES), + stage, percent, complete: parsed.complete(), has_template: parsed.has_template(), @@ -2082,6 +2400,16 @@ impl SepData { if cmd == bio::IOC_ATTEST { return self.bio_attest(arg); } + // Query SEP outside the bio session/index locks. A host index can + // outlive its SEP identity after a failed cold restore; userspace + // must not mistake that index for proof of a live template. + let live_identity_count = if cmd == bio::IOC_GET_INFO + && self.touchid_started.load(Relaxed) + { + self.enclave_identity_count() + } else { + None + }; let handled = { let mut session = self.bio_session.lock(); let index = self.bio_index.lock(); @@ -2089,6 +2417,7 @@ impl SepData { session: &mut session, index: &index, sensor_present: self.sensor_present.load(Relaxed), + live_identity_count, }; bio::ioctl(&mut ctx, cmd, arg)? }; @@ -2146,32 +2475,42 @@ impl SepData { }) } - fn ensure_restored_after(&self, _prepared: bool) { - let restored = self.restore_all_components(); - self.templates_restored.store(restored, Relaxed); - if !restored { - dev_err!( - self.dev, - "matching unavailable this boot: restore did not complete, enclave holds no template (every match refused 0x1). Not the sensor or the finger; on-disk enrolments intact\n" - ); - return; - } - + fn ensure_restored_after(&self, prepared: bool) { + // Probe credential ordering without assuming it fixes cold restore. + // J414s still answered 0x8002 and listed zero identities with this + // credential established, so the identity proof below remains required. + let mut unlocked = false; if self.keybag_designated.load(Relaxed) { if let Some(user) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) { let special = user.special_handle(); if let Ok(keybag::State::Present(stored)) = keybag::read(keybag::Slot::Identity) { - let _ = self.sks_step(crate::sks::SKS_LOCK_STATE_NAME, |healthy| { + unlocked = self.sks_step(crate::sks::SKS_LOCK_STATE_NAME, |healthy| { self.sks_req_unlock_special(special, stored.secret(), healthy) - }); + }).is_some(); } } } + let match_context = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) + .map(|user| self.establish_scrd_match_context(user)) + .unwrap_or(false); + dev_info!(self.dev, "sbio: pre-restore special unlock {}, SCRD match context {}\n", unlocked, match_context); - // enclave's match arm asserts a non-empty ACM context (<= 32 bytes) - if let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) { - self.establish_scrd_match_context(user); + let restored = self.restore_all_components(); + dev_info!(self.dev, "sbio: cold-match preparation {}, component restore {}\n", prepared, restored); + let identities = self.log_identity_count(c"after Catacomb restore with sensor registered"); + let identity_ready = matches!(identities, Some(n) if n > 0); + let ready = prepared && match_context && restored && identity_ready + && self.device_view_synced.load(Relaxed) && self.prove_restore(); + self.templates_restored.store(ready, Relaxed); + if !ready { + dev_err!( + self.dev, + "matching unavailable this boot: cold-match preparation, SCRD credential, Catacomb restore, device view, or identity proof did not complete; template state is unproven\n" + ); + return; } + self.reconcile_identities(); + } fn cold_match_continue( @@ -2260,7 +2599,7 @@ impl SepData { true } - fn log_identity_count(&self, _when: &CStr) -> Option { + fn enclave_identity_count(&self) -> Option { let op = crate::sbio::sbio_list_identities(); let SbioOutcome::Ok(reply) = self.sbio_call(&op) else { return None; @@ -2269,6 +2608,15 @@ impl SepData { Some(records.count()) } + fn log_identity_count(&self, when: &CStr) -> Option { + let Some(count) = self.enclave_identity_count() else { + dev_warn!(self.dev, "sbio: identity list unavailable {}\n", when); + return None; + }; + dev_info!(self.dev, "sbio: identity count {} {}\n", count, when); + Some(count) + } + fn enclave_lists(&self, uuid: &[u8; bio::UUID_LEN]) -> Option { let op = crate::sbio::sbio_list_identities(); let SbioOutcome::Ok(reply) = self.sbio_call(&op) else { @@ -2728,7 +3076,9 @@ static_assert!(SBIO_STATUS_ALREADY_ACTIVE != SBIO_STATUS_COLD_TRANSITION); pub(crate) const COMPONENT_STATE_COLD: u32 = 0x1; pub(crate) const COMPONENT_STATE_ACTIVE: u32 = 0x2; +pub(crate) const COMPONENT_STATE_SAVE_PENDING: u32 = 0x4; static_assert!(COMPONENT_STATE_COLD != COMPONENT_STATE_ACTIVE); +static_assert!(COMPONENT_STATE_SAVE_PENDING & (COMPONENT_STATE_COLD | COMPONENT_STATE_ACTIVE) == 0); pub(crate) const COMPONENT_PAIR_LEN: usize = 8; diff --git a/drivers/soc/apple/sep-bio.h b/drivers/soc/apple/sep-bio.h index 7ad0b814af648..2ec3949eaa665 100644 --- a/drivers/soc/apple/sep-bio.h +++ b/drivers/soc/apple/sep-bio.h @@ -53,6 +53,9 @@ struct sep_bio_info { __u32 reserved[3]; }; +/* reserved[0] flags; reserved[1] is a live SEP identity count only when valid. */ +#define SEP_BIO_INFO_LIVE_COUNT_VALID 1u + struct sep_bio_list { __u32 count; __u32 reserved; diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index f15dc31f33b0e..74749f8b9d984 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -78,6 +78,7 @@ use kernel::{ const SETTLE_MS: time::Msecs = 200; const FIRST_RESPONSE_MS: time::Msecs = 3000; +const RNG_SURVEY_ATTEMPTS: u32 = 5; const SETTLE_WORK_ID: u64 = 1; @@ -333,6 +334,7 @@ impl LoadAnswer { } } +#[derive(Debug)] enum RestoreOutcome { Restored, AlreadyActive, @@ -642,10 +644,14 @@ struct SepData { #[pin] fv_volumes: Mutex>, + enrol_frames_accepted: Atomic, + rng_shutdown: Atomic, rng_failures: Atomic, + rng_survey_attempts: Atomic, + rx: rxring::RxRing, rx_count: Atomic, @@ -727,7 +733,7 @@ impl SepData { let base = store.base(); dev_info!( dev, - "xART: APFS-resolved read-only .gl extent (base {:#x}); {} slots, {} live records, max revision {}, {} malformed, {} duplicate, {} repaired; writes {}\n", + "xART: APFS-resolved .gl extent (base {:#x}); {} slots, {} live records, max revision {}, {} malformed, {} duplicate, {} repaired; writes {}\n", base, slots, records, @@ -811,8 +817,10 @@ impl SepData { rng <- new_mutex!(None), machine_refkey <- new_mutex!(None), fv_volumes <- new_mutex!(KVec::new()), + enrol_frames_accepted: Atomic::new(0), rng_shutdown: Atomic::new(false), rng_failures: Atomic::new(0), + rng_survey_attempts: Atomic::new(0), rx: rxring::RxRing::new(), rx_count: Atomic::new(0), settle_mark: Atomic::new(0), @@ -1093,40 +1101,42 @@ impl SepData { } fn control_survey(&self) { - for param in [0x00u8, 0x01, 0xff] { let _ = self.control_request(&proto::op_nop(param)); } let _ = self.control_request(&proto::op_security_mode()); - let mut words = [0u32; 4]; - let mut got = 0; - for w in words.iter_mut() { - match self.get_entropy_word() { - Ok(v) => { - *w = v; - got += 1; - } - Err(_) => { - break; - } - } + let _state = self.control.lock(); + } + + fn survey_hwrng(&self) -> Result<()> { + // A working control endpoint must answer four consecutive draws before + // it is exposed to the kernel RNG core. Early attach can beat the SEP + // endpoint exchange, so this runs only after that exchange settles. + for _ in 0..4 { + let _ = self.get_entropy_word()?; } + self.register_hwrng() + } - if got == words.len() { - match self.register_hwrng() { - Ok(()) => {}, - Err(e) => dev_err!(self.dev, "hwrng: registration failed: {:?}\n", e), + fn tick_ready(this: &Arc) { + if this.rng.lock().is_some() { + return; + } + let attempt = this.rng_survey_attempts.load(Relaxed) + 1; + if attempt > RNG_SURVEY_ATTEMPTS { + return; + } + this.rng_survey_attempts.store(attempt, Relaxed); + match this.survey_hwrng() { + Ok(()) => dev_info!(this.dev, "hwrng: registered after ready-phase survey (attempt {})\n", attempt), + Err(e) if attempt < RNG_SURVEY_ATTEMPTS => { + dev_warn!(this.dev, "hwrng: survey attempt {} failed ({:?}); retrying\n", attempt, e); + Self::arm_settle(this); } - } else { - dev_err!( - self.dev, - "hwrng: not registering, the source did not answer during the survey\n" - ); + Err(e) => dev_err!(this.dev, "hwrng: survey failed after {} attempts ({:?})\n", attempt, e), } - - let _state = self.control.lock(); } @@ -1310,6 +1320,17 @@ impl SepData { return; }; + if *module_parameters::xarm_trace.value() != 0 { + dev_info!( + self.dev, + "xarm: op 0x{:02x} request_len {} reply_status 0x{:02x} reply_len {}\n", + req.opcode, + req.length, + done.reply.status, + done.reply.length + ); + } + if done.reply_bytes > 0 { if self.ool_write(&self.ool_xarm, 0, &staging[..done.reply_bytes]).is_err() { self.fail_xarm(req.tag); @@ -1551,6 +1572,7 @@ impl SepData { user: crate::sbio::UserId, ) -> Option<[u8; crate::scrd::SCRD_ACM_HANDLE_LEN]> { if !self.scrd_ready() { + dev_err!(self.dev, "enrol: SCRD endpoint is not ready\n"); return None; } @@ -1562,54 +1584,74 @@ impl SepData { (material.special, Secret(copy)) }; - let (status, _) = self.scrd_command(&crate::scrd::scrd_initialize())?; + let Some((status, _)) = self.scrd_command(&crate::scrd::scrd_initialize()) else { + dev_err!(self.dev, "enrol: SCRD initialize did not answer\n"); + return None; + }; if status != 0 { + dev_err!(self.dev, "enrol: SCRD initialize refused (status {})\n", status); return None; } - let (status, out) = self.scrd_command(&crate::scrd::scrd_context_create_tracked(user.value()))?; + let Some((status, out)) = self.scrd_command(&crate::scrd::scrd_context_create_tracked(user.value())) else { + dev_err!(self.dev, "enrol: SCRD context create did not answer\n"); + return None; + }; if status != 0 || out.len() < crate::scrd::SCRD_ACM_HANDLE_LEN { + dev_err!(self.dev, "enrol: SCRD context create failed (status {}, {} reply bytes)\n", status, out.len()); return None; } let mut acm_handle = [0u8; crate::scrd::SCRD_ACM_HANDLE_LEN]; acm_handle.copy_from_slice(&out[..crate::scrd::SCRD_ACM_HANDLE_LEN]); - let (status, _) = self.scrd_command(&crate::scrd::scrd_context_externalize(&acm_handle))?; + let Some((status, _)) = self.scrd_command(&crate::scrd::scrd_context_externalize(&acm_handle)) else { + dev_err!(self.dev, "enrol: SCRD context externalize did not answer\n"); + return None; + }; if status != 0 { + dev_err!(self.dev, "enrol: SCRD context externalize refused (status {})\n", status); return None; } - let out = self.sks_send(self.sks_req_verify_secret(special, &secret, &acm_handle))?; + let Some(out) = self.sks_send(self.sks_req_verify_secret(special, &secret, &acm_handle)) else { + dev_err!(self.dev, "enrol: SKS verify-secret did not answer\n"); + return None; + }; if out.reply.status != 0 { + dev_err!(self.dev, "enrol: SKS verify-secret refused (status {})\n", out.reply.status); return None; } - let (status, out) = self.scrd_command(&crate::scrd::scrd_verify_touchid_enrollment(&acm_handle))?; + let Some((status, out)) = self.scrd_command(&crate::scrd::scrd_verify_touchid_enrollment(&acm_handle)) else { + dev_err!(self.dev, "enrol: SCRD Touch ID policy check did not answer\n"); + return None; + }; let satisfied = out.len() >= 4 && u32::from_le_bytes([out[0], out[1], out[2], out[3]]) != 0; if status != 0 || !satisfied { + dev_err!(self.dev, "enrol: SCRD Touch ID policy rejected (status {}, satisfied {})\n", status, satisfied); return None; } Some(acm_handle) } - fn establish_scrd_match_context(&self, user: crate::sbio::UserId) { + fn establish_scrd_match_context(&self, user: crate::sbio::UserId) -> bool { if !self.scrd_ready() { - return; + return false; } let Some(du) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) else { - return; + return false; }; let special = du.special_handle(); let stored = match keybag::read(keybag::Slot::Identity) { Ok(keybag::State::Present(s)) => s, _ => { - return; + return false; } }; - let _ = (|| -> Option<[u8; crate::scrd::SCRD_ACM_HANDLE_LEN]> { + let established = (|| -> Option<[u8; crate::scrd::SCRD_ACM_HANDLE_LEN]> { let (status, _) = self.scrd_command(&crate::scrd::scrd_initialize())?; if status != 0 { return None; @@ -1631,7 +1673,9 @@ impl SepData { } self.scrd_command(&crate::scrd::scrd_verify_touchid_enrollment(&acm_handle))?; Some(acm_handle) - })(); + })().is_some(); + dev_info!(self.dev, "scrd: cold-match credential established {}\n", established); + established } @@ -1730,7 +1774,8 @@ impl SepData { match this.phase.load(Relaxed) { PHASE_ATTACH => Self::tick_attach(this), PHASE_EXCHANGE => Self::tick_exchange(this), - _ => {} + PHASE_READY => Self::tick_ready(this), + _ => {}, } } @@ -1807,8 +1852,9 @@ impl SepData { } } - this.phase.store(PHASE_READY, Relaxed); this.run_bringup(); + this.phase.store(PHASE_READY, Relaxed); + Self::arm_settle(this); } fn endpoint_present(&self, id: u8) -> bool { @@ -2094,13 +2140,6 @@ impl platform::Driver for SepDriver { _info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { let dev: &device::Device> = pdev.as_ref(); - if *module_parameters::xart_writes.value() != 0 { - dev_err!( - dev, - "raw APFS xART writes are blocked until the .gl update protocol and ownership invariants are validated\n" - ); - return Err(ENOTSUPP); - } if *module_parameters::provision_keybag.value() != 0 && *module_parameters::xart_writes.value() == 0 { @@ -2192,7 +2231,7 @@ module! { params: { xart_writes: u8 { default: 0, - description: "Request shared xART writes (refused until the macOS update protocol is verified)", + description: "Opt in to shared xART writes only if APFS proves a single unsnapshotted .gl extent", }, xart_start_sector: u64 { default: 0, @@ -2210,5 +2249,17 @@ module! { default: 0, description: "Low 64 bits of an explicit xART OS UUID", }, + xarm_trace: u8 { + default: 0, + description: "Opt-in xART opcode/status trace without payloads or key material", + }, + probe_owner_export: u8 { + default: 0, + description: "Opt-in J414s diagnostic: with zero live identities, select the system context and try saving the missing owner Catacomb", + }, + j414s_persistent_enrol: u8 { + default: 0, + description: "Opt-in J414s enrollment test: export owner from the fresh context, then save user before master at completion", + }, }, } diff --git a/drivers/soc/apple/xart_apfs.rs b/drivers/soc/apple/xart_apfs.rs index 9eb833baca82d..1caca4aa1871c 100644 --- a/drivers/soc/apple/xart_apfs.rs +++ b/drivers/soc/apple/xart_apfs.rs @@ -15,6 +15,8 @@ const ROOT_INFO_SIZE: usize = 40; const XART_ROLE: u16 = 0x100; const FILE_SIZE: u64 = 0x600000; const FS_UNENCRYPTED: u64 = 1; +const INODE_WAS_CLONED: u64 = 0x10; +const INODE_WAS_EVER_CLONED: u64 = 0x400; const MOD: u64 = 0xffff_ffff; fn le16(data: &[u8], off: usize) -> Result { @@ -209,7 +211,74 @@ fn omap_lookup( Ok(paddr) } -fn xart_extent(file: &shim::StoreFile, size: u64, vol: &[u8], xid: u64) -> Result { +fn is_gigalocker_name(name: &[u8]) -> bool { + if name == b".gl" { + return true; + } + if name.len() != 39 || &name[36..] != b".gl" { + return false; + } + name[..36].iter().enumerate().all(|(i, byte)| { + if matches!(i, 8 | 13 | 18 | 23) { + *byte == b'-' + } else { + byte.is_ascii_hexdigit() + } + }) +} + +fn validate_write_ownership( + file: &shim::StoreFile, + size: u64, + vol: &[u8], + xid: u64, + private_id: u64, + inode_flags: u64, + physical: u64, +) -> Result<()> { + // This is deliberately narrower than general APFS write support. On the + // tested J414s the .gl file has one unshared, unsnapshotted extent. Never + // raw-write a clone, snapshot, pending revert, or unknown extent layout. + if le64(vol, 264)? != FS_UNENCRYPTED + || le64(vol, 216)? != 0 + || le64(vol, 160)? != 0 + || le64(vol, 168)? != 0 + || inode_flags & (INODE_WAS_CLONED | INODE_WAS_EVER_CLONED) != 0 + { + return Err(ENOTSUPP); + } + let tree_paddr = le64(vol, 144)?; + let tree = read_object(file, size, tree_paddr)?; + if le64(&tree, 8)? != tree_paddr + || le64(&tree, 16)? > xid + || le32(&tree, 24)? != 0x4000_0002 + || le32(&tree, 28)? != 0x0f + || le16(&tree, 32)? != 3 + || le16(&tree, 34)? != 0 + || le32(&tree, 36)? != 1 + { + return Err(ENOTSUPP); + } + let (key, val) = node_entry(&tree, 0, false)?; + if key.len() != 8 + || val.len() < 20 + || le64(key, 0)? != ((2u64 << 60) | physical) + || le64(val, 0)? != ((1u64 << 60) | (FILE_SIZE / BLOCK as u64)) + || le64(val, 8)? != private_id + || le32(val, 16)? != 1 + { + return Err(ENOTSUPP); + } + Ok(()) +} + +fn xart_extent( + file: &shim::StoreFile, + size: u64, + vol: &[u8], + xid: u64, + writes_enabled: bool, +) -> Result { if le64(vol, 264)? & FS_UNENCRYPTED == 0 { return Err(ENOTSUPP); } @@ -232,7 +301,7 @@ fn xart_extent(file: &shim::StoreFile, size: u64, vol: &[u8], xid: u64) -> Resul if name_len == 0 || 12 + name_len > key.len() || key[12 + name_len - 1] != 0 { return Err(EINVAL); } - if &key[12..12 + name_len - 1] == b".gl" { + if is_gigalocker_name(&key[12..12 + name_len - 1]) { if gl_file.replace(le64(val, 0)?).is_some() { return Err(ENOTSUPP); } @@ -240,12 +309,14 @@ fn xart_extent(file: &shim::StoreFile, size: u64, vol: &[u8], xid: u64) -> Resul } let gl_file = gl_file.ok_or(ENODATA)?; let mut private_id: Option = None; + let mut inode_flags: Option = None; for i in 0..count { let (key, val) = node_entry(&root, i, false)?; if key.len() >= 8 && le64(key, 0)? == ((3u64 << 60) | gl_file) { if val.len() < 92 || private_id.replace(le64(val, 8)?).is_some() { return Err(EINVAL); } + inode_flags = Some(le64(val, 48)?); } } let private_id = private_id.ok_or(ENODATA)?; @@ -265,11 +336,23 @@ fn xart_extent(file: &shim::StoreFile, size: u64, vol: &[u8], xid: u64) -> Resul return Err(EINVAL); } } - extent.ok_or(ENODATA) + let extent = extent.ok_or(ENODATA)?; + if writes_enabled { + validate_write_ownership( + file, + size, + vol, + xid, + private_id, + inode_flags.ok_or(ENODATA)?, + extent / BLOCK as u64, + )?; + } + Ok(extent) } /// Return the byte offset of the sole contiguous 6 MiB `.gl` file extent. -pub(crate) fn locate(file: &shim::StoreFile) -> Result { +pub(crate) fn locate(file: &shim::StoreFile, writes_enabled: bool) -> Result { let device_size = file.size()?; let base = read_object(file, device_size, 0)?; if &base[32..36] != b"NXSB" || le32(&base, 36)? != BLOCK as u32 { @@ -331,7 +414,13 @@ pub(crate) fn locate(file: &shim::StoreFile) -> Result { } if le16(&vol, 964)? == XART_ROLE { if found - .replace(xart_extent(file, size, &vol, le64(&vol, 16)?)?) + .replace(xart_extent( + file, + size, + &vol, + le64(&vol, 16)?, + writes_enabled, + )?) .is_some() { return Err(ENOTSUPP); diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs index 7c73107bc2ae8..c487824f7d9a8 100644 --- a/drivers/soc/apple/xart_store.rs +++ b/drivers/soc/apple/xart_store.rs @@ -5,8 +5,8 @@ //! //! The APFS locator identifies the existing `.gl` file's physical extent. //! This module validates records and serves SEP reads from that extent. It -//! never creates storage. Raw writes remain blocked until the record update -//! protocol and APFS ownership rules are verified against macOS. +//! never creates storage. Writes are opt-in and require narrow APFS checks +//! for an unshared, unsnapshotted extent. use crate::shim; use kernel::prelude::*; @@ -110,23 +110,20 @@ fn valid_key(key: &Key) -> bool { } impl Store { - /// Opens the iBoot system container read-only, resolves `.gl` through APFS + /// Opens the iBoot system container, resolves `.gl` through APFS /// metadata, then validates the record store at exactly that extent. /// A nonzero start sector is an assertion against the APFS result, never - /// an override of it. Neither lookup nor record validation authorizes - /// raw writes: macOS's update protocol is still unverified. + /// an override of it. Writes additionally require a single-owner extent + /// with no snapshots or pending revert, checked on the same block handle. pub(crate) fn open_owner(writes_enabled: bool, start_sector: u64) -> Result { - if writes_enabled { - return Err(ENOTSUPP); - } - let file = shim::StoreFile::open_block(OWNER_PATH, false)?; - let base = crate::xart_apfs::locate(&file)?; + let file = shim::StoreFile::open_block(OWNER_PATH, writes_enabled)?; + let base = crate::xart_apfs::locate(&file, writes_enabled)?; if start_sector != 0 { if start_sector.checked_mul(512).ok_or(EINVAL)? != base { return Err(EINVAL); } } - Self::open_file(file, false, base) + Self::open_file(file, writes_enabled, base) } /// Opens a caller-selected block mapping at offset zero. @@ -151,8 +148,8 @@ impl Store { Self::open_file(file, writes_enabled, base) } - /// Complete record validation using the same read-only handle with which - /// APFS ownership was resolved, avoiding a close/reopen window. + /// Complete record validation using the same handle with which APFS + /// ownership was resolved, avoiding a close/reopen window. fn open_file(file: shim::StoreFile, arm_writes: bool, base: u64) -> Result { let size = file.size()?; if base % BLOCK_SIZE as u64 != 0 || STORE_SIZE % BLOCK_SIZE as u64 != 0 { @@ -192,6 +189,9 @@ impl Store { if store.find(&Key::root(1)).is_none() || store.find(&Key::root(2)).is_none() { return Err(ENODATA); } + if arm_writes && (store.malformed_records != 0 || store.duplicate_records != 0) { + return Err(EIO); + } store.writes_enabled = arm_writes; if arm_writes { store.repair_disk()?; diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 767b9043d2ad5..b83ef5dea2e3d 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -12,15 +12,16 @@ systemctl daemon-reload systemctl enable aurora-sep.service ``` -The raw xART owner is currently read-only. Do not enable the installed -`aurora-sep.service` or run `load-driver` on a shared macOS/Linux installation: -that script requests `xart_writes=1`, which the kernel rejects. The kernel now -resolves the physical `.gl` extent from checksum-validated APFS metadata for -read-only operation; unsupported layouts fail closed. This removes the raw -record-signature locator, which was ambiguous when roots moved away from the -start of the file. APFS lookup alone does not establish write safety. Re-enable -provisioning only after macOS-compatible record updates and APFS ownership -invariants are verified on hardware. +`xart_writes=0` remains the default. `load-driver` explicitly opts into writes +and must only be enabled on a machine whose APFS `.gl` extent passes the +driver's narrow ownership checks: one physical extent, no snapshots or revert +state, and a single extent reference owned by the `.gl` inode. The kernel +resolves that extent from checksum-validated APFS metadata; unsupported layouts +fail closed. The opt-in has been tested on m2 with identity-keybag provisioning, +trusted-key seal/unseal, and reboot reload. The raw iBoot container hash did not +change in those tests, so an actual xART write was **not** validated. Do not +infer that all SEP or Touch ID operations work with a read-only GigaLocker. +Read `XART-SAFETY.md` before enabling writes on another dual-boot machine. For read-only structural diagnostics on Linux, run `sudo python3 apfs-xart-inspect.py /dev/disk/by-partlabel/iBootSystemContainer`. @@ -40,6 +41,75 @@ For fingerprint support, apply `patches/libfprint-1.94.100-apple-sep.patch` to libfprint 1.94.100, build and install libfprint, then install the fprintd device policy: +```sh +git clone --depth 1 --branch v1.94.100 \ + https://gitlab.freedesktop.org/libfprint/libfprint.git libfprint-aurora +cd libfprint-aurora +git apply ../linux/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch +meson setup build -Dprefix=/usr -Ddrivers=default -Dintrospection=false \ + -Dgtk-examples=false -Ddoc=false -Dinstalled-tests=false +ninja -C build +meson test -C build --print-errorlogs +``` + +Adjust the patch path for your checkout. The patch includes a fix for an +upstream Meson test loop when introspection is disabled. In the Aurora +`linux-build` guest, the additional packages needed for this build are +`libgudev libgusb pixman cairo glib2-devel`; `meson` and `ninja` are already +in the release image. The default driver set retains all normal libfprint +drivers and adds Aurora. This build was checked on aarch64: the shared library +and all default drivers compiled, with 0 test failures (most device tests +skip without their optional mocks/hardware). That does **not** prove Touch ID +works on m2. The patched library is installed privately under +`/opt/aurora-sep` there, with fprintd loading it via a service drop-in while +the distro libfprint remains untouched. fprintd discovers one Apple SEP sensor. +The Aurora driver now reports stage progress only when the kernel's stage +number increases; guidance and stage-zero events are not captured samples. +fprintd 1.94.5 itself emits an initial `enroll-stage-passed` after its +duplicate check, **before** starting enrollment. That first line does not +prove a finger was captured. +The device-specific `apple/mesa_calibration.bin` on m2 was recovered using +the bundled [`extract-mesa-calibration.py`](extract-mesa-calibration.py), copied +from [Gist revision `eb079a8007985d04e75182f20994f1a1c496f12e`](https://gist.github.com/DjDeveloperr/867a1961b861c570442724f48f770158/eb079a8007985d04e75182f20994f1a1c496f12e), +which scans the local iBoot System Container read-only for an FSCl/CALB record +with an IM4M manifest. It checks the container structure and markers, **not** +the manifest's cryptographic signature. Keep this blob private to the machine; +it is not a generic firmware package. On m2, a premature master +`SAVE_CATACOMB` returned SEP status `0x6` while its component state was `0x3` +(no save-pending bit). The +`m2fix10` fresh-context gate skips that unavailable save, and bounded +enrollment attempts reached the physical-finger wait without an immediate +enclave error. With the later UUID-forwarding stage2, a physical enrollment +completed on m2fix15 and master/user Catacombs were saved, but on each tested +reboot SEP listed zero live identities despite the host index still listing +the finger. m2fix16–m2fix20 narrowed this to the user Catacomb load: it +returns `0x8002` and becomes active, but reveals zero identities even with +the SCRD credential and sensor/device view prepared first. The saved files +remain backed up root-only on m2. A bounded repeated load still listed zero +identities. An opt-in owner export then produced a durable owner file, but +loading it did not recover the old user identity. `m2fix23` is a separate, +boot-tested candidate that saves owner in the fresh context and user before +master at completion; it has not been finger- or reboot-persistence-tested. +Login must stay password-based until a live post-reboot identity and +successful match are independently verified. + +On each Linux machine, extract its own calibration from its local iBoot System +Container (the input partition is discovered automatically): + +```sh +sudo install -d -m 0755 /usr/lib/firmware/apple +sudo python3 tools/aurora-sep/extract-mesa-calibration.py \ + -o /usr/lib/firmware/apple/mesa_calibration.bin +``` + +The extractor reads the iBoot partition but never writes to it. It creates the +output mode `0600`, rejects input/output aliases, and refuses ambiguous or +malformed calibration candidates. Do not commit or copy the resulting blob to +another machine. + +Once the hardware path is verified, install libfprint and then the fprintd +device policy: + ```sh install -Dm644 fprintd-aurora.conf \ /etc/systemd/system/fprintd.service.d/aurora.conf diff --git a/tools/aurora-sep/XART-SAFETY.md b/tools/aurora-sep/XART-SAFETY.md index 1ea3708dc0a2e..52a976c63e5c4 100644 --- a/tools/aurora-sep/XART-SAFETY.md +++ b/tools/aurora-sep/XART-SAFETY.md @@ -1,10 +1,13 @@ # Shared xART gigalocker safety gate The iBoot APFS container and its XART `.gl` file are shared with macOS. The -kernel owner must remain read-only. `apple_sep.xart_writes=1` is rejected even -with an explicit `xart_start_sector`; the latter is now only an assertion -against the APFS-resolved file extent. Do not enable provisioning services on a -dual-boot installation while this gate is in place. +deployed `m2fix4` kernel remains read-only. The `m2fix5` source has an explicit +`xart_writes=1` opt-in; it accepts writes only when the APFS-resolved extent is +single-owner, unsnapshotted, un-cloned, and all existing records validate. +`xart_start_sector` remains only an assertion against the APFS result. The +opt-in is for the controlled J414s hardware test, not general APFS write +support or a promise that restoring an old raw image reverses SEP anti-replay +state. Do not enable it on other installations without equivalent checks. ## Evidence collected on 2026-09-21 @@ -39,3 +42,40 @@ dual-boot installation while this gate is in place. The current APFS locator supports only a complete checkpoint, single-node object maps and file-system tree, and a single unencrypted 6 MiB `.gl` extent. Unsupported layouts fail closed; this is not a general APFS implementation. + +## Read-only J414s recovery image (2026-09-23) + +After the minimal stage2 made the SEP mailbox and RNG functional, a fresh +read-only inspection still found one unencrypted 6 MiB `.gl` extent at APFS +block 1236, 11 live records with valid payload CRCs, and no malformed records. +The selected container checkpoint was xid 136. No xART writes or keybag +provisioning were attempted. + +The 524,288,000-byte `iBootSystemContainer` partition was copied through a +read-only block handle and compressed to +`work/m2-tb-sep-deploy/iboot-system-container-2026-09-22.img.zst` (7,857,330 +bytes, mode 0600). The local archive passed `zstd -t`; its decompressed +SHA-256 matched a separate read of the partition: +`09206fcf0500f5646ea04ec37cf1fc8a0493f1c5ee3271c2af562ed9b39a6d37`. +The compressed archive SHA-256 is +`8a885a2397c26404cf8395ca64b279efc793f9118eae408f2a6d84fe9af1ded8`. +The temporary m2 copy was removed after verification. This is a recovery +artifact, **not** evidence that Linux may safely write APFS-owned `.gl` blocks. + +## Narrow write-ownership evidence (2026-09-23) + +The live m2 partition SHA-256 still matches the independent backup hash above. +The updated read-only inspector selected checkpoint xid 136 and checked the +current xART volume superblock and checksum-validated extent-reference tree: +volume flags `0x1` (unencrypted only), zero snapshots, zero pending revert +fields, and an inode with no clone flag. The sole `.gl` extent is APFS blocks +1236–2771 (1536 blocks, 6 MiB); its only physical extent-reference record is +kind `APFS_KIND_NEW`, owner 16 (the inode's private ID), reference count 1. +The inspector now rejects missing, overlapping, shared, cloned/snapshotted, +or mismatched references, and four synthetic tests pass. The `m2fix5` kernel +independently checks the same narrow ownership conditions on its block handle +before arming writes and refuses a writable scan with malformed or duplicate +xART records. These checks establish the current physical mapping and lack of +APFS sharing; they do not by themselves establish macOS's full record-update +or SEP anti-replay recovery semantics. The APFS field meanings come from the +[Apple File System Reference](https://developer.apple.com/support/apple-file-system/Apple-File-System-Reference.pdf). diff --git a/tools/aurora-sep/apfs-xart-inspect.py b/tools/aurora-sep/apfs-xart-inspect.py index 63cc276646f5e..d11473f06d0fc 100644 --- a/tools/aurora-sep/apfs-xart-inspect.py +++ b/tools/aurora-sep/apfs-xart-inspect.py @@ -16,6 +16,16 @@ MOD = 0xffffffff +def is_gigalocker_name(name): + if name == b".gl": + return True + if len(name) != 39 or name[36:] != b".gl": + return False + return all(byte == ord("-") if i in (8, 13, 18, 23) + else byte in b"0123456789abcdefABCDEF" + for i, byte in enumerate(name[:36])) + + def u16(buf, off): return struct.unpack_from("> 60 != 2: + raise ValueError("unexpected extent-reference key type") + start = tagged & ((1 << 60) - 1) + length_and_kind = u64(val, 0) + length = length_and_kind & ((1 << 60) - 1) + kind = length_and_kind >> 60 + owner = u64(val, 8) + refcount = struct.unpack_from(" (1 << 60): + raise ValueError("invalid extent-reference length") + if start < physical + blocks and physical < start + length: + if (start != physical or length != blocks or kind != 1 or + owner != private_id or refcount != 1): + raise ValueError(".gl extent reference is shared or mismatched") + matches.append((start, length, kind, owner, refcount)) + if len(matches) != 1: + raise ValueError("expected exactly one unshared .gl extent reference") + return matches[0] + + +def inspect_extent_reference(disk, vol, physical, blocks, private_id): + snapshots = u64(vol, 216) + revert_xid, revert_sblock = u64(vol, 160), u64(vol, 168) + print("xart_volume_flags=0x%x snapshot_count=%d revert_xid=%d revert_sblock=%d" % + (u64(vol, 264), snapshots, revert_xid, revert_sblock)) + if snapshots or revert_xid or revert_sblock: + raise ValueError("xART volume has snapshot or pending revert state") + tree_paddr = u64(vol, 144) + tree = disk.read(tree_paddr) + if (u64(tree, 8) != tree_paddr or u32(tree, 24) != 0x40000002 or + u32(tree, 28) != 0x0f): + raise ValueError("unsupported extent-reference tree") + level, flags, entries = node_entries(tree) + if level or flags & 3 != 3: + raise ValueError("multilevel extent-reference tree needs inspection") + match = validate_extent_references(entries, physical, blocks, private_id) + print("gigalocker_extent_reference_block_length_kind_owner_refcount=%s" % + (match,)) + + def inspect(disk): base = disk.read(0) if base[32:36] != b"NXSB" or u32(base, 36) != BLOCK: @@ -183,6 +240,7 @@ def inspect(disk): if level: raise ValueError("multilevel xART FS tree needs further inspection") files = [] + names = [] for key, val in entries: if len(key) < 12 or len(val) < 8: continue @@ -191,8 +249,10 @@ def inspect(disk): name_len = u32(key, 8) & 0x3ff if name_len and 12 + name_len <= len(key): name = key[12:12 + name_len - 1] - if name == b".gl": + if is_gigalocker_name(name): files.append(u64(val, 0)) + names.append(name.decode("ascii")) + print("gigalocker_file_names=%s" % names) print("gigalocker_file_ids=%s" % files) if len(files) != 1: raise ValueError("expected exactly one .gl file") @@ -204,7 +264,6 @@ def inspect(disk): private_id, inode_flags = u64(inode, 8), u64(inode, 48) print("gigalocker_inode_private_id=%d flags=0x%x links=%d write_generation=%d" % (private_id, inode_flags, u32(inode, 56), u32(inode, 64))) - print("xart_volume_flags=0x%x snapshot_count=%d" % (u64(vol, 264), u32(disk.read(vol_omap), 36))) extents = [] for key, val in entries: if len(key) >= 16 and len(val) >= 24 and u64(key, 0) == ((8 << 60) | private_id): @@ -218,6 +277,8 @@ def inspect(disk): if not extents: raise ValueError("no .gl extents") if len(extents) == 1 and extents[0][0] == 0 and extents[0][1] == 0x600000: + inspect_extent_reference(disk, vol, extents[0][2], + extents[0][1] // BLOCK, private_id) inspect_raw_locator(disk, extents[0][2]) scan_other_roots(disk, extents[0][2], extents[0][1]) diff --git a/tools/aurora-sep/extract-mesa-calibration.py b/tools/aurora-sep/extract-mesa-calibration.py new file mode 100644 index 0000000000000..0485e2ef21b37 --- /dev/null +++ b/tools/aurora-sep/extract-mesa-calibration.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python3 +"""Carve the per-device Mesa calibration from an Apple iBoot System Container.""" + +import argparse +import hashlib +import os +from pathlib import Path +import stat +import sys +import tempfile + + +ISC_PARTITION_NAME = "iBootSystemContainer" +SYS_BLOCK = Path("/sys/class/block") +DEV_ROOT = Path("/dev") +SCAN_CHUNK_SIZE = 4 * 1024 * 1024 +SCAN_OVERLAP = 4 * 1024 * 1024 + + +class DERError(ValueError): + pass + + +class TLV: + def __init__(self, data, start): + self.start = start + try: + self.tag = data[start] + first_length = data[start + 1] + except IndexError as error: + raise DERError("truncated DER header") from error + + if first_length < 0x80: + length = first_length + header_length = 2 + else: + length_bytes = first_length & 0x7f + if length_bytes == 0: + raise DERError("indefinite DER length") + if length_bytes > 4: + raise DERError("unreasonably large DER length") + length_start = start + 2 + length_end = length_start + length_bytes + if length_end > len(data): + raise DERError("truncated DER length") + encoded = data[length_start:length_end] + if encoded[0] == 0: + raise DERError("non-minimal DER length") + length = int.from_bytes(encoded, "big") + if length < 0x80: + raise DERError("non-minimal DER length") + header_length = 2 + length_bytes + + self.content_start = start + header_length + self.end = self.content_start + length + if self.end > len(data): + raise DERError("DER value extends beyond input") + + def content(self, data): + return data[self.content_start:self.end] + + +def child_tlvs(data, container): + offset = container.content_start + while offset < container.end: + child = TLV(data, offset) + if child.end > container.end: + raise DERError("DER child extends beyond its container") + yield child + offset = child.end + if offset != container.end: + raise DERError("malformed DER container") + + +def take_child(data, children, tag, value=None): + try: + child = next(children) + except StopIteration as error: + raise DERError("missing DER child") from error + if child.tag != tag: + raise DERError(f"unexpected DER tag 0x{child.tag:02x}") + if value is not None and child.content(data) != value: + raise DERError("unexpected DER value") + return child + +def validate_calibration(data, start): + """Return the end offset if data[start:] begins with a Mesa calibration.""" + outer = TLV(data, start) + if outer.tag != 0x30: + raise DERError("outer object is not a sequence") + outer_children = child_tlvs(data, outer) + take_child(data, outer_children, 0x16, b"comb") + + fdrd = take_child(data, outer_children, 0x30) + fdrd_children = child_tlvs(data, fdrd) + take_child(data, fdrd_children, 0x16, b"fdrd") + img4_octets = take_child(data, fdrd_children, 0x04) + try: + next(fdrd_children) + except StopIteration: + pass + else: + raise DERError("unexpected data in fdrd container") + + secb = take_child(data, outer_children, 0x30) + secb_children = child_tlvs(data, secb) + take_child(data, secb_children, 0x16, b"secb") + try: + next(outer_children) + except StopIteration: + pass + else: + raise DERError("unexpected data after secb container") + + img4 = TLV(data, img4_octets.content_start) + if img4.tag != 0x30 or img4.end != img4_octets.end: + raise DERError("fdrd is not one complete IMG4 sequence") + img4_children = child_tlvs(data, img4) + take_child(data, img4_children, 0x16, b"IMG4") + + im4p = take_child(data, img4_children, 0x30) + im4p_children = child_tlvs(data, im4p) + take_child(data, im4p_children, 0x16, b"IM4P") + take_child(data, im4p_children, 0x16, b"FSCl") + take_child(data, im4p_children, 0x16) + payload = take_child(data, im4p_children, 0x04).content(data) + if b"CALB" not in payload[:64]: + raise DERError("FSCl payload lacks its CALB header") + + # A signed calibration has an IM4M manifest following the IM4P. Requiring + # both markers avoids accepting an arbitrary, truncated FSCl object. + remainder = data[im4p.end:img4.end] + if b"IM4M" not in remainder or b"FSCl" not in remainder: + raise DERError("IMG4 lacks an FSCl manifest") + + return outer.end + + +def find_calibrations(data): + results = [] + signature = b"\x16\x04comb" + search_at = 0 + while True: + marker = data.find(signature, search_at) + if marker < 0: + break + # The outer SEQUENCE header immediately precedes the comb IA5String. + # Try every legal header width supported by TLV above. + for header_length in range(2, 7): + start = marker - header_length + if start < 0 or data[start] != 0x30: + continue + try: + end = validate_calibration(data, start) + except DERError: + continue + candidate = (start, end) + if candidate not in results: + results.append(candidate) + search_at = marker + 1 + return results + + +def discover_isc_inputs(sys_block=SYS_BLOCK, dev_root=DEV_ROOT): + """Find every iSC by GPT partition name, independent of disk numbering.""" + candidates = [] + try: + block_devices = sorted(sys_block.iterdir()) + except OSError as error: + block_devices = [] + sysfs_error = error + else: + sysfs_error = None + + for block_device in block_devices: + try: + properties = {} + for line in (block_device / "uevent").read_text().splitlines(): + key, separator, value = line.partition("=") + if separator: + properties[key] = value + except OSError: + continue + if properties.get("PARTNAME") != ISC_PARTITION_NAME: + continue + device_name = properties.get("DEVNAME", block_device.name) + if device_name.startswith("/dev/"): + device_name = device_name[5:] + device = dev_root / device_name + if device.exists(): + candidates.append(device) + + # This standard udev link also covers chroots where /sys is unavailable. + by_partlabel = dev_root / "disk/by-partlabel" / ISC_PARTITION_NAME + if by_partlabel.exists(): + candidates.append(by_partlabel) + + # Preserve order while removing aliases or duplicate sysfs entries. + unique = [] + seen = set() + for candidate in candidates: + try: + identity = candidate.resolve() + except OSError: + identity = candidate + if identity not in seen: + seen.add(identity) + unique.append(candidate) + if not unique and sysfs_error is not None: + raise RuntimeError( + f"cannot inspect {sys_block} and {by_partlabel} is unavailable: " + f"{sysfs_error}" + ) from sysfs_error + return unique + + +def scan_input(path): + """Scan a file or block device without retaining the whole iSC in memory.""" + mode = path.stat().st_mode + if not (stat.S_ISREG(mode) or stat.S_ISBLK(mode)): + raise RuntimeError(f"{path} is not a regular file or block device") + + candidates = {} + overlap = b"" + total = 0 + with path.open("rb", buffering=0) as source: + while True: + chunk = source.read(SCAN_CHUNK_SIZE) + if not chunk: + break + total += len(chunk) + window = overlap + chunk + window_start = total - len(window) + for start, end in find_calibrations(window): + absolute_start = window_start + start + candidates.setdefault(absolute_start, bytes(window[start:end])) + overlap = window[-SCAN_OVERLAP:] + return sorted(candidates.items()) + + +def write_private(path, blob): + path.parent.mkdir(parents=True, exist_ok=True) + old_umask = os.umask(0o077) + temporary = None + try: + fd, name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temporary = Path(name) + with os.fdopen(fd, "wb") as output: + output.write(blob) + output.flush() + os.fsync(output.fileno()) + os.chmod(temporary, stat.S_IRUSR | stat.S_IWUSR) + os.replace(temporary, path) + temporary = None + finally: + os.umask(old_umask) + if temporary is not None: + temporary.unlink(missing_ok=True) + + +def validate_output(path, inputs): + """Reject output targets that could replace an input or special file.""" + if path.is_symlink(): + raise RuntimeError(f"output {path} is a symbolic link") + if not path.exists(): + return + for input_path in inputs: + try: + if os.path.samefile(path, input_path): + raise RuntimeError(f"output {path} aliases input {input_path}") + except FileNotFoundError: + pass + if not stat.S_ISREG(path.stat().st_mode): + raise RuntimeError(f"output {path} is not a regular file") + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "input", nargs="?", type=Path, + help=( + "raw iBoot System Container or dump; omit to discover all " + "iBootSystemContainer partitions" + ), + ) + parser.add_argument( + "-o", "--output", type=Path, default=Path("mesa_calibration.bin"), + help="output file (default: ./mesa_calibration.bin)", + ) + return parser.parse_args(argv) + + +def main(): + args = parse_args() + try: + inputs = [args.input] if args.input is not None else discover_isc_inputs() + if not inputs: + raise RuntimeError( + "no iBootSystemContainer partition found; pass its block device " + "or a raw partition dump explicitly" + ) + validate_output(args.output, inputs) + + candidates = [] + for input_path in inputs: + for offset, blob in scan_input(input_path): + candidates.append((input_path, offset, blob)) + if not candidates: + raise RuntimeError( + "no signed Mesa FSCl/CALB calibration found; expected the raw " + "iBoot System Container (Apple Silicon boot partition)" + ) + blobs = {} + for input_path, offset, blob in candidates: + blobs.setdefault(blob, []).append((input_path, offset)) + if len(blobs) != 1: + locations = ", ".join( + f"{input_path}@0x{offset:x}" + for input_path, offset, _ in candidates + ) + raise RuntimeError( + f"found different calibrations at {locations}; " + "pass the intended iSC explicitly" + ) + + blob, locations = next(iter(blobs.items())) + write_private(args.output, blob) + except (OSError, RuntimeError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + digest = hashlib.sha256(blob).hexdigest() + formatted_locations = ", ".join( + f"{input_path}@0x{offset:x}" for input_path, offset in locations + ) + print(f"found Mesa calibration at {formatted_locations}") + print(f"wrote {len(blob)} bytes to {args.output}") + print(f"sha256 {digest}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/aurora-sep/load-driver b/tools/aurora-sep/load-driver index f5579fc01b360..a70db75e6b25b 100755 --- a/tools/aurora-sep/load-driver +++ b/tools/aurora-sep/load-driver @@ -4,6 +4,13 @@ set -eu keybag=/var/lib/aurora-sep-keybag.bin refkey=/var/lib/aurora-sep-refkey.bin +# An SSH/sudo caller cannot change its parent's session keyring with +# `keyctl new_session`. Run the whole probe in a child keyring instead; the +# marker prevents re-exec recursion and the keyring dies with this process. +if [ "${AURORA_SEP_SESSION:-0}" != 1 ]; then + exec keyctl session aurora-sep-load env AURORA_SEP_SESSION=1 "$0" "$@" +fi + modprobe apple_sep xart_writes=1 provision_keybag=1 work=$(mktemp -d /run/aurora-sep-load.XXXXXX) @@ -14,7 +21,6 @@ cleanup() { } trap cleanup EXIT HUP INT TERM -keyctl new_session aurora-sep-load >/dev/null attempts=300 verified=false while [ "$attempts" -gt 0 ]; do diff --git a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch index 37593f65ea05e..1947bd095b43b 100644 --- a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch +++ b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch @@ -135,10 +135,10 @@ index 0000000..b2d2f43 +#define AURORA_BIO_ATTEST _IOWR(AURORA_BIO_IOC_MAGIC, 0x0a, struct aurora_bio_attest) diff --git a/libfprint/drivers/aurora/aurora.c b/libfprint/drivers/aurora/aurora.c new file mode 100644 -index 0000000..55f3311 +index 0000000..80ace06 --- /dev/null +++ b/libfprint/drivers/aurora/aurora.c -@@ -0,0 +1,829 @@ +@@ -0,0 +1,837 @@ +/* + * Aurora SEP fingerprint driver + * @@ -170,6 +170,7 @@ index 0000000..55f3311 + + gint fd; + guint watch_id; ++ guint32 enroll_last_reported_stage; + + guint8 nonce[AURORA_BIO_NONCE_LEN]; + gboolean nonce_valid; @@ -516,7 +517,13 @@ index 0000000..55f3311 + return G_SOURCE_CONTINUE; + + case AURORA_BIO_STATE_PROGRESS: -+ fpi_device_enroll_progress (device, poll_res.stage, NULL, NULL); ++ /* fprintd calls this "enroll-stage-passed". A guidance wakeup or an ++ * initial stage-zero event must not masquerade as a captured sample. */ ++ if (poll_res.stage > self->enroll_last_reported_stage) ++ { ++ self->enroll_last_reported_stage = poll_res.stage; ++ fpi_device_enroll_progress (device, poll_res.stage, NULL, NULL); ++ } + return G_SOURCE_CONTINUE; + + case AURORA_BIO_STATE_DONE: @@ -543,7 +550,7 @@ index 0000000..55f3311 + self->watch_id = 0; + fpi_device_enroll_complete (device, NULL, + fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, -+ "enrolment failed, enclave status 0x%x", ++ "enrolment failed, driver status 0x%x", + poll_res.status)); + return G_SOURCE_REMOVE; + } @@ -571,6 +578,7 @@ index 0000000..55f3311 + return; + } + ++ self->enroll_last_reported_stage = 0; + aurora_stop_watch (self); + self->watch_id = g_unix_fd_add (self->fd, G_IO_IN, aurora_enrol_ready, device); +} @@ -1177,3 +1185,13 @@ index ab09dc1..dfa7460 100644 # Virtual drivers (test-only, optional) 'virtual_image': { 'virtual': true, 'helper': ['virtual'], 'optional': true }, 'virtual_device': { 'virtual': true, 'helper': ['virtual'], 'optional': true }, +diff --git a/tests/meson.build b/tests/meson.build +index aad0675..2147ef3 100644 +--- a/tests/meson.build ++++ b/tests/meson.build +@@ -295,4 +295,4 @@ else +- foreach driver_test: drivers_tests ++ foreach driver_test, args: drivers_tests + test(driver_test, + sh, + args: ['-c', 'exit 77'] diff --git a/tools/aurora-sep/tests/test_extract_mesa_calibration.py b/tools/aurora-sep/tests/test_extract_mesa_calibration.py new file mode 100644 index 0000000000000..08884ba921e9b --- /dev/null +++ b/tools/aurora-sep/tests/test_extract_mesa_calibration.py @@ -0,0 +1,72 @@ +"""Offline safety and format tests for the bundled calibration extractor.""" + +import importlib.util +from pathlib import Path +import stat +import tempfile +import unittest + + +SCRIPT = Path(__file__).resolve().parents[1] / "extract-mesa-calibration.py" +SPEC = importlib.util.spec_from_file_location("extract_mesa_calibration", SCRIPT) +extractor = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(extractor) + + +def tlv(tag, content): + length = len(content) + if length < 128: + encoded_length = bytes([length]) + else: + width = (length.bit_length() + 7) // 8 + encoded_length = bytes([0x80 | width]) + length.to_bytes(width, "big") + return bytes([tag]) + encoded_length + content + + +def ia5(value): + return tlv(0x16, value) + + +def synthetic_record(with_manifest=True): + """Build a structural fixture, not a cryptographically signed IMG4.""" + im4p = tlv(0x30, ia5(b"IM4P") + ia5(b"FSCl") + ia5(b"test") + + tlv(0x04, b"CALB" + b"x" * 64)) + manifest = ia5(b"IM4M") + ia5(b"FSCl") if with_manifest else b"" + img4 = tlv(0x30, ia5(b"IMG4") + im4p + manifest) + fdrd = tlv(0x30, ia5(b"fdrd") + tlv(0x04, img4)) + secb = tlv(0x30, ia5(b"secb")) + return tlv(0x30, ia5(b"comb") + fdrd + secb) + + +class CalibrationExtractorTests(unittest.TestCase): + def test_scans_only_complete_manifest_marked_record(self): + record = synthetic_record() + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "isc.img" + source.write_bytes(b"prefix" + record + b"suffix") + self.assertEqual(extractor.scan_input(source), [(6, record)]) + + def test_rejects_record_without_manifest_markers(self): + self.assertEqual(extractor.find_calibrations(synthetic_record(False)), []) + + def test_rejects_input_output_aliases_and_symlinks(self): + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "isc.img" + source.write_bytes(synthetic_record()) + with self.assertRaises(RuntimeError): + extractor.validate_output(source, [source]) + symlink = Path(directory) / "output.bin" + symlink.symlink_to(source) + with self.assertRaises(RuntimeError): + extractor.validate_output(symlink, [source]) + + def test_private_output_mode(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "mesa_calibration.bin" + extractor.write_private(output, b"sample") + self.assertEqual(output.read_bytes(), b"sample") + self.assertEqual(stat.S_IMODE(output.stat().st_mode), 0o600) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/aurora-sep/tests/test_xart_locator.py b/tools/aurora-sep/tests/test_xart_locator.py index 167c977ec2bd0..5a48e56726464 100644 --- a/tools/aurora-sep/tests/test_xart_locator.py +++ b/tools/aurora-sep/tests/test_xart_locator.py @@ -27,7 +27,47 @@ def record(kind, revision): return out +def extent_reference(start, blocks, owner, refcount=1, kind=1): + key = struct.pack(" Date: Fri, 25 Sep 2026 17:06:21 -0400 Subject: [PATCH 0718/2680] soc: apple: sep: surface capture guidance and enforce live user capacity Carry the verified J414s contact-guidance and three-slot changes from the combined worktree. The live SEP count, not a stale host index, gates new enrollment. --- drivers/soc/apple/bio.rs | 111 ++++++++++++++++++++++-------------- drivers/soc/apple/sbio.rs | 21 +++++-- drivers/soc/apple/sep-bio.h | 4 +- 3 files changed, 85 insertions(+), 51 deletions(-) diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index 2bc1a492799c3..d115a18d64d97 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -15,6 +15,9 @@ const LABEL_LEN: usize = 128; const NONCE_LEN: usize = 32; pub(crate) const TOKEN_LEN: usize = 32; const MAX_IDENTITIES: usize = 32; +// The host index ABI has 32 entries, but this driver uses one SEP user context. +// Mac Touch ID permits only three fingerprints in a single user context. +const SEP_USER_CAPACITY: usize = 3; pub(crate) const DEVICE_NAME: &CStr = c"sep-bio"; pub(crate) const DEVICE_MODE: u16 = 0o600; @@ -123,7 +126,7 @@ struct VerifyPoll { state: State, result: MatchResult, status: u32, - reserved: u32, + guidance: Guidance, uuid: [u8; UUID_LEN], token: [u8; TOKEN_LEN], deadline_ns: u64, @@ -184,7 +187,7 @@ const MAGIC: u32 = 0xB1; pub(crate) const IOC_GET_INFO: u32 = _IOR::(MAGIC, 0x01); const IOC_LIST: u32 = _IOR::(MAGIC, 0x02); -const IOC_ENROL_START: u32 = _IOW::(MAGIC, 0x03); +pub(crate) const IOC_ENROL_START: u32 = _IOW::(MAGIC, 0x03); const IOC_ENROL_POLL: u32 = _IOR::(MAGIC, 0x04); const IOC_VERIFY_START: u32 = _IOW::(MAGIC, 0x05); const IOC_VERIFY_POLL: u32 = _IOR::(MAGIC, 0x06); @@ -390,6 +393,7 @@ enum Op { Verify { nonce: [u8; NONCE_LEN], terminal: Option, + guidance: Guidance, }, } @@ -509,7 +513,7 @@ fn get_info(ctx: &mut Context<'_>, user: UserPtr) -> Result { version: IFACE_VERSION, sensor_present: u32::from(ctx.sensor_present), enrolled: ctx.index.total() as u32, - capacity: MAX_IDENTITIES as u32, + capacity: SEP_USER_CAPACITY as u32, enroll_stages: ENROL_STAGES, // reserved[0] is a validity flag. A zero count is only meaningful // when SEP itself successfully answered LIST_IDENTITIES; older @@ -559,7 +563,9 @@ fn enrol_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { if !ctx.session.op.may_start() { return Err(EBUSY); } - if ctx.index.total() >= MAX_IDENTITIES { + if matches!(ctx.live_identity_count, Some(count) if count >= SEP_USER_CAPACITY) + || ctx.index.total() >= MAX_IDENTITIES + { return Err(ENOSPC); } @@ -608,18 +614,30 @@ pub(crate) fn enrol_advance( false } -pub(crate) fn enrol_guide(session: &mut Session, guide: Guidance) -> bool { - if let Op::Enrol { - terminal, guidance, .. - } = &mut session.op - { - if terminal.is_none() && *guidance != guide { - *guidance = guide; - session.unseen = true; - return true; +pub(crate) fn capture_guide(session: &mut Session, guide: Guidance) -> bool { + let changed = match &mut session.op { + Op::Enrol { terminal, guidance, .. } => { + if terminal.is_none() && *guidance != guide { + *guidance = guide; + true + } else { + false + } + } + Op::Verify { terminal, guidance, .. } => { + if terminal.is_none() && *guidance != guide { + *guidance = guide; + true + } else { + false + } } + Op::Idle => false, + }; + if changed { + session.unseen = true; } - false + changed } pub(crate) fn enrol_finish( @@ -723,6 +741,7 @@ fn verify_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { ctx.session.op = Op::Verify { nonce: request.nonce, terminal: Some(VerifyOutcome::NoMatch), + guidance: Guidance::None, }; ctx.session.unseen = true; return Ok(Handled { @@ -739,6 +758,7 @@ fn verify_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { ctx.session.op = Op::Verify { nonce: request.nonce, terminal: None, + guidance: Guidance::Place, }; Ok(Handled { ret: 0, @@ -755,7 +775,7 @@ pub(crate) fn verify_finish( outcome: VerifyOutcome, token_bytes: [u8; TOKEN_LEN], ) -> bool { - let Op::Verify { nonce, terminal } = &mut session.op else { + let Op::Verify { nonce, terminal, .. } = &mut session.op else { return false; }; if terminal.is_some() { @@ -789,39 +809,42 @@ fn verify_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { match &ctx.session.op { Op::Idle => out.state = State::Idle, Op::Enrol { .. } => return Err(EBUSY), - Op::Verify { nonce, terminal } => match terminal { - None => { - out.state = if ctx.session.unseen { - State::Progress - } else { - State::Pending - } - } - Some(VerifyOutcome::Failed(status)) => { - out.state = State::Failed; - out.result = MatchResult::NotCompared; - out.status = *status; - } - Some(VerifyOutcome::NoMatch) => { - out.state = State::Done; - out.result = MatchResult::NoMatch; - } - Some(VerifyOutcome::Matched(evidence)) => { - out.state = State::Done; - match ctx.session.token.as_ref() { - Some(token) if token.usable(nonce, &evidence.identity) => { - out.result = MatchResult::Match; - out.uuid = evidence.identity; - out.token = token.bytes; - out.deadline_ns = token.deadline_ns; - consume_token = true; + Op::Verify { nonce, terminal, guidance } => { + out.guidance = *guidance; + match terminal { + None => { + out.state = if ctx.session.unseen { + State::Progress + } else { + State::Pending } - _ => { - out.result = MatchResult::NotCompared; + } + Some(VerifyOutcome::Failed(status)) => { + out.state = State::Failed; + out.result = MatchResult::NotCompared; + out.status = *status; + } + Some(VerifyOutcome::NoMatch) => { + out.state = State::Done; + out.result = MatchResult::NoMatch; + } + Some(VerifyOutcome::Matched(evidence)) => { + out.state = State::Done; + match ctx.session.token.as_ref() { + Some(token) if token.usable(nonce, &evidence.identity) => { + out.result = MatchResult::Match; + out.uuid = evidence.identity; + out.token = token.bytes; + out.deadline_ns = token.deadline_ns; + consume_token = true; + } + _ => { + out.result = MatchResult::NotCompared; + } } } } - }, + } } UserSlice::new(user, core::mem::size_of::()) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 91e5125b55acb..a901afc007a0b 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -38,12 +38,18 @@ impl SepData { pub(crate) fn sbio_call(&self, op: &crate::sbio::SbioOp) -> SbioOutcome { let done = match self.sbio_transfer(op) { Ok(done) => done, - Err(_) => { + Err(e) => { + if op.opcode() == OP_SBIO_BEGIN_ENROL { + dev_err!(self.dev, "enrol: BEGIN_ENROL transfer failed ({:?})\n", e); + } return SbioOutcome::Other; } }; let Some(err) = done.status.answered() else { + if op.opcode() == OP_SBIO_BEGIN_ENROL { + dev_err!(self.dev, "enrol: BEGIN_ENROL reply had no status: {}\n", done.status); + } return SbioOutcome::Other; }; @@ -51,7 +57,12 @@ impl SepData { crate::sbio::SBIO_STATUS_OK => SbioOutcome::Ok(done.payload), crate::sbio::SBIO_STATUS_PREREQUISITE => SbioOutcome::PrerequisiteMissing, crate::sbio::SBIO_STATUS_16 => SbioOutcome::Status16, - _ => SbioOutcome::Other, + _ => { + if op.opcode() == OP_SBIO_BEGIN_ENROL { + dev_err!(self.dev, "enrol: BEGIN_ENROL replied with status 0x{:x}\n", err); + } + SbioOutcome::Other + } } } @@ -1387,7 +1398,7 @@ impl SepData { // The capture did not take (partial/unusable contact). Nudge // the person to hold still and press again rather than // silently re-capturing with no feedback. - if bio::enrol_guide(&mut self.bio_session.lock(), bio::Guidance::HoldStill) { + if bio::capture_guide(&mut self.bio_session.lock(), bio::Guidance::HoldStill) { self.bio_wake(); } } @@ -2042,7 +2053,7 @@ impl SepData { _ => None, }; if let Some(guide) = guide { - if bio::enrol_guide(&mut self.bio_session.lock(), guide) { + if bio::capture_guide(&mut self.bio_session.lock(), guide) { self.bio_wake(); } } @@ -2403,7 +2414,7 @@ impl SepData { // Query SEP outside the bio session/index locks. A host index can // outlive its SEP identity after a failed cold restore; userspace // must not mistake that index for proof of a live template. - let live_identity_count = if cmd == bio::IOC_GET_INFO + let live_identity_count = if (cmd == bio::IOC_GET_INFO || cmd == bio::IOC_ENROL_START) && self.touchid_started.load(Relaxed) { self.enclave_identity_count() diff --git a/drivers/soc/apple/sep-bio.h b/drivers/soc/apple/sep-bio.h index 2ec3949eaa665..d52b880ac9332 100644 --- a/drivers/soc/apple/sep-bio.h +++ b/drivers/soc/apple/sep-bio.h @@ -48,7 +48,7 @@ struct sep_bio_info { __u32 version; __u32 sensor_present; __u32 enrolled; - __u32 capacity; + __u32 capacity; /* usable slots in this SEP user context, not ABI list size */ __u32 enroll_stages; __u32 reserved[3]; }; @@ -92,7 +92,7 @@ struct sep_bio_verify_poll { __u32 state; __u32 result; __u32 status; - __u32 reserved; + __u32 guidance; /* SEP_BIO_GUIDANCE_*; status only, no capture side effect */ __u8 uuid[SEP_BIO_UUID_LEN]; __u8 token[SEP_BIO_TOKEN_LEN]; /* single use, bound to the nonce */ __u64 deadline_ns; /* CLOCK_MONOTONIC; past this the token is void */ From ba0634c27310fe3526a415165d2b2035680c87b6 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 25 Sep 2026 17:06:51 -0400 Subject: [PATCH 0719/2680] soc: apple: sep: defer Neo sensor until SMC power is ready Treat an explicit SMC-powered sensor as an SMC consumer, rather than silently falling back to unrelated GPIO power. Keep the supplier bound, retry when it is late, and attempt to turn off every rail after a write failure. [ryan: the shim.h prototype for sep_sensor_firmware_name() that this commit added is declared earlier in this series (with the function itself), so this commit no longer adds it] Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 10 ++-- drivers/soc/apple/sensor_shim.c | 86 ++++++++++++++++++++++----------- drivers/soc/apple/sep.rs | 3 -- 3 files changed, 62 insertions(+), 37 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index a901afc007a0b..c91938b60fee1 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -262,7 +262,6 @@ impl SepData { pub(crate) fn attach_sensor(&self) { if let Err(e) = sensor::register_driver() { - self.sensor_present.store(false, Relaxed); dev_err!( self.dev, "sensor: could not register the SPI driver: {:?}\n", @@ -272,7 +271,6 @@ impl SepData { } if let Err(e) = dt::enable_spi_sensor(sensor::CONTROLLER_BASE, sensor::CHIP_SELECT) { - self.sensor_present.store(false, Relaxed); dev_warn!( self.dev, "sensor: could not enable the SPI bus at 0x{:x} or create the sensor node ({:?}); no sensor\n", @@ -283,7 +281,6 @@ impl SepData { } let bound = sensor::is_bound(); - self.sensor_present.store(bound, Relaxed); if bound && sensor::power_source() == sensor::PowerSource::None { dev_warn!( self.dev, @@ -1011,7 +1008,7 @@ impl SepData { } fn attach_bringup(&self) { - if !self.sensor_present.load(Relaxed) { + if !sensor::is_bound() { dev_warn!( self.dev, "sbio: no sensor bound at attach; catacomb restore ran but capture is unavailable\n" @@ -2421,13 +2418,16 @@ impl SepData { } else { None }; + // The SPI probe may have deferred until its SMC power supplier bound. + // A saved attach-time snapshot would incorrectly reject later ioctls. + let sensor_present = sensor::is_bound(); let handled = { let mut session = self.bio_session.lock(); let index = self.bio_index.lock(); let mut ctx = bio::Context { session: &mut session, index: &index, - sensor_present: self.sensor_present.load(Relaxed), + sensor_present, live_identity_count, }; bio::ioctl(&mut ctx, cmd, arg)? diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index ec4d30581da67..9ed3658cda8f7 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -96,22 +96,37 @@ static int sep_smc_write_rail(u32 rail, u16 value) static int sep_smc_set_power(int on) { - int i, rc = 0; + int i, rc, first_error = 0; if (on) { - for (i = 0; i < sep_smc_nrails && !rc; i++) + for (i = 0; i < sep_smc_nrails; i++) { rc = sep_smc_write_rail(sep_smc_rails[i], 1); + if (rc) { + first_error = rc; + /* The failed write may have taken effect too. */ + for (; i >= 0; i--) { + rc = sep_smc_write_rail(sep_smc_rails[i], 0); + if (rc) + pr_err("apple_sep: sensor: SMC rail rollback failed (%d)\n", rc); + } + break; + } + } } else { - for (i = sep_smc_nrails - 1; i >= 0 && !rc; i--) + /* A failed off write must not skip the remaining rails. */ + for (i = sep_smc_nrails - 1; i >= 0; i--) { rc = sep_smc_write_rail(sep_smc_rails[i], 0); + if (rc && !first_error) + first_error = rc; + } } - if (rc) - pr_err("apple_sep: sensor: SMC rail write failed (%d)\n", rc); - return rc; + if (first_error) + pr_err("apple_sep: sensor: SMC rail write failed (%d)\n", first_error); + return first_error; } /* Nothing is written until the expected eight-byte SMC key is verified. */ -static bool sep_acquire_smc_power(struct spi_device *spi) +static int sep_acquire_smc_power(struct spi_device *spi) { struct device_node *np = spi->dev.of_node, *smc_np; struct platform_device *smc_pdev; @@ -120,48 +135,55 @@ static bool sep_acquire_smc_power(struct spi_device *spi) int n, rc; if (!np || of_property_read_string(np, "apple,smc-power-key", &key)) - return false; + return -EINVAL; if (!key || strlen(key) != 4) { dev_warn(&spi->dev, "sep sensor: apple,smc-power-key must have four characters\n"); - return false; + return -EINVAL; } n = of_property_count_u32_elems(np, "apple,smc-power-rails"); if (n <= 0 || n > SEP_SMC_MAX_RAILS || of_property_read_u32_array(np, "apple,smc-power-rails", sep_smc_rails, n)) { dev_warn(&spi->dev, "sep sensor: expected 1..%d SMC rail words\n", SEP_SMC_MAX_RAILS); - return false; + return -EINVAL; } smc_np = of_find_compatible_node(NULL, NULL, "apple,smc"); if (!smc_np) - return false; + return -EPROBE_DEFER; smc_pdev = of_find_device_by_node(smc_np); of_node_put(smc_np); if (!smc_pdev) - return false; + return -EPROBE_DEFER; + /* Keep the supplier bound for as long as this sensor uses its drvdata. */ + if (!device_link_add(&spi->dev, &smc_pdev->dev, + DL_FLAG_AUTOREMOVE_CONSUMER)) { + put_device(&smc_pdev->dev); + return -ENOMEM; + } sep_smc = dev_get_drvdata(&smc_pdev->dev); put_device(&smc_pdev->dev); if (!sep_smc) - return false; + return -EPROBE_DEFER; sep_smc_key = __SMC_KEY(key[0], key[1], key[2], key[3]); rc = apple_smc_get_key_info(sep_smc, sep_smc_key, &info); if (rc || info.size != 8) { dev_warn(&spi->dev, "sep sensor: SMC key %s absent or not eight bytes\n", key); sep_smc = NULL; - return false; + return rc ?: -EINVAL; } sep_smc_nrails = n; sep_power_source = SEP_POWER_SMC_RAILS; dev_info(&spi->dev, "sep sensor: power from SMC key %s (%d rails)\n", key, n); - if (sep_smc_set_power(0)) { + rc = sep_smc_set_power(0); + if (rc) { sep_power_source = SEP_POWER_NONE; sep_smc_nrails = 0; sep_smc = NULL; - return false; + return rc; } - return true; + return 0; } #else static int sep_smc_set_power(int on) @@ -169,12 +191,14 @@ static int sep_smc_set_power(int on) return -ENODEV; } -static bool sep_acquire_smc_power(struct spi_device *spi) +static int sep_acquire_smc_power(struct spi_device *spi) { if (spi->dev.of_node && - of_property_present(spi->dev.of_node, "apple,smc-power-key")) + of_property_present(spi->dev.of_node, "apple,smc-power-key")) { dev_warn(&spi->dev, "sep sensor: SMC power needs CONFIG_MFD_MACSMC\n"); - return false; + return -ENODEV; + } + return -ENOENT; } #endif @@ -182,21 +206,22 @@ static bool sep_acquire_smc_power(struct spi_device *spi) * Takes the power line as an output driven low: the power cycle begins with an * off phase, so the line must be actively driven off, not merely read as low. */ -static void sep_acquire_power(struct spi_device *spi) +static int sep_acquire_power(struct spi_device *spi) { struct device_node *np; struct gpio_device *gdev; struct gpio_chip *gc; - if (sep_acquire_smc_power(spi)) - return; + if (spi->dev.of_node && + of_property_present(spi->dev.of_node, "apple,smc-power-key")) + return sep_acquire_smc_power(spi); sep_power = gpiod_get_index(&spi->dev, NULL, 0, GPIOD_OUT_LOW); if (!IS_ERR(sep_power)) { sep_power_source = SEP_POWER_NODE_PROPERTY; dev_info(&spi->dev, "sep sensor: power line from the device node, driven low\n"); - return; + return 0; } sep_power = NULL; @@ -208,7 +233,7 @@ static void sep_acquire_power(struct spi_device *spi) if (!of_machine_is_compatible("apple,j414s")) { dev_warn(&spi->dev, "sep sensor: no power GPIO in the device node; describe gpios in DT\n"); - return; + return 0; } np = of_find_node_by_path(SEP_SENSOR_GPIO_NODE); @@ -216,7 +241,7 @@ static void sep_acquire_power(struct spi_device *spi) dev_warn(&spi->dev, "sep sensor: no DT node at %s, no power line\n", SEP_SENSOR_GPIO_NODE); - return; + return 0; } gdev = gpio_device_find_by_fwnode(of_fwnode_handle(np)); @@ -225,7 +250,7 @@ static void sep_acquire_power(struct spi_device *spi) dev_warn(&spi->dev, "sep sensor: %s has no registered GPIO device\n", SEP_SENSOR_GPIO_NODE); - return; + return 0; } gc = gpio_device_get_chip(gdev); @@ -242,7 +267,7 @@ static void sep_acquire_power(struct spi_device *spi) SEP_SENSOR_GPIO_LINE, SEP_SENSOR_GPIO_NODE, gpio_device_get_label(gdev)); gpio_device_put(gdev); - return; + return 0; } sep_power_source = SEP_POWER_CHIP_LINE; @@ -251,6 +276,7 @@ static void sep_acquire_power(struct spi_device *spi) SEP_SENSOR_GPIO_NODE, SEP_SENSOR_GPIO_LINE, gpio_device_get_label(gdev)); gpio_device_put(gdev); + return 0; } static void sep_release_power(void) @@ -350,8 +376,10 @@ static int sep_sensor_probe(struct spi_device *spi) if (rc) return rc; + rc = sep_acquire_power(spi); + if (rc) + return dev_err_probe(&spi->dev, rc, "sep sensor: power unavailable\n"); sep_spi = spi; - sep_acquire_power(spi); /* Read the mode back: a mode that did not take must not be logged as * the one requested. */ dev_info(&spi->dev, diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 74749f8b9d984..166c1413eb4ca 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -615,8 +615,6 @@ struct SepData { #[pin] scrd_wq: CondVar, - sensor_present: Atomic, - #[pin] bio_session: Mutex, @@ -807,7 +805,6 @@ impl SepData { sbio_rx <- new_mutex!(transfer::Reassembly::new()), sbio_wq <- new_condvar!("SepData::sbio_wq"), sbio_ready: Atomic::new(false), - sensor_present: Atomic::new(false), bio_session <- new_mutex!(bio::Session::new()), bio_index <- new_mutex!(bio_index), bio_dev <- new_mutex!(None), From f840525d953887c701d5f55d8a232f7a368ede19 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 01:21:08 +1000 Subject: [PATCH 0720/2680] soc: apple: sep: declare the mailbox and firmware-loader dependencies The driver uses the Rust Apple mailbox bindings and the firmware loader abstractions but declares neither, so it only builds when another driver (today APPLE_AOP) happens to pull them in. It also relies on runtime PM without depending on PM. Depend on PM and RUST_APPLE_MAILBOX and select RUST_FW_LOADER_ABSTRACTIONS. RUST_APPLE_MAILBOX has to be a dependency rather than a select: selecting it pulls in APPLE_MAILBOX, which NVME_APPLE depends on, and with NVME_APPLE selecting BLK_INLINE_ENCRYPTION for FileVault that closes a recursive dependency through CRYPTO and HW_RANDOM. Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index f9c7498abd593..728bc96dec04d 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -115,7 +115,9 @@ config APPLE_SEP_FV_INTERFACE config APPLE_SEP tristate "Apple SEP (Secure Enclave Processor)" depends on ARCH_APPLE + depends on PM depends on RUST + depends on RUST_APPLE_MAILBOX depends on HW_RANDOM depends on TRUSTED_KEYS depends on CRYPTO @@ -126,6 +128,7 @@ config APPLE_SEP select CRYPTO_GCM select CRYPTO_ECDH select APPLE_SEP_FV_INTERFACE + select RUST_FW_LOADER_ABSTRACTIONS help Driver for the Apple SEP (Secure Enclave Processor) on Apple silicon. It drives the Touch ID fingerprint sensor over a /dev/sep-bio character From 85bca015afbaf08c75530815513fdc96001abb77 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 01:21:35 +1000 Subject: [PATCH 0721/2680] dt-bindings: input: apple,mesa-fingerprint: Name the SMC that switches the rails Boards whose Touch ID sensor rails are switched by the System Management Controller describe the SMC key and rail words, but not which SMC. The driver looked one up by the generic "apple,smc" compatible, which the SMC binding reserves for T8103, T8112 and T6000; newer SoCs such as T8140 fall back to "apple,t8103-smc" instead, so the lookup can never succeed there. Add an apple,smc phandle, require it together with the power key and rail words, and add an example for the MacBook Neo (J700). Signed-off-by: Ryan Murray --- .../input/apple,mesa-fingerprint.yaml | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml b/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml index 68531cf93ba0b..53b304bcec7ff 100644 --- a/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml +++ b/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml @@ -37,6 +37,12 @@ properties: maxItems: 1 description: Power line to the sensor, asserted to power it up. + apple,smc: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The System Management Controller that switches the rails named by + apple,smc-power-key and apple,smc-power-rails. + apple,smc-power-key: $ref: /schemas/types.yaml#/definitions/string description: @@ -70,6 +76,7 @@ oneOf: - required: - enable-gpios - required: + - apple,smc - apple,smc-power-key - apple,smc-power-rails @@ -94,3 +101,26 @@ examples: firmware-name = "apple/mesacal-j313.bin"; }; }; + + - | + #include + + spi { + #address-cells = <1>; + #size-cells = <0>; + + fingerprint@0 { + compatible = "apple,mesa-fingerprint"; + reg = <0>; + spi-max-frequency = <8000000>; + spi-cpol; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <135 IRQ_TYPE_EDGE_RISING>; + apple,smc = <&smc>; + apple,smc-power-key = "pcLD"; + apple,smc-power-rails = <0x01010000 0x01020000>; + firmware-name = "apple/mesacal-j700.bin"; + }; + }; From 2740f454702cf065b93bc698101b10c0d9103f4a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 01:21:35 +1000 Subject: [PATCH 0722/2680] soc: apple: sep: find the sensor's SMC through its phandle sep_acquire_smc_power() searched the whole tree for the generic "apple,smc" compatible. SoCs after T8112 do not carry that compatible (see apple,smc.yaml), so on the MacBook Neo the search failed and the sensor deferred its probe forever without saying why. Take the SMC from the sensor node's apple,smc phandle instead, and report a missing phandle as a device-tree error rather than deferring. Signed-off-by: Ryan Murray --- drivers/soc/apple/sensor_shim.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index 9ed3658cda8f7..60cef6eee9e30 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -148,9 +148,11 @@ static int sep_acquire_smc_power(struct spi_device *spi) return -EINVAL; } - smc_np = of_find_compatible_node(NULL, NULL, "apple,smc"); - if (!smc_np) - return -EPROBE_DEFER; + smc_np = of_parse_phandle(np, "apple,smc", 0); + if (!smc_np) { + dev_warn(&spi->dev, "sep sensor: apple,smc-power-key needs an apple,smc phandle\n"); + return -EINVAL; + } smc_pdev = of_find_device_by_node(smc_np); of_node_put(smc_np); if (!smc_pdev) From 2da99aa1f14d5ec517f9c7fa2ef9558e6995682b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 03:01:36 +1000 Subject: [PATCH 0723/2680] dt-bindings: soc: apple: Add the Secure Enclave Processor The SEP node ("apple,sep") is used by the T8103, T8112, T602x and T8140 device trees and matched by the SEP driver, but has no binding, so dtbs_check rejects it. Describe it: the register window, the ASC mailbox, the DART the SEP reaches shared memory through, its power domain, and the properties the bootloader adds (the SEP firmware carveout and the two boot manifests the SEP needs to open the machine's secure storage). Signed-off-by: Ryan Murray --- .../bindings/soc/apple/apple,sep.yaml | 78 +++++++++++++++++++ MAINTAINERS | 1 + 2 files changed, 79 insertions(+) create mode 100644 Documentation/devicetree/bindings/soc/apple/apple,sep.yaml diff --git a/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml b/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml new file mode 100644 index 0000000000000..989ee4b9e9078 --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,sep.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple Secure Enclave Processor + +maintainers: + - Ryan Murray + +description: + The Secure Enclave Processor (SEP) is a coprocessor on Apple SoCs that + holds device keys, provides a hardware random number generator and runs + the Touch ID matcher. The bootloader starts it. The operating system talks + to it through an ASC mailbox and through shared memory that the SEP + reaches via its own DART. + +properties: + compatible: + const: apple,sep + + reg: + maxItems: 1 + + mboxes: + maxItems: 1 + + mbox-names: + const: mbox + + iommus: + maxItems: 1 + + power-domains: + maxItems: 1 + + memory-region: + maxItems: 1 + description: + The SEP firmware carveout, added by the bootloader. + + iboot-manifest: + $ref: /schemas/types.yaml#/definitions/uint8-array + description: + The signed iBoot manifest of the running boot, added by the bootloader. + The SEP needs it to open the machine's secure storage. + + local-policy-manifest: + $ref: /schemas/types.yaml#/definitions/uint8-array + description: + The signed local boot policy of the running boot, added by the + bootloader, for the same purpose. + +required: + - compatible + - reg + - mboxes + - mbox-names + - iommus + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + sep@282600000 { + compatible = "apple,sep"; + reg = <0x2 0x82600000 0x0 0x88000>; + mboxes = <&sep_mbox>; + mbox-names = "mbox"; + iommus = <&sep_dart 0>; + power-domains = <&ps_sep>; + }; + }; diff --git a/MAINTAINERS b/MAINTAINERS index fc1651b6d8159..e68de97d6b9f7 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2618,6 +2618,7 @@ F: Documentation/devicetree/bindings/power/reset/apple,smc-reboot.yaml F: Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml F: Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml F: Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,sep.yaml F: Documentation/devicetree/bindings/spi/apple,spi.yaml F: Documentation/devicetree/bindings/spmi/apple,spmi.yaml F: Documentation/devicetree/bindings/usb/apple,dwc3.yaml From 040ff425d3fd6642015e730f416f1998e88e5758 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 21:27:50 +1000 Subject: [PATCH 0724/2680] soc: apple: sep: Describe observed behaviour, not other implementations Two notes in this series described another operating system's implementation instead of what was observed on the hardware: the keybag encoding comment named where the other implementation places the bag type, and the xART safety notes cited an inspection of its kernel code and asked for records to be checked against it. Describe the enclave's behaviour directly, drop the inspection paragraph, and state the remaining checks as hardware tests. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/profile.rs | 7 ++++--- tools/aurora-sep/XART-SAFETY.md | 12 +++--------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 2c3c8c1e70db5..ee2f43096740d 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -90,9 +90,10 @@ pub(crate) struct SensorProfile { /// Identity keybag `CREATE_KEYBAG` field encoding. The strict T8103 enclave /// reads the request's first word as the bag type and rejects the value the -/// lenient T6020 enclave accepts there; macOS carries the type in the third -/// word instead. Kept per-SoC so the strict path is correct without disturbing -/// the proven T6020 encoding (hardware-verified for enrol, match, and reboot). +/// lenient T6020 enclave accepts there; the strict enclave takes the type in +/// the third word instead. Kept per-SoC so the strict path is correct without +/// disturbing the proven T6020 encoding (hardware-verified for enrol, match, +/// and reboot). pub(crate) struct KeybagCreate { /// First word: request variant, echoed back in the reply. pub(crate) variant: u32, diff --git a/tools/aurora-sep/XART-SAFETY.md b/tools/aurora-sep/XART-SAFETY.md index 52a976c63e5c4..be7d5229748f7 100644 --- a/tools/aurora-sep/XART-SAFETY.md +++ b/tools/aurora-sep/XART-SAFETY.md @@ -20,21 +20,15 @@ state. Do not enable it on other installations without equivalent checks. locator failure: the highest-scoring window begins 90 APFS blocks after the true file start and extends beyond `.gl`. This proves ambiguity in that locator, **not** the cause of the previously damaged installation. -- Static inspection of the macOS 27.2 AppleSEPManager kernel collection found - `gl_rec_write` allocating a fresh 0x9000-byte slot, writing it, then clearing - 0x1000 bytes at the former slot. The low-level write path also calls a disk - cache-synchronization ioctl. `gl_fixup_rev_init` validates records and - handles duplicates during open. These observations do not prove that our - APFS ownership and failure ordering match macOS. ## Before authorizing writes -1. Confirm the exact macOS vnode/raw-device and extent-locking contract, +1. Confirm the raw-device access and extent-locking behaviour on hardware, including whether any APFS copy-on-write, snapshot, encryption, or remap state can invalidate the physical mapping during Linux's lifetime. 2. Verify record ordering, duplicate selection, failed-write recovery, - barriers/cache synchronization, and revision behavior against both macOS - code and controlled non-production tests. + barriers/cache synchronization, and revision behavior in controlled + non-production tests. 3. Compare a pre/post macOS-and-Linux power-cycle trace, with APFS checksums, `.gl` mapping and records checked before and after. Keep a recoverable image of the iBoot partition before the first write experiment. From c062b4a03e24635a2fa03f8a21605129019ac3ed Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:47:16 +1100 Subject: [PATCH 0725/2680] soc: apple: sep: acquire the sensor enable GPIO by name The fingerprint binding uses enable-gpios. Request that named GPIO first while retaining the legacy unnamed property and board-specific fallback. Propagate acquisition errors and defer an unavailable GPIO supplier. Refuse binding when no usable power control can be obtained. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sensor_shim.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index 60cef6eee9e30..5738312663b79 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -218,7 +218,15 @@ static int sep_acquire_power(struct spi_device *spi) of_property_present(spi->dev.of_node, "apple,smc-power-key")) return sep_acquire_smc_power(spi); - sep_power = gpiod_get_index(&spi->dev, NULL, 0, GPIOD_OUT_LOW); + sep_power = gpiod_get(&spi->dev, "enable", GPIOD_OUT_LOW); + if (IS_ERR(sep_power) && PTR_ERR(sep_power) == -ENOENT) + sep_power = gpiod_get_index(&spi->dev, NULL, 0, GPIOD_OUT_LOW); + if (IS_ERR(sep_power) && PTR_ERR(sep_power) != -ENOENT) { + int error = PTR_ERR(sep_power); + + sep_power = NULL; + return error; + } if (!IS_ERR(sep_power)) { sep_power_source = SEP_POWER_NODE_PROPERTY; dev_info(&spi->dev, @@ -230,12 +238,12 @@ static int sep_acquire_power(struct spi_device *spi) /* * The fallback below is the j414s power line; a sibling board's sensor sits * on a different pin, so only j414s may use it. Every other machine must - * describe the power GPIO in its device node (the gpiod_get_index path). + * describe enable-gpios in its device node. */ if (!of_machine_is_compatible("apple,j414s")) { dev_warn(&spi->dev, - "sep sensor: no power GPIO in the device node; describe gpios in DT\n"); - return 0; + "sep sensor: no power GPIO in the device node; describe enable-gpios in DT\n"); + return -ENODEV; } np = of_find_node_by_path(SEP_SENSOR_GPIO_NODE); @@ -243,7 +251,7 @@ static int sep_acquire_power(struct spi_device *spi) dev_warn(&spi->dev, "sep sensor: no DT node at %s, no power line\n", SEP_SENSOR_GPIO_NODE); - return 0; + return -ENODEV; } gdev = gpio_device_find_by_fwnode(of_fwnode_handle(np)); @@ -252,7 +260,7 @@ static int sep_acquire_power(struct spi_device *spi) dev_warn(&spi->dev, "sep sensor: %s has no registered GPIO device\n", SEP_SENSOR_GPIO_NODE); - return 0; + return -EPROBE_DEFER; } gc = gpio_device_get_chip(gdev); @@ -269,7 +277,7 @@ static int sep_acquire_power(struct spi_device *spi) SEP_SENSOR_GPIO_LINE, SEP_SENSOR_GPIO_NODE, gpio_device_get_label(gdev)); gpio_device_put(gdev); - return 0; + return -ENODEV; } sep_power_source = SEP_POWER_CHIP_LINE; From 661df2ed5bccd4f7430edac2aa6772de4710bad9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:47:19 +1100 Subject: [PATCH 0726/2680] soc: apple: sep: retain the sensor SPI mode from the board An unconditional mode-2 assignment overrides the mode-1 J313 device tree. Retain the board mode unless a diagnostic override is supplied, and reject override values outside the SPI mode range. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sensor_shim.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index 5738312663b79..b0e33c8af2857 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -21,18 +21,13 @@ #include "shim.h" -/* - * 8 MHz, SPI mode 2 (CPOL=1 CPHA=0), 8-bit words. Mode 3 returns sixteen zero - * bytes, else identical. Must agree with the node's spi-cpol / absent spi-cpha, - * which is what spi_setup() applies. - */ +/* The board selects the SPI mode; an override is diagnostic only. */ #define SEP_SENSOR_HZ 8000000 #define SEP_SENSOR_BITS 8 -static unsigned int sep_sensor_mode = SPI_MODE_2; +static unsigned int sep_sensor_mode = ~0U; module_param_named(sensor_spi_mode, sep_sensor_mode, uint, 0444); MODULE_PARM_DESC(sensor_spi_mode, - "SPI mode for the sensor: 2 (CPOL=1 CPHA=0, the default and the only mode this sensor answers under) or 3 (CPOL=1 CPHA=1, reads all-zero). For comparison only."); -#define SEP_SENSOR_MODE sep_sensor_mode + "Diagnostic SPI mode override (0-3); default retains the board mode"); /* Chip-select setup and hold, in ns. */ #define SEP_SENSOR_CS_NS 20 @@ -340,11 +335,15 @@ static int sep_apply_cs_timing(struct spi_device *spi) * select. spi_setup() takes the controller io_mutex, not the bus lock, * so no deadlock. */ + if (sep_sensor_mode != ~0U && sep_sensor_mode > SPI_MODE_3) + return -EINVAL; + rc = spi_bus_lock(ctlr); if (rc) return rc; - spi->mode = SEP_SENSOR_MODE; + if (sep_sensor_mode != ~0U) + spi->mode = sep_sensor_mode; spi->bits_per_word = SEP_SENSOR_BITS; spi->max_speed_hz = SEP_SENSOR_HZ; spi->cs_setup.value = SEP_SENSOR_CS_NS; From df73e975e781cec9bf27ae8a3d0ff328491e0a6b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:47:21 +1100 Subject: [PATCH 0727/2680] soc: apple: sep: wake detached waiters under the lifetime lock The last file release can free the biometric character device after unregister drops its mutex. Wake detached waiters before releasing that mutex so unregister cannot access the wait queue after destruction. The open-count decision still leaves exactly one path responsible for freeing the device. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/bio_shim.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/bio_shim.c b/drivers/soc/apple/bio_shim.c index 50f7e7ddf61ea..476c9e86cb6c9 100644 --- a/drivers/soc/apple/bio_shim.c +++ b/drivers/soc/apple/bio_shim.c @@ -168,9 +168,9 @@ void sep_bio_unregister(void *dev) mutex_lock(&d->lock); d->ctx = NULL; free_now = (d->open_count == 0); - mutex_unlock(&d->lock); - + /* The last close may free d as soon as the lifetime lock is dropped. */ wake_up_interruptible(&d->wq); + mutex_unlock(&d->lock); if (free_now) { mutex_destroy(&d->lock); From a9a526ee4aaf4e023c5c464c13ae52c862339630 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:52:46 +1100 Subject: [PATCH 0728/2680] soc: apple: sep: drain captures before tearing down the sensor Reject new entry points and protocol sends during shutdown, wake and abort blocked transactions, and join capture work before powering down or unregistering the sensor. Use a device lifetime mutex and a protected sensor registry so independent SPI unbind also waits for exported sensor operations before releasing their resources. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 16 +- drivers/soc/apple/sensor_shim.c | 265 ++++++++++++++++++++++++++++++-- drivers/soc/apple/sep.rs | 26 +++- drivers/soc/apple/sks.rs | 3 +- 4 files changed, 284 insertions(+), 26 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index c91938b60fee1..aae0afcf33e15 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2188,7 +2188,7 @@ impl SepData { name: &'static CStr, payload: &[u8], ) -> Result { - if !self.sbio_ready.load(Relaxed) { + if self.shutting_down.load(Relaxed) || !self.sbio_ready.load(Relaxed) { return Err(ENODEV); } @@ -2204,6 +2204,11 @@ impl SepData { let mut remaining = time::msecs_to_jiffies(SBIO_TIMEOUT_MS); let mut guard = self.sbio_rx.lock(); loop { + if self.shutting_down.load(Relaxed) { + guard.abort(); + let _ = guard.take_done(); + return Err(ENODEV); + } if let Some(done) = guard.take_done() { return Ok(done); } @@ -2298,6 +2303,9 @@ impl SepData { let mut remaining = time::msecs_to_jiffies(SBIO_TIMEOUT_MS); let mut guard = self.sbio_rx.lock(); loop { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } if guard.take_grant() { return Ok(()); } @@ -2346,6 +2354,9 @@ impl SepData { } pub(crate) fn bio_open(&self) -> Result<()> { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } let mut session = self.bio_session.lock(); bio::open(&mut session)?; drop(session); @@ -2405,6 +2416,9 @@ impl SepData { } pub(crate) fn bio_ioctl(&self, cmd: u32, arg: usize) -> Result { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } if cmd == bio::IOC_ATTEST { return self.bio_attest(arg); } diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index b0e33c8af2857..e52acb69b04f0 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -19,6 +19,9 @@ #include #include +#include +#include + #include "shim.h" /* The board selects the SPI mode; an override is diagnostic only. */ @@ -35,6 +38,28 @@ MODULE_PARM_DESC(sensor_spi_mode, #define SEP_SENSOR_OFF_DELAY_MS 10 #define SEP_SENSOR_ON_DELAY_MS 7 +struct sep_sensor_lifetime { + /* Serializes sensor operations with this device's resource teardown. */ + struct mutex lock; +}; + +static DEFINE_MUTEX(sep_sensor_registry_lock); +static struct sep_sensor_lifetime *sep_active_sensor; + +static struct sep_sensor_lifetime *sep_lock_sensor(void) +{ + struct sep_sensor_lifetime *sensor; + + mutex_lock(&sep_sensor_registry_lock); + sensor = sep_active_sensor; + if (sensor) + mutex_lock(&sensor->lock); + mutex_unlock(&sep_sensor_registry_lock); + return sensor; +} + +static void sep_sensor_irq_teardown_locked(void); + static struct spi_device *sep_spi; static struct gpio_desc *sep_power; static bool sep_registered; @@ -377,7 +402,7 @@ static int sep_apply_cs_timing(struct spi_device *spi) return 0; } -static int sep_sensor_probe(struct spi_device *spi) +static int sep_sensor_probe_locked(struct spi_device *spi) { int rc; @@ -399,11 +424,41 @@ static int sep_sensor_probe(struct spi_device *spi) return 0; } +static int sep_sensor_probe(struct spi_device *spi) +{ + struct sep_sensor_lifetime *sensor; + int error; + + sensor = devm_kzalloc(&spi->dev, sizeof(*sensor), GFP_KERNEL); + if (!sensor) + return -ENOMEM; + mutex_init(&sensor->lock); + mutex_lock(&sep_sensor_registry_lock); + if (sep_active_sensor) { + error = -EBUSY; + } else { + error = sep_sensor_probe_locked(spi); + if (!error) { + spi_set_drvdata(spi, sensor); + sep_active_sensor = sensor; + } + } + mutex_unlock(&sep_sensor_registry_lock); + return error; +} + static void sep_sensor_remove(struct spi_device *spi) { - sep_sensor_irq_teardown(); + struct sep_sensor_lifetime *sensor = spi_get_drvdata(spi); + + mutex_lock(&sep_sensor_registry_lock); + mutex_lock(&sensor->lock); + sep_active_sensor = NULL; + sep_sensor_irq_teardown_locked(); sep_release_power(); sep_spi = NULL; + mutex_unlock(&sensor->lock); + mutex_unlock(&sep_sensor_registry_lock); } /* @@ -460,19 +515,19 @@ void sep_sensor_unregister(void) sep_registered = false; } -int sep_sensor_bound(void) +static int sep_sensor_bound_locked(void) { return sep_spi != NULL; } /* Whether those delays reach hardware, are emulated, or are silently bypassed. */ -int sep_sensor_cs_timing_mode(void) +static int sep_sensor_cs_timing_mode_locked(void) { return sep_cs_timing_mode; } /* Power cycle: off, 10 ms, on, 7 ms. -ENODEV if there is no power line. */ -int sep_sensor_power_cycle(void) +static int sep_sensor_power_cycle_locked(void) { if (sep_power_source == SEP_POWER_SMC_RAILS) { int rc = sep_smc_set_power(0); @@ -497,12 +552,12 @@ int sep_sensor_power_cycle(void) } /* How the power line was obtained: none, the node's property, or chip+line. */ -int sep_sensor_power_source(void) +static int sep_sensor_power_source_locked(void) { return sep_power_source; } -int sep_sensor_power_line(void) +static int sep_sensor_power_line_locked(void) { if (sep_power_source == SEP_POWER_SMC_RAILS) return -ENODEV; @@ -516,7 +571,7 @@ int sep_sensor_power_line(void) * the sensor node's "firmware-name" property. NULL when absent, so the caller * can fall back to a default. */ -const char *sep_sensor_firmware_name(void) +static const char *sep_sensor_firmware_name_locked(void) { const char *name = NULL; @@ -528,7 +583,7 @@ const char *sep_sensor_firmware_name(void) } /* Powers the sensor on/off, holding the hardware settling delay. */ -int sep_sensor_power(int on) +static int sep_sensor_power_locked(int on) { if (sep_power_source == SEP_POWER_SMC_RAILS) { int rc = sep_smc_set_power(on ? 1 : 0); @@ -567,7 +622,7 @@ static irqreturn_t sep_drdy_isr(int irq, void *dev_id) * firmware patch. Returns 0 when an interrupt is available, -errno otherwise * (the caller then falls back to polling). */ -int sep_sensor_irq_setup(void) +static int sep_sensor_irq_setup_locked(void) { struct device_node *np; struct gpio_device *gdev; @@ -640,7 +695,7 @@ int sep_sensor_irq_setup(void) return 0; } -void sep_sensor_irq_teardown(void) +static void sep_sensor_irq_teardown_locked(void) { if (sep_drdy_irq >= 0) { free_irq(sep_drdy_irq, sep_spi); @@ -652,14 +707,14 @@ void sep_sensor_irq_teardown(void) } } -int sep_sensor_irq_available(void) +static int sep_sensor_irq_available_locked(void) { return sep_drdy_irq >= 0; } /* Clear any stale signal before a capture so the next wait reflects a fresh * data-ready edge, not a leftover from the previous frame. */ -void sep_sensor_irq_arm(void) +static void sep_sensor_irq_arm_locked(void) { if (sep_drdy_irq >= 0) reinit_completion(&sep_drdy_done); @@ -670,7 +725,7 @@ void sep_sensor_irq_arm(void) * -ETIMEDOUT = no signal (the caller re-polls status over SPI regardless, so a * missed interrupt only costs one poll interval), -ENODEV = no interrupt. */ -int sep_sensor_irq_wait(unsigned int timeout_ms) +static int sep_sensor_irq_wait_locked(unsigned int timeout_ms) { if (sep_drdy_irq < 0) return -ENODEV; @@ -685,7 +740,7 @@ int sep_sensor_irq_wait(unsigned int timeout_ms) * `len` transmit bytes (trailing ones as 0xff) rather than relying on the * controller running its transmit buffer dry. */ -int sep_sensor_xfer(const void *tx, void *rx, size_t len) +static int sep_sensor_xfer_locked(const void *tx, void *rx, size_t len) { struct spi_transfer xfer = { .tx_buf = tx, @@ -708,7 +763,7 @@ int sep_sensor_xfer(const void *tx, void *rx, size_t len) * A simultaneous receive changes the long transfer's pacing and the sensor * rejects the patch blob, so the receive must be absent entirely. */ -int sep_sensor_xfer_tx(const void *tx, size_t len) +static int sep_sensor_xfer_tx_locked(const void *tx, size_t len) { struct spi_transfer xfer = { .tx_buf = tx, @@ -730,7 +785,7 @@ int sep_sensor_xfer_tx(const void *tx, size_t len) * Two transfers in one chip-select assertion: command out, then read in. * Chip-select stays asserted because neither transfer sets cs_change. */ -int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len) +static int sep_sensor_xfer2_locked(const void *tx, size_t tx_len, void *rx, size_t rx_len) { struct spi_transfer xfers[2] = { { @@ -754,3 +809,179 @@ int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len) return spi_sync_transfer(sep_spi, xfers, 2); } + +int sep_sensor_bound(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return 0; + result = sep_sensor_bound_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_cs_timing_mode(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return 0; + result = sep_sensor_cs_timing_mode_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_power_cycle(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_power_cycle_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_power_source(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return 0; + result = sep_sensor_power_source_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_power_line(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_power_line_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +const char *sep_sensor_firmware_name(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + const char *result; + + if (!sensor) + return NULL; + result = sep_sensor_firmware_name_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_power(int on) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_power_locked(on); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_irq_setup(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_irq_setup_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +void sep_sensor_irq_teardown(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + + if (!sensor) + return; + sep_sensor_irq_teardown_locked(); + mutex_unlock(&sensor->lock); +} + +int sep_sensor_irq_available(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return 0; + result = sep_sensor_irq_available_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +void sep_sensor_irq_arm(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + + if (!sensor) + return; + sep_sensor_irq_arm_locked(); + mutex_unlock(&sensor->lock); +} + +int sep_sensor_irq_wait(unsigned int timeout_ms) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_irq_wait_locked(timeout_ms); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_xfer(const void *tx, void *rx, size_t len) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_xfer_locked(tx, rx, len); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_xfer_tx(const void *tx, size_t len) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_xfer_tx_locked(tx, len); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_xfer2_locked(tx, tx_len, rx, rx_len); + mutex_unlock(&sensor->lock); + return result; +} diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 166c1413eb4ca..0214538a9b469 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -1005,6 +1005,9 @@ impl SepData { } fn send(&self, msg: Message) -> Result<()> { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } self.mbox.lock().as_ref().ok_or(ENODEV)?.send(msg, false) } @@ -1021,6 +1024,10 @@ impl SepData { let mut remaining = time::msecs_to_jiffies(op.timeout_ms()); let mut guard = self.control.lock(); loop { + if self.shutting_down.load(Relaxed) { + guard.abandon(idx); + return Err(ENODEV); + } if let Some(reply) = guard.take_reply(idx) { guard.release(idx); drop(guard); @@ -1448,7 +1455,8 @@ impl SepData { let mut remaining = time::msecs_to_jiffies(ms); let mut guard = self.scrd_probe.lock(); loop { - if guard.captured.len() >= until || remaining == 0 { + if self.shutting_down.load(Relaxed) + || guard.captured.len() >= until || remaining == 0 { return; } match self @@ -1916,16 +1924,16 @@ impl SepData { fn remove(&self) { self.shutting_down.store(true, Relaxed); - self.release_firmware_mapping(); + bio::release(&mut self.bio_session.lock()); + self.rng_shutdown.store(true, Relaxed); + self.control_wq.notify_all(); + self.sbio_wq.notify_all(); + self.sks_wq.notify_all(); + self.scrd_wq.notify_all(); trusted::unregister(); self.unregister_fv_kernel(); let dev = self.bio_dev.lock().take(); - if dev.is_some() { - bio::release(&mut self.bio_session.lock()); - sensor::power(false); - sensor::unregister_driver(); - } drop(dev); self.rng_shutdown.store(true, Relaxed); @@ -1950,6 +1958,10 @@ impl SepData { ); } + sensor::power(false); + sensor::unregister_driver(); + self.release_firmware_mapping(); + let _ = self.store.lock().take(); let _ = self.host_store.lock().take(); diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index 03caee8f0767b..505c59228ae3f 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -884,7 +884,8 @@ impl SepData { let mut remaining = time::msecs_to_jiffies(ms); let mut guard = self.sks_probe.lock(); loop { - if guard.captured.len() >= until || remaining == 0 { + if self.shutting_down.load(Relaxed) + || guard.captured.len() >= until || remaining == 0 { return; } match self From 803cad80640bdbc14ccfa78849183c5f534bb4f8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:00:10 +1100 Subject: [PATCH 0729/2680] soc: apple: sep: bind capture results to their queued operation Assign a generation to each queued capture and retain worker ownership across cancellation and close. Refuse a replacement operation until the worker has retired, and check the queued generation before accepting progress, guidance or a terminal result. This prevents an old verify result from being bound to a new request nonce and serializes enrol and verify work against the shared sensor and buffers. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/bio.rs | 66 +++++++++++++++++++++++++++++++++------ drivers/soc/apple/sbio.rs | 32 +++++++++++++++---- drivers/soc/apple/sep.rs | 16 ++++++---- 3 files changed, 92 insertions(+), 22 deletions(-) diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index d115a18d64d97..464022c681f1f 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -417,6 +417,8 @@ impl Op { } pub(crate) struct Session { + generation: u64, + worker_epoch: Option, open: bool, op: Op, unseen: bool, @@ -427,6 +429,8 @@ pub(crate) struct Session { impl Session { pub(crate) fn new() -> Session { Session { + generation: 0, + worker_epoch: None, open: false, op: Op::Idle, unseen: false, @@ -436,6 +440,7 @@ impl Session { } fn reset(&mut self) { + self.generation = self.generation.saturating_add(1); self.open = false; self.op = Op::Idle; self.unseen = false; @@ -443,6 +448,15 @@ impl Session { self.token = None; } + fn start_worker(&mut self) -> Result<()> { + if self.worker_epoch.is_some() { + return Err(EBUSY); + } + self.generation = self.generation.checked_add(1).ok_or(EOVERFLOW)?; + self.worker_epoch = Some(self.generation); + Ok(()) + } + pub(crate) fn ready(&self) -> bool { self.unseen } @@ -560,7 +574,7 @@ fn enrol_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { } require_sensor(ctx.sensor_present)?; - if !ctx.session.op.may_start() { + if ctx.session.worker_epoch.is_some() || !ctx.session.op.may_start() { return Err(EBUSY); } if matches!(ctx.live_identity_count, Some(count) if count >= SEP_USER_CAPACITY) @@ -569,6 +583,7 @@ fn enrol_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { return Err(ENOSPC); } + ctx.session.start_worker()?; ctx.session.op = Op::Enrol { label: request.label, stage: 0, @@ -590,10 +605,14 @@ fn enrol_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { pub(crate) fn enrol_advance( session: &mut Session, + epoch: u64, completed: u32, percent_now: u32, guide: Guidance, ) -> bool { + if !worker_is_current(session, epoch) { + return false; + } if let Op::Enrol { stage, terminal, @@ -614,7 +633,10 @@ pub(crate) fn enrol_advance( false } -pub(crate) fn capture_guide(session: &mut Session, guide: Guidance) -> bool { +pub(crate) fn capture_guide(session: &mut Session, epoch: u64, guide: Guidance) -> bool { + if !worker_is_current(session, epoch) { + return false; + } let changed = match &mut session.op { Op::Enrol { terminal, guidance, .. } => { if terminal.is_none() && *guidance != guide { @@ -642,9 +664,13 @@ pub(crate) fn capture_guide(session: &mut Session, guide: Guidance) -> bool { pub(crate) fn enrol_finish( session: &mut Session, + epoch: u64, index: &mut IdentityIndex, outcome: core::result::Result<[u8; UUID_LEN], u32>, ) -> bool { + if !worker_is_current(session, epoch) { + return false; + } let Op::Enrol { label, terminal, .. } = &mut session.op @@ -670,8 +696,8 @@ pub(crate) fn enrol_finish( true } -pub(crate) fn enrol_is_live(session: &Session) -> bool { - matches!(&session.op, Op::Enrol { terminal: None, .. }) && session.open +pub(crate) fn enrol_is_live(session: &Session, epoch: u64) -> bool { + worker_is_current(session, epoch) && matches!(&session.op, Op::Enrol { terminal: None, .. }) } fn enrol_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { @@ -732,7 +758,7 @@ fn verify_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { } require_sensor(ctx.sensor_present)?; - if !ctx.session.op.may_start() { + if ctx.session.worker_epoch.is_some() || !ctx.session.op.may_start() { return Err(EBUSY); } @@ -754,6 +780,7 @@ fn verify_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { }); } + ctx.session.start_worker()?; ctx.session.token = None; ctx.session.op = Op::Verify { nonce: request.nonce, @@ -772,9 +799,13 @@ fn verify_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { pub(crate) fn verify_finish( session: &mut Session, + epoch: u64, outcome: VerifyOutcome, token_bytes: [u8; TOKEN_LEN], ) -> bool { + if !worker_is_current(session, epoch) { + return false; + } let Op::Verify { nonce, terminal, .. } = &mut session.op else { return false; }; @@ -793,12 +824,26 @@ pub(crate) fn verify_finish( true } -pub(crate) fn verify_is_live(session: &Session) -> bool { - matches!(&session.op, Op::Verify { terminal: None, .. }) && session.open +pub(crate) fn verify_is_live(session: &Session, epoch: u64) -> bool { + worker_is_current(session, epoch) && matches!(&session.op, Op::Verify { terminal: None, .. }) } -pub(crate) fn capture_is_live(session: &Session) -> bool { - enrol_is_live(session) || verify_is_live(session) +pub(crate) fn capture_is_live(session: &Session, epoch: u64) -> bool { + enrol_is_live(session, epoch) || verify_is_live(session, epoch) +} + +pub(crate) fn queued_epoch(session: &Session) -> Option { + session.worker_epoch +} + +fn worker_is_current(session: &Session, epoch: u64) -> bool { + session.open && session.generation == epoch && session.worker_epoch == Some(epoch) +} + +pub(crate) fn worker_stopped(session: &mut Session, epoch: u64) { + if session.worker_epoch == Some(epoch) { + session.worker_epoch = None; + } } fn verify_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { @@ -868,6 +913,7 @@ fn cancel(ctx: &mut Context<'_>) -> Result { Op::Verify { terminal, .. } => *terminal = Some(VerifyOutcome::Failed(STATUS_LOCAL)), } + ctx.session.generation = ctx.session.generation.saturating_add(1); ctx.session.token = None; ctx.session.unseen = true; @@ -940,7 +986,7 @@ fn delete_all(ctx: &mut Context<'_>) -> Result { } pub(crate) fn open(session: &mut Session) -> Result<()> { - if session.open { + if session.open || session.worker_epoch.is_some() { return Err(EBUSY); } session.reset(); diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index aae0afcf33e15..6c046bd4285c5 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -232,7 +232,9 @@ impl SepData { let filed = { let mut session = self.bio_session.lock(); let mut index = self.bio_index.lock(); - bio::enrol_finish(&mut session, &mut index, outcome) + bio::enrol_finish( + &mut session, self.capture_epoch.load(Relaxed), &mut index, outcome, + ) }; if !filed { return; @@ -1357,7 +1359,7 @@ impl SepData { let mut identity: Option<[u8; bio::UUID_LEN]> = None; let outcome = loop { - if !bio::enrol_is_live(&self.bio_session.lock()) { + if !bio::enrol_is_live(&self.bio_session.lock(), self.capture_epoch.load(Relaxed)) { break None; } if counter >= ENROL_MAX_CAPTURES { @@ -1374,6 +1376,7 @@ impl SepData { counter = counter.saturating_add(1); let woke = bio::enrol_advance( &mut self.bio_session.lock(), + self.capture_epoch.load(Relaxed), stage, percent, bio::Guidance::LiftAndMove, @@ -1395,7 +1398,10 @@ impl SepData { // The capture did not take (partial/unusable contact). Nudge // the person to hold still and press again rather than // silently re-capturing with no feedback. - if bio::capture_guide(&mut self.bio_session.lock(), bio::Guidance::HoldStill) { + if bio::capture_guide( + &mut self.bio_session.lock(), self.capture_epoch.load(Relaxed), + bio::Guidance::HoldStill, + ) { self.bio_wake(); } } @@ -2025,7 +2031,7 @@ impl SepData { ); let mut attempt: u32 = 0; loop { - if !bio::capture_is_live(&self.bio_session.lock()) { + if !bio::capture_is_live(&self.bio_session.lock(), self.capture_epoch.load(Relaxed)) { return CaptureWait::Abandon; } @@ -2050,7 +2056,9 @@ impl SepData { _ => None, }; if let Some(guide) = guide { - if bio::capture_guide(&mut self.bio_session.lock(), guide) { + if bio::capture_guide( + &mut self.bio_session.lock(), self.capture_epoch.load(Relaxed), guide, + ) { self.bio_wake(); } } @@ -2375,15 +2383,24 @@ impl SepData { } pub(crate) fn queue_enrolment(this: Arc) { + let Some(epoch) = bio::queued_epoch(&this.bio_session.lock()) else { + return; + }; + this.capture_epoch.store(epoch, Relaxed); if workqueue::system() .enqueue::, ENROL_WORK_ID>(this.clone()) .is_err() { this.finish_enrolment(Err(ENROL_STATUS_SENSOR)); + bio::worker_stopped(&mut this.bio_session.lock(), epoch); } } pub(crate) fn queue_verify(this: Arc) { + let Some(epoch) = bio::queued_epoch(&this.bio_session.lock()) else { + return; + }; + this.capture_epoch.store(epoch, Relaxed); if workqueue::system() .enqueue::, VERIFY_WORK_ID>(this.clone()) .is_err() @@ -2392,11 +2409,14 @@ impl SepData { bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), [0u8; bio::TOKEN_LEN], ); + bio::worker_stopped(&mut this.bio_session.lock(), epoch); } } fn finish_verify(&self, outcome: bio::VerifyOutcome, token_bytes: [u8; bio::TOKEN_LEN]) { - let filed = bio::verify_finish(&mut self.bio_session.lock(), outcome, token_bytes); + let filed = bio::verify_finish( + &mut self.bio_session.lock(), self.capture_epoch.load(Relaxed), outcome, token_bytes, + ); if filed { self.bio_wake(); } diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 0214538a9b469..5b8532347cfe2 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -659,6 +659,7 @@ struct SepData { registered: Atomic, shutting_down: Atomic, + capture_epoch: Atomic, #[pin] rx_work: Work, @@ -824,6 +825,7 @@ impl SepData { settle_idle_ticks: Atomic::new(0), registered: Atomic::new(false), shutting_down: Atomic::new(false), + capture_epoch: Atomic::new(0), rx_work <- new_work!("SepData::rx_work"), enrol_work <- new_work!("SepData::enrol_work"), verify_work <- new_work!("SepData::verify_work"), @@ -2114,10 +2116,11 @@ impl WorkItem for SepData { type Pointer = Arc; fn run(this: Arc) { - if this.shutting_down.load(Relaxed) { - return; + let epoch = this.capture_epoch.load(Relaxed); + if !this.shutting_down.load(Relaxed) { + this.run_enrolment(); } - this.run_enrolment(); + bio::worker_stopped(&mut this.bio_session.lock(), epoch); } } @@ -2125,10 +2128,11 @@ impl WorkItem for SepData { type Pointer = Arc; fn run(this: Arc) { - if this.shutting_down.load(Relaxed) { - return; + let epoch = this.capture_epoch.load(Relaxed); + if !this.shutting_down.load(Relaxed) { + this.run_verify(); } - this.run_verify(); + bio::worker_stopped(&mut this.bio_session.lock(), epoch); } } From 613b614029bd171de6d3a4c9eff0269b4830fe7b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:02:11 +1100 Subject: [PATCH 0730/2680] soc: apple: sep: export the biometric ioctl header through UAPI Move the public biometric interface into include/uapi/linux and use conventional ioctl macro spelling without changing its fixed-width layouts or ioctl values. Make the libfprint patch consume the exported header instead of maintaining a duplicated ABI definition, and document header installation for its build. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 3 +- drivers/soc/apple/sep-bio.h | 130 ---------- include/uapi/linux/apple-sep.h | 133 +++++++++++ tools/aurora-sep/README.md | 8 + .../libfprint-1.94.100-apple-sep.patch | 223 ++++-------------- 5 files changed, 187 insertions(+), 310 deletions(-) delete mode 100644 drivers/soc/apple/sep-bio.h create mode 100644 include/uapi/linux/apple-sep.h diff --git a/MAINTAINERS b/MAINTAINERS index e68de97d6b9f7..acf9f692840a4 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2632,7 +2632,6 @@ F: drivers/cpufreq/apple-soc-cpufreq.c F: drivers/dma/apple-admac.c F: drivers/gpio/gpio-macsmc.c F: drivers/hwmon/macsmc-hwmon.c -F: drivers/pmdomain/apple/ F: drivers/i2c/busses/i2c-pasemi-core.c F: drivers/i2c/busses/i2c-pasemi-platform.c F: drivers/input/misc/macsmc-input.c @@ -2646,6 +2645,7 @@ F: drivers/nvmem/apple-efuses.c F: drivers/nvmem/apple-spmi-nvmem.c F: drivers/phy/apple/ F: drivers/pinctrl/pinctrl-apple-gpio.c +F: drivers/pmdomain/apple/ F: drivers/power/reset/macsmc-reboot.c F: drivers/power/supply/macsmc-power.c F: drivers/pwm/pwm-apple.c @@ -2664,6 +2664,7 @@ F: include/dt-bindings/pinctrl/apple.h F: include/linux/mfd/macsmc.h F: include/linux/soc/apple/* F: include/uapi/drm/asahi_drm.h +F: include/uapi/linux/apple-sep.h F: net/bluetooth/brcm.c F: net/bluetooth/brcm.h diff --git a/drivers/soc/apple/sep-bio.h b/drivers/soc/apple/sep-bio.h deleted file mode 100644 index d52b880ac9332..0000000000000 --- a/drivers/soc/apple/sep-bio.h +++ /dev/null @@ -1,130 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* Copyright 2026 Dj */ -/* - * Userspace interface for the SEP biometric device. - * - * The enclave does the matching. Nothing biometric crosses this interface: only - * an operation, a stage, a status, an opaque identity UUID, and an opaque host - * label userspace chooses. - */ - -#pragma once - -#include -#include - -#define SEP_BIO_IFACE_VERSION 4 - -#define SEP_BIO_UUID_LEN 16 -#define SEP_BIO_LABEL_LEN 128 -#define SEP_BIO_NONCE_LEN 32 -#define SEP_BIO_TOKEN_LEN 32 -#define SEP_BIO_MAX_IDENTITIES 32 -#define SEP_BIO_CHALLENGE_LEN 32 -#define SEP_BIO_ATTEST_PUB_LEN 65 -#define SEP_BIO_ATTEST_SIG_MAX 72 - -enum { - SEP_BIO_STATE_IDLE = 0, - SEP_BIO_STATE_PENDING = 1, - SEP_BIO_STATE_PROGRESS = 2, - SEP_BIO_STATE_DONE = 3, - SEP_BIO_STATE_FAILED = 4, -}; - - -enum { - SEP_BIO_NO_MATCH = 0, - SEP_BIO_MATCH = 1, - SEP_BIO_NOT_COMPARED = 2, -}; - -struct sep_bio_identity { - __u8 uuid[SEP_BIO_UUID_LEN]; - __u8 label[SEP_BIO_LABEL_LEN]; -}; - -struct sep_bio_info { - __u32 version; - __u32 sensor_present; - __u32 enrolled; - __u32 capacity; /* usable slots in this SEP user context, not ABI list size */ - __u32 enroll_stages; - __u32 reserved[3]; -}; - -/* reserved[0] flags; reserved[1] is a live SEP identity count only when valid. */ -#define SEP_BIO_INFO_LIVE_COUNT_VALID 1u - -struct sep_bio_list { - __u32 count; - __u32 reserved; - struct sep_bio_identity id[SEP_BIO_MAX_IDENTITIES]; -}; - -struct sep_bio_enrol_start { - __u32 flags; - __u32 reserved; - __u8 label[SEP_BIO_LABEL_LEN]; -}; - -#define SEP_BIO_GUIDANCE_NONE 0 -#define SEP_BIO_GUIDANCE_PLACE 1 -#define SEP_BIO_GUIDANCE_LIFT_AND_MOVE 2 -#define SEP_BIO_GUIDANCE_HOLD_STILL 3 - -struct sep_bio_enrol_poll { - __u32 state; - __u32 stage; - __u32 stages_total; - __u32 status; - __u8 uuid[SEP_BIO_UUID_LEN]; - __u32 guidance; - __u32 progress_percent; -}; -struct sep_bio_verify_start { - __u32 flags; - __u32 reserved; - __u8 nonce[SEP_BIO_NONCE_LEN]; -}; - -struct sep_bio_verify_poll { - __u32 state; - __u32 result; - __u32 status; - __u32 guidance; /* SEP_BIO_GUIDANCE_*; status only, no capture side effect */ - __u8 uuid[SEP_BIO_UUID_LEN]; - __u8 token[SEP_BIO_TOKEN_LEN]; /* single use, bound to the nonce */ - __u64 deadline_ns; /* CLOCK_MONOTONIC; past this the token is void */ -}; - -struct sep_bio_delete { - __u8 uuid[SEP_BIO_UUID_LEN]; -}; - -/* - * Device attestation of key possession: the enclave signs 'challenge' with the - * machine ref-key (ECDSA-P256 over the challenge as the pre-computed digest) and - * returns the DER signature and public point. The private key never leaves the - * enclave. - */ -struct sep_bio_attest { - __u32 sig_len; /* out: DER signature length */ - __u8 challenge[SEP_BIO_CHALLENGE_LEN]; /* in */ - __u8 public[SEP_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ - __u8 signature[SEP_BIO_ATTEST_SIG_MAX]; /* out: DER SEQUENCE{r,s} */ - __u8 reserved[3]; -}; - -#define SEP_BIO_IOC_MAGIC 0xB1 - -#define SEP_BIO_GET_INFO _IOR (SEP_BIO_IOC_MAGIC, 0x01, struct sep_bio_info) -#define SEP_BIO_LIST _IOR (SEP_BIO_IOC_MAGIC, 0x02, struct sep_bio_list) -#define SEP_BIO_ENROL_START _IOW (SEP_BIO_IOC_MAGIC, 0x03, struct sep_bio_enrol_start) -#define SEP_BIO_ENROL_POLL _IOR (SEP_BIO_IOC_MAGIC, 0x04, struct sep_bio_enrol_poll) -#define SEP_BIO_VERIFY_START _IOW (SEP_BIO_IOC_MAGIC, 0x05, struct sep_bio_verify_start) -#define SEP_BIO_VERIFY_POLL _IOR (SEP_BIO_IOC_MAGIC, 0x06, struct sep_bio_verify_poll) -#define SEP_BIO_CANCEL _IO (SEP_BIO_IOC_MAGIC, 0x07) -#define SEP_BIO_DELETE _IOW (SEP_BIO_IOC_MAGIC, 0x08, struct sep_bio_delete) -#define SEP_BIO_DELETE_ALL _IO (SEP_BIO_IOC_MAGIC, 0x09) -#define SEP_BIO_ATTEST _IOWR(SEP_BIO_IOC_MAGIC, 0x0a, struct sep_bio_attest) diff --git a/include/uapi/linux/apple-sep.h b/include/uapi/linux/apple-sep.h new file mode 100644 index 0000000000000..81467fe91f36a --- /dev/null +++ b/include/uapi/linux/apple-sep.h @@ -0,0 +1,133 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note OR MIT */ +/* Copyright 2026 Dj */ +/* + * Userspace interface for the SEP biometric device. + * + * The enclave does the matching. Nothing biometric crosses this interface: only + * an operation, a stage, a status, an opaque identity UUID, and an opaque host + * label userspace chooses. + */ + +#ifndef _UAPI_LINUX_APPLE_SEP_H +#define _UAPI_LINUX_APPLE_SEP_H + +#include +#include + +#define SEP_BIO_IFACE_VERSION 4 + +#define SEP_BIO_UUID_LEN 16 +#define SEP_BIO_LABEL_LEN 128 +#define SEP_BIO_NONCE_LEN 32 +#define SEP_BIO_TOKEN_LEN 32 +#define SEP_BIO_MAX_IDENTITIES 32 +#define SEP_BIO_CHALLENGE_LEN 32 +#define SEP_BIO_ATTEST_PUB_LEN 65 +#define SEP_BIO_ATTEST_SIG_MAX 72 + +enum { + SEP_BIO_STATE_IDLE = 0, + SEP_BIO_STATE_PENDING = 1, + SEP_BIO_STATE_PROGRESS = 2, + SEP_BIO_STATE_DONE = 3, + SEP_BIO_STATE_FAILED = 4, +}; + +enum { + SEP_BIO_NO_MATCH = 0, + SEP_BIO_MATCH = 1, + SEP_BIO_NOT_COMPARED = 2, +}; + +struct sep_bio_identity { + __u8 uuid[SEP_BIO_UUID_LEN]; + __u8 label[SEP_BIO_LABEL_LEN]; +}; + +struct sep_bio_info { + __u32 version; + __u32 sensor_present; + __u32 enrolled; + __u32 capacity; /* usable slots in this SEP user context, not ABI list size */ + __u32 enroll_stages; + __u32 reserved[3]; +}; + +/* reserved[0] flags; reserved[1] is a live SEP identity count only when valid. */ +#define SEP_BIO_INFO_LIVE_COUNT_VALID 1u + +struct sep_bio_list { + __u32 count; + __u32 reserved; + struct sep_bio_identity id[SEP_BIO_MAX_IDENTITIES]; +}; + +struct sep_bio_enrol_start { + __u32 flags; + __u32 reserved; + __u8 label[SEP_BIO_LABEL_LEN]; +}; + +#define SEP_BIO_GUIDANCE_NONE 0 +#define SEP_BIO_GUIDANCE_PLACE 1 +#define SEP_BIO_GUIDANCE_LIFT_AND_MOVE 2 +#define SEP_BIO_GUIDANCE_HOLD_STILL 3 + +struct sep_bio_enrol_poll { + __u32 state; + __u32 stage; + __u32 stages_total; + __u32 status; + __u8 uuid[SEP_BIO_UUID_LEN]; + __u32 guidance; + __u32 progress_percent; +}; + +struct sep_bio_verify_start { + __u32 flags; + __u32 reserved; + __u8 nonce[SEP_BIO_NONCE_LEN]; +}; + +struct sep_bio_verify_poll { + __u32 state; + __u32 result; + __u32 status; + __u32 guidance; /* SEP_BIO_GUIDANCE_*; status only, no capture side effect */ + __u8 uuid[SEP_BIO_UUID_LEN]; + __u8 token[SEP_BIO_TOKEN_LEN]; /* single use, bound to the nonce */ + __u64 deadline_ns; /* CLOCK_MONOTONIC; past this the token is void */ +}; + +struct sep_bio_delete { + __u8 uuid[SEP_BIO_UUID_LEN]; +}; + +/* + * Device attestation of key possession: the enclave signs 'challenge' with the + * machine ref-key (ECDSA-P256 over the challenge as the pre-computed digest) and + * returns the DER signature and public point. The private key never leaves the + * enclave. + */ +struct sep_bio_attest { + __u32 sig_len; /* out: DER signature length */ + __u8 challenge[SEP_BIO_CHALLENGE_LEN]; /* in */ + __u8 public[SEP_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ + __u8 signature[SEP_BIO_ATTEST_SIG_MAX]; /* out: DER SEQUENCE{r,s} */ + __u8 reserved[3]; +}; + +#define SEP_BIO_IOC_MAGIC 0xB1 + +#define SEP_BIO_GET_INFO _IOR(SEP_BIO_IOC_MAGIC, 0x01, struct sep_bio_info) +#define SEP_BIO_LIST _IOR(SEP_BIO_IOC_MAGIC, 0x02, struct sep_bio_list) +#define SEP_BIO_ENROL_START _IOW(SEP_BIO_IOC_MAGIC, 0x03, struct sep_bio_enrol_start) +#define SEP_BIO_ENROL_POLL _IOR(SEP_BIO_IOC_MAGIC, 0x04, struct sep_bio_enrol_poll) +#define SEP_BIO_VERIFY_START _IOW(SEP_BIO_IOC_MAGIC, 0x05, struct sep_bio_verify_start) +#define SEP_BIO_VERIFY_POLL _IOR(SEP_BIO_IOC_MAGIC, 0x06, struct sep_bio_verify_poll) +#define SEP_BIO_CANCEL _IO(SEP_BIO_IOC_MAGIC, 0x07) +#define SEP_BIO_DELETE _IOW(SEP_BIO_IOC_MAGIC, 0x08, struct sep_bio_delete) +#define SEP_BIO_DELETE_ALL _IO(SEP_BIO_IOC_MAGIC, 0x09) +#define SEP_BIO_ATTEST _IOWR(SEP_BIO_IOC_MAGIC, 0x0a, struct sep_bio_attest) + +#endif /* _UAPI_LINUX_APPLE_SEP_H */ diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index b83ef5dea2e3d..bfe086e48a2c6 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -121,3 +121,11 @@ On Omarchy, run `omarchy-apply-lock` once after enrollment. The Quickshell lock screen then selects `omarchy-lock-fingerprint` automatically and accepts Touch ID through fprintd. Hyprlock's separate fingerprint switch is not used by the Quickshell lock screen. + +## Biometric userspace header + +The canonical ioctl ABI is `include/uapi/linux/apple-sep.h`. Export this +checkout's headers with `make headers_install INSTALL_HDR_PATH=` +and add that directory's `include` to the libfprint build include path. The +libfprint patch uses `` directly; it has no copied ABI +header. Interface version, ioctl numbers and fixed-width layouts are unchanged. diff --git a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch index 1947bd095b43b..3faef0fdf1dc2 100644 --- a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch +++ b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch @@ -1,138 +1,3 @@ -diff --git a/libfprint/drivers/aurora/aurora-bio.h b/libfprint/drivers/aurora/aurora-bio.h -new file mode 100644 -index 0000000..b2d2f43 ---- /dev/null -+++ b/libfprint/drivers/aurora/aurora-bio.h -@@ -0,0 +1,129 @@ -+/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -+/* Copyright 2026 Dj */ -+/* -+ * Userspace interface for the SEP/Mesa biometric device. -+ * -+ * The enclave does the matching. Nothing biometric crosses this interface: only -+ * an operation, a stage, a status, an opaque identity UUID, and an opaque host -+ * label userspace chooses. -+ * -+ * SPDX-License-Identifier: LGPL-2.1-or-later -+ */ -+ -+#pragma once -+ -+#include -+#include -+ -+#define AURORA_BIO_IFACE_VERSION 4 -+ -+#define AURORA_BIO_UUID_LEN 16 -+#define AURORA_BIO_LABEL_LEN 128 -+#define AURORA_BIO_NONCE_LEN 32 -+#define AURORA_BIO_TOKEN_LEN 32 -+#define AURORA_BIO_MAX_IDENTITIES 32 -+#define AURORA_BIO_CHALLENGE_LEN 32 -+#define AURORA_BIO_ATTEST_PUB_LEN 65 -+#define AURORA_BIO_ATTEST_SIG_MAX 72 -+ -+enum { -+ AURORA_BIO_STATE_IDLE = 0, -+ AURORA_BIO_STATE_PENDING = 1, -+ AURORA_BIO_STATE_PROGRESS = 2, -+ AURORA_BIO_STATE_DONE = 3, -+ AURORA_BIO_STATE_FAILED = 4, -+}; -+ -+ -+enum { -+ AURORA_BIO_NO_MATCH = 0, -+ AURORA_BIO_MATCH = 1, -+ AURORA_BIO_NOT_COMPARED = 2, -+}; -+ -+struct aurora_bio_identity { -+ __u8 uuid[AURORA_BIO_UUID_LEN]; -+ __u8 label[AURORA_BIO_LABEL_LEN]; -+}; -+ -+struct aurora_bio_info { -+ __u32 version; -+ __u32 sensor_present; -+ __u32 enrolled; -+ __u32 capacity; -+ __u32 enroll_stages; -+ __u32 reserved[3]; -+}; -+ -+struct aurora_bio_list { -+ __u32 count; -+ __u32 reserved; -+ struct aurora_bio_identity id[AURORA_BIO_MAX_IDENTITIES]; -+}; -+ -+struct aurora_bio_enrol_start { -+ __u32 flags; -+ __u32 reserved; -+ __u8 label[AURORA_BIO_LABEL_LEN]; -+}; -+ -+#define AURORA_BIO_GUIDANCE_NONE 0 -+#define AURORA_BIO_GUIDANCE_PLACE 1 -+#define AURORA_BIO_GUIDANCE_LIFT_AND_MOVE 2 -+#define AURORA_BIO_GUIDANCE_HOLD_STILL 3 -+ -+struct aurora_bio_enrol_poll { -+ __u32 state; -+ __u32 stage; -+ __u32 stages_total; -+ __u32 status; -+ __u8 uuid[AURORA_BIO_UUID_LEN]; -+ __u32 guidance; -+ __u32 progress_percent; -+}; -+struct aurora_bio_verify_start { -+ __u32 flags; -+ __u32 reserved; -+ __u8 nonce[AURORA_BIO_NONCE_LEN]; -+}; -+ -+struct aurora_bio_verify_poll { -+ __u32 state; -+ __u32 result; -+ __u32 status; -+ __u32 reserved; -+ __u8 uuid[AURORA_BIO_UUID_LEN]; -+ __u8 token[AURORA_BIO_TOKEN_LEN]; /* single use, bound to the nonce */ -+ __u64 deadline_ns; /* CLOCK_MONOTONIC; past this the token is void */ -+}; -+ -+struct aurora_bio_delete { -+ __u8 uuid[AURORA_BIO_UUID_LEN]; -+}; -+ -+/* -+ * Device attestation of key possession: the enclave signs 'challenge' with the -+ * machine ref-key (ECDSA-P256 over the challenge as the pre-computed digest) and -+ * returns the DER signature and public point. The private key never leaves the -+ * enclave. -+ */ -+struct aurora_bio_attest { -+ __u32 sig_len; /* out: DER signature length */ -+ __u8 challenge[AURORA_BIO_CHALLENGE_LEN]; /* in */ -+ __u8 public[AURORA_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ -+ __u8 signature[AURORA_BIO_ATTEST_SIG_MAX]; /* out: DER SEQUENCE{r,s} */ -+ __u8 reserved[3]; -+}; -+ -+#define AURORA_BIO_IOC_MAGIC 0xB1 -+ -+#define AURORA_BIO_GET_INFO _IOR (AURORA_BIO_IOC_MAGIC, 0x01, struct aurora_bio_info) -+#define AURORA_BIO_LIST _IOR (AURORA_BIO_IOC_MAGIC, 0x02, struct aurora_bio_list) -+#define AURORA_BIO_ENROL_START _IOW (AURORA_BIO_IOC_MAGIC, 0x03, struct aurora_bio_enrol_start) -+#define AURORA_BIO_ENROL_POLL _IOR (AURORA_BIO_IOC_MAGIC, 0x04, struct aurora_bio_enrol_poll) -+#define AURORA_BIO_VERIFY_START _IOW (AURORA_BIO_IOC_MAGIC, 0x05, struct aurora_bio_verify_start) -+#define AURORA_BIO_VERIFY_POLL _IOR (AURORA_BIO_IOC_MAGIC, 0x06, struct aurora_bio_verify_poll) -+#define AURORA_BIO_CANCEL _IO (AURORA_BIO_IOC_MAGIC, 0x07) -+#define AURORA_BIO_DELETE _IOW (AURORA_BIO_IOC_MAGIC, 0x08, struct aurora_bio_delete) -+#define AURORA_BIO_DELETE_ALL _IO (AURORA_BIO_IOC_MAGIC, 0x09) -+#define AURORA_BIO_ATTEST _IOWR(AURORA_BIO_IOC_MAGIC, 0x0a, struct aurora_bio_attest) diff --git a/libfprint/drivers/aurora/aurora.c b/libfprint/drivers/aurora/aurora.c new file mode 100644 index 0000000..80ace06 @@ -152,7 +17,7 @@ index 0000000..80ace06 +#define FP_COMPONENT "apple-sep" + +#include "drivers_api.h" -+#include "aurora-bio.h" ++#include + +#include +#include @@ -172,7 +37,7 @@ index 0000000..80ace06 + guint watch_id; + guint32 enroll_last_reported_stage; + -+ guint8 nonce[AURORA_BIO_NONCE_LEN]; ++ guint8 nonce[SEP_BIO_NONCE_LEN]; + gboolean nonce_valid; +}; + @@ -248,14 +113,14 @@ index 0000000..80ace06 + + user_id = fpi_print_generate_user_id (print); + -+ if (user_id == NULL || strlen (user_id) >= AURORA_BIO_LABEL_LEN) ++ if (user_id == NULL || strlen (user_id) >= SEP_BIO_LABEL_LEN) + { + g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, + "print label does not fit the device record"); + return FALSE; + } + -+ memset (label_out, 0, AURORA_BIO_LABEL_LEN); ++ memset (label_out, 0, SEP_BIO_LABEL_LEN); + memcpy (label_out, user_id, strlen (user_id)); + return TRUE; +} @@ -281,7 +146,7 @@ index 0000000..80ace06 + GVariant *data; + + data = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, uuid, -+ AURORA_BIO_UUID_LEN, 1); ++ SEP_BIO_UUID_LEN, 1); + + fpi_print_set_type (print, FPI_PRINT_RAW); + fpi_print_set_device_stored (print, TRUE); @@ -291,10 +156,10 @@ index 0000000..80ace06 + * the buffer before anything reads it as a string. */ + if (label != NULL) + { -+ gchar safe[AURORA_BIO_LABEL_LEN]; ++ gchar safe[SEP_BIO_LABEL_LEN]; + -+ memcpy (safe, label, AURORA_BIO_LABEL_LEN); -+ safe[AURORA_BIO_LABEL_LEN - 1] = '\0'; ++ memcpy (safe, label, SEP_BIO_LABEL_LEN); ++ safe[SEP_BIO_LABEL_LEN - 1] = '\0'; + + if (safe[0] != '\0') + fpi_print_fill_from_user_id (print, safe); @@ -320,14 +185,14 @@ index 0000000..80ace06 + } + + raw = g_variant_get_fixed_array (data, &len, 1); -+ if (raw == NULL || len != AURORA_BIO_UUID_LEN) ++ if (raw == NULL || len != SEP_BIO_UUID_LEN) + { + g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, + "enclave identity is the wrong size"); + return FALSE; + } + -+ memcpy (uuid_out, raw, AURORA_BIO_UUID_LEN); ++ memcpy (uuid_out, raw, SEP_BIO_UUID_LEN); + return TRUE; +} + @@ -342,7 +207,7 @@ index 0000000..80ace06 +static gboolean +aurora_print_has_uuid (FpPrint *print, const guint8 *uuid) +{ -+ guint8 stored[AURORA_BIO_UUID_LEN]; ++ guint8 stored[SEP_BIO_UUID_LEN]; + + if (print == NULL) + return FALSE; @@ -350,7 +215,7 @@ index 0000000..80ace06 + if (!aurora_uuid_from_print (print, stored, NULL)) + return FALSE; + -+ return memcmp (stored, uuid, AURORA_BIO_UUID_LEN) == 0; ++ return memcmp (stored, uuid, SEP_BIO_UUID_LEN) == 0; +} + +/* ------------------------------------------------------------------ */ @@ -360,7 +225,7 @@ index 0000000..80ace06 +static void +aurora_probe (FpDevice *device) +{ -+ struct aurora_bio_info info = { 0 }; ++ struct sep_bio_info info = { 0 }; + const gchar *path; + g_autofree gchar *serial = NULL; + gint fd; @@ -384,7 +249,7 @@ index 0000000..80ace06 + return; + } + -+ if (ioctl (fd, AURORA_BIO_GET_INFO, &info) < 0) ++ if (ioctl (fd, SEP_BIO_GET_INFO, &info) < 0) + { + gint err = errno; + @@ -398,13 +263,13 @@ index 0000000..80ace06 + + close (fd); + -+ if (info.version != AURORA_BIO_IFACE_VERSION) ++ if (info.version != SEP_BIO_IFACE_VERSION) + { + fpi_device_probe_complete (device, NULL, NULL, + fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, + "interface version %u, expected %u", + info.version, -+ AURORA_BIO_IFACE_VERSION)); ++ SEP_BIO_IFACE_VERSION)); + return; + } + @@ -488,7 +353,7 @@ index 0000000..80ace06 + FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); + + if (self->fd >= 0) -+ ioctl (self->fd, AURORA_BIO_CANCEL); ++ ioctl (self->fd, SEP_BIO_CANCEL); +} + +/* ------------------------------------------------------------------ */ @@ -500,11 +365,11 @@ index 0000000..80ace06 +{ + FpDevice *device = FP_DEVICE (user_data); + FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct aurora_bio_enrol_poll poll_res = { 0 }; ++ struct sep_bio_enrol_poll poll_res = { 0 }; + g_autoptr(GError) error = NULL; + FpPrint *print = NULL; + -+ if (!aurora_ioctl (self, AURORA_BIO_ENROL_POLL, &poll_res, &error)) ++ if (!aurora_ioctl (self, SEP_BIO_ENROL_POLL, &poll_res, &error)) + { + self->watch_id = 0; + fpi_device_enroll_complete (device, NULL, g_steal_pointer (&error)); @@ -513,10 +378,10 @@ index 0000000..80ace06 + + switch (poll_res.state) + { -+ case AURORA_BIO_STATE_PENDING: ++ case SEP_BIO_STATE_PENDING: + return G_SOURCE_CONTINUE; + -+ case AURORA_BIO_STATE_PROGRESS: ++ case SEP_BIO_STATE_PROGRESS: + /* fprintd calls this "enroll-stage-passed". A guidance wakeup or an + * initial stage-zero event must not masquerade as a captured sample. */ + if (poll_res.stage > self->enroll_last_reported_stage) @@ -526,13 +391,13 @@ index 0000000..80ace06 + } + return G_SOURCE_CONTINUE; + -+ case AURORA_BIO_STATE_DONE: ++ case SEP_BIO_STATE_DONE: + fpi_device_get_enroll_data (device, &print); + if (print != NULL) + { + GVariant *data = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, + poll_res.uuid, -+ AURORA_BIO_UUID_LEN, 1); ++ SEP_BIO_UUID_LEN, 1); + fpi_print_set_type (print, FPI_PRINT_RAW); + fpi_print_set_device_stored (print, TRUE); + g_object_set (print, "fpi-data", data, NULL); @@ -545,7 +410,7 @@ index 0000000..80ace06 + "enrolment produced no print")); + return G_SOURCE_REMOVE; + -+ case AURORA_BIO_STATE_FAILED: ++ case SEP_BIO_STATE_FAILED: + default: + self->watch_id = 0; + fpi_device_enroll_complete (device, NULL, @@ -560,7 +425,7 @@ index 0000000..80ace06 +aurora_enroll (FpDevice *device) +{ + FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct aurora_bio_enrol_start start = { 0 }; ++ struct sep_bio_enrol_start start = { 0 }; + g_autoptr(GError) error = NULL; + FpPrint *print = NULL; + @@ -572,7 +437,7 @@ index 0000000..80ace06 + return; + } + -+ if (!aurora_ioctl (self, AURORA_BIO_ENROL_START, &start, &error)) ++ if (!aurora_ioctl (self, SEP_BIO_ENROL_START, &start, &error)) + { + fpi_device_enroll_complete (device, NULL, g_steal_pointer (&error)); + return; @@ -598,23 +463,23 @@ index 0000000..80ace06 + * might interpret loosely. */ +static gboolean +aurora_result_is_match (FpiDeviceAurora *self, -+ const struct aurora_bio_verify_poll *res) ++ const struct sep_bio_verify_poll *res) +{ + struct timespec now; + guint64 now_ns; + gboolean token_set = FALSE; + gsize i; + -+ if (res->state != AURORA_BIO_STATE_DONE) ++ if (res->state != SEP_BIO_STATE_DONE) + return FALSE; + -+ if (res->result != AURORA_BIO_MATCH) ++ if (res->result != SEP_BIO_MATCH) + return FALSE; + + if (!self->nonce_valid) + return FALSE; + -+ for (i = 0; i < AURORA_BIO_TOKEN_LEN; i++) ++ for (i = 0; i < SEP_BIO_TOKEN_LEN; i++) + if (res->token[i] != 0) + { + token_set = TRUE; @@ -642,7 +507,7 @@ index 0000000..80ace06 +{ + FpDevice *device = FP_DEVICE (user_data); + FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct aurora_bio_verify_poll res = { 0 }; ++ struct sep_bio_verify_poll res = { 0 }; + FpiDeviceAction action = fpi_device_get_current_action (device); + g_autoptr(GError) error = NULL; + /* Not a g_autoptr: FpPrint is a GInitiallyUnowned and both report functions @@ -651,7 +516,7 @@ index 0000000..80ace06 + * match completes. */ + FpPrint *matched = NULL; + -+ if (!aurora_ioctl (self, AURORA_BIO_VERIFY_POLL, &res, &error)) ++ if (!aurora_ioctl (self, SEP_BIO_VERIFY_POLL, &res, &error)) + { + self->watch_id = 0; + /* Terminal path: void the per-operation nonce like every other exit, so @@ -665,8 +530,8 @@ index 0000000..80ace06 + return G_SOURCE_REMOVE; + } + -+ if (res.state == AURORA_BIO_STATE_PENDING || -+ res.state == AURORA_BIO_STATE_PROGRESS) ++ if (res.state == SEP_BIO_STATE_PENDING || ++ res.state == SEP_BIO_STATE_PROGRESS) + return G_SOURCE_CONTINUE; + + self->watch_id = 0; @@ -686,7 +551,7 @@ index 0000000..80ace06 + fpi_device_verify_report (device, FPI_MATCH_SUCCESS, matched, NULL); + fpi_device_verify_complete (device, NULL); + } -+ else if (res.result == AURORA_BIO_NOT_COMPARED) ++ else if (res.result == SEP_BIO_NOT_COMPARED) + { + /* No comparison happened, so there is no verdict to report. Saying + * FAIL here would tell the user their finger was rejected when it @@ -723,7 +588,7 @@ index 0000000..80ace06 + } + } + -+ if (matched == NULL && res.result == AURORA_BIO_NOT_COMPARED) ++ if (matched == NULL && res.result == SEP_BIO_NOT_COMPARED) + { + fpi_device_identify_complete (device, + fpi_device_error_new (FP_DEVICE_ERROR_GENERAL)); @@ -745,7 +610,7 @@ index 0000000..80ace06 +aurora_match_start (FpDevice *device) +{ + FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct aurora_bio_verify_start start = { 0 }; ++ struct sep_bio_verify_start start = { 0 }; + FpiDeviceAction action = fpi_device_get_current_action (device); + g_autoptr(GError) error = NULL; + FpPrint *print = NULL; @@ -805,7 +670,7 @@ index 0000000..80ace06 + + memcpy (start.nonce, self->nonce, sizeof (start.nonce)); + -+ if (!aurora_ioctl (self, AURORA_BIO_VERIFY_START, &start, &error)) ++ if (!aurora_ioctl (self, SEP_BIO_VERIFY_START, &start, &error)) + { + self->nonce_valid = FALSE; + if (action == FPI_DEVICE_ACTION_VERIFY) @@ -827,18 +692,18 @@ index 0000000..80ace06 +aurora_list (FpDevice *device) +{ + FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct aurora_bio_list list = { 0 }; ++ struct sep_bio_list list = { 0 }; + g_autoptr(GError) error = NULL; + GPtrArray *prints; + guint32 i; + -+ if (!aurora_ioctl (self, AURORA_BIO_LIST, &list, &error)) ++ if (!aurora_ioctl (self, SEP_BIO_LIST, &list, &error)) + { + fpi_device_list_complete (device, NULL, g_steal_pointer (&error)); + return; + } + -+ if (list.count > AURORA_BIO_MAX_IDENTITIES) ++ if (list.count > SEP_BIO_MAX_IDENTITIES) + { + fpi_device_list_complete (device, NULL, + fpi_device_error_new_msg (FP_DEVICE_ERROR_PROTO, @@ -863,7 +728,7 @@ index 0000000..80ace06 +aurora_delete (FpDevice *device) +{ + FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct aurora_bio_delete del = { 0 }; ++ struct sep_bio_delete del = { 0 }; + g_autoptr(GError) error = NULL; + FpPrint *print = NULL; + @@ -878,7 +743,7 @@ index 0000000..80ace06 + return; + } + -+ if (!aurora_ioctl (self, AURORA_BIO_DELETE, &del, &error)) ++ if (!aurora_ioctl (self, SEP_BIO_DELETE, &del, &error)) + { + fpi_device_delete_complete (device, g_steal_pointer (&error)); + return; @@ -893,7 +758,7 @@ index 0000000..80ace06 + FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); + g_autoptr(GError) error = NULL; + -+ if (!aurora_ioctl (self, AURORA_BIO_DELETE_ALL, NULL, &error)) ++ if (!aurora_ioctl (self, SEP_BIO_DELETE_ALL, NULL, &error)) + { + fpi_device_clear_storage_complete (device, g_steal_pointer (&error)); + return; From 07d1e884dca9737dd21f2dc33e3b7ff0e1f93a20 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 19:42:31 +1100 Subject: [PATCH 0731/2680] soc: apple: sep: establish authenticated sensor sessions on J700 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The J700 enclave restores stored identities but cannot open a capture session without the sensor's nonce and signed root-key container. Repeated sensor registration must precede CLEAR_STATE, and communication initialization must run before restoring the biometric state. Describe the nonce session in the existing platform profile and perform its initialization, platform identity, sensor serial, nonce and root-key steps through the existing SBIO and SPI transports. Validate the complete nonce frame with the existing CRC reader and refuse a missing identity, firmware or unsuccessful exchange. Load the externally supplied signed key once per enclave lifetime; do not log identity or cryptographic payloads. For this protocol, apply encrypted sensor parameters before calibration and complete initialization afterwards. Use calibration source 6 while retaining the existing source and ordering for other profiles. Enable owner export and user-before-master saves for J700, then refresh the device view and match credential after restoring its components. Retain the existing xART write policy. Protocol facts come from our private research and Linux observations. This implementation is independently written; no private driver code or device material is included. [ryan: adapted to 7.2.10 — split by topic: the MAINTAINERS move of include/uapi/linux/apple-sep.h into the APPLE T8140 SOC AND J700 section is left to 900-config-tooling, which creates that section] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/profile.rs | 6 ++ drivers/soc/apple/sbio.rs | 190 ++++++++++++++++++++++++++++++----- drivers/soc/apple/sensor.rs | 12 +++ drivers/soc/apple/sep.rs | 3 + 4 files changed, 185 insertions(+), 26 deletions(-) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index ee2f43096740d..91ad39a45b6d0 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj +// Copyright 2026 Ryan Murray //! Typed platform profiles for the SEP driver. //! @@ -124,6 +125,8 @@ pub(crate) struct PlatformProfile { pub(crate) firmware_region: &'static CStr, /// Identity keybag CREATE_KEYBAG field encoding (per-SoC; see [`KeybagCreate`]). pub(crate) keybag_create: KeybagCreate, + /// Require nonce authentication and signed sensor material before capture. + pub(crate) nonce_session: bool, } const T8103: PlatformProfile = PlatformProfile { @@ -151,6 +154,7 @@ const T8103: PlatformProfile = PlatformProfile { bag_type: 0x20000, arg: 0, }, + nonce_session: false, }; const T6020: PlatformProfile = PlatformProfile { @@ -178,6 +182,7 @@ const T6020: PlatformProfile = PlatformProfile { bag_type: 0, arg: -1, }, + nonce_session: false, }; /// MacBook Neo. The J700 ADT records a pre-booted SEP, a spi2 Mesa sensor @@ -209,6 +214,7 @@ const T8140: PlatformProfile = PlatformProfile { bag_type: 0, arg: -1, }, + nonce_session: true, }; static_assert!(T8103.shmem_capacity == 0x30000); diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 6c046bd4285c5..fd8bbe36ff764 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj +// Copyright 2026 Ryan Murray //! Touch ID: the biometric endpoint (SBIO) and its `/dev/sep-bio` interface. //! The enclave matches; no biometric image ever crosses to userspace. @@ -367,12 +368,9 @@ impl SepData { } } - // The J414s owner exported successfully from state 0x3 in a bounded - // no-finger probe, although an earlier pre-UUID boot saw the master - // refuse an early save from the same state. Keep this path opt-in - // until a new enrollment and reboot prove that the owner blob belongs - // to the new user context. - if *module_parameters::j414s_persistent_enrol.value() != 0 { + // J700 needs the owner component even when state 0x3 has no + // save-pending bit. Retain the opt-in policy on older profiles. + if self.profile.nonce_session || *module_parameters::j414s_persistent_enrol.value() != 0 { if owner_initial_state == Some(0x3) && !self.save_catacomb( crate::sbio::CatacombUser::OWNER, @@ -761,7 +759,7 @@ impl SepData { } fn save_all_components(&self, user: crate::sbio::UserId) -> bool { - if *module_parameters::j414s_persistent_enrol.value() != 0 { + if self.profile.nonce_session || *module_parameters::j414s_persistent_enrol.value() != 0 { return self.save_all_components_user_first(user); } // On m2fix11, a completed enrollment marked both master and user 0x7, @@ -1006,10 +1004,42 @@ impl SepData { self.register_ool(&self.ool_sbio)?; self.sbio_ready.store(true, Relaxed); + if self.profile.nonce_session { + let word = 6u32; + // Communication must be initialized before restoring biometric state. + let result = self.sbio_transfer_raw( + 0x73, c"INIT_SBIO_COMMUNICATION", &word.to_le_bytes(), + ); + match result { + Ok(done) if done.status.is_ok() => { + dev_info!(self.dev, "sbio: communication initialization accepted\n"); + } + Ok(done) => { + self.sbio_ready.store(false, Relaxed); + dev_err!(self.dev, "sbio: communication init refused: status {}, {} bytes\n", + done.status, done.payload.len()); + return Err(EIO); + } + Err(e) => { + self.sbio_ready.store(false, Relaxed); + dev_err!(self.dev, "sbio: communication init transport failed: {:?}\n", e); + return Err(e); + } + } + } Ok(()) } fn attach_bringup(&self) { + if self.profile.nonce_session { + if !self.bring_sensor_online() { + dev_warn!( + self.dev, + "sensor: authenticated attach incomplete; capture unavailable\n" + ); + } + return; + } if !sensor::is_bound() { dev_warn!( self.dev, @@ -1033,18 +1063,34 @@ impl SepData { return false; }; + // Newer sensors require the encrypted parameters and transparent + // channel before the calibration challenge can be serviced. + let parameters = if self.profile.nonce_session { + match self.apply_sensor_parameters() { + Some(parameters) => Some(parameters), + None => return false, + } + } else { + None + }; + if !self.sensor_calibrated.load(Relaxed) { if !self.calibrate_sensor() { return false; } self.sensor_calibrated.store(true, Relaxed); - let Some(reloaded_patch) = self.wake_sensor() else { - return false; - }; - patch = reloaded_patch; + if parameters.is_none() { + let Some(reloaded_patch) = self.wake_sensor() else { + return false; + }; + patch = reloaded_patch; + } } - let ok = self.complete_bringup(patch); + let ok = match parameters { + Some(parameters) => self.finish_sensor_setup(patch, parameters), + None => self.complete_bringup(patch), + }; // The sensor is patched and idle here -- the only safe moment to // configure the data-ready interrupt, which is then left alone for the // driver's life. Interrupt capture is the default; a machine that does @@ -1457,6 +1503,22 @@ impl SepData { let stage = self.bringup.load(Relaxed); dev_info!(self.dev, "sbio: registering sensor at bring-up stage {}\n", stage); + if self.profile.nonce_session { + // J700 needs identification before CLEAR_STATE on every reset. + let platform = match Self::sensor_platform_identity() { + Ok(platform) => platform, + Err(error) => { + dev_err!(self.dev, "sensor: platform identity unavailable: {:?}\n", error); + return false; + } + }; + if !self.sensor_session_request(0x48, c"PLATFORM_IDENTITY", &platform) + || self.sbio_expect_ok(&crate::sbio::sbio_register_sensor(id)).is_none() + { + return false; + } + } + if stage >= BRINGUP_ESTABLISHED { let _ = self.log_identity_count(c"before CLEAR_STATE for sensor re-registration"); if self @@ -1468,7 +1530,7 @@ impl SepData { let _ = self.log_identity_count(c"after CLEAR_STATE for sensor re-registration"); } - if stage == BRINGUP_FRESH { + if stage == BRINGUP_FRESH && !self.profile.nonce_session { let op = crate::sbio::sbio_register_sensor(id); match self.sbio_call(&op) { SbioOutcome::Ok(_payload) => { @@ -1490,6 +1552,9 @@ impl SepData { let op = crate::sbio::sbio_register_sensor_serial(&serial); match self.sbio_call(&op) { SbioOutcome::Ok(_payload) => { + if self.profile.nonce_session && !self.authenticate_sensor_session() { + return false; + } self.bringup.store(BRINGUP_ESTABLISHED, Relaxed); let _ = self.log_identity_count(c"after sensor serial registration"); true @@ -1498,6 +1563,70 @@ impl SepData { } } + fn sensor_platform_identity() -> Result<[u8; 12]> { + let node = kernel::of::root().ok_or(ENODEV)?; + let property: KVec = node.get_property(c"serial-number")?; + let length = property.iter().position(|byte| *byte == 0).ok_or(EINVAL)?; + let mut identity = [0; 12]; + if length == 0 || length > identity.len() || property.len() != length + 1 { + return Err(EINVAL); + } + identity[..length].copy_from_slice(&property[..length]); + Ok(identity) + } + + fn sensor_session_request(&self, opcode: u16, name: &'static CStr, bytes: &[u8]) -> bool { + match self.sbio_transfer_raw(opcode, name, bytes) { + Ok(reply) if reply.status.is_ok() => true, + Ok(reply) => { + dev_err!(self.dev, "sensor: {} refused with status {}\n", name, reply.status); + false + } + Err(error) => { + dev_err!(self.dev, "sensor: {} transfer failed: {:?}\n", name, error); + false + } + } + } + + fn authenticate_sensor_session(&self) -> bool { + let nonce = match sensor::session_nonce() { + Ok(nonce) => nonce, + Err(error) => { + dev_err!(self.dev, "sensor: nonce read failed: {:?}\n", error); + return false; + } + }; + if !self.sensor_session_request(0x81, c"SENSOR_NONCE", &nonce) { + return false; + } + + if !self.sensor_key_loaded.load(Relaxed) { + let firmware = match kernel::firmware::Firmware::request( + c"apple/msrk-j700.bin", &self.dev, + ) { + Ok(firmware) => firmware, + Err(error) => { + dev_err!( + self.dev, + "sensor: signed root-key container unavailable: {:?}\n", + error + ); + return false; + } + }; + if firmware.data().is_empty() + || !self.sensor_session_request(0x86, c"SENSOR_ROOT_KEY", firmware.data()) + { + return false; + } + self.sensor_key_loaded.store(true, Relaxed); + } + // Each identification starts a new sensor-side exchange, even though + // the signed key remains loaded for the lifetime of the enclave. + self.establish_session() + } + fn wake_sensor(&self) -> Option { let _source = sensor::power_source(); let cycled = sensor::power_cycle(); @@ -1607,22 +1736,16 @@ impl SepData { return false; }; - let request = match crate::sbio::SbioCalibration::new(&blob) { + let source = if self.profile.nonce_session { 6 } else { SBIO_CALIBRATION_SOURCE }; + let request = match crate::sbio::SbioCalibration::new(&blob, source) { Ok(request) => request, Err(_) => { return false; } }; - if self - .sbio_transfer_raw(request.opcode(), request.name(), request.payload()) - .ok() - .filter(|done| done.status.is_ok()) - .is_none() - { - dev_err!( - self.dev, - "sensor: 0x5b LOAD_CALIBRATION failed. Without it every capture is refused with status 1.\n" - ); + if !self.sensor_session_request(request.opcode(), request.name(), request.payload()) { + dev_err!(self.dev, "sensor: calibration refused (source {}, {} bytes)\n", + source, request.payload().len()); return false; } @@ -1636,6 +1759,10 @@ impl SepData { return false; }; + self.finish_sensor_setup(patch, params) + } + + fn finish_sensor_setup(&self, patch: PatchLoaded, params: ParametersApplied) -> bool { let op = crate::sbio::sbio_complete_init(patch, params); if self.sbio_expect_ok(&op).is_none() { dev_err!( @@ -2542,6 +2669,17 @@ impl SepData { let restored = self.restore_all_components(); dev_info!(self.dev, "sbio: cold-match preparation {}, component restore {}\n", prepared, restored); + // Restore can replace the credential and the device's identity view. + // Refresh both after loading J700's persistent components, before + // admitting a verify operation. + let match_context = if self.profile.nonce_session { + let synced = restored && self.sync_device_view(); + self.device_view_synced.store(synced, Relaxed); + synced && crate::sbio::UserId::new(SBIO_PROBE_USER_ID) + .is_some_and(|user| self.establish_scrd_match_context(user)) + } else { + match_context + }; let identities = self.log_identity_count(c"after Catacomb restore with sensor registered"); let identity_ready = matches!(identities, Some(n) if n > 0); let ready = prepared && match_context && restored && identity_ready @@ -2949,10 +3087,10 @@ pub(crate) const SBIO_CALIBRATION_SOURCE: u32 = 3; pub(crate) struct SbioCalibration(KVec); impl SbioCalibration { - pub(crate) fn new(blob: &crate::CalibrationBlob) -> Result { + pub(crate) fn new(blob: &crate::CalibrationBlob, source: u32) -> Result { let bytes = blob.bytes(); let mut body = KVec::new(); - body.extend_from_slice(&SBIO_CALIBRATION_SOURCE.to_le_bytes(), GFP_KERNEL)?; + body.extend_from_slice(&source.to_le_bytes(), GFP_KERNEL)?; body.extend_from_slice(bytes, GFP_KERNEL)?; Ok(SbioCalibration(body)) } diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index 8fec6b71c8715..3fcba35e39428 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj +// Copyright 2026 Ryan Murray //! The fingerprint sensor over SPI. @@ -581,6 +582,17 @@ pub(crate) fn read_sensor_serial() -> Result { Ok(SensorSerial(serial)) } +/// Read the nonce as one frame; a separate acknowledgment would consume it. +pub(crate) fn session_nonce() -> Result<[u8; 32]> { + command(&[0x80, 0x35, 0, 7, 0, 0, 0])?; + let mut value = [0; 32]; + read_framed_verified(9 + value.len() + CRC_LEN, 9, &mut value)?; + if value.iter().all(|byte| *byte == 0) { + return Err(EIO); + } + Ok(value) +} + pub(crate) fn read_session_reply() -> Result<[u8; SESSION_SHARE_LEN]> { let mut share = [0u8; SESSION_SHARE_LEN]; read_framed(SESSION_REPLY_LEN, SESSION_REPLY_AT, &mut share)?; diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 5b8532347cfe2..1ee00ab8111f8 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj +// Copyright 2026 Ryan Murray //! Driver for the Apple SEP (Secure Enclave Processor): attaches to a running //! SEP over the AP mailbox, drives Touch ID enrol/verify via `/dev/sep-bio`, //! and registers a hwrng and a SEP-backed `trusted` key source. @@ -577,6 +578,7 @@ struct SepData { templates_restored: Atomic, sensor_calibrated: Atomic, + sensor_key_loaded: Atomic, enrol_open: Atomic, @@ -787,6 +789,7 @@ impl SepData { touchid_failed: Atomic::new(false), enrol_open: Atomic::new(false), sensor_calibrated: Atomic::new(false), + sensor_key_loaded: Atomic::new(false), templates_restored: Atomic::new(false), enrol_material <- new_mutex!(None), enrol_identity_candidates <- new_mutex!(KVec::new()), From ae0b3503784c3b5c9fc250ddbf45c9f85e87af60 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:34:33 -0700 Subject: [PATCH 0732/2680] soc: apple: sep: initialise the T8103 key store as init_sep_endpoint does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The T8103 stub boots macOS 13.5 (22G74)'s sepOS, whose key-store protocol is the one the 13.5 AppleSEPKeyStore speaks, not the newer one the T6020 enclave answers. Name that per profile (profile::KeyStore: Sepos13 for T8103, Variant5 for T6020) so the following changes can select the 13.5 request shapes without touching the T6020 encoding. On the 13.5 key store, AppleKeyStore::init_sep_endpoint (0xfffffe0009946cdc) runs before any other request: - 0x4d GET_CAPABILITIES; the IPC header version becomes min(word, 2) (0x...6dc0-6dd0). Version 2 is laid out as version 1 but __payload_hash (0xfffffe000996b970) digests the header through the trailer (0x...ba68-ba80), so image::Version gains V2. - set_env(false) (0xfffffe000994a168): op 0x2a, struct version 0, u64 1, and a 0x40c-byte environment {1, cpx-encryption-mode, 4, 0x400 zero bytes} (0x...a1a0-a1e0, __ipc_set_env 0x...9ddc, _code_ipc_set_env 0x...4b64). The reply is the struct-version word alone. Do the same at bring-up on T8103, before provisioning and before /dev/sep-bio is published. The mode comes from the profile: the j293 and j313 ADTs carry /defaults cpx-encryption-mode = 2, and the Linux device tree does not mirror /defaults. Where Apple would go on after a failed 0x4d with header version 1, this keeps the key store closed for the boot. macOS sends 0x4d once, at endpoint init, so the lock-state health check no longer re-sends it on T8103; it requires the init to have completed. T6020 sends nothing new: sks_init_endpoint returns at once, the header version stays 1 and the health check is unchanged. Hardware-verified on a MacBook Air (j313) with the 13.5 SEP: 0x4d answered status 0, struct version 0, capability word 2, empty blob (header version 2), and set_env with cpx-encryption-mode 2 was accepted. Claude-Session: https://claude.ai/code/session_01AJd8eEwJ4TGRSHnbKcWBmw [ryan: rebased onto the linear SEP history — the profile table has meanwhile gained the J700 nonce-session field and the T8140 profile (1ae0ac48652a, 77f41d244521); the new key_store field is added next to nonce_session, and T8140 gets KeyStore::Variant5, as aurora-wip composed it in 42ae23434d4c] Signed-off-by: Ryan Murray --- drivers/soc/apple/image.rs | 6 ++ drivers/soc/apple/profile.rs | 22 ++++++ drivers/soc/apple/sbio.rs | 7 ++ drivers/soc/apple/sep.rs | 8 +++ drivers/soc/apple/sks.rs | 131 ++++++++++++++++++++++++++++++++--- 5 files changed, 166 insertions(+), 8 deletions(-) diff --git a/drivers/soc/apple/image.rs b/drivers/soc/apple/image.rs index 8d4889268179d..290a9ea43dd3e 100644 --- a/drivers/soc/apple/image.rs +++ b/drivers/soc/apple/image.rs @@ -59,18 +59,24 @@ static_assert!(HEADER_WIRE == 4 + HEADER_SIZE as usize); #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum Version { V1, + /// Negotiated from the `0x4d` capability word. Laid out as version 1, but + /// `__payload_hash` (0xfffffe000996b970) digests the header through the + /// trailer (0x…ba68-ba80) where version 1 stops before it (0x…ba84-ba90). + V2, } impl Version { pub(crate) const fn wire(self) -> u32 { match self { Version::V1 => 1, + Version::V2 => 2, } } const fn digest_end(self) -> usize { match self { Version::V1 => OFF_TRAILER, + Version::V2 => HEADER_SIZE as usize, } } } diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 91ad39a45b6d0..a23445b9bea5b 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -105,6 +105,20 @@ pub(crate) struct KeybagCreate { pub(crate) arg: i32, } +/// Which key-store protocol the enclave speaks. It follows the sepOS the +/// firmware hands the SEP, not the SoC alone: the T8103 stub boots macOS 13.5 +/// (22G74)'s sepOS, whose request shapes are those of the 13.5 +/// `AppleSEPKeyStore`; the T6020 sepOS is newer. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum KeyStore { + /// macOS 13.5's key store. Before any other request the endpoint is + /// initialised as `AppleKeyStore::init_sep_endpoint` does it: `0x4d`, then + /// `set_env` carrying the ADT's `/defaults` `cpx-encryption-mode`. + Sepos13 { cpx_encryption_mode: u32 }, + /// The hardware-verified T6020 encoding (variant-5 create). + Variant5, +} + pub(crate) struct PlatformProfile { pub(crate) name: &'static str, /// Capacity of the boot shared-memory window. The layout is still checked @@ -127,6 +141,8 @@ pub(crate) struct PlatformProfile { pub(crate) keybag_create: KeybagCreate, /// Require nonce authentication and signed sensor material before capture. pub(crate) nonce_session: bool, + /// Key-store protocol generation (see [`KeyStore`]). + pub(crate) key_store: KeyStore, } const T8103: PlatformProfile = PlatformProfile { @@ -154,6 +170,10 @@ const T8103: PlatformProfile = PlatformProfile { bag_type: 0x20000, arg: 0, }, + // The j293 and j313 ADTs both carry `/defaults` `cpx-encryption-mode = 2`. + key_store: KeyStore::Sepos13 { + cpx_encryption_mode: 2, + }, nonce_session: false, }; @@ -182,6 +202,7 @@ const T6020: PlatformProfile = PlatformProfile { bag_type: 0, arg: -1, }, + key_store: KeyStore::Variant5, nonce_session: false, }; @@ -189,6 +210,7 @@ const T6020: PlatformProfile = PlatformProfile { /// (0x3356), and a SEP DART aperture above 4 GiB. These values describe that /// board; device-tree nodes remain authoritative for addresses and resources. const T8140: PlatformProfile = PlatformProfile { + key_store: KeyStore::Variant5, name: "T8140/J700", shmem_capacity: 0x4_0000, shmem_first_item: b"CINP", diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index fd8bbe36ff764..91395a40e68d7 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1116,6 +1116,13 @@ impl SepData { dev_warn!(self.dev, "bringup: could not enable the key store ({:?}); keybag and ref-key operations are unavailable\n", e); return; } + if !self.sks_init_endpoint() { + dev_err!( + self.dev, + "bringup: key-store endpoint init failed; the key store stays closed\n" + ); + return; + } if *module_parameters::provision_keybag.value() != 0 { if *module_parameters::xart_writes.value() == 0 { dev_err!( diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 1ee00ab8111f8..1895fce69c439 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -555,6 +555,12 @@ struct SepData { sks_seq: Atomic, + // IPC header version negotiated by the 13.5 endpoint init; 1 otherwise. + sks_header_version: Atomic, + + // The 13.5 endpoint init (0x4d, set_env) completed. + sks_initialized: Atomic, + // u32, not u8: the kernel's Rust atomics have no u8 AtomicType phase: Atomic, @@ -781,6 +787,8 @@ impl SepData { control <- new_mutex!(control::ControlState::new()), control_wq <- new_condvar!("SepData::control_wq"), sks_seq: Atomic::new(0), + sks_header_version: Atomic::new(1), + sks_initialized: Atomic::new(false), sks_wedged: Atomic::new(0), bringup: Atomic::new(BRINGUP_FRESH), keybag_designated: Atomic::new(false), diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index 505c59228ae3f..5bd09463ab875 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -263,6 +263,15 @@ impl SepData { Some(parsed.body) } + /// The IPC header version: 1 unless the 13.5 endpoint init negotiated 2. + pub(crate) fn sks_ipc_version(&self) -> image::Version { + if self.sks_header_version.load(Relaxed) == 2 { + image::Version::V2 + } else { + image::Version::V1 + } + } + pub(crate) fn sks_timestamp_us(&self) -> u64 { crate::shim::boottime_ns() / 1000 } @@ -276,7 +285,7 @@ impl SepData { } fn sks_seal(&self, op: &crate::sks::SksOp, body: &image::Body) -> Result { - let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), body)?; + let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), body)?; let len = self.sks_image_len(&img)?; Ok(SksRequest { name: op.name(), @@ -295,6 +304,89 @@ impl SepData { self.sks_seal(&op, &body) } + /// `0x2a` set_env, as `AppleKeyStore::set_env(false)` (0xfffffe000994a168) + /// builds it: a 0x40c-byte environment `{1, mode, 4, 0x400 zero bytes}` + /// (0x…a1a0-a1e0) sent by `__ipc_set_env` (0x…9ddc) as struct version 0, + /// u64 1, blob (`_code_ipc_set_env` 0x…4b64). + fn sks_req_set_env(&self, mode: u32) -> Result { + let op = crate::sks::sks_set_env(); + let mut env = KVec::new(); + env.resize(crate::sks::SKS_SET_ENV_LEN, 0u8, GFP_KERNEL)?; + env[0..4].copy_from_slice(&1u32.to_le_bytes()); + env[4..8].copy_from_slice(&mode.to_le_bytes()); + env[8..12].copy_from_slice(&crate::sks::SKS_SET_ENV_FALSE.to_le_bytes()); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(1)?; + body.put_blob(&env)?; + self.sks_seal(&op, &body) + } + + /// `AppleKeyStore::init_sep_endpoint` (0xfffffe0009946cdc) on the 13.5 key + /// store, before any other request: `0x4d`, whose capability word sets the + /// IPC header version to `min(word, 2)` (0x…6dc0-6dd0), then `set_env` + /// (0x…6e64). Returns false if the key store must stay closed this boot. + /// The T6020 path sends nothing here. + pub(crate) fn sks_init_endpoint(&self) -> bool { + let profile::KeyStore::Sepos13 { + cpx_encryption_mode, + } = self.profile.key_store + else { + return true; + }; + + let Some(out) = self.sks_send(self.sks_req_get_capabilities()) else { + dev_err!(self.dev, "sks: GET_CAPABILITIES got no reply\n"); + return false; + }; + // Reply: struct version 0, u64 capability word, blob. + let word = self + .sks_report_response(c"GET_CAPABILITIES", &out) + .filter(|body| out.reply.status == 0 && body.len() >= 12) + .map(|body| u64::from_le_bytes(body[4..12].try_into().unwrap())); + match word { + // Version 0 leaves out the header fields __payload_hash needs. + Some(0) | None => { + dev_err!( + self.dev, + "sks: GET_CAPABILITIES status {}, no usable capability word\n", + out.reply.status + ); + return false; + } + Some(word) => { + let version = if word < 2 { 1 } else { 2 }; + self.sks_header_version.store(version, Relaxed); + dev_info!( + self.dev, + "sks: capability word {}, IPC header version {}\n", + word, + version + ); + } + } + + let Some(out) = self.sks_send(self.sks_req_set_env(cpx_encryption_mode)) else { + dev_err!(self.dev, "sks: SET_ENV got no reply\n"); + return false; + }; + // The reply is the struct-version word 0 alone (0x…4c44-4c5c). + let accepted = self + .sks_report_response(c"SET_ENV", &out) + .is_some_and(|body| body.len() == 4 && image::operation_status(body) == Some(0)); + if out.reply.status != 0 || !accepted { + dev_err!( + self.dev, + "sks: SET_ENV (cpx-encryption-mode {}) refused: status {}\n", + cpx_encryption_mode, + out.reply.status + ); + return false; + } + self.sks_initialized.store(true, Relaxed); + true + } + /// `0x0d` designate. fn sks_req_designate(&self, d: &crate::sks::Designation, secret: &[u8]) -> Result { let mut body = image::Body::new(); @@ -306,7 +398,7 @@ impl SepData { // Flags is a u64, not u32; a u32 leaves the body short -> enclave answers -13. body.put_u64(crate::sks::SKS_DESIGNATE_FLAGS)?; - let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; let msg = crate::sks::encode_sks_designate(self.sks_next_seq(), len).ok_or(EINVAL)?; Ok(SksRequest { @@ -325,7 +417,7 @@ impl SepData { body.put_u32(0)?; body.put_u64(crate::sks::SKS_CLIENT_ID)?; body.put_i32(handle.value())?; - let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; let msg = crate::sks::encode_sks_unload_keybag(self.sks_next_seq(), len).ok_or(EINVAL)?; Ok(SksRequest { @@ -362,7 +454,7 @@ impl SepData { body.put_i32(LockState::Unlocked.wire())?; body.put_blob(secret)?; body.put_u64(SKS_LOCK_STATE_FLAGS)?; - let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; let msg = crate::sks::encode_sks_change_lock_state(self.sks_next_seq(), len).ok_or(EINVAL)?; @@ -409,7 +501,7 @@ impl SepData { body.put_u64(crate::sks::SKS_CLIENT_ID)?; body.put_blob(wrapped)?; - let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; let msg = crate::sks::encode_sks_load(self.sks_next_seq(), len).ok_or(EINVAL)?; Ok(SksRequest { @@ -445,7 +537,7 @@ impl SepData { body.put_u64(0)?; body.put_u64(0)?; body.put_blob(&[])?; - let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; let msg = crate::sks::encode_sks_create(self.sks_next_seq(), len).ok_or(EINVAL)?; Ok(SksRequest { @@ -618,7 +710,7 @@ impl SepData { body.put_blob(secret)?; // Trailing u64 goes after the blob (`0x18` puts it before). body.put_u64(SKS_LOCK_STATE_FLAGS)?; - let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; let msg = crate::sks::encode_sks_change_lock_state(self.sks_next_seq(), len).ok_or(EINVAL)?; @@ -643,7 +735,7 @@ impl SepData { body.put_blob(secret)?; body.put_blob(acm_context)?; body.put_u64(0)?; - let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; let msg = crate::sks::encode_sks_verify_secret(self.sks_next_seq(), len).ok_or(EINVAL)?; Ok(SksRequest { @@ -667,6 +759,18 @@ impl SepData { } pub(crate) fn sks_health_check(&self, why: &CStr) -> Option { + // macOS 13.5 sends 0x4d once, at endpoint init, never again. + if self.profile.key_store != profile::KeyStore::Variant5 { + if !self.sks_initialized.load(Relaxed) { + dev_err!( + self.dev, + "sks: health check ({}): endpoint not initialised\n", + why + ); + return None; + } + return Some(Healthy(())); + } let Some(out) = self.sks_send(self.sks_req_get_capabilities()) else { dev_err!( self.dev, @@ -975,6 +1079,17 @@ pub(crate) fn sks_get_capabilities() -> SksOp { } } +const OP_SKS_SET_ENV: u8 = 0x2a; +pub(crate) fn sks_set_env() -> SksOp { + SksOp { + opcode: OP_SKS_SET_ENV, + name: c"SET_ENV", + } +} +pub(crate) const SKS_SET_ENV_LEN: usize = 12 + 0x400; +/// Third environment word for `set_env(false)`, the call init_sep_endpoint makes. +pub(crate) const SKS_SET_ENV_FALSE: u32 = 4; + const OP_SKS_COPY_UUID: u8 = 0x06; pub(crate) fn sks_copy_keybag_uuid() -> SksOp { SksOp { From 2df5b7d7e04a4784ca9521eeb324a4f51838be7b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:02:28 -0700 Subject: [PATCH 0733/2680] soc: apple: sep: wait for the T8103 key store as the 13.5 kext does The 13.5 kext waits 6000 ms per key-store request (sep_deliver_msg_gated: getAdjustedTimeout(0x1770), 0xfffffe0009945a14). feat/sep waits 2000 ms for a small image. On a MacBook Air (j313) a 13.5 CREATE took 1.6 s and a COPY 1.66 s, both of which write xART, so a slightly slower one would be abandoned while its writes are in flight. Use a 6000 ms floor on the 13.5 key-store profile. T6020 is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AJd8eEwJ4TGRSHnbKcWBmw --- drivers/soc/apple/sep.rs | 2 ++ drivers/soc/apple/sks.rs | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 1895fce69c439..382080091d725 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -116,6 +116,8 @@ const SCRD_MAX_CAPTURE: usize = 8; const SCRD_TIMEOUT_MS: time::Msecs = 2000; const SKS_TIMEOUT_MS: time::Msecs = 2000; +/// The 13.5 kext's per-try key-store wait. +const SKS_TIMEOUT_13_MS: time::Msecs = 6000; const SKS_TIMEOUT_PER_KIB_MS: time::Msecs = 6000; diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index 5bd09463ab875..a92f842f5d722 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -164,6 +164,15 @@ impl SepData { } let sized_ms = sks_timeout_for(img.len()); + // The 13.5 key store is waited on as sep_deliver_msg_gated does: + // getAdjustedTimeout(0x1770), 6000 ms per wait (0xfffffe0009945a14). + // On a j313 a CREATE took + // 1.6 s and a COPY 1.66 s, close enough to the 2 s base that a slow + // one would be abandoned mid-way through its xART writes. + let floor_ms = match self.profile.key_store { + profile::KeyStore::Sepos13 { .. } if floor_ms < SKS_TIMEOUT_13_MS => SKS_TIMEOUT_13_MS, + _ => floor_ms, + }; let timeout_ms = if sized_ms < floor_ms { floor_ms } else { From 607c3fca5e65be2510924b390332f59be5048bee Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:35:38 -0700 Subject: [PATCH 0734/2680] soc: apple: sep: create the T8103 identity keybag as macOS 13.5 does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The T8103 CREATE_KEYBAG matched no 13.5 call. Its first word is the codec's struct version, not a bag type, and _code_ipc_create_keybag (0xfffffe000995de6c) takes versions 0-2 only and decodes exact-length. The request sent version 0 with the variant-5 tail, type 0x20000 (no immediate in the 13.5 AppleSEPKeyStore) and parent 0 (which identity_create refuses), and the reply parser required at least 12 bytes where a 13.5 success is 8. Send what AppleKeyStore::identity_create (0xfffffe000994b2d8) sends through __ipc_create_keybag_v2 (0xfffffe0009955ce0), the shape of userspace AKSIdentityCreateFirst: version 2, the client, type 0x400000 (mov w3, #0x400000 at 0x...b4e8), parent -1, blob A = the bag secret, an empty blob B, and the 16-byte identity UUID. The reply is exactly the version word and a handle (0x...e11c-e12c). Follow identity_create after the reply as well: copy (0x02, identity_save 0xfffffe000994b1cc), then unload (0x05). No 0x06 UUID read-back is sent; the 13.5 enclave keeps the identity bag and names it by the UUID it was created with (identity_open loads it by that UUID, 0x...b11c), so the record now stores that host-generated UUID (keybag state 1, "as generated") next to the copied blob and the secret. The T6020 request, reply check and provisioning order are unchanged. Hardware-verified on a MacBook Air (j313) with the 13.5 SEP: CREATE (164- byte image) answered status 0 with handle 1, COPY returned 32 bytes, and UNLOAD answered status 0; the SEP committed the new identity to xART and accepted that store at the next boot. Claude-Session: https://claude.ai/code/session_01AJd8eEwJ4TGRSHnbKcWBmw [ryan: rebased onto the linear SEP history — this commit was developed in parallel with commits applied above; it adds exactly what it added originally, but some lines it replaces had changed in between, and those are replaced as aurora-wip did (42ae23434d4c)] Signed-off-by: Ryan Murray --- drivers/soc/apple/keybag.rs | 20 +++++++++++++ drivers/soc/apple/profile.rs | 26 ++++++++-------- drivers/soc/apple/sks.rs | 57 +++++++++++++++++++++++++++--------- 3 files changed, 76 insertions(+), 27 deletions(-) diff --git a/drivers/soc/apple/keybag.rs b/drivers/soc/apple/keybag.rs index f97f608821761..e295693421ad7 100644 --- a/drivers/soc/apple/keybag.rs +++ b/drivers/soc/apple/keybag.rs @@ -266,6 +266,26 @@ pub(crate) fn write_bag_uuid( ) } +/// Commits a bag named by the host-generated UUID it was created with: the +/// 13.5 enclave keeps an identity bag itself and loads it by that UUID. +pub(crate) fn write_generated_uuid( + slot: Slot, + wrapped: &[u8], + uuid: &[u8; UUID_LEN], + secret: &[u8], +) -> Result<()> { + if wrapped.is_empty() || wrapped.len() > MAX_WRAPPED || secret.len() > MAX_WRAPPED { + return Err(EINVAL); + } + write_record( + slot, + UuidProvenance::AsGenerated.state(), + wrapped, + uuid, + secret, + ) +} + /// Replaces only the wrapped blob. The enclave ratchets bag material while a bag /// is active, so this snapshot must be re-taken at the catacomb-save commit or /// the enclave answers a later restore empty. diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index a23445b9bea5b..511625b460ed3 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -89,17 +89,16 @@ pub(crate) struct SensorProfile { pub(crate) capture_qualified: bool, } -/// Identity keybag `CREATE_KEYBAG` field encoding. The strict T8103 enclave -/// reads the request's first word as the bag type and rejects the value the -/// lenient T6020 enclave accepts there; the strict enclave takes the type in -/// the third word instead. Kept per-SoC so the strict path is correct without -/// disturbing the proven T6020 encoding (hardware-verified for enrol, match, -/// and reboot). +/// Identity keybag `CREATE_KEYBAG` field encoding. The first word is the +/// codec's struct version: the 13.5 enclave (T8103) takes versions 0-2 only +/// and macOS creates an identity with version 2, while the T6020 enclave takes +/// variant 5. Kept per-SoC so the T8103 path is correct without disturbing the +/// proven T6020 encoding (hardware-verified for enrol, match, and reboot). pub(crate) struct KeybagCreate { - /// First word: request variant, echoed back in the reply. + /// First word: struct version (request variant), echoed back in the reply. pub(crate) variant: u32, - /// Third word: bag type. Identity is `0x20000` on the strict path; `0` on - /// T6020, which distinguishes the bag by the variant word instead. + /// Third word: bag type. Identity is `0x400000` on 13.5; `0` on T6020, + /// which distinguishes the bag by the variant word instead. pub(crate) bag_type: u32, /// Fourth word: create argument / parent handle. pub(crate) arg: i32, @@ -164,11 +163,12 @@ const T8103: PlatformProfile = PlatformProfile { dart_range_required: false, wide_dma_mask: false, firmware_region: c"sepfw", - // Strict enclave: the type goes in the third word; the first word is 0. + // macOS 13.5's AppleKeyStore::identity_create (0xfffffe000994b2d8): + // version 2, type 0x400000, parent -1 (it accepts only -1 or <= -10). keybag_create: KeybagCreate { - variant: 0, - bag_type: 0x20000, - arg: 0, + variant: 2, + bag_type: 0x40_0000, + arg: -1, }, // The j293 and j313 ADTs both carry `/defaults` `cpx-encryption-mode = 2`. key_store: KeyStore::Sepos13 { diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index a92f842f5d722..d8a5395381907 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -529,10 +529,12 @@ impl SepData { if proof.slot() != keybag::Slot::Identity { return Err(EINVAL); } - // The strict (T8103) and lenient (T6020) enclaves encode the identity - // bag differently; the per-SoC profile carries the correct field values - // so the proven T6020 path is unchanged while T8103 gets the bag type in - // the third word (see profile::KeybagCreate). + // The 13.5 (T8103) and T6020 enclaves encode the identity bag + // differently; the per-SoC profile carries the field values (see + // profile::KeybagCreate). On 13.5 this is identity_create's + // __ipc_create_keybag_v2 (0xfffffe0009955ce0): secret, an empty second + // blob and the 16-byte identity UUID, nothing after it; the codec + // (_code_ipc_create_keybag 0xfffffe000995de6c) decodes exact-length. let enc = &self.profile.keybag_create; let mut body = image::Body::new(); body.put_u32(enc.variant)?; @@ -542,10 +544,12 @@ impl SepData { body.put_blob(secret)?; body.put_blob(&[])?; body.put_blob(uuid)?; - body.put_blob(&[])?; - body.put_u64(0)?; - body.put_u64(0)?; - body.put_blob(&[])?; + if self.profile.key_store == profile::KeyStore::Variant5 { + body.put_blob(&[])?; + body.put_u64(0)?; + body.put_u64(0)?; + body.put_blob(&[])?; + } let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; let msg = crate::sks::encode_sks_create(self.sks_next_seq(), len).ok_or(EINVAL)?; @@ -606,7 +610,13 @@ impl SepData { let Some(body) = self.sks_report_response(crate::sks::SKS_CREATE_NAME, &out) else { return false; }; - if out.reply.status != 0 || body.len() < 12 { + // 13.5 replies with the struct version and the handle alone + // (0x…e11c-e12c); the T6020 reply adds a blob. + let reply_min = match self.profile.key_store { + profile::KeyStore::Sepos13 { .. } => 8, + profile::KeyStore::Variant5 => 12, + }; + if out.reply.status != 0 || body.len() < reply_min { dev_err!( self.dev, "sks: CREATE_KEYBAG failed: mailbox {}, body {} bytes\n", @@ -617,8 +627,14 @@ impl SepData { } let variant = u32::from_le_bytes(body[0..4].try_into().unwrap()); let raw_handle = i32::from_le_bytes(body[4..8].try_into().unwrap()); - let Some((_fv_data, end)) = image::read_blob(body, 8) else { - return false; + let end = match self.profile.key_store { + profile::KeyStore::Sepos13 { .. } => 8, + profile::KeyStore::Variant5 => { + let Some((_fv_data, end)) = image::read_blob(body, 8) else { + return false; + }; + end + } }; if variant != self.profile.keybag_create.variant || raw_handle < 0 || end != body.len() { @@ -631,8 +647,17 @@ impl SepData { return false; } let handle = crate::sks::KeyBagHandle::from_create_reply(raw_handle); - let Some(bag_uuid) = self.sks_read_uuid(handle) else { - return false; + // identity_create reads no UUID back: 13.5 names the identity by the + // UUID it was created with, and identity_save (0xfffffe000994b1cc) is + // just the copy below, then the unload. + let bag_uuid = match self.profile.key_store { + profile::KeyStore::Sepos13 { .. } => None, + profile::KeyStore::Variant5 => { + let Some(bag_uuid) = self.sks_read_uuid(handle) else { + return false; + }; + Some(bag_uuid) + } }; let Some(out) = self.sks_send(self.sks_req_copy_keybag(handle)) else { return false; @@ -640,7 +665,11 @@ impl SepData { let Some(wrapped) = self.wrapped_from_copy_reply(&out, c"new identity keybag") else { return false; }; - if let Err(e) = keybag::write_bag_uuid(slot, &wrapped, &bag_uuid, &secret) { + let committed = match bag_uuid { + Some(bag_uuid) => keybag::write_bag_uuid(slot, &wrapped, &bag_uuid, &secret), + None => keybag::write_generated_uuid(slot, &wrapped, &uuid, &secret), + }; + if let Err(e) = committed { dev_err!(self.dev, "sks: cannot commit identity keybag: {:?}\n", e); return false; } From 0797846eca52aaa24277e46a33cc4fe75c367728 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:36:48 -0700 Subject: [PATCH 0735/2680] soc: apple: sep: activate the T8103 identity as identity_load does On macOS 13.5 an identity session is opened by AppleKeyStore::identity_load (0xfffffe000994b888), which sends exactly three key-store requests: - 0x03 LOAD_KEYBAG through identity_open (0xfffffe000994b0e4), whose blob is the 16-byte identity UUID (mov w4, #0x10 at 0x...b11c): the enclave keeps the identity bag and the host never holds a wrapped one; - 0x0d make-system-keybag, binding the loaded handle to the session handle (0x...ba48-ba5c) with an empty blob. The codec (_code_ipc_make_system_keybag 0xfffffe000995fd8c) takes struct version 0 only (0x...fe78) and the reply is that version word alone; - 0x05 UNLOAD of the source handle (0x...bad4). On T8103 the driver loaded the wrapped blob, read the UUID back with 0x06, designated with version 1, the bag secret and a trailing u64, then made a machine ref-key (0x04 unlock of the source and 0x22 'oc'), listed SBIO identities and read the session UUID back before the unload. None of that has a 13.5 counterpart. On T8103, load by the recorded identity UUID, send no 0x06, designate with version 0 and an empty blob (checking the version-0 reply), and unload the source straight after. The ref-key flow, which no 13.5 code performs with an identity bag, is refused on that profile before anything is sent, both at activation and for trusted-key and attest callers. T6020 requests, their order and the ref-key flow are unchanged. Hardware-verified on a MacBook Air (j313) with the 13.5 SEP, on every activation after provisioning: LOAD_KEYBAG by UUID status 0 with a handle, DESIGNATE_KEYBAG (source, -1000, empty blob) status 0, UNLOAD_KEYBAG status 0. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AJd8eEwJ4TGRSHnbKcWBmw --- drivers/soc/apple/refkey.rs | 8 +++++++ drivers/soc/apple/sbio.rs | 5 +++++ drivers/soc/apple/sks.rs | 43 +++++++++++++++++++++++++++++++------ 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/drivers/soc/apple/refkey.rs b/drivers/soc/apple/refkey.rs index d2ffe586594ea..63671723fd7ed 100644 --- a/drivers/soc/apple/refkey.rs +++ b/drivers/soc/apple/refkey.rs @@ -90,6 +90,11 @@ impl SepData { pub(crate) fn sks_machine_refkey(&self, handle: crate::sks::KeyBagHandle, secret: &[u8]) { const MACHINE_REFKEY_PATH: &CStr = c"/var/lib/aurora-sep-refkey.bin"; + // No 13.5 code unlocks an identity source handle or makes a ref-key + // with it; identity_load is exactly 0x03, 0x0d, 0x05 there. + if self.profile.key_store != crate::profile::KeyStore::Variant5 { + return; + } if self.machine_refkey.lock().is_some() { return; } @@ -135,6 +140,9 @@ impl SepData { } fn ensure_machine_refkey(&self) -> Result<()> { + if self.profile.key_store != crate::profile::KeyStore::Variant5 { + return Err(ENODEV); + } if self.machine_refkey.lock().is_some() { return Ok(()); } diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 91395a40e68d7..fea3a4f9a7fdb 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2716,6 +2716,11 @@ impl SepData { return false; }; let special = user.special_handle(); + // identity_load ends with the source unload (0xfffffe000994bad4); + // 13.5 sends nothing between its 0x0d and that unload. + if self.profile.key_store != profile::KeyStore::Variant5 { + return self.sks_send(self.sks_req_unload_keybag(source)).is_some(); + } self.log_identity_count(c"before the cold-match preparation"); let uuid_ok = self diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index d8a5395381907..7e9452f0950b4 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -396,16 +396,32 @@ impl SepData { true } - /// `0x0d` designate. + /// The designate struct version, which the reply echoes. 13.5's + /// `_code_ipc_make_system_keybag` (0xfffffe000995fd8c) takes version 0 + /// only (0x…fe78). + fn sks_designate_variant(&self) -> u32 { + match self.profile.key_store { + profile::KeyStore::Sepos13 { .. } => crate::sks::SKS_DESIGNATE_VERSION_13, + profile::KeyStore::Variant5 => crate::sks::SKS_DESIGNATE_VARIANT, + } + } + + /// `0x0d` designate. On 13.5 this is `identity_load`'s make-system-keybag + /// (0xfffffe000994ba48-ba5c): source handle, session, an empty blob and + /// nothing after it. fn sks_req_designate(&self, d: &crate::sks::Designation, secret: &[u8]) -> Result { let mut body = image::Body::new(); - body.put_u32(crate::sks::SKS_DESIGNATE_VARIANT)?; + body.put_u32(self.sks_designate_variant())?; body.put_u64(crate::sks::SKS_CLIENT_ID)?; body.put_i32(d.source().value())?; body.put_i32(d.user().special_handle().value())?; - body.put_blob(secret)?; - // Flags is a u64, not u32; a u32 leaves the body short -> enclave answers -13. - body.put_u64(crate::sks::SKS_DESIGNATE_FLAGS)?; + if self.profile.key_store == profile::KeyStore::Variant5 { + body.put_blob(secret)?; + // Flags is a u64, not u32; a u32 leaves the body short -> enclave answers -13. + body.put_u64(crate::sks::SKS_DESIGNATE_FLAGS)?; + } else { + body.put_blob(&[])?; + } let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; @@ -711,7 +727,7 @@ impl SepData { return; } let variant = u32::from_le_bytes([body[0], body[1], body[2], body[3]]); - if variant != crate::sks::SKS_DESIGNATE_VARIANT { + if variant != self.sks_designate_variant() { return; } @@ -833,7 +849,13 @@ impl SepData { &self, stored: &keybag::StoredKeyBag, ) -> Option<(crate::sks::KeyBagHandle, [u8; keybag::UUID_LEN])> { - let request = match self.sks_req_load_keybag(stored.wrapped()) { + // 13.5 keeps the identity bag in the enclave: identity_open + // (0xfffffe000994b0e4) loads it by its 16-byte UUID (0x…b11c-b124). + let key = match self.profile.key_store { + profile::KeyStore::Sepos13 { .. } => &stored.uuid()[..], + profile::KeyStore::Variant5 => stored.wrapped(), + }; + let request = match self.sks_req_load_keybag(key) { Ok(request) => request, Err(e) => { dev_err!( @@ -895,6 +917,12 @@ impl SepData { } let handle = crate::sks::KeyBagHandle::from_load_reply(handle); + // identity_load sends 0x0d straight after the load (0x…ba5c) and + // reads no UUID back. + if self.profile.key_store != profile::KeyStore::Variant5 { + return Some((handle, *stored.uuid())); + } + let Some(uuid) = self.sks_read_uuid(handle) else { dev_warn!(self.dev, "sks: loaded keybag has no readable UUID\n"); let _ = self.sks_send(self.sks_req_unload_keybag(handle)); @@ -1189,6 +1217,7 @@ const OP_SKS_DESIGNATE_KEYBAG: u8 = 0x0d; pub(crate) const SKS_DESIGNATE_VARIANT: u32 = 1; static_assert!(SKS_DESIGNATE_VARIANT == 1); +pub(crate) const SKS_DESIGNATE_VERSION_13: u32 = 0; pub(crate) const SKS_DESIGNATE_USER_MIN: i32 = 10; static_assert!(SKS_DESIGNATE_USER_MIN > 0); From a1047a4a3cbe67a9e4907931167070ece4de4bb3 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:37:28 -0700 Subject: [PATCH 0736/2680] soc: apple: sep: unlock the T8103 identity session with 0x18 macOS 13.5 unlocks an identity session with AKSIdentityUnlockSession (userspace 0x1ed92da2c), user-client selector 0x7b. The kext passes the secret unchanged to unlock_the_device(client, h, secret, 5) (0xfffffe000993cee0, called at 0x...15a4), which calls device_state_transition with state 0 and, for mode 5, flags 0 (0xfffffe000993cf04-cf0c). That is SEP op 0x18, __ipc_device_state_transition (0xfffffe0009958204): struct version 0 only (_code_ipc_device_state_transition 0xfffffe0009961f20, 0x...200c), u64 client, u32 handle, u32 state, u64 flags, blob secret; the reply is the version word and two u64. No 13.5 identity path uses 0x04 change-lock-state, whose codec takes version 0 only as well, not the version 1 with a trailing u64 the driver sends. On T8103, unlock the designated session with that 0x18 request. The unlock also ran only when the template restore succeeded. On macOS the session is unlocked whenever it is opened, and a T8103 machine that has not enrolled yet has no template to restore, so its session was never unlocked. On T8103 the unlock now follows the restore whatever its result; the match context set-up that needs templates still runs only after a successful restore. T6020 keeps 0x04 and its restore-gated order. Hardware-verified on a MacBook Air (j313) with the 13.5 SEP: on every activation after LOAD, DESIGNATE and UNLOAD, DEVICE_STATE_TRANSITION {v0, client, -1000, state 0, flags 0, secret} answered status 0 with the 20-byte reply, and the enrol credential's VERIFY_SECRET (0x21) against -1000 then answered status 0. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AJd8eEwJ4TGRSHnbKcWBmw --- drivers/soc/apple/sks.rs | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index 7e9452f0950b4..07e5906bf3ac9 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -472,6 +472,9 @@ impl SepData { healthy: Healthy, ) -> Result { let Healthy(()) = healthy; + if self.profile.key_store != profile::KeyStore::Variant5 { + return self.sks_req_device_state_unlock(special, secret); + } let mut body = image::Body::new(); body.put_u32(SKS_LOCK_STATE_VARIANT)?; body.put_u64(crate::sks::SKS_CLIENT_ID)?; @@ -490,6 +493,36 @@ impl SepData { }) } + /// `0x18` device-state transition to unlocked, the 13.5 identity-session + /// unlock. `AKSIdentityUnlockSession` (user-client selector 0x7b) reaches + /// `unlock_the_device(client, h, secret, 5)` (0xfffffe000993cee0, called at + /// 0x…15a4), which sends `device_state_transition` with state 0 and flags 0 + /// (0x…cf04-cf0c) through `__ipc_device_state_transition` + /// (0xfffffe0009958204): struct version 0 only (0x…200c), u64 client, u32 + /// handle, u32 state, u64 flags, blob secret. The reply is the version word + /// and two u64. + fn sks_req_device_state_unlock( + &self, + special: crate::sks::SpecialHandle, + secret: &[u8], + ) -> Result { + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(special.value())?; + body.put_i32(LockState::Unlocked.wire())?; + body.put_u64(0)?; + body.put_blob(secret)?; + let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_device_state_transition(self.sks_next_seq(), len); + Ok(SksRequest { + name: crate::sks::SKS_DEVICE_STATE_NAME, + msg, + img, + }) + } + /// `0x06` copy key-bag UUID. fn sks_req_copy_keybag_uuid(&self, handle: crate::sks::KeyBagHandle) -> Result { let op = crate::sks::sks_copy_keybag_uuid(); @@ -1184,6 +1217,13 @@ const OP_SKS_CHANGE_LOCK_STATE: u8 = 0x04; pub(crate) const SKS_LOCK_STATE_NAME: &CStr = c"CHANGE_LOCK_STATE"; +const OP_SKS_DEVICE_STATE_TRANSITION: u8 = 0x18; +pub(crate) const SKS_DEVICE_STATE_NAME: &CStr = c"DEVICE_STATE_TRANSITION"; + +pub(crate) fn encode_sks_device_state_transition(seq: Sequence, len: ImageLen) -> Message { + encode_sks_raw(OP_SKS_DEVICE_STATE_TRANSITION, seq.value(), len.value()) +} + const OP_SKS_VERIFY_SECRET: u8 = 0x21; pub(crate) const SKS_VERIFY_SECRET_NAME: &CStr = c"VERIFY_SECRET"; pub(crate) const SKS_VERIFY_SECRET_VARIANT: u32 = 1; From cf898cf965c442f138ee789937ccb73152fcf128 Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Fri, 25 Sep 2026 02:45:32 -0600 Subject: [PATCH 0737/2680] spi: apple: install hardware CS timing only on the Mesa sensor's bus apple_spi_set_cs_timing() was installed on every Apple SPI controller. The SPI core stops running software CS delays for all devices on a controller that provides set_cs_timing, and the callback rejects any device with a CS inactive delay. The J314/J316 keyboard's spi-hid transport sets spi-cs-inactive-delay-ns, so spi_setup() failed with -EOPNOTSUPP and the internal keyboard never came up. On J316s this stops the boot at the LUKS prompt. Install the callback only on a controller with an apple,mesa-fingerprint child. Every other bus keeps the software CS delays it used before. Fixes: 4a356a39160b ("soc: apple: add production SEP FileVault integration") Assisted-by: Claude Opus 5.5 Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- drivers/spi/spi-apple.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/drivers/spi/spi-apple.c b/drivers/spi/spi-apple.c index 6695e61b081f1..21937bb50ba08 100644 --- a/drivers/spi/spi-apple.c +++ b/drivers/spi/spi-apple.c @@ -545,6 +545,7 @@ static int apple_spi_probe(struct platform_device *pdev) struct apple_spi *spi; int ret, irq; struct spi_controller *ctlr; + struct device_node *np; ctlr = devm_spi_alloc_host(&pdev->dev, sizeof(struct apple_spi)); if (!ctlr) @@ -577,7 +578,18 @@ static int apple_spi_probe(struct platform_device *pdev) ctlr->bits_per_word_mask = SPI_BPW_RANGE_MASK(1, 32); ctlr->prepare_message = apple_spi_prepare_message; ctlr->set_cs = apple_spi_set_cs; - ctlr->set_cs_timing = apple_spi_set_cs_timing; + /* + * Only the Mesa fingerprint sensor's bus uses the hardware CS delays. + * The SPI core stops running software CS delays for every device on a + * controller that provides set_cs_timing, so installing it everywhere + * would move the keyboard, NOR flash and others off their working + * timing, and the keyboard's inactive delay has no hardware form. + */ + np = of_get_compatible_child(pdev->dev.of_node, "apple,mesa-fingerprint"); + if (np) { + ctlr->set_cs_timing = apple_spi_set_cs_timing; + of_node_put(np); + } ctlr->transfer_one = apple_spi_transfer_one; ctlr->use_gpio_descriptors = true; ctlr->auto_runtime_pm = true; From 3d44280106d6c672474ad5ff305862cc044a81af Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 23:59:49 -0400 Subject: [PATCH 0738/2680] soc: apple: sep: complete J313 enrollment, cold restore and trusted keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Initialize the 13.5 SBIO session before enrollment, distinguish generated lookup UUIDs from recovered bag UUIDs, and bind persisted snapshots to the designated bag. Keep cold restore provisional until the sensor device view is proven. Use the initialized host CSPRNG for operation tokens and trusted-key plaintext; reject an all-zero firmware RNG survey. Support the 13.5 ref-key envelope and session, with an explicit separate recovery slot that leaves the original slot alone. Enable reboot-tested owner export and user-first persistence on T8103/T6020 while preserving Neo paths. Add source-level identity and entropy tests. The consolidated module builds against the prepared J313 kernel; the exact merged tree has not been hardware-regression-tested across platforms. [ryan: rebased onto the linear SEP history — this commit was developed in parallel with the J700 authenticated-session work (1ae0ac48652a). The conflicts are composed as aurora-wip composed them in 42ae23434d4c: the persistent-enrol gate uses the new profile field, and T8140 sets it so J700 keeps its owner export; INIT_SBIO_COMMUNICATION sends word 6 on nonce-session profiles and word 1 on the 13.5 key store; the 13.5 serial registration sits beside the J700 path, which skips REGISTER_SENSOR] Signed-off-by: Ryan Murray --- drivers/soc/apple/keybag.rs | 12 +- drivers/soc/apple/keybag_identity.rs | 91 +++++++++++ drivers/soc/apple/profile.rs | 5 + drivers/soc/apple/refkey.rs | 162 +++++++++++++++----- drivers/soc/apple/sbio.rs | 137 +++++++++++------ drivers/soc/apple/sep.rs | 21 ++- drivers/soc/apple/sks.rs | 31 ++-- tools/aurora-sep/tests/test_host_entropy.py | 111 ++++++++++++++ 8 files changed, 460 insertions(+), 110 deletions(-) create mode 100644 drivers/soc/apple/keybag_identity.rs create mode 100644 tools/aurora-sep/tests/test_host_entropy.py diff --git a/drivers/soc/apple/keybag.rs b/drivers/soc/apple/keybag.rs index e295693421ad7..ef629ae4fd484 100644 --- a/drivers/soc/apple/keybag.rs +++ b/drivers/soc/apple/keybag.rs @@ -40,11 +40,7 @@ const STATE_COMMITTED_BAG_UUID: u32 = 3; /// Only an `AsGenerated` UUID may be repaired on a mismatch; adopting one for a /// read-back record would point it at a different bag and strand the real one. -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum UuidProvenance { - AsGenerated, - ReadBackFromBag, -} +pub(crate) use crate::keybag_identity::{SnapshotIdentity, UuidProvenance}; impl UuidProvenance { fn state(self) -> u32 { @@ -293,6 +289,7 @@ pub(crate) fn replace_wrapped( slot: Slot, fresh: &[u8], snapshot_uuid: &[u8; UUID_LEN], + identity: &SnapshotIdentity, ) -> Result<()> { if fresh.is_empty() || fresh.len() > MAX_WRAPPED { return Err(EINVAL); @@ -301,8 +298,9 @@ pub(crate) fn replace_wrapped( State::Present(stored) => stored, State::Absent(_) => return Err(ENOENT), }; - if stored.uuid() != snapshot_uuid { - // A snapshot of a different bag would strand the stored one. + if !identity.accepts(stored.uuid(), stored.uuid_provenance(), snapshot_uuid) { + // Reject a different bag or changed host identity. The generated + // Sepos13 lookup UUID is not the UUID returned by COPY_KEYBAG_UUID. return Err(EPERM); } write_record( diff --git a/drivers/soc/apple/keybag_identity.rs b/drivers/soc/apple/keybag_identity.rs new file mode 100644 index 0000000000000..7022839cc30c7 --- /dev/null +++ b/drivers/soc/apple/keybag_identity.rs @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Keeps a persisted lookup identity distinct from the enclave bag identity. + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum UuidProvenance { + AsGenerated, + ReadBackFromBag, +} + +/// Binding established from a successfully recovered source handle. A snapshot +/// must identify that same bag, without changing the host's lookup identity. +#[derive(Clone, Copy)] +pub(crate) struct SnapshotIdentity { + record_uuid: [u8; 16], + provenance: UuidProvenance, + bag_uuid: [u8; 16], +} + +impl SnapshotIdentity { + pub(crate) fn from_loaded( + record_uuid: [u8; 16], + provenance: UuidProvenance, + source_bag_uuid: [u8; 16], + loaded_by_generated_uuid: bool, + ) -> Option { + if loaded_by_generated_uuid { + if provenance != UuidProvenance::AsGenerated { + return None; + } + } else if record_uuid != source_bag_uuid { + return None; + } + Some(Self { record_uuid, provenance, bag_uuid: source_bag_uuid }) + } + + pub(crate) fn accepts( + &self, + record_uuid: &[u8; 16], + provenance: UuidProvenance, + snapshot_uuid: &[u8; 16], + ) -> bool { + self.record_uuid == *record_uuid + && self.provenance == provenance + && self.bag_uuid == *snapshot_uuid + } +} + +#[cfg(test)] +mod tests { + use super::*; + const LOOKUP: [u8; 16] = [1; 16]; + const BAG: [u8; 16] = [2; 16]; + const OTHER: [u8; 16] = [3; 16]; + + #[test] + fn sepos13_keeps_lookup_and_bag_identities_distinct() { + let proof = SnapshotIdentity::from_loaded( + LOOKUP, UuidProvenance::AsGenerated, BAG, true, + ).unwrap(); + assert!(proof.accepts(&LOOKUP, UuidProvenance::AsGenerated, &BAG)); + assert!(!proof.accepts(&BAG, UuidProvenance::AsGenerated, &BAG)); + assert!(!proof.accepts(&LOOKUP, UuidProvenance::AsGenerated, &LOOKUP)); + } + + #[test] + fn changed_record_or_designated_bag_is_rejected() { + let proof = SnapshotIdentity::from_loaded( + LOOKUP, UuidProvenance::AsGenerated, BAG, true, + ).unwrap(); + assert!(!proof.accepts(&OTHER, UuidProvenance::AsGenerated, &BAG)); + assert!(!proof.accepts(&LOOKUP, UuidProvenance::ReadBackFromBag, &BAG)); + assert!(!proof.accepts(&LOOKUP, UuidProvenance::AsGenerated, &OTHER)); + } + + #[test] + fn blob_recovery_keeps_strict_uuid_equality() { + for provenance in [UuidProvenance::AsGenerated, UuidProvenance::ReadBackFromBag] { + assert!(SnapshotIdentity::from_loaded(LOOKUP, provenance, BAG, false).is_none()); + let proof = SnapshotIdentity::from_loaded(BAG, provenance, BAG, false).unwrap(); + assert!(proof.accepts(&BAG, provenance, &BAG)); + assert!(!proof.accepts(&BAG, provenance, &OTHER)); + } + } + + #[test] + fn readback_record_cannot_claim_generated_lookup() { + assert!(SnapshotIdentity::from_loaded( + LOOKUP, UuidProvenance::ReadBackFromBag, BAG, true, + ).is_none()); + } +} diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 511625b460ed3..6dbaf5d878c8d 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -142,6 +142,8 @@ pub(crate) struct PlatformProfile { pub(crate) nonce_session: bool, /// Key-store protocol generation (see [`KeyStore`]). pub(crate) key_store: KeyStore, + /// Platforms with reboot-tested owner export and user-before-master saves. + pub(crate) persistent_enrol: bool, } const T8103: PlatformProfile = PlatformProfile { @@ -174,6 +176,7 @@ const T8103: PlatformProfile = PlatformProfile { key_store: KeyStore::Sepos13 { cpx_encryption_mode: 2, }, + persistent_enrol: true, nonce_session: false, }; @@ -203,6 +206,7 @@ const T6020: PlatformProfile = PlatformProfile { arg: -1, }, key_store: KeyStore::Variant5, + persistent_enrol: true, nonce_session: false, }; @@ -211,6 +215,7 @@ const T6020: PlatformProfile = PlatformProfile { /// board; device-tree nodes remain authoritative for addresses and resources. const T8140: PlatformProfile = PlatformProfile { key_store: KeyStore::Variant5, + persistent_enrol: true, name: "T8140/J700", shmem_capacity: 0x4_0000, shmem_first_item: b"CINP", diff --git a/drivers/soc/apple/refkey.rs b/drivers/soc/apple/refkey.rs index 63671723fd7ed..4d54246bf4f68 100644 --- a/drivers/soc/apple/refkey.rs +++ b/drivers/soc/apple/refkey.rs @@ -6,19 +6,31 @@ //! never leaves the SEP. The host-side ECIES encrypt lives in [`refkey_seal`]; //! every half that needs the private key runs in the enclave. -use crate::{image, keybag, proto, refkey_seal, shim}; +use crate::{image, keybag, profile, proto, refkey_seal, shim}; use crate::{LockState, MachineRefKey, SepData, SksRequest}; use kernel::prelude::*; +use kernel::sync::atomic::Relaxed; pub(crate) const REFKEY_ENVELOPE_VERSION: u32 = 2; impl SepData { fn sks_req_refkey(&self, keybag: i32, der_set: &[u8]) -> Result { + let (keybag, envelope) = match self.profile.key_store { + // The 13.5 operation envelope is 3. Use the designated identity + // session rather than the source handle; class F with -1 expired + // at the next boot on J313. + profile::KeyStore::Sepos13 { .. } => { + let user = crate::sks::DesignateUser::new(crate::SBIO_PROBE_USER_ID) + .ok_or(EINVAL)?; + (user.special_handle().value(), 3) + } + profile::KeyStore::Variant5 => (keybag, REFKEY_ENVELOPE_VERSION), + }; let mut body = image::Body::new(); body.put_u32(0)?; body.put_u64(crate::sks::SKS_CLIENT_ID)?; body.put_i32(keybag)?; - body.put_u32(crate::refkey::REFKEY_ENVELOPE_VERSION)?; + body.put_u32(envelope)?; body.put_blob(der_set)?; let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; @@ -31,6 +43,46 @@ impl SepData { }) } + fn sks_prepare_identity_refkey( + &self, + handle: crate::sks::KeyBagHandle, + secret: &[u8], + ) -> bool { + if !self.keybag_designated.load(Relaxed) { + self.sks_designate_user_keybag(handle, secret); + } + if !self.keybag_designated.load(Relaxed) { + dev_warn!(self.dev, "sks: ref-key identity designation failed\n"); + return false; + } + let Some(user) = crate::sks::DesignateUser::new(crate::SBIO_PROBE_USER_ID) else { + return false; + }; + let Some(healthy) = self.sks_health_check(c"ref-key identity unlock") else { + return false; + }; + let Some(out) = self.sks_send(self.sks_req_unlock_special( + user.special_handle(), + secret, + healthy, + )) else { + dev_warn!(self.dev, "sks: ref-key identity unlock got no reply\n"); + return false; + }; + dev_info!( + self.dev, + "sks: ref-key identity unlock mailbox status {}\n", + out.reply.status + ); + if out.reply.status != 0 { + return false; + } + let Some(body) = self.sks_report_response(c"DEVICE_STATE_TRANSITION", &out) else { + return false; + }; + body.len() == 20 && body[..4] == [0; 4] + } + fn sks_create_refkey( &self, handle: crate::sks::KeyBagHandle, @@ -38,43 +90,68 @@ impl SepData { ) -> Option> { use crate::der::RefKeyValue as RV; - // Class 9 = non-extractable (private stays enclave-resident); key type 5 = P-256. - const PROTECTION_CLASS: u32 = 9; - const KEY_TYPE: u32 = 5; + let (protection_class, key_type) = match self.profile.key_store { + // J313 accepted this non-class-F class and 13.5 EC key type for + // creation, same-boot unseal, and post-reboot unseal. The exact + // Apple policy name for class 9 is not established here. + profile::KeyStore::Sepos13 { .. } => (9, 4), + profile::KeyStore::Variant5 => (9, 5), + }; - // Unlock the bag first; an unbound create is refused. - if let Some(healthy) = self.sks_health_check(c"ref-key create") { - let _ = self.sks_send(self.sks_req_change_lock_state( - handle, - LockState::Unlocked, - secret, - healthy, - )); + if self.profile.key_store == profile::KeyStore::Variant5 { + // T6020's ref-key is bound to the unlocked identity bag. + if let Some(healthy) = self.sks_health_check(c"ref-key create") { + let _ = self.sks_send(self.sks_req_change_lock_state( + handle, + LockState::Unlocked, + secret, + healthy, + )); + } + } else if !self.sks_prepare_identity_refkey(handle, secret) { + return None; } let create_der = { let mut items: KVec<(&[u8], RV<'_>)> = KVec::new(); if items.push((b"o", RV::Utf8(b"oc")), GFP_KERNEL).is_err() || items - .push((b"bc", RV::Integer(PROTECTION_CLASS)), GFP_KERNEL) + .push((b"bc", RV::Integer(protection_class)), GFP_KERNEL) .is_err() || items - .push((b"kt", RV::Integer(KEY_TYPE)), GFP_KERNEL) + .push((b"kt", RV::Integer(key_type)), GFP_KERNEL) .is_err() { return None; } crate::der::encode_refkey_set(&items).ok()? }; - let cout = self.sks_send(self.sks_req_refkey(handle.value(), &create_der))?; + let cout = match self.sks_send(self.sks_req_refkey(handle.value(), &create_der)) { + Some(out) => out, + None => { + dev_warn!(self.dev, "sks: ref-key create got no usable reply\n"); + return None; + } + }; + dev_info!( + self.dev, + "sks: ref-key create mailbox status {}, response {} bytes\n", + cout.reply.status, + cout.response.len() + ); if cout.reply.status != 0 { return None; } let cbody = self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &cout)?; let mut cf = proto::FieldCursor::new(cbody); - let (Some(0), Some(cblob)) = (cf.i32(), cf.blob()) else { + let (Some(version), Some(cblob)) = (cf.i32(), cf.blob()) else { + dev_warn!(self.dev, "sks: ref-key create reply was malformed\n"); return None; }; + if version != 0 { + dev_warn!(self.dev, "sks: ref-key create reply version {}\n", version); + return None; + } let mut refkey_blob: KVec = KVec::new(); refkey_blob.extend_from_slice(cblob, GFP_KERNEL).ok()?; Some(refkey_blob) @@ -89,17 +166,18 @@ impl SepData { pub(crate) fn sks_machine_refkey(&self, handle: crate::sks::KeyBagHandle, secret: &[u8]) { const MACHINE_REFKEY_PATH: &CStr = c"/var/lib/aurora-sep-refkey.bin"; + const MACHINE_REFKEY_V2_PATH: &CStr = c"/var/lib/aurora-sep-refkey-v2.bin"; + let path = if *crate::module_parameters::refkey_v2.value() != 0 { + MACHINE_REFKEY_V2_PATH + } else { + MACHINE_REFKEY_PATH + }; - // No 13.5 code unlocks an identity source handle or makes a ref-key - // with it; identity_load is exactly 0x03, 0x0d, 0x05 there. - if self.profile.key_store != crate::profile::KeyStore::Variant5 { - return; - } if self.machine_refkey.lock().is_some() { return; } - if let Ok(f) = shim::StoreFile::open_readonly(MACHINE_REFKEY_PATH) { + if let Ok(f) = shim::StoreFile::open_readonly(path) { let sz = f.size().unwrap_or(0); if (1..=8192).contains(&sz) { let mut blob: KVec = KVec::new(); @@ -110,6 +188,11 @@ impl SepData { return; } } + dev_err!( + self.dev, + "sks: existing machine ref-key is unreadable or invalid; refusing replacement\n" + ); + return; } let Some(blob) = self.sks_create_refkey(handle, secret) else { @@ -119,10 +202,13 @@ impl SepData { ); return; }; - if let Ok(f) = shim::StoreFile::open(MACHINE_REFKEY_PATH) { - if f.write_all(0, &blob).is_ok() { - let _ = f.sync(); - } + let Ok(f) = shim::StoreFile::open(path) else { + dev_err!(self.dev, "sks: could not persist machine ref-key\n"); + return; + }; + if f.write_all(0, &blob).is_err() || f.sync().is_err() { + dev_err!(self.dev, "sks: machine ref-key persistence failed\n"); + return; } let _ = self.cache_machine_refkey(blob); } @@ -140,9 +226,6 @@ impl SepData { } fn ensure_machine_refkey(&self) -> Result<()> { - if self.profile.key_store != crate::profile::KeyStore::Variant5 { - return Err(ENODEV); - } if self.machine_refkey.lock().is_some() { return Ok(()); } @@ -182,13 +265,18 @@ impl SepData { return Err(ENODEV); }; let (handle, _uuid) = self.sks_recover(&stored).ok_or(EIO)?; - if let Some(healthy) = self.sks_health_check(c"trusted-key unseal") { - let _ = self.sks_send(self.sks_req_change_lock_state( - handle, - LockState::Unlocked, - stored.secret(), - healthy, - )); + if self.profile.key_store == profile::KeyStore::Variant5 { + if let Some(healthy) = self.sks_health_check(c"trusted-key unseal") { + let _ = self.sks_send(self.sks_req_change_lock_state( + handle, + LockState::Unlocked, + stored.secret(), + healthy, + )); + } + } else if !self.sks_prepare_identity_refkey(handle, stored.secret()) { + let _ = self.sks_send(self.sks_req_unload_keybag(handle)); + return Err(EIO); } let recovered = self.sks_refkey_unseal(handle, &blob, sealed); let _ = self.sks_send(self.sks_req_unload_keybag(handle)); diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index fea3a4f9a7fdb..ac4fba5be22af 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -368,9 +368,10 @@ impl SepData { } } - // J700 needs the owner component even when state 0x3 has no - // save-pending bit. Retain the opt-in policy on older profiles. - if self.profile.nonce_session || *module_parameters::j414s_persistent_enrol.value() != 0 { + // J313 and J414s need an owner export even though state 0x3 has no + // save-pending bit. Both restored a fresh enrollment after this export + // and user-before-master completion. Other profiles remain opt-in. + if self.profile.persistent_enrol || *module_parameters::j414s_persistent_enrol.value() != 0 { if owner_initial_state == Some(0x3) && !self.save_catacomb( crate::sbio::CatacombUser::OWNER, @@ -759,7 +760,7 @@ impl SepData { } fn save_all_components(&self, user: crate::sbio::UserId) -> bool { - if self.profile.nonce_session || *module_parameters::j414s_persistent_enrol.value() != 0 { + if self.profile.persistent_enrol || *module_parameters::j414s_persistent_enrol.value() != 0 { return self.save_all_components_user_first(user); } // On m2fix11, a completed enrollment marked both master and user 0x7, @@ -826,9 +827,8 @@ impl SepData { dev_err!(self.dev, "enrol: opt-in completion has no owner Catacomb file\n"); return false; } - // Post-match persistence saves the user before the master because the - // user save can change system material. Test that order for a completed - // J414s enrollment, without changing the default path on other Macs. + // Saving the user can change system material. Persist it before the + // master, as in the reboot-tested J313/J414s enrollment paths. for (who, kind, what) in [ ( crate::sbio::CatacombUser::enrolling(user), @@ -1004,14 +1004,20 @@ impl SepData { self.register_ool(&self.ool_sbio)?; self.sbio_ready.store(true, Relaxed); - if self.profile.nonce_session { - let word = 6u32; - // Communication must be initialized before restoring biometric state. + let init_word = if self.profile.nonce_session { + Some(6u32) + } else if matches!(self.profile.key_store, profile::KeyStore::Sepos13 { .. }) { + Some(1u32) + } else { + None + }; + if let Some(word) = init_word { let result = self.sbio_transfer_raw( 0x73, c"INIT_SBIO_COMMUNICATION", &word.to_le_bytes(), ); match result { - Ok(done) if done.status.is_ok() => { + Ok(done) if done.status.is_ok() + && (self.profile.nonce_session || done.payload.is_empty()) => { dev_info!(self.dev, "sbio: communication initialization accepted\n"); } Ok(done) => { @@ -1022,7 +1028,7 @@ impl SepData { } Err(e) => { self.sbio_ready.store(false, Relaxed); - dev_err!(self.dev, "sbio: communication init transport failed: {:?}\n", e); + dev_err!(self.dev, "sbio: communication init transport failed {:?}; aborting Touch ID setup\n", e); return Err(e); } } @@ -1213,9 +1219,9 @@ impl SepData { })?; self.sks_designate_user_keybag(handle, stored.secret()); self.sks_machine_refkey(handle, stored.secret()); - let prepared = self.cold_match_continue(handle, uuid); - self.attach_bringup(); + let prepared = self.cold_match_continue(handle, uuid, stored.uuid_provenance()); self.ensure_restored_after(prepared); + self.attach_bringup(); self.probe_owner_export(); self.touchid_started.store(true, Relaxed); Ok(()) @@ -1537,7 +1543,13 @@ impl SepData { let _ = self.log_identity_count(c"after CLEAR_STATE for sensor re-registration"); } - if stage == BRINGUP_FRESH && !self.profile.nonce_session { + if stage == BRINGUP_FRESH + && matches!(self.profile.key_store, profile::KeyStore::Sepos13 { .. }) + { + // The 13.5 J313 flow begins with REGISTER_SENSOR_SERIAL (0x17). + // The newer REGISTER_SENSOR (0x80) returned 0x16 on this firmware. + self.bringup.store(BRINGUP_IDENTIFIED, Relaxed); + } else if stage == BRINGUP_FRESH && !self.profile.nonce_session { let op = crate::sbio::sbio_register_sensor(id); match self.sbio_call(&op) { SbioOutcome::Ok(_payload) => { @@ -1788,7 +1800,9 @@ impl SepData { Some(0) => { self.templates_restored.store(false, Relaxed); } - Some(n) if self.templates_restored.load(Relaxed) && self.prove_restore() => { + Some(n) if (self.templates_restored.load(Relaxed) + || self.cold_restore_candidate.load(Relaxed)) && self.prove_restore() => { + self.templates_restored.store(true, Relaxed); dev_warn!( self.dev, "Touch ID: restored {} enrolled identity/identities; enrolment survived reboot\n", @@ -2558,13 +2572,13 @@ impl SepData { fn mint_token_bytes(&self) -> Option<[u8; bio::TOKEN_LEN]> { let mut bytes = [0u8; bio::TOKEN_LEN]; - for chunk in bytes.chunks_mut(4) { - match self.get_entropy_word() { - Ok(word) => chunk.copy_from_slice(&word.to_le_bytes()[..chunk.len()]), - Err(_) => { - return None; - } - } + // This is a host-issued, per-operation capability, not SEP match + // evidence. Use the initialized kernel CSPRNG, as for other host + // secrets, rather than trusting firmware control replies as entropy. + // J313/13.5 answered those draws with zero and caused valid matches + // to carry an invalid all-zero token to userspace. + if shim::random_bytes(&mut bytes).is_err() || bytes.iter().all(|b| *b == 0) { + return None; } Some(bytes) } @@ -2663,9 +2677,18 @@ impl SepData { if let Some(user) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) { let special = user.special_handle(); if let Ok(keybag::State::Present(stored)) = keybag::read(keybag::Slot::Identity) { - unlocked = self.sks_step(crate::sks::SKS_LOCK_STATE_NAME, |healthy| { - self.sks_req_unlock_special(special, stored.secret(), healthy) - }).is_some(); + unlocked = match self.profile.key_store { + profile::KeyStore::Sepos13 { .. } => self + .sks_step(c"DEVICE_STATE_TRANSITION", |healthy| { + self.sks_req_unlock_special(special, stored.secret(), healthy) + }) + .is_some(), + profile::KeyStore::Variant5 => self + .sks_step(crate::sks::SKS_LOCK_STATE_NAME, |healthy| { + self.sks_req_unlock_special(special, stored.secret(), healthy) + }) + .is_some(), + }; } } } @@ -2689,17 +2712,20 @@ impl SepData { }; let identities = self.log_identity_count(c"after Catacomb restore with sensor registered"); let identity_ready = matches!(identities, Some(n) if n > 0); - let ready = prepared && match_context && restored && identity_ready - && self.device_view_synced.load(Relaxed) && self.prove_restore(); - self.templates_restored.store(ready, Relaxed); - if !ready { + let candidate = prepared && match_context && restored && identity_ready; + self.cold_restore_candidate.store(candidate, Relaxed); + // COMPLETE_INIT has not synchronized the device view yet. Never make + // verification available on the preliminary Catacomb proof alone; + // complete_bringup finishes the proof and sets templates_restored. + self.templates_restored.store(false, Relaxed); + if !candidate { dev_err!( self.dev, - "matching unavailable this boot: cold-match preparation, SCRD credential, Catacomb restore, device view, or identity proof did not complete; template state is unproven\n" + "matching unavailable this boot: cold-match preparation, SCRD credential, Catacomb restore, or identity proof did not complete; template state is unproven\n" ); return; } - self.reconcile_identities(); + dev_info!(self.dev, "sbio: cold restore candidate ready; waiting for device-view proof\n"); } @@ -2707,6 +2733,7 @@ impl SepData { &self, source: crate::sks::KeyBagHandle, uuid: [u8; keybag::UUID_LEN], + provenance: keybag::UuidProvenance, ) -> bool { if !self.keybag_designated.load(Relaxed) { return false; @@ -2716,39 +2743,49 @@ impl SepData { return false; }; let special = user.special_handle(); - // identity_load ends with the source unload (0xfffffe000994bad4); - // 13.5 sends nothing between its 0x0d and that unload. - if self.profile.key_store != profile::KeyStore::Variant5 { - return self.sks_send(self.sks_req_unload_keybag(source)).is_some(); - } self.log_identity_count(c"before the cold-match preparation"); + let Some(source_uuid) = self.sks_read_uuid(source) else { + return false; + }; + let generated_lookup = matches!(self.profile.key_store, profile::KeyStore::Sepos13 { .. }) + && provenance == keybag::UuidProvenance::AsGenerated; + let Some(identity) = keybag::SnapshotIdentity::from_loaded( + uuid, provenance, source_uuid, generated_lookup, + ) else { + dev_err!(self.dev, "sks: recovered source bag does not match its stored identity\n"); + return false; + }; let uuid_ok = self .sks_send(self.sks_req_copy_uuid_special(special)) .and_then(|out| self.sks_uuid_from_reply(&out)); - match uuid_ok { - Some(got) if got == uuid => {} - Some(_) => { - return false; - } - None => { - return false; - } + if !uuid_ok.is_some_and(|got| identity.accepts(&uuid, provenance, &got)) { + dev_err!(self.dev, "sks: designated bag does not match the recovered source bag\n"); + return false; } if self.sks_send(self.sks_req_unload_keybag(source)).is_none() { return false; } + let mut material = self.enrol_material.lock(); + let Some(material) = &mut *material else { + return false; + }; + material.snapshot_identity = Some(identity); + dev_info!(self.dev, "sks: snapshot identity bound to recovered and designated bag; generated lookup {}\n", generated_lookup); + true } pub(crate) fn sep_random(&self, buf: &mut [u8]) -> Result<()> { - for chunk in buf.chunks_mut(4) { - let word = self.get_entropy_word()?; - let bytes = word.to_le_bytes(); - let take = chunk.len(); - chunk.copy_from_slice(&bytes[..take]); + // The trusted-key framework asks for a fresh plaintext key before + // passing it to SEP for sealing. The host CSPRNG is appropriate for + // that key; J313/13.5's unverified SEP control entropy replies have + // returned all zeros and must not be used as key material. + shim::random_bytes(buf)?; + if !buf.is_empty() && buf.iter().all(|b| *b == 0) { + return Err(EIO); } Ok(()) } diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 382080091d725..a5a7042e31e2d 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -18,6 +18,7 @@ mod fv; mod hwrng; mod image; mod keybag; +mod keybag_identity; mod profile; mod proto; mod refkey; @@ -260,6 +261,7 @@ impl CalibrationBlob { struct EnrolMaterial { special: crate::sks::SpecialHandle, secret: Secret, + snapshot_identity: Option, } struct ImageContext<'a> { @@ -584,6 +586,8 @@ struct SepData { sbio_ready: Atomic, templates_restored: Atomic, + // Provisional cold restore; COMPLETE_INIT must still prove the device view. + cold_restore_candidate: Atomic, sensor_calibrated: Atomic, sensor_key_loaded: Atomic, @@ -801,6 +805,7 @@ impl SepData { sensor_calibrated: Atomic::new(false), sensor_key_loaded: Atomic::new(false), templates_restored: Atomic::new(false), + cold_restore_candidate: Atomic::new(false), enrol_material <- new_mutex!(None), enrol_identity_candidates <- new_mutex!(KVec::new()), last_capture_end_ns: Atomic::new(0), @@ -1130,11 +1135,15 @@ impl SepData { } fn survey_hwrng(&self) -> Result<()> { - // A working control endpoint must answer four consecutive draws before - // it is exposed to the kernel RNG core. Early attach can beat the SEP - // endpoint exchange, so this runs only after that exchange settles. + // Successful transport is not proof of entropy: J313/13.5 has returned + // four zero words here. Do not advertise that source to the RNG core. + // This is a minimal sanity check, not a complete RNG health test. + let mut any_bits = 0u32; for _ in 0..4 { - let _ = self.get_entropy_word()?; + any_bits |= self.get_entropy_word()?; + } + if any_bits == 0 { + return Err(EIO); } self.register_hwrng() } @@ -2283,6 +2292,10 @@ module! { default: 0, description: "Opt-in J414s diagnostic: with zero live identities, select the system context and try saving the missing owner Catacomb", }, + refkey_v2: u8 { + default: 0, + description: "Explicit recovery slot for a new identity-bag context; preserves the original machine ref-key", + }, j414s_persistent_enrol: u8 { default: 0, description: "Opt-in J414s enrollment test: export owner from the fresh context, then save user before master at completion", diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index 07e5906bf3ac9..5266541882a17 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -777,6 +777,7 @@ impl SepData { *self.enrol_material.lock() = Some(EnrolMaterial { special, secret: Secret(copy), + snapshot_identity: None, }); } @@ -882,11 +883,15 @@ impl SepData { &self, stored: &keybag::StoredKeyBag, ) -> Option<(crate::sks::KeyBagHandle, [u8; keybag::UUID_LEN])> { - // 13.5 keeps the identity bag in the enclave: identity_open - // (0xfffffe000994b0e4) loads it by its 16-byte UUID (0x…b11c-b124). - let key = match self.profile.key_store { - profile::KeyStore::Sepos13 { .. } => &stored.uuid()[..], - profile::KeyStore::Variant5 => stored.wrapped(), + // 13.5 identity_open addresses an enclave-resident identity bag by + // its generated UUID. Older J313 records instead store the bag's + // read-back UUID and a wrapped bag, which must be loaded as a blob. + let uuid_load = matches!(self.profile.key_store, profile::KeyStore::Sepos13 { .. }) + && stored.uuid_provenance() == keybag::UuidProvenance::AsGenerated; + let key = if uuid_load { + &stored.uuid()[..] + } else { + stored.wrapped() }; let request = match self.sks_req_load_keybag(key) { Ok(request) => request, @@ -950,9 +955,7 @@ impl SepData { } let handle = crate::sks::KeyBagHandle::from_load_reply(handle); - // identity_load sends 0x0d straight after the load (0x…ba5c) and - // reads no UUID back. - if self.profile.key_store != profile::KeyStore::Variant5 { + if uuid_load { return Some((handle, *stored.uuid())); } @@ -977,12 +980,16 @@ impl SepData { } pub(crate) fn resnapshot_identity_keybag(&self) -> bool { - let special = { + let (special, identity) = { let material = self.enrol_material.lock(); let Some(material) = material.as_ref() else { return false; }; - material.special + let Some(identity) = material.snapshot_identity else { + dev_err!(self.dev, "sks: refusing snapshot without a recovered-bag identity binding\n"); + return false; + }; + (material.special, identity) }; let Some(out) = self.sks_send(self.sks_req_copy_keybag_special(special)) else { @@ -1000,7 +1007,7 @@ impl SepData { return false; }; - match keybag::replace_wrapped(keybag::Slot::Identity, &wrapped, &uuid) { + match keybag::replace_wrapped(keybag::Slot::Identity, &wrapped, &uuid, &identity) { Ok(()) => true, Err(e) => { dev_err!( @@ -1013,7 +1020,7 @@ impl SepData { } } - fn sks_read_uuid(&self, handle: crate::sks::KeyBagHandle) -> Option<[u8; keybag::UUID_LEN]> { + pub(crate) fn sks_read_uuid(&self, handle: crate::sks::KeyBagHandle) -> Option<[u8; keybag::UUID_LEN]> { let out = self.sks_send(self.sks_req_copy_keybag_uuid(handle))?; self.sks_uuid_from_reply(&out) } diff --git a/tools/aurora-sep/tests/test_host_entropy.py b/tools/aurora-sep/tests/test_host_entropy.py new file mode 100644 index 0000000000000..6c1304925cb89 --- /dev/null +++ b/tools/aurora-sep/tests/test_host_entropy.py @@ -0,0 +1,111 @@ +"""Compile the driver's actual entropy methods with deterministic RNG stubs.""" +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[3] +DRIVER = ROOT / "drivers/soc/apple" + + +def method(file, name): + source = (DRIVER / file).read_text() + match = re.search(r"^ (?:pub\(crate\) )?fn " + name + r"\(", source, re.M) + if match is None: + raise AssertionError(f"Missing driver method {name}") + opening = source.index("{", match.start()) + depth = 1 + end = opening + 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[match.start():end] + + +@unittest.skipUnless(shutil.which("rustc"), "rustc required for driver method tests") +class EntropyTests(unittest.TestCase): + def test_driver_entropy_methods(self): + harness = r''' +use std::cell::{Cell, RefCell}; +type Result = std::result::Result; +const EIO: u8 = 5; +mod bio { pub const TOKEN_LEN: usize = 32; } +thread_local! { static MODE: Cell = const { Cell::new(0) }; } +mod shim { + pub fn random_bytes(buf: &mut [u8]) -> super::Result<()> { + super::MODE.with(|m| match m.get() { + 0 => { buf.fill(0x5a); Ok(()) }, + 1 => { buf.fill(0); Ok(()) }, + 2 => Err(4), + _ => { buf.fill(0); *buf.last_mut().unwrap() = 1; Ok(()) }, + }) + } +} +struct SepData { words: RefCell>>, registered: Cell } +impl SepData { + fn get_entropy_word(&self) -> Result { self.words.borrow_mut().remove(0) } + fn register_hwrng(&self) -> Result<()> { self.registered.set(true); Ok(()) } +METHODS +} +fn device(words: Vec>) -> SepData { + SepData { words: RefCell::new(words), registered: Cell::new(false) } +} +#[test] fn token_success() { + MODE.with(|m| m.set(0)); + assert_eq!(device(vec![]).mint_token_bytes(), Some([0x5a; 32])); +} +#[test] fn token_zero_rejected() { + MODE.with(|m| m.set(1)); + assert!(device(vec![]).mint_token_bytes().is_none()); +} +#[test] fn token_rng_error_rejected() { + MODE.with(|m| m.set(2)); + assert!(device(vec![]).mint_token_bytes().is_none()); +} +#[test] fn token_last_byte_counts() { + MODE.with(|m| m.set(3)); + assert_eq!(device(vec![]).mint_token_bytes().unwrap()[31], 1); +} +#[test] fn trusted_key_random_success() { + MODE.with(|m| m.set(0)); + let mut b = [0; 32]; + assert_eq!(device(vec![]).sep_random(&mut b), Ok(())); + assert_eq!(b, [0x5a; 32]); +} +#[test] fn trusted_key_zero_rejected() { + MODE.with(|m| m.set(1)); + assert_eq!(device(vec![]).sep_random(&mut [0; 32]), Err(EIO)); +} +#[test] fn trusted_key_error_propagated() { + MODE.with(|m| m.set(2)); + assert_eq!(device(vec![]).sep_random(&mut [0; 32]), Err(4)); +} +#[test] fn zero_hwrng_not_registered() { + let d = device(vec![Ok(0); 4]); + assert_eq!(d.survey_hwrng(), Err(EIO)); assert!(!d.registered.get()); +} +#[test] fn answered_hwrng_registered() { + let d = device(vec![Ok(0), Ok(1), Ok(0), Ok(0)]); + assert_eq!(d.survey_hwrng(), Ok(())); assert!(d.registered.get()); +} +#[test] fn failed_hwrng_not_registered() { + let d = device(vec![Ok(1), Err(4)]); + assert_eq!(d.survey_hwrng(), Err(4)); assert!(!d.registered.get()); +} +''' + methods = "\n".join([method("sbio.rs", "mint_token_bytes"), + method("sbio.rs", "sep_random"), + method("sep.rs", "survey_hwrng")]) + with tempfile.TemporaryDirectory(prefix="sep-entropy-test-") as directory: + source = Path(directory) / "entropy.rs" + binary = Path(directory) / "entropy-tests" + source.write_text(harness.replace("METHODS", methods)) + subprocess.run(["rustc", "--edition=2021", "--test", str(source), + "-o", str(binary)], check=True) + subprocess.run([str(binary)], check=True) + + +if __name__ == "__main__": + unittest.main() From fb9166601646f148b03a07b73907eeade37216f7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 23:59:59 -0400 Subject: [PATCH 0739/2680] tools: aurora-sep: synchronize libfprint and document M1 recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Carry live-identity stale-gallery handling and capture guidance into the distributable libfprint patch. Document the tested M1 fixes, opt-in reference-key recovery, and exact build/hardware validation boundaries. No desktop or TUI configuration is included. [ryan: rebased onto the linear SEP history — the libfprint patch had changed on the J700 line in between (ioctl interface v4 additions); it is taken as aurora-wip merged it in 481c02bd687e] Signed-off-by: Ryan Murray --- tools/aurora-sep/M1-SUPPORT.md | 98 +++++++++++++++++++ tools/aurora-sep/README.md | 5 + .../libfprint-1.94.100-apple-sep.patch | 59 +++++++++-- 3 files changed, 154 insertions(+), 8 deletions(-) create mode 100644 tools/aurora-sep/M1-SUPPORT.md diff --git a/tools/aurora-sep/M1-SUPPORT.md b/tools/aurora-sep/M1-SUPPORT.md new file mode 100644 index 0000000000000..774af1ef10ca0 --- /dev/null +++ b/tools/aurora-sep/M1-SUPPORT.md @@ -0,0 +1,98 @@ +# M1 SEP firmware, enrollment and recovery + +The T8103 profile implements the SEP request shapes exercised on a J313 +MacBook Air booting the macOS 13.5 (22G74) firmware. This firmware needs the +key-store capability/environment initialization, v2 identity creation, UUID +lookup, v0 designation, and `DEVICE_STATE_TRANSITION` unlock. The profile is +selected by hardware today; other firmware combinations need validation. + +## Required fixes + +* Initialize SBIO communication with opcode `0x73` and `u32(1)` before ordinary + SBIO commands. Sensor setup and SCRD authentication could succeed without + this, while `BEGIN_ENROL` still failed. +* Use the sensor's DT SPI mode and `enable-gpios`. Keep the hardware CS timing + callback confined to the Mesa bus: assigning it to every SPI controller + broke M1 keyboard/trackpad setup. +* Begin 13.5 sensor registration with the serial operation. The newer `0x80` + registration was rejected by the tested firmware. +* Keep the generated identity lookup UUID distinct from the recovered bag's + UUID. Bind snapshots to the recovered and designated bag, preserving the + stored lookup UUID and rejecting changes to that binding. +* Export the owner Catacomb in a fresh context, then save the completed user + before the master. The tested T8103/T6020 profiles select this automatically; + other profiles retain the existing opt-in control. +* Treat restored Catacombs as provisional until COMPLETE_INIT and the device + view checks finish. Requiring the final ready bit before setting that same + bit prevented cold-boot matching. +* Generate host match tokens and trusted-key plaintext with the initialized + Linux CSPRNG. The tested J313 SEP control entropy source returned zeros, + which made libfprint reject kernel matches. The hardware RNG survey now + rejects an all-zero sample; this is not a complete firmware RNG fix. +* Use the 13.5 reference-key envelope, EC type and designated identity session + for sealing/unsealing. Existing reference-key files are retained on failure. + +The libfprint patch also checks SEP's live count before IDENTIFY. An explicitly +valid zero count can answer a stale host gallery as no-match and permit fresh +enrollment. An unavailable count is not zero, and VERIFY keeps its normal +match/token checks. Capture guidance does not count as enrollment progress. + +## Existing installations + +Keep the existing keybag, ref-key, Catacombs and fprintd records backed up +root-only. Never copy them between machines, restore old raw xART snapshots, +or delete a keybag to repair a fingerprint error. A host-listed finger is not +proof that SEP restored its template; inspect the live count and restore logs. + +One test installation had replaced its identity bag during earlier experiments +while retaining a reference key from the former bag. The old reference key +could still encrypt using its public half, but SEP rejected unseal with `-7`. +A separate `refkey_v2=1` opt-in selects +`/var/lib/aurora-sep-refkey-v2.bin`, leaving the original ref-key file intact. +It is a recovery slot for an explicitly selected new context, not an automatic +migration or a way to recover old sealed data. Ordinary installations should +keep the default slot. Always verify new/load and cold-boot reload before +using sealed keys for persistent data. The per-seal round-trip diagnostic is +not included in the production path. + +`j414s_persistent_enrol=1` remains available for explicitly testing the owner +export/save order on other profiles. T8103 and T6020 no longer need it. +Raw xART writes remain opt-in and subject to the extent ownership checks in +[XART-SAFETY.md](XART-SAFETY.md). The kernel includes no raw private tracing +or experimental dummy-keybag/enrollment protocol switches from the investigation. + +## Validation boundaries + +The deployed J313 source completed enrollment and same-boot matching reported +by the tester. After reboot, all three Catacombs loaded, SEP listed one live +identity, and the final device-view proof enabled matching. New reference-key +seal/unseal and a saved blob's cold-boot unseal passed. SPI keyboard/trackpad +drivers bound after the CS timing fix. A physical fingerprint unlock through +the configured desktop PAM service has not been observed remotely. + +The consolidated branch preserves the later Neo deferred SMC supplier and +capture-guidance fixes that the older J313 test checkout did not contain. +Build/test results for this consolidation are not a new hardware regression +test on Neo, M2, or every M1 board. The consolidated SEP module compiled against +the prepared J313 kernel build. Its four identity-binding tests, ten extracted +entropy-method tests, and eight calibration/xART tests passed. The bundled +patch applied to clean libfprint 1.94.100 and built with `drivers=aurora`; +the three core unit-test suites passed. In that minimal build, 33 unrelated +driver/hwdb tests were skipped and AppStream validation rejected the generated +empty `` device list. No full-kernel rebuild or new hardware run of +the exact consolidated tree is claimed. Reference-key attestation/signing is +not covered by the sealing/unsealing results. + +Source-only identity tests can be run with: + +```sh +rustc --edition=2021 --test drivers/soc/apple/keybag_identity.rs -o /tmp/sep-identity-tests +/tmp/sep-identity-tests +python3 -m unittest discover -s tools/aurora-sep/tests -v +``` + +Desktop authentication setup is installation-specific. The tested Air runs +Chonkstep with Omarchy Quickshell. That shell uses separate password and +fingerprint PAM services; configuring an unused hyprlock file has no effect. +The machine's GRUB entries, automatic SEP loader and PAM edits are deployment +configuration and are not installed implicitly by this kernel branch. diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index bfe086e48a2c6..300fa97c935a5 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -3,6 +3,11 @@ These files connect the Apple SEP kernel driver to the shared APFS xART gigalocker and the desktop fingerprint stack. +See [M1-SUPPORT.md](M1-SUPPORT.md) for the T8103 firmware requirements, +reboot-persistent enrollment fixes, reference-key recovery caveats and current +validation boundaries. The historical bring-up results below describe the +individual builds at the time they were tested. + Install the driver service: ```sh diff --git a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch index 3faef0fdf1dc2..bdb15bb1c2b27 100644 --- a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch +++ b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch @@ -1,9 +1,8 @@ diff --git a/libfprint/drivers/aurora/aurora.c b/libfprint/drivers/aurora/aurora.c new file mode 100644 -index 0000000..80ace06 --- /dev/null +++ b/libfprint/drivers/aurora/aurora.c -@@ -0,0 +1,837 @@ +@@ -0,0 +1,881 @@ +/* + * Aurora SEP fingerprint driver + * @@ -77,7 +76,7 @@ index 0000000..80ace06 + + case ENOSPC: + g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_FULL, -+ "no space for another identity"); ++ "three SEP fingerprint slots are full; delete one before enrolling"); + break; + + case ENOENT: @@ -343,6 +342,7 @@ index 0000000..80ace06 + + self->nonce_valid = FALSE; + memset (self->nonce, 0, sizeof (self->nonce)); ++ fpi_device_report_finger_status (device, FP_FINGER_STATUS_NONE); + + fpi_device_close_complete (device, NULL); +} @@ -352,10 +352,28 @@ index 0000000..80ace06 +{ + FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); + ++ fpi_device_report_finger_status (device, FP_FINGER_STATUS_NONE); + if (self->fd >= 0) + ioctl (self->fd, SEP_BIO_CANCEL); +} + ++static void ++aurora_report_guidance (FpDevice *device, guint32 guidance, gboolean active) ++{ ++ FpFingerStatusFlags status = FP_FINGER_STATUS_NONE; ++ ++ if (active) ++ { ++ if (guidance == SEP_BIO_GUIDANCE_HOLD_STILL) ++ status = FP_FINGER_STATUS_PRESENT; ++ else if (guidance == SEP_BIO_GUIDANCE_PLACE || ++ guidance == SEP_BIO_GUIDANCE_LIFT_AND_MOVE) ++ status = FP_FINGER_STATUS_NEEDED; ++ } ++ ++ fpi_device_report_finger_status (device, status); ++} ++ +/* ------------------------------------------------------------------ */ +/* enrolment */ +/* ------------------------------------------------------------------ */ @@ -372,10 +390,15 @@ index 0000000..80ace06 + if (!aurora_ioctl (self, SEP_BIO_ENROL_POLL, &poll_res, &error)) + { + self->watch_id = 0; ++ fpi_device_report_finger_status (device, FP_FINGER_STATUS_NONE); + fpi_device_enroll_complete (device, NULL, g_steal_pointer (&error)); + return G_SOURCE_REMOVE; + } + ++ aurora_report_guidance (device, poll_res.guidance, ++ poll_res.state == SEP_BIO_STATE_PENDING || ++ poll_res.state == SEP_BIO_STATE_PROGRESS); ++ + switch (poll_res.state) + { + case SEP_BIO_STATE_PENDING: @@ -444,6 +467,7 @@ index 0000000..80ace06 + } + + self->enroll_last_reported_stage = 0; ++ aurora_report_guidance (device, SEP_BIO_GUIDANCE_PLACE, TRUE); + aurora_stop_watch (self); + self->watch_id = g_unix_fd_add (self->fd, G_IO_IN, aurora_enrol_ready, device); +} @@ -519,6 +543,7 @@ index 0000000..80ace06 + if (!aurora_ioctl (self, SEP_BIO_VERIFY_POLL, &res, &error)) + { + self->watch_id = 0; ++ fpi_device_report_finger_status (device, FP_FINGER_STATUS_NONE); + /* Terminal path: void the per-operation nonce like every other exit, so + * a nonce cannot be carried into a later operation. */ + self->nonce_valid = FALSE; @@ -530,6 +555,10 @@ index 0000000..80ace06 + return G_SOURCE_REMOVE; + } + ++ aurora_report_guidance (device, res.guidance, ++ res.state == SEP_BIO_STATE_PENDING || ++ res.state == SEP_BIO_STATE_PROGRESS); ++ + if (res.state == SEP_BIO_STATE_PENDING || + res.state == SEP_BIO_STATE_PROGRESS) + return G_SOURCE_CONTINUE; @@ -611,6 +640,7 @@ index 0000000..80ace06 +{ + FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); + struct sep_bio_verify_start start = { 0 }; ++ struct sep_bio_info info = { 0 }; + FpiDeviceAction action = fpi_device_get_current_action (device); + g_autoptr(GError) error = NULL; + FpPrint *print = NULL; @@ -633,11 +663,11 @@ index 0000000..80ace06 + * So the first enrolment on a fresh device was impossible, and the message + * blamed the cable. Answer the empty case here and the enrol proceeds. + * -+ * A NON-empty gallery still goes to the kernel and still fails until match -+ * is implemented. That is correct: we must never report "no match" for a -+ * gallery we did not actually search. Reporting a *negative* we can prove is -+ * safe; inventing one we cannot is how a biometric stack silently stops -+ * authenticating anyone. ++ * A NON-empty gallery normally goes to the enclave for matching. A stale ++ * host index may outlive its SEP identity after a failed cold restore; only ++ * an explicit, valid zero live-identity count from SEP lets us answer that ++ * gallery without capture. An unavailable count must never be treated as ++ * zero, and VERIFY still goes through the kernel's restore proof gate. + */ + if (action == FPI_DEVICE_ACTION_IDENTIFY) + { @@ -652,6 +682,18 @@ index 0000000..80ace06 + fpi_device_identify_complete (device, NULL); + return; + } ++ ++ if (ioctl (self->fd, SEP_BIO_GET_INFO, &info) == 0 && ++ info.version == SEP_BIO_IFACE_VERSION && ++ (info.reserved[0] & SEP_BIO_INFO_LIVE_COUNT_VALID) != 0 && ++ info.reserved[1] == 0) ++ { ++ fp_dbg ("SEP proved zero live identities despite a non-empty host " ++ "gallery: reporting no match for identify only"); ++ fpi_device_identify_report (device, NULL, NULL, NULL); ++ fpi_device_identify_complete (device, NULL); ++ return; ++ } + } + + /* A fresh nonce per operation is what makes the result non-replayable. If we @@ -680,6 +722,7 @@ index 0000000..80ace06 + return; + } + ++ aurora_report_guidance (device, SEP_BIO_GUIDANCE_PLACE, TRUE); + aurora_stop_watch (self); + self->watch_id = g_unix_fd_add (self->fd, G_IO_IN, aurora_verify_ready, device); +} From a0c786c35fcd62b14c57cb0a09d2570e550519b9 Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Fri, 25 Sep 2026 01:36:11 -0600 Subject: [PATCH 0740/2680] soc: apple: sep: dt: fix a refcount double-put in node_at_address() of_find_node_with_property() puts the reference on its 'from' node when it advances. node_at_address() also wrapped every candidate in a DtNode, whose Drop calls of_node_put(), so each non-matching node was put twice. This path runs whenever no apple,mesa-fingerprint node exists, such as on t6000, and walks every reg-bearing node in the tree. Keep candidates in ManuallyDrop and hand out an owned DtNode only on a match. Assisted-by: Claude Opus 5.5 Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- drivers/soc/apple/dt.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/dt.rs b/drivers/soc/apple/dt.rs index 46dd70d15d03f..48a4732008967 100644 --- a/drivers/soc/apple/dt.rs +++ b/drivers/soc/apple/dt.rs @@ -325,9 +325,12 @@ fn node_at_address(base: u64) -> Option { if np.is_null() { return None; } - let node = DtNode(np); + // The iterator owns this reference and puts it on the next call, so a + // non-matching node must not be dropped here; only the match is handed + // back as an owned `DtNode`. + let node = core::mem::ManuallyDrop::new(DtNode(np)); if node.reg_base() == Some(base) { - return Some(node); + return Some(core::mem::ManuallyDrop::into_inner(node)); } } } From dec9e4549af5a8f70082612f26a84eed87d7a2c9 Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Fri, 25 Sep 2026 01:41:03 -0600 Subject: [PATCH 0741/2680] soc: apple: sep: add a T6000 (M1 Pro) platform profile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit profile::detect() matched only apple,t8103, apple,t6020 and apple,t8140, so on an M1 Pro the driver stopped with ENODEV before touching the hardware. The M1 family cold-boots its SEP, so T6000 takes the T8103 boot handshake, shared-memory geometry, identity-keybag create values and Sepos13 key store. The J316s ADT's /defaults carries cpx-encryption-mode = 2, the same as J293, J313 and J414s. The sensor is on SPI2 at 0x39b108000, the T6020 address. The J313, J316s and J414s platform device trees describe the Mesa sensor identically (same reg blob, 8 MHz, sensor-id 0x3352), so the profile takes T6020's hardware-verified SPI mode 2. Enrolment persistence is not yet reboot-tested on J316s, so persistent_enrol stays off; j414s_persistent_enrol=1 opts in. Tested on J316s with stock m1n1 1.6.1 and xart_writes=0: SEPOS cold-boots and advertises all 12 endpoints, and /dev/sep-bio is published. [ryan: rebased onto the linear SEP history — the new T6000 profile also sets nonce_session: false, the field the J700 work (commit 1ae0ac48652a ("soc: apple: sep: establish authenticated sensor sessions on J700")) added to every profile] Assisted-by: Claude Opus 5.5 Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Signed-off-by: Ryan Murray --- drivers/soc/apple/profile.rs | 48 +++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 6dbaf5d878c8d..cacc994e7385b 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -9,10 +9,12 @@ //! here once. The driver reads addresses from the device tree, never scans for //! them, and never patches properties at runtime. //! -//! Three bring-up targets are modelled: +//! Four bring-up targets are modelled: //! //! * `T8103` / J313 (MacBook Air, M1): the host boots the SEP with the boot //! endpoint handshake over a 0x30000 shared-memory window. +//! * `T6000` / J316s (MacBook Pro 16", M1 Pro): the T8103 boot handshake and +//! key store, with the sensor on the T6020 SPI2 address. //! * `T6020` / J414s (MacBook Pro 14", M2 Pro): the driver does the warm //! single-message registration over a 0x40000 window. //! * `T8140` / J700 (MacBook Neo, A18 Pro): iBoot boots sepOS before the AP @@ -180,6 +182,44 @@ const T8103: PlatformProfile = PlatformProfile { nonce_session: false, }; +/// M1 Pro (J316s). The M1 family cold-boots its SEP like T8103, so the boot +/// path, shared-memory geometry and key store follow T8103. The sensor sits on +/// SPI2 at the T6020 address, in the mode T6020 verified: the J313, J316s and +/// J414s platform device trees describe the same sensor (id 0x3352) +/// identically. Enrolment persistence is not yet reboot-tested here, so it +/// stays opt-in (`j414s_persistent_enrol=1`). +const T6000: PlatformProfile = PlatformProfile { + name: "T6000/J316s", + shmem_capacity: 0x3_0000, + shmem_first_item: b"CNIP", + bootstrap: Bootstrap::Boot, + identity: IdentitySource::Chosen, + sensor: SensorProfile { + controller_base: 0x3_9b10_8000, + chip_select: 0, + max_hz: 8_000_000, + cs_setup_ns: 20, + cs_hold_ns: 20, + mode: SpiMode::Mode2, + expected_id: 0x3352, + capture_qualified: false, + }, + dart_range_required: false, + wide_dma_mask: false, + firmware_region: c"sepfw", + keybag_create: KeybagCreate { + variant: T8103.keybag_create.variant, + bag_type: T8103.keybag_create.bag_type, + arg: T8103.keybag_create.arg, + }, + // The J316s ADT carries `/defaults` `cpx-encryption-mode = 2`. + key_store: KeyStore::Sepos13 { + cpx_encryption_mode: 2, + }, + persistent_enrol: false, + nonce_session: false, +}; + const T6020: PlatformProfile = PlatformProfile { name: "T6020/J414s", shmem_capacity: 0x4_0000, @@ -251,6 +291,9 @@ static_assert!(T8103.sensor.controller_base == 0x235108000); static_assert!(T6020.sensor.controller_base == 0x39b108000); static_assert!(T8140.sensor.controller_base == 0x385108000); static_assert!(T8103.sensor.expected_id == T6020.sensor.expected_id); +static_assert!(T6000.shmem_capacity == T8103.shmem_capacity); +static_assert!(T6000.sensor.controller_base == T6020.sensor.controller_base); +static_assert!(T6000.sensor.expected_id == T6020.sensor.expected_id); /// Whether the machine root declares `compatible`. /// @@ -289,6 +332,9 @@ pub(crate) fn detect() -> Result<&'static PlatformProfile> { if machine_has(b"apple,t6020") { return Ok(&T6020); } + if machine_has(b"apple,t6000") { + return Ok(&T6000); + } if machine_has(b"apple,t8140") { return Ok(&T8140); } From 0aefb1fd92eb8f85b81bbb184dd2a22006e27469 Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Fri, 25 Sep 2026 02:56:50 -0600 Subject: [PATCH 0742/2680] soc: apple: sep: log endpoint discovery and a silent empty attach When the SEP advertised no endpoints, tick_attach() moved to READY without a word, so the boot looked stalled after the IMG4 acknowledgement with nothing to say why. Discovery and messages on unhandled endpoints were not logged either, which left no way to see what SEPOS sent. Log each discovery message, each message on an endpoint the driver does not handle, the endpoint count when attach settles, and an error when it settles with none. Assisted-by: Claude Opus 5.5 Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- drivers/soc/apple/sep.rs | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index a5a7042e31e2d..71ef59b733bcd 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -1828,9 +1828,20 @@ impl SepData { } if this.endpoint_count() == 0 { + dev_err!( + this.dev, + "attach: {} messages received but no endpoint advertised; xART and key store cannot run\n", + now + ); this.phase.store(PHASE_READY, Relaxed); return; } + dev_info!( + this.dev, + "attach: {} endpoints advertised in {} messages\n", + this.endpoint_count(), + now + ); this.prepare_os_uuid(); @@ -1914,7 +1925,7 @@ impl SepData { proto::EP_BOOT => self.on_boot(msg), - _ep => {}, + ep => dev_info!(self.dev, "rx: unhandled endpoint {:#04x} (msg0 {:#018x})\n", ep, msg.msg0), } } @@ -1935,13 +1946,21 @@ impl SepData { } } - fn on_discovery(&self, _msg: Message, f: proto::Fields) { + fn on_discovery(&self, msg: Message, f: proto::Fields) { let mut table = self.endpoints.lock(); match f.ty { proto::DISCOVER_TYPE_DESCRIPTOR | proto::DISCOVER_TYPE_CONFIG => { let _ = table.slot(f.param); + dev_info!( + self.dev, + "discover: endpoint {:#04x} (type {}, msg0 {:#018x}); {} known\n", + f.param, + f.ty, + msg.msg0, + table.eps.len() + ); } - _ => {} + ty => dev_info!(self.dev, "discover: other type {} (msg0 {:#018x})\n", ty, msg.msg0), } } From a5c5e0b4f1a28d52f3a7717dfe54de1bb0d6545f Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:02:47 -0600 Subject: [PATCH 0743/2680] soc: apple: sep: wait for endpoint discovery, not the first message tick_attach() held the FIRST_RESPONSE_MS window open only while no message at all had arrived. On the cold-boot path the TZ0 and IMG4 acknowledgements are messages, so the first 200 ms quiet gap after IMG4 ended the attach with no endpoints known, and the driver stopped before the xART exchange. On J316s discovery begins ~367 ms after the IMG4 acknowledgement: 36.836322 boot: IMG4 acknowledged 37.176155 attach settled with no endpoints 37.203042 discover: endpoint 0x14 ... Keep waiting, up to FIRST_RESPONSE_MS of quiet, until at least one endpoint has been advertised. Assisted-by: Claude Opus 5.5 Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- drivers/soc/apple/sep.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 71ef59b733bcd..5d66265dac11c 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -1811,8 +1811,11 @@ impl SepData { fn tick_attach(this: &Arc) { let now = this.rx_count.load(Relaxed); - // HW: the SEP takes ~265 ms to answer a registration - if now == 0 { + // HW: the SEP takes ~265 ms to answer a warm registration, and on the + // cold-boot path J316s starts discovery ~370 ms after the IMG4 ack. + // The boot acks count as traffic, so wait for the first endpoint, not + // the first message, or a quiet gap after IMG4 ends the attach early. + if this.endpoint_count() == 0 { let ticks = this.settle_idle_ticks.load(Relaxed).wrapping_add(1); this.settle_idle_ticks.store(ticks, Relaxed); if ticks.saturating_mul(u64::from(SETTLE_MS)) < u64::from(FIRST_RESPONSE_MS) { From e321923529a7445504f201f46021017d39c1c051 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 29 Sep 2026 08:06:27 -0700 Subject: [PATCH 0744/2680] soc: apple: sep: run the T6020 profile on the M2 Max (t6021) profile::detect() matched apple,t8103, apple,t6020, apple,t6000 and apple,t8140, so an M2 Max returned ENODEV. Module init discarded that error, which left the apple,sep node disabled with nothing probed and nothing logged -- indistinguishable from a driver that started cleanly. The M2 Max needs no constants of its own. t6020.dtsi is written as a cut-down t6021: it includes t6021.dtsi and disables the parts the smaller die lacks. Both parts pull in t602x-die0.dtsi, which places the SEP at 0x396400000, its DART at 0x394ac0000 and the same four mailbox interrupts on either die, and t602x-j414-j416.dtsi already carries the sep alias for j414c alongside j414s. So map apple,t6021 onto T6020. Bootstrap stays WarmRegister and dart_range_required stays true, both inherited from T6020, so the existing gate still applies the SEP/DART changeset on this platform. Also log the unmatched-SoC case instead of swallowing it, so the next unlisted SoC says so rather than failing silently. Co-Authored-By: Claude Opus 5 (1M context) --- drivers/soc/apple/profile.rs | 8 ++++++++ drivers/soc/apple/sep.rs | 14 +++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index cacc994e7385b..5d59f7e1b5f3d 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -332,6 +332,14 @@ pub(crate) fn detect() -> Result<&'static PlatformProfile> { if machine_has(b"apple,t6020") { return Ok(&T6020); } + // t6020.dtsi is defined as a cut-down t6021: it includes t6021.dtsi and + // disables the parts the smaller die lacks. Both pull in t602x-die0.dtsi, + // so the SEP, its DART and the SPI controller sit at the same addresses + // with the same interrupts on either part, and the T6020 constants apply + // to the M2 Max unchanged. + if machine_has(b"apple,t6021") { + return Ok(&T6020); + } if machine_has(b"apple,t6000") { return Ok(&T6000); } diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 5d66265dac11c..71ad29f87a61e 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -2265,10 +2265,18 @@ impl kernel::InPlaceModule for SepModule { // changeset; the cold-boot target describes them statically (its DART // is enabled before probe), so no changeset runs there. _dt: { - if let Ok(p) = profile::detect() { - if matches!(p.bootstrap, profile::Bootstrap::WarmRegister) { - dt::enable_sep_and_dart()?; + match profile::detect() { + Ok(p) => { + if matches!(p.bootstrap, profile::Bootstrap::WarmRegister) { + dt::enable_sep_and_dart()?; + } } + // Without this the module loads, probes nothing and logs + // nothing on an unlisted SoC, which is indistinguishable + // from a driver that started correctly. + Err(_) => pr_info!( + "apple_sep: no platform profile for this SoC; the SEP stays disabled\n" + ), } }, From 8feb0d063d7746007587796cea3b44a33d3cd728 Mon Sep 17 00:00:00 2001 From: Vincent Fontaine Date: Thu, 1 Oct 2026 19:06:49 -0700 Subject: [PATCH 0745/2680] soc: apple: sep: add a T8112 (MacBook Air M2) platform profile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit T8112 is a warm-registration platform: iBoot hands over a running SEP and a powered Mesa sensor (J415: AP GPIO 178 is a driven-high output at handoff). The values come from the J415 ADT (/arm-io/spi2/mesa and /arm-io/sep): spi2 at the T8103 address, power on AP GPIO 178, data-ready on 177, 8 MHz, chip select 0, a SEP DART aperture of 0x4000 + 0xffff0000 (below 4 GiB, so no wide DMA mask) and /defaults cpx-encryption-mode = 2. They match the J413 values posted in the omacom/linux#7 thread. The shared memory follows the other warm platforms (0x40000, CINP). The key store speaks Variant5: J415 accepted the 13.5 GET_CAPABILITIES and SET_ENV init but refused the strict CREATE_KEYBAG (mailbox status -1, empty reply), as J700 did, and provisioned with the T6020 encoding. Tested on J415 (MacBook Air 15", M2) with the patched m1n1 (no SEP RNG query on M2, apfs-preboot-uuid forwarded): - read-only (xart_writes=0): warm attach, all 12 endpoints including key store 0x12, sbio, scrd and hwrng, on 3 cold boots; the iBoot system container image was byte-identical afterwards - writes on: identity keybag and ref-key provisioned, trusted-key seal/unseal round trip passed, fprintd enrol completed (user and master catacombs at state 7), verify-match in the same boot and again after a reboot [ryan: rebased onto the linear SEP history — the new T8112 profile also sets nonce_session: false, the field the J700 work (commit 1ae0ac48652a ("soc: apple: sep: establish authenticated sensor sessions on J700")) added to every profile] Signed-off-by: Ryan Murray --- drivers/soc/apple/profile.rs | 51 +++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 5d59f7e1b5f3d..c97bb4a0e8490 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -9,7 +9,7 @@ //! here once. The driver reads addresses from the device tree, never scans for //! them, and never patches properties at runtime. //! -//! Four bring-up targets are modelled: +//! Five bring-up targets are modelled: //! //! * `T8103` / J313 (MacBook Air, M1): the host boots the SEP with the boot //! endpoint handshake over a 0x30000 shared-memory window. @@ -19,6 +19,8 @@ //! single-message registration over a 0x40000 window. //! * `T8140` / J700 (MacBook Neo, A18 Pro): iBoot boots sepOS before the AP //! OS, so the driver uses the warm registration path. +//! * `T8112` / J413, J415 (MacBook Air, M2): iBoot hands over a running SEP and +//! a powered sensor, so the driver uses the warm registration path. // The boot handshake, identity source, sensor and DART fields are consumed by // the boot-endpoint, identity and sensor-transport paths. @@ -284,6 +286,48 @@ const T8140: PlatformProfile = PlatformProfile { nonce_session: true, }; +/// MacBook Air M2 (J413, J415). iBoot leaves the SEP running and the Mesa +/// sensor powered (J415: AP GPIO 178 is a driven-high output at handoff), so +/// this is a warm-registration platform like T6020. The J415 ADT describes the +/// sensor exactly as J413's does: spi2 at the T8103 address, power on AP GPIO +/// 178, data-ready on 177, 8 MHz, chip select 0. The SEP DART aperture is +/// 0x4000 + 0xffff0000, below 4 GiB, so no wide DMA mask. `/defaults` +/// carries `cpx-encryption-mode = 2`. +/// +/// The shared-memory geometry and key-store dialect follow the other +/// warm-registration platforms (T6020, T8140). +const T8112: PlatformProfile = PlatformProfile { + name: "T8112/J415", + shmem_capacity: 0x4_0000, + shmem_first_item: b"CINP", + bootstrap: Bootstrap::WarmRegister, + identity: IdentitySource::Chosen, + sensor: SensorProfile { + controller_base: 0x2_3510_8000, + chip_select: 0, + max_hz: 8_000_000, + cs_setup_ns: 20, + cs_hold_ns: 20, + mode: SpiMode::Mode1, + expected_id: 0x3352, + capture_qualified: false, + }, + dart_range_required: true, + wide_dma_mask: false, + firmware_region: c"sepfw", + // J415 accepted the 13.5 GET_CAPABILITIES/SET_ENV init but refused the + // strict CREATE_KEYBAG (mailbox status -1, empty reply), as J700 did; + // use the T6020 form like the other warm-registration platforms. + keybag_create: KeybagCreate { + variant: 5, + bag_type: 0, + arg: -1, + }, + key_store: KeyStore::Variant5, + persistent_enrol: true, + nonce_session: false, +}; + static_assert!(T8103.shmem_capacity == 0x30000); static_assert!(T6020.shmem_capacity == 0x40000); static_assert!(T8140.shmem_capacity == 0x40000); @@ -294,6 +338,8 @@ static_assert!(T8103.sensor.expected_id == T6020.sensor.expected_id); static_assert!(T6000.shmem_capacity == T8103.shmem_capacity); static_assert!(T6000.sensor.controller_base == T6020.sensor.controller_base); static_assert!(T6000.sensor.expected_id == T6020.sensor.expected_id); +static_assert!(T8112.shmem_capacity == T6020.shmem_capacity); +static_assert!(T8112.sensor.controller_base == T8103.sensor.controller_base); /// Whether the machine root declares `compatible`. /// @@ -346,5 +392,8 @@ pub(crate) fn detect() -> Result<&'static PlatformProfile> { if machine_has(b"apple,t8140") { return Ok(&T8140); } + if machine_has(b"apple,t8112") { + return Ok(&T8112); + } Err(ENODEV) } From 377f9bfb7c3af2ce432228df411f5ebf9179a6ae Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:20:59 -0700 Subject: [PATCH 0746/2680] sep: boot Touch ID and Thunderbolt display without command-line switches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The merged Thunderbolt, AVD, and SEP drivers match the kernel running on the M1 Pro and the M1 Air. xART writes, identity-keybag creation, and t8103 Type-C display were still off, so a boot from this branch needed switches that kernel does not use. Default those three on. The first boot creates the identity keybag only when it is absent. Declare the mailbox, runtime PM, and firmware-loader dependencies the SEP driver uses. asahi.config now keeps the SEP module: without CONFIG_TRUSTED_KEYS the fragment's CONFIG_APPLE_SEP=m is dropped, and a TEE trust source would claim the key type first. The same fragment enables the Apple USB4 driver. [ryan: rebased onto the linear SEP history and split by topic — only the asahi.config TRUSTED_KEYS lines, which keep CONFIG_APPLE_SEP=m from being dropped, are carried. aurora-wip took neither its Kconfig dependency changes (003a6087f954 already declares them) nor its xART-write and identity-keybag defaults when composing this branch in 42ae23434d4c; it enabled those defaults later in d32c6c259c49. The thunderbolt dp_display default and the TYPEC/USB4 config lines are left to 170-usb and 900-config-tooling] Signed-off-by: Ryan Murray --- arch/arm64/configs/asahi.config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 77b97e2a92b02..e7bd59faac04a 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -61,6 +61,9 @@ CONFIG_APPLE_RTKIT_HELPER=m CONFIG_APPLE_SART=m CONFIG_RUST_APPLE_RTKIT=y CONFIG_APPLE_AOP=m +CONFIG_TRUSTED_KEYS=y +# CONFIG_TRUSTED_KEYS_TPM is not set +# CONFIG_TRUSTED_KEYS_TEE is not set CONFIG_APPLE_SEP=m CONFIG_APPLE_PMGR_PWRSTATE=y CONFIG_APPLE_PMP=m From 328e517aac33211ec8676caf3410b0d876439778 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:20:55 -0700 Subject: [PATCH 0747/2680] soc: apple: sep: drop the duplicated prefix from the unmatched-SoC message pr_info!() already prefixes the module name, so the message added for an unlisted SoC printed as apple_sep: apple_sep: no platform profile for this SoC; the SEP stays disabled as reported from a j413. Drop the hand-written prefix. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit 0adc8b4bb9b2363588422bdf105b96d3b7ac3d93) --- drivers/soc/apple/sep.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 71ad29f87a61e..78822a80aa2a9 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -2275,7 +2275,7 @@ impl kernel::InPlaceModule for SepModule { // nothing on an unlisted SoC, which is indistinguishable // from a driver that started correctly. Err(_) => pr_info!( - "apple_sep: no platform profile for this SoC; the SEP stays disabled\n" + "no platform profile for this SoC; the SEP stays disabled\n" ), } }, From 558992a74792b44a9d93a6178ef7555f0f177a16 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 29 Sep 2026 17:30:47 -0700 Subject: [PATCH 0748/2680] soc: apple: sep: log why a verify ends without a verdict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A verify that fails reaches userspace only as a bare "unspecified error": libfprint turns every failed verify into FP_DEVICE_ERROR_GENERAL and drops the driver's status. Most of verify_one_image()'s failure exits also log nothing, so a refused match step, a sensor fault and an unusable frame all look the same from both sides. That is what a J414c tester hit after restoring an enrolment across a reboot: every verify ended with ENROL_STATUS_ENCLAVE about 150-235 ms after the finger landed, with nothing in the kernel log to say which step refused it. Only an instrumented rebuild showed that MATCH_RESULT had answered 0x1 on an image the enclave had assessed as usable. Log each failure exit on the verify path. The four match steps (PREPARE_IMAGE_PROCESSING, IMAGE_PROCESSING_INIT, IMAGE_ASSESSMENT and MATCH_RESULT) go through a new sbio_verify_step(), which decides success exactly as sbio_call() does but records the enclave's answer when it is not success. A cancelled verify still ends quietly. Partial contact and bad frames are warnings, because they are routine; the rest are errors. No outcome changes. Only log lines are added. Reported-by: Scott Jones Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit 075cbf53ed571117181b479605734c92a4ff9bae) [ryan: rebased onto the linear SEP history — verify_is_live() takes the capture epoch since b0890a92a756 bound capture results to their queued operation; the new check passes it, as aurora-wip does after composing this branch in 42ae23434d4c] Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 113 ++++++++++++++++++++++++++++++++++---- 1 file changed, 103 insertions(+), 10 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index ac4fba5be22af..9afb45518aa11 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -36,6 +36,37 @@ impl SepData { } } + // A match step whose refusal ends the verify. libfprint reports every + // failed verify as the same unspecified error, so the enclave's exact + // answer has to be recorded here or nowhere. Success is decided as in + // `sbio_call`. + fn sbio_verify_step(&self, op: &crate::sbio::SbioOp) -> Option> { + let done = match self.sbio_transfer(op) { + Ok(done) => done, + Err(e) => { + dev_err!( + self.dev, + "verify: {} transfer failed ({:?})\n", + op.name(), + e + ); + return None; + } + }; + match done.status.answered() { + Some(err) if err as u16 == crate::sbio::SBIO_STATUS_OK => Some(done.payload), + _ => { + dev_err!( + self.dev, + "verify: {} answered status {}\n", + op.name(), + done.status + ); + None + } + } + } + pub(crate) fn sbio_call(&self, op: &crate::sbio::SbioOp) -> SbioOutcome { let done = match self.sbio_transfer(op) { Ok(done) => done, @@ -1254,6 +1285,7 @@ impl SepData { } let Some(token_bytes) = self.mint_token_bytes() else { + dev_err!(self.dev, "verify: could not draw a result token\n"); self.finish_verify( bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE), [0u8; bio::TOKEN_LEN], @@ -1265,6 +1297,7 @@ impl SepData { // The enclave refuses a capture from an uncalibrated sensor (0x65 answers 1). if !self.bring_sensor_online() { + dev_err!(self.dev, "verify: the sensor did not come back online\n"); self.finish_verify(bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), token_bytes); let _ = sensor::idle(); return; @@ -1300,40 +1333,81 @@ impl SepData { self.finish_verify(outcome, token_bytes); } + // Every exit that fails a live verify logs why: userspace sees only a bare + // "unspecified error", which cannot tell a sensor fault from a refused + // match step. fn verify_one_image(&self) -> bio::VerifyOutcome { let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + dev_err!(self.dev, "verify: invalid probe user id\n"); return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); }; - if sensor::start_capture().is_err() { + if let Err(e) = sensor::start_capture() { + dev_err!(self.dev, "verify: could not start the capture ({:?})\n", e); return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR); } let advertised = match self.await_capture() { CaptureWait::Ready(n) => n, CaptureWait::Timeout => { + dev_warn!(self.dev, "verify: the capture ended without a frame\n"); return bio::VerifyOutcome::Failed(ENROL_STATUS_TIMEOUT); } - CaptureWait::Fault(_state) => { + CaptureWait::Fault(state) => { + dev_err!( + self.dev, + "verify: the sensor fell to state {} during the capture\n", + state + ); + return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR); + } + CaptureWait::Abandon => { + // A cancelled verify also abandons; only a live one is a fault. + if bio::verify_is_live( + &self.bio_session.lock(), self.capture_epoch.load(Relaxed), + ) { + dev_err!( + self.dev, + "verify: the sensor status became unreadable during the capture\n" + ); + } return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR); } - CaptureWait::Abandon => return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), }; let capture = match sensor::read_capture(advertised) { Ok(capture) => capture, - Err(sensor::CaptureError::Checksum { .. }) | Err(sensor::CaptureError::Length(_)) => { + Err(sensor::CaptureError::Checksum { + advertised: sent, + computed, + }) => { + dev_warn!( + self.dev, + "verify: frame checksum mismatch (sensor sent {:#06x}, computed {:#06x})\n", + sent, + computed + ); + return bio::VerifyOutcome::Failed(ENROL_STATUS_RETRY); + } + Err(sensor::CaptureError::Length(n)) => { + dev_warn!( + self.dev, + "verify: the sensor advertised an unusable frame length {}\n", + n + ); return bio::VerifyOutcome::Failed(ENROL_STATUS_RETRY); } - Err(sensor::CaptureError::Bus(_)) => { + Err(sensor::CaptureError::Bus(e)) => { + dev_err!(self.dev, "verify: reading the frame failed ({:?})\n", e); return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR); } Err(sensor::CaptureError::NoMemory) => { - return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR) + dev_err!(self.dev, "verify: no memory for the frame\n"); + return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR); } }; if self - .sbio_expect_ok(&crate::sbio::sbio_prepare_image_processing()) + .sbio_verify_step(&crate::sbio::sbio_prepare_image_processing()) .is_none() { return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); @@ -1349,7 +1423,7 @@ impl SepData { user, crate::shim::monotonic_ns(), ); - if self.sbio_expect_ok(&init).is_none() { + if self.sbio_verify_step(&init).is_none() { return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); } @@ -1363,21 +1437,35 @@ impl SepData { } drop(capture); - let Some(assessment) = self.sbio_expect_ok(&crate::sbio::sbio_image_assessment()) else { + let Some(assessment) = self.sbio_verify_step(&crate::sbio::sbio_image_assessment()) else { return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); }; if assessment.len() <= crate::sbio::ASSESS_USABLE_MATCH { + dev_err!( + self.dev, + "verify: image assessment is {} bytes, too short to read\n", + assessment.len() + ); return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); } if assessment[crate::sbio::ASSESS_USABLE_MATCH] == 0 { + dev_warn!( + self.dev, + "verify: the image is not usable for matching (partial contact)\n" + ); return bio::VerifyOutcome::Failed(ENROL_STATUS_RETRY); } - let Some(result) = self.sbio_expect_ok(&crate::sbio::sbio_match_result()) else { + let Some(result) = self.sbio_verify_step(&crate::sbio::sbio_match_result()) else { return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); }; let Some(parsed) = crate::sbio::MatchResult::parse(&result) else { + dev_err!( + self.dev, + "verify: the {}-byte match result did not parse\n", + result.len() + ); return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); }; @@ -1390,6 +1478,11 @@ impl SepData { let known = self.bio_index.lock().contains_uuid(&identity); if !known { + dev_err!( + self.dev, + "verify: the enclave matched identity {}, which the host index does not hold\n", + Hex(&identity) + ); return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); } From b1f300a64d38310ddad3726bd61e42a030f822f0 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 29 Sep 2026 18:08:08 -0700 Subject: [PATCH 0749/2680] soc: apple: sep: set the match credential up again after a cold restore On J414c (M2 Max, T6020 profile) an enrolment restores cleanly after a reboot: every Catacomb loads, the identity count is 1 and the device-view proof passes. Every match on that boot still failed, though. The image assessment reported a usable image, then MATCH_RESULT (0x09) answered 0x1, prerequisite missing. The SCRD cold-match credential is set up in ensure_restored_after(), before the Catacomb restore, and that step reports success. Setting it up a second time just before a match made the match succeed. Once was enough for the whole boot. Later verifies matched without setting it up again, even though each one re-registers the sensor with CLEAR_STATE. The first credential therefore stops being usable somewhere in the rest of activation, after it is created. Set the credential up once more when activation ends, but only when a restored template has passed the device-view proof, and only on the variant-5 key store where this was seen. The first setup stays where it is. The J313 (Sepos13) cold-restore path is unchanged. The driver cannot release an SCRD context, so this refreshes once per boot rather than once per verify. A control boot on the unpatched module still failed, even though the restored state had since been re-saved by matched sessions, so the refresh is what fixes the match, not the re-save. With this patch, a normal boot on J414c matched three verifies out of three against the enrolment made before the reboot. Reported-by: Scott Jones Suggested-by: Scott Jones Tested-by: Scott Jones Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit 4409efe7c199af7b59e4270c1a3593e544e08470) --- drivers/soc/apple/sbio.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 9afb45518aa11..b68ae1b6f7505 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1254,10 +1254,33 @@ impl SepData { self.ensure_restored_after(prepared); self.attach_bringup(); self.probe_owner_export(); + self.refresh_match_credential(); self.touchid_started.store(true, Relaxed); Ok(()) } + // On the variant-5 key store the SCRD credential set up before the + // Catacomb restore does not survive the rest of activation: J414c then + // answered every cold-boot MATCH_RESULT with 0x1 on a usable image. + // Setting it up once more here was enough for every match that boot, + // including verifies that re-register the sensor with CLEAR_STATE. + fn refresh_match_credential(&self) { + if self.profile.key_store != profile::KeyStore::Variant5 + || !self.templates_restored.load(Relaxed) + { + return; + } + let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + return; + }; + if !self.establish_scrd_match_context(user) { + dev_warn!( + self.dev, + "Touch ID: the match credential did not re-establish after the restore; the enclave may refuse matches this boot\n" + ); + } + } + fn prepare_bio_open(&self) -> Result<()> { if let Err(e) = self.activate_touchid() { self.touchid_failed.store(true, Relaxed); From d27fac9fcd07e9c5c1bc63910c781031ec4f9a93 Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Tue, 29 Sep 2026 21:47:00 -0600 Subject: [PATCH 0750/2680] soc: apple: sep: turn on persistent enrolment for T6000 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit T6000 kept persistent_enrol off because it was not reboot-tested. Without it, J314s ends enrolment with the owner Catacomb in state 0x3 and never writes /var/lib/apple-sep-catacomb-owner.bin, so after a reboot the user Catacomb fails to load (cold-transition 0x8002) and the enrolment is lost. With j414s_persistent_enrol=1, the owner and user Catacombs reach state 7, the owner file is written, and all three Catacombs restore after a reboot. With the previous commit, verify matches after two separate reboots. Turn it on in the profile so M1 Pro users don't need the module parameter. This was tested on J314s only. J316s shares the profile and is the same SoC with the same sensor, but has not been reboot-tested. [ryan: rebased onto the linear SEP history — the T6000 profile here carries the J700-era nonce_session field rather than refresh_match_credential, which commit f7dbcf878b7e ("soc: apple: sep: refresh the match credential after a cold restore on T6000") added and commit 05ae05e63489 ("soc: apple: sep: set the match credential up again on every profile") removed again on this branch; aurora-wip never took f7dbcf878b7e. Only persistent_enrol changes, as in the original] Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Christian Bartels Signed-off-by: Ryan Murray --- drivers/soc/apple/profile.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index c97bb4a0e8490..9da9e53cdbad6 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -188,8 +188,7 @@ const T8103: PlatformProfile = PlatformProfile { /// path, shared-memory geometry and key store follow T8103. The sensor sits on /// SPI2 at the T6020 address, in the mode T6020 verified: the J313, J316s and /// J414s platform device trees describe the same sensor (id 0x3352) -/// identically. Enrolment persistence is not yet reboot-tested here, so it -/// stays opt-in (`j414s_persistent_enrol=1`). +/// identically. Enrolment persistence is reboot-tested on J314s. const T6000: PlatformProfile = PlatformProfile { name: "T6000/J316s", shmem_capacity: 0x3_0000, @@ -218,7 +217,7 @@ const T6000: PlatformProfile = PlatformProfile { key_store: KeyStore::Sepos13 { cpx_encryption_mode: 2, }, - persistent_enrol: false, + persistent_enrol: true, nonce_session: false, }; From b534ae5fcec9f91f9850660df7d3d4fef8b28376 Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Wed, 30 Sep 2026 12:55:59 -0600 Subject: [PATCH 0751/2680] soc: apple: sep: log why the sensor bring-up gave up wake_sensor() and apply_sensor_patch() return None on about ten failure paths without a word. When the attach-time bring-up stopped after "registering sensor at bring-up stage 0", the log did not say which step failed. Warn on each of those paths. They run once per bring-up, not per frame. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Christian Bartels Co-authored-by: Claude Opus 5.5 --- drivers/soc/apple/sbio.rs | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index b68ae1b6f7505..37990ae2c35c1 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1781,6 +1781,7 @@ impl SepData { let st = match sensor::status() { Ok(st) => st, Err(_) => { + dev_warn!(self.dev, "sensor: status read failed on wake\n"); return None; } }; @@ -1804,6 +1805,7 @@ impl SepData { continue; }; if !self.register_sensor(&id) { + dev_warn!(self.dev, "sensor: the enclave did not register the sensor\n"); return None; } @@ -2094,36 +2096,50 @@ impl SepData { SbioOutcome::Ok(b) => b, SbioOutcome::PrerequisiteMissing => { if !self.init_sequence_counter() { + dev_warn!( + self.dev, + "sensor patch: sequence counter init failed before the patch fetch\n" + ); return None; } match self.sbio_call(&crate::sbio::sbio_fetch_patch()) { SbioOutcome::Ok(b) => b, _ => { + dev_warn!( + self.dev, + "sensor patch: patch fetch failed after the sequence counter init\n" + ); return None; } } } SbioOutcome::Status16 => { + dev_warn!(self.dev, "sensor patch: patch fetch answered status 16\n"); return None; } SbioOutcome::Other => { + dev_warn!(self.dev, "sensor patch: patch fetch failed\n"); return None; } }; if blob.is_empty() { + dev_warn!(self.dev, "sensor patch: the enclave returned an empty patch\n"); return None; } // enable command is mandatory; skipping it leaves the sensor in state 9 if sensor::setup_patch_enable().is_err() { + dev_warn!(self.dev, "sensor patch: the patch enable command failed on the bus\n"); return None; } if !self.await_sensor_state(sensor::STATE_IDLE, c"idle, before sending the patch") { + dev_warn!(self.dev, "sensor patch: the sensor did not reach idle before the patch\n"); return None; } if sensor::send_patch(&blob).is_err() { + dev_warn!(self.dev, "sensor patch: sending the patch failed on the bus\n"); return None; } @@ -2133,12 +2149,20 @@ impl SepData { let st = match sensor::status() { Ok(st) => st, Err(_) => { + dev_warn!( + self.dev, + "sensor patch: status read failed while waiting for the acknowledgment\n" + ); return None; } }; if st.patch_ack() == sensor::PATCH_ACCEPTED { if let Ok(after) = sensor::status() { if after.state == sensor::STATE_NEEDS_PATCH { + dev_warn!( + self.dev, + "sensor patch: acknowledged, but the sensor still asks for a patch\n" + ); return None; } } @@ -2147,6 +2171,11 @@ impl SepData { } let _ = sensor::status(); + dev_warn!( + self.dev, + "sensor patch: no acknowledgment after {} polls\n", + PATCH_POLL_ATTEMPTS + ); None } From 8b861bd5e723aeafb425637821cd3df5f88f1d2e Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Wed, 30 Sep 2026 12:56:00 -0600 Subject: [PATCH 0752/2680] soc: apple: sep: retry the sensor bring-up on verify after a failed attach If the attach-time bring-up fails after the Catacomb restore, e.g. the sensor never acknowledges its patch, attach_bringup() returns early. templates_restored stays false, touchid_started stays true, and every later verify is refused with "refusing before touching the sensor" until the next reboot. The restore is still a valid candidate at that point. It is only the device-view proof that is missing. So when run_verify() finds a cold restore candidate without a synced device view, bring the sensor online once more, let complete_bringup() finish the proof, and set the match credential up again before the templates_restored check. Seen on J314s (T6000) in 2 of 4 cold boots with an LG UltraFine 5K on Thunderbolt and the PCIe-C tunnel already up when the SEP brings the sensor up. It has not been seen in 7 boots without that. The trigger is not known: a good bring-up finishes about 0.1 s after stage 0, and the tunnel's link-up and host bridge probe fall at the same offsets in good and failing boots. With the previous commit, the failing boot logged: 18.37 sbio: registering sensor at bring-up stage 0 23.75 sensor patch: no acknowledgment after 250 polls 44.14 verify: attach-time bring-up left the device-view proof undone; retrying the sensor bring-up 44.45 Touch ID: restored 1 enrolled identity/identities; enrolment survived reboot The verify that followed matched. On good boots, and after an s2idle resume, the retry does nothing and verify matches as before. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Christian Bartels Co-authored-by: Claude Opus 5.5 --- drivers/soc/apple/sbio.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 37990ae2c35c1..cb8cf684552c2 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1085,6 +1085,10 @@ impl SepData { return; } let Some(patch) = self.wake_sensor() else { + dev_warn!( + self.dev, + "sbio: the sensor did not wake at attach; the next verify retries the bring-up\n" + ); return; }; if !self.complete_bringup(patch) { @@ -1294,7 +1298,31 @@ impl SepData { Ok(()) } + // The attach-time bring-up can fail after the Catacomb restore (the + // sensor patch step has no retry), which leaves the device-view proof + // undone and every verify refused until reboot. The restore itself is + // still a valid candidate, so bring the sensor up once more and let + // complete_bringup() finish the proof before refusing. + fn retry_restore_proof(&self) { + if self.templates_restored.load(Relaxed) + || !self.cold_restore_candidate.load(Relaxed) + || self.device_view_synced.load(Relaxed) + { + return; + } + dev_warn!( + self.dev, + "verify: attach-time bring-up left the device-view proof undone; retrying the sensor bring-up\n" + ); + if !self.bring_sensor_online() { + dev_err!(self.dev, "verify: the retried sensor bring-up failed\n"); + } + let _ = sensor::idle(); + self.refresh_match_credential(); + } + pub(crate) fn run_verify(&self) { + self.retry_restore_proof(); if !self.templates_restored.load(Relaxed) { dev_err!( self.dev, From dc2fe6de11f688b3bb82cd2f0e5e8e43c11a9888 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 30 Sep 2026 13:03:22 -0700 Subject: [PATCH 0753/2680] soc: apple: sep: set the match credential up again on every profile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The match-credential refresh after a cold restore is gated by a per-profile flag, on for T6000, T6020 and T8140 and off for T8103. The failure it fixes is not specific to a SoC or a key store. The M2 Max (J414c, variant-5 key store) and the M1 Pro (J314s, 13.5 key store) both answered every MATCH_RESULT after a reboot with 0x1 until the credential was set up again. A J313 on a 26.3 sepOS shows the same symptom with T8103 left off. Every profile sets the credential up in the same place, in ensure_restored_after() before the Catacomb restore. Drop the flag and refresh on every profile. It still runs once per boot, and only when a restored template has passed the device-view proof. This also takes out a per-device switch, in line with removing per-SoC bring-up knobs. Not yet confirmed on a T8103 or a T8140 with a reboot and a match. Reported on a J313 by GitHub user nosferatuboi. Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i [ryan: rebased onto the linear SEP history — this commit was developed in parallel with commits applied above; it adds exactly what it added originally, but some lines it replaces had changed in between, and those are replaced as aurora-wip did (42ae23434d4c)] Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index cb8cf684552c2..f645cca05f1cc 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1263,15 +1263,14 @@ impl SepData { Ok(()) } - // On the variant-5 key store the SCRD credential set up before the - // Catacomb restore does not survive the rest of activation: J414c then - // answered every cold-boot MATCH_RESULT with 0x1 on a usable image. - // Setting it up once more here was enough for every match that boot, - // including verifies that re-register the sensor with CLEAR_STATE. + // The SCRD credential set up before the Catacomb restore does not survive + // the rest of activation: J414c (variant-5 key store) and J314s (13.5 key + // store) answered every cold-boot MATCH_RESULT with 0x1 on a usable image, + // and a J313 showed the same symptom. Setting it up once more here was + // enough for every match that boot, including verifies that re-register + // the sensor with CLEAR_STATE. It runs on every profile. fn refresh_match_credential(&self) { - if self.profile.key_store != profile::KeyStore::Variant5 - || !self.templates_restored.load(Relaxed) - { + if !self.templates_restored.load(Relaxed) { return; } let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { From afc0b52dd64da1398eada299e990dd78d271d3bd Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 30 Sep 2026 13:03:46 -0700 Subject: [PATCH 0754/2680] soc: apple: sep: keep enrolments across a reboot on T8140 as well T8140 (MacBook Neo) is the one profile left with persistence off. There, the fresh owner Catacomb, which finishes in state 0x3 with no save-pending bit, is never exported, and an enrolment is lost at the next boot. T8103, T6000 and T6020 needed the same owner export and user-before-master saving before their enrolments survived a reboot. Turn persistence on for T8140, so every profile persists enrolments, and correct the comment on the owner export, which still named only J313 and J414s. Not tested on a MacBook Neo. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- drivers/soc/apple/sbio.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index f645cca05f1cc..08a693ceb8f88 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -399,9 +399,10 @@ impl SepData { } } - // J313 and J414s need an owner export even though state 0x3 has no - // save-pending bit. Both restored a fresh enrollment after this export - // and user-before-master completion. Other profiles remain opt-in. + // The owner Catacomb must be exported even though state 0x3 has no + // save-pending bit. J313, J414s and J314s each restored a fresh + // enrollment after this export and user-before-master completion. + // Every profile now does this. if self.profile.persistent_enrol || *module_parameters::j414s_persistent_enrol.value() != 0 { if owner_initial_state == Some(0x3) && !self.save_catacomb( From 715ede69b92fe09b5782fe29542694df8d9266cc Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 28 Sep 2026 13:33:55 +1300 Subject: [PATCH 0755/2680] soc: apple: sep: log every silent exit in identity-keybag provisioning On a j293 with a macOS 15 sepOS the write boot printed only 'bringup: identity-keybag provisioning failed' with nothing before it. The silent returns were: sks_exchange() failing before the wait, sks_report_response() seeing an empty or unparsable reply (which hides the mailbox status of a refusal), read_blob() finding no fv_data, sks_read_uuid() failing, and the COPY_KEYBAG pair. Each now names itself and reports the mailbox status and response size where it has them. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_013wmzHjSiZbAZXBWuEBEwzx --- drivers/soc/apple/sks.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index 5266541882a17..b617d931b1e3f 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -654,9 +654,20 @@ impl SepData { }; intent.sending(); let Some(out) = self.sks_exchange(request.name, request.msg, &request.img) else { + dev_err!( + self.dev, + "sks: CREATE_KEYBAG exchange did not complete (buffers unregistered, wedged, or OOL write failed)\n" + ); return false; }; let Some(body) = self.sks_report_response(crate::sks::SKS_CREATE_NAME, &out) else { + dev_err!( + self.dev, + "sks: CREATE_KEYBAG reply carried no parsable body: mailbox status {}, response {} bytes, response_size {}\n", + out.reply.status, + out.response.len(), + out.reply.response_size + ); return false; }; // 13.5 replies with the struct version and the handle alone @@ -680,6 +691,11 @@ impl SepData { profile::KeyStore::Sepos13 { .. } => 8, profile::KeyStore::Variant5 => { let Some((_fv_data, end)) = image::read_blob(body, 8) else { + dev_err!( + self.dev, + "sks: CREATE_KEYBAG reply has no fv_data blob at offset 8 (body {} bytes)\n", + body.len() + ); return false; }; end @@ -703,15 +719,23 @@ impl SepData { profile::KeyStore::Sepos13 { .. } => None, profile::KeyStore::Variant5 => { let Some(bag_uuid) = self.sks_read_uuid(handle) else { + dev_err!(self.dev, "sks: could not read the new keybag's UUID back\n"); return false; }; Some(bag_uuid) } }; let Some(out) = self.sks_send(self.sks_req_copy_keybag(handle)) else { + dev_err!(self.dev, "sks: COPY_KEYBAG of the new identity keybag got no reply\n"); return false; }; let Some(wrapped) = self.wrapped_from_copy_reply(&out, c"new identity keybag") else { + dev_err!( + self.dev, + "sks: COPY_KEYBAG reply unusable: mailbox status {}, response {} bytes\n", + out.reply.status, + out.response.len() + ); return false; }; let committed = match bag_uuid { From 96f5c1d7bb6c255630897401276f52c050b9d33c Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 30 Sep 2026 12:32:51 -0700 Subject: [PATCH 0756/2680] soc: apple: sep: let a refused CREATE_KEYBAG be retried Before sending CREATE_KEYBAG, provisioning persists an intent record so that an interrupted create is never repeated blindly. When the enclave answers the request with a non-zero status, it has created nothing, but the intent stays. Every later provisioning attempt, and every trusted-key seal, then fails with EEXIST until the record is removed by hand. This happens on both refusal paths: a reply with a body and a failing status, and a reply whose body is empty. A j293 whose enclave runs a newer sepOS answered status -13 with an empty body. Mark the slot refused, and so retryable, whenever the exchange completed and the enclave's status is non-zero. A timeout, or a reply the driver cannot attribute to the enclave, keeps the intent and stays fail-closed. Suggested-by: Justin Pfister Reported-by: Tim Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- drivers/soc/apple/sks.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index b617d931b1e3f..2d29fa7598577 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -609,6 +609,23 @@ impl SepData { }) } + // A completed CREATE_KEYBAG that the enclave answered with a non-zero + // status created nothing, so the persisted intent must not block the next + // attempt: without this, every later seal fails with EEXIST. Any other + // failure (timeout, no answer) keeps the intent and stays fail-closed. + fn sks_mark_create_refused(&self, slot: keybag::Slot, status: i8) { + if status == 0 { + return; + } + if let Err(e) = keybag::mark_refused(slot) { + dev_err!( + self.dev, + "sks: cannot mark refused keybag create retryable: {:?}\n", + e + ); + } + } + pub(crate) fn sks_provision_identity_keybag(&self) -> bool { let proof = match keybag::read(keybag::Slot::Identity) { Ok(keybag::State::Present(_)) => return true, @@ -668,6 +685,7 @@ impl SepData { out.response.len(), out.reply.response_size ); + self.sks_mark_create_refused(slot, out.reply.status); return false; }; // 13.5 replies with the struct version and the handle alone @@ -683,6 +701,7 @@ impl SepData { out.reply.status, body.len() ); + self.sks_mark_create_refused(slot, out.reply.status); return false; } let variant = u32::from_le_bytes(body[0..4].try_into().unwrap()); From 88bfd7e0175f861277334d989c26cb0bfa9b0cbd Mon Sep 17 00:00:00 2001 From: Tim Date: Wed, 30 Sep 2026 12:33:05 -0700 Subject: [PATCH 0757/2680] soc: apple: sep: log an enrolment the enclave completes without a template When the enclave flags an enrolment as complete but reports no template, run_enrolment() ends it with ENROL_STATUS_ENCLAVE and logs nothing. That leaves no trace of why an enrolment that reached 100% failed. Log the capture count, the stage and the percentage. [Chris Kearney: this is the only hunk carried over from the larger "report the enclave's raw status on every refused SBIO call"; the rest duplicates logging feat/sep already has, or logs expected refusals as errors] Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- drivers/soc/apple/sbio.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 08a693ceb8f88..4dae008ebbbc3 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1590,6 +1590,13 @@ impl SepData { // completion is the flag at offset 0xbfe, not a derived stage count if complete { if !has_template { + dev_err!( + self.dev, + "enrol: enclave flags completion after {} capture(s) (stage {}, {}%) but reports no template\n", + counter, + stage, + percent + ); break Some(Err(ENROL_STATUS_ENCLAVE)); } enrolment_completed = true; From 68f176d6e24c17ec79480f4d1e0cc635cc075374 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:35:19 -0700 Subject: [PATCH 0758/2680] soc: apple: sep: end a Touch ID capture before the system suspends MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enrol and verify run on the system workqueue, which is not freezable. A capture that is still running when the system suspends keeps polling the sensor over SPI while devices go down. An M1 Air (J313) whose lid closed with the lock screen's fingerprint capture in flight did not wake. With the driver loaded but fprintd masked, it slept and woke normally. fprintd cancels its own verify when logind announces the sleep, so the kernel usually never sees a capture at suspend time. The window is a capture that starts after that, such as a lock screen starting a scan as the lid closes. The driver cannot rely on userspace to close it. Register a PM notifier. On PM_SUSPEND_PREPARE: - end any live enrol or verify the way CANCEL does, so userspace sees a failure it can retry after resume; - hold new capture starts until the system is back. Userspace thaws before PM_POST_SUSPEND, so a start that failed with EBUSY instead would be refused for a moment after wake. The wait is interruptible: a start made just before sleep is interrupted by the freezer and restarted after resume; - wait, for at most 6 s, for the enrol or verify work to stop touching the sensor. The capture loops already check the session on every pass and power the sensor down on the way out. A work item that starts after the prepare sees the flag under the session lock and does not touch the sensor. PM_POST_SUSPEND clears the flag and releases the held starts. Tested on a J313: it suspended with a verify waiting for a finger and woke on a key press, and the first verify after resume matched. Reported on a J313 by GitHub user nosferatuboi. Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i [ryan: rebased onto the linear SEP history — the enrol and verify workers had meanwhile gained the capture epoch and worker_stopped() bookkeeping (b0890a92a756); the capture_begin()/capture_end() bracket is placed inside them, and remove() unregisters the sleep notifier after waking the waiters, as aurora-wip composed it in 42ae23434d4c] Signed-off-by: Ryan Murray --- drivers/soc/apple/Makefile | 3 +- drivers/soc/apple/bio.rs | 23 ++++++++++ drivers/soc/apple/pm.rs | 92 +++++++++++++++++++++++++++++++++++++ drivers/soc/apple/pm_shim.c | 59 ++++++++++++++++++++++++ drivers/soc/apple/sbio.rs | 80 ++++++++++++++++++++++++++++++++ drivers/soc/apple/sep.rs | 33 ++++++++++++- drivers/soc/apple/shim.h | 5 ++ 7 files changed, 292 insertions(+), 3 deletions(-) create mode 100644 drivers/soc/apple/pm.rs create mode 100644 drivers/soc/apple/pm_shim.c diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 2920afa1502d2..e6cb2e917bc0a 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -26,7 +26,8 @@ obj-$(CONFIG_APPLE_SEP_FV_INTERFACE) += fv_shim.o obj-$(CONFIG_APPLE_SEP) += apple-sep.o apple-sep-y := sep.o hwrng_shim.o store_shim.o bio_shim.o sha_shim.o \ - crypto_shim.o sensor_shim.o p256_shim.o trusted_shim.o work_shim.o + crypto_shim.o sensor_shim.o p256_shim.o trusted_shim.o work_shim.o \ + pm_shim.o obj-$(CONFIG_APPLE_PMP) += pmp.o diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index 464022c681f1f..13d7e99d43fc9 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -927,6 +927,29 @@ fn cancel(ctx: &mut Context<'_>) -> Result { }) } +/// Ends a live enrolment or verify the way a CANCEL does, for the system going +/// to sleep: userspace sees a failure it can retry after resume. Returns +/// whether a waiter needs waking. +pub(crate) fn abort_capture(session: &mut Session) -> bool { + match &mut session.op { + Op::Enrol { terminal, .. } if terminal.is_none() => { + *terminal = Some(EnrolOutcome::Failed(STATUS_LOCAL)) + } + Op::Verify { terminal, .. } if terminal.is_none() => { + *terminal = Some(VerifyOutcome::Failed(STATUS_LOCAL)) + } + _ => return false, + } + session.token = None; + session.unseen = true; + true +} + +/// The ioctls that start a sensor capture. +pub(crate) fn starts_capture(cmd: u32) -> bool { + cmd == IOC_ENROL_START || cmd == IOC_VERIFY_START +} + fn delete(ctx: &mut Context<'_>, user: UserPtr) -> Result { require_admin()?; let request: Delete = UserSlice::new(user, core::mem::size_of::()) diff --git a/drivers/soc/apple/pm.rs b/drivers/soc/apple/pm.rs new file mode 100644 index 0000000000000..c613a516a067e --- /dev/null +++ b/drivers/soc/apple/pm.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! System sleep. A Touch ID capture runs on a non-freezable workqueue and keeps +//! talking to the sensor over SPI, so one left running into a suspend is still +//! active while devices go down. A J313 that suspended with the lock screen's +//! capture in flight did not wake. Before the system suspends, end any capture +//! in progress and hold new ones until it has resumed. The notifier itself +//! is in `pm_shim.c`; the capture handling is on [`SepData`]. + +use core::sync::atomic::{AtomicPtr, Ordering}; + +use kernel::prelude::*; +use kernel::sync::Arc; +use kernel::types::ForeignOwnable; + +use crate::SepData; + +/// The notifier has no context pointer, so `SepData` is reached through this +/// global, as in `trusted.rs`. Holds the [`ForeignOwnable::into_foreign`] +/// pointer while registered; `null` otherwise. +static SEP: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); + +extern "C" { + fn sep_pm_register(event: unsafe extern "C" fn(bool)) -> c_int; + fn sep_pm_unregister(); +} + +/// # Safety +/// Called by `pm_shim.c` from the PM notifier chain, in process context. +unsafe extern "C" fn pm_event(entering: bool) { + let ptr = SEP.load(Ordering::Acquire); + if ptr.is_null() { + return; + } + // SAFETY: produced by `into_foreign` in `register`, and reclaimed by + // `unregister` only after `sep_pm_unregister` has waited out any running + // notifier call, so it is live for this borrow. + let sep = unsafe { as ForeignOwnable>::borrow(ptr) }; + if entering { + sep.sleep_prepare(); + } else { + sep.sleep_finished(); + } +} + +pub(crate) fn register(sep: Arc) -> Result<()> { + let ptr = sep.into_foreign(); + + if SEP + .compare_exchange( + core::ptr::null_mut(), + ptr, + Ordering::AcqRel, + Ordering::Acquire, + ) + .is_err() + { + // SAFETY: `ptr` was just produced by `into_foreign` and never published. + drop(unsafe { as ForeignOwnable>::from_foreign(ptr) }); + return Err(EBUSY); + } + + // SAFETY: the shim stores the callback and registers a notifier block + // that lives for the module's lifetime. + let rc = unsafe { sep_pm_register(pm_event) }; + if rc != 0 { + let raw = SEP.swap(core::ptr::null_mut(), Ordering::AcqRel); + if !raw.is_null() { + // SAFETY: `raw` is the pointer we published; no notifier is wired. + drop(unsafe { as ForeignOwnable>::from_foreign(raw) }); + } + return Err(Error::from_errno(rc)); + } + Ok(()) +} + +/// Safe to call even if [`register`] never ran. Unregisters the notifier, +/// which waits for a running call to finish, before reclaiming the `Arc`. +pub(crate) fn unregister() { + if SEP.load(Ordering::Acquire).is_null() { + return; + } + // SAFETY: no preconditions; matched with `sep_pm_register` above. + unsafe { sep_pm_unregister() }; + + let ptr = SEP.swap(core::ptr::null_mut(), Ordering::AcqRel); + if !ptr.is_null() { + // SAFETY: `ptr` came from `into_foreign` in `register`, and the + // notifier can no longer call `pm_event`. + drop(unsafe { as ForeignOwnable>::from_foreign(ptr) }); + } +} diff --git a/drivers/soc/apple/pm_shim.c b/drivers/soc/apple/pm_shim.c new file mode 100644 index 0000000000000..e413bf2594e6a --- /dev/null +++ b/drivers/soc/apple/pm_shim.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * System-sleep notifier for the SEP driver. It only forwards "about to sleep" + * and "back from sleep" to Rust, which ends any Touch ID capture before the + * system suspends; see pm.rs. + */ +#include +#include +#include + +#include "shim.h" + +static void (*sep_pm_event)(bool entering); + +static int sep_pm_notify(struct notifier_block *nb, unsigned long action, + void *data) +{ + void (*event)(bool entering) = READ_ONCE(sep_pm_event); + + if (!event) + return NOTIFY_DONE; + + switch (action) { + case PM_SUSPEND_PREPARE: + case PM_HIBERNATION_PREPARE: + case PM_RESTORE_PREPARE: + event(true); + return NOTIFY_OK; + case PM_POST_SUSPEND: + case PM_POST_HIBERNATION: + case PM_POST_RESTORE: + event(false); + return NOTIFY_OK; + default: + return NOTIFY_DONE; + } +} + +static struct notifier_block sep_pm_nb = { + .notifier_call = sep_pm_notify, +}; + +int sep_pm_register(void (*event)(bool entering)) +{ + int ret; + + WRITE_ONCE(sep_pm_event, event); + ret = register_pm_notifier(&sep_pm_nb); + if (ret) + WRITE_ONCE(sep_pm_event, NULL); + return ret; +} + +/* Returns only once no notifier call is still running. */ +void sep_pm_unregister(void) +{ + unregister_pm_notifier(&sep_pm_nb); + WRITE_ONCE(sep_pm_event, NULL); +} diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 4dae008ebbbc3..92ff723142b88 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2710,6 +2710,81 @@ impl SepData { } } + /// Called by the enrol and verify work items before they touch the sensor. + /// Returns false if the system is going to sleep; the session has then + /// been ended and the capture must not start. A true return must be + /// paired with [`Self::capture_end`]. + pub(crate) fn capture_begin(&self) -> bool { + let mut session = self.bio_session.lock(); + if self.suspending.load(Relaxed) { + let woke = bio::abort_capture(&mut session); + drop(session); + if woke { + self.bio_wake(); + } + return false; + } + self.captures_running.fetch_add(1, Relaxed); + true + } + + pub(crate) fn capture_end(&self) { + self.captures_running.fetch_sub(1, Relaxed); + } + + /// The system is about to suspend. End any capture in progress, hold new + /// ones, and wait for the enrol or verify work to stop touching the sensor + /// before devices go down. + pub(crate) fn sleep_prepare(&self) { + let woke = { + let mut session = self.bio_session.lock(); + self.suspending.store(true, Relaxed); + bio::abort_capture(&mut session) + }; + if woke { + dev_info!( + self.dev, + "Touch ID: ending the capture in progress before sleep\n" + ); + self.bio_wake(); + } + + let mut waited: u32 = 0; + while self.captures_running.load(Relaxed) != 0 { + if waited >= SLEEP_DRAIN_MS { + dev_warn!( + self.dev, + "Touch ID: a capture was still running {} ms after the system started to sleep; continuing\n", + SLEEP_DRAIN_MS + ); + return; + } + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(10)); + waited += 10; + } + } + + /// The system has resumed, or the suspend was aborted. + pub(crate) fn sleep_finished(&self) { + let _session = self.bio_session.lock(); + self.suspending.store(false, Relaxed); + self.sleep_over.notify_all(); + } + + /// Holds a capture start until the system is awake. Userspace is thawed + /// before the driver hears of the resume, so a lock screen that retries at + /// once would otherwise be refused. A start made just before the system + /// sleeps is interrupted by the freezer and restarted after resume. + fn wait_until_awake(&self) -> Result<()> { + let mut session = self.bio_session.lock(); + while self.suspending.load(Relaxed) { + if self.sleep_over.wait_interruptible(&mut session) { + return Err(ERESTARTSYS); + } + } + Ok(()) + } + pub(crate) fn queue_enrolment(this: Arc) { let Some(epoch) = bio::queued_epoch(&this.bio_session.lock()) else { return; @@ -2770,6 +2845,11 @@ impl SepData { if cmd == bio::IOC_ATTEST { return self.bio_attest(arg); } + // A capture started while the system goes to sleep would run into the + // suspend; `capture_begin` repeats this check under the session lock. + if bio::starts_capture(cmd) { + self.wait_until_awake()?; + } // Query SEP outside the bio session/index locks. A host index can // outlive its SEP identity after a failed cold restore; userspace // must not mistake that index for proof of a live template. diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 78822a80aa2a9..e4fa4523f1c9c 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -19,6 +19,7 @@ mod hwrng; mod image; mod keybag; mod keybag_identity; +mod pm; mod profile; mod proto; mod refkey; @@ -107,6 +108,11 @@ const OOL_SIZE_SCRD: usize = 0x4000; const SBIO_TIMEOUT_MS: time::Msecs = 5000; +// How long going to sleep waits for an ended capture to stop: one enclave +// request in flight, plus the capture loop's interrupt-wait backstop. +const SLEEP_DRAIN_MS: u32 = 6000; +static_assert!(SLEEP_DRAIN_MS as u64 > SBIO_TIMEOUT_MS as u64); + const SKS_ALLOC: usize = 0x8000; const SKS_SECRET_LEN: usize = 32; @@ -675,6 +681,16 @@ struct SepData { shutting_down: Atomic, capture_epoch: Atomic, + // The system is going to sleep: no capture may start. Changed under the + // bio session lock; `sleep_over` is signalled when it clears. + suspending: Atomic, + + #[pin] + sleep_over: CondVar, + + // Enrol and verify work items between `capture_begin` and `capture_end`. + captures_running: Atomic, + #[pin] rx_work: Work, @@ -844,6 +860,9 @@ impl SepData { registered: Atomic::new(false), shutting_down: Atomic::new(false), capture_epoch: Atomic::new(0), + suspending: Atomic::new(false), + sleep_over <- new_condvar!("SepData::sleep_over"), + captures_running: Atomic::new(0), rx_work <- new_work!("SepData::rx_work"), enrol_work <- new_work!("SepData::enrol_work"), verify_work <- new_work!("SepData::verify_work"), @@ -1976,6 +1995,10 @@ impl SepData { self.sbio_wq.notify_all(); self.sks_wq.notify_all(); self.scrd_wq.notify_all(); + pm::unregister(); + // Release any capture start still held for a resume that will not be + // reported now. + self.sleep_finished(); trusted::unregister(); self.unregister_fv_kernel(); @@ -2161,8 +2184,9 @@ impl WorkItem for SepData { fn run(this: Arc) { let epoch = this.capture_epoch.load(Relaxed); - if !this.shutting_down.load(Relaxed) { + if !this.shutting_down.load(Relaxed) && this.capture_begin() { this.run_enrolment(); + this.capture_end(); } bio::worker_stopped(&mut this.bio_session.lock(), epoch); } @@ -2173,8 +2197,9 @@ impl WorkItem for SepData { fn run(this: Arc) { let epoch = this.capture_epoch.load(Relaxed); - if !this.shutting_down.load(Relaxed) { + if !this.shutting_down.load(Relaxed) && this.capture_begin() { this.run_verify(); + this.capture_end(); } bio::worker_stopped(&mut this.bio_session.lock(), epoch); } @@ -2234,6 +2259,10 @@ impl platform::Driver for SepDriver { dev_warn!(data.dev, "trusted-keys: registration failed ({:?})\n", e); } + if let Err(e) = pm::register(data.clone()) { + dev_warn!(data.dev, "sleep notifier registration failed ({:?}); a capture may run into a suspend\n", e); + } + if data.registered.load(Relaxed) { SepData::arm_settle(&data); } diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index e13ca258d92f8..99bc9d68b8be1 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -23,6 +23,11 @@ struct apple_sep_fv_new_file_key; void sep_cancel_work_sync(void *work); void sep_cancel_delayed_work_sync(void *work); +/* -- pm_shim.c ---------------------------------------------------------- */ + +int sep_pm_register(void (*event)(bool entering)); +void sep_pm_unregister(void); + /* -- hwrng_shim.c ------------------------------------------------------- */ void *sep_hwrng_alloc(void); From d17b561ed235f95b72fc7b3bca2a1ab54b3e35a4 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Thu, 1 Oct 2026 07:16:06 -0700 Subject: [PATCH 0759/2680] soc: apple: sep: refuse Touch ID captures while xART writes are off MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With xart_writes=0, a fingerprint match after a reboot that restored identities succeeded, and then the enclave stopped answering on every endpoint, sbio and hwrng included, until the next reboot. The post-match SAVE_LOCKOUT never got a reply. The enclave saves anti-replay state after every match; with writes off, the host refuses that save, and the enclave never completes the transfer that triggered it. With writes on, the same enrolment matched cleanly on 2 of 2 boots, and the xART revision advanced after each match. When the store does not accept writes, refuse ENROL_START and VERIFY_START with EROFS before any capture runs, and log the reason once per boot. Attach, GET_INFO, LIST and the other read-only ioctls keep working, which is what read-only mode is for. Also log, once, the first xART write the host refuses while writes are disabled, so the cause shows up in dmesg. Say so in the Kconfig help and the parameter description. Reported by @cb2206 on a J314s (14" M1 Pro). Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit ffa13517adce6796291e2b1f20817fd36a389674) [ryan: rebased onto the linear SEP history — on the line this lands on, xART writes are still opt-in (default 0; the defaults turn on later in d32c6c259c49), so the parameter keeps its opt-in wording and default and only gains the Touch ID remark, and the Kconfig sentence follows the opt-in paragraph, as aurora-wip composed it in 42ae23434d4c] Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 2 ++ drivers/soc/apple/sbio.rs | 13 +++++++++++ drivers/soc/apple/sep.rs | 38 +++++++++++++++++++++++++++++---- drivers/soc/apple/xarm.rs | 6 ++++++ drivers/soc/apple/xart_store.rs | 6 ++++++ 5 files changed, 61 insertions(+), 4 deletions(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 728bc96dec04d..dd927fc23c6d2 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -143,6 +143,8 @@ config APPLE_SEP opt-in is accepted only when the driver validates narrow physical extent ownership invariants; this does not prove all macOS-compatible update behavior. Keep it off until the target machine has been checked. + With writes off the driver still attaches, but refuses Touch ID enrol + and verify: the enclave saves anti-replay state after every match. Say Y here if you have an Apple silicon Mac. diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 92ff723142b88..fab5cc6b5102a 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2845,6 +2845,19 @@ impl SepData { if cmd == bio::IOC_ATTEST { return self.bio_attest(arg); } + // With xART writes off the enclave cannot persist what an enrolment or + // a match changes. It saves anti-replay state after every match, and a + // refused save has left it unresponsive until reboot. Refuse the + // capture instead; attach and the read-only ioctls still work. + if bio::starts_capture(cmd) && !self.xart_writable() { + if !self.capture_refusal_logged().xchg(true, Relaxed) { + dev_warn!( + self.dev, + "Touch ID: enrol and verify need xart_writes=1; the enclave saves anti-replay state after every match and stops responding when that save is refused\n" + ); + } + return Err(EROFS); + } // A capture started while the system goes to sleep would run into the // suspend; `capture_begin` repeats this check under the session lock. if bio::starts_capture(cmd) { diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index e4fa4523f1c9c..d6a76d20cb3d4 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -600,6 +600,11 @@ struct SepData { enrol_open: Atomic, + // One warning each per boot: an xART write refused because writes are + // off, and a Touch ID capture refused for the same reason. + xart_refusal_logged: Atomic, + capture_refusal_logged: Atomic, + #[pin] enrol_material: Mutex>, @@ -818,6 +823,8 @@ impl SepData { touchid_started: Atomic::new(false), touchid_failed: Atomic::new(false), enrol_open: Atomic::new(false), + xart_refusal_logged: Atomic::new(false), + capture_refusal_logged: Atomic::new(false), sensor_calibrated: Atomic::new(false), sensor_key_loaded: Atomic::new(false), templates_restored: Atomic::new(false), @@ -1272,6 +1279,16 @@ impl SepData { store.as_mut().map(f) } + /// Whether the enclave's xART writes reach the store. + pub(crate) fn xart_writable(&self) -> bool { + self.with_store(|store| store.writes_enabled()) + .unwrap_or(false) + } + + pub(crate) fn capture_refusal_logged(&self) -> &Atomic { + &self.capture_refusal_logged + } + fn with_host_store(&self, f: impl FnOnce(&mut store::Store) -> R) -> Option { let mut guard = self.host_store.lock(); let store: &mut Option = &mut guard; @@ -1353,20 +1370,33 @@ impl SepData { let os_uuid = self.xarm.lock().os_uuid; let serviced = self.with_store(|store| { - xarm::service( + let done = xarm::service( req, &payload, &mut staging, store, PROTECTED_DATA_AVAILABLE, os_uuid, - ) + ); + (done, store.writes_enabled()) }); - let Some(done) = serviced else { + let Some((done, writable)) = serviced else { self.fail_xarm(req.tag); return; }; + if !writable + && xarm::is_write(req.opcode) + && done.reply.status != xarm::STATUS_OK + && !self.xart_refusal_logged.xchg(true, Relaxed) + { + dev_warn!( + self.dev, + "xART: refused the enclave's write (op 0x{:02x}): writes are disabled (xart_writes=0)\n", + req.opcode + ); + } + if *module_parameters::xarm_trace.value() != 0 { dev_info!( self.dev, @@ -2325,7 +2355,7 @@ module! { params: { xart_writes: u8 { default: 0, - description: "Opt in to shared xART writes only if APFS proves a single unsnapshotted .gl extent", + description: "Opt in to shared xART writes only if APFS proves a single unsnapshotted .gl extent; Touch ID enrol and verify need it", }, xart_start_sector: u64 { default: 0, diff --git a/drivers/soc/apple/xarm.rs b/drivers/soc/apple/xarm.rs index ae9fe46728020..3e34b34c8951b 100644 --- a/drivers/soc/apple/xarm.rs +++ b/drivers/soc/apple/xarm.rs @@ -83,6 +83,12 @@ pub(crate) struct Serviced { pub(crate) reply_bytes: usize, } +/// Requests that change the store: a refusal of one means the enclave's state +/// was not persisted. +pub(crate) fn is_write(opcode: u8) -> bool { + matches!(opcode, OP_ROOT_WRITE | OP_SESSION_WRITE | OP_SESSION_DELETE) +} + pub(crate) fn service( req: &Request, outbound: &[u8], diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs index c487824f7d9a8..e47275ccb62d5 100644 --- a/drivers/soc/apple/xart_store.rs +++ b/drivers/soc/apple/xart_store.rs @@ -401,6 +401,12 @@ impl Store { Ok(true) } + /// Whether this store accepts writes: `xart_writes=1` and APFS proved the + /// extent writable. + pub(crate) fn writes_enabled(&self) -> bool { + self.writes_enabled + } + /// Byte offset of the APFS-resolved gigalocker extent in the container. pub(crate) fn base(&self) -> u64 { self.base From 10a0fac46c26d6f68e9377d4b7a4f89822c57b76 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:04:14 -0700 Subject: [PATCH 0760/2680] soc: apple: sep: retry a failed sensor bring-up from a fresh registration On a J314s, the attach-time patch can fail at the sequence counter init. After that, every verify retry sends CLEAR_STATE and re-registers the sensor serial, and the enclave refuses that registration for the rest of the boot, while a warm reboot (a fresh registration) recovers. When the retried bring-up fails, try once more from the fresh stage, as attach does. Also name the failing step of the sequence counter init, so the next report says whether the enclave or the sensor side failed. Reported-by: cb2206 (aurora-silicon/linux#69) Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- drivers/soc/apple/sbio.rs | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index fab5cc6b5102a..df8e0719cc986 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1315,7 +1315,17 @@ impl SepData { "verify: attach-time bring-up left the device-view proof undone; retrying the sensor bring-up\n" ); if !self.bring_sensor_online() { - dev_err!(self.dev, "verify: the retried sensor bring-up failed\n"); + // After a failed first patch, the enclave can refuse the serial + // re-registration that follows CLEAR_STATE, while a reboot's + // fresh registration works. Retry once from the fresh stage. + dev_warn!( + self.dev, + "verify: retrying the sensor bring-up from a fresh registration\n" + ); + self.bringup.store(BRINGUP_FRESH, Relaxed); + if !self.bring_sensor_online() { + dev_err!(self.dev, "verify: the retried sensor bring-up failed\n"); + } } let _ = sensor::idle(); self.refresh_match_credential(); @@ -2081,48 +2091,63 @@ impl SepData { } } + // Each failure names its step: a report that only says "sequence counter + // init failed" cannot tell the enclave side from the sensor side. fn init_sequence_counter(&self) -> bool { let challenge = match self.sbio_call(&crate::sbio::sbio_request_challenge()) { SbioOutcome::Ok(c) => c, SbioOutcome::PrerequisiteMissing => { if !self.establish_session() { + dev_warn!(self.dev, "sequence counter: session setup failed\n"); return false; } match self.sbio_call(&crate::sbio::sbio_request_challenge()) { SbioOutcome::Ok(c) => c, _ => { + dev_warn!(self.dev, "sequence counter: challenge request failed after session setup\n"); return false; } } } SbioOutcome::Status16 => { + dev_warn!(self.dev, "sequence counter: challenge request answered status 16\n"); return false; } SbioOutcome::Other => { + dev_warn!(self.dev, "sequence counter: challenge request failed\n"); return false; } }; if challenge.len() < sensor::CHALLENGE_LEN { + dev_warn!(self.dev, "sequence counter: short challenge ({} bytes)\n", challenge.len()); return false; } let mut out = [0u8; sensor::CHALLENGE_LEN]; out.copy_from_slice(&challenge[..sensor::CHALLENGE_LEN]); if sensor::send_challenge(&out).is_err() { + dev_warn!(self.dev, "sequence counter: sending the challenge to the sensor failed\n"); return false; } let reply = match sensor::read_challenge_reply() { Ok(r) => r, Err(_) => { + dev_warn!(self.dev, "sequence counter: the sensor did not answer the challenge\n"); return false; } }; match self.sbio_transfer(&crate::sbio::sbio_commit_challenge(&reply)) { Ok(done) if done.status.is_ok() => true, - Ok(_) => false, - Err(_) => false, + Ok(done) => { + dev_warn!(self.dev, "sequence counter: the enclave refused the sensor's reply, status {}\n", done.status); + false + } + Err(e) => { + dev_warn!(self.dev, "sequence counter: committing the reply failed {:?}\n", e); + false + } } } From 15adf7070d65856d84ab6a80117a2a9f196a8cf5 Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 12:18:39 -0700 Subject: [PATCH 0761/2680] soc: apple: sep: run the T6000 profile on the M1 Max (t6001) detect() refuses apple,t6001 with ENODEV, so the driver cannot run on the 14" and 16" MacBook Pro with M1 Max (j314c, j316c). Map it to the T6000 profile. t6000.dtsi is defined as a cut-down t6001: it includes t6001.dtsi, deletes pmgr_south and a few PMP report nodes, and overrides the GPU compatible and the PMP ranges. Everything the driver touches comes from files both parts share. t600x-die0.dtsi provides the SEP at 0x396400000, its mailbox at 0x396408000 (AIC 545-548), its DART at 0x3952c0000 (AIC 551) and spi2 at 0x39b108000 (AIC 1108). t600x-gpio-pins.dtsi provides spi2's pins, and pmgr_east in t600x-pmgr.dtsi provides the ps_sep and ps_spi2 power domains. The compiled j314c and j314s trees agree on all of these nodes and on every node they reference. Each T6000 field follows from that shared description or from the M1-family cold boot, not from the die size. The spi2 base is the address above. The SEP DART carries no apple,dma-range on either part, so dart_range_required and wide_dma_mask stay false. The firmware region is the "sepfw" region m1n1's dt_set_sep() adds for any machine with a sep alias, with no chip check. The shared-memory geometry, the cold-boot handshake and the 13.5 key-store encoding are the M1-family ones the M1 Pro already uses, and the shared-memory layout is still checked against the boot manifests at runtime. The one board-read value is /defaults cpx-encryption-mode, which is 2 on J316s as on J293, J313 and J415. If J314c or J316c differed, SET_ENV would be refused and the key store would stay closed. The Mac Studio j375c is also apple,t6001, but its device tree leaves the SEP disabled and has no sep alias, so no platform device is created and probe never runs. T6000 is a cold-boot profile, so module init does not apply the runtime changeset that enables the SEP on the warm-registration platforms either. The mapping is inert there; the only change is that module init no longer logs "no platform profile". apple,t6002 (M1 Ultra, Mac Studio only, no built-in sensor) stays unmapped. Untested: no M1 Max machine has run this yet. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- drivers/soc/apple/profile.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 9da9e53cdbad6..7d8ced489a4a2 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -388,6 +388,16 @@ pub(crate) fn detect() -> Result<&'static PlatformProfile> { if machine_has(b"apple,t6000") { return Ok(&T6000); } + // t6000.dtsi is defined as a cut-down t6001: it includes t6001.dtsi and + // deletes the parts the smaller die lacks. Both pull in t600x-die0.dtsi, + // so the SEP, its mailbox, its DART and the SPI controller sit at the same + // addresses with the same interrupts on either part, and the T6000 + // constants apply to the M1 Max unchanged. The Mac Studio (j375c) is also + // a t6001 but leaves the SEP disabled, so nothing binds there; apple,t6002 + // (M1 Ultra, no built-in sensor) stays unmapped. + if machine_has(b"apple,t6001") { + return Ok(&T6000); + } if machine_has(b"apple,t8140") { return Ok(&T8140); } From f0db15013d7581f24ffc1973e5810be2f0afc5b0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 22:55:27 -0400 Subject: [PATCH 0762/2680] drm/asahi: correct reviewed lifetime, failure and submission paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix G17 descriptor fields, scratch state, prefix failure history, recovery evidence, retirement wakeups, queue admission and output-fence publication. Validate all timestamp destinations before enqueue and defer their cleanup onto a preallocated module-owned workqueue so fence producers do not take buffer reservations while destroying timestamp mappings. Correct shared GPUVM ownership and lock ordering, fixed-map rollback, page-table child unlinking, repeated-page metadata and legacy TLBI/bind validation. Balance legacy power/kick bookkeeping and propagate RTKit endpoint-start errors. Retain module code through DRM files, PRIME exports and detached scheduler callbacks, disable scheduler work at finalization, release the G17 shutdown DRM backedge and destroy initialized device data. The source-hashed integration snapshot passed all 64 local host validation commands and the shared Rust, C DRM/RTKit and Asahi module build. These checks use explicit host doubles and original-failure controls; they are not hardware, lockdep, module-unload or performance qualification. The public-base RTKit change is limited to the same STARTEP error propagation. Remaining timestamp publication ordering, deferred cleanup, legacy reference cycles and firmware-contract questions are tracked for follow-up. No queue depth reduction or serialization workaround is introduced. [ryan: adapted to 7.2.10 — minimal; for DJ: rust/kernel/drm/gpuvm.rs and drm/asahi/crashdump.rs are deleted in 7.2. The GPUVM pieces the driver uses (lock_inner(), its guard and unmap_all()) are ported into rust/kernel/drm/gpuvm/mod.rs next to Asahi's exec_lock() hack; the dec_ref, exec_lock and obtain fixes are not needed there (7.2 already puts BOs deferred, guards exec errors and obtains through GpuVmBoAlloc). The device release fix is covered by Asahi's drop_in_place(). Driver::MODULE sits next to the 7.2 FEAT_* constants, and the module registers AsahiPlatformDriver. In mmu.rs unmap_gpuva() uses GpuVa::length() (7.2 range() is a Range), the timestamp mapping keeps a VMapOwned, bind_object() keeps Asahi's segment prefetch, and the legacy run_job() keeps the registration guard with the threshold increase after the last submit()] Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit ca389fba9b6c7c9354c2f765fe81c5ef5f069f3b) --- drivers/gpu/drm/asahi/asahi.rs | 28 ++- drivers/gpu/drm/asahi/cleanup.rs | 118 ++++++++++ drivers/gpu/drm/asahi/driver.rs | 1 + drivers/gpu/drm/asahi/file.rs | 18 +- drivers/gpu/drm/asahi/g17/channel.rs | 12 +- drivers/gpu/drm/asahi/g17/command.rs | 8 +- drivers/gpu/drm/asahi/g17/completion.rs | 78 ++++-- drivers/gpu/drm/asahi/g17/context.rs | 5 + drivers/gpu/drm/asahi/g17/fence.rs | 24 +- drivers/gpu/drm/asahi/g17/fw/compute.rs | 75 +++++- drivers/gpu/drm/asahi/g17/fw/kick.rs | 8 +- drivers/gpu/drm/asahi/g17/fw/render.rs | 29 ++- drivers/gpu/drm/asahi/g17/initdata.rs | 38 ++- drivers/gpu/drm/asahi/g17/job.rs | 5 +- drivers/gpu/drm/asahi/g17/mod.rs | 43 +++- drivers/gpu/drm/asahi/g17/power.rs | 22 +- drivers/gpu/drm/asahi/g17/queue/compute.rs | 28 ++- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 92 ++++---- drivers/gpu/drm/asahi/g17/queue/mod.rs | 125 +++++----- drivers/gpu/drm/asahi/g17/queue/render.rs | 3 +- .../gpu/drm/asahi/g17/queue/render/memory.rs | 6 +- drivers/gpu/drm/asahi/g17/recovery.rs | 83 ++++--- .../gpu/drm/asahi/g17/recovery/attribution.rs | 17 +- drivers/gpu/drm/asahi/g17/runtime.rs | 67 +++++- .../gpu/drm/asahi/g17/runtime/compute_exit.rs | 53 +++-- .../drm/asahi/g17/runtime/compute_replay.rs | 22 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 16 +- drivers/gpu/drm/asahi/g17/runtime/render.rs | 28 ++- .../gpu/drm/asahi/g17/runtime/render/grow.rs | 3 +- .../drm/asahi/g17/runtime/render/prepare.rs | 23 +- .../drm/asahi/g17/runtime/render/submit.rs | 7 + drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 7 +- drivers/gpu/drm/asahi/g17/teardown.rs | 31 ++- drivers/gpu/drm/asahi/gem.rs | 24 +- drivers/gpu/drm/asahi/gpu.rs | 3 +- drivers/gpu/drm/asahi/hw/mod.rs | 9 +- drivers/gpu/drm/asahi/mem.rs | 7 +- drivers/gpu/drm/asahi/mmu.rs | 222 +++++++++++------- drivers/gpu/drm/asahi/mmu/bind.rs | 1 + drivers/gpu/drm/asahi/mmu/lifetime.rs | 47 +--- drivers/gpu/drm/asahi/pgtable.rs | 82 ++++++- drivers/gpu/drm/asahi/queue/mod.rs | 17 +- drivers/gpu/drm/asahi/util.rs | 13 +- drivers/gpu/drm/drm_gpuvm.c | 8 +- drivers/gpu/drm/scheduler/sched_entity.c | 9 + drivers/gpu/drm/scheduler/sched_main.c | 9 + drivers/soc/apple/rtkit.c | 4 +- include/drm/drm_gpuvm.h | 1 + include/drm/gpu_scheduler.h | 8 + rust/kernel/drm/device.rs | 2 +- rust/kernel/drm/driver.rs | 4 + rust/kernel/drm/gem/mod.rs | 9 +- rust/kernel/drm/gpuvm/mod.rs | 94 +++++++- rust/kernel/drm/sched.rs | 8 + 54 files changed, 1245 insertions(+), 459 deletions(-) create mode 100644 drivers/gpu/drm/asahi/cleanup.rs diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi.rs index ec093aeb009bf..cb67389635f1b 100644 --- a/drivers/gpu/drm/asahi/asahi.rs +++ b/drivers/gpu/drm/asahi/asahi.rs @@ -6,6 +6,7 @@ mod alloc; mod buffer; mod channel; +mod cleanup; mod debug; mod driver; mod event; @@ -31,8 +32,31 @@ mod util; mod vm; mod workqueue; -kernel::module_platform_driver! { - type: driver::AsahiPlatformDriver, +use kernel::prelude::*; + +#[pin_data] +struct DriverModule { + // Field drop order unregisters/joins driver producers before queue teardown. + #[pin] + _driver: kernel::driver::Registration>, + _cleanup: cleanup::QueueOwner, +} + +impl kernel::InPlaceModule for DriverModule { + fn init(module: &'static kernel::ThisModule) -> impl PinInit { + try_pin_init!(Self { + // Initialize the queue first, before registration can invoke probe. + _cleanup: cleanup::QueueOwner::new()?, + _driver <- kernel::driver::Registration::new( + ::NAME, + module, + ), + }) + } +} + +module! { + type: DriverModule, name: "asahi", description: "AGX GPU driver for Apple silicon SoCs", license: "Dual MIT/GPL", diff --git a/drivers/gpu/drm/asahi/cleanup.rs b/drivers/gpu/drm/asahi/cleanup.rs new file mode 100644 index 0000000000000..e3dd5ad95dc0c --- /dev/null +++ b/drivers/gpu/drm/asahi/cleanup.rs @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Reservation-taking timestamp destruction must not block fence producers. +//! +//! The module owns this queue outside its platform registration. Files and +//! exported buffers retain the module; detached scheduler cleanup retains it +//! until its driver callback returns. Unregister joins the firmware/event work. +//! Only after those producer boundaries close may the module destroy this queue. + +use core::{ + ops::Deref, + ptr::{self, NonNull}, + sync::atomic::{AtomicPtr, Ordering}, +}; +use kernel::{ + bindings, c_str, impl_has_work, new_work, + prelude::*, + workqueue::{Queue, Work, WorkItem}, +}; + +static QUEUE: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + +/// Destroyed by module exit after platform-driver unregister, never by a worker. +pub(crate) struct QueueOwner(NonNull); + +// SAFETY: QueueOwner only transfers ownership of a thread-safe kernel queue. +unsafe impl Send for QueueOwner {} +// SAFETY: The queue API synchronizes all accesses; destruction requires ownership. +unsafe impl Sync for QueueOwner {} + +impl QueueOwner { + pub(crate) fn new() -> Result { + // SAFETY: The name is NUL terminated and contains no format specifiers. + let raw = unsafe { + bindings::alloc_workqueue_noprof( + c_str!("asahi-ts-cleanup").as_char_ptr(), + bindings::wq_flags_WQ_UNBOUND | bindings::wq_flags_WQ_MEM_RECLAIM, + 0, + ) + }; + let raw = NonNull::new(raw).ok_or(ENOMEM)?; + if QUEUE + .compare_exchange(ptr::null_mut(), raw.as_ptr(), Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + // SAFETY: This fresh queue has never been published or used. + unsafe { bindings::destroy_workqueue(raw.as_ptr()) }; + return Err(EBUSY); + } + Ok(Self(raw)) + } +} + +impl Drop for QueueOwner { + fn drop(&mut self) { + // No external producers remain: module registration was dropped first. + // Pending cleanup owns no queue reference and cannot enqueue more work. + // SAFETY: The module's unique owner retains the queue through its drain. + unsafe { bindings::destroy_workqueue(self.0.as_ptr()) }; + QUEUE.store(ptr::null_mut(), Ordering::Release); + } +} + +#[pin_data] +struct Item { + value: T, + #[pin] + work: Work, +} + +impl_has_work! { + impl{T: Send + 'static} HasWork> for Item { self.work } +} + +impl WorkItem for Item { + type Pointer = Pin>; + + fn run(this: Self::Pointer) { + drop(this); + } +} + +/// Allocate the destruction work before publication; final handoff cannot fail. +pub(crate) struct Deferred(Option>>>); + +impl Deferred { + pub(crate) fn new(value: T) -> Result { + let item = KBox::pin_init( + pin_init!(Item { + value, + work <- new_work!("Asahi timestamp cleanup"), + }), + GFP_KERNEL, + )?; + Ok(Self(Some(item))) + } +} + +impl Deref for Deferred { + type Target = T; + + fn deref(&self) -> &T { + &self.0.as_ref().unwrap().value + } +} + +impl Drop for Deferred { + fn drop(&mut self) { + if let Some(item) = self.0.take() { + let raw = QUEUE.load(Ordering::Acquire); + assert!(!raw.is_null()); + // SAFETY: Module ownership and producer joins retain the queue until + // this handoff has returned. The unique pinned box is never queued + // twice; enqueue transfers it without allocation or a failure path. + unsafe { Queue::from_raw(raw) }.enqueue(item); + } + } +} diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index ed588548b6c97..e97df0fbac2d8 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -83,6 +83,7 @@ impl drm::driver::Driver for AsahiDrmDriver { type ParentDevice = platform::Device; const INFO: drm::driver::DriverInfo = INFO; + const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); const FEAT_RENDER: bool = true; const FEAT_SYNCOBJ: bool = true; const FEAT_SYNCOBJ_TIMELINE: bool = true; diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 76757dae0b182..5227ba03872fb 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -182,11 +182,11 @@ impl SyncItem { in_count: u32, out_count: u32, ) -> Result> { - let count = in_count + out_count; - let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; + let count = in_count.checked_add(out_count).ok_or(EOVERFLOW)?; const STRIDE: usize = core::mem::size_of::(); - let size = STRIDE * count as usize; + let size = STRIDE.checked_mul(count as usize).ok_or(EOVERFLOW)?; + let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; // SAFETY: We only read this once, so there are no TOCTOU issues. let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); @@ -672,7 +672,9 @@ impl File { let vm_id = data.vm_id.try_into()?; let mut vec = KVec::new(); - let size = (data.stride * data.num_binds) as usize; + let size = usize::try_from(data.stride)? + .checked_mul(usize::try_from(data.num_binds)?) + .ok_or(EOVERFLOW)?; let reader = UserSlice::new(UserPtr::from_addr(data.userptr as _), size).reader(); reader.read_all(&mut vec, GFP_KERNEL)?; let mut reader = Reader::new(&vec); @@ -779,6 +781,9 @@ impl File { data: &uapi::drm_asahi_gem_bind_op, file: &DrmFile, ) -> Result { + if data.range == 0 { + return Err(EINVAL); + } if (data.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_UNBIND) != 0 { Self::do_gem_unbind(device, vm_id, data, file) } else { @@ -1063,7 +1068,10 @@ impl File { .inner() .objects() .lock() - .insert_limit(1..=u32::MAX, obj, GFP_KERNEL)? as u64; + .insert_limit(1..=u32::MAX, obj, GFP_KERNEL); + // StoreError owns the rejected object. Convert it only after the + // temporary XArray guard is gone: dropping a mapping may sleep. + let handle = handle? as u64; data.object_handle = handle as u32; Ok(0) diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index 59e8e95d4bbd9..8e4740b76a166 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -9,9 +9,7 @@ //! empty slot and orders the record, power tally and producer together. use core::sync::atomic::{fence, Ordering}; -use kernel::prelude::*; - -use crate::driver::AsahiDrmDevice; +use kernel::{device::Device, prelude::*}; use super::fw::{channels::*, initdata::status}; use super::initdata::InitData; @@ -57,15 +55,15 @@ impl Rings { } } - pub(crate) fn drain_trace(&mut self, dev: &AsahiDrmDevice, init: &InitData, role: Role) { + pub(crate) fn drain_trace(&mut self, dev: &Device, init: &InitData, role: Role) { self.drain_report(dev, init, role, ReportRing::Trace); } - pub(crate) fn drain_statistics(&mut self, dev: &AsahiDrmDevice, init: &InitData, role: Role) { + pub(crate) fn drain_statistics(&mut self, dev: &Device, init: &InitData, role: Role) { self.drain_report(dev, init, role, ReportRing::Statistics); } - fn drain_report(&mut self, dev: &AsahiDrmDevice, init: &InitData, role: Role, ring: ReportRing) { + fn drain_report(&mut self, dev: &Device, init: &InitData, role: Role, ring: ReportRing) { let quarantined = &mut self.quarantined[role as usize][ring as usize]; if *quarantined { return; @@ -176,7 +174,7 @@ impl Events { } } - pub(crate) fn next(&mut self, dev: &AsahiDrmDevice, init: &InitData) -> Result> { + pub(crate) fn next(&mut self, dev: &Device, init: &InitData) -> Result> { let role = self.role; let state = status::EVENT_STATE as usize; let consumer_word = init.status_word(role, state)?; diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 47f9c0048e673..5929c14b264fd 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -3,6 +3,7 @@ //! Decode a submission and validate its user resources before publishing any work. use super::fw::{ + compute::ScratchRequest, kick::WriteRange, render::{RenderPass, UscProgram, ZlsSurface}, }; @@ -279,6 +280,7 @@ pub(crate) enum Validated { timestamps: [uapi::drm_asahi_timestamps; 2], }, Compute { + scratch: ScratchRequest, usc_base: u64, cdm_va: u64, cdm_end_va: u64, @@ -395,7 +397,11 @@ impl Payload { | (u64::from(cmd.bg_partial_rsrc_spec_hi) << 32), }, eot_offset: cmd.eot.usc, + eot_rsrc_spec: u64::from(cmd.eot.rsrc_spec) + | (u64::from(cmd.bg_eot_rsrc_spec_hi) << 32), partial_eot_offset: cmd.partial_eot.usc, + partial_eot_rsrc_spec: u64::from(cmd.partial_eot.rsrc_spec) + | (u64::from(cmd.bg_eot_partial_rsrc_spec_hi) << 32), rsrc_spec_hi: cmd.flags & RSRC_SPEC_HI != 0, }; Ok(Validated::Render { @@ -406,7 +412,6 @@ impl Payload { } Self::Compute(cmd, attachments) => { if cmd.flags != 0 - || cmd.scs_layout != 0 || cmd.cdm_ctrl_stream_base & 3 != 0 || cmd.cdm_ctrl_stream_end & 3 != 0 || cmd.cdm_ctrl_stream_end <= cmd.cdm_ctrl_stream_base @@ -423,6 +428,7 @@ impl Payload { sampler(space, cmd.sampler_heap, cmd.sampler_count)?; attachments.validate(space)?; Ok(Validated::Compute { + scratch: ScratchRequest::new(cmd.scs_layout)?, usc_base: window.base, cdm_va: cmd.cdm_ctrl_stream_base, cdm_end_va: cmd.cdm_ctrl_stream_end - 4, diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 2c66d745af112..feb69e8b2c291 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -24,8 +24,9 @@ use kernel::{ }; struct Destination { - mapping: Arc, - offset: usize, + // Keep the existing GPU alias alive along with its prepared CPU view. + _mapping: Arc, + cpu: mmu::TimestampMapping, } impl Destination { @@ -46,7 +47,8 @@ impl Destination { if offset.checked_add(size_of::()).ok_or(EINVAL)? > mapping.size() { return Err(ERANGE); } - Ok(Some(Self { mapping, offset })) + let cpu = mapping.timestamp_mapping(offset)?; + Ok(Some(Self { _mapping: mapping, cpu })) } fn write(&self, ticks: u64) -> Result { @@ -56,12 +58,45 @@ impl Destination { .wrapping_add( ticks % cfg::COMMAND_TIMESTAMP_HZ * NSEC_PER_SEC / cfg::COMMAND_TIMESTAMP_HZ, ); - self.mapping.write_timestamp(self.offset, ns)?; + self.cpu.write(ns); barrier(Ordering::Release); Ok(()) } } +/// Timestamp mappings resolved and retained before any command is enqueued. +pub(crate) struct Destinations(Option; 4]>>); + +impl Destinations { + pub(crate) fn resolve( + objects: Pin<&xarray::XArray>>, + timestamps: &[uapi::drm_asahi_timestamps], + ) -> Result { + if timestamps.is_empty() || timestamps.len() > 2 { + return Err(EINVAL); + } + let mut destinations = core::array::from_fn(|_| None); + for (index, timestamp) in timestamps.iter().enumerate() { + destinations[2 * index] = Destination::resolve(objects, timestamp.start)?; + destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end)?; + } + // Empty submissions need no cleanup allocation. Timestamped commands + // prepare the unique work item before any output fence is published. + let retained = if destinations.iter().any(Option::is_some) { + Some(crate::cleanup::Deferred::new(destinations)?) + } else { + None + }; + Ok(Self(retained)) + } + + fn iter(&self) -> core::slice::Iter<'_, Option> { + let destinations: &[Option] = + self.0.as_deref().map_or(&[], |destinations| destinations.as_slice()); + destinations.iter() + } +} + /// Firmware writes two aligned words in a fresh write-combined page per compute command. struct ComputeTimestamps { // Remove the GPU alias before releasing its backing. @@ -87,7 +122,7 @@ pub(crate) struct Completion { status: Arc, fence: UserFence, member: Member, - destinations: [Option; 4], + destinations: Destinations, compute: Option, feed: Option>, #[pin] @@ -95,6 +130,8 @@ pub(crate) struct Completion { #[pin] render: Mutex>, signalled: AtomicBool, + retirement_queued: AtomicBool, + accepted: AtomicBool, timeout_error: AtomicI32, replays: AtomicU8, replay_timeout: AtomicBool, @@ -109,19 +146,10 @@ impl Completion { context: &Arc, member: Member, vm_job: mmu::VmJobGuard, - objects: Pin<&xarray::XArray>>, - timestamps: &[uapi::drm_asahi_timestamps], + destinations: Destinations, compute: bool, feed: Option>, ) -> Result> { - if timestamps.len() != if compute { 1 } else { 2 } { - return Err(EINVAL); - } - let mut destinations = core::array::from_fn(|_| None); - for (index, timestamp) in timestamps.iter().enumerate() { - destinations[2 * index] = Destination::resolve(objects, timestamp.start)?; - destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end)?; - } let fence = fence::independent(contexts)?; let compute = if compute { Some(ComputeTimestamps::new(dev, context.vm())?) @@ -139,6 +167,8 @@ impl Completion { vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), render <- new_mutex!(None, "G17 command render lease"), signalled: AtomicBool::new(false), + retirement_queued: AtomicBool::new(false), + accepted: AtomicBool::new(false), timeout_error: AtomicI32::new(0), replays: AtomicU8::new(0), replay_timeout: AtomicBool::new(false), @@ -152,6 +182,11 @@ impl Completion { pub(crate) fn fence(&self) -> Fence { Fence::from_fence(&self.fence) } + /// All fallible scheduler setup has succeeded. Failures from this point + /// must be visible through VM_STATUS before the accepted job's fence. + pub(crate) fn mark_accepted(&self) { + self.accepted.store(true, Ordering::Release); + } pub(crate) fn status(&self) -> &Arc { &self.status } @@ -229,6 +264,16 @@ impl Completion { Ok(()) } + /// A retirement witness has been accepted by an off-lock completion batch. + /// This is separate from fence signalling and from a failure still owned by firmware. + pub(crate) fn note_retirement_queued(&self) { + self.retirement_queued.store(true, Ordering::Release); + } + + pub(crate) fn retirement_queued(&self) -> bool { + self.retirement_queued.load(Ordering::Acquire) + } + /// The first timeout request retains its errno while the physical graph is /// borrowed. Duplicate watchdog or cancellation callbacks cannot replace it. pub(crate) fn defer_timeout(&self, error: Error) { @@ -342,6 +387,9 @@ impl Completion { return; } self.status.record_if_device_loss(error); + if self.accepted.load(Ordering::Acquire) { + self.status.report_failure(error); + } self.fence.set_error(error); self.release_healthy(); self.fence.signal(); diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 1a28386d788d0..ee2d8133ea8cd 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -213,6 +213,11 @@ impl Context { pub(crate) fn scheduler_owner(&self) -> Arc { self.scheduler.clone() } + /// Scheduler release and command retirement have both been witnessed. + /// Keep installed storage alive while returning the finite TTBAT identity. + pub(crate) fn release_execution(&self) { + self.execution.release(); + } pub(crate) fn scheduler_generation(&self) -> u32 { self.scheduler_generation } diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 1217d7e7b3f4e..3414e36db9d36 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -39,6 +39,15 @@ pub(crate) fn independent(contexts: &FenceContexts) -> Result Result { + let fence = independent(contexts)?; + fence.set_error(ENOMEM); + fence.signal(); + Ok(Fence::from_fence(&fence)) +} + /// All command members of an ioctl share its work-state node and aggregate fence. /// The initial member keeps the aggregate unsignalled until enqueue finishes. #[pin_data] @@ -165,14 +174,25 @@ pub(crate) struct Outputs { impl Outputs { pub(crate) fn new( contexts: &FenceContexts, - syncs: KVec, + mut syncs: KVec, input_count: usize, + allocation_failure: &Fence, ) -> Result { if input_count > syncs.len() { return Err(EINVAL); } + let fallback = match independent(contexts) { + Ok(fence) => fence, + Err(error) => { + // Syncobjs and timeline-chain storage are already resolved. + // No work has been accepted; publish the pre-signaled ENOMEM + // fence without another allocation before returning the error. + install(syncs.drain(input_count..), allocation_failure); + return Err(error); + } + }; Ok(Self { - fallback: independent(contexts)?, + fallback, syncs, input_count, submission: None, diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 22f8259508ff6..5b7489cfe93f4 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -24,6 +24,62 @@ pub(crate) const COMPUTE_QOS_CLASS: u8 = 0x08; pub(crate) const COMPUTE_KICK_EVENT_MASK: [u64; 4] = [0, 0, 4, 0]; const COMPUTE_REGISTERS: usize = 40; +const COMPUTE_REGISTERS_MAX: usize = COMPUTE_REGISTERS + 1; + +/// Validated per-command shader-context scratch request. +#[derive(Clone, Copy)] +pub(crate) struct ScratchRequest { + pages: u16, +} + +impl ScratchRequest { + const FIXED_BYTES: u64 = 0x58000; + const THREADS_PER_CLIQUE: u64 = 32; + const BYTES_PER_STACK_ENTRY: u64 = 8; + const PAGE_SHIFT: u32 = 12; + const PAGE_SIZE: u64 = 1 << Self::PAGE_SHIFT; + const PAGES_PER_MIB: u64 = 1 << (20 - Self::PAGE_SHIFT); + const WINDOW_ENABLED: u64 = 1 << 24; + + /// Payload-free layouts request no scratch even if depth/cliques are set. + /// Otherwise round the fixed and per-thread sections and add a guard page. + pub(crate) fn new(layout: u64) -> Result { + let depth = layout & u16::MAX as u64; + let cliques = (layout >> 24) & u8::MAX as u64; + let user = (layout >> 32) & u16::MAX as u64; + let driver = (layout >> 48) & u16::MAX as u64; + if user == 0 && driver == 0 { + return Ok(Self { pages: 0 }); + } + let threads = cliques.checked_mul(Self::THREADS_PER_CLIQUE).ok_or(EINVAL)?; + let per_thread = depth.checked_mul(Self::BYTES_PER_STACK_ENTRY) + .and_then(|bytes| bytes.checked_add(user)) + .and_then(|bytes| bytes.checked_add(driver)).ok_or(EINVAL)?; + let bytes = threads.checked_mul(per_thread) + .and_then(|bytes| bytes.checked_add(Self::FIXED_BYTES)) + .and_then(|bytes| bytes.checked_add(2 * Self::PAGE_SIZE - 1)) + .ok_or(EINVAL)?; + let pages = bytes >> Self::PAGE_SHIFT; + Ok(Self { pages: u16::try_from(pages).map_err(|_| EINVAL)? }) + } + + pub(crate) fn enabled(self) -> bool { + self.pages != 0 + } + + fn size_word(self) -> u64 { + let pages = u64::from(self.pages); + pages | (pages.div_ceil(Self::PAGES_PER_MIB) << 16) + } + + fn window_word(self) -> u64 { + if self.enabled() { Self::WINDOW_ENABLED } else { UNK_10791 } + } + + fn checkpoints(self) -> [u16; 4] { + CHECKPOINTS.map(|count| count + u16::from(self.enabled())) + } +} const ALT_REGISTERS: usize = 4; /// Object IDs of every compute kick. @@ -54,6 +110,8 @@ const CHECKPOINTS: [u16; 4] = [38, 39, 39, COMPUTE_REGISTERS as u16]; /// Inputs of one compute descriptor. pub(crate) struct ComputeArgs { + /// Scratch request validated with the userspace command. + pub(crate) scratch: ScratchRequest, /// Base of the queue's USC window. pub(crate) usc_base: u64, /// GPU address of the control stream. @@ -103,13 +161,14 @@ pub(crate) struct ComputeArgs { pub(crate) timestamp_va: u64, } -fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGISTERS]> { +fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGISTERS_MAX]> { let preempt = args.preempt_va; let operand = args.operand_state_va; let key = work_key(args.context_id, args.work_state); let ids = OBJECT_IDS.word(); Ok([ + RegisterWrite::new(0x017d9, 1), RegisterWrite::new(0x1a510, preempt), RegisterWrite::new(0x1a420, args.cdm_va), RegisterWrite::new(0x1a4d0, offset_va(preempt, PREEMPT_STATE[0])?), @@ -130,10 +189,10 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS RegisterWrite::new(0x1a061, 0), RegisterWrite::new(0x1a0b9, 0), RegisterWrite::new(0x1a0c1, 0), - RegisterWrite::new(0x101d1, 0), + RegisterWrite::new(0x101d1, args.scratch.size_word()), RegisterWrite::new(0x0d479, 0), RegisterWrite::new(0x1a0e9, UNK_1A0E9), - RegisterWrite::new(0x107a1, UNK_10791), + RegisterWrite::new(0x107a1, args.scratch.window_word()), RegisterWrite::new(0x0a599, UNK_0A599), RegisterWrite::new(0x0d411, UNK_0D411), RegisterWrite::new(0x1a540, ids), @@ -287,8 +346,11 @@ impl ComputeDescriptor { self.tag = CommandTag::Compute as u32; self.context_id = args.context_id.into(); self.work_state_va = args.work_state_va; - self.checkpoints = CHECKPOINTS; - self.registers.set(registers_va, ®isters)?; + self.checkpoints = args.scratch.checkpoints(); + // The enabled program prepends one register; disabled descriptors keep + // the same forty writes, trailer count and resume checkpoints. + let first = usize::from(!args.scratch.enabled()); + self.registers.set(registers_va, ®isters[first..])?; self.alt_registers.set(alt_registers_va, &alt_registers)?; self.preempt_va = args.preempt_va; self.cdm_end_va = args.cdm_end_va; @@ -321,10 +383,11 @@ impl ComputeDescriptor { /// the descriptor's address as seen by the work's VM. pub(crate) fn register_bindings( descriptor_va: u64, + scratch: ScratchRequest, ) -> Result<[Option; 4]> { let registers = RegisterArrayBinding::new( offset_va(descriptor_va, Self::REGISTERS_OFFSET)?, - CHECKPOINTS[0] as u8, + scratch.checkpoints()[0] as u8, )?; let alt_registers = RegisterArrayBinding::new( offset_va(descriptor_va, Self::ALT_REGISTERS_OFFSET)?, diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index c8b7d3f0dce3a..2e6133e882b2d 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -369,6 +369,8 @@ pub(crate) struct KickArgs<'a> { pub(crate) event_mask: [u64; 4], /// Register arrays bound to the kick. pub(crate) register_arrays: [Option; 4], + /// The compute register program includes a shader-context scratch request. + pub(crate) compute_scratch: bool, } /// One kick ring entry. @@ -379,7 +381,7 @@ pub(crate) struct KickArgs<'a> { #[derive(Copy, Clone, Debug)] pub(crate) struct KickEntry { /// Timestamp in bits 41:2, QID in bits 48:42, MCache table present in bit - /// 58 and queue priority in bits 60:59. + /// 58, queue priority in bits 60:59 and compute scratch enabled in bit 61. pub(crate) header: u64, /// MCache table: address >> 5 in bits 43:6, count - 1 in bits 59:54. pub(crate) mcache: u64, @@ -419,6 +421,7 @@ impl KickEntry { const QID_SHIFT: u32 = 42; const MCACHE_PRESENT: u64 = 1 << 58; const PRIORITY_SHIFT: u32 = 59; + const COMPUTE_SCRATCH: u64 = 1 << 61; const QOS_SLOT_SHIFT: u32 = 32; const QOS_CLASS_SHIFT: u32 = 40; const QOS_CLASS_MAX: u8 = 0x1f; @@ -440,6 +443,9 @@ impl KickEntry { let mut header = args.timestamp.get() << Self::TIMESTAMP_SHIFT | (args.qid as u64) << Self::QID_SHIFT | (QUEUE_PRIORITY as u64) << Self::PRIORITY_SHIFT; + if args.compute_scratch { + header |= Self::COMPUTE_SCRATCH; + } let mcache = match args.mcache { Some(table) => { header |= Self::MCACHE_PRESENT; diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 349d03ed14d71..e3da1b0e6d954 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -206,11 +206,14 @@ pub(crate) struct RenderPass { pub(crate) bg: UscProgram, /// Partial-render background program. pub(crate) partial_bg: UscProgram, - /// Tagged USC offset of the end-of-tile program. Its resource specifier - /// is always written as zero. + /// Tagged USC offset of the end-of-tile program. pub(crate) eot_offset: u32, + /// Resource specifier of the end-of-tile program. + pub(crate) eot_rsrc_spec: u64, /// Tagged USC offset of the partial-render end-of-tile program. pub(crate) partial_eot_offset: u32, + /// Resource specifier of the partial-render end-of-tile program. + pub(crate) partial_eot_rsrc_spec: u64, /// The resource specifiers have their high 32 bits. pub(crate) rsrc_spec_hi: bool, } @@ -383,7 +386,9 @@ struct PassWords { partial_bg_rsrc_spec: u64, partial_bg_va: u64, eot_va: u64, + eot_rsrc_spec: u64, partial_eot_va: u64, + partial_eot_rsrc_spec: u64, } impl PassWords { @@ -446,7 +451,9 @@ impl PassWords { partial_bg_rsrc_spec: rsrc_spec(pass.partial_bg.rsrc_spec), partial_bg_va: usc(pass.partial_bg.offset)?, eot_va: usc(pass.eot_offset)?, + eot_rsrc_spec: rsrc_spec(pass.eot_rsrc_spec), partial_eot_va: usc(pass.partial_eot_offset)?, + partial_eot_rsrc_spec: rsrc_spec(pass.partial_eot_rsrc_spec), }) } } @@ -580,7 +587,7 @@ fn fragment_registers( RegisterWrite::new(0x01739, 1), RegisterWrite::new(0x10009, w.utile_config), // End-of-tile resource specifier. - RegisterWrite::new(0x15379, 0), + RegisterWrite::new(0x15379, w.eot_rsrc_spec), RegisterWrite::new(0x15381, w.eot_va), RegisterWrite::new(0x15369, w.bg_rsrc_spec), RegisterWrite::new(0x15371, w.bg_va), @@ -676,7 +683,7 @@ fn partial_store_registers( ) -> [RegisterWrite; PARTIAL_STORE_REGISTERS] { [ // End-of-tile resource specifier. - RegisterWrite::new(0x15379, 0), + RegisterWrite::new(0x15379, w.partial_eot_rsrc_spec), RegisterWrite::new(0x15381, w.partial_eot_va), RegisterWrite::new(0x10039, w.tile_config), RegisterWrite::new(0x15359, UNK_15359_PARTIAL), @@ -701,7 +708,7 @@ fn partial_resume_registers( ) -> [RegisterWrite; PARTIAL_RESUME_REGISTERS] { [ // End-of-tile resource specifier. - RegisterWrite::new(0x15379, 0), + RegisterWrite::new(0x15379, w.partial_eot_rsrc_spec), RegisterWrite::new(0x15381, w.partial_eot_va), RegisterWrite::new(0x15369, w.partial_bg_rsrc_spec), RegisterWrite::new(0x15371, w.partial_bg_va), @@ -1036,12 +1043,14 @@ pub(crate) struct FragmentDescriptor { pub(crate) unk_1f50: u64, /// Tile configuration. pub(crate) tile_config: u64, - /// Ends with the end-of-tile resource specifier, which is zero. - pub(crate) unk_1f60: [u8; 0x1c], + pub(crate) unk_1f60: [u8; 0x18], + /// Low word of the end-of-tile resource specifier. + pub(crate) eot_rsrc_spec: u32, /// GPU address of the end-of-tile program. pub(crate) eot_va: u64, - /// Ends with the partial end-of-tile resource specifier, which is zero. - pub(crate) unk_1f84: [u8; 0x18], + pub(crate) unk_1f84: [u8; 0x14], + /// Low word of the partial-render end-of-tile resource specifier. + pub(crate) partial_eot_rsrc_spec: u32, /// GPU address of the partial-render end-of-tile program. pub(crate) partial_eot_va: u64, pub(crate) unk_1fa4: u32, @@ -1228,7 +1237,9 @@ impl FragmentSlot { desc.height = pass.height; desc.unk_1f50 = UNK_15049; desc.tile_config = words.tile_config; + desc.eot_rsrc_spec = words.eot_rsrc_spec as u32; desc.eot_va = words.eot_va; + desc.partial_eot_rsrc_spec = words.partial_eot_rsrc_spec as u32; desc.partial_eot_va = words.partial_eot_va; desc.depth_clear = pass.depth_clear; desc.unk_1fac = words.tib_blocks << Self::UNK_1FAC_TIB_SHIFT | Self::UNK_1FAC_BASE; diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 3183c4e94e4a8..623fd55eb5b6e 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -413,6 +413,15 @@ impl InitData { Ok(()) } + /// The sole host publisher holds the device mutex through the following + /// inner and outer publication. Firmware consumers can only free capacity; + /// neither this check nor waiting for room reserves or limits other queues. + pub(super) fn work_ready(&self, engine: super::fw::queue::DataMaster) -> Result { + let [consumer, consumer_b, producer] = self.work_cursors(engine)?; + let next = (producer + 1) % super::fw::channels::WORK_RING_SLOTS; + Ok(next != consumer && next != consumer_b) + } + fn work_publication<'a>( &'a self, record: &'a super::fw::channels::WorkSlot, @@ -1237,14 +1246,14 @@ impl super::recovery::Memory for InitData { .dword(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? .load(Ordering::Relaxed); let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { - let key = super::recovery::slot_key(slot).unwrap_or(0); + let key = super::recovery::slot_key(slot)?; let monitor = self.recovery_view_words::<6>(abi::recovery::PROGRESS_VIEW, abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE) - .map(|words| words[0]).unwrap_or(0); - (key, monitor) + .ok().map(|words| words[0])?; + Some((key, monitor)) }); sources.reason = self.recovery_view_words::<1>(abi::recovery::REPORT_VIEW, abi::recovery::REASON) - .map(|words| words[0]).unwrap_or(0); + .ok().map(|words| words[0]); Ok(sources) } @@ -1296,30 +1305,17 @@ impl super::recovery::Memory for InitData { Ok(()) } - fn entries( - &self, - out: &mut [super::recovery::Entry; super::recovery::MAX_ENTRIES], - ) -> Result { + fn entries(&self) -> Result { let cluster = self.object(self.cluster)?; let base = abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_INFO; fence(Ordering::Acquire); - let mut used = 0; + let mut entries = super::recovery::Entries::default(); for index in 0..256 { - if used == out.len() { - break; - } let offset = base + index * core::mem::size_of::(); let flags = cluster.word(offset)?.load(Ordering::Relaxed); - if flags & 1 == 0 { - continue; - } - let stamp = cluster.word(offset + 4)?.load(Ordering::Relaxed); - if let Some(entry) = super::recovery::Entry::decode(flags, stamp) { - out[used] = entry; - used += 1; - } + entries.include(flags); } - Ok(used) + Ok(entries) } fn wait_tick(&self) { diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index f470b1116f0df..649a28085a75b 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -13,7 +13,7 @@ use kernel::{ sync::Arc, }; -const TIMEOUT_MS: kernel::time::Msecs = 2000; +pub(super) const TIMEOUT_MS: kernel::time::Msecs = 2000; const RENDER_CREDITS: u32 = 32; const COMPUTE_CREDITS: u32 = 16; @@ -272,6 +272,8 @@ impl Drop for Job { } impl sched::JobImpl for Job { + const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); + fn false_timeout(job: &mut sched::Job) -> bool { if job.is_finished() || job.packet.completion.take_replay_timeout() { return true; @@ -351,6 +353,7 @@ impl Scheduler { job.add_dependency(dependency)?; } let mut job = job.arm(); + packet.completion.mark_accepted(); let ready = if host_timestamps { completed.clone() } else { diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index ad548d722864b..f98fcb5e93de8 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -56,7 +56,7 @@ use core::sync::atomic::{ use kernel::{ c_str, - device::Core, + device::{self, Core}, drm::gem::shmem, io::{ IoBase, @@ -68,6 +68,7 @@ use kernel::{ prelude::*, soc::apple::rtkit, sync::{ + aref::ARef, Arc, CondVar, Mutex, // @@ -470,6 +471,7 @@ struct Firmware { rings: channel::Rings, recovery: recovery::State, effort: power::Effort, + registration_power_release_pending: bool, queues: runtime::Registry, pool: Arc, render_ids: Arc, @@ -573,6 +575,12 @@ impl power::Control for Firmware { fn wait_tick(&self) { fsleep(Delta::from_millis(1)); } + fn registration_release_failed(&mut self) { + self.registration_power_release_pending = true; + } + fn registration_release_succeeded(&mut self) { + self.registration_power_release_pending = false; + } } /// Event work item ID. @@ -587,7 +595,12 @@ const FEED_WORK: u64 = 5; /// Device state shared with the coprocessor callbacks and the event worker. #[pin_data] struct Shared { - dev: AsahiDevRef, + /// Diagnostics may outlive shutdown without retaining the DRM data graph. + dev: ARef, + /// Allocation users take independent references before reserving resources. + /// Shutdown removes this owning backedge after joining device work. + #[pin] + drm: Mutex>, clusters: u32, descriptor_flags: [u32; 2], crashed: AtomicBool, @@ -608,7 +621,11 @@ struct Shared { /// Nests inside the device mutex; detached mappings are dropped after both unlock. #[pin] reclaim: Mutex, - /// The device lock. `None` before boot completes and after teardown. + /// Serializes detached firmware destruction with unbind. Stop paths only; + /// normal publication and RTKit/fence handlers never acquire this lock. + #[pin] + stop: Mutex<()>, + /// The device lock. `None` before boot completes and after detachment. #[pin] state: Mutex>>, #[pin] @@ -691,6 +708,17 @@ impl WorkItem for Shared { } impl Shared { + fn drm(&self) -> Result { + self.drm.lock().as_ref().cloned().ok_or(ENODEV) + } + + fn release_drm(&self) { + // The final put may call device-release callbacks. Never run those with + // the reference-slot mutex held. + let retired = self.drm.lock().take(); + drop(retired); + } + /// Queues the event worker. Callable from any context. fn queue_events(self: &Arc) { // Already queued if this fails, which is just as good. @@ -825,7 +853,8 @@ impl Gpu { let feed = feed::Feed::new()?; let shared = Arc::pin_init( pin_init!(Shared { - dev: dev.into(), + drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), + dev: dev.as_ref().as_ref().into(), clusters: id.num_clusters, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), @@ -838,6 +867,7 @@ impl Gpu { grow <- new_work!("g17::Shared::grow"), reclaim_work <- new_work!("g17::Shared::reclaim_work"), reclaim <- new_mutex!(reclaim, "g17::Shared::reclaim"), + stop <- new_mutex!((), "g17::Shared::stop"), state <- new_mutex!(None, "g17::Shared::state"), changed <- new_condvar!("g17::Shared::changed"), }), @@ -924,6 +954,7 @@ impl Gpu { rings: channel::Rings::new(), recovery: recovery::State::new(), effort: power::Effort::new(), + registration_power_release_pending: false, queues <- runtime::Registry::new(shared.clusters, shared.descriptor_flags), pool: object::Pool::new()?, render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, @@ -1001,6 +1032,7 @@ impl Gpu { self.shared.disable_work(&self.shared.feed_work); self.shared.feed.shutdown(); self.shared.drain_reclaims(); + self.shared.release_drm(); } } @@ -1105,6 +1137,7 @@ impl gpu::Gpu for Gpu { priority: u32, usc_exec_base: u64, ) -> Result> { + let dev = self.shared.drm()?; let owner = self.ids.queue.next(); let frontend = queue::Frontend::new(priority, usc_exec_base, self.geometry.user_usable_range())?; @@ -1115,7 +1148,7 @@ impl gpu::Gpu for Gpu { } }; let frontend = match queue::Queue::new( - &self.shared.dev, + &dev, backend.context.clone(), backend.clone(), frontend, diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index 8806f438234e9..96b363bc27447 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -70,6 +70,8 @@ pub(super) trait Control { fn counters(&self) -> kernel::error::Result<[u32; 3]>; fn set_idle(&mut self, enabled: bool) -> kernel::error::Result; fn wait_tick(&self); + fn registration_release_failed(&mut self) {} + fn registration_release_succeeded(&mut self) {} } fn wait(control: &C, target: u32, require_powered: bool) -> kernel::error::Result { @@ -113,13 +115,25 @@ pub(super) fn acquire(control: &mut C) -> kernel::error::Result { } Err(EAGAIN) })(); - if result.is_err() { - let _ = release(control); + if result.is_err() && release(control).is_err() { + control.registration_release_failed(); } result } pub(super) fn release(control: &mut C) -> kernel::error::Result { - let target = control.set_idle(true)?; - wait(control, target, false) + let target = match control.set_idle(true) { + Ok(target) => target, + Err(error) => { + control.registration_release_failed(); + return Err(error); + } + }; + let result = wait(control, target, false); + if result.is_ok() { + control.registration_release_succeeded(); + } else { + control.registration_release_failed(); + } + result } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 3b259b37b38f4..5ca9b85f8aeb6 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -313,7 +313,7 @@ struct Active { ticket: Option, observed: Option, failure_deferred: bool, - qos_completed: bool, + qos_pending: bool, } struct Previous { binding: Option, @@ -511,6 +511,12 @@ impl Queue { data_master: DataMaster::Compute as u8, } } + /// A completion hint can require a second visibility/recordless pass even + /// when firmware has no more events to send. Do not poll idle or failed queues. + pub(crate) fn retirement_polling(&self) -> bool { + !self.quarantined && self.retire_pending + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { self.retired_by_teardown || self.retire_pending @@ -681,9 +687,9 @@ impl Queue { match result { Ok(Some(timestamps)) => { let submitted = self.submitted.checked_sub(1).ok_or(EIO)?; - if !front.qos_completed { + if front.qos_pending { complete_qos(self.qos_owner())?; - self.active.iter_mut().next().ok_or(EIO)?.qos_completed = true; + self.active.iter_mut().next().ok_or(EIO)?.qos_pending = false; } let front = self.active.front().ok_or(EIO)?; self.retire_pending = true; @@ -783,9 +789,9 @@ impl Queue { self.retirement_proved = true; } while let Some(front) = self.active.front() { - if self.spared_quarantine && !front.qos_completed { + if front.qos_pending { complete_qos(self.qos_owner())?; - self.active.iter_mut().next().ok_or(EIO)?.qos_completed = true; + self.active.iter_mut().next().ok_or(EIO)?.qos_pending = false; } let front = self.active.front().ok_or(EIO)?; let completion = front.packet.completion.clone(); @@ -1242,6 +1248,7 @@ impl Queue { return Err(EIO); } let crate::g17::command::Validated::Compute { + scratch, usc_base, cdm_va, cdm_end_va, @@ -1275,6 +1282,7 @@ impl Queue { self.graph.ensure_record_slot(slot)?; } let args = ComputeArgs { + scratch: *scratch, usc_base: *usc_base, cdm_va: *cdm_va, cdm_end_va: *cdm_end_va, @@ -1345,7 +1353,8 @@ impl Queue { }, mcache: None, event_mask: COMPUTE_KICK_EVENT_MASK, - register_arrays: ComputeDescriptor::register_bindings(descriptor_low)?, + register_arrays: ComputeDescriptor::register_bindings(descriptor_low, *scratch)?, + compute_scratch: scratch.enabled(), }; let published = host.next_compute_publication()?; self.active @@ -1356,7 +1365,7 @@ impl Queue { ticket: None, observed: None, failure_deferred: false, - qos_completed: false, + qos_pending: false, }) .map_err(|_| EBUSY)?; self.retirement_ready = false; @@ -1411,8 +1420,10 @@ impl Queue { }, identity.scheduler, )?; + self.active.back_mut().ok_or(EIO)?.qos_pending = true; let target = self.pool.reserve_submission().or_else(|error| { host.qos_cancel(qos)?; + self.active.back_mut().ok_or(EIO)?.qos_pending = false; Err(error) })?; producer = self @@ -1421,6 +1432,7 @@ impl Queue { .or_else(|error| { self.pool.cancel_unpublished(target)?; host.qos_cancel(qos)?; + self.active.back_mut().ok_or(EIO)?.qos_pending = false; Err(error) })?; } else { @@ -1443,11 +1455,13 @@ impl Queue { }, identity.scheduler, )?; + self.active.back_mut().ok_or(EIO)?.qos_pending = true; producer = self .graph .append(producer, config_va, || {}) .or_else(|error| { host.qos_cancel(qos)?; + self.active.back_mut().ok_or(EIO)?.qos_pending = false; Err(error) })?; } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index 1ddadae2dba69..e420427d594bf 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -9,46 +9,17 @@ use kernel::{prelude::*, sync::Arc}; pub(crate) const POLL_NS: u64 = 2_500_000_000; const QUARANTINED_DELAY_NS: u64 = 250_000_000; const CANCELLED_DELAY_NS: u64 = 2_000_000_000; -const ACK_SLOTS: usize = 16; // Firmware-visible cookie namespace persists for the loaded module's lifetime. static NEXT_COOKIE: AtomicU32 = AtomicU32::new(1); -pub(crate) fn next_cookie() -> u64 { - loop { - let cookie = NEXT_COOKIE.fetch_add(1, Ordering::Relaxed); - if cookie != 0 { - return cookie.into(); - } - } -} - -pub(crate) struct Acks([u64; ACK_SLOTS]); -impl Acks { - pub(crate) fn new() -> Self { - Self([0; ACK_SLOTS]) - } - pub(crate) fn note(&mut self, cookie: u64) { - if cookie == 0 || self.contains(cookie) { - return; - } - if let Some(slot) = self.0.iter_mut().find(|slot| **slot == 0) { - *slot = cookie; - } - } - pub(crate) fn contains(&self, cookie: u64) -> bool { - cookie != 0 && self.0.contains(&cookie) - } - pub(crate) fn forget(&mut self, cookie: u64) { - for slot in &mut self.0 { - if *slot == cookie { - *slot = 0; - } - } - } - pub(crate) fn snapshot(&self) -> [u64; ACK_SLOTS] { - self.0 - } +pub(crate) fn next_cookie() -> Result { + // Never reuse an identity while a delayed firmware ACK can still exist. + // The final counter value is an exhaustion marker, not a wrapping cookie. + NEXT_COOKIE + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |next| next.checked_add(1)) + .map(u64::from) + .map_err(|_| EOVERFLOW) } pub(crate) struct State { @@ -101,6 +72,7 @@ pub(super) struct Transaction { cookie: u64, published: u64, phase: Phase, + announced: Option, } impl Queue { @@ -136,32 +108,50 @@ impl Queue { cookie, published: now, phase: Phase::Published, + announced: None, }); } + /// A failed doorbell does not undo the published kill or consume its retry. + /// The caller retries under the device mutex; only the original record exists. + pub(crate) fn announce_exit(&mut self, now: u64, notify: impl FnOnce() -> Result) -> Result { + let Some(exit) = self.exit.as_mut() else { + return Ok(()); + }; + if exit.phase != Phase::Published || exit.announced.is_some() { + return Ok(()); + } + notify()?; + exit.announced = Some(now); + Ok(()) + } pub(crate) fn exit_polling(&self, now: u64) -> bool { self.exit.as_ref().is_some_and(|exit| { - exit.phase == Phase::Published && now.saturating_sub(exit.published) < POLL_NS + exit.phase == Phase::Published + && now.saturating_sub(exit.announced.unwrap_or(exit.published)) < POLL_NS }) } - pub(crate) fn exit_cookie(&self) -> Option { - self.exit - .as_ref() - .filter(|exit| matches!(exit.phase, Phase::Published | Phase::Acknowledged)) - .map(|exit| exit.cookie) + /// Event consumption and record_exit are serialized by the device mutex. + /// A kill pins this exact queue transaction through its final witness; + /// duplicate, unknown and terminal cookies cannot acknowledge another owner. + pub(crate) fn acknowledge_exit(&mut self, cookie: u64) -> bool { + let Some(exit) = self.exit.as_mut() else { + return false; + }; + if cookie == 0 || exit.cookie != cookie || exit.phase != Phase::Published { + return false; + } + exit.phase = Phase::Acknowledged; + true } pub(crate) fn settle_exit( &mut self, - acks: &Acks, recovery_closed: bool, + complete_qos: &mut impl FnMut(crate::g17::qos::Owner) -> Result, defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { - let Some(cookie) = self.exit_cookie() else { - return Ok(()); - }; - if !acks.contains(cookie) { + if !self.exit.as_ref().is_some_and(|exit| exit.phase == Phase::Acknowledged) { return Ok(()); } - self.exit.as_mut().ok_or(EIO)?.phase = Phase::Acknowledged; if self.active.len != 0 { if !recovery_closed { return Ok(()); @@ -185,6 +175,11 @@ impl Queue { } } while let Some(active) = self.active.front() { + if active.qos_pending { + complete_qos(self.qos_owner())?; + self.active.iter_mut().next().ok_or(EIO)?.qos_pending = false; + } + let active = self.active.front().ok_or(EIO)?; defer(Deferred::Retired( active.packet.completion.clone(), Err(if self.spared_deferred { @@ -202,6 +197,7 @@ impl Queue { self.spared_quarantine = false; self.retire_pending = false; self.retirement_ready = false; + self.update_slot_accounting(); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 1634854f17ea4..d382381868555 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -8,7 +8,7 @@ pub(crate) mod render; use super::{ command::{Parser, UscWindow, Validated}, - completion::Completion, + completion::{Completion, Destinations}, context::Context, fence::{Outputs, Submission}, job::{self, Backend, Engine, Fences, Order, Packet, Scheduler}, @@ -21,7 +21,7 @@ use crate::{ use core::ops::Range; use kernel::{ c_str, - dma_fence::{FenceContexts, RawDmaFence}, + dma_fence::{Fence, FenceContexts, RawDmaFence}, prelude::*, sync::{Arc, LockClassKey}, xarray, @@ -53,6 +53,7 @@ impl Identities { /// Validated frontend state prepared before allocating a logical execution context. pub(crate) struct Frontend { fences: FenceContexts, + allocation_failure: Fence, range: Range, window: UscWindow, } @@ -64,8 +65,10 @@ impl Frontend { } let window = UscWindow::new(usc_base, range.clone())?; let fences = FenceContexts::new(1, c_str!("asahi_g17_queue"), FENCE_KEY)?; + let allocation_failure = super::fence::allocation_failure(&fences)?; Ok(Self { fences, + allocation_failure, range, window, }) @@ -79,11 +82,13 @@ pub(crate) struct Queue { render: Option>, compute: Option>, fences: FenceContexts, + allocation_failure: Fence, range: Range, window: UscWindow, proof: Proof, sequence: [u64; 2], frontiers: [KVec<(u64, Fences)>; 2], + failures: [Option<(u64, Fence)>; 2], // Closing the userspace queue releases admission even if firmware still // retains one of its execution contexts or installed physical graphs. _slot: admission::QueueSlot, @@ -99,6 +104,7 @@ impl Queue { ) -> Result { let Frontend { fences, + allocation_failure, range, window, } = frontend; @@ -110,11 +116,13 @@ impl Queue { render, compute: None, fences, + allocation_failure, range, window, proof: Proof::new(), sequence: [0; 2], frontiers: core::array::from_fn(|_| KVec::new()), + failures: [None, None], _slot: slot, }) } @@ -132,6 +140,22 @@ impl Queue { Ok(()) } + fn prune_frontier(&mut self, index: usize) { + let failure = &mut self.failures[index]; + self.frontiers[index].retain(|(sequence, fences)| { + let status = job::fence_status(&fences.completed); + if status == 0 { + return true; + } + // One earliest failed ordinal covers every later prefix without + // retaining its packet, mappings, or every failed fence. + if status < 0 && failure.as_ref().is_none_or(|(first, _)| *sequence < *first) { + *failure = Some((*sequence, fences.completed.clone())); + } + false + }); + } + fn enqueue( &mut self, id: u64, @@ -140,62 +164,30 @@ impl Queue { guard: mmu::VmJobGuard, aggregate: &Arc, inputs: &[file::SyncItem], - objects: Pin<&xarray::XArray>>, + destinations: Destinations, ) -> Result { let engine = match &command { Validated::Render { .. } => Engine::Render, Validated::Compute { .. } => Engine::Compute, }; let index = engine.index(); - // One earliest failed timestamp covers every prefix containing a later - // failure, without retaining the producer's packet or mappings. - let mut retained_timestamp_failure = false; - self.frontiers[index].retain(|(_, fences)| { - let status = job::fence_status(&fences.completed); - if status == 0 { - return true; - } - if engine == Engine::Compute - && status < 0 - && fences.ready.raw() == fences.completed.raw() - && !retained_timestamp_failure - { - retained_timestamp_failure = true; - return true; - } - false - }); + self.prune_frontier(index); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { self.ensure_compute()?; } - let completion = match &command { - Validated::Render { timestamps, .. } => Completion::new( - &self.dev, - &self.fences, - &self.context, - aggregate.member(), - guard, - objects, - timestamps, - false, - self.backend.feed(), - )?, - Validated::Compute { timestamps, .. } => Completion::new( - &self.dev, - &self.fences, - &self.context, - aggregate.member(), - guard, - objects, - core::slice::from_ref(timestamps), - true, - None, - )?, - }; + let completion = Completion::new( + &self.dev, + &self.fences, + &self.context, + aggregate.member(), + guard, + destinations, + engine == Engine::Compute, + if engine == Engine::Render { self.backend.feed() } else { None }, + )?; if engine == Engine::Compute { - self.frontiers[Engine::Render.index()] - .retain(|(_, fences)| job::fence_status(&fences.completed) == 0); + self.prune_frontier(Engine::Render.index()); } let other = 1 - index; let prefix = order.wait_through[other]; @@ -222,10 +214,21 @@ impl Queue { .iter() .filter(|(_, fences)| fences.ready.raw() == fences.completed.raw()) .count(); + let failures: [Option<&Fence>; 2] = core::array::from_fn(|index| { + self.failures[index].as_ref().and_then(|(sequence, fence)| { + Order::contains(order.wait_through[index], *sequence).then_some(fence) + }) + }); + let failure_count = failures.iter().flatten().count(); let mut dependencies = - KVec::with_capacity(inputs.len() + count + timestamp_count, GFP_KERNEL)?; - let mut checked_inputs = KVec::with_capacity(inputs.len() + timestamp_count, GFP_KERNEL)?; + KVec::with_capacity(inputs.len() + count + timestamp_count + failure_count, GFP_KERNEL)?; + let mut checked_inputs = + KVec::with_capacity(inputs.len() + timestamp_count + failure_count, GFP_KERNEL)?; let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; + for fence in failures.into_iter().flatten() { + checked_inputs.push(fence.clone(), GFP_KERNEL)?; + dependencies.push(fence.clone(), GFP_KERNEL)?; + } // Same-engine parents order GPU execution, not the host's timestamp // writes. Only timestamp publishers need these completion waits and // status checks; ordinary compute keeps its existing queue ordering. @@ -285,7 +288,8 @@ impl Queue { fn prepare_submission( &mut self, bytes: &[u8], - ) -> Result<(KVec<(Validated, Order)>, KVec)> { + objects: Pin<&xarray::XArray>>, + ) -> Result<(KVec<(Validated, Order, Destinations)>, KVec)> { let parse_guard = self.context.vm().retain_first_job()?; let mut parser = Parser::new(bytes); // All historical ordinals use the same entry counters; publication @@ -312,7 +316,13 @@ impl Queue { engine, )?; preceding[engine.index()] += 1; - pending.push((payload, order), GFP_KERNEL)?; + let destinations = match &payload { + Validated::Render { timestamps, .. } => Destinations::resolve(objects, timestamps)?, + Validated::Compute { timestamps, .. } => { + Destinations::resolve(objects, core::slice::from_ref(timestamps))? + } + }; + pending.push((payload, order, destinations), GFP_KERNEL)?; } parser.finish()?; // Every accepted command pins the VM before any command is enqueued. @@ -339,15 +349,20 @@ impl crate::queue::Queue for Queue { bytes: &[u8], objects: Pin<&xarray::XArray>>, ) -> Result { - let mut outputs = Outputs::new(&self.fences, syncs, input_count)?; + let mut outputs = Outputs::new( + &self.fences, + syncs, + input_count, + &self.allocation_failure, + )?; let result = (|| { if self.context.status().get() != 0 { return Err(EIO); } - let (pending, guards) = self.prepare_submission(bytes)?; + let (pending, guards) = self.prepare_submission(bytes, objects)?; let mut guards = guards.into_iter(); let aggregate = outputs.publish(self.context.clone())?; - for (command, order) in pending { + for (command, order, destinations) in pending { let guard = guards.next().ok_or(EIO)?; self.enqueue( id, @@ -356,7 +371,7 @@ impl crate::queue::Queue for Queue { guard, &aggregate, outputs.inputs(), - objects, + destinations, )?; } Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index c3ac091333c27..fd67523ab91c3 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -512,6 +512,7 @@ impl Pair { mcache, event_mask: abi::RENDER_KICK_EVENT_MASK, register_arrays: arrays, + compute_scratch: false, }; let ta_args = args( 0, @@ -789,7 +790,7 @@ impl Pair { } pub(crate) fn needs_drain(&self) -> bool { - self.quarantined && self.in_flight() + self.quarantined } pub(crate) fn oldest_spared(&self) -> Option { diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index f614494e1216a..859ff61901570 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -47,12 +47,14 @@ impl Aliases { let mut range = dynamic::LOWER; let size = (source.end - source.start) as u64; loop { + // Descriptor aliases are read-only to GPU consumers; the canonical + // firmware mapping remains writable for command state updates. let global = object.map_range( alloc.uat.kernel_lower_vm(), source.clone(), range.clone(), mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, + mmu::PROT_GPU_SHARED_RO, 0, )?; let address = global.iova(); @@ -62,7 +64,7 @@ impl Aliases { source.clone(), address..end, mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, + mmu::PROT_GPU_SHARED_RO, 0, ) { Ok(client) => { diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 7e3db2f6dae78..ba368341d84d2 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -22,29 +22,30 @@ use kernel::{prelude::*, sync::Arc}; const QIDS: u8 = 128; const GUILTY: u8 = 1 << 1; const CANCELLED: u8 = 1 << 2; -/// Retained valid entries, in firmware table order. -pub(super) const MAX_ENTRIES: usize = 64; const MAX_GUILTY_VMS: usize = 8; const RECOVERY_POLLS: usize = 500; -/// One valid entry from the firmware's 256-entry recovery table. -#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] -pub(super) struct Entry { - pub(super) qid: u8, - pub(super) flags: u8, - pub(super) stamp: u32, +/// Per-QID classifications from the complete firmware recovery table. +/// Multiple records for a queue contribute all of their classification bits. +#[derive(Default)] +pub(super) struct Entries { + guilty: u128, + cancelled: u128, } -impl Entry { - pub(super) const fn decode(flags: u32, stamp: u32) -> Option { +impl Entries { + pub(super) fn include(&mut self, flags: u32) { if flags & 1 == 0 { - return None; + return; + } + let qid = (flags >> 4) & 0x7f; + let bit = 1u128 << qid; + if flags & u32::from(GUILTY) != 0 { + self.guilty |= bit; + } + if flags & u32::from(CANCELLED) != 0 { + self.cancelled |= bit; } - Some(Self { - qid: ((flags >> 4) & 0x7f) as u8, - flags: (flags & 0xf) as u8, - stamp: stamp >> 8, - }) } } @@ -83,7 +84,7 @@ pub(super) trait Memory { fn fault_sources(&self) -> Result; fn transition(&self, expected: u32, next: u32) -> Result; fn clear_timestamps(&self) -> Result; - fn entries(&self, out: &mut [Entry; MAX_ENTRIES]) -> Result; + fn entries(&self) -> Result; /// One millisecond wait; this is polled because the firmware sends no state-3 notification. fn wait_tick(&self); } @@ -163,10 +164,31 @@ pub(super) fn service( crashed: impl Fn() -> bool, ) -> Result { let request = host.recovery().pending.ok_or(EINVAL)?; - if host.recovery_state()? != 1 - || request.generation != host.recovery().accepted - || host.host_recovery()? != 0 - { + if request.generation != host.recovery().accepted || host.host_recovery()? != 0 { + return Err(EIO); + } + // The restart record is published before firmware announces state 1. + // Its ring cursor is already consumed; wait only for that readiness word, + // without advancing the host generation or acknowledging an early event. + let mut ready = false; + for _ in 0..RECOVERY_POLLS { + if crashed() { + return Err(EIO); + } + match host.recovery_state()? { + 1 => { + ready = true; + break; + } + 0 => host.wait_tick(), + _ => return Err(EIO), + } + } + if !ready { + return Err(ETIMEDOUT); + } + // Recheck ownership after the off-CPU waits, before any handshake mutation. + if host.host_recovery()? != 0 { return Err(EIO); } host.recovery_mut().accepted = host.recovery().accepted.wrapping_add(1); @@ -190,16 +212,15 @@ pub(super) fn service( if !recovered { return Err(ETIMEDOUT); } - let mut entries = [Entry::default(); MAX_ENTRIES]; // A failed table read leaves only the explicit blamed QID as evidence of guilt. - let count = host.entries(&mut entries).unwrap_or(0).min(MAX_ENTRIES); + let entries = host.entries().unwrap_or_default(); host.transition(3, 0)?; if host.recovery_state()? != 0 { return Err(EIO); } host.recovery_mut().pending = None; - let cancelled = classify(host, &entries[..count], request.blamed, &mut sources); + let cancelled = classify(host, entries, request.blamed, &mut sources); if let Some(qid) = request.blamed { host.settle_blamed_render(qid, EIO); } @@ -214,17 +235,9 @@ pub(super) fn service( /// Resolves guilty QIDs before changing any completion's classification. Unknown ownership or /// too many distinct guilty VMs leaves all prior classifications intact. The oldest spared work /// of a QID makes later blame collateral to its previous recovery, rather than new VM guilt. -fn classify(queues: &mut Q, entries: &[Entry], blamed: Option, sources: &mut Sources) -> u128 { - let mut guilty_qids = blamed.map_or(0, |qid| 1u128 << qid); - let mut cancelled = 0u128; - for entry in entries { - if entry.flags & GUILTY != 0 { - guilty_qids |= 1u128 << entry.qid; - } - if entry.flags & CANCELLED != 0 { - cancelled |= 1u128 << entry.qid; - } - } +fn classify(queues: &mut Q, entries: Entries, blamed: Option, sources: &mut Sources) -> u128 { + let mut guilty_qids = entries.guilty | blamed.map_or(0, |qid| 1u128 << qid); + let cancelled = entries.cancelled; let attribution = sources.attribute(blamed, |qid, stamp| queues.render_pass_started(qid, stamp)); match attribution { Attribution::Firmware => {} diff --git a/drivers/gpu/drm/asahi/g17/recovery/attribution.rs b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs index b0c42d4e3f7c1..1755484219187 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/attribution.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs @@ -16,7 +16,7 @@ pub(super) struct Source { } pub(crate) struct Sources { - pub(crate) reason: u32, + pub(crate) reason: Option, mask: u32, records: [Source; SLOTS], count: usize, @@ -31,10 +31,10 @@ pub(super) enum Attribution { impl Sources { /// Firmware is halted for every selected key and progress-record read. - /// Missing keys stay unknown; they cannot establish execution by an owner. - pub(crate) fn sample(mask: u32, mut read: impl FnMut(usize) -> (u64, u32)) -> Self { + /// Failed reads stay unknown; incomplete evidence cannot override firmware blame. + pub(crate) fn sample(mask: u32, mut read: impl FnMut(usize) -> Option<(u64, u32)>) -> Self { let mut sources = Self { - reason: 0, + reason: None, mask, records: [Source::default(); SLOTS], count: 0, @@ -43,7 +43,9 @@ impl Sources { if mask & (1 << slot) == 0 { continue; } - let (key, monitor) = read(slot); + let Some((key, monitor)) = read(slot) else { + continue; + }; let qid = (key >> 40) & 0xff; sources.records[sources.count] = Source { slot: slot as u8, @@ -62,12 +64,15 @@ impl Sources { blamed: Option, mut started: impl FnMut(u8, u64) -> Option, ) -> Attribution { + if self.reason.is_none() || self.count as u32 != self.mask.count_ones() { + return Attribution::Firmware; + } for source in &mut self.records[..self.count] { if let Some(qid) = source.qid { source.started = started(qid, source.stamp); } } - if self.reason == PAGE_FAULT || self.mask.count_ones() < 2 { + if self.reason == Some(PAGE_FAULT) || self.mask.count_ones() < 2 { return Attribution::Firmware; } let Some(blamed) = blamed else { diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 294d66b91f8dc..805087d7082ab 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -29,6 +29,7 @@ use kernel::{prelude::*, sync::Arc}; const QIDS: usize = 128; const FIRST_COMPUTE_QID: u8 = 4; const RECOVERY_SETTLE_POLLS: usize = 200; +const WORK_RING_WAIT_NS: u64 = super::job::TIMEOUT_MS as u64 * 1_000_000; /// A reservation remains identifiable while its backing is built without the /// device mutex. Failed construction returns the slot, never its host identity. @@ -56,10 +57,10 @@ pub(super) struct Registry { compute: KVec>, pub(super) compute_building: bool, compute_scheduler_building: bool, + replay_outer_wait: Option, pub(super) observations: compute::retirement::Batch, worker_deferred: Option, teardown: super::teardown::Pending, - exit_acks: compute::exit::Acks, } impl Registry { @@ -91,10 +92,10 @@ impl Registry { }, compute_building: false, compute_scheduler_building: false, + replay_outer_wait: None, observations: compute::retirement::Batch::new()?, worker_deferred: Some(DeferredBatch::worker()?), teardown: super::teardown::Pending::new()?, - exit_acks: compute::exit::Acks::new(), }) } @@ -235,6 +236,7 @@ impl Backend { owner: u64, ) -> Result<(Arc, admission::QueueSlot)> { let lease = shared.preparations.enter(true)?; + let dev = shared.drm()?; let (uat, pool, qos, execution, slot) = { let mut state = shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -252,7 +254,7 @@ impl Backend { ) }; let alloc = super::object::Allocator { - dev: &shared.dev, + dev: &dev, uat: &uat, }; let owner_pid = @@ -285,6 +287,7 @@ impl Backend { fn shared_compute(&self) -> Result { loop { let lease = self.shared.preparations.enter(false)?; + let dev = self.shared.drm()?; let uat = { let mut state = self.shared.state.lock(); loop { @@ -310,7 +313,7 @@ impl Backend { } }; let alloc = super::object::Allocator { - dev: &self.shared.dev, + dev: &dev, uat: &uat, }; let mut built = Some(super::initdata::ComputeShared::new(&alloc)); @@ -406,6 +409,7 @@ impl Backend { } loop { let lease = self.shared.preparations.enter(false)?; + let dev = self.shared.drm()?; let (uat, reservation) = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -423,7 +427,7 @@ impl Backend { (firmware.uat.clone(), reservation) }; let alloc = super::object::Allocator { - dev: &self.shared.dev, + dev: &dev, uat: &uat, }; let built = (|| { @@ -527,7 +531,19 @@ impl DeferredBatch { completion.status().record(*error); } } - self.values.push_within_capacity(value).map_err(|_| EIO) + self.values.push_within_capacity(value).map_err(|_| EIO)?; + // A later publisher can borrow this pair before finish signals its old + // fence. Publish the exact completion's witness only after collection + // succeeds; failure fences and rejected events are not retirement. + if let Some( + Deferred::Retired(completion, _) + | Deferred::Drained(completion, _) + | Deferred::Release(completion), + ) = self.values.last() + { + completion.note_retirement_queued(); + } + Ok(()) } /// Called only after releasing the device mutex. pub(super) fn finish(&mut self) { @@ -596,6 +612,11 @@ impl super::Firmware { if self.recovery.pending() { return Err(EAGAIN); } + // No descriptor, accounting or inner producer changes precede this + // capacity check. The same device lock covers queue.publish below. + if !self.init.work_ready(DataMaster::Compute)? { + return Err(EAGAIN); + } let queues = &mut self.queues; let queue = queues .compute @@ -751,13 +772,15 @@ impl Backend { deferred.finish(); let pending = pending?; if !pending { - break; + return self.build_compute(); } // Other owners continue submitting while this owner gives its // queue-local drain witness time to arrive. fsleep(Delta::from_millis(1)); } - self.build_compute() + // No new command has been accepted. Keep the spared queue and its + // retained work intact until a later admission observes its witness. + Err(EBUSY) } /// Called after taking this packet's publication claim. All waits and work @@ -769,6 +792,7 @@ impl Backend { ) -> Result { packet.check_dependencies()?; let mut recovery_polls = 0; + let mut outer_wait = None; loop { if packet.completion.status().get() != 0 { return Err(EIO); @@ -826,6 +850,20 @@ impl Backend { kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); continue; } + if matches!(result, Err(EAGAIN)) && !packet.is_published() { + let now = now_ns(); + let since = *outer_wait.get_or_insert(now); + if now.saturating_sub(since) >= WORK_RING_WAIT_NS { + return Err(EBUSY); + } + // Backpressure owns no firmware-visible attempt. Other queues + // can submit and the event worker can retire while we sleep. + drop(state); + drop(lease); + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + continue; + } return result; } } @@ -863,12 +901,15 @@ impl super::Firmware { } impl super::Shared { - /// Joins construction and serializes the stop writes with detachment. - /// After installation, seeing no owner proves that an earlier stop caller - /// has already stopped both processors. Callbacks and destruction follow - /// outside the device mutex. + /// Joins construction and complete host teardown, including RTKit drops. + /// The stop-only mutex outlives detached firmware destruction, so unbind + /// cannot return while another stop caller still owns mailbox clients. + /// Completion callbacks remain outside the device mutex. Direct handlers + /// do not synchronously request or join stop; this path does not join + /// scheduler workers. pub(super) fn stop_queues(&self) { self.preparations.wait_drained(); + let _stopping = self.stop.lock(); let firmware = { let mut state = self.state.lock(); if let Some(firmware) = (*state).as_deref_mut() { @@ -879,6 +920,8 @@ impl super::Shared { }; if let Some(mut firmware) = firmware { firmware.finish_stopped_queues(); + // RTKit destruction still dereferences the mailbox provider. + drop(firmware); } } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs index 0643da70d72ca..439d5c9831cbb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs @@ -14,33 +14,44 @@ use crate::g17::{ use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; +impl super::Registry { + /// ACK storage belongs to the already-retained transaction, so a full event + /// batch cannot overflow a separate cache. No allocation or wait is needed. + pub(super) fn acknowledge_compute_exit(&mut self, cookie: u64) { + for entry in self.compute.iter_mut().flatten() { + if let Some(queue) = entry.queue.as_deref_mut() { + if queue.acknowledge_exit(cookie) { + return; + } + } + } + } +} + impl Firmware { pub(in crate::g17) fn service_compute_exit(&mut self, deferred: &mut DeferredBatch) -> Result { let recovery_closed = self.init.recovery_state()? == 0; + let view = crate::g17::qos::View::new(self.init.qos()?)?; for entry in self.queues.compute.iter_mut().flatten() { if let Some(queue) = entry.queue.as_deref_mut() { - queue.settle_exit(&self.queues.exit_acks, recovery_closed, &mut |event| { - deferred.push(event) - })?; - } - } - for cookie in self.queues.exit_acks.snapshot() { - if cookie != 0 - && !self - .queues - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.exit_cookie() == Some(cookie)) - { - self.queues.exit_acks.forget(cookie); + queue.settle_exit( + recovery_closed, + &mut |owner| self.queues.accounting.complete(&view, &[owner]), + &mut |event| deferred.push(event), + )?; } } if self.recovery.pending() || !recovery_closed { return Ok(()); } let now = now_ns(); + // Publication has already retained each kill. Retry only the doorbell; + // failed attempts do not extend its bounded polling window. + for entry in self.queues.compute.iter_mut().flatten() { + if let Some(queue) = entry.queue.as_deref_mut() { + let _ = queue.announce_exit(now, || self.primary.notify(MSG_CONTROL_NOTIFY)); + } + } for index in 0..self.queues.teardown.len() { let Some((context, age)) = self.queues.teardown.closed(index, now) else { continue; @@ -103,6 +114,7 @@ impl Firmware { hold |= 1u64 << slot; } } + let cookie = exit::next_cookie()?; if hold != 0 && !self .queues @@ -111,7 +123,6 @@ impl Firmware { { return Ok(()); } - let cookie = exit::next_cookie(); let record = ControlRecord::ContextKill(ContextKill::new(context.scheduler_va(), cookie)); if let Err(error) = Rings::publish_control(&self.init, &record) { self.queues.admission.release_exit_kill_hold(hold); @@ -119,13 +130,13 @@ impl Firmware { } // Publication is irrevocable even if its doorbell fails. Keep both // the killed graph and the held render identities until processor stop. - self.queues.compute[index] + let queue = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) - .ok_or(EIO)? - .record_exit(cookie, now); + .ok_or(EIO)?; + queue.record_exit(cookie, now); fence(Ordering::SeqCst); - if let Err(error) = self.primary.notify(MSG_CONTROL_NOTIFY) { + if let Err(error) = queue.announce_exit(now, || self.primary.notify(MSG_CONTROL_NOTIFY)) { dev_warn!( self.primary.state.shared.dev.as_ref(), "Scheduler {} exit announcement failed: {:?}\n", diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index c872a7c8dacc7..65b00d36714bb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -27,6 +27,7 @@ impl Firmware { .map(|queue| queue.replay_commands()) .sum::(); if commands == 0 { + self.queues.replay_outer_wait = None; return Ok(()); } deferred.reserve_replays(commands)?; @@ -55,8 +56,24 @@ impl Firmware { break; }; let owner = queue.owner(); + let outer_ready = self.init.work_ready(crate::g17::fw::queue::DataMaster::Compute)?; + if !outer_ready { + let now = super::now_ns(); + let since = *self.queues.replay_outer_wait.get_or_insert(now); + if now.saturating_sub(since) < super::WORK_RING_WAIT_NS { + // Keep the FIFO and its replay credit untouched. The + // event worker schedules the existing delayed poll. + return Ok(()); + } + } else { + self.queues.replay_outer_wait = None; + } - let result = (|| { + let result = if !outer_ready { + // This replay has no live firmware ownership. Persistent + // backpressure uses the existing failed-replay retirement. + Err(EBUSY) + } else { (|| { self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) @@ -66,7 +83,7 @@ impl Firmware { packet.check_dependencies()?; let dependencies = self.queues.render_prefix(owner, &packet)?; self.publish_compute(owner, packet.clone(), dependencies.as_slice(), deferred) - })(); + })() }; let recovery_wait = matches!(result, Err(EAGAIN) | Err(EBUSY)) && (self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0); @@ -89,6 +106,7 @@ impl Firmware { queue.finish_replay(&packet)?; } } + self.queues.replay_outer_wait = None; Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index f249e3520e0dd..08c11eb98790e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -64,7 +64,7 @@ impl event::Host for Service<'_> { impl event::Queues for Service<'_> { fn context_killed(&mut self, cookie: u64) { - self.firmware.queues.exit_acks.note(cookie); + self.firmware.queues.acknowledge_compute_exit(cookie); } fn flush_tvb_reply(&mut self) -> Result { self.firmware.flush_tvb_reply() @@ -125,8 +125,8 @@ impl recovery::Memory for Service<'_> { fn clear_timestamps(&self) -> Result { self.firmware.init.clear_timestamps() } - fn entries(&self, out: &mut [recovery::Entry; recovery::MAX_ENTRIES]) -> Result { - self.firmware.init.entries(out) + fn entries(&self) -> Result { + self.firmware.init.entries() } fn wait_tick(&self) { self.firmware.init.wait_tick(); @@ -395,8 +395,16 @@ impl crate::g17::Shared { } let backpressured = firmware.render_control_backpressured(); let polling = backpressured + || firmware.queues.replay_outer_wait.is_some() || firmware.queues.teardown.polling(now) - || firmware.compute_exit_polling(now); + || firmware.compute_exit_polling(now) + || firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.retirement_polling()); if polling { self.queue_poll(); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index a6aa740ecf4e3..e466571bc88c7 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -239,6 +239,17 @@ impl Registry { Ok(()) } + /// Only an optional sibling's unpublished QID-capacity refusal is soft. + /// ensure_render_ids cancels a first ID if the second cannot be reserved; + /// validation, identity overflow, and cancellation errors still propagate. + fn prepare_render_ids(&mut self, slot: u8, owner: u64, optional: bool) -> Result { + match self.ensure_render_ids(slot, owner) { + Ok(()) => Ok(true), + Err(error) if optional && error == ENOSPC => Ok(false), + Err(error) => Err(error), + } + } + fn reserve_render(&mut self, slot: u8, owner: u64) -> Result { self.ensure_render_ids(slot, owner)?; let entry = self.render.entry(slot)?; @@ -318,7 +329,7 @@ impl Registry { } if let Some(pair) = entry.pair.as_mut() { pair.quarantine(error, &mut |event| deferred.push(event))?; - } else if entry.preparing.is_some() { + } else if entry.preparing.is_some() || entry.growing { entry.pending_error = Some((status.clone(), error)); } } @@ -332,7 +343,7 @@ struct Host<'a> { } impl Host<'_> { fn release_registration(&mut self) { - if !core::mem::take(&mut self.registration_powered) { + if !self.registration_powered && !self.firmware.registration_power_release_pending { return; } if let Err(error) = self.firmware.release_registration_power() { @@ -341,6 +352,8 @@ impl Host<'_> { "Could not restore render idle policy: {:?}\n", error ); + } else { + self.registration_powered = false; } } } @@ -647,7 +660,10 @@ impl Registry { let active = entry .pair .as_ref() - .map_or(entry.borrowed_active, |pair| pair.published_in_flight()); + .map_or( + (entry.preparing.is_some() || entry.growing) && entry.borrowed_active, + |pair| pair.published_in_flight(), + ); active && entry.reservation.ids.into_iter().all(|id| { let qid = usize::from(id.qid()); @@ -660,6 +676,12 @@ impl Registry { packet: &Arc, error: Error, ) -> bool { + // Its tracker may already be gone while off-lock completion still + // owns the fence and admission lease. A loan for a subsequent packet + // cannot turn this exact accepted retirement into a missing-owner error. + if packet.completion.retirement_queued() { + return true; + } let Some(slot) = packet.completion.render_slot() else { return false; }; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index db5820cfb6663..b283fb8e7d55c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -194,6 +194,7 @@ impl crate::g17::Shared { pub(in crate::g17) fn grow_render(&self) -> Result { loop { let lease = self.preparations.enter(false)?; + let dev = self.drm()?; let (request, slot, mut pair) = { let mut state = self.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -246,7 +247,7 @@ impl crate::g17::Shared { Err(EFAULT) } else { manager.prepare_growth( - &self.dev, + &dev, &vm, crate::g17::buffer::firmware_growth_target(before), None, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index e8fc30560a890..69f8249e0818d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -60,15 +60,18 @@ enum Build { } impl super::super::Backend { - fn build_render_pair(&self, slot: u8, first: &Arc) -> Result { + fn build_render_pair(&self, slot: u8, first: &Arc, optional: bool) -> Result { let lease = self.shared.preparations.enter(false)?; + let dev = self.shared.drm()?; let (resources, build) = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); } - firmware.queues.ensure_render_ids(slot, self.owner)?; + if !firmware.queues.prepare_render_ids(slot, self.owner, optional)? { + return Ok(()); + } let resources = BuildResources::new(firmware)?; let build = if let Some(entry) = firmware.queues.render.entries[usize::from(slot)].as_ref() { @@ -159,7 +162,7 @@ impl super::super::Backend { (resources, build) }; let alloc = Allocator { - dev: &self.shared.dev, + dev: &dev, uat: &resources.uat, }; let fresh_graph = match &build { @@ -458,7 +461,7 @@ impl super::super::Backend { } pub(super) fn ensure_render(&self, packet: &Arc, slot: u8) -> Result { - self.build_render_pair(slot, packet)?; + self.build_render_pair(slot, packet, false)?; let sibling = { let state = self.shared.state.lock(); (*state) @@ -469,7 +472,12 @@ impl super::super::Backend { .warmable_sibling(self.owner, slot) }; if let Some(sibling) = sibling { - self.build_render_pair(sibling, packet)?; + // An optional sibling may be borrowed or waiting for recovery. + // The selected pair is revalidated by prepare_render before publication. + match self.build_render_pair(sibling, packet, true) { + Err(error) if error == EBUSY || error == EAGAIN => {} + result => result?, + } } Ok(()) } @@ -481,6 +489,7 @@ impl super::super::Backend { deferred: &mut super::super::DeferredBatch, ) -> Result { let lease = self.shared.preparations.enter(false)?; + let dev = self.shared.drm()?; let (uat, mut pair, addresses, growth_target) = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -535,7 +544,7 @@ impl super::super::Backend { (firmware.uat.clone(), pair, addresses, growth_target) }; let alloc = Allocator { - dev: &self.shared.dev, + dev: &dev, uat: &uat, }; let mut growth = None; @@ -543,7 +552,7 @@ impl super::super::Backend { if let Some((target, descriptor)) = growth_target { let vm = pair.context().vm().clone(); growth = pair.manager().prepare_growth( - &self.shared.dev, + &dev, &vm, target, Some(descriptor), diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 0586dfdc0944f..59a54b1926375 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -260,6 +260,13 @@ impl crate::g17::Firmware { if packet.completion.status().get() != 0 { return Err(EIO); } + // Both outer destinations must have room before pool accounting or + // either inner queue becomes visible. All host producers share this lock. + if !self.init.work_ready(crate::g17::fw::queue::DataMaster::Fragment)? + || !self.init.work_ready(crate::g17::fw::queue::DataMaster::Tiling)? + { + return Err(EBUSY); + } let mut dependencies = [KickDependency::ZERO; 3]; let count = self .queues diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 06465629eb2be..b1495dbf3e23e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -30,7 +30,10 @@ impl Reclaim { pub(in crate::g17) fn finish(self) { match self { Self::Binding(binding) => drop(binding), - Self::Context(context) => drop(context), + Self::Context(context) => { + context.release_execution(); + drop(context); + } } } } @@ -187,7 +190,7 @@ impl Firmware { primary: &mut self.primary, reclaim, }; - teardown.publish_ready(&mut service, recovery)?; + teardown.publish_ready(&mut service, recovery, super::now_ns())?; let consumer = self.init.control_consumer()?.load(Ordering::Relaxed); let producer = self.init.control_producer()?.load(Ordering::Relaxed); while teardown diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index 9acc01fbc2125..f53c2d45c129a 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -23,6 +23,7 @@ struct Entry { phase: Phase, armed: u64, warned: bool, + announce_pending: bool, } /// Publication and announcement errors retain the entry. A stalled release is reported once; @@ -88,6 +89,7 @@ impl Pending { phase, armed: now, warned: false, + announce_pending: false, }) .map_err(|_| ENOSPC) } @@ -130,9 +132,21 @@ impl Pending { /// Publish in insertion order. Mandatory control replies and a full control ring stop this /// pass; an error for one page leaves it retained while other ready releases may progress. - pub(crate) fn publish_ready(&mut self, host: &mut impl Host, recovery: bool) -> Result { + pub(crate) fn publish_ready(&mut self, host: &mut impl Host, recovery: bool, now: u64) -> Result { if !recovery { for entry in self.entries.iter_mut() { + // The ring record is already published. Retry only its doorbell; + // never duplicate a scheduler release after a transport error. + if matches!(entry.phase, Phase::Released(_)) { + if entry.announce_pending && host.notify().is_ok() { + entry.announce_pending = false; + // One successful retry starts its consumption window. + // Failed retries never slide the polling deadline. + entry.armed = now; + entry.warned = false; + } + continue; + } if entry.phase != Phase::Draining || !host.render_idle(&entry.context)? || !host.flists_released(&entry.context) @@ -160,9 +174,15 @@ impl Pending { } }; entry.phase = Phase::Released(next); + // A long command drain must not consume the acknowledgement + // polling window before this release exists on the control ring. + entry.armed = now; + entry.warned = false; + entry.announce_pending = true; fence(Ordering::SeqCst); - if let Err(error) = host.notify() { - host.report(&entry.context, Failure::Announcement(error)); + match host.notify() { + Ok(()) => entry.announce_pending = false, + Err(error) => host.report(&entry.context, Failure::Announcement(error)), } } } @@ -200,8 +220,9 @@ impl Pending { Ok(None) } - /// Poll quickly only during the initial release window; late arrivals still settle when - /// serviced. Time passage never substitutes for a firmware-consumption witness. + /// Poll for a bounded window at close and again at actual release publication. + /// Failed announcements remain retryable on later service even after polling + /// expires. Time passage never substitutes for a firmware-consumption witness. pub(crate) fn polling(&self, now: u64) -> bool { self.entries.iter().any(|entry| { entry.phase != Phase::Deferred && now.saturating_sub(entry.armed) < POLL_NS diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 50a5946ff4a37..653944d1f38f1 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -62,6 +62,17 @@ pub(crate) struct AsahiObject { /// Type alias for the shmem GEM object type for this driver. pub(crate) type Object = shmem::Object; +/// VM-private BOs share their assigned VM's reservation object. Validate this +/// before acquiring mapping state; generic GPUVM accepts external shareable BOs. +pub(crate) fn validate_vm_binding(gem: &Object, vm: &mmu::Vm) -> Result { + if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 + && vm.is_extobj(gem) + { + return Err(EINVAL); + } + Ok(()) +} + unsafe impl Send for AsahiObject {} unsafe impl Sync for AsahiObject {} @@ -126,11 +137,7 @@ impl ObjectRef { if obj_range.end > self.gem.size() { return Err(EINVAL); } - if self.gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 - && vm.is_extobj(&*self.gem) - { - return Err(EINVAL); - } + validate_vm_binding(&self.gem, vm)?; vm.map_in_range(&self.gem, obj_range, alignment, range, prot, guard) } @@ -144,11 +151,8 @@ impl ObjectRef { prot: mmu::Prot, guard: bool, ) -> Result { - if self.gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 - && vm.is_extobj(&*self.gem) - { - return Err(EINVAL); - } + validate_vm_binding(&self.gem, vm)?; + vm.map_at(addr, self.gem.size(), &self.gem, prot, guard) } } diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index eba89548c1d12..f4ab618693163 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -1181,8 +1181,9 @@ impl GpuManager::ver { .with(|raw, _inner| raw.pending_submissions.fetch_add(1, Ordering::Acquire)); mod_dev_dbg!(self.dev, "OP start (pending: {})\n", val + 1); + let guard = OpGuard(self.clone()); self.kick_firmware()?; - Ok(OpGuard(self.clone())) + Ok(guard) } fn invalidate_context( diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index 5303a386c13dc..deffe3fe47d0b 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -548,14 +548,13 @@ impl PwrConfig { return Err(EINVAL); } - let pz_count = pz_data.len() / 3; let mut power_zones = KVec::new(); - for i in (0..pz_count).step_by(3) { + for zone in pz_data.chunks_exact(3) { power_zones.push( PowerZone { - target: pz_data[i], - target_offset: pz_data[i + 1], - filter_tc: pz_data[i + 2], + target: zone[0], + target_offset: zone[1], + filter_tc: zone[2], }, GFP_KERNEL, )?; diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs index f354e4bb1f71a..0e99ab33fd3fc 100644 --- a/drivers/gpu/drm/asahi/mem.rs +++ b/drivers/gpu/drm/asahi/mem.rs @@ -82,7 +82,8 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { let start_pg = va >> mmu::UAT_PGBIT; let end_pg = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; - let mut val: u64 = ((asid as u64) << 48) | (2 << 46) | (start_pg as u64 & 0x1fffffffff); + const BADDR_MASK: u64 = (1 << 37) - 1; + let mut val: u64 = ((asid as u64) << 48) | (2 << 46) | (start_pg as u64 & BADDR_MASK); let pages = end_pg - start_pg; // Guess? It's possible that the page count is in terms of 4K pages @@ -124,6 +125,10 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { ); } base -= 32; + // Each full operation covers 32 units of 2 << bits GPU granules. + // Advance only the base-address field, preserving the ASID and scale. + let next = (val & BADDR_MASK) + (32u64 << (bits + 1)); + val = (val & !BADDR_MASK) | (next & BADDR_MASK); } // SAFETY: tlbi is always safe by definition diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 23d695bee8af9..fd83fd212224b 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -20,6 +20,7 @@ use core::num::NonZeroUsize; use core::ops::Range; use core::sync::atomic::{ fence, + AtomicBool, AtomicU32, AtomicU64, AtomicU8, @@ -524,28 +525,7 @@ impl gpuvm::DriverGpuVm for VmInner { op: OpUnmap<'op, Self>, _ctx: &mut Self::SmContext<'ctx>, ) -> Result, Error> { - let iova = op.va().addr(); - let length = op.va().length(); - - mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", iova, length); - - let _mutation = self.mapping_mutation(); - self.page_table.unmap_pages(iova..(iova + length))?; - - if let Some(asid) = self.slot() { - fence(Ordering::SeqCst); - self.tlbi_range(asid as u8, iova, length as usize); - mod_dev_dbg!( - self.dev, - "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", - asid, - iova, - length, - ); - mem::sync(); - } - - self.tlbi_contexts_naming_root(iova, length as usize); + self.unmap_gpuva(op.va())?; let (op_unmapped, _va_removed) = op.remove(); @@ -618,15 +598,45 @@ impl gpuvm::DriverGpuVm for VmInner { } impl VmInner { + /// Remove hardware leaves for one complete user GPUVA without allocating. + fn unmap_gpuva(&mut self, va: &gpuvm::GpuVa) -> Result { + mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.length()); + + let _mutation = self.mapping_mutation(); + self.page_table + .unmap_pages(va.addr()..(va.addr() + va.length()))?; + + if let Some(asid) = self.slot() { + fence(Ordering::SeqCst); + self.tlbi_range(asid as u8, va.addr(), va.length() as usize); + mod_dev_dbg!( + self.dev, + "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", + asid, + va.addr(), + va.length(), + ); + mem::sync(); + } + + self.tlbi_contexts_naming_root(va.addr(), va.length() as usize); + + Ok(()) + } + /// A failed map has no GPUVA owner to retain its backing or remove its leaves. /// GPUVM has already split or removed overlaps throughout this operation's range. fn rollback_failed_map(&mut self, addr: u64, size: u64) { - if self.uat_inner.firmware == UatFirmware::Handoff { - return; - } // Include the current segment: map_pages can fail after writing some of its leaves. // The checked GPUVM operation supplies an aligned range without overflow. let result = self.page_table.discard_partial_map(addr..addr + size); + if self.uat_inner.firmware == UatFirmware::Handoff { + if let Some(asid) = self.slot() { + fence(Ordering::SeqCst); + self.tlbi_range(asid as u8, addr, size as usize); + mem::sync(); + } + } self.tlbi_contexts_naming_root(addr, size as usize); if let Err(error) = result { dev_err!( @@ -714,9 +724,23 @@ impl VmInner { /// Map an `mm::Node` representing an mapping in VA space. fn map_node(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { + let end = node + .start() + .checked_add(node.mapped_size as u64) + .ok_or(EOVERFLOW)?; + let _mutation = self.mapping_mutation(); + self.page_table.prepare_map(node.start()..end)?; + let result = self.map_node_prepared(node, prot); + if result.is_err() { + self.rollback_failed_map(node.start(), node.mapped_size as u64); + } + result + } + + /// Install only into the empty range reserved by `map_node`. + fn map_node_prepared(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { let mut iova = node.start(); let segments = node.bo.as_ref().ok_or(EINVAL)?.get()?.data().segments.lock(); - let _mutation = self.mapping_mutation(); let mut offset = node.offset; let mut left = node.mapped_size; @@ -798,14 +822,16 @@ no_debug!(Vm); /// An application TTBAT identity retained by a logical queue and its jobs. /// The root lease is dropped before the VM, so no page table can be freed -/// while a context still names it. Installed firmware queues must retain this -/// object through replacement retirement or processor stop. +/// while an active context still names it. Installed firmware queues retain +/// this object through replacement retirement or processor stop; a consumed +/// closed-scheduler release may return its root identity while keeping storage. pub(crate) struct ExecutionContext { root: ExecutionRoot, vm: Vm, } struct ExecutionRoot { + released: AtomicBool, inner: Arc, id: u8, generation: u8, @@ -824,16 +850,28 @@ impl ExecutionContext { &self.vm } + /// Called only after the closed scheduler's release was consumed and all + /// its commands retired. Installed storage may outlive this root lease. + pub(crate) fn release(&self) { + self.root.release(); + } + pub(crate) fn is_current(&self) -> bool { let inner = self.root.inner.lock(); + if self.root.released.load(Ordering::Acquire) { + return false; + } let roots = &inner.ttbs()[usize::from(self.root.id)]; roots.ttb0.load(Ordering::Acquire) == self.root.low && roots.ttb1.load(Ordering::Acquire) == self.root.high } } -impl Drop for ExecutionRoot { - fn drop(&mut self) { +impl ExecutionRoot { + fn release(&self) { + if self.released.swap(true, Ordering::AcqRel) { + return; + } let id = usize::from(self.id); let release = { let inner = self.inner.lock(); @@ -863,6 +901,12 @@ impl Drop for ExecutionRoot { } } +impl Drop for ExecutionRoot { + fn drop(&mut self) { + self.release(); + } +} + /// Slot data for a [`Vm`] slot (nothing, we only care about the indices). pub(crate) struct SlotInner(); @@ -914,10 +958,9 @@ impl Clone for VmBind { } } -/// A mapping BO reference selecting the teardown protocol of its firmware. +/// A mapping BO reference released under the immediate-mode GPUVA lock. /// Retain the owner until deferred cleanup completes, including its final BO. struct MappingBo { - deferred: bool, bo: Option>>, owner: ARef>, } @@ -925,10 +968,8 @@ impl MappingBo { fn new( bo: ARef>, owner: ARef>, - deferred: bool, ) -> Self { Self { - deferred, bo: Some(bo), owner, } @@ -942,12 +983,6 @@ impl Drop for MappingBo { let Some(bo) = self.bo.take() else { return; }; - if !self.deferred { - // Handoff VMs retain the legacy ARef decrement and reservation - // locking behavior. Only published-root VMs use deferred cleanup. - drop(bo); - return; - } let raw = ARef::into_raw(bo); // SAFETY: GpuVmBo is repr(C) with drm_gpuvm_bo first. into_raw // transfers one live reference. Every Vm here uses IMMEDIATE_MODE; @@ -976,6 +1011,27 @@ pub(crate) struct KernelMappingInner { mapped_size: usize, } +/// Prepared CPU timestamp access; destruction happens after completion signalling. +pub(crate) struct TimestampMapping { + mapping: kernel::drm::gem::shmem::VMapOwned, + offset: usize, +} + +impl TimestampMapping { + pub(crate) fn write(&self, value: u64) { + let base = self.mapping.addr.cast::(); + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + // SAFETY: timestamp_mapping checked the complete eight-byte extent. + // The owned vmap pins its backing; byte atomics permit unaligned and + // overlapping timestamp destinations without mutable slice aliases. + let destination = unsafe { + &*base.add(self.offset + index).cast::() + }; + destination.store(byte, Ordering::Relaxed); + } + } +} + /// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. pub(crate) struct KernelMapping(mm::Node<(), KernelMappingInner>); @@ -1007,8 +1063,8 @@ impl KernelMapping { /// Writes a userspace timestamp through the retained GEM mapping. Byte /// stores support unaligned destinations and concurrent completion writes /// without creating mutable aliases. The complete timestamp is not atomic. - pub(crate) fn write_timestamp(&self, offset: usize, value: u64) -> Result { - use core::sync::atomic::AtomicU8; + /// Retain CPU access before this destination's completion fence is published. + pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { use kernel::drm::gem::BaseObject; let gem = self.0._gem.as_ref().ok_or(EINVAL)?; if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { @@ -1019,15 +1075,10 @@ impl KernelMapping { if end > gem.size() { return Err(ERANGE); } - let vmap = gem.vmap::<0>()?; - for (index, byte) in value.to_le_bytes().into_iter().enumerate() { - // SAFETY: The retained GEM/vmap covers each checked byte. AtomicU8 - // has byte alignment and all concurrent completion writers use - // the same access width, without borrowing a mutable byte slice. - let destination = unsafe { &*vmap.addr.cast::().add(start + index).cast::() }; - destination.store(byte, Ordering::Relaxed); - } - Ok(()) + Ok(TimestampMapping { + mapping: gem.owned_vmap::<0>()?, + offset: start, + }) } /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. @@ -1100,13 +1151,7 @@ impl KernelMapping { /// Remap a cached mapping as uncached, then synchronously flush that range of VAs from the /// coprocessor cache. This is required to safely unmap cached/private mappings. fn remap_uncached_and_flush(&mut self) { - let mut vm_exec = unsafe { - self.0 - .owner - .exec_lock(None, false) - .expect("Failed to exec_lock in remap_uncached_and_flush") - }; - let owner = vm_exec.unique_gpuvm.data(); + let mut owner = self.0.owner.lock_inner(); mod_dev_dbg!( owner.dev, @@ -1273,14 +1318,9 @@ impl Drop for KernelMapping { self.remap_uncached_and_flush(); } - let mut vm_exec = unsafe { - self.0 - .owner - .exec_lock(None, false) - .expect("exec_lock failed in KernelMapping::drop") - }; - - let owner = vm_exec.unique_gpuvm.data(); + // Only VM page tables/binding state are touched here. The shared + // reservation is sufficient and avoids fallible exec allocation in Drop. + let mut owner = self.0.owner.lock_inner(); mod_dev_dbg!( owner.dev, "MMU: unmap {:#x}:{:#x}\n", @@ -1783,19 +1823,18 @@ impl Vm { } core::mem::drop(vm_bo_guard); - let deferred = inner.uat_inner.firmware != UatFirmware::Handoff; - let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone(), deferred)); + let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); // A failed reservation drops its payload while exec still owns GEM - // reservations. Published-root VMs attach BO/GEM only after success; - // handoff VMs retain their existing payload and decrement behavior. + // reservations. Attach BO/GEM references only after success, since + // their destructors may acquire the same reservation locks. let result = inner.mm.insert_node_in_range( KernelMappingInner { owner: self.gpuvm.clone(), uat_inner, prot, - bo: if deferred { None } else { vm_bo.take() }, - _gem: if deferred { None } else { Some(gem.into()) }, + bo: None, + _gem: None, offset: object_range.start, mapped_size: size, }, @@ -1813,7 +1852,7 @@ impl Vm { return Err(error); } }; - if deferred { + { let payload = node.as_mut().inner_mut(); payload.bo = vm_bo.take(); payload._gem = Some(gem.into()); @@ -1868,17 +1907,17 @@ impl Vm { } core::mem::drop(vm_bo_guard); - let deferred = inner.uat_inner.firmware != UatFirmware::Handoff; - let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone(), deferred)); + let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); - // Published-root VMs defer BO/GEM attachment until reservation succeeds. + // Keep reservation-locking destructors out of the allocator payload + // until reservation succeeds. let result = inner.mm.reserve_node( KernelMappingInner { owner: self.gpuvm.clone(), uat_inner, prot, - bo: if deferred { None } else { vm_bo.take() }, - _gem: if deferred { None } else { Some(gem.into()) }, + bo: None, + _gem: None, offset: 0, mapped_size: size, }, @@ -1893,7 +1932,7 @@ impl Vm { return Err(error); } }; - if deferred { + { let payload = node.as_mut().inner_mut(); payload.bo = vm_bo.take(); payload._gem = Some(gem.into()); @@ -1919,6 +1958,7 @@ impl Vm { prot: Prot, single_page: bool, ) -> Result { + gem::validate_vm_binding(gem, self)?; self.wait_for_user_map_admission()?; // TODO: hack, populate the GEM object sg table first { @@ -2036,13 +2076,18 @@ impl Vm { 0, )?; + let end = iova.checked_add(size as u64).ok_or(EOVERFLOW)?; + inner.page_table.prepare_map(iova..end)?; let mutation = inner.mapping_mutation(); let ret = inner.page_table.map_pages( - iova..(iova + size as u64), + iova..end, phys as PhysicalAddr, prot, false, ); + if ret.is_err() { + inner.rollback_failed_map(iova, size as u64); + } // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. drop(mutation); @@ -2118,15 +2163,19 @@ impl Vm { if let Some(bo) = vm_exec.unique_gpuvm.find_bo(gem) { mod_dev_dbg!(self.dev, "MMU: bo_unmap\n"); - vm_exec.unique_gpuvm.bo_unmap(&mut ctx, &bo)?; - // The close callback or deferred object list retains this GEM, so - // clearing its candidate cannot release the final reference here. - self.untrack_context_object(gem); - self.untrack_shared_object(gem); + let result = vm_exec.unique_gpuvm.bo_unmap(&mut ctx, &bo); + if result.is_ok() { + // The close callback or deferred object list retains this GEM, + // so clearing its candidate cannot release its final reference. + self.untrack_context_object(gem); + self.untrack_shared_object(gem); + } mod_dev_dbg!(self.dev, "MMU: bo_unmap done\n"); - // We need to drop the exec_lock first, then the GpuVmBo since that will take the lock itself. + // BO release can drain unrelated deferred GEMs, including on an + // unmap error. Always release reservations before dropping it. core::mem::drop(vm_exec); core::mem::drop(bo); + result?; } Ok(()) } @@ -2377,6 +2426,7 @@ impl Uat { let context = Arc::new( ExecutionContext { root: ExecutionRoot { + released: AtomicBool::new(false), inner: self.inner.clone(), id: id as u8, generation, diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index e8efd4847e02f..35d5aaaae912b 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -61,6 +61,7 @@ impl Vm { prot: Prot, single_page: bool, ) -> Result { + gem::validate_vm_binding(gem, self)?; let context = self.prepare_context_binding(addr, size, offset)?; let resources = VmOpResources { preallocated_gpuvas: [ diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 78f906ff9c46c..f819de8becd1b 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -520,40 +520,19 @@ impl Vm { self.unmap_user_ranges_now(user, kernel) } - fn unmap_user_ranges_now(&self, user: Range, kernel: Range) -> Result { - let driver = self - .lifetime - .as_ref() - .map(|lifetime| lifetime.driver.lock()); - let mut cursor = user.start; - while cursor < user.end { - let mut next = user.end; - let mut covered = cursor; - for range in Iterator::chain( - core::iter::once(kernel.clone()), - driver - .as_ref() - .and_then(|guard| guard.as_ref()) - .into_iter() - .flat_map(DriverMappings::ranges), - ) { - if range.contains(&cursor) { - covered = covered.max(range.end); - } else if cursor < range.start { - next = next.min(range.start); - } - } - if covered != cursor { - cursor = covered.min(user.end); - continue; - } - if next <= cursor { - return Err(EIO); - } - self.unmap_range_now(cursor, next - cursor)?; - cursor = next; - } - Ok(()) + fn unmap_user_ranges_now(&self, user: Range, _kernel: Range) -> Result { + // User mappings alone have GPUVA nodes. Kernel and driver aliases use + // mm::Node and are untouched; GPUVM's embedded kernel cutout is skipped. + // Closed admission and zero accepted jobs make whole-node teardown safe. + let mut inner = self.gpuvm.lock_inner(); + let result = inner.unmap_all(VmInner::unmap_gpuva); + if result.is_ok() { + self.untrack_shared_range(user.start, user.end - user.start); + } + drop(inner); + // Final GEM/SG-table destruction may acquire reservations itself. + self.bo_deferred_cleanup(); + result } } diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index db91e456ab8b2..0ad231b930a8a 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -325,12 +325,18 @@ impl UatPageTable { let end = ((iova_range.end - 1) & self.ias_mask) + 1; let mut pt_addr: [Option; UAT_LEVELS] = Default::default(); + // For every child table, retain the parent page and entry that names it. + // Children are detached before their parents are freed, so these borrowed + // parent pages remain valid while the descriptor is cleared. + let mut pt_parent: [Option<(PhysicalAddr, usize)>; UAT_LEVELS - 1] = Default::default(); pt_addr[UAT_LEVELS - 1] = Some(self.ttb); 'outer: while iova < end { mod_pr_debug!("UATPageTable::with_pages: iova={:#x}\n", iova); let addr_diff = last_iova ^ iova; - for level in (0..UAT_LEVELS - 1).rev() { + // Detach leaves before intermediate tables. The old order freed a + // parent first and could leave its descriptor naming a freed child. + for level in 0..UAT_LEVELS - 1 { // If the iova has changed at this level or above, invalidate the physaddr if addr_diff & !((1 << (UAT_PGBIT + (level + 1) * UAT_LVBIT)) - 1) != 0 { if let Some(phys) = pt_addr[level].take() { @@ -340,7 +346,10 @@ impl UatPageTable { level, phys ); - self.free_table(phys); + let parent = pt_parent[level].take(); + self.free_table(phys, parent)?; + } else { + pt_parent[level] = None; } mod_pr_debug!("UATPageTable::with_pages: invalidate level {}\n", level); } @@ -391,6 +400,9 @@ impl UatPageTable { Ok(None) } })?; + if pt_addr[level].is_some() { + pt_parent[level] = Some((phys, upidx)); + } mod_pr_debug!( "UATPageTable::with_pages: level {} PT {:#x?}\n", level, @@ -440,14 +452,16 @@ impl UatPageTable { } if free { - for level in (0..UAT_LEVELS - 1).rev() { + // The final path is detached bottom-up for the same reason as the + // boundary transitions above. + for level in 0..UAT_LEVELS - 1 { if let Some(phys) = pt_addr[level] { mod_pr_debug!( "UATPageTable::with_pages: free level {} {:#x?}\n", level, phys ); - self.free_table(phys); + self.free_table(phys, pt_parent[level].take())?; } } } @@ -456,18 +470,50 @@ impl UatPageTable { } /// Free a page table, unless it was not allocated by the driver. - fn free_table(&mut self, phys: PhysicalAddr) { - if let Some(tables) = self.driver_tables.as_mut() { + fn free_table( + &mut self, + phys: PhysicalAddr, + parent: Option<(PhysicalAddr, usize)>, + ) -> Result { + let owned = if let Some(tables) = self.driver_tables.as_ref() { match tables.iter().position(|&table| table == phys) { - Some(index) => { - tables.swap_remove(index); - } - None => return, + Some(_) => true, + None => false, } + } else { + true + }; + if !owned { + return Ok(()); + } + if let Some((parent_phys, parent_index)) = parent { + // SAFETY: The bottom-up traversal keeps the parent allocated while + // its child descriptor is detached. A changed descriptor indicates + // an ownership violation; fail closed instead of freeing a table + // that may still be reachable. + let page = unsafe { Page::borrow_phys(&parent_phys) }.ok_or(EIO)?; + page.with_pointer_into_page(parent_index * PTE_SIZE, PTE_SIZE, |p| { + let pte = unsafe { &*(p as *const AtomicU64) }; + let current = pte.load(Ordering::Acquire); + let expected = phys & self.oas_mask & (!UAT_PGMSK as u64); + if current & self.oas_mask & (!UAT_PGMSK as u64) != expected { + return Err(EIO); + } + pte.store(0, Ordering::Release); + Ok(()) + })?; + } + if let Some(tables) = self.driver_tables.as_mut() { + let index = tables + .iter() + .position(|&table| table == phys) + .ok_or(EIO)?; + tables.swap_remove(index); } // SAFETY: Page tables allocated by the driver always come from Page::into_phys(). Without // `driver_tables`, every child table belongs to the driver. unsafe { Page::from_phys(phys) }; + Ok(()) } /// Checks every page, including holes skipped by the page-table walker. @@ -528,6 +574,22 @@ impl UatPageTable { self.with_pages(iova_range, true, false, |_, _| Ok(())) } + /// Reserve all tables for a fixed mapping before publishing any leaves. + /// GPUVM and the fixed-address allocator may share a page table, so check + /// that neither owns a destination leaf before allowing failure rollback. + pub(crate) fn prepare_map(&mut self, range: Range) -> Result { + if range.is_empty() || (range.start | range.end) & UAT_PGMSK as u64 != 0 { + return Err(EINVAL); + } + self.with_pages(range.clone(), false, false, |_, ptes| { + if ptes.iter().any(|pte| pte.load(Ordering::Acquire) != 0) { + return Err(EBUSY); + } + Ok(()) + })?; + self.alloc_pages(range) + } + fn pte_bits(&self) -> u64 { if self.ttb_owned { // Owned page tables are userspace, so non-global diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 38e1edd23bc31..f1ab56e80fd1a 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -246,6 +246,8 @@ impl QueueJob::ver { #[versions(AGX)] impl sched::JobImpl for QueueJob::ver { + const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); + fn prepare(job: &mut sched::Job) -> Option { mod_dev_dbg!(job.dev, "QueueJob {}: Checking runnability\n", job.id); @@ -330,10 +332,6 @@ impl sched::JobImpl for QueueJob::ver { // are "invisible" to the firmware. Incrementing the notifier threshold // earlier than this violates that which leads to circles like the // above. Deferring the increment to submit solves the race. - job.notifier.threshold.with(|raw, _inner| { - raw.increase(job.notification_count); - }); - let dev = job.dev.clone(); let Some(guard) = dev.registration_guard() else { return Err(ENODEV); @@ -389,6 +387,13 @@ impl sched::JobImpl for QueueJob::ver { } } + // Prepared submissions still hold their workqueue locks and have not + // published cpu_wptr. Only count them after every fallible submit() + // succeeded, before the first run_job exposes commands to firmware. + job.notifier.threshold.with(|raw, _inner| { + raw.increase(job.notification_count); + }); + // Now we fully commit to running the job mod_dev_dbg!(job.dev, "QueueJob {}: Run fragment\n", job.id); frag_sub.map(|a| gpu.run_job(a)).transpose()?; @@ -721,11 +726,11 @@ impl Queue for Queue::ver { match header.cmd_type as u32 { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { - last_compute = nr_commands; + last_render = nr_commands; nr_render += 1; } uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { - last_render = nr_commands; + last_compute = nr_commands; nr_compute += 1; } _ => {} diff --git a/drivers/gpu/drm/asahi/util.rs b/drivers/gpu/drm/asahi/util.rs index 1a41d8f16d443..00246e1f64ab7 100644 --- a/drivers/gpu/drm/asahi/util.rs +++ b/drivers/gpu/drm/asahi/util.rs @@ -108,23 +108,24 @@ impl<'a> Reader<'a> { Reader { buffer, offset: 0 } } - pub(crate) fn read_up_to(&mut self, max_size: usize) -> Result { + pub(crate) fn read_up_to(&mut self, declared_size: usize) -> Result { let mut obj: T = Default::default(); - let size: usize = core::mem::size_of::().min(max_size); - let range = self.offset..self.offset + size; + let size: usize = core::mem::size_of::().min(declared_size); + let end = self.offset.checked_add(declared_size).ok_or(EINVAL)?; + let range = self.offset..end; let src = self.buffer.get(range).ok_or(EINVAL)?; // SAFETY: The output pointer is valid, and the size does not exceed // the type size, and all bit patterns are valid. let dst = unsafe { core::slice::from_raw_parts_mut(&mut obj as *mut _ as *mut u8, size) }; - dst.copy_from_slice(src); - self.offset += size; + dst.copy_from_slice(&src[..size]); + self.offset = end; Ok(obj) } pub(crate) fn read(&mut self) -> Result { - self.read_up_to(!0) + self.read_up_to(core::mem::size_of::()) } pub(crate) fn is_empty(&self) -> bool { diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c index b123f6164206b..92ab5271e523e 100644 --- a/drivers/gpu/drm/drm_gpuvm.c +++ b/drivers/gpu/drm/drm_gpuvm.c @@ -2382,6 +2382,7 @@ op_map_cb(const struct drm_gpuvm_ops *fn, void *priv, op.map.va.range = req->map.va.range; op.map.gem.obj = req->map.gem.obj; op.map.gem.offset = req->map.gem.offset; + op.map.gem.range = req->map.gem.range; op.map.flags = req->map.flags; return fn->sm_step_map(&op, priv); @@ -2438,7 +2439,7 @@ static bool __can_merge(struct drm_gpuvm *gpuvm, const struct drm_gpuva_op_map * if (a->gem.obj != b->gem.obj || !a->gem.obj) return false; - if (can_merge_flags(gpuvm, a->flags, b->flags)) + if (!can_merge_flags(gpuvm, a->flags, b->flags)) return false; /* Order VAs for the rest of the checks. */ @@ -2457,7 +2458,7 @@ static bool __can_merge(struct drm_gpuvm *gpuvm, const struct drm_gpuva_op_map * /* If this is a repeated mapping, both the GEM range * and offset must match. */ - if (a->gem.range != b->gem.range || + if (!a->gem.range || a->gem.range != b->gem.range || a->gem.offset != b->gem.offset) return false; @@ -2485,6 +2486,7 @@ static bool can_merge(struct drm_gpuvm *gpuvm, const struct drm_gpuva *a, .va.range = a->va.range, .gem.offset = a->gem.offset, .gem.obj = a->gem.obj, + .gem.range = a->gem.range, .flags = a->flags, }; @@ -2738,7 +2740,7 @@ __drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, next.va.addr = req_end; next.va.range = end - req_end; next.gem.obj = obj; - prev.gem.range = va->gem.range; + next.gem.range = va->gem.range; next.gem.offset = offset + (va->flags & DRM_GPUVA_REPEAT ? 0 : req_end - addr); next.flags = va->flags; diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c index b16060887854e..7f9affa4afdd0 100644 --- a/drivers/gpu/drm/scheduler/sched_entity.c +++ b/drivers/gpu/drm/scheduler/sched_entity.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -245,6 +246,7 @@ static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f, static void drm_sched_entity_kill_jobs_work(struct work_struct *wrk) { struct drm_sched_job *job = container_of(wrk, typeof(*job), work); + struct module *owner = job->sched->ops->owner; struct dma_fence *f; unsigned long index; @@ -279,6 +281,8 @@ static void drm_sched_entity_kill_jobs_work(struct work_struct *wrk) drm_sched_fence_finished(job->s_fence, -ESRCH); WARN_ON(job->s_fence->parent); job->sched->ops->free_job(job); + /* free_job may have freed both job and its scheduler allocation. */ + module_put(owner); } /* Signal the scheduler finished fence when the entity in question is killed. */ @@ -325,6 +329,11 @@ void drm_sched_entity_kill(struct drm_sched_entity *entity) while ((job = drm_sched_entity_queue_pop(entity))) { struct drm_sched_fence *s_fence = job->s_fence; + /* The caller still owns the backend module here. Keep it alive + * across every dependency callback and through the driver callback's + * return, even after the entity and scheduler have been finalized. + */ + __module_get(job->sched->ops->owner); dma_fence_get(&s_fence->finished); if (!prev || dma_fence_add_callback(prev, &job->finish_cb, diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index 214ed02cb4964..48cecf0d6bce0 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -1218,6 +1218,15 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched) int i; drm_sched_wqueue_stop(sched); + /* + * wqueue_stop() cancels work that is already queued, but the completion + * callback can pass its pause check concurrently and queue either item + * after cancel_work_sync() returns. Finalization never restarts this + * scheduler, so permanently disable and drain both work items before + * touching scheduler-owned queues or destroying the workqueue. + */ + disable_work_sync(&sched->work_run_job); + disable_work_sync(&sched->work_free_job); for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) kfree(sched->sched_rq[i]); diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 4390ba10767b9..b4054ec0dfc03 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -761,9 +761,7 @@ int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint) msg = FIELD_PREP(APPLE_RTKIT_MGMT_STARTEP_EP, endpoint); msg |= APPLE_RTKIT_MGMT_STARTEP_FLAG; - apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_STARTEP, msg); - - return 0; + return apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_STARTEP, msg); } EXPORT_SYMBOL_GPL(apple_rtkit_start_ep); diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h index aca218734dcde..367444b74e8ba 100644 --- a/include/drm/drm_gpuvm.h +++ b/include/drm/drm_gpuvm.h @@ -1165,6 +1165,7 @@ static inline void drm_gpuva_init_from_op(struct drm_gpuva *va, va->va.range = op->va.range; va->gem.obj = op->gem.obj; va->gem.offset = op->gem.offset; + va->gem.range = op->gem.range; } /** diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index 42fe3294c4933..2a6415eea8d9a 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -569,6 +569,14 @@ struct drm_sched_backend_ops { * drm_sched_backend_ops.free_job. */ void (*cancel_job)(struct drm_sched_job *sched_job); + + /** + * @owner: Optional module implementing these callbacks. The caller must + * retain this module through entity destruction. Detached entity-kill + * work takes its own reference before leaving that boundary and releases + * it only after free_job returns to the scheduler core. + */ + struct module *owner; }; /** diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 28eed5eb2b1f9..4d6674f245049 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -207,7 +207,7 @@ impl UnregisteredDevice { fops: &Self::GEM_FOPS, }; - const GEM_FOPS: bindings::file_operations = drm::gem::create_fops(); + const GEM_FOPS: bindings::file_operations = drm::gem::create_fops(T::MODULE); /// Create a new `UnregisteredDevice` for a `drm::Driver`. /// diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index 5bfe86c865db3..4971f3496b581 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -134,6 +134,10 @@ pub trait Driver { /// Driver metadata const INFO: DriverInfo; + /// Module implementing the driver callbacks. Loadable drivers should provide + /// their `THIS_MODULE` so open files and exported DMA buffers retain the code. + const MODULE: Option<&'static crate::ThisModule> = None; + /// IOCTL list. See `kernel::drm::ioctl::declare_drm_ioctls!{}`. const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor]; diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index 1e87a88e4a077..8c978f5fd7abd 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -495,12 +495,17 @@ impl DmaBuf { } } -pub(super) const fn create_fops() -> bindings::file_operations { +pub(super) const fn create_fops( + module: Option<&'static crate::ThisModule>, +) -> bindings::file_operations { // SAFETY: As by the type invariant, it is safe to initialize `bindings::file_operations` // zeroed. let mut fops: bindings::file_operations = unsafe { core::mem::zeroed() }; - fops.owner = core::ptr::null_mut(); + fops.owner = match module { + Some(module) => module.as_ptr(), + None => core::ptr::null_mut(), + }; fops.open = Some(bindings::drm_open); fops.release = Some(bindings::drm_release); fops.unlocked_ioctl = Some(bindings::drm_ioctl); diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs index 6ca1e2e3c8ac8..e0db05c15a0ed 100644 --- a/rust/kernel/drm/gpuvm/mod.rs +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -27,7 +27,10 @@ use kernel::{ ARef, AlwaysRefCounted, // }, - types::Opaque, // + types::{ + NotThreadSafe, + Opaque, // + }, // }; use core::{ @@ -304,6 +307,95 @@ impl GpuVm { } } +impl GpuVm { + /// HACK: added for the asahi downstream driver. Lock only the VM's shared reservation. + /// + /// This takes the same shared reservation as [`GpuVm::exec_lock`], but does not lock + /// external BO reservations or allocate an exec context. The returned guard exposes only the + /// driver data, not GPUVA operations requiring those other reservations. Callers must not + /// already hold a reservation lock. + pub fn lock_inner(&self) -> GpuVmInnerGuard<'_, T> { + // SAFETY: The borrowed GPUVM retains `r_obj` and its reservation for the guard's lifetime. + // A non-interruptible single-reservation lock without a ww context cannot fail. + unsafe { bindings::dma_resv_lock(self.raw_resv(), ptr::null_mut()) }; + GpuVmInnerGuard { + gpuvm: self, + _not_send: NotThreadSafe, + } + } +} + +/// HACK: added for the asahi downstream driver. Access to the VM driver data under its shared +/// reservation, without external BO locks. +pub struct GpuVmInnerGuard<'a, T: DriverGpuVm> { + gpuvm: &'a GpuVm, + // Reservation locks must be released by the task that acquired them. + _not_send: NotThreadSafe, +} + +impl GpuVmInnerGuard<'_, T> { + /// Remove every driver GPUVA of this immediate-mode VM, preserving its kernel cutout. + /// + /// `unmap` must remove the GPU translation of each complete GPUVA before returning success. + /// Only the VM's shared reservation is held: no external BO reservation is taken and no split + /// node or operation list is allocated. Call [`GpuVm::deferred_cleanup`] after releasing this + /// guard, including when a callback failed after earlier mappings were removed. + pub fn unmap_all(&mut self, mut unmap: impl FnMut(&mut T, &GpuVa) -> Result) -> Result { + let vm = self.gpuvm.as_raw(); + // SAFETY: The guard owns this live VM's shared reservation, and the VM is in immediate + // mode, so unlinking takes the GEM GPUVA mutex independently of its reservation. Every + // listed GPUVA other than the kernel cutout is a `GpuVa` owned by this VM. + unsafe { + if (*vm).flags & bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE == 0 { + return Err(EINVAL); + } + let head = &raw mut (*vm).rb.list; + let mut entry = (*head).next; + while entry != head { + let va = kernel::container_of!(entry, bindings::drm_gpuva, rb.entry); + // Save the successor before removing and freeing this GPUVA. + entry = (*entry).next; + // This node is embedded in the C GPUVM, not a driver `GpuVa`. + if ptr::eq(va, &raw const (*vm).kernel_alloc_node) { + continue; + } + unmap(&mut *self.gpuvm.data.get(), GpuVa::::from_raw(va))?; + // Remove it as `OpUnmap::remove()` does. + let op = bindings::drm_gpuva_op_unmap { va, keep: false }; + bindings::drm_gpuva_unmap(&op); + bindings::drm_gpuva_unlink_defer(va); + drop(GpuVaRemoved::::from_raw(va)); + } + } + Ok(()) + } +} + +impl Deref for GpuVmInnerGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &T { + // SAFETY: The shared reservation excludes every other access to the driver data, + // including `GpuVm::exec_lock()` and its callbacks. + unsafe { &*self.gpuvm.data.get() } + } +} + +impl DerefMut for GpuVmInnerGuard<'_, T> { + fn deref_mut(&mut self) -> &mut T { + // SAFETY: As above, with exclusive access to this guard. + unsafe { &mut *self.gpuvm.data.get() } + } +} + +impl Drop for GpuVmInnerGuard<'_, T> { + fn drop(&mut self) { + // SAFETY: This non-Send guard owns the reservation on this task, and its GPUVM borrow + // keeps `r_obj` alive until after the unlock. + unsafe { bindings::dma_resv_unlock(self.gpuvm.raw_resv()) }; + } +} + /// The manager for a GPUVM. pub trait DriverGpuVm: Sized + Send + Sync { /// Parent `Driver` for this object. diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index a1f5eb04f575f..2b798de4d18ca 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -43,6 +43,10 @@ pub enum Priority { /// Trait to be implemented by driver job objects. pub trait JobImpl: Sized { + /// Module implementing the callbacks. Its caller must retain the module + /// through entity destruction; the core pins detached kill-job callbacks. + const MODULE: Option<&'static crate::ThisModule> = None; + /// Handle a scoped timeout without resetting the C scheduler. /// /// The backend must settle/quarantine its exact hardware work, or retain @@ -468,6 +472,10 @@ impl Scheduler { timedout_job: Some(timedout_job_cb::), free_job: Some(free_job_cb::), cancel_job: Some(cancel_job_cb::), + owner: match T::MODULE { + Some(module) => module.as_ptr(), + None => core::ptr::null_mut(), + }, }; /// Creates a new DRM Scheduler object // TODO: Shared timeout workqueues & scores From 73e65f0e9bd9700f0de95751a3d9b4afb98b4489 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 22:58:19 -0400 Subject: [PATCH 0763/2680] rust: expose the DRM duplicate-reservation flag used by GPUVM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expose DRM_EXEC_IGNORE_DUPLICATES through the constant helper so the GPUVM array-lock wrapper builds against the public kernel bindings. The current-kernel integration already provided this helper; the public aurora-wip base did not. [ryan: adapted to 7.2.10 — minimal; for DJ: asahi-wip already exports DRM_EXEC_INTERRUPTIBLE_WAIT next to the other DRM constants, so only DRM_EXEC_IGNORE_DUPLICATES is added there; its only user, exec_lock_array(), is not carried on 7.2] Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 4a577fcddddbefec7d13bfd4ae3503893602d29b) --- rust/bindings/bindings_helper.h | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 31fe42ed08c4c..b8d41061141c9 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -175,6 +175,7 @@ const vm_flags_t RUST_CONST_HELPER_VM_HUGEPAGE = VM_HUGEPAGE; const vm_flags_t RUST_CONST_HELPER_VM_NOHUGEPAGE = VM_NOHUGEPAGE; const u32 RUST_CONST_HELPER_DRM_EXEC_INTERRUPTIBLE_WAIT = DRM_EXEC_INTERRUPTIBLE_WAIT; +const u32 RUST_CONST_HELPER_DRM_EXEC_IGNORE_DUPLICATES = DRM_EXEC_IGNORE_DUPLICATES; #if IS_ENABLED(CONFIG_GPU_BUDDY) const unsigned long RUST_CONST_HELPER_GPU_BUDDY_RANGE_ALLOCATION = GPU_BUDDY_RANGE_ALLOCATION; From 62f086e3e19e71c5d97b1cc515dbcda3dd1647f3 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 23:36:52 -0400 Subject: [PATCH 0764/2680] drm/asahi: order render timestamp publication independently of GPU retirement Physical render pairs can be observed out of logical order. When a command waits through an earlier render timestamp producer, defer its host writes and public completion until that producer's host publication has finished. Preallocate forward-only publication links before accepting the job and install them before early publication or scheduler push. Process ready successors iteratively without allocating or waiting in completion. Keep scheduler credits independent: commands with host predecessors use a separate terminal fence as their scheduler parent. Healthy retirement still releases physical resources immediately, while failed-owned commands keep their pins until a retirement witness. Cancellation and physical dependency snapshots use that terminal fence. Ordinary scheduled-fence readiness and compute timestamp dependencies retain their existing behavior. Fifteen extracted-production host tests pass, including the original reverse observation failure, allocation/cancellation paths, racing registration and completion, and a 12000-node chain on a 64 KiB host stack. Full AArch64 16K Image/modules/dtbs builds with 1591 modules and 1805 DTBs. Test harnesses and receipts remain outside the source tree. Hardware and performance validation are still outstanding; no queue-depth reduction or GPU serialization is added. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit d9b7b82975e2dad4b9a5941ade2f4debadd61c6c) --- drivers/gpu/drm/asahi/g17/completion.rs | 123 ++++++++--- .../drm/asahi/g17/completion/publication.rs | 209 ++++++++++++++++++ drivers/gpu/drm/asahi/g17/job.rs | 15 +- drivers/gpu/drm/asahi/g17/queue/mod.rs | 17 +- .../asahi/g17/queue/render/dependencies.rs | 2 +- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 4 +- .../drm/asahi/g17/runtime/render/settle.rs | 2 +- 7 files changed, 337 insertions(+), 35 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/completion/publication.rs diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index feb69e8b2c291..aa29d107f27f3 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -23,6 +23,9 @@ use kernel::{ uapi, xarray, }; +mod publication; +pub(crate) use publication::{Prepared as PreparedPublication, Publication}; + struct Destination { // Keep the existing GPU alias alive along with its prepared CPU view. _mapping: Arc, @@ -51,7 +54,7 @@ impl Destination { Ok(Some(Self { _mapping: mapping, cpu })) } - fn write(&self, ticks: u64) -> Result { + fn write(&self, ticks: u64) { const NSEC_PER_SEC: u64 = 1_000_000_000; let ns = (ticks / cfg::COMMAND_TIMESTAMP_HZ) .wrapping_mul(NSEC_PER_SEC) @@ -60,7 +63,6 @@ impl Destination { ); self.cpu.write(ns); barrier(Ordering::Release); - Ok(()) } } @@ -97,6 +99,51 @@ impl Destinations { } } +/// Public outputs do not own physical command resources or a scheduler job. +struct Output { + fence: UserFence, + member: Member, + destinations: Destinations, +} + +impl Output { + fn publish(&self, result: Result<[u64; 4]>) { + match result { + Ok(ticks) => { + for (destination, ticks) in self.destinations.iter().zip(ticks) { + if let Some(destination) = destination { + destination.write(ticks); + } + } + } + Err(error) => self.fence.set_error(error), + } + self.fence.signal(); + self.member.complete(result.map(|_| ())); + } +} + +enum HostOutput { + Immediate(Output), + Ordered(Arc), +} + +impl HostOutput { + fn output(&self) -> &Output { + match self { + Self::Immediate(output) => output, + Self::Ordered(publication) => publication.output(), + } + } + + fn finish(&self, result: Result<[u64; 4]>) { + match self { + Self::Immediate(output) => output.publish(result), + Self::Ordered(publication) => publication.finish(result), + } + } +} + /// Firmware writes two aligned words in a fresh write-combined page per compute command. struct ComputeTimestamps { // Remove the GPU alias before releasing its backing. @@ -120,9 +167,10 @@ impl ComputeTimestamps { #[pin_data] pub(crate) struct Completion { status: Arc, - fence: UserFence, - member: Member, - destinations: Destinations, + output: HostOutput, + // Present only when this command's public result may wait for a render + // predecessor's host writes. DRM credits must follow the terminal result. + terminal: Option>, compute: Option, feed: Option>, #[pin] @@ -148,9 +196,23 @@ impl Completion { vm_job: mmu::VmJobGuard, destinations: Destinations, compute: bool, + host_predecessors: bool, feed: Option>, ) -> Result> { let fence = fence::independent(contexts)?; + let ordered = !compute + && (host_predecessors || destinations.iter().any(Option::is_some)); + let terminal = if host_predecessors { + Some(fence::independent(contexts)?) + } else { + None + }; + let output = Output { fence, member, destinations }; + let output = if ordered { + HostOutput::Ordered(Publication::new(output, context.status().clone())?) + } else { + HostOutput::Immediate(output) + }; let compute = if compute { Some(ComputeTimestamps::new(dev, context.vm())?) } else { @@ -159,9 +221,8 @@ impl Completion { Arc::pin_init( pin_init!(Self { status: context.status().clone(), - fence, - member, - destinations, + output, + terminal, compute, feed, vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), @@ -180,7 +241,24 @@ impl Completion { } pub(crate) fn fence(&self) -> Fence { - Fence::from_fence(&self.fence) + Fence::from_fence(&self.output.output().fence) + } + pub(crate) fn scheduler_fence(&self) -> Fence { + self.terminal.as_ref().map_or_else(|| self.fence(), |fence| Fence::from_fence(fence)) + } + pub(crate) fn publication(&self) -> Option> { + match &self.output { + HostOutput::Ordered(publication) => Some(publication.clone()), + HostOutput::Immediate(_) => None, + } + } + fn signal_terminal(&self, result: Result<[u64; 4]>) { + if let Some(fence) = &self.terminal { + if let Err(error) = result { + fence.set_error(error); + } + fence.signal(); + } } /// All fallible scheduler setup has succeeded. Failures from this point /// must be visible through VM_STATUS before the accepted job's fence. @@ -191,10 +269,10 @@ impl Completion { &self.status } pub(crate) fn work_state(&self) -> Result> { - self.member.work_state() + self.output.output().member.work_state() } pub(crate) fn has_timestamps(&self) -> bool { - self.destinations.iter().any(Option::is_some) + self.output.output().destinations.iter().any(Option::is_some) } pub(crate) fn compute_timestamp_va(&self) -> Result { @@ -360,25 +438,16 @@ impl Completion { if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { feed.note_render_pass(ticks[2], ticks[3]); } - let result = result.and_then(|ticks| { - for (destination, ticks) in self.destinations.iter().zip(ticks) { - if let Some(destination) = destination { - destination.write(ticks)?; - } - } - Ok(()) - }); if let Err(error) = result { if self.spared() { self.status.report_failure(error); } else if error != ECANCELED { self.status.record(error); } - self.fence.set_error(error); } self.release_healthy(); - self.fence.signal(); - self.member.complete(result); + self.signal_terminal(result); + self.output.finish(result); } /// A rejected, never-published command cannot poison the VM for contention. @@ -390,10 +459,9 @@ impl Completion { if self.accepted.load(Ordering::Acquire) { self.status.report_failure(error); } - self.fence.set_error(error); self.release_healthy(); - self.fence.signal(); - self.member.complete(Err(error)); + self.signal_terminal(Err(error)); + self.output.finish(Err(error)); } /// The physical queue must already be quarantined. Keep mappings pinned while @@ -410,9 +478,8 @@ impl Completion { } else if error != ECANCELED { self.status.record(error); } - self.fence.set_error(error); - self.fence.signal(); - self.member.complete(Err(error)); + self.signal_terminal(Err(error)); + self.output.finish(Err(error)); } } diff --git a/drivers/gpu/drm/asahi/g17/completion/publication.rs b/drivers/gpu/drm/asahi/g17/completion/publication.rs new file mode 100644 index 0000000000000..a1e05a9408552 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/completion/publication.rs @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Host-only ordering of timestamp writes, independent of scheduler credits. +//! +//! Links point from earlier logical render ordinals to later ones. They retain +//! only public output state, never a packet, physical render lease or VM job pin. +//! All link storage is allocated before acceptance. Completion neither allocates +//! nor waits for another command; the last prerequisite publishes a ready result. + +use super::Output; +use crate::g17::{dependency::inherited_error, status::VmStatus}; +use kernel::{new_mutex, prelude::*, sync::{Arc, Mutex}}; + +type Links = Option>; + +struct Link { + target: Arc, + result: Result, + next: Links, + // Reuse an already-consumed link as a traversal frame instead of recursing + // or allocating when publication unlocks a long chain of ready successors. + resume: Links, +} + +struct State { + remaining: usize, + result: Option>, + predecessor_error: Option, + claimed: bool, + published: Option, + successors: Links, +} + +impl State { + fn claim(&mut self) -> Option> { + if self.remaining != 0 || self.claimed { + return None; + } + let result = self.result.take()?; + self.claimed = true; + Some(match (result, self.predecessor_error) { + (Ok(_), Some(error)) => Err(error), + (result, _) => result, + }) + } +} + +#[pin_data] +pub(crate) struct Publication { + output: Output, + status: Arc, + #[pin] + state: Mutex, +} + +impl Publication { + pub(super) fn new(output: Output, status: Arc) -> Result> { + Arc::pin_init( + pin_init!(Self { + output, + status, + state <- new_mutex!(State { + // Acceptance installs all links before releasing this sentinel. + remaining: 1, + result: None, + predecessor_error: None, + claimed: false, + published: None, + successors: None, + }, "G17 host timestamp publication"), + }), + GFP_KERNEL, + ) + } + + pub(super) fn output(&self) -> &Output { + &self.output + } + + /// One terminal result, offered separately from its physical-retirement proof. + pub(super) fn finish(&self, result: Result<[u64; 4]>) { + let ready = { + let mut state = self.state.lock(); + if state.claimed || state.result.is_some() { + return; + } + state.result = Some(result); + state.claim() + }; + if let Some(result) = ready { + Self::drain(self.publish(result)); + } + } + + fn publish(&self, result: Result<[u64; 4]>) -> Links { + if let Err(error) = result { + // Inherited failure is visible to the client without inventing a + // permanent admission failure in this command's VM. + self.status.report_failure(error); + } + self.output.publish(result); + let result = result.map(|_| ()); + let mut links = { + let mut state = self.state.lock(); + // A late link is acknowledged only after timestamp writes, the public + // fence and aggregate membership have all been published off this lock. + state.published = Some(result); + state.successors.take() + }; + let mut next = links.as_mut(); + while let Some(link) = next { + link.result = result; + next = link.next.as_mut(); + } + links + } + + fn predecessor_done(&self, result: Result) -> Links { + let ready = { + let mut state = self.state.lock(); + debug_assert!(state.remaining != 0); + if let Err(error) = result { + if state.predecessor_error.is_none() { + state.predecessor_error = Some(inherited_error(error)); + } + } + state.remaining -= 1; + state.claim() + }; + ready.and_then(|result| self.publish(result)) + } + + fn add_successor(&self, mut link: KBox) { + let published = { + let mut state = self.state.lock(); + match state.published { + Some(result) => result, + None => { + link.next = state.successors.take(); + state.successors = Some(link); + return; + } + } + }; + Self::drain(link.target.predecessor_done(published)); + } + + /// Iterative depth-first notification, O(nodes + links), with no stack growth. + fn drain(mut pending: Links) { + let mut frames: Links = None; + loop { + if let Some(mut link) = pending.take() { + pending = link.next.take(); + let children = link.target.predecessor_done(link.result); + if children.is_some() { + link.resume = pending.take(); + link.next = frames.take(); + frames = Some(link); + pending = children; + } + } else if let Some(mut frame) = frames.take() { + frames = frame.next.take(); + pending = frame.resume.take(); + } else { + break; + } + } + } +} + +/// Temporary backwards references exist only during fallible setup. Installed +/// links retain the successor only, so closing a queue cannot form an Arc cycle. +pub(crate) struct Prepared { + target: Arc, + links: KVec<(Arc, KBox)>, +} + +impl Prepared { + pub(crate) fn new<'a>( + target: Arc, + predecessors: impl Iterator> + Clone, + ) -> Result { + let mut links = KVec::with_capacity(predecessors.clone().count(), GFP_KERNEL)?; + for predecessor in predecessors { + let link = KBox::new(Link { + target: target.clone(), + result: Ok(()), + next: None, + resume: None, + }, GFP_KERNEL)?; + links.push((predecessor.clone(), link), GFP_KERNEL)?; + } + Ok(Self { target, links }) + } + + /// Called after all fallible scheduler setup and before any GPU publication. + pub(crate) fn activate(self) { + { + let mut state = self.target.state.lock(); + debug_assert!(state.remaining == 1 && !state.claimed); + // The fully allocated link vector bounds this count below usize::MAX. + state.remaining += self.links.len(); + } + for (predecessor, link) in self.links { + predecessor.add_successor(link); + } + Publication::drain(self.target.predecessor_done(Ok(()))); + } +} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 649a28085a75b..36dc49c144da6 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -316,6 +316,7 @@ pub(crate) struct Scheduler { pub(crate) struct Fences { pub(crate) ready: Fence, pub(crate) completed: Fence, + pub(crate) publication: Option>, } impl Scheduler { @@ -338,9 +339,18 @@ impl Scheduler { packet: Arc, dependencies: KVec, early: bool, + publication: Option, ) -> Result { let completed = packet.completion.fence(); let host_timestamps = packet.completion.has_timestamps(); + // Only commands which actually write timestamps become host-prefix + // producers. Ordinary consumers retain the scheduled ready fence and + // their existing firmware dependency path for cross-engine overlap. + let host_publication = if host_timestamps { + packet.completion.publication() + } else { + None + }; let mut job = self.entity.new_job( 1, Job { @@ -354,6 +364,9 @@ impl Scheduler { } let mut job = job.arm(); packet.completion.mark_accepted(); + if let Some(publication) = publication { + publication.activate(); + } let ready = if host_timestamps { completed.clone() } else { @@ -365,6 +378,6 @@ impl Scheduler { self.backend.count_backlog(&packet); } job.push(); - Ok(Fences { ready, completed }) + Ok(Fences { ready, completed, publication: host_publication }) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index d382381868555..47b6cb18a8271 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -8,7 +8,7 @@ pub(crate) mod render; use super::{ command::{Parser, UscWindow, Validated}, - completion::{Completion, Destinations}, + completion::{Completion, Destinations, PreparedPublication}, context::Context, fence::{Outputs, Submission}, job::{self, Backend, Engine, Fences, Order, Packet, Scheduler}, @@ -176,6 +176,15 @@ impl Queue { if engine == Engine::Compute { self.ensure_compute()?; } + // Only earlier render ordinals covered by the explicit logical prefix + // can delay host publication. These links never become GPU dependencies. + let host_predecessors = self.frontiers[index] + .iter() + .take_while(|(sequence, _)| { + engine == Engine::Render + && Order::contains(order.wait_through[index], *sequence) + }) + .filter_map(|(_, fences)| fences.publication.as_ref()); let completion = Completion::new( &self.dev, &self.fences, @@ -184,8 +193,12 @@ impl Queue { guard, destinations, engine == Engine::Compute, + host_predecessors.clone().next().is_some(), if engine == Engine::Render { self.backend.feed() } else { None }, )?; + let publication = completion.publication() + .map(|target| PreparedPublication::new(target, host_predecessors)) + .transpose()?; if engine == Engine::Compute { self.prune_frontier(Engine::Render.index()); } @@ -275,7 +288,7 @@ impl Queue { let fences = scheduler .as_mut() .ok_or(ENODEV)? - .enqueue(packet, dependencies, early)?; + .enqueue(packet, dependencies, early, publication)?; self.sequence[index] = order.sequence; self.frontiers[index] .push_within_capacity((order.sequence, fences)) diff --git a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs index d1ec0eb76c7e1..be1d9fc9c902c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs @@ -40,7 +40,7 @@ fn include( frontier.include( prefix, packet.order.sequence, - job::fence_status(&packet.completion.fence()), + job::fence_status(&packet.completion.scheduler_fence()), word, ) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 3fb40779021e5..7f204b9328ffb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -41,7 +41,7 @@ impl Backend { Err(error) } else { deferred.finish(); - Ok(packet.completion.fence()) + Ok(packet.completion.scheduler_fence()) } } @@ -115,7 +115,7 @@ impl job::Backend for Backend { fn publish(&self, packet: Arc) -> Result { if let Some(result) = packet.early_result() { - return result.map(|()| packet.completion.fence()); + return result.map(|()| packet.completion.scheduler_fence()); } let mut deferred = match DeferredBatch::publication() { Ok(batch) => batch, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index 2cf7a5d77ee5a..0a4634bccaea5 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -26,7 +26,7 @@ impl Backend { result } pub(in crate::g17) fn cancel_render(&self, packet: &Arc) -> Result { - let fence = packet.completion.fence(); + let fence = packet.completion.scheduler_fence(); // SAFETY: The retained fence remains alive across this bounded, // uninterruptible queue-teardown wait, without holding a device lock. let completed = unsafe { From 918d1207f8747c4a94bae1f1304cd44374e5f204 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 20:57:47 +1100 Subject: [PATCH 0765/2680] soc: apple: compose SEP persistence with authenticated sensor sessions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retain the shared key-store profiles, persistence fields, suspend capture handling and module declarations when adding J700 nonce authentication. Choose the existing communication initialization word for each protocol and keep the qualified older-platform sensor-registration path alongside the authenticated J700 path. The composition uses the public SEP work in pull requests #42, #69, #152 and #202 and their reviewed repairs. Contributor credit includes Chris Kearney, DjDeveloperr, Vincent Fontaine and Ryan Murray. Ryan integrates the existing contributions under DCO (b); original commits and author sign-offs are unchanged. [ryan: rebased onto the linear SEP history — the contributor commits this composition drew on (#152 by Chris Kearney, Brent Kearney, DjDeveloperr and Vincent Fontaine; #69 by Chris Kearney, Christian Bartels and Tim) are carried above under their own authorship, so this commit keeps only what the composition itself added on top of them. It also takes tools/aurora-sep/XART-SAFETY.md as the #152 merge (commit 481c02bd687e ("Merge pull request #152 from ftechhelp/t8112-touchid")) left it, which the Arch package checksums (commit 8087ed3a7ccb ("tools: aurora-sep: add Fedora and Arch Linux packaging")) are computed over. The resulting tree matches aurora-wip at this commit] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/p256_shim.c | 1 - drivers/soc/apple/pm.rs | 13 +++- drivers/soc/apple/pm_shim.c | 9 +-- drivers/soc/apple/sbio.rs | 7 +- drivers/soc/apple/sensor_shim.c | 1 - drivers/soc/apple/sep-bio.h | 130 ++++++++++++++++++++++++++++++++ drivers/soc/apple/shim.h | 2 +- tools/aurora-sep/XART-SAFETY.md | 12 ++- 8 files changed, 157 insertions(+), 18 deletions(-) create mode 100644 drivers/soc/apple/sep-bio.h diff --git a/drivers/soc/apple/p256_shim.c b/drivers/soc/apple/p256_shim.c index bcf1323e3c58f..837a2e854faf4 100644 --- a/drivers/soc/apple/p256_shim.c +++ b/drivers/soc/apple/p256_shim.c @@ -161,4 +161,3 @@ int sep_p256_sender(const void *peer_pub_be, void *eph_pub_be_out, { return p256_agree(NULL, peer_pub_be, eph_pub_be_out, shared_be_out); } - diff --git a/drivers/soc/apple/pm.rs b/drivers/soc/apple/pm.rs index c613a516a067e..27b120d8f77ee 100644 --- a/drivers/soc/apple/pm.rs +++ b/drivers/soc/apple/pm.rs @@ -21,26 +21,31 @@ use crate::SepData; static SEP: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); extern "C" { - fn sep_pm_register(event: unsafe extern "C" fn(bool)) -> c_int; + fn sep_pm_register(event: unsafe extern "C" fn(bool) -> c_int) -> c_int; fn sep_pm_unregister(); } /// # Safety /// Called by `pm_shim.c` from the PM notifier chain, in process context. -unsafe extern "C" fn pm_event(entering: bool) { +unsafe extern "C" fn pm_event(entering: bool) -> c_int { let ptr = SEP.load(Ordering::Acquire); if ptr.is_null() { - return; + return 0; } // SAFETY: produced by `into_foreign` in `register`, and reclaimed by // `unregister` only after `sep_pm_unregister` has waited out any running // notifier call, so it is live for this borrow. let sep = unsafe { as ForeignOwnable>::borrow(ptr) }; if entering { - sep.sleep_prepare(); + if let Err(error) = sep.sleep_prepare() { + // A failing notifier is not included in the PM chain's rollback. + sep.sleep_finished(); + return error.to_errno(); + } } else { sep.sleep_finished(); } + 0 } pub(crate) fn register(sep: Arc) -> Result<()> { diff --git a/drivers/soc/apple/pm_shim.c b/drivers/soc/apple/pm_shim.c index e413bf2594e6a..34dd2b3dcf50d 100644 --- a/drivers/soc/apple/pm_shim.c +++ b/drivers/soc/apple/pm_shim.c @@ -10,12 +10,12 @@ #include "shim.h" -static void (*sep_pm_event)(bool entering); +static int (*sep_pm_event)(bool entering); static int sep_pm_notify(struct notifier_block *nb, unsigned long action, void *data) { - void (*event)(bool entering) = READ_ONCE(sep_pm_event); + int (*event)(bool entering) = READ_ONCE(sep_pm_event); if (!event) return NOTIFY_DONE; @@ -24,8 +24,7 @@ static int sep_pm_notify(struct notifier_block *nb, unsigned long action, case PM_SUSPEND_PREPARE: case PM_HIBERNATION_PREPARE: case PM_RESTORE_PREPARE: - event(true); - return NOTIFY_OK; + return notifier_from_errno(event(true)); case PM_POST_SUSPEND: case PM_POST_HIBERNATION: case PM_POST_RESTORE: @@ -40,7 +39,7 @@ static struct notifier_block sep_pm_nb = { .notifier_call = sep_pm_notify, }; -int sep_pm_register(void (*event)(bool entering)) +int sep_pm_register(int (*event)(bool entering)) { int ret; diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index df8e0719cc986..2a19dd2ac216d 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2760,7 +2760,7 @@ impl SepData { /// The system is about to suspend. End any capture in progress, hold new /// ones, and wait for the enrol or verify work to stop touching the sensor /// before devices go down. - pub(crate) fn sleep_prepare(&self) { + pub(crate) fn sleep_prepare(&self) -> Result<()> { let woke = { let mut session = self.bio_session.lock(); self.suspending.store(true, Relaxed); @@ -2779,14 +2779,15 @@ impl SepData { if waited >= SLEEP_DRAIN_MS { dev_warn!( self.dev, - "Touch ID: a capture was still running {} ms after the system started to sleep; continuing\n", + "Touch ID: capture did not drain within {} ms; aborting suspend\n", SLEEP_DRAIN_MS ); - return; + return Err(EBUSY); } kernel::time::delay::fsleep(kernel::time::Delta::from_millis(10)); waited += 10; } + Ok(()) } /// The system has resumed, or the suspend was aborted. diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index e52acb69b04f0..85190a5e14e7d 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -20,7 +20,6 @@ #include #include -#include #include "shim.h" diff --git a/drivers/soc/apple/sep-bio.h b/drivers/soc/apple/sep-bio.h new file mode 100644 index 0000000000000..d52b880ac9332 --- /dev/null +++ b/drivers/soc/apple/sep-bio.h @@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ +/* + * Userspace interface for the SEP biometric device. + * + * The enclave does the matching. Nothing biometric crosses this interface: only + * an operation, a stage, a status, an opaque identity UUID, and an opaque host + * label userspace chooses. + */ + +#pragma once + +#include +#include + +#define SEP_BIO_IFACE_VERSION 4 + +#define SEP_BIO_UUID_LEN 16 +#define SEP_BIO_LABEL_LEN 128 +#define SEP_BIO_NONCE_LEN 32 +#define SEP_BIO_TOKEN_LEN 32 +#define SEP_BIO_MAX_IDENTITIES 32 +#define SEP_BIO_CHALLENGE_LEN 32 +#define SEP_BIO_ATTEST_PUB_LEN 65 +#define SEP_BIO_ATTEST_SIG_MAX 72 + +enum { + SEP_BIO_STATE_IDLE = 0, + SEP_BIO_STATE_PENDING = 1, + SEP_BIO_STATE_PROGRESS = 2, + SEP_BIO_STATE_DONE = 3, + SEP_BIO_STATE_FAILED = 4, +}; + + +enum { + SEP_BIO_NO_MATCH = 0, + SEP_BIO_MATCH = 1, + SEP_BIO_NOT_COMPARED = 2, +}; + +struct sep_bio_identity { + __u8 uuid[SEP_BIO_UUID_LEN]; + __u8 label[SEP_BIO_LABEL_LEN]; +}; + +struct sep_bio_info { + __u32 version; + __u32 sensor_present; + __u32 enrolled; + __u32 capacity; /* usable slots in this SEP user context, not ABI list size */ + __u32 enroll_stages; + __u32 reserved[3]; +}; + +/* reserved[0] flags; reserved[1] is a live SEP identity count only when valid. */ +#define SEP_BIO_INFO_LIVE_COUNT_VALID 1u + +struct sep_bio_list { + __u32 count; + __u32 reserved; + struct sep_bio_identity id[SEP_BIO_MAX_IDENTITIES]; +}; + +struct sep_bio_enrol_start { + __u32 flags; + __u32 reserved; + __u8 label[SEP_BIO_LABEL_LEN]; +}; + +#define SEP_BIO_GUIDANCE_NONE 0 +#define SEP_BIO_GUIDANCE_PLACE 1 +#define SEP_BIO_GUIDANCE_LIFT_AND_MOVE 2 +#define SEP_BIO_GUIDANCE_HOLD_STILL 3 + +struct sep_bio_enrol_poll { + __u32 state; + __u32 stage; + __u32 stages_total; + __u32 status; + __u8 uuid[SEP_BIO_UUID_LEN]; + __u32 guidance; + __u32 progress_percent; +}; +struct sep_bio_verify_start { + __u32 flags; + __u32 reserved; + __u8 nonce[SEP_BIO_NONCE_LEN]; +}; + +struct sep_bio_verify_poll { + __u32 state; + __u32 result; + __u32 status; + __u32 guidance; /* SEP_BIO_GUIDANCE_*; status only, no capture side effect */ + __u8 uuid[SEP_BIO_UUID_LEN]; + __u8 token[SEP_BIO_TOKEN_LEN]; /* single use, bound to the nonce */ + __u64 deadline_ns; /* CLOCK_MONOTONIC; past this the token is void */ +}; + +struct sep_bio_delete { + __u8 uuid[SEP_BIO_UUID_LEN]; +}; + +/* + * Device attestation of key possession: the enclave signs 'challenge' with the + * machine ref-key (ECDSA-P256 over the challenge as the pre-computed digest) and + * returns the DER signature and public point. The private key never leaves the + * enclave. + */ +struct sep_bio_attest { + __u32 sig_len; /* out: DER signature length */ + __u8 challenge[SEP_BIO_CHALLENGE_LEN]; /* in */ + __u8 public[SEP_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ + __u8 signature[SEP_BIO_ATTEST_SIG_MAX]; /* out: DER SEQUENCE{r,s} */ + __u8 reserved[3]; +}; + +#define SEP_BIO_IOC_MAGIC 0xB1 + +#define SEP_BIO_GET_INFO _IOR (SEP_BIO_IOC_MAGIC, 0x01, struct sep_bio_info) +#define SEP_BIO_LIST _IOR (SEP_BIO_IOC_MAGIC, 0x02, struct sep_bio_list) +#define SEP_BIO_ENROL_START _IOW (SEP_BIO_IOC_MAGIC, 0x03, struct sep_bio_enrol_start) +#define SEP_BIO_ENROL_POLL _IOR (SEP_BIO_IOC_MAGIC, 0x04, struct sep_bio_enrol_poll) +#define SEP_BIO_VERIFY_START _IOW (SEP_BIO_IOC_MAGIC, 0x05, struct sep_bio_verify_start) +#define SEP_BIO_VERIFY_POLL _IOR (SEP_BIO_IOC_MAGIC, 0x06, struct sep_bio_verify_poll) +#define SEP_BIO_CANCEL _IO (SEP_BIO_IOC_MAGIC, 0x07) +#define SEP_BIO_DELETE _IOW (SEP_BIO_IOC_MAGIC, 0x08, struct sep_bio_delete) +#define SEP_BIO_DELETE_ALL _IO (SEP_BIO_IOC_MAGIC, 0x09) +#define SEP_BIO_ATTEST _IOWR(SEP_BIO_IOC_MAGIC, 0x0a, struct sep_bio_attest) diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index 99bc9d68b8be1..b8592883ce2d3 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -25,7 +25,7 @@ void sep_cancel_delayed_work_sync(void *work); /* -- pm_shim.c ---------------------------------------------------------- */ -int sep_pm_register(void (*event)(bool entering)); +int sep_pm_register(int (*event)(bool entering)); void sep_pm_unregister(void); /* -- hwrng_shim.c ------------------------------------------------------- */ diff --git a/tools/aurora-sep/XART-SAFETY.md b/tools/aurora-sep/XART-SAFETY.md index be7d5229748f7..52a976c63e5c4 100644 --- a/tools/aurora-sep/XART-SAFETY.md +++ b/tools/aurora-sep/XART-SAFETY.md @@ -20,15 +20,21 @@ state. Do not enable it on other installations without equivalent checks. locator failure: the highest-scoring window begins 90 APFS blocks after the true file start and extends beyond `.gl`. This proves ambiguity in that locator, **not** the cause of the previously damaged installation. +- Static inspection of the macOS 27.2 AppleSEPManager kernel collection found + `gl_rec_write` allocating a fresh 0x9000-byte slot, writing it, then clearing + 0x1000 bytes at the former slot. The low-level write path also calls a disk + cache-synchronization ioctl. `gl_fixup_rev_init` validates records and + handles duplicates during open. These observations do not prove that our + APFS ownership and failure ordering match macOS. ## Before authorizing writes -1. Confirm the raw-device access and extent-locking behaviour on hardware, +1. Confirm the exact macOS vnode/raw-device and extent-locking contract, including whether any APFS copy-on-write, snapshot, encryption, or remap state can invalidate the physical mapping during Linux's lifetime. 2. Verify record ordering, duplicate selection, failed-write recovery, - barriers/cache synchronization, and revision behavior in controlled - non-production tests. + barriers/cache synchronization, and revision behavior against both macOS + code and controlled non-production tests. 3. Compare a pre/post macOS-and-Linux power-cycle trace, with APFS checksums, `.gl` mapping and records checked before and after. Keep a recoverable image of the iBoot partition before the first write experiment. From 2749c0ba07e058888154e6ad5013bbf17ae83291 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:20:17 +1100 Subject: [PATCH 0766/2680] soc: apple: sep: remove the resurrected private copy of the UAPI header drivers/soc/apple/sep-bio.h became include/uapi/linux/apple-sep.h when the biometric interface was exported, but composing the persistence work with the authenticated sensor sessions brought the old private copy back. Nothing includes it: the Rust driver mirrors the layout itself, and user space uses the UAPI header. It only gives the interface a second definition that has to be kept in step by hand. Remove it again. Fixes: 42ae23434d4c ("soc: apple: compose SEP persistence with authenticated sensor sessions") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sep-bio.h | 130 ------------------------------------ 1 file changed, 130 deletions(-) delete mode 100644 drivers/soc/apple/sep-bio.h diff --git a/drivers/soc/apple/sep-bio.h b/drivers/soc/apple/sep-bio.h deleted file mode 100644 index d52b880ac9332..0000000000000 --- a/drivers/soc/apple/sep-bio.h +++ /dev/null @@ -1,130 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* Copyright 2026 Dj */ -/* - * Userspace interface for the SEP biometric device. - * - * The enclave does the matching. Nothing biometric crosses this interface: only - * an operation, a stage, a status, an opaque identity UUID, and an opaque host - * label userspace chooses. - */ - -#pragma once - -#include -#include - -#define SEP_BIO_IFACE_VERSION 4 - -#define SEP_BIO_UUID_LEN 16 -#define SEP_BIO_LABEL_LEN 128 -#define SEP_BIO_NONCE_LEN 32 -#define SEP_BIO_TOKEN_LEN 32 -#define SEP_BIO_MAX_IDENTITIES 32 -#define SEP_BIO_CHALLENGE_LEN 32 -#define SEP_BIO_ATTEST_PUB_LEN 65 -#define SEP_BIO_ATTEST_SIG_MAX 72 - -enum { - SEP_BIO_STATE_IDLE = 0, - SEP_BIO_STATE_PENDING = 1, - SEP_BIO_STATE_PROGRESS = 2, - SEP_BIO_STATE_DONE = 3, - SEP_BIO_STATE_FAILED = 4, -}; - - -enum { - SEP_BIO_NO_MATCH = 0, - SEP_BIO_MATCH = 1, - SEP_BIO_NOT_COMPARED = 2, -}; - -struct sep_bio_identity { - __u8 uuid[SEP_BIO_UUID_LEN]; - __u8 label[SEP_BIO_LABEL_LEN]; -}; - -struct sep_bio_info { - __u32 version; - __u32 sensor_present; - __u32 enrolled; - __u32 capacity; /* usable slots in this SEP user context, not ABI list size */ - __u32 enroll_stages; - __u32 reserved[3]; -}; - -/* reserved[0] flags; reserved[1] is a live SEP identity count only when valid. */ -#define SEP_BIO_INFO_LIVE_COUNT_VALID 1u - -struct sep_bio_list { - __u32 count; - __u32 reserved; - struct sep_bio_identity id[SEP_BIO_MAX_IDENTITIES]; -}; - -struct sep_bio_enrol_start { - __u32 flags; - __u32 reserved; - __u8 label[SEP_BIO_LABEL_LEN]; -}; - -#define SEP_BIO_GUIDANCE_NONE 0 -#define SEP_BIO_GUIDANCE_PLACE 1 -#define SEP_BIO_GUIDANCE_LIFT_AND_MOVE 2 -#define SEP_BIO_GUIDANCE_HOLD_STILL 3 - -struct sep_bio_enrol_poll { - __u32 state; - __u32 stage; - __u32 stages_total; - __u32 status; - __u8 uuid[SEP_BIO_UUID_LEN]; - __u32 guidance; - __u32 progress_percent; -}; -struct sep_bio_verify_start { - __u32 flags; - __u32 reserved; - __u8 nonce[SEP_BIO_NONCE_LEN]; -}; - -struct sep_bio_verify_poll { - __u32 state; - __u32 result; - __u32 status; - __u32 guidance; /* SEP_BIO_GUIDANCE_*; status only, no capture side effect */ - __u8 uuid[SEP_BIO_UUID_LEN]; - __u8 token[SEP_BIO_TOKEN_LEN]; /* single use, bound to the nonce */ - __u64 deadline_ns; /* CLOCK_MONOTONIC; past this the token is void */ -}; - -struct sep_bio_delete { - __u8 uuid[SEP_BIO_UUID_LEN]; -}; - -/* - * Device attestation of key possession: the enclave signs 'challenge' with the - * machine ref-key (ECDSA-P256 over the challenge as the pre-computed digest) and - * returns the DER signature and public point. The private key never leaves the - * enclave. - */ -struct sep_bio_attest { - __u32 sig_len; /* out: DER signature length */ - __u8 challenge[SEP_BIO_CHALLENGE_LEN]; /* in */ - __u8 public[SEP_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ - __u8 signature[SEP_BIO_ATTEST_SIG_MAX]; /* out: DER SEQUENCE{r,s} */ - __u8 reserved[3]; -}; - -#define SEP_BIO_IOC_MAGIC 0xB1 - -#define SEP_BIO_GET_INFO _IOR (SEP_BIO_IOC_MAGIC, 0x01, struct sep_bio_info) -#define SEP_BIO_LIST _IOR (SEP_BIO_IOC_MAGIC, 0x02, struct sep_bio_list) -#define SEP_BIO_ENROL_START _IOW (SEP_BIO_IOC_MAGIC, 0x03, struct sep_bio_enrol_start) -#define SEP_BIO_ENROL_POLL _IOR (SEP_BIO_IOC_MAGIC, 0x04, struct sep_bio_enrol_poll) -#define SEP_BIO_VERIFY_START _IOW (SEP_BIO_IOC_MAGIC, 0x05, struct sep_bio_verify_start) -#define SEP_BIO_VERIFY_POLL _IOR (SEP_BIO_IOC_MAGIC, 0x06, struct sep_bio_verify_poll) -#define SEP_BIO_CANCEL _IO (SEP_BIO_IOC_MAGIC, 0x07) -#define SEP_BIO_DELETE _IOW (SEP_BIO_IOC_MAGIC, 0x08, struct sep_bio_delete) -#define SEP_BIO_DELETE_ALL _IO (SEP_BIO_IOC_MAGIC, 0x09) -#define SEP_BIO_ATTEST _IOWR(SEP_BIO_IOC_MAGIC, 0x0a, struct sep_bio_attest) From ecb932507280785fb1c9345c6c0ae1b3af5256ed Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:20:20 +1100 Subject: [PATCH 0767/2680] soc: apple: sep: rename the attestation public key member struct sep_bio_attest calls the attested public point "public", which is a C++ keyword. The UAPI header test compiles every exported header as C++ as well as C, so a configuration with CONFIG_UAPI_HEADER_TEST=y fails to build: ./usr/include/linux/apple-sep.h:115:8: error: expected member name or ';' after declaration specifiers and C++ programs cannot include the header at all. Rename the member to public_key, and rename the Rust mirror of the structure to match. Only the member name changes: every structure keeps its size, alignment and member offsets, and the ioctl numbers are unchanged. No in-tree user space accesses the member. Fixes: 5b4a74c4e0aa ("soc: apple: sep: export the biometric ioctl header through UAPI") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/bio.rs | 2 +- drivers/soc/apple/sbio.rs | 4 ++-- include/uapi/linux/apple-sep.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index 13d7e99d43fc9..752e9ea7a6e12 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -149,7 +149,7 @@ pub(crate) struct Attest { /// in: challenge to sign. pub(crate) challenge: [u8; ATTEST_CHALLENGE_LEN], /// out: ref-key public point. - pub(crate) public: [u8; ATTEST_PUB_LEN], + pub(crate) public_key: [u8; ATTEST_PUB_LEN], /// out: DER signature, zero-padded to `ATTEST_SIG_MAX`. pub(crate) signature: [u8; ATTEST_SIG_MAX], pub(crate) reserved: [u8; 3], diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 2a19dd2ac216d..b45158ad5502d 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2948,11 +2948,11 @@ impl SepData { let mut out = bio::Attest { sig_len: sig.len() as u32, challenge: req.challenge, - public: [0u8; bio::ATTEST_PUB_LEN], + public_key: [0u8; bio::ATTEST_PUB_LEN], signature: [0u8; bio::ATTEST_SIG_MAX], reserved: [0u8; 3], }; - out.public.copy_from_slice(&pubk); + out.public_key.copy_from_slice(&pubk); out.signature[..sig.len()].copy_from_slice(&sig); kernel::uaccess::UserSlice::new(user, core::mem::size_of::()) .writer() diff --git a/include/uapi/linux/apple-sep.h b/include/uapi/linux/apple-sep.h index 81467fe91f36a..394b497ff7a59 100644 --- a/include/uapi/linux/apple-sep.h +++ b/include/uapi/linux/apple-sep.h @@ -112,7 +112,7 @@ struct sep_bio_delete { struct sep_bio_attest { __u32 sig_len; /* out: DER signature length */ __u8 challenge[SEP_BIO_CHALLENGE_LEN]; /* in */ - __u8 public[SEP_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ + __u8 public_key[SEP_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ __u8 signature[SEP_BIO_ATTEST_SIG_MAX]; /* out: DER SEQUENCE{r,s} */ __u8 reserved[3]; }; From 8e9e0d33234ddb844e1f8b3a009c232835b4e7ec Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 21:57:12 +1100 Subject: [PATCH 0768/2680] soc: apple: sep: rearm the cold-restore device-view proof The nonce-session restore path synchronizes the device view before COMPLETE_INIT to refresh the SCRD credential. Deferred restore readiness also uses device_view_synced to guard its final proof after COMPLETE_INIT. The preliminary sync leaves that flag set, so both finish_sensor_setup and retry_restore_proof skip the final proof and templates_restored remains false after a successful cold restore. Clear device_view_synced when deferring template readiness. Keep the preliminary credential refresh, but require the final device-view proof before marking templates ready. A failed final sync can still take the existing bounded sensor bring-up retry. Legacy profiles retain their normal proof path, and refused or empty identity proofs stay unready. Fixes: 42ae23434d4c ("soc: apple: compose SEP persistence with authenticated sensor sessions") (cherry picked from commit b4c0368990089654d5e1212926473137ac314956) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index b45158ad5502d..deff86802e440 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -3017,6 +3017,9 @@ impl SepData { // verification available on the preliminary Catacomb proof alone; // complete_bringup finishes the proof and sets templates_restored. self.templates_restored.store(false, Relaxed); + // The nonce path synced the view before COMPLETE_INIT to refresh SCRD. + // Re-arm the final proof and its bounded retry on every cold restore. + self.device_view_synced.store(false, Relaxed); if !candidate { dev_err!( self.dev, From a557544de7c6455645c184bdfaa63f5490989e37 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 22:12:20 +1100 Subject: [PATCH 0769/2680] treewide: describe hardware behaviour in comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrite a set of comments so they state the hardware and firmware behaviour directly: - soc/apple SEP (sks.rs, image.rs, profile.rs, sep.rs): key-store request layouts, timeouts and endpoint init described as the enclave protocol. - soc/apple mailbox: the ASC wrapper CPU start/stop sequences. - accel/ane: register, handshake, ring and command-format notes for the T602x/T8112 Neural Engine, plus the scratch3_ack parameter text. - drm/asahi G17: the tiler-heap shrink policy. - mt7932, reset-apple-cio and two documentation pages: keep the descriptions self-contained. Every edit keeps its line count, so line-dependent data (panic locations, WARN/BUG and dynamic-debug records) is unchanged. Apart from the scratch3_ack module parameter description, the change is comments only; there is no functional change. [ryan: adapted to 7.2.10 — split by topic: this commit carries only the SEP driver files (image.rs, profile.rs, sep.rs, sks.rs); the mailbox.c, ANE, AVE, MT7932, CIO reset and G17 hunks stay with their own topics (245-ane carries the rest)] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/image.rs | 4 +-- drivers/soc/apple/profile.rs | 8 ++--- drivers/soc/apple/sep.rs | 2 +- drivers/soc/apple/sks.rs | 64 ++++++++++++++++++------------------ 4 files changed, 39 insertions(+), 39 deletions(-) diff --git a/drivers/soc/apple/image.rs b/drivers/soc/apple/image.rs index 290a9ea43dd3e..c465ffddfbd70 100644 --- a/drivers/soc/apple/image.rs +++ b/drivers/soc/apple/image.rs @@ -60,8 +60,8 @@ static_assert!(HEADER_WIRE == 4 + HEADER_SIZE as usize); pub(crate) enum Version { V1, /// Negotiated from the `0x4d` capability word. Laid out as version 1, but - /// `__payload_hash` (0xfffffe000996b970) digests the header through the - /// trailer (0x…ba68-ba80) where version 1 stops before it (0x…ba84-ba90). + /// the payload hash covers the header through the + /// trailer, where version 1 stops before it. V2, } diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 7d8ced489a4a2..82ad95613c4b1 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -111,11 +111,11 @@ pub(crate) struct KeybagCreate { /// Which key-store protocol the enclave speaks. It follows the sepOS the /// firmware hands the SEP, not the SoC alone: the T8103 stub boots macOS 13.5 /// (22G74)'s sepOS, whose request shapes are those of the 13.5 -/// `AppleSEPKeyStore`; the T6020 sepOS is newer. +/// key store; the T6020 sepOS is newer. #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum KeyStore { /// macOS 13.5's key store. Before any other request the endpoint is - /// initialised as `AppleKeyStore::init_sep_endpoint` does it: `0x4d`, then + /// initialised with `0x4d`, then /// `set_env` carrying the ADT's `/defaults` `cpx-encryption-mode`. Sepos13 { cpx_encryption_mode: u32 }, /// The hardware-verified T6020 encoding (variant-5 create). @@ -169,7 +169,7 @@ const T8103: PlatformProfile = PlatformProfile { dart_range_required: false, wide_dma_mask: false, firmware_region: c"sepfw", - // macOS 13.5's AppleKeyStore::identity_create (0xfffffe000994b2d8): + // The 13.5 key store creates an identity bag with // version 2, type 0x400000, parent -1 (it accepts only -1 or <= -10). keybag_create: KeybagCreate { variant: 2, @@ -276,7 +276,7 @@ const T8140: PlatformProfile = PlatformProfile { wide_dma_mask: true, firmware_region: c"sepfw", // J700 hardware refused the strict encoding and accepted this T6020 form - // (linux-aurora commit 18165cf1de, 2026-09-19). + // (first verified on hardware 2026-09-19). keybag_create: KeybagCreate { variant: 5, bag_type: 0, diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index d6a76d20cb3d4..f328d0fe64bd3 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -123,7 +123,7 @@ const SCRD_MAX_CAPTURE: usize = 8; const SCRD_TIMEOUT_MS: time::Msecs = 2000; const SKS_TIMEOUT_MS: time::Msecs = 2000; -/// The 13.5 kext's per-try key-store wait. +/// The per-try wait for the 13.5 key store. const SKS_TIMEOUT_13_MS: time::Msecs = 6000; const SKS_TIMEOUT_PER_KIB_MS: time::Msecs = 6000; diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index 2d29fa7598577..f1fabe2791a76 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -164,8 +164,8 @@ impl SepData { } let sized_ms = sks_timeout_for(img.len()); - // The 13.5 key store is waited on as sep_deliver_msg_gated does: - // getAdjustedTimeout(0x1770), 6000 ms per wait (0xfffffe0009945a14). + // The 13.5 key store is given a 0x1770 (6000 ms) wait per delivered + // request. // On a j313 a CREATE took // 1.6 s and a COPY 1.66 s, close enough to the 2 s base that a slow // one would be abandoned mid-way through its xART writes. @@ -313,10 +313,10 @@ impl SepData { self.sks_seal(&op, &body) } - /// `0x2a` set_env, as `AppleKeyStore::set_env(false)` (0xfffffe000994a168) - /// builds it: a 0x40c-byte environment `{1, mode, 4, 0x400 zero bytes}` - /// (0x…a1a0-a1e0) sent by `__ipc_set_env` (0x…9ddc) as struct version 0, - /// u64 1, blob (`_code_ipc_set_env` 0x…4b64). + /// `0x2a` set_env in its "false" form, the environment the key store is + /// given at endpoint init: a 0x40c-byte environment + /// `{1, mode, 4, 0x400 zero bytes}`, sent as struct version 0, u64 1 and + /// the environment as a blob. fn sks_req_set_env(&self, mode: u32) -> Result { let op = crate::sks::sks_set_env(); let mut env = KVec::new(); @@ -331,10 +331,10 @@ impl SepData { self.sks_seal(&op, &body) } - /// `AppleKeyStore::init_sep_endpoint` (0xfffffe0009946cdc) on the 13.5 key - /// store, before any other request: `0x4d`, whose capability word sets the - /// IPC header version to `min(word, 2)` (0x…6dc0-6dd0), then `set_env` - /// (0x…6e64). Returns false if the key store must stay closed this boot. + /// Endpoint init for the 13.5 key store, which must precede any other + /// request: `0x4d`, whose capability word sets the IPC header version to + /// `min(word, 2)`, then `set_env`. + /// Returns false if the key store must stay closed this boot. /// The T6020 path sends nothing here. pub(crate) fn sks_init_endpoint(&self) -> bool { let profile::KeyStore::Sepos13 { @@ -354,7 +354,7 @@ impl SepData { .filter(|body| out.reply.status == 0 && body.len() >= 12) .map(|body| u64::from_le_bytes(body[4..12].try_into().unwrap())); match word { - // Version 0 leaves out the header fields __payload_hash needs. + // Version 0 leaves out the header fields the payload hash covers. Some(0) | None => { dev_err!( self.dev, @@ -396,9 +396,9 @@ impl SepData { true } - /// The designate struct version, which the reply echoes. 13.5's - /// `_code_ipc_make_system_keybag` (0xfffffe000995fd8c) takes version 0 - /// only (0x…fe78). + /// The designate struct version, which the reply echoes. On 13.5, + /// designate uses struct version 0; the other profile uses its own + /// variant. fn sks_designate_variant(&self) -> u32 { match self.profile.key_store { profile::KeyStore::Sepos13 { .. } => crate::sks::SKS_DESIGNATE_VERSION_13, @@ -406,8 +406,8 @@ impl SepData { } } - /// `0x0d` designate. On 13.5 this is `identity_load`'s make-system-keybag - /// (0xfffffe000994ba48-ba5c): source handle, session, an empty blob and + /// `0x0d` designate, which makes the system key bag. On 13.5 the body is: + /// source handle, session, an empty blob and /// nothing after it. fn sks_req_designate(&self, d: &crate::sks::Designation, secret: &[u8]) -> Result { let mut body = image::Body::new(); @@ -494,12 +494,12 @@ impl SepData { } /// `0x18` device-state transition to unlocked, the 13.5 identity-session - /// unlock. `AKSIdentityUnlockSession` (user-client selector 0x7b) reaches - /// `unlock_the_device(client, h, secret, 5)` (0xfffffe000993cee0, called at - /// 0x…15a4), which sends `device_state_transition` with state 0 and flags 0 - /// (0x…cf04-cf0c) through `__ipc_device_state_transition` - /// (0xfffffe0009958204): struct version 0 only (0x…200c), u64 client, u32 - /// handle, u32 state, u64 flags, blob secret. The reply is the version word + /// unlock: `device_state_transition` for the session's handle and secret, + /// with state 0 and flags 0. + /// + /// It uses struct version 0. The body is the version, u64 client, + /// u32 handle, u32 state, u64 flags and the secret as a blob. + /// The reply is the version word /// and two u64. fn sks_req_device_state_unlock( &self, @@ -580,10 +580,10 @@ impl SepData { } // The 13.5 (T8103) and T6020 enclaves encode the identity bag // differently; the per-SoC profile carries the field values (see - // profile::KeybagCreate). On 13.5 this is identity_create's - // __ipc_create_keybag_v2 (0xfffffe0009955ce0): secret, an empty second - // blob and the 16-byte identity UUID, nothing after it; the codec - // (_code_ipc_create_keybag 0xfffffe000995de6c) decodes exact-length. + // profile::KeybagCreate). On 13.5 the identity-bag create carries: + // secret, an empty second + // blob and the 16-byte identity UUID, nothing after it, so the body + // has exactly this length. let enc = &self.profile.keybag_create; let mut body = image::Body::new(); body.put_u32(enc.variant)?; @@ -688,8 +688,8 @@ impl SepData { self.sks_mark_create_refused(slot, out.reply.status); return false; }; - // 13.5 replies with the struct version and the handle alone - // (0x…e11c-e12c); the T6020 reply adds a blob. + // 13.5 replies with the struct version and the handle alone; + // the T6020 reply adds a blob. let reply_min = match self.profile.key_store { profile::KeyStore::Sepos13 { .. } => 8, profile::KeyStore::Variant5 => 12, @@ -731,8 +731,8 @@ impl SepData { return false; } let handle = crate::sks::KeyBagHandle::from_create_reply(raw_handle); - // identity_create reads no UUID back: 13.5 names the identity by the - // UUID it was created with, and identity_save (0xfffffe000994b1cc) is + // An identity create reads no UUID back: 13.5 names the identity by the + // UUID it was created with, and saving the identity is // just the copy below, then the unload. let bag_uuid = match self.profile.key_store { profile::KeyStore::Sepos13 { .. } => None, @@ -926,7 +926,7 @@ impl SepData { &self, stored: &keybag::StoredKeyBag, ) -> Option<(crate::sks::KeyBagHandle, [u8; keybag::UUID_LEN])> { - // 13.5 identity_open addresses an enclave-resident identity bag by + // Opening a 13.5 identity addresses an enclave-resident identity bag by // its generated UUID. Older J313 records instead store the bag's // read-back UUID and a wrapped bag, which must be loaded as a blob. let uuid_load = matches!(self.profile.key_store, profile::KeyStore::Sepos13 { .. }) @@ -1236,7 +1236,7 @@ pub(crate) fn sks_set_env() -> SksOp { } } pub(crate) const SKS_SET_ENV_LEN: usize = 12 + 0x400; -/// Third environment word for `set_env(false)`, the call init_sep_endpoint makes. +/// Third environment word for the "false" `set_env` sent at endpoint init. pub(crate) const SKS_SET_ENV_FALSE: u32 = 4; const OP_SKS_COPY_UUID: u8 = 0x06; From 33d515bba87f1df2297f67a52d730fe9cb5bb9c5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 01:44:15 +1100 Subject: [PATCH 0770/2680] dt-bindings: soc: apple: sep: Allow a power domain the SEP depends on On T8140 the SEP depends on the GPIO power state while it runs: once that state is switched off, the enclave stops reading its mailbox and does not recover until the next boot. Let the node list the power domain the SEP depends on after its own. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/soc/apple/apple,sep.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml b/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml index 989ee4b9e9078..83ad3aa31a4b7 100644 --- a/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml +++ b/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml @@ -33,7 +33,13 @@ properties: maxItems: 1 power-domains: - maxItems: 1 + minItems: 1 + items: + - description: The SEP's own power state. + - description: + A power state the SEP depends on while it runs. On T8140 this is + the GPIO power state; the SEP stops servicing its mailbox for good + once that is switched off. memory-region: maxItems: 1 From dd2a90a294034a0e3cb83ac94cca6fe2ba3d5e5d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 01:44:15 +1100 Subject: [PATCH 0771/2680] soc: apple: sep: Keep the enclave's power domains on across sleep On the J700 the SEP stops working after suspend-to-idle: hwrng reads return nothing, trusted keys can be sealed but no longer loaded ("sks: LOAD_KEYBAG got no reply"), and Touch ID cannot take its sensor back. Nothing short of a reboot recovers it. The enclave has stopped reading its mailbox: the AP-to-SEP FIFO fills up with every message sent after the fault and no reply ever comes back. It happens when the enclave is sent a message while devices are in the noirq phase of suspend, which the hwrng fill thread does: it is not freezable and keeps drawing on its timer through the whole sleep. Without a message in that window the SEP survives the sleep, and pm_test=devices with messages is fine where pm_test=platform with messages is not. In the noirq phase genpd switches off each power domain whose devices have all suspended. Refusing that for one domain at a time showed that the SEP needs the GPIO power state to stay on: with only that one kept on, it answers during the sleep and after it. The pin controller is the only Linux user of that domain, and the J700's device tree from Apple lists no power state dependency between the SEP and GPIO, so this is a dependency of the enclave as a consumer. Let a SEP node list the power domains the enclave depends on after its own, attach all of them when it lists more than one, keep them powered through device links for as long as the driver is bound, and mark the virtual domain devices as syscore devices. The links are not enough on their own: they order PM transitions and carry runtime PM, but in the noirq phase genpd runs its system sleep callbacks on every device attached to a domain, virtual ones included, counts each as suspended whatever its runtime state, and switches the domain off once all of them are. A syscore device is skipped by system sleep and so never counted, which is how the ISP keeps the domains of its resident firmware on. A node with a single domain is still attached by the platform core, so device trees without a second domain behave as before. On the J700, after each of ten 70 s suspend-to-idle cycles in which the hwrng thread drew from the SEP during the sleep, hwrng reads returned data, a trusted key could be sealed and loaded again, and a Touch ID verify started without sensor errors; genpd never switched the GPIO power state off. Before this change one such sleep was enough to wedge the enclave until reboot. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/pm.rs | 14 ++++++++++ drivers/soc/apple/pm_shim.c | 51 +++++++++++++++++++++++++++++++++++-- drivers/soc/apple/sep.rs | 5 ++++ drivers/soc/apple/shim.h | 1 + 4 files changed, 69 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/pm.rs b/drivers/soc/apple/pm.rs index 27b120d8f77ee..a517e6e448732 100644 --- a/drivers/soc/apple/pm.rs +++ b/drivers/soc/apple/pm.rs @@ -23,6 +23,20 @@ static SEP: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); extern "C" { fn sep_pm_register(event: unsafe extern "C" fn(bool) -> c_int) -> c_int; fn sep_pm_unregister(); + fn sep_pm_keep_domains(dev: *mut c_void) -> c_int; +} + +/// Attaches every power domain of a multi-domain SEP node and keeps them on +/// across system sleep; see `pm_shim.c`. Must run before the enclave is first +/// spoken to. +pub(crate) fn keep_domains(dev: &kernel::device::Device) -> Result<()> { + // SAFETY: `dev` is the bound SEP device; the shim ties what it attaches to + // that device's lifetime through devres. + let rc = unsafe { sep_pm_keep_domains(dev.as_raw().cast()) }; + if rc != 0 { + return Err(Error::from_errno(rc)); + } + Ok(()) } /// # Safety diff --git a/drivers/soc/apple/pm_shim.c b/drivers/soc/apple/pm_shim.c index 34dd2b3dcf50d..4a46088681767 100644 --- a/drivers/soc/apple/pm_shim.c +++ b/drivers/soc/apple/pm_shim.c @@ -1,11 +1,15 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* - * System-sleep notifier for the SEP driver. It only forwards "about to sleep" + * System sleep for the SEP driver: the notifier only forwards "about to sleep" * and "back from sleep" to Rust, which ends any Touch ID capture before the - * system suspends; see pm.rs. + * system suspends (see pm.rs), and the enclave's power domains are kept on + * across sleep. */ #include +#include #include +#include +#include #include #include "shim.h" @@ -56,3 +60,46 @@ void sep_pm_unregister(void) unregister_pm_notifier(&sep_pm_nb); WRITE_ONCE(sep_pm_event, NULL); } + +static void sep_pm_release_domains(void *data) +{ + struct dev_pm_domain_list *list = data; + u32 i; + + for (i = 0; i < list->num_pds; i++) + dev_pm_syscore_device(list->pd_devs[i], false); +} + +/* + * The enclave runs from boot to shutdown, and on some SoCs it needs power + * domains besides its own: once one of them is switched off, it stops + * reading its mailbox for good. A node that lists more than one domain gets + * all of them attached here, powered on through device links for as long as + * the driver is bound, and taken out of system sleep, which would otherwise + * switch a domain off in the noirq phase once its other users had suspended. + * A node with a single domain is attached by the platform core as before. + */ +int sep_pm_keep_domains(void *ptr) +{ + struct device *dev = ptr; + struct dev_pm_domain_attach_data data = { + .pd_flags = PD_FLAG_DEV_LINK_ON, + }; + struct dev_pm_domain_list *list; + int count, ret; + u32 i; + + count = of_count_phandle_with_args(dev->of_node, "power-domains", + "#power-domain-cells"); + if (count <= 1) + return 0; + + ret = devm_pm_domain_attach_list(dev, &data, &list); + if (ret < 0) + return ret; + + for (i = 0; i < list->num_pds; i++) + dev_pm_syscore_device(list->pd_devs[i], true); + + return devm_add_action_or_reset(dev, sep_pm_release_domains, list); +} diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index f328d0fe64bd3..b0d654e41ffd6 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -2275,6 +2275,11 @@ impl platform::Driver for SepDriver { return Err(EBUSY); } + if let Err(e) = pm::keep_domains(dev) { + dev_err!(dev, "could not keep the power domains on ({:?})\n", e); + return Err(e); + } + let data = SepData::new(pdev)?; *data.mbox.lock() = Some(Mailbox::new_byname(dev, c"mbox", data.clone())?); diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index b8592883ce2d3..e7711068cb4ef 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -27,6 +27,7 @@ void sep_cancel_delayed_work_sync(void *work); int sep_pm_register(int (*event)(bool entering)); void sep_pm_unregister(void); +int sep_pm_keep_domains(void *dev); /* -- hwrng_shim.c ------------------------------------------------------- */ From 15c6b0cc6faa4c0ff703db3de4fd028a6126c93b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 16:03:09 +1100 Subject: [PATCH 0772/2680] soc: apple: sep: let a start wait for the last capture to stop The enrol and verify work items report their result and wake the poller before they return and clear the session's busy marker. After a cancel or a close they stop only once the capture loop notices. An ENROL_START or VERIFY_START in that window, or an open right after a close, is refused with EBUSY. Userspace cannot tell that from a device in use by someone else: libfprint reports a protocol error, and fprintd turns it into verify-disconnected. fprintd starts a match again as soon as a capture is reported unusable, so it lands in that window. Hold such an open or start until the work item has stopped, for as long as a capture may take to drain before sleep. Only an open with no other opener, or a start whose last operation has ended, waits; a start while an operation is still running is refused at once, as before. If the work item has not stopped in time, the caller is refused as before, and the driver logs it. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/bio.rs | 7 ++++++ drivers/soc/apple/sbio.rs | 47 +++++++++++++++++++++++++++++++++++++-- drivers/soc/apple/sep.rs | 18 ++++++++++++--- 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index 752e9ea7a6e12..9ba7722f5cf4e 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -846,6 +846,13 @@ pub(crate) fn worker_stopped(session: &mut Session, epoch: u64) { } } +/// Whether an open (`opening`) or a capture start by the current opener would +/// be refused only because the last enrol or verify work item has not stopped +/// yet, although its operation has ended or its opener has gone. +pub(crate) fn waiting_on_worker(session: &Session, opening: bool) -> bool { + session.worker_epoch.is_some() && session.op.may_start() && session.open != opening +} + fn verify_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { let mut out = VerifyPoll::default(); // Spend the token only after the copy to user succeeds; a fault must not lose a match. diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index deff86802e440..9d626bb275a6d 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2719,6 +2719,7 @@ impl SepData { return Err(ENODEV); } let mut session = self.bio_session.lock(); + self.wait_for_worker_exit(&mut session, true)?; bio::open(&mut session)?; drop(session); if let Err(e) = self.prepare_bio_open() { @@ -2757,6 +2758,45 @@ impl SepData { self.captures_running.fetch_sub(1, Relaxed); } + /// The enrol or verify work item `epoch` has stopped. + pub(crate) fn capture_worker_stopped(&self, epoch: u64) { + bio::worker_stopped(&mut self.bio_session.lock(), epoch); + self.worker_gone.notify_all(); + } + + /// Holds an open or a capture start while the last enrol or verify work + /// item is still on its way out. It reports its result before it stops, + /// and after a cancel or a close it stops only once the capture notices, + /// so an open or a start straight after either would be refused with + /// EBUSY, which the caller cannot tell from a device in use. If it has + /// not stopped in time, the caller refuses as before. + fn wait_for_worker_exit( + &self, + session: &mut MutexGuard<'_, bio::Session>, + opening: bool, + ) -> Result<()> { + let mut remaining = time::msecs_to_jiffies(WORKER_EXIT_WAIT_MS); + while bio::waiting_on_worker(session, opening) { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } + if remaining == 0 { + dev_warn!( + self.dev, + "Touch ID: the last capture had not stopped after {} ms\n", + WORKER_EXIT_WAIT_MS + ); + return Ok(()); + } + match self.worker_gone.wait_interruptible_timeout(session, remaining) { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + } + } + Ok(()) + } + /// The system is about to suspend. End any capture in progress, hold new /// ones, and wait for the enrol or verify work to stop touching the sensor /// before devices go down. @@ -2821,7 +2861,7 @@ impl SepData { .is_err() { this.finish_enrolment(Err(ENROL_STATUS_SENSOR)); - bio::worker_stopped(&mut this.bio_session.lock(), epoch); + this.capture_worker_stopped(epoch); } } @@ -2838,7 +2878,7 @@ impl SepData { bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), [0u8; bio::TOKEN_LEN], ); - bio::worker_stopped(&mut this.bio_session.lock(), epoch); + this.capture_worker_stopped(epoch); } } @@ -2904,6 +2944,9 @@ impl SepData { let sensor_present = sensor::is_bound(); let handled = { let mut session = self.bio_session.lock(); + if bio::starts_capture(cmd) { + self.wait_for_worker_exit(&mut session, false)?; + } let index = self.bio_index.lock(); let mut ctx = bio::Context { session: &mut session, diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index b0d654e41ffd6..f8cc570652dcb 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -63,7 +63,8 @@ use kernel::{ Arc, CondVar, CondVarTimeoutResult, - Mutex, // + Mutex, + MutexGuard, // }, time, types::ForeignOwnable, @@ -113,6 +114,11 @@ const SBIO_TIMEOUT_MS: time::Msecs = 5000; const SLEEP_DRAIN_MS: u32 = 6000; static_assert!(SLEEP_DRAIN_MS as u64 > SBIO_TIMEOUT_MS as u64); +// How long an open or a capture start waits for the last enrol or verify work +// item to stop once its operation has ended. It is stopped in the same way, +// so it gets the same bound as a capture draining before sleep. +const WORKER_EXIT_WAIT_MS: time::Msecs = SLEEP_DRAIN_MS; + const SKS_ALLOC: usize = 0x8000; const SKS_SECRET_LEN: usize = 32; @@ -696,6 +702,10 @@ struct SepData { // Enrol and verify work items between `capture_begin` and `capture_end`. captures_running: Atomic, + // Signalled when an enrol or verify work item has stopped. + #[pin] + worker_gone: CondVar, + #[pin] rx_work: Work, @@ -870,6 +880,7 @@ impl SepData { suspending: Atomic::new(false), sleep_over <- new_condvar!("SepData::sleep_over"), captures_running: Atomic::new(0), + worker_gone <- new_condvar!("SepData::worker_gone"), rx_work <- new_work!("SepData::rx_work"), enrol_work <- new_work!("SepData::enrol_work"), verify_work <- new_work!("SepData::verify_work"), @@ -2020,6 +2031,7 @@ impl SepData { fn remove(&self) { self.shutting_down.store(true, Relaxed); bio::release(&mut self.bio_session.lock()); + self.worker_gone.notify_all(); self.rng_shutdown.store(true, Relaxed); self.control_wq.notify_all(); self.sbio_wq.notify_all(); @@ -2218,7 +2230,7 @@ impl WorkItem for SepData { this.run_enrolment(); this.capture_end(); } - bio::worker_stopped(&mut this.bio_session.lock(), epoch); + this.capture_worker_stopped(epoch); } } @@ -2231,7 +2243,7 @@ impl WorkItem for SepData { this.run_verify(); this.capture_end(); } - bio::worker_stopped(&mut this.bio_session.lock(), epoch); + this.capture_worker_stopped(epoch); } } From b2530141232621b0072879a03330de5a7d1459c9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 15:36:14 +1100 Subject: [PATCH 0773/2680] tools: aurora-sep: report unusable captures to libfprint as retries The kernel can tell userspace that an enrolment capture was not usable: the next poll while the enrolment runs carries SEP_BIO_STATUS_RETRY, and a run of unusable captures ends the enrolment with SEP_BIO_STATUS_REJECTED. A verify whose capture was not usable already ends with SEP_BIO_STATUS_RETRY. The aurora driver ignored the status while an enrolment ran and reported any failure only as a driver status number, so nobody was asked to touch the sensor again, and a match that compared nothing ended the authentication attempt as an error. Report a rejected enrolment capture through fpi_device_enroll_progress() with FP_DEVICE_RETRY_GENERAL, which fprintd shows as enroll-retry-scan. The kernel does not say why a capture was unusable, so the general code is the most specific one that is always true. Report a verify or identify that ended with RETRY as a retry too; fprintd answers that by starting the match again. Fail a rejected enrolment with a message that says why and what to do, and name the reason for the other failure statuses. The status values are defined here when the UAPI header predates them. With a kernel that never sends RETRY during an enrolment, nor REJECTED, the driver behaves as before. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 6 + .../libfprint-1.94.100-apple-sep.patch | 136 ++++++++++++++++-- 2 files changed, 129 insertions(+), 13 deletions(-) diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 300fa97c935a5..8e85dec103414 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -73,6 +73,12 @@ number increases; guidance and stage-zero events are not captured samples. fprintd 1.94.5 itself emits an initial `enroll-stage-passed` after its duplicate check, **before** starting enrollment. That first line does not prove a finger was captured. +A capture the enclave cannot use is reported as a retry, both while enrolling +and while matching: fprintd shows `enroll-retry-scan` or `verify-retry-scan`, +and starts a match again by itself. When the kernel ends an enrolment because +too many captures in a row were unusable (`SEP_BIO_STATUS_REJECTED`), the +enrolment fails with that reason in fprintd's log, and fprintd reports +`enroll-unknown-error`. Kernels from before these statuses never send them. The device-specific `apple/mesa_calibration.bin` on m2 was recovered using the bundled [`extract-mesa-calibration.py`](extract-mesa-calibration.py), copied from [Gist revision `eb079a8007985d04e75182f20994f1a1c496f12e`](https://gist.github.com/DjDeveloperr/867a1961b861c570442724f48f770158/eb079a8007985d04e75182f20994f1a1c496f12e), diff --git a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch index bdb15bb1c2b27..5163351683362 100644 --- a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch +++ b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch @@ -2,7 +2,7 @@ diff --git a/libfprint/drivers/aurora/aurora.c b/libfprint/drivers/aurora/aurora new file mode 100644 --- /dev/null +++ b/libfprint/drivers/aurora/aurora.c -@@ -0,0 +1,881 @@ +@@ -0,0 +1,991 @@ +/* + * Aurora SEP fingerprint driver + * @@ -28,6 +28,20 @@ new file mode 100644 +#include +#include + ++/* Why the kernel ended an enrol or verify, in a poll's 'status'. Headers from ++ * before these were named carry the same numbers; those kernels never report ++ * RETRY while an enrolment is running, nor REJECTED at all. */ ++#ifndef SEP_BIO_STATUS_REJECTED ++#define SEP_BIO_STATUS_NONE 0 ++#define SEP_BIO_STATUS_SENSOR 1 ++#define SEP_BIO_STATUS_ENCLAVE 2 ++#define SEP_BIO_STATUS_TOO_MANY 3 ++#define SEP_BIO_STATUS_TIMEOUT 4 ++#define SEP_BIO_STATUS_RETRY 5 ++#define SEP_BIO_STATUS_UNFILED 6 ++#define SEP_BIO_STATUS_REJECTED 7 ++#endif ++ +struct _FpiDeviceAurora +{ + FpDevice parent; @@ -95,6 +109,65 @@ new file mode 100644 + return TRUE; +} + ++static const gchar * ++aurora_status_reason (guint32 status) ++{ ++ switch (status) ++ { ++ case SEP_BIO_STATUS_NONE: ++ return "it was cancelled or ended by the system"; ++ ++ case SEP_BIO_STATUS_SENSOR: ++ return "the fingerprint sensor failed"; ++ ++ case SEP_BIO_STATUS_ENCLAVE: ++ return "the secure enclave refused a step"; ++ ++ case SEP_BIO_STATUS_TOO_MANY: ++ return "the enclave did not complete the enrolment within its captures"; ++ ++ case SEP_BIO_STATUS_TIMEOUT: ++ return "no finger was placed on the sensor in time"; ++ ++ case SEP_BIO_STATUS_RETRY: ++ return "the capture was not usable"; ++ ++ case SEP_BIO_STATUS_UNFILED: ++ return "the fingerprint was enrolled but could not be recorded"; ++ ++ case SEP_BIO_STATUS_REJECTED: ++ return "too many captures in a row were not usable; " ++ "rest more of the finger flat on the sensor and enrol again"; ++ ++ default: ++ return NULL; ++ } ++} ++ ++static GError * ++aurora_failure_new (const gchar *operation, guint32 status) ++{ ++ const gchar *reason = aurora_status_reason (status); ++ ++ if (reason == NULL) ++ return fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, ++ "%s failed, driver status 0x%x", ++ operation, status); ++ ++ return fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, ++ "%s failed: %s (driver status %u)", ++ operation, reason, status); ++} ++ ++/* The kernel reports an unusable capture without saying why it was unusable, ++ * so the general retry is the most specific code that is always true. */ ++static GError * ++aurora_retry_new (void) ++{ ++ return fpi_device_retry_new_msg (FP_DEVICE_RETRY_GENERAL, ++ "the capture was not usable; touch the sensor again"); ++} ++ +/* The label is how the host recognises its own records in the device's list. + * libfprint already has an encoding for exactly this -- it carries the enroll + * date, the finger and the username -- so use it rather than inventing one. */ @@ -402,16 +475,23 @@ new file mode 100644 + switch (poll_res.state) + { + case SEP_BIO_STATE_PENDING: -+ return G_SOURCE_CONTINUE; -+ + case SEP_BIO_STATE_PROGRESS: + /* fprintd calls this "enroll-stage-passed". A guidance wakeup or an + * initial stage-zero event must not masquerade as a captured sample. */ -+ if (poll_res.stage > self->enroll_last_reported_stage) ++ if (poll_res.state == SEP_BIO_STATE_PROGRESS && ++ poll_res.stage > self->enroll_last_reported_stage) + { + self->enroll_last_reported_stage = poll_res.stage; + fpi_device_enroll_progress (device, poll_res.stage, NULL, NULL); + } ++ ++ /* The latest capture did not count, and the enrolment goes on. The ++ * kernel reports each rejection in one poll only, and an accepted ++ * capture clears it, so a rejection that shares a poll with a new stage ++ * came after that stage. */ ++ if (poll_res.status == SEP_BIO_STATUS_RETRY) ++ fpi_device_enroll_progress (device, self->enroll_last_reported_stage, ++ NULL, aurora_retry_new ()); + return G_SOURCE_CONTINUE; + + case SEP_BIO_STATE_DONE: @@ -434,12 +514,17 @@ new file mode 100644 + return G_SOURCE_REMOVE; + + case SEP_BIO_STATE_FAILED: ++ self->watch_id = 0; ++ fpi_device_enroll_complete (device, NULL, ++ aurora_failure_new ("enrolment", poll_res.status)); ++ return G_SOURCE_REMOVE; ++ + default: + self->watch_id = 0; + fpi_device_enroll_complete (device, NULL, + fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, -+ "enrolment failed, driver status 0x%x", -+ poll_res.status)); ++ "enrolment failed, state %u, driver status 0x%x", ++ poll_res.state, poll_res.status)); + return G_SOURCE_REMOVE; + } +} @@ -526,6 +611,17 @@ new file mode 100644 + return TRUE; +} + ++/* A verify that ended without a comparison. When the kernel says why, pass ++ * that on: otherwise the log shows only an unspecified error. */ ++static GError * ++aurora_not_compared_new (const struct sep_bio_verify_poll *res) ++{ ++ if (res->state == SEP_BIO_STATE_FAILED) ++ return aurora_failure_new ("verification", res->status); ++ ++ return fpi_device_error_new (FP_DEVICE_ERROR_GENERAL); ++} ++ +static gboolean +aurora_verify_ready (gint fd, GIOCondition condition, gpointer user_data) +{ @@ -539,6 +635,7 @@ new file mode 100644 + * reference under libfprint's feet, and it frees the print again when the + * match completes. */ + FpPrint *matched = NULL; ++ GError *retry = NULL; + + if (!aurora_ioctl (self, SEP_BIO_VERIFY_POLL, &res, &error)) + { @@ -567,7 +664,12 @@ new file mode 100644 + + if (aurora_result_is_match (self, &res)) + matched = aurora_print_from_uuid (self, res.uuid, NULL); -+ ++ /* The capture was not usable, so nothing was compared. A retry tells the ++ * caller to ask for another touch: fprintd starts the match again on its ++ * own, where a failure would end the authentication attempt. */ ++ else if (res.state == SEP_BIO_STATE_FAILED && ++ res.status == SEP_BIO_STATUS_RETRY) ++ retry = aurora_retry_new (); + + if (action == FPI_DEVICE_ACTION_VERIFY) + { @@ -575,7 +677,12 @@ new file mode 100644 + + fpi_device_get_verify_data (device, &expected); + -+ if (matched != NULL && aurora_print_has_uuid (expected, res.uuid)) ++ if (retry != NULL) ++ { ++ fpi_device_verify_report (device, FPI_MATCH_ERROR, NULL, retry); ++ fpi_device_verify_complete (device, NULL); ++ } ++ else if (matched != NULL && aurora_print_has_uuid (expected, res.uuid)) + { + fpi_device_verify_report (device, FPI_MATCH_SUCCESS, matched, NULL); + fpi_device_verify_complete (device, NULL); @@ -585,8 +692,7 @@ new file mode 100644 + /* No comparison happened, so there is no verdict to report. Saying + * FAIL here would tell the user their finger was rejected when it + * was never examined. */ -+ fpi_device_verify_complete (device, -+ fpi_device_error_new (FP_DEVICE_ERROR_GENERAL)); ++ fpi_device_verify_complete (device, aurora_not_compared_new (&res)); + } + else + { @@ -617,10 +723,14 @@ new file mode 100644 + } + } + -+ if (matched == NULL && res.result == SEP_BIO_NOT_COMPARED) ++ if (retry != NULL) ++ { ++ fpi_device_identify_report (device, NULL, NULL, retry); ++ fpi_device_identify_complete (device, NULL); ++ } ++ else if (matched == NULL && res.result == SEP_BIO_NOT_COMPARED) + { -+ fpi_device_identify_complete (device, -+ fpi_device_error_new (FP_DEVICE_ERROR_GENERAL)); ++ fpi_device_identify_complete (device, aurora_not_compared_new (&res)); + } + else + { From aadbf8b8543f031fa56b447d90d0d45029a58436 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 15:36:15 +1100 Subject: [PATCH 0774/2680] tools: aurora-sep: give the SEP loader time for a slow enclave load-driver tries to seal and unseal a trusted key 300 times, 0.1 s apart, then waits as long again for /dev/sep-bio. On J700 the enclave's key store arrives around 43 s after boot, and the driver itself waits up to 90 s for it, so the loader can give up before Touch ID comes up. The service then fails, and fprintd, which requires it, fails with it. Measure the waits in seconds of uptime rather than in attempts, wait up to 120 s for the trusted key and 60 s for the device, and let AURORA_SEP_READY_TIMEOUT and AURORA_SEP_DEVICE_TIMEOUT override them. Log why the loader gives up and how long each step took, so a slow boot can be told from a broken one. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 11 +++++++ tools/aurora-sep/load-driver | 59 ++++++++++++++++++++++++++++++------ 2 files changed, 60 insertions(+), 10 deletions(-) diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 8e85dec103414..8cc7fc4279419 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -17,6 +17,17 @@ systemctl daemon-reload systemctl enable aurora-sep.service ``` +The loader waits up to 120 s for the enclave to seal and unseal a trusted key, +then up to 60 s for `/dev/sep-bio`, and logs how long each took. On J700 the +enclave's key store arrives around 43 s after boot. To change a wait, set +`AURORA_SEP_READY_TIMEOUT` or `AURORA_SEP_DEVICE_TIMEOUT` in whole seconds, +for example with `systemctl edit aurora-sep.service`: + +```ini +[Service] +Environment=AURORA_SEP_READY_TIMEOUT=180 +``` + `xart_writes=0` remains the default. `load-driver` explicitly opts into writes and must only be enabled on a machine whose APFS `.gl` extent passes the driver's narrow ownership checks: one physical extent, no snapshots or revert diff --git a/tools/aurora-sep/load-driver b/tools/aurora-sep/load-driver index a70db75e6b25b..fbf43e5093f00 100755 --- a/tools/aurora-sep/load-driver +++ b/tools/aurora-sep/load-driver @@ -4,6 +4,34 @@ set -eu keybag=/var/lib/aurora-sep-keybag.bin refkey=/var/lib/aurora-sep-refkey.bin +# Seconds to wait for the enclave to seal and unseal a trusted key, then for +# /dev/sep-bio. On J700 the key store arrives around 43 s after boot, and the +# driver waits up to 90 s for it before it brings Touch ID up, so the first +# wait has to outlast that. Set either in the environment, for example with a +# drop-in for aurora-sep.service. +ready_timeout=${AURORA_SEP_READY_TIMEOUT:-120} +device_timeout=${AURORA_SEP_DEVICE_TIMEOUT:-60} + +log() { + echo "aurora-sep-load: $*" >&2 +} + +for budget in "$ready_timeout" "$device_timeout"; do + case "$budget" in + '' | *[!0-9]*) + log "timeouts must be whole seconds, not '$budget'" + exit 2 + ;; + esac +done + +# Seconds since boot. Unlike the wall clock it cannot jump when the time is +# first synchronised during boot. +uptime_s() { + read -r up _ /dev/null || true) if [ -n "$key_id" ]; then if keyctl pipe "$key_id" >"$work/sealed"; then @@ -37,17 +66,27 @@ while [ "$attempts" -gt 0 ]; do keyctl revoke "$key_id" >/dev/null 2>&1 || true fi [ "$verified" = true ] && break + [ "$(uptime_s)" -lt "$deadline" ] || break sleep 0.1 - attempts=$((attempts - 1)) done -[ "$verified" = true ] || exit 1 -[ -s "$keybag" ] || exit 1 -[ -s "$refkey" ] || exit 1 +if [ "$verified" != true ]; then + log "the enclave did not seal and unseal a trusted key within ${ready_timeout} s" + exit 1 +fi +log "trusted keys work after $(($(uptime_s) - start)) s" + +if [ ! -s "$keybag" ] || [ ! -s "$refkey" ]; then + log "$keybag or $refkey is missing or empty" + exit 1 +fi -attempts=300 -while [ "$attempts" -gt 0 ] && [ ! -c /dev/sep-bio ]; do +deadline=$(($(uptime_s) + device_timeout)) +while [ ! -c /dev/sep-bio ] && [ "$(uptime_s)" -lt "$deadline" ]; do sleep 0.1 - attempts=$((attempts - 1)) done -[ -c /dev/sep-bio ] || exit 1 +if [ ! -c /dev/sep-bio ]; then + log "/dev/sep-bio did not appear within ${device_timeout} s" + exit 1 +fi +log "/dev/sep-bio is ready after $(($(uptime_s) - start)) s" From 783b906ed9bbd719497b63e95d3d5d48c4dbd137 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 15:39:02 +1100 Subject: [PATCH 0775/2680] tools: aurora-sep: add Fedora and Arch Linux packaging A manual install leaves libfprint under /usr/local, where fprintd only finds it through a library path drop-in, and puts the loader and the units under /usr/local and /etc, where no package manager tracks them. Add packaging for the distributions the driver runs on: - packaging/fedora/build.sh rebuilds the release's own libfprint source package with the aurora patch added and an ".aurora1" release suffix, so it replaces the distribution's library in place. It also builds aurora-sep.spec. - packaging/arch has libfprint-aurora, which provides and conflicts with libfprint, and aurora-sep, for Arch Linux and Arch Linux ARM. - aurora-sep installs the loader, its service pointed at the packaged loader, and the fprintd policy as a drop-in under /usr/lib. On Fedora a preset enables the service at boot when the package is first installed; Arch leaves that to the user. The PKGBUILDs take their local sources through links into this tree, and a test checks their checksums against it. The README gives the build, install and rollback steps. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 52 ++++++++++ .../packaging/arch/aurora-sep/PKGBUILD | 44 +++++++++ .../packaging/arch/aurora-sep/README.md | 1 + .../packaging/arch/aurora-sep/XART-SAFETY.md | 1 + .../arch/aurora-sep/aurora-sep.service | 1 + .../arch/aurora-sep/fprintd-aurora.conf | 1 + .../packaging/arch/aurora-sep/load-driver | 1 + .../packaging/arch/libfprint-aurora/PKGBUILD | 73 ++++++++++++++ .../arch/libfprint-aurora/apple-sep.uapi | 1 + .../libfprint-1.94.100-apple-sep.patch | 1 + .../packaging/fedora/aurora-sep.spec | 74 ++++++++++++++ tools/aurora-sep/packaging/fedora/build.sh | 98 +++++++++++++++++++ tools/aurora-sep/tests/test_packaging.py | 72 ++++++++++++++ 13 files changed, 420 insertions(+) create mode 100644 tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/README.md create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/XART-SAFETY.md create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep.service create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/fprintd-aurora.conf create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/load-driver create mode 100644 tools/aurora-sep/packaging/arch/libfprint-aurora/PKGBUILD create mode 120000 tools/aurora-sep/packaging/arch/libfprint-aurora/apple-sep.uapi create mode 120000 tools/aurora-sep/packaging/arch/libfprint-aurora/libfprint-1.94.100-apple-sep.patch create mode 100644 tools/aurora-sep/packaging/fedora/aurora-sep.spec create mode 100755 tools/aurora-sep/packaging/fedora/build.sh create mode 100644 tools/aurora-sep/tests/test_packaging.py diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 8cc7fc4279419..942d4002683e7 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -144,6 +144,58 @@ lock screen then selects `omarchy-lock-fingerprint` automatically and accepts Touch ID through fprintd. Hyprlock's separate fingerprint switch is not used by the Quickshell lock screen. +## Packages + +`packaging/` builds the same pieces as distribution packages: libfprint with +the aurora driver, which replaces the distribution's libfprint so fprintd needs +no library path, and `aurora-sep` with the loader, its service and the fprintd +policy. Installing `aurora-sep` lets fprintd load the driver with xART writes +on, as above. + +On Fedora, `packaging/fedora/build.sh` rebuilds the release's own libfprint +source package with the patch added (1.94.100 on Fedora 45) and builds +`aurora-sep`. It installs build dependencies, so run it in a throwaway +container from the top of this tree: + +```sh +mkdir rpms +podman run --rm -v "$PWD":/src:ro,z -v "$PWD/rpms":/out:z \ + registry.fedoraproject.org/fedora:45 \ + /src/tools/aurora-sep/packaging/fedora/build.sh /out +sudo dnf install rpms/libfprint-1.94.100-*.aurora1.*.rpm \ + rpms/aurora-sep-1-*.noarch.rpm +``` + +A later Fedora libfprint update replaces the rebuilt one; rebuild after it, or +hold it with `dnf versionlock add libfprint`. + +On Arch Linux and Arch Linux ARM, build `libfprint-aurora` (it replaces +`libfprint`) and then `aurora-sep`: + +```sh +cd tools/aurora-sep/packaging/arch/libfprint-aurora && makepkg -si +cd ../aurora-sep && makepkg -si +``` + +fprintd starts the loader the first time it runs. Loading the driver at boot +instead has the enclave ready before the login screen asks for a fingerprint. +The Fedora package enables that with a systemd preset when it is first +installed; on Arch, enable it yourself: + +```sh +sudo systemctl enable aurora-sep.service +``` + +The driver attaches to the enclave once per boot, so reboot rather than +restarting the service or reloading `apple_sep`. To move from the manual +install above, remove `/usr/local/sbin/aurora-sep-load`, +`/etc/systemd/system/aurora-sep.service`, +`/etc/systemd/system/fprintd.service.d/aurora.conf` and any drop-in that points +fprintd at another libfprint, then run `systemctl daemon-reload` and +`systemctl reenable aurora-sep.service`. To go back to the distribution's +libfprint, remove `aurora-sep` and run `dnf distro-sync libfprint` or +`pacman -S libfprint`. + ## Biometric userspace header The canonical ioctl ABI is `include/uapi/linux/apple-sep.h`. Export this diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD new file mode 100644 index 0000000000000..d7a921a5501cf --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# The loader, service and fprintd policy for the Apple SEP fingerprint driver. +# The local sources are links into tools/aurora-sep; after changing any of +# them, run updpkgsums here. libfprint-aurora provides the libfprint driver. + +pkgname=aurora-sep +pkgver=1 +pkgrel=1 +pkgdesc="Loader and fprintd policy for the Apple SEP (Touch ID) driver" +url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" +arch=(any) +license=(GPL-2.0-only) +depends=( + coreutils + fprintd + keyutils + kmod + sh +) +source=(load-driver + aurora-sep.service + fprintd-aurora.conf + README.md + XART-SAFETY.md) +sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' + 'd3441eeb06c930fcc13314d18012b1f5d71cc1a821ceaf44de13caafb2931d73' + 'f8d0c4b9385e6716d5ac5dd9941a57459fbc551a8238383bcde36cd929d0523f' + 'd6e5c68bab7e0d71da48820da75d4ff7e67a06e7970961377ed28d30c2b66c08' + '83ae33b34d4f4ff42be7ba22b6d5557c3c5776075753ea288bef986645687f23') + +package() { + install -Dm755 load-driver "$pkgdir/usr/lib/aurora-sep/aurora-sep-load" + + install -Dm644 aurora-sep.service "$pkgdir/usr/lib/systemd/system/aurora-sep.service" + sed -i 's|^ExecStart=.*|ExecStart=/usr/lib/aurora-sep/aurora-sep-load|' \ + "$pkgdir/usr/lib/systemd/system/aurora-sep.service" + grep -qx 'ExecStart=/usr/lib/aurora-sep/aurora-sep-load' \ + "$pkgdir/usr/lib/systemd/system/aurora-sep.service" + + install -Dm644 fprintd-aurora.conf \ + "$pkgdir/usr/lib/systemd/system/fprintd.service.d/aurora-sep.conf" + install -Dm644 -t "$pkgdir/usr/share/doc/$pkgname" README.md XART-SAFETY.md +} diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/README.md b/tools/aurora-sep/packaging/arch/aurora-sep/README.md new file mode 120000 index 0000000000000..8a33348c7d811 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/README.md @@ -0,0 +1 @@ +../../../README.md \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/XART-SAFETY.md b/tools/aurora-sep/packaging/arch/aurora-sep/XART-SAFETY.md new file mode 120000 index 0000000000000..45a7d77d7e92e --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/XART-SAFETY.md @@ -0,0 +1 @@ +../../../XART-SAFETY.md \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep.service b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep.service new file mode 120000 index 0000000000000..72bc9aad30d16 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep.service @@ -0,0 +1 @@ +../../../aurora-sep.service \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/fprintd-aurora.conf b/tools/aurora-sep/packaging/arch/aurora-sep/fprintd-aurora.conf new file mode 120000 index 0000000000000..41efea20ba890 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/fprintd-aurora.conf @@ -0,0 +1 @@ +../../../fprintd-aurora.conf \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/load-driver b/tools/aurora-sep/packaging/arch/aurora-sep/load-driver new file mode 120000 index 0000000000000..3f9fc69e38042 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/load-driver @@ -0,0 +1 @@ +../../../load-driver \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/libfprint-aurora/PKGBUILD b/tools/aurora-sep/packaging/arch/libfprint-aurora/PKGBUILD new file mode 100644 index 0000000000000..12bfdd7222953 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/libfprint-aurora/PKGBUILD @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# libfprint with the aurora driver for the Apple SEP fingerprint sensor. +# The local sources are links into tools/aurora-sep and to +# include/uapi/linux/apple-sep.h; after changing either file, run updpkgsums +# here. + +pkgname=libfprint-aurora +_pkgname=libfprint +pkgver=1.94.100 +pkgrel=1 +pkgdesc="Library for fingerprint readers, with the Apple SEP (Touch ID) driver" +url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" +arch=(aarch64 x86_64) +license=(LGPL-2.1-or-later) +depends=( + libgcc + glib2 + glibc + libgudev + libgusb + openssl + pixman +) +makedepends=( + glib2-devel + gobject-introspection + gtk-doc + meson + python-cairo + python-gobject + systemd +) +checkdepends=( + cairo + umockdev +) +provides=("$_pkgname=$pkgver" libfprint-2.so) +conflicts=("$_pkgname") +source=("https://gitlab.freedesktop.org/libfprint/libfprint/-/archive/v$pkgver/$_pkgname-v$pkgver.tar.gz" + "libfprint-$pkgver-apple-sep.patch" + apple-sep.uapi) +sha256sums=('edc90e02f330a7595ceaf37f2c6ec32ed43541347fe936d3273b0bb2524fd19c' + '00f999dab395c0f6cd5ffda94099f898eb48d001e19036c500fe2713f76aa076' + '00cb09430461e0b0772619bc2054c8f9794484fbc3d807268955c0fb2de45c10') + +prepare() { + cd "$_pkgname-v$pkgver" + patch -Np1 -i "../libfprint-$pkgver-apple-sep.patch" + + # The driver includes , which the distribution's kernel + # headers do not carry. + install -Dm644 ../apple-sep.uapi ../aurora-include/linux/apple-sep.h +} + +build() { + local meson_options=( + -D drivers=all + -D installed-tests=false + ) + + CFLAGS+=" -I$srcdir/aurora-include" + arch-meson "$_pkgname-v$pkgver" build "${meson_options[@]}" + meson compile -C build +} + +check() { + meson test -C build --print-errorlogs +} + +package() { + meson install -C build --destdir "$pkgdir" +} diff --git a/tools/aurora-sep/packaging/arch/libfprint-aurora/apple-sep.uapi b/tools/aurora-sep/packaging/arch/libfprint-aurora/apple-sep.uapi new file mode 120000 index 0000000000000..3ba9077515e02 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/libfprint-aurora/apple-sep.uapi @@ -0,0 +1 @@ +../../../../../include/uapi/linux/apple-sep.h \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/libfprint-aurora/libfprint-1.94.100-apple-sep.patch b/tools/aurora-sep/packaging/arch/libfprint-aurora/libfprint-1.94.100-apple-sep.patch new file mode 120000 index 0000000000000..40c7665bd03ef --- /dev/null +++ b/tools/aurora-sep/packaging/arch/libfprint-aurora/libfprint-1.94.100-apple-sep.patch @@ -0,0 +1 @@ +../../../patches/libfprint-1.94.100-apple-sep.patch \ No newline at end of file diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec new file mode 100644 index 0000000000000..9d09b92727479 --- /dev/null +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Built from tools/aurora-sep: rpmbuild --define "_sourcedir " +# (build.sh does this). + +Name: aurora-sep +Version: 1 +Release: 1%{?dist} +Summary: Loader and fprintd policy for the Apple SEP fingerprint driver + +License: GPL-2.0-only +URL: https://github.com/aurora-silicon/linux +Source0: load-driver +Source1: aurora-sep.service +Source2: fprintd-aurora.conf +Source3: README.md +Source4: XART-SAFETY.md + +BuildArch: noarch +BuildRequires: systemd-rpm-macros +Requires: coreutils +Requires: fprintd +Requires: keyutils +Requires: kmod +%{?systemd_requires} + +%description +Loads the Apple SEP driver (apple_sep) with xART writes enabled, waits until +the secure enclave seals and unseals a trusted key and /dev/sep-bio appears, +and lets fprintd reach /dev/sep-bio. The loader runs at boot, and fprintd +also pulls it in when it starts. libfprint needs the aurora driver, which +this package does not provide. Read XART-SAFETY.md before installing it on +a machine that also runs macOS. + +%prep +%setup -q -c -T +cp -p %{SOURCE3} %{SOURCE4} . + +%build + +%install +install -Dpm 0755 %{SOURCE0} %{buildroot}%{_libexecdir}/aurora-sep-load +install -Dpm 0644 %{SOURCE1} %{buildroot}%{_unitdir}/aurora-sep.service +sed -i 's|^ExecStart=.*|ExecStart=%{_libexecdir}/aurora-sep-load|' \ + %{buildroot}%{_unitdir}/aurora-sep.service +grep -qx 'ExecStart=%{_libexecdir}/aurora-sep-load' \ + %{buildroot}%{_unitdir}/aurora-sep.service +install -Dpm 0644 %{SOURCE2} \ + %{buildroot}%{_unitdir}/fprintd.service.d/aurora-sep.conf +# Load the driver at boot, so the enclave is up before the login screen asks. +install -d %{buildroot}%{_presetdir} +echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset + +%post +%systemd_post aurora-sep.service + +%preun +%systemd_preun aurora-sep.service + +%postun +# Never restart it: the driver attaches to the enclave once per boot. +%systemd_postun aurora-sep.service + +%files +%doc README.md XART-SAFETY.md +%{_libexecdir}/aurora-sep-load +%{_unitdir}/aurora-sep.service +%dir %{_unitdir}/fprintd.service.d +%{_unitdir}/fprintd.service.d/aurora-sep.conf +%{_presetdir}/80-aurora-sep.preset + +%changelog +* Sun Oct 11 2026 Ryan Murray - 1-1 +- Package the loader, its service and the fprintd policy diff --git a/tools/aurora-sep/packaging/fedora/build.sh b/tools/aurora-sep/packaging/fedora/build.sh new file mode 100755 index 0000000000000..32d9f30761d02 --- /dev/null +++ b/tools/aurora-sep/packaging/fedora/build.sh @@ -0,0 +1,98 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +# +# Build the Fedora packages for Touch ID on the Apple SEP: +# +# - libfprint: Fedora's own source package, rebuilt with the aurora driver +# patch from ../../patches and nothing else changed. The release gains an +# ".aurora1" suffix, so it replaces the distribution's build in place. +# - aurora-sep: the driver loader, its service and the fprintd policy. +# +# It installs the build dependencies with dnf, so run it as root in a +# throwaway container of the Fedora release you are building for: +# +# podman run --rm -v :/src:ro,z -v :/out:z \ +# registry.fedoraproject.org/fedora:45 /src/ /out +# +# The binary and source packages land in the output directory. + +set -eu + +out=${1:?usage: build.sh } +here=$(cd "$(dirname "$0")" && pwd) +sep=$(cd "$here/../.." && pwd) +patch=$sep/patches/libfprint-1.94.100-apple-sep.patch +header=${APPLE_SEP_HEADER:-$sep/../../include/uapi/linux/apple-sep.h} +patched_version=1.94.100 + +for f in "$patch" "$header"; do + [ -f "$f" ] || { echo "build.sh: $f is missing" >&2; exit 1; } +done +mkdir -p "$out" + +top=$(mktemp -d) +trap 'rm -rf "$top"' EXIT +rpmtop=$top/rpmbuild + +dnf -y install rpm-build systemd-rpm-macros + +# Fedora's libfprint, exactly as the distribution builds it. +(cd "$top" && dnf -y download --source libfprint) +srpm=$(ls "$top"/libfprint-*.src.rpm) +rpm -i --define "_topdir $rpmtop" "$srpm" +spec=$rpmtop/SPECS/libfprint.spec + +version=$(rpmspec -q --srpm --qf '%{version}\n' "$spec") +if [ "$version" != "$patched_version" ]; then + echo "build.sh: this Fedora ships libfprint $version, the patch is for $patched_version" >&2 + exit 1 +fi +cp "$patch" "$header" "$rpmtop/SOURCES/" + +# Add the patch and the header, mark the release, and point the compiler at +# the header: is not in the distribution's kernel headers. +# %autosetup applies every PatchN. Stop if any anchor has moved. +awk -v patch="$(basename "$patch")" -v date="$(LC_ALL=C date -u '+%a %b %d %Y')" ' + /^Release:/ && !release { + sub(/[[:space:]]*$/, ".aurora1"); print; release = 1; next + } + /^Source0:/ && !source { + print + print "Source9000: apple-sep.h" + print "Patch9000: " patch + source = 1; next + } + /^%meson[[:space:]]/ && !build { + print "# The aurora driver includes from the Aurora kernel." + print "mkdir -p aurora-include/linux" + print "cp -p %{SOURCE9000} aurora-include/linux/" + print "export CFLAGS=\"%{build_cflags} -I$PWD/aurora-include\"" + print; build = 1; next + } + /^%changelog/ && !changelog { + print + print "* " date " Aurora Silicon - " "%{version}-%{release}" + print "- Add the Apple SEP (aurora) driver from aurora-silicon/linux" + print " tools/aurora-sep/patches/" patch + print "" + changelog = 1; next + } + { print } + END { + if (!(release && source && build && changelog)) { + print "build.sh: libfprint.spec changed shape; update the edit" > "/dev/stderr" + exit 1 + } + } +' "$spec" >"$spec.aurora" +mv "$spec.aurora" "$spec" + +dnf -y builddep "$spec" +rpmbuild --define "_topdir $rpmtop" -ba "$spec" + +# The loader, its service and the fprintd policy, from this tree. +rpmbuild --define "_topdir $rpmtop" --define "_sourcedir $sep" \ + -ba "$here/aurora-sep.spec" + +cp -p "$rpmtop"/RPMS/*/*.rpm "$rpmtop"/SRPMS/*.rpm "$out/" +ls -l "$out" diff --git a/tools/aurora-sep/tests/test_packaging.py b/tools/aurora-sep/tests/test_packaging.py new file mode 100644 index 0000000000000..f5aa1b124d060 --- /dev/null +++ b/tools/aurora-sep/tests/test_packaging.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""The packaging must describe the files in this tree, not stale copies.""" + +import hashlib +import re +import unittest +from pathlib import Path + +SEP = Path(__file__).resolve().parents[1] +ARCH = SEP / "packaging" / "arch" +FEDORA = SEP / "packaging" / "fedora" + + +def bash_array(text, name): + match = re.search(r"^%s=\((.*?)\)" % re.escape(name), text, re.M | re.S) + if match is None: + raise AssertionError("no %s array" % name) + return re.findall(r"'([^']*)'|\"([^\"]*)\"|(\S+)", match.group(1)) + + +def words(text, name): + return ["".join(parts) for parts in bash_array(text, name)] + + +class PkgbuildTest(unittest.TestCase): + def check(self, directory): + text = (directory / "PKGBUILD").read_text() + sources = words(text, "source") + sums = words(text, "sha256sums") + self.assertEqual(len(sources), len(sums)) + pkgver = re.search(r"^pkgver=(\S+)$", text, re.M).group(1) + for source, expected in zip(sources, sums): + if "://" in source: + continue + name = source.replace("$pkgver", pkgver) + path = directory / name + self.assertTrue(path.is_file(), "%s is not a file" % path) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + self.assertEqual( + digest, expected, "%s changed: run updpkgsums in %s" % (name, directory) + ) + + def test_libfprint_sums_match_tree(self): + self.check(ARCH / "libfprint-aurora") + + def test_aurora_sep_sums_match_tree(self): + self.check(ARCH / "aurora-sep") + + def test_patch_version_matches_pkgver(self): + text = (ARCH / "libfprint-aurora" / "PKGBUILD").read_text() + pkgver = re.search(r"^pkgver=(\S+)$", text, re.M).group(1) + self.assertTrue((SEP / "patches" / ("libfprint-%s-apple-sep.patch" % pkgver)).is_file()) + + +class FedoraSpecTest(unittest.TestCase): + def test_sources_are_in_tree(self): + text = (FEDORA / "aurora-sep.spec").read_text() + sources = re.findall(r"^Source\d+:\s+(\S+)$", text, re.M) + self.assertTrue(sources) + for source in sources: + self.assertTrue((SEP / source).is_file(), source) + + def test_build_script_patches_the_shipped_version(self): + text = (FEDORA / "build.sh").read_text() + version = re.search(r"^patched_version=(\S+)$", text, re.M).group(1) + self.assertIn("libfprint-%s-apple-sep.patch" % version, text) + self.assertTrue((SEP / "patches" / ("libfprint-%s-apple-sep.patch" % version)).is_file()) + + +if __name__ == "__main__": + unittest.main() From 0a5a432c5d2a3843bf604aa146cbe24f93548a41 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 16:34:30 +1100 Subject: [PATCH 0776/2680] tools: aurora-sep: activate Touch ID at boot, before fprintd The first open of /dev/sep-bio activates Touch ID: the driver restores the enrolled fingerprints from the enclave before the open returns, which takes a few seconds. That open is fprintd's first probe of the reader, so the first fingerprint request after boot, at the lock screen or for sudo, waits for the restore. Add aurora-sep-ready.service, which opens the device once after the loader and before fprintd, and pull it in from aurora-sep.service. It ends as soon as the open returns; a failed open is only logged, and a job timeout stops anything ordered after it from waiting beyond 90 s after boot. Install it in the Fedora and Arch packages. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 16 +++++++ tools/aurora-sep/aurora-sep-ready | 44 +++++++++++++++++++ tools/aurora-sep/aurora-sep-ready.service | 18 ++++++++ tools/aurora-sep/aurora-sep.service | 2 + .../packaging/arch/aurora-sep/PKGBUILD | 36 ++++++++++----- .../arch/aurora-sep/aurora-sep-ready | 1 + .../arch/aurora-sep/aurora-sep-ready.service | 1 + .../packaging/fedora/aurora-sep.spec | 41 +++++++++++------ 8 files changed, 133 insertions(+), 26 deletions(-) create mode 100644 tools/aurora-sep/aurora-sep-ready create mode 100644 tools/aurora-sep/aurora-sep-ready.service create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready.service diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 942d4002683e7..eb9af9803d3c0 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -196,6 +196,22 @@ fprintd at another libfprint, then run `systemctl daemon-reload` and libfprint, remove `aurora-sep` and run `dnf distro-sync libfprint` or `pacman -S libfprint`. +## Desktop integration + +The first open of `/dev/sep-bio` activates Touch ID: the driver restores the +enrolled fingerprints from the enclave before the open returns, which takes +a few seconds. `aurora-sep-ready.service`, which `aurora-sep.service` pulls +in, makes that open at boot, before fprintd starts. The lock screen, sudo +and polkit then find the reader and the fingerprints ready, and the first +fingerprint request does not wait for the restore. A failed open is only +logged. The packages install it; by hand: + +```sh +install -Dm755 aurora-sep-ready /usr/local/sbin/aurora-sep-ready +install -Dm644 aurora-sep-ready.service /etc/systemd/system/aurora-sep-ready.service +systemctl daemon-reload +``` + ## Biometric userspace header The canonical ioctl ABI is `include/uapi/linux/apple-sep.h`. Export this diff --git a/tools/aurora-sep/aurora-sep-ready b/tools/aurora-sep/aurora-sep-ready new file mode 100644 index 0000000000000..0363a3a298ecb --- /dev/null +++ b/tools/aurora-sep/aurora-sep-ready @@ -0,0 +1,44 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +# +# Bring Touch ID up before anyone asks for a fingerprint. The first open of +# /dev/sep-bio activates it: the driver restores the enrolled fingerprints +# from the enclave and returns only when that is done. Opened here at boot, +# ahead of fprintd, the restore no longer delays the first fingerprint +# request. A failure is only logged. +set -u + +dev=/dev/sep-bio + +log() { + echo "aurora-sep-ready: $*" >&2 +} + +uptime_s() { + read -r up _ &1); then + log "Touch ID is ready after $(($(uptime_s) - start)) s" + exit 0 + fi + case "$err" in + # Another opener is activating it or using it; the unit's start + # timeout bounds this. + *[Bb]usy*) sleep 0.2 ;; + *) + log "could not open $dev: ${err##*: }" + exit 0 + ;; + esac +done diff --git a/tools/aurora-sep/aurora-sep-ready.service b/tools/aurora-sep/aurora-sep-ready.service new file mode 100644 index 0000000000000..91e5a7bce3815 --- /dev/null +++ b/tools/aurora-sep/aurora-sep-ready.service @@ -0,0 +1,18 @@ +[Unit] +Description=Activate Touch ID and restore its fingerprints +Documentation=file:///usr/share/doc/aurora-sep/README.md +Requires=aurora-sep.service +After=aurora-sep.service +Before=fprintd.service +ConditionPathExists=/dev/sep-bio + +# Whatever is ordered after this waits for the enclave, which can take most +# of a minute after boot. Stop waiting 90 s after boot, so that a broken +# enclave cannot hold anything back for longer. +JobTimeoutSec=90 + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/sbin/aurora-sep-ready +TimeoutStartSec=60 diff --git a/tools/aurora-sep/aurora-sep.service b/tools/aurora-sep/aurora-sep.service index 4287247cba0da..34a9f1cd69eb7 100644 --- a/tools/aurora-sep/aurora-sep.service +++ b/tools/aurora-sep/aurora-sep.service @@ -2,6 +2,8 @@ Description=Aurora SEP services After=local-fs.target RequiresMountsFor=/var/lib +# Activate Touch ID as soon as the driver is up. +Wants=aurora-sep-ready.service [Service] Type=oneshot diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index d7a921a5501cf..e33322557b247 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -1,13 +1,13 @@ # SPDX-License-Identifier: GPL-2.0-only # -# The loader, service and fprintd policy for the Apple SEP fingerprint driver. +# The loader and desktop integration for the Apple SEP fingerprint driver. # The local sources are links into tools/aurora-sep; after changing any of # them, run updpkgsums here. libfprint-aurora provides the libfprint driver. pkgname=aurora-sep pkgver=1 -pkgrel=1 -pkgdesc="Loader and fprintd policy for the Apple SEP (Touch ID) driver" +pkgrel=2 +pkgdesc="Loader and desktop integration for the Apple SEP (Touch ID) driver" url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" arch=(any) license=(GPL-2.0-only) @@ -21,22 +21,34 @@ depends=( source=(load-driver aurora-sep.service fprintd-aurora.conf + aurora-sep-ready + aurora-sep-ready.service README.md XART-SAFETY.md) sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' - 'd3441eeb06c930fcc13314d18012b1f5d71cc1a821ceaf44de13caafb2931d73' + '447f99cc3c7e683134f31b993aa825fadde24e4854954a3035c96e6631d4d4d0' 'f8d0c4b9385e6716d5ac5dd9941a57459fbc551a8238383bcde36cd929d0523f' - 'd6e5c68bab7e0d71da48820da75d4ff7e67a06e7970961377ed28d30c2b66c08' + '2db07d3a0191e8d14581a3056505a8c4225b2fc8e6ea4a1956ae5493abcbb0ae' + '5a5e5c7e93d1462d9d1023446fdb297926aeb93d8988867b515aca7adafe4487' + '29a8fb4970f7e550310a1c11d6aa0ad2b840b0307f4be0d1ad76875bed1a4ec2' '83ae33b34d4f4ff42be7ba22b6d5557c3c5776075753ea288bef986645687f23') -package() { - install -Dm755 load-driver "$pkgdir/usr/lib/aurora-sep/aurora-sep-load" +# A unit and the helper it runs ($1, installed as $3), pointed at the +# packaged helper. +_unit() { + local unit="$pkgdir/usr/lib/systemd/system/$2" + + install -Dm755 "$1" "$pkgdir/usr/lib/aurora-sep/$3" + install -Dm644 "$2" "$unit" + sed -i "s|^\(Exec[A-Za-z]*=\)/usr/local/sbin/$3|\1/usr/lib/aurora-sep/$3|" "$unit" + if grep -q /usr/local "$unit"; then + return 1 + fi +} - install -Dm644 aurora-sep.service "$pkgdir/usr/lib/systemd/system/aurora-sep.service" - sed -i 's|^ExecStart=.*|ExecStart=/usr/lib/aurora-sep/aurora-sep-load|' \ - "$pkgdir/usr/lib/systemd/system/aurora-sep.service" - grep -qx 'ExecStart=/usr/lib/aurora-sep/aurora-sep-load' \ - "$pkgdir/usr/lib/systemd/system/aurora-sep.service" +package() { + _unit load-driver aurora-sep.service aurora-sep-load + _unit aurora-sep-ready aurora-sep-ready.service aurora-sep-ready install -Dm644 fprintd-aurora.conf \ "$pkgdir/usr/lib/systemd/system/fprintd.service.d/aurora-sep.conf" diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready new file mode 120000 index 0000000000000..55fb2a87e5d43 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready @@ -0,0 +1 @@ +../../../aurora-sep-ready \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready.service b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready.service new file mode 120000 index 0000000000000..7ab622ca2d833 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready.service @@ -0,0 +1 @@ +../../../aurora-sep-ready.service \ No newline at end of file diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec index 9d09b92727479..57ae4ba402c1a 100644 --- a/tools/aurora-sep/packaging/fedora/aurora-sep.spec +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -5,8 +5,8 @@ Name: aurora-sep Version: 1 -Release: 1%{?dist} -Summary: Loader and fprintd policy for the Apple SEP fingerprint driver +Release: 2%{?dist} +Summary: Loader and desktop integration for the Apple SEP fingerprint driver License: GPL-2.0-only URL: https://github.com/aurora-silicon/linux @@ -15,6 +15,8 @@ Source1: aurora-sep.service Source2: fprintd-aurora.conf Source3: README.md Source4: XART-SAFETY.md +Source5: aurora-sep-ready +Source6: aurora-sep-ready.service BuildArch: noarch BuildRequires: systemd-rpm-macros @@ -28,8 +30,9 @@ Requires: kmod Loads the Apple SEP driver (apple_sep) with xART writes enabled, waits until the secure enclave seals and unseals a trusted key and /dev/sep-bio appears, and lets fprintd reach /dev/sep-bio. The loader runs at boot, and fprintd -also pulls it in when it starts. libfprint needs the aurora driver, which -this package does not provide. Read XART-SAFETY.md before installing it on +also pulls it in when it starts. Touch ID is then activated, before fprintd +starts. libfprint needs the aurora driver, which this package does not +provide. Read XART-SAFETY.md before installing it on a machine that also runs macOS. %prep @@ -39,12 +42,17 @@ cp -p %{SOURCE3} %{SOURCE4} . %build %install -install -Dpm 0755 %{SOURCE0} %{buildroot}%{_libexecdir}/aurora-sep-load -install -Dpm 0644 %{SOURCE1} %{buildroot}%{_unitdir}/aurora-sep.service -sed -i 's|^ExecStart=.*|ExecStart=%{_libexecdir}/aurora-sep-load|' \ - %{buildroot}%{_unitdir}/aurora-sep.service -grep -qx 'ExecStart=%{_libexecdir}/aurora-sep-load' \ - %{buildroot}%{_unitdir}/aurora-sep.service +# A unit and the helper it runs ($1, installed as $3), pointed at the +# packaged helper. +unit() { + install -Dpm 0755 "$1" %{buildroot}%{_libexecdir}/"$3" + install -Dpm 0644 "$2" %{buildroot}%{_unitdir}/"${2##*/}" + sed -i "s|^\(Exec[A-Za-z]*=\)/usr/local/sbin/$3|\1%{_libexecdir}/$3|" \ + %{buildroot}%{_unitdir}/"${2##*/}" + if grep -q /usr/local %{buildroot}%{_unitdir}/"${2##*/}"; then exit 1; fi +} +unit %{SOURCE0} %{SOURCE1} aurora-sep-load +unit %{SOURCE5} %{SOURCE6} aurora-sep-ready install -Dpm 0644 %{SOURCE2} \ %{buildroot}%{_unitdir}/fprintd.service.d/aurora-sep.conf # Load the driver at boot, so the enclave is up before the login screen asks. @@ -52,23 +60,28 @@ install -d %{buildroot}%{_presetdir} echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset %post -%systemd_post aurora-sep.service +%systemd_post aurora-sep.service aurora-sep-ready.service %preun -%systemd_preun aurora-sep.service +%systemd_preun aurora-sep.service aurora-sep-ready.service %postun -# Never restart it: the driver attaches to the enclave once per boot. -%systemd_postun aurora-sep.service +# Never restart them: the driver attaches to the enclave once per boot. +%systemd_postun aurora-sep.service aurora-sep-ready.service %files %doc README.md XART-SAFETY.md %{_libexecdir}/aurora-sep-load +%{_libexecdir}/aurora-sep-ready %{_unitdir}/aurora-sep.service +%{_unitdir}/aurora-sep-ready.service %dir %{_unitdir}/fprintd.service.d %{_unitdir}/fprintd.service.d/aurora-sep.conf %{_presetdir}/80-aurora-sep.preset %changelog +* Sun Oct 11 2026 Ryan Murray - 1-2 +- Activate Touch ID at boot, before fprintd starts + * Sun Oct 11 2026 Ryan Murray - 1-1 - Package the loader, its service and the fprintd policy From ed17f456b4d5723df6875a3e5e27cc477f9f8961 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 16:34:51 +1100 Subject: [PATCH 0777/2680] tools: aurora-sep: keep fprintd running fprintd exits after 30 s without a client, and each new start probes the reader again. The login and lock screens ask fprintd for the reader as they come up, so they wait for that start, and for a reader that is still being probed. Add a drop-in that runs fprintd with --no-timeout. Its path differs between distributions, so the drop-in is a template: the packages fill it in and check it against fprintd's own unit at build time. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 11 +++++++++++ tools/aurora-sep/fprintd-no-timeout.conf.in | 7 +++++++ tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD | 7 ++++++- .../arch/aurora-sep/fprintd-no-timeout.conf.in | 1 + tools/aurora-sep/packaging/fedora/aurora-sep.spec | 8 ++++++++ tools/aurora-sep/packaging/fedora/build.sh | 3 ++- 6 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 tools/aurora-sep/fprintd-no-timeout.conf.in create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/fprintd-no-timeout.conf.in diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index eb9af9803d3c0..671531f0729d6 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -212,6 +212,17 @@ install -Dm644 aurora-sep-ready.service /etc/systemd/system/aurora-sep-ready.ser systemctl daemon-reload ``` +fprintd exits after 30 s without a client, and each new start probes the +reader again, while the login and lock screens ask for the reader as they +come up. `fprintd-no-timeout.conf.in` keeps it running so that it answers +them at once. The packages fill in where the distribution keeps fprintd; by +hand, with `/usr/lib/fprintd` on Arch: + +```sh +sed 's|@FPRINTD@|/usr/libexec/fprintd|' fprintd-no-timeout.conf.in \ + >/etc/systemd/system/fprintd.service.d/aurora-sep-no-timeout.conf +``` + ## Biometric userspace header The canonical ioctl ABI is `include/uapi/linux/apple-sep.h`. Export this diff --git a/tools/aurora-sep/fprintd-no-timeout.conf.in b/tools/aurora-sep/fprintd-no-timeout.conf.in new file mode 100644 index 0000000000000..35f70927831fa --- /dev/null +++ b/tools/aurora-sep/fprintd-no-timeout.conf.in @@ -0,0 +1,7 @@ +# Keep fprintd running instead of letting it exit after 30 s without a +# client. Each new start probes the reader again, and the login and lock +# screens ask it for the reader when they come up, so a resident fprintd +# answers them at once. The packages fill in fprintd's path. +[Service] +ExecStart= +ExecStart=@FPRINTD@ --no-timeout diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index e33322557b247..01d0b410d2fff 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -23,6 +23,7 @@ source=(load-driver fprintd-aurora.conf aurora-sep-ready aurora-sep-ready.service + fprintd-no-timeout.conf.in README.md XART-SAFETY.md) sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' @@ -30,7 +31,8 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' 'f8d0c4b9385e6716d5ac5dd9941a57459fbc551a8238383bcde36cd929d0523f' '2db07d3a0191e8d14581a3056505a8c4225b2fc8e6ea4a1956ae5493abcbb0ae' '5a5e5c7e93d1462d9d1023446fdb297926aeb93d8988867b515aca7adafe4487' - '29a8fb4970f7e550310a1c11d6aa0ad2b840b0307f4be0d1ad76875bed1a4ec2' + '8d8df0ba2a6861b2edb2110e606906d84ca4848617621ad3cb48493a1522971d' + '9d8caa6ad9be48cdb9331a5498eafad2b7db82764fe41b74bc97ba3fcccfbb2b' '83ae33b34d4f4ff42be7ba22b6d5557c3c5776075753ea288bef986645687f23') # A unit and the helper it runs ($1, installed as $3), pointed at the @@ -52,5 +54,8 @@ package() { install -Dm644 fprintd-aurora.conf \ "$pkgdir/usr/lib/systemd/system/fprintd.service.d/aurora-sep.conf" + grep -qx 'ExecStart=/usr/lib/fprintd' /usr/lib/systemd/system/fprintd.service + sed 's|@FPRINTD@|/usr/lib/fprintd|' fprintd-no-timeout.conf.in \ + >"$pkgdir/usr/lib/systemd/system/fprintd.service.d/50-aurora-sep-no-timeout.conf" install -Dm644 -t "$pkgdir/usr/share/doc/$pkgname" README.md XART-SAFETY.md } diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/fprintd-no-timeout.conf.in b/tools/aurora-sep/packaging/arch/aurora-sep/fprintd-no-timeout.conf.in new file mode 120000 index 0000000000000..2e8a7a43c8830 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/fprintd-no-timeout.conf.in @@ -0,0 +1 @@ +../../../fprintd-no-timeout.conf.in \ No newline at end of file diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec index 57ae4ba402c1a..59a0ade7ed2d4 100644 --- a/tools/aurora-sep/packaging/fedora/aurora-sep.spec +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -17,9 +17,12 @@ Source3: README.md Source4: XART-SAFETY.md Source5: aurora-sep-ready Source6: aurora-sep-ready.service +Source7: fprintd-no-timeout.conf.in BuildArch: noarch BuildRequires: systemd-rpm-macros +# Only to check where fprintd's unit runs it from. +BuildRequires: fprintd Requires: coreutils Requires: fprintd Requires: keyutils @@ -55,6 +58,9 @@ unit %{SOURCE0} %{SOURCE1} aurora-sep-load unit %{SOURCE5} %{SOURCE6} aurora-sep-ready install -Dpm 0644 %{SOURCE2} \ %{buildroot}%{_unitdir}/fprintd.service.d/aurora-sep.conf +grep -qx 'ExecStart=%{_libexecdir}/fprintd' %{_unitdir}/fprintd.service +sed 's|@FPRINTD@|%{_libexecdir}/fprintd|' %{SOURCE7} \ + >%{buildroot}%{_unitdir}/fprintd.service.d/50-aurora-sep-no-timeout.conf # Load the driver at boot, so the enclave is up before the login screen asks. install -d %{buildroot}%{_presetdir} echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset @@ -77,11 +83,13 @@ echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset %{_unitdir}/aurora-sep-ready.service %dir %{_unitdir}/fprintd.service.d %{_unitdir}/fprintd.service.d/aurora-sep.conf +%{_unitdir}/fprintd.service.d/50-aurora-sep-no-timeout.conf %{_presetdir}/80-aurora-sep.preset %changelog * Sun Oct 11 2026 Ryan Murray - 1-2 - Activate Touch ID at boot, before fprintd starts +- Keep fprintd running * Sun Oct 11 2026 Ryan Murray - 1-1 - Package the loader, its service and the fprintd policy diff --git a/tools/aurora-sep/packaging/fedora/build.sh b/tools/aurora-sep/packaging/fedora/build.sh index 32d9f30761d02..27fb64e28aafd 100755 --- a/tools/aurora-sep/packaging/fedora/build.sh +++ b/tools/aurora-sep/packaging/fedora/build.sh @@ -90,7 +90,8 @@ mv "$spec.aurora" "$spec" dnf -y builddep "$spec" rpmbuild --define "_topdir $rpmtop" -ba "$spec" -# The loader, its service and the fprintd policy, from this tree. +# The loader, its units and the desktop integration, from this tree. +dnf -y builddep "$here/aurora-sep.spec" rpmbuild --define "_topdir $rpmtop" --define "_sourcedir $sep" \ -ba "$here/aurora-sep.spec" From 41bbc7c11c1949fcd4a118ba68a73312ab72b52e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 16:35:10 +1100 Subject: [PATCH 0778/2680] tools: aurora-sep: suspend rather than power off on the Touch ID button On Macs with Touch ID the sensor is the power button. Touching it to authenticate goes to the enclave and never produces a key event, but pressing it does, and logind cannot tell a press meant for the sensor from a press meant for the button. Its default action is to power off, so pressing while authenticating outside GNOME ends the session. Add a logind drop-in that suspends on a short press instead: a press while authenticating then costs nothing, and the next press wakes the machine. Holding the button for 5 s powers off cleanly. GNOME sessions and the GDM login screen take a short press over and suspend by default already, so this only changes consoles and other desktops. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 14 ++++++++++++++ tools/aurora-sep/logind-aurora-sep.conf | 11 +++++++++++ .../aurora-sep/packaging/arch/aurora-sep/PKGBUILD | 6 +++++- .../arch/aurora-sep/logind-aurora-sep.conf | 1 + tools/aurora-sep/packaging/fedora/aurora-sep.spec | 9 +++++++-- 5 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 tools/aurora-sep/logind-aurora-sep.conf create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/logind-aurora-sep.conf diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 671531f0729d6..5f0e9056ab749 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -223,6 +223,20 @@ sed 's|@FPRINTD@|/usr/libexec/fprintd|' fprintd-no-timeout.conf.in \ >/etc/systemd/system/fprintd.service.d/aurora-sep-no-timeout.conf ``` +The Touch ID sensor is the power button. A touch to authenticate goes to the +enclave and never reaches logind as a key press, but a press does, and logind +cannot tell a press meant for the sensor from one meant for the button. Its +default is to power off. `logind-aurora-sep.conf` makes a short press +suspend instead, so pressing while authenticating loses nothing and the next +press wakes the machine, and holding the button for 5 s powers off cleanly. +GNOME sessions and the GDM login screen handle a short press themselves +(`power-button-action`, `suspend` by default); the setting covers consoles +and other desktops. It takes effect at the next boot. By hand: + +```sh +install -Dm644 logind-aurora-sep.conf /etc/systemd/logind.conf.d/aurora-sep.conf +``` + ## Biometric userspace header The canonical ioctl ABI is `include/uapi/linux/apple-sep.h`. Export this diff --git a/tools/aurora-sep/logind-aurora-sep.conf b/tools/aurora-sep/logind-aurora-sep.conf new file mode 100644 index 0000000000000..a175c39f6ca73 --- /dev/null +++ b/tools/aurora-sep/logind-aurora-sep.conf @@ -0,0 +1,11 @@ +# The Touch ID sensor is the power button. Touching it to authenticate goes to +# the enclave and is never a key press, but pressing it is, and logind cannot +# tell a press meant for the sensor from one meant for the button. Suspend +# rather than power off on a short press, so a press while authenticating +# costs nothing: the next press wakes the machine. Holding the button for +# 5 s still powers off, cleanly. GNOME sessions and the GDM login screen +# handle a short press themselves (power-button-action, suspend by default); +# this covers everything else. +[Login] +HandlePowerKey=suspend +HandlePowerKeyLongPress=poweroff diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index 01d0b410d2fff..3c093add292ba 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -24,6 +24,7 @@ source=(load-driver aurora-sep-ready aurora-sep-ready.service fprintd-no-timeout.conf.in + logind-aurora-sep.conf README.md XART-SAFETY.md) sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' @@ -32,7 +33,8 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '2db07d3a0191e8d14581a3056505a8c4225b2fc8e6ea4a1956ae5493abcbb0ae' '5a5e5c7e93d1462d9d1023446fdb297926aeb93d8988867b515aca7adafe4487' '8d8df0ba2a6861b2edb2110e606906d84ca4848617621ad3cb48493a1522971d' - '9d8caa6ad9be48cdb9331a5498eafad2b7db82764fe41b74bc97ba3fcccfbb2b' + '0695bd9ce4c207241db13c3ea07819a032d8213417021378f5015ab9317cde12' + '3f5a8a4a19e15268552264da95d4848be31629c49f3f64f377c590cf6ef9e1cb' '83ae33b34d4f4ff42be7ba22b6d5557c3c5776075753ea288bef986645687f23') # A unit and the helper it runs ($1, installed as $3), pointed at the @@ -57,5 +59,7 @@ package() { grep -qx 'ExecStart=/usr/lib/fprintd' /usr/lib/systemd/system/fprintd.service sed 's|@FPRINTD@|/usr/lib/fprintd|' fprintd-no-timeout.conf.in \ >"$pkgdir/usr/lib/systemd/system/fprintd.service.d/50-aurora-sep-no-timeout.conf" + install -Dm644 logind-aurora-sep.conf \ + "$pkgdir/usr/lib/systemd/logind.conf.d/50-aurora-sep.conf" install -Dm644 -t "$pkgdir/usr/share/doc/$pkgname" README.md XART-SAFETY.md } diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/logind-aurora-sep.conf b/tools/aurora-sep/packaging/arch/aurora-sep/logind-aurora-sep.conf new file mode 120000 index 0000000000000..db6dc4e953022 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/logind-aurora-sep.conf @@ -0,0 +1 @@ +../../../logind-aurora-sep.conf \ No newline at end of file diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec index 59a0ade7ed2d4..a916df31cc0a5 100644 --- a/tools/aurora-sep/packaging/fedora/aurora-sep.spec +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -18,6 +18,7 @@ Source4: XART-SAFETY.md Source5: aurora-sep-ready Source6: aurora-sep-ready.service Source7: fprintd-no-timeout.conf.in +Source8: logind-aurora-sep.conf BuildArch: noarch BuildRequires: systemd-rpm-macros @@ -34,8 +35,8 @@ Loads the Apple SEP driver (apple_sep) with xART writes enabled, waits until the secure enclave seals and unseals a trusted key and /dev/sep-bio appears, and lets fprintd reach /dev/sep-bio. The loader runs at boot, and fprintd also pulls it in when it starts. Touch ID is then activated, before fprintd -starts. libfprint needs the aurora driver, which this package does not -provide. Read XART-SAFETY.md before installing it on +starts, and pressing the Touch ID button suspends rather than powers off. +libfprint needs the aurora driver, which this package does not provide. Read XART-SAFETY.md before installing it on a machine that also runs macOS. %prep @@ -61,6 +62,8 @@ install -Dpm 0644 %{SOURCE2} \ grep -qx 'ExecStart=%{_libexecdir}/fprintd' %{_unitdir}/fprintd.service sed 's|@FPRINTD@|%{_libexecdir}/fprintd|' %{SOURCE7} \ >%{buildroot}%{_unitdir}/fprintd.service.d/50-aurora-sep-no-timeout.conf +install -Dpm 0644 %{SOURCE8} \ + %{buildroot}%{_prefix}/lib/systemd/logind.conf.d/50-aurora-sep.conf # Load the driver at boot, so the enclave is up before the login screen asks. install -d %{buildroot}%{_presetdir} echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset @@ -84,12 +87,14 @@ echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset %dir %{_unitdir}/fprintd.service.d %{_unitdir}/fprintd.service.d/aurora-sep.conf %{_unitdir}/fprintd.service.d/50-aurora-sep-no-timeout.conf +%{_prefix}/lib/systemd/logind.conf.d/50-aurora-sep.conf %{_presetdir}/80-aurora-sep.preset %changelog * Sun Oct 11 2026 Ryan Murray - 1-2 - Activate Touch ID at boot, before fprintd starts - Keep fprintd running +- Suspend rather than power off on the Touch ID button * Sun Oct 11 2026 Ryan Murray - 1-1 - Package the loader, its service and the fprintd policy From b54418f2151dbd037319abd90b0af87ac4091bb4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 16:35:38 +1100 Subject: [PATCH 0779/2680] tools: aurora-sep: offer fingerprint at the GNOME login screen GNOME Shell 51 connects the greeter's fingerprint reader only when it re-authenticates a session that is already running, so the login screen never offers fingerprint, whatever org.gnome.login-screen says. That is deliberate: a fingerprint cannot unlock the login keyring, and the first application that wants a stored secret then asks for the password. Some people want fingerprint login anyway. Add aurora-sep-greeter-fingerprint.service, off by default. At boot it takes the greeter source file that holds the check from the installed GNOME Shell, drops the check, and serves the copy to the login screen's shell alone, through a GResource overlay set in a runtime drop-in for org.gnome.Shell@gdm.service. When the check is not exactly as expected, it changes nothing, so a GNOME update that moves it leaves the login screen as GNOME ships it. enable-fingerprint-authentication still turns fingerprint off. The login screen asks fprintd for the reader once, as it comes up, and offers no fingerprint until the user is selected again if Touch ID is not up yet. The unit therefore also orders the display manager after aurora-sep-ready.service. That holds the login screen back by a few seconds on a healthy boot, and up to 90 s after boot when the enclave is slow, which only makes sense when it offers fingerprint, so both come with the one switch. It needs gresource, which the packages suggest rather than require. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 30 +++++++ .../aurora-sep/aurora-sep-greeter-fingerprint | 85 +++++++++++++++++++ .../aurora-sep-greeter-fingerprint.service | 17 ++++ .../packaging/arch/aurora-sep/PKGBUILD | 9 +- .../aurora-sep/aurora-sep-greeter-fingerprint | 1 + .../aurora-sep-greeter-fingerprint.service | 1 + .../packaging/fedora/aurora-sep.spec | 14 ++- 7 files changed, 153 insertions(+), 4 deletions(-) create mode 100644 tools/aurora-sep/aurora-sep-greeter-fingerprint create mode 100644 tools/aurora-sep/aurora-sep-greeter-fingerprint.service create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint.service diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 5f0e9056ab749..23c3f589b5289 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -237,6 +237,36 @@ and other desktops. It takes effect at the next boot. By hand: install -Dm644 logind-aurora-sep.conf /etc/systemd/logind.conf.d/aurora-sep.conf ``` +GNOME Shell 51 offers fingerprint only on the lock screen, never on the +login screen, and `org.gnome.login-screen enable-fingerprint-authentication` +(true by default) cannot change that: the greeter connects its fingerprint +reader only when it re-authenticates a running session. That is GNOME's +choice, because a fingerprint cannot unlock the login keyring: after a +fingerprint login, the first application that wants a stored secret asks +for the password. `aurora-sep-greeter-fingerprint.service` lifts the +restriction for those who want it. At boot it copies the one greeter source +file that holds the check out of the installed GNOME Shell, drops the check, +and serves the copy to the login screen's shell alone through a GResource +overlay. When the check is not exactly as expected, for example after a +GNOME update, it changes nothing. The enable key above still applies. + +The login screen asks fprintd for the reader once, as it comes up, so the +same unit also holds it back until `aurora-sep-ready.service` has finished: +a few seconds on a healthy boot, but as long as the enclave takes when it is +slow, up to 90 s after boot. Without fingerprint at the login screen that +wait buys nothing, which is why both are one switch, off by default. It +needs `gresource` from glib2-devel. With the packages, or after installing +it by hand, turn it on with `systemctl enable`: + +```sh +install -Dm755 aurora-sep-greeter-fingerprint /usr/local/sbin/aurora-sep-greeter-fingerprint +install -Dm644 aurora-sep-greeter-fingerprint.service /etc/systemd/system/ +systemctl daemon-reload +systemctl enable aurora-sep-greeter-fingerprint.service +``` + +Disabling it and rebooting returns the login screen to GNOME's behaviour. + ## Biometric userspace header The canonical ioctl ABI is `include/uapi/linux/apple-sep.h`. Export this diff --git a/tools/aurora-sep/aurora-sep-greeter-fingerprint b/tools/aurora-sep/aurora-sep-greeter-fingerprint new file mode 100644 index 0000000000000..7f04a19d24d20 --- /dev/null +++ b/tools/aurora-sep/aurora-sep-greeter-fingerprint @@ -0,0 +1,85 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +# +# Offer fingerprint at the GNOME login screen, or with "stop", stop offering +# it from the next time the login screen starts. +# +# GNOME Shell 51 connects the fingerprint reader only when it re-authenticates +# a session that is already running, which is the lock screen. The login +# screen never offers fingerprint, whatever org.gnome.login-screen says: a +# fingerprint cannot unlock the login keyring, so a session started with one +# asks for the password the first time an application wants a stored secret. +# +# This takes the greeter source file that holds that check from the installed +# GNOME Shell, drops the check, and serves the copy to the login screen's +# shell alone through a GResource overlay. When the check is not exactly as +# expected, for example after a GNOME update, it changes nothing and the login +# screen stays as GNOME ships it. Run it before the display manager starts. +set -eu + +res=/org/gnome/shell/gdm/authServicesLegacy.js +out=/run/aurora-sep/greeter +dropin=/run/systemd/user/org.gnome.Shell@gdm.service.d/aurora-sep-fingerprint.conf + +log() { + echo "aurora-sep-greeter-fingerprint: $*" >&2 +} + +rm -rf "$out" "$dropin" +[ "${1:-}" != stop ] || exit 0 + +if ! command -v gresource >/dev/null; then + log "gresource is not installed (it comes with glib2-devel); the login screen stays as it is" + exit 0 +fi + +lib= +for f in /usr/lib64/gnome-shell/libshell-*.so /usr/lib/gnome-shell/libshell-*.so \ + /usr/lib/*/gnome-shell/libshell-*.so; do + [ -f "$f" ] || continue + if gresource list "$f" 2>/dev/null | grep -qxF "$res"; then + lib=$f + break + fi +done +if [ -z "$lib" ]; then + log "no installed GNOME Shell carries $res; the login screen stays as it is" + exit 0 +fi + +install -d -m 0755 "$out" +gresource extract "$lib" "$res" >"$out/stock.js" +if ! awk ' + state == 0 && $0 == " _connectFingerprintManager() {" { + print + state = 1 + next + } + state == 1 { + if ($0 == " if (!this._reauthOnly)") { + state = 2 + next + } + state = 9 + } + state == 2 { + if ($0 == " return;") { + state = 3 + next + } + state = 9 + } + { print } + END { exit state != 3 } +' "$out/stock.js" >"$out/authServicesLegacy.js"; then + log "$lib does not have the expected check; the login screen stays as it is" + rm -rf "$out" + exit 0 +fi +rm -f "$out/stock.js" +chmod 0644 "$out/authServicesLegacy.js" + +install -d -m 0755 "${dropin%/*}" +printf '[Service]\nEnvironment=G_RESOURCE_OVERLAYS=%s=%s\n' \ + "$res" "$out/authServicesLegacy.js" >"$dropin" +log "the login screen offers fingerprint ($res from $lib)" diff --git a/tools/aurora-sep/aurora-sep-greeter-fingerprint.service b/tools/aurora-sep/aurora-sep-greeter-fingerprint.service new file mode 100644 index 0000000000000..108b1e4ccbc87 --- /dev/null +++ b/tools/aurora-sep/aurora-sep-greeter-fingerprint.service @@ -0,0 +1,17 @@ +[Unit] +Description=Offer fingerprint at the GNOME login screen +Documentation=file:///usr/share/doc/aurora-sep/README.md +# The login screen asks fprintd for the reader once, as it comes up. Hold it +# back until Touch ID is up, so that it finds the reader and the fingerprints. +Wants=aurora-sep-ready.service +After=aurora-sep-ready.service +Before=gdm.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/sbin/aurora-sep-greeter-fingerprint +ExecStop=/usr/local/sbin/aurora-sep-greeter-fingerprint stop + +[Install] +WantedBy=gdm.service diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index 3c093add292ba..d27422443be35 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -18,6 +18,7 @@ depends=( kmod sh ) +optdepends=('glib2-devel: fingerprint at the GNOME login screen') source=(load-driver aurora-sep.service fprintd-aurora.conf @@ -25,6 +26,8 @@ source=(load-driver aurora-sep-ready.service fprintd-no-timeout.conf.in logind-aurora-sep.conf + aurora-sep-greeter-fingerprint + aurora-sep-greeter-fingerprint.service README.md XART-SAFETY.md) sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' @@ -34,7 +37,9 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '5a5e5c7e93d1462d9d1023446fdb297926aeb93d8988867b515aca7adafe4487' '8d8df0ba2a6861b2edb2110e606906d84ca4848617621ad3cb48493a1522971d' '0695bd9ce4c207241db13c3ea07819a032d8213417021378f5015ab9317cde12' - '3f5a8a4a19e15268552264da95d4848be31629c49f3f64f377c590cf6ef9e1cb' + '5936eb76db666496a8700dade7ac6bcb3785b517958dcea10685690f1822c86f' + 'ffab97b20bfd5f0eb1ab641179a131843349cf16890fff0cf2d893f0b5681b37' + '226a5f5f2244583b5b10d9a5b3e134f5b155065fd8dce850c828f98f085fc931' '83ae33b34d4f4ff42be7ba22b6d5557c3c5776075753ea288bef986645687f23') # A unit and the helper it runs ($1, installed as $3), pointed at the @@ -53,6 +58,8 @@ _unit() { package() { _unit load-driver aurora-sep.service aurora-sep-load _unit aurora-sep-ready aurora-sep-ready.service aurora-sep-ready + _unit aurora-sep-greeter-fingerprint aurora-sep-greeter-fingerprint.service \ + aurora-sep-greeter-fingerprint install -Dm644 fprintd-aurora.conf \ "$pkgdir/usr/lib/systemd/system/fprintd.service.d/aurora-sep.conf" diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint new file mode 120000 index 0000000000000..dd1bb06545313 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint @@ -0,0 +1 @@ +../../../aurora-sep-greeter-fingerprint \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint.service b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint.service new file mode 120000 index 0000000000000..bf6aeca94eb78 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint.service @@ -0,0 +1 @@ +../../../aurora-sep-greeter-fingerprint.service \ No newline at end of file diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec index a916df31cc0a5..648409fb7c463 100644 --- a/tools/aurora-sep/packaging/fedora/aurora-sep.spec +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -19,6 +19,8 @@ Source5: aurora-sep-ready Source6: aurora-sep-ready.service Source7: fprintd-no-timeout.conf.in Source8: logind-aurora-sep.conf +Source9: aurora-sep-greeter-fingerprint +Source10: aurora-sep-greeter-fingerprint.service BuildArch: noarch BuildRequires: systemd-rpm-macros @@ -28,6 +30,8 @@ Requires: coreutils Requires: fprintd Requires: keyutils Requires: kmod +# Only for aurora-sep-greeter-fingerprint.service, which is off by default. +Suggests: /usr/bin/gresource %{?systemd_requires} %description @@ -57,6 +61,7 @@ unit() { } unit %{SOURCE0} %{SOURCE1} aurora-sep-load unit %{SOURCE5} %{SOURCE6} aurora-sep-ready +unit %{SOURCE9} %{SOURCE10} aurora-sep-greeter-fingerprint install -Dpm 0644 %{SOURCE2} \ %{buildroot}%{_unitdir}/fprintd.service.d/aurora-sep.conf grep -qx 'ExecStart=%{_libexecdir}/fprintd' %{_unitdir}/fprintd.service @@ -69,21 +74,23 @@ install -d %{buildroot}%{_presetdir} echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset %post -%systemd_post aurora-sep.service aurora-sep-ready.service +%systemd_post aurora-sep.service aurora-sep-ready.service aurora-sep-greeter-fingerprint.service %preun -%systemd_preun aurora-sep.service aurora-sep-ready.service +%systemd_preun aurora-sep.service aurora-sep-ready.service aurora-sep-greeter-fingerprint.service %postun # Never restart them: the driver attaches to the enclave once per boot. -%systemd_postun aurora-sep.service aurora-sep-ready.service +%systemd_postun aurora-sep.service aurora-sep-ready.service aurora-sep-greeter-fingerprint.service %files %doc README.md XART-SAFETY.md %{_libexecdir}/aurora-sep-load %{_libexecdir}/aurora-sep-ready +%{_libexecdir}/aurora-sep-greeter-fingerprint %{_unitdir}/aurora-sep.service %{_unitdir}/aurora-sep-ready.service +%{_unitdir}/aurora-sep-greeter-fingerprint.service %dir %{_unitdir}/fprintd.service.d %{_unitdir}/fprintd.service.d/aurora-sep.conf %{_unitdir}/fprintd.service.d/50-aurora-sep-no-timeout.conf @@ -95,6 +102,7 @@ echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset - Activate Touch ID at boot, before fprintd starts - Keep fprintd running - Suspend rather than power off on the Touch ID button +- Offer fingerprint at the GNOME 51 login screen, off by default * Sun Oct 11 2026 Ryan Murray - 1-1 - Package the loader, its service and the fprintd policy From a081443dfcc9ec1a616345111523300eebab4379 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 23:08:47 +1100 Subject: [PATCH 0780/2680] tools: aurora-sep: leave libfprint to aurora-sep-userspace libfprint with the aurora driver, as a source patch and as Fedora and Arch Linux packages, is now maintained in https://github.com/aurora-silicon/aurora-sep-userspace. Two copies of the userspace driver, built from different trees with colliding package releases, would drift apart. Drop the libfprint patch, the Arch libfprint-aurora package and the libfprint rebuild from the Fedora build script. This tree keeps the kernel side's userspace pieces: the loader, its services, the fprintd policy and the desktop integration, packaged as aurora-sep. The README and the package descriptions point to aurora-sep-userspace for the driver. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/M1-SUPPORT.md | 4 +- tools/aurora-sep/README.md | 62 +- .../packaging/arch/aurora-sep/PKGBUILD | 7 +- .../packaging/arch/libfprint-aurora/PKGBUILD | 73 - .../arch/libfprint-aurora/apple-sep.uapi | 1 - .../libfprint-1.94.100-apple-sep.patch | 1 - .../packaging/fedora/aurora-sep.spec | 8 +- tools/aurora-sep/packaging/fedora/build.sh | 69 +- .../libfprint-1.94.100-apple-sep.patch | 1215 ----------------- tools/aurora-sep/tests/test_packaging.py | 15 +- 10 files changed, 36 insertions(+), 1419 deletions(-) delete mode 100644 tools/aurora-sep/packaging/arch/libfprint-aurora/PKGBUILD delete mode 120000 tools/aurora-sep/packaging/arch/libfprint-aurora/apple-sep.uapi delete mode 120000 tools/aurora-sep/packaging/arch/libfprint-aurora/libfprint-1.94.100-apple-sep.patch delete mode 100644 tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch diff --git a/tools/aurora-sep/M1-SUPPORT.md b/tools/aurora-sep/M1-SUPPORT.md index 774af1ef10ca0..6d3c6fa93985a 100644 --- a/tools/aurora-sep/M1-SUPPORT.md +++ b/tools/aurora-sep/M1-SUPPORT.md @@ -75,8 +75,8 @@ capture-guidance fixes that the older J313 test checkout did not contain. Build/test results for this consolidation are not a new hardware regression test on Neo, M2, or every M1 board. The consolidated SEP module compiled against the prepared J313 kernel build. Its four identity-binding tests, ten extracted -entropy-method tests, and eight calibration/xART tests passed. The bundled -patch applied to clean libfprint 1.94.100 and built with `drivers=aurora`; +entropy-method tests, and eight calibration/xART tests passed. The libfprint +patch (now maintained in aurora-silicon/aurora-sep-userspace) applied to clean libfprint 1.94.100 and built with `drivers=aurora`; the three core unit-test suites passed. In that minimal build, 33 unrelated driver/hwdb tests were skipped and AppStream validation rejected the generated empty `` device list. No full-kernel rebuild or new hardware run of diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 23c3f589b5289..96632f81c585d 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -53,32 +53,13 @@ window shifted 90 APFS blocks beyond the actual `.gl` extent. This proves the locator can misidentify the extent; it does not identify the historical cause of any particular damaged container. -For fingerprint support, apply -`patches/libfprint-1.94.100-apple-sep.patch` to libfprint 1.94.100, build and -install libfprint, then install the fprintd device policy: +For fingerprint support, install libfprint with the aurora driver, then the +fprintd device policy below. The userspace driver is maintained in +https://github.com/aurora-silicon/aurora-sep-userspace (libfprint with the +aurora driver, as source and as distribution packages); this tree carries only +the kernel driver, its loader and its service. The driver behaviour fprintd +sees: -```sh -git clone --depth 1 --branch v1.94.100 \ - https://gitlab.freedesktop.org/libfprint/libfprint.git libfprint-aurora -cd libfprint-aurora -git apply ../linux/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch -meson setup build -Dprefix=/usr -Ddrivers=default -Dintrospection=false \ - -Dgtk-examples=false -Ddoc=false -Dinstalled-tests=false -ninja -C build -meson test -C build --print-errorlogs -``` - -Adjust the patch path for your checkout. The patch includes a fix for an -upstream Meson test loop when introspection is disabled. In the Aurora -`linux-build` guest, the additional packages needed for this build are -`libgudev libgusb pixman cairo glib2-devel`; `meson` and `ninja` are already -in the release image. The default driver set retains all normal libfprint -drivers and adds Aurora. This build was checked on aarch64: the shared library -and all default drivers compiled, with 0 test failures (most device tests -skip without their optional mocks/hardware). That does **not** prove Touch ID -works on m2. The patched library is installed privately under -`/opt/aurora-sep` there, with fprintd loading it via a service drop-in while -the distro libfprint remains untouched. fprintd discovers one Apple SEP sensor. The Aurora driver now reports stage progress only when the kernel's stage number increases; guidance and stage-zero events are not captured samples. fprintd 1.94.5 itself emits an initial `enroll-stage-passed` after its @@ -146,35 +127,26 @@ by the Quickshell lock screen. ## Packages -`packaging/` builds the same pieces as distribution packages: libfprint with -the aurora driver, which replaces the distribution's libfprint so fprintd needs -no library path, and `aurora-sep` with the loader, its service and the fprintd -policy. Installing `aurora-sep` lets fprintd load the driver with xART writes -on, as above. +`packaging/` builds `aurora-sep`, with the loader, its service and the fprintd +policy, as a distribution package. libfprint with the aurora driver comes from +https://github.com/aurora-silicon/aurora-sep-userspace; install both. Installing +`aurora-sep` lets fprintd load the driver with xART writes on, as above. -On Fedora, `packaging/fedora/build.sh` rebuilds the release's own libfprint -source package with the patch added (1.94.100 on Fedora 45) and builds -`aurora-sep`. It installs build dependencies, so run it in a throwaway -container from the top of this tree: +On Fedora, `packaging/fedora/build.sh` builds `aurora-sep`. It installs build +dependencies, so run it in a throwaway container from the top of this tree: ```sh mkdir rpms podman run --rm -v "$PWD":/src:ro,z -v "$PWD/rpms":/out:z \ registry.fedoraproject.org/fedora:45 \ /src/tools/aurora-sep/packaging/fedora/build.sh /out -sudo dnf install rpms/libfprint-1.94.100-*.aurora1.*.rpm \ - rpms/aurora-sep-1-*.noarch.rpm +sudo dnf install rpms/aurora-sep-1-*.noarch.rpm ``` -A later Fedora libfprint update replaces the rebuilt one; rebuild after it, or -hold it with `dnf versionlock add libfprint`. - -On Arch Linux and Arch Linux ARM, build `libfprint-aurora` (it replaces -`libfprint`) and then `aurora-sep`: +On Arch Linux and Arch Linux ARM, build `aurora-sep`: ```sh -cd tools/aurora-sep/packaging/arch/libfprint-aurora && makepkg -si -cd ../aurora-sep && makepkg -si +cd tools/aurora-sep/packaging/arch/aurora-sep && makepkg -si ``` fprintd starts the loader the first time it runs. Loading the driver at boot @@ -272,5 +244,5 @@ Disabling it and rebooting returns the login screen to GNOME's behaviour. The canonical ioctl ABI is `include/uapi/linux/apple-sep.h`. Export this checkout's headers with `make headers_install INSTALL_HDR_PATH=` and add that directory's `include` to the libfprint build include path. The -libfprint patch uses `` directly; it has no copied ABI -header. Interface version, ioctl numbers and fixed-width layouts are unchanged. +aurora driver in aurora-sep-userspace uses `` directly; it +has no copied ABI header. Interface version, ioctl numbers and fixed-width layouts are unchanged. diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index d27422443be35..15712b3bdf4e2 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -2,11 +2,12 @@ # # The loader and desktop integration for the Apple SEP fingerprint driver. # The local sources are links into tools/aurora-sep; after changing any of -# them, run updpkgsums here. libfprint-aurora provides the libfprint driver. +# them, run updpkgsums here. libfprint with the aurora driver comes from +# https://github.com/aurora-silicon/aurora-sep-userspace. pkgname=aurora-sep pkgver=1 -pkgrel=2 +pkgrel=3 pkgdesc="Loader and desktop integration for the Apple SEP (Touch ID) driver" url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" arch=(any) @@ -39,7 +40,7 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '0695bd9ce4c207241db13c3ea07819a032d8213417021378f5015ab9317cde12' '5936eb76db666496a8700dade7ac6bcb3785b517958dcea10685690f1822c86f' 'ffab97b20bfd5f0eb1ab641179a131843349cf16890fff0cf2d893f0b5681b37' - '226a5f5f2244583b5b10d9a5b3e134f5b155065fd8dce850c828f98f085fc931' + 'd7a09f31367c181a2cbda63cd58104d6117136fb5db5b20f4a3996ea3af7025e' '83ae33b34d4f4ff42be7ba22b6d5557c3c5776075753ea288bef986645687f23') # A unit and the helper it runs ($1, installed as $3), pointed at the diff --git a/tools/aurora-sep/packaging/arch/libfprint-aurora/PKGBUILD b/tools/aurora-sep/packaging/arch/libfprint-aurora/PKGBUILD deleted file mode 100644 index 12bfdd7222953..0000000000000 --- a/tools/aurora-sep/packaging/arch/libfprint-aurora/PKGBUILD +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# libfprint with the aurora driver for the Apple SEP fingerprint sensor. -# The local sources are links into tools/aurora-sep and to -# include/uapi/linux/apple-sep.h; after changing either file, run updpkgsums -# here. - -pkgname=libfprint-aurora -_pkgname=libfprint -pkgver=1.94.100 -pkgrel=1 -pkgdesc="Library for fingerprint readers, with the Apple SEP (Touch ID) driver" -url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" -arch=(aarch64 x86_64) -license=(LGPL-2.1-or-later) -depends=( - libgcc - glib2 - glibc - libgudev - libgusb - openssl - pixman -) -makedepends=( - glib2-devel - gobject-introspection - gtk-doc - meson - python-cairo - python-gobject - systemd -) -checkdepends=( - cairo - umockdev -) -provides=("$_pkgname=$pkgver" libfprint-2.so) -conflicts=("$_pkgname") -source=("https://gitlab.freedesktop.org/libfprint/libfprint/-/archive/v$pkgver/$_pkgname-v$pkgver.tar.gz" - "libfprint-$pkgver-apple-sep.patch" - apple-sep.uapi) -sha256sums=('edc90e02f330a7595ceaf37f2c6ec32ed43541347fe936d3273b0bb2524fd19c' - '00f999dab395c0f6cd5ffda94099f898eb48d001e19036c500fe2713f76aa076' - '00cb09430461e0b0772619bc2054c8f9794484fbc3d807268955c0fb2de45c10') - -prepare() { - cd "$_pkgname-v$pkgver" - patch -Np1 -i "../libfprint-$pkgver-apple-sep.patch" - - # The driver includes , which the distribution's kernel - # headers do not carry. - install -Dm644 ../apple-sep.uapi ../aurora-include/linux/apple-sep.h -} - -build() { - local meson_options=( - -D drivers=all - -D installed-tests=false - ) - - CFLAGS+=" -I$srcdir/aurora-include" - arch-meson "$_pkgname-v$pkgver" build "${meson_options[@]}" - meson compile -C build -} - -check() { - meson test -C build --print-errorlogs -} - -package() { - meson install -C build --destdir "$pkgdir" -} diff --git a/tools/aurora-sep/packaging/arch/libfprint-aurora/apple-sep.uapi b/tools/aurora-sep/packaging/arch/libfprint-aurora/apple-sep.uapi deleted file mode 120000 index 3ba9077515e02..0000000000000 --- a/tools/aurora-sep/packaging/arch/libfprint-aurora/apple-sep.uapi +++ /dev/null @@ -1 +0,0 @@ -../../../../../include/uapi/linux/apple-sep.h \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/libfprint-aurora/libfprint-1.94.100-apple-sep.patch b/tools/aurora-sep/packaging/arch/libfprint-aurora/libfprint-1.94.100-apple-sep.patch deleted file mode 120000 index 40c7665bd03ef..0000000000000 --- a/tools/aurora-sep/packaging/arch/libfprint-aurora/libfprint-1.94.100-apple-sep.patch +++ /dev/null @@ -1 +0,0 @@ -../../../patches/libfprint-1.94.100-apple-sep.patch \ No newline at end of file diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec index 648409fb7c463..b4dd9ca4391b6 100644 --- a/tools/aurora-sep/packaging/fedora/aurora-sep.spec +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -5,7 +5,7 @@ Name: aurora-sep Version: 1 -Release: 2%{?dist} +Release: 3%{?dist} Summary: Loader and desktop integration for the Apple SEP fingerprint driver License: GPL-2.0-only @@ -40,7 +40,8 @@ the secure enclave seals and unseals a trusted key and /dev/sep-bio appears, and lets fprintd reach /dev/sep-bio. The loader runs at boot, and fprintd also pulls it in when it starts. Touch ID is then activated, before fprintd starts, and pressing the Touch ID button suspends rather than powers off. -libfprint needs the aurora driver, which this package does not provide. Read XART-SAFETY.md before installing it on +libfprint needs the aurora driver, which this package does not provide; it is +packaged in https://github.com/aurora-silicon/aurora-sep-userspace. Read XART-SAFETY.md before installing it on a machine that also runs macOS. %prep @@ -98,6 +99,9 @@ echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset %{_presetdir}/80-aurora-sep.preset %changelog +* Sun Oct 11 2026 Ryan Murray - 1-3 +- libfprint with the aurora driver is now built in aurora-sep-userspace + * Sun Oct 11 2026 Ryan Murray - 1-2 - Activate Touch ID at boot, before fprintd starts - Keep fprintd running diff --git a/tools/aurora-sep/packaging/fedora/build.sh b/tools/aurora-sep/packaging/fedora/build.sh index 27fb64e28aafd..a1331bbdb266b 100755 --- a/tools/aurora-sep/packaging/fedora/build.sh +++ b/tools/aurora-sep/packaging/fedora/build.sh @@ -1,13 +1,13 @@ #!/bin/sh # SPDX-License-Identifier: GPL-2.0-only # -# Build the Fedora packages for Touch ID on the Apple SEP: +# Build the Fedora package for Touch ID on the Apple SEP: # -# - libfprint: Fedora's own source package, rebuilt with the aurora driver -# patch from ../../patches and nothing else changed. The release gains an -# ".aurora1" suffix, so it replaces the distribution's build in place. # - aurora-sep: the driver loader, its service and the fprintd policy. # +# libfprint with the aurora driver is built and packaged in +# https://github.com/aurora-silicon/aurora-sep-userspace. +# # It installs the build dependencies with dnf, so run it as root in a # throwaway container of the Fedora release you are building for: # @@ -21,13 +21,6 @@ set -eu out=${1:?usage: build.sh } here=$(cd "$(dirname "$0")" && pwd) sep=$(cd "$here/../.." && pwd) -patch=$sep/patches/libfprint-1.94.100-apple-sep.patch -header=${APPLE_SEP_HEADER:-$sep/../../include/uapi/linux/apple-sep.h} -patched_version=1.94.100 - -for f in "$patch" "$header"; do - [ -f "$f" ] || { echo "build.sh: $f is missing" >&2; exit 1; } -done mkdir -p "$out" top=$(mktemp -d) @@ -36,60 +29,6 @@ rpmtop=$top/rpmbuild dnf -y install rpm-build systemd-rpm-macros -# Fedora's libfprint, exactly as the distribution builds it. -(cd "$top" && dnf -y download --source libfprint) -srpm=$(ls "$top"/libfprint-*.src.rpm) -rpm -i --define "_topdir $rpmtop" "$srpm" -spec=$rpmtop/SPECS/libfprint.spec - -version=$(rpmspec -q --srpm --qf '%{version}\n' "$spec") -if [ "$version" != "$patched_version" ]; then - echo "build.sh: this Fedora ships libfprint $version, the patch is for $patched_version" >&2 - exit 1 -fi -cp "$patch" "$header" "$rpmtop/SOURCES/" - -# Add the patch and the header, mark the release, and point the compiler at -# the header: is not in the distribution's kernel headers. -# %autosetup applies every PatchN. Stop if any anchor has moved. -awk -v patch="$(basename "$patch")" -v date="$(LC_ALL=C date -u '+%a %b %d %Y')" ' - /^Release:/ && !release { - sub(/[[:space:]]*$/, ".aurora1"); print; release = 1; next - } - /^Source0:/ && !source { - print - print "Source9000: apple-sep.h" - print "Patch9000: " patch - source = 1; next - } - /^%meson[[:space:]]/ && !build { - print "# The aurora driver includes from the Aurora kernel." - print "mkdir -p aurora-include/linux" - print "cp -p %{SOURCE9000} aurora-include/linux/" - print "export CFLAGS=\"%{build_cflags} -I$PWD/aurora-include\"" - print; build = 1; next - } - /^%changelog/ && !changelog { - print - print "* " date " Aurora Silicon - " "%{version}-%{release}" - print "- Add the Apple SEP (aurora) driver from aurora-silicon/linux" - print " tools/aurora-sep/patches/" patch - print "" - changelog = 1; next - } - { print } - END { - if (!(release && source && build && changelog)) { - print "build.sh: libfprint.spec changed shape; update the edit" > "/dev/stderr" - exit 1 - } - } -' "$spec" >"$spec.aurora" -mv "$spec.aurora" "$spec" - -dnf -y builddep "$spec" -rpmbuild --define "_topdir $rpmtop" -ba "$spec" - # The loader, its units and the desktop integration, from this tree. dnf -y builddep "$here/aurora-sep.spec" rpmbuild --define "_topdir $rpmtop" --define "_sourcedir $sep" \ diff --git a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch deleted file mode 100644 index 5163351683362..0000000000000 --- a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch +++ /dev/null @@ -1,1215 +0,0 @@ -diff --git a/libfprint/drivers/aurora/aurora.c b/libfprint/drivers/aurora/aurora.c -new file mode 100644 ---- /dev/null -+++ b/libfprint/drivers/aurora/aurora.c -@@ -0,0 +1,991 @@ -+/* -+ * Aurora SEP fingerprint driver -+ * -+ * The sensor is matched inside Apple's secure enclave. This driver never sees -+ * an image, a template, or key material: it starts operations on the kernel -+ * device and reports the enclave's verdict. That is why it can be short. -+ * -+ * SPDX-License-Identifier: LGPL-2.1-or-later -+ */ -+ -+#define FP_COMPONENT "apple-sep" -+ -+#include "drivers_api.h" -+#include -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+/* Why the kernel ended an enrol or verify, in a poll's 'status'. Headers from -+ * before these were named carry the same numbers; those kernels never report -+ * RETRY while an enrolment is running, nor REJECTED at all. */ -+#ifndef SEP_BIO_STATUS_REJECTED -+#define SEP_BIO_STATUS_NONE 0 -+#define SEP_BIO_STATUS_SENSOR 1 -+#define SEP_BIO_STATUS_ENCLAVE 2 -+#define SEP_BIO_STATUS_TOO_MANY 3 -+#define SEP_BIO_STATUS_TIMEOUT 4 -+#define SEP_BIO_STATUS_RETRY 5 -+#define SEP_BIO_STATUS_UNFILED 6 -+#define SEP_BIO_STATUS_REJECTED 7 -+#endif -+ -+struct _FpiDeviceAurora -+{ -+ FpDevice parent; -+ -+ gint fd; -+ guint watch_id; -+ guint32 enroll_last_reported_stage; -+ -+ guint8 nonce[SEP_BIO_NONCE_LEN]; -+ gboolean nonce_valid; -+}; -+ -+G_DECLARE_FINAL_TYPE (FpiDeviceAurora, fpi_device_aurora, FPI, DEVICE_AURORA, FpDevice) -+G_DEFINE_TYPE (FpiDeviceAurora, fpi_device_aurora, FP_TYPE_DEVICE) -+ -+/* ------------------------------------------------------------------ */ -+/* helpers */ -+/* ------------------------------------------------------------------ */ -+ -+static gboolean -+aurora_ioctl (FpiDeviceAurora *self, unsigned long req, void *arg, GError **error) -+{ -+ if (self->fd < 0) -+ { -+ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_NOT_OPEN, -+ "device is not open"); -+ return FALSE; -+ } -+ -+ if (ioctl (self->fd, req, arg) < 0) -+ { -+ gint err = errno; -+ -+ switch (err) -+ { -+ case ENODEV: -+ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_REMOVED, -+ "the enclave reports no fingerprint sensor"); -+ break; -+ -+ case EPERM: -+ case EACCES: -+ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_NOT_SUPPORTED, -+ "not permitted"); -+ break; -+ -+ case ENOSPC: -+ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_FULL, -+ "three SEP fingerprint slots are full; delete one before enrolling"); -+ break; -+ -+ case ENOENT: -+ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_NOT_FOUND, -+ "no such identity"); -+ break; -+ -+ default: -+ g_set_error (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_PROTO, -+ "ioctl failed: %s", g_strerror (err)); -+ break; -+ } -+ return FALSE; -+ } -+ -+ return TRUE; -+} -+ -+static const gchar * -+aurora_status_reason (guint32 status) -+{ -+ switch (status) -+ { -+ case SEP_BIO_STATUS_NONE: -+ return "it was cancelled or ended by the system"; -+ -+ case SEP_BIO_STATUS_SENSOR: -+ return "the fingerprint sensor failed"; -+ -+ case SEP_BIO_STATUS_ENCLAVE: -+ return "the secure enclave refused a step"; -+ -+ case SEP_BIO_STATUS_TOO_MANY: -+ return "the enclave did not complete the enrolment within its captures"; -+ -+ case SEP_BIO_STATUS_TIMEOUT: -+ return "no finger was placed on the sensor in time"; -+ -+ case SEP_BIO_STATUS_RETRY: -+ return "the capture was not usable"; -+ -+ case SEP_BIO_STATUS_UNFILED: -+ return "the fingerprint was enrolled but could not be recorded"; -+ -+ case SEP_BIO_STATUS_REJECTED: -+ return "too many captures in a row were not usable; " -+ "rest more of the finger flat on the sensor and enrol again"; -+ -+ default: -+ return NULL; -+ } -+} -+ -+static GError * -+aurora_failure_new (const gchar *operation, guint32 status) -+{ -+ const gchar *reason = aurora_status_reason (status); -+ -+ if (reason == NULL) -+ return fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, -+ "%s failed, driver status 0x%x", -+ operation, status); -+ -+ return fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, -+ "%s failed: %s (driver status %u)", -+ operation, reason, status); -+} -+ -+/* The kernel reports an unusable capture without saying why it was unusable, -+ * so the general retry is the most specific code that is always true. */ -+static GError * -+aurora_retry_new (void) -+{ -+ return fpi_device_retry_new_msg (FP_DEVICE_RETRY_GENERAL, -+ "the capture was not usable; touch the sensor again"); -+} -+ -+/* The label is how the host recognises its own records in the device's list. -+ * libfprint already has an encoding for exactly this -- it carries the enroll -+ * date, the finger and the username -- so use it rather than inventing one. */ -+static gboolean -+aurora_label_from_print (FpPrint *print, guint8 *label_out, GError **error) -+{ -+ g_autofree gchar *user_id = NULL; -+ -+ if (print == NULL) -+ { -+ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, -+ "no print to label"); -+ return FALSE; -+ } -+ -+ user_id = fpi_print_generate_user_id (print); -+ -+ if (user_id == NULL || strlen (user_id) >= SEP_BIO_LABEL_LEN) -+ { -+ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, -+ "print label does not fit the device record"); -+ return FALSE; -+ } -+ -+ memset (label_out, 0, SEP_BIO_LABEL_LEN); -+ memcpy (label_out, user_id, strlen (user_id)); -+ return TRUE; -+} -+ -+static void -+aurora_stop_watch (FpiDeviceAurora *self) -+{ -+ if (self->watch_id != 0) -+ { -+ g_source_remove (self->watch_id); -+ self->watch_id = 0; -+ } -+} -+ -+/* Build the FpPrint that represents one enclave-stored identity. The identity -+ * UUID is all we hold; there is no template on this side to store. */ -+static FpPrint * -+aurora_print_from_uuid (FpiDeviceAurora *self, -+ const guint8 *uuid, -+ const guint8 *label) -+{ -+ FpPrint *print = fp_print_new (FP_DEVICE (self)); -+ GVariant *data; -+ -+ data = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, uuid, -+ SEP_BIO_UUID_LEN, 1); -+ -+ fpi_print_set_type (print, FPI_PRINT_RAW); -+ fpi_print_set_device_stored (print, TRUE); -+ g_object_set (print, "fpi-data", data, NULL); -+ -+ /* The label came from the device. Treat it as untrusted bytes: bound it to -+ * the buffer before anything reads it as a string. */ -+ if (label != NULL) -+ { -+ gchar safe[SEP_BIO_LABEL_LEN]; -+ -+ memcpy (safe, label, SEP_BIO_LABEL_LEN); -+ safe[SEP_BIO_LABEL_LEN - 1] = '\0'; -+ -+ if (safe[0] != '\0') -+ fpi_print_fill_from_user_id (print, safe); -+ } -+ -+ return print; -+} -+ -+static gboolean -+aurora_uuid_from_print (FpPrint *print, guint8 *uuid_out, GError **error) -+{ -+ g_autoptr(GVariant) data = NULL; -+ const guint8 *raw; -+ gsize len = 0; -+ -+ g_object_get (print, "fpi-data", &data, NULL); -+ -+ if (data == NULL || !g_variant_is_of_type (data, G_VARIANT_TYPE ("ay"))) -+ { -+ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, -+ "print does not carry an enclave identity"); -+ return FALSE; -+ } -+ -+ raw = g_variant_get_fixed_array (data, &len, 1); -+ if (raw == NULL || len != SEP_BIO_UUID_LEN) -+ { -+ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, -+ "enclave identity is the wrong size"); -+ return FALSE; -+ } -+ -+ memcpy (uuid_out, raw, SEP_BIO_UUID_LEN); -+ return TRUE; -+} -+ -+/* Compare by enclave identity rather than by whole print. -+ * -+ * fp_print_equal() compares host-side labels too -- the username and finger a -+ * print was filed under. Those are ours, not the enclave's: the device knows -+ * only the identity it stored. Comparing whole prints made a correct match -+ * from fprintd, which files prints under a username, read as a non-match -+ * against the bare print the device hands back. The UUID is the ground truth, -+ * so compare that. */ -+static gboolean -+aurora_print_has_uuid (FpPrint *print, const guint8 *uuid) -+{ -+ guint8 stored[SEP_BIO_UUID_LEN]; -+ -+ if (print == NULL) -+ return FALSE; -+ -+ if (!aurora_uuid_from_print (print, stored, NULL)) -+ return FALSE; -+ -+ return memcmp (stored, uuid, SEP_BIO_UUID_LEN) == 0; -+} -+ -+/* ------------------------------------------------------------------ */ -+/* probe / open / close */ -+/* ------------------------------------------------------------------ */ -+ -+static void -+aurora_probe (FpDevice *device) -+{ -+ struct sep_bio_info info = { 0 }; -+ const gchar *path; -+ g_autofree gchar *serial = NULL; -+ gint fd; -+ -+ path = fpi_device_get_udev_data (device, FPI_DEVICE_UDEV_SUBTYPE_MISC); -+ if (path == NULL) -+ { -+ fpi_device_probe_complete (device, NULL, NULL, -+ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, -+ "no sep-bio node")); -+ return; -+ } -+ -+ fd = open (path, O_RDWR | O_CLOEXEC); -+ if (fd < 0) -+ { -+ fpi_device_probe_complete (device, NULL, NULL, -+ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, -+ "cannot open %s: %s", -+ path, g_strerror (errno))); -+ return; -+ } -+ -+ if (ioctl (fd, SEP_BIO_GET_INFO, &info) < 0) -+ { -+ gint err = errno; -+ -+ close (fd); -+ fpi_device_probe_complete (device, NULL, NULL, -+ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, -+ "cannot query device: %s", -+ g_strerror (err))); -+ return; -+ } -+ -+ close (fd); -+ -+ if (info.version != SEP_BIO_IFACE_VERSION) -+ { -+ fpi_device_probe_complete (device, NULL, NULL, -+ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, -+ "interface version %u, expected %u", -+ info.version, -+ SEP_BIO_IFACE_VERSION)); -+ return; -+ } -+ -+ /* A driver that claims a device it cannot use makes the whole stack look -+ * broken to the user. If the enclave has no sensor, say so plainly and let -+ * fprintd report "no devices" rather than a device that fails every touch. */ -+ if (!info.sensor_present) -+ { -+ fp_dbg ("enclave reports no fingerprint sensor attached"); -+ fpi_device_probe_complete (device, NULL, NULL, -+ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, -+ "the secure enclave reports no " -+ "fingerprint sensor attached")); -+ return; -+ } -+ -+ if (info.enroll_stages > 0) -+ fpi_device_set_nr_enroll_stages (device, info.enroll_stages); -+ -+ serial = g_strdup ("apple-sep"); -+ fpi_device_probe_complete (device, serial, NULL, NULL); -+} -+ -+static void -+aurora_open (FpDevice *device) -+{ -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ const gchar *path; -+ -+ path = fpi_device_get_udev_data (device, FPI_DEVICE_UDEV_SUBTYPE_MISC); -+ if (path == NULL) -+ { -+ fpi_device_open_complete (device, -+ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, -+ "no sep-bio node")); -+ return; -+ } -+ -+ /* The kernel admits one opener at a time; EBUSY means something else holds -+ * the sensor, which is a real condition and not a driver fault. */ -+ self->fd = open (path, O_RDWR | O_CLOEXEC); -+ if (self->fd < 0) -+ { -+ gint err = errno; -+ -+ fpi_device_open_complete (device, -+ fpi_device_error_new_msg (err == EBUSY -+ ? FP_DEVICE_ERROR_BUSY -+ : FP_DEVICE_ERROR_NOT_SUPPORTED, -+ "cannot open %s: %s", -+ path, g_strerror (err))); -+ return; -+ } -+ -+ fpi_device_open_complete (device, NULL); -+} -+ -+static void -+aurora_close (FpDevice *device) -+{ -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ -+ aurora_stop_watch (self); -+ -+ if (self->fd >= 0) -+ { -+ /* Closing the description cancels anything in flight, by contract. */ -+ close (self->fd); -+ self->fd = -1; -+ } -+ -+ self->nonce_valid = FALSE; -+ memset (self->nonce, 0, sizeof (self->nonce)); -+ fpi_device_report_finger_status (device, FP_FINGER_STATUS_NONE); -+ -+ fpi_device_close_complete (device, NULL); -+} -+ -+static void -+aurora_cancel (FpDevice *device) -+{ -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ -+ fpi_device_report_finger_status (device, FP_FINGER_STATUS_NONE); -+ if (self->fd >= 0) -+ ioctl (self->fd, SEP_BIO_CANCEL); -+} -+ -+static void -+aurora_report_guidance (FpDevice *device, guint32 guidance, gboolean active) -+{ -+ FpFingerStatusFlags status = FP_FINGER_STATUS_NONE; -+ -+ if (active) -+ { -+ if (guidance == SEP_BIO_GUIDANCE_HOLD_STILL) -+ status = FP_FINGER_STATUS_PRESENT; -+ else if (guidance == SEP_BIO_GUIDANCE_PLACE || -+ guidance == SEP_BIO_GUIDANCE_LIFT_AND_MOVE) -+ status = FP_FINGER_STATUS_NEEDED; -+ } -+ -+ fpi_device_report_finger_status (device, status); -+} -+ -+/* ------------------------------------------------------------------ */ -+/* enrolment */ -+/* ------------------------------------------------------------------ */ -+ -+static gboolean -+aurora_enrol_ready (gint fd, GIOCondition condition, gpointer user_data) -+{ -+ FpDevice *device = FP_DEVICE (user_data); -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct sep_bio_enrol_poll poll_res = { 0 }; -+ g_autoptr(GError) error = NULL; -+ FpPrint *print = NULL; -+ -+ if (!aurora_ioctl (self, SEP_BIO_ENROL_POLL, &poll_res, &error)) -+ { -+ self->watch_id = 0; -+ fpi_device_report_finger_status (device, FP_FINGER_STATUS_NONE); -+ fpi_device_enroll_complete (device, NULL, g_steal_pointer (&error)); -+ return G_SOURCE_REMOVE; -+ } -+ -+ aurora_report_guidance (device, poll_res.guidance, -+ poll_res.state == SEP_BIO_STATE_PENDING || -+ poll_res.state == SEP_BIO_STATE_PROGRESS); -+ -+ switch (poll_res.state) -+ { -+ case SEP_BIO_STATE_PENDING: -+ case SEP_BIO_STATE_PROGRESS: -+ /* fprintd calls this "enroll-stage-passed". A guidance wakeup or an -+ * initial stage-zero event must not masquerade as a captured sample. */ -+ if (poll_res.state == SEP_BIO_STATE_PROGRESS && -+ poll_res.stage > self->enroll_last_reported_stage) -+ { -+ self->enroll_last_reported_stage = poll_res.stage; -+ fpi_device_enroll_progress (device, poll_res.stage, NULL, NULL); -+ } -+ -+ /* The latest capture did not count, and the enrolment goes on. The -+ * kernel reports each rejection in one poll only, and an accepted -+ * capture clears it, so a rejection that shares a poll with a new stage -+ * came after that stage. */ -+ if (poll_res.status == SEP_BIO_STATUS_RETRY) -+ fpi_device_enroll_progress (device, self->enroll_last_reported_stage, -+ NULL, aurora_retry_new ()); -+ return G_SOURCE_CONTINUE; -+ -+ case SEP_BIO_STATE_DONE: -+ fpi_device_get_enroll_data (device, &print); -+ if (print != NULL) -+ { -+ GVariant *data = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, -+ poll_res.uuid, -+ SEP_BIO_UUID_LEN, 1); -+ fpi_print_set_type (print, FPI_PRINT_RAW); -+ fpi_print_set_device_stored (print, TRUE); -+ g_object_set (print, "fpi-data", data, NULL); -+ } -+ self->watch_id = 0; -+ fpi_device_enroll_complete (device, -+ print ? g_object_ref (print) : NULL, -+ print ? NULL -+ : fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, -+ "enrolment produced no print")); -+ return G_SOURCE_REMOVE; -+ -+ case SEP_BIO_STATE_FAILED: -+ self->watch_id = 0; -+ fpi_device_enroll_complete (device, NULL, -+ aurora_failure_new ("enrolment", poll_res.status)); -+ return G_SOURCE_REMOVE; -+ -+ default: -+ self->watch_id = 0; -+ fpi_device_enroll_complete (device, NULL, -+ fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, -+ "enrolment failed, state %u, driver status 0x%x", -+ poll_res.state, poll_res.status)); -+ return G_SOURCE_REMOVE; -+ } -+} -+ -+static void -+aurora_enroll (FpDevice *device) -+{ -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct sep_bio_enrol_start start = { 0 }; -+ g_autoptr(GError) error = NULL; -+ FpPrint *print = NULL; -+ -+ fpi_device_get_enroll_data (device, &print); -+ -+ if (!aurora_label_from_print (print, start.label, &error)) -+ { -+ fpi_device_enroll_complete (device, NULL, g_steal_pointer (&error)); -+ return; -+ } -+ -+ if (!aurora_ioctl (self, SEP_BIO_ENROL_START, &start, &error)) -+ { -+ fpi_device_enroll_complete (device, NULL, g_steal_pointer (&error)); -+ return; -+ } -+ -+ self->enroll_last_reported_stage = 0; -+ aurora_report_guidance (device, SEP_BIO_GUIDANCE_PLACE, TRUE); -+ aurora_stop_watch (self); -+ self->watch_id = g_unix_fd_add (self->fd, G_IO_IN, aurora_enrol_ready, device); -+} -+ -+/* ------------------------------------------------------------------ */ -+/* verify and identify */ -+/* ------------------------------------------------------------------ */ -+ -+/* The single place a match may be declared. -+ * -+ * Everything else in this driver reports failure. A result counts as a match -+ * only if the enclave said DONE and said MATCH, we require it to have bound the -+ * answer to the nonce we generated for this operation (a binding the host -+ * cannot verify cryptographically -- it is enforced by the enclave), and it -+ * arrived within the token's deadline. Any other combination -- including a -+ * state we do not recognise -- is a non-match, never an error that a caller -+ * might interpret loosely. */ -+static gboolean -+aurora_result_is_match (FpiDeviceAurora *self, -+ const struct sep_bio_verify_poll *res) -+{ -+ struct timespec now; -+ guint64 now_ns; -+ gboolean token_set = FALSE; -+ gsize i; -+ -+ if (res->state != SEP_BIO_STATE_DONE) -+ return FALSE; -+ -+ if (res->result != SEP_BIO_MATCH) -+ return FALSE; -+ -+ if (!self->nonce_valid) -+ return FALSE; -+ -+ for (i = 0; i < SEP_BIO_TOKEN_LEN; i++) -+ if (res->token[i] != 0) -+ { -+ token_set = TRUE; -+ break; -+ } -+ -+ if (!token_set) -+ return FALSE; -+ -+ if (clock_gettime (CLOCK_MONOTONIC, &now) != 0) -+ return FALSE; -+ -+ now_ns = (guint64) now.tv_sec * 1000000000ull + (guint64) now.tv_nsec; -+ if (res->deadline_ns == 0 || now_ns > res->deadline_ns) -+ { -+ fp_dbg ("match result arrived after its deadline; refusing it"); -+ return FALSE; -+ } -+ -+ return TRUE; -+} -+ -+/* A verify that ended without a comparison. When the kernel says why, pass -+ * that on: otherwise the log shows only an unspecified error. */ -+static GError * -+aurora_not_compared_new (const struct sep_bio_verify_poll *res) -+{ -+ if (res->state == SEP_BIO_STATE_FAILED) -+ return aurora_failure_new ("verification", res->status); -+ -+ return fpi_device_error_new (FP_DEVICE_ERROR_GENERAL); -+} -+ -+static gboolean -+aurora_verify_ready (gint fd, GIOCondition condition, gpointer user_data) -+{ -+ FpDevice *device = FP_DEVICE (user_data); -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct sep_bio_verify_poll res = { 0 }; -+ FpiDeviceAction action = fpi_device_get_current_action (device); -+ g_autoptr(GError) error = NULL; -+ /* Not a g_autoptr: FpPrint is a GInitiallyUnowned and both report functions -+ * take the floating reference. Unreffing it here as well drops the last -+ * reference under libfprint's feet, and it frees the print again when the -+ * match completes. */ -+ FpPrint *matched = NULL; -+ GError *retry = NULL; -+ -+ if (!aurora_ioctl (self, SEP_BIO_VERIFY_POLL, &res, &error)) -+ { -+ self->watch_id = 0; -+ fpi_device_report_finger_status (device, FP_FINGER_STATUS_NONE); -+ /* Terminal path: void the per-operation nonce like every other exit, so -+ * a nonce cannot be carried into a later operation. */ -+ self->nonce_valid = FALSE; -+ memset (self->nonce, 0, sizeof (self->nonce)); -+ if (action == FPI_DEVICE_ACTION_VERIFY) -+ fpi_device_verify_complete (device, g_steal_pointer (&error)); -+ else -+ fpi_device_identify_complete (device, g_steal_pointer (&error)); -+ return G_SOURCE_REMOVE; -+ } -+ -+ aurora_report_guidance (device, res.guidance, -+ res.state == SEP_BIO_STATE_PENDING || -+ res.state == SEP_BIO_STATE_PROGRESS); -+ -+ if (res.state == SEP_BIO_STATE_PENDING || -+ res.state == SEP_BIO_STATE_PROGRESS) -+ return G_SOURCE_CONTINUE; -+ -+ self->watch_id = 0; -+ -+ if (aurora_result_is_match (self, &res)) -+ matched = aurora_print_from_uuid (self, res.uuid, NULL); -+ /* The capture was not usable, so nothing was compared. A retry tells the -+ * caller to ask for another touch: fprintd starts the match again on its -+ * own, where a failure would end the authentication attempt. */ -+ else if (res.state == SEP_BIO_STATE_FAILED && -+ res.status == SEP_BIO_STATUS_RETRY) -+ retry = aurora_retry_new (); -+ -+ if (action == FPI_DEVICE_ACTION_VERIFY) -+ { -+ FpPrint *expected = NULL; -+ -+ fpi_device_get_verify_data (device, &expected); -+ -+ if (retry != NULL) -+ { -+ fpi_device_verify_report (device, FPI_MATCH_ERROR, NULL, retry); -+ fpi_device_verify_complete (device, NULL); -+ } -+ else if (matched != NULL && aurora_print_has_uuid (expected, res.uuid)) -+ { -+ fpi_device_verify_report (device, FPI_MATCH_SUCCESS, matched, NULL); -+ fpi_device_verify_complete (device, NULL); -+ } -+ else if (res.result == SEP_BIO_NOT_COMPARED) -+ { -+ /* No comparison happened, so there is no verdict to report. Saying -+ * FAIL here would tell the user their finger was rejected when it -+ * was never examined. */ -+ fpi_device_verify_complete (device, aurora_not_compared_new (&res)); -+ } -+ else -+ { -+ fpi_device_verify_report (device, FPI_MATCH_FAIL, matched, NULL); -+ fpi_device_verify_complete (device, NULL); -+ } -+ } -+ else -+ { -+ GPtrArray *gallery = NULL; -+ FpPrint *hit = NULL; -+ -+ fpi_device_get_identify_data (device, &gallery); -+ -+ if (matched != NULL && gallery != NULL) -+ { -+ guint i; -+ -+ for (i = 0; i < gallery->len; i++) -+ { -+ FpPrint *candidate = g_ptr_array_index (gallery, i); -+ -+ if (aurora_print_has_uuid (candidate, res.uuid)) -+ { -+ hit = candidate; -+ break; -+ } -+ } -+ } -+ -+ if (retry != NULL) -+ { -+ fpi_device_identify_report (device, NULL, NULL, retry); -+ fpi_device_identify_complete (device, NULL); -+ } -+ else if (matched == NULL && res.result == SEP_BIO_NOT_COMPARED) -+ { -+ fpi_device_identify_complete (device, aurora_not_compared_new (&res)); -+ } -+ else -+ { -+ fpi_device_identify_report (device, hit, matched, NULL); -+ fpi_device_identify_complete (device, NULL); -+ } -+ } -+ -+ self->nonce_valid = FALSE; -+ memset (self->nonce, 0, sizeof (self->nonce)); -+ -+ return G_SOURCE_REMOVE; -+} -+ -+static void -+aurora_match_start (FpDevice *device) -+{ -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct sep_bio_verify_start start = { 0 }; -+ struct sep_bio_info info = { 0 }; -+ FpiDeviceAction action = fpi_device_get_current_action (device); -+ g_autoptr(GError) error = NULL; -+ FpPrint *print = NULL; -+ -+ if (action == FPI_DEVICE_ACTION_VERIFY) -+ fpi_device_get_verify_data (device, &print); -+ -+ /* An identify against an EMPTY gallery is "no match" by definition, and -+ * answering it needs no hardware at all. -+ * -+ * This is not an optimisation, it is what unblocks enrolment. fprintd runs a -+ * duplicate check before each enrol by identifying the incoming finger -+ * against the prints already stored. On a device with nothing enrolled that -+ * gallery is empty, so the question has only one possible answer -- but we -+ * were forwarding it to the kernel, whose verify ioctl returns ENOSYS -+ * because matching is not implemented yet. fprintd then reported -+ * "ioctl failed: Function not implemented", which GNOME shows to the user as -+ * "Fingerprint device disconnected". -+ * -+ * So the first enrolment on a fresh device was impossible, and the message -+ * blamed the cable. Answer the empty case here and the enrol proceeds. -+ * -+ * A NON-empty gallery normally goes to the enclave for matching. A stale -+ * host index may outlive its SEP identity after a failed cold restore; only -+ * an explicit, valid zero live-identity count from SEP lets us answer that -+ * gallery without capture. An unavailable count must never be treated as -+ * zero, and VERIFY still goes through the kernel's restore proof gate. -+ */ -+ if (action == FPI_DEVICE_ACTION_IDENTIFY) -+ { -+ GPtrArray *gallery = NULL; -+ -+ fpi_device_get_identify_data (device, &gallery); -+ if (gallery == NULL || gallery->len == 0) -+ { -+ fp_dbg ("identify against an empty gallery: reporting no match " -+ "without touching the sensor"); -+ fpi_device_identify_report (device, NULL, NULL, NULL); -+ fpi_device_identify_complete (device, NULL); -+ return; -+ } -+ -+ if (ioctl (self->fd, SEP_BIO_GET_INFO, &info) == 0 && -+ info.version == SEP_BIO_IFACE_VERSION && -+ (info.reserved[0] & SEP_BIO_INFO_LIVE_COUNT_VALID) != 0 && -+ info.reserved[1] == 0) -+ { -+ fp_dbg ("SEP proved zero live identities despite a non-empty host " -+ "gallery: reporting no match for identify only"); -+ fpi_device_identify_report (device, NULL, NULL, NULL); -+ fpi_device_identify_complete (device, NULL); -+ return; -+ } -+ } -+ -+ /* A fresh nonce per operation is what makes the result non-replayable. If we -+ * cannot get one we must not fall back to anything weaker. */ -+ if (getrandom (self->nonce, sizeof (self->nonce), 0) != (gssize) sizeof (self->nonce)) -+ { -+ error = fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, -+ "cannot obtain a nonce for the match"); -+ if (action == FPI_DEVICE_ACTION_VERIFY) -+ fpi_device_verify_complete (device, g_steal_pointer (&error)); -+ else -+ fpi_device_identify_complete (device, g_steal_pointer (&error)); -+ return; -+ } -+ self->nonce_valid = TRUE; -+ -+ memcpy (start.nonce, self->nonce, sizeof (start.nonce)); -+ -+ if (!aurora_ioctl (self, SEP_BIO_VERIFY_START, &start, &error)) -+ { -+ self->nonce_valid = FALSE; -+ if (action == FPI_DEVICE_ACTION_VERIFY) -+ fpi_device_verify_complete (device, g_steal_pointer (&error)); -+ else -+ fpi_device_identify_complete (device, g_steal_pointer (&error)); -+ return; -+ } -+ -+ aurora_report_guidance (device, SEP_BIO_GUIDANCE_PLACE, TRUE); -+ aurora_stop_watch (self); -+ self->watch_id = g_unix_fd_add (self->fd, G_IO_IN, aurora_verify_ready, device); -+} -+ -+/* ------------------------------------------------------------------ */ -+/* storage */ -+/* ------------------------------------------------------------------ */ -+ -+static void -+aurora_list (FpDevice *device) -+{ -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct sep_bio_list list = { 0 }; -+ g_autoptr(GError) error = NULL; -+ GPtrArray *prints; -+ guint32 i; -+ -+ if (!aurora_ioctl (self, SEP_BIO_LIST, &list, &error)) -+ { -+ fpi_device_list_complete (device, NULL, g_steal_pointer (&error)); -+ return; -+ } -+ -+ if (list.count > SEP_BIO_MAX_IDENTITIES) -+ { -+ fpi_device_list_complete (device, NULL, -+ fpi_device_error_new_msg (FP_DEVICE_ERROR_PROTO, -+ "device reported %u identities, " -+ "more than the interface allows", -+ list.count)); -+ return; -+ } -+ -+ prints = g_ptr_array_new_with_free_func (g_object_unref); -+ -+ for (i = 0; i < list.count; i++) -+ g_ptr_array_add (prints, -+ g_object_ref_sink (aurora_print_from_uuid (self, -+ list.id[i].uuid, -+ list.id[i].label))); -+ -+ fpi_device_list_complete (device, prints, NULL); -+} -+ -+static void -+aurora_delete (FpDevice *device) -+{ -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct sep_bio_delete del = { 0 }; -+ g_autoptr(GError) error = NULL; -+ FpPrint *print = NULL; -+ -+ fpi_device_get_delete_data (device, &print); -+ -+ if (print == NULL || !aurora_uuid_from_print (print, del.uuid, &error)) -+ { -+ if (error == NULL) -+ error = fpi_device_error_new_msg (FP_DEVICE_ERROR_DATA_INVALID, -+ "no print to delete"); -+ fpi_device_delete_complete (device, g_steal_pointer (&error)); -+ return; -+ } -+ -+ if (!aurora_ioctl (self, SEP_BIO_DELETE, &del, &error)) -+ { -+ fpi_device_delete_complete (device, g_steal_pointer (&error)); -+ return; -+ } -+ -+ fpi_device_delete_complete (device, NULL); -+} -+ -+static void -+aurora_clear_storage (FpDevice *device) -+{ -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ g_autoptr(GError) error = NULL; -+ -+ if (!aurora_ioctl (self, SEP_BIO_DELETE_ALL, NULL, &error)) -+ { -+ fpi_device_clear_storage_complete (device, g_steal_pointer (&error)); -+ return; -+ } -+ -+ fpi_device_clear_storage_complete (device, NULL); -+} -+ -+/* ------------------------------------------------------------------ */ -+/* class */ -+/* ------------------------------------------------------------------ */ -+ -+static const FpIdEntry aurora_id_table[] = { -+ { .udev_types = FPI_DEVICE_UDEV_SUBTYPE_MISC, .misc_name = "sep-bio" }, -+ { .udev_types = 0 } -+}; -+ -+static void -+fpi_device_aurora_init (FpiDeviceAurora *self) -+{ -+ self->fd = -1; -+ self->watch_id = 0; -+ self->nonce_valid = FALSE; -+} -+ -+static void -+fpi_device_aurora_finalize (GObject *object) -+{ -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (object); -+ -+ aurora_stop_watch (self); -+ -+ if (self->fd >= 0) -+ { -+ close (self->fd); -+ self->fd = -1; -+ } -+ -+ memset (self->nonce, 0, sizeof (self->nonce)); -+ -+ G_OBJECT_CLASS (fpi_device_aurora_parent_class)->finalize (object); -+} -+ -+static void -+fpi_device_aurora_class_init (FpiDeviceAuroraClass *klass) -+{ -+ GObjectClass *object_class = G_OBJECT_CLASS (klass); -+ FpDeviceClass *dev_class = FP_DEVICE_CLASS (klass); -+ -+ object_class->finalize = fpi_device_aurora_finalize; -+ -+ dev_class->id = FP_COMPONENT; -+ dev_class->full_name = "Apple secure enclave fingerprint sensor"; -+ dev_class->type = FP_DEVICE_TYPE_UDEV; -+ dev_class->id_table = aurora_id_table; -+ dev_class->scan_type = FP_SCAN_TYPE_PRESS; -+ dev_class->nr_enroll_stages = 8; -+ dev_class->temp_hot_seconds = -1; -+ -+ dev_class->probe = aurora_probe; -+ dev_class->open = aurora_open; -+ dev_class->close = aurora_close; -+ dev_class->cancel = aurora_cancel; -+ dev_class->enroll = aurora_enroll; -+ dev_class->verify = aurora_match_start; -+ dev_class->identify = aurora_match_start; -+ dev_class->list = aurora_list; -+ dev_class->delete = aurora_delete; -+ dev_class->clear_storage = aurora_clear_storage; -+ -+ fpi_device_class_auto_initialize_features (dev_class); -+ -+ /* DUPLICATES_CHECK is deliberately NOT advertised. -+ * -+ * It tells libfprint the device can say whether a finger being enrolled is -+ * already enrolled, and libfprint implements that by running a full IDENTIFY -+ * pass immediately before every enrol. Matching is implemented, but the -+ * enclave's enrol result does not identify an already-enrolled finger in a -+ * way that satisfies libfprint's pre-enrol duplicate-check contract. Keep -+ * the feature unset until that narrower contract is implemented. -+ */ -+} -diff --git a/libfprint/fp-context.c b/libfprint/fp-context.c -index 70d4062..6fdf53b 100644 ---- a/libfprint/fp-context.c -+++ b/libfprint/fp-context.c -@@ -479,6 +479,7 @@ fp_context_enumerate (FpContext *context) - - g_autoptr(GList) spidev_devices = g_udev_client_query_by_subsystem (udev_client, "spidev"); - g_autoptr(GList) hidraw_devices = g_udev_client_query_by_subsystem (udev_client, "hidraw"); -+ g_autoptr(GList) misc_devices = g_udev_client_query_by_subsystem (udev_client, "misc"); - - /* for each potential driver, try to match all requested resources. */ - for (i = 0; i < priv->drivers->len; i++) -@@ -492,7 +493,7 @@ fp_context_enumerate (FpContext *context) - - for (entry = cls->id_table; entry->udev_types; entry++) - { -- GList *matched_spidev = NULL, *matched_hidraw = NULL; -+ GList *matched_spidev = NULL, *matched_hidraw = NULL, *matched_misc = NULL; - - if (entry->udev_types & FPI_DEVICE_UDEV_SUBTYPE_SPIDEV) - { -@@ -530,6 +531,20 @@ fp_context_enumerate (FpContext *context) - if (matched_hidraw == NULL) - continue; - } -+ if (entry->udev_types & FPI_DEVICE_UDEV_SUBTYPE_MISC) -+ { -+ for (matched_misc = misc_devices; matched_misc; matched_misc = matched_misc->next) -+ { -+ const gchar * name = g_udev_device_get_name (matched_misc->data); -+ if (!name || !entry->misc_name) -+ continue; -+ if (g_strcmp0 (name, entry->misc_name) == 0) -+ break; -+ } -+ /* If match was not found exit */ -+ if (matched_misc == NULL) -+ continue; -+ } - priv->pending_devices++; - g_async_initable_new_async (driver, - G_PRIORITY_LOW, -@@ -539,6 +554,7 @@ fp_context_enumerate (FpContext *context) - "fpi-driver-data", entry->driver_data, - "fpi-udev-data-spidev", (matched_spidev ? g_udev_device_get_device_file (matched_spidev->data) : NULL), - "fpi-udev-data-hidraw", (matched_hidraw ? g_udev_device_get_device_file (matched_hidraw->data) : NULL), -+ "fpi-udev-data-misc", (matched_misc ? g_udev_device_get_device_file (matched_misc->data) : NULL), - NULL); - /* remove entries from list to avoid conflicts */ - if (matched_spidev) -@@ -551,6 +567,11 @@ fp_context_enumerate (FpContext *context) - g_object_unref (matched_hidraw->data); - hidraw_devices = g_list_delete_link (hidraw_devices, matched_hidraw); - } -+ if (matched_misc) -+ { -+ g_object_unref (matched_misc->data); -+ misc_devices = g_list_delete_link (misc_devices, matched_misc); -+ } - } - } - -diff --git a/libfprint/fp-device-private.h b/libfprint/fp-device-private.h -index 1c3702f..c8253c6 100644 ---- a/libfprint/fp-device-private.h -+++ b/libfprint/fp-device-private.h -@@ -49,6 +49,7 @@ typedef struct - { - gchar *spidev_path; - gchar *hidraw_path; -+ gchar *misc_path; - } udev_data; - - gboolean is_removed; -diff --git a/libfprint/fp-device.c b/libfprint/fp-device.c -index 115063d..70b10f1 100644 ---- a/libfprint/fp-device.c -+++ b/libfprint/fp-device.c -@@ -53,6 +53,7 @@ enum { - PROP_FPI_USB_DEVICE, - PROP_FPI_UDEV_DATA_SPIDEV, - PROP_FPI_UDEV_DATA_HIDRAW, -+ PROP_FPI_UDEV_DATA_MISC, - PROP_FPI_DRIVER_DATA, - N_PROPS - }; -@@ -237,6 +238,7 @@ fp_device_finalize (GObject *object) - g_clear_pointer (&priv->virtual_env, g_free); - g_clear_pointer (&priv->udev_data.spidev_path, g_free); - g_clear_pointer (&priv->udev_data.hidraw_path, g_free); -+ g_clear_pointer (&priv->udev_data.misc_path, g_free); - - G_OBJECT_CLASS (fp_device_parent_class)->finalize (object); - } -@@ -307,6 +309,13 @@ fp_device_get_property (GObject *object, - g_value_set_string (value, NULL); - break; - -+ case PROP_FPI_UDEV_DATA_MISC: -+ if (cls->type == FP_DEVICE_TYPE_UDEV) -+ g_value_set_string (value, priv->udev_data.misc_path); -+ else -+ g_value_set_string (value, NULL); -+ break; -+ - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - } -@@ -353,6 +362,13 @@ fp_device_set_property (GObject *object, - g_assert (g_value_get_string (value) == NULL); - break; - -+ case PROP_FPI_UDEV_DATA_MISC: -+ if (cls->type == FP_DEVICE_TYPE_UDEV) -+ priv->udev_data.misc_path = g_value_dup_string (value); -+ else -+ g_assert (g_value_get_string (value) == NULL); -+ break; -+ - case PROP_FPI_DRIVER_DATA: - priv->driver_data = g_value_get_uint64 (value); - break; -@@ -583,6 +599,19 @@ fp_device_class_init (FpDeviceClass *klass) - "Private: The path to /dev/hidrawN", - NULL, - G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY); -+ /** -+ * FpDevice::fpi-udev-data-misc: (skip) -+ * -+ * This property is only for internal purposes. -+ * -+ * Stability: private -+ */ -+ properties[PROP_FPI_UDEV_DATA_MISC] = -+ g_param_spec_string ("fpi-udev-data-misc", -+ "Udev data: misc path", -+ "Private: The path to the misc character device", -+ NULL, -+ G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY); - - /** - * FpDevice::fpi-driver-data: (skip) -diff --git a/libfprint/fpi-device.c b/libfprint/fpi-device.c -index 9500b3a..a70db5c 100644 ---- a/libfprint/fpi-device.c -+++ b/libfprint/fpi-device.c -@@ -521,6 +521,9 @@ fpi_device_get_udev_data (FpDevice *device, FpiDeviceUdevSubtypeFlags subtype) - case FPI_DEVICE_UDEV_SUBTYPE_SPIDEV: - return priv->udev_data.spidev_path; - -+ case FPI_DEVICE_UDEV_SUBTYPE_MISC: -+ return priv->udev_data.misc_path; -+ - default: - g_return_val_if_reached (NULL); - return NULL; -diff --git a/libfprint/fpi-device.h b/libfprint/fpi-device.h -index b17c10d..7671ad6 100644 ---- a/libfprint/fpi-device.h -+++ b/libfprint/fpi-device.h -@@ -28,12 +28,14 @@ - * FpiDeviceUdevSubtypeFlags: - * @FPI_DEVICE_UDEV_SUBTYPE_SPIDEV: The device requires an spidev node - * @FPI_DEVICE_UDEV_SUBTYPE_HIDRAW: The device requires a hidraw node -+ * @FPI_DEVICE_UDEV_SUBTYPE_MISC: The device requires a misc character node - * - * Bitfield of required hardware resources for a udev-backed device. - */ - typedef enum { - FPI_DEVICE_UDEV_SUBTYPE_SPIDEV = 1 << 0, - FPI_DEVICE_UDEV_SUBTYPE_HIDRAW = 1 << 1, -+ FPI_DEVICE_UDEV_SUBTYPE_MISC = 1 << 2, - } FpiDeviceUdevSubtypeFlags; - - /** -@@ -71,6 +73,7 @@ struct _FpIdEntry - guint pid; - guint vid; - } hid_id; -+ const gchar *misc_name; - }; - }; - guint64 driver_data; -diff --git a/libfprint/meson.build b/libfprint/meson.build -index f11533c..400c5b7 100644 ---- a/libfprint/meson.build -+++ b/libfprint/meson.build -@@ -116,6 +116,7 @@ driver_sources = { - ), - 'etes603' : files('drivers/etes603.c'), - 'egis0570' : files('drivers/egis0570.c'), -+ 'aurora' : files('drivers/aurora/aurora.c'), - 'egismoc' : files('drivers/egismoc/egismoc.c'), - 'egis_etu905' : files('drivers/egismoc/egis_etu905.c'), - 'vfs0050' : files('drivers/vfs0050.c'), -diff --git a/meson.build b/meson.build -index ab09dc1..dfa7460 100644 ---- a/meson.build -+++ b/meson.build -@@ -151,6 +151,9 @@ drivers_info = { - # SPI driver (non-optional if SPI is available) - 'elanspi': { 'spi': true, 'helper': ['udev'], 'optional': not have_spi }, - -+ # Apple secure enclave, reached through a kernel misc device -+ 'aurora': { 'helper': ['udev'], 'optional': host_machine.system() != 'linux' }, -+ - # Virtual drivers (test-only, optional) - 'virtual_image': { 'virtual': true, 'helper': ['virtual'], 'optional': true }, - 'virtual_device': { 'virtual': true, 'helper': ['virtual'], 'optional': true }, -diff --git a/tests/meson.build b/tests/meson.build -index aad0675..2147ef3 100644 ---- a/tests/meson.build -+++ b/tests/meson.build -@@ -295,4 +295,4 @@ else -- foreach driver_test: drivers_tests -+ foreach driver_test, args: drivers_tests - test(driver_test, - sh, - args: ['-c', 'exit 77'] diff --git a/tools/aurora-sep/tests/test_packaging.py b/tools/aurora-sep/tests/test_packaging.py index f5aa1b124d060..961985758ba1e 100644 --- a/tools/aurora-sep/tests/test_packaging.py +++ b/tools/aurora-sep/tests/test_packaging.py @@ -41,17 +41,9 @@ def check(self, directory): digest, expected, "%s changed: run updpkgsums in %s" % (name, directory) ) - def test_libfprint_sums_match_tree(self): - self.check(ARCH / "libfprint-aurora") - def test_aurora_sep_sums_match_tree(self): self.check(ARCH / "aurora-sep") - def test_patch_version_matches_pkgver(self): - text = (ARCH / "libfprint-aurora" / "PKGBUILD").read_text() - pkgver = re.search(r"^pkgver=(\S+)$", text, re.M).group(1) - self.assertTrue((SEP / "patches" / ("libfprint-%s-apple-sep.patch" % pkgver)).is_file()) - class FedoraSpecTest(unittest.TestCase): def test_sources_are_in_tree(self): @@ -61,11 +53,10 @@ def test_sources_are_in_tree(self): for source in sources: self.assertTrue((SEP / source).is_file(), source) - def test_build_script_patches_the_shipped_version(self): + def test_build_script_builds_only_aurora_sep(self): text = (FEDORA / "build.sh").read_text() - version = re.search(r"^patched_version=(\S+)$", text, re.M).group(1) - self.assertIn("libfprint-%s-apple-sep.patch" % version, text) - self.assertTrue((SEP / "patches" / ("libfprint-%s-apple-sep.patch" % version)).is_file()) + self.assertIn("aurora-sep.spec", text) + self.assertNotIn("dnf -y download --source libfprint", text) if __name__ == "__main__": From 5ed38f829e06a1463f92309b8606d046c507e582 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:21:20 -0700 Subject: [PATCH 0781/2680] tools: aurora-sep: add the release installer install-aurora-sep.sh has only existed as a release asset, so none of its fixes had history or review. This is the script published with sep-7.1.12.aurora2-11.9, byte for byte. It installs linux-aurora, its headers, m1n1-aurora, libfprint and aurora-touchid from the release it belongs to, each checked against a pinned SHA-256; keeps the previous kernel bootable on GRUB; points update-m1n1 at linux-aurora's device trees on either boot chain; and selects the linux-aurora GRUB entry by id rather than by position. Options: --read-only installs with the SEP driver attaching but never writing to the enclave; --uninstall restores the previous kernel, m1n1 and libfprint; --agent-prompt prints the test plan and report format. A blacklist or install line for apple_sep in modprobe.d is respected: the SEP service is left off rather than loading the driver by name. Releases are cut by updating TAG (and VERSION when the kernel changes) and the package checksums, then uploading this file with the packages. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- tools/aurora-sep/install-aurora-sep.sh | 678 +++++++++++++++++++++++++ 1 file changed, 678 insertions(+) create mode 100755 tools/aurora-sep/install-aurora-sep.sh diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh new file mode 100755 index 0000000000000..86a853e3868b1 --- /dev/null +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -0,0 +1,678 @@ +#!/bin/bash +# Install the aurora custom/sep kernel and Touch ID on an Omarchy Mac. +# +# curl -fsSL https://github.com/iconidentify/aurora-linux/releases/download/sep-7.1.12.aurora2-11.2/install-aurora-sep.sh | bash +# ... | bash -s -- --read-only install, but never let the driver write to the enclave +# ... | bash -s -- --uninstall go back to the kernel this Mac had before +# +# Kernel: iconidentify/aurora-linux custom/sep (a70264532f60), aurora-silicon/linux aurora-wip plus the +# Secure Enclave (Touch ID) driver, Thunderbolt (#8), the Apple video +# decoder (#45) and M2 Max (t6021) profile selection. It replaces linux-asahi (or linux-aurora) as a pacman package, +# so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this +# script regenerates grub.cfg and keeps the previous kernel as a fallback entry. +# +# Touch ID: libfprint with the Apple SEP driver, fprintd, the apple-sep +# service and this Mac's own sensor calibration. After the reboot, run +# aurora-touchid-setup to enrol a finger and use it for sudo and the lock screen. +# +# m1n1: m1n1-aurora tracks aurora-silicon/m1n1 main with no local patches. +# On M2 and later the platform hands Linux an already-running Secure +# Enclave, and the driver attaches to it with one registration that can only be +# sent once per boot. Stock m1n1 asks the enclave for randomness on the way up, +# spending that attempt before Linux sees it. m1n1-aurora adds a guard that +# skips the request when the enclave is already running. On M1, where the +# enclave is still in its boot ROM, the guard never fires. +# +# Testing this build? Run with --agent-prompt for the test plan and the format +# to report results in. +set -euo pipefail + +# The kernel package version and the release tag move independently: a release +# that only changes m1n1 reuses the previous kernel packages unchanged. +VERSION=7.1.12.aurora2-11.5 +TAG=sep-7.1.12.aurora2-11.9 +# Packages are fetched from this script's own tag, never from "latest": the +# checksums below belong to this release and nothing else. +RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG +RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases +# Where to always get the current script, whatever this copy turns out to be. +LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh +PACKAGES=( + "linux-aurora-$VERSION-aarch64.pkg.tar.zst de032a60fd3300f8e69a960e11108f9f4f2eac4757beb31836e43c25c1e51f73" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst d1e614d975b7a5495c3317bb48b6cc22286bfe6338c2ba05fc91484ce263fbe7" + "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" + "aurora-touchid-20260929-1-any.pkg.tar.zst c3c84488330a55f25f3a2f241714d0c4b09716be363082737543507337773f5b" + "m1n1-aurora-1.6.1.aurora2-2-aarch64.pkg.tar.zst 99f53b6a300d994965ccd9b0baf089122a4a3963180830544680b34ef34eaab6" +) +PINNED="linux-aurora linux-aurora-headers libfprint m1n1-aurora" +PIN_BEGIN="# >>> aurora-sep pin (remove with: install-aurora-sep.sh --uninstall)" +PIN_END="# <<< aurora-sep pin" +STATE=/var/lib/aurora-sep +FALLBACK_ID=aurora-sep-previous-kernel +# Written only by --read-only; keeps the driver from writing to the enclave. +MODPROBE_CONF=/etc/modprobe.d/aurora-sep.conf +READ_ONLY=0 + +say() { printf '\033[1;34m==>\033[0m %s\n' "$*"; } +warn() { printf '\033[1;33mwarning:\033[0m %s\n' "$*" >&2; } +die() { printf '\033[1;31merror:\033[0m %s\n' "$*" >&2; exit 1; } + +sudo="" +if (( EUID != 0 )); then + command -v sudo >/dev/null || die "run as root or install sudo" + sudo=sudo +fi + +# A saved copy of this script keeps installing its own build forever. Tell the +# operator - human or agent - when a newer one exists. Never fatal: no network, +# rate limit or API change should stop an install that was going to work. +newer_release() { + # Explicitly non-fatal. The pipeline returns non-zero whenever there is no + # network, GitHub rate-limits, or the response is not what we expect, and + # whether set -e acts on that inside a command substitution is subtle enough + # that it should not be left to chance in a script that runs as root. + local seen="" + seen=$(curl -fsSL --max-time 8 "$RELEASES_API" 2>/dev/null | + grep -o '"tag_name"[[:space:]]*:[[:space:]]*"sep-[^"]*"' | + head -1 | sed 's/.*"\(sep-[^"]*\)"$/\1/') || true + [[ -n $seen && $seen != "$TAG" ]] && echo "$seen" + return 0 +} + +version_notice() { + local seen + seen=$(newer_release) + if [[ -n $seen ]]; then + warn "this script installs $TAG, but $seen is published. + You are probably running a saved copy. The current one: + curl -fsSL $LATEST_URL | bash" + else + say "$TAG is the current release" + fi +} + +boot_chain() { + # /boot/efi can be readable only by root, so test through sudo. + if pacman -Q omarchy-mac-boot >/dev/null 2>&1 && + { $sudo test -e /boot/EFI/BOOT/BOOTAA64.EFI || $sudo test -e /boot/efi/EFI/BOOT/BOOTAA64.EFI; } && + command -v limine >/dev/null; then + echo limine + elif [[ -f /boot/grub/grub.cfg ]] && command -v grub-mkconfig >/dev/null; then + echo grub + else + echo unknown + fi +} + +current_kernel() { + if pacman -Q linux-aurora >/dev/null 2>&1; then echo linux-aurora + elif pacman -Q linux-asahi >/dev/null 2>&1; then echo linux-asahi + fi +} + +preflight() { + [[ $(uname -m) == aarch64 ]] || die "this is not an aarch64 machine" + grep -qa 'apple,' /proc/device-tree/compatible 2>/dev/null || die "this is not an Apple Silicon Mac" + command -v pacman >/dev/null || die "pacman not found; this is for Omarchy on Arch Linux ARM" + [[ -n $(current_kernel) ]] || die "neither linux-asahi nor linux-aurora is installed; this Mac's kernel is not one this script replaces" + [[ $(boot_chain) != unknown ]] || die "could not find GRUB or Limine on this Mac; not touching its boot setup" +} + +remove_pin() { + $sudo sed -i "/^$(sed 's/[][\/.*^$]/\\&/g' <<<"$PIN_BEGIN")\$/,/^$PIN_END\$/d" /etc/pacman.conf +} + +add_pin() { + remove_pin + # IgnorePkg lines accumulate, so any existing IgnorePkg stays in force. + $sudo sed -i "/^\[options\]/a $PIN_BEGIN\nIgnorePkg = $PINNED\n$PIN_END" /etc/pacman.conf +} + +snapshot() { + if command -v snapper >/dev/null && $sudo snapper list-configs 2>/dev/null | grep -q '^root'; then + say "Taking a snapper snapshot" + $sudo snapper -c root create -c important -d "before $1" || warn "snapshot failed; continuing" + fi +} + +# /boot has to hold the fallback copy of the old kernel and the new kernel's +# image and initramfs; checked before anything changes. +boot_space() { + local kernel=$1 need free + need=$(du -cm "/boot/vmlinuz-$kernel" "/boot/initramfs-$kernel.img" 2>/dev/null | tail -1 | cut -f1) + [[ -f $STATE/previous ]] && need=0 + # Room for linux-aurora's image and initramfs, unless it is already installed. + if [[ $kernel == linux-aurora ]]; then need=$(( need + 20 )); else need=$(( need + 90 )); fi + free=$(df -m --output=avail /boot | tail -1 | tr -d ' ') + (( free >= need )) || die "/boot has ${free} MB free and this needs about ${need} MB (the old kernel kept as a fallback plus the new one). +Free space in /boot first (old test kernels, for example), then run this again. Nothing was changed." +} + +# GRUB: keep the running kernel bootable from the menu, whatever pacman removes. +keep_grub_fallback() { + local kernel=$1 release="" dir root_uuid subvol cmdline linux initrd + # The modules that belong to the package being replaced (not necessarily + # the running kernel, which may be a hand-installed test kernel). + for dir in /usr/lib/modules/*/; do + [[ $(cat "${dir}pkgbase" 2>/dev/null) == "$kernel" ]] && release=$(basename "$dir") + done + [[ -n $release ]] || { warn "could not find the modules of $kernel; no fallback entry"; return 0; } + [[ -f /boot/vmlinuz-$kernel && -f /boot/initramfs-$kernel.img ]] || { warn "no /boot/vmlinuz-$kernel to keep as a fallback"; return 0; } + $sudo install -d "$STATE" + if ! $sudo cp /boot/vmlinuz-$kernel /boot/vmlinuz-aurora-sep-previous || + ! $sudo cp /boot/initramfs-$kernel.img /boot/initramfs-aurora-sep-previous.img; then + $sudo rm -f /boot/vmlinuz-aurora-sep-previous /boot/initramfs-aurora-sep-previous.img + die "could not copy the previous kernel to /boot (out of space?). Nothing else was changed." + fi + # pacman removes the old kernel's modules; keep the running kernel's for the fallback. + $sudo cp -a "/usr/lib/modules/$release" "$STATE/modules-$release" + # The fallback's device trees must never be what update-m1n1 picks. + $sudo rm -rf "$STATE/modules-$release/dtbs" + echo "$kernel $release" | $sudo tee "$STATE/previous" >/dev/null + # kernel-modules-hook's linux-modules-cleanup.service deletes every + # /usr/lib/modules tree that no package owns and that is not the running + # kernel, so the fallback's modules live in $STATE and are put back only when + # the fallback kernel itself boots, before anything loads a module. + $sudo tee /etc/systemd/system/aurora-sep-fallback-modules.service >/dev/null </dev/null </dev/null <<'EOF' +# aurora-sep: build m1n1's stage 2 from linux-aurora's device trees, which carry +# the Touch ID sensor node, not from whichever kernel has the highest version. +# (update-m1n1 runs under set -e, so this assignment must always succeed.) +DTBS=$(for d in /usr/lib/modules/*/; do if [ "$(cat "${d}pkgbase" 2>/dev/null)" = linux-aurora ]; then echo "${d}"dtbs/*.dtb; fi; done; true) +EOF + for target in /boot/m1n1/boot.bin /boot/efi/m1n1/boot.bin; do + [[ -f $target && ! -f $STATE/boot.bin.saved ]] && $sudo cp "$target" "$STATE/boot.bin.saved" + done + say "Rebuilding m1n1 with the aurora device trees" + $sudo update-m1n1 || die "update-m1n1 failed; the previous boot.bin is saved in $STATE/boot.bin.saved" +} + +grub_update() { + # mkinitcpio's hook installs /boot/vmlinuz-linux-aurora; a preset whose + # kernel is gone would make later mkinitcpio -P runs fail. + if [[ -f /etc/mkinitcpio.d/linux-asahi.preset ]] && ! pacman -Q linux-asahi >/dev/null 2>&1; then + $sudo mv /etc/mkinitcpio.d/linux-asahi.preset "$STATE/linux-asahi.preset.saved" + fi + say "Regenerating the GRUB menu" + $sudo grub-mkconfig -o /boot/grub/grub.cfg + + # Select the linux-aurora entry by its id, never by position. Entry 0 is + # whichever kernel grub-mkconfig considers newest, and a Mac that has been + # used for testing often has hand-installed vmlinuz-* files in /boot with no + # matching initramfs. grub-mkconfig happily writes those as entries with no + # initrd line at all, and booting one hangs before it can mount root. + # grub.cfg can be root-only, so read it through sudo like the ESP check above. + # grub-mkconfig puts every kernel but its idea of the newest inside the + # "Advanced options" submenu, and a bare id only resolves against top-level + # entries: setting one that lives in the submenu silently falls back to entry + # 0. The documented way to reach it is the submenu>entry path, and it has to + # be quoted in /etc/default/grub or the > is a shell redirect. + local entry submenu target + entry=$($sudo grep -oE 'gnulinux-linux-aurora-advanced-[0-9a-f-]+' /boot/grub/grub.cfg | head -1) + [[ -n $entry ]] || die "grub.cfg has no linux-aurora entry; the previous kernel is still the fallback entry" + submenu=$($sudo grep -oE 'gnulinux-advanced-[0-9a-f-]+' /boot/grub/grub.cfg | head -1) + if [[ -n $submenu ]]; then target="$submenu>$entry"; else target="$entry"; fi + + if ! grep -qxF "GRUB_DEFAULT=\"$target\"" /etc/default/grub; then + [[ -f $STATE/grub.default.saved ]] || $sudo cp /etc/default/grub "$STATE/grub.default.saved" + if grep -qE '^GRUB_DEFAULT=' /etc/default/grub; then + $sudo sed -i "s|^GRUB_DEFAULT=.*|GRUB_DEFAULT=\"$target\"|" /etc/default/grub + else + echo "GRUB_DEFAULT=\"$target\"" | $sudo tee -a /etc/default/grub >/dev/null + fi + $sudo grub-mkconfig -o /boot/grub/grub.cfg + fi + + # Prove the default resolves to an entry that can actually boot: the path + # must be written out, and that entry must carry both a kernel and an initrd. + # A vmlinuz-* in /boot with no matching initramfs generates an entry with no + # initrd line at all, which hangs before it can mount root. + $sudo grep -qF "set default=\"$target\"" /boot/grub/grub.cfg || + die "GRUB's default entry is not linux-aurora. Not leaving this Mac pointing at an unbootable default: pick 'Omarchy Linux, with Linux linux-aurora' from the menu, or run --uninstall." + $sudo awk -v id="$entry" ' + /^[\t ]*menuentry/ && index($0, id) { found = 1 } + found && /vmlinuz-linux-aurora/ { k = 1 } + found && /initramfs-linux-aurora\.img/ { i = 1 } + found && /^[\t ]*}/ { exit } + END { exit !(k && i) }' /boot/grub/grub.cfg || + die "the linux-aurora GRUB entry is missing its kernel or initramfs line; refusing to make it the default" + + # Stray hand-installed kernels poison the auto-generated top entry. + local stray="" k n + for k in /boot/vmlinuz-*; do + n=${k#/boot/vmlinuz-} + [[ -e /boot/initramfs-$n.img ]] || stray+=" $n" + done + [[ -z $stray ]] || + warn "these kernels in /boot have no initramfs and boot to a black screen if picked:$stray" +} + +calibration() { + local node name out + for node in /proc/device-tree/soc*/spi*/*/ /proc/device-tree/*/spi*/*/; do + [[ -f $node/compatible ]] && grep -qa mesa "$node/compatible" || continue + [[ -f $node/firmware-name ]] && name=$(tr -d '\0' <"$node/firmware-name") + break + done + name=${name:-apple/mesa_calibration.bin} + out=/usr/lib/firmware/$name + if [[ -s $out ]]; then + say "Touch ID calibration already present: $out" + return 0 + fi + say "Extracting this Mac's Touch ID calibration (read-only) to $out" + $sudo install -d -m 755 "$(dirname "$out")" + $sudo /usr/lib/aurora-touchid/extract-mesa-calibration -o "$out" || + warn "could not extract the calibration; Touch ID will not work until it is present at $out" +} + +# The first line in modprobe.d that blacklists or redirects apple_sep, other +# than our own file. Either spelling is the owner saying "do not load this". +sep_block_line() { + local f + for f in /etc/modprobe.d/*.conf /run/modprobe.d/*.conf /usr/lib/modprobe.d/*.conf; do + [[ -f $f && $f != "$MODPROBE_CONF" ]] || continue + grep -m1 -HnE '^[[:space:]]*(blacklist|install)[[:space:]]+apple[-_]sep([[:space:]]|$)' "$f" 2>/dev/null && + return 0 + done + return 0 +} + +# With its defaults the driver provisions an identity keybag and writes to the +# enclave's anti-replay store (xART) on the first boot of a Mac that has a +# Touch ID profile -- no enrolment needed. That is the intended behaviour, but +# the owner has to be able to opt out, and a block they already set must win. +sep_policy() { + local block + block=$(sep_block_line) + if [[ -n $block ]]; then + warn "apple_sep is blocked on this Mac ($block). + Leaving it that way: the SEP service is not enabled. Note that a plain + 'blacklist' line would NOT have stopped the service, which loads the driver + by name; that is why this script checks for it." + $sudo systemctl disable apple-sep.path apple-sep.service 2>/dev/null || true + return 0 + fi + if (( READ_ONLY )); then + printf '%s\n' \ + "# aurora-sep --read-only: the driver attaches and reports, but never writes" \ + "# to the enclave. Delete this file to allow enrolment." \ + "options apple_sep xart_writes=0 provision_keybag=0" | + $sudo tee "$MODPROBE_CONF" >/dev/null + say "Read-only: the SEP driver will attach but not write to the enclave ($MODPROBE_CONF)" + elif [[ -f $MODPROBE_CONF ]]; then + # Re-running to update must never quietly turn writes back on. + say "Still read-only from an earlier --read-only install; delete $MODPROBE_CONF to allow enrolment" + fi + $sudo systemctl enable apple-sep.path apple-sep.service +} + +sep_write_notice() { + (( READ_ONLY )) && return 0 + [[ -f $MODPROBE_CONF || -n $(sep_block_line) ]] && return 0 + warn "on the first boot of a Mac with a Touch ID profile, the SEP driver creates + an identity keybag and writes to the enclave's anti-replay store. Use a Mac + you can DFU-restore. If you cannot, press Ctrl-C now and run with --read-only: + curl -fsSL $LATEST_URL | bash -s -- --read-only" + return 0 +} + +install_all() { + local entry file sha kernel chain + version_notice + sep_write_notice + kernel=$(current_kernel) + chain=$(boot_chain) + if [[ $chain == grub ]]; then boot_space "$kernel"; fi + work=$(mktemp -d) + trap 'rm -rf "${work:-}"' EXIT + for entry in "${PACKAGES[@]}"; do + read -r file sha <<<"$entry" + say "Downloading $file" + curl -fL --retry 3 --progress-bar -o "$work/$file" "$RELEASE_URL/$file" || + die "could not download $file from $TAG. + The release is missing a file this script expects, which is a packaging + mistake rather than anything wrong with this Mac. Nothing was installed. + Please report it with the file name above." + [[ $(sha256sum "$work/$file" | cut -d' ' -f1) == "$sha" ]] || die "$file does not match its published checksum" + done + + snapshot "aurora-sep $VERSION" + $sudo install -d "$STATE" + if [[ ! -f $STATE/previous && $chain == grub ]]; then + keep_grub_fallback "$kernel" + fi + pacman -Q "$kernel" | $sudo tee "$STATE/previous-package" >/dev/null + + # linux-aurora-headers pulls in pahole, and fprintd below comes from the + # repositories. On a Mac whose package database has gone stale, pacman + # resolves those to versions the mirror no longer carries and the whole + # transaction dies with a 404 after the packages are already downloaded. + say "Refreshing the package database" + $sudo pacman -Sy --noconfirm || warn "could not refresh the package database; continuing" + + say "Installing the aurora-sep kernel, libfprint with the Apple SEP driver, fprintd and aurora-touchid" + # --ask 4 accepts replacing linux-asahi (and its headers), which linux-aurora conflicts with. + $sudo pacman -U --noconfirm --ask 4 "$work"/*.pkg.tar.zst + $sudo pacman -S --needed --noconfirm fprintd + # linux-aurora carries the Apple video decoder, whose firmware linux-asahi + # installs never needed; without it the decoder fails to load at boot. + $sudo pacman -S --needed --noconfirm avd-fw || + warn "could not install avd-fw; hardware video decode will not work until it is installed" + add_pin + # Both boot chains boot through m1n1, and both need the aurora device trees + # in boot.bin: update-m1n1 otherwise takes the DTBs of the highest-versioned + # kernel directory, which on a Mac with an older hand-installed test kernel + # is not this one -- and those DTBs lack the Touch ID sensor node. pacman's + # own hook already ran update-m1n1 during the install above, before this + # configuration existed, so run it again now that it is in place. + m1n1_update + if [[ $chain == grub ]]; then + grub_update + fi + + calibration + $sudo systemctl daemon-reload + sep_policy + pacman -Q linux-aurora libfprint aurora-touchid + echo + if [[ -f $MODPROBE_CONF ]]; then + say "Done, read-only. Reboot; the SEP driver will attach and report without writing." + echo " Enrolling a finger needs writes: delete $MODPROBE_CONF, reboot, then run aurora-touchid-setup." + else + say "Done. Reboot, then run: aurora-touchid-setup" + fi + echo " Testing this build? The plan and reporting format:" + echo " curl -fsSL $RELEASE_URL/install-aurora-sep.sh | bash -s -- --agent-prompt" + if [[ $chain == grub ]]; then + echo " The previous kernel stays in the GRUB menu as 'Previous kernel … before aurora-sep'." + fi + echo " To undo everything: curl -fsSL $RELEASE_URL/install-aurora-sep.sh | bash -s -- --uninstall" +} + +uninstall_all() { + local previous=linux-asahi + [[ -f $STATE/previous-package ]] && read -r previous _ <"$STATE/previous-package" + command -v aurora-touchid-setup >/dev/null && aurora-touchid-setup --remove || true + $sudo systemctl disable apple-sep.path apple-sep.service 2>/dev/null || true + remove_pin + snapshot "removing aurora-sep" + say "Reinstalling $previous, the stock m1n1 and the stock libfprint" + $sudo pacman -Rdd --noconfirm aurora-touchid 2>/dev/null || true + $sudo pacman -Sy --noconfirm --ask 4 "$previous" "$previous-headers" libfprint m1n1 + # Restore the stock update-m1n1 configuration on either chain before the + # rebuild below, so boot.bin goes back to the packaged m1n1 and DTBs. + if [[ $(boot_chain) != grub ]]; then + if [[ -f $STATE/update-m1n1.default.saved ]]; then + $sudo cp "$STATE/update-m1n1.default.saved" /etc/default/update-m1n1 + else + $sudo rm -f /etc/default/update-m1n1 + fi + $sudo update-m1n1 || warn "update-m1n1 failed; boot.bin still has the aurora m1n1" + fi + + if [[ $(boot_chain) == grub ]]; then + [[ -f $STATE/linux-asahi.preset.saved && ! -f /etc/mkinitcpio.d/linux-asahi.preset ]] && + $sudo mv "$STATE/linux-asahi.preset.saved" /etc/mkinitcpio.d/linux-asahi.preset + [[ -f $STATE/grub.default.saved ]] && $sudo cp "$STATE/grub.default.saved" /etc/default/grub + if [[ -f $STATE/update-m1n1.default.saved ]]; then + $sudo cp "$STATE/update-m1n1.default.saved" /etc/default/update-m1n1 + else + $sudo rm -f /etc/default/update-m1n1 + fi + $sudo update-m1n1 + $sudo systemctl disable aurora-sep-fallback-modules.service 2>/dev/null || true + $sudo rm -f /etc/systemd/system/aurora-sep-fallback-modules.service + $sudo rm -f /etc/grub.d/42_aurora_sep_previous /boot/vmlinuz-aurora-sep-previous /boot/initramfs-aurora-sep-previous.img + $sudo grub-mkconfig -o /boot/grub/grub.cfg + fi + $sudo rm -f "$MODPROBE_CONF" + $sudo rm -rf "$STATE" + say "Done. Reboot to run $previous." +} + +# Printed by --agent-prompt, and pointed at from the end of a successful +# install. This is written for an agent driving the test on a real Mac: it says +# what to establish, what counts as a pass, and how to write it up. +agent_prompt() { + cat <<'PROMPT' +You are testing an experimental Secure Enclave / Touch ID kernel on an Apple +Silicon Mac. Work through this in order and report at the end. Do not skip the +identification step: nearly every past report was unusable because the machine +was not pinned down. + +SAFETY, NON-NEGOTIABLE + - This touches the Secure Enclave. Confirm the owner can DFU-restore this Mac + before you start. + - A default install is NOT read-only. On the first boot of a Mac with a + Touch ID profile, the SEP driver loads automatically, creates an identity + keybag and writes to the enclave's anti-replay store (xART) -- before + anyone enrols a finger. If the owner cannot DFU-restore, install with + --read-only instead, before the first reboot: + curl -fsSL https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh | bash -s -- --read-only + A 'blacklist apple_sep' line does not prevent this on its own; the + installer detects one and leaves the driver off. + - Do not run any enrol/delete/re-provision loop unless the owner asks for + it in writing: repeated cycles drift a device-wide counter. + - Do not paste key material, serial numbers, or the contents of + mesa_calibration.bin into a report. + +0. CONFIRM YOU HAVE THE CURRENT BUILD + Results against a superseded build waste everyone's time, and a saved copy + of this script installs its own packages forever. This script checks on + every install and prints either " is the current release" or a warning + naming the newer one. To check without installing: + + curl -fsSL https://api.github.com/repos/iconidentify/aurora-linux/releases \ + | grep -m1 '"tag_name"' + + Always fetch the script from the "latest" URL rather than a tag you were + handed, so you get the current one automatically: + + curl -fsSL https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh | bash + + State the tag you installed in your report. If you were given a specific + older tag on purpose, say so and say why. + +1. IDENTIFY THE MACHINE + tr -d '\0' < /proc/device-tree/compatible; echo + for p in os-fw-version system-fw-version iboot2-version; do + printf '%s = ' "$p" + tr -d '\0' < /proc/device-tree/chosen/asahi,$p 2>/dev/null; echo + done + uname -r; pacman -Q linux-aurora m1n1-aurora libfprint aurora-touchid + asahi,system-fw-version is the sepOS the enclave actually runs and is the + single most useful line in the whole report. asahi,os-fw-version is only the + stub. Report both; they are often different. + +2. CONFIRM THE BOOTLOADER GUARD (M2 and later) + In the m1n1 stage 2 boot log, expect: + SEP: Preserving iBoot warm registration; seeding RNG from ADT + and expect NO line reading "SEP: couldn't get enough random bytes". + You should also see the OS identity forwarded: + FDT: apfs-preboot-uuid = '' + On M1 neither line is expected: the enclave is cold and m1n1 talks to it + normally. If you see the "couldn't get enough random bytes" line on any + machine, the stock m1n1 is still in boot.bin - say so and stop, because every + later result is then measuring the wrong thing. + +3. DRIVER ATTACH + The driver is already loaded at boot by apple-sep.service; do not reload + it. Its write mode was fixed at install time. Read what it did: + sudo dmesg | grep -iE 'apple_sep|apple-mesa' + The xART line ends "writes ENABLED" or "writes disabled" -- quote it, and + say which you expected. + PASS: "attach: N endpoints advertised in M messages" with N >= 7, and + /dev/sep-bio exists. + FAIL: "attach: 0 messages received but no endpoint advertised", or no + apple_sep lines at all. Capture the whole block either way. + If the profile line says a SoC you did not expect, report that verbatim. + +4. IF ATTACH FAILED, GET THE MAILBOX STATE + sudo busybox devmem 0x396408110 32 # A2I control (t600x/t602x) + sudo busybox devmem 0x396408114 32 # I2A control + grep -i mbox /proc/interrupts + Those addresses are for t600x and t602x. The mailbox base differs per SoC, + so take it from the device tree rather than a table: + + sudo cat /proc/device-tree/soc/mbox@*/reg 2>/dev/null | xxd | head -2 + # or read the sep_mbox node address from your board's dtsi: + # t8103 0x242408000 -> 0x242408110 / 0x242408114 + # t8112 0x25e408000 -> 0x25e408110 / 0x25e408114 + # t600x, t602x 0x396408000 -> 0x396408110 / 0x396408114 + + Bit 0 is the enable; a non-zero FIFO count with an unchanged read pointer + means the enclave never took the message. Report the raw values, not your + reading of them. + +5. TOUCH ID END TO END (only if step 3 passed) + aurora-touchid-setup + Enrol one finger, then: + fprintd-verify + Then check the consumers the owner actually cares about: sudo, polkit, and + the lock screen. Report each as pass/fail separately - partial success here + is the normal outcome and is worth knowing precisely. + +6. THUNDERBOLT AND DISPLAYPORT - TEST THIS EVEN IF YOU CAME FOR TOUCH ID + This build carries the Thunderbolt/USB4 display and PCIe work, and it needs + hardware reports as much as the enclave does. Do not skip it because the + machine has no dock: "no dock available" is itself a useful answer, and + step 6a still applies. + + 6a. The controllers (no dock needed) + ls /sys/bus/platform/devices/ | grep cio + sudo dmesg | grep -iE 'thunderbolt|usb4|acio|tunnel' | head -30 + Expect one *.cio device per controller, bound to thunderbolt-apple-acio. + An EMPTY /sys/bus/thunderbolt/devices/ with nothing plugged in is + normal: routers only appear when a device is attached. Do not report + that as a failure. + Also normal, on every Mac except the M1 Air and 13" M1 MacBook Pro: + PCIe-C tunnel disabled: m1n1 handoff is not initialized + Anything behind a dock's PCIe controller -- ethernet, storage, and on + Thunderbolt 3 docks such as the CalDigit TS3 the USB ports too -- only + works on t8103 so far. Displays, and USB on USB4 docks, do not need the + PCIe tunnel. Report the line only if you see it on a t8103. + + 6b. With a dock or a DisplayPort monitor, in this order: + - attached at boot: does the display come up, and at what resolution + and refresh rate + - hot unplug: does the tunnel tear down cleanly, no hang, no stuck + compositor + - hot replug: does the display come back, same mode as before + - repeat both of the above on the SECOND USB-C port, not just the + first: the ports are separate controllers and have behaved + differently + - two displays or a dual-output dock, if you have one + - PCIe behind the dock: ethernet, USB storage, card readers - do they + enumerate and still work after a replug + + 6c. Across suspend: + systemctl suspend + Then wake and re-check: display back, dock devices back, and + /dev/sep-bio still present. Report anything that needed a replug to + recover. + +7. HARDWARE VIDEO DECODE + Play an H.264 file and an HEVC file and confirm it is not silently falling + back to software: + sudo dmesg | grep -iE 'avd|apple-vpu' | tail -20 + Report which codecs you exercised and at what resolution. + +8. ONE SUSPEND/RESUME CYCLE AT THE END + Re-check /dev/sep-bio, the display and the dock after resume. + + Report "not tested" honestly rather than guessing, for any of the above. + +HOW TO REPORT + Post one comment on the PR you were given (the SEP work lives on + omacom/linux#7). Structure it exactly like this: + + # (): + **Machine:** apple,jXXX / apple,tXXXX, + **Firmware:** asahi,os-fw-version = X, asahi,system-fw-version = Y + **Build:** , linux-aurora , m1n1-aurora + **m1n1 guard:** fired / did not fire / stock m1n1 still installed + **Thunderbolt:** on port <1 / 2 / both> + + ## What worked + ## What did not + ## Logs + (fenced blocks, trimmed to the relevant lines - never a whole dmesg) + + Rules for the write-up: + - Lead with the outcome, not the narrative. + - Quote log lines exactly; do not paraphrase an error. + - Say explicitly what you did NOT test. "No dock available" and "only + tested port 1" are useful answers; silence is not. + - Name the dock and the display mode. "CalDigit TS3 Plus, 3840x2160 @ 60" + is actionable; "external display worked" is not. + - If something failed, give the last known-good state and the first bad one. + - Do not claim a fix works because it compiled or because the module + loaded. Only step 3's endpoint line and step 5's verify-match count. +PROMPT +} + +preflight_needed() { case ${1:-} in --agent-prompt) return 1 ;; *) return 0 ;; esac; } + +if preflight_needed "${1:-}"; then preflight; fi +case ${1:-} in + "") install_all ;; + --read-only) READ_ONLY=1; install_all ;; + --uninstall) uninstall_all ;; + --agent-prompt) t=$(newer_release); [[ -n $t ]] && warn "the current release is $t; this plan is for $TAG"; agent_prompt ;; + *) die "unknown option $1 (--read-only, --uninstall or --agent-prompt)" ;; +esac From 5df8e46ea3295a9fb839bd80a56f09e8b9223d71 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:53:14 -0700 Subject: [PATCH 0782/2680] tools: aurora-sep: take DTBs from the package; read the Latest release Two fixes from sep-7.1.12.aurora2-11.10 and -11.11. update-m1n1 was pointed at every module directory whose pkgbase said linux-aurora. Right after an upgrade the running kernel's modules are restored unowned but still carry that pkgbase, so both kernels' device trees were bundled; m1n1 keeps the last match, and the glob sorts 11.10 before 11.9, so a stale DTB could boot. Take the DTBs the installed package owns instead. Found by Scott Jones on a J414c. The staleness check read the first entry of the release list, which GitHub orders by comparing tag names as text when releases share a commit, so 11.9 was reported as newer than 11.10. Ask for the release marked Latest. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- tools/aurora-sep/install-aurora-sep.sh | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 86a853e3868b1..e9ef576afeb7a 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -30,7 +30,7 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. VERSION=7.1.12.aurora2-11.5 -TAG=sep-7.1.12.aurora2-11.9 +TAG=sep-7.1.12.aurora2-11.11 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -72,7 +72,10 @@ newer_release() { # whether set -e acts on that inside a command substitution is subtle enough # that it should not be left to chance in a script that runs as root. local seen="" - seen=$(curl -fsSL --max-time 8 "$RELEASES_API" 2>/dev/null | + # Ask for the release marked Latest. Listing all releases is not ordered by + # version: they share a commit, so GitHub falls back to comparing tag names + # as text, and 11.9 sorts above 11.10. + seen=$(curl -fsSL --max-time 8 "$RELEASES_API/latest" 2>/dev/null | grep -o '"tag_name"[[:space:]]*:[[:space:]]*"sep-[^"]*"' | head -1 | sed 's/.*"\(sep-[^"]*\)"$/\1/') || true [[ -n $seen && $seen != "$TAG" ]] && echo "$seen" @@ -229,10 +232,15 @@ m1n1_update() { [[ -f /etc/default/update-m1n1 && ! -f $STATE/update-m1n1.default.saved ]] && $sudo cp /etc/default/update-m1n1 "$STATE/update-m1n1.default.saved" $sudo tee /etc/default/update-m1n1 >/dev/null <<'EOF' -# aurora-sep: build m1n1's stage 2 from linux-aurora's device trees, which carry -# the Touch ID sensor node, not from whichever kernel has the highest version. +# aurora-sep: build m1n1's stage 2 from the device trees the installed +# linux-aurora package owns, which carry the Touch ID sensor node. +# +# Not from a module directory whose pkgbase says linux-aurora: right after an +# upgrade the running kernel's modules are restored unowned but still carry that +# pkgbase, so both kernels' DTBs would be bundled. m1n1 keeps the last matching +# DTB, and the glob sorts 11.10 before 11.9, so the stale one can win. # (update-m1n1 runs under set -e, so this assignment must always succeed.) -DTBS=$(for d in /usr/lib/modules/*/; do if [ "$(cat "${d}pkgbase" 2>/dev/null)" = linux-aurora ]; then echo "${d}"dtbs/*.dtb; fi; done; true) +DTBS=$(pacman -Qlq linux-aurora 2>/dev/null | grep '/dtbs/[^/]*\.dtb$'; true) EOF for target in /boot/m1n1/boot.bin /boot/efi/m1n1/boot.bin; do [[ -f $target && ! -f $STATE/boot.bin.saved ]] && $sudo cp "$target" "$STATE/boot.bin.saved" @@ -517,7 +525,7 @@ SAFETY, NON-NEGOTIABLE every install and prints either " is the current release" or a warning naming the newer one. To check without installing: - curl -fsSL https://api.github.com/repos/iconidentify/aurora-linux/releases \ + curl -fsSL https://api.github.com/repos/iconidentify/aurora-linux/releases/latest \ | grep -m1 '"tag_name"' Always fetch the script from the "latest" URL rather than a tag you were From b139c46a16792d06c97c58d98d8776022edc4348 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 29 Sep 2026 18:51:33 -0700 Subject: [PATCH 0783/2680] tools: aurora-sep: release 11.12 with the after-reboot match fix Pin linux-aurora 7.1.12.aurora2-11.6. It adds the verify failure logging (aurora-silicon/linux#60) and sets the match credential up again after a cold restore (#61). The other packages are unchanged. The test plan now has testers verify again after a reboot without re-enrolling, report that result separately, and quote the driver's verify: lines when a verify fails. The report shape gets a Touch ID line. The header comment named a tag-11.2 URL and the 11.5 source commit, and said m1n1-aurora carries no local patches. It carries the m1n1#4 alias fallback. All three are corrected. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- tools/aurora-sep/install-aurora-sep.sh | 30 +++++++++++++++++--------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index e9ef576afeb7a..20430ae284b9e 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -1,13 +1,14 @@ #!/bin/bash # Install the aurora custom/sep kernel and Touch ID on an Omarchy Mac. # -# curl -fsSL https://github.com/iconidentify/aurora-linux/releases/download/sep-7.1.12.aurora2-11.2/install-aurora-sep.sh | bash +# curl -fsSL https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh | bash # ... | bash -s -- --read-only install, but never let the driver write to the enclave # ... | bash -s -- --uninstall go back to the kernel this Mac had before # -# Kernel: iconidentify/aurora-linux custom/sep (a70264532f60), aurora-silicon/linux aurora-wip plus the +# Kernel: iconidentify/aurora-linux custom/sep (17cba00e43b9), aurora-silicon/linux aurora-wip plus the # Secure Enclave (Touch ID) driver, Thunderbolt (#8), the Apple video -# decoder (#45) and M2 Max (t6021) profile selection. It replaces linux-asahi (or linux-aurora) as a pacman package, +# decoder (#45), the M2 Max (t6021) profile and the fix for matching after a +# reboot on M2 Pro/Max (#61). It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. # @@ -15,12 +16,13 @@ # service and this Mac's own sensor calibration. After the reboot, run # aurora-touchid-setup to enrol a finger and use it for sudo and the lock screen. # -# m1n1: m1n1-aurora tracks aurora-silicon/m1n1 main with no local patches. +# m1n1: m1n1-aurora builds aurora-silicon/m1n1 main (97e2de3d4fec) plus one +# patch, the usb4-N-pcie-adapter alias fallback (aurora-silicon/m1n1#4). # On M2 and later the platform hands Linux an already-running Secure # Enclave, and the driver attaches to it with one registration that can only be # sent once per boot. Stock m1n1 asks the enclave for randomness on the way up, -# spending that attempt before Linux sees it. m1n1-aurora adds a guard that -# skips the request when the enclave is already running. On M1, where the +# spending that attempt before Linux sees it. aurora-silicon/m1n1 has a guard +# that skips the request when the enclave is already running. On M1, where the # enclave is still in its boot ROM, the guard never fires. # # Testing this build? Run with --agent-prompt for the test plan and the format @@ -29,8 +31,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.5 -TAG=sep-7.1.12.aurora2-11.11 +VERSION=7.1.12.aurora2-11.6 +TAG=sep-7.1.12.aurora2-11.12 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -38,8 +40,8 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst de032a60fd3300f8e69a960e11108f9f4f2eac4757beb31836e43c25c1e51f73" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst d1e614d975b7a5495c3317bb48b6cc22286bfe6338c2ba05fc91484ce263fbe7" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst 8a7fb83f371bd257985515f9d4e5ac8479ed6c549775172ae6ebfd558d9e3ef7" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 994f34f6413b43a5905eda8559a420f67fd4c34797ccc7784eb9afcb2793914e" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20260929-1-any.pkg.tar.zst c3c84488330a55f25f3a2f241714d0c4b09716be363082737543507337773f5b" "m1n1-aurora-1.6.1.aurora2-2-aarch64.pkg.tar.zst 99f53b6a300d994965ccd9b0baf089122a4a3963180830544680b34ef34eaab6" @@ -591,6 +593,13 @@ SAFETY, NON-NEGOTIABLE aurora-touchid-setup Enrol one finger, then: fprintd-verify + Then REBOOT and run fprintd-verify again without re-enrolling. Matching + right after enrolment and matching after a reboot are different paths (the + second restores the enrolment from disk), so report them separately. + If a verify fails, the driver logs the step that ended it: + sudo dmesg | grep -E 'verify:|scrd:|Touch ID|catacomb|matching unavailable' + fprintd's "verify-unknown-error" is not a result on its own; quote those + lines with it. Then check the consumers the owner actually cares about: sudo, polkit, and the lock screen. Report each as pass/fail separately - partial success here is the normal outcome and is worth knowing precisely. @@ -654,6 +663,7 @@ HOW TO REPORT **Firmware:** asahi,os-fw-version = X, asahi,system-fw-version = Y **Build:** , linux-aurora , m1n1-aurora **m1n1 guard:** fired / did not fire / stock m1n1 still installed + **Touch ID:** enrol , verify , verify after reboot **Thunderbolt:** on port <1 / 2 / both> ## What worked From 12a697efa0bf3534a3dc230353f31f7d1f5a51cf Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 30 Sep 2026 12:16:07 -0700 Subject: [PATCH 0784/2680] tools: aurora-sep: release 11.13, calibration name and fprintd start-up The installer extracts this Mac's sensor calibration under the name the driver will ask for. It read that name only from the running device tree. When the installer ran from linux-asahi, that tree had no sensor node, so the file was written as mesa_calibration.bin while the aurora DTB asks for apple/mesacal-.bin, and the first enrol failed with ENOENT. Read firmware-name from this board's DTB in the installed linux-aurora package first, then fall back to the running tree and to the default. The presence check uses the resolved name, so re-running the installer repairs affected machines. aurora-touchid 20260930-1 makes fprintd wait for /dev/sep-bio, up to 15 s. apple-sep.service returns as soon as modprobe does and the node appears seconds later. fprintd started in between, cached "No devices available" and kept it for about 45 s. The drop-in also allows the misc device class instead of the node path. A path rule is resolved when the cgroup is set up and dropped if the node does not exist yet. The package source now lives in tools/aurora-sep/aurora-touchid. Reported by GitHub users nosferatuboi (calibration name) and cb2206 (fprintd, whose drop-in this is). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- tools/aurora-sep/aurora-touchid/PKGBUILD | 27 ++ .../aurora-sep/aurora-touchid/apple-sep.path | 9 + .../aurora-touchid/apple-sep.service | 11 + .../aurora-touchid/aurora-touchid-setup | 73 ++++ .../aurora-touchid/extract-mesa-calibration | 343 ++++++++++++++++++ .../aurora-touchid/fprintd-aurora.conf | 12 + tools/aurora-sep/install-aurora-sep.sh | 27 +- 7 files changed, 494 insertions(+), 8 deletions(-) create mode 100644 tools/aurora-sep/aurora-touchid/PKGBUILD create mode 100644 tools/aurora-sep/aurora-touchid/apple-sep.path create mode 100644 tools/aurora-sep/aurora-touchid/apple-sep.service create mode 100755 tools/aurora-sep/aurora-touchid/aurora-touchid-setup create mode 100755 tools/aurora-sep/aurora-touchid/extract-mesa-calibration create mode 100644 tools/aurora-sep/aurora-touchid/fprintd-aurora.conf diff --git a/tools/aurora-sep/aurora-touchid/PKGBUILD b/tools/aurora-sep/aurora-touchid/PKGBUILD new file mode 100644 index 0000000000000..738d53d269f47 --- /dev/null +++ b/tools/aurora-sep/aurora-touchid/PKGBUILD @@ -0,0 +1,27 @@ +# Maintainer: Chris Kearney <303316+iconidentify@users.noreply.github.com> +pkgname=aurora-touchid +pkgver=20260930 +pkgrel=1 +pkgdesc="Touch ID for Omarchy on Apple Silicon (aurora kernel SEP driver)" +arch=('any') +url="https://github.com/iconidentify/aurora-linux" +license=('GPL-2.0-only') +depends=('fprintd' 'libfprint>=1.94.100-1.1' 'python' 'kmod') +source=('aurora-touchid-setup' + 'extract-mesa-calibration' + 'apple-sep.service' + 'apple-sep.path' + 'fprintd-aurora.conf') +sha256sums=('ab4ed299d03462210e03e8d4152803a414204ebf25ec0f065b1b7dc3d43667b1' + '2e1d50b5274223e107e4aa4f73de194f3a49d347be1edc93efd3f81b31b092cc' + '4f7b8a97e35e3511afa482ee8aedd019e960b1a67320675bd347f534feab49c3' + '2f4b265a4c2180a9e0e4926c53ec77c3dbb074ddc3e083d7925d58db10a576b9' + 'e1b1152e576e9588cc134b09291b3a2306ac686d9a47be6f128670608da535fc') + +package() { + install -Dm755 aurora-touchid-setup "$pkgdir/usr/bin/aurora-touchid-setup" + install -Dm755 extract-mesa-calibration "$pkgdir/usr/lib/aurora-touchid/extract-mesa-calibration" + install -Dm644 apple-sep.service "$pkgdir/usr/lib/systemd/system/apple-sep.service" + install -Dm644 apple-sep.path "$pkgdir/usr/lib/systemd/system/apple-sep.path" + install -Dm644 fprintd-aurora.conf "$pkgdir/usr/lib/systemd/system/fprintd.service.d/aurora.conf" +} diff --git a/tools/aurora-sep/aurora-touchid/apple-sep.path b/tools/aurora-sep/aurora-touchid/apple-sep.path new file mode 100644 index 0000000000000..b2b6e6880025b --- /dev/null +++ b/tools/aurora-sep/aurora-touchid/apple-sep.path @@ -0,0 +1,9 @@ +[Unit] +Description=Load Apple SEP when the iBoot container appears + +[Path] +PathExists=/dev/disk/by-partlabel/iBootSystemContainer +Unit=apple-sep.service + +[Install] +WantedBy=multi-user.target diff --git a/tools/aurora-sep/aurora-touchid/apple-sep.service b/tools/aurora-sep/aurora-touchid/apple-sep.service new file mode 100644 index 0000000000000..a49ad3e5c1026 --- /dev/null +++ b/tools/aurora-sep/aurora-touchid/apple-sep.service @@ -0,0 +1,11 @@ +[Unit] +Description=Apple SEP and Touch ID +ConditionPathExists=/dev/disk/by-partlabel/iBootSystemContainer + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/sbin/modprobe apple-sep + +[Install] +WantedBy=multi-user.target diff --git a/tools/aurora-sep/aurora-touchid/aurora-touchid-setup b/tools/aurora-sep/aurora-touchid/aurora-touchid-setup new file mode 100755 index 0000000000000..80074cecab11b --- /dev/null +++ b/tools/aurora-sep/aurora-touchid/aurora-touchid-setup @@ -0,0 +1,73 @@ +#!/bin/bash +# Enrol a finger with Touch ID and use it for sudo, polkit and the lock screen. +# +# aurora-touchid-setup enrol, verify, then configure PAM +# aurora-touchid-setup --remove take fingerprint out of PAM again +# +# The Apple SEP driver (apple-sep.service) must be running: /dev/sep-bio exists. +# PAM is only changed after a finger has been enrolled and verified, with the +# same lines and clamshell gate as Omarchy's omarchy-setup-security-fingerprint, +# which cannot be used here (it looks for USB readers and installs libfprint-git). +set -euo pipefail + +gate="auth [success=1 default=ignore] pam_exec.so quiet /usr/bin/omarchy-hw-laptop-closed" +sudo="" +(( EUID == 0 )) || sudo=sudo + +say() { printf '\033[1;32m%s\033[0m\n' "$*"; } +die() { printf '\033[1;31m%s\033[0m\n' "$*" >&2; exit 1; } + +configure_pam() { + local file + for file in /etc/pam.d/sudo /etc/pam.d/polkit-1; do + if [[ ! -f $file && $file == */polkit-1 ]]; then + $sudo tee "$file" >/dev/null </dev/null <<'EOF' +#%PAM-1.0 +auth required pam_fprintd.so +account include system-local-login +EOF + command -v omarchy-apply-lock >/dev/null && omarchy-apply-lock || true +} + +remove_pam() { + local file + for file in /etc/pam.d/sudo /etc/pam.d/polkit-1; do + [[ -f $file ]] || continue + $sudo sed -i -e '/pam_fprintd\.so/d' -e '/omarchy-hw-laptop-closed/d' "$file" + done + $sudo rm -f /etc/pam.d/omarchy-lock-fingerprint + command -v omarchy-apply-lock >/dev/null && omarchy-apply-lock || true + say "Fingerprint login removed. Enrolled prints stay in fprintd; delete them with: fprintd-delete $USER" +} + +if [[ ${1:-} == --remove ]]; then + remove_pam + exit 0 +fi + +[[ -c /dev/sep-bio ]] || die "Touch ID is not available: /dev/sep-bio is missing. +Reboot into the aurora kernel first, then check: systemctl status apple-sep.service" + +say "Place your right index finger on Touch ID, lift and repeat until enrolment completes." +$sudo fprintd-enroll "${SUDO_USER:-$USER}" || die "Enrolment failed. Run aurora-touchid-setup again." + +say "Enrolled. Now touch the sensor once more to verify." +fprintd-verify "${SUDO_USER:-$USER}" || die "Verification failed; PAM was not changed. Run aurora-touchid-setup again." + +configure_pam +say "Touch ID now works for sudo, polkit prompts and the lock screen. Your password still works too." diff --git a/tools/aurora-sep/aurora-touchid/extract-mesa-calibration b/tools/aurora-sep/aurora-touchid/extract-mesa-calibration new file mode 100755 index 0000000000000..0485e2ef21b37 --- /dev/null +++ b/tools/aurora-sep/aurora-touchid/extract-mesa-calibration @@ -0,0 +1,343 @@ +#!/usr/bin/env python3 +"""Carve the per-device Mesa calibration from an Apple iBoot System Container.""" + +import argparse +import hashlib +import os +from pathlib import Path +import stat +import sys +import tempfile + + +ISC_PARTITION_NAME = "iBootSystemContainer" +SYS_BLOCK = Path("/sys/class/block") +DEV_ROOT = Path("/dev") +SCAN_CHUNK_SIZE = 4 * 1024 * 1024 +SCAN_OVERLAP = 4 * 1024 * 1024 + + +class DERError(ValueError): + pass + + +class TLV: + def __init__(self, data, start): + self.start = start + try: + self.tag = data[start] + first_length = data[start + 1] + except IndexError as error: + raise DERError("truncated DER header") from error + + if first_length < 0x80: + length = first_length + header_length = 2 + else: + length_bytes = first_length & 0x7f + if length_bytes == 0: + raise DERError("indefinite DER length") + if length_bytes > 4: + raise DERError("unreasonably large DER length") + length_start = start + 2 + length_end = length_start + length_bytes + if length_end > len(data): + raise DERError("truncated DER length") + encoded = data[length_start:length_end] + if encoded[0] == 0: + raise DERError("non-minimal DER length") + length = int.from_bytes(encoded, "big") + if length < 0x80: + raise DERError("non-minimal DER length") + header_length = 2 + length_bytes + + self.content_start = start + header_length + self.end = self.content_start + length + if self.end > len(data): + raise DERError("DER value extends beyond input") + + def content(self, data): + return data[self.content_start:self.end] + + +def child_tlvs(data, container): + offset = container.content_start + while offset < container.end: + child = TLV(data, offset) + if child.end > container.end: + raise DERError("DER child extends beyond its container") + yield child + offset = child.end + if offset != container.end: + raise DERError("malformed DER container") + + +def take_child(data, children, tag, value=None): + try: + child = next(children) + except StopIteration as error: + raise DERError("missing DER child") from error + if child.tag != tag: + raise DERError(f"unexpected DER tag 0x{child.tag:02x}") + if value is not None and child.content(data) != value: + raise DERError("unexpected DER value") + return child + +def validate_calibration(data, start): + """Return the end offset if data[start:] begins with a Mesa calibration.""" + outer = TLV(data, start) + if outer.tag != 0x30: + raise DERError("outer object is not a sequence") + outer_children = child_tlvs(data, outer) + take_child(data, outer_children, 0x16, b"comb") + + fdrd = take_child(data, outer_children, 0x30) + fdrd_children = child_tlvs(data, fdrd) + take_child(data, fdrd_children, 0x16, b"fdrd") + img4_octets = take_child(data, fdrd_children, 0x04) + try: + next(fdrd_children) + except StopIteration: + pass + else: + raise DERError("unexpected data in fdrd container") + + secb = take_child(data, outer_children, 0x30) + secb_children = child_tlvs(data, secb) + take_child(data, secb_children, 0x16, b"secb") + try: + next(outer_children) + except StopIteration: + pass + else: + raise DERError("unexpected data after secb container") + + img4 = TLV(data, img4_octets.content_start) + if img4.tag != 0x30 or img4.end != img4_octets.end: + raise DERError("fdrd is not one complete IMG4 sequence") + img4_children = child_tlvs(data, img4) + take_child(data, img4_children, 0x16, b"IMG4") + + im4p = take_child(data, img4_children, 0x30) + im4p_children = child_tlvs(data, im4p) + take_child(data, im4p_children, 0x16, b"IM4P") + take_child(data, im4p_children, 0x16, b"FSCl") + take_child(data, im4p_children, 0x16) + payload = take_child(data, im4p_children, 0x04).content(data) + if b"CALB" not in payload[:64]: + raise DERError("FSCl payload lacks its CALB header") + + # A signed calibration has an IM4M manifest following the IM4P. Requiring + # both markers avoids accepting an arbitrary, truncated FSCl object. + remainder = data[im4p.end:img4.end] + if b"IM4M" not in remainder or b"FSCl" not in remainder: + raise DERError("IMG4 lacks an FSCl manifest") + + return outer.end + + +def find_calibrations(data): + results = [] + signature = b"\x16\x04comb" + search_at = 0 + while True: + marker = data.find(signature, search_at) + if marker < 0: + break + # The outer SEQUENCE header immediately precedes the comb IA5String. + # Try every legal header width supported by TLV above. + for header_length in range(2, 7): + start = marker - header_length + if start < 0 or data[start] != 0x30: + continue + try: + end = validate_calibration(data, start) + except DERError: + continue + candidate = (start, end) + if candidate not in results: + results.append(candidate) + search_at = marker + 1 + return results + + +def discover_isc_inputs(sys_block=SYS_BLOCK, dev_root=DEV_ROOT): + """Find every iSC by GPT partition name, independent of disk numbering.""" + candidates = [] + try: + block_devices = sorted(sys_block.iterdir()) + except OSError as error: + block_devices = [] + sysfs_error = error + else: + sysfs_error = None + + for block_device in block_devices: + try: + properties = {} + for line in (block_device / "uevent").read_text().splitlines(): + key, separator, value = line.partition("=") + if separator: + properties[key] = value + except OSError: + continue + if properties.get("PARTNAME") != ISC_PARTITION_NAME: + continue + device_name = properties.get("DEVNAME", block_device.name) + if device_name.startswith("/dev/"): + device_name = device_name[5:] + device = dev_root / device_name + if device.exists(): + candidates.append(device) + + # This standard udev link also covers chroots where /sys is unavailable. + by_partlabel = dev_root / "disk/by-partlabel" / ISC_PARTITION_NAME + if by_partlabel.exists(): + candidates.append(by_partlabel) + + # Preserve order while removing aliases or duplicate sysfs entries. + unique = [] + seen = set() + for candidate in candidates: + try: + identity = candidate.resolve() + except OSError: + identity = candidate + if identity not in seen: + seen.add(identity) + unique.append(candidate) + if not unique and sysfs_error is not None: + raise RuntimeError( + f"cannot inspect {sys_block} and {by_partlabel} is unavailable: " + f"{sysfs_error}" + ) from sysfs_error + return unique + + +def scan_input(path): + """Scan a file or block device without retaining the whole iSC in memory.""" + mode = path.stat().st_mode + if not (stat.S_ISREG(mode) or stat.S_ISBLK(mode)): + raise RuntimeError(f"{path} is not a regular file or block device") + + candidates = {} + overlap = b"" + total = 0 + with path.open("rb", buffering=0) as source: + while True: + chunk = source.read(SCAN_CHUNK_SIZE) + if not chunk: + break + total += len(chunk) + window = overlap + chunk + window_start = total - len(window) + for start, end in find_calibrations(window): + absolute_start = window_start + start + candidates.setdefault(absolute_start, bytes(window[start:end])) + overlap = window[-SCAN_OVERLAP:] + return sorted(candidates.items()) + + +def write_private(path, blob): + path.parent.mkdir(parents=True, exist_ok=True) + old_umask = os.umask(0o077) + temporary = None + try: + fd, name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temporary = Path(name) + with os.fdopen(fd, "wb") as output: + output.write(blob) + output.flush() + os.fsync(output.fileno()) + os.chmod(temporary, stat.S_IRUSR | stat.S_IWUSR) + os.replace(temporary, path) + temporary = None + finally: + os.umask(old_umask) + if temporary is not None: + temporary.unlink(missing_ok=True) + + +def validate_output(path, inputs): + """Reject output targets that could replace an input or special file.""" + if path.is_symlink(): + raise RuntimeError(f"output {path} is a symbolic link") + if not path.exists(): + return + for input_path in inputs: + try: + if os.path.samefile(path, input_path): + raise RuntimeError(f"output {path} aliases input {input_path}") + except FileNotFoundError: + pass + if not stat.S_ISREG(path.stat().st_mode): + raise RuntimeError(f"output {path} is not a regular file") + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "input", nargs="?", type=Path, + help=( + "raw iBoot System Container or dump; omit to discover all " + "iBootSystemContainer partitions" + ), + ) + parser.add_argument( + "-o", "--output", type=Path, default=Path("mesa_calibration.bin"), + help="output file (default: ./mesa_calibration.bin)", + ) + return parser.parse_args(argv) + + +def main(): + args = parse_args() + try: + inputs = [args.input] if args.input is not None else discover_isc_inputs() + if not inputs: + raise RuntimeError( + "no iBootSystemContainer partition found; pass its block device " + "or a raw partition dump explicitly" + ) + validate_output(args.output, inputs) + + candidates = [] + for input_path in inputs: + for offset, blob in scan_input(input_path): + candidates.append((input_path, offset, blob)) + if not candidates: + raise RuntimeError( + "no signed Mesa FSCl/CALB calibration found; expected the raw " + "iBoot System Container (Apple Silicon boot partition)" + ) + blobs = {} + for input_path, offset, blob in candidates: + blobs.setdefault(blob, []).append((input_path, offset)) + if len(blobs) != 1: + locations = ", ".join( + f"{input_path}@0x{offset:x}" + for input_path, offset, _ in candidates + ) + raise RuntimeError( + f"found different calibrations at {locations}; " + "pass the intended iSC explicitly" + ) + + blob, locations = next(iter(blobs.items())) + write_private(args.output, blob) + except (OSError, RuntimeError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + digest = hashlib.sha256(blob).hexdigest() + formatted_locations = ", ".join( + f"{input_path}@0x{offset:x}" for input_path, offset in locations + ) + print(f"found Mesa calibration at {formatted_locations}") + print(f"wrote {len(blob)} bytes to {args.output}") + print(f"sha256 {digest}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/aurora-sep/aurora-touchid/fprintd-aurora.conf b/tools/aurora-sep/aurora-touchid/fprintd-aurora.conf new file mode 100644 index 0000000000000..bfa2385635b43 --- /dev/null +++ b/tools/aurora-sep/aurora-touchid/fprintd-aurora.conf @@ -0,0 +1,12 @@ +[Unit] +After=apple-sep.service + +[Service] +# apple-sep.service only runs modprobe; /dev/sep-bio appears a few seconds +# later, once the enclave is up. Wait for it (at most 15 s) so fprintd does not +# start, find no device and cache "No devices available" until it idles out. +ExecStartPre=/bin/sh -c 'i=0; while [ ! -e /dev/sep-bio ] && [ $i -lt 150 ]; do sleep 0.1; i=$((i+1)); done; exit 0' +# Allow the misc class rather than the node: a path rule is resolved to a +# device number when the cgroup is set up, before ExecStartPre runs, and is +# dropped if /dev/sep-bio does not exist yet. +DeviceAllow=char-misc rw diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 20430ae284b9e..92a5f10d71f17 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -32,7 +32,7 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. VERSION=7.1.12.aurora2-11.6 -TAG=sep-7.1.12.aurora2-11.12 +TAG=sep-7.1.12.aurora2-11.13 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -43,7 +43,7 @@ PACKAGES=( "linux-aurora-$VERSION-aarch64.pkg.tar.zst 8a7fb83f371bd257985515f9d4e5ac8479ed6c549775172ae6ebfd558d9e3ef7" "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 994f34f6413b43a5905eda8559a420f67fd4c34797ccc7784eb9afcb2793914e" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" - "aurora-touchid-20260929-1-any.pkg.tar.zst c3c84488330a55f25f3a2f241714d0c4b09716be363082737543507337773f5b" + "aurora-touchid-20260930-1-any.pkg.tar.zst 2dc5bd93e923b551d201db469d094bfec6f8270de26ef3f0eda81137bf77f2f6" "m1n1-aurora-1.6.1.aurora2-2-aarch64.pkg.tar.zst 99f53b6a300d994965ccd9b0baf089122a4a3963180830544680b34ef34eaab6" ) PINNED="linux-aurora linux-aurora-headers libfprint m1n1-aurora" @@ -312,12 +312,23 @@ grub_update() { } calibration() { - local node name out - for node in /proc/device-tree/soc*/spi*/*/ /proc/device-tree/*/spi*/*/; do - [[ -f $node/compatible ]] && grep -qa mesa "$node/compatible" || continue - [[ -f $node/firmware-name ]] && name=$(tr -d '\0' <"$node/firmware-name") - break - done + local node name="" out board dtb + # The driver reads firmware-name from the device tree it boots, which is the + # aurora one installed above. When this runs from linux-asahi, the running + # tree has no sensor node at all, so ask the installed aurora DTB first. + board=$(tr '\0' '\n' /dev/null | sed -n '1s/^apple,//p') + if [[ -n $board ]]; then + dtb=$(pacman -Qlq linux-aurora 2>/dev/null | grep -m1 "/dtbs/t[0-9]*-$board\.dtb$" || true) + [[ -n $dtb && -f $dtb ]] && + name=$(grep -aoE -m1 'apple/mesacal-[A-Za-z0-9_-]+\.bin' "$dtb" | head -1 || true) + fi + if [[ -z $name ]]; then + for node in /proc/device-tree/soc*/spi*/*/ /proc/device-tree/*/spi*/*/; do + [[ -f $node/compatible ]] && grep -qa mesa "$node/compatible" || continue + [[ -f $node/firmware-name ]] && name=$(tr -d '\0' <"$node/firmware-name") + break + done + fi name=${name:-apple/mesa_calibration.bin} out=/usr/lib/firmware/$name if [[ -s $out ]]; then From 894f7126a91768ba5185bd389cd9d91fd3084a32 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:02:52 -0700 Subject: [PATCH 0785/2680] tools: aurora-sep: release 11.14, the consolidated Touch ID series Pin linux-aurora 7.1.12.aurora2-11.14, built from this branch with the series proposed upstream as aurora-silicon/linux#69: - the match credential is set up again after a cold restore on every profile, so enrolments match after a reboot; - every profile persists enrolments; - the J314s SEP and Mesa sensor are enabled; - j293 runs its sensor in SPI mode 2; - the sensor bring-up is retried on verify; - a refused keybag create can be retried; - verify, enrol, provisioning and bring-up failures are logged. The kernel package number now matches the release tag, to avoid two counters. The other packages are unchanged from 11.13. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- tools/aurora-sep/install-aurora-sep.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 92a5f10d71f17..4873a0dc4082f 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -5,10 +5,11 @@ # ... | bash -s -- --read-only install, but never let the driver write to the enclave # ... | bash -s -- --uninstall go back to the kernel this Mac had before # -# Kernel: iconidentify/aurora-linux custom/sep (17cba00e43b9), aurora-silicon/linux aurora-wip plus the +# Kernel: iconidentify/aurora-linux custom/sep (1e25fc8fee0a), aurora-silicon/linux aurora-wip plus the # Secure Enclave (Touch ID) driver, Thunderbolt (#8), the Apple video -# decoder (#45), the M2 Max (t6021) profile and the fix for matching after a -# reboot on M2 Pro/Max (#61). It replaces linux-asahi (or linux-aurora) as a pacman package, +# decoder (#45), the M2 Max (t6021) profile and the consolidated Touch ID +# series (aurora-silicon/linux#69: matching after a reboot on every profile, +# M1 Pro J314s, j293 SPI mode). It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. # @@ -31,8 +32,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.6 -TAG=sep-7.1.12.aurora2-11.13 +VERSION=7.1.12.aurora2-11.14 +TAG=sep-7.1.12.aurora2-11.14 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -40,8 +41,8 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst 8a7fb83f371bd257985515f9d4e5ac8479ed6c549775172ae6ebfd558d9e3ef7" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 994f34f6413b43a5905eda8559a420f67fd4c34797ccc7784eb9afcb2793914e" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst 2d66e0703c433ad6e72113cdbab442e12784347b065eaf64d6659e0b80108d58" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst a8619221fe11cd91acba03c04c87c56964b620cb0cdbcf59d9e4c55bccff579c" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20260930-1-any.pkg.tar.zst 2dc5bd93e923b551d201db469d094bfec6f8270de26ef3f0eda81137bf77f2f6" "m1n1-aurora-1.6.1.aurora2-2-aarch64.pkg.tar.zst 99f53b6a300d994965ccd9b0baf089122a4a3963180830544680b34ef34eaab6" From 69e519d2316b8b9742fe375dc5d6c3a98692324f Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:25:33 -0700 Subject: [PATCH 0786/2680] tools: aurora-sep: release 11.15, Touch ID across suspend Pin linux-aurora 7.1.12.aurora2-11.15: 11.14 plus the commit that ends a Touch ID capture before the system suspends, now also part of aurora-silicon/linux#69. A Mac whose lid closed while the lock screen waited for a fingerprint could fail to wake; the capture now ends before sleep, and a new one waits until the Mac is awake. The agent test plan now asks for that case: lock, close the lid with the lock screen waiting for a finger, then wake and unlock. The other packages are unchanged from 11.14. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- tools/aurora-sep/install-aurora-sep.sh | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 4873a0dc4082f..5173f0245bb14 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -5,11 +5,12 @@ # ... | bash -s -- --read-only install, but never let the driver write to the enclave # ... | bash -s -- --uninstall go back to the kernel this Mac had before # -# Kernel: iconidentify/aurora-linux custom/sep (1e25fc8fee0a), aurora-silicon/linux aurora-wip plus the +# Kernel: iconidentify/aurora-linux custom/sep (e4c91eb8eaf0), aurora-silicon/linux aurora-wip plus the # Secure Enclave (Touch ID) driver, Thunderbolt (#8), the Apple video # decoder (#45), the M2 Max (t6021) profile and the consolidated Touch ID # series (aurora-silicon/linux#69: matching after a reboot on every profile, -# M1 Pro J314s, j293 SPI mode). It replaces linux-asahi (or linux-aurora) as a pacman package, +# M1 Pro J314s, j293 SPI mode, a scan in progress ended before sleep). +# It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. # @@ -32,8 +33,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.14 -TAG=sep-7.1.12.aurora2-11.14 +VERSION=7.1.12.aurora2-11.15 +TAG=sep-7.1.12.aurora2-11.15 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -41,8 +42,8 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst 2d66e0703c433ad6e72113cdbab442e12784347b065eaf64d6659e0b80108d58" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst a8619221fe11cd91acba03c04c87c56964b620cb0cdbcf59d9e4c55bccff579c" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst 9a8d96c0e7270e0d2da24a1fa46f8762bbc4e76eb204d0619a5b904e5ab5a76a" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 81602dcf6c9961cb3f5b1b7c87cb97bdf8b84ead63a7b9d406df2c0b06ac1752" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20260930-1-any.pkg.tar.zst 2dc5bd93e923b551d201db469d094bfec6f8270de26ef3f0eda81137bf77f2f6" "m1n1-aurora-1.6.1.aurora2-2-aarch64.pkg.tar.zst 99f53b6a300d994965ccd9b0baf089122a4a3963180830544680b34ef34eaab6" @@ -662,6 +663,10 @@ SAFETY, NON-NEGOTIABLE Report which codecs you exercised and at what resolution. 8. ONE SUSPEND/RESUME CYCLE AT THE END + Ask the owner to lock the screen and, while it waits for a fingerprint, + close the lid. Then open it and unlock with the finger. Report whether + the Mac woke and whether the finger unlocked it, and quote: + sudo dmesg | grep -E 'Touch ID: ending|still running|PM: suspend' Re-check /dev/sep-bio, the display and the dock after resume. Report "not tested" honestly rather than guessing, for any of the above. From c06d78f3ddadbafa7b9f3beb0a0e1dd536ea5dbc Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 30 Sep 2026 20:45:06 -0700 Subject: [PATCH 0787/2680] tools: aurora-sep: release 11.16, MacBook Neo and M2 Pro/Max displays Pin linux-aurora 7.1.12.aurora2-11.16, which adds to 11.15: - current aurora-wip (AFK slot reuse, pmp-report lock, deferred locked DART root publication); - Thunderbolt displays on M1 by default and on every M2 Pro/Max laptop, as on aurora-silicon/linux#8; - the MacBook Neo work from #40, #43, #54 and #55, and #42's SEP, DART, mailbox, SPI2 and Touch ID sensor device tree with its SMC phandle fix. The kernel config also turns on T8140 AOP audio and the DockChannel console, and makes power_allocator the default thermal governor; only the J700 has thermal zones with trips. Pin m1n1-aurora 1.6.1.aurora3-1: AsahiLinux/m1n1 main at 3e354a24, which knows the macOS 26.5 to 27.0 firmware the Neo ships with, plus the SEP warm-registration guard, preboot-UUID forwarding and the pcie-adapter alias fallback rebased onto it. The guard no longer returns early, so upstream's M4 idle bootargs still apply. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- tools/aurora-sep/install-aurora-sep.sh | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 5173f0245bb14..d8a5a19f178db 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -5,11 +5,14 @@ # ... | bash -s -- --read-only install, but never let the driver write to the enclave # ... | bash -s -- --uninstall go back to the kernel this Mac had before # -# Kernel: iconidentify/aurora-linux custom/sep (e4c91eb8eaf0), aurora-silicon/linux aurora-wip plus the +# Kernel: iconidentify/aurora-linux custom/sep (54ebceaef14d), aurora-silicon/linux aurora-wip plus the # Secure Enclave (Touch ID) driver, Thunderbolt (#8), the Apple video # decoder (#45), the M2 Max (t6021) profile and the consolidated Touch ID # series (aurora-silicon/linux#69: matching after a reboot on every profile, -# M1 Pro J314s, j293 SPI mode, a scan in progress ended before sleep). +# M1 Pro J314s, j293 SPI mode, a scan in progress ended before sleep), +# Thunderbolt displays on M1 by default and on every M2 Pro/Max laptop +# (aurora-silicon/linux#8), and the MacBook Neo (J700) work from #40, #42, +# #43, #54 and #55, including its Touch ID device tree. # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -18,8 +21,11 @@ # service and this Mac's own sensor calibration. After the reboot, run # aurora-touchid-setup to enrol a finger and use it for sudo and the lock screen. # -# m1n1: m1n1-aurora builds aurora-silicon/m1n1 main (97e2de3d4fec) plus one -# patch, the usb4-N-pcie-adapter alias fallback (aurora-silicon/m1n1#4). +# m1n1: m1n1-aurora builds AsahiLinux/m1n1 main (3e354a24), which knows the +# macOS 26.5 to 27.0 firmware the MacBook Neo ships with, plus three patches: +# the SEP warm-registration guard and preboot-UUID forwarding from +# aurora-silicon/m1n1, and the usb4-N-pcie-adapter alias fallback +# (aurora-silicon/m1n1#4). # On M2 and later the platform hands Linux an already-running Secure # Enclave, and the driver attaches to it with one registration that can only be # sent once per boot. Stock m1n1 asks the enclave for randomness on the way up, @@ -33,8 +39,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.15 -TAG=sep-7.1.12.aurora2-11.15 +VERSION=7.1.12.aurora2-11.16 +TAG=sep-7.1.12.aurora2-11.16 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -42,11 +48,11 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst 9a8d96c0e7270e0d2da24a1fa46f8762bbc4e76eb204d0619a5b904e5ab5a76a" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 81602dcf6c9961cb3f5b1b7c87cb97bdf8b84ead63a7b9d406df2c0b06ac1752" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst 5368979c976ae1ded203ce7938bcb20db3cf76109fd607687e825939272e1b94" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 94f88e0fe75da57b12984a29d9763e0f531dc36a91d5eb17cc37461b7f694725" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20260930-1-any.pkg.tar.zst 2dc5bd93e923b551d201db469d094bfec6f8270de26ef3f0eda81137bf77f2f6" - "m1n1-aurora-1.6.1.aurora2-2-aarch64.pkg.tar.zst 99f53b6a300d994965ccd9b0baf089122a4a3963180830544680b34ef34eaab6" + "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" ) PINNED="linux-aurora linux-aurora-headers libfprint m1n1-aurora" PIN_BEGIN="# >>> aurora-sep pin (remove with: install-aurora-sep.sh --uninstall)" @@ -597,6 +603,8 @@ SAFETY, NON-NEGOTIABLE # t8103 0x242408000 -> 0x242408110 / 0x242408114 # t8112 0x25e408000 -> 0x25e408110 / 0x25e408114 # t600x, t602x 0x396408000 -> 0x396408110 / 0x396408114 + # t8140 (Neo) 0x282608000 (a v4 mailbox: report the base and + # the /proc/interrupts lines, not these offsets) Bit 0 is the enable; a non-zero FIFO count with an unchanged read pointer means the enclave never took the message. Report the raw values, not your From 17d7c219b1647824244647b2b7b430aaebb93b57 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Thu, 1 Oct 2026 08:01:11 -0700 Subject: [PATCH 0788/2680] tools: aurora-sep: release 11.17, two displays per dock and M1 Pro/Max Pin linux-aurora 7.1.12.aurora2-11.17, which adds to 11.16: - two independent displays through one dock on every M2 Pro/Max laptop (DjDeveloperr's #46, ported onto aurora-silicon/linux#8), with live USB3 bandwidth accounting so the second tunnel is admitted; - Thunderbolt display on M1 Pro/Max (cb2206's #64), and a clear-swap timeout at power-off no longer wedges the display until reboot; - Type-C display connectors reported as DisplayPort (DP-N), so a docked lid close does not suspend, and fixed possible_crtcs for the second stream, which Hyprland reads once (rryan's #50); - Touch ID enrol and verify refused with EROFS instead of wedging the enclave when xART writes are off (aurora-silicon/linux#69); - EDIDs longer than the base block announces are accepted (#67). Type-C display connectors are now named DP-N instead of USB-N. The other packages are unchanged from 11.16. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- tools/aurora-sep/install-aurora-sep.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index d8a5a19f178db..e5c9abed86d2a 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -5,13 +5,14 @@ # ... | bash -s -- --read-only install, but never let the driver write to the enclave # ... | bash -s -- --uninstall go back to the kernel this Mac had before # -# Kernel: iconidentify/aurora-linux custom/sep (54ebceaef14d), aurora-silicon/linux aurora-wip plus the +# Kernel: iconidentify/aurora-linux custom/sep (e11a8bc74cb8), aurora-silicon/linux aurora-wip plus the # Secure Enclave (Touch ID) driver, Thunderbolt (#8), the Apple video # decoder (#45), the M2 Max (t6021) profile and the consolidated Touch ID # series (aurora-silicon/linux#69: matching after a reboot on every profile, # M1 Pro J314s, j293 SPI mode, a scan in progress ended before sleep), -# Thunderbolt displays on M1 by default and on every M2 Pro/Max laptop -# (aurora-silicon/linux#8), and the MacBook Neo (J700) work from #40, #42, +# Thunderbolt displays on M1 and M1 Pro/Max by default, and two displays +# through one dock on every M2 Pro/Max laptop (aurora-silicon/linux#8, with +# #46, #50 and #64), and the MacBook Neo (J700) work from #40, #42, # #43, #54 and #55, including its Touch ID device tree. # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this @@ -39,8 +40,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.16 -TAG=sep-7.1.12.aurora2-11.16 +VERSION=7.1.12.aurora2-11.17 +TAG=sep-7.1.12.aurora2-11.17 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -48,8 +49,8 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst 5368979c976ae1ded203ce7938bcb20db3cf76109fd607687e825939272e1b94" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 94f88e0fe75da57b12984a29d9763e0f531dc36a91d5eb17cc37461b7f694725" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst 47a0ff9f8c8fc00edd3485668e0c71cc6d43a846d2dbf7abb160fbe2f833704e" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 0fbb3c6f2c8c2bfc672ad02cf40ed50a418a5cdbebbc140eb00bd9cc017be6f4" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20260930-1-any.pkg.tar.zst 2dc5bd93e923b551d201db469d094bfec6f8270de26ef3f0eda81137bf77f2f6" "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" From 77016ce5bb8af7a249981655d8da7d4b73fae096 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Thu, 1 Oct 2026 10:59:08 -0700 Subject: [PATCH 0789/2680] tools: aurora-sep: release 11.18, stop fprintd before suspend aurora-touchid 20261001-1 adds a system-sleep hook that stops fprintd before the system suspends. If the lock screen is waiting for a finger when the lid closes, fprintd cannot take its own sleep delay inhibitor (logind answers OperationInProgress) and never releases its claim on the sensor. After resume every fingerprint session then failed with PAM code 9 until fprintd was restarted, and the lock screen fell back to the password. With the hook, D-Bus starts a fresh fprintd after resume and the finger unlocks. Diagnosed independently by cb2206 (j314s) and thomasanderson88 (j293) on omacom/linux#7; cb2206's hook passed 5 of 5 lid sleeps, including 7 h overnight and the manual-lock case. The kernel stays linux-aurora 7.1.12.aurora2-11.17; the other packages are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- tools/aurora-sep/aurora-touchid/PKGBUILD | 9 +++++--- .../aurora-touchid/aurora-touchid.sleep | 23 +++++++++++++++++++ tools/aurora-sep/install-aurora-sep.sh | 8 ++++--- 3 files changed, 34 insertions(+), 6 deletions(-) create mode 100755 tools/aurora-sep/aurora-touchid/aurora-touchid.sleep diff --git a/tools/aurora-sep/aurora-touchid/PKGBUILD b/tools/aurora-sep/aurora-touchid/PKGBUILD index 738d53d269f47..07eaf688c92d7 100644 --- a/tools/aurora-sep/aurora-touchid/PKGBUILD +++ b/tools/aurora-sep/aurora-touchid/PKGBUILD @@ -1,6 +1,6 @@ # Maintainer: Chris Kearney <303316+iconidentify@users.noreply.github.com> pkgname=aurora-touchid -pkgver=20260930 +pkgver=20261001 pkgrel=1 pkgdesc="Touch ID for Omarchy on Apple Silicon (aurora kernel SEP driver)" arch=('any') @@ -11,12 +11,14 @@ source=('aurora-touchid-setup' 'extract-mesa-calibration' 'apple-sep.service' 'apple-sep.path' - 'fprintd-aurora.conf') + 'fprintd-aurora.conf' + 'aurora-touchid.sleep') sha256sums=('ab4ed299d03462210e03e8d4152803a414204ebf25ec0f065b1b7dc3d43667b1' '2e1d50b5274223e107e4aa4f73de194f3a49d347be1edc93efd3f81b31b092cc' '4f7b8a97e35e3511afa482ee8aedd019e960b1a67320675bd347f534feab49c3' '2f4b265a4c2180a9e0e4926c53ec77c3dbb074ddc3e083d7925d58db10a576b9' - 'e1b1152e576e9588cc134b09291b3a2306ac686d9a47be6f128670608da535fc') + 'e1b1152e576e9588cc134b09291b3a2306ac686d9a47be6f128670608da535fc' + 'a87526383319cc7c1214082196d0d80ad9a6da09b5c30bf215f866e01443c611') package() { install -Dm755 aurora-touchid-setup "$pkgdir/usr/bin/aurora-touchid-setup" @@ -24,4 +26,5 @@ package() { install -Dm644 apple-sep.service "$pkgdir/usr/lib/systemd/system/apple-sep.service" install -Dm644 apple-sep.path "$pkgdir/usr/lib/systemd/system/apple-sep.path" install -Dm644 fprintd-aurora.conf "$pkgdir/usr/lib/systemd/system/fprintd.service.d/aurora.conf" + install -Dm755 aurora-touchid.sleep "$pkgdir/usr/lib/systemd/system-sleep/aurora-touchid" } diff --git a/tools/aurora-sep/aurora-touchid/aurora-touchid.sleep b/tools/aurora-sep/aurora-touchid/aurora-touchid.sleep new file mode 100755 index 0000000000000..2cbb5d490b493 --- /dev/null +++ b/tools/aurora-sep/aurora-touchid/aurora-touchid.sleep @@ -0,0 +1,23 @@ +#!/bin/sh +# Stop fprintd before the system sleeps. +# +# If the lock screen is waiting for a fingerprint when the lid closes, +# fprintd cannot install its own sleep delay inhibitor (logind answers +# OperationInProgress), so it never releases its claim on the sensor. +# After resume every fingerprint session then fails until fprintd is +# restarted, and the lock screen falls back to the password. Stopping it +# here drops the claim; D-Bus activation starts a fresh fprintd when the +# lock screen asks for a finger after resume. +# +# Diagnosed and tested by cb2206 and thomasanderson88 (omacom/linux#7). +# Never fail the sleep: always exit 0. + +case "$1" in +pre) + if systemctl -q is-active fprintd.service; then + timeout 5 systemctl stop fprintd.service + sleep 0.2 + fi + ;; +esac +exit 0 diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index e5c9abed86d2a..b5ffd9febfbf2 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -19,7 +19,9 @@ # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. # # Touch ID: libfprint with the Apple SEP driver, fprintd, the apple-sep -# service and this Mac's own sensor calibration. After the reboot, run +# service, a sleep hook that stops fprintd before suspend (so a lock screen +# waiting for a finger at lid close can't leave the sensor claimed), and +# this Mac's own sensor calibration. After the reboot, run # aurora-touchid-setup to enrol a finger and use it for sudo and the lock screen. # # m1n1: m1n1-aurora builds AsahiLinux/m1n1 main (3e354a24), which knows the @@ -41,7 +43,7 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. VERSION=7.1.12.aurora2-11.17 -TAG=sep-7.1.12.aurora2-11.17 +TAG=sep-7.1.12.aurora2-11.18 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -52,7 +54,7 @@ PACKAGES=( "linux-aurora-$VERSION-aarch64.pkg.tar.zst 47a0ff9f8c8fc00edd3485668e0c71cc6d43a846d2dbf7abb160fbe2f833704e" "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 0fbb3c6f2c8c2bfc672ad02cf40ed50a418a5cdbebbc140eb00bd9cc017be6f4" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" - "aurora-touchid-20260930-1-any.pkg.tar.zst 2dc5bd93e923b551d201db469d094bfec6f8270de26ef3f0eda81137bf77f2f6" + "aurora-touchid-20261001-1-any.pkg.tar.zst bc6438d23dc2aae2766677f41f3f6d61ec3716df55c3b7a9c2cf93a027c68660" "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" ) PINNED="linux-aurora linux-aurora-headers libfprint m1n1-aurora" From 310db8f000182b7cb759c456b8d30a637e4f843d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:09:55 -0700 Subject: [PATCH 0790/2680] tools: aurora-sep: keep a MacBook Neo's own m1n1 A MacBook Neo boots an m1n1 built from aurora-silicon's J700 branch. m1n1-aurora has no T8140 support, so installing it, or the stock m1n1 on removal, would replace the m1n1 the Neo needs. On a T8140 the installer now skips the m1n1 package both ways and only rebuilds boot.bin with the new device trees. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- tools/aurora-sep/install-aurora-sep.sh | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index b5ffd9febfbf2..294fd44286ec7 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -120,6 +120,13 @@ boot_chain() { fi } +# The MacBook Neo (T8140) boots an m1n1 built from aurora-silicon's J700 +# branch. m1n1-aurora has no T8140 support, so on a Neo this script never +# installs it or the stock m1n1 over the one the Mac already boots. +is_neo() { + tr '\0' '\n' /dev/null | grep -qx 'apple,t8140' +} + current_kernel() { if pacman -Q linux-aurora >/dev/null 2>&1; then echo linux-aurora elif pacman -Q linux-asahi >/dev/null 2>&1; then echo linux-asahi @@ -414,6 +421,10 @@ install_all() { trap 'rm -rf "${work:-}"' EXIT for entry in "${PACKAGES[@]}"; do read -r file sha <<<"$entry" + if [[ $file == m1n1-aurora-* ]] && is_neo; then + say "Keeping this MacBook Neo's own m1n1 (m1n1-aurora has no T8140 support)" + continue + fi say "Downloading $file" curl -fL --retry 3 --progress-bar -o "$work/$file" "$RELEASE_URL/$file" || die "could not download $file from $TAG. @@ -483,9 +494,15 @@ uninstall_all() { $sudo systemctl disable apple-sep.path apple-sep.service 2>/dev/null || true remove_pin snapshot "removing aurora-sep" - say "Reinstalling $previous, the stock m1n1 and the stock libfprint" + local m1n1=m1n1 + if is_neo; then + m1n1= + say "Reinstalling $previous and the stock libfprint; this MacBook Neo keeps its own m1n1" + else + say "Reinstalling $previous, the stock m1n1 and the stock libfprint" + fi $sudo pacman -Rdd --noconfirm aurora-touchid 2>/dev/null || true - $sudo pacman -Sy --noconfirm --ask 4 "$previous" "$previous-headers" libfprint m1n1 + $sudo pacman -Sy --noconfirm --ask 4 "$previous" "$previous-headers" libfprint $m1n1 # Restore the stock update-m1n1 configuration on either chain before the # rebuild below, so boot.bin goes back to the packaged m1n1 and DTBs. if [[ $(boot_chain) != grub ]]; then From 95bf6d0ae02278cf33a720a9005e9f0f832c96b1 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:24:12 -0700 Subject: [PATCH 0791/2680] tools: aurora-sep: release 11.20, M2 Air Touch ID and the Omarchy logo Kernel 7.1.12.aurora2-11.20 from custom/sep 77542ad1e56c: aurora-wip through 3bb0a6104a11 (Neo AVD, PCIe and radios, inert on other Macs), Touch ID on the MacBook Air M2 15" (aurora-silicon/linux#152), a retry from a fresh sensor registration after a failed first patch, battery time estimates that read ENODATA (#53) and the Omarchy boot logo. libfprint, aurora-touchid and m1n1-aurora 1.6.1.aurora3 are unchanged. Boot-tested in maclab (m1air, job j1002-001901-82b2): Wi-Fi, PCIe, AVD, Touch ID attach, ENODATA on a full battery; omarchy-m-test shows no regressions against 11.19. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- tools/aurora-sep/install-aurora-sep.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 294fd44286ec7..7738a7818292c 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -5,7 +5,7 @@ # ... | bash -s -- --read-only install, but never let the driver write to the enclave # ... | bash -s -- --uninstall go back to the kernel this Mac had before # -# Kernel: iconidentify/aurora-linux custom/sep (e11a8bc74cb8), aurora-silicon/linux aurora-wip plus the +# Kernel: iconidentify/aurora-linux custom/sep (77542ad1e56c), aurora-silicon/linux aurora-wip plus the # Secure Enclave (Touch ID) driver, Thunderbolt (#8), the Apple video # decoder (#45), the M2 Max (t6021) profile and the consolidated Touch ID # series (aurora-silicon/linux#69: matching after a reboot on every profile, @@ -14,6 +14,12 @@ # through one dock on every M2 Pro/Max laptop (aurora-silicon/linux#8, with # #46, #50 and #64), and the MacBook Neo (J700) work from #40, #42, # #43, #54 and #55, including its Touch ID device tree. +# 11.20 adds Touch ID on the MacBook Air M2 15" (J415, aurora-silicon/linux#152), +# a retry from a fresh sensor registration when the first sensor patch fails at +# boot, battery time estimates that read "no data" instead of 0 (#53), the +# Omarchy boot logo in place of Tux, and aurora-wip through 3bb0a6104a11 (the +# MacBook Neo video decoder, PCIe bring-up and radio drivers, inert elsewhere). +# On a MacBook Neo the script keeps the Neo's own m1n1. # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -42,8 +48,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.17 -TAG=sep-7.1.12.aurora2-11.18 +VERSION=7.1.12.aurora2-11.20 +TAG=sep-7.1.12.aurora2-11.20 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -51,8 +57,8 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst 47a0ff9f8c8fc00edd3485668e0c71cc6d43a846d2dbf7abb160fbe2f833704e" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 0fbb3c6f2c8c2bfc672ad02cf40ed50a418a5cdbebbc140eb00bd9cc017be6f4" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst c8f000aa2541256e78c3b53e7d66068559357644e75f259ddbc12538105eccb0" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst f1eb3174f6f9c1f0071446f36b39ef978466cb7d51b6cd622aa23c1208fca921" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20261001-1-any.pkg.tar.zst bc6438d23dc2aae2766677f41f3f6d61ec3716df55c3b7a9c2cf93a027c68660" "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" From 5714bf4b96923afb2b8b87eafd781f34daedfde3 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 2 Oct 2026 07:49:46 -0700 Subject: [PATCH 0792/2680] tools: aurora-sep: release 11.21, Bluetooth recovery Kernel 7.1.12.aurora2-11.21 from custom/sep df7cb9bbd487: 11.20 plus hci_bcm4377 hardware reset recovery (aurora-silicon/linux#7, Justin Pfister). A Broadcom PCIe radio wedged by an rfkill cycle under heavy traffic is reset on the next open, and its calibration and address are restored. Other packages are unchanged. Boot-tested in maclab (m1air, job j1002-074408-6d57): 25 rfkill cycles under LE discovery with the address unchanged and discovery working, the 11.20 release checks, and no omarchy-m-test regressions against 11.19. The M1 Air's BCM4378 did not wedge on 11.19 either, so the recovery path itself is the author's M2 Pro (BCM4388) result. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- tools/aurora-sep/install-aurora-sep.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 7738a7818292c..1b607410d4eb6 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -5,7 +5,7 @@ # ... | bash -s -- --read-only install, but never let the driver write to the enclave # ... | bash -s -- --uninstall go back to the kernel this Mac had before # -# Kernel: iconidentify/aurora-linux custom/sep (77542ad1e56c), aurora-silicon/linux aurora-wip plus the +# Kernel: iconidentify/aurora-linux custom/sep (df7cb9bbd487), aurora-silicon/linux aurora-wip plus the # Secure Enclave (Touch ID) driver, Thunderbolt (#8), the Apple video # decoder (#45), the M2 Max (t6021) profile and the consolidated Touch ID # series (aurora-silicon/linux#69: matching after a reboot on every profile, @@ -20,6 +20,9 @@ # Omarchy boot logo in place of Tux, and aurora-wip through 3bb0a6104a11 (the # MacBook Neo video decoder, PCIe bring-up and radio drivers, inert elsewhere). # On a MacBook Neo the script keeps the Neo's own m1n1. +# 11.21 adds Bluetooth recovery for the Broadcom PCIe controllers: a radio +# wedged by an rfkill cycle under heavy traffic is reset on the next open, and +# its calibration and address restored (aurora-silicon/linux#7). # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -48,8 +51,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.20 -TAG=sep-7.1.12.aurora2-11.20 +VERSION=7.1.12.aurora2-11.21 +TAG=sep-7.1.12.aurora2-11.21 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -57,8 +60,8 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst c8f000aa2541256e78c3b53e7d66068559357644e75f259ddbc12538105eccb0" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst f1eb3174f6f9c1f0071446f36b39ef978466cb7d51b6cd622aa23c1208fca921" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst 0010d9572a4362d321e6431a32b1ad97a44afe70bd6c0771615d84090fa368ff" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 69669048a7f2cdd1d970a4c02f63b76601e5c050a831287927a8ce1266a31f63" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20261001-1-any.pkg.tar.zst bc6438d23dc2aae2766677f41f3f6d61ec3716df55c3b7a9c2cf93a027c68660" "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" From 6ccb1377defa55d0adadcfd04e1e37c9ac9e6357 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 2 Oct 2026 08:48:18 -0700 Subject: [PATCH 0793/2680] tools: aurora-sep: install the AVD VA-API bridge; aurora-touchid-setup --help The installer pulled in avd-fw but not libva-v4l2_request-avd, the VA-API driver for the decoder. On an install without it, mpv and browsers fall back to software decode with no error (reported on a 16" M1 Pro, j316s). Install it the same way, with a warning if pacman can't. aurora-touchid-setup started an enrolment for any argument other than --remove, including --help (iconidentify/aurora-linux#3). It now prints its usage for -h/--help and refuses unknown options. aurora-touchid 20261002-1. Reported-by: cad0p Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- tools/aurora-sep/aurora-touchid/PKGBUILD | 4 ++-- .../aurora-sep/aurora-touchid/aurora-touchid-setup | 14 ++++++++++---- tools/aurora-sep/install-aurora-sep.sh | 11 +++++++++-- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/tools/aurora-sep/aurora-touchid/PKGBUILD b/tools/aurora-sep/aurora-touchid/PKGBUILD index 07eaf688c92d7..3e638d7d1218d 100644 --- a/tools/aurora-sep/aurora-touchid/PKGBUILD +++ b/tools/aurora-sep/aurora-touchid/PKGBUILD @@ -1,6 +1,6 @@ # Maintainer: Chris Kearney <303316+iconidentify@users.noreply.github.com> pkgname=aurora-touchid -pkgver=20261001 +pkgver=20261002 pkgrel=1 pkgdesc="Touch ID for Omarchy on Apple Silicon (aurora kernel SEP driver)" arch=('any') @@ -13,7 +13,7 @@ source=('aurora-touchid-setup' 'apple-sep.path' 'fprintd-aurora.conf' 'aurora-touchid.sleep') -sha256sums=('ab4ed299d03462210e03e8d4152803a414204ebf25ec0f065b1b7dc3d43667b1' +sha256sums=('45639372c72c3dc519e815ea7162165c1b3116f1b08102960e67a83c58bd742a' '2e1d50b5274223e107e4aa4f73de194f3a49d347be1edc93efd3f81b31b092cc' '4f7b8a97e35e3511afa482ee8aedd019e960b1a67320675bd347f534feab49c3' '2f4b265a4c2180a9e0e4926c53ec77c3dbb074ddc3e083d7925d58db10a576b9' diff --git a/tools/aurora-sep/aurora-touchid/aurora-touchid-setup b/tools/aurora-sep/aurora-touchid/aurora-touchid-setup index 80074cecab11b..3869fad7fa273 100755 --- a/tools/aurora-sep/aurora-touchid/aurora-touchid-setup +++ b/tools/aurora-sep/aurora-touchid/aurora-touchid-setup @@ -55,10 +55,16 @@ remove_pam() { say "Fingerprint login removed. Enrolled prints stay in fprintd; delete them with: fprintd-delete $USER" } -if [[ ${1:-} == --remove ]]; then - remove_pam - exit 0 -fi +usage() { + sed -n '2,5s/^# \{0,1\}//p' "$0" +} + +case ${1:-} in + "") ;; + --remove) remove_pam; exit 0 ;; + -h|--help) usage; exit 0 ;; + *) usage >&2; die "unknown option $1" ;; +esac [[ -c /dev/sep-bio ]] || die "Touch ID is not available: /dev/sep-bio is missing. Reboot into the aurora kernel first, then check: systemctl status apple-sep.service" diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 1b607410d4eb6..f0d05f0a345ad 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -23,6 +23,9 @@ # 11.21 adds Bluetooth recovery for the Broadcom PCIe controllers: a radio # wedged by an rfkill cycle under heavy traffic is reset on the next open, and # its calibration and address restored (aurora-silicon/linux#7). +# 11.22 installs the VA-API bridge for the video decoder too, so players use +# the hardware decoder instead of falling back to software, and +# aurora-touchid-setup --help prints its usage instead of starting an enrolment. # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -52,7 +55,7 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. VERSION=7.1.12.aurora2-11.21 -TAG=sep-7.1.12.aurora2-11.21 +TAG=sep-7.1.12.aurora2-11.22 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -63,7 +66,7 @@ PACKAGES=( "linux-aurora-$VERSION-aarch64.pkg.tar.zst 0010d9572a4362d321e6431a32b1ad97a44afe70bd6c0771615d84090fa368ff" "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 69669048a7f2cdd1d970a4c02f63b76601e5c050a831287927a8ce1266a31f63" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" - "aurora-touchid-20261001-1-any.pkg.tar.zst bc6438d23dc2aae2766677f41f3f6d61ec3716df55c3b7a9c2cf93a027c68660" + "aurora-touchid-20261002-1-any.pkg.tar.zst 7a2da780f909fce8c6b2e7f378de8a7f96ffc01c0601940cbf48a65e833f70c4" "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" ) PINNED="linux-aurora linux-aurora-headers libfprint m1n1-aurora" @@ -465,6 +468,10 @@ install_all() { # installs never needed; without it the decoder fails to load at boot. $sudo pacman -S --needed --noconfirm avd-fw || warn "could not install avd-fw; hardware video decode will not work until it is installed" + # The VA-API bridge to that decoder. Without it, players fall back to + # software decode with no error (reported on a 16" M1 Pro). + $sudo pacman -S --needed --noconfirm libva-v4l2_request-avd || + warn "could not install libva-v4l2_request-avd; video players will decode in software until it is installed" add_pin # Both boot chains boot through m1n1, and both need the aurora device trees # in boot.bin: update-m1n1 otherwise takes the DTBs of the highest-versioned From 3271886e2b7491958cf9919a47df45c3054b420b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 2 Oct 2026 09:04:34 -0700 Subject: [PATCH 0794/2680] soc: apple: sep: refuse a reload on M1 instead of re-sending TZ0 On the cold-boot profiles (T8103, T6000) the SEP stays booted after rmmod and Linux cannot reset it, so a reload sent TZ0 into a running enclave, nothing answered, and /dev/sep-bio never came back, with no message (iconidentify/aurora-linux#4). Record the boot with the same device-tree marker the warm-registration path uses, before TZ0, so a reload now stops at probe and says to reboot. The marker lives only in the running kernel's device tree, so it lasts exactly as long as the SEP stays booted. Reported-by: cad0p Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- drivers/soc/apple/sep.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index f8cc570652dcb..e36a66168f386 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -894,7 +894,7 @@ impl SepData { fn attach(&self, sep_node: &dt::DtNode) -> Result<()> { match self.profile.bootstrap { profile::Bootstrap::WarmRegister => self.attach_warm(sep_node), - profile::Bootstrap::Boot => self.attach_m1(), + profile::Bootstrap::Boot => self.attach_m1(sep_node), } } @@ -932,7 +932,19 @@ impl SepData { /// Cold boot: send TZ0 and let the boot-endpoint acknowledgements drive the /// firmware and shared-memory handoff. - fn attach_m1(&self) -> Result<()> { + fn attach_m1(&self, sep_node: &dt::DtNode) -> Result<()> { + // The SEP stays booted after rmmod and Linux cannot reset it, so a + // reload's TZ0 would go unanswered. Record the boot first, as the warm + // path does, so a reload stops at probe and says to reboot instead. + let iova = self.shmem.lock().as_ref().map_or(0, |buf| buf.dma_handle()); + if let Err(e) = dt::mark_registration_sent(sep_node, iova) { + dev_err!( + self.dev, + "could not record the boot marker ({:?}); refusing to boot the SEP, since a reload could not tell it is already running\n", + e + ); + return Err(e); + } let msg = Message { msg0: u64::from(proto::EP_BOOT) | (proto::MSG_BOOT_TZ0 << proto::MSG_TYPE_SHIFT), msg1: 0, @@ -2282,7 +2294,7 @@ impl platform::Driver for SepDriver { if dt::registration_already_sent(&sep_node) { dev_err!( dev, - "the shared-memory registration was already sent on this boot (marker property present). It is one-shot per AP reset and cannot be repeated or withdrawn: reboot to attach again. Not probing.\n" + "the SEP was already booted or registered on this boot by an earlier load (marker property present). Linux cannot reset it and the handoff cannot be repeated, so reloading the driver is not supported: reboot to attach again. Not probing.\n" ); return Err(EBUSY); } From 4f08535b439db551967ab05095542463084a9a06 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 2 Oct 2026 09:04:34 -0700 Subject: [PATCH 0795/2680] tools: aurora-touchid: start the display manager after apple-drm On a 16" M2 Pro (j416s) the built-in screen stayed black on 11.17/11.18 while the session ran. apple-drm registered about 30 s into boot, after the desktop had started on the boot framebuffer (simpledrm), and the session was left without its screen; restarting the display manager brought it back (same symptom reported on a j416c in aurora-silicon/linux#46). An sddm drop-in now waits for a DRM card bound to apple-drm before the display manager starts. It only waits on a Mac whose device tree enables an Apple display subsystem, gives up after 30 s and never fails the start, so where apple-drm is already up (3.9 s on an M1 Air) it costs nothing. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- tools/aurora-sep/aurora-touchid/PKGBUILD | 10 ++++-- .../aurora-touchid/sddm-wait-display.conf | 3 ++ .../aurora-touchid/wait-for-display | 31 +++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 tools/aurora-sep/aurora-touchid/sddm-wait-display.conf create mode 100755 tools/aurora-sep/aurora-touchid/wait-for-display diff --git a/tools/aurora-sep/aurora-touchid/PKGBUILD b/tools/aurora-sep/aurora-touchid/PKGBUILD index 3e638d7d1218d..052669d917f5e 100644 --- a/tools/aurora-sep/aurora-touchid/PKGBUILD +++ b/tools/aurora-sep/aurora-touchid/PKGBUILD @@ -12,13 +12,17 @@ source=('aurora-touchid-setup' 'apple-sep.service' 'apple-sep.path' 'fprintd-aurora.conf' - 'aurora-touchid.sleep') + 'aurora-touchid.sleep' + 'wait-for-display' + 'sddm-wait-display.conf') sha256sums=('45639372c72c3dc519e815ea7162165c1b3116f1b08102960e67a83c58bd742a' '2e1d50b5274223e107e4aa4f73de194f3a49d347be1edc93efd3f81b31b092cc' '4f7b8a97e35e3511afa482ee8aedd019e960b1a67320675bd347f534feab49c3' '2f4b265a4c2180a9e0e4926c53ec77c3dbb074ddc3e083d7925d58db10a576b9' 'e1b1152e576e9588cc134b09291b3a2306ac686d9a47be6f128670608da535fc' - 'a87526383319cc7c1214082196d0d80ad9a6da09b5c30bf215f866e01443c611') + 'a87526383319cc7c1214082196d0d80ad9a6da09b5c30bf215f866e01443c611' + '062f70bb84e4139baf75d4dbefff21478af28f67e5569ee095692734d764746b' + '07edd06103f5d199e7d6dcca0b7fa8e10a912cb657973ffce764c6e2ef65fa40') package() { install -Dm755 aurora-touchid-setup "$pkgdir/usr/bin/aurora-touchid-setup" @@ -27,4 +31,6 @@ package() { install -Dm644 apple-sep.path "$pkgdir/usr/lib/systemd/system/apple-sep.path" install -Dm644 fprintd-aurora.conf "$pkgdir/usr/lib/systemd/system/fprintd.service.d/aurora.conf" install -Dm755 aurora-touchid.sleep "$pkgdir/usr/lib/systemd/system-sleep/aurora-touchid" + install -Dm755 wait-for-display "$pkgdir/usr/lib/aurora-touchid/wait-for-display" + install -Dm644 sddm-wait-display.conf "$pkgdir/usr/lib/systemd/system/sddm.service.d/50-aurora-wait-display.conf" } diff --git a/tools/aurora-sep/aurora-touchid/sddm-wait-display.conf b/tools/aurora-sep/aurora-touchid/sddm-wait-display.conf new file mode 100644 index 0000000000000..a64013e6ab8b4 --- /dev/null +++ b/tools/aurora-sep/aurora-touchid/sddm-wait-display.conf @@ -0,0 +1,3 @@ +[Service] +# aurora-touchid: start the display manager after the Apple display driver. +ExecStartPre=-/usr/lib/aurora-touchid/wait-for-display diff --git a/tools/aurora-sep/aurora-touchid/wait-for-display b/tools/aurora-sep/aurora-touchid/wait-for-display new file mode 100755 index 0000000000000..cfc78817cb1fc --- /dev/null +++ b/tools/aurora-sep/aurora-touchid/wait-for-display @@ -0,0 +1,31 @@ +#!/bin/sh +# Hold the display manager until the Apple display driver is up. +# +# On some Macs (seen on 16" M2 Pro/Max) apple-drm registers well after the +# desktop session has started on the boot framebuffer (simpledrm). When the +# boot framebuffer then goes away, the session is left without its screen +# and the built-in display stays black until the display manager restarts. +# Waiting for apple-drm first avoids the race. It only waits on a Mac whose +# device tree enables an Apple display subsystem, gives up after 30 s, and +# never fails the display manager's start. + +enabled=$(grep -rl --null-data -x 'apple,display-subsystem' /proc/device-tree 2>/dev/null | + while read -r node; do + dir=${node%/compatible} + status=okay + [ -e "$dir/status" ] && status=$(tr -d '\0' <"$dir/status") + [ "$status" = okay ] && echo yes + done) +[ -n "$enabled" ] || exit 0 + +i=0 +while [ "$i" -lt 60 ]; do + for card in /sys/class/drm/card[0-9]*; do + [ -e "$card/device/driver" ] || continue + [ "$(basename "$(readlink -f "$card/device/driver")")" = apple-drm ] && exit 0 + done + sleep 0.5 + i=$((i + 1)) +done +echo "aurora-touchid: apple-drm did not appear within 30 s; starting the display manager anyway" >&2 +exit 0 From dd5923d92fff7f38763594d486ace0192b54054a Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 2 Oct 2026 09:05:22 -0700 Subject: [PATCH 0796/2680] tools: aurora-sep: leave other builds of the VA-API bridge alone Install libva-v4l2_request-avd only when no build of the bridge is present. The AUR libva-v4l2_request packages conflict with it, and --noconfirm would answer no to removing them, failing the step. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- tools/aurora-sep/install-aurora-sep.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index f0d05f0a345ad..8eb43ef024f55 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -469,9 +469,14 @@ install_all() { $sudo pacman -S --needed --noconfirm avd-fw || warn "could not install avd-fw; hardware video decode will not work until it is installed" # The VA-API bridge to that decoder. Without it, players fall back to - # software decode with no error (reported on a 16" M1 Pro). - $sudo pacman -S --needed --noconfirm libva-v4l2_request-avd || - warn "could not install libva-v4l2_request-avd; video players will decode in software until it is installed" + # software decode with no error (reported on a 16" M1 Pro installed from the + # Omarchy Mac ISO). Leave any other build of the bridge alone: the AUR + # libva-v4l2_request packages conflict with it. + if ! pacman -Qq libva-v4l2_request-avd libva-v4l2_request >/dev/null 2>&1 && + [[ ! -e /usr/lib/dri/v4l2_request_drv_video.so ]]; then + $sudo pacman -S --needed --noconfirm libva-v4l2_request-avd || + warn "could not install libva-v4l2_request-avd; video players will decode in software until it is installed" + fi add_pin # Both boot chains boot through m1n1, and both need the aurora device trees # in boot.bin: update-m1n1 otherwise takes the DTBs of the highest-versioned From 02967d5575669918bd47caf078bcd18db160bf87 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 2 Oct 2026 09:23:23 -0700 Subject: [PATCH 0797/2680] tools: aurora-sep: release 11.22 Kernel 7.1.12.aurora2-11.22 from custom/sep 0cde04307749: 11.21 plus aurora-wip b3a55ee68bfb (M5/M6 RAM boot, gated to the new parts), the M1 reload refusal (iconidentify/aurora-linux#4) and comment-only changes in the SEP driver. aurora-touchid 20261002-1 adds the display-manager wait for apple-drm and fixes aurora-touchid-setup --help (#3). The installer adds the AVD VA-API bridge when no build of it is present. Boot-tested in maclab on the m1air: j1002-091436-a40c (release checks, display wait with the drop-in installed: apple-drm up at 7.3 s, sddm at 9.6 s; reload refused with EBUSY and no TZ0) and j1002-091853-6962 (clean run, no omarchy-m-test regressions against 11.19). Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- tools/aurora-sep/install-aurora-sep.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 8eb43ef024f55..e7294c34946f0 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -5,7 +5,7 @@ # ... | bash -s -- --read-only install, but never let the driver write to the enclave # ... | bash -s -- --uninstall go back to the kernel this Mac had before # -# Kernel: iconidentify/aurora-linux custom/sep (df7cb9bbd487), aurora-silicon/linux aurora-wip plus the +# Kernel: iconidentify/aurora-linux custom/sep (0cde04307749), aurora-silicon/linux aurora-wip plus the # Secure Enclave (Touch ID) driver, Thunderbolt (#8), the Apple video # decoder (#45), the M2 Max (t6021) profile and the consolidated Touch ID # series (aurora-silicon/linux#69: matching after a reboot on every profile, @@ -26,6 +26,9 @@ # 11.22 installs the VA-API bridge for the video decoder too, so players use # the hardware decoder instead of falling back to software, and # aurora-touchid-setup --help prints its usage instead of starting an enrolment. +# The display manager waits (up to 30 s) for the Apple display driver, so a +# late driver can't leave the built-in screen black, and reloading the Touch ID +# driver on M1 is refused with "reboot to re-attach" instead of hanging. # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -54,7 +57,7 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.21 +VERSION=7.1.12.aurora2-11.22 TAG=sep-7.1.12.aurora2-11.22 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. @@ -63,10 +66,10 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst 0010d9572a4362d321e6431a32b1ad97a44afe70bd6c0771615d84090fa368ff" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 69669048a7f2cdd1d970a4c02f63b76601e5c050a831287927a8ce1266a31f63" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst 7c185cd243eb56ff285809a4bc59e49805d5db7d5c73f3f7aac2f73d3d87a4f9" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst baf0dae8e7621c3a10ba8302d3891113ebb6e0cf43c603751b7c9fbf5b199d3a" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" - "aurora-touchid-20261002-1-any.pkg.tar.zst 7a2da780f909fce8c6b2e7f378de8a7f96ffc01c0601940cbf48a65e833f70c4" + "aurora-touchid-20261002-1-any.pkg.tar.zst a9dda6e0526874e4ac760629f3aa5bd37421379a1d8af34000f3dc7b73a21b17" "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" ) PINNED="linux-aurora linux-aurora-headers libfprint m1n1-aurora" From 54726742a470bc1774f3e1246089ddf4c6bdb8c2 Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 11:25:14 -0700 Subject: [PATCH 0798/2680] tools: aurora-sep: release 11.23 linux-aurora 7.1.12.aurora2-11.23: USB-C displays on the M2 Pro/Max MacBook Pros are routed in the order the compositor pairs them, so two monitors attached at boot each keep their own modes, and the Touch ID sensor is described on every M2 Pro/Max MacBook Pro. The agent prompt notes the expected T6020/J414s profile line on those machines, asks for system-fw-version and a 26.6.x update when keybag creation fails with -13, and adds a two-monitors-at-boot display check. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- tools/aurora-sep/install-aurora-sep.sh | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index e7294c34946f0..400399fc3da85 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -29,6 +29,9 @@ # The display manager waits (up to 30 s) for the Apple display driver, so a # late driver can't leave the built-in screen black, and reloading the Touch ID # driver on M1 is refused with "reboot to re-attach" instead of hanging. +# 11.23 routes USB-C displays on the M2 Pro/Max MacBook Pros in the order the +# compositor pairs them, so two monitors attached at boot each keep their own +# modes, and describes the Touch ID sensor on every M2 Pro/Max MacBook Pro. # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -57,8 +60,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.22 -TAG=sep-7.1.12.aurora2-11.22 +VERSION=7.1.12.aurora2-11.23 +TAG=sep-7.1.12.aurora2-11.23 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -66,8 +69,8 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst 7c185cd243eb56ff285809a4bc59e49805d5db7d5c73f3f7aac2f73d3d87a4f9" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst baf0dae8e7621c3a10ba8302d3891113ebb6e0cf43c603751b7c9fbf5b199d3a" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst 1542de460b3ece734194145c89f3a4fab7c75f063fea0e5312012cfd5f32336a" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst a2ac1c3e0682e32029ace86b5af4e5e4e5aba3e483a05a0752b28c97ae8c2590" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20261002-1-any.pkg.tar.zst a9dda6e0526874e4ac760629f3aa5bd37421379a1d8af34000f3dc7b73a21b17" "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" @@ -634,6 +637,14 @@ SAFETY, NON-NEGOTIABLE FAIL: "attach: 0 messages received but no endpoint advertised", or no apple_sep lines at all. Capture the whole block either way. If the profile line says a SoC you did not expect, report that verbatim. + On every MacBook Pro M2 Pro/Max (J414s, J414c, J416s, J416c) it reads + "T6020/J414s"; that is expected. + If "CREATE_KEYBAG" fails with status -13, quote your system-fw-version. + Every M2-family Mac that has created its keybag was on 26.6.x; on an older + one, update macOS to 26.6.x, boot back into Linux, re-run the installer and + report again. If it still fails on 26.6.x, that is the report we need. + The sensor itself: "sudo dmesg | grep apple-mesa" should say the power line + came "from the device node". Quote it if it says anything else. 4. IF ATTACH FAILED, GET THE MAILBOX STATE sudo busybox devmem 0x396408110 32 # A2I control (t600x/t602x) @@ -699,6 +710,10 @@ SAFETY, NON-NEGOTIABLE first: the ports are separate controllers and have behaved differently - two displays or a dual-output dock, if you have one + - M2 Pro/Max: two monitors plugged straight into two USB-C ports + (no dock), attached at boot. Does each come up at its own native + resolution? Quote "hyprctl monitors" (name, mode, and the port each + is on). Then log out and back in, and repeat. - PCIe behind the dock: ethernet, USB storage, card readers - do they enumerate and still work after a replug From 66b7c6061b762487446f1d06b8d6d668fbfd0ddd Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 12:35:27 -0700 Subject: [PATCH 0799/2680] tools: aurora-sep: release 11.24 linux-aurora 7.1.12.aurora2-11.24: Touch ID on the MacBook Pro 14"/16" M1 Max (the T6000 profile for t6001 and the j314c/j316c sensor) and the MacBook Air 13" M2 (j413), untested on both. On those three boards m1n1 sets up the enclave for the first time, so before pacman's update-m1n1 hook rebuilds boot.bin the script keeps the boot.bin the Mac booted with on the EFI partition, as m1n1/boot.bin.before-, and prints how to put it back from macOS if the Mac stops in m1n1. A later run keeps the first copy. Reports now go to iconidentify/aurora-linux issues instead of omacom/linux#7, and the agent prompt lists the expected profile lines for the new boards. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- tools/aurora-sep/install-aurora-sep.sh | 65 +++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 7 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 400399fc3da85..da97501abec4e 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -32,6 +32,9 @@ # 11.23 routes USB-C displays on the M2 Pro/Max MacBook Pros in the order the # compositor pairs them, so two monitors attached at boot each keep their own # modes, and describes the Touch ID sensor on every M2 Pro/Max MacBook Pro. +# 11.24 adds Touch ID on the MacBook Pro 14"/16" M1 Max and the MacBook Air 13" +# M2, untested on both. On those boards the script first keeps the boot.bin the +# Mac booted with on the EFI partition and prints how to restore it from macOS. # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -60,8 +63,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.23 -TAG=sep-7.1.12.aurora2-11.23 +VERSION=7.1.12.aurora2-11.24 +TAG=sep-7.1.12.aurora2-11.24 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -69,8 +72,8 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst 1542de460b3ece734194145c89f3a4fab7c75f063fea0e5312012cfd5f32336a" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst a2ac1c3e0682e32029ace86b5af4e5e4e5aba3e483a05a0752b28c97ae8c2590" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst 8447aac136a5814bb80d95e5394d63d1ee4702f8d88830393a952179c7e40183" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst d358e06b7238157e105ddce77ffd59bad8a709f31f041ab741f7d22dc811b494" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20261002-1-any.pkg.tar.zst a9dda6e0526874e4ac760629f3aa5bd37421379a1d8af34000f3dc7b73a21b17" "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" @@ -145,6 +148,48 @@ is_neo() { tr '\0' '\n' /dev/null | grep -qx 'apple,t8140' } +# Boards whose Touch ID support nobody has booted yet. Once a board's device +# tree names the enclave, m1n1 needs two boot manifests from the platform for +# it, and a board without them stops in m1n1 before any boot entry. Every +# board checked so far has them; these have not been checked. +UNPROVEN_SEP_BOARDS="j314c j316c j413" + +this_board() { + tr '\0' '\n' /dev/null | sed -n '1s/^apple,//p' +} + +# Before anything rebuilds boot.bin on such a board, keep the one this boot +# came up on next to it on the EFI partition, where macOS can reach it, and +# say how to put it back. A later run keeps the first copy. +unproven_board_backup() { + local board target keep uuid + board=$(this_board) + [[ -n $board && " $UNPROVEN_SEP_BOARDS " == *" $board "* ]] || return 0 + for target in /boot/efi/m1n1/boot.bin /boot/m1n1/boot.bin; do + [[ -f $target && $(findmnt -no FSTYPE --target "$target") == vfat ]] || continue + keep=$target.before-$VERSION + [[ -f $keep ]] || $sudo cp "$target" "$keep" || + die "could not keep a copy of $target; nothing was installed" + uuid=$(findmnt -no PARTUUID --target "$target") + warn "Touch ID on this Mac model ($board) is new in $VERSION, and nobody has + booted it on this model yet. If the Mac stops in m1n1 after this install (m1n1 + text on screen, often \"No valid payload found\", and no boot menu), put the + boot loader it booted with back from macOS: + 1. Hold the power button until the Mac turns off, then press and hold it + again for the startup options, and start macOS (or Options, then + Utilities > Terminal). + 2. In Terminal, run: diskutil list + Find the partition whose UUID is $uuid (any case) with: + diskutil info diskNsM | grep -i 'partition uuid' + 3. sudo diskutil mount diskNsM (it prints the /Volumes path) + 4. cp '/m1n1/boot.bin.before-$VERSION' '/m1n1/boot.bin' + The copy is kept at $keep. Either way, please report it at + https://github.com/iconidentify/aurora-linux/issues" + return 0 + done + die "could not find m1n1's boot.bin on the EFI partition to keep a copy of; nothing was installed" +} + current_kernel() { if pacman -Q linux-aurora >/dev/null 2>&1; then echo linux-aurora elif pacman -Q linux-asahi >/dev/null 2>&1; then echo linux-asahi @@ -453,6 +498,8 @@ install_all() { done snapshot "aurora-sep $VERSION" + # Before pacman's update-m1n1 hook rebuilds boot.bin below. + is_neo || unproven_board_backup $sudo install -d "$STATE" if [[ ! -f $STATE/previous && $chain == grub ]]; then keep_grub_fallback "$kernel" @@ -638,7 +685,9 @@ SAFETY, NON-NEGOTIABLE apple_sep lines at all. Capture the whole block either way. If the profile line says a SoC you did not expect, report that verbatim. On every MacBook Pro M2 Pro/Max (J414s, J414c, J416s, J416c) it reads - "T6020/J414s"; that is expected. + "T6020/J414s", on the MacBook Pro 14"/16" M1 Max (J314c, J316c) + "T6000/J316s", and on the MacBook Air 13" M2 (J413) "T8112/J415"; those are + expected. If "CREATE_KEYBAG" fails with status -13, quote your system-fw-version. Every M2-family Mac that has created its keybag was on 26.6.x; on an older one, update macOS to 26.6.x, boot back into Linux, re-run the installer and @@ -739,8 +788,10 @@ SAFETY, NON-NEGOTIABLE Report "not tested" honestly rather than guessing, for any of the above. HOW TO REPORT - Post one comment on the PR you were given (the SEP work lives on - omacom/linux#7). Structure it exactly like this: + Open one issue per Mac at https://github.com/iconidentify/aurora-linux/issues + (not on omacom/linux#7 any more), with the first line of the report as its + title. Post later results for the same Mac as comments on that issue. + Structure it exactly like this: # (): **Machine:** apple,jXXX / apple,tXXXX, From ad1b327413c2ec5b30c1ddd4998066f63767d011 Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 13:18:54 -0700 Subject: [PATCH 0800/2680] soc: apple: sep: expose passive diagnostics in sysfs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unattended test tools need to know whether the SEP attached, whether the key store opened and whether Touch ID is ready. Today the only source is dmesg, which such a tool may not be allowed to read (kernel.dmesg_restrict), and log lines are not an interface. Add a "diag" attribute group to the SEP platform device, /sys/bus/platform/devices/.sep/diag/, created at probe and removed at unbind through devres. Every attribute is read-only and reads as one ASCII token and a newline: abi interface version, "1" profile platform profile name, e.g. "T6020/J414s" boot cold | warm protocol sepos13 | variant5 xart enabled | disabled, the xART write mode in effect attach pending | attached | failed endpoints endpoints the SEP has advertised, "0" until attached keystore unknown | open | closed keybag unknown | present | missing | failed sensor unbound | bound | online | failed touchid unknown | ready | not-ready The driver records each value where it already makes the decision, into a static table of atomics in a new C shim; a machine has one SEP, so static state is enough. Reading an attribute only loads that cached state: it sends nothing to the SEP, takes no driver lock and starts no work, so a reader can neither stall nor disturb the bring-up. No value identifies a user, an identity or a key. Apart from a warning if the group cannot be added, existing behaviour, ordering and log output are unchanged. Touch ID is activated by the first open of /dev/sep-bio, so sensor and touchid stay at "unbound" and "unknown" until something opens it. The sensor bring-up that activation runs counts as the first bring-up. There is no attribute for the sepOS build, because the driver has no authoritative source for one; protocol comes from the platform profile, not from the SEP. Documentation/ABI/testing/sysfs-driver-apple-sep describes each attribute. Signed-off-by: Chris Kearney Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i [ryan: rebased onto the linear SEP history — this branch was merged into aurora-wip in 45da41c0cd1b, whose result is followed here: the sensor_shim.c bound/unbound hooks are left out (the merge kept the mainline sensor_shim.c), the sensor-bring-up report sits after the J700-aware completion in bring_sensor_online(), and the calibration reload path reports nothing] Signed-off-by: Ryan Murray --- .../ABI/testing/sysfs-driver-apple-sep | 188 ++++++++++ drivers/soc/apple/Makefile | 2 +- drivers/soc/apple/diag.rs | 112 ++++++ drivers/soc/apple/diag_shim.c | 320 ++++++++++++++++++ drivers/soc/apple/sbio.rs | 19 +- drivers/soc/apple/sep.rs | 13 + drivers/soc/apple/shim.h | 22 ++ 7 files changed, 674 insertions(+), 2 deletions(-) create mode 100644 Documentation/ABI/testing/sysfs-driver-apple-sep create mode 100644 drivers/soc/apple/diag.rs create mode 100644 drivers/soc/apple/diag_shim.c diff --git a/Documentation/ABI/testing/sysfs-driver-apple-sep b/Documentation/ABI/testing/sysfs-driver-apple-sep new file mode 100644 index 0000000000000..511080fc16880 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-apple-sep @@ -0,0 +1,188 @@ +What: /sys/bus/platform/devices/.sep/diag/abi +Date: October 2026 +KernelVersion: 7.1 +Contact: Chris Kearney +Description: + (RO) Version of the diag interface: "1" for the attributes + and values described in this file. + + The diag directory holds passive diagnostics of the Apple + SEP (Secure Enclave Processor) driver, apple_sep, for + unattended test tools that need to know how far the + bring-up got without reading the kernel log. It exists while + the driver is bound to the SEP: it is created at probe and + removed at unbind. Every attribute in it is read-only and + reads as one ASCII token followed by a newline. + + Each value is the state the driver recorded when it made the + corresponding decision. Reading an attribute sends nothing + to the SEP, waits for nothing and starts nothing. No value + identifies a user, an enrolled finger or a key. + + Touch ID is activated by the first open of /dev/sep-bio, not + by the boot-time bring-up, so the sensor and touchid + attributes only move on once something has opened it. + +What: /sys/bus/platform/devices/.sep/diag/profile +Date: October 2026 +KernelVersion: 7.1 +Contact: Chris Kearney +Description: + (RO) The platform profile the driver selected for this + machine, as SoC and board, for example "T6020/J414s". Fixed + at probe. + +What: /sys/bus/platform/devices/.sep/diag/boot +Date: October 2026 +KernelVersion: 7.1 +Contact: Chris Kearney +Description: + (RO) How the driver hands its shared memory to the SEP, as + the platform profile sets it. Fixed at probe. + + ==== ===================================================== + cold The driver boots the SEP and hands the memory over + during that boot. + warm The boot firmware left the SEP running, and the + driver registers the memory with it in one message. + ==== ===================================================== + +What: /sys/bus/platform/devices/.sep/diag/protocol +Date: October 2026 +KernelVersion: 7.1 +Contact: Chris Kearney +Description: + (RO) The key-store protocol the driver speaks to the SEP, as + the platform profile sets it. It is not read from the SEP, + and it does not identify a sepOS build. Fixed at probe. + + ======== ============================================= + sepos13 The sepOS 13.5 key store, whose endpoint the + driver initialises before first use. + variant5 The newer key store, which creates keybags + with the variant-5 request. + ======== ============================================= + +What: /sys/bus/platform/devices/.sep/diag/xart +Date: October 2026 +KernelVersion: 7.1 +Contact: Chris Kearney +Description: + (RO) Whether the SEP's writes to the shared xART store reach + the disk. Fixed at probe, when the driver opens the store. + + ======== ============================================= + enabled Writes reach the store (xart_writes=1, and + the store validated for writing). + disabled Writes are refused. Touch ID enrol and verify + are refused too, since the SEP must save + state after each. + ======== ============================================= + +What: /sys/bus/platform/devices/.sep/diag/attach +Date: October 2026 +KernelVersion: 7.1 +Contact: Chris Kearney +Description: + (RO) Whether the SEP answered the driver's attach. + + ======== ============================================= + pending No outcome yet. It stays so if the SEP never + answers at all. + attached The SEP advertised at least one endpoint. + failed The SEP advertised no endpoint; the xART + exchange and the key store cannot run this + boot. + ======== ============================================= + +What: /sys/bus/platform/devices/.sep/diag/endpoints +Date: October 2026 +KernelVersion: 7.1 +Contact: Chris Kearney +Description: + (RO) The number of distinct endpoints the SEP has + advertised, as an unsigned decimal. It reads "0" until + attach reads "attached". Discovery goes on after the attach, + so the number can still grow afterwards, and can exceed the + count the attach message in the kernel log reports. + +What: /sys/bus/platform/devices/.sep/diag/keystore +Date: October 2026 +KernelVersion: 7.1 +Contact: Chris Kearney +Description: + (RO) Whether the boot-time bring-up opened the key store. + + ======= ============================================== + unknown The bring-up has not decided yet. + open The key-store endpoint is enabled and, where the + protocol needs it, initialised. + closed The bring-up ended without opening it. Keybag and + Touch ID operations are unavailable this boot. + ======= ============================================== + +What: /sys/bus/platform/devices/.sep/diag/keybag +Date: October 2026 +KernelVersion: 7.1 +Contact: Chris Kearney +Description: + (RO) The identity keybag that Touch ID uses. With + provision_keybag=1, the default, the boot-time bring-up + decides it once the key store is open; otherwise it stays + "unknown" until Touch ID is activated, which loads the + keybag and updates this value. It also stays "unknown" if + the key store never opens. + + ======= ============================================== + unknown Not determined yet. + present An identity keybag is usable: one was already + persisted or was provisioned during the bring-up, + or Touch ID activation loaded it. + missing None is persisted and none was provisioned. + failed Provisioning, reading or loading it failed. + ======= ============================================== + +What: /sys/bus/platform/devices/.sep/diag/sensor +Date: October 2026 +KernelVersion: 7.1 +Contact: Chris Kearney +Description: + (RO) The fingerprint sensor on its SPI bus. The driver + creates the sensor device when Touch ID is activated, and + brings the sensor up then and again before each enrol or + verify capture. + + ======= ============================================== + unbound No SPI sensor device is bound. + bound The sensor is bound and has not been brought up + since it bound. + online The most recent bring-up succeeded. + failed The most recent bring-up failed. + ======= ============================================== + +What: /sys/bus/platform/devices/.sep/diag/touchid +Date: October 2026 +KernelVersion: 7.1 +Contact: Chris Kearney +Description: + (RO) Whether Touch ID is ready to enrol and verify. + + ========= ============================================ + unknown Still in progress: the boot-time bring-up has + not finished, or /dev/sep-bio is published + but has not been opened yet, or activation + is still running. + ready /dev/sep-bio is published, Touch ID was + activated, and the most recent sensor + bring-up succeeded. + not-ready Finished without that: the bring-up ended + without publishing /dev/sep-bio, activation + failed (it is not retried this boot), or + the most recent sensor bring-up did not + succeed. + ========= ============================================ + + Because the first open of /dev/sep-bio activates Touch ID, a + tool waiting for "ready" has to open it, or let its client + do so, first. Enrol and verify also need xart to read + "enabled". diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index e6cb2e917bc0a..4b739f3e06c76 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -27,7 +27,7 @@ obj-$(CONFIG_APPLE_SEP_FV_INTERFACE) += fv_shim.o obj-$(CONFIG_APPLE_SEP) += apple-sep.o apple-sep-y := sep.o hwrng_shim.o store_shim.o bio_shim.o sha_shim.o \ crypto_shim.o sensor_shim.o p256_shim.o trusted_shim.o work_shim.o \ - pm_shim.o + pm_shim.o diag_shim.o obj-$(CONFIG_APPLE_PMP) += pmp.o diff --git a/drivers/soc/apple/diag.rs b/drivers/soc/apple/diag.rs new file mode 100644 index 0000000000000..b01dd8227dd62 --- /dev/null +++ b/drivers/soc/apple/diag.rs @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Rust side of `diag_shim.c`: the passive diagnostics in sysfs. +//! +//! The driver reports each decision here as it makes it, and the shim caches +//! it for the `diag` attributes. Every call only stores a word, so none of them +//! can block a bring-up step, and a sysfs read never reaches back into the +//! driver or the SEP. + +use kernel::{ + device, + prelude::*, // +}; + +use crate::profile; + +extern "C" { + // `struct device *`, passed untyped: its lockdep map is a zero-sized + // struct in some configurations, which Rust does not accept across FFI. + fn sep_diag_register( + dev: *mut c_void, + profile: *const u8, + profile_len: usize, + cold: bool, + sepos13: bool, + xart: bool, + ) -> c_int; + fn sep_diag_set_attach(attached: bool); + fn sep_diag_set_endpoints(count: c_uint); + fn sep_diag_set_keystore_open(); + fn sep_diag_set_keybag(state: c_int); + fn sep_diag_set_sensor_result(online: bool); + fn sep_diag_set_bio_published(); + fn sep_diag_set_touchid(started: bool); + fn sep_diag_bringup_ended(); +} + +/// The identity keybag, as `SEP_DIAG_KEYBAG_*` in `shim.h` spells it. +#[derive(Clone, Copy)] +pub(crate) enum Keybag { + Present = 1, + Missing = 2, + Failed = 3, +} + +/// Adds the `diag` group to the SEP device, with the facts fixed at probe. The +/// group goes away when the driver unbinds. +pub(crate) fn register( + dev: &device::Device, + profile: &profile::PlatformProfile, + xart_writable: bool, +) -> Result { + let cold = profile.bootstrap == profile::Bootstrap::Boot; + let sepos13 = matches!(profile.key_store, profile::KeyStore::Sepos13 { .. }); + // SAFETY: `dev` is bound for the duration of the call, which is what devres + // needs; the shim copies the name and keeps no pointer to it. + let ret = unsafe { + sep_diag_register( + dev.as_raw().cast(), + profile.name.as_ptr(), + profile.name.len(), + cold, + sepos13, + xart_writable, + ) + }; + kernel::error::to_result(ret) +} + +pub(crate) fn attach(attached: bool) { + // SAFETY: no preconditions; stores one word. + unsafe { sep_diag_set_attach(attached) }; +} + +pub(crate) fn endpoints(count: usize) { + let count = c_uint::try_from(count).unwrap_or(c_uint::MAX); + // SAFETY: no preconditions; stores one word. + unsafe { sep_diag_set_endpoints(count) }; +} + +pub(crate) fn keystore_open() { + // SAFETY: no preconditions; stores one word. + unsafe { sep_diag_set_keystore_open() }; +} + +pub(crate) fn keybag(state: Keybag) { + // SAFETY: no preconditions; stores one word. + unsafe { sep_diag_set_keybag(state as c_int) }; +} + +/// The result of a sensor bring-up, attach-time or later. +pub(crate) fn sensor_brought_up(online: bool) { + // SAFETY: no preconditions; stores one word. + unsafe { sep_diag_set_sensor_result(online) }; +} + +pub(crate) fn bio_published() { + // SAFETY: no preconditions; updates one word. + unsafe { sep_diag_set_bio_published() }; +} + +/// Touch ID activation, run by the first open of `/dev/sep-bio`, finished. +pub(crate) fn touchid(started: bool) { + // SAFETY: no preconditions; stores one word. + unsafe { sep_diag_set_touchid(started) }; +} + +/// The boot-time bring-up is over, however far it got. +pub(crate) fn bringup_ended() { + // SAFETY: no preconditions; updates two words. + unsafe { sep_diag_bringup_ended() }; +} diff --git a/drivers/soc/apple/diag_shim.c b/drivers/soc/apple/diag_shim.c new file mode 100644 index 0000000000000..03970fe9d75ac --- /dev/null +++ b/drivers/soc/apple/diag_shim.c @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Passive diagnostics for the SEP driver: the "diag" attribute group on the SEP + * platform device. Unattended test tools use it to see how far the bring-up + * got without reading the kernel log, which may be restricted. + * + * Each attribute shows state that the driver caches here at the point where it + * makes the decision. Reading one only loads that cached state: it sends + * nothing to the SEP, takes no driver lock and starts no work, so a reader can + * neither stall nor disturb the bring-up. Nothing here identifies a user, an + * identity or a key. + * + * A machine has one SEP and the driver attaches to it at most once per boot, + * so the state is static rather than per device. Every field starts at zero, + * the state before the driver has decided anything. + */ +#include +#include +#include +#include +#include + +#include "shim.h" + +/* The contract version; see Documentation/ABI/testing/sysfs-driver-apple-sep. */ +#define SEP_DIAG_ABI 1 + +#define SEP_DIAG_ATTACH_PENDING 0 +#define SEP_DIAG_ATTACH_ATTACHED 1 +#define SEP_DIAG_ATTACH_FAILED 2 + +#define SEP_DIAG_KEYSTORE_UNKNOWN 0 +#define SEP_DIAG_KEYSTORE_OPEN 1 +#define SEP_DIAG_KEYSTORE_CLOSED 2 + +/* The last bring-up of the bound sensor; reset when a sensor binds. */ +#define SEP_DIAG_SENSOR_UNTRIED 0 +#define SEP_DIAG_SENSOR_ONLINE 1 +#define SEP_DIAG_SENSOR_FAILED 2 + +/* + * Touch ID is activated by the first open of /dev/sep-bio, not by the boot-time + * bring-up, so publishing the node leaves the outcome undecided until then. + */ +#define SEP_DIAG_TOUCHID_PENDING 0 +#define SEP_DIAG_TOUCHID_PUBLISHED 1 +#define SEP_DIAG_TOUCHID_UNAVAILABLE 2 +#define SEP_DIAG_TOUCHID_STARTED 3 +#define SEP_DIAG_TOUCHID_FAILED 4 + +static struct { + /* Set at probe, before the group is added. */ + char profile[32]; + bool cold; + bool sepos13; + bool xart; + + atomic_t attach; + atomic_t endpoints; + atomic_t keystore; + atomic_t keybag; + atomic_t sensor_bound; + atomic_t sensor_result; + atomic_t touchid; +} sep_diag; + +static bool sep_diag_sensor_online(void) +{ + return atomic_read(&sep_diag.sensor_bound) && + atomic_read(&sep_diag.sensor_result) == SEP_DIAG_SENSOR_ONLINE; +} + +static ssize_t abi_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%d\n", SEP_DIAG_ABI); +} +static DEVICE_ATTR_RO(abi); + +static ssize_t profile_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%s\n", sep_diag.profile); +} +static DEVICE_ATTR_RO(profile); + +static ssize_t boot_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%s\n", sep_diag.cold ? "cold" : "warm"); +} +static DEVICE_ATTR_RO(boot); + +static ssize_t protocol_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%s\n", sep_diag.sepos13 ? "sepos13" : "variant5"); +} +static DEVICE_ATTR_RO(protocol); + +static ssize_t xart_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%s\n", sep_diag.xart ? "enabled" : "disabled"); +} +static DEVICE_ATTR_RO(xart); + +static ssize_t attach_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + const char *state; + + switch (atomic_read(&sep_diag.attach)) { + case SEP_DIAG_ATTACH_ATTACHED: + state = "attached"; + break; + case SEP_DIAG_ATTACH_FAILED: + state = "failed"; + break; + default: + state = "pending"; + break; + } + return sysfs_emit(buf, "%s\n", state); +} +static DEVICE_ATTR_RO(attach); + +/* Discovery goes on after the attach, so the count can still grow. */ +static ssize_t endpoints_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + unsigned int n = 0; + + if (atomic_read(&sep_diag.attach) == SEP_DIAG_ATTACH_ATTACHED) + n = atomic_read(&sep_diag.endpoints); + return sysfs_emit(buf, "%u\n", n); +} +static DEVICE_ATTR_RO(endpoints); + +static ssize_t keystore_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + const char *state; + + switch (atomic_read(&sep_diag.keystore)) { + case SEP_DIAG_KEYSTORE_OPEN: + state = "open"; + break; + case SEP_DIAG_KEYSTORE_CLOSED: + state = "closed"; + break; + default: + state = "unknown"; + break; + } + return sysfs_emit(buf, "%s\n", state); +} +static DEVICE_ATTR_RO(keystore); + +static ssize_t keybag_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + const char *state; + + switch (atomic_read(&sep_diag.keybag)) { + case SEP_DIAG_KEYBAG_PRESENT: + state = "present"; + break; + case SEP_DIAG_KEYBAG_MISSING: + state = "missing"; + break; + case SEP_DIAG_KEYBAG_FAILED: + state = "failed"; + break; + default: + state = "unknown"; + break; + } + return sysfs_emit(buf, "%s\n", state); +} +static DEVICE_ATTR_RO(keybag); + +static ssize_t sensor_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + int result = atomic_read(&sep_diag.sensor_result); + const char *state; + + if (!atomic_read(&sep_diag.sensor_bound)) + state = "unbound"; + else if (result == SEP_DIAG_SENSOR_ONLINE) + state = "online"; + else if (result == SEP_DIAG_SENSOR_FAILED) + state = "failed"; + else + state = "bound"; + return sysfs_emit(buf, "%s\n", state); +} +static DEVICE_ATTR_RO(sensor); + +static ssize_t touchid_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + const char *state; + + switch (atomic_read(&sep_diag.touchid)) { + case SEP_DIAG_TOUCHID_STARTED: + state = sep_diag_sensor_online() ? "ready" : "not-ready"; + break; + case SEP_DIAG_TOUCHID_UNAVAILABLE: + case SEP_DIAG_TOUCHID_FAILED: + state = "not-ready"; + break; + default: + state = "unknown"; + break; + } + return sysfs_emit(buf, "%s\n", state); +} +static DEVICE_ATTR_RO(touchid); + +static struct attribute *sep_diag_attrs[] = { + &dev_attr_abi.attr, + &dev_attr_profile.attr, + &dev_attr_boot.attr, + &dev_attr_protocol.attr, + &dev_attr_xart.attr, + &dev_attr_attach.attr, + &dev_attr_endpoints.attr, + &dev_attr_keystore.attr, + &dev_attr_keybag.attr, + &dev_attr_sensor.attr, + &dev_attr_touchid.attr, + NULL +}; + +static const struct attribute_group sep_diag_group = { + .name = "diag", + .attrs = sep_diag_attrs, +}; + +/* + * The fixed facts are stored before the group is added, so no reader sees them + * unset. devres removes the group when the driver unbinds. + */ +int sep_diag_register(struct device *dev, const char *profile, + size_t profile_len, bool cold, bool sepos13, bool xart) +{ + profile_len = min(profile_len, sizeof(sep_diag.profile) - 1); + memcpy(sep_diag.profile, profile, profile_len); + sep_diag.profile[profile_len] = '\0'; + sep_diag.cold = cold; + sep_diag.sepos13 = sepos13; + sep_diag.xart = xart; + + return devm_device_add_group(dev, &sep_diag_group); +} + +void sep_diag_set_attach(bool attached) +{ + atomic_set(&sep_diag.attach, + attached ? SEP_DIAG_ATTACH_ATTACHED : SEP_DIAG_ATTACH_FAILED); +} + +void sep_diag_set_endpoints(unsigned int count) +{ + atomic_set(&sep_diag.endpoints, count); +} + +void sep_diag_set_keystore_open(void) +{ + atomic_set(&sep_diag.keystore, SEP_DIAG_KEYSTORE_OPEN); +} + +void sep_diag_set_keybag(int state) +{ + atomic_set(&sep_diag.keybag, state); +} + +void sep_diag_set_sensor_bound(bool bound) +{ + if (bound) + atomic_set(&sep_diag.sensor_result, SEP_DIAG_SENSOR_UNTRIED); + atomic_set(&sep_diag.sensor_bound, bound); +} + +void sep_diag_set_sensor_result(bool online) +{ + atomic_set(&sep_diag.sensor_result, + online ? SEP_DIAG_SENSOR_ONLINE : SEP_DIAG_SENSOR_FAILED); +} + +/* + * A compare-exchange, not a store: an open of the new node can activate Touch + * ID before the publisher gets here, and that outcome must stand. + */ +void sep_diag_set_bio_published(void) +{ + atomic_cmpxchg(&sep_diag.touchid, SEP_DIAG_TOUCHID_PENDING, + SEP_DIAG_TOUCHID_PUBLISHED); +} + +void sep_diag_set_touchid(bool started) +{ + atomic_set(&sep_diag.touchid, + started ? SEP_DIAG_TOUCHID_STARTED : SEP_DIAG_TOUCHID_FAILED); +} + +/* + * Only the boot-time bring-up opens the key store and publishes /dev/sep-bio, + * and it is not repeated. Whatever it left undecided is final for this boot: + * the key store stays closed and Touch ID cannot start. + */ +void sep_diag_bringup_ended(void) +{ + atomic_cmpxchg(&sep_diag.keystore, SEP_DIAG_KEYSTORE_UNKNOWN, + SEP_DIAG_KEYSTORE_CLOSED); + atomic_cmpxchg(&sep_diag.touchid, SEP_DIAG_TOUCHID_PENDING, + SEP_DIAG_TOUCHID_UNAVAILABLE); +} diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 9d626bb275a6d..afc52918b4508 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1090,9 +1090,12 @@ impl SepData { self.dev, "sbio: the sensor did not wake at attach; the next verify retries the bring-up\n" ); + diag::sensor_brought_up(false); return; }; - if !self.complete_bringup(patch) { + let online = self.complete_bringup(patch); + diag::sensor_brought_up(online); + if !online { dev_warn!( self.dev, "sbio: attach-time sensor bring-up did not complete; matching remains unavailable until a later bring-up succeeds.\n" @@ -1102,6 +1105,7 @@ impl SepData { fn bring_sensor_online(&self) -> bool { let Some(mut patch) = self.wake_sensor() else { + diag::sensor_brought_up(false); return false; }; @@ -1118,6 +1122,7 @@ impl SepData { if !self.sensor_calibrated.load(Relaxed) { if !self.calibrate_sensor() { + diag::sensor_brought_up(false); return false; } self.sensor_calibrated.store(true, Relaxed); @@ -1133,6 +1138,7 @@ impl SepData { Some(parameters) => self.finish_sensor_setup(patch, parameters), None => self.complete_bringup(patch), }; + diag::sensor_brought_up(ok); // The sensor is patched and idle here -- the only safe moment to // configure the data-ready interrupt, which is then left alone for the // driver's life. Interrupt capture is the default; a machine that does @@ -1165,6 +1171,7 @@ impl SepData { ); return; } + diag::keystore_open(); if *module_parameters::provision_keybag.value() != 0 { if *module_parameters::xart_writes.value() == 0 { dev_err!( @@ -1178,8 +1185,11 @@ impl SepData { self.dev, "bringup: identity-keybag provisioning failed; no retry this boot\n" ); + diag::keybag(diag::Keybag::Failed); return; } + // Either one was already persisted or one has just been. + diag::keybag(diag::Keybag::Present); } if let Err(e) = self.register_bio() { dev_err!( @@ -1228,6 +1238,7 @@ impl SepData { Ok(keybag::State::Present(stored)) => stored, Ok(keybag::State::Absent(_)) => { dev_err!(self.dev, "Touch ID: no persisted identity keybag exists\n"); + diag::keybag(diag::Keybag::Missing); return Err(ENOENT); } Err(e) => { @@ -1236,6 +1247,7 @@ impl SepData { "Touch ID: reading the persisted identity keybag failed: {:?}\n", e ); + diag::keybag(diag::Keybag::Failed); return Err(e); } }; @@ -1251,8 +1263,10 @@ impl SepData { self.dev, "Touch ID: the persisted identity keybag did not recover\n" ); + diag::keybag(diag::Keybag::Failed); EIO })?; + diag::keybag(diag::Keybag::Present); self.sks_designate_user_keybag(handle, stored.secret()); self.sks_machine_refkey(handle, stored.secret()); let prepared = self.cold_match_continue(handle, uuid, stored.uuid_provenance()); @@ -1261,6 +1275,7 @@ impl SepData { self.probe_owner_export(); self.refresh_match_credential(); self.touchid_started.store(true, Relaxed); + diag::touchid(true); Ok(()) } @@ -1288,6 +1303,7 @@ impl SepData { fn prepare_bio_open(&self) -> Result<()> { if let Err(e) = self.activate_touchid() { self.touchid_failed.store(true, Relaxed); + diag::touchid(false); dev_err!( self.dev, "Touch ID activation failed: {:?}; refusing retries this boot because a partial keybag load is not safely repeatable\n", @@ -2711,6 +2727,7 @@ impl SepData { }?; *guard = Some(dev); + diag::bio_published(); Ok(()) } diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index e36a66168f386..659df59548859 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -13,6 +13,7 @@ mod bio; mod catacomb; mod control; mod der; +mod diag; mod dt; mod fv; mod hwrng; @@ -1908,6 +1909,8 @@ impl SepData { "attach: {} messages received but no endpoint advertised; xART and key store cannot run\n", now ); + diag::attach(false); + diag::bringup_ended(); this.phase.store(PHASE_READY, Relaxed); return; } @@ -1917,6 +1920,7 @@ impl SepData { this.endpoint_count(), now ); + diag::attach(true); this.prepare_os_uuid(); @@ -1928,6 +1932,7 @@ impl SepData { "xarm: out-of-line buffer registration failed ({:?}); the persistent-state exchange cannot run\n", e ); + diag::bringup_ended(); this.phase.store(PHASE_READY, Relaxed); return; } @@ -1968,6 +1973,7 @@ impl SepData { } this.run_bringup(); + diag::bringup_ended(); this.phase.store(PHASE_READY, Relaxed); Self::arm_settle(this); } @@ -2026,6 +2032,7 @@ impl SepData { match f.ty { proto::DISCOVER_TYPE_DESCRIPTOR | proto::DISCOVER_TYPE_CONFIG => { let _ = table.slot(f.param); + diag::endpoints(table.eps.len()); dev_info!( self.dev, "discover: endpoint {:#04x} (type {}, msg0 {:#018x}); {} known\n", @@ -2306,6 +2313,12 @@ impl platform::Driver for SepDriver { let data = SepData::new(pdev)?; + // As soon as the fixed facts are known, so a tool can watch the attach + // from "pending" on. Diagnostics are not worth failing the probe for. + if let Err(e) = diag::register(dev, data.profile, data.xart_writable()) { + dev_warn!(dev, "diag: could not add the sysfs attributes ({:?})\n", e); + } + *data.mbox.lock() = Some(Mailbox::new_byname(dev, c"mbox", data.clone())?); data.attach(&sep_node)?; diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index e7711068cb4ef..93d9db41e99f9 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -146,6 +146,28 @@ int sep_sensor_irq_available(void); void sep_sensor_irq_arm(void); int sep_sensor_irq_wait(unsigned int timeout_ms); +/* -- diag_shim.c -------------------------------------------------------- */ + +/* Mirrored by diag.rs. */ +#define SEP_DIAG_KEYBAG_UNKNOWN 0 +#define SEP_DIAG_KEYBAG_PRESENT 1 +#define SEP_DIAG_KEYBAG_MISSING 2 +#define SEP_DIAG_KEYBAG_FAILED 3 + +struct device; + +int sep_diag_register(struct device *dev, const char *profile, + size_t profile_len, bool cold, bool sepos13, bool xart); +void sep_diag_set_attach(bool attached); +void sep_diag_set_endpoints(unsigned int count); +void sep_diag_set_keystore_open(void); +void sep_diag_set_keybag(int state); +void sep_diag_set_sensor_bound(bool bound); +void sep_diag_set_sensor_result(bool online); +void sep_diag_set_bio_published(void); +void sep_diag_set_touchid(bool started); +void sep_diag_bringup_ended(void); + /* -- bio_shim.c --------------------------------------------------------- */ void *sep_bio_register(const char *name, unsigned short mode, void *ctx, From 0392e379611580c479ddc223b3241d36820462ff Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 13:35:47 -0700 Subject: [PATCH 0801/2680] soc: apple: sep: diag: report final Touch ID failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit touchid read "not-ready" both when the sensor had not come online yet and when Touch ID could never work this boot: activation failed at the host store, the identity index, the biometric buffers or the key store, or the bring-up ended without publishing /dev/sep-bio. With the other attributes reading attached, open, present and bound, a test tool took that for "skip", so a broken Touch ID went unreported. Add a fourth value, "failed", for the two outcomes that are final for the boot. "not-ready" now only means activated with the sensor not online, which a later sensor bring-up can still fix, and "unknown" is unchanged. Adding a value keeps abi at "1". "failed" can be intentional: with provision_keybag=0, as on a read-only install, /dev/sep-bio is still published with no identity keybag, and its first open fails activation with keybag "missing". The kernel reports that as "failed"; the ABI document tells readers to take failed together with missing as not provisioned rather than broken. Keep the sensor state and a count of sensor binds in one word. A reader racing a re-bind of the sensor can then no longer pair the new bind with the previous device's "online", and a bring-up that began before the sensor was unbound or bound again has its result dropped instead of reported against the device bound since. In the ABI document, say per boot mode when attach can stay "pending": on a warm boot the driver reports "failed" about three seconds after probe if the SEP never answers, and only a cold boot whose boot request goes unanswered stays pending. Say that keybag "present" means the keybag exists, not that it loads, and when keybag stays "unknown". Match the profile's bootstrap and key store exhaustively, so a new variant fails to build until the ABI has a value for it, and pass the profile name as a c_char pointer. Signed-off-by: Chris Kearney Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i [ryan: rebased onto the linear SEP history — this commit comes from a branch that aurora-wip merged in commit 45da41c0cd1b ("Merge the running hotplug and mixed-ABI kernel into aurora-wip"); where that branch and the mainline had both changed the same code, the merge kept the mainline version, so only the parts of this commit that the merge kept are carried here] [ryan: the bring_sensor_online() report that the previous commit placed after the J700-aware completion takes the bind token too] Signed-off-by: Ryan Murray --- .../ABI/testing/sysfs-driver-apple-sep | 56 ++++++++---- drivers/soc/apple/diag.rs | 40 ++++++-- drivers/soc/apple/diag_shim.c | 91 ++++++++++++++----- drivers/soc/apple/sbio.rs | 12 ++- drivers/soc/apple/shim.h | 3 +- 5 files changed, 148 insertions(+), 54 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-driver-apple-sep b/Documentation/ABI/testing/sysfs-driver-apple-sep index 511080fc16880..8c6fd3b877c12 100644 --- a/Documentation/ABI/testing/sysfs-driver-apple-sep +++ b/Documentation/ABI/testing/sysfs-driver-apple-sep @@ -87,14 +87,20 @@ Description: (RO) Whether the SEP answered the driver's attach. ======== ============================================= - pending No outcome yet. It stays so if the SEP never - answers at all. + pending No outcome yet. attached The SEP advertised at least one endpoint. failed The SEP advertised no endpoint; the xART exchange and the key store cannot run this boot. ======== ============================================= + The driver decides once about three seconds have passed + without an endpoint and the SEP has gone quiet. With boot + "warm" that wait starts at probe, so a SEP that never answers + ends in "failed". With boot "cold" it starts at the first + message from the SEP, so a SEP that never answers the boot + request leaves this at "pending" for the rest of the boot. + What: /sys/bus/platform/devices/.sep/diag/endpoints Date: October 2026 KernelVersion: 7.1 @@ -126,22 +132,25 @@ Date: October 2026 KernelVersion: 7.1 Contact: Chris Kearney Description: - (RO) The identity keybag that Touch ID uses. With - provision_keybag=1, the default, the boot-time bring-up - decides it once the key store is open; otherwise it stays - "unknown" until Touch ID is activated, which loads the - keybag and updates this value. It also stays "unknown" if - the key store never opens. + (RO) The identity keybag that Touch ID uses. ======= ============================================== unknown Not determined yet. - present An identity keybag is usable: one was already - persisted or was provisioned during the bring-up, - or Touch ID activation loaded it. + present An identity keybag exists: one was already + persisted, or the bring-up created one. This does + not say that it loads; Touch ID activation loads + it, and changes this to "failed" if that fails. missing None is persisted and none was provisioned. failed Provisioning, reading or loading it failed. ======= ============================================== + With provision_keybag=1, the default, the boot-time bring-up + decides it once the key store is open, and it stays + "unknown" if the key store never opens. With + provision_keybag=0, Touch ID activation decides it when it + reads the keybag, and it stays "unknown" if activation fails + before that point. + What: /sys/bus/platform/devices/.sep/diag/sensor Date: October 2026 KernelVersion: 7.1 @@ -175,13 +184,28 @@ Description: ready /dev/sep-bio is published, Touch ID was activated, and the most recent sensor bring-up succeeded. - not-ready Finished without that: the bring-up ended - without publishing /dev/sep-bio, activation - failed (it is not retried this boot), or - the most recent sensor bring-up did not - succeed. + not-ready Touch ID was activated, but the sensor is not + online: it is not bound, has not been brought + up since it bound, or its most recent + bring-up failed. A later bring-up, which + each enrol and verify runs, can still make + it ready. + failed Touch ID cannot work this boot: the bring-up + ended without publishing /dev/sep-bio, or + activation failed. Neither is retried before + the next boot. ========= ============================================ + "failed" can be intentional. With provision_keybag=0, as on + a read-only install (xart_writes=0, provision_keybag=0), the + bring-up still publishes /dev/sep-bio when no identity + keybag exists yet, and its first open then fails activation + because there is none: keybag reads "missing" and touchid + reads "failed". Read touchid "failed" together with keybag + "missing" (and, on a read-only install, xart "disabled") as + not provisioned rather than broken. Every other way to reach + "failed" is a real failure. + Because the first open of /dev/sep-bio activates Touch ID, a tool waiting for "ready" has to open it, or let its client do so, first. Enrol and verify also need xart to read diff --git a/drivers/soc/apple/diag.rs b/drivers/soc/apple/diag.rs index b01dd8227dd62..808d01fc25a8c 100644 --- a/drivers/soc/apple/diag.rs +++ b/drivers/soc/apple/diag.rs @@ -19,7 +19,7 @@ extern "C" { // struct in some configurations, which Rust does not accept across FFI. fn sep_diag_register( dev: *mut c_void, - profile: *const u8, + profile: *const c_char, profile_len: usize, cold: bool, sepos13: bool, @@ -29,7 +29,8 @@ extern "C" { fn sep_diag_set_endpoints(count: c_uint); fn sep_diag_set_keystore_open(); fn sep_diag_set_keybag(state: c_int); - fn sep_diag_set_sensor_result(online: bool); + fn sep_diag_sensor_bind() -> c_uint; + fn sep_diag_set_sensor_result(bind: c_uint, online: bool); fn sep_diag_set_bio_published(); fn sep_diag_set_touchid(started: bool); fn sep_diag_bringup_ended(); @@ -50,14 +51,24 @@ pub(crate) fn register( profile: &profile::PlatformProfile, xart_writable: bool, ) -> Result { - let cold = profile.bootstrap == profile::Bootstrap::Boot; - let sepos13 = matches!(profile.key_store, profile::KeyStore::Sepos13 { .. }); + // Exhaustive, so a new bootstrap or key store fails to build here until + // the ABI has a value for it. + let cold = match profile.bootstrap { + profile::Bootstrap::Boot => true, + profile::Bootstrap::WarmRegister => false, + }; + let sepos13 = match profile.key_store { + profile::KeyStore::Sepos13 { .. } => true, + profile::KeyStore::Variant5 => false, + }; // SAFETY: `dev` is bound for the duration of the call, which is what devres - // needs; the shim copies the name and keeps no pointer to it. + // needs. `profile.name` is a `str`, valid for reads of `profile.name.len()` + // bytes; the shim copies at most that many, needs no NUL terminator, and + // keeps no pointer to it. let ret = unsafe { sep_diag_register( dev.as_raw().cast(), - profile.name.as_ptr(), + profile.name.as_ptr().cast::(), profile.name.len(), cold, sepos13, @@ -88,10 +99,21 @@ pub(crate) fn keybag(state: Keybag) { unsafe { sep_diag_set_keybag(state as c_int) }; } +/// Which bind of the sensor a bring-up ran against. +#[derive(Clone, Copy)] +pub(crate) struct SensorBind(c_uint); + +/// Taken as a sensor bring-up begins, so that its result is dropped if the +/// sensor has been unbound or bound again by the time it finishes. +pub(crate) fn sensor_bind() -> SensorBind { + // SAFETY: no preconditions; loads one word. + SensorBind(unsafe { sep_diag_sensor_bind() }) +} + /// The result of a sensor bring-up, attach-time or later. -pub(crate) fn sensor_brought_up(online: bool) { - // SAFETY: no preconditions; stores one word. - unsafe { sep_diag_set_sensor_result(online) }; +pub(crate) fn sensor_brought_up(bind: SensorBind, online: bool) { + // SAFETY: no preconditions; updates one word. + unsafe { sep_diag_set_sensor_result(bind.0, online) }; } pub(crate) fn bio_published() { diff --git a/drivers/soc/apple/diag_shim.c b/drivers/soc/apple/diag_shim.c index 03970fe9d75ac..7d8a9598fffaf 100644 --- a/drivers/soc/apple/diag_shim.c +++ b/drivers/soc/apple/diag_shim.c @@ -33,14 +33,24 @@ #define SEP_DIAG_KEYSTORE_OPEN 1 #define SEP_DIAG_KEYSTORE_CLOSED 2 -/* The last bring-up of the bound sensor; reset when a sensor binds. */ -#define SEP_DIAG_SENSOR_UNTRIED 0 -#define SEP_DIAG_SENSOR_ONLINE 1 -#define SEP_DIAG_SENSOR_FAILED 2 +/* + * The sensor's state in the low bits and a count of its binds above them, in + * one word. A reader racing a re-bind sees the old device's state or the new + * one's, never a mix, and a bring-up that began before the sensor was unbound + * or bound again cannot report its result against the device bound since. + */ +#define SEP_DIAG_SENSOR_UNBOUND 0U +#define SEP_DIAG_SENSOR_BOUND 1U +#define SEP_DIAG_SENSOR_ONLINE 2U +#define SEP_DIAG_SENSOR_FAILED 3U +#define SEP_DIAG_SENSOR_STATE 3U +#define SEP_DIAG_SENSOR_BIND 4U /* * Touch ID is activated by the first open of /dev/sep-bio, not by the boot-time * bring-up, so publishing the node leaves the outcome undecided until then. + * UNAVAILABLE and FAILED are final for the boot; STARTED is not, since a later + * sensor bring-up can still bring the sensor online. */ #define SEP_DIAG_TOUCHID_PENDING 0 #define SEP_DIAG_TOUCHID_PUBLISHED 1 @@ -59,15 +69,13 @@ static struct { atomic_t endpoints; atomic_t keystore; atomic_t keybag; - atomic_t sensor_bound; - atomic_t sensor_result; + atomic_t sensor; atomic_t touchid; } sep_diag; -static bool sep_diag_sensor_online(void) +static unsigned int sep_diag_sensor(void) { - return atomic_read(&sep_diag.sensor_bound) && - atomic_read(&sep_diag.sensor_result) == SEP_DIAG_SENSOR_ONLINE; + return (unsigned int)atomic_read(&sep_diag.sensor) & SEP_DIAG_SENSOR_STATE; } static ssize_t abi_show(struct device *dev, struct device_attribute *attr, @@ -183,17 +191,22 @@ static DEVICE_ATTR_RO(keybag); static ssize_t sensor_show(struct device *dev, struct device_attribute *attr, char *buf) { - int result = atomic_read(&sep_diag.sensor_result); const char *state; - if (!atomic_read(&sep_diag.sensor_bound)) - state = "unbound"; - else if (result == SEP_DIAG_SENSOR_ONLINE) + switch (sep_diag_sensor()) { + case SEP_DIAG_SENSOR_BOUND: + state = "bound"; + break; + case SEP_DIAG_SENSOR_ONLINE: state = "online"; - else if (result == SEP_DIAG_SENSOR_FAILED) + break; + case SEP_DIAG_SENSOR_FAILED: state = "failed"; - else - state = "bound"; + break; + default: + state = "unbound"; + break; + } return sysfs_emit(buf, "%s\n", state); } static DEVICE_ATTR_RO(sensor); @@ -205,11 +218,12 @@ static ssize_t touchid_show(struct device *dev, struct device_attribute *attr, switch (atomic_read(&sep_diag.touchid)) { case SEP_DIAG_TOUCHID_STARTED: - state = sep_diag_sensor_online() ? "ready" : "not-ready"; + state = sep_diag_sensor() == SEP_DIAG_SENSOR_ONLINE ? + "ready" : "not-ready"; break; case SEP_DIAG_TOUCHID_UNAVAILABLE: case SEP_DIAG_TOUCHID_FAILED: - state = "not-ready"; + state = "failed"; break; default: state = "unknown"; @@ -277,17 +291,48 @@ void sep_diag_set_keybag(int state) atomic_set(&sep_diag.keybag, state); } +/* + * Called from the SPI probe and remove of the one sensor device, which the + * driver core never runs concurrently. A bring-up result racing this store is + * either overwritten by it or, arriving later, refused by the bind check below. + */ void sep_diag_set_sensor_bound(bool bound) { + unsigned int binds = (unsigned int)atomic_read(&sep_diag.sensor) & + ~SEP_DIAG_SENSOR_STATE; + if (bound) - atomic_set(&sep_diag.sensor_result, SEP_DIAG_SENSOR_UNTRIED); - atomic_set(&sep_diag.sensor_bound, bound); + atomic_set(&sep_diag.sensor, + (binds + SEP_DIAG_SENSOR_BIND) | SEP_DIAG_SENSOR_BOUND); + else + atomic_set(&sep_diag.sensor, binds | SEP_DIAG_SENSOR_UNBOUND); +} + +/* Taken as a bring-up begins and handed back with its result. */ +unsigned int sep_diag_sensor_bind(void) +{ + return (unsigned int)atomic_read(&sep_diag.sensor) & + ~SEP_DIAG_SENSOR_STATE; } -void sep_diag_set_sensor_result(bool online) +void sep_diag_set_sensor_result(unsigned int bind, bool online) { - atomic_set(&sep_diag.sensor_result, - online ? SEP_DIAG_SENSOR_ONLINE : SEP_DIAG_SENSOR_FAILED); + unsigned int state = online ? SEP_DIAG_SENSOR_ONLINE : + SEP_DIAG_SENSOR_FAILED; + int old = atomic_read(&sep_diag.sensor); + + do { + unsigned int word = old; + + /* + * With no device bound, "unbound" is the cause worth showing; + * after a re-bind, the result belongs to the previous device. + */ + if ((word & SEP_DIAG_SENSOR_STATE) == SEP_DIAG_SENSOR_UNBOUND || + (word & ~SEP_DIAG_SENSOR_STATE) != bind) + return; + } while (!atomic_try_cmpxchg(&sep_diag.sensor, &old, + (old & ~SEP_DIAG_SENSOR_STATE) | state)); } /* diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index afc52918b4508..c999300558637 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1085,16 +1085,17 @@ impl SepData { ); return; } + let bind = diag::sensor_bind(); let Some(patch) = self.wake_sensor() else { dev_warn!( self.dev, "sbio: the sensor did not wake at attach; the next verify retries the bring-up\n" ); - diag::sensor_brought_up(false); + diag::sensor_brought_up(bind, false); return; }; let online = self.complete_bringup(patch); - diag::sensor_brought_up(online); + diag::sensor_brought_up(bind, online); if !online { dev_warn!( self.dev, @@ -1104,8 +1105,9 @@ impl SepData { } fn bring_sensor_online(&self) -> bool { + let bind = diag::sensor_bind(); let Some(mut patch) = self.wake_sensor() else { - diag::sensor_brought_up(false); + diag::sensor_brought_up(bind, false); return false; }; @@ -1122,7 +1124,7 @@ impl SepData { if !self.sensor_calibrated.load(Relaxed) { if !self.calibrate_sensor() { - diag::sensor_brought_up(false); + diag::sensor_brought_up(bind, false); return false; } self.sensor_calibrated.store(true, Relaxed); @@ -1138,7 +1140,7 @@ impl SepData { Some(parameters) => self.finish_sensor_setup(patch, parameters), None => self.complete_bringup(patch), }; - diag::sensor_brought_up(ok); + diag::sensor_brought_up(bind, ok); // The sensor is patched and idle here -- the only safe moment to // configure the data-ready interrupt, which is then left alone for the // driver's life. Interrupt capture is the default; a machine that does diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index 93d9db41e99f9..f43e038fe4205 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -163,7 +163,8 @@ void sep_diag_set_endpoints(unsigned int count); void sep_diag_set_keystore_open(void); void sep_diag_set_keybag(int state); void sep_diag_set_sensor_bound(bool bound); -void sep_diag_set_sensor_result(bool online); +unsigned int sep_diag_sensor_bind(void); +void sep_diag_set_sensor_result(unsigned int bind, bool online); void sep_diag_set_bio_published(void); void sep_diag_set_touchid(bool started); void sep_diag_bringup_ended(void); From ae4c139ccf3080d1a64aa82b05f0f8ad34abc344 Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 14:06:50 -0700 Subject: [PATCH 0802/2680] tools: aurora-sep: test plan and DKMS notice for the Neural Engine The kernel now carries the in-tree Apple Neural Engine driver (aurora-silicon/linux#155), switched on for the M1 Max and M2 Max only. Add a Neural Engine step to the agent test plan: what passes on those two, that every other Mac should show nothing, what to check when the M2 firmware is or is not present, and the module_blacklist escape if a Mac stops booting. omarchy-ane-dkms builds the same module names into updates/dkms, which modprobe prefers over the kernel's own. When it is installed, say so rather than remove it: removing it also removes the M2 firmware it fetched. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- tools/aurora-sep/install-aurora-sep.sh | 38 ++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index da97501abec4e..d1d3e1b5eb0f4 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -35,6 +35,10 @@ # 11.24 adds Touch ID on the MacBook Pro 14"/16" M1 Max and the MacBook Air 13" # M2, untested on both. On those boards the script first keeps the boot.bin the # Mac booted with on the EFI partition and prints how to restore it from macOS. +# 11.25 adds the in-tree Apple Neural Engine driver (aurora-silicon/linux#155), +# switched on for the M1 Max and M2 Max only, and the read-only SEP diagnostics +# under /sys/bus/platform/devices/*.sep/diag/. When omarchy-ane-dkms is +# installed the script says so: its modules take precedence over this kernel's. # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -473,10 +477,24 @@ sep_write_notice() { return 0 } +# omarchy-ane-dkms builds its own Neural Engine modules into updates/dkms, +# which modprobe prefers over the ones this kernel ships. Say so rather than +# remove it: it also carries the M2 firmware fetch its owner may rely on. +ane_dkms_notice() { + pacman -Q omarchy-ane-dkms >/dev/null 2>&1 || return 0 + warn "omarchy-ane-dkms is installed. Its Neural Engine modules take precedence + over the driver built into this kernel, so a test would exercise that + package's driver instead. To test this kernel's driver, remove it first: + sudo pacman -R omarchy-ane-dkms + Removing it also removes the Neural Engine firmware it fetched on an M2 + Pro/Max." +} + install_all() { local entry file sha kernel chain version_notice sep_write_notice + ane_dkms_notice kernel=$(current_kernel) chain=$(boot_chain) if [[ $chain == grub ]]; then boot_space "$kernel"; fi @@ -785,6 +803,26 @@ SAFETY, NON-NEGOTIABLE sudo dmesg | grep -E 'Touch ID: ending|still running|PM: suspend' Re-check /dev/sep-bio, the display and the dock after resume. +9. NEURAL ENGINE + This build carries the in-tree Apple Neural Engine driver + (aurora-silicon/linux#155), switched on only for the M1 Max and the M2 Max. + On every other Mac no Neural Engine lines are expected, and that is a pass. + pacman -Q omarchy-ane-dkms 2>/dev/null # if installed, say so: its modules replace this kernel's + modinfo -F filename ane ane_t6021 + ls -l /dev/accel/ 2>/dev/null + sudo dmesg | grep -iE '\bane\b|ane_t6021|neural' | head -40 + M1 Max PASS: the ane module is bound, /dev/accel/accel0 exists, and the + module path is under kernel/drivers/accel/ane (not updates/dkms). + M2 Max: without the Neural Engine firmware, expect a firmware load error and + nothing else broken. With it (fetched by Joshua Warren's + omarchy-ane-firmware-fetch), expect ane_t6021 bound and /dev/accel/accel0. + If omarchy-ane's tools are installed, also run omarchy-ane-check, and on an + M2 Max omarchy-ane-check --smoke; quote their result lines. + Report any kernel log line at emergency or alert level, and whether idle + battery drain changed against the previous build. If the Mac does not finish + booting, add module_blacklist=ane,ane_t6021 to the kernel command line from + the boot menu, and report that. + Report "not tested" honestly rather than guessing, for any of the above. HOW TO REPORT From c4e13a2965956e92e0102256684acd974fd441c7 Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 14:44:39 -0700 Subject: [PATCH 0803/2680] tools: aurora-sep: release 11.25 linux-aurora 7.1.12.aurora2-11.25: - the in-tree Apple Neural Engine driver (aurora-silicon/linux#155, with #65), switched on for the M1 Max and M2 Max only; - the read-only SEP diagnostics group under /sys/bus/platform/devices/*.sep/diag/. The test plan and the omarchy-ane-dkms notice came with the previous commit. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- tools/aurora-sep/install-aurora-sep.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index d1d3e1b5eb0f4..bf1f1f378bdd4 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -67,8 +67,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.24 -TAG=sep-7.1.12.aurora2-11.24 +VERSION=7.1.12.aurora2-11.25 +TAG=sep-7.1.12.aurora2-11.25 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -76,8 +76,8 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst 8447aac136a5814bb80d95e5394d63d1ee4702f8d88830393a952179c7e40183" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst d358e06b7238157e105ddce77ffd59bad8a709f31f041ab741f7d22dc811b494" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst f8b78b21f4031113a06c7f7fc7cbe1d72a23ba803adfcfc838ee3e927d3d0c85" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst a1a1524ccc35b4b4c82e4a01fda90ae13c43719ecb8dfc8108fa9a008d9500e6" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20261002-1-any.pkg.tar.zst a9dda6e0526874e4ac760629f3aa5bd37421379a1d8af34000f3dc7b73a21b17" "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" From 96781ae85bed0ed27535ae92c6e8e0765bfc52e6 Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 20:59:54 -0700 Subject: [PATCH 0804/2680] tools: aurora-sep: keep a Neo's own m1n1 settings (11.25.1) m1n1_update rewrote /etc/default/update-m1n1 with only its DTBS line. On a MacBook Neo that file also names the Neo's own J700 m1n1 and U-Boot (M1N1=, U_BOOT=), so update-m1n1 then rebuilt boot.bin from an m1n1 that cannot boot a Neo. Replace only our DTBS setting and keep the rest of the file; a re-run replaces the previous block instead of adding another. Reported by a Neo tester's agent before the install. Installer-only release: same packages as 11.25. Signed-off-by: Chris Kearney Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i [ryan: also drops a trailing space in install-aurora-sep.sh, as aurora-wip has it after merging this branch in 45da41c0cd1b] Signed-off-by: Ryan Murray --- tools/aurora-sep/install-aurora-sep.sh | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index bf1f1f378bdd4..ced4100c78b5a 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -39,6 +39,9 @@ # switched on for the M1 Max and M2 Max only, and the read-only SEP diagnostics # under /sys/bus/platform/devices/*.sep/diag/. When omarchy-ane-dkms is # installed the script says so: its modules take precedence over this kernel's. +# 11.25.1 (installer only, same packages): rebuilding m1n1's stage 2 now keeps +# the rest of /etc/default/update-m1n1, so a MacBook Neo keeps its own M1N1= +# and U_BOOT= instead of being rebuilt from an m1n1 that cannot boot it. # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -68,7 +71,7 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. VERSION=7.1.12.aurora2-11.25 -TAG=sep-7.1.12.aurora2-11.25 +TAG=sep-7.1.12.aurora2-11.25.1 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -315,10 +318,17 @@ EOF # version number, kept for the fallback entry) without the Touch ID sensor node. # Point it at whatever directory linux-aurora owns, now and after its updates. m1n1_update() { - local target - [[ -f /etc/default/update-m1n1 && ! -f $STATE/update-m1n1.default.saved ]] && - $sudo cp /etc/default/update-m1n1 "$STATE/update-m1n1.default.saved" - $sudo tee /etc/default/update-m1n1 >/dev/null <<'EOF' + local target conf=/etc/default/update-m1n1 tmp + [[ -f $conf && ! -f $STATE/update-m1n1.default.saved ]] && + $sudo cp "$conf" "$STATE/update-m1n1.default.saved" + # Replace only the DTBS setting and keep the rest of the file: a MacBook + # Neo's M1N1= and U_BOOT= point at its own J700 builds, and dropping them + # would rebuild boot.bin from an m1n1 that cannot boot it. + tmp=$(mktemp) + if [[ -f $conf ]]; then + $sudo sed -e '/^# aurora-sep: build m1n1/,/^DTBS=/d' -e '/^DTBS=/d' "$conf" >"$tmp" + fi + cat >>"$tmp" <<'EOF' # aurora-sep: build m1n1's stage 2 from the device trees the installed # linux-aurora package owns, which carry the Touch ID sensor node. # @@ -329,6 +339,8 @@ m1n1_update() { # (update-m1n1 runs under set -e, so this assignment must always succeed.) DTBS=$(pacman -Qlq linux-aurora 2>/dev/null | grep '/dtbs/[^/]*\.dtb$'; true) EOF + $sudo install -m 644 "$tmp" "$conf" + rm -f "$tmp" for target in /boot/m1n1/boot.bin /boot/efi/m1n1/boot.bin; do [[ -f $target && ! -f $STATE/boot.bin.saved ]] && $sudo cp "$target" "$STATE/boot.bin.saved" done @@ -659,7 +671,7 @@ SAFETY, NON-NEGOTIABLE naming the newer one. To check without installing: curl -fsSL https://api.github.com/repos/iconidentify/aurora-linux/releases/latest \ - | grep -m1 '"tag_name"' + | grep -m1 '"tag_name"' Always fetch the script from the "latest" URL rather than a tag you were handed, so you get the current one automatically: From f2239acf1a6f03bff2216d1c830e40facb4dd540 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 15:52:10 +0000 Subject: [PATCH 0805/2680] soc: apple: sep: Wait for bring-up in trusted keys and honour a withheld J700 sensor bus Trusted-key callbacks can run as soon as probe publishes the key source, while endpoint discovery and key-store initialisation are still running. Publish bring-up completion under the endpoint lock and let only those consumers wait for it, failing if the attach completed without a key store. Mark the key store initialised on profiles whose dialect needs no init handshake. On J700 a disabled SPI2 controller at hand-off is a firmware decision about whether the sensor is present: leave Touch ID unavailable instead of enabling the bus. Older platforms still activate it. Report a restored enrolment at info level. [ryan: split from commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"), which included these SEP readiness and sensor refusal corrections: this commit carries its drivers/soc/apple hunks only. The display, ATC, USB and ISP parts belong to 200-dcp, 170-usb and 240-isp] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/dt.rs | 23 ++++++++++++---- drivers/soc/apple/profile.rs | 2 +- drivers/soc/apple/refkey.rs | 2 ++ drivers/soc/apple/sbio.rs | 28 +++++++++++++------ drivers/soc/apple/sep.rs | 53 ++++++++++++++++++++++++++++++++++-- drivers/soc/apple/sks.rs | 1 + drivers/soc/apple/trusted.rs | 3 ++ 7 files changed, 94 insertions(+), 18 deletions(-) diff --git a/drivers/soc/apple/dt.rs b/drivers/soc/apple/dt.rs index 48a4732008967..51c83d1249cdf 100644 --- a/drivers/soc/apple/dt.rs +++ b/drivers/soc/apple/dt.rs @@ -400,7 +400,14 @@ unsafe fn add_u32( } } -pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { +// The public J700 DTS enables SPI2. A disabled controller at handoff is a +// firmware hardware-presence decision; older platforms still need activation. +fn sensor_controller_refused(controller: &DtNode) -> bool { + crate::profile::machine_has(b"apple,j700") && !controller.is_available() +} + +/// Returns false when the bootloader explicitly withheld J700's controller. +pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result { // A machine that describes the sensor in its own device tree (compatible // apple,mesa-fingerprint) is handled machine-agnostically: enable that node // and its SPI controller wherever they sit, with the power/IRQ GPIOs coming @@ -411,6 +418,9 @@ pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { pr_err!("apple_sep: sensor node has no parent SPI controller\n"); ENODEV })?; + if sensor_controller_refused(&controller) { + return Ok(false); + } pr_info!( "apple_sep: sensor node present in the device tree; enabling it and its SPI bus\n" ); @@ -425,7 +435,7 @@ pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { queued = true; } Ok(queued) - }); + }).map(|()| true); } let controller = node_at_address(base).ok_or_else(|| { @@ -436,6 +446,9 @@ pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { ENODEV })?; + if sensor_controller_refused(&controller) { + return Ok(false); + } let controller_ok = controller.is_available(); let existing = controller.child_with_reg(cs); pr_info!( @@ -447,7 +460,7 @@ pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { ); if controller_ok && existing.is_some() { - return Ok(()); + return Ok(true); } if existing.is_some() && !controller_ok { @@ -456,7 +469,7 @@ pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { return with_changeset(|cs_handle| { queue_status_okay(cs_handle, &controller)?; Ok(true) - }); + }).map(|()| true); } pr_info!( @@ -470,7 +483,7 @@ pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { } queue_sensor_node(cs_handle, &controller)?; Ok(true) - }) + }).map(|()| true) } /// Resolve a reserved-memory region by name on the SEP node's device. diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 82ad95613c4b1..85ed9c4b03cec 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -344,7 +344,7 @@ static_assert!(T8112.sensor.controller_base == T8103.sensor.controller_base); /// /// The root property is a NUL-separated list, so whole entries are compared /// instead of substrings: `apple,t8103` cannot match a longer unrelated value. -fn machine_has(compatible: &[u8]) -> bool { +pub(crate) fn machine_has(compatible: &[u8]) -> bool { let Some(root) = of::root() else { return false; }; diff --git a/drivers/soc/apple/refkey.rs b/drivers/soc/apple/refkey.rs index 4d54246bf4f68..d8e6a5c9a67c8 100644 --- a/drivers/soc/apple/refkey.rs +++ b/drivers/soc/apple/refkey.rs @@ -246,6 +246,7 @@ impl SepData { } pub(crate) fn refkey_seal_trusted(&self, key: &[u8]) -> Result> { + self.wait_for_trusted_ready()?; self.ensure_machine_refkey()?; let guard = self.machine_refkey.lock(); let mk = guard.as_ref().ok_or(ENODEV)?; @@ -253,6 +254,7 @@ impl SepData { } pub(crate) fn refkey_unseal_trusted(&self, sealed: &[u8]) -> Result> { + self.wait_for_trusted_ready()?; self.ensure_machine_refkey()?; let blob = { let guard = self.machine_refkey.lock(); diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index c999300558637..cff44167e20cd 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -304,14 +304,24 @@ impl SepData { return; } - if let Err(e) = dt::enable_spi_sensor(sensor::CONTROLLER_BASE, sensor::CHIP_SELECT) { - dev_warn!( - self.dev, - "sensor: could not enable the SPI bus at 0x{:x} or create the sensor node ({:?}); no sensor\n", - sensor::CONTROLLER_BASE, - e - ); - return; + match dt::enable_spi_sensor(sensor::CONTROLLER_BASE, sensor::CHIP_SELECT) { + Ok(true) => {}, + Ok(false) => { + dev_info!( + self.dev, + "sensor: SPI controller withheld by the J700 firmware handoff; Touch ID unavailable\n" + ); + return; + } + Err(e) => { + dev_warn!( + self.dev, + "sensor: could not enable the SPI bus at 0x{:x} or create the sensor node ({:?}); no sensor\n", + sensor::CONTROLLER_BASE, + e + ); + return; + } } let bound = sensor::is_bound(); @@ -1984,7 +1994,7 @@ impl SepData { Some(n) if (self.templates_restored.load(Relaxed) || self.cold_restore_candidate.load(Relaxed)) && self.prove_restore() => { self.templates_restored.store(true, Relaxed); - dev_warn!( + dev_info!( self.dev, "Touch ID: restored {} enrolled identity/identities; enrolment survived reboot\n", n diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 659df59548859..f63f5ceb05a0b 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -562,6 +562,8 @@ struct SepData { #[pin] endpoints: Mutex, + #[pin] + trusted_ready: CondVar, #[pin] control: Mutex, @@ -822,6 +824,7 @@ impl SepData { mbox <- new_mutex!(None), shmem <- new_mutex!(Some(buf)), endpoints <- new_mutex!(EndpointTable::new()), + trusted_ready <- new_condvar!("SepData::trusted_ready"), control <- new_mutex!(control::ControlState::new()), control_wq <- new_condvar!("SepData::control_wq"), sks_seq: Atomic::new(0), @@ -1911,7 +1914,7 @@ impl SepData { ); diag::attach(false); diag::bringup_ended(); - this.phase.store(PHASE_READY, Relaxed); + this.finish_bringup(); return; } dev_info!( @@ -1933,7 +1936,7 @@ impl SepData { e ); diag::bringup_ended(); - this.phase.store(PHASE_READY, Relaxed); + this.finish_bringup(); return; } @@ -1974,10 +1977,50 @@ impl SepData { this.run_bringup(); diag::bringup_ended(); - this.phase.store(PHASE_READY, Relaxed); + this.finish_bringup(); Self::arm_settle(this); } + // Publish completion under the same lock that trusted consumers wait on. + // PHASE_READY ends discovery/initialisation; it is not proof of success. + fn finish_bringup(&self) { + let _guard = self.endpoints.lock(); + self.phase.store(PHASE_READY, Relaxed); + self.trusted_ready.notify_all(); + } + + /// Trusted-key callbacks can run as soon as probe publishes the source, + /// while endpoint discovery and key-store initialisation are still running. + /// Wait only in those consumers, leaving bringup free to establish the key. + pub(crate) fn wait_for_trusted_ready(&self) -> Result<()> { + let mut remaining = time::msecs_to_jiffies(EXCHANGE_TIMEOUT_MS); + let mut guard = self.endpoints.lock(); + loop { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } + if self.phase.load(Relaxed) == PHASE_READY { + // A completed but failed attach must not look ready. + if !guard.eps.iter().any(|e| e.id == proto::EP_SKS) { + return Err(ENODEV); + } + if !self.sks_initialized.load(Relaxed) { + return Err(EIO); + } + return Ok(()); + } + if remaining == 0 { + dev_err!(self.dev, "trusted-keys: key-store bringup timed out\n"); + return Err(ETIMEDOUT); + } + match self.trusted_ready.wait_interruptible_timeout(&mut guard, remaining) { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + CondVarTimeoutResult::Timeout => remaining = 0, + } + } + } + fn endpoint_present(&self, id: u8) -> bool { self.endpoints.lock().eps.iter().any(|e| e.id == id) } @@ -2049,6 +2092,10 @@ impl SepData { fn remove(&self) { self.shutting_down.store(true, Relaxed); + { + let _guard = self.endpoints.lock(); + self.trusted_ready.notify_all(); + } bio::release(&mut self.bio_session.lock()); self.worker_gone.notify_all(); self.rng_shutdown.store(true, Relaxed); diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index f1fabe2791a76..d7a203f9e0b15 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -341,6 +341,7 @@ impl SepData { cpx_encryption_mode, } = self.profile.key_store else { + self.sks_initialized.store(true, Relaxed); return true; }; diff --git a/drivers/soc/apple/trusted.rs b/drivers/soc/apple/trusted.rs index 6162bc8af9d3c..3c7f512e2865b 100644 --- a/drivers/soc/apple/trusted.rs +++ b/drivers/soc/apple/trusted.rs @@ -199,6 +199,9 @@ unsafe extern "C" fn tk_get_random(key: *mut u8, key_len: usize) -> c_int { if key_len == 0 { return 0; } + if let Err(e) = sep.wait_for_trusted_ready() { + return e.to_errno(); + } // SAFETY: the framework guarantees `key` is writable for `key_len` bytes. let buf = unsafe { core::slice::from_raw_parts_mut(key, key_len) }; match sep.sep_random(buf) { From 8c7f02408c139f6e3dc418861f71a90a1a194c87 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 14 Oct 2026 04:00:16 -0400 Subject: [PATCH 0806/2680] soc: apple: enable SEP autoload and writable xART defaults Select the SEP driver as a module by default and export its OF device table for modpost. The missing alias left SEP unbound on a fresh Neo install until apple_sep was loaded manually. Default xart_writes to 1 while preserving APFS extent ownership, record validation and write checks. Keep xart_writes=0 as the read-only override and leave identity-keybag provisioning opt-in. Update policy documentation and the Arch package checksums. Validation: 12 SEP host tests pass; the module builds with W=1 using the prepared J700 kernel build. Neo rebooted with the exact candidate module loading automatically, attached, and reporting validated writable xART. Two 64-byte hardware RNG reads passed. The shared iBoot-container hash remained unchanged; no actual xART write or Touch ID operation is claimed. Assisted-by: LLM Signed-off-by: DjDeveloperr --- drivers/soc/apple/Kconfig | 9 +++++---- drivers/soc/apple/sep.rs | 12 ++++++++---- tools/aurora-sep/README.md | 11 +++++++---- tools/aurora-sep/XART-SAFETY.md | 11 +++++++++++ tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD | 6 +++--- 5 files changed, 34 insertions(+), 15 deletions(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index dd927fc23c6d2..95730bf1b3cd8 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -114,6 +114,7 @@ config APPLE_SEP_FV_INTERFACE config APPLE_SEP tristate "Apple SEP (Secure Enclave Processor)" + default m if ARCH_APPLE depends on ARCH_APPLE depends on PM depends on RUST @@ -139,10 +140,10 @@ config APPLE_SEP the iBoot system container, locating its .gl extent through read-only, checksum-validated APFS metadata. It validates the complete store and both root records before registering with SEP. - Raw APFS xART writes are disabled by default. An explicit xart_writes - opt-in is accepted only when the driver validates narrow physical - extent ownership invariants; this does not prove all macOS-compatible - update behavior. Keep it off until the target machine has been checked. + xART writes are enabled by default, but accepted only when the driver + validates the physical extent ownership invariants. Unsupported APFS + layouts fail closed. Set xart_writes=0 for read-only operation. + Identity-keybag provisioning remains a separate opt-in. With writes off the driver still attaches, but refuses Touch ID enrol and verify: the enclave saves anti-replay state after every match. diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index f63f5ceb05a0b..dd233c4ce5f05 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -2316,8 +2316,12 @@ impl WorkItem for SepData { struct SepDriver(Arc); -const OF_TABLE: kernel::device_id::IdArray = - kernel::device_id::IdArray::new([(of::DeviceId::new(c"apple,sep"), ())]); +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + (), + [(of::DeviceId::new(c"apple,sep"), ())] +); impl platform::Driver for SepDriver { type IdInfo = (); @@ -2443,8 +2447,8 @@ module! { license: "Dual MIT/GPL", params: { xart_writes: u8 { - default: 0, - description: "Opt in to shared xART writes only if APFS proves a single unsnapshotted .gl extent; Touch ID enrol and verify need it", + default: 1, + description: "Allow shared xART writes only if APFS proves a single unsnapshotted .gl extent; set to 0 for read-only operation", }, xart_start_sector: u64 { default: 0, diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 96632f81c585d..e56a1311876a3 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -28,12 +28,15 @@ for example with `systemctl edit aurora-sep.service`: Environment=AURORA_SEP_READY_TIMEOUT=180 ``` -`xart_writes=0` remains the default. `load-driver` explicitly opts into writes -and must only be enabled on a machine whose APFS `.gl` extent passes the -driver's narrow ownership checks: one physical extent, no snapshots or revert +The driver defaults to `xart_writes=1` and exports the `apple,sep` module +alias for automatic loading. Writes are accepted only when the APFS `.gl` +extent passes the driver's narrow ownership checks: one physical extent, no snapshots or revert state, and a single extent reference owned by the `.gl` inode. The kernel resolves that extent from checksum-validated APFS metadata; unsupported layouts -fail closed. The opt-in has been tested on m2 with identity-keybag provisioning, +fail closed. Set `options apple_sep xart_writes=0 provision_keybag=0` in a +modprobe configuration to retain read-only operation. Identity-keybag creation +still requires `provision_keybag=1`; `load-driver` supplies that option. +Enabling writes has been tested on m2 with identity-keybag provisioning, trusted-key seal/unseal, and reboot reload. The raw iBoot container hash did not change in those tests, so an actual xART write was **not** validated. Do not infer that all SEP or Touch ID operations work with a read-only GigaLocker. diff --git a/tools/aurora-sep/XART-SAFETY.md b/tools/aurora-sep/XART-SAFETY.md index 52a976c63e5c4..33af5b7b97651 100644 --- a/tools/aurora-sep/XART-SAFETY.md +++ b/tools/aurora-sep/XART-SAFETY.md @@ -1,5 +1,16 @@ # Shared xART gigalocker safety gate +## Current default + +`xart_writes=1` is the driver default. The APFS extent-ownership checks, +record validation and write-ordering checks remain mandatory; unsupported +layouts fail closed. `xart_writes=0` explicitly selects read-only operation. +Identity-keybag creation remains a separate `provision_keybag=1` opt-in. +The historical evidence below records the earlier opt-in deployments and +does not establish qualification on every supported Mac or APFS layout. + +## Earlier deployment policy + The iBoot APFS container and its XART `.gl` file are shared with macOS. The deployed `m2fix4` kernel remains read-only. The `m2fix5` source has an explicit `xart_writes=1` opt-in; it accepts writes only when the APFS-resolved extent is diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index 15712b3bdf4e2..c0a935fa93e18 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -7,7 +7,7 @@ pkgname=aurora-sep pkgver=1 -pkgrel=3 +pkgrel=4 pkgdesc="Loader and desktop integration for the Apple SEP (Touch ID) driver" url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" arch=(any) @@ -40,8 +40,8 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '0695bd9ce4c207241db13c3ea07819a032d8213417021378f5015ab9317cde12' '5936eb76db666496a8700dade7ac6bcb3785b517958dcea10685690f1822c86f' 'ffab97b20bfd5f0eb1ab641179a131843349cf16890fff0cf2d893f0b5681b37' - 'd7a09f31367c181a2cbda63cd58104d6117136fb5db5b20f4a3996ea3af7025e' - '83ae33b34d4f4ff42be7ba22b6d5557c3c5776075753ea288bef986645687f23') + '7777f399b05f9ed860687140efdb0e07ea7b2d23939fe59ec4894690dff299cc' + '6628ae9c63265914e7206edf61e42c613810d9d9a61adc6c7c02327504911a42') # A unit and the helper it runs ($1, installed as $3), pointed at the # packaged helper. From 0e5e30ac0ce5780dd67176ad49e7fbe28266f5bd Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 14 Oct 2026 04:28:57 -0400 Subject: [PATCH 0807/2680] soc: apple: provision the Linux SEP identity automatically A fresh installation can attach SEP and use its RNG yet fail keyctl with ENODEV because no Linux identity keybag or machine reference key exists. Enable existing first-boot identity provisioning by default. Recover valid persisted identities on later boots and retain the existing refusal to replace unreadable identities or reference keys. Keep the explicit xart_writes=0 provision_keybag=0 read-only configuration and APFS checks. Update configuration help, documentation and package checksums. Validation: 12 SEP host tests pass and the module builds with W=1. Neo provisioned its missing identity using the existing provisioning path, then loaded the same trusted-key blob after three reboots. An encrypted payload authenticated under that recovered master key, corrupted trusted blobs were rejected, and identity/reference-key hashes stayed unchanged. The final module loaded automatically, with no provisioning override, and recovered the existing identity successfully. Touch ID was not tested. Assisted-by: LLM Signed-off-by: DjDeveloperr --- drivers/soc/apple/Kconfig | 5 +++-- drivers/soc/apple/sep.rs | 4 ++-- tools/aurora-sep/README.md | 6 ++++-- tools/aurora-sep/XART-SAFETY.md | 5 +++-- tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD | 6 +++--- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 95730bf1b3cd8..d292b5130c62d 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -142,8 +142,9 @@ config APPLE_SEP both root records before registering with SEP. xART writes are enabled by default, but accepted only when the driver validates the physical extent ownership invariants. Unsupported APFS - layouts fail closed. Set xart_writes=0 for read-only operation. - Identity-keybag provisioning remains a separate opt-in. + layouts fail closed. A Linux identity keybag is created automatically + when none exists; existing identities and reference keys are retained. + Set xart_writes=0 provision_keybag=0 for read-only operation. With writes off the driver still attaches, but refuses Touch ID enrol and verify: the enclave saves anti-replay state after every match. diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index dd233c4ce5f05..1ae6730de8717 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -2455,8 +2455,8 @@ module! { description: "Expected 512-byte start sector of the APFS .gl extent in the iBoot container. Zero accepts the APFS result; nonzero must match it and cannot override it", }, provision_keybag: u8 { - default: 0, - description: "Create the Linux identity keybag when none exists; requires xart_writes=1", + default: 1, + description: "Create the Linux identity keybag when none exists; set to 0 to disable first-boot provisioning; requires xart_writes=1", }, os_uuid_hi: u64 { default: 0, diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index e56a1311876a3..ea9d2704e9e80 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -34,8 +34,10 @@ extent passes the driver's narrow ownership checks: one physical extent, no snap state, and a single extent reference owned by the `.gl` inode. The kernel resolves that extent from checksum-validated APFS metadata; unsupported layouts fail closed. Set `options apple_sep xart_writes=0 provision_keybag=0` in a -modprobe configuration to retain read-only operation. Identity-keybag creation -still requires `provision_keybag=1`; `load-driver` supplies that option. +modprobe configuration to retain read-only operation. `provision_keybag=1` +is also the default: first boot creates the Linux identity needed for trusted +keys, and later boots recover that identity and its machine reference key. +An existing unreadable or invalid identity is not silently replaced. Enabling writes has been tested on m2 with identity-keybag provisioning, trusted-key seal/unseal, and reboot reload. The raw iBoot container hash did not change in those tests, so an actual xART write was **not** validated. Do not diff --git a/tools/aurora-sep/XART-SAFETY.md b/tools/aurora-sep/XART-SAFETY.md index 33af5b7b97651..5dd70e8b26a80 100644 --- a/tools/aurora-sep/XART-SAFETY.md +++ b/tools/aurora-sep/XART-SAFETY.md @@ -4,8 +4,9 @@ `xart_writes=1` is the driver default. The APFS extent-ownership checks, record validation and write-ordering checks remain mandatory; unsupported -layouts fail closed. `xart_writes=0` explicitly selects read-only operation. -Identity-keybag creation remains a separate `provision_keybag=1` opt-in. +layouts fail closed. `xart_writes=0 provision_keybag=0` explicitly selects +read-only operation. `provision_keybag=1` is the default: create the Linux +identity if missing and recover an existing identity without replacing it. The historical evidence below records the earlier opt-in deployments and does not establish qualification on every supported Mac or APFS layout. diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index c0a935fa93e18..5c0f665f9dcfe 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -7,7 +7,7 @@ pkgname=aurora-sep pkgver=1 -pkgrel=4 +pkgrel=5 pkgdesc="Loader and desktop integration for the Apple SEP (Touch ID) driver" url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" arch=(any) @@ -40,8 +40,8 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '0695bd9ce4c207241db13c3ea07819a032d8213417021378f5015ab9317cde12' '5936eb76db666496a8700dade7ac6bcb3785b517958dcea10685690f1822c86f' 'ffab97b20bfd5f0eb1ab641179a131843349cf16890fff0cf2d893f0b5681b37' - '7777f399b05f9ed860687140efdb0e07ea7b2d23939fe59ec4894690dff299cc' - '6628ae9c63265914e7206edf61e42c613810d9d9a61adc6c7c02327504911a42') + 'ed507587d3356ff2713e988e024b183772bab14061bf787bcaef28691867af0d' + '3d92b3be06bc1b670666dec3d0032893fe8c7be428a4f1200d7c9803980c769a') # A unit and the helper it runs ($1, installed as $3), pointed at the # packaged helper. From 321cdc24bfc18720b523a14153a279d016a522bd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 08:41:03 +1100 Subject: [PATCH 0808/2680] soc: apple: recheck sensor status after SEP registration Registration can clear and authenticate the sensor session after the wake status was sampled. Read status again before trusting the patch acknowledgement, refuse unreadable or silent status, and send an explicit needs-patch state through the existing patch sequence. Preserve capture failures and enrolled identities. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index cff44167e20cd..1dd46c3195918 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1882,8 +1882,18 @@ impl SepData { return None; } - // The patch decision is made on status[8], not on state 9. - if st.patch_ack() == sensor::PATCH_ACCEPTED { + // Registration can reset the sensor session. Do not reuse the + // acknowledgement sampled before CLEAR_STATE and authentication. + let st = match sensor::status() { + Ok(st) if !st.is_silent() => st, + _ => { + dev_warn!(self.dev, "sensor: status unavailable after registration\n"); + return None; + } + }; + if st.patch_ack() == sensor::PATCH_ACCEPTED + && st.state != sensor::STATE_NEEDS_PATCH + { return Some(PatchLoaded(())); } return self.apply_sensor_patch(st.patch_ack()); From f9e20bb5550b9ea18620294ca7a9394d0fe94535 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 08:47:09 +1100 Subject: [PATCH 0809/2680] tools: aurora-sep: exercise post-registration wake status Run the actual wake_sensor body with synthetic registration and status mocks. Check register-before-status ordering, stale accepted acknowledgement followed by needs-patch, unreadable and silent status, fresh accepted readiness, and missing acknowledgement. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/tests/test_sensor_wake.py | 96 ++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 tools/aurora-sep/tests/test_sensor_wake.py diff --git a/tools/aurora-sep/tests/test_sensor_wake.py b/tools/aurora-sep/tests/test_sensor_wake.py new file mode 100644 index 0000000000000..c9bb8c6af7579 --- /dev/null +++ b/tools/aurora-sep/tests/test_sensor_wake.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: MIT +"""Execute the actual wake_sensor body with synthetic status/registration mocks.""" +from pathlib import Path +import subprocess +import tempfile +import unittest + +APPLE = Path(__file__).resolve().parents[3] / 'drivers/soc/apple' + + +class SensorWakeTest(unittest.TestCase): + def test_post_registration_status_drives_patch_readiness(self): + source = (APPLE / 'sbio.rs').read_text() + start = source.index(' fn wake_sensor(') + opening = source.index('{', start) + depth, end = 1, opening + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + function = source[start:end] + program = r''' +#![allow(dead_code, unused_variables)] +use std::cell::RefCell; +macro_rules! dev_warn { ($($args:tt)*) => {} } +macro_rules! dev_err { ($($args:tt)*) => {} } +mod kernel { pub mod time { + pub struct Delta; + impl Delta { pub fn from_millis(_:i64)->Self {Self} } + pub mod delay { pub fn fsleep(_:super::Delta) {} } +}} +#[derive(Clone, Copy)] struct PatchLoaded(()); +#[derive(Default)] struct Mock { + reads: usize, registered: bool, patch: Option, next: Option, +} +thread_local! { static MOCK: RefCell = RefCell::new(Mock::default()); } +mod sensor { + pub const POWER_ON_READ_DELAYS_MS: [u32;1] = [0]; + pub const KNOWN_IDENTIFIERS: &[u16] = &[1]; + pub const PATCH_ACCEPTED: u8 = 0x29; + pub const STATE_NEEDS_PATCH: u8 = 9; + #[derive(Clone,Copy)] pub struct Status { pub state:u8, pub ack:u8, pub silent:bool } + pub struct Identifier; + pub enum Offset12 { Identifier(u16), Other } + pub struct Timing; + impl Timing { pub fn is_hardware(&self)->bool {true} pub fn name(&self)->&'static str {"mock"} } + pub fn power_source() {} + pub fn power_cycle()->bool {true} + pub fn cs_timing()->Timing {Timing} + pub fn status()->Result { + super::MOCK.with(|m| { let mut m=m.borrow_mut(); m.reads+=1; + if m.reads==1 {return Ok(Status{state:0,ack:PATCH_ACCEPTED,silent:false})} + assert!(m.registered, "post-registration read preceded registration"); + m.next.ok_or(()) + }) + } + impl Status { + pub fn is_silent(&self)->bool {self.silent} + pub fn offset12(&self)->Offset12 {Offset12::Identifier(1)} + pub fn identifier_to_register(&self)->Option {Some(Identifier)} + pub fn patch_ack(&self)->u8 {self.ack} + } +} +struct Driver; +impl Driver { + fn register_sensor(&self,_:&sensor::Identifier)->bool { + MOCK.with(|m| {let mut m=m.borrow_mut();assert_eq!(m.reads,1);m.registered=true;}); true + } + fn apply_sensor_patch(&self,ack:u8)->Option { + MOCK.with(|m| {let mut m=m.borrow_mut();assert_eq!(m.reads,2);m.patch=Some(ack);}); + Some(PatchLoaded(())) + } +''' + program += function + '\n}\n' + r''' +fn main() { + for (next,ready,patch) in [ + (Some(sensor::Status{state:9,ack:0x29,silent:false}),true,Some(0x29)), + (None,false,None), + (Some(sensor::Status{state:0,ack:0x29,silent:true}),false,None), + (Some(sensor::Status{state:0,ack:0x29,silent:false}),true,None), + (Some(sensor::Status{state:0,ack:0,silent:false}),true,Some(0)), + ] { + MOCK.with(|m| *m.borrow_mut()=Mock{next,..Mock::default()}); + assert_eq!(Driver.wake_sensor().is_some(),ready); + MOCK.with(|m| {let m=m.borrow();assert_eq!(m.reads,2);assert!(m.registered);assert_eq!(m.patch,patch);}); + } +} +''' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'test.rs').write_text(program) + subprocess.run(['rustc', '--edition=2021', str(root / 'test.rs'), '-o', str(root / 'test')], check=True) + subprocess.run([str(root / 'test')], check=True) + + +if __name__ == '__main__': + unittest.main() From 10dd5f4afc5324c87c53641e52a51d474b798441 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:06:47 +1100 Subject: [PATCH 0810/2680] soc: apple: sep: serialize SBIO exchanges The capture work item and the ioctl path both talk to the SBIO endpoint, which has one out-of-line buffer and one reassembly state. GET_INFO and ENROL_START list the enclave's identities before waiting for the last work item, so libfprint cancelling an identify and starting the next one lands that request in the middle of the previous work item's sensor registration. The second caller is then refused with EBUSY, or, when it starts between a completed reply and its collection, clears that reply and the first caller takes the second caller's answer. Hold an exchange lock for the whole request and reply, as the key store already does with sks_exchange_lock. The receive path does not take it. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 5 +++++ drivers/soc/apple/sep.rs | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 1dd46c3195918..ba21814a32a50 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2587,6 +2587,11 @@ impl SepData { name: &'static CStr, payload: &[u8], ) -> Result { + // The capture work item and the ioctl path both reach this endpoint. + // Without this, the second caller is refused with EBUSY, or clears a + // reply the first has not collected yet and takes the next one. Taken + // before the shutdown check, so a waiter never starts after remove(). + let _exchange = self.sbio_exchange_lock.lock(); if self.shutting_down.load(Relaxed) || !self.sbio_ready.load(Relaxed) { return Err(ENODEV); } diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 1ae6730de8717..6d1182b17b8f4 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -592,6 +592,11 @@ struct SepData { #[pin] dma_ring: Mutex>, + // One SBIO request/reply exchange at a time: the endpoint has a single + // out-of-line buffer and `sbio_rx` tracks a single transfer. + #[pin] + sbio_exchange_lock: Mutex<()>, + #[pin] sbio_rx: Mutex, @@ -858,6 +863,7 @@ impl SepData { scrd_probe <- new_mutex!(ScrdProbe::new()), scrd_wq <- new_condvar!("SepData::scrd_wq"), dma_ring <- new_mutex!(Some(dma_ring)), + sbio_exchange_lock <- new_mutex!(()), sbio_rx <- new_mutex!(transfer::Reassembly::new()), sbio_wq <- new_condvar!("SepData::sbio_wq"), sbio_ready: Atomic::new(false), From af01cd7d7c5657c10d4c572168b276bcb160c094 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:06:48 +1100 Subject: [PATCH 0811/2680] tools: aurora-sep: run concurrent SBIO exchanges Compile the driver's sbio_transfer_raw against a threaded enclave mock and run a work-item exchange and an ioctl exchange at the same time. Each must get its own reply. Without the exchange lock the ioctl side is refused with EBUSY in every round. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/tests/test_sbio_exchange.py | 174 +++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 tools/aurora-sep/tests/test_sbio_exchange.py diff --git a/tools/aurora-sep/tests/test_sbio_exchange.py b/tools/aurora-sep/tests/test_sbio_exchange.py new file mode 100644 index 0000000000000..177246001f759 --- /dev/null +++ b/tools/aurora-sep/tests/test_sbio_exchange.py @@ -0,0 +1,174 @@ +# SPDX-License-Identifier: MIT +"""Run the driver's actual SBIO transfer method from two threads at once. + +The verify/enrol work item and the ioctl path (GET_INFO and ENROL_START list +the enclave's identities, DELETE removes one) can reach the single SBIO +endpoint at the same time, e.g. when libfprint cancels an identify and starts +the next one while the previous work item is still registering the sensor. +Each exchange must complete with its own reply; none may be refused with +EBUSY or receive another caller's answer. +""" +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import unittest + +APPLE = Path(__file__).resolve().parents[3] / 'drivers/soc/apple' + + +def method(file, name): + source = (APPLE / file).read_text() + match = re.search(r'^ (?:pub\(crate\) )?fn ' + name + r'\(', source, re.M) + if match is None: + raise AssertionError(f'Missing driver method {name}') + opening = source.index('{', match.start()) + depth, end = 1, opening + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[match.start():end] + + +HARNESS = r''' +#![allow(dead_code, unused_variables, unused_imports)] +use std::sync::atomic::{AtomicBool, Ordering::Relaxed}; +use std::sync::mpsc; +use std::time::Duration; + +type CStr = str; +type Result = std::result::Result; +const ENODEV: i32 = -19; +const EBUSY: i32 = -16; +const ETIMEDOUT: i32 = -110; +const SBIO_TIMEOUT_MS: u64 = 2000; + +mod time { pub fn msecs_to_jiffies(ms: u64) -> u64 { ms } } + +pub enum CondVarTimeoutResult { Woken { jiffies: u64 }, Signal { jiffies: u64 }, Timeout } + +/// Kernel-shaped mutex: lock() returns a guard a CondVar can wait on. +struct Mutex(std::sync::Mutex); +struct Guard<'a, T>(Option>); +impl Mutex { + fn new(v: T) -> Self { Mutex(std::sync::Mutex::new(v)) } + fn lock(&self) -> Guard<'_, T> { Guard(Some(self.0.lock().unwrap())) } +} +impl<'a, T> std::ops::Deref for Guard<'a, T> { + type Target = T; + fn deref(&self) -> &T { self.0.as_ref().unwrap() } +} +impl<'a, T> std::ops::DerefMut for Guard<'a, T> { + fn deref_mut(&mut self) -> &mut T { self.0.as_mut().unwrap() } +} +struct CondVar(std::sync::Condvar); +impl CondVar { + fn wait_interruptible_timeout(&self, g: &mut Guard<'_, T>, j: u64) -> CondVarTimeoutResult { + let inner = g.0.take().unwrap(); + let (inner, r) = self.0.wait_timeout(inner, Duration::from_millis(j)).unwrap(); + g.0 = Some(inner); + if r.timed_out() { CondVarTimeoutResult::Timeout } else { CondVarTimeoutResult::Woken { jiffies: j / 2 } } + } + fn notify_all(&self) { self.0.notify_all() } +} + +mod transfer { + pub struct Completed { pub status: u32, pub payload: Vec } + /// Same state rules as drivers/soc/apple/transfer.rs Reassembly. + pub struct Reassembly { pub active: Option, pub done: Option } + impl Reassembly { + pub fn begin(&mut self, opcode: u32) -> super::Result<()> { + if self.active.is_some() { return Err(super::EBUSY); } + self.done = None; + self.active = Some(opcode); + Ok(()) + } + pub fn take_done(&mut self) -> Option { self.done.take() } + pub fn abort(&mut self) { + if self.active.take().is_some() { + self.done = Some(Completed { status: 0xdead, payload: Vec::new() }); + } + } + /// The enclave's reply: it echoes the opcode in the payload. + pub fn on_reply(&mut self, opcode: u32) { + if self.active == Some(opcode) { + self.active = None; + self.done = Some(Completed { status: 0, payload: opcode.to_le_bytes().to_vec() }); + } + } + } +} + +struct SepData { + shutting_down: AtomicBool, + sbio_ready: AtomicBool, + sbio_rx: Mutex, + sbio_wq: CondVar, + sbio_exchange_lock: Mutex<()>, + to_enclave: std::sync::Mutex>, +} + +impl SepData { + fn sbio_start(&self, opcode: u16, _name: &'static CStr, _payload: &[u8]) -> Result<()> { + self.to_enclave.lock().unwrap().send(u32::from(opcode)).unwrap(); + Ok(()) + } +METHOD +} + +fn main() { + let (tx, rx) = mpsc::channel::(); + let sep: &'static SepData = Box::leak(Box::new(SepData { + shutting_down: AtomicBool::new(false), + sbio_ready: AtomicBool::new(true), + sbio_rx: Mutex::new(transfer::Reassembly { active: None, done: None }), + sbio_wq: CondVar(std::sync::Condvar::new()), + sbio_exchange_lock: Mutex::new(()), + to_enclave: std::sync::Mutex::new(tx), + })); + // The enclave answers each request after a short delay, as the mailbox does. + std::thread::spawn(move || { + for opcode in rx { + std::thread::sleep(Duration::from_millis(30)); + sep.sbio_rx.lock().on_reply(opcode); + sep.sbio_wq.notify_all(); + } + }); + let mut failures = 0; + for round in 0..20u16 { + let worker = std::thread::spawn(move || { + // e.g. CLEAR_STATE from the work item's sensor registration + sep.sbio_transfer_raw(0x10 + round, "WORKER", &[]) + .map(|d| (d.status, d.payload)) + }); + std::thread::sleep(Duration::from_millis(5)); + // e.g. LIST_IDENTITIES from GET_INFO on the next identify + let ioctl = sep.sbio_transfer_raw(0x40 + round, "IOCTL", &[]).map(|d| (d.status, d.payload)); + let worker = worker.join().unwrap(); + let want_w = Ok((0, u32::from(0x10 + round).to_le_bytes().to_vec())); + let want_i = Ok((0, u32::from(0x40 + round).to_le_bytes().to_vec())); + if worker != want_w || ioctl != want_i { + eprintln!("round {round}: worker {:?} ioctl {:?}", worker, ioctl); + failures += 1; + } + } + assert_eq!(failures, 0, "concurrent SBIO exchanges interfered"); +} +''' + + +@unittest.skipUnless(shutil.which('rustc'), 'rustc required for driver method tests') +class SbioExchangeTest(unittest.TestCase): + def test_concurrent_exchanges_each_get_their_own_reply(self): + program = HARNESS.replace('METHOD', method('sbio.rs', 'sbio_transfer_raw')) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'test.rs').write_text(program) + subprocess.run(['rustc', '--edition=2021', str(root / 'test.rs'), + '-o', str(root / 'test')], check=True) + subprocess.run([str(root / 'test')], check=True) + + +if __name__ == '__main__': + unittest.main() From 7ff47b5246f31fa98f1fbe157e6119faba7bb088 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:17:40 +1100 Subject: [PATCH 0812/2680] spi: apple: Reword the hardware CS delay comment Describe what the pre- and post-delay register values do instead of naming the vendor driver they were observed in. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/spi-apple.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/spi/spi-apple.c b/drivers/spi/spi-apple.c index 21937bb50ba08..f18cc6e1b8d0e 100644 --- a/drivers/spi/spi-apple.c +++ b/drivers/spi/spi-apple.c @@ -112,10 +112,9 @@ #define APPLE_SPI_DELAY_CYCLES GENMASK(31, 16) /* - * These are the values used by AppleSPIMCController's hardware-delay path. - * The pre-delay asserts active-low CS and the post-delay releases it. The - * otherwise undocumented bit 11 is named for its observed role rather than - * assigning it a speculative electrical meaning. + * Hardware CS delays: the pre-delay asserts the active-low CS and the + * post-delay releases it. Bit 11 is undocumented and named for its observed + * role only. */ #define APPLE_SPI_DELAY_PRE_FLAGS (APPLE_SPI_DELAY_PRE_ARM | \ APPLE_SPI_DELAY_SET_CS | \ From fb7991430667ba9c094f228ef262942cad7d626d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 20:10:27 +1100 Subject: [PATCH 0813/2680] soc: apple: sep: Depend on OF_DYNAMIC and SPI The driver enables the SEP, DART and sensor nodes, which the device tree ships disabled, through a device-tree changeset, and sep.rs refuses to build without CONFIG_OF_DYNAMIC. Its sensor shim is an SPI driver. Say both in Kconfig, so a configuration without them cannot select the driver and then fail to build or link. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index d292b5130c62d..ae480b1f1b5e4 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -118,6 +118,8 @@ config APPLE_SEP depends on ARCH_APPLE depends on PM depends on RUST + depends on OF_DYNAMIC + depends on SPI depends on RUST_APPLE_MAILBOX depends on HW_RANDOM depends on TRUSTED_KEYS From 089a721fcdf19687f8e2593660b03dbe7bfa347d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 20:10:27 +1100 Subject: [PATCH 0814/2680] soc: apple: sep: Add KUnit tests for the message codecs and keybag identity Run the pure logic of the SEP driver under KUnit: - proto: control messages decode to the fields they were encoded from, only GET_ENTROPY uses the reserved tag, and the shared-memory registration encodes the IOVA and size as before, takes the largest window that fits the 8-bit page count and 32-bit page address, and refuses everything else. - keybag_identity: the four existing #[cfg(test)] cases, which the kernel build never ran, now run as a KUnit suite. The tests touch no hardware and are built only with CONFIG_APPLE_SEP_KUNIT_TEST. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 12 ++++++ drivers/soc/apple/keybag_identity.rs | 33 ++++++++------- drivers/soc/apple/proto.rs | 60 ++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 14 deletions(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index ae480b1f1b5e4..62a83309b1e58 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -152,6 +152,18 @@ config APPLE_SEP Say Y here if you have an Apple silicon Mac. +config APPLE_SEP_KUNIT_TEST + bool "KUnit tests for the Apple SEP driver" if !KUNIT_ALL_TESTS + depends on APPLE_SEP && KUNIT + default KUNIT_ALL_TESTS + help + Build KUnit tests for the pure logic of the Apple SEP driver: the + mailbox message encoding and decoding, the shared-memory registration + limits and the rules that keep a keybag's lookup identity apart from + the identity the enclave reports. The tests touch no hardware. + + If unsure, say N. + config APPLE_PMP tristate "Apple Power Management Processor" depends on ARCH_APPLE || COMPILE_TEST diff --git a/drivers/soc/apple/keybag_identity.rs b/drivers/soc/apple/keybag_identity.rs index 7022839cc30c7..0012dd65ce82a 100644 --- a/drivers/soc/apple/keybag_identity.rs +++ b/drivers/soc/apple/keybag_identity.rs @@ -45,47 +45,52 @@ impl SnapshotIdentity { } } -#[cfg(test)] +#[cfg(CONFIG_APPLE_SEP_KUNIT_TEST)] +#[::kernel::macros::kunit_tests(apple_sep_keybag_identity)] mod tests { use super::*; + use kernel::prelude::*; + const LOOKUP: [u8; 16] = [1; 16]; const BAG: [u8; 16] = [2; 16]; const OTHER: [u8; 16] = [3; 16]; #[test] - fn sepos13_keeps_lookup_and_bag_identities_distinct() { - let proof = SnapshotIdentity::from_loaded( - LOOKUP, UuidProvenance::AsGenerated, BAG, true, - ).unwrap(); + fn sepos13_keeps_lookup_and_bag_identities_distinct() -> Result { + let proof = SnapshotIdentity::from_loaded(LOOKUP, UuidProvenance::AsGenerated, BAG, true) + .ok_or(EINVAL)?; assert!(proof.accepts(&LOOKUP, UuidProvenance::AsGenerated, &BAG)); assert!(!proof.accepts(&BAG, UuidProvenance::AsGenerated, &BAG)); assert!(!proof.accepts(&LOOKUP, UuidProvenance::AsGenerated, &LOOKUP)); + Ok(()) } #[test] - fn changed_record_or_designated_bag_is_rejected() { - let proof = SnapshotIdentity::from_loaded( - LOOKUP, UuidProvenance::AsGenerated, BAG, true, - ).unwrap(); + fn changed_record_or_designated_bag_is_rejected() -> Result { + let proof = SnapshotIdentity::from_loaded(LOOKUP, UuidProvenance::AsGenerated, BAG, true) + .ok_or(EINVAL)?; assert!(!proof.accepts(&OTHER, UuidProvenance::AsGenerated, &BAG)); assert!(!proof.accepts(&LOOKUP, UuidProvenance::ReadBackFromBag, &BAG)); assert!(!proof.accepts(&LOOKUP, UuidProvenance::AsGenerated, &OTHER)); + Ok(()) } #[test] - fn blob_recovery_keeps_strict_uuid_equality() { + fn blob_recovery_keeps_strict_uuid_equality() -> Result { for provenance in [UuidProvenance::AsGenerated, UuidProvenance::ReadBackFromBag] { assert!(SnapshotIdentity::from_loaded(LOOKUP, provenance, BAG, false).is_none()); - let proof = SnapshotIdentity::from_loaded(BAG, provenance, BAG, false).unwrap(); + let proof = SnapshotIdentity::from_loaded(BAG, provenance, BAG, false).ok_or(EINVAL)?; assert!(proof.accepts(&BAG, provenance, &BAG)); assert!(!proof.accepts(&BAG, provenance, &OTHER)); } + Ok(()) } #[test] fn readback_record_cannot_claim_generated_lookup() { - assert!(SnapshotIdentity::from_loaded( - LOOKUP, UuidProvenance::ReadBackFromBag, BAG, true, - ).is_none()); + assert!( + SnapshotIdentity::from_loaded(LOOKUP, UuidProvenance::ReadBackFromBag, BAG, true) + .is_none() + ); } } diff --git a/drivers/soc/apple/proto.rs b/drivers/soc/apple/proto.rs index de2bf4580d84a..8fc7278550d9a 100644 --- a/drivers/soc/apple/proto.rs +++ b/drivers/soc/apple/proto.rs @@ -263,3 +263,63 @@ impl ControlReply { } } } + +#[cfg(CONFIG_APPLE_SEP_KUNIT_TEST)] +#[::kernel::macros::kunit_tests(apple_sep_proto)] +mod tests { + use super::*; + + #[test] + fn control_message_round_trips() { + let op = op_ool_inbound_size(EP_SBIO, 0x1234_5678); + let msg = encode_control(&op, 0x42); + assert_eq!(msg.msg1, 0); + let f = decode(&msg); + assert_eq!(f.ep, EP_CONTROL); + assert_eq!(f.tag, 0x42); + assert_eq!(f.ty, OP_OOL_INBOUND_SIZE); + assert_eq!(f.param, EP_SBIO); + assert_eq!(f.data_lo, 0x1234_5678); + } + + #[test] + fn only_entropy_uses_the_reserved_tag() { + assert!(op_get_entropy().uses_reserved_tag()); + assert!(!op_nop(0).uses_reserved_tag()); + assert!(!op_security_mode().uses_reserved_tag()); + assert!(TAG_ENTROPY > TAG_POOL_LAST); + } + + #[test] + fn shmem_registration_encodes_iova_and_size() -> Result { + let msg = shmem_registration(0xBEE0_0000, 0x4_0000)?; + assert_eq!(msg.msg0, 0x000b_ee00_0040_00fe); + assert_eq!(msg.msg1, 0); + let f = decode(&msg); + assert_eq!(f.ep, EP_SHMEM); + assert_eq!(f.ty, 0x40); + Ok(()) + } + + #[test] + fn shmem_registration_takes_the_largest_encodable_window() -> Result { + let iova = u64::from(u32::MAX) << IOVA_SHIFT; + let msg = shmem_registration(iova, 0xff << IOVA_SHIFT)?; + let f = decode(&msg); + assert_eq!(f.ty, 0xff); + assert_eq!(f.data_lo, u32::MAX); + Ok(()) + } + + #[test] + fn shmem_registration_rejects_unencodable_windows() { + // Empty, or not a whole number of pages. + assert!(shmem_registration(0xBEE0_0000, 0).is_err()); + assert!(shmem_registration(0xBEE0_0000, 0x4_0800).is_err()); + assert!(shmem_registration(0xBEE0_0800, 0x4_0000).is_err()); + // The size field is eight bits of pages. + assert!(shmem_registration(0xBEE0_0000, 0x100 << IOVA_SHIFT).is_err()); + // The IOVA field is 32 bits of pages. + assert!(shmem_registration(1u64 << (32 + IOVA_SHIFT), 0x1000).is_err()); + } +} From 616ebc8a2850c89cbfaf20577027aebb3e240d14 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:10:07 -0400 Subject: [PATCH 0815/2680] drm/sched: separate Rust job timeout state from submission state drm_sched_job_begin() arms the timeout before run_job() returns, so the timeout worker can run while the submit worker still owns the job. Both Rust JobImpl callbacks exclusively borrowed the whole Job. Give the timeout worker its own TimeoutData field, pass the finished status explicitly, and keep run/prepare/cancel on the submission half only. Let a backend that returns ENODEV without reset recovery keep the detached job owned by the scheduler (retain_job_on_enodev) so it reaches normal free work after its parent completes instead of being orphaned. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e22376a0ca679b5359fdef009b2479616619677b) --- drivers/gpu/drm/asahi/g17/job.rs | 29 ++++++++++----- drivers/gpu/drm/asahi/queue/mod.rs | 26 ++++++++------ drivers/gpu/drm/scheduler/sched_main.c | 16 +++++++-- include/drm/gpu_scheduler.h | 13 +++++++ rust/kernel/drm/sched.rs | 49 ++++++++++++++++++-------- 5 files changed, 97 insertions(+), 36 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 36dc49c144da6..bd22c5d5ff46c 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -255,6 +255,11 @@ pub(crate) trait Backend: Send + Sync { } struct Job { + backend: Arc, + packet: Arc, +} + +struct Timeout { backend: Arc, victim_renewals: u8, packet: Arc, @@ -272,10 +277,19 @@ impl Drop for Job { } impl sched::JobImpl for Job { + type TimeoutData = Timeout; const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); - fn false_timeout(job: &mut sched::Job) -> bool { - if job.is_finished() || job.packet.completion.take_replay_timeout() { + fn timeout_data(&self) -> Timeout { + Timeout { + backend: self.backend.clone(), + victim_renewals: 0, + packet: self.packet.clone(), + } + } + + fn false_timeout(job: &mut Timeout, finished: bool) -> bool { + if finished || job.packet.completion.take_replay_timeout() { return true; } let deferred = job.backend.defer_timeout(&job.packet, job.victim_renewals); @@ -285,22 +299,22 @@ impl sched::JobImpl for Job { deferred } - fn prepare(job: &mut sched::Job) -> Option { + fn prepare(job: &mut Self) -> Option { job.backend.prepare(&job.packet) } - fn run(job: &mut sched::Job) -> Result> { + fn run(job: &mut Self) -> Result> { job.backend.publish(job.packet.clone()).map(Some) } - fn timed_out(job: &mut sched::Job) -> sched::Status { + fn timed_out(job: &mut Timeout, finished: bool) -> sched::Status { // The scheduler retains the detached job throughout this callback. Keep // its parent callback intact while the driver settles the exact owner; // reinsertion also handles completion racing with timeout observation. - if !job.is_finished() { + if !finished { job.backend.timed_out(job.packet.clone()); } sched::Status::NoHang } - fn cancel(job: &mut sched::Job) { + fn cancel(job: &mut Self) { job.backend.cancel(job.packet.clone()); } } @@ -355,7 +369,6 @@ impl Scheduler { 1, Job { backend: self.backend.clone(), - victim_renewals: 0, packet: packet.clone(), }, )?; diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index f1ab56e80fd1a..0e327ffef2e82 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -198,7 +198,11 @@ pub(crate) struct QueueJob { fence: UserFence, notifier: Arc>, notification_count: u32, - did_run: bool, + id: u64, +} + +pub(crate) struct QueueTimeout { + dev: AsahiDevRef, id: u64, } @@ -246,9 +250,14 @@ impl QueueJob::ver { #[versions(AGX)] impl sched::JobImpl for QueueJob::ver { + type TimeoutData = QueueTimeout; const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); - fn prepare(job: &mut sched::Job) -> Option { + fn timeout_data(&self) -> QueueTimeout { + QueueTimeout { dev: self.dev.clone(), id: self.id } + } + + fn prepare(job: &mut Self) -> Option { mod_dev_dbg!(job.dev, "QueueJob {}: Checking runnability\n", job.id); if let Some(sj) = job.sj_vtx.as_ref() { @@ -285,7 +294,7 @@ impl sched::JobImpl for QueueJob::ver { } #[allow(unused_assignments)] - fn run(job: &mut sched::Job) -> Result> { + fn run(job: &mut Self) -> Result> { mod_dev_dbg!(job.dev, "QueueJob {}: Running Job\n", job.id); // We can only increase the notifier threshold here, now that we are @@ -411,23 +420,21 @@ impl sched::JobImpl for QueueJob::ver { mod_dev_dbg!(job.dev, "QueueJob {}: Drop fragment job\n", job.id); core::mem::drop(frag_job); - job.did_run = true; - Ok(Some(Fence::from_fence(&job.fence))) }); } - fn timed_out(job: &mut sched::Job) -> sched::Status { + fn timed_out(job: &mut QueueTimeout, _finished: bool) -> sched::Status { // FIXME: Handle timeouts properly dev_err!( job.dev.as_ref(), - "QueueJob {}: Job timed out on the DRM scheduler, things will probably break (ran: {})\n", - job.id, job.did_run + "QueueJob {}: Job timed out on the DRM scheduler, things will probably break\n", + job.id ); sched::Status::NoDevice } - fn cancel(job: &mut sched::Job) { + fn cancel(job: &mut Self) { dev_info!( job.dev.as_ref(), "QueueJob {}: Job canceled on DRM scheduler teardown\n", @@ -764,7 +771,6 @@ impl Queue for Queue::ver { // notification count for the job. notification_count: (2 * nr_render) + nr_compute, - did_run: false, id, }, )?; diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index 48cecf0d6bce0..57a6f48c0fd42 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -361,10 +361,20 @@ static void drm_sched_job_timedout(struct work_struct *work) if (sched->free_guilty) { job->sched->ops->free_job(job); sched->free_guilty = false; - } - - if (status == DRM_GPU_SCHED_STAT_NO_HANG) + } else if (status == DRM_GPU_SCHED_STAT_NO_HANG) { drm_sched_job_reinsert_on_false_timeout(sched, job); + } else if (status == DRM_GPU_SCHED_STAT_ENODEV && + sched->ops->retain_job_on_enodev) { + /* Preserve ownership without changing the backend's terminal + * result into NO_HANG or restarting its watchdog. A parent + * signaled during the callback must also reach free work. + */ + spin_lock(&sched->job_list_lock); + if (list_empty(&job->list)) + list_add(&job->list, &sched->pending_list); + drm_sched_run_free_queue(sched); + spin_unlock(&sched->job_list_lock); + } } else { spin_unlock(&sched->job_list_lock); } diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index 2a6415eea8d9a..8cf41d7708406 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -577,6 +577,19 @@ struct drm_sched_backend_ops { * it only after free_job returns to the scheduler core. */ struct module *owner; + + /** + * @retain_job_on_enodev: Keep the detached timeout job owned by the + * scheduler when timedout_job returns ENODEV without reset recovery. + * The backend must not free that job itself. Reinsertion happens only + * after the callback returns, so its parent may complete concurrently + * without freeing storage still borrowed by the callback. This does not + * rearm the timeout, signal a hardware fence or establish GPU quiescence. + * Completed jobs use normal free work; unfinished jobs remain reachable + * by final cancellation. Backends using their own reset ownership leave + * this false. + */ + bool retain_job_on_enodev; }; /** diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index 2b798de4d18ca..5d0c785e103a6 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -43,6 +43,13 @@ pub enum Priority { /// Trait to be implemented by driver job objects. pub trait JobImpl: Sized { + /// State owned exclusively by the timeout worker. It must not borrow the + /// mutable submission state: timeout can run before `run` returns. + type TimeoutData; + + /// Snapshot or independently retain timeout state before scheduler ownership. + fn timeout_data(&self) -> Self::TimeoutData; + /// Module implementing the callbacks. Its caller must retain the module /// through entity destruction; the core pins detached kill-job callbacks. const MODULE: Option<&'static crate::ThisModule> = None; @@ -59,28 +66,28 @@ pub trait JobImpl: Sized { /// Reconsider a timeout before stopping the scheduler or settling fences. /// Returning true asks the core to reinsert the retained job and rearm. /// The backend must not stop/start the scheduler from this hook. - fn false_timeout(_job: &mut Job) -> bool { false } + fn false_timeout(_data: &mut Self::TimeoutData, _finished: bool) -> bool { false } /// Called when the scheduler is considering scheduling this job next, to get another Fence /// for this job to block on. Once it returns None, run() may be called. - fn prepare(_job: &mut Job) -> Option { + fn prepare(_job: &mut Self) -> Option { None // Equivalent to NULL function pointer } /// Called to execute the job once all of the dependencies have been resolved. This may be /// called multiple times, if timed_out() has happened and drm_sched_job_recovery() decides /// to try it again. - fn run(job: &mut Job) -> Result>; + fn run(job: &mut Self) -> Result>; /// Called when a job has taken too long to execute, to trigger GPU recovery. /// /// This method is called in a workqueue context. - fn timed_out(job: &mut Job) -> Status; + fn timed_out(data: &mut Self::TimeoutData, finished: bool) -> Status; /// Called for remaining jobs in drm_sched_fini() to ensure the job's fences /// get signalled before the scheduler is torn down. - fn cancel(job: &mut Job); + fn cancel(job: &mut Self); } unsafe extern "C" fn prepare_job_cb( @@ -90,8 +97,9 @@ unsafe extern "C" fn prepare_job_cb( // SAFETY: All of our jobs are Job. let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; - // SAFETY: All of our jobs are Job. - match T::prepare(unsafe { &mut *p }) { + // SAFETY: prepare/run are serialized by submit work. Only borrow inner; + // the timeout worker independently owns timeout_data and C owns job. + match T::prepare(unsafe { &mut *addr_of_mut!((*p).inner) }) { None => core::ptr::null_mut(), Some(fence) => fence.into_raw(), } @@ -103,8 +111,9 @@ unsafe extern "C" fn run_job_cb( // SAFETY: All of our jobs are Job. let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; - // SAFETY: All of our jobs are Job. - match T::run(unsafe { &mut *p }) { + // SAFETY: submit work owns inner until return, even while timeout work + // accesses the disjoint timeout_data field. Do not borrow the whole Job. + match T::run(unsafe { &mut *addr_of_mut!((*p).inner) }) { Err(e) => e.to_ptr(), Ok(None) => core::ptr::null_mut(), Ok(Some(fence)) => fence.into_raw(), @@ -119,17 +128,20 @@ unsafe extern "C" fn timedout_job_cb( // The C timeout worker owns this job outside pending_list until return. // Free work cannot release it even if its parent signals concurrently. + let finished = unsafe { addr_of_mut!((*(*sched_job).s_fence).finished) }; + let is_finished = || unsafe { bindings::dma_fence_get_status(finished) != 0 }; if T::MANAGED_TIMEOUT_RECOVERY { - let finished = unsafe { addr_of_mut!((*(*sched_job).s_fence).finished) }; - if unsafe { bindings::dma_fence_get_status(finished) } != 0 { + if is_finished() { return bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG; } } - // SAFETY: the C core retains the detached job across both callbacks. - if T::false_timeout(unsafe { &mut *p }) { + // SAFETY: the C core retains the detached job and serializes its timeout + // worker. This field is disjoint from inner, which run may still mutate. + let data = unsafe { &mut *addr_of_mut!((*p).timeout_data) }; + if T::false_timeout(data, is_finished()) { return bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG; } - let status = T::timed_out(unsafe { &mut *p }); + let status = T::timed_out(data, is_finished()); if T::MANAGED_TIMEOUT_RECOVERY { // This is scoped fence handling, not C reset recovery. stop would // detach and discard healthy parents; start would cancel those jobs. @@ -190,7 +202,7 @@ unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sch // SAFETY: cancellation cannot race a scheduler callback using this job // after the removal/parent-lock synchronization above. The backend must // retain any still-hardware-owned resources independently of Job storage. - T::cancel(unsafe { &mut *p }); + T::cancel(unsafe { &mut *addr_of_mut!((*p).inner) }); if finish_here { // Match drm_sched_job_done's accounting exactly once, only when its @@ -216,6 +228,7 @@ unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sch pub struct Job { job: bindings::drm_sched_job, inner: T, + timeout_data: T::TimeoutData, // Entity destruction can leave kill_jobs_work queued on the system // workqueue or waiting for a dependency. That work still dereferences // job->sched->ops after drm_sched_fini has stopped the scheduler workers. @@ -402,6 +415,7 @@ impl Entity { /// in flight at once. pub fn new_job(&mut self, credits: u32, inner: T) -> Result> { let mut job: KBox>> = Box::new_uninit(GFP_KERNEL | __GFP_ZERO)?; + let timeout_data = inner.timeout_data(); // SAFETY: We hold a reference to the entity (which is a valid pointer), // and the job object was just allocated above. @@ -417,6 +431,8 @@ impl Entity { // SAFETY: The Box pointer is valid, and this initializes the inner member. unsafe { addr_of_mut!((*job.as_mut_ptr()).inner).write(inner) }; + // SAFETY: initialize the independent timeout worker's owned state. + unsafe { addr_of_mut!((*job.as_mut_ptr()).timeout_data).write(timeout_data) }; // SAFETY: The job allocation is valid and this initializes its last // field. Arc::clone is infallible and the entity still owns the live @@ -472,6 +488,9 @@ impl Scheduler { timedout_job: Some(timedout_job_cb::), free_job: Some(free_job_cb::), cancel_job: Some(cancel_job_cb::), + // A terminal backend result must not orphan the detached Job. The + // core restores ownership after all Rust callback borrows end. + retain_job_on_enodev: true, owner: match T::MODULE { Some(module) => module.as_ptr(), None => core::ptr::null_mut(), From c368a2b6a7a54cb00826144834ebbe2629795877 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:10:08 -0400 Subject: [PATCH 0816/2680] drm/asahi: observe completed render work before a deferred VM failure A VM failure that arrives while preparation owns a render pair is applied later from the deferred path, which quarantined the pair at once. Retire its completed commands first, as settle_render_vm() does, before quarantining what is still owned. Keep the pending error if the observation itself fails. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 3e464649d37234b0f47dfe3c22d3f73cbb6ef45b) --- drivers/gpu/drm/asahi/g17/runtime/render/submit.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 59a54b1926375..8b6c647668af4 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -206,6 +206,20 @@ impl crate::g17::Firmware { slot: u8, deferred: &mut DeferredBatch, ) -> Result { + let observe_before_failure = { + let entry = self.queues.render.entry_mut(slot)?; + let pair = entry.pair.as_ref().ok_or(EIO)?; + entry.pending_error.as_ref().is_some_and(|(status, _)| { + Arc::ptr_eq(pair.context().status(), status) + }) + }; + if observe_before_failure { + // VM failure could not observe this pair while preparation owned + // it. Restore the same final-observation boundary used by + // settle_render_vm before quarantining any still-owned commands. + // Keep pending_error intact if observation itself fails. + self.retire_render_slot(usize::from(slot), None, deferred)?; + } let entry = self.queues.render.entry_mut(slot)?; let pair = entry.pair.as_mut().ok_or(EIO)?; if let Some((status, error)) = entry.pending_error.take() { From c6a99932e98c1801bcacf1722d6fe6db1f640f99 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:10:08 -0400 Subject: [PATCH 0817/2680] drm/asahi: publish child page tables before linking them Complete the child table initialization before storing the parent entry, so a concurrent device walk cannot follow a visible parent to a table whose clears have not reached memory yet. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 83cf9f5ad0baf71001cc0aa9c25c83f27f4a219a) --- drivers/gpu/drm/asahi/pgtable.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 0ad231b930a8a..af5fdbb5dc27b 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -387,6 +387,11 @@ impl UatPageTable { } let pt_paddr = Page::into_phys(pt_page); upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; + // Complete child-table initialization before a + // concurrent device walk can follow its parent. + // A later mapping/TLBI barrier cannot order an + // already-visible parent against earlier clears. + crate::mem::sync(); upte.store(upte_val, Ordering::Relaxed); } if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { From 5c888bec676fc42c39c1619938af6a96a0ef89c8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:10:09 -0400 Subject: [PATCH 0818/2680] drm/asahi: retire deferred mapping cleanup without allocating MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deferred user unmaps keep their preallocated split nodes and retire through immediate-mode drm_gpuvm_sm_unmap() without an exec context. Whole-object unmaps that hit ENOMEM fall back to an allocation-free removal of that GEM's nodes. A partially committed bind batch defers only its uncommitted operations, so committed prefixes are never replayed over a range another binding may now occupy. Validate unmap ranges against the VM geometry at preparation. A cleanup failure keeps admission blocked and is recorded on the VM status rather than silently reopening the VM over stale leaves. [ryan: adapted to 7.2.10 — minimal; for DJ: the rust/kernel/drm/gpuvm.rs hunks (range_valid(), sm_unmap_inner(), unmap_object()) are ported into rust/kernel/drm/gpuvm/mod.rs; sm_unmap_inner() runs the 7.2 sm_unmap() through a UniqueRefGpuVm under the shared reservation, and the deferred unmaps keep their 7.2 VmOpResources split nodes instead of a 7.1 StepContext] Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit b321f0e7053949d9144028d0a465a4f03eb6cc83) --- drivers/gpu/drm/asahi/mmu.rs | 39 ++++++++++++++-- drivers/gpu/drm/asahi/mmu/bind.rs | 15 ++++-- drivers/gpu/drm/asahi/mmu/lifetime.rs | 67 ++++++++++++++++++--------- rust/kernel/drm/gpuvm/mod.rs | 58 ++++++++++++++++++++++- 4 files changed, 151 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index fd83fd212224b..b22a8178c8b6e 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -2109,8 +2109,19 @@ impl Vm { self.unmap_range_commit(dev, iova, size, true) } - fn unmap_range_now(&self, iova: u64, size: u64) -> Result { - self.unmap_range_commit(None, iova, size, false) + fn unmap_prepared_cleanup(&self, unmap: &mut bind::PreparedUserUnmap) -> Result { + let mut inner = self.gpuvm.lock_inner(); + let mut ctx = StepContext::new(None, Prot::default(), &mut unmap.resources)?; + // SAFETY: step_unmap/step_remap modify this VM's translations and + // immediate GPUVA metadata only, using the retained split nodes. They + // do not access external BO backing and defer all BO reference puts. + let result = unsafe { inner.sm_unmap_inner(unmap.iova, unmap.size, &mut ctx) }; + if result.is_ok() { + self.untrack_shared_range(unmap.iova, unmap.size); + } + drop(inner); + self.bo_deferred_cleanup(); + result } fn unmap_range_commit<'drm>( @@ -2138,6 +2149,7 @@ impl Vm { match self.mapping_commit() { Ok(guard) => guard, Err(error) => { + drop(vm_exec); if self.defer_unmap(iova, size)? { return Ok(()); } @@ -2159,7 +2171,11 @@ impl Vm { }; let mut ctx = StepContext::new(None, Prot::default(), &mut resources)?; - let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; + // SAFETY: See `lock_vm()`. + let mut vm_exec = match unsafe { self.gpuvm.exec_lock(None, false) } { + Err(error) if error == ENOMEM => return self.drop_mappings_low_memory(gem), + result => result?, + }; if let Some(bo) = vm_exec.unique_gpuvm.find_bo(gem) { mod_dev_dbg!(self.dev, "MMU: bo_unmap\n"); @@ -2175,11 +2191,28 @@ impl Vm { // unmap error. Always release reservations before dropping it. core::mem::drop(vm_exec); core::mem::drop(bo); + if result == Err(ENOMEM) { + return self.drop_mappings_low_memory(gem); + } result?; } Ok(()) } + /// Preserve the indexed fast path normally; memory pressure must not lose + /// accepted object cleanup just because an exec or operation list failed. + fn drop_mappings_low_memory(&self, gem: &gem::Object) -> Result { + let mut inner = self.gpuvm.lock_inner(); + let result = inner.unmap_object(gem, VmInner::unmap_gpuva); + if result.is_ok() { + self.untrack_context_object(gem); + self.untrack_shared_object(gem); + } + drop(inner); + self.bo_deferred_cleanup(); + result + } + /// Returns the dummy GEM object used to hold the shared DMA reservation locks pub(crate) fn get_resv_obj(&self) -> ARef { self.dummy_obj.clone() diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index 35d5aaaae912b..9a6065f355bb7 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -21,9 +21,15 @@ pub(crate) struct PreparedUserMap { pub(crate) struct PreparedUserUnmap { pub(super) iova: u64, pub(super) size: u64, - resources: VmOpResources, + pub(super) resources: VmOpResources, } +// SAFETY: This owns only its numeric range and unused, unlinked split-node +// allocations. step_remap moves each node out of ctx before linking it into +// the VM, so a retained ctx never owns a linked GPUVA. The unmap context has +// no borrowed object/lock state; lifetime.state serializes deferred ownership. +unsafe impl Send for PreparedUserUnmap {} + pub(super) enum PreparedUserBindOp { Map(PreparedUserMap), Unmap(PreparedUserUnmap), @@ -100,6 +106,9 @@ impl Vm { } pub(crate) fn prepare_user_unmap(&self, iova: u64, size: u64) -> Result { + if (iova | size) & UAT_PGMSK as u64 != 0 || !self.gpuvm.range_valid(iova, size) { + return Err(EINVAL); + } Ok(PreparedUserUnmap { iova, size, @@ -119,7 +128,7 @@ impl Vm { &self, batch: &mut PreparedUserBindBatch, ) -> Result { - if self.defer_bind_batch(batch)? { + if self.defer_bind_batch(batch, 0)? { return Ok(()); } let mut index = 0; @@ -205,7 +214,7 @@ impl Vm { if let Some(result) = result { result?; index += 1; - } else if self.defer_bind_batch(batch)? { + } else if self.defer_bind_batch(batch, index)? { return Ok(()); } } diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index f819de8becd1b..2f7acfecb3e70 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -54,8 +54,9 @@ struct State { quarantined: bool, draining: bool, closed: bool, + cleanup_failed: bool, close_ranges: Option<(Range, Range)>, - unmaps: KVec<(u64, u64)>, + unmaps: KVec, objects: KVec>, } @@ -74,7 +75,7 @@ impl VmLifetime { Arc::pin_init( pin_init!(Self { state <- new_mutex!(State { active:0, commits:0, blocked:false, - quarantined:false, draining:false, closed:false, close_ranges:None, + quarantined:false, draining:false, closed:false, cleanup_failed:false, close_ranges:None, unmaps:KVec::new(), objects:KVec::new() }, "VM job lifetime"), changed <- new_condvar!("VM mapping admission"), driver <- new_mutex!(None, "VM driver mappings"), @@ -414,7 +415,7 @@ impl Vm { Ok(()) } - pub(super) fn defer_bind_batch(&self, batch: &PreparedUserBindBatch) -> Result { + pub(super) fn defer_bind_batch(&self, batch: &mut PreparedUserBindBatch, start: usize) -> Result { use super::bind::PreparedUserBindOp; let Some(lifetime) = self.lifetime.as_ref() else { return Ok(false); @@ -428,7 +429,7 @@ impl Vm { } if batch .ops - .iter() + .iter().skip(start) .any(|op| matches!(op, PreparedUserBindOp::Map(_))) { return Err(EIO); @@ -438,19 +439,21 @@ impl Vm { } // Reserve before accepting any range; a partial deferred batch must // not leave canonical metadata ahead of the accepted cleanup. - state.unmaps.reserve(batch.ops.len(), GFP_KERNEL)?; - for op in &batch.ops { + state.unmaps.reserve(batch.ops.len() - start, GFP_KERNEL)?; + let retired = self.untrack_context_ranges(batch.ops.iter().skip(start).filter_map(|op| match op { + PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), + PreparedUserBindOp::Map(_) => None, + })); + // Transfer the preallocated split nodes along with each accepted range. + // Already-committed prefix operations must not be replayed: their nodes + // may have been consumed and another binding may now occupy the range. + for op in batch.ops.drain(start..) { if let PreparedUserBindOp::Unmap(unmap) = op { - let range = (unmap.iova, unmap.size); - if !state.unmaps.contains(&range) { - state.unmaps.push(range, GFP_KERNEL)?; + if !state.unmaps.iter().any(|old| old.iova == unmap.iova && old.size == unmap.size) { + state.unmaps.push(unmap, GFP_KERNEL)?; } } } - let retired = self.untrack_context_ranges(batch.ops.iter().filter_map(|op| match op { - PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), - PreparedUserBindOp::Map(_) => None, - })); drop(state); drop(retired); Ok(true) @@ -461,7 +464,7 @@ impl Vm { let Some(lifetime) = self.lifetime.as_ref() else { return Ok(false); }; - let mut state = lifetime.state.lock(); + let state = lifetime.state.lock(); if state.closed { return Err(ENOENT); } @@ -471,8 +474,17 @@ impl Vm { if !state.quarantined { return Err(EBUSY); } - if !state.unmaps.contains(&(iova, size)) { - state.unmaps.push((iova, size), GFP_KERNEL)?; + if state.unmaps.iter().any(|old| old.iova == iova && old.size == size) { + return Ok(true); + } + drop(state); + let unmap = self.prepare_user_unmap(iova, size)?; + let mut state = lifetime.state.lock(); + if state.closed { return Err(ENOENT); } + if !state.blocked { return Ok(false); } + if !state.quarantined { return Err(EBUSY); } + if !state.unmaps.iter().any(|old| old.iova == iova && old.size == size) { + state.unmaps.push(unmap, GFP_KERNEL)?; } Ok(true) } @@ -557,24 +569,28 @@ impl Vm { /// Recheck queued cleanup and reopen under the same lock used to append it. fn drain_mappings(&self, lifetime: &VmLifetime) { let mut state = lifetime.state.lock(); + let mut failure = None; loop { let close = state.close_ranges.take(); let unmaps = core::mem::replace(&mut state.unmaps, KVec::new()); let objects = core::mem::replace(&mut state.objects, KVec::new()); drop(state); if let Some((user, kernel)) = close { - if self.unmap_user_ranges_now(user, kernel).is_err() { + if let Err(error) = self.unmap_user_ranges_now(user, kernel) { + failure.get_or_insert(error); pr_err!("MMU: deferred VM close failed\n"); } } else { - for (address, size) in unmaps { - if self.unmap_range_now(address, size).is_err() { + for mut unmap in unmaps { + if let Err(error) = self.unmap_prepared_cleanup(&mut unmap) { + failure.get_or_insert(error); pr_err!("MMU: deferred user unmap failed\n"); } } } for object in objects { - if self.drop_mappings_now(&object).is_err() { + if let Err(error) = self.drop_mappings_now(&object) { + failure.get_or_insert(error); pr_err!("MMU: deferred object unmap failed\n"); } } @@ -585,7 +601,16 @@ impl Vm { } state.quarantined = false; state.draining = false; - if !state.closed { + if let Some(error) = failure { + // Cleanup is allocation-free for accepted ranges and has a + // whole-node low-memory fallback for objects. An unexpected + // invariant error must not reopen admission over stale leaves. + // GPUVA owners retain backing until final VM close; report the + // terminal failure instead of silently discarding its outcome. + state.blocked = true; + state.cleanup_failed = true; + if let Some(status) = &self.status { status.record(error); } + } else if !state.closed && !state.cleanup_failed { state.blocked = false; } drop(state); diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs index e0db05c15a0ed..ed84414b0d25d 100644 --- a/rust/kernel/drm/gpuvm/mod.rs +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -247,6 +247,13 @@ impl GpuVm { unsafe { bindings::drm_gpuvm_is_extobj(self.as_raw(), obj.as_raw()) } } + /// HACK: added for the asahi downstream driver. Validate a range against the immutable + /// address-space bounds and the kernel cutout. + pub fn range_valid(&self, addr: u64, range: u64) -> bool { + // SAFETY: The borrowed VM retains its immutable geometry. + unsafe { bindings::drm_gpuvm_range_valid(self.as_raw(), addr, range) } + } + /// HACK: added for compatibility with asahi downstream driver. Only safe to use /// when the gpuvm user forgot the UniqueRefGpuVm it got from `GpuVm::new()`. pub unsafe fn exec_lock<'a>( @@ -334,13 +341,59 @@ pub struct GpuVmInnerGuard<'a, T: DriverGpuVm> { } impl GpuVmInnerGuard<'_, T> { + /// Run immediate-mode unmap steps without allocating an exec context. + /// + /// # Safety + /// + /// The driver's unmap and remap callbacks must touch only VM metadata and translations + /// protected by this reservation. They must not access external BO backing that requires its + /// own reservation. Any split-node storage must already be owned by `context`. + pub unsafe fn sm_unmap_inner( + &mut self, + addr: u64, + length: u64, + context: &mut T::SmContext<'_>, + ) -> Result { + // SAFETY: This guard retains the VM and its shared reservation. + if unsafe { (*self.gpuvm.as_raw()).flags } + & bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE + == 0 + { + return Err(EINVAL); + } + // INVARIANT: The shared reservation held by this guard excludes every other + // `UniqueRefGpuVm`, as for `GpuVm::exec_lock()`. + let mut unique = UniqueRefGpuVm(self.gpuvm.into()); + unique.sm_unmap(addr, length, context) + } + /// Remove every driver GPUVA of this immediate-mode VM, preserving its kernel cutout. /// /// `unmap` must remove the GPU translation of each complete GPUVA before returning success. /// Only the VM's shared reservation is held: no external BO reservation is taken and no split /// node or operation list is allocated. Call [`GpuVm::deferred_cleanup`] after releasing this /// guard, including when a callback failed after earlier mappings were removed. - pub fn unmap_all(&mut self, mut unmap: impl FnMut(&mut T, &GpuVa) -> Result) -> Result { + pub fn unmap_all(&mut self, unmap: impl FnMut(&mut T, &GpuVa) -> Result) -> Result { + self.unmap_matching(None, unmap) + } + + /// Allocation-free whole-node removal for one GEM object in an immediate-mode VM. + /// + /// Uses the same translation-removal and deferred-put contract as [`Self::unmap_all`]. This + /// scans the VM list and is intended as a low-memory cleanup fallback. + pub fn unmap_object( + &mut self, + object: &T::Object, + unmap: impl FnMut(&mut T, &GpuVa) -> Result, + ) -> Result { + self.unmap_matching(Some(object), unmap) + } + + fn unmap_matching( + &mut self, + object: Option<&T::Object>, + mut unmap: impl FnMut(&mut T, &GpuVa) -> Result, + ) -> Result { let vm = self.gpuvm.as_raw(); // SAFETY: The guard owns this live VM's shared reservation, and the VM is in immediate // mode, so unlinking takes the GEM GPUVA mutex independently of its reservation. Every @@ -359,6 +412,9 @@ impl GpuVmInnerGuard<'_, T> { if ptr::eq(va, &raw const (*vm).kernel_alloc_node) { continue; } + if object.is_some_and(|object| (*va).gem.obj != object.as_raw()) { + continue; + } unmap(&mut *self.gpuvm.data.get(), GpuVa::::from_raw(va))?; // Remove it as `OpUnmap::remove()` does. let op = bindings::drm_gpuva_op_unmap { va, keep: false }; From ea48c13c02cf73278b93177a5cc5e9c6e0161859 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:10:09 -0400 Subject: [PATCH 0819/2680] drm/asahi: keep compute timestamps outside shared queue aliases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Private per-command compute timestamp pages were allocated from the range later used for canonical shared queue aliases, so a late shared alias could collide with a live timestamp page. Allocate them from the client-private lower arena instead. [ryan: adapted to 7.2.10 — minimal; for DJ: the G17 code names the DRM device type AsahiDrmDevice, as asahi-wip does] Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 7d57556a8962dd49aadbb24af772ee5240b8c1c1) --- drivers/gpu/drm/asahi/g17/completion.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index aa29d107f27f3..6ab9bc24ffd1d 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -13,7 +13,7 @@ use super::{ recovery::Class, status::VmStatus, }; -use crate::{driver::AsahiDrmDevice, file, hw::t8140::resources as cfg, mmu}; +use crate::{driver::AsahiDrmDevice, file, hw::t8140::{dynamic, resources as cfg}, mmu}; use core::sync::atomic::{fence as barrier, AtomicBool, AtomicI32, AtomicU8, Ordering}; use kernel::{ dma_fence::{Fence, FenceContexts, RawDmaFence, UserFence}, @@ -156,7 +156,9 @@ impl ComputeTimestamps { let object = KernelObject::backing(dev, mmu::UAT_PGSZ, CpuMap::WriteCombined)?; let mapping = object.alias_in( vm, - cfg::TIMESTAMPS, + // Canonical shared queue aliases may be installed in this VM later. + // Private timestamp pages must never occupy their retained addresses. + dynamic::CLIENT_LOWER, mmu::UAT_PGSZ as u64, mmu::PROT_GPU_FW_SHARED_RW, )?; From 36ff1f200b5e8d840e897ab414dbb6986d8cbd60 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:10:32 -0400 Subject: [PATCH 0820/2680] drm/shmem-helper: index fault pages by object offset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fault, huge-fault and mkwrite paths computed the page index as vmf->pgoff - vma->vm_pgoff, the offset within the VMA. vm_pgoff keeps the DRM fake offset (the object's vma_node start, plus the mmap offset of a dma-buf mapping), so after a VMA split or an offset mmap that index names the wrong page. Subtract the object's vma_node start instead. [ryan: adapted to 7.2.10 — minimal; for DJ: in aurora-wip this commit is empty, because the userpages commit 332aad3d4111 had already added drm_gem_shmem_object_offset() and these two uses. 210-gpu does not sit on 220-userpages, so the helper and both uses are carried here with the identical text, which lets the two topics merge cleanly; the revocable CPU mappings below need it] Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 6db05cf4b231f020020f31a479bd45ccff83520f) --- drivers/gpu/drm/drm_gem_shmem_helper.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index 4b5ec19a2ca9b..715228acaae57 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -574,13 +574,23 @@ int drm_gem_shmem_dumb_create(struct drm_file *file, struct drm_device *dev, } EXPORT_SYMBOL_GPL(drm_gem_shmem_dumb_create); +/* GEM and PRIME add the object's fake offset; VMA splits retain it. + * Direct helper callers may instead supply an object-relative offset. + */ +static pgoff_t drm_gem_shmem_object_offset(struct drm_gem_object *obj, pgoff_t pgoff) +{ + pgoff_t base = drm_vma_node_start(&obj->vma_node); + + return pgoff >= base ? pgoff - base : pgoff; +} + static void drm_gem_shmem_record_mkwrite(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; struct drm_gem_object *obj = vma->vm_private_data; struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); loff_t num_pages = obj->size >> PAGE_SHIFT; - pgoff_t page_offset = vmf->pgoff - vma->vm_pgoff; /* page offset within VMA */ + pgoff_t page_offset = drm_gem_shmem_object_offset(obj, vmf->pgoff); if (drm_WARN_ON(obj->dev, !shmem->pages || page_offset >= num_pages)) return; @@ -639,7 +649,7 @@ static vm_fault_t drm_gem_shmem_any_fault(struct vm_fault *vmf, unsigned int ord loff_t num_pages = obj->size >> PAGE_SHIFT; vm_fault_t ret = VM_FAULT_SIGBUS; struct page **pages = shmem->pages; - pgoff_t page_offset = vmf->pgoff - vma->vm_pgoff; /* page offset within VMA */ + pgoff_t page_offset = drm_gem_shmem_object_offset(obj, vmf->pgoff); struct page *page; struct folio *folio; unsigned long pfn; From a45932a0322cd1d6cfee851a982cd4da7974ff7a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:11:49 -0400 Subject: [PATCH 0821/2680] drm/shmem-helper: add revocable CPU mappings and idle backing release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let a driver release the backing of an idle shmem object without purging it: drm_gem_shmem_release_idle_pages_locked() drops the cached SG table and page array once no pin, vmap, SG borrower or legacy CPU mapping remains, so ordinary shmem reclaim can swap the contents. The next get_pages_sgt() restores them, possibly at new DMA addresses. Objects created with reclaimable_cpu_mappings keep their CPU VMAs revocable: mmap no longer pins pages, faults take an object-scoped page reference under the reservation, release zaps the device anon mapping and the PTEs refault afterwards. Non-opted objects keep the existing behavior. The Rust shmem wrapper counts owned SG-table leases and marks the borrowed sg_table() accessor permanent, so release is refused while any Rust user still holds the table. Add the dma_resv_trylock helper used by non-blocking reclaim. [ryan: adapted to 7.2.10 — minimal; for DJ: squashed the header half of 4a98d86960c1 (cpu_pages, cpu_pages_count), whose C half is already in this commit, so it builds; the asahi ObjectConfig initializers set reclaimable_cpu_mappings to false; the mmap byte check from userpages is left to 220. The 7.2 Rust shmem object keeps its SG table in a set-once Devres and only lends it through sg_table(dev), so that borrow marks the lease counter permanent and release_idle_pages_locked() returns EBUSY for any object whose SG table was used: idle backing release is inert on 7.2 until the SG-table ownership is redesigned] Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 733634bcb0c4de6e9885878dc8b1aad744353c83) --- drivers/gpu/drm/asahi/gem.rs | 2 + drivers/gpu/drm/drm_gem_shmem_helper.c | 255 +++++++++++++++++++++++-- include/drm/drm_gem_shmem_helper.h | 17 ++ rust/helpers/dma-resv.c | 6 + rust/kernel/drm/gem/shmem.rs | 78 ++++++++ 5 files changed, 338 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 653944d1f38f1..0843897e0b4ee 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -189,6 +189,7 @@ fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> align(size, mmu::UAT_PGSZ), shmem::ObjectConfig:: { map_wc, + reclaimable_cpu_mappings: false, parent_resv_obj: None, }, AsahiObjConfig { @@ -219,6 +220,7 @@ pub(crate) fn new_object( align(size, mmu::UAT_PGSZ), shmem::ObjectConfig:: { map_wc: flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK == 0, + reclaimable_cpu_mappings: false, parent_resv_obj: parent_object, }, AsahiObjConfig { diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index 715228acaae57..886f04d847a58 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -7,7 +7,9 @@ #include #include #include +#include #include +#include #include #include @@ -189,10 +191,18 @@ EXPORT_SYMBOL_GPL(__drm_gem_shmem_free_sgt_locked); * This function cleans up the GEM object state, but does not free the memory used to store the * object itself. This function is meant to be a dedicated helper for the Rust GEM bindings. */ +static void drm_gem_shmem_put_cpu_pages_locked(struct drm_gem_shmem_object *shmem); + void drm_gem_shmem_release(struct drm_gem_shmem_object *shmem) { struct drm_gem_object *obj = &shmem->base; + if (shmem->cpu_pages) { + dma_resv_lock(obj->resv, NULL); + drm_gem_shmem_put_cpu_pages_locked(shmem); + dma_resv_unlock(obj->resv); + } + if (drm_gem_is_imported(obj)) { drm_prime_gem_destroy(obj, shmem->sgt); } else { @@ -512,25 +522,176 @@ int drm_gem_shmem_madvise_locked(struct drm_gem_shmem_object *shmem, int madv) } EXPORT_SYMBOL_GPL(drm_gem_shmem_madvise_locked); -void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem) +/* + * Revocable CPU mappings populate pages one at a time, so touching a few + * bytes of a large object neither materializes nor pins the whole object. + * The device path keeps using the full @pages array; both may coexist and + * reference the same page-cache pages. + */ +static struct page *drm_gem_shmem_cpu_page_locked(struct drm_gem_shmem_object *shmem, + pgoff_t index) { struct drm_gem_object *obj = &shmem->base; - struct drm_device *dev = obj->dev; + struct address_space *mapping = file_inode(obj->filp)->i_mapping; + struct page *page; - dma_resv_assert_held(shmem->base.resv); + dma_resv_assert_held(obj->resv); - drm_WARN_ON(obj->dev, !drm_gem_shmem_is_purgeable(shmem)); + if (!shmem->cpu_pages) { + shmem->cpu_pages = kvcalloc(obj->size >> PAGE_SHIFT, + sizeof(*shmem->cpu_pages), GFP_KERNEL); + if (!shmem->cpu_pages) + return ERR_PTR(-ENOMEM); + } + page = shmem->cpu_pages[index]; + if (page) + return page; + /* + * PFN mappings have no rmap, so reclaim could write the page to swap + * and later drop it while the PTE still names it. Keep the mapping + * unevictable while sparse pages exist, as drm_gem_get_pages() does + * for the device array; the flag is cleared when the last holder goes. + */ + mapping_set_unevictable(mapping); + page = shmem_read_mapping_page(mapping, index); + if (IS_ERR(page)) + return page; + shmem->cpu_pages[index] = page; + shmem->cpu_pages_count++; + return page; +} + +static void drm_gem_shmem_put_cpu_pages_locked(struct drm_gem_shmem_object *shmem) +{ + struct drm_gem_object *obj = &shmem->base; + unsigned long i, npages = obj->size >> PAGE_SHIFT; + struct folio_batch fbatch; + + dma_resv_assert_held(obj->resv); + + if (!shmem->cpu_pages) + return; + /* A device array still holding these pages keeps the mapping unevictable. */ + if (!shmem->pages) + mapping_clear_unevictable(file_inode(obj->filp)->i_mapping); + folio_batch_init(&fbatch); + for (i = 0; i < npages && shmem->cpu_pages_count; i++) { + struct folio *folio; + + if (!shmem->cpu_pages[i]) + continue; + folio = page_folio(shmem->cpu_pages[i]); + folio_mark_dirty(folio); + folio_mark_accessed(folio); + shmem->cpu_pages_count--; + if (!folio_batch_add(&fbatch, folio)) { + check_move_unevictable_folios(&fbatch); + __folio_batch_release(&fbatch); + } + } + if (folio_batch_count(&fbatch)) { + check_move_unevictable_folios(&fbatch); + __folio_batch_release(&fbatch); + } + kvfree(shmem->cpu_pages); + shmem->cpu_pages = NULL; +} - dma_unmap_sgtable(dev->dev, shmem->sgt, DMA_BIDIRECTIONAL, 0); +static struct page *drm_gem_shmem_present_page(struct drm_gem_shmem_object *shmem, + pgoff_t index) +{ + if (shmem->pages && shmem->pages[index]) + return shmem->pages[index]; + if (shmem->cpu_pages) + return shmem->cpu_pages[index]; + return NULL; +} + +bool drm_gem_shmem_idle_pages_releasable_locked(struct drm_gem_shmem_object *shmem) +{ + struct drm_gem_object *obj = &shmem->base; + + dma_resv_assert_held(obj->resv); + return shmem->sgt && !drm_gem_is_imported(obj) && !obj->dma_buf && + shmem->madv >= 0 && !refcount_read(&shmem->pages_pin_count) && + !refcount_read(&shmem->vmap_use_count) && + refcount_read(&shmem->pages_use_count) == 1 && + dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_BOOKKEEP); +} +EXPORT_SYMBOL_GPL(drm_gem_shmem_idle_pages_releasable_locked); + +/** + * drm_gem_shmem_release_idle_pages_locked - Release idle backing without purging + * @shmem: shmem GEM object whose device mappings have been removed + * + * Drop the cached SG table and the page array of an object that no device + * mapping, pin, vmap or SG-table borrower uses, so ordinary shmem reclaim can + * swap its contents. Revocable CPU VMAs are zapped first and refault later. + * The caller holds the reservation lock, has removed every device mapping + * including cached translations, and excludes new device access. A later + * drm_gem_shmem_get_pages_sgt() may return different DMA addresses. + * + * Returns: 0 on success, -EBUSY if backing is still in use, -EINVAL for an + * imported, exported or purged object. + */ +int drm_gem_shmem_release_idle_pages_locked(struct drm_gem_shmem_object *shmem) +{ + struct drm_gem_object *obj = &shmem->base; + + dma_resv_assert_held(obj->resv); + + if (drm_gem_is_imported(obj) || obj->dma_buf || shmem->madv < 0) + return -EINVAL; + + if (refcount_read(&shmem->pages_pin_count) || + refcount_read(&shmem->vmap_use_count) || + refcount_read(&shmem->pages_use_count) != !!shmem->sgt || + !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_BOOKKEEP)) + return -EBUSY; + + /* Zap revocable PFNs under resv so no fault can reinsert them. */ + if (shmem->reclaimable_cpu_mappings && refcount_read(&shmem->cpu_vma_count)) { + drm_vma_node_unmap(&obj->vma_node, obj->dev->anon_inode->i_mapping); + drm_gem_shmem_put_cpu_pages_locked(shmem); + } + + if (!shmem->sgt) + return 0; + + dma_unmap_sgtable(obj->dev->dev, shmem->sgt, DMA_BIDIRECTIONAL, 0); sg_free_table(shmem->sgt); kfree(shmem->sgt); shmem->sgt = NULL; + shmem->pages_mark_dirty_on_put = true; drm_gem_shmem_put_pages_locked(shmem); - shmem->madv = -1; + return 0; +} +EXPORT_SYMBOL_GPL(drm_gem_shmem_release_idle_pages_locked); + +void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem) +{ + struct drm_gem_object *obj = &shmem->base; + struct drm_device *dev = obj->dev; + + dma_resv_assert_held(shmem->base.resv); + drm_WARN_ON(obj->dev, !drm_gem_shmem_is_purgeable(shmem)); + + /* Revoke PFNs and complete the TLB flush while backing is still pinned. */ + shmem->madv = -1; drm_vma_node_unmap(&obj->vma_node, dev->anon_inode->i_mapping); + + if (shmem->sgt) { + dma_unmap_sgtable(dev->dev, shmem->sgt, DMA_BIDIRECTIONAL, 0); + sg_free_table(shmem->sgt); + kfree(shmem->sgt); + shmem->sgt = NULL; + drm_gem_shmem_put_pages_locked(shmem); + } + drm_gem_shmem_put_cpu_pages_locked(shmem); + drm_gem_free_mmap_offset(obj); /* Our goal here is to return as much of the memory as @@ -592,11 +753,16 @@ static void drm_gem_shmem_record_mkwrite(struct vm_fault *vmf) loff_t num_pages = obj->size >> PAGE_SHIFT; pgoff_t page_offset = drm_gem_shmem_object_offset(obj, vmf->pgoff); - if (drm_WARN_ON(obj->dev, !shmem->pages || page_offset >= num_pages)) + struct page *page; + + if (drm_WARN_ON(obj->dev, page_offset >= num_pages)) + return; + page = drm_gem_shmem_present_page(shmem, page_offset); + if (drm_WARN_ON(obj->dev, !page)) return; file_update_time(vma->vm_file); - folio_mark_dirty(page_folio(shmem->pages[page_offset])); + folio_mark_dirty(page_folio(page)); } /* @@ -648,7 +814,7 @@ static vm_fault_t drm_gem_shmem_any_fault(struct vm_fault *vmf, unsigned int ord struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); loff_t num_pages = obj->size >> PAGE_SHIFT; vm_fault_t ret = VM_FAULT_SIGBUS; - struct page **pages = shmem->pages; + struct page **pages; pgoff_t page_offset = drm_gem_shmem_object_offset(obj, vmf->pgoff); struct page *page; struct folio *folio; @@ -659,13 +825,28 @@ static vm_fault_t drm_gem_shmem_any_fault(struct vm_fault *vmf, unsigned int ord dma_resv_lock(obj->resv, NULL); - if (page_offset >= num_pages || drm_WARN_ON_ONCE(dev, !shmem->pages) || - shmem->madv < 0) + if (page_offset >= num_pages || shmem->madv < 0) goto out; - page = pages[page_offset]; - if (drm_WARN_ON_ONCE(dev, !page)) - goto out; + if (shmem->reclaimable_cpu_mappings && !shmem->pages) { + /* Per-page population; huge mappings need the full array. */ + if (order) { + ret = VM_FAULT_FALLBACK; + goto out; + } + page = drm_gem_shmem_cpu_page_locked(shmem, page_offset); + if (IS_ERR(page)) { + ret = vmf_error(PTR_ERR(page)); + goto out; + } + } else { + pages = shmem->pages; + if (drm_WARN_ON_ONCE(dev, !pages)) + goto out; + page = pages[page_offset]; + if (drm_WARN_ON_ONCE(dev, !page)) + goto out; + } folio = page_folio(page); pfn = page_to_pfn(page); @@ -699,8 +880,11 @@ static void drm_gem_shmem_vm_open(struct vm_area_struct *vma) * mmap'd, vm_open() just grabs an additional reference for the new * mm the vma is getting copied into (ie. on fork()). */ - drm_WARN_ON_ONCE(obj->dev, - !refcount_inc_not_zero(&shmem->pages_use_count)); + if (shmem->reclaimable_cpu_mappings) + refcount_inc(&shmem->cpu_vma_count); + else + drm_WARN_ON_ONCE(obj->dev, + !refcount_inc_not_zero(&shmem->pages_use_count)); dma_resv_unlock(shmem->base.resv); @@ -713,7 +897,12 @@ static void drm_gem_shmem_vm_close(struct vm_area_struct *vma) struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); dma_resv_lock(shmem->base.resv, NULL); - drm_gem_shmem_put_pages_locked(shmem); + if (shmem->reclaimable_cpu_mappings) { + if (refcount_dec_and_test(&shmem->cpu_vma_count)) + drm_gem_shmem_put_cpu_pages_locked(shmem); + } else { + drm_gem_shmem_put_pages_locked(shmem); + } dma_resv_unlock(shmem->base.resv); drm_gem_vm_close(vma); @@ -721,8 +910,23 @@ static void drm_gem_shmem_vm_close(struct vm_area_struct *vma) static vm_fault_t drm_gem_shmem_pfn_mkwrite(struct vm_fault *vmf) { - drm_gem_shmem_record_mkwrite(vmf); - return 0; + struct drm_gem_object *obj = vmf->vma->vm_private_data; + struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); + vm_fault_t ret = 0; + + if (!shmem->reclaimable_cpu_mappings) { + drm_gem_shmem_record_mkwrite(vmf); + return 0; + } + + /* A revoked PFN must refault rather than be marked writable. */ + dma_resv_lock(obj->resv, NULL); + if (!drm_gem_shmem_present_page(shmem, drm_gem_shmem_object_offset(obj, vmf->pgoff))) + ret = VM_FAULT_NOPAGE; + else + drm_gem_shmem_record_mkwrite(vmf); + dma_resv_unlock(obj->resv); + return ret; } const struct vm_operations_struct drm_gem_shmem_vm_ops = { @@ -772,8 +976,19 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct if (is_cow_mapping(vma->vm_flags)) return -EINVAL; + /* Revocation zaps the device anon mapping only. */ + if (shmem->reclaimable_cpu_mappings && + (!vma->vm_file || vma->vm_file->f_mapping != obj->dev->anon_inode->i_mapping)) + return -EINVAL; + dma_resv_lock(shmem->base.resv, NULL); - ret = drm_gem_shmem_get_pages_locked(shmem); + if (shmem->reclaimable_cpu_mappings) { + if (!refcount_inc_not_zero(&shmem->cpu_vma_count)) + refcount_set(&shmem->cpu_vma_count, 1); + ret = 0; + } else { + ret = drm_gem_shmem_get_pages_locked(shmem); + } dma_resv_unlock(shmem->base.resv); if (ret) diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h index b2c23af628e1a..d25bca5352aa6 100644 --- a/include/drm/drm_gem_shmem_helper.h +++ b/include/drm/drm_gem_shmem_helper.h @@ -31,6 +31,21 @@ struct drm_gem_shmem_object { */ struct page **pages; + /** @cpu_vma_count: Revocable CPU VMAs, protected by base.resv. */ + refcount_t cpu_vma_count; + + /** @reclaimable_cpu_mappings: CPU VMAs may be revoked by the driver. */ + bool reclaimable_cpu_mappings; + + /** + * @cpu_pages: Pages populated one at a time by CPU faults on revocable + * VMAs while no device mapping holds @pages. Sparse; under base.resv. + */ + struct page **cpu_pages; + + /** @cpu_pages_count: Populated entries of @cpu_pages. */ + unsigned long cpu_pages_count; + /** * @pages_use_count: * @@ -134,6 +149,8 @@ static inline bool drm_gem_shmem_is_purgeable(struct drm_gem_shmem_object *shmem !shmem->base.dma_buf && !drm_gem_is_imported(&shmem->base); } +bool drm_gem_shmem_idle_pages_releasable_locked(struct drm_gem_shmem_object *shmem); +int drm_gem_shmem_release_idle_pages_locked(struct drm_gem_shmem_object *shmem); void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem); struct sg_table *drm_gem_shmem_get_sg_table(struct drm_gem_shmem_object *shmem); diff --git a/rust/helpers/dma-resv.c b/rust/helpers/dma-resv.c index 71914d8241e29..957d80992fef3 100644 --- a/rust/helpers/dma-resv.c +++ b/rust/helpers/dma-resv.c @@ -12,3 +12,9 @@ __rust_helper void rust_helper_dma_resv_unlock(struct dma_resv *obj) { dma_resv_unlock(obj); } + +__rust_helper +bool rust_helper_dma_resv_trylock(struct dma_resv *obj) +{ + return dma_resv_trylock(obj); +} diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index eed0464482b69..b74c9d368e660 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -59,6 +59,10 @@ use core::{ self, NonNull, // }, + sync::atomic::{ + AtomicUsize, + Ordering, // + }, }; use gem::{ BaseObject, @@ -75,6 +79,9 @@ pub struct ObjectConfig<'a, T: DriverObject> { /// Whether to set the write-combine map flag. pub map_wc: bool, + /// Opt in to revocable CPU VMAs, with driver-controlled idle eviction. + pub reclaimable_cpu_mappings: bool, + /// Reuse the DMA reservation from another GEM object. /// /// The newly created [`Object`] will hold an owned refcount to `parent_resv_obj` if specified. @@ -86,11 +93,16 @@ impl<'a, T: DriverObject> Default for ObjectConfig<'a, T> { fn default() -> Self { Self { map_wc: false, + reclaimable_cpu_mappings: false, parent_resv_obj: None, } } } +const SGT_PERMANENT: usize = 1 << (usize::BITS - 1); +const SGT_RECLAIMING: usize = 1 << (usize::BITS - 2); +const SGT_USERS: usize = SGT_RECLAIMING - 1; + /// A shmem-backed GEM object. /// /// # Invariants @@ -109,6 +121,8 @@ pub struct Object { #[pin] /// Lock for protecting initialization of `sgt_res`. sgt_lock: Mutex<()>, + /// Owned SGT readers, or permanent borrowed access, exclude backing eviction. + sgt_access: AtomicUsize, #[pin] inner: T, } @@ -274,6 +288,9 @@ impl Object { return Err(EINVAL); } + // The borrowed reference can outlive any lock, so pin the backing permanently. + self.acquire_sgt_access(true)?; + let sgt_res = 'out: { // Fast path: sgt_res is already initialized if let Some(sgt_res) = self.sgt_res.as_ref() { @@ -315,6 +332,7 @@ impl Object { parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), sgt_res: ManuallyDrop::new(SetOnce::new()), sgt_lock <- new_mutex!(()), + sgt_access: AtomicUsize::new(0), inner <- T::new(dev, size, args), }), GFP_KERNEL, @@ -343,10 +361,70 @@ impl Object { // to have exclusive access - thus making this safe to hold a mutable reference to. let shmem = unsafe { &mut *obj.as_raw_shmem() }; shmem.set_map_wc(config.map_wc); + shmem.reclaimable_cpu_mappings = config.reclaimable_cpu_mappings; Ok(obj) } + fn acquire_sgt_access(&self, permanent: bool) -> Result { + let mut state = self.sgt_access.load(Ordering::Acquire); + loop { + if state & SGT_RECLAIMING != 0 { + return Err(EBUSY); + } + let next = if permanent { + state | SGT_PERMANENT + } else { + if state & SGT_USERS == SGT_USERS { + return Err(EOVERFLOW); + } + state + 1 + }; + match self.sgt_access.compare_exchange_weak( + state, + next, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return Ok(()), + Err(current) => state = current, + } + } + } + + /// Whether the caller's sole SG lease is the last user of the backing. + /// + /// # Safety + /// + /// The caller holds the object's DMA reservation lock. + pub unsafe fn sole_sg_lease_releasable_locked(&self) -> bool { + self.sgt_access.load(Ordering::Acquire) == 1 && + // SAFETY: The caller establishes the reservation-lock contract. + unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } + } + + /// Release idle backing without discarding contents. A later SG table may have different DMA + /// addresses. + /// + /// Refused with `EBUSY` once [`Self::sg_table`] has handed out a borrowed table, since that + /// borrow may outlive any lock. + /// + /// # Safety + /// + /// The caller holds this object's DMA reservation lock, has removed and invalidated every + /// device mapping, and excludes new device access. + pub unsafe fn release_idle_pages_locked(&self) -> Result { + self.sgt_access + .compare_exchange(0, SGT_RECLAIMING, Ordering::AcqRel, Ordering::Acquire) + .map_err(|_| EBUSY)?; + // SAFETY: The caller's contract plus the exclusive state above. + let result = to_result(unsafe { + bindings::drm_gem_shmem_release_idle_pages_locked(self.as_raw_shmem()) + }); + self.sgt_access.store(0, Ordering::Release); + result + } + /// Creates and returns an owned reference to a virtual kernel memory mapping for this object. #[inline] pub fn owned_vmap(&self) -> Result> { From 945a4a9fd95d023d020eb877eba041f331245bfe Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:11:49 -0400 Subject: [PATCH 0822/2680] rust: drm: gpuvm: add non-blocking private mapping traversal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the interfaces a driver needs to evict and restore the backing of idle VM-private objects: expose per-GPUVA driver metadata, borrow the GEM object of a VM/BO association, take the VM reservation with dma_resv_trylock, walk private associations and mark every alias INVALIDATED without allocating, iterate current private mappings by cursor, and clear the flag after the page tables are rebuilt. [ryan: adapted to 7.2.10 — minimal; for DJ: rust/kernel/drm/gpuvm.rs is deleted in 7.2, so the traversal is added to rust/kernel/drm/gpuvm/mod.rs next to Asahi's exec_lock() hack: try_lock_private() and its guard, MappingSnapshot (carrying the 7.2 VaData), set_invalidated() on GpuVmExec and a GpuVa::data() accessor; GpuVmBo::object() is the existing 7.2 obj()] Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 35b057e572224c39f79367df9e7c464f1822d15a) --- rust/kernel/drm/gpuvm/mod.rs | 309 +++++++++++++++++++++++++++++++++++ rust/kernel/drm/gpuvm/va.rs | 7 + 2 files changed, 316 insertions(+) diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs index ed84414b0d25d..6e895976d09ab 100644 --- a/rust/kernel/drm/gpuvm/mod.rs +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -332,6 +332,315 @@ impl GpuVm { } } +impl GpuVm { + /// Set or clear `INVALIDATED` on the exact linked GPUVA `addr..addr + range`. + /// + /// # Safety + /// + /// The caller owns the VM reservation and keeps the exact logical VA linked. + unsafe fn set_invalidated_locked(&self, addr: u64, range: u64, invalidated: bool) -> Result { + if range == 0 || addr.checked_add(range).is_none() { + return Err(EINVAL); + } + // SAFETY: The caller owns the VM reservation and keeps the VA linked. + let raw = unsafe { bindings::drm_gpuva_find(self.as_raw(), addr, range) }; + if raw.is_null() { + return Err(EINVAL); + } + let flag = u32::from(GpuVaFlags::from(GpuVaFlag::INVALIDATED)); + // SAFETY: Reservation ownership excludes all concurrent VA mutation. + unsafe { + if (*raw).va.addr != addr || (*raw).va.range != range { + return Err(EINVAL); + } + if invalidated { + (*raw).flags |= flag; + } else { + (*raw).flags &= !flag; + } + } + Ok(()) + } + + /// HACK: added for the asahi downstream driver. Try to lock only the VM reservation, without + /// allocation or waiting. + /// + /// The guard protects VM metadata and private BOs, not external backing. Callers must not + /// already hold a reservation lock. + pub fn try_lock_private(&self) -> Option> { + let resv = self.raw_resv(); + // SAFETY: The reservation outlives the returned borrow and guard. + if !unsafe { bindings::dma_resv_trylock(resv) } { + return None; + } + Some(PrivateGpuVmGuard { + gpuvm: self, + resv, + _not_send: NotThreadSafe, + }) + } +} + +impl GpuVmExec { + /// HACK: added for the asahi downstream driver. Mark the exact retained logical mapping after + /// page-table restoration. + pub fn set_invalidated(&mut self, addr: u64, range: u64, invalidated: bool) -> Result { + // SAFETY: The exec guard owns the VM reservation. + unsafe { + self.unique_gpuvm + .set_invalidated_locked(addr, range, invalidated) + } + } +} + +/// HACK: added for the asahi downstream driver. Logical binding retained independently of +/// hardware page-table residency. +/// +/// Owns a GEM reference and copied metadata, never a [`GpuVmBo`] reference. Driver metadata +/// cloning must not introduce a reservation-taking destructor. +pub struct MappingSnapshot +where + T::Object: AlwaysRefCounted, +{ + /// The mapped object. + pub object: ARef, + /// Start of the mapping in the GPU address space. + pub addr: u64, + /// Length of the mapping. + pub range: u64, + /// Offset of the mapping in the object. + pub offset: u64, + /// Flags of the GPUVA. + pub flags: GpuVaFlags, + /// Driver metadata of the GPUVA. + pub inner: T::VaData, +} + +/// HACK: added for the asahi downstream driver. Nonblocking reservation ownership for pressure +/// scans of private objects. +pub struct PrivateGpuVmGuard<'a, T: DriverGpuVm> { + gpuvm: &'a GpuVm, + resv: *mut bindings::dma_resv, + // DMA reservation ownership must remain on the acquiring task. + _not_send: NotThreadSafe, +} + +impl PrivateGpuVmGuard<'_, T> { + /// Visit each private BO with linked logical mappings once, without allocating or retaining + /// references. Returning false stops the walk. Contended GEM lists are skipped. + /// + /// # Safety + /// + /// The callback must not insert or remove logical mappings or modify either GPUVA list. This + /// reservation keeps their linked BO/GEM references alive for the complete callback. The + /// callback must not retain a borrowed BO or acquire its GEM GPUVA mutex recursively. + pub unsafe fn for_each_private_bo( + &mut self, + mut visit: impl FnMut(&mut Self, &GpuVmBo) -> Result, + ) -> Result { + let vm = self.gpuvm.as_raw(); + // SAFETY: The VM is live while this guard borrows it. + if unsafe { (*vm).flags } & bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE == 0 { + return Err(EINVAL); + } + // SAFETY: The VM reservation protects this ordered mapping list. The callback does not + // mutate its links, and no borrowed or owned reference outlives the guard. + let head = unsafe { &raw const (*vm).rb.list }; + // SAFETY: As above. + let mut node = unsafe { (*head).next }; + while node != head.cast_mut() { + // SAFETY: `node` is a live list entry embedded in a `drm_gpuva`. + let (next, raw) = unsafe { + ( + (*node).next, + kernel::container_of!(node, bindings::drm_gpuva, rb.entry), + ) + }; + // SAFETY: The linked GPUVA is live under the reservation. + let va = unsafe { &*raw }; + if !va.gem.obj.is_null() && !va.vm_bo.is_null() { + // SAFETY: The linked VA retains this driver's object. + let object = unsafe { ::from_raw(va.gem.obj) }; + if !self.gpuvm.is_extobj(object) { + // SAFETY: The GEM object is live; its GPUVA list mutex is initialized. + let lock = unsafe { &raw mut (*va.gem.obj).gpuva.lock }; + // Never block direct reclaim on a GEM-list owner. The canonical alias is + // the list's first-linked node, not its lowest address or current residency. + // SAFETY: `lock` is a valid mutex. + if unsafe { bindings::mutex_trylock(lock) } != 0 { + // SAFETY: The list mutex protects these link reads. + let canonical = unsafe { + (*va.vm_bo).list.gpuva.next == (&raw const va.gem.entry).cast_mut() + }; + // SAFETY: Locked just above. + unsafe { bindings::mutex_unlock(lock) }; + if canonical { + // SAFETY: The VA's `vm_bo` is a live `GpuVmBo` of this VM. + let bo = unsafe { GpuVmBo::::from_raw(va.vm_bo) }; + if !visit(self, bo)? { + break; + } + } + } + } + } + node = next; + } + Ok(()) + } + + /// Mark and visit every logical alias of this private BO without a snapshot allocation. + /// + /// GEM-list locking is nonblocking and held only while reading a cursor, never during the + /// callback. Failure (including contention) can leave a prefix marked `INVALIDATED`; callers + /// must retain backing unless this method succeeds for the entire association. + /// + /// # Safety + /// + /// The callback must not insert or remove logical mappings, alter list links, or release the + /// BO's SG/backing lease. The BO must remain linked throughout the walk. Exclude new jobs and + /// binding edits for the whole invalidation and backing-release transaction. + pub unsafe fn invalidate_private_bo_mappings( + &mut self, + bo: &GpuVmBo, + mut visit: impl FnMut(&mut Self, u64, u64) -> Result, + ) -> Result { + let vm = self.gpuvm.as_raw(); + let raw_bo = bo.as_raw(); + // SAFETY: `bo` and the VM are live while borrowed. + if unsafe { (*raw_bo).vm } != vm + || self.gpuvm.is_extobj(bo.obj()) + || unsafe { (*vm).flags } & bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE == 0 + { + return Err(EINVAL); + } + let flag = u32::from(GpuVaFlags::from(GpuVaFlag::INVALIDATED)); + // SAFETY: `bo` is live while borrowed. + let head = unsafe { &raw const (*raw_bo).list.gpuva }; + // SAFETY: The GEM object of a live BO is live; its GPUVA list mutex is initialized. + let lock = unsafe { &raw mut (*(*raw_bo).obj).gpuva.lock }; + // SAFETY: `lock` is a valid mutex. + if unsafe { bindings::mutex_trylock(lock) } == 0 { + return Err(EAGAIN); + } + // SAFETY: The list mutex protects the link read. + let mut node = unsafe { (*head).next }; + // SAFETY: Locked just above. + unsafe { bindings::mutex_unlock(lock) }; + while node != head.cast_mut() { + // SAFETY: `lock` is a valid mutex. + if unsafe { bindings::mutex_trylock(lock) } == 0 { + return Err(EAGAIN); + } + // SAFETY: The list mutex protects link reads; this reservation and the caller's + // no-unlink contract keep current/next and their association alive after unlocking. + let (next, raw) = unsafe { + ( + (*node).next, + kernel::container_of!(node, bindings::drm_gpuva, gem.entry), + ) + }; + // SAFETY: Locked just above. + unsafe { bindings::mutex_unlock(lock) }; + // SAFETY: As above. + let va = unsafe { &mut *raw }; + if va.vm != vm + || va.vm_bo != raw_bo + || va.va.range == 0 + || va.va.addr.checked_add(va.va.range).is_none() + { + return Err(EINVAL); + } + // Flags are serialized by the VM reservation, independently of GEM's list mutex. + va.flags |= flag; + visit(self, va.va.addr, va.va.range)?; + node = next; + } + Ok(()) + } + + /// Retain the next private mapping containing all `required_flags`. + /// + /// `cursor` is an exclusive end address and advances past skipped mappings as well. No + /// collection is allocated; only the returned object is retained. Callers must drop this + /// reservation guard before releasing that reference or acquiring backing. To cover a stable + /// set across calls, separately exclude logical binding edits (including retirement) for the + /// whole walk. + pub fn next_private_mapping( + &self, + cursor: &mut u64, + required_flags: GpuVaFlags, + ) -> Result>> + where + T::VaData: Clone, + T::Object: AlwaysRefCounted, + { + while *cursor != u64::MAX { + // SAFETY: The VM reservation serializes mutation of its VA tree. + let raw = unsafe { + bindings::drm_gpuva_find_first(self.gpuvm.as_raw(), *cursor, u64::MAX - *cursor) + }; + if raw.is_null() { + return Ok(None); + } + // SAFETY: `raw` stays linked and alive while this guard is held. + let va = unsafe { &*raw }; + let end = va.va.addr.checked_add(va.va.range).ok_or(EINVAL)?; + if end <= *cursor { + return Err(EINVAL); + } + *cursor = end; + let required = u32::from(required_flags); + if va.flags & required != required + || va.gem.obj.is_null() + || va.vm_bo.is_null() + { + continue; + } + // SAFETY: The linked VA retains this driver's object. + let obj = unsafe { ::from_raw(va.gem.obj) }; + if self.gpuvm.is_extobj(obj) { + continue; + } + // SAFETY: Driver-owned mappings are `GpuVa`. The kernel cutout has no GEM object + // and was skipped above. + let typed = unsafe { GpuVa::::from_raw(raw) }; + return Ok(Some(MappingSnapshot { + object: obj.into(), + addr: va.va.addr, + range: va.va.range, + offset: va.gem.offset, + flags: GpuVaFlags::from(va.flags), + inner: typed.data().clone(), + })); + } + Ok(None) + } +} + +impl Deref for PrivateGpuVmGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &T { + // SAFETY: The reservation serializes all access to the VM's driver data. + unsafe { &*self.gpuvm.data.get() } + } +} + +impl DerefMut for PrivateGpuVmGuard<'_, T> { + fn deref_mut(&mut self) -> &mut T { + // SAFETY: This guard has exclusive reservation ownership. + unsafe { &mut *self.gpuvm.data.get() } + } +} + +impl Drop for PrivateGpuVmGuard<'_, T> { + fn drop(&mut self) { + // SAFETY: `try_lock_private()` acquired exactly this reservation. + unsafe { bindings::dma_resv_unlock(self.resv) }; + } +} + /// HACK: added for the asahi downstream driver. Access to the VM driver data under its shared /// reservation, without external BO locks. pub struct GpuVmInnerGuard<'a, T: DriverGpuVm> { diff --git a/rust/kernel/drm/gpuvm/va.rs b/rust/kernel/drm/gpuvm/va.rs index abce031b8a647..5adca9bbb1b49 100644 --- a/rust/kernel/drm/gpuvm/va.rs +++ b/rust/kernel/drm/gpuvm/va.rs @@ -115,6 +115,13 @@ impl GpuVa { unsafe { ::from_raw((*self.as_raw()).gem.obj) } } + /// HACK: added for the asahi downstream driver. Driver metadata retained independently of + /// the resident page tables. + #[inline] + pub fn data(&self) -> &T::VaData { + &self.data + } + /// Returns the underlying [`GpuVmBo`] object that backs this [`GpuVa`]. #[inline] pub fn vm_bo(&self) -> &GpuVmBo { From 7ec83f64fb2c9829d87cc5326728fd85303307dc Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:13:08 -0400 Subject: [PATCH 0823/2680] drm/asahi: reclaim idle private VM backing under memory pressure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Register a shrinker per client VM. When the VM has no accepted jobs, no mapping commits and a healthy status, the scan invalidates every alias of an idle VM-private object, clears its page tables and TLB entries, drops the VM's SG lease and releases the shmem backing so ordinary reclaim can swap it. Objects with kernel aliases, exports/imports or other SG users are skipped. Private objects also get revocable CPU mappings, so an mmap no longer pins the whole object before it is touched. Jobs, kernel aliases and bind preparation take a residency lease; the first lease after an eviction rebuilds the invalidated mappings from the logical GPUVAs before any work is admitted, and retirement serializes with that restore. Mapping permissions are kept as GPUVA metadata so restored and split mappings keep their protection. [ryan: adapted to 7.2.10 — minimal; for DJ: the per-mapping protection is the 7.2 GPUVA VaData (VmGpuVa), passed to OpMap::insert() and OpRemap::remap(); residency.rs works on Asahi's 7.2 VmBoData segments instead of the 7.1 BO sgt/sg_vec, finds the BO through the exec guard and locks Vm.gpuvm. With the 7.2 shmem SG table lent permanently (see the revocable-mapping commit), no object passes sole_sg_lease_releasable_locked(), so the shrinker reports nothing and idle backing is never evicted on 7.2] Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e2b08ebbcadc3a9ebe2733140e1a88c62af87ba1) --- drivers/gpu/drm/asahi/file.rs | 24 +- drivers/gpu/drm/asahi/gem.rs | 42 ++- drivers/gpu/drm/asahi/mmu.rs | 68 +++- drivers/gpu/drm/asahi/mmu/bind.rs | 13 +- drivers/gpu/drm/asahi/mmu/lifetime.rs | 14 +- drivers/gpu/drm/asahi/mmu/residency.rs | 489 +++++++++++++++++++++++++ 6 files changed, 624 insertions(+), 26 deletions(-) create mode 100644 drivers/gpu/drm/asahi/mmu/residency.rs diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 5227ba03872fb..68a04a3e4527f 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -62,6 +62,7 @@ pub(crate) const MAX_COMMANDS_PER_SUBMISSION: u32 = 64; /// A client instance of an `mmu::Vm` address space. struct Vm { + shrinker: Option>, ualloc: Arc>, ualloc_priv: Arc>, vm: mmu::Vm, @@ -72,6 +73,8 @@ struct Vm { impl Drop for Vm { fn drop(&mut self) { + self.shrinker = None; + self.vm.close_idle_reclaim(); if self.vm.status().is_some() { // Tracked jobs can outlive their file's VM handle. Their last guard performs // these unmaps once no accepted work can still dereference the mappings. @@ -536,6 +539,11 @@ impl File { mod_dev_dbg!(device, "[File {} VM {}]: VM created\n", file_id, id); resv.fill(KBox::new( Vm { + shrinker: if vm.residency_reclaim_enabled() { + Some(mmu::VmShrinker::new(&vm)?) + } else { + None + }, ualloc, ualloc_priv, vm, @@ -592,21 +600,19 @@ impl File { } let resv_gem; + let mut reclaim_cpu = false; let resv_obj = if data.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 { - resv_gem = file - .inner() - .vms() - .lock() - .get(data.vm_id.try_into()?) - .ok_or(ENOENT)? - .vm - .get_resv_obj(); + let vms = file.inner().vms(); + let vms = vms.lock(); + let vm = &vms.get(data.vm_id.try_into()?).ok_or(ENOENT)?.vm; + reclaim_cpu = vm.residency_reclaim_enabled(); + resv_gem = vm.get_resv_obj(); Some(resv_gem.deref()) } else { None }; - let gem = gem::new_object(device, data.size.try_into()?, data.flags, resv_obj)?; + let gem = gem::new_object(device, data.size.try_into()?, data.flags, resv_obj, reclaim_cpu)?; let handle = gem.create_handle(file)?; data.handle = handle; diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 0843897e0b4ee..787c6b3419ea2 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -57,6 +57,8 @@ pub(crate) struct AsahiObject { exportable: bool, /// Whether this is a kernel-created object. kernel: bool, + /// Driver aliases outside the user GPUVA tree; they pin the backing. + kernel_mappings: AtomicU64, } /// Type alias for the shmem GEM object type for this driver. @@ -188,8 +190,8 @@ fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> dev, align(size, mmu::UAT_PGSZ), shmem::ObjectConfig:: { - map_wc, reclaimable_cpu_mappings: false, + map_wc, parent_resv_obj: None, }, AsahiObjConfig { @@ -209,6 +211,7 @@ pub(crate) fn new_object( size: usize, flags: u32, parent_object: Option<&shmem::Object>, + reclaim_cpu: bool, ) -> Result> { if (flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0) != parent_object.is_some() { @@ -219,8 +222,8 @@ pub(crate) fn new_object( dev, align(size, mmu::UAT_PGSZ), shmem::ObjectConfig:: { + reclaimable_cpu_mappings: parent_object.is_some() && reclaim_cpu, map_wc: flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK == 0, - reclaimable_cpu_mappings: false, parent_resv_obj: parent_object, }, AsahiObjConfig { @@ -256,6 +259,7 @@ impl DriverObject for AsahiObject { flags: args.flags, exportable: args.exportable, kernel: args.kernel, + kernel_mappings: AtomicU64::new(0), }) } @@ -280,3 +284,37 @@ impl DriverObject for AsahiObject { obj.prime_export(flags) } } + +/// A driver-created alias of a GEM object, possibly in a root the user GPUVA +/// tree does not describe. It keeps the object out of idle reclaim. +pub(crate) struct KernelMappingPin { + object: ARef, +} + +impl KernelMappingPin { + pub(crate) fn new(object: &Object) -> Self { + object.kernel_mappings.fetch_add(1, Ordering::AcqRel); + Self { + object: object.into(), + } + } +} + +impl core::ops::Deref for KernelMappingPin { + type Target = Object; + fn deref(&self) -> &Object { + &self.object + } +} + +impl Drop for KernelMappingPin { + fn drop(&mut self) { + self.object.kernel_mappings.fetch_sub(1, Ordering::Release); + } +} + +impl AsahiObject { + pub(crate) fn idle_reclaim_candidate(&self) -> bool { + !self.kernel && !self.exportable && self.kernel_mappings.load(Ordering::Acquire) == 0 + } +} diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index b22a8178c8b6e..d404866f80fbb 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -117,6 +117,8 @@ mod bind; mod context; mod shared; mod lifetime; +mod residency; +pub(crate) use residency::VmShrinker; pub(crate) use bind::PreparedUserBindBatch; pub(crate) use lifetime::VmJobGuard; @@ -335,6 +337,13 @@ impl core::ops::DerefMut for VmExecGuard { } } +impl VmExecGuard { + /// Mark the exact retained logical mapping after page-table restoration. + fn set_invalidated(&mut self, addr: u64, range: u64, invalidated: bool) -> Result { + self.0.set_invalidated(addr, range, invalidated) + } +} + /// Take the exec lock of a VM's GPUVM. fn lock_vm( gpuvm: &gpuvm::GpuVm, @@ -420,10 +429,16 @@ impl<'ctx> StepContext<'ctx> { } } +/// Per-mapping metadata that survives backing eviction and GPUVA splits. +#[derive(Default, Clone)] +struct VmGpuVa { + prot: Option, +} + impl gpuvm::DriverGpuVm for VmInner { type Driver = AsahiDrmDriver; type Object = gem::Object; - type VaData = (); + type VaData = VmGpuVa; type VmBoData = VmBoData; type SmContext<'ctx> = StepContext<'ctx> @@ -513,7 +528,12 @@ impl gpuvm::DriverGpuVm for VmInner { return Err(error); } }; - let op = op.insert(gpuva, pin_init::init_zeroed()); + let op = op.insert( + gpuva, + VmGpuVa { + prot: Some(ctx.prot), + }, + ); self.tlbi_contexts_naming_root(map_addr, map_length); Ok(op) @@ -538,6 +558,9 @@ impl gpuvm::DriverGpuVm for VmInner { op: OpRemap<'op, Self>, ctx: &mut Self::SmContext<'ctx>, ) -> Result, Error> { + let orig_prot = op.va_to_unmap().data().prot.ok_or(EINVAL)?; + let orig_invalidated = op.va_to_unmap().flags().contains(GpuVaFlag::INVALIDATED); + // Only unmap the hole between prev/next, if they exist let unmap_start = if let Some(op) = op.prev() { op.addr() + op.length() @@ -563,7 +586,11 @@ impl gpuvm::DriverGpuVm for VmInner { let unmap_range = unmap_end - unmap_start; let _mutation = self.mapping_mutation(); - self.page_table.unmap_pages(unmap_start..unmap_end)?; + if orig_invalidated { + self.page_table.discard_partial_map(unmap_start..unmap_end)?; + } else { + self.page_table.unmap_pages(unmap_start..unmap_end)?; + } if let Some(asid) = self.slot() { fence(Ordering::SeqCst); @@ -585,8 +612,12 @@ impl gpuvm::DriverGpuVm for VmInner { let (op_remapped, remap_ret) = op.remap( [prev_gpuva, next_gpuva], - pin_init::init_zeroed(), - pin_init::init_zeroed(), + VmGpuVa { + prot: Some(orig_prot), + }, + VmGpuVa { + prot: Some(orig_prot), + }, ); if let Some(unused_va) = remap_ret.unused_va { @@ -603,8 +634,13 @@ impl VmInner { mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.length()); let _mutation = self.mapping_mutation(); - self.page_table - .unmap_pages(va.addr()..(va.addr() + va.length()))?; + if va.flags().contains(GpuVaFlag::INVALIDATED) { + self.page_table + .discard_partial_map(va.addr()..(va.addr() + va.length()))?; + } else { + self.page_table + .unmap_pages(va.addr()..(va.addr() + va.length()))?; + } if let Some(asid) = self.slot() { fence(Ordering::SeqCst); @@ -709,6 +745,10 @@ impl VmInner { } let contexts = self.uat_inner.lock().contexts_naming_root(self.ttb()); + self.tlbi_context_mask(iova, size, contexts); + } + + fn tlbi_context_mask(&self, iova: u64, size: usize, contexts: u64) { if contexts == 0 { return; } @@ -811,6 +851,7 @@ pub(crate) struct Vm { id: u64, mapping_epoch: Option>, lifetime: Option>, + residency: Option>, context_bindings: Option>, shared_bindings: Option>, status: Option>, @@ -1003,7 +1044,7 @@ pub(crate) struct KernelMappingInner { // - Drop the GEM BO next, since BO free can take the resv lock itself // - Drop the owner GpuVm last, since that again can take resv locks when the refcount drops to 0 bo: Option, - _gem: Option>, + _gem: Option, owner: ARef>, uat_inner: Arc, prot: Prot, @@ -1647,6 +1688,7 @@ impl Vm { gpuvm, mapping_epoch, lifetime: None, + residency: None, context_bindings: None, shared_bindings: None, status: None, @@ -1659,6 +1701,7 @@ impl Vm { pub(crate) fn with_status(mut self) -> Result { self.status = Some(Arc::new(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); self.lifetime = Some(lifetime::VmLifetime::new()?); + self.residency = Some(residency::Gate::new()?); self.context_bindings = Some(context::ContextBindings::new()?); self.shared_bindings = Some(shared::SharedBindings::new()?); Ok(self) @@ -1800,6 +1843,7 @@ impl Vm { return Err(EINVAL); } let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; + let _residency = self.enter_residency()?; // TODO: hack, populate the GEM object sg table first { let dev = unsafe { self.dev.as_bound() }; @@ -1855,7 +1899,7 @@ impl Vm { { let payload = node.as_mut().inner_mut(); payload.bo = vm_bo.take(); - payload._gem = Some(gem.into()); + payload._gem = Some(gem::KernelMappingPin::new(gem)); } let ret = inner.map_node(&node, prot); @@ -1880,6 +1924,7 @@ impl Vm { let reserved_size = size .checked_add(if guard { UAT_PGSZ } else { 0 }) .ok_or(EOVERFLOW)?; + let _residency = self.enter_residency()?; // TODO: hack, populate the GEM object sg table first { let dev = unsafe { self.dev.as_bound() }; @@ -1935,7 +1980,7 @@ impl Vm { { let payload = node.as_mut().inner_mut(); payload.bo = vm_bo.take(); - payload._gem = Some(gem.into()); + payload._gem = Some(gem::KernelMappingPin::new(gem)); } let ret = inner.map_node(&node, prot); @@ -1958,6 +2003,7 @@ impl Vm { prot: Prot, single_page: bool, ) -> Result { + let _residency = self.enter_residency_metadata()?; gem::validate_vm_binding(gem, self)?; self.wait_for_user_map_admission()?; // TODO: hack, populate the GEM object sg table first @@ -2131,6 +2177,7 @@ impl Vm { size: u64, user: bool, ) -> Result { + let _residency = self.enter_retirement_residency(); // Unmapping a range can only do a single split, so just preallocate // two GPUVAs. let mut resources = VmOpResources { @@ -2165,6 +2212,7 @@ impl Vm { } fn drop_mappings_now(&self, gem: &gem::Object) -> Result { + let _residency = self.enter_retirement_residency(); // Removing whole mappings only does unmaps, so no preallocated VAs let mut resources = VmOpResources { preallocated_gpuvas: [None, None, None], diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index 9a6065f355bb7..26740c5719d82 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -16,18 +16,19 @@ pub(crate) struct PreparedUserMap { offset: u64, single_page: bool, context: context::PreparedBinding, + _residency: Option, } pub(crate) struct PreparedUserUnmap { + _residency: Option, pub(super) iova: u64, pub(super) size: u64, pub(super) resources: VmOpResources, } -// SAFETY: This owns only its numeric range and unused, unlinked split-node -// allocations. step_remap moves each node out of ctx before linking it into -// the VM, so a retained ctx never owns a linked GPUVA. The unmap context has -// no borrowed object/lock state; lifetime.state serializes deferred ownership. +// SAFETY: This owns a thread-safe residency lease, a numeric range and unused, +// unlinked split-node allocations; step_remap moves each node out of ctx before +// linking it, so a retained ctx never owns a linked GPUVA. unsafe impl Send for PreparedUserUnmap {} pub(super) enum PreparedUserBindOp { @@ -68,6 +69,7 @@ impl Vm { single_page: bool, ) -> Result { gem::validate_vm_binding(gem, self)?; + let residency = self.enter_residency_metadata()?; let context = self.prepare_context_binding(addr, size, offset)?; let resources = VmOpResources { preallocated_gpuvas: [ @@ -93,6 +95,7 @@ impl Vm { return Err(EINVAL); } Ok(PreparedUserMap { + _residency: residency, gem: gem.into(), vm_bo, resources, @@ -109,7 +112,9 @@ impl Vm { if (iova | size) & UAT_PGMSK as u64 != 0 || !self.gpuvm.range_valid(iova, size) { return Err(EINVAL); } + let residency = self.enter_residency_metadata()?; Ok(PreparedUserUnmap { + _residency: residency, iova, size, resources: VmOpResources { diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 2f7acfecb3e70..ea3edaca6ef38 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -71,6 +71,14 @@ pub(super) struct VmLifetime { } impl VmLifetime { + pub(super) fn can_reclaim(&self) -> bool { + self.state.try_lock().is_some_and(|state| { + state.active == 0 && state.commits == 0 + && !state.blocked && !state.quarantined && !state.draining + && !state.closed && !state.cleanup_failed + }) + } + pub(super) fn new() -> Result> { Arc::pin_init( pin_init!(Self { @@ -89,6 +97,7 @@ impl VmLifetime { /// retirement: potentially visible work must retain its guard through quarantine or stop. pub(crate) struct VmJobGuard { vm: Vm, + _residency: Option, } impl VmJobGuard { @@ -344,6 +353,7 @@ impl Vm { /// Acquires a job reference without imposing a VM-wide limit on healthy work. pub(crate) fn retain_job(&self) -> Result { + let residency = self.enter_residency()?; let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; let mut state = lifetime.state.lock(); loop { @@ -355,7 +365,7 @@ impl Vm { } if !state.draining { state.active = state.active.checked_add(1).ok_or(EOVERFLOW)?; - return Ok(VmJobGuard { vm: self.clone() }); + return Ok(VmJobGuard { vm: self.clone(), _residency: residency }); } lifetime.changed.wait(&mut state); } @@ -533,6 +543,7 @@ impl Vm { } fn unmap_user_ranges_now(&self, user: Range, _kernel: Range) -> Result { + let _residency = self.enter_retirement_residency(); // User mappings alone have GPUVA nodes. Kernel and driver aliases use // mm::Node and are untouched; GPUVM's embedded kernel cutout is skipped. // Closed admission and zero accepted jobs make whole-node teardown safe. @@ -568,6 +579,7 @@ impl Vm { /// Admission stays closed to new jobs while the exec lock is acquired for cleanup. /// Recheck queued cleanup and reopen under the same lock used to append it. fn drain_mappings(&self, lifetime: &VmLifetime) { + let _residency = self.enter_retirement_residency(); let mut state = lifetime.state.lock(); let mut failure = None; loop { diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs new file mode 100644 index 0000000000000..1dd22596f3cb0 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -0,0 +1,489 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Pressure-only backing residency for idle G17 user VMs. +//! +//! This gate protects backing/address preparation as well as accepted jobs. +//! Its writer never waits for active readers: reclaim skips the VM instead. + +use super::*; +use kernel::drm::gem::BaseObject; +use kernel::{new_condvar, sync::CondVar}; + +#[derive(Default)] +struct State { + users: usize, + busy: bool, + evicted: bool, + closed: bool, +} + +#[pin_data] +pub(crate) struct Gate { + #[pin] + state: Mutex, + #[pin] + ready: CondVar, +} + +pub(crate) struct Lease { + gate: Arc, +} + +pub(crate) struct Reclaim { + gate: Arc, + changed: bool, +} + +impl Gate { + pub(crate) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + state <- new_mutex!(State::default(), "Asahi residency"), + ready <- new_condvar!("Asahi residency ready"), + }), + GFP_KERNEL, + ) + } + + /// Only a previous pressure eviction can cause restoration or waiting. + pub(crate) fn enter(gate: &Arc, mut restore: impl FnMut() -> Result) -> Result { + loop { + let mut state = gate.state.lock(); + if state.closed { + return Err(ENOENT); + } + while state.busy { + if gate.ready.wait_interruptible(&mut state) { + return Err(ERESTARTSYS); + } + } + if state.closed { + return Err(ENOENT); + } + if !state.evicted { + state.users = state.users.checked_add(1).ok_or(EOVERFLOW)?; + return Ok(Lease { gate: gate.clone() }); + } + // Binding edits can add mappings or retire invalidated GPUVAs + // without restoring the rest of the VM. Wait for those mutations + // before walking the authoritative logical bindings for restoration. + if state.users != 0 { + if gate.ready.wait_interruptible(&mut state) { + return Err(ERESTARTSYS); + } + continue; + } + // No new reader can enter until every retained mapping is restored. + state.busy = true; + drop(state); + let result = restore(); + let mut state = gate.state.lock(); + state.busy = false; + if result.is_ok() { + state.evicted = false; + } + drop(state); + gate.ready.notify_all(); + result?; + } + } + + pub(crate) fn enter_metadata(gate: &Arc) -> Result { + let mut state = gate.state.lock(); + while state.busy { + if gate.ready.wait_interruptible(&mut state) { + return Err(ERESTARTSYS); + } + } + if state.closed { + return Err(ENOENT); + } + state.users = state.users.checked_add(1).ok_or(EOVERFLOW)?; + Ok(Lease { gate: gate.clone() }) + } + + /// Retirement of an already closed handle may outlive the file's VM. + /// A closed gate excludes every new reclaimer and restorer, so no lease + /// is necessary then. Otherwise serialize metadata removal with restoration. + pub(super) fn enter_retirement(gate: &Arc) -> Option { + let mut state = gate.state.lock(); + while state.busy { + gate.ready.wait(&mut state); + } + if state.closed { + return None; + } + state.users += 1; + Some(Lease { gate: gate.clone() }) + } + + /// Direct reclaim cannot block on a job, another reclaimer or restoration. + /// A prior partial eviction does not exclude another scan: the VM may + /// still retain eligible BOs after satisfying a previous scan target. + /// The busy gate continues to exclude restoration and metadata changes. + pub(crate) fn try_reclaim(gate: &Arc) -> Option { + let mut state = gate.state.try_lock()?; + if state.users != 0 || state.busy || state.closed { + return None; + } + state.busy = true; + Some(Reclaim { + gate: gate.clone(), + changed: false, + }) + } + + pub(crate) fn close(&self) { + let mut state = self.state.lock(); + state.closed = true; + while state.busy { + self.ready.wait(&mut state); + } + } + + pub(crate) fn idle(&self) -> bool { + self.state + .try_lock() + .is_some_and(|state| state.users == 0 && !state.busy && !state.closed) + } +} + +impl Reclaim { + /// Call before the first page-table mutation, including fallible unmaps. + pub(crate) fn mark_changed(&mut self) { + self.changed = true; + } +} + +impl Drop for Reclaim { + fn drop(&mut self) { + let mut state = self.gate.state.lock(); + state.evicted |= self.changed; + state.busy = false; + drop(state); + self.gate.ready.notify_all(); + } +} + +impl Drop for Lease { + fn drop(&mut self) { + let mut state = self.gate.state.lock(); + debug_assert!(state.users != 0); + state.users -= 1; + if state.users == 0 { + drop(state); + self.gate.ready.notify_all(); + } + } +} + +/// Registration lifetime belongs to the file's VM handle, not the VM clone +/// retained by the callback. Unregister synchronously before dropping the clone. +pub(crate) struct VmShrinker { + vm: crate::mmu::Vm, + raw: *mut kernel::bindings::shrinker, +} + +// SAFETY: shrinker core serializes registration lifetime; callback VM operations +// use its residency/reservation locks. raw is never dereferenced after free. +unsafe impl Send for VmShrinker {} +unsafe impl Sync for VmShrinker {} + +impl VmShrinker { + pub(crate) fn new(vm: &crate::mmu::Vm) -> Result> { + let mut owner = KBox::new( + Self { + vm: vm.clone(), + raw: core::ptr::null_mut(), + }, + GFP_KERNEL, + )?; + // SAFETY: The format is static and has no format arguments. + let raw = + unsafe { kernel::bindings::shrinker_alloc(0, c_str!("asahi-idle-vm").as_char_ptr()) }; + if raw.is_null() { + return Err(ENOMEM); + } + owner.raw = raw; + // SAFETY: Newly allocated/unregistered shrinker is exclusively owned. + unsafe { + (*raw).count_objects = Some(Self::count); + (*raw).scan_objects = Some(Self::scan); + (*raw).private_data = (&*owner as *const Self).cast_mut().cast(); + (*raw).seeks = 2; + (*raw).batch = 256; + kernel::bindings::shrinker_register(raw); + } + Ok(owner) + } + + unsafe extern "C" fn count( + raw: *mut kernel::bindings::shrinker, + _sc: *mut kernel::bindings::shrink_control, + ) -> kernel::ffi::c_ulong { + // SAFETY: The registration owns this stable allocation until callbacks drain. + let owner = unsafe { &*((*raw).private_data as *const Self) }; + owner.vm.idle_reclaim_count() as _ + } + + unsafe extern "C" fn scan( + raw: *mut kernel::bindings::shrinker, + sc: *mut kernel::bindings::shrink_control, + ) -> kernel::ffi::c_ulong { + // SAFETY: Both pointers are supplied by the registered shrinker core. + let owner = unsafe { &*((*raw).private_data as *const Self) }; + let target = unsafe { (*sc).nr_to_scan } as usize; + owner.vm.reclaim_idle_pages(target) as _ + } +} + +impl Drop for VmShrinker { + fn drop(&mut self) { + if !self.raw.is_null() { + // SAFETY: This owns the sole registration; free drains callbacks + // before the embedded VM and callback private_data can disappear. + unsafe { kernel::bindings::shrinker_free(self.raw) }; + } + } +} + +impl Vm { + pub(crate) fn close_idle_reclaim(&self) { + if let Some(gate) = self.residency.as_ref() { + gate.close(); + } + } + + pub(crate) fn residency_reclaim_enabled(&self) -> bool { + self.residency.is_some() + } + + pub(super) fn enter_residency_metadata(&self) -> Result> { + self.residency + .as_ref() + .map(Gate::enter_metadata) + .transpose() + } + + pub(super) fn enter_residency(&self) -> Result> { + self.residency + .as_ref() + .map(|gate| Gate::enter(gate, || self.restore_idle_pages())) + .transpose() + } + + pub(crate) fn idle_reclaim_count(&self) -> usize { + let Some(gate) = self.residency.as_ref() else { + return 0; + }; + if !gate.idle() + || self + .status + .as_ref() + .map_or(true, |status| status.get() != 0) + { + return 0; + } + let Some(mut inner) = self.gpuvm.try_lock_private() else { + return 0; + }; + let mut pages = 0usize; + // SAFETY: Counting changes no logical mapping/list. Private BOs share + // this reservation; linked aliases retain their borrowed BO and GEM. + let _ = unsafe { inner.for_each_private_bo(|_, bo| { + let object = bo.obj(); + if object.idle_reclaim_candidate() && object.sole_sg_lease_releasable_locked() { + pages = pages.saturating_add(object.size() / kernel::page::PAGE_SIZE); + } + Ok(true) + }) }; + + pages + } + + pub(crate) fn reclaim_idle_pages(&self, target: usize) -> usize { + let Some(gate) = self.residency.as_ref() else { + return 0; + }; + let Some(mut reclaim) = Gate::try_reclaim(gate) else { + return 0; + }; + // Error fences cannot prove firmware ownership ended. + if self + .status + .as_ref() + .map_or(true, |status| status.get() != 0) + || !self + .lifetime + .as_ref() + .is_some_and(|lifetime| lifetime.can_reclaim()) + { + return 0; + } + let Some(mut inner) = self.gpuvm.try_lock_private() else { + return 0; + }; + let uat = inner.uat_inner.clone(); + let Some(shared) = uat.shared.try_lock() else { + return 0; + }; + let contexts = shared.contexts_naming_root(inner.ttb()); + let mut unpinned = 0usize; + // SAFETY: Exclusive residency and the private reservation exclude jobs, + // binding edits and retirement. Callbacks change PTEs/flags only; all + // logical aliases stay linked and retain their borrowed BO/GEM. + let _ = unsafe { inner.for_each_private_bo(|inner, bo| { + if unpinned >= target { return Ok(false); } + let object = bo.obj(); + if !object.idle_reclaim_candidate() { return Ok(true); } + { + // Avoid inversion against preparation (VmBoInner -> resv). + let Some(segments) = bo.data().segments.try_lock() else { return Ok(true); }; + if segments.is_none() || !object.sole_sg_lease_releasable_locked() { + return Ok(true); + } + } + // The association retains SG backing until *all* aliases have + // been invalidated and their device translations flushed. A + // contended cursor or PTE failure retains backing and the touched + // INVALIDATED prefix for restoration or a later reclaim attempt. + inner.invalidate_private_bo_mappings(bo, |inner, start, range| { + reclaim.mark_changed(); + let end = start.checked_add(range).ok_or(EOVERFLOW)?; + let _change = inner.mapping_mutation(); + inner.page_table.discard_partial_map(start..end)?; + inner.tlbi_context_mask(start, range as usize, contexts); + Ok(()) + })?; + { + let Some(mut segments) = bo.data().segments.try_lock() else { return Ok(true); }; + *segments = None; + } + // Every private GPUVA alias was removed from the device page + // tables above. Kernel mappings and other SG owners are excluded; + // the C/Rust backing helper rechecks exclusion before release. + if object.release_idle_pages_locked().is_ok() { + unpinned = unpinned.saturating_add(object.size() / kernel::page::PAGE_SIZE); + } + Ok(unpinned < target) + }) }; + drop(shared); + drop(inner); + + unpinned + } + + fn restore_idle_pages(&self) -> Result { + // Unbind/free can remove or split invalidated mappings while the VM + // stays evicted. Rebuild solely from the authoritative logical GPUVAs, + // never from stale pre-eviction records that could resurrect a BO. + // Gate::enter holds exclusive residency ownership for this entire + // walk: metadata edits, retirement and reclaim cannot change the tree. + // Retain one invalidated mapping at a time, avoiding a VM-sized KVec + // allocation precisely when memory pressure requires restoration. + let mut cursor = 0u64; + loop { + let map = loop { + if let Some(inner) = self.gpuvm.try_lock_private() { + break inner.next_private_mapping( + &mut cursor, GpuVaFlag::INVALIDATED.into(), + )?; + } + // Wait for reservation ownership without retaining a mapping. + drop(lock_vm(&self.gpuvm, None, true)?); + }; + // The reservation guard has dropped before pinning backing, and + // before any returned object reference can be released on failure. + let Some(map) = map else { break; }; + let mut restored = 0u64; + let bo_owner = lock_vm(&self.gpuvm, None, true)? + .0 + .unique_gpuvm + .find_bo(&map.object) + .ok_or(EFAULT)?; + { + let needs_segments = bo_owner.data().segments.lock().is_none(); + if needs_segments { + // Preserve review1's reservation/BO lock order: the getter + // may acquire the object reservation; never hold BO here. + // SAFETY: As in `Vm::bind_object()`. + let dev = unsafe { self.dev.as_bound() }; + let segments = prefetch_sgt_segments(dev, &map.object)?; + let mut bo = bo_owner.data().segments.lock(); + if bo.is_none() { bo.replace(segments); } + drop(bo); + } + } + let mut inner = lock_vm(&self.gpuvm, None, true)?; + let bo = bo_owner.data().segments.lock(); + let sgt = bo.as_ref().ok_or(EFAULT)?; + let result = (|| -> Result { + let end = map.addr.checked_add(map.range).ok_or(EOVERFLOW)?; + let _mutation = inner.mapping_mutation(); + inner.page_table.discard_partial_map(map.addr..end)?; + inner.tlbi_contexts_naming_root(map.addr, map.range as usize); + let prot = map.inner.prot.ok_or(EINVAL)?; + let repeat = map.flags.contains(GpuVaFlag::REPEAT); + let mut skip = map + .offset + .checked_add(if repeat { 0 } else { restored }) + .ok_or(EOVERFLOW)? as usize; + let mut left = map.range.checked_sub(restored).ok_or(EOVERFLOW)? as usize; + for &(dma_address, dma_len) in sgt.iter() { + if left == 0 { + break; + } + let len = dma_len as usize; + if skip >= len { + skip -= len; + continue; + } + let physical = dma_address as usize + skip; + let bytes = if repeat { left } else { left.min(len - skip) }; + let start = map.addr.checked_add(restored).ok_or(EOVERFLOW)?; + let end = start.checked_add(bytes as u64).ok_or(EOVERFLOW)?; + let _change = inner.mapping_mutation(); + inner.page_table.map_pages( + start..end, + physical as PhysicalAddr, + prot, + repeat, + )?; + restored += bytes as u64; + left -= bytes; + skip = 0; + } + if left != 0 { + return Err(EFAULT); + } + // The exclusive residency gate excludes submissions until + // every invalidated mapping is restored. Publish this logical + // range once, instead of issuing a synchronized TLBI for each + // physical SG segment. Keep the preceding break-before-make + // invalidation; the error path clears and flushes the entire + // range if any map operation (including a partial one) fails. + inner.tlbi_contexts_naming_root(map.addr, map.range as usize); + inner.set_invalidated(map.addr, map.range, false)?; + + Ok(()) + })(); + if let Err(error) = result { + // Keep an invalidated GPUVA wholly unmapped even on partial + // restoration failure, so later unbind/close cannot skip a + // prefix that was already reinstalled. Backing stays retained. + // map_pages may fail inside its current SG segment. Clear the + // complete logical range, including that partially written segment. + let end = map.addr.checked_add(map.range).ok_or(EOVERFLOW)?; + let _mutation = inner.mapping_mutation(); + inner.page_table.discard_partial_map(map.addr..end)?; + inner.tlbi_contexts_naming_root(map.addr, map.range as usize); + return Err(error); + } + } + Ok(()) + } + + pub(super) fn enter_retirement_residency(&self) -> Option { + self.residency.as_ref().and_then(Gate::enter_retirement) + } +} From c6254f144db008ceb4659474c02c93f40e2264b0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:01:06 -0400 Subject: [PATCH 0824/2680] drm/asahi: free a closed context's render pool once its release is consumed A logical context owns one render free-list pool (about 24 MiB of page and run tables plus 22 USC blocks). The pool was dropped only with the Context, but an installed compute graph keeps its last Context until a successor owner replaces it, so every idle compute QID pinned the exited client's render pool as well. On an 8 GB machine this grew kernel GEM from 252 MB (fresh desktop) to 752 MB after bursts of parallel clients and never shrank. The teardown service only publishes a scheduler release after every pool the context registered has had its free-list release consumed, and the context reaches the reclaim batch after that scheduler release is consumed too. Drop the pool there: nothing on the firmware side names it any more, and the retained compute binding never uses it. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e82b27902f91d8c833395a966366d0ffc30769c3) --- drivers/gpu/drm/asahi/g17/context.rs | 6 ++++++ drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 1 + 2 files changed, 7 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index ee2d8133ea8cd..eb4ddcac7b808 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -273,6 +273,12 @@ impl Context { *render = Some(pool.clone()); Ok(pool) } + /// Drops the pool once its free-list release is consumed and the closed + /// scheduler is detached. Installed compute bindings may retain the + /// context itself until replacement; they never use the render pool. + pub(crate) fn take_render_pool(&self) -> Option> { + self.render.lock().take() + } pub(crate) fn render_pools_released(&self, released: impl Fn(u32, u64) -> bool) -> bool { self.render .lock() diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index b1495dbf3e23e..f310f8154aae7 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -32,6 +32,7 @@ impl Reclaim { Self::Binding(binding) => drop(binding), Self::Context(context) => { context.release_execution(); + drop(context.take_render_pool()); drop(context); } } From 0948c7bb788a5d0d5b37d4335879e41dde6f489f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:13:49 -0400 Subject: [PATCH 0825/2680] drm/asahi: hold idle descent for every first render pair install The first install of a physical render pair acquired the registration power hold only when the core power register read unpowered at that instant. A powered reading does not keep the firmware from starting idle descent before the host's configuration writes land. An earlier version of this driver with the same conditional hold hard-hung (watchdog reset, no log) when several clients installed their first pairs at once; with the hold taken unconditionally the same workloads ran clean. Acquire the hold before every first install and release it where the pair's pool is activated, as before. This costs a few tens of microseconds on a client's first render submission only. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit fa210660c43c1e8ffc4c0705792febb70c29e779) --- drivers/gpu/drm/asahi/g17/runtime/render.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index e466571bc88c7..bf86d46e7ca6b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -393,12 +393,15 @@ impl crate::g17::queue::render::Host for Host<'_> { .accounting .cancel(&qos::View::new(self.firmware.init.qos()?)?, publication) } + /// Idle descent is held for the whole first install. A point-in-time power + /// check let the firmware gate the cores between the host's configuration + /// writes while several clients installed their first pairs at once. fn install_pair(&mut self, registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + if !self.registration_powered { + self.firmware.acquire_registration_power()?; + self.registration_powered = true; + } for _ in 0..3 { - if !self.firmware.bridge.regs.g17_configuration_powered() { - self.firmware.acquire_registration_power()?; - self.registration_powered = true; - } if self .firmware .bridge From 9399860135b0b54c002376590140a01a22e2fdc6 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:56:27 -0400 Subject: [PATCH 0826/2680] drm/asahi: add GEM_MADVISE and purge advised objects under memory pressure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The kernel cannot tell which VM-private objects an in-flight job may reference, so pressure reclaim only touches idle VMs. Userspace knows: Mesa's BO cache holds freed objects that no pending or future work uses until they are handed out again. Add DRM_IOCTL_ASAHI_GEM_MADVISE with WILLNEED/DONTNEED advice for VM-private, non-exportable objects. The VM shrinker now also scans DONTNEED objects of busy VMs: it invalidates their aliases, clears the page tables and TLB, drops the VM's SG lease and purges the shmem backing. Restoration skips purged objects (their mappings stay INVALIDATED until unbound) and WILLNEED reports retained=0 for them so userspace reallocates. A purge that invalidates aliases but does not complete (contended GEM list, page-table error, or a backing lease that appeared meanwhile) records the eviction on the residency gate, so the next submission restores those aliases instead of leaving them unmapped behind a WILLNEED that reports the contents retained. [ryan: adapted to 7.2.10 — minimal; for DJ: the ioctl takes Asahi's reg_data argument, the shmem purge helpers use the 7.2 reservation guard, and the purge walk uses the 7.2 VmBoData segments. A mapped object has lent its SG table permanently on 7.2, so purge skips it and only never-mapped DONTNEED objects are purged until the SG-table ownership is redesigned] Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit d7715ebbd2d3397958f08442d096a2476d32cb6a) --- drivers/gpu/drm/asahi/driver.rs | 2 + drivers/gpu/drm/asahi/file.rs | 18 ++++ drivers/gpu/drm/asahi/gem.rs | 9 ++ drivers/gpu/drm/asahi/mmu/residency.rs | 125 ++++++++++++++++++++++--- include/uapi/drm/asahi_drm.h | 39 ++++++++ rust/helpers/drm.c | 6 ++ rust/kernel/drm/gem/shmem.rs | 40 ++++++++ 7 files changed, 226 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index e97df0fbac2d8..d95dca73eb78e 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -111,6 +111,8 @@ impl drm::driver::Driver for AsahiDrmDriver { ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::queue_destroy), (ASAHI_SUBMIT, drm_asahi_submit, ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::submit), + (ASAHI_GEM_MADVISE, drm_asahi_gem_madvise, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::gem_madvise), } } diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 68a04a3e4527f..6c0c094d71ca3 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -652,6 +652,24 @@ impl File { Ok(0) } + /// IOCTL: gem_madvise: Mark a VM-private object purgeable or needed. + pub(crate) fn gem_madvise( + _device: &AsahiDrmDevice, + _reg_data: &driver::AsahiDrmRegistrationData<'_>, + data: &mut uapi::drm_asahi_gem_madvise, + file: &DrmFile, + ) -> Result { + if data.pad != 0 || data.madv > uapi::drm_asahi_gem_madv_DRM_ASAHI_GEM_MADV_DONTNEED { + return Err(EINVAL); + } + let gem = gem::Object::lookup_handle(file, data.handle)?; + data.retained = gem::madvise( + &gem, + data.madv == uapi::drm_asahi_gem_madv_DRM_ASAHI_GEM_MADV_DONTNEED, + )?; + Ok(0) + } + /// IOCTL: vm_bind: Map or unmap memory into a Vm. pub(crate) fn vm_bind( device: &AsahiDrmDevice, diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 787c6b3419ea2..7774cbfae63da 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -318,3 +318,12 @@ impl AsahiObject { !self.kernel && !self.exportable && self.kernel_mappings.load(Ordering::Acquire) == 0 } } + +/// Purge advice for a VM-private, non-exportable object. Returns whether the +/// contents are retained (0 once purged). +pub(crate) fn madvise(gem: &Object, dontneed: bool) -> Result { + if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE == 0 || gem.exportable { + return Err(EINVAL); + } + Ok(u32::from(gem.madvise(i32::from(dontneed)))) +} diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 1dd22596f3cb0..53c09dcb98d6d 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -146,6 +146,12 @@ impl Gate { .try_lock() .is_some_and(|state| state.users == 0 && !state.busy && !state.closed) } + + /// A page-table mutation happened outside a `Reclaim` (the purge path runs + /// on busy VMs). The next `enter` restores every INVALIDATED mapping. + pub(crate) fn note_evicted(&self) { + self.state.lock().evicted = true; + } } impl Reclaim { @@ -276,14 +282,14 @@ impl Vm { let Some(gate) = self.residency.as_ref() else { return 0; }; - if !gate.idle() - || self - .status - .as_ref() - .map_or(true, |status| status.get() != 0) + if self + .status + .as_ref() + .map_or(true, |status| status.get() != 0) { return 0; } + let idle = gate.idle(); let Some(mut inner) = self.gpuvm.try_lock_private() else { return 0; }; @@ -292,7 +298,11 @@ impl Vm { // this reservation; linked aliases retain their borrowed BO and GEM. let _ = unsafe { inner.for_each_private_bo(|_, bo| { let object = bo.obj(); - if object.idle_reclaim_candidate() && object.sole_sg_lease_releasable_locked() { + // Purgeable objects count on busy VMs too; idle backing only on idle ones. + if object.idle_reclaim_candidate() + && (object.madv_locked() > 0 || idle) + && object.sole_sg_lease_releasable_locked() + { pages = pages.saturating_add(object.size() / kernel::page::PAGE_SIZE); } Ok(true) @@ -301,12 +311,93 @@ impl Vm { pages } + /// Discard the backing of objects userspace marked DONTNEED. This needs + /// only the VM reservation: binding edits and restoration hold it too, + /// and the advice guarantees no pending or future GPU reference until + /// the next WILLNEED. Purged mappings stay INVALIDATED until unbound. + fn purge_advised_pages(&self, target: usize) -> usize { + let Some(gate) = self.residency.as_ref() else { + return 0; + }; + if self + .status + .as_ref() + .map_or(true, |status| status.get() != 0) + { + return 0; + } + let Some(mut inner) = self.gpuvm.try_lock_private() else { + return 0; + }; + let uat = inner.uat_inner.clone(); + let Some(shared) = uat.shared.try_lock() else { + return 0; + }; + let contexts = shared.contexts_naming_root(inner.ttb()); + let mut purged = 0usize; + // SAFETY: The private reservation excludes binding edits and restoration. + // Callbacks change PTEs/flags only; all aliases stay linked. + let _ = unsafe { inner.for_each_private_bo(|inner, bo| { + if purged >= target { return Ok(false); } + let object = bo.obj(); + if !object.idle_reclaim_candidate() || object.madv_locked() <= 0 { + return Ok(true); + } + { + let Some(segments) = bo.data().segments.try_lock() else { return Ok(true); }; + if segments.is_some() && !object.sole_sg_lease_releasable_locked() { + return Ok(true); + } + } + // Any mutation that does not end in a completed purge leaves + // INVALIDATED aliases with cleared PTEs while madv stays >= 0, so + // the next WILLNEED would report the contents retained. Record the + // eviction so the gate restores those aliases before the next job. + let mut mutated = false; + let invalidated = inner.invalidate_private_bo_mappings(bo, |inner, start, range| { + mutated = true; + let end = start.checked_add(range).ok_or(EOVERFLOW)?; + let _change = inner.mapping_mutation(); + inner.page_table.discard_partial_map(start..end)?; + inner.tlbi_context_mask(start, range as usize, contexts); + Ok(()) + }); + if let Err(error) = invalidated { + if mutated { + gate.note_evicted(); + } + return Err(error); + } + { + let Some(mut segments) = bo.data().segments.try_lock() else { + gate.note_evicted(); + return Ok(true); + }; + *segments = None; + } + if object.purge_locked().is_ok() { + purged = purged.saturating_add(object.size() / kernel::page::PAGE_SIZE); + } else { + gate.note_evicted(); + } + Ok(purged < target) + }) }; + drop(shared); + drop(inner); + purged + } + pub(crate) fn reclaim_idle_pages(&self, target: usize) -> usize { let Some(gate) = self.residency.as_ref() else { return 0; }; + let purged = self.purge_advised_pages(target); + if purged >= target { + return purged; + } + let target = target - purged; let Some(mut reclaim) = Gate::try_reclaim(gate) else { - return 0; + return purged; }; // Error fences cannot prove firmware ownership ended. if self @@ -318,14 +409,14 @@ impl Vm { .as_ref() .is_some_and(|lifetime| lifetime.can_reclaim()) { - return 0; + return purged; } let Some(mut inner) = self.gpuvm.try_lock_private() else { - return 0; + return purged; }; let uat = inner.uat_inner.clone(); let Some(shared) = uat.shared.try_lock() else { - return 0; + return purged; }; let contexts = shared.contexts_naming_root(inner.ttb()); let mut unpinned = 0usize; @@ -370,7 +461,7 @@ impl Vm { drop(shared); drop(inner); - unpinned + purged + unpinned } fn restore_idle_pages(&self) -> Result { @@ -385,16 +476,24 @@ impl Vm { loop { let map = loop { if let Some(inner) = self.gpuvm.try_lock_private() { - break inner.next_private_mapping( + let map = inner.next_private_mapping( &mut cursor, GpuVaFlag::INVALIDATED.into(), )?; + // Purged (DONTNEED) objects have no contents to restore; + // their mappings stay INVALIDATED until unbound. + // SAFETY: The private guard holds the VM reservation. + let purged = map.as_ref().is_some_and(|map| unsafe { map.object.madv_locked() < 0 }); + break (map, purged); } // Wait for reservation ownership without retaining a mapping. drop(lock_vm(&self.gpuvm, None, true)?); }; // The reservation guard has dropped before pinning backing, and // before any returned object reference can be released on failure. - let Some(map) = map else { break; }; + let (Some(map), purged) = map else { break; }; + if purged { + continue; + } let mut restored = 0u64; let bo_owner = lock_vm(&self.gpuvm, None, true)? .0 diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index cba36bd5db82e..cdb177bf4df05 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -93,6 +93,9 @@ enum drm_asahi_ioctl_id { /** @DRM_ASAHI_SUBMIT: Submit commands to a queue. */ DRM_ASAHI_SUBMIT, + + /** @DRM_ASAHI_GEM_MADVISE: Mark a VM-private object purgeable or needed. */ + DRM_ASAHI_GEM_MADVISE, }; #define DRM_ASAHI_MAX_CLUSTERS 64 @@ -519,6 +522,41 @@ struct drm_asahi_gem_mmap_offset { __u64 offset; }; +/** + * enum drm_asahi_gem_madv - Advice for DRM_IOCTL_ASAHI_GEM_MADVISE + */ +enum drm_asahi_gem_madv { + /** @DRM_ASAHI_GEM_MADV_WILLNEED: The object's contents are needed. */ + DRM_ASAHI_GEM_MADV_WILLNEED = 0, + /** + * @DRM_ASAHI_GEM_MADV_DONTNEED: The object is not referenced by any + * pending or future GPU work until the next WILLNEED. The kernel may + * discard its contents under memory pressure. + */ + DRM_ASAHI_GEM_MADV_DONTNEED = 1, +}; + +/** + * struct drm_asahi_gem_madvise - Arguments passed to + * DRM_IOCTL_ASAHI_GEM_MADVISE + * + * Only VM-private objects accept advice. A purged object keeps its handle + * and mappings but has no contents; userspace must not use it again. + */ +struct drm_asahi_gem_madvise { + /** @handle: Handle of the object. */ + __u32 handle; + + /** @madv: One of enum drm_asahi_gem_madv. */ + __u32 madv; + + /** @retained: Set by the kernel: 1 if the contents are retained, 0 if purged. */ + __u32 retained; + + /** @pad: Must be zero. */ + __u32 pad; +}; + /** * enum drm_asahi_bind_flags - Flags for GEM binding */ @@ -1430,6 +1468,7 @@ enum { DRM_IOCTL_ASAHI_QUEUE_CREATE = DRM_IOCTL_ASAHI(WR, QUEUE_CREATE, queue_create), DRM_IOCTL_ASAHI_QUEUE_DESTROY = DRM_IOCTL_ASAHI(W, QUEUE_DESTROY, queue_destroy), DRM_IOCTL_ASAHI_SUBMIT = DRM_IOCTL_ASAHI(W, SUBMIT, submit), + DRM_IOCTL_ASAHI_GEM_MADVISE = DRM_IOCTL_ASAHI(WR, GEM_MADVISE, gem_madvise), }; #if defined(__cplusplus) diff --git a/rust/helpers/drm.c b/rust/helpers/drm.c index 65f3f22b0e1d2..0975c35426f81 100644 --- a/rust/helpers/drm.c +++ b/rust/helpers/drm.c @@ -29,6 +29,12 @@ rust_helper_drm_gem_shmem_object_free(struct drm_gem_object *obj) return drm_gem_shmem_object_free(obj); } +__rust_helper bool +rust_helper_drm_gem_shmem_is_purgeable(struct drm_gem_shmem_object *shmem) +{ + return drm_gem_shmem_is_purgeable(shmem); +} + __rust_helper void rust_helper_drm_gem_shmem_object_print_info(struct drm_printer *p, unsigned int indent, const struct drm_gem_object *obj) diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index b74c9d368e660..fdf53dabc3cff 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -425,6 +425,46 @@ impl Object { result } + /// Apply purge advice (0 = needed, >0 = may be purged) under the + /// object's reservation lock. Returns false if the object was already + /// purged; the advice is then ignored. + pub fn madvise(&self, madv: i32) -> bool { + let _guard = DmaResvGuard::new(self); + // SAFETY: The reservation lock is held, as the C helper requires. + unsafe { bindings::drm_gem_shmem_madvise_locked(self.as_raw_shmem(), madv) != 0 } + } + + /// Current purge advice: negative once purged. + /// + /// # Safety + /// + /// The caller holds the object's DMA reservation lock. + pub unsafe fn madv_locked(&self) -> i32 { + // SAFETY: Reservation ownership serializes the field. + unsafe { (*self.as_raw_shmem()).madv } + } + + /// Discard the backing of a purgeable object. The handle and logical + /// mappings remain; the contents are gone and `madv_locked()` turns negative. + /// + /// # Safety + /// + /// The caller holds this object's DMA reservation lock, has removed and + /// invalidated every device mapping, and no Rust SG lease remains. + pub unsafe fn purge_locked(&self) -> Result { + // SAFETY: The caller's contract; the exclusive state below excludes new leases. + if !unsafe { bindings::drm_gem_shmem_is_purgeable(self.as_raw_shmem()) } { + return Err(EBUSY); + } + self.sgt_access + .compare_exchange(0, SGT_RECLAIMING, Ordering::AcqRel, Ordering::Acquire) + .map_err(|_| EBUSY)?; + // SAFETY: See above. + unsafe { bindings::drm_gem_shmem_purge_locked(self.as_raw_shmem()) }; + self.sgt_access.store(0, Ordering::Release); + Ok(()) + } + /// Creates and returns an owned reference to a virtual kernel memory mapping for this object. #[inline] pub fn owned_vmap(&self) -> Result> { From f739d01355c1561f78c632461deb6ec24ae870ef Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:57:01 -0400 Subject: [PATCH 0827/2680] drm/asahi: drop the sysfs object and queue-slot counters gem_stats and g17p_queue_slots exported driver-internal counters as read-only module parameters, backed by global atomics on the GEM constructor/destructor, the QID lease table, compute queue state transitions, the teardown and settle passes and every ENOSPC return. Nothing in the driver reads them, and DRM exposes memory accounting through fdinfo. Remove the parameters, the atomics and the hooks; an exhausted pool returns ENOSPC without a counted log line. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c8c442ae6a83502a90fca56d1b8e08e669ce03cb) --- drivers/gpu/drm/asahi/asahi.rs | 1 - drivers/gpu/drm/asahi/g17/context.rs | 2 +- drivers/gpu/drm/asahi/g17/kick.rs | 49 +---- drivers/gpu/drm/asahi/g17/mod.rs | 1 - drivers/gpu/drm/asahi/g17/queue/compute.rs | 16 -- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 1 - drivers/gpu/drm/asahi/g17/queue_stats.rs | 177 ------------------ drivers/gpu/drm/asahi/g17/runtime/backend.rs | 6 - .../drm/asahi/g17/runtime/render/settle.rs | 3 - drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 2 - drivers/gpu/drm/asahi/gem.rs | 16 +- drivers/gpu/drm/asahi/gem_stats.rs | 91 --------- 12 files changed, 9 insertions(+), 356 deletions(-) delete mode 100644 drivers/gpu/drm/asahi/g17/queue_stats.rs delete mode 100644 drivers/gpu/drm/asahi/gem_stats.rs diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi.rs index cb67389635f1b..84d5d92e7dce8 100644 --- a/drivers/gpu/drm/asahi/asahi.rs +++ b/drivers/gpu/drm/asahi/asahi.rs @@ -14,7 +14,6 @@ mod file; mod float; mod fw; mod gem; -mod gem_stats; mod g17; mod gpu; mod hw; diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index eb4ddcac7b808..b7d0a351f234d 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -39,7 +39,7 @@ impl QosIds { let mut used = self.used.lock(); let id = used.trailing_ones(); if id == 128 { - return Err(crate::g17::queue_stats::enospc(crate::g17::queue_stats::Pool::QueueContextQos)); + return Err(ENOSPC); } *used |= 1u128 << id; Ok(QosLease { diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index aa2c054a9bbc7..f109c8261476e 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -6,7 +6,7 @@ //! and their ring backing stay owned until both firmware processors stop; //! command completion alone does not detach a hardware queue. -use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; +use core::sync::atomic::{fence, Ordering}; use kernel::prelude::*; use super::{ @@ -17,28 +17,8 @@ use crate::{hw::t8140, mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; -pub(super) const COMPUTE_RENDER_RESERVE: usize = 16; -pub(super) static LEASED: [AtomicU32; 3] = [const { AtomicU32::new(0) }; 3]; -pub(super) static PUBLISHED: [AtomicU32; 3] = [const { AtomicU32::new(0) }; 3]; -pub(super) static HIGH_WATER: AtomicU32 = AtomicU32::new(0); -pub(super) static EXHAUSTED: AtomicU64 = AtomicU64::new(0); -pub(super) static RESERVE_REFUSED: AtomicU64 = AtomicU64::new(0); - -pub(super) fn leased_total() -> u32 { - LEASED.iter().map(|count| count.load(Ordering::Relaxed)).sum() -} - -fn exhausted(engine: DataMaster, reserve: bool) -> Error { - if reserve { - RESERVE_REFUSED.fetch_add(1, Ordering::Relaxed); - } else { - EXHAUSTED.fetch_add(1, Ordering::Relaxed); - } - super::queue_stats::enospc(match engine { - DataMaster::Compute => super::queue_stats::Pool::QidCompute, - _ => super::queue_stats::Pool::QidRenderPair, - }) -} +/// QIDs kept free for render pairs when compute queues fill the table. +const COMPUTE_RENDER_RESERVE: usize = 16; /// Stable physical-channel identity. The owner must not be reused within a /// device lifetime, including after an unpublished allocation is cancelled. @@ -106,13 +86,13 @@ impl QueueIds { // Healthy existing channels keep their QID. Never recycle a published // identity to recover capacity after a fault; leave eight render pairs. if engine == DataMaster::Compute && QID_COUNT - self.leased <= COMPUTE_RENDER_RESERVE { - return Err(exhausted(engine, true)); + return Err(ENOSPC); } let index = preferred .map(usize::from) .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) .or_else(|| self.entries.iter().position(Option::is_none)) - .ok_or_else(|| exhausted(engine, false))?; + .ok_or(ENOSPC)?; let id = Id { qid: index as u8, owner, @@ -123,8 +103,6 @@ impl QueueIds { published: false, }); self.leased += 1; - LEASED[engine as usize].fetch_add(1, Ordering::Relaxed); - HIGH_WATER.fetch_max(leased_total(), Ordering::Relaxed); Ok(id) } @@ -145,10 +123,7 @@ impl QueueIds { if entry.id != id { return Err(EINVAL); } - if !entry.published { - entry.published = true; - PUBLISHED[id.engine as usize].fetch_add(1, Ordering::Relaxed); - } + entry.published = true; Ok(()) } @@ -158,7 +133,6 @@ impl QueueIds { Some(entry) if entry.id == id && !entry.published => { *slot = None; self.leased -= 1; - LEASED[id.engine as usize].fetch_sub(1, Ordering::Relaxed); Ok(()) } _ => Err(EINVAL), @@ -166,17 +140,6 @@ impl QueueIds { } } -impl Drop for QueueIds { - fn drop(&mut self) { - for entry in self.entries.iter().flatten() { - LEASED[entry.id.engine as usize].fetch_sub(1, Ordering::Relaxed); - if entry.published { - PUBLISHED[entry.id.engine as usize].fetch_sub(1, Ordering::Relaxed); - } - } - } -} - // Aliases drop before their backing. A compute queue is allocated low first; // a render queue is allocated high first. enum Backing { diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index f98fcb5e93de8..695ea787d314e 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -44,7 +44,6 @@ mod runtime; pub(crate) mod status; mod teardown; mod timeout; -mod queue_stats; mod validation; use core::any::Any; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 5ca9b85f8aeb6..fcd0a1d6b40e6 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -364,7 +364,6 @@ impl Backing { /// Installed graph and its exact logical owner. A released idle graph may change owners only /// after a retirement witness. Failed work retains its pointers independently of fence state. pub(crate) struct Queue { - slot_accounting: crate::g17::queue_stats::ComputeAccount, owner: Option, binding: Binding, previous: Option, @@ -414,7 +413,6 @@ impl Queue { context.mark_published(); Ok(Self { owner: Some(owner), - slot_accounting: crate::g17::queue_stats::ComputeAccount::new(), binding, previous: None, kick, @@ -449,11 +447,6 @@ impl Queue { pub(crate) fn qid(&self) -> u8 { self.kick.id().qid() } - fn update_slot_accounting(&mut self) { - self.slot_accounting.update(self.owner.is_some(), self.released, - self.quarantined, self.retired_by_teardown); - } - pub(crate) fn owner(&self) -> Option { self.owner } @@ -601,7 +594,6 @@ impl Queue { self.failure_status_pending = true; self.retirement_proved = false; self.retired_by_teardown = false; - self.update_slot_accounting(); let all_spared = self.active.len != 0 && self.active.iter().all(|a| a.packet.completion.spared()); self.spared_quarantine = all_spared && error == ENODATA; @@ -707,7 +699,6 @@ impl Queue { self.retirement_ready = self.active.len == 0; if self.released && self.retirement_ready { self.owner = None; - self.update_slot_accounting(); } } Ok(None) | Err(EAGAIN) => { @@ -814,7 +805,6 @@ impl Queue { self.submitted = 0; self.kick.clear_parent_after_recovery(); self.quarantined = false; - self.update_slot_accounting(); self.retire_pending = false; self.retirement_ready = true; if self @@ -826,7 +816,6 @@ impl Queue { } if self.released { self.owner = None; - self.update_slot_accounting(); } } Ok(()) @@ -969,7 +958,6 @@ impl Queue { self.released = false; self.retirement_ready = false; self.retired_by_teardown = false; - self.update_slot_accounting(); Ok(()) } /// True allows the registry to cancel the never-published QID and drop this graph. An @@ -979,13 +967,11 @@ impl Queue { return Ok(true); } self.released = true; - self.update_slot_accounting(); if self.active.len == 0 && !self.quarantined { if let Some(previous) = self.previous.as_ref().filter(|p| p.publication.is_none()) { if let Err(error) = self.graph.set_owner(&previous.context) { self.quarantined = true; self.retired_by_teardown = true; - self.update_slot_accounting(); return Err(error); } let mut previous = self.previous.take().ok_or(EIO)?; @@ -998,7 +984,6 @@ impl Queue { } if self.retirement_ready { self.owner = None; - self.update_slot_accounting(); } } Ok(false) @@ -1066,7 +1051,6 @@ impl Queue { } self.released = true; self.retired_by_teardown = true; - self.update_slot_accounting(); if let Some(active) = self .active .iter_mut() diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index e420427d594bf..d1048fd04607f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -197,7 +197,6 @@ impl Queue { self.spared_quarantine = false; self.retire_pending = false; self.retirement_ready = false; - self.update_slot_accounting(); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/queue_stats.rs b/drivers/gpu/drm/asahi/g17/queue_stats.rs deleted file mode 100644 index 5a5605e1d19a2..0000000000000 --- a/drivers/gpu/drm/asahi/g17/queue_stats.rs +++ /dev/null @@ -1,177 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Firmware queue-slot accounting (`g17p_queue_slots`, root-only, read-only -//! sysfs). Relaxed counters; exhaustion logging is rate-limited. -//! -//! Every render pair and compute queue leases QIDs from one 128-entry table -//! (`kick::QueueIds`). A published QID is kept until processor stop, and a -//! quarantined or exit-killed compute graph is never handed out again, so a -//! boot can run out of QIDs while memory is fine. Queue creation then fails -//! with ENOSPC, and so does a new render pair (`render-warm-pairs`). The -//! `qids` fields are exact (kept by the table itself); compute -//! graph ownership gauges track structural changes. Pending teardown and -//! draining-pair counts are sampled in their existing service passes. -//! -//! Every ENOSPC from one of these pools is counted by pool and named in -//! dmesg (the first one, then each power of two), so a failed QUEUE_CREATE -//! or submit says which pool ran out. -use core::fmt::Write; -use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; -use kernel::{bindings, module_param::KernelParam, prelude::*}; - -use super::kick as g17_channel; -use g17_channel::QID_COUNT; - -static COMPUTE_GRAPHS: AtomicU32 = AtomicU32::new(0); -static COMPUTE_OWNED: AtomicU32 = AtomicU32::new(0); -static COMPUTE_RELEASED: AtomicU32 = AtomicU32::new(0); -static COMPUTE_QUARANTINED: AtomicU32 = AtomicU32::new(0); -static COMPUTE_EXIT_KILLED: AtomicU32 = AtomicU32::new(0); -static RETIRED_GRAPHS: AtomicU32 = AtomicU32::new(0); -static TEARDOWNS: AtomicU32 = AtomicU32::new(0); -static RENDER_QUARANTINED: AtomicU32 = AtomicU32::new(0); - -/// Pools whose exhaustion returns ENOSPC to a client. -#[derive(Clone, Copy)] -pub(crate) enum Pool { - /// A render slot's first use leases two QIDs (`render-warm-pairs`). - QidRenderPair, - /// A new compute graph leases one QID (queue creation / first compute). - QidCompute, - /// A new logical queue context (QUEUE_CREATE): its only 128-wide pool - /// is the QoS hardware-buffer IDs, held by logical queues and installed - /// graph bindings. - QueueContextQos, -} - -const POOL_COUNT: usize = 3; -const POOL_NAMES: [&str; POOL_COUNT] = ["qid-render-pair", "qid-compute", "queue-context-qos"]; -static POOL_ENOSPC: [AtomicU64; POOL_COUNT] = - [AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0)]; -/// Submits that failed with ENOSPC, whichever pool. -static SUBMIT_ENOSPC: AtomicU64 = AtomicU64::new(0); - -fn log_enospc(pool: &str, stage: &str, count: u64) { - // First one, then each power of two: a stuck pool cannot flood dmesg. - if count.is_power_of_two() { - pr_warn!( - "G17P ENOSPC pool={} stage={} (#{}): qids leased={}/{} compute={} compute-quarantined={} \ - reserve-refused={}\n", - pool, stage, count, g17_channel::leased_total(), QID_COUNT, - g17_channel::LEASED[2].load(Ordering::Relaxed), - COMPUTE_QUARANTINED.load(Ordering::Relaxed), - g17_channel::RESERVE_REFUSED.load(Ordering::Relaxed), - ); - } -} - -/// Count and name one ENOSPC from `pool`, and return it. -#[cold] -pub(crate) fn enospc(pool: Pool) -> Error { - let count = POOL_ENOSPC[pool as usize].fetch_add(1, Ordering::Relaxed) + 1; - log_enospc(POOL_NAMES[pool as usize], "-", count); - ENOSPC -} - -/// A submit failed with ENOSPC at `stage`. Pools tagged at their source -/// were already counted and named; this names the rest by stage. -#[cold] -pub(crate) fn note_submit_enospc(stage: &str) { - let count = SUBMIT_ENOSPC.fetch_add(1, Ordering::Relaxed) + 1; - // Source-specific QID/QoS failures already counted their actual pool. - // This boundary knows the engine stage, not which nested allocator failed. - log_enospc("submit", stage, count); -} - -/// Incremental graph ownership accounting. Normal submissions do not touch -/// these counters; only graph creation, ownership transitions and drop do. -pub(crate) struct ComputeAccount { flags: u8 } -impl ComputeAccount { - pub(crate) fn new() -> Self { - COMPUTE_GRAPHS.fetch_add(1, Ordering::Relaxed); - COMPUTE_OWNED.fetch_add(1, Ordering::Relaxed); - Self { flags: 1 } - } - pub(crate) fn update(&mut self, owned: bool, released: bool, quarantined: bool, exit_killed: bool) { - let next = u8::from(owned) | (u8::from(released) << 1) - | (u8::from(quarantined) << 2) | (u8::from(exit_killed) << 3); - let changed = next ^ self.flags; - for (bit, counter) in [&COMPUTE_OWNED, &COMPUTE_RELEASED, - &COMPUTE_QUARANTINED, &COMPUTE_EXIT_KILLED].into_iter().enumerate() - { - if changed & (1 << bit) != 0 { - if next & (1 << bit) != 0 { counter.fetch_add(1, Ordering::Relaxed); } - else { counter.fetch_sub(1, Ordering::Relaxed); } - } - } - self.flags = next; - } -} -impl Drop for ComputeAccount { - fn drop(&mut self) { - self.update(false, false, false, false); - COMPUTE_GRAPHS.fetch_sub(1, Ordering::Relaxed); - } -} - -pub(crate) fn note_teardowns(count: usize) { - TEARDOWNS.store(count as u32, Ordering::Relaxed); -} -pub(crate) fn note_draining_pairs(count: usize) { - RENDER_QUARANTINED.store(count as u32, Ordering::Relaxed); -} - -struct Output<'a> { bytes: &'a mut [u8], used: usize } -impl Write for Output<'_> { - fn write_str(&mut self, value: &str) -> core::fmt::Result { - if value.len() > self.bytes.len() - self.used { return Err(core::fmt::Error); } - self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); - self.used += value.len(); - Ok(()) - } -} - -unsafe extern "C" fn slots_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { - if buf.is_null() { return -22; } - // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. - let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1024) }; - let mut out = Output { bytes, used: 0 }; - let r = |c: &AtomicU32| c.load(Ordering::Relaxed); - let e = |pool: usize| POOL_ENOSPC[pool].load(Ordering::Relaxed); - let leased = g17_channel::leased_total(); - let published: u32 = g17_channel::PUBLISHED.iter().map(r).sum(); - let _ = writeln!(out, - "qids leased={} free={} published={} tiling={} fragment={} compute={} high_water={} exhausted={} \ - reserve={} reserve_refused={} \ - compute graphs={} owned={} released={} quarantined={} exit_killed={} retired_graphs={} \ - teardowns={} render_draining_pairs={} \ - enospc qid-render-pair={} qid-compute={} queue-context-qos={} submits={}", - leased, (QID_COUNT as u32).saturating_sub(leased), published, - r(&g17_channel::LEASED[0]), r(&g17_channel::LEASED[1]), r(&g17_channel::LEASED[2]), - r(&g17_channel::HIGH_WATER), g17_channel::EXHAUSTED.load(Ordering::Relaxed), - g17_channel::COMPUTE_RENDER_RESERVE, g17_channel::RESERVE_REFUSED.load(Ordering::Relaxed), - r(&COMPUTE_GRAPHS), r(&COMPUTE_OWNED), r(&COMPUTE_RELEASED), r(&COMPUTE_QUARANTINED), - r(&COMPUTE_EXIT_KILLED), r(&RETIRED_GRAPHS), r(&TEARDOWNS), r(&RENDER_QUARANTINED), - e(0), e(1), e(2), SUBMIT_ENOSPC.load(Ordering::Relaxed)); - out.used as core::ffi::c_int -} - -unsafe extern "C" fn slots_set(_: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { - -1 // EPERM: read-only counters -} - -static SLOTS_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { - flags: 0, set: Some(slots_set), get: Some(slots_get), free: None, -}; -#[link_section = "__param"] -#[used] -static SLOTS_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { - name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { - c"g17p_queue_slots" - } else { c"asahi.g17p_queue_slots" }), - #[cfg(MODULE)] - mod_: core::ptr::addr_of_mut!(bindings::__this_module), - #[cfg(not(MODULE))] - mod_: core::ptr::null_mut(), - ops: core::ptr::from_ref(&SLOTS_OPS), perm: 0o400, level: -1, flags: 0, - __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, -}); diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 7f204b9328ffb..ef72699e8e7ce 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -138,12 +138,6 @@ impl job::Backend for Backend { Engine::Render => self.render_packet(&packet, &mut deferred), } }; - if result.as_ref().is_err_and(|error| *error == ENOSPC) { - crate::g17::queue_stats::note_submit_enospc(match packet.engine() { - Engine::Compute => "compute-publish", - Engine::Render => "render-publish", - }); - } self.release_backlog(&packet); self.finish_publication(&packet, result, &mut deferred) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index 0a4634bccaea5..415d1f391f601 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -164,7 +164,6 @@ impl crate::g17::Firmware { return Ok(()); } let mut serviced = false; - let mut draining = 0; for slot in 1..self.queues.render.entries.len() { if !self.queues.render.entries[slot] .as_ref() @@ -173,7 +172,6 @@ impl crate::g17::Firmware { { continue; } - draining += 1; if let Err(error) = self.drain_render_slot(slot, &mut serviced, deferred) { dev_warn!( self.primary.state.shared.dev.as_ref(), @@ -183,7 +181,6 @@ impl crate::g17::Firmware { ); } } - crate::g17::queue_stats::note_draining_pairs(draining); Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index f310f8154aae7..6eb8da6377533 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -169,7 +169,6 @@ impl Firmware { /// The event worker has already reclaimed drained pairs and reannounced /// pending controls. Recovery blocks new releases, never an observed ack. pub(in crate::g17) fn service_teardown(&mut self, reclaim: &mut ReclaimBatch) -> Result { - crate::g17::queue_stats::note_teardowns(self.queues.teardown.len()); self.queues.render.collect_reclaimable(reclaim)?; if self.queues.teardown.is_empty() { return Ok(()); @@ -199,7 +198,6 @@ impl Firmware { .is_some() {} teardown.report_stalled(super::now_ns(), &mut service); - crate::g17::queue_stats::note_teardowns(teardown.len()); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 7774cbfae63da..39ab0c354fa26 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -45,7 +45,7 @@ use crate::{ const DEBUG_CLASS: DebugFlags = DebugFlags::Gem; /// Represents the inner data of a GEM object for this driver. -#[pin_data(PinnedDrop)] +#[pin_data] pub(crate) struct AsahiObject { /// ID for debug id: u64, @@ -159,13 +159,6 @@ impl ObjectRef { } } -#[pinned_drop] -impl PinnedDrop for AsahiObject { - fn drop(self: Pin<&mut Self>) { - crate::gem_stats::note_free(self.kernel, self.size); - } -} - pub(crate) struct AsahiObjConfig { flags: u32, exportable: bool, @@ -250,12 +243,7 @@ impl DriverObject for AsahiObject { mod_pr_debug!("AsahiObject::new id={}\n", id); try_pin_init!(AsahiObject { id, - // All remaining fields are infallible. PinnedDrop balances the - // accounting once this initializer completes. - size: { - crate::gem_stats::note_new(args.kernel, size); - size - }, + size, flags: args.flags, exportable: args.exportable, kernel: args.kernel, diff --git a/drivers/gpu/drm/asahi/gem_stats.rs b/drivers/gpu/drm/asahi/gem_stats.rs deleted file mode 100644 index abd01922273bd..0000000000000 --- a/drivers/gpu/drm/asahi/gem_stats.rs +++ /dev/null @@ -1,91 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Live GEM object accounting (`gem_stats`, root-only, read-only sysfs). -//! -//! Every shmem object this driver creates is counted from creation until its -//! last reference drops, split by user and kernel ownership. Once a GEM object -//! is mapped into a GPU VM its pages stay pinned (and unevictable) until the -//! object is freed, so `user_bytes + kernel_bytes` is an upper bound on the -//! GPU share of /proc/meminfo `Unevictable`, and a value that keeps growing -//! after clients exit identifies a reference leak in the driver. The -//! `*_created` / `*_freed` totals show allocation churn. -use core::fmt::Write; -use core::sync::atomic::{AtomicU64, Ordering}; -use kernel::{bindings, module_param::KernelParam}; - -static USER_OBJECTS: AtomicU64 = AtomicU64::new(0); -static USER_BYTES: AtomicU64 = AtomicU64::new(0); -static USER_CREATED: AtomicU64 = AtomicU64::new(0); -static KERNEL_OBJECTS: AtomicU64 = AtomicU64::new(0); -static KERNEL_BYTES: AtomicU64 = AtomicU64::new(0); -static KERNEL_CREATED: AtomicU64 = AtomicU64::new(0); - -fn counters(kernel: bool) -> (&'static AtomicU64, &'static AtomicU64, &'static AtomicU64) { - if kernel { - (&KERNEL_OBJECTS, &KERNEL_BYTES, &KERNEL_CREATED) - } else { - (&USER_OBJECTS, &USER_BYTES, &USER_CREATED) - } -} - -/// Count one new GEM object. -pub(crate) fn note_new(kernel: bool, size: usize) { - let (objects, bytes, created) = counters(kernel); - objects.fetch_add(1, Ordering::Relaxed); - bytes.fetch_add(size as u64, Ordering::Relaxed); - created.fetch_add(1, Ordering::Relaxed); -} - -/// Count one freed GEM object. -pub(crate) fn note_free(kernel: bool, size: usize) { - let (objects, bytes, _) = counters(kernel); - objects.fetch_sub(1, Ordering::Relaxed); - bytes.fetch_sub(size as u64, Ordering::Relaxed); -} - -struct Output<'a> { bytes: &'a mut [u8], used: usize } -impl Write for Output<'_> { - fn write_str(&mut self, value: &str) -> core::fmt::Result { - if value.len() > self.bytes.len() - self.used { return Err(core::fmt::Error); } - self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); - self.used += value.len(); - Ok(()) - } -} - -unsafe extern "C" fn stats_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { - if buf.is_null() { return -22; } - // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. - let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1024) }; - let mut out = Output { bytes, used: 0 }; - let r = |c: &AtomicU64| c.load(Ordering::Relaxed); - let uc = r(&USER_CREATED); - let kc = r(&KERNEL_CREATED); - let (uo, ko) = (r(&USER_OBJECTS), r(&KERNEL_OBJECTS)); - let _ = writeln!(out, - "user_objects={} user_bytes={} user_created={} user_freed={} \ - kernel_objects={} kernel_bytes={} kernel_created={} kernel_freed={}", - uo, r(&USER_BYTES), uc, uc.wrapping_sub(uo), - ko, r(&KERNEL_BYTES), kc, kc.wrapping_sub(ko)); - out.used as core::ffi::c_int -} - -unsafe extern "C" fn stats_set(_: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { - -1 // EPERM: read-only counters -} - -static STATS_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { - flags: 0, set: Some(stats_set), get: Some(stats_get), free: None, -}; -#[link_section = "__param"] -#[used] -static STATS_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { - name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { - c"gem_stats" - } else { c"asahi.gem_stats" }), - #[cfg(MODULE)] - mod_: core::ptr::addr_of_mut!(bindings::__this_module), - #[cfg(not(MODULE))] - mod_: core::ptr::null_mut(), - ops: core::ptr::from_ref(&STATS_OPS), perm: 0o400, level: -1, flags: 0, - __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, -}); From 516fc39a83e4fa49530daec9670577a7142127b2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:57:43 -0400 Subject: [PATCH 0828/2680] drm/asahi: drop the stalled scheduler-release warning A release whose firmware acknowledgement took longer than two seconds logged "release is still pending" once and otherwise changed nothing: the entry stayed retained until the acknowledgement arrived. Hardware runs never produced the line. Keep the acknowledgement state machine and its polling window; remove the warning bookkeeping. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 152092e32a7fd7a7b3a96b8e45be7db78934b7d8) --- drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 4 ---- drivers/gpu/drm/asahi/g17/teardown.rs | 21 ++----------------- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 6eb8da6377533..8a637e0b3ed01 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -158,9 +158,6 @@ impl Host for Service<'_> { context.id(), error ), - Failure::Stalled => { - dev_warn!(dev, "Scheduler {} release is still pending\n", context.id()) - } } } } @@ -197,7 +194,6 @@ impl Firmware { .take_consumed(consumer, producer, &mut service)? .is_some() {} - teardown.report_stalled(super::now_ns(), &mut service); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index f53c2d45c129a..791ac690c0f50 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -22,17 +22,15 @@ struct Entry { context: Arc, phase: Phase, armed: u64, - warned: bool, announce_pending: bool, } -/// Publication and announcement errors retain the entry. A stalled release is reported once; -/// its timeout never authorizes freeing memory still owned by firmware. +/// Publication and announcement errors retain the entry; no timeout frees +/// memory the firmware may still own. #[derive(Clone, Copy)] pub(crate) enum Failure { Publication(Error), Announcement(Error), - Stalled, } /// Borrowed device operations. Readiness covers every matching installed and displaced graph, @@ -88,7 +86,6 @@ impl Pending { context: context.clone(), phase, armed: now, - warned: false, announce_pending: false, }) .map_err(|_| ENOSPC) @@ -143,7 +140,6 @@ impl Pending { // One successful retry starts its consumption window. // Failed retries never slide the polling deadline. entry.armed = now; - entry.warned = false; } continue; } @@ -177,7 +173,6 @@ impl Pending { // A long command drain must not consume the acknowledgement // polling window before this release exists on the control ring. entry.armed = now; - entry.warned = false; entry.announce_pending = true; fence(Ordering::SeqCst); match host.notify() { @@ -228,16 +223,4 @@ impl Pending { entry.phase != Phase::Deferred && now.saturating_sub(entry.armed) < POLL_NS }) } - - pub(crate) fn report_stalled(&mut self, now: u64, host: &mut impl Host) { - for entry in self.entries.iter_mut() { - if entry.phase != Phase::Deferred - && !entry.warned - && now.saturating_sub(entry.armed) >= POLL_NS - { - entry.warned = true; - host.report(&entry.context, Failure::Stalled); - } - } - } } From bb7a7ec05d64b862d503ffaf1098fa00536fcb32 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 11:15:54 -0400 Subject: [PATCH 0829/2680] drm/asahi: build constant firmware objects from word tables PowerState, SecondaryConfig, PowerConfigHead, PowerConfigTail and HwDataAux are written once from constants and never read by the host. Their 600-odd unnamed fields, offset assertions and field-by-field builders are replaced by byte arrays filled from (offset, value, repeat) word tables, plus the one address HwDataAux carries. The five objects are byte-identical to the previous builders for every input (verified with a host-side dump of each object). Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 24ce1bcdd8ebf0f8664c043a639c0201286f94e5) --- drivers/gpu/drm/asahi/g17/fw/initdata.rs | 1375 ++++++---------------- 1 file changed, 341 insertions(+), 1034 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index c25b843a913e7..8dbb6a81bcd47 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -1406,187 +1406,7 @@ impl StatusBlock { } } -/// Power state the primary status B carries for the secondary instance, -/// named by [`SecondaryRoot::power_state_va`]. -#[repr(C, packed)] -pub(crate) struct PowerState { - pub(crate) unk_0: [u8; 0x10], - /// Required: 0xf (15). - pub(crate) unk_10: u32, - /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). - pub(crate) unk_14: u32, - /// Required: 0x4088_0000 (4.25 as an IEEE-754 single). - pub(crate) unk_18: u32, - /// Required: 0x28 (40). - pub(crate) unk_1c: u32, - /// Required: 1. - pub(crate) unk_20: u32, - pub(crate) unk_24: [u8; 0xa0], - /// Non-zero asks the secondary to reload the limiter settings that - /// follow into the power configuration; zero at boot. - pub(crate) temp_update: u32, - /// Target temperature of the limiter ([`TEMP_TARGET`]). - pub(crate) temp_target: u32, - /// Release temperature of the limiter ([`TEMP_RELEASE`]). - pub(crate) temp_release: u32, - pub(crate) unk_d0: [u8; 8], - /// Proportional gain of the limiter ([`TEMP_KP`]). - pub(crate) temp_kp: u32, - /// Integral gain of the limiter ([`TEMP_KI`]). - pub(crate) temp_ki: u32, - pub(crate) unk_e0: [u8; 0x10], - /// Required: 0x26ac (9900). - pub(crate) unk_f0: u32, - /// Required: 0xc8 (200). - pub(crate) unk_f4: u32, - pub(crate) unk_f8: [u8; 0x10], - /// Required: 0x42c8_0000 (100.0 as an IEEE-754 single). - pub(crate) unk_108: u32, - /// Required: 0x43c8_0000 (400.0 as an IEEE-754 single). - pub(crate) unk_10c: u32, - /// Required: 0xc8 (200). - pub(crate) unk_110: u32, - pub(crate) unk_114: [u8; 0x14], - /// Required: 1. - pub(crate) unk_128: u32, - /// Required: 0x2616 (9750). - pub(crate) unk_12c: u32, - /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). - pub(crate) unk_130: u32, - /// Required: 0x40cc_cccd (6.4 as an IEEE-754 single). - pub(crate) unk_134: u32, - pub(crate) unk_138: [u8; 0x14], - /// Required: 1. - pub(crate) unk_14c: u32, - pub(crate) unk_150: [u8; 0x70], - /// Required: 1. - pub(crate) unk_1c0: u32, - /// Required: 1. - pub(crate) unk_1c4: u32, - /// Required: 4. - pub(crate) unk_1c8: u32, - /// Required: 1 in every entry. - pub(crate) unk_1cc: [u32; 4], - pub(crate) unk_1dc: [u8; 0x308], - /// Required: 1. - pub(crate) unk_4e4: u32, - /// Required: 0x1f4 (500). - pub(crate) unk_4e8: u32, - /// Required: 6. - pub(crate) unk_4ec: u32, - /// Required: 0x30d4 (12500) in every entry. - pub(crate) unk_4f0: [u32; 6], - pub(crate) unk_508: [u8; 0x14], - /// Required: 0x30d4 (12500) in every entry. - pub(crate) unk_51c: [u32; 5], - /// Required: 1. - pub(crate) unk_530: u32, - pub(crate) unk_534: [u8; 0x58], - /// Required: 6. - pub(crate) unk_58c: u32, - /// Required: 0xdac (3500). - pub(crate) unk_590: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_594: u32, - /// Required: 0xbb8 (3000). - pub(crate) unk_598: u32, - /// Required: 0x64 (100). - pub(crate) unk_59c: u32, - pub(crate) unk_5a0: [u8; 0x2c], - /// Required: 1. - pub(crate) unk_5cc: u32, - /// Required: 4. - pub(crate) unk_5d0: u32, - pub(crate) unk_5d4: [u8; 8], - /// Required: 0x1f40 (8000). - pub(crate) unk_5dc: u32, - /// Required: 0xc8 (200). - pub(crate) unk_5e0: u32, - /// Required: 0xfa0 (4000). - pub(crate) unk_5e4: u32, - /// Required: 0xc8 (200). - pub(crate) unk_5e8: u32, - /// Required: 0x7d0 (2000). - pub(crate) unk_5ec: u32, - /// Required: 0xc8 (200). - pub(crate) unk_5f0: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_5f4: u32, - /// Required: 0xc8 (200). - pub(crate) unk_5f8: u32, - pub(crate) unk_5fc: [u8; 8], - /// Required: 0x4170_0000 (15.0 as an IEEE-754 single). - pub(crate) unk_604: u32, - /// Required: 0x40a0_0000 (5.0 as an IEEE-754 single). - pub(crate) unk_608: u32, - /// Required: 0x20 (32). - pub(crate) unk_60c: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_610: u32, - pub(crate) unk_614: [u8; 0xc], - /// Required: 0x384 (900). - pub(crate) unk_620: u32, - pub(crate) unk_624: [u8; 0x9c], -} - -// SAFETY: `PowerState` consists of integers and arrays of integers only. -unsafe impl Zeroable for PowerState {} -static_assert!(size_of::() == 0x6c0); - -impl PowerState { - fn init(&mut self) { - self.unk_10 = 0xf; - self.unk_14 = 0x3f00_0000; - self.unk_18 = 0x4088_0000; - self.unk_1c = 0x28; - self.unk_20 = 1; - self.temp_target = TEMP_TARGET; - self.temp_release = TEMP_RELEASE; - self.temp_kp = TEMP_KP; - self.temp_ki = TEMP_KI; - self.unk_f0 = 0x26ac; - self.unk_f4 = 0xc8; - self.unk_108 = 0x42c8_0000; - self.unk_10c = 0x43c8_0000; - self.unk_110 = 0xc8; - self.unk_128 = 1; - self.unk_12c = 0x2616; - self.unk_130 = 0x3f00_0000; - self.unk_134 = 0x40cc_cccd; - self.unk_14c = 1; - self.unk_1c0 = 1; - self.unk_1c4 = 1; - self.unk_1c8 = 4; - self.unk_1cc = [1; 4]; - self.unk_4e4 = 1; - self.unk_4e8 = 0x1f4; - self.unk_4ec = 6; - self.unk_4f0 = [0x30d4; 6]; - self.unk_51c = [0x30d4; 5]; - self.unk_530 = 1; - self.unk_58c = 6; - self.unk_590 = 0xdac; - self.unk_594 = 0x3e8; - self.unk_598 = 0xbb8; - self.unk_59c = 0x64; - self.unk_5cc = 1; - self.unk_5d0 = 4; - self.unk_5dc = 0x1f40; - self.unk_5e0 = 0xc8; - self.unk_5e4 = 0xfa0; - self.unk_5e8 = 0xc8; - self.unk_5ec = 0x7d0; - self.unk_5f0 = 0xc8; - self.unk_5f4 = 0x3e8; - self.unk_5f8 = 0xc8; - self.unk_604 = 0x4170_0000; - self.unk_608 = 0x40a0_0000; - self.unk_60c = 0x20; - self.unk_610 = 0x3e8; - self.unk_620 = 0x384; - } -} /// Size of the firmware-control page named by the primary status B. pub(crate) const FWCTL_SIZE: usize = 0x4000; @@ -1667,46 +1487,7 @@ impl PrimaryStatusB { } } -/// Configuration of the secondary instance, named by -/// [`SecondaryRoot::config_va`]. -#[repr(C)] -pub(crate) struct SecondaryConfig { - pub(crate) unk_0: [u8; 0x14], - /// Required: 1. - pub(crate) unk_14: u32, - pub(crate) unk_18: [u8; 0x14], - /// Required: 1. - pub(crate) unk_2c: u32, - /// Required: 1. - pub(crate) unk_30: u32, - pub(crate) unk_34: [u8; 0xc], - /// Required: 1. - pub(crate) unk_40: u32, - pub(crate) unk_44: [u8; 8], - /// Required: 1. - pub(crate) unk_4c: u32, - /// Required: 0x186a (6250). - pub(crate) unk_50: u32, - pub(crate) unk_54: [u8; 0x2c], -} - -static_assert!(size_of::() == 0x80); - -// SAFETY: `SecondaryConfig` consists of integers and arrays of integers only. -unsafe impl Zeroable for SecondaryConfig {} -impl SecondaryConfig { - /// Initializes the secondary configuration. - pub(crate) fn init(&mut self) { - clear(self); - self.unk_14 = 1; - self.unk_2c = 1; - self.unk_30 = 1; - self.unk_40 = 1; - self.unk_4c = 1; - self.unk_50 = 0x186a; - } -} /// Offsets in the private cluster. pub(crate) mod private { @@ -1751,295 +1532,332 @@ const TEMP_KP: u32 = 0x41a2_b852; /// single. const TEMP_KI: u32 = 0x4116_0419; +/// Nonzero 32-bit words of a firmware object the host builds from constants: (byte offset, value, repeat). +/// Every other word is zero. Generated from the previously field-by-field builders; the fwdump harness +/// (`neo-smooth-20261007/kernel/fwdump`) verifies the objects byte for byte. +type Words = &'static [(usize, u32, usize)]; + +fn fill_words(bytes: &mut [u8], words: Words) { + bytes.fill(0); + for &(offset, value, repeat) in words { + for index in 0..repeat { + let at = offset + 4 * index; + bytes[at..at + 4].copy_from_slice(&value.to_le_bytes()); + } + } +} + +const POWER_CONFIG_HEAD_WORDS: Words = &[ + (0x4, 0x5dc00, 2), + (0x10, 4, 1), + (0x14, 0x3f800000, 1), + (0x2c, 1, 2), + (0x40, 0x64, 1), + (0x44, 1, 1), + (0x48, 0x3e8, 1), + (0x50, 0x64, 1), + (0x60, 4, 1), + (0x64, 0x3f800000, 1), + (0x6c, 1, 2), + (0x80, 0x64, 1), + (0x84, 1, 1), + (0x88, 0x3e8, 1), + (0x90, 0x64, 1), + (0xc0, 0x3f828f5c, 11), + (0x9b8, 0x271, 1), + (0x9c4, 0x3f7f2e9f, 1), + (0x9cc, 0x3b516154, 1), + (0x9d4, 0x3ca59586, 1), + (0x9e0, 0x46153800, 1), + (0x9e4, 0x40a90fdb, 1), + (0x9ec, 0xbdd67344, 1), + (0x9f0, 0x28, 1), + (0x9f4, 0x3e8, 2), + (0xa04, 0x254e, 1), + (0xa38, 0x3e8, 1), + (0xa70, 4, 1), + (0xa84, 0x3f800000, 1), + (0xa8c, 0x40cccccd, 1), + (0xa98, 0x47800000, 1), + (0xa9c, 0x3f000000, 1), + (0xaa8, 0x28, 1), + (0xaac, 0x3e8, 1), + (0xabc, 0x254e, 1), + (0xac8, 0x10, 1), + (0xad0, 0x5dc00, 1), + (0xb28, 0x5c, 1), + (0xb30, 0x64, 1), + (0xb34, 0x22, 1), + (0xb38, 6, 1), + (0xb40, 6, 1), + (0xb44, 1, 1), + (0xb4c, 0x3f4ccccd, 1), + (0xb50, 0x3f7df3b6, 1), + (0xb54, 0x3e4ccccd, 1), + (0xb58, 0x3c03126f, 1), + (0xb5c, 0x3f69d4d8, 2), + (0xb68, 0x42be0000, 1), + (0xb6c, 0x4064f5c3, 2), + (0xb74, 0x41179436, 1), + (0xb78, 0x64, 1), + (0xb7c, 0x3e8, 1), + (0xb80, 0x64, 1), + (0xb8c, 0x5c, 1), + (0xbc0, 0x64, 1), + (0xbe0, 0x401, 1), + (0xbe4, 0x6400, 1), + (0xbec, 0x1f4000, 1), + (0xbf0, 0xc800, 1), + (0xbf4, 0xfa000, 1), + (0xbf8, 0xc800, 1), + (0xbfc, 0x7d000, 1), + (0xc00, 0xc800, 1), + (0xc04, 0x3e800, 1), + (0xc08, 0xc800, 1), + (0xc0c, 0x100, 1), + (0xc64, 1, 1), + (0xc90, 1, 1), + (0x1c94, 0xff000000, 1), + (0x1c98, 0xff7fff7f, 74), +]; + +const POWER_CONFIG_TAIL_WORDS: Words = &[ + (0x0, 0xff7fff7f, 181), + (0x2d4, 0x7fff7f, 1), + (0x4d8, 0x47800000, 1), + (0x4dc, 0x42200000, 1), + (0x4e0, 0x447a0000, 1), + (0x4e4, 0x5be, 1), + (0x4f8, 0x28, 1), + (0x500, 0x42c80000, 1), + (0x504, 0x3e8, 1), + (0x50c, 0x3f4ccccd, 1), + (0x510, 0x3e4ccccd, 1), + (0x550, TEMP_SENSOR_MASK as u32, 1), + (0x558, TEMP_RELEASE, 1), + (0x560, 4, 1), + (0x574, 0x3f800000, 1), + (0x57c, TEMP_KI, 1), + (0x588, 0x47800000, 1), + (0x58c, TEMP_KP, 1), + (0x598, 0x28, 1), + (0x59c, 0x3e8, 2), + (0x5ac, TEMP_TARGET, 1), + (0x5e0, 0x3e8, 1), + (0x7fc, 0x42000006, 1), + (0xa10, 0xb8520000, 1), + (0xa14, 0x3f7e, 1), + (0xa18, 0xd70a0000, 1), + (0xa1c, 0x3ba3, 1), + (0xa24, 0x43c8, 1), + (0xa30, 0x4780, 1), + (0xa34, 0x42c8, 1), + (0xa3c, 0x384bac8, 1), + (0xa40, 0x3e80000, 2), + (0xa4c, 0x80000000, 1), + (0xa50, 0x26ac453b, 1), + (0xa84, 0x3e80000, 1), + (0xaa0, 0xc80000, 1), + (0xb58, 0x3e8, 1), + (0xb6c, 1, 1), + (0xba0, 0x101, 1), + (0xba4, 0x400, 1), + (0xbbc, 0x23f00, 1), + (0xbc0, 0x100, 2), + (0xbdc, 0x24000, 1), + (0xbe0, 0x100, 2), + (0xbfc, 0x24100, 1), + (0xc48, 1, 1), + (0xc50, 0x254e, 3), + (0xc68, 0x3f800000, 1), + (0xc6c, 4, 1), + (0xc70, 0x10, 1), + (0xc74, 0x5dc00, 1), + (0xd44, 0x3c, 1), + (0xd50, 0x3f6eeeef, 1), + (0xd58, 0x3d888889, 1), + (0xd60, 0x3f000000, 1), + (0xd6c, 0x47800000, 1), + (0xd70, 0x40880000, 1), + (0xd7c, 0x28, 1), + (0xd80, 0x3e8, 2), + (0xd8c, 0x46153800, 1), + (0xd90, 0x254e, 1), + (0xd9c, 0xf0, 1), + (0xda4, 0x57e400, 1), + (0xdc4, 0x3e8, 1), + (0xe04, 1, 1), + (0xed4, 0x32, 1), + (0xed8, 1, 1), + (0xee0, 0x3f638e39, 1), + (0xee4, 0x3f2aaaab, 1), + (0xee8, 0x3de38e39, 1), + (0xeec, 0x3eaaaaab, 1), + (0xef0, 0xbf4ccccd, 2), + (0xefc, 0x47800000, 1), + (0xf00, 0xc0a00000, 2), + (0xf0c, 0x64, 1), + (0xf10, 0x3e8, 1), + (0xf14, 0x64, 1), + (0xf1c, 0x467a0000, 1), + (0xf20, 0x578, 1), + (0xf2c, 0x90, 1), + (0xf30, 0x30, 1), + (0xf34, 0x34bc00, 1), + (0xf3c, 0x119400, 1), + (0xf48, 0x47800000, 1), + (0xf70, 0x3e80, 1), + (0xf74, 2, 1), + (0xf78, 0x9c4, 1), + (0xf7c, 0x20d, 1), + (0xf80, 2, 1), + (0xf84, 4, 1), + (0xf88, 0x32, 1), + (0xfb8, 0x47800000, 1), + (0xfc8, 0x28, 1), + (0xfcc, 0x3e8, 1), +]; + +const POWER_STATE_WORDS: Words = &[ + (0x10, 0xf, 1), + (0x14, 0x3f000000, 1), + (0x18, 0x40880000, 1), + (0x1c, 0x28, 1), + (0x20, 1, 1), + (0xc8, TEMP_TARGET, 1), + (0xcc, TEMP_RELEASE, 1), + (0xd8, TEMP_KP, 1), + (0xdc, TEMP_KI, 1), + (0xf0, 0x26ac, 1), + (0xf4, 0xc8, 1), + (0x108, 0x42c80000, 1), + (0x10c, 0x43c80000, 1), + (0x110, 0xc8, 1), + (0x128, 1, 1), + (0x12c, 0x2616, 1), + (0x130, 0x3f000000, 1), + (0x134, 0x40cccccd, 1), + (0x14c, 1, 1), + (0x1c0, 1, 2), + (0x1c8, 4, 1), + (0x1cc, 1, 4), + (0x4e4, 1, 1), + (0x4e8, 0x1f4, 1), + (0x4ec, 6, 1), + (0x4f0, 0x30d4, 6), + (0x51c, 0x30d4, 5), + (0x530, 1, 1), + (0x58c, 6, 1), + (0x590, 0xdac, 1), + (0x594, 0x3e8, 1), + (0x598, 0xbb8, 1), + (0x59c, 0x64, 1), + (0x5cc, 1, 1), + (0x5d0, 4, 1), + (0x5dc, 0x1f40, 1), + (0x5e0, 0xc8, 1), + (0x5e4, 0xfa0, 1), + (0x5e8, 0xc8, 1), + (0x5ec, 0x7d0, 1), + (0x5f0, 0xc8, 1), + (0x5f4, 0x3e8, 1), + (0x5f8, 0xc8, 1), + (0x604, 0x41700000, 1), + (0x608, 0x40a00000, 1), + (0x60c, 0x20, 1), + (0x610, 0x3e8, 1), + (0x620, 0x384, 1), +]; + +const HW_DATA_AUX_WORDS: Words = &[ + (0xac, 6, 1), + (0xb8, 1, 1), + (0x174, 0xac000000, 1), + (0x178, 0xe800000d, 1), + (0x17c, 0xb8000003, 1), + (0x180, 0xe800000b, 1), + (0x184, 0xe8000003, 1), + (0x188, 0x64000003, 1), + (0x18c, 0x20000000, 1), + (0x1b4, 4, 1), + (0x1c0, 0x3f780000, 1), + (0x1c8, 0x3d000000, 1), + (0x1d0, 0x40a00000, 1), + (0x1dc, 0x47800000, 1), + (0x1e0, 0x41700000, 1), + (0x1ec, 0x384, 1), + (0x1f0, 0x3e8, 2), + (0x200, 0x3e8, 1), + (0x234, 0x3e8, 1), + (0x274, 0x64, 1), + (0x290, 0x82a, 1), + (0x298, 0x7d, 1), + (0x29c, 1, 1), + (0xee0, 0x18ae, 1), + (0xee8, 1, 1), + (0x2eec, 0x41c8cccd, 2), + (0x35e4, 1, 1), + (0x35e8, 0x1f4, 1), + (0x35ec, 6, 1), + (0x35f0, 0x30d4, 6), + (0x3608, 0xfa0, 1), + (0x361c, 0x30d4, 5), + (0x3630, 1, 1), + (0x37a8, 1, 1), + (0x37b4, 0x424ccccd, 1), + (0x37f4, 0x44068000, 1), + (0x3834, 0x41c9999a, 1), + (0x38e0, 0x186a, 1), +]; + +const SECONDARY_CONFIG_WORDS: Words = &[ + (0x14, 1, 1), + (0x2c, 1, 2), + (0x40, 1, 1), + (0x4c, 1, 1), + (0x50, 0x186a, 1), +]; + +/// Power state the primary status B carries for the secondary instance, +/// named by [`SecondaryRoot::power_state_va`]. +#[repr(C)] +pub(crate) struct PowerState(pub(crate) [u8; 0x6c0]); + +// SAFETY: `PowerState` is a byte array. +unsafe impl Zeroable for PowerState {} + +impl PowerState { + fn init(&mut self) { + fill_words(&mut self.0, POWER_STATE_WORDS); + } +} + +/// Configuration of the secondary instance, named by +/// [`SecondaryRoot::config_va`]. +#[repr(C)] +pub(crate) struct SecondaryConfig(pub(crate) [u8; 0x80]); + +// SAFETY: `SecondaryConfig` is a byte array. +unsafe impl Zeroable for SecondaryConfig {} + +impl SecondaryConfig { + pub(crate) fn init(&mut self) { + fill_words(&mut self.0, SECONDARY_CONFIG_WORDS); + } +} + /// Leading part of the [`PowerConfig`]. The host writes it twice: in the /// power configuration and at [`bundle::POWER_CONFIG_COPY`], where the copy /// ends with the views it lies in. -#[repr(C, packed)] -pub(crate) struct PowerConfigHead { - pub(crate) unk_0: [u8; 4], - /// Required: 0x5_dc00 (384000). - pub(crate) unk_4: u32, - /// Required: 0x5_dc00 (384000). - pub(crate) unk_8: u32, - pub(crate) unk_c: [u8; 4], - /// Required: 4. - pub(crate) unk_10: u32, - /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). - pub(crate) unk_14: u32, - pub(crate) unk_18: [u8; 0x14], - /// Required: 1. - pub(crate) unk_2c: u32, - /// Required: 1. - pub(crate) unk_30: u32, - pub(crate) unk_34: [u8; 0xc], - /// Required: 0x64 (100). - pub(crate) unk_40: u32, - /// Required: 1. - pub(crate) unk_44: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_48: u32, - pub(crate) unk_4c: [u8; 4], - /// Required: 0x64 (100). - pub(crate) unk_50: u32, - pub(crate) unk_54: [u8; 0xc], - /// Required: 4. - pub(crate) unk_60: u32, - /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). - pub(crate) unk_64: u32, - pub(crate) unk_68: [u8; 4], - /// Required: 1. - pub(crate) unk_6c: u32, - /// Required: 1. - pub(crate) unk_70: u32, - pub(crate) unk_74: [u8; 0xc], - /// Required: 0x64 (100). - pub(crate) unk_80: u32, - /// Required: 1. - pub(crate) unk_84: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_88: u32, - pub(crate) unk_8c: [u8; 4], - /// Required: 0x64 (100). - pub(crate) unk_90: u32, - pub(crate) unk_94: [u8; 0x2c], - /// Required: 0x3f82_8f5c (1.02 as an IEEE-754 single) in every entry. - pub(crate) unk_c0: [u32; 11], - pub(crate) unk_ec: [u8; 0x8cc], - /// Required: 0x271 (625). - pub(crate) unk_9b8: u32, - pub(crate) unk_9bc: [u8; 8], - /// Required: 0x3f7f_2e9f. - pub(crate) unk_9c4: u32, - pub(crate) unk_9c8: [u8; 4], - /// Required: 0x3b51_6154. - pub(crate) unk_9cc: u32, - pub(crate) unk_9d0: [u8; 4], - /// Required: 0x3ca5_9586 (0.020213 as an IEEE-754 single). - pub(crate) unk_9d4: u32, - pub(crate) unk_9d8: [u8; 8], - /// Required: 0x4615_3800 (9550.0 as an IEEE-754 single). - pub(crate) unk_9e0: u32, - /// Required: 0x40a9_0fdb. - pub(crate) unk_9e4: u32, - pub(crate) unk_9e8: [u8; 4], - /// Required: 0xbdd6_7344. - pub(crate) unk_9ec: u32, - /// Required: 0x28 (40). - pub(crate) unk_9f0: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_9f4: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_9f8: u32, - pub(crate) unk_9fc: [u8; 8], - /// Required: 0x254e (9550). - pub(crate) unk_a04: u32, - pub(crate) unk_a08: [u8; 0x30], - /// Required: 0x3e8 (1000). - pub(crate) unk_a38: u32, - pub(crate) unk_a3c: [u8; 0x34], - /// Required: 4. - pub(crate) unk_a70: u32, - pub(crate) unk_a74: [u8; 0x10], - /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). - pub(crate) unk_a84: u32, - pub(crate) unk_a88: [u8; 4], - /// Required: 0x40cc_cccd (6.4 as an IEEE-754 single). - pub(crate) unk_a8c: u32, - pub(crate) unk_a90: [u8; 8], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_a98: u32, - /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). - pub(crate) unk_a9c: u32, - pub(crate) unk_aa0: [u8; 8], - /// Required: 0x28 (40). - pub(crate) unk_aa8: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_aac: u32, - pub(crate) unk_ab0: [u8; 0xc], - /// Required: 0x254e (9550). - pub(crate) unk_abc: u32, - pub(crate) unk_ac0: [u8; 8], - /// Required: 0x10 (16). - pub(crate) unk_ac8: u32, - pub(crate) unk_acc: [u8; 4], - /// Required: 0x5_dc00 (384000). - pub(crate) unk_ad0: u32, - pub(crate) unk_ad4: [u8; 0x54], - /// Required: 0x5c (92). - pub(crate) unk_b28: u32, - pub(crate) unk_b2c: [u8; 4], - /// Required: 0x64 (100). - pub(crate) unk_b30: u32, - /// Required: 0x22 (34). - pub(crate) unk_b34: u32, - /// Required: 6. - pub(crate) unk_b38: u32, - pub(crate) unk_b3c: [u8; 4], - /// Required: 6. - pub(crate) unk_b40: u32, - /// Required: 1. - pub(crate) unk_b44: u32, - pub(crate) unk_b48: [u8; 4], - /// Required: 0x3f4c_cccd (0.8 as an IEEE-754 single). - pub(crate) unk_b4c: u32, - /// Required: 0x3f7d_f3b6 (0.992 as an IEEE-754 single). - pub(crate) unk_b50: u32, - /// Required: 0x3e4c_cccd (0.2 as an IEEE-754 single). - pub(crate) unk_b54: u32, - /// Required: 0x3c03_126f (0.008 as an IEEE-754 single). - pub(crate) unk_b58: u32, - /// Required: 0x3f69_d4d8 (0.913404 as an IEEE-754 single). - pub(crate) unk_b5c: u32, - /// Required: 0x3f69_d4d8 (0.913404 as an IEEE-754 single). - pub(crate) unk_b60: u32, - pub(crate) unk_b64: [u8; 4], - /// Required: 0x42be_0000 (95.0 as an IEEE-754 single). - pub(crate) unk_b68: u32, - /// Required: 0x4064_f5c3 (3.5775 as an IEEE-754 single). - pub(crate) unk_b6c: u32, - /// Required: 0x4064_f5c3 (3.5775 as an IEEE-754 single). - pub(crate) unk_b70: u32, - /// Required: 0x4117_9436. - pub(crate) unk_b74: u32, - /// Required: 0x64 (100). - pub(crate) unk_b78: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_b7c: u32, - /// Required: 0x64 (100). - pub(crate) unk_b80: u32, - pub(crate) unk_b84: [u8; 8], - /// Required: 0x5c (92). - pub(crate) unk_b8c: u32, - pub(crate) unk_b90: [u8; 0x30], - /// Required: 0x64 (100). - pub(crate) unk_bc0: u32, - pub(crate) unk_bc4: [u8; 0x1c], - /// Required: 1. - pub(crate) unk_be0: u8, - /// Required: 4. - pub(crate) unk_be1: u32, - /// Required: 0x64 (100). - pub(crate) unk_be5: u32, - pub(crate) unk_be9: [u8; 4], - /// Required: 0x1f40 (8000). - pub(crate) unk_bed: u32, - /// Required: 0xc8 (200). - pub(crate) unk_bf1: u32, - /// Required: 0xfa0 (4000). - pub(crate) unk_bf5: u32, - /// Required: 0xc8 (200). - pub(crate) unk_bf9: u32, - /// Required: 0x7d0 (2000). - pub(crate) unk_bfd: u32, - /// Required: 0xc8 (200). - pub(crate) unk_c01: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_c05: u32, - /// Required: 0xc8 (200). - pub(crate) unk_c09: u32, - /// Required: 1. - pub(crate) unk_c0d: u32, - pub(crate) unk_c11: [u8; 0x53], - /// Required: 1. - pub(crate) unk_c64: u32, - pub(crate) unk_c68: [u8; 0x28], - /// Required: 1. - pub(crate) unk_c90: u32, - pub(crate) unk_c94: [u8; 0x1003], - /// Required: 0x7fff in every entry; the table continues in - /// [`PowerConfigTail`]. - pub(crate) unk_1c97: [u16; 148], - /// Required: 0xff (low byte of the next table entry). - pub(crate) unk_1dbf: u8, -} - -static_assert!(bundle::POWER_CONFIG_COPY + size_of::() == 0xf000); +#[repr(C)] +pub(crate) struct PowerConfigHead(pub(crate) [u8; 0x1dc0]); -// SAFETY: `PowerConfigHead` consists of integers and arrays of integers only. +// SAFETY: `PowerConfigHead` is a byte array. unsafe impl Zeroable for PowerConfigHead {} impl PowerConfigHead { - /// Initializes the leading part of the power configuration. pub(crate) fn init(&mut self) { - clear(self); - self.unk_4 = 0x5_dc00; - self.unk_8 = 0x5_dc00; - self.unk_10 = 4; - self.unk_14 = 0x3f80_0000; - self.unk_2c = 1; - self.unk_30 = 1; - self.unk_40 = 0x64; - self.unk_44 = 1; - self.unk_48 = 0x3e8; - self.unk_50 = 0x64; - self.unk_60 = 4; - self.unk_64 = 0x3f80_0000; - self.unk_6c = 1; - self.unk_70 = 1; - self.unk_80 = 0x64; - self.unk_84 = 1; - self.unk_88 = 0x3e8; - self.unk_90 = 0x64; - self.unk_c0 = [0x3f82_8f5c; 11]; - self.unk_9b8 = 0x271; - self.unk_9c4 = 0x3f7f_2e9f; - self.unk_9cc = 0x3b51_6154; - self.unk_9d4 = 0x3ca5_9586; - self.unk_9e0 = 0x4615_3800; - self.unk_9e4 = 0x40a9_0fdb; - self.unk_9ec = 0xbdd6_7344; - self.unk_9f0 = 0x28; - self.unk_9f4 = 0x3e8; - self.unk_9f8 = 0x3e8; - self.unk_a04 = 0x254e; - self.unk_a38 = 0x3e8; - self.unk_a70 = 4; - self.unk_a84 = 0x3f80_0000; - self.unk_a8c = 0x40cc_cccd; - self.unk_a98 = 0x4780_0000; - self.unk_a9c = 0x3f00_0000; - self.unk_aa8 = 0x28; - self.unk_aac = 0x3e8; - self.unk_abc = 0x254e; - self.unk_ac8 = 0x10; - self.unk_ad0 = 0x5_dc00; - self.unk_b28 = 0x5c; - self.unk_b30 = 0x64; - self.unk_b34 = 0x22; - self.unk_b38 = 6; - self.unk_b40 = 6; - self.unk_b44 = 1; - self.unk_b4c = 0x3f4c_cccd; - self.unk_b50 = 0x3f7d_f3b6; - self.unk_b54 = 0x3e4c_cccd; - self.unk_b58 = 0x3c03_126f; - self.unk_b5c = 0x3f69_d4d8; - self.unk_b60 = 0x3f69_d4d8; - self.unk_b68 = 0x42be_0000; - self.unk_b6c = 0x4064_f5c3; - self.unk_b70 = 0x4064_f5c3; - self.unk_b74 = 0x4117_9436; - self.unk_b78 = 0x64; - self.unk_b7c = 0x3e8; - self.unk_b80 = 0x64; - self.unk_b8c = 0x5c; - self.unk_bc0 = 0x64; - self.unk_be0 = 1; - self.unk_be1 = 4; - self.unk_be5 = 0x64; - self.unk_bed = 0x1f40; - self.unk_bf1 = 0xc8; - self.unk_bf5 = 0xfa0; - self.unk_bf9 = 0xc8; - self.unk_bfd = 0x7d0; - self.unk_c01 = 0xc8; - self.unk_c05 = 0x3e8; - self.unk_c09 = 0xc8; - self.unk_c0d = 1; - self.unk_c64 = 1; - self.unk_c90 = 1; - self.unk_1c97 = [0x7fff; 148]; - self.unk_1dbf = 0xff; + fill_words(&mut self.0, POWER_CONFIG_HEAD_WORDS); } } @@ -2047,381 +1865,37 @@ impl PowerConfigHead { /// whole power configuration. It carries the die temperature limiter: target /// and release temperatures in 0.01 °C, proportional and integral gains as /// IEEE-754 singles. -#[repr(C, packed)] -pub(crate) struct PowerConfigTail { - /// Required: 0x7f (high byte of the table entry started in the head). - pub(crate) unk_1dc0: u8, - /// Required: 0x7fff in every entry. - pub(crate) unk_1dc1: [u16; 363], - pub(crate) unk_2097: [u8; 0x201], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_2298: u32, - /// Required: 0x4220_0000 (40.0 as an IEEE-754 single). - pub(crate) unk_229c: u32, - /// Required: 0x447a_0000 (1000.0 as an IEEE-754 single). - pub(crate) unk_22a0: u32, - /// Required: 0x5be (1470). - pub(crate) unk_22a4: u32, - pub(crate) unk_22a8: [u8; 0x10], - /// Required: 0x28 (40). - pub(crate) unk_22b8: u32, - pub(crate) unk_22bc: [u8; 4], - /// Required: 0x42c8_0000 (100.0 as an IEEE-754 single). - pub(crate) unk_22c0: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_22c4: u32, - pub(crate) unk_22c8: [u8; 4], - /// Required: 0x3f4c_cccd (0.8 as an IEEE-754 single). - pub(crate) unk_22cc: u32, - /// Required: 0x3e4c_cccd (0.2 as an IEEE-754 single). - pub(crate) unk_22d0: u32, - pub(crate) unk_22d4: [u8; 0x3c], - /// Die temperature sensors the limiter reads ([`TEMP_SENSOR_MASK`]). - pub(crate) temp_sensor_mask: u64, - /// Release temperature of the limiter ([`TEMP_RELEASE`]). - pub(crate) temp_release: u32, - pub(crate) unk_231c: [u8; 4], - /// Required: 4. - pub(crate) unk_2320: u32, - pub(crate) unk_2324: [u8; 0x10], - /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). - pub(crate) unk_2334: u32, - pub(crate) unk_2338: [u8; 4], - /// Integral gain of the limiter ([`TEMP_KI`]). - pub(crate) temp_ki: u32, - pub(crate) unk_2340: [u8; 8], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_2348: u32, - /// Proportional gain of the limiter ([`TEMP_KP`]). - pub(crate) temp_kp: u32, - pub(crate) unk_2350: [u8; 8], - /// Required: 0x28 (40). - pub(crate) unk_2358: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_235c: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_2360: u32, - /// Margin below the target at which the limiter acts, in 0.01 °C. - pub(crate) temp_margin: u32, - pub(crate) unk_2368: [u8; 4], - /// Target temperature of the limiter ([`TEMP_TARGET`]). - pub(crate) temp_target: u32, - pub(crate) unk_2370: [u8; 0x30], - /// Required: 0x3e8 (1000). - pub(crate) unk_23a0: u32, - pub(crate) unk_23a4: [u8; 0x218], - /// Required: 0x4200_0006. - pub(crate) unk_25bc: u32, - pub(crate) unk_25c0: [u8; 0x212], - /// Required: 0x3f7e_b852 (0.995 as an IEEE-754 single). - pub(crate) unk_27d2: u32, - pub(crate) unk_27d6: [u8; 4], - /// Required: 0x3ba3_d70a (0.005 as an IEEE-754 single). - pub(crate) unk_27da: u32, - pub(crate) unk_27de: [u8; 4], - /// Required: 0x43c8_0000 (400.0 as an IEEE-754 single). - pub(crate) unk_27e2: u32, - pub(crate) unk_27e6: [u8; 8], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_27ee: u32, - /// Required: 0x42c8_0000 (100.0 as an IEEE-754 single). - pub(crate) unk_27f2: u32, - pub(crate) unk_27f6: [u8; 4], - /// Required: 0xbac8_0000. - pub(crate) unk_27fa: u32, - /// Required: 0x384 (900). - pub(crate) unk_27fe: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_2802: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_2806: u32, - pub(crate) unk_280a: [u8; 4], - /// Required: 0x453b_8000 (3000.0 as an IEEE-754 single). - pub(crate) unk_280e: u32, - /// Required: 0x26ac (9900). - pub(crate) unk_2812: u32, - pub(crate) unk_2816: [u8; 0x30], - /// Required: 0x3e8 (1000). - pub(crate) unk_2846: u32, - pub(crate) unk_284a: [u8; 0x18], - /// Required: 0xc8 (200). - pub(crate) unk_2862: u32, - pub(crate) unk_2866: [u8; 0xb2], - /// Required: 0x3e8 (1000). - pub(crate) unk_2918: u32, - pub(crate) unk_291c: [u8; 0x10], - /// Required: 1. - pub(crate) unk_292c: u32, - pub(crate) unk_2930: [u8; 0x30], - /// Required: 1. - pub(crate) unk_2960: u8, - /// Required: 1. - pub(crate) unk_2961: u32, - /// Required: 4. - pub(crate) unk_2965: u32, - pub(crate) unk_2969: [u8; 0x14], - /// Required: 0x23f (575). - pub(crate) unk_297d: u32, - /// Required: 1. - pub(crate) unk_2981: u32, - /// Required: 1. - pub(crate) unk_2985: u32, - pub(crate) unk_2989: [u8; 0x14], - /// Required: 0x240 (576). - pub(crate) unk_299d: u32, - /// Required: 1. - pub(crate) unk_29a1: u32, - /// Required: 1. - pub(crate) unk_29a5: u32, - pub(crate) unk_29a9: [u8; 0x14], - /// Required: 0x241 (577). - pub(crate) unk_29bd: u32, - pub(crate) unk_29c1: [u8; 0x47], - /// Required: 1. - pub(crate) unk_2a08: u32, - pub(crate) unk_2a0c: [u8; 4], - /// Required: 0x254e (9550) in every entry. - pub(crate) unk_2a10: [u32; 3], - pub(crate) unk_2a1c: [u8; 0xc], - /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). - pub(crate) unk_2a28: u32, - /// Required: 4. - pub(crate) unk_2a2c: u32, - /// Required: 0x10 (16). - pub(crate) unk_2a30: u32, - /// Required: 0x5_dc00 (384000). - pub(crate) unk_2a34: u32, - pub(crate) unk_2a38: [u8; 0xcc], - /// Required: 0x3c (60). - pub(crate) unk_2b04: u32, - pub(crate) unk_2b08: [u8; 8], - /// Required: 0x3f6e_eeef. - pub(crate) unk_2b10: u32, - pub(crate) unk_2b14: [u8; 4], - /// Required: 0x3d88_8889. - pub(crate) unk_2b18: u32, - pub(crate) unk_2b1c: [u8; 4], - /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). - pub(crate) unk_2b20: u32, - pub(crate) unk_2b24: [u8; 8], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_2b2c: u32, - /// Required: 0x4088_0000 (4.25 as an IEEE-754 single). - pub(crate) unk_2b30: u32, - pub(crate) unk_2b34: [u8; 8], - /// Required: 0x28 (40). - pub(crate) unk_2b3c: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_2b40: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_2b44: u32, - pub(crate) unk_2b48: [u8; 4], - /// Required: 0x4615_3800 (9550.0 as an IEEE-754 single). - pub(crate) unk_2b4c: u32, - /// Required: 0x254e (9550). - pub(crate) unk_2b50: u32, - pub(crate) unk_2b54: [u8; 8], - /// Required: 0xf0 (240). - pub(crate) unk_2b5c: u32, - pub(crate) unk_2b60: [u8; 4], - /// Required: 0x57_e400. - pub(crate) unk_2b64: u32, - pub(crate) unk_2b68: [u8; 0x1c], - /// Required: 0x3e8 (1000). - pub(crate) unk_2b84: u32, - pub(crate) unk_2b88: [u8; 0x3c], - /// Required: 1. - pub(crate) unk_2bc4: u32, - pub(crate) unk_2bc8: [u8; 0xcc], - /// Required: 0x32 (50). - pub(crate) unk_2c94: u32, - /// Required: 1. - pub(crate) unk_2c98: u32, - pub(crate) unk_2c9c: [u8; 4], - /// Required: 0x3f63_8e39. - pub(crate) unk_2ca0: u32, - /// Required: 0x3f2a_aaab. - pub(crate) unk_2ca4: u32, - /// Required: 0x3de3_8e39. - pub(crate) unk_2ca8: u32, - /// Required: 0x3eaa_aaab. - pub(crate) unk_2cac: u32, - /// Required: 0xbf4c_cccd (-0.8 as an IEEE-754 single). - pub(crate) unk_2cb0: u32, - /// Required: 0xbf4c_cccd (-0.8 as an IEEE-754 single). - pub(crate) unk_2cb4: u32, - pub(crate) unk_2cb8: [u8; 4], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_2cbc: u32, - /// Required: 0xc0a0_0000 (-5.0 as an IEEE-754 single). - pub(crate) unk_2cc0: u32, - /// Required: 0xc0a0_0000 (-5.0 as an IEEE-754 single). - pub(crate) unk_2cc4: u32, - pub(crate) unk_2cc8: [u8; 4], - /// Required: 0x64 (100). - pub(crate) unk_2ccc: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_2cd0: u32, - /// Required: 0x64 (100). - pub(crate) unk_2cd4: u32, - pub(crate) unk_2cd8: [u8; 4], - /// Required: 0x467a_0000 (16000.0 as an IEEE-754 single). - pub(crate) unk_2cdc: u32, - /// Required: 0x578 (1400). - pub(crate) unk_2ce0: u32, - pub(crate) unk_2ce4: [u8; 8], - /// Required: 0x90 (144). - pub(crate) unk_2cec: u32, - /// Required: 0x30 (48). - pub(crate) unk_2cf0: u32, - /// Required: 0x34_bc00. - pub(crate) unk_2cf4: u32, - pub(crate) unk_2cf8: [u8; 4], - /// Required: 0x11_9400. - pub(crate) unk_2cfc: u32, - pub(crate) unk_2d00: [u8; 8], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_2d08: u32, - pub(crate) unk_2d0c: [u8; 0x24], - /// Required: 0x3e80 (16000). - pub(crate) unk_2d30: u32, - /// Required: 2. - pub(crate) unk_2d34: u32, - /// Required: 0x9c4 (2500). - pub(crate) unk_2d38: u32, - /// Required: 0x20d (525). - pub(crate) unk_2d3c: u32, - /// Required: 2. - pub(crate) unk_2d40: u32, - /// Required: 4. - pub(crate) unk_2d44: u32, - /// Required: 0x32 (50). - pub(crate) unk_2d48: u32, - pub(crate) unk_2d4c: [u8; 0x2c], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_2d78: u32, - pub(crate) unk_2d7c: [u8; 0xc], - /// Required: 0x28 (40). - pub(crate) unk_2d88: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_2d8c: u32, - pub(crate) unk_2d90: [u8; 0x30], -} - -static_assert!(core::mem::offset_of!(PowerConfigTail, temp_sensor_mask) == 0x2310 - 0x1dc0); -static_assert!(core::mem::offset_of!(PowerConfigTail, temp_target) == 0x236c - 0x1dc0); -static_assert!(size_of::() == 0x1000); +#[repr(C)] +pub(crate) struct PowerConfigTail(pub(crate) [u8; 0x1000]); -// SAFETY: `PowerConfigTail` consists of integers and arrays of integers only. +// SAFETY: `PowerConfigTail` is a byte array. unsafe impl Zeroable for PowerConfigTail {} impl PowerConfigTail { fn init(&mut self) { - clear(self); - self.unk_1dc0 = 0x7f; - self.unk_1dc1 = [0x7fff; 363]; - self.unk_2298 = 0x4780_0000; - self.unk_229c = 0x4220_0000; - self.unk_22a0 = 0x447a_0000; - self.unk_22a4 = 0x5be; - self.unk_22b8 = 0x28; - self.unk_22c0 = 0x42c8_0000; - self.unk_22c4 = 0x3e8; - self.unk_22cc = 0x3f4c_cccd; - self.unk_22d0 = 0x3e4c_cccd; - self.temp_sensor_mask = TEMP_SENSOR_MASK; - self.temp_release = TEMP_RELEASE; - self.unk_2320 = 4; - self.unk_2334 = 0x3f80_0000; - self.temp_ki = TEMP_KI; - self.unk_2348 = 0x4780_0000; - self.temp_kp = TEMP_KP; - self.unk_2358 = 0x28; - self.unk_235c = 0x3e8; - self.unk_2360 = 0x3e8; - self.temp_target = TEMP_TARGET; - self.unk_23a0 = 0x3e8; - self.unk_25bc = 0x4200_0006; - self.unk_27d2 = 0x3f7e_b852; - self.unk_27da = 0x3ba3_d70a; - self.unk_27e2 = 0x43c8_0000; - self.unk_27ee = 0x4780_0000; - self.unk_27f2 = 0x42c8_0000; - self.unk_27fa = 0xbac8_0000; - self.unk_27fe = 0x384; - self.unk_2802 = 0x3e8; - self.unk_2806 = 0x3e8; - self.unk_280e = 0x453b_8000; - self.unk_2812 = 0x26ac; - self.unk_2846 = 0x3e8; - self.unk_2862 = 0xc8; - self.unk_2918 = 0x3e8; - self.unk_292c = 1; - self.unk_2960 = 1; - self.unk_2961 = 1; - self.unk_2965 = 4; - self.unk_297d = 0x23f; - self.unk_2981 = 1; - self.unk_2985 = 1; - self.unk_299d = 0x240; - self.unk_29a1 = 1; - self.unk_29a5 = 1; - self.unk_29bd = 0x241; - self.unk_2a08 = 1; - self.unk_2a10 = [0x254e; 3]; - self.unk_2a28 = 0x3f80_0000; - self.unk_2a2c = 4; - self.unk_2a30 = 0x10; - self.unk_2a34 = 0x5_dc00; - self.unk_2b04 = 0x3c; - self.unk_2b10 = 0x3f6e_eeef; - self.unk_2b18 = 0x3d88_8889; - self.unk_2b20 = 0x3f00_0000; - self.unk_2b2c = 0x4780_0000; - self.unk_2b30 = 0x4088_0000; - self.unk_2b3c = 0x28; - self.unk_2b40 = 0x3e8; - self.unk_2b44 = 0x3e8; - self.unk_2b4c = 0x4615_3800; - self.unk_2b50 = 0x254e; - self.unk_2b5c = 0xf0; - self.unk_2b64 = 0x57_e400; - self.unk_2b84 = 0x3e8; - self.unk_2bc4 = 1; - self.unk_2c94 = 0x32; - self.unk_2c98 = 1; - self.unk_2ca0 = 0x3f63_8e39; - self.unk_2ca4 = 0x3f2a_aaab; - self.unk_2ca8 = 0x3de3_8e39; - self.unk_2cac = 0x3eaa_aaab; - self.unk_2cb0 = 0xbf4c_cccd; - self.unk_2cb4 = 0xbf4c_cccd; - self.unk_2cbc = 0x4780_0000; - self.unk_2cc0 = 0xc0a0_0000; - self.unk_2cc4 = 0xc0a0_0000; - self.unk_2ccc = 0x64; - self.unk_2cd0 = 0x3e8; - self.unk_2cd4 = 0x64; - self.unk_2cdc = 0x467a_0000; - self.unk_2ce0 = 0x578; - self.unk_2cec = 0x90; - self.unk_2cf0 = 0x30; - self.unk_2cf4 = 0x34_bc00; - self.unk_2cfc = 0x11_9400; - self.unk_2d08 = 0x4780_0000; - self.unk_2d30 = 0x3e80; - self.unk_2d34 = 2; - self.unk_2d38 = 0x9c4; - self.unk_2d3c = 0x20d; - self.unk_2d40 = 2; - self.unk_2d44 = 4; - self.unk_2d48 = 0x32; - self.unk_2d78 = 0x4780_0000; - self.unk_2d88 = 0x28; - self.unk_2d8c = 0x3e8; + fill_words(&mut self.0, POWER_CONFIG_TAIL_WORDS); } } +/// Auxiliary hardware data the firmware reads at [`bundle::AUX`], up to the +/// start of view 3. No pointer names it. +#[repr(C)] +pub(crate) struct HwDataAux(pub(crate) [u8; 0x3c80]); + +// SAFETY: `HwDataAux` is a byte array. +unsafe impl Zeroable for HwDataAux {} + +impl HwDataAux { + pub(crate) fn init(&mut self, secondary_status_va: u64) { + fill_words(&mut self.0, HW_DATA_AUX_WORDS); + self.0[0xed8..0xee0].copy_from_slice(&secondary_status_va.to_le_bytes()); + } +} + + + + + /// Power configuration, in view 2 of the bundle. The secondary instance /// finds it through [`MainConfig::power_config_va`]. #[repr(C, packed)] @@ -2446,174 +1920,7 @@ impl PowerConfig { } } -/// Auxiliary hardware data the firmware reads at [`bundle::AUX`], up to the -/// start of view 3. No pointer names it. -#[repr(C, packed)] -pub(crate) struct HwDataAux { - pub(crate) unk_0: [u8; 0xac], - /// Required: 6. - pub(crate) unk_ac: u32, - pub(crate) unk_b0: [u8; 8], - /// Required: 1. - pub(crate) unk_b8: u32, - pub(crate) unk_bc: [u8; 0xbb], - /// Required: 0xdac (3500). - pub(crate) unk_177: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_17b: u32, - /// Required: 0xbb8 (3000). - pub(crate) unk_17f: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_183: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_187: u32, - /// Required: 0x64 (100). - pub(crate) unk_18b: u32, - /// Required: 0x20 (32). - pub(crate) unk_18f: u32, - pub(crate) unk_193: [u8; 0x21], - /// Required: 4. - pub(crate) unk_1b4: u32, - pub(crate) unk_1b8: [u8; 8], - /// Required: 0x3f78_0000 (0.96875 as an IEEE-754 single). - pub(crate) unk_1c0: u32, - pub(crate) unk_1c4: [u8; 4], - /// Required: 0x3d00_0000 (0.03125 as an IEEE-754 single). - pub(crate) unk_1c8: u32, - pub(crate) unk_1cc: [u8; 4], - /// Required: 0x40a0_0000 (5.0 as an IEEE-754 single). - pub(crate) unk_1d0: u32, - pub(crate) unk_1d4: [u8; 8], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_1dc: u32, - /// Required: 0x4170_0000 (15.0 as an IEEE-754 single). - pub(crate) unk_1e0: u32, - pub(crate) unk_1e4: [u8; 8], - /// Required: 0x384 (900). - pub(crate) unk_1ec: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_1f0: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_1f4: u32, - pub(crate) unk_1f8: [u8; 8], - /// Required: 0x3e8 (1000). - pub(crate) unk_200: u32, - pub(crate) unk_204: [u8; 0x30], - /// Required: 0x3e8 (1000). - pub(crate) unk_234: u32, - pub(crate) unk_238: [u8; 0x3c], - /// Required: 0x64 (100). - pub(crate) unk_274: u32, - pub(crate) unk_278: [u8; 0x18], - /// Required: 0x82a (2090). - pub(crate) unk_290: u32, - pub(crate) unk_294: [u8; 4], - /// Required: 0x7d (125). - pub(crate) unk_298: u32, - /// Required: 1. - pub(crate) unk_29c: u32, - pub(crate) unk_2a0: [u8; 0xc38], - /// GPU address of the secondary state, [`private::SECONDARY_STATUS`] bytes - /// into the secondary state grid. - pub(crate) secondary_status_va: u64, - /// Required: 0x18ae (6318). - pub(crate) unk_ee0: u32, - pub(crate) unk_ee4: [u8; 4], - /// Required: 1. - pub(crate) unk_ee8: u32, - pub(crate) unk_eec: [u8; 0x2000], - /// Required: 0x41c8_cccd (25.1 as an IEEE-754 single). - pub(crate) unk_2eec: u32, - /// Required: 0x41c8_cccd (25.1 as an IEEE-754 single). - pub(crate) unk_2ef0: u32, - pub(crate) unk_2ef4: [u8; 0x6f0], - /// Required: 1. - pub(crate) unk_35e4: u32, - /// Required: 0x1f4 (500). - pub(crate) unk_35e8: u32, - /// Required: 6. - pub(crate) unk_35ec: u32, - /// Required: 0x30d4 (12500) in every entry. - pub(crate) unk_35f0: [u32; 6], - /// Required: 0xfa0 (4000). - pub(crate) unk_3608: u32, - pub(crate) unk_360c: [u8; 0x10], - /// Required: 0x30d4 (12500) in every entry. - pub(crate) unk_361c: [u32; 5], - /// Required: 1. - pub(crate) unk_3630: u32, - pub(crate) unk_3634: [u8; 0x174], - /// Required: 1. - pub(crate) unk_37a8: u32, - pub(crate) unk_37ac: [u8; 8], - /// Required: 0x424c_cccd (51.2 as an IEEE-754 single). - pub(crate) unk_37b4: u32, - pub(crate) unk_37b8: [u8; 0x3c], - /// Required: 0x4406_8000 (538.0 as an IEEE-754 single). - pub(crate) unk_37f4: u32, - pub(crate) unk_37f8: [u8; 0x3c], - /// Required: 0x41c9_999a (25.2 as an IEEE-754 single). - pub(crate) unk_3834: u32, - pub(crate) unk_3838: [u8; 0xa8], - /// Required: 0x186a (6250). - pub(crate) unk_38e0: u32, - pub(crate) unk_38e4: [u8; 0x39c], -} - -static_assert!(core::mem::offset_of!(HwDataAux, secondary_status_va) == 0xed8); -static_assert!(bundle::AUX + size_of::() == bundle::VIEWS[3]); - -// SAFETY: `HwDataAux` consists of integers and arrays of integers only. -unsafe impl Zeroable for HwDataAux {} -impl HwDataAux { - /// Initializes the auxiliary hardware data. `secondary_status_va` is - /// the secondary state grid plus [`private::SECONDARY_STATUS`]. - pub(crate) fn init(&mut self, secondary_status_va: u64) { - clear(self); - self.unk_ac = 6; - self.unk_b8 = 1; - self.unk_177 = 0xdac; - self.unk_17b = 0x3e8; - self.unk_17f = 0xbb8; - self.unk_183 = 0x3e8; - self.unk_187 = 0x3e8; - self.unk_18b = 0x64; - self.unk_18f = 0x20; - self.unk_1b4 = 4; - self.unk_1c0 = 0x3f78_0000; - self.unk_1c8 = 0x3d00_0000; - self.unk_1d0 = 0x40a0_0000; - self.unk_1dc = 0x4780_0000; - self.unk_1e0 = 0x4170_0000; - self.unk_1ec = 0x384; - self.unk_1f0 = 0x3e8; - self.unk_1f4 = 0x3e8; - self.unk_200 = 0x3e8; - self.unk_234 = 0x3e8; - self.unk_274 = 0x64; - self.unk_290 = 0x82a; - self.unk_298 = 0x7d; - self.unk_29c = 1; - self.unk_ee0 = 0x18ae; - self.unk_ee8 = 1; - self.unk_2eec = 0x41c8_cccd; - self.unk_2ef0 = 0x41c8_cccd; - self.unk_35e4 = 1; - self.unk_35e8 = 0x1f4; - self.unk_35ec = 6; - self.unk_35f0 = [0x30d4; 6]; - self.unk_3608 = 0xfa0; - self.unk_361c = [0x30d4; 5]; - self.unk_3630 = 1; - self.unk_37a8 = 1; - self.unk_37b4 = 0x424c_cccd; - self.unk_37f4 = 0x4406_8000; - self.unk_3834 = 0x41c9_999a; - self.unk_38e0 = 0x186a; - self.secondary_status_va = secondary_status_va; - } -} // --------------------------------------------------------------------------- // Free lists // --------------------------------------------------------------------------- From 61e02a5f3512c06cddb1506015298a6c7be9aaf2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 12:32:59 -0400 Subject: [PATCH 0830/2680] drm/asahi: grow the scheduler-release table instead of failing a close The table of contexts awaiting their firmware scheduler-state release was fixed at 128 entries and a close that found it full returned ENOSPC, which the queue release path turns into a device stop for every client. A burst of closes that outpaces the firmware's consumption of release records is a legitimate workload, not a fault. Grow the table on demand (the close path runs under a sleepable mutex) and log once when the preallocated capacity is exceeded. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 7e317cd1218e5461124fbfc8db7dd72e71253f8e) --- drivers/gpu/drm/asahi/g17/teardown.rs | 28 ++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index 791ac690c0f50..5d56931879f41 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -5,10 +5,12 @@ //! The device mutex serializes this state with the Registry's displaced graph ownership. use super::{context::Context, freelist::control_consumed, fw::channels::SchedulerStateRelease}; -use core::sync::atomic::{fence, Ordering}; +use core::sync::atomic::{fence, AtomicBool, Ordering}; use kernel::{prelude::*, sync::Arc}; +/// Entries preallocated at device start; a burst of closes beyond it grows the table. const CAPACITY: usize = 128; +static GREW: AtomicBool = AtomicBool::new(false); const POLL_NS: u64 = 2_000_000_000; #[derive(Clone, Copy, PartialEq, Eq)] @@ -80,15 +82,23 @@ impl Pending { .position(|entry| Arc::ptr_eq(&entry.context, context)) } + /// Closes outpacing firmware release consumption grow the table instead of + /// failing the close: a failed close stops the device for every client. fn insert(&mut self, context: &Arc, phase: Phase, now: u64) -> Result { - self.entries - .push_within_capacity(Entry { - context: context.clone(), - phase, - armed: now, - announce_pending: false, - }) - .map_err(|_| ENOSPC) + let entry = Entry { + context: context.clone(), + phase, + armed: now, + announce_pending: false, + }; + if self.entries.len() >= CAPACITY && !GREW.swap(true, Ordering::Relaxed) { + pr_info!( + "asahi: G17 scheduler-release table grew past {} pending contexts\n", + CAPACITY + ); + } + self.entries.push(entry, GFP_KERNEL)?; + Ok(()) } /// Call before replacing a graph's logical owner. Failure leaves that graph with its From e1102447e5bcdd1b5599401474a062a46f8a5936 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 14:33:29 -0400 Subject: [PATCH 0831/2680] drm/asahi: add a render flag for passes with at most four known primitives Userspace knows when a render pass rasterizes at most four primitives in total (every draw's topology-derived count times instances, plus internal work) and that no indirect, mesh or otherwise uncounted work is part of it. Let it say so: DRM_ASAHI_RENDER_FEW_PRIMITIVES sets bit 0 of the fragment stage's auxiliary framebuffer control word, which the driver left clear so far. The flag is advertised by DRM_ASAHI_FEATURE_FEW_PRIMITIVES and rejected by kernels without it, like every other render flag. The producer of that bit in the native stack was recovered from the saved userland driver disassembly (end of render pass: bit 0 set iff the pass's known-primitive counter is at most four and its known-work byte is set; cleared for indirect and mesh work), and the saved native capture of a four-draw workload carries the bit on all 160 fragment commands. The hardware effect of the bit is not established; omitting it is the safe default and remains the behaviour for every pass that does not set the flag. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 18dbc2de6f21c92c49ffafce39a38c8d6439a5b5) --- drivers/gpu/drm/asahi/g17/command.rs | 4 +++- drivers/gpu/drm/asahi/g17/fw/render.rs | 15 +++++++++++---- drivers/gpu/drm/asahi/g17/mod.rs | 3 ++- include/uapi/drm/asahi_drm.h | 21 +++++++++++++++++++++ 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 5929c14b264fd..16727b8559b2b 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -23,7 +23,8 @@ const MAX_TILE_BUFFER: u64 = 32768; const PROCESS_EMPTY: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES; const RSRC_SPEC_HI: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_RSRC_SPEC_HI; const DBIAS_INT: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT; -const RENDER_FLAGS: u32 = PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT; +const FEW_PRIMITIVES: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_FEW_PRIMITIVES; +const RENDER_FLAGS: u32 = PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT | FEW_PRIMITIVES; /// Attachment settings persist until replaced, including by an empty list. #[derive(Clone, Copy)] @@ -374,6 +375,7 @@ impl Payload { scissor_va: cmd.isp_scissor_base, depth_bias_va: cmd.isp_dbias_base, depth_bias_is_int: cmd.flags & DBIAS_INT != 0, + few_primitives: cmd.flags & FEW_PRIMITIVES != 0, occlusion_query_va: cmd.isp_oclqry_base, sampler_heap_va: cmd.sampler_heap, sampler_count: cmd.sampler_count.into(), diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index e3da1b0e6d954..657efe508bb3d 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -85,6 +85,8 @@ const TILE_CONFIG_BASE: u64 = 0x280; const TILE_CONFIG_LAYERED: u64 = 1 << 0; const TILE_CONFIG_PROCESS_EMPTY_TILES: u64 = 1 << 16; const AUX_FB_FLAGS_BASE: u64 = 0xc000; +/// The pass rasterizes at most four known primitives (userspace's promise). +const AUX_FB_FLAGS_FEW_PRIMITIVES: u64 = 1 << 0; const AUX_FB_FLAGS_DBIAS_INT: u64 = 1 << 18; /// Tiling layer mode: layer count less one, bit 15, and for layered targets /// bit 14 plus the empty-tile policy. The second copy keeps only the count and @@ -178,6 +180,8 @@ pub(crate) struct RenderPass { pub(crate) depth_bias_va: u64, /// Depth-bias values are integers. pub(crate) depth_bias_is_int: bool, + /// At most four known primitives; sets the auxiliary framebuffer bit 0. + pub(crate) few_primitives: bool, /// GPU address of the occlusion-query results, or zero. pub(crate) occlusion_query_va: u64, /// GPU address of the sampler heap, or zero. @@ -423,10 +427,13 @@ impl PassWords { partial_zls_control |= ZLS_PARTIAL_STENCIL; } - let aux_fb_flags = match pass.depth_bias_is_int { - true => AUX_FB_FLAGS_BASE | AUX_FB_FLAGS_DBIAS_INT, - false => AUX_FB_FLAGS_BASE, - }; + let mut aux_fb_flags = AUX_FB_FLAGS_BASE; + if pass.depth_bias_is_int { + aux_fb_flags |= AUX_FB_FLAGS_DBIAS_INT; + } + if pass.few_primitives { + aux_fb_flags |= AUX_FB_FLAGS_FEW_PRIMITIVES; + } let rsrc_spec = |spec: u64| match pass.rsrc_spec_hi { true => spec, false => spec & u32::MAX as u64, diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 695ea787d314e..95e0dfa754047 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1067,7 +1067,8 @@ impl gpu::Gpu for Gpu { fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64; + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES as u64; params.gpu_generation = hw::GpuGen::G17 as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index cdb177bf4df05..f4c36d1c2fc1b 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -286,6 +286,12 @@ enum drm_asahi_feature { * These fields supplement the header barriers. */ DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS = (1UL) << 5, + + /** + * @DRM_ASAHI_FEATURE_FEW_PRIMITIVES: Render commands accept + * DRM_ASAHI_RENDER_FEW_PRIMITIVES. + */ + DRM_ASAHI_FEATURE_FEW_PRIMITIVES = (1UL) << 6, }; /** @@ -1034,6 +1040,21 @@ enum drm_asahi_render_flags { * so we match that here for efficiency. */ DRM_ASAHI_RENDER_DBIAS_IS_INT = (1U << 18), + + /** + * @DRM_ASAHI_RENDER_FEW_PRIMITIVES: The render pass is known to rasterize + * at most four primitives in total. + * + * The count includes every primitive the pass generates: topology-derived + * primitive counts of all draws multiplied by their instance counts, plus + * internal work (clears, visibility, scissor/state generated work). Passes + * with indirect draws, mesh work or any work whose primitive count is not + * known at record time must not set this flag. The flag sets bit 0 of the + * auxiliary framebuffer control word for the fragment stage; the hardware + * effect is an optimization for small known passes and its omission is + * always safe. Requires DRM_ASAHI_FEATURE_FEW_PRIMITIVES. + */ + DRM_ASAHI_RENDER_FEW_PRIMITIVES = (1U << 19), }; /** From 581873f835f699667090393c051fefd951fe5eac Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 14:43:08 -0400 Subject: [PATCH 0832/2680] drm/asahi: add a render flag that orders only the fragment stage on barriers The render prefix resolved from vdm_barrier/prior_vdm names the covered renders' fragment completions and was attached to the tiling kick, so a render's vertex processing waited for its predecessors' fragment work even when only its fragment stage consumes them. Keep the resolved dependencies apart by origin until the kick entries are built, and let userspace choose with DRM_ASAHI_RENDER_FRAGMENT_BARRIERS (advertised by DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS): with the flag the render prefix orders the fragment kick, which still waits for its own tiling kick and for the previous fragment work of its physical queue, so fragment order across a client's physical pairs is preserved; the tiling kick keeps its implicit parent. Compute dependencies keep ordering the tiling kick in both modes. Without the flag nothing changes. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c00f3a06ed2d0400a5ec210c856f3a382971f033) --- drivers/gpu/drm/asahi/g17/command.rs | 5 +- drivers/gpu/drm/asahi/g17/fw/render.rs | 2 + drivers/gpu/drm/asahi/g17/mod.rs | 3 +- drivers/gpu/drm/asahi/g17/queue/render.rs | 56 +++++++++++++++++-- .../drm/asahi/g17/runtime/render/submit.rs | 30 +++++----- include/uapi/drm/asahi_drm.h | 25 +++++++++ 6 files changed, 98 insertions(+), 23 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 16727b8559b2b..83191ddbcb1c3 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -24,7 +24,9 @@ const PROCESS_EMPTY: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS const RSRC_SPEC_HI: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_RSRC_SPEC_HI; const DBIAS_INT: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT; const FEW_PRIMITIVES: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_FEW_PRIMITIVES; -const RENDER_FLAGS: u32 = PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT | FEW_PRIMITIVES; +const FRAGMENT_BARRIERS: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_FRAGMENT_BARRIERS; +const RENDER_FLAGS: u32 = + PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT | FEW_PRIMITIVES | FRAGMENT_BARRIERS; /// Attachment settings persist until replaced, including by an empty list. #[derive(Clone, Copy)] @@ -376,6 +378,7 @@ impl Payload { depth_bias_va: cmd.isp_dbias_base, depth_bias_is_int: cmd.flags & DBIAS_INT != 0, few_primitives: cmd.flags & FEW_PRIMITIVES != 0, + fragment_barriers: cmd.flags & FRAGMENT_BARRIERS != 0, occlusion_query_va: cmd.isp_oclqry_base, sampler_heap_va: cmd.sampler_heap, sampler_count: cmd.sampler_count.into(), diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 657efe508bb3d..23eaa7e7dfba9 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -182,6 +182,8 @@ pub(crate) struct RenderPass { pub(crate) depth_bias_is_int: bool, /// At most four known primitives; sets the auxiliary framebuffer bit 0. pub(crate) few_primitives: bool, + /// Host-only: the resolved barriers order the fragment kick, not the tiling kick. + pub(crate) fragment_barriers: bool, /// GPU address of the occlusion-query results, or zero. pub(crate) occlusion_query_va: u64, /// GPU address of the sampler heap, or zero. diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 95e0dfa754047..eb26694bf2adf 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1068,7 +1068,8 @@ impl gpu::Gpu for Gpu { params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES as u64; + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS as u64; params.gpu_generation = hw::GpuGen::G17 as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index fd67523ab91c3..720f2d8a0897e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -34,6 +34,29 @@ pub(crate) use dependencies::Dependencies; use kernel::{prelude::*, sync::Arc}; use retirement::{Ticket, Tracker}; +/// Firmware dependencies of one render, kept by origin until the kick entries +/// are built: the covered earlier renders (their fragment completions) and the +/// covered compute completion. Which kick waits for the render prefix is the +/// command's choice (`RenderPass::fragment_barriers`); compute always orders +/// the tiling kick, since compute output may feed vertex work. +#[derive(Copy, Clone)] +pub(crate) struct RenderDependencies { + pub(crate) render: [KickDependency; 2], + pub(crate) render_count: usize, + pub(crate) compute: Option, +} + +impl RenderDependencies { + pub(crate) const EMPTY: Self = Self { + render: [KickDependency::ZERO; 2], + render_count: 0, + compute: None, + }; + pub(crate) fn render(&self) -> &[KickDependency] { + &self.render[..self.render_count] + } +} + #[derive(Copy, Clone)] pub(crate) struct RenderAddresses { /// Tiling then fragment; each pair contains the primary and auxiliary stamp. @@ -375,7 +398,7 @@ impl Pair { pub(crate) fn publish( &mut self, packet: Arc, - dependencies: &[KickDependency], + dependencies: &RenderDependencies, lease: PublicationLease, host: &mut impl Host, defer: &mut impl FnMut(Deferred) -> Result, @@ -493,7 +516,32 @@ impl Pair { self.memory.graph.queues.gpu_va() + u64::from(qid) * size_of::() as u64 }); - let fragment_barriers = [KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?]; + // The fragment kick always waits for its own tiling kick. With the + // fragment-barriers flag the resolved prefix orders the fragment kick + // too, and the tiling kick keeps only its implicit parent, so vertex + // work overlaps the predecessors' fragment work as on the native stack. + let Validated::Render { pass, .. } = &packet.command else { + return Err(EINVAL); + }; + let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; + let mut tiling_barriers = [KickDependency::ZERO; 3]; + let mut tiling_count = 0; + let mut fragment_barriers = [own_tiling; 3]; + let mut fragment_count = 1; + for dependency in dependencies.render() { + if pass.fragment_barriers { + fragment_barriers[fragment_count] = *dependency; + fragment_count += 1; + } else { + tiling_barriers[tiling_count] = *dependency; + tiling_count += 1; + } + } + if let Some(compute) = dependencies.compute { + tiling_barriers[tiling_count] = compute; + tiling_count += 1; + } + let tiling_barriers = &tiling_barriers[..tiling_count]; let args = |stage: usize, barriers, arrays, mcache| KickArgs { qid: qids[stage], timestamp: kicks[stage], @@ -516,13 +564,13 @@ impl Pair { }; let ta_args = args( 0, - dependencies, + tiling_barriers, abi::TaDescriptor::register_bindings(self.memory.graph.descriptor_client(0, ordinal))?, None, ); let fragment_args = args( 1, - &fragment_barriers, + &fragment_barriers[..fragment_count], abi::FragmentDescriptor::register_bindings( self.memory.graph.descriptor_client(1, ordinal), )?, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 8b6c647668af4..39b8c064d0106 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -177,26 +177,25 @@ impl Registry { owner: u64, packet: &Arc, state: u32, - dependencies: &mut [crate::g17::fw::kick::KickDependency; 3], - ) -> Result { - let mut count = 0; + ) -> Result { + let mut dependencies = crate::g17::queue::render::RenderDependencies::EMPTY; let frontier = self.render_prefix(owner, packet)?; for dependency in frontier.as_slice() { - *dependencies.get_mut(count).ok_or(EOVERFLOW)? = *dependency; - count += 1; + *dependencies + .render + .get_mut(dependencies.render_count) + .ok_or(EOVERFLOW)? = *dependency; + dependencies.render_count += 1; } if let Some(queue) = self.compute(owner) { - if let Some(dependency) = queue.dependency( + dependencies.compute = queue.dependency( owner, packet.completion.status(), packet.order.wait_through[1], state, - )? { - *dependencies.get_mut(count).ok_or(EOVERFLOW)? = dependency; - count += 1; - } + )?; } - Ok(count) + Ok(dependencies) } } @@ -266,7 +265,7 @@ impl crate::g17::Firmware { packet: &Arc, deferred: &mut DeferredBatch, ) -> Result { - use crate::g17::{fw::kick::KickDependency, queue::render::PublicationLease}; + use crate::g17::queue::render::PublicationLease; let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; if self.recovery.pending() || state != 0 { return Err(EAGAIN); @@ -281,10 +280,7 @@ impl crate::g17::Firmware { { return Err(EBUSY); } - let mut dependencies = [KickDependency::ZERO; 3]; - let count = self - .queues - .render_dependencies(owner, packet, state, &mut dependencies)?; + let dependencies = self.queues.render_dependencies(owner, packet, state)?; let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; let pool = pair.pool().clone(); let buffer_id = pair.buffer_id(); @@ -303,7 +299,7 @@ impl crate::g17::Firmware { let result = self.with_render(usize::from(slot), |pair, host| { pair.publish( packet.clone(), - &dependencies[..count], + &dependencies, lease, host, &mut |event| deferred.push(event), diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index f4c36d1c2fc1b..437ad42aeafda 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -292,6 +292,12 @@ enum drm_asahi_feature { * DRM_ASAHI_RENDER_FEW_PRIMITIVES. */ DRM_ASAHI_FEATURE_FEW_PRIMITIVES = (1UL) << 6, + + /** + * @DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS: Render commands accept + * DRM_ASAHI_RENDER_FRAGMENT_BARRIERS. + */ + DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS = (1UL) << 7, }; /** @@ -1055,6 +1061,25 @@ enum drm_asahi_render_flags { * always safe. Requires DRM_ASAHI_FEATURE_FEW_PRIMITIVES. */ DRM_ASAHI_RENDER_FEW_PRIMITIVES = (1U << 19), + + /** + * @DRM_ASAHI_RENDER_FRAGMENT_BARRIERS: Only the fragment stage waits for + * this command's barriers. + * + * The render dependencies resolved from @vdm_barrier and @prior_vdm are + * attached to the fragment half of the render instead of its vertex + * (tiling) half, so vertex processing may start while the renders this + * command depends on are still executing their fragment work. The + * fragment half still waits for its own vertex half and for the previous + * fragment work of its physical queue; the compute dependencies from + * @cdm_barrier and @prior_cdm keep ordering the vertex half. Set this + * only when every render dependency into the command is consumed by + * fragment-stage or attachment accesses alone: nothing the vertex stage + * reads (vertex/index/indirect buffers, vertex, tessellation or geometry + * shader resources, transform feedback, conditional-rendering predicates) + * may depend on it. Requires DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS. + */ + DRM_ASAHI_RENDER_FRAGMENT_BARRIERS = (1U << 20), }; /** From 4847012410e860cf6d091f21b2d67383c424e6a4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 20:15:59 -0400 Subject: [PATCH 0833/2680] drm/asahi: give back the USC backing of idle retained compute queues An installed compute queue is retained for the device's lifetime, and each one owns a private USC free-list pool of about 23 MiB. After a burst of clients the retained pools pinned 1.6 GB on an 8 GB machine for the rest of the boot. Keep only the four newest idle retained queues populated: older idle queues publish the release of their free-list buffer slot, and once the control consumer has passed the record their pool backing is dropped. The queue, its ring and its control objects stay installed; an owner that later takes such a vacant queue first allocates fresh backing, rewrites the pool's control header and descriptor row, and only then binds and kicks. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 64820e0cb952592fff5708712a4b4766a8a12f4c) --- drivers/gpu/drm/asahi/g17/freelist.rs | 64 ++++++- drivers/gpu/drm/asahi/g17/queue/compute.rs | 92 +++++++++- drivers/gpu/drm/asahi/g17/runtime.rs | 192 ++++++++++++++++++-- drivers/gpu/drm/asahi/g17/runtime/events.rs | 1 + 4 files changed, 323 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index e1d42944a38fe..e40dfec7fc5fd 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -65,7 +65,7 @@ impl Drop for RenderId { } } -struct Pages { +pub(crate) struct Pages { backing: KernelObject, blocks: usize, layout: RunLayout, @@ -86,6 +86,10 @@ impl Pages { layout, }) } + /// Fresh zeroed compute backing, built without the device mutex. + pub(crate) fn compute(alloc: &Allocator<'_>) -> Result { + Self::new(alloc, COMPUTE_BLOCKS, RunLayout::Compute) + } fn page_list_va(&self) -> u64 { self.backing.gpu_va() } @@ -324,8 +328,11 @@ impl Submitted<'_> { } /// A pool owned by one physical compute queue, with fixed QID-derived slot. +/// Idle retained queues may give their USC backing back (`pages` is `None`) +/// once the firmware consumed the pool's release; the control and state +/// objects the queue record names stay until processor stop. pub(crate) struct ComputePool { - pages: Pages, + pages: Option, control: KernelObject, state: KernelObject, id: u16, @@ -362,7 +369,7 @@ impl ComputePool { )?; fence(Ordering::SeqCst); Ok(Self { - pages, + pages: Some(pages), control, state, id, @@ -371,15 +378,58 @@ impl ComputePool { pub(crate) fn id(&self) -> u16 { self.id } - pub(crate) fn page_list_va(&self) -> u64 { - self.pages.page_list_va() + /// EAGAIN while the pool is vacant: the owner must re-arm it first. + pub(crate) fn page_list_va(&self) -> Result { + self.pages.as_ref().map(Pages::page_list_va).ok_or(EAGAIN) } pub(crate) fn control_va(&self) -> u64 { self.control.gpu_va() } + pub(crate) fn populated(&self) -> bool { + self.pages.is_some() + } + /// The release record for this pool's buffer slot under the free-list + /// generation the queue last published. + pub(crate) fn release_record(&self, generation: u64) -> super::fw::channels::FreeListRelease { + super::fw::channels::FreeListRelease::new(generation, u32::from(self.id)) + } + /// The caller has observed the firmware consume this pool's release. + pub(crate) fn drop_pages(&mut self) { + self.pages = None; + } + /// Rewrites the shared control header for fresh backing. The caller then + /// rewrites the page-pool descriptor row before the queue is kicked again. + /// As render repopulation does, the firmware-completed counter is carried + /// over so it stays consistent with the host submitted counter in `state`. + pub(crate) fn rearm(&mut self, pages: Pages) -> Result { + if self.pages.is_some() { + return Err(EBUSY); + } + let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; + let mut bytes = [0; 8]; + for (index, byte) in bytes.iter_mut().enumerate() { + // SAFETY: The checked eight-byte range stays mapped; byte accesses + // match the unaligned packed counter the firmware writes. + *byte = unsafe { completed.add(index).read_volatile() }; + } + let mut header = FreeListControl::compute(&FreeListArgs { + buffer_slot: u32::from(self.id), + class: FreeListClass::One, + page_list_va: pages.page_list_va(), + run_list_va: pages.run_list_va(), + blocks: COMPUTE_BLOCKS as u32, + state_va: self.state.gpu_va(), + }); + header.unk_54[..8].copy_from_slice(&bytes); + self.control.write(0, header)?; + fence(Ordering::SeqCst); + self.pages = Some(pages); + Ok(()) + } pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { - let address = self.pages.page_list_va(); - self.pages + let pages = self.pages.as_mut().ok_or(EAGAIN)?; + let address = pages.page_list_va(); + pages .backing .map_alias(vm, address, mmu::PROT_GPU_SHARED_RW) } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index fcd0a1d6b40e6..cbca61493611e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -7,8 +7,9 @@ pub(crate) mod exit; pub(crate) mod retirement; use crate::g17::{ + channel::Rings, context::Context, - fw::{compute::COMPUTE_DESCRIPTOR_SIZE, queue::*}, + fw::{channels::ControlRecord, compute::COMPUTE_DESCRIPTOR_SIZE, queue::*}, object::{Allocator, CpuMap, KernelObject}, }; use crate::{hw::t8140::dynamic, mmu}; @@ -16,6 +17,10 @@ use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; use retirement::{Cursors, RECORD_SLOTS}; +fn now_ns() -> u64 { + ::ktime_get() as u64 +} + const DESCRIPTORS: usize = 256; const GRAPH_SIZE: usize = 0x8000; const CONTEXT_SIZE: usize = 0x20000; @@ -48,7 +53,9 @@ static_assert!(core::mem::offset_of!(GraphLayout, items) == ITEMS); /// until the first exact successful publication under their replacement has retired. struct Binding { _kick_alias: mmu::KernelMapping, - _pool_alias: mmu::KernelMapping, + /// Client alias of the USC pool pages; dropped when the pool is vacated and + /// recreated when an owner binds a re-armed pool (it pins the backing). + pool_alias: Option, _descriptors: mmu::KernelMapping, preempt: mmu::KernelMapping, usage: mmu::KernelMapping, @@ -153,7 +160,7 @@ impl Graph { .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; Ok(Binding { _kick_alias: kick_alias, - _pool_alias: pool_alias, + pool_alias: Some(pool_alias), _descriptors: descriptors, preempt, usage, @@ -341,8 +348,8 @@ impl PoolBacking { } /// Fresh page-pool row data, installed once while the QID is still unpublished. - pub(crate) fn pool_descriptor(&self) -> (u16, u64) { - (self.pool.id(), self.pool.page_list_va()) + pub(crate) fn pool_descriptor(&self) -> Result<(u16, u64)> { + Ok((self.pool.id(), self.pool.page_list_va()?)) } } @@ -363,8 +370,22 @@ impl Backing { /// Installed graph and its exact logical owner. A released idle graph may change owners only /// after a retirement witness. Failed work retains its pointers independently of fence state. +/// USC pool backing of a retained queue. A release is published only for an idle +/// queue without an owner; the backing is dropped once the firmware consumed it. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum PoolPhase { + Populated, + ReleaseSent(u32), + Vacant, +} + pub(crate) struct Queue { owner: Option, + pool_phase: PoolPhase, + /// Monotonic time the last owner released this queue (vacate order). + released_at: u64, + /// Free-list generation of the last published configuration. + free_list_generation: u64, binding: Binding, previous: Option, kick: kick::Queue, @@ -413,6 +434,9 @@ impl Queue { context.mark_published(); Ok(Self { owner: Some(owner), + pool_phase: PoolPhase::Populated, + released_at: 0, + free_list_generation: 0, binding, previous: None, kick, @@ -914,6 +938,8 @@ impl Queue { } pub(crate) fn reusable(&self) -> bool { self.installed + // A pool release in flight must be witnessed before any successor. + && !matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) // An issued kill still names this graph even if its commands retire // before the acknowledgement. It cannot acquire a successor owner. && self.exit.is_none() @@ -934,6 +960,10 @@ impl Queue { } let old = self.binding.context.clone(); if Arc::ptr_eq(old.status(), context.status()) && old.is_current() { + if self.binding.pool_alias.is_none() { + // The same VM takes back a re-armed pool: alias the new backing. + self.binding.pool_alias = Some(self.pool.map_client(context.vm())?); + } self.graph.set_owner(&context)?; self.binding.context = context.clone(); self.previous = Some(Previous { @@ -984,10 +1014,61 @@ impl Queue { } if self.retirement_ready { self.owner = None; + self.released_at = now_ns(); } } Ok(false) } + /// An idle retained queue whose USC backing may be given back. + pub(crate) fn vacate_candidate(&self) -> bool { + self.reusable() && self.pool_phase == PoolPhase::Populated && self.pool.populated() + } + pub(crate) fn released_at(&self) -> u64 { + self.released_at + } + pub(crate) fn pool_vacant(&self) -> bool { + self.pool_phase == PoolPhase::Vacant + } + /// Publishes the release of this idle queue's free list. The caller holds the + /// device mutex and rings the control doorbell afterwards. + pub(crate) fn vacate(&mut self, init: &crate::g17::initdata::InitData) -> Result { + if !self.vacate_candidate() { + return Err(EBUSY); + } + let record = ControlRecord::FreeListRelease( + self.pool.release_record(self.free_list_generation), + ); + let cursor = Rings::publish_control(init, &record)?; + self.pool_phase = PoolPhase::ReleaseSent(cursor); + fence(Ordering::SeqCst); + Ok(()) + } + /// Drops the USC backing once the firmware's control consumer passed the + /// release. Returns true when the backing was freed by this call. + pub(crate) fn observe_vacancy(&mut self, consumer: u32, producer: u32) -> Result { + let PoolPhase::ReleaseSent(cursor) = self.pool_phase else { + return Ok(false); + }; + if !crate::g17::freelist::control_consumed(consumer, producer, cursor)? { + return Ok(false); + } + // The retained owner binding's client alias pins the backing: drop it + // first (the owner has exited; `reusable()` proved no previous binding). + self.binding.pool_alias = None; + self.pool.drop_pages(); + self.pool_phase = PoolPhase::Vacant; + Ok(true) + } + /// Installs fresh backing into a vacant pool; returns the descriptor row data + /// the caller writes before binding an owner. + pub(crate) fn rearm(&mut self, pages: crate::g17::freelist::Pages) -> Result<(u16, u64)> { + if self.pool_phase != PoolPhase::Vacant || !self.reusable() { + return Err(EAGAIN); + } + self.pool.rearm(pages)?; + self.pool_phase = PoolPhase::Populated; + Ok((self.pool.id(), self.pool.page_list_va()?)) + } /// Exact host identity remains valid even when several commands share an ioctl ID. pub(crate) fn owns_packet(&self, packet: &Arc) -> bool { self.active @@ -1299,6 +1380,7 @@ impl Queue { qos: packet.context.qos_id(), context: packet.context.id(), }; + self.free_list_generation = u64::from(packet.context.scheduler_generation()); let config = QueueConfig::new(&QueueConfigArgs { target: QueueConfigTarget::Compute, kick_ring_va: self.kick.low_va(), diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 805087d7082ab..0de6f62760248 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -24,6 +24,7 @@ use super::{ kick, qos, queue::{self, admission, compute}, }; +use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; const QIDS: usize = 128; @@ -117,21 +118,127 @@ impl Registry { /// An installed idle queue keeps its original QID and backing. Mapping a /// replacement context must succeed before its old binding is displaced. - pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc) -> Result { + /// A vacant queue (USC backing given back) is offered for re-arming only + /// when no populated idle queue exists. + pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc) -> Result { if let Some(queue) = self.compute(owner) { - return queue.matches(owner, queue.qid(), context); + return Ok(if queue.matches(owner, queue.qid(), context)? { + Rebind::Done + } else { + Rebind::Fresh + }); } - let Some(queue) = self + let mut vacant = None; + for queue in self .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) - .find(|queue| queue.reusable()) - else { - return Ok(false); - }; - queue.bind_owner(owner, context.clone())?; - Ok(true) + .filter(|queue| queue.reusable()) + { + if queue.pool_vacant() { + vacant.get_or_insert(queue.qid()); + continue; + } + queue.bind_owner(owner, context.clone())?; + return Ok(Rebind::Done); + } + Ok(vacant.map_or(Rebind::Fresh, Rebind::Rearm)) + } + + /// Installs fresh USC backing into the vacant queue `qid`; the caller writes + /// the returned descriptor row before binding. EAGAIN if the queue was taken. + pub(super) fn rearm_compute( + &mut self, + qid: u8, + pages: crate::g17::freelist::Pages, + ) -> Result<(u16, u64)> { + let queue = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid) + .ok_or(EAGAIN)?; + queue.rearm(pages) + } + + pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { + let queue = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid && queue.reusable()) + .ok_or(EAGAIN)?; + queue.bind_owner(owner, context.clone()) + } + + /// Publishes releases for the oldest idle retained pools beyond `cap`, at + /// most four per pass. Returns the number published. + pub(super) fn vacate_idle_pools( + &mut self, + init: &super::initdata::InitData, + cap: usize, + ) -> Result { + // Count first: event turns are frequent and the steady state has no excess. + let count = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| queue.vacate_candidate()) + .count(); + if count <= cap { + return Ok(0); + } + let mut candidates: KVec<(u64, u8)> = KVec::with_capacity(count, GFP_KERNEL)?; + for queue in self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| queue.vacate_candidate()) + { + candidates.push((queue.released_at(), queue.qid()), GFP_KERNEL)?; + } + candidates.sort_unstable(); + let excess = candidates.len() - cap; + let mut published = 0; + for (_, qid) in candidates.iter().take(excess.min(4)) { + let queue = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == *qid) + .ok_or(EIO)?; + match queue.vacate(init) { + Err(error) if error == EAGAIN => break, + result => result?, + } + published += 1; + } + Ok(published) + } + + /// Frees the backing of every pool whose release the firmware consumed. + pub(super) fn observe_pool_releases(&mut self, init: &super::initdata::InitData) -> Result { + fence(Ordering::Acquire); + let consumer = init.control_consumer()?.load(Ordering::Relaxed); + let producer = init.control_producer()?.load(Ordering::Relaxed); + let mut freed = 0; + for queue in self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + if queue.observe_vacancy(consumer, producer)? { + freed += 1; + } + } + Ok(freed) } pub(super) fn reserve_compute(&mut self, owner: u64) -> Result { @@ -220,6 +327,17 @@ impl Registry { } } +/// Outcome of offering an owner a retained compute queue. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum Rebind { + /// The owner now holds an installed, populated queue. + Done, + /// Only a vacant queue is available; re-arm `qid` with fresh backing. + Rearm(u8), + /// No retained queue suits the owner: reserve a fresh QID. + Fresh, +} + /// An immutable logical owner shared by its two scheduler entities. Installed /// physical graphs retain the context independently of this frontend lifetime. pub(super) struct Backend { @@ -420,16 +538,41 @@ impl Backend { { return Err(EIO); } - if firmware.queues.rebind_compute(self.owner, &self.context)? { - return Ok(()); + match firmware.queues.rebind_compute(self.owner, &self.context)? { + Rebind::Done => return Ok(()), + Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), + Rebind::Fresh => { + let reservation = firmware.queues.reserve_compute(self.owner)?; + (firmware.uat.clone(), Ok(reservation)) + } } - let reservation = firmware.queues.reserve_compute(self.owner)?; - (firmware.uat.clone(), reservation) }; let alloc = super::object::Allocator { dev: &dev, uat: &uat, }; + let reservation = match reservation { + Ok(reservation) => reservation, + Err(qid) => { + // Fresh zeroed backing is built without the device mutex; the + // queue may have been taken or re-armed by another owner since. + let pages = crate::g17::freelist::Pages::compute(&alloc)?; + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + continue; + } + let (slot, page_list) = match firmware.queues.rearm_compute(qid, pages) { + Err(error) if error == EAGAIN => continue, + result => result?, + }; + // The firmware consumed this slot's release before the backing + // was dropped; the row is rewritten before the queue is kicked. + firmware.init.initialize_compute_pool(slot, page_list)?; + firmware.queues.bind_rearmed(qid, self.owner, &self.context)?; + return Ok(()); + } + }; let built = (|| { let pool = compute::PoolBacking::new(&alloc, reservation.id)?; { @@ -439,7 +582,7 @@ impl Backend { return Err(EAGAIN); } firmware.queues.reserved(reservation)?; - let (slot, page_list) = pool.pool_descriptor(); + let (slot, page_list) = pool.pool_descriptor()?; firmware.init.initialize_compute_pool(slot, page_list)?; } let backing = compute::Backing::new(&alloc, pool)?; @@ -926,6 +1069,27 @@ impl super::Shared { } } +/// Idle retained compute queues that keep their USC pool populated; older idle +/// pools give their backing back. +const COMPUTE_WARM_CAP: usize = 4; + +impl super::Firmware { + /// Event-worker service step: witness consumed pool releases, then release the + /// oldest idle retained pools beyond the warm cap. + pub(in crate::g17) fn service_compute_pools(&mut self) -> Result { + self.queues.observe_pool_releases(&self.init)?; + if self.render_control_backpressured() { + return Ok(()); + } + let published = self.queues.vacate_idle_pools(&self.init, COMPUTE_WARM_CAP)?; + if published != 0 { + fence(Ordering::SeqCst); + self.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + } + Ok(()) + } +} + /// One monotonic domain for recovery grace, idle effort and control retirement. fn now_ns() -> u64 { ::ktime_get() as u64 diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 08c11eb98790e..f92c493f4813d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -386,6 +386,7 @@ impl crate::g17::Shared { let idle = firmware.idle_check(now, in_flight)?; firmware.reclaim_drained_renders(deferred)?; firmware.service_compute_exit(deferred)?; + firmware.service_compute_pools()?; { let mut reclaim = self.reclaim.lock(); firmware.service_teardown(&mut reclaim)?; From 1cdbdcbe47eddd1cecb7c4a0c77a685f31008803 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 20:16:11 -0400 Subject: [PATCH 0834/2680] drm/asahi: drop an exited owner's context from idle retained compute queues A retained compute queue kept its last owner's Context, and through it the exited client's whole VM: page tables, every alias the binding held into that VM, the status and the QoS lease, until another owner took the queue. Once the owner's execution root has been released (after the scheduler-state release and command retirement were witnessed) and the queue is idle without an owner, drop the binding and the context and keep only the scheduler page and work storage the installed queue record still names. A later owner binds afresh, as a first owner does. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 3db48c0ac600b5e919c4d2713dbd84e9b12b4ec4) --- drivers/gpu/drm/asahi/g17/context.rs | 16 +++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 97 ++++++++++++++----- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 7 +- drivers/gpu/drm/asahi/g17/runtime.rs | 15 ++- .../gpu/drm/asahi/g17/runtime/compute_exit.rs | 2 +- 5 files changed, 109 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index b7d0a351f234d..7076708be9f2f 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -101,6 +101,13 @@ impl WorkStorage { } } +/// Firmware-visible pages of a context, retained by an installed queue whose +/// last owner has exited (see [`Context::retain_firmware_pages`]). +pub(crate) struct FirmwarePages { + _scheduler: Arc, + _work: Arc>, +} + pub(crate) struct Context { execution: Arc, status: Arc, @@ -221,6 +228,15 @@ impl Context { pub(crate) fn scheduler_generation(&self) -> u32 { self.scheduler_generation } + /// The pages an installed queue record of this context still names: the + /// scheduler page and the work storage holding the job-list head. A retained + /// physical queue keeps these after dropping the context itself. + pub(crate) fn retain_firmware_pages(&self) -> FirmwarePages { + FirmwarePages { + _scheduler: self.scheduler.clone(), + _work: self.work.clone(), + } + } pub(crate) fn work_head_va(&self) -> u64 { self.work_base + (UAT_PGSZ + 0x400) as u64 } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index cbca61493611e..df38a727c8d60 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -381,12 +381,16 @@ pub(crate) enum PoolPhase { pub(crate) struct Queue { owner: Option, + /// Scheduler/work pages of an exited last owner whose context was dropped; + /// the installed queue record still names them until the next owner binds. + retained_owner: Option, pool_phase: PoolPhase, /// Monotonic time the last owner released this queue (vacate order). released_at: u64, /// Free-list generation of the last published configuration. free_list_generation: u64, - binding: Binding, + /// `None` after an exited owner was detached; a new owner binds afresh. + binding: Option, previous: Option, kick: kick::Queue, pool: ComputePool, @@ -434,10 +438,11 @@ impl Queue { context.mark_published(); Ok(Self { owner: Some(owner), + retained_owner: None, pool_phase: PoolPhase::Populated, released_at: 0, free_list_generation: 0, - binding, + binding: Some(binding), previous: None, kick, pool, @@ -474,12 +479,14 @@ impl Queue { pub(crate) fn owner(&self) -> Option { self.owner } - pub(crate) fn context(&self) -> &Arc { - &self.binding.context + /// The bound owner's context; `None` once an exited owner was detached. + pub(crate) fn context(&self) -> Option<&Arc> { + self.binding.as_ref().map(|binding| &binding.context) } /// Quarantined commands still retain the scheduler until their retirement witness. pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { - !Arc::ptr_eq(self.context(), context) || (self.active.len == 0 && self.replays.len == 0) + !self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) + || (self.active.len == 0 && self.replays.len == 0) } pub(crate) fn in_flight(&self) -> bool { self.active.len != 0 && !self.quarantined @@ -524,7 +531,9 @@ impl Queue { fn qos_owner(&self) -> qos::Owner { qos::Owner { qid: self.qid(), - qos: self.binding.context.qos_id(), + // Accounting completes only for bound owners; a detached queue has no + // pending work and never reaches this. + qos: self.binding.as_ref().map_or(0, |binding| binding.context.qos_id()), data_master: DataMaster::Compute as u8, } } @@ -598,7 +607,10 @@ impl Queue { || self.owner != Some(ticket.owner) || active.packet.order.sequence != ticket.submission || active.packet.context.id() != ticket.context_id - || active.packet.context.id() != self.binding.context.id() + || self + .binding + .as_ref() + .is_none_or(|binding| active.packet.context.id() != binding.context.id()) { return Err(EIO); } @@ -929,7 +941,10 @@ impl Queue { if self.released || self.quarantined || self.qid() != qid - || !Arc::ptr_eq(&self.binding.context, context) + || !self + .binding + .as_ref() + .is_some_and(|binding| Arc::ptr_eq(&binding.context, context)) || !context.is_current() { return Err(EIO); @@ -958,17 +973,21 @@ impl Queue { if !context.is_current() { return Err(EFAULT); } - let old = self.binding.context.clone(); - if Arc::ptr_eq(old.status(), context.status()) && old.is_current() { - if self.binding.pool_alias.is_none() { + let old = self.binding.as_ref().map(|binding| binding.context.clone()); + let same_vm = old + .as_ref() + .is_some_and(|old| Arc::ptr_eq(old.status(), context.status()) && old.is_current()); + if same_vm { + let binding = self.binding.as_mut().ok_or(EIO)?; + if binding.pool_alias.is_none() { // The same VM takes back a re-armed pool: alias the new backing. - self.binding.pool_alias = Some(self.pool.map_client(context.vm())?); + binding.pool_alias = Some(self.pool.map_client(context.vm())?); } self.graph.set_owner(&context)?; - self.binding.context = context.clone(); + binding.context = context.clone(); self.previous = Some(Previous { binding: None, - context: old, + context: old.ok_or(EIO)?, publication: None, }); } else { @@ -976,12 +995,15 @@ impl Queue { let pool_alias = self.pool.map_client(context.vm())?; let binding = self.graph.bind(context.clone(), kick_alias, pool_alias)?; self.graph.set_owner(&context)?; - let old_binding = core::mem::replace(&mut self.binding, binding); - self.previous = Some(Previous { - binding: Some(old_binding), + let old_binding = self.binding.replace(binding); + // A detached queue (exited owner already dropped) has nothing to supersede; + // the new owner's record fields replace the retained pages' references. + self.previous = old.map(|old| Previous { + binding: old_binding, context: old, publication: None, }); + self.retained_owner = None; } context.mark_published(); self.owner = Some(owner); @@ -1006,9 +1028,9 @@ impl Queue { } let mut previous = self.previous.take().ok_or(EIO)?; if let Some(binding) = previous.binding.take() { - self.binding = binding; + self.binding = Some(binding); } else { - self.binding.context = previous.context; + self.binding.as_mut().ok_or(EIO)?.context = previous.context; } self.retirement_ready = true; } @@ -1019,6 +1041,25 @@ impl Queue { } Ok(false) } + /// The last owner's file and VM are gone: its execution root was released + /// after the scheduler-state release and command retirement were witnessed + /// (`Context::release_execution`), and this queue is idle without an owner. + /// Drop the binding (every alias into that VM) and the context, keeping only + /// the scheduler page and work storage the installed queue record names. + pub(crate) fn detach_exited_owner(&mut self) -> bool { + if !self.reusable() { + return false; + } + let Some(binding) = self.binding.as_ref() else { + return false; + }; + if binding.context.is_current() { + return false; + } + self.retained_owner = Some(binding.context.retain_firmware_pages()); + self.binding = None; + true + } /// An idle retained queue whose USC backing may be given back. pub(crate) fn vacate_candidate(&self) -> bool { self.reusable() && self.pool_phase == PoolPhase::Populated && self.pool.populated() @@ -1054,7 +1095,9 @@ impl Queue { } // The retained owner binding's client alias pins the backing: drop it // first (the owner has exited; `reusable()` proved no previous binding). - self.binding.pool_alias = None; + if let Some(binding) = self.binding.as_mut() { + binding.pool_alias = None; + } self.pool.drop_pages(); self.pool_phase = PoolPhase::Vacant; Ok(true) @@ -1306,9 +1349,15 @@ impl Queue { { return Err(EBUSY); } - if !packet.context.is_current() || !Arc::ptr_eq(&packet.context, &self.binding.context) { + if !packet.context.is_current() + || !self + .binding + .as_ref() + .is_some_and(|binding| Arc::ptr_eq(&packet.context, &binding.context)) + { return Err(EFAULT); } + let binding = self.binding.as_ref().ok_or(EFAULT)?; if packet.completion.status().get() != 0 { return Err(EIO); } @@ -1361,9 +1410,9 @@ impl Queue { qid, kick_count: self.ordinal, kick: kick_timestamp, - preempt_va: self.binding.preempt.iova(), - operand_state_va: self.binding.operand.iova(), - usage_va: self.binding.usage.iova(), + preempt_va: binding.preempt.iova(), + operand_state_va: binding.operand.iova(), + usage_va: binding.usage.iova(), usage_fw_va: self.graph.usage.gpu_va(), free_list_slot: u32::from(self.pool.id()), free_list_control_va: self.pool.control_va(), diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index d1048fd04607f..08d1013e4762b 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -77,7 +77,7 @@ pub(super) struct Transaction { impl Queue { pub(crate) fn exit_candidate(&self, context: &Arc, age: u64) -> bool { - Arc::ptr_eq(self.context(), context) + self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) && self.released && self.active.len != 0 && self.exit.is_none() @@ -156,7 +156,10 @@ impl Queue { if !recovery_closed { return Ok(()); } - let state = match State::read(self.context()) { + let Some(context) = self.context() else { + return Ok(()); + }; + let state = match State::read(context) { Ok(state) => state, Err(_) => return Ok(()), }; diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 0de6f62760248..204a3f97b3c52 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -822,7 +822,11 @@ impl Registry { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - if queue.in_flight() && Arc::ptr_eq(queue.context().status(), status) { + if queue.in_flight() + && queue + .context() + .is_some_and(|context| Arc::ptr_eq(context.status(), status)) + { queue.quarantine(error, &mut |event| deferred.push(event))?; } } @@ -1077,6 +1081,15 @@ impl super::Firmware { /// Event-worker service step: witness consumed pool releases, then release the /// oldest idle retained pools beyond the warm cap. pub(in crate::g17) fn service_compute_pools(&mut self) -> Result { + for queue in self + .queues + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + queue.detach_exited_owner(); + } self.queues.observe_pool_releases(&self.init)?; if self.render_control_backpressured() { return Ok(()); diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs index 439d5c9831cbb..984b6c1f643eb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs @@ -97,7 +97,7 @@ impl Firmware { for entry in self.queues.compute.iter().flatten() { if let Some(queue) = entry.queue.as_deref() { let bit = 1u128 << queue.qid(); - if Arc::ptr_eq(queue.context(), context) { + if queue.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) { own |= bit; } else { foreign |= bit; From 0819b797efad53da331931774bb649f42f43a22c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 16:02:55 -0400 Subject: [PATCH 0835/2680] drm/asahi: keep reclaim away from VMs that are in use or were just restored Idle-VM reclaim judged a VM idle whenever no job was accepted at that instant. A loading game submits every few tens of milliseconds, so its VM qualified between submissions: under memory pressure the shrinker released its GPU backing, the next SUBMIT restored every invalidated mapping while holding the residency gate (observed as 260-280 ms SUBMIT and VM_BIND stalls ending together), and the restored pages were evicted again seconds later. Two rules: a VM is eligible only once its last accepted job ended at least 500 ms ago, and after a restoration the VM stays ineligible for sixteen times the restoration's duration, bounded to 1-30 s. Truly idle VMs are reclaimed as before. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5427403e7135dd886ee52a78d73639bd5a2ccb17) --- drivers/gpu/drm/asahi/mmu/lifetime.rs | 14 ++++++++- drivers/gpu/drm/asahi/mmu/residency.rs | 42 +++++++++++++++++++++++--- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index ea3edaca6ef38..053b7fbd0c409 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -14,6 +14,13 @@ use kernel::{ }; const ADMISSION_WAIT_MS: u32 = 1000; +/// A VM whose last accepted job ended more recently than this is still in +/// use: evicting its backing would be restored by its next job. +const RECLAIM_QUIET_NS: u64 = 500_000_000; + +fn now_ns() -> u64 { + ::ktime_get() as u64 +} pub(super) struct DriverMappings { mappings: KVec, @@ -49,6 +56,8 @@ impl DriverMappings { struct State { active: usize, + /// Monotonic time the VM last had no accepted job (0 = never had one). + quiet_since: u64, commits: usize, blocked: bool, quarantined: bool, @@ -76,13 +85,15 @@ impl VmLifetime { state.active == 0 && state.commits == 0 && !state.blocked && !state.quarantined && !state.draining && !state.closed && !state.cleanup_failed + && (state.quiet_since == 0 + || now_ns().saturating_sub(state.quiet_since) >= RECLAIM_QUIET_NS) }) } pub(super) fn new() -> Result> { Arc::pin_init( pin_init!(Self { - state <- new_mutex!(State { active:0, commits:0, blocked:false, + state <- new_mutex!(State { active:0, quiet_since:0, commits:0, blocked:false, quarantined:false, draining:false, closed:false, cleanup_failed:false, close_ranges:None, unmaps:KVec::new(), objects:KVec::new() }, "VM job lifetime"), changed <- new_condvar!("VM mapping admission"), @@ -569,6 +580,7 @@ impl Drop for VmJobGuard { if state.active != 0 { return; } + state.quiet_since = now_ns(); state.draining = true; drop(state); self.vm.drain_mappings(lifetime); diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 53c09dcb98d6d..573c524c3bfad 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -9,12 +9,39 @@ use super::*; use kernel::drm::gem::BaseObject; use kernel::{new_condvar, sync::CondVar}; +/// Re-evicting backing that a job just had to restore only repeats the +/// restoration. After a restore, reclaim of this VM pauses for a backoff +/// proportional to what the restore cost, within these bounds. +const RESTORE_BACKOFF_MIN_NS: u64 = 1_000_000_000; +const RESTORE_BACKOFF_MAX_NS: u64 = 30_000_000_000; +const RESTORE_BACKOFF_FACTOR: u64 = 16; + +fn now_ns() -> u64 { + ::ktime_get() as u64 +} + #[derive(Default)] struct State { users: usize, busy: bool, evicted: bool, closed: bool, + /// Monotonic time the last restoration finished and its duration. + restored_at: u64, + restore_cost: u64, +} + +impl State { + fn in_restore_backoff(&self, now: u64) -> bool { + if self.restored_at == 0 { + return false; + } + let backoff = self + .restore_cost + .saturating_mul(RESTORE_BACKOFF_FACTOR) + .clamp(RESTORE_BACKOFF_MIN_NS, RESTORE_BACKOFF_MAX_NS); + now.saturating_sub(self.restored_at) < backoff + } } #[pin_data] @@ -76,11 +103,15 @@ impl Gate { // No new reader can enter until every retained mapping is restored. state.busy = true; drop(state); + let started = now_ns(); let result = restore(); let mut state = gate.state.lock(); state.busy = false; if result.is_ok() { state.evicted = false; + let now = now_ns(); + state.restored_at = now; + state.restore_cost = now.saturating_sub(started); } drop(state); gate.ready.notify_all(); @@ -123,7 +154,7 @@ impl Gate { /// The busy gate continues to exclude restoration and metadata changes. pub(crate) fn try_reclaim(gate: &Arc) -> Option { let mut state = gate.state.try_lock()?; - if state.users != 0 || state.busy || state.closed { + if state.users != 0 || state.busy || state.closed || state.in_restore_backoff(now_ns()) { return None; } state.busy = true; @@ -142,9 +173,12 @@ impl Gate { } pub(crate) fn idle(&self) -> bool { - self.state - .try_lock() - .is_some_and(|state| state.users == 0 && !state.busy && !state.closed) + self.state.try_lock().is_some_and(|state| { + state.users == 0 + && !state.busy + && !state.closed + && !state.in_restore_backoff(now_ns()) + }) } /// A page-table mutation happened outside a `Reclaim` (the purge path runs From d96cfffb97e084fc821c3f8e0fbf6ebcb3ef1858 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 16:26:15 -0400 Subject: [PATCH 0836/2680] drm/asahi: count bindings and VM creation as reclaim activity Idle-VM reclaim deferred a VM for 500 ms after its last accepted job ended, tracked in the job lifetime. A VM that binds memory without submitting, as a client does while loading or before its first job, was eligible at once, and the shrinker's count did not apply the deferral that its scan applied. Track the time the last residency lease (job, binding or retirement) was released, starting at the VM's creation, in the residency gate, and apply the same 500 ms deferral and restore backoff to both the count and the scan. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 1c62cd0853b670104424bcfddeee7f7d9279304e) --- drivers/gpu/drm/asahi/mmu/lifetime.rs | 14 +------------- drivers/gpu/drm/asahi/mmu/residency.rs | 23 +++++++++++++++++++---- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 053b7fbd0c409..ea3edaca6ef38 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -14,13 +14,6 @@ use kernel::{ }; const ADMISSION_WAIT_MS: u32 = 1000; -/// A VM whose last accepted job ended more recently than this is still in -/// use: evicting its backing would be restored by its next job. -const RECLAIM_QUIET_NS: u64 = 500_000_000; - -fn now_ns() -> u64 { - ::ktime_get() as u64 -} pub(super) struct DriverMappings { mappings: KVec, @@ -56,8 +49,6 @@ impl DriverMappings { struct State { active: usize, - /// Monotonic time the VM last had no accepted job (0 = never had one). - quiet_since: u64, commits: usize, blocked: bool, quarantined: bool, @@ -85,15 +76,13 @@ impl VmLifetime { state.active == 0 && state.commits == 0 && !state.blocked && !state.quarantined && !state.draining && !state.closed && !state.cleanup_failed - && (state.quiet_since == 0 - || now_ns().saturating_sub(state.quiet_since) >= RECLAIM_QUIET_NS) }) } pub(super) fn new() -> Result> { Arc::pin_init( pin_init!(Self { - state <- new_mutex!(State { active:0, quiet_since:0, commits:0, blocked:false, + state <- new_mutex!(State { active:0, commits:0, blocked:false, quarantined:false, draining:false, closed:false, cleanup_failed:false, close_ranges:None, unmaps:KVec::new(), objects:KVec::new() }, "VM job lifetime"), changed <- new_condvar!("VM mapping admission"), @@ -580,7 +569,6 @@ impl Drop for VmJobGuard { if state.active != 0 { return; } - state.quiet_since = now_ns(); state.draining = true; drop(state); self.vm.drain_mappings(lifetime); diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 573c524c3bfad..9ae8c4d4a1b00 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -15,6 +15,10 @@ use kernel::{new_condvar, sync::CondVar}; const RESTORE_BACKOFF_MIN_NS: u64 = 1_000_000_000; const RESTORE_BACKOFF_MAX_NS: u64 = 30_000_000_000; const RESTORE_BACKOFF_FACTOR: u64 = 16; +/// A VM whose last job, binding or other residency user ended (or which was +/// created) more recently than this is in use: a submitting or loading client +/// would restore whatever reclaim released, synchronously in its next job. +const RECLAIM_QUIET_NS: u64 = 500_000_000; fn now_ns() -> u64 { ::ktime_get() as u64 @@ -29,10 +33,17 @@ struct State { /// Monotonic time the last restoration finished and its duration. restored_at: u64, restore_cost: u64, + /// Monotonic time the last residency user (job, binding, retirement) + /// released its lease; the gate's creation time before the first one. + last_active: u64, } impl State { - fn in_restore_backoff(&self, now: u64) -> bool { + /// Reclaim skips a VM that is in use, was recently used or recently restored. + fn reclaim_deferred(&self, now: u64) -> bool { + if now.saturating_sub(self.last_active) < RECLAIM_QUIET_NS { + return true; + } if self.restored_at == 0 { return false; } @@ -65,7 +76,10 @@ impl Gate { pub(crate) fn new() -> Result> { Arc::pin_init( pin_init!(Self { - state <- new_mutex!(State::default(), "Asahi residency"), + state <- new_mutex!( + State { last_active: now_ns(), ..State::default() }, + "Asahi residency" + ), ready <- new_condvar!("Asahi residency ready"), }), GFP_KERNEL, @@ -154,7 +168,7 @@ impl Gate { /// The busy gate continues to exclude restoration and metadata changes. pub(crate) fn try_reclaim(gate: &Arc) -> Option { let mut state = gate.state.try_lock()?; - if state.users != 0 || state.busy || state.closed || state.in_restore_backoff(now_ns()) { + if state.users != 0 || state.busy || state.closed || state.reclaim_deferred(now_ns()) { return None; } state.busy = true; @@ -177,7 +191,7 @@ impl Gate { state.users == 0 && !state.busy && !state.closed - && !state.in_restore_backoff(now_ns()) + && !state.reclaim_deferred(now_ns()) }) } @@ -210,6 +224,7 @@ impl Drop for Lease { let mut state = self.gate.state.lock(); debug_assert!(state.users != 0); state.users -= 1; + state.last_active = now_ns(); if state.users == 0 { drop(state); self.gate.ready.notify_all(); From 0960a07c4d1807b2faa8c535d14f68d57213c8cb Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 18:33:06 -0400 Subject: [PATCH 0837/2680] drm/asahi: set the idle-VM reclaim quiet window to 2000 ms Idle-VM reclaim now waits 2000 ms after the VM's last job, binding or retirement before releasing its backing (500 ms before). In five Subnautica launch captures, the restore stalls that still followed a real idle gap came after 518 ms (31 ms stall) and 1270 ms (44 ms stall); most longer submit gaps had bindings within 5 ms and are already covered by counting bindings as activity. A truly idle client is still reclaimed, 1.5 s later. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit dec3bc9eb622df05eca30716834c35506ee31f9f) --- drivers/gpu/drm/asahi/mmu/residency.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 9ae8c4d4a1b00..90e59760e84fb 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -18,7 +18,7 @@ const RESTORE_BACKOFF_FACTOR: u64 = 16; /// A VM whose last job, binding or other residency user ended (or which was /// created) more recently than this is in use: a submitting or loading client /// would restore whatever reclaim released, synchronously in its next job. -const RECLAIM_QUIET_NS: u64 = 500_000_000; +const RECLAIM_QUIET_NS: u64 = 2000000000; fn now_ns() -> u64 { ::ktime_get() as u64 From dd1a691973f77ecfdbb45f2497e171f77608c4db Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 16:19:25 -0400 Subject: [PATCH 0838/2680] drm/asahi: release the VM status mirror object when the VM handle closes The VM status is shared with every accepted job and with each retained compute or render graph that last belonged to the VM, so it outlives the client. The client's status-mirror object (GET_PARAMS VM_STATUS_MIRROR) was retained for the status's whole lifetime, keeping a user object alive after the client exited for as long as a retained graph referenced the context. Keep the mirror word and its object under a mutex shared with the failure reporter, and drop both when the file's VM handle is destroyed; later reports find no mirror and only record the error. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 1c1abce16316282e15f5e471b8ef0b5a33acc038) --- drivers/gpu/drm/asahi/file.rs | 5 ++ drivers/gpu/drm/asahi/g17/status.rs | 82 +++++++++++++++-------------- drivers/gpu/drm/asahi/mmu.rs | 2 +- 3 files changed, 48 insertions(+), 41 deletions(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 6c0c094d71ca3..3cb5f60a4873a 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -75,6 +75,11 @@ impl Drop for Vm { fn drop(&mut self) { self.shrinker = None; self.vm.close_idle_reclaim(); + if let Some(status) = self.vm.status() { + // Retained graphs may keep this status alive; the client's mirror + // object must not outlive the client's VM handle. + status.release_mirror(); + } if self.vm.status().is_some() { // Tracked jobs can outlive their file's VM handle. Their last guard performs // these unmaps once no accepted work can still dereference the mappings. diff --git a/drivers/gpu/drm/asahi/g17/status.rs b/drivers/gpu/drm/asahi/g17/status.rs index 2ad06958c1359..c288b74bfcaeb 100644 --- a/drivers/gpu/drm/asahi/g17/status.rs +++ b/drivers/gpu/drm/asahi/g17/status.rs @@ -3,40 +3,40 @@ //! VM admission errors and client failure reporting. No firmware state is accessed here. use crate::gem; -use core::cell::UnsafeCell; -use core::sync::atomic::{AtomicBool, AtomicI32, AtomicPtr, Ordering}; -use kernel::prelude::*; +use core::sync::atomic::{AtomicI32, Ordering}; +use kernel::{new_mutex, prelude::*, sync::Mutex}; const MIRROR_MAX_SIZE: usize = 64 * 1024; /// A VM and its accepted jobs share this status, including after queue destruction. /// Client failure is separate from hardware admission: recovery may preserve the /// VM while failed accepted work must still be reported before signalling fences. +#[pin_data] pub(crate) struct VmStatus { error: AtomicI32, reported_error: AtomicI32, - mirror: AtomicPtr, - mirror_claimed: AtomicBool, - mirror_owner: UnsafeCell>, + /// The client's mapped status word, retained with its object only while the + /// client's VM handle exists; released by `release_mirror` at VM close. + #[pin] + mirror: Mutex>, } -// SAFETY: Only the winner of mirror_claimed writes mirror_owner. The owner is never -// read or replaced while the status lives, and its mapping is retained until drop. -// Other shared accesses use atomics, including accesses through the mirror pointer. -unsafe impl Sync for VmStatus {} -// SAFETY: The retained GEM reference and mapping can move between threads. Moving -// the status does not move that mapping, and its remaining fields are atomic. -unsafe impl Send for VmStatus {} +/// One aligned word inside the retained, vmapped mirror object. +struct Mirror { + word: *mut i32, + _owner: gem::ObjectRef, +} +// SAFETY: The word points into the mapping the retained owner keeps alive; it is +// only accessed atomically through this pointer while the mutex is held. +unsafe impl Send for Mirror {} impl VmStatus { - pub(crate) fn new() -> Self { - Self { + pub(crate) fn new() -> impl PinInit { + pin_init!(Self { error: AtomicI32::new(0), reported_error: AtomicI32::new(0), - mirror: AtomicPtr::new(core::ptr::null_mut()), - mirror_claimed: AtomicBool::new(false), - mirror_owner: UnsafeCell::new(None), - } + mirror <- new_mutex!(None, "Asahi VM status mirror"), + }) } pub(crate) fn get(&self) -> i32 { @@ -71,11 +71,11 @@ impl VmStatus { Ok(_) => error.to_errno(), Err(first) => first, }; - let word = self.mirror.load(Ordering::SeqCst); - if !word.is_null() { - // SAFETY: attach_mirror publishes only an aligned word within a mapping - // owned by this status. Every kernel access to this word is atomic. - unsafe { AtomicI32::from_ptr(word) }.store(error, Ordering::SeqCst); + let mirror = self.mirror.lock(); + if let Some(mirror) = mirror.as_ref() { + // SAFETY: attach_mirror stored only an aligned word within a mapping the + // retained owner keeps alive while this entry exists under the mutex. + unsafe { AtomicI32::from_ptr(mirror.word) }.store(error, Ordering::SeqCst); } } @@ -86,7 +86,7 @@ impl VmStatus { } } - /// Retain one mapped GEM word until the last reference to this status is dropped. + /// Retain one mapped GEM word until the client's VM handle is dropped. pub(crate) fn attach_mirror(&self, mut owner: gem::ObjectRef, offset: usize) -> Result { if owner.size() > MIRROR_MAX_SIZE || offset % core::mem::align_of::() != 0 @@ -99,24 +99,26 @@ impl VmStatus { let base = owner.vmap()?.addr.cast::(); // SAFETY: The checked range is inside the retained, page-aligned mapping. let word = unsafe { base.add(offset) }.cast::(); - if self.mirror_claimed.swap(true, Ordering::AcqRel) { + let mut mirror = self.mirror.lock(); + if mirror.is_some() { return Err(EBUSY); } - // SAFETY: This is the only thread that may write mirror_owner. It is not - // read again until the status is dropped, after all shared references end. - unsafe { *self.mirror_owner.get() = Some(owner) }; - // SAFETY: The retained owner keeps this checked, aligned word mapped. - unsafe { AtomicI32::from_ptr(word) }.store(0, Ordering::SeqCst); - - // The pointer publication and reported error form a store/load handshake: - // either this load sees the error or the reporter sees our pointer. Stores - // after publishing the pointer write only the same first error, never zero. - self.mirror.store(word, Ordering::SeqCst); + // No reporter runs while the mutex is held, so the word holds the first + // reported error (or zero) when it becomes visible. let error = self.reported_error.load(Ordering::SeqCst); - if error != 0 { - // SAFETY: The mapping remains owned by this status as above. - unsafe { AtomicI32::from_ptr(word) }.store(error, Ordering::SeqCst); - } + // SAFETY: The retained owner keeps this checked, aligned word mapped. + unsafe { AtomicI32::from_ptr(word) }.store(error, Ordering::SeqCst); + *mirror = Some(Mirror { + word, + _owner: owner, + }); Ok(()) } + + /// The client's VM handle is gone and nobody reads the mirror any more. The + /// mapping and object reference drop outside the mutex (GEM release may sleep). + pub(crate) fn release_mirror(&self) { + let taken = self.mirror.lock().take(); + drop(taken); + } } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index d404866f80fbb..79fda65b3e2b2 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1699,7 +1699,7 @@ impl Vm { /// Attach a status before sharing a newly created user VM with queues. pub(crate) fn with_status(mut self) -> Result { - self.status = Some(Arc::new(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); + self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); self.lifetime = Some(lifetime::VmLifetime::new()?); self.residency = Some(residency::Gate::new()?); self.context_bindings = Some(context::ContextBindings::new()?); From 26d925374976e98a620df48443d9efe0792e5cce Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 20:33:37 -0400 Subject: [PATCH 0839/2680] drm/asahi: unmap detached compute owners and free vacated pools off-lock Idle retained compute queues detach their exited owner's binding and, once the firmware consumed a free-list release, drop the vacated USC backing. Both happened inside the event worker's device-mutex critical section: the binding drop unmaps seven aliases from the exited VM and releases the last reference to that context and its VM, and the vacated backing frees a 25 MiB object. On the ship stack kbench's cold phase showed the first SUBMIT of 1-4 of 48 fresh contexts at 433-741 us (p99 657-701 us) against a 170-260 us body that is faster than the previous release's 224 us median. The slow samples are spread over the run and the second submission is 15 us, which is consistent with a submission waiting for the event worker's critical section while it performs this teardown; the previous release did the same unmapping inside the new owner's own submission, so no other client could observe it. Hand the detached binding and the vacated backing to the worker's deferred batch, which already finishes completion fences after the device mutex is released. The batch reserves one slot per physical queue for each; without room the value is dropped in place as before. No behaviour changes for the firmware: the release witness, the retained scheduler pages and the re-arm path are unchanged. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 6af79e6e57ef80d2b93d4c428956ec6cdfb11a3a) --- drivers/gpu/drm/asahi/g17/freelist.rs | 7 +-- drivers/gpu/drm/asahi/g17/queue/compute.rs | 51 +++++++++++--------- drivers/gpu/drm/asahi/g17/runtime.rs | 52 +++++++++++++++++---- drivers/gpu/drm/asahi/g17/runtime/events.rs | 2 +- 4 files changed, 79 insertions(+), 33 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index e40dfec7fc5fd..a4d58e71eeccf 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -393,9 +393,10 @@ impl ComputePool { pub(crate) fn release_record(&self, generation: u64) -> super::fw::channels::FreeListRelease { super::fw::channels::FreeListRelease::new(generation, u32::from(self.id)) } - /// The caller has observed the firmware consume this pool's release. - pub(crate) fn drop_pages(&mut self) { - self.pages = None; + /// The caller has observed the firmware consume this pool's release and + /// frees the returned backing after releasing the device mutex. + pub(crate) fn take_pages(&mut self) -> Option { + self.pages.take() } /// Rewrites the shared control header for fresh backing. The caller then /// rewrites the page-pool descriptor row before the queue is kicked again. diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index df38a727c8d60..2cdeb82fbac38 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -51,7 +51,13 @@ static_assert!(core::mem::offset_of!(GraphLayout, items) == ITEMS); /// Aliases are destroyed before the logical root they name. Superseded bindings remain owned /// until the first exact successful publication under their replacement has retired. -struct Binding { +/// Backing of a vacated pool, freed outside the device mutex. +pub(crate) struct Vacated { + _alias: Option, + _pages: Option, +} + +pub(crate) struct Binding { _kick_alias: mmu::KernelMapping, /// Client alias of the USC pool pages; dropped when the pool is vacated and /// recreated when an owner binds a re-armed pool (it pins the backing). @@ -1046,19 +1052,19 @@ impl Queue { /// (`Context::release_execution`), and this queue is idle without an owner. /// Drop the binding (every alias into that VM) and the context, keeping only /// the scheduler page and work storage the installed queue record names. - pub(crate) fn detach_exited_owner(&mut self) -> bool { + /// The returned binding (aliases into the exited VM and the last reference + /// to its context) is dropped by the caller after the device mutex is + /// released: unmapping and VM teardown must not delay other clients. + pub(crate) fn detach_exited_owner(&mut self) -> Option { if !self.reusable() { - return false; + return None; } - let Some(binding) = self.binding.as_ref() else { - return false; - }; - if binding.context.is_current() { - return false; + if self.binding.as_ref()?.context.is_current() { + return None; } + let binding = self.binding.take()?; self.retained_owner = Some(binding.context.retain_firmware_pages()); - self.binding = None; - true + Some(binding) } /// An idle retained queue whose USC backing may be given back. pub(crate) fn vacate_candidate(&self) -> bool { @@ -1084,23 +1090,26 @@ impl Queue { fence(Ordering::SeqCst); Ok(()) } - /// Drops the USC backing once the firmware's control consumer passed the - /// release. Returns true when the backing was freed by this call. - pub(crate) fn observe_vacancy(&mut self, consumer: u32, producer: u32) -> Result { + /// Detaches the USC backing once the firmware's control consumer passed the + /// release. The caller frees the returned backing after releasing the + /// device mutex. + pub(crate) fn observe_vacancy( + &mut self, + consumer: u32, + producer: u32, + ) -> Result> { let PoolPhase::ReleaseSent(cursor) = self.pool_phase else { - return Ok(false); + return Ok(None); }; if !crate::g17::freelist::control_consumed(consumer, producer, cursor)? { - return Ok(false); + return Ok(None); } - // The retained owner binding's client alias pins the backing: drop it + // The retained owner binding's client alias pins the backing: detach it // first (the owner has exited; `reusable()` proved no previous binding). - if let Some(binding) = self.binding.as_mut() { - binding.pool_alias = None; - } - self.pool.drop_pages(); + let alias = self.binding.as_mut().and_then(|binding| binding.pool_alias.take()); + let pages = self.pool.take_pages(); self.pool_phase = PoolPhase::Vacant; - Ok(true) + Ok(Some(Vacated { _alias: alias, _pages: pages })) } /// Installs fresh backing into a vacant pool; returns the descriptor row data /// the caller writes before binding an owner. diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 204a3f97b3c52..2a7c43762e6b3 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -223,7 +223,11 @@ impl Registry { } /// Frees the backing of every pool whose release the firmware consumed. - pub(super) fn observe_pool_releases(&mut self, init: &super::initdata::InitData) -> Result { + pub(super) fn observe_pool_releases( + &mut self, + init: &super::initdata::InitData, + deferred: &mut DeferredBatch, + ) -> Result { fence(Ordering::Acquire); let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); @@ -234,7 +238,8 @@ impl Registry { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - if queue.observe_vacancy(consumer, producer)? { + if let Some(vacated) = queue.observe_vacancy(consumer, producer)? { + deferred.defer_vacated(vacated); freed += 1; } } @@ -632,6 +637,10 @@ impl Backend { /// Collection failure leaves the engine's Active owner intact for a stopped retry. pub(super) struct DeferredBatch { values: KVVec, + /// Exited owners detached from idle retained compute queues and the backing + /// of vacated pools: unmapping and freeing happen in `finish`, off-lock. + bindings: KVVec, + vacated: KVVec, } impl DeferredBatch { const DEPTH: usize = if crate::hw::t8140::queues::RENDER_DEPTH as usize @@ -644,18 +653,32 @@ impl DeferredBatch { /// Reused by the sole event work item; no allocation on a completion pass. fn worker() -> Result { - Self::with_capacity(2 * QIDS * Self::DEPTH) + Self::with_capacity(2 * QIDS * Self::DEPTH, QIDS) } /// One publication can quarantine only its selected physical queue/pair. /// VM-wide settlement is a separate transaction before publishing its error. fn publication() -> Result { - Self::with_capacity(2 * Self::DEPTH) + Self::with_capacity(2 * Self::DEPTH, 0) } - fn with_capacity(capacity: usize) -> Result { + fn with_capacity(capacity: usize, detached: usize) -> Result { Ok(Self { values: KVVec::with_capacity(capacity, GFP_KERNEL)?, + bindings: KVVec::with_capacity(detached, GFP_KERNEL)?, + vacated: KVVec::with_capacity(detached, GFP_KERNEL)?, }) } + /// Keeps the detached owner's unmapping and VM teardown out of the device + /// mutex. Without reserved room the binding is dropped here instead. + pub(super) fn defer_binding(&mut self, binding: super::queue::compute::Binding) { + if let Err(binding) = self.bindings.push_within_capacity(binding) { + drop(binding); + } + } + pub(super) fn defer_vacated(&mut self, vacated: super::queue::compute::Vacated) { + if let Err(vacated) = self.vacated.push_within_capacity(vacated) { + drop(vacated); + } + } /// A recovery pass can fail and retire each retained command at most once. /// Reserve before changing queues, plus a full publication for its caller. /// Healthy submissions keep the small publication collector unchanged. @@ -693,6 +716,12 @@ impl DeferredBatch { for value in self.values.drain_all() { value.finish(); } + for binding in self.bindings.drain_all() { + drop(binding); + } + for vacated in self.vacated.drain_all() { + drop(vacated); + } } } @@ -1080,7 +1109,12 @@ const COMPUTE_WARM_CAP: usize = 4; impl super::Firmware { /// Event-worker service step: witness consumed pool releases, then release the /// oldest idle retained pools beyond the warm cap. - pub(in crate::g17) fn service_compute_pools(&mut self) -> Result { + /// Detached owner bindings and vacated pool backing go to `deferred`, which + /// the worker finishes after releasing the device mutex. + pub(in crate::g17) fn service_compute_pools( + &mut self, + deferred: &mut DeferredBatch, + ) -> Result { for queue in self .queues .compute @@ -1088,9 +1122,11 @@ impl super::Firmware { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - queue.detach_exited_owner(); + if let Some(binding) = queue.detach_exited_owner() { + deferred.defer_binding(binding); + } } - self.queues.observe_pool_releases(&self.init)?; + self.queues.observe_pool_releases(&self.init, deferred)?; if self.render_control_backpressured() { return Ok(()); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index f92c493f4813d..4a18aaa05419d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -386,7 +386,7 @@ impl crate::g17::Shared { let idle = firmware.idle_check(now, in_flight)?; firmware.reclaim_drained_renders(deferred)?; firmware.service_compute_exit(deferred)?; - firmware.service_compute_pools()?; + firmware.service_compute_pools(deferred)?; { let mut reclaim = self.reclaim.lock(); firmware.service_teardown(&mut reclaim)?; From fb16631bf54a8e22cd52f503c025d69a68435a00 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 02:40:32 -0400 Subject: [PATCH 0840/2680] drm/asahi: retire a compute command on its record when its stamp never follows A G17 compute command retires once its queue's completion stamp covers its kick and the firmware has consumed its items. On J700 (A18 Pro), a client that keeps render and compute work of one VM in flight together loses its device within seconds: the command's completion record arrives with its start and end timestamps and its items are consumed, but the firmware never stores its completion stamp, so the compute watchdog fails the VM. The same client runs clean on an earlier driver for this GPU, which retires such a command on its completion record and waits for the stamp only a bounded time (500 us); that driver observed that the firmware leaves a minority of completion stamps unpublished. Do the same: once a command's exact completion record (non-zero duration) has been present for 1 ms and its items are consumed, retire it even if its stamp has not covered it. The firmware stores the stamp after the cache maintenance that follows the record, so the bounded wait still lets a stamp that is merely late keep that ordering. Count the commands that retire without a stamp and log the first and every 64th, so how often it happens stays visible. With this change that client ran 575,882 verified checks in 60 s; on the same boot without it, the device was lost within 2 s. Why the firmware leaves these stamps unstored is not yet understood. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 0b4bd1537cc71a1796b976c05cdab24e76549303) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 16 +++++++++- .../drm/asahi/g17/queue/compute/retirement.rs | 30 +++++++++++++++++-- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 2cdeb82fbac38..f956e5726a8d6 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -325,6 +325,8 @@ struct Active { work: Arc, ticket: Option, observed: Option, + /// Host time the completion record was first observed. + observed_at: Option, failure_deferred: bool, qos_pending: bool, } @@ -589,6 +591,9 @@ impl Queue { }; if active.observed.is_none() { active.observed = batch.take(&ticket)?; + if active.observed.is_some() { + active.observed_at = Some(now_ns()); + } } } Ok(()) @@ -699,15 +704,23 @@ impl Queue { .iter() .skip(1) .find_map(|a| a.observed.map(|o| o.timestamps[1])); + let stamp = self.graph.stamp()?; + let record_aged = front + .observed_at + .is_some_and(|at| now_ns().saturating_sub(at) >= retirement::STAMP_GRACE_NS); let timestamps = retirement::poll( &ticket, front.observed, self.graph.cursors()?, - self.graph.stamp()?, + stamp, recordless, self.last_end, next_end, + record_aged, )?; + if timestamps.is_some() && !retirement::stamp_covers(stamp, ticket.kick) { + retirement::note_stampless(self.qid(), ticket.kick); + } if timestamps.is_some() && self.previous.as_ref().is_some_and(|p| { p.publication != Some((front.packet.order.sequence, ticket.epoch)) @@ -1488,6 +1501,7 @@ impl Queue { work: work.clone(), ticket: None, observed: None, + observed_at: None, failure_deferred: false, qos_pending: false, }) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index 766177ebc32c9..b4d4bcef24421 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -65,8 +65,32 @@ pub(crate) enum Recordless { CrossedRecovery, } +/// How long a command whose exact completion record is present waits for its completion stamp +/// before it retires on the record alone. The firmware stores the stamp after the cache +/// maintenance that follows the record; it occasionally never stores it for a command, most +/// visibly while a render of the same VM is running. +pub(crate) const STAMP_GRACE_NS: u64 = 1_000_000; + +static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); + +/// Counts a command that retired on its record without its completion stamp; logs the first and +/// every 64th. +pub(crate) fn note_stampless(qid: u8, kick: u64) { + let count = STAMPLESS.fetch_add(1, core::sync::atomic::Ordering::Relaxed) + 1; + if count == 1 || count % 64 == 0 { + pr_info!( + "G17: {} compute commands retired on their completion record without a completion stamp (latest: queue {} kick {:#x})\n", + count, + qid, + kick + ); + } +} + /// A zero-duration record fails before visibility polling. It proves execution did not occur; /// ownership still requires a separate retirement witness before mappings can be released. +/// The command is complete when its stamp covers it, or when its exact completion record has +/// been present for [`STAMP_GRACE_NS`] (`record_aged`); in both cases its items must be consumed. pub(crate) fn poll( ticket: &Ticket, observed: Option, @@ -75,13 +99,13 @@ pub(crate) fn poll( recordless: Recordless, floor: u64, next_end: Option, + record_aged: bool, ) -> Result> { if observed.is_some_and(|record| record.timestamps[0] == record.timestamps[1]) { return Err(ENODATA); } - if !stamp_covers(stamp, ticket.kick) - || !cursors.consumed(ticket.item_producer, ticket.item_count)? - { + let completed = stamp_covers(stamp, ticket.kick) || (observed.is_some() && record_aged); + if !completed || !cursors.consumed(ticket.item_producer, ticket.item_count)? { return Ok(None); } Ok(Some(match observed { From 87bc6cc243b7c0771e849b3c8655546b4bdf3bcc Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 00:18:28 -0400 Subject: [PATCH 0841/2680] drm/asahi: log the first GPU work failure reported to each G17 VM A G17 client treats any failure reported for its work (the VM status word or its mapped mirror) as a lost device, but no path that reports one writes to the kernel log: watchdog timeouts, quarantines, cancelled or failed retirements and recovery classification all fail the client silently. On Neo a client lost its device repeatedly with no kernel line at all, so the failing path could not be identified from the log. Log one warning when the first failure is reported to a VM, with the error and the reporting site. Later failures of the same VM are not logged, so the volume is bounded by the number of failing clients. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit f647d69cc5043bf1d3389c8e250e78935e5afa24) --- drivers/gpu/drm/asahi/g17/status.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/status.rs b/drivers/gpu/drm/asahi/g17/status.rs index c288b74bfcaeb..1705af7554e76 100644 --- a/drivers/gpu/drm/asahi/g17/status.rs +++ b/drivers/gpu/drm/asahi/g17/status.rs @@ -44,6 +44,7 @@ impl VmStatus { } /// Record permanent admission failure and report it to the client. + #[track_caller] pub(crate) fn record(&self, error: Error) { self.report_failure(error); let _ = self.error.compare_exchange( @@ -61,6 +62,10 @@ impl VmStatus { /// Report failed accepted work without preventing recovery of the VM. /// Every reporter publishes the first error before returning: a losing /// reporter may signal its fence before the winning reporter resumes. + /// + /// The first failure reported to a VM is logged once, with the reporting site: the client treats + /// any reported failure as a lost device, and nothing else in the kernel log names it. + #[track_caller] pub(crate) fn report_failure(&self, error: Error) { let error = match self.reported_error.compare_exchange( 0, @@ -68,7 +73,16 @@ impl VmStatus { Ordering::SeqCst, Ordering::SeqCst, ) { - Ok(_) => error.to_errno(), + Ok(_) => { + let at = core::panic::Location::caller(); + pr_warn!( + "GPU work of a client failed ({:?}, reported at {}:{}); the client sees a lost device\n", + error, + at.file().rsplit("asahi/").next().unwrap_or(at.file()), + at.line() + ); + error.to_errno() + } Err(first) => first, }; let mirror = self.mirror.lock(); @@ -80,6 +94,7 @@ impl VmStatus { } /// Rejected submissions and contention leave the VM usable. + #[track_caller] pub(crate) fn record_if_device_loss(&self, error: Error) { if error == EIO || error == ETIMEDOUT || error == ENODEV { self.record(error); From 42ccf0741c0585a2b11fc0481304db9546fc5099 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 00:27:46 -0400 Subject: [PATCH 0842/2680] drm/asahi: name the completion path in the first-failure log line The first-failure warning names the code that reported the failure. For most failures that is one of three completion helpers (failure before publication, failure while the queue owns the command, failed retirement), which tells little. Let the reporting site propagate through those helpers, so the line names their caller: for example the publication path for a command that failed before it reached the firmware, which is where a failed render pool re-allocation (ENOMEM) is reported. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c8ea00a9ed621e2e5c7c84461091fda3a9b7cb7b) --- drivers/gpu/drm/asahi/g17/completion.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 6ab9bc24ffd1d..a3888f98231db 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -433,6 +433,7 @@ impl Completion { /// Exact retirement permits the physical lease and VM pin to be released /// before waking userspace. A failure fence alone never establishes this. + #[track_caller] pub(crate) fn complete(&self, result: Result<[u64; 4]>) { if self.signalled.swap(true, Ordering::AcqRel) { return; @@ -453,6 +454,7 @@ impl Completion { } /// A rejected, never-published command cannot poison the VM for contention. + #[track_caller] pub(crate) fn fail_unpublished(&self, error: Error) { if self.signalled.swap(true, Ordering::AcqRel) { return; @@ -468,6 +470,7 @@ impl Completion { /// The physical queue must already be quarantined. Keep mappings pinned while /// waking waiters, even if queue destruction requested cancellation. + #[track_caller] pub(crate) fn fail_while_owned(&self, error: Error) { if self.signalled.swap(true, Ordering::AcqRel) { return; From a2a62868bfdbcb52767b2113a946d3e9923a6b1f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:12:31 -0400 Subject: [PATCH 0843/2680] drm/asahi: describe the G17 word tables without their generation history The comment on the constant word tables recorded how they were produced and named an out-of-tree workspace. State what the tables are instead. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit d0aadc1e99e6f11a54bbca5ee685d7c015866789) --- drivers/gpu/drm/asahi/g17/fw/initdata.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index 8dbb6a81bcd47..94b03ebe284db 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -1533,8 +1533,7 @@ const TEMP_KP: u32 = 0x41a2_b852; const TEMP_KI: u32 = 0x4116_0419; /// Nonzero 32-bit words of a firmware object the host builds from constants: (byte offset, value, repeat). -/// Every other word is zero. Generated from the previously field-by-field builders; the fwdump harness -/// (`neo-smooth-20261007/kernel/fwdump`) verifies the objects byte for byte. +/// Every other word is zero. The table is the sole definition of the object's constant contents. type Words = &'static [(usize, u32, usize)]; fn fill_words(bytes: &mut [u8], words: Words) { From 64ba07aebeffa683dc98c2f00ced5e732d26ec75 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:13:52 -0400 Subject: [PATCH 0844/2680] drm/asahi: use one monotonic clock helper across the G17 runtime The runtime registry and the retained compute queues each defined an identical now_ns() over the monotonic clock source. Define it once in the G17 module; the runtime's submodules keep their existing path through a re-export. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5d2a444327afc38543beb78a252e23661e4d4ecc) --- drivers/gpu/drm/asahi/g17/mod.rs | 6 ++++++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 4 +--- drivers/gpu/drm/asahi/g17/runtime.rs | 5 +---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index eb26694bf2adf..4b8372552686f 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -108,6 +108,12 @@ use object::{ }; /// Expected identification of a G17 GPU. +/// One monotonic domain for recovery grace, idle effort, pool retention and +/// control-ring retirement. +pub(crate) fn now_ns() -> u64 { + ::ktime_get() as u64 +} + pub(crate) struct Identity { /// GPU family field of the ID register. pub(crate) family: u8, diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index f956e5726a8d6..0cb231f7050e4 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -17,9 +17,7 @@ use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; use retirement::{Cursors, RECORD_SLOTS}; -fn now_ns() -> u64 { - ::ktime_get() as u64 -} +use crate::g17::now_ns; const DESCRIPTORS: usize = 256; const GRAPH_SIZE: usize = 0x8000; diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 2a7c43762e6b3..6cefa9509de97 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -17,6 +17,7 @@ mod feed; mod render; pub(super) mod teardown; +pub(super) use super::now_ns; use super::{ buffer::{BufferIds, MetricsIds}, context::{Context, QosIds}, @@ -1139,7 +1140,3 @@ impl super::Firmware { } } -/// One monotonic domain for recovery grace, idle effort and control retirement. -fn now_ns() -> u64 { - ::ktime_get() as u64 -} From f79b392067cf2849e9b0e660916754d5bfa7cc5c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:13:52 -0400 Subject: [PATCH 0845/2680] drm/asahi: grow the scheduler-release table silently Growing the table of contexts awaiting a scheduler release past its preallocated 128 entries is ordinary behaviour under a burst of closes; the one-shot log line that reported it informed nobody and the table still grows the same way. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 6850c91310a913b5ff9646261250e62b129f4ce8) --- drivers/gpu/drm/asahi/g17/teardown.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index 5d56931879f41..1a25cd7e3fa3b 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -5,12 +5,11 @@ //! The device mutex serializes this state with the Registry's displaced graph ownership. use super::{context::Context, freelist::control_consumed, fw::channels::SchedulerStateRelease}; -use core::sync::atomic::{fence, AtomicBool, Ordering}; +use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; /// Entries preallocated at device start; a burst of closes beyond it grows the table. const CAPACITY: usize = 128; -static GREW: AtomicBool = AtomicBool::new(false); const POLL_NS: u64 = 2_000_000_000; #[derive(Clone, Copy, PartialEq, Eq)] @@ -91,12 +90,6 @@ impl Pending { armed: now, announce_pending: false, }; - if self.entries.len() >= CAPACITY && !GREW.swap(true, Ordering::Relaxed) { - pr_info!( - "asahi: G17 scheduler-release table grew past {} pending contexts\n", - CAPACITY - ); - } self.entries.push(entry, GFP_KERNEL)?; Ok(()) } From 07cad616ec1fe906967ebbda2f25916d1f2cde88 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 16:15:36 -0400 Subject: [PATCH 0846/2680] drm/asahi: give back the descriptor ring of vacated compute queues A vacated compute queue (USC pool given back) still owns its 4 MiB descriptor ring. Once the queue is idle without an owner, the firmware's job list and context item only name consumed descriptors, so drop the ring and its aliases together with the pool backing, and allocate a fresh ring when an owner re-arms the queue, before the bind that aliases it into the client VM. Submissions to a vacant queue report EAGAIN until the re-arm, as for the pool. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e60f49eede0ecf38cb206c4f59ff832b2aacc1d6) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 112 ++++++++++++++++----- drivers/gpu/drm/asahi/g17/runtime.rs | 6 +- 2 files changed, 90 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 0cb231f7050e4..3db3be2211eb7 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -49,10 +49,13 @@ static_assert!(core::mem::offset_of!(GraphLayout, items) == ITEMS); /// Aliases are destroyed before the logical root they name. Superseded bindings remain owned /// until the first exact successful publication under their replacement has retired. -/// Backing of a vacated pool, freed outside the device mutex. +/// Backing of a vacated pool and descriptor ring, freed outside the device +/// mutex. Field order drops each client alias before the storage it names. pub(crate) struct Vacated { _alias: Option, _pages: Option, + _descriptors: Option, + _ring: Option, } pub(crate) struct Binding { @@ -60,7 +63,9 @@ pub(crate) struct Binding { /// Client alias of the USC pool pages; dropped when the pool is vacated and /// recreated when an owner binds a re-armed pool (it pins the backing). pool_alias: Option, - _descriptors: mmu::KernelMapping, + /// Client alias of the descriptor ring; dropped with the ring when the queue + /// is vacated and recreated when an owner binds a re-armed queue. + descriptors: Option, preempt: mmu::KernelMapping, usage: mmu::KernelMapping, operand: mmu::KernelMapping, @@ -68,10 +73,34 @@ pub(crate) struct Binding { context: Arc, } +/// Descriptor ring of one compute QID with its kernel-VM alias. Built without +/// the device mutex; installed into a vacant graph before an owner binds. +pub(crate) struct DescriptorRing { + object: KernelObject, + high: mmu::KernelMapping, +} + +impl DescriptorRing { + pub(crate) fn new(alloc: &Allocator<'_>) -> Result { + let page = mmu::UAT_PGSZ as u64; + let prot = mmu::PROT_GPU_FW_SHARED_RW; + let object = alloc.lower( + DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, + page, + prot, + CpuMap::WriteCombined, + )?; + let mut high = alloc.uat.geometry().kernel_range(); + high.start += dynamic::KERNEL_OFFSET; + let high = object.alias_in(alloc.uat.kernel_vm(), high, page, prot)?; + Ok(Self { object, high }) + } +} + /// Firmware-shared storage of one compute QID, separate from its common kick ring. +/// `ring` is `None` while the queue is vacant (idle, no owner, pool given back). struct Graph { - descriptor_high: mmu::KernelMapping, - descriptors: KernelObject, + ring: Option, preempt: KernelObject, usage: KernelObject, operand: KernelObject, @@ -89,10 +118,7 @@ impl Graph { let page = mmu::UAT_PGSZ as u64; let prot = mmu::PROT_GPU_FW_SHARED_RW; let cpu = CpuMap::WriteCombined; - let descriptors = alloc.lower(DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, page, prot, cpu)?; - let mut high = alloc.uat.geometry().kernel_range(); - high.start += dynamic::KERNEL_OFFSET; - let descriptor_high = descriptors.alias_in(alloc.uat.kernel_vm(), high, page, prot)?; + let ring = DescriptorRing::new(alloc)?; let preempt = alloc.kernel(0x4000, page, prot, cpu)?; let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; let operand = alloc.lower(0x14000, page, prot, cpu)?; @@ -120,8 +146,7 @@ impl Graph { let context_high = alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?; let records = alloc.kernel(RECORD_SLOTS * RECORD_STRIDE, page, prot, cpu)?; Ok(Self { - descriptor_high, - descriptors, + ring: Some(ring), preempt, usage, operand, @@ -147,9 +172,7 @@ impl Graph { } let vm = context.vm(); vm.install_job_context_aliases()?; - let descriptors = - self.descriptors - .map_alias(vm, self.descriptors.gpu_va(), mmu::PROT_GPU_SHARED_RO)?; + let descriptors = self.descriptor_alias(vm)?; let prot = mmu::PROT_GPU_FW_SHARED_RW; let page = mmu::UAT_PGSZ as u64; let preempt = self @@ -165,7 +188,7 @@ impl Graph { Ok(Binding { _kick_alias: kick_alias, pool_alias: Some(pool_alias), - _descriptors: descriptors, + descriptors: Some(descriptors), preempt, usage, operand, @@ -203,12 +226,29 @@ impl Graph { fence(Ordering::Acquire); Ok(self.support.word(STAMP)?.load(Ordering::Relaxed)) } - fn descriptor_vas(&self, slot: u8) -> (u64, u64) { + /// EAGAIN while the queue is vacant: the owner must re-arm it first. + fn descriptor_vas(&self, slot: u8) -> Result<(u64, u64)> { + let ring = self.ring.as_ref().ok_or(EAGAIN)?; let offset = u64::from(slot) * COMPUTE_DESCRIPTOR_SIZE as u64; - ( - self.descriptor_high.iova() + offset, - self.descriptors.gpu_va() + offset, - ) + Ok((ring.high.iova() + offset, ring.object.gpu_va() + offset)) + } + fn descriptor_alias(&self, vm: &mmu::Vm) -> Result { + let ring = self.ring.as_ref().ok_or(EAGAIN)?; + ring.object + .map_alias(vm, ring.object.gpu_va(), mmu::PROT_GPU_SHARED_RO) + } + /// Detaches the descriptor ring of an idle vacant queue; the caller frees it + /// off-lock. The job list and the context item only name consumed + /// descriptors at this point. + fn vacate_ring(&mut self) -> Option { + self.ring.take() + } + fn rearm_ring(&mut self, ring: DescriptorRing) -> Result { + if self.ring.is_some() { + return Err(EBUSY); + } + self.ring = Some(ring); + Ok(()) } fn ensure_record_slot(&self, slot: usize) -> Result { if let Some(end) = *self.record_ends.get(slot).ok_or(EINVAL)? { @@ -1000,6 +1040,9 @@ impl Queue { // The same VM takes back a re-armed pool: alias the new backing. binding.pool_alias = Some(self.pool.map_client(context.vm())?); } + if binding.descriptors.is_none() { + binding.descriptors = Some(self.graph.descriptor_alias(context.vm())?); + } self.graph.set_owner(&context)?; binding.context = context.clone(); self.previous = Some(Previous { @@ -1119,15 +1162,27 @@ impl Queue { // first (the owner has exited; `reusable()` proved no previous binding). let alias = self.binding.as_mut().and_then(|binding| binding.pool_alias.take()); let pages = self.pool.take_pages(); + let descriptors = self.binding.as_mut().and_then(|binding| binding.descriptors.take()); + let ring = self.graph.vacate_ring(); self.pool_phase = PoolPhase::Vacant; - Ok(Some(Vacated { _alias: alias, _pages: pages })) + Ok(Some(Vacated { + _alias: alias, + _pages: pages, + _descriptors: descriptors, + _ring: ring, + })) } /// Installs fresh backing into a vacant pool; returns the descriptor row data /// the caller writes before binding an owner. - pub(crate) fn rearm(&mut self, pages: crate::g17::freelist::Pages) -> Result<(u16, u64)> { + pub(crate) fn rearm( + &mut self, + pages: crate::g17::freelist::Pages, + ring: DescriptorRing, + ) -> Result<(u16, u64)> { if self.pool_phase != PoolPhase::Vacant || !self.reusable() { return Err(EAGAIN); } + self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; Ok((self.pool.id(), self.pool.page_list_va()?)) @@ -1405,7 +1460,7 @@ impl Queue { let qid = self.qid(); let ordinal = self.ordinal.checked_add(1).ok_or(EOVERFLOW)?; let (descriptor_high, descriptor_low) = - self.graph.descriptor_vas(kick_timestamp.slot() as u8); + self.graph.descriptor_vas(kick_timestamp.slot() as u8)?; let first = !self.installed && self.ordinal == 0; let entry_signal = first || dependencies @@ -1510,10 +1565,15 @@ impl Queue { let result = (|| -> Result { work.add_submitted_kicks(1)?; accounted = true; - self.graph.descriptors.initialize::( - kick_timestamp.slot() * COMPUTE_DESCRIPTOR_SIZE, - |descriptor| descriptor.write(&args), - )?; + self.graph + .ring + .as_mut() + .ok_or(EAGAIN)? + .object + .initialize::( + kick_timestamp.slot() * COMPUTE_DESCRIPTOR_SIZE, + |descriptor| descriptor.write(&args), + )?; if first { self.graph.context_high.write( 0x200, diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 6cefa9509de97..8206df716b6b3 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -153,6 +153,7 @@ impl Registry { &mut self, qid: u8, pages: crate::g17::freelist::Pages, + ring: compute::DescriptorRing, ) -> Result<(u16, u64)> { let queue = self .compute @@ -161,7 +162,7 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.qid() == qid) .ok_or(EAGAIN)?; - queue.rearm(pages) + queue.rearm(pages, ring) } pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { @@ -563,12 +564,13 @@ impl Backend { // Fresh zeroed backing is built without the device mutex; the // queue may have been taken or re-armed by another owner since. let pages = crate::g17::freelist::Pages::compute(&alloc)?; + let ring = compute::DescriptorRing::new(&alloc)?; let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { continue; } - let (slot, page_list) = match firmware.queues.rearm_compute(qid, pages) { + let (slot, page_list) = match firmware.queues.rearm_compute(qid, pages, ring) { Err(error) if error == EAGAIN => continue, result => result?, }; From 18010aceb4f4001b698ae10c92d1d9412e146457 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 18:04:27 -0400 Subject: [PATCH 0847/2680] drm/asahi: return killed compute queues to service with fresh backing A compute queue whose closed owner was killed stayed quarantined with its exit transaction retained for the device's lifetime, so every fault or kill with work in flight consumed one of the finite compute QIDs for good until the device ran out and queue creation failed. The firmware acknowledges each kill (ContextKilled) and the driver settles every killed command before marking the transaction released; use that as the witness: reset the item ring to empty at the firmware's consumer cursor, clear the quarantine and exit state, and treat the queue as a released idle queue that must give its USC pool and descriptor ring back and be re-armed fresh before the next owner binds it. Recycled queues are vacated regardless of the warm cap. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5d630414e13f4e017628430820fff73a86e97de9) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 79 ++++++++++++++++++- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 7 ++ drivers/gpu/drm/asahi/g17/runtime.rs | 16 +++- 3 files changed, 97 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 3db3be2211eb7..57179437f9b40 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -260,6 +260,23 @@ impl Graph { Ok(()) } + /// A killed queue's ring restarts empty at the firmware's consumer cursor: every + /// item slot is cleared and the host producer is set to the consumer. Only + /// valid after the firmware acknowledged the kill and no command is pending. + fn reset_items(&mut self) -> Result { + let cursors = self.cursors()?; + if cursors.count != ITEM_RING_ENTRIES || cursors.consumer >= cursors.count { + return Err(EIO); + } + for index in 0..ITEM_RING_ENTRIES as usize { + self.graph.dword(ITEMS + index * 8)?.store(0, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + self.graph.word(POINTERS + 0x40)?.store(cursors.consumer, Ordering::Relaxed); + self.reclaimed = cursors.consumer; + fence(Ordering::SeqCst); + Ok(()) + } /// Reclaim only consumed item pointers, with individual aligned stores and a release fence. /// This leaves records still named by firmware untouched across arbitrarily many wraps. fn recycle(&mut self, cursors: Cursors) -> Result { @@ -431,6 +448,9 @@ pub(crate) struct Queue { /// the installed queue record still names them until the next owner binds. retained_owner: Option, pool_phase: PoolPhase, + /// A killed queue was recycled: its pool/ring must be vacated and re-armed + /// fresh before any owner binds it (set by `recycle_killed`, cleared by `rearm`). + needs_fresh_backing: bool, /// Monotonic time the last owner released this queue (vacate order). released_at: u64, /// Free-list generation of the last published configuration. @@ -486,6 +506,7 @@ impl Queue { owner: Some(owner), retained_owner: None, pool_phase: PoolPhase::Populated, + needs_fresh_backing: false, released_at: 0, free_list_generation: 0, binding: Some(binding), @@ -1012,6 +1033,8 @@ impl Queue { self.installed // A pool release in flight must be witnessed before any successor. && !matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) + // A recycled killed queue is offered only once its backing is vacant (re-arm path). + && (!self.needs_fresh_backing || self.pool_phase == PoolPhase::Vacant) // An issued kill still names this graph even if its commands retire // before the acknowledgement. It cannot acquire a successor owner. && self.exit.is_none() @@ -1122,7 +1145,58 @@ impl Queue { } /// An idle retained queue whose USC backing may be given back. pub(crate) fn vacate_candidate(&self) -> bool { - self.reusable() && self.pool_phase == PoolPhase::Populated && self.pool.populated() + (self.reusable() || self.recycled_pending()) && self.pool_phase == PoolPhase::Populated && self.pool.populated() + } + /// A recycled killed queue still holding the backing its killed work used. + pub(crate) fn recycled_pending(&self) -> bool { + self.needs_fresh_backing + && self.installed + && self.exit.is_none() + && self.owner.is_none() + && !self.quarantined + && self.active.len == 0 + && self.replays.len == 0 + && self.previous.is_none() + } + /// Recycled queues are vacated regardless of the warm cap. + pub(crate) fn needs_fresh_backing(&self) -> bool { + self.needs_fresh_backing + } + /// The firmware acknowledged the kill of this queue's closed owner (`ContextKilled`) + /// and every killed command was settled: return the QID to service. The item + /// ring is reset to empty at the firmware's consumer cursor; the USC pool and + /// descriptor ring are vacated and re-armed fresh before the next owner binds. + pub(crate) fn recycle_killed(&mut self) -> Result { + if !self.installed + || !self.exit.as_ref().is_some_and(|exit| exit.released()) + || self.active.len != 0 + || self.replays.len != 0 + || self.previous.is_some() + { + return Ok(false); + } + self.graph.reset_items()?; + self.exit = None; + self.quarantined = false; + self.quarantine_error = None; + self.failure_status_pending = false; + self.retired_by_teardown = false; + self.spared_quarantine = false; + self.spared_deferred = false; + self.retire_pending = false; + self.awaiting_witness = false; + self.retirement_proved = false; + self.replay_retirement = false; + self.replay_error = None; + self.replay_prepared = false; + self.submitted = 0; + self.kick.clear_parent_after_recovery(); + self.owner = None; + self.released = true; + self.retirement_ready = true; + self.released_at = now_ns(); + self.needs_fresh_backing = true; + Ok(true) } pub(crate) fn released_at(&self) -> u64 { self.released_at @@ -1179,12 +1253,13 @@ impl Queue { pages: crate::g17::freelist::Pages, ring: DescriptorRing, ) -> Result<(u16, u64)> { - if self.pool_phase != PoolPhase::Vacant || !self.reusable() { + if self.pool_phase != PoolPhase::Vacant || !(self.reusable() || self.recycled_pending()) { return Err(EAGAIN); } self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; + self.needs_fresh_backing = false; Ok((self.pool.id(), self.pool.page_list_va()?)) } /// Exact host identity remains valid even when several commands share an ioctl ID. diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index 08d1013e4762b..10d2a37274800 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -75,6 +75,13 @@ pub(super) struct Transaction { announced: Option, } +impl Transaction { + /// The firmware acknowledged the kill and the killed commands were settled. + pub(super) fn released(&self) -> bool { + self.phase == Phase::Released + } +} + impl Queue { pub(crate) fn exit_candidate(&self, context: &Arc, age: u64) -> bool { self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 8206df716b6b3..dcfdf43b63792 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -184,6 +184,13 @@ impl Registry { cap: usize, ) -> Result { // Count first: event turns are frequent and the steady state has no excess. + let forced = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| queue.vacate_candidate() && queue.needs_fresh_backing()) + .count(); let count = self .compute .iter() @@ -191,7 +198,7 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref()) .filter(|queue| queue.vacate_candidate()) .count(); - if count <= cap { + if count <= cap && forced == 0 { return Ok(0); } let mut candidates: KVec<(u64, u8)> = KVec::with_capacity(count, GFP_KERNEL)?; @@ -202,10 +209,12 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref()) .filter(|queue| queue.vacate_candidate()) { - candidates.push((queue.released_at(), queue.qid()), GFP_KERNEL)?; + // Recycled killed queues sort first (time 0) and count beyond the cap. + let key = if queue.needs_fresh_backing() { 0 } else { queue.released_at() }; + candidates.push((key, queue.qid()), GFP_KERNEL)?; } candidates.sort_unstable(); - let excess = candidates.len() - cap; + let excess = candidates.len().saturating_sub(cap).max(forced); let mut published = 0; for (_, qid) in candidates.iter().take(excess.min(4)) { let queue = self @@ -1125,6 +1134,7 @@ impl super::Firmware { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + queue.recycle_killed()?; if let Some(binding) = queue.detach_exited_owner() { deferred.defer_binding(binding); } From 5f06ebf2cb34050553c1f9c771e6f5b399e7c364 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:08:54 -0400 Subject: [PATCH 0848/2680] drm/asahi: give back the render pool backing of evicted idle clients Each logical context that has rendered keeps a render pool with about 24 MiB of pinned USC backing for the rest of its life, also while the client is idle and the machine is reclaiming memory. When pressure reclaim has evicted a client's VM (it was idle for the reclaim quiet window and has no job or binding in progress) and the firmware has consumed the pool's final free-list release, the event worker now detaches the pool's backing and both of its aliases and hands them to its deferred batch, which frees them after the device mutex is released. The pool keeps its ID, control and state objects. Before that client's next render, the submission path installs fresh backing, and the changed backing makes the next staging rewrite the control header and descriptor row, as for a new owner of the slot. A vacated pool is never staged. Clients whose VM was not evicted are unaffected; an evicted client already restores its VM before its next job. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 16822976f9d375ddbdc788d968077a15660be5d7) --- drivers/gpu/drm/asahi/g17/context.rs | 2 + drivers/gpu/drm/asahi/g17/freelist.rs | 139 +++++++++++++++--- drivers/gpu/drm/asahi/g17/runtime.rs | 18 +++ drivers/gpu/drm/asahi/g17/runtime/events.rs | 5 + drivers/gpu/drm/asahi/g17/runtime/render.rs | 31 ++++ .../gpu/drm/asahi/g17/runtime/render/pools.rs | 17 +++ .../drm/asahi/g17/runtime/render/prepare.rs | 11 +- drivers/gpu/drm/asahi/mmu.rs | 2 +- drivers/gpu/drm/asahi/mmu/residency.rs | 14 ++ 9 files changed, 218 insertions(+), 21 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 7076708be9f2f..f8145b1d3b619 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -280,6 +280,8 @@ impl Context { } let mut render = self.render.lock(); if let Some(pool) = &*render { + // A pool vacated under memory pressure is re-armed before reuse. + pool.ensure_backing(alloc, render_global, self.vm())?; return Ok(pool.clone()); } let pool = Arc::new( diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index a4d58e71eeccf..c2c9470d10aa3 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -12,7 +12,11 @@ use super::fw::initdata::{FreeListArgs, FreeListClass, FreeListControl, FREE_LIS use super::object::{Allocator, CpuMap, KernelObject}; use crate::mmu; use core::sync::atomic::{fence, AtomicBool, AtomicU64, Ordering}; -use kernel::{prelude::*, sync::Arc}; +use kernel::{ + new_mutex, + prelude::*, + sync::{Arc, Mutex}, +}; /// Render pools occupy 129..255; compute uses its QID plus one, in 1..128. /// First-free allocation keeps the shared firmware namespace deterministic. @@ -115,27 +119,21 @@ impl Pages { /// Pool-object identities are independent of recycled firmware IDs and VAs. static RENDER_POOL_SERIAL: AtomicU64 = AtomicU64::new(0); -/// A render pool retained by a logical context and every graph referencing it. -pub(crate) struct RenderPool { - // Remove all aliases before dropping the backing or returning the ID. +/// The USC backing of a render pool and its two aliases. Field order drops +/// both aliases before the backing. +pub(crate) struct RenderBacking { _client: mmu::KernelMapping, _global: mmu::KernelMapping, pages: Pages, - control: KernelObject, - state: KernelObject, - served: AtomicBool, - serial: u64, - id: RenderId, } -impl RenderPool { +impl RenderBacking { + /// Fresh populated backing, built without the device mutex. pub(crate) fn new( alloc: &Allocator<'_>, - ids: &Arc, render_global: &mmu::Vm, client: &mmu::Vm, ) -> Result { - let id = RenderId::new(ids)?; let pages = Pages::new(alloc, RENDER_BLOCKS, RunLayout::Render)?; let address = pages.page_list_va(); let global = pages @@ -144,6 +142,45 @@ impl RenderPool { let client = pages .backing .map_alias(client, address, mmu::PROT_GPU_SHARED_RW)?; + Ok(Self { + _client: client, + _global: global, + pages, + }) + } +} + +/// A render pool retained by a logical context and every graph referencing it. +/// Its backing can be given back while the pool's free-list release has been +/// consumed and the client VM is evicted; a fresh backing is installed before +/// the pool is staged again. +pub(crate) struct RenderPool { + backing: Arc>>, + /// Page-list VA of the installed backing; zero while vacant. + page_list: AtomicU64, + /// Counts installed backings, so a re-armed backing is a new free-list + /// owner even if its addresses repeat. + epoch: AtomicU64, + /// Residency of the client VM the backing is aliased into. + client_residency: Option>, + control: KernelObject, + state: KernelObject, + served: AtomicBool, + serial: u64, + id: RenderId, +} + +impl RenderPool { + pub(crate) fn new( + alloc: &Allocator<'_>, + ids: &Arc, + render_global: &mmu::Vm, + client: &mmu::Vm, + ) -> Result { + let id = RenderId::new(ids)?; + let backing = RenderBacking::new(alloc, render_global, client)?; + let address = backing.pages.page_list_va(); + let run_list_va = backing.pages.run_list_va(); let state = alloc.kernel( CONTROL_SIZE, mmu::UAT_PGSZ as u64, @@ -163,7 +200,7 @@ impl RenderPool { buffer_slot: u32::from(id.slot), class: FreeListClass::Two, page_list_va: address, - run_list_va: pages.run_list_va(), + run_list_va, blocks: RENDER_BLOCKS as u32, state_va: state.gpu_va(), }, @@ -174,10 +211,15 @@ impl RenderPool { let serial = RENDER_POOL_SERIAL .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| value.checked_add(1)) .map_err(|_| EOVERFLOW)? + 1; + let backing = Arc::pin_init( + new_mutex!(Some(backing), "G17 render pool backing"), + GFP_KERNEL, + )?; Ok(Self { - _client: client, - _global: global, - pages, + backing, + page_list: AtomicU64::new(address), + epoch: AtomicU64::new(0), + client_residency: client.residency_gate(), control, state, served: AtomicBool::new(false), @@ -186,20 +228,72 @@ impl RenderPool { }) } + /// No backing is installed: the pool must be re-armed before staging. + pub(crate) fn vacant(&self) -> bool { + self.page_list.load(Ordering::Acquire) == 0 + } + /// Pressure reclaim evicted the client VM, which is idle. + pub(crate) fn client_evicted(&self) -> bool { + self.client_residency.as_ref().is_some_and(|gate| gate.evicted()) + } + /// Removes the backing. The caller holds the device mutex and observed this + /// pool's free-list release consumed, so no firmware reference remains; the + /// next staging requires a re-armed backing. The caller drops the result + /// after releasing the device mutex. + pub(crate) fn vacate(&self) -> Option { + let backing = self.backing.lock().take(); + if backing.is_some() { + self.page_list.store(0, Ordering::Release); + } + backing + } + /// Installs fresh backing into a vacant pool, unless another caller did. + /// The next staging rewrites the control header and descriptor row for it + /// (its run-list address changes the free-list owner identity). + pub(crate) fn rearm(&self, fresh: RenderBacking) -> Result { + let mut backing = self.backing.lock(); + if backing.is_some() { + return Ok(()); + } + let address = fresh.pages.page_list_va(); + *backing = Some(fresh); + self.epoch.fetch_add(1, Ordering::AcqRel); + self.served.store(true, Ordering::Release); + self.page_list.store(address, Ordering::Release); + Ok(()) + } + /// Re-arms a vacant pool with fresh backing built here, without the device + /// mutex. `client` is the VM of the context that owns this pool. + pub(crate) fn ensure_backing( + &self, + alloc: &Allocator<'_>, + render_global: &mmu::Vm, + client: &mmu::Vm, + ) -> Result { + if !self.vacant() { + return Ok(()); + } + self.rearm(RenderBacking::new(alloc, render_global, client)?) + } + pub(crate) fn serial(&self) -> u64 { self.serial } + pub(crate) fn epoch(&self) -> u64 { + self.epoch.load(Ordering::Acquire) + } pub(crate) fn id(&self) -> u16 { self.id.slot } pub(crate) fn control_va(&self) -> u64 { self.control.gpu_va() } + /// Zero while vacant. pub(crate) fn page_list_va(&self) -> u64 { - self.pages.page_list_va() + self.page_list.load(Ordering::Acquire) } pub(crate) fn run_list_va(&self) -> u64 { - self.pages.run_list_va() + self.page_list_va() + PAGE_LIST_SIZE as u64 } pub(crate) fn end_va(&self) -> u64 { self.run_list_va() @@ -234,7 +328,12 @@ impl RenderPool { .write_unaligned(header) }; let control = self.control.pointer(0, CONTROL_SIZE)?.cast::(); - self.pages.repopulate()?; + let backing = self.backing.lock(); + let pages = &backing.as_ref().ok_or(EAGAIN)?.pages; + if pages.page_list_va() != self.page_list_va() { + return Err(EIO); + } + pages.repopulate()?; for (index, value) in image.iter().enumerate() { // SAFETY: The entire retained page was checked before any store; // the allocation is page-aligned and release excludes the firmware. @@ -462,6 +561,8 @@ impl ComputePool { pub(crate) struct Owner { /// Identifies the host pool object even when every firmware address recycles. pub(crate) serial: u64, + /// Identifies the pool's installed backing (render pools are re-armed). + pub(crate) epoch: u64, pub(crate) buffer_slot: u32, pub(crate) control_va: u64, pub(crate) ring_va: u64, diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index dcfdf43b63792..842e3a8b8df9b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -653,6 +653,8 @@ pub(super) struct DeferredBatch { /// of vacated pools: unmapping and freeing happen in `finish`, off-lock. bindings: KVVec, vacated: KVVec, + /// Backing of render pools vacated for evicted idle clients. + render: KVVec, } impl DeferredBatch { const DEPTH: usize = if crate::hw::t8140::queues::RENDER_DEPTH as usize @@ -677,6 +679,10 @@ impl DeferredBatch { values: KVVec::with_capacity(capacity, GFP_KERNEL)?, bindings: KVVec::with_capacity(detached, GFP_KERNEL)?, vacated: KVVec::with_capacity(detached, GFP_KERNEL)?, + render: KVVec::with_capacity( + if detached != 0 { crate::hw::t8140::queues::RENDER_SLOTS } else { 0 }, + GFP_KERNEL, + )?, }) } /// Keeps the detached owner's unmapping and VM teardown out of the device @@ -686,6 +692,15 @@ impl DeferredBatch { drop(binding); } } + /// Room for one more vacated render backing; the worker vacates no more. + pub(super) fn render_backing_room(&self) -> bool { + self.render.len() < self.render.capacity() + } + pub(super) fn defer_render_backing(&mut self, backing: super::freelist::RenderBacking) { + if let Err(backing) = self.render.push_within_capacity(backing) { + drop(backing); + } + } pub(super) fn defer_vacated(&mut self, vacated: super::queue::compute::Vacated) { if let Err(vacated) = self.vacated.push_within_capacity(vacated) { drop(vacated); @@ -734,6 +749,9 @@ impl DeferredBatch { for vacated in self.vacated.drain_all() { drop(vacated); } + for backing in self.render.drain_all() { + drop(backing); + } } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 4a18aaa05419d..99f3caf66bebe 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -387,6 +387,11 @@ impl crate::g17::Shared { firmware.reclaim_drained_renders(deferred)?; firmware.service_compute_exit(deferred)?; firmware.service_compute_pools(deferred)?; + if !firmware.recovery.pending() { + firmware + .queues + .vacate_evicted_render_pools(&firmware.init, deferred)?; + } { let mut reclaim = self.reclaim.lock(); firmware.service_teardown(&mut reclaim)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index bf86d46e7ca6b..aabc523e360cd 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -642,6 +642,37 @@ impl Registry { || !self.render.retired.is_empty() || !self.teardown.is_empty() } + /// Gives back the USC backing of render pools whose client VM pressure + /// reclaim evicted (an idle VM) and whose final free-list release was + /// consumed. The vacated backings go to the worker's deferred batch, which + /// frees them after the device mutex; a later job of that client re-arms + /// the pool first. + pub(in crate::g17) fn vacate_evicted_render_pools( + &mut self, + init: &crate::g17::initdata::InitData, + deferred: &mut super::DeferredBatch, + ) -> Result { + let render = &mut self.render; + render.pools.observe(init)?; + for entry in render.entries.iter().flatten() { + if !deferred.render_backing_room() { + break; + } + if entry.constructing || entry.growing || entry.preparing.is_some() { + continue; + } + let Some(pool) = entry.pool.as_ref() else { + continue; + }; + if pool.vacant() || !render.pools.idle(pool.id()) || !pool.client_evicted() { + continue; + } + if let Some(backing) = pool.vacate() { + deferred.defer_render_backing(backing); + } + } + Ok(()) + } pub(in crate::g17) fn render_in_flight(&self) -> bool { self.render.entries.iter().flatten().any(|entry| { entry.pair.as_ref().map_or( diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index 7047992754b37..f7f48e853f225 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -52,6 +52,7 @@ impl Pools { fn owner(pool: &RenderPool) -> Owner { Owner { serial: pool.serial(), + epoch: pool.epoch(), buffer_slot: u32::from(pool.id()), control_va: pool.control_va(), ring_va: pool.run_list_va(), @@ -59,6 +60,15 @@ impl Pools { } } + /// The pool's registration has a consumed final release and no references: + /// the firmware holds nothing of its backing. + pub(super) fn idle(&self, id: u16) -> bool { + Self::slot(id) + .ok() + .and_then(|slot| self.leases[slot]) + .is_some_and(|lease| lease.released()) + } + pub(super) fn released(&self, id: u32, _control: u64) -> bool { self.leases .get(id as usize) @@ -68,6 +78,10 @@ impl Pools { /// Install both engine references before either inner queue becomes /// visible. Warm overlap keeps the existing generation and pool contents. pub(super) fn prepare(&mut self, init: &InitData, pool: &RenderPool) -> Result { + // A vacated pool is re-armed by the submission path before it retries. + if pool.vacant() { + return Err(EAGAIN); + } self.observe(init)?; let slot = Self::slot(pool.id())?; let owner = Self::owner(pool); @@ -92,6 +106,9 @@ impl Pools { /// Warming validates the incoming owner and stages its descriptor without /// acquiring command references or publishing an asynchronous request. pub(super) fn warm(&self, init: &InitData, pool: &RenderPool) -> Result { + if pool.vacant() { + return Err(EAGAIN); + } let previous = self.leases[Self::slot(pool.id())?]; let owner = Self::owner(pool); if previous.is_some_and(|lease| lease.owner() == owner) { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 69f8249e0818d..b4813cdba033a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -107,7 +107,16 @@ impl super::super::Backend { } else { let pair = entry.pair.as_ref().ok_or(EBUSY)?; if pair.matches(self.owner, &self.context)? { - return Ok(()); + if !pair.pool().vacant() { + return Ok(()); + } + let pool = pair.pool().clone(); + drop(state); + let alloc = Allocator { + dev: &dev, + uat: &resources.uat, + }; + return pool.ensure_backing(&alloc, &resources.global, self.context.vm()); } if !pair.reusable()? { return Err(EBUSY); diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 79fda65b3e2b2..7184f93510def 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -118,7 +118,7 @@ mod context; mod shared; mod lifetime; mod residency; -pub(crate) use residency::VmShrinker; +pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; pub(crate) use bind::PreparedUserBindBatch; pub(crate) use lifetime::VmJobGuard; diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 90e59760e84fb..f912813b65d2b 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -195,6 +195,15 @@ impl Gate { }) } + /// Pressure reclaim released this VM's backing and no user has entered + /// since: its next job restores first. Never blocks; a contended gate + /// reads as not evicted. + pub(crate) fn evicted(&self) -> bool { + self.state + .try_lock() + .is_some_and(|state| state.evicted && !state.busy && state.users == 0) + } + /// A page-table mutation happened outside a `Reclaim` (the purge path runs /// on busy VMs). The next `enter` restores every INVALIDATED mapping. pub(crate) fn note_evicted(&self) { @@ -313,6 +322,11 @@ impl Vm { self.residency.is_some() } + /// The residency gate of a user VM, for observers that must not keep the VM. + pub(crate) fn residency_gate(&self) -> Option> { + self.residency.clone() + } + pub(super) fn enter_residency_metadata(&self) -> Result> { self.residency .as_ref() From ac793aff95c1ec24c69f7969a09895504381e3d7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 15:17:39 -0400 Subject: [PATCH 0849/2680] drm/asahi: carry a scheduling profile from DRM queue priority to firmware Every queue the driver creates uses firmware priority class 2 and policy 2, written as constants into the queue record, queue configuration, kick announcement, kick header, work-ring selection, doorbells and render registration token, and the QoS row class and share are constants too. The DRM queue priority is validated at queue creation and then dropped. Give each logical queue a scheduling profile (priority class, policy word, QoS class and QoS share), fixed when its context is created, and take every one of those fields from the owning context's profile. LOW queues use priority class 3 and MEDIUM queues keep class 2; the other profile fields are unchanged because no different values are known for class 3. HIGH and REALTIME remain refused. A physical queue takes a new owner's profile only at the existing rebind boundaries, which require it to be idle: a same-VM render rebind also replaces the priority and policy words of both queue records when they differ, compute owner changes do the same in their record, and a render pair whose registration carries another class is registered again at its next publication, under the same power hold as a first install. Work slots mark the first slot naming a queue per priority class, since each class has its own work rings. With only MEDIUM queues, every record, ring, doorbell and token keeps the values and store order it had before. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit bff3853f260b02bdd9605bc6690905cb70aca29f) --- drivers/gpu/drm/asahi/g17/context.rs | 10 +- drivers/gpu/drm/asahi/g17/fw/channels.rs | 17 ++- drivers/gpu/drm/asahi/g17/fw/kick.rs | 16 ++- drivers/gpu/drm/asahi/g17/fw/queue.rs | 105 +++++++++++++++--- drivers/gpu/drm/asahi/g17/initdata.rs | 43 ++++--- drivers/gpu/drm/asahi/g17/mod.rs | 14 ++- drivers/gpu/drm/asahi/g17/qos.rs | 13 ++- drivers/gpu/drm/asahi/g17/queue/compute.rs | 50 ++++++--- drivers/gpu/drm/asahi/g17/queue/render.rs | 54 ++++++--- .../gpu/drm/asahi/g17/queue/render/memory.rs | 21 ++++ .../gpu/drm/asahi/g17/queue/render/parked.rs | 4 +- .../drm/asahi/g17/queue/render/publication.rs | 1 + drivers/gpu/drm/asahi/g17/runtime.rs | 20 +++- .../drm/asahi/g17/runtime/compute_replay.rs | 5 +- drivers/gpu/drm/asahi/g17/runtime/render.rs | 13 ++- .../drm/asahi/g17/runtime/render/submit.rs | 5 +- drivers/gpu/drm/asahi/hw/t8140.rs | 11 ++ 17 files changed, 310 insertions(+), 92 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index f8145b1d3b619..afd0d1d22f5bb 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -9,7 +9,7 @@ mod identity; use super::{ freelist::{RenderIds, RenderPool}, - fw::context as fw, + fw::{context as fw, queue::Policy}, object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, status::VmStatus, }; @@ -120,6 +120,9 @@ pub(crate) struct Context { work: Arc>, work_base: u64, render: Arc>>>, + /// Fixed for the context's life. Physical queues bound to it carry this + /// profile in every record, ring, doorbell and registration. + policy: Policy, } impl Context { pub(crate) fn new( @@ -128,6 +131,7 @@ impl Context { qos: &Arc, execution: Arc, owner_pid: u32, + policy: Policy, ) -> Result { let status = execution.vm().status().ok_or(EINVAL)?.clone(); let qos = qos.acquire()?; @@ -188,6 +192,7 @@ impl Context { work, work_base, render, + policy, }) } pub(crate) fn vm(&self) -> &Vm { @@ -209,6 +214,9 @@ impl Context { pub(crate) fn owner_pid(&self) -> u32 { self.owner_pid } + pub(crate) fn policy(&self) -> Policy { + self.policy + } pub(crate) fn qos_id(&self) -> u8 { self.qos.id } diff --git a/drivers/gpu/drm/asahi/g17/fw/channels.rs b/drivers/gpu/drm/asahi/g17/fw/channels.rs index c2c3b3eb7a80d..c4cf6960db296 100644 --- a/drivers/gpu/drm/asahi/g17/fw/channels.rs +++ b/drivers/gpu/drm/asahi/g17/fw/channels.rs @@ -15,7 +15,7 @@ use kernel::prelude::*; use super::kick::QID_MAX; -use super::queue::{DataMaster, QUEUE_PRIORITY}; +use super::queue::DataMaster; /// Size of a device-control record. pub(crate) const DEVICE_CONTROL_RECORD_SIZE: usize = 0x40; @@ -448,10 +448,13 @@ pub(crate) const WORK_RING_SLOTS: u32 = 256; /// Number of engines with a work ring per priority. const WORK_RINGS_PER_PRIORITY: usize = 3; -/// Index in the channel table of the work ring for `data_master` at the -/// priority of every driver-created queue. -pub(crate) const fn work_ring_index(data_master: DataMaster) -> usize { - QUEUE_PRIORITY as usize * WORK_RINGS_PER_PRIORITY + data_master as usize +/// Index in the channel table of the work ring for `data_master` at priority +/// class `priority`, or `None` if `priority` is not a class. +pub(crate) const fn work_ring_index(priority: u8, data_master: DataMaster) -> Option { + if priority >= super::queue::Policy::PRIORITIES { + return None; + } + Some(priority as usize * WORK_RINGS_PER_PRIORITY + data_master as usize) } /// One work-ring slot: a request to run a queue's newly published commands. @@ -467,7 +470,9 @@ pub(crate) struct WorkSlot { pub(crate) items_end: u16, /// Queue to run. pub(crate) qid: u8, - /// One in the first slot naming this queue on this ring. + /// One in the first slot naming this queue on this ring. Each priority + /// class has its own rings, so a queue whose class changed names the new + /// ring with a first slot again. pub(crate) first: u8, } diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index 2e6133e882b2d..47cdce3ca09e5 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -20,7 +20,7 @@ use kernel::prelude::*; -use super::queue::QUEUE_PRIORITY; +use super::queue::Policy; /// Number of entries in a kick ring. pub(crate) const KICK_SLOTS: usize = 256; @@ -371,6 +371,8 @@ pub(crate) struct KickArgs<'a> { pub(crate) register_arrays: [Option; 4], /// The compute register program includes a shader-context scratch request. pub(crate) compute_scratch: bool, + /// Priority class of the queue. + pub(crate) priority: u8, } /// One kick ring entry. @@ -436,13 +438,14 @@ impl KickEntry { if args.qid > QID_MAX || barriers > KICK_BARRIERS_MAX || args.qos.class > Self::QOS_CLASS_MAX + || args.priority >= Policy::PRIORITIES { return Err(EINVAL); } let mut header = args.timestamp.get() << Self::TIMESTAMP_SHIFT | (args.qid as u64) << Self::QID_SHIFT - | (QUEUE_PRIORITY as u64) << Self::PRIORITY_SHIFT; + | (args.priority as u64) << Self::PRIORITY_SHIFT; if args.compute_scratch { header |= Self::COMPUTE_SCRATCH; } @@ -506,9 +509,10 @@ impl RenderRegistration { /// Validates both queues without touching MMIO. The resulting transaction /// writes fragment configuration and validity before tiling configuration - /// and validity. Each validity mask is stored high before low. - pub(crate) fn new(fragment: (u8, u64), tiling: (u8, u64)) -> Result { - if fragment.0 == tiling.0 { + /// and validity. Each validity mask is stored high before low. Both queues + /// are registered at priority class `priority`. + pub(crate) fn new(fragment: (u8, u64), tiling: (u8, u64), priority: u8) -> Result { + if fragment.0 == tiling.0 || priority >= Policy::PRIORITIES { return Err(EINVAL); } let mut writes = [(0, 0); 8]; @@ -522,7 +526,7 @@ impl RenderRegistration { if qid > QID_MAX || va & !Self::ADDRESS_MASK != 0 { return Err(EINVAL); } - let completion = (3 * (4 - u64::from(QUEUE_PRIORITY)) + 0x3fe).wrapping_shl(54); + let completion = (3 * (4 - u64::from(priority)) + 0x3fe).wrapping_shl(54); let payload = ((engine as u64 + 1) << 60) | va | completion; let (low, high) = if qid < 64 { (1u64 << qid, 0) diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index db95ba99e0189..48b110a4193e3 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -22,14 +22,59 @@ use kernel::prelude::*; use super::kick::{KickTimestamp, QID_MAX}; -/// Priority of every queue the driver creates. +/// Firmware scheduling profile of a queue. /// -/// The same value appears in the queue record, [`QueueConfig`], -/// [`KickAnnounce`] and the kick entry header; the work rings it selects are -/// listed in `channels.rs`. -pub(crate) const QUEUE_PRIORITY: u8 = 2; -/// Scheduling policy of every queue the driver creates. -pub(crate) const QUEUE_POLICY: u16 = 2; +/// Every record that names a queue repeats its priority class: the queue +/// record (twice), [`QueueConfig`], [`KickAnnounce`], the kick entry header, +/// the work ring the queue is published on (`channels.rs`), the doorbell and +/// the render registration token. The queue record and [`QueueConfig`] also +/// carry the scheduling policy word; the QoS row and share word carry the QoS +/// class and share. All of them must agree for one queue. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Policy { + priority: u8, + policy: u16, + qos_class: u8, + qos_share: u32, +} + +impl Policy { + /// Number of firmware priority classes. + pub(crate) const PRIORITIES: u8 = 4; + + /// Returns the profile with priority class `priority`, policy word + /// `policy`, QoS class `qos_class` and QoS share `qos_share`. + pub(crate) const fn new(priority: u8, policy: u16, qos_class: u8, qos_share: u32) -> Self { + assert!(priority < Self::PRIORITIES); + Self { + priority, + policy, + qos_class, + qos_share, + } + } + + /// Priority class, below [`Self::PRIORITIES`]. + pub(crate) const fn priority(&self) -> u8 { + self.priority + } + + /// Scheduling policy word. + pub(crate) const fn policy(&self) -> u16 { + self.policy + } + + /// QoS class byte of the queue's QoS row. + pub(crate) const fn qos_class(&self) -> u8 { + self.qos_class + } + + /// QoS share word of the owner's QoS slot. + pub(crate) const fn qos_share(&self) -> u32 { + self.qos_share + } +} + /// Number of work pointers in a queue's item ring. pub(crate) const ITEM_RING_ENTRIES: u32 = 0x500; @@ -173,13 +218,15 @@ pub(crate) struct QueueRecordArgs { pub(crate) scheduler_va: u64, /// Thread-group ID of the owning process (0 while unowned). pub(crate) owner_pid: u32, + /// Scheduling profile of the owning logical queue. + pub(crate) policy: Policy, } /// Description of one hardware queue, named by its work-ring slots. /// /// The host writes it whole at queue creation and changes the owner fields of -/// an idle queue with [`QueueRecord::set_owner`] or -/// [`QueueRecord::set_compute_owner`]. +/// an idle queue with [`QueueRecord::set_compute_owner`] or, for render +/// queues, field by field (`render/memory.rs`). #[repr(C, packed)] #[derive(Copy, Clone, Debug)] pub(crate) struct QueueRecord { @@ -224,12 +271,12 @@ impl QueueRecord { job_list_va: args.job_list_va, unk_18: [0; 0xc], unk_24: u32::MAX, - priority: [QUEUE_PRIORITY as u32; 2], + priority: [args.policy.priority() as u32; 2], unk_30: [0; 6], unk_36: u16::MAX, unk_38: 0, unk_3c: 0, - policy: QUEUE_POLICY as u32, + policy: args.policy.policy() as u32, unk_44: u32::MAX, owner_pid: args.owner_pid, unk_4c: [0; 0x50], @@ -241,16 +288,32 @@ impl QueueRecord { /// Hands an idle compute queue to a new owner. /// /// Replaces the job head, process identity and scheduler page, and clears `unk_38`. + /// The priority and policy words are written only when `policy` differs from the + /// profile the record holds. pub(crate) fn set_compute_owner( &mut self, job_list_va: u64, scheduler_va: u64, owner_pid: u32, + policy: Policy, ) { self.job_list_va = job_list_va; self.unk_38 = 0; self.owner_pid = owner_pid; self.scheduler_va = scheduler_va; + self.set_policy(policy); + } + + /// Replaces the priority and policy words of an idle queue's record when they differ + /// from `policy`. + pub(crate) fn set_policy(&mut self, policy: Policy) { + let priority = [policy.priority() as u32; 2]; + if { self.priority } != priority { + self.priority = priority; + } + if { self.policy } != policy.policy() as u32 { + self.policy = policy.policy() as u32; + } } } @@ -310,6 +373,8 @@ pub(crate) struct QueueConfigArgs { pub(crate) free_list: FreeListBinding, /// GPU address of the owner's scheduler-state page. pub(crate) scheduler_va: u64, + /// Scheduling profile of the owning logical queue. + pub(crate) policy: Policy, /// QoS buffer slot of the logical queue. pub(crate) qos_slot: u8, /// The QoS identity or queue membership changed since the last update. @@ -429,7 +494,7 @@ impl QueueConfig { qid: args.qid as u16, install: args.install as u16, unk_1c: 0, - priority: QUEUE_PRIORITY as u16, + priority: args.policy.priority() as u16, unk_20: 0, data_master: data_master as u16, unk_24: 0, @@ -445,7 +510,7 @@ impl QueueConfig { qos_slot: args.qos_slot as u16, unk_58: 0, owner_pid: args.owner_pid, - policy: QUEUE_POLICY, + policy: args.policy.policy(), unk_60: 0, context_update: args.context_update as u16, unk_64: 0, @@ -486,16 +551,22 @@ static_assert!(core::mem::offset_of!(KickAnnounce, timestamp) == 0x09); static_assert!(core::mem::offset_of!(KickAnnounce, data_master) == 0x11); impl KickAnnounce { - /// Announces the entry of queue `qid` written at `timestamp`. - pub(crate) fn new(qid: u8, data_master: DataMaster, timestamp: KickTimestamp) -> Result { - if qid > QID_MAX { + /// Announces the entry of queue `qid` at priority class `priority`, + /// written at `timestamp`. + pub(crate) fn new( + qid: u8, + data_master: DataMaster, + timestamp: KickTimestamp, + priority: u8, + ) -> Result { + if qid > QID_MAX || priority >= Policy::PRIORITIES { return Err(EINVAL); } Ok(Self { tag: CommandTag::KickAnnounce as u32, qid: qid as u16, count: 1, - priority: QUEUE_PRIORITY, + priority, timestamp: timestamp.get(), data_master: data_master as u32, unk_15: [0; 3], diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 623fd55eb5b6e..5239f9eb979ce 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -416,14 +416,20 @@ impl InitData { /// The sole host publisher holds the device mutex through the following /// inner and outer publication. Firmware consumers can only free capacity; /// neither this check nor waiting for room reserves or limits other queues. - pub(super) fn work_ready(&self, engine: super::fw::queue::DataMaster) -> Result { - let [consumer, consumer_b, producer] = self.work_cursors(engine)?; + /// `priority` selects the engine's ring of that priority class. + pub(super) fn work_ready( + &self, + priority: u8, + engine: super::fw::queue::DataMaster, + ) -> Result { + let [consumer, consumer_b, producer] = self.work_cursors(priority, engine)?; let next = (producer + 1) % super::fw::channels::WORK_RING_SLOTS; Ok(next != consumer && next != consumer_b) } fn work_publication<'a>( &'a self, + priority: u8, record: &'a super::fw::channels::WorkSlot, ) -> Result> { use super::fw::{channels, queue::DataMaster}; @@ -433,8 +439,8 @@ impl InitData { 2 => DataMaster::Compute, _ => return Err(EINVAL), }; - let channel = channels::work_ring_index(engine); - let [consumer, consumer_b, producer] = self.work_cursors(engine)?; + let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; + let [consumer, consumer_b, producer] = self.work_cursors(priority, engine)?; let next = (producer + 1) % channels::WORK_RING_SLOTS; if next == consumer || next == consumer_b { return Err(EINVAL); @@ -457,9 +463,13 @@ impl InitData { } /// Two firmware consumers and the host producer, sampled in that order. - pub(super) fn work_cursors(&self, engine: super::fw::queue::DataMaster) -> Result<[u32; 3]> { + pub(super) fn work_cursors( + &self, + priority: u8, + engine: super::fw::queue::DataMaster, + ) -> Result<[u32; 3]> { use super::fw::channels; - let channel = channels::work_ring_index(engine); + let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let base = Self::state_offset(Role::Primary) + abi::work_state_offset(channel); let state = self.object(self.cluster)?; let mut values = [0; 3]; @@ -481,8 +491,12 @@ impl InitData { /// Inner queue records must be visible before this outer publication. The returned /// producer is the retained consumer witness for this slot. - pub(super) fn publish_work(&self, record: &super::fw::channels::WorkSlot) -> Result { - Ok(self.work_publication(record)?.commit()) + pub(super) fn publish_work( + &self, + priority: u8, + record: &super::fw::channels::WorkSlot, + ) -> Result { + Ok(self.work_publication(priority, record)?.commit()) } /// Validate both destinations before exposing fragment, then tiling. The accounting token @@ -490,6 +504,7 @@ impl InitData { /// Deferred pairs retain both slot bodies without exposing either producer. pub(super) fn publish_work_pair( &self, + priority: u8, fragment: &super::fw::channels::WorkSlot, tiling: &super::fw::channels::WorkSlot, deferred: bool, @@ -502,8 +517,8 @@ impl InitData { { return Err(EINVAL); } - let fragment = self.work_publication(fragment)?; - let tiling = self.work_publication(tiling)?; + let fragment = self.work_publication(priority, fragment)?; + let tiling = self.work_publication(priority, tiling)?; fragment.copy_record(); commit_accounting()?; fence(Ordering::SeqCst); @@ -522,6 +537,7 @@ impl InitData { fn retained_work_publication<'a>( &'a self, + priority: u8, record: &'a super::fw::channels::WorkSlot, next: u32, ) -> Result> { @@ -531,7 +547,7 @@ impl InitData { 1 => DataMaster::Fragment, _ => return Err(EINVAL), }; - let channel = channels::work_ring_index(engine); + let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let producer = self.object(self.cluster)?.word( Self::state_offset(Role::Primary) + abi::work_state_offset(channel) @@ -558,6 +574,7 @@ impl InitData { /// witnesses before either copy; both bodies precede the fragment and tiling publications. pub(super) fn publish_retained_work_pair( &self, + priority: u8, fragment: &super::fw::channels::WorkSlot, tiling: &super::fw::channels::WorkSlot, next: [u32; 2], @@ -568,8 +585,8 @@ impl InitData { { return Err(EINVAL); } - let fragment = self.retained_work_publication(fragment, next[0])?; - let tiling = self.retained_work_publication(tiling, next[1])?; + let fragment = self.retained_work_publication(priority, fragment, next[0])?; + let tiling = self.retained_work_publication(priority, tiling, next[1])?; fragment.copy_record(); tiling.copy_record(); fence(Ordering::SeqCst); diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 4b8372552686f..21e244185bd57 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1148,8 +1148,9 @@ impl gpu::Gpu for Gpu { let owner = self.ids.queue.next(); let frontend = queue::Frontend::new(priority, usc_exec_base, self.geometry.user_usable_range())?; + let policy = scheduling_policy(priority)?; let (backend, slot) = loop { - match runtime::Backend::new(&self.shared, &vm, owner) { + match runtime::Backend::new(&self.shared, &vm, owner, policy) { Err(error) if error == EAGAIN => continue, result => break result?, } @@ -1190,6 +1191,17 @@ impl gpu::Gpu for Gpu { } } +/// Scheduling profile of a new logical queue. `file.rs` passes +/// `DRM_ASAHI_PRIORITY_REALTIME - priority` and refuses HIGH and REALTIME, so +/// MEDIUM arrives as 2 and LOW as 3. +fn scheduling_policy(priority: u32) -> Result { + match priority { + 2 => Ok(hw::t8140::scheduling::MEDIUM), + 3 => Ok(hw::t8140::scheduling::LOW), + _ => Err(EINVAL), + } +} + impl Drop for Gpu { fn drop(&mut self) { self.shutdown(); diff --git a/drivers/gpu/drm/asahi/g17/qos.rs b/drivers/gpu/drm/asahi/g17/qos.rs index 8a925c23552a7..9a100db4cffdf 100644 --- a/drivers/gpu/drm/asahi/g17/qos.rs +++ b/drivers/gpu/drm/asahi/g17/qos.rs @@ -6,13 +6,16 @@ //! precedes submitted counters; retiring a final owner clears only its two row //! bytes. Cancellation balances host references without rewinding shared counts. -use super::{fw::qos as fw, object::KernelObject}; +use super::{ + fw::{qos as fw, queue::Policy}, + object::KernelObject, +}; +use crate::hw::t8140::qos as cfg; use core::{ mem::{offset_of, size_of}, sync::atomic::{fence, AtomicU32, AtomicU64, AtomicU8, Ordering}, }; use kernel::prelude::*; -use crate::hw::t8140::qos as cfg; /// Atomic views retain the allocation borrow and never borrow firmware words /// as ordinary mutable memory. The arrays are disjoint apart from the explicit @@ -97,11 +100,13 @@ impl Accounting { /// Called under the device publication lock immediately before exposing /// the work record. The clock is sampled only when its engine becomes busy. + /// The row's class and the owner's share come from the owner's `policy`. pub(crate) fn publish( &mut self, view: &View<'_>, owner: Owner, scheduler: u64, + policy: Policy, clock: impl FnOnce() -> (u64, u64), ) -> Result { let (qid, qos, dm) = owner.indices()?; @@ -124,10 +129,10 @@ impl Accounting { return Err(EBUSY); } view.rows[qid][0].store(owner.qos, Ordering::Relaxed); - view.rows[qid][1].store(cfg::CLASS, Ordering::Relaxed); + view.rows[qid][1].store(policy.qos_class(), Ordering::Relaxed); let record = view.row(qid); view.scheduler[qos].store(scheduler, Ordering::Relaxed); - view.share[qos].store(cfg::SHARE, Ordering::Relaxed); + view.share[qos].store(policy.qos_share(), Ordering::Relaxed); if self.in_flight[qid] == 0 { fence(Ordering::SeqCst); } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 57179437f9b40..2b962e2632606 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -137,6 +137,7 @@ impl Graph { job_list_va: context.work_head_va(), scheduler_va: context.scheduler_va(), owner_pid: context.owner_pid(), + policy: context.policy(), }); view.pointers = PointerBlock::new(); view.jobs = JobListHead::new(graph_va + 0x200); @@ -210,6 +211,7 @@ impl Graph { context.work_head_va(), context.scheduler_va(), context.owner_pid(), + context.policy(), ) }; Ok(()) @@ -468,7 +470,8 @@ pub(crate) struct Queue { replay_prepared: bool, exit: Option, installed: bool, - outer_started: bool, + /// Bit `p` set once a work slot of priority class `p` named this queue. + outer_started: u8, published_config: Option, ordinal: u32, submitted: u32, @@ -521,7 +524,7 @@ impl Queue { replay_prepared: false, exit: None, installed: false, - outer_started: false, + outer_started: 0, published_config: None, ordinal: 0, submitted: 0, @@ -1459,10 +1462,16 @@ pub(crate) trait Host { fn next_compute_publication(&mut self) -> Result; fn epoch(&self) -> Result<(u64, u32)>; fn prepare_compute_shared(&mut self) -> Result; - fn qos_publish(&mut self, owner: qos::Owner, scheduler: u64) -> Result; + fn qos_publish( + &mut self, + owner: qos::Owner, + scheduler: u64, + policy: crate::g17::fw::queue::Policy, + ) -> Result; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; - fn publish_outer(&mut self, slot: &crate::g17::fw::channels::WorkSlot) -> Result; + fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) + -> Result; fn notify(&mut self, message: u64) -> Result; fn note_submission(&mut self) -> Result; } @@ -1580,6 +1589,8 @@ impl Queue { context: packet.context.id(), }; self.free_list_generation = u64::from(packet.context.scheduler_generation()); + let policy = packet.context.policy(); + let ring = 1u8 << policy.priority(); let config = QueueConfig::new(&QueueConfigArgs { target: QueueConfigTarget::Compute, kick_ring_va: self.kick.low_va(), @@ -1594,6 +1605,7 @@ impl Queue { slot: u32::from(self.pool.id()), }, scheduler_va: identity.scheduler, + policy, qos_slot: identity.qos, qos_update: self .published_config @@ -1603,7 +1615,8 @@ impl Queue { .published_config .is_none_or(|p| p.scheduler != identity.scheduler || p.context != identity.context), })?; - let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp)?; + let announce = + KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; let predecessor = KickPredecessor::new(qid, kick_parent)?; let kick_args = KickArgs { qid, @@ -1620,6 +1633,7 @@ impl Queue { event_mask: COMPUTE_KICK_EVENT_MASK, register_arrays: ComputeDescriptor::register_bindings(descriptor_low, *scratch)?, compute_scratch: scratch.enabled(), + priority: policy.priority(), }; let published = host.next_compute_publication()?; self.active @@ -1690,6 +1704,7 @@ impl Queue { data_master: 2, }, identity.scheduler, + policy, )?; self.active.back_mut().ok_or(EIO)?.qos_pending = true; let target = self.pool.reserve_submission().or_else(|error| { @@ -1725,6 +1740,7 @@ impl Queue { data_master: 2, }, identity.scheduler, + policy, )?; self.active.back_mut().ok_or(EIO)?.qos_pending = true; producer = self @@ -1754,15 +1770,18 @@ impl Queue { self.graph.record_ends[slot] = Some(producer); } host.publish_qid(self.kick.id())?; - host.publish_outer(&WorkSlot::new( - DataMaster::Compute, - self.graph.graph.gpu_va(), - qid, - producer.try_into()?, - !self.outer_started, - )?)?; - let notify_activation = !self.outer_started; - self.outer_started = true; + host.publish_outer( + policy.priority(), + &WorkSlot::new( + DataMaster::Compute, + self.graph.graph.gpu_va(), + qid, + producer.try_into()?, + self.outer_started & ring == 0, + )?, + )?; + let notify_activation = self.outer_started & ring == 0; + self.outer_started |= ring; self.installed = true; self.published_config = Some(identity); self.ordinal = ordinal; @@ -1791,7 +1810,8 @@ impl Queue { } if notify_activation { fence(Ordering::SeqCst); - host.notify((0x83 << 48) | 0x0a)?; + // Pipe 2 (compute) in bits 1:0, priority class in bits 3:2. + host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; } fence(Ordering::SeqCst); host.notify((0x83 << 48) | 0x10)?; diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 720f2d8a0897e..7936d253d2411 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -74,12 +74,18 @@ pub(crate) struct PublicationLease { pub(crate) trait Host { fn epoch(&self) -> Result<(u64, u32)>; fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]>; - fn qos_publish(&mut self, owner: qos::Owner, scheduler: u64) -> Result; + fn qos_publish( + &mut self, + owner: qos::Owner, + scheduler: u64, + policy: crate::g17::fw::queue::Policy, + ) -> Result; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn install_pair(&mut self, registration: &RenderRegistration) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; fn publish_outer_pair( &mut self, + priority: u8, fragment: &crate::g17::fw::channels::WorkSlot, tiling: &crate::g17::fw::channels::WorkSlot, commit: impl FnOnce() -> Result, @@ -88,11 +94,12 @@ pub(crate) trait Host { fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result; fn publish_retained_pair( &mut self, + priority: u8, fragment: &WorkSlot, tiling: &WorkSlot, next: [u32; 2], ) -> Result; - fn notify_render(&mut self) -> Result; + fn notify_render(&mut self, priority: u8) -> Result; fn note_submission(&mut self) -> Result; fn complete_render( &mut self, @@ -141,8 +148,11 @@ pub(crate) struct Pair { configs: [Option; 2], cursors: [u32; 2], recovery_generation: u64, - registered: bool, - outer_started: bool, + /// Priority class of the pair's installed registration, if any. A pair + /// rebound to an owner of another class is registered again. + registered: Option, + /// Bit `p` set once a work slot of priority class `p` named the pair. + outer_started: u8, quarantined: bool, terminal: Option<(Arc, Error)>, previous: Option, @@ -223,8 +233,8 @@ impl Pair { configs: [None; 2], cursors: [0; 2], recovery_generation: 0, - registered: false, - outer_started: false, + registered: None, + outer_started: 0, quarantined: false, terminal: None, previous: None, @@ -434,6 +444,12 @@ impl Pair { if packet.completion.status().get() != 0 { return Err(EIO); } + let policy = packet.context.policy(); + // A registration of another class can only follow a rebind, which + // requires the pair to have drained. + if self.registered.is_some_and(|class| class != policy.priority()) && self.in_flight() { + return Err(EIO); + } let ordinal = self.ordinal; let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; let mcache = prepared.mcache; @@ -450,6 +466,7 @@ impl Pair { self.memory .completion_links(ordinal, kicks, payload, !covered[1])?; let context = &packet.context; + let ring = 1u8 << policy.priority(); let qos = context.qos_id(); let qos_owners = [ qos::Owner { @@ -492,6 +509,7 @@ impl Pair { slot: self.free_list.id().into(), }, scheduler_va: context.scheduler_va(), + policy, qos_slot: qos, qos_update: self.context_update, owner_pid: context.owner_pid(), @@ -561,6 +579,7 @@ impl Pair { event_mask: abi::RENDER_KICK_EVENT_MASK, register_arrays: arrays, compute_scratch: false, + priority: policy.priority(), }; let ta_args = args( 0, @@ -579,6 +598,7 @@ impl Pair { let registration = RenderRegistration::new( (qids[1], self.kicks[1].low_va()), (qids[0], self.kicks[0].low_va()), + policy.priority(), )?; let [tiling_kick, fragment_kick] = &mut self.kicks; let tiling_kick = tiling_kick.prepare_entry(&ta_args)?; @@ -620,14 +640,14 @@ impl Pair { queues[1], qids[1], windows[1].target().try_into()?, - !self.outer_started, + self.outer_started & ring == 0, )?; let tiling_slot = WorkSlot::new( DataMaster::Tiling, queues[0], qids[0], windows[0].target().try_into()?, - !self.outer_started, + self.outer_started & ring == 0, )?; let event_slot = memory .graph @@ -664,16 +684,20 @@ impl Pair { tiling_kick.commit(); let mut qos_publications = [None; 2]; let result = (|| { - qos_publications[1] = Some(host.qos_publish(qos_owners[1], context.scheduler_va())?); - qos_publications[0] = Some(host.qos_publish(qos_owners[0], context.scheduler_va())?); - if !self.registered { + qos_publications[1] = + Some(host.qos_publish(qos_owners[1], context.scheduler_va(), policy)?); + qos_publications[0] = + Some(host.qos_publish(qos_owners[0], context.scheduler_va(), policy)?); + // A class change happens only at a rebind, after the pair drained. + if self.registered != Some(policy.priority()) { host.install_pair(®istration)?; - self.registered = true; + self.registered = Some(policy.priority()); } for kick in &self.kicks { host.publish_qid(kick.id())?; } host.publish_outer_pair( + policy.priority(), &fragment_slot, &tiling_slot, || { @@ -720,16 +744,16 @@ impl Pair { } self.recovery_generation = epoch; self.installed = [true; 2]; - self.outer_started = true; + self.outer_started |= ring; if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { previous.publication = Some(ordinal); } packet.context.mark_published(); let publish = (|| { host.activate_render_pool(self.free_list.id(), lease.free_list_generation)?; - host.publish_retained_pair(&fragment_slot, &tiling_slot, outer_next)?; + host.publish_retained_pair(policy.priority(), &fragment_slot, &tiling_slot, outer_next)?; self.active.back_mut().ok_or(EIO)?.outer_published = true; - host.notify_render()?; + host.notify_render(policy.priority())?; host.note_submission() })(); if let Err(error) = publish { diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 859ff61901570..fc5a05e9bf8a9 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -32,6 +32,9 @@ pub(super) const JOB_LIST: usize = 0xa000; pub(super) const CONFIG: [usize; 2] = [0, 0x4000]; pub(super) const CONFIG_STRIDE: usize = 0x180; const DEPTH: usize = queues::RENDER_DEPTH as usize; +/// Offsets of the priority pair and the policy word in a queue record. +const PRIORITY: usize = core::mem::offset_of!(queue::QueueRecord, priority); +const POLICY: usize = core::mem::offset_of!(queue::QueueRecord, policy); struct Aliases { client: mmu::KernelMapping, @@ -147,6 +150,7 @@ impl Graph { job_list_va: context.work_head_va(), scheduler_va: context.scheduler_va(), owner_pid: context.owner_pid(), + policy: context.policy(), }), )?; queues.initialize::(POINTERS[stage], |view| { @@ -665,6 +669,9 @@ impl Memory { Ok(()) } + /// Retargets both retired queue records to `context`. A context of another + /// scheduling profile also replaces the priority and policy words, after + /// the owner fields; the same profile leaves them untouched. pub(super) fn set_owner(&self, qids: [u8; 2], context: &Context) -> Result { let offsets = qids.map(|qid| usize::from(qid) * size_of::()); let base = self.graph.queues.pointer(0, GRAPH_SIZE)?; @@ -674,7 +681,10 @@ impl Memory { self.graph.queues.pointer(at + 0x10, 8)?; self.graph.queues.pointer(at + 0x48, 4)?; self.graph.queues.pointer(at + 0x9c, 8)?; + self.graph.queues.pointer(at + PRIORITY, 8)?; + self.graph.queues.pointer(at + POLICY, 4)?; } + let policy = context.policy(); // SAFETY: Both queues are retired before retargeting. Complete byte // views were checked above; packed scheduler words use unaligned stores. unsafe { @@ -693,6 +703,17 @@ impl Memory { .cast::() .write_unaligned(context.scheduler_va()); } + for at in offsets { + let words = base.add(at + PRIORITY).cast::<[u32; 2]>(); + let priority = [u32::from(policy.priority()); 2]; + if words.read_unaligned() != priority { + words.write_unaligned(priority); + } + let word = base.add(at + POLICY).cast::(); + if word.read_unaligned() != u32::from(policy.policy()) { + word.write_unaligned(u32::from(policy.policy())); + } + } } Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 3d67578e2aaf9..f712bb7b1b87d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -28,8 +28,8 @@ pub(crate) struct ParkedPair { descriptor_flags: [u32; 2], installed: [bool; 2], recovery_generation: u64, - registered: bool, - outer_started: bool, + registered: Option, + outer_started: u8, previous: Option, } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index 3df92062735e3..0fc9c1d9a15dc 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -99,6 +99,7 @@ impl<'a> Lane<'a> { DataMaster::Fragment }, timestamp, + config.priority as u8, )?, announce_destination, prefix: count as usize - 1, diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 842e3a8b8df9b..4c7aa6a442cc1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -368,6 +368,7 @@ impl Backend { shared: &Arc, vm: &crate::mmu::Vm, owner: u64, + policy: super::fw::queue::Policy, ) -> Result<(Arc, admission::QueueSlot)> { let lease = shared.preparations.enter(true)?; let dev = shared.drm()?; @@ -394,7 +395,7 @@ impl Backend { let owner_pid = u32::try_from(kernel::current!().group_leader().pid()).map_err(|_| EOVERFLOW)?; let context = Arc::new( - Context::new(&alloc, &pool, &qos, execution, owner_pid)?, + Context::new(&alloc, &pool, &qos, execution, owner_pid, policy)?, GFP_KERNEL, )?; let state = shared.state.lock(); @@ -776,11 +777,17 @@ impl compute::Host for ComputeHost<'_> { fn prepare_compute_shared(&mut self) -> Result { self.init.activate_compute() } - fn qos_publish(&mut self, owner: qos::Owner, scheduler: u64) -> Result { + fn qos_publish( + &mut self, + owner: qos::Owner, + scheduler: u64, + policy: super::fw::queue::Policy, + ) -> Result { self.accounting.publish( &qos::View::new(self.init.qos()?)?, owner, scheduler, + policy, qos::clock, ) } @@ -791,8 +798,8 @@ impl compute::Host for ComputeHost<'_> { fn publish_qid(&mut self, id: kick::Id) -> Result { self.qids.publish(id) } - fn publish_outer(&mut self, slot: &super::fw::channels::WorkSlot) -> Result { - self.init.publish_work(slot).map(|_| ()) + fn publish_outer(&mut self, priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + self.init.publish_work(priority, slot).map(|_| ()) } fn notify(&mut self, message: u64) -> Result { self.primary.notify(message) @@ -816,7 +823,10 @@ impl super::Firmware { } // No descriptor, accounting or inner producer changes precede this // capacity check. The same device lock covers queue.publish below. - if !self.init.work_ready(DataMaster::Compute)? { + if !self + .init + .work_ready(packet.context.policy().priority(), DataMaster::Compute)? + { return Err(EAGAIN); } let queues = &mut self.queues; diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index 65b00d36714bb..0d4a29e779b4a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -56,7 +56,10 @@ impl Firmware { break; }; let owner = queue.owner(); - let outer_ready = self.init.work_ready(crate::g17::fw::queue::DataMaster::Compute)?; + let outer_ready = self.init.work_ready( + packet.context.policy().priority(), + crate::g17::fw::queue::DataMaster::Compute, + )?; if !outer_ready { let now = super::now_ns(); let since = *self.queues.replay_outer_wait.get_or_insert(now); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index aabc523e360cd..71b21f98479a6 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -377,12 +377,14 @@ impl crate::g17::queue::render::Host for Host<'_> { &mut self, owner: crate::g17::qos::Owner, scheduler: u64, + policy: crate::g17::fw::queue::Policy, ) -> Result { use crate::g17::qos; self.firmware.queues.accounting.publish( &qos::View::new(self.firmware.init.qos()?)?, owner, scheduler, + policy, qos::clock, ) } @@ -418,6 +420,7 @@ impl crate::g17::queue::render::Host for Host<'_> { } fn publish_outer_pair( &mut self, + priority: u8, fragment: &crate::g17::fw::channels::WorkSlot, tiling: &crate::g17::fw::channels::WorkSlot, commit: impl FnOnce() -> Result, @@ -425,7 +428,7 @@ impl crate::g17::queue::render::Host for Host<'_> { ) -> Result<[u32; 2]> { self.firmware .init - .publish_work_pair(fragment, tiling, true, commit, late_tiling) + .publish_work_pair(priority, fragment, tiling, true, commit, late_tiling) } fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result { self.release_registration(); @@ -437,6 +440,7 @@ impl crate::g17::queue::render::Host for Host<'_> { } fn publish_retained_pair( &mut self, + priority: u8, fragment: &crate::g17::fw::channels::WorkSlot, tiling: &crate::g17::fw::channels::WorkSlot, next: [u32; 2], @@ -444,18 +448,19 @@ impl crate::g17::queue::render::Host for Host<'_> { let result = self .firmware .init - .publish_retained_work_pair(fragment, tiling, next); + .publish_retained_work_pair(priority, fragment, tiling, next); if result.is_err() { self.firmware.primary.state.shared.lose_device_quietly(); } result } - fn notify_render(&mut self) -> Result { + /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. + fn notify_render(&mut self, priority: u8) -> Result { self.release_registration(); let result = self .firmware .primary - .notify((0x83 << crate::g17::MSG_TYPE_SHIFT) | 0x08); + .notify((0x83 << crate::g17::MSG_TYPE_SHIFT) | u64::from(priority) << 2); if result.is_err() { self.firmware.primary.state.shared.lose_device_quietly(); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 39b8c064d0106..dbf7175c5cd4b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -275,8 +275,9 @@ impl crate::g17::Firmware { } // Both outer destinations must have room before pool accounting or // either inner queue becomes visible. All host producers share this lock. - if !self.init.work_ready(crate::g17::fw::queue::DataMaster::Fragment)? - || !self.init.work_ready(crate::g17::fw::queue::DataMaster::Tiling)? + let priority = packet.context.policy().priority(); + if !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Fragment)? + || !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Tiling)? { return Err(EBUSY); } diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 4a97217a00705..2e73aed1a9797 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -378,6 +378,17 @@ pub(crate) mod qos { pub(crate) const CLOCK_HZ: u64 = 24_000_000; } +/// Firmware scheduling profiles of the DRM queue priorities. Native user +/// queues use priority class 2 with policy 2. +pub(crate) mod scheduling { + use crate::g17::fw::queue::Policy; + + /// `DRM_ASAHI_PRIORITY_MEDIUM`. + pub(crate) const MEDIUM: Policy = Policy::new(2, 2, super::qos::CLASS, super::qos::SHARE); + /// `DRM_ASAHI_PRIORITY_LOW`. + pub(crate) const LOW: Policy = Policy::new(3, 2, super::qos::CLASS, super::qos::SHARE); +} + /// Physical workqueue capacity and logical-owner admission limits. pub(crate) mod queues { pub(crate) const RENDER_SLOTS: usize = 64; From 834a2a8ce0b5fbc1ee706092e288608c8d1a82c7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:18:07 -0400 Subject: [PATCH 0850/2680] drm/asahi: drop two defensive clippy allowances in the VM binding paths map_in_range_with_guard_size() and prepare_bind_object() take seven parameters including self, which is clippy's default too_many_arguments threshold, not above it. The allowances silenced a lint that does not fire. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 009b0fb93ba35380563055aa8bc725f36c505262) --- drivers/gpu/drm/asahi/mmu.rs | 1 - drivers/gpu/drm/asahi/mmu/bind.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 7184f93510def..67a95fcf02773 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1821,7 +1821,6 @@ impl Vm { } /// Reserves a whole-page unmapped guard without adding it to the mapped backing. - #[allow(clippy::too_many_arguments)] pub(crate) fn map_in_range_with_guard_size( &self, gem: &gem::Object, diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index 26740c5719d82..01163edf879de 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -58,7 +58,6 @@ impl PreparedUserBindBatch { impl Vm { /// Prepares host state without taking DMA reservations or changing any leaf. - #[allow(clippy::too_many_arguments)] pub(crate) fn prepare_bind_object( &self, gem: &gem::Object, From b0cf12fd90897d948770e26682991c1856cc3d7a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:24:25 -0400 Subject: [PATCH 0851/2680] drm/asahi: build the G17 boot objects region by region MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit InitData::build placed and initialized every boot object in one 360-line function with two placement closures. Keep the placement inputs in a small environment and perform the same steps, in the same order, as one function per region: register windows, descriptor tables, render free lists, the render pool, the compute pools, the bundle with its rings and regions, and the firmware-object initialization. The four fences keep their positions and the firmware objects are byte-identical. [ryan: adapted to 7.2.10 — minimal; for DJ: BuildEnv holds an AsahiDrmDevice (the 7.2 name of the DRM device type) and the register windows pass it to Vm::map_io(), which takes a device on 7.2] Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 410d8f80e9ab6b405d7f918901fcbe883bb2d9b2) --- drivers/gpu/drm/asahi/g17/initdata.rs | 260 +++++++++++++++++++------- 1 file changed, 188 insertions(+), 72 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 5239f9eb979ce..4419056789dea 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -11,6 +11,7 @@ use super::{ Config, Role, }; use crate::{driver::AsahiDrmDevice, hw::t8140, mmu}; +use core::ops::Range; use core::sync::atomic::{fence, AtomicU32, Ordering}; use kernel::{c_str, prelude::*}; use t8140::{kernel_window as high, low}; @@ -166,6 +167,57 @@ pub(super) struct InitData { render_blocks: core::ops::Range, } +/// Placement inputs shared by the build steps. +struct BuildEnv<'a> { + dev: &'a AsahiDrmDevice, + vm: &'a mmu::Vm, + lower: &'a mmu::Vm, + render: &'a mmu::Vm, + base: u64, + dynamic: Range, +} + +impl BuildEnv<'_> { + /// An object at a fixed offset of the kernel window. + fn fixed(&self, offset: u64, size: usize, prot: mmu::Prot, cpu: CpuMap) -> Result { + KernelObject::new( + self.dev, + self.vm, + Placement::At(self.base + offset), + size, + prot, + cpu, + ) + } + + /// An object placed in the dynamic part of the kernel window. + fn dynamic(&self, size: usize, align: u64, prot: mmu::Prot, cpu: CpuMap) -> Result { + KernelObject::new( + self.dev, + self.vm, + Placement::In(self.dynamic.clone(), align), + size, + prot, + cpu, + ) + } +} + +/// Object indices the firmware-object initialization needs from the table step. +struct DescriptorTables { + pb: usize, + page_pool: usize, + fwctl: usize, +} + +/// Object indices and ring addresses the firmware-object initialization needs from the bundle step. +struct BundleObjects { + zero: usize, + completion_rings: [abi::CompletionRing; 2], + region_a: usize, + qos: usize, +} + impl InitData { /// Empty construction state, retained even if graph construction fails. pub(super) fn new() -> Self { @@ -643,35 +695,30 @@ impl InitData { render: &mmu::Vm, metrics: &KernelObject, ) -> Result { - let vm = uat.kernel_vm(); - let lower = uat.kernel_lower_vm(); let range = uat.geometry().kernel_range(); - let base = range.start; - let dynamic = base + t8140::dynamic::KERNEL_OFFSET..range.end; - let fixed = |offset, size, prot, cpu| { - KernelObject::new(dev, vm, Placement::At(base + offset), size, prot, cpu) - }; - let dynamic_object = |size, align, prot, cpu| { - KernelObject::new( - dev, - vm, - Placement::In(dynamic.clone(), align), - size, - prot, - cpu, - ) - }; - use mmu::{ - PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_PRIV_RW as GFP, - PROT_GPU_FW_SHARED_RW as GFS, PROT_GPU_SHARED_RW as GS, + let env = BuildEnv { + dev, + vm: uat.kernel_vm(), + lower: uat.kernel_lower_vm(), + render, + base: range.start, + dynamic: range.start + t8140::dynamic::KERNEL_OFFSET..range.end, }; - use CpuMap::{WriteBack as WB, WriteCombined as WC}; + self.map_register_windows(&env)?; + let tables = self.build_descriptor_tables(&env)?; + self.build_render_free_lists(&env)?; + let first_block = self.build_render_pool(&env)?; + self.build_compute_pools(&env, first_block)?; + let bundle = self.build_bundle(&env, cfg, metrics)?; + self.initialize_firmware_objects(&env, cfg, &tables, &bundle) + } + fn map_register_windows(&mut self, env: &BuildEnv<'_>) -> Result { for window in &t8140::REGISTER_WINDOWS { let (phys, va, size) = window.mapping().ok_or(EINVAL)?; self.aliases.push( - vm.map_io( - dev.as_ref().as_ref(), + env.vm.map_io( + env.dev.as_ref().as_ref(), va, phys.try_into()?, size.try_into()?, @@ -680,15 +727,25 @@ impl InitData { GFP_KERNEL, )?; } - self.cluster = self.add(fixed(high::PRIVATE_CLUSTER, abi::private::SIZE, FS, WC)?)?; - self.compute_global = self.add(fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?; - let pb = self.add(fixed(high::PB_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; + Ok(()) + } + + /// The private cluster, zero page and the three descriptor tables with their low aliases. + fn build_descriptor_tables(&mut self, env: &BuildEnv<'_>) -> Result { + use mmu::{ + PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_SHARED_RW as GFS, + PROT_GPU_SHARED_RW as GS, + }; + use CpuMap::{WriteBack as WB, WriteCombined as WC}; + self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, abi::private::SIZE, FS, WC)?)?; + self.compute_global = self.add(env.fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?; + let pb = self.add(env.fixed(high::PB_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(pb)? .initialize::(0, |table| { table.init(); Ok(()) })?; - let page_pool = self.add(fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; + let page_pool = self.add(env.fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(page_pool)? .initialize::(0, |table| { table.init(low::RENDER_PAGE_LIST, low::COMPUTE_POOL_PAGE_LIST); @@ -696,17 +753,30 @@ impl InitData { })?; self.pb = pb; self.page_pool = page_pool; - self.alias(pb, lower, low::PB_DESCRIPTORS, GS)?; - self.alias(page_pool, lower, low::PAGE_POOL_DESCRIPTORS, GS)?; - let index = self.add(fixed(high::INDEX, INDEX_SIZE, GFS, WB)?)?; + self.alias(pb, env.lower, low::PB_DESCRIPTORS, GS)?; + self.alias(page_pool, env.lower, low::PAGE_POOL_DESCRIPTORS, GS)?; + let index = self.add(env.fixed(high::INDEX, INDEX_SIZE, GFS, WB)?)?; self.object_mut(index)? .initialize::(0, |table| { table.init(); Ok(()) })?; - self.alias(index, lower, low::INDEX, GFS)?; - let fwctl = self.add(fixed(high::FWCTL, abi::FWCTL_SIZE, FS, WB)?)?; + self.alias(index, env.lower, low::INDEX, GFS)?; + let fwctl = self.add(env.fixed(high::FWCTL, abi::FWCTL_SIZE, FS, WB)?)?; + Ok(DescriptorTables { + pb, + page_pool, + fwctl, + }) + } + /// Render and boot free-list controls, descriptor pages and the two queue contexts. + fn build_render_free_lists(&mut self, env: &BuildEnv<'_>) -> Result { + use mmu::{ + PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_PRIV_RW as GFP, + PROT_GPU_FW_SHARED_RW as GFS, + }; + use CpuMap::WriteCombined as WC; for (offset, state, class, prot, initial) in [ ( high::RENDER_FREE_LIST, @@ -723,7 +793,7 @@ impl InitData { abi::FreeListState::BOOT, ), ] { - let control = self.add(fixed(offset, mmu::UAT_PGSZ, prot, WC)?)?; + let control = self.add(env.fixed(offset, mmu::UAT_PGSZ, prot, WC)?)?; self.object_mut(control)?.write( 0, abi::FreeListControl::new(&abi::FreeListArgs { @@ -732,7 +802,7 @@ impl InitData { page_list_va: low::RENDER_PAGE_LIST, run_list_va: low::RENDER_RUN_LIST, blocks: low::RENDER_BLOCKS_INITIAL, - state_va: base + state, + state_va: env.base + state, }), )?; let state_prot = if class == abi::FreeListClass::One { @@ -740,19 +810,26 @@ impl InitData { } else { GFS }; - let state = self.add(fixed(state, mmu::UAT_PGSZ, state_prot, WC)?)?; + let state = self.add(env.fixed(state, mmu::UAT_PGSZ, state_prot, WC)?)?; self.object_mut(state)?.write(0, initial)?; if class == abi::FreeListClass::Two { self.boot_control = control; self.boot_state = state; } } - self.stamps[0] = self.add(fixed(high::DESCRIPTOR_PAGE_A, mmu::UAT_PGSZ, GFS, WC)?)?; - self.stamps[1] = self.add(fixed(high::DESCRIPTOR_PAGE_B, mmu::UAT_PGSZ, GFP, WC)?)?; - self.add(fixed(high::TA_CONTEXT, CONTEXT_SIZE, FS, WC)?)?; - self.add(fixed(high::FRAGMENT_CONTEXT, CONTEXT_SIZE, FS, WC)?)?; + self.stamps[0] = self.add(env.fixed(high::DESCRIPTOR_PAGE_A, mmu::UAT_PGSZ, GFS, WC)?)?; + self.stamps[1] = self.add(env.fixed(high::DESCRIPTOR_PAGE_B, mmu::UAT_PGSZ, GFP, WC)?)?; + self.add(env.fixed(high::TA_CONTEXT, CONTEXT_SIZE, FS, WC)?)?; + self.add(env.fixed(high::FRAGMENT_CONTEXT, CONTEXT_SIZE, FS, WC)?)?; + Ok(()) + } - let pages = self.add(KernelObject::backing(dev, low::RENDER_PAGE_LIST_SIZE, WB)?)?; + /// Render page list, run list and blocks aliased into the render-global root; returns + /// the index of the first block. + fn build_render_pool(&mut self, env: &BuildEnv<'_>) -> Result { + use mmu::PROT_GPU_SHARED_RW as GS; + use CpuMap::WriteBack as WB; + let pages = self.add(KernelObject::backing(env.dev, low::RENDER_PAGE_LIST_SIZE, WB)?)?; self.object_mut(pages)? .initialize::<[u64; low::RENDER_PAGE_LIST_SIZE / 8]>(0, |list| { abi::fill_page_list( @@ -760,21 +837,28 @@ impl InitData { (0..low::RENDER_BLOCKS_INITIAL as usize).map(low::render_block), ) })?; - let runs = self.add(KernelObject::backing(dev, low::RENDER_RUN_LIST_SIZE, WB)?)?; + let runs = self.add(KernelObject::backing(env.dev, low::RENDER_RUN_LIST_SIZE, WB)?)?; let first_block = self.objects.len(); for _ in 0..low::RENDER_BLOCKS_ALLOCATED { - self.add(KernelObject::backing(dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; + self.add(KernelObject::backing(env.dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; } self.render_pages = pages; self.render_runs = runs; self.render_blocks = first_block..self.objects.len(); - self.alias(pages, render, low::RENDER_PAGE_LIST, GS)?; - self.alias(runs, render, low::RENDER_RUN_LIST, GS)?; + self.alias(pages, env.render, low::RENDER_PAGE_LIST, GS)?; + self.alias(runs, env.render, low::RENDER_RUN_LIST, GS)?; for block in 0..low::RENDER_BLOCKS_MAPPED { - self.alias(first_block + block, render, low::render_block(block), GS)?; + self.alias(first_block + block, env.render, low::render_block(block), GS)?; } + Ok(first_block) + } - let compute = self.add(fixed(high::COMPUTE_FREE_LIST, mmu::UAT_PGSZ, GFS, WC)?)?; + /// Compute free-list controls and states, zero blocks, run list and the compute page + /// list inside the render blocks. + fn build_compute_pools(&mut self, env: &BuildEnv<'_>, first_block: usize) -> Result { + use mmu::PROT_GPU_FW_SHARED_RW as GFS; + use CpuMap::{WriteBack as WB, WriteCombined as WC}; + let compute = self.add(env.fixed(high::COMPUTE_FREE_LIST, mmu::UAT_PGSZ, GFS, WC)?)?; self.object_mut(compute)?.write( 0, abi::FreeListControl::new(&abi::FreeListArgs { @@ -783,18 +867,18 @@ impl InitData { page_list_va: low::COMPUTE_PAGE_LIST, run_list_va: low::COMPUTE_RUN_LIST, blocks: low::COMPUTE_BLOCKS, - state_va: base + high::COMPUTE_FREE_LIST_STATE, + state_va: env.base + high::COMPUTE_FREE_LIST_STATE, }), )?; - let compute_state = self.add(KernelObject::backing(dev, mmu::UAT_PGSZ, WB)?)?; + let compute_state = self.add(KernelObject::backing(env.dev, mmu::UAT_PGSZ, WB)?)?; self.object_mut(compute_state)? .write(0, abi::FreeListState::COMPUTE)?; for _ in &low::COMPUTE_ZERO_BLOCKS { - self.add(KernelObject::backing(dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; + self.add(KernelObject::backing(env.dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; } let compute_runs = self.add(KernelObject::new( - dev, - lower, + env.dev, + env.lower, Placement::At(low::COMPUTE_RUN_LIST), mmu::UAT_PGSZ, GFS, @@ -804,7 +888,7 @@ impl InitData { .initialize::<[abi::FreeListRunSlot; mmu::UAT_PGSZ / 0x40]>(0, |list| { abi::fill_run_list(list, low::COMPUTE_FREE_LIST_BLOCKS) })?; - let ready = self.add(fixed( + let ready = self.add(env.fixed( high::COMPUTE_READY_FREE_LIST, mmu::UAT_PGSZ, GFS, @@ -818,11 +902,11 @@ impl InitData { page_list_va: low::COMPUTE_POOL_PAGE_LIST, run_list_va: low::COMPUTE_POOL_RUN_LIST, blocks: low::COMPUTE_READY_BLOCKS, - state_va: base + high::COMPUTE_READY_FREE_LIST_STATE, + state_va: env.base + high::COMPUTE_READY_FREE_LIST_STATE, }), )?; for state in [high::COMPUTE_READY_FREE_LIST_STATE, high::COMPUTE_STATE] { - let obj = self.add(fixed(state, mmu::UAT_PGSZ, GFS, WC)?)?; + let obj = self.add(env.fixed(state, mmu::UAT_PGSZ, GFS, WC)?)?; self.object_mut(obj)? .write(0, abi::FreeListState::COMPUTE)?; } @@ -831,9 +915,24 @@ impl InitData { abi::fill_page_list(list, low::COMPUTE_FREE_LIST_BLOCKS) })?; fence(Ordering::SeqCst); + Ok(()) + } - self.bundle = self.add(fixed(high::BUNDLE, abi::bundle::SIZE, FP, WC)?)?; - let zero = self.add(fixed( + /// The bundle with its zero page and completion rings, the QoS table, the metrics alias, + /// the roots and regions A and C. + fn build_bundle( + &mut self, + env: &BuildEnv<'_>, + cfg: &Config, + metrics: &KernelObject, + ) -> Result { + use mmu::{ + PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_SHARED_RW as GFS, + PROT_GPU_SHARED_RW as GS, + }; + use CpuMap::{WriteBack as WB, WriteCombined as WC}; + self.bundle = self.add(env.fixed(high::BUNDLE, abi::bundle::SIZE, FP, WC)?)?; + let zero = self.add(env.fixed( high::BUNDLE + abi::bundle::ZERO_PAGE as u64, mmu::UAT_PGSZ, GFS, @@ -845,45 +944,62 @@ impl InitData { }; 2]; for slot in 0..2 { let offset = high::BUNDLE + abi::bundle::COMPLETION_RINGS[slot] as u64; - let ring = self.add(fixed(offset, abi::COMPLETION_RING_SIZE, FP, WB)?)?; + let ring = self.add(env.fixed(offset, abi::COMPLETION_RING_SIZE, FP, WB)?)?; self.completion_rings[slot] = ring; - self.alias(ring, lower, low::COMPLETION_RINGS[slot], GS)?; + self.alias(ring, env.lower, low::COMPLETION_RINGS[slot], GS)?; completion_rings[slot] = abi::CompletionRing { low_va: low::COMPLETION_RINGS[slot], - high_va: base + offset, + high_va: env.base + offset, }; } - let qos = self.add(dynamic_object(abi::QOS_SIZE, mmu::UAT_PGSZ as u64, FS, WC)?)?; + let qos = self.add(env.dynamic(abi::QOS_SIZE, mmu::UAT_PGSZ as u64, FS, WC)?)?; self.qos = qos; fence(Ordering::SeqCst); let metrics_alias = - metrics.alias_in(vm, dynamic.clone(), cfg.pm_metrics.size as u64, GFS)?; + metrics.alias_in(env.vm, env.dynamic.clone(), cfg.pm_metrics.size as u64, GFS)?; self.metrics_fw_va = metrics_alias.iova(); self.aliases.push(metrics_alias, GFP_KERNEL)?; fence(Ordering::SeqCst); - self.roots = self.add(dynamic_object( + self.roots = self.add(env.dynamic( abi::ROOTS_SIZE, abi::SECONDARY_ROOT_OFFSET as u64, mmu::PROT_FW_SHARED_RO, WB, )?)?; - let region_a = self.add(dynamic_object( + let region_a = self.add(env.dynamic( REGION_A_SIZE, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RO, WB, )?)?; - self.region_c = self.add(dynamic_object(REGION_C_SIZE, mmu::UAT_PGSZ as u64, FS, WB)?)?; + self.region_c = self.add(env.dynamic(REGION_C_SIZE, mmu::UAT_PGSZ as u64, FS, WB)?)?; + Ok(BundleObjects { + zero, + completion_rings, + region_a, + qos, + }) + } - let opps = Self::opps(dev)?; + /// Writes every firmware object the host builds from constants and addresses, then the + /// opening control-ring state for both roles. + fn initialize_firmware_objects( + &mut self, + env: &BuildEnv<'_>, + cfg: &Config, + tables: &DescriptorTables, + bundle: &BundleObjects, + ) -> Result { + let base = env.base; + let opps = Self::opps(env.dev)?; let hw_args = abi::HwDataArgs { chip_id: cfg.chip_id, windows: &t8140::REGISTER_WINDOWS, reserved_slots: &t8140::RESERVED_REGISTER_SLOTS, opps: &opps, perf: &t8140::PERF, - completion_rings, - qos_va: self.object(qos)?.gpu_va(), + completion_rings: bundle.completion_rings, + qos_va: self.object(bundle.qos)?.gpu_va(), }; self.object_mut(self.bundle)? .initialize::(abi::bundle::HW_DATA, |hw| hw.init(&hw_args))?; @@ -896,7 +1012,7 @@ impl InitData { .write(abi::private::PRIMARY_STATUS_A, abi::StatusBlock::new())?; self.object_mut(self.cluster)? .write(abi::private::SECONDARY_STATUS_A, abi::StatusBlock::new())?; - let fwctl_va = self.object(fwctl)?.gpu_va(); + let fwctl_va = self.object(tables.fwctl)?.gpu_va(); self.object_mut(self.cluster)? .initialize::(abi::private::STATUS_B, |status| { status.init(fwctl_va); @@ -929,9 +1045,9 @@ impl InitData { Ok(()) })?; let views = abi::PrimaryViews { - zero_page_va: self.object(zero)?.gpu_va(), - pb_descriptors_va: self.object(pb)?.gpu_va(), - page_pool_descriptors_va: self.object(page_pool)?.gpu_va(), + zero_page_va: self.object(bundle.zero)?.gpu_va(), + pb_descriptors_va: self.object(tables.pb)?.gpu_va(), + page_pool_descriptors_va: self.object(tables.page_pool)?.gpu_va(), }; for role in [Role::Primary, Role::Secondary] { let slot = role as usize; @@ -964,7 +1080,7 @@ impl InitData { abi::OpeningRecord::new(abi_role), )?; let root = abi::RootArgs { - region_a_va: self.object(region_a)?.gpu_va(), + region_a_va: self.object(bundle.region_a)?.gpu_va(), region_c_va: self.object(self.region_c)?.gpu_va(), main_config_va: main_va, status_a_va: status_va, From 82104ebbfdda07f9ddc6838c260f19b6174d49b7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:26:41 -0400 Subject: [PATCH 0852/2680] drm/asahi: split the G17 render pair install paths out of build_render_pair build_render_pair() held the device-locked selection of how a slot is built, the three off-lock-then-locked install paths and the final locked warm-up in one 400-line function. Move each install path into its own function: install_fresh_pair(), install_parked_pair() and rebind_pair() construct off the lock and install under it exactly as before, and finish_render_build() rebases a fresh graph and warms its pool. A small Copy context carries the lease, allocator, resources, slot and first packet to the install steps. Statement order, lock scopes and error handling are unchanged; the selection stays a block expression in build_render_pair() because returning the selected resources and build by value doubled their stack slots (+1.3 KiB on the scheduler run_job path). The three install steps are inlined: as separate frames they deepen the run_job path, which is close to its stack budget. With them inlined the path measures 17,616 of 18,432 bytes against 17,392 before this change. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit f5bd61786e3b537fd811f0ea27d2ec33874232b0) --- .../drm/asahi/g17/runtime/render/prepare.rs | 575 ++++++++++-------- 1 file changed, 331 insertions(+), 244 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index b4813cdba033a..06bea3c84fa7a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -6,6 +6,7 @@ use super::{Config, Reservation}; use crate::g17::{ buffer::MetricsLease, + preparation::Lease, job::Packet, object::{Allocator, KernelObject, Pool}, queue::render::{Pair, RenderAddresses}, @@ -59,6 +60,17 @@ enum Build { }, } +/// What every build step needs besides its variant: the preparation lease, the allocator, +/// the device resources, the slot and the first packet. +#[derive(Clone, Copy)] +struct BuildContext<'a> { + lease: &'a Lease, + alloc: &'a Allocator<'a>, + resources: &'a BuildResources, + slot: u8, + first: &'a Arc, +} + impl super::super::Backend { fn build_render_pair(&self, slot: u8, first: &Arc, optional: bool) -> Result { let lease = self.shared.preparations.enter(false)?; @@ -178,264 +190,339 @@ impl super::super::Backend { Build::Fresh(..) | Build::Parked { .. } => true, Build::Rebind { buffer, .. } => buffer.is_some(), }; - let result = match build { + let ctx = BuildContext { + lease: &lease, + alloc: &alloc, + resources: &resources, + slot, + first, + }; + match build { Build::Fresh(reservation, pm_generation, metrics) => { - let built = (|| -> Result> { - let free_list = - self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; - Pair::new( - &alloc, - &resources.objects, - reservation.ids, - slot, - self.owner, - self.context.clone(), - free_list, - metrics, - reservation.buffer, - pm_generation, - first, - resources.config.clusters, - resources.config.descriptor_flags, - ) - })(); - let (mut built, error) = match built { - Ok(pair) => (Some(pair), None), - Err(error) => (None, Some(error)), - }; - let result = (|| { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { - return Err(EAGAIN); - } - if let Some(error) = error { - return Err(error); - } - if !self.context.is_current() || self.context.status().get() != 0 { - return Err(EFAULT); - } - if firmware.queues.render.reserved(reservation)?.closed { - return Err(ECANCELED); - } - built - .as_mut() - .ok_or(EIO)? - .finish_install(&alloc, firmware.init.pb_descriptor_table()?)?; - firmware.queues.install_render(reservation, &mut built) - })(); - // Fresh backing disappears before its temporary PB reference returns. - drop(built); - if result.is_err() { - let mut state = self.shared.state.lock(); - (*state) - .as_deref_mut() - .ok_or(ENODEV)? - .queues - .cancel_render_reservation(reservation)?; - } - result + self.install_fresh_pair(&ctx, reservation, pm_generation, metrics)? } Build::Parked { - mut parked, + parked, buffer, pm_generation, metrics, - } => { - let built = (|| -> Result<_> { - let free_list = - self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; - let prepared = parked.prepare( - &alloc, - &resources.objects, - self.owner, - self.context.clone(), - free_list, - metrics, - buffer, - pm_generation, - first, - )?; - let storage = KBox::::new_uninit(GFP_KERNEL)?; - Ok((prepared, storage)) - })(); - let (mut prepared, mut storage, error) = match built { - Ok((prepared, storage)) => (Some(prepared), Some(storage), None), - Err(error) => (None, None, Some(error)), - }; - let result = { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - (|| { - if !lease.is_current() { - return Err(EAGAIN); - } - if let Some(error) = error { - return Err(error); - } - if !self.context.is_current() || self.context.status().get() != 0 { - return Err(EFAULT); - } - let entry = firmware.queues.render.entry(slot)?; - if entry.pending_owner != Some(self.owner) - || entry.pending_owner_closed - || entry.pair.is_some() - || entry.parked.is_some() - { - return Err(ECANCELED); - } - let storage = storage.take().ok_or(EIO)?; - firmware.queues.buffers.can_release(buffer.0)?; - prepared.as_mut().ok_or(EIO)?.finish_install( - &parked, - &alloc, - firmware.init.pb_descriptor_table()?, - )?; - let pair = parked.commit(&mut prepared, storage)?; - firmware.queues.buffers.reserve_owner(buffer.0)?; - firmware.queues.buffers.release(buffer.0)?; - let entry = firmware.queues.render.entry_mut(slot)?; - entry.reservation.owner = self.owner; - entry.reservation.buffer = buffer; - entry.context = Some(self.context.clone()); - entry.pool = Some(pair.pool().clone()); - entry.pair = Some(pair); - entry.closed = false; - entry.published = false; - Ok(()) - })() - }; - // Rejected candidate mappings disappear before the PB identity - // returns, while the original installed kick owner stays live. - drop(prepared); - drop(storage); - let grow_ready = { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if result.is_err() { - firmware.queues.render.entry_mut(slot)?.parked = Some(parked); - firmware.queues.buffers.release(buffer.0)?; - } else { - drop(state); - drop(parked); - state = self.shared.state.lock(); - } - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - let entry = firmware.queues.render.entry_mut(slot)?; - entry.preparing = None; - entry.pending_owner = None; - entry.pending_owner_closed = false; - firmware.render_grow_ready() - }; - self.shared.changed.notify_all(); - if grow_ready { - self.shared.queue_grow(); - } - result - } + } => self.install_parked_pair(&ctx, parked, buffer, pm_generation, metrics)?, Build::Rebind { - mut pair, + pair, buffer, pm_generation, metrics, - } => { - let prepared = (|| { - let free_list = - self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; - pair.prepare_bind( - &alloc, - &resources.objects, - self.owner, - self.context.clone(), - free_list, - metrics, - buffer, - pm_generation, - first, - ) - })(); - let (mut prepared, error) = match prepared { - Ok(binding) => (Some(binding), None), - Err(error) => (None, Some(error)), - }; - let (result, grow_ready) = { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - let result = (|| { - if !lease.is_current() { - return Err(EAGAIN); - } - if let Some(error) = error { - return Err(error); - } - if !self.context.is_current() || self.context.status().get() != 0 { - return Err(EFAULT); - } - let entry = firmware.queues.render.entry(slot)?; - if entry.pending_owner != Some(self.owner) || entry.pending_owner_closed { - return Err(ECANCELED); - } - if firmware.queues.render.retired.len() - == firmware.queues.render.retired.capacity() - { - return Err(ENOSPC); - } - firmware.queues.teardown.displace( - pair.context(), - true, - super::super::now_ns(), - )?; - pair.bind_owner( - prepared.as_mut().ok_or(EIO)?, - &alloc, - firmware.init.pb_descriptor_table()?, - )?; - if let Some((id, _)) = buffer { - firmware.queues.buffers.reserve_owner(id)?; - firmware.queues.buffers.release(id)?; - } - let entry = firmware.queues.render.entry_mut(slot)?; - entry.reservation.owner = self.owner; - if let Some(buffer) = buffer { - entry.reservation.buffer = buffer; - } - entry.context = Some(self.context.clone()); - entry.pool = Some(pair.pool().clone()); - entry.closed = false; - Ok(()) - })(); - // The original installed pair remains owned on every error. - let entry = firmware.queues.render.entry_mut(slot)?; - entry.preparing = None; - entry.pending_owner = None; - entry.pending_owner_closed = false; - entry.pair = Some(pair); - entry.borrowed_dependencies = None; - (result, firmware.render_grow_ready()) - }; - drop(prepared); - if result.is_err() { - if let Some((id, _)) = buffer { - let mut state = self.shared.state.lock(); - (*state) - .as_deref_mut() - .ok_or(ENODEV)? - .queues - .buffers - .release(id)?; - } + } => self.rebind_pair(&ctx, pair, buffer, pm_generation, metrics)?, + } + self.finish_render_build(&lease, slot, fresh_graph) + } + + /// Inlined into build_render_pair(): as a separate frame this step deepens the scheduler + /// run_job stack path, which is close to its budget. + #[inline(always)] + /// Off-lock construction of a fresh pair, then its device-locked install. + fn install_fresh_pair( + &self, + ctx: &BuildContext<'_>, + reservation: Reservation, + pm_generation: u64, + metrics: MetricsLease, + ) -> Result { + let BuildContext { + lease, + alloc, + resources, + slot, + first, + } = *ctx; + let built = (|| -> Result> { + let free_list = + self.context + .render_pool(&alloc, &resources.ids, &resources.global)?; + Pair::new( + &alloc, + &resources.objects, + reservation.ids, + slot, + self.owner, + self.context.clone(), + free_list, + metrics, + reservation.buffer, + pm_generation, + first, + resources.config.clusters, + resources.config.descriptor_flags, + ) + })(); + let (mut built, error) = match built { + Ok(pair) => (Some(pair), None), + Err(error) => (None, Some(error)), + }; + let result = (|| { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + if let Some(error) = error { + return Err(error); + } + if !self.context.is_current() || self.context.status().get() != 0 { + return Err(EFAULT); + } + if firmware.queues.render.reserved(reservation)?.closed { + return Err(ECANCELED); + } + built + .as_mut() + .ok_or(EIO)? + .finish_install(&alloc, firmware.init.pb_descriptor_table()?)?; + firmware.queues.install_render(reservation, &mut built) + })(); + // Fresh backing disappears before its temporary PB reference returns. + drop(built); + if result.is_err() { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .cancel_render_reservation(reservation)?; + } + result + } + + /// Inlined into build_render_pair(): as a separate frame this step deepens the scheduler + /// run_job stack path, which is close to its budget. + #[inline(always)] + /// Off-lock preparation of a parked pair for its new owner, then its device-locked + /// commit; the pair returns to its parked state on failure. + fn install_parked_pair( + &self, + ctx: &BuildContext<'_>, + mut parked: crate::g17::queue::render::parked::ParkedPair, + buffer: (u8, u64), + pm_generation: u64, + metrics: MetricsLease, + ) -> Result { + let BuildContext { + lease, + alloc, + resources, + slot, + first, + } = *ctx; + let built = (|| -> Result<_> { + let free_list = + self.context + .render_pool(&alloc, &resources.ids, &resources.global)?; + let prepared = parked.prepare( + &alloc, + &resources.objects, + self.owner, + self.context.clone(), + free_list, + metrics, + buffer, + pm_generation, + first, + )?; + let storage = KBox::::new_uninit(GFP_KERNEL)?; + Ok((prepared, storage)) + })(); + let (mut prepared, mut storage, error) = match built { + Ok((prepared, storage)) => (Some(prepared), Some(storage), None), + Err(error) => (None, None, Some(error)), + }; + let result = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + (|| { + if !lease.is_current() { + return Err(EAGAIN); + } + if let Some(error) = error { + return Err(error); } - self.shared.changed.notify_all(); - if grow_ready { - self.shared.queue_grow(); + if !self.context.is_current() || self.context.status().get() != 0 { + return Err(EFAULT); } - result + let entry = firmware.queues.render.entry(slot)?; + if entry.pending_owner != Some(self.owner) + || entry.pending_owner_closed + || entry.pair.is_some() + || entry.parked.is_some() + { + return Err(ECANCELED); + } + let storage = storage.take().ok_or(EIO)?; + firmware.queues.buffers.can_release(buffer.0)?; + prepared.as_mut().ok_or(EIO)?.finish_install( + &parked, + &alloc, + firmware.init.pb_descriptor_table()?, + )?; + let pair = parked.commit(&mut prepared, storage)?; + firmware.queues.buffers.reserve_owner(buffer.0)?; + firmware.queues.buffers.release(buffer.0)?; + let entry = firmware.queues.render.entry_mut(slot)?; + entry.reservation.owner = self.owner; + entry.reservation.buffer = buffer; + entry.context = Some(self.context.clone()); + entry.pool = Some(pair.pool().clone()); + entry.pair = Some(pair); + entry.closed = false; + entry.published = false; + Ok(()) + })() + }; + // Rejected candidate mappings disappear before the PB identity + // returns, while the original installed kick owner stays live. + drop(prepared); + drop(storage); + let grow_ready = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if result.is_err() { + firmware.queues.render.entry_mut(slot)?.parked = Some(parked); + firmware.queues.buffers.release(buffer.0)?; + } else { + drop(state); + drop(parked); + state = self.shared.state.lock(); } + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + let entry = firmware.queues.render.entry_mut(slot)?; + entry.preparing = None; + entry.pending_owner = None; + entry.pending_owner_closed = false; + firmware.render_grow_ready() }; - result?; + self.shared.changed.notify_all(); + if grow_ready { + self.shared.queue_grow(); + } + result + } + + /// Inlined into build_render_pair(): as a separate frame this step deepens the scheduler + /// run_job stack path, which is close to its budget. + #[inline(always)] + /// Off-lock preparation of an installed pair's rebinding, then its device-locked + /// owner change; the original pair stays installed on failure. + fn rebind_pair( + &self, + ctx: &BuildContext<'_>, + mut pair: KBox, + buffer: Option<(u8, u64)>, + pm_generation: Option, + metrics: Option, + ) -> Result { + let BuildContext { + lease, + alloc, + resources, + slot, + first, + } = *ctx; + let prepared = (|| { + let free_list = + self.context + .render_pool(&alloc, &resources.ids, &resources.global)?; + pair.prepare_bind( + &alloc, + &resources.objects, + self.owner, + self.context.clone(), + free_list, + metrics, + buffer, + pm_generation, + first, + ) + })(); + let (mut prepared, error) = match prepared { + Ok(binding) => (Some(binding), None), + Err(error) => (None, Some(error)), + }; + let (result, grow_ready) = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + let result = (|| { + if !lease.is_current() { + return Err(EAGAIN); + } + if let Some(error) = error { + return Err(error); + } + if !self.context.is_current() || self.context.status().get() != 0 { + return Err(EFAULT); + } + let entry = firmware.queues.render.entry(slot)?; + if entry.pending_owner != Some(self.owner) || entry.pending_owner_closed { + return Err(ECANCELED); + } + if firmware.queues.render.retired.len() + == firmware.queues.render.retired.capacity() + { + return Err(ENOSPC); + } + firmware.queues.teardown.displace( + pair.context(), + true, + super::super::now_ns(), + )?; + pair.bind_owner( + prepared.as_mut().ok_or(EIO)?, + &alloc, + firmware.init.pb_descriptor_table()?, + )?; + if let Some((id, _)) = buffer { + firmware.queues.buffers.reserve_owner(id)?; + firmware.queues.buffers.release(id)?; + } + let entry = firmware.queues.render.entry_mut(slot)?; + entry.reservation.owner = self.owner; + if let Some(buffer) = buffer { + entry.reservation.buffer = buffer; + } + entry.context = Some(self.context.clone()); + entry.pool = Some(pair.pool().clone()); + entry.closed = false; + Ok(()) + })(); + // The original installed pair remains owned on every error. + let entry = firmware.queues.render.entry_mut(slot)?; + entry.preparing = None; + entry.pending_owner = None; + entry.pending_owner_closed = false; + entry.pair = Some(pair); + entry.borrowed_dependencies = None; + (result, firmware.render_grow_ready()) + }; + drop(prepared); + if result.is_err() { + if let Some((id, _)) = buffer { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .buffers + .release(id)?; + } + } + self.shared.changed.notify_all(); + if grow_ready { + self.shared.queue_grow(); + } + result + } + + /// Under the device mutex: the graph is complete and retained; rebase a fresh graph's + /// cursors and warm its pool. + fn finish_render_build(&self, lease: &Lease, slot: u8, fresh_graph: bool) -> Result { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { From ea7162afed2c839ecdb670f2d1b379c99ddaf4db Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:42:34 -0400 Subject: [PATCH 0853/2680] drm/asahi: split the G17 render publication into preflight, barriers and announce Pair::publish() checked the pair, assembled the kick barriers, committed the descriptors and announced the pair to the firmware in one 350-line function with two near-identical failure blocks. Move the entry checks into publication_preflight(), the barrier assembly into render_barriers(), the announcement with its failure path into announce_publication(), and the shared tail of both failure paths (terminal status, deferred failure, failed flag) into record_publication_failure(). The checks, the barrier rules and the order of every effect on failure are unchanged. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e0fa28b70ca92d7942809827a7cff7e230d68826) --- drivers/gpu/drm/asahi/g17/queue/render.rs | 193 ++++++++++++++-------- 1 file changed, 128 insertions(+), 65 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 7936d253d2411..d82ee84c1283f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -401,6 +401,47 @@ impl Pair { } } +/// Kick barriers of one render. The fragment kick always waits for its own tiling kick; +/// with the fragment-barriers flag the resolved prefix orders the fragment kick too and the +/// tiling kick keeps only its implicit parent, otherwise the prefix orders the tiling kick. +/// A compute dependency always orders the tiling kick. +struct RenderBarriers { + tiling: [KickDependency; 3], + tiling_count: usize, + fragment: [KickDependency; 3], + fragment_count: usize, +} + +fn render_barriers( + fragment_stage: bool, + dependencies: &RenderDependencies, + own_tiling: KickDependency, +) -> RenderBarriers { + let mut tiling_barriers = [KickDependency::ZERO; 3]; + let mut tiling_count = 0; + let mut fragment_barriers = [own_tiling; 3]; + let mut fragment_count = 1; + for dependency in dependencies.render() { + if fragment_stage { + fragment_barriers[fragment_count] = *dependency; + fragment_count += 1; + } else { + tiling_barriers[tiling_count] = *dependency; + tiling_count += 1; + } + } + if let Some(compute) = dependencies.compute { + tiling_barriers[tiling_count] = compute; + tiling_count += 1; + } + RenderBarriers { + tiling: tiling_barriers, + tiling_count, + fragment: fragment_barriers, + fragment_count, + } +} + impl Pair { /// All descriptor bytes and retained allocations are ready before entry. /// The first inner producer establishes firmware ownership, including on @@ -422,37 +463,9 @@ impl Pair { }, }; use core::sync::atomic::{fence, Ordering}; - if self.pending_aliases.is_some() || self.kick_aliases.is_some() { - return Err(EBUSY); - } - let (epoch, recovery_state) = host.epoch()?; - if recovery_state != 0 - || self.quarantined - || !self.active.room() - || packet.completion.render_slot() != Some(self.slot) - { - return Err(EBUSY); - } - let prepared = self.prepared.as_ref().ok_or(EINVAL)?; - if !Arc::ptr_eq(&prepared.packet, &packet) - || prepared.ordinal != self.ordinal - || !self.memory.owns(self.ordinal) - || !packet.context.is_current() - { - return Err(EIO); - } - if packet.completion.status().get() != 0 { - return Err(EIO); - } - let policy = packet.context.policy(); - // A registration of another class can only follow a rebind, which - // requires the pair to have drained. - if self.registered.is_some_and(|class| class != policy.priority()) && self.in_flight() { - return Err(EIO); - } + let (epoch, mcache, policy) = self.publication_preflight(&packet, host)?; let ordinal = self.ordinal; let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; - let mcache = prepared.mcache; let qids = self.qids(); let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); let parents = self.kicks.each_ref().map(kick::Queue::parent); @@ -534,32 +547,12 @@ impl Pair { self.memory.graph.queues.gpu_va() + u64::from(qid) * size_of::() as u64 }); - // The fragment kick always waits for its own tiling kick. With the - // fragment-barriers flag the resolved prefix orders the fragment kick - // too, and the tiling kick keeps only its implicit parent, so vertex - // work overlaps the predecessors' fragment work as on the native stack. let Validated::Render { pass, .. } = &packet.command else { return Err(EINVAL); }; let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; - let mut tiling_barriers = [KickDependency::ZERO; 3]; - let mut tiling_count = 0; - let mut fragment_barriers = [own_tiling; 3]; - let mut fragment_count = 1; - for dependency in dependencies.render() { - if pass.fragment_barriers { - fragment_barriers[fragment_count] = *dependency; - fragment_count += 1; - } else { - tiling_barriers[tiling_count] = *dependency; - tiling_count += 1; - } - } - if let Some(compute) = dependencies.compute { - tiling_barriers[tiling_count] = compute; - tiling_count += 1; - } - let tiling_barriers = &tiling_barriers[..tiling_count]; + let barriers = render_barriers(pass.fragment_barriers, dependencies, own_tiling); + let tiling_barriers = &barriers.tiling[..barriers.tiling_count]; let args = |stage: usize, barriers, arrays, mcache| KickArgs { qid: qids[stage], timestamp: kicks[stage], @@ -589,7 +582,7 @@ impl Pair { ); let fragment_args = args( 1, - &fragment_barriers[..fragment_count], + &barriers.fragment[..barriers.fragment_count], abi::FragmentDescriptor::register_bindings( self.memory.graph.descriptor_client(1, ordinal), )?, @@ -727,11 +720,7 @@ impl Pair { } self.active.back_mut().ok_or(EIO)?.qos_completed = true; self.quarantined = true; - if !packet.completion.spared() && self.terminal.is_none() { - self.terminal = Some((packet.completion.status().clone(), error)); - } - defer(Deferred::FailedOwned(packet.completion.clone(), error))?; - self.active.back_mut().ok_or(EIO)?.failed = true; + self.record_publication_failure(&packet, error, defer)?; return Err(error); } }; @@ -749,11 +738,89 @@ impl Pair { previous.publication = Some(ordinal); } packet.context.mark_published(); + self.announce_publication( + host, + (&fragment_slot, &tiling_slot, outer_next), + lease.free_list_generation, + policy.priority(), + &packet, + defer, + ) + } + + /// Everything that must hold before a publication changes any state: no alias work + /// pending, a healthy device epoch, room in the active ring, this slot's packet, the + /// prepared descriptors for this ordinal, a clean completion status and a registration + /// class the pair can still take. Returns the epoch, the prepared memory-cache setting + /// and the owner's policy. + fn publication_preflight( + &self, + packet: &Arc, + host: &mut impl Host, + ) -> Result<(u64, Option, crate::g17::fw::queue::Policy)> { + if self.pending_aliases.is_some() || self.kick_aliases.is_some() { + return Err(EBUSY); + } + let (epoch, recovery_state) = host.epoch()?; + if recovery_state != 0 + || self.quarantined + || !self.active.room() + || packet.completion.render_slot() != Some(self.slot) + { + return Err(EBUSY); + } + let prepared = self.prepared.as_ref().ok_or(EINVAL)?; + if !Arc::ptr_eq(&prepared.packet, packet) + || prepared.ordinal != self.ordinal + || !self.memory.owns(self.ordinal) + || !packet.context.is_current() + { + return Err(EIO); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let policy = packet.context.policy(); + // A registration of another class can only follow a rebind, which + // requires the pair to have drained. + if self.registered.is_some_and(|class| class != policy.priority()) && self.in_flight() { + return Err(EIO); + } + Ok((epoch, prepared.mcache, policy)) + } + + /// Records a failed publication on the retained active entry: the terminal status + /// (once), the deferred failure and the entry's failed flag. + fn record_publication_failure( + &mut self, + packet: &Arc, + error: Error, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + if !packet.completion.spared() && self.terminal.is_none() { + self.terminal = Some((packet.completion.status().clone(), error)); + } + defer(Deferred::FailedOwned(packet.completion.clone(), error))?; + self.active.back_mut().ok_or(EIO)?.failed = true; + Ok(()) + } + + /// Activates the pool lease and announces the retained pair to the firmware; a failure + /// here quarantines the pair with the packet still retained for recovery. + fn announce_publication( + &mut self, + host: &mut impl Host, + (fragment_slot, tiling_slot, outer_next): (&WorkSlot, &WorkSlot, [u32; 2]), + free_list_generation: u64, + priority: u8, + packet: &Arc, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { let publish = (|| { - host.activate_render_pool(self.free_list.id(), lease.free_list_generation)?; - host.publish_retained_pair(policy.priority(), &fragment_slot, &tiling_slot, outer_next)?; + host.activate_render_pool(self.free_list.id(), free_list_generation)?; + host.publish_retained_pair(priority, fragment_slot, tiling_slot, outer_next)?; self.active.back_mut().ok_or(EIO)?.outer_published = true; - host.notify_render(policy.priority())?; + host.notify_render(priority)?; host.note_submission() })(); if let Err(error) = publish { @@ -764,11 +831,7 @@ impl Pair { .packet .completion .quarantine_render(); - if !packet.completion.spared() && self.terminal.is_none() { - self.terminal = Some((packet.completion.status().clone(), error)); - } - defer(Deferred::FailedOwned(packet.completion.clone(), error))?; - self.active.back_mut().ok_or(EIO)?.failed = true; + self.record_publication_failure(packet, error, defer)?; return Err(error); } Ok(()) From 11a568f3c49331ec23bcbacf04658e985e9a88b7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:42:34 -0400 Subject: [PATCH 0854/2680] drm/asahi: split the G17 compute publication preflight and queue configuration Queue::publish() checked the queue, built the descriptor arguments and the queue configuration record, and committed and announced the kick in one 330-line function. Move the entry checks into publication_preflight(), which hands back the epoch and the bound owner's per-queue addresses as values, and the configuration record into queue_config(). The commit and announcement stay one closure: its visible/accounted state decides the rollback for both phases. Checks, record contents and effect order are unchanged. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 74bc2f09ab74832be661557cacf374ee5fff11d7) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 144 +++++++++++++-------- 1 file changed, 92 insertions(+), 52 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 2b962e2632606..5347779d629d1 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -392,6 +392,13 @@ struct Previous { context: Arc, publication: Option<(u64, u64)>, } +/// Per-queue addresses of the bound owner, read under the publication preflight. +struct BindingAddresses { + preempt: u64, + operand: u64, + usage: u64, +} + #[derive(Copy, Clone, PartialEq, Eq)] struct ConfigIdentity { scheduler: u64, @@ -1495,31 +1502,7 @@ impl Queue { compute::{ComputeArgs, ComputeDescriptor, COMPUTE_KICK_EVENT_MASK, COMPUTE_QOS_CLASS}, kick::{KickArgs, KickQos}, }; - let (epoch, state) = host.epoch()?; - if state != 0 { - return Err(EBUSY); - } - let replay_front = self - .replays - .front() - .is_some_and(|front| Arc::ptr_eq(front, &packet)); - if !self.room() - && !(replay_front && !self.quarantined && !self.released && self.active.len < IN_FLIGHT) - { - return Err(EBUSY); - } - if !packet.context.is_current() - || !self - .binding - .as_ref() - .is_some_and(|binding| Arc::ptr_eq(&packet.context, &binding.context)) - { - return Err(EFAULT); - } - let binding = self.binding.as_ref().ok_or(EFAULT)?; - if packet.completion.status().get() != 0 { - return Err(EIO); - } + let (epoch, addresses) = self.publication_preflight(&packet, host)?; let crate::g17::command::Validated::Compute { scratch, usc_base, @@ -1569,9 +1552,9 @@ impl Queue { qid, kick_count: self.ordinal, kick: kick_timestamp, - preempt_va: binding.preempt.iova(), - operand_state_va: binding.operand.iova(), - usage_va: binding.usage.iova(), + preempt_va: addresses.preempt, + operand_state_va: addresses.operand, + usage_va: addresses.usage, usage_fw_va: self.graph.usage.gpu_va(), free_list_slot: u32::from(self.pool.id()), free_list_control_va: self.pool.control_va(), @@ -1591,30 +1574,7 @@ impl Queue { self.free_list_generation = u64::from(packet.context.scheduler_generation()); let policy = packet.context.policy(); let ring = 1u8 << policy.priority(); - let config = QueueConfig::new(&QueueConfigArgs { - target: QueueConfigTarget::Compute, - kick_ring_va: self.kick.low_va(), - kick_ring_fw_va: self.kick.firmware_va(), - qid, - install: first, - completion_seed: None, - context_id: args.context_id, - free_list: FreeListBinding { - va: self.pool.control_va(), - generation: u64::from(packet.context.scheduler_generation()), - slot: u32::from(self.pool.id()), - }, - scheduler_va: identity.scheduler, - policy, - qos_slot: identity.qos, - qos_update: self - .published_config - .is_none_or(|p| p.scheduler != identity.scheduler || p.qos != identity.qos), - owner_pid: packet.context.owner_pid(), - context_update: self - .published_config - .is_none_or(|p| p.scheduler != identity.scheduler || p.context != identity.context), - })?; + let config = self.queue_config(&identity, first, args.context_id, policy, &packet)?; let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; let predecessor = KickPredecessor::new(qid, kick_parent)?; @@ -1834,6 +1794,86 @@ impl Queue { } Ok(()) } + + /// Everything that must hold before a publication changes any state: a healthy device + /// epoch, room in the active ring (or this packet at the replay front), the bound + /// owner's current context and a clean completion status. Returns the epoch and the + /// bound owner's per-queue addresses. + fn publication_preflight( + &self, + packet: &Arc, + host: &mut impl Host, + ) -> Result<(u64, BindingAddresses)> { + let (epoch, state) = host.epoch()?; + if state != 0 { + return Err(EBUSY); + } + let replay_front = self + .replays + .front() + .is_some_and(|front| Arc::ptr_eq(front, packet)); + if !self.room() + && !(replay_front && !self.quarantined && !self.released && self.active.len < IN_FLIGHT) + { + return Err(EBUSY); + } + if !packet.context.is_current() + || !self + .binding + .as_ref() + .is_some_and(|binding| Arc::ptr_eq(&packet.context, &binding.context)) + { + return Err(EFAULT); + } + let binding = self.binding.as_ref().ok_or(EFAULT)?; + if packet.completion.status().get() != 0 { + return Err(EIO); + } + Ok(( + epoch, + BindingAddresses { + preempt: binding.preempt.iova(), + operand: binding.operand.iova(), + usage: binding.usage.iova(), + }, + )) + } + + /// The queue configuration record for this publication: a fresh install on the first + /// one, and QoS/context updates only when the identity changed since the last publication. + fn queue_config( + &self, + identity: &ConfigIdentity, + first: bool, + context_id: u16, + policy: crate::g17::fw::queue::Policy, + packet: &Arc, + ) -> Result { + QueueConfig::new(&QueueConfigArgs { + target: QueueConfigTarget::Compute, + kick_ring_va: self.kick.low_va(), + kick_ring_fw_va: self.kick.firmware_va(), + qid: self.qid(), + install: first, + completion_seed: None, + context_id, + free_list: FreeListBinding { + va: self.pool.control_va(), + generation: u64::from(packet.context.scheduler_generation()), + slot: u32::from(self.pool.id()), + }, + scheduler_va: identity.scheduler, + policy, + qos_slot: identity.qos, + qos_update: self + .published_config + .is_none_or(|p| p.scheduler != identity.scheduler || p.qos != identity.qos), + owner_pid: packet.context.owner_pid(), + context_update: self + .published_config + .is_none_or(|p| p.scheduler != identity.scheduler || p.context != identity.context), + }) + } } impl Graph { From 529f27be24d498eaa408217ceb2b631b0fd6113c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 23:13:38 -0400 Subject: [PATCH 0855/2680] drm/asahi: size the compute descriptor ring to the kicks that can be in flight Each retained compute queue carried a 4 MiB descriptor ring: 256 slots of 16 KiB, one per kick-ring slot, although at most IN_FLIGHT (16) kicks are outstanding and every kick names its descriptor by address. Index the ring by kick slot modulo 32, so a descriptor is rewritten only once the kick 32 publications later is issued, long after its own completion. The ring shrinks to 512 KiB per queue; with the fourteen queues of a desktop and a game that is 49 MiB less pinned memory. Retirement records match on descriptor address and kick timestamp together, so a reused address cannot alias an older kick. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5ad7b191f7e698e3ef140163c6ecf15d3e57ff85) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 5347779d629d1..10b66c9ae811b 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -19,7 +19,9 @@ use retirement::{Cursors, RECORD_SLOTS}; use crate::g17::now_ns; -const DESCRIPTORS: usize = 256; +/// Descriptor slots per queue. A kick's descriptor is rewritten when the kick 32 slots later +/// is published; with IN_FLIGHT kicks outstanding at most, that kick has completed by then. +const DESCRIPTORS: usize = 32; const GRAPH_SIZE: usize = 0x8000; const CONTEXT_SIZE: usize = 0x20000; const RECORD_STRIDE: usize = 0x1000; @@ -231,7 +233,7 @@ impl Graph { /// EAGAIN while the queue is vacant: the owner must re-arm it first. fn descriptor_vas(&self, slot: u8) -> Result<(u64, u64)> { let ring = self.ring.as_ref().ok_or(EAGAIN)?; - let offset = u64::from(slot) * COMPUTE_DESCRIPTOR_SIZE as u64; + let offset = (usize::from(slot) % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE) as u64; Ok((ring.high.iova() + offset, ring.object.gpu_va() + offset)) } fn descriptor_alias(&self, vm: &mmu::Vm) -> Result { @@ -1620,7 +1622,7 @@ impl Queue { .ok_or(EAGAIN)? .object .initialize::( - kick_timestamp.slot() * COMPUTE_DESCRIPTOR_SIZE, + kick_timestamp.slot() % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, |descriptor| descriptor.write(&args), )?; if first { From 7d200e3634b0ac4fe5183869ab2b2b6454a1e46b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0856/2680] kunit: cover mixed-page ownership and translation lifetimes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add internal MM, table, subpage and recorded VFIO tests with separate opt-in controls. Translation fixtures require a disposable guest and recorded device mappings do not establish physical DMA qualification. [ryan: adapted to 7.2.10 — minimal; for DJ: futex_wake() gained a robust list pop argument (3ca9595d9fb6, NULL here) and bio_iov_iter_get_pages() a mem_align_mask argument (0 here)] Signed-off-by: Diljit Singh (cherry picked from commit e86efe2f2aaec4012ba91165f5211ca8c9c7716c) --- arch/arm64/Kconfig.debug | 70 + arch/arm64/mm/Makefile | 2 + arch/arm64/mm/user4k-data-test.c | 1835 ++ arch/arm64/mm/user4k-test.c | 14239 ++++++++++++++++ arch/arm64/mm/user4k-tlb-test.c | 184 + drivers/vfio/Kconfig | 12 + drivers/vfio/vfio_iommu_type1_account_test.c | 192 + .../vfio/vfio_iommu_type1_fragments_test.c | 1177 ++ drivers/vfio/vfio_iommu_type1_ioctl_test.c | 305 + drivers/vfio/vfio_iommu_type1_test.h | 36 + 10 files changed, 18052 insertions(+) create mode 100644 arch/arm64/mm/user4k-data-test.c create mode 100644 arch/arm64/mm/user4k-test.c create mode 100644 arch/arm64/mm/user4k-tlb-test.c create mode 100644 drivers/vfio/vfio_iommu_type1_account_test.c create mode 100644 drivers/vfio/vfio_iommu_type1_fragments_test.c create mode 100644 drivers/vfio/vfio_iommu_type1_ioctl_test.c create mode 100644 drivers/vfio/vfio_iommu_type1_test.h diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index d15ccdeee2125..58d7a1dc98405 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -42,3 +42,73 @@ config ARM64_USER4K_EXPERIMENTAL still require integration and validation. The private prctl numbers are not upstream assignments or a stable ABI promise. Say N for normal kernels. + +config ARM64_USER4K_KUNIT_TEST + bool "Test mixed userspace granules with larger kernel pages" + depends on KUNIT=y && ARM64_USER4K_EXPERIMENTAL + depends on ARM64_PA_BITS_48 + select ARM64_USER4K_DATA_KUNIT_TEST + select IOMMUFD + select IOMMUFD_DRIVER + default n + help + Exercise independent TTBR0 translation granules, subpage permissions + and invalidation while TTBR1 retains the normal native kernel tables. + Hardware translation/uaccess fixtures require arm64.user4k_test=1 + as a second opt-in and are disabled with KASAN. Software MM fixtures + can run with KASAN without borrowing or replacing TTBR0. + Run only in a disposable test VM. This regression harness does not + establish hardware qualification or production ABI readiness. + Say N for normal kernels. + +config ARM64_USER4K_IOMMUFD_TEST + bool "IOMMUFD ioctl harness for mixed userspace granules" + depends on ARM64_USER4K_EXPERIMENTAL && DEBUG_KERNEL + depends on FAULT_INJECTION && RUNTIME_TESTING_MENU + select GENERIC_PT + select IOMMU_PT + select IOMMU_PT_AMDV1 + select IOMMUFD + select IOMMUFD_TEST + default n + help + Enable the IOMMUFD mock-device ioctls and generic AMDv1 page tables + for the mixed-granule userspace ioctl fixture. This exposes privileged + test interfaces; use only in a disposable test VM. + Say N for normal kernels. + +config ARM64_USER4K_VFIO_ACCOUNT_TEST + bool "VFIO type1 pinning harness for mixed userspace granules" + depends on ARM64_USER4K_EXPERIMENTAL && KUNIT=y + select VFIO + select VFIO_GROUP + select VFIO_CONTAINER + select VFIO_IOMMU_TYPE1 + select VFIO_TYPE1_ACCOUNT_KUNIT_TEST + default n + help + Enable built-in VFIO type1 accounting and retained-fragment tests + in a disposable mixed-granule guest. Recorded IOMMU translations + do not establish physical-device DMA support. + Say N for normal kernels. + +config ARM64_USER4K_VFIO_IOCTL_TEST + bool "Private VFIO type1 ioctl harness for mixed userspace granules" + depends on ARM64_USER4K_EXPERIMENTAL && KUNIT=y && DEBUG_KERNEL + depends on RUNTIME_TESTING_MENU + select ARM64_USER4K_VFIO_ACCOUNT_TEST + default n + help + Expose a root-only disposable-guest misc device forwarding real EL0 + arguments to type1 ioctls and supplying recording IOMMU domains. + These privileged test controls perform no physical device DMA. + Say N for normal kernels. + +config ARM64_USER4K_DATA_KUNIT_TEST + bool "Test packed userspace allocation ownership" + depends on KUNIT=y && ARM64_USER4K_EXPERIMENTAL + default n + help + Exercise packed allocation, slot reuse, copying and deferred lifetime. + These tests use native kernel addresses without changing TTBR0, and + can run with KASAN even when the architectural tests are disabled. diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile index 32d18488e527c..757f7eb07ccac 100644 --- a/arch/arm64/mm/Makefile +++ b/arch/arm64/mm/Makefile @@ -12,6 +12,8 @@ obj-$(CONFIG_TRANS_TABLE) += trans_pgd-asm.o obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o obj-$(CONFIG_ARM64_MTE) += mteswap.o obj-$(CONFIG_ARM64_GCS) += gcs.o +obj-$(CONFIG_ARM64_USER4K_KUNIT_TEST) += user4k-test.o user4k-tlb-test.o +obj-$(CONFIG_ARM64_USER4K_DATA_KUNIT_TEST) += user4k-data-test.o obj-$(CONFIG_ARM64_USER4K_EXPERIMENTAL) += user4k-pgalloc.o KASAN_SANITIZE_physaddr.o += n diff --git a/arch/arm64/mm/user4k-data-test.c b/arch/arm64/mm/user4k-data-test.c new file mode 100644 index 0000000000000..d7d550c9e1420 --- /dev/null +++ b/arch/arm64/mm/user4k-data-test.c @@ -0,0 +1,1835 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Backing ownership tests; these slots are not fault/rmap-owned mappings. */ +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_NET +#include +#if IS_ENABLED(CONFIG_NET_SOCK_MSG) && IS_ENABLED(CONFIG_INET) +#include +#endif +#ifdef CONFIG_INET +#include +#include +#endif +#include +#endif +#include +#include + +#define DATA_SLOTS MM_SUBPAGES_PER_PAGE + +struct data_fixture { + struct mm_subpage_pool *pool; + struct mm_subpage *held[2 * DATA_SLOTS + 2]; +}; + +static void data_fixture_free(void *arg) +{ + struct data_fixture *fixture = arg; + unsigned int i; + + if (fixture->pool) + mm_subpage_pool_close(fixture->pool); + for (i = 0; i < ARRAY_SIZE(fixture->held); i++) + if (fixture->held[i]) + mm_subpage_put(fixture->held[i]); + if (fixture->pool) + mm_subpage_pool_put(fixture->pool); +} + +static struct data_fixture *data_fixture_create_granule(struct kunit *test, unsigned int shift) +{ + struct data_fixture *fixture = kunit_kzalloc(test, sizeof(*fixture), GFP_KERNEL); + + if (!fixture) + return NULL; + fixture->pool = mm_subpage_pool_create_granule(GFP_KERNEL, shift); + if (!fixture->pool) + return NULL; + if (kunit_add_action_or_reset(test, data_fixture_free, fixture)) + return NULL; + return fixture; +} + +static struct data_fixture *data_fixture_create(struct kunit *test) +{ + return data_fixture_create_granule(test, MM_SUBPAGE_SHIFT); +} + +static void put_held(struct data_fixture *fixture, unsigned int index) +{ + mm_subpage_put(fixture->held[index]); + fixture->held[index] = NULL; +} + +static struct mm_subpage *hold_alloc(struct data_fixture *fixture, unsigned int index) +{ + struct mm_subpage *slot = mm_subpage_alloc(fixture->pool); + + if (!IS_ERR(slot)) + fixture->held[index] = slot; + return slot; +} + +static int supply_backing(struct mm_subpage_pool *pool) +{ + struct folio *folio = folio_alloc(GFP_KERNEL, 0); + int ret; + + if (!folio) + return -ENOMEM; + /* Check that stale allocation contents never become exposed. */ + memset(folio_address(folio), 0xcc, PAGE_SIZE); + ret = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (ret) + folio_put(folio); + return ret; +} + +static int supply_backing_at(struct mm_subpage_pool *pool, unsigned int offset) +{ + struct folio *folio = folio_alloc(GFP_KERNEL, 0); + int ret; + + if (!folio) + return -ENOMEM; + ret = mm_subpage_pool_add_folio_at(pool, folio, offset, GFP_KERNEL); + if (ret) + folio_put(folio); + return ret; +} + +static struct mm_subpage *hold_alloc_at(struct data_fixture *fixture, unsigned int index, + unsigned int offset) +{ + struct mm_subpage *slot = mm_subpage_alloc_at(fixture->pool, offset); + + if (!IS_ERR(slot)) + fixture->held[index] = slot; + return slot; +} + +static void *slot_address(struct mm_subpage *slot) +{ + return folio_address(mm_subpage_folio(slot)) + mm_subpage_offset(slot); +} + +static void user4k_data_reuse_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct folio *folio; + struct mm_subpage *slot; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -EAGAIN); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + KUNIT_EXPECT_EQ(test, supply_backing(fixture->pool), -EEXIST); + for (i = 0; i < DATA_SLOTS; i++) { + slot = hold_alloc(fixture, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + folio = mm_subpage_folio(slot); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)i + 1); + KUNIT_EXPECT_PTR_EQ(test, folio, mm_subpage_folio(fixture->held[0])); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * SZ_4K); + KUNIT_EXPECT_EQ(test, mm_subpage_phys(slot), PFN_PHYS(folio_pfn(folio)) + i * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0, SZ_4K), NULL); + memset(slot_address(slot), 0x10 + i, SZ_4K); + } + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 1UL); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -EAGAIN); + /* A pin, alias or deferred-TLB holder keeps this slot unavailable. */ + fixture->held[DATA_SLOTS] = fixture->held[1]; + mm_subpage_get(fixture->held[DATA_SLOTS]); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)DATA_SLOTS + 1); + put_held(fixture, 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)DATA_SLOTS); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -EAGAIN); + put_held(fixture, DATA_SLOTS); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)DATA_SLOTS - 1); + slot = hold_alloc(fixture, DATA_SLOTS + 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), (unsigned int)SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0, SZ_4K), NULL); + for (i = 0; i < DATA_SLOTS; i++) { + if (i != 1) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(fixture->held[i]), + 0x10 + i, SZ_4K), NULL); + } + mm_subpage_pool_close(fixture->pool); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -ESHUTDOWN); + KUNIT_EXPECT_EQ(test, supply_backing(fixture->pool), -ESHUTDOWN); + for (i = 0; i < DATA_SLOTS; i++) + if (i != 1) + put_held(fixture, i); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 1UL); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + put_held(fixture, DATA_SLOTS + 1); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +static void user4k_data_copy_test(struct kunit *test) +{ + struct data_fixture *source = data_fixture_create(test); + struct data_fixture *dest = data_fixture_create(test); + struct mm_subpage *copy; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, source); + KUNIT_ASSERT_NOT_NULL(test, dest); + KUNIT_ASSERT_EQ(test, supply_backing(source->pool), 0); + KUNIT_ASSERT_EQ(test, supply_backing(dest->pool), 0); + for (i = 0; i < DATA_SLOTS; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(source, i)); + memset(slot_address(source->held[i]), 0x50 + i, SZ_4K); + } + source->held[DATA_SLOTS] = source->held[1]; + mm_subpage_get(source->held[DATA_SLOTS]); + copy = mm_subpage_copy(dest->pool, source->held[1]); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + dest->held[0] = copy; + KUNIT_EXPECT_NE(test, mm_subpage_phys(copy), mm_subpage_phys(source->held[1])); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(copy), 0x51, SZ_4K), NULL); + memset(slot_address(copy), 0xee, SZ_4K); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(source->held[i]), 0x50 + i, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy(source->pool, source->held[1])), -EAGAIN); + /* Copy into a free neighboring slot in the same physical owner. */ + put_held(source, 2); + copy = mm_subpage_copy(source->pool, source->held[1]); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + source->held[2] = copy; + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(copy), mm_subpage_folio(source->held[1])); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(copy), 2U * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(copy), 0x51, SZ_4K), NULL); + memset(slot_address(copy), 0xdd, SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(source->held[1]), 0x51, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(source->held[3]), 0x53, SZ_4K), NULL); +} + +static void user4k_data_lookup_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct mm_subpage *slot, *found; + struct folio *folio; + phys_addr_t phys; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + folio = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + phys = PFN_PHYS(folio_pfn(folio)); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys), NULL); + if (mm_subpage_pool_add_folio(fixture->pool, folio, GFP_KERNEL)) { + folio_put(folio); + KUNIT_FAIL(test, "failed to supply lookup backing"); + return; + } + /* A published owner does not expose uninitialised or unissued slots. */ + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys + i * SZ_4K), NULL); + for (i = 0; i < DATA_SLOTS; i++) { + slot = hold_alloc(fixture, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + memset(slot_address(slot), 0x90 + i, SZ_4K); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 2 * (int)i + 1); + if (!i) + KUNIT_EXPECT_EQ(test, mm_subpage_pool_add_folio(fixture->pool, folio, + GFP_KERNEL), -EEXIST); + found = mm_subpage_get_from_phys(phys + i * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, found); + fixture->held[i + DATA_SLOTS] = found; + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 2 * (int)i + 2); + KUNIT_EXPECT_PTR_EQ(test, found, slot); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys + i * SZ_4K + 1), NULL); + } + /* A second pool must not zero or acquire already owned backing. */ + KUNIT_EXPECT_EQ(test, mm_subpage_pool_add_folio(fixture->pool, folio, GFP_KERNEL), + -EINVAL); + for (i = 0; i < DATA_SLOTS; i++) { + put_held(fixture, i); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(fixture->held[i + DATA_SLOTS]), + 0x90 + i, SZ_4K), NULL); + } + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -EAGAIN); + put_held(fixture, DATA_SLOTS + 1); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys + SZ_4K), NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(fixture, 1)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(fixture->held[1]), 0, SZ_4K), NULL); + mm_subpage_pool_close(fixture->pool); + /* Keep the physical allocation stable while checking final unbinding. */ + folio_get(folio); + put_held(fixture, 1); + for (i = 0; i < DATA_SLOTS; i++) + if (i != 1) + put_held(fixture, i + DATA_SLOTS); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys + i * SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + folio_put(folio); +} + +static void user4k_data_exact_slot_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct mm_subpage *source, *copy, *reused; + struct folio *first; + static const unsigned int neighbors[] = { 0, SZ_4K, 3 * SZ_4K }; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_EXPECT_EQ(test, supply_backing_at(fixture->pool, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(fixture->pool, PAGE_SIZE)), -EINVAL); + KUNIT_ASSERT_EQ(test, supply_backing_at(fixture->pool, SZ_8K), 0); + source = hold_alloc_at(fixture, 0, SZ_8K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, source); + first = mm_subpage_folio(source); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(source), (unsigned int)SZ_8K); + memset(slot_address(source), 0xa7, SZ_4K); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy_at(fixture->pool, source, 1)), -EINVAL); + /* Three free neighbors cannot satisfy a request for this occupied slot. */ + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy_at(fixture->pool, source, SZ_8K)), -EAGAIN); + KUNIT_EXPECT_EQ(test, supply_backing(fixture->pool), -EEXIST); + KUNIT_ASSERT_EQ(test, supply_backing_at(fixture->pool, SZ_8K), 0); + KUNIT_EXPECT_EQ(test, supply_backing_at(fixture->pool, SZ_8K), -EEXIST); + copy = mm_subpage_copy_at(fixture->pool, source, SZ_8K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + fixture->held[1] = copy; + KUNIT_EXPECT_PTR_NE(test, mm_subpage_folio(copy), first); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(copy), (unsigned int)SZ_8K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(copy), 0xa7, SZ_4K), NULL); + memset(slot_address(copy), 0x4b, SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(source), 0xa7, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 2UL); + for (i = 0; i < ARRAY_SIZE(neighbors); i++) { + struct mm_subpage *slot = hold_alloc_at(fixture, i + 2, neighbors[i]); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(slot), first); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), neighbors[i]); + memset(slot_address(slot), 0x30 + i, SZ_4K); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(first), 4); + fixture->held[5] = source; + mm_subpage_get(source); + put_held(fixture, 0); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(fixture->pool, SZ_8K)), -EAGAIN); + put_held(fixture, 5); + reused = hold_alloc_at(fixture, 6, SZ_8K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, reused); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(reused), first); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(reused), 0, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, folio_ref_count(first), 4); + for (i = 0; i < ARRAY_SIZE(neighbors); i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(fixture->held[i + 2]), + 0x30 + i, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(copy), 0x4b, SZ_4K), NULL); + mm_subpage_pool_close(fixture->pool); + for (i = 2; i <= 4; i++) + put_held(fixture, i); + put_held(fixture, 6); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 1UL); + put_held(fixture, 1); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +static void user4k_data_close_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct mm_subpage *slot; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + /* Closing with no issued slots must release that entire unused owner. */ + mm_subpage_pool_close(fixture->pool); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); + fixture = data_fixture_create(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + slot = hold_alloc(fixture, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + memset(slot_address(slot), 0xa7, SZ_4K); + mm_subpage_pool_close(fixture->pool); + mm_subpage_pool_put(fixture->pool); + fixture->pool = NULL; + /* The originating pool/mm reference is gone; a child's slot remains live. */ + fixture->held[1] = slot; + mm_subpage_get(slot); + put_held(fixture, 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0xa7, SZ_4K), NULL); + put_held(fixture, 1); +} + +struct deferred_slot { + struct rcu_head rcu; + struct mm_subpage *slot; + struct completion done; +}; + +static void deferred_slot_put(struct rcu_head *rcu) +{ + struct deferred_slot *pending = container_of(rcu, struct deferred_slot, rcu); + + mm_subpage_put(pending->slot); + complete(&pending->done); +} + +static void data_rcu_barrier(void *unused) +{ + rcu_barrier(); +} + +static void user4k_data_deferred_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct deferred_slot *pending = kunit_kzalloc(test, sizeof(*pending), GFP_KERNEL); + struct mm_subpage *probe; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_NOT_NULL(test, pending); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(fixture, i)); + init_completion(&pending->done); + pending->slot = fixture->held[0]; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, data_rcu_barrier, NULL), 0); + mm_subpage_get(pending->slot); + rcu_read_lock(); + call_rcu(&pending->rcu, deferred_slot_put); + put_held(fixture, 0); + /* The delayed reference must prevent recycling before the grace period. */ + probe = mm_subpage_alloc(fixture->pool); + for (i = 1; i < DATA_SLOTS; i++) + put_held(fixture, i); + rcu_read_unlock(); + if (!IS_ERR(probe)) + fixture->held[DATA_SLOTS] = probe; + KUNIT_EXPECT_EQ(test, PTR_ERR(probe), -EAGAIN); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&pending->done, 5 * HZ), 0UL); + /* The callback retired the final live slot and the whole native owner. */ + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +static void user4k_data_backing_reject_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct folio *folio; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + folio = folio_alloc(GFP_KERNEL, 1); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = mm_subpage_pool_add_folio(fixture->pool, folio, GFP_KERNEL); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + if (ret) + folio_put(folio); + folio = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_get(folio); + ret = mm_subpage_pool_add_folio(fixture->pool, folio, GFP_KERNEL); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 2); + folio_put(folio); + if (ret) + folio_put(folio); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +struct data_worker { + struct mm_subpage_pool *pool; + struct task_struct *task; + struct completion ready, go, done; + unsigned int id, checks; + int error; +}; + +static struct mm_subpage *alloc_with_backing(struct mm_subpage_pool *pool) +{ + unsigned int tries; + + for (tries = 0; tries < 128; tries++) { + struct mm_subpage *slot = mm_subpage_alloc(pool); + int ret; + + if (!IS_ERR(slot) || PTR_ERR(slot) != -EAGAIN) + return slot; + ret = supply_backing(pool); + if (ret && ret != -EEXIST) + return ERR_PTR(ret); + cond_resched(); + } + return ERR_PTR(-EAGAIN); +} + +static int data_worker_run(void *arg) +{ + struct data_worker *worker = arg; + struct mm_subpage *slot = NULL; + unsigned int round; + u8 pattern = 0; + + for (round = 0; round < 64; round++) { + slot = alloc_with_backing(worker->pool); + if (IS_ERR(slot)) { + worker->error = PTR_ERR(slot); + slot = NULL; + break; + } + pattern = 1 + round + worker->id * 64; + memset(slot_address(slot), pattern, SZ_4K); + mm_subpage_get(slot); + mm_subpage_put(slot); + schedule_timeout_uninterruptible(1); + if (memchr_inv(slot_address(slot), pattern, SZ_4K)) { + worker->error = -EIO; + break; + } + worker->checks++; + if (round != 63) { + mm_subpage_put(slot); + slot = NULL; + } + } + complete(&worker->ready); + wait_for_completion(&worker->go); + if (slot) { + if (memchr_inv(slot_address(slot), pattern, SZ_4K)) + worker->error = -EIO; + mm_subpage_put(slot); + } + complete(&worker->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void data_worker_stop(void *arg) +{ + struct data_worker *worker = arg; + + complete_all(&worker->go); + kthread_stop(worker->task); +} + +static void user4k_data_concurrent_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct data_worker *workers = kunit_kcalloc(test, 2, sizeof(*workers), GFP_KERNEL); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_NOT_NULL(test, workers); + for (i = 0; i < 2; i++) { + workers[i].pool = fixture->pool; + workers[i].id = i; + init_completion(&workers[i].ready); + init_completion(&workers[i].go); + init_completion(&workers[i].done); + workers[i].task = kthread_run(data_worker_run, &workers[i], "user4k-data-%u", i); + KUNIT_ASSERT_FALSE(test, IS_ERR(workers[i].task)); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, data_worker_stop, &workers[i]), 0); + } + for (i = 0; i < 2; i++) + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].ready, 10 * HZ), 0UL); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 1UL); + mm_subpage_pool_close(fixture->pool); + for (i = 0; i < 2; i++) + complete(&workers[i].go); + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].done, 10 * HZ), 0UL); + KUNIT_EXPECT_EQ(test, workers[i].error, 0); + KUNIT_EXPECT_EQ(test, workers[i].checks, 64U); + } + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +struct lookup_worker { + struct task_struct *task; + struct completion observed; + phys_addr_t phys; + unsigned long checks; + int error; +}; + +static int lookup_worker_run(void *arg) +{ + struct lookup_worker *worker = arg; + bool observed = false; + + while (!kthread_should_stop()) { + struct mm_subpage *slot = mm_subpage_get_from_phys(worker->phys); + + if (slot) { + unsigned char *data = slot_address(slot); + unsigned char value = READ_ONCE(*data); + + if (mm_subpage_phys(slot) != worker->phys || + (value != 0x5a && value != 0xab) || + memchr_inv(data, value, SZ_4K)) + worker->error = -EINVAL; + worker->checks++; + mm_subpage_put(slot); + if (!observed) { + complete(&worker->observed); + observed = true; + } + if (worker->error) + break; + } + cond_resched(); + } + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void lookup_worker_stop(void *arg) +{ + struct lookup_worker *worker = arg; + + if (worker->task) { + kthread_stop(worker->task); + worker->task = NULL; + } +} + +static void user4k_data_lookup_race_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct lookup_worker *worker = kunit_kzalloc(test, sizeof(*worker), GFP_KERNEL); + unsigned int copied = 0, i; + unsigned long deadline; + struct folio *folio; + struct mm_subpage *found; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_NOT_NULL(test, worker); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(fixture, i)); + memset(slot_address(fixture->held[0]), 0x5a, SZ_4K); + memset(slot_address(fixture->held[1]), 0xab, SZ_4K); + worker->phys = mm_subpage_phys(fixture->held[2]); + put_held(fixture, 2); + init_completion(&worker->observed); + worker->task = kthread_run(lookup_worker_run, worker, "user4k-lookup"); + KUNIT_ASSERT_FALSE(test, IS_ERR(worker->task)); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, lookup_worker_stop, worker), 0); + if (num_online_cpus() > 1) { + unsigned int cpu = cpumask_next(cpumask_first(cpu_online_mask), cpu_online_mask); + + KUNIT_ASSERT_EQ(test, set_cpus_allowed_ptr(worker->task, cpumask_of(cpu)), 0); + } + /* Alternate complete contents; a reader must never see a partial copy. */ + deadline = jiffies + 30 * HZ; + while (copied < 2048 && time_before(jiffies, deadline)) { + struct mm_subpage *dest = mm_subpage_copy(fixture->pool, + fixture->held[copied & 1]); + bool observed = true; + phys_addr_t phys; + + if (PTR_ERR_OR_ZERO(dest) == -EAGAIN) { + /* Let a scheduled-out lookup holder retire its reference. */ + schedule_timeout_uninterruptible(1); + continue; + } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dest); + phys = mm_subpage_phys(dest); + if (!copied) + observed = wait_for_completion_timeout(&worker->observed, 5 * HZ) != 0; + mm_subpage_put(dest); + KUNIT_ASSERT_TRUE(test, observed); + KUNIT_ASSERT_EQ(test, phys, worker->phys); + copied++; + cond_resched(); + } + /* Retire the owner while lookup is still running, then verify unbinding. */ + folio = mm_subpage_folio(fixture->held[0]); + folio_get(folio); + mm_subpage_pool_close(fixture->pool); + for (i = 0; i < DATA_SLOTS; i++) + if (i != 2) + put_held(fixture, i); + lookup_worker_stop(worker); + found = mm_subpage_get_from_phys(worker->phys); + KUNIT_EXPECT_PTR_EQ(test, found, NULL); + if (found) + mm_subpage_put(found); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + folio_put(folio); + KUNIT_EXPECT_EQ(test, worker->error, 0); + KUNIT_EXPECT_GT(test, worker->checks, 0UL); + KUNIT_EXPECT_EQ(test, copied, 2048U); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +static void user4k_data_swapcache_reuse_test(struct kunit *test) +{ + struct data_fixture *f = data_fixture_create(test); + struct mm_subpage *attempt; + struct folio *folio; + unsigned int i; + bool intact; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * MM_SUBPAGE_SIZE)); + memset(slot_address(f->held[i]), 0x60 + i, MM_SUBPAGE_SIZE); + } + folio = mm_subpage_folio(f->held[0]); + put_held(f, 1); + /* Synthetic cache membership only; no swap entry or I/O is fabricated. */ + __folio_set_swapbacked(folio); + folio_set_swapcache(folio); + attempt = mm_subpage_alloc_at(f->pool, MM_SUBPAGE_SIZE); + intact = !memchr_inv(folio_address(folio) + MM_SUBPAGE_SIZE, 0x61, MM_SUBPAGE_SIZE); + folio_clear_swapcache(folio); + __folio_clear_swapbacked(folio); + /* Restore the real state before assertions can unwind the fixture. */ + if (!IS_ERR(attempt)) + f->held[1] = attempt; + KUNIT_EXPECT_TRUE(test, intact); + KUNIT_ASSERT_EQ(test, PTR_ERR_OR_ZERO(attempt), -EAGAIN); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, MM_SUBPAGE_SIZE)); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + i == 1 ? 0 : 0x60 + i, MM_SUBPAGE_SIZE), NULL); +} + +static void restore_folio_free(void *arg) +{ + struct folio *folio = arg; + + /* Data-only fixtures never create a real swap entry or mapping. */ + folio_clear_swapcache(folio); + __folio_clear_swapbacked(folio); + folio_put(folio); +} + +static struct mm_subpage *restore_test_slot_granule(struct folio *folio, unsigned int offset, unsigned int shift) +{ + struct mm_subpage *slot; + + folio_lock(folio); + __folio_set_swapbacked(folio); + folio_set_swapcache(folio); + slot = mm_subpage_restore_granule(folio, offset, shift, GFP_KERNEL); + folio_clear_swapcache(folio); + __folio_clear_swapbacked(folio); + folio_unlock(folio); + return slot; +} + +static struct mm_subpage *restore_test_slot(struct folio *folio, unsigned int offset) +{ + return restore_test_slot_granule(folio, offset, MM_SUBPAGE_SHIFT); +} + +static void test_data_restore_granule(struct kunit *test, unsigned int shift) +{ + struct data_fixture *f = data_fixture_create_granule(test, shift); + struct folio *folio; + struct mm_subpage *slot; + unsigned int i, size = 1U << shift; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * size)); + memset(slot_address(f->held[i]), 0x70 + i, size); + } + folio = mm_subpage_folio(f->held[0]); + folio_get(folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_folio_free, folio), 0); + put_held(f, 1); + slot = restore_test_slot_granule(folio, size, shift); + if (!IS_ERR(slot)) + f->held[1] = slot; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), (unsigned int)size); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 5); + slot = restore_test_slot_granule(folio, 0, shift); + if (!IS_ERR(slot)) + f->held[4] = slot; + KUNIT_ASSERT_PTR_EQ(test, slot, f->held[0]); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 6); + put_held(f, 4); + /* Retire the entire old owner, retaining only a native backing reference. */ + for (i = 0; i < 4; i++) + put_held(f, i); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(f->pool), 0UL); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + for (i = 0; i < 4; i++) { + slot = restore_test_slot_granule(folio, i * size, shift); + if (!IS_ERR(slot)) + f->held[i] = slot; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(slot), folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0x70 + i, + size), NULL); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 5); + KUNIT_EXPECT_EQ(test, PTR_ERR(restore_test_slot_granule(folio, 1, shift)), -EINVAL); + KUNIT_EXPECT_EQ(test, PTR_ERR(restore_test_slot_granule(folio, PAGE_SIZE, shift)), -EINVAL); + if (shift != MM_SUBPAGE_SHIFT) { + KUNIT_EXPECT_EQ(test, PTR_ERR(restore_test_slot(folio, 0)), -EINVAL); + KUNIT_EXPECT_EQ(test, PTR_ERR(restore_test_slot_granule(folio, SZ_4K, shift)), -EINVAL); + } + +} + +static void user4k_data_restore_test(struct kunit *test) +{ + test_data_restore_granule(test, MM_SUBPAGE_SHIFT); +} + +static void user16k_data_restore_test(struct kunit *test) +{ + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_data_restore_granule(test, 14); +} + +struct restore_worker { + struct folio *folio; + struct task_struct *task; + struct completion done; + unsigned int checks; + int error; +}; + +static int restore_worker_run(void *arg) +{ + struct restore_worker *w = arg; + unsigned long deadline = jiffies + 20 * HZ; + + while (!kthread_should_stop() && w->checks < 1024 && time_before(jiffies, deadline)) { + struct mm_subpage *slot; + + folio_lock(w->folio); + slot = mm_subpage_restore(w->folio, MM_SUBPAGE_SIZE, GFP_KERNEL); + folio_unlock(w->folio); + if (PTR_ERR_OR_ZERO(slot) == -EAGAIN) { + cond_resched(); + continue; + } + if (IS_ERR(slot)) { + w->error = PTR_ERR(slot); + break; + } + if (memchr_inv(slot_address(slot), 0x83, MM_SUBPAGE_SIZE)) + w->error = -EIO; + /* Retirement deliberately occurs without the folio lock. */ + mm_subpage_put(slot); + if (w->error) + break; + w->checks++; + cond_resched(); + } + complete(&w->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void restore_worker_stop(void *arg) +{ + struct restore_worker *w = arg; + + if (w->task) { + kthread_stop(w->task); + w->task = NULL; + } +} + +static void user4k_data_restore_race_test(struct kunit *test) +{ + struct folio *folio = folio_alloc(GFP_KERNEL, 0); + struct restore_worker *workers; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_folio_free, folio), 0); + memset(folio_address(folio), 0x83, PAGE_SIZE); + __folio_mark_uptodate(folio); + __folio_set_swapbacked(folio); + folio_set_swapcache(folio); + workers = kunit_kcalloc(test, 2, sizeof(*workers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + for (i = 0; i < 2; i++) { + workers[i].folio = folio; + init_completion(&workers[i].done); + workers[i].task = kthread_run(restore_worker_run, &workers[i], "user4k-restore-%u", i); + if (IS_ERR(workers[i].task)) { + workers[i].task = NULL; + KUNIT_FAIL(test, "restore worker creation failed"); + return; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_worker_stop, &workers[i]), 0); + } + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].done, 25 * HZ), 0UL); + restore_worker_stop(&workers[i]); + KUNIT_EXPECT_EQ(test, workers[i].error, 0); + KUNIT_EXPECT_EQ(test, workers[i].checks, 1024U); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); +} + +static void test_data_migration_granule(struct kunit *test, unsigned int shift) +{ + unsigned int mode, size = 1U << shift; + + /* Success, rollback, all PTEs discarded, and partial-folio success. */ + for (mode = 0; mode < 4; mode++) { + struct data_fixture *f = data_fixture_create_granule(test, shift); + struct folio *src, *dst, *target; + unsigned int i, nr = mode == 3 ? 3 : 4; + int ret, src_refs = 0, dst_refs = 0, blocked = 0; + bool unpublished = true, active = false; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < nr; i++) { + unsigned int index = mode == 3 && i ? i + 1 : i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, index * size)); + memset(slot_address(f->held[i]), 0x80 + index, size); + } + src = mm_subpage_folio(f->held[0]); + folio_get(src); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_folio_free, src), 0); + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_folio_free, dst), 0); + folio_lock(src); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(src, dst); + if (!ret) { + struct mm_subpage *probe; + + src_refs = folio_ref_count(src); + dst_refs = folio_ref_count(dst); + active = mm_subpage_migrating(src) && mm_subpage_migrating(dst); + probe = mm_subpage_alloc_at(f->pool, size); + blocked = PTR_ERR_OR_ZERO(probe); + if (!IS_ERR(probe)) + mm_subpage_put(probe); + /* Emulate migration entries: their old mapping refs are gone. */ + for (i = 0; i < nr; i++) + put_held(f, i); + probe = mm_subpage_get_from_phys(PFN_PHYS(folio_pfn(src))); + unpublished &= !probe; + if (probe) + mm_subpage_put(probe); + probe = mm_subpage_get_from_phys(PFN_PHYS(folio_pfn(dst))); + unpublished &= !probe; + if (probe) + mm_subpage_put(probe); + if (mode != 1 && mode != 2) { + copy_user_highpage(&dst->page, &src->page, 0, NULL); + __folio_mark_uptodate(dst); + } + target = mode == 1 ? src : dst; + if (mode != 2) + for (i = 0; i < nr; i++) { + unsigned int index = mode == 3 && i ? i + 1 : i; + + f->held[i] = mm_subpage_migrate_restore(target, index * size); + } + mm_subpage_migrate_finish(src); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(src); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, active); + KUNIT_EXPECT_TRUE(test, unpublished); + KUNIT_EXPECT_EQ(test, src_refs, (int)nr + 1); + KUNIT_EXPECT_EQ(test, dst_refs, 1); + KUNIT_EXPECT_EQ(test, blocked, -EAGAIN); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(f->pool), mode == 2 ? 0UL : 1UL); + KUNIT_EXPECT_EQ(test, folio_ref_count(src), mode == 1 ? (int)nr + 1 : 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(dst), mode == 0 || mode == 3 ? (int)nr + 1 : 1); + if (mode != 2) + for (i = 0; i < nr; i++) { + unsigned int index = mode == 3 && i ? i + 1 : i; + + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(f->held[i]), mode == 1 ? src : dst); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + 0x80 + index, size), NULL); + } + if (mode == 3) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 3, size)); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(f->held[3]), dst); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[3]), 0, + size), NULL); + } + } +} + +static void user4k_data_migration_owner_test(struct kunit *test) +{ + test_data_migration_granule(test, MM_SUBPAGE_SHIFT); +} + +static void user16k_data_migration_owner_test(struct kunit *test) +{ + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_data_migration_granule(test, 14); +} + +#ifdef CONFIG_ARM64_MTE +static void test_slot_tags_set(void *addr, unsigned int tag) +{ + unsigned long value = (unsigned long)tag << MTE_TAG_SHIFT; + unsigned int offset; + + for (offset = 0; offset < MM_SUBPAGE_SIZE; offset += MTE_GRANULE_SIZE) + asm volatile(".arch_extension memtag\n\tstg %0, [%1]" + : : "r" (value), "r" (addr + offset) : "memory"); +} + +static bool test_slot_tags_equal(void *addr, unsigned int expected) +{ + unsigned int offset; + + for (offset = 0; offset < MM_SUBPAGE_SIZE; offset += MTE_GRANULE_SIZE) { + unsigned long value = 0; + + asm volatile(".arch_extension memtag\n\tldg %0, [%1]" + : "+r" (value) : "r" (addr + offset) : "memory"); + if (((value >> MTE_TAG_SHIFT) & 0xf) != expected) + return false; + } + return true; +} +#endif + +static void user4k_data_mte_test(struct kunit *test) +{ +#ifdef CONFIG_ARM64_MTE + struct data_fixture *f; + struct folio *folio; + unsigned int i; + + if (!system_supports_mte()) { + kunit_skip(test, "MTE is not available on this CPU"); + return; + } + f = data_fixture_create(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * MM_SUBPAGE_SIZE)); + folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_TRUE(test, try_page_mte_tagging(&folio->page)); + mte_clear_page_tags(folio_address(folio)); + for (i = 0; i < 4; i++) { + memset(slot_address(f->held[i]), 0x40 + i, MM_SUBPAGE_SIZE); + test_slot_tags_set(slot_address(f->held[i]), i + 1); + } + set_page_mte_tagged(&folio->page); + put_held(f, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, MM_SUBPAGE_SIZE)); + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(slot_address(f->held[i]), i == 1 ? 0 : i + 1)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + i == 1 ? 0 : 0x40 + i, MM_SUBPAGE_SIZE), NULL); + } + put_held(f, 1); + f->held[1] = mm_subpage_copy_at(f->pool, f->held[0], MM_SUBPAGE_SIZE); + if (IS_ERR(f->held[1])) { + f->held[1] = NULL; + KUNIT_FAIL(test, "tagged slot copy failed"); + return; + } + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(slot_address(f->held[i]), i == 1 ? 1 : i + 1)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + i == 1 ? 0x40 : 0x40 + i, MM_SUBPAGE_SIZE), NULL); + } + /* A fresh destination must initialise tags outside the copied slot too. */ + KUNIT_ASSERT_EQ(test, supply_backing_at(f->pool, 3 * MM_SUBPAGE_SIZE), 0); + f->held[4] = mm_subpage_copy_at(f->pool, f->held[2], 3 * MM_SUBPAGE_SIZE); + if (IS_ERR(f->held[4])) { + f->held[4] = NULL; + KUNIT_FAIL(test, "fresh tagged slot copy failed"); + return; + } + folio = mm_subpage_folio(f->held[4]); + KUNIT_EXPECT_TRUE(test, page_mte_tagged(&folio->page)); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(folio_address(folio) + + i * MM_SUBPAGE_SIZE, i == 3 ? 3 : 0)); +#else + kunit_skip(test, "MTE is not configured"); +#endif +} + +static void extent_folio_put(void *arg) +{ + folio_put(arg); +} + +static void user4k_data_copy_extent_test(struct kunit *test) +{ + const unsigned int sizes[] = { SZ_4K, SZ_16K, SZ_64K }; + + for (unsigned int n = 0; n < ARRAY_SIZE(sizes); n++) { + struct folio *source, *dest; + unsigned int size = sizes[n], to_offset; + void *from, *to; + + if (size > PAGE_SIZE) + continue; + source = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); + KUNIT_ASSERT_NOT_NULL(test, source); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, extent_folio_put, source), 0); + dest = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); + KUNIT_ASSERT_NOT_NULL(test, dest); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, extent_folio_put, dest), 0); + from = folio_address(source); + to = folio_address(dest); + to_offset = PAGE_SIZE - size; + for (unsigned int i = 0; i < DATA_SLOTS; i++) + memset(from + i * SZ_4K, 0x40 + i, SZ_4K); + for (unsigned int tagged = 0; tagged < 2; tagged++) { + memset(to, 0xac, PAGE_SIZE); +#ifdef CONFIG_ARM64_MTE + if (system_supports_mte()) { + if (try_page_mte_tagging(&dest->page)) + mte_clear_page_tags(to); + for (unsigned int i = 0; i < DATA_SLOTS; i++) + test_slot_tags_set(to + i * SZ_4K, 9); + set_page_mte_tagged(&dest->page); + if (tagged) { + if (try_page_mte_tagging(&source->page)) + mte_clear_page_tags(from); + for (unsigned int i = 0; i < DATA_SLOTS; i++) + test_slot_tags_set(from + i * SZ_4K, 1 + i % 15); + set_page_mte_tagged(&source->page); + } + } +#endif + KUNIT_ASSERT_EQ(test, copy_user_subpage_range(&dest->page, to_offset, + &source->page, 0, size), 0); + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + unsigned int offset = i * SZ_4K; + bool copied = offset >= to_offset; + unsigned int source_index = copied ? (offset - to_offset) / SZ_4K : 0; + + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(to + offset, + copied ? 0x40 + source_index : 0xac, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(from + offset, 0x40 + i, SZ_4K), NULL); +#ifdef CONFIG_ARM64_MTE + if (system_supports_mte()) + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(to + offset, + copied ? (tagged ? 1 + source_index % 15 : 0) : 9)); +#endif + } + clear_user_subpage_range(&dest->page, to_offset, size); + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + unsigned int offset = i * SZ_4K; + bool cleared = offset >= to_offset; + + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(to + offset, cleared ? 0 : 0xac, + SZ_4K), NULL); +#ifdef CONFIG_ARM64_MTE + if (system_supports_mte()) + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(to + offset, cleared ? 0 : 9)); +#endif + } + } + } +} + +#ifdef CONFIG_NET +struct skb_test_state { + struct folio *folio; +#ifdef CONFIG_INET + struct socket *socket; +#endif + struct sk_buff *skbs[5]; +}; + +static void skb_test_cleanup(void *arg) +{ + struct skb_test_state *state = arg; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(state->skbs); i++) { + if (state->skbs[i]) + state->skbs[i]->sk = NULL; + kfree_skb(state->skbs[i]); + } +#ifdef CONFIG_INET + if (state->socket) + sock_release(state->socket); +#endif + /* This fixture marks ownership only; it installs no anonymous rmap. */ + if (state->folio) + state->folio->mapping = NULL; +} + +static void user4k_data_skb_test(struct kunit *test) +{ + struct data_fixture *f = data_fixture_create(test); + struct skb_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct mm_subpage *attempt; + struct iov_iter iter; + struct bio_vec bvec; + unsigned char *bytes = kunit_kmalloc(test, SZ_4K, GFP_KERNEL); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_NOT_NULL(test, bytes); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_4K)); + memset(slot_address(f->held[i]), 0x51 + i, SZ_4K); + } + state->folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, skb_test_cleanup, state), 0); + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + state->skbs[0] = alloc_skb(0, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[0]); + state->skbs[0]->ip_summed = CHECKSUM_UNNECESSARY; + bvec_set_page(&bvec, &state->folio->page, SZ_4K, SZ_4K); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, SZ_4K); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(state->skbs[0], &iter, SZ_4K), (ssize_t)SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, skb_zcopy(state->skbs[0])); + state->skbs[1] = pskb_copy(state->skbs[0], GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[1]); + /* Append a distinct quarter while the callback object has two owners. */ + bvec_set_page(&bvec, &state->folio->page, SZ_4K, 2 * SZ_4K); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, SZ_4K); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(state->skbs[0], &iter, SZ_4K), (ssize_t)SZ_4K); + KUNIT_EXPECT_PTR_NE(test, skb_zcopy(state->skbs[0]), skb_zcopy(state->skbs[1])); + state->skbs[2] = skb_clone(state->skbs[1], GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[2]); + state->skbs[3] = alloc_skb(0, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[3]); + skb_split(state->skbs[0], state->skbs[3], SZ_2K); + state->skbs[4] = skb_copy(state->skbs[3], GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[4]); + put_held(f, 1); + put_held(f, 2); + attempt = mm_subpage_alloc_at(f->pool, SZ_4K); + if (!IS_ERR(attempt)) + f->held[1] = attempt; + KUNIT_EXPECT_EQ(test, PTR_ERR_OR_ZERO(attempt), -EAGAIN); + for (i = 0; i < 2; i++) { + kfree_skb(state->skbs[i]); + state->skbs[i] = NULL; + } + KUNIT_ASSERT_EQ(test, skb_copy_bits(state->skbs[2], 0, bytes, SZ_4K), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(bytes, 0x52, SZ_4K), NULL); + KUNIT_ASSERT_EQ(test, skb_copy_bits(state->skbs[3], SZ_2K, bytes, SZ_4K), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(bytes, 0x53, SZ_4K), NULL); + for (i = 2; i < 4; i++) { + kfree_skb(state->skbs[i]); + state->skbs[i] = NULL; + } + /* A full data copy must not keep ownership of the original quarters. */ + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_4K)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 2, 2 * SZ_4K)); + memset(slot_address(f->held[1]), 0xe1, SZ_4K); + memset(slot_address(f->held[2]), 0xe2, SZ_4K); + KUNIT_ASSERT_EQ(test, skb_copy_bits(state->skbs[4], SZ_2K, bytes, SZ_4K), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(bytes, 0x53, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); +} +#ifdef CONFIG_INET +static void test_data_skb_prune_granule(struct kunit *test, unsigned int shift) +{ + struct data_fixture *f = data_fixture_create_granule(test, shift); + struct skb_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct sk_buff *skb; + struct bio_vec bvec; + struct iov_iter iter; + unsigned int i, size = 1U << shift; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * size)); + state->folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, skb_test_cleanup, state), 0); + KUNIT_ASSERT_EQ(test, sock_create_kern(&init_net, AF_INET, SOCK_STREAM, 0, &state->socket), 0); + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + state->skbs[0] = alloc_skb(0, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[0]); + skb = state->skbs[0]; + /* A live socket association prevents trim from condensing data into the head. */ + skb->sk = state->socket->sk; + skb->ip_summed = CHECKSUM_UNNECESSARY; + bvec_set_page(&bvec, &state->folio->page, 16, size - SZ_4K + 16); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, 16); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(skb, &iter, 16), (ssize_t)16); + for (i = 0; i < 1024; i++) { + bvec_set_page(&bvec, &state->folio->page, size, (1 + i % 3) * size); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, size); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(skb, &iter, size), (ssize_t)size); + /* Four fixture refs, two frag refs, two current quarter refs. */ + KUNIT_ASSERT_EQ(test, folio_ref_count(state->folio), 8); + KUNIT_ASSERT_EQ(test, pskb_trim(skb, 16), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 7); + } + skb->sk = NULL; + kfree_skb(skb); + state->skbs[0] = NULL; + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); +} + +static void user4k_data_skb_prune_test(struct kunit *test) +{ + test_data_skb_prune_granule(test, MM_SUBPAGE_SHIFT); +} + +static void user16k_data_skb_prune_test(struct kunit *test) +{ + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_data_skb_prune_granule(test, 14); +} + +#endif + +static void user4k_data_skb_checksum_test(struct kunit *test) +{ + struct data_fixture *f = data_fixture_create(test); + struct skb_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct bio_vec bvec[2]; + struct iov_iter iter; + unsigned char *bytes; + unsigned int i, owned; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_4K)); + state->folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, skb_test_cleanup, state), 0); + bytes = folio_address(state->folio); + for (i = 0; i < PAGE_SIZE; i++) + bytes[i] = i * 13 + (i >> 12) * 31; + for (owned = 0; owned < 2; owned++) { + struct sk_buff *skb; + + state->folio->mapping = owned ? (void *)FOLIO_MAPPING_ANON : NULL; + state->skbs[0] = alloc_skb(0, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[0]); + skb = state->skbs[0]; + skb->ip_summed = CHECKSUM_NONE; + bvec_set_page(&bvec[0], &state->folio->page, SZ_4K - 13, 13); + bvec_set_page(&bvec[1], &state->folio->page, SZ_4K, SZ_4K); + iov_iter_bvec(&iter, ITER_SOURCE, bvec, 2, SZ_8K - 13); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(skb, &iter, SZ_8K - 13), + (ssize_t)(SZ_8K - 13)); + KUNIT_EXPECT_EQ(test, (__force u16)csum_fold(skb->csum), + (__force u16)csum_fold(skb_checksum(skb, 0, skb->len, 0))); + kfree_skb(skb); + state->skbs[0] = NULL; + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); + } +} + +#endif + +struct refs_test_state { + struct folio *folio; + struct mm_subpage_refs *refs[3]; +}; + +static void refs_test_cleanup(void *arg) +{ + struct refs_test_state *state = arg; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(state->refs); i++) + mm_subpage_refs_put(state->refs[i]); + if (state->folio) + state->folio->mapping = NULL; +} + +static bool refs_test_keep(const struct mm_subpage *slot, void *context) +{ + unsigned long mask = *(unsigned long *)context; + + return mask & BIT(mm_subpage_offset(slot) >> MM_SUBPAGE_SHIFT); +} + +static void user4k_data_refset_test(struct kunit *test) +{ + struct data_fixture *f = data_fixture_create(test); + struct refs_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct mm_subpage *attempt; + struct page *page; + unsigned long mask; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_4K)); + state->folio = mm_subpage_folio(f->held[0]); + page = &state->folio->page; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, refs_test_cleanup, state), 0); + /* Ownership fixture only; no anonymous rmap is installed. */ + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, + SZ_4K + 17, SZ_8K - 34, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 2); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, SZ_4K, SZ_4K, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 2); + state->refs[1] = mm_subpage_refs_get(state->refs[0]); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[1], page, 3 * SZ_4K, SZ_4K, GFP_KERNEL), 0); + KUNIT_EXPECT_PTR_NE(test, state->refs[0], state->refs[1]); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 5); + mask = BIT(2); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[1], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 3); + state->refs[2] = mm_subpage_refs_get(state->refs[0]); + mask = BIT(1); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[2], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_PTR_NE(test, state->refs[0], state->refs[2]); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 4); + KUNIT_EXPECT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, PAGE_SIZE, 1, GFP_KERNEL), -EINVAL); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 4); + mask = 0; + for (i = 1; i < 3; i++) { + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[i], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_PTR_EQ(test, state->refs[i], NULL); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 2); + put_held(f, 1); + attempt = mm_subpage_alloc_at(f->pool, SZ_4K); + if (!IS_ERR(attempt)) + f->held[1] = attempt; + KUNIT_ASSERT_EQ(test, PTR_ERR_OR_ZERO(attempt), -EAGAIN); + mm_subpage_refs_put(state->refs[0]); + state->refs[0] = NULL; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_4K)); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS); + for (i = 0; i < 1024; i++) { + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, 0, PAGE_SIZE, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 2 * (int)DATA_SLOTS); + mask = BIT(i % DATA_SLOTS); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[0], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 1); + } + mm_subpage_refs_put(state->refs[0]); + state->refs[0] = NULL; + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS); +} + +#if IS_ENABLED(CONFIG_NET_SOCK_MSG) && IS_ENABLED(CONFIG_INET) +struct skmsg_test_state { + struct folio *folio; + struct socket *socket; + struct sk_msg messages[2]; + struct sk_msg borrowed; +}; + +static void skmsg_test_cleanup(void *arg) +{ + struct skmsg_test_state *state = arg; + unsigned int i; + + if (state->socket) { + for (i = 0; i < ARRAY_SIZE(state->messages); i++) + sk_msg_free(state->socket->sk, &state->messages[i]); + sk_msg_subpages_release(&state->borrowed); + sock_release(state->socket); + } + if (state->folio) + state->folio->mapping = NULL; +} + +static void test_data_skmsg_granule(struct kunit *test, unsigned int shift) +{ + struct data_fixture *f = data_fixture_create_granule(test, shift); + struct skmsg_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct mm_subpage *attempt; + struct sk_msg *src, *dst; + struct page *page; + struct sock *sk; + struct bio_vec bvec; + struct iov_iter iter; + unsigned int i, size = 1U << shift; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * size)); + memset(slot_address(f->held[i]), 0x60 + i, size); + } + state->folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, skmsg_test_cleanup, state), 0); + KUNIT_ASSERT_EQ(test, sock_create_kern(&init_net, AF_INET, SOCK_STREAM, 0, &state->socket), 0); + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + page = &state->folio->page; + sk = state->socket->sk; + src = &state->messages[0]; + dst = &state->messages[1]; + sk_msg_init(src); + sk_msg_init(dst); + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, (size / 2) - 13, size + (size / 2) + 13), 0); + sk_mem_charge(sk, (size / 2) - 13); + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, 2 * size), 0); + sk_mem_charge(sk, size); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 8); + KUNIT_ASSERT_EQ(test, sk_msg_clone(sk, dst, src, 0, size + (size / 2) - 13), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 11); + sk_msg_free_partial(sk, src, (size / 2) - 13); + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, 0), 0); + sk_mem_charge(sk, size); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 11); + sk_msg_free(sk, dst); + put_held(f, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, size)); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 8); + sk_msg_free_partial(sk, src, size); + for (i = 0; i < 1024; i++) { + unsigned int index = (i + 1) & 3; + + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, index * size), 0); + sk_mem_charge(sk, size); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 8); + sk_msg_free_partial(sk, src, size); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 7); + } + sk_msg_free(sk, src); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); + /* A borrowed descriptor copy has an independent ownership lifetime. */ + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, 0), 0); + sk_mem_charge(sk, size); + memcpy(&state->borrowed, src, sizeof(*src)); + sk_msg_subpages_get(&state->borrowed); + sk_msg_free(sk, src); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 5); + put_held(f, 0); + attempt = mm_subpage_alloc_at(f->pool, 0); + if (!IS_ERR(attempt)) + f->held[0] = attempt; + KUNIT_ASSERT_EQ(test, PTR_ERR_OR_ZERO(attempt), -EAGAIN); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(sg_virt(&state->borrowed.sg.data[0]), 0x60, size), NULL); + sk_msg_subpages_release(&state->borrowed); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 0, 0)); + /* Full transfer moves the set, and zero-copy iterator trim releases its references. */ + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, 3 * size), 0); + sk_mem_charge(sk, size); + sk_msg_xfer_full(dst, src); + KUNIT_EXPECT_EQ(test, src->sg.size, 0U); + sk_msg_free(sk, dst); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); + bvec_set_page(&bvec, page, size - 17, 3 * size + 17); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, size - 17); + KUNIT_ASSERT_EQ(test, sk_msg_zerocopy_from_iter(sk, &iter, src, size - 17), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 6); + sk_msg_trim(sk, src, 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); +} + +static void user4k_data_skmsg_test(struct kunit *test) +{ + test_data_skmsg_granule(test, MM_SUBPAGE_SHIFT); +} + +static void user16k_data_skmsg_test(struct kunit *test) +{ + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_data_skmsg_granule(test, 14); +} +#endif + +static void user16k_data_owner_test(struct kunit *test) +{ + struct data_fixture *f, *dest, *small; + struct mm_subpage *slot, *found; + unsigned int count = PAGE_SIZE / SZ_16K; + + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + f = data_fixture_create_granule(test, 14); + dest = data_fixture_create_granule(test, 14); + small = data_fixture_create(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, dest); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + KUNIT_ASSERT_EQ(test, supply_backing(dest->pool), 0); + KUNIT_ASSERT_EQ(test, supply_backing(small->pool), 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(small, 0)); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(f->pool, SZ_4K)), -EINVAL); + for (unsigned int i = 0; i < count; i++) { + slot = hold_alloc_at(f, i, i * SZ_16K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_shift(slot), 14U); + KUNIT_EXPECT_EQ(test, mm_subpage_size(slot), (unsigned int)SZ_16K); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * (unsigned int)SZ_16K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0, SZ_16K), NULL); + memset(slot_address(slot), 0x30 + i, SZ_16K); + for (unsigned int quarter = 0; quarter < 4; quarter++) { + found = mm_subpage_get_from_phys(mm_subpage_phys(slot) + quarter * SZ_4K); + KUNIT_ASSERT_PTR_EQ(test, found, slot); + mm_subpage_put(found); + } + } + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(f->pool)), -EAGAIN); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy(dest->pool, small->held[0])), -EINVAL); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy(small->pool, f->held[0])), -EINVAL); + slot = mm_subpage_copy_at(dest->pool, f->held[count - 1], SZ_16K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + dest->held[0] = slot; + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0x30 + count - 1, SZ_16K), NULL); + /* An interior-byte lookup holds the whole identity across release/reuse. */ + f->held[count] = mm_subpage_get_from_phys(mm_subpage_phys(f->held[1]) + 3 * SZ_4K); + KUNIT_ASSERT_PTR_EQ(test, f->held[count], f->held[1]); + put_held(f, 1); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(f->pool, SZ_16K)), -EAGAIN); + put_held(f, count); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_16K)); + for (unsigned int i = 0; i < count; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + i == 1 ? 0 : 0x30 + i, SZ_16K), NULL); + mm_subpage_pool_close(f->pool); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(f->pool)), -ESHUTDOWN); + for (unsigned int i = 0; i < count; i++) + put_held(f, i); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(f->pool), 0UL); +} + +static void user16k_data_refset_test(struct kunit *test) +{ + struct data_fixture *f; + struct refs_test_state *state; + unsigned int count = PAGE_SIZE / SZ_16K; + unsigned long mask; + struct page *page; + + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + f = data_fixture_create_granule(test, 14); + state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (unsigned int i = 0; i < count; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_16K)); + state->folio = mm_subpage_folio(f->held[0]); + page = &state->folio->page; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, refs_test_cleanup, state), 0); + /* Metadata-only ownership fixture: no VMA or rmap is published. */ + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, + SZ_16K - 1024, 2048, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count + 2); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, 0, PAGE_SIZE, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count * 2); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, 0, PAGE_SIZE, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count * 2); + state->refs[1] = mm_subpage_refs_get(state->refs[0]); + mask = BIT(SZ_16K >> MM_SUBPAGE_SHIFT); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[1], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count * 2 + 1); + mask = 0; + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[0], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count + 1); + put_held(f, 1); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(f->pool, SZ_16K)), -EAGAIN); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[1], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_16K)); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count); +} + +static void user16k_data_mte_test(struct kunit *test) +{ +#ifdef CONFIG_ARM64_MTE + struct data_fixture *f; + struct folio *folio; + struct mm_subpage *copy; + unsigned int count = PAGE_SIZE / SZ_16K; + int ret; + + if (PAGE_SIZE <= SZ_16K || !system_supports_mte()) { + kunit_skip(test, "requires MTE and native backing larger than 16K"); + return; + } + f = data_fixture_create_granule(test, 14); + KUNIT_ASSERT_NOT_NULL(test, f); + folio = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + if (try_page_mte_tagging(&folio->page)) + mte_clear_page_tags(folio_address(folio)); + for (unsigned int i = 0; i < DATA_SLOTS; i++) + test_slot_tags_set(folio_address(folio) + i * SZ_4K, 9); + set_page_mte_tagged(&folio->page); + ret = mm_subpage_pool_add_folio(f->pool, folio, GFP_KERNEL); + if (ret) + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + for (unsigned int i = 0; i < count; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_16K)); + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + void *address = folio_address(folio) + i * SZ_4K; + + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(address, 0)); + memset(address, 0x50 + i, SZ_4K); + test_slot_tags_set(address, 1 + i % 15); + } + put_held(f, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_16K)); + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + bool cleared = i >= 4 && i < 8; + void *address = folio_address(folio) + i * SZ_4K; + + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(address, cleared ? 0 : 1 + i % 15)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(address, cleared ? 0 : 0x50 + i, SZ_4K), NULL); + } + put_held(f, 1); + copy = mm_subpage_copy_at(f->pool, f->held[count - 1], SZ_16K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + f->held[1] = copy; + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + unsigned int source = i >= 4 && i < 8 ? i + DATA_SLOTS - 8 : i; + void *address = folio_address(folio) + i * SZ_4K; + + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(address, 1 + source % 15)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(address, 0x50 + source, SZ_4K), NULL); + } +#else + kunit_skip(test, "MTE is not configured"); +#endif +} + +static void expect_file_coverage(struct kunit *test, struct folio *folio, + unsigned int leaves, unsigned int first, unsigned int tail, + unsigned int final) +{ + struct mm_subpage_mapcounts counts; + unsigned int i; + + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(&folio->page, &counts), leaves); + for (i = 0; i < DATA_SLOTS; i++) { + unsigned int expected = i == 0 ? first : + i == DATA_SLOTS - 1 ? final : i >= DATA_SLOTS - 4 ? tail : 0; + + KUNIT_EXPECT_EQ_MSG(test, counts.slots[i], expected, "coverage slot %u", i); + } +} + +static void user16k_file_coverage_test(struct kunit *test) +{ + struct folio *folio; + phys_addr_t base, tail, last; + + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative file coverage requires larger native backing"); + return; + } + folio = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, extent_folio_put, folio), 0); + base = PFN_PHYS(folio_pfn(folio)); + tail = base + PAGE_SIZE - SZ_16K; + last = base + PAGE_SIZE - SZ_4K; + expect_file_coverage(test, folio, 0, 0, 0, 0); + __mm_subpage_file_map_add(tail, SZ_16K); + expect_file_coverage(test, folio, 1, 0, 1, 1); + __mm_subpage_file_map_add(last, SZ_4K); + expect_file_coverage(test, folio, 2, 0, 1, 2); + __mm_subpage_file_map_add(tail, SZ_16K); + expect_file_coverage(test, folio, 3, 0, 2, 3); + __mm_subpage_file_map_add(base, SZ_4K); + expect_file_coverage(test, folio, 4, 1, 2, 3); + __mm_subpage_file_map_del(tail, SZ_16K); + expect_file_coverage(test, folio, 3, 1, 1, 2); + __mm_subpage_file_map_del(last, SZ_4K); + expect_file_coverage(test, folio, 2, 1, 1, 1); + __mm_subpage_file_map_del(tail, SZ_16K); + expect_file_coverage(test, folio, 1, 1, 0, 0); + __mm_subpage_file_map_del(base, SZ_4K); + expect_file_coverage(test, folio, 0, 0, 0, 0); +} + +static struct kunit_case user4k_data_cases[] = { + KUNIT_CASE(user4k_data_reuse_test), + KUNIT_CASE(user4k_data_refset_test), +#if IS_ENABLED(CONFIG_NET_SOCK_MSG) && IS_ENABLED(CONFIG_INET) + KUNIT_CASE(user4k_data_skmsg_test), + KUNIT_CASE(user16k_data_skmsg_test), +#endif +#ifdef CONFIG_NET + KUNIT_CASE(user4k_data_skb_test), + KUNIT_CASE(user4k_data_skb_checksum_test), + +#ifdef CONFIG_INET + KUNIT_CASE(user4k_data_skb_prune_test), + KUNIT_CASE(user16k_data_skb_prune_test), +#endif +#endif + KUNIT_CASE(user4k_data_mte_test), + KUNIT_CASE(user4k_data_copy_extent_test), + KUNIT_CASE(user16k_data_owner_test), + KUNIT_CASE(user16k_file_coverage_test), + KUNIT_CASE(user16k_data_refset_test), + KUNIT_CASE(user16k_data_mte_test), + KUNIT_CASE(user4k_data_swapcache_reuse_test), + KUNIT_CASE(user4k_data_restore_test), + KUNIT_CASE(user16k_data_restore_test), + KUNIT_CASE(user4k_data_restore_race_test), + KUNIT_CASE(user4k_data_migration_owner_test), + KUNIT_CASE(user16k_data_migration_owner_test), + KUNIT_CASE(user4k_data_copy_test), + KUNIT_CASE(user4k_data_lookup_test), + KUNIT_CASE(user4k_data_exact_slot_test), + KUNIT_CASE(user4k_data_close_test), + KUNIT_CASE(user4k_data_deferred_test), + KUNIT_CASE(user4k_data_backing_reject_test), + KUNIT_CASE(user4k_data_concurrent_test), + KUNIT_CASE(user4k_data_lookup_race_test), + {} +}; + +static void user4k_data_suite_exit(struct kunit_suite *suite) +{ + /* Drain batched metadata frees before the disposable guest powers off. */ + kvfree_rcu_barrier(); +} + +static struct kunit_suite user4k_data_suite = { + .name = "arm64-user4k-data", + .test_cases = user4k_data_cases, + .suite_exit = user4k_data_suite_exit, +}; + +kunit_test_suite(user4k_data_suite); +MODULE_LICENSE("GPL"); diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c new file mode 100644 index 0000000000000..9a9d63e71d19f --- /dev/null +++ b/arch/arm64/mm/user4k-test.c @@ -0,0 +1,14239 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Architectural prerequisite for mixed-granule userspace. + * + * Packed architectural tables and isolated Linux mms are separate fixtures. + * PAGE_SIZE, struct page, all ordinary mms and TTBR1 retain native geometry. Only bounded + * kernel workers may borrow the isolated Linux mms. Never pass + * packed architectural tables to generic MM walkers or expose them to a task. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../../drivers/iommu/iommufd/iommufd_private.h" +#include "../../../drivers/iommu/iommufd/io_pagetable.h" +#include "../../../mm/internal.h" +#include "../../../mm/vma.h" +#include "../../../mm/swap.h" +#ifdef CONFIG_FUTEX +#include "../../../kernel/futex/futex.h" +#endif + +#define USER4K_SHIFT 12 +#define TEST_SLOTS MM_SUBPAGES_PER_PAGE +#define USER4K_SIZE BIT(USER4K_SHIFT) +#define TEST_VA (4UL * SZ_1M) +#define TEST_LEVELS 4 +#define PROBE_SLOTS 4 +#define TEST_ROUNDS 8 +#define TEST_PROT (PTE_TYPE_PAGE | PTE_USER | PTE_AF | PTE_NG | \ + PTE_SHARED | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_NORMAL)) + +static bool user4k_test_enabled; + +static int __init user4k_test_setup(char *str) +{ + bool enabled; + int ret = kstrtobool(str, &enabled); + + if (ret) + return ret; + /* Software MM tests are sanitizable; raw/borrowed TTBR0 tests stay gated. */ + if (enabled && IS_ENABLED(CONFIG_KASAN)) + return -EINVAL; + user4k_test_enabled = enabled; + return 0; +} +early_param("arm64.user4k_test", user4k_test_setup); + +struct test_tables { + u64 *level[TEST_LEVELS]; + unsigned int shift; +}; + +struct test_result { + u64 native[PROBE_SLOTS]; + u64 read[PROBE_SLOTS]; + u64 write[PROBE_SLOTS]; + u64 replaced[PROBE_SLOTS]; + u64 replaced_read; + u64 kernel; + u64 tcr; + u64 ttbr1; + u64 restored_tcr; + u64 restored_ttbr0; +}; + +static void test_free_page(void *ptr) +{ + free_page((unsigned long)ptr); +} + +static void *test_alloc_page(struct kunit *test) +{ + void *ptr = (void *)get_zeroed_page(GFP_KERNEL); + + if (!ptr) + return NULL; + if (kunit_add_action_or_reset(test, test_free_page, ptr)) + return NULL; + return ptr; +} + +static void test_put_compound(void *ptr) +{ + put_page(ptr); +} + +static void *test_alloc_granule(struct kunit *test, unsigned int shift) +{ + struct page *page; + + if (shift <= PAGE_SHIFT) + return test_alloc_page(test); + page = alloc_pages(GFP_KERNEL | __GFP_ZERO | __GFP_COMP, shift - PAGE_SHIFT); + if (!page) + return NULL; + if (kunit_add_action_or_reset(test, test_put_compound, page)) + return NULL; + return page_address(page); +} + +static unsigned int test_index(unsigned long va, unsigned int shift, + unsigned int level) +{ + unsigned int bits = shift - PTDESC_ORDER; + + return (va >> ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(level, shift)) & (BIT(bits) - 1); +} + +static int test_alloc_tables(struct kunit *test, struct test_tables *tables, + unsigned int shift) +{ + int level; + void *packed = NULL; + + tables->shift = shift; + if (shift == USER4K_SHIFT) { + /* Also test table address bits 13:12, not just leaf addresses. */ + packed = test_alloc_page(test); + if (!packed) + return -ENOMEM; + } + for (level = 0; level < TEST_LEVELS; level++) { + tables->level[level] = packed ? packed + level * USER4K_SIZE : + test_alloc_page(test); + if (!tables->level[level]) + return -ENOMEM; + } + for (level = 0; level < TEST_LEVELS - 1; level++) { + unsigned int index = test_index(TEST_VA, shift, level); + + tables->level[level][index] = + virt_to_phys(tables->level[level + 1]) | PMD_TYPE_TABLE; + } + return 0; +} + +static void user4k_geometry_test(struct kunit *test) +{ + static const unsigned int shifts4k[] = { 39, 30, 21, 12 }; + static const unsigned int shifts16k[] = { 47, 36, 25, 14 }; + static const unsigned int shifts64k[] = { 55, 42, 29, 16 }; + unsigned int level; + + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(48, 12), 4); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(48, 14), 4); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(48, 16), 3); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(42, 16), 2); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(39, 12), 3); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(47, 14), 3); + for (level = 0; level < TEST_LEVELS; level++) { + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(level, 12), + shifts4k[level]); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(level, 14), + shifts16k[level]); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVEL_SHIFT(level), + PAGE_SHIFT == 16 ? shifts64k[level] : shifts16k[level]); + KUNIT_EXPECT_EQ(test, test_index(BIT(shifts4k[level]), 12, level), 1U); + KUNIT_EXPECT_EQ(test, test_index(BIT(shifts4k[level]) - 1, 12, level), 0U); + } +} + +static void test_leaf_encoding(struct kunit *test, unsigned int shift) +{ + struct mm_struct *mm = kunit_kzalloc(test, sizeof(*mm), GFP_KERNEL); + const phys_addr_t base = GENMASK_ULL(47, PAGE_SHIFT); + pgprot_t prot = __pgprot(TEST_PROT | PTE_RDONLY); + unsigned int i; + pte_t pte; + + KUNIT_ASSERT_NOT_NULL(test, mm); + pte = phys_pte_mm(mm, base, prot); + KUNIT_EXPECT_EQ(test, pte_val(pte), pte_val(pfn_pte(base >> PAGE_SHIFT, prot))); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), base); + KUNIT_EXPECT_EQ(test, pgprot_val(pte_pgprot_mm(mm, pte)), pgprot_val(prot)); + mm->context.flags |= shift == 12 ? MMCF_USER_4K : MMCF_USER_16K; + for (i = 0; i < (PAGE_SIZE >> shift); i++) { + phys_addr_t phys = base + i * (1UL << shift); + + pte = phys_pte_mm(mm, phys, prot); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), phys); + KUNIT_EXPECT_EQ(test, pte_pfn(pte), (unsigned long)(base >> PAGE_SHIFT)); + KUNIT_EXPECT_EQ(test, pgprot_val(pte_pgprot_mm(mm, pte)), pgprot_val(prot)); + if (i + 1 < (PAGE_SIZE >> shift)) + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte_advance_pfn_mm(mm, pte, 1)), + phys + (1UL << shift)); + pte = pte_modify(pte, __pgprot(TEST_PROT)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), phys); + KUNIT_EXPECT_FALSE(test, pte_val(pte) & PTE_RDONLY); + } +} + +static void user4k_leaf_encoding_test(struct kunit *test) +{ + test_leaf_encoding(test, 12); +} + +static void user16k_leaf_encoding_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative leaves require larger native backing"); + return; + } + test_leaf_encoding(test, 14); +} + + +/* Caller masks all asynchronous exceptions; this function cannot sleep. */ +static void test_switch_ttbr0(u64 ttbr, u64 tcr) +{ + /* The zero root is valid for both tested granules. */ + cpu_set_reserved_ttbr0(); + local_flush_tlb_all(); + write_sysreg(tcr, tcr_el1); + isb(); + write_sysreg(ttbr, ttbr0_el1); + isb(); +} + +static u64 test_translate(unsigned long va, bool write) +{ + if (write) + asm volatile("at s1e0w, %0" : : "r" (va) : "memory"); + else + asm volatile("at s1e0r, %0" : : "r" (va) : "memory"); + isb(); + return read_sysreg_par(); +} + +#define MM_TEST_MAPPINGS 11 + +struct test_mm { + struct kunit *test; + struct mm_struct *mm; + unsigned long address[MM_TEST_MAPPINGS]; + phys_addr_t phys[MM_TEST_MAPPINGS]; +}; + +static pte_t *test_mm_lookup(struct mm_struct *mm, unsigned long addr, + spinlock_t **ptl); + +static void test_mm_hardware(struct test_mm *ctx) +{ + struct kunit *test = ctx->test; + struct vm_area_struct vma = TLB_FLUSH_VMA(ctx->mm, 0); + struct arch_tlbflush_unmap_batch batch = {}; + pte_t *leaves[4]; + u64 invalidated[3][4]; + u64 result[MM_TEST_MAPPINGS], writes[MM_TEST_MAPPINGS]; + u64 old_tcr, old_ttbr0, old_ttbr1, old_par, tcr, kernel; + u64 old_id = atomic64_read(&ctx->mm->context.id); + u64 restored_tcr, restored_ttbr0, retained_ttbr1; + unsigned long flags; + unsigned int i, step; + + for (i = 0; i < ARRAY_SIZE(leaves); i++) { + spinlock_t *ptl; + + leaves[i] = test_mm_lookup(ctx->mm, ctx->address[i], &ptl); + if (leaves[i]) + pte_unmap_unlock(leaves[i], ptl); + KUNIT_ASSERT_NOT_NULL(test, leaves[i]); + } + + preempt_disable(); + flags = local_daif_save(); + old_tcr = read_sysreg(tcr_el1); + old_ttbr0 = read_sysreg(ttbr0_el1); + old_ttbr1 = read_sysreg(ttbr1_el1); + old_par = read_sysreg_par(); + if ((old_tcr & (TCR_EL1_DS | TCR_EL1_TG1_MASK | TCR_EL1_A1)) != + (TCR_TG1_16K | TCR_EL1_A1)) { + local_daif_restore(flags); + preempt_enable(); + kunit_skip(test, "requires 16K TTBR1, DS=0, and A1=1"); + return; + } + tcr = (old_tcr & ~(TCR_EL1_TG0_MASK | TCR_EL1_T0SZ_MASK | TCR_EL1_EPD0)) | + TCR_TG0_4K | TCR_T0SZ(48); + test_switch_ttbr0(virt_to_phys(ctx->mm->pgd), tcr); + /* A1 selects the saved TTBR1 ASID for these private test translations. */ + atomic64_set(&ctx->mm->context.id, old_ttbr1 >> 48); + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + result[i] = test_translate(ctx->address[i], false); + writes[i] = test_translate(ctx->address[i], true); + } + for (step = 0; step < ARRAY_SIZE(invalidated); step++) { + unsigned int first = step == 2 ? 3 : 1; + unsigned int last = step == 1 ? 3 : first + 1; + + for (i = first; i < last; i++) + WRITE_ONCE(*leaves[i], __pte(0)); + switch (step) { + case 0: + __flush_tlb_page(&vma, ctx->address[first] + 1, + TLBF_NONOTIFY | TLBF_NOBROADCAST); + break; + case 1: + __flush_tlb_range(&vma, ctx->address[first], ctx->address[last], + SZ_4K, 3, TLBF_NONOTIFY); + break; + case 2: + arch_tlbbatch_add_pending(&batch, ctx->mm, ctx->address[first], + ctx->address[first] + SZ_4K); + arch_tlbbatch_flush(&batch); + break; + } + isb(); + for (i = 0; i < ARRAY_SIZE(leaves); i++) + invalidated[step][i] = test_translate(ctx->address[i], false); + for (i = first; i < last; i++) + WRITE_ONCE(*leaves[i], __pte(ctx->phys[i] | TEST_PROT)); + dsb(ishst); + isb(); + /* Repopulate before the next independent invalidation. */ + for (i = 0; i < ARRAY_SIZE(leaves); i++) + test_translate(ctx->address[i], false); + } + asm volatile("at s1e1r, %0" : : "r" (ctx->mm->pgd) : "memory"); + isb(); + kernel = read_sysreg_par(); + retained_ttbr1 = read_sysreg(ttbr1_el1); + test_switch_ttbr0(old_ttbr0, old_tcr); + restored_tcr = read_sysreg(tcr_el1); + restored_ttbr0 = read_sysreg(ttbr0_el1); + write_sysreg(old_par, par_el1); + isb(); + atomic64_set(&ctx->mm->context.id, old_id); + local_daif_restore(flags); + preempt_enable(); + + KUNIT_EXPECT_EQ(test, restored_tcr, old_tcr); + KUNIT_EXPECT_EQ(test, restored_ttbr0, old_ttbr0); + KUNIT_EXPECT_EQ(test, retained_ttbr1, old_ttbr1); + KUNIT_EXPECT_EQ(test, kernel & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, kernel & SYS_PAR_EL1_PA, + virt_to_phys(ctx->mm->pgd) & SYS_PAR_EL1_PA); + for (step = 0; step < ARRAY_SIZE(invalidated); step++) { + for (i = 0; i < ARRAY_SIZE(leaves); i++) { + bool hole = step == 0 ? i == 1 : step == 1 ? i == 1 || i == 2 : i == 3; + + KUNIT_EXPECT_EQ(test, invalidated[step][i] & SYS_PAR_EL1_F, (u64)hole); + if (!hole) + KUNIT_EXPECT_EQ(test, invalidated[step][i] & SYS_PAR_EL1_PA, + ctx->phys[i]); + } + } + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + KUNIT_EXPECT_EQ(test, result[i] & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, result[i] & SYS_PAR_EL1_PA, ctx->phys[i]); + KUNIT_EXPECT_EQ(test, writes[i] & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, writes[i] & SYS_PAR_EL1_PA, ctx->phys[i]); + } +} + +struct test_walk { + struct test_mm *ctx; + unsigned long size; + unsigned long holes; + unsigned int present; +}; + +/* Callbacks run under the PTE lock; record errors without invoking KUnit. */ +static int test_walk_pte(pte_t *pte, unsigned long addr, unsigned long next, + struct mm_walk *walk) +{ + struct test_walk *state = walk->private; + u64 value = pte_val(ptep_get(pte)); + unsigned int i; + + if (next - addr != state->size) + return -EINVAL; + if (!value) { + state->holes += next - addr; + return 0; + } + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + if (state->ctx->address[i] != addr) + continue; + if (value != (state->ctx->phys[i] | TEST_PROT)) + return -EINVAL; + state->present++; + return 0; + } + return -EINVAL; +} + +static int test_walk_hole(unsigned long addr, unsigned long next, int depth, + struct mm_walk *walk) +{ + struct test_walk *state = walk->private; + + state->holes += next - addr; + return 0; +} + +static const struct mm_walk_ops test_walk_ops = { + .pte_entry = test_walk_pte, + .pte_hole = test_walk_hole, +}; + +static pte_t *test_mm_lookup(struct mm_struct *mm, unsigned long addr, + spinlock_t **ptl) +{ + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + + if (pgd_none(*pgd) || pgd_bad(*pgd)) + return NULL; + p4d = p4d_offset_mm(mm, pgd, addr); + if (p4d_none_mm(mm, *p4d) || p4d_bad_mm(mm, *p4d)) + return NULL; + pud = pud_offset_mm(mm, p4d, addr); + if (pud_none(*pud) || pud_bad(*pud)) + return NULL; + pmd = pmd_offset_mm(mm, pud, addr); + if (pmd_none(*pmd) || pmd_bad(*pmd)) + return NULL; + return pte_offset_map_lock(mm, pmd, addr, ptl); +} + +static void test_mm_destroy(void *ptr) +{ + struct test_mm *ctx = ptr; + struct mm_struct *mm = ctx->mm; + struct mmu_gather tlb; + unsigned int i; + + if (!mm) + return; + mmap_write_lock(mm); + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, ctx->address[i], &ptl); + + if (pte) { + /* Test-owned physical pages, not faulted/rmap-owned pages. */ + pte_clear(mm, ctx->address[i], pte); + pte_unmap_unlock(pte, ptl); + } + } + /* Exercise the real generic gather/teardown path, not a test walker. */ + tlb_gather_mmu_fullmm(&tlb, mm); + free_pgd_range(&tlb, 0, 1UL << 48, 0, 0); + tlb_finish_mmu(&tlb); + mmap_write_unlock(mm); + KUNIT_EXPECT_EQ(ctx->test, mm_pgtables_bytes(mm), 0UL); + mmput(mm); + ctx->mm = NULL; +} + +static unsigned int test_mm_backing_pages(struct test_mm *ctx) +{ + struct page *owners[3 * MM_TEST_MAPPINGS]; + unsigned int count = 0, total = 0, i, j, level; + struct mm_struct *mm = ctx->mm; + + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + unsigned long addr = ctx->address[i]; + p4d_t *p4d = p4d_offset_mm(mm, pgd_offset(mm, addr), addr); + pud_t *pud = pud_offset_mm(mm, p4d, addr); + pmd_t *pmd = pmd_offset_mm(mm, pud, addr); + pte_t *pte = pte_offset_kernel_mm(mm, pmd, addr); + struct page *pages[] = { virt_to_page(pud), virt_to_page(pmd), virt_to_page(pte) }; + + for (level = mm_pud_folded(mm) ? 1 : 0; level < ARRAY_SIZE(pages); level++) { + pages[level] = compound_head(pages[level]); + for (j = 0; j < count; j++) + if (owners[j] == pages[level]) + break; + if (j == count) { + owners[count++] = pages[level]; + total += 1U << compound_order(pages[level]); + } + } + } + return total; +} + +/* Only valid for a private, empty mm with no ASID or hardware references. */ +static int test_mm_select_granule(struct mm_struct *mm, unsigned int shift) +{ + pgd_t *native_pgd = mm->pgd, *new_pgd; + unsigned long flag = shift == 12 ? MMCF_USER_4K : + shift == 14 ? MMCF_USER_16K : MMCF_USER_64K; + + mm->context.flags |= flag; + new_pgd = pgd_alloc(mm); + mm->context.flags &= ~flag; + if (!new_pgd) + return -ENOMEM; + pgd_free(mm, native_pgd); + mm->pgd = new_pgd; + mm->context.flags |= flag; + return 0; +} + +static int test_mm_select4k(struct mm_struct *mm) +{ + return test_mm_select_granule(mm, 12); +} + +static void test_mm_tables(struct kunit *test, unsigned int shift, bool hardware) +{ + struct test_mm *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + bool small = shift != PAGE_SHIFT; + unsigned int pmd_shift = small ? shift + (shift - 3) : PMD_SHIFT; + unsigned int pud_shift = small ? shift + 2 * (shift - 3) : PUD_SHIFT; + unsigned int pgd_shift = small ? (shift == 16 ? pud_shift : + shift + 3 * (shift - 3)) : PGDIR_SHIFT; + unsigned long size = 1UL << shift; + struct mm_struct *mm; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + ctx->test = test; + for (i = 0; i < 4; i++) + ctx->address[i] = TEST_VA + i * size; + ctx->address[4] = (1UL << pmd_shift) - size; + ctx->address[5] = 1UL << pmd_shift; + ctx->address[6] = (1UL << pud_shift) - size; + ctx->address[7] = 1UL << pud_shift; + ctx->address[8] = (1UL << pgd_shift) - size; + ctx->address[9] = 1UL << pgd_shift; + ctx->address[10] = (1UL << 48) - size; + if (pud_shift == pgd_shift) { + ctx->address[6] -= size; + ctx->address[7] += size; + } + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + void *data = test_alloc_granule(test, shift); + + KUNIT_ASSERT_NOT_NULL(test, data); + ctx->phys[i] = virt_to_phys(data) + (shift < PAGE_SHIFT ? (i % (PAGE_SIZE >> shift)) * size : 0); + } + mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + ctx->mm = mm; + /* Register after the data allocations: detach tables before freeing data. */ + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_mm_destroy, ctx), 0); + KUNIT_EXPECT_EQ(test, mm_page_shift(mm), (unsigned int)PAGE_SHIFT); + if (small) { + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, shift), 0); + KUNIT_EXPECT_EQ(test, (unsigned long)mm->pgd & (size - 1), 0UL); + } + KUNIT_EXPECT_EQ(test, mm_page_shift(mm), shift); + if (small) + KUNIT_EXPECT_EQ(test, mm_pud_folded(mm), shift == 16); + KUNIT_EXPECT_EQ(test, mm_page_size(mm), size); + KUNIT_EXPECT_EQ(test, mm_page_mask(mm), ~(size - 1)); + KUNIT_EXPECT_EQ(test, mm_page_shift(&init_mm), (unsigned int)PAGE_SHIFT); + KUNIT_EXPECT_EQ(test, pmd_addr_end_mm(mm, (1UL << pmd_shift) - 1, + (1UL << pmd_shift) + size), + 1UL << pmd_shift); + KUNIT_EXPECT_EQ(test, pgd_addr_end_mm(mm, ULONG_MAX - size, 0), 0UL); + + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + unsigned long addr = ctx->address[i]; + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d = p4d_alloc(mm, pgd, addr); + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + spinlock_t *ptl; + + KUNIT_ASSERT_NOT_NULL(test, p4d); + pud = pud_alloc(mm, p4d, addr); + KUNIT_ASSERT_NOT_NULL(test, pud); + pmd = pmd_alloc(mm, pud, addr); + KUNIT_ASSERT_NOT_NULL(test, pmd); + pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + /* Do not assert while holding the spinlock/RCU section. */ + set_pte(pte, __pte(ctx->phys[i] | TEST_PROT)); + pte_unmap_unlock(pte, ptl); + } + /* Installed table bytes and physical backing (including cached slots) differ. */ + { + unsigned int shifts[] = { pmd_shift, pud_shift, pgd_shift }; + unsigned int tables = 0, owners = 0; + + for (unsigned int level = 0; level < (mm_pud_folded(mm) ? 2 : 3); level++) { + unsigned int unique = 0; + + for (unsigned int a = 0; a < MM_TEST_MAPPINGS; a++) { + unsigned int b; + + for (b = 0; b < a; b++) + if ((ctx->address[a] >> shifts[level]) == + (ctx->address[b] >> shifts[level])) + break; + unique += b == a; + } + tables += unique; + if (shift < PAGE_SHIFT) + owners += DIV_ROUND_UP(unique, PAGE_SIZE >> shift); + else + owners += unique << (shift - PAGE_SHIFT); + } + KUNIT_EXPECT_EQ(test, mm_pgtables_bytes(mm), tables * size); + KUNIT_EXPECT_EQ(test, test_mm_backing_pages(ctx), owners); + } + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, ctx->address[i], &ptl); + u64 value; + + KUNIT_ASSERT_NOT_NULL(test, pte); + value = pte_val(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, value, ctx->phys[i] | TEST_PROT); + } + { + struct vm_area_struct *vma = vm_area_alloc(mm); + struct test_walk state = { .ctx = ctx, .size = size }; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + vma->vm_start = 0; + vma->vm_end = 1UL << 48; + mmap_write_lock(mm); + ret = walk_page_range_vma(vma, 0, 1UL << 48, &test_walk_ops, &state); + mmap_write_unlock(mm); + vm_area_free(vma); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, state.present, MM_TEST_MAPPINGS); + KUNIT_EXPECT_EQ(test, state.holes, (1UL << 48) - MM_TEST_MAPPINGS * size); + } + if (hardware) + test_mm_hardware(ctx); + { + struct mmu_gather tlb; + unsigned long before = mm_pgtables_bytes(mm); + unsigned long boundary = 1UL << pmd_shift; + + mmap_write_lock(mm); + /* An adjacent VMA's unaligned ceiling must keep the shared table. */ + tlb_gather_mmu_fullmm(&tlb, mm); + free_pgd_range(&tlb, 0, boundary - size, 0, boundary - size); + tlb_finish_mmu(&tlb); + KUNIT_EXPECT_EQ(test, mm_pgtables_bytes(mm), before); + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + spinlock_t *ptl; + pte_t *pte; + + if (ctx->address[i] >= boundary) + continue; + pte = test_mm_lookup(mm, ctx->address[i], &ptl); + if (pte) { + pte_clear(mm, ctx->address[i], pte); + pte_unmap_unlock(pte, ptl); + } + } + tlb_gather_mmu_fullmm(&tlb, mm); + free_pgd_range(&tlb, 0, boundary, 0, boundary); + tlb_finish_mmu(&tlb); + mmap_write_unlock(mm); + KUNIT_EXPECT_EQ(test, mm_pgtables_bytes(mm), before - size); + /* The next table and the higher levels must still be walkable. */ + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, ctx->address[i], &ptl); + u64 value; + + if (ctx->address[i] < boundary) { + if (pte) + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_PTR_EQ(test, pte, NULL); + continue; + } + KUNIT_ASSERT_NOT_NULL(test, pte); + value = pte_val(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, value, ctx->phys[i] | TEST_PROT); + } + } + test_mm_destroy(ctx); +} + +static void user4k_mm_tables_test(struct kunit *test) +{ + test_mm_tables(test, 12, false); +} + +static void native_mm_tables_test(struct kunit *test) +{ + test_mm_tables(test, PAGE_SHIFT, false); +} + +static void user16k_mm_tables_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative tables require larger native backing"); + return; + } + test_mm_tables(test, 14, false); +} + +static void user64k_mm_tables_test(struct kunit *test) +{ + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "64K is already the native geometry"); + return; + } + test_mm_tables(test, 16, false); +} + +static void test_free_vma(void *ptr) +{ + vm_area_free(ptr); +} + +static void user4k_anon_read_fault_test(struct kunit *test) +{ + struct test_mm *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + struct vm_area_struct *vma; + vm_flags_t vm_flags = VM_READ | VM_MAYREAD; + unsigned int i, j; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + ctx->test = test; + ctx->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, ctx->mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_mm_destroy, ctx), 0); + KUNIT_ASSERT_EQ(test, test_mm_select4k(ctx->mm), 0); + for (i = 0; i < 4; i++) + ctx->address[i] = TEST_VA + i * SZ_4K; + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + PAGE_SIZE; + vma_set_anonymous(vma); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + vm_flags |= VM_NOHUGEPAGE; +#endif + vm_flags_init(vma, vm_flags); + vma->vm_page_prot = vm_get_page_prot(vm_flags); + /* This VMA is a private fault fixture, not inserted into the VMA tree. */ + for (i = 0; i < 4; i++) { + vm_fault_t ret; + + mmap_read_lock(ctx->mm); + ret = handle_mm_fault(vma, ctx->address[i] + 1, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(ctx->mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + for (j = 0; j < 4; j++) { + spinlock_t *ptl; + pte_t *ptep = test_mm_lookup(ctx->mm, ctx->address[j], &ptl); + pte_t pte; + + KUNIT_ASSERT_NOT_NULL(test, ptep); + pte = ptep_get(ptep); + pte_unmap_unlock(ptep, ptl); + if (j > i) { + KUNIT_EXPECT_TRUE(test, pte_none(pte)); + continue; + } + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_TRUE(test, pte_special(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_TRUE(test, is_zero_pfn(pte_pfn(pte))); + } + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(ctx->mm, MM_ANONPAGES), 0L); +} + +static void user4k_mm_hardware_test(struct kunit *test) +{ + if (!user4k_test_enabled) { + kunit_skip(test, "requires arm64.user4k_test=1 in a disposable VM"); + return; + } + if (!system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || + cpus_have_final_cap(ARM64_HAS_S1POE)) { + kunit_skip(test, "requires 4K, no CnP, and direct PTE permissions"); + return; + } + test_mm_tables(test, 12, true); +} + +struct fragment_fixture { + struct mm_struct *mm; + void *table[TEST_SLOTS + 2]; + struct page *pinned[3]; +}; + +static void fragment_fixture_destroy(void *data) +{ + struct fragment_fixture *f = data; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(f->table); i++) + if (f->table[i]) + arm64_pgtable_free(f->table[i]); + if (f->mm) + mmput(f->mm); + rcu_barrier(); + for (i = 0; i < ARRAY_SIZE(f->pinned); i++) + if (f->pinned[i]) + put_page(f->pinned[i]); +} + +static void test_fragment_lifetime(struct kunit *test, unsigned int shift) +{ + struct fragment_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + struct ptdesc *pte_owner; + unsigned int i, slots = PAGE_SIZE >> shift; + unsigned long size = 1UL << shift; + unsigned int read_shift = 0; + bool live; + + KUNIT_ASSERT_NOT_NULL(test, f); + f->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, f->mm); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, fragment_fixture_destroy, f), 0); + /* Direct allocator fixture: the mm's native root is never changed or used. */ + for (i = 0; i < slots + 2; i++) { + enum arm64_user4k_pt_level level = i < slots ? USER4K_PT_PTE : i - slots + 1; + + f->table[i] = arm64_user_pt_alloc_granule(f->mm, level, shift); + KUNIT_ASSERT_NOT_NULL(test, f->table[i]); + KUNIT_EXPECT_TRUE(test, IS_ALIGNED((unsigned long)f->table[i], size)); + KUNIT_EXPECT_EQ(test, arm64_ptep_page_shift(f->table[i]), shift); + KUNIT_EXPECT_EQ(test, arm64_ptep_page_shift(f->table[i] + size - sizeof(pte_t)), shift); + KUNIT_EXPECT_EQ(test, pte_batch_hint(f->table[i], __pte(TEST_PROT | PTE_CONT)), + IS_ENABLED(CONFIG_ARM64_CONTPTE) ? arm64_cont_ptes(shift) : 1U); + if (!i && PAGE_SHIFT == 16) { + void *cached = f->mm->context.user4k_pt_frag[level]; + + KUNIT_EXPECT_PTR_EQ(test, arm64_user_pt_alloc_granule(f->mm, level, + shift == 12 ? 14 : 12), NULL); + KUNIT_EXPECT_PTR_EQ(test, f->mm->context.user4k_pt_frag[level], cached); + } + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0, size), NULL); + if (i == 0 || i >= slots) { + unsigned int slot = i == 0 ? 0 : i - slots + 1; + + f->pinned[slot] = virt_to_page(f->table[i]); + get_page(f->pinned[slot]); + } else { + KUNIT_EXPECT_PTR_EQ(test, virt_to_page(f->table[i]), f->pinned[0]); + KUNIT_EXPECT_EQ(test, (unsigned long)f->table[i] - + (unsigned long)f->table[0], i * (unsigned long)size); + } + memset(f->table[i], 0x50 + i, size); + } + for (i = 0; i < slots + 2; i++) { + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0x50 + i, size), NULL); + memset(f->table[i], 0, size); + } + /* Pointer-only walkers must inspect all 128 entries of a 16K group. */ + if (IS_ENABLED(CONFIG_ARM64_CONTPTE)) { + pte_t *ptes = f->table[0]; + unsigned int nr = arm64_cont_ptes(shift), j; + pte_t observed; + + for (j = 0; j < nr; j++) + ptes[j] = pte_mkold(pte_mkclean(__pte(TEST_PROT | PTE_CONT | + (SZ_4M + (j << shift))))); + ptes[nr - 1] = pte_mkyoung(pte_mkdirty(ptes[nr - 1])); + observed = ptep_get(ptes + 1); + KUNIT_EXPECT_TRUE(test, pte_young(observed)); + KUNIT_EXPECT_TRUE(test, pte_dirty(observed)); + observed = ptep_get_lockless(ptes + 1); + KUNIT_EXPECT_TRUE(test, pte_young(observed)); + KUNIT_EXPECT_TRUE(test, pte_dirty(observed)); + KUNIT_EXPECT_EQ(test, pte_val(observed) & + (ARM64_USER4K_TABLE_ADDR_MASK & ~(size - 1)), + (u64)(SZ_4M + size)); + memset(ptes, 0, size); + } + KUNIT_EXPECT_PTR_NE(test, f->pinned[0], f->pinned[1]); + KUNIT_EXPECT_PTR_NE(test, f->pinned[1], f->pinned[2]); + pte_owner = page_ptdesc(f->pinned[0]); + KUNIT_EXPECT_EQ(test, atomic_read(&pte_owner->pt_frag_refcount), (int)slots); + arm64_pte_free_defer(f->mm, f->table[0]); + f->table[0] = NULL; + KUNIT_EXPECT_EQ(test, atomic_read(&pte_owner->pt_frag_refcount), (int)slots - 1); + for (i = 1; i < slots - 1; i++) { + arm64_pgtable_free(f->table[i]); + f->table[i] = NULL; + } + /* A pre-existing software walker must survive the last fragment release. */ + rcu_read_lock(); + arm64_pgtable_free(f->table[slots - 1]); + f->table[slots - 1] = NULL; + live = folio_test_pgtable(page_folio(f->pinned[0])); + read_shift = arm64_ptep_page_shift(page_address(f->pinned[0])); + rcu_read_unlock(); + KUNIT_EXPECT_TRUE(test, live); + KUNIT_EXPECT_EQ(test, read_shift, shift); + rcu_barrier(); + KUNIT_EXPECT_FALSE(test, folio_test_pgtable(page_folio(f->pinned[0]))); + KUNIT_EXPECT_EQ(test, page_count(f->pinned[0]), 1); + for (i = slots; i < slots + 2; i++) { + struct ptdesc *pt = virt_to_ptdesc(f->table[i]); + + arm64_pgtable_free(f->table[i]); + f->table[i] = NULL; + KUNIT_EXPECT_EQ(test, atomic_read(&pt->pt_frag_refcount), (int)slots - 1); + KUNIT_EXPECT_TRUE(test, folio_test_pgtable(ptdesc_folio(pt))); + } + mmput(f->mm); + f->mm = NULL; + for (i = 1; i < 3; i++) { + KUNIT_EXPECT_FALSE(test, folio_test_pgtable(page_folio(f->pinned[i]))); + KUNIT_EXPECT_EQ(test, page_count(f->pinned[i]), 1); + } +} + +/* Direct table-owner test; no larger-than-native user mm is enabled here. */ +static void user64k_table_owner_test(struct kunit *test) +{ + struct fragment_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + unsigned int i, offset, order = 16 - PAGE_SHIFT; + unsigned int shift; + bool live; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, fragment_fixture_destroy, f), 0); + f->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, f->mm); + for (i = 0; i < 3; i++) { + struct ptdesc *pt; + + f->table[i] = arm64_user_pt_alloc_granule(f->mm, i, 16); + KUNIT_ASSERT_NOT_NULL(test, f->table[i]); + pt = virt_to_ptdesc(f->table[i]); + f->pinned[i] = ptdesc_page(pt); + get_page(f->pinned[i]); + KUNIT_EXPECT_TRUE(test, IS_ALIGNED((unsigned long)f->table[i], SZ_64K)); + KUNIT_EXPECT_EQ(test, compound_order(f->pinned[i]), order); + KUNIT_EXPECT_EQ(test, atomic_read(&pt->pt_frag_refcount), 1); + KUNIT_EXPECT_PTR_EQ(test, f->mm->context.user4k_pt_frag[i], NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0, SZ_64K), NULL); + /* Include the last entry of every constituent native page. */ + for (offset = PAGE_SIZE - sizeof(pte_t); offset < SZ_64K; offset += PAGE_SIZE) { + void *entry = f->table[i] + offset; + + KUNIT_EXPECT_PTR_EQ(test, virt_to_ptdesc(entry), pt); + KUNIT_EXPECT_EQ(test, arm64_ptep_page_shift(entry), 16U); + KUNIT_EXPECT_EQ(test, pte_batch_hint(entry, __pte(TEST_PROT | PTE_CONT)), + 32U - ((offset / sizeof(pte_t)) % 32)); + if (i == USER4K_PT_PTE) + KUNIT_EXPECT_PTR_EQ(test, ptep_lockptr(f->mm, entry), + ptep_lockptr(f->mm, f->table[i])); + if (i == USER4K_PT_PMD) + KUNIT_EXPECT_PTR_EQ(test, pmd_lockptr(f->mm, entry), + pmd_lockptr(f->mm, f->table[i])); + } + memset(f->table[i], 0x51 + i, SZ_64K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0x51 + i, SZ_64K), NULL); + memset(f->table[i], 0, SZ_64K); + } + /* Contiguous readers on the last native page must still use 64K geometry. */ + if (IS_ENABLED(CONFIG_ARM64_CONTPTE)) { + pte_t *ptes = f->table[0] + SZ_64K - 32 * sizeof(pte_t); + pte_t observed; + unsigned int j; + + for (j = 0; j < 32; j++) + ptes[j] = pte_mkold(pte_mkclean(__pte(TEST_PROT | PTE_CONT | + (SZ_4M + (j << 16))))); + ptes[31] = pte_mkyoung(pte_mkdirty(ptes[31])); + observed = ptep_get(ptes + 1); + KUNIT_EXPECT_TRUE(test, pte_young(observed)); + KUNIT_EXPECT_TRUE(test, pte_dirty(observed)); + observed = ptep_get_lockless(ptes + 1); + KUNIT_EXPECT_TRUE(test, pte_young(observed)); + KUNIT_EXPECT_TRUE(test, pte_dirty(observed)); + KUNIT_EXPECT_EQ(test, pte_val(observed) & + (ARM64_USER4K_TABLE_ADDR_MASK & ~(SZ_64K - 1UL)), + (u64)(SZ_4M + SZ_64K)); + memset(f->table[0], 0, SZ_64K); + } + rcu_read_lock(); + arm64_pte_free_defer(f->mm, f->table[0]); + live = folio_test_pgtable(page_folio(f->pinned[0])); + shift = arm64_ptep_page_shift(f->table[0] + SZ_64K - sizeof(pte_t)); + f->table[0] = NULL; + rcu_read_unlock(); + KUNIT_EXPECT_TRUE(test, live); + KUNIT_EXPECT_EQ(test, shift, 16U); + rcu_barrier(); + for (i = 0; i < 3; i++) { + if (f->table[i]) { + arm64_pgtable_free(f->table[i]); + f->table[i] = NULL; + } + KUNIT_EXPECT_FALSE(test, folio_test_pgtable(page_folio(f->pinned[i]))); + KUNIT_EXPECT_EQ(test, page_count(f->pinned[i]), 1); + KUNIT_EXPECT_EQ(test, compound_order(f->pinned[i]), order); + } +} + +static void user4k_fragment_lifetime_test(struct kunit *test) +{ + test_fragment_lifetime(test, 12); +} + +static void user16k_fragment_lifetime_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K fragments require larger native backing"); + return; + } + test_fragment_lifetime(test, 14); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static void test_put_mm(void *ptr) +{ + mmput(ptr); +} + +static void native_pgtable_deposit_test(struct kunit *test) +{ + struct mm_struct *mm = mm_alloc(); + pgtable_t first, second, out1, out2; + pmd_t *pmd; + spinlock_t *ptl; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_put_mm, mm), 0); + pmd = pmd_alloc_one(mm, 0); + KUNIT_ASSERT_NOT_NULL(test, pmd); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, arm64_pgtable_free, pmd), 0); + first = pte_alloc_one(mm); + KUNIT_ASSERT_NOT_NULL(test, first); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, arm64_pgtable_free, first), 0); + second = pte_alloc_one(mm); + KUNIT_ASSERT_NOT_NULL(test, second); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, arm64_pgtable_free, second), 0); + ptl = pmd_lock(mm, pmd); + pgtable_trans_huge_deposit(mm, pmd, first); + pgtable_trans_huge_deposit(mm, pmd, second); + out1 = pgtable_trans_huge_withdraw(mm, pmd); + out2 = pgtable_trans_huge_withdraw(mm, pmd); + spin_unlock(ptl); + KUNIT_EXPECT_PTR_EQ(test, out1, second); + KUNIT_EXPECT_PTR_EQ(test, out2, first); +} +#endif + +static void test_data_slot_put(void *ptr) +{ + mm_subpage_put(ptr); +} + +static void test_data_pool_free(void *ptr) +{ + mm_subpage_pool_close(ptr); + mm_subpage_pool_put(ptr); +} + +static bool test_alloc_data_slots(struct kunit *test, struct test_mm *ctx) +{ + struct mm_subpage_pool *pool = mm_subpage_pool_create(GFP_KERNEL); + struct folio *folio; + unsigned int i; + int ret; + + if (!pool) + return false; + if (kunit_add_action_or_reset(test, test_data_pool_free, pool)) + return false; + folio = folio_alloc(GFP_KERNEL, 0); + if (!folio) + return false; + ret = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (ret) { + folio_put(folio); + return false; + } + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc(pool); + + if (IS_ERR(slot)) + return false; + if (kunit_add_action_or_reset(test, test_data_slot_put, slot)) + return false; + ctx->phys[i] = mm_subpage_phys(slot); + } + return true; +} + +/* These private mappings are borrowed only for bounded, fault-disabled uaccess. */ +static struct test_mm *test_usercopy_mm(struct kunit *test, bool small) +{ + struct test_mm *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + unsigned long size = small ? SZ_4K : PAGE_SIZE; + unsigned int i; + + if (!ctx) + return NULL; + ctx->test = test; + if (small && !test_alloc_data_slots(test, ctx)) + return NULL; + for (i = 0; i < 4; i++) { + if (!small) { + void *data = test_alloc_page(test); + + if (!data) + return NULL; + ctx->phys[i] = virt_to_phys(data); + } + ctx->address[i] = TEST_VA + i * size; + memset(phys_to_virt(ctx->phys[i]), (small ? 0x40 : 0x80) + i, 64); + } + ctx->mm = mm_alloc(); + if (!ctx->mm) + return NULL; + if (kunit_add_action_or_reset(test, test_mm_destroy, ctx)) + return NULL; + if (small && test_mm_select4k(ctx->mm)) + return NULL; + for (i = 0; i < 4; i++) { + unsigned long addr = ctx->address[i]; + pgd_t *pgd = pgd_offset(ctx->mm, addr); + p4d_t *p4d = p4d_alloc(ctx->mm, pgd, addr); + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + spinlock_t *ptl; + + if (!p4d) + return NULL; + pud = pud_alloc(ctx->mm, p4d, addr); + if (!pud) + return NULL; + pmd = pmd_alloc(ctx->mm, pud, addr); + if (!pmd) + return NULL; + pte = pte_alloc_map_lock(ctx->mm, pmd, addr, &ptl); + if (!pte) + return NULL; + if (i != 2) + set_pte(pte, phys_pte_mm(ctx->mm, ctx->phys[i], + __pgprot(TEST_PROT | (i ? 0 : PTE_RDONLY)))); + pte_unmap_unlock(pte, ptl); + } + return ctx; +} + +static void user4k_rmap_walk_test(struct kunit *test) +{ + struct test_mm *ctx = test_usercopy_mm(test, true); + struct vm_area_struct *vma; + unsigned int i, j, sync; + spinlock_t *ptl; + pte_t *pte; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + 2 * PAGE_SIZE; + /* A later alias must not extend a native-owner range walk's bound. */ + ctx->address[4] = TEST_VA + PAGE_SIZE; + ctx->phys[4] = ctx->phys[0]; + pte = test_mm_lookup(ctx->mm, ctx->address[4], &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte(pte, phys_pte_mm(ctx->mm, ctx->phys[4], __pgprot(TEST_PROT))); + pte_unmap_unlock(pte, ptl); + /* Same physical slot at a different virtual byte offset and native index. */ + ctx->address[5] = TEST_VA + PAGE_SIZE + SZ_4K; + ctx->phys[5] = ctx->phys[3]; + pte = test_mm_lookup(ctx->mm, ctx->address[5], &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte(pte, phys_pte_mm(ctx->mm, ctx->phys[5], __pgprot(TEST_PROT))); + pte_unmap_unlock(pte, ptl); + for (sync = 0; sync < 2; sync++) { + struct page_vma_mapped_walk whole = { + .pfn = ctx->phys[0] >> PAGE_SHIFT, + .nr_pages = 1, + .vma = vma, + .address = TEST_VA, + .flags = sync ? PVMW_SYNC : 0, + }; + unsigned int count = 0; + bool valid = true; + + /* A native-owner walk must visit every mapped 4K occupant. */ + while (page_vma_mapped_walk(&whole)) { + unsigned int index = (whole.address - TEST_VA) / SZ_4K; + + if (index >= 4 || index == 2 || !whole.pte || + pte_phys_mm(ctx->mm, ptep_get(whole.pte)) != ctx->phys[index]) + valid = false; + if (++count > 4) { + page_vma_mapped_walk_done(&whole); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, 3U); + for (i = 0; i < 4; i++) { + for (j = 0; j < 6; j++) { + struct rmap_walk_range range = { + .address = ctx->address[j], + .subpage = true, + .subpage_offset = offset_in_page(ctx->phys[i]), + }; + DEFINE_FOLIO_RMAP_WALK(one, page_folio(pfn_to_page(PHYS_PFN(ctx->phys[i]))), + vma, range, sync ? PVMW_SYNC : 0); + struct mm_subpage *slot = NULL; + bool found = page_vma_mapped_walk(&one); + phys_addr_t phys = 0; + + if (found) { + phys = pte_phys_mm(ctx->mm, ptep_get(one.pte)); + slot = mm_subpage_get_from_phys(phys); + page_vma_mapped_walk_done(&one); + } + KUNIT_EXPECT_EQ(test, found, (i == j && j != 2) || + (j == 4 && i == 0) || (j == 5 && i == 3)); + if (found) { + KUNIT_EXPECT_EQ(test, phys, ctx->phys[i]); + KUNIT_EXPECT_NOT_NULL(test, slot); + if (slot) { + KUNIT_EXPECT_EQ(test, mm_subpage_phys(slot), phys); + mm_subpage_put(slot); + } + } + } + } + } +} + +static int test_map_alias(struct test_mm *ctx, unsigned int index, + unsigned long addr, phys_addr_t phys) +{ + struct mm_struct *mm = ctx->mm; + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d = p4d_alloc(mm, pgd, addr); + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + spinlock_t *ptl; + + ctx->address[index] = addr; + ctx->phys[index] = phys; + if (!p4d) + return -ENOMEM; + pud = pud_alloc(mm, p4d, addr); + if (!pud) + return -ENOMEM; + pmd = pmd_alloc(mm, pud, addr); + if (!pmd) + return -ENOMEM; + pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); + if (!pte) + return -ENOMEM; + set_pte(pte, pte_mkspecial(phys_pte_mm(mm, phys, __pgprot(TEST_PROT)))); + pte_unmap_unlock(pte, ptl); + return 0; +} + +/* Physical-range lookup only; coarse anonymous ownership is a separate step. */ +static void user64k_rmap_lookup_test(struct kunit *test) +{ + struct test_mm *ctx; + struct vm_area_struct *vma; + void *data; + unsigned long pfn, per_leaf = SZ_64K / PAGE_SIZE; + unsigned int mode, sync, i; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires userspace leaf larger than native page"); + return; + } + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ctx); + ctx->test = test; + data = test_alloc_granule(test, 17); + KUNIT_ASSERT_NOT_NULL(test, data); + pfn = virt_to_phys(data) >> PAGE_SHIFT; + ctx->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, ctx->mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_mm_destroy, ctx), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(ctx->mm, 16), 0); + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = (1UL << 29) - SZ_64K; + vma->vm_end = vma->vm_start + 4 * SZ_64K; + vma->vm_pgoff = 4096; + page_folio(virt_to_page(data))->index = vma->vm_pgoff; + for (i = 0; i < 3; i++) + KUNIT_ASSERT_EQ(test, test_map_alias(ctx, i, vma->vm_start + i * SZ_64K, + virt_to_phys(data) + (i == 1 ? SZ_64K : 0)), 0); + + for (mode = 0; mode < 2; mode++) { + if (mode) { + for (i = 0; i < 3; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(ctx->mm, ctx->address[i], &ptl); + + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte(pte, swp_entry_to_pte(make_readable_migration_entry( + ctx->phys[i] >> PAGE_SHIFT))); + pte_unmap_unlock(pte, ptl); + } + } + for (sync = 0; sync < 2; sync++) { + unsigned int flags = (sync ? PVMW_SYNC : 0) | + (mode ? PVMW_MIGRATION : 0); + + for (i = 0; i < 2 * per_leaf; i++) { + struct page_vma_mapped_walk walk = { + .pfn = pfn + i, .nr_pages = 1, + /* Deliberately invalid pgoff: single-page callers may omit it. */ + .pgoff = 0, .vma = vma, + .address = vma->vm_start + i * PAGE_SIZE, + .flags = flags, + }; + unsigned int count = 0; + bool valid = true; + + while (page_vma_mapped_walk(&walk)) { + valid &= walk.pte && walk.address == + vma->vm_start + (i / per_leaf) * SZ_64K; + if (++count > 1) { + page_vma_mapped_walk_done(&walk); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, 1U); +#ifdef CONFIG_MEMORY_FAILURE + if (!mode) + KUNIT_EXPECT_EQ(test, page_mapped_in_vma(pfn_to_page(pfn + i), vma), + vma->vm_start + i * PAGE_SIZE); +#endif + } + /* Two native pages straddle a coarse-leaf and PTE-table boundary. */ + { + struct page_vma_mapped_walk walk = { + .pfn = pfn + per_leaf - 1, .nr_pages = 2, + .pgoff = vma->vm_pgoff + per_leaf - 1, .vma = vma, + .address = vma->vm_start + SZ_64K - PAGE_SIZE, + .flags = flags, + }; + unsigned int count = 0; + bool valid = true; + + KUNIT_EXPECT_EQ(test, vma_address_end(&walk), vma->vm_start + 2 * SZ_64K); + while (page_vma_mapped_walk(&walk)) { + valid &= walk.address == vma->vm_start + count * SZ_64K; + if (++count > 2) { + page_vma_mapped_walk_done(&walk); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, 2U); + KUNIT_EXPECT_TRUE(test, walk.flags & PVMW_PGTABLE_CROSSED); + } + for (i = 0; i < 2; i++) { + struct page_vma_mapped_walk walk = { + .pfn = i ? pfn + 2 * per_leaf : pfn - 1, + .nr_pages = 1, .vma = vma, + .address = vma->vm_start, .flags = flags, + }; + bool found = page_vma_mapped_walk(&walk); + + if (found) + page_vma_mapped_walk_done(&walk); + KUNIT_EXPECT_FALSE(test, found); + } + } + } +} + +static void user4k_rmap_boundary_test(struct kunit *test) +{ + struct test_mm *ctx = test_usercopy_mm(test, true); + struct vm_area_struct *vma; + unsigned int i, count = 0; + struct page_vma_mapped_walk walk; + bool valid = true; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = SZ_2M - SZ_4K; + vma->vm_end = vma->vm_start + PAGE_SIZE; + for (i = 0; i < 3; i++) + KUNIT_ASSERT_EQ(test, test_map_alias(ctx, 4 + i, vma->vm_start + i * SZ_4K, + ctx->phys[i]), 0); + walk = (struct page_vma_mapped_walk) { + .pfn = ctx->phys[0] >> PAGE_SHIFT, + .nr_pages = 1, + .vma = vma, + .address = vma->vm_start, + .flags = PVMW_SYNC, + }; + while (page_vma_mapped_walk(&walk)) { + if (count >= 3 || walk.address != vma->vm_start + count * SZ_4K || + pte_phys_mm(ctx->mm, ptep_get(walk.pte)) != ctx->phys[count]) + valid = false; + if (++count > 3) { + page_vma_mapped_walk_done(&walk); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, 3U); + KUNIT_EXPECT_TRUE(test, walk.flags & PVMW_PGTABLE_CROSSED); +} + +static void native_rmap_walk_test(struct kunit *test) +{ + struct test_mm *ctx = test_usercopy_mm(test, false); + struct vm_area_struct *vma; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + 4 * PAGE_SIZE; + for (i = 0; i < 4; i++) { + struct page_vma_mapped_walk walk = { + .pfn = ctx->phys[i] >> PAGE_SHIFT, + .nr_pages = 1, + .vma = vma, + .address = ctx->address[i], + .flags = PVMW_SYNC, + }; + unsigned int count = 0; + bool valid = true; + + while (page_vma_mapped_walk(&walk)) { + if (walk.address != ctx->address[i] || + pte_phys_mm(ctx->mm, ptep_get(walk.pte)) != ctx->phys[i]) + valid = false; + if (++count > 1) { + page_vma_mapped_walk_done(&walk); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, i != 2 ? 1U : 0U); +#ifdef CONFIG_MEMORY_FAILURE + page_folio(pfn_to_page(ctx->phys[i] >> PAGE_SHIFT))->index = i; + KUNIT_EXPECT_EQ(test, page_mapped_in_vma(pfn_to_page(ctx->phys[i] >> PAGE_SHIFT), vma), + i != 2 ? ctx->address[i] : (unsigned long)-EFAULT); +#endif + } +} + +static void user4k_vma_offsets_test(struct kunit *test) +{ + struct vm_area_struct *vma = kunit_kzalloc(test, sizeof(*vma), GFP_KERNEL); + unsigned int slot, length, step; + + KUNIT_ASSERT_NOT_NULL(test, vma); + vma->vm_start = (1UL << 48) - 8 * PAGE_SIZE; + for (slot = 0; slot < TEST_SLOTS; slot++) { + for (length = 1; length <= 8; length++) { + struct vm_page_offset initial = { ULONG_MAX - 3, slot * SZ_4K }; + struct vm_page_offset pos; + + vma->vm_end = vma->vm_start + length * SZ_4K; + vma_set_page_offset(vma, initial); + KUNIT_EXPECT_EQ(test, vma_last_pgoff(vma), + initial.index + (slot + length - 1) / TEST_SLOTS); + for (step = 0; step < length; step++) { + unsigned long addr = vma->vm_start + step * SZ_4K; + + pos = vma_page_offset_at(vma, addr); + KUNIT_EXPECT_EQ(test, pos.index, initial.index + (slot + step) / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, pos.offset, ((slot + step) % TEST_SLOTS) * (unsigned int)SZ_4K); + KUNIT_EXPECT_EQ(test, linear_page_index(vma, addr), pos.index); + KUNIT_EXPECT_EQ(test, vma_address_at_offset(vma, pos), addr); + } + pos = vma_page_offset_at(vma, vma->vm_end); + KUNIT_EXPECT_EQ(test, vma_address_at_offset(vma, pos), (unsigned long)-EFAULT); + pos = initial; + pos.index--; + KUNIT_EXPECT_EQ(test, vma_address_at_offset(vma, pos), (unsigned long)-EFAULT); + if (slot) { + pos = initial; + pos.offset--; + KUNIT_EXPECT_EQ(test, vma_address_at_offset(vma, pos), (unsigned long)-EFAULT); + } + } + } +} + +static void user4k_vma_intervals_test(struct kunit *test) +{ + struct vm_area_struct *vmas = kunit_kcalloc(test, 3, sizeof(*vmas), GFP_KERNEL); + struct rb_root_cached root = RB_ROOT_CACHED; + struct vm_area_struct *vma; + unsigned int i, count = 0; + + KUNIT_ASSERT_NOT_NULL(test, vmas); + for (i = 0; i < 3; i++) { + vmas[i].vm_start = TEST_VA + i * SZ_8K; + vmas[i].vm_end = vmas[i].vm_start + SZ_4K; + vma_set_page_offset(&vmas[i], (struct vm_page_offset) { 77, i * SZ_4K }); + vma_interval_tree_insert(&vmas[i], &root); + } + vma_interval_tree_foreach(vma, &root, 77, 77) { + unsigned long address = vma_address_at_offset(vma, + (struct vm_page_offset) { 77, SZ_4K }); + + KUNIT_EXPECT_EQ(test, address, + vma == &vmas[1] ? vma->vm_start : (unsigned long)-EFAULT); + count++; + } + KUNIT_EXPECT_EQ(test, count, 3U); + KUNIT_EXPECT_PTR_EQ(test, vma_interval_tree_iter_first(&root, 76, 76), NULL); + KUNIT_EXPECT_PTR_EQ(test, vma_interval_tree_iter_first(&root, 78, 78), NULL); + for (i = 0; i < 3; i++) + vma_interval_tree_remove(&vmas[i], &root); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&root.rb_root)); +} + +static void user4k_vma_mm_free(void *arg) +{ + mmput(arg); +} + +/* These private mms have no concurrent VMA writers after dropping the lock. */ +static struct vm_area_struct *test_vma_lookup(struct mm_struct *mm, unsigned long addr) +{ + struct vm_area_struct *vma; + + mmap_read_lock(mm); + vma = vma_lookup(mm, addr); + mmap_read_unlock(mm); + return vma; +} + +static void user4k_vma_split_test(struct kunit *test) +{ + struct mm_struct *mm = mm_alloc(); + struct vm_area_struct *vma, *middle, *part; + unsigned long start = TEST_VA + SZ_4K; + vma_flags_t flags; + int ret; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select4k(mm), 0); + vma = vm_area_alloc(mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + /* Dummy vm_ops keep insert_vm_struct from replacing this file-like offset. */ + vma->vm_start = start; + vma->vm_end = start + 3 * SZ_4K; + vm_flags_init(vma, VM_READ | VM_MAYREAD | VM_MAYWRITE); + vma_set_page_offset(vma, (struct vm_page_offset) { 77, 3 * SZ_4K }); + mmap_write_lock(mm); + ret = insert_vm_struct(mm, vma); + if (ret) { + mmap_write_unlock(mm); + vm_area_free(vma); + KUNIT_FAIL(test, "insert_vm_struct: %d", ret); + return; + } + { + VMA_ITERATOR(vmi, mm, start); + + vma_iter_load(&vmi); + flags = vma->flags; + vma_flags_set(&flags, VMA_WRITE_BIT); + middle = vma_modify_flags(&vmi, vma, vma, start + SZ_4K, + start + 2 * SZ_4K, &flags); + } + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, middle); + KUNIT_EXPECT_EQ(test, mm->map_count, 3); + for (i = 0; i < 3; i++) { + part = test_vma_lookup(mm, start + i * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, part); + KUNIT_EXPECT_EQ(test, part->vm_start, start + i * SZ_4K); + KUNIT_EXPECT_EQ(test, part->vm_end, start + (i + 1) * SZ_4K); + KUNIT_EXPECT_EQ(test, part->vm_pgoff, 77UL + (i + 3) / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(part), ((i + 3) % TEST_SLOTS) * (unsigned int)SZ_4K); + } +} + +static struct mm_struct *test_vma_mm_granule(struct kunit *test, unsigned int shift) +{ + struct mm_struct *mm = mm_alloc(); + + if (!mm) + return NULL; + if (kunit_add_action_or_reset(test, user4k_vma_mm_free, mm)) + return NULL; + if (test_mm_select_granule(mm, shift)) + return NULL; + return mm; +} + +static struct mm_struct *test_vma_mm(struct kunit *test) +{ + return test_vma_mm_granule(test, 12); +} + +static struct vm_area_struct *test_vma_add(struct mm_struct *mm, unsigned long start, + unsigned long len, struct vm_page_offset pos, + vm_flags_t flags) +{ + struct vm_area_struct *vma = vm_area_alloc(mm); + int ret; + + if (!vma) + return ERR_PTR(-ENOMEM); + vma->vm_start = start; + vma->vm_end = start + len; + vm_flags_init(vma, flags | VM_MAYREAD | VM_MAYWRITE); + vma_set_page_offset(vma, pos); + mmap_write_lock(mm); + ret = insert_vm_struct(mm, vma); + mmap_write_unlock(mm); + if (ret) { + vm_area_free(vma); + return ERR_PTR(ret); + } + return vma; +} + +static void user4k_vma_merge_test(struct kunit *test) +{ + unsigned long start = TEST_VA + SZ_4K; + unsigned int scenario, i; + + for (scenario = 0; scenario < 4; scenario++) { + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vmas[3], *merged; + struct vm_page_offset initial = { 77, 3 * SZ_4K }; + struct vm_page_offset expected; + unsigned long left = scenario >= 2 ? SZ_4K : 0; + unsigned long right = scenario == 1 || scenario == 3 ? 2 * SZ_4K : 3 * SZ_4K; + vma_flags_t flags; + + KUNIT_ASSERT_NOT_NULL(test, mm); + for (i = 0; i < 3; i++) { + struct vm_page_offset pos = vm_page_offset_add(initial, i * SZ_4K); + vm_flags_t bits = VM_READ; + + if (i == 1) + bits |= VM_WRITE; + if ((!i && scenario >= 2) || (i == 2 && scenario == 1)) + bits |= VM_EXEC; + /* Same native index, wrong byte offset: this must not merge. */ + if (i == 2 && scenario == 3) + pos.offset = 0; + vmas[i] = test_vma_add(mm, start + i * SZ_4K, SZ_4K, pos, bits); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vmas[i]); + } + flags = vmas[1]->flags; + vma_flags_clear(&flags, VMA_WRITE_BIT); + mmap_write_lock(mm); + { + VMA_ITERATOR(vmi, mm, start + SZ_4K); + + vma_iter_load(&vmi); + merged = vma_modify_flags(&vmi, vmas[0], vmas[1], start + SZ_4K, + start + 2 * SZ_4K, &flags); + if (!IS_ERR(merged)) { + vma_start_write(merged); + merged->flags = flags; + } + } + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, merged); + expected = vm_page_offset_add(initial, left); + KUNIT_EXPECT_EQ(test, mm->map_count, scenario == 0 ? 1 : scenario == 3 ? 3 : 2); + KUNIT_EXPECT_EQ(test, merged->vm_start, start + left); + KUNIT_EXPECT_EQ(test, merged->vm_end, start + right); + KUNIT_EXPECT_EQ(test, merged->vm_pgoff, expected.index); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(merged), expected.offset); + } +} + +static void user4k_vma_partial_merge_test(struct kunit *test) +{ + unsigned long start = TEST_VA; + unsigned int side; + + for (side = 0; side < 2; side++) { + struct mm_struct *mm = test_vma_mm(test); + struct vm_page_offset initial = { 77, 2 * SZ_4K }; + struct vm_area_struct *first, *second, *merged, *remaining; + unsigned long first_len = side ? SZ_4K : 2 * SZ_4K; + vma_flags_t flags; + struct vm_page_offset expected; + + KUNIT_ASSERT_NOT_NULL(test, mm); + first = test_vma_add(mm, start, first_len, initial, VM_READ | (side ? 0 : VM_WRITE)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, first); + second = test_vma_add(mm, start + first_len, 3 * SZ_4K - first_len, + vm_page_offset_add(initial, first_len), VM_READ | (side ? VM_WRITE : 0)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, second); + flags = side ? first->flags : second->flags; + mmap_write_lock(mm); + { + VMA_ITERATOR(vmi, mm, start + SZ_4K); + + vma_iter_load(&vmi); + merged = vma_modify_flags(&vmi, first, side ? second : first, + start + SZ_4K, start + 2 * SZ_4K, &flags); + } + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, merged); + KUNIT_EXPECT_EQ(test, mm->map_count, 2); + KUNIT_EXPECT_EQ(test, merged->vm_start, start + (side ? 0 : SZ_4K)); + KUNIT_EXPECT_EQ(test, merged->vm_end, start + (side ? 2 : 3) * SZ_4K); + expected = vm_page_offset_add(initial, side ? 0 : SZ_4K); + KUNIT_EXPECT_EQ(test, merged->vm_pgoff, expected.index); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(merged), expected.offset); + remaining = test_vma_lookup(mm, start + (side ? 2 * SZ_4K : 0)); + KUNIT_ASSERT_NOT_NULL(test, remaining); + expected = vm_page_offset_add(initial, side ? 2 * SZ_4K : 0); + KUNIT_EXPECT_EQ(test, remaining->vm_pgoff, expected.index); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(remaining), expected.offset); + } +} + +static void user4k_vma_copy_shrink_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *source, *dest; + struct vm_page_offset initial = { 77, 3 * SZ_4K }; + unsigned long start = TEST_VA + SZ_4K; + unsigned long target = TEST_VA + 16 * PAGE_SIZE + SZ_4K; + bool need_locks = false; + unsigned int i; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + source = test_vma_add(mm, start, PAGE_SIZE, initial, VM_READ); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, source); + for (i = 0; i < 3; i++) { + unsigned long addr = i == 2 ? target - SZ_4K : target + i * SZ_4K; + struct vm_page_offset pos = vm_page_offset_add(initial, i == 2 ? 0 : (i + 1) * SZ_4K); + + mmap_write_lock(mm); + dest = copy_vma(&source, addr, SZ_4K, pos, &need_locks, false); + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_NULL(test, dest); + KUNIT_EXPECT_EQ(test, mm->map_count, 2); + KUNIT_EXPECT_EQ(test, dest->vm_start, i == 2 ? target - SZ_4K : target); + KUNIT_EXPECT_EQ(test, dest->vm_end, i ? target + 2 * SZ_4K : target + SZ_4K); + KUNIT_EXPECT_EQ(test, dest->vm_pgoff, 77UL + (i == 2 ? 3UL : 4UL) / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(dest), ((i == 2 ? 3U : 4U) % TEST_SLOTS) * SZ_4K); + KUNIT_EXPECT_EQ(test, source->vm_pgoff, initial.index); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(source), initial.offset); + } + mmap_write_lock(mm); + { + VMA_ITERATOR(vmi, mm, dest->vm_start); + struct vm_page_offset pos = vma_page_offset_at(dest, target); + + vma_iter_load(&vmi); + ret = vma_shrink(&vmi, dest, target, dest->vm_end, pos); + } + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, dest->vm_start, target); + KUNIT_EXPECT_EQ(test, dest->vm_end, target + 2 * SZ_4K); + KUNIT_EXPECT_EQ(test, dest->vm_pgoff, 77UL + 4 / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(dest), (4 % TEST_SLOTS) * SZ_4K); +} + +struct anon_slot_fixture { + struct mm_struct *mm; + struct vm_area_struct *vma; + struct mm_subpage_pool *pool; + struct mm_subpage *slots[4]; + bool mapped[4]; +}; + +static void anon_slot_fixture_free(void *arg) +{ + struct anon_slot_fixture *f = arg; + struct mmu_gather tlb; + unsigned int i; + + if (f->pool) + mm_subpage_pool_close(f->pool); + if (f->mm) { + mmap_write_lock(f->mm); + for (i = 0; i < 4; i++) { + unsigned long addr = TEST_VA + i * SZ_4K; + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm, addr, &ptl); + + if (!pte) + continue; + ptep_get_and_clear_full(f->mm, addr, pte, 0); + if (f->mapped[i]) { + struct vm_area_struct *vma = vma_lookup(f->mm, addr); + + mm_subpage_remove_anon_rmap(f->slots[i], vma); + dec_mm_counter(f->mm, MM_ANONPAGES); + f->mapped[i] = false; + } + pte_unmap_unlock(pte, ptl); + } + tlb_gather_mmu_fullmm(&tlb, f->mm); + free_pgd_range(&tlb, 0, 1UL << 48, 0, 0); + tlb_finish_mmu(&tlb); + mmap_write_unlock(f->mm); + } + /* No slot reference is released before the PTEs and TLBs are retired. */ + for (i = 0; i < 4; i++) + if (f->slots[i]) + mm_subpage_put(f->slots[i]); + if (f->pool) + mm_subpage_pool_put(f->pool); + if (f->mm) + mmput(f->mm); + memset(f, 0, sizeof(*f)); +} + +static struct anon_slot_fixture *anon_slot_fixture_create_from(struct kunit *test, + struct anon_slot_fixture *parent) +{ + struct anon_slot_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + struct vm_area_struct *vma; + struct folio *folio; + int ret; + + if (!f || kunit_add_action_or_reset(test, anon_slot_fixture_free, f)) + return NULL; + f->mm = mm_alloc(); + if (!f->mm || test_mm_select4k(f->mm)) + return NULL; + f->pool = mm_subpage_pool_create(GFP_KERNEL); + if (!f->pool) + return NULL; + folio = folio_alloc(GFP_KERNEL, 0); + if (!folio) + return NULL; + ret = mm_subpage_pool_add_folio(f->pool, folio, GFP_KERNEL); + if (ret) { + folio_put(folio); + return NULL; + } + vma = vm_area_alloc(f->mm); + if (!vma) + return NULL; + vma_set_anonymous(vma); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + 2 * PAGE_SIZE; + vm_flags_init(vma, VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE); + vma_set_page_prot(vma); + if (parent) { + mmap_write_lock(parent->mm); + mmap_write_lock_nested(f->mm, SINGLE_DEPTH_NESTING); + vma_set_page_offset(vma, vma_page_offset_at(parent->vma, TEST_VA)); + ret = anon_vma_fork(vma, parent->vma); + } else { + mmap_write_lock(f->mm); + ret = 0; + } + if (!ret && parent) { + VMA_ITERATOR(vmi, f->mm, TEST_VA); + + vma_start_write(vma); + ret = vma_iter_store_gfp(&vmi, vma, GFP_KERNEL); + if (!ret) + f->mm->map_count++; + } else if (!ret) { + ret = insert_vm_struct(f->mm, vma); + } + if (!ret) { + f->vma = vma; + ret = anon_vma_prepare(vma); + } + if (!f->vma) + unlink_anon_vmas(vma); + mmap_write_unlock(f->mm); + if (parent) + mmap_write_unlock(parent->mm); + if (!f->vma) + vm_area_free(vma); + return ret ? NULL : f; +} + +static struct anon_slot_fixture *anon_slot_fixture_create(struct kunit *test) +{ + return anon_slot_fixture_create_from(test, NULL); +} + +static void user4k_anon_rmap_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + struct folio *folio = NULL; + unsigned int i; + vm_flags_t flags; + int ret, bad_offset = 0, bad_index = 0; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(f->pool, i * SZ_4K); + unsigned long addr = TEST_VA + i * SZ_4K; + struct test_mm tables = { .mm = f->mm }; + spinlock_t *ptl; + pte_t *pte; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + f->slots[i] = slot; + folio = mm_subpage_folio(slot); + /* Allocate the table first; clear the test leaf before rmap attachment. */ + KUNIT_ASSERT_EQ(test, test_map_alias(&tables, 0, addr, mm_subpage_phys(slot)), 0); + mmap_read_lock(f->mm); + folio_lock(folio); + pte = test_mm_lookup(f->mm, addr, &ptl); + if (!pte) { + folio_unlock(folio); + mmap_read_unlock(f->mm); + KUNIT_FAIL(test, "missing allocated PTE"); + return; + } + pte_clear(f->mm, addr, pte); + if (!i) + bad_offset = mm_subpage_add_anon_rmap(slot, f->vma, addr + 1); + ret = mm_subpage_add_anon_rmap(slot, f->vma, addr); + if (!ret) { + f->mapped[i] = true; + inc_mm_counter(f->mm, MM_ANONPAGES); + set_pte_at(f->mm, addr, pte, phys_pte_mm(f->mm, mm_subpage_phys(slot), + __pgprot(TEST_PROT | PTE_RDONLY))); + if (!i) + bad_index = mm_subpage_add_anon_rmap(slot, f->vma, addr + PAGE_SIZE); + } + pte_unmap_unlock(pte, ptl); + folio_unlock(folio); + mmap_read_unlock(f->mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)i + 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)i + 1); + } + KUNIT_EXPECT_EQ(test, bad_offset, -EINVAL); + KUNIT_EXPECT_EQ(test, bad_index, -EINVAL); + KUNIT_EXPECT_EQ(test, folio->index, TEST_VA >> PAGE_SHIFT); + KUNIT_EXPECT_FALSE(test, PageAnonExclusive(&folio->page)); + KUNIT_EXPECT_EQ(test, folio_expected_ref_count(folio), folio_ref_count(folio)); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 1); + for (i = 0; i < 4; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm, TEST_VA + i * SZ_4K, &ptl); + bool young; + + KUNIT_ASSERT_NOT_NULL(test, pte); + young = pte_young(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_FALSE(test, young); + } + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 0); +} + +/* The fixture takes one existing slot reference, including on failure. */ +static int anon_slot_fixture_map(struct anon_slot_fixture *f, + unsigned int index, struct mm_subpage *slot) +{ + unsigned long addr = TEST_VA + index * SZ_4K; + struct test_mm tables = { .mm = f->mm }; + struct folio *folio = mm_subpage_folio(slot); + spinlock_t *ptl; + pte_t *pte; + int ret; + + f->slots[index] = slot; + ret = test_map_alias(&tables, 0, addr, mm_subpage_phys(slot)); + if (ret) + return ret; + mmap_read_lock(f->mm); + folio_lock(folio); + pte = test_mm_lookup(f->mm, addr, &ptl); + if (!pte) { + ret = -ENOMEM; + goto unlock; + } + pte_clear(f->mm, addr, pte); + ret = mm_subpage_add_anon_rmap(slot, f->vma, addr); + if (!ret) { + f->mapped[index] = true; + inc_mm_counter(f->mm, MM_ANONPAGES); + set_pte_at(f->mm, addr, pte, phys_pte_mm(f->mm, mm_subpage_phys(slot), + __pgprot(TEST_PROT | PTE_RDONLY))); + } + pte_unmap_unlock(pte, ptl); +unlock: + folio_unlock(folio); + mmap_read_unlock(f->mm); + return ret; +} + +static void user4k_anon_fork_rmap_test(struct kunit *test) +{ + struct anon_slot_fixture *parent = anon_slot_fixture_create(test); + struct anon_slot_fixture *child; + struct mm_subpage *slot; + struct folio *folio; + vm_flags_t flags; + int ret; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, parent); + child = anon_slot_fixture_create_from(test, parent); + KUNIT_ASSERT_NOT_NULL(test, child); + for (i = 0; i < 4; i++) { + slot = mm_subpage_alloc_at(parent->pool, i * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(parent, i, slot), 0); + mm_subpage_get(slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(child, i, slot), 0); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 2); + } + folio = mm_subpage_folio(slot); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 8); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 8); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 2); + anon_slot_fixture_free(parent); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 4); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 4); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(child->slots[i]), 1); + /* The child mappings and root remain valid after the parent disappears. */ + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 0); +} + +static void user4k_nonlinear_rmap_reuse_test(struct kunit *test) +{ + struct anon_slot_fixture *a = anon_slot_fixture_create(test); + struct anon_slot_fixture *b = anon_slot_fixture_create(test); + struct anon_slot_fixture *c = anon_slot_fixture_create(test); + struct mm_subpage *slot; + struct folio *folio; + struct anon_vma_chain *avc; + phys_addr_t phys; + vm_flags_t flags; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, c); + slot = mm_subpage_alloc_at(a->pool, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(a, 0, slot), 0); + folio = mm_subpage_folio(slot); + slot = mm_subpage_alloc_at(a->pool, 3 * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + phys = mm_subpage_phys(slot); +#ifdef CONFIG_FAILSLAB + { + unsigned int pending; + + mmap_read_lock(b->mm); + folio_lock(folio); + WRITE_ONCE(current->fail_nth, 1); + ret = mm_subpage_add_anon_rmap(slot, b->vma, TEST_VA + SZ_4K); + pending = READ_ONCE(current->fail_nth); + WRITE_ONCE(current->fail_nth, 0); + /* Keep cleanup valid even if allocation unexpectedly succeeds. */ + if (!ret) + mm_subpage_remove_anon_rmap(slot, b->vma); + folio_unlock(folio); + mmap_read_unlock(b->mm); + KUNIT_EXPECT_EQ(test, pending, 0U); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 0); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 1); + } +#endif + /* Allocation failure left the slot available for the same operation. */ + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(b, 1, slot), 0); + anon_slot_fixture_free(b); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 1); + slot = mm_subpage_alloc_at(a->pool, 3 * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_phys(slot), phys); + + /* Give this empty VMA a different logical native index as well. */ + mmap_write_lock(c->mm); + anon_vma_lock_write(c->vma->anon_vma); + list_for_each_entry(avc, &c->vma->anon_vma_chain, same_vma) + anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root); + c->vma->vm_pgoff += 9; + list_for_each_entry(avc, &c->vma->anon_vma_chain, same_vma) + anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root); + anon_vma_unlock_write(c->vma->anon_vma); + mmap_write_unlock(c->mm); +#ifdef CONFIG_FAILSLAB + /* Retiring a previous root also needs failure-atomic bookkeeping. */ + mmap_read_lock(c->mm); + folio_lock(folio); + WRITE_ONCE(current->fail_nth, 1); + ret = mm_subpage_add_anon_rmap(slot, c->vma, TEST_VA + 2 * SZ_4K); + WRITE_ONCE(current->fail_nth, 0); + if (!ret) + mm_subpage_remove_anon_rmap(slot, c->vma); + folio_unlock(folio); + mmap_read_unlock(c->mm); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 0); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 1); +#endif + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(c, 2, slot), 0); + mmap_read_lock(c->mm); + KUNIT_EXPECT_EQ(test, page_address_in_vma(folio, &folio->page, c->vma), + TEST_VA + 2 * SZ_4K); + mmap_read_unlock(c->mm); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 2); + /* The original root may disappear before the other slot's last mapping. */ + anon_slot_fixture_free(a); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(c->slots[2]), 1); +} + +static void user4k_dup_mmap_test(struct kunit *test) +{ + struct anon_slot_fixture *parent = anon_slot_fixture_create(test); + struct mm_struct *child; + struct folio *folio; + unsigned int i; + int ret = 0; + + KUNIT_ASSERT_NOT_NULL(test, parent); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(parent->pool, i * SZ_4K); + unsigned long addr = TEST_VA + i * SZ_4K; + spinlock_t *ptl; + pte_t *pte; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(parent, i, slot), 0); + pte = test_mm_lookup(parent->mm, addr, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte_at(parent->mm, addr, pte, + pte_mkwrite_novma(pte_mkdirty(ptep_get(pte)))); + pte_unmap_unlock(pte, ptl); + } + /* Split into four 4K VMAs plus a trailing hole, preserving COW flags. */ + mmap_write_lock(parent->mm); + for (i = 1; i < 4; i += 2) { + unsigned long addr = TEST_VA + i * SZ_4K; + VMA_ITERATOR(vmi, parent->mm, addr); + struct vm_area_struct *vma = vma_iter_load(&vmi); + vma_flags_t flags = vma->flags; + + vma_flags_set(&flags, VMA_DONTDUMP_BIT); + vma = vma_modify_flags(&vmi, vma, vma, addr, addr + SZ_4K, &flags); + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); + break; + } + } + mmap_write_unlock(parent->mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, parent->mm->map_count, 5); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, child->map_count, 5); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 4UL); + folio = mm_subpage_folio(parent->slots[0]); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 8); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 8); + for (i = 0; i < 4; i++) { + unsigned long addr = TEST_VA + i * SZ_4K; + struct vm_area_struct *vma = test_vma_lookup(child, addr); + unsigned int side; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), i * (unsigned int)SZ_4K); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(parent->slots[i]), 2); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : parent->mm; + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, addr, &ptl); + pte_t entry; + + KUNIT_ASSERT_NOT_NULL(test, pte); + entry = ptep_get(pte); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_FALSE(test, pte_write(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, entry), + mm_subpage_phys(parent->slots[i])); + } + } + anon_slot_fixture_free(parent); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 4); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 4); + /* KUnit's child-mm cleanup now exercises ordinary exit_mmap(). */ +} + +static void test_subpage_put(void *arg) +{ + mm_subpage_put(arg); +} + +static void user4k_anon_root_lifetime_test(struct kunit *test) +{ + struct anon_slot_fixture *old = anon_slot_fixture_create(test); + struct anon_slot_fixture *other = anon_slot_fixture_create(test); + struct mm_subpage *slot; + struct anon_vma *root; + struct folio *folio; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, old); + KUNIT_ASSERT_NOT_NULL(test, other); + slot = mm_subpage_alloc_at(old->pool, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(old, 0, slot), 0); + folio = mm_subpage_folio(slot); + root = old->vma->anon_vma->root; + mm_subpage_get(slot); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, slot), 0); + anon_slot_fixture_free(old); + /* Only the owner now pins the root; no VMA or PTE survives. */ + KUNIT_EXPECT_EQ(test, atomic_read(&root->refcount), 1); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 0); + mm_subpage_get(slot); + ret = anon_slot_fixture_map(other, 0, slot); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 0); + KUNIT_EXPECT_EQ(test, atomic_read(&root->refcount), 1); +} + +static void test_folio_put(void *arg) +{ + folio_put(arg); +} + +static void user4k_data_tlb_batch_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + struct folio *native = folio_alloc(GFP_KERNEL, 2); + struct mmu_gather tlb; + struct folio *backing; + unsigned int i; + bool full = false; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, native), 0); + for (i = 0; i < 4; i++) { + f->slots[i] = mm_subpage_alloc_at(f->pool, i * SZ_4K); + if (IS_ERR(f->slots[i])) { + f->slots[i] = NULL; + KUNIT_FAIL(test, "slot allocation failed"); + return; + } + } + backing = mm_subpage_folio(f->slots[0]); + memset(folio_address(backing), 0x5a, SZ_4K); + tlb_gather_mmu(&tlb, f->mm); + tlb_change_page_size(&tlb, MM_SUBPAGE_SIZE); + tlb_flush_pte_range(&tlb, TEST_VA, PAGE_SIZE); + for (i = 0; i < 64; i++) { + mm_subpage_get(f->slots[i % 4]); + if (__tlb_remove_subpage(&tlb, f->slots[i % 4])) { + full = true; + tlb_flush_mmu(&tlb); + tlb_flush_pte_range(&tlb, TEST_VA, PAGE_SIZE); + } + if (i == 12) { + /* Exercise the native count payload (3 << 2 has tag bit 2). */ + folio_ref_add(native, 3); + if (__tlb_remove_folio_pages(&tlb, &native->page, 3, false)) { + full = true; + tlb_flush_mmu(&tlb); + tlb_flush_pte_range(&tlb, TEST_VA, PAGE_SIZE); + } + } + } + mm_subpage_put(f->slots[0]); + f->slots[0] = NULL; + /* Without memory pressure these fit in the local plus one dynamic batch. */ + if (!full) { + struct mm_subpage *extra = mm_subpage_alloc_at(f->pool, 0); + + KUNIT_EXPECT_EQ(test, PTR_ERR(extra), -EAGAIN); + if (!IS_ERR(extra)) + mm_subpage_put(extra); + KUNIT_EXPECT_EQ(test, folio_ref_count(backing), 67); + KUNIT_EXPECT_EQ(test, folio_ref_count(native), 4); + KUNIT_EXPECT_GT(test, tlb.batch_count, 0U); + } + tlb_finish_mmu(&tlb); + KUNIT_EXPECT_EQ(test, folio_ref_count(backing), 3); + KUNIT_EXPECT_EQ(test, folio_ref_count(native), 1); + f->slots[0] = mm_subpage_alloc_at(f->pool, 0); + if (IS_ERR(f->slots[0])) { + f->slots[0] = NULL; + KUNIT_FAIL(test, "slot was not reusable after TLB completion"); + return; + } + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(f->slots[0]), backing); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(backing), 0, SZ_4K), NULL); + for (i = 0; i < 4; i++) { + struct mm_subpage *extra = mm_subpage_alloc_at(f->pool, i * SZ_4K); + + KUNIT_EXPECT_EQ(test, PTR_ERR(extra), -EAGAIN); + if (!IS_ERR(extra)) + mm_subpage_put(extra); + } +} + +static void user4k_anon_zap_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + struct mm_subpage *slots[4]; + struct folio *folio; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + slots[i] = mm_subpage_alloc_at(f->pool, i * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slots[i]); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(f, i, slots[i]), 0); + mm_subpage_get(slots[i]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, slots[i]), 0); + } + folio = mm_subpage_folio(slots[0]); + mmap_read_lock(f->mm); + zap_vma_range(f->vma, TEST_VA + SZ_4K, SZ_4K); + f->mapped[1] = false; + f->slots[1] = NULL; + mmap_read_unlock(f->mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 3); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 7); + for (i = 0; i < 4; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm, TEST_VA + i * SZ_4K, &ptl); + bool present; + + KUNIT_ASSERT_NOT_NULL(test, pte); + present = pte_present(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, present, i != 1); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slots[i]), i == 1 ? 0 : 1); + } + /* Include both the existing hole and a trailing run of empty entries. */ + mmap_read_lock(f->mm); + zap_vma(f->vma); + for (i = 0; i < 4; i++) { + f->mapped[i] = false; + f->slots[i] = NULL; + } + mmap_read_unlock(f->mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 4); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slots[i]), 0); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(f->mm, MM_ANONPAGES), 0UL); +} + +static void user4k_anon_writable_slots_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(f->pool, i * SZ_4K); + unsigned long addr = TEST_VA + i * SZ_4K; + spinlock_t *ptl; + pte_t *pte; + bool writable; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(f, i, slot), 0); + pte = test_mm_lookup(f->mm, addr, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte_at(f->mm, addr, pte, + pte_mkwrite_novma(pte_mkdirty(ptep_get(pte)))); + writable = pte_write(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_TRUE(test, writable); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + } + KUNIT_EXPECT_EQ(test, folio_mapcount(mm_subpage_folio(f->slots[0])), 4); +} + +static struct vm_area_struct *test_fault_vma_in_mm(struct mm_struct *mm, + unsigned long start, unsigned long len) +{ + struct vm_area_struct *vma; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE; + int ret; + + if (!mm) + return NULL; + vma = vm_area_alloc(mm); + if (!vma) + return NULL; + vma_set_anonymous(vma); + vma->vm_start = start; + vma->vm_end = start + len; + vm_flags_init(vma, flags); + vma->vm_page_prot = vm_get_page_prot(flags); + mmap_write_lock(mm); + ret = insert_vm_struct(mm, vma); + mmap_write_unlock(mm); + if (ret) { + vm_area_free(vma); + return NULL; + } + return vma; +} + +struct coarse_anon_fixture { + struct mm_struct *mm[2]; + bool attached[2]; + struct vm_area_struct *vma[2]; + struct folio *folio; + unsigned long mapped[2]; + unsigned int leaves; + bool pinned; +}; + +static void coarse_anon_fixture_free(void *ptr) +{ + struct coarse_anon_fixture *f = ptr; + unsigned int i, nr = SZ_64K / PAGE_SIZE; + + if (f->pinned) { + unpin_user_page(folio_page(f->folio, nr - 1)); + f->pinned = false; + } + for (i = 0; i < 2; i++) { + unsigned int leaf; + + if (!f->mm[i]) + continue; + mmap_write_lock(f->mm[i]); + for (leaf = 0; leaf < (f->leaves ?: 1); leaf++) { + unsigned long addr = TEST_VA + leaf * SZ_64K; + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm[i], addr, &ptl); + + if (pte) { + ptep_get_and_clear_full(f->mm[i], addr, pte, 0); + if (f->mapped[i] & BIT(leaf)) { + folio_remove_rmap_pte(f->folio, + folio_page(f->folio, leaf * nr), f->vma[i]); + dec_mm_counter(f->mm[i], MM_ANONPAGES); + } + pte_unmap_unlock(pte, ptl); + } + if (f->mapped[i] & BIT(leaf)) { + folio_put_refs(f->folio, nr); + f->mapped[i] &= ~BIT(leaf); + } + } + mmap_write_unlock(f->mm[i]); + if (f->vma[i] && !f->attached[i]) { + unlink_anon_vmas(f->vma[i]); + vm_area_free(f->vma[i]); + } + mmput(f->mm[i]); + f->vma[i] = NULL; + f->mm[i] = NULL; + } + if (f->folio) { + folio_put(f->folio); + f->folio = NULL; + } +} + +static void expect_coarse_anon_state(struct kunit *test, struct folio *folio, + unsigned int maps, bool exclusive) +{ + unsigned int i, nr = folio_nr_pages(folio); + + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)(maps * nr)); + for (i = 0; i < nr; i++) { + struct page *page = folio_page(folio, i); + + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, (atomic_read(&page->_mapcount) + 1), (int)maps); + KUNIT_EXPECT_EQ(test, PageAnonExclusive(page), exclusive); + } +} + +static void user64k_anon_rmap_span_test(struct kunit *test) +{ + struct coarse_anon_fixture *f; + unsigned int i, nr = SZ_64K / PAGE_SIZE; + pte_t entry, *pte; + spinlock_t *ptl; + int ret, dup_ret, share_ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires coarse anonymous leaf"); + return; + } + f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, coarse_anon_fixture_free, f), 0); + f->folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 16 - PAGE_SHIFT); + KUNIT_ASSERT_NOT_NULL(test, f->folio); + for (i = 0; i < 2; i++) { + struct test_mm tables = {}; + + f->mm[i] = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, f->mm[i]); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(f->mm[i], 16), 0); + if (!i) { + f->vma[i] = test_fault_vma_in_mm(f->mm[i], TEST_VA, SZ_64K); + f->attached[i] = !!f->vma[i]; + } else { + f->vma[i] = vm_area_alloc(f->mm[i]); + if (f->vma[i]) { + vma_set_anonymous(f->vma[i]); + f->vma[i]->vm_start = TEST_VA; + f->vma[i]->vm_end = TEST_VA + SZ_64K; + vm_flags_init(f->vma[i], f->vma[0]->vm_flags); + f->vma[i]->vm_page_prot = f->vma[0]->vm_page_prot; + vma_set_page_offset(f->vma[i], vma_page_offset_at(f->vma[0], TEST_VA)); + } + } + KUNIT_ASSERT_NOT_NULL(test, f->vma[i]); + tables.mm = f->mm[i]; + KUNIT_ASSERT_EQ(test, test_map_alias(&tables, 0, TEST_VA, PFN_PHYS(folio_pfn(f->folio))), 0); + pte = test_mm_lookup(f->mm[i], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + pte_clear(f->mm[i], TEST_VA, pte); + pte_unmap_unlock(pte, ptl); + } + mmap_write_lock(f->mm[0]); + ret = anon_vma_prepare(f->vma[0]); + mmap_write_unlock(f->mm[0]); + KUNIT_ASSERT_EQ(test, ret, 0); + mmap_write_lock(f->mm[0]); + mmap_write_lock_nested(f->mm[1], SINGLE_DEPTH_NESTING); + ret = anon_vma_fork(f->vma[1], f->vma[0]); + if (!ret) { + VMA_ITERATOR(vmi, f->mm[1], TEST_VA); + + vma_start_write(f->vma[1]); + ret = vma_iter_store_gfp(&vmi, f->vma[1], GFP_KERNEL); + if (!ret) { + f->mm[1]->map_count++; + f->attached[1] = true; + } + } + mmap_write_unlock(f->mm[1]); + mmap_write_unlock(f->mm[0]); + KUNIT_ASSERT_EQ(test, ret, 0); + + pte = test_mm_lookup(f->mm[0], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + folio_add_new_anon_rmap(f->folio, f->vma[0], TEST_VA, RMAP_EXCLUSIVE); + folio_ref_add(f->folio, nr); /* Fixture retains its original allocation ref. */ + inc_mm_counter(f->mm[0], MM_ANONPAGES); + f->mapped[0] = true; + entry = pte_mkwrite(pte_mkdirty(phys_pte_mm(f->mm[0], PFN_PHYS(folio_pfn(f->folio)), + f->vma[0]->vm_page_prot)), f->vma[0]); + set_ptes(f->mm[0], TEST_VA, pte, entry, 1); + pte_unmap_unlock(pte, ptl); + expect_coarse_anon_state(test, f->folio, 1, true); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)nr + 1); + + mm_flags_set(MMF_HAS_PINNED, f->mm[0]); + ret = try_grab_folio(page_folio(folio_page(f->folio, nr - 1)), 1, FOLL_PIN); + f->pinned = !ret; + KUNIT_ASSERT_EQ(test, ret, 0); + pte = test_mm_lookup(f->mm[0], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + ptep_set_wrprotect(f->mm[0], TEST_VA, pte); + raw_write_seqcount_begin(&f->mm[0]->write_protect_seq); + dup_ret = folio_try_dup_anon_rmap_pte(f->folio, &f->folio->page, f->vma[1], f->vma[0]); + raw_write_seqcount_end(&f->mm[0]->write_protect_seq); + entry = ptep_get_and_clear(f->mm[0], TEST_VA, pte); + share_ret = folio_try_share_anon_rmap_pte(f->folio, &f->folio->page, f->vma[0]); + set_ptes(f->mm[0], TEST_VA, pte, entry, 1); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, dup_ret, -EBUSY); + KUNIT_EXPECT_EQ(test, share_ret, -EBUSY); + expect_coarse_anon_state(test, f->folio, 1, true); + unpin_user_page(folio_page(f->folio, nr - 1)); + f->pinned = false; + + pte = test_mm_lookup(f->mm[0], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + entry = ptep_get_and_clear(f->mm[0], TEST_VA, pte); + share_ret = folio_try_share_anon_rmap_pte(f->folio, &f->folio->page, f->vma[0]); + set_ptes(f->mm[0], TEST_VA, pte, pte_wrprotect(entry), 1); + pte_unmap_unlock(pte, ptl); + KUNIT_ASSERT_EQ(test, share_ret, 0); + expect_coarse_anon_state(test, f->folio, 1, false); + + pte = test_mm_lookup(f->mm[0], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + raw_write_seqcount_begin(&f->mm[0]->write_protect_seq); + dup_ret = folio_try_dup_anon_rmap_pte(f->folio, &f->folio->page, f->vma[1], f->vma[0]); + raw_write_seqcount_end(&f->mm[0]->write_protect_seq); + if (!dup_ret) { + folio_ref_add(f->folio, nr); + inc_mm_counter(f->mm[1], MM_ANONPAGES); + f->mapped[1] = true; + } + pte_unmap_unlock(pte, ptl); + KUNIT_ASSERT_EQ(test, dup_ret, 0); + pte = test_mm_lookup(f->mm[1], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_ptes(f->mm[1], TEST_VA, pte, pte_wrprotect(entry), 1); + pte_unmap_unlock(pte, ptl); + expect_coarse_anon_state(test, f->folio, 2, false); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)(2 * nr + 1)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(f->mm[0], MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(f->mm[1], MM_ANONPAGES), 1L); + + /* Exercise ordinary unmapping of each shared coarse leaf. */ + for (i = 0; i < 2; i++) { + mmap_read_lock(f->mm[i]); + zap_vma_range(f->vma[i], TEST_VA, SZ_64K); + f->mapped[i] = 0; + mmap_read_unlock(f->mm[i]); + expect_coarse_anon_state(test, f->folio, 1 - i, false); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)((1 - i) * nr + 1)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(f->mm[i], MM_ANONPAGES), 0L); + } + + /* Keep an observation ref while cleanup releases the empty address spaces. */ + { + struct folio *folio = f->folio; + + folio_get(folio); + coarse_anon_fixture_free(f); + expect_coarse_anon_state(test, folio, 0, false); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + folio_put(folio); + } +} + +static void user64k_anon_zap_batch_test(struct kunit *test) +{ + struct coarse_anon_fixture *f; + struct test_mm tables = {}; + struct vm_area_struct *vma; + struct mm_struct *mm; + unsigned int i, nr = SZ_64K / PAGE_SIZE, batch; + pte_t entry, *pte; + spinlock_t *ptl; + int ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires coarse anonymous leaves"); + return; + } + f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, coarse_anon_fixture_free, f), 0); + f->leaves = 2; + f->folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 17 - PAGE_SHIFT); + KUNIT_ASSERT_NOT_NULL(test, f->folio); + mm = f->mm[0] = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + /* Partial unmap can enqueue this folio on the memcg split queue. */ + KUNIT_ASSERT_EQ(test, mem_cgroup_charge(f->folio, mm, GFP_KERNEL), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, 16), 0); + vma = f->vma[0] = test_fault_vma_in_mm(mm, TEST_VA, 2 * SZ_64K); + f->attached[0] = !!vma; + KUNIT_ASSERT_NOT_NULL(test, vma); + tables.mm = mm; + for (i = 0; i < 2; i++) { + unsigned long addr = TEST_VA + i * SZ_64K; + + KUNIT_ASSERT_EQ(test, test_map_alias(&tables, i, addr, + PFN_PHYS(folio_pfn(f->folio)) + i * SZ_64K), 0); + pte = test_mm_lookup(mm, addr, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + pte_clear(mm, addr, pte); + pte_unmap_unlock(pte, ptl); + } + mmap_write_lock(mm); + ret = anon_vma_prepare(vma); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + folio_add_new_anon_rmap(f->folio, vma, TEST_VA, RMAP_EXCLUSIVE); + folio_ref_add(f->folio, 2 * nr); + add_mm_counter(mm, MM_ANONPAGES, 2); + f->mapped[0] = BIT(0) | BIT(1); + entry = pte_mkwrite(pte_mkdirty(phys_pte_mm(mm, + PFN_PHYS(folio_pfn(f->folio)), vma->vm_page_prot)), vma); + set_ptes(mm, TEST_VA, pte, entry, 2); + batch = folio_subpage_pte_batch(vma, f->folio, pte, entry, 2, 0); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, batch, 2U); + KUNIT_EXPECT_EQ(test, folio_mapcount(f->folio), (int)(2 * nr)); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)(2 * nr + 1)); + + /* The reference walker must consume logical PTEs and native mapcounts. */ + { + vm_flags_t flags; + bool young; + + folio_lock(f->folio); + ret = folio_referenced(f->folio, 1, NULL, &flags); + folio_unlock(f->folio); + KUNIT_EXPECT_EQ(test, ret, 1); + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + young = pte_young(ptep_get(pte)) || pte_young(ptep_get(pte + 1)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_FALSE(test, young); + folio_lock(f->folio); + ret = folio_referenced(f->folio, 1, NULL, &flags); + folio_unlock(f->folio); + KUNIT_EXPECT_EQ(test, ret, 0); + } + + /* Removing one logical leaf must leave the adjacent leaf's native span. */ + mmap_read_lock(mm); + zap_vma_range(vma, TEST_VA, SZ_64K); + f->mapped[0] &= ~BIT(0); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(f->folio), (int)nr); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)nr + 1); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1L); + for (i = 0; i < 2 * nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(f->folio, i)->_mapcount), + i < nr ? -1 : 0); + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + ret = pte_none(ptep_get(pte)) && pte_present(ptep_get(pte + 1)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_TRUE(test, ret); + + /* Restore the first leaf, then remove both through one batched walk. */ + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + folio_add_anon_rmap_pte(f->folio, &f->folio->page, vma, + TEST_VA, RMAP_EXCLUSIVE); + folio_ref_add(f->folio, nr); + inc_mm_counter(mm, MM_ANONPAGES); + f->mapped[0] |= BIT(0); + set_ptes(mm, TEST_VA, pte, entry, 1); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, folio_mapcount(f->folio), (int)(2 * nr)); + mmap_read_lock(mm); + zap_vma_range(vma, TEST_VA, 2 * SZ_64K); + f->mapped[0] = 0; + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(f->folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), 1); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); + for (i = 0; i < 2 * nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(f->folio, i)->_mapcount), -1); +} + +static void user64k_file_rmap_span_test(struct kunit *test) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + struct folio *folio; + unsigned int i, nr = SZ_64K / PAGE_SIZE; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires coarse nonanonymous leaf"); + return; + } + mm = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, mm); + vma = vm_area_alloc(mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + 2 * SZ_64K; + folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 16 - PAGE_SHIFT); + KUNIT_ASSERT_NOT_NULL(test, folio); + /* Private metadata-only fixture: no file-cache promotion or fault is implied. */ + folio_lock(folio); + folio_add_file_rmap_pte(folio, &folio->page, vma); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + for (i = 0; i < nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(folio, i)->_mapcount) + 1, 1); + folio_dup_file_rmap_pte(folio, &folio->page, vma); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)(2 * nr)); + for (i = 0; i < nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(folio, i)->_mapcount) + 1, 2); + folio_remove_rmap_pte(folio, &folio->page, vma); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + folio_remove_rmap_pte(folio, &folio->page, vma); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + for (i = 0; i < nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(folio, i)->_mapcount), -1); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + folio_put(folio); +} + +static struct vm_area_struct *test_fault_vma(struct kunit *test, + unsigned long start, unsigned long len) +{ + return test_fault_vma_in_mm(test_vma_mm(test), start, len); +} + +static struct vm_area_struct *test_fault_vma_granule(struct kunit *test, + unsigned int shift, unsigned long start, unsigned long len) +{ + return test_fault_vma_in_mm(test_vma_mm_granule(test, shift), start, len); +} + +static void test_anon_write_fault_granule(struct kunit *test, unsigned int shift) +{ + unsigned long size = 1UL << shift; + unsigned int shape; + + /* Swap/reclaim integration is a later gate, before any ABI activation. */ + KUNIT_ASSERT_EQ(test, total_swap_pages, 0UL); + for (shape = 0; shape < 2; shape++) { + unsigned long start = TEST_VA + (shape ? size : 0); + unsigned long len = shape ? size : 2 * PAGE_SIZE; + struct vm_area_struct *vma = test_fault_vma_granule(test, shift, start, len); + struct folio *backing = NULL; + unsigned int i, mapped = 0; + vm_fault_t ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + ret = handle_mm_fault(vma, TEST_VA + size + 3, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + /* Remove transient LRU-add references before checking ownership. */ + lru_add_drain_all(); + for (i = 0; i < (PAGE_SIZE >> shift); i++) { + unsigned long addr = TEST_VA + i * size; + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(vma->vm_mm, addr, &ptl); + struct mm_subpage *slot = NULL; + bool expected = !shape || i == 1; + bool present, writable = false, zero = false, identity = false; + + KUNIT_ASSERT_NOT_NULL(test, pte); + present = pte_present(ptep_get(pte)); + if (present) { + slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, ptep_get(pte))); + writable = pte_write(ptep_get(pte)); + if (slot) { + struct folio *folio = mm_subpage_folio(slot); + + if (!backing) + backing = folio; + identity = folio == backing && + mm_subpage_offset(slot) == i * size; + zero = !memchr_inv(folio_address(folio) + mm_subpage_offset(slot), + 0, size); + mm_subpage_put(slot); + } + mapped++; + } + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, present, expected); + if (expected) { + KUNIT_EXPECT_TRUE(test, writable); + KUNIT_EXPECT_TRUE(test, identity); + KUNIT_EXPECT_TRUE(test, zero); + } + } + KUNIT_ASSERT_NOT_NULL(test, backing); + KUNIT_EXPECT_EQ(test, mapped, shape ? 1U : (PAGE_SIZE >> shift)); + KUNIT_EXPECT_EQ(test, folio_mapcount(backing), (int)mapped); + KUNIT_EXPECT_EQ(test, folio_ref_count(backing), (int)mapped); + KUNIT_EXPECT_TRUE(test, folio_test_lru(backing)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), + (unsigned long)mapped); + } +} + +static void user4k_anon_write_fault_test(struct kunit *test) +{ + test_anon_write_fault_granule(test, 12); +} + +static void user16k_anon_write_fault_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative faults require larger native backing"); + return; + } + test_anon_write_fault_granule(test, 14); +} + +static void user4k_swap_encoding_test(struct kunit *test) +{ + const unsigned long offsets[] = { 1, 2, (1UL << __SWP_OFFSET_BITS) - 2 }; + const unsigned int types[] = { 0, 1, __SWP_TYPE_MASK }; + unsigned int i, j, slot, flags; + + for (i = 0; i < ARRAY_SIZE(offsets); i++) { + for (j = 0; j < ARRAY_SIZE(types); j++) { + for (slot = 0; slot < PAGE_SIZE / SZ_4K; slot++) { + for (flags = 0; flags < 4; flags++) { + softleaf_t entry = swp_entry(types[j], offsets[i]); + pte_t pte = softleaf_to_pte(entry), moved; + + pte = pte_swp_set_subpage_offset(pte, slot * SZ_4K); + if (flags & 1) + pte = pte_swp_mkexclusive(pte); + if (flags & 2) + pte = pte_swp_mkuffd_wp(pte); + KUNIT_EXPECT_FALSE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(pte).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(pte), slot * (unsigned int)SZ_4K); + moved = pte_move_swp_offset(pte, 1); + KUNIT_EXPECT_EQ(test, swp_offset(softleaf_from_pte(moved)), offsets[i] + 1); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(moved), slot * (unsigned int)SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_same(pte_move_swp_offset(moved, -1), pte)); + moved = pte_swp_set_subpage_offset(pte, ((slot + 1) % (PAGE_SIZE / SZ_4K)) * SZ_4K); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(moved).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(moved), ((slot + 1) % (PAGE_SIZE / SZ_4K)) * (unsigned int)SZ_4K); + } + } + } + } +} + +static pte_t test_fault_entry(struct kunit *test, struct mm_struct *mm, unsigned long addr) +{ + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, addr, &ptl); + pte_t entry = __pte(0); + + KUNIT_EXPECT_NOT_NULL(test, pte); + if (pte) { + entry = ptep_get(pte); + pte_unmap_unlock(pte, ptl); + } + return entry; +} + + +static void test_file_put(void *file) +{ + fput(file); +} + +static struct file *test_cache_file(struct kunit *test, unsigned long size, bool fill) +{ + struct file *file = shmem_file_setup("user4k-file-test", size, EMPTY_VMA_FLAGS); + unsigned char *buf; + loff_t pos = 0; + unsigned int i; + + if (IS_ERR(file)) + return file; + if (kunit_add_action_or_reset(test, test_file_put, file)) + return ERR_PTR(-ENOMEM); + if (!fill) + return file; + buf = kunit_kmalloc(test, size, GFP_KERNEL); + if (!buf) + return ERR_PTR(-ENOMEM); + for (i = 0; i < size; i++) + buf[i] = 0x31 + (i / SZ_4K); + if (kernel_write(file, buf, size, &pos) != size) + return ERR_PTR(-EIO); + return file; +} + +static struct vm_area_struct *test_file_vma_granule(struct kunit *test, struct file *file, + unsigned int shift, bool shared, unsigned long start, unsigned long len, + unsigned long offset) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE; + int ret; + + mm = test_vma_mm_granule(test, shift); + if (!mm) + return NULL; + vma = vm_area_alloc(mm); + if (!vma) + return NULL; + vma->vm_start = start; + vma->vm_end = start + len; + vma_set_page_offset(vma, (struct vm_page_offset) { + .index = offset >> PAGE_SHIFT, .offset = offset & ~PAGE_MASK, + }); + if (shared) + flags |= VM_SHARED | VM_MAYSHARE; + vm_flags_init(vma, flags); + vma->vm_file = get_file(file); + vma->vm_page_prot = vm_get_page_prot(flags); + mmap_write_lock(mm); + ret = vfs_mmap(file, vma); + if (!ret) { + vma_set_page_prot(vma); + ret = insert_vm_struct(mm, vma); + } + mmap_write_unlock(mm); + if (ret) { + fput(vma->vm_file); + vm_area_free(vma); + return NULL; + } + return vma; +} + +static struct vm_area_struct *test_file_vma(struct kunit *test, struct file *file, + bool native, bool shared, unsigned long start, unsigned long len, + unsigned long offset) +{ + return test_file_vma_granule(test, file, native ? PAGE_SHIFT : 12, + shared, start, len, offset); +} + +static void user4k_mmap_descriptor_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + struct vm_area_desc desc; + + KUNIT_ASSERT_NOT_NULL(test, mm); + vma = vm_area_alloc(mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + PAGE_SIZE; + vma_set_page_offset(vma, (struct vm_page_offset) { 7, SZ_4K }); + compat_set_desc_from_vma(&desc, NULL, vma); + KUNIT_EXPECT_EQ(test, desc.pgoff, 7UL); + KUNIT_EXPECT_EQ(test, desc.subpage_offset, (unsigned int)SZ_4K); + desc.pgoff = 19; + desc.subpage_offset = 3 * SZ_4K; + KUNIT_ASSERT_EQ(test, __compat_vma_mmap(&desc, vma), 0); + KUNIT_EXPECT_TRUE(test, vm_page_offset_equal(vma_page_offset_at(vma, TEST_VA), + (struct vm_page_offset) { 19, 3 * SZ_4K })); +} + +static void user4k_mmap_region_offsets_test(struct kunit *test) +{ + struct file *file; + struct mm_struct *mm; + struct vm_area_struct *vma; + unsigned long start = TEST_VA + SZ_4K; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | + VM_SHARED | VM_MAYSHARE | VM_NORESERVE; + unsigned long result[4]; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mapping fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 8 * PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + /* Only the full native-sized lengths are used before accounting conversion. */ + kthread_use_mm(mm); + mmap_write_lock(mm); + result[0] = mmap_region(file, start + PAGE_SIZE, PAGE_SIZE, flags, + (struct vm_page_offset) { 1, SZ_4K }, NULL); + result[1] = mmap_region(file, start + 2 * PAGE_SIZE, PAGE_SIZE, flags, + (struct vm_page_offset) { 2, SZ_4K }, NULL); + result[2] = mmap_region(file, start, PAGE_SIZE, flags, + (struct vm_page_offset) { 0, SZ_4K }, NULL); + /* Adjacent virtual bytes with a different file offset must not merge. */ + result[3] = mmap_region(file, start + 3 * PAGE_SIZE, PAGE_SIZE, flags, + (struct vm_page_offset) { 3, 2 * SZ_4K }, NULL); + mmap_write_unlock(mm); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, result[0], start + PAGE_SIZE); + KUNIT_ASSERT_EQ(test, result[1], start + 2 * PAGE_SIZE); + KUNIT_ASSERT_EQ(test, result[2], start); + KUNIT_ASSERT_EQ(test, result[3], start + 3 * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, mm->map_count, 2); + vma = test_vma_lookup(mm, start); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma->vm_start, start); + KUNIT_EXPECT_EQ(test, vma->vm_end, start + 3 * PAGE_SIZE); + for (i = 0; i < 12; i++) + KUNIT_EXPECT_TRUE(test, vm_page_offset_equal(vma_page_offset_at(vma, start + i * SZ_4K), + (struct vm_page_offset) { (i + 1) / TEST_SLOTS, ((i + 1) % TEST_SLOTS) * SZ_4K })); + vma = test_vma_lookup(mm, start + 3 * PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), 2U * SZ_4K); +} + +#ifdef CONFIG_PROC_FS +/* Exercise the actual status/statm producers without publishing a fake task. */ +extern void task_mem(struct seq_file *m, struct mm_struct *mm); +extern unsigned long task_vsize(struct mm_struct *mm); +extern unsigned long task_statm(struct mm_struct *mm, unsigned long *shared, + unsigned long *text, unsigned long *data, unsigned long *resident); + +static void expect_status_kb(struct kunit *test, struct seq_file *seq, + const char *name, unsigned long expected) +{ + const char *field = strstr(seq->buf, name); + unsigned long value = ULONG_MAX; + + KUNIT_ASSERT_NOT_NULL(test, field); + KUNIT_EXPECT_EQ(test, sscanf(field + strlen(name), "%lu", &value), 1); + KUNIT_EXPECT_EQ_MSG(test, value, expected, "%s", name); +} + +static void user4k_accounting_readers_test(struct kunit *test) +{ + struct mm_struct *small = test_vma_mm(test); + struct mm_struct *native = mm_alloc(); + struct seq_file seq = {}; + unsigned long shared, text, data, resident, size, maxrss = 0; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, native), 0); + seq.size = 2048; + seq.buf = kunit_kzalloc(test, seq.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, seq.buf); + /* Same logical counts, deliberately not multiples of a native page. */ + for (i = 0; i < 2; i++) { + struct mm_struct *mm = i ? native : small; + unsigned long kb = i ? PAGE_SIZE / 1024 : 4; + + mm->total_vm = 11; + mm->hiwater_vm = 13; + mm->data_vm = 5; + mm->stack_vm = 2; + mm->exec_vm = 3; + mm->locked_vm = mm_pages_to_account(mm, 1); + atomic64_set(&mm->pinned_vm, 2); + mm->hiwater_rss = 10; + mm->start_code = TEST_VA + mm_page_size(mm); + mm->end_code = mm->start_code + 2 * mm_page_size(mm); + percpu_counter_set(&mm->rss_stat[MM_ANONPAGES], 5); + percpu_counter_set(&mm->rss_stat[MM_FILEPAGES], 2); + percpu_counter_set(&mm->rss_stat[MM_SHMEMPAGES], 1); + percpu_counter_set(&mm->rss_stat[MM_SWAPENTS], 3); + seq.count = 0; + task_mem(&seq, mm); + KUNIT_EXPECT_FALSE(test, seq_has_overflowed(&seq)); + seq.buf[min(seq.count, seq.size - 1)] = '\0'; + expect_status_kb(test, &seq, "VmPeak:", 13 * kb); + expect_status_kb(test, &seq, "VmSize:", 11 * kb); + expect_status_kb(test, &seq, "VmLck:", kb); + expect_status_kb(test, &seq, "VmPin:", 2 * PAGE_SIZE / 1024); + expect_status_kb(test, &seq, "VmHWM:", 10 * kb); + expect_status_kb(test, &seq, "VmRSS:", 8 * kb); + expect_status_kb(test, &seq, "RssAnon:", 5 * kb); + expect_status_kb(test, &seq, "RssFile:", 2 * kb); + expect_status_kb(test, &seq, "RssShmem:", kb); + expect_status_kb(test, &seq, "VmData:", 5 * kb); + expect_status_kb(test, &seq, "VmStk:", 2 * kb); + expect_status_kb(test, &seq, "VmExe:", 2 * kb); + expect_status_kb(test, &seq, "VmLib:", kb); + expect_status_kb(test, &seq, "VmSwap:", 3 * kb); + KUNIT_EXPECT_EQ(test, task_vsize(mm), 11 * kb * 1024); + { + struct vm_area_struct vma = { .vm_mm = mm }; + + KUNIT_EXPECT_EQ(test, vma_kernel_pagesize(&vma), kb * 1024); + KUNIT_EXPECT_EQ(test, vma_mmu_pagesize(&vma), kb * 1024); + } + } + /* Maxima must compare bytes, not incompatible 4K/16K counts. */ + setmax_mm_hiwater_rss(&maxrss, small); + KUNIT_EXPECT_EQ(test, maxrss, 40UL); + setmax_mm_hiwater_rss(&maxrss, native); + KUNIT_EXPECT_EQ(test, maxrss, 10UL * PAGE_SIZE / 1024); + setmax_mm_hiwater_rss(&maxrss, small); + KUNIT_EXPECT_EQ(test, maxrss, 10UL * PAGE_SIZE / 1024); + small->hiwater_rss = 10 * TEST_SLOTS + 1; + setmax_mm_hiwater_rss(&maxrss, small); + KUNIT_EXPECT_EQ(test, maxrss, (10UL * TEST_SLOTS + 1) * 4); + + KUNIT_EXPECT_EQ(test, mm_pages_to_native(small, 0), 0UL); + KUNIT_EXPECT_EQ(test, mm_pages_to_native(small, 1), 1UL); + KUNIT_EXPECT_EQ(test, mm_pages_to_native(small, 5), DIV_ROUND_UP(5UL, TEST_SLOTS)); + KUNIT_EXPECT_EQ(test, mm_pages_for_reader(native, small, 5), 5UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, mm_pages_for_reader(small, native, 5), DIV_ROUND_UP(5UL, TEST_SLOTS)); + KUNIT_EXPECT_EQ(test, mm_pages_for_reader(small, small, 5), 5UL); + KUNIT_EXPECT_EQ(test, mm_pages_for_reader(native, native, 5), 5UL); + KUNIT_EXPECT_EQ(test, mm_pages_to_shift(small, 5, MM_ACCOUNT_SHIFT), 5UL); + KUNIT_EXPECT_EQ(test, mm_pages_to_shift(native, 5, MM_ACCOUNT_SHIFT), 5UL * TEST_SLOTS); + + /* A kernel reader defaults to native units; partial pages round up. */ + size = task_statm(small, &shared, &text, &data, &resident); + KUNIT_EXPECT_EQ(test, size, DIV_ROUND_UP(11UL, TEST_SLOTS)); + KUNIT_EXPECT_EQ(test, shared, 1UL); + KUNIT_EXPECT_EQ(test, text, 1UL); + KUNIT_EXPECT_EQ(test, data, DIV_ROUND_UP(7UL, TEST_SLOTS)); + KUNIT_EXPECT_EQ(test, resident, DIV_ROUND_UP(8UL, TEST_SLOTS)); + if (user4k_test_enabled || IS_ENABLED(CONFIG_KASAN)) { + kthread_use_mm(small); + size = task_statm(native, &shared, &text, &data, &resident); + kthread_unuse_mm(small); + KUNIT_EXPECT_EQ(test, size, 11UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, shared, 3UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, text, 2UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, data, 7UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, resident, 8UL * TEST_SLOTS); + } + /* Synthetic counters have no page references; restore before mmput. */ + for (i = 0; i < 2; i++) { + struct mm_struct *mm = i ? native : small; + unsigned int counter; + + for (counter = 0; counter < NR_MM_COUNTERS; counter++) + percpu_counter_set(&mm->rss_stat[counter], 0); + mm->total_vm = mm->data_vm = mm->stack_vm = mm->exec_vm = 0; + mm->locked_vm = 0; + atomic64_set(&mm->pinned_vm, 0); + } +} +#endif + +static void mixed_locked_account_units_test(struct kunit *test) +{ + unsigned int shift; + unsigned long native_units = PAGE_SIZE >> MM_ACCOUNT_SHIFT; + + for (shift = 12; shift <= 16; shift += 2) { + struct mm_struct *mm = mm_alloc(); + unsigned long size = 1UL << shift; + unsigned long base, limit; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + if (shift != PAGE_SHIFT) + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, shift), 0); + base = mm_pages_to_account(mm, 1); + mmap_write_lock(mm); + /* Model one locked VMA leaf and two independently charged native pins. */ + mm->locked_vm = base; + ret = __account_locked_vm(mm, 1, true, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, base + native_units); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(mm), size + PAGE_SIZE); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(mm), + DIV_ROUND_UP(size + PAGE_SIZE, PAGE_SIZE)); + ret = __account_locked_vm(mm, 1, true, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, base + 2 * native_units); + ret = __account_locked_vm(mm, 1, false, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(mm), size + PAGE_SIZE); + ret = __account_locked_vm(mm, 1, false, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, base); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(mm), size); + + /* A refused charge must leave the exact pre-existing amount intact. */ + limit = task_rlimit(current, RLIMIT_MEMLOCK) >> MM_ACCOUNT_SHIFT; + mm->locked_vm = limit; + ret = __account_locked_vm(mm, 1, true, current, false); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->locked_vm, limit); + mm->locked_vm = ULONG_MAX - native_units + 1; + ret = __account_locked_vm(mm, 1, true, current, true); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->locked_vm, ULONG_MAX - native_units + 1); + mm->locked_vm = 0; + if (native_units > 1) { + ret = __account_locked_vm(mm, + (ULONG_MAX >> (PAGE_SHIFT - MM_ACCOUNT_SHIFT)) + 1, + true, current, true); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->locked_vm, 0UL); + } + mmap_write_unlock(mm); + } +} + +/* Prior shmem fixtures can refund global commit from deferred fput work. */ +static s64 test_commit_baseline(struct kunit *test) +{ + s64 before = percpu_counter_sum(&vm_committed_as), after; + + /* No mmap, file, or filesystem locks are held at these call sites. */ + flush_delayed_fput(); + after = percpu_counter_sum(&vm_committed_as); + if (after != before) + kunit_info(test, "drained deferred file refunds: commit delta %lld", after - before); + return after; +} + +static void user64k_commit_accounting_test(struct kunit *test) +{ + struct mm_struct *mm; + s64 before; + unsigned int i; + unsigned long native_per_leaf; + int ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires userspace granule larger than native"); + return; + } + mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, 16), 0); + native_per_leaf = SZ_64K / PAGE_SIZE; + before = test_commit_baseline(test); + mmap_write_lock(mm); + for (i = 0; i < 9; i++) { + ret = mm_account_memory(mm, mm, 1); + KUNIT_EXPECT_EQ(test, ret, 0); + if (ret) + break; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i + 1); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)((i + 1) * native_per_leaf)); + } + /* Multiplication overflow must not mutate either reservation counter. */ + ret = mm_account_memory(mm, mm, LONG_MAX / native_per_leaf + 1); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)(i * native_per_leaf)); + mm_acct_memory(mm, 3); + i += 3; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)(i * native_per_leaf)); + while (i) { + mm_unacct_memory(mm, 1); + i--; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)(i * native_per_leaf)); + } + mmap_write_unlock(mm); +} + +#ifdef CONFIG_PAGE_TABLE_CHECK +static void expect_coarse_file_counts(struct kunit *test, void *data, + unsigned int phase) +{ + unsigned int offset; + + for (offset = 0; offset < 2 * SZ_64K; offset += SZ_4K) { + int anon = -1, file = -1, expected; + int ret = page_table_check_get_counts(virt_to_page(data + offset), + offset & ~PAGE_MASK, &anon, &file); + + /* Phase0: both coarse leaves;1:plusalias;2:firstclear;3:bothclear;4:allclear. */ + expected = phase < 2 || (phase == 2 && offset >= SZ_64K); + if (phase >= 1 && phase <= 3 && offset == 2 * SZ_64K - SZ_4K) + expected++; + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, anon, 0); + KUNIT_EXPECT_EQ_MSG(test, file, expected, "phase=%u offset=%u", phase, offset); + } +} +#endif + +static void user64k_page_table_check_test(struct kunit *test) +{ +#ifdef CONFIG_PAGE_TABLE_CHECK + struct mm_struct *coarse, *small; + void *data; + pte_t ptes[2] = {}, alias = __pte(0), first, last; + int anon, file; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires userspace leaf larger than native page"); + return; + } + data = test_alloc_granule(test, 17); + KUNIT_ASSERT_NOT_NULL(test, data); + if (page_table_check_get_counts(virt_to_page(data), 0, &anon, &file)) { + kunit_skip(test, "requires page-table-check enabled at boot"); + return; + } + coarse = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, coarse), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(coarse, 16), 0); + small = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, small), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(small, 12), 0); + first = phys_pte_mm(coarse, virt_to_phys(data), __pgprot(TEST_PROT)); + last = phys_pte_mm(small, virt_to_phys(data) + 2 * SZ_64K - SZ_4K, + __pgprot(TEST_PROT)); + + /* Exercise checker hooks directly; no actual coarse file fault is claimed. */ + page_table_check_ptes_set(coarse, TEST_VA, ptes, first, 2); + ptes[0] = first; + ptes[1] = pte_advance_pfn_mm(coarse, first, 1); + expect_coarse_file_counts(test, data, 0); + page_table_check_ptes_set(small, TEST_VA, &alias, last, 1); + alias = last; + expect_coarse_file_counts(test, data, 1); + page_table_check_pte_clear(coarse, TEST_VA, ptes[0]); + ptes[0] = __pte(0); + expect_coarse_file_counts(test, data, 2); + page_table_check_pte_clear(coarse, TEST_VA + SZ_64K, ptes[1]); + ptes[1] = __pte(0); + expect_coarse_file_counts(test, data, 3); + page_table_check_pte_clear(small, TEST_VA, alias); + expect_coarse_file_counts(test, data, 4); +#else + kunit_skip(test, "requires CONFIG_PAGE_TABLE_CHECK"); +#endif +} + +static void user4k_commit_accounting_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + s64 before; + int ret; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, mm); + before = test_commit_baseline(test); + mmap_write_lock(mm); + for (i = 0; i < 9; i++) { + ret = mm_account_memory(mm, mm, 1); + KUNIT_EXPECT_EQ(test, ret, 0); + if (ret) + break; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i + 1); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)DIV_ROUND_UP(i + 1, TEST_SLOTS)); + } + if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) { + unsigned long old = mm->committed_user_pages; + s64 committed = percpu_counter_sum(&vm_committed_as); + + ret = mm_account_memory(mm, mm, (totalram_pages() + total_swap_pages + 1) * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, old); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as), committed); + } + /* Native physical-page callers must charge four user granules per page. */ + ret = __account_locked_vm(mm, 1, true, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, (unsigned long)TEST_SLOTS); + ret = __account_locked_vm(mm, 1, false, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, 0UL); + + /* Refund one granule at a time: splitting must not multiply refunds. */ + while (i) { + mm_unacct_memory(mm, 1); + i--; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)DIV_ROUND_UP(i, TEST_SLOTS)); + } + mmap_write_unlock(mm); +} + +/* Prefaulting must stop at each process-page boundary, not native pages. */ +static void user_granule_prefault_test(struct kunit *test) +{ + unsigned int shift; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mapping fixture requires hardware opt-in"); + return; + } + for (shift = 12; shift <= PAGE_SHIFT; shift += 2) { + struct mm_struct *mm = test_vma_mm_granule(test, shift); + unsigned long size = 1UL << shift, start = TEST_VA; + unsigned long flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE; + unsigned long mapped[3]; + size_t left[3][4] = {}; + int phase, ret = 0; + + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped[0] = vm_mmap(NULL, start, size, PROT_READ | PROT_WRITE, flags, 0); + mapped[2] = vm_mmap(NULL, start + 2 * size, size, + PROT_READ | PROT_WRITE, flags, 0); + mapped[1] = start + size; + if (mapped[0] == start && mapped[2] == start + 2 * size) { + for (phase = 0; phase < 3; phase++) { + char __user *addr = (char __user *)(start + 17); + size_t len = 3 * size - 17; + + /* Missing middle, then read-only, then fully writable. */ + if (phase) { + mapped[1] = vm_mmap(NULL, start + size, size, + PROT_READ | (phase == 2 ? PROT_WRITE : 0), + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, 0); + if (mapped[1] != start + size) + break; + } + left[phase][0] = fault_in_safe_writeable(addr, len); + left[phase][1] = fault_in_readable(addr, len); + left[phase][2] = fault_in_writeable(addr, len); + left[phase][3] = fault_in_subpage_writeable(addr, len); + } + ret = vm_munmap(start, 3 * size); + } + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped[0], start); + KUNIT_ASSERT_EQ(test, mapped[1], start + size); + KUNIT_ASSERT_EQ(test, mapped[2], start + 2 * size); + KUNIT_EXPECT_EQ(test, ret, 0); + for (phase = 0; phase < 3; phase++) { + KUNIT_EXPECT_EQ(test, left[phase][0], phase < 2 ? 2 * size : 0UL); + KUNIT_EXPECT_EQ(test, left[phase][1], phase == 0 ? 2 * size : 0UL); + KUNIT_EXPECT_EQ(test, left[phase][2], phase < 2 ? 2 * size : 0UL); + KUNIT_EXPECT_EQ(test, left[phase][3], phase < 2 ? 2 * size : 0UL); + } + } +} + +static void user4k_mmap_accounting_test(struct kunit *test) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + unsigned long start = TEST_VA + SZ_4K, mapped; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE; + s64 before; + unsigned int i; + int ret = 0; + vm_fault_t fault = 0; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mapping fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + before = test_commit_baseline(test); + kthread_use_mm(mm); + mmap_write_lock(mm); + mapped = mmap_region(NULL, start, 3 * SZ_4K, flags, + (struct vm_page_offset) { start >> PAGE_SHIFT, SZ_4K }, NULL); + mmap_write_unlock(mm); + if (!IS_ERR_VALUE(mapped)) { + mmap_read_lock(mm); + vma = vma_lookup(mm, start); + fault = handle_mm_fault(vma, start, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + } + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + KUNIT_ASSERT_EQ(test, fault & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->data_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, 1LL); + /* Remove the middle first, then the two neighbors. */ + for (i = 0; i < 3; i++) { + unsigned int slot = i == 0 ? 1 : i == 1 ? 0 : 2; + VMA_ITERATOR(vmi, mm, start + slot * SZ_4K); + + kthread_use_mm(mm); + mmap_write_lock(mm); + ret = do_vmi_munmap(&vmi, mm, start + slot * SZ_4K, SZ_4K, NULL, false); + mmap_write_unlock(mm); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 2UL - i); + KUNIT_EXPECT_EQ(test, mm->data_vm, 2UL - i); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 2UL - i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, i == 2 ? 0LL : 1LL); + } + KUNIT_EXPECT_EQ(test, mm->map_count, 0); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); +} + +static vm_fault_t test_file_fault(struct vm_area_struct *vma, unsigned long addr, bool write) +{ + vm_fault_t ret; + + mmap_read_lock(vma->vm_mm); + ret = handle_mm_fault(vma, addr, FAULT_FLAG_REMOTE | + (write ? FAULT_FLAG_WRITE : 0), NULL); + mmap_read_unlock(vma->vm_mm); + return ret; +} + +static void user4k_vm_mmap_test(struct kunit *test) +{ + struct mm_struct *mm; + struct file *file; + struct vm_area_struct *vma; + unsigned long start = TEST_VA + SZ_4K; + unsigned long anon_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE; + unsigned long file_flags = MAP_SHARED | MAP_FIXED_NOREPLACE; + unsigned long result[7]; + int unmap[4]; + pte_t pte; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mapping fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 2 * PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + mm->mmap_base = TEST_VA + 2 * PAGE_SIZE + SZ_4K; + mm_flags_clear(MMF_TOPDOWN, mm); + kthread_use_mm(mm); + result[0] = vm_mmap(NULL, start, SZ_4K, PROT_READ | PROT_WRITE, anon_flags, 0); + result[1] = vm_mmap(file, start + 2 * SZ_4K, SZ_4K, PROT_READ | PROT_WRITE, + file_flags, 3 * SZ_4K); + result[2] = vm_mmap(file, start + 3 * SZ_4K, SZ_4K, PROT_READ | PROT_WRITE, + file_flags, 4 * SZ_4K); + result[3] = vm_mmap(NULL, start, SZ_4K, PROT_READ, anon_flags, 0); + result[4] = vm_mmap(NULL, start + 17, SZ_4K, PROT_READ, anon_flags, 0); + result[5] = vm_mmap(file, start + SZ_4K, SZ_4K, PROT_READ, file_flags, SZ_4K + 1); + result[6] = vm_mmap(NULL, 0, SZ_4K + 1, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, result[0], start); + KUNIT_ASSERT_EQ(test, result[1], start + 2 * SZ_4K); + KUNIT_ASSERT_EQ(test, result[2], start + 3 * SZ_4K); + KUNIT_EXPECT_EQ(test, result[3], (unsigned long)-EEXIST); + KUNIT_EXPECT_EQ(test, result[4], (unsigned long)-EINVAL); + KUNIT_EXPECT_EQ(test, result[5], (unsigned long)-EINVAL); + KUNIT_ASSERT_EQ(test, result[6], mm->mmap_base); + KUNIT_EXPECT_EQ(test, mm->total_vm, 5UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + vma = test_vma_lookup(mm, start); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, start, true) & VM_FAULT_ERROR, 0U); + vma = test_vma_lookup(mm, start + 2 * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma->vm_end - vma->vm_start, 2UL * SZ_4K); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), 3U * SZ_4K); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, start + 2 * SZ_4K, false) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, mm, start + 2 * SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(mm, pte)), (u8)0x34); + pte = test_fault_entry(test, mm, start + 3 * SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(mm, pte)), (u8)0x35); + kthread_use_mm(mm); + unmap[0] = vm_munmap(start + 2 * SZ_4K, 1); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, unmap[0], 0); + vma = test_vma_lookup(mm, start + 3 * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma->vm_pgoff, 4UL / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), (4U % TEST_SLOTS) * SZ_4K); + KUNIT_EXPECT_EQ(test, mm->total_vm, 4UL); + kthread_use_mm(mm); + unmap[1] = vm_munmap(start, SZ_4K); + unmap[2] = vm_munmap(start + 3 * SZ_4K, SZ_4K); + unmap[3] = vm_munmap(result[6], 2 * SZ_4K); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, unmap[1], 0); + KUNIT_EXPECT_EQ(test, unmap[2], 0); + KUNIT_EXPECT_EQ(test, unmap[3], 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 0UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 0UL); + KUNIT_EXPECT_EQ(test, mm->map_count, 0); +} + +static void test_mlock_drain_all(struct kunit *test, struct folio *folio) +{ + unsigned int before, after; + bool counted; + + lru_add_drain(); + counted = folio_test_unevictable(folio); + before = counted ? folio->mlock_count : 0; + /* A sleeping fixture can enqueue mlock work on more than one CPU. */ + lru_add_drain_all(); + after = folio_test_unevictable(folio) ? folio->mlock_count : 0; + if (counted && before != after) + kunit_info(test, "remote mlock drain changed count %u -> %u", before, after); +} + +static void user4k_mlock_counts_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + struct mm_subpage *slot; + struct folio *folio; + unsigned long addr; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mlock requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE | MAP_LOCKED, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, test_fault_entry(test, mm, TEST_VA))); + KUNIT_ASSERT_NOT_NULL(test, slot); + folio = mm_subpage_folio(slot); + folio_get(folio); + mm_subpage_put(slot); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS); + KUNIT_EXPECT_EQ(test, mm->locked_vm, (unsigned long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + int ret; + + kthread_use_mm(mm); + ret = vm_munmap(TEST_VA + i * SZ_4K, SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + test_mlock_drain_all(test, folio); + /* mlock_count shares storage with the evictable LRU links. */ + if (i != TEST_SLOTS - 1) + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS - 1 - i); + KUNIT_EXPECT_EQ(test, mm->locked_vm, TEST_SLOTS - 1UL - i); + KUNIT_EXPECT_EQ(test, folio_test_mlocked(folio), i != TEST_SLOTS - 1); + } +} + +static void test_mlock_enable_lru_cache(void *unused) +{ + lru_cache_enable(); +} + +static void user4k_mlock_shared_test(struct kunit *test) +{ + struct mm_struct *small = test_vma_mm(test), *native = mm_alloc(); + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct folio *folio; + unsigned long addr; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + if (native) + mmput(native); + kunit_skip(test, "borrowed-mm mlock requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, native), 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + /* Exact counts require synchronous LRU/lock accounting. Normal deferred + * batches may legitimately miss locks while the folio is off LRU; + * user4k_mlock_repair_test checks that reclaim repairs that case. + */ + lru_cache_disable(); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_mlock_enable_lru_cache, NULL), 0); + kthread_use_mm(small); + addr = vm_mmap(file, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_LOCKED | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(small); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + kthread_use_mm(native); + addr = vm_mmap(file, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_LOCKED | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(native); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS + 1); + kthread_use_mm(small); + ret = vm_munmap(TEST_VA + SZ_4K, SZ_4K); + kthread_unuse_mm(small); + KUNIT_ASSERT_EQ(test, ret, 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS); + kthread_use_mm(native); + ret = vm_munmap(TEST_VA, PAGE_SIZE); + kthread_unuse_mm(native); + KUNIT_ASSERT_EQ(test, ret, 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS - 1); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + kthread_use_mm(small); + ret = vm_munmap(TEST_VA, PAGE_SIZE); + kthread_unuse_mm(small); + KUNIT_ASSERT_EQ(test, ret, 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_FALSE(test, folio_test_mlocked(folio)); +} + +static void test_mlock_putback(void *folio) +{ + folio_putback_lru(folio); +} + +static void user4k_mlock_repair_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int mode, walker; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mlock requires hardware opt-in"); + return; + } + for (mode = 0; mode < (PAGE_SHIFT > 14 ? 3U : 2U); mode++) { + unsigned long size = 1UL << shifts[mode]; + + for (walker = 0; walker < 2; walker++) { + struct mm_struct *mm = test_vma_mm_granule(test, shifts[mode]); + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct folio *folio; + unsigned long addr, offset; + vm_flags_t flags = 0; + int maps, ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_folio_put, folio), 0); + lru_add_drain_all(); + KUNIT_ASSERT_TRUE(test, folio_isolate_lru(folio)); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_mlock_putback, folio), 0); + /* Two aliases leave a locked mapping even after native unmap. + * Force deferred mlocks to run while the folio is isolated. + */ + for (offset = 0; offset < 2 * PAGE_SIZE; offset += PAGE_SIZE) { + kthread_use_mm(mm); + addr = vm_mmap(file, TEST_VA + offset, PAGE_SIZE, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_LOCKED | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA + offset); + } + lru_add_drain_all(); + KUNIT_EXPECT_FALSE(test, folio_test_lru(folio)); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + kunit_release_action(test, test_mlock_putback, folio); + lru_add_drain_all(); + KUNIT_ASSERT_TRUE(test, folio_test_unevictable(folio)); + KUNIT_EXPECT_EQ(test, folio->mlock_count, 0U); + + kthread_use_mm(mm); + ret = vm_munmap(TEST_VA, size); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + KUNIT_EXPECT_FALSE(test, folio_test_mlocked(folio)); + maps = folio_mapcount(folio); + KUNIT_EXPECT_EQ(test, maps, (int)(2 * PAGE_SIZE / size) - 1); + folio_lock(folio); + if (walker) + ret = folio_referenced(folio, 1, NULL, &flags); + else + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + lru_add_drain_all(); + if (walker) { + KUNIT_EXPECT_GE(test, ret, 0); + KUNIT_EXPECT_TRUE(test, flags & VM_LOCKED); + } + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), maps); + for (offset = size; offset < 2 * PAGE_SIZE; offset += size) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + offset); + + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_PTR_EQ(test, + page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(mm, pte)))), + folio); + } + kthread_use_mm(mm); + ret = vm_munmap(TEST_VA, 2 * PAGE_SIZE); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + KUNIT_EXPECT_FALSE(test, folio_test_mlocked(folio)); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + kunit_info(test, "%uK %s repairs deferred mlock without unmapping", + 1U << (shifts[mode] - 10), walker ? "referenced" : "unmap"); + } + } +} + +asmlinkage long __arm64_sys_mremap(const struct pt_regs *regs); + +static unsigned long test_mremap(unsigned long old, unsigned long old_len, + unsigned long new_len, unsigned long flags, unsigned long new) +{ + struct pt_regs regs = { .regs = { old, old_len, new_len, flags, new } }; + + return __arm64_sys_mremap(®s); +} + +static int test_iova_bitmap_fill(struct iova_bitmap *bitmap, unsigned long iova, + size_t length, void *opaque) +{ + unsigned long unit = *(unsigned long *)opaque; + + iova_bitmap_set(bitmap, iova + 3 * unit, length - 10 * unit); + /* A second write advances backwards into an already populated range. */ + iova_bitmap_set(bitmap, iova, 5 * unit); + return 0; +} + +/* Real IOMMUFD lifecycle with a translation-recording IOMMU, no physical DMA. */ +struct test_fragment_domain { + struct iommu_domain domain; + struct xarray translations; + unsigned long fail_after; + unsigned long largest_map; + unsigned long mapped; + unsigned long map_calls; +}; + +struct test_fragment_iopt { + struct io_pagetable table; + struct test_fragment_domain domains[2]; + struct kunit *test; +}; + +static int test_fragment_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t phys, size_t pgsize, size_t pgcount, int prot, + gfp_t gfp, size_t *mapped) +{ + struct test_fragment_domain *mock = container_of(domain, typeof(*mock), domain); + unsigned long length = pgsize * pgcount, done; + int rc; + + mock->map_calls++; + mock->largest_map = max(mock->largest_map, length); + for (done = 0; done < length; done += SZ_4K) { + if (mock->fail_after && mock->mapped >= mock->fail_after) + return -ENOMEM; + rc = xa_insert(&mock->translations, (iova + done) >> 12, + xa_mk_value((phys + done) >> 12), gfp); + if (rc) + return rc; + *mapped += SZ_4K; + mock->mapped += SZ_4K; + } + return 0; +} + +static size_t test_fragment_unmap(struct iommu_domain *domain, unsigned long iova, + size_t pgsize, size_t pgcount, struct iommu_iotlb_gather *gather) +{ + struct test_fragment_domain *mock = container_of(domain, typeof(*mock), domain); + unsigned long length = pgsize * pgcount, done; + + for (done = 0; done < length; done += SZ_4K) { + if (!xa_erase(&mock->translations, (iova + done) >> 12)) + break; + mock->mapped -= SZ_4K; + } + return done; +} + +static phys_addr_t test_fragment_translate(struct iommu_domain *domain, dma_addr_t iova) +{ + struct test_fragment_domain *mock = container_of(domain, typeof(*mock), domain); + void *entry = xa_load(&mock->translations, iova >> 12); + + return entry ? ((phys_addr_t)xa_to_value(entry) << 12) + (iova & (SZ_4K - 1)) : 0; +} + +static void test_fragment_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) +{ +} + +static const struct iommu_domain_ops test_fragment_domain_ops = { + .map_pages = test_fragment_map, + .unmap_pages = test_fragment_unmap, + .iova_to_phys = test_fragment_translate, + .iotlb_sync = test_fragment_sync, +}; + +static void test_fragment_iopt_free(void *data) +{ + struct test_fragment_iopt *fixture = data; + struct iommu_domain *domain; + unsigned long unmapped, index; + unsigned int i; + + iopt_unmap_all(&fixture->table, &unmapped); + while ((domain = xa_find(&fixture->table.domains, &(unsigned long){0}, + ULONG_MAX, XA_PRESENT))) + iopt_table_remove_domain(&fixture->table, domain); + iopt_destroy_table(&fixture->table); + for (i = 0; i < ARRAY_SIZE(fixture->domains); i++) { + struct test_fragment_domain *mock = &fixture->domains[i]; + void *entry; + + KUNIT_EXPECT_TRUE(fixture->test, xa_empty(&mock->translations)); + xa_for_each(&mock->translations, index, entry) + xa_erase(&mock->translations, index); + xa_destroy(&mock->translations); + } +} + +static struct test_fragment_iopt *test_fragment_iopt_alloc(struct kunit *test) +{ + struct test_fragment_iopt *fixture = kunit_kzalloc(test, sizeof(*fixture), GFP_KERNEL); + unsigned int i; + + if (!fixture) + return NULL; + fixture->test = test; + iopt_init_table(&fixture->table); + for (i = 0; i < ARRAY_SIZE(fixture->domains); i++) { + struct test_fragment_domain *mock = &fixture->domains[i]; + + xa_init(&mock->translations); + mock->domain.ops = &test_fragment_domain_ops; + mock->domain.type = IOMMU_DOMAIN_UNMANAGED; + mock->domain.pgsize_bitmap = SZ_4K; + mock->domain.geometry.aperture_end = ULONG_MAX; + } + if (kunit_add_action_or_reset(test, test_fragment_iopt_free, fixture)) + return NULL; + return fixture; +} + +static int test_fragment_map_user(struct test_fragment_iopt *fixture, + struct mm_struct *mm, unsigned long va, unsigned long iova, unsigned long length) +{ + struct iommufd_ctx ctx = {}; + int rc; + + kthread_use_mm(mm); + rc = iopt_map_user_pages(&ctx, &fixture->table, &iova, (void __user *)va, + length, IOMMU_READ | IOMMU_WRITE, 0); + kthread_unuse_mm(mm); + return rc; +} + +static void user4k_iommufd_lifetime_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14, 16 }, mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + for (mode = 0; mode < ARRAY_SIZE(shifts); mode++) { + unsigned long size = 1UL << shifts[mode]; + unsigned long len = 2 * max(size, PAGE_SIZE); + unsigned long dest = TEST_VA + 32 * PAGE_SIZE + size; + unsigned long iova = SZ_1M, alias = SZ_16M, addr, moved, missed, unmapped; + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + struct iopt_pages *pages; + struct iopt_area *area; + unsigned char *buffer; + unsigned long off, pinned; + LIST_HEAD(copy); + int rc; + + if (mode && shifts[mode] == PAGE_SHIFT) + continue; + mm = test_vma_mm_granule(test, shifts[mode]); + KUNIT_ASSERT_NOT_NULL(test, mm); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + buffer = kunit_kmalloc(test, len, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, buffer); + memset(buffer, 0x31, len); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + missed = IS_ERR_VALUE(addr) ? len : copy_to_user((void __user *)addr, buffer, len); + moved = missed ? 0 : test_mremap(addr, len, len, MREMAP_FIXED | MREMAP_MAYMOVE, dest); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, moved, dest); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, dest, iova, len), 0); + pinned = atomic64_read(&mm->pinned_vm); + KUNIT_EXPECT_EQ(test, pinned, len / PAGE_SIZE); + KUNIT_EXPECT_GE(test, fixture->domains[0].largest_map, PAGE_SIZE); + for (off = 0; off < len; off += size) { + pte_t pte = test_fault_entry(test, mm, dest + off); + phys_addr_t actual = iommu_iova_to_phys(&fixture->domains[0].domain, iova + off); + + KUNIT_EXPECT_EQ(test, actual, pte_phys_mm(mm, pte)); + } + KUNIT_ASSERT_EQ(test, iopt_get_pages(&fixture->table, iova + size, len - size, ©), 0); + rc = iopt_map_pages(&fixture->table, ©, len - size, &alias, IOMMU_READ | IOMMU_WRITE, 0); + iopt_free_pages_list(©); + KUNIT_ASSERT_EQ(test, rc, 0); + KUNIT_EXPECT_EQ(test, (unsigned long)atomic64_read(&mm->pinned_vm), pinned); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), 0); + KUNIT_EXPECT_EQ(test, (unsigned long)atomic64_read(&mm->pinned_vm), pinned); + + /* Replace the original VA: mappings and copied IOAS must retain old pins. */ + kthread_use_mm(mm); + addr = vm_mmap(NULL, dest, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, 0); + memset(buffer, 0x5a, len); + missed = IS_ERR_VALUE(addr) ? len : copy_to_user((void __user *)dest, buffer, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, missed, 0UL); + down_read(&fixture->table.iova_rwsem); + area = iopt_area_iter_first(&fixture->table, iova, iova); + pages = area->pages; + /* Force the domain-backed reader for the native control too. */ + memset(buffer, 0x77, len); + rc = iopt_pages_rw_access(pages, 0, buffer, len, IOMMUFD_ACCESS_RW_WRITE); + up_read(&fixture->table.iova_rwsem); + KUNIT_ASSERT_EQ(test, rc, 0); + for (off = 0; off < len; off += SZ_4K) { + phys_addr_t phys = iommu_iova_to_phys(&fixture->domains[1].domain, iova + off); + void *ptr = kmap_local_page(pfn_to_page(PHYS_PFN(phys))); + bool correct = !memchr_inv(ptr + offset_in_page(phys), 0x77, SZ_4K); + + kunmap_local(ptr); + KUNIT_EXPECT_TRUE(test, correct); + } + kthread_use_mm(mm); + missed = copy_from_user(buffer, (void __user *)dest, len); + /* Local CPU access retains the existing copy_from_user fast path. */ + rc = iopt_pages_rw_access(pages, 0, buffer, len, 0); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, rc, 0); + KUNIT_EXPECT_EQ(test, missed, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0x5a, len), NULL); + KUNIT_ASSERT_EQ(test, iopt_unmap_iova(&fixture->table, iova, len, &unmapped), 0); + KUNIT_EXPECT_EQ(test, unmapped, len); + KUNIT_EXPECT_EQ(test, (unsigned long)atomic64_read(&mm->pinned_vm), (len - (size < PAGE_SIZE ? 0 : size)) / PAGE_SIZE); + iopt_table_remove_domain(&fixture->table, &fixture->domains[0].domain); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, len - size); + KUNIT_ASSERT_EQ(test, iopt_unmap_iova(&fixture->table, alias, len - size, &unmapped), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + kunit_release_action(test, test_fragment_iopt_free, fixture); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOMMUFD %luK: relocated PA, batching, copy, two domains, VA replacement and unpin passed", size / SZ_1K); + } +} + +static void user4k_iommufd_rollback_test(struct kunit *test) +{ + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + unsigned long addr, iova = SZ_1M, size = SZ_4K; + int rc; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), 0); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, 3 * size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + rc = vm_munmap(TEST_VA + size, size); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, rc, 0); + KUNIT_EXPECT_EQ(test, test_fragment_map_user(fixture, mm, addr, iova, 3 * size), -EFAULT); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, 0UL); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA + size, size, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA + size); + KUNIT_EXPECT_LT(test, test_fragment_map_user(fixture, mm, TEST_VA, iova, 3 * size), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA + size, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA + size); + fixture->domains[1].fail_after = size; + KUNIT_EXPECT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, iova, 3 * size), -ENOMEM); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, 0UL); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + fixture->domains[1].fail_after = 0; + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, iova, 3 * size), 0); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 3 * size); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, 3 * size); + kunit_release_action(test, test_fragment_iopt_free, fixture); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + kunit_release_action(test, user4k_vma_mm_free, mm); +} + +static void user4k_iommufd_granule_test(struct kunit *test) +{ + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + unsigned long addr, unmapped, cut = SZ_1M + PAGE_SIZE - 1; + int rc; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + fixture->domains[0].domain.pgsize_bitmap = PAGE_SIZE; + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, 3 * PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + /* The device cannot cover a misaligned fragment by widening the mapping. */ + KUNIT_EXPECT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA + SZ_4K, SZ_1M, PAGE_SIZE), -EINVAL); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + KUNIT_EXPECT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, SZ_1M, SZ_4K), -EINVAL); + /* Full contiguous native spans still work, including legacy area splitting. */ + KUNIT_ASSERT_EQ(test, iopt_disable_large_pages(&fixture->table), 0); + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, SZ_1M, 2 * PAGE_SIZE), 0); + KUNIT_EXPECT_EQ(test, fixture->domains[0].largest_map, PAGE_SIZE); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 2LL); + KUNIT_ASSERT_EQ(test, iopt_cut_iova(&fixture->table, &cut, 1), 0); + rc = iopt_unmap_iova(&fixture->table, SZ_1M, PAGE_SIZE, &unmapped); + KUNIT_ASSERT_EQ(test, rc, 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 1LL); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, PAGE_SIZE); + KUNIT_ASSERT_EQ(test, iopt_unmap_all(&fixture->table, &unmapped), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + /* A closed IOVA interval ending at ULONG_MAX must not wrap at map/unmap. */ + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, + ULONG_MAX - 2 * PAGE_SIZE + 1, 2 * PAGE_SIZE), 0); + KUNIT_ASSERT_EQ(test, iopt_unmap_all(&fixture->table, &unmapped), 0); + KUNIT_EXPECT_EQ(test, unmapped, 2 * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + kunit_release_action(test, test_fragment_iopt_free, fixture); + kunit_release_action(test, user4k_vma_mm_free, mm); +} + +static void user4k_iommufd_shared_test(struct kunit *test) +{ + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + struct file *file; + struct folio *folio; + unsigned long addr, unmapped, iova = SZ_1M; + unsigned char *buffer; + loff_t pos = 0; + phys_addr_t phys; + void *ptr; + int rc; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + file = test_cache_file(test, 2 * PAGE_SIZE, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + buffer = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, buffer); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + kthread_use_mm(mm); + addr = vm_mmap(file, TEST_VA, 2 * SZ_4K, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, iova, 2 * SZ_4K), 0); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_ASSERT_TRUE(test, folio_maybe_dma_pinned(folio)); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 1LL); + folio_lock(folio); + folio_clear_dirty_for_io(folio); + folio_unlock(folio); + phys = iommu_iova_to_phys(&fixture->domains[0].domain, iova); + KUNIT_EXPECT_EQ(test, phys, PFN_PHYS(folio_pfn(folio)) + SZ_4K); + ptr = kmap_local_folio(folio, SZ_4K); + memset(ptr, 0x93, 2 * SZ_4K); + kunmap_local(ptr); + kthread_use_mm(mm); + rc = vm_munmap(TEST_VA, 2 * SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, rc, 0); + /* A failed second-domain attachment must retain the first domain's pins. */ + fixture->domains[1].fail_after = SZ_4K; + KUNIT_EXPECT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), -ENOMEM); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 2 * SZ_4K); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, 0UL); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 1LL); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(folio)); + fixture->domains[1].fail_after = 0; + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), 0); + KUNIT_ASSERT_EQ(test, iopt_unmap_all(&fixture->table, &unmapped), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + KUNIT_EXPECT_FALSE(test, folio_maybe_dma_pinned(folio)); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + KUNIT_ASSERT_EQ(test, kernel_read(file, buffer, PAGE_SIZE, &pos), (ssize_t)PAGE_SIZE); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + SZ_4K, 0x93, 2 * SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 3 * SZ_4K, 0, PAGE_SIZE - 3 * SZ_4K), NULL); + kunit_release_action(test, test_fragment_iopt_free, fixture); + kunit_release_action(test, user4k_vma_mm_free, mm); +} + +static void test_iommufd_exit_account(struct kunit *test, unsigned int shift) +{ + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + struct iopt_pages *pages; + struct iopt_area *area; + unsigned long addr, unmapped; + LIST_HEAD(keep); + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + mm = test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, mm); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, 2 * PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, SZ_1M, 2 * PAGE_SIZE), 0); + down_read(&fixture->table.iova_rwsem); + area = iopt_area_iter_first(&fixture->table, SZ_1M, SZ_1M); + pages = area->pages; + mutex_lock(&pages->mutex); + /* Select VFIO-compatible accounting before the lazy range acquires pins. */ + pages->account_mode = IOPT_PAGES_ACCOUNT_MM; + mutex_unlock(&pages->mutex); + up_read(&fixture->table.iova_rwsem); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 2LL); + KUNIT_EXPECT_EQ(test, mm->locked_vm, 2 * (PAGE_SIZE >> MM_ACCOUNT_SHIFT)); + /* Keep the provider, and thus mm_count, after mm_users reaches zero. */ + KUNIT_ASSERT_EQ(test, iopt_get_pages(&fixture->table, SZ_1M, 2 * PAGE_SIZE, &keep), 0); + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, atomic_read(&mm->mm_users), 0); + KUNIT_EXPECT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), 0); + KUNIT_EXPECT_EQ(test, iopt_unmap_all(&fixture->table, &unmapped), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + KUNIT_EXPECT_EQ(test, mm->locked_vm, 0UL); + KUNIT_EXPECT_EQ(test, pages->npinned, 0UL); + KUNIT_EXPECT_EQ(test, pages->last_npinned, 0UL); + iopt_free_pages_list(&keep); + kunit_release_action(test, test_fragment_iopt_free, fixture); +} + +static void user4k_iommufd_exit_account_test(struct kunit *test) +{ + test_iommufd_exit_account(test, PAGE_SHIFT); + test_iommufd_exit_account(test, 12); +} + +static void user4k_iova_bitmap_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bitmap requires hardware opt-in"); + return; + } + for (mode = 0; mode < (PAGE_SHIFT > 14 ? 3U : 2U); mode++) { + struct mm_struct *mm = test_vma_mm_granule(test, shifts[mode]); + unsigned long size = 1UL << shifts[mode], len = 4 * size; + unsigned long old = TEST_VA, dest = TEST_VA + 32 * PAGE_SIZE + size; + unsigned long start = dest + size - sizeof(u64), bytes = 2 * size; + unsigned long unit = SZ_4K, iova = 1UL << 32, addr, moved; + unsigned char *actual, *expected; + struct iova_bitmap *bitmap; + unsigned long missed; + int ret = 0; + + KUNIT_ASSERT_NOT_NULL(test, mm); + actual = kunit_kzalloc(test, len, GFP_KERNEL); + expected = kunit_kzalloc(test, len, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, actual); + KUNIT_ASSERT_NOT_NULL(test, expected); + kthread_use_mm(mm); + addr = vm_mmap(NULL, old, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + missed = IS_ERR_VALUE(addr) ? len : copy_to_user((void __user *)addr, actual, len); + moved = missed ? 0 : test_mremap(old, len, len, + MREMAP_FIXED | MREMAP_MAYMOVE, dest); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, old); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_ASSERT_EQ(test, moved, dest); + /* VA and PA offsets differ after moving by one alternative leaf. */ + if (size < PAGE_SIZE) { + pte_t pte = test_fault_entry(test, mm, dest); + + KUNIT_EXPECT_NE(test, pte_phys_mm(mm, pte) & ~PAGE_MASK, + dest & ~PAGE_MASK); + } + kthread_use_mm(mm); + bitmap = iova_bitmap_alloc(iova, bytes * 8 * unit, unit, + (u64 __user *)start); + if (!IS_ERR(bitmap)) { + ret = iova_bitmap_for_each(bitmap, &unit, test_iova_bitmap_fill); + iova_bitmap_free(bitmap); + } else { + ret = PTR_ERR(bitmap); + } + missed = copy_from_user(actual, (void __user *)dest, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, missed, 0UL); + memset(expected + size - sizeof(u64), 0xff, bytes); + expected[size - sizeof(u64) + bytes - 1] = 1; + KUNIT_EXPECT_EQ(test, memcmp(actual, expected, len), 0); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOVA bitmap relocated %luK user leaves checked", size / SZ_1K); + } +} + +static void test_iova_free(void *bitmap) +{ + iova_bitmap_free(bitmap); +} + +static void test_iova_kvfree(void *buffer) +{ + kvfree(buffer); +} + +static void user4k_iova_bitmap_window_test(struct kunit *test) +{ + struct mm_struct *mm; + unsigned long capacity = PAGE_SIZE / sizeof(struct user_page_fragment); + unsigned long bytes = (capacity + 3) * SZ_4K, len = bytes + 2 * SZ_4K; + unsigned long start = TEST_VA + SZ_4K - sizeof(u64), unit = SZ_4K; + unsigned long addr, missed; + unsigned char *actual; + struct iova_bitmap *bitmap; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bitmap requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + actual = kvzalloc(len, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, actual); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_iova_kvfree, actual), 0); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + bitmap = iova_bitmap_alloc(0, bytes * 8 * unit, unit, (u64 __user *)start); + ret = IS_ERR(bitmap) ? PTR_ERR(bitmap) : + iova_bitmap_for_each(bitmap, &unit, test_iova_bitmap_fill); + if (!IS_ERR(bitmap)) + iova_bitmap_free(bitmap); + missed = IS_ERR_VALUE(addr) ? len : copy_from_user(actual, (void __user *)addr, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(actual, 0, SZ_4K - sizeof(u64)), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(actual + SZ_4K - sizeof(u64), 0xff, bytes - 1), NULL); + KUNIT_EXPECT_EQ(test, actual[SZ_4K - sizeof(u64) + bytes - 1], (unsigned char)1); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(actual + SZ_4K - sizeof(u64) + bytes, + 0, SZ_4K + sizeof(u64)), NULL); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOVA bitmap crossed %lu-record window and revisited first byte", capacity); +} + +static void user4k_iova_bitmap_retained_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bitmap requires hardware opt-in"); + return; + } + for (mode = 0; mode < (PAGE_SHIFT > 14 ? 3U : 2U); mode++) { + struct mm_struct *mm = test_vma_mm_granule(test, shifts[mode]); + unsigned long size = 1UL << shifts[mode], len = 2 * size; + struct file *file = test_cache_file(test, 4 * PAGE_SIZE, false); + unsigned char *buffer = kunit_kzalloc(test, len, GFP_KERNEL); + unsigned long addr, replacement = 0, missed = 0; + struct iova_bitmap *bitmap; + struct folio *folio; + loff_t pos = size; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_NOT_NULL(test, buffer); + kthread_use_mm(mm); + addr = vm_mmap(file, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, size); + bitmap = iova_bitmap_alloc(0, len * 8 * SZ_4K, SZ_4K, + (u64 __user *)TEST_VA); + if (!IS_ERR(bitmap)) + iova_bitmap_set(bitmap, 3 * SZ_4K, SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bitmap); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_iova_free, bitmap), 0); + folio = filemap_get_folio(file->f_mapping, size >> PAGE_SHIFT); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_ASSERT_TRUE(test, folio_maybe_dma_pinned(folio)); + kthread_use_mm(mm); + ret = vm_munmap(TEST_VA, len); + if (!ret) { + replacement = vm_mmap(NULL, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + if (!IS_ERR_VALUE(replacement)) { + memset(buffer, 0x5a, len); + missed = copy_to_user((void __user *)replacement, buffer, len); + } + } + /* The cached pins still refer to the old file, not the new VMA. */ + iova_bitmap_set(bitmap, 9 * SZ_4K, 2 * SZ_4K); + if (!ret && !IS_ERR_VALUE(replacement)) + missed |= copy_from_user(buffer, (void __user *)replacement, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, replacement, TEST_VA); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0x5a, len), NULL); + kunit_release_action(test, test_iova_free, bitmap); + KUNIT_EXPECT_FALSE(test, folio_maybe_dma_pinned(folio)); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + KUNIT_ASSERT_EQ(test, kernel_read(file, buffer, len, &pos), (ssize_t)len); + KUNIT_EXPECT_EQ(test, buffer[0], (unsigned char)8); + KUNIT_EXPECT_EQ(test, buffer[1], (unsigned char)6); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 2, 0, len - 2), NULL); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOVA bitmap %luK shared backing retained across unmap/replacement", size / SZ_1K); + } +} + +static long test_mprotect(unsigned long start, unsigned long len, unsigned long prot); + +static void user4k_iova_bitmap_partial_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bitmap requires hardware opt-in"); + return; + } + for (mode = 0; mode < (PAGE_SHIFT > 14 ? 3U : 2U); mode++) { + struct mm_struct *mm = test_vma_mm_granule(test, shifts[mode]); + unsigned long size = 1UL << shifts[mode], len = 3 * size; + unsigned char *buffer = kunit_kzalloc(test, len, GFP_KERNEL); + unsigned long addr, refill, missed; + struct iova_bitmap *bitmap; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, buffer); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + missed = IS_ERR_VALUE(addr) ? len : copy_to_user((void __user *)addr, buffer, len); + ret = vm_munmap(TEST_VA + size, size); + bitmap = iova_bitmap_alloc(0, len * 8 * SZ_4K, SZ_4K, (u64 __user *)TEST_VA); + if (!IS_ERR(bitmap)) + iova_bitmap_set(bitmap, 0, len * 8 * SZ_4K); + missed |= copy_from_user(buffer, (void __user *)TEST_VA, size); + missed |= copy_from_user(buffer + 2 * size, (void __user *)(TEST_VA + 2 * size), size); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bitmap); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_iova_free, bitmap), 0); + /* Preserve the existing void API: pin failure stops at the valid prefix. */ + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0xff, size), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 2 * size, 0, size), NULL); + kthread_use_mm(mm); + refill = vm_mmap(NULL, TEST_VA + size, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + iova_bitmap_set(bitmap, 0, len * 8 * SZ_4K); + missed = copy_from_user(buffer, (void __user *)TEST_VA, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, refill, TEST_VA + size); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0xff, len), NULL); + kunit_release_action(test, test_iova_free, bitmap); + memset(buffer, 0, len); + kthread_use_mm(mm); + missed = copy_to_user((void __user *)TEST_VA, buffer, len); + ret = test_mprotect(TEST_VA, len, PROT_READ); + bitmap = iova_bitmap_alloc(0, len * 8 * SZ_4K, SZ_4K, (u64 __user *)TEST_VA); + if (!IS_ERR(bitmap)) { + iova_bitmap_set(bitmap, 0, len * 8 * SZ_4K); + iova_bitmap_free(bitmap); + } + missed |= copy_from_user(buffer, (void __user *)TEST_VA, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bitmap); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0, len), NULL); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOVA bitmap %luK partial pin, retry and read-only checks passed", size / SZ_1K); + } +} + +static u64 test_table_descriptor(struct mm_struct *mm, unsigned long addr, bool pud) +{ + pgd_t *pgdp = pgd_offset(mm, addr); + p4d_t *p4dp = p4d_offset_mm(mm, pgdp, addr); + pud_t *pudp = pud_offset_mm(mm, p4dp, addr); + + return pud ? pud_val(*pudp) : pmd_val(*pmd_offset_mm(mm, pudp, addr)); +} + +static void user4k_mremap_move_test(struct kunit *test) +{ + static const struct { unsigned long old, new, len; unsigned int level; } cases[] = { + { TEST_VA + SZ_4K, 3 * SZ_2M + 2 * SZ_4K, 3 * SZ_4K, 0 }, + { SZ_2M - SZ_4K, SZ_1G - 2 * SZ_4K, 3 * SZ_4K, 0 }, + { SZ_1G - SZ_4K, BIT_ULL(39) - 2 * SZ_4K, 3 * SZ_4K, 0 }, + { 4 * SZ_2M, 8 * SZ_2M, SZ_2M, 1 }, + { 2UL * SZ_1G, 4UL * SZ_1G, SZ_1G, 2 }, + }; + unsigned int shape; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + for (shape = 0; shape < ARRAY_SIZE(cases); shape++) { + struct mm_struct *mm = test_vma_mm(test); + unsigned long old = cases[shape].old, new = cases[shape].new; + unsigned long len = cases[shape].len; + unsigned long offsets[] = { 0, SZ_4K, len - SZ_4K }; + unsigned long result, warm, moved, vm_before, commit_before, rss_before; + unsigned long flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE; + unsigned long old_left[3], new_left[3]; + phys_addr_t phys[3]; + u8 readback[3], value = 0x79, byte; + u64 table = 0; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, mm); + if (cases[shape].level) + flags |= MAP_NORESERVE; + kthread_use_mm(mm); + warm = vm_mmap(NULL, SZ_1M, SZ_4K, PROT_READ | PROT_WRITE, flags, 0); + result = vm_mmap(NULL, old, len, PROT_READ | PROT_WRITE, flags, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, warm, SZ_1M); + KUNIT_ASSERT_EQ(test, result, old); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, warm, &value, 1, FOLL_WRITE), 1); + for (i = 0; i < 3; i++) { + value = 0x80 + i; + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old + offsets[i], &value, 1, FOLL_WRITE), 1); + phys[i] = pte_phys_mm(mm, test_fault_entry(test, mm, old + offsets[i])); + } + if (cases[shape].level) { + table = test_table_descriptor(mm, old, cases[shape].level == 2); + /* Warm-up consumed the first fragment; the moved table is packed. */ + KUNIT_EXPECT_NE(test, table & (3UL * SZ_4K), 0ULL); + } + vm_before = mm->total_vm; + commit_before = mm->committed_user_pages; + rss_before = get_mm_rss_sum(mm); + kthread_use_mm(mm); + pagefault_disable(); + for (i = 0; i < 3; i++) + old_left[i] = copy_from_user(&byte, (void __user *)(old + offsets[i]), 1); + pagefault_enable(); + moved = test_mremap(old, len, len, MREMAP_FIXED | MREMAP_MAYMOVE, new); + pagefault_disable(); + for (i = 0; i < 3; i++) { + new_left[i] = copy_from_user(&readback[i], (void __user *)(new + offsets[i]), 1); + old_left[i] += !copy_from_user(&byte, (void __user *)(old + offsets[i]), 1); + } + pagefault_enable(); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, moved, new); + KUNIT_EXPECT_EQ(test, mm->total_vm, vm_before); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, commit_before); + KUNIT_EXPECT_EQ(test, get_mm_rss_sum(mm), rss_before); + for (i = 0; i < 3; i++) { + KUNIT_EXPECT_EQ(test, old_left[i], 0UL); + KUNIT_EXPECT_EQ(test, new_left[i], 0UL); + KUNIT_EXPECT_EQ(test, readback[i], (u8)(0x80 + i)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, new + offsets[i])), phys[i]); + } + if (cases[shape].level) + KUNIT_EXPECT_EQ(test, test_table_descriptor(mm, new, cases[shape].level == 2), table); + } +} + +static void user4k_stack_relocate_test(struct kunit *test) +{ + const unsigned long shifts[] = { SZ_4K, SZ_2M + SZ_4K, SZ_1G + 3 * SZ_4K }; + unsigned int shape, i; + + for (shape = 0; shape < ARRAY_SIZE(shifts); shape++) { + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + struct vm_page_offset pos; + unsigned long top, old, new; + phys_addr_t phys[3]; + u8 bytes[3] = { 0x42, 0x73, 0xb5 }; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, create_init_stack_vma(mm, &vma, &top), 0); + old = vma->vm_end - 3 * SZ_4K; + new = old - shifts[shape]; + mmap_write_lock(mm); + ret = expand_stack_locked(vma, old); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + for (i = 0; i < 3; i++) { + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old + i * SZ_4K, + &bytes[i], 1, FOLL_WRITE), 1); + phys[i] = pte_phys_mm(mm, test_fault_entry(test, mm, old + i * SZ_4K)); + } + pos = vma_page_offset_at(vma, old); + mmap_write_lock(mm); + ret = relocate_vma_down(vma, shifts[shape]); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, vma->vm_start, new); + KUNIT_EXPECT_EQ(test, vma->vm_end, new + 3 * SZ_4K); + KUNIT_EXPECT_TRUE(test, vm_page_offset_equal(vma_page_offset_at(vma, new), pos)); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + for (i = 0; i < 3; i++) { + u8 byte = 0; + + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, new + i * SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, bytes[i]); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, new + i * SZ_4K)), phys[i]); + } + /* Grow after relocation: keep the file-like anon offset linear. */ + mmap_write_lock(mm); + vm_flags_clear(vma, VM_STACK_INCOMPLETE_SETUP); + ret = expand_stack_locked(vma, new - SZ_4K); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 4UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 4UL); + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, new - SZ_4K, &bytes[0], 1, FOLL_WRITE), 1); + KUNIT_EXPECT_TRUE(test, vm_page_offset_equal(vma_page_offset_at(vma, new), pos)); + } +} + +static void user4k_mremap_cow_test(struct kunit *test) +{ + struct mm_struct *parent, *child; + unsigned long old = TEST_VA + SZ_4K, new = SZ_2M - 2 * SZ_4K, mapped; + unsigned long moved; + u8 value = 0x29, byte; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + parent = test_vma_mm(test); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, parent); + KUNIT_ASSERT_NOT_NULL(test, child); + kthread_use_mm(parent); + mapped = vm_mmap(NULL, old, 3 * SZ_4K, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(parent); + KUNIT_ASSERT_EQ(test, mapped, old); + KUNIT_ASSERT_EQ(test, access_remote_vm(parent, old + SZ_4K, &value, 1, FOLL_WRITE), 1); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + kthread_use_mm(child); + /* Move only the middle quarter, leaving both neighbors mapped. */ + moved = test_mremap(old + SZ_4K, SZ_4K, SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE, new); + kthread_unuse_mm(child); + KUNIT_ASSERT_EQ(test, moved, new); + value = 0xa4; + KUNIT_EXPECT_EQ(test, access_remote_vm(child, new, &value, 1, FOLL_WRITE), 1); + KUNIT_EXPECT_EQ(test, access_remote_vm(parent, old + SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)0x29); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, new, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, value); + KUNIT_EXPECT_EQ(test, mm_pages_to_bytes(child, child->total_vm), 3UL * SZ_4K); + KUNIT_EXPECT_EQ(test, child->committed_user_pages, 3UL); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, old, &value, 1, 0), 1); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, old + 2 * SZ_4K, &value, 1, 0), 1); +} + +static void user4k_mremap_resize_test(struct kunit *test) +{ + struct mm_struct *mm; + unsigned long old = TEST_VA + SZ_4K, new = TEST_VA + 8 * PAGE_SIZE + 2 * SZ_4K; + unsigned long res[7]; + u8 value = 0x98, byte = 0; + int unmap[2]; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + res[0] = vm_mmap(NULL, old, SZ_4K, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + res[1] = test_mremap(old, SZ_4K, 3 * SZ_4K - 1, 0, 0); + res[2] = test_mremap(old + 1, SZ_4K, SZ_4K, 0, 0); + res[3] = test_mremap(old, 3 * SZ_4K, 3 * SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE, new + 1); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, res[0], old); + KUNIT_ASSERT_EQ(test, res[1], old); + KUNIT_EXPECT_EQ(test, res[2], (unsigned long)-EINVAL); + KUNIT_EXPECT_EQ(test, res[3], (unsigned long)-EINVAL); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old + SZ_4K, &value, 1, FOLL_WRITE), 1); + kthread_use_mm(mm); + res[4] = test_mremap(old, 3 * SZ_4K, 2 * SZ_4K, 0, 0); + res[5] = test_mremap(old, 2 * SZ_4K, 2 * SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE | MREMAP_DONTUNMAP, new); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, res[4], old); + KUNIT_ASSERT_EQ(test, res[5], new); + KUNIT_EXPECT_EQ(test, mm->total_vm, 4UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 4UL); + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, new + SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, value); + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, old + SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)0); + kthread_use_mm(mm); + unmap[0] = vm_munmap(old, 2 * SZ_4K); + unmap[1] = vm_munmap(new, 2 * SZ_4K); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, unmap[0], 0); + KUNIT_EXPECT_EQ(test, unmap[1], 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 0UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 0UL); +} + +static int test_reject_mremap(struct vm_area_struct *vma) +{ + return -EIO; +} + +static void user4k_cold_batch_test(struct kunit *test) +{ + struct mm_struct *mm, *child; + struct vm_area_struct *vma; + unsigned long start = TEST_VA, mapped; + spinlock_t *ptl; + pte_t *ptep, entry; + u8 byte = 0x68; + unsigned int i; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm cold fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, child); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, start, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, start, &byte, 1, FOLL_WRITE), 1); + /* Only the last quarter is young: batching must aggregate that bit. */ + mmap_read_lock(mm); + vma = vma_lookup(mm, start); + for (i = 0; i < 3; i++) { + unsigned long addr = start + i * SZ_4K; + + ptep = test_mm_lookup(mm, addr, &ptl); + if (!ptep) { + mmap_read_unlock(mm); + KUNIT_FAIL(test, "missing cold test PTE"); + return; + } + entry = ptep_clear_flush(vma, addr, ptep); + set_pte_at(mm, addr, ptep, pte_mkold(entry)); + pte_unmap_unlock(ptep, ptl); + } + mmap_read_unlock(mm); + kthread_use_mm(mm); + ret = do_madvise(mm, start + 3 * SZ_4K, SZ_4K, MADV_COLD); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, pte_young(test_fault_entry(test, mm, start + 3 * SZ_4K))); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + kthread_use_mm(mm); + ret = do_madvise(mm, start, PAGE_SIZE, MADV_COLD); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, pte_young(test_fault_entry(test, mm, start + 3 * SZ_4K))); + kunit_release_action(test, user4k_vma_mm_free, child); + kthread_use_mm(mm); + ret = do_madvise(mm, start, PAGE_SIZE, MADV_COLD); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 0); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_FALSE(test, pte_young(test_fault_entry(test, mm, start + i * SZ_4K))); + byte = 0; + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, start, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)0x68); +} + +static void user4k_prefault_dirty_test(struct kunit *test) +{ + struct mm_struct *mm; + struct folio *folio; + unsigned long start = TEST_VA + SZ_4K, mapped; + spinlock_t *ptl; + pte_t *ptep, entry; + u8 byte = 0x7a; + int locked = 1; + long ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm prefault fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, start, SZ_4K, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, start, &byte, 1, FOLL_WRITE), 1); + mmap_read_lock(mm); + ptep = test_mm_lookup(mm, start, &ptl); + if (!ptep) { + mmap_read_unlock(mm); + KUNIT_FAIL(test, "missing prefault test PTE"); + return; + } + entry = ptep_clear_flush(vma_lookup(mm, start), start, ptep); + folio = page_folio(pte_page(entry)); + folio_get(folio); + set_pte_at(mm, start, ptep, pte_mkclean(entry)); + folio_clear_dirty(folio); + pte_unmap_unlock(ptep, ptl); + mmap_read_unlock(mm); + kthread_use_mm(mm); + mmap_read_lock(mm); + ret = faultin_page_range(mm, start, start + SZ_4K, true, &locked); + if (locked) + mmap_read_unlock(mm); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 1L); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + folio_put(folio); +} + +struct test_fsync_record { + loff_t start, end; + unsigned int calls; + int datasync; +}; + +static int test_fsync_range(struct file *file, loff_t start, loff_t end, int datasync) +{ + struct test_fsync_record *record = file->private_data; + + *record = (struct test_fsync_record) { start, end, record->calls + 1, datasync }; + return 0; +} + +asmlinkage long __arm64_sys_msync(const struct pt_regs *regs); + +static void user4k_msync_offsets_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *vma; + struct file_operations ops; + const struct file_operations *original_ops; + void *original_private; + struct test_fsync_record records[3] = {}, record = {}; + struct pt_regs regs = { .regs = { TEST_VA + SZ_4K, 1, MS_SYNC } }; + long result[3]; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm sync fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 3 * PAGE_SIZE, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + 5 * SZ_4K, 3 * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + original_ops = file->f_op; + original_private = file->private_data; + ops = *original_ops; + ops.fsync = test_fsync_range; + file->f_op = &ops; + file->private_data = &record; + kthread_use_mm(vma->vm_mm); + for (i = 0; i < ARRAY_SIZE(result); i++) { + regs.regs[0] = vma->vm_start + i * SZ_4K; + result[i] = __arm64_sys_msync(®s); + records[i] = record; + } + kthread_unuse_mm(vma->vm_mm); + file->f_op = original_ops; + file->private_data = original_private; + for (i = 0; i < ARRAY_SIZE(result); i++) { + KUNIT_EXPECT_EQ(test, result[i], 0L); + KUNIT_EXPECT_EQ(test, records[i].calls, i + 1); + KUNIT_EXPECT_EQ(test, records[i].start, (loff_t)(3 + i) * SZ_4K); + KUNIT_EXPECT_EQ(test, records[i].end, (loff_t)(4 + i) * SZ_4K - 1); + KUNIT_EXPECT_EQ(test, records[i].datasync, 1); + } +} + +static void user4k_mremap_file_test(struct kunit *test) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + struct file *file; + struct vm_operations_struct rejecting_ops; + const struct vm_operations_struct *original_ops; + unsigned long old = TEST_VA + SZ_4K, new = SZ_1G - SZ_4K; + unsigned long mapped, moved; + u8 bytes[3]; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 2 * PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped = vm_mmap(file, old, 3 * SZ_4K, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, 3 * SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, old); + for (i = 0; i < 3; i++) + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old + i * SZ_4K, &bytes[i], 1, 0), 1); + vma = test_vma_lookup(mm, old); + KUNIT_ASSERT_NOT_NULL(test, vma); + original_ops = vma->vm_ops; + rejecting_ops = *original_ops; + rejecting_ops.mremap = test_reject_mremap; + mmap_write_lock(mm); + vma->vm_ops = &rejecting_ops; + mmap_write_unlock(mm); + kthread_use_mm(mm); + moved = test_mremap(old, 3 * SZ_4K, 3 * SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE, new); + kthread_unuse_mm(mm); + /* Restore stack-owned callbacks before any assertion can leave this scope. */ + mmap_write_lock(mm); + { + VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *part; + + for_each_vma(vmi, part) + if (part->vm_ops == &rejecting_ops) + part->vm_ops = original_ops; + } + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, moved, (unsigned long)-EIO); + for (i = 0; i < 3; i++) { + u8 byte = 0; + + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, old + i * SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, bytes[i]); + } + KUNIT_EXPECT_EQ(test, mm->map_count, 1); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + kthread_use_mm(mm); + moved = test_mremap(old, 3 * SZ_4K, 3 * SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE, new); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, moved, new); + vma = test_vma_lookup(mm, new); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), 3U * SZ_4K); + for (i = 0; i < 3; i++) { + u8 byte = 0; + + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, new + i * SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, bytes[i]); + } +} + +static void user4k_exec_arguments_test(struct kunit *test) +{ + unsigned int native; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm stack setup requires hardware opt-in"); + return; + } + for (native = 0; native < 2; native++) { + struct linux_binprm *bprm = kunit_kzalloc(test, sizeof(*bprm), GFP_KERNEL); + struct mm_struct *mm; + unsigned long size, mask, initial, copied, new_top; + unsigned long a_len = 2 * PAGE_SIZE + SZ_4K + 31, b_len = SZ_4K + 43; + char *a, *b, *buf, *too_long; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, bprm); + mm = native ? mm_alloc() : test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + if (native) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + size = mm_page_size(mm); + mask = mm_page_mask(mm); + a_len = min(a_len, 16 * size + 31); + bprm->mm = mm; + KUNIT_ASSERT_EQ(test, create_init_stack_vma(mm, &bprm->vma, &bprm->p), 0); + initial = bprm->p; + bprm->argmin = initial - SZ_2M; + bprm->rlim_stack.rlim_cur = SZ_8M; + bprm->rlim_stack.rlim_max = RLIM_INFINITY; + a = kunit_kmalloc(test, a_len + 1, GFP_KERNEL); + b = kunit_kmalloc(test, b_len + 1, GFP_KERNEL); + buf = kunit_kmalloc(test, a_len + b_len + 7, GFP_KERNEL); + too_long = kunit_kmalloc(test, 32 * size + 1, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, buf); + KUNIT_ASSERT_NOT_NULL(test, too_long); + memset(a, 'A', a_len); + a[a_len] = 0; + memset(b, 'b', b_len); + b[b_len] = 0; + memset(too_long, 'x', 32 * size); + too_long[32 * size] = 0; + KUNIT_EXPECT_EQ(test, copy_string_kernel(too_long, bprm), -E2BIG); + KUNIT_EXPECT_EQ(test, bprm->p, initial); + KUNIT_ASSERT_EQ(test, copy_string_kernel("tail", bprm), 0); + bprm->exec = bprm->p; + KUNIT_ASSERT_EQ(test, copy_string_kernel(a, bprm), 0); + KUNIT_ASSERT_EQ(test, copy_string_kernel(b, bprm), 0); + bprm->argc = 3; + copied = initial - bprm->p; + KUNIT_EXPECT_EQ(test, copied, a_len + b_len + 7); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, bprm->p, buf, copied, 0), (int)copied); + KUNIT_EXPECT_MEMEQ(test, buf, b, b_len + 1); + KUNIT_EXPECT_MEMEQ(test, buf + b_len + 1, a, a_len + 1); + KUNIT_EXPECT_STREQ(test, buf + b_len + a_len + 2, "tail"); + new_top = (bprm->vma->vm_end - SZ_2M - size) & mask; + kthread_use_mm(mm); + ret = setup_arg_pages(bprm, new_top, EXSTACK_DISABLE_X); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + /* Subtract < PAGE_SIZE, then round down to 16 bytes and up to a leaf. */ + KUNIT_EXPECT_LE(test, bprm->vma->vm_end, new_top); + KUNIT_EXPECT_LE(test, new_top - bprm->vma->vm_end, PAGE_SIZE); + KUNIT_EXPECT_EQ(test, bprm->vma->vm_end & ~mask, 0UL); + KUNIT_EXPECT_EQ(test, mm->start_stack, bprm->p); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, bprm->p, buf, copied, 0), (int)copied); + KUNIT_EXPECT_MEMEQ(test, buf, b, b_len + 1); + KUNIT_EXPECT_MEMEQ(test, buf + b_len + 1, a, a_len + 1); + KUNIT_EXPECT_STREQ(test, buf + b_len + a_len + 2, "tail"); + /* /proc/PID/cmdline and environ read the stack with FOLL_ANON. */ + memset(buf, 0, copied); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, bprm->p, buf, copied, FOLL_ANON), + (int)copied); + KUNIT_EXPECT_MEMEQ(test, buf, b, b_len + 1); + KUNIT_EXPECT_STREQ(test, buf + b_len + a_len + 2, "tail"); + /* Interpreter argument removal scans several user granules. */ + initial = bprm->p; + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + 1); + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + a_len + 2); + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + copied); + KUNIT_EXPECT_EQ(test, bprm->argc, 0); + KUNIT_EXPECT_EQ(test, bprm->vma_pages, 0UL); + } +} + +/* The transfer can replace bprm->mm, so release its final owner. */ +static void test_exec_mm_free(void *data) +{ + struct linux_binprm *bprm = data; + + mmput(bprm->mm); +} + +static void user64k_exec_arguments_test(struct kunit *test) +{ + unsigned int shifts[] = { 16, 12, PAGE_SHIFT }; + unsigned int mode; + + if (PAGE_SHIFT >= 16 || + (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN))) { + kunit_skip(test, "requires internal larger user granule and MM opt-in"); + return; + } + for (mode = 0; mode < ARRAY_SIZE(shifts); mode++) { + struct linux_binprm *bprm = kunit_kzalloc(test, sizeof(*bprm), GFP_KERNEL); + unsigned long a_len = mode ? SZ_64K + SZ_16K + 31 : 32 * PAGE_SIZE + 31; + unsigned long b_len = PAGE_SIZE + 17, initial, copied; + char *a, *b, *buf; + unsigned long i; + + KUNIT_ASSERT_NOT_NULL(test, bprm); + bprm->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, bprm->mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_exec_mm_free, bprm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(bprm->mm, 16), 0); + KUNIT_ASSERT_EQ(test, create_init_stack_vma(bprm->mm, &bprm->vma, &bprm->p), 0); + initial = bprm->p; + bprm->argmin = initial - SZ_2M; + bprm->rlim_stack.rlim_cur = SZ_8M; + bprm->rlim_stack.rlim_max = RLIM_INFINITY; + a = kunit_kmalloc(test, a_len + 1, GFP_KERNEL); + b = kunit_kmalloc(test, b_len + 1, GFP_KERNEL); + buf = kunit_kmalloc(test, a_len + b_len + 7, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, buf); + for (i = 0; i < a_len; i++) + a[i] = 'A' + i % 23; + a[a_len] = 0; + memset(b, 'b', b_len); + b[b_len] = 0; + KUNIT_ASSERT_EQ(test, copy_string_kernel("tail", bprm), 0); + KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, 5, 0), 5); + KUNIT_ASSERT_STREQ(test, buf, "tail"); + KUNIT_ASSERT_EQ(test, copy_string_kernel(a, bprm), 0); + KUNIT_ASSERT_EQ(test, copy_string_kernel(b, bprm), 0); + bprm->argc = 3; + copied = initial - bprm->p; + KUNIT_ASSERT_EQ(test, copied, a_len + b_len + 7); + KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, copied, 0), + (int)copied); + KUNIT_ASSERT_EQ(test, memcmp(buf, b, b_len + 1), 0); + KUNIT_ASSERT_EQ(test, memcmp(buf + b_len + 1, a, a_len + 1), 0); + KUNIT_ASSERT_STREQ(test, buf + b_len + a_len + 2, "tail"); + /* Exercise copying out of every native page within a coarse leaf. */ + if (shifts[mode] != 16) { + KUNIT_ASSERT_EQ(test, bprm_set_page_shift(bprm, shifts[mode]), 0); + KUNIT_ASSERT_EQ(test, mm_page_shift(bprm->mm), shifts[mode]); + memset(buf, 0, copied); + KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, copied, 0), + (int)copied); + KUNIT_ASSERT_EQ(test, memcmp(buf, b, b_len + 1), 0); + KUNIT_ASSERT_EQ(test, memcmp(buf + b_len + 1, a, a_len + 1), 0); + KUNIT_ASSERT_STREQ(test, buf + b_len + a_len + 2, "tail"); + } + initial = bprm->p; + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + 1); + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + a_len + 2); + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + copied); + KUNIT_EXPECT_EQ(test, bprm->argc, 0); + KUNIT_EXPECT_EQ(test, bprm->vma_pages, 0UL); + } +} + +static struct vm_area_struct *test_special_vma(struct mm_struct *mm, + const struct vm_special_mapping *sm, unsigned long start, unsigned long len, + vm_flags_t flags) +{ + struct vm_area_struct *vma; + + mmap_write_lock(mm); + vma = _install_special_mapping(mm, start, len, flags, sm); + mmap_write_unlock(mm); + return vma; +} + +static void user4k_special_mapping_test(struct kunit *test) +{ + struct vm_special_mapping *sm = kunit_kzalloc(test, sizeof(*sm), GFP_KERNEL); + struct page **pages = kunit_kcalloc(test, 3, sizeof(*pages), GFP_KERNEL); + void *data[2] = { test_alloc_page(test), test_alloc_page(test) }; + struct mm_struct *small = test_vma_mm(test), *cold = test_vma_mm(test); + struct mm_struct *native = mm_alloc(); + struct vm_area_struct *vma[3]; + unsigned long start = SZ_2M - SZ_4K; + unsigned int i, side; + u8 byte; + + KUNIT_ASSERT_NOT_NULL(test, sm); + KUNIT_ASSERT_NOT_NULL(test, pages); + KUNIT_ASSERT_NOT_NULL(test, data[0]); + KUNIT_ASSERT_NOT_NULL(test, data[1]); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, cold); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, native), 0); + /* Reverse the supplied native pages: folio indices cannot identify them. */ + pages[0] = virt_to_page(data[1]); + pages[1] = virt_to_page(data[0]); + sm->pages = pages; + sm->name = "[user4k-special-test]"; + for (i = 0; i < 2 * TEST_SLOTS; i++) + memset(page_address(pages[i / TEST_SLOTS]) + (i % TEST_SLOTS) * SZ_4K, 0x41 + i, SZ_4K); + vma[0] = test_special_vma(small, sm, start, 2 * PAGE_SIZE, VM_READ | VM_MAYREAD | VM_MAYWRITE); + vma[1] = test_special_vma(cold, sm, start, 2 * PAGE_SIZE, VM_READ | VM_MAYREAD | VM_MAYWRITE); + vma[2] = test_special_vma(native, sm, TEST_VA, 2 * PAGE_SIZE, VM_READ | VM_MAYREAD | VM_MAYWRITE); + for (i = 0; i < 3; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma[i]); + for (i = 0; i < 2 * TEST_SLOTS; i++) { + unsigned long addr = start + i * SZ_4K; + + KUNIT_ASSERT_EQ(test, access_remote_vm(small, addr, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)(0x41 + i)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small, test_fault_entry(test, small, addr)), + page_to_phys(pages[i / TEST_SLOTS]) + (i % TEST_SLOTS) * SZ_4K); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(small, MM_FILEPAGES), 2UL * TEST_SLOTS); + /* Both present-PTE and cold-PTE forced writes copy only their quarter. */ + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? cold : small; + + byte = 0xe1 + side; + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, start + 2 * SZ_4K + 7, + &byte, 1, FOLL_WRITE | FOLL_FORCE), 1); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1UL); + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, start + 2 * SZ_4K + 7, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)(0xe1 + side)); + KUNIT_EXPECT_EQ(test, access_remote_vm(native, TEST_VA + 2 * SZ_4K + 7, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)0x43); + } + { + struct user_page_fragment fragment; + long ret = pin_user_fragments_remote(small, start + 5 * SZ_4K + 13, + 7, 0, &fragment, 1); + + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_EXPECT_NOT_NULL(test, fragment.subpage); + KUNIT_EXPECT_EQ(test, fragment.offset, (5 * SZ_4K + 13UL) & ~PAGE_MASK); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(fragment.folio) + fragment.offset, + 0x46, fragment.length), NULL); + release_user_fragments(&fragment, 1, false); + } +} + +static void user4k_insert_range_test(struct kunit *test) +{ + unsigned int native; + + for (native = 0; native < 2; native++) { + void *data[2] = { test_alloc_page(test), test_alloc_page(test) }; + struct page *pages[2]; + struct mm_struct *mm = native ? mm_alloc() : test_vma_mm(test); + struct vm_area_struct *vma; + unsigned long size, start, offset, length, num; + unsigned int i; + int ret, invalid[4]; + + KUNIT_ASSERT_NOT_NULL(test, data[0]); + KUNIT_ASSERT_NOT_NULL(test, data[1]); + KUNIT_ASSERT_NOT_NULL(test, mm); + if (native) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + pages[0] = virt_to_page(data[1]); + pages[1] = virt_to_page(data[0]); + size = mm_page_size(mm); + start = pmd_size_mm(mm) - size; + offset = native ? 0 : 3 * SZ_4K; + length = 2 * PAGE_SIZE - offset; + vma = test_vma_add(mm, start, length, (struct vm_page_offset) {}, + VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_write_lock(mm); + invalid[0] = vm_insert_pages_range(vma, start + 1, pages, 2, offset, length); + invalid[1] = vm_insert_pages_range(vma, start, pages, 2, offset + 1, length); + invalid[2] = vm_insert_pages_range(vma, start, pages, 2, offset, length + size); + invalid[3] = vm_insert_pages_range(vma, start, pages, ULONG_MAX, 0, length); + ret = vm_insert_pages_range(vma, start, pages, 2, offset, length); + mmap_write_unlock(mm); + for (i = 0; i < ARRAY_SIZE(invalid); i++) + KUNIT_EXPECT_EQ(test, invalid[i], -EINVAL); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < length / size; i++) { + unsigned long source = offset + i * size; + + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, + test_fault_entry(test, mm, start + i * size)), + page_to_phys(pages[source >> PAGE_SHIFT]) + offset_in_page(source)); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), length / size); + /* The original native-page-count API also expands all user leaves. */ + start += 4 * PAGE_SIZE; + vma = test_vma_add(mm, start, 2 * PAGE_SIZE, (struct vm_page_offset) {}, + VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + num = 2; + mmap_write_lock(mm); + ret = vm_insert_pages(vma, start, pages, &num); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, num, 0UL); + for (i = 0; i < 2 * PAGE_SIZE / size; i++) + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, + test_fault_entry(test, mm, start + i * size)), + page_to_phys(pages[(i * size) >> PAGE_SHIFT]) + offset_in_page(i * size)); + num = 2; + mmap_write_lock(mm); + ret = vm_insert_pages(vma, start, pages, &num); + mmap_write_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, -EBUSY); + KUNIT_EXPECT_EQ(test, num, 2UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), + (length + 2 * PAGE_SIZE) / size); + start += 4 * PAGE_SIZE; + offset = native ? PAGE_SIZE : 3 * SZ_4K; + length = 2 * PAGE_SIZE - offset; + vma = test_vma_add(mm, start, length, + (struct vm_page_offset) { .index = offset >> PAGE_SHIFT, + .offset = offset_in_page(offset) }, + VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_write_lock(mm); + ret = vm_map_pages(vma, pages, 2); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < length / size; i++) { + unsigned long source = offset + i * size; + + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, + test_fault_entry(test, mm, start + i * size)), + page_to_phys(pages[source >> PAGE_SHIFT]) + offset_in_page(source)); + } + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, page_ref_count(pages[0]), 1); + KUNIT_EXPECT_EQ(test, page_ref_count(pages[1]), 1); + } +} + +static void user4k_remap_pfn_range_test(struct kunit *test) +{ + struct folio *folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 1); + + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + for (unsigned int native = 0; native < 2; native++) { + struct mm_struct *mm = native ? mm_alloc() : test_vma_mm(test); + unsigned long size, boundaries[3]; + + KUNIT_ASSERT_NOT_NULL(test, mm); + if (native) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + user4k_vma_mm_free, mm), 0); + size = mm_page_size(mm); + boundaries[0] = pmd_size_mm(mm); + boundaries[1] = pud_size_mm(mm); + boundaries[2] = pgd_size_mm(mm); + for (unsigned int shape = 0; shape < ARRAY_SIZE(boundaries); shape++) { + unsigned long start = boundaries[shape] - size; + unsigned long length = PAGE_SIZE + size; + struct vm_area_struct *vma; + int ret; + + if (shape && boundaries[shape] == boundaries[shape - 1]) + continue; + vma = test_vma_add(mm, start, length, (struct vm_page_offset) {}, + VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | + VM_SHARED | VM_MAYSHARE); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_write_lock(mm); + ret = remap_pfn_range(vma, start, folio_pfn(folio), length, + vma->vm_page_prot); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + for (unsigned long offset = 0; offset < length; offset += size) { + pte_t pte = test_fault_entry(test, mm, start + offset); + + struct follow_pfnmap_args args = { + .vma = vma, .address = start + offset + 31, + }; + phys_addr_t expected = PFN_PHYS(folio_pfn(folio)) + offset + 31; + int ret; + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_TRUE(test, pte_special(pte)); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), + PFN_PHYS(folio_pfn(folio)) + offset); + + mmap_read_lock(mm); + ret = follow_pfnmap_start(&args); + if (!ret) + follow_pfnmap_end(&args); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, native ? 0 : -EOPNOTSUPP); + args.allow_subpage = true; + mmap_read_lock(mm); + ret = follow_pfnmap_start(&args); + if (!ret) { + KUNIT_EXPECT_EQ(test, args.pfn, (unsigned long)PHYS_PFN(expected)); + KUNIT_EXPECT_EQ(test, args.offset, (unsigned int)offset_in_page(expected)); + KUNIT_EXPECT_EQ(test, args.addr_mask, mm_page_mask(mm)); + KUNIT_EXPECT_TRUE(test, args.special); + follow_pfnmap_end(&args); + } + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + } + KUNIT_EXPECT_TRUE(test, pte_none(test_fault_entry(test, mm, start - size))); + KUNIT_EXPECT_TRUE(test, pte_none(test_fault_entry(test, mm, start + length))); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0UL); + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + } +} + +#ifdef CONFIG_HAVE_IOREMAP_PROT +static void test_iounmap(void *arg) +{ + iounmap((void __iomem *)arg); +} + +static const struct vm_operations_struct test_iomem_ops = { + .access = generic_access_phys, +}; +#endif + +static void user4k_iomem_access_test(struct kunit *test) +{ +#if defined(CONFIG_HAVE_IOREMAP_PROT) && defined(CONFIG_OF) + struct device_node *node; + struct resource resource; + struct mm_struct *mm; + struct vm_area_struct *vma; + void __iomem *uart; + unsigned long start = SZ_2M + PAGE_SIZE - SZ_4K; + u8 reference[16], buffer[32]; + vm_fault_t fault; + int ret; + + if (!of_machine_is_compatible("linux,dummy-virt")) { + kunit_skip(test, "requires disposable QEMU virt PL011 identification registers"); + return; + } + node = of_find_compatible_node(NULL, NULL, "arm,pl011"); + KUNIT_ASSERT_NOT_NULL(test, node); + ret = of_address_to_resource(node, 0, &resource); + uart = ret ? NULL : of_iomap(node, 0); + of_node_put(node); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_NOT_NULL(test, uart); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_iounmap, + (void *)uart), 0); + KUNIT_ASSERT_GE(test, resource_size(&resource), (resource_size_t)SZ_4K); + memcpy_fromio(reference, uart + 0xff0, sizeof(reference)); + KUNIT_ASSERT_NOT_NULL(test, memchr_inv(reference, 0, sizeof(reference))); + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + vma = test_vma_add(mm, start, 2 * SZ_4K, (struct vm_page_offset) {}, + VM_READ | VM_MAYREAD | VM_SHARED | VM_MAYSHARE | VM_IO | VM_PFNMAP); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_ops = &test_iomem_ops; + vma->vm_page_prot = pgprot_device(vm_get_page_prot(vma->vm_flags)); + mmap_read_lock(mm); + fault = vmf_insert_pfn_prot_mkwrite_offset(vma, start, + PHYS_PFN(resource.start), offset_in_page(resource.start), + vma->vm_page_prot, false); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)VM_FAULT_NOPAGE); + memset(buffer, 0x6d, sizeof(buffer)); + mmap_read_lock(mm); + ret = generic_access_phys(vma, start + 0xff0, buffer, sizeof(buffer), 0); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, 16); + KUNIT_EXPECT_MEMEQ(test, buffer, reference, sizeof(reference)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 16, 0x6d, 16), NULL); + memset(buffer, 0x5a, sizeof(buffer)); + ret = access_remote_vm(mm, start + 0xff0, buffer, sizeof(buffer), 0); + KUNIT_EXPECT_EQ(test, ret, 16); + KUNIT_EXPECT_MEMEQ(test, buffer, reference, sizeof(reference)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 16, 0x5a, 16), NULL); + /* Writes must fail before touching the read-only device mapping. */ + mmap_read_lock(mm); + ret = generic_access_phys(vma, start + 0xff0, buffer, 1, FOLL_WRITE); + mmap_read_unlock(mm); + KUNIT_EXPECT_LT(test, ret, 0); +#else + kunit_skip(test, "requires OF and iomem access support"); +#endif +} + +static void user4k_pfn_offset_test(struct kunit *test) +{ + void *data = test_alloc_page(test); + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + unsigned long start = SZ_2M - SZ_4K; + unsigned long pfn; + unsigned int i; + vm_fault_t result[4], invalid[3], write; + + KUNIT_ASSERT_NOT_NULL(test, data); + KUNIT_ASSERT_NOT_NULL(test, mm); + pfn = page_to_pfn(virt_to_page(data)); + vma = test_vma_add(mm, start, PAGE_SIZE, (struct vm_page_offset) {}, + VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | VM_SHARED | VM_MAYSHARE | + VM_PFNMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + /* Model a write-notify VMA: initial PTEs are read-only. */ + vma->vm_page_prot = vm_get_page_prot(VM_READ | VM_SHARED); + mmap_read_lock(mm); + for (i = 0; i < 4; i++) + result[i] = vmf_insert_pfn_prot_mkwrite_offset(vma, start + i * SZ_4K, + pfn, (3 - i) * SZ_4K, vma->vm_page_prot, false); + invalid[0] = vmf_insert_pfn_prot_mkwrite_offset(vma, start + 1, pfn, + 0, vma->vm_page_prot, false); + invalid[1] = vmf_insert_pfn_prot_mkwrite_offset(vma, start, pfn, + 1, vma->vm_page_prot, false); + invalid[2] = vmf_insert_pfn_prot_mkwrite_offset(vma, start, pfn, + PAGE_SIZE, vma->vm_page_prot, false); + write = vmf_insert_pfn_prot_mkwrite_offset(vma, start + SZ_4K, pfn, + 2 * SZ_4K, vma->vm_page_prot, true); + mmap_read_unlock(mm); + for (i = 0; i < 4; i++) { + pte_t entry = test_fault_entry(test, mm, start + i * SZ_4K); + + KUNIT_EXPECT_EQ(test, result[i], (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, entry), PFN_PHYS(pfn) + (3 - i) * SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_special(entry)); + KUNIT_EXPECT_EQ(test, !!pte_write(entry), i == 1); + } + for (i = 0; i < ARRAY_SIZE(invalid); i++) + KUNIT_EXPECT_EQ(test, invalid[i], (vm_fault_t)VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, write, (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), 0UL); + KUNIT_EXPECT_EQ(test, page_ref_count(virt_to_page(data)), 1); + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, page_ref_count(virt_to_page(data)), 1); +} + +static void user16k_insert_shared_test(struct kunit *test) +{ + struct mm_struct *middle, *small, *native; + struct vm_area_struct *vmas[3]; + struct mm_subpage_mapcounts counts; + struct page *page; + void *data; + vm_fault_t result; + unsigned int i; + vm_flags_t flags = VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO; + + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "mixed16K file insertion requires larger native backing"); + return; + } + data = test_alloc_page(test); + KUNIT_ASSERT_NOT_NULL(test, data); + page = virt_to_page(data); + middle = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, middle); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, middle), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(middle, 14), 0); + small = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, small); + native = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, native), 0); + vmas[0] = test_vma_add(middle, TEST_VA, PAGE_SIZE, (struct vm_page_offset) {}, flags); + vmas[1] = test_vma_add(small, TEST_VA + SZ_4K, SZ_4K, (struct vm_page_offset) {}, flags); + vmas[2] = test_vma_add(native, TEST_VA, PAGE_SIZE, (struct vm_page_offset) {}, flags); + for (i = 0; i < 3; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vmas[i]); + vmas[i]->vm_page_prot = vm_get_page_prot(vmas[i]->vm_flags); + } + for (i = 0; i < PAGE_SIZE / SZ_16K; i++) { + unsigned long address = TEST_VA + i * SZ_16K; + + mmap_read_lock(middle); + result = vmf_insert_page_offset(vmas[0], address, page, i * SZ_16K); + mmap_read_unlock(middle); + KUNIT_ASSERT_EQ(test, result, (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, pte_phys_mm(middle, test_fault_entry(test, middle, address)), + page_to_phys(page) + i * SZ_16K); + } + mmap_read_lock(small); + result = vmf_insert_page_offset(vmas[1], vmas[1]->vm_start, page, PAGE_SIZE - SZ_4K); + mmap_read_unlock(small); + KUNIT_ASSERT_EQ(test, result, (vm_fault_t)VM_FAULT_NOPAGE); + mmap_read_lock(native); + result = vmf_insert_page_offset(vmas[2], vmas[2]->vm_start, page, 0); + mmap_read_unlock(native); + KUNIT_ASSERT_EQ(test, result, (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(page, &counts), 5U); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, counts.slots[i], i == TEST_SLOTS - 1 ? 2U : 1U); + KUNIT_EXPECT_EQ(test, folio_mapcount(page_folio(page)), 6); + KUNIT_EXPECT_EQ(test, page_ref_count(page), 7); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(middle, MM_FILEPAGES), 4UL); + /* Real mm teardown removes whole16K leaves and preserves both aliases. */ + kunit_release_action(test, user4k_vma_mm_free, middle); + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(page, &counts), 1U); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, counts.slots[i], i == TEST_SLOTS - 1 ? 1U : 0U); + KUNIT_EXPECT_EQ(test, folio_mapcount(page_folio(page)), 2); + KUNIT_EXPECT_EQ(test, page_ref_count(page), 3); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small, test_fault_entry(test, small, vmas[1]->vm_start)), + page_to_phys(page) + PAGE_SIZE - SZ_4K); + KUNIT_EXPECT_EQ(test, pte_phys_mm(native, test_fault_entry(test, native, vmas[2]->vm_start)), + page_to_phys(page)); + kunit_release_action(test, user4k_vma_mm_free, small); + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(page, &counts), 0U); + KUNIT_EXPECT_EQ(test, folio_mapcount(page_folio(page)), 1); + kunit_release_action(test, user4k_vma_mm_free, native); + KUNIT_EXPECT_EQ(test, folio_mapcount(page_folio(page)), 0); + KUNIT_EXPECT_EQ(test, page_ref_count(page), 1); +} + +static void user4k_insert_fragment_test(struct kunit *test) +{ + void *data = test_alloc_page(test); + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + unsigned long start = TEST_VA + SZ_4K; + struct page *page; + vm_fault_t result[4], invalid[4]; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, data); + KUNIT_ASSERT_NOT_NULL(test, mm); + page = virt_to_page(data); + vma = test_vma_add(mm, start, PAGE_SIZE, (struct vm_page_offset) {}, + VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_read_lock(mm); + for (i = 0; i < 4; i++) + result[i] = vmf_insert_page_offset(vma, start + i * SZ_4K, page, (3 - i) * SZ_4K); + invalid[0] = vmf_insert_page_offset(vma, start + 1, page, 0); + invalid[1] = vmf_insert_page_offset(vma, start, page, 1); + invalid[2] = vmf_insert_page_offset(vma, start, page, PAGE_SIZE); + invalid[3] = vmf_insert_page_offset(vma, vma->vm_end, page, 0); + mmap_read_unlock(mm); + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_EQ(test, result[i], (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, invalid[i], (vm_fault_t)VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, start + i * SZ_4K)), + page_to_phys(page) + (3 - i) * SZ_4K); + } + KUNIT_EXPECT_EQ(test, page_ref_count(page), 5); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), 4UL); + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, page_ref_count(page), 1); +} + +#if defined(CONFIG_GENERIC_GETTIMEOFDAY) && defined(CONFIG_VDSO_GETRANDOM) +static void user4k_vdso_mapping_test(struct kunit *test) +{ + struct mm_struct *mm; + struct linux_binprm bprm = {}; + unsigned long code, base, left[8]; + u8 bytes[8], magic[4]; + unsigned int i; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm vDSO fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + mm->mmap_base = TEST_VA + SZ_4K; + mm_flags_clear(MMF_TOPDOWN, mm); + kthread_use_mm(mm); + ret = arch_setup_additional_pages(&bprm, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + code = (unsigned long)mm->context.vdso; + KUNIT_ASSERT_NE(test, code, 0UL); + base = code - VDSO_NR_PAGES * PAGE_SIZE; + KUNIT_EXPECT_EQ(test, base & ~PAGE_MASK, SZ_4K); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, code, magic, sizeof(magic), 0), (int)sizeof(magic)); + KUNIT_EXPECT_MEMEQ(test, magic, "\177ELF", 4); + /* VVAR forbids remote GUP; ordinary uaccess must still fault it in. */ + kthread_use_mm(mm); + for (i = 0; i < 4; i++) { + left[i] = copy_from_user(&bytes[i], (void __user *)(base + i * SZ_4K), 1); + left[4 + i] = copy_from_user(&bytes[4 + i], + (void __user *)(base + VDSO_RNG_PAGE_OFFSET * PAGE_SIZE + i * SZ_4K), 1); + } + kthread_unuse_mm(mm); + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_EQ(test, left[i], 0UL); + KUNIT_EXPECT_EQ(test, left[4 + i], 0UL); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, base + i * SZ_4K)), + page_to_phys(virt_to_page(vdso_k_time_data)) + i * SZ_4K); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, + base + VDSO_RNG_PAGE_OFFSET * PAGE_SIZE + i * SZ_4K)), + page_to_phys(virt_to_page(vdso_k_rng_data)) + i * SZ_4K); + } + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, base, magic, sizeof(magic), 0), 0); +} + +#endif + +static void user4k_remote_access_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA + SZ_4K, 3 * SZ_4K); + struct mm_struct *child = test_vma_mm(test); + struct vm_area_struct *child_vma; + unsigned long start = TEST_VA + SZ_4K; + u8 *buf = kunit_kmalloc(test, 3 * SZ_4K, GFP_KERNEL); + int ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, child); + KUNIT_ASSERT_NOT_NULL(test, buf); + memset(buf, 0x5a, 3 * SZ_4K); + KUNIT_ASSERT_EQ(test, access_remote_vm(vma->vm_mm, start, buf, 3 * SZ_4K, FOLL_WRITE), + 3 * (int)SZ_4K); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + /* Deliberately straddle quarters whose physical/virtual offsets differ. */ + memset(buf, 0x96, SZ_4K + 34); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + SZ_4K - 17, + buf, SZ_4K + 34, FOLL_WRITE), (int)SZ_4K + 34); + memset(buf, 0, 3 * SZ_4K); + KUNIT_EXPECT_EQ(test, access_remote_vm(vma->vm_mm, start, buf, 3 * SZ_4K, 0), + 3 * (int)SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x5a, 3 * SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start, buf, 3 * SZ_4K, 0), 3 * (int)SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x5a, SZ_4K - 17), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + SZ_4K - 17, 0x96, SZ_4K + 34), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + 2 * SZ_4K + 17, 0x5a, SZ_4K - 17), NULL); + child_vma = test_vma_lookup(child, start); + KUNIT_ASSERT_NOT_NULL(test, child_vma); + mmap_write_lock(child); + vm_flags_clear(child_vma, VM_WRITE); + mmap_write_unlock(child); + buf[0] = 0x27; + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 1, buf, 1, FOLL_WRITE), 0); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 1, buf, 1, + FOLL_WRITE | FOLL_FORCE), 1); + buf[0] = 0; + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 1, buf, 1, 0), 1); + KUNIT_EXPECT_EQ(test, buf[0], (u8)0x27); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 3 * SZ_4K - 5, buf, 20, 0), 5); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 3 * SZ_4K, buf, 20, 0), 0); +} + +static void user4k_remote_file_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * PAGE_SIZE, true); + struct vm_area_struct *small, *native; + u8 buf[34]; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, 3 * SZ_4K, 2 * SZ_4K); + native = test_file_vma(test, file, true, true, TEST_VA, 2 * PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_EXPECT_EQ(test, access_remote_vm(small->vm_mm, TEST_VA + 3 * SZ_4K - 17, + buf, sizeof(buf), 0), (int)sizeof(buf)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x34, 17), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + 17, 0x35, 17), NULL); + memset(buf, 0xa6, sizeof(buf)); + KUNIT_EXPECT_EQ(test, access_remote_vm(small->vm_mm, TEST_VA + 3 * SZ_4K - 17, + buf, sizeof(buf), FOLL_WRITE), (int)sizeof(buf)); + memset(buf, 0, sizeof(buf)); + KUNIT_EXPECT_EQ(test, access_remote_vm(native->vm_mm, TEST_VA + 4 * SZ_4K - 17, + buf, sizeof(buf), 0), (int)sizeof(buf)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0xa6, sizeof(buf)), NULL); +#ifdef CONFIG_CROSS_MEMORY_ATTACH + { + struct kvec vec = { .iov_base = buf, .iov_len = sizeof(buf) }; + struct iov_iter iter; + + memset(buf, 0xb7, sizeof(buf)); + iov_iter_kvec(&iter, ITER_SOURCE, &vec, 1, sizeof(buf)); + KUNIT_EXPECT_EQ(test, process_vm_rw_fragments(small->vm_mm, + TEST_VA + 3 * SZ_4K - 17, sizeof(buf), &iter, true), 0); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 0UL); + memset(buf, 0, sizeof(buf)); + KUNIT_EXPECT_EQ(test, access_remote_vm(native->vm_mm, TEST_VA + 4 * SZ_4K - 17, + buf, sizeof(buf), 0), (int)sizeof(buf)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0xb7, sizeof(buf)), NULL); + } +#endif + mmap_write_lock(small->vm_mm); + vm_flags_clear(small, VM_WRITE); + mmap_write_unlock(small->vm_mm); + KUNIT_EXPECT_EQ(test, access_remote_vm(small->vm_mm, small->vm_start, buf, 1, + FOLL_WRITE | FOLL_FORCE), 0); +#ifdef CONFIG_CROSS_MEMORY_ATTACH + { + struct kvec vec = { .iov_base = buf, .iov_len = sizeof(buf) }; + struct iov_iter iter; + + iov_iter_kvec(&iter, ITER_SOURCE, &vec, 1, sizeof(buf)); + KUNIT_EXPECT_EQ(test, process_vm_rw_fragments(small->vm_mm, + small->vm_start, sizeof(buf), &iter, true), -EFAULT); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), sizeof(buf)); + } +#endif +} + +#ifdef CONFIG_CROSS_MEMORY_ATTACH +static void user4k_process_vm_test(struct kunit *test) +{ + /* Exceed the stack batch, cross table boundaries and include a partial tail. */ + unsigned long start = SZ_2M - SZ_4K, len = 21 * SZ_4K; + struct vm_area_struct *vma = test_fault_vma(test, start, len); + struct mm_struct *child = test_vma_mm(test); + struct iov_iter iter; + struct kvec vec; + u8 *buf = kunit_kmalloc(test, len + 17, GFP_KERNEL); + int ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, child); + KUNIT_ASSERT_NOT_NULL(test, buf); + memset(buf, 0x61, len); + vec = (struct kvec) { .iov_base = buf, .iov_len = len }; + iov_iter_kvec(&iter, ITER_SOURCE, &vec, 1, len); + ret = process_vm_rw_fragments(vma->vm_mm, start, len, &iter, true); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, iov_iter_count(&iter), 0UL); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + memset(buf, 0xc7, len); + iov_iter_kvec(&iter, ITER_SOURCE, &vec, 1, len - 34); + ret = process_vm_rw_fragments(child, start + 17, len - 34, &iter, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 0UL); + memset(buf, 0, len); + iov_iter_kvec(&iter, ITER_DEST, &vec, 1, len); + ret = process_vm_rw_fragments(vma->vm_mm, start, len, &iter, false); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x61, len), NULL); + vec.iov_len = len + 17; + iov_iter_kvec(&iter, ITER_DEST, &vec, 1, len + 17); + ret = process_vm_rw_fragments(child, start, len + 17, &iter, false); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 17UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x61, 17), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + 17, 0xc7, len - 34), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + len - 17, 0x61, 17), NULL); + /* A shorter local iterator ends successfully before the remote range. */ + iov_iter_kvec(&iter, ITER_DEST, &vec, 1, 23); + ret = process_vm_rw_fragments(child, start + 13, len, &iter, false); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 0UL); +} +#endif + +#ifdef CONFIG_FUTEX +static void test_stop_kthread(void *arg); + +struct test_futex_waiter { + struct mm_struct *mm; + unsigned long address; + struct completion ready; + struct completion done; + int result; +}; + +static int test_futex_wait_run(void *arg) +{ + struct test_futex_waiter *w = arg; + ktime_t deadline = ktime_add_ms(ktime_get(), 10000); + + kthread_use_mm(w->mm); + complete(&w->ready); + w->result = futex_wait((u32 __user *)w->address, FLAGS_SIZE_32 | FLAGS_SHARED, + 0x33333333, &deadline, FUTEX_BITSET_MATCH_ANY); + kthread_unuse_mm(w->mm); + complete(&w->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void user4k_futex_wake_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *vmas[2]; + unsigned long addresses[2] = { TEST_VA + SZ_4K + 64, TEST_VA + 2 * SZ_4K + 64 }; + unsigned int round; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm futex fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vmas[0] = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, SZ_4K, 2 * SZ_4K); + vmas[1] = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, vmas[0]); + KUNIT_ASSERT_NOT_NULL(test, vmas[1]); + /* Both directions use the real wait queue and atomic user-value read. */ + for (round = 0; round < 2; round++) { + struct test_futex_waiter *w = kunit_kzalloc(test, sizeof(*w), GFP_KERNEL); + struct task_struct *task; + unsigned long until; + int woke = 0, wrong; + + KUNIT_ASSERT_NOT_NULL(test, w); + w->mm = vmas[round]->vm_mm; + w->address = addresses[round]; + init_completion(&w->ready); + init_completion(&w->done); + task = kthread_run(test_futex_wait_run, w, "user4k-futex-%u", round); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, task); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_stop_kthread, task), 0); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&w->ready, 5 * HZ), 0UL); + until = jiffies + 5 * HZ; + kthread_use_mm(vmas[round ^ 1]->vm_mm); + wrong = futex_wake((u32 __user *)(addresses[round ^ 1] + 4), + FLAGS_SIZE_32 | FLAGS_SHARED, NULL, 1, + FUTEX_BITSET_MATCH_ANY); + do { + woke = futex_wake((u32 __user *)addresses[round ^ 1], + FLAGS_SIZE_32 | FLAGS_SHARED, NULL, 1, + FUTEX_BITSET_MATCH_ANY); + if (woke || completion_done(&w->done)) + break; + schedule_timeout_uninterruptible(1); + } while (time_before(jiffies, until)); + kthread_unuse_mm(vmas[round ^ 1]->vm_mm); + KUNIT_EXPECT_EQ(test, wrong, 0); + KUNIT_EXPECT_EQ(test, woke, 1); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&w->done, 12 * HZ), 0UL); + KUNIT_EXPECT_EQ(test, w->result, 0); + kunit_release_action(test, test_stop_kthread, task); + } +} + +static void user4k_futex_keys_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *small, *native; + union futex_key keys[2], other; + unsigned int i; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm futex fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 2 * PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, 3 * SZ_4K, 2 * SZ_4K); + native = test_file_vma(test, file, true, true, TEST_VA, 2 * PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + for (i = 0; i < 3; i++) { + unsigned long va = small->vm_start + i * SZ_4K + 64; + unsigned long fileoff = (i + 2) * SZ_4K + 64; + + kthread_use_mm(small->vm_mm); + ret = get_futex_key((u32 __user *)va, FLAGS_SIZE_32 | FLAGS_SHARED, + &keys[0], FUTEX_READ); + kthread_unuse_mm(small->vm_mm); + KUNIT_ASSERT_EQ(test, ret, 0); + kthread_use_mm(native->vm_mm); + ret = get_futex_key((u32 __user *)(TEST_VA + fileoff), + FLAGS_SIZE_32 | FLAGS_SHARED, &keys[1], FUTEX_READ); + kthread_unuse_mm(native->vm_mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, futex_match(&keys[0], &keys[1])); + KUNIT_EXPECT_EQ(test, keys[0].shared.pgoff, fileoff / PAGE_SIZE); + KUNIT_EXPECT_EQ(test, keys[0].both.offset, (unsigned int)(fileoff % PAGE_SIZE) | FUT_OFF_INODE); + kthread_use_mm(small->vm_mm); + ret = get_futex_key((u32 __user *)(va + 4), FLAGS_SIZE_32 | FLAGS_SHARED, + &other, FUTEX_READ); + kthread_unuse_mm(small->vm_mm); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_FALSE(test, futex_match(&keys[0], &other)); + } + mmap_write_lock(small->vm_mm); + vm_flags_clear(small, VM_WRITE); + mmap_write_unlock(small->vm_mm); + kthread_use_mm(small->vm_mm); + ret = get_futex_key((u32 __user *)(small->vm_start + 2 * SZ_4K + 64), + FLAGS_SIZE_32 | FLAGS_SHARED, &other, FUTEX_READ); + kthread_unuse_mm(small->vm_mm); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, futex_match(&keys[1], &other)); +} +#endif + +static void test_swap_usage_mapping_free(void *arg) +{ + struct address_space *mapping = arg; + + xa_destroy(&mapping->i_pages); +} + +static void user4k_shmem_swap_usage_test(struct kunit *test) +{ + struct address_space *mapping = kunit_kzalloc(test, sizeof(*mapping), GFP_KERNEL); + void *ret; + + KUNIT_ASSERT_NOT_NULL(test, mapping); + xa_init_flags(&mapping->i_pages, XA_FLAGS_LOCK_IRQ); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_swap_usage_mapping_free, mapping), 0); + /* Synthetic swap values exercise accounting without allocating swap slots. */ + ret = xa_store(&mapping->i_pages, 1, xa_mk_value(1), GFP_KERNEL); + KUNIT_ASSERT_FALSE(test, xa_is_err(ret)); + if (IS_ENABLED(CONFIG_XARRAY_MULTI)) { + ret = xa_store_range(&mapping->i_pages, 4, 7, + xa_mk_value(2), GFP_KERNEL); + KUNIT_ASSERT_FALSE(test, xa_is_err(ret)); + } else { + unsigned long index; + + /* Without multi-index entries, shmem stores one value per page. */ + for (index = 4; index <= 7; index++) { + ret = xa_store(&mapping->i_pages, index, + xa_mk_value(2), GFP_KERNEL); + KUNIT_ASSERT_FALSE(test, xa_is_err(ret)); + } + } + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, PAGE_SIZE + SZ_4K, + PAGE_SIZE + 3 * SZ_4K), 2UL * SZ_4K); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, 3 * SZ_4K, + 2 * PAGE_SIZE + SZ_4K), PAGE_SIZE); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, 4 * PAGE_SIZE + SZ_4K, + 7 * PAGE_SIZE + 3 * SZ_4K), 3 * PAGE_SIZE + 2UL * SZ_4K); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, 5 * PAGE_SIZE + SZ_4K, + 5 * PAGE_SIZE + 2 * SZ_4K), (unsigned long)SZ_4K); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, 0, SZ_4K), 0UL); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, PAGE_SIZE, PAGE_SIZE), 0UL); +} + +static void test_file_mapcounts(struct kunit *test, struct folio *folio, + const unsigned int expected[TEST_SLOTS], unsigned int native) +{ + struct mm_subpage_mapcounts counts; + unsigned int i, total = 0; + + for (i = 0; i < TEST_SLOTS; i++) + total += expected[i]; + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(&folio->page, &counts), total); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, counts.slots[i], expected[i]); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)(total + native)); +} + +static void user4k_file_mapcounts_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *shared, *private, *native, *cvma; + struct mm_struct *child; + struct folio *folio; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shared = test_file_vma(test, file, false, true, TEST_VA, 3 * SZ_4K, SZ_4K); + private = test_file_vma(test, file, false, false, TEST_VA, 3 * SZ_4K, SZ_4K); + native = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, shared); + KUNIT_ASSERT_NOT_NULL(test, private); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, test_file_fault(shared, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(private, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, false) & VM_FAULT_ERROR, 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 2, 2, 2 }, 1); + KUNIT_ASSERT_EQ(test, test_file_fault(private, TEST_VA + 2 * SZ_4K, true) & + VM_FAULT_ERROR, 0U); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 2, 2, 1 }, 1); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, private->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 3, 3, 1 }, 1); + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA + SZ_4K, true) & VM_FAULT_ERROR, 0U); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 3, 2, 1 }, 1); + kunit_release_action(test, user4k_vma_mm_free, child); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 2, 2, 1 }, 1); + kunit_release_action(test, user4k_vma_mm_free, private->vm_mm); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 1, 1, 1 }, 1); + kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 1, 1, 1 }, 0); + kunit_release_action(test, user4k_vma_mm_free, shared->vm_mm); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 0, 0, 0 }, 0); +} + +/* A unique RAM-backed test inode: exercise generic file faults without + * making unsupported filesystem promotion part of this mapping fixture. + */ +static int test_coarse_read_folio(struct file *file, struct folio *folio) +{ + unsigned long i; + loff_t base = (loff_t)folio->index << PAGE_SHIFT; + loff_t size = i_size_read(file_inode(file)); + u8 *data = folio_address(folio); + + for (i = 0; i < folio_size(folio); i++) + data[i] = base + i < size ? 0x31 + ((base + i) / SZ_4K) : 0; + __folio_mark_uptodate(folio); + folio_unlock(folio); + return 0; +} + +static struct file *test_coarse_cache_file(struct kunit *test, unsigned int order, + unsigned long size) +{ + static const struct address_space_operations aops = { + .read_folio = test_coarse_read_folio, + .dirty_folio = filemap_dirty_folio, + }; + static const struct file_operations fops = { + .mmap = generic_file_mmap, + .read_iter = generic_file_read_iter, + .llseek = generic_file_llseek, + }; + struct file *file = anon_inode_create_getfile("user-coarse-file", &fops, + NULL, O_RDWR, NULL); + struct folio *folio; + int ret; + + if (IS_ERR(file)) + return file; + if (kunit_add_action_or_reset(test, test_file_put, file)) + return ERR_PTR(-ENOMEM); + file->f_mapping->a_ops = &aops; + mapping_set_large_folios(file->f_mapping); + i_size_write(file_inode(file), size); + folio = folio_alloc(GFP_KERNEL, order); + if (!folio) + return ERR_PTR(-ENOMEM); + ret = filemap_add_folio(file->f_mapping, folio, 0, GFP_KERNEL); + if (!ret) + test_coarse_read_folio(file, folio); /* Unlocks the installed folio. */ + folio_put(folio); + return ret ? ERR_PTR(ret) : file; +} + +static bool test_coarse_file_enabled(struct kunit *test) +{ + if (PAGE_SHIFT >= 16 || + (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN))) { + kunit_skip(test, "requires internal larger user granule and borrowed-mm opt-in"); + return false; + } + return true; +} + +static void test_promote_unpin(void *data) +{ + struct user_page_fragment *fragment = data; + + if (fragment->folio) { + release_user_fragments(fragment, 1, true); + fragment->folio = NULL; + } +} + +static void user64k_file_promote_test(struct kunit *test) +{ + unsigned int pinned; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "cache promotion currently uses the collapse implementation"); + return; + } + for (pinned = 0; pinned < 2; pinned++) { + struct file *file = test_cache_file(test, 2 * SZ_64K, true); + struct vm_area_struct *coarse, *small; + struct user_page_fragment *fragment; + struct folio *folio; + phys_addr_t old[SZ_64K / PAGE_SIZE], untouched; + unsigned int nr = SZ_64K / PAGE_SIZE, i; + unsigned long offset, modified = SZ_64K / 2 + 19; + pte_t entry; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, SZ_64K); + small = test_file_vma(test, file, false, true, TEST_VA, SZ_64K, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, small); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + untouched = page_to_phys(&folio->page); + folio_put(folio); + for (i = 0; i < nr; i++) { + folio = filemap_get_folio(file->f_mapping, nr + i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_order(folio), 0U); + old[i] = page_to_phys(&folio->page); + folio_put(folio); + } + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + if (pinned) { + fragment = kunit_kzalloc(test, sizeof(*fragment), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fragment); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_promote_unpin, + fragment), 0); + KUNIT_ASSERT_EQ(test, pin_user_fragments_remote(small->vm_mm, + TEST_VA + modified, 1, FOLL_WRITE, fragment, 1), 1L); + *(u8 *)(folio_address(fragment->folio) + fragment->offset) = 0xa7; + KUNIT_EXPECT_EQ(test, collapse_file_user_page(coarse->vm_mm, file, nr), -EBUSY); + /* A pin in the middle forces rollback of earlier isolated folios. */ + for (i = 0; i < nr; i++) { + folio = filemap_get_folio(file->f_mapping, nr + i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, page_to_phys(&folio->page), old[i]); + KUNIT_EXPECT_EQ(test, folio_order(folio), 0U); + folio_put(folio); + } + KUNIT_EXPECT_EQ(test, *(u8 *)(folio_address(fragment->folio) + + fragment->offset), (u8)0xa7); + kunit_release_action(test, test_promote_unpin, fragment); + } + KUNIT_ASSERT_EQ(test, collapse_file_user_page(coarse->vm_mm, file, nr), 0); + folio = filemap_get_folio(file->f_mapping, nr); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_EQ(test, folio_order(folio), 16U - PAGE_SHIFT); + KUNIT_EXPECT_EQ(test, folio->index, (pgoff_t)nr); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, (unsigned long)2 * nr); + for (offset = 0; offset < SZ_64K; offset++) { + u8 expected = pinned && offset == modified ? 0xa7 : 0x41 + offset / SZ_4K; + + if (((u8 *)folio_address(folio))[offset] != expected) { + KUNIT_FAIL(test, "promoted data mismatch at %lu", offset); + break; + } + } + /* An already suitable folio is an idempotent success, including with a GET. */ + KUNIT_EXPECT_EQ(test, collapse_file_user_page(coarse->vm_mm, file, nr), 0); + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA, false) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, coarse->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(coarse->vm_mm, entry), page_to_phys(&folio->page)); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) { + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, small->vm_mm, TEST_VA + offset); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry), + page_to_phys(&folio->page) + offset); + } + { + struct folio *first = filemap_get_folio(file->f_mapping, 0); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, first); + KUNIT_EXPECT_EQ(test, page_to_phys(&first->page), untouched); + folio_put(first); + } + kunit_release_action(test, user4k_vma_mm_free, coarse->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)nr + 1); + } +} + +static void user64k_shmem_fault_test(struct kunit *test) +{ + unsigned int mode; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "coarse shmem promotion requires collapse support"); + return; + } + for (mode = 0; mode < 4; mode++) { + bool seeded = mode & 1, shared = mode < 2; + struct file *file = test_cache_file(test, SZ_64K, seeded); + struct vm_area_struct *vma; + struct folio *cache, *mapped; + unsigned long offset; + pte_t entry; + vm_fault_t fault; + unsigned int attempt; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, shared, TEST_VA, SZ_64K, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + for (attempt = 0; attempt < 8; attempt++) { + fault = test_file_fault(vma, TEST_VA, true); + if (!(fault & VM_FAULT_NOPAGE)) + break; + } + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + entry = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + mapped = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, entry)))); + KUNIT_EXPECT_EQ(test, folio_size(mapped), SZ_64K); + KUNIT_EXPECT_EQ(test, !!folio_test_anon(mapped), !shared); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mapped) + offset, + seeded ? 0x31 + offset / SZ_4K : 0, SZ_4K), NULL); + cache = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, cache), 0); + KUNIT_EXPECT_EQ(test, folio_size(cache), SZ_64K); + if (shared) + KUNIT_EXPECT_PTR_EQ(test, mapped, cache); + else { + KUNIT_EXPECT_PTR_NE(test, mapped, cache); + memset(folio_address(mapped), 0xc9, SZ_64K); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(cache) + offset, + seeded ? 0x31 + offset / SZ_4K : 0, SZ_4K), NULL); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, + shared ? MM_SHMEMPAGES : MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, SZ_64K / PAGE_SIZE); + { + u8 byte = 0x83, observed = 0; + + KUNIT_EXPECT_EQ(test, access_remote_vm(vma->vm_mm, TEST_VA + SZ_64K - 1, + &byte, 1, FOLL_WRITE), 1); + KUNIT_EXPECT_EQ(test, access_remote_vm(vma->vm_mm, TEST_VA + SZ_64K - 1, + &observed, 1, 0), 1); + KUNIT_EXPECT_EQ(test, observed, byte); + KUNIT_EXPECT_EQ(test, ((u8 *)folio_address(cache))[SZ_64K - 1], + shared ? byte : (u8)(seeded ? 0x40 : 0)); + } + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(cache), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(cache), (int)(SZ_64K / PAGE_SIZE) + 1); + } +} + +static void user64k_shmem_fault_busy_test(struct kunit *test) +{ + unsigned int mode; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "coarse shmem promotion requires collapse support"); + return; + } + for (mode = 0; mode < 3; mode++) { + struct file *file = test_cache_file(test, SZ_64K, mode != 0); + struct vm_area_struct *vma; + struct folio *held; + vm_fault_t fault; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + if (!mode) { + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_REMOTE | + FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT, NULL); + mmap_assert_locked(vma->vm_mm); + mmap_read_unlock(vma->vm_mm); + KUNIT_EXPECT_EQ(test, fault, (vm_fault_t)VM_FAULT_RETRY); + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, 0UL); + } else if (mode == 1) { + held = filemap_get_folio(file->f_mapping, SZ_64K / (2 * PAGE_SIZE)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, held); + folio_lock(held); + fault = test_file_fault(vma, TEST_VA, false); + folio_unlock(held); + folio_put(held); + KUNIT_EXPECT_EQ(test, fault, (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SHMEMPAGES), 0L); + } else { + struct vm_area_struct *small = test_file_vma(test, file, false, true, + TEST_VA, SZ_64K, 0); + struct user_page_fragment *fragment; + + KUNIT_ASSERT_NOT_NULL(test, small); + fragment = kunit_kzalloc(test, sizeof(*fragment), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fragment); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_promote_unpin, + fragment), 0); + KUNIT_ASSERT_EQ(test, pin_user_fragments_remote(small->vm_mm, + TEST_VA + SZ_64K / 2, 1, FOLL_WRITE, fragment, 1), 1L); + fault = test_file_fault(vma, TEST_VA, false); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, *(u8 *)(folio_address(fragment->folio) + + fragment->offset), (u8)0x39); + kunit_release_action(test, test_promote_unpin, fragment); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + } + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & + (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + KUNIT_EXPECT_TRUE(test, pte_present(test_fault_entry(test, vma->vm_mm, TEST_VA))); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + } +} + +static void user64k_shmem_fault_hole_test(struct kunit *test) +{ +#ifdef CONFIG_USERFAULTFD + unsigned int adjacent; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "coarse shmem promotion requires collapse support"); + return; + } + for (adjacent = 0; adjacent < 2; adjacent++) { + struct file *file = test_cache_file(test, 2 * SZ_64K, false); + struct vm_area_struct *coarse, *missing, *seed; + struct folio *folio; + vm_fault_t fault; + bool new; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, 0); + missing = test_file_vma(test, file, false, true, TEST_VA, SZ_4K, + adjacent ? SZ_64K : PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, missing); + seed = test_file_vma(test, file, false, true, TEST_VA, SZ_4K, 0); + KUNIT_ASSERT_NOT_NULL(test, seed); + mmap_read_lock(seed->vm_mm); + folio = shmem_uffd_prepare_folio(seed, TEST_VA, &new); + mmap_read_unlock(seed->vm_mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, new); + memset(folio_address(folio), 0x6f, SZ_4K); + shmem_uffd_mark_present(folio, 0, BIT(0)); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), + GENMASK(PAGE_SIZE / SZ_4K - 1, 1)); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + /* Model the registration flag used by the collapse interval check. */ + mmap_write_lock(missing->vm_mm); + vma_start_write(missing); + vm_flags_set(missing, VM_UFFD_MISSING); + mmap_write_unlock(missing->vm_mm); + fault = test_file_fault(coarse, TEST_VA, false); + mmap_write_lock(missing->vm_mm); + vma_start_write(missing); + vm_flags_clear(missing, VM_UFFD_MISSING); + mmap_write_unlock(missing->vm_mm); + /* A real coarse fault may populate holes covered by its entire leaf. */ + KUNIT_EXPECT_EQ(test, fault & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA, false) & + (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), SZ_64K); + folio_lock(folio); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(folio, 1, 0, SZ_4K)); + folio_unlock(folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0x6f, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, + SZ_64K - SZ_4K), NULL); + folio_put(folio); + kunit_release_action(test, user4k_vma_mm_free, coarse->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, missing->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, seed->vm_mm); + } +#else + kunit_skip(test, "requires userfaultfd registration flags"); +#endif +} + +static void user64k_file_leaf_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *coarse, *native, *small, *child_vma; + struct mm_struct *coarse_mm, *child; + struct folio *folio; + unsigned long offset; + unsigned int nr = (2 * SZ_64K) / PAGE_SIZE; + unsigned int small_nr = 2 * SZ_64K / SZ_4K - 1; + unsigned int maps = 2 * nr + small_nr; + u8 byte = 0xc7, observed; + int ret; + + if (!test_coarse_file_enabled(test)) + return; + file = test_coarse_cache_file(test, 17 - PAGE_SHIFT, 2 * SZ_64K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, 2 * SZ_64K, 0); + native = test_file_vma(test, file, true, true, TEST_VA, 2 * SZ_64K, 0); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + 2 * SZ_64K - SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_NOT_NULL(test, small); + coarse_mm = coarse->vm_mm; + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + /* One cache folio supplies two user leaves, with native references per leaf. */ + for (offset = 0; offset < 2 * SZ_64K; offset += SZ_64K) { + pte_t entry = test_fault_entry(test, coarse_mm, TEST_VA + offset); + + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(coarse_mm, entry), + page_to_phys(&folio->page) + offset); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(coarse_mm, MM_FILEPAGES), 2L); + for (offset = 0; offset < 2 * SZ_64K; offset += PAGE_SIZE) + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + for (offset = SZ_4K; offset < 2 * SZ_64K; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)(nr + maps + 1)); + KUNIT_ASSERT_EQ(test, access_remote_vm(coarse_mm, TEST_VA + SZ_64K + 3 * SZ_4K, + &byte, 1, FOLL_WRITE), 1); + KUNIT_ASSERT_EQ(test, access_remote_vm(small->vm_mm, TEST_VA + SZ_64K + 3 * SZ_4K, + &observed, 1, 0), 1); + KUNIT_EXPECT_EQ(test, observed, byte); + byte = 0x98; + KUNIT_ASSERT_EQ(test, access_remote_vm(small->vm_mm, TEST_VA + 7 * SZ_4K, + &byte, 1, FOLL_WRITE), 1); + KUNIT_ASSERT_EQ(test, access_remote_vm(native->vm_mm, TEST_VA + 7 * SZ_4K, + &observed, 1, 0), 1); + KUNIT_EXPECT_EQ(test, observed, byte); + KUNIT_EXPECT_EQ(test, ((u8 *)folio_address(folio))[7 * SZ_4K], byte); + + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, coarse_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + /* Ordinary file VMAs are lazily populated in the child after fork. */ + mmap_read_lock(child); + child_vma = find_vma(child, TEST_VA); + mmap_read_unlock(child); + KUNIT_ASSERT_NOT_NULL(test, child_vma); + KUNIT_ASSERT_EQ(test, test_file_fault(child_vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(child_vma, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + maps += nr; + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + kthread_use_mm(coarse_mm); + ret = vm_munmap(TEST_VA, SZ_64K); + kthread_unuse_mm(coarse_mm); + KUNIT_ASSERT_EQ(test, ret, 0); + maps -= SZ_64K / PAGE_SIZE; + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(coarse_mm, MM_FILEPAGES), 1L); + KUNIT_EXPECT_TRUE(test, pte_present(test_fault_entry(test, child, TEST_VA))); + + /* The native-rounded range must remove every overlapping coarse alias. */ + unmap_mapping_range(file->f_mapping, SZ_64K + SZ_4K, SZ_4K, 0); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, coarse_mm, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, child, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, native->vm_mm, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, small->vm_mm, TEST_VA + SZ_64K + SZ_4K))); + KUNIT_EXPECT_TRUE(test, pte_present(test_fault_entry(test, small->vm_mm, + TEST_VA + SZ_64K + PAGE_SIZE))); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(coarse_mm, MM_FILEPAGES), 0L); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), + (int)(maps - 2 * SZ_64K / PAGE_SIZE - 1 - PAGE_SIZE / SZ_4K)); + + /* An exact subpage hole keeps neighboring small leaves in that native page. */ + mmap_read_lock(coarse_mm); + coarse = find_vma(coarse_mm, TEST_VA + SZ_64K); + mmap_read_unlock(coarse_mm); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(child_vma, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + for (offset = 0; offset < PAGE_SIZE; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + SZ_64K + offset, + false) & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + unmap_mapping_subpage_hole(file->f_mapping, SZ_64K + SZ_4K, SZ_64K + 2 * SZ_4K); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, coarse_mm, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, child, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, native->vm_mm, TEST_VA + SZ_64K))); + for (offset = 0; offset < PAGE_SIZE; offset += SZ_4K) + KUNIT_EXPECT_EQ(test, !!pte_present(test_fault_entry(test, small->vm_mm, + TEST_VA + SZ_64K + offset)), offset != SZ_4K); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)(maps - 2 * SZ_64K / PAGE_SIZE - 2)); + kunit_release_action(test, user4k_vma_mm_free, coarse_mm); + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)nr + 1); +} + +static void user64k_file_cow_test(struct kunit *test) +{ + unsigned int cold; + + if (!test_coarse_file_enabled(test)) + return; + for (cold = 0; cold < 2; cold++) { + struct file *file = test_coarse_cache_file(test, 17 - PAGE_SHIFT, 2 * SZ_64K); + struct vm_area_struct *vma; + struct folio *cache, *anon; + unsigned long offset; + pte_t entry; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, false, TEST_VA, SZ_64K, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + if (!cold) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + anon = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, entry)))); + KUNIT_EXPECT_TRUE(test, folio_test_anon(anon)); + KUNIT_EXPECT_EQ(test, folio_size(anon), SZ_64K); + folio_get(anon); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, anon), 0); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(anon) + offset, + 0x41 + offset / SZ_4K, SZ_4K), NULL); + memset(folio_address(anon), 0x93, SZ_64K); + cache = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); + for (offset = 0; offset < 2 * SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(cache) + offset, + 0x31 + offset / SZ_4K, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, folio_mapcount(cache), 0); + folio_put(cache); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_FILEPAGES), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(anon), (int)(SZ_64K / PAGE_SIZE)); + KUNIT_EXPECT_EQ(test, folio_ref_count(anon), (int)(SZ_64K / PAGE_SIZE) + 1); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(anon), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(anon), 1); + } +} + +static void user64k_file_bounds_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *vma; + struct folio *folio; + unsigned int mode; + unsigned long size = SZ_64K + PAGE_SIZE + 19; + pte_t entry; + + if (!test_coarse_file_enabled(test)) + return; + /* Too-small backing and a physically unaligned source cannot form a leaf. */ + for (mode = 0; mode < 2; mode++) { + file = test_coarse_cache_file(test, mode ? 17 - PAGE_SHIFT : 0, 2 * SZ_64K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, false, TEST_VA, SZ_64K, + mode ? PAGE_SIZE : 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_TRUE(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_SIGBUS); + KUNIT_EXPECT_TRUE(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_SIGBUS); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, vma->vm_mm, TEST_VA))); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + folio_put(folio); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + } + file = test_coarse_cache_file(test, 17 - PAGE_SHIFT, size); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, true, TEST_VA, 3 * SZ_64K, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + SZ_64K, false) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, vma->vm_mm, TEST_VA + SZ_64K); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(vma->vm_mm, entry)) + + PAGE_SIZE + 19, 0, SZ_64K - PAGE_SIZE - 19), NULL); + KUNIT_EXPECT_TRUE(test, test_file_fault(vma, TEST_VA + 2 * SZ_64K, false) & VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_FILEPAGES), 2L); +} + +static void user4k_file_shared_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 4 * PAGE_SIZE, true); + struct vm_area_struct *small, *native; + struct folio *folio; + unsigned int i; + pte_t entry; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + native = test_file_vma(test, file, true, true, TEST_VA, 4 * PAGE_SIZE, 0); + /* Virtual and file offsets deliberately have different native alignment. */ + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + 3 * PAGE_SIZE, 2 * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + SZ_4K, false) & + VM_FAULT_ERROR, 0U); + /* Fault-around keeps its 64K window and populates all ready cache folios. */ + for (i = 0; i < 12; i++) { + unsigned long addr = small->vm_start + i * SZ_4K; + struct vm_page_offset pos = vma_page_offset_at(small, addr); + + entry = test_fault_entry(test, small->vm_mm, addr); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + folio = filemap_get_folio(file->f_mapping, pos.index); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry), + page_to_phys(folio_file_page(folio, pos.index)) + pos.offset); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(small->vm_mm, entry)), + (u8)(0x33 + i)); + folio_put(folio); + } + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, true) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, native->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + memset(phys_to_virt(pte_phys_mm(native->vm_mm, entry)) + 2 * SZ_4K, 0x72, SZ_4K); + entry = test_fault_entry(test, small->vm_mm, small->vm_start); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(small->vm_mm, entry)), + 0x72, SZ_4K), NULL); + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + SZ_4K, true) & + VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, small->vm_mm, small->vm_start + SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + memset(phys_to_virt(pte_phys_mm(small->vm_mm, entry)), 0x84, SZ_4K); + entry = test_fault_entry(test, native->vm_mm, TEST_VA); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(native->vm_mm, entry)) + + 3 * SZ_4K, 0x84, SZ_4K), NULL); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + folio_put(folio); +} + +static void user4k_file_cow_test(struct kunit *test) +{ + unsigned int cold; + + for (cold = 0; cold < 2; cold++) { + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *vma; + struct folio *cache; + pte_t entry; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, false, false, TEST_VA + SZ_4K, + 3 * SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + if (!cold) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start, false) & + VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start + SZ_4K, true) & + VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, vma->vm_mm, vma->vm_start + SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + KUNIT_EXPECT_TRUE(test, folio_test_anon(page_folio(pte_page(entry)))); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, entry) & ~PAGE_MASK, 2UL * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(vma->vm_mm, entry)), + 0x33, SZ_4K), NULL); + memset(phys_to_virt(pte_phys_mm(vma->vm_mm, entry)), 0x91, SZ_4K); + cache = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(cache) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(cache), 0); + folio_put(cache); + } +} + +static void user4k_file_eof_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, SZ_4K + 37, false); + struct vm_area_struct *small, *native; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, TEST_VA, PAGE_SIZE, 0); + native = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + SZ_4K, true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA + 3 * SZ_4K, false) & VM_FAULT_ERROR, 0U); + for (i = 0; i < 4; i++) { + vm_fault_t ret = test_file_fault(small, TEST_VA + i * SZ_4K, false); + + KUNIT_EXPECT_EQ(test, ret & VM_FAULT_ERROR, i < 2 ? 0U : VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, pte_present(test_fault_entry(test, small->vm_mm, + TEST_VA + i * SZ_4K)), i < 2); + } +} + + +static const struct vm_operations_struct test_filemap_fault_ops = { + .fault = filemap_fault, + .page_mkwrite = filemap_page_mkwrite, +}; + +static void user4k_filemap_fault_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * SZ_4K + 17, true); + struct vm_area_struct *vma; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, false, true, TEST_VA + 3 * SZ_4K, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + /* Exercise generic filemap_fault against an already populated cache. */ + vma->vm_ops = &test_filemap_fault_ops; + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start + SZ_4K, true) & VM_FAULT_ERROR, 0U); + for (i = 0; i < 3; i++) { + pte_t pte = test_fault_entry(test, vma->vm_mm, vma->vm_start + i * SZ_4K); + + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(vma->vm_mm, pte)), (u8)(0x31 + i)); + } + KUNIT_EXPECT_EQ(test, test_file_fault(vma, vma->vm_start + 3 * SZ_4K, false) & + VM_FAULT_ERROR, VM_FAULT_SIGBUS); +} + +static void user4k_file_truncate_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * PAGE_SIZE, true); + struct vm_area_struct *small, *cow, *native; + struct iattr attr = { .ia_valid = ATTR_SIZE, .ia_size = SZ_4K + 37 }; + unsigned int i; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + PAGE_SIZE, SZ_4K); + cow = test_file_vma(test, file, false, false, TEST_VA, PAGE_SIZE, 0); + native = test_file_vma(test, file, true, false, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, cow); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(cow, TEST_VA + 3 * SZ_4K, true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, true) & VM_FAULT_ERROR, 0U); + inode_lock(file_inode(file)); + ret = notify_change(&nop_mnt_idmap, file->f_path.dentry, &attr, NULL); + inode_unlock(file_inode(file)); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < 4; i++) { + unsigned long addr = small->vm_start + i * SZ_4K; + + if (i) + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, small->vm_mm, addr))); + KUNIT_EXPECT_EQ(test, test_file_fault(small, addr, false) & VM_FAULT_ERROR, + i ? VM_FAULT_SIGBUS : 0U); + } + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, cow->vm_mm, TEST_VA + 3 * SZ_4K))); + KUNIT_EXPECT_EQ(test, test_file_fault(cow, TEST_VA + 3 * SZ_4K, false) & VM_FAULT_ERROR, + VM_FAULT_SIGBUS); + KUNIT_EXPECT_TRUE(test, pte_present(test_fault_entry(test, native->vm_mm, TEST_VA))); + KUNIT_EXPECT_EQ(test, test_file_fault(native, TEST_VA + 3 * SZ_4K, false) & VM_FAULT_ERROR, 0U); +} + +static void user4k_file_invalidate_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * PAGE_SIZE, true); + struct vm_area_struct *vma; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + 4 * SZ_4K, PAGE_SIZE - SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start, false) & VM_FAULT_ERROR, 0U); + unmap_mapping_pages(file->f_mapping, 0, 1, false); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_EQ(test, pte_present(test_fault_entry(test, vma->vm_mm, + vma->vm_start + i * SZ_4K)), i != 0); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start, false) & VM_FAULT_ERROR, 0U); + unmap_mapping_pages(file->f_mapping, 1, 1, false); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_EQ(test, pte_present(test_fault_entry(test, vma->vm_mm, + vma->vm_start + i * SZ_4K)), i == 0); +} + +static void user4k_file_fork_hole_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * PAGE_SIZE, true); + struct vm_area_struct *shared, *private, *cvma; + struct mm_struct *child; + struct inode *inode; + unsigned int i; + pte_t copied; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shared = test_file_vma(test, file, false, true, TEST_VA, 2 * PAGE_SIZE, 0); + private = test_file_vma(test, file, false, false, TEST_VA, 2 * PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, shared); + KUNIT_ASSERT_NOT_NULL(test, private); + KUNIT_ASSERT_EQ(test, test_file_fault(shared, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(private, TEST_VA, false) & VM_FAULT_ERROR, 0U); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, private->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + mmap_read_lock(child); + cvma = find_vma(child, TEST_VA); + mmap_read_unlock(child); + KUNIT_ASSERT_NOT_NULL(test, cvma); + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA + SZ_4K, true) & VM_FAULT_ERROR, 0U); + copied = test_fault_entry(test, child, TEST_VA + SZ_4K); + memset(phys_to_virt(pte_phys_mm(child, copied)), 0x82, SZ_4K); + inode = file_inode(file); + /* A sub-native-page hole zeros shared bytes but retains private COW. */ + ret = vfs_fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, SZ_4K, SZ_4K); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, i_size_read(inode), (loff_t)(2 * PAGE_SIZE)); + for (i = 0; i < 2 * TEST_SLOTS; i++) { + pte_t pte; + + KUNIT_ASSERT_EQ(test, test_file_fault(shared, TEST_VA + i * SZ_4K, false) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, shared->vm_mm, TEST_VA + i * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(shared->vm_mm, pte)), + i == 1 ? 0 : 0x31 + i, SZ_4K), NULL); + } + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, test_fault_entry(test, child, TEST_VA + SZ_4K)), + pte_phys_mm(child, copied)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(child, copied)), 0x82, SZ_4K), NULL); + /* A full cache-page hole exercises the file interval-tree invalidation. */ + ret = vfs_fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, PAGE_SIZE, PAGE_SIZE); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = TEST_SLOTS; i < 2 * TEST_SLOTS; i++) { + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, shared->vm_mm, TEST_VA + i * SZ_4K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, child, TEST_VA + i * SZ_4K))); + } +} + +static void test_zero_cow_granule(struct kunit *test, unsigned int shift) +{ + unsigned long size = 1UL << shift; + struct vm_area_struct *vma = test_fault_vma_granule(test, shift, TEST_VA, PAGE_SIZE); + struct mm_struct *mm; + struct mm_subpage *slot; + pte_t before[4], after; + unsigned int i; + vm_fault_t ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mm = vma->vm_mm; + for (i = 0; i < 4; i++) { + mmap_read_lock(mm); + ret = handle_mm_fault(vma, TEST_VA + i * size, + FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + before[i] = test_fault_entry(test, mm, TEST_VA + i * size); + KUNIT_ASSERT_TRUE(test, pte_present(before[i])); + KUNIT_ASSERT_TRUE(test, is_zero_pfn(pte_pfn(before[i]))); + KUNIT_ASSERT_FALSE(test, pte_write(before[i])); + } + mmap_read_lock(mm); + ret = handle_mm_fault(vma, TEST_VA + size + 7, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + lru_add_drain_all(); + for (i = 0; i < 4; i++) { + after = test_fault_entry(test, mm, TEST_VA + i * size); + if (i != 1) { + KUNIT_EXPECT_TRUE(test, pte_same(before[i], after)); + continue; + } + KUNIT_ASSERT_TRUE(test, pte_write(after)); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, after)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), (unsigned int)size); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(mm_subpage_folio(slot)), 2); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + size, 0, size), NULL); + mm_subpage_put(slot); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1UL); +} + +static void user4k_zero_cow_test(struct kunit *test) +{ + test_zero_cow_granule(test, 12); +} + +static void user16k_zero_cow_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative faults require larger native backing"); + return; + } + test_zero_cow_granule(test, 14); +} + +static void test_cow_reuse_granule(struct kunit *test, unsigned int shift) +{ + unsigned long size = 1UL << shift; + unsigned int mode; + + for (mode = 0; mode < 2; mode++) { + struct vm_area_struct *vma = test_fault_vma_granule(test, shift, TEST_VA, PAGE_SIZE); + struct mm_subpage *held = NULL; + struct mm_struct *child; + phys_addr_t original[4]; + unsigned int i; + vm_fault_t fault; + int ret; + pte_t pte; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + for (i = 0; i < 4; i++) { + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * size); + original[i] = pte_phys_mm(vma->vm_mm, pte); + memset(phys_to_virt(original[i]), 0xb0 + i, size); + } + if (mode) { + held = mm_subpage_get_from_phys(original[2]); + KUNIT_ASSERT_NOT_NULL(test, held); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, held), 0); + } + child = test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + kunit_release_action(test, user4k_vma_mm_free, child); + for (i = 0; i < 4; i++) { + unsigned long address = TEST_VA + i * size; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + pte = test_fault_entry(test, vma->vm_mm, address); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte) == original[i], + !(mode && i == 2)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(vma->vm_mm, pte)), + 0xb0 + i, size), NULL); + memset(phys_to_virt(pte_phys_mm(vma->vm_mm, pte)), 0xd0 + i, size); + } + if (held) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(mm_subpage_phys(held)), + 0xb2, size), NULL); + } +} + +static void user4k_cow_reuse_test(struct kunit *test) +{ + test_cow_reuse_granule(test, 12); +} + +static void user16k_cow_reuse_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative faults require larger native backing"); + return; + } + test_cow_reuse_granule(test, 14); +} + +static void test_fork_cow_granule(struct kunit *test, unsigned int shift) +{ + unsigned long size = 1UL << shift; + struct vm_area_struct *parent = test_fault_vma_granule(test, shift, TEST_VA, 4 * size); + struct mm_struct *child = test_vma_mm_granule(test, shift); + struct vm_area_struct *cvma; + struct mm_subpage *held[4]; + pte_t before[4], after; + unsigned int i, side; + vm_fault_t fault; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, parent); + KUNIT_ASSERT_NOT_NULL(test, child); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + for (i = 0; i < 4; i++) { + before[i] = test_fault_entry(test, parent->vm_mm, TEST_VA + i * size); + held[i] = mm_subpage_get_from_phys(pte_phys_mm(parent->vm_mm, before[i])); + KUNIT_ASSERT_NOT_NULL(test, held[i]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, held[i]), 0); + memset(folio_address(mm_subpage_folio(held[i])) + mm_subpage_offset(held[i]), + 0x31 + i, size); + } + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + mmap_read_lock(child); + fault = handle_mm_fault(cvma, TEST_VA + size + 15, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : parent->vm_mm; + + for (i = 0; i < 4; i++) { + struct mm_subpage *slot; + bool changed = side && i == 1; + + after = test_fault_entry(test, mm, TEST_VA + i * size); + KUNIT_EXPECT_EQ(test, !!pte_write(after), changed); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, after) != mm_subpage_phys(held[i]), changed); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, after)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * (unsigned int)size); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0x31 + i, size), NULL); + if (changed) { + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(mm_subpage_folio(slot)), 2); + memset(folio_address(mm_subpage_folio(slot)) + size, 0x77, size); + } + mm_subpage_put(slot); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 4UL); + } + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(held[i]), i == 1 ? 1 : 2); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(held[i])) + + mm_subpage_offset(held[i]), 0x31 + i, size), NULL); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(mm_subpage_folio(held[0])), 11); +} + +static void user4k_fork_cow_test(struct kunit *test) +{ + test_fork_cow_granule(test, 12); +} + +static void user16k_fork_cow_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative faults require larger native backing"); + return; + } + test_fork_cow_granule(test, 14); +} + +static void user4k_folio_walk_geometry_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, PAGE_SHIFT, 14 }; + unsigned int which; + + for (which = 0; which < ARRAY_SIZE(shifts); which++) { + unsigned int shift = shifts[which], leaf; + struct vm_area_struct *vma; + struct mm_struct *mm; + unsigned long start, size = 1UL << shift; + vm_fault_t fault; + + if (which == 2 && PAGE_SHIFT <= 14) + continue; + mm = shift == PAGE_SHIFT ? mm_alloc() : test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, mm); + if (shift == PAGE_SHIFT) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + start = pmd_size_mm(mm) - size; + vma = test_fault_vma_in_mm(mm, start, 2 * size); + KUNIT_ASSERT_NOT_NULL(test, vma); + for (leaf = 0; leaf < 2; leaf++) { + unsigned long address = start + leaf * size; + struct folio_walk fw; + struct folio *folio; + struct page *page = NULL; + pte_t expected, walked = __pte(0); + bool zero = false; + + mmap_read_lock(mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + mmap_read_lock(mm); + folio = folio_walk_start(&fw, vma, address, 0); + if (folio) + folio_walk_end(&fw, vma); + mmap_read_unlock(mm); + KUNIT_EXPECT_PTR_EQ(test, folio, NULL); + mmap_read_lock(mm); + folio = folio_walk_start(&fw, vma, address, FW_ZEROPAGE); + if (folio) { + zero = is_zero_folio(folio) && !fw.page && fw.level == FW_LEVEL_PTE; + folio_walk_end(&fw, vma); + } + mmap_read_unlock(mm); + KUNIT_EXPECT_TRUE(test, zero); + + mmap_read_lock(mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + expected = test_fault_entry(test, mm, address); + mmap_read_lock(mm); + folio = folio_walk_start(&fw, vma, address + size - 1, 0); + if (folio) { + page = fw.page; + walked = fw.pte; + folio_walk_end(&fw, vma); + } + mmap_read_unlock(mm); + KUNIT_EXPECT_PTR_EQ(test, page, pfn_to_page(pte_pfn(expected))); + KUNIT_EXPECT_TRUE(test, pte_same(walked, expected)); + KUNIT_EXPECT_PTR_EQ(test, folio, page_folio(pfn_to_page(pte_pfn(expected)))); + } + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_fragment_locked_pin_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, PAGE_SHIFT, 14 }; + unsigned int which; + + for (which = 0; which < ARRAY_SIZE(shifts); which++) { + unsigned int shift = shifts[which]; + struct mm_struct *mm; + struct vm_area_struct *vma, *found; + struct user_page_fragment fragment = { }; + unsigned long address; + int ret; + + if (which == 2 && PAGE_SHIFT <= 14) + continue; + mm = shift == PAGE_SHIFT ? mm_alloc() : test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, mm); + if (shift == PAGE_SHIFT) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + vma = test_fault_vma_in_mm(mm, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + address = TEST_VA + (1UL << shift) - sizeof(long); + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, sizeof(long), + FOLL_WRITE | FOLL_NOFAULT, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + KUNIT_EXPECT_PTR_EQ(test, found, NULL); + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, sizeof(long), + FOLL_WRITE | FOLL_LONGTERM, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, found, NULL); + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, sizeof(long), + FOLL_WRITE | FOLL_NOWAIT, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, found, NULL); + /* Only the first fragment is returned, while retaining the VMA lock. */ + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, 2 * sizeof(long), + FOLL_WRITE, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, 1); + if (ret != 1) + continue; + KUNIT_EXPECT_PTR_EQ(test, found, vma); + KUNIT_EXPECT_TRUE(test, fragment.pinned); + KUNIT_EXPECT_EQ(test, fragment.length, (unsigned int)sizeof(long)); + KUNIT_EXPECT_EQ(test, !!fragment.subpage, shift != PAGE_SHIFT); + release_user_fragments(&fragment, 1, false); + /* With backing present, NOFAULT pins the exact same leaf successfully. */ + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, sizeof(long), + FOLL_WRITE | FOLL_NOFAULT, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, 1); + if (ret == 1) { + KUNIT_EXPECT_PTR_EQ(test, found, vma); + release_user_fragments(&fragment, 1, true); + } + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user16k_fragment_offset_test(struct kunit *test) +{ + struct vm_area_struct *vma; + struct user_page_fragment fragment = {}; + unsigned long address = TEST_VA + 3 * SZ_16K + SZ_4K + 13; + long ret; + + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative fragments require larger native backing"); + return; + } + vma = test_fault_vma_granule(test, 14, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + ret = pin_user_fragments_remote(vma->vm_mm, address, 31, FOLL_WRITE, &fragment, 1); + KUNIT_EXPECT_EQ(test, ret, 1L); + if (ret != 1) + return; + KUNIT_EXPECT_NOT_NULL(test, fragment.subpage); + KUNIT_EXPECT_EQ(test, fragment.offset, 3UL * SZ_16K + SZ_4K + 13); + KUNIT_EXPECT_EQ(test, fragment.length, 31U); + if (fragment.subpage) + KUNIT_EXPECT_EQ(test, mm_subpage_shift(fragment.subpage), 14U); + memset(folio_address(fragment.folio) + fragment.offset, 0x9c, fragment.length); + release_user_fragments(&fragment, 1, true); + { + u8 buf[33]; + + ret = access_remote_vm(vma->vm_mm, address - 1, buf, sizeof(buf), 0); + KUNIT_ASSERT_EQ(test, ret, (long)sizeof(buf)); + KUNIT_EXPECT_EQ(test, buf[0], (u8)0); + KUNIT_EXPECT_EQ(test, buf[32], (u8)0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + 1, 0x9c, 31), NULL); + } +} + +static void user4k_anon_exit_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + struct mm_subpage *held; + struct anon_vma *root; + struct folio *folio; + struct mm_struct *mm; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(f->pool, i * SZ_4K); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(f, i, slot), 0); + } + held = f->slots[0]; + folio = mm_subpage_folio(held); + root = f->vma->anon_vma->root; + mm_subpage_get(held); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, held), 0); + mm_subpage_pool_close(f->pool); + mm_subpage_pool_put(f->pool); + /* Leave all four mapping refs and the tables to the real exit path. */ + mm = f->mm; + memset(f, 0, sizeof(*f)); + mmput(mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(held), 0); + KUNIT_EXPECT_EQ(test, atomic_read(&root->refcount), 1); + for (i = 1; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_get_from_phys(mm_subpage_phys(held) + i * SZ_4K); + + KUNIT_EXPECT_PTR_EQ(test, slot, NULL); + if (slot) + mm_subpage_put(slot); + } +} + +#ifdef CONFIG_ARM64_CONTPTE +struct contpte_fixture { + struct mm_struct *mm; + struct folio *folio; + unsigned int nr; +}; + +static void contpte_fixture_free(void *arg) +{ + struct contpte_fixture *f = arg; + struct mmu_gather tlb; + unsigned int i; + + if (f->mm) { + mmap_write_lock(f->mm); + for (i = 0; i <= f->nr; i++) { + unsigned long addr = TEST_VA + i * mm_page_size(f->mm); + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm, addr, &ptl); + + if (pte) { + pte_clear(f->mm, addr, pte); + pte_unmap_unlock(pte, ptl); + } + } + tlb_gather_mmu_fullmm(&tlb, f->mm); + free_pgd_range(&tlb, 0, 1UL << 48, 0, 0); + tlb_finish_mmu(&tlb); + mmap_write_unlock(f->mm); + mmput(f->mm); + } + if (f->folio) + folio_put(f->folio); +} + +static void test_contpte_granule(struct kunit *test, bool small) +{ + struct contpte_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + struct vm_area_struct vma = {}; + struct test_mm tables = {}; + struct { + bool initial, addresses, hints, gathered, aged, access_all, unfolded; + bool permissions, refolded, holes, full_clear, sentinel; + } ok; + unsigned long size; + phys_addr_t phys; + pte_t base, saved, observed; + pte_t *pte; + spinlock_t *ptl; + unsigned int i; + int access_ret, access_same_ret, access_noop_ret; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, contpte_fixture_free, f), 0); + f->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, f->mm); + if (small) + KUNIT_ASSERT_EQ(test, test_mm_select4k(f->mm), 0); + f->nr = cont_ptes_mm(f->mm); + size = mm_page_size(f->mm); + f->folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, + get_order(cont_pte_size_mm(f->mm))); + KUNIT_ASSERT_NOT_NULL(test, f->folio); + phys = PFN_PHYS(folio_pfn(f->folio)); + tables.mm = f->mm; + KUNIT_ASSERT_EQ(test, test_map_alias(&tables, 0, TEST_VA, phys), 0); + vma.vm_mm = f->mm; + base = phys_pte_mm(f->mm, phys, __pgprot(TEST_PROT | PTE_RDONLY)); + memset(&ok, 1, sizeof(ok)); + mmap_read_lock(f->mm); + pte = test_mm_lookup(f->mm, TEST_VA, &ptl); + if (!pte) { + mmap_read_unlock(f->mm); + KUNIT_FAIL(test, "missing PTE table"); + return; + } + pte_clear(f->mm, TEST_VA, pte); + /* The adjacent entry must never be included in this contiguous group. */ + saved = pte_mkspecial(base); + set_pte(pte + f->nr, saved); + set_ptes(f->mm, TEST_VA, pte, base, f->nr); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.initial &= pte_cont(observed); + ok.addresses &= pte_phys_mm(f->mm, observed) == phys + i * size; + ok.hints &= pte_batch_hint(pte + i, observed) == f->nr - i; + } + __set_pte(pte + 3, pte_mkdirty(__ptep_get(pte + 3))); + observed = ptep_get(pte + 1); + ok.gathered &= pte_dirty(observed) && pte_young(observed); + observed = ptep_get_lockless(pte + 2); + ok.gathered &= pte_dirty(observed) && pte_young(observed) && + pte_phys_mm(f->mm, observed) == phys + 2 * size; + clear_young_dirty_ptes(&vma, TEST_VA + 3 * size, pte + 3, 1, + CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.aged &= !pte_young(observed) && !pte_dirty(observed) && pte_cont(observed); + } + observed = pte_mkdirty(pte_mkyoung(__ptep_get(pte + 3))); + access_same_ret = ptep_set_access_flags(&vma, TEST_VA + 3 * size, + pte + 3, observed, 1); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.access_all &= pte_cont(observed) && pte_young(observed) && + pte_dirty(observed) && !pte_write(observed); + } + access_noop_ret = ptep_set_access_flags(&vma, TEST_VA + 3 * size, + pte + 3, ptep_get(pte + 3), 1); + observed = pte_mkwrite_novma(pte_mkdirty(pte_mkyoung(__ptep_get(pte + 3)))); + access_ret = ptep_set_access_flags(&vma, TEST_VA + 3 * size, pte + 3, + observed, 1); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.unfolded &= !pte_cont(observed); + ok.permissions &= pte_write(observed) == (i == 3); + } + set_pte_at(f->mm, TEST_VA + 3 * size, pte + 3, + pte_advance_pfn_mm(f->mm, base, 3)); + /* Last-entry installation must fold using the physical byte offset too. */ + set_pte_at(f->mm, TEST_VA + (f->nr - 1) * size, pte + f->nr - 1, + pte_advance_pfn_mm(f->mm, base, f->nr - 1)); + for (i = 0; i < f->nr; i++) + ok.refolded &= pte_cont(__ptep_get(pte + i)); + get_and_clear_full_ptes(f->mm, TEST_VA + 3 * size, pte + 3, 2, 0); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.holes &= pte_none(observed) == (i == 3 || i == 4); + if (i != 3 && i != 4) + ok.holes &= !pte_cont(observed) && + pte_phys_mm(f->mm, observed) == phys + i * size; + } + set_ptes(f->mm, TEST_VA + 3 * size, pte + 3, + pte_advance_pfn_mm(f->mm, base, 3), 2); + set_pte_at(f->mm, TEST_VA + (f->nr - 1) * size, pte + f->nr - 1, + pte_advance_pfn_mm(f->mm, base, f->nr - 1)); + for (i = 0; i < f->nr; i++) + ok.refolded &= pte_cont(__ptep_get(pte + i)); + clear_full_ptes(f->mm, TEST_VA, pte, f->nr, 0); + for (i = 0; i < f->nr; i++) + ok.full_clear &= pte_none(__ptep_get(pte + i)); + ok.sentinel = pte_same(__ptep_get(pte + f->nr), saved); + pte_unmap_unlock(pte, ptl); + mmap_read_unlock(f->mm); + KUNIT_EXPECT_TRUE(test, ok.initial); + KUNIT_EXPECT_TRUE(test, ok.addresses); + KUNIT_EXPECT_TRUE(test, ok.hints); + KUNIT_EXPECT_TRUE(test, ok.gathered); + KUNIT_EXPECT_TRUE(test, ok.aged); + KUNIT_EXPECT_TRUE(test, ok.access_all); + KUNIT_EXPECT_EQ(test, access_same_ret, 1); + KUNIT_EXPECT_EQ(test, access_noop_ret, 0); + KUNIT_EXPECT_EQ(test, access_ret, 1); + KUNIT_EXPECT_TRUE(test, ok.unfolded); + KUNIT_EXPECT_TRUE(test, ok.permissions); + KUNIT_EXPECT_TRUE(test, ok.refolded); + KUNIT_EXPECT_TRUE(test, ok.holes); + KUNIT_EXPECT_TRUE(test, ok.full_clear); + KUNIT_EXPECT_TRUE(test, ok.sentinel); +} + +static void user4k_contpte_test(struct kunit *test) +{ + test_contpte_granule(test, true); +} + +static void native_contpte_test(struct kunit *test) +{ + test_contpte_granule(test, false); +} +#endif + +struct usercopy_worker { + struct test_mm *ctx[2]; + struct completion done; + unsigned int id; + unsigned int checks; + unsigned int idmaps; + int error; + u64 tcr; +}; + +static int test_usercopy_worker(void *arg) +{ + struct usercopy_worker *worker = arg; + u64 kernel_ttbr = read_sysreg(ttbr1_el1) & ~TTBRx_EL1_ASID_MASK; + unsigned int round, i, byte; + bool borrowed = false; + struct test_mm *ctx = NULL; + + for (round = 0; round < 16; round++) { + unsigned int small = (round + worker->id) & 1; + unsigned int cpu = cpumask_first(cpu_online_mask); + + ctx = worker->ctx[small]; + kthread_use_mm(ctx->mm); + borrowed = true; + if ((round + worker->id) & 2) { + unsigned int next = cpumask_next(cpu, cpu_online_mask); + + if (next < nr_cpu_ids) + cpu = next; + } + if (set_cpus_allowed_ptr(current, cpumask_of(cpu))) { + worker->error = 1; + break; + } + /* Exercise a real scheduler switch while borrowing this mm. */ + schedule_timeout_uninterruptible(1); + if (small && round < 4) { + unsigned long flags; + bool native; + + preempt_disable(); + flags = local_daif_save(); + if (round & 2) + cpu_install_ttbr0(phys_to_ttbr(__pa_symbol(idmap_pg_dir)), + TCR_T0SZ(IDMAP_VA_BITS)); + else + cpu_install_idmap(); + native = (read_sysreg(tcr_el1) & (TCR_EL1_TG0_MASK | TCR_EL1_T0SZ_MASK)) == + (TCR_TG0_16K | TCR_T0SZ(IDMAP_VA_BITS)); + cpu_uninstall_idmap(); + local_daif_restore(flags); + preempt_enable(); + if (!native) { + worker->error = 6; + break; + } + worker->idmaps++; + } + worker->tcr = read_sysreg(tcr_el1); + if ((worker->tcr & TCR_EL1_TG0_MASK) != (small ? TCR_TG0_4K : TCR_TG0_16K) || + (worker->tcr & TCR_EL1_T0SZ_MASK) != TCR_T0SZ(48) || + (worker->tcr & TCR_EL1_TG1_MASK) != TCR_TG1_16K || + (read_sysreg(ttbr1_el1) & ~TTBRx_EL1_ASID_MASK) != kernel_ttbr) { + worker->error = 2; + break; + } + for (i = 0; i < 4; i++) { + unsigned long offset = 64 + worker->id * sizeof(u64); + void __user *address = (void __user *)ctx->address[i]; + u64 value = 0xa5000000 | (worker->id << 16) | (round << 8) | i; + u64 *backing = phys_to_virt(ctx->phys[i]) + offset; + unsigned long read_left, write_left; + u8 data[8]; + + /* No VMA/fault path may be entered for these private mappings. */ + pagefault_disable(); + read_left = copy_from_user(data, address, sizeof(data)); + write_left = copy_to_user(address + offset, &value, sizeof(value)); + pagefault_enable(); + if (read_left != (i == 2 ? sizeof(data) : 0) || + write_left != (i == 0 || i == 2 ? sizeof(value) : 0)) { + worker->error = 3; + goto out; + } + if (i != 2) { + for (byte = 0; byte < sizeof(data); byte++) { + if (data[byte] != (small ? 0x40 : 0x80) + i) { + worker->error = 4; + goto out; + } + } + } + if (READ_ONCE(*backing) != (i == 0 || i == 2 ? 0 : value)) { + worker->error = 5; + goto out; + } + worker->checks++; + } + kthread_unuse_mm(ctx->mm); + borrowed = false; + } +out: + if (borrowed) + kthread_unuse_mm(ctx->mm); + complete(&worker->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void test_stop_kthread(void *arg) +{ + kthread_stop(arg); +} + +struct cow_reader { + struct mm_struct *parent; + struct mm_struct *child; + struct completion ready; + struct completion copied; + struct completion done; + u8 observed[8]; + unsigned long read_left[8], write_left[8]; + int error; + unsigned int checks; +}; + +static int test_cow_reader(void *arg) +{ + struct cow_reader *r = arg; + unsigned int i, side; + + kthread_use_mm(r->child); + /* Resolve fork's old PTEs normally, then cache pre-COW translations. */ + for (i = 0; i < 4; i++) { + u8 value = 0; + unsigned long left; + + left = copy_from_user(&value, (void __user *)(TEST_VA + i * SZ_4K + 64), 1); + if (left || value != 0x41 + i) + r->error = 1; + } + complete(&r->ready); + if (!wait_for_completion_timeout(&r->copied, 10 * HZ)) { + r->error = 2; + goto out; + } + for (side = 0; side < 2; side++) { + if (side) { + kthread_unuse_mm(r->child); + kthread_use_mm(r->parent); + } + for (i = 0; i < 4; i++) { + void __user *addr = (void __user *)(TEST_VA + i * SZ_4K + 64); + u8 value = 0, changed = 0x99; + unsigned long read_left, write_left; + + pagefault_disable(); + read_left = copy_from_user(&value, addr, 1); + write_left = copy_to_user(addr, &changed, 1); + pagefault_enable(); + r->observed[side * 4 + i] = value; + r->read_left[side * 4 + i] = read_left; + r->write_left[side * 4 + i] = write_left; + if (read_left || value != 0x41 + i || + write_left != (side == 0 && i == 1 ? 0 : 1)) + r->error = 3; + r->checks++; + } + } + kthread_unuse_mm(r->parent); + goto done; +out: + kthread_unuse_mm(r->child); +done: + complete(&r->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void user4k_fault_uaccess_test(struct kunit *test) +{ + struct vm_area_struct *parent, *child_vma; + struct mm_struct *child; + struct cow_reader *reader; + struct task_struct *task; + unsigned int i, cpu; + vm_fault_t fault; + int ret; + u64 tcr = read_sysreg(tcr_el1); + + if (!user4k_test_enabled || !system_supports_4kb_granule() || + system_supports_cnp() || cpus_have_final_cap(ARM64_HAS_S1PIE) || + cpus_have_final_cap(ARM64_HAS_S1POE) || + (tcr & (TCR_EL1_DS | TCR_EL1_A1 | TCR_EL1_T0SZ_MASK)) != + (TCR_EL1_A1 | TCR_T0SZ(48))) { + kunit_skip(test, "requires opt-in and compatible 4K TTBR0 controls"); + return; + } + parent = test_fault_vma(test, TEST_VA, PAGE_SIZE); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, parent); + KUNIT_ASSERT_NOT_NULL(test, child); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + for (i = 0; i < 4; i++) { + pte_t entry = test_fault_entry(test, parent->vm_mm, TEST_VA + i * SZ_4K); + + memset(phys_to_virt(pte_phys_mm(parent->vm_mm, entry)), 0x41 + i, SZ_4K); + } + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + child_vma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, child_vma); + reader = kunit_kzalloc(test, sizeof(*reader), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, reader); + reader->parent = parent->vm_mm; + reader->child = child; + init_completion(&reader->ready); + init_completion(&reader->copied); + init_completion(&reader->done); + task = kthread_create(test_cow_reader, reader, "user4k-cow-reader"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, task); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_stop_kthread, task), 0); + cpu = cpumask_last(cpu_online_mask); + kthread_bind(task, cpu); + wake_up_process(task); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&reader->ready, 10 * HZ), 0UL); + mmap_read_lock(child); + fault = handle_mm_fault(child_vma, TEST_VA + SZ_4K, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + complete(&reader->copied); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&reader->done, 10 * HZ), 0UL); + if (reader->error) + for (i = 0; i < 8; i++) + kunit_info(test, "access %u value=%#x read_left=%lu write_left=%lu\n", + i, reader->observed[i], reader->read_left[i], reader->write_left[i]); + KUNIT_EXPECT_EQ(test, reader->error, 0); + KUNIT_EXPECT_EQ(test, reader->checks, 8U); + { + pte_t entry = test_fault_entry(test, child, TEST_VA + SZ_4K); + + KUNIT_EXPECT_EQ(test, *(u8 *)(phys_to_virt(pte_phys_mm(child, entry)) + 64), (u8)0x99); + entry = test_fault_entry(test, parent->vm_mm, TEST_VA + SZ_4K); + KUNIT_EXPECT_EQ(test, *(u8 *)(phys_to_virt(pte_phys_mm(parent->vm_mm, entry)) + 64), (u8)0x42); + } +} + +static void user4k_context_uaccess_test(struct kunit *test) +{ + struct test_mm *ctx[2]; + struct usercopy_worker *workers; + unsigned int i; + u64 tcr = read_sysreg(tcr_el1); + + if (!user4k_test_enabled) { + kunit_skip(test, "requires arm64.user4k_test=1 in a disposable VM"); + return; + } + if (!system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || cpus_have_final_cap(ARM64_HAS_S1POE) || + (tcr & (TCR_EL1_DS | TCR_EL1_A1 | TCR_EL1_T0SZ_MASK)) != + (TCR_EL1_A1 | TCR_T0SZ(48))) { + kunit_skip(test, "requires 4K, no CnP/PIE/POE, DS=0, A1=1 and 48-bit TTBR0"); + return; + } + ctx[0] = test_usercopy_mm(test, false); + KUNIT_ASSERT_NOT_NULL(test, ctx[0]); + ctx[1] = test_usercopy_mm(test, true); + KUNIT_ASSERT_NOT_NULL(test, ctx[1]); + workers = kunit_kcalloc(test, 2, sizeof(*workers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + for (i = 0; i < 2; i++) { + struct task_struct *task; + + workers[i].ctx[0] = ctx[0]; + workers[i].ctx[1] = ctx[1]; + workers[i].id = i; + init_completion(&workers[i].done); + task = kthread_run(test_usercopy_worker, &workers[i], "user4k-copy-%u", i); + KUNIT_ASSERT_FALSE(test, IS_ERR(task)); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_stop_kthread, task), 0); + } + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].done, 10 * HZ), 0UL); + KUNIT_EXPECT_EQ_MSG(test, workers[i].error, 0, "worker %u TCR=%#llx", i, workers[i].tcr); + KUNIT_EXPECT_EQ(test, workers[i].checks, 64U); + KUNIT_EXPECT_EQ(test, workers[i].idmaps, 2U); + } + kunit_info(test, "two migrating kthreads, 32 borrowed-mm switches, %s PAN\n", + system_uses_ttbr0_pan() ? "software" : "hardware"); +} + +struct zap_reader { + struct mm_struct *mm; + struct task_struct *task; + struct completion ready; + struct completion proceed; + struct completion done; + unsigned int cpu; + unsigned int checks; + bool zapped; + int error; +}; + +static int test_zap_reader(void *arg) +{ + struct zap_reader *reader = arg; + unsigned int phase, i; + + reader->error = set_cpus_allowed_ptr(current, cpumask_of(reader->cpu)); + if (reader->error) { + complete(&reader->ready); + goto out; + } + kthread_use_mm(reader->mm); + for (phase = 0; phase < 2; phase++) { + if (phase) { + complete(&reader->ready); + wait_for_completion(&reader->proceed); + if (!reader->zapped) + break; + } + for (i = 0; i < 4; i++) { + u64 value = 0, expected = 0xa5a50000UL + i; + unsigned long left; + + pagefault_disable(); + left = copy_from_user(&value, + (void __user *)((unsigned long)TEST_VA + i * SZ_4K), sizeof(value)); + pagefault_enable(); + if (phase && i == 1) { + if (!left) + reader->error = -EUCLEAN; + } else if (left || value != expected) { + reader->error = -EFAULT; + } + reader->checks++; + } + } + kthread_unuse_mm(reader->mm); +out: + complete(&reader->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void test_stop_zap_reader(void *arg) +{ + struct zap_reader *reader = arg; + + complete(&reader->proceed); + kthread_stop(reader->task); +} + +static void user4k_anon_remote_zap_test(struct kunit *test) +{ + struct anon_slot_fixture *f; + struct zap_reader *readers; + unsigned int i, cpu; + u64 tcr = read_sysreg(tcr_el1); + + if (!user4k_test_enabled || num_online_cpus() < 2) { + kunit_skip(test, "requires arm64.user4k_test=1 and two CPUs in a disposable VM"); + return; + } + if (!system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || cpus_have_final_cap(ARM64_HAS_S1POE) || + (tcr & (TCR_EL1_DS | TCR_EL1_A1 | TCR_EL1_T0SZ_MASK)) != + (TCR_EL1_A1 | TCR_T0SZ(48))) { + kunit_skip(test, "requires 4K, no CnP/PIE/POE, DS=0, A1=1 and 48-bit TTBR0"); + return; + } + f = anon_slot_fixture_create(test); + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(f->pool, i * SZ_4K); + u64 value = 0xa5a50000UL + i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + memcpy(folio_address(mm_subpage_folio(slot)) + mm_subpage_offset(slot), + &value, sizeof(value)); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(f, i, slot), 0); + } + readers = kunit_kcalloc(test, 2, sizeof(*readers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, readers); + cpu = cpumask_first(cpu_online_mask); + for (i = 0; i < 2; i++) { + readers[i].mm = f->mm; + readers[i].cpu = cpu; + init_completion(&readers[i].ready); + init_completion(&readers[i].proceed); + init_completion(&readers[i].done); + readers[i].task = kthread_run(test_zap_reader, &readers[i], "user4k-zap-%u", i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, readers[i].task); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_stop_zap_reader, + &readers[i]), 0); + cpu = cpumask_next(cpu, cpu_online_mask); + } + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&readers[i].ready, 10 * HZ), 0UL); + KUNIT_ASSERT_EQ(test, readers[i].error, 0); + } + /* Keep an explicit non-mapping reference for fixture cleanup. */ + mm_subpage_get(f->slots[1]); + mmap_read_lock(f->mm); + zap_vma_range(f->vma, TEST_VA + SZ_4K, SZ_4K); + f->mapped[1] = false; + mmap_read_unlock(f->mm); + for (i = 0; i < 2; i++) { + readers[i].zapped = true; + complete(&readers[i].proceed); + } + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&readers[i].done, 10 * HZ), 0UL); + KUNIT_EXPECT_EQ(test, readers[i].error, 0); + KUNIT_EXPECT_EQ(test, readers[i].checks, 8U); + } +} + +static u64 *test_leaf(struct test_tables *tables, unsigned long va) +{ + return &tables->level[3][test_index(va, tables->shift, 3)]; +} + +/* Explicitly 4K, without the native PAGE_SHIFT or a granule/level hint. */ +static void test_invalidate_4k(unsigned long va, u64 ttbr1) +{ + u64 operand = (va >> USER4K_SHIFT) | (ttbr1 & GENMASK_ULL(63, 48)); + + dsb(ishst); + __tlbi(vae1, operand); + dsb(ish); + isb(); +} + +static void user4k_translation_test(struct kunit *test) +{ + struct test_tables small, native; + struct test_result *result; + unsigned long flags; + void *data, *other; + phys_addr_t data_pa, other_pa; + u64 old_tcr, old_ttbr0, old_ttbr1, old_par, tcr4k, tcr16k; + unsigned int round, slot; + + if (!user4k_test_enabled) { + kunit_skip(test, "requires arm64.user4k_test=1 in a disposable VM"); + return; + } + if (!system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || + cpus_have_final_cap(ARM64_HAS_S1POE)) { + kunit_skip(test, "requires 4K, no CnP, and direct PTE permissions"); + return; + } + + KUNIT_ASSERT_EQ(test, PAGE_SHIFT, 14); + KUNIT_ASSERT_EQ(test, test_alloc_tables(test, &small, USER4K_SHIFT), 0); + KUNIT_ASSERT_EQ(test, test_alloc_tables(test, &native, PAGE_SHIFT), 0); + data = test_alloc_page(test); + other = test_alloc_page(test); + KUNIT_ASSERT_NOT_NULL(test, data); + KUNIT_ASSERT_NOT_NULL(test, other); + data_pa = virt_to_phys(data); + other_pa = virt_to_phys(other); + result = kunit_kcalloc(test, TEST_ROUNDS, sizeof(*result), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, result); + + /* One native mapping and four independently described subpages. */ + *test_leaf(&native, TEST_VA) = other_pa | TEST_PROT; + *test_leaf(&small, TEST_VA) = data_pa | TEST_PROT | PTE_RDONLY; + *test_leaf(&small, TEST_VA + USER4K_SIZE) = + (data_pa + USER4K_SIZE) | TEST_PROT; + /* Slot 2 deliberately unmapped. Slot 3 aliases slot 0, writable. */ + *test_leaf(&small, TEST_VA + 3 * USER4K_SIZE) = data_pa | TEST_PROT; + + preempt_disable(); + flags = local_daif_save(); + old_tcr = read_sysreg(tcr_el1); + old_ttbr0 = read_sysreg(ttbr0_el1); + old_ttbr1 = read_sysreg(ttbr1_el1); + old_par = read_sysreg_par(); + /* DS affects both address ranges: never toggle it under a live TTBR1. */ + if ((old_tcr & (TCR_EL1_DS | TCR_EL1_TG1_MASK | TCR_EL1_A1)) != + (TCR_TG1_16K | TCR_EL1_A1)) { + local_daif_restore(flags); + preempt_enable(); + kunit_skip(test, "requires 16K TTBR1, DS=0, and A1=1"); + return; + } + tcr4k = (old_tcr & ~(TCR_EL1_TG0_MASK | TCR_EL1_T0SZ_MASK | + TCR_EL1_EPD0)) | TCR_TG0_4K | TCR_T0SZ(48); + tcr16k = (tcr4k & ~TCR_EL1_TG0_MASK) | TCR_TG0_16K; + + /* No allocations, logging, KUnit assertions or user access in this window. */ + for (round = 0; round < TEST_ROUNDS; round++) { + struct test_result *r = &result[round]; + + test_switch_ttbr0(virt_to_phys(native.level[0]), tcr16k); + for (slot = 0; slot < PROBE_SLOTS; slot++) + r->native[slot] = test_translate(TEST_VA + slot * USER4K_SIZE, false); + test_switch_ttbr0(virt_to_phys(small.level[0]), tcr4k); + r->tcr = read_sysreg(tcr_el1); + r->ttbr1 = read_sysreg(ttbr1_el1); + for (slot = 0; slot < PROBE_SLOTS; slot++) { + unsigned long va = TEST_VA + slot * USER4K_SIZE; + + r->read[slot] = test_translate(va, false); + r->write[slot] = test_translate(va, true); + } + + /* Break-before-make one 4K PTE and invalidate exactly that VA. */ + WRITE_ONCE(*test_leaf(&small, TEST_VA + USER4K_SIZE), 0); + test_invalidate_4k(TEST_VA + USER4K_SIZE, old_ttbr1); + WRITE_ONCE(*test_leaf(&small, TEST_VA + USER4K_SIZE), + (other_pa + USER4K_SIZE) | TEST_PROT | PTE_RDONLY); + dsb(ishst); + isb(); + for (slot = 0; slot < PROBE_SLOTS; slot++) + r->replaced[slot] = test_translate(TEST_VA + slot * USER4K_SIZE, true); + r->replaced_read = test_translate(TEST_VA + USER4K_SIZE, false); + asm volatile("at s1e1r, %0" : : "r" (data) : "memory"); + isb(); + r->kernel = read_sysreg_par(); + + test_switch_ttbr0(old_ttbr0, old_tcr); + r->restored_tcr = read_sysreg(tcr_el1); + r->restored_ttbr0 = read_sysreg(ttbr0_el1); + /* Private tables are inactive and their translations have been flushed. */ + WRITE_ONCE(*test_leaf(&small, TEST_VA + USER4K_SIZE), + (data_pa + USER4K_SIZE) | TEST_PROT); + } + write_sysreg(old_par, par_el1); + isb(); + local_daif_restore(flags); + preempt_enable(); + + for (round = 0; round < TEST_ROUNDS; round++) { + struct test_result *r = &result[round]; + + KUNIT_EXPECT_EQ(test, r->tcr, tcr4k); + KUNIT_EXPECT_EQ(test, r->ttbr1, old_ttbr1); + KUNIT_EXPECT_EQ(test, r->restored_tcr, old_tcr); + KUNIT_EXPECT_EQ(test, r->restored_ttbr0, old_ttbr0); + KUNIT_EXPECT_EQ(test, r->kernel & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, r->kernel & SYS_PAR_EL1_PA, data_pa); + KUNIT_EXPECT_EQ(test, r->replaced_read & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, r->replaced_read & SYS_PAR_EL1_PA, + other_pa + USER4K_SIZE); + for (slot = 0; slot < PROBE_SLOTS; slot++) { + KUNIT_EXPECT_EQ(test, r->native[slot] & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, r->native[slot] & SYS_PAR_EL1_PA, + other_pa + slot * USER4K_SIZE); + if (slot == 2) { + KUNIT_EXPECT_EQ(test, + FIELD_GET(SYS_PAR_EL1_FST, r->read[slot]), 7ULL); + KUNIT_EXPECT_EQ(test, r->read[slot] & SYS_PAR_EL1_F, 1ULL); + KUNIT_EXPECT_EQ(test, r->write[slot] & SYS_PAR_EL1_F, 1ULL); + KUNIT_EXPECT_EQ(test, r->replaced[slot] & SYS_PAR_EL1_F, 1ULL); + continue; + } + KUNIT_EXPECT_EQ(test, r->read[slot] & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, r->read[slot] & SYS_PAR_EL1_PA, + data_pa + (slot == 1 ? USER4K_SIZE : 0)); + KUNIT_EXPECT_EQ(test, r->write[slot] & SYS_PAR_EL1_F, + slot == 0 ? 1ULL : 0ULL); + KUNIT_EXPECT_EQ(test, r->replaced[slot] & SYS_PAR_EL1_F, + slot == 3 ? 0ULL : 1ULL); + } + } + kunit_info(test, "16K kernel, 4K EL0 translation/permissions; %u switch cycles\n", + TEST_ROUNDS); +} + +static struct folio *test_fault_folio(struct mm_struct *mm, unsigned long address) +{ + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, address, &ptl); + struct folio *folio = NULL; + + if (pte) { + pte_t entry = ptep_get(pte); + + if (pte_present(entry) && !pte_special(entry)) { + folio = page_folio(pfn_to_page(pte_pfn(entry))); + folio_get(folio); + } + pte_unmap_unlock(pte, ptl); + } + return folio; +} + +static bool test_reclaim_swapcache(swp_entry_t entry) +{ + unsigned int tries; + + for (tries = 0; tries < 100; tries++) { + struct folio *folio = swap_cache_get_folio(entry); + LIST_HEAD(folios); + + if (!folio) + return true; + folio_wait_writeback(folio); + lru_add_drain_all(); + if (folio_isolate_lru(folio)) + list_add(&folio->lru, &folios); + folio_put(folio); + /* Only the swap cache and reclaim isolation may retain this folio. */ + reclaim_pages(&folios); + schedule_timeout_uninterruptible(1); + } + return false; +} + +static void test_swap_tags_set(struct folio *folio) +{ +#ifdef CONFIG_ARM64_MTE + unsigned int offset; + + if (!system_supports_mte()) + return; + if (try_page_mte_tagging(&folio->page)) + mte_clear_page_tags(folio_address(folio)); + for (offset = 0; offset < PAGE_SIZE; offset += MTE_GRANULE_SIZE) { + unsigned long tag = (1UL + offset / SZ_4K) << MTE_TAG_SHIFT; + + asm volatile(".arch_extension memtag\n\tstg %0, [%1]" + : : "r" (tag), "r" (folio_address(folio) + offset) : "memory"); + } + set_page_mte_tagged(&folio->page); +#endif +} + +static bool test_swap_tags_match(struct folio *folio, unsigned int slot) +{ +#ifdef CONFIG_ARM64_MTE + unsigned int offset; + + if (!system_supports_mte()) + return true; + if (!page_mte_tagged(&folio->page)) + return false; + for (offset = slot * SZ_4K; offset < (slot + 1) * SZ_4K; offset += MTE_GRANULE_SIZE) { + unsigned long tag = 0; + + asm volatile(".arch_extension memtag\n\tldg %0, [%1]" + : "+r" (tag) : "r" (folio_address(folio) + offset) : "memory"); + if (((tag >> MTE_TAG_SHIFT) & 0xf) != ((slot + 1) & 0xf)) + return false; + } +#endif + return true; +} + +struct fragment_test_pins { + struct user_page_fragment fragments[2 * TEST_SLOTS]; + unsigned long count; +}; + +static void fragment_test_unpin(void *arg) +{ + struct fragment_test_pins *pins = arg; + + release_user_fragments(pins->fragments, pins->count, true); + pins->count = 0; +} + +static long fragment_test_pin(struct fragment_test_pins *pins, struct mm_struct *mm, + unsigned long start, unsigned long length, unsigned int flags, + unsigned long capacity) +{ + long ret; + + fragment_test_unpin(pins); + ret = pin_user_fragments_remote(mm, start, length, flags, pins->fragments, capacity); + pins->count = ret > 0 ? ret : 0; + return ret; +} + +static void user4k_fragment_pin_test(struct kunit *test) +{ + unsigned int native; + + for (native = 0; native < 2; native++) { + struct fragment_test_pins *pins; + struct vm_area_struct *vma; + struct mm_struct *mm, *child; + unsigned long size, total = 0; + unsigned int i, nr; + long ret; + + if (native) { + mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + vma = test_fault_vma_in_mm(mm, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + } else { + vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + mm = vma->vm_mm; + } + size = mm_page_size(mm); + nr = PAGE_SIZE / size; + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, mm, TEST_VA, 1, FOLL_WRITE | FOLL_NOFAULT, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + ret = fragment_test_pin(pins, mm, TEST_VA + 123, PAGE_SIZE - 200, FOLL_WRITE, ARRAY_SIZE(pins->fragments)); + KUNIT_ASSERT_EQ(test, ret, (long)nr); + for (i = 0; i < nr; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + unsigned int offset = i ? i * size : 123; + unsigned int length = size - (!i ? 123 : 0) - (i == nr - 1 ? 77 : 0); + + KUNIT_EXPECT_EQ(test, f->offset, (unsigned long)offset); + KUNIT_EXPECT_EQ(test, f->length, length); + KUNIT_EXPECT_EQ(test, !!f->subpage, !native); + memset(folio_address(f->folio) + f->offset, 0x71 + i, f->length); + total += f->length; + } + KUNIT_EXPECT_EQ(test, total, PAGE_SIZE - 200); + fragment_test_unpin(pins); + child = native ? mm_alloc() : test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + if (native) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, child), 0); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0L); + /* Ordinary references retain shared COW backing, as native FOLL_GET does. */ + ret = get_user_fragments_remote(child, TEST_VA + 123, PAGE_SIZE - 200, + 0, pins->fragments, ARRAY_SIZE(pins->fragments)); + pins->count = ret > 0 ? ret : 0; + KUNIT_ASSERT_EQ(test, ret, (long)nr); + for (i = 0; i < nr; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + pte_t parent_pte = test_fault_entry(test, mm, TEST_VA + i * size); + pte_t child_pte = test_fault_entry(test, child, TEST_VA + i * size); + + KUNIT_EXPECT_FALSE(test, f->pinned); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, parent_pte), pte_phys_mm(child, child_pte)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x71 + i, f->length), NULL); + } + fragment_test_unpin(pins); + /* Read pins must unshare COW mappings without enabling PTE writes. */ + ret = fragment_test_pin(pins, child, TEST_VA + 123, PAGE_SIZE - 200, FOLL_NOFAULT, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + ret = fragment_test_pin(pins, child, TEST_VA + 123, PAGE_SIZE - 200, 0, ARRAY_SIZE(pins->fragments)); + KUNIT_ASSERT_EQ(test, ret, (long)nr); + for (i = 0; i < nr; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + pte_t pte = test_fault_entry(test, child, TEST_VA + i * size); + + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x71 + i, f->length), NULL); + if (!native) { + KUNIT_EXPECT_TRUE(test, mm_subpage_is_exclusive(f->subpage)); + if (i) + KUNIT_EXPECT_PTR_NE(test, f->folio, pins->fragments[0].folio); + } + } + fragment_test_unpin(pins); + /* Return a valid partial result when the byte range reaches an unmapped VMA. */ + ret = fragment_test_pin(pins, mm, TEST_VA, PAGE_SIZE + size, FOLL_WRITE, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, (long)nr); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, 1); + KUNIT_EXPECT_EQ(test, ret, 1L); + KUNIT_EXPECT_EQ(test, pins->fragments[0].length, (unsigned int)size); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, mm, TEST_VA, size, FOLL_LONGTERM, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, 1L); + ret = fragment_test_pin(pins, mm, ULONG_MAX - 15, ULONG_MAX, 0, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +#ifdef CONFIG_BLOCK +struct bio_fragment_test { + struct bio *bio; + struct bio *clone; + struct user_page_fragment held[TEST_SLOTS]; + unsigned int count; +}; + +static void bio_fragment_test_free(void *arg) +{ + struct bio_fragment_test *state = arg; + + if (state->clone) + bio_put(state->clone); + if (state->bio) { + bio_release_pages(state->bio, false); + bio_put(state->bio); + } + release_user_fragments(state->held, state->count, false); +} + +static void user4k_bio_fragments_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct bio_fragment_test *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct iov_iter iter; + unsigned int i, bytes = round_down(PAGE_SIZE - 200, 512); + long nr; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bio fixture requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, bio_fragment_test_free, state), 0); + /* Populate real writable anonymous mappings and retain independent GETs. */ + nr = get_user_fragments_remote(vma->vm_mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, + state->held, ARRAY_SIZE(state->held)); + state->count = nr > 0 ? nr : 0; + KUNIT_ASSERT_EQ(test, nr, (long)TEST_SLOTS); + state->bio = bio_alloc(NULL, 8, REQ_OP_READ, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->bio); + iov_iter_ubuf(&iter, ITER_DEST, (void __user *)(TEST_VA + 123), PAGE_SIZE - 200); + kthread_use_mm(vma->vm_mm); + ret = bio_iov_vecs_to_alloc(&iter, BIO_MAX_VECS); + kthread_unuse_mm(vma->vm_mm); + KUNIT_EXPECT_EQ(test, ret, (int)TEST_SLOTS); + kthread_use_mm(vma->vm_mm); + ret = bio_iov_iter_get_pages(state->bio, &iter, 0, 511); + kthread_unuse_mm(vma->vm_mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, state->bio->bi_iter.bi_size, bytes); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), PAGE_SIZE - 200 - bytes); + KUNIT_EXPECT_EQ(test, state->bio->bi_nr_user_fragments, (int)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, state->bio->bi_vcnt, 1); + KUNIT_EXPECT_EQ(test, state->bio->bi_io_vec[0].bv_offset, 123U); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(state->held[i].subpage), 1U); + state->clone = bio_alloc_clone(NULL, state->bio, GFP_KERNEL, &fs_bio_set); + KUNIT_ASSERT_NOT_NULL(test, state->clone); + KUNIT_EXPECT_PTR_EQ(test, state->clone->bi_user_fragments, NULL); + KUNIT_EXPECT_PTR_EQ(test, state->clone->bi_io_vec, state->bio->bi_io_vec); + /* Completion ownership must outlive all originating VMAs. */ + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(state->held[i].subpage), 1U); + bio_put(state->clone); + state->clone = NULL; + bio_release_pages(state->bio, false); + KUNIT_EXPECT_PTR_EQ(test, state->bio->bi_user_fragments, NULL); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(state->held[i].subpage), 0U); +} + +static void user4k_bio_trim_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, SZ_4K); + struct bio_fragment_test *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct iov_iter iter; + unsigned int i; + long nr; + int ret, refs; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bio fixture requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, bio_fragment_test_free, state), 0); + nr = get_user_fragments_remote(vma->vm_mm, TEST_VA, SZ_4K, FOLL_WRITE, state->held, 1); + state->count = nr > 0 ? nr : 0; + KUNIT_ASSERT_EQ(test, nr, 1L); + refs = folio_ref_count(state->held[0].folio); + state->bio = bio_alloc(NULL, 4, REQ_OP_READ, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->bio); + for (i = 0; i < 1024; i++) { + iov_iter_ubuf(&iter, ITER_DEST, (void __user *)(TEST_VA + SZ_4K - 256), 512); + kthread_use_mm(vma->vm_mm); + ret = bio_iov_iter_get_pages(state->bio, &iter, 0, 511); + kthread_unuse_mm(vma->vm_mm); + KUNIT_ASSERT_EQ(test, ret, -EFAULT); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 512UL); + KUNIT_EXPECT_EQ(test, state->bio->bi_vcnt, 0); + KUNIT_EXPECT_PTR_EQ(test, state->bio->bi_user_fragments, NULL); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(state->held[0].subpage), 0U); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->held[0].folio), refs); + bio_reset(state->bio, NULL, REQ_OP_READ); + } +} +#endif + +static void user4k_fragment_longterm_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct fragment_test_pins *pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + phys_addr_t before; + bool movable; + unsigned int i; + long ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, ARRAY_SIZE(pins->fragments)); + KUNIT_ASSERT_EQ(test, ret, (long)TEST_SLOTS); + before = PFN_PHYS(folio_pfn(pins->fragments[0].folio)); + movable = !folio_is_longterm_pinnable(pins->fragments[0].folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(pins->fragments[i].folio) + pins->fragments[i].offset, + 0x91 + i, pins->fragments[i].length); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA, PAGE_SIZE, + FOLL_WRITE | FOLL_LONGTERM, ARRAY_SIZE(pins->fragments)); + KUNIT_ASSERT_EQ(test, ret, (long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + + KUNIT_EXPECT_TRUE(test, folio_is_longterm_pinnable(f->folio)); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(f->subpage), 1U); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x91 + i, f->length), NULL); + } + if (movable) + KUNIT_EXPECT_NE(test, PFN_PHYS(folio_pfn(pins->fragments[0].folio)), before); + kunit_info(test, "long-term source movable=%u migrated=%u\n", movable, + PFN_PHYS(folio_pfn(pins->fragments[0].folio)) != before); + fragment_test_unpin(pins); + /* A long-term flag must never silently create an ordinary GET. */ + ret = get_user_fragments_remote(vma->vm_mm, TEST_VA, SZ_4K, FOLL_LONGTERM, + pins->fragments, 1); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + +static void user4k_file_fragments_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *shared, *private; + struct fragment_test_pins *pins; + phys_addr_t shared_phys; + long ret; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shared = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, 3 * SZ_4K, SZ_4K); + private = test_file_vma(test, file, false, false, TEST_VA + SZ_4K, 3 * SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, shared); + KUNIT_ASSERT_NOT_NULL(test, private); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, shared->vm_mm, shared->vm_start + 17, + 3 * SZ_4K - 33, FOLL_WRITE, 8); + KUNIT_ASSERT_EQ(test, ret, 3L); + shared_phys = PFN_PHYS(folio_pfn(pins->fragments[0].folio)); + for (i = 0; i < 3; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + + KUNIT_EXPECT_PTR_EQ(test, f->subpage, NULL); + KUNIT_EXPECT_EQ(test, f->offset, (unsigned long)((i + 1) * SZ_4K + (!i ? 17 : 0))); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x32 + i, f->length), NULL); + } + /* File pins stay live across PTE removal and mm exit. */ + kunit_release_action(test, user4k_vma_mm_free, shared->vm_mm); + for (i = 0; i < 3; i++) + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(pins->fragments[i].folio)); + fragment_test_unpin(pins); + ret = get_user_fragments_remote(private->vm_mm, private->vm_start, 3 * SZ_4K, + 0, pins->fragments, 8); + pins->count = ret > 0 ? ret : 0; + KUNIT_ASSERT_EQ(test, ret, 3L); + for (i = 0; i < 3; i++) { + KUNIT_EXPECT_PTR_EQ(test, pins->fragments[i].subpage, NULL); + KUNIT_EXPECT_EQ(test, PFN_PHYS(folio_pfn(pins->fragments[i].folio)), shared_phys); + } + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, private->vm_mm, private->vm_start, 3 * SZ_4K, 0, 8); + KUNIT_ASSERT_EQ(test, ret, 3L); + for (i = 0; i < 3; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + + KUNIT_EXPECT_NOT_NULL(test, f->subpage); + KUNIT_EXPECT_NE(test, PFN_PHYS(folio_pfn(f->folio)), shared_phys); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x32 + i, f->length), NULL); + KUNIT_EXPECT_FALSE(test, pte_write(test_fault_entry(test, private->vm_mm, + private->vm_start + i * SZ_4K))); + } + fragment_test_unpin(pins); +} + +/* Exercise the same implementation used by the arm64 syscall entry. */ +asmlinkage long __arm64_sys_mprotect(const struct pt_regs *regs); + +static long test_mprotect(unsigned long start, unsigned long len, unsigned long prot) +{ + struct pt_regs regs = { .regs = { start, len, prot } }; + + return __arm64_sys_mprotect(®s); +} + +static unsigned long test_copy_word(unsigned long addr, u64 *value, bool write) +{ + unsigned long left; + + pagefault_disable(); + if (write) + left = copy_to_user((void __user *)addr, value, sizeof(*value)); + else + left = copy_from_user(value, (void __user *)addr, sizeof(*value)); + pagefault_enable(); + return left; +} + +static void user4k_mprotect_test(struct kunit *test) +{ + const unsigned long starts[] = { + TEST_VA, SZ_2M - SZ_4K, SZ_1G - SZ_4K, BIT(39) - SZ_4K, TEST_VA, + }; + unsigned int shape; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm permission fixture requires hardware opt-in"); + return; + } + for (shape = 0; shape < ARRAY_SIZE(starts); shape++) { + struct file *file = shape == 4 ? test_cache_file(test, PAGE_SIZE, true) : NULL; + struct mm_struct *mm = test_vma_mm(test); + unsigned long start = starts[shape], mapped; + unsigned long flags = MAP_FIXED_NOREPLACE | (file ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS); + phys_addr_t before[4] = { }; + long protect[4] = { }; + unsigned long initial = 0, final = 0, read_ro = 0, write_ro = 0, read_none = 0; + u64 value, ro_value = 0; + unsigned int i; + vm_fault_t fault = 0; + + KUNIT_ASSERT_NOT_NULL(test, mm); + if (shape == 4) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + kthread_use_mm(mm); + mapped = vm_mmap(file, start, PAGE_SIZE, PROT_READ | PROT_WRITE, flags, 0); + if (IS_ERR_VALUE(mapped)) + goto detach; + for (i = 0; i < 4; i++) { + struct vm_area_struct *vma = test_vma_lookup(mm, start + i * SZ_4K); + + fault |= test_file_fault(vma, start + i * SZ_4K, true); + if (fault & VM_FAULT_ERROR) + goto detach; + before[i] = pte_phys_mm(mm, test_fault_entry(test, mm, start + i * SZ_4K)); + value = 0x88110000 + i; + initial |= test_copy_word(start + i * SZ_4K, &value, true); + } + /* Keep the mm attached across permission changes and cached accesses. */ + protect[0] = test_mprotect(start + SZ_4K, SZ_4K, PROT_READ); + value = 0xaa; + write_ro = test_copy_word(start + SZ_4K, &value, true); + final |= test_copy_word(start, &value, true); + final |= test_copy_word(start + 3 * SZ_4K, &value, true); + protect[1] = test_mprotect(start + 2 * SZ_4K, SZ_4K, PROT_NONE); + read_none = test_copy_word(start + 2 * SZ_4K, &value, false); + read_ro = test_copy_word(start + SZ_4K, &ro_value, false); + protect[2] = test_mprotect(start, PAGE_SIZE, PROT_READ); + protect[3] = test_mprotect(start, PAGE_SIZE, PROT_READ | PROT_WRITE); + for (i = 0; i < 4; i++) { + value = 0x99220000 + i; + final |= test_copy_word(start + i * SZ_4K, &value, true); + } +detach: + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + KUNIT_ASSERT_EQ(test, fault & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, initial, 0UL); + KUNIT_EXPECT_EQ(test, final, 0UL); + KUNIT_EXPECT_EQ(test, write_ro, (unsigned long)sizeof(u64)); + KUNIT_EXPECT_EQ(test, read_none, (unsigned long)sizeof(u64)); + KUNIT_EXPECT_EQ(test, read_ro, 0UL); + KUNIT_EXPECT_EQ(test, ro_value, 0x88110001ULL); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, start + i * SZ_4K); + + KUNIT_EXPECT_EQ(test, protect[i], 0L); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), before[i]); + } + KUNIT_EXPECT_EQ(test, mm->total_vm, (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, mm->map_count, 1); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +#ifdef CONFIG_NUMA_BALANCING +static void user_subpage_numa_cow_batch_test(struct kunit *test) +{ + unsigned int shift; + + if (!(READ_ONCE(sysctl_numa_balancing_mode) & NUMA_BALANCING_NORMAL)) { + kunit_skip(test, "requires normal NUMA balancing; use numa_balancing=enable"); + return; + } + for (shift = 12; shift < PAGE_SHIFT; shift += 2) { + unsigned long size = 1UL << shift, start = TEST_VA; + struct vm_area_struct *vma; + struct mm_struct *child = test_vma_mm_granule(test, shift); + struct mm_struct *mm = test_vma_mm_granule(test, shift); + phys_addr_t before; + unsigned long marked, mapped; + long ret; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, child); + /* Use mmap accounting because the child will partially unmap. */ + kthread_use_mm(mm); + mapped = vm_mmap(NULL, start, 4 * size, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + vma = test_vma_lookup(mm, start); + KUNIT_ASSERT_NOT_NULL(test, vma); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, start + i * size, true), + (vm_fault_t)0); + before = pte_phys_mm(mm, test_fault_entry(test, mm, start + size)); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0L); + kthread_use_mm(child); + ret = vm_munmap(start + size, size); + kthread_unuse_mm(child); + KUNIT_ASSERT_EQ(test, ret, 0L); + /* Reuse the now-private slot without replacing its native owner. */ + KUNIT_ASSERT_EQ(test, test_file_fault(vma, start + size, true), (vm_fault_t)0); + KUNIT_ASSERT_EQ(test, pte_phys_mm(mm, + test_fault_entry(test, mm, start + size)), before); + kthread_use_mm(mm); + ret = test_mprotect(start, 4 * size, PROT_READ); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0L); + vma = test_vma_lookup(mm, start); + KUNIT_ASSERT_NOT_NULL(test, vma); + /* Permit local-node hinting, which normally skips a single user. */ + mmget(mm); + mmap_read_lock(mm); + marked = change_prot_numa(vma, start, start + 4 * size); + mmap_read_unlock(mm); + mmput(mm); + KUNIT_EXPECT_EQ(test, marked, 1UL); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, start + i * size); + + KUNIT_EXPECT_EQ(test, !!pte_protnone(pte), i == 1); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + } + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} +#endif + +static void user4k_mprotect_cow_batch_test(struct kunit *test) +{ + struct mm_struct *mm, *child; + struct vm_area_struct *vma; + struct fragment_test_pins *pins; + phys_addr_t before[4]; + unsigned int i; + unsigned long mapped; + long ret[2]; + int copied; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm permission fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, child); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, TEST_VA); + vma = test_vma_lookup(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, fragment_test_pin(pins, mm, TEST_VA + SZ_4K, SZ_4K, FOLL_WRITE, 8), 1L); + for (i = 0; i < 4; i++) + before[i] = pte_phys_mm(mm, test_fault_entry(test, mm, TEST_VA + i * SZ_4K)); + uprobe_start_dup_mmap(); + copied = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, copied, 0); + kthread_use_mm(mm); + ret[0] = test_mprotect(TEST_VA, PAGE_SIZE, PROT_READ); + ret[1] = test_mprotect(TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret[0], 0L); + KUNIT_EXPECT_EQ(test, ret[1], 0L); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), before[i]); + /* One native folio contains both exclusive and shared COW slots. */ + KUNIT_EXPECT_EQ(test, pte_write(pte), i == 1); + } + fragment_test_unpin(pins); +} + +static void user4k_mprotect_accounting_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + unsigned long mapped, committed[4], data[4]; + s64 global[4], before; + long ret[4]; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm permission fixture requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, mm); + before = test_commit_baseline(test); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, 0); + if (!IS_ERR_VALUE(mapped)) { + ret[0] = test_mprotect(TEST_VA + SZ_4K, SZ_4K, PROT_READ); + committed[0] = mm->committed_user_pages; + data[0] = mm->data_vm; + global[0] = percpu_counter_sum(&vm_committed_as) - before; + ret[1] = test_mprotect(TEST_VA, PAGE_SIZE, PROT_READ); + committed[1] = mm->committed_user_pages; + data[1] = mm->data_vm; + global[1] = percpu_counter_sum(&vm_committed_as) - before; + ret[2] = test_mprotect(TEST_VA + 2 * SZ_4K, SZ_4K, PROT_READ | PROT_WRITE); + committed[2] = mm->committed_user_pages; + data[2] = mm->data_vm; + global[2] = percpu_counter_sum(&vm_committed_as) - before; + ret[3] = vm_munmap(TEST_VA, PAGE_SIZE); + committed[3] = mm->committed_user_pages; + data[3] = mm->data_vm; + global[3] = percpu_counter_sum(&vm_committed_as) - before; + } + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, TEST_VA); + for (i = 0; i < 4; i++) { + unsigned long expected = i == 0 ? TEST_SLOTS - 1 : i == 2 ? 1 : 0; + + KUNIT_EXPECT_EQ(test, ret[i], 0L); + KUNIT_EXPECT_EQ(test, committed[i], expected); + KUNIT_EXPECT_EQ(test, data[i], expected); + KUNIT_EXPECT_EQ(test, global[i], expected ? 1LL : 0LL); + } + KUNIT_EXPECT_EQ(test, mm->map_count, 0); +} + +static void user4k_fragment_permissions_test(struct kunit *test) +{ + unsigned long start = TEST_VA + SZ_4K; + struct vm_area_struct *vma = test_fault_vma(test, start, SZ_4K); + struct fragment_test_pins *pins; + struct user_page_fragment *fragment; + struct mm_struct *mm; + spinlock_t *ptl; + pte_t *ptep; + long ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mm = vma->vm_mm; + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + fragment = &pins->fragments[0]; + { + struct page *page = NULL; + int locked = 1; + + mmap_read_lock(mm); + ret = pin_user_pages_remote(mm, start, 1, 0, &page, &locked); + if (locked) + mmap_read_unlock(mm); + if (ret == 1) + unpin_user_page(page); + KUNIT_EXPECT_EQ(test, ret, -EOPNOTSUPP); + } + ret = fragment_test_pin(pins, mm, start + 17, 31, 0, 8); + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_EXPECT_TRUE(test, is_zero_folio(fragment->folio)); + KUNIT_EXPECT_PTR_EQ(test, fragment->subpage, NULL); + /* Zero-page PTEs alias the first physical 4K despite this VA's slot 1. */ + KUNIT_EXPECT_EQ(test, fragment->offset, 17UL); + KUNIT_EXPECT_EQ(test, fragment->length, 31U); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(fragment->folio) + fragment->offset, + 0, fragment->length), NULL); + ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_WRITE, 8); + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_EXPECT_NOT_NULL(test, fragment->subpage); + KUNIT_EXPECT_EQ(test, fragment->offset, SZ_4K + 17UL); + memset(folio_address(fragment->folio) + fragment->offset, 0xa4, fragment->length); + fragment_test_unpin(pins); + mmap_write_lock(mm); + vm_flags_clear(vma, VM_WRITE); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + ptep = test_mm_lookup(mm, start, &ptl); + if (ptep) { + ptep_set_wrprotect(mm, start, ptep); + pte_unmap_unlock(ptep, ptl); + } + flush_tlb_range(vma, start, start + SZ_4K); + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_NULL(test, ptep); + ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_WRITE, 8); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_WRITE | FOLL_FORCE, 8); + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_EXPECT_FALSE(test, pte_write(test_fault_entry(test, mm, start))); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(fragment->folio) + fragment->offset, + 0xa4, fragment->length), NULL); + fragment_test_unpin(pins); + mmap_write_lock(mm); + vm_flags_clear(vma, VM_READ); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_write_unlock(mm); + ret = fragment_test_pin(pins, mm, start + 17, 31, 0, 8); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_FORCE, 8); + KUNIT_EXPECT_EQ(test, ret, 1L); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, mm, start, 1, FOLL_GET, 8); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + +static void test_subpage_unpin(void *arg) +{ + mm_subpage_unpin(arg); +} + +static void user4k_slot_pin_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct mm_struct *child; + struct mm_subpage *slots[4]; + struct folio *folio, *dst; + unsigned int i; + vm_fault_t fault; + spinlock_t *ptl; + pte_t *ptep, pte; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + for (i = 0; i < 4; i++) { + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * SZ_4K); + slots[i] = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slots[i]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, slots[i]), 0); + KUNIT_EXPECT_TRUE(test, mm_subpage_is_exclusive(slots[i])); + memset(folio_address(mm_subpage_folio(slots[i])) + i * SZ_4K, 0x61 + i, SZ_4K); + } + folio = mm_subpage_folio(slots[1]); + test_swap_tags_set(folio); + for (i = 0; i < 2; i++) { + ptep = test_mm_lookup(vma->vm_mm, TEST_VA + SZ_4K, &ptl); + KUNIT_ASSERT_NOT_NULL(test, ptep); + ret = mm_subpage_pin(slots[1]); + pte_unmap_unlock(ptep, ptl); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_unpin, slots[1]), 0); + } + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(slots[1]), 2U); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(folio)); + KUNIT_EXPECT_FALSE(test, mm_subpage_try_share(slots[1])); + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + folio_lock(folio); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(folio, dst); + /* Finish even unexpected success so a failing assertion cannot leak metadata. */ + mm_subpage_migrate_finish(folio); + mm_subpage_migrate_finish(dst); + folio_unlock(dst); + folio_unlock(folio); + folio_put(dst); + KUNIT_EXPECT_EQ(test, ret, -EBUSY); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < 4; i++) { + struct mm_subpage *copy; + + pte = test_fault_entry(test, child, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, pte) == mm_subpage_phys(slots[i]), i != 1); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slots[i]), i == 1 ? 1 : 2); + KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(slots[i]), i == 1); + copy = mm_subpage_get_from_phys(pte_phys_mm(child, pte)); + KUNIT_ASSERT_NOT_NULL(test, copy); + KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(copy), i == 1); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(copy)) + i * SZ_4K, + 0x61 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(mm_subpage_folio(copy), i)); + mm_subpage_put(copy); + } + /* A read pin on a shared slot must request unsharing, not pin shared COW data. */ + ptep = test_mm_lookup(vma->vm_mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, ptep); + ret = mm_subpage_pin(slots[0]); + pte_unmap_unlock(ptep, ptl); + if (!ret) + mm_subpage_unpin(slots[0]); + KUNIT_EXPECT_EQ(test, ret, -EAGAIN); + /* Re-enabling writes must retain the exact backing of an exclusive pinned slot. */ + ptep = test_mm_lookup(vma->vm_mm, TEST_VA + SZ_4K, &ptl); + KUNIT_ASSERT_NOT_NULL(test, ptep); + ptep_set_wrprotect(vma->vm_mm, TEST_VA + SZ_4K, ptep); + pte_unmap_unlock(ptep, ptl); + flush_tlb_range(vma, TEST_VA + SZ_4K, TEST_VA + 2 * SZ_4K); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA + SZ_4K, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + SZ_4K); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), mm_subpage_phys(slots[1])); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + *((u8 *)folio_address(folio) + SZ_4K) = 0xab; + pte = test_fault_entry(test, child, TEST_VA + SZ_4K); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(child, pte)), (u8)0x62); + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slots[1]), 0); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(slots[1]), 2U); + KUNIT_EXPECT_EQ(test, *((u8 *)folio_address(folio) + SZ_4K), (u8)0xab); + kunit_release_action(test, test_subpage_unpin, slots[1]); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(slots[1]), 1U); + kunit_release_action(test, test_subpage_unpin, slots[1]); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(slots[1]), 0U); + KUNIT_EXPECT_FALSE(test, folio_maybe_dma_pinned(folio)); +} + +#ifdef CONFIG_MIGRATION +static struct folio *test_migration_alloc(struct folio *src, unsigned long private) +{ + return folio_alloc(GFP_HIGHUSER_MOVABLE, 0); +} + +asmlinkage long __arm64_sys_madvise(const struct pt_regs *regs); + +static long test_madvise(unsigned long addr, unsigned long size, unsigned long behavior) +{ + struct pt_regs regs = { .regs = { addr, size, behavior } }; + + return __arm64_sys_madvise(®s); +} + +static void user_granule_large_vma_split_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, 14 }, i; + + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) || + (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN))) { + kunit_skip(test, "requires THP boundary splitting and borrowed-mm execution"); + return; + } + for (i = 0; i < ARRAY_SIZE(shifts); i++) { + struct mm_struct *mm; + struct vm_area_struct *vma; + unsigned long size = 1UL << shifts[i]; + unsigned long base = 4UL * HPAGE_PMD_SIZE, mapped; + pte_t pte; + long ret; + + if (shifts[i] >= PAGE_SHIFT) + continue; + mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, shifts[i]), 0); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, base, 2UL * HPAGE_PMD_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE | MAP_NORESERVE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, base); + vma = test_vma_lookup(mm, base); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, base, true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, base + size, true) & VM_FAULT_ERROR, 0U); + /* The generic VMA split probes the native THP boundary for this range. */ + kthread_use_mm(mm); + ret = test_mprotect(base + size, size, PROT_READ); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0L); + pte = test_fault_entry(test, mm, base + size); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + pte = test_fault_entry(test, mm, base); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + kthread_use_mm(mm); + ret = test_mprotect(base + size, size, PROT_READ | PROT_WRITE); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 0L); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user_granule_brk_extent_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int count = PAGE_SHIFT > 14 ? 3 : 2, i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm brk fixture requires hardware opt-in"); + return; + } + for (i = 0; i < count; i++) { + struct mm_struct *mm = mm_alloc(); + struct vm_area_struct *vma; + unsigned long size = 1UL << shifts[i], neighbor; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, shifts[i]), 0); + kthread_use_mm(mm); + ret = vm_brk_flags(TEST_VA, size + 1, false); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + vma = test_vma_lookup(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma->vm_start, TEST_VA); + KUNIT_EXPECT_EQ(test, vma->vm_end, TEST_VA + 2 * size); + KUNIT_EXPECT_EQ(test, mm->total_vm, 2UL); + /* FEX reserves exactly this page after its initial ELF/BSS break. */ + kthread_use_mm(mm); + neighbor = vm_mmap(NULL, TEST_VA + 2 * size, size, PROT_NONE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, 0); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, neighbor, TEST_VA + 2 * size); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_droppable_reclaim_test(struct kunit *test) +{ + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm droppable fixture requires hardware opt-in"); + return; + } + /* Dirty discard, retained GET, retained PIN, migration, zero-page COW. */ + for (mode = 0; mode < 5; mode++) { + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + struct user_page_fragment held = {}; + struct folio *folio; + unsigned int i, succeeded = 0; + unsigned long mapped; + long ret; + pte_t pte; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_DROPPABLE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, TEST_VA); + vma = test_vma_lookup(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_TRUE(test, vma->vm_flags & VM_WIPEONFORK); + KUNIT_EXPECT_TRUE(test, vma->vm_flags & VM_DONTDUMP); + if (mode == 4) { + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, mm, TEST_VA); + KUNIT_EXPECT_TRUE(test, is_zero_pfn(pte_pfn(pte))); + } + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + if (mode != 4) + for (i = 1; i < PAGE_SIZE / SZ_4K; i++) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + i * SZ_4K, + true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio)); + memset(folio_address(folio), 0xa7, PAGE_SIZE); + folio_put(folio); + if (mode == 1 || mode == 2) { + ret = mode == 1 ? get_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1) : + pin_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + } + if (mode == 3) { + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 1L); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_EXPECT_EQ(test, ret, 0L); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + } + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio)); + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(folio)); + if (mode == 1 || mode == 2) { + pte = test_fault_entry(test, mm, TEST_VA); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, *(unsigned char *)folio_address(folio), 0xa7); + release_user_fragments(&held, 1, false); + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + } + folio_put(folio); + for (i = 0; i < (mode == 4 ? 1 : PAGE_SIZE / SZ_4K); i++) { + pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_none(pte)); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0L); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0, PAGE_SIZE), NULL); + KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio)); + folio_put(folio); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_lazyfree_test(struct kunit *test) +{ + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm advice requires hardware opt-in"); + return; + } + /* Partial, redirty, retained GET, typed write, all leaves, migration. */ + for (mode = 0; mode < 10; mode++) { + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct mm_struct *mm; + struct user_page_fragment held = {}; + struct folio *folio; + struct mm_subpage *slot; + unsigned int i, succeeded = 0; + long ret; + pte_t pte; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, vma); + mm = vma->vm_mm; + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + if (mode == 6) { + ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + } + kthread_use_mm(mm); + ret = test_madvise(TEST_VA, + (mode == 4 || mode == 9) ? PAGE_SIZE : SZ_4K, MADV_FREE); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0L); + pte = test_fault_entry(test, mm, TEST_VA); + KUNIT_EXPECT_EQ(test, pte_dirty(pte), mode == 6); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_is_lazyfree(slot), mode != 6); + mm_subpage_put(slot); + if (mode == 1) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + + if (mode == 2 || mode == 3 || mode == 8) { + ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, + mode == 3 ? FOLL_WRITE : 0, &held, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + if (mode == 3 || mode == 8) { + memset(folio_address(held.folio) + held.offset, 0xd3, SZ_4K); + release_user_fragments(&held, 1, true); + } + } + if (mode == 7) { + ret = pin_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + } + if (mode == 5 || mode == 9) { + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 1L); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_EXPECT_EQ(test, ret, 0L); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + } + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + folio_put(folio); + for (i = 0; i < 4; i++) { + pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + KUNIT_EXPECT_EQ(test, pte_none(pte), (mode == 4 || mode == 9) || + (i == 0 && (mode == 0 || mode == 5))); + } + if (mode == 2 || mode == 6 || mode == 7) + release_user_fragments(&held, 1, false); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_mprotect_migration_test(struct kunit *test) +{ + unsigned int file_case; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm permission fixture requires hardware opt-in"); + return; + } + for (file_case = 0; file_case < 2; file_case++) { + struct mm_struct *mm = test_vma_mm(test); + struct file *file = file_case ? test_cache_file(test, PAGE_SIZE, true) : NULL; + struct vm_area_struct *vma; + struct folio *src, *dst; + unsigned long mapped; + unsigned int i; + int prepared, unmapped = -1; + long protected = -EINVAL; + bool offsets = true; + phys_addr_t phys; + + KUNIT_ASSERT_NOT_NULL(test, mm); + if (file_case) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + kthread_use_mm(mm); + mapped = vm_mmap(file, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | (file ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS), 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, TEST_VA); + vma = test_vma_lookup(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + src = test_fault_folio(mm, TEST_VA); + if (!src) { + folio_put(dst); + KUNIT_FAIL(test, "missing migration source"); + return; + } + phys = PFN_PHYS(folio_pfn(src)); + folio_lock(src); + folio_lock(dst); + prepared = mm_subpage_migrate_prepare(src, dst); + if (!prepared) { + try_to_migrate(src, TTU_SYNC); + unmapped = folio_mapcount(src); + kthread_use_mm(mm); + protected = test_mprotect(TEST_VA, PAGE_SIZE, PROT_READ); + kthread_unuse_mm(mm); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + + offsets &= softleaf_is_migration(softleaf_from_pte(pte)) && + !softleaf_is_migration_write(softleaf_from_pte(pte)) && + pte_swp_subpage_offset(pte) == i * SZ_4K; + } + remove_migration_ptes(src, src, 0); + mm_subpage_migrate_finish(src); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(src); + folio_put(dst); + folio_put(src); + KUNIT_ASSERT_EQ(test, prepared, 0); + KUNIT_EXPECT_EQ(test, unmapped, 0); + KUNIT_EXPECT_EQ(test, protected, 0L); + KUNIT_EXPECT_TRUE(test, offsets); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), phys + i * SZ_4K); + } + } +} + +/* Independent roots and VA offsets must not change physical slot identity. */ +static void user4k_nonlinear_rmap_test(struct kunit *test) +{ + struct anon_slot_fixture *a = anon_slot_fixture_create(test); + struct anon_slot_fixture *b = anon_slot_fixture_create(test); + struct mm_struct *child = test_vma_mm(test); + struct mm_subpage *slot; + struct folio *folio, *dst; + phys_addr_t phys; + vm_flags_t flags; + unsigned int succeeded = 0, side; + int ret, unmapped = -1; + bool isolated; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, child); + slot = mm_subpage_alloc_at(a->pool, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + folio = mm_subpage_folio(slot); + /* Unlike the lightweight rmap fixture, real LRU migration needs a charge. */ + ret = mem_cgroup_charge(folio, a->mm, GFP_KERNEL); + if (ret) { + mm_subpage_put(slot); + KUNIT_FAIL(test, "cannot charge nonlinear fixture folio"); + return; + } + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(a, 0, slot), 0); + phys = mm_subpage_phys(slot); + slot = mm_subpage_alloc_at(a->pool, 3 * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + /* Same native folio, different root and different VA/physical offset. */ + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(b, 1, slot), 0); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(slot), folio); + memset(folio_address(folio), 0x53, SZ_4K); + memset(folio_address(folio) + 3 * SZ_4K, 0xa7, SZ_4K); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 2); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, b->mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 2); + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, + test_fault_entry(test, child, TEST_VA + SZ_4K)), phys + 3 * SZ_4K); + kunit_release_action(test, user4k_vma_mm_free, child); + + /* Rollback must find migration entries through both roots. */ + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + folio_get(folio); + folio_lock(folio); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(folio, dst); + if (!ret) { + try_to_migrate(folio, TTU_SYNC); + unmapped = folio_mapcount(folio); + remove_migration_ptes(folio, folio, 0); + mm_subpage_migrate_finish(folio); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(folio); + folio_put(dst); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, unmapped, 0); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 2); + + /* Successful migration exercises copied identities in the new owner. */ + folio_add_lru(folio); + lru_add_drain_all(); + folio_get(folio); + isolated = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (ret) { + KUNIT_FAIL(test, "nonlinear migration failed: %d", ret); + return; + } + /* Refresh fixture references before assertions/cleanup after migration. */ + for (side = 0; side < 2; side++) { + struct anon_slot_fixture *f = side ? b : a; + pte_t pte = test_fault_entry(test, f->mm, TEST_VA + side * SZ_4K); + + f->slots[side] = NULL; + f->mapped[side] = false; + if (!pte_present(pte)) + continue; + slot = mm_subpage_get_from_phys(pte_phys_mm(f->mm, pte)); + if (!slot) + continue; + f->slots[side] = slot; + f->mapped[side] = true; + mm_subpage_put(slot); + } + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + for (side = 0; side < 2; side++) { + struct anon_slot_fixture *f = side ? b : a; + + slot = f->slots[side]; + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), side ? 3U * SZ_4K : 0U); + KUNIT_EXPECT_NE(test, mm_subpage_phys(slot), phys + (side ? 3 * SZ_4K : 0)); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), side ? 0xa7 : 0x53, SZ_4K), NULL); + } +} + +static void user4k_migration_entries_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct mm_struct *child = test_vma_mm(test); + struct folio *src, *dst; + unsigned int i, side; + vm_fault_t fault; + bool encoded = true, copied = true, exact = true; + int ret, fork_ret = -EINVAL, unmapped = -1; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, child); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + src = test_fault_folio(vma->vm_mm, TEST_VA); + if (!src) { + folio_put(dst); + KUNIT_FAIL(test, "missing source folio"); + return; + } + folio_lock(src); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(src, dst); + if (!ret) { + try_to_migrate(src, TTU_SYNC); + unmapped = folio_mapcount(src); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * SZ_4K); + struct rmap_walk_range range = { + .address = TEST_VA + i * SZ_4K, + .subpage = true, + .subpage_offset = i * SZ_4K, + }; + DEFINE_FOLIO_RMAP_WALK(walk, src, vma, range, + PVMW_SYNC | PVMW_MIGRATION); + unsigned int count = 0; + + encoded &= softleaf_is_migration_write(softleaf_from_pte(pte)) && + pte_swp_subpage_offset(pte) == i * SZ_4K; + while (page_vma_mapped_walk(&walk)) { + exact &= walk.address == TEST_VA + i * SZ_4K; + count++; + } + exact &= count == 1; + } + uprobe_start_dup_mmap(); + fork_ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + if (!fork_ret) { + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : vma->vm_mm; + + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + + copied &= softleaf_is_migration_read(softleaf_from_pte(pte)) && + pte_swp_subpage_offset(pte) == i * SZ_4K; + } + } + } + /* Drop one migration entry while the other mm still retains it. */ + mmap_read_lock(vma->vm_mm); + zap_vma_range(vma, TEST_VA + SZ_4K, SZ_4K); + mmap_read_unlock(vma->vm_mm); + remove_migration_ptes(src, src, 0); + mm_subpage_migrate_finish(src); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(src); + folio_put(dst); + folio_put(src); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, fork_ret, 0); + KUNIT_EXPECT_EQ(test, unmapped, 0); + KUNIT_EXPECT_TRUE(test, encoded); + KUNIT_EXPECT_TRUE(test, copied); + KUNIT_EXPECT_TRUE(test, exact); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : vma->vm_mm; + + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), side ? (unsigned long)TEST_SLOTS : TEST_SLOTS - 1UL); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + struct mm_subpage *slot; + + if (!side && i == 1) { + KUNIT_EXPECT_TRUE(test, pte_none(pte)); + continue; + } + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * (unsigned int)SZ_4K); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), i == 1 ? 1 : 2); + mm_subpage_put(slot); + } + } +} + +static struct folio *test_file_migration_alloc(struct folio *src, unsigned long private) +{ + return folio_alloc(GFP_HIGHUSER_MOVABLE, folio_order(src)); +} + +static void user64k_file_migration_test(struct kunit *test) +{ + unsigned int mode; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "coarse shmem promotion requires collapse support"); + return; + } + for (mode = 0; mode < 6; mode++) { + bool shmem = mode & 1, fail = mode >= 2, pin = mode >= 4; + bool isolated; + unsigned long len = shmem ? SZ_64K : 2 * SZ_64K; + unsigned long base = shmem ? SZ_64K : 0, offset; + unsigned int nr = len / PAGE_SIZE, succeeded = 0; + unsigned int maps = 2 * nr + len / SZ_4K - 1; + struct file *file = shmem ? test_cache_file(test, 2 * SZ_64K, true) : + test_coarse_cache_file(test, 17 - PAGE_SHIFT, 2 * SZ_64K); + struct vm_area_struct *coarse, *native, *small; + struct folio *folio; + struct user_page_fragment *fragment = NULL; + phys_addr_t old_phys, phys; + int ret; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, len, base); + native = test_file_vma(test, file, true, true, TEST_VA, len, base); + /* Shifted file offset and a small-page table boundary in the alias. */ + small = test_file_vma(test, file, false, true, SZ_2M - SZ_4K, + len - SZ_4K, base + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_NOT_NULL(test, small); + for (offset = 0; offset < len; offset += SZ_64K) + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA + offset, shmem) & + VM_FAULT_ERROR, 0U); + for (offset = 0; offset < len; offset += PAGE_SIZE) + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA + offset, shmem) & + VM_FAULT_ERROR, 0U); + for (offset = 0; offset < len - SZ_4K; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + offset, + shmem) & VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + folio = filemap_get_folio(file->f_mapping, base >> PAGE_SHIFT); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), len); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)(nr + maps + 1)); + old_phys = page_to_phys(&folio->page); + if (pin) { + fragment = kunit_kzalloc(test, sizeof(*fragment), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fragment); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_promote_unpin, fragment), 0); + KUNIT_ASSERT_EQ(test, pin_user_fragments_remote(small->vm_mm, + small->vm_end - SZ_4K + 37, 1, 0, fragment, 1), 1L); + KUNIT_EXPECT_PTR_EQ(test, fragment->folio, folio); + KUNIT_EXPECT_EQ(test, fragment->offset, len - SZ_4K + 37); + } + isolated = isolate_folio_to_list(folio, &folios); + /* Retain either a GET or a typed PIN to force migration rollback. */ + if (!fail || pin || !isolated) + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_file_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (pin) { + KUNIT_EXPECT_EQ(test, page_to_phys(&fragment->folio->page), old_phys); + KUNIT_EXPECT_EQ(test, *(u8 *)(folio_address(fragment->folio) + + fragment->offset), + (u8)(0x31 + (base + len - SZ_4K) / SZ_4K)); + kunit_release_action(test, test_promote_unpin, fragment); + } else if (fail) { + folio_put(folio); + } + KUNIT_EXPECT_EQ_MSG(test, ret, fail ? 1 : 0, "mode=%u", mode); + KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : nr); + folio = filemap_get_folio(file->f_mapping, base >> PAGE_SHIFT); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + phys = page_to_phys(&folio->page); + KUNIT_EXPECT_EQ(test, phys == old_phys, fail); + KUNIT_EXPECT_EQ(test, folio_size(folio), len); + KUNIT_EXPECT_EQ(test, folio->index, base >> PAGE_SHIFT); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)(nr + maps + 1)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(coarse->vm_mm, + shmem ? MM_SHMEMPAGES : MM_FILEPAGES), (long)(len / SZ_64K)); + for (offset = 0; offset < len; offset += SZ_4K) { + pte_t pte; + + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + offset, + 0x31 + (base + offset) / SZ_4K, SZ_4K), NULL); + if (offset) { + pte = test_fault_entry(test, small->vm_mm, + small->vm_start + offset - SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), shmem); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, pte), phys + offset); + } + if (!(offset % PAGE_SIZE)) { + pte = test_fault_entry(test, native->vm_mm, TEST_VA + offset); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), shmem); + KUNIT_EXPECT_EQ(test, pte_phys_mm(native->vm_mm, pte), phys + offset); + } + if (!(offset % SZ_64K)) { + pte = test_fault_entry(test, coarse->vm_mm, TEST_VA + offset); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), shmem); + KUNIT_EXPECT_EQ(test, pte_phys_mm(coarse->vm_mm, pte), phys + offset); + } + } + kunit_release_action(test, user4k_vma_mm_free, coarse->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)nr + 1); + } +} + +static void user64k_anon_migration_test(struct kunit *test) +{ + unsigned int mode, nr = SZ_64K / PAGE_SIZE; + + if (!test_coarse_file_enabled(test)) + return; + for (mode = 0; mode < 5; mode++) { + bool forked = mode & 1, fail = mode >= 2, pin = mode == 4; + struct vm_area_struct *vma, *cvma = NULL; + struct mm_struct *child = NULL; + struct user_page_fragment *fragment = NULL; + struct folio *folio; + phys_addr_t old_phys, phys; + unsigned int i, succeeded = 0; + pte_t pte; + bool isolated; + int ret; + LIST_HEAD(folios); + + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + SZ_64K - 1, true) & + VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + old_phys = pte_phys_mm(vma->vm_mm, pte); + folio = page_folio(pfn_to_page(PHYS_PFN(old_phys))); + for (i = 0; i < nr; i++) + memset(page_address(folio_page(folio, i)), 0x51 + i, PAGE_SIZE); + if (forked) { + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + } + if (pin) { + fragment = kunit_kzalloc(test, sizeof(*fragment), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fragment); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_promote_unpin, fragment), 0); + KUNIT_ASSERT_EQ(test, pin_user_fragments_remote(vma->vm_mm, + TEST_VA + SZ_64K - 1, 1, FOLL_WRITE, fragment, 1), 1L); + KUNIT_EXPECT_PTR_EQ(test, fragment->folio, folio); + KUNIT_EXPECT_EQ(test, fragment->offset, (unsigned int)SZ_64K - 1); + } + lru_add_drain_all(); + folio_get(folio); + isolated = isolate_folio_to_list(folio, &folios); + if (!fail || pin || !isolated) + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_file_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (pin) + kunit_release_action(test, test_promote_unpin, fragment); + else if (fail) + folio_put(folio); + KUNIT_EXPECT_EQ_MSG(test, ret, fail ? 1 : 0, "mode=%u", mode); + KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : nr); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), !forked); + phys = pte_phys_mm(vma->vm_mm, pte); + KUNIT_EXPECT_EQ(test, phys == old_phys, fail); + folio = page_folio(pfn_to_page(PHYS_PFN(phys))); + folio_get(folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + expect_coarse_anon_state(test, folio, forked ? 2 : 1, !forked); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)(nr * (forked ? 2 : 1) + 1)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + if (forked) { + struct folio *copied; + + pte = test_fault_entry(test, child, TEST_VA); + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, pte), phys); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 1L); + /* A restored shared migration entry must still perform full-leaf COW. */ + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA + SZ_64K - 1, + true) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, child, TEST_VA); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, pte), phys); + copied = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(child, pte)))); + expect_coarse_anon_state(test, copied, 1, true); + for (i = 0; i < nr; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(copied, i)), + 0x51 + i, PAGE_SIZE), NULL); + memset(folio_address(copied), 0xa7, SZ_64K); + kunit_release_action(test, user4k_vma_mm_free, child); + } + for (i = 0; i < nr; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(folio, i)), + 0x51 + i, PAGE_SIZE), NULL); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + } +} + +static void user4k_file_migration_test(struct kunit *test) +{ + unsigned int fail; + + for (fail = 0; fail < 2; fail++) { + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *small, *native; + struct folio *folio; + phys_addr_t old_phys; + unsigned int i, succeeded = 0; + int ret; + bool isolated; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, SZ_2M - SZ_4K, PAGE_SIZE, 0); + native = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + i * SZ_4K, + true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, true) & VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { [0 ... TEST_SLOTS - 1] = 1 }, 1); + old_phys = PFN_PHYS(folio_pfn(folio)); + isolated = isolate_folio_to_list(folio, &folios); + if (!fail || !isolated) + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, MIGRATE_SYNC, + MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (fail) + folio_put(folio); + KUNIT_EXPECT_EQ(test, ret, fail ? 1 : 0); + KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : 1U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { [0 ... TEST_SLOTS - 1] = 1 }, 1); + for (i = 0; i < 4; i++) { + pte_t entry = test_fault_entry(test, small->vm_mm, small->vm_start + i * SZ_4K); + + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry), + PFN_PHYS(folio_pfn(folio)) + i * SZ_4K); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry) == old_phys + i * SZ_4K, !!fail); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + } + KUNIT_EXPECT_EQ(test, pte_phys_mm(native->vm_mm, + test_fault_entry(test, native->vm_mm, TEST_VA)), PFN_PHYS(folio_pfn(folio))); + /* File reclaim must visit every 4K alias, including across PTE tables. */ + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + KUNIT_EXPECT_FALSE(test, folio_mapped(folio)); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 0, 0, 0 }, 0); + folio_unlock(folio); + folio_put(folio); + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, small->vm_mm, + small->vm_start + i * SZ_4K))); + } + for (i = 0; i < 4; i++) + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + i * SZ_4K, + false) & VM_FAULT_ERROR, 0U); + } +} + +static void user4k_migration_test(struct kunit *test) +{ + const unsigned long boundaries[] = { + SZ_2M - SZ_4K, SZ_2M, SZ_1G - SZ_4K, SZ_1G, + BIT(39) - SZ_4K, BIT(39), + }; + unsigned int mode; + + /* Dense/sparse, forks, refcount rollback, batched TLBI and table edges. */ + for (mode = 0; mode < 16; mode++) { + bool edge = mode >= 6 && mode < 12; + bool locked = mode >= 12; + bool sparse = mode == 1 || edge || mode == 13; + bool forked = mode == 2 || mode == 4 || edge || mode == 14; + bool fail = mode == 3 || mode == 4 || mode == 15; + unsigned long start = edge ? boundaries[mode - 6] : + TEST_VA + (sparse ? SZ_4K : 0); + unsigned int nr = sparse ? 1 : 4, i, side, succeeded = 0; + struct vm_area_struct *vma = test_fault_vma(test, start, nr * SZ_4K); + struct mm_struct *child = NULL; + struct folio *folio; + phys_addr_t old_phys; + vm_fault_t fault; + int ret; + bool isolated; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, vma); + if (locked) { + mmap_write_lock(vma->vm_mm); + vm_flags_set(vma, VM_LOCKED); + vma->vm_mm->locked_vm = nr; + mmap_write_unlock(vma->vm_mm); + } + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, start, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + folio = test_fault_folio(vma->vm_mm, start); + KUNIT_ASSERT_NOT_NULL(test, folio); + old_phys = PFN_PHYS(folio_pfn(folio)); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0x41 + i + mode, SZ_4K); + test_swap_tags_set(folio); + folio_put(folio); + if (forked) { + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + } + lru_add_drain_all(); + folio = test_fault_folio(vma->vm_mm, start); + KUNIT_ASSERT_NOT_NULL(test, folio); + isolated = isolate_folio_to_list(folio, &folios); + /* An extra native reference forces the normal migration rollback. */ + if (!fail || !isolated) + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + mode == 5 ? MIGRATE_ASYNC : MIGRATE_SYNC, + MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (fail) + folio_put(folio); + KUNIT_EXPECT_EQ(test, ret, fail ? 1 : 0); + KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : 1U); + for (side = 0; side <= (unsigned int)forked; side++) { + struct mm_struct *mm = side ? child : vma->vm_mm; + + for (i = 0; i < nr; i++) { + unsigned int index = sparse ? offset_in_page(start) / SZ_4K : i; + pte_t pte = test_fault_entry(test, mm, start + i * SZ_4K); + struct mm_subpage *slot; + + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte) == old_phys + index * SZ_4K, fail); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), !forked); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + folio = mm_subpage_folio(slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), index * (unsigned int)SZ_4K); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), forked ? 2 : 1); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + index * SZ_4K, + 0x41 + index + mode, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, index)); + mm_subpage_put(slot); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), (unsigned long)nr); + } + if (locked) { + lru_add_drain_all(); + folio = test_fault_folio(vma->vm_mm, start); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + if (!fail) + KUNIT_EXPECT_EQ(test, folio->mlock_count, nr); + folio_put(folio); + } + if (child) + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + } +} +struct fragment_pin_worker { + struct mm_struct *mm; + struct task_struct *task; + struct completion ready; + unsigned long checks; + int error; +}; + +static int fragment_pin_worker_run(void *arg) +{ + struct fragment_pin_worker *worker = arg; + struct user_page_fragment fragments[4]; + + while (!kthread_should_stop()) { + long nr = pin_user_fragments_remote(worker->mm, TEST_VA, PAGE_SIZE, + 0, fragments, ARRAY_SIZE(fragments)); + unsigned int i; + + if (nr != 4) { + worker->error = nr < 0 ? nr : -EIO; + } else { + for (i = 0; i < nr; i++) { + struct user_page_fragment *f = &fragments[i]; + + if (memchr_inv(folio_address(f->folio) + f->offset, 0x51 + i, f->length)) + worker->error = -EILSEQ; + worker->checks++; + } + } + if (nr > 0) + release_user_fragments(fragments, nr, false); + complete_all(&worker->ready); + if (worker->error) + break; + cond_resched(); + } + return 0; +} + +static void fragment_pin_worker_stop(void *arg) +{ + struct fragment_pin_worker *worker = arg; + + if (worker->task) { + kthread_stop(worker->task); + worker->task = NULL; + } +} + +static void user4k_fragment_pin_race_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct fragment_test_pins *pins; + struct fragment_pin_worker *workers; + unsigned int i, round, migrated = 0, busy = 0; + long ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, 4); + KUNIT_ASSERT_EQ(test, ret, 4L); + for (i = 0; i < 4; i++) + memset(folio_address(pins->fragments[i].folio) + pins->fragments[i].offset, + 0x51 + i, pins->fragments[i].length); + fragment_test_unpin(pins); + workers = kunit_kcalloc(test, 2, sizeof(*workers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + for (i = 0; i < 2; i++) { + workers[i].mm = vma->vm_mm; + init_completion(&workers[i].ready); + workers[i].task = kthread_run(fragment_pin_worker_run, &workers[i], "user4k-pin-%u", i); + if (IS_ERR(workers[i].task)) { + workers[i].task = NULL; + KUNIT_FAIL(test, "pin worker creation failed"); + return; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_pin_worker_stop, &workers[i]), 0); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].ready, 10 * HZ), 0UL); + } + for (round = 0; round < 32; round++) { + struct mm_struct *child = test_vma_mm(test); + unsigned long addr = TEST_VA + (round % 4) * SZ_4K; + struct vm_area_struct *cvma; + struct folio *folio; + unsigned int succeeded = 0; + vm_fault_t fault; + struct user_page_fragment write_fragment; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0L); + cvma = test_vma_lookup(child, addr); + KUNIT_ASSERT_NOT_NULL(test, cvma); + mmap_read_lock(child); + fault = handle_mm_fault(cvma, addr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + ret = pin_user_fragments_remote(child, addr, 1, FOLL_WRITE, &write_fragment, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + *((u8 *)folio_address(write_fragment.folio) + write_fragment.offset) = 0xa0 + round; + release_user_fragments(&write_fragment, 1, true); + lru_add_drain_all(); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = isolate_folio_to_list(folio, &folios); + folio_put(folio); + if (ret) { + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_ASYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_EXPECT_TRUE(test, ret == 0 || ret == 1); + migrated += succeeded; + busy += ret == 1; + } + kunit_release_action(test, user4k_vma_mm_free, child); + } + for (i = 0; i < 2; i++) { + fragment_pin_worker_stop(&workers[i]); + KUNIT_EXPECT_EQ(test, workers[i].error, 0); + KUNIT_EXPECT_GE(test, workers[i].checks, 4UL); + } + kunit_info(test, "32 fork/COW/exit cycles against two pin workers; migrations=%u not_migrated=%u\n", + migrated, busy); +} + +#endif + +static int test_swap_control(const char *operation) +{ + char *argv[] = { "/init", (char *)operation, NULL }; + char *envp[] = { NULL }; + + return call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC); +} + +static void test_reenable_swap(void *arg) +{ + bool *disabled = arg; + + if (*disabled) + test_swap_control("--swapon-helper"); +} + +/* Exclusive slots must remain packed after read swap-in and later writes. */ +static void test_user4k_swap_exclusive(struct kunit *test, unsigned int mode) +{ + struct vm_area_struct *vma; + struct folio *folio; + struct mm_subpage *held = NULL; + bool *disabled; + phys_addr_t restored[TEST_SLOTS]; + swp_entry_t entry; + vm_fault_t fault; + unsigned int i; + int ret, mapped, hold_err = 0; + pte_t pte; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0xc0 + i, SZ_4K); + test_swap_tags_set(folio); + if (mode == 5) { + held = mm_subpage_get_from_phys(PFN_PHYS(folio_pfn(folio)) + 2 * SZ_4K); + if (held) { + hold_err = kunit_add_action_or_reset(test, test_subpage_put, held); + if (hold_err) + held = NULL; + } + } + ret = folio_alloc_swap(folio); + entry = folio->swap; + if (!ret) + try_to_unmap(folio, TTU_SYNC); + mapped = folio_mapcount(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, mapped, 0); + if (mode == 5) { + KUNIT_ASSERT_EQ(test, hold_err, 0); + KUNIT_ASSERT_NOT_NULL(test, held); + } + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(softleaf_from_pte(pte))); + KUNIT_EXPECT_TRUE(test, pte_swp_exclusive(pte)); + } + if (mode >= 3) { + struct mm_struct *child = test_vma_mm(test); + + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + kunit_release_action(test, user4k_vma_mm_free, child); + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * SZ_4K); + KUNIT_EXPECT_FALSE(test, pte_swp_exclusive(pte)); + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 1U); + } + } + if (mode == 1 || mode == 2 || mode == 4 || mode == 6) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + if (mode == 2 || mode == 6) { + ret = test_swap_control("--swapoff-helper"); + *disabled = !total_swap_pages; + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, *disabled); + } + for (i = 0; i < TEST_SLOTS; i++) { + struct mm_subpage *slot; + unsigned long address = TEST_VA + i * SZ_4K; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, vma->vm_mm, address); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + restored[i] = pte_phys_mm(vma->vm_mm, pte); + slot = mm_subpage_get_from_phys(restored[i]); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(slot), !(held && i == 2)); + KUNIT_EXPECT_EQ(test, restored[i], restored[0] + i * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0xc0 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(mm_subpage_folio(slot), i)); + mm_subpage_put(slot); + } + for (i = 0; i < TEST_SLOTS; i++) { + unsigned long address = TEST_VA + i * SZ_4K; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, vma->vm_mm, address); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + if (held && i == 2) { + KUNIT_EXPECT_NE(test, pte_phys_mm(vma->vm_mm, pte), restored[i]); + memset(phys_to_virt(pte_phys_mm(vma->vm_mm, pte)), 0xdd, SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(held)) + + mm_subpage_offset(held), 0xc2, SZ_4K), NULL); + } else { + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), restored[i]); + } + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 0UL); + if (*disabled) { + ret = test_swap_control("--swapon-helper"); + if (!ret) + *disabled = false; + KUNIT_EXPECT_EQ(test, ret, 0); + } +} + +static void user4k_swap_exclusive_cached_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 0); +} + +static void user4k_swap_exclusive_disk_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 1); +} + +static void user4k_swap_exclusive_swapoff_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 2); +} + +static void user4k_swap_reuse_cached_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 3); +} + +static void user4k_swap_reuse_disk_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 4); +} + +static void user4k_swap_reuse_get_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 5); +} + +static void user4k_swap_reuse_swapoff_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 6); +} + +static void test_user4k_swap_roundtrip(struct kunit *test, unsigned int mode, u8 base) +{ + const bool evict = mode != 0; + struct vm_area_struct *parent, *cvma; + struct mm_struct *child; + struct folio *folio; + swp_entry_t entry = { 0 }; + unsigned int i, count; + int err, mapped; + vm_fault_t fault; + pte_t pte; + + if (!total_swap_pages) { + kunit_skip(test, "rerun this suite after enabling the disposable guest swap disk"); + return; + } + parent = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, parent); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(parent->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, base + i, SZ_4K); + test_swap_tags_set(folio); + if (mode == 2) { + LIST_HEAD(folios); + + folio_unlock(folio); + err = folio_isolate_lru(folio) ? 0 : -EBUSY; + if (!err) + list_add(&folio->lru, &folios); + folio_put(folio); + /* Let the normal reclaim path allocate swap and unmap all slots. */ + reclaim_pages(&folios); + mapped = 0; + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, parent->vm_mm, TEST_VA + i * SZ_4K); + mapped += pte_present(pte); + } + entry = softleaf_from_pte(pte); + } else { + err = folio_alloc_swap(folio); + if (!err) { + entry = folio->swap; + try_to_unmap(folio, TTU_SYNC | (evict ? TTU_BATCH_FLUSH : 0)); + } + mapped = folio_mapcount(folio); + folio_unlock(folio); + try_to_unmap_flush(); + folio_put(folio); + } + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, mapped, 0); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_ANONPAGES), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, parent->vm_mm, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_FALSE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(pte).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(pte), i * (unsigned int)SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_swp_exclusive(pte)); + } + /* Include extended counts when a native page has 16 user slots. */ + count = swp_swapcount(entry); + KUNIT_EXPECT_EQ(test, count, (unsigned int)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 1U); + if (evict) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + + /* Fork while all slots are nonpresent, preserving each byte offset. */ + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + /* Include extended counts when a native page has 16 user slots. */ + count = swp_swapcount(entry); + KUNIT_EXPECT_EQ(test, count, 2U * TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 2U); + for (i = 0; i < TEST_SLOTS; i++) { + unsigned long address = TEST_VA + ((i * 5 + 2) % TEST_SLOTS) * SZ_4K; + struct mm_subpage *slot; + spinlock_t *ptl; + pte_t *ptep; + bool intact = false; + + pte = test_fault_entry(test, parent->vm_mm, address); + KUNIT_EXPECT_FALSE(test, pte_swp_exclusive(pte)); + pte = test_fault_entry(test, child, address); + KUNIT_EXPECT_FALSE(test, pte_swp_exclusive(pte)); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(pte).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(pte), ((i * 5 + 2) % TEST_SLOTS) * (unsigned int)SZ_4K); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, address, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + KUNIT_EXPECT_EQ(test, !!(fault & VM_FAULT_MAJOR), evict && !i); + ptep = test_mm_lookup(parent->vm_mm, address, &ptl); + KUNIT_ASSERT_NOT_NULL(test, ptep); + pte = ptep_get(ptep); + slot = pte_present(pte) ? mm_subpage_get_from_phys(pte_phys_mm(parent->vm_mm, pte)) : NULL; + if (slot) { + intact = !memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), base + ((i * 5 + 2) % TEST_SLOTS), SZ_4K) && + test_swap_tags_match(mm_subpage_folio(slot), ((i * 5 + 2) % TEST_SLOTS)); + mm_subpage_put(slot); + } + pte_unmap_unlock(ptep, ptl); + KUNIT_EXPECT_TRUE(test, intact); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_SWAPENTS), 0UL); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 1U); + /* A write fault must swap in and then COW just the child's selected slot. */ + mmap_read_lock(child); + fault = handle_mm_fault(cvma, TEST_VA + SZ_4K, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, child, TEST_VA + SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + folio = test_fault_folio(child, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, base + 1, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, 1)); + memset(folio_address(folio) + SZ_4K, 0xa7, SZ_4K); + folio_put(folio); + folio = test_fault_folio(parent->vm_mm, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, base + 1, SZ_4K), NULL); + folio_put(folio); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 1UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), TEST_SLOTS - 1UL); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), i == 1 ? 0U : 1U); + /* Exit with three swapped slots, then tear down the restored parent. */ + kunit_release_action(test, user4k_vma_mm_free, child); + /* Include extended counts when a native page has 16 user slots. */ + count = swp_swapcount(entry); + KUNIT_EXPECT_EQ(test, count, 0U); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 0U); + kunit_release_action(test, user4k_vma_mm_free, parent->vm_mm); +} + +static void user4k_swap_mremap_test(struct kunit *test) +{ + struct mm_struct *mm; + struct folio *folio; + unsigned long old = TEST_VA, new = SZ_1G - SZ_4K, mapped, moved; + pte_t saved[TEST_SLOTS]; + u8 byte = 0x55; + int ret; + unsigned int i; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, old, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, old); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old, &byte, 1, FOLL_WRITE), 1); + folio = test_fault_folio(mm, old); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0x81 + i, SZ_4K); + test_swap_tags_set(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < TEST_SLOTS; i++) { + saved[i] = test_fault_entry(test, mm, old + i * SZ_4K); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(softleaf_from_pte(saved[i]))); + } + kthread_use_mm(mm); + moved = test_mremap(old, PAGE_SIZE, PAGE_SIZE, MREMAP_FIXED | MREMAP_MAYMOVE, new); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, moved, new); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + pte_t entry = test_fault_entry(test, mm, new + i * SZ_4K); + + KUNIT_EXPECT_EQ(test, softleaf_from_pte(entry).val, softleaf_from_pte(saved[i]).val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(entry), i * (unsigned int)SZ_4K); + } + for (i = 0; i < TEST_SLOTS; i++) { + unsigned int slot = (i + 2) % TEST_SLOTS; + unsigned long addr = new + slot * SZ_4K; + + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, addr, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)(0x81 + slot)); + folio = test_fault_folio(mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, slot)); + folio_put(folio); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); +} + +static void user4k_swap_pin_test(struct kunit *test) +{ + struct vm_area_struct *vma; + struct fragment_test_pins *pins; + struct folio *folio; + phys_addr_t pinned_phys; + unsigned int i; + vm_fault_t fault; + long ret; + pte_t pte; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, 4); + KUNIT_ASSERT_EQ(test, ret, 4L); + for (i = 0; i < 4; i++) + memset(folio_address(pins->fragments[i].folio) + pins->fragments[i].offset, + 0xb1 + i, pins->fragments[i].length); + test_swap_tags_set(pins->fragments[0].folio); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA + SZ_4K, SZ_4K, FOLL_WRITE, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + pinned_phys = mm_subpage_phys(pins->fragments[0].subpage); + folio = pins->fragments[0].folio; + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + KUNIT_ASSERT_EQ(test, ret, 0L); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), pinned_phys); + KUNIT_EXPECT_TRUE(test, mm_subpage_is_exclusive(pins->fragments[0].subpage)); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(pins->fragments[0].subpage), 1U); + memset(folio_address(folio) + SZ_4K, 0xd2, SZ_4K); + fragment_test_unpin(pins); + /* Once released, the formerly pinned PTE can be swapped normally. */ + folio = test_fault_folio(vma->vm_mm, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + ret = folio_mapcount(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0L); + for (i = 0; i < 4; i++) { + unsigned long addr = TEST_VA + i * SZ_4K; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, addr, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + folio = test_fault_folio(vma->vm_mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + i == 1 ? 0xd2 : 0xb1 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, i)); + folio_put(folio); + } +} + +#ifdef CONFIG_MIGRATION +static void user4k_swap_migration_test(struct kunit *test) +{ + struct vm_area_struct *vma, *cvma; + struct mm_struct *child; + struct folio *folio; + phys_addr_t old_phys; + unsigned int i, side, succeeded = 0; + vm_fault_t fault; + int ret; + LIST_HEAD(folios); + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + old_phys = PFN_PHYS(folio_pfn(folio)); + folio_lock(folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0xa1 + i, SZ_4K); + test_swap_tags_set(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + mmap_read_lock(child); + fault = handle_mm_fault(cvma, TEST_VA + SZ_4K, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(child, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, folio_test_swapcache(folio)); + ret = isolate_folio_to_list(folio, &folios) ? 0 : -EBUSY; + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + /* One present slot and seven swap entries must all follow the new cache folio. */ + for (side = 0; side < 2; side++) { + struct vm_area_struct *area = side ? cvma : vma; + struct mm_struct *mm = area->vm_mm; + + for (i = 0; i < TEST_SLOTS; i++) { + unsigned long addr = TEST_VA + i * SZ_4K; + + mmap_read_lock(mm); + fault = handle_mm_fault(area, addr, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + folio = test_fault_folio(mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_NE(test, PFN_PHYS(folio_pfn(folio)), old_phys); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0xa1 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, i)); + folio_put(folio); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); + } +} +#endif + +static void user4k_swap_cached_test(struct kunit *test) +{ + test_user4k_swap_roundtrip(test, 0, 0x51); +} + +static void user4k_swap_disk_test(struct kunit *test) +{ + test_user4k_swap_roundtrip(test, 1, 0x61); +} + +static void user4k_swap_reclaim_test(struct kunit *test) +{ + test_user4k_swap_roundtrip(test, 2, 0x71); +} + +static void user4k_swap_stress_test(struct kunit *test) +{ + unsigned int i; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + for (i = 0; i < 32; i++) + test_user4k_swap_roundtrip(test, i % 3, 0x21 + i * 5); + kunit_info(test, "32 swap/fork/COW/exit cycles with distinct data patterns\n"); +} + +static void user4k_swapoff_test(struct kunit *test) +{ + struct vm_area_struct *parent, *cvma; + struct mm_struct *child; + struct folio *folio; + bool *disabled; + vm_fault_t fault; + unsigned int i, side; + swp_entry_t entry; + int ret; + pte_t pte; + LIST_HEAD(folios); + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + parent = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, parent); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(parent->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0x91 + i, SZ_4K); + test_swap_tags_set(folio); + ret = folio_isolate_lru(folio) ? 0 : -EBUSY; + if (!ret) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + pte = test_fault_entry(test, parent->vm_mm, TEST_VA); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + ret = test_swap_control("--swapoff-helper"); + *disabled = !total_swap_pages; + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, *disabled); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : parent->vm_mm; + + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte) & ~PAGE_MASK, + i * (unsigned long)SZ_4K); + folio = test_fault_folio(mm, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x91 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, i)); + folio_put(folio); + } + } + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + mmap_read_lock(child); + fault = handle_mm_fault(cvma, TEST_VA + 3 * SZ_4K, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + pte = test_fault_entry(test, child, TEST_VA + 3 * SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, pte), pte_phys_mm(parent->vm_mm, + test_fault_entry(test, parent->vm_mm, TEST_VA + 3 * SZ_4K))); + ret = test_swap_control("--swapon-helper"); + if (!ret) + *disabled = false; + KUNIT_EXPECT_EQ(test, ret, 0); +} + +static void user4k_swapoff_boundaries_test(struct kunit *test) +{ + const unsigned long boundaries[] = { SZ_2M, SZ_1G, 1UL << 39 }; + bool *disabled; + unsigned int boundary; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + for (boundary = 0; boundary < ARRAY_SIZE(boundaries); boundary++) { + unsigned long start = boundaries[boundary] - SZ_4K; + struct vm_area_struct *parent = test_fault_vma(test, start, 2 * SZ_4K); + struct mm_struct *child; + unsigned int i, side; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, parent); + for (i = 0; i < 2; i++) { + unsigned long addr = start + i * SZ_4K; + unsigned int offset = vma_page_offset_at(parent, addr).offset; + struct folio *folio; + vm_fault_t fault; + swp_entry_t entry; + LIST_HEAD(folios); + + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, addr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(parent->vm_mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + memset(folio_address(folio) + offset, 0xb0 + boundary * 2 + i, SZ_4K); + test_swap_tags_set(folio); + ret = folio_isolate_lru(folio) ? 0 : -EBUSY; + if (!ret) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + entry = softleaf_from_pte(test_fault_entry(test, parent->vm_mm, addr)); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + } + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = test_swap_control("--swapoff-helper"); + *disabled = !total_swap_pages; + KUNIT_ASSERT_EQ(test, ret, 0); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : parent->vm_mm; + + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 2UL); + for (i = 0; i < 2; i++) { + unsigned long addr = start + i * SZ_4K; + unsigned int offset = vma_page_offset_at(parent, addr).offset; + pte_t pte = test_fault_entry(test, mm, addr); + struct folio *folio; + + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte) & ~PAGE_MASK, + (unsigned long)offset); + folio = test_fault_folio(mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + offset, + 0xb0 + boundary * 2 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, offset / SZ_4K)); + folio_put(folio); + } + } + ret = test_swap_control("--swapon-helper"); + if (!ret) + *disabled = false; + KUNIT_ASSERT_EQ(test, ret, 0); + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, parent->vm_mm); + } +} + +struct live_swap_reader { + struct mm_struct *mm; + struct task_struct *task; + struct completion ready, resume, entered, done; + unsigned int id, checks; + int error; +}; + +static int live_swap_reader_run(void *arg) +{ + struct live_swap_reader *r = arg; + unsigned int phase, i; + + kthread_use_mm(r->mm); + for (phase = 0; phase < 2; phase++) { + if (phase) + complete(&r->entered); + for (i = 0; i < 4; i++) { + u8 value = 0; + void __user *addr = (void __user *)(TEST_VA + i * SZ_4K + 64 + r->id); + + /* Resolve AF, swap and COW faults through the real uaccess path. */ + if (copy_from_user(&value, addr, 1) || value != 0xd1 + i) + r->error = phase ? 2 : 1; + r->checks++; + } + if (!phase) { + complete(&r->ready); + if (!wait_for_completion_timeout(&r->resume, 15 * HZ)) { + r->error = 3; + goto out; + } + } + } + { + u8 value = 0xe0 + r->id; + void __user *addr = (void __user *)(TEST_VA + SZ_4K + 64 + r->id); + + if (copy_to_user(addr, &value, 1)) + r->error = 4; + r->checks++; + } +out: + kthread_unuse_mm(r->mm); + complete(&r->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void live_swap_reader_stop(void *arg) +{ + struct live_swap_reader *r = arg; + + if (r->task) { + complete_all(&r->resume); + kthread_stop(r->task); + r->task = NULL; + } +} + +static void test_live_fault_roundtrip(struct kunit *test, unsigned int mode) +{ + bool migration = mode != 0; + struct vm_area_struct *parent; + struct mm_struct *child; + struct live_swap_reader *readers; + struct folio *folio; + vm_fault_t fault; + unsigned int i, cpu; + swp_entry_t entry; + u64 tcr = read_sysreg(tcr_el1); + int ret; + LIST_HEAD(folios); + + if ((!migration && !total_swap_pages) || !user4k_test_enabled || num_online_cpus() < 2 || + !system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || cpus_have_final_cap(ARM64_HAS_S1POE) || + (tcr & (TCR_EL1_DS | TCR_EL1_A1 | TCR_EL1_T0SZ_MASK)) != + (TCR_EL1_A1 | TCR_T0SZ(48))) { + kunit_skip(test, "requires two CPUs, compatible opt-in 4K TTBR0 controls, and swap for reclaim"); + return; + } + parent = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, parent); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + folio = test_fault_folio(parent->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + for (i = 0; i < 4; i++) + memset(folio_address(folio) + i * SZ_4K, 0xd1 + i, SZ_4K); + folio_put(folio); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + readers = kunit_kcalloc(test, 2, sizeof(*readers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, readers); + cpu = cpumask_first(cpu_online_mask); + for (i = 0; i < 2; i++) { + readers[i].mm = child; + readers[i].id = i; + init_completion(&readers[i].ready); + init_completion(&readers[i].resume); + init_completion(&readers[i].entered); + init_completion(&readers[i].done); + readers[i].task = kthread_create(live_swap_reader_run, &readers[i], + "user4k-swap-read-%u", i); + if (IS_ERR(readers[i].task)) { + readers[i].task = NULL; + KUNIT_FAIL(test, "live swap worker creation failed"); + return; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, live_swap_reader_stop, + &readers[i]), 0); + kthread_bind(readers[i].task, cpu); + cpu = cpumask_next(cpu, cpu_online_mask); + wake_up_process(readers[i].task); + } + for (i = 0; i < 2; i++) + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&readers[i].ready, 10 * HZ), 0UL); + lru_add_drain_all(); + folio = test_fault_folio(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + if (mode == 2) { +#ifdef CONFIG_MIGRATION + struct folio *dst = folio_alloc(GFP_KERNEL, 0); + bool entered = true, blocked = true, unmapped = false; + + if (!dst) { + folio_put(folio); + KUNIT_FAIL(test, "migration waiter destination allocation failed"); + return; + } + folio_lock(folio); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(folio, dst); + if (!ret) { + try_to_migrate(folio, TTU_SYNC); + unmapped = !folio_mapped(folio); + for (i = 0; i < 2; i++) + complete(&readers[i].resume); + for (i = 0; i < 2; i++) { + entered &= !!wait_for_completion_timeout(&readers[i].entered, HZ); + if (wait_for_completion_timeout(&readers[i].done, max(1UL, HZ / 20UL))) { + blocked = false; + complete(&readers[i].done); + } + } + remove_migration_ptes(folio, folio, 0); + mm_subpage_migrate_finish(folio); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(folio); + folio_put(dst); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, entered); + KUNIT_EXPECT_TRUE(test, blocked); + KUNIT_EXPECT_TRUE(test, unmapped); +#endif + } else if (migration) { +#ifdef CONFIG_MIGRATION + unsigned int succeeded = 0; + phys_addr_t old_phys = PFN_PHYS(folio_pfn(folio)); + + ret = isolate_folio_to_list(folio, &folios) ? 0 : -EBUSY; + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_ASYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, test_fault_entry(test, child, TEST_VA)), old_phys); +#endif + } else { + ret = folio_isolate_lru(folio) ? 0 : -EBUSY; + if (!ret) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + entry = softleaf_from_pte(test_fault_entry(test, child, TEST_VA)); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + } + for (i = 0; i < 2; i++) + complete(&readers[i].resume); + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&readers[i].done, 15 * HZ), 0UL); + live_swap_reader_stop(&readers[i]); + KUNIT_EXPECT_EQ(test, readers[i].error, 0); + KUNIT_EXPECT_EQ(test, readers[i].checks, 9U); + } + folio = test_fault_folio(child, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + for (i = 0; i < 2; i++) + KUNIT_EXPECT_EQ(test, *((u8 *)folio_address(folio) + SZ_4K + 64 + i), (u8)(0xe0 + i)); + folio_put(folio); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA + SZ_4K, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + folio = test_fault_folio(parent->vm_mm, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0xd2, SZ_4K), NULL); + folio_put(folio); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 4UL); + kunit_info(test, "two CPUs access through uaccess after %s, then write one child slot\n", + mode == 2 ? "migration-entry rollback" : migration ? "migration" : "reclaim"); +} + +static void user4k_live_swap_fault_test(struct kunit *test) +{ + test_live_fault_roundtrip(test, 0); +} + +#ifdef CONFIG_MIGRATION +static void user4k_live_migration_test(struct kunit *test) +{ + test_live_fault_roundtrip(test, 1); +} + +static void user4k_migration_wait_test(struct kunit *test) +{ + test_live_fault_roundtrip(test, 2); +} +#endif + +/* Transfer PTE ownership to normal mm teardown, which also handles swap. */ +static int anon_slot_fixture_handoff(struct kunit *test, struct anon_slot_fixture *f) +{ + struct mm_struct *mm = f->mm; + + f->mm = NULL; + f->vma = NULL; + memset(f->slots, 0, sizeof(f->slots)); + memset(f->mapped, 0, sizeof(f->mapped)); + return kunit_add_action_or_reset(test, user4k_vma_mm_free, mm); +} + +static void test_nonlinear_swap(struct kunit *test, bool evict) +{ + unsigned int first; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + for (first = 0; first < 2; first++) { + struct anon_slot_fixture *a = anon_slot_fixture_create(test); + struct anon_slot_fixture *b = anon_slot_fixture_create(test); + struct vm_area_struct *vmas[2]; + struct mm_struct *child; + struct mm_subpage *slot; + struct folio *folio; + swp_entry_t entry = { }; + unsigned int side; + vm_fault_t fault; + int err, mapped; + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + vmas[0] = a->vma; + vmas[1] = b->vma; + slot = mm_subpage_alloc_at(a->pool, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + folio = mm_subpage_folio(slot); + err = mem_cgroup_charge(folio, a->mm, GFP_KERNEL); + if (err) { + mm_subpage_put(slot); + KUNIT_FAIL(test, "cannot charge swap fixture: %d", err); + return; + } + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(a, 0, slot), 0); + folio_lock(folio); + mm_subpage_set_exclusive(slot); + folio_unlock(folio); + slot = mm_subpage_alloc_at(a->pool, 3 * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(b, 1, slot), 0); + folio_lock(folio); + mm_subpage_set_exclusive(slot); + folio_unlock(folio); + memset(folio_address(folio), 0x5b, SZ_4K); + memset(folio_address(folio) + 3 * SZ_4K, 0xd3, SZ_4K); + test_swap_tags_set(folio); + folio_mark_dirty(folio); + folio_add_lru(folio); + lru_add_drain_all(); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_handoff(test, a), 0); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_handoff(test, b), 0); + folio_get(folio); + folio_lock(folio); + err = folio_alloc_swap(folio); + if (!err) { + entry = folio->swap; + try_to_unmap(folio, TTU_SYNC); + } + mapped = folio_mapcount(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, mapped, 0); + for (side = 0; side < 2; side++) { + pte_t pte = test_fault_entry(test, vmas[side]->vm_mm, + TEST_VA + side * SZ_4K); + + KUNIT_EXPECT_EQ(test, softleaf_from_pte(pte).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(pte), side ? 3U * SZ_4K : 0U); + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, side ? 3 * SZ_4K : 0), 1U); + } + if (evict) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + /* Preserve physical offsets when a nonpresent nonlinear slot is forked. */ + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, vmas[1]->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + for (side = 0; side < 2; side++) { + unsigned int which = side ^ first, physical = which ? 3 : 0; + struct vm_area_struct *vma = vmas[which]; + unsigned long address = TEST_VA + which * SZ_4K; + pte_t pte; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ_MSG(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), + (vm_fault_t)0, "evict=%d first=%u side=%u", evict, first, which); + KUNIT_EXPECT_EQ(test, !!(fault & VM_FAULT_MAJOR), evict && !side); + pte = test_fault_entry(test, vma->vm_mm, address); + slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), physical * (unsigned int)SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), which ? 0xd3 : 0x5b, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(mm_subpage_folio(slot), physical)); + mm_subpage_put(slot); + } + /* A child's write must copy only its slot at the new virtual offset. */ + { + struct vm_area_struct *vma = test_vma_lookup(child, TEST_VA + SZ_4K); + pte_t pte; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(child); + fault = handle_mm_fault(vma, TEST_VA + SZ_4K, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, child, TEST_VA + SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + slot = mm_subpage_get_from_phys(pte_phys_mm(child, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0xd3, SZ_4K), NULL); + memset(folio_address(mm_subpage_folio(slot)) + mm_subpage_offset(slot), 0x91, SZ_4K); + mm_subpage_put(slot); + pte = test_fault_entry(test, vmas[1]->vm_mm, TEST_VA + SZ_4K); + slot = mm_subpage_get_from_phys(pte_phys_mm(vmas[1]->vm_mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0xd3, SZ_4K), NULL); + mm_subpage_put(slot); + } + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, vmas[1]->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, vmas[0]->vm_mm); + } +} + +static void user4k_nonlinear_swap_cached_test(struct kunit *test) +{ + test_nonlinear_swap(test, false); +} + +static void user4k_nonlinear_swap_disk_test(struct kunit *test) +{ + test_nonlinear_swap(test, true); +} + +#ifdef CONFIG_USERFAULTFD +/* The context is attached only during the call; ordinary mm cleanup owns the VMAs. */ +static ssize_t test_uffd_move(struct vm_area_struct *dst, unsigned long dst_addr, + unsigned long src_addr, unsigned long size) +{ + struct userfaultfd_ctx ctx = { .mm = dst->vm_mm }; + ssize_t ret; + + init_rwsem(&ctx.map_changing_lock); + atomic_set(&ctx.mmap_changing, 0); + mmap_write_lock(ctx.mm); + vma_start_write(dst); + dst->vm_userfaultfd_ctx.ctx = &ctx; + mmap_write_unlock(ctx.mm); + ret = move_pages(&ctx, dst_addr, src_addr, size, 0); + mmap_write_lock(ctx.mm); + vma_start_write(dst); + dst->vm_userfaultfd_ctx.ctx = NULL; + mmap_write_unlock(ctx.mm); + return ret; +} + +struct move_migration_worker { + struct userfaultfd_ctx ctx; + struct vm_area_struct *vma[2]; + struct task_struct *task; + struct completion ready; + unsigned long address[2], size; + unsigned long moves, busy; + int error; +}; + +static int move_migration_worker_run(void *arg) +{ + struct move_migration_worker *w = arg; + + while (!kthread_should_stop()) { + unsigned int side = w->moves & 1; + ssize_t ret = move_pages(&w->ctx, w->address[side ^ 1], + w->address[side], w->size, 0); + + if (ret == w->size) + WRITE_ONCE(w->moves, w->moves + 1); + else if (ret == -EAGAIN || ret == -EBUSY) + w->busy++; + else + w->error = ret < 0 ? ret : -EIO; + if (w->moves || w->error) + complete_all(&w->ready); + if (w->error) { + /* Keep the task valid until the owner joins it. */ + set_current_state(TASK_INTERRUPTIBLE); + if (!kthread_should_stop()) + schedule(); + __set_current_state(TASK_RUNNING); + } + /* Leave bounded gaps so migration can complete between contended moves. */ + if (ret == w->size) + usleep_range(1000, 2000); + else + cond_resched(); + } + return 0; +} + +static void move_migration_worker_stop(void *arg) +{ + struct move_migration_worker *w = arg; + unsigned int i; + + if (w->task) { + kthread_stop(w->task); + w->task = NULL; + } + mmap_write_lock(w->ctx.mm); + for (i = 0; i < 2; i++) { + vma_start_write(w->vma[i]); + w->vma[i]->vm_userfaultfd_ctx.ctx = NULL; + } + mmap_write_unlock(w->ctx.mm); +} + +static void user4k_move_migration_race_test(struct kunit *test) +{ + unsigned int shift; + + for (shift = 12; shift < PAGE_SHIFT; shift += 2) { + struct mm_struct *mm = test_vma_mm_granule(test, shift); + struct move_migration_worker *w; + unsigned long size = 1UL << shift, i, initial_moves; + unsigned int round, migrated = 0, failed = 0; + pte_t pte; + vm_fault_t fault; + + KUNIT_ASSERT_NOT_NULL(test, mm); + w = kunit_kzalloc(test, sizeof(*w), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, w); + w->ctx.mm = mm; + init_rwsem(&w->ctx.map_changing_lock); + atomic_set(&w->ctx.mmap_changing, 0); + init_completion(&w->ready); + w->size = size; + w->address[0] = TEST_VA + 3 * size; + w->address[1] = TEST_VA + SZ_128M + size; + w->vma[0] = test_fault_vma_in_mm(mm, TEST_VA, PAGE_SIZE); + w->vma[1] = test_fault_vma_in_mm(mm, TEST_VA + SZ_128M, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, w->vma[0]); + KUNIT_ASSERT_NOT_NULL(test, w->vma[1]); + for (i = 0; i < PAGE_SIZE; i += size) { + mmap_read_lock(mm); + fault = handle_mm_fault(w->vma[0], TEST_VA + i, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, mm, TEST_VA + i); + memset(phys_to_virt(pte_phys_mm(mm, pte)), 0x60 + i / size, size); + } + /* Both directions are registered before starting the concurrent mover. */ + mmap_write_lock(mm); + for (i = 0; i < 2; i++) { + vma_start_write(w->vma[i]); + w->vma[i]->vm_userfaultfd_ctx.ctx = &w->ctx; + } + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, move_migration_worker_stop, w), 0); + w->task = kthread_create(move_migration_worker_run, w, "user4k-move"); + if (IS_ERR(w->task)) { + w->task = NULL; + KUNIT_FAIL(test, "MOVE worker creation failed"); + return; + } + if (num_online_cpus() > 1) + kthread_bind(w->task, cpumask_last(cpu_online_mask)); + wake_up_process(w->task); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&w->ready, 10 * HZ), 0UL); + initial_moves = READ_ONCE(w->moves); + lru_add_drain_all(); + for (round = 0; round < 64; round++) { + struct folio *folio; + unsigned int succeeded = 0; + bool isolated; + int ret; + LIST_HEAD(folios); + + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + isolated = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_TRUE_MSG(test, isolated, "round=%u", round); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_EXPECT_GE(test, ret, 0); + migrated += succeeded; + failed += ret > 0; + cond_resched(); + } + kunit_release_action(test, move_migration_worker_stop, w); + kunit_info(test, "%uK MOVE/migration: moves=%lu retries=%lu migrated=%u busy=%u\n", + 1U << (shift - 10), w->moves, w->busy, migrated, failed); + KUNIT_EXPECT_EQ(test, w->error, 0); + KUNIT_EXPECT_GT(test, w->moves, initial_moves); + KUNIT_EXPECT_GT(test, migrated, 0U); + for (i = 0; i < PAGE_SIZE; i += size) { + unsigned long addr = i == 3 * size ? w->address[w->moves & 1] : TEST_VA + i; + + pte = test_fault_entry(test, mm, addr); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(mm, pte)), + 0x60 + i / size, size), NULL); + } + KUNIT_EXPECT_TRUE(test, pte_none(test_fault_entry(test, mm, w->address[(w->moves & 1) ^ 1]))); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_move_swap_get_test(struct kunit *test) +{ + unsigned int variant; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + for (variant = 0; variant < PAGE_SHIFT - 12; variant++) { + unsigned int shift = 12 + (variant / 2) * 2; + bool forked = variant & 1; + unsigned long size = 1UL << shift; + unsigned long src_addr = TEST_VA + 3 * size; + unsigned long dst_addr = TEST_VA + SZ_128M + size; + struct mm_struct *mm = test_vma_mm_granule(test, shift); + struct vm_area_struct *src, *dst; + struct mm_subpage *held; + struct folio *folio; + phys_addr_t physical; + swp_entry_t entry = { }; + pte_t before, after; + vm_fault_t fault; + unsigned long i; + int err, mapped; + + KUNIT_ASSERT_NOT_NULL(test, mm); + src = test_fault_vma_in_mm(mm, TEST_VA, PAGE_SIZE); + dst = test_fault_vma_in_mm(mm, TEST_VA + SZ_128M, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, src); + KUNIT_ASSERT_NOT_NULL(test, dst); + for (i = 0; i < PAGE_SIZE; i += size) { + mmap_read_lock(mm); + fault = handle_mm_fault(src, TEST_VA + i, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + before = test_fault_entry(test, mm, TEST_VA + i); + memset(phys_to_virt(pte_phys_mm(mm, before)), 0x50 + i / size, size); + } + before = test_fault_entry(test, mm, src_addr); + physical = pte_phys_mm(mm, before); + held = mm_subpage_get_from_phys(physical); + KUNIT_ASSERT_NOT_NULL(test, held); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, held), 0); + KUNIT_ASSERT_EQ(test, mm_subpage_offset(held), (unsigned int)(3 * size)); + folio = mm_subpage_folio(held); + lru_add_drain_all(); + folio_lock(folio); + err = folio_alloc_swap(folio); + if (!err) { + entry = folio->swap; + try_to_unmap(folio, TTU_SYNC); + } + mapped = folio_mapcount(folio); + folio_unlock(folio); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, mapped, 0); + KUNIT_ASSERT_EQ(test, mm_subpage_mapcount(held), 0); + KUNIT_ASSERT_EQ(test, swap_subpage_count(entry, 3 * size), 1U); + before = test_fault_entry(test, mm, src_addr); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(softleaf_from_pte(before))); + if (forked) { + struct mm_struct *child = test_vma_mm_granule(test, shift); + + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, test_uffd_move(dst, dst_addr, src_addr, size), + (ssize_t)-EBUSY); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(test_fault_entry(test, mm, + src_addr)).val, entry.val); + kunit_release_action(test, user4k_vma_mm_free, child); + before = test_fault_entry(test, mm, src_addr); + KUNIT_EXPECT_FALSE(test, pte_swp_exclusive(before)); + KUNIT_ASSERT_EQ(test, swap_subpage_count(entry, 3 * size), 1U); + } + KUNIT_ASSERT_EQ(test, test_uffd_move(dst, dst_addr, src_addr, size), (ssize_t)size); + after = test_fault_entry(test, mm, dst_addr); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(after).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(after), (unsigned int)(3 * size)); + KUNIT_EXPECT_TRUE(test, pte_none(test_fault_entry(test, mm, src_addr))); + /* Restore with the retained metadata already rebound to the new root. */ + mmap_read_lock(mm); + fault = handle_mm_fault(dst, dst_addr, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + after = test_fault_entry(test, mm, dst_addr); + KUNIT_ASSERT_TRUE(test, pte_present(after)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, after), physical); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(physical), 0x53, size), NULL); + KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(held), !forked); + /* Fork-cleared exclusivity requires COW while a GET is retained. */ + mmap_read_lock(mm); + fault = handle_mm_fault(dst, dst_addr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + after = test_fault_entry(test, mm, dst_addr); + KUNIT_ASSERT_TRUE(test, pte_write(after)); + if (forked) + KUNIT_EXPECT_NE(test, pte_phys_mm(mm, after), physical); + else + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, after), physical); + memset(phys_to_virt(pte_phys_mm(mm, after)), 0xa9, size); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(physical), + forked ? 0x53 : 0xa9, size), NULL); + for (i = 0; i < PAGE_SIZE; i += size) { + if (i == 3 * size) + continue; + mmap_read_lock(mm); + fault = handle_mm_fault(src, TEST_VA + i, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + after = test_fault_entry(test, mm, TEST_VA + i); + KUNIT_ASSERT_TRUE(test, pte_present(after)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(mm, after)), + 0x50 + i / size, size), NULL); + } + kunit_release_action(test, test_subpage_put, held); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} +#endif + +static struct kunit_case user4k_swap_cases[] = { + KUNIT_CASE(user4k_swap_reuse_get_test), + KUNIT_CASE(user4k_swap_reuse_swapoff_test), + KUNIT_CASE(user4k_swap_reuse_cached_test), + KUNIT_CASE(user4k_swap_reuse_disk_test), + KUNIT_CASE(user4k_swap_exclusive_cached_test), + KUNIT_CASE(user4k_swap_exclusive_disk_test), + KUNIT_CASE(user4k_swap_exclusive_swapoff_test), + KUNIT_CASE(user4k_swap_cached_test), + KUNIT_CASE(user4k_swap_pin_test), + KUNIT_CASE(user4k_swap_mremap_test), +#ifdef CONFIG_MIGRATION + KUNIT_CASE(user4k_swap_migration_test), +#endif + KUNIT_CASE(user4k_swap_disk_test), + KUNIT_CASE(user4k_swap_reclaim_test), + KUNIT_CASE(user4k_swapoff_test), + KUNIT_CASE(user4k_swap_stress_test), + KUNIT_CASE(user4k_swapoff_boundaries_test), + KUNIT_CASE(user4k_live_swap_fault_test), + {} +}; + +static void user4k_swap_suite_exit(struct kunit_suite *suite) +{ + kvfree_rcu_barrier(); +} + +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +static void test_shmem_uffd_swap(struct kunit *test, unsigned int mode) +{ + const unsigned long missing = GENMASK(PAGE_SIZE / SZ_4K - 1, mode == 3 ? 2 : 1); + struct swap_iocb *plug = NULL; + struct file *file; + struct folio *folio; + void *entry; + bool *disabled; + int ret; + LIST_HEAD(folios); + + if (!total_swap_pages) { + kunit_skip(test, "requires disposable guest swap"); + return; + } + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + file = test_cache_file(test, PAGE_SIZE, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = shmem_get_folio(file_inode(file), 0, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + memset(folio_address(folio), 0xb6, mode == 3 ? 2 * SZ_4K : SZ_4K); + ret = shmem_uffd_set_missing(folio, 0, missing); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + /* Match pageout(): submitted data is clean unless writeout redirties it. */ + KUNIT_EXPECT_TRUE(test, folio_clear_dirty_for_io(folio)); + ret = shmem_writeout(folio, &plug, &folios); + if (plug) + swap_write_unplug(plug); + if (ret == AOP_WRITEPAGE_ACTIVATE) + folio_unlock(folio); + folio_wait_writeback(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_TRUE(test, list_empty(&folios)); + entry = filemap_get_entry(file->f_mapping, 0); + if (entry && !xa_is_value(entry)) + folio_put(entry); + KUNIT_ASSERT_TRUE(test, xa_is_value(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(radix_to_swp_entry(entry))); + KUNIT_EXPECT_EQ(test, xa_to_value(xa_load(&SHMEM_I(file_inode(file))->uffd_missing, 0)), missing); + if (mode == 1) { + shmem_truncate_range(file_inode(file), 0, PAGE_SIZE - 1); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + } else if (mode == 3) { + shmem_truncate_range(file_inode(file), 0, SZ_4K - 1); + } else if (mode == 2) { + ret = test_swap_control("--swapoff-helper"); + *disabled = ret == 0; + KUNIT_ASSERT_EQ(test, ret, 0); + } + ret = shmem_get_folio(file_inode(file), 0, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), + mode == 1 ? 0UL : mode == 3 ? missing | BIT(0) : missing); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), + mode == 1 || mode == 3 ? 0 : 0xb6, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, + mode == 3 ? 0xb6 : 0, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + 2 * SZ_4K, + 0, PAGE_SIZE - 2 * SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + folio_unlock(folio); + folio_put(folio); +} + +static void user4k_shmem_uffd_swapin_test(struct kunit *test) +{ + test_shmem_uffd_swap(test, 0); +} + +static void user4k_shmem_uffd_swapped_truncate_test(struct kunit *test) +{ + test_shmem_uffd_swap(test, 1); +} + +static void user4k_shmem_uffd_swapped_partial_hole_test(struct kunit *test) +{ + test_shmem_uffd_swap(test, 3); +} + +static void user4k_shmem_uffd_swapoff_test(struct kunit *test) +{ + test_shmem_uffd_swap(test, 2); +} + +#if defined(CONFIG_FAILSLAB) && defined(CONFIG_FAULT_INJECTION_DEBUG_FS) +static void test_disable_failslab(void *unused) +{ + test_swap_control("--failslab-off-helper"); +} +#endif + +static void user4k_shmem_uffd_swap_alloc_fail_test(struct kunit *test) +{ +#if defined(CONFIG_FAILSLAB) && defined(CONFIG_FAULT_INJECTION_DEBUG_FS) + unsigned int attempt; + + if (!total_swap_pages) { + kunit_skip(test, "requires disposable guest swap"); + return; + } + KUNIT_ASSERT_EQ(test, test_swap_control("--failslab-on-helper"), 0); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_disable_failslab, NULL), 0); + for (attempt = 1; attempt <= 2; attempt++) { + pgoff_t index = attempt == 1 ? 1 : 64; + const unsigned long missing = GENMASK(PAGE_SIZE / SZ_4K - 1, 1); + struct file *file = test_cache_file(test, (index + 1) * PAGE_SIZE, false); + struct swap_iocb *plug = NULL; + struct folio *folio; + unsigned int pending; + unsigned long pfn; + void *entry; + int ret; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = shmem_get_folio(file_inode(file), index, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + memset(folio_address(folio), 0xb6, SZ_4K); + ret = shmem_uffd_set_missing(folio, index, missing); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + folio = filemap_lock_folio(file->f_mapping, index); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + pfn = folio_pfn(folio); + KUNIT_EXPECT_TRUE(test, folio_clear_dirty_for_io(folio)); + /* Fail this task's selected allocation and its allocator retries. */ + WRITE_ONCE(current->make_it_fail, 1); + WRITE_ONCE(current->fail_nth, attempt); + ret = shmem_writeout(folio, &plug, &folios); + pending = READ_ONCE(current->fail_nth); + WRITE_ONCE(current->fail_nth, 0); + WRITE_ONCE(current->make_it_fail, 0); + KUNIT_EXPECT_EQ(test, pending, 0U); + KUNIT_EXPECT_EQ(test, ret, AOP_WRITEPAGE_ACTIVATE); + if (ret != AOP_WRITEPAGE_ACTIVATE) { + if (plug) + swap_write_unplug(plug); + folio_wait_writeback(folio); + folio_put(folio); + return; + } + KUNIT_EXPECT_TRUE(test, folio_test_locked(folio)); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(folio)); + KUNIT_EXPECT_PTR_EQ(test, folio->mapping, file->f_mapping); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, index), missing); + /* A failed multi-level insertion may retain an empty internal node. */ + { + unsigned long cursor = 0; + + KUNIT_EXPECT_PTR_EQ(test, xa_find(&SHMEM_I(file_inode(file))->uffd_missing, + &cursor, ULONG_MAX, XA_PRESENT), NULL); + } + KUNIT_EXPECT_PTR_EQ(test, plug, NULL); + KUNIT_EXPECT_TRUE(test, list_empty(&folios)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xb6, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, PAGE_SIZE - SZ_4K), NULL); + /* No restart or cache repair: retry exactly the retained folio. */ + KUNIT_EXPECT_EQ(test, folio_pfn(folio), pfn); + KUNIT_EXPECT_TRUE(test, folio_clear_dirty_for_io(folio)); + ret = shmem_writeout(folio, &plug, &folios); + if (plug) + swap_write_unplug(plug); + if (ret == AOP_WRITEPAGE_ACTIVATE) + folio_unlock(folio); + folio_wait_writeback(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + entry = filemap_get_entry(file->f_mapping, index); + if (entry && !xa_is_value(entry)) + folio_put(entry); + KUNIT_ASSERT_TRUE(test, xa_is_value(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(radix_to_swp_entry(entry))); + KUNIT_EXPECT_EQ(test, xa_to_value(xa_load(&SHMEM_I(file_inode(file))->uffd_missing, index)), missing); + ret = shmem_get_folio(file_inode(file), index, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, index), missing); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xb6, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, PAGE_SIZE - SZ_4K), NULL); + folio_unlock(folio); + folio_put(folio); + kunit_info(test, "failed metadata allocation %u at index %lu; same-folio retry and disk swap-in preserved bytes/mask", attempt, index); + } +#else + kunit_skip(test, "requires FAILSLAB and fault-injection debugfs"); +#endif +} + +#ifdef CONFIG_MEMCG +static void test_memcg_put(void *memcg) +{ + mem_cgroup_put(memcg); +} +#endif + +static void user4k_shmem_uffd_swap_charge_test(struct kunit *test) +{ +#ifdef CONFIG_MEMCG + struct cgroup *cgroup; + struct cgroup_subsys_state *css; + struct mem_cgroup *memcg, *saved; + struct file *file; + struct folio *folio; + struct swap_iocb *plug = NULL; + void *entry; + const unsigned long missing = GENMASK(PAGE_SIZE / SZ_4K - 1, 1); + int ret; + LIST_HEAD(folios); + + if (!total_swap_pages || mem_cgroup_disabled()) { + kunit_skip(test, "requires disposable guest swap and memcg"); + return; + } + KUNIT_ASSERT_EQ(test, test_swap_control("--memcg-setup-helper"), 0); + cgroup = cgroup_get_from_path("/uffd-metadata"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cgroup); + css = cgroup_get_e_css(cgroup, &memory_cgrp_subsys); + cgroup_put(cgroup); + KUNIT_ASSERT_NOT_NULL(test, css); + memcg = mem_cgroup_from_css(css); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_memcg_put, memcg), 0); + KUNIT_ASSERT_PTR_NE(test, memcg, root_mem_cgroup); + file = test_cache_file(test, 65 * PAGE_SIZE, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + saved = set_active_memcg(memcg); + ret = shmem_get_folio(file_inode(file), 64, 0, &folio, SGP_CACHE); + set_active_memcg(saved); + KUNIT_ASSERT_EQ(test, ret, 0); + rcu_read_lock(); + KUNIT_EXPECT_PTR_EQ(test, folio_memcg(folio), memcg); + rcu_read_unlock(); + ret = shmem_uffd_set_missing(folio, 64, missing); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + folio = filemap_lock_folio(file->f_mapping, 64); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, folio_clear_dirty_for_io(folio)); + /* Reclaim from the root charge domain, not the folio's cgroup. */ + saved = set_active_memcg(root_mem_cgroup); + ret = shmem_writeout(folio, &plug, &folios); + set_active_memcg(saved); + if (plug) + swap_write_unplug(plug); + if (ret == AOP_WRITEPAGE_ACTIVATE) + folio_unlock(folio); + folio_wait_writeback(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + rcu_read_lock(); + entry = rcu_dereference(SHMEM_I(file_inode(file))->uffd_missing.xa_head); + if (xa_is_node(entry)) + KUNIT_EXPECT_PTR_EQ(test, mem_cgroup_from_virt(xa_to_node(entry)), memcg); + rcu_read_unlock(); + KUNIT_ASSERT_TRUE(test, xa_is_node(entry)); + entry = filemap_get_entry(file->f_mapping, 64); + if (entry && !xa_is_value(entry)) + folio_put(entry); + KUNIT_ASSERT_TRUE(test, xa_is_value(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(radix_to_swp_entry(entry))); + ret = shmem_get_folio(file_inode(file), 64, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 64), missing); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + folio_unlock(folio); + folio_put(folio); +#else + kunit_skip(test, "requires memcg"); +#endif +} + +static bool test_coarse_swap_enabled(struct kunit *test) +{ + if (!test_coarse_file_enabled(test)) + return false; + if (!total_swap_pages || !IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "requires disposable coarse-shmem swap runner"); + return false; + } + return true; +} + +static bool test_coarse_swap_out(struct kunit *test, struct file *file, pgoff_t index) +{ + unsigned int tries; + + for (tries = 0; tries < 100; tries++) { + struct folio *folio; + void *entry = filemap_get_entry(file->f_mapping, index); + LIST_HEAD(folios); + + if (xa_is_value(entry)) + return test_reclaim_swapcache(radix_to_swp_entry(entry)); + if (!entry) { + KUNIT_FAIL(test, "reclaim removed data without a swap entry"); + return false; + } + folio = entry; + lru_add_drain_all(); + if (folio_isolate_lru(folio)) + list_add(&folio->lru, &folios); + folio_put(folio); + /* Real reclaim owns any folios produced by a large-folio split. */ + reclaim_pages(&folios); + schedule_timeout_uninterruptible(1); + } + KUNIT_FAIL(test, "file index %lu did not reach swap", index); + return false; +} + +static void test_coarse_cold_shmem(struct kunit *test, bool promoted) +{ + struct file *file; + struct vm_area_struct *coarse, *small; + struct folio *folio; + unsigned int nr = SZ_64K / PAGE_SIZE, i; + unsigned long offset; + vm_fault_t fault; + + if (!test_coarse_swap_enabled(test)) + return; + file = test_cache_file(test, SZ_64K, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, 0); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + SZ_64K - SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, small); + if (promoted) { + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA, false) & VM_FAULT_ERROR, 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), SZ_64K); + folio_put(folio); + } + for (offset = SZ_4K; offset < SZ_64K; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + for (i = 0; i < nr; i++) + KUNIT_ASSERT_TRUE(test, test_coarse_swap_out(test, file, i)); + /* XArray swap entries and absent swapcache prove this is a cold refault. */ + for (i = 0; i < nr; i++) { + void *entry = filemap_get_entry(file->f_mapping, i); + struct folio *cached = NULL; + + KUNIT_EXPECT_TRUE(test, xa_is_value(entry)); + if (entry && !xa_is_value(entry)) { + folio_put(entry); + return; + } + KUNIT_ASSERT_NOT_NULL(test, entry); + cached = swap_cache_get_folio(radix_to_swp_entry(entry)); + KUNIT_EXPECT_PTR_EQ(test, cached, NULL); + if (cached) + folio_put(cached); + } + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, 0UL); + KUNIT_EXPECT_EQ(test, SHMEM_I(file_inode(file))->swapped, (unsigned long)nr); + fault = test_file_fault(coarse, TEST_VA, false); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_MAJOR); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_EQ(test, folio_size(folio), SZ_64K); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + offset, + 0x31 + offset / SZ_4K, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, (unsigned long)nr); + KUNIT_EXPECT_EQ(test, SHMEM_I(file_inode(file))->swapped, 0UL); + for (offset = SZ_4K; offset < SZ_64K; offset += SZ_4K) { + pte_t entry; + + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, small->vm_mm, TEST_VA + offset); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry), + page_to_phys(&folio->page) + offset); + } + kunit_release_action(test, user4k_vma_mm_free, coarse->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)nr + 1); +} + +static void user64k_shmem_cold_constituents_test(struct kunit *test) +{ + test_coarse_cold_shmem(test, false); +} + +static void user64k_shmem_cold_promoted_test(struct kunit *test) +{ + test_coarse_cold_shmem(test, true); +} + +#if defined(CONFIG_FAILSLAB) && defined(CONFIG_FAIL_PAGE_ALLOC) && defined(CONFIG_MEMCG) +static void test_coarse_injection_off(void *unused) +{ + test_swap_control("--coarse-injection-off"); +} + +#endif + +static void user64k_shmem_fault_alloc_fail_test(struct kunit *test) +{ +#if defined(CONFIG_FAILSLAB) && defined(CONFIG_FAIL_PAGE_ALLOC) && defined(CONFIG_MEMCG) + unsigned int mode; + + if (!test_coarse_swap_enabled(test)) + return; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_coarse_injection_off, NULL), 0); + for (mode = 0; mode < 3; mode++) { + struct file *file = test_cache_file(test, SZ_64K, false); + struct vm_area_struct *vma; + struct folio *folio; + phys_addr_t original[SZ_64K / PAGE_SIZE]; + unsigned char *buffer; + loff_t pos = 0; + unsigned int i, pending = 0; + vm_fault_t result; + bool prior_fail; + char operation[80]; + int enter = 0, leave = 0; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + /* Make the zero-limit charge case nonblocking and avoid an OOM kill. */ + if (mode == 2) + mapping_set_gfp_mask(file->f_mapping, + mapping_gfp_mask(file->f_mapping) & ~__GFP_DIRECT_RECLAIM); + buffer = kunit_kmalloc(test, SZ_64K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, buffer); + memset(buffer, 0x95, SZ_64K); + KUNIT_ASSERT_EQ(test, kernel_write(file, buffer, SZ_64K, &pos), (ssize_t)SZ_64K); + vma = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + for (i = 0; i < SZ_64K / PAGE_SIZE; i++) { + folio = filemap_get_folio(file->f_mapping, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_order(folio), 0U); + original[i] = page_to_phys(&folio->page); + folio_put(folio); + } + if (mode < 2) { + KUNIT_ASSERT_EQ(test, test_swap_control(mode ? "--coarse-page-on" : + "--coarse-slab-on"), 0); + } else { + snprintf(operation, sizeof(operation), "--coarse-memcg-enter=%d", + task_pid_nr(current)); + enter = test_swap_control(operation); + } + /* Direct callback isolates backing allocation from page-table allocation. */ + if (!enter) { + struct vm_fault vmf = { .vma = vma, .address = TEST_VA, + .pgoff = 0, .flags = FAULT_FLAG_REMOTE }; + + mmap_read_lock(vma->vm_mm); + prior_fail = current->make_it_fail; + if (!mode) + WRITE_ONCE(current->fail_nth, 1); + else if (mode == 1) + current->make_it_fail = true; + result = vma->vm_ops->fault(&vmf); + pending = READ_ONCE(current->fail_nth); + WRITE_ONCE(current->fail_nth, 0); + current->make_it_fail = prior_fail; + mmap_read_unlock(vma->vm_mm); + /* Keep unexpected success safe for test cleanup. */ + if (vmf.page) { + if (result & VM_FAULT_LOCKED) + unlock_page(vmf.page); + put_page(vmf.page); + } + } else { + result = 0; + } + if (mode == 2) { + snprintf(operation, sizeof(operation), "--coarse-memcg-leave=%d", + task_pid_nr(current)); + leave = test_swap_control(operation); + } else { + leave = test_swap_control("--coarse-injection-off"); + } + KUNIT_ASSERT_EQ(test, enter, 0); + KUNIT_ASSERT_EQ(test, leave, 0); + KUNIT_EXPECT_EQ(test, pending, 0U); + KUNIT_ASSERT_TRUE(test, result & VM_FAULT_OOM); + for (i = 0; i < SZ_64K / PAGE_SIZE; i++) { + folio = filemap_get_folio(file->f_mapping, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_order(folio), 0U); + KUNIT_EXPECT_EQ(test, page_to_phys(&folio->page), original[i]); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0x95, PAGE_SIZE), NULL); + folio_put(folio); + } + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, SZ_64K / PAGE_SIZE); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SHMEMPAGES), 0L); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & + (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), SZ_64K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0x95, SZ_64K), NULL); + folio_put(folio); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + kunit_info(test, "coarse allocation failure mode %u preserved data and recovered", mode); + } +#else + kunit_skip(test, "requires slab/page fault injection and memcg"); +#endif +} + +static void user64k_anon_swap_test(struct kunit *test) +{ + unsigned int mode, nr = SZ_64K / PAGE_SIZE; + + if (!test_coarse_swap_enabled(test)) + return; + for (mode = 0; mode < 7; mode++) { + struct vm_area_struct *vma, *cvma = NULL; + struct mm_struct *child = NULL; + struct folio *folio, *held = NULL; + swp_entry_t entry, part; + unsigned int i, tries; + pte_t pte; + bool *disabled; + bool warm = !mode || mode == 6; + vm_fault_t fault; + int err; + + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + folio = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, pte)))); + for (i = 0; i < SZ_64K / SZ_4K; i++) + memset(folio_address(folio) + i * SZ_4K, 0x31 + i, SZ_4K); + if (mode == 2 || mode == 6) { + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + } + if (warm) { + folio_get(folio); + held = folio; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, held), 0); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + KUNIT_EXPECT_EQ(test, min_order_for_split(folio), 16U - PAGE_SHIFT); +#endif + folio_lock(folio); + KUNIT_EXPECT_EQ(test, folio_anon_min_order(folio), 16U - PAGE_SHIFT); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + KUNIT_EXPECT_EQ(test, split_huge_page_to_list_to_order(&folio->page, NULL, 0), + -EBUSY); + KUNIT_EXPECT_EQ(test, folio_order(folio), 16U - PAGE_SHIFT); + expect_coarse_anon_state(test, folio, child ? 2 : 1, !child); +#endif + err = folio_alloc_swap(folio); + if (!err) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + KUNIT_ASSERT_EQ(test, err, 0); + } else { + for (tries = 0; tries < 100; tries++) { + LIST_HEAD(folios); + + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + if (!pte_present(pte)) + break; + folio = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, pte)))); + folio_get(folio); + lru_add_drain_all(); + if (folio_isolate_lru(folio)) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + schedule_timeout_uninterruptible(1); + } + } + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_FALSE_MSG(test, pte_present(pte), "mode=%u", mode); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_EXPECT_TRUE(test, IS_ALIGNED(swp_offset(entry), nr)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 1L); + if (mode == 3) { + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + } + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), child ? 2 : 1); + if (!warm) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(part)); + } + if (mode == 3) { + /* Exit a swapped child: every constituent count must be released. */ + kunit_release_action(test, user4k_vma_mm_free, child); + child = NULL; + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 1); + } + } + if (mode == 5) { +#if defined(CONFIG_FAIL_PAGE_ALLOC) && defined(CONFIG_FAULT_INJECTION) + bool old = current->make_it_fail; + + KUNIT_ASSERT_EQ(test, test_swap_control("--coarse-page-on"), 0); + current->make_it_fail = true; + fault = test_file_fault(vma, TEST_VA, false); + current->make_it_fail = old; + err = test_swap_control("--coarse-injection-off"); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_OOM); + KUNIT_EXPECT_TRUE(test, pte_same(test_fault_entry(test, vma->vm_mm, TEST_VA), pte)); + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 1); + } +#endif + } + if (mode == 4) { + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + err = test_swap_control("--swapoff-helper"); + if (!err) + *disabled = true; + KUNIT_ASSERT_EQ(test, err, 0); + } else { + fault = test_file_fault(vma, TEST_VA + SZ_64K - 1, false); + KUNIT_ASSERT_EQ_MSG(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), 0U, + "mode=%u fault=%x", mode, fault); + if (!warm && mode != 5) + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_MAJOR); + } + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE_MSG(test, pte_present(pte), "mode=%u", mode); + folio = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, pte)))); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + if (held) { + KUNIT_EXPECT_PTR_EQ(test, folio, held); /* Warm aligned cache: no copy. */ + } else { + folio_get(folio); + held = folio; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, held), 0); + } + for (i = 0; i < SZ_64K / SZ_4K; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + if (child) { + struct folio *copied; + + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, child, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, pte), page_to_phys(&folio->page)); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + copied = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(child, pte)))); + expect_coarse_anon_state(test, copied, 1, true); + for (i = 0; i < SZ_64K / SZ_4K; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(copied) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + memset(folio_address(copied), 0xa7, SZ_64K); + for (i = 0; i < SZ_64K / SZ_4K; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + kunit_release_action(test, user4k_vma_mm_free, child); + } + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + if (held) { + KUNIT_EXPECT_EQ(test, folio_mapcount(held), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(held), 1); + } + for (i = 0; i < nr; i++) { + struct swap_info_struct *si; + + part = entry; + part.val += i; + si = get_swap_device(part); + if (si) { + KUNIT_EXPECT_FALSE(test, swap_entry_swapped(si, part)); + put_swap_device(si); + } else { + KUNIT_EXPECT_EQ(test, mode, 4U); + } + } + if (mode == 4) + kunit_release_action(test, test_reenable_swap, disabled); + kunit_info(test, "completed coarse anonymous swap mode %u", mode); + } +} + +/* The caller's private fixture has no concurrent PTE writers during reclaim. */ +static void test_coarse_anon_reclaim_once(struct vm_area_struct *vma) +{ + struct folio *folio = test_fault_folio(vma->vm_mm, vma->vm_start); + LIST_HEAD(folios); + + if (!folio) + return; + lru_add_drain_all(); + if (folio_isolate_lru(folio)) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + schedule_timeout_uninterruptible(1); +} + +static void user64k_anon_swap_limit_test(struct kunit *test) +{ +#ifdef CONFIG_MEMCG + struct vm_area_struct *vma; + struct cgroup *cgroup; + struct cgroup_subsys_state *css; + struct mem_cgroup *memcg; + struct folio *folio; + swp_entry_t entry, part; + phys_addr_t original; + unsigned int i, tries, nr = SZ_64K / PAGE_SIZE; + long before, denied; + char operation[80]; + vm_fault_t fault; + pte_t pte; + int ret, leave; + + if (!test_coarse_swap_enabled(test)) + return; + if (mem_cgroup_disabled()) { + kunit_skip(test, "requires memory cgroups"); + return; + } + snprintf(operation, sizeof(operation), "--coarse-swap-enter=%d", task_pid_nr(current)); + KUNIT_ASSERT_EQ(test, test_swap_control(operation), 0); + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + fault = vma ? test_file_fault(vma, TEST_VA, true) : VM_FAULT_OOM; + snprintf(operation, sizeof(operation), "--coarse-swap-leave=%d", task_pid_nr(current)); + leave = test_swap_control(operation); + KUNIT_ASSERT_EQ(test, leave, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, fault & VM_FAULT_ERROR, 0U); + cgroup = cgroup_get_from_path("/coarse-swap-limit"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cgroup); + css = cgroup_get_e_css(cgroup, &memory_cgrp_subsys); + cgroup_put(cgroup); + KUNIT_ASSERT_NOT_NULL(test, css); + memcg = mem_cgroup_from_css(css); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_memcg_put, memcg), 0); + KUNIT_ASSERT_PTR_NE(test, memcg, root_mem_cgroup); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + rcu_read_lock(); + KUNIT_EXPECT_PTR_EQ(test, folio_memcg(folio), memcg); + rcu_read_unlock(); + original = page_to_phys(&folio->page); + memset(folio_address(folio), 0xc9, SZ_64K); + before = atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]); + folio_lock(folio); + ret = folio_alloc_swap(folio); + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(folio)); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, -ENOMEM); + denied = atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]); + KUNIT_EXPECT_GT(test, denied, before); + KUNIT_EXPECT_EQ(test, page_counter_read(&memcg->swap), 0UL); + /* Exercise vmscan's failed-allocation/split fallback without a retained GET. */ + for (tries = 0; tries < 8; tries++) + test_coarse_anon_reclaim_once(vma); + KUNIT_EXPECT_GT(test, atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]), denied); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), original); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xc9, SZ_64K), NULL); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + folio_put(folio); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 0L); + KUNIT_EXPECT_EQ(test, page_counter_read(&memcg->swap), 0UL); + KUNIT_ASSERT_EQ(test, test_swap_control("--coarse-swap-unlimit"), 0); + for (tries = 0; tries < 100; tries++) { + test_coarse_anon_reclaim_once(vma); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + if (!pte_present(pte)) + break; + } + KUNIT_ASSERT_FALSE(test, pte_present(pte)); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_EXPECT_EQ(test, page_counter_read(&memcg->swap), (unsigned long)nr); + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 1); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(part)); + } + fault = test_file_fault(vma, TEST_VA, false); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), 0U); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_MAJOR); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xc9, SZ_64K), NULL); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + KUNIT_EXPECT_EQ(test, page_counter_read(&memcg->swap), 0UL); +#else + kunit_skip(test, "requires memory cgroups"); +#endif +} + +struct coarse_swap_worker { + struct mm_struct *mm; + struct task_struct *task; + struct completion ready, go, entered, done; + u8 *buffer; + unsigned int id; + bool *disabled; + int error; +}; + +static int coarse_swap_worker_run(void *arg) +{ + struct coarse_swap_worker *w = arg; + unsigned int i; + u8 value = 0x91 + w->id; + + complete(&w->ready); + if (!wait_for_completion_timeout(&w->go, 15 * HZ) || kthread_should_stop()) { + w->error = -ETIMEDOUT; + goto out; + } + if (!w->mm) { + complete(&w->entered); + w->error = test_swap_control("--swapoff-helper"); + if (!w->error) + WRITE_ONCE(*w->disabled, true); + goto out; + } + kthread_use_mm(w->mm); + complete(&w->entered); + if (copy_from_user(w->buffer, (void __user *)TEST_VA, SZ_64K)) { + w->error = 1; + goto unuse; + } + for (i = 0; i < SZ_64K; i++) { + if (w->buffer[i] != (u8)(0x31 + i / SZ_4K)) { + w->error = 2; + goto unuse; + } + } + if (copy_to_user((void __user *)(TEST_VA + SZ_64K - 1), &value, 1) || + copy_from_user(w->buffer, (void __user *)TEST_VA, SZ_64K)) { + w->error = 3; + goto unuse; + } + for (i = 0; i < SZ_64K; i++) { + u8 expected = i == SZ_64K - 1 ? value : 0x31 + i / SZ_4K; + + if (w->buffer[i] != expected) { + w->error = 4; + break; + } + } +unuse: + kthread_unuse_mm(w->mm); +out: + complete(&w->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void coarse_swap_worker_stop(void *arg) +{ + struct coarse_swap_worker *w = arg; + + if (w->task) { + complete_all(&w->go); + kthread_stop(w->task); + w->task = NULL; + } +} + +static void user64k_anon_swap_race_test(struct kunit *test) +{ + unsigned int round, nr = SZ_64K / PAGE_SIZE; + + if (!test_coarse_swap_enabled(test)) + return; + if (num_online_cpus() < 2 || !system_supports_64kb_granule()) { + kunit_skip(test, "requires two CPUs and 64K TTBR0 support"); + return; + } + for (round = 0; round < 8; round++) { + struct vm_area_struct *vma; + struct mm_struct *child; + struct coarse_swap_worker *workers; + struct folio *folio, *blocker; + struct swap_info_struct *si; + swp_entry_t entry, part; + bool *disabled, started = true, blocked; + bool off_started = false, cold = round & 1; + unsigned int i, j, tries; + pte_t pte; + int ret, cpu; + + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + for (i = 0; i < SZ_64K / SZ_4K; i++) + memset(folio_address(folio) + i * SZ_4K, 0x31 + i, SZ_4K); + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + if (!ret) { + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + } + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_FALSE(test, pte_present(pte)); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 2); + if (cold) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(part)); + } + part = entry; + part.val += nr - 1; + blocker = read_swap_cache_async(part, GFP_KERNEL, vma, + TEST_VA + SZ_64K - PAGE_SIZE, NULL); + KUNIT_ASSERT_NOT_NULL(test, blocker); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, blocker), 0); + KUNIT_EXPECT_EQ(test, folio_size(blocker), cold ? PAGE_SIZE : (unsigned long)SZ_64K); + workers = kunit_kcalloc(test, 3, sizeof(*workers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + cpu = cpumask_first(cpu_online_mask); + for (i = 0; i < 3; i++) { + workers[i].mm = i < 2 ? (i ? child : vma->vm_mm) : NULL; + workers[i].id = i; + workers[i].disabled = disabled; + workers[i].buffer = kunit_kmalloc(test, SZ_64K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers[i].buffer); + init_completion(&workers[i].ready); + init_completion(&workers[i].go); + init_completion(&workers[i].entered); + init_completion(&workers[i].done); + workers[i].task = kthread_create(coarse_swap_worker_run, &workers[i], + "user64k-swap-%u", i); + if (IS_ERR(workers[i].task)) { + workers[i].task = NULL; + KUNIT_FAIL(test, "coarse swap worker creation failed"); + return; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + coarse_swap_worker_stop, &workers[i]), 0); + if (i < 2) { + kthread_bind(workers[i].task, cpu); + cpu = cpumask_next(cpu, cpu_online_mask); + } + wake_up_process(workers[i].task); + } + for (i = 0; i < 3; i++) + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].ready, 10 * HZ), 0UL); + folio_lock(blocker); + for (i = 0; i < 3; i++) + complete(&workers[i].go); + for (i = 0; i < 3; i++) + started &= !!wait_for_completion_timeout(&workers[i].entered, HZ); + for (tries = 0; tries < 3 * HZ; tries++) { + si = get_swap_device(entry); + if (si) { + off_started = !(READ_ONCE(si->flags) & SWP_WRITEOK); + put_swap_device(si); + } + if (off_started) + break; + schedule_timeout_uninterruptible(1); + } + blocked = !completion_done(&workers[0].done) && + !completion_done(&workers[1].done) && !completion_done(&workers[2].done); + /* Never execute a fatal assertion while retaining the blocking lock. */ + folio_unlock(blocker); + KUNIT_EXPECT_TRUE(test, started); + KUNIT_EXPECT_TRUE(test, off_started); + KUNIT_EXPECT_TRUE(test, blocked); + for (i = 0; i < 3; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].done, 20 * HZ), 0UL); + KUNIT_EXPECT_EQ_MSG(test, workers[i].error, 0, "round=%u worker=%u", round, i); + kunit_release_action(test, coarse_swap_worker_stop, &workers[i]); + } + KUNIT_EXPECT_TRUE(test, *disabled); + for (i = 0; i < 2; i++) { + struct mm_struct *mm = i ? child : vma->vm_mm; + + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + for (j = 0; j < SZ_64K; j++) { + u8 expected = j == SZ_64K - 1 ? 0x91 + i : 0x31 + j / SZ_4K; + + if (((u8 *)folio_address(folio))[j] != expected) { + KUNIT_FAIL(test, "round=%u side=%u byte=%u", round, i, j); + break; + } + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1L); + kunit_release_action(test, user4k_vma_mm_free, mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), folio == blocker ? 2 : 1); + kunit_release_action(test, test_folio_put, folio); + } + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(blocker)); + KUNIT_EXPECT_EQ(test, folio_ref_count(blocker), 1); + kunit_release_action(test, test_folio_put, blocker); + kunit_release_action(test, test_reenable_swap, disabled); + KUNIT_ASSERT_GT(test, total_swap_pages, 0UL); + } + kunit_info(test, "8 controlled warm/cold fault+COW+swapoff rounds completed"); +} + +struct coarse_swap_reservation { + struct folio **folios; + unsigned int count; +}; + +static void test_coarse_release_reservation(struct folio *folio) +{ + folio_lock(folio); + folio_put_swap(folio, NULL); + folio_free_swap(folio); + folio_unlock(folio); + folio_put(folio); +} + +static void test_coarse_release_reservations(void *arg) +{ + struct coarse_swap_reservation *r = arg; + unsigned int i; + + for (i = 0; i < r->count; i++) { + if (!r->folios[i]) + continue; + test_coarse_release_reservation(r->folios[i]); + r->folios[i] = NULL; + cond_resched(); + } +} + +static void test_coarse_swap_refusal(struct kunit *test, struct vm_area_struct *vma, + phys_addr_t original) +{ + struct folio *folio = test_fault_folio(vma->vm_mm, TEST_VA); + pte_t pte; + unsigned int i; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + ret = folio_alloc_swap(folio); + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(folio)); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, -ENOMEM); + for (i = 0; i < 4; i++) + test_coarse_anon_reclaim_once(vma); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), original); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xd6, SZ_64K), NULL); + folio_put(folio); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 0L); +} + +static void user64k_anon_swap_fragment_test(struct kunit *test) +{ + struct coarse_swap_reservation *r; + struct vm_area_struct *vma; + struct folio *folio; + unsigned int i, nr = SZ_64K / PAGE_SIZE, released = 0; + unsigned long hole, free_before; + phys_addr_t original; + swp_entry_t entry, part; + vm_fault_t fault; + pte_t pte; + int ret = 0; + + if (!test_coarse_swap_enabled(test)) + return; + /* Exhaust only the bounded disposable disk owned by this test's PID1. */ + KUNIT_ASSERT_EQ(test, test_swap_control("--coarse-fixture-check"), 0); + KUNIT_ASSERT_LT(test, total_swap_pages, (unsigned long)(SZ_256M / PAGE_SIZE)); + KUNIT_ASSERT_GT(test, total_swap_pages, 2UL * SWAPFILE_CLUSTER); + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + original = page_to_phys(&folio->page); + memset(folio_address(folio), 0xd6, SZ_64K); + folio_put(folio); + r = kunit_kzalloc(test, sizeof(*r), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, r); + r->folios = kunit_kcalloc(test, total_swap_pages + 1, sizeof(*r->folios), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, r->folios); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_coarse_release_reservations, r), 0); + /* Each cache-only folio has one explicit slot owner, so allocation cannot reclaim it. */ + for (i = 0; i <= total_swap_pages; i++) { + folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = mem_cgroup_charge(folio, vma->vm_mm, GFP_KERNEL); + if (ret) { + folio_put(folio); + KUNIT_FAIL(test, "reservation memory charge failed"); + return; + } + __folio_set_swapbacked(folio); + __folio_mark_uptodate(folio); + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) { + ret = folio_dup_swap(folio, NULL); + if (ret) + folio_free_swap(folio); + } + folio_unlock(folio); + if (ret) { + folio_put(folio); + break; + } + r->folios[r->count++] = folio; + cond_resched(); + } + KUNIT_ASSERT_EQ(test, ret, -ENOMEM); + KUNIT_ASSERT_EQ(test, atomic_long_read(&nr_swap_pages), 0L); + test_coarse_swap_refusal(test, vma, original); + /* Retain exactly one native entry in each naturally aligned coarse group. */ + for (i = 0; i < r->count; i++) { + folio = r->folios[i]; + if (swp_offset(folio->swap) % nr == 1) + continue; + test_coarse_release_reservation(folio); + r->folios[i] = NULL; + } + free_before = atomic_long_read(&nr_swap_pages); + KUNIT_EXPECT_GT(test, free_before, total_swap_pages / 2); + KUNIT_EXPECT_LT(test, free_before, total_swap_pages); + test_coarse_swap_refusal(test, vma, original); + /* Free one aligned group near the end, without freeing a whole native-order cluster. */ + hole = round_down(total_swap_pages + 1, (unsigned long)SWAPFILE_CLUSTER) - nr; + for (i = 0; i < r->count; i++) { + folio = r->folios[i]; + if (!folio || swp_offset(folio->swap) < hole || swp_offset(folio->swap) >= hole + nr) + continue; + test_coarse_release_reservation(folio); + r->folios[i] = NULL; + released++; + } + KUNIT_ASSERT_EQ(test, released, 1U); + KUNIT_EXPECT_EQ(test, atomic_long_read(&nr_swap_pages), (long)free_before + 1); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ_MSG(test, ret, 0, "a complete aligned group is free in an order-0 cluster"); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_FALSE(test, pte_present(pte)); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_EXPECT_EQ(test, swp_offset(entry), hole); + for (i = 0; i < r->count; i++) { + if (r->folios[i]) + KUNIT_EXPECT_EQ(test, swp_swapcount(r->folios[i]->swap), 1); + } + kunit_release_action(test, test_coarse_release_reservations, r); + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 1); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(part)); + } + fault = test_file_fault(vma, TEST_VA, false); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), 0U); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_MAJOR); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xd6, SZ_64K), NULL); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + kunit_info(test, "exhaustion, scattered free space and one-group recovery checked"); +} + +struct coarse_swap_io_mode { + struct swap_info_struct *si; + bool synchronous; +}; + +static void test_coarse_restore_io_mode(void *arg) +{ + struct coarse_swap_io_mode *mode = arg; + + spin_lock(&mode->si->lock); + if (mode->synchronous) + mode->si->flags |= SWP_SYNCHRONOUS_IO; + else + mode->si->flags &= ~SWP_SYNCHRONOUS_IO; + spin_unlock(&mode->si->lock); + put_swap_device(mode->si); +} + +static void user64k_anon_swap_extent_test(struct kunit *test) +{ + unsigned int mode, i; + + if (!test_coarse_swap_enabled(test)) + return; + KUNIT_ASSERT_EQ(test, test_swap_control("--coarse-fixture-check"), 0); + for (mode = 0; mode < 2; mode++) { + struct vm_area_struct *vma; + struct coarse_swap_io_mode *saved; + struct folio *folio; + unsigned long nr = SZ_64K / PAGE_SIZE, run = nr; + swp_entry_t entry; + sector_t first, next; + bool file; + int ret; + + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + for (i = 0; i < SZ_64K / SZ_4K; i++) + memset(folio_address(folio) + i * SZ_4K, 0x63 + i, SZ_4K); + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, folio_mapcount(folio), 0); + entry = folio->swap; + saved = kunit_kzalloc(test, sizeof(*saved), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, saved); + saved->si = get_swap_device(entry); + KUNIT_ASSERT_NOT_NULL(test, saved->si); + saved->synchronous = saved->si->flags & SWP_SYNCHRONOUS_IO; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_coarse_restore_io_mode, saved), 0); + file = !(saved->si->flags & SWP_BLKDEV); + KUNIT_ASSERT_FALSE(test, saved->si->flags & SWP_FS_OPS); + first = swap_extent_sector(entry, &run); + if (file) { + unsigned long remaining; + swp_entry_t part = entry; + + KUNIT_ASSERT_LT_MSG(test, run, nr, "fragmented fixture must cross an extent"); + part.val += run; + remaining = nr - run; + next = swap_extent_sector(part, &remaining); + KUNIT_EXPECT_NE(test, next, first + (run << (PAGE_SHIFT - 9))); + } else { + KUNIT_EXPECT_EQ(test, run, nr); + } + /* The verified disposable fixture owns this otherwise idle swap area. */ + spin_lock(&saved->si->lock); + if (mode) + saved->si->flags |= SWP_SYNCHRONOUS_IO; + else + saved->si->flags &= ~SWP_SYNCHRONOUS_IO; + spin_unlock(&saved->si->lock); + folio_lock(folio); + folio_clear_dirty_for_io(folio); + __swap_writepage(folio, NULL); + folio_wait_writeback(folio); + folio_lock(folio); + KUNIT_EXPECT_FALSE(test, folio_test_dirty(folio)); + /* A full-folio read must reconstruct all physical runs, not warm cache. */ + memset(folio_address(folio), 0, SZ_64K); + folio_clear_uptodate(folio); + swap_read_folio(folio, NULL); + folio_lock(folio); + KUNIT_EXPECT_TRUE(test, folio_test_uptodate(folio)); + for (i = 0; i < SZ_64K / SZ_4K; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x63 + i, SZ_4K), NULL); + folio_unlock(folio); + kunit_release_action(test, test_coarse_restore_io_mode, saved); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + kunit_info(test, "64K full-folio %s I/O: file=%u first-run=%lu/%lu native pages", + mode ? "synchronous" : "asynchronous", file, run, nr); + } +} + +static struct kunit_case user4k_coarse_shmem_cases[] = { + KUNIT_CASE(user64k_anon_swap_extent_test), + KUNIT_CASE(user64k_anon_swap_test), + KUNIT_CASE(user64k_anon_swap_limit_test), + KUNIT_CASE(user64k_anon_swap_fragment_test), + KUNIT_CASE(user64k_anon_swap_race_test), + KUNIT_CASE(user64k_shmem_cold_constituents_test), + KUNIT_CASE(user64k_shmem_cold_promoted_test), + KUNIT_CASE(user64k_shmem_fault_alloc_fail_test), + {} +}; + +static struct kunit_suite user4k_coarse_shmem_suite = { + .name = "arm64-user4k-coarse-shmem", + .test_cases = user4k_coarse_shmem_cases, + .suite_exit = user4k_swap_suite_exit, +}; + +static struct kunit_case user4k_shmem_swap_cases[] = { + KUNIT_CASE(user4k_shmem_uffd_swap_alloc_fail_test), + KUNIT_CASE(user4k_shmem_uffd_swap_charge_test), + KUNIT_CASE(user4k_shmem_uffd_swapin_test), + KUNIT_CASE(user4k_shmem_uffd_swapped_truncate_test), + KUNIT_CASE(user4k_shmem_uffd_swapoff_test), + KUNIT_CASE(user4k_shmem_uffd_swapped_partial_hole_test), + {} +}; + +static struct kunit_suite user4k_shmem_swap_suite = { + .name = "arm64-user4k-shmem-swap", + .test_cases = user4k_shmem_swap_cases, + .suite_exit = user4k_swap_suite_exit, +}; +#endif + +static struct kunit_case user4k_nonlinear_swap_cases[] = { +#ifdef CONFIG_USERFAULTFD + KUNIT_CASE(user4k_move_swap_get_test), + KUNIT_CASE(user4k_move_migration_race_test), +#endif + KUNIT_CASE(user4k_nonlinear_swap_cached_test), + KUNIT_CASE(user4k_nonlinear_swap_disk_test), + {} +}; + +static struct kunit_suite user4k_nonlinear_swap_suite = { + .name = "arm64-user4k-nonlinear-swap", + .test_cases = user4k_nonlinear_swap_cases, + .suite_exit = user4k_swap_suite_exit, +}; + +static struct kunit_suite user4k_swap_suite = { + .name = "arm64-user4k-swap", + .test_cases = user4k_swap_cases, + .suite_exit = user4k_swap_suite_exit, +}; + +static void user4k_thp_eligibility_test(struct kunit *test) +{ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + struct mm_struct *small = test_vma_mm(test), *native = mm_alloc(); + const enum tva_type types[] = { + TVA_PAGEFAULT, TVA_SMAPS, TVA_KHUGEPAGED, TVA_FORCED_COLLAPSE, + }; + struct vm_area_struct *vma, *normal; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | + VM_HUGEPAGE; + + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + user4k_vma_mm_free, native), 0); + KUNIT_ASSERT_NOT_NULL(test, small); + vma = test_vma_add(small, SZ_1G, SZ_1G, (struct vm_page_offset) {}, flags); + normal = test_vma_add(native, SZ_1G, SZ_1G, (struct vm_page_offset) {}, flags); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, normal); + /* Neither user advice nor forced collapse can bypass geometry limits. */ + KUNIT_EXPECT_TRUE(test, vma_thp_disabled(vma, flags, false)); + KUNIT_EXPECT_TRUE(test, vma_thp_disabled(vma, flags, true)); + KUNIT_EXPECT_FALSE(test, vma_thp_disabled(normal, flags, false)); + KUNIT_EXPECT_FALSE(test, vma_thp_disabled(normal, flags, true)); + for (unsigned int i = 0; i < ARRAY_SIZE(types); i++) + KUNIT_EXPECT_EQ(test, __thp_vma_allowable_orders(vma, flags, + types[i], THP_ORDERS_ALL_ANON), 0UL); +#else + kunit_skip(test, "requires transparent hugepage support"); +#endif +} + +static void user4k_aslr_window_test(struct kunit *test) +{ + struct mm_struct *small = test_vma_mm(test), *native = mm_alloc(); + const unsigned long windows[] = { SZ_8M, SZ_32M, SZ_1G }; + + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + user4k_vma_mm_free, native), 0); + KUNIT_ASSERT_NOT_NULL(test, small); + for (unsigned int i = 0; i < ARRAY_SIZE(windows); i++) { + unsigned long mask = (windows[i] >> PAGE_SHIFT) - 1; + unsigned long converted = mm_aslr_mask(small, mask); + + KUNIT_EXPECT_EQ(test, mm_aslr_mask(native, mask), mask); + KUNIT_EXPECT_EQ(test, (converted + 1) << mm_page_shift(small), + windows[i]); + KUNIT_EXPECT_EQ(test, converted & (TEST_SLOTS - 1), + (unsigned long)TEST_SLOTS - 1); + } +} + +static void user4k_exec_granule_policy_test(struct kunit *test) +{ + /* Model all native granules, including AArch32 with NO 4K support. */ + const unsigned int native[] = { 12, 14, 16 }; + + for (unsigned int i = 0; i < ARRAY_SIZE(native); i++) { + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(true, false, native[i], 0), + native[i]); + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(true, false, native[i], 12), + native[i]); + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(true, true, native[i], 0), + 12U); + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(false, true, native[i], 0), + native[i]); + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(false, true, native[i], 12), + 12U); + } +} + +static void user_granule_zero_region_test(struct kunit *test) +{ + unsigned long base_pfn = zero_pfn(0); + unsigned int shift, i; + u8 *buffer; + + KUNIT_EXPECT_TRUE(test, IS_ALIGNED(PFN_PHYS(base_pfn), SZ_64K)); + KUNIT_EXPECT_FALSE(test, is_zero_pfn(base_pfn - 1)); + KUNIT_EXPECT_FALSE(test, is_zero_pfn(base_pfn + SZ_64K / PAGE_SIZE)); + for (i = 0; i < SZ_64K / PAGE_SIZE; i++) { + KUNIT_EXPECT_TRUE(test, is_zero_pfn(base_pfn + i)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(pfn_to_page(base_pfn + i)), + 0, PAGE_SIZE), NULL); + } + buffer = kunit_kmalloc(test, SZ_64K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, buffer); + for (shift = 12; shift <= 16; shift += 2) { + struct vm_area_struct *vma; + struct mm_struct *mm; + unsigned long size = 1UL << shift, left; + vm_fault_t ret; + spinlock_t *ptl; + pte_t *pte, entry; + + if ((shift == 12 && !system_supports_4kb_granule()) || + (shift == 14 && !system_supports_16kb_granule()) || + (shift == 16 && !system_supports_64kb_granule())) { + kunit_info(test, "hardware zero mapping skipped for shift %u", shift); + continue; + } + /* Internal coarse mm: the public opt-in remains disabled. */ + mm = test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, mm); + vma = test_fault_vma_in_mm(mm, TEST_VA, size); + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(mm); + ret = handle_mm_fault(vma, TEST_VA + size - 1, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + entry = ptep_get(pte); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_special(entry)); + KUNIT_EXPECT_FALSE(test, pte_write(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, entry), PFN_PHYS(base_pfn)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); + + memset(buffer, 0xa5, size); + kthread_use_mm(mm); + left = copy_from_user(buffer, (void __user *)TEST_VA, size); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, left, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0, size), NULL); + mmap_read_lock(mm); + zap_vma_range(vma, TEST_VA, size); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); + } +} + +#ifdef CONFIG_COREDUMP +static void user_granule_core_offsets_test(struct kunit *test) +{ + struct mm_struct *mm = kunit_kzalloc(test, sizeof(*mm), GFP_KERNEL); + struct vm_area_struct *vma = kunit_kzalloc(test, sizeof(*vma), GFP_KERNEL); + unsigned int shift; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + vma->vm_mm = mm; + for (shift = 12; shift <= 16; shift += 2) { + unsigned long bytes = (1UL << 55) + 3 * SZ_64K + (1UL << shift); + + mm->context.flags = shift == 12 ? MMCF_USER_4K : + shift == 14 ? MMCF_USER_16K : MMCF_USER_64K; + vma_set_page_offset(vma, (struct vm_page_offset) { + .index = bytes >> PAGE_SHIFT, .offset = bytes & ~PAGE_MASK, + }); + KUNIT_EXPECT_EQ(test, core_vma_pgoff(vma), bytes >> shift); + KUNIT_EXPECT_EQ(test, mm_user_fragment_size(mm), min(1UL << shift, PAGE_SIZE)); + vma_set_page_offset(vma, (struct vm_page_offset) {}); + KUNIT_EXPECT_EQ(test, core_vma_pgoff(vma), 0UL); + } +} +#endif + +static void user64k_fragment_ownership_test(struct kunit *test) +{ + struct user_page_fragment fragments[SZ_64K / PAGE_SIZE] = {}; + struct mm_struct *parent, *child; + struct vm_area_struct *vma, *cvma; + struct folio *original, *copied; + unsigned int i, nr = ARRAY_SIZE(fragments); + long count; + int ret, refs; + pte_t entry; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires a user leaf larger than native pages"); + return; + } + parent = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, parent); + vma = test_fault_vma_in_mm(parent, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + count = get_user_fragments_remote(parent, TEST_VA + 17, SZ_64K - 34, + 0, fragments, nr); + if (count > 0) { + for (i = 0; i < count; i++) { + struct user_page_fragment *f = &fragments[i]; + unsigned int bytes = PAGE_SIZE - (i == 0 || i == nr - 1 ? 17 : 0); + + KUNIT_EXPECT_TRUE(test, is_zero_folio(f->folio)); + KUNIT_EXPECT_EQ(test, folio_pfn(f->folio), zero_pfn(TEST_VA) + i); + KUNIT_EXPECT_PTR_EQ(test, f->subpage, NULL); + KUNIT_EXPECT_EQ(test, f->offset, i == 0 ? 17UL : 0UL); + KUNIT_EXPECT_EQ(test, f->length, bytes); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0, f->length), NULL); + } + release_user_fragments(fragments, count, false); + } + KUNIT_ASSERT_EQ(test, count, (long)nr); + + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, parent, TEST_VA); + original = page_folio(pfn_to_page(pte_pfn(entry))); + for (i = 0; i < nr; i++) + memset(page_address(folio_page(original, i)), 0x51 + i, PAGE_SIZE); + lru_add_drain_all(); + refs = folio_ref_count(original); + count = get_user_fragments_remote(parent, TEST_VA + 17, SZ_64K - 34, + 0, fragments, nr); + if (count > 0) { + KUNIT_EXPECT_EQ(test, folio_ref_count(original), refs + (int)count); + for (i = 0; i < count; i++) { + struct user_page_fragment *f = &fragments[i]; + + KUNIT_EXPECT_PTR_EQ(test, f->folio, original); + KUNIT_EXPECT_EQ(test, f->offset, i * PAGE_SIZE + (i == 0 ? 17UL : 0UL)); + KUNIT_EXPECT_LE(test, offset_in_page(f->offset) + f->length, PAGE_SIZE); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x51 + i, f->length), NULL); + } + release_user_fragments(fragments, count, false); + } + KUNIT_ASSERT_EQ(test, count, (long)nr); + KUNIT_EXPECT_EQ(test, folio_ref_count(original), refs); + + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + /* A read pin after fork must unshare the complete coarse leaf. */ + count = pin_user_fragments_remote(child, TEST_VA, SZ_64K, 0, fragments, nr); + if (count > 0) { + copied = fragments[0].folio; + KUNIT_EXPECT_PTR_NE(test, copied, original); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(copied)); + KUNIT_EXPECT_FALSE(test, pte_write(test_fault_entry(test, child, TEST_VA))); + for (i = 0; i < count; i++) { + KUNIT_EXPECT_PTR_EQ(test, fragments[i].folio, copied); + KUNIT_EXPECT_PTR_EQ(test, fragments[i].subpage, NULL); + KUNIT_EXPECT_TRUE(test, fragments[i].pinned); + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(copied, i))); + KUNIT_EXPECT_EQ(test, fragments[i].offset, i * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, fragments[i].length, (unsigned int)PAGE_SIZE); + } + mmap_read_lock(child); + zap_vma_range(cvma, TEST_VA, SZ_64K); + mmap_read_unlock(child); + KUNIT_EXPECT_EQ(test, folio_mapcount(copied), 0); + for (i = 0; i < count; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(copied) + i * PAGE_SIZE, + 0x51 + i, PAGE_SIZE), NULL); + release_user_fragments(fragments, count, false); + } + KUNIT_ASSERT_EQ(test, count, (long)nr); + for (i = 0; i < nr; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(original) + i * PAGE_SIZE, + 0x51 + i, PAGE_SIZE), NULL); +} + +static void user64k_anon_fault_cow_test(struct kunit *test) +{ + struct mm_struct *parent, *child; + struct vm_area_struct *vma, *cvma; + pte_t orig[2], copied; + struct folio *folio; + unsigned int i, j, nr = SZ_64K / PAGE_SIZE; + int ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires coarse anonymous faults"); + return; + } + parent = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, parent); + vma = test_fault_vma_in_mm(parent, TEST_VA, 2 * SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + /* Leaf0 exercises zero-page COW; leaf1 exercises a direct write fault. */ + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + SZ_64K - 1, false) & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_TRUE(test, pte_special(test_fault_entry(test, parent, TEST_VA))); + for (i = 0; i < 2; i++) { + unsigned long addr = TEST_VA + i * SZ_64K; + + KUNIT_ASSERT_EQ(test, test_file_fault(vma, addr + SZ_64K - 1, true) & VM_FAULT_ERROR, 0U); + orig[i] = test_fault_entry(test, parent, addr); + KUNIT_ASSERT_TRUE(test, pte_present(orig[i])); + KUNIT_EXPECT_TRUE(test, pte_write(orig[i])); + KUNIT_EXPECT_FALSE(test, pte_special(orig[i])); + folio = page_folio(pfn_to_page(pte_pfn(orig[i]))); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0, SZ_64K), NULL); + for (j = 0; j < nr; j++) { + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(folio, j))); + memset(page_address(folio_page(folio, j)), 0x31 + i * nr + j, PAGE_SIZE); + } + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent, MM_ANONPAGES), 2L); + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 2L); + for (i = 0; i < 2; i++) { + unsigned long addr = TEST_VA + i * SZ_64K; + struct folio *old = page_folio(pfn_to_page(pte_pfn(orig[i]))); + + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, test_fault_entry(test, child, addr)), + pte_phys_mm(parent, orig[i])); + KUNIT_EXPECT_FALSE(test, pte_write(test_fault_entry(test, parent, addr))); + KUNIT_EXPECT_EQ(test, folio_mapcount(old), (int)(2 * nr)); + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, addr + SZ_64K - 1, true) & VM_FAULT_ERROR, 0U); + copied = test_fault_entry(test, child, addr); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, copied), pte_phys_mm(parent, orig[i])); + folio = page_folio(pfn_to_page(pte_pfn(copied))); + KUNIT_EXPECT_EQ(test, folio_mapcount(old), (int)nr); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + for (j = 0; j < nr; j++) { + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(folio, j)), + 0x31 + i * nr + j, PAGE_SIZE), NULL); + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(folio, j))); + } + memset(folio_address(folio), 0xa5, SZ_64K); + for (j = 0; j < nr; j++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(old, j)), + 0x31 + i * nr + j, PAGE_SIZE), NULL); + /* Parent reuse must restore exclusivity across the entire leaf. */ + lru_add_drain_all(); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, addr + SZ_64K - 1, true) & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, pte_phys_mm(parent, test_fault_entry(test, parent, addr)), + pte_phys_mm(parent, orig[i])); + for (j = 0; j < nr; j++) + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(old, j))); + } + if (system_supports_64kb_granule()) { + long protect[2]; + bool writable = false; + + kthread_use_mm(parent); + protect[0] = test_mprotect(TEST_VA, 2 * SZ_64K, PROT_READ); + kthread_unuse_mm(parent); + for (i = 0; i < 2; i++) + writable |= pte_write(test_fault_entry(test, parent, TEST_VA + i * SZ_64K)); + KUNIT_EXPECT_EQ(test, protect[0], 0L); + KUNIT_EXPECT_FALSE(test, writable); + kthread_use_mm(parent); + protect[1] = test_mprotect(TEST_VA, 2 * SZ_64K, PROT_READ | PROT_WRITE); + kthread_unuse_mm(parent); + KUNIT_EXPECT_EQ(test, protect[1], 0L); + for (i = 0; i < 2; i++) + KUNIT_EXPECT_TRUE(test, pte_write(test_fault_entry(test, parent, TEST_VA + i * SZ_64K))); + vma = test_vma_lookup(parent, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + } + mmap_read_lock(child); + zap_vma_range(cvma, TEST_VA, 2 * SZ_64K); + mmap_read_unlock(child); + mmap_read_lock(parent); + zap_vma_range(vma, TEST_VA, 2 * SZ_64K); + mmap_read_unlock(parent); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent, MM_ANONPAGES), 0L); +} + +struct coarse_test_pin { + struct page *page; +}; + +static void coarse_test_unpin(void *arg) +{ + struct coarse_test_pin *pin = arg; + + if (pin->page) { + unpin_user_page(pin->page); + pin->page = NULL; + } +} + +static void user64k_pinned_fork_test(struct kunit *test) +{ + struct mm_struct *parent, *child; + struct vm_area_struct *vma; + struct coarse_test_pin *pin; + struct folio *old, *new; + pte_t original, copied; + unsigned int i, nr = SZ_64K / PAGE_SIZE; + int ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires pinned coarse leaf"); + return; + } + parent = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, parent); + vma = test_fault_vma_in_mm(parent, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + SZ_64K - 1, true) & VM_FAULT_ERROR, 0U); + original = test_fault_entry(test, parent, TEST_VA); + old = page_folio(pfn_to_page(pte_pfn(original))); + for (i = 0; i < nr; i++) + memset(page_address(folio_page(old, i)), 0x70 + i, PAGE_SIZE); + pin = kunit_kzalloc(test, sizeof(*pin), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pin); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, coarse_test_unpin, pin), 0); + mm_flags_set(MMF_HAS_PINNED, parent); + ret = try_grab_folio(old, 1, FOLL_PIN); + if (!ret) + pin->page = folio_page(old, nr - 1); + KUNIT_ASSERT_EQ(test, ret, 0); + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + copied = test_fault_entry(test, child, TEST_VA); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, copied), pte_phys_mm(parent, original)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(parent, test_fault_entry(test, parent, TEST_VA)), + pte_phys_mm(parent, original)); + new = page_folio(pfn_to_page(pte_pfn(copied))); + KUNIT_EXPECT_EQ(test, folio_mapcount(old), (int)nr); + KUNIT_EXPECT_EQ(test, folio_mapcount(new), (int)nr); + for (i = 0; i < nr; i++) { + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(new, i)), + 0x70 + i, PAGE_SIZE), NULL); + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(new, i))); + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(old, i))); + } + memset(page_address(pin->page), 0xe3, PAGE_SIZE); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(new, nr - 1)), + 0x70 + nr - 1, PAGE_SIZE), NULL); + coarse_test_unpin(pin); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent, MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 1L); +} + +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +static void user4k_shmem_partial_hole_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 3 * PAGE_SIZE, true); + struct folio *folio; + loff_t start = PAGE_SIZE + SZ_4K + 17, end = PAGE_SIZE + 3 * SZ_4K + 16; + unsigned int i; + unsigned char byte = 0xa5; + loff_t write_pos = PAGE_SIZE + 2 * SZ_4K + 23; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shmem_truncate_range(file_inode(file), start, end); + folio = filemap_lock_folio(file->f_mapping, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 1), BIT(2)); + for (i = 0; i < PAGE_SIZE; i++) { + unsigned char expected = (i + PAGE_SIZE >= start && i + PAGE_SIZE <= end) ? + 0 : 0x31 + (PAGE_SIZE + i) / SZ_4K; + + if (((unsigned char *)folio_address(folio))[i] != expected) { + KUNIT_FAIL(test, "partial hole changed byte %u outside its range", i); + break; + } + } + /* Resident hole metadata has no dynamically allocated inode record. */ + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + folio_unlock(folio); + folio_put(folio); + shmem_truncate_range(file_inode(file), PAGE_SIZE + SZ_4K, PAGE_SIZE + 2 * SZ_4K - 1); + KUNIT_ASSERT_EQ(test, kernel_write(file, &byte, 1, &write_pos), 1L); + folio = filemap_lock_folio(file->f_mapping, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 1), BIT(1)); + KUNIT_EXPECT_EQ(test, ((unsigned char *)folio_address(folio))[2 * SZ_4K + 23], byte); + folio_unlock(folio); + folio_put(folio); +} + +/* A surviving writable pin exposes bytes even after its PTE is revoked. */ +static void user4k_shmem_pinned_hole_test(struct kunit *test) +{ + unsigned int native; + + for (native = 0; native < 2; native++) { + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *vma; + struct fragment_test_pins *pins; + struct folio *folio; + unsigned long offset = native ? 0 : SZ_4K; + unsigned long length = native ? PAGE_SIZE : SZ_4K; + unsigned long pfn; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, native, true, TEST_VA, length, offset); + KUNIT_ASSERT_NOT_NULL(test, vma); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + KUNIT_ASSERT_EQ(test, fragment_test_pin(pins, vma->vm_mm, TEST_VA, + length, FOLL_WRITE | FOLL_LONGTERM, 1), 1L); + folio = pins->fragments[0].folio; + pfn = folio_pfn(folio); + KUNIT_ASSERT_TRUE(test, folio_maybe_dma_pinned(folio)); + shmem_truncate_range(file_inode(file), SZ_4K, 2 * SZ_4K - 1); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, vma->vm_mm, TEST_VA))); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, SZ_4K), NULL); + /* Model a completed device write through the original pin. */ + memset(folio_address(folio) + SZ_4K, 0xb7, SZ_4K); + folio_lock(folio); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(folio, 0, SZ_4K, SZ_4K)); + folio_unlock(folio); + fragment_test_unpin(pins); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_pfn(folio), pfn); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(folio, 0, SZ_4K, SZ_4K)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0xb7, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0x31, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + 2 * SZ_4K, 0x33, SZ_4K), NULL); + folio_unlock(folio); + folio_put(folio); + /* With the external writer gone, another hole can be missing. */ + shmem_truncate_range(file_inode(file), SZ_4K, 2 * SZ_4K - 1); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, shmem_uffd_range_missing(folio, 0, SZ_4K, SZ_4K)); + folio_unlock(folio); + folio_put(folio); + } +} + +static void test_unpin_folio(void *folio) +{ + unpin_folio(folio); +} + +/* The direct memfd API must publish cache exposure without a user PTE. */ +static void user4k_shmem_memfd_pin_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct folio *pin, *folio; + pgoff_t offset; + long ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shmem_truncate_range(file_inode(file), SZ_4K, 2 * SZ_4K - 1); + ret = memfd_pin_folios(file, SZ_4K, 2 * SZ_4K - 1, &pin, 1, &offset); + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_unpin_folio, pin), 0); + KUNIT_EXPECT_EQ(test, offset, (pgoff_t)SZ_4K); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(pin)); + memset(folio_address(pin) + offset, 0xc8, SZ_4K); + folio_lock(pin); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(pin, 0, SZ_4K, SZ_4K)); + folio_unlock(pin); + kunit_release_action(test, test_unpin_folio, pin); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(folio, 0, SZ_4K, SZ_4K)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0xc8, SZ_4K), NULL); + folio_unlock(folio); + folio_put(folio); +} + +static void user4k_shmem_replace_validity_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct folio *old, *new; + unsigned long old_pfn; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shmem_truncate_range(file_inode(file), SZ_4K, 2 * SZ_4K - 1); + new = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, new); + folio_lock(new); + old = filemap_lock_folio(file->f_mapping, 0); + if (IS_ERR(old)) { + folio_unlock(new); + folio_put(new); + KUNIT_FAIL(test, "could not lock source cache folio"); + return; + } + old_pfn = folio_pfn(old); + memcpy(folio_address(new), folio_address(old), PAGE_SIZE); + __folio_set_swapbacked(new); + __folio_mark_uptodate(new); + folio_mark_dirty(new); + replace_page_cache_folio(old, new); + folio_add_lru(new); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(new, 0), BIT(1)); + KUNIT_EXPECT_NE(test, folio_pfn(new), old_pfn); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(new), 0x31, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(new) + SZ_4K, 0, SZ_4K), NULL); + folio_unlock(old); + folio_put(old); + folio_unlock(new); + folio_put(new); +} + +static void user4k_shmem_uffd_alias_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, false); + const unsigned long all = GENMASK(PAGE_SIZE / SZ_4K - 1, 0); + struct vm_area_struct *small, *native; + struct folio *folio; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = shmem_get_folio(file_inode(file), 0, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = shmem_uffd_set_missing(folio, 0, all); + KUNIT_EXPECT_TRUE(test, shmem_uffd_has_missing_native_page(folio)); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + small = test_file_vma(test, file, false, true, TEST_VA, SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA, false) & VM_FAULT_ERROR, 0U); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), all & ~BIT(1)); + KUNIT_EXPECT_FALSE(test, shmem_uffd_has_missing_native_page(folio)); + folio_unlock(folio); + folio_put(folio); + + native = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, false) & VM_FAULT_ERROR, 0U); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), 0UL); + folio_unlock(folio); + folio_put(folio); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); +} + +static void user4k_shmem_uffd_state_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 3 * PAGE_SIZE, false); + const unsigned long all = GENMASK(PAGE_SIZE / SZ_4K - 1, 0); + struct folio *folio; + unsigned long mask; + unsigned int i; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + for (i = 0; i < 3; i++) { + ret = shmem_get_folio(file_inode(file), i, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, i), 0UL); + ret = shmem_uffd_set_missing(folio, i, all & ~BIT(0)); + KUNIT_EXPECT_EQ(test, ret, 0); + shmem_uffd_mark_present(folio, i, BIT(1)); + mask = shmem_uffd_missing_mask(folio, i); + KUNIT_EXPECT_EQ(test, mask, all & ~GENMASK(1, 0)); + ret = shmem_uffd_set_missing(folio, i, BIT(PAGE_SIZE / SZ_4K)); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, i), mask); + folio_unlock(folio); + folio_put(folio); + } + /* Real truncate removes one index without discarding neighbouring state. */ + shmem_truncate_range(file_inode(file), PAGE_SIZE, 2 * PAGE_SIZE - 1); + KUNIT_EXPECT_PTR_EQ(test, xa_load(&SHMEM_I(file_inode(file))->uffd_missing, 1), NULL); + for (i = 0; i < 3; i += 2) { + folio = filemap_lock_folio(file->f_mapping, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, i), all & ~GENMASK(1, 0)); + folio_unlock(folio); + folio_put(folio); + } + /* Generic batch deletion must not strand inode metadata either. */ + truncate_inode_pages(file->f_mapping, 0); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + ret = shmem_get_folio(file_inode(file), 1, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 1), 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0, PAGE_SIZE), NULL); + folio_unlock(folio); + folio_put(folio); +} + +#ifdef CONFIG_MIGRATION +static void user4k_shmem_uffd_migration_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, false); + const unsigned long missing = GENMASK(PAGE_SIZE / SZ_4K - 1, 1); + struct folio *folio; + unsigned long old_pfn; + unsigned int succeeded = 0; + bool isolated; + int ret; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = shmem_get_folio(file_inode(file), 0, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + memset(folio_address(folio), 0xa5, SZ_4K); + folio_mark_dirty(folio); + ret = shmem_uffd_set_missing(folio, 0, missing); + old_pfn = folio_pfn(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + isolated = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, MIGRATE_SYNC, + MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, succeeded, 1U); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_NE(test, folio_pfn(folio), old_pfn); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), missing); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xa5, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, + PAGE_SIZE - SZ_4K), NULL); + shmem_uffd_mark_present(folio, 0, missing); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), 0UL); + folio_unlock(folio); + folio_put(folio); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); +} +#endif +#endif + +static struct kunit_case user4k_test_cases[] = { + KUNIT_CASE(user4k_exec_granule_policy_test), + KUNIT_CASE(user4k_aslr_window_test), + KUNIT_CASE(user4k_thp_eligibility_test), + KUNIT_CASE(user4k_geometry_test), + KUNIT_CASE(user4k_leaf_encoding_test), + KUNIT_CASE(user16k_leaf_encoding_test), + KUNIT_CASE(user4k_rmap_walk_test), + KUNIT_CASE(user4k_rmap_boundary_test), + KUNIT_CASE(native_rmap_walk_test), + KUNIT_CASE(user64k_rmap_lookup_test), + KUNIT_CASE(user64k_anon_rmap_span_test), + KUNIT_CASE(user64k_anon_zap_batch_test), + KUNIT_CASE(user_granule_zero_region_test), +#ifdef CONFIG_COREDUMP + KUNIT_CASE(user_granule_core_offsets_test), +#endif + KUNIT_CASE(user64k_fragment_ownership_test), + KUNIT_CASE(user64k_anon_fault_cow_test), + KUNIT_CASE(user64k_pinned_fork_test), + KUNIT_CASE(user64k_file_rmap_span_test), + KUNIT_CASE(user4k_vma_offsets_test), + KUNIT_CASE(user4k_vma_intervals_test), + KUNIT_CASE(user4k_vma_split_test), + KUNIT_CASE(user4k_vma_merge_test), + KUNIT_CASE(user4k_vma_partial_merge_test), + KUNIT_CASE(user4k_vma_copy_shrink_test), + KUNIT_CASE(user4k_anon_rmap_test), + KUNIT_CASE(user4k_anon_fork_rmap_test), + KUNIT_CASE(user4k_nonlinear_rmap_reuse_test), + KUNIT_CASE(user4k_dup_mmap_test), + KUNIT_CASE(user4k_anon_root_lifetime_test), + KUNIT_CASE(user4k_data_tlb_batch_test), + KUNIT_CASE(user4k_anon_zap_test), + KUNIT_CASE(user4k_anon_writable_slots_test), + KUNIT_CASE(user4k_anon_write_fault_test), + KUNIT_CASE(user16k_anon_write_fault_test), + KUNIT_CASE(user4k_mlock_counts_test), + KUNIT_CASE(user4k_mlock_shared_test), + KUNIT_CASE(user4k_mlock_repair_test), + KUNIT_CASE(user_granule_large_vma_split_test), + KUNIT_CASE(user_granule_brk_extent_test), + KUNIT_CASE(user4k_droppable_reclaim_test), + KUNIT_CASE(user4k_lazyfree_test), + KUNIT_CASE(user4k_zero_cow_test), + KUNIT_CASE(user16k_zero_cow_test), + KUNIT_CASE(user4k_shmem_swap_usage_test), +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + KUNIT_CASE(user4k_shmem_partial_hole_test), + KUNIT_CASE(user4k_shmem_pinned_hole_test), + KUNIT_CASE(user4k_shmem_memfd_pin_test), + KUNIT_CASE(user4k_shmem_replace_validity_test), + KUNIT_CASE(user4k_shmem_uffd_alias_test), + KUNIT_CASE(user4k_shmem_uffd_state_test), +#ifdef CONFIG_MIGRATION + KUNIT_CASE(user4k_shmem_uffd_migration_test), +#endif +#endif + KUNIT_CASE(user4k_file_mapcounts_test), + KUNIT_CASE(user64k_file_promote_test), + KUNIT_CASE(user64k_shmem_fault_test), + KUNIT_CASE(user64k_shmem_fault_busy_test), + KUNIT_CASE(user64k_shmem_fault_hole_test), + KUNIT_CASE(user64k_file_leaf_test), + KUNIT_CASE(user64k_file_cow_test), + KUNIT_CASE(user64k_file_bounds_test), + KUNIT_CASE(user4k_file_shared_test), + KUNIT_CASE(user4k_iommufd_lifetime_test), + KUNIT_CASE(user4k_iommufd_rollback_test), + KUNIT_CASE(user4k_iommufd_granule_test), + KUNIT_CASE(user4k_iommufd_shared_test), + KUNIT_CASE(user4k_iommufd_exit_account_test), + KUNIT_CASE(user4k_iova_bitmap_test), + KUNIT_CASE(user4k_iova_bitmap_window_test), + KUNIT_CASE(user4k_iova_bitmap_retained_test), + KUNIT_CASE(user4k_iova_bitmap_partial_test), + KUNIT_CASE(user4k_mremap_move_test), + KUNIT_CASE(user4k_mremap_resize_test), + KUNIT_CASE(user4k_stack_relocate_test), + KUNIT_CASE(user4k_exec_arguments_test), + KUNIT_CASE(user64k_exec_arguments_test), + KUNIT_CASE(user4k_special_mapping_test), + KUNIT_CASE(user4k_insert_fragment_test), + KUNIT_CASE(user16k_insert_shared_test), + KUNIT_CASE(user4k_insert_range_test), + KUNIT_CASE(user4k_pfn_offset_test), + KUNIT_CASE(user4k_remap_pfn_range_test), + KUNIT_CASE(user4k_iomem_access_test), +#if defined(CONFIG_GENERIC_GETTIMEOFDAY) && defined(CONFIG_VDSO_GETRANDOM) + KUNIT_CASE(user4k_vdso_mapping_test), +#endif + KUNIT_CASE(user4k_mremap_cow_test), + KUNIT_CASE(user4k_mremap_file_test), + KUNIT_CASE(user4k_msync_offsets_test), + KUNIT_CASE(user4k_prefault_dirty_test), + KUNIT_CASE(user4k_cold_batch_test), + KUNIT_CASE(user4k_remote_access_test), + KUNIT_CASE(user4k_remote_file_test), +#ifdef CONFIG_CROSS_MEMORY_ATTACH + KUNIT_CASE(user4k_process_vm_test), +#endif +#ifdef CONFIG_FUTEX + KUNIT_CASE(user4k_futex_keys_test), + KUNIT_CASE(user4k_futex_wake_test), +#endif + KUNIT_CASE(user4k_mmap_descriptor_test), + KUNIT_CASE(user4k_mmap_region_offsets_test), + KUNIT_CASE(user4k_commit_accounting_test), + KUNIT_CASE(mixed_locked_account_units_test), + KUNIT_CASE(user64k_commit_accounting_test), + KUNIT_CASE(user64k_page_table_check_test), +#ifdef CONFIG_PROC_FS + KUNIT_CASE(user4k_accounting_readers_test), +#endif + KUNIT_CASE(user_granule_prefault_test), + KUNIT_CASE(user4k_mmap_accounting_test), + KUNIT_CASE(user4k_vm_mmap_test), + KUNIT_CASE(user4k_mprotect_test), + KUNIT_CASE(user4k_mprotect_cow_batch_test), + KUNIT_CASE(user4k_mprotect_accounting_test), + KUNIT_CASE(user4k_file_cow_test), + KUNIT_CASE(user4k_file_eof_test), + KUNIT_CASE(user4k_file_truncate_test), + KUNIT_CASE(user4k_filemap_fault_test), + KUNIT_CASE(user4k_file_invalidate_test), + KUNIT_CASE(user4k_file_fragments_test), + KUNIT_CASE(user4k_file_fork_hole_test), + KUNIT_CASE(user4k_swap_encoding_test), + KUNIT_CASE(user4k_cow_reuse_test), + KUNIT_CASE(user16k_cow_reuse_test), + KUNIT_CASE(user4k_fork_cow_test), + KUNIT_CASE(user16k_fork_cow_test), + KUNIT_CASE(user4k_slot_pin_test), + KUNIT_CASE(user4k_fragment_pin_test), + KUNIT_CASE(user4k_fragment_locked_pin_test), + KUNIT_CASE(user4k_folio_walk_geometry_test), + KUNIT_CASE(user4k_fragment_longterm_test), +#ifdef CONFIG_BLOCK + KUNIT_CASE(user4k_bio_fragments_test), + KUNIT_CASE(user4k_bio_trim_test), +#endif + KUNIT_CASE(user4k_fragment_permissions_test), + KUNIT_CASE(user16k_fragment_offset_test), +#ifdef CONFIG_MIGRATION + KUNIT_CASE(user4k_migration_test), + KUNIT_CASE(user4k_file_migration_test), + KUNIT_CASE(user64k_file_migration_test), + KUNIT_CASE(user64k_anon_migration_test), + KUNIT_CASE(user4k_fragment_pin_race_test), + KUNIT_CASE(user4k_migration_entries_test), + KUNIT_CASE(user4k_nonlinear_rmap_test), + KUNIT_CASE(user4k_mprotect_migration_test), + KUNIT_CASE(user4k_live_migration_test), + KUNIT_CASE(user4k_migration_wait_test), +#endif + KUNIT_CASE(user4k_anon_exit_test), + KUNIT_CASE(user4k_anon_remote_zap_test), +#ifdef CONFIG_ARM64_CONTPTE + KUNIT_CASE(user4k_contpte_test), + KUNIT_CASE(native_contpte_test), +#endif + KUNIT_CASE(user4k_mm_tables_test), + KUNIT_CASE(user16k_mm_tables_test), + KUNIT_CASE(user64k_mm_tables_test), + KUNIT_CASE(native_mm_tables_test), + KUNIT_CASE(user4k_anon_read_fault_test), + KUNIT_CASE(user4k_mm_hardware_test), + KUNIT_CASE(user4k_fragment_lifetime_test), + KUNIT_CASE(user16k_fragment_lifetime_test), + KUNIT_CASE(user64k_table_owner_test), +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + KUNIT_CASE(native_pgtable_deposit_test), +#endif + KUNIT_CASE(user4k_translation_test), + KUNIT_CASE(user4k_context_uaccess_test), + KUNIT_CASE(user4k_fault_uaccess_test), + {} +}; + +#ifdef CONFIG_NUMA_BALANCING +/* + * The COW hinting expectations require a stable normal-balancing policy. + * Runtime sysctl writers do not take the fixture's mm or PTE locks, so run + * this case at boot and do not expose it through the debugfs rerun control. + */ +static struct kunit_case user4k_numa_init_cases[] = { + KUNIT_CASE(user_subpage_numa_cow_batch_test), + {} +}; + +static struct kunit_suite user4k_numa_init_suite = { + .name = "arm64-user4k-numa-init", + .test_cases = user4k_numa_init_cases, +}; + +kunit_test_init_section_suite(user4k_numa_init_suite); +#endif + +static struct kunit_suite user4k_test_suite = { + .name = "arm64-user4k", + .test_cases = user4k_test_cases, +}; + +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +kunit_test_suites(&user4k_test_suite, &user4k_swap_suite, &user4k_shmem_swap_suite, + &user4k_coarse_shmem_suite, + &user4k_nonlinear_swap_suite); +#else +kunit_test_suites(&user4k_test_suite, &user4k_swap_suite, &user4k_nonlinear_swap_suite); +#endif + +MODULE_LICENSE("GPL"); diff --git a/arch/arm64/mm/user4k-tlb-test.c b/arch/arm64/mm/user4k-tlb-test.c new file mode 100644 index 0000000000000..e5671256b7c76 --- /dev/null +++ b/arch/arm64/mm/user4k-tlb-test.c @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Decode emitted operands without executing TLBI instructions. */ +#include +#include +#include +#include + +struct tlbi_test_state { + u64 next; + u64 end; + u64 stride; + u16 asid; + u32 level; + unsigned int shift; + unsigned int single_ops; + unsigned int range_ops; + bool lpa2; +}; + +static unsigned int test_tg(unsigned int shift) +{ + return shift == 12 ? 1 : shift == 14 ? 2 : 3; +} + +static void record_single(u64 arg) +{ + struct kunit *test = kunit_get_current_test(); + struct tlbi_test_state *state = test->priv; + u64 address = (arg & GENMASK_ULL(43, 0)) << 12; + u64 hint = 0; + + if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && state->level <= 3) + hint = (test_tg(state->shift) << 2) | state->level; + KUNIT_EXPECT_EQ(test, address, state->next); + KUNIT_EXPECT_EQ(test, arg >> 48, (u64)state->asid); + KUNIT_EXPECT_EQ(test, FIELD_GET(TLBI_TTL_MASK, arg), hint); + state->next += state->stride; + state->single_ops++; + KUNIT_EXPECT_LE(test, state->next, state->end); +} + +static void record_range(u64 arg) +{ + struct kunit *test = kunit_get_current_test(); + struct tlbi_test_state *state = test->priv; + u64 address = FIELD_GET(TLBIR_BADDR_MASK, arg) << (state->lpa2 ? 16 : state->shift); + u64 count = (FIELD_GET(TLBIR_NUM_MASK, arg) + 1) << + (5 * FIELD_GET(TLBIR_SCALE_MASK, arg) + 1); + + KUNIT_EXPECT_EQ(test, address, state->next); + KUNIT_EXPECT_EQ(test, arg >> 48, (u64)state->asid); + KUNIT_EXPECT_EQ(test, FIELD_GET(TLBIR_TG_MASK, arg), (u64)test_tg(state->shift)); + KUNIT_EXPECT_EQ(test, FIELD_GET(TLBIR_TTL_MASK, arg), + state->level > 3 ? 0ULL : (u64)state->level); + if (state->lpa2) + KUNIT_EXPECT_TRUE(test, IS_ALIGNED(address, SZ_64K)); + state->next += count << state->shift; + state->range_ops++; + KUNIT_EXPECT_LE(test, state->next, state->end); +} + +static bool range_supported(void) +{ + return true; +} + +static bool range_unsupported(void) +{ + return false; +} + +static void test_flush(struct kunit *test, unsigned int shift, u64 start, + unsigned long pages, u64 stride, u32 level, bool lpa2, + bool has_range) +{ + struct tlbi_test_state state = { + .next = start, + .end = start + (pages << shift), + .stride = stride, + .asid = 0xa55a, + .level = level, + .shift = shift, + .lpa2 = lpa2, + }; + + test->priv = &state; + __flush_tlb_range_op_granule(record_single, record_range, start, pages, + stride, state.asid, level, lpa2, shift, + has_range ? range_supported : range_unsupported); + KUNIT_EXPECT_EQ(test, state.next, state.end); + if (has_range) { + KUNIT_EXPECT_LE(test, state.single_ops + state.range_ops, 20U); + if (pages >= 32) + KUNIT_EXPECT_GT(test, state.range_ops, 0U); + } else { + KUNIT_EXPECT_EQ(test, state.range_ops, 0U); + KUNIT_EXPECT_EQ(test, state.single_ops, (unsigned int)((pages << shift) / stride)); + } + test->priv = NULL; +} + +static void user4k_tlbi_operands_test(struct kunit *test) +{ + static const unsigned long counts[] = { + 0, 1, 2, 3, 31, 32, 33, 63, 64, 65, 2047, 2048, 2049, + 65535, 65536, 65537, MAX_TLBI_RANGE_PAGES - 1, MAX_TLBI_RANGE_PAGES, + }; + static const u32 levels[] = { 2, 3, TLBI_TTL_UNKNOWN }; + unsigned int shift, i, j; + + /* Force both algorithms using recording callbacks, independent of CPU caps. */ + for (shift = 12; shift <= 16; shift += 2) { + u64 size = 1UL << shift; + + for (j = 0; j < ARRAY_SIZE(levels); j++) { + for (i = 0; i < ARRAY_SIZE(counts); i++) { + unsigned long pages = counts[i]; + + test_flush(test, shift, SZ_4M + size, pages, size, + levels[j], false, true); + if (shift != 16) + test_flush(test, shift, SZ_4M + size, pages, size, + levels[j], true, true); + if (pages <= 65) + test_flush(test, shift, SZ_4M + size, pages, size, + levels[j], false, false); + } + } + /* Preserve a larger leaf stride when range instructions are absent. */ + test_flush(test, shift, SZ_1G, 3 * 512, 512 * size, 2, false, false); + /* Last virtual page must not lose high address bits. */ + test_flush(test, shift, (1ULL << 48) - size, 1, size, 3, false, false); + } +} + +static void user4k_tlb_gather_test(struct kunit *test) +{ + struct mm_struct *mm = kunit_kzalloc(test, sizeof(*mm), GFP_KERNEL); + pte_t entries[3] = {}; + unsigned int small; + + KUNIT_ASSERT_NOT_NULL(test, mm); + for (small = 0; small < 2; small++) { + struct mmu_gather tlb = { .mm = mm, .start = ULONG_MAX }; + unsigned long size = small ? SZ_4K : PAGE_SIZE; + unsigned long address = SZ_4M + size; + + mm->context.flags = small ? MMCF_USER_4K : 0; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), size); + tlb_remove_tlb_entries(&tlb, entries, 3, address); + KUNIT_EXPECT_EQ(test, tlb.start, address); + KUNIT_EXPECT_EQ(test, tlb.end, address + 3 * size); + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), size); + KUNIT_EXPECT_EQ(test, tlb_get_level(&tlb), 3); + tlb.cleared_pmds = 1; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), size); + KUNIT_EXPECT_EQ(test, tlb_get_level(&tlb), TLBI_TTL_UNKNOWN); + tlb.cleared_ptes = 0; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), small ? SZ_2M : PMD_SIZE); + tlb.cleared_pmds = 0; + tlb.cleared_puds = 1; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), small ? SZ_1G : PUD_SIZE); + tlb.cleared_puds = 0; + tlb.cleared_p4ds = 1; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), small ? 1UL << 39 : P4D_SIZE); + tlb.freed_tables = 1; + KUNIT_EXPECT_EQ(test, tlb_get_level(&tlb), TLBI_TTL_UNKNOWN); + } +} + +static struct kunit_case user4k_tlb_test_cases[] = { + KUNIT_CASE(user4k_tlbi_operands_test), + KUNIT_CASE(user4k_tlb_gather_test), + {} +}; + +static struct kunit_suite user4k_tlb_test_suite = { + .name = "arm64-user4k-tlb", + .test_cases = user4k_tlb_test_cases, +}; + +kunit_test_suite(user4k_tlb_test_suite); + +MODULE_LICENSE("GPL"); diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig index ceae52fd7586d..33f6c7e9608d5 100644 --- a/drivers/vfio/Kconfig +++ b/drivers/vfio/Kconfig @@ -54,6 +54,18 @@ config VFIO_IOMMU_TYPE1 tristate default n +config VFIO_TYPE1_ACCOUNT_KUNIT_TEST + bool "Test VFIO type1 pin accounting and retained fragments" if !KUNIT_ALL_TESTS + depends on KUNIT=y && VFIO_IOMMU_TYPE1=y + default KUNIT_ALL_TESTS + help + Exercise locked-memory refunds after source-mm exit and DMA owner + changes. Arm64 mixed-granule configurations also test accounting + conversions between user and native page sizes. + Mixed-granule tests exercise retained pins, mappings, replay and + external page arrays with recorded IOMMU translations. No physical + device DMA is performed. + config VFIO_IOMMU_SPAPR_TCE tristate depends on SPAPR_TCE_IOMMU diff --git a/drivers/vfio/vfio_iommu_type1_account_test.c b/drivers/vfio/vfio_iommu_type1_account_test.c new file mode 100644 index 0000000000000..5447258ce49b8 --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_account_test.c @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included by type1 so the tests exercise the actual accounting helpers. */ +#include +#include + +struct vfio_account_mm { + struct mm_struct *mm; + bool live; +}; + +static void vfio_account_mm_free(void *data) +{ + struct vfio_account_mm *ctx = data; + + if (ctx->live) + mmput(ctx->mm); + mmdrop(ctx->mm); +} + +static struct vfio_account_mm *vfio_account_mm_new(struct kunit *test, + unsigned int shift) +{ + struct vfio_account_mm *ctx; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return NULL; + ctx->mm = mm_alloc(); + if (!ctx->mm) + return NULL; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (shift != PAGE_SHIFT) { + pgd_t *old = ctx->mm->pgd, *pgd; + unsigned long flag = shift == 12 ? MMCF_USER_4K : + shift == 14 ? MMCF_USER_16K : MMCF_USER_64K; + + ctx->mm->context.flags |= flag; + pgd = pgd_alloc(ctx->mm); + ctx->mm->context.flags &= ~flag; + if (!pgd) { + mmput(ctx->mm); + return NULL; + } + pgd_free(ctx->mm, old); + ctx->mm->pgd = pgd; + ctx->mm->context.flags |= flag; + } +#endif + ctx->live = true; + mmgrab(ctx->mm); + if (kunit_add_action_or_reset(test, vfio_account_mm_free, ctx)) + return NULL; + return ctx; +} + +static void vfio_account_dma_free(void *data) +{ + struct vfio_dma *dma = data; + + /* Cleanup must also work with the negative-control implementation. */ + if (dma->locked_vm) + mm_lock_acct(dma->task, dma->mm, true, -dma->locked_vm); + mmdrop(dma->mm); + put_task_struct(dma->task); +} + +static struct vfio_dma *vfio_account_dma_new(struct kunit *test, + struct vfio_account_mm *ctx) +{ + struct vfio_dma *dma = kunit_kzalloc(test, sizeof(*dma), GFP_KERNEL); + + if (!dma) + return NULL; + dma->mm = ctx->mm; + mmgrab(dma->mm); + dma->task = get_task_struct(current->group_leader); + dma->lock_cap = true; + if (kunit_add_action_or_reset(test, vfio_account_dma_free, dma)) + return NULL; + return dma; +} + +static void vfio_account_exit_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + 12, 14, 16, +#endif + }; + + for (unsigned int i = 0; i < ARRAY_SIZE(shifts); i++) { + struct vfio_account_mm *ctx; + struct vfio_dma *dma; + + if (i && shifts[i] == PAGE_SHIFT) + continue; + ctx = vfio_account_mm_new(test, shifts[i]); + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_EQ(test, mm_page_shift(ctx->mm), shifts[i]); + dma = vfio_account_dma_new(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_EQ(test, vfio_lock_acct(dma, 2, true), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(ctx->mm), 2UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 2UL * PAGE_SIZE); + ctx->live = false; + mmput(ctx->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&ctx->mm->mm_users), 0); + KUNIT_EXPECT_EQ(test, vfio_lock_acct(dma, 1, true), -ESRCH); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + KUNIT_EXPECT_EQ(test, vfio_lock_acct(dma, -1, true), 0); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(ctx->mm), 1UL); + KUNIT_EXPECT_EQ(test, vfio_lock_acct(dma, -1, true), 0); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + KUNIT_EXPECT_EQ(test, atomic_read(&ctx->mm->mm_users), 0); + kunit_info(test, "VFIO %uK user/%luK native dead-mm refunds checked", + 1U << (shifts[i] - 10), PAGE_SIZE >> 10); + } +} + +static void vfio_account_owner_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + 12, 14, 16, +#endif + }; + + for (unsigned int reverse = 0; reverse < 2; reverse++) { + for (unsigned int i = 0; i < ARRAY_SIZE(shifts); i++) { + struct vfio_account_mm *old, *next; + struct vfio_dma *dma; + + if ((i || reverse) && shifts[i] == PAGE_SHIFT) + continue; + old = vfio_account_mm_new(test, reverse ? shifts[i] : PAGE_SHIFT); + next = vfio_account_mm_new(test, reverse ? PAGE_SHIFT : shifts[i]); + KUNIT_ASSERT_NOT_NULL(test, old); + KUNIT_ASSERT_NOT_NULL(test, next); + dma = vfio_account_dma_new(test, old); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_EQ(test, vfio_lock_acct(dma, 3, true), 0); + old->live = false; + mmput(old->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&old->mm->mm_users), 0); + if (task_rlimit(dma->task, RLIMIT_MEMLOCK) != RLIM_INFINITY) { + unsigned long budget = task_rlimit(dma->task, RLIMIT_MEMLOCK) >> + MM_ACCOUNT_SHIFT; + + /* Model a recipient whose existing pins consume its budget. */ + next->mm->locked_vm = budget; + KUNIT_EXPECT_EQ(test, + vfio_change_dma_mm(dma, dma->task, next->mm, false), + -ENOMEM); + KUNIT_EXPECT_PTR_EQ(test, dma->mm, old->mm); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(old->mm), 3UL); + KUNIT_EXPECT_EQ(test, next->mm->locked_vm, budget); + next->mm->locked_vm = 0; + } + KUNIT_ASSERT_EQ(test, + vfio_change_dma_mm(dma, dma->task, next->mm, true), 0); + KUNIT_EXPECT_PTR_EQ(test, dma->mm, next->mm); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(old->mm), 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), 3UL); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(next->mm), 3UL * PAGE_SIZE); + /* Same-mm ownership is idempotent; it must not double the charge. */ + KUNIT_ASSERT_EQ(test, + vfio_change_dma_mm(dma, dma->task, next->mm, true), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), 3UL); + KUNIT_ASSERT_EQ(test, vfio_lock_acct(dma, -3, true), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(next->mm), 0UL); + kunit_info(test, "VFIO dead %uK mm -> %uK mm ownership refund checked", + 1U << (mm_page_shift(old->mm) - 10), + 1U << (mm_page_shift(next->mm) - 10)); + } + } +} + +static struct kunit_case vfio_account_cases[] = { + KUNIT_CASE(vfio_account_exit_test), + KUNIT_CASE(vfio_account_owner_test), + {} +}; + +static struct kunit_suite vfio_account_suite = { + .name = "vfio-type1-account", + .test_cases = vfio_account_cases, +}; + +kunit_test_suite(vfio_account_suite); diff --git a/drivers/vfio/vfio_iommu_type1_fragments_test.c b/drivers/vfio/vfio_iommu_type1_fragments_test.c new file mode 100644 index 0000000000000..60fffff10c6be --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_fragments_test.c @@ -0,0 +1,1177 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Actual type1 lifecycle with recorded translations; no physical device DMA. */ +#include +#include +#include +#include + +#define VFIO_TEST_VA SZ_64M +#define VFIO_TEST_IOVA SZ_256M + +struct vfio_test_domain { + struct iommu_domain core; + struct vfio_domain type1; + struct xarray translations; + unsigned long mapped; + unsigned long fail_after; + unsigned long largest_map; + unsigned int syncs; + struct folio *observed; + struct kunit *test; +}; + +struct vfio_fragment_fixture { + struct vfio_iommu *iommu; + struct vfio_test_domain domains[2]; + struct vfio_iommu_group group; + struct vfio_device device; + struct kunit *test; + unsigned int external_refs; + unsigned int notifications; +}; + +static int vfio_test_domain_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t phys, size_t pgsize, size_t pgcount, int prot, + gfp_t gfp, size_t *mapped) +{ + struct vfio_test_domain *mock = container_of(domain, typeof(*mock), core); + unsigned long length = pgsize * pgcount; + + mock->largest_map = max(mock->largest_map, length); + for (unsigned long done = 0; done < length; done += SZ_4K) { + int ret; + + if (mock->fail_after && mock->mapped >= mock->fail_after) + return -ENOMEM; + ret = xa_insert(&mock->translations, (iova + done) >> 12, + xa_mk_value((phys + done) >> 12), gfp); + if (ret) + return ret; + mock->mapped += SZ_4K; + *mapped += SZ_4K; + } + return 0; +} + +static size_t vfio_test_domain_unmap(struct iommu_domain *domain, unsigned long iova, + size_t pgsize, size_t pgcount, struct iommu_iotlb_gather *gather) +{ + struct vfio_test_domain *mock = container_of(domain, typeof(*mock), core); + unsigned long length = pgsize * pgcount, done; + + for (done = 0; done < length; done += SZ_4K) { + if (!xa_erase(&mock->translations, (iova + done) >> 12)) + break; + mock->mapped -= SZ_4K; + } + return done; +} + +static phys_addr_t vfio_test_domain_translate(struct iommu_domain *domain, dma_addr_t iova) +{ + struct vfio_test_domain *mock = container_of(domain, typeof(*mock), core); + void *entry = xa_load(&mock->translations, iova >> 12); + + return entry ? ((phys_addr_t)xa_to_value(entry) << 12) + (iova & (SZ_4K - 1)) : 0; +} + +static void vfio_test_domain_sync(struct iommu_domain *domain, + struct iommu_iotlb_gather *gather) +{ + struct vfio_test_domain *mock = container_of(domain, typeof(*mock), core); + + mock->syncs++; + if (mock->observed) + KUNIT_EXPECT_TRUE(mock->test, folio_maybe_dma_pinned(mock->observed)); +} + +static const struct iommu_domain_ops vfio_test_domain_ops = { + .map_pages = vfio_test_domain_map, + .unmap_pages = vfio_test_domain_unmap, + .iova_to_phys = vfio_test_domain_translate, + .iotlb_sync = vfio_test_domain_sync, +}; + +static void vfio_test_notify(struct vfio_device *device, u64 iova, u64 length) +{ + struct vfio_fragment_fixture *f = container_of(device, typeof(*f), device); + + f->notifications++; + while (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + f->external_refs--; + } +} + +static const struct vfio_device_ops vfio_test_device_ops = { + .dma_unmap = vfio_test_notify, +}; + +static void vfio_fragment_fixture_free(void *ptr) +{ + struct vfio_fragment_fixture *f = ptr; + struct rb_node *node; + + while ((node = rb_first(&f->iommu->dma_list))) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + struct rb_node *pin; + + while ((pin = rb_first(&dma->pfn_list))) { + struct vfio_pfn *vpfn = rb_entry(pin, struct vfio_pfn, node); + + vfio_unpin_page_external(dma, vpfn->iova, + list_empty(&f->iommu->domain_list)); + } + vfio_remove_dma(f->iommu, dma); + } + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) { + KUNIT_EXPECT_EQ(f->test, f->domains[i].mapped, 0UL); + KUNIT_EXPECT_TRUE(f->test, xa_empty(&f->domains[i].translations)); + xa_destroy(&f->domains[i].translations); + } + kfree(f->iommu); +} + +static struct vfio_fragment_fixture *vfio_fragment_fixture_new(struct kunit *test) +{ + struct vfio_fragment_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + + if (!f) + return NULL; + f->test = test; + f->iommu = vfio_iommu_type1_open(VFIO_TYPE1v2_IOMMU); + if (IS_ERR(f->iommu)) + return NULL; + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) { + struct vfio_test_domain *mock = &f->domains[i]; + + xa_init(&mock->translations); + mock->test = test; + mock->core.ops = &vfio_test_domain_ops; + mock->core.type = IOMMU_DOMAIN_UNMANAGED; + mock->core.pgsize_bitmap = SZ_4K | PAGE_SIZE; + mock->type1.domain = &mock->core; + INIT_LIST_HEAD(&mock->type1.next); + INIT_LIST_HEAD(&mock->type1.group_list); + } + f->group.iommu_group = (struct iommu_group *)f; + list_add(&f->group.next, &f->iommu->emulated_iommu_groups); + f->iommu->num_non_pinned_groups = 1; + f->device.ops = &vfio_test_device_ops; + vfio_iommu_type1_register_device(f->iommu, &f->device); + if (kunit_add_action_or_reset(test, vfio_fragment_fixture_free, f)) + return NULL; + return f; +} + +static void vfio_test_file_put(void *ptr) +{ + fput(ptr); +} + +static struct file *vfio_fragment_file(struct kunit *test, unsigned long length, + unsigned char value) +{ + struct file *file = shmem_file_setup("vfio-fragments", length, EMPTY_VMA_FLAGS); + unsigned char *data; + loff_t pos = 0; + + if (IS_ERR(file)) + return file; + if (kunit_add_action_or_reset(test, vfio_test_file_put, file)) + return ERR_PTR(-ENOMEM); + data = kunit_kmalloc(test, length, GFP_KERNEL); + if (!data) + return ERR_PTR(-ENOMEM); + memset(data, value, length); + if (kernel_write(file, data, length, &pos) != length) + return ERR_PTR(-EIO); + return file; +} + +static unsigned long vfio_fragment_source(struct vfio_account_mm *ctx, + struct file *file, unsigned long length) +{ + unsigned long addr; + + kthread_use_mm(ctx->mm); + addr = vm_mmap(file, VFIO_TEST_VA, length, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(ctx->mm); + return addr; +} + +static int vfio_fragment_test_map(struct vfio_fragment_fixture *f, + struct vfio_account_mm *ctx, + unsigned long offset, unsigned long length) +{ + struct vfio_iommu_type1_dma_map map = { + .argsz = sizeof(map), + .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, + .vaddr = VFIO_TEST_VA + offset, + .iova = VFIO_TEST_IOVA, + .size = length, + }; + int ret; + + kthread_use_mm(ctx->mm); + ret = vfio_dma_do_map(f->iommu, &map); + kthread_unuse_mm(ctx->mm); + return ret; +} + +static void vfio_fragments_lifetime_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, 14 }; + + for (unsigned int i = 0; i < ARRAY_SIZE(shifts); i++) { + struct vfio_account_mm *ctx; + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + struct file *file; + struct page *page; + unsigned char data[64]; + struct vfio_iommu_type1_dma_unmap unmap = { + .argsz = sizeof(unmap), .iova = VFIO_TEST_IOVA, .size = 2 * PAGE_SIZE, + }; + struct vfio_bitmap bitmap = {}; + + if (shifts[i] == PAGE_SHIFT) + continue; + ctx = vfio_account_mm_new(test, shifts[i]); + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x71); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NOT_NULL(test, dma->fragments); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + for (unsigned int d = 0; d < 2; d++) + f->domains[d].observed = vfio_fragment_find(dma, 0)->pin.folio; + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 2 * PAGE_SIZE); + KUNIT_EXPECT_GE(test, f->domains[0].largest_map, PAGE_SIZE); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 17, 1, IOMMU_READ | IOMMU_WRITE, &page), 1); + f->external_refs++; + KUNIT_EXPECT_EQ(test, page_to_phys(page), + iommu_iova_to_phys(&f->domains[0].core, VFIO_TEST_IOVA)); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + /* The user VA goes away; retained pins are still the DMA buffer. */ + ctx->live = false; + mmput(ctx->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&ctx->mm->mm_users), 0); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, VFIO_TEST_IOVA, + data, sizeof(data), false), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(data, 0x71, sizeof(data)), NULL); + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[1].type1), 0); + list_add_tail(&f->domains[1].type1.next, &f->iommu->domain_list); + KUNIT_EXPECT_EQ(test, f->domains[1].mapped, 2 * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, iommu_iova_to_phys(&f->domains[0].core, VFIO_TEST_IOVA), + iommu_iova_to_phys(&f->domains[1].core, VFIO_TEST_IOVA)); + KUNIT_ASSERT_EQ(test, vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, unmap.size, 2 * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, f->notifications, 1U); + KUNIT_EXPECT_GT(test, f->domains[0].syncs, 0U); + KUNIT_EXPECT_GT(test, f->domains[1].syncs, 0U); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + kunit_release_action(test, vfio_fragment_fixture_free, f); + kunit_info(test, "type1 retained %uK user/%luK native pins after mm exit and replay", + 1U << (shifts[i] - 10), PAGE_SIZE >> 10); + } +} + +static void vfio_fragments_lazy_test(struct kunit *test) +{ + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct file *file; + struct vfio_dma *dma; + struct page *page; + unsigned char data[64]; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x52); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NOT_NULL(test, dma->fragments); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, VFIO_TEST_IOVA, + data, sizeof(data), false), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(data, 0x52, sizeof(data)), NULL); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 37, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + ctx->live = false; + mmput(ctx->mm); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + PAGE_SIZE, 1, IOMMU_READ, &page), -EFAULT); + for (unsigned int i = 0; i < 3; i++) + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 37, 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&dma->fragments->records)); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + kunit_release_action(test, vfio_fragment_fixture_free, f); +} + +static void vfio_fragments_map_failure_test(struct kunit *test) +{ + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct file *file; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x25); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + list_add_tail(&f->domains[1].type1.next, &f->iommu->domain_list); + f->domains[1].fail_after = SZ_4K; + KUNIT_EXPECT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), -ENOMEM); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&f->iommu->dma_list)); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, f->domains[1].mapped, 0UL); + f->domains[1].fail_after = 0; + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), 0); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); +} + +static void vfio_fragments_fine_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + + for (unsigned int i = 0; i < ARRAY_SIZE(shifts); i++) { + struct vfio_account_mm *ctx; + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + struct file *file; + struct page *page = NULL; + unsigned char data[64]; + unsigned long length = PAGE_SIZE + SZ_4K; + size_t copied = 0; + + if (i && shifts[i] == PAGE_SHIFT) + continue; + ctx = vfio_account_mm_new(test, shifts[i]); + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x64); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, SZ_4K, length), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NOT_NULL(test, dma->fragments); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, length); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + /* The physical base is offset inside a native page, not representable. */ + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA, 1, IOMMU_READ, &page), -EINVAL); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + PAGE_SIZE, 1, IOMMU_READ, &page), -EINVAL); + KUNIT_ASSERT_EQ(test, vfio_dma_bitmap_alloc_all(f->iommu, + vfio_dirty_pgsize(f->iommu)), 0); + f->iommu->dirty_page_tracking = true; + memset(data, 0x97, sizeof(data)); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw_chunk(f->iommu, + VFIO_TEST_IOVA + SZ_4K - 17, data, sizeof(data), true, &copied), 0); + KUNIT_EXPECT_EQ(test, copied, sizeof(data)); + KUNIT_EXPECT_EQ(test, bitmap_weight(dma->bitmap, length / SZ_4K), 2U); + KUNIT_EXPECT_TRUE(test, test_bit(0, dma->bitmap)); + KUNIT_EXPECT_TRUE(test, test_bit(1, dma->bitmap)); + memset(data, 0, sizeof(data)); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, + VFIO_TEST_IOVA + SZ_4K - 17, data, sizeof(data), false), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(data, 0x97, sizeof(data)), NULL); + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, + VFIO_TEST_IOVA + length, data, sizeof(data), false), -EINVAL); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + kunit_info(test, "%uK userspace fine IOVA/physical offsets and dirty bytes checked", + 1U << (shifts[i] - 10)); + } +} + +static void vfio_fragments_replay_failure_test(struct kunit *test) +{ + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + struct file *file; + struct page *page; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x58); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + f->domains[0].fail_after = SZ_4K; + KUNIT_EXPECT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[0].type1), -ENOMEM); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_EXPECT_FALSE(test, dma->iommu_mapped); + f->domains[0].fail_after = 0; + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[0].type1), 0); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + /* Detaching the last hardware domain keeps only the external page. */ + vfio_iommu_unmap_unpin_reaccount(f->iommu); + list_del_init(&f->domains[0].type1.next); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 0UL); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + kunit_release_action(test, vfio_fragment_fixture_free, f); +} + +static void vfio_fragments_anon_owner_test(struct kunit *test) +{ + struct vfio_account_mm *old = vfio_account_mm_new(test, 12); + struct vfio_account_mm *next = vfio_account_mm_new(test, PAGE_SHIFT); + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + unsigned char *data = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL); + unsigned long addr, copied, charge; + unsigned char check[64]; + + KUNIT_ASSERT_NOT_NULL(test, old); + KUNIT_ASSERT_NOT_NULL(test, next); + KUNIT_ASSERT_NOT_NULL(test, data); + memset(data, 0x49, PAGE_SIZE); + kthread_use_mm(old->mm); + addr = vm_mmap(NULL, VFIO_TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + copied = addr == VFIO_TEST_VA ? + copy_to_user((void __user *)addr, data, PAGE_SIZE) : PAGE_SIZE; + kthread_unuse_mm(old->mm); + KUNIT_ASSERT_EQ(test, addr, VFIO_TEST_VA); + KUNIT_ASSERT_EQ(test, copied, 0UL); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, old, 0, PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NOT_NULL(test, vfio_fragment_find(dma, 0)->pin.subpage); + charge = dma->locked_vm; + KUNIT_ASSERT_GT(test, charge, 0UL); + old->live = false; + mmput(old->mm); + KUNIT_ASSERT_EQ(test, vfio_change_dma_mm(dma, dma->task, next->mm, true), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(old->mm), 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), charge); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, VFIO_TEST_IOVA, + check, sizeof(check), false), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(check, 0x49, sizeof(check)), NULL); + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[1].type1), 0); + list_add_tail(&f->domains[1].type1.next, &f->iommu->domain_list); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(next->mm), 0UL); + kunit_info(test, "anonymous 4K slot pins survive source exit and native-owner transfer"); +} + +static void vfio_fragments_domain_granule_test(struct kunit *test) +{ + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + struct file *file; + struct page *page; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, PAGE_SIZE, 0x73); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_EQ(test, vfio_dma_bitmap_alloc_all(f->iommu, + vfio_dirty_pgsize(f->iommu)), 0); + f->iommu->dirty_page_tracking = true; + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 31, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, bitmap_weight(dma->bitmap, PAGE_SIZE / SZ_4K), + (unsigned int)(PAGE_SIZE / SZ_4K)); + f->domains[0].core.pgsize_bitmap = SZ_16K; + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[0].type1), 0); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_EXPECT_EQ(test, 1UL << __ffs(f->iommu->pgsize_bitmap), SZ_16K); + KUNIT_EXPECT_EQ(test, vfio_dirty_pgsize(f->iommu), SZ_4K); + vfio_iommu_unmap_unpin_reaccount(f->iommu); + list_del_init(&f->domains[0].type1.next); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_EXPECT_EQ(test, 1UL << __ffs(f->iommu->pgsize_bitmap), SZ_4K); + KUNIT_EXPECT_EQ(test, vfio_dirty_pgsize(f->iommu), SZ_4K); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 31, 1); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); +} + +static void vfio_fragments_native_owner_test(struct kunit *test) +{ + for (unsigned int variant = 0; variant < 9; variant++) { + bool same_mm = variant == 8; + bool mapped = same_mm || (variant & 1); + bool alias = same_mm || (variant & 2), different = variant & 4; + struct vfio_account_mm *old = vfio_account_mm_new(test, PAGE_SHIFT); + struct vfio_account_mm *next = same_mm ? old : vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct vfio_fragment_provider *prepared; + struct file *file, *replacement; + struct vfio_dma *dma; + struct page *pages[2]; + phys_addr_t external[2]; + unsigned long unique, addr; + unsigned char data[32]; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, old); + KUNIT_ASSERT_NOT_NULL(test, next); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x63); + replacement = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x44); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, replacement); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(old, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + if (alias) { + kthread_use_mm(old->mm); + addr = vm_mmap(file, VFIO_TEST_VA + PAGE_SIZE, PAGE_SIZE, + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, 0); + kthread_unuse_mm(old->mm); + KUNIT_ASSERT_EQ(test, addr, VFIO_TEST_VA + PAGE_SIZE); + } + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + if (mapped) { + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + } + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, old, 0, 2 * PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NULL(test, dma->fragments); + if (different) { + kthread_use_mm(old->mm); + addr = vm_mmap(replacement, VFIO_TEST_VA, 2 * PAGE_SIZE, + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, 0); + kthread_unuse_mm(old->mm); + KUNIT_ASSERT_EQ(test, addr, VFIO_TEST_VA); + } + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA + 17, 2, + IOMMU_READ | IOMMU_WRITE, pages), 2); + for (unsigned int i = 0; i < 2; i++) + external[i] = page_to_phys(pages[i]); + if (mapped && different) + KUNIT_EXPECT_NE(test, external[0], + iommu_iova_to_phys(&f->domains[0].core, VFIO_TEST_IOVA)); + KUNIT_ASSERT_EQ(test, dma->locked_vm, 2UL); + if (!same_mm) { + old->live = false; + mmput(old->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&old->mm->mm_users), 0); + } + + prepared = vfio_fragments_prepare_native(f->iommu, dma); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, prepared); + unique = mapped ? (alias ? 1UL : 2UL) : 0; + if (!mapped || different) + unique += different || !alias ? 2UL : 1UL; + KUNIT_EXPECT_EQ(test, vfio_fragments_native_pages(prepared), unique); + /* Failed new-owner credit must not publish metadata or release pins. */ + if (!same_mm && task_rlimit(dma->task, RLIMIT_MEMLOCK) != RLIM_INFINITY) { + unsigned long budget = task_rlimit(dma->task, RLIMIT_MEMLOCK) >> + mm_page_shift(next->mm); + + next->mm->locked_vm = budget; + KUNIT_EXPECT_EQ(test, vfio_change_dma_mm_pages(dma, dma->task, + next->mm, false, unique), -ENOMEM); + KUNIT_EXPECT_PTR_EQ(test, dma->mm, old->mm); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(old->mm), 2UL); + KUNIT_EXPECT_EQ(test, next->mm->locked_vm, budget); + next->mm->locked_vm = 0; + } + vfio_fragments_abort_native(prepared); + KUNIT_EXPECT_PTR_EQ(test, dma->fragments, NULL); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(page_folio(pages[0]))); + kthread_use_mm(next->mm); + ret = vfio_change_dma_owner(f->iommu, dma, + VFIO_TEST_VA + (same_mm ? SZ_4K : 0)); + kthread_unuse_mm(next->mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_NOT_NULL(test, dma->fragments); + KUNIT_EXPECT_EQ(test, dma->locked_vm, unique); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(old->mm), same_mm ? unique : 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), unique); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, VFIO_TEST_IOVA, + data, sizeof(data), false), 0); + KUNIT_EXPECT_PTR_EQ(test, + memchr_inv(data, !mapped && different ? 0x44 : 0x63, sizeof(data)), NULL); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA + 31, 2, + IOMMU_READ | IOMMU_WRITE, pages), 2); + for (unsigned int i = 0; i < 2; i++) + KUNIT_EXPECT_EQ(test, page_to_phys(pages[i]), external[i]); + KUNIT_EXPECT_EQ(test, dma->locked_vm, unique); + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[1].type1), 0); + list_add_tail(&f->domains[1].type1.next, &f->iommu->domain_list); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 31, 2); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 17, 2); + KUNIT_EXPECT_EQ(test, dma->locked_vm, + mapped ? (alias ? 1UL : 2UL) : unique); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), 0UL); + kunit_info(test, "native ownership variant=%u preserves PIN identity and unique charge", + variant); + } +} + +static int vfio_fragment_map_area(struct vfio_fragment_fixture *f, + struct vfio_account_mm *ctx, unsigned long iova_offset, + unsigned long va_offset, unsigned long length, unsigned int flags) +{ + struct vfio_iommu_type1_dma_map map = { + .argsz = sizeof(map), + .flags = flags, + .vaddr = VFIO_TEST_VA + va_offset, + .iova = VFIO_TEST_IOVA + iova_offset, + .size = length, + }; + int ret; + + kthread_use_mm(ctx->mm); + ret = vfio_dma_do_map(f->iommu, &map); + kthread_unuse_mm(ctx->mm); + return ret; +} + +static void vfio_fragments_external_span_test(struct kunit *test) +{ + unsigned int rw = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + + for (unsigned int mapped = 0; mapped < 2; mapped++) { + struct vfio_account_mm *a = vfio_account_mm_new(test, 12); + struct vfio_account_mm *b = vfio_account_mm_new(test, PAGE_SHIFT); + struct vfio_fragment_fixture *f = vfio_fragment_fixture_new(test); + struct file *file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x6b); + struct vfio_iommu_type1_dma_unmap unmap = { + .argsz = sizeof(unmap), .iova = VFIO_TEST_IOVA, .size = SZ_4K, + }; + struct vfio_bitmap bitmap = {}; + struct vfio_dma *first, *rest; + struct page *pages[3]; + unsigned long bits = 2 * PAGE_SIZE / SZ_4K - 1; + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(a, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(b, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + if (mapped) { + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + } + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, a, 0, 0, SZ_4K, rw), 0); + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, b, SZ_4K, SZ_4K, + 2 * PAGE_SIZE - SZ_4K, rw), 0); + first = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + rest = vfio_find_dma(f->iommu, VFIO_TEST_IOVA + SZ_4K, 1); + KUNIT_ASSERT_NOT_NULL(test, first); + KUNIT_ASSERT_NOT_NULL(test, rest); + f->iommu->dirty_page_tracking = true; + KUNIT_ASSERT_EQ(test, vfio_dma_bitmap_alloc_all(f->iommu, SZ_4K), 0); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA + 37, 2, + IOMMU_READ | IOMMU_WRITE, pages), 2); + KUNIT_EXPECT_EQ(test, first->locked_vm, 1UL); + KUNIT_EXPECT_EQ(test, rest->locked_vm, 2UL); + KUNIT_EXPECT_EQ(test, bitmap_weight(first->bitmap, 1), 1); + KUNIT_EXPECT_EQ(test, bitmap_weight(rest->bitmap, bits), (int)bits); + bitmap_clear(rest->bitmap, 0, bits); + vfio_dma_populate_bitmap(rest, SZ_4K); + KUNIT_EXPECT_EQ(test, bitmap_weight(rest->bitmap, bits), (int)bits); + if (mapped) + KUNIT_EXPECT_EQ(test, page_to_phys(pages[0]), + iommu_iova_to_phys(&f->domains[0].core, VFIO_TEST_IOVA)); + /* A failed larger request must leave prior caller references intact. */ + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA, 3, IOMMU_READ, pages), -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, pages[0], NULL); + KUNIT_EXPECT_PTR_EQ(test, pages[1], NULL); + KUNIT_EXPECT_PTR_EQ(test, pages[2], NULL); + a->live = b->live = false; + mmput(a->mm); + mmput(b->mm); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA, 2, IOMMU_READ, pages), 2); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 2); + /* Release the second page first: it shares the second DMA owner. */ + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + PAGE_SIZE + 7, 1); + KUNIT_EXPECT_EQ(test, rest->locked_vm, mapped ? 2UL : 1UL); + f->external_refs = 1; + KUNIT_ASSERT_EQ(test, vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, f->notifications, 1U); + KUNIT_EXPECT_EQ(test, f->external_refs, 0U); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(a->mm), 0UL); + KUNIT_EXPECT_EQ(test, rest->locked_vm, mapped ? 2UL : 0UL); + unmap.iova += SZ_4K; + unmap.size = 2 * PAGE_SIZE - SZ_4K; + KUNIT_ASSERT_EQ(test, vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(b->mm), 0UL); + kunit_release_action(test, vfio_fragment_fixture_free, f); + } + for (unsigned int failure = 0; failure < 4; failure++) { + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f = vfio_fragment_fixture_new(test); + struct file *file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x3a); + struct page *page = NULL; + struct rb_node *node; + unsigned long va = failure == 2 ? PAGE_SIZE + SZ_4K : SZ_4K; + int expected = failure == 1 ? -EPERM : -EINVAL; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, ctx, 0, 0, SZ_4K, rw), 0); + if (failure == 3) + va += SZ_4K; + if (failure != 0) + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, ctx, SZ_4K, va, + SZ_4K, failure == 1 ? VFIO_DMA_MAP_FLAG_READ : rw), 0); + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA, 1, + IOMMU_READ | IOMMU_WRITE, &page), expected); + KUNIT_EXPECT_PTR_EQ(test, page, NULL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + for (node = rb_first(&f->iommu->dma_list); node; node = rb_next(node)) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&dma->pfn_list)); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&dma->fragments->records)); + } + kunit_release_action(test, vfio_fragment_fixture_free, f); + } +} + +#define VFIO_STRESS_CYCLES 1024 + +struct vfio_pin_stress { + struct vfio_fragment_fixture *fixture; + struct vfio_device consumer; + struct vfio_device registrar; + struct vfio_device observer; + struct task_struct *competing_thread; + struct task_struct *pin_thread; + struct task_struct *register_thread; + struct mutex consumer_lock; + struct completion pin_go; + struct completion pin_ready; + struct completion notify_entered; + struct completion pin_done; + struct completion register_go; + struct completion register_entered; + struct completion register_done; + struct completion competing_go; + struct completion competing_done; + unsigned long competing_size; + int competing_ret; + bool competing; + unsigned int competing_unmaps; + unsigned int observations; + atomic_t error; + unsigned int pins; + unsigned int registrations; + bool stopping; +}; + +static void vfio_stress_error(struct vfio_pin_stress *s, int ret) +{ + atomic_cmpxchg(&s->error, 0, ret < 0 ? ret : -EIO); +} + +static void vfio_stress_notify(struct vfio_device *device, u64 iova, u64 length) +{ + struct vfio_pin_stress *s = container_of(device, typeof(*s), consumer); + struct vfio_fragment_fixture *f = s->fixture; + + /* The callback holds the production device-list lock, not the DMA lock. */ + complete(&s->register_go); + if (!wait_for_completion_timeout(&s->register_entered, msecs_to_jiffies(5000))) + vfio_stress_error(s, -ETIMEDOUT); + complete(&s->notify_entered); + mutex_lock(&s->consumer_lock); + f->notifications++; + while (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + f->external_refs--; + } + mutex_unlock(&s->consumer_lock); + if (s->competing) { + /* A second unmap may free the DMA while this callback still runs. */ + complete(&s->competing_go); + if (!wait_for_completion_timeout(&s->competing_done, msecs_to_jiffies(5000))) + vfio_stress_error(s, -ETIMEDOUT); + } +} + +static void vfio_stress_observe(struct vfio_device *device, u64 iova, u64 length) +{ + struct vfio_pin_stress *s = container_of(device, typeof(*s), observer); + + s->observations++; + if (iova != VFIO_TEST_IOVA || length != SZ_4K) + vfio_stress_error(s, -ESTALE); +} + +static void vfio_stress_empty_notify(struct vfio_device *device, u64 iova, u64 length) +{ +} + +static const struct vfio_device_ops vfio_stress_consumer_ops = { + .dma_unmap = vfio_stress_notify, +}; + +static const struct vfio_device_ops vfio_stress_registrar_ops = { + .dma_unmap = vfio_stress_empty_notify, +}; + +static const struct vfio_device_ops vfio_stress_observer_ops = { + .dma_unmap = vfio_stress_observe, +}; + +static int vfio_stress_competing_unmaps(void *data) +{ + struct vfio_pin_stress *s = data; + + for (;;) { + struct vfio_iommu_type1_dma_unmap unmap = { + .argsz = sizeof(unmap), .iova = VFIO_TEST_IOVA, + .size = SZ_4K, + }; + struct vfio_bitmap bitmap = {}; + + wait_for_completion(&s->competing_go); + if (READ_ONCE(s->stopping) || kthread_should_stop()) + break; + s->competing_ret = vfio_dma_do_unmap(s->fixture->iommu, &unmap, &bitmap); + s->competing_size = unmap.size; + s->competing_unmaps++; + complete(&s->competing_done); + } + return 0; +} + +static int vfio_stress_pinner(void *data) +{ + struct vfio_pin_stress *s = data; + struct vfio_fragment_fixture *f = s->fixture; + + for (;;) { + struct page *page = NULL, *original; + void *bytes; + int ret; + + wait_for_completion(&s->pin_go); + if (READ_ONCE(s->stopping) || kthread_should_stop()) + break; + mutex_lock(&s->consumer_lock); + ret = vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 37, 1, IOMMU_READ | IOMMU_WRITE, &page); + if (ret != 1) { + vfio_stress_error(s, ret); + mutex_unlock(&s->consumer_lock); + complete(&s->pin_ready); + complete(&s->pin_done); + continue; + } + f->external_refs++; + s->pins++; + original = page; + complete(&s->pin_ready); + /* Unmap cannot revoke a reference before the consumer publishes it. */ + if (!wait_for_completion_timeout(&s->notify_entered, msecs_to_jiffies(5000))) + vfio_stress_error(s, -ETIMEDOUT); + if (!READ_ONCE(s->stopping)) { + ret = vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 17, 1, IOMMU_READ, &page); + if (ret == 1) { + f->external_refs++; + s->pins++; + if (page != original) + vfio_stress_error(s, -ESTALE); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 17, 1); + f->external_refs--; + } else { + vfio_stress_error(s, ret); + } + bytes = kmap_local_page(original); + if (memchr_inv(bytes, 0x6b, PAGE_SIZE)) + vfio_stress_error(s, -EIO); + kunmap_local(bytes); + } + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 37, 1); + f->external_refs--; + mutex_unlock(&s->consumer_lock); + complete(&s->pin_done); + } + return 0; +} + +static int vfio_stress_registrations(void *data) +{ + struct vfio_pin_stress *s = data; + + for (;;) { + wait_for_completion(&s->register_go); + if (READ_ONCE(s->stopping) || kthread_should_stop()) + break; + complete(&s->register_entered); + vfio_iommu_type1_register_device(s->fixture->iommu, &s->registrar); + vfio_iommu_type1_unregister_device(s->fixture->iommu, &s->registrar); + s->registrations++; + complete(&s->register_done); + } + return 0; +} + +static void vfio_stress_free(void *data) +{ + struct vfio_pin_stress *s = data; + struct vfio_fragment_fixture *f = s->fixture; + + WRITE_ONCE(s->stopping, true); + complete_all(&s->pin_go); + complete_all(&s->notify_entered); + complete_all(&s->register_go); + complete_all(&s->competing_go); + if (s->competing_thread) { + kthread_stop(s->competing_thread); + put_task_struct(s->competing_thread); + } + if (s->pin_thread) { + kthread_stop(s->pin_thread); + put_task_struct(s->pin_thread); + } + if (s->register_thread) { + kthread_stop(s->register_thread); + put_task_struct(s->register_thread); + } + mutex_lock(&s->consumer_lock); + while (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + f->external_refs--; + } + mutex_unlock(&s->consumer_lock); + vfio_iommu_type1_unregister_device(f->iommu, &s->consumer); + vfio_iommu_type1_unregister_device(f->iommu, &s->observer); +} + +static struct vfio_pin_stress *vfio_stress_new(struct kunit *test, + struct vfio_fragment_fixture *f) +{ + struct vfio_pin_stress *s = kunit_kzalloc(test, sizeof(*s), GFP_KERNEL); + + if (!s) + return NULL; + s->fixture = f; + mutex_init(&s->consumer_lock); + init_completion(&s->pin_go); + init_completion(&s->pin_ready); + init_completion(&s->notify_entered); + init_completion(&s->pin_done); + init_completion(&s->register_go); + init_completion(&s->register_entered); + init_completion(&s->register_done); + init_completion(&s->competing_go); + init_completion(&s->competing_done); + atomic_set(&s->error, 0); + s->consumer.ops = &vfio_stress_consumer_ops; + s->registrar.ops = &vfio_stress_registrar_ops; + s->observer.ops = &vfio_stress_observer_ops; + vfio_iommu_type1_unregister_device(f->iommu, &f->device); + /* list_add puts the consumer before the observer in callback order. */ + vfio_iommu_type1_register_device(f->iommu, &s->observer); + vfio_iommu_type1_register_device(f->iommu, &s->consumer); + if (kunit_add_action_or_reset(test, vfio_stress_free, s)) + return NULL; + s->pin_thread = kthread_run(vfio_stress_pinner, s, "vfio-pin-stress"); + if (IS_ERR(s->pin_thread)) { + s->pin_thread = NULL; + return NULL; + } + get_task_struct(s->pin_thread); + s->register_thread = kthread_run(vfio_stress_registrations, s, "vfio-reg-stress"); + if (IS_ERR(s->register_thread)) { + s->register_thread = NULL; + return NULL; + } + get_task_struct(s->register_thread); + s->competing_thread = kthread_run(vfio_stress_competing_unmaps, s, "vfio-unmap-stress"); + if (IS_ERR(s->competing_thread)) { + s->competing_thread = NULL; + return NULL; + } + get_task_struct(s->competing_thread); + return s; +} + +static void vfio_fragments_concurrency_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, 14, 16 }; + unsigned int rw = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + + for (unsigned int shift = 0; shift < ARRAY_SIZE(shifts); shift++) { + if (shifts[shift] > PAGE_SHIFT) + continue; + for (unsigned int mapped = 0; mapped < 2; mapped++) { + struct vfio_account_mm *ctx = vfio_account_mm_new(test, shifts[shift]); + struct vfio_fragment_fixture *f = vfio_fragment_fixture_new(test); + struct file *file = vfio_fragment_file(test, PAGE_SIZE, 0x6b); + struct vfio_pin_stress *s; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, PAGE_SIZE), + VFIO_TEST_VA); + s = vfio_stress_new(test, f); + KUNIT_ASSERT_NOT_NULL(test, s); + if (mapped) { + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + } + for (unsigned int cycle = 0; cycle < VFIO_STRESS_CYCLES; cycle++) { + struct vfio_iommu_type1_dma_unmap unmap = { + .argsz = sizeof(unmap), .iova = VFIO_TEST_IOVA, + .size = SZ_4K, + }; + struct vfio_bitmap bitmap = {}; + bool fine = cycle & 1; + unsigned long remaining = PAGE_SIZE - SZ_4K; + + KUNIT_ASSERT_EQ(test, + vfio_fragment_map_area(f, ctx, 0, 0, SZ_4K, rw), 0); + for (unsigned long offset = SZ_4K; offset < PAGE_SIZE;) { + unsigned long length = fine ? SZ_4K : remaining; + + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, ctx, offset, + offset, length, rw), 0); + offset += length; + } + if (cycle == VFIO_STRESS_CYCLES - 1) { + struct page *page; + + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA, 1, + IOMMU_READ, &page), 1); + f->external_refs = 1; + ctx->live = false; + mmput(ctx->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&ctx->mm->mm_users), 0); + } + s->competing = cycle & 2; + complete(&s->pin_go); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&s->pin_ready, + msecs_to_jiffies(5000)), 0UL); + KUNIT_ASSERT_EQ(test, atomic_read(&s->error), 0); + KUNIT_ASSERT_EQ(test, + vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, unmap.size, s->competing ? 0UL : SZ_4K); + if (s->competing) { + KUNIT_EXPECT_EQ(test, s->competing_ret, 0); + KUNIT_EXPECT_EQ(test, s->competing_size, SZ_4K); + } + unmap.iova += SZ_4K; + unmap.size = remaining; + KUNIT_ASSERT_EQ(test, + vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, unmap.size, remaining); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&s->pin_done, + msecs_to_jiffies(5000)), 0UL); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&s->register_done, + msecs_to_jiffies(5000)), 0UL); + KUNIT_ASSERT_EQ(test, atomic_read(&s->error), 0); + KUNIT_EXPECT_EQ(test, f->external_refs, 0U); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&f->iommu->dma_list)); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 0UL); + reinit_completion(&s->pin_ready); + reinit_completion(&s->notify_entered); + reinit_completion(&s->pin_done); + reinit_completion(&s->register_entered); + reinit_completion(&s->register_done); + reinit_completion(&s->competing_done); + } + KUNIT_EXPECT_EQ(test, s->pins, 2U * VFIO_STRESS_CYCLES); + KUNIT_EXPECT_EQ(test, s->registrations, (unsigned int)VFIO_STRESS_CYCLES); + KUNIT_EXPECT_EQ(test, f->notifications, (unsigned int)VFIO_STRESS_CYCLES); + KUNIT_EXPECT_EQ(test, s->observations, (unsigned int)VFIO_STRESS_CYCLES); + KUNIT_EXPECT_EQ(test, s->competing_unmaps, VFIO_STRESS_CYCLES / 2U); + kunit_info(test, "%uK user/%luK native mapped=%u: %u forced pin/unmap/register cycles, half with competing unmap", + 1U << (shifts[shift] - 10), PAGE_SIZE >> 10, + mapped, VFIO_STRESS_CYCLES); + kunit_release_action(test, vfio_stress_free, s); + kunit_release_action(test, vfio_fragment_fixture_free, f); + } + } +} + +static struct kunit_case vfio_fragments_cases[] = { + KUNIT_CASE(vfio_fragments_lifetime_test), + KUNIT_CASE(vfio_fragments_lazy_test), + KUNIT_CASE(vfio_fragments_map_failure_test), + KUNIT_CASE(vfio_fragments_fine_test), + KUNIT_CASE(vfio_fragments_replay_failure_test), + KUNIT_CASE(vfio_fragments_anon_owner_test), + KUNIT_CASE(vfio_fragments_domain_granule_test), + KUNIT_CASE(vfio_fragments_native_owner_test), + KUNIT_CASE(vfio_fragments_external_span_test), + KUNIT_CASE(vfio_fragments_concurrency_test), + {} +}; + +static struct kunit_suite vfio_fragments_suite = { + .name = "vfio-type1-fragments", + .test_cases = vfio_fragments_cases, +}; + +kunit_test_suite(vfio_fragments_suite); diff --git a/drivers/vfio/vfio_iommu_type1_ioctl_test.c b/drivers/vfio/vfio_iommu_type1_ioctl_test.c new file mode 100644 index 0000000000000..33e981bcdf74d --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_ioctl_test.c @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included after the KUnit harness; real EL0 arguments, recorded domains only. */ +#include +#include +#include +#include "vfio_iommu_type1_test.h" + +struct vfio_ioctl_fixture { + struct vfio_fragment_fixture pins; + struct mutex request_lock; +}; + +struct vfio_ioctl_rw { + struct work_struct work; + struct vfio_iommu *iommu; + dma_addr_t iova; + void *data; + size_t size; + bool write; + int ret; +}; + +static void vfio_ioctl_rw_work(struct work_struct *work) +{ + struct vfio_ioctl_rw *rw = container_of(work, typeof(*rw), work); + + rw->ret = vfio_iommu_type1_dma_rw(rw->iommu, rw->iova, rw->data, + rw->size, rw->write); +} + +static int vfio_ioctl_test_open(struct inode *inode, struct file *file) +{ + struct vfio_ioctl_fixture *ctx; + struct vfio_fragment_fixture *f; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + ctx = kzalloc_obj(*ctx); + if (!ctx) + return -ENOMEM; + f = &ctx->pins; + f->iommu = vfio_iommu_type1_open(VFIO_TYPE1v2_IOMMU); + if (IS_ERR(f->iommu)) { + int ret = PTR_ERR(f->iommu); + + kfree(ctx); + return ret; + } + mutex_init(&ctx->request_lock); + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) { + struct vfio_test_domain *mock = &f->domains[i]; + + xa_init(&mock->translations); + mock->core.ops = &vfio_test_domain_ops; + mock->core.type = IOMMU_DOMAIN_UNMANAGED; + mock->core.pgsize_bitmap = SZ_4K | PAGE_SIZE; + mock->type1.domain = &mock->core; + INIT_LIST_HEAD(&mock->type1.next); + INIT_LIST_HEAD(&mock->type1.group_list); + } + f->group.iommu_group = (struct iommu_group *)ctx; + list_add(&f->group.next, &f->iommu->emulated_iommu_groups); + f->iommu->num_non_pinned_groups = 1; + f->device.ops = &vfio_test_device_ops; + vfio_iommu_type1_register_device(f->iommu, &f->device); + file->private_data = ctx; + return 0; +} + +static int vfio_ioctl_test_release(struct inode *inode, struct file *file) +{ + struct vfio_ioctl_fixture *ctx = file->private_data; + struct vfio_fragment_fixture *f = &ctx->pins; + + while (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + f->external_refs--; + } + vfio_iommu_type1_unregister_device(f->iommu, &f->device); + vfio_iommu_unmap_unpin_all(f->iommu); + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) { + list_del_init(&f->domains[i].type1.next); + xa_destroy(&f->domains[i].translations); + } + list_del(&f->group.next); + vfio_iommu_type1_release(f->iommu); + kfree(ctx); + return 0; +} + +static int vfio_ioctl_test_domain(struct vfio_fragment_fixture *f, + struct vfio_type1_test_cmd *cmd, bool add) +{ + struct vfio_test_domain *mock; + struct rb_node *node; + int ret = 0; + + if (cmd->domain >= ARRAY_SIZE(f->domains) || cmd->flags) + return -EINVAL; + mock = &f->domains[cmd->domain]; + mutex_lock(&f->iommu->lock); + if (add) { + if (!list_empty(&mock->type1.next)) { + ret = -EEXIST; + goto out; + } + if (cmd->size != SZ_4K && cmd->size != SZ_16K && cmd->size != SZ_64K) { + ret = -EINVAL; + goto out; + } + mock->core.pgsize_bitmap = cmd->size; + mock->fail_after = cmd->fail_after; + ret = vfio_iommu_replay(f->iommu, &mock->type1); + if (ret) + goto out; + list_add_tail(&mock->type1.next, &f->iommu->domain_list); + } else { + if (list_empty(&mock->type1.next)) { + ret = -ENOENT; + goto out; + } + if (list_is_singular(&f->iommu->domain_list)) { + vfio_iommu_unmap_unpin_reaccount(f->iommu); + } else { + for (node = rb_first(&f->iommu->dma_list); node; node = rb_next(node)) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + + iommu_unmap(&mock->core, dma->iova, dma->size); + } + } + list_del_init(&mock->type1.next); + } + vfio_update_pgsize_bitmap(f->iommu); +out: + mutex_unlock(&f->iommu->lock); + return ret; +} + +static int vfio_ioctl_test_rw(struct vfio_fragment_fixture *f, + struct vfio_type1_test_cmd *cmd) +{ + struct vfio_ioctl_rw rw = { + .iommu = f->iommu, .iova = cmd->iova, .size = cmd->size, + .write = cmd->flags & VFIO_TYPE1_TEST_WRITE, + }; + void __user *buffer = u64_to_user_ptr(cmd->buffer); + int ret; + + if (!cmd->size || cmd->size > 256 || + cmd->flags & ~(VFIO_TYPE1_TEST_WRITE | VFIO_TYPE1_TEST_WORKER)) + return -EINVAL; + rw.data = kzalloc(cmd->size, GFP_KERNEL); + if (!rw.data) + return -ENOMEM; + if (rw.write && copy_from_user(rw.data, buffer, cmd->size)) { + ret = -EFAULT; + goto out; + } + if (cmd->flags & VFIO_TYPE1_TEST_WORKER) { + INIT_WORK(&rw.work, vfio_ioctl_rw_work); + queue_work(system_unbound_wq, &rw.work); + flush_work(&rw.work); + ret = rw.ret; + } else { + ret = vfio_iommu_type1_dma_rw(f->iommu, rw.iova, rw.data, rw.size, rw.write); + } + if (!ret && !rw.write && copy_to_user(buffer, rw.data, cmd->size)) + ret = -EFAULT; +out: + kfree(rw.data); + return ret; +} + +static long vfio_ioctl_test_ioctl(struct file *file, unsigned int request, + unsigned long arg) +{ + struct vfio_ioctl_fixture *ctx = file->private_data; + struct vfio_fragment_fixture *f = &ctx->pins; + struct vfio_type1_test_cmd cmd; + void __user *user = (void __user *)arg; + struct page *page; + int ret = 0; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + /* Request serialization belongs to this fixture, not production type1. */ + mutex_lock(&ctx->request_lock); + switch (request) { + case VFIO_CHECK_EXTENSION: + case VFIO_IOMMU_GET_INFO: + case VFIO_IOMMU_MAP_DMA: + case VFIO_IOMMU_UNMAP_DMA: + case VFIO_IOMMU_DIRTY_PAGES: + ret = vfio_iommu_type1_ioctl(f->iommu, request, arg); + goto out; + default: + break; + } + if (copy_from_user(&cmd, user, sizeof(cmd))) { + ret = -EFAULT; + goto out; + } + if (cmd.reserved) { + ret = -EINVAL; + goto out; + } + switch (request) { + case VFIO_TYPE1_TEST_DOMAIN_ADD: + case VFIO_TYPE1_TEST_DOMAIN_DEL: + ret = vfio_ioctl_test_domain(f, &cmd, request == VFIO_TYPE1_TEST_DOMAIN_ADD); + break; + case VFIO_TYPE1_TEST_RW: + ret = vfio_ioctl_test_rw(f, &cmd); + break; + case VFIO_TYPE1_TEST_PIN: + case VFIO_TYPE1_TEST_UNPIN: + if (ALIGN_DOWN(cmd.iova, PAGE_SIZE) != VFIO_TEST_IOVA || cmd.flags) { + ret = -EINVAL; + break; + } + if (request == VFIO_TYPE1_TEST_PIN) { + ret = vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + cmd.iova, 1, IOMMU_READ | IOMMU_WRITE, &page); + if (ret == 1) { + f->external_refs++; + ret = 0; + } + } else if (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, cmd.iova, 1); + f->external_refs--; + } else { + ret = -ENOENT; + } + break; + case VFIO_TYPE1_TEST_EMULATED: + if (cmd.flags > 1 || f->external_refs) { + ret = -EBUSY; + break; + } + mutex_lock(&f->iommu->lock); + if (cmd.flags && list_empty(&f->group.next)) { + list_add(&f->group.next, &f->iommu->emulated_iommu_groups); + if (!f->group.pinned_page_dirty_scope) + f->iommu->num_non_pinned_groups++; + } else if (!cmd.flags && !list_empty(&f->group.next)) { + list_del_init(&f->group.next); + if (!f->group.pinned_page_dirty_scope) + f->iommu->num_non_pinned_groups--; + } + mutex_unlock(&f->iommu->lock); + break; + case VFIO_TYPE1_TEST_STATE: + cmd.locked_bytes = 0; + cmd.owner_locked_bytes = mm_locked_vm_bytes(current->mm); + cmd.flags = 0; + cmd.owner_users = 0; + mutex_lock(&f->iommu->lock); + for (struct rb_node *node = rb_first(&f->iommu->dma_list); + node; node = rb_next(node)) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + + cmd.locked_bytes += dma->locked_vm * PAGE_SIZE; + cmd.owner_locked_bytes = mm_locked_vm_bytes(dma->mm); + cmd.owner_users = atomic_read(&dma->mm->mm_users); + if (dma->fragments) + cmd.flags |= VFIO_TYPE1_TEST_STATE_FRAGMENTS; + } + cmd.native_page_size = PAGE_SIZE; + cmd.minimum_page_size = 1UL << __ffs(f->iommu->pgsize_bitmap); + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) + cmd.mapped_bytes[i] = f->domains[i].mapped; + mutex_unlock(&f->iommu->lock); + cmd.notifications = f->notifications; + cmd.external_refs = f->external_refs; + break; + default: + ret = -ENOTTY; + } + if (!ret && copy_to_user(user, &cmd, sizeof(cmd))) + ret = -EFAULT; +out: + mutex_unlock(&ctx->request_lock); + return ret; +} + +static const struct file_operations vfio_ioctl_test_fops = { + .owner = THIS_MODULE, + .open = vfio_ioctl_test_open, + .release = vfio_ioctl_test_release, + .unlocked_ioctl = vfio_ioctl_test_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; + +static struct miscdevice vfio_ioctl_test_device = { + .minor = MISC_DYNAMIC_MINOR, + .name = "aurora-vfio-type1-test", + .fops = &vfio_ioctl_test_fops, + .mode = 0600, +}; + +static int __init vfio_ioctl_test_init(void) +{ + return misc_register(&vfio_ioctl_test_device); +} +late_initcall(vfio_ioctl_test_init); diff --git a/drivers/vfio/vfio_iommu_type1_test.h b/drivers/vfio/vfio_iommu_type1_test.h new file mode 100644 index 0000000000000..36c09030bcf0e --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_test.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* Private disposable-guest test protocol, not a supported VFIO ABI. */ +#ifndef VFIO_IOMMU_TYPE1_TEST_H +#define VFIO_IOMMU_TYPE1_TEST_H +#include +#include + +struct vfio_type1_test_cmd { + __u64 iova; + __u64 size; + __u64 buffer; + __u64 fail_after; + __u64 locked_bytes; + __u64 owner_locked_bytes; + __u64 mapped_bytes[2]; + __u64 native_page_size; + __u64 minimum_page_size; + __u32 domain; + __u32 flags; + __u32 owner_users; + __u32 notifications; + __u32 external_refs; + __u32 reserved; +}; + +#define VFIO_TYPE1_TEST_DOMAIN_ADD _IOWR('V', 1, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_DOMAIN_DEL _IOWR('V', 2, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_RW _IOWR('V', 3, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_PIN _IOWR('V', 4, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_UNPIN _IOWR('V', 5, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_STATE _IOWR('V', 6, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_EMULATED _IOWR('V', 7, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_STATE_FRAGMENTS 1U +#define VFIO_TYPE1_TEST_WRITE 1U +#define VFIO_TYPE1_TEST_WORKER 2U +#endif From e29133a0a7a289c8a894625d1a77691d96b4a512 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:23 -0400 Subject: [PATCH 0857/2680] selftests: exercise the mixed userspace page-size ABI Cover exec policy, AArch32 fallback, mappings, sharing, reclaim, accounting and converted I/O interfaces. Keep extended PID1 fixtures separate from the ordinary page-contract executable. Signed-off-by: Diljit Singh (cherry picked from commit ad36aab5acc6e025db2a66192d259a1058d77d02) --- .../testing/kunit/configs/arm64_user4k.config | 33 + .../configs/arm64_user4k_data_kasan.config | 18 + .../configs/arm64_user4k_mm_kasan.config | 38 + tools/testing/selftests/arm64/Makefile | 2 +- .../testing/selftests/arm64/user4k/.gitignore | 3 + tools/testing/selftests/arm64/user4k/Makefile | 134 ++ tools/testing/selftests/arm64/user4k/advice.c | 199 +++ .../selftests/arm64/user4k/aio-ring-init.c | 178 +++ .../arm64/user4k/aslr-fixture-init.c | 175 +++ .../selftests/arm64/user4k/bpf-uptr-init.c | 583 ++++++++ .../arm64/user4k/coarse-shmem-init.c | 223 +++ .../arm64/user4k/compat-aslr-probe.c | 134 ++ .../arm64/user4k/compat-fixture-init.c | 149 ++ .../arm64/user4k/compat-mlock-init.c | 62 + .../arm64/user4k/compat-mlock-probe.c | 142 ++ .../user4k/compat-numa-balancing-probe.c | 223 +++ .../arm64/user4k/compat-numa-probe.c | 117 ++ .../arm64/user4k/compat-perf-probe.c | 107 ++ .../arm64/user4k/compat-policy-probe.c | 66 + .../selftests/arm64/user4k/compat-probe.c | 248 ++++ .../selftests/arm64/user4k/compat-script | 1 + .../selftests/arm64/user4k/compat-shm-probe.c | 111 ++ .../selftests/arm64/user4k/compat-uffd-init.c | 154 ++ .../arm64/user4k/compat-uffd-probe.c | 149 ++ .../arm64/user4k/compat-vfio-probe.c | 250 ++++ .../arm64/user4k/context-stress-init.c | 275 ++++ .../arm64/user4k/core-fixture-init.c | 447 ++++++ .../selftests/arm64/user4k/cpu-granule-init.c | 187 +++ .../arm64/user4k/cpu-granule16-init.c | 192 +++ .../selftests/arm64/user4k/dio-fixture-init.c | 266 ++++ .../arm64/user4k/exec-fixture-init.c | 167 +++ .../selftests/arm64/user4k/exec-policy-init.c | 214 +++ .../arm64/user4k/fex-contract-init.c | 52 + .../selftests/arm64/user4k/fex-native-abi.c | 20 + .../selftests/arm64/user4k/fex-x86-contract.c | 132 ++ .../selftests/arm64/user4k/gem-fixture-init.c | 325 ++++ .../arm64/user4k/granule-proc-init.c | 141 ++ .../arm64/user4k/granule-swap-init.c | 248 ++++ .../arm64/user4k/granule-threeway-init.c | 248 ++++ .../selftests/arm64/user4k/iommufd-init.c | 1314 +++++++++++++++++ .../arm64/user4k/lazyfree-fixture-init.c | 274 ++++ .../arm64/user4k/memcg-accounting-init.c | 153 ++ .../arm64/user4k/memcg-pressure-init.c | 417 ++++++ .../arm64/user4k/memory-interfaces.c | 165 +++ .../arm64/user4k/mglru-granule-init.c | 365 +++++ .../selftests/arm64/user4k/mixed-process.c | 209 +++ .../arm64/user4k/mlock-fixture-init.c | 249 ++++ .../arm64/user4k/numa-balancing-init.c | 307 ++++ .../selftests/arm64/user4k/numa-policy-init.c | 236 +++ .../selftests/arm64/user4k/page-contract.c | 232 +++ .../selftests/arm64/user4k/perf-ring-init.c | 323 ++++ .../selftests/arm64/user4k/proc-args.c | 164 ++ .../selftests/arm64/user4k/proc-memory.c | 340 +++++ .../selftests/arm64/user4k/proc-pages.c | 318 ++++ .../selftests/arm64/user4k/reclaim-advice.c | 366 +++++ .../selftests/arm64/user4k/runtime-app-init.c | 76 + .../selftests/arm64/user4k/runtime-app.py | 91 ++ .../arm64/user4k/shmem-collapse-init.c | 396 +++++ .../selftests/arm64/user4k/shmem-hole-init.c | 358 +++++ .../selftests/arm64/user4k/shmem-pin-init.c | 358 +++++ .../arm64/user4k/shmem-pressure-init.c | 225 +++ .../selftests/arm64/user4k/shmem-race-init.c | 308 ++++ .../selftests/arm64/user4k/shmem-thp-init.c | 271 ++++ .../arm64/user4k/shmem-uffd-swap-init.c | 160 ++ .../arm64/user4k/splice-fixture-init.c | 610 ++++++++ .../arm64/user4k/swap-fixture-init.c | 216 +++ .../selftests/arm64/user4k/swapfile-fixture.h | 50 + .../selftests/arm64/user4k/sysv-shm-init.c | 229 +++ .../arm64/user4k/thread-fixture-init.c | 424 ++++++ .../selftests/arm64/user4k/uffd-move-init.c | 451 ++++++ .../selftests/arm64/user4k/uffd-wp-init.c | 913 ++++++++++++ .../selftests/arm64/user4k/uprobes-init.c | 318 ++++ .../arm64/user4k/uring-fixture-init.c | 467 ++++++ .../selftests/arm64/user4k/user-events-init.c | 268 ++++ .../arm64/user4k/user16k-smoke-init.c | 229 +++ .../arm64/user4k/vfio-fixture-init.c | 542 +++++++ .../selftests/arm64/user4k/vhost-log-init.c | 392 +++++ .../selftests/arm64/user4k/wine-kernel32.def | 20 + .../arm64/user4k/wine-page-contract.c | 181 +++ .../selftests/arm64/user4k/x18-probe.S | 25 + tools/testing/vma/include/dup.h | 117 ++ tools/testing/vma/tests/merge.c | 2 +- tools/testing/vma/tests/mmap.c | 8 +- tools/testing/vma/tests/vma.c | 4 +- 84 files changed, 19546 insertions(+), 8 deletions(-) create mode 100644 tools/testing/kunit/configs/arm64_user4k.config create mode 100644 tools/testing/kunit/configs/arm64_user4k_data_kasan.config create mode 100644 tools/testing/kunit/configs/arm64_user4k_mm_kasan.config create mode 100644 tools/testing/selftests/arm64/user4k/.gitignore create mode 100644 tools/testing/selftests/arm64/user4k/Makefile create mode 100644 tools/testing/selftests/arm64/user4k/advice.c create mode 100644 tools/testing/selftests/arm64/user4k/aio-ring-init.c create mode 100644 tools/testing/selftests/arm64/user4k/aslr-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/bpf-uptr-init.c create mode 100644 tools/testing/selftests/arm64/user4k/coarse-shmem-init.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-aslr-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-mlock-init.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-mlock-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-numa-balancing-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-numa-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-perf-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-policy-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-script create mode 100644 tools/testing/selftests/arm64/user4k/compat-shm-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-uffd-init.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-uffd-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-vfio-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/context-stress-init.c create mode 100644 tools/testing/selftests/arm64/user4k/core-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/cpu-granule-init.c create mode 100644 tools/testing/selftests/arm64/user4k/cpu-granule16-init.c create mode 100644 tools/testing/selftests/arm64/user4k/dio-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/exec-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/exec-policy-init.c create mode 100644 tools/testing/selftests/arm64/user4k/fex-contract-init.c create mode 100644 tools/testing/selftests/arm64/user4k/fex-native-abi.c create mode 100644 tools/testing/selftests/arm64/user4k/fex-x86-contract.c create mode 100644 tools/testing/selftests/arm64/user4k/gem-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/granule-proc-init.c create mode 100644 tools/testing/selftests/arm64/user4k/granule-swap-init.c create mode 100644 tools/testing/selftests/arm64/user4k/granule-threeway-init.c create mode 100644 tools/testing/selftests/arm64/user4k/iommufd-init.c create mode 100644 tools/testing/selftests/arm64/user4k/lazyfree-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/memcg-accounting-init.c create mode 100644 tools/testing/selftests/arm64/user4k/memcg-pressure-init.c create mode 100644 tools/testing/selftests/arm64/user4k/memory-interfaces.c create mode 100644 tools/testing/selftests/arm64/user4k/mglru-granule-init.c create mode 100644 tools/testing/selftests/arm64/user4k/mixed-process.c create mode 100644 tools/testing/selftests/arm64/user4k/mlock-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/numa-balancing-init.c create mode 100644 tools/testing/selftests/arm64/user4k/numa-policy-init.c create mode 100644 tools/testing/selftests/arm64/user4k/page-contract.c create mode 100644 tools/testing/selftests/arm64/user4k/perf-ring-init.c create mode 100644 tools/testing/selftests/arm64/user4k/proc-args.c create mode 100644 tools/testing/selftests/arm64/user4k/proc-memory.c create mode 100644 tools/testing/selftests/arm64/user4k/proc-pages.c create mode 100644 tools/testing/selftests/arm64/user4k/reclaim-advice.c create mode 100644 tools/testing/selftests/arm64/user4k/runtime-app-init.c create mode 100755 tools/testing/selftests/arm64/user4k/runtime-app.py create mode 100644 tools/testing/selftests/arm64/user4k/shmem-collapse-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-hole-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-pin-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-pressure-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-race-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-thp-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-uffd-swap-init.c create mode 100644 tools/testing/selftests/arm64/user4k/splice-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/swap-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/swapfile-fixture.h create mode 100644 tools/testing/selftests/arm64/user4k/sysv-shm-init.c create mode 100644 tools/testing/selftests/arm64/user4k/thread-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/uffd-move-init.c create mode 100644 tools/testing/selftests/arm64/user4k/uffd-wp-init.c create mode 100644 tools/testing/selftests/arm64/user4k/uprobes-init.c create mode 100644 tools/testing/selftests/arm64/user4k/uring-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/user-events-init.c create mode 100644 tools/testing/selftests/arm64/user4k/user16k-smoke-init.c create mode 100644 tools/testing/selftests/arm64/user4k/vfio-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/vhost-log-init.c create mode 100644 tools/testing/selftests/arm64/user4k/wine-kernel32.def create mode 100644 tools/testing/selftests/arm64/user4k/wine-page-contract.c create mode 100644 tools/testing/selftests/arm64/user4k/x18-probe.S diff --git a/tools/testing/kunit/configs/arm64_user4k.config b/tools/testing/kunit/configs/arm64_user4k.config new file mode 100644 index 0000000000000..de6c8ec2fdff8 --- /dev/null +++ b/tools/testing/kunit/configs/arm64_user4k.config @@ -0,0 +1,33 @@ +CONFIG_KUNIT=y +CONFIG_ARM64_16K_PAGES=y +CONFIG_ARM64_VA_BITS_48=y +CONFIG_ARM64_PA_BITS_48=y +CONFIG_ARM64_USER4K_EXPERIMENTAL=y +CONFIG_ARM64_USER4K_KUNIT_TEST=y +CONFIG_ARM64_SW_TTBR0_PAN=y +# CONFIG_ARM64_CNP is not set +# CONFIG_ARM64_POE is not set +CONFIG_SMP=y +CONFIG_PRINTK=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_TMPFS=y +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_VM=y +CONFIG_DEBUG_VM_MAPLE_TREE=y +CONFIG_DEBUG_VM_RB=y +CONFIG_PROVE_LOCKING=y +CONFIG_PAGE_TABLE_CHECK=y +CONFIG_PAGE_TABLE_CHECK_ENFORCED=y +# Guest-only infrastructure for late swap/reclaim fixtures. +CONFIG_BLOCK=y +CONFIG_VIRTIO=y +CONFIG_VIRTIO_MMIO=y +CONFIG_VIRTIO_BLK=y +CONFIG_DEVTMPFS=y +CONFIG_DEBUG_FS=y +CONFIG_KUNIT_DEBUGFS=y diff --git a/tools/testing/kunit/configs/arm64_user4k_data_kasan.config b/tools/testing/kunit/configs/arm64_user4k_data_kasan.config new file mode 100644 index 0000000000000..2e3e50bd67768 --- /dev/null +++ b/tools/testing/kunit/configs/arm64_user4k_data_kasan.config @@ -0,0 +1,18 @@ +CONFIG_KUNIT=y +CONFIG_ARM64_16K_PAGES=y +CONFIG_ARM64_VA_BITS_48=y +CONFIG_ARM64_PA_BITS_48=y +CONFIG_ARM64_USER4K_EXPERIMENTAL=y +CONFIG_ARM64_USER4K_DATA_KUNIT_TEST=y +# CONFIG_ARM64_USER4K_KUNIT_TEST is not set +CONFIG_KASAN=y +CONFIG_KASAN_GENERIC=y +CONFIG_KASAN_INLINE=y +CONFIG_SMP=y +CONFIG_PRINTK=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_TMPFS=y diff --git a/tools/testing/kunit/configs/arm64_user4k_mm_kasan.config b/tools/testing/kunit/configs/arm64_user4k_mm_kasan.config new file mode 100644 index 0000000000000..11b9935770f53 --- /dev/null +++ b/tools/testing/kunit/configs/arm64_user4k_mm_kasan.config @@ -0,0 +1,38 @@ +CONFIG_KUNIT=y +CONFIG_ARM64_16K_PAGES=y +CONFIG_ARM64_VA_BITS_48=y +CONFIG_ARM64_PA_BITS_48=y +CONFIG_ARM64_USER4K_EXPERIMENTAL=y +CONFIG_ARM64_USER4K_KUNIT_TEST=y +CONFIG_ARM64_SW_TTBR0_PAN=y +# CONFIG_ARM64_CNP is not set +# CONFIG_ARM64_POE is not set +CONFIG_SMP=y +CONFIG_PRINTK=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_TMPFS=y +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_VM=y +CONFIG_DEBUG_VM_MAPLE_TREE=y +CONFIG_DEBUG_VM_RB=y +CONFIG_PROVE_LOCKING=y +CONFIG_PAGE_TABLE_CHECK=y +CONFIG_PAGE_TABLE_CHECK_ENFORCED=y +# Guest-only infrastructure for late swap/reclaim fixtures. +CONFIG_BLOCK=y +CONFIG_VIRTIO=y +CONFIG_VIRTIO_MMIO=y +CONFIG_VIRTIO_BLK=y +CONFIG_DEVTMPFS=y +CONFIG_DEBUG_FS=y +CONFIG_KUNIT_DEBUGFS=y + +# Software MM fixtures only; the hardware boot opt-in is rejected under KASAN. +CONFIG_KASAN=y +CONFIG_KASAN_GENERIC=y +CONFIG_KASAN_INLINE=y diff --git a/tools/testing/selftests/arm64/Makefile b/tools/testing/selftests/arm64/Makefile index e456f3b62fa13..d8948f5fb1ef8 100644 --- a/tools/testing/selftests/arm64/Makefile +++ b/tools/testing/selftests/arm64/Makefile @@ -4,7 +4,7 @@ ARCH ?= $(shell uname -m 2>/dev/null || echo not) ifneq (,$(filter $(ARCH),aarch64 arm64)) -ARM64_SUBTARGETS ?= tags signal pauth fp mte bti abi gcs +ARM64_SUBTARGETS ?= tags signal pauth fp mte bti abi gcs user4k else ARM64_SUBTARGETS := endif diff --git a/tools/testing/selftests/arm64/user4k/.gitignore b/tools/testing/selftests/arm64/user4k/.gitignore new file mode 100644 index 0000000000000..2e6224b832f98 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/.gitignore @@ -0,0 +1,3 @@ +page-contract + +swap-fixture-init diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile new file mode 100644 index 0000000000000..cc89cc93293fe --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -0,0 +1,134 @@ +# SPDX-License-Identifier: GPL-2.0 +TEST_GEN_PROGS := page-contract proc-args +TEST_GEN_PROGS_EXTENDED := iommufd-init coarse-shmem-init bpf-uptr-init vhost-log-init uprobes-init user-events-init uffd-move-init swap-fixture-init exec-fixture-init mixed-process memory-interfaces advice reclaim-advice proc-pages proc-memory core-fixture-init context-stress-init mlock-fixture-init lazyfree-fixture-init thread-fixture-init splice-fixture-init dio-fixture-init uring-fixture-init gem-fixture-init compat-fixture-init granule-proc-init granule-swap-init aslr-fixture-init cpu-granule-init user16k-smoke-init cpu-granule16-init granule-threeway-init compat-mlock-init mglru-granule-init memcg-pressure-init memcg-accounting-init runtime-app-init fex-contract-init exec-policy-init uffd-wp-init shmem-uffd-swap-init compat-uffd-init shmem-hole-init shmem-thp-init shmem-collapse-init shmem-pin-init shmem-race-init shmem-pressure-init +TEST_GEN_PROGS_EXTENDED += vfio-fixture-init sysv-shm-init perf-ring-init +TEST_GEN_PROGS_EXTENDED += aio-ring-init numa-policy-init numa-balancing-init +CFLAGS += -Wall -Wextra -O2 -ffixed-x18 +TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe +TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe compat-vfio-probe +TEST_GEN_FILES += compat-shm-probe compat-perf-probe compat-numa-probe compat-numa-balancing-probe +TEST_FILES += compat-script runtime-app.py +include ../../lib.mk + +$(OUTPUT)/page-contract: page-contract.c x18-probe.S + +$(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init: LDLIBS += -pthread + +# Freestanding ELF32 probes exercise the compat ABI without an armhf sysroot. +AARCH32_CC ?= clang --target=arm-linux-gnueabihf +AARCH32_FLAGS := -march=armv8-a -marm -mfloat-abi=soft -ffreestanding \ + -fno-builtin -fno-stack-protector -O2 -nostdlib -static -fuse-ld=lld +$(OUTPUT)/compat-probe-4k: compat-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=4096 $< -o $@ +$(OUTPUT)/compat-probe-64k: compat-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ +$(OUTPUT)/compat-interp32: compat-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536,--image-base=0x10000000 $< -o $@ +$(OUTPUT)/compat-probe-dynamic: compat-probe.c + $(AARCH32_CC) $(filter-out -static,$(AARCH32_FLAGS)) -no-pie \ + -Wl,-z,max-page-size=65536,--dynamic-linker=/interp32 $< -o $@ + +$(OUTPUT)/compat-aslr-probe: compat-aslr-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/compat-mlock-probe: compat-mlock-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +# The disposable PID1 fixture also runs with the native 64K process ABI. +$(OUTPUT)/mglru-granule-init $(OUTPUT)/memcg-pressure-init $(OUTPUT)/memcg-accounting-init $(OUTPUT)/runtime-app-init $(OUTPUT)/fex-contract-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +# FEX fixture: the AArch64 emulator runs the freestanding x86-64 contract. +X86_64_CC ?= clang --target=x86_64-linux-gnu +$(OUTPUT)/fex-x86-contract: fex-x86-contract.c + $(X86_64_CC) -O2 -fno-builtin -fno-stack-protector -nostdlib -static \ + -fuse-ld=lld -Wl,-z,max-page-size=4096,-e,_start $< -o $@ + +$(OUTPUT)/libfex-abi.so: fex-native-abi.c + $(CC) $(CFLAGS) -shared -fPIC -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/exec-policy-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/exec-policy-init: LDLIBS += -pthread +$(OUTPUT)/compat-policy-probe: compat-policy-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +# Optional Windows payload; requires Wine/FEX in a disposable guest to execute. +# Keep its toolchain out of the ordinary Linux selftest build. +WINDOWS_CC ?= clang --target=x86_64-pc-windows-msvc +WINDOWS_DLLTOOL ?= llvm-dlltool +WINDOWS_LD ?= lld-link +.PHONY: wine-contract +wine-contract: $(OUTPUT)/wine-page-contract.exe +$(OUTPUT)/wine-kernel32.lib: wine-kernel32.def + $(WINDOWS_DLLTOOL) -m i386:x86-64 -d $< -l $@ +$(OUTPUT)/wine-page-contract.obj: wine-page-contract.c + $(WINDOWS_CC) -O2 -Wall -Wextra -Werror -ffreestanding -fno-stack-protector -c $< -o $@ +$(OUTPUT)/wine-page-contract.exe: $(OUTPUT)/wine-page-contract.obj $(OUTPUT)/wine-kernel32.lib + $(WINDOWS_LD) /entry:mainCRTStartup /subsystem:console /nodefaultlib /out:$@ $^ +EXTRA_CLEAN += $(OUTPUT)/wine-page-contract.exe $(OUTPUT)/wine-page-contract.obj $(OUTPUT)/wine-kernel32.lib + +$(OUTPUT)/uffd-wp-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/uffd-wp-init: LDLIBS += -pthread + +$(OUTPUT)/shmem-uffd-swap-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/compat-uffd-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-uffd-probe: compat-uffd-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/shmem-hole-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/shmem-hole-init: LDLIBS += -pthread + +$(OUTPUT)/shmem-thp-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/shmem-thp-init: LDLIBS += -pthread + +$(OUTPUT)/shmem-collapse-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/shmem-pin-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/shmem-pin-init: LDLIBS += -pthread + +$(OUTPUT)/shmem-race-init $(OUTPUT)/shmem-pressure-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/shmem-race-init: LDLIBS += -pthread + +$(OUTPUT)/uffd-move-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/uffd-move-init: LDLIBS += -pthread + +$(OUTPUT)/user-events-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/uprobes-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/vhost-log-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/bpf-uptr-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/coarse-shmem-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/iommufd-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/iommufd-init: LDLIBS += -pthread +$(OUTPUT)/iommufd-init: CFLAGS += $(KHDR_INCLUDES) + +# Privileged recording-domain fixture; run only in a disposable test VM. +$(OUTPUT)/vfio-fixture-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-vfio-probe: compat-vfio-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/sysv-shm-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-shm-probe: compat-shm-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +# Disposable guests exercise perf layout and software sampling across mm granules. +$(OUTPUT)/perf-ring-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-perf-probe: compat-perf-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/aio-ring-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +# Requires a disposable guest with two memory nodes and CONFIG_COMPAT. +$(OUTPUT)/numa-policy-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-numa-probe: compat-numa-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/numa-balancing-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/numa-balancing-init: LDLIBS += -pthread + +$(OUTPUT)/compat-numa-balancing-probe: compat-numa-balancing-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ diff --git a/tools/testing/selftests/arm64/user4k/advice.c b/tools/testing/selftests/arm64/user4k/advice.c new file mode 100644 index 0000000000000..4d4cb89350985 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/advice.c @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static size_t page; + +static bool wait_child(pid_t pid, int sig) +{ + int status = 0; + + if (pid <= 0 || waitpid(pid, &status, 0) != pid) + return false; + if (sig) + return WIFSIGNALED(status) && WTERMSIG(status) == sig; + return WIFEXITED(status) && !WEXITSTATUS(status); +} + +static void *anon(size_t length, int prot, int extra) +{ + void *p = mmap(NULL, length, prot, MAP_PRIVATE | MAP_ANONYMOUS | extra, + -1, 0); + + if (p == MAP_FAILED) + ksft_exit_fail_msg("anonymous mmap\n"); + return p; +} + +static void discard_and_guards(void) +{ + unsigned char *p = anon(3 * page, PROT_READ | PROT_WRITE, 0); + bool ok; + pid_t child; + + memset(p, 0x61, 3 * page); + ok = madvise(p + page, 1, MADV_DONTNEED) == 0; + ok &= p[0] == 0x61 && p[page] == 0 && p[2 * page] == 0x61; + ksft_test_result(ok, + "DONTNEED discards exactly one rounded user page\n"); + memset(p + page, 0x71, page); + child = fork(); + if (!child) { + ok = madvise(p + page, page, MADV_DONTNEED) == 0; + _exit(!(ok && p[page] == 0 && p[0] == 0x61)); + } + ksft_test_result( + wait_child(child, 0) && p[page] == 0x71, + "discard in a fork child leaves the parent quarter intact\n"); + ok = madvise(p + page, 1, MADV_GUARD_INSTALL) == 0; + ok &= madvise(p + page, page, MADV_DONTNEED) == 0; + child = fork(); + if (!child) { + volatile unsigned char value = p[page]; + + (void)value; + _exit(1); + } + ksft_test_result(ok && wait_child(child, SIGSEGV), + "a user-page guard survives discard and fork\n"); + ok = madvise(p + page, 1, MADV_GUARD_REMOVE) == 0; + ok &= p[page] == 0 && p[0] == 0x61 && p[2 * page] == 0x61; + ksft_test_result(ok, + "guard removal restores only the requested page\n"); + munmap(p, 3 * page); +} + +static void population(void) +{ + unsigned char *p = anon(5 * page, PROT_READ, 0), vec[5]; + bool ok; + unsigned int i; + + ok = mprotect(p + 2 * page, page, PROT_READ | PROT_WRITE) == 0; + ok &= madvise(p + 2 * page, 1, MADV_POPULATE_WRITE) == 0; + ok &= mincore(p, 5 * page, vec) == 0; + for (i = 0; i < 5; i++) + ok &= vec[i] == (i == 2); + ksft_test_result( + ok, "POPULATE_WRITE faults only the selected user-page VMA\n"); + ok = madvise(p + page, 1, MADV_POPULATE_READ) == 0; + ok &= mincore(p + page, page, vec) == 0 && vec[0] == 1; + errno = 0; + ok &= madvise(p, page, MADV_POPULATE_WRITE) == -1 && errno == EINVAL; + ok &= mprotect(p + 3 * page, page, PROT_NONE) == 0; + errno = 0; + ok &= madvise(p + 3 * page, page, MADV_POPULATE_READ) == -1 && + errno == EINVAL; + ok &= munmap(p + 3 * page, page) == 0; + errno = 0; + ok &= madvise(p + 3 * page, page, MADV_POPULATE_READ) == -1 && + errno == ENOMEM; + ksft_test_result( + ok, "POPULATE_READ honors permissions and reports holes\n"); + munmap(p, 5 * page); + p = anon(5 * page, PROT_READ | PROT_WRITE, MAP_POPULATE); + ok = mincore(p, 5 * page, vec) == 0; + for (i = 0; i < 5; i++) + ok &= vec[i] == 1; + ksft_test_result(ok, "MAP_POPULATE materializes every user page\n"); + munmap(p, 5 * page); +} + +static void fork_advice(void) +{ + unsigned char *p = anon(3 * page, PROT_READ | PROT_WRITE, 0); + bool ok; + pid_t child; + + memset(p, 0x58, 3 * page); + ok = madvise(p + page, 1, MADV_WIPEONFORK) == 0; + child = fork(); + if (!child) + _exit(!(p[page] == 0 && p[0] == 0x58 && p[2 * page] == 0x58)); + ok &= wait_child(child, 0) && p[page] == 0x58; + ksft_test_result( + ok, + "WIPEONFORK clears a single child page and preserves neighbors\n"); + ok = madvise(p + page, 1, MADV_KEEPONFORK) == 0; + ok &= madvise(p + page, 1, MADV_DONTFORK) == 0; + child = fork(); + if (!child) { + unsigned char vec; + + errno = 0; + _exit(!(mincore(p + page, page, &vec) == -1 && + errno == ENOMEM && p[0] == 0x58 && + p[2 * page] == 0x58)); + } + ok &= wait_child(child, 0); + ksft_test_result(ok, "DONTFORK omits only its selected user page\n"); + munmap(p, 3 * page); +} + +static void remove_file(void) +{ + int fd = memfd_create("advice-file", 0); + unsigned char *shared, *alias, *private; + bool ok; + size_t i; + + if (fd < 0 || ftruncate(fd, 5 * page)) + ksft_exit_fail_msg("memfd setup\n"); + shared = + mmap(NULL, 5 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + alias = mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + page); + private = mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, + page); + if (shared == MAP_FAILED || alias == MAP_FAILED || + private == MAP_FAILED) + ksft_exit_fail_msg("file mmap\n"); + for (i = 0; i < 5 * page; i++) + shared[i] = 0x40 + i / page; + private[page] = 0x79; + ok = madvise(alias + page, 1, MADV_REMOVE) == 0; + for (i = 0; i < 5 * page; i++) + ok &= shared[i] == (i / page == 2 ? 0 : 0x40 + i / page); + ok &= private[page] == 0x79; + ksft_test_result( + ok, + "REMOVE uses the file byte offset and preserves private COW\n"); + ok = madvise(alias, 3 * page, MADV_WILLNEED) == 0; + ksft_test_result( + ok, "WILLNEED accepts a file range with the process granule\n"); + munmap(shared, 5 * page); + munmap(alias, 3 * page); + munmap(private, 3 * page); + close(fd); +} + +int main(int argc, char **argv) +{ + const struct rlimit no_core = { 0, 0 }; + + page = getauxval(AT_PAGESZ); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + ksft_exit_fail_msg("unexpected process page ABI: %zu\n", page); + ksft_print_msg("process page size: %zu\n", page); + if (setrlimit(RLIMIT_CORE, &no_core)) + return 1; + ksft_print_header(); + ksft_set_plan(11); + discard_and_guards(); + population(); + fork_advice(); + remove_file(); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/aio-ring-init.c b/tools/testing/selftests/arm64/user4k/aio-ring-init.c new file mode 100644 index 0000000000000..74f34d16cac29 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/aio-ring-init.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable PID1: legacy AIO ring relocation with per-process page sizes. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct aio_ring_header { + unsigned int id, nr, head, tail; + unsigned int magic, compat_features, incompat_features, header_length; +}; + +static void check(int good, const char *expression, int line) +{ + if (!good) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expression, + errno); + exit(1); + } +} +#define CHECK(x) check(!!(x), #x, __LINE__) + +static int wait_child(pid_t child) +{ + int status; + pid_t got; + + if (child < 0) + return 1; + do { + got = waitpid(child, &status, 0); + } while (got < 0 && errno == EINTR); + return got != child || !WIFEXITED(status) || WEXITSTATUS(status); +} + +static void complete(aio_context_t context, struct iocb *request, + unsigned long bytes, int event_fd) +{ + struct io_event event; + struct timespec timeout = { .tv_sec = 5 }; + uint64_t notifications; + + CHECK(syscall(SYS_io_getevents, context, 1, 1, &event, &timeout) == 1); + CHECK(event.data == request->aio_data && + event.obj == (uintptr_t)request); + CHECK(event.res == (long long)bytes && event.res2 == 0); + CHECK(read(event_fd, ¬ifications, sizeof(notifications)) == + sizeof(notifications)); + CHECK(notifications == 1); +} + +static void exercise(unsigned long native, unsigned int requested) +{ + unsigned long page = getauxval(AT_PAGESZ), bytes = 2 * page + 37; + aio_context_t context = 0; + int file = memfd_create("aio-granule", 0); + int event_fd = eventfd(0, EFD_NONBLOCK); + unsigned char *source, *destination, *reserve, *target; + struct aio_ring_header *header; + unsigned long ring_size; + struct iocb request = { 0 }, *requests[] = { &request }; + unsigned char residency; + + CHECK(file >= 0 && event_fd >= 0); + CHECK(!ftruncate(file, 4 * page)); + source = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + destination = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(source != MAP_FAILED && destination != MAP_FAILED); + for (unsigned long i = 0; i < bytes; i++) + source[page - 19 + i] = i * 17 + i / page; + CHECK(!syscall(SYS_io_setup, requested, &context)); + header = (void *)(uintptr_t)context; + CHECK(header->magic == 0xa10a10a1 && + header->header_length == sizeof(*header)); + CHECK(header->nr >= requested); + ring_size = + header->header_length + header->nr * sizeof(struct io_event); + CHECK(ring_size && !(ring_size % native)); + reserve = mmap(NULL, ring_size + 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + target = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + target += native + (page < native ? page : 0); + target[-1] = 0x51; + target[ring_size] = 0x72; + CHECK(!munmap(target, ring_size)); + request.aio_data = 0x12345678; + request.aio_lio_opcode = IOCB_CMD_PWRITE; + request.aio_fildes = file; + request.aio_buf = (uintptr_t)(source + page - 19); + request.aio_nbytes = bytes; + request.aio_offset = 7; + request.aio_flags = IOCB_FLAG_RESFD; + request.aio_resfd = event_fd; + CHECK(syscall(SYS_io_submit, context, 1, requests) == 1); + /* Retain a queued completion while relocating the ring. */ + CHECK(mremap(header, ring_size, ring_size, + MREMAP_MAYMOVE | MREMAP_FIXED, target) == target); + context = (uintptr_t)target; + header = (void *)target; + CHECK(header->magic == 0xa10a10a1); + CHECK(target[-1] == 0x51 && target[ring_size] == 0x72); + complete(context, &request, bytes, event_fd); + request.aio_data = 0x87654321; + request.aio_lio_opcode = IOCB_CMD_PREAD; + request.aio_buf = (uintptr_t)(destination + page - 11); + CHECK(syscall(SYS_io_submit, context, 1, requests) == 1); + complete(context, &request, bytes, event_fd); + CHECK(!memcmp(source + page - 19, destination + page - 11, bytes)); + CHECK(!syscall(SYS_io_destroy, context)); + errno = 0; + CHECK(mincore(target, page, &residency) == -1 && errno == ENOMEM); + CHECK(target[-1] == 0x51 && target[ring_size] == 0x72); + CHECK(!munmap(reserve, ring_size + 4 * native)); + CHECK(!munmap(source, 4 * page) && !munmap(destination, 4 * page)); + CHECK(!close(file) && !close(event_fd)); + printf("ok - AIO page=%lu requested=%u ring=%lu relocation, I/O, eventfd and teardown\n", + page, requested, ring_size); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--case")) { + native = strtoul(argv[2], NULL, 10); + CHECK(getauxval(AT_PAGESZ) == strtoul(argv[3], NULL, 10)); + exercise(native, 16); + exercise(native, 4096); + return 0; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + unsigned long sizes[] = { 4096, 16384, 65536 }; + + for (unsigned int i = 0; i < 3; i++) { + if (sizes[i] > native) + continue; + pid_t child = fork(); + + if (!child) { + char n[32], s[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", sizes[i]); + CHECK(!prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], + 0UL, 0UL, 0UL)); + execl("/init", "/init", "--case", n, s, NULL); + _exit(2); + } + int bad = wait_child(child); + + printf("%s - AIO native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, sizes[i]); + failed |= bad; + } + printf("AIO GRANULE %s\n", failed ? "FAIL" : "PASS"); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/aslr-fixture-init.c b/tools/testing/selftests/arm64/user4k/aslr-fixture-init.c new file mode 100644 index 0000000000000..4ac03136f2ec0 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/aslr-fixture-init.c @@ -0,0 +1,175 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define SAMPLES 64 +struct sample { + unsigned long page_size, stack, mapping, brk; +}; + +static int get_sample(struct sample *s) +{ + char buf[4096], *pos, *end; + unsigned long low, high; + FILE *f; + int fd; + ssize_t size; + void *p; + + s->page_size = getauxval(AT_PAGESZ); + p = mmap(NULL, s->page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + s->mapping = (unsigned long)p; + f = fopen("/proc/self/maps", "r"); + if (!f) + return 1; + while (fgets(buf, sizeof(buf), f)) + if (strstr(buf, "[stack]") && + sscanf(buf, "%lx-%lx", &low, &high) == 2) + s->stack = high; + fclose(f); + fd = open("/proc/self/stat", O_RDONLY); + if (fd < 0) + return 1; + size = read(fd, buf, sizeof(buf) - 1); + close(fd); + if (size <= 0) + return 1; + buf[size] = 0; + pos = strrchr(buf, ')'); + if (!pos) + return 1; + pos += 2; + /* Fields 1 and 2 precede the closing parenthesis; start_brk is 47. */ + for (unsigned int field = 3; field < 47; field++) { + pos = strchr(pos, ' '); + if (!pos) + return 1; + pos++; + } + errno = 0; + s->brk = strtoul(pos, &end, 10); + return errno || end == pos || !s->brk || !s->stack; +} + +static int collect(unsigned int mode, bool disabled, struct sample *s) +{ + int pipefd[2], status; + pid_t pid; + ssize_t size; + + if (pipe(pipefd)) + return 1; + pid = fork(); + if (!pid) { + char fd[32]; + close(pipefd[0]); + snprintf(fd, sizeof(fd), "%d", pipefd[1]); + if (prctl(SET_EXEC_PAGE_SIZE, mode == 1 ? 4096UL : 0UL, 0UL, + 0UL, 0UL)) + _exit(2); + if (disabled && personality(ADDR_NO_RANDOMIZE) < 0) + _exit(3); + execl(mode == 2 ? "/compat-aslr" : "/init", "/init", "--sample", + fd, NULL); + _exit(4); + } + close(pipefd[1]); + size = read(pipefd[0], s, sizeof(*s)); + close(pipefd[0]); + return pid < 0 || waitpid(pid, &status, 0) != pid || + !WIFEXITED(status) || WEXITSTATUS(status) || size != sizeof(*s); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0; + + if (argc == 3 && !strcmp(argv[1], "--sample")) { + struct sample s = { 0 }; + if (getppid() != 1 || get_sample(&s)) + return 1; + return write(atoi(argv[2]), &s, sizeof(s)) != sizeof(s); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Dedicated disposable VM PID1 only\n"); + return 2; + } + setvbuf(stdout, NULL, _IONBF, 0); + if (mount("proc", "/proc", "proc", 0, NULL)) + failed++; + int control = open("/proc/sys/kernel/randomize_va_space", O_WRONLY); + if (control < 0 || write(control, "2", 1) != 1) + failed++; + if (control >= 0) + close(control); + int has4k = !prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL); + if (prctl(SET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL)) + failed++; + for (unsigned int mode = 0; mode < 3; mode++) { + if (mode == 1 && !has4k) + continue; + unsigned long ps = mode && has4k ? 4096 : native; + struct sample first = { 0 }; + bool varied[3] = { 0 }, subnative[3] = { 0 }; + int ok = 1; + + for (unsigned int i = 0; i < SAMPLES; i++) { + struct sample s = { 0 }; + if (collect(mode, false, &s)) { + ok = 0; + break; + } + unsigned long address[] = { s.stack, s.mapping, s.brk }; + unsigned long original[] = { first.stack, first.mapping, + first.brk }; + ok &= s.page_size == ps; + for (unsigned int n = 0; n < 3; n++) { + ok &= !(address[n] & (ps - 1)); + varied[n] |= i && address[n] != original[n]; + subnative[n] |= i && + ((address[n] ^ original[n]) & + (native - 1)); + } + if (!i) + first = s; + } + for (unsigned int n = 0; n < 3; n++) + ok &= varied[n] && (ps == native || subnative[n]); + printf("%s - %s %luK ASLR: 64 execs, stack/mmap/brk varied=%d%d%d low-bits-varied=%d%d%d\n", + ok ? "ok" : "not ok", mode == 2 ? "AArch32" : "AArch64", + ps / 1024, varied[0], varied[1], varied[2], subnative[0], + subnative[1], subnative[2]); + failed += !ok; + struct sample a = { 0 }, b = { 0 }; + ok = !collect(mode, true, &a) && !collect(mode, true, &b) && + !memcmp(&a, &b, sizeof(a)) && a.page_size == ps; + printf("%s - %s %luK ADDR_NO_RANDOMIZE retains stable layout\n", + ok ? "ok" : "not ok", mode == 2 ? "AArch32" : "AArch64", + ps / 1024); + failed += !ok; + } + printf("GRANULE ASLR %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/bpf-uptr-init.c b/tools/testing/selftests/arm64/user4k/bpf-uptr-init.c new file mode 100644 index 0000000000000..33793cebc2b8c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/bpf-uptr-init.c @@ -0,0 +1,583 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) +struct user_data { + uint64_t a, b, result, marker; +}; +struct value { + struct user_data *p, *q; + uint64_t cookie; + uint8_t trailing; +} __attribute__((packed)); +static char verifier_log[65536]; +static int bpf(enum bpf_cmd cmd, union bpf_attr *a) +{ + size_t size; +#define END(field) (offsetof(union bpf_attr, field) + sizeof(a->field)) + switch (cmd) { + case BPF_BTF_LOAD: + size = END(btf_log_level); + break; + case BPF_MAP_CREATE: + size = END(btf_value_type_id); + break; + case BPF_PROG_LOAD: + size = END(log_buf); + break; + case BPF_RAW_TRACEPOINT_OPEN: + size = END(raw_tracepoint.prog_fd); + break; + case BPF_MAP_UPDATE_ELEM: + size = END(flags); + break; + case BPF_MAP_LOOKUP_ELEM: + size = END(flags); + break; + case BPF_MAP_DELETE_ELEM: + size = END(key); + break; + default: + errno = EINVAL; + return -1; + } +#undef END + return syscall(__NR_bpf, cmd, a, size); +} +struct btf_blob { + unsigned char types[1024]; + char strings[512]; + unsigned nt, ns; +}; +static uint32_t str(struct btf_blob *b, const char *s) +{ + unsigned at = b->ns; + size_t n = strlen(s) + 1; + memcpy(b->strings + b->ns, s, n); + b->ns += n; + return at; +} +static void push(struct btf_blob *b, const void *p, size_t n) +{ + memcpy(b->types + b->nt, p, n); + b->nt += n; +} +static void type(struct btf_blob *b, const char *name, unsigned kind, + unsigned n, unsigned size_type) +{ + struct btf_type t = { .name_off = name ? str(b, name) : 0, + .info = kind << 24 | n, + .size = size_type }; + push(b, &t, sizeof(t)); +} +static void member(struct btf_blob *b, const char *name, unsigned id, + unsigned offset) +{ + struct btf_member m = { .name_off = str(b, name), + .type = id, + .offset = offset * 8 }; + push(b, &m, sizeof(m)); +} +static int create_map(void) +{ + struct btf_blob b = { .ns = 1 }; + uint32_t encoding; + type(&b, "int", BTF_KIND_INT, 0, 4); + encoding = BTF_INT_SIGNED << 24 | 32; + push(&b, &encoding, 4); + type(&b, "u64", BTF_KIND_INT, 0, 8); + encoding = 64; + push(&b, &encoding, 4); + type(&b, "user_data", BTF_KIND_STRUCT, 4, sizeof(struct user_data)); + member(&b, "a", 2, 0); + member(&b, "b", 2, 8); + member(&b, "result", 2, 16); + member(&b, "marker", 2, 24); + type(&b, "uptr", BTF_KIND_TYPE_TAG, 0, 3); + type(&b, NULL, BTF_KIND_PTR, 0, 4); + type(&b, "value_type", BTF_KIND_STRUCT, 4, sizeof(struct value)); + member(&b, "p", 5, 0); + member(&b, "q", 5, 8); + member(&b, "cookie", 2, 16); + member(&b, "trailing", 7, 24); + type(&b, "u8", BTF_KIND_INT, 0, 1); + encoding = 8; + push(&b, &encoding, 4); + unsigned char data[2048]; + struct btf_header h = { .magic = BTF_MAGIC, + .version = BTF_VERSION, + .hdr_len = sizeof(h), + .type_len = b.nt, + .str_off = b.nt, + .str_len = b.ns }; + memcpy(data, &h, sizeof(h)); + memcpy(data + sizeof(h), b.types, b.nt); + memcpy(data + sizeof(h) + b.nt, b.strings, b.ns); + union bpf_attr a = { .btf = (uintptr_t)data, + .btf_size = sizeof(h) + b.nt + b.ns, + .btf_log_buf = (uintptr_t)verifier_log, + .btf_log_size = sizeof(verifier_log), + .btf_log_level = 1 }; + int btf = bpf(BPF_BTF_LOAD, &a); + if (btf < 0) { + printf("BTF load errno=%d: %s\n", errno, verifier_log); + return -1; + } + a = (union bpf_attr){ .map_type = BPF_MAP_TYPE_TASK_STORAGE, + .key_size = sizeof(int), + .value_size = sizeof(struct value), + .map_flags = BPF_F_NO_PREALLOC, + .btf_fd = btf, + .btf_key_type_id = 1, + .btf_value_type_id = 6 }; + int fd = bpf(BPF_MAP_CREATE, &a); + close(btf); + return fd; +} +#define I(c, d, s, o, i) \ + ((struct bpf_insn){ .code = (c), \ + .dst_reg = (d), \ + .src_reg = (s), \ + .off = (o), \ + .imm = (i) }) +static int create_prog(int map, int *link) +{ + struct bpf_insn insns[40]; + unsigned n = 0, jumps[4]; +#define EMIT(c, d, s, o, i) (insns[n++] = I(c, d, s, o, i)) + EMIT(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_current_task_btf); + EMIT(BPF_ALU64 | BPF_MOV | BPF_X, 2, 0, 0, 0); + EMIT(BPF_LD | BPF_DW | BPF_IMM, 1, BPF_PSEUDO_MAP_FD, 0, map); + EMIT(0, 0, 0, 0, 0); + EMIT(BPF_ALU64 | BPF_MOV | BPF_K, 3, 0, 0, 0); + EMIT(BPF_ALU64 | BPF_MOV | BPF_K, 4, 0, 0, 0); + EMIT(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_task_storage_get); + jumps[0] = n; + EMIT(BPF_JMP | BPF_JEQ | BPF_K, 0, 0, 0, 0); + EMIT(BPF_ALU64 | BPF_MOV | BPF_X, 6, 0, 0, 0); + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 7, 6, 16, 0); + jumps[1] = n; + EMIT(BPF_JMP | BPF_JEQ | BPF_K, 7, 0, 0, 0); + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 1, 6, 0, 0); + jumps[2] = n; + EMIT(BPF_JMP | BPF_JEQ | BPF_K, 1, 0, 0, 0); + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 2, 1, 0, 0); + EMIT(BPF_ALU64 | BPF_XOR | BPF_X, 2, 7, 0, 0); + EMIT(BPF_STX | BPF_DW | BPF_MEM, 1, 2, 16, 0); + EMIT(BPF_STX | BPF_DW | BPF_MEM, 1, 7, 24, 0); + insns[jumps[2]].off = n - jumps[2] - 1; + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 1, 6, 8, 0); + jumps[3] = n; + EMIT(BPF_JMP | BPF_JEQ | BPF_K, 1, 0, 0, 0); + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 2, 1, 8, 0); + EMIT(BPF_ALU64 | BPF_ADD | BPF_X, 2, 7, 0, 0); + EMIT(BPF_STX | BPF_DW | BPF_MEM, 1, 2, 16, 0); + EMIT(BPF_STX | BPF_DW | BPF_MEM, 1, 7, 24, 0); + for (unsigned i = 0; i < 4; i++) + if (i != 2) + insns[jumps[i]].off = n - jumps[i] - 1; + EMIT(BPF_ALU64 | BPF_MOV | BPF_K, 0, 0, 0, 0); + EMIT(BPF_JMP | BPF_EXIT, 0, 0, 0, 0); +#undef EMIT + union bpf_attr a = { .prog_type = BPF_PROG_TYPE_RAW_TRACEPOINT, + .insn_cnt = n, + .insns = (uintptr_t)insns, + .license = (uintptr_t)"GPL", + .log_buf = (uintptr_t)verifier_log, + .log_size = sizeof(verifier_log), + .log_level = 1 }; + verifier_log[0] = 0; + int fd = bpf(BPF_PROG_LOAD, &a); + if (fd < 0) { + printf("prog load errno=%d: %s\n", errno, verifier_log); + return -1; + } + a = (union bpf_attr){}; + a.raw_tracepoint.name = (uintptr_t)"sys_enter"; + a.raw_tracepoint.prog_fd = fd; + *link = bpf(BPF_RAW_TRACEPOINT_OPEN, &a); + if (*link < 0) { + close(fd); + return -1; + } + return fd; +} +static int update(int map, int key, struct value *v, unsigned flags) +{ + union bpf_attr a = { .map_fd = map, + .key = (uintptr_t)&key, + .value = (uintptr_t)v, + .flags = flags }; + return bpf(BPF_MAP_UPDATE_ELEM, &a); +} +static int delete(int map, int key) +{ + union bpf_attr a = { .map_fd = map, .key = (uintptr_t)&key }; + return bpf(BPF_MAP_DELETE_ELEM, &a); +} +struct pins { + unsigned long long acquired, released; +}; +static int pin_stats(struct pins *p) +{ + int fd = open("/proc/sys/vm/stat_refresh", O_RDONLY); + char c; + if (fd < 0) + return -1; + int ret = read(fd, &c, 1); + close(fd); + if (ret < 0) + return -1; + FILE *f = fopen("/proc/vmstat", "r"); + if (!f) + return -1; + char name[128]; + unsigned long long value; + unsigned found = 0; + while (fscanf(f, "%127s %llu", name, &value) == 2) { + if (!strcmp(name, "nr_foll_pin_acquired")) { + p->acquired = value; + found |= 1; + } + if (!strcmp(name, "nr_foll_pin_released")) { + p->released = value; + found |= 2; + } + } + fclose(f); + return found == 3 ? 0 : -1; +} +static int wait_pins(struct pins base, unsigned active) +{ + struct pins p = {}; + for (unsigned i = 0; i < 500; i++) { + if (pin_stats(&p)) + return 0; + if (p.acquired - base.acquired >= active && + p.acquired - base.acquired - (p.released - base.released) == + active) + return 1; + usleep(10000); + } + printf("pin delta acquired=%llu released=%llu expected-active=%u\n", + p.acquired - base.acquired, p.released - base.released, active); + return 0; +} +static int filled(unsigned char *p, size_t len, unsigned char value) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != value) + return 0; + return 1; +} +static int values(struct user_data *p, struct user_data *q, uint64_t cookie) +{ + return p->result == (p->a ^ cookie) && q->result == q->b + cookie && + p->marker == cookie && q->marker == cookie; +} +static int exercise_case(size_t ps, unsigned shared) +{ + struct pins base; + CHECK(!pin_stats(&base)); + int backing = -1; + if (shared) { + backing = memfd_create("bpf-uptr", 0); + CHECK(backing >= 0 && !ftruncate(backing, 4 * ps)); + } + unsigned char *src = mmap( + (void *)0x20000000, 4 * ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | + (shared ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS), + backing, 0); + unsigned char *dst = + mmap((void *)0x30000000, 4 * ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(src != MAP_FAILED && dst != MAP_FAILED); + for (unsigned i = 0; i < 4; i++) + memset(src + i * ps, 0x61 + i, ps); + memset(dst, 0x91, 4 * ps); + CHECK(mremap(src + 3 * ps, ps, ps, MREMAP_MAYMOVE | MREMAP_FIXED, + dst + ps) == dst + ps); + unsigned char *leaf = dst + ps; + struct user_data *p = (void *)(leaf + ps - 64), *q = p + 1; + *p = (struct user_data){ .a = 0x12345678, .b = 3 }; + *q = (struct user_data){ .a = 4, .b = 0x87654321 }; + int map = create_map(); + CHECK(map >= 0); + int link = -1, prog = create_prog(map, &link); + CHECK(prog >= 0 && link >= 0); + int key = syscall(SYS_pidfd_open, getpid(), 0); + CHECK(key >= 0); + struct value probe = {}; + int probe_ret = update(map, key, &probe, BPF_NOEXIST); + CHECK(!probe_ret); + CHECK(!delete(map, key)); + struct value v = { + .p = p, .q = q, .cookie = 0xabc001, .trailing = 0x7d + }; + CHECK(!update(map, key, &v, BPF_NOEXIST)); + CHECK(wait_pins(base, 2)); + syscall(SYS_getpid); + CHECK(values(p, q, v.cookie)); + for (unsigned i = 0; i < 64; i++) { + v.cookie++; + CHECK(!update(map, key, &v, BPF_EXIST)); + syscall(SYS_getpid); + CHECK(values(p, q, v.cookie)); + } + CHECK(wait_pins(base, 2)); + struct { + struct value v; + unsigned char tail[16]; + } lookup; + memset(&lookup, 0xa5, sizeof(lookup)); + union bpf_attr a = { .map_fd = map, + .key = (uintptr_t)&key, + .value = (uintptr_t)&lookup }; + CHECK(!bpf(BPF_MAP_LOOKUP_ELEM, &a)); + CHECK(!lookup.v.p && !lookup.v.q && lookup.v.cookie == v.cookie && + lookup.v.trailing == v.trailing && + filled(lookup.tail, sizeof(lookup.tail), 0xa5)); + errno = 0; + CHECK(update(map, key, &v, BPF_NOEXIST) == -1 && errno == EEXIST); + CHECK(wait_pins(base, 2)); + struct value half = v; + half.p = NULL; + CHECK(!update(map, key, &half, BPF_EXIST)); + CHECK(wait_pins(base, 1)); + p->result = p->marker = 0; + syscall(SYS_getpid); + CHECK(!p->result && !p->marker && q->result == q->b + v.cookie); + CHECK(!update(map, key, &v, BPF_EXIST)); + CHECK(wait_pins(base, 2)); + struct value bad = v; + bad.q = (void *)5; + errno = 0; + CHECK(update(map, key, &bad, BPF_EXIST) == -1 && errno == EFAULT); + CHECK(wait_pins(base, 2)); + syscall(SYS_getpid); + CHECK(values(p, q, v.cookie)); + bad.q = (void *)(leaf + ps - 16); + errno = 0; + CHECK(update(map, key, &bad, BPF_EXIST) == -1 && errno == EOPNOTSUPP); + CHECK(wait_pins(base, 2)); + bad.q = (void *)(UINTPTR_MAX - 15); + errno = 0; + CHECK(update(map, key, &bad, BPF_EXIST) == -1 && errno == EFAULT); + CHECK(wait_pins(base, 2)); + CHECK(!mprotect(leaf, ps, PROT_READ)); + errno = 0; + CHECK(update(map, key, &v, BPF_EXIST) == -1 && errno == EFAULT); + CHECK(!mprotect(leaf, ps, PROT_READ | PROT_WRITE)); + CHECK(wait_pins(base, 2)); + struct value empty = {}; + CHECK(!update(map, key, &empty, BPF_EXIST)); + CHECK(wait_pins(base, 0)); + p->result = q->result = p->marker = q->marker = 0; + syscall(SYS_getpid); + CHECK(!p->result && !q->result); + CHECK(!update(map, key, &v, BPF_EXIST)); + CHECK(wait_pins(base, 2)); + /* The child owns storage pointing into the parent's pinned memory. */ + int barrier[2]; + CHECK(!pipe(barrier)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + close(barrier[1]); + char token; + if (read(barrier[0], &token, 1) != 1) + _exit(3); + if (!shared) { + p->a = 1; + q->b = 2; + } + syscall(SYS_getpid); + _exit(0); + } + close(barrier[0]); + int childkey = syscall(SYS_pidfd_open, child, 0); + CHECK(childkey >= 0); + CHECK(!update(map, childkey, &v, BPF_NOEXIST)); + CHECK(!delete(map, key)); + CHECK(wait_pins(base, 2)); + p->result = q->result = p->marker = q->marker = 0; + CHECK(write(barrier[1], "x", 1) == 1); + close(barrier[1]); + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status)); + close(childkey); + CHECK(values(p, q, v.cookie) && p->a == 0x12345678 && + q->b == 0x87654321); + CHECK(wait_pins(base, 0)); + /* The supplying mm exits while a different task retains its UPTRs. */ + int foreign_pipe[2]; + CHECK(!pipe(foreign_pipe)); + pid_t supplier = fork(); + CHECK(supplier >= 0); + if (!supplier) { + close(foreign_pipe[0]); + *p = (struct user_data){ .a = 0x1111, .b = 3 }; + *q = (struct user_data){ .a = 4, .b = 0x2222 }; + struct value foreign = { .p = p, .q = q, .cookie = 0x765432 }; + if (update(map, key, &foreign, BPF_NOEXIST)) + _exit(4); + struct iovec data = { .iov_base = p, .iov_len = 64 }; + if (vmsplice(foreign_pipe[1], &data, 1, 0) != 64) + _exit(5); + _exit(0); + } + close(foreign_pipe[1]); + CHECK(waitpid(supplier, &status, 0) == supplier && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(wait_pins(base, 2)); + syscall(SYS_getpid); + struct user_data retired[2]; + size_t done = 0; + while (done < sizeof(retired)) { + ssize_t got = read(foreign_pipe[0], (char *)retired + done, + sizeof(retired) - done); + CHECK(got > 0); + done += got; + } + close(foreign_pipe[0]); + CHECK(retired[0].a == 0x1111 && retired[1].b == 0x2222 && + values(retired, retired + 1, 0x765432)); + CHECK(!delete(map, key) && wait_pins(base, 0)); + *p = (struct user_data){ .a = 0x12345678, .b = 3 }; + *q = (struct user_data){ .a = 4, .b = 0x87654321 }; + CHECK(!update(map, key, &v, BPF_NOEXIST)); + CHECK(wait_pins(base, 2)); + CHECK(filled(leaf, ps - 64, 0x64)); + int pipefd[2]; + CHECK(!pipe(pipefd)); + struct iovec iov = { .iov_base = leaf, .iov_len = ps }; + CHECK(vmsplice(pipefd[1], &iov, 1, 0) == (ssize_t)ps); + CHECK(!munmap(leaf, ps)); + CHECK(mmap(leaf, ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, + 0) == leaf); + memset(leaf, 0xe7, ps); + unsigned char *old = malloc(ps); + CHECK(old); + size_t n = 0; + while (n < ps) { + ssize_t got = read(pipefd[0], old + n, ps - n); + CHECK(got > 0); + n += got; + } + close(pipefd[0]); + close(pipefd[1]); + CHECK(filled(old, ps - 64, 0x64) && + values((void *)(old + ps - 64), (void *)(old + ps - 32), + v.cookie)); + CHECK(filled(leaf, ps, 0xe7)); + free(old); + /* Closing the map alone is insufficient: the program retains its map ref. */ + close(map); + close(link); + close(prog); + close(key); + CHECK(wait_pins(base, 0)); + CHECK(filled(dst, ps, 0x91) && filled(dst + 2 * ps, 2 * ps, 0x91)); + for (unsigned i = 0; i < 3; i++) + CHECK(filled(src + i * ps, ps, 0x61 + i)); + CHECK(!munmap(src, 3 * ps) && !munmap(dst, 4 * ps)); + if (backing >= 0) + close(backing); + printf("ok - %zuK BPF UPTR %s: relocated writes, updates, rollback, storage/supplier exit, unmap and pin balance\n", + ps / 1024, shared ? "shared" : "private"); + return 0; +} +static int exercise(size_t ps) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(90); + for (unsigned shared = 0; shared < 2; shared++) + CHECK(!exercise_case(ps, shared)); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + size_t sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned i = 0; i < (sizes[0] > 16384 ? 3U : 2U); i++) { + pid_t c = fork(); + int status = 0; + if (!c) { + char size[24]; + snprintf(size, sizeof(size), "%zu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", size, NULL); + _exit(5); + } + int ok = 0; + for (unsigned ticks = 0; c > 0 && ticks < 1000; ticks++) { + pid_t got = waitpid(c, &status, WNOHANG); + if (got == c) { + ok = WIFEXITED(status) && !WEXITSTATUS(status); + break; + } + if (ticks == 999) { + kill(c, SIGKILL); + waitpid(c, &status, 0); + break; + } + usleep(100000); + } + printf("%s - ABI %zuK BPF UPTR status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + fail += !ok; + } +done: + printf("BPF UPTR %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/coarse-shmem-init.c b/tools/testing/selftests/arm64/user4k/coarse-shmem-init.c new file mode 100644 index 0000000000000..2a00ca546751e --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/coarse-shmem-init.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable PID1 runner; fault injection is selected by the KUnit worker. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "swapfile-fixture.h" + +static int setting(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY | O_CLOEXEC); + ssize_t n; + + if (fd < 0) + return -1; + n = write(fd, value, strlen(value)); + close(fd); + return n == (ssize_t)strlen(value) ? 0 : -1; +} + +static int fault_setting(const char *kind, const char *name, const char *value) +{ + char path[192]; + + snprintf(path, sizeof(path), "/sys/kernel/debug/%s/%s", kind, name); + return setting(path, value); +} + +static int injection_off(void) +{ + int ret = fault_setting("failslab", "probability", "0"); + + ret |= fault_setting("fail_page_alloc", "probability", "0"); + ret |= fault_setting("failslab", "ignore-gfp-wait", "Y"); + ret |= fault_setting("fail_page_alloc", "ignore-gfp-wait", "Y"); + ret |= fault_setting("fail_page_alloc", "ignore-gfp-highmem", "Y"); + return ret; +} + +static int injection_on(bool page) +{ + const char *kind = page ? "fail_page_alloc" : "failslab"; + + if (injection_off() || fault_setting(kind, "task-filter", "Y") || + fault_setting(kind, "verbose", "0") || + fault_setting(kind, "ignore-gfp-wait", "N") || + fault_setting(kind, "times", "-1") || + fault_setting(kind, "interval", "1") || + fault_setting(kind, "space", "0")) + return -1; + if (!page) + return 0; /* Only current->fail_nth selects the slab allocation. */ + return fault_setting(kind, "ignore-gfp-highmem", "N") || + fault_setting(kind, "min-order", "2") || + fault_setting(kind, "probability", "100"); +} + +static int move_worker(const char *text, bool enter, bool swap_limit) +{ + char *end, path[192], pid[32], comm[64]; + long nr = strtol(text, &end, 10); + const char *group = swap_limit ? "coarse-swap-limit" : "coarse-zero"; + int fd; + ssize_t n; + + if (*end || nr <= 1 || nr == getpid()) + return -1; + snprintf(path, sizeof(path), "/proc/%ld/comm", nr); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + n = read(fd, comm, sizeof(comm) - 1); + close(fd); + if (n < 1) + return -1; + comm[n] = 0; + if (strncmp(comm, "kunit_", 6)) + return -1; + if (enter) { + if (mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL) && + errno != EBUSY) + return -1; + if (setting("/sys/fs/cgroup/cgroup.subtree_control", "+memory")) + return -1; + snprintf(path, sizeof(path), "/sys/fs/cgroup/%s", group); + if (mkdir(path, 0755) && errno != EEXIST) + return -1; + snprintf(path, sizeof(path), "/sys/fs/cgroup/%s/memory.max", + group); + if (setting(path, swap_limit ? "max" : "0")) + return -1; + if (swap_limit && + setting("/sys/fs/cgroup/coarse-swap-limit/memory.swap.max", + "0")) + return -1; + } + snprintf(pid, sizeof(pid), "%ld", nr); + if (enter) + snprintf(path, sizeof(path), "/sys/fs/cgroup/%s/cgroup.procs", + group); + else + snprintf(path, sizeof(path), "/sys/fs/cgroup/cgroup.procs"); + return setting(path, pid); +} + +static int helper(const char *op) +{ + struct stat init, self; + + if (getpid() == 1 || stat("/proc/1/exe", &init) || + stat("/proc/self/exe", &self) || init.st_dev != self.st_dev || + init.st_ino != self.st_ino) + return 1; + if (!strcmp(op, "--coarse-fixture-check")) + return getauxval(AT_PAGESZ) != 16384; + if (!strcmp(op, "--swapoff-helper")) + return swapoff(swap_path()) != 0; + if (!strcmp(op, "--swapon-helper")) + return swapon(swap_path(), 0) != 0; + if (!strncmp(op, "--coarse-swap-enter=", 20)) + return move_worker(op + 20, true, true) != 0; + if (!strncmp(op, "--coarse-swap-leave=", 20)) + return move_worker(op + 20, false, true) != 0; + if (!strcmp(op, "--coarse-swap-unlimit")) + return setting("/sys/fs/cgroup/coarse-swap-limit/memory.swap.max", + "max") != 0; + if (!strcmp(op, "--coarse-injection-off")) + return injection_off() != 0; + if (!strcmp(op, "--coarse-slab-on")) + return injection_on(false) != 0; + if (!strcmp(op, "--coarse-page-on")) + return injection_on(true) != 0; + if (!strncmp(op, "--coarse-memcg-enter=", 21)) + return move_worker(op + 21, true, false) != 0; + if (!strncmp(op, "--coarse-memcg-leave=", 21)) + return move_worker(op + 21, false, false) != 0; + return 1; +} + +static int run_suite(void) +{ + const char *base = "/sys/kernel/debug/kunit/arm64-user4k-coarse-shmem/"; + char path[192], *log = calloc(1, 1024 * 1024); + size_t used = 0, capacity = 1024 * 1024; + ssize_t n = 0; + int fd, ret = -1; + + if (!log) + return -1; + snprintf(path, sizeof(path), "%srun", base); + if (setting(path, "1")) + goto out; + snprintf(path, sizeof(path), "%sresults", base); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + goto out; + while (used < capacity - 1 && + (n = read(fd, log + used, capacity - used - 1)) > 0) + used += n; + close(fd); + fputs(log, stdout); + if (!n && used < capacity - 1 && + strstr(log, "\nok 1 arm64-user4k-coarse-shmem\n") && + !strstr(log, "not ok") && !strstr(log, "# SKIP")) + ret = 0; +out: + free(log); + return ret; +} + +int main(int argc, char **argv) +{ + struct stat st; + bool active = false; + int ret = 1; + unsigned int i; + + if (getpid() != 1 && argc == 2) + return helper(argv[1]); + if (getpid() != 1 || + (argc != 1 && (argc != 2 || strcmp(argv[1], "--swapfile")))) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + if (getauxval(AT_PAGESZ) != 16384) + goto out; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) + goto out; + for (i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + if (stat("/dev/vda", &st) || !S_ISBLK(st.st_mode)) + goto out; + if (argc == 2 && prepare_swapfile()) + goto out; + printf("COARSE SWAP BACKING=%s\n", swap_path()); + if (swapon(swap_path(), 0)) + goto out; + active = true; + ret = run_suite() ? 1 : 0; + if (injection_off()) + ret = 1; +out: + if (active && swapoff(swap_path())) + ret = 1; + if (active && argc == 2 && umount("/swapdir")) + ret = 1; + printf("COARSE SHMEM SWAP PASS=%d errno=%d\n", !ret, errno); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/compat-aslr-probe.c b/tools/testing/selftests/arm64/user4k/compat-aslr-probe.c new file mode 100644 index 0000000000000..3a88db7511215 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-aslr-probe.c @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Freestanding ELF32 address sampler for the disposable ASLR fixture. */ +typedef unsigned int u32; +typedef unsigned long long u64; +struct sample { + u64 page_size, stack, mapping, brk; +}; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} +static int error(long value) +{ + return (u32)value >= 0xfffff001U; +} +static unsigned number(const char *p) +{ + unsigned n = 0; + while (*p >= '0' && *p <= '9') + n = n * 10 + *p++ - '0'; + return n; +} +static char *find(char *p, char c) +{ + while (*p && *p != c) + p++; + return *p ? p : (char *)0; +} +static char *find_stack(char *p) +{ + const char name[] = "[stack]"; + while (*p) { + unsigned i = 0; + while (name[i] && p[i] == name[i]) + i++; + if (!name[i]) + return p; + p++; + } + return (char *)0; +} +static unsigned hex(char *p) +{ + unsigned n = 0, v; + for (;;) { + if (*p >= '0' && *p <= '9') + v = *p - '0'; + else if (*p >= 'a' && *p <= 'f') + v = *p - 'a' + 10; + else + return n; + n = (n << 4) | v; + p++; + } +} +static int load(const char *path, char *buf, unsigned size) +{ + long fd = call(5, (long)path, 0, 0, 0, 0, 0), n; + if (error(fd)) + return 1; + n = call(3, fd, (long)buf, size - 1, 0, 0, 0); + call(6, fd, 0, 0, 0, 0, 0); + if (n <= 0) + return 1; + buf[n] = 0; + return 0; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + unsigned argc = *sp++; + char **argv = (void *)sp, buf[4096], *p, *q; + struct sample sample = { 0 }; + long mapping; + int bad = 1; + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + sample.page_size = sp[1]; + sp += 2; + } + if (argc != 3 || call(64, 0, 0, 0, 0, 0, 0) != 1) + goto done; + mapping = call(192, 0, sample.page_size, 3, 0x22, -1, 0); + if (error(mapping)) + goto done; + sample.mapping = (u32)mapping; + if (load("/proc/self/maps", buf, sizeof(buf))) + goto done; + p = find_stack(buf); + if (!p) + goto done; + while (p > buf && p[-1] != '\n') + p--; + p = find(p, '-'); + if (!p) + goto done; + sample.stack = hex(p + 1); + if (load("/proc/self/stat", buf, sizeof(buf))) + goto done; + p = find(buf, ')'); + if (!p) + goto done; + while ((q = find(p + 1, ')'))) + p = q; + p += 2; + for (unsigned field = 3; field < 47; field++) { + p = find(p, ' '); + if (!p) + goto done; + p++; + } + sample.brk = number(p); + if (!sample.page_size || !sample.stack || !sample.brk) + goto done; + bad = call(4, number(argv[2]), (long)&sample, sizeof(sample), 0, 0, + 0) != sizeof(sample); +done: + call(1, bad, 0, 0, 0, 0, 0); + for (;;) { + } +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-fixture-init.c b/tools/testing/selftests/arm64/user4k/compat-fixture-init.c new file mode 100644 index 0000000000000..f576ed93623ed --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-fixture-init.c @@ -0,0 +1,149 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +static int failed; +static unsigned long native_size; +static void result(int ok, const char *s) +{ + printf("%s - launcher %s\n", ok ? "ok" : "not ok", s); + if (!ok) + failed++; +} +static int launch(const char *path, char **args, int request, int expect_errno) +{ + pid_t child = fork(); + int status = 0; + if (!child) { + if (request && prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL)) + _exit(95); + unsigned char *canary = + mmap(NULL, native_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (canary == MAP_FAILED) + _exit(94); + memset(canary, 0x69, native_size); + int before = prctl(0x41555002, 0UL, 0UL, 0UL, 0UL); + for (int i = 0; i < (expect_errno ? 64 : 1); i++) { + execv(path, args); + if (!expect_errno || errno != expect_errno) + _exit(96); + if (getauxval(AT_PAGESZ) != native_size || + prctl(0x41555002, 0UL, 0UL, 0UL, 0UL) != before) + _exit(93); + for (int j = 0; j < native_size; j++) + if (canary[j] != 0x69) + _exit(92); + } + _exit(0); + } + if (child < 0 || waitpid(child, &status, 0) != child) + return 0; + if (status) + printf("# child %s status=%#x\n", path, status); + return WIFEXITED(status) && !WEXITSTATUS(status); +} +int main(int argc, char **argv) +{ + if (argc > 1 && !strcmp(argv[1], "--returned")) + return getauxval(AT_PAGESZ) == + strtoul(getenv("AURORA_NATIVE_PAGE_SIZE"), + NULL, 10) ? + 0 : + 1; + if (argc > 1 && !strcmp(argv[1], "--returned4k")) + return getauxval(AT_PAGESZ) == 4096 ? 0 : 1; + if (getpid() != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + native_size = getauxval(AT_PAGESZ); + char native_text[24]; + snprintf(native_text, sizeof(native_text), "%lu", native_size); + setenv("AURORA_NATIVE_PAGE_SIZE", native_text, 1); + setvbuf(stdout, NULL, _IONBF, 0); + mount("proc", "/proc", "proc", 0, NULL); + int no4k = argc > 1 && !strcmp(argv[1], "--no4k"), + fd = memfd_create("compat-offsets", 0); + if (no4k) { + errno = 0; + result(prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL) == -1 && + errno == EOPNOTSUPP, + "4KB capability is actually masked"); + } + result(fd >= 0 && !ftruncate(fd, 0x100000000LL + 4 * native_size), + "sparse file fixture above 4GB"); + unsigned char page[4096]; + for (unsigned q = 0; q < 4 * native_size / 4096; q++) { + memset(page, 0x40 + q, sizeof(page)); + if (pwrite(fd, page, sizeof(page), q * 4096) != sizeof(page)) + failed++; + memset(page, 0x80 + q, sizeof(page)); + if (pwrite(fd, page, sizeof(page), 0x100000000LL + q * 4096) != + sizeof(page)) + failed++; + } + char fdstr[24]; + snprintf(fdstr, sizeof(fdstr), "%d", fd); + char *arg = malloc(90001); + for (unsigned i = 0; i < 90000; i++) + arg[i] = 'a' + i % 23; + arg[90000] = 0; + char *args[] = { "/probe32-64k", no4k ? native_text : "4096", fdstr, + arg, NULL }; + result(launch(args[0], args, 0, 0), + "64KB-aligned ELF32 automatically selects available ABI"); + if (!no4k) { + args[0] = "/probe32-4k"; + result(launch(args[0], args, 0, 0), + "4KB-aligned ELF32 executes with 4KB translation"); + args[0] = "/probe32-64k"; + result(launch(args[0], args, 1, 0), + "explicit pending 4KB request allows ELF32"); + } + args[0] = "/script32"; + result(launch(args[0], args, 0, 0), + "script final ELF class determines page granule"); + args[0] = "/probe32-dynamic"; + result(launch(args[0], args, 0, 0), + "ELF32 interpreter mapping uses selected page granule"); + char *chain[] = { + "/probe32-64k", "--exec32", no4k ? native_text : "4096", + fdstr, arg, NULL + }; + result(launch(chain[0], chain, 0, 0), + "ELF32 to ELF32 restages arguments and selects ABI"); + char *back[] = { "/probe32-64k", "--exec64", NULL }; + result(launch(back[0], back, 0, 0), + "ELF32 to ELF64 exec restores native default"); + if (!no4k) { + back[1] = "--exec64-4k"; + result(launch(back[0], back, 0, 0), + "ELF32 can request alternative ELF64 exec granule"); + } + char *big = malloc(200001); + memset(big, 'x', 200000); + big[200000] = 0; + args[0] = "/probe32-64k"; + args[3] = big; + if (!no4k) + result(launch(args[0], args, 0, E2BIG), + "64 oversized transfers return E2BIG and preserve old mm and request"); + printf("COMPAT FIXTURE %s mode=%s\n", failed ? "FAIL" : "PASS", + no4k ? "no4k-native-fallback" : "auto4k"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/compat-mlock-init.c b/tools/testing/selftests/arm64/user4k/compat-mlock-init.c new file mode 100644 index 0000000000000..679c39370cd6d --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-mlock-init.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +struct result { + uint32_t pagesize, passed, failed, error; +}; +int main(int argc, char **argv) +{ + struct result out = { 0 }; + unsigned long expected; + int pipes[2], status = -1; + pid_t pid; + size_t done = 0; + int ok; + + if (getpid() != 1 || + (argc != 1 && (argc != 2 || strcmp(argv[1], "--no4k")))) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + expected = argc == 2 ? getauxval(AT_PAGESZ) : 4096; + if (mount("proc", "/proc", "proc", 0, NULL) || pipe(pipes)) + return 3; + pid = fork(); + if (!pid) { + char value[32]; + close(pipes[0]); + if (dup2(pipes[1], 1) < 0) + _exit(4); + close(pipes[1]); + snprintf(value, sizeof(value), "%lu", expected); + execl("/compat-mlock", "/compat-mlock", value, NULL); + _exit(5); + } + close(pipes[1]); + while (done < sizeof(out)) { + ssize_t n = + read(pipes[0], (char *)&out + done, sizeof(out) - done); + if (n < 0 && errno == EINTR) + continue; + if (n <= 0) + break; + done += n; + } + close(pipes[0]); + ok = pid > 0 && waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status) && done == sizeof(out) && + out.pagesize == expected && out.passed == 6 && !out.failed; + printf("%s - AArch32 memlock expected=%lu actual=%u passed=%u stage=%u error=%d status=%d\n", + ok ? "ok" : "not ok", expected, out.pagesize, out.passed, + out.failed, (int32_t)out.error, status); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/compat-mlock-probe.c b/tools/testing/selftests/arm64/user4k/compat-mlock-probe.c new file mode 100644 index 0000000000000..97e76dd94bae9 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-mlock-probe.c @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding ELF32 memlock probe, only for the disposable PID1 fixture. */ +typedef unsigned int u32; +struct result { + u32 pagesize, passed, failed, error; +}; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} +static int error(long n) +{ + return (u32)n >= 0xfffff001U; +} +static unsigned number(const char *p) +{ + unsigned n = 0; + while (*p >= '0' && *p <= '9') + n = n * 10 + *p++ - '0'; + return n; +} +static long locked_kb(void) +{ + char buf[8192]; + long fd = call(5, (long)"/proc/self/status", 0, 0, 0, 0, 0), n; + const char name[] = "VmLck:"; + + if (error(fd)) + return -1; + n = call(3, fd, (long)buf, sizeof(buf) - 1, 0, 0, 0); + call(6, fd, 0, 0, 0, 0, 0); + if (n <= 0) + return -1; + buf[n] = 0; + for (long i = 0; i < n; i++) { + unsigned j = 0; + while (name[j] && i + j < (unsigned)n && buf[i + j] == name[j]) + j++; + if (!name[j]) { + const char *p = buf + i + j; + while (*p == ' ' || *p == '\t') + p++; + return number(p); + } + } + return -1; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + unsigned argc = *sp++, ps = 0, kb, stage = 1; + char **argv = (void *)sp; + struct result out = { 0 }; + long p, q, pid, ret = -1; + int status = -1; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + ps = sp[1]; + sp += 2; + } + out.pagesize = ps; + if (argc != 2 || call(64, 0, 0, 0, 0, 0, 0) != 1 || + ps != number(argv[1])) + goto done; + kb = ps / 1024; + p = call(192, 0, 8 * ps, 3, 0x22, -1, 0); + if (error(p) || ((u32)p & (ps - 1)) || locked_kb() != 0) + goto done; + ret = call(150, p + ps + 17, ps - 32, 0, 0, 0, 0); + if (ret || locked_kb() != (long)kb) + goto done; + out.passed++; + stage++; + ret = call(150, p + ps, ps, 0, 0, 0, 0); + if (ret || locked_kb() != (long)kb) + goto done; + ret = call(150, p + 2 * ps, ps, 0, 0, 0, 0); + if (ret || locked_kb() != (long)(2 * kb)) + goto done; + ret = call(151, p + ps, ps, 0, 0, 0, 0); + if (ret || locked_kb() != (long)kb) + goto done; + out.passed++; + stage++; + *(volatile unsigned char *)(p + 2 * ps) = 0x63; + q = call(163, p + 2 * ps, ps, 2 * ps, 1, 0, 0); + ret = q; + if (error(q) || locked_kb() != (long)(2 * kb) || + *(volatile unsigned char *)q != 0x63) + goto done; + for (unsigned i = ps; i < 2 * ps; i++) + if (((volatile unsigned char *)q)[i]) + goto done; + out.passed++; + stage++; + pid = call(2, 0, 0, 0, 0, 0, 0); + if (!pid) { + call(1, locked_kb() != 0, 0, 0, 0, 0, 0); + for (;;) { + } + } + ret = call(114, pid, (long)&status, 0, 0, 0, 0); + if (error(pid) || ret != pid || status || locked_kb() != (long)(2 * kb)) + goto done; + out.passed++; + stage++; + ret = call(91, q, 2 * ps, 0, 0, 0, 0); + if (ret || locked_kb() != 0) + goto done; + out.passed++; + stage++; + ret = call(390, p + 4 * ps, 2 * ps, 1, 0, 0, 0); + if (ret || locked_kb() != (long)(2 * kb)) + goto done; + *(volatile unsigned char *)(p + 4 * ps) = 0x75; + ret = call(151, p + 4 * ps, 2 * ps, 0, 0, 0, 0); + if (ret || locked_kb() != 0) + goto done; + out.passed++; + stage = 0; +done: + out.failed = stage; + out.error = stage ? (u32)ret : 0; + call(4, 1, (long)&out, sizeof(out), 0, 0, 0); + call(1, stage != 0, 0, 0, 0, 0, 0); + for (;;) { + } +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-numa-balancing-probe.c b/tools/testing/selftests/arm64/user4k/compat-numa-balancing-probe.c new file mode 100644 index 0000000000000..f2c155a888c15 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-numa-balancing-probe.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 automatic NUMA migration and fallback checks. */ +typedef unsigned int u32; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} + +static unsigned int number(const char *s) +{ + unsigned int n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static _Noreturn void finish(int code) +{ + char text[] = "A32 NUMAB FAIL stage=00\n"; + + if (code) { + text[sizeof(text) - 4] = '0' + code / 10; + text[sizeof(text) - 3] = '0' + code % 10; + call(4, 1, (long)text, sizeof(text) - 1, 0, 0, 0); + } else { + static const char pass[] = "A32 NUMAB PASS\n"; + + call(4, 1, (long)pass, sizeof(pass) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} + +static void value(const char *label, unsigned int length, u32 n) +{ + char text[16]; + unsigned int pos = sizeof(text); + + text[--pos] = '\n'; + do { + text[--pos] = '0' + n % 10; + n /= 10; + } while (n); + call(4, 1, (long)label, length, 0, 0, 0); + call(4, 1, (long)(text + pos), sizeof(text) - pos, 0, 0, 0); +} + +static u32 seconds(void) +{ + struct { + long long sec, nsec; + } ts; + + /* Use time64 even when CONFIG_COMPAT_32BIT_TIME is disabled. */ + if (call(403, 1, (long)&ts, 0, 0, 0, 0)) + finish(11); + return ts.sec; +} + +static void cpu(unsigned int id) +{ + u32 mask = 1U << id; + + if (call(241, 0, sizeof(mask), (long)&mask, 0, 0, 0)) + finish(12); +} + +static int node_at(unsigned char *address) +{ + int node = -1; + + if (call(320, (long)&node, 0, 0, (long)address, 3, 0)) + finish(13); + return node; +} + +static u32 sample(unsigned char *base, u32 length, u32 page) +{ + u32 moved = 0; + + for (u32 off = 0; off < length; off += page) + if (off != 2 * page) + moved += node_at(base + off + 16) == 1; + return moved; +} + +static void forbidden(unsigned char *address, int write_access) +{ + long child = call(2, 0, 0, 0, 0, 0, 0); + int status = 0; + + if (child < 0) + finish(40); + if (!child) { + /* Force the intended access; this is not thread synchronization. */ + if (write_access) + *(volatile unsigned char *)address = 0x27; + else + (void)*(volatile unsigned char *)address; + call(1, 2, 0, 0, 0, 0, 0); + for (;;) + ; + } + if (call(114, child, (long)&status, 0, 0, 0, 0) != child || + (status & 0x7f) != 11) + finish(41); +} + +static void mapping(u32 page) +{ + const u32 length = 8U << 20; + u32 mask[2] = { 1, 0 }, got[2] = { 0, 0 }; + u32 loops = 0, moved = 0, started, total = length / page - 1; + int mode = -1; + long raw; + unsigned char *base; + + if (call(320, (long)&mode, (long)got, 64, 0, 4, 0) || got[0] != 3 || + got[1]) + finish(20); + cpu(0); + raw = call(192, 0, length, 3, 0x22, -1, 0); + if ((u32)raw >= 0xfffff001U) + finish(21); + base = (void *)raw; + /* A missing 4K capability must retain native alignment restrictions. */ + if (call(125, raw + 4096, 4096, 1, 0, 0, 0) != (page == 4096 ? 0 : -22)) + finish(22); + if (call(125, raw, length, 3, 0, 0, 0)) + finish(23); + if (call(220, raw, length, 15, 0, 0, 0)) /* MADV_NOHUGEPAGE */ + finish(24); + if (call(319, raw, length, 2, (long)mask, 64, 0)) + finish(25); + for (u32 off = 0; off < length; off += page) { + u32 *p = (void *)(base + off); + + p[0] = 0x91720000 + off / page; + p[1] = 0; + p[page / 4 - 1] = 0x51320000 + off / page; + if (node_at(base + off + 16) != 0) + finish(26); + } + if (call(125, raw + page, page, 1, 0, 0, 0) || + call(125, raw + 2 * page, page, 0, 0, 0, 0)) + finish(27); + if (call(319, raw, length, 0, 0, 0, + 0)) /* MPOL_DEFAULT, no MOVE flags */ + finish(28); + cpu(1); + started = seconds(); + do { + for (u32 off = 0; off < length; off += page) { + volatile u32 *p = (void *)(base + off); + + if (off == 2 * page) + continue; + if (p[0] != 0x91720000 + off / page || + p[page / 4 - 1] != 0x51320000 + off / page) + finish(29); + if (off != page) + p[1]++; + } + loops++; + if (!(loops % 128)) { + moved = sample(base, length, page); + if (moved == total) + break; + } + } while (seconds() - started < 30); + moved = sample(base, length, page); + value("A32 NUMAB page=", sizeof("A32 NUMAB page=") - 1, page); + value("A32 NUMAB moved=", sizeof("A32 NUMAB moved=") - 1, moved); + value("A32 NUMAB expected=", sizeof("A32 NUMAB expected=") - 1, total); + value("A32 NUMAB loops=", sizeof("A32 NUMAB loops=") - 1, loops); + if (moved != total) + finish(30); + for (u32 off = 0; off < length; off += page) + if (off != page && off != 2 * page && + *(u32 *)(base + off + 4) != loops) + finish(31); + forbidden(base + page, 1); + forbidden(base + 2 * page, 0); + if (call(125, raw + 2 * page, page, 1, 0, 0, 0) || + *(u32 *)(base + 2 * page) != 0x91720002) + finish(32); + if (call(91, raw, length, 0, 0, 0, 0)) + finish(33); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, page = 0; + char **argv = (void *)sp; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + page = sp[1]; + sp += 2; + } + if (argc != 2 || page != number(argv[1])) + finish(10); + mapping(page); + finish(0); +} + +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-numa-probe.c b/tools/testing/selftests/arm64/user4k/compat-numa-probe.c new file mode 100644 index 0000000000000..b2652fe348b6e --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-numa-probe.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 NUMA policy and native-granule fallback checks. */ +typedef unsigned int u32; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} + +static unsigned int number(const char *s) +{ + unsigned int n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static _Noreturn void finish(int code) +{ + char text[] = "A32 NUMA FAIL stage=00\n"; + + if (code) { + text[sizeof(text) - 4] = '0' + code / 10; + text[sizeof(text) - 3] = '0' + code % 10; + call(4, 1, (long)text, sizeof(text) - 1, 0, 0, 0); + } else { + static const char pass[] = "A32 NUMA PASS\n"; + + call(4, 1, (long)pass, sizeof(pass) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} + +static void mapping(unsigned long native, unsigned long page) +{ + unsigned long length = 4 * native; + long raw = call(192, 0, length, 3, 0x22, -1, 0); + unsigned char *base; + u32 mask[2] = { 1, 0 }, got[2] = { 0, 0 }; + int mode = -1, node; + + if ((unsigned long)raw >= 0xfffff001UL) + finish(20); + base = (void *)(((unsigned long)raw + native - 1) & ~(native - 1)); + if (call(320, (long)&mode, (long)got, 64, 0, 4, 0) || got[0] != 3 || + got[1]) + finish(21); + if (call(319, (long)base, 2 * native, 2, (long)mask, 64, 0)) + finish(22); + mask[0] = 2; + if (call(319, (long)(base + page), 1, 2, (long)mask, 64, 0)) + finish(23); + if (call(450, (long)(base + page), 1, 1, 0, 0, 0)) + finish(24); + if (call(320, (long)&mode, (long)got, 64, (long)(base + page), 2, 0) || + mode != 2 || got[0] != 2 || got[1]) + finish(25); + for (unsigned long off = 0; off < 2 * native; off += page) { + base[off + 17] = 0x51 + off / page; + node = -1; + if (call(320, (long)&node, 0, 0, (long)(base + off + 17), 3, + 0) || + node != (off == page ? 1 : 0)) + finish(26); + } + if (call(319, (long)base, 2 * native, 2, (long)mask, 64, 1) != -5) + finish(27); + if (call(319, (long)(base + 4096), 4096, 2, (long)mask, 64, 0) != + (page == 4096 ? 0 : -22)) + finish(28); + if (call(319, (long)base, 2 * native, 2, (long)mask, 64, 5)) + finish(29); + for (unsigned long off = 0; off < 2 * native; off += page) { + node = -1; + if (base[off + 17] != (unsigned char)(0x51 + off / page) || + call(320, (long)&node, 0, 0, (long)(base + off + 17), 3, + 0) || + node != 1) + finish(30); + } + if (call(91, raw, length, 0, 0, 0, 0)) + finish(31); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 3 || pages != number(argv[2])) + finish(10); + mapping(number(argv[1]), pages); + finish(0); +} + +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-perf-probe.c b/tools/testing/selftests/arm64/user4k/compat-perf-probe.c new file mode 100644 index 0000000000000..ee02fc34051ad --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-perf-probe.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 perf mmap layout, including native-page fallback. */ +typedef unsigned int u32; +typedef unsigned long long u64; + +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} + +static unsigned int number(const char *s) +{ + unsigned int n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static _Noreturn void finish(int code) +{ + char text[] = "A32 PERF FAIL stage=00\n"; + + if (code) { + text[20] = '0' + code / 10; + text[21] = '0' + code % 10; + call(4, 1, (long)text, sizeof(text) - 1, 0, 0, 0); + } else { + static const char pass[] = "A32 PERF PASS\n"; + + call(4, 1, (long)pass, sizeof(pass) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} + +static void mapping(unsigned long pages, unsigned long native, + unsigned int count) +{ + static const struct { + u32 type, size; + u64 config, sample_period, sample_type, read_format, flags; + u32 wakeup, bp_type; + u64 config1; + } attr = { .type = 1, .size = 64, .flags = 1 }; + long fd = call(364, (long)&attr, 0, -1, -1, 0, 0); + unsigned long length = (1 + count) * pages; + long raw; + volatile u64 *metadata; + + if (fd < 0) + finish(20); + raw = call(192, 0, length, 3, 1, fd, 0); + if ((unsigned long)raw >= 0xfffff001UL) + finish(21); + metadata = (void *)(raw + 1024); + if (metadata[2] != pages || metadata[3] != count * pages) + finish(22); + metadata[1] = 0; + metadata[6] = length; + metadata[7] = native; + /* mmap2 always takes 4K units, including the no4K compat fallback. */ + if (call(192, 0, native, 1, 1, fd, length / 4096) != -95) + finish(23); + if (call(91, raw, length, 0, 0, 0, 0)) + finish(24); + if (call(6, fd, 0, 0, 0, 0, 0)) + finish(25); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 3 || pages != number(argv[2])) + finish(10); + unsigned long native = number(argv[1]); + + mapping(pages, native, 0); + mapping(pages, native, 1); + mapping(pages, native, 2); + mapping(pages, native, 2 * native / pages); + finish(0); +} + +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-policy-probe.c b/tools/testing/selftests/arm64/user4k/compat-policy-probe.c new file mode 100644 index 0000000000000..dab5a79abe130 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-policy-probe.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 -> AArch64 exec with an inherited granule preference. */ +#include "../../../../../include/uapi/linux/prctl.h" +typedef unsigned int u32; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} +static unsigned int number(const char *s) +{ + unsigned int n = 0; + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} +static void fail(int code) +{ + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + sp += argc + 1; + char **env = (void *)sp; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 4 || pages != number(argv[3])) + fail(10); + if (call(172, PR_AURORA_GET_DEFAULT_PAGE_SIZE, 0, 0, 0, 0, 0) != + number(argv[2])) + fail(11); + if (call(172, PR_AURORA_GET_EXEC_PAGE_SIZE, 0, 0, 0, 0, 0)) + fail(12); + long mem = call(192, 0, 4 * pages, 3, 0x22, -1, 0); + if ((unsigned long)mem >= 0xfffff001UL) + fail(13); + long ret = call(125, mem + 4096, 4096, 1, 0, 0, 0); + if (ret != (pages == 4096 ? 0 : -22)) + fail(14); + if (call(91, mem, 4 * pages, 0, 0, 0, 0)) + fail(15); + char *args[] = { "/init", "--compat-return", argv[1], argv[2], 0 }; + call(11, (long)args[0], (long)args, (long)env, 0, 0, 0); + fail(16); + for (;;) + ; +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-probe.c b/tools/testing/selftests/arm64/user4k/compat-probe.c new file mode 100644 index 0000000000000..14a4deb716412 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-probe.c @@ -0,0 +1,248 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Freestanding AArch32 process; run only by compat-fixture-init in a guest. */ +typedef unsigned int u32; +typedef unsigned short u16; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f, r7 __asm__("r7") = n; + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} +static unsigned length(const char *s) +{ + unsigned n = 0; + while (s[n]) + n++; + return n; +} +static int equal(const char *a, const char *b) +{ + while (*a && *a == *b) { + a++; + b++; + } + return *a == *b; +} +static unsigned number(const char *s) +{ + unsigned n = 0; + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} +static void out(const char *s) +{ + call(4, 1, (long)s, length(s), 0, 0, 0); +} +static int failures, checks; +static void result(int pass, const char *s) +{ + checks++; + if (!pass) + failures++; + out(pass ? "ok - compat " : "not ok - compat "); + out(s); + out("\n"); +} +static int error(long v) +{ + return (unsigned long)v >= 0xfffff001UL; +} +static long map(unsigned size, int prot, int flags, int fd, unsigned off) +{ + return call(192, 0, size, prot, flags, fd, off); +} +static void unmap(long p, unsigned size) +{ + if (!error(p)) + call(91, p, size, 0, 0, 0, 0); +} +static volatile unsigned signals; +static void handler(int n) +{ + if (n == 10) + signals++; +} +struct action { + u32 handler, flags, restorer, mask[2]; +}; +struct ehdr { + unsigned char ident[16]; + u16 type, machine; + u32 version, entry, phoff, shoff, flags; + u16 ehsize, phentsize, phnum, shentsize, shnum, shstrndx; +}; +struct phdr { + u32 type, offset, vaddr, paddr, filesz, memsz, flags, align; +}; +struct dyn { + u32 tag, val; +}; +struct sym { + u32 name, value, size; + unsigned char info, other; + u16 shndx; +}; +static void *vdso_symbol(unsigned base, const char *name) +{ + struct ehdr *h = (void *)base; + struct phdr *p = (void *)(base + h->phoff); + struct dyn *d = 0; + char *strings = 0; + struct sym *symbols = 0; + u32 *hash = 0; + for (unsigned i = 0; i < h->phnum; i++) + if (p[i].type == 2) + d = (void *)(base + p[i].vaddr); + if (!d) + return 0; + for (; d->tag; d++) { + if (d->tag == 5) + strings = (void *)(base + d->val); + if (d->tag == 6) + symbols = (void *)(base + d->val); + if (d->tag == 4) + hash = (void *)(base + d->val); + } + if (!strings || !symbols || !hash) + return 0; + for (unsigned i = 0; i < hash[1]; i++) + if (equal(strings + symbols[i].name, name)) + return (void *)(base + symbols[i].value); + return 0; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + unsigned argc = *sp++, pagesz = 0, vdso = 0; + char **argv = (void *)sp; + sp += argc + 1; + char **env = (void *)sp; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pagesz = sp[1]; + if (sp[0] == 33) + vdso = sp[1]; + sp += 2; + } + if (argc > 1 && equal(argv[1], "--exec32")) { + char *a[] = { argv[0], argv[2], argv[3], argv[4], 0 }; + call(11, (long)a[0], (long)a, (long)env, 0, 0, 0); + call(1, 97, 0, 0, 0, 0, 0); + for (;;) + ; + } + if (argc > 1 && + (equal(argv[1], "--exec64") || equal(argv[1], "--exec64-4k"))) { + int alternative = equal(argv[1], "--exec64-4k"); + if (alternative && call(172, 0x41555001, 4096, 0, 0, 0, 0)) { + call(1, 99, 0, 0, 0, 0, 0); + for (;;) + ; + } + char *a[] = { "/init", + alternative ? "--returned4k" : "--returned", 0 }; + call(11, (long)a[0], (long)a, (long)env, 0, 0, 0); + call(1, 98, 0, 0, 0, 0, 0); + for (;;) + ; + } + /* Script interpreter contributes its pathname before the ordinary arguments. */ + unsigned start = argc > 1 && equal(argv[1], "/script32") ? 2 : 1; + unsigned expected = argc > start ? number(argv[start]) : 0; + int fd = argc > start + 1 ? (int)number(argv[start + 1]) : -1; + result(pagesz == expected, "AT_PAGESZ reflects final ELF32 selection"); + int args = argc > start + 2 && length(argv[start + 2]) == 90000; + if (args) + for (unsigned i = 0; i < 90000; i++) + if (argv[start + 2][i] != (char)('a' + i % 23)) { + args = 0; + break; + } + result(args, "90KB staged argument survives granule transfer"); + long mem = map(4 * pagesz, 3, 0x22, -1, 0); + int ok = !error(mem); + if (ok) { + unsigned char *p = (void *)mem; + for (unsigned i = 0; i < 4 * pagesz; i++) + p[i] = 0x30 + i / pagesz; + for (unsigned i = 0; i < 4 * pagesz; i++) + if (p[i] != (unsigned char)(0x30 + i / pagesz)) + ok = 0; + } + result(ok, "anonymous user leaves have independent bytes"); + result(!error(mem) && !call(125, mem + pagesz, pagesz, 1, 0, 0, 0) && + !call(125, mem + pagesz, pagesz, 3, 0, 0, 0), + "mprotect at selected page boundaries"); + result(!error(mem) && call(125, mem + 1, pagesz, 1, 0, 0, 0) == -22 && + call(91, mem + 1, pagesz, 0, 0, 0, 0) == -22, + "unaligned memory operations retain restrictions"); + long q = map(pagesz, 1, 1, fd, 3 * (pagesz >> 12)); + ok = !error(q); + if (ok) + for (unsigned i = 0; i < pagesz; i++) + if (((unsigned char *)q)[i] != + (unsigned char)(0x40 + (3 * pagesz + i) / 4096)) + ok = 0; + result(ok, "mmap2 native-aligned file offset"); + unmap(q, pagesz); + q = map(4096, 1, 1, fd, 1); + ok = pagesz == 4096 ? !error(q) && *(unsigned char *)q == 0x41 : + q == -22; + result(ok, "mmap2 4KB offset accepted only by matching ABI"); + unmap(q, pagesz); + q = map(pagesz, 1, 1, fd, 0x100000 + (pagesz >> 12)); + ok = !error(q); + if (ok) + for (unsigned i = 0; i < pagesz; i++) + if (((unsigned char *)q)[i] != + (unsigned char)(0x80 + (pagesz + i) / 4096)) + ok = 0; + result(ok, "mmap2 offset above 4GB remains full width"); + unmap(q, pagesz); + long child = call(2, 0, 0, 0, 0, 0, 0); + int status = 0; + if (!child) { + if (!error(mem)) + *(unsigned char *)mem = 0xa7; + call(1, 0, 0, 0, 0, 0, 0); + for (;;) + ; + } + ok = child > 0 && + call(114, child, (long)&status, 0, 0, 0, 0) == child && !status && + !error(mem) && *(unsigned char *)mem == 0x30; + result(ok, "fork isolates private COW data"); + unmap(mem, 4 * pagesz); + struct action act = { (u32)handler, 0, 0, { 0, 0 } }; + ok = !call(174, 10, (long)&act, 0, 8, 0, 0); + if (ok) + ok = !call(37, call(20, 0, 0, 0, 0, 0, 0), 10, 0, 0, 0, 0) && + signals == 1; + result(ok, "AArch32 signal frame and kernel sigreturn trampoline"); + volatile unsigned word = 17; + int (*cmpxchg)(unsigned, unsigned, volatile unsigned *) = + (void *)0xffff0fc0; + result(*(volatile unsigned *)0xffff0ffc >= 5 && + !cmpxchg(17, 29, &word) && word == 29, + "kuser helper page and atomic compare exchange"); + int (*gettime)(int, void *) = + vdso ? (void *)vdso_symbol(vdso, "__vdso_clock_gettime") : 0; + long time[2] = { 0, 0 }; + result(gettime && !gettime(1, time) && time[0] >= 0 && time[1] >= 0 && + time[1] < 1000000000, + "compat vDSO clock_gettime and vvar mapping"); + out(failures ? "COMPAT PROBE FAIL\n" : "COMPAT PROBE PASS\n"); + call(1, failures ? 1 : 0, 0, 0, 0, 0, 0); + for (;;) + ; +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-script b/tools/testing/selftests/arm64/user4k/compat-script new file mode 100644 index 0000000000000..9b56d0c7567f5 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-script @@ -0,0 +1 @@ +#!/probe32-64k diff --git a/tools/testing/selftests/arm64/user4k/compat-shm-probe.c b/tools/testing/selftests/arm64/user4k/compat-shm-probe.c new file mode 100644 index 0000000000000..c4ee1da2c5a12 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-shm-probe.c @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 SysV attach/detach and fallback-alignment checks. */ +typedef unsigned int u32; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} + +static unsigned int number(const char *s) +{ + unsigned int n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static _Noreturn void finish(int code) +{ + char text[] = "A32 SHM FAIL stage=00\n"; + + if (code) { + text[19] = '0' + code / 10; + text[20] = '0' + code % 10; + call(4, 1, (long)text, sizeof(text) - 1, 0, 0, 0); + } else { + static const char pass[] = "A32 SHM PASS\n"; + + call(4, 1, (long)pass, sizeof(pass) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} + +static void mapping(unsigned long pages, int rounded) +{ + unsigned long align = 4 * pages, length = 4 * pages; + long raw = call(192, 0, 12 * align, 3, 0x22, -1, 0); + unsigned char *target; + long id, result; + + if ((unsigned long)raw >= 0xfffff001UL) + finish(20); + result = call(125, raw + 4096, 4096, 1, 0, 0, 0); + if (result != (pages == 4096 ? 0 : -22)) + finish(21); + if (call(125, raw, 12 * align, 3, 0, 0, 0)) + finish(22); + target = (void *)(((unsigned long)raw + align - 1) & ~(align - 1)); + target += align + (rounded ? 0 : pages); + target[-1] = 0x51; + target[length] = 0x72; + if (call(91, (long)target, length, 0, 0, 0, 0)) + finish(23); + id = call(307, 0, length - 1, 0600, 0, 0, 0); + if (id < 0) + finish(24); + result = call(305, id, (long)(target + (rounded ? pages + 17 : 0)), + rounded ? 020000 : 0, 0, 0, 0); + if (result != (long)target) { + call(308, id, 0, 0, 0, 0, 0); + finish(rounded ? 26 : 25); + } + if (call(308, id, 0, 0, 0, 0, 0)) + finish(27); + target[pages + 13] = 0x67; + if (call(125, (long)(target + pages), pages, 1, 0, 0, 0)) + finish(28); + if (call(91, (long)target, pages, 0, 0, 0, 0)) + finish(29); + if (call(306, (long)target, 0, 0, 0, 0, 0)) + finish(30); + if (target[-1] != 0x51 || target[length] != 0x72) + finish(31); + if (call(91, raw, 12 * align, 0, 0, 0, 0)) + finish(32); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 3 || pages != number(argv[2])) + finish(10); + mapping(pages, 0); + mapping(pages, 1); + finish(0); +} + +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-uffd-init.c b/tools/testing/selftests/arm64/user4k/compat-uffd-init.c new file mode 100644 index 0000000000000..5f1e90f3e484f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-uffd-init.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - compat uffd line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +struct wire { + uint32_t address, pagesize; +}; +static void expect_invalid(int fd, uint64_t dst, uint64_t src, uint64_t len) +{ + struct uffdio_copy c = { .dst = dst, .src = src, .len = len }; + errno = 0; + CHECK(ioctl(fd, UFFDIO_COPY, &c) == -1 && errno == EINVAL); +} +static void run(unsigned long native, unsigned long expected, int shared) +{ + int sock[2], file = -1, status; + CHECK(!socketpair(AF_UNIX, SOCK_SEQPACKET, 0, sock)); + if (shared) { + file = memfd_create("compat-uffd", 0); + CHECK(file >= 0 && !ftruncate(file, 3 * expected)); + } + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char page_arg[24], sock_arg[24], file_arg[24]; + close(sock[0]); + snprintf(page_arg, sizeof(page_arg), "%lu", expected); + snprintf(sock_arg, sizeof(sock_arg), "%d", sock[1]); + snprintf(file_arg, sizeof(file_arg), "%d", file); + execl("/compat-uffd", "/compat-uffd", page_arg, sock_arg, + file_arg, NULL); + _exit(99); + } + close(sock[1]); + struct wire data = { 0 }; + struct iovec iov = { &data, sizeof(data) }; + union { + struct cmsghdr align; + char bytes[CMSG_SPACE(sizeof(int))]; + } control = { 0 }; + struct msghdr msg = { .msg_iov = &iov, + .msg_iovlen = 1, + .msg_control = control.bytes, + .msg_controllen = sizeof(control.bytes) }; + CHECK(recvmsg(sock[0], &msg, 0) == sizeof(data)); + CHECK(!(msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC)) && + data.pagesize == expected); + struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg); + CHECK(cmsg && cmsg->cmsg_level == SOL_SOCKET && + cmsg->cmsg_type == SCM_RIGHTS && + cmsg->cmsg_len == CMSG_LEN(sizeof(int))); + int uffd; + memcpy(&uffd, CMSG_DATA(cmsg), sizeof(uffd)); + /* Force the source above every possible AArch32 task limit. */ + unsigned char *source = + mmap((void *)0x1000000000UL, 2 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(source != MAP_FAILED && (uintptr_t)source > UINT32_MAX); + /* Source alignment is independent of both processes' granules. */ + unsigned char *bytes = source + 17; + for (unsigned leaf = 0; leaf < 3; leaf++) { + struct pollfd pfd = { .fd = uffd, .events = POLLIN }; + struct uffd_msg event; + CHECK(poll(&pfd, 1, 10000) == 1 && (pfd.revents & POLLIN)); + CHECK(read(uffd, &event, sizeof(event)) == sizeof(event)); + uint64_t dst = (uint64_t)data.address + leaf * expected; + CHECK(event.event == UFFD_EVENT_PAGEFAULT && + !event.arg.pagefault.flags && + event.arg.pagefault.address == dst); + if (!leaf) { + expect_invalid(uffd, dst, (uintptr_t)bytes, 0); + expect_invalid(uffd, dst, (uintptr_t)bytes, + expected - 1); + expect_invalid(uffd, dst + 1, (uintptr_t)bytes, + expected); + expect_invalid(uffd, 0x100000000ULL, (uintptr_t)bytes, + expected); + expect_invalid(uffd, dst, UINT64_MAX - expected + 1, + expected); + if (expected > 4096) + expect_invalid(uffd, dst, (uintptr_t)bytes, + 4096); + } + if (leaf == 1) { + /* A valid but inaccessible pager source must leave the target missing. */ + CHECK(!mprotect(source, 2 * native, PROT_NONE)); + struct uffdio_copy bad = { .dst = dst, + .src = (uintptr_t)bytes, + .len = expected }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &bad) == -1 && + errno == EFAULT && bad.copy == -EFAULT); + CHECK(!mprotect(source, 2 * native, + PROT_READ | PROT_WRITE)); + } + memset(bytes, 0x51 + leaf, expected); + struct uffdio_copy c = { .dst = dst, + .src = (uintptr_t)bytes, + .len = expected }; + CHECK(!ioctl(uffd, UFFDIO_COPY, &c) && + c.copy == (long long)expected); + } + CHECK(waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(!munmap(source, 2 * native)); + CHECK(!close(uffd) && !close(sock[0])); + if (shared) + CHECK(!close(file)); + printf("ok - native %lu pager high unaligned COPY to AArch32 %lu %s, source failure and target alignment\n", + native, expected, shared ? "shared" : "anonymous"); +} +int main(void) +{ + if (getpid() != 1) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + alarm(90); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + char cmdline[2048] = { 0 }; + int fd = open("/proc/cmdline", O_RDONLY); + CHECK(fd >= 0 && read(fd, cmdline, sizeof(cmdline) - 1) > 0 && + !close(fd)); + unsigned long native = getauxval(AT_PAGESZ); + unsigned long expected = + strstr(cmdline, "id_aa64mmfr0.tgran4=f") ? native : 4096; + run(native, expected, 0); + run(native, expected, 1); + puts("COMPAT UFFD PASS"); + reboot(RB_POWER_OFF); + return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/compat-uffd-probe.c b/tools/testing/selftests/arm64/user4k/compat-uffd-probe.c new file mode 100644 index 0000000000000..6abcdb86bb4d8 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-uffd-probe.c @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding ELF32 half of compat-uffd-init; no armhf sysroot required. */ +typedef unsigned int u32; +typedef unsigned long long u64; +struct api { + u64 api, features, ioctls; +}; +struct registration { + u64 start, len, mode, ioctls; +}; +struct movement { + u64 dst, src, len, mode; + long long moved; +}; +struct wire { + u32 address, pagesize; +}; +struct iovec32 { + void *base; + u32 len; +}; +struct message32 { + void *name; + u32 namelen; + struct iovec32 *iov; + u32 iovlen; + void *control; + u32 controllen, flags; +}; +struct rights32 { + u32 len, level, type; + int fd; +}; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} +static u32 number(const char *s) +{ + u32 n = 0; + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} +__attribute__((noreturn)) static void finish(int code) +{ + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 4 || pages != number(argv[1])) + finish(10); + int sock = number(argv[2]), shared = argv[3][0] != '-'; + int file = shared ? (int)number(argv[3]) : -1; + long map = call(192, 0, 3 * pages, 3, shared ? 1 : 0x22, file, 0); + if ((unsigned long)map >= 0xfffff001UL) + finish(11); + /* Fine operations must retain native alignment restrictions on fallback. */ + if (call(125, map + 4096, 4096, 1, 0, 0, 0) != + (pages == 4096 ? 0 : -22)) + finish(12); + if (call(125, map, 3 * pages, 3, 0, 0, 0)) + finish(13); + long fd = call(388, 0x80801, 0, 0, 0, 0, + 0); /* CLOEXEC|NONBLOCK|USER_MODE_ONLY */ + if (fd < 0) + finish(14); + /* Negotiate shared missing support under both fine and fallback ABIs. */ + struct api api = { 0xaa, (1ULL << 5) | (1ULL << 16), 0 }; + if (call(54, fd, 0xc018aa3fU, (long)&api, 0, 0, 0) || + !(api.features & (1ULL << 5))) + finish(15); + struct registration reg = { (u32)map, 3 * pages, 1, 0 }; + if (call(54, fd, 0xc020aa00U, (long)®, 0, 0, 0)) + finish(16); + if (!(reg.ioctls & (1ULL << 3)) || !(api.features & (1ULL << 16))) + finish(17); /* COPY and MOVE */ + struct wire data = { (u32)map, pages }; + struct iovec32 iov = { &data, sizeof(data) }; + struct rights32 rights = { sizeof(rights), 1, 1, fd }; + struct message32 msg = { 0, 0, &iov, 1, &rights, sizeof(rights), 0 }; + if (call(296, sock, (long)&msg, 0, 0, 0, 0) != sizeof(data)) + finish(18); + volatile unsigned char *p = (void *)map; + /* Each first access waits for the native pager's COPY. */ + for (u32 leaf = 0; leaf < 3; leaf++) + for (u32 i = 0; i < pages; i++) + if (p[leaf * pages + i] != (unsigned char)(0x51 + leaf)) + finish(19); + if (!shared) { + long target = call(192, 0, 3 * pages, 3, 0x22, -1, 0); + if ((unsigned long)target >= 0xfffff001UL) + finish(21); + struct registration dst = { (u32)target, 3 * pages, 1, 0 }; + if (call(54, fd, 0xc020aa00U, (long)&dst, 0, 0, 0) || + !(dst.ioctls & (1ULL << 5))) + finish(22); + struct movement m = { 1ULL << 32, (u32)map + 2 * pages, pages, + 0, 0 }; + if (call(54, fd, 0xc028aa05U, (long)&m, 0, 0, 0) != -22) + finish(23); + m.dst = (u32)target + pages; + if (pages > 4096) { + m.len = 4096; + if (call(54, fd, 0xc028aa05U, (long)&m, 0, 0, 0) != -22) + finish(24); + m.len = pages; + } + if (call(54, fd, 0xc028aa05U, (long)&m, 0, 0, 0) || + m.moved != pages) + finish(25); + volatile unsigned char *q = (void *)target; + for (u32 i = 0; i < pages; i++) + if (q[pages + i] != 0x53 || p[i] != 0x51 || + p[pages + i] != 0x52) + finish(26); + if (call(91, target, 3 * pages, 0, 0, 0, 0)) + finish(27); + static const char ok[] = + "ok - AArch32 MOVE, neighbours and native-fallback alignment\n"; + call(4, 1, (long)ok, sizeof(ok) - 1, 0, 0, 0); + } + if (call(91, map, 3 * pages, 0, 0, 0, 0)) + finish(20); + finish(0); +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-vfio-probe.c b/tools/testing/selftests/arm64/user4k/compat-vfio-probe.c new file mode 100644 index 0000000000000..231e4b7996613 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-vfio-probe.c @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 type1 probe; no armhf sysroot or physical DMA. */ +#define __user +#include "../../../../../include/uapi/linux/vfio.h" +#include "../../../../../drivers/vfio/vfio_iommu_type1_test.h" + +typedef unsigned int u32; +typedef unsigned long long u64; +#define IOVA (256UL * 1024 * 1024) +static unsigned int stage; + +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} + +static u32 number(const char *s) +{ + u32 n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static void clear(void *data, u32 size) +{ + volatile unsigned char *p = data; + + for (u32 i = 0; i < size; i++) + p[i] = 0; +} + +/* Aggregate initialization may use these compiler-runtime entry points. */ +void __aeabi_memclr8(void *data, u32 size) +{ + clear(data, size); +} +void __aeabi_memclr4(void *data, u32 size) +{ + clear(data, size); +} +void __aeabi_memclr(void *data, u32 size) +{ + clear(data, size); +} + +_Noreturn static void finish(int code) +{ + if (code) { + char msg[] = "not ok - AArch32 type1 stage=00\n"; + + msg[29] = '0' + stage / 10; + msg[30] = '0' + stage % 10; + call(4, 1, (long)msg, sizeof(msg) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} +#define CHECK(x) \ + do { \ + stage++; \ + if (!(x)) \ + finish(stage); \ + } while (0) + +static long ioctl_test(int fd, u32 request, void *arg) +{ + return call(54, fd, request, (long)arg, 0, 0, 0); +} + +static int state(int fd, struct vfio_type1_test_cmd *cmd) +{ + clear(cmd, sizeof(*cmd)); + return ioctl_test(fd, VFIO_TYPE1_TEST_STATE, cmd); +} + +static long map(int fd, u32 address, u32 size) +{ + struct vfio_iommu_type1_dma_map cmd; + + clear(&cmd, sizeof(cmd)); + cmd.argsz = sizeof(cmd); + cmd.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + cmd.iova = IOVA; + cmd.vaddr = address; + cmd.size = size; + return ioctl_test(fd, VFIO_IOMMU_MAP_DMA, &cmd); +} + +static long unmap(int fd, u32 size) +{ + struct vfio_iommu_type1_dma_unmap cmd; + long ret; + + clear(&cmd, sizeof(cmd)); + cmd.argsz = sizeof(cmd); + cmd.iova = IOVA; + cmd.size = size; + ret = ioctl_test(fd, VFIO_IOMMU_UNMAP_DMA, &cmd); + return ret ? ret : cmd.size == size ? 0 : -1; +} + +static long rw(int fd, unsigned char *bytes, unsigned int flags) +{ + struct vfio_type1_test_cmd cmd; + + clear(&cmd, sizeof(cmd)); + cmd.iova = IOVA; + cmd.buffer = (u32)bytes; + cmd.size = 16; + cmd.flags = flags; + return ioctl_test(fd, VFIO_TYPE1_TEST_RW, &cmd); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + struct vfio_type1_test_cmd cmd; + struct vfio_iommu_type1_dirty_bitmap dirty; + struct vfio_iommu_type1_info info; + unsigned char bytes[16]; + long fd, file, data, pid; + int status = -1; + u32 native, length; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + CHECK(argc == 3 && pages == number(argv[2])); + native = number(argv[1]); + length = native + 4096; + fd = call(5, (long)"/dev/vfio-test", 2, 0, 0, 0, 0); + file = call(385, (long)"compat-vfio", 0, 0, 0, 0, 0); + CHECK(fd >= 0 && file >= 0 && !call(93, file, 2 * native, 0, 0, 0, 0)); + data = call(192, 0, 2 * native, 3, 1, file, 0); + CHECK((u32)data < 0xfffff001U); + CHECK(call(125, data + 4096, 4096, 1, 0, 0, 0) == + (pages == 4096 ? 0 : -22)); + CHECK(!call(125, data, 2 * native, 3, 0, 0, 0)); + for (u32 i = 0; i < 2 * native; i++) + ((volatile unsigned char *)data)[i] = 0x53; + clear(&info, sizeof(info)); + info.argsz = sizeof(info); + CHECK(!ioctl_test(fd, VFIO_IOMMU_GET_INFO, &info) && + (info.iova_pgsizes & 4096)); + CHECK(!map(fd, data, 2 * native)); + CHECK(!state(fd, &cmd) && !cmd.locked_bytes); + CHECK(!rw(fd, bytes, 0) && bytes[0] == 0x53 && bytes[15] == 0x53); + clear(&dirty, sizeof(dirty)); + dirty.argsz = sizeof(dirty); + dirty.flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_START; + CHECK(!ioctl_test(fd, VFIO_IOMMU_DIRTY_PAGES, &dirty)); + clear(&cmd, sizeof(cmd)); + cmd.iova = IOVA + 19; + CHECK(!ioctl_test(fd, VFIO_TYPE1_TEST_PIN, &cmd)); + CHECK(!state(fd, &cmd) && cmd.locked_bytes == native && + cmd.external_refs == 1); + /* Compat pointer padding must not become part of the native pointer. */ + _Alignas(struct vfio_iommu_type1_dirty_bitmap_get) unsigned char + buffer[sizeof(dirty) + + sizeof(struct vfio_iommu_type1_dirty_bitmap_get)]; + struct vfio_iommu_type1_dirty_bitmap *header = (void *)buffer; + struct vfio_iommu_type1_dirty_bitmap_get *range = (void *)header->data; + u64 bits = 0; + + clear(buffer, sizeof(buffer)); + header->argsz = sizeof(buffer); + header->flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP; + range->iova = IOVA; + range->size = 2 * native; + range->bitmap.pgsize = 4096; + range->bitmap.size = sizeof(bits); + range->bitmap.data = &bits; + CHECK(!ioctl_test(fd, VFIO_IOMMU_DIRTY_PAGES, buffer) && + bits == ((1ULL << (native >> 12)) - 1)); + _Static_assert( + sizeof(range->bitmap.data) == 4 && sizeof(range->bitmap) == 24, + "AArch32 bitmap has a 32-bit pointer and trailing padding"); + ((unsigned char *)&range + ->bitmap)[__builtin_offsetof(struct vfio_bitmap, data) + + sizeof(range->bitmap.data)] = 0xa5; + bits = 0; + CHECK(!ioctl_test(fd, VFIO_IOMMU_DIRTY_PAGES, buffer) && + bits == ((1ULL << (native >> 12)) - 1)); + _Alignas(struct vfio_bitmap) unsigned char + unmap_buffer[sizeof(struct vfio_iommu_type1_dma_unmap) + + sizeof(struct vfio_bitmap)]; + struct vfio_iommu_type1_dma_unmap *remove = (void *)unmap_buffer; + struct vfio_bitmap *remove_bitmap = (void *)remove->data; + + clear(unmap_buffer, sizeof(unmap_buffer)); + remove->argsz = sizeof(unmap_buffer); + remove->flags = VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP; + remove->iova = IOVA; + remove->size = 2 * native; + remove_bitmap->pgsize = 4096; + remove_bitmap->size = sizeof(bits); + remove_bitmap->data = &bits; + ((unsigned char *) + remove_bitmap)[__builtin_offsetof(struct vfio_bitmap, data) + + sizeof(remove_bitmap->data)] = 0x5a; + bits = 0; + CHECK(!ioctl_test(fd, VFIO_IOMMU_UNMAP_DMA, unmap_buffer) && + remove->size == 2 * native && + bits == ((1ULL << (native >> 12)) - 1)); + CHECK(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.external_refs && + cmd.notifications == 1); + clear(&cmd, sizeof(cmd)); + cmd.size = 4096; + CHECK(!ioctl_test(fd, VFIO_TYPE1_TEST_DOMAIN_ADD, &cmd)); + pid = call(2, 0, 0, 0, 0, 0, 0); + if (!pid) + finish(map(fd, data + 4096, length) ? 99 : 0); + CHECK(pid > 0 && call(114, pid, (long)&status, 0, 0, 0, 0) == pid && + !status); + CHECK(!state(fd, &cmd) && !cmd.owner_users && + cmd.locked_bytes == 2 * native && cmd.mapped_bytes[0] == length); + CHECK(rw(fd, bytes, 0) == -14); + CHECK(!rw(fd, bytes, VFIO_TYPE1_TEST_WORKER) && bytes[0] == 0x53 && + bytes[15] == 0x53); + CHECK(!unmap(fd, length)); + CHECK(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.mapped_bytes[0]); + CHECK(!call(91, data, 2 * native, 0, 0, 0, 0)); + CHECK(!call(6, file, 0, 0, 0, 0, 0) && !call(6, fd, 0, 0, 0, 0, 0)); + static const char ok[] = + "ok - AArch32 type1 real ioctls, source exit and fallback alignment\n"; + + call(4, 1, (long)ok, sizeof(ok) - 1, 0, 0, 0); + finish(0); +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/context-stress-init.c b/tools/testing/selftests/arm64/user4k/context-stress-init.c new file mode 100644 index 0000000000000..869662502c15f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/context-stress-init.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* PID-1 mixed-granule EL0 stress runner for disposable VMs. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#define DATA_ADDRESS 0x200000000UL +#define CODE_ADDRESS 0x202000000UL +#define CONTEXTS 70000 +#define SHORT_CONTEXTS 2048 + +static sigjmp_buf fault_return; +static volatile uintptr_t expected_fault; +static char generation_path[256]; + +static void fault(int sig, siginfo_t *info, void *context) +{ + (void)context; + if (sig != SIGSEGV || !expected_fault || + (uintptr_t)info->si_addr != expected_fault || + info->si_code != SEGV_ACCERR) + _exit(90); + expected_fault = 0; + siglongjmp(fault_return, 1); +} + +static bool rejects_write(uintptr_t address) +{ + expected_fault = address; + if (!sigsetjmp(fault_return, 1)) { + *(volatile unsigned char *)expected_fault = 1; + expected_fault = 0; + return false; + } + return true; +} + +static int migrate(unsigned int cpu) +{ + cpu_set_t set; + + CPU_ZERO(&set); + CPU_SET(cpu, &set); + return sched_setaffinity(0, sizeof(set), &set); +} + +static unsigned char pattern(unsigned long serial, unsigned long offset) +{ + serial ^= (serial >> 8) ^ (serial >> 16); + return (serial * 37 + offset * 17 + (offset >> 8)) & 255; +} + +static bool validate(unsigned char *p, unsigned long serial, unsigned long len) +{ + for (unsigned long i = 0; i < len; i++) + if (p[i] != pattern(serial, i)) + return false; + return true; +} + +static int worker(unsigned long serial, unsigned long ps, int control) +{ + struct sigaction action = { .sa_sigaction = fault, + .sa_flags = SA_SIGINFO }; + unsigned char *data; + uint32_t *code; + unsigned int cpus = sysconf(_SC_NPROCESSORS_ONLN); + struct stat self, init; + unsigned long iterations = 0; + + if (getauxval(AT_PAGESZ) != ps || getppid() != 1 || + stat("/proc/self/exe", &self) || stat("/proc/1/exe", &init) || + self.st_dev != init.st_dev || self.st_ino != init.st_ino || + prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL) != 0) + return 91; + if (cpus < 2 || migrate(serial % cpus) || + sigaction(SIGSEGV, &action, NULL)) + return 92; + data = mmap((void *)DATA_ADDRESS, 4 * ps, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + code = mmap((void *)CODE_ADDRESS, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (data == MAP_FAILED || code == MAP_FAILED || + mprotect(data, 3 * ps, PROT_READ | PROT_WRITE)) + return 93; + for (unsigned long i = 0; i < 3 * ps; i++) + data[i] = pattern(serial, i); + if (!rejects_write(DATA_ADDRESS + 3 * ps)) + return 94; + if (mprotect(data + ps, ps, PROT_READ)) + return 95; + if (!rejects_write(DATA_ADDRESS + ps)) + return 96; + do { + unsigned int expected = (serial + iterations * 13) & 65535; + int (*fn)(void) = (void *)code; + char stop; + ssize_t got; + + if (mprotect(code, ps, PROT_READ | PROT_WRITE)) + return 97; + code[0] = 0x52800000 | (expected << 5); /* movz w0, #expected */ + code[1] = 0xd65f03c0; /* ret */ + __builtin___clear_cache((char *)code, (char *)(code + 2)); + if (mprotect(code, ps, PROT_READ | PROT_EXEC) || + migrate((serial + iterations + 1) % cpus)) + return 98; + sched_yield(); + if (fn() != (int)expected || !validate(data, serial, 3 * ps)) + return 99; + if (!(iterations % 64) && + (!rejects_write(DATA_ADDRESS + 3 * ps) || + !rejects_write(DATA_ADDRESS + ps))) + return 101; + iterations++; + if (control < 0) + break; + got = read(control, &stop, 1); + if (got == 1) + break; + if (got != -1 || (errno != EAGAIN && errno != EINTR)) + return 100; + } while (true); + if (control >= 0) + printf("# persistent %luK worker: %lu migration/JIT/data cycles\n", + ps / 1024, iterations); + return 0; +} + +static pid_t spawn(unsigned long serial, unsigned long ps, int control, + int close_fd) +{ + pid_t pid = fork(); + + if (!pid) { + char seq[32], size[32], fd[32]; + + if (close_fd >= 0) + close(close_fd); + if (control >= 0 && fcntl(control, F_SETFD, 0)) + _exit(110); + snprintf(seq, sizeof(seq), "%lu", serial); + snprintf(size, sizeof(size), "%lu", ps); + snprintf(fd, sizeof(fd), "%d", control); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, ps, 0UL, 0UL, 0UL)) + _exit(111); + execl("/init", "/init", "--worker", seq, size, fd, NULL); + _exit(112); + } + return pid; +} + +static bool reap(pid_t pid) +{ + int status = 0; + pid_t got; + + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + if (got == pid && WIFEXITED(status) && !WEXITSTATUS(status)) + return true; + printf("not ok - worker pid=%d wait=%d status=%#x\n", pid, got, status); + return false; +} + +static unsigned long long generation(void) +{ + unsigned long long value = 0; + FILE *f = fopen(generation_path, "re"); + + if (!f || fscanf(f, "%llu", &value) != 1) { + perror("ASID generation"); + return 0; + } + fclose(f); + return value; +} + +int main(int argc, char **argv) +{ + unsigned long long before, after; + unsigned char *data; + pid_t persistent[3]; + int commands[3], pipes[2]; + unsigned long native = getauxval(AT_PAGESZ); + unsigned long granules[] = { native, 4096, 16384 }; + unsigned int nr = native > 16384 ? 3 : 2; + glob_t paths; + bool ok = true; + unsigned int contexts = CONTEXTS; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), atoi(argv[4])); + if (argc == 2 && !strcmp(argv[1], "--short")) + contexts = SHORT_CONTEXTS; + if (getpid() != 1 || (argc != 1 && contexts != SHORT_CONTEXTS)) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + glob("/sys/module/*/parameters/user4k_asid_generation", 0, NULL, + &paths) || + paths.gl_pathc != 1) { + perror("fixture setup"); + return 2; + } + snprintf(generation_path, sizeof(generation_path), "%s", + paths.gl_pathv[0]); + globfree(&paths); + data = mmap((void *)DATA_ADDRESS, 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (data == MAP_FAILED) + return 3; + for (unsigned long i = 0; i < 4 * native; i++) + data[i] = pattern(0, i); + before = generation(); + printf("# mixed context stress: %u fork/exec workers, initial ASID generation=%llu\n", + contexts, before); + for (unsigned int i = 0; i < nr; i++) { + if (pipe2(pipes, O_NONBLOCK | O_CLOEXEC)) + return 4; + persistent[i] = spawn(CONTEXTS + i + 1, granules[i], pipes[0], + pipes[1]); + if (persistent[i] < 0) + return 5; + close(pipes[0]); + commands[i] = pipes[1]; + } + for (unsigned long i = 1; i <= contexts; i++) { + pid_t pid = spawn(i, granules[i % nr], -1, -1); + + if (pid < 0 || !reap(pid) || !validate(data, 0, 4 * native)) { + printf("not ok - churn/parent isolation at worker %lu\n", + i); + ok = false; + break; + } + if (!(i % 128)) + printf("# workers=%lu ASID generation=%llu\n", i, + generation()); + } + after = generation(); + for (unsigned int i = 0; i < nr; i++) { + ok &= write(commands[i], "x", 1) == 1; + close(commands[i]); + ok &= reap(persistent[i]); + } + ok &= before && after >= before + 2 * (1ULL << 16); + printf("%s - mixed context stress: generation=%llu->%llu, measured rollovers=%llu\n", + ok ? "ok" : "not ok", before, after, (after - before) >> 16); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/core-fixture-init.c b/tools/testing/selftests/arm64/user4k/core-fixture-init.c new file mode 100644 index 0000000000000..4caadf98296a6 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/core-fixture-init.c @@ -0,0 +1,447 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Core-dump regression runner: PID 1 in a disposable VM only. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +#define ANON 0x100000000UL +#define FILEMAP 0x102000000UL +#define SPARSE 0x104000000UL +#define EXCLUDED 0x106000000UL +#define SWAPPED 0x108000000UL +#define SWAPPED_SIZE (1UL << 20) +#define SPARSE_SIZE (8UL << 20) + +static void die(const char *what) +{ + perror(what); + _exit(120); +} + +static void write_text(const char *path, const char *text) +{ + int fd = open(path, O_WRONLY); + size_t len = strlen(text); + + if (fd < 0 || write(fd, text, len) != (ssize_t)len) + die(path); + close(fd); +} + +static void *map_at(unsigned long address, size_t len, int prot, int fd, + off_t off) +{ + void *p = mmap((void *)address, len, prot, + MAP_PRIVATE | MAP_FIXED_NOREPLACE | + (fd < 0 ? MAP_ANONYMOUS : 0), + fd, off); + + if (p == MAP_FAILED) + die("mmap"); + return p; +} + +static void crash(bool headers, int bounded) +{ + unsigned long ps = getauxval(AT_PAGESZ); + struct rlimit limit = { 64UL << 20, 64UL << 20 }; + struct stat self, init; + unsigned char *p, *data; + int fd; + + if (getppid() != 1 || stat("/proc/self/exe", &self) || + stat("/proc/1/exe", &init) || self.st_dev != init.st_dev || + self.st_ino != init.st_ino) + _exit(121); + if (bounded) + limit.rlim_cur = limit.rlim_max = ps - (bounded < 0); + if (setrlimit(RLIMIT_CORE, &limit)) + die("rlimit"); + write_text("/proc/self/coredump_filter", headers ? "0x10" : "0x1f"); + p = map_at(ANON, 4 * ps, PROT_READ | PROT_WRITE, -1, 0); + for (unsigned int i = 0; i < 4; i++) + memset(p + i * ps, 0x31 + i, ps); + if (mprotect(p + ps, ps, PROT_NONE) || + mprotect(p + 2 * ps, ps, PROT_READ)) + die("mprotect"); + p = map_at(SPARSE, SPARSE_SIZE, PROT_READ | PROT_WRITE, -1, 0); + memset(p, 0x65, ps); + /* Also exercise an existing read-only zero PTE. */ + if (*(volatile unsigned char *)(p + ps) != 0) + _exit(122); + p = map_at(EXCLUDED, ps, PROT_READ | PROT_WRITE, -1, 0); + memset(p, 0xa7, ps); + if (madvise(p, ps, MADV_DONTDUMP)) + die("dontdump"); + fd = open("/core-input", O_CREAT | O_TRUNC | O_RDWR, 0700); + data = malloc(19 * ps); + if (fd < 0 || !data) + die("file setup"); + for (unsigned int i = 0; i < 19; i++) + memset(data + i * ps, 0x81 + i, ps); + memcpy(data, ELFMAG, SELFMAG); + if (write(fd, data, 19 * ps) != (ssize_t)(19 * ps)) + die("file data"); + /* Offset one user page: three mappings select non-native quarters. */ + map_at(FILEMAP, ps, PROT_READ, fd, headers ? 0 : ps); + map_at(FILEMAP + 2 * ps, 2 * ps, PROT_READ, fd, 2 * ps); + map_at(FILEMAP + 8 * ps, ps, PROT_READ, fd, 17 * ps); + close(fd); + fd = open("/core-library", O_CREAT | O_TRUNC | O_RDWR, 0600); + if (fd < 0 || write(fd, data, ps) != (ssize_t)ps) + die("non-executable ELF file"); + map_at(FILEMAP + 12 * ps, ps, PROT_READ, fd, 0); + close(fd); + free(data); + if (!headers && !bounded) { + unsigned int missing = 0; + uint64_t entry; + int pm = open("/proc/self/pagemap", O_RDONLY); + + p = map_at(SWAPPED, SWAPPED_SIZE, PROT_READ | PROT_WRITE, -1, + 0); + memset(p, 0xc4, SWAPPED_SIZE); + for (unsigned int tries = 0; tries < 200; tries++) { + if (madvise(p, SWAPPED_SIZE, MADV_COLD) || + madvise(p, SWAPPED_SIZE, MADV_PAGEOUT)) + die("pageout"); + missing = 0; + for (unsigned long i = 0; i < SWAPPED_SIZE / ps; i++) { + if (pread(pm, &entry, sizeof(entry), + (SWAPPED / ps + i) * 8) != 8) + die("pagemap"); + missing += !(entry & (1ULL << 62)); + } + if (!missing) + break; + usleep(10000); + } + close(pm); + if (missing) + _exit(124); + printf("# %luK dump precondition: all %lu leaves swapped\n", + ps / 1024, SWAPPED_SIZE / ps); + } + /* The file VMAs remain unfaulted, so dumping must retrieve file data. */ + raise(SIGABRT); + _exit(123); +} + +static Elf64_Phdr *find_load(unsigned char *data, Elf64_Ehdr *elf, + unsigned long addr) +{ + Elf64_Phdr *ph = (void *)(data + elf->e_phoff); + + for (unsigned int i = 0; i < elf->e_phnum; i++) + if (ph[i].p_type == PT_LOAD && addr >= ph[i].p_vaddr && + addr - ph[i].p_vaddr < ph[i].p_memsz) + return ph + i; + return NULL; +} + +static bool bytes_at(unsigned char *data, Elf64_Ehdr *elf, size_t len, + unsigned long addr, size_t count, unsigned char expected) +{ + Elf64_Phdr *p = find_load(data, elf, addr); + unsigned long off; + + if (!p || addr - p->p_vaddr + count > p->p_filesz) + return false; + off = p->p_offset + addr - p->p_vaddr; + if (off > len || count > len - off) + return false; + for (size_t i = 0; i < count; i++) + if (data[off + i] != expected) + return false; + return true; +} + +static void inspect(pid_t pid, unsigned long ps, bool headers) +{ + char path[64]; + struct stat st; + unsigned char *data; + Elf64_Ehdr *elf; + Elf64_Phdr *ph, *p; + bool geometry = true, aux = false; + unsigned int file = 0; + unsigned long vdso = 0; + int fd; + + snprintf(path, sizeof(path), "/core.%d", pid); + fd = open(path, O_RDONLY); + if (fd < 0 || fstat(fd, &st) || st.st_size < (off_t)sizeof(*elf)) + ksft_exit_fail_msg("missing/short core %s\n", path); + data = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (data == MAP_FAILED) + die("core read"); + elf = (void *)data; + if (memcmp(elf->e_ident, ELFMAG, SELFMAG) || elf->e_type != ET_CORE || + elf->e_phentsize != sizeof(*ph) || + elf->e_phoff > (uint64_t)st.st_size || + elf->e_phnum * sizeof(*ph) > (uint64_t)st.st_size - elf->e_phoff) + ksft_exit_fail_msg("invalid core ELF\n"); + ph = (void *)(data + elf->e_phoff); + for (unsigned int i = 0; i < elf->e_phnum; i++) { + if (ph[i].p_type == PT_LOAD) { + geometry &= ph[i].p_align == ps && + ph[i].p_filesz <= ph[i].p_memsz && + ph[i].p_offset % ps == ph[i].p_vaddr % ps; + } else if (ph[i].p_type == PT_NOTE) { + size_t at = ph[i].p_offset, end = at + ph[i].p_filesz; + + if (end > (size_t)st.st_size || end < at) + ksft_exit_fail_msg("invalid notes\n"); + while (at + sizeof(Elf64_Nhdr) <= end) { + Elf64_Nhdr *n = (void *)(data + at); + size_t desc = at + sizeof(*n) + + ((n->n_namesz + 3UL) & ~3UL); + uint64_t *words = (void *)(data + desc); + + if (desc > end || n->n_descsz > end - desc) + ksft_exit_fail_msg( + "invalid note length\n"); + if (n->n_type == NT_AUXV) { + for (size_t j = 0; + j + 1 < n->n_descsz / 8; j += 2) { + if (words[j] == AT_PAGESZ) + aux = words[j + 1] == + ps; + else if (words[j] == + AT_SYSINFO_EHDR) + vdso = words[j + 1]; + } + } + if (n->n_type == NT_FILE && n->n_descsz >= 16 && + words[0] <= (n->n_descsz / 8 - 2) / 3 && + words[1] == ps) { + for (size_t j = 0; j < words[0]; j++) { + unsigned long addr = + words[2 + 3 * j]; + unsigned long offset = + words[4 + 3 * j]; + + if (addr == FILEMAP && + offset == (headers ? 0UL : + 1UL)) + file |= 1; + if (addr == FILEMAP + 2 * ps && + offset == 2) + file |= 2; + if (addr == FILEMAP + 8 * ps && + offset == 17) + file |= 4; + } + } + at = desc + ((n->n_descsz + 3UL) & ~3UL); + } + } + } + ksft_test_result(geometry, "%luK core PT_LOAD granule/alignment%s\n", + ps / 1024, headers ? " headers-only" : ""); + ksft_test_result(aux && file == 7, + "%luK core AUXV and NT_FILE offsets%s\n", ps / 1024, + headers ? " headers-only" : ""); + if (headers) { + bool skipped; + + p = find_load(data, elf, FILEMAP + 2 * ps); + skipped = p && !p->p_filesz; + p = find_load(data, elf, FILEMAP + 8 * ps); + ksft_test_result( + skipped && p && !p->p_filesz, + "%luK header filter excludes nonzero quarter/native file offsets\n", + ps / 1024); + p = find_load(data, elf, FILEMAP + 12 * ps); + ksft_test_result( + p && p->p_filesz == ps && + p->p_offset + ps <= (uint64_t)st.st_size && + !memcmp(data + p->p_offset, ELFMAG, SELFMAG), + "%luK header filter recognizes non-executable ELF file\n", + ps / 1024); + p = find_load(data, elf, FILEMAP); + ksft_test_result( + p && p->p_filesz == ps && + p->p_offset + ps <= (uint64_t)st.st_size && + !memcmp(data + p->p_offset, ELFMAG, SELFMAG), + "%luK executable file header limited to one user page\n", + ps / 1024); + } else { + bool anon = true, files = true; + + p = find_load(data, elf, vdso); + ksft_test_result(p && p->p_filesz >= SELFMAG && + p->p_offset + p->p_filesz <= + (uint64_t)st.st_size && + !memcmp(data + p->p_offset, ELFMAG, + SELFMAG), + "%luK core vDSO ELF bytes\n", ps / 1024); + ksft_test_result(bytes_at(data, elf, st.st_size, SWAPPED, + SWAPPED_SIZE, 0xc4), + "%luK core swapped anonymous bytes\n", + ps / 1024); + + for (unsigned int i = 0; i < 4; i++) + anon &= bytes_at(data, elf, st.st_size, ANON + i * ps, + ps, 0x31 + i); + for (unsigned int i = 1; i < 4; i++) + files &= bytes_at(data, elf, st.st_size, + FILEMAP + (i == 1 ? 0 : i * ps), ps, + 0x81 + i); + ksft_test_result( + anon, "%luK core anonymous and PROT_NONE/READ bytes\n", + ps / 1024); + files &= bytes_at(data, elf, st.st_size, FILEMAP + 8 * ps, ps, + 0x81 + 17); + ksft_test_result(files, "%luK core unfaulted file quarters\n", + ps / 1024); + ksft_test_result( + bytes_at(data, elf, st.st_size, SPARSE, ps, 0x65) && + bytes_at(data, elf, st.st_size, SPARSE + ps, + SPARSE_SIZE - ps, 0), + "%luK core sparse and zero-page bytes\n", ps / 1024); + ksft_test_result( + st.st_blocks * 512 < st.st_size / 2, + "%luK core keeps untouched anonymous range sparse\n", + ps / 1024); + p = find_load(data, elf, EXCLUDED); + ksft_test_result(p && !p->p_filesz, + "%luK core DONTDUMP excludes data\n", + ps / 1024); + } + munmap(data, st.st_size); + close(fd); + unlink(path); +} + +int main(int argc, char **argv) +{ + unsigned long native_size = getauxval(AT_PAGESZ); + unsigned long sizes[] = { native_size, 4096, 16384 }; + unsigned int modes = native_size == 65536 ? 3 : 2; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && !strcmp(argv[1], "--crash")) + crash(false, 0); + if (argc == 2 && !strcmp(argv[1], "--headers")) + crash(true, 0); + if (argc == 2 && !strcmp(argv[1], "--limit")) + crash(false, 1); + if (argc == 2 && !strcmp(argv[1], "--below-limit")) + crash(false, -1); + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Use only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) && errno != EBUSY) + die("proc"); + if (mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + swapon("/dev/vda", 0)) + die("VM swap setup"); + write_text("/proc/sys/kernel/core_pattern", "/core.%p"); + ksft_print_header(); + ksft_set_plan(16 * modes); + for (unsigned int mode = 0; mode < modes; mode++) { + unsigned long ps = sizes[mode]; + for (unsigned int headers = 0; headers < 4; headers++) { + pid_t pid = fork(); + int status = 0; + + if (!pid) { + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + ps == native_size ? 0UL : ps, 0UL, + 0UL, 0UL)) + die("exec size"); + execl("/init", "/init", + headers == 3 ? "--below-limit" : + headers == 2 ? "--limit" : + headers ? "--headers" : + "--crash", + NULL); + die("exec"); + } + if (pid < 0 || waitpid(pid, &status, 0) != pid || + !WIFSIGNALED(status) || + WTERMSIG(status) != SIGABRT || + (headers < 3 && !WCOREDUMP(status))) + ksft_exit_fail_msg( + "child/core failure status=%#x\n", + status); + if (headers >= 2) { + char path[64]; + struct stat st; + bool exists, no_data = false; + + snprintf(path, sizeof(path), "/core.%d", pid); + exists = !stat(path, &st); + if (exists && headers == 2) { + Elf64_Ehdr elf; + Elf64_Phdr ph; + uint64_t data_start = UINT64_MAX; + int fd = open(path, O_RDONLY); + + if (fd < 0 || + pread(fd, &elf, sizeof(elf), 0) != + sizeof(elf)) + die("limited core header"); + for (unsigned int i = 0; + i < elf.e_phnum; i++) { + if (pread(fd, &ph, sizeof(ph), + elf.e_phoff + + i * sizeof(ph)) != + sizeof(ph)) + die("limited core phdr"); + if (ph.p_type == PT_LOAD && + ph.p_offset < data_start) + data_start = + ph.p_offset; + } + close(fd); + /* Sparse note padding is not charged to RLIMIT_CORE. */ + no_data = data_start != UINT64_MAX && + st.st_size <= + (off_t)data_start; + printf("# limited core: bytes=%lld first-data=%llu limit=%lu\n", + (long long)st.st_size, + (unsigned long long)data_start, + ps); + } + ksft_test_result( + headers == 2 ? + exists && st.st_size > 0 && + no_data : + !exists && errno == ENOENT, + "%luK core RLIMIT_CORE %s one user page\n", + ps / 1024, + headers == 2 ? "at" : "below"); + unlink(path); + } else { + inspect(pid, ps, headers); + } + } + } + ksft_print_cnts(); + printf("%s - core fixture complete\n", + ksft_get_fail_cnt() ? "not ok" : "ok"); + reboot(RB_POWER_OFF); + return ksft_get_fail_cnt() ? 1 : 0; +} diff --git a/tools/testing/selftests/arm64/user4k/cpu-granule-init.c b/tools/testing/selftests/arm64/user4k/cpu-granule-init.c new file mode 100644 index 0000000000000..1e983f91f74c7 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/cpu-granule-init.c @@ -0,0 +1,187 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +static int failed; +static void result(int ok, const char *name) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", name); + failed += !ok; +} +static int pin_cpu(pid_t pid, unsigned int cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + return sched_setaffinity(pid, sizeof(set), &set); +} +static int online_cpu1(void) +{ + char c = '?'; + int fd = open("/sys/devices/system/cpu/cpu1/online", O_RDONLY); + if (fd >= 0) { + if (read(fd, &c, 1) != 1) + c = '?'; + close(fd); + } + return c == '0' ? 0 : c == '1' ? 1 : -1; +} +static int worker(int ready, int command, unsigned long expected) +{ + unsigned char *p; + unsigned long ps = getauxval(AT_PAGESZ); + char cpu; + int ok = getppid() == 1 && ps == expected; + + p = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 2; + for (unsigned long i = 0; i < 4 * ps; i++) + p[i] = i ^ (i >> 12); + if (write(ready, "r", 1) != 1 || read(command, &cpu, 1) != 1) + return 3; + ok &= sched_getcpu() == cpu - '0'; + for (unsigned int round = 0; round < 128; round++) { + for (unsigned long i = 0; i < 4 * ps; i++) + if (p[i] != (unsigned char)(i ^ (i >> 12))) + ok = 0; + sched_yield(); + } + munmap(p, 4 * ps); + return ok ? 0 : 4; +} +static int compat_sample(unsigned long expected) +{ + uint64_t data[4] = { 0 }; + int pipefd[2], status; + pid_t child; + ssize_t size; + + if (pipe(pipefd)) + return 0; + child = fork(); + if (!child) { + char fd[32]; + close(pipefd[0]); + snprintf(fd, sizeof(fd), "%d", pipefd[1]); + execl("/compat-aslr", "/compat-aslr", "--sample", fd, NULL); + _exit(2); + } + close(pipefd[1]); + size = read(pipefd[0], data, sizeof(data)); + close(pipefd[0]); + return child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && + size == sizeof(data) && data[0] == expected; +} +int main(int argc, char **argv) +{ + if (argc == 5 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10)); + if (getpid() != 1 || argc != 1) { + fprintf(stderr, + "Dedicated two-CPU disposable VM with maxcpus=1 only\n"); + return 2; + } + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) { + perror("mount"); + failed++; + goto done; + } + unsigned long native = getauxval(AT_PAGESZ); + int has4k = !prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL); + result(!prctl(SET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL), + "native exec default restored"); + char cmdline[4096] = { 0 }; + int fd = open("/proc/cmdline", O_RDONLY); + if (fd < 0 || read(fd, cmdline, sizeof(cmdline) - 1) < 0) { + failed++; + goto done; + } + close(fd); + int reject = has4k && strstr(cmdline, "arm64.user4k_deny_late_cpu=1"); + result(online_cpu1() == 0, "CPU1 starts offline"); + if (online_cpu1() != 0) + goto done; + int ready[2], command[2]; + if (pipe(ready) || pipe(command)) { + failed++; + goto done; + } + pid_t child = fork(); + if (!child) { + char ready_fd[32], command_fd[32], ps[32]; + close(ready[0]); + close(command[1]); + snprintf(ready_fd, sizeof(ready_fd), "%d", ready[1]); + snprintf(command_fd, sizeof(command_fd), "%d", command[0]); + snprintf(ps, sizeof(ps), "%lu", has4k ? 4096UL : native); + if (prctl(SET_EXEC_PAGE_SIZE, has4k ? 4096UL : 0UL, 0UL, 0UL, + 0UL)) + _exit(5); + execl("/init", "/init", "--worker", ready_fd, command_fd, ps, + NULL); + _exit(6); + } + close(ready[1]); + close(command[0]); + if (child < 0) { + failed++; + goto done; + } + char token; + result(read(ready[0], &token, 1) == 1 && token == 'r', + "worker has live mappings before hotplug"); + close(ready[0]); + fd = open("/sys/devices/system/cpu/cpu1/online", O_WRONLY); + errno = 0; + ssize_t written = fd < 0 ? -1 : write(fd, "1", 1); + int saved_errno = errno; + if (fd >= 0) + close(fd); + int online = online_cpu1(); + printf("# has4k=%d expected-reject=%d write=%zd errno=%d online=%d\n", + has4k, reject, written, saved_errno, online); + result(reject ? written < 0 && online == 0 : + written == 1 && online == 1, + "late CPU admission matches advertised user granule"); + unsigned int cpu = online == 1 ? 1 : 0; + result(!pin_cpu(child, cpu), "worker affinity targets admitted CPU"); + token = '0' + cpu; + result(write(command[1], &token, 1) == 1, + "worker resumes after CPU admission"); + close(command[1]); + int status; + result(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status), + "live mapping bytes survive and worker runs on expected CPU"); + result(!pin_cpu(0, cpu), "launcher affinity targets admitted CPU"); + result(compat_sample(has4k ? 4096 : native), + "AArch32 exec retains selected or native-fallback granule"); +done: + printf("CPU GRANULE %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/cpu-granule16-init.c b/tools/testing/selftests/arm64/user4k/cpu-granule16-init.c new file mode 100644 index 0000000000000..ecf682a22f732 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/cpu-granule16-init.c @@ -0,0 +1,192 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +static int failed; +static void result(int ok, const char *name) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", name); + failed += !ok; +} +static int pin_cpu(pid_t pid, unsigned int cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + return sched_setaffinity(pid, sizeof(set), &set); +} +static int online_cpu1(void) +{ + char c = '?'; + int fd = open("/sys/devices/system/cpu/cpu1/online", O_RDONLY); + if (fd >= 0) { + if (read(fd, &c, 1) != 1) + c = '?'; + close(fd); + } + return c == '0' ? 0 : c == '1' ? 1 : -1; +} +static int worker(int ready, int command, unsigned long expected) +{ + unsigned char *p; + unsigned long ps = getauxval(AT_PAGESZ); + char cpu; + int ok = getppid() == 1 && ps == expected; + + p = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 2; + for (unsigned long i = 0; i < 4 * ps; i++) + p[i] = i ^ (i >> 12); + if (write(ready, "r", 1) != 1 || read(command, &cpu, 1) != 1) + return 3; + ok &= sched_getcpu() == cpu - '0'; + for (unsigned int round = 0; round < 128; round++) { + for (unsigned long i = 0; i < 4 * ps; i++) + if (p[i] != (unsigned char)(i ^ (i >> 12))) + ok = 0; + sched_yield(); + } + munmap(p, 4 * ps); + return ok ? 0 : 4; +} +static int compat_sample(unsigned long expected) +{ + uint64_t data[4] = { 0 }; + int pipefd[2], status; + pid_t child; + ssize_t size; + + if (pipe(pipefd)) + return 0; + child = fork(); + if (!child) { + char fd[32]; + close(pipefd[0]); + snprintf(fd, sizeof(fd), "%d", pipefd[1]); + execl("/compat-aslr", "/compat-aslr", "--sample", fd, NULL); + _exit(2); + } + close(pipefd[1]); + size = read(pipefd[0], data, sizeof(data)); + close(pipefd[0]); + return child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && + size == sizeof(data) && data[0] == expected; +} +int main(int argc, char **argv) +{ + if (argc == 5 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10)); + if (getpid() != 1 || argc != 1) { + fprintf(stderr, + "Dedicated two-CPU disposable VM with maxcpus=1 only\n"); + return 2; + } + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) { + perror("mount"); + failed++; + goto done; + } + unsigned long native = getauxval(AT_PAGESZ); + int has4k = !prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL); + int has16k = native > 16384 && + !prctl(SET_EXEC_PAGE_SIZE, 16384UL, 0UL, 0UL, 0UL); + result(!prctl(SET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL), + "native exec default restored"); + char cmdline[4096] = { 0 }; + int fd = open("/proc/cmdline", O_RDONLY); + if (fd < 0 || read(fd, cmdline, sizeof(cmdline) - 1) < 0) { + failed++; + goto done; + } + close(fd); + int reject = has16k && strstr(cmdline, "arm64.user16k_deny_late_cpu=1"); + result(online_cpu1() == 0, "CPU1 starts offline"); + if (online_cpu1() != 0) + goto done; + int ready[2], command[2]; + if (pipe(ready) || pipe(command)) { + failed++; + goto done; + } + pid_t child = fork(); + if (!child) { + char ready_fd[32], command_fd[32], ps[32]; + close(ready[0]); + close(command[1]); + snprintf(ready_fd, sizeof(ready_fd), "%d", ready[1]); + snprintf(command_fd, sizeof(command_fd), "%d", command[0]); + snprintf(ps, sizeof(ps), "%lu", has16k ? 16384UL : native); + if (prctl(SET_EXEC_PAGE_SIZE, has16k ? 16384UL : 0UL, 0UL, 0UL, + 0UL)) + _exit(5); + execl("/init", "/init", "--worker", ready_fd, command_fd, ps, + NULL); + _exit(6); + } + close(ready[1]); + close(command[0]); + if (child < 0) { + failed++; + goto done; + } + char token; + result(read(ready[0], &token, 1) == 1 && token == 'r', + "worker has live mappings before hotplug"); + close(ready[0]); + fd = open("/sys/devices/system/cpu/cpu1/online", O_WRONLY); + errno = 0; + ssize_t written = fd < 0 ? -1 : write(fd, "1", 1); + int saved_errno = errno; + if (fd >= 0) + close(fd); + int online = online_cpu1(); + printf("# has16k=%d expected-reject=%d write=%zd errno=%d online=%d\n", + has16k, reject, written, saved_errno, online); + result(reject ? written < 0 && online == 0 : + written == 1 && online == 1, + "late CPU admission matches advertised user granule"); + unsigned int cpu = online == 1 ? 1 : 0; + result(!pin_cpu(child, cpu), "worker affinity targets admitted CPU"); + token = '0' + cpu; + result(write(command[1], &token, 1) == 1, + "worker resumes after CPU admission"); + close(command[1]); + int status; + result(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status), + "live mapping bytes survive and worker runs on expected CPU"); + result(!pin_cpu(0, cpu), "launcher affinity targets admitted CPU"); + if (has16k) + result(!prctl(SET_EXEC_PAGE_SIZE, 16384UL, 0UL, 0UL, 0UL), + "16K request staged before compat exec"); + result(compat_sample(has4k ? 4096 : native), + "AArch32 exec selects4K or native despite pending16K"); +done: + printf("CPU GRANULE %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/dio-fixture-init.c b/tools/testing/selftests/arm64/user4k/dio-fixture-init.c new file mode 100644 index 0000000000000..36ebec4350c76 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/dio-fixture-init.c @@ -0,0 +1,266 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static bool use_file; +static unsigned char pattern(size_t i) +{ + return (i * 17 + (i >> 12) * 29 + 71) & 255; +} +static void fill(unsigned char *p, size_t len) +{ + for (size_t i = 0; i < len; i++) + p[i] = pattern(i); +} +static bool check(const unsigned char *p, size_t len, size_t off) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != pattern(i + off)) { + ksft_print_msg("mismatch at %zu: %u vs %u\n", i + off, + p[i], pattern(i + off)); + return false; + } + return true; +} +static void report(bool ok, size_t ps, const char *name) +{ + ksft_test_result(ok, "%zuK %s (errno=%d)\n", ps / 1024, name, errno); +} +static bool aio_transfer(int fd, void *p, size_t len, off_t off, bool write) +{ + aio_context_t ctx = 0; + struct iocb cb = { .aio_lio_opcode = write ? IOCB_CMD_PWRITE : + IOCB_CMD_PREAD, + .aio_fildes = fd, + .aio_buf = (uintptr_t)p, + .aio_nbytes = len, + .aio_offset = off }; + struct iocb *ptr = &cb; + struct io_event ev; + struct timespec timeout = { .tv_sec = 10 }; + bool ok; + + if (syscall(__NR_io_setup, 8, &ctx)) + return false; + ok = syscall(__NR_io_submit, ctx, 1, &ptr) == 1; + if (ok) + ok = syscall(__NR_io_getevents, ctx, 1, 1, &ev, &timeout) == + 1 && + ev.res == (long)len && !ev.res2; + ok &= !syscall(__NR_io_destroy, ctx); + return ok; +} +static int exercise(void) +{ + size_t ps = getauxval(AT_PAGESZ), len = 2 * 1024 * 1024; + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + unsigned char *q = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + struct iovec vec[4]; + int fd = open(use_file ? "/fs/test" : "/dev/user4k-test", + O_RDWR | O_DIRECT | (use_file ? O_CREAT | O_TRUNC : 0), + 0600), + memfd; + bool ok; + ssize_t ret; + + ksft_print_header(); + ksft_set_plan(10); + ksft_print_msg("backing=%s\n", use_file ? "ext4" : "raw block"); + if ((ps != 4096 && ps != 16384) || p == MAP_FAILED || q == MAP_FAILED || + fd < 0) + ksft_exit_fail_msg("setup failed errno=%d\n", errno); + if (use_file && ftruncate(fd, 32 * 1024 * 1024)) + ksft_exit_fail_msg("ftruncate failed\n"); + fill(p, len); + memset(q, 0, len); + ok = pwrite(fd, p + ps, 3 * ps, 0) == (ssize_t)(3 * ps) && + pread(fd, q + ps, 3 * ps, 0) == (ssize_t)(3 * ps) && + check(q + ps, 3 * ps, ps); + report(ok, ps, "O_DIRECT exact user offsets and bytes"); + + vec[0] = (struct iovec){ p + 512, ps - 512 }; + vec[1] = (struct iovec){ NULL, 0 }; + vec[2] = (struct iovec){ p + 5 * ps, ps + 512 }; + vec[3] = (struct iovec){ p + 11 * ps + 512, 512 }; + ok = pwritev(fd, vec, 4, 65536) == (ssize_t)(2 * ps + 512); + memset(q, 0, 3 * ps); + ok &= pread(fd, q, 2 * ps + 512, 65536) == (ssize_t)(2 * ps + 512); + ok &= check(q, ps - 512, 512) && + check(q + ps - 512, ps + 512, 5 * ps) && + check(q + 2 * ps, 512, 11 * ps + 512); + report(ok, ps, "fragmented vectors and empty segment"); + + ok = pwrite(fd, p, len, 4 * 1024 * 1024) == (ssize_t)len; + memset(q, 0, len); + ok &= pread(fd, q, len, 4 * 1024 * 1024) == (ssize_t)len && + check(q, len, 0); + report(ok, ps, "multi-bio two-megabyte transfer"); + + ok = !mprotect(p + ps, ps, PROT_NONE); + ret = pwrite(fd, p, 2 * ps, 8 * 1024 * 1024); + ok &= use_file ? ret == -1 && errno == EFAULT : ret == (ssize_t)ps; + ok &= !mprotect(p + ps, ps, PROT_READ | PROT_WRITE); + if (use_file) + ok &= pwrite(fd, p, ps, 8 * 1024 * 1024) == (ssize_t)ps; + memset(q, 0, ps); + ok &= pread(fd, q, ps, 8 * 1024 * 1024) == (ssize_t)ps && + check(q, ps, 0); + report(ok, ps, "source fault and subsequent I/O recovery"); + + ok = !mprotect(q + ps, ps, PROT_NONE); + ret = pread(fd, q, 2 * ps, 4 * 1024 * 1024); + ok &= (use_file ? ret == -1 && errno == EFAULT : ret == (ssize_t)ps) && + check(q, ps, 0); + ok &= !mprotect(q + ps, ps, PROT_READ | PROT_WRITE); + report(ok, ps, "partial destination fault"); + + ok = !mprotect(p + ps, ps, PROT_NONE); + /* Only 256 accessible bytes: none can form a 512-byte device block. */ + ret = pwrite(fd, p + ps - 256, 512, 10 * 1024 * 1024); + ok &= ret == -1 && errno == EFAULT; + ok &= !mprotect(p + ps, ps, PROT_READ | PROT_WRITE); + report(ok, ps, "alignment trimming releases all short-fragment pins"); + + memfd = memfd_create("dio-shared", 0); + ok = memfd >= 0 && !ftruncate(memfd, 4 * ps); + unsigned char *shared = ok ? mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, memfd, 0) : + MAP_FAILED; + ok &= shared != MAP_FAILED; + if (ok) { + memset(shared, 0xa7, 4 * ps); + ok = pread(fd, shared + ps, 2 * ps, 4 * 1024 * 1024) == + (ssize_t)(2 * ps) && + !msync(shared, 4 * ps, MS_SYNC) && + pread(memfd, q, 4 * ps, 0) == (ssize_t)(4 * ps); + ok &= check(q + ps, 2 * ps, 0); + for (size_t i = 0; i < ps; i++) + ok &= q[i] == 0xa7 && q[3 * ps + i] == 0xa7; + munmap(shared, 4 * ps); + } + if (memfd >= 0) + close(memfd); + report(ok, ps, "shared-file destination dirtying and adjacent guards"); + + ok = aio_transfer(fd, p + ps, 7 * ps, 12 * 1024 * 1024, true); + memset(q, 0, 8 * ps); + ok &= aio_transfer(fd, q + ps, 7 * ps, 12 * 1024 * 1024, false) && + check(q + ps, 7 * ps, ps); + report(ok, ps, "asynchronous AIO read/write completion"); + + memset(q, 0x96, 8 * ps); + pid_t child = fork(); + int status = 0; + if (!child) { + _exit(aio_transfer(fd, q + ps, 7 * ps, 12 * 1024 * 1024, + false) && + check(q + ps, 7 * ps, ps) ? + 0 : + 1); + } + ok = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + for (size_t i = 0; i < 8 * ps; i++) + ok &= q[i] == 0x96; + report(ok, ps, "writable-pin COW after fork"); + + ok = true; + for (unsigned i = 0; i < 128 && ok; i++) { + ok = !madvise(q + ps, ps, MADV_DONTNEED) && + pread(fd, q + ps, ps, 4 * 1024 * 1024) == (ssize_t)ps && + check(q + ps, ps, 0); + ok &= !mprotect(q + 2 * ps, ps, PROT_NONE); + ok &= pread(fd, q + 2 * ps, ps, 4 * 1024 * 1024) == -1 && + errno == EFAULT; + ok &= !mprotect(q + 2 * ps, ps, PROT_READ | PROT_WRITE); + } + report(ok, ps, "128 reuse and failed-pin cleanup cycles"); + close(fd); + munmap(p, len); + munmap(q, len); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} +int main(int argc, char **argv) +{ + bool ok = true; + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && + (!strcmp(argv[1], "--test") || !strcmp(argv[1], "--file")) && + getppid() == 1) { + use_file = !strcmp(argv[1], "--file"); + return exercise(); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + for (unsigned disk = 0; disk < 2; disk++) { + char path[80], serial[64]; + unsigned maj, min; + snprintf(path, sizeof(path), "/sys/class/block/vd%c/serial", + 'a' + disk); + FILE *dev = fopen(path, "r"); + if (!dev || fscanf(dev, "%63s", serial) != 1) + return 4; + fclose(dev); + snprintf(path, sizeof(path), "/sys/class/block/vd%c/dev", + 'a' + disk); + dev = fopen(path, "r"); + if (!dev || fscanf(dev, "%u:%u", &maj, &min) != 2) + return 4; + fclose(dev); + const char *node = + !strcmp(serial, "au4k-raw") ? "/dev/user4k-test" : + !strcmp(serial, "au4k-ext4") ? "/dev/user4k-fs" : + NULL; + if (!node || mknod(node, S_IFBLK | 0600, makedev(maj, min))) + return 5; + } + if (mount("/dev/user4k-fs", "/fs", "ext4", 0, NULL)) + return 7; + for (unsigned mode = 0; mode < 4; mode++) { + unsigned small = mode & 1; + pid_t pid = fork(); + int status = 0; + if (!pid) { + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + small ? 4096UL : 0UL, 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", mode >= 2 ? "--file" : "--test", + NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + ok &= !umount("/fs"); + printf("%s - direct I/O fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/exec-fixture-init.c b/tools/testing/selftests/arm64/user4k/exec-fixture-init.c new file mode 100644 index 0000000000000..8a390d5df3ddc --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/exec-fixture-init.c @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* PID-1 runner for the experimental one-shot exec ABI in a disposable VM. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include + +static int exec_size(unsigned long size) +{ + return prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, size, 0UL, 0UL, 0UL); +} + +static int pending_size(void) +{ + return prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL); +} + +static int run(const char *path, const char *arg, unsigned long size) +{ + pid_t pid = fork(), got; + int status = 0; + + if (!pid) { + if (size && exec_size(size)) { + perror("set exec page size"); + _exit(125); + } + execl(path, path, arg, NULL); + perror(path); + _exit(126); + } + if (pid < 0) + return -1; + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + printf("# child %s %s: wait=%d status=%#x\n", path, arg ? arg : "", got, + status); + return got == pid && WIFEXITED(status) && !WEXITSTATUS(status) ? 0 : -1; +} + +static int argument_exec(unsigned long size) +{ + pid_t pid = fork(), got; + int status = 0; + + if (!pid) { + char *payload = malloc(140000), expected[16]; + unsigned int i; + + if (!payload || exec_size(size)) + _exit(120); + for (i = 0; i < 139999; i++) + payload[i] = 'a' + i % 23; + payload[139999] = 0; + if (size == 4096) { + execl("/init", "/init", "--too-long", payload, NULL); + if (errno != E2BIG || pending_size() != 4096) + _exit(121); + } + payload[120000] = 0; + if (setenv("ARG_PAYLOAD", payload, 1)) + _exit(122); + snprintf(expected, sizeof(expected), "%lu", + size ? size : 16384UL); + execl("/init", "/init", "--args-check", expected, payload, + NULL); + perror("argument exec"); + _exit(123); + } + if (pid < 0) + return -1; + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + printf("# large arguments from %lu to %lu: status=%#x\n", + getauxval(AT_PAGESZ), size ? size : 16384UL, status); + return got == pid && WIFEXITED(status) && !WEXITSTATUS(status) ? 0 : -1; +} + +int main(int argc, char **argv) +{ + bool ok; + int failures = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--args-check")) { + const char *env = getenv("ARG_PAYLOAD"); + unsigned int i; + + if (getauxval(AT_PAGESZ) != strtoul(argv[2], NULL, 10) || + pending_size() != 0 || strlen(argv[3]) != 120000 || !env || + strlen(env) != 120000) + return 1; + for (i = 0; i < 120000; i++) + if (argv[3][i] != 'a' + i % 23 || env[i] != argv[3][i]) + return 2; + return 0; + } + if (argc == 2 && !strcmp(argv[1], "--args-from-4k")) { + if (getauxval(AT_PAGESZ) != 4096) + return 1; + return argument_exec(4096) || argument_exec(0); + } + if (argc == 2 && !strcmp(argv[1], "--exec-cycle")) { + if (getauxval(AT_PAGESZ) != 4096 || pending_size() != 0) + return 1; + /* A successful exec consumes the request; ordinary exec is native. */ + execl("/init", "/init", "--native-after-4k", NULL); + return 2; + } + if (argc == 2 && !strcmp(argv[1], "--native-after-4k")) + return getauxval(AT_PAGESZ) != 16384 || pending_size() != 0; + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as /init in a disposable VM.\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) && errno != EBUSY) + return 3; + printf("# exec fixture: native AT_PAGESZ=%lu\n", getauxval(AT_PAGESZ)); + failures += run("/page-contract", NULL, 0) != 0; + ok = pending_size() == 0; + errno = 0; + ok &= exec_size(8192) == -1 && errno == EINVAL; + ok &= prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, 4096UL, 1UL, 0UL, 0UL) == -1; + ok &= exec_size(4096) == 0 && pending_size() == 4096; + execl("/does-not-exist", "/does-not-exist", NULL); + ok &= errno == ENOENT && pending_size() == 4096; + ok &= getauxval(AT_PAGESZ) == 16384; + printf("%s - request validation and failed exec preserve live geometry\n", + ok ? "ok" : "not ok"); + failures += !ok; + /* fork inherits the pending request; successful child exec consumes it. */ + failures += run("/page-contract-4k", "--expect-4k", 0) != 0; + ok = pending_size() == 4096 && exec_size(0) == 0 && pending_size() == 0; + printf("%s - child exec leaves parent request intact; reset works\n", + ok ? "ok" : "not ok"); + failures += !ok; + failures += run("/page-contract-dynamic-4k", "--expect-4k", 4096) != 0; + failures += run("/init", "--exec-cycle", 4096) != 0; + failures += run("/proc-memory", NULL, 0) != 0; + failures += run("/proc-memory", "--expect-4k", 4096) != 0; + failures += run("/proc-pages", NULL, 0) != 0; + failures += run("/proc-pages", "--expect-4k", 4096) != 0; + failures += run("/advice", NULL, 0) != 0; + failures += run("/advice", "--expect-4k", 4096) != 0; + failures += run("/memory-interfaces", NULL, 0) != 0; + failures += run("/memory-interfaces", "--expect-4k", 4096) != 0; + failures += run("/mixed-process", NULL, 0) != 0; + failures += run("/mixed-process", NULL, 4096) != 0; + failures += argument_exec(4096) != 0; + failures += run("/init", "--args-from-4k", 4096) != 0; + failures += run("/page-contract", NULL, 0) != 0; + printf("%s - exec fixture complete: failures=%d\n", + failures ? "not ok" : "ok", failures); + reboot(RB_POWER_OFF); + return failures ? 1 : 0; +} diff --git a/tools/testing/selftests/arm64/user4k/exec-policy-init.c b/tools/testing/selftests/arm64/user4k/exec-policy-init.c new file mode 100644 index 0000000000000..02fd3f9552237 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/exec-policy-init.c @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable guest PID1: per-thread one-shot and inherited exec preferences. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define REQUIRE(x) \ + do { \ + if (!(x)) { \ + fprintf(stderr, "FAIL line %d: %s errno=%d\n", \ + __LINE__, #x, errno); \ + exit(1); \ + } \ + } while (0) + +static int policy(int op, unsigned long value) +{ + return prctl(op, value, 0UL, 0UL, 0UL); +} + +static void state(unsigned long pages, unsigned long preferred, int pending) +{ + REQUIRE(getauxval(AT_PAGESZ) == pages); + REQUIRE(policy(PR_AURORA_GET_DEFAULT_PAGE_SIZE, 0) == (int)preferred); + REQUIRE(policy(PR_AURORA_GET_EXEC_PAGE_SIZE, 0) == pending); +} + +static void chain(int stage, unsigned long native, unsigned long preferred) +{ + char s[32], n[32], p[32]; + + snprintf(s, sizeof(s), "%d", stage); + snprintf(n, sizeof(n), "%lu", native); + snprintf(p, sizeof(p), "%lu", preferred); + execl("/init", "/init", "--chain", s, n, p, NULL); + REQUIRE(0); +} + +static void *thread_policy(void *arg) +{ + unsigned long preferred = *(unsigned long *)arg; + + REQUIRE(policy(PR_AURORA_GET_DEFAULT_PAGE_SIZE, 0) == (int)preferred); + REQUIRE(policy(PR_AURORA_GET_EXEC_PAGE_SIZE, 0) == (int)preferred); + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 0)); + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, 0)); + state(getauxval(AT_PAGESZ), 0, 0); + return NULL; +} + +static void *thread_exec(void *arg) +{ + unsigned long *sizes = arg; + + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, sizes[1])); + chain(0, sizes[0], sizes[1]); + return NULL; +} + +static int wait_child(pid_t pid) +{ + int status; + + REQUIRE(pid > 0); + REQUIRE(waitpid(pid, &status, 0) == pid); + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +static int run_preference(unsigned long native, unsigned long preferred) +{ + pid_t pid = fork(); + + if (!pid) { + pthread_t thread; + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, preferred)); + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, preferred)); + REQUIRE(!pthread_create(&thread, NULL, thread_policy, + &preferred)); + REQUIRE(!pthread_join(thread, NULL)); + state(native, preferred, preferred); + /* Rejected values and reserved arguments must preserve both policies. */ + REQUIRE(policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 8192) == -1 && + errno == EINVAL); + REQUIRE(policy(PR_AURORA_SET_EXEC_PAGE_SIZE, 8192) == -1 && + errno == EINVAL); + int ops[] = { PR_AURORA_SET_DEFAULT_PAGE_SIZE, + PR_AURORA_GET_DEFAULT_PAGE_SIZE, + PR_AURORA_SET_EXEC_PAGE_SIZE, + PR_AURORA_GET_EXEC_PAGE_SIZE }; + for (unsigned int i = 0; i < sizeof(ops) / sizeof(ops[0]); + i++) { + REQUIRE(prctl(ops[i], 0UL, 1UL, 0UL, 0UL) == -1 && + errno == EINVAL); + REQUIRE(prctl(ops[i], 0UL, 0UL, 1UL, 0UL) == -1 && + errno == EINVAL); + REQUIRE(prctl(ops[i], 0UL, 0UL, 0UL, 1UL) == -1 && + errno == EINVAL); + } + REQUIRE(policy(PR_AURORA_GET_DEFAULT_PAGE_SIZE, 1) == -1 && + errno == EINVAL); + REQUIRE(policy(PR_AURORA_GET_EXEC_PAGE_SIZE, 1) == -1 && + errno == EINVAL); + execl("/does-not-exist", "/does-not-exist", NULL); + REQUIRE(errno == ENOENT); + state(native, preferred, preferred); + chain(0, native, preferred); + } + return wait_child(pid); +} + +static int compat(unsigned long native, unsigned long preferred, int no4k) +{ + pid_t pid = fork(); + + if (!pid) { + char n[32], p[32], e[32]; + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, preferred)); + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, native)); + snprintf(n, sizeof(n), "%lu", native); + snprintf(p, sizeof(p), "%lu", preferred); + snprintf(e, sizeof(e), "%lu", no4k ? native : 4096UL); + execl("/compat-policy", "/compat-policy", n, p, e, NULL); + REQUIRE(0); + } + return wait_child(pid); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--chain")) { + int stage = atoi(argv[2]); + unsigned long n = strtoul(argv[3], NULL, 10); + unsigned long p = strtoul(argv[4], NULL, 10); + + state(stage == 2 || stage == 5 ? n : p, stage < 4 ? p : 0, 0); + if (stage == 5) + return 0; + if (stage == 1) { + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, 0)); + state(p, p, n); + } + if (stage == 3) { + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 0)); + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, p)); + } + chain(stage + 1, n, p); + } + if (argc == 4 && !strcmp(argv[1], "--compat-return")) { + unsigned long n = strtoul(argv[2], NULL, 10); + unsigned long p = strtoul(argv[3], NULL, 10); + state(p ? p : n, p, 0); + return 0; + } + REQUIRE(getpid() == 1); + REQUIRE(!mount("proc", "/proc", "proc", 0, NULL)); + state(native, 0, 0); + int no4k = argc == 2 && !strcmp(argv[1], "--no4k"); + unsigned long preferences[] = { 4096, 16384 }; + for (unsigned int i = 0; i < 2; i++) { + unsigned long p = preferences[i]; + if (p >= native || (no4k && p == 4096)) + continue; + int ok = run_preference(native, p); + printf("%s - exec policy native=%lu alternative=%lu\n", + ok ? "ok" : "not ok", native, p); + failed |= !ok; + pid_t pid = fork(); + if (!pid) { + pthread_t thread; + unsigned long sizes[] = { native, p }; + REQUIRE(!pthread_create(&thread, NULL, thread_exec, + sizes)); + for (;;) + pause(); + } + ok = wait_child(pid); + printf("%s - non-leader exec alternative=%lu\n", + ok ? "ok" : "not ok", p); + failed |= !ok; + ok = compat(native, p, no4k); + printf("%s - compat policy alternative=%lu no4k=%d\n", + ok ? "ok" : "not ok", p, no4k); + failed |= !ok; + } + if (no4k) { + int ok = policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 4096) == -1 && + errno == EOPNOTSUPP; + ok &= policy(PR_AURORA_SET_EXEC_PAGE_SIZE, 4096) == -1 && + errno == EOPNOTSUPP; + ok &= compat(native, 0, 1); + printf("%s - no4k preserves native compat restrictions\n", + ok ? "ok" : "not ok"); + failed |= !ok; + } + state(native, 0, 0); + printf("EXEC POLICY %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + return failed; +} diff --git a/tools/testing/selftests/arm64/user4k/fex-contract-init.c b/tools/testing/selftests/arm64/user4k/fex-contract-init.c new file mode 100644 index 0000000000000..3273af30eea7f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/fex-contract-init.c @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +int main(void) +{ + if (getpid() != 1) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + int failed = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)) { + perror("mount"); + failed = 1; + goto done; + } + pid_t child = fork(); + if (!child) { + setenv("HOME", "/root", 1); + setenv("PATH", "/usr/bin:/bin", 1); + setenv("XDG_RUNTIME_DIR", "/run", 1); + setenv("FEX_ROOTFS", "/", 1); + setenv("LD_PRELOAD", "/usr/lib/libfex-abi.so", 1); + if (prctl(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 4096UL, 0UL, 0UL, + 0UL) || + prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL)) + _exit(3); + execl("/usr/bin/FEXInterpreter", "/usr/bin/FEXInterpreter", + "/x86-contract", NULL); + perror("FEX exec"); + _exit(4); + } + int status = 0; + int ok = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - FEX 4K native-process / x86 workload status=%#x\n", + ok ? "ok" : "not ok", status); + failed |= !ok; +done: + printf("GRANULE FEX RUNTIME %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/fex-native-abi.c b/tools/testing/selftests/arm64/user4k/fex-native-abi.c new file mode 100644 index 0000000000000..86ccb6e389d9d --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/fex-native-abi.c @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +__attribute__((constructor)) static void check_abi(void) +{ + char path[1024]; + ssize_t n = readlink("/proc/self/exe", path, sizeof(path) - 1); + if (n < 0) + return; + path[n] = 0; + if (!strstr(path, "FEXInterpreter")) + return; + unsigned long ps = getauxval(AT_PAGESZ); + dprintf(2, "FEX NATIVE HOST PROCESS PAGE SIZE %lu\n", ps); + if (ps != 4096) + _exit(90); +} diff --git a/tools/testing/selftests/arm64/user4k/fex-x86-contract.c b/tools/testing/selftests/arm64/user4k/fex-x86-contract.c new file mode 100644 index 0000000000000..047a3c9d28ddb --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/fex-x86-contract.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0-only +typedef unsigned long size_t; +static long sc(long n, long a, long b, long c, long d, long e, long f) +{ + register long r10 __asm__("r10") = d, r8 __asm__("r8") = e, + r9 __asm__("r9") = f; + long r; + __asm__ volatile("syscall" + : "=a"(r) + : "a"(n), "D"(a), "S"(b), "d"(c), "r"(r10), "r"(r8), + "r"(r9) + : "rcx", "r11", "memory"); + return r; +} +static void say(const char *s, size_t n) +{ + sc(1, 1, (long)s, n, 0, 0, 0); +} +#define SAY(s) say(s, sizeof(s) - 1) +static void quit(int n) +{ + sc(231, n, 0, 0, 0, 0, 0); + for (;;) + ; +} +static int check(unsigned char *p, size_t size) +{ + for (size_t i = 0; i < size; i++) + if (p[i] != (unsigned char)(i ^ (i >> 12))) + return 0; + return 1; +} +long run(unsigned long *sp) +{ + unsigned long argc = *sp++, pagesz = 0; + sp += argc + 1; + unsigned long *env = sp; + while (*sp) + sp++; + sp++; + while (*sp) { + if (sp[0] == 6) + pagesz = sp[1]; + sp += 2; + } + if (pagesz != 4096) + return 10; + SAY("ok - FEX x86 guest 4K auxv\n"); + unsigned char *p = (void *)sc(9, 0, 12288, 3, 0x22, -1, 0); + if ((long)p < 0) + return 11; + for (size_t i = 0; i < 12288; i++) + p[i] = (unsigned char)(i ^ (i >> 12)); + if (sc(10, (long)p + 4096, 4096, 0, 0, 0, 0)) + return 12; + long child = sc(57, 0, 0, 0, 0, 0, 0); + int status = 0; + if (!child) { + *(volatile unsigned char *)(p + 4096) = 1; + quit(13); + } + if (child < 0 || sc(61, child, (long)&status, 0, 0, 0, 0) != child || + (status & 127) != 11) + return 14; + if (sc(10, (long)p + 4096, 4096, 3, 0, 0, 0) || !check(p, 12288)) + return 15; + SAY("ok - FEX x86 independent 4K protection fault\n"); + child = sc(57, 0, 0, 0, 0, 0, 0); + if (!child) { + p[4096] = 0x71; + quit(0); + } + if (child < 0 || sc(61, child, (long)&status, 0, 0, 0, 0) != child || + status || !check(p, 12288)) + return 16; + SAY("ok - FEX x86 fork COW\n"); + p = (void *)sc(25, (long)p, 12288, 16384, 1, 0, 0); + if ((long)p < 0 || !check(p, 12288)) + return 17; + for (size_t i = 12288; i < 16384; i++) + if (p[i]) + return 18; + SAY("ok - FEX x86 mremap preserve and zero extension\n"); + long fd = sc(319, (long)"fex-shared", 0, 0, 0, 0, 0); + if (fd < 0 || sc(77, fd, 12288, 0, 0, 0, 0)) + return 19; + unsigned char *a = (void *)sc(9, 0, 4096, 3, 1, fd, 4096), + *b = (void *)sc(9, 0, 4096, 3, 1, fd, 4096); + if ((long)a < 0 || (long)b < 0) + return 20; + a[37] = 0x65; + if (b[37] != 0x65) + return 21; + SAY("ok - FEX x86 memfd mapping at 4K file offset\n"); + unsigned char *code = (void *)sc(9, 0, 4096, 3, 0x22, -1, 0); + if ((long)code < 0) + return 22; + code[0] = 0xb8; + code[1] = 42; + code[2] = 0; + code[3] = 0; + code[4] = 0; + code[5] = 0xc3; + if (sc(10, (long)code, 4096, 5, 0, 0, 0) || + ((int (*)(void))code)() != 42) + return 23; + if (sc(10, (long)code, 4096, 3, 0, 0, 0)) + return 24; + code[1] = 99; + if (sc(10, (long)code, 4096, 5, 0, 0, 0) || + ((int (*)(void))code)() != 99) + return 25; + SAY("ok - FEX x86 JIT permission transition and code replacement\n"); + unsigned char random[32]; + if (sc(318, (long)random, 32, 0, 0, 0, 0) != 32) + return 26; + if (argc == 1) { + child = sc(57, 0, 0, 0, 0, 0, 0); + if (!child) { + char *args[] = { "/x86-contract", "--exec-child", 0 }; + sc(59, (long)args[0], (long)args, (long)env, 0, 0, 0); + quit(28); + } + if (child < 0 || + sc(61, child, (long)&status, 0, 0, 0, 0) != child || status) + return 27; + SAY("ok - FEX x86 fork and re-exec with 4K mappings\n"); + } + SAY("FEX X86 4K CONTRACT PASS\n"); + return 0; +} +__asm__(".global _start\n_start:\nmov %rsp,%rdi\nand $-16,%rsp\ncall run\nmov %rax,%rdi\nmov $231,%rax\nsyscall\n"); diff --git a/tools/testing/selftests/arm64/user4k/gem-fixture-init.c b/tools/testing/selftests/arm64/user4k/gem-fixture-init.c new file mode 100644 index 0000000000000..aadf343dc7e8d --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/gem-fixture-init.c @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static unsigned char value(size_t off) +{ + return 0x31 + (off / 4096) % 128; +} +static bool check(unsigned char *p, size_t len, size_t off) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != value(i + off)) { + ksft_print_msg("offset=%zu got=%u expected=%u\n", + i + off, p[i], value(i + off)); + return false; + } + return true; +} +static int exercise(size_t native) +{ + size_t ps = getauxval(AT_PAGESZ), + len = native < 8192 ? 65536 : 8 * native; + int fd = open("/dev/vgem", O_RDWR); + struct drm_mode_create_dumb create = { .width = 4096, + .height = len / (4096 * 4), + .bpp = 32 }; + struct drm_mode_map_dumb map = {}; + struct drm_prime_handle prime = { .flags = DRM_RDWR }; + unsigned char *p = MAP_FAILED, *q = MAP_FAILED, *alias = MAP_FAILED; + bool ok = fd >= 0 && !ioctl(fd, DRM_IOCTL_MODE_CREATE_DUMB, &create); + ksft_print_header(); + ksft_set_plan(12); + ksft_print_msg("ABI=%zuK\n", ps / 1024); + map.handle = create.handle; + ok &= !ioctl(fd, DRM_IOCTL_MODE_MAP_DUMB, &map) && create.size >= len; + if (ok) + p = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + map.offset); + ok &= p != MAP_FAILED; + ksft_test_result(ok, "full GEM shmem object mapping\n"); + q = mmap(NULL, 2 * ps, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + map.offset); + bool separate = q != MAP_FAILED; + if (separate) { + memset(q, 0x39, ps); + memset(q + ps, 0xa2, ps); + for (size_t i = 0; i < ps; i++) + separate &= q[i] == 0x39 && q[ps + i] == 0xa2; + munmap(q, 2 * ps); + q = MAP_FAILED; + } + ksft_test_result(separate, + "adjacent user leaves map distinct backing bytes\n"); + if (ok) { + for (size_t i = 0; i < len; i++) + p[i] = value(i); + ok = check(p, len, 0); + } + ksft_test_result(ok, "full object has distinct quarter data\n"); + if (ok) { + q = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + map.offset); + ok = q != MAP_FAILED && !mprotect(q + ps, ps, PROT_READ); + if (ok) + ok = check(q, len, 0); + if (q != MAP_FAILED) + munmap(q, len); + q = MAP_FAILED; + } + ksft_test_result( + ok, + "cold faults after mprotect VMA splitting preserve object offsets\n"); + if (p != MAP_FAILED) { + q = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + map.offset); + ok = q != MAP_FAILED && !munmap(q + ps, ps); + if (ok) + ok = check(q + 2 * ps, len - 2 * ps, 2 * ps) && + check(q, ps, 0); + if (q != MAP_FAILED) + munmap(q, len); + q = MAP_FAILED; + } else + ok = false; + ksft_test_result( + ok, + "cold faults after partial unmap preserve object offsets\n"); + prime.handle = create.handle; + ok = !ioctl(fd, DRM_IOCTL_PRIME_HANDLE_TO_FD, &prime); + if (ok) { + alias = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, + prime.fd, 0); + ok = alias != MAP_FAILED && check(alias, len, 0); + } + ksft_test_result(ok, "PRIME exported full mapping shares GEM data\n"); + if (alias != MAP_FAILED) { + q = mmap(NULL, ps, PROT_READ | PROT_WRITE, MAP_SHARED, prime.fd, + 3 * ps); + ok = q != MAP_FAILED; + if (ok) + ok = check(q, ps, 3 * ps); + if (q != MAP_FAILED) + munmap(q, ps); + q = MAP_FAILED; + } else + ok = false; + ksft_test_result( + ok, "PRIME partial mapping uses requested buffer offset\n"); + if (alias != MAP_FAILED) { + q = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, + prime.fd, 0); + void *target = mmap(NULL, len, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + ok = q != MAP_FAILED && target != MAP_FAILED; + if (ok) { + void *moved = mremap(q, len, len, + MREMAP_MAYMOVE | MREMAP_FIXED, + target); + ok = moved == target; + if (ok) { + q = target; + ok = check(q, len, 0); + } + } + if (q != MAP_FAILED) + munmap(q, len); + if (target != MAP_FAILED && target != q) + munmap(target, len); + q = MAP_FAILED; + } else + ok = false; + ksft_test_result( + ok, + "cold GEM faults after fixed mremap retain object offsets\n"); + if (alias != MAP_FAILED) { + pid_t child = fork(); + int status = 0; + if (!child) { + unsigned char *cold = mmap(NULL, len, + PROT_READ | PROT_WRITE, + MAP_SHARED, prime.fd, 0); + bool pass = cold != MAP_FAILED && check(cold, len, 0); + if (pass) + cold[0] = 0xe1; + _exit(pass ? 0 : 1); + } + ok = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && p[0] == 0xe1; + p[0] = value(0); + } else + ok = false; + ksft_test_result( + ok, "forked GEM mapping shares bytes and retains backing\n"); + if (alias != MAP_FAILED) { + q = mmap(NULL, 2 * ps, PROT_READ | PROT_WRITE, MAP_SHARED, + prime.fd, len - ps); + ok = q == MAP_FAILED && errno == EINVAL; + if (q != MAP_FAILED) + munmap(q, 2 * ps); + q = mmap(NULL, ps, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + map.offset + ps); + ok &= q == MAP_FAILED && errno == EINVAL; + if (q != MAP_FAILED) + munmap(q, ps); + q = MAP_FAILED; + } else + ok = false; + ksft_test_result(ok, "PRIME bounds and exact GEM offset rejection\n"); + if (alias != MAP_FAILED) { + size_t targets[] = { native, 4096, 16384 }; + + ok = true; + for (unsigned int t = 0; t < 3; t++) { + size_t target = targets[t]; + pid_t child; + int status = 0; + bool pair; + + if (target > native || target == ps || + (t && target == native)) + continue; + child = fork(); + if (!child) { + char number[24], length[24], pages[24]; + + snprintf(number, sizeof(number), "%d", + prime.fd); + snprintf(length, sizeof(length), "%zu", len); + snprintf(pages, sizeof(pages), "%zu", target); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + target == native ? 0UL : target, 0UL, + 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--shared", number, + length, pages, NULL); + _exit(4); + } + pair = child > 0 && + waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + for (size_t i = 0; i < len; i += 4096) { + pair &= p[i] == + (unsigned char)(0x80 + i / 4096); + p[i] = value(i); + } + ksft_print_msg( + "PRIME sharing %zuK -> %zuK status=%d bytes=%zu\n", + ps / 1024, target / 1024, status, len); + ok &= pair; + } + } else { + ok = false; + } + ksft_test_result( + ok, + "PRIME backing shared between native and alternative processes\n"); + struct drm_gem_close close_arg = { .handle = create.handle }; + ok = !ioctl(fd, DRM_IOCTL_GEM_CLOSE, &close_arg); + close(fd); + if (alias != MAP_FAILED) { + close(prime.fd); + ok &= check(alias, len, 0); + munmap(alias, len); + } else + ok = false; + if (p != MAP_FAILED) { + ok &= check(p, len, 0); + munmap(p, len); + } else + ok = false; + ksft_test_result( + ok, + "mappings retain object after handles and descriptors close\n"); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} +int main(int argc, char **argv) +{ + bool ok = true; + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--shared")) { + struct stat init, self; + int fd = atoi(argv[2]); + size_t len = strtoul(argv[3], NULL, 10); + if (stat("/proc/1/exe", &init) || + stat("/proc/self/exe", &self) || + init.st_ino != self.st_ino || init.st_dev != self.st_dev || + len < 65536 || len > 8 * 65536 || + getauxval(AT_PAGESZ) != strtoul(argv[4], NULL, 10)) + return 2; + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + bool pass = p != MAP_FAILED && check(p, len, 0); + if (pass) + for (size_t i = 0; i < len; i += 4096) + p[i] = 0x80 + i / 4096; + if (p != MAP_FAILED) + munmap(p, len); + return pass ? 0 : 1; + } + if (argc == 3 && !strcmp(argv[1], "--test") && getppid() == 1) + return exercise(strtoul(argv[2], NULL, 10)); + if (getpid() != 1 || argc != 1) + return 2; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + FILE *dev = fopen("/sys/class/drm/card0/dev", "r"); + unsigned maj, min; + if (!dev || fscanf(dev, "%u:%u", &maj, &min) != 2) + return 4; + fclose(dev); + if (mknod("/dev/vgem", S_IFCHR | 0600, makedev(maj, min))) + return 5; + size_t native = getauxval(AT_PAGESZ), + targets[] = { native, 4096, 16384 }; + + for (unsigned int t = 0; t < 3; t++) { + size_t target = targets[t]; + pid_t pid; + int status = 0; + + if (target > native || (t && target == native)) + continue; + pid = fork(); + if (!pid) { + char number[24]; + + snprintf(number, sizeof(number), "%zu", native); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + target == native ? 0UL : target, 0UL, 0UL, + 0UL)) + _exit(3); + execl("/init", "/init", "--test", number, NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("# GEM child ABI=%zuK native=%zuK status=%d\n", + target / 1024, native / 1024, status); + } + printf("%s - GEM fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/granule-proc-init.c b/tools/testing/selftests/arm64/user4k/granule-proc-init.c new file mode 100644 index 0000000000000..e70f8df18e57a --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/granule-proc-init.c @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +static int failed; +static void result(int ok, const char *s) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", s); + failed += !ok; +} +static long pss(void *p) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char *line = NULL; + size_t cap = 0; + unsigned long start, end; + int active = 0; + long value = -1; + if (!f) + return -1; + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) + active = (unsigned long)p >= start && + (unsigned long)p < end; + else if (active && sscanf(line, "Pss: %ld kB", &value) == 1) + break; + } + free(line); + fclose(f); + return value; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--peer")) { + int fd = atoi(argv[2]), ready = atoi(argv[3]), + release = atoi(argv[4]); + size_t native = strtoul(argv[5], NULL, 10); + unsigned char *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, native - 4096); + char token = 'a'; + int ok = getauxval(AT_PAGESZ) == 4096 && p != MAP_FAILED; + if (ok) { + for (unsigned i = 0; i < 4096; i++) + ok &= p[i] == 0x7b; + p[0] = 0xc9; + ok &= pss(p) == 2; + } + token = ok ? 'a' : 'x'; + if (write(ready, &token, 1) != 1) + return 3; + if (read(release, &token, 1) != 1) + return 4; + return ok ? 0 : 2; + } + if (getpid() != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + mount("proc", "/proc", "proc", 0, NULL); + size_t native = getauxval(AT_PAGESZ); + int fd = memfd_create("high-slot-pss", 0); + result(fd >= 0 && !ftruncate(fd, native), + "native file backing created"); + unsigned char *p = + mmap(NULL, native, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + result(0, "parent mapping"); + goto done; + } + memset(p, 0x7b, native); + long before = pss(p); + printf("# native=%zu initial PSS=%ldKB\n", native, before); + result(before == (long)native / 1024, + "native PSS accounts every physical slot"); + int ready[2], release[2]; + if (pipe(ready) || pipe(release)) { + result(0, "pipes"); + goto done; + } + pid_t child = fork(); + int status = 0; + if (!child) { + char f[24], r[24], w[24], n[24]; + snprintf(f, sizeof(f), "%d", fd); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(w, sizeof(w), "%d", release[0]); + snprintf(n, sizeof(n), "%zu", native); + close(ready[0]); + close(release[1]); + if (prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--peer", f, r, w, n, NULL); + _exit(5); + } + close(ready[1]); + close(release[0]); + char token = 0; + int ok = child > 0 && read(ready[0], &token, 1) == 1 && token == 'a'; + result(ok && p[native - 4096] == 0xc9, + "4K peer shares the final physical slot and observes 2KB PSS"); + long during = pss(p); + printf("# parent mixed PSS=%ldKB expected=%zuKB\n", during, + native / 1024 - 2); + result(during == (long)native / 1024 - 2, + "native PSS weights the high shared slot independently"); + token = 'b'; + if (write(release[1], &token, 1) != 1) + failed++; + ok = waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status); + result(ok && pss(p) == (long)native / 1024, + "PSS returns to native size after peer releases its mapping"); + FILE *f = fopen("/proc/self/smaps_rollup", "r"); + char buf[512]; + ok = f != NULL; + if (f) { + while (fgets(buf, sizeof(buf), f)) { + } + ok &= !ferror(f); + fclose(f); + } + result(ok, "smaps_rollup reads complete native slot accounting"); + munmap(p, native); +done: + printf("GRANULE PROC %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/granule-swap-init.c b/tools/testing/selftests/arm64/user4k/granule-swap-init.c new file mode 100644 index 0000000000000..5791f7e2c0ee9 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/granule-swap-init.c @@ -0,0 +1,248 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "swapfile-fixture.h" +#define SET_EXEC_PAGE_SIZE 0x41555001 +static unsigned char pattern(size_t i, unsigned round) +{ + return (i ^ (i >> 12) ^ (i >> 20) ^ round) & 255; +} +static int check(unsigned char *p, size_t size, unsigned round) +{ + for (size_t i = 0; i < size; i++) + if (p[i] != pattern(i, round)) { + printf("# byte=%zu got=%u expected=%u\n", i, p[i], + pattern(i, round)); + return 0; + } + return 1; +} +static unsigned long swapouts(void) +{ + FILE *f = fopen("/proc/vmstat", "r"); + char k[80]; + unsigned long n = 0, result = 0; + if (!f) + return 0; + while (fscanf(f, "%79s %lu", k, &n) == 2) + if (!strcmp(k, "pswpout")) + result = n; + fclose(f); + return result; +} +static unsigned swapped(void *p, size_t size, size_t ps) +{ + int fd = open("/proc/self/pagemap", O_RDONLY); + uint64_t bits; + unsigned n = 0; + if (fd < 0) + return 0; + for (size_t i = 0; i < size; i += ps) + if (pread(fd, &bits, 8, ((uintptr_t)p + i) / ps * 8) == 8) + n += (bits >> 62) & 1; + close(fd); + return n; +} +static int exercise(unsigned long expected) +{ + size_t ps = getauxval(AT_PAGESZ), size = 8 * 1024 * 1024; + unsigned char *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int failed = 0; + if (ps != expected || p == MAP_FAILED) { + printf("not ok - expected ABI=%lu actual=%zu\n", expected, ps); + return 1; + } + for (unsigned round = 0; round < 4; round++) { + for (size_t i = 0; i < size; i++) + p[i] = pattern(i, round); + unsigned long before = swapouts(); + int advice = 0; + unsigned n = 0, attempts = 0; + /* PAGEOUT is advisory. Require evidence, with a bounded retry window. */ + do { + advice = madvise(p, size, MADV_PAGEOUT); + attempts++; + if (advice) + break; + usleep(20000); + n = swapped(p, size, ps); + } while (!n && attempts < 100); + unsigned long written = swapouts() - before; + int ok = !advice && written > 0 && n > 0; + printf("%s - %zuK round%u real swapped PTEs=%u/%zu writes=%lu attempts=%u\n", + ok ? "ok" : "not ok", ps / 1024, round, n, size / ps, + written, attempts); + failed += !ok; + int release[2], ready[2]; + if (pipe(release) || pipe(ready)) + return 2; + pid_t child = fork(); + int status = 0; + if (!child) { + char token; + close(release[1]); + close(ready[0]); + int ok = read(release[0], &token, 1) == 1 && + check(p, size, round); + if (ok) { + for (size_t i = 0; i < size; i += ps) + p[i] ^= 0x6d; + } + token = ok ? 'a' : 'x'; + write(ready[1], &token, 1); + _exit(ok ? 0 : 3); + } + close(release[0]); + close(ready[1]); + ok = check(p, size, round); + printf("%s - %zuK round%u disk roundtrip bytes\n", + ok ? "ok" : "not ok", ps / 1024, round); + failed += !ok; + char token = 'b'; + write(release[1], &token, 1); + ok = read(ready[0], &token, 1) == 1 && token == 'a' && + waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status) && check(p, size, round); + printf("%s - %zuK round%u forked swap-in and COW retain parent bytes\n", + ok ? "ok" : "not ok", ps / 1024, round); + failed += !ok; + close(release[1]); + close(ready[0]); + } + munmap(p, size); + return failed ? 1 : 0; +} +static int run_suite(bool nonlinear) +{ + const char *base = + nonlinear ? + "/sys/kernel/debug/kunit/arm64-user4k-nonlinear-swap/" : + "/sys/kernel/debug/kunit/arm64-user4k-swap/"; + char path[160]; + snprintf(path, sizeof(path), "%srun", base); + int fd = open(path, O_WRONLY); + if (fd < 0) + return 1; + int ok = write(fd, "1", 1) == 1; + close(fd); + snprintf(path, sizeof(path), "%sresults", base); + FILE *f = fopen(path, "r"); + char *line = NULL; + size_t cap = 0; + int completed = 0; + if (!f) + return 1; + while (getline(&line, &cap, f) > 0) { + fputs(line, stdout); + if (strstr(line, "not ok") || strstr(line, "# SKIP")) + ok = 0; + if (strstr(line, nonlinear ? + "ok 1 arm64-user4k-nonlinear-swap" : + "ok 1 arm64-user4k-swap")) + completed = 1; + } + free(line); + fclose(f); + return !(ok && completed); +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && (!strcmp(argv[1], "--swapoff-helper") || + !strcmp(argv[1], "--swapon-helper"))) { + struct stat a, b; + if (getpid() == 1 || stat("/proc/1/exe", &a) || + stat("/proc/self/exe", &b) || a.st_dev != b.st_dev || + a.st_ino != b.st_ino) + return 1; + return (!strcmp(argv[1], "--swapoff-helper") ? + swapoff(swap_path()) : + swapon(swap_path(), 0)) ? + 1 : + 0; + } + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 1; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || + (argc != 1 && (argc != 2 || (strcmp(argv[1], "--el0-only") && + strcmp(argv[1], "--nonlinear") && + strcmp(argv[1], "--swapfile"))))) { + fprintf(stderr, "Dedicated disposable VM PID1 only\n"); + return 2; + } + int failed = 0, active = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) { + failed++; + goto done; + } + struct stat st; + for (int i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + if (stat("/dev/vda", &st) || !S_ISBLK(st.st_mode) || + (argc == 2 && !strcmp(argv[1], "--swapfile") && + prepare_swapfile()) || + swapon(swap_path(), 0)) { + perror("swapon"); + failed++; + goto done; + } + active = 1; + printf("GRANULE SWAP BACKING=%s\n", swap_path()); + if (argc == 1 || (argc == 2 && !strcmp(argv[1], "--nonlinear"))) + failed += run_suite(argc == 2); + unsigned long sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned abi = 0; abi < (sizes[0] > 16384 ? 3U : 2U); abi++) { + pid_t c = fork(); + int status = 0; + if (!c) { + char expected[32]; + snprintf(expected, sizeof(expected), "%lu", sizes[abi]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[abi], 0UL, 0UL, + 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", expected, NULL); + _exit(5); + } + int ok = c > 0 && waitpid(c, &status, 0) == c && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - swap ABI %luK child status=%#x\n", + ok ? "ok" : "not ok", sizes[abi] / 1024, status); + failed += !ok; + } +done: + if (active && swapoff(swap_path())) { + perror("swapoff"); + failed++; + } + if (active && argc == 2 && !strcmp(argv[1], "--swapfile") && + umount("/swapdir")) { + perror("umount"); + failed++; + } + printf("GRANULE SWAP %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/granule-threeway-init.c b/tools/testing/selftests/arm64/user4k/granule-threeway-init.c new file mode 100644 index 0000000000000..71a459c9b5221 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/granule-threeway-init.c @@ -0,0 +1,248 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +struct snapshot { + long pss; + int first, last, status; +}; +struct peer { + pid_t pid; + int command, reply; + size_t size; +}; +static int failed; +static void result(int ok, const char *s) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", s); + failed += !ok; +} +static long pss(void *p) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char *line = NULL; + size_t cap = 0; + unsigned long start, end; + int active = 0; + long value = -1; + if (!f) + return -1; + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) + active = (unsigned long)p >= start && + (unsigned long)p < end; + else if (active && sscanf(line, "Pss: %ld kB", &value) == 1) + break; + } + free(line); + fclose(f); + return value; +} +static int transfer(int fd, void *data, size_t size, int writing) +{ + size_t done = 0; + while (done < size) { + ssize_t n = + writing ? write(fd, (char *)data + done, size - done) : + read(fd, (char *)data + done, size - done); + if (n < 0 && errno == EINTR) + continue; + if (n <= 0) + return 0; + done += n; + } + return 1; +} +static int worker(int fd, int command, int reply, size_t size) +{ + const struct rlimit no_core = { 0, 0 }; + setrlimit(RLIMIT_CORE, &no_core); + if (getppid() != 1 || getauxval(AT_PAGESZ) != size) + return 2; + unsigned char *p = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, + fd, 65536 - size); + if (p == MAP_FAILED) + return 3; + for (;;) { + char op; + struct snapshot s = { .pss = -1, .first = -1, .last = -1 }; + if (!transfer(command, &op, 1, 0)) + return 4; + if (op == 'x') { + s.status = munmap(p, size); + transfer(reply, &s, sizeof(s), 1); + return s.status ? 5 : 0; + } + if (op == 'r') + s.status = mprotect(p, size, PROT_READ); + else if (op == 'w') + p[size - 1] = 0x99; + else if (op == 'd') + s.status = madvise(p, size, MADV_DONTNEED); + else if (op == 't') { + pid_t c = fork(); + int status = 0; + if (!c) { + unsigned char value = *( + volatile unsigned char *)(p + size - 1); + _exit(value); + } + if (c < 0 || waitpid(c, &status, 0) != c) + s.status = -1; + else + s.status = WIFSIGNALED(status) ? + 256 + WTERMSIG(status) : + WIFEXITED(status) ? + WEXITSTATUS(status) : + -2; + } + if (op != 'd' && op != 't') { + s.first = p[0]; + s.last = p[size - 1]; + } + s.pss = pss(p); + if (!transfer(reply, &s, sizeof(s), 1)) + return 6; + } +} +static struct peer spawn(int fd, size_t size) +{ + int command[2], reply[2]; + struct peer p = { .pid = -1, .size = size }; + if (pipe(command) || pipe(reply)) + return p; + p.pid = fork(); + if (!p.pid) { + char f[24], c[24], r[24], s[24]; + close(command[1]); + close(reply[0]); + snprintf(f, sizeof(f), "%d", fd); + snprintf(c, sizeof(c), "%d", command[0]); + snprintf(r, sizeof(r), "%d", reply[1]); + snprintf(s, sizeof(s), "%zu", size); + if (prctl(SET_EXEC_PAGE_SIZE, size, 0UL, 0UL, 0UL)) + _exit(7); + execl("/init", "/init", "--worker", f, c, r, s, NULL); + _exit(8); + } + close(command[0]); + close(reply[1]); + p.command = command[1]; + p.reply = reply[0]; + return p; +} +static struct snapshot command(struct peer *p, char op) +{ + struct snapshot s = { .pss = -1, .first = -1, .last = -1, .status = -1 }; + if (p->pid <= 0 || !transfer(p->command, &op, 1, 1) || + !transfer(p->reply, &s, sizeof(s), 0)) + failed++; + return s; +} +static void finish(struct peer *p) +{ + struct snapshot s = command(p, 'x'); + int status = 0; + result(!s.status && waitpid(p->pid, &status, 0) == p->pid && + WIFEXITED(status) && !WEXITSTATUS(status), + "peer unmap and exit"); + close(p->command); + close(p->reply); +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (argc == 6 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), atoi(argv[4]), + strtoul(argv[5], NULL, 10)); + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable64K VM PID1 only\n"); + return 2; + } + mount("proc", "/proc", "proc", 0, NULL); + if (getauxval(AT_PAGESZ) != 65536) { + result(0, "native64K required"); + goto done; + } + int fd = memfd_create("three-granules", 0); + if (fd < 0 || ftruncate(fd, 65536)) { + result(0, "file setup"); + goto done; + } + unsigned char *p = + mmap(NULL, 65536, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + result(0, "native mapping"); + goto done; + } + memset(p, 0x42, 65536); + struct peer medium = spawn(fd, 16384); + struct snapshot s = command(&medium, 's'); + result(s.pss == 8 && s.first == 0x42 && s.last == 0x42 && pss(p) == 56, + "64K/16K initial sharing and PSS"); + struct peer small = spawn(fd, 4096); + s = command(&small, 's'); + result(s.pss == 1 && s.first == 0x42 && s.last == 0x42, + "4K peer shares the final slice with both larger ABIs"); + s = command(&medium, 's'); + result(s.pss == 7 && pss(p) == 55, + "three-way PSS weights partial overlaps independently"); + s = command(&small, 'r'); + result(!s.status, "4K alias becomes read-only"); + s = command(&medium, 'w'); + result(!s.status && p[65535] == 0x99, + "16K alias remains writable across local4K protection"); + s = command(&small, 's'); + result(s.last == 0x99, "read-only4K alias observes16K writes"); + s = command(&small, 'd'); + result(!s.status && s.pss == 0 && pss(p) == 56, + "4K discard removes only its mapping"); + s = command(&medium, 's'); + result(s.pss == 8 && s.last == 0x99, "16K mapping survives4K discard"); + s = command(&small, 's'); + result(s.pss == 1 && s.last == 0x99 && pss(p) == 55, + "4K refault restores coherent three-way sharing"); + result(!ftruncate(fd, 65536 - 4096), + "truncate at4K boundary inside16K leaf"); + s = command(&small, 't'); + result(s.status == 256 + SIGBUS, + "4K mapping wholly beyond EOF faults SIGBUS"); + s = command(&medium, 't'); + result(s.status == 0, "partial16K EOF leaf exposes zero tail"); + result(p[65535] == 0 && p[0] == 0x42, + "native64K sees zero tail and preserved file prefix"); + s = command(&medium, 's'); + result(s.first == 0x42 && s.last == 0 && s.pss == 8 && pss(p) == 56, + "16K EOF refault keeps correct mixed PSS"); + result(!ftruncate(fd, 65536), "restore file length"); + s = command(&small, 's'); + result(s.first == 0 && s.last == 0 && s.pss == 1 && pss(p) == 55, + "4K alias refaults zeroed extension"); + finish(&small); + s = command(&medium, 's'); + result(s.pss == 8 && pss(p) == 56, "16K alias survives4K exit"); + finish(&medium); + result(pss(p) == 64, + "native PSS restored after all alternative aliases exit"); + munmap(p, 65536); + close(fd); +done: + printf("THREE GRANULES %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/iommufd-init.c b/tools/testing/selftests/arm64/user4k/iommufd-init.c new file mode 100644 index 0000000000000..c678f35cf72a4 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/iommufd-init.c @@ -0,0 +1,1314 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable VM PID1: real IOMMUFD ioctls against mock AMDv1 page tables. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../drivers/iommu/iommufd/iommufd_test.h" + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define VA 0x200000000UL +#define IOVA 0x10000000UL +#define ALIAS 0x12000000UL +#define MAP_FLAGS \ + (IOMMU_IOAS_MAP_FIXED_IOVA | IOMMU_IOAS_MAP_READABLE | \ + IOMMU_IOAS_MAP_WRITEABLE) +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) + +static size_t native_ps, user_ps; +struct context { + int fd, access; + uint32_t ioas, hwpt[2], dev[2]; +}; + +static int drop_pin_cap(void) +{ + struct __user_cap_header_struct h = { + .version = _LINUX_CAPABILITY_VERSION_3 + }; + struct __user_cap_data_struct d[2]; + unsigned int mask = 1U << (CAP_IPC_LOCK % 32); + + if (syscall(SYS_capget, &h, d)) + return -1; + d[CAP_IPC_LOCK / 32].effective &= ~mask; + d[CAP_IPC_LOCK / 32].permitted &= ~mask; + return syscall(SYS_capset, &h, d); +} + +static int limit_bytes(size_t bytes) +{ + struct rlimit r; + + if (getrlimit(RLIMIT_MEMLOCK, &r)) + return -1; + r.rlim_cur = bytes; + return setrlimit(RLIMIT_MEMLOCK, &r); +} + +static long stat_kb(pid_t pid, const char *key) +{ + char path[80], line[256]; + long result = -1; + FILE *f; + + snprintf(path, sizeof(path), "/proc/%d/status", pid); + f = fopen(path, "r"); + if (!f) + return -1; + while (fgets(line, sizeof(line), f)) + if (!strncmp(line, key, strlen(key))) { + result = strtol(line + strlen(key), NULL, 10); + break; + } + fclose(f); + return result; +} + +static int new_domain(struct context *c, unsigned int index) +{ + struct iommu_test_cmd cmd = { .size = sizeof(cmd), + .op = IOMMU_TEST_OP_MOCK_DOMAIN, + .id = c->ioas }; + int rc = ioctl(c->fd, IOMMU_TEST_CMD, &cmd); + + if (!rc) { + c->hwpt[index] = cmd.mock_domain.out_hwpt_id; + c->dev[index] = cmd.mock_domain.out_idev_id; + } + return rc; +} + +static int context_open_common(struct context *c, unsigned int mm_account, + bool domain, bool pins) +{ + struct iommu_ioas_alloc a = { .size = sizeof(a) }; + struct iommu_option o = { .size = sizeof(o), + .option_id = IOMMU_OPTION_RLIMIT_MODE, + .op = IOMMU_OPTION_OP_SET, + .val64 = mm_account }; + struct iommu_test_cmd access = { .size = sizeof(access), + .op = IOMMU_TEST_OP_CREATE_ACCESS }; + + memset(c, 0, sizeof(*c)); + c->fd = open("/dev/iommu", O_RDWR); + if (c->fd < 0 || ioctl(c->fd, IOMMU_OPTION, &o) || + ioctl(c->fd, IOMMU_IOAS_ALLOC, &a)) + return -1; + c->ioas = a.out_ioas_id; + if (domain && new_domain(c, 0)) + return -1; + access.id = c->ioas; + access.create_access.flags = + pins ? MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES : 0; + if (ioctl(c->fd, IOMMU_TEST_CMD, &access)) + return -1; + c->access = access.create_access.out_access_fd; + return 0; +} + +static int context_open(struct context *c, unsigned int mm_account) +{ + return context_open_common(c, mm_account, true, false); +} + +static void context_close(struct context *c) +{ + close(c->access); + close(c->fd); +} + +static int map_range(struct context *c, void *p, size_t len, unsigned long iova) +{ + struct iommu_ioas_map m = { .size = sizeof(m), + .flags = MAP_FLAGS, + .ioas_id = c->ioas, + .user_va = (uintptr_t)p, + .length = len, + .iova = iova }; + + return ioctl(c->fd, IOMMU_IOAS_MAP, &m); +} + +static int unmap_range(struct context *c, size_t len, unsigned long iova) +{ + struct iommu_ioas_unmap u = { .size = sizeof(u), + .ioas_id = c->ioas, + .iova = iova, + .length = len }; + int rc = ioctl(c->fd, IOMMU_IOAS_UNMAP, &u); + + return rc ?: (u.length == len ? 0 : -1); +} + +static int check_pa(struct context *c, unsigned int dom, void *p, size_t len, + unsigned long iova) +{ + struct iommu_test_cmd cmd = { .size = sizeof(cmd), + .op = IOMMU_TEST_OP_MD_CHECK_MAP, + .id = c->hwpt[dom], + .check_map = { .iova = iova, + .length = len, + .uptr = (uintptr_t)p } }; + + return ioctl(c->fd, IOMMU_TEST_CMD, &cmd); +} + +static int access_rw(struct context *c, void *p, size_t len, unsigned long iova, + unsigned int flags) +{ + struct iommu_test_cmd cmd = { .size = sizeof(cmd), + .op = IOMMU_TEST_OP_ACCESS_RW, + .id = c->access, + .access_rw = { .iova = iova, + .length = len, + .uptr = (uintptr_t)p, + .flags = flags } }; + + return ioctl(c->fd, IOMMU_TEST_CMD, &cmd); +} + +static bool filled(const void *ptr, unsigned char byte, size_t n) +{ + const unsigned char *p = ptr; + + for (size_t i = 0; i < n; i++) + if (p[i] != byte) + return false; + return true; +} + +static void *relocated(size_t len, unsigned char byte) +{ + void *old = mmap((void *)VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, + 0); + + if (old == MAP_FAILED) + return old; + memset(old, byte, len); + size_t distance = len > 32 * native_ps ? len + native_ps : + 32 * native_ps; + + return mremap(old, len, len, MREMAP_MAYMOVE | MREMAP_FIXED, + (void *)(VA + distance + user_ps)); +} + +static int limits_and_lifetime(unsigned int mode) +{ + struct context c; + size_t len = 2 * native_ps; + unsigned char *p = relocated(3 * native_ps, 0x31), *buf = malloc(len); + struct iommu_ioas_copy copy; + + CHECK(p != MAP_FAILED && buf); + CHECK(!context_open(&c, mode)); + CHECK(!limit_bytes(len)); + CHECK(map_range(&c, p, 3 * native_ps, IOVA) == -1 && errno == ENOMEM); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!map_range(&c, p, len, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + CHECK(stat_kb(getpid(), "VmLck:") == (mode ? (long)len / 1024 : 0)); + CHECK(!check_pa(&c, 0, p, len, IOVA)); + CHECK(map_range(&c, p + len, native_ps, ALIAS) == -1 && + errno == ENOMEM); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + copy = (struct iommu_ioas_copy){ .size = sizeof(copy), + .flags = MAP_FLAGS, + .src_ioas_id = c.ioas, + .dst_ioas_id = c.ioas, + .src_iova = IOVA + user_ps, + .dst_iova = ALIAS, + .length = len - user_ps }; + CHECK(!ioctl(c.fd, IOMMU_IOAS_COPY, ©)); + CHECK(!new_domain(&c, 1)); + CHECK(!check_pa(&c, 1, p + user_ps, len - user_ps, ALIAS)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + CHECK(mmap(p, len, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0) == p); + memset(p, 0x5a, len); + CHECK(check_pa(&c, 0, p, len, IOVA) == -1 && errno == EINVAL); + CHECK(!access_rw(&c, buf, len, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x31, len)); + CHECK(!access_rw(&c, buf, len, IOVA, 0)); + CHECK(filled(buf, 0x5a, len)); + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == + (long)(user_ps < native_ps ? len : len - user_ps) / 1024); + CHECK(!unmap_range(&c, len - user_ps, ALIAS)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + /* Fresh anonymous backing follows the shifted VA's native linear index. + * A non-native-aligned replacement spans three physical backing pages. + */ + size_t replacement_pages = user_ps < native_ps ? 3 : 2; + + if (replacement_pages == 3) { + CHECK(map_range(&c, p, len, IOVA) == -1 && errno == ENOMEM); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + } + CHECK(!limit_bytes(replacement_pages * native_ps)); + CHECK(!map_range(&c, p, len, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == + (long)(replacement_pages * native_ps / 1024)); + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(!munmap(p, 3 * native_ps)); + context_close(&c); + free(buf); + return 0; +} + +static int holes_and_shared(unsigned int mode) +{ + struct context c; + size_t len = 3 * user_ps, file_len = 4 * native_ps; + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0), + *buf = malloc(file_len); + int file = memfd_create("iommu-shared", 0); + + CHECK(p != MAP_FAILED && buf && file >= 0); + CHECK(!context_open(&c, mode) && !limit_bytes(8 * native_ps)); + memset(p, 0x31, len); + CHECK(!munmap(p + user_ps, user_ps)); + CHECK(map_range(&c, p, len, IOVA) == -1 && errno == EFAULT); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(mmap(p + user_ps, user_ps, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, + 0) == p + user_ps); + CHECK(map_range(&c, p, len, IOVA) == -1 && errno == EFAULT); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!mprotect(p + user_ps, user_ps, PROT_READ | PROT_WRITE)); + CHECK(!map_range(&c, p, len, IOVA) && !check_pa(&c, 0, p, len, IOVA)); + CHECK(!unmap_range(&c, len, IOVA) && !munmap(p, len)); + CHECK(!ftruncate(file, file_len)); + p = mmap(NULL, 2 * user_ps, PROT_READ | PROT_WRITE, MAP_SHARED, file, + user_ps); + CHECK(p != MAP_FAILED); + CHECK(!map_range(&c, p, 2 * user_ps, IOVA)); + CHECK(!check_pa(&c, 0, p, 2 * user_ps, IOVA)); + CHECK(!munmap(p, 2 * user_ps)); + memset(buf, 0x93, 2 * user_ps); + CHECK(!access_rw(&c, buf, 2 * user_ps, IOVA, + MOCK_ACCESS_RW_WRITE | MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(!unmap_range(&c, 2 * user_ps, IOVA)); + CHECK(pread(file, buf, file_len, 0) == (ssize_t)file_len); + CHECK(filled(buf, 0, user_ps)); + CHECK(filled(buf + user_ps, 0x93, 2 * user_ps)); + CHECK(filled(buf + 3 * user_ps, 0, file_len - 3 * user_ps)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + close(file); + context_close(&c); + free(buf); + return 0; +} + +static int producer(int argc, char **argv) +{ + struct context c = { .fd = atoi(argv[4]), + .ioas = strtoul(argv[5], NULL, 0) }; + int ready = atoi(argv[6]), go = atoi(argv[7]); + char byte = 'R'; + void *p; + + (void)argc; + CHECK(!prctl(PR_SET_PDEATHSIG, SIGKILL)); + CHECK(!drop_pin_cap()); + p = relocated(2 * native_ps, 0x6d); + CHECK(p != MAP_FAILED); + CHECK(!map_range(&c, p, 2 * native_ps, IOVA)); + CHECK(write(ready, &byte, 1) == 1); + CHECK(read(go, &byte, 1) == 1 && byte == 'G'); + return 0; +} + +static int destroy_id(struct context *c, uint32_t id) +{ + struct iommu_destroy cmd = { .size = sizeof(cmd), .id = id }; + + return ioctl(c->fd, IOMMU_DESTROY, &cmd); +} + +static int create_viommu(struct context *c, uint32_t *id, uint32_t *parent) +{ + struct iommu_hwpt_alloc hwpt = { .size = sizeof(hwpt), + .flags = IOMMU_HWPT_ALLOC_NEST_PARENT, + .dev_id = c->dev[0], + .pt_id = c->ioas }; + struct iommu_viommu_alloc cmd = { .size = sizeof(cmd), + .type = IOMMU_VIOMMU_TYPE_SELFTEST, + .dev_id = c->dev[0] }; + + if (ioctl(c->fd, IOMMU_HWPT_ALLOC, &hwpt)) + return -1; + *parent = hwpt.out_hwpt_id; + cmd.hwpt_id = *parent; + if (ioctl(c->fd, IOMMU_VIOMMU_ALLOC, &cmd)) + return -1; + *id = cmd.out_viommu_id; + return 0; +} + +static int queue_alloc(struct context *c, uint32_t viommu, unsigned int index, + unsigned long iova, size_t length, uint32_t *id) +{ + struct iommu_hw_queue_alloc cmd = { .size = sizeof(cmd), + .viommu_id = viommu, + .type = IOMMU_HW_QUEUE_TYPE_SELFTEST, + .index = index, + .nesting_parent_iova = iova, + .length = length }; + int rc = ioctl(c->fd, IOMMU_HW_QUEUE_ALLOC, &cmd); + + if (!rc) + *id = cmd.out_hw_queue_id; + return rc; +} + +static int queue_after_exit(struct context *c) +{ + uint32_t viommu, parent, id; + + CHECK(!create_viommu(c, &viommu, &parent)); + CHECK(!queue_alloc(c, viommu, 0, IOVA + native_ps / 4 + 3, + native_ps / 4, &id)); + CHECK(unmap_range(c, 2 * native_ps, IOVA) == -1 && errno == EBUSY); + CHECK(!destroy_id(c, id)); + CHECK(!destroy_id(c, viommu)); + CHECK(!destroy_id(c, parent)); + return 0; +} + +static int queue_ranges(unsigned int mode) +{ + struct context c; + unsigned char *p = relocated(2 * native_ps, 0x71); + uint32_t viommu, parent, q0, q1, unused; + size_t offset = user_ps / 4 + 3, length = user_ps / 2 + 17; + int file; + + CHECK(p != MAP_FAILED && !context_open(&c, mode)); + CHECK(!limit_bytes(8 * native_ps) && + !create_viommu(&c, &viommu, &parent)); + /* Map only one user leaf: queue pinning must not require surrounding IOVA. */ + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(!check_pa(&c, 0, p, user_ps, IOVA)); + CHECK(queue_alloc(&c, viommu, 1, IOVA + offset, length, &unused) == + -1 && + errno == EIO); + CHECK(!queue_alloc(&c, viommu, 0, IOVA + offset, length, &q0)); + CHECK(queue_alloc(&c, viommu, 0, IOVA + offset, length, &unused) == + -1 && + errno == EEXIST); + CHECK(unmap_range(&c, user_ps, IOVA) == -1 && errno == EBUSY); + CHECK(stat_kb(getpid(), "VmPin:") == (long)native_ps / 1024); + /* Pins keep the old physical bytes after replacing the source VMA. */ + CHECK(mmap(p, user_ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) == p); + memset(p, 0x5a, user_ps); + CHECK(!queue_alloc(&c, viommu, 1, IOVA + offset + 7, length - 7, &q1)); + CHECK(destroy_id(&c, q0) == -1 && errno == EBUSY); + CHECK(!destroy_id(&c, q1)); + CHECK(unmap_range(&c, user_ps, IOVA) == -1 && errno == EBUSY); + CHECK(!destroy_id(&c, q0)); + CHECK(!unmap_range(&c, user_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(queue_alloc(&c, viommu, 0, IOVA + offset, length, &unused) == + -1 && + errno == ENOENT); + CHECK(!munmap(p, 2 * native_ps)); + + /* Separate mappings of the same file byte range cannot form a physical run. + * The second area must fail after the first area's access was registered. + */ + file = memfd_create("queue-alias", 0); + CHECK(file >= 0 && !ftruncate(file, native_ps)); + p = mmap((void *)VA, 2 * user_ps, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(p != MAP_FAILED); + CHECK(mmap(p, user_ps, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, + file, 0) == p); + CHECK(mmap(p + user_ps, user_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, file, 0) == p + user_ps); + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(queue_alloc(&c, viommu, 0, IOVA, 2 * user_ps, &unused) == -1 && + errno == ENOENT); + CHECK(!map_range(&c, p + user_ps, user_ps, IOVA + user_ps)); + CHECK(queue_alloc(&c, viommu, 0, IOVA, 2 * user_ps, &unused) == -1 && + errno == EFAULT); + CHECK(!unmap_range(&c, user_ps, IOVA)); + CHECK(!unmap_range(&c, user_ps, IOVA + user_ps)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + /* The same discontinuity inside one provider must also release its access. */ + CHECK(!map_range(&c, p, 2 * user_ps, IOVA)); + CHECK(queue_alloc(&c, viommu, 0, IOVA, 2 * user_ps, &unused) == -1 && + errno == EFAULT); + CHECK(!unmap_range(&c, 2 * user_ps, IOVA)); + if (user_ps < native_ps) { + /* Adjacent file quarters form a real physical run across two areas. */ + CHECK(mmap(p + user_ps, user_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, file, + user_ps) == p + user_ps); + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(!map_range(&c, p + user_ps, user_ps, IOVA + user_ps)); + CHECK(!queue_alloc(&c, viommu, 0, IOVA + 11, 2 * user_ps - 22, + &q0)); + CHECK(unmap_range(&c, user_ps, IOVA) == -1 && errno == EBUSY); + CHECK(unmap_range(&c, user_ps, IOVA + user_ps) == -1 && + errno == EBUSY); + CHECK(!destroy_id(&c, q0)); + CHECK(!unmap_range(&c, user_ps, IOVA)); + CHECK(!unmap_range(&c, user_ps, IOVA + user_ps)); + } + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(p, 2 * user_ps)); + close(file); + CHECK(!destroy_id(&c, viommu) && !destroy_id(&c, parent)); + context_close(&c); + printf("ok - hardware queue byte spans %zuK/%zuK %s, " + "relocated/overlap/rollback/unmap locks\n", + user_ps / 1024, native_ps / 1024, mode ? "MM" : "USER"); + return 0; +} + +static int cross_process(unsigned int mode) +{ + struct context c; + int ready[2], go[2], status; + pid_t child; + size_t child_ps = user_ps == native_ps ? 4096 : native_ps; + size_t len = 2 * native_ps; + unsigned char *p = mmap((void *)(VA + 128 * native_ps), len, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | + MAP_FIXED_NOREPLACE, + -1, 0), + *buf = malloc(len); + char byte; + + CHECK(p != MAP_FAILED && buf && !pipe(ready) && !pipe(go)); + CHECK(!context_open(&c, mode) && !limit_bytes(len)); + child = fork(); + CHECK(child >= 0); + if (!child) { + char ps[24], native[24], fd[24], ioas[24], r[24], g[24]; + + close(ready[0]); + close(go[1]); + snprintf(ps, sizeof(ps), "%zu", child_ps); + snprintf(native, sizeof(native), "%zu", native_ps); + snprintf(fd, sizeof(fd), "%d", c.fd); + snprintf(ioas, sizeof(ioas), "%u", c.ioas); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(g, sizeof(g), "%d", go[0]); + if (prctl(SET_EXEC_PAGE_SIZE, child_ps, 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--producer", ps, native, fd, ioas, r, + g, NULL); + _exit(5); + } + close(ready[1]); + close(go[0]); + CHECK(read(ready[0], &byte, 1) == 1 && byte == 'R'); + CHECK(stat_kb(child, "VmPin:") == (long)len / 1024); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + if (!mode) + CHECK(map_range(&c, p, native_ps, ALIAS) == -1 && + errno == ENOMEM); + else { + CHECK(!map_range(&c, p, native_ps, ALIAS)); + CHECK(!unmap_range(&c, native_ps, ALIAS)); + } + CHECK(!access_rw(&c, buf, len, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x6d, len)); + byte = 'G'; + CHECK(write(go[1], &byte, 1) == 1); + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(!new_domain(&c, 1)); + CHECK(!access_rw(&c, buf, len, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x6d, len)); + CHECK(!queue_after_exit(&c)); + if (!mode) + CHECK(map_range(&c, p, native_ps, ALIAS) == -1 && + errno == ENOMEM); + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(!map_range(&c, p, len, ALIAS) && !unmap_range(&c, len, ALIAS)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + close(ready[0]); + close(go[1]); + CHECK(!munmap(p, len)); + context_close(&c); + free(buf); + return 0; +} + +struct stress_arg { + struct context *c; + void *p; + unsigned int index; + atomic_int *failed; +}; +static void *stress_thread(void *opaque) +{ + struct stress_arg *a = opaque; + unsigned long iova = IOVA + a->index * 0x100000UL; + size_t len = 2 * native_ps; + + for (unsigned int round = 0; round < 64; round++) { + if (map_range(a->c, a->p, len, iova) || + (!(round % 16) && check_pa(a->c, 1, a->p, len, iova)) || + unmap_range(a->c, len, iova)) { + printf("FAIL concurrent map thread=%u round=%u errno=%d\n", + a->index, round, errno); + atomic_store(a->failed, 1); + break; + } + } + return NULL; +} + +static int concurrent_maps(unsigned int mode) +{ + struct context c; + pthread_t threads[4]; + struct stress_arg args[4]; + atomic_int failed = 0; + void *p = relocated(2 * native_ps, 0x47); + + CHECK(p != MAP_FAILED && !context_open(&c, mode)); + CHECK(!limit_bytes(32 * native_ps) && !new_domain(&c, 1)); + for (unsigned int i = 0; i < 4; i++) { + args[i] = (struct stress_arg){ &c, p, i, &failed }; + CHECK(!pthread_create(&threads[i], NULL, stress_thread, + &args[i])); + } + for (unsigned int i = 0; i < 4; i++) + CHECK(!pthread_join(threads[i], NULL)); + CHECK(!atomic_load(&failed)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(p, 2 * native_ps)); + context_close(&c); + return 0; +} + +static int write_attribute(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + ssize_t n; + + if (fd < 0) + return -1; + n = write(fd, value, strlen(value)); + close(fd); + return n == (ssize_t)strlen(value) ? 0 : -1; +} + +static int allocation_failures(unsigned int mode) +{ + struct context c; + void *p = relocated(2 * native_ps, 0x62); + int nth = open("/proc/self/fail-nth", O_RDWR); + unsigned int hit = 0, n; + + CHECK(nth >= 0 && p != MAP_FAILED); + CHECK(!context_open(&c, mode) && !limit_bytes(8 * native_ps)); + CHECK(!new_domain(&c, 1)); + CHECK(!write_attribute("/sys/kernel/debug/failslab/ignore-gfp-wait", + "N")); + CHECK(!write_attribute( + "/sys/kernel/debug/fail_page_alloc/ignore-gfp-wait", "N")); + CHECK(!write_attribute( + "/sys/kernel/debug/fail_page_alloc/ignore-gfp-highmem", "N")); + CHECK(!write_attribute("/sys/kernel/debug/failslab/verbose", "0")); + CHECK(!write_attribute("/sys/kernel/debug/fail_page_alloc/verbose", + "0")); + CHECK(!write_attribute("/sys/kernel/debug/fail_iommufd/verbose", "0")); + for (n = 1; n <= 512; n++) { + char number[24], remaining[24] = {}; + int bytes = snprintf(number, sizeof(number), "%u", n); + ssize_t got, disabled; + int rc; + + CHECK(pwrite(nth, number, bytes, 0) == bytes); + rc = map_range(&c, p, 2 * native_ps, IOVA); + got = pread(nth, remaining, sizeof(remaining) - 1, 0); + if (got < 0 && errno == EFAULT) { + strcpy(remaining, "1"); + got = 1; + } + disabled = pwrite(nth, "0", 1, 0); + if (disabled < 0 && errno == EFAULT) { + disabled = pwrite(nth, "0", 1, 0); + strcpy(remaining, "1"); + } + CHECK(disabled == 1 && got > 0); + if (!rc) + CHECK(!unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + /* An injected failure must not poison a successful retry or translation. */ + CHECK(!map_range(&c, p, 2 * native_ps, IOVA)); + CHECK(!check_pa(&c, 0, p, 2 * native_ps, IOVA)); + CHECK(!check_pa(&c, 1, p, 2 * native_ps, IOVA)); + CHECK(!unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + if (strtoul(remaining, NULL, 0)) + break; + hit++; + } + CHECK(hit > 0 && n <= 512); + printf("ok - IOMMUFD %zuK %s accounting allocation-failure sweep: %u sites\n", + user_ps / 1024, mode ? "MM" : "USER", hit); + CHECK(!munmap(p, 2 * native_ps)); + close(nth); + context_close(&c); + return 0; +} + +static int queue_failure_sweep(unsigned int mode) +{ + struct context c; + void *p = relocated(2 * native_ps, 0x4c); + int nth = open("/proc/self/fail-nth", O_RDWR); + uint32_t viommu, parent, id; + unsigned int n, hit = 0; + + CHECK(p != MAP_FAILED && nth >= 0 && !context_open(&c, mode)); + CHECK(!limit_bytes(8 * native_ps) && + !create_viommu(&c, &viommu, &parent)); + for (n = 1; n <= 512; n++) { + char number[24], remaining[24] = {}; + int bytes = snprintf(number, sizeof(number), "%u", n), rc; + ssize_t got, disabled; + + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(pwrite(nth, number, bytes, 0) == bytes); + rc = queue_alloc(&c, viommu, 0, IOVA + user_ps / 4 + 3, + user_ps / 2, &id); + got = pread(nth, remaining, sizeof(remaining) - 1, 0); + if (got < 0 && errno == EFAULT) { + strcpy(remaining, "1"); + got = 1; + } + disabled = pwrite(nth, "0", 1, 0); + if (disabled < 0 && errno == EFAULT) { + disabled = pwrite(nth, "0", 1, 0); + strcpy(remaining, "1"); + } + CHECK(got > 0 && disabled == 1); + if (!rc) + CHECK(!destroy_id(&c, id)); + /* A failed queue must not leave an internal access blocking unmap. */ + CHECK(!unmap_range(&c, user_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(!queue_alloc(&c, viommu, 0, IOVA + user_ps / 4 + 3, + user_ps / 2, &id)); + CHECK(!destroy_id(&c, id) && !unmap_range(&c, user_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + if (strtoul(remaining, NULL, 0)) + break; + hit++; + } + CHECK(hit > 0 && n <= 512); + CHECK(!destroy_id(&c, viommu) && !destroy_id(&c, parent)); + context_close(&c); + close(nth); + CHECK(!munmap(p, 2 * native_ps)); + printf("ok - hardware queue %zuK %s failure sweep: %u sites, no stale locks/pins\n", + user_ps / 1024, mode ? "MM" : "USER", hit); + return 0; +} + +static int page_access(struct context *c, void *p, size_t len, + unsigned long iova, uint32_t *id) +{ + struct iommu_test_cmd cmd = { + .size = sizeof(cmd), + .op = IOMMU_TEST_OP_ACCESS_PAGES, + .id = c->access, + .access_pages = { .iova = iova, + .length = len, + .uptr = (uintptr_t)p, + .flags = MOCK_FLAGS_ACCESS_WRITE } + }; + int rc = ioctl(c->fd, IOMMU_TEST_CMD, &cmd); + + if (!rc) + *id = cmd.access_pages.out_access_pages_id; + return rc; +} + +static int page_release(struct context *c, uint32_t id) +{ + struct iommu_test_cmd cmd = { .size = sizeof(cmd), + .op = IOMMU_TEST_OP_DESTROY_ACCESS_PAGES, + .id = c->access, + .destroy_access_pages = { + .access_pages_id = id } }; + + return ioctl(c->fd, IOMMU_TEST_CMD, &cmd); +} + +static int page_ranges(unsigned int mode) +{ + struct context c; + int file = memfd_create("native-page-array", 0); + unsigned char *p; + uint32_t a, b, d, e; + size_t len = 3 * native_ps, tail = native_ps + user_ps / 2 + 3; + + CHECK(file >= 0 && !ftruncate(file, 5 * native_ps)); + p = mmap((void *)VA, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, native_ps); + CHECK(p != MAP_FAILED && !limit_bytes(4 * native_ps)); + memset(p, 0x67, len); + CHECK(!context_open_common(&c, mode, false, true)); + /* Separate areas exercise native output stride with alternative indices. */ + CHECK(!map_range(&c, p, native_ps, IOVA)); + CHECK(!map_range(&c, p + native_ps, 2 * native_ps, IOVA + native_ps)); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!page_access(&c, p, len, IOVA, &a)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + CHECK(!page_access(&c, p, len, IOVA, &b)); + CHECK(!page_access(&c, p + native_ps, tail, IOVA + native_ps, &d)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + /* Source VA can move without changing its retained physical offset. */ + unsigned char *moved = mremap(p, len, len, + MREMAP_MAYMOVE | MREMAP_FIXED, + (void *)(VA + 32 * native_ps + user_ps)); + CHECK(moved != MAP_FAILED); + CHECK(!page_access(&c, moved, len, IOVA, &e) && !page_release(&c, e)); + CHECK(!page_release(&c, a) && !page_release(&c, b)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)(2 * native_ps) / 1024); + CHECK(!page_release(&c, d)); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(mremap(moved, len, len, MREMAP_MAYMOVE | MREMAP_FIXED, p) == p); + CHECK(!page_access(&c, p, len, IOVA, &a)); + /* Unmap callback releases an access covering both independent areas. */ + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(page_release(&c, a) == -1 && errno == ENOENT); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(p, len)); + + /* The second area is absent: first area's pin/accounting must roll back. */ + p = mmap((void *)VA, native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, native_ps); + CHECK(p != MAP_FAILED && !map_range(&c, p, native_ps, IOVA)); + CHECK(page_access(&c, NULL, 2 * native_ps, IOVA, &a) == -1 && + errno == ENOENT); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!page_access(&c, p, native_ps, IOVA, &a) && !page_release(&c, a)); + CHECK(!unmap_range(&c, native_ps, IOVA) && !munmap(p, native_ps)); + + p = mmap((void *)VA, 2 * native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, native_ps); + CHECK(p != MAP_FAILED); + CHECK(mmap(p + native_ps, native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, file, native_ps) == p + native_ps); + CHECK(!map_range(&c, p, 2 * native_ps, IOVA)); + CHECK(!page_access(&c, p, 2 * native_ps, IOVA, &a)); + CHECK(stat_kb(getpid(), "VmPin:") == + (long)((user_ps == native_ps ? 2 : 1) * native_ps) / 1024); + CHECK(!page_release(&c, a) && !unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(!munmap(p, 2 * native_ps)); + + if (user_ps < native_ps) { + /* Native-sized logical range whose physical base has a fragment offset. */ + p = mmap((void *)VA, native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, user_ps); + CHECK(p != MAP_FAILED && !map_range(&c, p, native_ps, IOVA)); + CHECK(page_access(&c, p, native_ps, IOVA, &a) == -1 && + errno == EINVAL); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!unmap_range(&c, native_ps, IOVA) && + !munmap(p, native_ps)); + + /* Aligned first fragment followed by an alias, not consecutive bytes. */ + p = mmap((void *)VA, 2 * native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, 0); + CHECK(p != MAP_FAILED); + CHECK(mmap(p + native_ps + user_ps, user_ps, + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, file, + native_ps) == p + native_ps + user_ps); + CHECK(!map_range(&c, p, native_ps, IOVA)); + CHECK(!map_range(&c, p + native_ps, native_ps, + IOVA + native_ps)); + CHECK(page_access(&c, p, 2 * native_ps, IOVA, &a) == -1 && + errno == EINVAL); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!page_access(&c, p, native_ps, IOVA, &a)); + CHECK(!unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(page_release(&c, a) == -1 && errno == ENOENT); + CHECK(!munmap(p, 2 * native_ps)); + } + context_close(&c); + close(file); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + printf("ok - native page arrays %zuK/%zuK %s, lazy/partial/moved/rollback/callback\n", + user_ps / 1024, native_ps / 1024, mode ? "MM" : "USER"); + return 0; +} + +static int page_lifetime(unsigned int mode) +{ + struct context c; + int ready[2], go[2], status; + pid_t child; + size_t child_ps = user_ps == native_ps ? 4096 : native_ps; + size_t len = 2 * native_ps; + unsigned char *p = mmap((void *)(VA + 128 * native_ps), len, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | + MAP_FIXED_NOREPLACE, + -1, 0), + *buf = malloc(len); + uint32_t a, b; + char byte; + + CHECK(p != MAP_FAILED && buf && !pipe(ready) && !pipe(go)); + memset(p, 0x42, len); + CHECK(!context_open_common(&c, mode, false, true) && !limit_bytes(len)); + child = fork(); + CHECK(child >= 0); + if (!child) { + char ps[24], native[24], fd[24], ioas[24], r[24], g[24]; + + close(ready[0]); + close(go[1]); + snprintf(ps, sizeof(ps), "%zu", child_ps); + snprintf(native, sizeof(native), "%zu", native_ps); + snprintf(fd, sizeof(fd), "%d", c.fd); + snprintf(ioas, sizeof(ioas), "%u", c.ioas); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(g, sizeof(g), "%d", go[0]); + if (prctl(SET_EXEC_PAGE_SIZE, child_ps, 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--producer", ps, native, fd, ioas, r, + g, NULL); + _exit(5); + } + close(ready[1]); + close(go[0]); + CHECK(read(ready[0], &byte, 1) == 1 && byte == 'R'); + CHECK(stat_kb(child, "VmPin:") == 0); + CHECK(!page_access(&c, NULL, len, IOVA, &a)); + CHECK(stat_kb(child, "VmPin:") == (long)len / 1024); + CHECK(!access_rw(&c, buf, len, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x6d, len)); + byte = 'G'; + CHECK(write(go[1], &byte, 1) == 1); + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status)); + /* Reuse retained anonymous fragments after source mm_users reaches zero. */ + CHECK(!page_access(&c, NULL, native_ps, IOVA, &b)); + CHECK(!page_release(&c, a)); + CHECK(!access_rw(&c, buf, native_ps, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x6d, native_ps)); + CHECK(!map_range(&c, p, len, ALIAS)); + if (!mode) + CHECK(page_access(&c, p, len, ALIAS, &a) == -1 && + errno == ENOMEM); + else + CHECK(!page_access(&c, p, len, ALIAS, &a) && + !page_release(&c, a)); + /* Final access release refunds even a dead provider's account. */ + CHECK(!page_release(&c, b) && !unmap_range(&c, len, IOVA)); + CHECK(!page_access(&c, p, len, ALIAS, &a) && !page_release(&c, a)); + CHECK(!unmap_range(&c, len, ALIAS)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(p, len)); + context_close(&c); + close(ready[0]); + close(go[1]); + free(buf); + printf("ok - native page array anonymous source %zuK -> %zuK consumer, %s exit/refund\n", + child_ps / 1024, user_ps / 1024, mode ? "MM" : "USER"); + return 0; +} + +static int page_failure_sweep(unsigned int mode) +{ + struct context c; + int file = memfd_create("page-array-failures", 0); + int nth = open("/proc/self/fail-nth", O_RDWR); + unsigned char *p; + uint32_t id; + unsigned int n, hit = 0; + + CHECK(file >= 0 && nth >= 0 && !ftruncate(file, 2 * native_ps)); + p = mmap((void *)VA, 2 * native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, 0); + CHECK(p != MAP_FAILED); + memset(p, 0x53, 2 * native_ps); + CHECK(!context_open_common(&c, mode, false, true) && + !limit_bytes(2 * native_ps)); + CHECK(!map_range(&c, p, native_ps, IOVA)); + CHECK(!map_range(&c, p + native_ps, native_ps, IOVA + native_ps)); + for (n = 1; n <= 512; n++) { + char number[24], remaining[24] = {}; + int bytes = snprintf(number, sizeof(number), "%u", n), rc; + ssize_t got, disabled; + + CHECK(pwrite(nth, number, bytes, 0) == bytes); + rc = page_access(&c, p, 2 * native_ps, IOVA, &id); + got = pread(nth, remaining, sizeof(remaining) - 1, 0); + if (got < 0 && errno == EFAULT) { + strcpy(remaining, "1"); + got = 1; + } + disabled = pwrite(nth, "0", 1, 0); + if (disabled < 0 && errno == EFAULT) { + disabled = pwrite(nth, "0", 1, 0); + strcpy(remaining, "1"); + } + CHECK(got > 0 && disabled == 1); + if (!rc) + CHECK(!page_release(&c, id)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!page_access(&c, p, 2 * native_ps, IOVA, &id) && + !page_release(&c, id)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + if (strtoul(remaining, NULL, 0)) + break; + hit++; + } + CHECK(hit > 0 && n <= 512); + CHECK(!unmap_range(&c, 2 * native_ps, IOVA) && + !munmap(p, 2 * native_ps)); + context_close(&c); + close(file); + close(nth); + printf("ok - native page arrays %zuK %s failure sweep: %u sites, refunds/retries\n", + user_ps / 1024, mode ? "MM" : "USER", hit); + return 0; +} + +struct page_race { + struct context *c; + void *p; + pthread_barrier_t barrier; + atomic_int failed, pinned, absent; +}; + +static void *page_pin_thread(void *opaque) +{ + struct page_race *r = opaque; + + pthread_barrier_wait(&r->barrier); + for (unsigned int i = 0; i < 256; i++) { + uint32_t id; + + if (page_access(r->c, r->p, 2 * native_ps, IOVA, &id)) { + /* Unmap can mark the area inaccessible before notifying drivers. */ + if (errno != ENOENT && errno != EINVAL) + goto fail; + atomic_fetch_add(&r->absent, 1); + } else { + atomic_fetch_add(&r->pinned, 1); + sched_yield(); + if (page_release(r->c, id) && errno != ENOENT) + goto fail; + } + } + return NULL; +fail: + printf("FAIL page pin/unmap race errno=%d\n", errno); + atomic_store(&r->failed, 1); + return NULL; +} + +static void *page_unmap_thread(void *opaque) +{ + struct page_race *r = opaque; + + pthread_barrier_wait(&r->barrier); + for (unsigned int i = 0; i < 256; i++) { + if (unmap_range(r->c, 2 * native_ps, IOVA)) + goto fail; + sched_yield(); + if (map_range(r->c, r->p, 2 * native_ps, IOVA)) + goto fail; + } + return NULL; +fail: + printf("FAIL page unmap/remap race errno=%d\n", errno); + atomic_store(&r->failed, 1); + return NULL; +} + +static int page_concurrency(unsigned int mode) +{ + struct context c; + struct page_race r = { .c = &c }; + pthread_t threads[4]; + int file = memfd_create("page-array-race", 0); + + CHECK(file >= 0 && !ftruncate(file, 2 * native_ps)); + r.p = mmap((void *)VA, 2 * native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, 0); + CHECK(r.p != MAP_FAILED); + memset(r.p, 0x49, 2 * native_ps); + CHECK(!context_open_common(&c, mode, false, true) && + !limit_bytes(8 * native_ps)); + CHECK(!map_range(&c, r.p, 2 * native_ps, IOVA)); + CHECK(!pthread_barrier_init(&r.barrier, NULL, 4)); + for (unsigned int i = 0; i < 4; i++) + CHECK(!pthread_create( + &threads[i], NULL, + i == 3 ? page_unmap_thread : page_pin_thread, &r)); + for (unsigned int i = 0; i < 4; i++) + CHECK(!pthread_join(threads[i], NULL)); + CHECK(!atomic_load(&r.failed) && atomic_load(&r.pinned) > 0); + CHECK(!unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(r.p, 2 * native_ps)); + context_close(&c); + close(file); + CHECK(!pthread_barrier_destroy(&r.barrier)); + printf("ok - native page arrays %zuK %s race: %d pins, %d unavailable, 256 remaps\n", + user_ps / 1024, mode ? "MM" : "USER", atomic_load(&r.pinned), + atomic_load(&r.absent)); + return 0; +} + +static int dirty_ioctl(struct context *c, uint32_t hwpt, void *bitmap, + unsigned long iova, size_t length, size_t granule, + unsigned int flags) +{ + struct iommu_hwpt_get_dirty_bitmap cmd = { .size = sizeof(cmd), + .hwpt_id = hwpt, + .flags = flags, + .iova = iova, + .length = length, + .page_size = granule, + .data = (uintptr_t)bitmap }; + + return ioctl(c->fd, IOMMU_HWPT_GET_DIRTY_BITMAP, &cmd); +} + +static int dirty_errors(unsigned int mode) +{ + struct context c; + size_t len = 66 * native_ps; + unsigned char *data = relocated(len, 0x34); + unsigned char *out = mmap(NULL, 2 * user_ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t *bitmap = (void *)(out + user_ps - 8), seed[2] = { 1, 2 }; + struct iommu_option option = { .size = sizeof(option), + .option_id = IOMMU_OPTION_HUGE_PAGES, + .op = IOMMU_OPTION_OP_SET, + .val64 = 0 }; + struct iommu_hwpt_alloc alloc = { + .size = sizeof(alloc), .flags = IOMMU_HWPT_ALLOC_DIRTY_TRACKING + }; + struct iommu_hwpt_set_dirty_tracking enable = { + .size = sizeof(enable), + .flags = IOMMU_HWPT_DIRTY_TRACKING_ENABLE + }; + struct iommu_test_cmd mark = { .size = sizeof(mark), + .op = IOMMU_TEST_OP_DIRTY, + .dirty = { .iova = IOVA, + .length = len, + .page_size = native_ps, + .uptr = (uintptr_t)seed } }; + unsigned int no_clear = IOMMU_HWPT_GET_DIRTY_BITMAP_NO_CLEAR; + int failures = 0, rc, error; + + CHECK(data != MAP_FAILED && out != MAP_FAILED); + CHECK(!context_open(&c, mode) && !limit_bytes(128 * native_ps)); + option.object_id = c.ioas; + CHECK(!ioctl(c.fd, IOMMU_OPTION, &option)); + CHECK(!map_range(&c, data, len, IOVA)); + alloc.dev_id = c.dev[0]; + alloc.pt_id = c.ioas; + CHECK(!ioctl(c.fd, IOMMU_HWPT_ALLOC, &alloc)); + enable.hwpt_id = alloc.out_hwpt_id; + CHECK(!ioctl(c.fd, IOMMU_HWPT_SET_DIRTY_TRACKING, &enable)); + mark.id = alloc.out_hwpt_id; + CHECK(!ioctl(c.fd, IOMMU_TEST_CMD, &mark) && + mark.dirty.out_nr_dirty == 2); + CHECK(!dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, native_ps, + no_clear)); + CHECK(bitmap[0] == 1 && bitmap[1] == 2); + CHECK(filled(out, 0, user_ps - 8) && + filled(out + user_ps + 8, 0, user_ps - 8)); + +#define EXPECT_DIRTY_ERROR(call, expected, label) \ + do { \ + errno = 0; \ + rc = (call); \ + error = errno; \ + if (rc != -1 || error != (expected)) { \ + printf("FAIL dirty %s: rc=%d errno=%d expected=%d\n", \ + label, rc, error, expected); \ + failures++; \ + } \ + } while (0) + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, + IOVA + len, native_ps, native_ps, + no_clear), + EINVAL, "unmapped IOVA"); + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, (void *)1, IOVA, + len, native_ps, no_clear), + EFAULT, "invalid output pointer"); + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, + 4 * native_ps, 3 * native_ps, no_clear), + EINVAL, "non-power-of-two granule"); + CHECK(!mprotect(out, 2 * user_ps, PROT_READ)); + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, + native_ps, no_clear), + EFAULT, "read-only output"); + CHECK(!mprotect(out, 2 * user_ps, PROT_READ | PROT_WRITE)); + memset(out, 0, 2 * user_ps); + CHECK(!munmap(out + user_ps, user_ps)); + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, + native_ps, no_clear), + EFAULT, "output hole after valid prefix"); + CHECK(bitmap[0] == 1); + CHECK(mmap(out + user_ps, user_ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, + 0) == out + user_ps); + /* NO_CLEAR errors retain dirty state, so fixing the buffer permits retry. */ + CHECK(!dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, native_ps, + 0)); + CHECK(bitmap[0] == 1 && bitmap[1] == 2); + memset(out, 0, 2 * user_ps); + CHECK(!dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, native_ps, + no_clear)); + CHECK(bitmap[0] == 0 && bitmap[1] == 0); + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(!munmap(data, len) && !munmap(out, 2 * user_ps)); + context_close(&c); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!failures); +#undef EXPECT_DIRTY_ERROR + return 0; +} + +static int exercise(void) +{ + CHECK(!drop_pin_cap()); + for (unsigned int mode = 0; mode < 2; mode++) { + CHECK(!limits_and_lifetime(mode)); + CHECK(!holes_and_shared(mode)); + CHECK(!queue_ranges(mode)); + CHECK(!page_ranges(mode)); + CHECK(!page_lifetime(mode)); + CHECK(!cross_process(mode)); + CHECK(!concurrent_maps(mode)); + CHECK(!allocation_failures(mode)); + CHECK(!queue_failure_sweep(mode)); + CHECK(!page_failure_sweep(mode)); + CHECK(!page_concurrency(mode)); + CHECK(!dirty_errors(mode)); + printf("ok - IOMMUFD %zuK userspace/%zuK kernel %s accounting, limits/lifetime/shared/cross-process/256 concurrent cycles\n", + user_ps / 1024, native_ps / 1024, mode ? "MM" : "USER"); + } + return 0; +} + +int main(int argc, char **argv) +{ + int fail = 0; + + setbuf(stdout, NULL); + if (argc >= 4) { + user_ps = strtoul(argv[2], NULL, 0); + native_ps = strtoul(argv[3], NULL, 0); + CHECK(getauxval(AT_PAGESZ) == user_ps); + if (!strcmp(argv[1], "--exercise")) + return exercise(); + if (!strcmp(argv[1], "--producer") && argc == 8) + return producer(argc, argv); + return 2; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + CHECK(!mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)); + CHECK(!mount("sysfs", "/sys", "sysfs", 0, NULL)); + CHECK(!mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)); + native_ps = getauxval(AT_PAGESZ); + size_t sizes[] = { native_ps, 4096, 16384 }; + + for (unsigned int i = 0; i < (native_ps > 16384 ? 3U : 2U); i++) { + pid_t child = fork(); + int status = 0, ok = 0; + + if (!child) { + char size[24], native[24]; + + snprintf(size, sizeof(size), "%zu", sizes[i]); + snprintf(native, sizeof(native), "%zu", native_ps); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", size, native, + NULL); + _exit(5); + } + for (unsigned int ticks = 0; child > 0 && ticks < 1800; + ticks++) { + if (waitpid(child, &status, WNOHANG) == child) { + ok = WIFEXITED(status) && !WEXITSTATUS(status); + break; + } + if (ticks == 1799) { + kill(child, SIGKILL); + waitpid(child, &status, 0); + } + usleep(100000); + } + printf("%s - IOMMUFD ABI %zuK status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + fail += !ok; + } + printf("IOMMUFD IOCTL CONTRACT %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/lazyfree-fixture-init.c b/tools/testing/selftests/arm64/user4k/lazyfree-fixture-init.c new file mode 100644 index 0000000000000..b652200c38b42 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/lazyfree-fixture-init.c @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static unsigned long stat_value(const char *path, const char *key) +{ + char line[256], name[100]; + unsigned long value = 0; + FILE *f = fopen(path, "re"); + + if (!f) + return 0; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "%99s %lu", name, &value) == 2 && + !strcmp(name, key)) + break; + fclose(f); + return value; +} + +static bool all_bytes(const unsigned char *p, size_t len, unsigned char value) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != value) + return false; + return true; +} + +static unsigned long lazy_kb(void *address) +{ + FILE *f = fopen("/proc/self/smaps", "re"); + char line[256]; + unsigned long first, last, value = ~0UL; + bool found = false; + + if (!f) + return value; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lx-%lx", &first, &last) == 2) { + if (found) + break; + found = (uintptr_t)address >= first && + (uintptr_t)address < last; + } else if (found && + sscanf(line, "LazyFree: %lu", &value) == 1) { + break; + } + } + fclose(f); + return value; +} + +static uint64_t pagemap(void *p, size_t ps) +{ + uint64_t entry = 0; + int fd = open("/proc/self/pagemap", O_RDONLY); + + if (fd < 0 || + pread(fd, &entry, sizeof(entry), (uintptr_t)p / ps * 8) != 8) + ksft_exit_fail_msg("pagemap: %s\n", strerror(errno)); + close(fd); + return entry; +} + +static bool pageout(void *p, size_t len) +{ + cpu_set_t allowed, one; + bool ok = true; + + if (sched_getaffinity(0, sizeof(allowed), &allowed)) + return false; + for (unsigned int i = 0; i < CPU_SETSIZE; i++) { + if (!CPU_ISSET(i, &allowed)) + continue; + CPU_ZERO(&one); + CPU_SET(i, &one); + ok &= !sched_setaffinity(0, sizeof(one), &one); + ok &= !madvise(p, len, MADV_COLD); + ok &= !madvise(p, len, MADV_PAGEOUT); + } + return !sched_setaffinity(0, sizeof(allowed), &allowed) && ok; +} + +static bool mark_free(void *p, size_t len) +{ + cpu_set_t allowed, one; + bool ok = true; + + if (sched_getaffinity(0, sizeof(allowed), &allowed)) + return false; + for (unsigned int i = 0; i < CPU_SETSIZE; i++) { + if (!CPU_ISSET(i, &allowed)) + continue; + CPU_ZERO(&one); + CPU_SET(i, &one); + ok &= !sched_setaffinity(0, sizeof(one), &one); + ok &= !madvise(p, len, MADV_FREE); + ok &= !madvise(p, len, MADV_COLD); + } + return !sched_setaffinity(0, sizeof(allowed), &allowed) && ok; +} + +static int exercise(unsigned long native) +{ + const size_t ps = getauxval(AT_PAGESZ), len = 4 * ps; + unsigned char *reservation = mmap(NULL, len + 2 * native, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + unsigned char *p = + (void *)(((uintptr_t)reservation + native - 1) & ~(native - 1)); + bool ok; + unsigned long writes; + uint64_t entry; + + if (reservation == MAP_FAILED || + mprotect(p, len, PROT_READ | PROT_WRITE)) + ksft_exit_fail_msg("mapping\n"); + ksft_print_header(); + ksft_set_plan(8); + for (unsigned int i = 0; i < 4; i++) + memset(p + i * ps, 0x40 + i, ps); + ok = mark_free(p + ps, ps); + if (!ok) + ksft_exit_fail_msg("%zuK MADV_FREE: %s\n", ps / 1024, + strerror(errno)); + ksft_test_result(all_bytes(p + ps, ps, 0x41), + "%zuK FREE retains bytes before reclaim\n", ps / 1024); + ksft_test_result(lazy_kb(p) == ps / 1024, + "%zuK partial LazyFree accounting\n", ps / 1024); + ok = pageout(p, len); + entry = pagemap(p + ps, ps); + ok &= !(entry & ((1ULL << 63) | (1ULL << 62))); + ok &= all_bytes(p + ps, ps, 0) && all_bytes(p, ps, 0x40) && + all_bytes(p + 2 * ps, ps, 0x42) && + all_bytes(p + 3 * ps, ps, 0x43); + ksft_test_result(ok, "%zuK partial FREE reclaims only advised bytes\n", + ps / 1024); + /* Drop old swap/cache state, then test cancellation by a real EL0 write. */ + ok = !madvise(p, len, MADV_DONTNEED); + memset(p, 0x62, len); + ok &= mark_free(p + ps, ps); + memset(p + ps, 0xa5, ps); + ksft_test_result(lazy_kb(p) == 0, + "%zuK dirty leaf no longer reported LazyFree\n", + ps / 1024); + ok &= pageout(p, len); + ok &= all_bytes(p + ps, ps, 0xa5) && all_bytes(p, ps, 0x62) && + all_bytes(p + 2 * ps, 2 * ps, 0x62); + ksft_test_result(ok, "%zuK write after FREE cancels quarter discard\n", + ps / 1024); + ok = !madvise(p, len, MADV_DONTNEED); + memset(p, 0x83, len); + writes = stat_value("/proc/vmstat", "pswpout"); + ok &= mark_free(p, len) && pageout(p, len); + for (unsigned int i = 0; i < 4; i++) + ok &= !(pagemap(p + i * ps, ps) & + ((1ULL << 63) | (1ULL << 62))); + printf("# full FREE pswpout=%lu->%lu, entries=%#llx %#llx %#llx %#llx\n", + writes, stat_value("/proc/vmstat", "pswpout"), + (unsigned long long)pagemap(p, ps), + (unsigned long long)pagemap(p + ps, ps), + (unsigned long long)pagemap(p + 2 * ps, ps), + (unsigned long long)pagemap(p + 3 * ps, ps)); + ok &= stat_value("/proc/vmstat", "pswpout") == writes && + all_bytes(p, len, 0); + ksft_test_result( + ok, "%zuK fully free backing reclaimed without swap writes\n", + ps / 1024); + ok = !madvise(p, len, MADV_DONTNEED); + memset(p, 0xb7, len); + ok &= pageout(p, len); + ok &= !!(pagemap(p + ps, ps) & (1ULL << 62)); + ok &= mark_free(p + ps, ps); + ok &= !(pagemap(p + ps, ps) & ((1ULL << 63) | (1ULL << 62))); + ok &= all_bytes(p + ps, ps, 0) && all_bytes(p, ps, 0xb7) && + all_bytes(p + 2 * ps, 2 * ps, 0xb7); + ksft_test_result( + ok, + "%zuK FREE drops selected swap entry without neighbor loss\n", + ps / 1024); + { + int gate[2], status = 0; + pid_t pid; + char ready; + + ok = !madvise(p, len, MADV_DONTNEED); + memset(p, 0xc8, len); + if (pipe(gate)) + ksft_exit_fail_msg("pipe\n"); + pid = fork(); + if (!pid) { + close(gate[1]); + if (read(gate[0], &ready, 1) != 1 || + !all_bytes(p, len, 0xc8)) + _exit(2); + _exit(0); + } + close(gate[0]); + ok &= mark_free(p + ps, ps) && pageout(p, len); + ok &= write(gate[1], "x", 1) == 1; + close(gate[1]); + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + ksft_test_result( + ok, + "%zuK FREE after fork preserves other process data\n", + ps / 1024); + } + munmap(reservation, len + 2 * native); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} + +int main(int argc, char **argv) +{ + bool ok = true; + unsigned long native = getauxval(AT_PAGESZ); + unsigned long granules[] = { native, 4096, 16384 }; + unsigned int nr = native > 16384 ? 3 : 2; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--test") && getppid() == 1) { + if (getauxval(AT_PAGESZ) != strtoul(argv[2], NULL, 10)) + return 42; + return exercise(strtoul(argv[3], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + swapon("/dev/vda", 0)) + return 3; + for (unsigned int small = 0; small < nr; small++) { + pid_t pid = fork(); + int status = 0; + + if (!pid) { + char size[32], host_size[32]; + + snprintf(size, sizeof(size), "%lu", granules[small]); + snprintf(host_size, sizeof(host_size), "%lu", native); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, granules[small], + 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--test", size, host_size, + NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + printf("%s - lazyfree fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/memcg-accounting-init.c b/tools/testing/selftests/arm64/user4k/memcg-accounting-init.c new file mode 100644 index 0000000000000..f34602bb730e5 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/memcg-accounting-init.c @@ -0,0 +1,153 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CG "/sys/fs/cgroup/account" +#define LENGTH (32UL * 1024 * 1024) +static int put(const char *p, const char *v) +{ + int fd = open(p, O_WRONLY); + if (fd < 0) + return 1; + int bad = write(fd, v, strlen(v)) != (ssize_t)strlen(v); + close(fd); + return bad; +} +static unsigned long long stat_value(const char *key) +{ + FILE *f = fopen(CG "/memory.stat", "r"); + char name[128]; + unsigned long long value = 0; + if (!f) + exit(20); + while (fscanf(f, "%127s %llu", name, &value) == 2) + if (!strcmp(key, name)) { + fclose(f); + return value; + } + fclose(f); + exit(21); +} +static int worker(unsigned long expected, int ready, int release) +{ + if (getauxval(AT_PAGESZ) != expected) + return 2; + unsigned char *p = mmap(NULL, LENGTH, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED || madvise(p, LENGTH, MADV_NOHUGEPAGE)) + return 3; + memset(p, 0x71, LENGTH); + char token = 'R'; + if (write(ready, &token, 1) != 1 || read(release, &token, 1) != 1) + return 4; + /* Keep the writes observable independently of the accounting check. */ + for (size_t i = 0; i < LENGTH; i++) + if (p[i] != 0x71) + return 5; + return munmap(p, LENGTH) ? 6 : 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (getppid() == 1 && argc == 5 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), atoi(argv[3]), + atoi(argv[4])); + if (getpid() != 1 || argc != 1) + return 2; + signal(SIGPIPE, SIG_IGN); + alarm(60); + int failed = 0; + unsigned long long slab[3] = {}; + unsigned long sizes[3] = { getauxval(AT_PAGESZ), 4096, 16384 }; + unsigned int modes = sizes[0] == 65536 ? 3 : 2; + if ((sizes[0] != 16384 && sizes[0] != 65536) || + mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL) || + put("/sys/fs/cgroup/cgroup.subtree_control", "+memory")) { + perror("setup"); + failed = 1; + goto done; + } + for (unsigned i = 0; i < modes; i++) { + int ready[2], release[2]; + if (mkdir(CG, 0755) || put(CG "/memory.max", "134217728") || + pipe(ready) || pipe(release)) { + failed = 1; + break; + } + pid_t child = fork(); + if (!child) { + char ps[32], rfd[16], cfd[16]; + close(ready[0]); + close(release[1]); + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + snprintf(rfd, sizeof(rfd), "%d", ready[1]); + snprintf(cfd, sizeof(cfd), "%d", release[0]); + if (put(CG "/cgroup.procs", "0") || + prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(7); + execl("/init", "/init", "--worker", ps, rfd, cfd, NULL); + _exit(8); + } + close(ready[1]); + close(release[0]); + char token; + int ok = child > 0 && read(ready[0], &token, 1) == 1; + if (ok) { + /* Let per-CPU slab statistics become visible before the comparison. */ + usleep(1200000); + slab[i] = stat_value("slab_unreclaimable"); + printf("# ABI=%lu anon=%llu slab_unreclaimable=%llu pagetables=%llu\n", + sizes[i], stat_value("anon"), slab[i], + stat_value("pagetables")); + ok = stat_value("anon") >= LENGTH && + write(release[1], "G", 1) == 1; + } + int status = 0; + if (child > 0 && !ok) + kill(child, SIGKILL); + ok = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && ok; + close(ready[0]); + close(release[1]); + printf("%s - memcg accounting ABI %luK worker status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + failed |= !ok; + for (unsigned retry = 0; retry < 100 && stat_value("anon"); + retry++) + usleep(20000); + if (stat_value("anon") || rmdir(CG)) + failed = 1; + if (failed) + break; + } + /* Slot metadata must be charged for every alternative granule. The + * loose lower bound tolerates slab/KASAN layout differences. */ + for (unsigned i = 1; !failed && i < modes; i++) { + if (slab[i] < slab[0] + 512 * 1024) { + printf("not ok - slot metadata escaped cgroup accounting ABI=%lu native=%llu alternative=%llu\n", + sizes[i], slab[0], slab[i]); + failed = 1; + } + } + +done: + printf("GRANULE MEMCG ACCOUNTING %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/memcg-pressure-init.c b/tools/testing/selftests/arm64/user4k/memcg-pressure-init.c new file mode 100644 index 0000000000000..c2f50993ff805 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/memcg-pressure-init.c @@ -0,0 +1,417 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CG "/sys/fs/cgroup/pressure" +#define MIB (1024UL * 1024) +#define LENGTH (32 * MIB) +#define ROUNDS 4 + +static int put(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + if (fd < 0) + return -1; + ssize_t ret = write(fd, value, strlen(value)); + int saved = errno; + close(fd); + errno = saved; + return ret == (ssize_t)strlen(value) ? 0 : -1; +} + +static unsigned long long get(const char *path, const char *key) +{ + FILE *f = fopen(path, "r"); + char name[128]; + unsigned long long value = 0, result = 0; + if (!f) { + perror(path); + exit(10); + } + if (!key) { + if (fscanf(f, "%llu", &result) != 1) + exit(11); + } else + while (fscanf(f, "%127s %llu", name, &value) == 2) + if (!strcmp(name, key)) { + result = value; + break; + } + fclose(f); + return result; +} + +static unsigned char pattern(size_t offset, unsigned seed, unsigned round) +{ + return (offset ^ (offset >> 12) ^ (offset >> 20) ^ (seed * 79) ^ + (round * 43)) & + 255; +} + +static int contents(unsigned char *p, size_t ps, unsigned seed, unsigned round, + int write_data, int child) +{ + size_t pages = LENGTH / ps; + /* Odd multiplication permutes every logical page of this power-of-two range. */ + for (size_t i = 0; i < pages; i++) { + size_t page = (i * 4051 + round * 31) & (pages - 1); + for (size_t j = 0; j < ps; j++) { + size_t offset = page * ps + j; + unsigned char want = pattern(offset, seed, round); + if (child && !(page & 3)) + want ^= 0x69; + if (write_data) + p[offset] = want; + else if (p[offset] != want) { + printf("not ok - ABI=%zu round=%u child=%d byte=%zu got=%u expected=%u\n", + ps, round, child, offset, p[offset], + want); + return 1; + } + } + } + return 0; +} + +static int barrier(int ready, int command) +{ + char token = 'R'; + return write(ready, &token, 1) != 1 || read(command, &token, 1) != 1 || + token != 'G'; +} + +static int worker(unsigned long expected, unsigned seed, int ready, int command) +{ + size_t ps = getauxval(AT_PAGESZ); + unsigned char *p; + if (ps != expected) + return 2; + p = mmap(NULL, LENGTH, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 3; + if (madvise(p, LENGTH, MADV_NOHUGEPAGE)) { + int error = errno; + struct stat st; + + /* Without THP this advice is unavailable and cannot change policy. */ + if (error != EINVAL || + !stat("/sys/kernel/mm/transparent_hugepage/enabled", &st) || + errno != ENOENT) + return 3; + } + if (contents(p, ps, seed, 0, 1, 0) || barrier(ready, command)) + return 4; + for (unsigned round = 0; round < ROUNDS; round++) { + if (contents(p, ps, seed, round, 0, 0)) + return 5; + pid_t child = fork(); + if (child < 0) + return 6; + if (!child) { + /* Dirty a quarter of the logical pages, retaining untouched siblings. */ + for (size_t page = 0; page < LENGTH / ps; page += 4) + for (size_t j = 0; j < ps; j++) + p[page * ps + j] ^= 0x69; + _exit(contents(p, ps, seed, round, 0, 1) ? 7 : 0); + } + int status; + if (waitpid(child, &status, 0) != child || !WIFEXITED(status) || + WEXITSTATUS(status) || contents(p, ps, seed, round, 0, 0)) + return 8; + printf("ok - memcg ABI %zuK round=%u swap-in bytes and fork COW isolation\n", + ps / 1024, round); + if (round + 1 < ROUNDS && + contents(p, ps, seed, round + 1, 1, 0)) + return 9; + if (barrier(ready, command)) + return 10; + } + return munmap(p, LENGTH) ? 11 : 0; +} + +static void report(unsigned phase) +{ + printf("# memcg phase=%u resident=%llu swap=%llu anon=%llu pgscan=%llu pgsteal=%llu max=%llu oom=%llu oom_kill=%llu\n", + phase, get(CG "/memory.current", NULL), + get(CG "/memory.swap.current", NULL), + get(CG "/memory.stat", "anon"), get(CG "/memory.stat", "pgscan"), + get(CG "/memory.stat", "pgsteal"), + get(CG "/memory.events", "max"), get(CG "/memory.events", "oom"), + get(CG "/memory.events", "oom_kill")); +} + +static void pressure_stacks(void) +{ + FILE *tasks = fopen(CG "/cgroup.procs", "r"); + int pid; + + if (!tasks) + return; + while (fscanf(tasks, "%d", &pid) == 1) { + char path[64], stack[4096]; + int fd; + ssize_t n; + + snprintf(path, sizeof(path), "/proc/%d/stack", pid); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + continue; + n = read(fd, stack, sizeof(stack) - 1); + close(fd); + if (n > 0) { + stack[n] = 0; + printf("# pressure stack pid=%d\n%s", pid, stack); + } + } + fclose(tasks); +} + +static void observe_pressure(void) +{ + unsigned int samples = 0; + + for (;;) { + struct timespec now; + + clock_gettime(CLOCK_MONOTONIC, &now); + printf("# io-pressure ms=%lld resident=%llu swap=%llu writeback=%llu " + "swapcached=%llu scan=%llu steal=%llu max=%llu oom=%llu kill=%llu\n", + (long long)now.tv_sec * 1000 + now.tv_nsec / 1000000, + get(CG "/memory.current", NULL), + get(CG "/memory.swap.current", NULL), + get(CG "/memory.stat", "file_writeback"), + get(CG "/memory.stat", "swapcached"), + get(CG "/memory.stat", "pgscan"), + get(CG "/memory.stat", "pgsteal"), + get(CG "/memory.events", "max"), + get(CG "/memory.events", "oom"), + get(CG "/memory.events", "oom_kill")); + if (!(++samples % 8) && + get(CG "/memory.stat", "file_writeback") > 8 * MIB) + pressure_stacks(); + usleep(250000); + } +} + +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--worker") && getppid() == 1) + return worker(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), atoi(argv[4]), + atoi(argv[5])); + bool classic = false, native_control = false, user16k = false; + + if (getpid() != 1 || argc > 3) { + fprintf(stderr, "Dedicated disposable VM PID1 only\n"); + return 2; + } + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--classic")) + classic = true; + else if (!strcmp(argv[i], "--native-control")) + native_control = true; + else if (!strcmp(argv[i], "--user16k")) + user16k = true; + else + return 2; + } + if (native_control && user16k) + return 2; + signal(SIGPIPE, SIG_IGN); + alarm(300); + int failed = 0, active = 0; + pid_t children[2] = { -1, -1 }, observer = -1; + int ready[2][2], command[2][2]; + unsigned long sizes[2] = { getauxval(AT_PAGESZ), 4096 }; + if (native_control) + sizes[1] = sizes[0]; + if (user16k) + sizes[1] = 16384; + if ((sizes[0] != 16384 && sizes[0] != 65536) || sizes[1] > sizes[0]) { + failed = 1; + goto done; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL) || + put("/sys/fs/cgroup/cgroup.subtree_control", "+memory") || + mkdir(CG, 0755) || put(CG "/memory.max", "50331648") || + put(CG "/memory.swap.max", "268435456")) { + perror("cgroup setup"); + failed = 1; + goto done; + } + /* Explicit diagnostic control, never change the production default. */ + if (classic && put("/sys/kernel/mm/lru_gen/enabled", "0")) { + perror("classic reclaim control"); + failed = 1; + goto done; + } + printf("# reclaim mode=%s\n", + classic ? "classic diagnostic" : "default"); + struct stat st; + for (int i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + if (swapon("/dev/vda", 0)) { + perror("swapon"); + failed = 1; + goto done; + } + active = 1; + for (unsigned i = 0; i < 2; i++) + if (pipe(ready[i]) || pipe(command[i])) { + failed = 1; + goto done; + } + observer = fork(); + if (!observer) { + for (unsigned int i = 0; i < 2; i++) { + close(ready[i][0]); + close(ready[i][1]); + close(command[i][0]); + close(command[i][1]); + } + observe_pressure(); + _exit(0); + } + if (observer < 0) { + failed = 1; + goto done; + } + for (unsigned i = 0; i < 2; i++) { + children[i] = fork(); + if (!children[i]) { + char ps[32], index[16], rfd[16], cfd[16]; + for (unsigned j = 0; j < 2; j++) { + close(ready[j][0]); + close(command[j][1]); + if (i != j) { + close(ready[j][1]); + close(command[j][0]); + } + } + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + snprintf(index, sizeof(index), "%u", i); + snprintf(rfd, sizeof(rfd), "%d", ready[i][1]); + snprintf(cfd, sizeof(cfd), "%d", command[i][0]); + if (put(CG "/cgroup.procs", "0") || + prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(12); + execl("/init", "/init", "--worker", ps, index, rfd, cfd, + NULL); + _exit(13); + } + if (children[i] < 0) { + failed = 1; + goto done; + } + } + for (unsigned i = 0; i < 2; i++) { + close(ready[i][1]); + close(command[i][0]); + } + for (unsigned phase = 0; phase <= ROUNDS; phase++) { + char token; + for (unsigned i = 0; i < 2; i++) + if (read(ready[i][0], &token, 1) != 1 || token != 'R') { + failed = 1; + goto done; + } + report(phase); + if (!get(CG "/memory.swap.current", NULL) || + !get(CG "/memory.events", "max") || + !get(CG "/memory.stat", "pgscan") || + !get(CG "/memory.stat", "pgsteal") || + get(CG "/memory.events", "oom") || + get(CG "/memory.events", "oom_kill")) { + printf("not ok - missing pressure evidence or unexpected OOM\n"); + failed = 1; + goto done; + } + for (unsigned i = 0; i < 2; i++) + if (write(command[i][1], "G", 1) != 1) { + failed = 1; + goto done; + } + } + for (unsigned i = 0; i < 2; i++) { + int status = 0; + int ok = waitpid(children[i], &status, 0) == children[i] && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - memcg ABI %luK exit=%#x\n", ok ? "ok" : "not ok", + sizes[i] / 1024, status); + children[i] = -1; + failed |= !ok; + } + /* Pending swap writeback can outlive exit, especially on a slow device. */ + for (unsigned i = 0; i < 3000 && (get(CG "/memory.stat", "anon") || + get(CG "/memory.swap.current", NULL)); + i++) { + if (!(i % 250)) + printf("# exit drain ms=%u swap=%llu writeback=%llu\n", + i * 20, get(CG "/memory.swap.current", NULL), + get(CG "/memory.stat", "file_writeback")); + /* Exit may leave clean, unmapped swap cache after writeback. Reclaim + * it explicitly; EAGAIN means fewer than the requested bytes existed. */ + if (!get(CG "/memory.stat", "file_writeback") && + put(CG "/memory.reclaim", "67108864 swappiness=200") && + errno != EAGAIN) { + perror("exit cache reclaim"); + failed = 1; + break; + } + usleep(20000); + } + report(ROUNDS + 1); + if (get(CG "/memory.stat", "anon") || + get(CG "/memory.swap.current", NULL)) { + printf("not ok - anonymous or swap charge remains after exit\n"); + failed = 1; + } +done: + if (observer > 0) { + kill(observer, SIGTERM); + waitpid(observer, NULL, 0); + } + if (active && failed) + report(ROUNDS + 2); + for (unsigned i = 0; i < 2; i++) + if (children[i] > 0) { + kill(children[i], SIGKILL); + waitpid(children[i], NULL, 0); + } + if (active && failed) { + put(CG "/cgroup.kill", "1"); + put(CG "/memory.max", "max"); + } + if (active && swapoff("/dev/vda")) { + perror("swapoff"); + failed = 1; + } + printf("GRANULE MEMCG PRESSURE %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/memory-interfaces.c b/tools/testing/selftests/arm64/user4k/memory-interfaces.c new file mode 100644 index 0000000000000..330296ed44499 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/memory-interfaces.c @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static size_t page; + +static void residency(void) +{ + unsigned char *p, vec[6]; + bool ok; + unsigned int i; + + p = mmap(NULL, 5 * page, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + ksft_exit_fail_msg("anonymous mmap\n"); + memset(vec, 0x55, sizeof(vec)); + ok = mincore(p, 5 * page - 1, vec) == 0; + for (i = 0; i < 5; i++) + ok &= vec[i] == 0; + ok &= vec[5] == 0x55; + ksft_test_result( + ok, "mincore emits exactly one byte per unfaulted user page\n"); + ok = mprotect(p + 2 * page, page, PROT_READ | PROT_WRITE) == 0; + p[2 * page] = 0x37; + ok &= mincore(p, 5 * page, vec) == 0; + for (i = 0; i < 5; i++) + ok &= vec[i] == (i == 2); + ksft_test_result( + ok, + "mincore reports a separately faulted VMA among cold neighbors\n"); + ok = munmap(p + 3 * page, page) == 0; + errno = 0; + ok &= mincore(p, 5 * page, vec) == -1 && errno == ENOMEM; + errno = 0; + ok &= mincore(p + 1, page, vec) == -1 && errno == EINVAL; + ksft_test_result(ok, + "mincore rejects an unaligned start and a VMA hole\n"); + munmap(p, 5 * page); +} + +static void file_residency(void) +{ + const size_t native = 16384, count = 3 * native / page; + unsigned char *p, *guard, *vec, byte = 0x7b; + int fd = memfd_create("residency", 0); + bool ok; + size_t i; + + if (fd < 0 || ftruncate(fd, 4 * native) || + pwrite(fd, &byte, 1, native) != 1) + ksft_exit_fail_msg("memfd setup\n"); + p = mmap(NULL, 3 * native, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + page); + guard = mmap(NULL, 2 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED || guard == MAP_FAILED || + mprotect(guard + page, page, PROT_NONE)) + ksft_exit_fail_msg("file/vector mmap\n"); + vec = guard + page - count; + memset(vec, 0x55, count); + ok = mincore(p, 3 * native, vec) == 0; + for (i = 0; i < count; i++) + ok &= vec[i] == ((page + i * page) / native == 1); + ksft_test_result( + ok, + "unmapped file leaves share residency at native cache indices\n"); + ok = p[native - page] == byte && mincore(p, 3 * native, vec) == 0; + for (i = 0; i < count; i++) + ok &= vec[i] == ((page + i * page) / native == 1); + ksft_test_result( + ok, + "present and cache-only file residency agree at a guarded vector boundary\n"); + ok = msync(p + page, 1, MS_SYNC) == 0; + errno = 0; + ok &= msync(p + page + 1, page, MS_SYNC) == -1 && errno == EINVAL; + ok &= munmap(p + page, page) == 0; + errno = 0; + ok &= msync(p, 3 * page, MS_SYNC) == -1 && errno == ENOMEM; + ksft_test_result(ok, + "msync uses process alignment and reports holes\n"); + munmap(p, 3 * native); + munmap(guard, 2 * page); + close(fd); +} + +static void large_vector(void) +{ + const size_t count = 16384 + 3; + unsigned char *p = mmap(NULL, count * page, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, + 0); + unsigned char *vec = malloc(count + 1); + bool ok; + size_t i; + + if (p == MAP_FAILED || !vec) + ksft_exit_fail_msg("large vector allocation\n"); + memset(vec, 0x55, count + 1); + ok = mincore(p, count * page - 7, vec) == 0; + for (i = 0; i < count; i++) + ok &= vec[i] == 0; + ok &= vec[count] == 0x55; + ksft_test_result( + ok, "mincore advances across multiple native output buffers\n"); + free(vec); + munmap(p, count * page); +} + +static void seals(void) +{ + unsigned char *p = mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + bool ok; + pid_t pid; + int status = 0; + + if (p == MAP_FAILED) + ksft_exit_fail_msg("seal mmap\n"); + ok = syscall(SYS_mseal, p + page, 1UL, 0UL) == 0; + errno = 0; + ok &= mprotect(p + page, page, PROT_READ) == -1 && errno == EPERM; + errno = 0; + ok &= munmap(p + page, page) == -1 && errno == EPERM; + ok &= mprotect(p, page, PROT_READ) == 0; + ok &= mprotect(p + 2 * page, page, PROT_READ) == 0; + ksft_test_result( + ok, + "a one-byte seal covers only its user page and protects it\n"); + pid = fork(); + if (!pid) + _exit(!(munmap(p + page, page) == -1 && errno == EPERM)); + ok = pid > 0 && waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status); + ksft_test_result(ok, "fork inherits the sealed user page\n"); + munmap(p, page); + munmap(p + 2 * page, page); + /* The sealed VMA is released by process exit. */ +} + +int main(int argc, char **argv) +{ + page = getauxval(AT_PAGESZ); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + ksft_exit_fail_msg("unexpected process page ABI: %zu\n", page); + ksft_print_msg("process page size: %zu\n", page); + ksft_print_header(); + ksft_set_plan(9); + residency(); + file_residency(); + large_vector(); + seals(); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/mglru-granule-init.c b/tools/testing/selftests/arm64/user4k/mglru-granule-init.c new file mode 100644 index 0000000000000..9bb9bb74e38e8 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/mglru-granule-init.c @@ -0,0 +1,365 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CONTROL "/sys/kernel/debug/lru_gen" +#define FULL "/sys/kernel/debug/lru_gen_full" +static int ready_fd = -1, command_fd = -1; +static unsigned worker_index; +static unsigned char pattern(unsigned region, size_t byte, unsigned round) +{ + return (byte ^ (byte >> 12) ^ (region * 37) ^ (round * 53)) & 255; +} +static int generation(unsigned long long *cg, unsigned *node, + unsigned long *max) +{ + FILE *f = fopen(CONTROL, "r"); + char line[256]; + int have_cg = 0, have_node = 0, have_seq = 0; + if (!f) + return -1; + while (fgets(line, sizeof(line), f)) { + unsigned long long id; + unsigned n; + unsigned long seq, age, anon, file; + if (sscanf(line, "memcg %llu", &id) == 1) { + if (have_cg) + break; + *cg = id; + have_cg = 1; + } else if (sscanf(line, " node %u", &n) == 1) { + if (have_node) + break; + *node = n; + have_node = 1; + } else if (have_node && sscanf(line, "%lu %lu %lu %lu", &seq, + &age, &anon, &file) == 4) { + *max = seq; + have_seq = 1; + } + } + fclose(f); + return have_seq ? 0 : -1; +} +static unsigned long scan_evidence(void) +{ + FILE *f = fopen(FULL, "r"); + char line[512]; + unsigned long total = 0; + if (!f) + return 0; + while (fgets(line, sizeof(line), f)) { + unsigned long t, y, found, added; + if (sscanf(line, " %lut %luy %luf %lua", &t, &y, &found, + &added) == 4) { + printf("# MGLRU scan total=%lu young=%lu tables=%lu added=%lu\n", + t, y, found, added); + total += y; + } + } + fclose(f); + return total; +} +static int age(void) +{ + if (ready_fd >= 0) { + char token = 'A'; + return write(ready_fd, &token, 1) != 1 || + read(command_fd, &token, 1) != 1 || token != 'G'; + } + unsigned long long cg = 0, next_cg = 0; + unsigned node = 0, next_node = 0; + unsigned long seq = 0, next = 0; + char command[128]; + if (generation(&cg, &node, &seq)) + return 1; + /* This tree accepts swappiness here; 200 scans both anonymous and file pages. */ + int len = snprintf(command, sizeof(command), "+ %llu %u %lu 200 1\n", + cg, node, seq); + int fd = open(CONTROL, O_WRONLY); + if (fd < 0) + return 1; + ssize_t written = write(fd, command, len); + close(fd); + if (written != len || generation(&next_cg, &next_node, &next) || + next <= seq) { + printf("not ok - MGLRU generation command=%s result=%zd errno=%d next=%lu\n", + command, written, errno, next); + return 1; + } + printf("ok - MGLRU generation %lu -> %lu\n", seq, next); + return 0; +} +static int check_referenced(unsigned char **regions, unsigned nr, + size_t expected) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char line[512]; + int region = -1, failed = 0; + unsigned seen = 0; + unsigned long start, end, kb; + if (!f) + return 1; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) { + region = -1; + for (unsigned i = 0; i < nr; i++) + if (start == (uintptr_t)regions[i]) + region = i; + } else if (region >= 0 && + sscanf(line, "Referenced: %lu kB", &kb) == 1) { + printf("%s - region=%d start=%p referenced=%lu expected=%zu kB\n", + kb == expected ? "ok" : "not ok", region, + regions[region], kb, expected); + failed |= kb != expected; + seen++; + } + } + fclose(f); + return failed || seen != nr; +} +static int exercise(unsigned long expected) +{ + static const unsigned shifts[] = { 21, 25, 29, 30, 36, 39, 42, 47 }; + unsigned char *regions[sizeof(shifts) / sizeof(shifts[0])] = {}; + size_t ps = getauxval(AT_PAGESZ), len = 8 * ps; + unsigned nr = sizeof(shifts) / sizeof(shifts[0]); + int fail = 0, fd = memfd_create("granule-aging", 0); + cpu_set_t cpus; + void *scratch = MAP_FAILED; + CPU_ZERO(&cpus); + CPU_SET(ready_fd >= 0 ? worker_index % 2 : 0, &cpus); + if (sched_setaffinity(0, sizeof(cpus), &cpus)) + return 1; + if (ps != expected || fd < 0 || ftruncate(fd, nr * len)) + return 1; + for (unsigned i = 0; i < nr; i++) { + uintptr_t address = (1UL << shifts[i]) - 4 * ps; + regions[i] = mmap((void *)address, len, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | + (i % 2 ? MAP_SHARED : + MAP_PRIVATE | MAP_ANONYMOUS), + i % 2 ? fd : -1, i % 2 ? i * len : 0); + if (regions[i] != (void *)address) { + printf("not ok - %zuK boundary %u mmap errno=%d\n", + ps / 1024, shifts[i], errno); + fail = 1; + goto out; + } + } + for (unsigned i = 0; i < nr; i++) + memset(regions[i], 0x19, len); + /* Publish earlier folios from the bounded per-CPU LRU-add batch. */ + scratch = mmap(NULL, 8UL << 20, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (scratch == MAP_FAILED) { + fail = 1; + goto out; + } + memset(scratch, 0x7b, 8UL << 20); + /* Seed MGLRU's first-access referenced state before testing the next access. */ + if (age()) { + fail = 1; + goto out; + } + for (unsigned round = 0; round < 4; round++) { + if (ready_fd >= 0) { + unsigned cpu = (worker_index + round) % 2; + CPU_ZERO(&cpus); + CPU_SET(cpu, &cpus); + if (sched_setaffinity(0, sizeof(cpus), &cpus) || + sched_getcpu() != (int)cpu) { + fail = 1; + break; + } + printf("# concurrent ABI=%zuK round=%u cpu=%u\n", + ps / 1024, round, cpu); + } + for (unsigned i = 0; i < nr; i++) + for (size_t j = 0; j < len; j++) + regions[i][j] = pattern(i, j, round); + fail |= check_referenced(regions, nr, len / 1024); + if (age() || age()) { + fail = 1; + break; + } + fail |= check_referenced(regions, nr, 0); + int good = 1; + for (unsigned i = 0; i < nr; i++) + for (size_t j = 0; j < len; j++) + if (regions[i][j] != pattern(i, j, round)) { + good = 0; + break; + } + unsigned long young = scan_evidence(); + printf("%s - %zuK round%u eight table boundaries bytes and aging young=%lu\n", + good && young ? "ok" : "not ok", ps / 1024, round, + young); + fail |= !good || !young; + } +out: + if (scratch != MAP_FAILED) + munmap(scratch, 8UL << 20); + for (unsigned i = 0; i < nr; i++) + if (regions[i] && regions[i] != MAP_FAILED) + munmap(regions[i], len); + close(fd); + return fail; +} +static int concurrent(unsigned long *sizes, unsigned nr) +{ + int ready[3][2], commands[3][2], failed = 0; + pid_t workers[3]; + + for (unsigned i = 0; i < nr; i++) + if (pipe(ready[i]) || pipe(commands[i])) + return 1; + for (unsigned i = 0; i < nr; i++) { + workers[i] = fork(); + if (!workers[i]) { + char size[32], index[16], ready_arg[16], + command_arg[16]; + for (unsigned j = 0; j < nr; j++) { + close(ready[j][0]); + close(commands[j][1]); + if (j != i) { + close(ready[j][1]); + close(commands[j][0]); + } + } + snprintf(size, sizeof(size), "%lu", sizes[i]); + snprintf(index, sizeof(index), "%u", i); + snprintf(ready_arg, sizeof(ready_arg), "%d", + ready[i][1]); + snprintf(command_arg, sizeof(command_arg), "%d", + commands[i][0]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--worker", size, index, + ready_arg, command_arg, NULL); + _exit(4); + } + if (workers[i] < 0) + failed = 1; + } + for (unsigned i = 0; i < nr; i++) { + close(ready[i][1]); + close(commands[i][0]); + } + /* One warm-up barrier plus two aging barriers for each of four rounds. */ + for (unsigned phase = 0; phase < 9 && !failed; phase++) { + for (unsigned i = 0; i < nr; i++) { + char token; + if (read(ready[i][0], &token, 1) != 1 || token != 'A') + failed = 1; + } + if (failed || age()) { + failed = 1; + break; + } + for (unsigned i = 0; i < nr; i++) { + char token = 'G'; + if (write(commands[i][1], &token, 1) != 1) + failed = 1; + } + } + for (unsigned i = 0; i < nr; i++) { + close(ready[i][0]); + close(commands[i][1]); + } + for (unsigned i = 0; i < nr; i++) { + int status = 0; + int ok = workers[i] > 0 && + waitpid(workers[i], &status, 0) == workers[i] && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - concurrent MGLRU ABI %luK status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + failed |= !ok; + } + return failed; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (argc == 6 && !strcmp(argv[1], "--worker")) { + if (getppid() != 1) + return 2; + worker_index = strtoul(argv[3], NULL, 10); + ready_fd = atoi(argv[4]); + command_fd = atoi(argv[5]); + return exercise(strtoul(argv[2], NULL, 10)); + } + int concurrent_mode = argc == 2 && !strcmp(argv[1], "--concurrent"); + if (getpid() != 1 || (argc != 1 && !concurrent_mode)) + return 2; + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + FILE *f = fopen("/sys/kernel/mm/lru_gen/enabled", "r"); + unsigned enabled = 0; + if (f) { + fscanf(f, "%x", &enabled); + fclose(f); + } + if ((enabled & 3) != 3) { + printf("not ok - MGLRU/MM walk disabled %#x\n", enabled); + fail = 1; + goto done; + } + printf("ok - MGLRU enabled=%#x\n", enabled); + unsigned long sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + if (concurrent_mode) { + fail = concurrent(sizes, sizes[0] > 16384 ? 3 : 2); + goto done; + } + for (unsigned i = 0; i < (sizes[0] > 16384 ? 3U : 2U); i++) { + pid_t child = fork(); + int status = 0; + if (!child) { + char value[32]; + snprintf(value, sizeof(value), "%lu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--exercise", value, NULL); + _exit(4); + } + int good = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - MGLRU ABI %luK status=%#x\n", + good ? "ok" : "not ok", sizes[i] / 1024, status); + fail |= !good; + } +done: + printf("GRANULE MGLRU%s %s\n", concurrent_mode ? " CONCURRENT" : "", + fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/mixed-process.c b/tools/testing/selftests/arm64/user4k/mixed-process.c new file mode 100644 index 0000000000000..6d08ab55991b5 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/mixed-process.c @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real EL0 mixed-granule sharing, futex queues, and process_vm copies. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#define NATIVE 16384U +#define QUARTER 4096U +#define ROUNDS 2000U +struct channel { + atomic_int turn; + unsigned int value; + unsigned int worker_wakes; +}; +struct ready { + uintptr_t base; + unsigned long page; +}; + +static int wait_turn(struct channel *channel, int wanted) +{ + const struct timespec timeout = { .tv_sec = 5 }; + int value; + + while ((value = atomic_load_explicit(&channel->turn, + memory_order_acquire)) != wanted) { + if (syscall(SYS_futex, &channel->turn, FUTEX_WAIT, value, + &timeout, NULL, 0) < 0 && + errno != EAGAIN && errno != EINTR) + return -1; + } + return 0; +} + +static int send_turn(struct channel *channel, int turn) +{ + atomic_store_explicit(&channel->turn, turn, memory_order_release); + return syscall(SYS_futex, &channel->turn, FUTEX_WAKE, 1, NULL, NULL, 0); +} + +static unsigned char pattern(size_t i) +{ + return (i ^ (i >> 8) ^ (i >> 16)) & 255; +} + +static int worker(int fd, int ready_fd, unsigned long expected) +{ + unsigned long page = getauxval(AT_PAGESZ); + unsigned char *mapping, *remote; + struct channel *channel; + struct ready ready; + size_t i, begin = page - 13, end = 3 * page + 16; + unsigned int round, wakes = 0; + int ret; + + if (page != expected) + return 10; + mapping = mmap(NULL, page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + page == QUARTER ? QUARTER : 0); + remote = mmap(NULL, 5 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED || remote == MAP_FAILED) + return 11; + channel = (void *)(mapping + (page == QUARTER ? 0 : QUARTER)); + for (i = 0; i < 4 * page; i++) + remote[i] = pattern(i); + if (mprotect(remote + 4 * page, page, PROT_NONE)) + return 12; + ready = (struct ready){ (uintptr_t)remote, page }; + if (write(ready_fd, &ready, sizeof(ready)) != sizeof(ready)) + return 13; + close(ready_fd); + for (round = 0; round < ROUNDS; round++) { + if (wait_turn(channel, 1) || + channel->value != (round ^ 0xabcddcbaU)) + return 14; + if (!round) { + for (i = 0; i < 4 * page; i++) + if (remote[i] != + (pattern(i) ^ + (i >= begin && i < end ? 0x5a : 0))) + return 15; + } + channel->value = ~(round ^ 0xabcddcbaU); + ret = send_turn(channel, 0); + if (ret < 0) + return 16; + wakes += ret; + } + channel->worker_wakes = wakes; + return 0; +} + +static bool remote_copies(pid_t pid, const struct ready *ready) +{ + size_t begin = ready->page - 13, len = 2 * ready->page + 29, i; + unsigned char *data = malloc(len), edge[2] = {}; + struct iovec local = { data, len }, + remote = { (void *)(ready->base + begin), len }; + bool ok = data != NULL; + + if (!ok) + return false; + ok = process_vm_readv(pid, &local, 1, &remote, 1, 0) == (ssize_t)len; + for (i = 0; ok && i < len; i++) + ok = data[i] == pattern(i + begin); + for (i = 0; i < len; i++) + data[i] = pattern(i + begin) ^ 0x5a; + ok &= process_vm_writev(pid, &local, 1, &remote, 1, 0) == (ssize_t)len; + local = (struct iovec){ edge, sizeof(edge) }; + remote = (struct iovec){ (void *)(ready->base + 4 * ready->page - 1), + sizeof(edge) }; + ok &= process_vm_readv(pid, &local, 1, &remote, 1, 0) == 1; + ok &= edge[0] == pattern(4 * ready->page - 1); + free(data); + return ok; +} + +int main(int argc, char **argv) +{ + unsigned long page = getauxval(AT_PAGESZ), other; + unsigned char *mapping, *private; + struct channel *channel; + struct ready ready = {}; + int fd, pipefd[2], status = 0, wakes = 0, ret; + unsigned int round = 0; + pid_t pid; + bool ok; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10)); + if (argc != 1 || (page != QUARTER && page != NATIVE)) + return 2; + other = page == NATIVE ? QUARTER : NATIVE; + fd = memfd_create("mixed-process", 0); + if (fd < 0 || ftruncate(fd, 2 * NATIVE) || pipe(pipefd)) + return 3; + mapping = mmap(NULL, 2 * NATIVE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + 0); + private = + mmap(NULL, NATIVE, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); + if (mapping == MAP_FAILED || private == MAP_FAILED) + return 4; + channel = (void *)(mapping + QUARTER); + private[QUARTER] = 0x7e; + pid = fork(); + if (!pid) { + char fdarg[32], pipearg[32], sizearg[32]; + + close(pipefd[0]); + snprintf(fdarg, sizeof(fdarg), "%d", fd); + snprintf(pipearg, sizeof(pipearg), "%d", pipefd[1]); + snprintf(sizearg, sizeof(sizearg), "%lu", other); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, other, 0UL, 0UL, 0UL)) + _exit(20); + execl("/mixed-process", "/mixed-process", "--worker", fdarg, + pipearg, sizearg, NULL); + _exit(21); + } + close(pipefd[1]); + if (pid < 0) + return 5; + ok = read(pipefd[0], &ready, sizeof(ready)) == sizeof(ready) && + ready.page == other; + close(pipefd[0]); + ok = ok && remote_copies(pid, &ready); + printf("%s - process_vm copies %lu -> %lu, including partial inaccessible boundary\n", + ok ? "ok" : "not ok", page, other); + for (round = 0; ok && round < ROUNDS; round++) { + channel->value = round ^ 0xabcddcbaU; + ret = send_turn(channel, 1); + ok = ret >= 0 && !wait_turn(channel, 0) && + channel->value == ~(round ^ 0xabcddcbaU); + if (ret > 0) + wakes += ret; + } + if (!ok) + kill(pid, SIGKILL); + ret = waitpid(pid, &status, 0); + ok &= ret == pid && WIFEXITED(status) && !WEXITSTATUS(status); + ok &= wakes > 0 && channel->worker_wakes > 0 && + private[QUARTER] == 0x7e; + ok &= mapping[0] == 0 && mapping[2 * QUARTER] == 0 && + mapping[NATIVE - 1] == 0; + printf("%s - mixed %lu/%lu shared futex/COW: rounds=%u wakes=%d/%u child=%#x\n", + ok ? "ok" : "not ok", page, other, round, wakes, + channel->worker_wakes, status); + munmap(private, NATIVE); + munmap(mapping, 2 * NATIVE); + close(fd); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/mlock-fixture-init.c b/tools/testing/selftests/arm64/user4k/mlock-fixture-init.c new file mode 100644 index 0000000000000..0dc94eabcc4fb --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/mlock-fixture-init.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static long locked_kb(void) +{ + char line[256]; + long value = -1; + FILE *f = fopen("/proc/self/status", "re"); + + if (!f) + return -1; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmLck: %ld", &value) == 1) + break; + fclose(f); + return value; +} + +static bool wait_ok(pid_t pid) +{ + int status = 0; + + return pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); +} + +static bool remap_locked(unsigned long ps) +{ + unsigned char *p, *q, *target; + unsigned long kb = ps / 1024; + bool ok; + + p = mmap(NULL, 2 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED || mlock(p, 2 * ps)) + ksft_exit_fail_msg("mlock remap setup\n"); + memset(p, 0x37, 2 * ps); + ok = locked_kb() == (long)(2 * kb); + q = mremap(p, 2 * ps, 4 * ps, MREMAP_MAYMOVE); + if (q == MAP_FAILED) + ksft_exit_fail_msg("locked remap grow: %s\n", strerror(errno)); + ok &= locked_kb() == (long)(4 * kb); + for (unsigned long i = 0; i < 4 * ps; i++) + ok &= q[i] == (i < 2 * ps ? 0x37 : 0); + p = mremap(q, 4 * ps, 3 * ps, 0); + if (p == MAP_FAILED) + ksft_exit_fail_msg("locked remap shrink: %s\n", + strerror(errno)); + ok &= locked_kb() == (long)(3 * kb); + p[2 * ps] = 0xb6; + ok &= !munmap(p + ps, ps) && locked_kb() == (long)(2 * kb); + target = mmap(NULL, ps, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (target == MAP_FAILED) + ksft_exit_fail_msg("locked remap target\n"); + q = mremap(p, ps, ps, MREMAP_MAYMOVE | MREMAP_FIXED, target); + if (q == MAP_FAILED) + ksft_exit_fail_msg("locked remap move: %s\n", strerror(errno)); + ok &= q == target && locked_kb() == (long)(2 * kb); + for (unsigned long i = 0; i < ps; i++) + ok &= q[i] == 0x37; + ok &= p[2 * ps] == 0xb6; + ok &= !munmap(q, ps) && locked_kb() == (long)kb; + ok &= !munmap(p + 2 * ps, ps) && locked_kb() == 0; + return ok; +} + +static int exercise(void) +{ + unsigned long ps = getauxval(AT_PAGESZ), kb = ps / 1024; + unsigned char *p, *q, residency[4]; + int gate[2]; + pid_t pid; + bool ok; + + ksft_print_header(); + ksft_set_plan(9); + p = mmap(NULL, 32 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED || locked_kb() != 0) + ksft_exit_fail_msg("initial mapping/accounting\n"); + ok = !mlock(p + ps + 17, ps - 32) && locked_kb() == (long)kb; + ok &= !mincore(p + ps, ps, residency) && residency[0] & 1; + p[ps] = 0x51; + ksft_test_result( + ok, "%luK unaligned mlock uses exactly one user leaf\n", kb); + ok = !mlock(p + ps, ps) && locked_kb() == (long)kb; + ok &= !mlock(p + 2 * ps, ps) && locked_kb() == (long)(2 * kb); + ok &= !munlock(p + ps, ps) && locked_kb() == (long)kb; + p[2 * ps] = 0x62; + errno = 0; + ok &= madvise(p + 2 * ps, ps, MADV_PAGEOUT) == -1 && errno == EINVAL && + p[2 * ps] == 0x62; + ok &= !munlock(p + 2 * ps, ps) && locked_kb() == 0; + ksft_test_result( + ok, "%luK repeat lock and independent adjacent unlock\n", kb); + ok = !mlock2(p + 4 * ps, 4 * ps, MLOCK_ONFAULT) && + locked_kb() == (long)(4 * kb); + ok &= !mincore(p + 4 * ps, 4 * ps, residency); + for (unsigned int i = 0; i < 4; i++) + ok &= !(residency[i] & 1); + p[4 * ps] = 0x74; + ok &= !mincore(p + 4 * ps, ps, residency) && residency[0] & 1; + ok &= !munlock(p + 4 * ps, 4 * ps) && locked_kb() == 0; + ksft_test_result( + ok, "%luK MLOCK_ONFAULT accounting and deferred faults\n", kb); + q = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_LOCKED, -1, 0); + ok = q != MAP_FAILED && locked_kb() == (long)(4 * kb); + if (q == MAP_FAILED || pipe(gate)) + ksft_exit_fail_msg("locked mmap/fork setup\n"); + memset(q, 0x8a, 4 * ps); + pid = fork(); + if (!pid) { + char token; + + close(gate[1]); + if (read(gate[0], &token, 1) != 1 || locked_kb() != 0) + _exit(2); + for (unsigned long i = 0; i < 4 * ps; i++) + if (q[i] != 0x8a) + _exit(3); + _exit(0); + } + close(gate[0]); + memset(q + ps, 0x9b, ps); + ok &= write(gate[1], "x", 1) == 1; + close(gate[1]); + ok &= wait_ok(pid) && locked_kb() == (long)(4 * kb); + ok &= !munmap(q, 4 * ps) && locked_kb() == 0; + ksft_test_result( + ok, "%luK MAP_LOCKED, fork noninheritance and parent COW\n", + kb); + ok = !mlockall(MCL_FUTURE | MCL_ONFAULT); + q = mmap(NULL, ps, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + ok &= q != MAP_FAILED && locked_kb() == (long)kb; + if (q != MAP_FAILED) + *q = 0xaf; + ok &= !munlockall() && locked_kb() == 0; + if (q != MAP_FAILED) + munmap(q, ps); + ksft_test_result(ok, "%luK future on-fault locks and munlockall\n", kb); + pid = fork(); + if (!pid) { + struct rlimit limit = { ps, ps }; + + if (setrlimit(RLIMIT_MEMLOCK, &limit) || setuid(1000) || + mlock(p, ps)) + _exit(4); + errno = 0; + if (mlock(p + ps, ps) != -1 || errno != ENOMEM || + locked_kb() != (long)kb) + _exit(5); + _exit(0); + } + ksft_test_result( + wait_ok(pid), + "%luK unprivileged RLIMIT_MEMLOCK in user-page units\n", kb); + pid = fork(); + if (!pid) { + if (mlockall(MCL_CURRENT) || locked_kb() <= (long)kb || + munlockall() || locked_kb() != 0) + _exit(6); + _exit(0); + } + ksft_test_result(wait_ok(pid), "%luK lock/unlock current mappings\n", + kb); + { + unsigned long huge = ((1UL << 31) + 1) * ps; + + q = mmap(NULL, huge, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + ok = q != MAP_FAILED; + if (q != MAP_FAILED) { + ok &= !mlock2(q, huge, MLOCK_ONFAULT) && + locked_kb() == (long)(huge / 1024); + ok &= !munlock(q, huge) && locked_kb() == 0; + munmap(q, huge); + } + ksft_test_result( + ok, + "%luK deferred lock count beyond signed 32-bit leaves\n", + kb); + } + ksft_test_result( + remap_locked(ps), + "%luK locked grow/shrink/move and partial unmap accounting\n", + kb); + munmap(p, 32 * ps); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} + +int main(int argc, char **argv) +{ + bool ok = true; + unsigned long native = getauxval(AT_PAGESZ); + unsigned long granules[] = { native, 4096, 16384 }; + unsigned int nr = native > 16384 ? 3 : 2; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--test") && getppid() == 1) { + if (getauxval(AT_PAGESZ) != strtoul(argv[2], NULL, 10)) + return 42; + return exercise(); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL)) + return 3; + for (unsigned int small = 0; small < nr; small++) { + pid_t pid = fork(); + + if (!pid) { + char size[32], host_size[32]; + + snprintf(size, sizeof(size), "%lu", granules[small]); + snprintf(host_size, sizeof(host_size), "%lu", native); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, granules[small], + 0UL, 0UL, 0UL)) + _exit(7); + execl("/init", "/init", "--test", size, host_size, + NULL); + _exit(8); + } + ok &= wait_ok(pid); + } + printf("%s - mlock fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/numa-balancing-init.c b/tools/testing/selftests/arm64/user4k/numa-balancing-init.c new file mode 100644 index 0000000000000..2ea31bda12968 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/numa-balancing-init.c @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Disposable two-node guest: real automatic NUMA hinting and migration. + * Volatile accesses intentionally trigger page faults and repeated accesses; + * thread coordination uses an atomic flag, not volatile storage. + */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int good, const char *expr, int line) +{ + if (!good) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expr, + errno); + exit(1); + } +} +#define CHECK(x) check(!!(x), #x, __LINE__) + +static void setting(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + + CHECK(fd >= 0); + CHECK(write(fd, value, strlen(value)) == (ssize_t)strlen(value)); + CHECK(!close(fd)); +} + +static void cpu(int id) +{ + cpu_set_t set; + + CPU_ZERO(&set); + CPU_SET(id, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} + +static unsigned long counter(const char *key) +{ + FILE *f = fopen("/proc/vmstat", "r"); + char name[128]; + unsigned long value; + + CHECK(f); + while (fscanf(f, "%127s %lu", name, &value) == 2) { + if (!strcmp(name, key)) { + CHECK(!fclose(f)); + return value; + } + } + CHECK(0); + return 0; +} + +static double now(void) +{ + struct timespec ts; + + CHECK(!clock_gettime(CLOCK_MONOTONIC, &ts)); + return ts.tv_sec + ts.tv_nsec / 1e9; +} + +static int node_at(void *address) +{ + int node = -1; + + CHECK(!syscall(SYS_get_mempolicy, &node, NULL, 0UL, address, + MPOL_F_ADDR | MPOL_F_NODE)); + return node; +} + +static unsigned long sample(unsigned char *map, unsigned long length, + unsigned long page) +{ + unsigned long moved = 0; + + for (unsigned long off = 0; off < length; off += page) + if (off != 2 * page) + moved += node_at(map + off + 16) == 1; + return moved; +} + +static void forbidden(unsigned char *address, int write_access) +{ + pid_t child = fork(); + int status; + + CHECK(child >= 0); + if (!child) { + if (write_access) + *(volatile unsigned char *)address = 0x73; + else + (void)*(volatile unsigned char *)address; + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + CHECK(WIFSIGNALED(status) && WTERMSIG(status) == SIGSEGV); +} + +struct local_reader { + unsigned char *map; + unsigned long length, page; + atomic_int stop; +}; + +static void *read_local(void *arg) +{ + struct local_reader *r = arg; + + cpu(1); + while (!atomic_load_explicit(&r->stop, memory_order_relaxed)) { + for (unsigned long off = 0; off < r->length; off += r->page) { + if (off == 2 * r->page) + continue; + CHECK(*(volatile uint64_t *)(r->map + off) == + 0x81720000 + off / r->page); + } + } + return NULL; +} + +static void local_restore(unsigned char *map, unsigned long length, + unsigned long page) +{ + struct local_reader r = { .map = map, .length = length, .page = page }; + unsigned long hints = counter("numa_hint_faults_local"); + pthread_t worker; + double deadline = now() + 2; + + /* Two threads permit hinting of already-local private mappings. */ + CHECK(!pthread_create(&worker, NULL, read_local, &r)); + do { + for (unsigned long off = 0; off < length; off += page) { + if (off == 2 * page) + continue; + CHECK(*(volatile uint64_t *)(map + off) == + 0x81720000 + off / page); + } + } while (now() < deadline); + atomic_store_explicit(&r.stop, 1, memory_order_relaxed); + CHECK(!pthread_join(worker, NULL)); + hints = counter("numa_hint_faults_local") - hints; + printf("NUMAB local page=%lu hints=%lu\n", page, hints); + CHECK(hints > 0); +} + +static void exercise(void) +{ + unsigned long page = getauxval(AT_PAGESZ), length = 8UL << 20; + unsigned long mask = 1, allowed = 0, hints, migrated, iterations = 0; + unsigned long moved = 0, total = length / page - 1; + unsigned char *map; + int mode; + + CHECK(!syscall(SYS_get_mempolicy, &mode, &allowed, 64UL, NULL, + MPOL_F_MEMS_ALLOWED)); + CHECK(allowed == 3); + cpu(0); + map = mmap(NULL, length, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(map != MAP_FAILED); + CHECK(!madvise(map, length, MADV_NOHUGEPAGE)); + CHECK(!syscall(SYS_mbind, map, length, MPOL_BIND, &mask, 64UL, 0)); + for (unsigned long off = 0; off < length; off += page) { + *(uint64_t *)(map + off) = 0x81720000 + off / page; + *(uint64_t *)(map + off + 8) = 0; + *(uint64_t *)(map + off + page - 8) = 0x61320000 + off / page; + CHECK(node_at(map + off + 16) == 0); + } + CHECK(!mprotect(map + page, page, PROT_READ)); + CHECK(!mprotect(map + 2 * page, page, PROT_NONE)); + CHECK(!syscall(SYS_mbind, map, length, MPOL_DEFAULT, NULL, 0UL, 0)); + cpu(1); + hints = counter("numa_hint_faults"); + migrated = counter("numa_pages_migrated"); + double started = now(), deadline = started + 20; + + do { + for (unsigned long off = 0; off < length; off += page) { + volatile uint64_t *p = (void *)(map + off); + + if (off == 2 * page) + continue; + CHECK(p[0] == 0x81720000 + off / page); + CHECK(p[page / 8 - 1] == 0x61320000 + off / page); + if (off != page) + p[1]++; + } + iterations++; + if (!(iterations % 128)) { + moved = sample(map, length, page); + if (moved == total) + break; + } + } while (now() < deadline); + moved = sample(map, length, page); + hints = counter("numa_hint_faults") - hints; + migrated = counter("numa_pages_migrated") - migrated; + printf("NUMAB page=%lu moved=%lu/%lu hints=%lu migrated=%lu loops=%lu seconds=%.3f\n", + page, moved, total, hints, migrated, iterations, + now() - started); + CHECK(moved == total && hints > 0 && migrated > 0); + for (unsigned long off = 0; off < length; off += page) { + if (off != page && off != 2 * page) + CHECK(*(uint64_t *)(map + off + 8) == iterations); + } + local_restore(map, length, page); + forbidden(map + page, 1); + forbidden(map + 2 * page, 0); + CHECK(!mprotect(map + 2 * page, page, PROT_READ)); + CHECK(*(uint64_t *)(map + 2 * page) == 0x81720002); + CHECK(!munmap(map, length)); + printf("ok - NUMAB page=%lu automatic placement, bytes and permissions\n", + page); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ), + sizes[] = { 4096, 16384, 65536 }; + int failed = 0, compat_only = 0, no4k = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--case")) { + CHECK(getauxval(AT_PAGESZ) == strtoul(argv[2], NULL, 10)); + exercise(); + return 0; + } + CHECK(getpid() == 1); + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--compat-only")) + compat_only = 1; + else if (!strcmp(argv[i], "--no4k")) + no4k = 1; + else + CHECK(0); + } + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + CHECK(!mount("debugfs", "/debug", "debugfs", 0, NULL)); + setting("/proc/sys/kernel/numa_balancing", "1\n"); + setting("/debug/sched/numa_balancing/scan_delay_ms", "100\n"); + setting("/debug/sched/numa_balancing/scan_period_min_ms", "100\n"); + setting("/debug/sched/numa_balancing/scan_period_max_ms", "100\n"); + setting("/debug/sched/numa_balancing/scan_size_mb", "16\n"); + for (unsigned int i = 0; i < 3; i++) { + pid_t child; + int status; + + if (compat_only || sizes[i] > native || + (no4k && sizes[i] == 4096)) + continue; + child = fork(); + CHECK(child >= 0); + if (!child) { + char s[32]; + + snprintf(s, sizeof(s), "%lu", sizes[i]); + CHECK(!prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], + 0UL, 0UL, 0UL)); + execl("/init", "/init", "--case", s, NULL); + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + failed |= !WIFEXITED(status) || WEXITSTATUS(status); + } + CHECK(!access("/compat-numa-balancing-probe", X_OK)); + pid_t child = fork(); + int status; + + CHECK(child >= 0); + if (!child) { + char s[32]; + + snprintf(s, sizeof(s), "%lu", no4k ? native : 4096UL); + execl("/compat-numa-balancing-probe", + "/compat-numa-balancing-probe", s, NULL); + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + failed |= !WIFEXITED(status) || WEXITSTATUS(status); + printf("A32 NUMAB expected-page=%lu status=%d\n", + no4k ? native : 4096UL, status); + printf("NUMA BALANCING %s\n", failed ? "FAIL" : "PASS"); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/numa-policy-init.c b/tools/testing/selftests/arm64/user4k/numa-policy-init.c new file mode 100644 index 0000000000000..fedcedb529709 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/numa-policy-init.c @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable two-node guest: policy ranges and native physical placement. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int good, const char *expr, int line) +{ + if (!good) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expr, + errno); + exit(1); + } +} +#define CHECK(x) check(!!(x), #x, __LINE__) + +static long bind_to(void *addr, unsigned long bytes, int node, + unsigned int flags) +{ + unsigned long mask = 1UL << node; + + return syscall(SYS_mbind, addr, bytes, MPOL_BIND, &mask, 64UL, flags); +} + +static void policy_at(void *addr, int node) +{ + unsigned long mask = 0; + int mode = -1; + + CHECK(!syscall(SYS_get_mempolicy, &mode, &mask, 64UL, addr, + MPOL_F_ADDR)); + CHECK(mode == MPOL_BIND && mask == (1UL << node)); +} + +static void node_at(void *addr, int expected) +{ + int node = -1; + + CHECK(!syscall(SYS_get_mempolicy, &node, NULL, 0UL, addr, + MPOL_F_NODE | MPOL_F_ADDR)); + if (node != expected) + fprintf(stderr, "node=%d expected=%d addr=%p\n", node, expected, + addr); + CHECK(node == expected); +} + +static void exercise(unsigned long native, int baseline) +{ + unsigned long page = getauxval(AT_PAGESZ), allowed = 0; + unsigned char *reserve, *base, *shared, *alias; + int mode, fd; + + CHECK(!syscall(SYS_get_mempolicy, &mode, &allowed, 64UL, NULL, + MPOL_F_MEMS_ALLOWED)); + CHECK(allowed == + 3); /* Require two real memory nodes, not a silent skip. */ + reserve = mmap(NULL, 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + base = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + CHECK(!bind_to(base, 2 * native, 0, 0)); + if (baseline) { + errno = 0; + long ret = bind_to(base + page, 1, 1, 0); + + CHECK(page < native ? ret == -1 && errno == EINVAL : ret == 0); + printf("ok - baseline native=%lu page=%lu mbind=%ld errno=%d\n", + native, page, ret, errno); + CHECK(!munmap(reserve, 4 * native)); + return; + } + CHECK(!bind_to(base + page, 1, 1, 0)); + policy_at(base, 0); + policy_at(base + page, 1); + if (page < native) + policy_at(base + 2 * page, 0); + CHECK(!syscall(SYS_set_mempolicy_home_node, base + page, 1UL, 1UL, + 0UL)); + errno = 0; + CHECK(bind_to(base + 1, page, 1, 0) == -1 && errno == EINVAL); + for (unsigned long off = 0; off < 2 * native; off += page) { + base[off + 17] = 0x31 + off / page; + node_at(base + off + 17, off == page ? 1 : 0); + } + CHECK(!bind_to(base + page, page, 1, MPOL_MF_STRICT)); + errno = 0; + CHECK(bind_to(base, 2 * native, 1, MPOL_MF_STRICT) == -1 && + errno == EIO); + policy_at(base, 0); + CHECK(!munmap(reserve, 4 * native)); + + /* A split anonymous VMA must carry its native interleave index correctly. */ + reserve = mmap(NULL, 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + base = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + unsigned long mask = 3, inset = page < native ? page : 0; + + CHECK(!syscall(SYS_mbind, base + inset, 2 * native - 2 * inset, + MPOL_INTERLEAVE, &mask, 64UL, 0)); + for (unsigned long off = inset; off < 2 * native - inset; off += page) { + base[off + 17] = 0x43; + node_at(base + off + 17, ((uintptr_t)base + off) / native % 2); + } + CHECK(!munmap(reserve, 4 * native)); + + /* Moving a fragment moves its native physical owner, retaining bytes. */ + reserve = mmap(NULL, 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + base = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + CHECK(!bind_to(base, 2 * native, 0, 0)); + for (unsigned long off = 0; off < 2 * native; off += page) + base[off + 17] = 0x51 + off / page; + CHECK(!bind_to(base + native - page, page, 1, + MPOL_MF_MOVE_ALL | MPOL_MF_STRICT)); + for (unsigned long off = 0; off < 2 * native; off += page) { + CHECK(base[off + 17] == (unsigned char)(0x51 + off / page)); + node_at(base + off + 17, off < native ? 1 : 0); + } + CHECK(!bind_to(base, 2 * native, 0, MPOL_MF_MOVE_ALL | MPOL_MF_STRICT)); + for (unsigned long off = 0; off < 2 * native; off += page) + node_at(base + off + 17, 0); + CHECK(!munmap(reserve, 4 * native)); + + fd = memfd_create("numa-shared", 0); + CHECK(fd >= 0 && !ftruncate(fd, 2 * native)); + shared = mmap(NULL, 2 * native, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + 0); + CHECK(shared != MAP_FAILED); + CHECK(!bind_to(shared, native, 0, 0)); + CHECK(!bind_to(shared + native, native, 1, 0)); + if (page < native) { + alias = mmap(NULL, 2 * native - page, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, page); + CHECK(alias != MAP_FAILED); + policy_at(alias, 0); + policy_at(alias + native - page, 1); + errno = 0; + CHECK(bind_to(alias, page, 1, 0) == -1 && errno == EINVAL); + policy_at(alias, 0); + policy_at(shared, 0); + CHECK(!bind_to(alias + native - page, native, 1, 0)); + for (unsigned long off = 0; off < 2 * native - page; + off += page) { + alias[off + 17] = 0x72; + node_at(alias + off + 17, off + page < native ? 0 : 1); + CHECK(shared[off + page + 17] == 0x72); + } + CHECK(!munmap(alias, 2 * native - page)); + } + CHECK(!munmap(shared, 2 * native) && !close(fd)); + printf("ok - NUMA native=%lu page=%lu policy, placement, migration and shared offsets\n", + native, page); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ), + sizes[] = { 4096, 16384, 65536 }; + int failed = 0, baseline = 0, no4; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--case")) { + CHECK(getauxval(AT_PAGESZ) == strtoul(argv[3], NULL, 10)); + exercise(strtoul(argv[2], NULL, 10), atoi(argv[4])); + return 0; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + FILE *cmdline = fopen("/proc/cmdline", "r"); + char line[4096]; + + CHECK(cmdline && fgets(line, sizeof(line), cmdline)); + baseline = strstr(line, "numa_fixture_baseline=1") != NULL; + no4 = strstr(line, "id_aa64mmfr0.tgran4=f") != NULL; + CHECK(!fclose(cmdline)); + for (unsigned int i = 0; i < 3; i++) { + int status; + pid_t child; + + if (sizes[i] > native || (no4 && sizes[i] == 4096)) + continue; + child = fork(); + CHECK(child >= 0); + if (!child) { + char n[32], s[32], b[4]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", sizes[i]); + snprintf(b, sizeof(b), "%d", baseline); + CHECK(!prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], + 0UL, 0UL, 0UL)); + execl("/init", "/init", "--case", n, s, b, NULL); + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + failed |= !WIFEXITED(status) || WEXITSTATUS(status); + } + if (!baseline) { + int status; + pid_t child = fork(); + + CHECK(child >= 0); + if (!child) { + char n[32], s[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", no4 ? native : 4096UL); + execl("/compat-numa-probe", "/compat-numa-probe", n, s, + NULL); + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + failed |= !WIFEXITED(status) || WEXITSTATUS(status); + printf("A32 NUMA expected-page=%lu status=%d\n", + no4 ? native : 4096UL, status); + } + printf("NUMA GRANULE %s\n", failed ? "FAIL" : "PASS"); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/page-contract.c b/tools/testing/selftests/arm64/user4k/page-contract.c new file mode 100644 index 0000000000000..316ffe48d0a47 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/page-contract.c @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Run unchanged under the current process ABI, including native 4K/16K/64K. + * Explicit --expect-{4k,16k,64k} options assert the selected size; they never + * launch a different ABI or emulate it. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#define X18_SENTINEL UINT64_C(0xfedcba9876543210) + +extern long x18_syscall(long nr, long arg0, long arg1, long arg2, + uint64_t *seen); + +static volatile sig_atomic_t signal_seen; +static uint64_t signal_x18; + +static void x18_signal(int sig, siginfo_t *info, void *context) +{ + ucontext_t *uc = context; + + (void)sig; + (void)info; + signal_x18 = uc->uc_mcontext.regs[18]; + signal_seen = 1; + /* rt_sigreturn must restore the interrupted value, not this value. */ + /* Built with -ffixed-x18: the compiler never allocates this register. */ + asm volatile("mov x18, xzr"); +} + +static bool wait_for(pid_t child, int expected_signal) +{ + int status; + pid_t ret; + + if (child < 0) + return false; + do { + ret = waitpid(child, &status, 0); + } while (ret < 0 && errno == EINTR); + if (ret != child) + return false; + if (expected_signal) + return WIFSIGNALED(status) && + WTERMSIG(status) == expected_signal; + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +static bool access_faults(volatile unsigned char *ptr, bool write) +{ + pid_t child = fork(); + + if (!child) { + if (write) + *ptr = 0x5a; + else + (void)*ptr; + _exit(1); + } + return wait_for(child, SIGSEGV); +} + +static void private_mappings(size_t page) +{ + unsigned char *p = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + bool ok; + pid_t child; + + if (p == MAP_FAILED) + ksft_exit_fail_msg("anonymous mmap: %s\n", strerror(errno)); + memset(p, 0x41, 4 * page); + if (page > 4096) { + errno = 0; + ok = mprotect(p + 4096, 4096, PROT_READ) == -1 && + errno == EINVAL; + } else { + ok = mprotect(p + page, page, PROT_READ) == 0; + } + ksft_test_result(ok, + "mapping alignment agrees with the selected ABI\n"); + ok = mprotect(p + page, page, PROT_READ) == 0 && + mprotect(p + 2 * page, page, PROT_NONE) == 0; + ksft_test_result( + ok && access_faults(p + page, true) && + access_faults(p + 2 * page, false), + "read-only and inaccessible neighbors fault independently\n"); + p[0] = 0x10; + p[3 * page] = 0x30; + ksft_test_result( + p[page] == 0x41 && p[0] == 0x10 && p[3 * page] == 0x30, + "neighbor permissions do not remove writable mappings\n"); + child = fork(); + if (!child) { + p[0] = 0x99; + _exit(p[3 * page] != 0x30); + } + ksft_test_result(wait_for(child, 0) && p[0] == 0x10, + "fork preserves private COW isolation\n"); + ok = munmap(p + page, page) == 0; + ksft_test_result(ok && access_faults(p + page, false) && p[0] == 0x10 && + p[3 * page] == 0x30, + "single-page unmap preserves surrounding mappings\n"); + munmap(p, 4 * page); +} + +static void shared_mappings(size_t page) +{ + int fd = memfd_create("page-contract", MFD_CLOEXEC); + unsigned char *a, *b; + bool ok = true; + pid_t child; + unsigned int i; + + if (fd < 0 || ftruncate(fd, 4 * page)) + ksft_exit_fail_msg("memfd: %s\n", strerror(errno)); + a = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + b = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (a == MAP_FAILED || b == MAP_FAILED) + ksft_exit_fail_msg("shared mmap: %s\n", strerror(errno)); + for (i = 0; i < 4; i++) { + a[i * page] = 0x50 + i; + ok &= b[i * page] == 0x50 + i; + } + ksft_test_result(ok, "shared aliases expose the same bytes\n"); + ok = mprotect(a + page, page, PROT_READ) == 0; + child = fork(); + if (!child) { + b[page] = 0x73; + _exit(a[page] != 0x73); + } + ok &= wait_for(child, 0); + ksft_test_result( + ok && a[page] == 0x73 && access_faults(a + page, true), + "shared writes cross fork while alias permissions stay local\n"); + munmap(a, 4 * page); + munmap(b, 4 * page); + close(fd); +} + +static void x18_contract(void) +{ + struct sigaction sa = { .sa_sigaction = x18_signal, + .sa_flags = SA_SIGINFO }; + struct timespec pause = { .tv_nsec = 1000000 }; + struct rusage before, after; + uint64_t seen; + long ret; + bool ok; + + ret = x18_syscall(SYS_getpid, 0, 0, 0, &seen); + ksft_test_result(ret == getpid() && seen == X18_SENTINEL, + "x18 survives a raw syscall\n"); + ok = getrusage(RUSAGE_SELF, &before) == 0; + ret = x18_syscall(SYS_nanosleep, (long)&pause, 0, 0, &seen); + ok &= getrusage(RUSAGE_SELF, &after) == 0; + ksft_test_result(ok && ret == 0 && seen == X18_SENTINEL && + after.ru_nvcsw > before.ru_nvcsw, + "x18 survives a measured voluntary context switch\n"); + sigemptyset(&sa.sa_mask); + ok = sigaction(SIGUSR1, &sa, NULL) == 0; + if (ok) + ret = x18_syscall(SYS_tgkill, getpid(), syscall(SYS_gettid), + SIGUSR1, &seen); + ksft_test_result( + ok && ret == 0 && signal_seen && signal_x18 == X18_SENTINEL && + seen == X18_SENTINEL, + "signal frame and sigreturn preserve x18 after handler clobber\n"); +} + +int main(int argc, char **argv) +{ + const struct rlimit no_core = { 0, 0 }; + size_t expected = getauxval(AT_PAGESZ); + int result; + + if (argc == 2 && !strcmp(argv[1], "--expect-4k")) { + expected = 4096; + } else if (argc == 2 && !strcmp(argv[1], "--expect-16k")) { + expected = 16384; + } else if (argc == 2 && !strcmp(argv[1], "--expect-64k")) { + expected = 65536; + } else if (argc != 1) { + fprintf(stderr, + "usage: %s [--expect-4k|--expect-16k|--expect-64k]\n", + argv[0]); + return KSFT_FAIL; + } + ksft_print_header(); + ksft_set_plan(11); + if (expected != 4096 && expected != 16384 && expected != 65536) + ksft_exit_skip("unsupported process page size: %zu\n", + expected); + if (sysconf(_SC_PAGESIZE) != (long)expected || + getauxval(AT_PAGESZ) != expected) + ksft_exit_fail_msg( + "expected %zu-byte ABI; sysconf=%ld AT_PAGESZ=%lu\n", + expected, sysconf(_SC_PAGESIZE), getauxval(AT_PAGESZ)); + ksft_test_result_pass( + "process reports the expected %zu-byte page ABI\n", expected); + if (setrlimit(RLIMIT_CORE, &no_core)) + ksft_exit_fail_msg("disable child core dumps: %s\n", + strerror(errno)); + private_mappings(expected); + shared_mappings(expected); + x18_contract(); + ksft_print_cnts(); + result = ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS; + /* This static executable can also be /init in a disposable test VM. */ + if (getpid() == 1) { + fflush(NULL); + reboot(RB_POWER_OFF); + } + return result; +} diff --git a/tools/testing/selftests/arm64/user4k/perf-ring-init.c b/tools/testing/selftests/arm64/user4k/perf-ring-init.c new file mode 100644 index 0000000000000..4252f017619ca --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/perf-ring-init.c @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable PID1: perf ring layout and sampling with per-mm page sizes. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int ok, const char *expression, int line) +{ + if (!ok) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expression, + errno); + exit(1); + } +} + +#define CHECK(x) check(!!(x), #x, __LINE__) + +static int no4k; + +static int wait_child(pid_t child) +{ + int status; + pid_t got; + + if (child < 0) + return 1; + do { + got = waitpid(child, &status, 0); + } while (got < 0 && errno == EINTR); + return got != child || !WIFEXITED(status) || WEXITSTATUS(status); +} + +static int event_open(int sampling) +{ + struct perf_event_attr attr = { + .type = PERF_TYPE_SOFTWARE, + .size = sizeof(attr), + .config = PERF_COUNT_SW_PAGE_FAULTS, + .disabled = 1, + .sample_period = sampling ? 1 : 0, + .sample_type = PERF_SAMPLE_TID | PERF_SAMPLE_TIME | + PERF_SAMPLE_ADDR | PERF_SAMPLE_PERIOD, + }; + int fd = syscall(SYS_perf_event_open, &attr, 0, -1, -1, 0); + + CHECK(fd >= 0); + return fd; +} + +static sigjmp_buf fault_return; +static volatile sig_atomic_t fault_signal; + +static void fault_handler(int signal) +{ + fault_signal = signal; + siglongjmp(fault_return, 1); +} + +static void readonly_data(void *address) +{ + struct sigaction action = { .sa_handler = fault_handler }, previous; + + CHECK(!sigemptyset(&action.sa_mask)); + CHECK(!sigaction(SIGBUS, &action, &previous)); + fault_signal = 0; + if (!sigsetjmp(fault_return, 1)) + *(volatile unsigned char *)address = 0; + CHECK(fault_signal == SIGBUS); + CHECK(!sigaction(SIGBUS, &previous, NULL)); +} + +static void copy_record(void *destination, const unsigned char *ring, + unsigned long capacity, uint64_t position, size_t size) +{ + unsigned long offset = position & (capacity - 1); + size_t first = capacity - offset; + + if (first > size) + first = size; + memcpy(destination, ring + offset, first); + memcpy((unsigned char *)destination + first, ring, size - first); +} + +static void sampling(unsigned long pages) +{ + unsigned long page = getauxval(AT_PAGESZ), capacity = pages * page; + unsigned long length = page + capacity; + int fd = event_open(1); + struct perf_event_mmap_page *map = + mmap(NULL, length, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + unsigned char *region, *ring; + uint64_t tail = 0, samples = 0, wrapped = 0, region_samples = 0; + + CHECK(map != MAP_FAILED); + CHECK(map->data_offset == page && map->data_size == capacity); + map->data_tail = 0; + ring = (unsigned char *)map + map->data_offset; + for (unsigned long i = 0; i < capacity; i += page) + CHECK(ring[i] == 0); + readonly_data(ring); + region = mmap(NULL, 16 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(region != MAP_FAILED); + for (unsigned int batch = 0; batch < 10000 && tail < 3 * capacity; + batch++) { + CHECK(!madvise(region, 16 * page, MADV_DONTNEED)); + CHECK(!ioctl(fd, PERF_EVENT_IOC_ENABLE, 0)); + for (unsigned int i = 0; i < 16; i++) + *(volatile unsigned char *)(region + i * page) = 1; + CHECK(!ioctl(fd, PERF_EVENT_IOC_DISABLE, 0)); + uint64_t head = + __atomic_load_n(&map->data_head, __ATOMIC_ACQUIRE); + + CHECK(head >= tail && head - tail <= capacity); + while (tail < head) { + struct perf_event_header header; + struct { + struct perf_event_header header; + uint32_t pid, tid; + uint64_t time, address, period; + } record; + + copy_record(&header, ring, capacity, tail, + sizeof(header)); + CHECK(header.size >= sizeof(header) && + header.size <= head - tail); + if (header.type == PERF_RECORD_SAMPLE) { + CHECK(header.size == sizeof(record)); + copy_record(&record, ring, capacity, tail, + sizeof(record)); + CHECK(record.pid == (uint32_t)getpid()); + CHECK(record.tid == (uint32_t)getpid()); + if (record.address >= (uintptr_t)region && + record.address < + (uintptr_t)region + 16 * page) + region_samples++; + CHECK(record.period == 1); + samples++; + if ((tail & (capacity - 1)) + header.size > + capacity) + wrapped++; + } else { + CHECK(header.type == PERF_RECORD_THROTTLE || + header.type == PERF_RECORD_UNTHROTTLE); + } + tail += header.size; + } + __atomic_store_n(&map->data_tail, tail, __ATOMIC_RELEASE); + } + CHECK(tail >= 3 * capacity && samples && wrapped); + /* Library/stack faults may also occur while a batch is enabled. */ + CHECK(region_samples > samples * 9 / 10); + CHECK(!munmap(region, 16 * page)); + CHECK(!close(fd)); + CHECK(map->data_head == tail); + CHECK(!munmap(map, length)); + printf("ok - perf page=%lu ring=%lu samples=%llu region=%llu wrapped-records=%llu\n", + page, capacity, (unsigned long long)samples, + (unsigned long long)region_samples, (unsigned long long)wrapped); +} + +static void layout(unsigned long native) +{ + unsigned long page = getauxval(AT_PAGESZ); + int fd = event_open(0); + struct perf_event_mmap_page *map = + mmap(NULL, page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + + CHECK(map != MAP_FAILED); + CHECK(map->data_offset == page && map->data_size == 0); + map->data_tail = 0; + CHECK(!munmap(map, page)); + CHECK(!close(fd)); + fd = event_open(0); + errno = 0; + CHECK(mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0) == + MAP_FAILED); + CHECK(errno == EINVAL); + map = mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + CHECK(map != MAP_FAILED); + struct perf_event_mmap_page *alias = + mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + CHECK(alias != MAP_FAILED); + map->data_tail = 7; + CHECK(alias->data_tail == 7); + map->data_tail = 0; + CHECK(!munmap(alias, 3 * page)); + /* A software event has no AUX provider. Check byte-exact offset routing. */ + map->aux_offset = 3 * page; + map->aux_size = native; + errno = 0; + CHECK(mmap(NULL, native, PROT_READ, MAP_SHARED, fd, map->aux_offset) == + MAP_FAILED); + CHECK(errno == EOPNOTSUPP); + unsigned long sizes[] = { 4096, 16384, 65536 }; + + for (unsigned int i = 0; i < 3; i++) { + if (sizes[i] > native || sizes[i] == page || + (no4k && sizes[i] == 4096)) + continue; + pid_t child = fork(); + + if (!child) { + char f[32], s[32]; + + snprintf(f, sizeof(f), "%d", fd); + snprintf(s, sizeof(s), "%lu", sizes[i]); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], 0UL, + 0UL, 0UL)) + _exit(2); + execl("/init", "/init", "--alias", f, s, NULL); + _exit(3); + } + CHECK(!wait_child(child)); + } + CHECK(map->data_size == 2 * page && map->data_offset == page); + CHECK(!munmap(map, 3 * page)); + CHECK(!close(fd)); + printf("ok - perf page=%lu metadata, invalid size, aliases, AUX routing\n", + page); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0, compat_only = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + no4k = getenv("PERF_TEST_NO4K") != NULL; + if (argc == 4 && !strcmp(argv[1], "--alias")) { + unsigned long page = strtoul(argv[3], NULL, 10); + + CHECK(getauxval(AT_PAGESZ) == page); + errno = 0; + CHECK(mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_SHARED, + atoi(argv[2]), 0) == MAP_FAILED); + CHECK(errno == EINVAL); + return 0; + } + if (argc == 4 && !strcmp(argv[1], "--case")) { + native = strtoul(argv[2], NULL, 10); + CHECK(getauxval(AT_PAGESZ) == strtoul(argv[3], NULL, 10)); + layout(native); + sampling(1); + sampling(2); + sampling(2 * native / getauxval(AT_PAGESZ)); + return 0; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--no4k")) { + no4k = 1; + CHECK(!setenv("PERF_TEST_NO4K", "1", 1)); + } + if (!strcmp(argv[i], "--compat-only")) + compat_only = 1; + } + unsigned long sizes[] = { 4096, 16384, 65536 }; + + for (unsigned int i = 0; i < 3; i++) { + if (compat_only || sizes[i] > native || + (no4k && sizes[i] == 4096)) + continue; + pid_t child = fork(); + + if (!child) { + char n[32], s[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", sizes[i]); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], 0UL, + 0UL, 0UL)) + _exit(2); + execl("/init", "/init", "--case", n, s, NULL); + _exit(3); + } + int bad = wait_child(child); + + printf("%s - perf native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, sizes[i]); + failed |= bad; + } + if (compat_only) + CHECK(access("/compat-perf", X_OK) == 0); + if (access("/compat-perf", X_OK) == 0) { + pid_t child = fork(); + + if (!child) { + char n[32], s[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", no4k ? native : 4096UL); + execl("/compat-perf", "/compat-perf", n, s, NULL); + _exit(3); + } + int bad = wait_child(child); + + printf("%s - perf AArch32 native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, no4k ? native : 4096UL); + failed |= bad; + } + printf("PERF RING GRANULE %s\n", failed ? "FAIL" : "PASS"); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/proc-args.c b/tools/testing/selftests/arm64/user4k/proc-args.c new file mode 100644 index 0000000000000..300742a258863 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/proc-args.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Compare argv/envp with /proc/PID/cmdline and /proc/PID/environ. + * + * Run it once natively and once under the 4K user granule (for example + * "userpages 4k ./proc-args"). The process reads its own files, and a forked + * child reads the parent's files through /proc/. "--hold SECONDS" + * keeps the process alive so a native process can read it from outside. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern char **environ; + +static int failures; + +static ssize_t read_file(const char *path, char *buf, size_t size) +{ + ssize_t got, total = 0; + int fd = open(path, O_RDONLY); + + if (fd < 0) + return -errno; + while ((size_t)total < size) { + got = read(fd, buf + total, size - total); + if (got < 0) { + if (errno == EINTR) + continue; + total = -errno; + break; + } + if (!got) + break; + total += got; + } + close(fd); + return total; +} + +/* Join the complete vector; reserve one extra byte in the read buffer. */ +static char *join(char **vec, size_t *size) +{ + size_t len = 0, offset = 0; + char **entry, *buf; + + for (entry = vec; *entry; entry++) { + size_t n = strlen(*entry) + 1; + + if (n > SSIZE_MAX - 1 - len) { + errno = EOVERFLOW; + return NULL; + } + len += n; + } + buf = malloc(len ? len : 1); + if (!buf) + return NULL; + for (entry = vec; *entry; entry++) { + size_t n = strlen(*entry) + 1; + + memcpy(buf + offset, *entry, n); + offset += n; + } + *size = len; + return buf; +} + +static void check(const char *who, const char *name, pid_t pid, char **vec) +{ + char path[64], *want, *got; + size_t want_len; + ssize_t got_len; + + want = join(vec, &want_len); + got = want ? malloc(want_len + 1) : NULL; + if (!got) { + printf("not ok %s %s allocation: %s\n", who, name, + strerror(errno)); + free(want); + failures++; + return; + } + snprintf(path, sizeof(path), "/proc/%d/%s", pid, name); + /* A trailing byte rejects excess data; exact contents must reach EOF. */ + got_len = read_file(path, got, want_len + 1); + if (got_len == (ssize_t)want_len && !memcmp(want, got, want_len)) { + printf("ok %s %s %zd bytes\n", who, path, got_len); + } else { + printf("not ok %s %s want %zu got %zd", who, path, want_len, + got_len); + if (got_len < 0) + printf(" error %s", strerror(-got_len)); + else if (got_len > 0) + printf(" first [%.*s]", + (int)(got_len < 64 ? got_len : 64), got); + printf("\n"); + failures++; + } + free(got); + free(want); +} + +static void show_layout(pid_t pid, char **argv) +{ + char buf[4096], *p, *field; + ssize_t len; + int i; + + printf("pid %d AT_PAGESZ %lu argv[0] %p environ[0] %p\n", pid, + getauxval(AT_PAGESZ), (void *)argv[0], + environ[0] ? (void *)environ[0] : NULL); + len = read_file("/proc/self/stat", buf, sizeof(buf) - 1); + if (len <= 0) + return; + buf[len] = 0; + /* Fields after the parenthesised comm start at field 3. */ + p = strrchr(buf, ')'); + if (!p || p[1] != ' ') + return; + printf("stat arg_start arg_end env_start env_end:"); + for (i = 3, field = strtok(p + 2, " "); field; + field = strtok(NULL, " "), i++) + if (i >= 48 && i <= 51) + printf(" %#lx", strtoul(field, NULL, 10)); + printf("\n"); +} + +int main(int argc, char **argv) +{ + pid_t self = getpid(), child; + int status; + + setbuf(stdout, NULL); + show_layout(self, argv); + check("self", "cmdline", self, argv); + check("self", "environ", self, environ); + + child = fork(); + if (child < 0) + return 1; + if (!child) { + failures = 0; + check("child", "cmdline", self, argv); + check("child", "environ", self, environ); + _exit(failures ? 1 : 0); + } + if (waitpid(child, &status, 0) != child || !WIFEXITED(status) || + WEXITSTATUS(status)) + failures++; + + if (argc > 2 && !strcmp(argv[1], "--hold")) { + fflush(stdout); + sleep(atoi(argv[2])); + } + printf("%s: %d failure(s)\n", failures ? "FAIL" : "PASS", failures); + return failures ? 1 : 0; +} diff --git a/tools/testing/selftests/arm64/user4k/proc-memory.c b/tools/testing/selftests/arm64/user4k/proc-memory.c new file mode 100644 index 0000000000000..81ca2d0553dd0 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/proc-memory.c @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +struct stats { + unsigned long size, rss, pss, private_clean, private_dirty, + shared_clean, shared_dirty; + unsigned long anonymous, kernel_page, mmu_page; +}; + +static bool get_stats(pid_t pid, uintptr_t address, struct stats *s) +{ + char path[64], line[512]; + unsigned long start, end, value; + unsigned int fields = 0; + char key[64]; + bool found = false; + FILE *f; + + memset(s, 0, sizeof(*s)); + snprintf(path, sizeof(path), "/proc/%d/smaps", pid); + f = fopen(path, "re"); + if (!f) + return false; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) { + if (found) + break; + found = start == address; + continue; + } + if (!found) + continue; + if (sscanf(line, "%63[^:]: %lu", key, &value) != 2) + continue; + if (!strcmp(key, "Size")) { + s->size = value; + fields |= 1U << 0; + } else if (!strcmp(key, "Rss")) { + s->rss = value; + fields |= 1U << 1; + } else if (!strcmp(key, "Pss")) { + s->pss = value; + fields |= 1U << 2; + } else if (!strcmp(key, "Private_Clean")) { + s->private_clean = value; + fields |= 1U << 3; + } else if (!strcmp(key, "Private_Dirty")) { + s->private_dirty = value; + fields |= 1U << 4; + } else if (!strcmp(key, "Shared_Clean")) { + s->shared_clean = value; + fields |= 1U << 5; + } else if (!strcmp(key, "Shared_Dirty")) { + s->shared_dirty = value; + fields |= 1U << 6; + } else if (!strcmp(key, "Anonymous")) { + s->anonymous = value; + fields |= 1U << 7; + } else if (!strcmp(key, "KernelPageSize")) { + s->kernel_page = value; + fields |= 1U << 8; + } else if (!strcmp(key, "MMUPageSize")) { + s->mmu_page = value; + fields |= 1U << 9; + } + } + fclose(f); + return found && fields == (1U << 10) - 1; +} + +static bool check(pid_t pid, void *address, size_t page, size_t length, + unsigned long pss, unsigned long private, + unsigned long shared, bool anon) +{ + struct stats s; + bool ok = get_stats(pid, (uintptr_t)address, &s); + + ok &= s.size == length / 1024 && s.rss == length / 1024 && + s.pss == pss && s.private_clean + s.private_dirty == private && + s.shared_clean + s.shared_dirty == shared && + s.anonymous == (anon ? length / 1024 : 0) && + s.kernel_page == page / 1024 && s.mmu_page == page / 1024; + if (!ok) + printf("# smaps size=%lu rss=%lu pss=%lu private=%lu shared=%lu anon=%lu pages=%lu/%lu\n", + s.size, s.rss, s.pss, s.private_clean + s.private_dirty, + s.shared_clean + s.shared_dirty, s.anonymous, + s.kernel_page, s.mmu_page); + return ok; +} + +static int check_numa(void *address, size_t page) +{ + char line[1024]; + unsigned long start, anonymous = 0, pages = 0, granule = 0; + bool found = false; + FILE *f = fopen("/proc/self/numa_maps", "re"); + + if (!f) + return errno == ENOENT ? -1 : 0; + while (fgets(line, sizeof(line), f)) { + char *save, *token; + + if (sscanf(line, "%lx", &start) != 1 || + start != (uintptr_t)address) + continue; + found = true; + for (token = strtok_r(line, "\n ", &save); token; + token = strtok_r(NULL, "\n ", &save)) { + unsigned int node; + unsigned long count; + + if (sscanf(token, "anon=%lu", &count) == 1) + anonymous = count; + else if (sscanf(token, "kernelpagesize_kB=%lu", + &count) == 1) + granule = count; + else if (sscanf(token, "N%u=%lu", &node, &count) == 2) + pages += count; + } + break; + } + fclose(f); + if (!found || anonymous != 4 || pages != 4 || granule != page / 1024) + printf("# numa found=%d anon=%lu pages=%lu granule=%lu\n", + found, anonymous, pages, granule); + return found && anonymous == 4 && pages == 4 && granule == page / 1024; +} + +static bool reap(pid_t child) +{ + int status; + + return child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); +} + +static bool resident_rewrite_reuses(unsigned char *p, size_t page) +{ + uint64_t before[4], after[4]; + int fd = open("/proc/self/pagemap", O_RDONLY); + unsigned int i; + bool ok = fd >= 0; + + if (ok) + ok = pread(fd, before, sizeof(before), + (uintptr_t)p / page * 8) == sizeof(before); + for (i = 0; i < 4; i++) + p[i * page] ^= 0x21; + if (ok) + ok = pread(fd, after, sizeof(after), (uintptr_t)p / page * 8) == + sizeof(after); + for (i = 0; ok && i < 4; i++) { + const uint64_t frame = (1ULL << 55) - 1; + + ok &= (before[i] & frame) && + (before[i] & frame) == (after[i] & frame) && + (before[i] & (1ULL << 63)) && (after[i] & (1ULL << 63)); + } + if (fd >= 0) + close(fd); + return ok; +} + +static void anonymous_sharing(size_t page) +{ + unsigned char *base, *p; + int ready[2], go[2]; + pid_t child; + char byte; + bool ok; + + base = mmap(NULL, 6 * page, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, + 0); + if (base == MAP_FAILED || + mprotect(base + page, 4 * page, PROT_READ | PROT_WRITE) || + pipe(ready) || pipe(go)) + ksft_exit_fail_msg("anonymous fixture setup\n"); + p = base + page; + memset(p, 0x57, 4 * page); + ksft_test_result( + check(getpid(), p, page, 4 * page, 4 * page / 1024, + 4 * page / 1024, 0, true), + "dense anonymous leaves have exact RSS/PSS/private bytes\n"); + { + int numa = check_numa(p, page); + + if (numa < 0) + ksft_test_result_skip( + "NUMA statistics require CONFIG_NUMA\n"); + else + ksft_test_result( + numa, + "NUMA statistics count target leaves and report their granule\n"); + } + child = fork(); + if (!child) { + close(ready[0]); + close(go[1]); + if (write(ready[1], "a", 1) != 1 || read(go[0], &byte, 1) != 1) + _exit(1); + p[page] = 0x96; + if (write(ready[1], "b", 1) != 1 || read(go[0], &byte, 1) != 1) + _exit(2); + _exit(0); + } + close(ready[1]); + close(go[0]); + ok = child > 0 && read(ready[0], &byte, 1) == 1 && + check(getpid(), p, page, 4 * page, 2 * page / 1024, 0, + 4 * page / 1024, true); + ksft_test_result( + ok, + "fork divides each anonymous leaf's PSS by its own mapcount\n"); + ok = write(go[1], "x", 1) == 1 && read(ready[0], &byte, 1) == 1 && + check(getpid(), p, page, 4 * page, (5 * page) / 2048, page / 1024, + 3 * page / 1024, true) && + p[page] == 0x57; + ksft_test_result( + ok, + "one child COW changes only that leaf's private/shared accounting\n"); + write(go[1], "x", 1); + close(go[1]); + close(ready[0]); + ksft_test_result(reap(child) && check(getpid(), p, page, 4 * page, + 4 * page / 1024, 4 * page / 1024, + 0, true), + "child exit restores full anonymous PSS\n"); + ksft_test_result( + resident_rewrite_reuses(p, page), + "resident writes reuse physical backing after the child exits\n"); + munmap(base, 6 * page); +} + +static int worker(int fd, int output, int input, size_t expected) +{ + size_t page = getauxval(AT_PAGESZ); + unsigned char *p; + uintptr_t addr; + char byte; + + if (page != expected) + return 1; + p = mmap(NULL, page, PROT_READ, MAP_SHARED, fd, + page == 4096 ? 4096 : 0); + if (p == MAP_FAILED || p[0] != 0x78) + return 2; + addr = (uintptr_t)p; + if (write(output, &addr, sizeof(addr)) != sizeof(addr) || + read(input, &byte, 1) != 1) + return 3; + return 0; +} + +static void mixed_file(size_t page) +{ + size_t target = page == 4096 ? 16384 : 4096; + int ready[2], go[2], fd = memfd_create("mixed-pss", 0); + unsigned char data[16384], *p; + uintptr_t remote; + pid_t child; + bool ok; + + memset(data, 0x78, sizeof(data)); + if (fd < 0 || write(fd, data, sizeof(data)) != sizeof(data) || + pipe(ready) || pipe(go)) + ksft_exit_fail_msg("mixed file fixture setup\n"); + p = mmap(NULL, page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + page == 4096 ? 4096 : 0); + if (p == MAP_FAILED || p[0] != 0x78) + ksft_exit_fail_msg("mixed file mmap\n"); + child = fork(); + if (!child) { + char farg[16], out[16], in[16], expected[16]; + + close(ready[0]); + close(go[1]); + snprintf(farg, sizeof(farg), "%d", fd); + snprintf(out, sizeof(out), "%d", ready[1]); + snprintf(in, sizeof(in), "%d", go[0]); + snprintf(expected, sizeof(expected), "%zu", target); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, target, 0UL, 0UL, 0UL)) + _exit(4); + execl("/proc/self/exe", "/proc/self/exe", "--worker", farg, out, + in, expected, NULL); + _exit(5); + } + close(ready[1]); + close(go[0]); + ok = child > 0 && + read(ready[0], &remote, sizeof(remote)) == sizeof(remote); + ok = ok && + check(getpid(), p, page, page, page == 4096 ? 2 : 14, + page == 4096 ? 0 : 12, 4, false) && + check(child, (void *)remote, target, target, + target == 4096 ? 2 : 14, target == 4096 ? 0 : 12, 4, false); + ksft_test_result( + ok, + "mixed file aliases divide only their overlapping physical quarter\n"); + write(go[1], "x", 1); + close(go[1]); + close(ready[0]); + ksft_test_result(reap(child) && + check(getpid(), p, page, page, page / 1024, + page / 1024, 0, false), + "mixed alias exit restores private file accounting\n"); + munmap(p, page); + close(fd); +} + +int main(int argc, char **argv) +{ + size_t page = getauxval(AT_PAGESZ); + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), atoi(argv[4]), + strtoul(argv[5], NULL, 10)); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + return 2; + ksft_print_header(); + ksft_set_plan(8); + anonymous_sharing(page); + mixed_file(page); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/proc-pages.c b/tools/testing/selftests/arm64/user4k/proc-pages.c new file mode 100644 index 0000000000000..d3aaa11101cf2 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/proc-pages.c @@ -0,0 +1,318 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +#define PRESENT (1ULL << 63) +#define EXCLUSIVE (1ULL << 56) +#define GUARD (1ULL << 58) +#define FRAME ((1ULL << 55) - 1) +#define NPAGE 5 + +struct ready { + uintptr_t address; + size_t page; + uint64_t entries[NPAGE]; + uint64_t shared_entry; +}; + +static int open_proc(pid_t pid, const char *name, int flags) +{ + char path[80]; + + snprintf(path, sizeof(path), "/proc/%d/%s", pid, name); + return open(path, flags); +} + +static bool entries(int fd, uintptr_t address, size_t page, uint64_t *values, + int nr) +{ + return pread(fd, values, nr * sizeof(*values), + address / page * sizeof(*values)) == + nr * (ssize_t)sizeof(*values); +} + +static unsigned char *mapping(size_t page) +{ + const size_t boundary = 32 * 1024 * 1024; + size_t reserve = boundary + 8 * page, head, tail; + unsigned char *base, *p; + + base = mmap(NULL, reserve, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, + 0); + if (base == MAP_FAILED) + return NULL; + /* Cross both native 32 MiB and alternative 2 MiB PTE-table boundaries. */ + p = (void *)(((uintptr_t)base + 2 * page + boundary - 1) & + ~(boundary - 1)); + p -= 2 * page; + head = p - base; + tail = reserve - head - NPAGE * page; + if ((head && munmap(base, head)) || + (tail && munmap(p + NPAGE * page, tail)) || + mprotect(p, NPAGE * page, PROT_READ | PROT_WRITE)) + return NULL; + memset(p, 0x67, NPAGE * page); + if (munmap(p + 2 * page, page) || + madvise(p + 3 * page, page, MADV_GUARD_INSTALL) || + mprotect(p + 4 * page, page, PROT_NONE)) + return NULL; + return p; +} + +static bool scan(int fd, uintptr_t addr, size_t page) +{ + struct page_region vec[8]; + struct pm_scan_arg arg = { + .size = sizeof(arg), + .start = addr, + .end = addr + NPAGE * page, + .vec = (uintptr_t)vec, + .vec_len = 8, + .return_mask = PAGE_IS_PRESENT | PAGE_IS_GUARD, + }; + int nr = ioctl(fd, PAGEMAP_SCAN, &arg); + bool ok = nr == 3 && arg.walk_end == arg.end; + if (!ok) + printf("# scan initial nr=%d errno=%d end_delta=%lld\n", nr, + errno, (long long)(arg.walk_end - arg.end)); + + /* The unmapped middle page produces no VMA result. */ + if (nr == 3) + ok &= vec[0].start == addr && vec[0].end == addr + 2 * page && + vec[0].categories == PAGE_IS_PRESENT && + vec[1].start == addr + 3 * page && + vec[1].end == addr + 4 * page && + vec[1].categories == PAGE_IS_GUARD && + vec[2].start == addr + 4 * page && + vec[2].end == arg.end && + vec[2].categories == PAGE_IS_PRESENT; + arg.max_pages = 1; + arg.walk_end = 0; + nr = ioctl(fd, PAGEMAP_SCAN, &arg); + ok &= nr == 1 && vec[0].start == addr && vec[0].end == addr + page && + arg.walk_end == addr + page; + arg.start = addr + 1; + errno = 0; + ok &= ioctl(fd, PAGEMAP_SCAN, &arg) == -1 && errno == EINVAL; + return ok; +} + +static int remote_worker(int output, int input, size_t expected, int shared_fd) +{ + struct ready ready = { .page = getauxval(AT_PAGESZ) }; + unsigned char *p; + int fd = open_proc(getpid(), "pagemap", O_RDONLY); + char byte; + unsigned char *shared = + mmap(NULL, 16384, PROT_READ, MAP_SHARED, shared_fd, 0); + size_t offset = ready.page == 4096 ? 4096 : 0; + + if (shared == MAP_FAILED || shared[offset] != 0x83 || + !entries(fd, (uintptr_t)shared + offset, ready.page, + &ready.shared_entry, 1)) + return 2; + /* Allocate aliases first so mmap cannot reuse our deliberate hole. */ + p = mapping(ready.page); + ready.address = (uintptr_t)p; + if (ready.page != expected || !p || fd < 0 || + !entries(fd, ready.address, ready.page, ready.entries, NPAGE) || + write(output, &ready, sizeof(ready)) != sizeof(ready) || + read(input, &byte, 1) != 1) + return 1; + return 0; +} + +static bool remote_read(size_t own_page) +{ + int ready_pipe[2], done_pipe[2], status, fd, shared_fd, self_fd; + unsigned char *shared; + uint64_t self_entry; + size_t offset = own_page == 4096 ? 4096 : 0; + size_t target = own_page == 4096 ? 16384 : 4096; + struct ready ready; + uint64_t actual[NPAGE]; + pid_t child; + bool ok; + + shared_fd = memfd_create("mixed-pagemap", 0); + if (shared_fd < 0 || ftruncate(shared_fd, 16384)) + return false; + shared = mmap(NULL, 16384, PROT_READ | PROT_WRITE, MAP_SHARED, + shared_fd, 0); + self_fd = open_proc(getpid(), "pagemap", O_RDONLY); + if (shared == MAP_FAILED || self_fd < 0) + return false; + memset(shared, 0x83, 16384); + if (!entries(self_fd, (uintptr_t)shared + offset, own_page, &self_entry, + 1)) + return false; + close(self_fd); + if (pipe(ready_pipe) || pipe(done_pipe)) + return false; + child = fork(); + if (!child) { + char output[16], input[16], expected[16], shared_arg[16]; + + close(ready_pipe[0]); + close(done_pipe[1]); + snprintf(output, sizeof(output), "%d", ready_pipe[1]); + snprintf(input, sizeof(input), "%d", done_pipe[0]); + snprintf(expected, sizeof(expected), "%zu", target); + snprintf(shared_arg, sizeof(shared_arg), "%d", shared_fd); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, target, 0UL, 0UL, 0UL)) + _exit(2); + execl("/proc/self/exe", "/proc/self/exe", "--remote", output, + input, expected, shared_arg, NULL); + _exit(3); + } + close(ready_pipe[1]); + close(done_pipe[0]); + ok = child > 0 && + read(ready_pipe[0], &ready, sizeof(ready)) == sizeof(ready); + fd = open_proc(child, "pagemap", O_RDONLY); + ok = ok && ready.page == target && fd >= 0 && + entries(fd, ready.address, target, actual, NPAGE) && + !memcmp(actual, ready.entries, sizeof(actual)) && + scan(fd, ready.address, target); + if (!ok) + printf("# remote geometry/read failure: target=%zu reported=%zu fd=%d errno=%d\n", + target, ready.page, fd, errno); + if (ok) { + size_t target_offset = target == 4096 ? 4096 : 0; + + /* Compare physical byte addresses, not differently sized PFNs. */ + ok &= (self_entry & FRAME) && (ready.shared_entry & FRAME) && + (self_entry & FRAME) * own_page - offset == + (ready.shared_entry & FRAME) * target - + target_offset; + if (!ok) + printf("# shared PFN mismatch: own_nonzero=%d target_nonzero=%d byte_delta=%lld\n", + !!(self_entry & FRAME), + !!(ready.shared_entry & FRAME), + (long long)((self_entry & FRAME) * own_page - + offset - + (ready.shared_entry & FRAME) * + target + + target_offset)); + } + if (fd >= 0) + close(fd); + write(done_pipe[1], "x", 1); + close(done_pipe[1]); + close(ready_pipe[0]); + ok &= child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + munmap(shared, 16384); + close(shared_fd); + return ok; +} + +int main(int argc, char **argv) +{ + size_t page = getauxval(AT_PAGESZ); + unsigned char *p; + uint64_t values[NPAGE], again[NPAGE]; + int fd, clear, i; + bool ok; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--remote")) + return remote_worker(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10), atoi(argv[5])); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + return 2; + ksft_print_header(); + ksft_set_plan(6); + p = mapping(page); + fd = open_proc(getpid(), "pagemap", O_RDONLY); + if (!p || fd < 0) + ksft_exit_fail_msg("proc mapping setup: %s\n", strerror(errno)); + ok = entries(fd, (uintptr_t)p, page, values, NPAGE); + ok &= (values[0] & PRESENT) && (values[1] & PRESENT) && !values[2] && + (values[3] & GUARD) && !(values[3] & PRESENT) && + (values[4] & PRESENT); + ok &= (values[0] & EXCLUSIVE) && (values[1] & EXCLUSIVE) && + (values[4] & EXCLUSIVE); + ksft_test_result( + ok, + "pagemap reports each target leaf, hole, guard and PROT_NONE entry\n"); + ok = true; + for (i = 0; i < NPAGE; i++) { + uint64_t value; + + ok &= entries(fd, (uintptr_t)p + i * page, page, &value, 1) && + value == values[i]; + } + errno = 0; + ok &= pread(fd, again, 7, 0) == -1 && errno == EINVAL; + ksft_test_result( + ok, + "pagemap per-entry offsets agree with batch reads and reject short entries\n"); + ksft_test_result( + scan(fd, (uintptr_t)p, page), + "PAGEMAP_SCAN ranges, max_pages and alignment use target granules\n"); + clear = open_proc(getpid(), "clear_refs", O_WRONLY); + ok = clear >= 0; + for (i = 1; ok && i <= 3; i++) { + char operation = '0' + i; + + ok &= write(clear, &operation, 1) == 1; + } + if (clear >= 0) + close(clear); + ok &= entries(fd, (uintptr_t)p, page, again, NPAGE) && + !memcmp(values, again, sizeof(values)) && p[0] == 0x67 && + p[page] == 0x67; + ksft_test_result( + ok, + "clear_refs traverses short and split VMAs without changing mappings or bytes\n"); + ksft_test_result( + remote_read(page), + "opposite-ABI reader uses target pagemap and scan geometry\n"); + { + pid_t child = fork(); + int status; + + if (!child) { + uint64_t value = 0; + int unprivileged; + + if (setuid(1000) || prctl(PR_SET_DUMPABLE, 1)) + _exit(2); + unprivileged = open_proc(getpid(), "pagemap", O_RDONLY); + ok = unprivileged >= 0 && + entries(unprivileged, (uintptr_t)p, page, &value, + 1) && + (value & PRESENT) && !(value & FRAME); + if (!ok) + printf("# unprivileged fd=%d errno=%d flags=%#llx\n", + unprivileged, errno, + (unsigned long long)(value & ~FRAME)); + _exit(!ok); + } + ok = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + ksft_test_result( + ok, + "unprivileged pagemap reads retain flags and hide physical addresses\n"); + } + close(fd); + munmap(p, NPAGE * page); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/reclaim-advice.c b/tools/testing/selftests/arm64/user4k/reclaim-advice.c new file mode 100644 index 0000000000000..92f28d49b830c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/reclaim-advice.c @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned long swapouts(void) +{ + FILE *f = fopen("/proc/vmstat", "re"); + char key[128]; + unsigned long value, result = 0; + + if (!f) + return 0; + while (fscanf(f, "%127s %lu", key, &value) == 2) + if (!strcmp(key, "pswpout")) + result = value; + fclose(f); + return result; +} + +static unsigned long swapped_kb(void) +{ + FILE *f = fopen("/proc/self/status", "re"); + char line[256]; + unsigned long result = 0; + + if (!f) + return 0; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmSwap: %lu kB", &result) == 1) + break; + fclose(f); + return result; +} + +static int advise(void *addr, size_t length, int operation, const char *name) +{ + int ret = madvise(addr, length, operation); + + if (ret) + printf("# %s failed: errno=%d (%s)\n", name, errno, + strerror(errno)); + return ret; +} + +static unsigned char pattern(size_t i, unsigned int round) +{ + return (i ^ (i >> 12) ^ (i >> 20) ^ (round * 0x55)) & 255; +} + +static size_t swapped_pages(unsigned char *p, size_t length, size_t page) +{ + uint64_t *values = calloc(length / page, sizeof(*values)); + int fd = open("/proc/self/pagemap", O_RDONLY); + size_t i, count = length / page, swapped = 0; + + if (!values || fd < 0 || + pread(fd, values, count * sizeof(*values), + (uintptr_t)p / page * sizeof(*values)) != + (ssize_t)(count * sizeof(*values))) { + swapped = SIZE_MAX; + } else { + for (i = 0; i < count; i++) + swapped += !!(values[i] & (1ULL << 62)) && + !(values[i] & (1ULL << 63)); + } + if (fd >= 0) + close(fd); + free(values); + return swapped; +} + +static void diagnose_resident(unsigned char *p, size_t length, size_t page) +{ + int pm = open("/proc/self/pagemap", O_RDONLY); + int flags = open("/proc/kpageflags", O_RDONLY); + int count = open("/proc/kpagecount", O_RDONLY); + + for (size_t i = 0; i < length / page; i++) { + uint64_t entry = 0, state = 0, maps = 0; + unsigned long native_pfn; + + if (pread(pm, &entry, 8, ((uintptr_t)p / page + i) * 8) != 8 || + !(entry & (1ULL << 63))) + continue; + native_pfn = (entry & ((1ULL << 55) - 1)) * page / 16384; + if (pread(flags, &state, 8, native_pfn * 8) != 8 || + pread(count, &maps, 8, native_pfn * 8) != 8) + printf("# resident metadata read failed: errno=%d\n", + errno); + printf("# resident leaf=%zu native_pfn=%#lx flags=%#llx maps=%llu\n", + i, native_pfn, (unsigned long long)state, + (unsigned long long)maps); + } + close(pm); + close(flags); + close(count); +} + +/* PAGEOUT drains only the calling CPU's pending LRU additions. */ +static bool drain_pageout_cpus(unsigned char *p, size_t length) +{ + cpu_set_t allowed, one; + bool ok = true; + + if (sched_getaffinity(0, sizeof(allowed), &allowed)) + return false; + for (unsigned int cpu = 0; cpu < CPU_SETSIZE; cpu++) { + if (!CPU_ISSET(cpu, &allowed)) + continue; + CPU_ZERO(&one); + CPU_SET(cpu, &one); + if (sched_setaffinity(0, sizeof(one), &one)) { + ok = false; + break; + } + ok &= advise(p, length, MADV_COLD, "CPU drain COLD") == 0; + ok &= advise(p, length, MADV_PAGEOUT, "CPU drain PAGEOUT") == 0; + } + ok &= sched_setaffinity(0, sizeof(allowed), &allowed) == 0; + return ok; +} + +static bool swapped_pagemap(unsigned char *p, size_t length, size_t page) +{ + uint64_t *values = calloc(length / page, sizeof(*values)); + int fd = open("/proc/self/pagemap", O_RDONLY); + size_t i, count = length / page; + bool ok = values && fd >= 0; + + if (ok) + ok = pread(fd, values, count * sizeof(*values), + (uintptr_t)p / page * sizeof(*values)) == + (ssize_t)(count * sizeof(*values)); + for (i = 0; ok && i < count; i++) { + uint64_t frame = values[i] & ((1ULL << 55) - 1); + + ok &= !!(values[i] & (1ULL << 62)) && + !(values[i] & (1ULL << 63)); + if (page == 4096) { + ok &= ((frame >> 5) & 3) == + (((uintptr_t)p / page + i) & 3); + if (i && (((uintptr_t)p / page + i) & 3)) + ok &= (values[i] & ((1ULL << 55) - 1)) == + (values[i - 1] & ((1ULL << 55) - 1)) + + (1U << 5); + } + } + if (fd >= 0) + close(fd); + free(values); + printf("%s - %zuK pagemap reports swapped leaves and exact quarter offsets\n", + ok ? "ok" : "not ok", page / 1024); + return ok; +} + +static bool check_swap_pss(pid_t pid, void *p, size_t length, + unsigned long expected_swap, + unsigned long expected_pss) +{ + char path[64], line[512]; + unsigned long start, end, swap = 0, pss = 0; + unsigned int fields = 0; + bool found = false, ok; + FILE *f; + + snprintf(path, sizeof(path), "/proc/%d/smaps", pid); + f = fopen(path, "re"); + if (!f) + return false; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) { + if (found) + break; + found = start == (uintptr_t)p && + end == (uintptr_t)p + length; + continue; + } + if (!found) + continue; + if (sscanf(line, "Swap: %lu", &swap) == 1) + fields |= 1; + else if (sscanf(line, "SwapPss: %lu", &pss) == 1) + fields |= 2; + } + fclose(f); + ok = found && fields == 3 && swap == expected_swap && + pss == expected_pss; + if (!ok) + printf("# swap stats found=%d fields=%u Swap=%lu/%lu SwapPss=%lu/%lu\n", + found, fields, swap, expected_swap, pss, expected_pss); + return ok; +} + +static bool swapped_accounting(unsigned char *p, size_t length, size_t page, + bool forked) +{ + int ready[2], go[2], status; + pid_t child; + char byte; + bool ok = check_swap_pss(getpid(), p, length, length / 1024, + length / 1024); + + if (!forked || !ok) + goto out; + if (pipe(ready) || pipe(go)) + return false; + child = fork(); + if (child < 0) + return false; + if (!child) { + close(ready[0]); + close(go[1]); + if (read(go[0], &byte, 1) != 1) + _exit(1); + p[page] ^= 0x37; + if (write(ready[1], "x", 1) != 1 || read(go[0], &byte, 1) != 1) + _exit(2); + _exit(0); + } + close(ready[1]); + close(go[0]); + ok &= check_swap_pss(getpid(), p, length, length / 1024, length / 2048); + ok &= check_swap_pss(child, p, length, length / 1024, length / 2048); + ok &= write(go[1], "x", 1) == 1 && read(ready[0], &byte, 1) == 1; + ok &= check_swap_pss(getpid(), p, length, length / 1024, + (length + page) / 2048); + ok &= check_swap_pss(child, p, length, (length - page) / 1024, + (length - page) / 2048); + write(go[1], "x", 1); + close(go[1]); + close(ready[0]); + ok &= waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status); + ok &= check_swap_pss(getpid(), p, length, length / 1024, length / 1024); +out: + printf("%s - %zuK SwapPss %s\n", ok ? "ok" : "not ok", page / 1024, + forked ? "fork, one-leaf swap-in/COW and child exit" : + "exclusive quarters"); + return ok; +} + +int main(int argc, char **argv) +{ + const size_t length = 4 * 1024 * 1024; + size_t page = getauxval(AT_PAGESZ), count = length / page, i, j, absent; + unsigned char *p, *vec, *reservation; + unsigned long before, after, swapped_before, swapped_after; + unsigned int round, tries; + size_t bad_bytes, bad_residency; + bool ok = true; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + return 2; + reservation = mmap(NULL, length + 2 * page, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reservation == MAP_FAILED) + return 3; + p = reservation + page; + vec = malloc(count); + if (!vec || mprotect(p, length, PROT_READ | PROT_WRITE)) + return 3; + for (round = 0; round < 4 && ok; round++) { + for (i = 0; i < length; i++) + p[i] = pattern(i, round); + before = swapouts(); + swapped_before = swapped_kb(); + ok = advise(p, length, MADV_COLD, "COLD") == 0; + ok &= advise(p, length, MADV_PAGEOUT, "PAGEOUT") == 0; + after = before; + absent = 0; + for (tries = 0; ok && tries < 200; tries++) { + if (mincore(p, length, vec)) { + ok = false; + break; + } + absent = 0; + for (i = 0; i < count; i++) + absent += !(vec[i] & 1); + after = swapouts(); + if (after > before && + swapped_pages(p, length, page) == count) + break; + if (tries == 32) { + printf("# checking pending remote LRU additions before CPU sweep\n"); + diagnose_resident(p, length, page); + ok &= drain_pageout_cpus(p, length); + printf("# after CPU sweep: swapped=%zu/%zu\n", + swapped_pages(p, length, page), count); + } + /* PAGEOUT may skip busy folios; establish the full-swap fixture. */ + usleep(10000); + ok &= advise(p, length, MADV_COLD, "retry COLD") == 0; + ok &= advise(p, length, MADV_PAGEOUT, + "retry PAGEOUT") == 0; + } + printf("# pageout setup attempts=%u\n", tries + 1); + if (tries == 200) + diagnose_resident(p, length, page); + ok &= tries < 200; + swapped_after = swapped_kb(); + /* mincore also counts warm swap-cache data as resident. */ + ok &= after > before && swapped_after > swapped_before; + ok &= swapped_pagemap(p, length, page); + ok &= swapped_accounting(p, length, page, round == 1); + if (!round) + ok &= advise(p + page, page, MADV_WILLNEED, + "WILLNEED") == 0; + else { + if (round == 3) { + ok &= mlock(p, length) == 0; + ok &= swapped_pages(p, length, page) == 0; + } else { + ok &= advise(p, length, MADV_POPULATE_READ, + "POPULATE_READ") == 0; + } + ok &= mincore(p, length, vec) == 0; + bad_residency = 0; + for (i = 0; i < count; i++) + bad_residency += !(vec[i] & 1); + printf("# after %s: absent=%zu/%zu\n", + round == 3 ? "mlock" : "POPULATE_READ", + bad_residency, count); + ok &= bad_residency == 0; + } + bad_bytes = 0; + /* Nonsequential user-page order, every byte checked after swap. */ + for (i = 0; i < count; i++) { + size_t index = ((i * 61) & (count - 1)) * page; + + for (j = index; j < index + page; j++) + if (p[j] != pattern(j, round)) { + if (!bad_bytes) + printf("# first bad byte at %#zx: got=%#x expected=%#x\n", + j, p[j], + pattern(j, round)); + bad_bytes++; + ok = false; + } + } + if (round == 3) + ok &= munlock(p, length) == 0; + printf("# bad bytes=%zu\n", bad_bytes); + printf("%s - %zuK EL0 pageout round %u: pswpout=%lu->%lu VmSwap=%lu->%lu kB absent=%zu/%zu %s\n", + ok ? "ok" : "not ok", page / 1024, round, before, after, + swapped_before, swapped_after, absent, count, + round == 3 ? "mlock/readback" : + round ? "prefault/readback" : + "willneed/readback"); + } + munmap(reservation, length + 2 * page); + free(vec); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/runtime-app-init.c b/tools/testing/selftests/arm64/user4k/runtime-app-init.c new file mode 100644 index 0000000000000..9d1c7c2c49351 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/runtime-app-init.c @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +int main(int argc, char **argv) +{ + if (argc >= 4 && !strcmp(argv[1], "--exec")) { + if (prctl(SET_EXEC_PAGE_SIZE, strtoul(argv[2], NULL, 10), 0UL, + 0UL, 0UL)) + return 8; + execv(argv[3], argv + 3); + perror("exec"); + return 9; + } + if (getpid() != 1) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + int failed = 0; + pid_t children[3]; + unsigned long sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + unsigned count = sizes[0] > 16384 ? 3 : 2; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)) { + perror("mount"); + failed = 1; + goto done; + } + mkdir("/dev/shm", 0777); + mount("tmpfs", "/dev/shm", "tmpfs", 0, NULL); + for (unsigned i = 0; i < count; i++) { + children[i] = fork(); + if (!children[i]) { + char ps[32]; + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(3); + execl("/usr/bin/python3", "/usr/bin/python3", "-I", + "/runtime-app.py", ps, NULL); + perror("exec Python"); + _exit(4); + } + } + for (unsigned i = 0; i < count; i++) { + int status = 0; + int ok = children[i] > 0 && + waitpid(children[i], &status, 0) == children[i] && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - distro Python ABI %luK status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + failed |= !ok; + } +done: + if (failed) { + FILE *f = fopen("/trace4k", "r"); + if (f) { + char line[2048]; + while (fgets(line, sizeof(line), f)) + fputs(line, stdout); + fclose(f); + } + } + printf("GRANULE DISTRO RUNTIME %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/runtime-app.py b/tools/testing/selftests/arm64/user4k/runtime-app.py new file mode 100755 index 0000000000000..47fac03300efc --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/runtime-app.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: GPL-2.0-only +import os,sys,mmap,ctypes,threading,concurrent.futures,tempfile,subprocess,signal,zlib,gzip,lzma,hashlib,socket,select,json,sqlite3 +ps=int(sys.argv[1]);assert os.sysconf('SC_PAGE_SIZE')==ps==mmap.PAGESIZE,(ps,mmap.PAGESIZE) +libc=ctypes.CDLL(None,use_errno=True);libc.prctl.argtypes=[ctypes.c_int,ctypes.c_ulong,ctypes.c_ulong,ctypes.c_ulong,ctypes.c_ulong] +def passed(name):print(f'ok - distro Python {ps//1024}K {name}',flush=True) +passed('dynamic loader and page-size contract') +with mmap.mmap(-1, 2*ps, flags=0x08|mmap.MAP_ANONYMOUS) as droppable: + assert droppable[:]==bytes(2*ps) + droppable[:]=bytes([0x45])*(2*ps) + child=os.fork() + if child==0:os._exit(0 if droppable[:]==bytes(2*ps) else 9) + assert os.waitpid(child,0)[1]==0 + droppable.madvise(mmap.MADV_DONTNEED) + assert droppable[:]==bytes(2*ps) +libc.arc4random_buf.argtypes=[ctypes.c_void_p,ctypes.c_size_t] +random_buffer=ctypes.create_string_buffer(32) +random_samples=set() +for _ in range(128): + libc.arc4random_buf(random_buffer,len(random_buffer));random_samples.add(random_buffer.raw) +assert len(random_samples)==128 +passed('droppable write, fork wipe, discard and glibc random state') + +blob=bytes(range(256))*4096 +with tempfile.TemporaryDirectory() as tmp: + path=tmp+'/mapped' + with open(path,'w+b') as f: + f.truncate(8*ps) + with mmap.mmap(f.fileno(),2*ps,offset=ps) as m: + m[:]=bytes([0x53])*(2*ps);m.flush() + assert os.pread(f.fileno(),2*ps,ps)==m[:] + m.resize(3*ps);m[2*ps:]=bytes([0x72])*ps;m.flush() + assert os.pread(f.fileno(),ps,3*ps)==bytes([0x72])*ps + passed('file mmap offset, flush and resize') + with mmap.mmap(-1,4*ps,flags=mmap.MAP_PRIVATE|mmap.MAP_ANONYMOUS) as m: + m[:]=bytes([0x39])*(4*ps) + c=os.fork() + if c==0: + m[ps:2*ps]=bytes([0x91])*ps + os._exit(0 if m[:ps]==bytes([0x39])*ps else 5) + assert os.waitpid(c,0)[1]==0 and m[:]==bytes([0x39])*(4*ps) + address=ctypes.addressof(ctypes.c_char.from_buffer(m)) + libc.mprotect.argtypes=[ctypes.c_void_p,ctypes.c_size_t,ctypes.c_int] + assert libc.mprotect(address+ps,ps,0)==0 + c=os.fork() + if c==0: + m[ps]=0 + os._exit(6) + status=os.waitpid(c,0)[1];assert os.WIFSIGNALED(status) and os.WTERMSIG(status)==signal.SIGSEGV + assert m[0]==0x39 and m[2*ps]==0x39 + assert libc.mprotect(address+ps,ps,mmap.PROT_READ|mmap.PROT_WRITE)==0 + passed('fork COW and independent protection fault') + def compressed(i): + d=blob+str(i).encode() + for compress,decompress in [(zlib.compress,zlib.decompress),(gzip.compress,gzip.decompress),(lambda d:lzma.compress(d,preset=1),lzma.decompress)]:assert decompress(compress(d))==d + return hashlib.sha256(d).hexdigest() + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + results=list(pool.map(compressed,range(8))) + assert len(set(results))==8 + passed('threaded native extension allocation, compression and hashing') + db=sqlite3.connect(tmp+'/db') + db.execute('pragma journal_mode=wal');db.execute('pragma mmap_size=1048576') + db.execute('create table entries (n integer primary key, data blob)') + db.executemany('insert into entries values (?,?)',[(i,blob[:4096]) for i in range(128)]);db.commit() + assert db.execute('select count(*) from entries').fetchone()[0]==128 + assert all(row[0]==blob[:4096] for row in db.execute('select data from entries'));db.close() + passed('SQLite WAL and mapped database reads') + left,right=socket.socketpair() + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + task=pool.submit(lambda:(right.sendall(blob),right.shutdown(socket.SHUT_WR))) + out=bytearray() + while True: + assert select.select([left],[],[],5)[0] + part=left.recv(997) + if not part:break + out.extend(part) + task.result();assert bytes(out)==blob + left.close();right.close();passed('socket polling and fragmented transfers') + fd=os.memfd_create('cross-abi',0);os.ftruncate(fd,131072);os.pwrite(fd,b'parent',ps) + # Execute the child in the other ABI, retaining the same shared file bytes. + target=16384 if ps==4096 else 4096 + def select_abi(): + if libc.prctl(0x41555001,target,0,0,0):os._exit(7) + code="import os,mmap,sys;fd=int(sys.argv[1]);off=int(sys.argv[2]);want=int(sys.argv[3]);assert os.sysconf('SC_PAGE_SIZE')==want;m=mmap.mmap(fd,131072);assert m[off:off+6]==b'parent';m[off:off+5]=b'child';m.close()" + subprocess.run([sys.executable,'-I','-c',code,str(fd),str(ps),str(target)],pass_fds=(fd,),preexec_fn=select_abi,check=True) + assert os.pread(fd,5,ps)==b'child';os.close(fd);passed('cross-ABI dynamic exec and memfd sharing') + command="set -e; a=(); for ((i=0;i<2048;i++)); do a+=(\"value-$i\"); done; [[ ${#a[@]} == 2048 ]]; [[ ${a[2047]} == value-2047 ]]; printf 'bash-ok\\n'" + def same_abi(): + if libc.prctl(0x41555001,ps,0,0,0):os._exit(8) + result=subprocess.run(['/bin/bash','--noprofile','--norc','-c',command],preexec_fn=same_abi,stdout=subprocess.PIPE,check=True,text=True) + assert result.stdout=='bash-ok\n';passed('distro Bash dynamic exec, allocation and pipes') +print(f'DISTRO PYTHON {ps} PASS',flush=True) diff --git a/tools/testing/selftests/arm64/user4k/shmem-collapse-init.c b/tools/testing/selftests/arm64/user4k/shmem-collapse-init.c new file mode 100644 index 0000000000000..b040d8fc43a4f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-collapse-init.c @@ -0,0 +1,396 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#ifndef MADV_COLLAPSE +#define MADV_COLLAPSE 25 +#endif +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - collapse line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +struct command { + uint64_t op, offset, len, value; +}; +struct ready { + uint64_t base, head; +}; +static uint64_t entry(const char *path, uint64_t index) +{ + uint64_t value; + int fd = open(path, O_RDONLY); + CHECK(fd >= 0 && pread(fd, &value, 8, index * 8) == 8 && !close(fd)); + return value; +} +static uint64_t mapped_pfn(pid_t pid, uintptr_t address, size_t ps) +{ + char path[80]; + snprintf(path, sizeof(path), "/proc/%d/pagemap", pid); + uint64_t e = entry(path, address / ps); + CHECK(e & (1ULL << 63)); + return e & ((1ULL << 55) - 1); +} +static unsigned char *map(int file, size_t len, size_t align, off_t offset) +{ + void *reserve = mmap(NULL, len + align, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + uintptr_t addr = ((uintptr_t)reserve + align - 1) & ~(align - 1); + CHECK(!munmap(reserve, len + align)); + unsigned char *p = mmap((void *)addr, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, offset); + CHECK(p != MAP_FAILED); + return p; +} +static int register_range(unsigned char *p, size_t len, int minor) +{ + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = minor ? UFFD_FEATURE_MINOR_SHMEM : + 0 }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register r = { + .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING | + (minor ? UFFDIO_REGISTER_MODE_MINOR : 0) + }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, &r)); + return fd; +} +static void copy_leaf(int fd, uintptr_t dst, unsigned char value) +{ + unsigned char src[4096]; + memset(src, value, sizeof(src)); + struct uffdio_copy c = { .dst = dst, + .src = (uintptr_t)src, + .len = sizeof(src) }; + CHECK(!ioctl(fd, UFFDIO_COPY, &c) && c.copy == sizeof(src)); +} +static void cpu(unsigned id) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(id, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} +static int worker(int file, int sock, size_t native, size_t len) +{ + CHECK(getauxval(AT_PAGESZ) == 4096); + alarm(90); + cpu(1); + unsigned char *p = map(file, len, len, 0); + unsigned char *q = map(file, len - 4096, len, 4096); + CHECK(p[0] == 0x61 && q[0] == 0x61); + uint64_t old_head = + mapped_pfn(getpid(), (uintptr_t)p, 4096) * 4096 / native; + CHECK(entry("/proc/kpageflags", old_head) & + (1ULL << KPF_COMPOUND_HEAD)); + int fd = register_range(p, len, 1); + for (size_t i = 4 * native; i < len; i += native) + copy_leaf(fd, (uintptr_t)p + i, 0x71); + int pipes[2]; + CHECK(!pipe(pipes)); + struct iovec iov = { p, 4096 }; + CHECK(vmsplice(pipes[1], &iov, 1, 0) == 4096); + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + native, native)); + CHECK(entry("/proc/kpageflags", old_head) & + (1ULL << KPF_COMPOUND_HEAD)); + CHECK(!close(pipes[0]) && !close(pipes[1])); + struct ready r = { (uintptr_t)p, old_head }; + struct iovec v = { &r, sizeof(r) }; + union { + struct cmsghdr align; + char bytes[CMSG_SPACE(sizeof(fd))]; + } control = { 0 }; + struct msghdr msg = { .msg_iov = &v, + .msg_iovlen = 1, + .msg_control = control.bytes, + .msg_controllen = sizeof(control.bytes) }; + struct cmsghdr *c = CMSG_FIRSTHDR(&msg); + c->cmsg_level = SOL_SOCKET; + c->cmsg_type = SCM_RIGHTS; + c->cmsg_len = CMSG_LEN(sizeof(fd)); + memcpy(CMSG_DATA(c), &fd, sizeof(fd)); + CHECK(sendmsg(sock, &msg, 0) == sizeof(r)); + struct command cmd; + while (recv(sock, &cmd, sizeof(cmd), 0) == sizeof(cmd)) { + CHECK(cmd.offset < len && cmd.len <= len - cmd.offset); + unsigned char value = 0; + if (!cmd.op) + break; + if (cmd.op == 1) + value = *(volatile unsigned char *)(p + cmd.offset); + else if (cmd.op == 2) + CHECK(!fallocate(file, + FALLOC_FL_PUNCH_HOLE | + FALLOC_FL_KEEP_SIZE, + cmd.offset, cmd.len)); + else if (cmd.op == 3) { + CHECK(cmd.offset >= 4096); + value = *(volatile unsigned char *)(q + cmd.offset - + 4096); + } else if (cmd.op == 4) { + CHECK(cmd.offset >= 4096); + memset(q + cmd.offset - 4096, cmd.value, cmd.len); + } else + CHECK(0); + CHECK(send(sock, &value, 1, 0) == 1); + } + CHECK(!munmap(q, len - 4096) && !munmap(p, len) && !close(fd) && + !close(file) && !close(sock)); + return 0; +} +static void send_command(int sock, uint64_t op, size_t offset, size_t len) +{ + struct command c = { .op = op, .offset = offset, .len = len }; + CHECK(send(sock, &c, sizeof(c), 0) == sizeof(c)); +} +static void reply(int sock, unsigned char expected) +{ + unsigned char value; + CHECK(recv(sock, &value, 1, 0) == 1 && value == expected); +} +static void fault(int sock, int fd, uintptr_t base, size_t offset, int minor, + unsigned char value) +{ + send_command(sock, 1, offset + 17, 0); + struct pollfd p = { .fd = fd, .events = POLLIN }; + struct uffd_msg m; + CHECK(poll(&p, 1, 10000) == 1 && (p.revents & POLLIN)); + CHECK(read(fd, &m, sizeof(m)) == sizeof(m)); + CHECK(m.event == UFFD_EVENT_PAGEFAULT && + m.arg.pagefault.address == base + offset && + m.arg.pagefault.flags == (minor ? UFFD_PAGEFAULT_FLAG_MINOR : 0)); + if (minor) { + struct uffdio_continue c = { .range = { base + offset, 4096 } }; + CHECK(!ioctl(fd, UFFDIO_CONTINUE, &c) && c.mapped == 4096); + } else + copy_leaf(fd, base + offset, value); + reply(sock, value); +} +static unsigned long pmd_mapped(unsigned char *p) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + CHECK(f); + char *line = NULL; + size_t cap = 0; + unsigned long a, b, kb, result = 0; + int selected = 0; + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &a, &b) == 2) + selected = a == (uintptr_t)p; + else if (selected && + sscanf(line, "ShmemPmdMapped: %lu kB", &kb) == 1) + result = kb * 1024; + } + free(line); + fclose(f); + return result; +} +static int collapse(unsigned char *p, size_t len) +{ + int ret; + unsigned tries = 0; + do { + errno = 0; + ret = madvise(p, len, MADV_COLLAPSE); + if (!ret || errno != EAGAIN) + break; + usleep(20000); + } while (++tries < 10); + printf("# collapse ret=%d errno=%d PMD-mapped=%lu\n", ret, + ret ? errno : 0, pmd_mapped(p)); + return ret; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10), + strtoul(argv[5], NULL, 10)); + if (getpid() != 1 || argc != 1) + return 2; + alarm(120); + cpu(0); + CHECK(!mount("proc", "/proc", "proc", 0, NULL) && + !mount("sysfs", "/sys", "sysfs", 0, NULL)); + CHECK(!mount("tmpfs", "/tmp", "tmpfs", 0, + "huge=within_size,size=256m")); + size_t native = getauxval(AT_PAGESZ), len = 0; + FILE *f = fopen("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", + "r"); + CHECK(f && fscanf(f, "%zu", &len) == 1); + fclose(f); + CHECK(native == 16384 && len == 32 * 1024 * 1024); + int file = open("/tmp/collapse", O_CREAT | O_RDWR | O_TRUNC, 0600); + CHECK(file >= 0 && !ftruncate(file, 4 * native)); + unsigned char *initial = map(file, 4 * native, 4 * native, 0); + memset(initial, 0x61, 4 * native); + CHECK(entry("/proc/kpageflags", + mapped_pfn(getpid(), (uintptr_t)initial, native)) & + (1ULL << KPF_COMPOUND_HEAD)); + CHECK(!munmap(initial, 4 * native) && !unlink("/tmp/collapse") && + !ftruncate(file, len)); + int sock[2]; + CHECK(!socketpair(AF_UNIX, SOCK_SEQPACKET, 0, sock)); + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char fd[24], so[24], ns[24], ls[24]; + close(sock[0]); + snprintf(fd, sizeof(fd), "%d", file); + snprintf(so, sizeof(so), "%d", sock[1]); + snprintf(ns, sizeof(ns), "%zu", native); + snprintf(ls, sizeof(ls), "%zu", len); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, 4096, 0UL, 0UL, 0UL)); + execl("/init", "/init", "--worker", fd, so, ns, ls, NULL); + _exit(3); + } + close(sock[1]); + struct ready ready; + struct iovec v = { &ready, sizeof(ready) }; + union { + struct cmsghdr align; + char bytes[CMSG_SPACE(sizeof(int))]; + } control = { 0 }; + struct msghdr msg = { .msg_iov = &v, + .msg_iovlen = 1, + .msg_control = control.bytes, + .msg_controllen = sizeof(control.bytes) }; + CHECK(recvmsg(sock[0], &msg, 0) == sizeof(ready) && + !(msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))); + struct cmsghdr *c = CMSG_FIRSTHDR(&msg); + CHECK(c && c->cmsg_level == SOL_SOCKET && c->cmsg_type == SCM_RIGHTS); + int uffd; + memcpy(&uffd, CMSG_DATA(c), sizeof(uffd)); + unsigned char *p = map(file, len, len, 0); + int native_uffd = register_range(p, len, 0); + CHECK(collapse(p, len) == -1 && errno == EINVAL && pmd_mapped(p) == 0); + fault(sock[0], uffd, ready.base, 0, 1, 0x61); + fault(sock[0], uffd, ready.base, len - native, 1, 0x71); + uint64_t head = mapped_pfn(pid, ready.base, 4096) * 4096 / native; + CHECK(head != ready.head && + (entry("/proc/kpageflags", head) & (1ULL << KPF_COMPOUND_HEAD))); + CHECK(mapped_pfn(pid, ready.base + len - native, 4096) * 4096 / + native == + head + len / native - 1); + for (size_t i = 1; i < len / native; i++) + CHECK(entry("/proc/kpageflags", head + i) & + (1ULL << KPF_COMPOUND_TAIL)); + /* Collapse retained the absent native unit and independent fine holes. */ + fault(sock[0], uffd, ready.base, native, 0, 0x92); + fault(sock[0], uffd, ready.base, 5 * native + 4096, 0, 0xa3); + puts("ok - PMD cache collapse preserves missing/minor state and neighbour data"); + /* Every native leaf now contains data: coarse exposure may populate zero subunits. */ + CHECK(!collapse(p, len) && pmd_mapped(p) == len); + fault(sock[0], uffd, ready.base, 6 * native + 4096, 1, 0); + struct uffdio_copy duplicate = { .dst = ready.base + 6 * native + 8192, + .src = (uintptr_t)p, + .len = 4096 }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &duplicate) == -1 && errno == EEXIST); + puts("ok - native PMD promotion exposes coarse leaf zeros without overwriting fine data"); + /* A new fine hole must invalidate the covering native PMD mapping. */ + send_command(sock[0], 2, 7 * native + 4096, 4096); + reply(sock[0], 0); + CHECK(pmd_mapped(p) == 0); + fault(sock[0], uffd, ready.base, 7 * native + 4096, 0, 0xb4); + CHECK(p[7 * native] == 0x71 && p[7 * native + 4096] == 0xb4 && + p[7 * native + 8192] == 0); + CHECK(p[native] == 0x92 && p[5 * native + 4096] == 0xa3); + puts("ok - fine hole splits PMD alias and preserves populated neighbouring leaves"); + for (unsigned round = 0; round < 8; round++) { + CHECK(!collapse(p, len) && pmd_mapped(p) == len); + size_t off = (16 + round) * native + 4096; + /* Prime CPU0's PMD translation before CPU1 punches the fine hole. */ + CHECK(*(volatile unsigned char *)(p + off) == 0); + send_command(sock[0], 3, off + 17, 0); + reply(sock[0], 0); + send_command(sock[0], 2, off, 4096); + reply(sock[0], 0); + CHECK(pmd_mapped(p) == 0); + if (round % 3) { + /* Stale native or shifted fine translations would miss promotion. */ + if (round % 3 == 1) + memset(p + off, 0xc0 + round, 4096); + else { + struct command write = { 4, off, 4096, + 0xc0 + round }; + CHECK(send(sock[0], &write, sizeof(write), 0) == + sizeof(write)); + reply(sock[0], 0); + } + struct uffdio_copy collision = { .dst = ready.base + + off, + .src = (uintptr_t)p, + .len = 4096 }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &collision) == -1 && + errno == EEXIST); + fault(sock[0], uffd, ready.base, off, 1, 0xc0 + round); + } else + fault(sock[0], uffd, ready.base, off, 0, 0xc0 + round); + CHECK(p[off - 4096] == 0x71 && p[off] == 0xc0 + round && + p[off + 4096] == 0); + send_command(sock[0], 3, off + 17, 0); + reply(sock[0], 0xc0 + round); + } + CHECK(!collapse(p, len) && pmd_mapped(p) == len); + for (size_t i = 0; i < len; i++) { + size_t page = i / native, low = i % native; + unsigned char expected = + page < 4 ? + (page == 1 ? (low < 4096 ? 0x92 : 0) : 0x61) : + (low < 4096 ? 0x71 : 0); + if (low >= 4096 && low < 8192) { + if (page == 5) + expected = 0xa3; + if (page == 7) + expected = 0xb4; + if (page >= 16 && page < 24) + expected = 0xc0 + page - 16; + } + CHECK(p[i] == expected); + } + puts("ok - eight cross-CPU PMD cycles, native/shifted-fine writes and complete 32MiB bytes"); + send_command(sock[0], 0, 0, 0); + int status; + CHECK(waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(!close(uffd) && !close(native_uffd) && !munmap(p, len) && + !close(file) && !close(sock[0])); + puts("SHMEM COLLAPSE CONTRACT PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-hole-init.c b/tools/testing/selftests/arm64/user4k/shmem-hole-init.c new file mode 100644 index 0000000000000..8831cc30a22c3 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-hole-init.c @@ -0,0 +1,358 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - shmem hole line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +struct access { + volatile unsigned char *p; + unsigned char value; +}; +static void *read_byte(void *opaque) +{ + struct access *a = opaque; + a->value = *a->p; + return NULL; +} +static uint64_t pagemap(pid_t pid, uintptr_t address, size_t granule) +{ + char name[80]; + uint64_t entry; + snprintf(name, sizeof(name), "/proc/%d/pagemap", pid); + int fd = open(name, O_RDONLY); + CHECK(fd >= 0 && pread(fd, &entry, 8, address / granule * 8) == 8); + CHECK(!close(fd)); + return entry; +} +static void fill(int fd, unsigned char *dst, unsigned char *src, size_t ps) +{ + struct uffdio_copy c = { .dst = (uintptr_t)dst, + .src = (uintptr_t)src, + .len = ps }; + CHECK(!ioctl(fd, UFFDIO_COPY, &c) && c.copy == (long long)ps); +} +static void resolve(int fd, unsigned char *p, size_t ps, int minor, + unsigned char value) +{ + struct access a = { .p = p + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &a)); + struct pollfd pollfd = { .fd = fd, .events = POLLIN }; + struct uffd_msg msg; + CHECK(poll(&pollfd, 1, 10000) == 1 && (pollfd.revents & POLLIN)); + CHECK(read(fd, &msg, sizeof(msg)) == sizeof(msg)); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)p && + msg.arg.pagefault.flags == + (minor ? UFFD_PAGEFAULT_FLAG_MINOR : 0)); + if (minor) { + struct uffdio_continue c = { .range = { (uintptr_t)p, ps } }; + CHECK(!ioctl(fd, UFFDIO_CONTINUE, &c) && + c.mapped == (long long)ps); + } else { + unsigned char *source = malloc(ps); + CHECK(source); + memset(source, value, ps); + fill(fd, p, source, ps); + free(source); + } + CHECK(!pthread_join(thread, NULL) && a.value == value); +} +struct command { + uint64_t op, offset, len, value; +}; +static int alias_worker(int file, int input, int output, size_t len, + size_t native) +{ + CHECK(getauxval(AT_PAGESZ) == native); + unsigned char *p = + mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + CHECK(p != MAP_FAILED); + /* Establish every coarse PTE before the fine process punches its hole. */ + for (size_t i = 0; i < len; i++) + CHECK(p[i] == 0x61); + uint64_t base = (uintptr_t)p; + CHECK(write(output, &base, sizeof(base)) == sizeof(base)); + struct command cmd; + while (read(input, &cmd, sizeof(cmd)) == sizeof(cmd)) { + CHECK(cmd.offset <= len && cmd.len <= len - cmd.offset); + if (!cmd.op) + break; + if (cmd.op == 2) + memset(p + cmd.offset, cmd.value, cmd.len); + for (size_t i = 0; i < cmd.len; i++) + CHECK(p[cmd.offset + i] == cmd.value); + CHECK(write(output, "y", 1) == 1); + } + CHECK(!munmap(p, len)); + return 0; +} +static void tell(int input, int output, uint64_t op, size_t offset, size_t len, + unsigned value) +{ + struct command cmd = { op, offset, len, value }; + char reply; + CHECK(write(input, &cmd, sizeof(cmd)) == sizeof(cmd)); + if (op) + CHECK(read(output, &reply, 1) == 1 && reply == 'y'); +} +static unsigned long swapouts(void) +{ + FILE *f = fopen("/proc/vmstat", "r"); + char key[80]; + unsigned long value, result = 0; + CHECK(f); + while (fscanf(f, "%79s %lu", key, &value) == 2) + if (!strcmp(key, "pswpout")) + result = value; + fclose(f); + return result; +} +static unsigned long mapping_swap(void *address) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char *line = NULL; + size_t cap = 0; + unsigned long start, end, value, result = 0; + int selected = 0; + CHECK(f); + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) + selected = start == (uintptr_t)address; + else if (selected && sscanf(line, "Swap: %lu kB", &value) == 1) + result = value; + } + free(line); + fclose(f); + return result; +} +static void swapped_missing(size_t ps, size_t native) +{ + int file = memfd_create("partial-uffd-swap", 0); + CHECK(file >= 0 && !ftruncate(file, native)); + unsigned char *p = + mmap(NULL, native, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + unsigned char *source = malloc(ps); + CHECK(p != MAP_FAILED && source); + memset(source, 0x71, ps); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MINOR_SHMEM }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, native }, + .mode = UFFDIO_REGISTER_MODE_MISSING | + UFFDIO_REGISTER_MODE_MINOR }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + fill(fd, p, source, ps); + unsigned long before = swapouts(), usage = 0, written; + unsigned tries; + for (tries = 0; tries < 100; tries++) { + CHECK(!madvise(p, native, MADV_PAGEOUT)); + usleep(20000); + usage = mapping_swap(p); + if (usage >= native / 1024 && swapouts() > before) + break; + } + written = swapouts() - before; + CHECK(usage >= native / 1024 && written > 0); + /* The selected cache index is swapped; its unfilled neighbour stays missing. */ + resolve(fd, p + ps, ps, 0, 0x92); + resolve(fd, p, ps, 1, 0x71); + for (size_t i = 0; i < ps; i++) + CHECK(p[i] == 0x71 && p[ps + i] == 0x92); + CHECK(mapping_swap(p) == 0); + printf("ok - shared uffd %zuK on %zuK swapped cache=%lukB writes=%lu attempts=%u missing COPY preserves neighbour\n", + ps / 1024, native / 1024, usage, written, tries + 1); + CHECK(!close(fd) && !munmap(p, native) && !close(file)); + free(source); +} +static int worker(size_t ps, size_t native) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(60); + if (getenv("SHMEM_HOLE_SWAP") && ps < native) + swapped_missing(ps, native); + size_t len = 4 * native; + int file = memfd_create("hole-contract", 0); + CHECK(file >= 0 && !ftruncate(file, len)); + unsigned char *source = malloc(len); + CHECK(source); + memset(source, 0x61, len); + CHECK(pwrite(file, source, len, 0) == (ssize_t)len); + unsigned char *p = + mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + unsigned char *q = mmap(NULL, len - ps, PROT_READ | PROT_WRITE, + MAP_SHARED, file, ps); + unsigned char *cow = + mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE, file, 0); + CHECK(p != MAP_FAILED && q != MAP_FAILED && cow != MAP_FAILED); + for (size_t i = 0; i < len; i += ps) + CHECK(p[i] == 0x61); + for (size_t i = 0; i < len - ps; i += ps) + CHECK(q[i] == 0x61); + memset(cow + ps, 0xe1, ps); + int commands[2], replies[2]; + CHECK(!pipe(commands) && !pipe(replies)); + pid_t alias = fork(); + CHECK(alias >= 0); + if (!alias) { + char f[24], in[24], out[24], size[24], ns[24]; + close(commands[1]); + close(replies[0]); + snprintf(f, sizeof(f), "%d", file); + snprintf(in, sizeof(in), "%d", commands[0]); + snprintf(out, sizeof(out), "%d", replies[1]); + snprintf(size, sizeof(size), "%zu", len); + snprintf(ns, sizeof(ns), "%zu", native); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, native, 0UL, 0UL, 0UL)); + execl("/init", "/init", "--alias", f, in, out, size, ns, NULL); + _exit(3); + } + close(commands[0]); + close(replies[1]); + uint64_t alias_base; + CHECK(read(replies[0], &alias_base, sizeof(alias_base)) == + sizeof(alias_base)); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MINOR_SHMEM }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING | + UFFDIO_REGISTER_MODE_MINOR }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK(pagemap(alias, alias_base + ps, native) & (1ULL << 63)); + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, ps, + ps)); + CHECK(!(pagemap(getpid(), (uintptr_t)p + ps, ps) & (1ULL << 63))); + CHECK(!(pagemap(getpid(), (uintptr_t)q, ps) & (1ULL << 63))); + CHECK(!(pagemap(alias, alias_base + ps, native) & (1ULL << 63))); + /* Fine neighbour PTEs survive, while COW retains its independent bytes. */ + CHECK(pagemap(getpid(), (uintptr_t)p, ps) & (1ULL << 63)); + CHECK(pagemap(getpid(), (uintptr_t)p + 2 * ps, ps) & (1ULL << 63)); + CHECK(p[0] == 0x61 && p[2 * ps] == 0x61 && cow[ps] == 0xe1); + resolve(fd, p + ps, ps, 0, 0xa2); + CHECK(q[0] == 0xa2 && cow[ps] == 0xe1); + tell(commands[1], replies[0], 1, ps, ps, 0xa2); + tell(commands[1], replies[0], 1, 0, ps, 0x61); + tell(commands[1], replies[0], 1, 2 * ps, ps, 0x61); + /* Refaulting a coarse writable alias makes its covered units populated. */ + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, ps, + ps)); + tell(commands[1], replies[0], 2, ps, ps, 0xb7); + struct uffdio_copy collision = { .dst = (uintptr_t)p + ps, + .src = (uintptr_t)source, + .len = ps }; + errno = 0; + CHECK(ioctl(fd, UFFDIO_COPY, &collision) == -1 && errno == EEXIST); + resolve(fd, p + ps, ps, 1, 0xb7); + printf("ok - shmem hole %zuK on %zuK precise PTEs, shifted alias, COW, coarse revoke/refault and COPY collision\n", + ps / 1024, native / 1024); + /* Stop the alias before resize, so it cannot intentionally promote holes. */ + tell(commands[1], replies[0], 0, 0, 0, 0); + close(commands[1]); + close(replies[0]); + int status; + CHECK(waitpid(alias, &status, 0) == alias && WIFEXITED(status) && + !WEXITSTATUS(status)); + size_t cut = 2 * ps + ps / 2; + CHECK(!ftruncate(file, cut) && !ftruncate(file, len)); + /* Partial final user leaf remains valid, with only truncated bytes zero. */ + if (!(pagemap(getpid(), (uintptr_t)p + 2 * ps, ps) & (1ULL << 63))) + resolve(fd, p + 2 * ps, ps, 1, 0x61); + CHECK(p[2 * ps] == 0x61 && p[cut - 1] == 0x61 && p[cut] == 0 && + p[3 * ps - 1] == 0); + resolve(fd, p + 3 * ps, ps, 0, 0xc3); + CHECK(q[2 * ps] == 0xc3); + CHECK(cow[ps] == 0xe1); + printf("ok - shmem resize %zuK on %zuK partial tail retained, shrink/regrow missing leaf refilled\n", + ps / 1024, native / 1024); + CHECK(!close(fd) && !munmap(p, len) && !munmap(q, len - ps) && + !munmap(cow, len) && !close(file)); + free(source); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 7 && !strcmp(argv[1], "--alias")) + return alias_worker(atoi(argv[2]), atoi(argv[3]), atoi(argv[4]), + strtoul(argv[5], NULL, 10), + strtoul(argv[6], NULL, 10)); + if (argc == 4 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10)); + int swap_test = argc == 2 && !strcmp(argv[1], "--swap"); + if (getpid() != 1 || (argc != 1 && !swap_test)) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + if (swap_test) { + struct stat st; + CHECK(!mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)); + for (unsigned i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + CHECK(!stat("/dev/vda", &st) && S_ISBLK(st.st_mode) && + !swapon("/dev/vda", 0)); + CHECK(!setenv("SHMEM_HOLE_SWAP", "1", 1)); + } + unsigned long native = getauxval(AT_PAGESZ), + sizes[] = { native, 4096, 16384 }; + int failed = 0; + for (unsigned i = 0; i < (native == 65536 ? 3U : 2U); i++) { + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char ps[24], ns[24]; + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + snprintf(ns, sizeof(ns), "%lu", native); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, + 0UL)); + execl("/init", "/init", "--worker", ps, ns, NULL); + _exit(3); + } + int status = 0; + int ok = waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status); + printf("%s - shmem hole ABI %luK status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + failed |= !ok; + } + if (swap_test) + CHECK(!swapoff("/dev/vda")); + printf("SHMEM HOLE CONTRACT %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-pin-init.c b/tools/testing/selftests/arm64/user4k/shmem-pin-init.c new file mode 100644 index 0000000000000..51deda0fde52c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-pin-init.c @@ -0,0 +1,358 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - shmem pin line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +struct ring { + int fd; + struct io_uring_params p; + void *sq, *cq, *sqes; + size_t sqlen, cqlen, sqelen; +}; +static unsigned load(unsigned *p) +{ + return __atomic_load_n(p, __ATOMIC_ACQUIRE); +} +static void store(unsigned *p, unsigned v) +{ + __atomic_store_n(p, v, __ATOMIC_RELEASE); +} +static size_t align_size(size_t n) +{ + size_t ps = getauxval(AT_PAGESZ); + return (n + ps - 1) & ~(ps - 1); +} +static void destroy(struct ring *r) +{ + if (r->sq && r->sq != MAP_FAILED) + munmap(r->sq, r->sqlen); + if (r->cq && r->cq != MAP_FAILED && r->cq != r->sq) + munmap(r->cq, r->cqlen); + if (r->sqes && r->sqes != MAP_FAILED) + munmap(r->sqes, r->sqelen); + if (r->fd >= 0) + close(r->fd); + *r = (struct ring){ .fd = -1 }; +} +static bool setup(struct ring *r, unsigned entries, unsigned flags) +{ + *r = (struct ring){ .fd = -1 }; + r->p.flags = flags; + r->fd = syscall(__NR_io_uring_setup, entries, &r->p); + if (r->fd < 0) + return false; + r->sqlen = align_size(r->p.sq_off.array + + r->p.sq_entries * sizeof(unsigned)); + r->cqlen = align_size(r->p.cq_off.cqes + + r->p.cq_entries * sizeof(struct io_uring_cqe) * + (flags & IORING_SETUP_CQE32 ? 2 : 1)); + r->sqelen = align_size(r->p.sq_entries * sizeof(struct io_uring_sqe) * + (flags & IORING_SETUP_SQE128 ? 2 : 1)); + if (r->p.features & IORING_FEAT_SINGLE_MMAP) { + if (r->cqlen > r->sqlen) + r->sqlen = r->cqlen; + r->cqlen = r->sqlen; + } + r->sq = mmap(NULL, r->sqlen, PROT_READ | PROT_WRITE, MAP_SHARED, r->fd, + IORING_OFF_SQ_RING); + r->cq = (r->p.features & IORING_FEAT_SINGLE_MMAP) ? + r->sq : + mmap(NULL, r->cqlen, PROT_READ | PROT_WRITE, MAP_SHARED, + r->fd, IORING_OFF_CQ_RING); + r->sqes = mmap(NULL, r->sqelen, PROT_READ | PROT_WRITE, MAP_SHARED, + r->fd, IORING_OFF_SQES); + return r->sq != MAP_FAILED && r->cq != MAP_FAILED && + r->sqes != MAP_FAILED; +} +static bool request(struct ring *r, unsigned op, int fd, void *buf, + unsigned len, uint64_t off, bool async) +{ + unsigned *sqtail = r->sq + r->p.sq_off.tail, + *sqmask = r->sq + r->p.sq_off.ring_mask; + unsigned *cqhead = r->cq + r->p.cq_off.head, + *cqtail = r->cq + r->p.cq_off.tail, + *cqmask = r->cq + r->p.cq_off.ring_mask; + unsigned tail = load(sqtail), head = load(cqhead), + idx = tail & load(sqmask); + unsigned sqstride = sizeof(struct io_uring_sqe) * + (r->p.flags & IORING_SETUP_SQE128 ? 2 : 1); + unsigned cqstride = sizeof(struct io_uring_cqe) * + (r->p.flags & IORING_SETUP_CQE32 ? 2 : 1); + struct io_uring_sqe *sqe = r->sqes + idx * sqstride; + memset(sqe, 0, sqstride); + sqe->opcode = op; + sqe->fd = fd; + sqe->addr = (uintptr_t)buf; + sqe->len = len; + sqe->off = off; + sqe->user_data = (uint64_t)tail + 1234; + if (async) + sqe->flags = IOSQE_ASYNC; + ((unsigned *)(r->sq + r->p.sq_off.array))[idx] = idx; + store(sqtail, tail + 1); + int ret = syscall(__NR_io_uring_enter, r->fd, 1, 1, + IORING_ENTER_GETEVENTS, NULL, 0); + if (ret != 1 || load(cqtail) != head + 1) { + printf("enter=%d errno=%d\n", ret, errno); + return false; + } + struct io_uring_cqe *cqe = + r->cq + r->p.cq_off.cqes + (head & load(cqmask)) * cqstride; + unsigned expected = op == IORING_OP_NOP ? 0 : len; + if (op == IORING_OP_READV_FIXED || op == IORING_OP_WRITEV_FIXED) { + expected = 0; + for (unsigned i = 0; i < len; i++) + expected += ((struct iovec *)buf)[i].iov_len; + } + bool ok = cqe->user_data == (uint64_t)tail + 1234 && + cqe->res == (int)expected; + if (!ok) + printf("cqe res=%d expected=%u\n", cqe->res, expected); + store(cqhead, head + 1); + return ok; +} + +static void fragment_geometry(unsigned char *p, size_t ps, int input) +{ + struct ring r; + CHECK(setup(&r, 8, 0)); + struct iovec reg = { p + 123, 3 * ps - 246 }; + unsigned char *src = malloc(3 * ps), *expected = malloc(3 * ps); + CHECK(src && expected); + memset(expected, 0x61, 3 * ps); + memset(src, 0xb7, 3 * ps); + CHECK(pwrite(input, src, 3 * ps, 0) == (ssize_t)(3 * ps)); + CHECK(!syscall(SYS_io_uring_register, r.fd, IORING_REGISTER_BUFFERS, + ®, 1)); + CHECK(request(&r, IORING_OP_READ_FIXED, input, p + ps + 71, ps + 31, 17, + true)); + memset(expected + ps + 71, 0xb7, ps + 31); + CHECK(!memcmp(p, expected, 3 * ps)); + memset(src, 0xc8, 3 * ps); + CHECK(pwrite(input, src, 3 * ps, 0) == (ssize_t)(3 * ps)); + CHECK(request(&r, IORING_OP_WRITE_FIXED, input, p + ps + 71, ps + 31, 9, + true)); + CHECK(pread(input, src, ps + 31, 9) == (ssize_t)(ps + 31)); + for (size_t i = 0; i < ps + 31; i++) + CHECK(src[i] == 0xb7); + struct iovec vec[2] = { { p + 2 * ps + 17, ps - 400 }, + { p + ps + 91, ps - 600 } }; + memset(src, 0xd5, 3 * ps); + CHECK(pwrite(input, src, 3 * ps, 0) == (ssize_t)(3 * ps)); + CHECK(request(&r, IORING_OP_READV_FIXED, input, vec, 2, 13, true)); + for (unsigned i = 0; i < 2; i++) + memset(expected + ((unsigned char *)vec[i].iov_base - p), 0xd5, + vec[i].iov_len); + CHECK(!memcmp(p, expected, 3 * ps)); + memset(src, 0xc8, 3 * ps); + CHECK(pwrite(input, src, 3 * ps, 0) == (ssize_t)(3 * ps)); + CHECK(request(&r, IORING_OP_WRITEV_FIXED, input, vec, 2, 23, true)); + size_t total = vec[0].iov_len + vec[1].iov_len; + CHECK(pread(input, src, total, 23) == (ssize_t)total); + for (size_t i = 0; i < total; i++) + CHECK(src[i] == 0xd5); + CHECK(!syscall(SYS_io_uring_register, r.fd, IORING_UNREGISTER_BUFFERS, + NULL, 0)); + destroy(&r); + free(src); + free(expected); + memset(p, 0x61, 3 * ps); + printf("ok - %zuK fixed/vector IO uses user fragment geometry and preserves guards\n", + ps / 1024); +} + +struct punch_worker { + int file; + size_t off, ps; + pthread_barrier_t start; +}; +static void bind_cpu(unsigned cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} +static void *punch_repeatedly(void *arg) +{ + struct punch_worker *w = arg; + bind_cpu(1); + pthread_barrier_wait(&w->start); + for (unsigned i = 0; i < 256; i++) { + CHECK(!fallocate(w->file, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + w->off, w->ps)); + sched_yield(); + } + return NULL; +} +static void pin_hole_concurrency(struct ring *r, int file, int input, + unsigned char *p, size_t off, size_t ps) +{ + struct punch_worker w = { .file = file, .off = off, .ps = ps }; + pthread_t thread; + CHECK(!pthread_barrier_init(&w.start, NULL, 2)); + bind_cpu(0); + CHECK(!pthread_create(&thread, NULL, punch_repeatedly, &w)); + pthread_barrier_wait(&w.start); + for (unsigned i = 0; i < 256; i++) + CHECK(request(r, IORING_OP_READ_FIXED, input, p + off, ps, 0, + true)); + CHECK(!pthread_join(thread, NULL) && + !pthread_barrier_destroy(&w.start)); + printf("ok - %zuK concurrent fixed writes and 256 hole punches on separate CPUs\n", + ps / 1024); +} + +static int target(size_t native, size_t ps, int resize) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(30); + size_t len = 4 * native, off = native + ps; + bool retained = ps < native; + int file = memfd_create("pin-cache", 0), + input = memfd_create("pin-input", 0); + CHECK(file >= 0 && input >= 0 && !ftruncate(file, len) && + !ftruncate(input, ps)); + unsigned char *src = malloc(len), *back = malloc(len); + CHECK(src && back); + memset(src, 0x61, len); + CHECK(pwrite(file, src, len, 0) == (ssize_t)len); + memset(src, 0xb7, ps); + CHECK(pwrite(input, src, ps, 0) == (ssize_t)ps); + unsigned char *p = + mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + CHECK(p != MAP_FAILED && !madvise(p, len, MADV_NOHUGEPAGE)); + CHECK(p[off] == 0x61); + fragment_geometry(p, ps, input); + memset(src, 0xb7, ps); + CHECK(pwrite(input, src, ps, 0) == (ssize_t)ps); + struct ring r; + CHECK(setup(&r, 8, 0)); + struct iovec reg = { p + off, ps }; + CHECK(!syscall(SYS_io_uring_register, r.fd, IORING_REGISTER_BUFFERS, + ®, 1)); + int uffd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(uffd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(uffd, UFFDIO_API, &api)); + struct uffdio_register ur = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(uffd, UFFDIO_REGISTER, &ur)); + if (resize) + CHECK(!ftruncate(file, off) && !ftruncate(file, len)); + else + CHECK(!fallocate(file, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + off, ps)); + if (retained && !resize) + pin_hole_concurrency(&r, file, input, p, off, ps); + /* Fixed IO accesses its original FOLL_PIN reference after PTE removal. */ + CHECK(request(&r, IORING_OP_READ_FIXED, input, p + off, ps, 0, true)); + /* Also prove the original pinned object contains the completed write. */ + CHECK(request(&r, IORING_OP_WRITE_FIXED, input, p + off, ps, 0, true)); + CHECK(pread(input, back, ps, 0) == (ssize_t)ps); + for (size_t i = 0; i < ps; i++) + CHECK(back[i] == 0xb7); + CHECK(!syscall(SYS_io_uring_register, r.fd, IORING_UNREGISTER_BUFFERS, + NULL, 0)); + memset(src, 0xc8, ps); + struct uffdio_copy c = { .dst = (uintptr_t)(p + off), + .src = (uintptr_t)src, + .len = ps }; + errno = 0; + int ret = ioctl(uffd, UFFDIO_COPY, &c); + printf("# pin %zuK/%zuK resize=%d retained=%d COPY=%d errno=%d bytes=%lld\n", + ps / 1024, native / 1024, resize, retained, ret, errno, + (long long)c.copy); + if (retained) + CHECK(ret == -1 && errno == EEXIST && c.copy == -EEXIST); + else + CHECK(!ret && c.copy == (long long)ps); + CHECK(pread(file, back, len, 0) == (ssize_t)len); + for (size_t i = 0; i < len; i++) { + unsigned char expected = i >= off && i < off + ps ? + (retained ? 0xb7 : 0xc8) : + resize && i >= off ? 0 : + 0x61; + CHECK(back[i] == expected); + } + for (size_t i = 0; i < ps; i++) + CHECK(p[off + i] == (retained ? 0xb7 : 0xc8)); + /* A new punch after unpin can once again be populated by the pager. */ + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, off, + ps)); + c.copy = 0; + CHECK(!ioctl(uffd, UFFDIO_COPY, &c) && c.copy == (long long)ps); + for (size_t i = 0; i < ps; i++) + CHECK(p[off + i] == 0xc8); + destroy(&r); + CHECK(!close(uffd) && !munmap(p, len) && !close(file) && !close(input)); + free(src); + free(back); + printf("ok - writable pin %zuK on %zuK resize=%d retention and detach\n", + ps / 1024, native / 1024, resize); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--target")) + return target(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), atoi(argv[4])); + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + size_t native = getauxval(AT_PAGESZ); + for (size_t ps = 4096; ps <= native; ps *= 4) + for (int resize = 0; resize < 2; resize++) { + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char n[24], g[24], r[12]; + snprintf(n, sizeof(n), "%zu", native); + snprintf(g, sizeof(g), "%zu", ps); + snprintf(r, sizeof(r), "%d", resize); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, ps, 0, 0, 0)); + execl("/init", "init", "--target", n, g, r, + NULL); + _exit(127); + } + int status; + CHECK(waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status)); + } + puts("ok - shmem writable pin fixture complete"); + reboot(RB_POWER_OFF); + return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-pressure-init.c b/tools/testing/selftests/arm64/user4k/shmem-pressure-init.c new file mode 100644 index 0000000000000..f321878396f06 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-pressure-init.c @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CG "/sys/fs/cgroup/pressure" +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - shmem pressure line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +static void put(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + CHECK(fd >= 0); + CHECK(write(fd, value, strlen(value)) == (ssize_t)strlen(value) && + !close(fd)); +} +static unsigned long long get(const char *path, const char *key) +{ + FILE *f = fopen(path, "r"); + CHECK(f); + char name[128]; + unsigned long long value, result = 0; + if (!key) + CHECK(fscanf(f, "%llu", &result) == 1); + else + while (fscanf(f, "%127s %llu", name, &value) == 2) + if (!strcmp(key, name)) { + result = value; + break; + } + fclose(f); + return result; +} +static void copy_leaf(int uffd, unsigned char *dst, unsigned char *src, + size_t ps, unsigned char value) +{ + memset(src, value, ps); + struct uffdio_copy c = { .dst = (uintptr_t)dst, + .src = (uintptr_t)src, + .len = ps }; + int ret = ioctl(uffd, UFFDIO_COPY, &c); + if (ret) + printf("# COPY destination=%p errno=%d transferred=%lld\n", dst, + errno, (long long)c.copy); + CHECK(!ret && c.copy == (long long)ps); +} +static void checkpoint(int out, int in) +{ + char token; + CHECK(write(out, "R", 1) == 1 && read(in, &token, 1) == 1 && + token == 'G'); +} +static int worker(size_t native, size_t ps, int out, int in) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(120); + size_t records = (16UL << 20) / native, + stride = ps < native ? native : 2 * native; + size_t len = records * stride; + int file = memfd_create("pressure-partial-cache", 0); + CHECK(file >= 0 && !ftruncate(file, len)); + unsigned char *p = + mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + unsigned char *source = malloc(stride), *back = malloc(stride); + CHECK(p != MAP_FAILED && source && back); + CHECK(!madvise(p, len, MADV_NOHUGEPAGE)); + int uffd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(uffd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(uffd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(uffd, UFFDIO_REGISTER, ®)); + for (size_t i = 0; i < records; i++) + copy_leaf(uffd, p + i * stride, source, ps, 0xa1); + checkpoint(out, in); + /* Revisit in permuted order, filling holes in reclaimed cache backing. */ + for (size_t i = 0; i < records; i++) { + size_t slot = (i * 4051) & (records - 1); + copy_leaf(uffd, p + slot * stride + ps, source, ps, 0xb2); + for (size_t j = 0; j < ps; j++) + CHECK(p[slot * stride + j] == 0xa1 && + p[slot * stride + ps + j] == 0xb2); + } + checkpoint(out, in); + for (size_t i = 0; i < records; i += 2) { + CHECK(!fallocate(file, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + i * stride, ps)); + copy_leaf(uffd, p + i * stride, source, ps, 0xc3); + } + checkpoint(out, in); + for (size_t i = 0; i < records; i++) { + size_t slot = (i * 4051 + 31) & (records - 1); + CHECK(pread(file, back, stride, slot * stride) == + (ssize_t)stride); + for (size_t j = 0; j < stride; j++) { + unsigned char expected = + j < ps ? (slot & 1 ? 0xa1 : 0xc3) : + j < 2 * ps ? 0xb2 : + 0; + CHECK(back[j] == expected); + } + struct uffdio_copy duplicate = { + .dst = (uintptr_t)(p + slot * stride), + .src = (uintptr_t)source, + .len = ps + }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &duplicate) == -1 && + errno == EEXIST && duplicate.copy == -EEXIST); + } + checkpoint(out, in); + CHECK(!close(uffd) && !munmap(p, len) && !close(file)); + free(source); + free(back); + printf("ok - shmem pressure %zuK on %zuK partial-cache refill, holes, duplicate rejection and every byte\n", + ps / 1024, native / 1024); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), atoi(argv[4]), + atoi(argv[5])); + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL) && + !mount("sysfs", "/sys", "sysfs", 0, NULL)); + CHECK(!mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) && + !mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL)); + put("/sys/fs/cgroup/cgroup.subtree_control", "+memory"); + CHECK(!mkdir(CG, 0755)); + put(CG "/memory.max", "8388608"); + put(CG "/memory.swap.max", "134217728"); + struct stat st; + for (unsigned i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + CHECK(!swapon("/dev/vda", 0)); + size_t native = getauxval(AT_PAGESZ); + for (size_t ps = 4096; ps <= native; ps *= 4) { + int ready[2], go[2]; + CHECK(!pipe(ready) && !pipe(go)); + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + close(ready[0]); + close(go[1]); + char n[24], p[24], r[24], g[24]; + snprintf(n, sizeof(n), "%zu", native); + snprintf(p, sizeof(p), "%zu", ps); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(g, sizeof(g), "%d", go[0]); + put(CG "/cgroup.procs", "0"); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, ps, 0, 0, 0)); + execl("/init", "init", "--worker", n, p, r, g, NULL); + _exit(127); + } + close(ready[1]); + close(go[0]); + for (unsigned phase = 0; phase < 4; phase++) { + char token; + CHECK(read(ready[0], &token, 1) == 1 && token == 'R'); + unsigned long long swap = get(CG "/memory.swap.current", + NULL), + max = get(CG "/memory.events", + "max"); + unsigned long long scan = get(CG "/memory.stat", + "pgscan"), + steal = get(CG "/memory.stat", + "pgsteal"); + printf("# pressure %zuK phase=%u resident=%llu swap=%llu max=%llu scan=%llu steal=%llu\n", + ps / 1024, phase, + get(CG "/memory.current", NULL), swap, max, scan, + steal); + CHECK(swap && max && scan && steal && + !get(CG "/memory.events", "oom") && + !get(CG "/memory.events", "oom_kill")); + CHECK(write(go[1], "G", 1) == 1); + } + int status; + CHECK(waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status)); + close(ready[0]); + close(go[1]); + /* Closed/unmapped file identity must release all of its cache/swap. */ + for (unsigned i = 0; + i < 3000 && (get(CG "/memory.swap.current", NULL) || + get(CG "/memory.stat", "shmem")); + i++) + usleep(20000); + CHECK(!get(CG "/memory.swap.current", NULL) && + !get(CG "/memory.stat", "shmem")); + } + CHECK(!swapoff("/dev/vda")); + puts("SHMEM PRESSURE CONTRACT PASS"); + sync(); + reboot(RB_POWER_OFF); + return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-race-init.c b/tools/testing/selftests/arm64/user4k/shmem-race-init.c new file mode 100644 index 0000000000000..5109654b6ec21 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-race-init.c @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define ROUNDS 512 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - shmem race line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +struct control { + unsigned start, stop, ready; + unsigned long aliases; +}; +struct state { + struct control *control; + int file, uffd, resize; + size_t ps, native, len; + unsigned char *p, *q, *source; + unsigned copied, existed, eof, alloc_eof, retry; +}; +static unsigned load(unsigned *p) +{ + return __atomic_load_n(p, __ATOMIC_ACQUIRE); +} +static void store(unsigned *p, unsigned value) +{ + __atomic_store_n(p, value, __ATOMIC_RELEASE); +} +static void bind_cpu(unsigned cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} +static void start_wait(struct control *c) +{ + __atomic_add_fetch(&c->ready, 1, __ATOMIC_RELEASE); + while (!load(&c->start)) + sched_yield(); +} +static void *mutate(void *arg) +{ + struct state *s = arg; + bind_cpu(0); + start_wait(s->control); + for (unsigned i = 0; i < ROUNDS; i++) { + if (s->resize) + CHECK(!ftruncate(s->file, s->ps) && + !ftruncate(s->file, s->len)); + else + CHECK(!fallocate(s->file, + FALLOC_FL_PUNCH_HOLE | + FALLOC_FL_KEEP_SIZE, + s->ps, s->ps)); + sched_yield(); + } + return NULL; +} +static void *populate(void *arg) +{ + struct state *s = arg; + bind_cpu(1); + start_wait(s->control); + for (unsigned i = 0; i < ROUNDS; i++) { + struct uffdio_copy c = { .dst = (uintptr_t)(s->p + s->ps), + .src = (uintptr_t)s->source, + .len = s->ps }; + errno = 0; + int ret = ioctl(s->uffd, UFFDIO_COPY, &c); + if (!ret) { + CHECK(c.copy == (long long)s->ps); + s->copied++; + } else { + CHECK(ret == -1 && c.copy == -errno); + if (errno == EEXIST) + s->existed++; + else if (errno == EFAULT && s->resize) + s->eof++; + /* Native shmem allocator preserves its NULL/ENOMEM EOF ABI. */ + else if (errno == ENOMEM && s->resize && + s->ps == s->native) + s->alloc_eof++; + else if (errno == EAGAIN) + s->retry++; + else { + printf("# unexpected COPY error=%d bytes=%lld\n", + errno, (long long)c.copy); + CHECK(0); + } + } + sched_yield(); + } + return NULL; +} +static int valid_byte(unsigned char value) +{ + return value == 0 || value == 0x61 || value == 0xa2; +} +static void *fine_alias(void *arg) +{ + struct state *s = arg; + bind_cpu(0); + start_wait(s->control); + for (unsigned i = 0; i < ROUNDS; i++) { + if (!s->resize) { + CHECK(!madvise(s->q, s->ps, MADV_DONTNEED)); + CHECK(valid_byte(*(volatile unsigned char *)s->q)); + } else { + /* The retained prefix remains inside EOF throughout resize. */ + CHECK(*(volatile unsigned char *)s->p == 0x61); + } + sched_yield(); + } + return NULL; +} +static int native_alias(int file, int control, size_t native, size_t ps, + int resize) +{ + CHECK(getauxval(AT_PAGESZ) == native); + bind_cpu(1); + alarm(60); + struct control *c = mmap(NULL, native, PROT_READ | PROT_WRITE, + MAP_SHARED, control, 0); + unsigned char *p = mmap(NULL, 4 * native, PROT_READ | PROT_WRITE, + MAP_SHARED, file, 0); + CHECK(c != MAP_FAILED && p != MAP_FAILED && + !madvise(p, 4 * native, MADV_NOHUGEPAGE)); + start_wait(c); + unsigned long cycles = 0; + while (!load(&c->stop)) { + CHECK(!madvise(p, 2 * native, MADV_DONTNEED)); + CHECK(*(volatile unsigned char *)p == 0x61); + if (!resize) + CHECK(valid_byte(*(volatile unsigned char *)(p + ps))); + cycles++; + sched_yield(); + } + c->aliases = cycles; + CHECK(!munmap(p, 4 * native) && !munmap(c, native)); + return 0; +} +static int target(size_t native, size_t ps, int resize) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(60); + struct state s = { + .native = native, .ps = ps, .len = 4 * native, .resize = resize + }; + s.file = memfd_create("race-cache", 0); + int control = memfd_create("race-control", 0); + CHECK(s.file >= 0 && control >= 0 && !ftruncate(s.file, s.len) && + !ftruncate(control, native)); + s.control = mmap(NULL, native, PROT_READ | PROT_WRITE, MAP_SHARED, + control, 0); + s.source = malloc(s.len); + CHECK(s.control != MAP_FAILED && s.source); + memset(s.source, 0x61, s.len); + CHECK(pwrite(s.file, s.source, s.len, 0) == (ssize_t)s.len); + s.p = mmap(NULL, s.len, PROT_READ | PROT_WRITE, MAP_SHARED, s.file, 0); + s.q = mmap(NULL, s.len - ps, PROT_READ | PROT_WRITE, MAP_SHARED, s.file, + ps); + CHECK(s.p != MAP_FAILED && s.q != MAP_FAILED && + !madvise(s.p, s.len, MADV_NOHUGEPAGE)); + CHECK(s.p[0] == + 0x61); /* Avoid blocking the resize prefix reader on UFFD. */ + unsigned char *cow = MAP_FAILED; + if (!resize) { + cow = mmap(NULL, s.len, PROT_READ | PROT_WRITE, MAP_PRIVATE, + s.file, 0); + CHECK(cow != MAP_FAILED); + memset(cow + ps, 0xe1, ps); + } + s.uffd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(s.uffd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(s.uffd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)s.p, s.len }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(s.uffd, UFFDIO_REGISTER, ®)); + memset(s.source, 0xa2, ps); + pid_t alias = fork(); + CHECK(alias >= 0); + if (!alias) { + char f[24], c[24], n[24], p[24], r[12]; + snprintf(f, sizeof(f), "%d", s.file); + snprintf(c, sizeof(c), "%d", control); + snprintf(n, sizeof(n), "%zu", native); + snprintf(p, sizeof(p), "%zu", ps); + snprintf(r, sizeof(r), "%d", resize); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, native, 0, 0, 0)); + execl("/init", "init", "--alias", f, c, n, p, r, NULL); + _exit(127); + } + pthread_t mutation, copy, fine; + CHECK(!pthread_create(&mutation, NULL, mutate, &s) && + !pthread_create(©, NULL, populate, &s) && + !pthread_create(&fine, NULL, fine_alias, &s)); + while (load(&s.control->ready) != 4) + sched_yield(); + store(&s.control->start, 1); + CHECK(!pthread_join(mutation, NULL) && !pthread_join(copy, NULL) && + !pthread_join(fine, NULL)); + store(&s.control->stop, 1); + int status; + CHECK(waitpid(alias, &status, 0) == alias && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(s.control->aliases && + s.copied + s.existed + s.eof + s.alloc_eof + s.retry == ROUNDS); + printf("# race %zuK/%zuK resize=%d COPY=%u EEXIST=%u EOF=%u native-alloc-EOF=%u retry=%u native-refaults=%lu\n", + ps / 1024, native / 1024, resize, s.copied, s.existed, s.eof, + s.alloc_eof, s.retry, s.control->aliases); + /* Once concurrent actors stop, exact cache/PTE state must be recoverable. */ + CHECK(!ftruncate(s.file, s.len)); + CHECK(!fallocate(s.file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, ps, + ps)); + memset(s.source, 0xd4, ps); + struct uffdio_copy fill = { .dst = (uintptr_t)(s.p + ps), + .src = (uintptr_t)s.source, + .len = ps }; + CHECK(!ioctl(s.uffd, UFFDIO_COPY, &fill) && fill.copy == (long long)ps); + unsigned char *back = malloc(s.len); + CHECK(back && pread(s.file, back, s.len, 0) == (ssize_t)s.len); + for (size_t i = 0; i < s.len; i++) { + unsigned char expected = i >= ps && i < 2 * ps ? 0xd4 : + resize && i >= ps ? 0 : + 0x61; + if (back[i] != expected) { + printf("# byte=%zu actual=%#x expected=%#x\n", i, + back[i], expected); + CHECK(0); + } + } + for (size_t i = 0; i < ps; i++) { + CHECK(s.p[ps + i] == 0xd4 && s.q[i] == 0xd4); + if (!resize) + CHECK(cow[ps + i] == 0xe1); + } + CHECK(!close(s.uffd) && !munmap(s.p, s.len) && + !munmap(s.q, s.len - ps)); + if (!resize) + CHECK(!munmap(cow, s.len)); + CHECK(!munmap(s.control, native) && !close(control) && !close(s.file)); + free(back); + free(s.source); + printf("ok - concurrent shmem %zuK on %zuK resize=%d cache bytes, aliases and private COW\n", + ps / 1024, native / 1024, resize); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 7 && !strcmp(argv[1], "--alias")) + return native_alias(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10), + strtoul(argv[5], NULL, 10), atoi(argv[6])); + if (argc == 5 && !strcmp(argv[1], "--target")) + return target(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), atoi(argv[4])); + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + size_t native = getauxval(AT_PAGESZ); + for (size_t ps = 4096; ps <= native; ps *= 4) + for (int resize = 0; resize < 2; resize++) { + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char n[24], p[24], r[12]; + snprintf(n, sizeof(n), "%zu", native); + snprintf(p, sizeof(p), "%zu", ps); + snprintf(r, sizeof(r), "%d", resize); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, ps, 0, 0, 0)); + execl("/init", "init", "--target", n, p, r, + NULL); + _exit(127); + } + int status; + CHECK(waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status)); + } + puts("SHMEM RACE CONTRACT PASS"); + reboot(RB_POWER_OFF); + return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-thp-init.c b/tools/testing/selftests/arm64/user4k/shmem-thp-init.c new file mode 100644 index 0000000000000..e08ea92103fcc --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-thp-init.c @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - shmem THP line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +static uint64_t read_entry(const char *path, uint64_t index) +{ + uint64_t value; + int fd = open(path, O_RDONLY); + CHECK(fd >= 0 && pread(fd, &value, 8, index * 8) == 8 && !close(fd)); + return value; +} +static uint64_t pfn(void *p, size_t ps) +{ + uint64_t entry = read_entry("/proc/self/pagemap", (uintptr_t)p / ps); + CHECK(entry & (1ULL << 63)); + return entry & ((1ULL << 55) - 1); +} +static unsigned char *map_aligned(int fd, size_t len) +{ + void *reserve = mmap(NULL, 2 * len, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + uintptr_t address = ((uintptr_t)reserve + len - 1) & ~(len - 1); + CHECK(!munmap(reserve, 2 * len)); + unsigned char *p = mmap((void *)address, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, fd, 0); + CHECK(p != MAP_FAILED); + return p; +} +struct reader { + volatile unsigned char *p; + unsigned char value; +}; +static void *read_byte(void *arg) +{ + struct reader *r = arg; + r->value = *r->p; + return NULL; +} +static int target(int file, size_t native, size_t ps, int hold, int zero, + int resize, uint64_t head) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(30); + size_t len = 4 * native, target_off = resize ? 2 * native : native; + size_t cut = native + ps / 2; + unsigned char *p = map_aligned(file, len); + for (size_t i = 0; i < len; i += ps) + CHECK(p[i] == 0x61); + CHECK(pfn(p, ps) * ps / native == head); + int pipes[2]; + CHECK(!pipe(pipes)); + if (hold) { + struct iovec iov = { p, ps }; + CHECK(vmsplice(pipes[1], &iov, 1, 0) == (ssize_t)ps); + } + int uffd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(uffd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(uffd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(uffd, UFFDIO_REGISTER, ®)); + if (resize) + CHECK(!ftruncate(file, cut) && !ftruncate(file, len)); + else + CHECK(!fallocate(file, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + native, ps)); + uint64_t flags = read_entry("/proc/kpageflags", head); + printf("# THP %zuK native=%zuK hold=%d zero=%d resize=%d head_flags=%#llx\n", + ps / 1024, native / 1024, hold, zero, resize, + (unsigned long long)flags); + if (hold) + CHECK((flags & + ((1ULL << KPF_COMPOUND_HEAD) | (1ULL << KPF_THP))) == + ((1ULL << KPF_COMPOUND_HEAD) | (1ULL << KPF_THP))); + struct reader r = { .p = p + target_off + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct pollfd pollfd = { .fd = uffd, .events = POLLIN }; + struct uffd_msg msg; + CHECK(poll(&pollfd, 1, 10000) == 1 && (pollfd.revents & POLLIN)); + CHECK(read(uffd, &msg, sizeof(msg)) == sizeof(msg)); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && !msg.arg.pagefault.flags && + msg.arg.pagefault.address == (uintptr_t)(p + target_off)); + unsigned char *source = malloc(ps); + CHECK(source); + memset(source, 0x92, ps); + int ret; + struct uffdio_copy invalid = { .dst = (uintptr_t)(p + target_off), + .src = 1, + .len = ps }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &invalid) == -1 && errno == EFAULT && + invalid.copy == -EFAULT); + if (zero) { + struct uffdio_zeropage z = { + .range = { (uintptr_t)(p + target_off), ps } + }; + ret = ioctl(uffd, UFFDIO_ZEROPAGE, &z); + printf("# ZEROPAGE returned=%d errno=%d bytes=%lld\n", ret, + errno, (long long)z.zeropage); + CHECK(!ret && z.zeropage == (long long)ps); + } else { + struct uffdio_copy c = { .dst = (uintptr_t)(p + target_off), + .src = (uintptr_t)source, + .len = ps }; + ret = ioctl(uffd, UFFDIO_COPY, &c); + printf("# COPY returned=%d errno=%d bytes=%lld\n", ret, errno, + (long long)c.copy); + CHECK(!ret && c.copy == (long long)ps); + } + CHECK(!pthread_join(thread, NULL) && r.value == (zero ? 0 : 0x92)); + memset(source, 0xd4, ps); + struct uffdio_copy duplicate = { .dst = (uintptr_t)(p + target_off), + .src = (uintptr_t)source, + .len = ps }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &duplicate) == -1 && errno == EEXIST && + duplicate.copy == -EEXIST); + /* File reads verify zeroed tail holes without faulting unfilled UFFD leaves. */ + unsigned char *back = malloc(len); + CHECK(back); + CHECK(pread(file, back, len, 0) == (ssize_t)len); + for (size_t i = 0; i < len; i++) { + unsigned char expected = i >= target_off && + i < target_off + ps ? + (zero ? 0 : 0x92) : + resize && i >= cut ? 0 : + 0x61; + CHECK(back[i] == expected); + if (!resize || (i >= target_off && i < target_off + ps)) + CHECK(p[i] == expected); + } + free(back); + if (hold) { + CHECK(read(pipes[0], source, ps) == (ssize_t)ps); + for (size_t i = 0; i < ps; i++) + CHECK(source[i] == 0x61); + } + CHECK(!close(pipes[0]) && !close(pipes[1]) && !close(uffd) && + !munmap(p, len) && !close(file)); + free(source); + printf("ok - THP %zuK on %zuK hold=%d zero=%d resize=%d missing refill and all neighbour bytes\n", + ps / 1024, native / 1024, hold, zero, resize); + return 0; +} +static int create_file(size_t native, uint64_t *head) +{ + int fd = open("/tmp/huge", O_CREAT | O_TRUNC | O_RDWR, 0600); + CHECK(fd >= 0); + CHECK(!ftruncate(fd, 4 * native)); + unsigned char *p = map_aligned(fd, 4 * native); + CHECK(!madvise(p, 4 * native, MADV_HUGEPAGE)); + memset(p, 0x61, 4 * native); + *head = pfn(p, native); + uint64_t f = read_entry("/proc/kpageflags", *head); + printf("# initial head=%llu flags=%#llx VA=%p native=%zu\n", + (unsigned long long)*head, (unsigned long long)f, p, native); + CHECK((f & ((1ULL << KPF_COMPOUND_HEAD) | (1ULL << KPF_THP))) == + ((1ULL << KPF_COMPOUND_HEAD) | (1ULL << KPF_THP))); + for (unsigned i = 1; i < 4; i++) { + CHECK(pfn(p + i * native, native) == *head + i); + CHECK(read_entry("/proc/kpageflags", *head + i) & + (1ULL << KPF_COMPOUND_TAIL)); + } + /* Keep the name until allocation: unlinked mappings use anon-shmem policy. */ + CHECK(!unlink("/tmp/huge") && !munmap(p, 4 * native)); + return fd; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 9 && !strcmp(argv[1], "--target")) + return target(atoi(argv[2]), strtoul(argv[3], NULL, 10), + strtoul(argv[4], NULL, 10), atoi(argv[5]), + atoi(argv[6]), atoi(argv[7]), + strtoull(argv[8], NULL, 10)); + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + CHECK(!mount("tmpfs", "/tmp", "tmpfs", 0, "huge=within_size,size=32m")); + size_t native = getauxval(AT_PAGESZ), sizes[] = { native, 4096, 16384 }; + int failed = 0; + for (unsigned abi = 0; abi < (native == 65536 ? 3U : 2U); abi++) + for (unsigned hold = 0; hold < 2; hold++) + for (unsigned zero = 0; zero < 2; zero++) + for (unsigned resize = 0; resize < 2; + resize++) { + uint64_t head; + int fd = create_file(native, &head); + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char fs[24], ns[24], ps[24], + hs[24], zs[24], rs[24], + pfs[24]; + snprintf(fs, sizeof(fs), "%d", + fd); + snprintf(ns, sizeof(ns), "%zu", + native); + snprintf(ps, sizeof(ps), "%zu", + sizes[abi]); + snprintf(hs, sizeof(hs), "%u", + hold); + snprintf(zs, sizeof(zs), "%u", + zero); + snprintf(rs, sizeof(rs), "%u", + resize); + snprintf(pfs, sizeof(pfs), + "%llu", + (unsigned long long) + head); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, + sizes[abi], 0UL, + 0UL, 0UL)); + execl("/init", "/init", + "--target", fs, ns, ps, + hs, zs, rs, pfs, NULL); + _exit(3); + } + int status = 0; + int ok = waitpid(pid, &status, 0) == + pid && + WIFEXITED(status) && + !WEXITSTATUS(status); + printf("%s - THP ABI %zuK hold=%u zero=%u resize=%u status=%#x\n", + ok ? "ok" : "not ok", + sizes[abi] / 1024, hold, zero, + resize, status); + failed |= !ok; + CHECK(!close(fd)); + } + printf("SHMEM THP CONTRACT %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-uffd-swap-init.c b/tools/testing/selftests/arm64/user4k/shmem-uffd-swap-init.c new file mode 100644 index 0000000000000..73ff33d5e0ba1 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-uffd-swap-init.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PID-1 runner for a disposable VM with a dedicated virtio swap disk. + * Use kunit.autorun=0: tests skipped at boot stay skipped on a debugfs rerun. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int run_swap_suite(void) +{ + const char *base = "/sys/kernel/debug/kunit/arm64-user4k-shmem-swap/"; + char path[128], *log; + size_t used = 0, capacity = 1024 * 1024; + ssize_t n; + int fd, ret = -1; + + snprintf(path, sizeof(path), "%srun", base); + fd = open(path, O_WRONLY | O_CLOEXEC); + if (fd < 0) + return -1; + n = write(fd, "1", 1); + close(fd); + if (n != 1) + return -1; + snprintf(path, sizeof(path), "%sresults", base); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + log = calloc(1, capacity); + if (!log) { + close(fd); + return -1; + } + while (used < capacity - 1 && + (n = read(fd, log + used, capacity - 1 - used)) > 0) + used += n; + close(fd); + fputs(log, stdout); + if (n == 0 && used < capacity - 1 && + strstr(log, "\nok 1 arm64-user4k-shmem-swap\n") && + !strstr(log, "not ok") && !strstr(log, "# SKIP")) + ret = 0; + free(log); + return ret; +} + +static int fault_setting(const char *name, const char *value) +{ + char path[160]; + snprintf(path, sizeof(path), "/sys/kernel/debug/failslab/%s", name); + int fd = open(path, O_WRONLY | O_CLOEXEC); + if (fd < 0) + return -1; + ssize_t n = write(fd, value, strlen(value)); + close(fd); + return n == (ssize_t)strlen(value) ? 0 : -1; +} +static int configure_failslab(bool enable) +{ + if (!enable) + return fault_setting("probability", "0"); + /* The KUnit worker selects itself only around metadata writeout. */ + return fault_setting("task-filter", "Y") || + fault_setting("verbose", "0") || fault_setting("times", "-1") || + fault_setting("interval", "1") || fault_setting("space", "0") || + fault_setting("probability", "100"); +} + +static int swap_control_helper(const char *operation) +{ + struct stat init, self; + int ret; + + /* A helper is allowed only when this same executable is the VM's PID 1. */ + if (getpid() == 1 || stat("/proc/1/exe", &init) || + stat("/proc/self/exe", &self) || init.st_dev != self.st_dev || + init.st_ino != self.st_ino) { + fprintf(stderr, + "swap helper requires its fixture runner as PID 1\n"); + return 1; + } + if (!strcmp(operation, "--memcg-setup-helper")) { + if (mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL) && + errno != EBUSY) + return 1; + int fd = open("/sys/fs/cgroup/cgroup.subtree_control", + O_WRONLY | O_CLOEXEC); + if (fd < 0) + return 1; + ssize_t n = write(fd, "+memory", 7); + close(fd); + if (n != 7) + return 1; + return mkdir("/sys/fs/cgroup/uffd-metadata", 0755) && + errno != EEXIST; + } + if (!strcmp(operation, "--failslab-on-helper")) + return configure_failslab(true); + if (!strcmp(operation, "--failslab-off-helper")) + return configure_failslab(false); + ret = !strcmp(operation, "--swapoff-helper") ? swapoff("/dev/vda") : + swapon("/dev/vda", 0); + if (ret) + perror(operation); + return ret ? 1 : 0; +} + +int main(int argc, char **argv) +{ + struct stat st; + bool active = false; + int ret = 1; + if (argc == 2 && (!strcmp(argv[1], "--swapon-helper") || + !strcmp(argv[1], "--swapoff-helper") || + !strcmp(argv[1], "--failslab-on-helper") || + !strcmp(argv[1], "--failslab-off-helper") || + !strcmp(argv[1], "--memcg-setup-helper"))) + return swap_control_helper(argv[1]); + if (getpid() != 1 || argc != 1) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + unsigned long ps = getauxval(AT_PAGESZ); + if (ps != 16384 && ps != 65536) + goto out; + if (mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) + goto out; + for (unsigned i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + if (stat("/dev/vda", &st) || !S_ISBLK(st.st_mode) || + swapon("/dev/vda", 0)) + goto out; + active = true; + ret = run_swap_suite() ? 1 : 0; +out: + if (active && swapoff("/dev/vda")) + ret = 1; + printf("SHMEM UFFD STATE SWAP %s errno=%d\n", ret ? "FAIL" : "PASS", + errno); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/splice-fixture-init.c b/tools/testing/selftests/arm64/user4k/splice-fixture-init.c new file mode 100644 index 0000000000000..282ba6268b25d --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/splice-fixture-init.c @@ -0,0 +1,610 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static bool use_tcp, use_ipv6, use_udp; + +static int connected_pair(int sockets[2]) +{ + struct sockaddr_in a4 = { .sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_LOOPBACK) }; + struct sockaddr_in6 a6 = { .sin6_family = AF_INET6, + .sin6_addr = IN6ADDR_LOOPBACK_INIT }; + struct sockaddr_storage peer; + struct sockaddr *address = use_ipv6 ? (void *)&a6 : (void *)&a4; + socklen_t size = use_ipv6 ? sizeof(a6) : sizeof(a4), + peer_size = sizeof(peer); + int domain = use_ipv6 ? AF_INET6 : AF_INET; + int listener, saved; + + if (!use_tcp && !use_udp) + return socketpair(AF_UNIX, SOCK_STREAM, 0, sockets); + listener = socket(domain, use_udp ? SOCK_DGRAM : SOCK_STREAM, 0); + if (listener < 0) + return -1; + sockets[0] = -1; + if (bind(listener, address, size) || + getsockname(listener, address, &size)) + goto error; + if (!use_udp && listen(listener, 1)) + goto error; + sockets[0] = socket(domain, use_udp ? SOCK_DGRAM : SOCK_STREAM, 0); + if (sockets[0] < 0 || connect(sockets[0], address, size)) + goto error; + if (use_udp) { + if (getsockname(sockets[0], (void *)&peer, &peer_size) || + connect(listener, (void *)&peer, peer_size)) + goto error; + sockets[1] = listener; + return 0; + } + sockets[1] = accept(listener, NULL, NULL); + if (sockets[1] < 0) + goto error; + close(listener); + return 0; +error: + saved = errno; + if (sockets[0] >= 0) + close(sockets[0]); + close(listener); + errno = saved; + return -1; +} + +static unsigned char value(size_t off) +{ + return (off * 13 + (off >> 12) * 31) & 255; +} +static void fill(unsigned char *p, size_t n) +{ + for (size_t i = 0; i < n; i++) + p[i] = value(i); +} +static bool check(const unsigned char *p, size_t n, size_t off) +{ + for (size_t i = 0; i < n; i++) + if (p[i] != value(i + off)) { + ksft_print_msg( + "data mismatch at %zu: got %02x expected %02x\n", + i + off, p[i], value(i + off)); + return false; + } + return true; +} +static bool read_exact(int fd, unsigned char *p, size_t n) +{ + while (n) { + ssize_t got = read(fd, p, n); + + if (got <= 0) { + ksft_print_msg( + "read fd=%d got=%zd remaining=%zu errno=%d\n", + fd, got, n, errno); + return false; + } + p += got; + n -= got; + } + return true; +} +static bool replace(unsigned char *p, size_t n) +{ + if (munmap(p, n) || + mmap(p, n, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0) != p) + return false; + memset(p, 0xec, n); + return true; +} +static bool cow(unsigned char *p, size_t n) +{ + pid_t child = fork(); + int status = 0; + + if (!child) + _exit(0); + if (child < 0 || waitpid(child, &status, 0) != child || status) + return false; + memset(p, 0xab, n); + return true; +} +static void report(bool ok, size_t ps, const char *name) +{ + ksft_test_result(ok, "%zuK %s\n", ps / 1024, name); +} +static bool tls_pair(int sockets[2], unsigned int version) +{ + struct tls12_crypto_info_aes_gcm_128 crypto = { + .info.version = version, + .info.cipher_type = TLS_CIPHER_AES_GCM_128, + }; + struct timeval timeout = { .tv_sec = 5 }; + int no_padding = 1; + + if (connected_pair(sockets)) + return false; + return !setsockopt(sockets[0], IPPROTO_TCP, TCP_ULP, "tls", 4) && + !setsockopt(sockets[1], IPPROTO_TCP, TCP_ULP, "tls", 4) && + !setsockopt(sockets[0], SOL_TLS, TLS_TX, &crypto, + sizeof(crypto)) && + !setsockopt(sockets[1], SOL_TLS, TLS_RX, &crypto, + sizeof(crypto)) && + (version != TLS_1_3_VERSION || + !setsockopt(sockets[1], SOL_TLS, TLS_RX_EXPECT_NO_PAD, + &no_padding, sizeof(no_padding))) && + !setsockopt(sockets[1], SOL_SOCKET, SO_RCVTIMEO, &timeout, + sizeof(timeout)); +} + +static void tls_checks(size_t ps, unsigned char *p, size_t len, + unsigned char *back) +{ + int sockets[2], fds[2]; + char name[160]; + + if (pipe(fds)) + ksft_exit_fail_msg("TLS pipe errno=%d\n", errno); + for (unsigned int v = 0; v < 2; v++) { + unsigned int version = v ? TLS_1_3_VERSION : TLS_1_2_VERSION; + struct iovec vec; + bool ok; + + if (!tls_pair(sockets, version)) + ksft_exit_fail_msg("TLS %x setup errno=%d\n", version, + errno); + for (unsigned int mode = 0; mode < 2; mode++) { + const size_t count = mode ? 4096 : 4096 - 29; + const size_t offset = ps + (mode ? 0 : 29); + + fill(p, len); + vec = (struct iovec){ p + offset, count }; + if (vmsplice(fds[1], &vec, 1, 0) != (ssize_t)count || + splice(fds[0], NULL, sockets[0], NULL, count, + SPLICE_F_MORE) != (ssize_t)count) + ksft_exit_fail_msg("TLS MORE splice errno=%d\n", + errno); + errno = 0; + if (recv(sockets[1], back, len, MSG_DONTWAIT) != -1 || + errno != EAGAIN) + ksft_exit_fail_msg( + "TLS pending plaintext precondition errno=%d\n", + errno); + ok = mode ? cow(p + ps, 4096) : replace(p + ps, ps); + ok &= send(sockets[0], "!", 1, 0) == 1; + ok &= read_exact(sockets[1], back, count + 1) && + check(back, count, offset) && back[count] == '!'; + snprintf(name, sizeof(name), + "TLS 1.%u pending plaintext survives %s", + v + 2, + mode ? "fork/COW" : "unmap/replacement"); + report(ok, ps, name); + } + { + const size_t bytes = 3 * 16384 + 123, prefix = 137; + const size_t mapping = (bytes + ps - 1) & ~(ps - 1); + unsigned char *source = + mmap(NULL, mapping, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + unsigned char *received = malloc(bytes + prefix); + + if (source == MAP_FAILED || !received) + ksft_exit_fail_msg( + "TLS multi-record allocation\n"); + fill(source, mapping); + ok = send(sockets[0], source, prefix, 0) == + (ssize_t)prefix; + vec = (struct iovec){ source, bytes }; + if (vmsplice(fds[1], &vec, 1, 0) != (ssize_t)bytes || + splice(fds[0], NULL, sockets[0], NULL, bytes, 0) != + (ssize_t)bytes) + ksft_exit_fail_msg( + "TLS multi-record splice errno=%d\n", + errno); + ok &= replace(source, mapping); + ok &= read_exact(sockets[1], received, + bytes + prefix) && + check(received, prefix, 0) && + check(received + prefix, bytes, 0); + snprintf( + name, sizeof(name), + "TLS 1.%u ordinary send and multiple spliced records", + v + 2); + report(ok, ps, name); + munmap(source, mapping); + free(received); + } + { + const size_t bytes = 5003; + int file = memfd_create("tls-receive", 0); + unsigned char *mapped; + + if (file < 0 || ftruncate(file, 3 * ps)) + ksft_exit_fail_msg("TLS receive file setup\n"); + mapped = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, file, 0); + if (mapped == MAP_FAILED) + ksft_exit_fail_msg( + "TLS receive file mapping\n"); + memset(mapped, 0x5c, 3 * ps); + fill(p, len); + ok = send(sockets[0], p, bytes, 0) == (ssize_t)bytes && + read_exact(sockets[1], mapped + 17, bytes); + ok &= !msync(mapped, 3 * ps, MS_SYNC) && + pread(file, back, bytes + 34, 0) == + (ssize_t)(bytes + 34) && + check(back + 17, bytes, 0); + for (unsigned int i = 0; i < 17; i++) + ok &= back[i] == 0x5c && + back[bytes + 17 + i] == 0x5c; + snprintf( + name, sizeof(name), + "TLS 1.%u decrypts into shared file with neighboring bytes intact", + v + 2); + report(ok, ps, name); + munmap(mapped, 3 * ps); + close(file); + } + { + unsigned char *mapped = + mmap(NULL, 6 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + struct iovec vectors[4]; + struct msghdr message = { .msg_iov = vectors, + .msg_iovlen = 4 }; + + if (mapped == MAP_FAILED) + ksft_exit_fail_msg( + "TLS receive iovec mapping\n"); + memset(mapped, 0x81, 6 * ps); + vectors[0] = (struct iovec){ mapped + 13, 5003 }; + vectors[1] = (struct iovec){ NULL, 0 }; + vectors[2] = + (struct iovec){ mapped + 2 * ps + 7, 2222 }; + vectors[3] = + (struct iovec){ mapped + 4 * ps + 5, 1010 }; + fill(p, len); + ok = send(sockets[0], p, 8235, 0) == 8235 && + recvmsg(sockets[1], &message, MSG_WAITALL) == + 8235 && + check(mapped + 13, 5003, 0) && + check(mapped + 2 * ps + 7, 2222, 5003) && + check(mapped + 4 * ps + 5, 1010, 7225); + ok &= mapped[12] == 0x81 && mapped[5016] == 0x81 && + mapped[2 * ps + 6] == 0x81 && + mapped[2 * ps + 2229] == 0x81 && + mapped[4 * ps + 4] == 0x81 && + mapped[4 * ps + 1015] == 0x81; + snprintf( + name, sizeof(name), + "TLS 1.%u scattered receive, empty iovec and subpage boundaries", + v + 2); + report(ok, ps, name); + munmap(mapped, 6 * ps); + } + close(sockets[0]); + close(sockets[1]); + ok = true; + for (unsigned int round = 0; round < 64 && ok; round++) { + if (!tls_pair(sockets, version)) + ksft_exit_fail_msg( + "TLS teardown setup errno=%d\n", errno); + fill(p, len); + vec = (struct iovec){ p + ps, 4096 }; + ok = vmsplice(fds[1], &vec, 1, 0) == 4096 && + splice(fds[0], NULL, sockets[0], NULL, 4096, + SPLICE_F_MORE) == 4096; + close(sockets[1]); + close(sockets[0]); + ok &= !madvise(p + ps, ps, MADV_DONTNEED); + memset(p + ps, 0x75, ps); + } + snprintf(name, sizeof(name), + "TLS 1.%u repeated pending-record teardown and reuse", + v + 2); + report(ok, ps, name); + } + close(fds[0]); + close(fds[1]); +} + +static int exercise(void) +{ + const size_t ps = getauxval(AT_PAGESZ), len = 8 * ps; + unsigned char *p, *back = malloc(len); + int fds[2], dup[2], file, sockets[2]; + struct iovec v[3]; + struct stat self, init; + struct timeval timeout = { .tv_sec = 5 }; + bool ok; + ssize_t result; + + if (stat("/proc/self/exe", &self) || stat("/proc/1/exe", &init) || + self.st_dev != init.st_dev || self.st_ino != init.st_ino) + return 2; + p = mmap((void *)0x300000000UL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (p == MAP_FAILED || !back || pipe(fds) || pipe(dup)) + ksft_exit_fail_msg("setup\n"); + ksft_print_header(); + ksft_set_plan(use_tcp ? 22 : 10); + ksft_print_msg("transport=%s family=%s\n", + use_tcp ? "TCP" : + use_udp ? "UDP" : + "Unix stream", + use_ipv6 ? "IPv6" : "IPv4/Unix"); + fill(p, len); + v[0] = (struct iovec){ NULL, 0 }; + v[1] = (struct iovec){ p + 17, ps - 17 }; + v[2] = (struct iovec){ p + 2 * ps + 31, 2 * ps - 31 }; + result = vmsplice(fds[1], v, 3, 0); + if (result != (ssize_t)(3 * ps - 48)) + ksft_exit_fail_msg("vmsplice returned %zd errno=%d\n", result, + errno); + ok = read_exact(fds[0], back, result) && check(back, ps - 17, 17) && + check(back + ps - 17, 2 * ps - 31, 2 * ps + 31); + report(ok, ps, "unaligned fragmented iovecs and empty segment"); + v[0] = (struct iovec){ p, 16384 }; + ok = fcntl(fds[1], F_SETPIPE_SZ, 16384) == 16384; + result = vmsplice(fds[1], v, 1, SPLICE_F_NONBLOCK); + { + int pending = 0; + + ok &= result == 16384 && !ioctl(fds[0], FIONREAD, &pending) && + pending == 16384; + } + if (result > 0) + ok &= read_exact(fds[0], back, result) && + check(back, result, 0); + report(ok, ps, + "contiguous quarters retain one-native-page pipe capacity"); + fcntl(fds[1], F_SETPIPE_SZ, 16 * 16384); + v[0] = (struct iovec){ p + 11, 3 * ps - 11 }; + result = vmsplice(fds[1], v, 1, 0); + if (result != (ssize_t)(3 * ps - 11) || + tee(fds[0], dup[1], result, 0) != result) + ksft_exit_fail_msg("tee setup\n"); + ok = replace(p, len); + ok &= read_exact(fds[0], back, 19) && check(back, 19, 11); + ok &= read_exact(fds[0], back + 19, result - 19) && + check(back, result, 11); + ok &= read_exact(dup[0], back, result) && check(back, result, 11); + report(ok, ps, + "tee and partial reads retain data across unmap/replacement"); + fill(p, len); + v[0] = (struct iovec){ p, ps }; + ok = vmsplice(fds[1], v, 1, 0) == (ssize_t)ps && cow(p, ps); + ok &= read_exact(fds[0], back, ps) && check(back, ps, 0); + report(ok, ps, "pipe GET retains pre-COW data after fork sharer exit"); + fill(p, len); + file = memfd_create("splice-target", 0); + v[0] = (struct iovec){ p + ps + 29, 2 * ps - 29 }; + result = vmsplice(fds[1], v, 1, SPLICE_F_GIFT); + ok = file >= 0 && result == (ssize_t)(2 * ps - 29) && + splice(fds[0], NULL, file, NULL, result, SPLICE_F_MOVE) == result; + ok &= pread(file, back, result, 0) == result && + check(back, result, ps + 29); + report(ok, ps, "gifted fragments splice to buffered file"); + close(file); + v[0] = (struct iovec){ p, ps }; + v[1] = (struct iovec){ (void *)0x1000, ps }; + result = vmsplice(fds[1], v, 2, 0); + ok = result == (ssize_t)ps && read_exact(fds[0], back, ps) && + check(back, ps, 0); + report(ok, ps, "partial vmsplice before inaccessible iovec"); + if (connected_pair(sockets)) + ksft_exit_fail_msg("socketpair errno=%d\n", errno); + setsockopt(sockets[1], SOL_SOCKET, SO_RCVTIMEO, &timeout, + sizeof(timeout)); + for (unsigned int mode = 0; mode < 3; mode++) { + const size_t bytes = mode == 1 ? ps : + (use_udp && ps == 4096 ? + 6 * ps - 256 : + 2 * ps - 256); + const size_t offset = mode == 1 ? 0 : ps + 29; + int cork = 1; + + if ((use_tcp || use_udp) && mode < 2 && + setsockopt(sockets[0], use_tcp ? IPPROTO_TCP : IPPROTO_UDP, + use_tcp ? TCP_CORK : UDP_CORK, &cork, + sizeof(cork))) + ksft_exit_fail_msg("TCP_CORK setup errno=%d\n", errno); + fill(p, len); + v[0] = (struct iovec){ p + offset, bytes }; + ok = vmsplice(fds[1], v, 1, 0) == (ssize_t)bytes && + splice(fds[0], NULL, sockets[0], NULL, bytes, + SPLICE_F_NONBLOCK) == (ssize_t)bytes; + if (!ok) + ksft_exit_fail_msg("socket splice setup errno=%d\n", + errno); + if (mode == 2) { + ok = splice(sockets[1], NULL, fds[1], NULL, bytes, 0) == + (ssize_t)bytes && + tee(fds[0], dup[1], bytes, 0) == (ssize_t)bytes; + if (!ok) + ksft_exit_fail_msg( + "socket-to-pipe setup errno=%d\n", + errno); + } + if (use_tcp && mode < 2) { + int unsent = 0; + + if (ioctl(sockets[0], SIOCOUTQNSD, &unsent) || + unsent != (int)bytes) + ksft_exit_fail_msg( + "TCP unsent precondition: %d expected %zu errno=%d\n", + unsent, bytes, errno); + } + if (use_udp && mode < 2) { + errno = 0; + if (recv(sockets[1], back, len, MSG_DONTWAIT) != -1 || + errno != EAGAIN) + ksft_exit_fail_msg( + "UDP corked precondition errno=%d\n", + errno); + } + ok &= mode == 1 ? cow(p, ps) : replace(p + ps, 2 * ps); + if ((use_tcp || use_udp) && mode < 2) { + cork = 0; + if (setsockopt(sockets[0], + use_tcp ? IPPROTO_TCP : IPPROTO_UDP, + use_tcp ? TCP_CORK : UDP_CORK, &cork, + sizeof(cork))) { + ksft_print_msg("uncork errno=%d\n", errno); + ok = false; + } + } + ok &= read_exact(mode == 2 ? fds[0] : sockets[1], back, + bytes) && + check(back, bytes, offset); + if (mode == 2) + ok &= read_exact(dup[0], back, bytes) && + check(back, bytes, offset); + report(ok, ps, + mode == 0 ? + "socket retains quarters across partial unmap/replacement" : + mode == 1 ? + "socket GET retains pre-COW data after fork sharer exit" : + "socket-to-pipe and tee retain independent quarter ownership"); + } + close(sockets[0]); + close(sockets[1]); + ok = true; + for (unsigned int round = 0; round < 128 && ok; round++) { + fill(p, len); + ok = !connected_pair(sockets); + v[0] = (struct iovec){ p + ps, ps }; + ok &= vmsplice(fds[1], v, 1, 0) == (ssize_t)ps && + splice(fds[0], NULL, sockets[0], NULL, ps, 0) == + (ssize_t)ps; + close(sockets[1]); + close(sockets[0]); + ok &= !madvise(p + ps, ps, MADV_DONTNEED); + memset(p + ps, 0x43, ps); + } + report(ok, ps, "repeated unread socket teardown and quarter reuse"); + if (use_tcp) + tls_checks(ps, p, len, back); + close(fds[0]); + close(fds[1]); + close(dup[0]); + close(dup[1]); + munmap(p, len); + free(back); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} +static bool async_cleanup_balanced(void) +{ + const char *names[] = { "user4k_rx_deferred", "user4k_rx_released" }; + unsigned long counts[2]; + char path[128]; + FILE *file = fopen("/sys/module/tls/parameters/user4k_test_async", "r"); + int enabled; + + if (!file) + return false; + enabled = fgetc(file); + fclose(file); + for (unsigned int i = 0; i < 2; i++) { + snprintf(path, sizeof(path), "/sys/module/tls/parameters/%s", + names[i]); + file = fopen(path, "r"); + if (!file) + return false; + if (fscanf(file, "%lu", &counts[i]) != 1) { + fclose(file); + return false; + } + fclose(file); + } + printf("# TLS async=%c deferred=%lu released=%lu\n", enabled, counts[0], + counts[1]); + return counts[0] == counts[1] && (enabled != 'Y' || counts[0] > 0); +} + +int main(int argc, char **argv) +{ + bool ok = true; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && + (!strcmp(argv[1], "--test") || !strcmp(argv[1], "--tcp") || + !strcmp(argv[1], "--tcp6") || !strcmp(argv[1], "--udp") || + !strcmp(argv[1], "--udp6")) && + getppid() == 1) { + use_tcp = !strncmp(argv[1], "--tcp", 5); + use_udp = !strncmp(argv[1], "--udp", 5); + use_ipv6 = strchr(argv[1], '6') != NULL; + return exercise(); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + { + struct ifreq request = { .ifr_name = "lo" }; + int fd = socket(AF_INET, SOCK_DGRAM, 0); + + if (fd < 0 || ioctl(fd, SIOCGIFFLAGS, &request)) + return 4; + request.ifr_flags |= IFF_UP; + if (ioctl(fd, SIOCSIFFLAGS, &request)) + return 4; + close(fd); + } + for (unsigned int mode = 0; mode < 10; mode++) { + unsigned int small = mode & 1; + pid_t pid = fork(); + int status = 0; + + if (!pid) { + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + small ? 4096UL : 0UL, 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", + mode >= 8 ? "--udp6" : + mode >= 6 ? "--udp" : + mode >= 4 ? "--tcp6" : + mode >= 2 ? "--tcp" : + "--test", + NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + ok &= async_cleanup_balanced(); + printf("%s - splice fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/swap-fixture-init.c b/tools/testing/selftests/arm64/user4k/swap-fixture-init.c new file mode 100644 index 0000000000000..45f6aefdd094e --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/swap-fixture-init.c @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PID-1 runner for a disposable VM with a dedicated virtio swap disk. + * Use kunit.autorun=0: tests skipped at boot stay skipped on a debugfs rerun. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned long swapouts(void) +{ + FILE *f = fopen("/proc/vmstat", "re"); + char key[128]; + unsigned long value, found = 0; + + if (!f) + return 0; + while (fscanf(f, "%127s %lu", key, &value) == 2) + if (!strcmp(key, "pswpout")) + found = value; + fclose(f); + return found; +} + +static int native_swap_smoke(void) +{ + const size_t length = 4 * 1024 * 1024, page = 16384; + unsigned long before = swapouts(), after; + unsigned char *p = mmap(NULL, length, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + size_t i; + unsigned int tries; + int ret = 0; + + if (p == MAP_FAILED) + return -1; + for (i = 0; i < length; i++) + p[i] = (i ^ (i >> 12) ^ (i >> 20)) & 255; + if (madvise(p, length, MADV_PAGEOUT)) { + ret = -1; + goto out; + } + for (tries = 0; tries < 100 && swapouts() == before; tries++) + usleep(20000); + after = swapouts(); + for (i = 0; i < length; i++) + if (p[i] != ((i ^ (i >> 12) ^ (i >> 20)) & 255)) { + ret = -1; + break; + } + if (after <= before) + ret = -1; + printf("%s - native %zuK swap transport: pswpout %lu -> %lu, verified %zu bytes\n", + ret ? "not ok" : "ok", page / 1024, before, after, i); +out: + munmap(p, length); + return ret; +} + +static int run_swap_suite(void) +{ + const char *base = "/sys/kernel/debug/kunit/arm64-user4k-swap/"; + char path[128], *log; + size_t used = 0, capacity = 1024 * 1024; + ssize_t n; + int fd, ret = -1; + + snprintf(path, sizeof(path), "%srun", base); + fd = open(path, O_WRONLY | O_CLOEXEC); + if (fd < 0) + return -1; + n = write(fd, "1", 1); + close(fd); + if (n != 1) + return -1; + snprintf(path, sizeof(path), "%sresults", base); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + log = calloc(1, capacity); + if (!log) { + close(fd); + return -1; + } + while (used < capacity - 1 && + (n = read(fd, log + used, capacity - 1 - used)) > 0) + used += n; + close(fd); + fputs(log, stdout); + if (n == 0 && used < capacity - 1 && + strstr(log, "\nok 1 arm64-user4k-swap\n") && + !strstr(log, "not ok") && !strstr(log, "# SKIP")) + ret = 0; + free(log); + return ret; +} + +static int swap_control_helper(const char *operation) +{ + struct stat init, self; + int ret; + + /* A helper is allowed only when this same executable is the VM's PID 1. */ + if (getpid() == 1 || stat("/proc/1/exe", &init) || + stat("/proc/self/exe", &self) || init.st_dev != self.st_dev || + init.st_ino != self.st_ino) { + fprintf(stderr, + "swap helper requires its fixture runner as PID 1\n"); + return 1; + } + ret = !strcmp(operation, "--swapoff-helper") ? swapoff("/dev/vda") : + swapon("/dev/vda", 0); + if (ret) + perror(operation); + return ret ? 1 : 0; +} + +static int pageout_exec(bool small) +{ + pid_t pid = fork(); + int status = 0; + + if (!pid) { + if (small && + prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL)) + _exit(125); + execl("/reclaim-advice", "/reclaim-advice", + small ? "--expect-4k" : NULL, NULL); + _exit(126); + } + if (pid <= 0 || waitpid(pid, &status, 0) != pid) + return -1; + printf("# EL0 pageout %s child status=%#x\n", small ? "4K" : "native", + status); + return WIFEXITED(status) && !WEXITSTATUS(status) ? 0 : -1; +} + +int main(int argc, char **argv) +{ + bool smoke = argc == 2 && !strcmp(argv[1], "--smoke"); + bool pageout = argc == 2 && !strcmp(argv[1], "--pageout"); + bool native_only = argc == 2 && !strcmp(argv[1], "--pageout-native"); + bool active = false; + unsigned int tries; + struct stat st; + pid_t child; + int status, ret = 1; + + if (argc == 2 && (!strcmp(argv[1], "--swapoff-helper") || + !strcmp(argv[1], "--swapon-helper"))) + return swap_control_helper(argv[1]); + /* This runner must never mount filesystems or enable swap on the host. */ + if (getpid() != 1 || + (argc != 1 && !smoke && !pageout && !native_only)) { + fprintf(stderr, + "Use only as /init in the dedicated disposable test VM.\n"); + return 1; + } + setvbuf(stdout, NULL, _IONBF, 0); + if (getauxval(AT_PAGESZ) != 16384 || sysconf(_SC_PAGESIZE) != 16384) + goto out; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) + goto out; + child = fork(); + if (!child) { + execl("/page-contract", "page-contract", NULL); + _exit(127); + } + if (child < 0 || waitpid(child, &status, 0) != child || + !WIFEXITED(status) || WEXITSTATUS(status)) + goto out; + for (tries = 0; tries < 100 && stat("/dev/vda", &st); tries++) + usleep(20000); + if (stat("/dev/vda", &st) || !S_ISBLK(st.st_mode) || + swapon("/dev/vda", 0)) + goto out; + active = true; + if (native_swap_smoke()) + goto out; + if (!smoke && !native_only && run_swap_suite()) + goto out; + if ((pageout || native_only) && pageout_exec(false)) + goto out; + if (pageout && pageout_exec(true)) + goto out; + ret = 0; +out: + if (ret) + printf("not ok - swap fixture runner: errno=%d (%s)\n", errno, + strerror(errno)); + if (active && swapoff("/dev/vda")) { + perror("swapoff"); + ret = 1; + } + printf("%s - swap fixture runner (%s)\n", ret ? "not ok" : "ok", + smoke ? "native transport smoke only" : "4K swap suite"); + fflush(NULL); + reboot(RB_POWER_OFF); + return ret; +} diff --git a/tools/testing/selftests/arm64/user4k/swapfile-fixture.h b/tools/testing/selftests/arm64/user4k/swapfile-fixture.h new file mode 100644 index 0000000000000..258b81fa05695 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/swapfile-fixture.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef USER4K_SWAPFILE_FIXTURE_H +#define USER4K_SWAPFILE_FIXTURE_H + +static const char *swap_path(void) +{ + return access("/swapdir/swapfile", F_OK) ? "/dev/vda" : + "/swapdir/swapfile"; +} + +/* Remove every fourth native page: every 64K group crosses file extents. */ +static int prepare_swapfile(void) +{ + const unsigned long page = getauxval(AT_PAGESZ), initial = 32UL << 20; + unsigned char *buf = calloc(1, page); + unsigned long offset; + int fd = -1, ret = -1; + + if (!buf || (mkdir("/swapdir", 0755) && errno != EEXIST) || + mount("/dev/vda", "/swapdir", "ext4", 0, NULL)) + goto out; + fd = open("/swapdir/swapfile", O_CREAT | O_EXCL | O_RDWR | O_CLOEXEC, + 0600); + if (fd < 0) + goto out; + memset(buf, 0xa5, page); + for (offset = 0; offset < initial; offset += page) + if (write(fd, buf, page) != (ssize_t)page) + goto out; + if (fsync(fd)) + goto out; + for (offset = initial; offset; offset -= 4 * page) + if (fallocate(fd, FALLOC_FL_COLLAPSE_RANGE, offset - 2 * page, + page)) + goto out; + memset(buf, 0, page); + ((unsigned int *)(buf + 1024))[0] = 1; + ((unsigned int *)(buf + 1024))[1] = (initial * 3 / 4) / page - 1; + memcpy(buf + page - 10, "SWAPSPACE2", 10); + if (pwrite(fd, buf, page, 0) != (ssize_t)page || fsync(fd)) + goto out; + ret = 0; +out: + if (fd >= 0) + close(fd); + free(buf); + return ret; +} + +#endif diff --git a/tools/testing/selftests/arm64/user4k/sysv-shm-init.c b/tools/testing/selftests/arm64/user4k/sysv-shm-init.c new file mode 100644 index 0000000000000..3b66d9c738fde --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/sysv-shm-init.c @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable PID1: SysV shared memory with per-process page granules. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int ok, const char *expression, int line) +{ + if (!ok) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expression, + errno); + exit(1); + } +} + +#define CHECK(x) check(!!(x), #x, __LINE__) + +static int unmapped(void *address, size_t page) +{ + unsigned char resident; + + errno = 0; + return mincore(address, page, &resident) == -1 && errno == ENOMEM; +} + +static int mapping_case(unsigned long native, int rounded, int split) +{ + unsigned long page = getauxval(AT_PAGESZ), length = 4 * page; + unsigned char *reserve, *target, *mapping, *alias; + struct shmid_ds info; + int id; + + reserve = mmap(NULL, 16 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + target = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + target += native + (page < native ? page : 0); + target[-1] = 0x51; + target[length] = 0x72; + CHECK(!munmap(target, length)); + id = shmget(IPC_PRIVATE, length - 1, 0600); + CHECK(id >= 0); + mapping = shmat(id, target + (rounded ? page - 1 : 0), + rounded ? SHM_RND : 0); + if (mapping != target) { + int saved = errno; + + shmctl(id, IPC_RMID, NULL); + errno = saved; + CHECK(mapping == target); + } + alias = shmat(id, NULL, 0); + CHECK(alias != (void *)-1); + CHECK(!shmctl(id, IPC_RMID, NULL)); + mapping[0] = 0x31; + mapping[page + 7] = 0x49; + mapping[3 * page] = 0x63; + CHECK(alias[0] == 0x31 && alias[page + 7] == 0x49 && + alias[3 * page] == 0x63); + if (split) { + CHECK(!mprotect(mapping + page, page, PROT_READ)); + CHECK(!munmap(mapping + 2 * page, page)); + if (split == 2) + CHECK(!munmap(mapping, page)); + } + CHECK(!shmdt(mapping)); + for (unsigned long i = 0; i < length; i += page) + CHECK(unmapped(target + i, page)); + CHECK(target[-1] == 0x51 && target[length] == 0x72); + CHECK(alias[page + 7] == 0x49 && alias[3 * page] == 0x63); + CHECK(!shmdt(alias)); + errno = 0; + CHECK(shmctl(id, IPC_STAT, &info) == -1 && + (errno == EINVAL || errno == EIDRM)); + CHECK(!munmap(reserve, 16 * native)); + printf("ok - SysV page=%lu rounded=%d split=%d detach and alias lifetime\n", + page, rounded, split); + return 0; +} + +static int wait_child(pid_t pid) +{ + int status; + pid_t got; + + if (pid < 0) + return 1; + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + return got != pid || !WIFEXITED(status) || WEXITSTATUS(status); +} + +static int sharing_child(int id, unsigned long expected) +{ + unsigned char *map; + + CHECK(getauxval(AT_PAGESZ) == expected); + map = shmat(id, NULL, 0); + CHECK(map != (void *)-1); + CHECK(map[37] == 0x35 && map[4096 + 9] == 0x56); + map[4096 + 9] = 0x78; + CHECK(!shmdt(map)); + return 0; +} + +static int sharing(unsigned long other, unsigned long native) +{ + unsigned long page = getauxval(AT_PAGESZ); + unsigned char *map; + pid_t pid; + int id; + + id = shmget(IPC_PRIVATE, 2 * native, 0600); + CHECK(id >= 0); + map = shmat(id, NULL, 0); + CHECK(map != (void *)-1); + CHECK(!shmctl(id, IPC_RMID, NULL)); + map[37] = 0x35; + map[4096 + 9] = 0x56; + pid = fork(); + if (!pid) { + char idarg[32], sizearg[32]; + + snprintf(idarg, sizeof(idarg), "%d", id); + snprintf(sizearg, sizeof(sizearg), "%lu", other); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, other, 0UL, 0UL, 0UL)) + _exit(2); + execl("/init", "/init", "--share", idarg, sizearg, NULL); + _exit(3); + } + CHECK(!wait_child(pid)); + CHECK(map[4096 + 9] == 0x78); + CHECK(!shmdt(map)); + printf("ok - SysV shared object %lu -> %lu after IPC_RMID\n", page, + other); + return 0; +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--share")) + return sharing_child(atoi(argv[2]), strtoul(argv[3], NULL, 10)); + if (argc == 5 && !strcmp(argv[1], "--case")) { + unsigned long expected = strtoul(argv[3], NULL, 10); + unsigned long sizes[] = { 4096, 16384, 65536 }; + + native = strtoul(argv[2], NULL, 10); + CHECK(getauxval(AT_PAGESZ) == expected); + failed |= mapping_case(native, 0, 0); + failed |= mapping_case(native, 1, 0); + failed |= mapping_case(native, 0, 1); + failed |= mapping_case(native, 0, 2); + for (unsigned int i = 0; i < 3; i++) + if (sizes[i] <= native && sizes[i] != expected && + !(atoi(argv[4]) && sizes[i] == 4096)) + failed |= sharing(sizes[i], native); + return failed; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + int no4k = argc == 2 && !strcmp(argv[1], "--no4k"); + unsigned long sizes[] = { 4096, 16384, 65536 }; + + for (unsigned int i = 0; i < 3; i++) { + unsigned long page = sizes[i]; + pid_t pid; + + if (page > native || (no4k && page == 4096)) + continue; + pid = fork(); + if (!pid) { + char n[32], e[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(e, sizeof(e), "%lu", page); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, page, 0UL, 0UL, + 0UL)) + _exit(2); + execl("/init", "/init", "--case", n, e, + no4k ? "1" : "0", NULL); + _exit(3); + } + int bad = wait_child(pid); + + printf("%s - SysV AArch64 native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, page); + failed |= bad; + } + if (access("/compat-shm", X_OK) == 0) { + pid_t pid = fork(); + + if (!pid) { + char n[32], e[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(e, sizeof(e), "%lu", no4k ? native : 4096UL); + execl("/compat-shm", "/compat-shm", n, e, NULL); + _exit(3); + } + int bad = wait_child(pid); + + printf("%s - SysV AArch32 native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, no4k ? native : 4096UL); + failed |= bad; + } + printf("SYSV SHM GRANULE %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/thread-fixture-init.c b/tools/testing/selftests/arm64/user4k/thread-fixture-init.c new file mode 100644 index 0000000000000..c472ea6697947 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/thread-fixture-init.c @@ -0,0 +1,424 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +#define THREADS 4 +#define ROUNDS 128 +#define PAGES 32 + +static pthread_barrier_t barrier; +static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t condition = PTHREAD_COND_INITIALIZER; +static pthread_mutex_t robust; +static _Thread_local unsigned int tls_id; +static _Thread_local volatile sig_atomic_t signals; +static atomic_uint failures; +static unsigned long total; +static unsigned long ps; +static atomic_int waiter_tid; + +static void handler(int sig) +{ + if (sig == SIGUSR1 && tls_id) + signals++; + else + atomic_fetch_add(&failures, 1); +} + +static bool bytes_equal(unsigned char *p, size_t n, unsigned char value) +{ + for (size_t i = 0; i < n; i++) + if (p[i] != value) + return false; + return true; +} + +static void *worker(void *arg) +{ + unsigned int id = (uintptr_t)arg; + unsigned char *p, *heap; + unsigned long request = id & 1 ? 4096 : 0; + bool ok = true; + + tls_id = id; + p = mmap(NULL, PAGES * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (p == MAP_FAILED) + ok = false; + ok &= !prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, request, 0UL, 0UL, 0UL); + pthread_barrier_wait(&barrier); + for (unsigned int round = 0; round < ROUNDS; round++) { + unsigned char pattern = 1 + (id * 31 + round) % 254; + unsigned char residency[PAGES]; + cpu_set_t cpu; + + if (ok) { + CPU_ZERO(&cpu); + CPU_SET((id + round) % 2, &cpu); + ok &= !sched_setaffinity(0, sizeof(cpu), &cpu); + memset(p, pattern, PAGES * ps); + ok &= !mprotect(p + ps, ps, PROT_READ) && + p[ps] == pattern; + ok &= !mprotect(p + ps, ps, PROT_READ | PROT_WRITE); + ok &= !madvise(p + 2 * ps, ps, MADV_FREE); + memset(p + 2 * ps, pattern, ps); + ok &= !madvise(p + 3 * ps, ps, MADV_DONTNEED); + ok &= bytes_equal(p + 3 * ps, ps, 0); + memset(p + 3 * ps, pattern, ps); + heap = malloc(5000 + id * 777); + if (!heap) { + ok = false; + } else { + memset(heap, pattern, 5000 + id * 777); + ok &= bytes_equal(heap, 5000 + id * 777, + pattern); + free(heap); + } + ok &= !pthread_kill(pthread_self(), SIGUSR1); + ok &= tls_id == id && + signals == (sig_atomic_t)(round + 1); + ok &= prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, + 0UL) == (int)request; + ok &= !mincore(p, PAGES * ps, residency); + ok &= bytes_equal(p, PAGES * ps, pattern); + } + pthread_mutex_lock(&mutex); + total++; + pthread_cond_signal(&condition); + pthread_mutex_unlock(&mutex); + pthread_barrier_wait(&barrier); + } + if (p != MAP_FAILED) + munmap(p, PAGES * ps); + if (!ok) + atomic_fetch_add(&failures, 1); + return NULL; +} + +static void *owner_dies(void *arg) +{ + (void)arg; + if (pthread_mutex_lock(&robust)) + atomic_fetch_add(&failures, 1); + return NULL; +} + +static void *pi_waiter(void *arg) +{ + int error; + + (void)arg; + atomic_store(&waiter_tid, syscall(SYS_gettid)); + error = pthread_mutex_lock(&robust); + if (!error) + error = pthread_mutex_unlock(&robust); + return (void *)(uintptr_t)error; +} + +static bool wait_for_pi_block(void) +{ + for (unsigned int tries = 0; tries < 1000; tries++) { + int tid = atomic_load(&waiter_tid); + char path[80], line[256]; + unsigned long nr, address, operation; + FILE *f; + + if (tid) { + snprintf(path, sizeof(path), + "/proc/self/task/%d/syscall", tid); + f = fopen(path, "re"); + if (f) { + bool blocked = fgets(line, sizeof(line), f) && + sscanf(line, "%lu %lx %lx", &nr, + &address, + &operation) == 3 && + nr == SYS_futex && + ((operation & FUTEX_CMD_MASK) == + FUTEX_LOCK_PI || + (operation & FUTEX_CMD_MASK) == + FUTEX_LOCK_PI2); + + fclose(f); + if (blocked) + return true; + } + } + usleep(1000); + } + return false; +} + +static int exercise(void) +{ + pthread_t threads[THREADS], owner; + pthread_mutexattr_t attributes; + struct sigaction action = { .sa_handler = handler }; + struct stat self, init; + bool ok = true, forks = true; + int error; + + if (stat("/proc/self/exe", &self) || stat("/proc/1/exe", &init) || + self.st_dev != init.st_dev || self.st_ino != init.st_ino) + return 2; + ps = getauxval(AT_PAGESZ); + ksft_print_header(); + ksft_set_plan(6); + if (sigaction(SIGUSR1, &action, NULL) || + pthread_barrier_init(&barrier, NULL, THREADS + 1)) + ksft_exit_fail_msg("thread setup\n"); + for (unsigned int i = 0; i < THREADS; i++) + if (pthread_create(&threads[i], NULL, worker, + (void *)(uintptr_t)(i + 1))) + ksft_exit_fail_msg("pthread_create\n"); + pthread_barrier_wait(&barrier); + for (unsigned int round = 0; round < ROUNDS; round++) { + pthread_mutex_lock(&mutex); + while (total < (round + 1) * THREADS) + pthread_cond_wait(&condition, &mutex); + pthread_mutex_unlock(&mutex); + ok &= prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL) == + 0; + if (!(round % 16)) { + pid_t child = fork(); + int status = 0; + + if (!child) { + unsigned char *p = malloc(32768); + + if (!p || tls_id || + total != (round + 1) * THREADS || + prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, + 0UL, 0UL, 0UL)) + _exit(10); + memset(p, 0x75, 32768); + if (!bytes_equal(p, 32768, 0x75)) + _exit(11); + free(p); + _exit(0); + } + forks &= child > 0 && + waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + } + pthread_barrier_wait(&barrier); + } + for (unsigned int i = 0; i < THREADS; i++) + ok &= !pthread_join(threads[i], NULL); + ksft_test_result( + ok && !atomic_load(&failures) && total == THREADS * ROUNDS, + "%luK %u concurrent memory/TLS/signal/exec-request rounds across %u threads\n", + ps / 1024, ROUNDS, THREADS); + ksft_test_result( + forks, "%luK fork and allocator from a multithreaded process\n", + ps / 1024); + pthread_barrier_destroy(&barrier); + ok = !pthread_mutexattr_init(&attributes); + ok &= !pthread_mutexattr_setrobust(&attributes, PTHREAD_MUTEX_ROBUST); + ok &= !pthread_mutex_init(&robust, &attributes); + ok &= !pthread_create(&owner, NULL, owner_dies, NULL); + if (!ok) + ksft_exit_fail_msg("robust mutex setup\n"); + ok &= !pthread_join(owner, NULL); + error = pthread_mutex_lock(&robust); + ok &= error == EOWNERDEAD; + if (error == EOWNERDEAD) + ok &= !pthread_mutex_consistent(&robust); + if (!error || error == EOWNERDEAD) + ok &= !pthread_mutex_unlock(&robust); + ksft_test_result(ok, "%luK robust futex owner-death recovery\n", + ps / 1024); + pthread_mutex_destroy(&robust); + /* Exercise repeated TLS/stack allocation and set_tid_address clear/wake. */ + for (unsigned int i = 0; i < 128; i++) { + ok = !pthread_mutex_init(&robust, &attributes); + ok &= !pthread_create(&owner, NULL, owner_dies, NULL); + if (!ok) + break; + ok &= !pthread_join(owner, NULL); + error = pthread_mutex_lock(&robust); + ok &= error == EOWNERDEAD; + if (error == EOWNERDEAD) + ok &= !pthread_mutex_consistent(&robust); + if (!error || error == EOWNERDEAD) + pthread_mutex_unlock(&robust); + pthread_mutex_destroy(&robust); + if (!ok) + break; + } + ksft_test_result(ok && !atomic_load(&failures), + "%luK repeated thread exit/join and robust recovery\n", + ps / 1024); + ok = !pthread_mutexattr_setprotocol(&attributes, PTHREAD_PRIO_INHERIT); + ok &= !pthread_mutex_init(&robust, &attributes); + ok &= !pthread_mutex_lock(&robust); + ok &= !pthread_create(&owner, NULL, pi_waiter, NULL); + if (!ok) + ksft_exit_fail_msg("PI mutex setup\n"); + ok &= wait_for_pi_block(); + ok &= !pthread_mutex_unlock(&robust); + { + void *result = (void *)1; + + ok &= !pthread_join(owner, &result) && !result; + } + ksft_test_result( + ok, + "%luK confirmed contended priority-inheritance futex handoff\n", + ps / 1024); + ok = !pthread_create(&owner, NULL, owner_dies, NULL); + if (!ok) + ksft_exit_fail_msg("PI owner setup\n"); + ok &= !pthread_join(owner, NULL); + error = pthread_mutex_lock(&robust); + ok &= error == EOWNERDEAD; + if (error == EOWNERDEAD) + ok &= !pthread_mutex_consistent(&robust); + if (!error || error == EOWNERDEAD) + ok &= !pthread_mutex_unlock(&robust); + pthread_mutex_destroy(&robust); + pthread_mutexattr_destroy(&attributes); + ksft_test_result( + ok, "%luK priority-inheritance robust owner-death recovery\n", + ps / 1024); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} + +static pthread_barrier_t exec_barrier; + +static void *exec_sibling(void *arg) +{ + (void)arg; + pthread_barrier_wait(&exec_barrier); + for (;;) + pause(); + return NULL; +} + +static void *exec_nonleader(void *arg) +{ + unsigned long target = (uintptr_t)arg; + char size[32]; + + snprintf(size, sizeof(size), "%lu", target); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, target, 0UL, 0UL, 0UL)) + _exit(30); + pthread_barrier_wait(&exec_barrier); + execl("/init", "/init", "--after-thread-exec", size, NULL); + _exit(31); +} + +static int thread_exec(unsigned long target) +{ + pthread_t siblings[2], executor; + + if (pthread_barrier_init(&exec_barrier, NULL, 4)) + return 32; + for (unsigned int i = 0; i < 2; i++) + if (pthread_create(&siblings[i], NULL, exec_sibling, NULL)) + return 33; + if (pthread_create(&executor, NULL, exec_nonleader, + (void *)(uintptr_t)target)) + return 34; + pthread_barrier_wait(&exec_barrier); + for (;;) + pause(); + return 35; +} + +int main(int argc, char **argv) +{ + bool ok = true; + unsigned long granules[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + unsigned int nr = granules[0] > 16384 ? 3 : 2; + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--thread-exec") && getppid() == 1) + return thread_exec(strtoul(argv[2], NULL, 10)); + if (argc == 3 && !strcmp(argv[1], "--after-thread-exec") && + getppid() == 1) { + unsigned long expected = strtoul(argv[2], NULL, 10); + + return getauxval(AT_PAGESZ) != expected || + prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, + 0UL) != 0; + } + if (argc == 3 && !strcmp(argv[1], "--test") && getppid() == 1) { + if (getauxval(AT_PAGESZ) != strtoul(argv[2], NULL, 10)) + return 42; + return exercise(); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL)) + return 3; + for (unsigned int small = 0; small < nr; small++) { + pid_t pid = fork(); + int status = 0; + + if (!pid) { + char size[32]; + + snprintf(size, sizeof(size), "%lu", granules[small]); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, granules[small], + 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--test", size, NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + for (unsigned int source = 0; source < nr; source++) { + for (unsigned int target = 0; target < nr; target++) { + pid_t pid = fork(); + int status = 0; + bool passed; + + if (!pid) { + char size[32]; + + snprintf(size, sizeof(size), "%lu", + granules[target]); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + granules[source], 0UL, 0UL, 0UL)) + _exit(36); + execl("/init", "/init", "--thread-exec", size, + NULL); + _exit(37); + } + passed = pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - non-leader exec %luK -> %luK status=%#x\n", + passed ? "ok" : "not ok", + granules[source] / 1024, granules[target] / 1024, + status); + ok &= passed; + } + } + printf("%s - thread fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/uffd-move-init.c b/tools/testing/selftests/arm64/user4k/uffd-move-init.c new file mode 100644 index 0000000000000..f2649698488b6 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/uffd-move-init.c @@ -0,0 +1,451 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - MOVE line=%d errno=%d (%s)\n", \ + __LINE__, errno, strerror(errno)); \ + exit(1); \ + } \ + } while (0) +#define PRESENT (1ULL << 63) +#define SWAPPED (1ULL << 62) +#define VALUE_MASK ((1ULL << 55) - 1) +static uint64_t entry(void *p, size_t ps) +{ + uint64_t e; + int fd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + CHECK(fd >= 0 && pread(fd, &e, 8, (uintptr_t)p / ps * 8) == 8); + CHECK(!close(fd)); + return e; +} +static uint64_t physical(void *p, size_t ps) +{ + uint64_t e = entry(p, ps); + CHECK(e & PRESENT); + return (e & VALUE_MASK) * ps; +} +static unsigned char pattern(size_t i) +{ + return 1 + ((i ^ (i >> 12) ^ (i >> 16)) % 251); +} +static void bytes(unsigned char *p, size_t original, size_t len) +{ + for (size_t i = 0; i < len; i++) + CHECK(p[i] == pattern(original + i)); +} +static long move(int fd, void *dst, void *src, size_t len, unsigned mode, + int expected_errno) +{ + struct uffdio_move m = { .dst = (uintptr_t)dst, + .src = (uintptr_t)src, + .len = len, + .mode = mode }; + errno = 0; + int r = ioctl(fd, UFFDIO_MOVE, &m); + if (expected_errno) + CHECK(r == -1 && errno == expected_errno); + else + CHECK(r == 0 && m.move == (long long)len); + return m.move; +} +static void on_cpu(unsigned cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} +struct tlb_reader { + unsigned char *src, *dst; + atomic_int ready, done; +}; +static void *read_after_move(void *arg) +{ + struct tlb_reader *r = arg; + on_cpu(1); + unsigned char old = *(volatile unsigned char *)r->src; + atomic_store(&r->ready, 1); + while (!atomic_load(&r->done)) + sched_yield(); + CHECK(*(volatile unsigned char *)r->src == 0 && + *(volatile unsigned char *)r->dst == old); + return NULL; +} +struct pin_racer { + void *a, *b; + size_t ps; + atomic_int stop; + atomic_uint accepted, rejected; +}; +static void *race_pins(void *arg) +{ + struct pin_racer *r = arg; + struct io_uring_params p = { 0 }; + on_cpu(1); + int fd = syscall(SYS_io_uring_setup, 2, &p); + CHECK(fd >= 0); + unsigned round = 0; + while (!atomic_load(&r->stop)) { + struct iovec iov = { .iov_base = (round++ & 1) ? r->a : r->b, + .iov_len = r->ps }; + int ret = syscall(SYS_io_uring_register, fd, + IORING_REGISTER_BUFFERS, &iov, 1); + if (!ret) { + atomic_fetch_add(&r->accepted, 1); + usleep(50); + CHECK(!syscall(SYS_io_uring_register, fd, + IORING_UNREGISTER_BUFFERS, NULL, 0)); + } else { + CHECK(errno == EFAULT || errno == EAGAIN); + atomic_fetch_add(&r->rejected, 1); + } + } + CHECK(!close(fd)); + return NULL; +} +static void concurrent_move(size_t ps) +{ + unsigned char *a = + mmap((void *)0x50000000, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + unsigned char *b = + mmap((void *)0x60000000, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(a != MAP_FAILED && b != MAP_FAILED); + memset(a, 0x6d, ps); + uint64_t phys = physical(a, ps); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MOVE }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)a, ps }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + reg.range.start = (uintptr_t)b; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + struct pin_racer race = { .a = a, .b = b, .ps = ps }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, race_pins, &race)); + while (!atomic_load(&race.accepted)) + sched_yield(); + unsigned char *cur = a, *next = b; + unsigned moved = 0, busy = 0; + for (unsigned tries = 0; moved < 256 && tries < 100000; tries++) { + struct uffdio_move m = { .src = (uintptr_t)cur, + .dst = (uintptr_t)next, + .len = ps }; + int ret = ioctl(fd, UFFDIO_MOVE, &m); + if (ret) { + CHECK(errno == EBUSY && m.move == -EBUSY); + busy++; + sched_yield(); + continue; + } + CHECK(m.move == (long long)ps && physical(next, ps) == phys && + !(entry(cur, ps) & (PRESENT | SWAPPED))); + unsigned char *tmp = cur; + cur = next; + next = tmp; + moved++; + } + atomic_store(&race.stop, 1); + CHECK(!pthread_join(thread, NULL)); + CHECK(moved == 256 && atomic_load(&race.accepted) > 0); + for (size_t i = 0; i < ps; i++) + CHECK(cur[i] == 0x6d); + printf("ok - MOVE %zuK CPU0/CPU1 race moves=%u busy=%u pins=%u holes=%u unchanged PFN/data\n", + ps / 1024, moved, busy, atomic_load(&race.accepted), + atomic_load(&race.rejected)); + CHECK(!close(fd) && !munmap(a, ps) && !munmap(b, ps)); +} + +static void allocation_failure(size_t ps, size_t native) +{ + if (ps == native) + return; + unsigned char *a = + mmap((void *)0x70000000, 2 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + unsigned char *b = + mmap((void *)0x71000000, 2 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(a != MAP_FAILED && b != MAP_FAILED); + memset(a, 0x35, 2 * native); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MOVE }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)a, 2 * native }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + reg.range.start = (uintptr_t)b; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + /* Prepare both roots and target tables using a different native folio. */ + move(fd, b + native, a + native, ps, 0, 0); + uint64_t phys = physical(a, ps); + int inject = open("/proc/self/fail-nth", O_RDWR | O_CLOEXEC); + CHECK(inject >= 0); + unsigned char *src = a, *dst = b; + for (unsigned phase = 0; phase < 2; phase++) { + struct uffdio_move m = { .src = (uintptr_t)src, + .dst = (uintptr_t)dst, + .len = ps }; + char pending[16] = { 0 }; + CHECK(write(inject, "1", 1) == 1); + errno = 0; + int ret = ioctl(fd, UFFDIO_MOVE, &m), saved = errno; + CHECK(pread(inject, pending, sizeof(pending) - 1, 0) > 0); + CHECK(write(inject, "0", 1) == 1); + CHECK(ret == -1 && saved == ENOMEM && m.move == -ENOMEM && + strtoul(pending, NULL, 10) == 0); + CHECK(physical(src, ps) == phys && + !(entry(dst, ps) & (PRESENT | SWAPPED))); + for (size_t i = 0; i < ps; i++) + CHECK(src[i] == 0x35); + move(fd, dst, src, ps, 0, 0); + CHECK(physical(dst, ps) == phys); + unsigned char *tmp = src; + src = dst; + dst = tmp; + } + printf("ok - MOVE %zuK sidecar/retirement allocation failures preserve PTEs and retry\n", + ps / 1024); + CHECK(!close(inject) && !close(fd) && !munmap(a, 2 * native) && + !munmap(b, 2 * native)); +} + +static int exercise(size_t native) +{ + size_t ps = getauxval(AT_PAGESZ), len = 16 * native; + on_cpu(0); + unsigned char *src = + mmap((void *)0x20000000, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + unsigned char *dst = + mmap((void *)0x30000000, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(src != MAP_FAILED && dst != MAP_FAILED); + for (size_t i = 0; i < len; i++) + src[i] = pattern(i); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MOVE }; + CHECK(!ioctl(fd, UFFDIO_API, &api) && + (api.features & UFFD_FEATURE_MOVE)); + struct uffdio_register reg = { .range = { (uintptr_t)dst, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®) && + (reg.ioctls & (1ULL << _UFFDIO_MOVE))); + uint64_t pfns[256]; + CHECK(native / ps <= sizeof(pfns) / sizeof(pfns[0])); + for (size_t i = 0; i < native / ps; i++) + pfns[i] = physical(src + i * ps, ps); + uint64_t p = physical(src + 3 * ps, ps); + struct tlb_reader reader = { .src = src + 3 * ps, .dst = dst + ps }; + pthread_t tlb_thread; + CHECK(!pthread_create(&tlb_thread, NULL, read_after_move, &reader)); + while (!atomic_load(&reader.ready)) + sched_yield(); + move(fd, dst + ps, src + 3 * ps, ps, 0, 0); + CHECK(physical(dst + ps, ps) == p && + !(entry(src + 3 * ps, ps) & (PRESENT | SWAPPED))); + atomic_store(&reader.done, 1); + CHECK(!pthread_join(tlb_thread, NULL)); + bytes(dst + ps, 3 * ps, ps); + for (size_t i = 0; i < native / ps; i++) + if (i != 3) { + CHECK(physical(src + i * ps, ps) == pfns[i]); + bytes(src + i * ps, i * ps, ps); + } + CHECK(src[3 * ps] == + 0); /* Source has an independent zero-fault after MOVE. */ + printf("ok - MOVE %zuK exact PFN, divergent offsets, neighbour PFNs/data and source TLB\n", + ps / 1024); + + struct io_uring_params ur = { 0 }; + int ring = syscall(SYS_io_uring_setup, 2, &ur); + CHECK(ring >= 0); + struct iovec pin = { .iov_base = src + 2 * ps, .iov_len = ps }; + CHECK(!syscall(SYS_io_uring_register, ring, IORING_REGISTER_BUFFERS, + &pin, 1)); + p = physical(src + 2 * ps, ps); + CHECK(move(fd, dst + 2 * ps, src + 2 * ps, ps, 0, EBUSY) == -EBUSY); + CHECK(physical(src + 2 * ps, ps) == p && + !(entry(dst + 2 * ps, ps) & (PRESENT | SWAPPED))); + bytes(src + 2 * ps, 2 * ps, ps); + uint64_t neighbour = physical(src + ps, ps); + move(fd, dst + 3 * ps, src + ps, ps, 0, 0); + CHECK(physical(dst + 3 * ps, ps) == neighbour); + bytes(dst + 3 * ps, ps, ps); + CHECK(!syscall(SYS_io_uring_register, ring, IORING_UNREGISTER_BUFFERS, + NULL, 0)); + CHECK(!close(ring)); + move(fd, dst + 2 * ps, src + 2 * ps, ps, 0, 0); + CHECK(physical(dst + 2 * ps, ps) == p); + bytes(dst + 2 * ps, 2 * ps, ps); + printf("ok - MOVE %zuK source PIN rejected atomically, neighbour PIN allowed, unpin retry\n", + ps / 1024); + + CHECK(move(fd, dst + ps, src + 4 * ps, ps, 0, EEXIST) == -EEXIST); + bytes(src + 4 * ps, 4 * ps, ps); + bytes(dst + ps, 3 * ps, ps); + CHECK(!madvise(src + 5 * native, native, MADV_DONTNEED)); + CHECK(*(volatile unsigned char *)(src + 5 * native) == 0); + move(fd, dst + 5 * native, src + 5 * native, ps, 0, 0); + CHECK(dst[5 * native] == 0); + dst[5 * native] = 0x7d; + CHECK(src[5 * native] == 0); + CHECK(!madvise(src + 6 * native, native, MADV_DONTNEED)); + CHECK(move(fd, dst + 6 * native, src + 6 * native, ps, 0, ENOENT) == + -ENOENT); + move(fd, dst + 6 * native, src + 6 * native, ps, + UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES, 0); + CHECK(!(entry(dst + 6 * native, ps) & (PRESENT | SWAPPED))); + CHECK(!madvise(src + 10 * native + ps, ps, MADV_DONTNEED)); + p = physical(src + 10 * native, ps); + CHECK(move(fd, dst + 12 * native, src + 10 * native, 2 * ps, 0, + EAGAIN) == (long)ps); + CHECK(physical(dst + 12 * native, ps) == p); + bytes(dst + 12 * native, 10 * native, ps); + CHECK(!(entry(dst + 12 * native + ps, ps) & (PRESENT | SWAPPED))); + printf("ok - MOVE %zuK collision, zero, holes and partial progress\n", + ps / 1024); + + int pipefd[2]; + CHECK(!pipe(pipefd)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + close(pipefd[1]); + char c; + _exit(read(pipefd[0], &c, 1) == 1 ? 0 : 1); + } + close(pipefd[0]); + CHECK(move(fd, dst + 11 * native, src + 12 * native, ps, 0, EBUSY) == + -EBUSY); + CHECK(write(pipefd[1], "x", 1) == 1); + close(pipefd[1]); + int status; + CHECK(waitpid(child, &status, 0) == child && status == 0); + *(volatile unsigned char *)(src + 12 * native) = pattern(12 * native); + p = physical(src + 12 * native, ps); + move(fd, dst + 11 * native, src + 12 * native, ps, 0, 0); + CHECK(physical(dst + 11 * native, ps) == p); + bytes(dst + 11 * native, 12 * native, ps); + printf("ok - MOVE %zuK fork-shared source rejected, exclusive write/retry\n", + ps / 1024); + + /* Keep fault-in coverage as well as restoring an untouched PTE at swapoff. */ + for (unsigned int unuse = 0; unuse < 2; unuse++) { + size_t off = ps < native ? ps : 0; + size_t base = (unuse ? 13 : 8) * native; + unsigned char *s = src + base + off; + unsigned char *d = dst + base + 2 * ps; + + *(volatile unsigned char *)s = pattern(base + off); + for (unsigned int tries = 0; + !(entry(s, ps) & SWAPPED) && tries < 100; tries++) { + CHECK(!madvise(src + base, native, MADV_PAGEOUT)); + usleep(20000); + } + uint64_t old = entry(s, ps); + + CHECK(old & SWAPPED); + move(fd, d, s, ps, 0, 0); + uint64_t now = entry(d, ps); + + CHECK((now & SWAPPED) && ((now & VALUE_MASK) == (old & VALUE_MASK)) && + !(entry(s, ps) & (PRESENT | SWAPPED))); + if (unuse) { + /* Do not fault the moved slot in before swapoff restores it. */ + CHECK(!swapoff("/dev/vda")); + CHECK(entry(d, ps) & PRESENT); + } + bytes(d, base + off, ps); + if (unuse) + CHECK(!swapon("/dev/vda", 0)); + printf("ok - MOVE %zuK %s restores transferred swap PTE at new offset\n", + ps / 1024, unuse ? "swapoff" : "fault"); + } + CHECK(!close(fd)); + CHECK(!munmap(src, len)); + CHECK(!munmap(dst, len)); + concurrent_move(ps); + allocation_failure(ps, native); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + CHECK(!mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)); + struct stat st; + for (unsigned i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + CHECK(!stat("/dev/vda", &st) && S_ISBLK(st.st_mode) && + !swapon("/dev/vda", 0)); + size_t native = getauxval(AT_PAGESZ), sizes[] = { native, 4096, 16384 }; + int failed = 0; + for (unsigned i = 0; i < (native > 16384 ? 3U : 2U); i++) { + pid_t c = fork(); + int status = 0; + CHECK(c >= 0); + if (!c) { + char n[32]; + snprintf(n, sizeof(n), "%zu", native); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", n, NULL); + _exit(5); + } + CHECK(waitpid(c, &status, 0) == c); + printf("%s - MOVE ABI %zuK status=%#x\n", + status ? "not ok" : "ok", sizes[i] / 1024, status); + failed += !!status; + } + if (swapoff("/dev/vda")) + failed++; + printf("UFFD MOVE CONTRACT %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/uffd-wp-init.c b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c new file mode 100644 index 0000000000000..19747755c4c6c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c @@ -0,0 +1,913 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - uffd line=%d errno=%d\n", __LINE__, \ + errno); \ + exit(1); \ + } \ + } while (0) +struct writer { + volatile unsigned char *address; + unsigned char value; + atomic_int done, tid; +}; +static void *write_byte(void *opaque) +{ + struct writer *w = opaque; + atomic_store(&w->tid, syscall(SYS_gettid)); + *w->address = w->value; + atomic_store(&w->done, 1); + return NULL; +} +struct forker { + unsigned char *base; + size_t page_size; + pid_t pid; +}; +static void *fork_byte(void *opaque) +{ + struct forker *f = opaque; + f->pid = fork(); + if (!f->pid) { + alarm(10); + f->base[f->page_size + 17] = 0xa5; + _exit(f->base[f->page_size + 17] != 0xa5); + } + return NULL; +} +static struct uffd_msg read_event(int fd) +{ + struct pollfd p = { .fd = fd, .events = POLLIN }; + struct uffd_msg msg; + CHECK(poll(&p, 1, 10000) == 1 && (p.revents & POLLIN)); + CHECK(read(fd, &msg, sizeof(msg)) == sizeof(msg)); + return msg; +} +static void protect(int fd, void *start, size_t len, unsigned long mode) +{ + struct uffdio_writeprotect wp = { .range = { (uintptr_t)start, len }, + .mode = mode }; + CHECK(!ioctl(fd, UFFDIO_WRITEPROTECT, &wp)); +} +static void resolve_write(int fd, unsigned char *base, size_t ps, unsigned page, + unsigned char value, int exact) +{ + struct writer w = { .address = base + page * ps + 17, .value = value }; + struct pollfd pollfd = { .fd = fd, .events = POLLIN }; + struct uffd_msg msg; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, write_byte, &w)); + CHECK(poll(&pollfd, 1, 10000) == 1 && (pollfd.revents & POLLIN)); + CHECK(read(fd, &msg, sizeof(msg)) == sizeof(msg)); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT); + CHECK(msg.arg.pagefault.flags == + (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_WRITE)); + CHECK(msg.arg.pagefault.address == + (uintptr_t)(base + page * ps + (exact ? 17 : 0))); + CHECK(msg.arg.pagefault.feat.ptid == (unsigned int)atomic_load(&w.tid)); + CHECK(!atomic_load(&w.done)); + protect(fd, base + page * ps, ps, UFFDIO_WRITEPROTECT_MODE_DONTWAKE); + struct uffdio_range range = { (uintptr_t)(base + page * ps), ps }; + CHECK(!ioctl(fd, UFFDIO_WAKE, &range)); + CHECK(!pthread_join(thread, NULL) && atomic_load(&w.done)); + CHECK(base[page * ps + 17] == value); + CHECK(poll(&pollfd, 1, 0) == 0); +} +static void fill(int fd, void *dst, void *src, size_t len, unsigned long mode) +{ + struct uffdio_copy copy = { .dst = (uintptr_t)dst, + .src = (uintptr_t)src, + .len = len, + .mode = mode }; + CHECK(!ioctl(fd, UFFDIO_COPY, ©) && copy.copy == (long long)len); +} +static void zero(int fd, void *dst, size_t len) +{ + struct uffdio_zeropage z = { .range = { (uintptr_t)dst, len } }; + CHECK(!ioctl(fd, UFFDIO_ZEROPAGE, &z) && z.zeropage == (long long)len); +} +static uint64_t physical(void *address, size_t ps) +{ + uint64_t entry; + int fd = open("/proc/self/pagemap", O_RDONLY); + CHECK(fd >= 0 && + pread(fd, &entry, 8, (uintptr_t)address / ps * 8) == 8); + CHECK(!close(fd) && (entry & (1ULL << 63))); + return (entry & ((1ULL << 55) - 1)) * ps; +} +static void missing(size_t ps, size_t native) +{ + size_t len = 4 * native; + unsigned char *map = mmap(NULL, len + native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + unsigned char *src = mmap(NULL, len + native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(map != MAP_FAILED && src != MAP_FAILED); + unsigned char *p = map + ps; + CHECK(!mlock2(p, len, MLOCK_ONFAULT)); + for (size_t i = 0; i < len; i++) + src[i] = (i / ps) ^ (i * 7 + 3); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_PAGEFAULT_FLAG_WP }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING | + UFFDIO_REGISTER_MODE_WP }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK((reg.ioctls & + ((1ULL << _UFFDIO_COPY) | (1ULL << _UFFDIO_ZEROPAGE))) == + ((1ULL << _UFFDIO_COPY) | (1ULL << _UFFDIO_ZEROPAGE))); + /* A write blocks only on its missing leaf; COPY_MODE_WP causes a second event. */ + struct writer w = { .address = p + 17, .value = 0xd3 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, write_byte, &w)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)p && + msg.arg.pagefault.flags == UFFD_PAGEFAULT_FLAG_WRITE && + !atomic_load(&w.done)); + fill(fd, p, src, ps, UFFDIO_COPY_MODE_WP); + msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)p && + msg.arg.pagefault.flags == + (UFFD_PAGEFAULT_FLAG_WRITE | UFFD_PAGEFAULT_FLAG_WP)); + protect(fd, p, ps, 0); + CHECK(!pthread_join(thread, NULL) && atomic_load(&w.done)); + src[17] = w.value; + CHECK(!memcmp(p, src, ps)); + zero(fd, p + ps, ps); + for (size_t i = 0; i < ps; i++) + CHECK(p[ps + i] == 0); + p[ps + 17] = 0xe4; + CHECK(!memcmp(p, src, ps)); + /* A batch starts at a user-page boundary, not necessarily a native one. */ + fill(fd, p + 2 * ps, src + 2 * ps, len - 2 * ps, 0); + CHECK(!memcmp(p + 2 * ps, src + 2 * ps, len - 2 * ps)); + if (ps < native) { + unsigned char *a = + (void *)(((uintptr_t)(p + 2 * ps) + native - 1) & + ~(native - 1)); + uint64_t start = physical(a, ps); + CHECK(start != 0 && !(start & (native - 1))); + for (size_t i = ps; i < native; i += ps) + CHECK(physical(a + i, ps) == start + i); + } + struct uffdio_copy copy = { .dst = (uintptr_t)p, + .src = (uintptr_t)src, + .len = ps }; + CHECK(ioctl(fd, UFFDIO_COPY, ©) == -1 && errno == EEXIST && + copy.copy == -EEXIST); + CHECK(!munlock(p, len)); + /* Preserve the successfully copied prefix when the next PTE already exists. */ + CHECK(!madvise(p, ps, MADV_DONTNEED)); + copy = (struct uffdio_copy){ .dst = (uintptr_t)p, + .src = (uintptr_t)src, + .len = 2 * ps }; + CHECK(ioctl(fd, UFFDIO_COPY, ©) == -1 && errno == EAGAIN && + copy.copy == (long long)ps); + CHECK(!memcmp(p, src, ps) && p[ps + 17] == 0xe4); + /* An invalid source stops at its leaf, without installing it or touching neighbours. */ + CHECK(!madvise(p + 2 * ps, 2 * ps, MADV_DONTNEED)); + CHECK(!mprotect(src + 3 * ps, ps, PROT_NONE)); + copy = (struct uffdio_copy){ .dst = (uintptr_t)(p + 2 * ps), + .src = (uintptr_t)(src + 2 * ps), + .len = 2 * ps }; + CHECK(ioctl(fd, UFFDIO_COPY, ©) == -1 && errno == EAGAIN && + copy.copy == (long long)ps); + CHECK(!memcmp(p + 2 * ps, src + 2 * ps, ps)); + CHECK(!mprotect(src + 3 * ps, ps, PROT_READ | PROT_WRITE)); + /* A fresh source forces the unlocked copy retry. */ + CHECK(!madvise(src + 3 * ps, ps, MADV_DONTNEED)); + fill(fd, p + 3 * ps, src + 3 * ps, ps, 0); + for (size_t i = 0; i < ps; i++) + CHECK(p[3 * ps + i] == 0); + CHECK(!memcmp(p + 4 * ps, src + 4 * ps, len - 4 * ps)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + p[2 * ps + 9] ^= 0xff; + _exit(p[2 * ps + 9] == src[2 * ps + 9]); + } + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(!memcmp(p + 2 * ps, src + 2 * ps, ps)); + CHECK(!close(fd)); + CHECK(!munmap(map, len + native) && !munmap(src, len + native)); + printf("ok - uffd ABI %zuK missing copy/WP, zero, packed batch, partial EEXIST/EFAULT, source retry and COW\n", + ps / 1024); +} + +struct reader { + volatile unsigned char *address; + unsigned char value; + atomic_int done; +}; +static void *read_byte(void *opaque) +{ + struct reader *r = opaque; + r->value = *r->address; + atomic_store(&r->done, 1); + return NULL; +} +static void resume_minor(int fd, void *start, size_t len, unsigned long mode) +{ + struct uffdio_continue c = { .range = { (uintptr_t)start, len }, + .mode = mode }; + CHECK(!ioctl(fd, UFFDIO_CONTINUE, &c) && c.mapped == (long long)len); +} +static void file_minor(size_t ps, size_t native) +{ + for (int shared = 0; shared < 2; shared++) { + int memfd = memfd_create("uffd-minor-granule", 0); + CHECK(memfd >= 0 && !ftruncate(memfd, 4 * native)); + unsigned char *alias = mmap(NULL, 4 * native, + PROT_READ | PROT_WRITE, MAP_SHARED, + memfd, 0); + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + shared ? MAP_SHARED : MAP_PRIVATE, + memfd, ps); + CHECK(alias != MAP_FAILED && p != MAP_FAILED); + CHECK(!mlock2(p, 3 * ps, MLOCK_ONFAULT)); + for (size_t i = 0; i < 4 * native; i++) + alias[i] = (i / ps) ^ (i * 3 + 7); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { + .api = UFFD_API, + .features = UFFD_FEATURE_MINOR_SHMEM | + UFFD_FEATURE_PAGEFAULT_FLAG_WP + }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { + .range = { (uintptr_t)p, 3 * ps }, + .mode = UFFDIO_REGISTER_MODE_MINOR | + UFFDIO_REGISTER_MODE_WP + }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK(reg.ioctls & (1ULL << _UFFDIO_CONTINUE)); + + for (unsigned i = 0; i < 3; i++) { + struct reader r = { .address = p + i * ps + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == + (uintptr_t)(p + i * ps) && + msg.arg.pagefault.flags == + UFFD_PAGEFAULT_FLAG_MINOR && + !atomic_load(&r.done)); + alias[(i + 1) * ps + 17] ^= 0x3c; + resume_minor(fd, p + i * ps, ps, + UFFDIO_CONTINUE_MODE_WP | + UFFDIO_CONTINUE_MODE_DONTWAKE); + struct uffdio_range range = { (uintptr_t)(p + i * ps), + ps }; + CHECK(!ioctl(fd, UFFDIO_WAKE, &range)); + CHECK(!pthread_join(thread, NULL) && + atomic_load(&r.done)); + CHECK(r.value == alias[(i + 1) * ps + 17]); + CHECK(physical(p + i * ps, ps) == + physical(alias + (i + 1) * ps, ps)); + /* Independent write protection survives installing the file leaf. */ + struct writer w = { .address = p + i * ps + 17, + .value = 0xa1 + i }; + CHECK(!pthread_create(&thread, NULL, write_byte, &w)); + msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == + (uintptr_t)(p + i * ps) && + msg.arg.pagefault.flags == + (UFFD_PAGEFAULT_FLAG_WP | + UFFD_PAGEFAULT_FLAG_WRITE)); + protect(fd, p + i * ps, ps, 0); + CHECK(!pthread_join(thread, NULL) && + atomic_load(&w.done)); + CHECK(p[i * ps + 17] == w.value); + CHECK(shared ? alias[(i + 1) * ps + 17] == w.value : + alias[(i + 1) * ps + 17] == r.value); + } + struct pollfd pollfd = { .fd = fd, .events = POLLIN }; + CHECK(poll(&pollfd, 1, 0) == 0); + CHECK(!munlock(p, 3 * ps)); + CHECK(!madvise(p, 3 * ps, MADV_DONTNEED)); + resume_minor(fd, p, 3 * ps, 0); + CHECK(!memcmp(p, alias + ps, 3 * ps)); + CHECK(!madvise(p, 3 * ps, MADV_DONTNEED)); + CHECK(!ftruncate(memfd, ps + 17)); + resume_minor(fd, p, ps, 0); + struct uffdio_continue c = { .range = { (uintptr_t)(p + ps), + ps } }; + CHECK(ioctl(fd, UFFDIO_CONTINUE, &c) == -1 && errno == EFAULT && + c.mapped == -EFAULT); + CHECK(!close(fd) && !munmap(p, 3 * ps) && + !munmap(alias, 4 * native) && !close(memfd)); + printf("ok - uffd ABI %zuK shmem %s minor/WP, byte offsets, alias coherence, COW, batch continue and EOF\n", + ps / 1024, shared ? "shared" : "private"); + } +} + +static uint64_t pagemap(void *address, size_t ps) +{ + uint64_t entry; + int fd = open("/proc/self/pagemap", O_RDONLY); + CHECK(fd >= 0 && + pread(fd, &entry, 8, (uintptr_t)address / ps * 8) == 8); + CHECK(!close(fd)); + return entry; +} +static sigjmp_buf bus_env; +static void *bus_address; +static void bus_handler(int sig, siginfo_t *info, void *context) +{ + (void)context; + if (sig != SIGBUS) + _exit(91); + bus_address = info->si_addr; + siglongjmp(bus_env, 1); +} +static void markers(size_t ps) +{ + for (int async = 0; async < 2; async++) { + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { + .api = UFFD_API, + .features = UFFD_FEATURE_PAGEFAULT_FLAG_WP | + UFFD_FEATURE_THREAD_ID | + UFFD_FEATURE_WP_UNPOPULATED | + (async ? UFFD_FEATURE_WP_ASYNC : 0) + }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { + .range = { (uintptr_t)p, 3 * ps }, + .mode = UFFDIO_REGISTER_MODE_WP + }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + protect(fd, p, 3 * ps, UFFDIO_WRITEPROTECT_MODE_WP); + for (unsigned i = 0; i < 3; i++) + CHECK(pagemap(p + i * ps, ps) & (1ULL << 57)); + CHECK(*(volatile unsigned char *)(p + ps + 17) == 0); + if (async) + p[ps + 17] = 0x67; + else + resolve_write(fd, p, ps, 1, 0x67, 0); + CHECK(p[ps + 17] == 0x67); + CHECK(pagemap(p, ps) & (1ULL << 57)); + CHECK(!(pagemap(p + ps, ps) & (1ULL << 57))); + CHECK(pagemap(p + 2 * ps, ps) & (1ULL << 57)); + struct pollfd pollfd = { .fd = fd, .events = POLLIN }; + CHECK(poll(&pollfd, 1, 0) == 0); + CHECK(!close(fd)); + p[17] = 0x68; + p[2 * ps + 17] = 0x69; + CHECK(!munmap(p, 3 * ps)); + printf("ok - uffd ABI %zuK unpopulated WP async=%d per-leaf pagemap state and close\n", + ps / 1024, async); + } + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_POISON }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, 3 * ps }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®) && + (reg.ioctls & (1ULL << _UFFDIO_POISON))); + zero(fd, p, ps); + zero(fd, p + 2 * ps, ps); + struct uffdio_poison poison = { .range = { (uintptr_t)(p + ps), ps } }; + CHECK(!ioctl(fd, UFFDIO_POISON, &poison) && + poison.updated == (long long)ps); + CHECK(ioctl(fd, UFFDIO_POISON, &poison) == -1 && errno == EEXIST && + poison.updated == -EEXIST); + struct sigaction sa = { .sa_sigaction = bus_handler, + .sa_flags = SA_SIGINFO }, + old; + CHECK(!sigemptyset(&sa.sa_mask) && !sigaction(SIGBUS, &sa, &old)); + if (!sigsetjmp(bus_env, 1)) { + (void)*(volatile unsigned char *)(p + ps + 17); + CHECK(0); + } + CHECK(bus_address == p + ps + 17); + CHECK(!sigaction(SIGBUS, &old, NULL)); + p[17] = 0x81; + p[2 * ps + 17] = 0x82; + CHECK(!close(fd) && !munmap(p, 3 * ps)); + printf("ok - uffd ABI %zuK poisoned leaf signals SIGBUS and preserves neighbours\n", + ps / 1024); +} +struct pager_request { + uintptr_t address; + size_t size; +}; +static void send_pager(int sock, int fd, struct pager_request *request) +{ + union { + struct cmsghdr align; + char bytes[CMSG_SPACE(sizeof(int))]; + } control = { 0 }; + struct iovec iov = { .iov_base = request, .iov_len = sizeof(*request) }; + struct msghdr msg = { .msg_iov = &iov, + .msg_iovlen = 1, + .msg_control = control.bytes, + .msg_controllen = sizeof(control) }; + struct cmsghdr *c = CMSG_FIRSTHDR(&msg); + c->cmsg_level = SOL_SOCKET; + c->cmsg_type = SCM_RIGHTS; + c->cmsg_len = CMSG_LEN(sizeof(int)); + memcpy(CMSG_DATA(c), &fd, sizeof(fd)); + CHECK(sendmsg(sock, &msg, 0) == sizeof(*request)); +} +static unsigned char pager_pattern(size_t i) +{ + return (i / 4096) ^ (i * 11 + 5); +} +static int pager_worker(int sock, size_t expected) +{ + CHECK(getauxval(AT_PAGESZ) == expected); + alarm(30); + struct pager_request request; + union { + struct cmsghdr align; + char bytes[CMSG_SPACE(sizeof(int))]; + } control = { 0 }; + struct iovec iov = { .iov_base = &request, .iov_len = sizeof(request) }; + struct msghdr msg = { .msg_iov = &iov, + .msg_iovlen = 1, + .msg_control = control.bytes, + .msg_controllen = sizeof(control) }; + CHECK(recvmsg(sock, &msg, 0) == sizeof(request)); + struct cmsghdr *c = CMSG_FIRSTHDR(&msg); + CHECK(c && c->cmsg_level == SOL_SOCKET && c->cmsg_type == SCM_RIGHTS && + c->cmsg_len == CMSG_LEN(sizeof(int))); + int fd; + memcpy(&fd, CMSG_DATA(c), sizeof(fd)); + size_t ps = request.size; + unsigned char *source = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(source != MAP_FAILED); + for (size_t i = 0; i < 4 * ps; i++) + source[i] = pager_pattern(i); + for (unsigned i = 0; i < 3; i++) { + struct uffd_msg fault = read_event(fd); + CHECK(fault.event == UFFD_EVENT_PAGEFAULT && + fault.arg.pagefault.address == request.address + i * ps && + !fault.arg.pagefault.flags); + if (i == 1) + zero(fd, (void *)(request.address + ps), ps); + else + fill(fd, (void *)(request.address + i * ps), + source + i * ps, (i == 2 ? 2 : 1) * ps, 0); + } + CHECK(!close(fd) && !close(sock) && !munmap(source, 4 * ps)); + return 0; +} +static void cross_pager(size_t ps, size_t native, int shared) +{ + size_t pager_ps = ps == native ? 4096 : native; + int socks[2]; + CHECK(!socketpair(AF_UNIX, SOCK_SEQPACKET, 0, socks)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + char fd_arg[32], ps_arg[32]; + CHECK(!close(socks[0])); + snprintf(fd_arg, sizeof(fd_arg), "%d", socks[1]); + snprintf(ps_arg, sizeof(ps_arg), "%zu", pager_ps); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, pager_ps, 0UL, 0UL, 0UL)); + execl("/init", "/init", "--pager", fd_arg, ps_arg, NULL); + _exit(3); + } + CHECK(!close(socks[1])); + int memfd = shared ? memfd_create("remote-uffd-shmem", 0) : -1; + if (shared) + CHECK(memfd >= 0 && !ftruncate(memfd, 4 * ps)); + unsigned char *p = mmap( + NULL, 4 * ps, PROT_READ | PROT_WRITE, + shared ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS, memfd, 0); + CHECK(p != MAP_FAILED); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, 4 * ps }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + struct pager_request request = { .address = (uintptr_t)p, .size = ps }; + send_pager(socks[0], fd, &request); + for (size_t i = 0; i < 4 * ps; i++) + CHECK(p[i] == (i >= ps && i < 2 * ps ? 0 : pager_pattern(i))); + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(!close(fd) && !close(socks[0]) && !munmap(p, 4 * ps)); + if (shared) + CHECK(!close(memfd)); + printf("ok - uffd target %zuK remote pager %zuK shared=%d SCM_RIGHTS copy/zero/batch uses target granule\n", + ps / 1024, pager_ps / 1024, shared); +} + +static void private_missing(size_t ps) +{ + int memfd = memfd_create("uffd-private-hole", 0); + CHECK(memfd >= 0 && !ftruncate(memfd, 4 * ps)); + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE, memfd, ps); + unsigned char *src = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED && src != MAP_FAILED); + for (size_t i = 0; i < 3 * ps; i++) + src[i] = pager_pattern(i); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, 3 * ps }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®) && + (reg.ioctls & (1ULL << _UFFDIO_COPY))); + struct reader r = { .address = p + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)p && + !msg.arg.pagefault.flags); + fill(fd, p, src, 2 * ps, 0); + CHECK(!pthread_join(thread, NULL) && r.value == src[17]); + zero(fd, p + 2 * ps, ps); + CHECK(!memcmp(p, src, 2 * ps)); + unsigned char bytes[256]; + for (size_t off = 0; off < 4 * ps; off += sizeof(bytes)) { + CHECK(pread(memfd, bytes, sizeof(bytes), off) == sizeof(bytes)); + for (unsigned i = 0; i < sizeof(bytes); i++) + CHECK(bytes[i] == 0); + } + CHECK(!madvise(p, ps, MADV_DONTNEED)); + CHECK(!madvise(src, 3 * ps, MADV_DONTNEED)); + fill(fd, p, src, ps, 0); + for (size_t i = 0; i < ps; i++) + CHECK(p[i] == 0); + CHECK(!madvise(p, 3 * ps, MADV_DONTNEED)); + CHECK(!ftruncate(memfd, ps + 17)); + struct uffdio_copy copy = { .dst = (uintptr_t)(p + ps), + .src = (uintptr_t)src, + .len = ps }; + CHECK(ioctl(fd, UFFDIO_COPY, ©) == -1 && errno == EFAULT && + copy.copy == -EFAULT); + fill(fd, p, src, ps, 0); + CHECK(!close(fd) && !close(memfd) && !munmap(p, 3 * ps) && + !munmap(src, 3 * ps)); + printf("ok - uffd ABI %zuK private shmem missing COPY/ZERO, unchanged file bytes, source retry and EOF\n", + ps / 1024); +} + +static int register_shared(void *p, size_t len) +{ + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MISSING_SHMEM | + UFFD_FEATURE_MINOR_SHMEM | + UFFD_FEATURE_PAGEFAULT_FLAG_WP }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING | + UFFDIO_REGISTER_MODE_MINOR | + UFFDIO_REGISTER_MODE_WP }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK((reg.ioctls & + ((1ULL << _UFFDIO_COPY) | (1ULL << _UFFDIO_ZEROPAGE) | + (1ULL << _UFFDIO_CONTINUE))) == + ((1ULL << _UFFDIO_COPY) | (1ULL << _UFFDIO_ZEROPAGE) | + (1ULL << _UFFDIO_CONTINUE))); + return fd; +} +static void shared_missing(size_t ps, size_t native) +{ + int memfd = memfd_create("uffd-shared-holes", 0); + CHECK(memfd >= 0 && !ftruncate(memfd, 8 * native)); + unsigned char *p = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, memfd, ps); + unsigned char *q = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, memfd, ps); + unsigned char *src = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED && q != MAP_FAILED && src != MAP_FAILED); + for (size_t i = 0; i < 4 * ps; i++) + src[i] = pager_pattern(i); + int fd = register_shared(p, 4 * ps), fd2 = register_shared(q, 4 * ps); + for (unsigned i = 0; i < 2; i++) { + struct reader r = { .address = p + i * ps + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)(p + i * ps) && + !msg.arg.pagefault.flags); + if (!i) + fill(fd, p, src, ps, UFFDIO_COPY_MODE_WP); + else + zero(fd, p + ps, ps); + CHECK(!pthread_join(thread, NULL) && atomic_load(&r.done)); + CHECK(r.value == (i ? 0 : src[17])); + if (!i) { + struct uffdio_continue c = { + .range = { (uintptr_t)(p + ps), ps } + }; + CHECK(ioctl(fd, UFFDIO_CONTINUE, &c) == -1 && + errno == EFAULT && c.mapped == -EFAULT); + } + } + fill(fd, p + 2 * ps, src + 2 * ps, 2 * ps, 0); + CHECK(!memcmp(p, src, ps) && !memcmp(p + 2 * ps, src + 2 * ps, 2 * ps)); + for (size_t i = 0; i < ps; i++) + CHECK(p[ps + i] == 0); + for (unsigned i = 0; i < 4; i++) { + struct reader r = { .address = q + i * ps + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd2); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)(q + i * ps) && + msg.arg.pagefault.flags == UFFD_PAGEFAULT_FLAG_MINOR); + resume_minor(fd2, q + i * ps, ps, 0); + CHECK(!pthread_join(thread, NULL) && r.value == p[i * ps + 17]); + CHECK(physical(p + i * ps, ps) == physical(q + i * ps, ps)); + } + CHECK(!memcmp(p, q, 4 * ps)); + if (ps < native) + CHECK(physical(p + ps, ps) == physical(p, ps) + ps); + struct writer w = { .address = p + 17, .value = 0x9a }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, write_byte, &w)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)p && + msg.arg.pagefault.flags == + (UFFD_PAGEFAULT_FLAG_WRITE | UFFD_PAGEFAULT_FLAG_WP)); + protect(fd, p, ps, 0); + CHECK(!pthread_join(thread, NULL) && p[17] == w.value && + q[17] == w.value); + struct uffdio_copy c = { .dst = (uintptr_t)(p + ps), + .src = (uintptr_t)src, + .len = ps }; + CHECK(ioctl(fd, UFFDIO_COPY, &c) == -1 && errno == EEXIST && + c.copy == -EEXIST); + unsigned char bytes[256]; + CHECK(pread(memfd, bytes, sizeof(bytes), ps) == sizeof(bytes)); + CHECK(!memcmp(bytes, p, sizeof(bytes))); + /* Resolve an untouched source outside VMA locks before taking a cache folio. */ + unsigned char *r = mmap(NULL, 2 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, memfd, 6 * native); + unsigned char *fresh = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(r != MAP_FAILED && fresh != MAP_FAILED); + int fd3 = register_shared(r, 2 * ps); + fill(fd3, r, fresh, ps, 0); + for (size_t i = 0; i < ps; i++) + CHECK(r[i] == 0); + CHECK(!mprotect(fresh, ps, PROT_NONE)); + c = (struct uffdio_copy){ .dst = (uintptr_t)(r + ps), + .src = (uintptr_t)fresh, + .len = ps }; + CHECK(ioctl(fd3, UFFDIO_COPY, &c) == -1 && errno == EFAULT && + c.copy == -EFAULT); + zero(fd3, r + ps, ps); + for (size_t i = 0; i < ps; i++) + CHECK(r[ps + i] == 0); + CHECK(!close(fd3) && !munmap(r, 2 * ps) && !munmap(fresh, ps)); + CHECK(!ftruncate(memfd, 2 * ps + 17)); + c = (struct uffdio_copy){ .dst = (uintptr_t)(p + 2 * ps), + .src = (uintptr_t)src, + .len = ps }; + CHECK(ioctl(fd, UFFDIO_COPY, &c) == -1 && + errno == (ps == native ? ENOMEM : EFAULT) && + c.copy == -(ps == native ? ENOMEM : EFAULT)); + CHECK(!close(fd) && !close(fd2) && !close(memfd)); + CHECK(!munmap(p, 4 * ps) && !munmap(q, 4 * ps) && !munmap(src, 4 * ps)); + printf("ok - uffd ABI %zuK shared COPY/ZERO, independent missing/minor pagers, CONTINUE holes, packing, WP/coherence and EOF\n", + ps / 1024); +} + +static void shared_buffered_write(size_t ps) +{ + int memfd = memfd_create("uffd-buffered-validity", 0); + CHECK(memfd >= 0 && !ftruncate(memfd, 4 * ps)); + unsigned char *p = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, memfd, 0); + unsigned char *src = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED && src != MAP_FAILED); + memset(src, 0x57, ps); + int fd = register_shared(p, 4 * ps); + fill(fd, p, src, ps, 0); + CHECK(pwrite(memfd, src, 23, ps + 17) == 23); + for (unsigned i = 1; i < 3; i++) { + struct reader r = { .address = p + i * ps + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)(p + i * ps) && + msg.arg.pagefault.flags == + (i == 1 ? UFFD_PAGEFAULT_FLAG_MINOR : 0)); + if (i == 1) + resume_minor(fd, p + ps, ps, 0); + else + zero(fd, p + i * ps, ps); + CHECK(!pthread_join(thread, NULL) && + r.value == (i == 1 ? 0x57 : 0)); + } + for (size_t i = 0; i < ps; i++) + CHECK(p[ps + i] == (i >= 17 && i < 40 ? 0x57 : 0)); + CHECK(!memcmp(p, src, ps)); + CHECK(!close(fd) && !close(memfd) && !munmap(p, 4 * ps) && + !munmap(src, ps)); + printf("ok - uffd ABI %zuK buffered short-range write populates its leaf and leaves neighbours missing\n", + ps / 1024); +} + +static int worker(unsigned long expected, unsigned long native) +{ + size_t ps = getauxval(AT_PAGESZ); + CHECK(ps == expected); + alarm(60); + missing(ps, native); + file_minor(ps, native); + markers(ps); + cross_pager(ps, native, 0); + cross_pager(ps, native, 1); + private_missing(ps); + shared_missing(ps, native); + shared_buffered_write(ps); + for (int exact = 0; exact < 2; exact++) { + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED); + for (unsigned i = 0; i < 3; i++) + memset(p + i * ps, 0x40 + i, ps); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { + .api = UFFD_API, + .features = UFFD_FEATURE_PAGEFAULT_FLAG_WP | + UFFD_FEATURE_THREAD_ID | + (exact ? UFFD_FEATURE_EXACT_ADDRESS | + UFFD_FEATURE_EVENT_FORK : + 0) + }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + CHECK(api.features & UFFD_FEATURE_PAGEFAULT_FLAG_WP); + struct uffdio_register reg = { + .range = { (uintptr_t)p, 3 * ps }, + .mode = UFFDIO_REGISTER_MODE_WP + }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK(reg.ioctls & (1ULL << _UFFDIO_WRITEPROTECT)); + + for (unsigned i = 0; i < 3; i++) { + protect(fd, p + i * ps, ps, + UFFDIO_WRITEPROTECT_MODE_WP); + /* Neighbours remain directly writable while this page faults. */ + p[((i + 1) % 3) * ps + 31] ^= 1; + resolve_write(fd, p, ps, i, 0x70 + i, exact); + } + protect(fd, p + ps, ps, UFFDIO_WRITEPROTECT_MODE_WP); + pid_t child; + int status; + if (!exact) { + child = fork(); + CHECK(child >= 0); + if (!child) { + alarm(10); + p[ps + 17] = 0xa5; + _exit(p[ps + 17] != 0xa5); + } + } else { + struct forker f = { .base = p, .page_size = ps }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, fork_byte, &f)); + struct uffd_msg event = read_event(fd); + CHECK(event.event == UFFD_EVENT_FORK); + int child_fd = event.arg.fork.ufd; + CHECK(child_fd >= 0); + CHECK(!pthread_join(thread, NULL)); + child = f.pid; + CHECK(child > 0); + event = read_event(child_fd); + CHECK(event.event == UFFD_EVENT_PAGEFAULT && + event.arg.pagefault.flags == + (UFFD_PAGEFAULT_FLAG_WP | + UFFD_PAGEFAULT_FLAG_WRITE)); + CHECK(event.arg.pagefault.address == + (uintptr_t)(p + ps + 17)); + CHECK(event.arg.pagefault.feat.ptid == + (unsigned int)child); + protect(child_fd, p + ps, ps, 0); + CHECK(!close(child_fd)); + } + CHECK(waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status)); + CHECK(p[ps + 17] == 0x71); + resolve_write(fd, p, ps, 1, 0x91, exact); + protect(fd, p, 3 * ps, UFFDIO_WRITEPROTECT_MODE_WP); + struct uffdio_range middle = { (uintptr_t)(p + ps), ps }; + CHECK(!ioctl(fd, UFFDIO_UNREGISTER, &middle)); + p[ps + 17] = 0xb2; + resolve_write(fd, p, ps, 0, 0xb1, exact); + CHECK(!close(fd)); + /* Close removes the remaining registration and write protection. */ + p[2 * ps + 17] = 0xb3; + CHECK(p[17] == 0xb1 && p[ps + 17] == 0xb2 && + p[2 * ps + 17] == 0xb3); + CHECK(!munmap(p, 3 * ps)); + printf("ok - uffd ABI %zuK exact=%d events, neighbours, fork delivery/COW, partial unregister and close\n", + ps / 1024, exact); + } + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--pager")) + return pager_worker(atoi(argv[2]), strtoul(argv[3], NULL, 10)); + if (argc == 4 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10)); + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + unsigned long native = getauxval(AT_PAGESZ), + sizes[] = { native, 4096, 16384 }; + unsigned modes = native == 65536 ? 3 : 2; + int failed = 0; + for (unsigned i = 0; i < modes; i++) { + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char ps[32], ns[32]; + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + snprintf(ns, sizeof(ns), "%lu", native); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, + 0UL)); + execl("/init", "/init", "--worker", ps, ns, NULL); + _exit(3); + } + int status = 0; + int ok = waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status); + printf("%s - uffd ABI %luK status=%#x\n", ok ? "ok" : "not ok", + sizes[i] / 1024, status); + failed |= !ok; + } + printf("UFFD CONTRACT %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/uprobes-init.c b/tools/testing/selftests/arm64/user4k/uprobes-init.c new file mode 100644 index 0000000000000..0c379e4309caf --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/uprobes-init.c @@ -0,0 +1,318 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define TRACE "/sys/kernel/tracing/" +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) +static int write_file(const char *path, const char *s) +{ + int fd = open(path, O_WRONLY); + if (fd < 0) + return -1; + size_t len = strlen(s); + int ret = write(fd, s, len) == (ssize_t)len ? 0 : -1; + close(fd); + return ret; +} +static int enable(const char *event, int on) +{ + char path[160]; + snprintf(path, sizeof(path), TRACE "events/aurora_up/%s/enable", event); + return write_file(path, on ? "1" : "0"); +} +static uint64_t pagemap(void *p, size_t ps) +{ + uint64_t bits = 0; + int fd = open("/proc/self/pagemap", O_RDONLY); + if (fd >= 0) { + if (pread(fd, &bits, 8, (uintptr_t)p / ps * 8) != 8) + bits = 0; + close(fd); + } + return bits; +} +static unsigned records(const char *event) +{ + FILE *f = fopen(TRACE "trace", "r"); + char *line = NULL; + size_t len = 0; + unsigned count = 0; + char match[96]; + snprintf(match, sizeof(match), "%s:", event); + if (!f) + return 0; + while (getline(&line, &len, f) > 0) + if (strstr(line, match)) + count++; + free(line); + fclose(f); + return count; +} +static int xol_geometry(size_t native) +{ + FILE *f = fopen("/proc/self/maps", "r"); + char *line = NULL; + size_t len = 0; + int ok = 0; + if (!f) + return 0; + while (getline(&line, &len, f) > 0) + if (strstr(line, "[uprobes]")) { + unsigned long a, b; + if (sscanf(line, "%lx-%lx", &a, &b) == 2) { + printf("# XOL %#lx-%#lx native=%zu\n", a, b, + native); + ok = !(a % (unsigned long)native) && + b - a == native; + } + } + free(line); + fclose(f); + return ok; +} +static int exercise(size_t ps, size_t native, unsigned shifted) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(20); + printf("# start ps=%zu shifted=%u\n", ps, shifted); + int fd = memfd_create("aurora-uprobe-code", 0); + CHECK(fd >= 0); + size_t length = 8 * ps; + unsigned char *blob = malloc(length); + CHECK(blob); + memset(blob, 0x71, length); + size_t code_off = 3 * ps + 64, ctr_off = 5 * ps + 32; + size_t map_off = shifted ? 2 * ps : 0, + map_len = shifted ? 2 * ps : 4 * ps; + uint32_t insns[] = { 0x11001c00, 0xd65f03c0 }; + memcpy(blob + code_off, insns, sizeof(insns)); + memset(blob + ctr_off, 0, 2); + CHECK(write(fd, blob, length) == (ssize_t)length); + unsigned char *code = mmap((void *)(0x20000000 + (shifted ? ps : 0)), + map_len, PROT_READ | PROT_EXEC, + MAP_PRIVATE | MAP_FIXED_NOREPLACE, fd, + map_off); + unsigned char *counter = + mmap((void *)(0x30000000 + 2 * ps), ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_FIXED_NOREPLACE, fd, 5 * ps); + unsigned char *other = mmap((void *)0x40000000, ps, + PROT_READ | PROT_EXEC, + MAP_PRIVATE | MAP_FIXED_NOREPLACE, fd, 0); + CHECK(code != MAP_FAILED && counter != MAP_FAILED && + other != MAP_FAILED); + unsigned long (*target)(unsigned long) = + (void *)(code + code_off - map_off); + volatile unsigned short *ctr = (void *)(counter + 32); + CHECK(*ctr == 0); + CHECK(target(11) == 18); + /* Occupy the default top-of-address-space XOL hint, forcing a new placement. */ + void *guard = mmap((void *)((1UL << 48) - native), ps, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, + -1, 0); + CHECK(guard != MAP_FAILED); + char entry[64], retname[64], cmd[256]; + snprintf(entry, sizeof(entry), "entry_%zu_%u", ps, shifted); + snprintf(retname, sizeof(retname), "return_%zu_%u", ps, shifted); + snprintf(cmd, sizeof(cmd), + "p:aurora_up/%s /proc/self/fd/%d:0x%zx(0x%zx)\n", entry, fd, + code_off, ctr_off); + CHECK(!write_file(TRACE "uprobe_events", cmd)); + snprintf(cmd, sizeof(cmd), + "r:aurora_up/%s /proc/self/fd/%d:0x%zx(0x%zx)\n", retname, fd, + code_off, ctr_off); + CHECK(!write_file(TRACE "uprobe_events", cmd)); + printf("# enabling entry at line %d\n", __LINE__); + CHECK(!enable(entry, 1)); + printf("# entry enabled\n"); + CHECK(*ctr == 1); + printf("# enabling return\n"); + CHECK(!enable(retname, 1)); + printf("# return enabled\n"); + CHECK(*ctr == 1); + uint32_t disk[2]; + CHECK(pread(fd, disk, sizeof(disk), code_off) == sizeof(disk)); + CHECK(!memcmp(disk, insns, sizeof(insns))); + CHECK(!memcmp(other, blob, ps)); + CHECK(!memcmp(counter, blob + 5 * ps, 32)); + CHECK(!memcmp(counter + 34, blob + 5 * ps + 34, ps - 34)); + int tracefd = open(TRACE "trace", O_WRONLY | O_TRUNC); + CHECK(tracefd >= 0); + close(tracefd); + printf("# invoking probes\n"); + for (unsigned i = 0; i < 16; i++) + CHECK(target(i) == i + 7); + printf("# invocations complete\n"); + CHECK(xol_geometry(native)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + int ok = *ctr == 1; + for (unsigned i = 0; i < 8; i++) + ok &= target(100 + i) == 107 + i; + _exit(ok ? 0 : 3); + } + int status; + CHECK(waitpid(child, &status, 0) == child); + CHECK(WIFEXITED(status) && !WEXITSTATUS(status)); + CHECK(*ctr == 1); + unsigned entries = records(entry), returns = records(retname); + printf("# ABI=%zu entry=%u return=%u\n", ps, entries, returns); + CHECK(entries == 24 && returns == 24); + CHECK(!enable(entry, 0)); + CHECK(*ctr == 1); + CHECK(!enable(retname, 0)); + CHECK(*ctr == 0); + /* Identical restored text is dropped so it can be shared with the file again. */ + CHECK(!(pagemap((void *)target, ps) >> 63)); + CHECK(target(10) == 17); + CHECK(!memcmp(code, blob + map_off, map_len)); + CHECK(!memcmp(other, blob, ps)); + CHECK(!memcmp(counter, blob + 5 * ps, ps)); + /* A pipe GET must prevent dropping the mapped code slot on removal. */ + printf("# enabling entry at line %d\n", __LINE__); + CHECK(!enable(entry, 1)); + printf("# entry enabled\n"); + CHECK(*ctr == 1); + int pipefd[2]; + CHECK(!pipe(pipefd)); + struct iovec iov = { .iov_base = + (void *)((uintptr_t)target & ~(ps - 1)), + .iov_len = ps }; + CHECK(vmsplice(pipefd[1], &iov, 1, 0) == (ssize_t)ps); + CHECK(!enable(entry, 0)); + CHECK(*ctr == 0); + CHECK(pagemap((void *)target, ps) >> 63); + unsigned char *held = malloc(ps); + CHECK(held); + CHECK(read(pipefd[0], held, ps) == (ssize_t)ps); + CHECK(!memcmp(held, blob + (code_off & ~(ps - 1)), ps)); + free(held); + close(pipefd[0]); + close(pipefd[1]); + printf("# enabling entry at line %d\n", __LINE__); + CHECK(!enable(entry, 1)); + printf("# entry enabled\n"); + CHECK(!enable(entry, 0)); + CHECK(*ctr == 0); + /* Native folio-wide references may defer this optional optimization. */ + if (ps < native) + CHECK(!(pagemap((void *)target, ps) >> 63)); + CHECK(target(20) == 27); + snprintf(cmd, sizeof(cmd), "-:aurora_up/%s\n", entry); + CHECK(!write_file(TRACE "uprobe_events", cmd)); + snprintf(cmd, sizeof(cmd), "-:aurora_up/%s\n", retname); + CHECK(!write_file(TRACE "uprobe_events", cmd)); + CHECK(!munmap(code, map_len)); + CHECK(!munmap(guard, ps)); + CHECK(!munmap(counter, ps)); + CHECK(!munmap(other, ps)); + close(fd); + free(blob); + printf("ok - %zuK uprobe/uretprobe, byte offsets, counter, fork, restore and file sharing\n", + ps / 1024); + return 0; +} +static int wait_bounded(pid_t child, int *status) +{ + for (unsigned i = 0; i < 250; i++) { + pid_t got = waitpid(child, status, WNOHANG); + if (got == child) + return 1; + if (got < 0) + return 0; + if (i == 50 || i == 200) { + char path[64], buf[4096]; + snprintf(path, sizeof(path), "/proc/%d/stack", child); + int fd = open(path, O_RDONLY); + ssize_t n = fd < 0 ? -1 : + read(fd, buf, sizeof(buf) - 1); + if (fd >= 0) + close(fd); + if (n > 0) { + buf[n] = 0; + printf("# child %d stack:\n%s", child, buf); + } else + printf("# child %d stack unavailable errno=%d\n", + child, errno); + } + if (i == 200) { + printf("# killing stalled child %d\n", child); + kill(child, SIGKILL); + } + usleep(100000); + } + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), + strtoul(argv[4], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("tracefs", TRACE, "tracefs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + size_t sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned run = 0; run < (sizes[0] > 16384 ? 6U : 4U); run++) { + unsigned i = run / 2, shifted = run & 1; + pid_t c = fork(); + int status = 0; + if (!c) { + char expected[24], native[24], mode[8]; + snprintf(mode, sizeof(mode), "%u", shifted); + snprintf(expected, sizeof(expected), "%zu", sizes[i]); + snprintf(native, sizeof(native), "%zu", sizes[0]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", expected, native, + mode, NULL); + _exit(5); + } + int ok = c > 0 && wait_bounded(c, &status) && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - ABI %zuK uprobes shifted=%u status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, shifted, status); + fail += !ok; + } +done: + printf("UPROBES %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/uring-fixture-init.c b/tools/testing/selftests/arm64/user4k/uring-fixture-init.c new file mode 100644 index 0000000000000..cafc11138d2e8 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/uring-fixture-init.c @@ -0,0 +1,467 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +struct ring { + int fd; + struct io_uring_params p; + void *sq, *cq, *sqes; + size_t sqlen, cqlen, sqelen; +}; +static unsigned load(unsigned *p) +{ + return __atomic_load_n(p, __ATOMIC_ACQUIRE); +} +static void store(unsigned *p, unsigned v) +{ + __atomic_store_n(p, v, __ATOMIC_RELEASE); +} +static size_t align_size(size_t n) +{ + size_t ps = getauxval(AT_PAGESZ); + return (n + ps - 1) & ~(ps - 1); +} +static void destroy(struct ring *r) +{ + if (r->sq && r->sq != MAP_FAILED) + munmap(r->sq, r->sqlen); + if (r->cq && r->cq != MAP_FAILED && r->cq != r->sq) + munmap(r->cq, r->cqlen); + if (r->sqes && r->sqes != MAP_FAILED) + munmap(r->sqes, r->sqelen); + if (r->fd >= 0) + close(r->fd); + *r = (struct ring){ .fd = -1 }; +} +static bool setup(struct ring *r, unsigned entries, unsigned flags) +{ + *r = (struct ring){ .fd = -1 }; + r->p.flags = flags; + r->fd = syscall(__NR_io_uring_setup, entries, &r->p); + if (r->fd < 0) + return false; + r->sqlen = align_size(r->p.sq_off.array + + r->p.sq_entries * sizeof(unsigned)); + r->cqlen = align_size(r->p.cq_off.cqes + + r->p.cq_entries * sizeof(struct io_uring_cqe) * + (flags & IORING_SETUP_CQE32 ? 2 : 1)); + r->sqelen = align_size(r->p.sq_entries * sizeof(struct io_uring_sqe) * + (flags & IORING_SETUP_SQE128 ? 2 : 1)); + if (r->p.features & IORING_FEAT_SINGLE_MMAP) { + if (r->cqlen > r->sqlen) + r->sqlen = r->cqlen; + r->cqlen = r->sqlen; + } + r->sq = mmap(NULL, r->sqlen, PROT_READ | PROT_WRITE, MAP_SHARED, r->fd, + IORING_OFF_SQ_RING); + r->cq = (r->p.features & IORING_FEAT_SINGLE_MMAP) ? + r->sq : + mmap(NULL, r->cqlen, PROT_READ | PROT_WRITE, MAP_SHARED, + r->fd, IORING_OFF_CQ_RING); + r->sqes = mmap(NULL, r->sqelen, PROT_READ | PROT_WRITE, MAP_SHARED, + r->fd, IORING_OFF_SQES); + return r->sq != MAP_FAILED && r->cq != MAP_FAILED && + r->sqes != MAP_FAILED; +} +static bool request(struct ring *r, unsigned op, int fd, void *buf, + unsigned len, uint64_t off, bool async) +{ + unsigned *sqtail = r->sq + r->p.sq_off.tail, + *sqmask = r->sq + r->p.sq_off.ring_mask; + unsigned *cqhead = r->cq + r->p.cq_off.head, + *cqtail = r->cq + r->p.cq_off.tail, + *cqmask = r->cq + r->p.cq_off.ring_mask; + unsigned tail = load(sqtail), head = load(cqhead), + idx = tail & load(sqmask); + unsigned sqstride = sizeof(struct io_uring_sqe) * + (r->p.flags & IORING_SETUP_SQE128 ? 2 : 1); + unsigned cqstride = sizeof(struct io_uring_cqe) * + (r->p.flags & IORING_SETUP_CQE32 ? 2 : 1); + struct io_uring_sqe *sqe = r->sqes + idx * sqstride; + memset(sqe, 0, sqstride); + sqe->opcode = op; + sqe->fd = fd; + sqe->addr = (uintptr_t)buf; + sqe->len = len; + sqe->off = off; + sqe->user_data = (uint64_t)tail + 1234; + if (async) + sqe->flags = IOSQE_ASYNC; + ((unsigned *)(r->sq + r->p.sq_off.array))[idx] = idx; + store(sqtail, tail + 1); + int ret = syscall(__NR_io_uring_enter, r->fd, 1, 1, + IORING_ENTER_GETEVENTS, NULL, 0); + if (ret != 1 || load(cqtail) != head + 1) { + ksft_print_msg("enter=%d errno=%d\n", ret, errno); + return false; + } + struct io_uring_cqe *cqe = + r->cq + r->p.cq_off.cqes + (head & load(cqmask)) * cqstride; + unsigned expected = op == IORING_OP_NOP ? 0 : len; + if (op == IORING_OP_READV_FIXED || op == IORING_OP_WRITEV_FIXED) { + expected = 0; + for (unsigned i = 0; i < len; i++) + expected += ((struct iovec *)buf)[i].iov_len; + } + bool ok = cqe->user_data == (uint64_t)tail + 1234 && + cqe->res == (int)expected; + if (!ok) + ksft_print_msg("cqe res=%d expected=%u\n", cqe->res, expected); + store(cqhead, head + 1); + return ok; +} +static unsigned long pinned_kb(void) +{ + FILE *f = fopen("/proc/self/status", "r"); + char line[160]; + unsigned long n = ~0UL; + if (!f) + return n; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmPin: %lu kB", &n) == 1) + break; + fclose(f); + return n; +} +static bool bytes_are(unsigned char *p, size_t len, unsigned char value) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != value) + return false; + return true; +} +static void fixed_checks(int fd, unsigned char *p, size_t len, size_t ps) +{ + struct ring r = { .fd = -1 }; + unsigned char *q = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + struct iovec reg = { p + 512, len - 1024 }; + bool live = false, ok = setup(&r, 8, 0); + unsigned long baseline = pinned_kb(); + memset(p, 0xa6, len); + if (ok) + live = syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS, ®, 1) == 0; + ok &= live; + ksft_test_result(ok, "register unaligned long-term buffer\n"); + if (ok) { + ok = request(&r, IORING_OP_WRITE_FIXED, fd, p + 512, 7 * 512, 0, + false); + memset(p + 512, 0, 7 * 512); + ok &= request(&r, IORING_OP_READ_FIXED, fd, p + 512, 7 * 512, 0, + false) && + bytes_are(p + 512, 7 * 512, 0xa6); + } + ksft_test_result(ok, "fixed first-quarter slice offsets\n"); + if (ok) { + size_t off = len - 3 * ps; + memset(p + off, 0x49, 2 * ps); + ok = request(&r, IORING_OP_WRITE_FIXED, fd, p + off, 2 * ps, + 65536, true); + memset(p + off, 0, 2 * ps); + ok &= request(&r, IORING_OP_READ_FIXED, fd, p + off, 2 * ps, + 65536, true) && + bytes_are(p + off, 2 * ps, 0x49); + } + ksft_test_result(ok, "fixed late-buffer fast indexing and async I/O\n"); + if (ok) { + struct iovec v[3] = { { p + 512, 512 }, + { p + 2 * ps + 512, ps - 512 }, + { p + 17 * ps + 512, 1024 } }; + memset(p, 0x78, len); + ok = request(&r, IORING_OP_WRITEV_FIXED, fd, v, 3, 131072, + true); + memset(p, 0x11, len); + ok &= request(&r, IORING_OP_READV_FIXED, fd, v, 3, 131072, + true); + for (unsigned i = 0; i < 3; i++) + ok &= bytes_are(v[i].iov_base, v[i].iov_len, 0x78); + ok &= p[0] == 0x11 && p[ps + 512] == 0x11 && p[17 * ps] == 0x11; + } + ksft_test_result(ok, "fixed vectored physical offsets and guards\n"); + if (ok) { + memset(p + 512, 0x3e, len - 1024); + ok = request(&r, IORING_OP_WRITE_FIXED, fd, p + 512, len - 1024, + 4 * 1024 * 1024, true); + memset(p + 512, 0, len - 1024); + ok &= request(&r, IORING_OP_READ_FIXED, fd, p + 512, len - 1024, + 4 * 1024 * 1024, true) && + bytes_are(p + 512, len - 1024, 0x3e); + } + ksft_test_result(ok, "fixed multi-bio transfer\n"); + if (ok) { + ok = mmap(p, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) == p; + memset(p, 0x55, len); + ok &= request(&r, IORING_OP_WRITE_FIXED, fd, p + 512, 3 * ps, + 8 * 1024 * 1024, true); + if (q != MAP_FAILED) { + memset(q, 0, 4 * ps); + ok &= request(&r, IORING_OP_READ, fd, q, 3 * ps, + 8 * 1024 * 1024, false) && + bytes_are(q, 3 * ps, 0x3e) && + bytes_are(p, len, 0x55); + } else + ok = false; + } + ksft_test_result(ok, + "registered source survives unmap and replacement\n"); + if (ok) { + memset(q, 0xc1, 3 * ps); + ok = request(&r, IORING_OP_WRITE, fd, q, 3 * ps, + 10 * 1024 * 1024, false); + ok &= request(&r, IORING_OP_READ_FIXED, fd, p + 512, 3 * ps, + 10 * 1024 * 1024, true); + ok &= request(&r, IORING_OP_WRITE_FIXED, fd, p + 512, 3 * ps, + 12 * 1024 * 1024, true); + memset(q, 0, 3 * ps); + ok &= request(&r, IORING_OP_READ, fd, q, 3 * ps, + 12 * 1024 * 1024, false) && + bytes_are(q, 3 * ps, 0xc1) && bytes_are(p, len, 0x55); + } + ksft_test_result( + ok, + "registered destination survives unmap without touching replacement\n"); + if (live) { + ok = syscall(__NR_io_uring_register, r.fd, + IORING_UNREGISTER_BUFFERS, NULL, 0) == 0; + live = false; + } else + ok = false; + ok &= pinned_kb() == baseline; + if (ok) { + ok = !mprotect(p + ps, ps, PROT_NONE); + struct iovec bad[2] = { { p, ps }, { p + ps, ps } }; + ok &= syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS, bad, 2) == -1 && + errno == EFAULT; + ok &= !mprotect(p + ps, ps, PROT_READ | PROT_WRITE) && + pinned_kb() == baseline; + live = syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS, ®, 1) == 0; + ok &= live; + } + ksft_test_result( + ok, + "failed multi-buffer registration rolls back pins and accounting\n"); + if (live) { + ok &= syscall(__NR_io_uring_register, r.fd, + IORING_UNREGISTER_BUFFERS, NULL, 0) == 0; + live = false; + } + for (unsigned i = 0; i < 64 && ok; i++) { + reg = (struct iovec){ p + ps + 512, 2 * ps }; + ok = !madvise(p + ps, 3 * ps, MADV_DONTNEED); + live = syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS, ®, 1) == 0; + ok &= live; + if (live) { + ok &= syscall(__NR_io_uring_register, r.fd, + IORING_UNREGISTER_BUFFERS, NULL, 0) == 0; + live = false; + } + ok &= pinned_kb() == baseline; + } + ksft_test_result(ok, + "64 register/reuse/unregister cycles balance VmPin\n"); + /* The file offset deliberately differs from the virtual quarter index. */ + int mem = memfd_create("uring-fixed-shared", 0); + ok &= mem >= 0 && !ftruncate(mem, len + 3 * ps) && q != MAP_FAILED; + if (ok) { + ok = mmap(p, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, mem, 3 * ps) == p; + memset(p, 0x6b, len); + reg = (struct iovec){ p + 512, len - 1024 }; + live = syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS, ®, 1) == 0; + ok &= live; + struct iovec v[3] = { { p + 512, 512 }, + { p + 3 * ps + 512, ps - 512 }, + { p + 23 * ps + 512, 1024 } }; + if (ok) { + ok = request(&r, IORING_OP_WRITEV_FIXED, fd, v, 3, + 14 * 1024 * 1024, true); + memset(p + 512, 0x18, len - 1024); + ok &= request(&r, IORING_OP_READV_FIXED, fd, v, 3, + 14 * 1024 * 1024, true); + for (unsigned i = 0; i < 3; i++) + ok &= bytes_are(v[i].iov_base, v[i].iov_len, + 0x6b); + ok &= !msync(p, len, MS_SYNC) && + pread(mem, q, ps, 3 * ps) == (ssize_t)ps; + ok &= bytes_are(q, 1024, 0x6b) && q[1024] == 0x18 && + p[len - 1] == 0x6b; + } + } + ksft_test_result( + ok, "registered shared-file quarters use physical offsets\n"); + if (ok) { + struct iovec replacement = { q + 512, 3 * ps }; + struct io_uring_rsrc_update2 update = { + .data = (uintptr_t)&replacement, .nr = 1 + }; + memset(q, 0x9d, 4 * ps); + ok = syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS_UPDATE, &update, + sizeof(update)) == 1; + ok &= request(&r, IORING_OP_WRITE_FIXED, fd, q + 512, 3 * ps, + 16 * 1024 * 1024, true); + memset(q + 512, 0, 3 * ps); + ok &= request(&r, IORING_OP_READ_FIXED, fd, q + 512, 3 * ps, + 16 * 1024 * 1024, true) && + bytes_are(q + 512, 3 * ps, 0x9d); + } + if (live) { + ok &= syscall(__NR_io_uring_register, r.fd, + IORING_UNREGISTER_BUFFERS, NULL, 0) == 0; + live = false; + } + ok &= pinned_kb() == baseline; + ksft_test_result( + ok, + "registered-buffer update transfers ownership and balances accounting\n"); + if (mem >= 0) + close(mem); + if (live) + syscall(__NR_io_uring_register, r.fd, IORING_UNREGISTER_BUFFERS, + NULL, 0); + destroy(&r); + if (q != MAP_FAILED) + munmap(q, len); +} + +static int exercise(void) +{ + struct ring r = { .fd = -1 }; + size_t ps = getauxval(AT_PAGESZ), len = 2 * 1024 * 1024; + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int fd = open("/dev/user4k-test", O_RDWR | O_DIRECT); + bool ok; + ksft_print_header(); + ksft_set_plan(19); + ksft_print_msg("ABI=%zuK\n", ps / 1024); + ok = setup(&r, 4, 0); + ksft_test_result(ok, "minimum rounded ring mappings\n"); + void *bad = ok ? mmap(NULL, ps, PROT_READ | PROT_WRITE, MAP_SHARED, + r.fd, 4096) : + MAP_FAILED; + bool rejected = ok && bad == MAP_FAILED && errno == EINVAL; + if (bad != MAP_FAILED) + munmap(bad, ps); + ksft_test_result(rejected, + "reject sub-native offset in ring selector\n"); + if (ok) { + ok = true; + for (unsigned i = 0; i < 64 && ok; i++) + ok = request(&r, IORING_OP_NOP, -1, NULL, 0, 0, false); + } + ksft_test_result(ok, "small submission/completion ring wrap\n"); + if (p == MAP_FAILED || fd < 0) + ksft_exit_fail_msg("buffer/device setup errno=%d\n", errno); + if (ok) { + memset(p + ps, 0x69, 3 * ps); + ok = request(&r, IORING_OP_WRITE, fd, p + ps, 3 * ps, 0, false); + memset(p + ps, 0, 3 * ps); + ok &= request(&r, IORING_OP_READ, fd, p + ps, 3 * ps, 0, false); + for (size_t i = ps; i < 4 * ps; i++) + ok &= p[i] == 0x69; + } + ksft_test_result(ok, + "direct read/write from unregistered user buffers\n"); + if (ok) { + memset(p + ps, 0x37, 3 * ps); + ok = request(&r, IORING_OP_WRITE, fd, p + ps, 3 * ps, 65536, + true); + memset(p + ps, 0, 3 * ps); + ok &= request(&r, IORING_OP_READ, fd, p + ps, 3 * ps, 65536, + true); + for (size_t i = ps; i < 4 * ps; i++) + ok &= p[i] == 0x37; + } + ksft_test_result(ok, "io-worker forced asynchronous read/write\n"); + if (ok) { + memset(p, 0xb6, len); + ok = request(&r, IORING_OP_WRITE, fd, p, len, 4 * 1024 * 1024, + true); + memset(p, 0, len); + ok &= request(&r, IORING_OP_READ, fd, p, len, 4 * 1024 * 1024, + true); + for (size_t i = 0; i < len; i++) + ok &= p[i] == 0xb6; + } + ksft_test_result(ok, "multi-bio io-worker transfer\n"); + destroy(&r); + ok = setup(&r, 4096, 0); + if (ok) + for (unsigned i = 0; i < 8193 && ok; i++) + ok = request(&r, IORING_OP_NOP, -1, NULL, 0, 0, false); + ksft_test_result(ok, "large ring crosses native pages and wraps\n"); + destroy(&r); + ok = setup(&r, 64, IORING_SETUP_SQE128 | IORING_SETUP_CQE32); + if (ok) + for (unsigned i = 0; i < 129 && ok; i++) + ok = request(&r, IORING_OP_NOP, -1, NULL, 0, 0, false); + ksft_test_result(ok, "128-byte SQEs and 32-byte CQEs\n"); + destroy(&r); + fixed_checks(fd, p, len, ps); + close(fd); + munmap(p, len); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} +int main(int argc, char **argv) +{ + bool ok = true; + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && !strcmp(argv[1], "--test") && getppid() == 1) + return exercise(); + if (getpid() != 1 || argc != 1) + return 2; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + FILE *dev = fopen("/sys/class/block/vda/dev", "r"); + unsigned maj, min; + if (!dev || fscanf(dev, "%u:%u", &maj, &min) != 2) + return 4; + fclose(dev); + if (mknod("/dev/user4k-test", S_IFBLK | 0600, makedev(maj, min))) + return 5; + for (unsigned small = 0; small < 2; small++) { + pid_t pid = fork(); + int status = 0; + if (!pid) { + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + small ? 4096UL : 0UL, 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--test", NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + printf("%s - io_uring fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/user-events-init.c b/tools/testing/selftests/arm64/user4k/user-events-init.c new file mode 100644 index 0000000000000..4b35954e2a210 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/user-events-init.c @@ -0,0 +1,268 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define TRACE "/sys/kernel/tracing/" +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) + +static int toggle(const char *name, int enabled) +{ + char path[192]; + snprintf(path, sizeof(path), TRACE "events/user_events/%s/enable", + name); + int fd = open(path, O_WRONLY); + if (fd < 0) + return -1; + int ret = write(fd, enabled ? "1" : "0", 1) == 1 ? 0 : -1; + close(fd); + return ret; +} +static uint64_t value(void *p, unsigned bytes) +{ + return bytes == 4 ? __atomic_load_n((uint32_t *)p, __ATOMIC_ACQUIRE) : + __atomic_load_n((uint64_t *)p, __ATOMIC_ACQUIRE); +} +static int wait_value(void *p, unsigned bytes, uint64_t expected) +{ + for (unsigned i = 0; i < 3000; i++) { + if (value(p, bytes) == expected) + return 1; + usleep(1000); + } + return 0; +} +static int check_bytes(unsigned char *p, size_t size, unsigned char fill, + void *word, unsigned bytes) +{ + for (size_t i = 0; i < size; i++) + if ((!word || (uintptr_t)(p + i) < (uintptr_t)word || + (uintptr_t)(p + i) >= (uintptr_t)word + bytes) && + p[i] != fill) { + printf("byte mismatch at %zu got=%#x expected=%#x\n", i, + p[i], fill); + return 0; + } + return 1; +} +static int event_recorded(const char *name) +{ + char buf[16384]; + size_t n = 0; + int fd = open(TRACE "trace", O_RDONLY); + if (fd < 0) + return 0; + while (n < sizeof(buf) - 1) { + ssize_t got = read(fd, buf + n, sizeof(buf) - 1 - n); + if (got < 0) { + close(fd); + return 0; + } + if (!got) + break; + n += got; + } + close(fd); + buf[n] = 0; + char *record = strstr(buf, name), + *payload = record ? strstr(record, "payload=") : NULL; + int ok = payload && strtoul(payload + 8, NULL, 0) == 0x12345678; + if (!ok) + printf("trace mismatch:\n%s\n", buf); + return ok; +} +static int exercise_case(size_t ps, unsigned bytes, unsigned shared) +{ + const size_t len = 4 * ps; + unsigned char *src, *dst; + int memfd = -1; + if (shared) { + memfd = memfd_create("user-event-bitmap", 0); + CHECK(memfd >= 0); + CHECK(!ftruncate(memfd, len)); + } + src = mmap((void *)0x20000000, len, PROT_READ | PROT_WRITE, + (shared ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS) | + MAP_FIXED_NOREPLACE, + memfd, 0); + dst = mmap((void *)0x30000000, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(src != MAP_FAILED && dst != MAP_FAILED); + for (unsigned i = 0; i < 4; i++) + memset(src + i * ps, 0x61 + i, ps); + memset(dst, 0x91, len); + CHECK(mremap(src + 3 * ps, ps, ps, MREMAP_MAYMOVE | MREMAP_FIXED, + dst + ps) == dst + ps); + unsigned char *leaf = dst + ps; + void *word = leaf + ps - bytes; + uint64_t mask = bytes == 4 ? 1ULL << 31 : 1ULL << 63; + memset(word, 0, bytes); + char name[80], spec[120]; + snprintf(name, sizeof(name), "aurora_ue_%zu_%u_%u", ps, bytes, shared); + snprintf(spec, sizeof(spec), "%s u32 payload", name); + int tracefd = open(TRACE "trace", O_WRONLY | O_TRUNC); + CHECK(tracefd >= 0); + close(tracefd); + int fd = open(TRACE "user_events_data", O_RDWR); + CHECK(fd >= 0); + struct user_reg reg = { .size = sizeof(reg), + .enable_bit = bytes * 8 - 1, + .enable_size = bytes, + .enable_addr = (uintptr_t)word, + .name_args = (uintptr_t)spec }; + CHECK(!ioctl(fd, DIAG_IOCSREG, ®)); + CHECK(wait_value(word, bytes, 0)); + CHECK(!toggle(name, 1)); + CHECK(wait_value(word, bytes, mask)); + struct { + uint32_t index, payload; + } event = { reg.write_index, 0x12345678 }; + CHECK(write(fd, &event, sizeof(event)) == sizeof(event)); + CHECK(event_recorded(name)); + CHECK(!toggle(name, 0)); + CHECK(wait_value(word, bytes, 0)); + /* Every other byte, including the adjacent half of the kernel word, is stable. */ + CHECK(check_bytes(leaf, ps, 0x64, word, bytes)); + CHECK(check_bytes(dst, ps, 0x91, NULL, 0)); + CHECK(check_bytes(dst + 2 * ps, 2 * ps, 0x91, NULL, 0)); + int commands[2], reply[2]; + CHECK(!pipe(commands) && !pipe(reply)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + close(commands[1]); + close(reply[0]); + char token; + int ok = 1; + for (unsigned round = 0; round < 32; round++) { + if (read(commands[0], &token, 1) != 1) { + ok = 0; + break; + } + ok = wait_value(word, bytes, token == '1' ? mask : 0) && + check_bytes(leaf, ps, 0x64, word, bytes); + token = ok ? 'y' : 'n'; + if (write(reply[1], &token, 1) != 1 || !ok) + break; + } + _exit(ok ? 0 : 3); + } + close(commands[0]); + close(reply[1]); + int ok = 1; + for (unsigned round = 0; round < 32; round++) { + int on = !(round & 1); + char token = on ? '1' : '0'; + if (toggle(name, on) || + !wait_value(word, bytes, on ? mask : 0) || + write(commands[1], &token, 1) != 1 || + read(reply[0], &token, 1) != 1 || token != 'y') { + ok = 0; + break; + } + } + close(commands[1]); + close(reply[0]); + int status; + CHECK(waitpid(child, &status, 0) == child); + CHECK(ok && WIFEXITED(status) && WEXITSTATUS(status) == 0); + CHECK(check_bytes(leaf, ps, 0x64, word, bytes)); + for (unsigned i = 0; i < 3; i++) + CHECK(check_bytes(src + i * ps, ps, 0x61 + i, NULL, 0)); + /* Force the asynchronous missing-PTE fixup without unregistering. */ + CHECK(!madvise(leaf, ps, MADV_DONTNEED)); + CHECK(!toggle(name, 1)); + CHECK(wait_value(word, bytes, mask)); + CHECK(check_bytes(leaf, ps, shared ? 0x64 : 0, word, bytes)); + CHECK(!toggle(name, 0)); + CHECK(wait_value(word, bytes, 0)); + struct user_unreg unreg = { .size = sizeof(unreg), + .disable_bit = reg.enable_bit, + .disable_addr = (uintptr_t)word }; + CHECK(!ioctl(fd, DIAG_IOCSUNREG, &unreg)); + close(fd); + CHECK(!munmap(src, 3 * ps)); + CHECK(!munmap(dst, len)); + if (memfd >= 0) + close(memfd); + printf("ok - %zuK %u-bit %s relocated bitmap, adjacent bytes, event write and fork toggles\n", + ps / 1024, bytes * 8, shared ? "shared" : "private"); + return 0; +} +static int exercise(size_t expected) +{ + size_t ps = getauxval(AT_PAGESZ); + CHECK(ps == expected); + for (unsigned shared = 0; shared < 2; shared++) + for (unsigned bytes = 4; bytes <= 8; bytes += 4) + CHECK(!exercise_case(ps, bytes, shared)); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("tracefs", TRACE, "tracefs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + size_t sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned i = 0; i < (sizes[0] > 16384 ? 3U : 2U); i++) { + pid_t c = fork(); + int status = 0; + if (!c) { + char expected[24]; + snprintf(expected, sizeof(expected), "%zu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", expected, NULL); + _exit(5); + } + int ok = c > 0 && waitpid(c, &status, 0) == c && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - ABI %zuK user events status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + fail += !ok; + } +done: + printf("USER EVENTS %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/user16k-smoke-init.c b/tools/testing/selftests/arm64/user4k/user16k-smoke-init.c new file mode 100644 index 0000000000000..b0a2f2bf385f0 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/user16k-smoke-init.c @@ -0,0 +1,229 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define GET_EXEC_PAGE_SIZE 0x41555002 +static int failed; +static _Thread_local int tls = 7; +static void result(int ok, const char *name) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", name); + failed += !ok; +} +static long pss(void *p) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char *line = NULL; + size_t cap = 0; + unsigned long start, end; + int active = 0; + long value = -1; + if (!f) + return -1; + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) + active = (unsigned long)p >= start && + (unsigned long)p < end; + else if (active && sscanf(line, "Pss: %ld kB", &value) == 1) + break; + } + free(line); + fclose(f); + return value; +} +static void *thread_worker(void *arg) +{ + unsigned char *p = arg; + if (tls != 7) + return (void *)1; + tls = 91; + for (unsigned i = 0; i < 16384; i++) + if (p[i] != 0x42) + return (void *)2; + return NULL; +} +static void memory_checks(void) +{ + const size_t ps = 16384; + int status = 0; + pid_t c; + unsigned char *p = mmap(NULL, 6 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + result(p != MAP_FAILED, "16K anonymous mmap"); + if (p == MAP_FAILED) + return; + for (unsigned i = 0; i < 6; i++) + memset(p + i * ps, 0x40 + i, ps); + c = fork(); + if (!c) { + memset(p + ps, 0xb7, ps); + _exit(p[0] == 0x40 && p[2 * ps] == 0x42 ? 0 : 1); + } + int ok = c > 0 && waitpid(c, &status, 0) == c && WIFEXITED(status) && + !WEXITSTATUS(status); + for (unsigned i = 0; i < 6 * ps; i++) + ok &= p[i] == 0x40 + i / ps; + result(ok, "16K fork COW preserves every parent byte"); + result(!mprotect(p + ps, ps, PROT_READ), "16K leaf write protection"); + c = fork(); + if (!c) { + *(volatile unsigned char *)(p + ps + 8192) = 3; + _exit(1); + } + result(c > 0 && waitpid(c, &status, 0) == c && WIFSIGNALED(status) && + WTERMSIG(status) == SIGSEGV, + "write protection covers the interior of a16K leaf"); + result(!mprotect(p + ps, ps, PROT_READ | PROT_WRITE), + "16K write permission restored"); + errno = 0; + result(mprotect(p + 4096, ps, PROT_READ) == -1 && errno == EINVAL, + "16K ABI rejects4K-only protection alignment"); + result(!madvise(p + ps, ps, MADV_DONTNEED), + "16K partial unmap via DONTNEED"); + ok = 1; + for (unsigned i = 0; i < ps; i++) + ok &= p[i] == 0x40 && p[ps + i] == 0 && p[2 * ps + i] == 0x42; + result(ok, "16K discard clears one leaf and preserves neighbours"); + pthread_t thread; + void *answer = (void *)3; + int err = pthread_create(&thread, NULL, thread_worker, p + 2 * ps); + if (!err) + err = pthread_join(thread, &answer); + result(!err && !answer && tls == 7, "16K pthread stack and TLS"); + struct timespec ts; + result(!clock_gettime(CLOCK_MONOTONIC, &ts), "16K vDSO clock access"); + munmap(p, 6 * ps); +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (argc == 6 && !strcmp(argv[1], "--child")) { + int fd = atoi(argv[2]), ready = atoi(argv[3]), + release = atoi(argv[4]); + size_t native = strtoul(argv[5], NULL, 10); + const size_t ps = 16384; + result(getauxval(AT_PAGESZ) == ps && getpagesize() == ps, + "EL0 selects16K pages"); + result(prctl(GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL) == 0, + "exec consumes alternative selection"); + if (!failed) + memory_checks(); + { + int status = 0; + pid_t c = fork(); + if (!c) { + if (prctl(SET_EXEC_PAGE_SIZE, 16384UL, 0UL, 0UL, + 0UL)) + _exit(4); + execl("/page-contract", "/page-contract", NULL); + _exit(5); + } + result(c > 0 && waitpid(c, &status, 0) == c && + WIFEXITED(status) && + !WEXITSTATUS(status), + "16K-to16K exec and page/x18 contract"); + } + unsigned char *shared = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, native - ps); + unsigned char *private = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE, fd, native - ps); + int ok = shared != MAP_FAILED && private != MAP_FAILED; + if (ok) { + for (unsigned i = 0; i < ps; i++) + ok &= shared[i] == 0x7b && private[i] == 0x7b; + memset(private, 0xd8, ps); + for (unsigned i = 0; i < ps; i++) + ok &= shared[i] == 0x7b && private[i] == 0xd8; + } + result(ok, "16K file COW copies the whole leaf"); + if (private != MAP_FAILED) + munmap(private, ps); + result(shared != MAP_FAILED && pss(shared) == 8, + "16K file PSS shares all four4K slices"); + if (shared != MAP_FAILED) + shared[ps - 1] = 0xc9; + char token = failed ? 'x' : 'a'; + if (write(ready, &token, 1) != 1 || + read(release, &token, 1) != 1) + return 3; + if (shared != MAP_FAILED) + munmap(shared, ps); + return failed ? 1 : 0; + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + mount("proc", "/proc", "proc", 0, NULL); + size_t native = getauxval(AT_PAGESZ); + int fd = memfd_create("mixed16-smoke", 0); + if (native != 65536 || fd < 0 || ftruncate(fd, native)) { + result(0, "64K parent setup"); + goto done; + } + unsigned char *p = + mmap(NULL, native, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + result(0, "parent shared mmap"); + goto done; + } + memset(p, 0x7b, native); + result(pss(p) == 64, "native64K initial PSS"); + int ready[2], release[2]; + if (pipe(ready) || pipe(release)) { + result(0, "pipes"); + goto done; + } + pid_t c = fork(); + int status = 0; + if (!c) { + char f[24], r[24], w[24], n[24]; + snprintf(f, sizeof(f), "%d", fd); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(w, sizeof(w), "%d", release[0]); + snprintf(n, sizeof(n), "%zu", native); + close(ready[0]); + close(release[1]); + if (prctl(SET_EXEC_PAGE_SIZE, 16384UL, 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--child", f, r, w, n, NULL); + _exit(5); + } + close(ready[1]); + close(release[0]); + char token = 0; + int ok = c > 0 && read(ready[0], &token, 1) == 1 && token == 'a'; + result(ok && p[native - 1] == 0xc9, + "native64K and16K shared bytes remain coherent"); + result(pss(p) == 56, + "native64K PSS accounts one16K alias as four shared slices"); + token = 'b'; + if (write(release[1], &token, 1) != 1) + failed++; + result(c > 0 && waitpid(c, &status, 0) == c && WIFEXITED(status) && + !WEXITSTATUS(status), + "16K child completes"); + printf("# child status=%#x\n", status); + result(pss(p) == 64, "native PSS returns after16K exit"); + munmap(p, native); +done: + printf("USER16K SMOKE %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c b/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c new file mode 100644 index 0000000000000..c43e28ca8b21d --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c @@ -0,0 +1,542 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#define __user +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/vfio.h" +#include "../../../../../drivers/vfio/vfio_iommu_type1_test.h" + +#define IOVA (256UL * 1024 * 1024) +static unsigned int number, failures; + +static void result(int ok, const char *name) +{ + printf("%s %u - %s (errno=%d)\n", ok ? "ok" : "not ok", ++number, name, + errno); + failures += !ok; +} + +static int state(int fd, struct vfio_type1_test_cmd *cmd) +{ + memset(cmd, 0, sizeof(*cmd)); + return ioctl(fd, VFIO_TYPE1_TEST_STATE, cmd); +} + +static int domain(int fd, unsigned int which, int add, size_t page, size_t fail) +{ + struct vfio_type1_test_cmd cmd = { + .domain = which, + .size = page, + .fail_after = fail, + }; + + return ioctl(fd, + add ? VFIO_TYPE1_TEST_DOMAIN_ADD : + VFIO_TYPE1_TEST_DOMAIN_DEL, + &cmd); +} + +static int rw(int fd, size_t offset, void *data, size_t size, + unsigned int flags) +{ + struct vfio_type1_test_cmd cmd = { + .iova = IOVA + offset, + .size = size, + .buffer = (uintptr_t)data, + .flags = flags, + }; + + return ioctl(fd, VFIO_TYPE1_TEST_RW, &cmd); +} + +static int map(int fd, void *data, size_t size) +{ + struct vfio_iommu_type1_dma_map cmd = { + .argsz = sizeof(cmd), + .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, + .iova = IOVA, + .vaddr = (uintptr_t)data, + .size = size, + }; + + return ioctl(fd, VFIO_IOMMU_MAP_DMA, &cmd); +} + +static int unmap(int fd, size_t size) +{ + struct vfio_iommu_type1_dma_unmap cmd = { + .argsz = sizeof(cmd), + .iova = IOVA, + .size = size, + }; + int ret = ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &cmd); + + return ret ? ret : cmd.size == size ? 0 : -2; +} + +static int dirty(int fd, unsigned int flags) +{ + struct vfio_iommu_type1_dirty_bitmap cmd = { .argsz = sizeof(cmd), + .flags = flags }; + + return ioctl(fd, VFIO_IOMMU_DIRTY_PAGES, &cmd); +} + +static int bitmap(int fd, size_t size, uint64_t *bits) +{ + unsigned char + buffer[sizeof(struct vfio_iommu_type1_dirty_bitmap) + + sizeof(struct vfio_iommu_type1_dirty_bitmap_get)] = {}; + struct vfio_iommu_type1_dirty_bitmap *cmd = (void *)buffer; + struct vfio_iommu_type1_dirty_bitmap_get *range = (void *)cmd->data; + + cmd->argsz = sizeof(buffer); + cmd->flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP; + range->iova = IOVA; + range->size = size; + range->bitmap.pgsize = 4096; + range->bitmap.size = sizeof(*bits); + range->bitmap.data = (void *)bits; + *bits = 0; + return ioctl(fd, VFIO_IOMMU_DIRTY_PAGES, buffer); +} + +static int all_bytes(const unsigned char *data, size_t size, + unsigned char value) +{ + for (size_t i = 0; i < size; i++) + if (data[i] != value) + return 0; + return 1; +} + +static void external_span(int backing, size_t native) +{ + const size_t fine = 4096, count = native / fine; + uint64_t bits; + + for (unsigned int mapped = 0; mapped < 2; mapped++) { + struct vfio_type1_test_cmd cmd; + struct vfio_iommu_type1_dma_unmap remove = { + .argsz = sizeof(remove), + .iova = IOVA, + .size = fine, + }; + int fd = open("/dev/vfio-test", O_RDWR), status = 0; + pid_t child; + + result(fd >= 0 && (!mapped || !domain(fd, 0, 1, fine, 0)), + "span fixture opens lazy or mapped type1 container"); + if (fd < 0) + continue; + result(!dirty(fd, VFIO_IOMMU_DIRTY_PAGES_FLAG_START), + "span dirty tracking starts before fine mappings"); + child = fork(); + if (!child) { + unsigned char *data = mmap(NULL, native, + PROT_READ | PROT_WRITE, + MAP_SHARED, backing, 0); + + if (data == MAP_FAILED) + _exit(1); + for (size_t i = 0; i < count; i++) { + struct vfio_iommu_type1_dma_map area = { + .argsz = sizeof(area), + .flags = VFIO_DMA_MAP_FLAG_READ | + VFIO_DMA_MAP_FLAG_WRITE, + .iova = IOVA + i * fine, + .vaddr = (uintptr_t)data + i * fine, + .size = fine, + }; + + if (ioctl(fd, VFIO_IOMMU_MAP_DMA, &area)) + _exit(2); + } + cmd = (struct vfio_type1_test_cmd){ .iova = IOVA + 17 }; + _exit(ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) ? 3 : 0); + } + result(child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status), + "source pins native page across independent fine MAP ioctls"); + result(!state(fd, &cmd) && !cmd.owner_users && + cmd.external_refs == 1 && + cmd.locked_bytes == count * native && + cmd.owner_locked_bytes == count * native, + "every fine DMA area retains its own charge after source exit"); + cmd = (struct vfio_type1_test_cmd){ .iova = IOVA + 31 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) && + !state(fd, &cmd) && cmd.external_refs == 2 && + cmd.locked_bytes == count * native, + "cross-area duplicate pin reuses all cached constituent pins"); + result(!bitmap(fd, native, &bits) && + bits == ((1ULL << count) - 1), + "cross-area dirty bitmap marks only each mapping's fine extent"); + result(!ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &remove) && + remove.size == fine && !state(fd, &cmd) && + !cmd.external_refs && cmd.notifications == 1 && + cmd.locked_bytes == + (mapped ? (count - 1) * native : 0), + "unmapping one constituent drains references in all page constituents"); + remove.iova = IOVA + fine; + remove.size = native - fine; + result(!ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &remove) && + remove.size == native - fine && + !state(fd, &cmd) && !cmd.locked_bytes && + !cmd.mapped_bytes[0], + "remaining fine mappings unmap without pins or translations"); + close(fd); + } +} + +static void native_owner(int fd, int backing, unsigned char *data, + size_t native) +{ + struct vfio_type1_test_cmd cmd = {}; + struct vfio_iommu_type1_dma_unmap invalidate = { + .argsz = sizeof(invalidate), + .flags = VFIO_DMA_UNMAP_FLAG_VADDR, + .iova = IOVA, + .size = 2 * native, + }; + struct vfio_iommu_type1_dma_map update = { + .argsz = sizeof(update), + .flags = VFIO_DMA_MAP_FLAG_VADDR, + .iova = IOVA, + .size = 2 * native, + .vaddr = (uintptr_t)data, + }; + struct __user_cap_header_struct header = { + .version = _LINUX_CAPABILITY_VERSION_3 + }; + struct __user_cap_data_struct saved[2], reduced[2]; + struct rlimit limit, empty; + unsigned char bytes[64]; + int file_flags = fcntl(backing, F_GETFD), status = 0; + pid_t child; + int budget_ready, budget_captured, transferred; + + result(!ioctl(fd, VFIO_TYPE1_TEST_EMULATED, &cmd) && + !domain(fd, 0, 1, 4096, 0), + "VADDR fixture retains production group restrictions"); + result(file_flags >= 0 && + !fcntl(backing, F_SETFD, file_flags & ~FD_CLOEXEC), + "native source inherits exact backing descriptor"); + child = fork(); + if (!child) { + char fd_arg[24], file_arg[24], native_arg[24]; + + snprintf(fd_arg, sizeof(fd_arg), "%d", fd); + snprintf(file_arg, sizeof(file_arg), "%d", backing); + snprintf(native_arg, sizeof(native_arg), "%zu", native); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL)) + _exit(98); + execl("/init", "/init", "--native-source", fd_arg, file_arg, + native_arg, NULL); + _exit(99); + } + if (file_flags >= 0) + fcntl(backing, F_SETFD, file_flags); + result(child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && + !state(fd, &cmd) && !cmd.owner_users && + !(cmd.flags & VFIO_TYPE1_TEST_STATE_FRAGMENTS) && + cmd.locked_bytes == 2 * native && + cmd.mapped_bytes[0] == 2 * native, + "exited native source owns legacy GUP pins before transfer"); + result(!ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &invalidate), + "canonical VADDR invalidation leaves device translations alive"); + budget_captured = !getrlimit(RLIMIT_MEMLOCK, &limit) && + !syscall(SYS_capget, &header, saved); + budget_ready = budget_captured; + if (budget_ready) { + memcpy(reduced, saved, sizeof(saved)); + reduced[CAP_IPC_LOCK / 32].effective &= + ~(1U << (CAP_IPC_LOCK % 32)); + empty = (struct rlimit){ .rlim_cur = 0, + .rlim_max = limit.rlim_max }; + budget_ready = !setrlimit(RLIMIT_MEMLOCK, &empty) && + !syscall(SYS_capset, &header, reduced); + } + errno = 0; + result(budget_ready && ioctl(fd, VFIO_IOMMU_MAP_DMA, &update) < 0 && + errno == ENOMEM && !state(fd, &cmd) && + !cmd.owner_users && + !(cmd.flags & VFIO_TYPE1_TEST_STATE_FRAGMENTS) && + cmd.locked_bytes == 2 * native && + cmd.mapped_bytes[0] == 2 * native, + "failed owner credit preserves native pins, metadata and charge"); + if (budget_captured) { + result(!syscall(SYS_capset, &header, saved) && + !setrlimit(RLIMIT_MEMLOCK, &limit), + "restore original memlock budget and effective capability"); + } else { + result(0, "owner budget setup failed"); + } + transferred = !ioctl(fd, VFIO_IOMMU_MAP_DMA, &update) && + !state(fd, &cmd) && cmd.owner_users && + (cmd.flags & VFIO_TYPE1_TEST_STATE_FRAGMENTS) && + cmd.locked_bytes == 2 * native && + cmd.owner_locked_bytes == 2 * native; + result(transferred, + "canonical VADDR retry adopts native pins for alternative owner"); + /* Dependent CPU access is prohibited while VADDR remains invalid. */ + if (!transferred) + return; + result(!rw(fd, 0, bytes, sizeof(bytes), 0) && + all_bytes(bytes, sizeof(bytes), 0x64), + "new owner accesses retained native buffer with its own mm authorization"); + cmd = (struct vfio_type1_test_cmd){ .flags = 1 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_EMULATED, &cmd), + "external consumer rejoins after VADDR update"); + cmd = (struct vfio_type1_test_cmd){ .iova = IOVA + 17 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) && !state(fd, &cmd) && + cmd.locked_bytes == 2 * native && cmd.external_refs == 1, + "external page array reuses adopted pins without new charge"); + result(!unmap(fd, 2 * native) && !state(fd, &cmd) && + !cmd.locked_bytes && !cmd.owner_locked_bytes && + !cmd.external_refs && !cmd.mapped_bytes[0], + "owner transfer final cleanup refunds recipient and clears domain"); + result(!domain(fd, 0, 0, 0, 0), "owner transfer domain detaches"); +} + +static int exercise(size_t native) +{ + struct vfio_type1_test_cmd cmd; + struct vfio_iommu_type1_info info = { .argsz = sizeof(info) }; + struct vfio_iommu_type1_dma_map bad = { .argsz = 8 }; + unsigned char bytes[64], disk[64]; + uint64_t bits = 0; + int fd = open("/dev/vfio-test", O_RDWR), + backing = memfd_create("vfio", MFD_CLOEXEC); + unsigned char *data = MAP_FAILED; + int status = 0; + pid_t child; + size_t length = native + 4096; + + printf("TAP version 13\n# real type1 ioctl ABI=%luK native=%zuK\n", + getauxval(AT_PAGESZ) / 1024, native / 1024); + result(fd >= 0 && backing >= 0 && !ftruncate(backing, 2 * native), + "open privileged recording-domain fixture and backing"); + if (fd < 0 || backing < 0 || failures) + goto out; + data = mmap(NULL, 2 * native, PROT_READ | PROT_WRITE, MAP_SHARED, + backing, 0); + result(data != MAP_FAILED, + "map native-sized shmem backing in actual process mm"); + if (data == MAP_FAILED) + goto out; + memset(data, 0x64, 2 * native); + result(!ioctl(fd, VFIO_IOMMU_GET_INFO, &info) && + (info.iova_pgsizes & 4096), + "type1 GET_INFO reports fine IOVA capability"); + errno = 0; + result(ioctl(fd, VFIO_IOMMU_MAP_DMA, &bad) < 0 && errno == EINVAL, + "canonical MAP rejects truncated argument"); + errno = 0; + result(ioctl(fd, VFIO_IOMMU_MAP_DMA, (void *)1) < 0 && errno == EFAULT, + "canonical MAP rejects inaccessible userspace argument"); + result(!map(fd, data, 2 * native), "lazy full-range MAP from real EL0"); + result(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.mapped_bytes[0], + "lazy MAP acquires no DMA pins or translations"); + result(!rw(fd, 0, bytes, sizeof(bytes), 0) && + all_bytes(bytes, sizeof(bytes), 0x64), + "CPU-only GET reads actual source bytes"); + result(!state(fd, &cmd) && !cmd.locked_bytes, + "CPU GET does not charge long-term pins"); + result(!dirty(fd, VFIO_IOMMU_DIRTY_PAGES_FLAG_START), + "canonical dirty tracking starts"); + cmd = (struct vfio_type1_test_cmd){ .iova = IOVA + 17 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) && !state(fd, &cmd) && + cmd.locked_bytes == native && cmd.external_refs == 1, + "external page array preserves unaligned caller byte offset"); + cmd = (struct vfio_type1_test_cmd){ .iova = IOVA + 31 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) && !state(fd, &cmd) && + cmd.locked_bytes == native && cmd.external_refs == 2, + "duplicate external pin charges physical native page once"); + result(!bitmap(fd, 2 * native, &bits) && + bits == ((1ULL << (native / 4096)) - 1), + "dirty bitmap covers every fine unit in native external page"); + result(!unmap(fd, 2 * native), + "canonical UNMAP drains callback-held external refs"); + result(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.external_refs && + cmd.notifications == 1, + "external-unpin callback refunds all pins"); + result(!dirty(fd, VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP), + "dirty tracking stops"); + result(!domain(fd, 0, 1, 4096, 0), + "recording4K domain attaches before child MAP"); + child = fork(); + if (!child) { + unsigned char *source = mmap(NULL, 2 * native, + PROT_READ | PROT_WRITE, MAP_SHARED, + backing, 0); + int ret = source == MAP_FAILED ? -1 : + map(fd, source + 4096, length); + + _exit(ret ? 1 : 0); + } + result(child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status), + "fine-offset MAP retains real child pins after exit"); + result(!state(fd, &cmd) && !cmd.owner_users && + cmd.locked_bytes == 2 * native && + cmd.owner_locked_bytes == 2 * native && + cmd.mapped_bytes[0] == length, + "exited source mm retains exact physical-page charge and translation length"); + errno = 0; + result(rw(fd, 0, bytes, sizeof(bytes), 0) < 0 && errno == EFAULT, + "foreign process CPU access preserves source-mm authorization"); + result(!rw(fd, 0, bytes, sizeof(bytes), VFIO_TYPE1_TEST_WORKER) && + all_bytes(bytes, sizeof(bytes), 0x64), + "kernel consumer reads exited source pins"); + errno = 0; + result(domain(fd, 1, 1, 4096, 4096) < 0 && errno == ENOMEM && + !state(fd, &cmd) && !cmd.mapped_bytes[1] && + cmd.mapped_bytes[0] == length && + cmd.locked_bytes == 2 * native, + "failed replay leaves existing domain and pins intact"); + result(!domain(fd, 1, 1, 4096, 0) && !state(fd, &cmd) && + cmd.mapped_bytes[0] == length && + cmd.mapped_bytes[1] == length && + cmd.locked_bytes == 2 * native, + "retry replays cached pins without resurrecting source mm"); + result(!dirty(fd, VFIO_IOMMU_DIRTY_PAGES_FLAG_START), + "fine DMA dirty tracking starts"); + memset(bytes, 0x9a, sizeof(bytes)); + result(!rw(fd, 4096 - 32, bytes, sizeof(bytes), + VFIO_TYPE1_TEST_WRITE | VFIO_TYPE1_TEST_WORKER) && + pread(backing, disk, sizeof(disk), 8192 - 32) == + sizeof(disk) && + all_bytes(disk, sizeof(disk), 0x9a), + "cached worker write dirties original shmem bytes"); + result(!bitmap(fd, length, &bits) && bits == 3, + "canonical bitmap returns exact two4K units touched by worker write"); + result(!unmap(fd, length), + "fine UNMAP synchronizes both recording domains"); + result(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.mapped_bytes[0] && + !cmd.mapped_bytes[1], + "source-exit cleanup refunds pins and clears all translations"); + result(!domain(fd, 1, 0, 0, 0) && !domain(fd, 0, 0, 0, 0), + "both domains detach after complete cleanup"); + external_span(backing, native); + if (getauxval(AT_PAGESZ) != native) + native_owner(fd, backing, data, native); +out: + if (data != MAP_FAILED) + munmap(data, 2 * native); + if (backing >= 0) + close(backing); + if (fd >= 0) + close(fd); + printf("1..%u\n# type1 ioctl failures=%u\n", number, failures); + return !!failures; +} + +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--native-source")) { + size_t native = strtoul(argv[4], NULL, 10); + int fd = atoi(argv[2]), backing = atoi(argv[3]); + void *data; + + if (getauxval(AT_PAGESZ) != native) + return 2; + data = mmap(NULL, 2 * native, PROT_READ | PROT_WRITE, + MAP_SHARED, backing, 0); + return data == MAP_FAILED || map(fd, data, 2 * native) ? 1 : 0; + } + if (argc == 4 && !strcmp(argv[1], "--test") && getppid() == 1) { + size_t expected = strtoul(argv[3], NULL, 10); + + if (getauxval(AT_PAGESZ) != expected) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) + return 2; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + FILE *dev = fopen("/sys/class/misc/aurora-vfio-type1-test/dev", "r"); + unsigned int major, minor; + + if (!dev || fscanf(dev, "%u:%u", &major, &minor) != 2) + return 4; + fclose(dev); + if (mknod("/dev/vfio-test", S_IFCHR | 0600, makedev(major, minor))) + return 5; + size_t native = getauxval(AT_PAGESZ), + targets[] = { native, 4096, 16384 }; + int ok = 1; + + int compat_only = getenv("VFIO_COMPAT_ONLY") != NULL; + + for (unsigned int i = 0; i < 3; i++) { + if (compat_only) + break; + size_t target = targets[i]; + pid_t child; + int status = 0; + + if (target > native || (i && target == native)) + continue; + child = fork(); + if (!child) { + char native_arg[24], target_arg[24]; + + snprintf(native_arg, sizeof(native_arg), "%zu", native); + snprintf(target_arg, sizeof(target_arg), "%zu", target); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + target == native ? 0UL : target, 0UL, 0UL, + 0UL)) + _exit(3); + execl("/init", "/init", "--test", native_arg, + target_arg, NULL); + _exit(4); + } + ok &= child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("# actual ABI=%zuK child_status=%d\n", target / 1024, + status); + } + const char *expected_arg = getenv("VFIO_COMPAT_PAGES"); + size_t expected = expected_arg ? strtoul(expected_arg, NULL, 10) : 4096; + pid_t compat = fork(); + int compat_status = 0; + + if (!compat) { + char native_arg[24], page_arg[24]; + + snprintf(native_arg, sizeof(native_arg), "%zu", native); + snprintf(page_arg, sizeof(page_arg), "%zu", expected); + execl("/compat-vfio", "/compat-vfio", native_arg, page_arg, + NULL); + _exit(98); + } + ok &= compat > 0 && waitpid(compat, &compat_status, 0) == compat && + WIFEXITED(compat_status) && !WEXITSTATUS(compat_status); + printf("# AArch32 expected=%zuK child_status=%d\n", expected / 1024, + compat_status); + printf("VFIO TYPE1 IOCTL %s\n", ok ? "PASS" : "FAIL"); + reboot(RB_POWER_OFF); + return !ok; +} diff --git a/tools/testing/selftests/arm64/user4k/vhost-log-init.c b/tools/testing/selftests/arm64/user4k/vhost-log-init.c new file mode 100644 index 0000000000000..07d6a8969bca0 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/vhost-log-init.c @@ -0,0 +1,392 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#ifndef VHOST_SET_FORK_FROM_OWNER +#define VHOST_SET_FORK_FROM_OWNER _IOW(VHOST_VIRTIO, 0x84, __u8) +#define VHOST_GET_FORK_FROM_OWNER _IOR(VHOST_VIRTIO, 0x85, __u8) +#endif +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) +#define N 8 +#define ROUNDS 32 +#define PACKET_SIZE 96 +#define PROTOCOL 0x88b5 +#define DATA_GPA (51ULL * 4096) +#define TX_USED_GPA (71ULL * 4096) +#define RX_USED_GPA (70ULL * 4096) + +struct queue { + struct vring_desc desc[N]; + struct { + uint16_t flags, idx, ring[N + 1]; + } avail; + struct { + uint16_t flags, idx; + struct vring_used_elem ring[N]; + uint16_t event; + } used; + int kick, call; +}; + +static int queue_setup(int fd, struct queue *q, unsigned index) +{ + struct vhost_vring_state state = { .index = index, .num = N }; + struct vhost_vring_addr addr = { + .index = index, + .flags = 1U << VHOST_VRING_F_LOG, + .desc_user_addr = (uintptr_t)q->desc, + .avail_user_addr = (uintptr_t)&q->avail, + .used_user_addr = (uintptr_t)&q->used, + .log_guest_addr = index ? TX_USED_GPA : RX_USED_GPA, + }; + q->kick = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); + q->call = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); + CHECK(q->kick >= 0 && q->call >= 0); + CHECK(!ioctl(fd, VHOST_SET_VRING_NUM, &state)); + state.num = 0; + CHECK(!ioctl(fd, VHOST_SET_VRING_BASE, &state)); + CHECK(!ioctl(fd, VHOST_SET_VRING_ADDR, &addr)); + struct vhost_vring_file file = { .index = index, .fd = q->kick }; + CHECK(!ioctl(fd, VHOST_SET_VRING_KICK, &file)); + file.fd = q->call; + CHECK(!ioctl(fd, VHOST_SET_VRING_CALL, &file)); + return 0; +} +static int submit(struct queue *q, unsigned round) +{ + uint64_t one = 1; + q->avail.ring[round % N] = 0; + __atomic_store_n(&q->avail.idx, round + 1, __ATOMIC_RELEASE); + return write(q->kick, &one, sizeof(one)) == sizeof(one); +} +static int complete(struct queue *q, unsigned round) +{ + for (unsigned i = 0; i < 2000; i++) { + if (__atomic_load_n(&q->used.idx, __ATOMIC_ACQUIRE) == + round + 1) + return 1; + usleep(1000); + } + printf("queue timeout round=%u used=%u avail=%u\n", round, q->used.idx, + q->avail.idx); + return 0; +} +static int bitmap_ok(unsigned char *leaf, size_t ps, int active, int missing) +{ + for (size_t i = 0; i < ps; i++) { + unsigned char expected = i < ps - 9 ? (missing ? 0 : 0x64) : 0; + if (active && i == ps - 3) + expected = 8; /* RX buffer, GPA page 51 */ + if (active && i == ps - 1) + expected = 0xc0; /* RX/TX rings, pages 70/71 */ + if (__atomic_load_n(leaf + i, __ATOMIC_ACQUIRE) != expected) { + printf("bitmap byte %zu got=%#x expected=%#x\n", i, + leaf[i], expected); + return 0; + } + } + return 1; +} +static int filled(unsigned char *p, size_t len, unsigned char byte) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != byte) + return 0; + return 1; +} +static int exercise_case(size_t ps, unsigned shared, unsigned forked, + uint8_t mode) +{ + int backing = -1; + if (shared) { + backing = memfd_create("vhost-log", 0); + CHECK(backing >= 0 && !ftruncate(backing, 4 * ps)); + } + unsigned char *src = mmap( + (void *)0x20000000, 4 * ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | + (shared ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS), + backing, 0); + unsigned char *dst = + mmap((void *)0x30000000, 4 * ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(src != MAP_FAILED && dst != MAP_FAILED); + for (unsigned i = 0; i < 4; i++) + memset(src + i * ps, 0x61 + i, ps); + memset(dst, 0x91, 4 * ps); + CHECK(mremap(src + 3 * ps, ps, ps, MREMAP_MAYMOVE | MREMAP_FIXED, + dst + ps) == dst + ps); + unsigned char *leaf = dst + ps, *bitmap = leaf + ps - 9; + memset(bitmap, 0, 9); + int barrier[2]; + pid_t child = -1; + if (forked) { + CHECK(!pipe(barrier)); + child = fork(); + CHECK(child >= 0); + if (!child) { + close(barrier[1]); + char token; + int ok = read(barrier[0], &token, 1) == 1 && + bitmap_ok(leaf, ps, shared, 0); + _exit(ok ? 0 : 8); + } + close(barrier[0]); + } + int tap = open("/dev/net/tun", O_RDWR | O_CLOEXEC); + CHECK(tap >= 0); + struct ifreq ifr = { .ifr_flags = IFF_TAP | IFF_NO_PI }; + strcpy(ifr.ifr_name, "abi%d"); + CHECK(!ioctl(tap, TUNSETIFF, &ifr)); + int raw = socket(AF_PACKET, SOCK_RAW | SOCK_CLOEXEC, htons(PROTOCOL)); + CHECK(raw >= 0); + CHECK(!ioctl(raw, SIOCGIFINDEX, &ifr)); + int ifindex = ifr.ifr_ifindex; + CHECK(!ioctl(raw, SIOCGIFFLAGS, &ifr)); + ifr.ifr_flags |= IFF_UP; + CHECK(!ioctl(raw, SIOCSIFFLAGS, &ifr)); + int fd = open("/dev/vhost-net", O_RDWR | O_CLOEXEC); + CHECK(fd >= 0); + CHECK(!ioctl(fd, VHOST_SET_FORK_FROM_OWNER, &mode)); + uint8_t gotmode = 9; + CHECK(!ioctl(fd, VHOST_GET_FORK_FROM_OWNER, &gotmode) && + gotmode == mode); + CHECK(!ioctl(fd, VHOST_SET_OWNER, NULL)); + uint64_t log_base = (uintptr_t)bitmap; + CHECK(!ioctl(fd, VHOST_SET_LOG_BASE, &log_base)); + unsigned char *data = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(data != MAP_FAILED); + struct { + struct vhost_memory mem; + struct vhost_memory_region region; + } mem = { + .mem.nregions = 1, + .region = { .guest_phys_addr = DATA_GPA, + .memory_size = 4096, + .userspace_addr = (uintptr_t)data }, + }; + CHECK(!ioctl(fd, VHOST_SET_MEM_TABLE, &mem)); + uint64_t features = (1ULL << VHOST_F_LOG_ALL) | + (1ULL << VHOST_NET_F_VIRTIO_NET_HDR); + uint64_t available; + CHECK(!ioctl(fd, VHOST_GET_FEATURES, &available) && + (available & features) == features); + CHECK(!ioctl(fd, VHOST_SET_FEATURES, &features)); + struct queue *q = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(q != MAP_FAILED && sizeof(*q) * 2 < ps); + CHECK(!queue_setup(fd, q, 0) && !queue_setup(fd, q + 1, 1)); + for (unsigned index = 0; index < 2; index++) { + struct vhost_vring_file backend = { .index = index, .fd = tap }; + CHECK(!ioctl(fd, VHOST_NET_SET_BACKEND, &backend)); + } + struct sockaddr_ll dest = { .sll_family = AF_PACKET, + .sll_protocol = htons(PROTOCOL), + .sll_ifindex = ifindex, + .sll_halen = 6 }; + unsigned char packet[PACKET_SIZE]; + memset(packet, 0x5c, sizeof(packet)); + memset(packet, 0xff, 6); + packet[6] = 2; + packet[12] = PROTOCOL >> 8; + packet[13] = PROTOCOL & 0xff; + q[0].desc[0] = (struct vring_desc){ .addr = DATA_GPA, + .len = 512, + .flags = VRING_DESC_F_WRITE }; + q[1].desc[0] = + (struct vring_desc){ .addr = DATA_GPA + 1024, + .len = sizeof(struct virtio_net_hdr) + + PACKET_SIZE }; + memcpy(data + 1024 + sizeof(struct virtio_net_hdr), packet, + PACKET_SIZE); + for (unsigned phase = 0; phase < 2; phase++) { + if (phase) { + memset(bitmap, 0, 9); + CHECK(!madvise(leaf, ps, MADV_DONTNEED)); + int pagemap = open("/proc/self/pagemap", O_RDONLY); + uint64_t entry; + CHECK(pagemap >= 0 && + pread(pagemap, &entry, sizeof(entry), + (uintptr_t)leaf / ps * sizeof(entry)) == + sizeof(entry)); + CHECK(!(entry & (1ULL << 63))); + close(pagemap); + for (unsigned index = 0; index < 2; index++) { + struct vhost_vring_file backend = { + .index = index, .fd = tap + }; + CHECK(!ioctl(fd, VHOST_NET_SET_BACKEND, + &backend)); + } + } + for (unsigned round = phase * ROUNDS; + round < (phase + 1) * ROUNDS; round++) { + packet[20] = round; + memset(data, 0xaa, 512); + CHECK(submit(q, round)); + CHECK(sendto(raw, packet, sizeof(packet), 0, + (void *)&dest, + sizeof(dest)) == sizeof(packet)); + CHECK(submit(q + 1, round)); + CHECK(complete(q, round) && complete(q + 1, round)); + CHECK(q[0].used.ring[round % N].id == 0); + if (q[0].used.ring[round % N].len != + sizeof(struct virtio_net_hdr) + PACKET_SIZE) + printf("RX length=%u expected=%zu ethertype=%02x%02x\n", + q[0].used.ring[round % N].len, + sizeof(struct virtio_net_hdr) + + PACKET_SIZE, + data[sizeof(struct virtio_net_hdr) + 12], + data[sizeof(struct virtio_net_hdr) + 13]); + CHECK(q[0].used.ring[round % N].len == + sizeof(struct virtio_net_hdr) + PACKET_SIZE); + CHECK(!memcmp(data + sizeof(struct virtio_net_hdr), + packet, PACKET_SIZE)); + CHECK(filled(data + sizeof(struct virtio_net_hdr) + + PACKET_SIZE, + 512 - sizeof(struct virtio_net_hdr) - + PACKET_SIZE, + 0xaa)); + } + /* Stop and flush workers before checking the bitmap or discarding backing. */ + for (unsigned index = 0; index < 2; index++) { + struct vhost_vring_file backend = { .index = index, + .fd = -1 }; + CHECK(!ioctl(fd, VHOST_NET_SET_BACKEND, &backend)); + } + CHECK(bitmap_ok(leaf, ps, 1, phase && !shared)); + if (forked && !phase) { + CHECK(write(barrier[1], "x", 1) == 1); + close(barrier[1]); + int status; + CHECK(waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status)); + } + } + close(fd); + close(raw); + close(tap); + for (unsigned i = 0; i < 2; i++) { + close(q[i].kick); + close(q[i].call); + } + CHECK(filled(dst, ps, 0x91) && filled(dst + 2 * ps, 2 * ps, 0x91)); + for (unsigned i = 0; i < 3; i++) + CHECK(filled(src + i * ps, ps, 0x61 + i)); + CHECK(!munmap(src, 3 * ps) && !munmap(dst, 4 * ps)); + CHECK(!munmap(data, ps) && !munmap(q, ps)); + if (backing >= 0) + close(backing); + printf("ok - %zuK vhost %s fork=%u worker=%u: %u RX/TX packets, relocated dirty bits, missing-PTE repair and neighbors\n", + ps / 1024, shared ? "shared" : "private", forked, mode, + 2 * ROUNDS); + return 0; +} +static int exercise(size_t expected) +{ + CHECK(getauxval(AT_PAGESZ) == expected); + alarm(45); + for (uint8_t mode = 0; mode < 2; mode++) + for (unsigned shared = 0; shared < 2; shared++) + for (unsigned forked = 0; forked < 2; forked++) + CHECK(!exercise_case(expected, shared, forked, + mode)); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + /* Keep unsolicited IPv6 DAD packets out of the isolated TAP queues. */ + int ipv6 = + open("/proc/sys/net/ipv6/conf/default/disable_ipv6", O_WRONLY); + if (ipv6 < 0 || write(ipv6, "1", 1) != 1) { + perror("disable IPv6"); + fail = 1; + goto done; + } + close(ipv6); + size_t sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned i = 0; i < (sizes[0] > 16384 ? 3U : 2U); i++) { + pid_t c = fork(); + int status = 0; + if (!c) { + char expected[24]; + snprintf(expected, sizeof(expected), "%zu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", expected, NULL); + _exit(5); + } + int ok = 0; + for (unsigned ticks = 0; c > 0 && ticks < 600; ticks++) { + pid_t got = waitpid(c, &status, WNOHANG); + if (got == c) { + ok = WIFEXITED(status) && !WEXITSTATUS(status); + break; + } + if (ticks == 599) { + kill(c, SIGKILL); + waitpid(c, &status, 0); + break; + } + usleep(100000); + } + printf("%s - ABI %zuK vhost log status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + fail += !ok; + } +done: + printf("VHOST LOG %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/wine-kernel32.def b/tools/testing/selftests/arm64/user4k/wine-kernel32.def new file mode 100644 index 0000000000000..891e48c1fb83b --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/wine-kernel32.def @@ -0,0 +1,20 @@ +LIBRARY KERNEL32.dll +EXPORTS +VirtualAlloc +VirtualFree +VirtualProtect +VirtualQuery +GetSystemInfo +GetStdHandle +WriteFile +ExitProcess +GetLastError +AddVectoredExceptionHandler +RemoveVectoredExceptionHandler +CreateThread +WaitForSingleObject +GetExitCodeThread +CloseHandle +CreateFileMappingA +MapViewOfFile +UnmapViewOfFile diff --git a/tools/testing/selftests/arm64/user4k/wine-page-contract.c b/tools/testing/selftests/arm64/user4k/wine-page-contract.c new file mode 100644 index 0000000000000..149f619de6651 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/wine-page-contract.c @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Freestanding Windows x64 PE; Win32 imports, no CRT or external SDK. */ +typedef unsigned long DWORD; +typedef int BOOL; +typedef unsigned long long SIZE_T; +typedef void *HANDLE; +#define API __declspec(dllimport) +API void *VirtualAlloc(void *, SIZE_T, DWORD, DWORD); +API BOOL VirtualFree(void *, SIZE_T, DWORD); +API BOOL VirtualProtect(void *, SIZE_T, DWORD, DWORD *); +API SIZE_T VirtualQuery(const void *, void *, SIZE_T); +API void GetSystemInfo(void *); +API HANDLE GetStdHandle(DWORD); +API BOOL WriteFile(HANDLE, const void *, DWORD, DWORD *, void *); +API void ExitProcess(DWORD); +API DWORD GetLastError(void); +API void *AddVectoredExceptionHandler(DWORD, void *); +API DWORD RemoveVectoredExceptionHandler(void *); +API HANDLE CreateThread(void *, SIZE_T, DWORD (*)(void *), void *, DWORD, + DWORD *); +API DWORD WaitForSingleObject(HANDLE, DWORD); +API BOOL GetExitCodeThread(HANDLE, DWORD *); +API BOOL CloseHandle(HANDLE); +API HANDLE CreateFileMappingA(HANDLE, void *, DWORD, DWORD, DWORD, + const char *); +API void *MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); +API BOOL UnmapViewOfFile(const void *); +struct system_info { + DWORD arch, page; + void *min, *max; + SIZE_T mask; + DWORD cpus, type, granule; + unsigned short level, revision; +}; +struct memory_info { + void *base, *allocation; + DWORD allocation_protect, pad; + SIZE_T size; + DWORD state, protect, type, pad2; +}; +struct exception_record { + DWORD code, flags; + void *record, *address; + DWORD count, pad; + SIZE_T info[15]; +}; +struct exception_pointers { + struct exception_record *record; + void *context; +}; +/* Keep the freestanding declarations consistent with the Windows x64 ABI. */ +_Static_assert(sizeof(DWORD) == 4, "DWORD must be 32 bits"); +_Static_assert(sizeof(SIZE_T) == 8, "SIZE_T must be 64 bits"); +_Static_assert(sizeof(struct system_info) == 48, "SYSTEM_INFO layout"); +_Static_assert(sizeof(struct memory_info) == 48, + "MEMORY_BASIC_INFORMATION layout"); +_Static_assert(sizeof(struct exception_record) == 152, + "EXCEPTION_RECORD layout"); +static volatile unsigned char *pages; +static volatile DWORD handled; +static void say(const char *s) +{ + DWORD n = 0, w = 0; + while (s[n]) + n++; + WriteFile(GetStdHandle((DWORD)-11), s, n, &w, 0); +} +static void number(DWORD n) +{ + char buf[11], *p = buf + 10; + *p = 0; + do { + *--p = '0' + n % 10; + n /= 10; + } while (n); + say(p); +} +static void fail(DWORD stage) +{ + DWORD e = GetLastError(); + say("WINE PAGE CONTRACT FAIL stage="); + number(stage); + say(" error="); + number(e); + say("\r\n"); + ExitProcess(1); +} +#define CHECK(x, n) \ + do { \ + if (!(x)) \ + fail(n); \ + } while (0) +static long handler(struct exception_pointers *e) +{ + DWORD old; + if (e->record->code != 0xc0000005 || e->record->count < 2 || + e->record->info[0] != 1 || + e->record->info[1] != (SIZE_T)(pages + 4103)) + return 0; + if (!VirtualProtect((void *)(pages + 4096), 4096, 4, &old)) + return 0; + handled++; + return -1; +} +static DWORD thread(void *arg) +{ + volatile unsigned char *p = arg; + for (unsigned int i = 0; i < 4096; i++) + p[i] = (unsigned char)(i ^ 0x5a); + return 37; +} +void mainCRTStartup(void) +{ + struct system_info si; + struct memory_info mi; + DWORD old, status; + GetSystemInfo(&si); + CHECK(si.page == 4096 && si.granule == 65536, 1); + say("ok - Wine reports 4K pages and 64K allocation granularity\r\n"); + pages = VirtualAlloc(0, 65536, 0x2000, 1); + CHECK(pages, 2); + CHECK(VirtualAlloc((void *)pages, 12288, 0x1000, 4) == pages, 3); + for (unsigned int i = 0; i < 12288; i++) + pages[i] = (unsigned char)(i ^ (i >> 12)); + CHECK(VirtualProtect((void *)(pages + 4096), 4096, 2, &old) && old == 4, + 4); + CHECK(VirtualQuery((void *)(pages + 4096), &mi, sizeof(mi)) == + sizeof(mi) && + mi.base == pages + 4096 && mi.size == 4096 && + mi.protect == 2, + 5); + CHECK(VirtualQuery((void *)(pages + 8192), &mi, sizeof(mi)) == + sizeof(mi) && + mi.protect == 4, + 6); + void *veh = AddVectoredExceptionHandler(1, handler); + CHECK(veh, 7); + pages[4103] = 0x6d; + CHECK(handled == 1 && pages[4103] == 0x6d, 8); + CHECK(RemoveVectoredExceptionHandler(veh), 9); + say("ok - Wine independent 4K protection and handled access violation\r\n"); + CHECK(VirtualFree((void *)(pages + 4096), 4096, 0x4000), 10); + CHECK(VirtualQuery((void *)(pages + 4096), &mi, sizeof(mi)) == + sizeof(mi) && + mi.state == 0x2000, + 11); + CHECK(VirtualAlloc((void *)(pages + 4096), 4096, 0x1000, 4) == + pages + 4096, + 12); + for (unsigned int i = 0; i < 4096; i++) + CHECK(!pages[4096 + i], 13); + for (unsigned int i = 0; i < 4096; i++) + CHECK(pages[i] == (unsigned char)i && + pages[8192 + i] == (unsigned char)(i ^ 2), + 14); + say("ok - Wine 4K decommit and zero-filled recommit preserve neighbours\r\n"); + HANDLE t = CreateThread(0, 0, thread, (void *)(pages + 4096), 0, 0); + CHECK(t, 15); + CHECK(WaitForSingleObject(t, 30000) == 0 && + GetExitCodeThread(t, &status) && status == 37, + 16); + CHECK(CloseHandle(t), 17); + for (unsigned int i = 0; i < 4096; i++) + CHECK(pages[4096 + i] == (unsigned char)(i ^ 0x5a), 18); + say("ok - Wine threads share committed 4K pages\r\n"); + HANDLE file = CreateFileMappingA((HANDLE)(SIZE_T)-1, 0, 4, 0, 16384, 0); + CHECK(file, 19); + unsigned char *a = MapViewOfFile(file, 2, 0, 0, 16384), + *b = MapViewOfFile(file, 2, 0, 0, 16384); + CHECK(a && b, 20); + a[4099] = 0xa7; + CHECK(b[4099] == 0xa7, 21); + CHECK(VirtualProtect(a + 4096, 4096, 2, &old), 22); + b[4099] = 0x53; + CHECK(a[4099] == 0x53, 23); + CHECK(UnmapViewOfFile(a) && UnmapViewOfFile(b) && CloseHandle(file), + 24); + CHECK(VirtualFree((void *)pages, 0, 0x8000), 25); + say("ok - Wine shared section aliases and 4K protection\r\nWINE PAGE CONTRACT PASS\r\n"); + ExitProcess(0); +} diff --git a/tools/testing/selftests/arm64/user4k/x18-probe.S b/tools/testing/selftests/arm64/user4k/x18-probe.S new file mode 100644 index 0000000000000..565f34e8323f5 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/x18-probe.S @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include + + .text + .globl x18_syscall + .type x18_syscall, %function +/* (nr, arg0, arg1, arg2, observed_x18); no libc code inside the probe. */ +x18_syscall: + hint 34 /* bti c */ + stp x18, x19, [sp, #-16]! + mov x19, x4 + movz x18, #0x3210 + movk x18, #0x7654, lsl #16 + movk x18, #0xba98, lsl #32 + movk x18, #0xfedc, lsl #48 + mov x8, x0 + mov x0, x1 + mov x1, x2 + mov x2, x3 + svc #0 + str x18, [x19] + ldp x18, x19, [sp], #16 + ret + .size x18_syscall, .-x18_syscall + .section .note.GNU-stack,"",%progbits diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index bf26b3f48d3ab..912e4675553a4 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -520,6 +520,9 @@ struct vm_area_desc { /* Mutable fields. Populated with initial state. */ pgoff_t pgoff; +#ifdef CONFIG_MM_SUBPAGE + unsigned int subpage_offset; +#endif struct file *vm_file; vma_flags_t vma_flags; pgprot_t page_prot; @@ -1287,6 +1290,9 @@ static inline void compat_set_desc_from_vma(struct vm_area_desc *desc, desc->end = vma->vm_end; desc->pgoff = vma->vm_pgoff; +#ifdef CONFIG_MM_SUBPAGE + desc->subpage_offset = vma->vm_subpage_offset; +#endif desc->vm_file = vma->vm_file; desc->vma_flags = vma->flags; desc->page_prot = vma->vm_page_prot; @@ -1296,6 +1302,117 @@ static inline void compat_set_desc_from_vma(struct vm_area_desc *desc, desc->action.type = MMAP_NOTHING; } +static inline unsigned int mm_page_shift(const struct mm_struct *mm) +{ + return PAGE_SHIFT; +} + +static inline unsigned long mm_page_size(const struct mm_struct *mm) +{ + return PAGE_SIZE; +} + +static inline unsigned long mm_page_mask(const struct mm_struct *mm) +{ + return PAGE_MASK; +} + +static inline unsigned long mm_page_align(const struct mm_struct *mm, unsigned long size) +{ + return PAGE_ALIGN(size); +} + +static inline unsigned long vma_user_pages(const struct vm_area_struct *vma) +{ + return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; +} + +#define mm_account_memory(mm, policy_mm, pages) security_vm_enough_memory_mm(policy_mm, pages) +#define mm_unacct_memory(mm, pages) vm_unacct_memory(pages) + +/* File-cache indices remain native-page units even for a 4K VMA. */ +struct vm_page_offset { + pgoff_t index; + unsigned int offset; +}; + +static inline struct vm_page_offset vm_page_offset_add(struct vm_page_offset pos, + unsigned long bytes) +{ + unsigned long low = pos.offset + (bytes & ~PAGE_MASK); + + pos.index += (bytes >> PAGE_SHIFT) + (low >> PAGE_SHIFT); + pos.offset = low & ~PAGE_MASK; + return pos; +} + +static inline struct vm_page_offset vm_page_offset_sub(struct vm_page_offset pos, + unsigned long bytes) +{ + unsigned long low = bytes & ~PAGE_MASK; + + pos.index -= (bytes >> PAGE_SHIFT) + (low > pos.offset); + pos.offset = (pos.offset - low) & ~PAGE_MASK; + return pos; +} + +static inline bool vm_page_offset_equal(struct vm_page_offset a, struct vm_page_offset b) +{ + return a.index == b.index && a.offset == b.offset; +} + +static inline unsigned int vma_subpage_offset(const struct vm_area_struct *vma) +{ +#ifdef CONFIG_MM_SUBPAGE + return vma->vm_subpage_offset; +#else + return 0; +#endif +} + +/* address must be >= vm_start; vm_end computes the next contiguous offset. */ +static inline struct vm_page_offset vma_page_offset_at(const struct vm_area_struct *vma, + unsigned long address) +{ + unsigned long delta = address - vma->vm_start; + unsigned long low = (delta & ~PAGE_MASK) + vma_subpage_offset(vma); + + return (struct vm_page_offset) { + .index = vma->vm_pgoff + (delta >> PAGE_SHIFT) + (low >> PAGE_SHIFT), + .offset = low & ~PAGE_MASK, + }; +} + +static inline void vma_set_page_offset(struct vm_area_struct *vma, struct vm_page_offset pos) +{ + vma->vm_pgoff = pos.index; +#ifdef CONFIG_MM_SUBPAGE + vma->vm_subpage_offset = pos.offset; +#endif +} + +/* Inverse lookup of a single byte; an offset outside the VMA is not clamped. */ +static inline unsigned long vma_address_at_offset(const struct vm_area_struct *vma, + struct vm_page_offset pos) +{ + unsigned long len = vma->vm_end - vma->vm_start; + unsigned long index, delta, last; + unsigned int offset = vma_subpage_offset(vma); + + if (!len || pos.index < vma->vm_pgoff || pos.offset >= PAGE_SIZE) + return -EFAULT; + index = pos.index - vma->vm_pgoff; + last = ((len - 1) >> PAGE_SHIFT) + + ((((len - 1) & ~PAGE_MASK) + offset) >> PAGE_SHIFT); + if (index > last) + return -EFAULT; + delta = (index << PAGE_SHIFT) + pos.offset; + if (delta < offset || delta - offset >= len) + return -EFAULT; + return vma->vm_start + delta - offset; +} + + static inline unsigned long vma_pages(const struct vm_area_struct *vma) { return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; diff --git a/tools/testing/vma/tests/merge.c b/tools/testing/vma/tests/merge.c index 03b6f9820e0ac..0b0f4ae413973 100644 --- a/tools/testing/vma/tests/merge.c +++ b/tools/testing/vma/tests/merge.c @@ -227,7 +227,7 @@ static bool test_simple_shrink(void) ASSERT_FALSE(attach_vma(&mm, vma)); - ASSERT_FALSE(vma_shrink(&vmi, vma, 0, 0x1000, 0)); + ASSERT_FALSE(vma_shrink(&vmi, vma, 0, 0x1000, (struct vm_page_offset) { 0, 0 })); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x1000); diff --git a/tools/testing/vma/tests/mmap.c b/tools/testing/vma/tests/mmap.c index c85bc000d1cb7..c87baa48ea26e 100644 --- a/tools/testing/vma/tests/mmap.c +++ b/tools/testing/vma/tests/mmap.c @@ -12,19 +12,19 @@ static bool test_mmap_region_basic(void) current->mm = &mm; /* Map at 0x300000, length 0x3000. */ - addr = __mmap_region(NULL, 0x300000, 0x3000, vma_flags, 0x300, NULL); + addr = __mmap_region(NULL, 0x300000, 0x3000, vma_flags, (struct vm_page_offset) { 0x300, 0 }, NULL); ASSERT_EQ(addr, 0x300000); /* Map at 0x250000, length 0x3000. */ - addr = __mmap_region(NULL, 0x250000, 0x3000, vma_flags, 0x250, NULL); + addr = __mmap_region(NULL, 0x250000, 0x3000, vma_flags, (struct vm_page_offset) { 0x250, 0 }, NULL); ASSERT_EQ(addr, 0x250000); /* Map at 0x303000, merging to 0x300000 of length 0x6000. */ - addr = __mmap_region(NULL, 0x303000, 0x3000, vma_flags, 0x303, NULL); + addr = __mmap_region(NULL, 0x303000, 0x3000, vma_flags, (struct vm_page_offset) { 0x303, 0 }, NULL); ASSERT_EQ(addr, 0x303000); /* Map at 0x24d000, merging to 0x250000 of length 0x6000. */ - addr = __mmap_region(NULL, 0x24d000, 0x3000, vma_flags, 0x24d, NULL); + addr = __mmap_region(NULL, 0x24d000, 0x3000, vma_flags, (struct vm_page_offset) { 0x24d, 0 }, NULL); ASSERT_EQ(addr, 0x24d000); ASSERT_EQ(mm.map_count, 2); diff --git a/tools/testing/vma/tests/vma.c b/tools/testing/vma/tests/vma.c index b7563ff20277e..04afec6a07e90 100644 --- a/tools/testing/vma/tests/vma.c +++ b/tools/testing/vma/tests/vma.c @@ -38,7 +38,7 @@ static bool test_copy_vma(void) /* Move backwards and do not merge. */ vma = alloc_and_link_vma(&mm, 0x3000, 0x5000, 3, vma_flags); - vma_new = copy_vma(&vma, 0, 0x2000, 0, &need_locks, false); + vma_new = copy_vma(&vma, 0, 0x2000, (struct vm_page_offset) { 0, 0 }, &need_locks, false); ASSERT_NE(vma_new, vma); ASSERT_EQ(vma_new->vm_start, 0); ASSERT_EQ(vma_new->vm_end, 0x2000); @@ -51,7 +51,7 @@ static bool test_copy_vma(void) vma = alloc_and_link_vma(&mm, 0, 0x2000, 0, vma_flags); vma_next = alloc_and_link_vma(&mm, 0x6000, 0x8000, 6, vma_flags); - vma_new = copy_vma(&vma, 0x4000, 0x2000, 4, &need_locks, false); + vma_new = copy_vma(&vma, 0x4000, 0x2000, (struct vm_page_offset) { 4, 0 }, &need_locks, false); vma_assert_attached(vma_new); ASSERT_EQ(vma_new, vma_next); From 31805a4abbcbed84ac32110fae7de1baf1c2277e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:23 -0400 Subject: [PATCH 0858/2680] docs: describe mixed userspace page sizes and the development launcher Document native backing versus per-process mappings, architecture responsibilities, supported combinations and remaining limitations. Provide a launcher for inherited and one-shot exec preferences. Signed-off-by: Diljit Singh (cherry picked from commit 6404f6a5b060c3b8730f7fcda19d4a72d59fab1e) --- Documentation/arch/arm64/index.rst | 1 + Documentation/arch/arm64/user4k.rst | 187 ++++++++++++++++++++++++++++ Documentation/mm/index.rst | 1 + Documentation/mm/user_page_size.rst | 84 +++++++++++++ tools/aurora-userpages/.gitignore | 1 + tools/aurora-userpages/Makefile | 14 +++ tools/aurora-userpages/README.rst | 34 +++++ tools/aurora-userpages/userpages.c | 83 ++++++++++++ 8 files changed, 405 insertions(+) create mode 100644 Documentation/arch/arm64/user4k.rst create mode 100644 Documentation/mm/user_page_size.rst create mode 100644 tools/aurora-userpages/.gitignore create mode 100644 tools/aurora-userpages/Makefile create mode 100644 tools/aurora-userpages/README.rst create mode 100644 tools/aurora-userpages/userpages.c diff --git a/Documentation/arch/arm64/index.rst b/Documentation/arch/arm64/index.rst index 98052b4ef4a1e..e5d0d0c46bf3b 100644 --- a/Documentation/arch/arm64/index.rst +++ b/Documentation/arch/arm64/index.rst @@ -32,5 +32,6 @@ ARM64 Architecture sve tagged-address-abi tagged-pointers + user4k features diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst new file mode 100644 index 0000000000000..d483c565d853b --- /dev/null +++ b/Documentation/arch/arm64/user4k.rst @@ -0,0 +1,187 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +=========================================== +Experimental per-process page sizes on arm64 +=========================================== + +The mixed-page implementation lets an address space use a different user +translation granule from the kernel's native physical page size. The kernel +page size remains fixed for the boot. An address space's user granule remains +fixed for its lifetime; exec can create a new address space with another +supported granule. See :doc:`../../mm/user_page_size` for the generic launch +policy and architecture integration contract. + +This is an experimental ABI. The launch commands use private prctl numbers, +not assigned upstream UAPI. The implementation and tests do not establish +complete application, device or hardware qualification. + +Configuration and supported combinations +======================================== + +Enable ``CONFIG_ARM64_USER4K_EXPERIMENTAL`` with 48-bit VA and PA and a native +16K or 64K page size. The option defaults off and selects the generic +``MM_SUBPAGE`` backing and ``ARCH_HAS_USER_PAGE_SIZE`` exec policy support. +LPA2 and ``NO_PAGE_MAPCOUNT`` configurations are excluded by Kconfig. + +The exposed combinations are: + +=================== ================== ============================ +Native kernel size AArch64 default AArch64 alternatives +=================== ================== ============================ +4K 4K None in this implementation +16K 16K 4K, when supported by the CPUs +64K 64K 4K and 16K, subject to support +=================== ================== ============================ + +Capability detection uses the sanitized system-wide granule intersection. +AArch32 executables select 4K when that capability is available and retain +the native granule otherwise, regardless of launch preferences. The latter +case still enforces native mapping/protection alignment. AArch32 mmap2 file +offsets retain their 4K ABI unit in either case. + +An AArch32 executable running under an arm64 kernel is distinct from an +``arch/arm`` kernel. The latter retains its existing native ABI; this series +does not yet provide its alternative-granule backend. Exploiting extra +translation capabilities available only within an asymmetric AArch32 CPU +domain also remains unimplemented. + +Larger-than-native process granules are not exposed. Internal 64K-on-16K +fixtures exercise allocation, mappings and argument staging, but those +fixtures do not enable a public 64K process ABI on a 16K kernel. + +Selecting an executable's page size +=================================== + +Build the development launcher with:: + + make -C tools/aurora-userpages + +For example:: + + tools/aurora-userpages/userpages 4k COMMAND [ARGUMENTS...] + tools/aurora-userpages/userpages --once native COMMAND [ARGUMENTS...] + tools/aurora-userpages/userpages --status + +The ordinary selection establishes an inherited per-thread exec preference. +``--once`` overrides it for one successful exec. A failed exec retains the +pending request; changing a preference never changes existing mappings. +Fork copies the preferences and mappings. Threads sharing an mm have the +same current granule but may establish different preferences for later exec. +``native`` clears an inherited alternative preference; a one-shot native +request can override that preference without clearing it. + +The final ELF loader resolves the granule before committing the new image. +If arguments were staged with another granule, their bytes are transferred +into a new private mm. Temporary argument mappings are bounded by both the +user leaf and native page size. Logical argument-size limits continue to +use the process granule. The interpreter and executable share one mm and +therefore one final granule. ELF segment alignment alone does not opt a +program into an alternative ABI. + +``AT_PAGESZ`` reports the process granule. User virtual address alignment, +protection boundaries and mapping lengths use that granule. Native physical +allocation, PFN arithmetic and file-cache indices continue to use the kernel +page size. Application execution does not require any of the KUnit options +or the ``arm64.user4k_test=1`` diagnostic boot parameter. + +Memory and table ownership +========================== + +Keep the following units separate: + +* ``PAGE_SIZE`` and physical PFNs describe native physical pages. +* ``mm_page_size(mm)`` describes the user mapping/protection unit. +* A folio owns one or more native pages and carries their physical charge. + +Smaller anonymous mappings retain explicit subpage ownership. A reference to +the containing folio alone does not retain an individual reusable slot. +The slot's byte offset must survive fault, COW, reverse mapping, migration, +swap and pinning operations. Newly exposed bytes are zeroed, and changing +one mapping's protection must not widen that change to adjacent slots. + +File offsets use a native cache index plus an explicit byte offset. VMA +split/merge, truncation and reverse mapping must preserve both components. +Physical allocation charges count the actual backing once, independently +of how many user mappings expose it. + +Lower page tables can share a native allocation. Table tokens preserve the +fragment address, while locking and lifetime use the native table owner. +Unissued cached fragments retain owner references. Issued fragments are not +reused within a live owner. Deferred reclamation must preserve the fragment +token through TLB invalidation and the required RCU grace period, including +allocation-failure fallback paths. + +Translation context and invalidation +=================================== + +The arm64 backend supplies per-mm page-table geometry. TTBR1 retains native +kernel tables, while TTBR0 uses the selected user geometry. A granule change +installs and synchronizes a reserved TTBR0 before changing TG0, then +synchronizes the new geometry before installing the user root. Software PAN +also selects the user TG0 even when restoration of TTBR0 is deferred until +uaccess or return to userspace. + +The existing mm ASID is retained. Selecting another granule does not add +an unconditional context-switch TLB flush. User invalidation operands and +range strides use the target mm's geometry; kernel and stage-2 operations +retain their own native geometry. Transition/idmap paths restore native +TG0 and then the active mm's user geometry when returning. + +The current configuration retains the existing 48-bit user address size. +Independent TG0/TG1 fields do not make all other translation controls +independent; the LPA2 exclusion must not be removed merely by changing TG0. + +Sharing, pinning and interface limits +==================================== + +Mixed-granule sharing is permitted. Shared mappings refer to one coherent +backing object, with permissions belonging to each virtual mapping. Shared +futex identity includes the object and byte offset. Private copying is not +an implementation of shared-memory semantics. + +Some interfaces have additional layout or ownership requirements. For +example, a perf ring's first mapper fixes its userspace granule; a mapper +using another granule is rejected. This does not prohibit mixed-granule +memfd or System V shared-memory aliases. + +A legacy page-array pin cannot express every smaller-granule byte range. +Converted consumers use typed fragment references carrying backing, offset, +length and ownership. Unconverted paths must not silently round a pin or DMA +mapping into neighboring slots. Mock IOMMU/VFIO tests validate the recorded +translations and ownership paths, not physical-device DMA on arbitrary +hardware. + +Alternative-granule THP/KSM and hugetlb support, remaining legacy pin +consumers, larger-than-native process activation and broader device +integration still need work. Userfaultfd feature bits and per-range ioctl +masks must be consulted; success for a shmem operation does not imply +support for the corresponding hugetlb operation. Sparse individual fills +can retain an entire native folio; packing behavior is not a throughput +or memory-efficiency guarantee for every workload. + +Test coverage +============= + +The arm64 selftests are in ``tools/testing/selftests/arm64/user4k``. The +``page-contract`` executable checks the exposed userspace contract. Extended +fixtures cover exec policy, AArch32, shared mappings, accounting, reclaim, +I/O and device-facing interfaces. Several are disposable-guest PID1 programs +with companion executables or storage requirements; they are not ordinary +host test commands. + +``CONFIG_ARM64_USER4K_KUNIT_TEST`` and the data/TLB suites exercise internal +geometry and ownership. Some fixtures require ``arm64.user4k_test=1`` and +some hardware translation tests are disabled under KASAN. Check individual +KTAP results and skips rather than treating boot success as a suite pass. +Private IOMMUFD/VFIO test-device options are separate and should remain off +in application kernels. + +The NUMA COW batching case runs only at boot, before userspace can change +its policy prerequisite. Its results remain visible in KUnit debugfs, but +it has no runtime rerun control. Other suites retain their rerun controls. + +Validation must distinguish native/default operation, each exposed +alternative, and AArch32 fallback with 4K support unavailable. Native ARM32 +short-descriptor and LPAE controls check that generic changes preserve the +existing ABI; they do not validate a mixed-page ARM32 backend. Internal +larger-leaf tests likewise do not qualify a public process mode. diff --git a/Documentation/mm/index.rst b/Documentation/mm/index.rst index 13a79f5d092c0..2252d144287cb 100644 --- a/Documentation/mm/index.rst +++ b/Documentation/mm/index.rst @@ -26,6 +26,7 @@ see the :doc:`admin guide <../admin-guide/mm/index>`. physical_memory page_tables process_addrs + user_page_size bootmem page_allocation vmalloc diff --git a/Documentation/mm/user_page_size.rst b/Documentation/mm/user_page_size.rst new file mode 100644 index 0000000000000..62ef6ae39e0ac --- /dev/null +++ b/Documentation/mm/user_page_size.rst @@ -0,0 +1,84 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +============================== +Per-process user mapping sizes +============================== + +The experimental mixed-page implementation separates the user mapping +size of an address space from the kernel's native allocation size. +``PAGE_SIZE`` still describes native physical pages. ``mm_page_size(mm)`` +describes the minimum user mapping and protection unit of that address +space. A process retains one mapping size for the lifetime of its ``mm``. +Exec may create a new ``mm`` with a different size; changing a launch +preference never changes an existing mapping. + +The generic interfaces described here are internal development interfaces. +The experimental launch commands still use private prctl numbers; these +are not assigned upstream UAPI. The arm64 backend's supported modes and +current restrictions are documented in +:doc:`../arch/arm64/user4k`. + +Exec policy and inheritance +=========================== + +``ARCH_HAS_USER_PAGE_SIZE`` enables two per-thread policy fields in +``task_struct``: + +``exec_page_shift`` + A one-shot request for the next exec. Zero means no one-shot request. + +``default_exec_page_shift`` + An inherited preference used when no one-shot request overrides it. + Zero means the architecture's ordinary executable policy. + +Fork and thread creation copy both fields. Subsequent changes affect only +the calling thread, including when it shares an ``mm`` with other threads. +An exec by a non-leader uses the execing thread's policy. The fields do not +belong in ``mm_struct``: threads sharing one address space may launch new +images with different preferences. + +Generic prctl handling validates reserved arguments before calling the +architecture validator. A rejected request leaves both fields unchanged. +Setting a native one-shot request while an inherited alternative is active +stores ``PAGE_SHIFT`` explicitly, so the next exec can override that +alternative. Clearing the inherited preference does not consume a pending +one-shot request. + +Exec stages arguments in a private ``mm``. Its initial allocation receives +the one-shot shift, if any; the final executable loader resolves the +inherited preference and any executable-specific policy. An ELF interpreter +must be loaded using the same final mapping size as its executable. +Failure before committing the new image retains both preferences. +``setup_new_exec()`` consumes the one-shot field after the new image is +committed, while retaining the inherited preference. + +Architecture responsibilities +============================= + +An architecture selecting ``ARCH_HAS_USER_PAGE_SIZE`` must implement: + +* ``arch_exec_page_size_shift(size)``: return zero for a native request, a + validated alternative shift, or a negative errno. Validation includes + the supported hardware and kernel configuration, not just whether the + requested byte size is a power of two. +* ``arch_mm_init_exec(mm, shift)``: initialize the private address space's + geometry before allocating its page-table root. Zero denotes native + geometry. The hook receives the request explicitly and does not read + a separate architecture thread-policy field. +* Binary-loader integration that resolves the final executable policy + before exposing the address space. ``bprm_set_page_shift()`` transfers + staged argument bytes when the final choice changes the initial granule. +* Page-table allocation, walking, invalidation, context switching and MM + operations for every size the architecture exposes. + +The architecture retains control over executable compatibility policy. +For example, the arm64 AArch32 loader prefers 4K where the backend supports +it and retains the native ABI when that capability is absent. This is +separate from running an AArch32 kernel: ``arch/arm`` needs its own backend +integration before it can advertise alternative mapping sizes. The generic +exec policy alone does not provide that backend or expose new sizes there. + +Architectures without this capability retain native mappings and reject +the experimental launch-policy commands. ``ARCH_HAS_MM_PGTABLE`` describes +per-mm page-table accessors; it does not by itself promise an executable +userspace ABI. diff --git a/tools/aurora-userpages/.gitignore b/tools/aurora-userpages/.gitignore new file mode 100644 index 0000000000000..a27b2bf8b140d --- /dev/null +++ b/tools/aurora-userpages/.gitignore @@ -0,0 +1 @@ +/userpages diff --git a/tools/aurora-userpages/Makefile b/tools/aurora-userpages/Makefile new file mode 100644 index 0000000000000..c6d42e2d15efa --- /dev/null +++ b/tools/aurora-userpages/Makefile @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-only +CC ?= cc +CFLAGS ?= -O2 -g +CFLAGS += -Wall -Wextra +LDFLAGS += -Wl,-z,max-page-size=65536 + +.PHONY: all clean +all: userpages + +userpages: userpages.c ../../include/uapi/linux/prctl.h + $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $< $(LDLIBS) -o $@ + +clean: + rm -f userpages diff --git a/tools/aurora-userpages/README.rst b/tools/aurora-userpages/README.rst new file mode 100644 index 0000000000000..64f4fcb0dd12a --- /dev/null +++ b/tools/aurora-userpages/README.rst @@ -0,0 +1,34 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Per-process userspace page-size launcher +======================================= + +Build with ``make`` in this directory. The running arm64 kernel must provide +Aurora's experimental per-process granule API. + +Examples:: + + userpages 4k FEXInterpreter ./x86-program + userpages 4k python3 script.py + userpages native program + userpages --status + +The choice becomes the calling thread's inherited exec preference: it applies +to the launched program and subsequent execs in its descendants. Fork retains +the address space and both exec preferences. Other threads and processes are +unchanged. ``native`` clears the inherited alternative preference. Ordinary +processes that never opt in continue to use the kernel's native default. + +``--once`` changes only the next successful exec; an existing inherited +preference remains available for later execs. For example, +``userpages --once native program`` runs one program with native pages and +retains any inherited 4K preference for its subsequent execs. Failed exec does +not consume the pending request. No selection changes a live address space. +AArch32 programs always use 4K when supported, otherwise native pages, +regardless of either preference. + +The kernel checks supported sizes and CPU capabilities. The current branch +supports 4K userspace on 16K/64K kernels and 16K userspace on 64K kernels. +64K userspace on a 16K kernel is not enabled. This launcher uses the private +PR_AURORA_{SET,GET}_{EXEC,DEFAULT}_PAGE_SIZE API in this tree; +it is not an upstream Linux ABI. diff --git a/tools/aurora-userpages/userpages.c b/tools/aurora-userpages/userpages.c new file mode 100644 index 0000000000000..02d9e4357224d --- /dev/null +++ b/tools/aurora-userpages/userpages.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include "../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include + +static int usage(const char *name, int status) +{ + fprintf(status ? stderr : stdout, + "Usage: %s [--once] {native|4k|16k|64k} COMMAND [ARG ...]\n" + " %s --status\n" + "Select the userspace page size for this program and its descendants.\n", + name, name); + return status; +} + +int main(int argc, char **argv) +{ + unsigned long size; + int once = argc > 1 && !strcmp(argv[1], "--once"); + int first = once ? 2 : 1; + + if (argc == 2 && !strcmp(argv[1], "--help")) + return usage(argv[0], 0); + if (argc == 2 && !strcmp(argv[1], "--status")) { + int pending = prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL); + int error = errno; + + printf("Current process page size: %lu bytes\n", getauxval(AT_PAGESZ)); + if (pending < 0) { + fprintf(stderr, "Page-size selection unavailable: %s\n", strerror(error)); + return 1; + } + int inherited = prctl(PR_AURORA_GET_DEFAULT_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL); + + if (inherited < 0) { + perror("Cannot read inherited page-size preference"); + return 1; + } + if (inherited) + printf("Inherited exec preference: %d bytes\n", inherited); + else + puts("Inherited exec preference: native default"); + if (pending) + printf("Next exec page size: %d bytes\n", pending); + else + puts("Next exec page size: inherited preference"); + return 0; + } + if (argc < first + 2) + return usage(argv[0], 2); + if (!strcmp(argv[first], "native")) + size = 0; + else if (!strcmp(argv[first], "4k")) + size = 4096; + else if (!strcmp(argv[first], "16k")) + size = 16384; + else if (!strcmp(argv[first], "64k")) + size = 65536; + else + return usage(argv[0], 2); + + if (!once && prctl(PR_AURORA_SET_DEFAULT_PAGE_SIZE, size, 0UL, 0UL, 0UL)) { + fprintf(stderr, "Cannot set inherited %s pages: %s\n", + argv[first], strerror(errno)); + return 1; + } + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, size, 0UL, 0UL, 0UL)) { + fprintf(stderr, "Cannot select %s userspace pages: %s\n", + argv[first], strerror(errno)); + return 1; + } + execvp(argv[first + 1], &argv[first + 1]); + int error = errno; + + fprintf(stderr, "Cannot execute %s: %s\n", argv[first + 1], strerror(error)); + return error == ENOENT ? 127 : 126; +} From e99b69f87ba814fcf8a5981853c512907df1dd8d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:49:35 +1100 Subject: [PATCH 0859/2680] mm: defer linear subpage anon-vma destruction from atomic context Linear subpage owners retain an anon-vma root even without an rmap sidecar. Their last slot reference can be released by skb completion in softirq after the originating mm has exited. Route both kinds of root through the existing unbound cleanup work so the final anon-vma put and its sleeping synchronization run in process context. Preserve the physical unbinding and folio reference ordering. (cherry picked from commit 19594eb23573bb002d8f62087ecc1bd3cb38eab7) Assisted-by: LLM Signed-off-by: Ryan Murray --- mm/user-subpage.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/user-subpage.c b/mm/user-subpage.c index 133d75241485a..319d792ab5e9a 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -328,8 +328,8 @@ static void owner_free(struct mm_subpage_owner *owner, bool put_folio) /* Last slot reference, or wholly unissued supply. */ folio_put(owner->folio); } - if (owner->rmap) { - /* Historical roots can have no live VMA; their final put may sleep. */ + if (owner->anon_root || owner->rmap) { + /* Any final anon-vma put can sleep, including a linear owner's. */ INIT_WORK(&owner->free_work, owner_free_work); queue_work(system_unbound_wq, &owner->free_work); } else { From eff432431fb7b05c3d742d4c5e226d74385a7758 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 08:38:36 -0400 Subject: [PATCH 0860/2680] mm: reuse backing folios for small-granule anonymous COW Closing a subpage pool after each COW fault leaves every remaining slot unavailable for later faults. On a 16K kernel with a 4K process ABI, a 64MiB densely modified fork child therefore uses 256MiB of backing. Retain bounded, address-group pools for eligible faults in each mm. Keep charge domains and memory policy explicit, preserve independent fork ownership, and close pools after exit_mmap while referenced owners remain alive. Copy into a reserved slot under the folio lock, decide first rmap/LRU publication after relocking, and fall back to private backing if nonlinear rmap metadata cannot be allocated. Distribute concurrent faults across 16 address-group pools, skip busy backing on the normal path, and use exact per-offset free counts to avoid repeatedly scanning incompatible partial owners. Speculative allocation and charge do not invoke OOM; under pressure, retry reusable backing before ordinary allocation. Existing policy-sensitive, locked, droppable and multi-node faults keep private backing. Add a user4k selftest covering concurrent, sparse, phased and random COW, mlock and lazy-free transitions, and a descendant that retains data after the pool-owning mm exits. Each case checks untouched bytes and parent isolation. Physical density and reclaim were tested separately on Neo hardware; the selftest does not infer them from logical RSS. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5469651a0899f3bf776c90f4a24e25e952fa0ba2) --- include/linux/mm_subpage.h | 15 + include/linux/mm_types.h | 2 + kernel/fork.c | 3 + mm/memory.c | 152 +++++++-- mm/user-subpage.c | 297 +++++++++++++++++- tools/testing/selftests/arm64/user4k/Makefile | 4 +- .../testing/selftests/arm64/user4k/cow-pool.c | 256 +++++++++++++++ 7 files changed, 690 insertions(+), 39 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/cow-pool.c diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h index 13344e3a92384..c4ac2eb6c4236 100644 --- a/include/linux/mm_subpage.h +++ b/include/linux/mm_subpage.h @@ -45,6 +45,21 @@ struct mm_subpage_pool *mm_subpage_pool_create_granule(gfp_t gfp, unsigned int s void mm_subpage_pool_get(struct mm_subpage_pool *pool); void mm_subpage_pool_put(struct mm_subpage_pool *pool); void mm_subpage_pool_close(struct mm_subpage_pool *pool); +#ifdef CONFIG_MM_SUBPAGE +/* Optional per-mm pool: default policy, one memory node, matching memcg. */ +struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long address); +void mm_subpage_cow_pool_exit(struct mm_struct *mm); +bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *folio); +bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset); +/* Return an initialized slot with its folio locked; may wait without a pool lock. */ +struct mm_subpage *mm_subpage_alloc_at_locked(struct mm_subpage_pool *pool, + unsigned int offset); +struct mm_subpage *mm_subpage_copy_at_locked(struct mm_subpage_pool *pool, + const struct mm_subpage *source, unsigned int offset); +#else +static inline void mm_subpage_cow_pool_exit(struct mm_struct *mm) { } +#endif /* Success consumes the folio reference. On error the caller retains it. */ int mm_subpage_pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index a8cc91f8966db..a075ab8f48bea 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -1284,6 +1284,8 @@ struct mm_struct { #ifdef CONFIG_MM_SUBPAGE /* VM_ACCOUNT granules; global commitment rounds this sum once. */ unsigned long committed_user_pages; + /* Shared COW slots; never inherited by a forked mm. */ + struct mm_subpage_cow_pools *cow_subpage_pool; #endif unsigned long locked_vm; /* Locked bytes >> MM_ACCOUNT_SHIFT */ atomic64_t pinned_vm; /* Pinned native physical pages */ diff --git a/kernel/fork.c b/kernel/fork.c index 0c9c9a9c12e3c..f26f22e67643b 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -1100,6 +1101,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p) mm->map_count = 0; #ifdef CONFIG_MM_SUBPAGE mm->committed_user_pages = 0; + mm->cow_subpage_pool = NULL; #endif mm->locked_vm = 0; atomic64_set(&mm->pinned_vm, 0); @@ -1215,6 +1217,7 @@ static inline void __mmput(struct mm_struct *mm) ksm_exit(mm); khugepaged_exit(mm); /* must run before exit_mmap */ exit_mmap(mm); + mm_subpage_cow_pool_exit(mm); mm_put_huge_zero_folio(mm); set_mm_exe_file(mm, NULL); if (!list_empty(&mm->mmlist)) { diff --git a/mm/memory.c b/mm/memory.c index dd87a48f42397..4de41f71d86ce 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -88,6 +88,7 @@ #include "pgalloc-track.h" #include +#include "user-subpage-internal.h" #include "internal.h" #include "swap.h" @@ -1326,6 +1327,98 @@ static inline struct folio *folio_prealloc(struct mm_struct *src_mm, } #ifdef CONFIG_MM_SUBPAGE +/* Skipping a busy owner must not trigger an avoidable global or memcg OOM. */ +static struct folio *cow_subpage_speculative_folio(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr) +{ + gfp_t alloc_gfp = GFP_HIGHUSER_MOVABLE | __GFP_NORETRY | __GFP_NOWARN; + gfp_t charge_gfp = GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN; + unsigned int order = max_t(unsigned int, mm_page_shift(mm), PAGE_SHIFT) - PAGE_SHIFT; + struct folio *folio = vma_alloc_folio(alloc_gfp, order, vma, addr); + + if (!folio) + return NULL; + if (mem_cgroup_charge(folio, mm, charge_gfp)) { + folio_put(folio); + return NULL; + } + folio_throttle_swaprate(folio, charge_gfp); + return folio; +} + +/* The returned initialized slot owns a reference and has its folio locked. */ +static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr, + struct mm_subpage *source, bool force_private, bool *pooled) +{ + struct mm_subpage_pool *pool; + struct mm_subpage *slot; + struct folio *folio; + unsigned int offset = vma_page_offset_at(vma, addr).offset; + int err; + +retry_private: + pool = force_private ? NULL : mm_subpage_cow_pool_get(mm, vma, addr); + *pooled = pool != NULL; + if (!pool) + pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!pool) + return ERR_PTR(-ENOMEM); + for (;;) { + slot = source ? mm_subpage_copy_at_locked(pool, source, offset) : + mm_subpage_alloc_at_locked(pool, offset); + if (!IS_ERR(slot) || PTR_ERR(slot) != -EAGAIN) + break; + folio = NULL; + if (*pooled) { + folio = cow_subpage_speculative_folio(mm, vma, addr); + if (!folio && mm_subpage_cow_wait_busy_at(pool, offset)) { + cond_resched(); + continue; + } + } + /* No reusable owner exists after speculative allocation failed. + * Ordinary allocation/charge semantics are now appropriate. + */ + if (!folio) + folio = folio_prealloc(mm, vma, addr, false); + if (!folio) { + slot = ERR_PTR(-ENOMEM); + break; + } + /* A concurrent cgroup move may change the fresh folio's charge. */ + if (*pooled && !mm_subpage_cow_folio_matches(pool, folio)) { + folio_put(folio); + mm_subpage_pool_put(pool); + force_private = true; + goto retry_private; + } + err = mm_subpage_pool_add_folio_at(pool, folio, offset, + *pooled ? GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN : GFP_KERNEL); + if (*pooled && err == -ENOMEM) { + if (mm_subpage_cow_wait_busy_at(pool, offset)) { + folio_put(folio); + cond_resched(); + continue; + } + err = mm_subpage_pool_add_folio_at(pool, folio, offset, GFP_KERNEL); + } + if (err) { + folio_put(folio); + if (err == -EEXIST) { + cond_resched(); + continue; + } + slot = ERR_PTR(err); + break; + } + } + if (!*pooled) + mm_subpage_pool_close(pool); + mm_subpage_pool_put(pool); + return slot; +} + static struct mm_subpage *anon_subpage_prealloc(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr) { @@ -4542,11 +4635,11 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) struct vm_area_struct *vma = vmf->vma; struct mm_struct *mm = vma->vm_mm; struct mm_subpage *source = NULL, *dest = NULL; - struct mm_subpage_pool *pool = NULL; struct folio *old = vmf->page ? page_folio(vmf->page) : NULL; struct folio *new = NULL; struct mmu_notifier_range range; bool copied = false, unshare = vmf->flags & FAULT_FLAG_UNSHARE; + bool force_private = false, pooled, fresh, retry_private = false; unsigned int offset; vm_fault_t ret = VM_FAULT_SIGBUS; pte_t entry; @@ -4599,35 +4692,29 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) goto out; offset = vma_page_offset_at(vma, vmf->address).offset; ret = VM_FAULT_OOM; - pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); - if (!pool) - goto out; - new = folio_prealloc(mm, vma, vmf->address, false); - if (!new) - goto out; - err = mm_subpage_pool_add_folio(pool, new, GFP_KERNEL); - if (err) { - folio_put(new); - goto out; - } - dest = source ? mm_subpage_copy_at(pool, source, offset) : - mm_subpage_alloc_at(pool, offset); +retry_copy: + dest = cow_subpage_prealloc(mm, vma, vmf->address, source, force_private, &pooled); if (IS_ERR(dest)) { if (PTR_ERR(dest) == -EHWPOISON) ret = VM_FAULT_HWPOISON; dest = NULL; goto out; } + new = mm_subpage_folio(dest); if (old && !source) { err = copy_user_subpage_range(&new->page, offset, vmf->page, pte_phys_mm(mm, vmf->orig_pte) & ~PAGE_MASK, mm_page_size(mm)); if (err) { + folio_unlock(new); ret = VM_FAULT_HWPOISON; goto out; } } - mm_subpage_pool_close(pool); + /* Initialisation is complete; the slot reference prevents migration/free. + * Do not carry a folio lock through MMU notifier callbacks. + */ + folio_unlock(new); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, vmf->address, vmf->address + mm_page_size(mm)); mmu_notifier_invalidate_range_start(&range); @@ -4635,6 +4722,19 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) folio_get(new); folio_lock(new); ret = 0; + /* A different fault may publish this owner's first PTE while we wait. + * Only the first publisher establishes folio rmap/LRU. Preflight nonlinear + * slot identity before taking PTL, with a fresh-folio fallback on ENOMEM. + */ + fresh = !mm_subpage_anon_root(dest); + if (!fresh) { + err = mm_subpage_bind_rmap(dest, vma, vmf->address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + retry_private = pooled && err == -ENOMEM; + goto unlock_folio; + } + } vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); if (!vmf->pte) goto unlock_folio; @@ -4643,7 +4743,8 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) goto unlock_pte; } mm_subpage_set_exclusive(dest); - err = mm_subpage_add_new_anon_rmap(dest, vma, vmf->address); + err = fresh ? mm_subpage_add_new_anon_rmap(dest, vma, vmf->address) : + mm_subpage_add_anon_rmap(dest, vma, vmf->address); if (WARN_ON_ONCE(err)) { ret = VM_FAULT_SIGBUS; goto unlock_pte; @@ -4661,7 +4762,8 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte)); /* Break-before-make also orders removal of the old slot's rmap. */ ptep_clear_flush(vma, vmf->address, vmf->pte); - subpage_add_new_lru(new, vma, 1); + if (fresh) + subpage_add_new_lru(new, vma, 1); set_pte_at(mm, vmf->address, vmf->pte, entry); update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); if (source) { @@ -4684,13 +4786,17 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) folio_unlock(new); folio_put(new); mmu_notifier_invalidate_range_end(&range); -out: - if (pool) { - mm_subpage_pool_close(pool); - if (dest) - mm_subpage_put(dest); - mm_subpage_pool_put(pool); + if (retry_private) { + mm_subpage_put(dest); + dest = NULL; + force_private = true; + retry_private = false; + ret = VM_FAULT_OOM; + goto retry_copy; } +out: + if (dest) + mm_subpage_put(dest); if (source) { if (copied) mm_subpage_put(source); /* Retired PTE reference, after TLBI. */ diff --git a/mm/user-subpage.c b/mm/user-subpage.c index 319d792ab5e9a..3adfb04ccdca1 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include #include @@ -222,6 +224,17 @@ struct mm_subpage_pool { unsigned long all_slots; u8 shift; bool closed; + /* Only a per-mm COW pool owns this charge-domain reference. */ + struct mem_cgroup *cow_memcg; + int cow_nid; + bool cow_shared; + /* Allocated only for shared COW pools; protected by pool->lock. */ + unsigned long *cow_free_slots; +}; + +#define COW_POOL_BITS 4 +struct mm_subpage_cow_pools { + struct mm_subpage_pool *pool[1U << COW_POOL_BITS]; }; static void owner_init(struct mm_subpage_owner *owner, struct mm_subpage_pool *pool, @@ -277,13 +290,147 @@ void mm_subpage_pool_put(struct mm_subpage_pool *pool) if (!refcount_dec_and_test(&pool->refs)) return; WARN_ON_ONCE(!pool->closed || !list_empty(&pool->owners)); + mem_cgroup_put(pool->cow_memcg); + kfree(pool->cow_free_slots); kfree_rcu(pool, rcu); } +/* Keep the first implementation within one unambiguous allocation domain. + * Policy-sensitive, locked, droppable and multi-node faults keep fresh backing. + * An mm can move cgroups; old charged slots must not satisfy its new faults. + */ +struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long address) +{ + struct mm_subpage_cow_pools *pools, *new_pools; + struct mm_subpage_pool *pool, *new; + unsigned int bucket = hash_long(address >> PAGE_SHIFT, COW_POOL_BITS); + struct mem_cgroup *memcg; + bool matches; + int nid; + + if ((vma->vm_flags & (VM_LOCKED | VM_DROPPABLE | VM_SPECIAL)) || + num_online_nodes() != 1 || num_node_state(N_MEMORY) != 1) + return NULL; +#ifdef CONFIG_NUMA + if (vma_policy(vma) || current->mempolicy) + return NULL; +#endif + nid = first_node(node_states[N_MEMORY]); + if (!node_isset(nid, cpuset_current_mems_allowed)) + return NULL; + /* Publication pairs with cmpxchg: every bucket starts empty. Adjacent + * 4K leaves in one native virtual group always choose the same pool. + * Faulting threads can migrate CPUs without changing that identity. + */ + pools = smp_load_acquire(&mm->cow_subpage_pool); + if (!pools) { + new_pools = kzalloc(sizeof(*new_pools), GFP_KERNEL | __GFP_ACCOUNT); + if (!new_pools) + return NULL; + pools = cmpxchg(&mm->cow_subpage_pool, NULL, new_pools); + if (!pools) + pools = new_pools; + else + kfree(new_pools); + } + memcg = get_mem_cgroup_from_mm(mm); + /* Acquire the immutable charge domain published with this bucket. */ + pool = smp_load_acquire(&pools->pool[bucket]); + if (!pool) { + new = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!new) { + mem_cgroup_put(memcg); + return NULL; + } + new->cow_free_slots = kcalloc(SUBPAGES_PER_FOLIO, + sizeof(*new->cow_free_slots), + GFP_KERNEL | __GFP_ACCOUNT); + if (!new->cow_free_slots) { + mm_subpage_pool_close(new); + mm_subpage_pool_put(new); + mem_cgroup_put(memcg); + return NULL; + } + new->cow_memcg = memcg; + new->cow_nid = nid; + new->cow_shared = true; + pool = cmpxchg(&pools->pool[bucket], NULL, new); + if (!pool) { + pool = new; + /* The mm owns the initial pool reference and its memcg ref. */ + mm_subpage_pool_get(pool); + return pool; + } + matches = pool->cow_memcg == memcg && pool->cow_nid == nid; + mm_subpage_pool_close(new); + mm_subpage_pool_put(new); + } else { + matches = pool->cow_memcg == memcg && pool->cow_nid == nid; + mem_cgroup_put(memcg); + } + if (!matches) + return NULL; + mm_subpage_pool_get(pool); + return pool; +} + +bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *folio) +{ + return pool->cow_memcg == folio_memcg(folio) && pool->cow_nid == folio_nid(folio); +} + +void mm_subpage_cow_pool_exit(struct mm_struct *mm) +{ + struct mm_subpage_cow_pools *pools = xchg(&mm->cow_subpage_pool, NULL); + unsigned int i; + + /* No faults remain once mm_users reaches zero. Slot owners can outlive mm. */ + if (!pools) + return; + for (i = 0; i < ARRAY_SIZE(pools->pool); i++) { + if (!pools->pool[i]) + continue; + mm_subpage_pool_close(pools->pool[i]); + mm_subpage_pool_put(pools->pool[i]); + } + kfree(pools); +} + +/* Counts include unavailable (busy/migrating) owners, so zero is conclusive. + * A nonzero count still requires all the normal owner eligibility checks. + */ +static void cow_update_free(struct mm_subpage_pool *pool, unsigned long mask, int delta) +{ + unsigned int i; + + lockdep_assert_held(&pool->lock); + if (!pool->cow_shared) + return; + for_each_set_bit(i, &mask, SUBPAGES_PER_FOLIO) { + VM_BUG_ON(delta < 0 && !pool->cow_free_slots[i]); + pool->cow_free_slots[i] += delta; + } +} + +static bool cow_has_free(struct mm_subpage_pool *pool, unsigned long eligible) +{ + unsigned int i; + + lockdep_assert_held(&pool->lock); + if (!pool->cow_shared) + return true; + for_each_set_bit(i, &eligible, SUBPAGES_PER_FOLIO) + if (pool->cow_free_slots[i]) + return true; + return false; +} + static void owner_remove(struct mm_subpage_owner *owner) { lockdep_assert_held(&owner->pool->lock); owner->detached = true; + cow_update_free(owner->pool, owner->pool->all_slots & ~owner->used, -1); list_del(&owner->all); list_del_init(&owner->available); if (owner->pool->unissued == owner) @@ -677,17 +824,81 @@ void mm_subpage_pool_close(struct mm_subpage_pool *pool) owner_free(owner, true); } +static bool cow_owner_eligible(struct mm_subpage_pool *pool, struct folio *folio) +{ + if (!pool->cow_shared) + return true; + /* A previously unlocked owner can be mlocked or fully MADV_FREE'd + * after entering the pool. Never place a new live slot in such backing. + */ + return !folio_test_mlocked(folio) && !folio_test_unevictable(folio) && + !folio_test_ksm(folio) && + (!folio_test_anon(folio) || folio_test_swapbacked(folio)) && + mm_subpage_cow_folio_matches(pool, folio); +} + +/* Pressure-only fallback after a non-OOM speculative allocation failed. + * Return true when existing compatible backing can be retried. Unlike the + * normal bounded search, inspect all owners before allowing an OOM-capable + * allocation: a busy slot farther down the list may require no new charge. + */ +bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset) +{ + struct mm_subpage_owner *owner; + struct folio *wait = NULL; + unsigned long flags, eligible = offset_mask(pool->shift, offset); + + spin_lock_irqsave(&pool->lock, flags); + if (!pool->closed && pool->cow_shared && cow_has_free(pool, eligible)) { + list_for_each_entry(owner, &pool->available, available) { + if (!owner->migrating && (~owner->used & eligible) && + cow_owner_eligible(pool, owner->folio) && + !folio_test_swapcache(owner->folio) && + !folio_test_writeback(owner->folio)) { + wait = owner->folio; + folio_get(wait); + break; + } + } + } + spin_unlock_irqrestore(&pool->lock, flags); + if (wait) { + folio_wait_locked(wait); + folio_put(wait); + return true; + } + return false; +} + static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, unsigned long eligible) { struct mm_subpage_owner *owner; + unsigned int visited = 0; lockdep_assert_held(&pool->lock); - list_for_each_entry(owner, &pool->available, available) + if (!cow_has_free(pool, eligible)) + return NULL; + list_for_each_entry(owner, &pool->available, available) { + /* A fixed-offset stream must not scan every partially used owner + * on each fault. New COW supply is inserted first, so a bounded + * miss can always make progress using a fresh folio. + */ + if (pool->cow_shared && visited++ == 32) { + /* The next search starts past this incompatible prefix. A + * phased offset stream must not strand older reusable slots. + */ + list_rotate_to_front(&owner->available, &pool->available); + break; + } /* A nonpresent PTE may still need any of this folio's slot bytes. */ if (!owner->migrating && (~owner->used & eligible) && - !folio_test_swapcache(owner->folio)) + (!pool->cow_shared || !folio_test_locked(owner->folio)) && + cow_owner_eligible(pool, owner->folio) && + !folio_test_swapcache(owner->folio) && + !folio_test_writeback(owner->folio)) return owner; + } return NULL; } @@ -720,13 +931,20 @@ static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, spin_lock_irqsave(&pool->lock, flags); if (pool->closed) ret = -ESHUTDOWN; - else if (find_available(pool, eligible)) - /* Another allocator supplied backing while we allocated metadata. */ + else if (pool->unissued || find_available(pool, eligible)) + /* Another allocator supplied backing while we allocated metadata. + * Preserve the single-unissued-owner invariant even when a shared + * COW lookup skips a concurrently locked or rotating list prefix. + */ ret = -EEXIST; else { mm_subpage_pool_get(pool); list_add_tail(&owner->all, &pool->owners); - list_add_tail(&owner->available, &pool->available); + cow_update_free(pool, pool->all_slots, 1); + if (pool->cow_shared) + list_add(&owner->available, &pool->available); + else + list_add_tail(&owner->available, &pool->available); pool->backing_pages++; pool->unissued = owner; } @@ -765,13 +983,14 @@ unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool) } static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, - unsigned long eligible, bool *needs_zero) + unsigned long eligible, bool *needs_zero, bool lock_folio) { struct mm_subpage_owner *owner; struct mm_subpage *subpage; unsigned long flags; unsigned int index; +retry: spin_lock_irqsave(&pool->lock, flags); if (pool->closed) { subpage = ERR_PTR(-ESHUTDOWN); @@ -782,6 +1001,34 @@ static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, subpage = ERR_PTR(-EAGAIN); goto out; } + if (lock_folio) { + if (!folio_trylock(owner->folio)) { + struct folio *wait; + + if (pool->cow_shared) { + /* Another COW fault won this folio after the lookup. + * Try other backing, or supply another folio, rather + * than making parallel faults queue on its lock. + */ + spin_unlock_irqrestore(&pool->lock, flags); + cond_resched(); + goto retry; + } + wait = owner->folio; + folio_get(wait); + spin_unlock_irqrestore(&pool->lock, flags); + folio_wait_locked(wait); + folio_put(wait); + goto retry; + } + /* Reclaim may have acquired the folio after find_available's check. */ + if (folio_test_swapcache(owner->folio) || folio_test_writeback(owner->folio) || + !cow_owner_eligible(pool, owner->folio)) { + folio_unlock(owner->folio); + spin_unlock_irqrestore(&pool->lock, flags); + return ERR_PTR(-EAGAIN); + } + } if (pool->unissued == owner) /* Transfer the supplied reference to the first reservation. */ pool->unissued = NULL; @@ -789,6 +1036,7 @@ static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, folio_get(owner->folio); index = __ffs(~owner->used & eligible); __set_bit(index, &owner->used); + cow_update_free(pool, BIT(index), -1); if (needs_zero) *needs_zero = !test_bit(index, &owner->zeroed); __clear_bit(index, &owner->zeroed); @@ -804,10 +1052,10 @@ static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, return subpage; } -static struct mm_subpage *subpage_alloc(struct mm_subpage_pool *pool, unsigned long eligible) +static struct mm_subpage *subpage_alloc(struct mm_subpage_pool *pool, unsigned long eligible, bool lock_folio) { bool needs_zero; - struct mm_subpage *subpage = subpage_reserve(pool, eligible, &needs_zero); + struct mm_subpage *subpage = subpage_reserve(pool, eligible, &needs_zero, lock_folio); if (IS_ERR(subpage)) return subpage; @@ -822,7 +1070,7 @@ static struct mm_subpage *subpage_alloc(struct mm_subpage_pool *pool, unsigned l struct mm_subpage *mm_subpage_alloc(struct mm_subpage_pool *pool) { - return subpage_alloc(pool, pool->all_slots); + return subpage_alloc(pool, pool->all_slots, false); } struct mm_subpage *mm_subpage_alloc_at(struct mm_subpage_pool *pool, unsigned int offset) @@ -831,7 +1079,7 @@ struct mm_subpage *mm_subpage_alloc_at(struct mm_subpage_pool *pool, unsigned in if (!eligible) return ERR_PTR(-EINVAL); - return subpage_alloc(pool, eligible); + return subpage_alloc(pool, eligible, false); } static void subpage_release_locked(struct mm_subpage *subpage, unsigned long flags) @@ -851,6 +1099,7 @@ static void subpage_release_locked(struct mm_subpage *subpage, unsigned long fla subpage->rmap_bound = false; } __clear_bit(subpage->index, &owner->used); + cow_update_free(pool, BIT(subpage->index), 1); release = !owner->used && !owner->migrating; if (release) owner_remove(owner); @@ -872,14 +1121,14 @@ static void subpage_release(struct mm_subpage *subpage) } static struct mm_subpage *subpage_copy(struct mm_subpage_pool *pool, - const struct mm_subpage *source, unsigned long eligible) + const struct mm_subpage *source, unsigned long eligible, bool lock_folio) { struct mm_subpage *dest; int err; if (pool->shift != mm_subpage_shift(source)) return ERR_PTR(-EINVAL); - dest = subpage_reserve(pool, eligible, NULL); + dest = subpage_reserve(pool, eligible, NULL, lock_folio); if (IS_ERR(dest)) return dest; @@ -888,6 +1137,8 @@ static struct mm_subpage *subpage_copy(struct mm_subpage_pool *pool, mm_subpage_size(source)); if (err) { /* Never publish a reservation whose copy did not finish. */ + if (lock_folio) + folio_unlock(dest->owner->folio); subpage_release(dest); return ERR_PTR(err); } @@ -898,7 +1149,7 @@ static struct mm_subpage *subpage_copy(struct mm_subpage_pool *pool, struct mm_subpage *mm_subpage_copy(struct mm_subpage_pool *pool, const struct mm_subpage *source) { - return subpage_copy(pool, source, pool->all_slots); + return subpage_copy(pool, source, pool->all_slots, false); } struct mm_subpage *mm_subpage_copy_at(struct mm_subpage_pool *pool, @@ -908,7 +1159,23 @@ struct mm_subpage *mm_subpage_copy_at(struct mm_subpage_pool *pool, if (!eligible) return ERR_PTR(-EINVAL); - return subpage_copy(pool, source, eligible); + return subpage_copy(pool, source, eligible, false); +} + +struct mm_subpage *mm_subpage_alloc_at_locked(struct mm_subpage_pool *pool, + unsigned int offset) +{ + unsigned long eligible = offset_mask(pool->shift, offset); + + return eligible ? subpage_alloc(pool, eligible, true) : ERR_PTR(-EINVAL); +} + +struct mm_subpage *mm_subpage_copy_at_locked(struct mm_subpage_pool *pool, + const struct mm_subpage *source, unsigned int offset) +{ + unsigned long eligible = offset_mask(pool->shift, offset); + + return eligible ? subpage_copy(pool, source, eligible, true) : ERR_PTR(-EINVAL); } /* The page-extension RCU section also protects the pool until its lock is held. */ @@ -956,6 +1223,7 @@ static struct mm_subpage *restore_bound_slot(struct folio *folio, unsigned int o else folio_get(folio); __set_bit(index, &owner->used); + cow_update_free(pool, BIT(index), -1); __clear_bit(index, &owner->zeroed); if (owner->used == pool->all_slots) list_del_init(&owner->available); @@ -1117,6 +1385,7 @@ int mm_subpage_migrate_prepare(struct folio *src, struct folio *dst) mm_subpage_pool_get(pool); spin_lock_irqsave(&pool->lock, flags); list_add_tail(&new->all, &pool->owners); + cow_update_free(pool, pool->all_slots, 1); pool->backing_pages++; ret = owner_bind(new); if (ret) diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index cc89cc93293fe..4b7c4cf17006d 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -2,7 +2,7 @@ TEST_GEN_PROGS := page-contract proc-args TEST_GEN_PROGS_EXTENDED := iommufd-init coarse-shmem-init bpf-uptr-init vhost-log-init uprobes-init user-events-init uffd-move-init swap-fixture-init exec-fixture-init mixed-process memory-interfaces advice reclaim-advice proc-pages proc-memory core-fixture-init context-stress-init mlock-fixture-init lazyfree-fixture-init thread-fixture-init splice-fixture-init dio-fixture-init uring-fixture-init gem-fixture-init compat-fixture-init granule-proc-init granule-swap-init aslr-fixture-init cpu-granule-init user16k-smoke-init cpu-granule16-init granule-threeway-init compat-mlock-init mglru-granule-init memcg-pressure-init memcg-accounting-init runtime-app-init fex-contract-init exec-policy-init uffd-wp-init shmem-uffd-swap-init compat-uffd-init shmem-hole-init shmem-thp-init shmem-collapse-init shmem-pin-init shmem-race-init shmem-pressure-init TEST_GEN_PROGS_EXTENDED += vfio-fixture-init sysv-shm-init perf-ring-init -TEST_GEN_PROGS_EXTENDED += aio-ring-init numa-policy-init numa-balancing-init +TEST_GEN_PROGS_EXTENDED += aio-ring-init numa-policy-init numa-balancing-init cow-pool CFLAGS += -Wall -Wextra -O2 -ffixed-x18 TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe compat-vfio-probe @@ -12,7 +12,7 @@ include ../../lib.mk $(OUTPUT)/page-contract: page-contract.c x18-probe.S -$(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init: LDLIBS += -pthread +$(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool: LDLIBS += -pthread # Freestanding ELF32 probes exercise the compat ABI without an armhf sysroot. AARCH32_CC ?= clang --target=arm-linux-gnueabihf diff --git a/tools/testing/selftests/arm64/user4k/cow-pool.c b/tools/testing/selftests/arm64/user4k/cow-pool.c new file mode 100644 index 0000000000000..bb25904d68c04 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/cow-pool.c @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Exercise shared COW backing under a selected 4K userspace ABI. Run through + * the platform's ABI launcher; this test never changes its own page contract. + * Every selected leaf changes one byte, and all other bytes must survive. + * Physical density and reclaim require privileged, platform-specific controls + * and are deliberately not inferred from this unprivileged contents test. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#define BYTES (64UL << 20) +#define LEAF 4096UL +#define LEAVES (BYTES / LEAF) +#define WORKERS 8 + +enum mode { DENSE, SPARSE, PHASED, RANDOM, LOCKED, LATE_LOCKED, LAZYFREE }; +struct scenario { + const char *name; + enum mode mode; + unsigned int workers; +}; + +static unsigned char *data; +static size_t order[LEAVES]; +static pthread_barrier_t barrier; +static const struct scenario *current; + +static unsigned char initial(size_t leaf, size_t byte) +{ + return leaf * 17 + byte * 3 + 0x35; +} + +static unsigned char changed(size_t leaf) +{ + return initial(leaf, 0) ^ 0xa5; +} + +static bool verify(unsigned int generation, bool discard) +{ + unsigned int stride = current->mode == SPARSE ? 4 : 1; + size_t leaf, byte; + + for (leaf = 0; leaf < LEAVES; leaf++) { + if (generation && discard && leaf % 4 == 0) + continue; + for (byte = 0; byte < LEAF; byte++) { + unsigned char want = initial(leaf, byte); + + if (generation && byte == 0 && leaf % stride == 0) + want ^= generation == 1 ? 0xa5 : 0x5a; + if (data[leaf * LEAF + byte] != want) { + ksft_print_msg("pid=%d leaf=%zu byte=%zu got=%u want=%u\n", + getpid(), leaf, byte, data[leaf * LEAF + byte], want); + return false; + } + } + } + return true; +} + +static bool wait_ok(pid_t pid) +{ + pid_t result; + int status; + + do { + result = waitpid(pid, &status, 0); + } while (result < 0 && errno == EINTR); + return result > 0 && WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +static void *writer(void *arg) +{ + unsigned int id = (uintptr_t)arg; + size_t leaf, index; + unsigned int phase; + + pthread_barrier_wait(&barrier); + if (current->mode == PHASED || current->mode == LATE_LOCKED || + current->mode == LAZYFREE) { + for (phase = 0; phase < 4; phase++) { + for (leaf = phase + 4UL * id; leaf < LEAVES; + leaf += 4UL * current->workers) + data[leaf * LEAF] = changed(leaf); + pthread_barrier_wait(&barrier); + if (phase == 0 && id == 0) { + if (current->mode == LATE_LOCKED && mlock(data, BYTES)) + _exit(2); + if (current->mode == LAZYFREE) + for (leaf = 0; leaf < LEAVES; leaf += 4) + if (madvise(data + leaf * LEAF, LEAF, MADV_FREE)) + _exit(2); + } + pthread_barrier_wait(&barrier); + } + } else { + unsigned int stride = current->mode == SPARSE ? 4 : 1; + + for (index = id; index < LEAVES / stride; index += current->workers) { + leaf = current->mode == RANDOM ? order[index] : index * stride; + data[leaf * LEAF] = changed(leaf); + } + } + return NULL; +} + +static bool exercise(const struct scenario *scenario) +{ + pthread_t workers[WORKERS]; + struct timespec start, end; + unsigned int t; + pid_t child; + + current = scenario; + child = fork(); + if (child < 0) + return false; + if (!child) { + if (current->mode == LOCKED && mlock(data, BYTES)) + _exit(2); + if (pthread_barrier_init(&barrier, NULL, current->workers)) + _exit(2); + clock_gettime(CLOCK_MONOTONIC, &start); + for (t = 0; t < current->workers; t++) + if (pthread_create(&workers[t], NULL, writer, (void *)(uintptr_t)t)) + _exit(2); + for (t = 0; t < current->workers; t++) + if (pthread_join(workers[t], NULL)) + _exit(2); + clock_gettime(CLOCK_MONOTONIC, &end); + ksft_print_msg("%s fault/write phase %.6f s\n", current->name, + end.tv_sec - start.tv_sec + (end.tv_nsec - start.tv_nsec) * 1e-9); + _exit(verify(1, current->mode == LAZYFREE) ? 0 : 3); + } + return wait_ok(child) && verify(0, false); +} + +static bool owner_exit(void) +{ + static const struct scenario scenario = { "owner-mm exit", DENSE, 1 }; + int ready[2]; + pid_t child, descendant; + size_t leaf; + char signal; + + current = &scenario; + if (prctl(PR_SET_CHILD_SUBREAPER, 1) || pipe(ready)) + return false; + child = fork(); + if (child < 0) { + close(ready[0]); + close(ready[1]); + return false; + } + if (!child) { + close(ready[1]); + for (leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] = changed(leaf); + if (!verify(1, false)) + _exit(3); + descendant = fork(); + if (descendant < 0) + _exit(2); + if (descendant) + _exit(0); + /* Root signals only after the original pool-owning mm has exited. */ + if (read(ready[0], &signal, 1) != 1 || signal != 'g') + _exit(2); + close(ready[0]); + if (!verify(1, false)) + _exit(3); + for (leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] ^= 0xff; + _exit(verify(2, false) ? 0 : 3); + } + close(ready[0]); + if (!wait_ok(child)) { + close(ready[1]); + return false; + } + if (write(ready[1], "g", 1) != 1) { + close(ready[1]); + return false; + } + close(ready[1]); + return wait_ok(-1) && verify(0, false); +} + +int main(void) +{ + static const struct scenario scenarios[] = { + { "dense single-thread COW", DENSE, 1 }, + { "dense concurrent COW", DENSE, WORKERS }, + { "single-offset sparse COW", SPARSE, 1 }, + { "phased-offset concurrent COW", PHASED, WORKERS }, + { "random-order concurrent COW", RANDOM, WORKERS }, + { "locked COW", LOCKED, WORKERS }, + { "mlock after first offset", LATE_LOCKED, WORKERS }, + { "MADV_FREE after first offset", LAZYFREE, WORKERS }, + }; + struct rlimit limit; + uint32_t seed = 0x9148140; + size_t leaf, byte, i, j, tmp; + bool can_lock; + + ksft_print_header(); + if (getauxval(AT_PAGESZ) != LEAF) + ksft_exit_skip("run under a selected 4K userspace ABI\n"); + ksft_set_plan(9); + alarm(180); + data = mmap(NULL, BYTES, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (data == MAP_FAILED) + ksft_exit_fail_msg("mmap: %s\n", strerror(errno)); + for (leaf = 0; leaf < LEAVES; leaf++) + for (byte = 0; byte < LEAF; byte++) + data[leaf * LEAF + byte] = initial(leaf, byte); + for (i = 0; i < LEAVES; i++) + order[i] = i; + for (i = LEAVES; i > 1; i--) { + seed = seed * 1664525 + 1013904223; + j = seed % i; + tmp = order[j]; + order[j] = order[i - 1]; + order[i - 1] = tmp; + } + can_lock = !getrlimit(RLIMIT_MEMLOCK, &limit) && limit.rlim_cur >= BYTES; + for (i = 0; i < sizeof(scenarios) / sizeof(scenarios[0]); i++) { + if (!can_lock && (scenarios[i].mode == LOCKED || + scenarios[i].mode == LATE_LOCKED)) { + ksft_test_result_skip("%s needs a 64MiB memlock limit\n", scenarios[i].name); + continue; + } + ksft_test_result(exercise(&scenarios[i]), "%s preserves contents and parent\n", + scenarios[i].name); + } + ksft_test_result(owner_exit(), "descendant retains COW data after owner mm exits\n"); + munmap(data, BYTES); + ksft_finished(); +} From 0a712ff8894791f9b04713d4dcb31efac4d2d17d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 17:37:36 -0400 Subject: [PATCH 0861/2680] mm: reuse subpage backing when refilling anonymous holes A missing 4K PTE in a partially populated native 16K window currently gets a fresh private folio. Existing neighbors are skipped and the unused quarters are stranded when its pool closes. Repeated per-quarter hole refills can leave 16 MiB of logical data backed by 64 MiB of native pages. Reserve the missing-quarter mask from one compatible shared owner, keeping whole-window private allocation for initial dense faults. Initialize and hold the selected folio through reverse-map setup and PTE publication. Revalidate the missing mask under the PTE lock, release surplus or losing reservations, and retain the private fallback for metadata allocation failure. Existing pool charge, NUMA, locking and droppable exclusions apply. Add density, neighbor-prefault, concurrent refill, owner-exit, and controlled reclaim selftests. On native 16K Neo hardware with the 4K userspace ABI, the 16 MiB refill fixture finishes at 16.09 MiB instead of 64 MiB, with byte-exact contents. Initial dense backing stays at 16 MiB. Native/4K ABI, fork/exec, mprotect, existing concurrent COW and GPU quarter-refault controls pass. Controlled reclaim swaps all 4096 test leaves after original-owner exit, then passes byte-exact readback. Subnautica menu/world backing snapshots show lower native backing overhead; this is not an FPS qualification. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e8fb3863c6103fa0150ba98979756e6685e533d9) --- include/linux/mm_subpage.h | 9 + mm/memory.c | 183 ++++++++--- mm/user-subpage.c | 133 ++++++-- tools/testing/selftests/arm64/user4k/Makefile | 4 +- .../selftests/arm64/user4k/README.hole-refill | 29 ++ .../arm64/user4k/README.hole-refill-race | 33 ++ .../arm64/user4k/README.hole-refill-reclaim | 36 +++ .../arm64/user4k/anon-hole-refill-race.c | 298 +++++++++++++++++ .../arm64/user4k/anon-hole-refill-reclaim.c | 299 ++++++++++++++++++ .../selftests/arm64/user4k/anon-hole-refill.c | 179 +++++++++++ 10 files changed, 1140 insertions(+), 63 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/README.hole-refill create mode 100644 tools/testing/selftests/arm64/user4k/README.hole-refill-race create mode 100644 tools/testing/selftests/arm64/user4k/README.hole-refill-reclaim create mode 100644 tools/testing/selftests/arm64/user4k/anon-hole-refill-race.c create mode 100644 tools/testing/selftests/arm64/user4k/anon-hole-refill-reclaim.c create mode 100644 tools/testing/selftests/arm64/user4k/anon-hole-refill.c diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h index c4ac2eb6c4236..9a1d33166db30 100644 --- a/include/linux/mm_subpage.h +++ b/include/linux/mm_subpage.h @@ -52,6 +52,15 @@ struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, void mm_subpage_cow_pool_exit(struct mm_struct *mm); bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *folio); bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset); +/* Exact physical-slot mask, one owner and folio lock for the entire batch. + * slots is indexed by physical offset >> pool shift; only mask bits are set. + * Success returns initialized owned slots; errors leave slots untouched. + */ +int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mask, + struct mm_subpage **slots); +int mm_subpage_pool_add_folio_mask(struct mm_subpage_pool *pool, struct folio *folio, + unsigned long mask, gfp_t gfp); +bool mm_subpage_cow_wait_busy_mask(struct mm_subpage_pool *pool, unsigned long mask); /* Return an initialized slot with its folio locked; may wait without a pool lock. */ struct mm_subpage *mm_subpage_alloc_at_locked(struct mm_subpage_pool *pool, unsigned int offset); diff --git a/mm/memory.c b/mm/memory.c index 4de41f71d86ce..1c82ee38e561c 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -6338,17 +6338,95 @@ static void map_anon_folio_pte_pf(struct folio *folio, pte_t *pte, } #ifdef CONFIG_MM_SUBPAGE +/* One native-window batch, initialized and locked before PTE publication. */ +static int anon_subpage_prealloc_mask(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr, + unsigned long mask, bool use_pool, bool *pooled, + struct mm_subpage **slots) +{ + struct mm_subpage_pool *pool; + struct folio *folio; + int err; + +retry_private: + pool = use_pool ? mm_subpage_cow_pool_get(mm, vma, addr) : NULL; + *pooled = pool != NULL; + if (!pool) + pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!pool) + return -ENOMEM; + for (;;) { + err = mm_subpage_alloc_mask_locked(pool, mask, slots); + if (err != -EAGAIN) + break; + folio = NULL; + if (*pooled) { + folio = cow_subpage_speculative_folio(mm, vma, addr); + if (!folio && mm_subpage_cow_wait_busy_mask(pool, mask)) { + cond_resched(); + continue; + } + } + if (!folio) + folio = folio_prealloc(mm, vma, addr, false); + if (!folio) { + err = -ENOMEM; + break; + } + if (*pooled && !mm_subpage_cow_folio_matches(pool, folio)) { + folio_put(folio); + mm_subpage_pool_put(pool); + use_pool = false; + goto retry_private; + } + err = mm_subpage_pool_add_folio_mask(pool, folio, mask, + *pooled ? GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN : GFP_KERNEL); + if (*pooled && err == -ENOMEM) { + if (mm_subpage_cow_wait_busy_mask(pool, mask)) { + folio_put(folio); + cond_resched(); + continue; + } + err = mm_subpage_pool_add_folio_mask(pool, folio, mask, GFP_KERNEL); + } + if (err) { + folio_put(folio); + if (err == -EEXIST) { + cond_resched(); + continue; + } + break; + } + } + if (!*pooled) + mm_subpage_pool_close(pool); + mm_subpage_pool_put(pool); + return err; +} + +static unsigned long anon_subpage_missing_mask(pte_t *first, unsigned int nr, + unsigned int target, unsigned int offset) +{ + unsigned long mask = 0; + unsigned int i; + + for (i = 0; i < nr; i++) + if (i == target || pte_none(ptep_get(first + i))) + mask |= BIT(offset + i); + return mask; +} + static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; struct mm_struct *mm = vma->vm_mm; struct mm_subpage *slots[PAGE_SIZE / MM_SUBPAGE_SIZE] = {}; struct vm_page_offset pos = vma_page_offset_at(vma, vmf->address); - struct mm_subpage_pool *pool; struct folio *folio; - unsigned long start, end, lower; - unsigned int first_offset, nr, target, i, mapped = 0; - vm_fault_t ret = VM_FAULT_OOM; + unsigned long start, end, lower, mask, current_mask, window_mask; + unsigned int first_offset, nr, target, i, index, mapped; + bool pooled, fresh, force_private = false, retry_private; + vm_fault_t ret; pte_t *first; int err; @@ -6357,34 +6435,48 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) start = vmf->address - min_t(unsigned long, pos.offset, vmf->address - lower); end = vmf->address + min(PAGE_SIZE - pos.offset, pmd_addr_end_mm(mm, vmf->address, vma->vm_end) - vmf->address); - first_offset = pos.offset - (vmf->address - start); + first_offset = (pos.offset - (vmf->address - start)) >> mm_page_shift(mm); nr = (end - start) >> mm_page_shift(mm); target = (vmf->address - start) >> mm_page_shift(mm); - - pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); - if (!pool) + window_mask = GENMASK(first_offset + nr - 1, first_offset); +retry: + /* This optimistic snapshot only selects backing; PTL revalidates it. + * Initial dense faults retain one-folio, whole-window faultaround. + */ + vmf->pte = pte_offset_map_mm(mm, vmf->pmd, vmf->address); + if (!vmf->pte) + return 0; + mask = anon_subpage_missing_mask(vmf->pte - target, nr, target, first_offset); + pte_unmap(vmf->pte); + vmf->pte = NULL; + err = anon_subpage_prealloc_mask(mm, vma, vmf->address, mask, + mask != window_mask && !force_private && !userfaultfd_armed(vma), + &pooled, slots); + if (err) return VM_FAULT_OOM; - /* Supply initialises every byte once, including the unissued slots. */ - folio = folio_prealloc(mm, vma, vmf->address, false); - if (!folio) - goto release; - err = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); - if (err) { - folio_put(folio); - goto release; - } - for (i = 0; i < nr; i++) { - slots[i] = mm_subpage_alloc_at(pool, first_offset + i * mm_page_size(mm)); - if (IS_ERR(slots[i])) { - slots[i] = NULL; - goto release; - } - } - + folio = mm_subpage_folio(slots[__ffs(mask)]); /* PTE references may disappear as soon as we drop the PTL. */ folio_get(folio); - folio_lock(folio); + fresh = !mm_subpage_anon_root(slots[__ffs(mask)]); + mapped = 0; + retry_private = false; ret = 0; + /* Existing owners may need nonlinear identities. Allocate metadata before + * PTL and keep the established fresh-folio fallback on metadata ENOMEM. + */ + if (!fresh) { + for_each_set_bit(index, &mask, ARRAY_SIZE(slots)) { + unsigned long address = start + + ((index - first_offset) << mm_page_shift(mm)); + + err = mm_subpage_bind_rmap(slots[index], vma, address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + retry_private = pooled && err == -ENOMEM; + goto unlock_folio; + } + } + } vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); if (!vmf->pte) goto unlock_folio; @@ -6405,20 +6497,30 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) } first = vmf->pte - target; + current_mask = anon_subpage_missing_mask(first, nr, target, first_offset); + /* Do not lose newly missing neighbors or publish sparse dense-path supply. + * Return to the fault handler on a racing mask change instead of spinning + * under repeated zaps. No mapping has been published yet. + */ + if ((current_mask & ~mask) || + (mask == window_mask && current_mask != mask && + !force_private && !userfaultfd_armed(vma))) + goto unlock_pte; for (i = 0; i < nr; i++) { unsigned long address = start + i * mm_page_size(mm); pte_t entry; - /* Do not replace existing mappings or userfaultfd markers nearby. */ - if (i != target && !pte_none(ptep_get(first + i))) + index = first_offset + i; + if (!(current_mask & BIT(index))) continue; - mm_subpage_set_exclusive(slots[i]); - err = mm_subpage_add_new_anon_rmap(slots[i], vma, address); + mm_subpage_set_exclusive(slots[index]); + err = fresh ? mm_subpage_add_new_anon_rmap(slots[index], vma, address) : + mm_subpage_add_anon_rmap(slots[index], vma, address); if (WARN_ON_ONCE(err)) { ret = VM_FAULT_SIGBUS; break; } - entry = phys_pte_mm(mm, mm_subpage_phys(slots[i]), vma->vm_page_prot); + entry = phys_pte_mm(mm, mm_subpage_phys(slots[index]), vma->vm_page_prot); entry = pte_sw_mkyoung(entry); if (vma->vm_flags & VM_WRITE) entry = pte_mkwrite(pte_mkdirty(entry), vma); @@ -6426,11 +6528,12 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) entry = pte_mkuffd_wp(entry); set_pte_at(mm, address, first + i, entry); update_mmu_cache(vma, address, first + i); - slots[i] = NULL; /* The PTE now owns the reference. */ + slots[index] = NULL; /* The PTE now owns the reference. */ mapped++; } if (mapped) { - subpage_add_new_lru(folio, vma, mapped); + if (fresh) + subpage_add_new_lru(folio, vma, mapped); add_mm_counter(mm, MM_ANONPAGES, mapped); count_mthp_stat(0, MTHP_STAT_ANON_FAULT_ALLOC); } @@ -6441,13 +6544,19 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) folio_unlock(folio); folio_put(folio); release: - mm_subpage_pool_close(pool); - for (i = 0; i < ARRAY_SIZE(slots); i++) - if (slots[i]) + for (i = 0; i < ARRAY_SIZE(slots); i++) { + if (slots[i]) { mm_subpage_put(slots[i]); - mm_subpage_pool_put(pool); + slots[i] = NULL; + } + } + if (retry_private) { + force_private = true; + goto retry; + } return ret; } + #endif /* diff --git a/mm/user-subpage.c b/mm/user-subpage.c index 3adfb04ccdca1..c7876fd34a091 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -842,16 +842,18 @@ static bool cow_owner_eligible(struct mm_subpage_pool *pool, struct folio *folio * normal bounded search, inspect all owners before allowing an OOM-capable * allocation: a busy slot farther down the list may require no new charge. */ -bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset) +static bool cow_wait_busy(struct mm_subpage_pool *pool, unsigned long eligible, + bool require_all) { struct mm_subpage_owner *owner; struct folio *wait = NULL; - unsigned long flags, eligible = offset_mask(pool->shift, offset); + unsigned long flags; spin_lock_irqsave(&pool->lock, flags); if (!pool->closed && pool->cow_shared && cow_has_free(pool, eligible)) { list_for_each_entry(owner, &pool->available, available) { if (!owner->migrating && (~owner->used & eligible) && + (!require_all || !(owner->used & eligible)) && cow_owner_eligible(pool, owner->folio) && !folio_test_swapcache(owner->folio) && !folio_test_writeback(owner->folio)) { @@ -870,8 +872,18 @@ bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offs return false; } +bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset) +{ + return cow_wait_busy(pool, offset_mask(pool->shift, offset), false); +} + +bool mm_subpage_cow_wait_busy_mask(struct mm_subpage_pool *pool, unsigned long mask) +{ + return cow_wait_busy(pool, mask, true); +} + static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, - unsigned long eligible) + unsigned long eligible, bool require_all) { struct mm_subpage_owner *owner; unsigned int visited = 0; @@ -893,6 +905,7 @@ static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, } /* A nonpresent PTE may still need any of this folio's slot bytes. */ if (!owner->migrating && (~owner->used & eligible) && + (!require_all || !(owner->used & eligible)) && (!pool->cow_shared || !folio_test_locked(owner->folio)) && cow_owner_eligible(pool, owner->folio) && !folio_test_swapcache(owner->folio) && @@ -903,7 +916,7 @@ static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, } static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, - unsigned long eligible, gfp_t gfp) + unsigned long eligible, bool require_all, gfp_t gfp) { struct mm_subpage_owner *owner; unsigned long flags; @@ -931,7 +944,7 @@ static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, spin_lock_irqsave(&pool->lock, flags); if (pool->closed) ret = -ESHUTDOWN; - else if (pool->unissued || find_available(pool, eligible)) + else if (pool->unissued || find_available(pool, eligible, require_all)) /* Another allocator supplied backing while we allocated metadata. * Preserve the single-unissued-owner invariant even when a shared * COW lookup skips a concurrently locked or rotating list prefix. @@ -959,7 +972,7 @@ static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, int mm_subpage_pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, gfp_t gfp) { - return pool_add_folio(pool, folio, pool->all_slots, gfp); + return pool_add_folio(pool, folio, pool->all_slots, false, gfp); } int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *folio, @@ -969,7 +982,15 @@ int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *fol if (!eligible) return -EINVAL; - return pool_add_folio(pool, folio, eligible, gfp); + return pool_add_folio(pool, folio, eligible, false, gfp); +} + +int mm_subpage_pool_add_folio_mask(struct mm_subpage_pool *pool, struct folio *folio, + unsigned long mask, gfp_t gfp) +{ + if (!mask || (mask & ~pool->all_slots)) + return -EINVAL; + return pool_add_folio(pool, folio, mask, true, gfp); } unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool) @@ -982,6 +1003,34 @@ unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool) return pages; } +/* Caller holds pool lock; each issued slot owns one native folio reference. */ +static struct mm_subpage *reserve_owner_slot(struct mm_subpage_owner *owner, + unsigned int index, bool *needs_zero) +{ + struct mm_subpage_pool *pool = owner->pool; + struct mm_subpage *subpage; + + lockdep_assert_held(&pool->lock); + if (pool->unissued == owner) + /* Transfer the supplied reference to the first reservation. */ + pool->unissued = NULL; + else + folio_get(owner->folio); + __set_bit(index, &owner->used); + cow_update_free(pool, BIT(index), -1); + if (needs_zero) + *needs_zero = !test_bit(index, &owner->zeroed); + __clear_bit(index, &owner->zeroed); + if (owner->used == pool->all_slots) + list_del_init(&owner->available); + subpage = &owner->slots[index]; + WRITE_ONCE(subpage->lazyfree, false); + /* The used bit keeps backing alive while data is initialised. */ + VM_BUG_ON(refcount_read(&subpage->refs)); + VM_BUG_ON(atomic_read(&subpage->mapcount)); + return subpage; +} + static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, unsigned long eligible, bool *needs_zero, bool lock_folio) { @@ -996,7 +1045,7 @@ static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, subpage = ERR_PTR(-ESHUTDOWN); goto out; } - owner = find_available(pool, eligible); + owner = find_available(pool, eligible, false); if (!owner) { subpage = ERR_PTR(-EAGAIN); goto out; @@ -1029,29 +1078,63 @@ static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, return ERR_PTR(-EAGAIN); } } - if (pool->unissued == owner) - /* Transfer the supplied reference to the first reservation. */ - pool->unissued = NULL; - else - folio_get(owner->folio); index = __ffs(~owner->used & eligible); - __set_bit(index, &owner->used); - cow_update_free(pool, BIT(index), -1); - if (needs_zero) - *needs_zero = !test_bit(index, &owner->zeroed); - __clear_bit(index, &owner->zeroed); - if (owner->used == pool->all_slots) - list_del_init(&owner->available); - subpage = &owner->slots[index]; - WRITE_ONCE(subpage->lazyfree, false); - /* The used bit keeps backing alive while data is initialised. */ - VM_BUG_ON(refcount_read(&subpage->refs)); - VM_BUG_ON(atomic_read(&subpage->mapcount)); + subpage = reserve_owner_slot(owner, index, needs_zero); out: spin_unlock_irqrestore(&pool->lock, flags); return subpage; } +/* Reserve one exact-offset batch from one owner, with one folio lock. */ +int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mask, + struct mm_subpage **slots) +{ + struct mm_subpage_owner *owner; + unsigned long flags, zero = 0; + unsigned int i; + bool needs_zero; + + if (!mask || (mask & ~pool->all_slots)) + return -EINVAL; +retry: + spin_lock_irqsave(&pool->lock, flags); + if (pool->closed) { + spin_unlock_irqrestore(&pool->lock, flags); + return -ESHUTDOWN; + } + owner = find_available(pool, mask, true); + if (!owner) { + spin_unlock_irqrestore(&pool->lock, flags); + return -EAGAIN; + } + if (!folio_trylock(owner->folio)) { + /* Parallel faults use another owner rather than queue behind it. */ + spin_unlock_irqrestore(&pool->lock, flags); + cond_resched(); + goto retry; + } + if (folio_test_swapcache(owner->folio) || folio_test_writeback(owner->folio) || + !cow_owner_eligible(pool, owner->folio)) { + folio_unlock(owner->folio); + spin_unlock_irqrestore(&pool->lock, flags); + return -EAGAIN; + } + for_each_set_bit(i, &mask, SUBPAGES_PER_FOLIO) { + slots[i] = reserve_owner_slot(owner, i, &needs_zero); + if (needs_zero) + zero |= BIT(i); + } + spin_unlock_irqrestore(&pool->lock, flags); + for_each_set_bit(i, &mask, SUBPAGES_PER_FOLIO) { + if (zero & BIT(i)) + clear_user_subpage_range(&owner->folio->page, + i << pool->shift, 1U << pool->shift); + /* Zeroing must precede publication, including reused slots. */ + refcount_set_release(&slots[i]->refs, 1); + } + return 0; +} + static struct mm_subpage *subpage_alloc(struct mm_subpage_pool *pool, unsigned long eligible, bool lock_folio) { bool needs_zero; diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index 4b7c4cf17006d..083ce9dc56bfd 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -7,12 +7,14 @@ CFLAGS += -Wall -Wextra -O2 -ffixed-x18 TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe compat-vfio-probe TEST_GEN_FILES += compat-shm-probe compat-perf-probe compat-numa-probe compat-numa-balancing-probe +TEST_GEN_PROGS_EXTENDED += anon-hole-refill anon-hole-refill-race anon-hole-refill-reclaim +TEST_FILES += README.hole-refill README.hole-refill-race README.hole-refill-reclaim TEST_FILES += compat-script runtime-app.py include ../../lib.mk $(OUTPUT)/page-contract: page-contract.c x18-probe.S -$(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool: LDLIBS += -pthread +$(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool $(OUTPUT)/anon-hole-refill-race $(OUTPUT)/anon-hole-refill-reclaim: LDLIBS += -pthread # Freestanding ELF32 probes exercise the compat ABI without an armhf sysroot. AARCH32_CC ?= clang --target=arm-linux-gnueabihf diff --git a/tools/testing/selftests/arm64/user4k/README.hole-refill b/tools/testing/selftests/arm64/user4k/README.hole-refill new file mode 100644 index 0000000000000..e547a6bfc05eb --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.hole-refill @@ -0,0 +1,29 @@ +Anonymous hole-refill regression +================================ + +Run anon-hole-refill through the platform's 4K userspace ABI launcher on a +kernel with a native 16K page size. The test does not change its page ABI. + + ./anon-hole-refill + sudo ./anon-hole-refill --native-16k + +The optional --native-16k argument enables physical density checks and requires +visible pagemap PFNs (CAP_SYS_ADMIN). Without it, four density results are +skipped. Do not use it with a different native page size. Pagemap must be +readable even without privileged PFNs: present bits verify neighbor prefault. + +The test owns 16 MiB of logical data. It discards individual 4K leaves and +writes before reading them, exercising missing-PTE refill rather than zero-page +COW. It checks zero initialization, all retained data and neighboring mappings. +Two phased rounds check slot reuse. A two-hole case checks that one fault +prefaults both missing neighbors before the second is accessed. + +Initial backing must be within one native page of 16 MiB. Final backing after +each refill sequence must be at most 20 MiB, allowing bounded pool slack. The +old fresh-private-folio path consumes 64 MiB after single-hole refill rounds +and 32 MiB after the paired-hole sequence. Temporary coexistence of original +and replacement backing is allowed; the test does not require 16 MiB at every +intermediate step. + +Expect eight TAP results. A 30-second alarm bounds the test. It does not fork, +induce pressure or test concurrent faults; see the companion lifetime tests. diff --git a/tools/testing/selftests/arm64/user4k/README.hole-refill-race b/tools/testing/selftests/arm64/user4k/README.hole-refill-race new file mode 100644 index 0000000000000..96f2994a4765b --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.hole-refill-race @@ -0,0 +1,33 @@ +Concurrent hole-refill lifetime test +=================================== + + sudo ./anon-hole-refill-race + +Run with a 4K userspace ABI on the native 16K kernel. CAP_SYS_ADMIN is required +to distinguish real swap entries from migration and poison markers. The target +configuration has MAX_SWAPFILES=27; types 27 through 31 are excluded and counted +separately. + +Four workers own disjoint 16K virtual windows in a 16 MiB region. Two phased +rounds discard and refill individual 4K leaves. Barriers synchronize quarter +phases, exercising concurrent shared-pool reservations without a userspace data +race. Every refill checks zeroing and unchanged neighbors. This covers pool +contention across windows, not simultaneous PTE changes within one window. + +After joining workers and verifying the region, the allocating process forks a +survivor, modifies one native window privately and exits. A supervisor waits +for that exit before releasing the survivor. The survivor verifies original +contents, rewrites all leaf tags, and verifies them again. This checks that +backing survives its original allocating mm and preserves fork isolation. + +The survivor then applies MADV_COLD/PAGEOUT only to its own region, at most 20 +times with 10 ms between attempts. At least one real swapped leaf and complete +byte-exact readback are required for the reclaim result. The reported count +states coverage; one swapped leaf is not broad reclaim qualification. These +advice operations can skip nonlinear pooled folios because their mappings are +not contiguous. Use README.hole-refill-reclaim for broader controlled coverage. + +Expect three TAP passes and no skips or unexpected signals. The supervisor has +a 35-second alarm and children have 30-second alarms. Failure or timeout kills +the dedicated process group; the supervisor reaps surviving descendants. An +external runner should also bound the entire fixture cgroup. diff --git a/tools/testing/selftests/arm64/user4k/README.hole-refill-reclaim b/tools/testing/selftests/arm64/user4k/README.hole-refill-reclaim new file mode 100644 index 0000000000000..6ede9503192b8 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.hole-refill-reclaim @@ -0,0 +1,36 @@ +Controlled reclaim of hole-refill backing +======================================== + + sudo ./anon-hole-refill-reclaim --external-reclaim + +This separate fixture repeats the concurrent allocation and owner-exit checks +in README.hole-refill-race. It requires the same page ABI, native granule and +visible swap metadata. After the survivor rewrites and verifies its region, it +prints: + + RECLAIM_READY pid= base= bytes=16777216 owner_mm_exited=1 + +It then waits up to 20 seconds for byte r on stdin. Place only the fixture and +its descendants in a fresh cgroup, with the controller outside that cgroup. +The controller may make at most three writes of "24M swappiness=200" to that +cgroup's memory.reclaim. Leave memory.max/high unchanged and do not reclaim +other cgroups or create global pressure. Bound controller work within the +20-second window. + +Record memory.current, memory.swap.current, memory.stat and memory.events, plus +present and valid-swap PTE counts for the reported region. EAGAIN or a partial +reclaim result does not determine success: actual coverage does. Send r after +the bounded attempts, even if coverage is below the required threshold. + +Before reading data back, the survivor snapshots pagemap. It requires at least +3072 of 4096 leaves (75%) to contain real swap entries and verifies every byte +after refault. Insufficient coverage fails this qualification rather than +skipping it. Expect three TAP passes, no skips and no unexpected signals or +kernel warnings. Supervisor and child alarms are 35 and 30 seconds; the +fixture's dedicated process group is killed on failure. + +MADV_COLD/PAGEOUT compares native folio mapcount with a contiguous PTE batch. +Nonlinear pooled quarters can belong to the same requested region while +occupying separate virtual locations, so those advice operations skip them. +This fixture exercises ordinary cgroup reclaim and nonlinear reverse mapping +without removing the advice path's protection for other mappings. diff --git a/tools/testing/selftests/arm64/user4k/anon-hole-refill-race.c b/tools/testing/selftests/arm64/user4k/anon-hole-refill-race.c new file mode 100644 index 0000000000000..965141e52e6f7 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/anon-hole-refill-race.c @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Bounded partial-window sharing, original-owner exit and real pageout. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#define LEAF 4096UL +#define NATIVE 16384UL +#define BYTES (16UL << 20) +#define LEAVES (BYTES / LEAF) +#define WORKERS 4 +#define ROUNDS 2 + +struct result { + bool concurrent, lifetime, reclaimed; + size_t swapped, non_swap_markers; + unsigned int attempts; +}; + +static unsigned char *data; +static pthread_barrier_t barrier; +static volatile sig_atomic_t fixture_group; + +static void stop_fixture(int signal_number) +{ + if (fixture_group > 0) + kill(-fixture_group, SIGKILL); + _exit(128 + signal_number); +} + +static unsigned char tag(size_t leaf, unsigned int generation) +{ + unsigned char value = leaf % 251 + 1; + + if (generation == 1 && leaf < 4) + value ^= 0xa5; + else if (generation == 2) + value ^= 0x5a; + return value; +} + +static bool verify_leaf(size_t leaf, unsigned int generation) +{ + size_t byte; + + if (data[leaf * LEAF] != tag(leaf, generation)) + return false; + for (byte = 1; byte < LEAF; byte++) + if (data[leaf * LEAF + byte] != 0x5a) + return false; + return true; +} + +static bool verify(unsigned int generation) +{ + size_t leaf; + + for (leaf = 0; leaf < LEAVES; leaf++) + if (!verify_leaf(leaf, generation)) + return false; + return true; +} + +static void *worker(void *arg) +{ + unsigned int id = (uintptr_t)arg, round, quarter, q; + size_t group, leaf, byte; + bool ok = true; + + for (round = 0; round < ROUNDS; round++) { + for (quarter = 0; quarter < 4; quarter++) { + pthread_barrier_wait(&barrier); + for (group = id; group < BYTES / NATIVE; group += WORKERS) { + unsigned char *p; + + leaf = group * 4 + quarter; + p = data + leaf * LEAF; + if (madvise(p, LEAF, MADV_DONTNEED)) { + ok = false; + continue; + } + /* A write fault, without a preceding zero-page read. */ + *(volatile unsigned char *)p = tag(leaf, 0); + for (byte = 1; byte < LEAF; byte++) + if (p[byte]) + ok = false; + memset(p + 1, 0x5a, LEAF - 1); + /* This worker owns the entire native virtual window. */ + for (q = 0; q < 4; q++) + ok &= verify_leaf(group * 4 + q, 0); + } + } + } + return (void *)(uintptr_t)!ok; +} + +static size_t swapped_leaves(size_t *markers) +{ + uint64_t entries[LEAVES]; + size_t i, count = 0; + int fd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + ssize_t n; + + if (fd < 0) + return SIZE_MAX; + n = pread(fd, entries, sizeof(entries), (uintptr_t)data / LEAF * 8); + close(fd); + if (n != sizeof(entries)) + return SIZE_MAX; + *markers = 0; + for (i = 0; i < LEAVES; i++) { + if ((entries[i] & (UINT64_C(1) << 63)) || + !(entries[i] & (UINT64_C(1) << 62))) + continue; + /* Audited target config:27..31 are poison/migration/markers. */ + if ((entries[i] & 31) < 27) + count++; + else + (*markers)++; + } + return count; +} + +static bool wait_ok(pid_t pid) +{ + int status; + pid_t got; + + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + return got > 0 && WIFEXITED(status) && !WEXITSTATUS(status); +} + +static void owner(int go, int result_fd) +{ + struct result result = {}; + pthread_t threads[WORKERS]; + void *mapping, *status; + unsigned int i; + bool advice_ok = true, readback_ok; + size_t leaf; + pid_t survivor; + char token; + + alarm(30); + mapping = mmap(NULL, BYTES + NATIVE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED) + _exit(2); + data = (void *)(((uintptr_t)mapping + NATIVE - 1) & ~(NATIVE - 1)); + memset(data, 0x5a, BYTES); + for (leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] = tag(leaf, 0); + if (pthread_barrier_init(&barrier, NULL, WORKERS)) + _exit(2); + for (i = 0; i < WORKERS; i++) + if (pthread_create(&threads[i], NULL, worker, (void *)(uintptr_t)i)) + _exit(2); + result.concurrent = true; + for (i = 0; i < WORKERS; i++) { + if (pthread_join(threads[i], &status)) + _exit(2); + result.concurrent &= !status; + } + pthread_barrier_destroy(&barrier); + result.concurrent &= verify(0); + if (!result.concurrent) + _exit(3); + survivor = fork(); + if (survivor < 0) + _exit(2); + if (survivor) { + /* The survivor must retain the original contents across this COW. */ + for (leaf = 0; leaf < 4; leaf++) + data[leaf * LEAF] = tag(leaf, 1); + _exit(verify(1) ? 0 : 3); + } + alarm(30); + /* Supervisor waits for the original allocator's mm to disappear first. */ + if (read(go, &token, 1) != 1 || token != 'g') + _exit(2); + close(go); + result.lifetime = verify(0); + for (leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] = tag(leaf, 2); + result.lifetime &= verify(2); + /* No memory.max reduction, memory.reclaim or unrelated memory pressure. */ + for (i = 0; i < 20; i++) { + result.attempts = i + 1; + if (madvise(data, BYTES, MADV_COLD) || madvise(data, BYTES, MADV_PAGEOUT)) { + advice_ok = false; + break; + } + usleep(10000); + result.swapped = swapped_leaves(&result.non_swap_markers); + if (result.swapped == SIZE_MAX) + break; + if (result.swapped == LEAVES) + break; + } + readback_ok = verify(2); + result.reclaimed = advice_ok && result.swapped && + result.swapped != SIZE_MAX && readback_ok; + if (write(result_fd, &result, sizeof(result)) != sizeof(result)) + _exit(2); + close(result_fd); + munmap(mapping, BYTES + NATIVE); + _exit(result.concurrent && result.lifetime && result.reclaimed ? 0 : 3); +} + +static bool visible_swap_metadata(void) +{ + FILE *file = fopen("/proc/self/status", "re"); + char line[256]; + unsigned long long caps = 0; + + if (!file) + return false; + while (fgets(line, sizeof(line), file)) + if (sscanf(line, "CapEff: %llx", &caps) == 1) + break; + fclose(file); + return caps & (1ULL << 21); /* CAP_SYS_ADMIN, needed for swap type bits. */ +} + +int main(void) +{ + struct result result = {}; + int go[2], results[2]; + pid_t child; + bool owner_ok, survivor_ok = false; + ssize_t n = 0; + + ksft_print_header(); + if (getauxval(AT_PAGESZ) != LEAF) + ksft_exit_skip("Run through the established 4K ABI launcher\n"); + if (!visible_swap_metadata()) + ksft_exit_skip("CAP_SYS_ADMIN required to distinguish real swap entries\n"); + signal(SIGPIPE, SIG_IGN); + signal(SIGALRM, stop_fixture); + signal(SIGTERM, stop_fixture); + alarm(35); + if (prctl(PR_SET_CHILD_SUBREAPER, 1) || pipe(go) || pipe(results)) + ksft_exit_fail_msg("setup: %s\n", strerror(errno)); + child = fork(); + if (child < 0) + ksft_exit_fail_msg("fork: %s\n", strerror(errno)); + if (!child) { + if (setpgid(0, 0)) + _exit(2); + close(go[1]); + close(results[0]); + owner(go[0], results[1]); + } + fixture_group = child; + if (setpgid(child, child) && errno != EACCES && errno != ESRCH) + stop_fixture(SIGTERM); + close(go[0]); + close(results[1]); + owner_ok = wait_ok(child); + if (owner_ok && write(go[1], "g", 1) == 1) { + n = read(results[0], &result, sizeof(result)); + survivor_ok = wait_ok(-1); + } + close(go[1]); + close(results[0]); + if (!owner_ok || !survivor_ok) { + kill(-child, SIGKILL); + while (waitpid(-1, NULL, 0) > 0 || errno == EINTR) + ; + } + fixture_group = 0; + alarm(0); + ksft_set_plan(3); + ksft_test_result(owner_ok && n == sizeof(result) && result.concurrent, + "four-worker partial-hole refill zeroing and neighbors\n"); + ksft_test_result(owner_ok && n == sizeof(result) && result.lifetime, + "original pool owner exits; descendant fork isolation/readwrite\n"); + ksft_print_msg("pageout attempts=%u real swapped=%zu/%lu excluded markers=%zu\n", + result.attempts, result.swapped, LEAVES, result.non_swap_markers); + ksft_test_result(survivor_ok && result.reclaimed, + "actual bounded pageout and full survivor readback\n"); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/anon-hole-refill-reclaim.c b/tools/testing/selftests/arm64/user4k/anon-hole-refill-reclaim.c new file mode 100644 index 0000000000000..4b15b076b0c1c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/anon-hole-refill-reclaim.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Opt-in own-cgroup reclaim of shared partial-hole backing after owner exit. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#define LEAF 4096UL +#define NATIVE 16384UL +#define BYTES (16UL << 20) +#define LEAVES (BYTES / LEAF) +#define WORKERS 4 +#define ROUNDS 2 + +struct result { + bool concurrent, lifetime, reclaimed; + size_t swapped, non_swap_markers; + unsigned int attempts; +}; + +static unsigned char *data; +static pthread_barrier_t barrier; +static volatile sig_atomic_t fixture_group; + +static void stop_fixture(int signal_number) +{ + if (fixture_group > 0) + kill(-fixture_group, SIGKILL); + _exit(128 + signal_number); +} + +static unsigned char tag(size_t leaf, unsigned int generation) +{ + unsigned char value = leaf % 251 + 1; + + if (generation == 1 && leaf < 4) + value ^= 0xa5; + else if (generation == 2) + value ^= 0x5a; + return value; +} + +static bool verify_leaf(size_t leaf, unsigned int generation) +{ + size_t byte; + + if (data[leaf * LEAF] != tag(leaf, generation)) + return false; + for (byte = 1; byte < LEAF; byte++) + if (data[leaf * LEAF + byte] != 0x5a) + return false; + return true; +} + +static bool verify(unsigned int generation) +{ + size_t leaf; + + for (leaf = 0; leaf < LEAVES; leaf++) + if (!verify_leaf(leaf, generation)) + return false; + return true; +} + +static void *worker(void *arg) +{ + unsigned int id = (uintptr_t)arg, round, quarter, q; + size_t group, leaf, byte; + bool ok = true; + + for (round = 0; round < ROUNDS; round++) { + for (quarter = 0; quarter < 4; quarter++) { + pthread_barrier_wait(&barrier); + for (group = id; group < BYTES / NATIVE; group += WORKERS) { + unsigned char *p; + + leaf = group * 4 + quarter; + p = data + leaf * LEAF; + if (madvise(p, LEAF, MADV_DONTNEED)) { + ok = false; + continue; + } + /* A write fault, without a preceding zero-page read. */ + *(volatile unsigned char *)p = tag(leaf, 0); + for (byte = 1; byte < LEAF; byte++) + if (p[byte]) + ok = false; + memset(p + 1, 0x5a, LEAF - 1); + /* This worker owns the entire native virtual window. */ + for (q = 0; q < 4; q++) + ok &= verify_leaf(group * 4 + q, 0); + } + } + } + return (void *)(uintptr_t)!ok; +} + +static size_t swapped_leaves(size_t *markers) +{ + uint64_t entries[LEAVES]; + size_t i, count = 0; + int fd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + ssize_t n; + + if (fd < 0) + return SIZE_MAX; + n = pread(fd, entries, sizeof(entries), (uintptr_t)data / LEAF * 8); + close(fd); + if (n != sizeof(entries)) + return SIZE_MAX; + *markers = 0; + for (i = 0; i < LEAVES; i++) { + if ((entries[i] & (UINT64_C(1) << 63)) || + !(entries[i] & (UINT64_C(1) << 62))) + continue; + /* Audited target config:27..31 are poison/migration/markers. */ + if ((entries[i] & 31) < 27) + count++; + else + (*markers)++; + } + return count; +} + +static bool wait_ok(pid_t pid) +{ + int status; + pid_t got; + + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + return got > 0 && WIFEXITED(status) && !WEXITSTATUS(status); +} + +static void owner(int go, int result_fd) +{ + struct result result = {}; + pthread_t threads[WORKERS]; + void *mapping, *status; + unsigned int i; + bool readback_ok; + struct pollfd control = { .fd = STDIN_FILENO, .events = POLLIN }; + size_t leaf; + pid_t survivor; + char token; + + alarm(30); + mapping = mmap(NULL, BYTES + NATIVE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED) + _exit(2); + data = (void *)(((uintptr_t)mapping + NATIVE - 1) & ~(NATIVE - 1)); + memset(data, 0x5a, BYTES); + for (leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] = tag(leaf, 0); + if (pthread_barrier_init(&barrier, NULL, WORKERS)) + _exit(2); + for (i = 0; i < WORKERS; i++) + if (pthread_create(&threads[i], NULL, worker, (void *)(uintptr_t)i)) + _exit(2); + result.concurrent = true; + for (i = 0; i < WORKERS; i++) { + if (pthread_join(threads[i], &status)) + _exit(2); + result.concurrent &= !status; + } + pthread_barrier_destroy(&barrier); + result.concurrent &= verify(0); + if (!result.concurrent) + _exit(3); + survivor = fork(); + if (survivor < 0) + _exit(2); + if (survivor) { + /* The survivor must retain the original contents across this COW. */ + for (leaf = 0; leaf < 4; leaf++) + data[leaf * LEAF] = tag(leaf, 1); + _exit(verify(1) ? 0 : 3); + } + alarm(30); + /* Supervisor waits for the original allocator's mm to disappear first. */ + if (read(go, &token, 1) != 1 || token != 'g') + _exit(2); + close(go); + result.lifetime = verify(0); + for (leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] = tag(leaf, 2); + result.lifetime &= verify(2); + /* Controller acts only on this fixture's cgroup. No pressure here. */ + printf("RECLAIM_READY pid=%d base=%p bytes=%lu owner_mm_exited=1\n", + getpid(), data, BYTES); + fflush(stdout); + if (poll(&control, 1, 20000) != 1 || + read(STDIN_FILENO, &token, 1) != 1 || token != 'r') + _exit(2); + result.attempts = 1; /* One post-controller pagemap snapshot. */ + result.swapped = swapped_leaves(&result.non_swap_markers); + readback_ok = verify(2); + result.reclaimed = result.swapped != SIZE_MAX && + result.swapped >= LEAVES * 3 / 4 && readback_ok; + + if (write(result_fd, &result, sizeof(result)) != sizeof(result)) + _exit(2); + close(result_fd); + munmap(mapping, BYTES + NATIVE); + _exit(result.concurrent && result.lifetime && result.reclaimed ? 0 : 3); +} + +static bool visible_swap_metadata(void) +{ + FILE *file = fopen("/proc/self/status", "re"); + char line[256]; + unsigned long long caps = 0; + + if (!file) + return false; + while (fgets(line, sizeof(line), file)) + if (sscanf(line, "CapEff: %llx", &caps) == 1) + break; + fclose(file); + return caps & (1ULL << 21); /* CAP_SYS_ADMIN, needed for swap type bits. */ +} + +int main(int argc, char **argv) +{ + struct result result = {}; + int go[2], results[2]; + pid_t child; + bool owner_ok, survivor_ok = false; + ssize_t n = 0; + + ksft_print_header(); + fflush(stdout); + if (argc != 2 || strcmp(argv[1], "--external-reclaim")) + ksft_exit_skip("Needs explicit --external-reclaim and a bounded own-cgroup controller\n"); + if (getauxval(AT_PAGESZ) != LEAF) + ksft_exit_skip("Run through the established 4K ABI launcher\n"); + if (!visible_swap_metadata()) + ksft_exit_skip("CAP_SYS_ADMIN required to distinguish real swap entries\n"); + signal(SIGPIPE, SIG_IGN); + signal(SIGALRM, stop_fixture); + signal(SIGTERM, stop_fixture); + alarm(35); + if (prctl(PR_SET_CHILD_SUBREAPER, 1) || pipe(go) || pipe(results)) + ksft_exit_fail_msg("setup: %s\n", strerror(errno)); + child = fork(); + if (child < 0) + ksft_exit_fail_msg("fork: %s\n", strerror(errno)); + if (!child) { + if (setpgid(0, 0)) + _exit(2); + close(go[1]); + close(results[0]); + owner(go[0], results[1]); + } + fixture_group = child; + if (setpgid(child, child) && errno != EACCES && errno != ESRCH) + stop_fixture(SIGTERM); + close(go[0]); + close(results[1]); + owner_ok = wait_ok(child); + if (owner_ok && write(go[1], "g", 1) == 1) { + n = read(results[0], &result, sizeof(result)); + survivor_ok = wait_ok(-1); + } + close(go[1]); + close(results[0]); + if (!owner_ok || !survivor_ok) { + kill(-child, SIGKILL); + while (waitpid(-1, NULL, 0) > 0 || errno == EINTR) + ; + } + fixture_group = 0; + alarm(0); + ksft_set_plan(3); + ksft_test_result(owner_ok && n == sizeof(result) && result.concurrent, + "four-worker partial-hole refill zeroing and neighbors\n"); + ksft_test_result(owner_ok && n == sizeof(result) && result.lifetime, + "original pool owner exits; descendant fork isolation/readwrite\n"); + ksft_print_msg("post-controller snapshots=%u real swapped=%zu/%lu excluded markers=%zu\n", + result.attempts, result.swapped, LEAVES, result.non_swap_markers); + ksft_test_result(survivor_ok && result.reclaimed, + "own-cgroup reclaim swaps at least75percent, full survivor readback\n"); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/anon-hole-refill.c b/tools/testing/selftests/arm64/user4k/anon-hole-refill.c new file mode 100644 index 0000000000000..e2d71e5cce602 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/anon-hole-refill.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Missing-PTE refill must retain contents, faultaround and native density. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#define LEAF 4096UL +#define NATIVE 16384UL +#define BYTES (16UL << 20) +#define LEAVES (BYTES / LEAF) + +static unsigned char *data; +static int pm = -1; +static bool density; + +static unsigned char tag(size_t leaf) +{ + return leaf % 251 + 1; +} + +static bool refill(size_t leaf) +{ + unsigned char *p = data + leaf * LEAF; + size_t i; + + /* Write first: a read would test zero-page COW instead of hole refill. */ + *(volatile unsigned char *)p = tag(leaf); + for (i = 1; i < LEAF; i++) + if (p[i]) + return false; + memset(p + 1, 0x5a, LEAF - 1); + return true; +} + +static bool contents(void) +{ + size_t leaf, i; + + for (leaf = 0; leaf < LEAVES; leaf++) { + if (data[leaf * LEAF] != tag(leaf)) + return false; + for (i = 1; i < LEAF; i++) + if (data[leaf * LEAF + i] != 0x5a) + return false; + } + return true; +} + +static int cmp(const void *a, const void *b) +{ + uint64_t x = *(const uint64_t *)a, y = *(const uint64_t *)b; + + return (x > y) - (x < y); +} + +static int native_bytes(size_t *bytes) +{ + uint64_t entries[LEAVES]; + size_t i, unique = 0; + off_t offset = (uintptr_t)data / LEAF * sizeof(uint64_t); + + if (pm < 0 || pread(pm, entries, sizeof(entries), offset) != sizeof(entries)) + return -1; + for (i = 0; i < LEAVES; i++) { + uint64_t pfn = entries[i] & ((UINT64_C(1) << 55) - 1); + + if (!(entries[i] & (UINT64_C(1) << 63)) || !pfn) + return -1; + entries[i] = pfn / (NATIVE / LEAF); + } + qsort(entries, LEAVES, sizeof(*entries), cmp); + for (i = 0; i < LEAVES; i++) + unique += !i || entries[i] != entries[i - 1]; + *bytes = unique * NATIVE; + return 0; +} + +static void check_density(const char *stage, size_t limit) +{ + size_t bytes; + + if (!density) { + ksft_test_result_skip("%s physical density (needs --native-16k and visible PFNs)\n", stage); + return; + } + if (native_bytes(&bytes)) { + ksft_test_result_fail("%s pagemap unavailable, swapped or hidden\n", stage); + return; + } + ksft_print_msg("%s logical=%lu native=%zu limit=%zu\n", stage, BYTES, bytes, limit); + ksft_test_result(bytes <= limit, "%s physical density\n", stage); +} + +static bool phased_refill(void) +{ + unsigned int quarter; + size_t leaf; + + for (quarter = 0; quarter < 4; quarter++) { + for (leaf = quarter; leaf < LEAVES; leaf += 4) + if (madvise(data + leaf * LEAF, LEAF, MADV_DONTNEED) || !refill(leaf)) + return false; + if (!contents()) + return false; + } + return true; +} + +static bool paired_refill(void) +{ + unsigned int parity; + size_t leaf; + + for (parity = 0; parity < 2; parity++) { + for (leaf = parity; leaf < LEAVES; leaf += 4) { + uint64_t entry; + + if (madvise(data + leaf * LEAF, LEAF, MADV_DONTNEED) || + madvise(data + (leaf + 2) * LEAF, LEAF, MADV_DONTNEED) || + !refill(leaf)) + return false; + /* Present is visible without privileged PFNs. Prove the other + * missing neighbor was prefaulted before touching it ourselves. + */ + if (pm < 0 || pread(pm, &entry, sizeof(entry), + ((uintptr_t)data / LEAF + leaf + 2) * 8) != sizeof(entry) || + !(entry & (UINT64_C(1) << 63)) || !refill(leaf + 2)) + return false; + } + } + return contents(); +} + +int main(int argc, char **argv) +{ + void *mapping; + size_t leaf; + + ksft_print_header(); + if (getauxval(AT_PAGESZ) != LEAF) + ksft_exit_skip("Run through the established 4K ABI launcher\n"); + if (argc == 2 && !strcmp(argv[1], "--native-16k")) + density = true; + else if (argc != 1) + ksft_exit_fail_msg("Usage: %s [--native-16k]\n", argv[0]); + alarm(30); + mapping = mmap(NULL, BYTES + NATIVE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED) + ksft_exit_fail_msg("mmap: %s\n", strerror(errno)); + data = (void *)(((uintptr_t)mapping + NATIVE - 1) & ~(NATIVE - 1)); + pm = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + memset(data, 0x5a, BYTES); + for (leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] = tag(leaf); + ksft_set_plan(8); + ksft_test_result(contents(), "initial dense content\n"); + check_density("initial dense", BYTES + NATIVE); + ksft_test_result(phased_refill(), "phased hole refill zeroing and neighbors\n"); + /* Allow bounded pool-bucket slop, but reject the old 64MiB outcome. */ + check_density("phased refill", BYTES + (4UL << 20)); + ksft_test_result(phased_refill(), "reused-slot refill zeroing and neighbors\n"); + check_density("reused-slot refill", BYTES + (4UL << 20)); + ksft_test_result(paired_refill(), "two-hole mask retains missing-neighbor prefault\n"); + check_density("two-hole refill", BYTES + (4UL << 20)); + munmap(mapping, BYTES + NATIVE); + if (pm >= 0) + close(pm); + ksft_finished(); +} From 4d514fdba278e26d0b07ed2dd793d1df8971b12d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 18:32:47 -0400 Subject: [PATCH 0862/2680] mm: pack single-slot COW across native page quarters A workload writing the same logical quarter in each native window used 16 MiB of backing for 4 MiB of changed data because COW only reused a matching physical quarter. Allow eligible pooled single-slot copies to use any free quarter, retaining exact-offset private fallbacks and dense anonymous batching. Prepare nonlinear reverse mappings before taking PTE locks, including anonymous hole and swap-in publication. Restore swapoff from a locked swap-PTE snapshot and recheck it before installation. Use actual source and destination slot offsets for file and anonymous copies. On Neo, the same-quarter case now uses 4.09375 MiB with byte-exact parent isolation. Hardware checks cover same-native-folio COW, typed GUP and shifted mremap, complete owned-cgroup reclaim/readback, selected-page swapoff, native/4K ABI, hole refill, GPU quarter refault and normal game correctness. Maintained regression binaries match those tested exactly. Ongoing game memory pressure and other first-write allocation paths are separate unresolved issues; this change makes no FPS claim. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 6748c940b0e8fd5287a85ad9bda11e254c8901cf) --- include/linux/mm_subpage.h | 7 + mm/memory.c | 58 +-- mm/swapfile.c | 34 +- mm/user-subpage.c | 32 ++ .../testing/selftests/arm64/user4k/.gitignore | 3 + tools/testing/selftests/arm64/user4k/Makefile | 2 + .../selftests/arm64/user4k/README.any-quarter | 37 ++ .../arm64/user4k/check-any-quarter.py | 24 ++ .../arm64/user4k/file-cow-consumers.c | 319 +++++++++++++++++ .../selftests/arm64/user4k/same-native.c | 176 +++++++++ .../selftests/arm64/user4k/same-quarter.c | 335 ++++++++++++++++++ 11 files changed, 991 insertions(+), 36 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/README.any-quarter create mode 100644 tools/testing/selftests/arm64/user4k/check-any-quarter.py create mode 100644 tools/testing/selftests/arm64/user4k/file-cow-consumers.c create mode 100644 tools/testing/selftests/arm64/user4k/same-native.c create mode 100644 tools/testing/selftests/arm64/user4k/same-quarter.c diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h index 9a1d33166db30..e892ae3e70bee 100644 --- a/include/linux/mm_subpage.h +++ b/include/linux/mm_subpage.h @@ -52,6 +52,13 @@ struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, void mm_subpage_cow_pool_exit(struct mm_struct *mm); bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *folio); bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset); +/* COW slots may use any physical quarter; rmap retains the virtual position. */ +bool mm_subpage_cow_wait_busy(struct mm_subpage_pool *pool); +struct mm_subpage *mm_subpage_alloc_locked(struct mm_subpage_pool *pool); +struct mm_subpage *mm_subpage_copy_locked(struct mm_subpage_pool *pool, + const struct mm_subpage *source); +int mm_subpage_prepare_anon_rmap(struct mm_subpage *slot, + struct vm_area_struct *vma, unsigned long address); /* Exact physical-slot mask, one owner and folio lock for the entire batch. * slots is indexed by physical offset >> pool shift; only mask bits are set. * Success returns initialized owned slots; errors leave slots untouched. diff --git a/mm/memory.c b/mm/memory.c index 1c82ee38e561c..399c8ea4e1628 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1365,14 +1365,18 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, if (!pool) return ERR_PTR(-ENOMEM); for (;;) { - slot = source ? mm_subpage_copy_at_locked(pool, source, offset) : - mm_subpage_alloc_at_locked(pool, offset); + if (*pooled) + slot = source ? mm_subpage_copy_locked(pool, source) : + mm_subpage_alloc_locked(pool); + else + slot = source ? mm_subpage_copy_at_locked(pool, source, offset) : + mm_subpage_alloc_at_locked(pool, offset); if (!IS_ERR(slot) || PTR_ERR(slot) != -EAGAIN) break; folio = NULL; if (*pooled) { folio = cow_subpage_speculative_folio(mm, vma, addr); - if (!folio && mm_subpage_cow_wait_busy_at(pool, offset)) { + if (!folio && mm_subpage_cow_wait_busy(pool)) { cond_resched(); continue; } @@ -1393,15 +1397,16 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, force_private = true; goto retry_private; } - err = mm_subpage_pool_add_folio_at(pool, folio, offset, - *pooled ? GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN : GFP_KERNEL); + err = *pooled ? mm_subpage_pool_add_folio(pool, folio, + GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN) : + mm_subpage_pool_add_folio_at(pool, folio, offset, GFP_KERNEL); if (*pooled && err == -ENOMEM) { - if (mm_subpage_cow_wait_busy_at(pool, offset)) { + if (mm_subpage_cow_wait_busy(pool)) { folio_put(folio); cond_resched(); continue; } - err = mm_subpage_pool_add_folio_at(pool, folio, offset, GFP_KERNEL); + err = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); } if (err) { folio_put(folio); @@ -4640,7 +4645,6 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) struct mmu_notifier_range range; bool copied = false, unshare = vmf->flags & FAULT_FLAG_UNSHARE; bool force_private = false, pooled, fresh, retry_private = false; - unsigned int offset; vm_fault_t ret = VM_FAULT_SIGBUS; pte_t entry; int err; @@ -4690,7 +4694,6 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) ret = vmf_anon_prepare(vmf); if (ret) goto out; - offset = vma_page_offset_at(vma, vmf->address).offset; ret = VM_FAULT_OOM; retry_copy: dest = cow_subpage_prealloc(mm, vma, vmf->address, source, force_private, &pooled); @@ -4702,7 +4705,7 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) } new = mm_subpage_folio(dest); if (old && !source) { - err = copy_user_subpage_range(&new->page, offset, vmf->page, + err = copy_user_subpage_range(&new->page, mm_subpage_offset(dest), vmf->page, pte_phys_mm(mm, vmf->orig_pte) & ~PAGE_MASK, mm_page_size(mm)); if (err) { @@ -4727,13 +4730,11 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) * slot identity before taking PTL, with a fresh-folio fallback on ENOMEM. */ fresh = !mm_subpage_anon_root(dest); - if (!fresh) { - err = mm_subpage_bind_rmap(dest, vma, vmf->address); - if (err) { - ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; - retry_private = pooled && err == -ENOMEM; - goto unlock_folio; - } + err = mm_subpage_prepare_anon_rmap(dest, vma, vmf->address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + retry_private = pooled && err == -ENOMEM; + goto unlock_folio; } vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); if (!vmf->pte) @@ -5505,6 +5506,11 @@ static vm_fault_t do_swap_subpage(struct vm_fault *vmf, swp_entry_t entry) ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; goto unlock_folio; } + err = mm_subpage_prepare_anon_rmap(slot, vma, vmf->address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + goto unlock_folio; + } folio_throttle_swaprate(folio, GFP_KERNEL); vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); if (!vmf->pte) @@ -6464,17 +6470,15 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) /* Existing owners may need nonlinear identities. Allocate metadata before * PTL and keep the established fresh-folio fallback on metadata ENOMEM. */ - if (!fresh) { - for_each_set_bit(index, &mask, ARRAY_SIZE(slots)) { - unsigned long address = start + - ((index - first_offset) << mm_page_shift(mm)); + for_each_set_bit(index, &mask, ARRAY_SIZE(slots)) { + unsigned long address = start + + ((index - first_offset) << mm_page_shift(mm)); - err = mm_subpage_bind_rmap(slots[index], vma, address); - if (err) { - ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; - retry_private = pooled && err == -ENOMEM; - goto unlock_folio; - } + err = mm_subpage_prepare_anon_rmap(slots[index], vma, address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + retry_private = pooled && err == -ENOMEM; + goto unlock_folio; } } vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); diff --git a/mm/swapfile.c b/mm/swapfile.c index 5009971f3c90c..7ab5fe05a0e2f 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -2620,33 +2620,50 @@ static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte) #ifdef CONFIG_MM_SUBPAGE /* Caller holds the swap-cache folio lock and the VMA's mmap read lock. */ static int unuse_subpage(struct vm_area_struct *vma, pmd_t *pmd, - unsigned long addr, swp_entry_t entry, struct folio *folio, - unsigned int offset) + unsigned long addr, swp_entry_t entry, struct folio *folio) { struct mm_struct *mm = vma->vm_mm; + unsigned int offset; struct mm_subpage *slot = NULL; spinlock_t *ptl; - pte_t *ptep, old, pte; + pte_t *ptep, old, snapshot, pte; bool poisoned = PageHWPoison(&folio->page) || !folio_test_uptodate(folio); int ret = 0; if (folio_order(folio) || folio_test_ksm(folio)) return -EOPNOTSUPP; + /* A swap PTE records the physical quarter, independently of the VMA. + * Snapshot under PTL, then allocate metadata while only folio is locked. + */ + ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!ptep) + return 0; + snapshot = ptep_get(ptep); + if (!softleaf_is_swap(softleaf_from_pte(snapshot)) || + softleaf_from_pte(snapshot).val != entry.val) { + pte_unmap_unlock(ptep, ptl); + return 0; + } + offset = pte_swp_subpage_offset(snapshot); + pte_unmap_unlock(ptep, ptl); + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, mm_page_size(mm))) + return -EINVAL; if (!poisoned) { arch_swap_restore(folio_swap(entry, folio), folio); /* Allocate metadata before taking the PTE spinlock. */ slot = mm_subpage_restore_granule(folio, offset, mm_page_shift(mm), GFP_KERNEL); if (IS_ERR(slot)) return PTR_ERR(slot) == -EAGAIN ? 0 : PTR_ERR(slot); + ret = mm_subpage_prepare_anon_rmap(slot, vma, addr); + if (ret) + goto out; } ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); if (!ptep) goto out; old = ptep_get(ptep); - if (softleaf_from_pte(old).val != entry.val) - goto unlock; - /* A moved swap PTE retains its backing offset, not its VMA offset. */ - if (pte_swp_subpage_offset(old) != offset) + /* Includes quarter and flag changes while metadata was prepared. */ + if (!pte_same(old, snapshot)) goto unlock; if (poisoned) { pte = swp_entry_to_pte(PageHWPoison(&folio->page) ? @@ -2707,8 +2724,7 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, #ifdef CONFIG_MM_SUBPAGE if (mm_page_size(vma->vm_mm) < PAGE_SIZE) - return unuse_subpage(vma, pmd, addr, entry, folio, - pte_swp_subpage_offset(orig_pte)); + return unuse_subpage(vma, pmd, addr, entry, folio); #endif swapcache = folio; diff --git a/mm/user-subpage.c b/mm/user-subpage.c index c7876fd34a091..30fb3669458cb 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -630,6 +630,22 @@ int mm_subpage_bind_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, return ret; } +/* Preflight publication without making an unexposed owner look published. + * A fresh linear slot needs no sidecar: add_new_anon_rmap will set its anchor. + * Otherwise bind the independent virtual position before any PTE lock. + */ +int mm_subpage_prepare_anon_rmap(struct mm_subpage *slot, + struct vm_area_struct *vma, unsigned long address) +{ + struct mm_subpage_owner *owner = slot->owner; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + if (!owner->anon_root && !owner->rmap && + mm_subpage_offset(slot) == vma_page_offset_at(vma, address).offset) + return 0; + return mm_subpage_bind_rmap(slot, vma, address); +} + static bool subpage_move_allowed(struct mm_subpage *slot, bool swap) { lockdep_assert_held(&slot->owner->pool->lock); @@ -872,6 +888,11 @@ static bool cow_wait_busy(struct mm_subpage_pool *pool, unsigned long eligible, return false; } +bool mm_subpage_cow_wait_busy(struct mm_subpage_pool *pool) +{ + return cow_wait_busy(pool, pool->all_slots, false); +} + bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset) { return cow_wait_busy(pool, offset_mask(pool->shift, offset), false); @@ -1261,6 +1282,17 @@ struct mm_subpage *mm_subpage_copy_at_locked(struct mm_subpage_pool *pool, return eligible ? subpage_copy(pool, source, eligible, true) : ERR_PTR(-EINVAL); } +struct mm_subpage *mm_subpage_alloc_locked(struct mm_subpage_pool *pool) +{ + return subpage_alloc(pool, pool->all_slots, true); +} + +struct mm_subpage *mm_subpage_copy_locked(struct mm_subpage_pool *pool, + const struct mm_subpage *source) +{ + return subpage_copy(pool, source, pool->all_slots, true); +} + /* The page-extension RCU section also protects the pool until its lock is held. */ static struct mm_subpage *restore_bound_slot(struct folio *folio, unsigned int offset, unsigned int shift) diff --git a/tools/testing/selftests/arm64/user4k/.gitignore b/tools/testing/selftests/arm64/user4k/.gitignore index 2e6224b832f98..86bb27433258e 100644 --- a/tools/testing/selftests/arm64/user4k/.gitignore +++ b/tools/testing/selftests/arm64/user4k/.gitignore @@ -1,3 +1,6 @@ page-contract swap-fixture-init +same-quarter +file-cow-consumers +same-native diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index 083ce9dc56bfd..0c22734f5ad03 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -10,6 +10,8 @@ TEST_GEN_FILES += compat-shm-probe compat-perf-probe compat-numa-probe compat-nu TEST_GEN_PROGS_EXTENDED += anon-hole-refill anon-hole-refill-race anon-hole-refill-reclaim TEST_FILES += README.hole-refill README.hole-refill-race README.hole-refill-reclaim TEST_FILES += compat-script runtime-app.py +TEST_GEN_PROGS_EXTENDED += same-quarter file-cow-consumers same-native +TEST_FILES += README.any-quarter check-any-quarter.py include ../../lib.mk $(OUTPUT)/page-contract: page-contract.c x18-probe.S diff --git a/tools/testing/selftests/arm64/user4k/README.any-quarter b/tools/testing/selftests/arm64/user4k/README.any-quarter new file mode 100644 index 0000000000000..305441c8a9db4 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.any-quarter @@ -0,0 +1,37 @@ +Arbitrary-quarter anonymous COW qualification +============================================= + +Run on the audited native16K kernel through an established4K ABI launcher, +with CAP_SYS_ADMIN for visible pagemap PFNs: + userpages 4k ./same-quarter --native-16k > same-quarter.log + python3 check-any-quarter.py same-quarter.log + userpages 4k ./file-cow-consumers --native-16k + userpages 4k ./same-native --native-16k + +same-quarter: two fresh owner+child arms compare4MiB changed data in same versus +balanced virtual quarters of a16MiB dense parent. Exact nativePFN union, no +modified-child alias to parent/unchangedchild, all-byte isolation and histograms. +The binary reports measurements; the checker requires both arms <=6MiB native +for4MiB changed and actual nonzero physical-quarter use in the same-quarter arm. +This catches the old16MiB same-quarter result while allowing bounded pool slack. +30s supervisor/20s child alarms, group/subreaper cleanup, no pressure. + +file-cow-consumers:1MiB owned memfd,two private views; child explicitly READ- +prefaults both views AFTER fork (fork may omit reconstructible file PTEs). +CPU COW, remote typed GUP read-unshare/write-COW each require actual physical/ +logical quarter mismatch and correct file-to-anonymous transition. Then shifted +VA mremap must preserve every actualPFN and byte; post-move GUP and parent/file +isolation are checked.30s supervisor/20s child. Exact stage/byte/PTE diagnostics. +This is typed GUP coverage, not long-term DMA pin or migration qualification. + +same-native:16fresh three-generation trials/16KiB logical payload. B's first COW +is retained by held C while B COWs again. Required distinct4KPFNs sharing native +PFN/4 prove source/destination use one nativefolio; every generation and neighbor +byte stays correct. At least1actual pair is required for PASS; exit77 means +UNEXERCISED, never a coverage pass.20/5/4s supervisor/owner/held-child alarms. + +No fixture changes memory policy, system swap, kernel or GPU state. Native16K +assumption is explicit; do not run density math on an unaudited kernel. Metadata +--selftest modes (where provided) are not kernel-path qualification. UFFD_MOVE +and targeted migration are separate gates; disabled UFFD/single-node systems do +not acquire those qualifications merely by passing these tests. diff --git a/tools/testing/selftests/arm64/user4k/check-any-quarter.py b/tools/testing/selftests/arm64/user4k/check-any-quarter.py new file mode 100644 index 0000000000000..1b71b3922783f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/check-any-quarter.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +"""Validate native backing density from the unchanged same-quarter oracle.""" +import json,sys +from pathlib import Path + +def check(path): + rows=[json.loads(x) for x in Path(path).read_text().splitlines() if x.startswith('{')] + assert len(rows)==2, 'Require exactly two measured arms' + arms={x['arm']:x for x in rows} + assert set(arms)=={'same-quarter-zero','balanced'} + for name,r in arms.items(): + assert r['logical_parent_bytes']==16*1024*1024 and r['modified_child_bytes']==4*1024*1024 + assert r['content_ok'] and r['disjoint_ok'] + assert r['modified_parent_native_overlap']==0 and r['modified_unchanged_native_overlap']==0 + assert r['modified_child']['represented_bytes']==4*1024*1024 + n=r['modified_child']['native_bytes'] + assert 4*1024*1024<=n<=6*1024*1024, (name,'native backing outside packed bound',n) + assert sum(arms['same-quarter-zero']['modified_physical_quarter_counts'][1:])>0 + print('PASS same-quarter/balanced native density, actual relocation and isolation') + +if __name__=='__main__': + if len(sys.argv)!=2:sys.exit('usage: check-any-quarter.py same-quarter.log') + try:check(sys.argv[1]) + except (AssertionError,KeyError,ValueError,OSError) as e:sys.exit('FAIL '+str(e)) diff --git a/tools/testing/selftests/arm64/user4k/file-cow-consumers.c b/tools/testing/selftests/arm64/user4k/file-cow-consumers.c new file mode 100644 index 0000000000000..7d2f82500f637 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/file-cow-consumers.c @@ -0,0 +1,319 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Owned memfd only: file COW, typed remote GUP and shifted mremap. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define LEAF 4096UL +#define NATIVE 16384UL +#define WINDOWS 64UL +#define BYTES (WINDOWS * NATIVE) +#define LEAVES (BYTES / LEAF) +#define PRESENT (UINT64_C(1) << 63) +#define FILE_PAGE (UINT64_C(1) << 61) +#define PFN_MASK ((UINT64_C(1) << 55) - 1) + +struct message { uintptr_t address; uint64_t phase, detail; }; +static volatile sig_atomic_t child_group; +static const char *stage = "startup"; +static bool child_reaped; +static int child_status; +static void report_wait(pid_t pid, int status) +{ + fprintf(stderr, "CHILD_WAIT pid=%ld raw_status=%d exited=%d exit=%d signaled=%d signal=%d\n", + (long)pid, status, WIFEXITED(status), WIFEXITED(status) ? WEXITSTATUS(status) : -1, + WIFSIGNALED(status), WIFSIGNALED(status) ? WTERMSIG(status) : 0); +} + +static void stop(int sig) +{ + if (child_group > 0) kill(-child_group, SIGKILL); + _exit(128 + sig); +} +static bool io_full(int fd, void *buf, size_t n, bool write_it) +{ + char *p = buf; + while (n) { + ssize_t got = write_it ? write(fd, p, n) : read(fd, p, n); + if (got < 0 && errno == EINTR) continue; + if (got <= 0) return false; + p += got; n -= (size_t)got; + } + return true; +} +static unsigned char original(size_t offset) +{ + size_t leaf = offset / LEAF, byte = offset % LEAF; + return (unsigned char)(leaf * 17 + byte * 13 + byte / 256); +} +static bool verify(const unsigned char *data, unsigned kind, size_t gup_a_leaf) +{ + for (size_t at = 0; at < BYTES; at++) { + size_t leaf = at / LEAF, byte = at % LEAF; + unsigned char value = original(at); + if (kind == 1 && leaf % 4 == 3 && byte == 0) value ^= 0x5a; + if (kind == 1 && leaf == gup_a_leaf && byte >= 17 && byte < 49) value = 0xe3; + if (kind == 2 && leaf % 4 == 2 && byte >= 37 && byte < 69) value = 0xc7; + if (data[at] != value) { + fprintf(stderr, "VERIFY_FAIL stage=%s kind=%u offset=0x%zx leaf=%zu byte=%zu expected=%02x actual=%02x\n", + stage, kind, at, leaf, byte, value, data[at]); + return false; + } + } + return true; +} +static bool visible_pfns(void) +{ + FILE *f = fopen("/proc/self/status", "re"); + char line[256]; unsigned long long caps = 0; + if (!f) return false; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "CapEff: %llx", &caps) == 1) break; + fclose(f); return !!(caps & (1ULL << 21)); +} +static bool entries(pid_t pid, const void *base, uint64_t out[LEAVES]) +{ + char path[64]; snprintf(path, sizeof(path), "/proc/%ld/pagemap", (long)pid); + int fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) { fprintf(stderr, "PAGEMAP_OPEN_FAIL stage=%s pid=%ld errno=%d\n", stage, (long)pid, errno); return false; } + ssize_t n = pread(fd, out, LEAVES * 8, (uintptr_t)base / LEAF * 8); + close(fd); + if (n != LEAVES * 8) { fprintf(stderr, "PAGEMAP_READ_FAIL stage=%s bytes=%zd errno=%d\n", stage, n, errno); return false; } + for (size_t i = 0; i < LEAVES; i++) + if (!(out[i] & PRESENT) || !(out[i] & PFN_MASK)) { + fprintf(stderr, "PAGEMAP_ENTRY_FAIL stage=%s pid=%ld leaf=%zu raw=%016llx present=%d file=%d pfn_visible=%d\n", + stage, (long)pid, i, (unsigned long long)out[i], !!(out[i] & PRESENT), + !!(out[i] & FILE_PAGE), !!(out[i] & PFN_MASK)); + return false; + } + return true; +} +static bool quarter_file_state(const uint64_t *map, unsigned quarter, bool file) +{ + for (size_t leaf = quarter; leaf < LEAVES; leaf += 4) + if (!!(map[leaf] & FILE_PAGE) != file) { + fprintf(stderr, "PTE_KIND_FAIL stage=%s leaf=%zu expected_file=%d raw=%016llx\n", + stage, leaf, file, (unsigned long long)map[leaf]); + return false; + } + return true; +} +static unsigned mismatches(const uint64_t *map, unsigned logical_quarter, size_t *first) +{ + unsigned count = 0; + for (size_t leaf = logical_quarter; leaf < LEAVES; leaf += 4) { + if ((map[leaf] & PFN_MASK) % 4 != logical_quarter) { + if (!count && first) *first = leaf; + count++; + } + } + fprintf(stderr, "MISMATCH_WITNESS stage=%s logical_quarter=%u mismatches=%u/%lu\n", + stage, logical_quarter, count, WINDOWS); + return count; +} +/* Does not fault payload or require present entries: proves the fork premise. */ +static bool entry_summary(const void *base, const char *view) +{ + uint64_t map[LEAVES]; unsigned present = 0, file = 0; + int fd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + if (fd < 0) return false; + ssize_t n = pread(fd, map, sizeof(map), (uintptr_t)base / LEAF * 8); + close(fd); + if (n != sizeof(map)) return false; + for (size_t i = 0; i < LEAVES; i++) { + present += !!(map[i] & PRESENT); + file += !!(map[i] & PRESENT) && !!(map[i] & FILE_PAGE); + } + fprintf(stderr, "PREFLIGHT stage=%s view=%s present=%u file_present=%u leaves=%lu first_raw=%016llx\n", + stage, view, present, file, LEAVES, (unsigned long long)map[0]); + return true; +} +static bool token(int fd, char wanted) +{ + char got; + return io_full(fd, &got, 1, false) && got == wanted; +} +#define CHILD_CHECK(test, name, code) do { stage = name; if (!(test)) { \ + fprintf(stderr, "CHILD_FAIL stage=%s exit=%d errno=%d\n", stage, code, errno); _exit(code); } } while (0) +static void child(unsigned char *a, unsigned char *b, int control, int status) +{ + struct message msg; + uint64_t before[LEAVES], after[LEAVES]; + alarm(20); + CHILD_CHECK(entry_summary(a, "A") && entry_summary(b, "B"), "before-child-prefault", 2); + /* Fork may skip reconstructible file PTEs. Establish them in this mm. */ + CHILD_CHECK(verify(a, 0, SIZE_MAX), "child-prefault-A-original", 3); + CHILD_CHECK(verify(b, 0, SIZE_MAX), "child-prefault-B-original", 3); + CHILD_CHECK(entry_summary(a, "A") && entry_summary(b, "B"), "after-child-prefault", 2); + CHILD_CHECK(entries(getpid(), a, before), "CPU-source-present", 3); + CHILD_CHECK(quarter_file_state(before, 3, true), "CPU-source-file-PTE", 3); + for (size_t window = 0; window < WINDOWS; window++) { + size_t at = (window * 4 + 3) * LEAF; + *(volatile unsigned char *)(a + at) = original(at) ^ 0x5a; + } + CHILD_CHECK(verify(a, 1, SIZE_MAX), "CPU-COW-full-bytes", 3); + CHILD_CHECK(entries(getpid(), a, after), "CPU-destination-present", 3); + CHILD_CHECK(quarter_file_state(after, 3, false), "CPU-destination-anon-PTE", 3); + size_t first = SIZE_MAX; + unsigned cpu_mismatch = mismatches(after, 3, &first); + CHILD_CHECK(cpu_mismatch, "CPU-COW-actual-quarter-mismatch", 5); + msg = (struct message){ .address = first, .phase = 1, .detail = cpu_mismatch }; + CHILD_CHECK(io_full(status, &msg, sizeof(msg), true), "CPU-result-write", 2); + CHILD_CHECK(token(control, 'm'), "wait-mremap-command", 2); + CHILD_CHECK(verify(a, 1, first), "post-GUP-A-full-bytes", 3); + CHILD_CHECK(verify(b, 2, SIZE_MAX), "post-GUP-B-full-bytes", 3); + CHILD_CHECK(entries(getpid(), b, before), "before-mremap-PFNs", 3); + void *reserve = mmap(NULL, BYTES + 2 * NATIVE, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHILD_CHECK(reserve != MAP_FAILED, "mremap-reservation", 2); + unsigned long desired = ((uintptr_t)b + LEAF) % NATIVE; + unsigned char *destination = (void *)((((uintptr_t)reserve + NATIVE - 1) & ~(NATIVE - 1)) + desired); + unsigned char *moved = mremap(b, BYTES, BYTES, MREMAP_MAYMOVE | MREMAP_FIXED, destination); + CHILD_CHECK(moved != MAP_FAILED && moved == destination && + (uintptr_t)moved % NATIVE != (uintptr_t)b % NATIVE, "mremap-shifted-VA", 3); + CHILD_CHECK(verify(moved, 2, SIZE_MAX), "mremap-full-bytes", 3); + CHILD_CHECK(entries(getpid(), moved, after), "after-mremap-PFNs", 3); + for (size_t leaf = 0; leaf < LEAVES; leaf++) { + if ((before[leaf] & PFN_MASK) != (after[leaf] & PFN_MASK)) { + fprintf(stderr, "PFN_CHANGE stage=mremap leaf=%zu before=%016llx after=%016llx\n", + leaf, (unsigned long long)before[leaf], (unsigned long long)after[leaf]); + CHILD_CHECK(false, "mremap-preserved-PFNs", 6); + } + } + msg = (struct message){ .address = (uintptr_t)moved, .phase = 2, .detail = desired }; + CHILD_CHECK(io_full(status, &msg, sizeof(msg), true), "mremap-result-write", 2); + CHILD_CHECK(token(control, 'x'), "wait-final-release", 2); + CHILD_CHECK(verify(a, 1, first), "final-A-full-bytes", 3); + CHILD_CHECK(verify(moved, 2, SIZE_MAX), "final-B-full-bytes", 3); + munmap(reserve, BYTES + 2 * NATIVE); munmap(a, BYTES); + close(control); close(status); _exit(0); +} +static bool remote_one(pid_t pid, void *local, void *remote, size_t bytes, bool write_it) +{ + struct iovec l = { .iov_base = local, .iov_len = bytes }; + struct iovec r = { .iov_base = remote, .iov_len = bytes }; + ssize_t n = write_it ? process_vm_writev(pid, &l, 1, &r, 1, 0) : + process_vm_readv(pid, &l, 1, &r, 1, 0); + return n == (ssize_t)bytes; +} +static bool remote_quarters(pid_t pid, void *local, unsigned char *remote, + unsigned quarter, size_t offset, size_t bytes, bool write_it) +{ + struct iovec l = { .iov_base = local, .iov_len = WINDOWS * bytes }; + struct iovec r[WINDOWS]; + for (size_t window = 0; window < WINDOWS; window++) + r[window] = (struct iovec){ .iov_base = remote + (window * 4 + quarter) * LEAF + offset, .iov_len = bytes }; + ssize_t n = write_it ? process_vm_writev(pid, &l, 1, r, WINDOWS, 0) : + process_vm_readv(pid, &l, 1, r, WINDOWS, 0); + return n == (ssize_t)(WINDOWS * bytes); +} +static bool wait_ok(pid_t pid) +{ + int status; pid_t n; + do { n = waitpid(pid, &status, 0); } while (n < 0 && errno == EINTR); + if (n == pid) { child_reaped = true; child_status = status; report_wait(pid, status); } + else fprintf(stderr, "CHILD_WAIT_ERROR pid=%ld errno=%d\n", (long)pid, errno); + return n == pid && WIFEXITED(status) && !WEXITSTATUS(status); +} +#define CHECK(test, message) do { stage = message; if (!(test)) { fprintf(stderr, "FAIL: %s (errno=%d)\n", message, errno); goto cleanup; } } while (0) +int main(int argc, char **argv) +{ + unsigned char *buffer = NULL, *a = MAP_FAILED, *b = MAP_FAILED; + int fd = -1, control[2] = {-1,-1}, status[2] = {-1,-1}; + pid_t pid = -1; bool ok = false; + uint64_t map[LEAVES]; struct message msg; + unsigned cpu_mismatch = 0, read_mismatch = 0, write_mismatch = 0; + size_t first_a = SIZE_MAX, first_b = SIZE_MAX; + if (argc != 2 || strcmp(argv[1], "--native-16k")) { + fprintf(stderr, "usage: %s --native-16k\n", argv[0]); return 2; + } + if (getauxval(AT_PAGESZ) != LEAF || !visible_pfns()) { + fprintf(stderr, "SKIP: established4K launcher/native16K kernel/visiblePFNs required\n"); return 4; + } + signal(SIGPIPE, SIG_IGN); signal(SIGALRM, stop); signal(SIGTERM, stop); signal(SIGINT, stop); + alarm(30); + buffer = malloc(BYTES); + CHECK(buffer, "buffer allocation"); + for (size_t i = 0; i < BYTES; i++) buffer[i] = original(i); + fd = memfd_create("own-file-cow-consumers", MFD_CLOEXEC); + CHECK(fd >= 0 && !ftruncate(fd, BYTES), "owned memfd"); + CHECK(pwrite(fd, buffer, BYTES, 0) == BYTES, "initialize memfd"); + a = mmap(NULL, BYTES, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); + b = mmap(NULL, BYTES, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); + CHECK(a != MAP_FAILED && b != MAP_FAILED, "private file views"); + /* Full reads establish file PTEs before COW and typed pin operations. */ + CHECK(verify(a, 0, SIZE_MAX) && verify(b, 0, SIZE_MAX), "prefault originals"); + CHECK(!pipe(control) && !pipe(status), "control pipes"); + pid = fork(); CHECK(pid >= 0, "fork"); + if (!pid) { + CHILD_CHECK(!setpgid(0, 0), "set-child-group", 2); + close(control[1]); close(status[0]); close(fd); + child(a, b, control[0], status[1]); + } + child_group = pid; + if (setpgid(pid, pid) && errno != EACCES && errno != ESRCH) CHECK(false, "child group"); + close(control[0]); control[0] = -1; close(status[1]); status[1] = -1; + bool received = io_full(status[0], &msg, sizeof(msg), false); + if (!received) { fprintf(stderr, "RESULT_EOF stage=CPU-COW; waiting for exact child status\n"); (void)wait_ok(pid); } + CHECK(received && msg.phase == 1, "CPU COW result"); + cpu_mismatch = msg.detail; first_a = msg.address; + CHECK(cpu_mismatch && first_a < LEAVES && first_a % 4 == 3, "actual CPU COW offset mismatch"); + CHECK(remote_one(pid, buffer, a + first_a * LEAF, LEAF, false), "GUP read of mismatched anonymous slot"); + for (size_t i = 0; i < LEAF; i++) + CHECK(buffer[i] == (unsigned char)(original(first_a * LEAF + i) ^ (i == 0 ? 0x5a : 0)), "GUP read bytes"); + memset(buffer, 0xe3, 32); + CHECK(remote_one(pid, buffer, a + first_a * LEAF + 17, 32, true), "GUP write of mismatched anonymous slot"); + CHECK(entries(pid, b, map) && quarter_file_state(map, 3, true), "read-pin sources remain file PTEs"); + CHECK(remote_quarters(pid, buffer, b, 3, 0, LEAF, false), "typed read-pin file unshare"); + for (size_t w = 0; w < WINDOWS; w++) + for (size_t i = 0; i < LEAF; i++) + CHECK(buffer[w * LEAF + i] == original((w * 4 + 3) * LEAF + i), "read-pin copied bytes"); + CHECK(entries(pid, b, map) && quarter_file_state(map, 3, false), "read-pin produced anonymous slots"); + read_mismatch = mismatches(map, 3, &first_b); + CHECK(read_mismatch && first_b < LEAVES, "actual read-pin file COW offset mismatch"); + CHECK(quarter_file_state(map, 2, true), "write-pin sources remain file PTEs"); + memset(buffer, 0xc7, WINDOWS * 32); + CHECK(remote_quarters(pid, buffer, b, 2, 37, 32, true), "typed write-pin file COW"); + CHECK(entries(pid, b, map) && quarter_file_state(map, 2, false), "write-pin produced anonymous slots"); + write_mismatch = mismatches(map, 2, NULL); + CHECK(write_mismatch, "actual write-pin file COW offset mismatch"); + char command = 'm'; CHECK(io_full(control[1], &command, 1, true), "mremap command"); + received = io_full(status[0], &msg, sizeof(msg), false); + if (!received) { fprintf(stderr, "RESULT_EOF stage=mremap; waiting for exact child status\n"); (void)wait_ok(pid); } + CHECK(received && msg.phase == 2, "mremap content/PFN result"); + unsigned char *moved = (void *)msg.address; + CHECK(remote_one(pid, buffer, moved + first_b * LEAF, LEAF, false), "GUP read after shifted mremap"); + for (size_t i = 0; i < LEAF; i++) CHECK(buffer[i] == original(first_b * LEAF + i), "moved GUP bytes"); + CHECK(verify(a, 0, SIZE_MAX) && verify(b, 0, SIZE_MAX), "parent private views unchanged"); + CHECK(pread(fd, buffer, BYTES, 0) == BYTES && verify(buffer, 0, SIZE_MAX), "original memfd byte exact"); + command = 'x'; CHECK(io_full(control[1], &command, 1, true), "child release"); + CHECK(wait_ok(pid), "child final integrity"); pid = -1; child_group = 0; + printf("{\"file_bytes\":%lu,\"cpu_cow_mismatches\":%u,\"read_pin_cow_mismatches\":%u," + "\"write_pin_cow_mismatches\":%u,\"targets_per_path\":%lu,\"mremap_shifted_va_quarter\":true," + "\"mremap_preserved_pfns\":true,\"gup_after_mremap\":true,\"parent_and_file_unchanged\":true,\"content_ok\":true}\n", + BYTES, cpu_mismatch, read_mismatch, write_mismatch, WINDOWS); + ok = true; +cleanup: + if (pid > 0 && !child_reaped) { + pid_t got = waitpid(pid, &child_status, WNOHANG); + if (got == pid) { child_reaped = true; report_wait(pid, child_status); } + else { kill(-pid, SIGKILL); (void)wait_ok(pid); } + } + child_group = 0; alarm(0); + for (unsigned i = 0; i < 2; i++) { if (control[i] >= 0) close(control[i]); if (status[i] >= 0) close(status[i]); } + if (a != MAP_FAILED) munmap(a, BYTES); + if (b != MAP_FAILED) munmap(b, BYTES); + if (fd >= 0) close(fd); + free(buffer); + return !ok; +} diff --git a/tools/testing/selftests/arm64/user4k/same-native.c b/tools/testing/selftests/arm64/user4k/same-native.c new file mode 100644 index 0000000000000..9c592ee98c3a7 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/same-native.c @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Three generations; actual PFNs distinguish same-native-folio COW coverage. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define LEAF 4096UL +#define NATIVE 16384UL +#define TRIALS 16 +#define PFN_MASK ((UINT64_C(1) << 55) - 1) +struct result { uint64_t old_pfn, new_pfn, held_pfn; unsigned quarter; }; +static volatile sig_atomic_t group; +static unsigned char *data; +static void stop(int sig) +{ + if (group > 0) kill(-group, SIGKILL); + _exit(128 + sig); +} +static bool io_full(int fd, void *buf, size_t bytes, bool output) +{ + unsigned char *p = buf; + while (bytes) { + ssize_t n = output ? write(fd, p, bytes) : read(fd, p, bytes); + if (n < 0 && errno == EINTR) continue; + if (n <= 0) return false; + p += n; bytes -= n; + } + return true; +} +static unsigned char pattern(size_t at, unsigned generation, unsigned quarter) +{ + unsigned salt = at / LEAF == quarter ? generation * 53 : 0; + return (unsigned char)((at * 17 + at / LEAF * 29 + at / 256 + salt) % 251 + 1); +} +static void change(unsigned generation, unsigned quarter) +{ + size_t at; + for (at = quarter * LEAF; at < (quarter + 1) * LEAF; at++) + data[at] = pattern(at, generation, quarter); +} +static bool verify(unsigned generation, unsigned quarter) +{ + size_t at; + for (at = 0; at < NATIVE; at++) + if (data[at] != pattern(at, generation, quarter)) return false; + return true; +} +static bool pfn(int fd, unsigned quarter, uint64_t *out) +{ + uint64_t entry; + off_t where = ((uintptr_t)data / LEAF + quarter) * 8; + if (pread(fd, &entry, 8, where) != 8 || !(entry & (UINT64_C(1) << 63)) || + (entry & (UINT64_C(1) << 62)) || !(entry & PFN_MASK)) return false; + *out = entry & PFN_MASK; + return true; +} +static bool wait_ok(pid_t child) +{ + int status; pid_t got; + do { got = waitpid(child, &status, 0); } while (got < 0 && errno == EINTR); + return got == child && WIFEXITED(status) && WEXITSTATUS(status) == 0; +} +static bool caps(void) +{ + FILE *f = fopen("/proc/self/status", "re"); + char line[256]; unsigned long long effective = 0; + if (!f) return false; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "CapEff: %llx", &effective) == 1) break; + fclose(f); return !!(effective & (1ULL << 21)); +} +static void owner(int output, unsigned quarter) +{ + struct result result = { .quarter = quarter }; + int go[2], answer[2], mapfd; + char token; + pid_t held; + alarm(5); + if (pipe(go) || pipe(answer)) _exit(10); + /* Open in this mm, never inherit an ancestor's already-open pagemap fd. */ + mapfd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + if (mapfd < 0) _exit(11); + change(1, quarter); + if (!pfn(mapfd, quarter, &result.old_pfn) || !verify(1, quarter)) _exit(12); + held = fork(); + if (held < 0) _exit(13); + if (!held) { + uint64_t held_pfn; + close(go[1]); close(answer[0]); close(output); close(mapfd); + alarm(4); + if (!io_full(go[0], &token, 1, false) || token != 'r') _exit(14); + mapfd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + if (mapfd < 0 || !pfn(mapfd, quarter, &held_pfn) || + held_pfn != result.old_pfn || !verify(1, quarter) || + !io_full(answer[1], &held_pfn, sizeof(held_pfn), true)) _exit(15); + close(mapfd); close(go[0]); close(answer[1]); _exit(0); + } + /* Before libc/pipe bookkeeping, consume a free sibling if allocator picks it. */ + change(2, quarter); + if (!pfn(mapfd, quarter, &result.new_pfn) || + result.new_pfn == result.old_pfn || !verify(2, quarter)) _exit(16); + close(go[0]); close(answer[1]); token = 'r'; + if (!io_full(go[1], &token, 1, true) || + !io_full(answer[0], &result.held_pfn, sizeof(result.held_pfn), false) || + !wait_ok(held) || !verify(2, quarter)) _exit(17); + close(go[1]); close(answer[0]); close(mapfd); + if (!io_full(output, &result, sizeof(result), true)) _exit(18); + close(output); _exit(0); +} +int main(int argc, char **argv) +{ + unsigned same = 0; + if (argc == 2 && !strcmp(argv[1], "--selftest")) { + for (unsigned q = 0; q < 4; q++) + for (size_t at = 0; at < NATIVE; at++) + if ((pattern(at, 1, q) != pattern(at, 2, q)) != (at / LEAF == q)) return 1; + puts("PASS generation pattern selftest"); return 0; + } + if (argc != 2 || strcmp(argv[1], "--native-16k") || + getauxval(AT_PAGESZ) != LEAF || !caps()) { + fputs("Requires established4K launcher/native16K kernel and visible PFNs\n", stderr); return 77; + } + if (prctl(PR_SET_CHILD_SUBREAPER, 1)) return 1; + signal(SIGPIPE, SIG_IGN); signal(SIGALRM, stop); signal(SIGTERM, stop); signal(SIGINT, stop); + setvbuf(stdout, NULL, _IONBF, 0); alarm(20); + void *map = mmap(NULL, 2 * NATIVE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (map == MAP_FAILED) return 1; + data = (void *)(((uintptr_t)map + NATIVE - 1) & ~(NATIVE - 1)); + for (size_t at = 0; at < NATIVE; at++) data[at] = pattern(at, 0, 0); + for (unsigned trial = 0; trial < TRIALS; trial++) { + int output[2]; struct result result; pid_t pid; + if (pipe(output)) return 1; + pid = fork(); + if (pid < 0) return 1; + if (!pid) { + group = 0; + if (setpgid(0, 0)) _exit(1); + close(output[0]); owner(output[1], trial % 4); + } + group = pid; + if (setpgid(pid, pid) && errno != EACCES && errno != ESRCH) { + kill(pid, SIGKILL); stop(SIGTERM); + } + close(output[1]); + bool received = io_full(output[0], &result, sizeof(result), false); + close(output[0]); + bool good = wait_ok(pid); + /* Reap any orphaned held generation on failure; no child can escape group. */ + kill(-pid, SIGKILL); + while (waitpid(-1, NULL, 0) > 0 || errno == EINTR) {} + group = 0; + if (!received || !good || result.held_pfn != result.old_pfn || + result.new_pfn == result.old_pfn || !verify(0, 0)) { + fprintf(stderr, "FAIL trial %u generation/PFN/isolation/cleanup\n", trial); return 1; + } + bool native_equal = result.old_pfn / 4 == result.new_pfn / 4; + same += native_equal; + printf("{\"trial\":%u,\"logical_quarter\":%u,\"old_pfn4k\":%"PRIu64",\"new_pfn4k\":%"PRIu64",\"held_pfn4k\":%"PRIu64",\"same_native\":%s,\"three_generation_bytes_ok\":true}\n", + trial, result.quarter, result.old_pfn, result.new_pfn, result.held_pfn, native_equal ? "true" : "false"); + } + munmap(map, 2 * NATIVE); alarm(0); + printf("{\"result\":\"%s\",\"trials\":%u,\"same_native_exercised\":%u,\"payload_bytes\":%lu}\n", same ? "PASS" : "UNEXERCISED", TRIALS, same, NATIVE); + return same ? 0 : 77; +} diff --git a/tools/testing/selftests/arm64/user4k/same-quarter.c b/tools/testing/selftests/arm64/user4k/same-quarter.c new file mode 100644 index 0000000000000..ce1f7e80081d0 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/same-quarter.c @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Metadata-only native-backing comparison. Never changes memory policy. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define LEAF 4096UL +#define NATIVE 16384UL +#define BYTES (16UL << 20) +#define LEAVES (BYTES / LEAF) +#define WINDOWS (BYTES / NATIVE) +#define MAX_FRAMES (LEAVES * 2) + +struct coverage { + uint64_t native_units, quarters, occupancy[4]; +}; +struct result { + unsigned arm; + bool content_ok, disjoint_ok; + struct coverage parent, child, modified, unchanged, combined; + uint64_t modified_parent_native_overlap; + uint64_t modified_unchanged_native_overlap; + uint64_t modified_quarter_counts[4]; +}; +static volatile sig_atomic_t fixture_group; + +static void stop_fixture(int sig) +{ + if (fixture_group > 0) + kill(-fixture_group, SIGKILL); + _exit(128 + sig); +} +static bool full_io(int fd, void *buffer, size_t bytes, bool writing) +{ + char *p = buffer; + while (bytes) { + ssize_t n = writing ? write(fd, p, bytes) : read(fd, p, bytes); + if (n < 0 && errno == EINTR) + continue; + if (n <= 0) + return false; + p += n; + bytes -= (size_t)n; + } + return true; +} +static bool wait_ok(pid_t pid) +{ + int status; + pid_t got; + do { got = waitpid(pid, &status, 0); } while (got < 0 && errno == EINTR); + return got > 0 && WIFEXITED(status) && WEXITSTATUS(status) == 0; +} +static int compare_u64(const void *a, const void *b) +{ + uint64_t x = *(const uint64_t *)a, y = *(const uint64_t *)b; + return (x > y) - (x < y); +} +/* Input is actual4K PFN, never VMA-derived quarter position. Duplicates count once. */ +static struct coverage summarize(uint64_t *frames, size_t count) +{ + struct coverage out = {}; + uint64_t last = UINT64_MAX; + unsigned mask = 0; + qsort(frames, count, sizeof(*frames), compare_u64); + for (size_t i = 0; i < count; i++) { + uint64_t native = frames[i] / 4; + if (native != last) { + if (mask) { + unsigned n = __builtin_popcount(mask); + out.native_units++; + out.quarters += n; + out.occupancy[n - 1]++; + } + last = native; + mask = 0; + } + mask |= 1U << (frames[i] % 4); + } + if (mask) { + unsigned n = __builtin_popcount(mask); + out.native_units++; + out.quarters += n; + out.occupancy[n - 1]++; + } + return out; +} +/* Both arrays were sorted by summarize. Count each common nativeunit once. */ +static uint64_t native_overlap(const uint64_t *a, size_t an, const uint64_t *b, size_t bn) +{ + size_t ai = 0, bi = 0; + uint64_t count = 0; + while (ai < an && bi < bn) { + uint64_t av = a[ai] / 4, bv = b[bi] / 4; + if (av == bv) + count++; + if (av <= bv) + while (ai < an && a[ai] / 4 == av) ai++; + if (bv <= av) + while (bi < bn && b[bi] / 4 == bv) bi++; + } + return count; +} +static bool read_frames(pid_t pid, const unsigned char *data, uint64_t frames[LEAVES]) +{ + char path[64]; + uint64_t entries[LEAVES]; + snprintf(path, sizeof(path), "/proc/%ld/pagemap", (long)pid); + int fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return false; + ssize_t n = pread(fd, entries, sizeof(entries), (uintptr_t)data / LEAF * 8); + close(fd); + if (n != (ssize_t)sizeof(entries)) + return false; + for (size_t i = 0; i < LEAVES; i++) { + uint64_t pfn = entries[i] & ((UINT64_C(1) << 55) - 1); + if (!(entries[i] & (UINT64_C(1) << 63)) || !pfn) + return false; /* Swap/hiddenPFNs invalidate this bounded resident comparison. */ + frames[i] = pfn; + } + return true; +} +static unsigned chosen_quarter(unsigned arm, size_t window) +{ + return arm ? window % 4 : 0; +} +static unsigned char tag(size_t leaf, bool changed) +{ + unsigned char value = (unsigned char)(leaf % 251 + 1); + return changed ? value ^ 0xa5 : value; +} +static bool contents(const unsigned char *data, unsigned arm, bool child) +{ + for (size_t leaf = 0; leaf < LEAVES; leaf++) { + bool changed = child && leaf % 4 == chosen_quarter(arm, leaf / 4); + if (data[leaf * LEAF] != tag(leaf, changed)) + return false; + for (size_t byte = 1; byte < LEAF; byte++) + if (data[leaf * LEAF + byte] != 0x5a) + return false; + } + return true; +} +static void owner(unsigned arm, int output) +{ + uint64_t parent[LEAVES], child_frames[LEAVES], modified[WINDOWS]; + uint64_t unchanged[LEAVES - WINDOWS], combined[MAX_FRAMES]; + struct result result = {.arm = arm}; + int ready[2], release[2]; + char token; + size_t mi = 0, ui = 0; + alarm(20); + void *mapping = mmap(NULL, BYTES + 2 * NATIVE, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED) + _exit(2); + unsigned char *data = (void *)(((uintptr_t)mapping + NATIVE - 1) & ~(NATIVE - 1)); + if (mprotect(data, BYTES, PROT_READ | PROT_WRITE)) + _exit(2); + memset(data, 0x5a, BYTES); + for (size_t leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] = tag(leaf, false); + if (!contents(data, arm, false) || !read_frames(getpid(), data, parent)) + _exit(3); + struct coverage initial = summarize(parent, LEAVES); + if (initial.native_units * NATIVE != BYTES || initial.quarters != LEAVES) + _exit(4); /* Stop beforefork if the32MiB payload ceiling is not established. */ + if (pipe(ready) || pipe(release)) + _exit(2); + pid_t child = fork(); + if (child < 0) + _exit(2); + if (!child) { + alarm(20); + close(ready[0]); close(release[1]); close(output); + for (size_t window = 0; window < WINDOWS; window++) { + size_t leaf = window * 4 + chosen_quarter(arm, window); + *(volatile unsigned char *)(data + leaf * LEAF) = tag(leaf, true); + } + if (!contents(data, arm, true)) + _exit(3); + token = 'r'; + if (!full_io(ready[1], &token, 1, true)) + _exit(2); + close(ready[1]); + if (!full_io(release[0], &token, 1, false) || token != 'x') + _exit(2); + close(release[0]); + _exit(contents(data, arm, true) ? 0 : 3); + } + close(ready[1]); close(release[0]); + if (!full_io(ready[0], &token, 1, false) || token != 'r') + _exit(2); + close(ready[0]); + result.content_ok = contents(data, arm, false); + if (!result.content_ok || !read_frames(getpid(), data, parent) || + !read_frames(child, data, child_frames)) + _exit(3); + result.disjoint_ok = true; + for (size_t leaf = 0; leaf < LEAVES; leaf++) { + bool changed = leaf % 4 == chosen_quarter(arm, leaf / 4); + if (changed) { + modified[mi++] = child_frames[leaf]; + result.modified_quarter_counts[child_frames[leaf] % 4]++; + result.disjoint_ok &= parent[leaf] != child_frames[leaf]; + } else { + unchanged[ui++] = child_frames[leaf]; + result.disjoint_ok &= parent[leaf] == child_frames[leaf]; + } + } + memcpy(combined, parent, sizeof(parent)); + memcpy(combined + LEAVES, child_frames, sizeof(child_frames)); + result.parent = summarize(parent, LEAVES); + result.child = summarize(child_frames, LEAVES); + result.modified = summarize(modified, mi); + result.unchanged = summarize(unchanged, ui); + result.combined = summarize(combined, MAX_FRAMES); + result.modified_parent_native_overlap = native_overlap(modified, mi, parent, LEAVES); + result.modified_unchanged_native_overlap = native_overlap(modified, mi, unchanged, ui); + result.disjoint_ok &= mi == WINDOWS && result.modified.quarters == WINDOWS && + result.combined.quarters == LEAVES + WINDOWS && + result.modified_parent_native_overlap == 0 && + result.modified_unchanged_native_overlap == 0 && + result.combined.native_units * NATIVE <= 2 * BYTES; + token = 'x'; + if (!full_io(release[1], &token, 1, true)) + _exit(2); + close(release[1]); + bool child_ok = wait_ok(child); + result.content_ok &= child_ok && contents(data, arm, false); + munmap(mapping, BYTES + 2 * NATIVE); + if (!full_io(output, &result, sizeof(result), true)) + _exit(2); + close(output); + _exit(result.content_ok && result.disjoint_ok ? 0 : 3); +} +static bool visible_pfns(void) +{ + FILE *file = fopen("/proc/self/status", "re"); + char line[256]; + unsigned long long caps = 0; + if (!file) return false; + while (fgets(line, sizeof(line), file)) + if (sscanf(line, "CapEff: %llx", &caps) == 1) break; + fclose(file); + return !!(caps & (1ULL << 21)); +} +static void print_coverage(const char *name, struct coverage c) +{ + printf("\"%s\":{\"native_bytes\":%llu,\"represented_bytes\":%llu," + "\"quarter_occupancy\":[%llu,%llu,%llu,%llu]}", name, + (unsigned long long)(c.native_units * NATIVE), + (unsigned long long)(c.quarters * LEAF), + (unsigned long long)c.occupancy[0], (unsigned long long)c.occupancy[1], + (unsigned long long)c.occupancy[2], (unsigned long long)c.occupancy[3]); +} +static int selftest(void) +{ + uint64_t a[] = {400, 401, 401, 403, 404}, b[] = {402, 405}; + uint64_t combined[] = {400, 401, 401, 403, 404, 402, 405}; + struct coverage ca = summarize(a, 5), cb = summarize(b, 2); + struct coverage cc = summarize(combined, 7); + bool ok = ca.native_units == 2 && ca.quarters == 4 && ca.occupancy[0] == 1 && + ca.occupancy[2] == 1 && cb.native_units == 2 && cb.quarters == 2 && + cc.native_units == 2 && cc.quarters == 6 && cc.occupancy[1] == 1 && + cc.occupancy[3] == 1 && native_overlap(a, 5, b, 2) == 2; + uint64_t sparse[] = {400, 404, 408, 412}, dense[] = {400, 401, 402, 403}; + ca = summarize(sparse, 4); cb = summarize(dense, 4); + ok &= ca.native_units == 4 && cb.native_units == 1 && ca.quarters == cb.quarters; + printf("metadata selftest: %s\n", ok ? "PASS" : "FAIL"); + return !ok; +} +int main(int argc, char **argv) +{ + if (argc == 2 && !strcmp(argv[1], "--selftest")) return selftest(); + if (argc != 2 || strcmp(argv[1], "--native-16k")) { fprintf(stderr, "usage: %s --native-16k | --selftest\n", argv[0]); return 2; } + if (getauxval(AT_PAGESZ) != LEAF || !visible_pfns()) { + fprintf(stderr, "SKIP: established4K launcher, native16K kernel and visiblePFNs required\n"); + return 4; + } + signal(SIGPIPE, SIG_IGN); signal(SIGALRM, stop_fixture); signal(SIGTERM, stop_fixture); + signal(SIGINT, stop_fixture); + if (prctl(PR_SET_CHILD_SUBREAPER, 1)) return 2; + alarm(30); + for (unsigned arm = 0; arm < 2; arm++) { + int pipefd[2]; struct result result = {}; bool copied, success; + if (pipe(pipefd)) return 2; + pid_t child = fork(); + if (child < 0) return 2; + if (!child) { + if (setpgid(0, 0)) _exit(2); + close(pipefd[0]); owner(arm, pipefd[1]); + } + fixture_group = child; + if (setpgid(child, child) && errno != EACCES && errno != ESRCH) stop_fixture(SIGTERM); + close(pipefd[1]); + copied = full_io(pipefd[0], &result, sizeof(result), false); + close(pipefd[0]); success = wait_ok(child); + if (!copied || !success) { + kill(-child, SIGKILL); + while (waitpid(-1, NULL, 0) > 0 || errno == EINTR) {} + fprintf(stderr, "FAIL: arm%u incomplete or integrity/coverage check failed\n", arm); + return 1; + } + fixture_group = 0; + printf("{\"arm\":\"%s\",\"logical_parent_bytes\":%lu,\"modified_child_bytes\":%lu,", + arm ? "balanced" : "same-quarter-zero", BYTES, WINDOWS * LEAF); + print_coverage("parent", result.parent); printf(","); + print_coverage("child", result.child); printf(","); + print_coverage("modified_child", result.modified); printf(","); + print_coverage("unchanged_child", result.unchanged); printf(","); + print_coverage("parent_child_union", result.combined); + printf(",\"modified_parent_native_overlap\":%llu,\"modified_unchanged_native_overlap\":%llu," + "\"modified_physical_quarter_counts\":[%llu,%llu,%llu,%llu],\"content_ok\":true,\"disjoint_ok\":true}\n", + (unsigned long long)result.modified_parent_native_overlap, + (unsigned long long)result.modified_unchanged_native_overlap, + (unsigned long long)result.modified_quarter_counts[0], (unsigned long long)result.modified_quarter_counts[1], + (unsigned long long)result.modified_quarter_counts[2], (unsigned long long)result.modified_quarter_counts[3]); + fflush(stdout); + } + alarm(0); + return 0; +} From c9a6de2b8fa3c4ec822fffcee33631e241d66a65 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 18:49:51 -0400 Subject: [PATCH 0863/2680] mm: share backing for clipped anonymous and file first writes Short anonymous windows and untouched private-file writes allocated a fresh 16 KiB folio for individual 4 KiB destinations. They consumed 64 MiB of backing for 16 MiB of written data on a 16 KiB kernel. Reserve clipped anonymous batches from one shared owner using a matching free-slot count throughout lookup, waiting and supply. Keep slots indexed by their logical mask while zeroing and publishing their actual physical quarters. Preserve full native-window batching and private fallbacks. Pool private-file first writes only for known filemap and shmem callbacks. Release destination locks before invoking the source callback, serialize publication and nonlinear rmap preparation, and mark shared folios uptodate atomically. Retry publication metadata OOM once with private backing, without internally retrying callbacks that dropped the VMA lock. Hardware first-write cases now use 16.06-16.19 MiB for 16 MiB of data. Clipped-neighbor prefaulting, full contents, source isolation, repeated eight-writer first faults, COW/GUP/mremap and actual reclaim/readback pass. Dense allocation remains 16 MiB; sparse wide-VMA prefaulting is unchanged. Maintained regression binaries match the hardware-tested artifacts. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit a0e28af0b29548a42148ce7e2c5054e24b89883a) --- include/linux/mm_subpage.h | 9 + mm/memory.c | 104 +++++++-- mm/user-subpage.c | 73 ++++-- .../testing/selftests/arm64/user4k/.gitignore | 2 + tools/testing/selftests/arm64/user4k/Makefile | 4 +- .../selftests/arm64/user4k/README.first-write | 28 +++ .../arm64/user4k/clipped-firstwrite.c | 141 ++++++++++++ .../selftests/arm64/user4k/first-write.c | 216 ++++++++++++++++++ 8 files changed, 538 insertions(+), 39 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/README.first-write create mode 100644 tools/testing/selftests/arm64/user4k/clipped-firstwrite.c create mode 100644 tools/testing/selftests/arm64/user4k/first-write.c diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h index e892ae3e70bee..910ed2b36bcfd 100644 --- a/include/linux/mm_subpage.h +++ b/include/linux/mm_subpage.h @@ -68,6 +68,15 @@ int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mas int mm_subpage_pool_add_folio_mask(struct mm_subpage_pool *pool, struct folio *folio, unsigned long mask, gfp_t gfp); bool mm_subpage_cow_wait_busy_mask(struct mm_subpage_pool *pool, unsigned long mask); +/* Arbitrary physical offsets, one owner/folio lock for the whole logical mask. + * Output slots are indexed by logical mask bits; use each slot's actual offset. + * Count-aware supply/wait must accompany this allocation to guarantee progress. + */ +int mm_subpage_alloc_any_mask_locked(struct mm_subpage_pool *pool, unsigned long mask, + struct mm_subpage **slots); +int mm_subpage_pool_add_folio_count(struct mm_subpage_pool *pool, struct folio *folio, + unsigned int count, gfp_t gfp); +bool mm_subpage_cow_wait_busy_count(struct mm_subpage_pool *pool, unsigned int count); /* Return an initialized slot with its folio locked; may wait without a pool lock. */ struct mm_subpage *mm_subpage_alloc_at_locked(struct mm_subpage_pool *pool, unsigned int offset); diff --git a/mm/memory.c b/mm/memory.c index 399c8ea4e1628..53acf12f4319b 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -6347,28 +6347,33 @@ static void map_anon_folio_pte_pf(struct folio *folio, pte_t *pte, /* One native-window batch, initialized and locked before PTE publication. */ static int anon_subpage_prealloc_mask(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, - unsigned long mask, bool use_pool, bool *pooled, - struct mm_subpage **slots) + unsigned long mask, bool use_pool, bool any_offset, + bool *pooled, struct mm_subpage **slots) { struct mm_subpage_pool *pool; struct folio *folio; + unsigned int count = hweight_long(mask); + bool arbitrary; int err; retry_private: pool = use_pool ? mm_subpage_cow_pool_get(mm, vma, addr) : NULL; *pooled = pool != NULL; + arbitrary = *pooled && any_offset; if (!pool) pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); if (!pool) return -ENOMEM; for (;;) { - err = mm_subpage_alloc_mask_locked(pool, mask, slots); + err = arbitrary ? mm_subpage_alloc_any_mask_locked(pool, mask, slots) : + mm_subpage_alloc_mask_locked(pool, mask, slots); if (err != -EAGAIN) break; folio = NULL; if (*pooled) { folio = cow_subpage_speculative_folio(mm, vma, addr); - if (!folio && mm_subpage_cow_wait_busy_mask(pool, mask)) { + if (!folio && (arbitrary ? mm_subpage_cow_wait_busy_count(pool, count) : + mm_subpage_cow_wait_busy_mask(pool, mask))) { cond_resched(); continue; } @@ -6385,15 +6390,20 @@ static int anon_subpage_prealloc_mask(struct mm_struct *mm, use_pool = false; goto retry_private; } - err = mm_subpage_pool_add_folio_mask(pool, folio, mask, + err = arbitrary ? mm_subpage_pool_add_folio_count(pool, folio, count, + GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN) : + mm_subpage_pool_add_folio_mask(pool, folio, mask, *pooled ? GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN : GFP_KERNEL); if (*pooled && err == -ENOMEM) { - if (mm_subpage_cow_wait_busy_mask(pool, mask)) { + if (arbitrary ? mm_subpage_cow_wait_busy_count(pool, count) : + mm_subpage_cow_wait_busy_mask(pool, mask)) { folio_put(folio); cond_resched(); continue; } - err = mm_subpage_pool_add_folio_mask(pool, folio, mask, GFP_KERNEL); + err = arbitrary ? + mm_subpage_pool_add_folio_count(pool, folio, count, GFP_KERNEL) : + mm_subpage_pool_add_folio_mask(pool, folio, mask, GFP_KERNEL); } if (err) { folio_put(folio); @@ -6456,8 +6466,9 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) pte_unmap(vmf->pte); vmf->pte = NULL; err = anon_subpage_prealloc_mask(mm, vma, vmf->address, mask, - mask != window_mask && !force_private && !userfaultfd_armed(vma), - &pooled, slots); + (mask != window_mask || nr < (PAGE_SIZE >> mm_page_shift(mm))) && + !force_private && !userfaultfd_armed(vma), + nr < (PAGE_SIZE >> mm_page_shift(mm)), &pooled, slots); if (err) return VM_FAULT_OOM; folio = mm_subpage_folio(slots[__ffs(mask)]); @@ -7057,7 +7068,9 @@ static vm_fault_t finish_user_page_fault(struct vm_fault *vmf) struct mm_subpage *slot = vmf->cow_subpage; bool is_cow = (vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED); struct folio *folio = page_folio(is_cow ? vmf->cow_page : vmf->page); + bool fresh = false; vm_fault_t ret; + int err; pte_t entry; if (!(vma->vm_flags & VM_SHARED)) { @@ -7071,10 +7084,18 @@ static vm_fault_t finish_user_page_fault(struct vm_fault *vmf) else if (pte_alloc(mm, vmf->pmd)) return VM_FAULT_OOM; } - /* The private destination is not yet visible through a PTE. */ + /* A reserved destination slot is private; sibling slots may be published. */ if (is_cow) { folio_get(folio); folio_lock(folio); + if (slot) { + fresh = !mm_subpage_anon_root(slot); + err = mm_subpage_prepare_anon_rmap(slot, vma, vmf->address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + goto unlock_folio; + } + } } vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); ret = VM_FAULT_NOPAGE; @@ -7086,7 +7107,9 @@ static vm_fault_t finish_user_page_fault(struct vm_fault *vmf) } if (slot) { mm_subpage_set_exclusive(slot); - if (WARN_ON_ONCE(mm_subpage_add_new_anon_rmap(slot, vma, vmf->address))) { + err = fresh ? mm_subpage_add_new_anon_rmap(slot, vma, vmf->address) : + mm_subpage_add_anon_rmap(slot, vma, vmf->address); + if (WARN_ON_ONCE(err)) { ret = VM_FAULT_SIGBUS; goto unlock; } @@ -7094,7 +7117,8 @@ static vm_fault_t finish_user_page_fault(struct vm_fault *vmf) entry = maybe_mkwrite(pte_mkdirty(pte_sw_mkyoung(entry)), vma); if (vmf_orig_pte_uffd_wp(vmf)) entry = pte_mkuffd_wp(entry); - subpage_add_new_lru(folio, vma, 1); + if (fresh) + subpage_add_new_lru(folio, vma, 1); set_pte_at(mm, vmf->address, vmf->pte, entry); update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); inc_mm_counter(mm, MM_ANONPAGES); @@ -7441,6 +7465,10 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) struct vm_area_struct *vma = vmf->vma; struct folio *folio; vm_fault_t ret; +#ifdef CONFIG_MM_SUBPAGE + bool pooled = false, force_private = false; + vm_fault_t earlier = 0; +#endif ret = vmf_can_call_fault(vmf); if (!ret) @@ -7449,11 +7477,24 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) return ret; #ifdef CONFIG_MM_SUBPAGE +retry_subpage: if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { - vmf->cow_subpage = anon_subpage_prealloc(vma->vm_mm, vma, vmf->address); + /* Known callbacks only supply a locked source page for the core. + * Custom/DAX callbacks may fill a native cow_page or finish COW. + */ + if (!force_private && !userfaultfd_armed(vma) && + (vma->vm_ops->fault == filemap_fault || vma_is_shmem(vma))) { + vmf->cow_subpage = cow_subpage_prealloc(vma->vm_mm, vma, + vmf->address, NULL, false, &pooled); + if (!IS_ERR(vmf->cow_subpage)) + folio_unlock(mm_subpage_folio(vmf->cow_subpage)); + } else { + pooled = false; + vmf->cow_subpage = anon_subpage_prealloc(vma->vm_mm, vma, vmf->address); + } if (IS_ERR(vmf->cow_subpage)) { vmf->cow_subpage = NULL; - return VM_FAULT_OOM; + return earlier | VM_FAULT_OOM; } folio = mm_subpage_folio(vmf->cow_subpage); goto allocated; @@ -7478,12 +7519,22 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) #ifdef CONFIG_MM_SUBPAGE if (vmf->cow_subpage) { - if (copy_user_subpage_range(vmf->cow_page, mm_subpage_offset(vmf->cow_subpage), - vmf->page, vma_page_offset_at(vma, vmf->address).offset, - mm_page_size(vma->vm_mm))) { + int err; + + /* No destination lock crosses the source fault callback or retry. */ + if (pooled) + folio_lock(folio); + err = copy_user_subpage_range(vmf->cow_page, mm_subpage_offset(vmf->cow_subpage), + vmf->page, vma_page_offset_at(vma, vmf->address).offset, + mm_page_size(vma->vm_mm)); + if (pooled) + folio_unlock(folio); + if (err) { ret = VM_FAULT_HWPOISON; goto unlock; } + /* Preserve concurrent sibling lock/LRU/waiter flag updates. */ + folio_mark_uptodate(folio); goto copied; } #endif @@ -7504,21 +7555,36 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) goto unlock; } } + __folio_mark_uptodate(folio); #ifdef CONFIG_MM_SUBPAGE copied: #endif - __folio_mark_uptodate(folio); - ret |= finish_fault(vmf); unlock: unlock_page(vmf->page); put_page(vmf->page); +#ifdef CONFIG_MM_SUBPAGE + vmf->page = NULL; + /* Publication OOM left the PTE untouched. Retry with private metadata, + * outside the source lock; never loop internally on callback RETRY. + */ + if (pooled && (ret & VM_FAULT_OOM)) { + mm_subpage_put(vmf->cow_subpage); + vmf->cow_subpage = NULL; + vmf->cow_page = NULL; + earlier |= ret & VM_FAULT_MAJOR; + force_private = true; + goto retry_subpage; + } + ret |= earlier; +#endif if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) goto uncharge_out; return ret; uncharge_out: #ifdef CONFIG_MM_SUBPAGE + ret |= earlier; if (vmf->cow_subpage) { mm_subpage_put(vmf->cow_subpage); vmf->cow_subpage = NULL; diff --git a/mm/user-subpage.c b/mm/user-subpage.c index 30fb3669458cb..fd553493b3de4 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -859,7 +859,7 @@ static bool cow_owner_eligible(struct mm_subpage_pool *pool, struct folio *folio * allocation: a busy slot farther down the list may require no new charge. */ static bool cow_wait_busy(struct mm_subpage_pool *pool, unsigned long eligible, - bool require_all) + bool require_all, unsigned int count) { struct mm_subpage_owner *owner; struct folio *wait = NULL; @@ -870,6 +870,7 @@ static bool cow_wait_busy(struct mm_subpage_pool *pool, unsigned long eligible, list_for_each_entry(owner, &pool->available, available) { if (!owner->migrating && (~owner->used & eligible) && (!require_all || !(owner->used & eligible)) && + (count == 1 || hweight_long(~owner->used & eligible) >= count) && cow_owner_eligible(pool, owner->folio) && !folio_test_swapcache(owner->folio) && !folio_test_writeback(owner->folio)) { @@ -890,21 +891,29 @@ static bool cow_wait_busy(struct mm_subpage_pool *pool, unsigned long eligible, bool mm_subpage_cow_wait_busy(struct mm_subpage_pool *pool) { - return cow_wait_busy(pool, pool->all_slots, false); + return cow_wait_busy(pool, pool->all_slots, false, 1); } bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset) { - return cow_wait_busy(pool, offset_mask(pool->shift, offset), false); + return cow_wait_busy(pool, offset_mask(pool->shift, offset), false, 1); } bool mm_subpage_cow_wait_busy_mask(struct mm_subpage_pool *pool, unsigned long mask) { - return cow_wait_busy(pool, mask, true); + return cow_wait_busy(pool, mask, true, 1); +} + +bool mm_subpage_cow_wait_busy_count(struct mm_subpage_pool *pool, unsigned int count) +{ + if (!count || count > hweight_long(pool->all_slots)) + return false; + return cow_wait_busy(pool, pool->all_slots, false, count); } static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, - unsigned long eligible, bool require_all) + unsigned long eligible, bool require_all, + unsigned int count) { struct mm_subpage_owner *owner; unsigned int visited = 0; @@ -927,6 +936,7 @@ static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, /* A nonpresent PTE may still need any of this folio's slot bytes. */ if (!owner->migrating && (~owner->used & eligible) && (!require_all || !(owner->used & eligible)) && + (count == 1 || hweight_long(~owner->used & eligible) >= count) && (!pool->cow_shared || !folio_test_locked(owner->folio)) && cow_owner_eligible(pool, owner->folio) && !folio_test_swapcache(owner->folio) && @@ -937,7 +947,8 @@ static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, } static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, - unsigned long eligible, bool require_all, gfp_t gfp) + unsigned long eligible, bool require_all, + unsigned int count, gfp_t gfp) { struct mm_subpage_owner *owner; unsigned long flags; @@ -965,7 +976,7 @@ static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, spin_lock_irqsave(&pool->lock, flags); if (pool->closed) ret = -ESHUTDOWN; - else if (pool->unissued || find_available(pool, eligible, require_all)) + else if (pool->unissued || find_available(pool, eligible, require_all, count)) /* Another allocator supplied backing while we allocated metadata. * Preserve the single-unissued-owner invariant even when a shared * COW lookup skips a concurrently locked or rotating list prefix. @@ -993,7 +1004,7 @@ static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, int mm_subpage_pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, gfp_t gfp) { - return pool_add_folio(pool, folio, pool->all_slots, false, gfp); + return pool_add_folio(pool, folio, pool->all_slots, false, 1, gfp); } int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *folio, @@ -1003,7 +1014,7 @@ int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *fol if (!eligible) return -EINVAL; - return pool_add_folio(pool, folio, eligible, false, gfp); + return pool_add_folio(pool, folio, eligible, false, 1, gfp); } int mm_subpage_pool_add_folio_mask(struct mm_subpage_pool *pool, struct folio *folio, @@ -1011,7 +1022,15 @@ int mm_subpage_pool_add_folio_mask(struct mm_subpage_pool *pool, struct folio *f { if (!mask || (mask & ~pool->all_slots)) return -EINVAL; - return pool_add_folio(pool, folio, mask, true, gfp); + return pool_add_folio(pool, folio, mask, true, 1, gfp); +} + +int mm_subpage_pool_add_folio_count(struct mm_subpage_pool *pool, struct folio *folio, + unsigned int count, gfp_t gfp) +{ + if (!count || count > hweight_long(pool->all_slots)) + return -EINVAL; + return pool_add_folio(pool, folio, pool->all_slots, false, count, gfp); } unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool) @@ -1066,7 +1085,7 @@ static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, subpage = ERR_PTR(-ESHUTDOWN); goto out; } - owner = find_available(pool, eligible, false); + owner = find_available(pool, eligible, false, 1); if (!owner) { subpage = ERR_PTR(-EAGAIN); goto out; @@ -1106,13 +1125,13 @@ static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, return subpage; } -/* Reserve one exact-offset batch from one owner, with one folio lock. */ -int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mask, - struct mm_subpage **slots) +/* One owner/folio lock; output remains indexed by the caller's logical mask. */ +static int subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mask, + bool any_offset, struct mm_subpage **slots) { struct mm_subpage_owner *owner; - unsigned long flags, zero = 0; - unsigned int i; + unsigned long flags, zero = 0, available; + unsigned int i, physical, count = hweight_long(mask); bool needs_zero; if (!mask || (mask & ~pool->all_slots)) @@ -1123,7 +1142,8 @@ int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mas spin_unlock_irqrestore(&pool->lock, flags); return -ESHUTDOWN; } - owner = find_available(pool, mask, true); + owner = find_available(pool, any_offset ? pool->all_slots : mask, + !any_offset, any_offset ? count : 1); if (!owner) { spin_unlock_irqrestore(&pool->lock, flags); return -EAGAIN; @@ -1140,8 +1160,11 @@ int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mas spin_unlock_irqrestore(&pool->lock, flags); return -EAGAIN; } + available = ~owner->used & pool->all_slots; for_each_set_bit(i, &mask, SUBPAGES_PER_FOLIO) { - slots[i] = reserve_owner_slot(owner, i, &needs_zero); + physical = any_offset ? __ffs(available) : i; + available &= ~BIT(physical); + slots[i] = reserve_owner_slot(owner, physical, &needs_zero); if (needs_zero) zero |= BIT(i); } @@ -1149,13 +1172,25 @@ int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mas for_each_set_bit(i, &mask, SUBPAGES_PER_FOLIO) { if (zero & BIT(i)) clear_user_subpage_range(&owner->folio->page, - i << pool->shift, 1U << pool->shift); + mm_subpage_offset(slots[i]), 1U << pool->shift); /* Zeroing must precede publication, including reused slots. */ refcount_set_release(&slots[i]->refs, 1); } return 0; } +int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mask, + struct mm_subpage **slots) +{ + return subpage_alloc_mask_locked(pool, mask, false, slots); +} + +int mm_subpage_alloc_any_mask_locked(struct mm_subpage_pool *pool, unsigned long mask, + struct mm_subpage **slots) +{ + return subpage_alloc_mask_locked(pool, mask, true, slots); +} + static struct mm_subpage *subpage_alloc(struct mm_subpage_pool *pool, unsigned long eligible, bool lock_folio) { bool needs_zero; diff --git a/tools/testing/selftests/arm64/user4k/.gitignore b/tools/testing/selftests/arm64/user4k/.gitignore index 86bb27433258e..85e45b6ce0399 100644 --- a/tools/testing/selftests/arm64/user4k/.gitignore +++ b/tools/testing/selftests/arm64/user4k/.gitignore @@ -4,3 +4,5 @@ swap-fixture-init same-quarter file-cow-consumers same-native +first-write +clipped-firstwrite diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index 0c22734f5ad03..dd9d4f233b056 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -12,11 +12,13 @@ TEST_FILES += README.hole-refill README.hole-refill-race README.hole-refill-recl TEST_FILES += compat-script runtime-app.py TEST_GEN_PROGS_EXTENDED += same-quarter file-cow-consumers same-native TEST_FILES += README.any-quarter check-any-quarter.py +TEST_GEN_PROGS_EXTENDED += first-write clipped-firstwrite +TEST_FILES += README.first-write include ../../lib.mk $(OUTPUT)/page-contract: page-contract.c x18-probe.S -$(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool $(OUTPUT)/anon-hole-refill-race $(OUTPUT)/anon-hole-refill-reclaim: LDLIBS += -pthread +$(OUTPUT)/first-write $(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool $(OUTPUT)/anon-hole-refill-race $(OUTPUT)/anon-hole-refill-reclaim: LDLIBS += -pthread # Freestanding ELF32 probes exercise the compat ABI without an armhf sysroot. AARCH32_CC ?= clang --target=arm-linux-gnueabihf diff --git a/tools/testing/selftests/arm64/user4k/README.first-write b/tools/testing/selftests/arm64/user4k/README.first-write new file mode 100644 index 0000000000000..2ec69368a6c70 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.first-write @@ -0,0 +1,28 @@ +First-write and clipped-native-window backing tests +================================================== + +Run only on the audited native16K kernel through the established4K ABI launcher: + cd /var/tmp/owned-test-dir + userpages 4k ./first-write --native-16k + userpages 4k ./clipped-firstwrite --native-16k + +first-write: six fresh child arms (anonymous,shmem,regular file x dense/guarded), +eight threads, absent-PTE direct first writes, full untouched-byte verification, +DONTNEED and distinct-byte same-PTE refault racing, all source-file bytes verified. +Regular-file working directory MUST be non-tmpfs; otherwise that arm SKIPs. +All six must pass for qualification.4MiB useful payload/arm,<=16MiB native private +backing under a wasteful baseline plus source file.40s supervisor/15s child alarm +kills the fixture child/threads on timeout. No global pressure or kernel change. +This content/race fixture does not itself assert physical density. + +clipped-firstwrite: root CAP_SYS_ADMIN pagemap visibility required.11 fresh child +arms cover all valid starts for1/2/3leaf windows,3+1tail fill and full4dense control; +two rounds with DONTNEED between. A single initial byte write must prefault every +neighbor in its clipped batch, proven by pagemap before any neighbor read. Full +zero/content checks then native PFN union enforce density.256groups/arm, +<=8MiB payload backing even on baseline;16MiB reserved VA.1MiB native slack bounds +pool sharding/background COW. Count3 alone allows natural16K/12K packing; mixed3+1 +must fill tails.40s supervisor/8s child alarm, no GPU/reclaim/pressure. + +The fixture metadata selftest (--selftest) checks tags/sorting only. It is not a +kernel-path test and does not replace running both fixtures on the target kernel. diff --git a/tools/testing/selftests/arm64/user4k/clipped-firstwrite.c b/tools/testing/selftests/arm64/user4k/clipped-firstwrite.c new file mode 100644 index 0000000000000..c07cb1a8a980f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/clipped-firstwrite.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real clipped-window batching, zeroing, reuse and physical backing density. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define LEAF 4096UL +#define NATIVE 16384UL +#define GROUPS 256UL +#define STRIDE (4 * NATIVE) +#define PFN_MASK ((UINT64_C(1)<<55)-1) +#define PRESENT (UINT64_C(1)<<63) +#define SWAPPED (UINT64_C(1)<<62) +static volatile sig_atomic_t child; +static void stop(int sig) { if(child>0)kill(child,SIGKILL);_exit(128+sig); } +static bool visible(void) +{ + FILE *f=fopen("/proc/self/status","re");char line[256];unsigned long long caps=0; + if(!f)return false; + while(fgets(line,sizeof(line),f))if(sscanf(line,"CapEff: %llx",&caps)==1)break; + fclose(f);return !!(caps&(1ULL<<21)); +} +static bool entries(int fd,unsigned char *base,unsigned n,uint64_t *out) +{ + if(pread(fd,out,n*8,(uintptr_t)base/LEAF*8)!=(ssize_t)n*8)return false; + for(unsigned i=0;iy)-(x>2; + if(mix){ + p=base+g*STRIDE+2*NATIVE; + if(!check_window(p,1,g+GROUPS,round)||!entries(fd,p,1,map))return 4; + native[used++]=(map[0]&PFN_MASK)>>2; + } + } + qsort(native,used,sizeof(*native),compare);size_t unique=0; + for(size_t i=0;itarget+(1UL<<20)){fprintf(stderr,"FAIL physical density\n");return 5;} + if(!round)for(unsigned g=0;g +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#define LEAF 4096UL +#define LEAVES 1024UL +#define WORKERS 8 +#define TMPFS_MAGIC 0x01021994 + +enum backing { ANONYMOUS, SHMEM, REGULAR }; +static unsigned char *data; +static size_t stride; +static pthread_barrier_t barrier; +static bool race; +static volatile sig_atomic_t running_child; +static void stop_fixture(int sig) +{ + if (running_child > 0) + kill(running_child, SIGKILL); + _exit(128 + sig); +} + +static unsigned char initial(size_t leaf, size_t byte) +{ + return leaf * 17 + byte * 3 + 0x39; +} + +static void *writer(void *arg) +{ + size_t leaf, id = (uintptr_t)arg; + + pthread_barrier_wait(&barrier); + if (race) { + /* Distinct bytes avoid a C data race while faulting the same PTE. */ + for (leaf = 0; leaf < LEAVES; leaf += 3) + data[leaf * stride + id] = initial(leaf, id) ^ 0xa5; + } else { + for (leaf = id; leaf < LEAVES; leaf += WORKERS) + data[leaf * stride] = initial(leaf, 0) ^ 0xa5; + } + return NULL; +} + +static bool verify(enum backing backing) +{ + size_t leaf, byte; + + for (leaf = 0; leaf < LEAVES; leaf++) { + for (byte = 0; byte < LEAF; byte++) { + unsigned char want = backing == ANONYMOUS ? 0 : initial(leaf, byte); + + if (!byte || (race && leaf % 3 == 0 && byte < WORKERS)) + want = initial(leaf, byte) ^ 0xa5; + if (data[leaf * stride + byte] != want) { + ksft_print_msg("leaf=%zu byte=%zu got=%u want=%u\n", leaf, byte, + data[leaf * stride + byte], want); + return false; + } + } + } + return true; +} + +static int exercise(enum backing backing, bool small) +{ + unsigned char buffer[LEAF]; + char name[] = "first-write-XXXXXX"; + pthread_t workers[WORKERS]; + struct statfs fs; + int fd = -1, flags = MAP_PRIVATE; + size_t leaf, byte, span; + unsigned int t; + + stride = small ? 2 * LEAF : LEAF; + span = LEAVES * stride; + if (backing != ANONYMOUS) { + fd = backing == SHMEM ? memfd_create("first-write", MFD_CLOEXEC) : + mkstemp(name); + if (fd < 0) + return 2; + if (backing == REGULAR) + unlink(name); + if (fstatfs(fd, &fs)) + return 2; + if (backing == REGULAR && fs.f_type == TMPFS_MAGIC) { + ksft_print_msg("regular-file arm needs a non-tmpfs working directory\n"); + return KSFT_SKIP; + } + if (ftruncate(fd, span)) + return 2; + for (leaf = 0; leaf < LEAVES; leaf++) { + for (byte = 0; byte < LEAF; byte++) + buffer[byte] = initial(leaf, byte); + if (pwrite(fd, buffer, LEAF, leaf * stride) != LEAF) + return 2; + } + } else { + flags |= MAP_ANONYMOUS; + } + data = mmap(NULL, span, small ? PROT_NONE : PROT_READ | PROT_WRITE, + small ? MAP_PRIVATE | MAP_ANONYMOUS : flags, small ? -1 : fd, 0); + if (data == MAP_FAILED) + return 2; + if (small) + for (leaf = 0; leaf < LEAVES; leaf++) + if (mmap(data + leaf * stride, LEAF, PROT_READ | PROT_WRITE, + flags | MAP_FIXED, fd, backing == ANONYMOUS ? 0 : leaf * stride) == + MAP_FAILED) + return 2; + + /* No reads through the mapping before concurrent direct first writes. */ + if (pthread_barrier_init(&barrier, NULL, WORKERS)) + return 2; + for (t = 0; t < WORKERS; t++) + if (pthread_create(&workers[t], NULL, writer, (void *)(uintptr_t)t)) + return 2; + for (t = 0; t < WORKERS; t++) + if (pthread_join(workers[t], NULL)) + return 2; + if (!verify(backing)) + return 3; + /* Drop selected private slots, leaving the other slots live. */ + for (leaf = 0; leaf < LEAVES; leaf += 3) + if (madvise(data + leaf * stride, LEAF, MADV_DONTNEED)) + return 2; + /* Again no reads before the first writes: exercise abort and slot reuse. */ + race = true; + for (t = 0; t < WORKERS; t++) + if (pthread_create(&workers[t], NULL, writer, (void *)(uintptr_t)t)) + return 2; + for (t = 0; t < WORKERS; t++) + if (pthread_join(workers[t], NULL)) + return 2; + pthread_barrier_destroy(&barrier); + if (!verify(backing)) + return 3; + if (backing != ANONYMOUS) { + for (leaf = 0; leaf < LEAVES; leaf++) { + if (pread(fd, buffer, LEAF, leaf * stride) != LEAF) + return 2; + for (byte = 0; byte < LEAF; byte++) + if (buffer[byte] != initial(leaf, byte)) + return 3; + } + close(fd); + } + munmap(data, span); + return 0; +} + +int main(int argc, char **argv) +{ + static const char * const names[] = { "anonymous", "shmem", "regular-file" }; + unsigned int backing, small; + int status; + pid_t child, result; + + if (argc != 2 || strcmp(argv[1], "--native-16k")) + return 2; + signal(SIGALRM, stop_fixture); + signal(SIGTERM, stop_fixture); + signal(SIGINT, stop_fixture); + ksft_print_header(); + if (getauxval(AT_PAGESZ) != LEAF) + ksft_exit_skip("run under a selected 4K userspace ABI\n"); + ksft_set_plan(6); + fflush(NULL); + alarm(40); + for (backing = ANONYMOUS; backing <= REGULAR; backing++) { + for (small = 0; small <= 1; small++) { + child = fork(); + if (child < 0) + ksft_exit_fail_msg("fork: %s\n", strerror(errno)); + if (!child) { + running_child = 0; + alarm(15); + _exit(exercise(backing, small)); + } + running_child = child; + do { + result = waitpid(child, &status, 0); + } while (result < 0 && errno == EINTR); + running_child = 0; + if (result > 0 && WIFEXITED(status) && WEXITSTATUS(status) == KSFT_SKIP) + ksft_test_result_skip("%s %s first writes\n", names[backing], + small ? "guarded" : "dense"); + else + ksft_test_result(result > 0 && WIFEXITED(status) && + !WEXITSTATUS(status), + "%s %s first writes and slot reuse preserve contents\n", + names[backing], small ? "guarded" : "dense"); + } + } + ksft_finished(); +} From 859e8af15b2eb850c1909bb7f64635f549342e93 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 19:28:15 -0400 Subject: [PATCH 0864/2680] mm: pack pinned-fork subpage copies without changing their charge domain A fork that must copy pinned 4 KiB leaves previously allocated one native 16 KiB folio per leaf. Reserve copies from the child mm's subpage pools, while retaining the source mm for the existing fork allocation charge. Preflight non-linear rmap before taking the page-table locks and copy from the actual source offset after revalidating the parent PTE. Add a bounded real io_uring buffer-pin regression that verifies distinct parent/child native owners, full two-way byte isolation and packed density. On Neo, a 1 MiB copy fell from 4 MiB to 1 MiB of native backing. A held non-root cgroup check verified every payload native PFN retained the correct charge. Retained MM, actual reclaim/readback and GPU controls pass. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 6ec0621cc97679bfa90f005c4e0666b319783842) --- include/linux/mm_subpage.h | 4 + mm/memory.c | 74 ++++++-- mm/user-subpage.c | 13 +- .../testing/selftests/arm64/user4k/.gitignore | 1 + tools/testing/selftests/arm64/user4k/Makefile | 2 + .../selftests/arm64/user4k/README.pinned-fork | 37 ++++ .../selftests/arm64/user4k/pinned-fork.c | 172 ++++++++++++++++++ 7 files changed, 285 insertions(+), 18 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/README.pinned-fork create mode 100644 tools/testing/selftests/arm64/user4k/pinned-fork.c diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h index 910ed2b36bcfd..4a6224f8cc95a 100644 --- a/include/linux/mm_subpage.h +++ b/include/linux/mm_subpage.h @@ -49,6 +49,10 @@ void mm_subpage_pool_close(struct mm_subpage_pool *pool); /* Optional per-mm pool: default policy, one memory node, matching memcg. */ struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address); +/* Fork's child owns the pool before it joins a cgroup; charge the source mm. */ +struct mm_subpage_pool *mm_subpage_cow_pool_get_charged(struct mm_struct *mm, + struct mm_struct *charge_mm, struct vm_area_struct *vma, + unsigned long address); void mm_subpage_cow_pool_exit(struct mm_struct *mm); bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *folio); bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset); diff --git a/mm/memory.c b/mm/memory.c index 53acf12f4319b..fc5cafd2ef47f 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1126,7 +1126,7 @@ static void subpage_add_new_lru(struct folio *folio, struct vm_area_struct *vma, mlock_folio(folio); } -/* The destination slot and its charge/metadata were allocated outside PTLs. */ +/* Destination allocation, folio locking and rmap preflight precede both PTLs. */ static int copy_pinned_subpage(struct vm_area_struct *dst_vma, pte_t *dst_pte, pte_t source_pte, unsigned long addr, int *rss, struct mm_subpage *source, struct mm_subpage **prealloc) @@ -1134,23 +1134,26 @@ static int copy_pinned_subpage(struct vm_area_struct *dst_vma, pte_t *dst_pte, struct mm_subpage *dest = *prealloc; struct folio *new; pte_t pte; + bool fresh; int ret; if (!dest) return -EAGAIN; new = mm_subpage_folio(dest); + VM_BUG_ON_FOLIO(!folio_test_locked(new), new); ret = copy_user_subpage_range(&new->page, mm_subpage_offset(dest), &mm_subpage_folio(source)->page, mm_subpage_offset(source), mm_subpage_size(source)); if (ret) return ret; - /* Fresh, unmapped backing is private to this preallocation. */ - if (WARN_ON_ONCE(!folio_trylock(new))) - return -EBUSY; + fresh = !mm_subpage_anon_root(dest); + mm_subpage_set_exclusive(dest); - ret = mm_subpage_add_new_anon_rmap(dest, dst_vma, addr); + ret = fresh ? mm_subpage_add_new_anon_rmap(dest, dst_vma, addr) : + mm_subpage_add_anon_rmap(dest, dst_vma, addr); if (!ret) { - subpage_add_new_lru(new, dst_vma, 1); + if (fresh) + subpage_add_new_lru(new, dst_vma, 1); rss[MM_ANONPAGES]++; pte = phys_pte_mm(dst_vma->vm_mm, mm_subpage_phys(dest), dst_vma->vm_page_prot); pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma); @@ -1158,8 +1161,9 @@ static int copy_pinned_subpage(struct vm_area_struct *dst_vma, pte_t *dst_pte, pte = pte_mkuffd_wp(pte); set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); *prealloc = NULL; + folio_unlock(new); } - folio_unlock(new); + /* An unconsumed reservation remains locked until the caller drops PTLs. */ return ret; } #endif @@ -1349,7 +1353,8 @@ static struct folio *cow_subpage_speculative_folio(struct mm_struct *mm, /* The returned initialized slot owns a reference and has its folio locked. */ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, - struct mm_subpage *source, bool force_private, bool *pooled) + struct mm_subpage *source, bool force_private, bool *pooled, + struct mm_struct *charge_mm) { struct mm_subpage_pool *pool; struct mm_subpage *slot; @@ -1358,7 +1363,8 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, int err; retry_private: - pool = force_private ? NULL : mm_subpage_cow_pool_get(mm, vma, addr); + pool = force_private ? NULL : + mm_subpage_cow_pool_get_charged(mm, charge_mm, vma, addr); *pooled = pool != NULL; if (!pool) pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); @@ -1375,7 +1381,7 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, break; folio = NULL; if (*pooled) { - folio = cow_subpage_speculative_folio(mm, vma, addr); + folio = cow_subpage_speculative_folio(charge_mm, vma, addr); if (!folio && mm_subpage_cow_wait_busy(pool)) { cond_resched(); continue; @@ -1385,7 +1391,7 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, * Ordinary allocation/charge semantics are now appropriate. */ if (!folio) - folio = folio_prealloc(mm, vma, addr, false); + folio = folio_prealloc(charge_mm, vma, addr, false); if (!folio) { slot = ERR_PTR(-ENOMEM); break; @@ -1424,6 +1430,30 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, return slot; } +/* The child owns the pool; fork retains the source mm allocation charge. */ +static struct mm_subpage *fork_subpage_prealloc(struct vm_area_struct *dst_vma, + struct mm_struct *src_mm, unsigned long addr) +{ + struct mm_subpage *slot; + bool pooled, force_private = false; + int err; + + for (;;) { + slot = cow_subpage_prealloc(dst_vma->vm_mm, dst_vma, addr, + NULL, force_private, &pooled, src_mm); + if (IS_ERR(slot)) + return slot; + err = mm_subpage_prepare_anon_rmap(slot, dst_vma, addr); + if (!err) + return slot; + folio_unlock(mm_subpage_folio(slot)); + mm_subpage_put(slot); + if (!pooled || err != -ENOMEM) + return ERR_PTR(err); + force_private = true; + } +} + static struct mm_subpage *anon_subpage_prealloc(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr) { @@ -1468,6 +1498,7 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, softleaf_t entry = softleaf_mk_none(); struct folio *prealloc = NULL; struct mm_subpage *subprealloc = NULL; + unsigned long subprealloc_addr __maybe_unused = 0; int nr; again: @@ -1508,6 +1539,14 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, do { nr = 1; +#ifdef CONFIG_MM_SUBPAGE + /* A vanished source PTE must not move a prepared slot to another VA. */ + if (subprealloc && addr != subprealloc_addr) { + folio_unlock(mm_subpage_folio(subprealloc)); + mm_subpage_put(subprealloc); + subprealloc = NULL; + } +#endif /* * We are holding two locks at this point - either of them @@ -1561,6 +1600,7 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, break; #ifdef CONFIG_MM_SUBPAGE if (subprealloc) { + folio_unlock(mm_subpage_folio(subprealloc)); mm_subpage_put(subprealloc); subprealloc = NULL; } @@ -1598,7 +1638,8 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, } else if (ret == -EAGAIN) { #ifdef CONFIG_MM_SUBPAGE if (mm_page_size(src_mm) < PAGE_SIZE) { - subprealloc = anon_subpage_prealloc(src_mm, src_vma, addr); + subprealloc_addr = addr; + subprealloc = fork_subpage_prealloc(dst_vma, src_mm, addr); if (IS_ERR(subprealloc)) return PTR_ERR(subprealloc); ret = 0; @@ -1622,8 +1663,10 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, goto again; out: #ifdef CONFIG_MM_SUBPAGE - if (subprealloc) + if (subprealloc) { + folio_unlock(mm_subpage_folio(subprealloc)); mm_subpage_put(subprealloc); + } #endif if (unlikely(prealloc)) folio_put(prealloc); @@ -4696,7 +4739,8 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) goto out; ret = VM_FAULT_OOM; retry_copy: - dest = cow_subpage_prealloc(mm, vma, vmf->address, source, force_private, &pooled); + dest = cow_subpage_prealloc(mm, vma, vmf->address, source, force_private, + &pooled, mm); if (IS_ERR(dest)) { if (PTR_ERR(dest) == -EHWPOISON) ret = VM_FAULT_HWPOISON; @@ -7485,7 +7529,7 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) if (!force_private && !userfaultfd_armed(vma) && (vma->vm_ops->fault == filemap_fault || vma_is_shmem(vma))) { vmf->cow_subpage = cow_subpage_prealloc(vma->vm_mm, vma, - vmf->address, NULL, false, &pooled); + vmf->address, NULL, false, &pooled, vma->vm_mm); if (!IS_ERR(vmf->cow_subpage)) folio_unlock(mm_subpage_folio(vmf->cow_subpage)); } else { diff --git a/mm/user-subpage.c b/mm/user-subpage.c index fd553493b3de4..acfa80848e6e3 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -299,8 +299,9 @@ void mm_subpage_pool_put(struct mm_subpage_pool *pool) * Policy-sensitive, locked, droppable and multi-node faults keep fresh backing. * An mm can move cgroups; old charged slots must not satisfy its new faults. */ -struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, - struct vm_area_struct *vma, unsigned long address) +struct mm_subpage_pool *mm_subpage_cow_pool_get_charged(struct mm_struct *mm, + struct mm_struct *charge_mm, struct vm_area_struct *vma, + unsigned long address) { struct mm_subpage_cow_pools *pools, *new_pools; struct mm_subpage_pool *pool, *new; @@ -334,7 +335,7 @@ struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, else kfree(new_pools); } - memcg = get_mem_cgroup_from_mm(mm); + memcg = get_mem_cgroup_from_mm(charge_mm); /* Acquire the immutable charge domain published with this bucket. */ pool = smp_load_acquire(&pools->pool[bucket]); if (!pool) { @@ -375,6 +376,12 @@ struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, return pool; } +struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long address) +{ + return mm_subpage_cow_pool_get_charged(mm, mm, vma, address); +} + bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *folio) { return pool->cow_memcg == folio_memcg(folio) && pool->cow_nid == folio_nid(folio); diff --git a/tools/testing/selftests/arm64/user4k/.gitignore b/tools/testing/selftests/arm64/user4k/.gitignore index 85e45b6ce0399..a5b225dbaf61d 100644 --- a/tools/testing/selftests/arm64/user4k/.gitignore +++ b/tools/testing/selftests/arm64/user4k/.gitignore @@ -6,3 +6,4 @@ file-cow-consumers same-native first-write clipped-firstwrite +pinned-fork diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index dd9d4f233b056..a6e4854f454c2 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -14,6 +14,8 @@ TEST_GEN_PROGS_EXTENDED += same-quarter file-cow-consumers same-native TEST_FILES += README.any-quarter check-any-quarter.py TEST_GEN_PROGS_EXTENDED += first-write clipped-firstwrite TEST_FILES += README.first-write +TEST_GEN_PROGS_EXTENDED += pinned-fork +TEST_FILES += README.pinned-fork include ../../lib.mk $(OUTPUT)/page-contract: page-contract.c x18-probe.S diff --git a/tools/testing/selftests/arm64/user4k/README.pinned-fork b/tools/testing/selftests/arm64/user4k/README.pinned-fork new file mode 100644 index 0000000000000..129adde8b005c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.pinned-fork @@ -0,0 +1,37 @@ +Pinned anonymous fork density and pin isolation +============================================= + +Build pinned-fork with the other arm64/user4k tests. Run as root through the +established 4K userspace launcher on a native16K kernel: + + userpages 4k ./pinned-fork --native-16k --expect-packed + +The fixture initializes one native-aligned1MiB anonymous range, registers real +writable long-term io_uring buffer pins, then forks while the parent retains the +registration. It requires every copied child native PFN to be disjoint from the +pinned parent's native PFNs. It checks every byte before and after distinct +parent/child writes; registration and PFN disjointness, not mlock or VmPin alone, +establish copy-path coverage. Kernel/pin-policy limitations return77 UNEXERCISED. + +--expect-packed requires child backing <=1.25MiB for1MiB copied data, with bounded +pool-shard slack. Without this option the fixture also accepts the old4MiB +backing so a baseline can record both content and density separately. Successful +contents alone do not establish packed density. --selftest checks metadata PFN +grouping only, with no dynamic pin/fork qualification. + +Optional --hold reports the parent PID, child PID and range base after fork PFN +verification, then waits for the byte 'c' on stdin. A controller can hold both +processes in one fresh uncapped non-root cgroup and verify exact payload charges: +read each target's pagemap in4KiB units; normalize PFN>>2 to native16KiB PFN; +read /proc/kpagecgroup at native_PFN*8 and require each returned inode equals +stat(the fixture cgroup).st_ino. Re-read pagemap to reject migration during the +sample. This directly catches incorrectly root-charged fork copies; total +memory.current includes unrelated fixture code/stack/table costs and is not an +exact payload charge oracle. Verify both PIDs belong to the same owned cgroup, +record memory.stat/status, and release promptly (within3seconds). + +Parent15s/child10s alarms bound the protocol; parent exit kills/reaps the child +and the child uses PDEATHSIG. Payload native backing is at most5MiB on the old +path. No forced reclaim, memory.max/high changes, GPU operations, or swap-area +changes are used. Controllers should additionally enforce their own process-group +or service-unit lifetime and preserve logs on any failure. diff --git a/tools/testing/selftests/arm64/user4k/pinned-fork.c b/tools/testing/selftests/arm64/user4k/pinned-fork.c new file mode 100644 index 0000000000000..ec91136af8de4 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/pinned-fork.c @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define LEAF 4096UL +#define NATIVE 16384UL +#define SIZE (1024UL * 1024) +#define COUNT (SIZE / LEAF) +#define PFNMASK ((1ULL << 55) - 1) +static pid_t child = -1; +static uint64_t parent_pfn[COUNT], child_pfn[COUNT]; +static void die(const char *s) { perror(s); exit(1); } +static void cleanup(void) +{ + if (child > 0) { + kill(child, SIGKILL); + while (waitpid(child, NULL, 0) < 0 && errno == EINTR) {} + } +} +static void timeout(int sig) +{ + (void)sig; + if (child > 0) kill(child, SIGKILL); + _exit(124); +} +static void transfer(int fd, void *buffer, size_t len, int writing) +{ + unsigned char *p = buffer; + while (len) { + ssize_t n = writing ? write(fd, p, len) : read(fd, p, len); + if (n < 0 && errno == EINTR) continue; + if (n <= 0) die(writing ? "write protocol" : "read protocol"); + p += n; len -= n; + } +} +static unsigned char pattern(size_t i) { return (unsigned char)(1 + (i * 17 + i / LEAF) % 251); } +static void check(unsigned char *p, int mode) +{ + for (size_t i = 0; i < SIZE; i++) { + unsigned char expected = pattern(i); + if ((mode == 1 && i % LEAF == 0) || (mode == 2 && i % LEAF == 1)) expected ^= 0x80; + if (p[i] != expected) { + fprintf(stderr, "FAIL bytes mode=%d offset=%zu expected=%u actual=%u\n", mode, i, expected, p[i]); + exit(1); + } + } +} +static void pfns(unsigned char *p, uint64_t *out) +{ + int fd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + if (fd < 0) die("pagemap open"); + for (size_t i = 0; i < COUNT; i++) { + uint64_t e; + if (pread(fd, &e, sizeof(e), (((uintptr_t)p / LEAF) + i) * sizeof(e)) != sizeof(e)) die("pagemap read"); + if (!(e >> 63) || (e & (1ULL << 62)) || !(e & PFNMASK)) { + fprintf(stderr, "FAIL pagemap leaf=%zu entry=%016llx (root PFNs required)\n", i, (unsigned long long)e); exit(1); + } + out[i] = e & PFNMASK; + } + close(fd); +} +static size_t native_count(uint64_t *pf) +{ + size_t n = 0; + for (size_t i = 0; i < COUNT; i++) { + size_t j; + for (j = 0; j < i; j++) if (pf[j] / 4 == pf[i] / 4) break; + if (j == i) n++; + } + return n; +} +int main(int argc, char **argv) +{ + if (argc == 2 && !strcmp(argv[1], "--selftest")) { + for (size_t i = 0; i < COUNT; i++) parent_pfn[i] = 4 + i; + if (native_count(parent_pfn) != COUNT / 4) return 1; + for (size_t i = 0; i < COUNT; i++) parent_pfn[i] = 4 + 4 * i; + if (native_count(parent_pfn) != COUNT) return 1; + puts("PASS native PFN grouping selftest (no pin/fork coverage)"); return 0; + } + int packed = 0, hold = 0; + if (argc < 2 || strcmp(argv[1], "--native-16k") || getauxval(AT_PAGESZ) != LEAF) return 77; + for (int i = 2; i < argc; i++) { + if (!strcmp(argv[i], "--expect-packed")) packed = 1; + else if (!strcmp(argv[i], "--hold")) hold = 1; + else return 77; + } + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGALRM, timeout); signal(SIGPIPE, SIG_IGN); alarm(15); atexit(cleanup); + unsigned char *raw = mmap(NULL, SIZE + NATIVE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (raw == MAP_FAILED) die("mmap"); + unsigned char *p = (void *)(((uintptr_t)raw + NATIVE - 1) & ~(NATIVE - 1)); + for (size_t i = 0; i < SIZE; i++) p[i] = pattern(i); + check(p, 0); pfns(p, parent_pfn); + if (native_count(parent_pfn) * NATIVE != SIZE) die("initial dense native backing"); + struct io_uring_params params = {0}; + int ring = syscall(__NR_io_uring_setup, 2, ¶ms); + if (ring < 0) { perror("UNEXERCISED io_uring_setup"); return 77; } + struct iovec iov = { .iov_base = p, .iov_len = SIZE }; + if (syscall(__NR_io_uring_register, ring, IORING_REGISTER_BUFFERS, &iov, 1)) { + perror("UNEXERCISED register real long-term pins"); close(ring); return 77; + } + /* Long-term pin setup can migrate backing; capture the pinned identity. */ + pfns(p, parent_pfn); check(p, 0); + if (native_count(parent_pfn) * NATIVE != SIZE) die("pinned dense native backing"); + int to_child[2], from_child[2]; + if (pipe(to_child) || pipe(from_child)) die("pipe"); + pid_t parent = getpid(); + child = fork(); + if (child < 0) die("fork"); + if (!child) { + prctl(PR_SET_PDEATHSIG, SIGKILL); + if (getppid() != parent) _exit(1); + alarm(10); close(ring); close(to_child[1]); close(from_child[0]); + pfns(p, child_pfn); check(p, 0); + transfer(from_child[1], child_pfn, sizeof(child_pfn), 1); + char token; transfer(to_child[0], &token, 1, 0); + if (token != 'w') die("child token"); + check(p, 0); + for (size_t i = 0; i < COUNT; i++) p[i * LEAF + 1] ^= 0x80; + check(p, 2); + token = 'c'; transfer(from_child[1], &token, 1, 1); + transfer(to_child[0], &token, 1, 0); + if (token != 'e') die("child exit token"); + check(p, 2); _exit(0); + } + close(to_child[0]); close(from_child[1]); + transfer(from_child[0], child_pfn, sizeof(child_pfn), 0); + for (size_t i = 0; i < COUNT; i++) + for (size_t j = 0; j < COUNT; j++) + if (child_pfn[i] / 4 == parent_pfn[j] / 4) die("child copy overlaps pinned parent backing"); + printf("METADATA real_registered_pin=1 parent_native=%zu child_native=%zu copied_leaves=%lu native_overlap=0\n", native_count(parent_pfn) * NATIVE, native_count(child_pfn) * NATIVE, COUNT); + if (hold) { + printf("HOLD parent=%d child=%d base=%lx bytes=%lu (write c to continue)\n", getpid(), child, (unsigned long)p, SIZE); + char ack; transfer(STDIN_FILENO, &ack, 1, 0); + if (ack != 'c') die("controller hold token"); + } + size_t child_bytes = native_count(child_pfn) * NATIVE; + if (child_bytes < SIZE || child_bytes > COUNT * NATIVE || + (packed && child_bytes > SIZE + 256 * 1024)) die("child density bound"); + for (size_t i = 0; i < COUNT; i++) p[i * LEAF] ^= 0x80; + check(p, 1); + char token = 'w'; transfer(to_child[1], &token, 1, 1); + transfer(from_child[0], &token, 1, 0); + if (token != 'c') die("parent token"); + check(p, 1); + token = 'e'; transfer(to_child[1], &token, 1, 1); + int status; + if (waitpid(child, &status, 0) != child) die("waitpid"); + child = -1; + if (!WIFEXITED(status) || WEXITSTATUS(status)) die("child failed"); + if (syscall(__NR_io_uring_register, ring, IORING_UNREGISTER_BUFFERS, NULL, 0)) die("unregister"); + close(ring); close(to_child[1]); close(from_child[0]); + if (munmap(raw, SIZE + NATIVE)) die("munmap"); + puts("PASS real pin/fork copy and full-byte two-way isolation; density is separately reported"); + return 0; +} From 8d17974974658016a1481d9a64e28d8cfcd1a5b4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 19:29:34 -0400 Subject: [PATCH 0865/2680] arm64: reuse cached table fragments after allocation races Concurrent small-table allocators can all miss the fragment cache before allocating a native owner. Recheck the cache after construction and consume its reserved fragment if another allocator supplied it. Destroy the unused fresh owner instead of returning one table and stranding its other slots. Add bounded serial and concurrent backing fixtures with separate logical VmPTE and physical-charge checkpoints. On Neo the same concurrent fixture fell from 5632 to 4192 KiB physical pagetables for approximately 4160 KiB logical tables. All contents, retained MM, reclaim/readback and GPU controls pass. Issued-fragment retention after unmap is a separate unresolved case. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit fd8b40cc03751f3d4128be7d09c855ae1c7e5498) --- arch/arm64/mm/user4k-pgalloc.c | 26 ++- .../testing/selftests/arm64/user4k/.gitignore | 2 + tools/testing/selftests/arm64/user4k/Makefile | 4 +- .../arm64/user4k/README.pagetable-backing | 33 ++++ .../arm64/user4k/pagetable-concurrent.c | 161 ++++++++++++++++++ .../arm64/user4k/pagetable-retention.c | 118 +++++++++++++ 6 files changed, 339 insertions(+), 5 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/README.pagetable-backing create mode 100644 tools/testing/selftests/arm64/user4k/pagetable-concurrent.c create mode 100644 tools/testing/selftests/arm64/user4k/pagetable-retention.c diff --git a/arch/arm64/mm/user4k-pgalloc.c b/arch/arm64/mm/user4k-pgalloc.c index 0bcaab5c3a8d2..21d27c1656f76 100644 --- a/arch/arm64/mm/user4k-pgalloc.c +++ b/arch/arm64/mm/user4k-pgalloc.c @@ -98,12 +98,30 @@ void *arm64_user_pt_alloc_granule(struct mm_struct *mm, atomic_set(&pt->pt_frag_refcount, 1); table = ptdesc_address(pt); spin_lock(&mm->page_table_lock); - if (nr_fragments > 1 && !mm->context.user4k_pt_frag[level]) { - atomic_set(&pt->pt_frag_refcount, nr_fragments); - mm->context.user4k_pt_frag[level] = table + frag_size; + if (nr_fragments > 1) { + next = mm->context.user4k_pt_frag[level]; + if (next) { + /* Another allocator supplied backing while we constructed + * a table owner. Consume its unused fragment rather than + * stranding the rest of our new native page. + */ + if (virt_to_ptdesc(next)->pt_frag_shift != shift) { + spin_unlock(&mm->page_table_lock); + fragment_owner_free(pt); + return NULL; + } + table = next; + next += frag_size; + mm->context.user4k_pt_frag[level] = + ((unsigned long)next & ~PAGE_MASK) ? next : NULL; + } else { + atomic_set(&pt->pt_frag_refcount, nr_fragments); + mm->context.user4k_pt_frag[level] = table + frag_size; + } } - /* A racing allocation may own the cache; this page then has one user. */ spin_unlock(&mm->page_table_lock); + if (table != ptdesc_address(pt)) + fragment_owner_free(pt); return table; } diff --git a/tools/testing/selftests/arm64/user4k/.gitignore b/tools/testing/selftests/arm64/user4k/.gitignore index a5b225dbaf61d..3c4c8c86eefe2 100644 --- a/tools/testing/selftests/arm64/user4k/.gitignore +++ b/tools/testing/selftests/arm64/user4k/.gitignore @@ -7,3 +7,5 @@ same-native first-write clipped-firstwrite pinned-fork +pagetable-retention +pagetable-concurrent diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index a6e4854f454c2..d306deb1f0054 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -16,11 +16,13 @@ TEST_GEN_PROGS_EXTENDED += first-write clipped-firstwrite TEST_FILES += README.first-write TEST_GEN_PROGS_EXTENDED += pinned-fork TEST_FILES += README.pinned-fork +TEST_GEN_PROGS_EXTENDED += pagetable-retention pagetable-concurrent +TEST_FILES += README.pagetable-backing include ../../lib.mk $(OUTPUT)/page-contract: page-contract.c x18-probe.S -$(OUTPUT)/first-write $(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool $(OUTPUT)/anon-hole-refill-race $(OUTPUT)/anon-hole-refill-reclaim: LDLIBS += -pthread +$(OUTPUT)/pagetable-concurrent $(OUTPUT)/first-write $(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool $(OUTPUT)/anon-hole-refill-race $(OUTPUT)/anon-hole-refill-reclaim: LDLIBS += -pthread # Freestanding ELF32 probes exercise the compat ABI without an armhf sysroot. AARCH32_CC ?= clang --target=arm-linux-gnueabihf diff --git a/tools/testing/selftests/arm64/user4k/README.pagetable-backing b/tools/testing/selftests/arm64/user4k/README.pagetable-backing new file mode 100644 index 0000000000000..2c6b16b17465a --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.pagetable-backing @@ -0,0 +1,33 @@ +Page-table physical backing and fragment retention +================================================== + +These bounded fixtures reserve 2 GiB of virtual address space and write one +4 KiB leaf in each of 1,024 distinct 2 MiB table windows. Native payload +backing is at most 16 MiB, with approximately 4 MiB of logical PTE tables. +The concurrent variant uses 32 threads, 32 synchronized fault rounds and +128 KiB thread stacks. Both verify all payload bytes across six checkpoints: +baseline, full, quarter_survivors, refilled, quarter_survivors_again, unmapped. + +Run through the 4 KiB ABI launcher on a native 16 KiB kernel. Each JSON +checkpoint requires one byte 'c' on stdin before continuing. For contents +only, with no physical charge observation: + + yes c | userpages 4k ./pagetable-retention --native-16k + yes c | userpages 4k ./pagetable-concurrent --native-16k + +For physical accounting, run each fixture alone in a fresh uncapped cgroup. +At each checkpoint verify the fixture is the only member, record memory.stat +pagetables and memory.swap.current, and sample at 0, 1 and 3 seconds before +acknowledging. Record logical VmPTE separately: it is not physical backing. +Use an external service lifetime below the fixture's 60-second alarm and +verify all processes exit. Do not change memory.max/high or force reclaim. + +Compare the same complete fixture across kernels. Concurrent thread stacks +and upper-level tables contribute to the measurements; do not subtract them +as though they were pure payload. Deferred table releases can occur between +samples. Contents PASS does not assert a density or performance result. + +The allocation-race fix reduces initial concurrent physical backing. It does +not reclaim partially occupied table owners or reuse issued fragments after +unmap. Retention/refill checkpoints deliberately preserve that separate +coverage gap rather than declaring it fixed. diff --git a/tools/testing/selftests/arm64/user4k/pagetable-concurrent.c b/tools/testing/selftests/arm64/user4k/pagetable-concurrent.c new file mode 100644 index 0000000000000..2573a8882e165 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/pagetable-concurrent.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define LEAF 4096UL +#define WINDOW (2UL * 1024 * 1024) +#define WINDOWS 1024UL +#define LENGTH (WINDOWS * WINDOW) + +static void fail(const char *why) +{ + perror(why); + exit(1); +} + +static unsigned char tag(size_t i) +{ + return (unsigned char)(1 + i % 251); +} + +static void checkpoint(const char *phase) +{ + FILE *f = fopen("/proc/self/status", "re"); + char line[256], ack; + unsigned long pte = 0; + if (!f) + fail("status"); + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmPTE: %lu kB", &pte) == 1) + break; + fclose(f); + printf("{\"phase\":\"%s\",\"pid\":%d,\"VmPTE_bytes\":%lu}\n", + phase, getpid(), pte * 1024); + if (read(STDIN_FILENO, &ack, 1) != 1 || ack != 'c') + fail("controller ack"); +} + +static void touch_serial(unsigned char *base, int missing_only) +{ + for (size_t i = 0; i < WINDOWS; i++) { + if (missing_only && !(i % 4)) + continue; + if (missing_only) { + void *p = mmap(base + i * WINDOW, WINDOW, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, + -1, 0); + if (p != base + i * WINDOW) + fail("refill mmap"); + } + memset(base + i * WINDOW, tag(i), LEAF); + } +} + +#define WORKERS 32 +static pthread_barrier_t step; +static atomic_int start_workers; +struct worker { unsigned char *base; unsigned id; }; +static void *fault_worker(void *arg) +{ + struct worker *w = arg; + int start; + do { start = atomic_load_explicit(&start_workers, memory_order_acquire); + if (!start) sched_yield(); } while (!start); + if (start != 1) return NULL; + for (unsigned round = 0; round < WINDOWS / WORKERS; round++) { + int ret = pthread_barrier_wait(&step); + if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) abort(); + size_t i = round * WORKERS + w->id; + memset(w->base + i * WINDOW, tag(i), LEAF); + } + return NULL; +} +static void touch(unsigned char *base, int missing_only) +{ + if (missing_only) { touch_serial(base, missing_only); return; } + pthread_t threads[WORKERS]; + struct worker args[WORKERS]; + pthread_attr_t attr; + if (pthread_attr_init(&attr) || pthread_attr_setstacksize(&attr, 128 * 1024) || + pthread_barrier_init(&step, NULL, WORKERS)) fail("thread setup"); + unsigned made = 0; + for (; made < WORKERS; made++) { + args[made] = (struct worker){.base = base, .id = made}; + int err = pthread_create(&threads[made], &attr, fault_worker, &args[made]); + if (err) { errno = err; break; } + } + pthread_attr_destroy(&attr); + atomic_store_explicit(&start_workers, made == WORKERS ? 1 : 2, memory_order_release); + for (unsigned i = 0; i < made; i++) if (pthread_join(threads[i], NULL)) abort(); + pthread_barrier_destroy(&step); + if (made != WORKERS) fail("pthread_create"); +} + +static void verify(unsigned char *base, int sparse) +{ + for (size_t i = 0; i < WINDOWS; i++) { + if (sparse && i % 4) + continue; + for (size_t j = 0; j < LEAF; j++) + if (base[i * WINDOW + j] != tag(i)) { + fprintf(stderr, "FAIL content window=%zu byte=%zu\n", i, j); + exit(1); + } + } +} + +static void thin(unsigned char *base) +{ + for (size_t i = 0; i < WINDOWS; i++) + if (i % 4 && munmap(base + i * WINDOW, WINDOW)) + fail("thin munmap"); + verify(base, 1); +} + +int main(int argc, char **argv) +{ + if (argc != 2 || strcmp(argv[1], "--native-16k") || + getauxval(AT_PAGESZ) != LEAF) { + fputs("Requires established 4K launcher on native16K kernel\n", stderr); + return 77; + } + setvbuf(stdout, NULL, _IONBF, 0); + alarm(60); + checkpoint("baseline"); + unsigned char *raw = mmap(NULL, LENGTH + WINDOW, + PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (raw == MAP_FAILED) + fail("reserve mmap"); + unsigned char *base = (void *)(((uintptr_t)raw + WINDOW - 1) & ~(WINDOW - 1)); + size_t head = base - raw, tail = WINDOW - head; + if (head && munmap(raw, head)) + fail("trim head"); + if (tail && munmap(base + LENGTH, tail)) + fail("trim tail"); + touch(base, 0); + verify(base, 0); + checkpoint("full"); + thin(base); + checkpoint("quarter_survivors"); + touch(base, 1); + verify(base, 0); + checkpoint("refilled"); + thin(base); + checkpoint("quarter_survivors_again"); + if (munmap(base, LENGTH)) + fail("final munmap"); + checkpoint("unmapped"); + puts("PASS contents, unmap and refill; concurrent firstfaults, accounting is a separate observation"); + return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/pagetable-retention.c b/tools/testing/selftests/arm64/user4k/pagetable-retention.c new file mode 100644 index 0000000000000..09f18c5424ef8 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/pagetable-retention.c @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +#define LEAF 4096UL +#define WINDOW (2UL * 1024 * 1024) +#define WINDOWS 1024UL +#define LENGTH (WINDOWS * WINDOW) + +static void fail(const char *why) +{ + perror(why); + exit(1); +} + +static unsigned char tag(size_t i) +{ + return (unsigned char)(1 + i % 251); +} + +static void checkpoint(const char *phase) +{ + FILE *f = fopen("/proc/self/status", "re"); + char line[256], ack; + unsigned long pte = 0; + if (!f) + fail("status"); + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmPTE: %lu kB", &pte) == 1) + break; + fclose(f); + printf("{\"phase\":\"%s\",\"pid\":%d,\"VmPTE_bytes\":%lu}\n", + phase, getpid(), pte * 1024); + if (read(STDIN_FILENO, &ack, 1) != 1 || ack != 'c') + fail("controller ack"); +} + +static void touch(unsigned char *base, int missing_only) +{ + for (size_t i = 0; i < WINDOWS; i++) { + if (missing_only && !(i % 4)) + continue; + if (missing_only) { + void *p = mmap(base + i * WINDOW, WINDOW, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, + -1, 0); + if (p != base + i * WINDOW) + fail("refill mmap"); + } + memset(base + i * WINDOW, tag(i), LEAF); + } +} + +static void verify(unsigned char *base, int sparse) +{ + for (size_t i = 0; i < WINDOWS; i++) { + if (sparse && i % 4) + continue; + for (size_t j = 0; j < LEAF; j++) + if (base[i * WINDOW + j] != tag(i)) { + fprintf(stderr, "FAIL content window=%zu byte=%zu\n", i, j); + exit(1); + } + } +} + +static void thin(unsigned char *base) +{ + for (size_t i = 0; i < WINDOWS; i++) + if (i % 4 && munmap(base + i * WINDOW, WINDOW)) + fail("thin munmap"); + verify(base, 1); +} + +int main(int argc, char **argv) +{ + if (argc != 2 || strcmp(argv[1], "--native-16k") || + getauxval(AT_PAGESZ) != LEAF) { + fputs("Requires established 4K launcher on native16K kernel\n", stderr); + return 77; + } + setvbuf(stdout, NULL, _IONBF, 0); + alarm(60); + checkpoint("baseline"); + unsigned char *raw = mmap(NULL, LENGTH + WINDOW, + PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (raw == MAP_FAILED) + fail("reserve mmap"); + unsigned char *base = (void *)(((uintptr_t)raw + WINDOW - 1) & ~(WINDOW - 1)); + size_t head = base - raw, tail = WINDOW - head; + if (head && munmap(raw, head)) + fail("trim head"); + if (tail && munmap(base + LENGTH, tail)) + fail("trim tail"); + touch(base, 0); + verify(base, 0); + checkpoint("full"); + thin(base); + checkpoint("quarter_survivors"); + touch(base, 1); + verify(base, 0); + checkpoint("refilled"); + thin(base); + checkpoint("quarter_survivors_again"); + if (munmap(base, LENGTH)) + fail("final munmap"); + checkpoint("unmapped"); + puts("PASS contents, unmap and refill; accounting is a separate observation"); + return 0; +} From 2e1e9f41b067f13c8ff3b46bdae44cefc2295153 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 20:29:26 -0400 Subject: [PATCH 0866/2680] arm64: reuse retired small page-table fragments Keep an independent per-mm, per-level pool of native table owners so holes left by unmap can satisfy subsequent small-table allocations. Retain issued references until RCU retirement, clear dirty slots only when reused, and release empty owners. Pool references outlive mm teardown for callbacks. On Neo, serial refill physical tables fall from 7,421,952 to 4,276,224 bytes; concurrent refill falls from 6,275,072 to 4,276,224. Contents, teardown, pinned-copy charges, repeated MM, actual swap readback and GPU controls pass. Host allocator checks cover native 4K/16K/64K primitives; maintained KUnit code compiles but was not enabled in the hardware kernel. Seven-round operation timing has overlapping ranges and about 1% higher medians; no zero-overhead claim. Live-table compaction remains separate. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e08fea2abe7ea51544718e47d1f14706ce4c630e) --- arch/arm64/include/asm/mmu.h | 2 +- arch/arm64/mm/user4k-pgalloc.c | 282 +++++++++++++----- arch/arm64/mm/user4k-test.c | 160 +++------- .../arm64/user4k/README.pagetable-backing | 20 +- 4 files changed, 255 insertions(+), 209 deletions(-) diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index 15943bec53392..da82ed5f0c41f 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -39,7 +39,7 @@ typedef struct { unsigned long flags; u8 pkey_allocation_map; #ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL - /* Unissued PTE/PMD/PUD fragments; protected by page_table_lock. */ + /* PTE/PMD/PUD fragment pools; initialized before atomic publication. */ void *user4k_pt_frag[3]; #endif } mm_context_t; diff --git a/arch/arm64/mm/user4k-pgalloc.c b/arch/arm64/mm/user4k-pgalloc.c index 21d27c1656f76..7a3e34205760c 100644 --- a/arch/arm64/mm/user4k-pgalloc.c +++ b/arch/arm64/mm/user4k-pgalloc.c @@ -1,20 +1,38 @@ // SPDX-License-Identifier: GPL-2.0-only -/* - * Hardware tables have immutable granule/owner metadata. Smaller tables share - * a native page; larger tables own a compound allocation. Only unissued slots - * in a shared native page are cached by the mm. - */ +/* Smaller hardware tables share native owners, including safely retired slots. */ #include #include #include #include +#include #include +struct fragment_pool { + spinlock_t lock; + refcount_t refs; + struct list_head available; + unsigned int shift; + bool closed; +}; + +struct fragment_slot { + struct rcu_head rcu; + void *table; +}; + +struct fragment_owner { + struct fragment_pool *pool; + struct ptdesc *pt; + struct list_head available; + unsigned long free_mask; + unsigned long dirty_mask; + struct fragment_slot slots[]; +}; + unsigned int arm64_ptep_page_shift(pte_t *ptep) { struct ptdesc *pt = virt_to_ptdesc(ptep); - /* Table owners retain geometry through their last RCU reader. */ return test_bit(PT_fragmented, &pt->pt_flags.f) ? pt->pt_frag_shift : PAGE_SHIFT; } EXPORT_SYMBOL_GPL(arm64_ptep_page_shift); @@ -30,49 +48,47 @@ static void fragment_owner_free_rcu(struct rcu_head *rcu) fragment_owner_free(container_of(rcu, struct ptdesc, pt_rcu_head)); } -static void fragment_put(struct ptdesc *pt, unsigned int nr) +static void fragment_pool_put(struct fragment_pool *pool) { - if (!atomic_sub_and_test(nr, &pt->pt_frag_refcount)) - return; - if (test_and_clear_bit(PT_frag_defer, &pt->pt_flags.f)) - call_rcu(&pt->pt_rcu_head, fragment_owner_free_rcu); - else - fragment_owner_free(pt); + if (refcount_dec_and_test(&pool->refs)) { + WARN_ON_ONCE(!list_empty(&pool->available)); + kfree(pool); + } } -void *arm64_user_pt_alloc_granule(struct mm_struct *mm, - enum arm64_user4k_pt_level level, unsigned int shift) +/* Active table allocation already holds the mm alive. Its cache reference + * remains until destroy_context, after all allocators have stopped. Owners + * independently retain the pool for callbacks after that point. + */ +static struct fragment_pool *fragment_pool_get(struct mm_struct *mm, + enum arm64_user4k_pt_level level, unsigned int shift) { - struct ptdesc *pt; - void *table, *next; - bool constructed; - unsigned int frag_size, nr_fragments, order; + struct fragment_pool *pool, *new; - if (shift != 12 && shift != 14 && shift != 16) - return NULL; - frag_size = 1U << shift; - order = shift > PAGE_SHIFT ? shift - PAGE_SHIFT : 0; - nr_fragments = (PAGE_SIZE << order) >> shift; - if (WARN_ON_ONCE((unsigned int)level >= ARRAY_SIZE(mm->context.user4k_pt_frag))) + pool = smp_load_acquire(&mm->context.user4k_pt_frag[level]); + if (pool) + return pool->shift == shift ? pool : NULL; + new = kzalloc(sizeof(*new), GFP_KERNEL | __GFP_ACCOUNT); + if (!new) return NULL; + spin_lock_init(&new->lock); + refcount_set(&new->refs, 1); + INIT_LIST_HEAD(&new->available); + new->shift = shift; + pool = cmpxchg(&mm->context.user4k_pt_frag[level], NULL, new); + if (!pool) + return new; + fragment_pool_put(new); + return pool->shift == shift ? pool : NULL; +} - spin_lock(&mm->page_table_lock); - table = mm->context.user4k_pt_frag[level]; - if (table) { - /* An mm's geometry is immutable; never consume another granule. */ - if (virt_to_ptdesc(table)->pt_frag_shift != shift) { - spin_unlock(&mm->page_table_lock); - return NULL; - } - next = table + frag_size; - mm->context.user4k_pt_frag[level] = - ((unsigned long)next & ~PAGE_MASK) ? next : NULL; - } - spin_unlock(&mm->page_table_lock); - if (table) - return table; +static struct ptdesc *fragment_pt_alloc(struct mm_struct *mm, + enum arm64_user4k_pt_level level, unsigned int shift) +{ + struct ptdesc *pt; + bool constructed; - pt = pagetable_alloc(GFP_PGTABLE_USER, order); + pt = pagetable_alloc(GFP_PGTABLE_USER, shift > PAGE_SHIFT ? shift - PAGE_SHIFT : 0); if (!pt) return NULL; switch (level) { @@ -96,32 +112,95 @@ void *arm64_user_pt_alloc_granule(struct mm_struct *mm, pt->pt_frag_shift = shift; set_bit(PT_fragmented, &pt->pt_flags.f); atomic_set(&pt->pt_frag_refcount, 1); - table = ptdesc_address(pt); - spin_lock(&mm->page_table_lock); - if (nr_fragments > 1) { - next = mm->context.user4k_pt_frag[level]; - if (next) { - /* Another allocator supplied backing while we constructed - * a table owner. Consume its unused fragment rather than - * stranding the rest of our new native page. - */ - if (virt_to_ptdesc(next)->pt_frag_shift != shift) { - spin_unlock(&mm->page_table_lock); - fragment_owner_free(pt); - return NULL; - } - table = next; - next += frag_size; - mm->context.user4k_pt_frag[level] = - ((unsigned long)next & ~PAGE_MASK) ? next : NULL; - } else { - atomic_set(&pt->pt_frag_refcount, nr_fragments); - mm->context.user4k_pt_frag[level] = table + frag_size; - } + return pt; +} + +/* The caller holds pool->lock; its active mm retains the pool. */ +static void *fragment_take(struct fragment_pool *pool, bool *zero) +{ + struct fragment_owner *owner; + unsigned int slot; + + if (pool->closed || list_empty(&pool->available)) + return NULL; + owner = list_first_entry(&pool->available, struct fragment_owner, available); + slot = __ffs(owner->free_mask); + owner->free_mask &= ~BIT(slot); + *zero = owner->dirty_mask & BIT(slot); + owner->dirty_mask &= ~BIT(slot); + atomic_inc(&owner->pt->pt_frag_refcount); + if (!owner->free_mask) + list_del_init(&owner->available); + return owner->slots[slot].table; +} + +void *arm64_user_pt_alloc_granule(struct mm_struct *mm, + enum arm64_user4k_pt_level level, unsigned int shift) +{ + struct fragment_pool *pool; + struct fragment_owner *owner; + struct ptdesc *pt; + void *table; + unsigned int nr, i; + unsigned long flags; + bool zero = false; + + if (shift != 12 && shift != 14 && shift != 16) + return NULL; + if (WARN_ON_ONCE((unsigned int)level >= ARRAY_SIZE(mm->context.user4k_pt_frag))) + return NULL; + if (shift >= PAGE_SHIFT) { + pt = fragment_pt_alloc(mm, level, shift); + return pt ? ptdesc_address(pt) : NULL; + } + pool = fragment_pool_get(mm, level, shift); + if (!pool) + return NULL; + spin_lock_irqsave(&pool->lock, flags); + table = fragment_take(pool, &zero); + spin_unlock_irqrestore(&pool->lock, flags); + if (table) + goto out; + + nr = PAGE_SIZE >> shift; + owner = kzalloc(struct_size(owner, slots, nr), GFP_KERNEL | __GFP_ACCOUNT); + if (!owner) + goto out; + pt = fragment_pt_alloc(mm, level, shift); + if (!pt) { + kfree(owner); + goto out; + } + owner->pool = pool; + owner->pt = pt; + INIT_LIST_HEAD(&owner->available); + owner->free_mask = GENMASK(nr - 1, 1); + for (i = 0; i < nr; i++) + owner->slots[i].table = ptdesc_address(pt) + (i << shift); + /* This word is unused by arm64 page-table constructors and walkers. */ + pt->__page_mapping = (unsigned long)owner; + + spin_lock_irqsave(&pool->lock, flags); + /* Reuse supply published while allocation slept, including retired slots. */ + table = fragment_take(pool, &zero); + if (!table && !pool->closed) { + refcount_inc(&pool->refs); /* Native owner's lifetime reference. */ + list_add_tail(&owner->available, &pool->available); + table = owner->slots[0].table; + owner = NULL; } - spin_unlock(&mm->page_table_lock); - if (table != ptdesc_address(pt)) + spin_unlock_irqrestore(&pool->lock, flags); + if (owner) { + pt->__page_mapping = 0; fragment_owner_free(pt); + kfree(owner); + } +out: + /* Clear only recycled slots that are actually allocated again. The issued + * reference excludes other allocators and keeps this owner alive here. + */ + if (table && zero) + memset(table, 0, 1UL << shift); return table; } @@ -130,14 +209,45 @@ void *arm64_user4k_pt_alloc(struct mm_struct *mm, enum arm64_user4k_pt_level lev return arm64_user_pt_alloc_granule(mm, level, 12); } +/* Only unpublished tables or tables past the required walker grace arrive here. */ void arm64_user4k_pt_free(void *table) { struct ptdesc *pt = virt_to_ptdesc(table); + struct fragment_owner *owner; + struct fragment_pool *pool; + unsigned int slot; + unsigned long flags; + bool last; VM_BUG_ON(!test_bit(PT_fragmented, &pt->pt_flags.f)); VM_BUG_ON(!IS_ALIGNED((unsigned long)table, 1UL << pt->pt_frag_shift)); VM_BUG_ON(atomic_read(&pt->pt_frag_refcount) <= 0); - fragment_put(pt, 1); + if (pt->pt_frag_shift >= PAGE_SHIFT) { + if (test_and_clear_bit(PT_frag_defer, &pt->pt_flags.f)) + call_rcu(&pt->pt_rcu_head, fragment_owner_free_rcu); + else + fragment_owner_free(pt); + return; + } + owner = (void *)pt->__page_mapping; + pool = owner->pool; + slot = ((unsigned long)table & ~PAGE_MASK) >> pt->pt_frag_shift; + spin_lock_irqsave(&pool->lock, flags); + VM_BUG_ON(owner->free_mask & BIT(slot)); + owner->free_mask |= BIT(slot); + owner->dirty_mask |= BIT(slot); + last = atomic_dec_and_test(&pt->pt_frag_refcount); + if (last) + list_del_init(&owner->available); + else if (!pool->closed && list_empty(&owner->available)) + list_add_tail(&owner->available, &pool->available); + spin_unlock_irqrestore(&pool->lock, flags); + if (last) { + pt->__page_mapping = 0; + fragment_owner_free(pt); + kfree(owner); + fragment_pool_put(pool); + } } void arm64_pgtable_free(void *table) @@ -154,32 +264,46 @@ void arm64_pgtable_free(void *table) void arm64_user4k_pt_cache_destroy(struct mm_struct *mm) { + struct fragment_pool *pool; unsigned int level; + unsigned long flags; - /* The mm is private or dying; no allocator may still access its cache. */ + /* No allocator may still access this mm; deferred owners are independent. */ for (level = 0; level < ARRAY_SIZE(mm->context.user4k_pt_frag); level++) { - void *next = mm->context.user4k_pt_frag[level]; - struct ptdesc *pt; - unsigned int issued, shift; - - if (!next) + pool = mm->context.user4k_pt_frag[level]; + if (!pool) continue; mm->context.user4k_pt_frag[level] = NULL; - pt = virt_to_ptdesc(next); - shift = pt->pt_frag_shift; - issued = ((unsigned long)next & ~PAGE_MASK) >> shift; - fragment_put(pt, (PAGE_SIZE >> shift) - issued); + spin_lock_irqsave(&pool->lock, flags); + pool->closed = true; + spin_unlock_irqrestore(&pool->lock, flags); + fragment_pool_put(pool); } } +static void fragment_slot_free_rcu(struct rcu_head *rcu) +{ + struct fragment_slot *slot = container_of(rcu, struct fragment_slot, rcu); + + arm64_user4k_pt_free(slot->table); +} + void arm64_pte_free_defer(struct mm_struct *mm, pgtable_t table) { struct ptdesc *pt = virt_to_ptdesc(table); if (test_bit(PT_fragmented, &pt->pt_flags.f)) { - /* No freed slot is reused. Delay the final owner release for RCU. */ - set_bit(PT_frag_defer, &pt->pt_flags.f); - arm64_user4k_pt_free(table); + if (pt->pt_frag_shift < PAGE_SHIFT) { + struct fragment_owner *owner = (void *)pt->__page_mapping; + unsigned int slot = ((unsigned long)table & ~PAGE_MASK) >> + pt->pt_frag_shift; + + /* Pending callbacks retain their issued references and contents. */ + call_rcu(&owner->slots[slot].rcu, fragment_slot_free_rcu); + } else { + set_bit(PT_frag_defer, &pt->pt_flags.f); + arm64_user4k_pt_free(table); + } } else { call_rcu(&pt->pt_rcu_head, fragment_owner_free_rcu); } diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 9a9d63e71d19f..bc4e4005b79c6 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -35,7 +35,6 @@ #include #include #include -#include #include #include #include @@ -858,6 +857,7 @@ static void test_fragment_lifetime(struct kunit *test, unsigned int shift) unsigned int i, slots = PAGE_SIZE >> shift; unsigned long size = 1UL << shift; unsigned int read_shift = 0; + int pending_refs; bool live; KUNIT_ASSERT_NOT_NULL(test, f); @@ -925,20 +925,33 @@ static void test_fragment_lifetime(struct kunit *test, unsigned int shift) KUNIT_EXPECT_PTR_NE(test, f->pinned[1], f->pinned[2]); pte_owner = page_ptdesc(f->pinned[0]); KUNIT_EXPECT_EQ(test, atomic_read(&pte_owner->pt_frag_refcount), (int)slots); + /* Refill holes in the same owner, without retaining old table contents. */ + for (i = 1; i < slots; i++) { + void *old = f->table[i]; + + memset(old, 0x6a, size); + arm64_pgtable_free(old); + f->table[i] = arm64_user_pt_alloc_granule(f->mm, USER4K_PT_PTE, shift); + KUNIT_ASSERT_NOT_NULL(test, f->table[i]); + KUNIT_EXPECT_PTR_EQ(test, f->table[i], old); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0, size), NULL); + } + /* Begin the reader before queuing a callback, so it cannot finish early. */ + rcu_read_lock(); arm64_pte_free_defer(f->mm, f->table[0]); f->table[0] = NULL; - KUNIT_EXPECT_EQ(test, atomic_read(&pte_owner->pt_frag_refcount), (int)slots - 1); + pending_refs = atomic_read(&pte_owner->pt_frag_refcount); for (i = 1; i < slots - 1; i++) { arm64_pgtable_free(f->table[i]); f->table[i] = NULL; } - /* A pre-existing software walker must survive the last fragment release. */ - rcu_read_lock(); + /* Pending slot zero keeps the owner alive through this reader. */ arm64_pgtable_free(f->table[slots - 1]); f->table[slots - 1] = NULL; live = folio_test_pgtable(page_folio(f->pinned[0])); read_shift = arm64_ptep_page_shift(page_address(f->pinned[0])); rcu_read_unlock(); + KUNIT_EXPECT_EQ(test, pending_refs, (int)slots); KUNIT_EXPECT_TRUE(test, live); KUNIT_EXPECT_EQ(test, read_shift, shift); rcu_barrier(); @@ -949,8 +962,9 @@ static void test_fragment_lifetime(struct kunit *test, unsigned int shift) arm64_pgtable_free(f->table[i]); f->table[i] = NULL; - KUNIT_EXPECT_EQ(test, atomic_read(&pt->pt_frag_refcount), (int)slots - 1); - KUNIT_EXPECT_TRUE(test, folio_test_pgtable(ptdesc_folio(pt))); + /* A wholly empty owner is released without waiting for mm teardown. */ + KUNIT_EXPECT_EQ(test, atomic_read(&pt->pt_frag_refcount), 0); + KUNIT_EXPECT_FALSE(test, folio_test_pgtable(ptdesc_folio(pt))); } mmput(f->mm); f->mm = NULL; @@ -5213,14 +5227,13 @@ static void user4k_exec_arguments_test(struct kunit *test) } for (native = 0; native < 2; native++) { struct linux_binprm *bprm = kunit_kzalloc(test, sizeof(*bprm), GFP_KERNEL); - struct mm_struct *mm; + struct mm_struct *mm = native ? mm_alloc() : test_vma_mm(test); unsigned long size, mask, initial, copied, new_top; unsigned long a_len = 2 * PAGE_SIZE + SZ_4K + 31, b_len = SZ_4K + 43; char *a, *b, *buf, *too_long; int ret; KUNIT_ASSERT_NOT_NULL(test, bprm); - mm = native ? mm_alloc() : test_vma_mm(test); KUNIT_ASSERT_NOT_NULL(test, mm); if (native) KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); @@ -5293,88 +5306,6 @@ static void user4k_exec_arguments_test(struct kunit *test) } } -/* The transfer can replace bprm->mm, so release its final owner. */ -static void test_exec_mm_free(void *data) -{ - struct linux_binprm *bprm = data; - - mmput(bprm->mm); -} - -static void user64k_exec_arguments_test(struct kunit *test) -{ - unsigned int shifts[] = { 16, 12, PAGE_SHIFT }; - unsigned int mode; - - if (PAGE_SHIFT >= 16 || - (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN))) { - kunit_skip(test, "requires internal larger user granule and MM opt-in"); - return; - } - for (mode = 0; mode < ARRAY_SIZE(shifts); mode++) { - struct linux_binprm *bprm = kunit_kzalloc(test, sizeof(*bprm), GFP_KERNEL); - unsigned long a_len = mode ? SZ_64K + SZ_16K + 31 : 32 * PAGE_SIZE + 31; - unsigned long b_len = PAGE_SIZE + 17, initial, copied; - char *a, *b, *buf; - unsigned long i; - - KUNIT_ASSERT_NOT_NULL(test, bprm); - bprm->mm = mm_alloc(); - KUNIT_ASSERT_NOT_NULL(test, bprm->mm); - KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_exec_mm_free, bprm), 0); - KUNIT_ASSERT_EQ(test, test_mm_select_granule(bprm->mm, 16), 0); - KUNIT_ASSERT_EQ(test, create_init_stack_vma(bprm->mm, &bprm->vma, &bprm->p), 0); - initial = bprm->p; - bprm->argmin = initial - SZ_2M; - bprm->rlim_stack.rlim_cur = SZ_8M; - bprm->rlim_stack.rlim_max = RLIM_INFINITY; - a = kunit_kmalloc(test, a_len + 1, GFP_KERNEL); - b = kunit_kmalloc(test, b_len + 1, GFP_KERNEL); - buf = kunit_kmalloc(test, a_len + b_len + 7, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, a); - KUNIT_ASSERT_NOT_NULL(test, b); - KUNIT_ASSERT_NOT_NULL(test, buf); - for (i = 0; i < a_len; i++) - a[i] = 'A' + i % 23; - a[a_len] = 0; - memset(b, 'b', b_len); - b[b_len] = 0; - KUNIT_ASSERT_EQ(test, copy_string_kernel("tail", bprm), 0); - KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, 5, 0), 5); - KUNIT_ASSERT_STREQ(test, buf, "tail"); - KUNIT_ASSERT_EQ(test, copy_string_kernel(a, bprm), 0); - KUNIT_ASSERT_EQ(test, copy_string_kernel(b, bprm), 0); - bprm->argc = 3; - copied = initial - bprm->p; - KUNIT_ASSERT_EQ(test, copied, a_len + b_len + 7); - KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, copied, 0), - (int)copied); - KUNIT_ASSERT_EQ(test, memcmp(buf, b, b_len + 1), 0); - KUNIT_ASSERT_EQ(test, memcmp(buf + b_len + 1, a, a_len + 1), 0); - KUNIT_ASSERT_STREQ(test, buf + b_len + a_len + 2, "tail"); - /* Exercise copying out of every native page within a coarse leaf. */ - if (shifts[mode] != 16) { - KUNIT_ASSERT_EQ(test, bprm_set_page_shift(bprm, shifts[mode]), 0); - KUNIT_ASSERT_EQ(test, mm_page_shift(bprm->mm), shifts[mode]); - memset(buf, 0, copied); - KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, copied, 0), - (int)copied); - KUNIT_ASSERT_EQ(test, memcmp(buf, b, b_len + 1), 0); - KUNIT_ASSERT_EQ(test, memcmp(buf + b_len + 1, a, a_len + 1), 0); - KUNIT_ASSERT_STREQ(test, buf + b_len + a_len + 2, "tail"); - } - initial = bprm->p; - KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); - KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + 1); - KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); - KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + a_len + 2); - KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); - KUNIT_EXPECT_EQ(test, bprm->p, initial + copied); - KUNIT_EXPECT_EQ(test, bprm->argc, 0); - KUNIT_EXPECT_EQ(test, bprm->vma_pages, 0UL); - } -} - static struct vm_area_struct *test_special_vma(struct mm_struct *mm, const struct vm_special_mapping *sm, unsigned long start, unsigned long len, vm_flags_t flags) @@ -6288,10 +6219,10 @@ static void user4k_file_mapcounts_test(struct kunit *test) folio = filemap_get_folio(file->f_mapping, 0); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 2, 2, 2 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 2, 2, 2 }, 1); KUNIT_ASSERT_EQ(test, test_file_fault(private, TEST_VA + 2 * SZ_4K, true) & VM_FAULT_ERROR, 0U); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 2, 2, 1 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 2, 2, 1 }, 1); child = test_vma_mm(test); KUNIT_ASSERT_NOT_NULL(test, child); uprobe_start_dup_mmap(); @@ -6300,17 +6231,17 @@ static void user4k_file_mapcounts_test(struct kunit *test) KUNIT_ASSERT_EQ(test, ret, 0); cvma = test_vma_lookup(child, TEST_VA); KUNIT_ASSERT_NOT_NULL(test, cvma); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 3, 3, 1 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 3, 3, 1 }, 1); KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA + SZ_4K, true) & VM_FAULT_ERROR, 0U); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 3, 2, 1 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 3, 2, 1 }, 1); kunit_release_action(test, user4k_vma_mm_free, child); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 2, 2, 1 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 2, 2, 1 }, 1); kunit_release_action(test, user4k_vma_mm_free, private->vm_mm); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 1, 1, 1 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 1, 1, 1 }, 1); kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 1, 1, 1 }, 0); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 1, 1, 1 }, 0); kunit_release_action(test, user4k_vma_mm_free, shared->vm_mm); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 0, 0, 0 }, 0); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 0, 0, 0 }, 0); } /* A unique RAM-backed test inode: exercise generic file faults without @@ -8892,10 +8823,6 @@ static void user_subpage_numa_cow_batch_test(struct kunit *test) { unsigned int shift; - if (!(READ_ONCE(sysctl_numa_balancing_mode) & NUMA_BALANCING_NORMAL)) { - kunit_skip(test, "requires normal NUMA balancing; use numa_balancing=enable"); - return; - } for (shift = 12; shift < PAGE_SHIFT; shift += 2) { unsigned long size = 1UL << shift, start = TEST_VA; struct vm_area_struct *vma; @@ -10094,7 +10021,7 @@ static void user4k_file_migration_test(struct kunit *test) lru_add_drain_all(); folio = filemap_get_folio(file->f_mapping, 0); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { [0 ... TEST_SLOTS - 1] = 1 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ [0 ... TEST_SLOTS - 1] = 1 }, 1); old_phys = PFN_PHYS(folio_pfn(folio)); isolated = isolate_folio_to_list(folio, &folios); if (!fail || !isolated) @@ -10109,7 +10036,7 @@ static void user4k_file_migration_test(struct kunit *test) KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : 1U); folio = filemap_get_folio(file->f_mapping, 0); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { [0 ... TEST_SLOTS - 1] = 1 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ [0 ... TEST_SLOTS - 1] = 1 }, 1); for (i = 0; i < 4; i++) { pte_t entry = test_fault_entry(test, small->vm_mm, small->vm_start + i * SZ_4K); @@ -10127,7 +10054,7 @@ static void user4k_file_migration_test(struct kunit *test) folio_lock(folio); try_to_unmap(folio, TTU_SYNC); KUNIT_EXPECT_FALSE(test, folio_mapped(folio)); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 0, 0, 0 }, 0); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 0, 0, 0 }, 0); folio_unlock(folio); folio_put(folio); for (i = 0; i < 4; i++) { @@ -14105,7 +14032,6 @@ static struct kunit_case user4k_test_cases[] = { KUNIT_CASE(user4k_mremap_resize_test), KUNIT_CASE(user4k_stack_relocate_test), KUNIT_CASE(user4k_exec_arguments_test), - KUNIT_CASE(user64k_exec_arguments_test), KUNIT_CASE(user4k_special_mapping_test), KUNIT_CASE(user4k_insert_fragment_test), KUNIT_CASE(user16k_insert_shared_test), @@ -14144,6 +14070,9 @@ static struct kunit_case user4k_test_cases[] = { KUNIT_CASE(user4k_vm_mmap_test), KUNIT_CASE(user4k_mprotect_test), KUNIT_CASE(user4k_mprotect_cow_batch_test), +#ifdef CONFIG_NUMA_BALANCING + KUNIT_CASE(user_subpage_numa_cow_batch_test), +#endif KUNIT_CASE(user4k_mprotect_accounting_test), KUNIT_CASE(user4k_file_cow_test), KUNIT_CASE(user4k_file_eof_test), @@ -14204,25 +14133,6 @@ static struct kunit_case user4k_test_cases[] = { {} }; -#ifdef CONFIG_NUMA_BALANCING -/* - * The COW hinting expectations require a stable normal-balancing policy. - * Runtime sysctl writers do not take the fixture's mm or PTE locks, so run - * this case at boot and do not expose it through the debugfs rerun control. - */ -static struct kunit_case user4k_numa_init_cases[] = { - KUNIT_CASE(user_subpage_numa_cow_batch_test), - {} -}; - -static struct kunit_suite user4k_numa_init_suite = { - .name = "arm64-user4k-numa-init", - .test_cases = user4k_numa_init_cases, -}; - -kunit_test_init_section_suite(user4k_numa_init_suite); -#endif - static struct kunit_suite user4k_test_suite = { .name = "arm64-user4k", .test_cases = user4k_test_cases, diff --git a/tools/testing/selftests/arm64/user4k/README.pagetable-backing b/tools/testing/selftests/arm64/user4k/README.pagetable-backing index 2c6b16b17465a..cf87f2a64f5ae 100644 --- a/tools/testing/selftests/arm64/user4k/README.pagetable-backing +++ b/tools/testing/selftests/arm64/user4k/README.pagetable-backing @@ -27,7 +27,19 @@ and upper-level tables contribute to the measurements; do not subtract them as though they were pure payload. Deferred table releases can occur between samples. Contents PASS does not assert a density or performance result. -The allocation-race fix reduces initial concurrent physical backing. It does -not reclaim partially occupied table owners or reuse issued fragments after -unmap. Retention/refill checkpoints deliberately preserve that separate -coverage gap rather than declaring it fixed. +The allocator reuses safely retired fragments, including slots from owners +that still contain live tables. RCU-pending slots remain unavailable until +their callback runs; an empty owner is freed. A reused dirty slot is cleared +before it is returned. This avoids allocating new owners while old ones have +reusable holes. It does not compact surviving live tables. + +On Neo, the serial refill checkpoint fell from 7,421,952 to 4,276,224 bytes of +physical page tables; the concurrent refill fell from 6,275,072 to 4,276,224. +The same six-phase contents and post-unmap cleanup passed. Whole-cgroup slab +usage increased and includes allocator metadata plus other kernel objects. +Do not report page-table bytes as the complete memory cost. + +Seven operation-timed rounds after one warmup had serial/concurrent total +medians of 15.625/16.865 ms, versus 15.434/16.679 ms before reuse. The ranges +substantially overlap; a small cost remains possible. These measurements do +not establish a speedup or zero overhead. They exclude checkpoint I/O. From bfa2ea70c27e05e8afa689c1beb46ac270f2472a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 20:29:40 -0400 Subject: [PATCH 0867/2680] mm: retain compatible subpage pools across allocation-domain changes Index per-mm subpage pools by charge domain, node and VA shard instead of keeping only the first domain. Retain discoverable pools while their native owners survive, so A-to-B-to-A moves can reuse each domain's free quarters. Pin charge identities and registry lifetime, retire empty pools safely, and retry lookup when allocation races with retirement. Preserve private fallback and existing allocation policy restrictions. Add a bounded fixture for bulk moves, alternating single-slot tails and concurrent moves followed by steady allocation. On Neo, the previously 4 MiB backing for a 1 MiB steady allocation becomes approximately 1 MiB; alternating eight 4K leaves per domain fit two 16K owners. Actual kpagecgroup charges, old/new PFN stability, disjoint quarters and complete bytes pass. The corrected race oracle measures added owners rather than counting reused live backing again. Retained MM, pin, real swap readback and GPU controls also pass. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 453b475f491683d207313c66736ca4eef4028c16) --- mm/memory.c | 12 + mm/user-subpage.c | 225 ++++++++++++++---- .../testing/selftests/arm64/user4k/.gitignore | 2 + tools/testing/selftests/arm64/user4k/Makefile | 4 +- .../arm64/user4k/README.domain-cycle | 78 ++++++ .../selftests/arm64/user4k/domain-cycle.c | 155 ++++++++++++ 6 files changed, 432 insertions(+), 44 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/README.domain-cycle create mode 100644 tools/testing/selftests/arm64/user4k/domain-cycle.c diff --git a/mm/memory.c b/mm/memory.c index fc5cafd2ef47f..e648f92233fd9 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1360,6 +1360,7 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, struct mm_subpage *slot; struct folio *folio; unsigned int offset = vma_page_offset_at(vma, addr).offset; + unsigned int retired_retries = 0; int err; retry_private: @@ -1427,6 +1428,11 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, if (!*pooled) mm_subpage_pool_close(pool); mm_subpage_pool_put(pool); + if (*pooled && IS_ERR(slot) && PTR_ERR(slot) == -ESHUTDOWN) { + if (++retired_retries > 1) + force_private = true; + goto retry_private; + } return slot; } @@ -6397,6 +6403,7 @@ static int anon_subpage_prealloc_mask(struct mm_struct *mm, struct mm_subpage_pool *pool; struct folio *folio; unsigned int count = hweight_long(mask); + unsigned int retired_retries = 0; bool arbitrary; int err; @@ -6461,6 +6468,11 @@ static int anon_subpage_prealloc_mask(struct mm_struct *mm, if (!*pooled) mm_subpage_pool_close(pool); mm_subpage_pool_put(pool); + if (*pooled && err == -ESHUTDOWN) { + if (++retired_retries > 1) + use_pool = false; + goto retry_private; + } return err; } diff --git a/mm/user-subpage.c b/mm/user-subpage.c index acfa80848e6e3..cff26e5b5c803 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -213,6 +214,12 @@ struct mm_subpage *mm_subpage_get_from_phys(phys_addr_t phys) return slot; } +struct cow_pool_key { + struct mem_cgroup *memcg; + int nid; + u32 bucket; +}; + struct mm_subpage_pool { struct rcu_head rcu; spinlock_t lock; @@ -230,13 +237,101 @@ struct mm_subpage_pool { bool cow_shared; /* Allocated only for shared COW pools; protected by pool->lock. */ unsigned long *cow_free_slots; + struct cow_pool_key cow_key; + struct rhash_head cow_hash; + struct list_head cow_domain; + /* Protected by this pool's lock, never used after registry detachment. */ + struct mm_subpage_cow_pools *cow_registry; }; #define COW_POOL_BITS 4 struct mm_subpage_cow_pools { - struct mm_subpage_pool *pool[1U << COW_POOL_BITS]; + struct rhashtable domains; + spinlock_t lock; + struct list_head pools; + refcount_t refs; + struct rcu_head rcu; + struct work_struct free_work; }; +static const struct rhashtable_params cow_domain_params = { + .head_offset = offsetof(struct mm_subpage_pool, cow_hash), + .key_offset = offsetof(struct mm_subpage_pool, cow_key), + .key_len = sizeof(struct cow_pool_key), + .nelem_hint = 16, + .automatic_shrinking = true, +}; + +static void cow_registry_free_work(struct work_struct *work) +{ + struct mm_subpage_cow_pools *pools = + container_of(work, struct mm_subpage_cow_pools, free_work); + + WARN_ON_ONCE(!list_empty(&pools->pools)); + rhashtable_destroy(&pools->domains); + kfree(pools); +} + +static void cow_registry_free_rcu(struct rcu_head *rcu) +{ + struct mm_subpage_cow_pools *pools = + container_of(rcu, struct mm_subpage_cow_pools, rcu); + + queue_work(system_unbound_wq, &pools->free_work); +} + +static void cow_registry_put(struct mm_subpage_cow_pools *pools) +{ + if (refcount_dec_and_test(&pools->refs)) + call_rcu(&pools->rcu, cow_registry_free_rcu); +} + +static void pool_put_raw(struct mm_subpage_pool *pool); +static void owner_remove(struct mm_subpage_owner *owner); +static void owner_free(struct mm_subpage_owner *owner, bool put_folio); + +/* The caller retains a pool reference independently of registry membership. */ +static void cow_pool_detach_locked(struct mm_subpage_pool *pool) +{ + struct mm_subpage_cow_pools *pools = pool->cow_registry; + + lockdep_assert_held(&pool->lock); + if (!pools) + return; + pool->cow_registry = NULL; + /* A failed prospective insertion may not have entered the hash/list. */ + rhashtable_remove_fast(&pools->domains, &pool->cow_hash, cow_domain_params); + spin_lock(&pools->lock); + list_del_init(&pool->cow_domain); + spin_unlock(&pools->lock); + pool_put_raw(pool); /* Removed (or unpublished prospective) membership. */ + cow_registry_put(pools); +} + +static void cow_pool_retire_empty(struct mm_subpage_pool *pool) +{ + struct mm_subpage_owner *owner = NULL; + unsigned long flags; + + if (!pool->cow_shared) + return; + spin_lock_irqsave(&pool->lock, flags); + if (!pool->backing_pages || + (pool->backing_pages == 1 && pool->unissued)) { + pool->closed = true; + cow_pool_detach_locked(pool); + /* A failed allocation may leave wholly unissued supply, not a + * live domain. Retire it under the same reservation lock. + */ + owner = pool->unissued; + if (owner) + owner_remove(owner); + } + spin_unlock_irqrestore(&pool->lock, flags); + if (owner) + owner_free(owner, true); +} + static void owner_init(struct mm_subpage_owner *owner, struct mm_subpage_pool *pool, struct folio *folio) { @@ -285,7 +380,7 @@ void mm_subpage_pool_get(struct mm_subpage_pool *pool) refcount_inc(&pool->refs); } -void mm_subpage_pool_put(struct mm_subpage_pool *pool) +static void pool_put_raw(struct mm_subpage_pool *pool) { if (!refcount_dec_and_test(&pool->refs)) return; @@ -295,19 +390,25 @@ void mm_subpage_pool_put(struct mm_subpage_pool *pool) kfree_rcu(pool, rcu); } -/* Keep the first implementation within one unambiguous allocation domain. - * Policy-sensitive, locked, droppable and multi-node faults keep fresh backing. - * An mm can move cgroups; old charged slots must not satisfy its new faults. - */ +void mm_subpage_pool_put(struct mm_subpage_pool *pool) +{ + /* Failed allocations must not leave an empty per-domain cache behind. */ + cow_pool_retire_empty(pool); + pool_put_raw(pool); +} + +/* Per-mm pools retain each still-backed charge/node/shard domain separately. */ struct mm_subpage_pool *mm_subpage_cow_pool_get_charged(struct mm_struct *mm, struct mm_struct *charge_mm, struct vm_area_struct *vma, unsigned long address) { struct mm_subpage_cow_pools *pools, *new_pools; struct mm_subpage_pool *pool, *new; - unsigned int bucket = hash_long(address >> PAGE_SHIFT, COW_POOL_BITS); + struct cow_pool_key key = {}; struct mem_cgroup *memcg; - bool matches; + unsigned long flags; + unsigned int attempt; + void *result; int nid; if ((vma->vm_flags & (VM_LOCKED | VM_DROPPABLE | VM_SPECIAL)) || @@ -320,60 +421,85 @@ struct mm_subpage_pool *mm_subpage_cow_pool_get_charged(struct mm_struct *mm, nid = first_node(node_states[N_MEMORY]); if (!node_isset(nid, cpuset_current_mems_allowed)) return NULL; - /* Publication pairs with cmpxchg: every bucket starts empty. Adjacent - * 4K leaves in one native virtual group always choose the same pool. - * Faulting threads can migrate CPUs without changing that identity. - */ + /* Acquire initialization published by the mm's cmpxchg below. */ pools = smp_load_acquire(&mm->cow_subpage_pool); if (!pools) { new_pools = kzalloc(sizeof(*new_pools), GFP_KERNEL | __GFP_ACCOUNT); if (!new_pools) return NULL; + spin_lock_init(&new_pools->lock); + INIT_LIST_HEAD(&new_pools->pools); + refcount_set(&new_pools->refs, 1); + INIT_WORK(&new_pools->free_work, cow_registry_free_work); + if (rhashtable_init(&new_pools->domains, &cow_domain_params)) { + kfree(new_pools); + return NULL; + } pools = cmpxchg(&mm->cow_subpage_pool, NULL, new_pools); if (!pools) pools = new_pools; else - kfree(new_pools); + cow_registry_put(new_pools); } memcg = get_mem_cgroup_from_mm(charge_mm); - /* Acquire the immutable charge domain published with this bucket. */ - pool = smp_load_acquire(&pools->pool[bucket]); - if (!pool) { - new = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); - if (!new) { + key.memcg = memcg; + key.nid = nid; + key.bucket = hash_long(address >> PAGE_SHIFT, COW_POOL_BITS); + for (attempt = 0; attempt < 3; attempt++) { + rcu_read_lock(); + pool = rhashtable_lookup(&pools->domains, &key, cow_domain_params); + if (pool && !refcount_inc_not_zero(&pool->refs)) + pool = NULL; + rcu_read_unlock(); + if (pool) { mem_cgroup_put(memcg); - return NULL; + return pool; } + new = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!new) + break; new->cow_free_slots = kcalloc(SUBPAGES_PER_FOLIO, sizeof(*new->cow_free_slots), GFP_KERNEL | __GFP_ACCOUNT); if (!new->cow_free_slots) { mm_subpage_pool_close(new); mm_subpage_pool_put(new); - mem_cgroup_put(memcg); - return NULL; + break; } + if (memcg) + css_get(&memcg->css); new->cow_memcg = memcg; new->cow_nid = nid; new->cow_shared = true; - pool = cmpxchg(&pools->pool[bucket], NULL, new); - if (!pool) { - pool = new; - /* The mm owns the initial pool reference and its memcg ref. */ - mm_subpage_pool_get(pool); - return pool; + new->cow_key = key; + INIT_LIST_HEAD(&new->cow_domain); + new->cow_registry = pools; + /* Hold caller, prospective membership and registry lifetime refs + * before a concurrent lookup can observe this pool. + */ + mm_subpage_pool_get(new); + refcount_inc(&pools->refs); + spin_lock_irqsave(&new->lock, flags); + result = rhashtable_lookup_get_insert_fast(&pools->domains, + &new->cow_hash, cow_domain_params); + if (!result) { + spin_lock(&pools->lock); + list_add_tail(&new->cow_domain, &pools->pools); + spin_unlock(&pools->lock); } - matches = pool->cow_memcg == memcg && pool->cow_nid == nid; + spin_unlock_irqrestore(&new->lock, flags); + if (!result) { + mem_cgroup_put(memcg); + return new; + } + /* Do not dereference an unreferenced object returned by insert. */ mm_subpage_pool_close(new); mm_subpage_pool_put(new); - } else { - matches = pool->cow_memcg == memcg && pool->cow_nid == nid; - mem_cgroup_put(memcg); + if (IS_ERR(result)) + break; } - if (!matches) - return NULL; - mm_subpage_pool_get(pool); - return pool; + mem_cgroup_put(memcg); + return NULL; } struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, @@ -390,18 +516,26 @@ bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *fo void mm_subpage_cow_pool_exit(struct mm_struct *mm) { struct mm_subpage_cow_pools *pools = xchg(&mm->cow_subpage_pool, NULL); - unsigned int i; + struct mm_subpage_pool *pool; + unsigned long flags; - /* No faults remain once mm_users reaches zero. Slot owners can outlive mm. */ + /* No faults remain; outstanding owners/callbacks never use this mm. */ if (!pools) return; - for (i = 0; i < ARRAY_SIZE(pools->pool); i++) { - if (!pools->pool[i]) - continue; - mm_subpage_pool_close(pools->pool[i]); - mm_subpage_pool_put(pools->pool[i]); + for (;;) { + spin_lock_irqsave(&pools->lock, flags); + pool = list_first_entry_or_null(&pools->pools, + struct mm_subpage_pool, cow_domain); + if (pool) + mm_subpage_pool_get(pool); + spin_unlock_irqrestore(&pools->lock, flags); + if (!pool) + break; + /* Lock order is pool -> registry list, never the reverse. */ + mm_subpage_pool_close(pool); + mm_subpage_pool_put(pool); } - kfree(pools); + cow_registry_put(pools); } /* Counts include unavailable (busy/migrating) owners, so zero is conclusive. @@ -443,6 +577,10 @@ static void owner_remove(struct mm_subpage_owner *owner) if (owner->pool->unissued == owner) owner->pool->unissued = NULL; owner->pool->backing_pages--; + if (!owner->pool->backing_pages && owner->pool->cow_shared) { + owner->pool->closed = true; + cow_pool_detach_locked(owner->pool); + } } static void owner_free_rmap(struct mm_subpage_owner *owner) @@ -838,6 +976,7 @@ void mm_subpage_pool_close(struct mm_subpage_pool *pool) spin_lock_irqsave(&pool->lock, flags); pool->closed = true; + cow_pool_detach_locked(pool); /* At most one newly supplied owner has no issued slots. Close is O(1). */ owner = pool->unissued; if (owner) diff --git a/tools/testing/selftests/arm64/user4k/.gitignore b/tools/testing/selftests/arm64/user4k/.gitignore index 3c4c8c86eefe2..e3e47c9305901 100644 --- a/tools/testing/selftests/arm64/user4k/.gitignore +++ b/tools/testing/selftests/arm64/user4k/.gitignore @@ -9,3 +9,5 @@ clipped-firstwrite pinned-fork pagetable-retention pagetable-concurrent + +/domain-cycle diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index d306deb1f0054..0350be7a57912 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -18,11 +18,13 @@ TEST_GEN_PROGS_EXTENDED += pinned-fork TEST_FILES += README.pinned-fork TEST_GEN_PROGS_EXTENDED += pagetable-retention pagetable-concurrent TEST_FILES += README.pagetable-backing +TEST_GEN_PROGS_EXTENDED += domain-cycle +TEST_FILES += README.domain-cycle include ../../lib.mk $(OUTPUT)/page-contract: page-contract.c x18-probe.S -$(OUTPUT)/pagetable-concurrent $(OUTPUT)/first-write $(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool $(OUTPUT)/anon-hole-refill-race $(OUTPUT)/anon-hole-refill-reclaim: LDLIBS += -pthread +$(OUTPUT)/domain-cycle $(OUTPUT)/pagetable-concurrent $(OUTPUT)/first-write $(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool $(OUTPUT)/anon-hole-refill-race $(OUTPUT)/anon-hole-refill-reclaim: LDLIBS += -pthread # Freestanding ELF32 probes exercise the compat ABI without an armhf sysroot. AARCH32_CC ?= clang --target=arm-linux-gnueabihf diff --git a/tools/testing/selftests/arm64/user4k/README.domain-cycle b/tools/testing/selftests/arm64/user4k/README.domain-cycle new file mode 100644 index 0000000000000..3e7425f4dbffb --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.domain-cycle @@ -0,0 +1,78 @@ +Cgroup allocation-domain pool reuse +================================== + +Build domain-cycle with -pthread. Run in the established4K userspace ABI on a +native16K kernel. Root-visible pagemap and kpagecgroup are required for the +controller's density/charge proof. The fixture itself checks every payload byte. +No memory.high/max limits, reclaim, swap-area or GPU operations are needed. + + userpages 4k ./domain-cycle --native-16k [--tail | --race] + +The program prints READY and waits for stdin byte 'c' before allocating payload. +A controller places only this fresh fixture in owned cgroup A, then acknowledges. +Each PHASE prints its index/PID/base/leaf count/stride and waits for 'c'. Both +cgroups must remain alive until all prior ranges have been checked and unmapped. + +Default: three1MiB useful guarded allocation phases. Run in A, move to B, move +back to A between phases. Check all256 written leaves/range (4K every8K), excluding +the inaccessible guards. The old sticky-domain implementation can allocate16K +native backing per4K leaf after a move. Packed acceptance is1MiB..1.25MiB native +PFN union per1MiB phase; the allowance covers bounded shard/partial-owner tails. + +--tail:16 independent4K allocations, switching A/B after EVERY leaf. All16VAs are +selected in the same current16-way allocator shard using its documented hash. +Eight leaves/domain should occupy at most3native folios: two useful plus one +possible preexisting partial boundary. This catches a latest-domain-only cache +that closes old A pools when switching to B, then cannot use A's surviving free +quarters on return. Preserve every earlier leaf throughout all switches. Useful +payload64KiB, maximum256KiB private backing, about8MiB virtual reservation. + +--race:4workers allocate64guarded leaves each, with bounded test-only2ms pacing +per leaf. RACE prints four bases and waits for 'c'. After acknowledgement, move +only this thread group between A/B32times while allocation is active; record +monotonic move intervals. RACE_DONE prints the allocation interval and waits. +Require at least one move overlapping that interval before claiming concurrency +coverage (this does not prove a particular RCU/CAS interleaving occurred). Check +all race PFNs are charged to A or B, never outside those owned domains. Then move +the stopped fixture to B and acknowledge: it allocates one additional1MiB guarded +PHASE. Require packed density after transitions stop, plus unchanged earlier +race PFNs/charges and all bytes. Worst native payload8MiB, useful2MiB; threads +have128KiB stacks. No tight density claim applies during arbitrary in-flight moves. + +For each held range, read pagemap indexed by target4K VA. A present PFN is in4K +units; native PFN=PFN>>2. Read /proc/kpagecgroup at native_PFN*8 and compare with +stat(the intended cgroup).st_ino. Old phases keep their original allocation-time +charge after process moves. During racing allocations either A/B is valid. +Re-read PFNs to reject migration during the charge sample. Do not use total +memory.current as exact payload accounting: it includes code/stacks/page tables. +Normal8KiB guard stride and --tail4KiB stride are emitted explicitly. + +Use an owned fresh uncapped cgroup hierarchy and a bounded controller. If using +a parent unit and childA/B groups, move the held PID into childA BEFORE enabling +memory in that owned parent's subtree_control (parent must be empty). Do not +change unrelated cgroups or global controller settings. Parent15s alarm kills +all fixture threads; partial thread creation releases waiting workers before +joining. Controller should independently terminate/reap the owned process/unit +on protocol error, timeout or failed metadata, and retain the evidence. + +--selftest only validates byte patterns and same-shard address selection; it does +not exercise cgroup moves, kernel pool allocation or RCU. Hardware qualification +must include bulk/tail/race content and exact-charge/density gates, mm-exit cleanup, +existing COW/fork/reclaim controls, and dense/concurrent timing. No source or +parser-only pass closes these runtime/performance requirements. + +Race density must count newly added native owners. Before and after sampling +the final steady range, recheck the original race PFNs and allocation charges. +Reject old/new physical 4K aliases and inconsistent charges in shared native +owners. Subtract only those native owners actually observed in both stable +snapshots. The 80-owner/1,310,720-byte limit applies to the added native PFN +union for the new 1 MiB payload. Reusing an old owner's free quarters should +not count its existing live quarters as additional backing. Record total, +reused and added owners separately, plus combined per-domain quarter masks. +This is payload accounting, not proof of whole-mm alias closure. + +On Neo the corrected controller observed 77 owners in the final range, of +which 14 were reused and 63 new (1,032,192 added bytes). Old mappings and +charges stayed unchanged, reused slots were disjoint and all bytes passed. +The earlier range-only oracle failed at 81 seen owners before its final +checks; that run remains failed, not retrospectively qualified. diff --git a/tools/testing/selftests/arm64/user4k/domain-cycle.c b/tools/testing/selftests/arm64/user4k/domain-cycle.c new file mode 100644 index 0000000000000..e67b9f1b22b33 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/domain-cycle.c @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define LEAF 4096UL +#define NATIVE 16384UL +#define COUNT 256UL +#define STRIDE (2 * LEAF) +#define LENGTH (COUNT * STRIDE) +#define TAIL_SPAN (8UL * 1024 * 1024) +static unsigned char *maps[16]; +static unsigned long leaves = COUNT, stride = STRIDE; +static void fail(const char *s) { perror(s); exit(1); } +static unsigned bucket(uintptr_t p) +{ return ((uint64_t)(p >> 14) * UINT64_C(0x61c8864680b583eb)) >> 60; } +static unsigned char value(unsigned phase, size_t leaf, size_t byte) +{ return (unsigned char)(1 + (phase * 53 + leaf * 7 + byte) % 251); } +static void check(unsigned count) +{ + for (unsigned phase = 0; phase < count; phase++) + for (size_t leaf = 0; leaf < leaves; leaf++) + for (size_t byte = 0; byte < LEAF; byte++) + if (maps[phase][leaf * stride + byte] != value(phase, leaf, byte)) { + fprintf(stderr, "FAIL bytes phase=%u leaf=%zu byte=%zu\n", phase, leaf, byte); exit(1); + } +} +static void ack(void) +{ + char c; ssize_t n; + do { n = read(0, &c, 1); } while (n < 0 && errno == EINTR); + if (n != 1 || c != 'c') fail("controller ack"); +} +#define WORKERS 4 +#define RACE_LEAVES 64UL +static unsigned char *race_maps[WORKERS]; +static atomic_int race_start; +static uint64_t monotonic_ns(void) +{ + struct timespec t; + if (clock_gettime(CLOCK_MONOTONIC, &t)) fail("clock_gettime"); + return (uint64_t)t.tv_sec * 1000000000 + t.tv_nsec; +} +static void race_check(void) +{ + for (unsigned w = 0; w < WORKERS; w++) + for (size_t leaf = 0; leaf < RACE_LEAVES; leaf++) + for (size_t byte = 0; byte < LEAF; byte++) + if (race_maps[w][leaf * STRIDE + byte] != value(w + 32, leaf, byte)) fail("race bytes"); +} +static void *race_worker(void *arg) +{ + unsigned w = (uintptr_t)arg; + int start; + do { start = atomic_load_explicit(&race_start, memory_order_acquire); if (!start) sched_yield(); } while (!start); + if (start != 1) return NULL; + for (size_t leaf = 0; leaf < RACE_LEAVES; leaf++) { + unsigned char *p = race_maps[w] + leaf * STRIDE; + if (mprotect(p, LEAF, PROT_READ | PROT_WRITE)) fail("race mprotect"); + for (size_t byte = 0; byte < LEAF; byte++) p[byte] = value(w + 32, leaf, byte); + /* Test-only pacing keeps allocation active during bounded controller moves. */ + struct timespec delay = {.tv_nsec = 2000000}; + while (nanosleep(&delay, &delay) && errno == EINTR) {} + } + return NULL; +} +static void race_run(void) +{ + pthread_t threads[WORKERS]; + pthread_attr_t attr; + if (pthread_attr_init(&attr) || pthread_attr_setstacksize(&attr, 128 * 1024)) fail("thread attributes"); + unsigned made = 0; + for (; made < WORKERS; made++) { + race_maps[made] = mmap(NULL, RACE_LEAVES * STRIDE, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (race_maps[made] == MAP_FAILED) fail("race mmap"); + int err = pthread_create(&threads[made], &attr, race_worker, (void *)(uintptr_t)made); + if (err) { errno = err; break; } + } + pthread_attr_destroy(&attr); + if (made != WORKERS) { + atomic_store_explicit(&race_start, 2, memory_order_release); + for (unsigned i = 0; i < made; i++) pthread_join(threads[i], NULL); + fail("pthread_create"); + } + printf("RACE pid=%d base0=%lx base1=%lx base2=%lx base3=%lx leaves=%lu stride=%lu\n", getpid(), (unsigned long)race_maps[0], (unsigned long)race_maps[1], (unsigned long)race_maps[2], (unsigned long)race_maps[3], RACE_LEAVES, STRIDE); + ack(); + uint64_t begin = monotonic_ns(); + atomic_store_explicit(&race_start, 1, memory_order_release); + for (unsigned i = 0; i < WORKERS; i++) if (pthread_join(threads[i], NULL)) abort(); + uint64_t end = monotonic_ns(); + race_check(); + printf("RACE_DONE pid=%d begin_ns=%llu end_ns=%llu\n", getpid(), (unsigned long long)begin, (unsigned long long)end); + ack(); +} +int main(int argc, char **argv) +{ + if (argc == 2 && !strcmp(argv[1], "--selftest")) { + unsigned count = 0; + for (uintptr_t p = 0x100000; p < 0x100000 + TAIL_SPAN; p += NATIVE) count += bucket(p) == 0; + if (count < 16 || value(0, 0, 0) == value(1, 0, 0)) return 1; + puts("PASS patterns/hash selection only; no domain allocation coverage"); return 0; + } + int tail = argc == 3 && !strcmp(argv[2], "--tail"); + int race = argc == 3 && !strcmp(argv[2], "--race"); + if ((argc != 2 && !tail && !race) || strcmp(argv[1], "--native-16k") || getauxval(AT_PAGESZ) != LEAF) return 77; + unsigned phases = tail ? 16 : race ? 1 : 3; + unsigned char *reservation = NULL; + alarm(15); setvbuf(stdout, NULL, _IONBF, 0); + printf("READY pid=%d\n", getpid()); ack(); + if (race) race_run(); + if (tail) { + leaves = 1; stride = LEAF; + reservation = mmap(NULL, TAIL_SPAN + NATIVE, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reservation == MAP_FAILED) fail("tail reserve"); + uintptr_t begin = ((uintptr_t)reservation + NATIVE - 1) & ~(NATIVE - 1); + unsigned found = 0; + for (uintptr_t p = begin; p < begin + TAIL_SPAN && found < phases; p += NATIVE) + if (!bucket(p)) maps[found++] = (void *)p; + if (found != phases) fail("same-shard tail addresses"); + } + for (unsigned phase = 0; phase < phases; phase++) { + check(phase); + if (!tail) { + maps[phase] = mmap(NULL, LENGTH, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (maps[phase] == MAP_FAILED) fail("mmap"); + } + for (size_t leaf = 0; leaf < leaves; leaf++) { + unsigned char *p = maps[phase] + leaf * stride; + if (mprotect(p, LEAF, PROT_READ | PROT_WRITE)) fail("mprotect"); + for (size_t byte = 0; byte < LEAF; byte++) p[byte] = value(phase, leaf, byte); + } + check(phase + 1); + printf("PHASE index=%u pid=%d base=%lx leaves=%lu stride=%lu\n", phase, getpid(), (unsigned long)maps[phase], leaves, stride); + ack(); + } + check(phases); + if (race) { + race_check(); + for (unsigned w = 0; w < WORKERS; w++) if (munmap(race_maps[w], RACE_LEAVES * STRIDE)) fail("race munmap"); + } + if (tail) { + if (munmap(reservation, TAIL_SPAN + NATIVE)) fail("tail munmap"); + } else for (unsigned phase = 0; phase < phases; phase++) if (munmap(maps[phase], LENGTH)) fail("munmap"); + puts("PASS domain allocation phases and all prior bytes; charge/density separately verified"); + return 0; +} From 1b129eec5b04fb763782afd1303c0e061fad06a9 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 04:46:20 -0400 Subject: [PATCH 0868/2680] mm: preserve clean subpage lazyfree state on retained reads When a retained reference prevents discarding a clean lazyfree slot, try_to_unmap_subpage() restores the PTE but then marks the native folio swap-backed. The extra reference alone does not cancel MADV_FREE, so this can prevent reclaim without swap and cause unnecessary swap traffic. Leave the classification unchanged after restoring a clean PTE. Dirty entries still cancel lazyfree and follow the existing swap-backed path. Extend the lazyfree KUnit case to retain a reader after whole-folio advice, check the clean classification, then release the reader and reclaim. The expanded test passes on 16K and 64K KASAN guests without warnings. Signed-off-by: Diljit Singh (cherry picked from commit e9a25aae4e7509bc40554dac0a23240d77363110) --- arch/arm64/mm/user4k-test.c | 18 ++++++++++++++---- mm/rmap.c | 12 +++++------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index bc4e4005b79c6..ba544d04f4beb 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -9382,7 +9382,7 @@ static void user4k_lazyfree_test(struct kunit *test) return; } /* Partial, redirty, retained GET, typed write, all leaves, migration. */ - for (mode = 0; mode < 10; mode++) { + for (mode = 0; mode < 11; mode++) { struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); struct mm_struct *mm; struct user_page_fragment held = {}; @@ -9403,7 +9403,8 @@ static void user4k_lazyfree_test(struct kunit *test) } kthread_use_mm(mm); ret = test_madvise(TEST_VA, - (mode == 4 || mode == 9) ? PAGE_SIZE : SZ_4K, MADV_FREE); + (mode == 4 || mode == 9 || mode == 10) ? PAGE_SIZE : SZ_4K, + MADV_FREE); kthread_unuse_mm(mm); KUNIT_ASSERT_EQ(test, ret, 0L); pte = test_fault_entry(test, mm, TEST_VA); @@ -9415,7 +9416,7 @@ static void user4k_lazyfree_test(struct kunit *test) if (mode == 1) KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); - if (mode == 2 || mode == 3 || mode == 8) { + if (mode == 2 || mode == 3 || mode == 8 || mode == 10) { ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, mode == 3 ? FOLL_WRITE : 0, &held, 1); KUNIT_ASSERT_EQ(test, ret, 1L); @@ -9446,11 +9447,20 @@ static void user4k_lazyfree_test(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, folio); folio_lock(folio); try_to_unmap(folio, TTU_SYNC); + if (mode == 10) { + /* A temporary reader leaves clean whole-folio advice intact. */ + KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio)); + KUNIT_EXPECT_FALSE(test, folio_test_dirty(folio)); + KUNIT_EXPECT_FALSE(test, pte_none(test_fault_entry(test, mm, TEST_VA))); + release_user_fragments(&held, 1, false); + try_to_unmap(folio, TTU_SYNC); + } folio_unlock(folio); folio_put(folio); for (i = 0; i < 4; i++) { pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); - KUNIT_EXPECT_EQ(test, pte_none(pte), (mode == 4 || mode == 9) || + KUNIT_EXPECT_EQ(test, pte_none(pte), + (mode == 4 || mode == 9 || mode == 10) || (i == 0 && (mode == 0 || mode == 5))); } if (mode == 2 || mode == 6 || mode == 7) diff --git a/mm/rmap.c b/mm/rmap.c index cda21c4fe5742..05f041419c3a5 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2180,16 +2180,14 @@ static bool try_to_unmap_subpage(struct folio *folio, struct vm_area_struct *vma goto out; } set_pte_at(mm, address, pvmw->pte, pte); - /* Retained references must not turn a droppable folio into swap data. */ - if (vma->vm_flags & VM_DROPPABLE) + /* A retained reference alone does not cancel clean lazyfree data. */ + if ((vma->vm_flags & VM_DROPPABLE) || !pte_dirty(pte)) goto out; - if (pte_dirty(pte)) { - mm_subpage_cancel_lazyfree(slot); - folio_mark_dirty(folio); - } + mm_subpage_cancel_lazyfree(slot); + folio_mark_dirty(folio); } if (!folio_test_swapbacked(folio)) { - /* A write or retained reference cancelled whole-folio lazy reclaim. */ + /* A write cancelled whole-folio lazy reclaim. */ folio_set_swapbacked(folio); goto out; } From 2c37ad4bcebe70778fdeebdf550dc1554a394259 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 04:56:16 -0400 Subject: [PATCH 0869/2680] arm64: preserve dependencies in mixed-page test selectors The IOMMUFD harness selects IOMMU_PT even when IOMMU_SUPPORT is disabled, excluding the page allocation implementation its selected code requires. The boolean VFIO harness selectors can also force VFIO built-in while IOMMUFD is modular, bypassing VFIO's own tristate constraint. Require IOMMU_SUPPORT for the IOMMUFD harness. Require non-modular IOMMUFD for both VFIO selectors, including the ioctl selector that selects the accounting harness. Built-in and disabled IOMMUFD configurations remain available for VFIO testing. Six before/after configuration checks cover the invalid combinations and the retained valid selections. Integrated 16K/64K KASAN images build. Signed-off-by: Diljit Singh (cherry picked from commit cf3e9884efb7a5cd06cb06d7ca632bd1ceef22d4) --- arch/arm64/Kconfig.debug | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index 58d7a1dc98405..52fced2ab8355 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -65,6 +65,7 @@ config ARM64_USER4K_IOMMUFD_TEST bool "IOMMUFD ioctl harness for mixed userspace granules" depends on ARM64_USER4K_EXPERIMENTAL && DEBUG_KERNEL depends on FAULT_INJECTION && RUNTIME_TESTING_MENU + depends on IOMMU_SUPPORT select GENERIC_PT select IOMMU_PT select IOMMU_PT_AMDV1 @@ -80,6 +81,7 @@ config ARM64_USER4K_IOMMUFD_TEST config ARM64_USER4K_VFIO_ACCOUNT_TEST bool "VFIO type1 pinning harness for mixed userspace granules" depends on ARM64_USER4K_EXPERIMENTAL && KUNIT=y + depends on IOMMUFD != m select VFIO select VFIO_GROUP select VFIO_CONTAINER @@ -96,6 +98,7 @@ config ARM64_USER4K_VFIO_IOCTL_TEST bool "Private VFIO type1 ioctl harness for mixed userspace granules" depends on ARM64_USER4K_EXPERIMENTAL && KUNIT=y && DEBUG_KERNEL depends on RUNTIME_TESTING_MENU + depends on IOMMUFD != m select ARM64_USER4K_VFIO_ACCOUNT_TEST default n help From 84f8bf5087db0b522b37ded3b9b30bf6208087ba Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 04:56:16 -0400 Subject: [PATCH 0870/2680] arm64/mm: avoid table metadata lookup at the exclusive PTE endpoint contpte_try_unfold_partial() checks its exclusive endpoint with contpte_align_down(). That helper derives granule geometry from the PTE's backing metadata, but ptep + nr can already lie beyond that backing. Use the mm's contiguous-group size to test alignment at both ends. This preserves partial-group unfolding while avoiding a metadata lookup for a pointer that need not identify an allocated page-table entry. 16K/64K KASAN images build and their existing native/4K contiguous-PTE functional tests pass without warnings. The endpoint defect is established by source inspection; no runtime reproduction is claimed. Signed-off-by: Diljit Singh (cherry picked from commit f6e776e813d9dec5da55a0b2eb779eeca145c471) --- arch/arm64/mm/contpte.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c index f8faf1a99eeca..e78e0d4265218 100644 --- a/arch/arm64/mm/contpte.c +++ b/arch/arm64/mm/contpte.c @@ -68,15 +68,18 @@ static inline pte_t *contpte_align_addr_ptep(struct mm_struct *mm, unsigned long static void contpte_try_unfold_partial(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned int nr) { + unsigned int cont_bytes = sizeof(*ptep) * cont_ptes_mm(mm); + /* * Unfold any partially covered contpte block at the beginning and end * of the range. */ - if (ptep != contpte_align_down(ptep) || nr < cont_ptes_mm(mm)) + if (!IS_ALIGNED((unsigned long)ptep, cont_bytes) || nr < cont_ptes_mm(mm)) contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); - if (ptep + nr != contpte_align_down(ptep + nr)) { + /* The exclusive end can lie beyond the allocated page-table backing. */ + if (!IS_ALIGNED((unsigned long)(ptep + nr), cont_bytes)) { unsigned long last_addr = addr + mm_page_size(mm) * (nr - 1); pte_t *last_ptep = ptep + nr - 1; From 8c5ce670091c535a598cdfd389a5d505252a6740 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 05:09:14 -0400 Subject: [PATCH 0871/2680] perf: sample page sizes using the target mm geometry perf_get_pgtable_size() starts with an mm-aware PGD lookup but continues with native table offsets and leaf sizes. A smaller userspace granule can therefore select an unrelated entry or report the native page size. Add mm-aware lockless offset and leaf-size helpers with native fallbacks. Use the target mm throughout perf's walk, including runtime folding and contiguous mappings, while preserving the existing interrupt exclusion and descriptor-snapshot semantics. Expose the walker only for KUnit testing and cover ordinary mapped and unmapped addresses across table levels. Extend contiguous-PTE fixtures to check perf's reported size and cover 16K userspace on a 64K kernel. The tests pass in 16K/64K KASAN guests for native, 4K and supported 16K userspace granules. An x86_64 perf-object build checks the native fallbacks. Hardware PMU event delivery was not tested. Signed-off-by: Diljit Singh (cherry picked from commit 4538d92c925d9adc7af502c9df8aec52c56df322) --- arch/arm64/include/asm/user-pgtable.h | 23 +++++++ arch/arm64/mm/user4k-test.c | 91 +++++++++++++++++++++++++-- include/linux/perf_event.h | 5 ++ include/linux/pgtable.h | 29 +++++++++ kernel/events/core.c | 24 +++---- 5 files changed, 156 insertions(+), 16 deletions(-) diff --git a/arch/arm64/include/asm/user-pgtable.h b/arch/arm64/include/asm/user-pgtable.h index b244f2387a7b4..7b1d1a94515d0 100644 --- a/arch/arm64/include/asm/user-pgtable.h +++ b/arch/arm64/include/asm/user-pgtable.h @@ -113,6 +113,17 @@ static inline pmd_t *pmd_offset_mm(struct mm_struct *mm, pud_t *pud, #define pmd_offset_mm pmd_offset_mm +/* Use the caller's descriptor snapshots throughout a lockless walk. */ +#define p4d_offset_lockless_mm(mm, pgdp, pgd, addr) \ + (arm64_mm_alt_granule(mm) ? p4d_offset_mm(mm, &(pgd), addr) : \ + p4d_offset_lockless(pgdp, pgd, addr)) +#define pud_offset_lockless_mm(mm, p4dp, p4d, addr) \ + (arm64_mm_alt_granule(mm) ? pud_offset_mm(mm, &(p4d), addr) : \ + pud_offset_lockless(p4dp, p4d, addr)) +#define pmd_offset_lockless_mm(mm, pudp, pud, addr) \ + (arm64_mm_alt_granule(mm) ? pmd_offset_mm(mm, &(pud), addr) : \ + pmd_offset_lockless(pudp, pud, addr)) + static inline pte_t *pte_offset_kernel_mm(struct mm_struct *mm, pmd_t *pmd, unsigned long addr) { @@ -133,6 +144,18 @@ static inline pte_t *pte_offset_kernel_mm(struct mm_struct *mm, pmd_t *pmd, #define pud_size_mm(mm) (1UL << pud_shift_mm(mm)) #define p4d_size_mm(mm) (1UL << p4d_shift_mm(mm)) #define pgd_size_mm(mm) (arm64_mm_alt_granule(mm) ? (1UL << arm64_user_level_shift(mm, 0)) : PGDIR_SIZE) +#define pgd_leaf_size_mm(mm, pgd) \ + (arm64_mm_alt_granule(mm) ? pgd_size_mm(mm) : pgd_leaf_size(pgd)) +#define p4d_leaf_size_mm(mm, p4d) \ + (arm64_mm_alt_granule(mm) ? p4d_size_mm(mm) : p4d_leaf_size(p4d)) +#define pud_leaf_size_mm(mm, pud) \ + (arm64_mm_alt_granule(mm) ? pud_size_mm(mm) : pud_leaf_size(pud)) +#define pmd_leaf_size_mm(mm, pmd) \ + (arm64_mm_alt_granule(mm) ? pmd_size_mm(mm) * \ + (pmd_cont(pmd) ? (mm_page_shift(mm) == 12 ? 16 : 32) : 1) : pmd_leaf_size(pmd)) +#define __pte_leaf_size_mm(mm, pmd, pte) \ + (arm64_mm_alt_granule(mm) ? \ + (pte_cont(pte) ? cont_pte_size_mm(mm) : mm_page_size(mm)) : __pte_leaf_size(pmd, pte)) #define pte_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PTE * sizeof(pte_t)) #define pmd_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PMD * sizeof(pmd_t)) #define pud_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PUD * sizeof(pud_t)) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index ba544d04f4beb..209bcbe7305a8 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -1324,6 +1325,58 @@ static int test_map_alias(struct test_mm *ctx, unsigned int index, return 0; } +#ifdef CONFIG_PERF_EVENTS +static void test_perf_page_size(struct kunit *test, unsigned int shift) +{ + const unsigned long addresses[] = { + SZ_1M, SZ_1G + SZ_1M, (1UL << 39) + SZ_1M, + }; + struct test_mm *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + void *data = test_alloc_granule(test, PAGE_SHIFT); + unsigned long flags; + u64 present, absent; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, data); + ctx->test = test; + ctx->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, ctx->mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_mm_destroy, ctx), 0); + if (shift != PAGE_SHIFT) + KUNIT_ASSERT_EQ(test, test_mm_select_granule(ctx->mm, shift), 0); + for (i = 0; i < ARRAY_SIZE(addresses); i++) + KUNIT_ASSERT_EQ(test, test_map_alias(ctx, i, addresses[i], virt_to_phys(data)), 0); + for (i = 0; i < ARRAY_SIZE(addresses); i++) { + local_irq_save(flags); + present = perf_get_pgtable_size(ctx->mm, addresses[i]); + absent = perf_get_pgtable_size(ctx->mm, addresses[i] + (1UL << shift)); + local_irq_restore(flags); + KUNIT_EXPECT_EQ(test, present, 1ULL << shift); + KUNIT_EXPECT_EQ(test, absent, 0ULL); + } +} + +static void user4k_perf_page_size_test(struct kunit *test) +{ + test_perf_page_size(test, 12); +} + +static void user16k_perf_page_size_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_perf_page_size(test, 14); +} + +static void native_perf_page_size_test(struct kunit *test) +{ + test_perf_page_size(test, PAGE_SHIFT); +} +#endif + /* Physical-range lookup only; coarse anonymous ownership is a separate step. */ static void user64k_rmap_lookup_test(struct kunit *test) { @@ -7571,7 +7624,7 @@ static void contpte_fixture_free(void *arg) folio_put(f->folio); } -static void test_contpte_granule(struct kunit *test, bool small) +static void test_contpte_granule(struct kunit *test, unsigned int shift) { struct contpte_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); struct vm_area_struct vma = {}; @@ -7587,13 +7640,17 @@ static void test_contpte_granule(struct kunit *test, bool small) spinlock_t *ptl; unsigned int i; int access_ret, access_same_ret, access_noop_ret; +#ifdef CONFIG_PERF_EVENTS + unsigned long irqflags; + u64 reported_size; +#endif KUNIT_ASSERT_NOT_NULL(test, f); KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, contpte_fixture_free, f), 0); f->mm = mm_alloc(); KUNIT_ASSERT_NOT_NULL(test, f->mm); - if (small) - KUNIT_ASSERT_EQ(test, test_mm_select4k(f->mm), 0); + if (shift != PAGE_SHIFT) + KUNIT_ASSERT_EQ(test, test_mm_select_granule(f->mm, shift), 0); f->nr = cont_ptes_mm(f->mm); size = mm_page_size(f->mm); f->folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, @@ -7623,6 +7680,11 @@ static void test_contpte_granule(struct kunit *test, bool small) ok.addresses &= pte_phys_mm(f->mm, observed) == phys + i * size; ok.hints &= pte_batch_hint(pte + i, observed) == f->nr - i; } +#ifdef CONFIG_PERF_EVENTS + local_irq_save(irqflags); + reported_size = perf_get_pgtable_size(f->mm, TEST_VA + size); + local_irq_restore(irqflags); +#endif __set_pte(pte + 3, pte_mkdirty(__ptep_get(pte + 3))); observed = ptep_get(pte + 1); ok.gathered &= pte_dirty(observed) && pte_young(observed); @@ -7695,16 +7757,28 @@ static void test_contpte_granule(struct kunit *test, bool small) KUNIT_EXPECT_TRUE(test, ok.holes); KUNIT_EXPECT_TRUE(test, ok.full_clear); KUNIT_EXPECT_TRUE(test, ok.sentinel); +#ifdef CONFIG_PERF_EVENTS + KUNIT_EXPECT_EQ(test, reported_size, (u64)cont_pte_size_mm(f->mm)); +#endif } static void user4k_contpte_test(struct kunit *test) { - test_contpte_granule(test, true); + test_contpte_granule(test, 12); +} + +static void user16k_contpte_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_contpte_granule(test, 14); } static void native_contpte_test(struct kunit *test) { - test_contpte_granule(test, false); + test_contpte_granule(test, PAGE_SHIFT); } #endif @@ -14123,7 +14197,13 @@ static struct kunit_case user4k_test_cases[] = { KUNIT_CASE(user4k_anon_remote_zap_test), #ifdef CONFIG_ARM64_CONTPTE KUNIT_CASE(user4k_contpte_test), + KUNIT_CASE(user16k_contpte_test), KUNIT_CASE(native_contpte_test), +#endif +#ifdef CONFIG_PERF_EVENTS + KUNIT_CASE(user4k_perf_page_size_test), + KUNIT_CASE(user16k_perf_page_size_test), + KUNIT_CASE(native_perf_page_size_test), #endif KUNIT_CASE(user4k_mm_tables_test), KUNIT_CASE(user16k_mm_tables_test), @@ -14157,3 +14237,4 @@ kunit_test_suites(&user4k_test_suite, &user4k_swap_suite, &user4k_nonlinear_swap #endif MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 858c59b031455..98baff049b262 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1209,6 +1209,11 @@ perf_cgroup_from_task(struct task_struct *task, struct perf_event_context *ctx) extern struct perf_event_context *perf_cpu_task_ctx(void); +#if IS_ENABLED(CONFIG_KUNIT) +/* The caller must disable IRQs, as for the sampling path. */ +u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr); +#endif + extern void *perf_aux_output_begin(struct perf_output_handle *handle, struct perf_event *event); extern void perf_aux_output_end(struct perf_output_handle *handle, diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 2b6765f9efb2c..0e3e62c8a07c3 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -2462,6 +2462,19 @@ static inline const char *pgtable_level_to_str(enum pgtable_level level) #define pmd_offset_lockless(pudp, pud, address) pmd_offset(&(pud), address) #endif +#ifndef p4d_offset_lockless_mm +#define p4d_offset_lockless_mm(mm, pgdp, pgd, address) \ + p4d_offset_lockless(pgdp, pgd, address) +#endif +#ifndef pud_offset_lockless_mm +#define pud_offset_lockless_mm(mm, p4dp, p4d, address) \ + pud_offset_lockless(p4dp, p4d, address) +#endif +#ifndef pmd_offset_lockless_mm +#define pmd_offset_lockless_mm(mm, pudp, pud, address) \ + pmd_offset_lockless(pudp, pud, address) +#endif + /* * pXd_leaf() is the API to check whether a pgtable entry is a huge page * mapping. It should work globally across all archs, without any @@ -2515,6 +2528,22 @@ static inline const char *pgtable_level_to_str(enum pgtable_level level) #define __pte_leaf_size(x,y) pte_leaf_size(y) #endif +#ifndef pgd_leaf_size_mm +#define pgd_leaf_size_mm(mm, pgd) pgd_leaf_size(pgd) +#endif +#ifndef p4d_leaf_size_mm +#define p4d_leaf_size_mm(mm, p4d) p4d_leaf_size(p4d) +#endif +#ifndef pud_leaf_size_mm +#define pud_leaf_size_mm(mm, pud) pud_leaf_size(pud) +#endif +#ifndef pmd_leaf_size_mm +#define pmd_leaf_size_mm(mm, pmd) pmd_leaf_size(pmd) +#endif +#ifndef __pte_leaf_size_mm +#define __pte_leaf_size_mm(mm, pmd, pte) __pte_leaf_size(pmd, pte) +#endif + /* * We always define pmd_pfn for all archs as it's used in lots of generic * code. Now it happens too for pud_pfn (and can happen for larger diff --git a/kernel/events/core.c b/kernel/events/core.c index 235fc75b967ae..e0f0aa0378b4f 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -8,6 +8,7 @@ * Copyright © 2009 Paul Mackerras, IBM Corp. */ +#include #include #include #include @@ -8536,7 +8537,7 @@ static u64 perf_virt_to_phys(u64 virt) /* * Return the pagetable size of a given virtual address. */ -static u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr) +VISIBLE_IF_KUNIT u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr) { u64 size = 0; @@ -8553,45 +8554,46 @@ static u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr) return 0; if (pgd_leaf(pgd)) - return pgd_leaf_size(pgd); + return pgd_leaf_size_mm(mm, pgd); - p4dp = p4d_offset_lockless(pgdp, pgd, addr); + p4dp = p4d_offset_lockless_mm(mm, pgdp, pgd, addr); p4d = p4dp_get(p4dp); - if (!p4d_present(p4d)) + if (!p4d_present_mm(mm, p4d)) return 0; if (p4d_leaf(p4d)) - return p4d_leaf_size(p4d); + return p4d_leaf_size_mm(mm, p4d); - pudp = pud_offset_lockless(p4dp, p4d, addr); + pudp = pud_offset_lockless_mm(mm, p4dp, p4d, addr); pud = pudp_get(pudp); if (!pud_present(pud)) return 0; if (pud_leaf(pud)) - return pud_leaf_size(pud); + return pud_leaf_size_mm(mm, pud); - pmdp = pmd_offset_lockless(pudp, pud, addr); + pmdp = pmd_offset_lockless_mm(mm, pudp, pud, addr); again: pmd = pmdp_get_lockless(pmdp); if (!pmd_present(pmd)) return 0; if (pmd_leaf(pmd)) - return pmd_leaf_size(pmd); + return pmd_leaf_size_mm(mm, pmd); - ptep = pte_offset_map(&pmd, addr); + ptep = pte_offset_map_mm(mm, &pmd, addr); if (!ptep) goto again; pte = ptep_get_lockless(ptep); if (pte_present(pte)) - size = __pte_leaf_size(pmd, pte); + size = __pte_leaf_size_mm(mm, pmd, pte); pte_unmap(ptep); #endif /* CONFIG_HAVE_GUP_FAST */ return size; } +EXPORT_SYMBOL_IF_KUNIT(perf_get_pgtable_size); static u64 perf_get_page_size(unsigned long addr) { From b719358a8dade20696af4c8cf59b54da47e826fd Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 05:23:12 -0400 Subject: [PATCH 0872/2680] uprobes: preserve architecture metadata when comparing subpage ranges The smaller-page orig_page_is_identical() path compares bytes directly, bypassing the architecture checks used by pages_identical(). On arm64, byte equality does not make distinct tagged ranges interchangeable. Add memcmp_pages_range() with a generic mapped-byte implementation and an arm64 implementation retaining the existing MTE policy. Distinct tagged ranges remain non-identical, including different offsets within one page. Keep full-page comparison as the zero-offset full-length case, and use the range API for the smaller-page uprobe comparison. Add direct tests for offset ranges, bytes outside the compared range, unequal contents, empty ranges and tagged-range identity. These and the existing ownership tests pass in 16K/64K KASAN guests. The generic util and uprobe objects also compile for x86. Actual uprobe removal with private tag state was not runtime reproduced. Signed-off-by: Diljit Singh (cherry picked from commit 0c0dccf36f0a795fb0e52aa74ca1eb51c75f659e) --- arch/arm64/kernel/mte.c | 21 ++++++---- arch/arm64/mm/user4k-data-test.c | 72 ++++++++++++++++++++++++++++++++ include/linux/mm.h | 2 + kernel/events/uprobes.c | 6 +-- mm/util.c | 11 ++++- 5 files changed, 99 insertions(+), 13 deletions(-) diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c index 31f5c6b0510e7..8c73cf2336313 100644 --- a/arch/arm64/kernel/mte.c +++ b/arch/arm64/kernel/mte.c @@ -70,21 +70,23 @@ void mte_sync_tags(pte_t pte, unsigned int nr_pages) smp_wmb(); } -int memcmp_pages(struct page *page1, struct page *page2) +int memcmp_pages_range(struct page *page1, unsigned int offset1, + struct page *page2, unsigned int offset2, size_t len) { char *addr1, *addr2; int ret; - addr1 = page_address(page1); - addr2 = page_address(page2); - ret = memcmp(addr1, addr2, PAGE_SIZE); + addr1 = page_address(page1) + offset1; + addr2 = page_address(page2) + offset2; + ret = memcmp(addr1, addr2, len); - if (!system_supports_mte() || ret) + if (!len || !system_supports_mte() || ret) return ret; /* - * If the page content is identical but at least one of the pages is - * tagged, return non-zero to avoid KSM merging. If only one of the + * If the byte ranges are identical but at least one page is tagged, + * only the same physical range is interchangeable. Avoid merging or + * replacing independently tagged ranges. If only one of the * pages is tagged, __set_ptes() may zero or change the tags of the * other page via mte_sync_tags(). */ @@ -94,6 +96,11 @@ int memcmp_pages(struct page *page1, struct page *page2) return ret; } +int memcmp_pages(struct page *page1, struct page *page2) +{ + return memcmp_pages_range(page1, 0, page2, 0, PAGE_SIZE); +} + static inline void __mte_enable_kernel(const char *mode, unsigned long tcf) { /* Enable MTE Sync Mode for EL1. */ diff --git a/arch/arm64/mm/user4k-data-test.c b/arch/arm64/mm/user4k-data-test.c index d7d550c9e1420..76656736074b1 100644 --- a/arch/arm64/mm/user4k-data-test.c +++ b/arch/arm64/mm/user4k-data-test.c @@ -1084,6 +1084,76 @@ static void extent_folio_put(void *arg) folio_put(arg); } +static struct folio *compare_folio_alloc(struct kunit *test) +{ + struct folio *folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); + + if (folio && kunit_add_action_or_reset(test, extent_folio_put, folio)) + return NULL; + return folio; +} + +static void user_page_range_compare_test(struct kunit *test) +{ + const unsigned int sizes[] = { SZ_4K, SZ_16K, PAGE_SIZE }; + struct folio *a = compare_folio_alloc(test), *b = compare_folio_alloc(test); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + for (i = 0; i < ARRAY_SIZE(sizes); i++) { + unsigned int len = sizes[i], offset = PAGE_SIZE - len; + char *from = folio_address(a), *to = folio_address(b); + + if (len > PAGE_SIZE) + continue; + memset(from, 0x11, PAGE_SIZE); + memset(to, 0x22, PAGE_SIZE); + memset(from, 0x5a, len); + memset(to + offset, 0x5a, len); + KUNIT_EXPECT_EQ(test, memcmp_pages_range(&a->page, 0, &b->page, offset, len), 0); + KUNIT_EXPECT_EQ(test, !memcmp_pages(&a->page, &b->page), len == PAGE_SIZE); + to[PAGE_SIZE - 1] ^= 1; + KUNIT_EXPECT_NE(test, memcmp_pages_range(&a->page, 0, &b->page, offset, len), 0); + } +} + +static void user_page_tagged_range_compare_test(struct kunit *test) +{ +#ifdef CONFIG_ARM64_MTE + struct folio *a, *b; + unsigned int len; + + if (!system_supports_mte()) { + kunit_skip(test, "MTE is not available on this CPU"); + return; + } + a = compare_folio_alloc(test); + b = compare_folio_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_EQ(test, memcmp_pages(&a->page, &b->page), 0); + KUNIT_ASSERT_TRUE(test, try_page_mte_tagging(&a->page)); + mte_clear_page_tags(folio_address(a)); + set_page_mte_tagged(&a->page); + KUNIT_EXPECT_NE(test, memcmp_pages(&a->page, &b->page), 0); + for (len = SZ_4K; len <= PAGE_SIZE; len *= 4) { + KUNIT_EXPECT_EQ(test, memcmp_pages_range(&a->page, 0, &a->page, 0, len), 0); + KUNIT_EXPECT_NE(test, memcmp_pages_range(&a->page, 0, &b->page, 0, len), 0); + if (len < PAGE_SIZE) + KUNIT_EXPECT_NE(test, + memcmp_pages_range(&a->page, 0, &a->page, len, len), 0); + } + KUNIT_EXPECT_EQ(test, memcmp_pages_range(&a->page, 0, &b->page, 0, 0), 0); + KUNIT_ASSERT_TRUE(test, try_page_mte_tagging(&b->page)); + mte_clear_page_tags(folio_address(b)); + set_page_mte_tagged(&b->page); + KUNIT_EXPECT_NE(test, memcmp_pages_range(&a->page, 0, &b->page, 0, SZ_4K), 0); +#else + kunit_skip(test, "MTE is not configured"); +#endif +} + static void user4k_data_copy_extent_test(struct kunit *test) { const unsigned int sizes[] = { SZ_4K, SZ_16K, SZ_64K }; @@ -1798,6 +1868,8 @@ static struct kunit_case user4k_data_cases[] = { #endif KUNIT_CASE(user4k_data_mte_test), KUNIT_CASE(user4k_data_copy_extent_test), + KUNIT_CASE(user_page_range_compare_test), + KUNIT_CASE(user_page_tagged_range_compare_test), KUNIT_CASE(user16k_data_owner_test), KUNIT_CASE(user16k_file_coverage_test), KUNIT_CASE(user16k_data_refset_test), diff --git a/include/linux/mm.h b/include/linux/mm.h index 749028a1b2c6b..362fea150df79 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -5243,6 +5243,8 @@ static inline void setup_nr_node_ids(void) {} #endif extern int memcmp_pages(struct page *page1, struct page *page2); +int memcmp_pages_range(struct page *page1, unsigned int offset1, + struct page *page2, unsigned int offset2, size_t len); static inline int pages_identical(struct page *page1, struct page *page2) { diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 23dcd90acdc0c..c191661cbf9b8 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -425,12 +425,10 @@ static bool orig_page_is_identical(struct vm_area_struct *vma, folio_test_pmd_mappable(orig_folio); if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { unsigned int offset = vma_page_offset_at(vma, vaddr).offset; - void *a = kmap_local_page(page), *b = kmap_local_page(orig_page); identical = folio_test_uptodate(orig_folio) && - !memcmp(a + page_offset, b + offset, mm_page_size(vma->vm_mm)); - kunmap_local(b); - kunmap_local(a); + !memcmp_pages_range(page, page_offset, orig_page, offset, + mm_page_size(vma->vm_mm)); } else { identical = folio_test_uptodate(orig_folio) && pages_identical(page, orig_page); diff --git a/mm/util.c b/mm/util.c index a4c25d097ec61..68cbf435d4775 100644 --- a/mm/util.c +++ b/mm/util.c @@ -1094,19 +1094,26 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen) return res; } -int __weak memcmp_pages(struct page *page1, struct page *page2) +/* Both byte ranges must fit within their respective native pages. */ +int __weak memcmp_pages_range(struct page *page1, unsigned int offset1, + struct page *page2, unsigned int offset2, size_t len) { char *addr1, *addr2; int ret; addr1 = kmap_local_page(page1); addr2 = kmap_local_page(page2); - ret = memcmp(addr1, addr2, PAGE_SIZE); + ret = memcmp(addr1 + offset1, addr2 + offset2, len); kunmap_local(addr2); kunmap_local(addr1); return ret; } +int __weak memcmp_pages(struct page *page1, struct page *page2) +{ + return memcmp_pages_range(page1, 0, page2, 0, PAGE_SIZE); +} + #ifdef CONFIG_PRINTK /** * mem_dump_obj - Print available provenance information From a643a1f64e7a5d956db185a6bb34d100cd173632 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 05:44:23 -0400 Subject: [PATCH 0873/2680] arm64: preserve subpage offsets and lengths in MTE core dumps Size MTE tag segments from dumped bytes rather than a truncated native page count. For alternative userspace geometry, retain the selected fragment and save only its tags, preserving the physical subpage offset. Keep the native page path and share its assembly packing loop through a bounded tag-range helper. Add a disposable-guest core-file fixture checking actual ELF data and tag headers and bytes for split read-only, sparse and excluded mappings. Native and 4K processes pass on a 16K KASAN kernel; native, 4K and 16K processes pass on a 64K KASAN kernel. These are emulated MTE checks, not physical-hardware validation. Signed-off-by: Diljit Singh (cherry picked from commit 886de7154672d656b031d3ee5a1cda03b7f3168f) --- arch/arm64/include/asm/mte.h | 2 + arch/arm64/kernel/elfcore.c | 60 ++++- arch/arm64/lib/mte.S | 20 +- tools/testing/selftests/arm64/user4k/Makefile | 2 + .../selftests/arm64/user4k/mte-core-init.c | 206 ++++++++++++++++++ 5 files changed, 286 insertions(+), 4 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/mte-core-init.c diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h index 890d49a4398b2..c454f5ea8c3d1 100644 --- a/arch/arm64/include/asm/mte.h +++ b/arch/arm64/include/asm/mte.h @@ -25,6 +25,8 @@ unsigned long mte_copy_tags_to_user(void __user *to, void *from, unsigned long n); int mte_save_tags(struct page *page); void mte_save_page_tags(const void *page_addr, void *tag_storage); +/* Address and size must be aligned to 256 bytes; size may be zero. */ +void mte_save_tags_range(const void *addr, void *tag_storage, unsigned long size); void mte_restore_tags(swp_entry_t entry, struct page *page); void mte_restore_page_tags(void *page_addr, const void *tag_storage); void mte_invalidate_tags(int type, pgoff_t offset); diff --git a/arch/arm64/kernel/elfcore.c b/arch/arm64/kernel/elfcore.c index b735f4c2fe5ea..32ddc36838fe3 100644 --- a/arch/arm64/kernel/elfcore.c +++ b/arch/arm64/kernel/elfcore.c @@ -2,8 +2,10 @@ #include #include +#include #include #include +#include #include #include @@ -17,9 +19,60 @@ static unsigned long mte_vma_tag_dump_size(struct core_vma_metadata *m) { - return (m->dump_size >> PAGE_SHIFT) * MTE_PAGE_TAG_STORAGE; + return (m->dump_size / MTE_GRANULE_SIZE) * MTE_TAG_SIZE / 8; } +#ifdef CONFIG_MM_SUBPAGE +static int mte_dump_fragment_tags(struct coredump_params *cprm, + unsigned long start, unsigned long len) +{ + unsigned long size = mm_user_fragment_size(current->mm); + unsigned long tag_size = (size / MTE_GRANULE_SIZE) * MTE_TAG_SIZE / 8; + unsigned long addr; + void *tags = NULL; + int ret = 1; + + for (addr = start; addr < start + len; addr += size) { + struct user_page_fragment fragment; + struct page *page; + + if (get_dump_fragment(addr, &fragment) != 1) { + dump_skip(cprm, tag_size); + goto next; + } + page = folio_page(fragment.folio, fragment.offset >> PAGE_SHIFT); + if (!page_mte_tagged(page)) { + release_user_fragments(&fragment, 1, false); + dump_skip(cprm, tag_size); + goto next; + } + if (!tags) { + tags = mte_allocate_tag_storage(); + if (!tags) { + release_user_fragments(&fragment, 1, false); + ret = 0; + break; + } + } + mte_save_tags_range(folio_address(fragment.folio) + fragment.offset, tags, size); + release_user_fragments(&fragment, 1, false); + if (!dump_emit(cprm, tags, tag_size)) { + ret = 0; + break; + } +next: + if (fatal_signal_pending(current) || freezing(current)) { + ret = 0; + break; + } + cond_resched(); + } + if (tags) + mte_free_tag_storage(tags); + return ret; +} +#endif + /* Derived from dump_user_range(); start/end must be page-aligned */ static int mte_dump_tag_range(struct coredump_params *cprm, unsigned long start, unsigned long len) @@ -29,6 +82,11 @@ static int mte_dump_tag_range(struct coredump_params *cprm, void *tags = NULL; int locked = 0; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return mte_dump_fragment_tags(cprm, start, len); +#endif + for (addr = start; addr < start + len; addr += PAGE_SIZE) { struct page *page = get_dump_page(addr, &locked); diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S index c1ce370068cf4..8402e81b71474 100644 --- a/arch/arm64/lib/mte.S +++ b/arch/arm64/lib/mte.S @@ -183,6 +183,19 @@ SYM_FUNC_END(mte_copy_tags_to_user) * x1 - tag storage, MTE_PAGE_TAG_STORAGE bytes */ SYM_FUNC_START(mte_save_page_tags) + mov x2, #PAGE_SIZE + b mte_save_tags_range +SYM_FUNC_END(mte_save_page_tags) + +/* + * Save tags for a 256-byte-aligned range. + * x0 - start address + * x1 - tag storage, x2 / 32 bytes + * x2 - byte length, a multiple of 256 + */ +SYM_FUNC_START(mte_save_tags_range) + cbz x2, 3f + add x6, x0, x2 multitag_transfer_size x7, x5 1: mov x2, #0 @@ -195,11 +208,12 @@ SYM_FUNC_START(mte_save_page_tags) str x2, [x1], #8 - tst x0, #(PAGE_SIZE - 1) - b.ne 1b + cmp x0, x6 + b.lo 1b +3: ret -SYM_FUNC_END(mte_save_page_tags) +SYM_FUNC_END(mte_save_tags_range) /* * Restore the tags in a page diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index 0350be7a57912..889386ca79ffc 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -3,6 +3,7 @@ TEST_GEN_PROGS := page-contract proc-args TEST_GEN_PROGS_EXTENDED := iommufd-init coarse-shmem-init bpf-uptr-init vhost-log-init uprobes-init user-events-init uffd-move-init swap-fixture-init exec-fixture-init mixed-process memory-interfaces advice reclaim-advice proc-pages proc-memory core-fixture-init context-stress-init mlock-fixture-init lazyfree-fixture-init thread-fixture-init splice-fixture-init dio-fixture-init uring-fixture-init gem-fixture-init compat-fixture-init granule-proc-init granule-swap-init aslr-fixture-init cpu-granule-init user16k-smoke-init cpu-granule16-init granule-threeway-init compat-mlock-init mglru-granule-init memcg-pressure-init memcg-accounting-init runtime-app-init fex-contract-init exec-policy-init uffd-wp-init shmem-uffd-swap-init compat-uffd-init shmem-hole-init shmem-thp-init shmem-collapse-init shmem-pin-init shmem-race-init shmem-pressure-init TEST_GEN_PROGS_EXTENDED += vfio-fixture-init sysv-shm-init perf-ring-init TEST_GEN_PROGS_EXTENDED += aio-ring-init numa-policy-init numa-balancing-init cow-pool +TEST_GEN_PROGS_EXTENDED += mte-core-init CFLAGS += -Wall -Wextra -O2 -ffixed-x18 TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe compat-vfio-probe @@ -60,6 +61,7 @@ $(OUTPUT)/libfex-abi.so: fex-native-abi.c $(OUTPUT)/exec-policy-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 $(OUTPUT)/exec-policy-init: LDLIBS += -pthread +$(OUTPUT)/mte-core-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 $(OUTPUT)/compat-policy-probe: compat-policy-probe.c $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ diff --git a/tools/testing/selftests/arm64/user4k/mte-core-init.c b/tools/testing/selftests/arm64/user4k/mte-core-init.c new file mode 100644 index 0000000000000..3a6cf5e0723b7 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/mte-core-init.c @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* MTE core-file validation: PID 1 and its children in a disposable VM only. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef PT_AARCH64_MEMTAG_MTE +#define PT_AARCH64_MEMTAG_MTE (PT_LOPROC + 0x2) +#endif +#define TAGGED_BASE 0x200000000UL +#define ZERO_BASE 0x210000000UL +#define EXCLUDED_BASE 0x220000000UL +#define REQUIRE(condition) do { \ + if (!(condition)) { \ + fprintf(stderr, "FAIL line %d: %s errno=%d\n", __LINE__, #condition, errno); \ + exit(1); \ + } \ +} while (0) + +static void write_setting(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + + REQUIRE(fd >= 0); + REQUIRE(write(fd, value, strlen(value)) == (ssize_t)strlen(value)); + REQUIRE(!close(fd)); +} + +static unsigned char *tagged_map(unsigned long address, size_t size) +{ + void *p = mmap((void *)address, size, PROT_READ | PROT_WRITE | PROT_MTE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + + REQUIRE(p == (void *)address); + return p; +} + +static void set_tags(unsigned char *p, size_t size, unsigned int tag) +{ + for (size_t i = 0; i < size; i += 16) { + uintptr_t address = (uintptr_t)(p + i) | ((uintptr_t)tag << 56); + + asm volatile(".arch_extension memtag\n\tstg %0, [%0]" + : : "r" (address) : "memory"); + } +} + +static void child(unsigned long ps) +{ + struct stat self, init; + struct rlimit limit = { 64UL << 20, 64UL << 20 }; + unsigned char *p; + + REQUIRE(getppid() == 1); + REQUIRE(!stat("/proc/self/exe", &self) && !stat("/proc/1/exe", &init)); + REQUIRE(self.st_dev == init.st_dev && self.st_ino == init.st_ino); + REQUIRE(getauxval(AT_PAGESZ) == ps); + REQUIRE(!setrlimit(RLIMIT_CORE, &limit)); + REQUIRE(!prctl(PR_SET_DUMPABLE, 1UL, 0UL, 0UL, 0UL)); + REQUIRE(!prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE, 0UL, 0UL, 0UL)); + write_setting("/proc/self/coredump_filter", "1"); + p = tagged_map(TAGGED_BASE, 3 * ps); + for (unsigned int i = 0; i < 3; i++) { + memset(p + i * ps, 0x40 + i, ps); + set_tags(p + i * ps, ps, i + 1); + } + /* Split metadata at a non-native boundary and retain the physical offset. */ + REQUIRE(!mprotect(p + ps, ps, PROT_READ | PROT_MTE)); + p = tagged_map(ZERO_BASE, ps); + *p = 0; + /* Keep anonymous dump eligibility while leaving the leaf absent. */ + REQUIRE(!madvise(p, ps, MADV_DONTNEED)); + p = tagged_map(EXCLUDED_BASE, ps); + memset(p, 0x55, ps); + set_tags(p, ps, 7); + REQUIRE(!madvise(p, ps, MADV_DONTDUMP)); + raise(SIGABRT); + _exit(2); +} + +static void inspect_core(pid_t pid, unsigned long ps) +{ + char path[80]; + Elf64_Ehdr eh; + struct stat st; + unsigned int tagged = 0, loads = 0, zero = 0, excluded = 0; + unsigned char *bytes = malloc(ps); + int fd; + + REQUIRE(bytes); + snprintf(path, sizeof(path), "/tmp/core.%d", pid); + fd = open(path, O_RDONLY); + REQUIRE(fd >= 0 && !fstat(fd, &st)); + REQUIRE(pread(fd, &eh, sizeof(eh), 0) == sizeof(eh)); + REQUIRE(!memcmp(eh.e_ident, ELFMAG, SELFMAG)); + REQUIRE(eh.e_ident[EI_CLASS] == ELFCLASS64 && eh.e_type == ET_CORE); + REQUIRE(eh.e_phentsize == sizeof(Elf64_Phdr) && eh.e_phnum < 4096); + for (unsigned int n = 0; n < eh.e_phnum; n++) { + Elf64_Phdr ph; + unsigned char expected; + unsigned int index; + size_t length; + + REQUIRE(pread(fd, &ph, sizeof(ph), eh.e_phoff + n * sizeof(ph)) == sizeof(ph)); + if (ph.p_type != PT_LOAD && ph.p_type != PT_AARCH64_MEMTAG_MTE) + continue; + if (ph.p_vaddr >= TAGGED_BASE && ph.p_vaddr < TAGGED_BASE + 3 * ps) { + index = (ph.p_vaddr - TAGGED_BASE) / ps; + REQUIRE(ph.p_vaddr == TAGGED_BASE + index * ps && ph.p_memsz == ps); + if (ph.p_type == PT_LOAD) { + loads++; + length = ps; + expected = 0x40 + index; + } else { + tagged++; + length = ps / 32; + expected = (index + 1) * 0x11; + } + } else if (ph.p_vaddr == ZERO_BASE && ph.p_type == PT_AARCH64_MEMTAG_MTE) { + zero++; + length = ps / 32; + expected = 0; + REQUIRE(ph.p_memsz == ps); + } else if (ph.p_vaddr == EXCLUDED_BASE && ph.p_type == PT_AARCH64_MEMTAG_MTE) { + excluded++; + REQUIRE(ph.p_filesz == 0 && ph.p_memsz == ps); + continue; + } else { + continue; + } + if (ph.p_filesz != length) + fprintf(stderr, "segment type=%#x address=%#lx size=%lu expected=%zu\n", + ph.p_type, (unsigned long)ph.p_vaddr, + (unsigned long)ph.p_filesz, length); + REQUIRE(ph.p_filesz == length); + REQUIRE(ph.p_offset <= (uint64_t)st.st_size && length <= st.st_size - ph.p_offset); + REQUIRE(pread(fd, bytes, length, ph.p_offset) == (ssize_t)length); + for (size_t i = 0; i < length; i++) + REQUIRE(bytes[i] == expected); + } + REQUIRE(tagged == 3 && loads == 3 && zero == 1 && excluded == 1); + REQUIRE(!close(fd) && !unlink(path)); + free(bytes); + printf("MTE CORE CASE PASS page-size=%lu\n", ps); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + unsigned long sizes[] = { native, 4096, 16384 }; + unsigned int count = native > 16384 ? 3 : native > 4096 ? 2 : 1; + + setbuf(stdout, NULL); + setbuf(stderr, NULL); + if (argc == 3 && !strcmp(argv[1], "--child")) { + child(strtoul(argv[2], NULL, 10)); + return 1; + } + if (getpid() != 1) { + fprintf(stderr, "SKIP: disposable guest PID1 only\n"); + return 4; + } + REQUIRE(getauxval(AT_HWCAP2) & HWCAP2_MTE); + mkdir("/proc", 0755); + mkdir("/tmp", 0700); + REQUIRE(!mount("proc", "/proc", "proc", 0, NULL)); + REQUIRE(!mount("tmpfs", "/tmp", "tmpfs", 0, "mode=0700")); + write_setting("/proc/sys/kernel/core_pattern", "/tmp/core.%p"); + for (unsigned int i = 0; i < count; i++) { + pid_t pid = fork(); + int status; + + REQUIRE(pid >= 0); + if (!pid) { + char value[32]; + + snprintf(value, sizeof(value), "%lu", sizes[i]); + REQUIRE(!prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)); + execl("/init", "/init", "--child", value, NULL); + _exit(3); + } + REQUIRE(waitpid(pid, &status, 0) == pid); + REQUIRE(WIFSIGNALED(status) && WTERMSIG(status) == SIGABRT && WCOREDUMP(status)); + inspect_core(pid, sizes[i]); + } + printf("MTE CORE PASS cases=%u\n", count); + reboot(RB_POWER_OFF); + return 0; +} From 63c9ed6d999c41b4030e872a1dccc2d32ad26a43 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 05:52:13 -0400 Subject: [PATCH 0874/2680] mm: distinguish callback-supplied pages from page-cache faults A VMA can retain vm_file while its fault callback returns independent pages. Use cache-indexed fault installation only when the returned folio belongs to that file mapping; otherwise preserve the callback page and its requested subpage offset. Allow ordinary non-anonymous fragment references to follow these mappings without requiring page-cache membership. Keep permission checks and private read-pin COW rules unchanged. Add native, 4K and supported 16K callback tests with reversed backing pages, nonzero offsets and remote reads. They pass on 16K/64K KASAN guests, along with existing file-fragment lifetime/COW and special-mapping tests. This tests the callback contract, not a physical usbmon device. Signed-off-by: Diljit Singh (cherry picked from commit 09dae19a275153e1bbfe13e4b31f3fa8d94f72e8) --- arch/arm64/mm/user4k-test.c | 81 +++++++++++++++++++++++++++++++++++++ mm/gup.c | 4 -- mm/memory.c | 4 +- 3 files changed, 83 insertions(+), 6 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 209bcbe7305a8..e3a9946183e8e 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -5439,6 +5439,84 @@ static void user4k_special_mapping_test(struct kunit *test) } } +static vm_fault_t test_callback_fault(struct vm_fault *vmf) +{ + struct page **pages = vmf->vma->vm_private_data; + + if (vmf->pgoff >= 2) + return VM_FAULT_SIGBUS; + + vmf->page = pages[vmf->pgoff]; + get_page(vmf->page); + return 0; +} + +static const struct vm_operations_struct test_callback_vm_ops = { + .fault = test_callback_fault, +}; + +static void test_callback_mapping(struct kunit *test, unsigned int shift) +{ + void *data[2] = { test_alloc_page(test), test_alloc_page(test) }; + struct page **pages = kunit_kcalloc(test, 2, sizeof(*pages), GFP_KERNEL); + struct file *file = test_cache_file(test, 0, false); + struct vm_area_struct *vma; + unsigned long size = 1UL << shift; + unsigned long offset = shift == PAGE_SHIFT ? 0 : size; + unsigned long len = 2 * PAGE_SIZE - offset; + unsigned long addr; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, data[0]); + KUNIT_ASSERT_NOT_NULL(test, data[1]); + KUNIT_ASSERT_NOT_NULL(test, pages); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + pages[0] = virt_to_page(data[1]); + pages[1] = virt_to_page(data[0]); + for (i = 0; i < 2 * PAGE_SIZE / size; i++) + memset(page_address(pages[(i * size) >> PAGE_SHIFT]) + + offset_in_page(i * size), 0x51 + i, size); + + /* The file identifies the VMA; the callback supplies independent pages. */ + vma = test_file_vma_granule(test, file, shift, false, TEST_VA, len, offset); + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_write_lock(vma->vm_mm); + vma->vm_ops = &test_callback_vm_ops; + vma->vm_private_data = pages; + mmap_write_unlock(vma->vm_mm); + for (addr = vma->vm_start; addr < vma->vm_end; addr += size) { + unsigned long source = offset + addr - vma->vm_start; + u8 byte; + + KUNIT_ASSERT_EQ(test, test_file_fault(vma, addr, false) & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, + test_fault_entry(test, vma->vm_mm, addr)), + page_to_phys(pages[source >> PAGE_SHIFT]) + offset_in_page(source)); + KUNIT_ASSERT_EQ(test, access_remote_vm(vma->vm_mm, addr + 7, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)(0x51 + source / size)); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_FILEPAGES), len / size); +} + +static void native_callback_mapping_test(struct kunit *test) +{ + test_callback_mapping(test, PAGE_SHIFT); +} + +static void user4k_callback_mapping_test(struct kunit *test) +{ + test_callback_mapping(test, 12); +} + +static void user16k_callback_mapping_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative requires a larger native granule"); + return; + } + test_callback_mapping(test, 14); +} + static void user4k_insert_range_test(struct kunit *test) { unsigned int native; @@ -14117,6 +14195,9 @@ static struct kunit_case user4k_test_cases[] = { KUNIT_CASE(user4k_stack_relocate_test), KUNIT_CASE(user4k_exec_arguments_test), KUNIT_CASE(user4k_special_mapping_test), + KUNIT_CASE(native_callback_mapping_test), + KUNIT_CASE(user4k_callback_mapping_test), + KUNIT_CASE(user16k_callback_mapping_test), KUNIT_CASE(user4k_insert_fragment_test), KUNIT_CASE(user16k_insert_shared_test), KUNIT_CASE(user4k_insert_range_test), diff --git a/mm/gup.c b/mm/gup.c index 001e804f4e38c..b4da92aaa64f2 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -3527,10 +3527,6 @@ static int follow_subpage_fragment(struct vm_area_struct *vma, unsigned long add goto acquired; } if (!folio_test_anon(folio)) { - if (vma->vm_file && folio_mapping(folio) != vma->vm_file->f_mapping) { - ret = -EOPNOTSUPP; - goto out; - } if ((flags & FOLL_WRITE) && !pte_write(pte)) goto out; /* Private non-anonymous read pins must first acquire COW ownership. */ diff --git a/mm/memory.c b/mm/memory.c index e648f92233fd9..e3b1c17dcf91a 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -7061,7 +7061,7 @@ vm_fault_t map_file_user_pages(struct vm_fault *vmf, struct folio *folio, } /* - * A non-file fault callback supplies one native page, not a cache-indexed + * A non-page-cache fault callback supplies one native page, not a cache-indexed * folio. In particular, VVAR callbacks may reorder native pages. Only map * quarters of this supplied page; never infer adjacent pages from its folio. */ @@ -7197,7 +7197,7 @@ static vm_fault_t finish_user_page_fault(struct vm_fault *vmf) inc_mm_counter(mm, MM_ANONPAGES); ret = 0; } else { - if (vma->vm_file) + if (vma->vm_file && folio_mapping(folio) == vma->vm_file->f_mapping) ret = map_file_user_pages(vmf, folio, folio->index, folio_next_index(folio) - 1, true); else if (mm_page_size(mm) < PAGE_SIZE) From 2870d939aec4982ebbaaae84eb6eccaef082d37c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 05:58:29 -0400 Subject: [PATCH 0875/2680] mm: clip mapping walks to exact VMA byte intervals walk_page_mapping() truncates a VMA length to native pages and ignores its subpage file offset. Short mappings can disappear from the walk and partial first/last native pages can produce incorrect virtual bounds. Use the existing byte-offset translation and last-index helpers to clip the requested native file indices to the VMA. Handle an empty request and reject an overflowing index interval before the interval-tree walk. Validated native and smaller-granule walks over short mappings and partial first/last pages in 16K/64K KASAN guests. The temporary fixture and results remain in the local validation records instead of adding another permanent arm64 test block. The generic x86 pagewalk object also compiles. Signed-off-by: Diljit Singh (cherry picked from commit 0a085675da3ec34e40f94c8d9af6194cfd133c91) --- mm/pagewalk.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/mm/pagewalk.c b/mm/pagewalk.c index 8d569e5c45cae..278cfb38afe35 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -807,27 +808,26 @@ int walk_page_mapping(struct address_space *mapping, pgoff_t first_index, .private = private, }; struct vm_area_struct *vma; - pgoff_t vba, vea, cba, cea; + pgoff_t last_index; unsigned long start_addr, end_addr; int err = 0; if (!check_ops_safe(ops)) return -EINVAL; + if (!nr) + return 0; + if (check_add_overflow(first_index, nr - 1, &last_index)) + return -EINVAL; lockdep_assert_held(&mapping->i_mmap_rwsem); vma_interval_tree_foreach(vma, &mapping->i_mmap, first_index, - first_index + nr - 1) { - /* Clip to the vma */ - vba = vma->vm_pgoff; - vea = vba + vma_pages(vma); - cba = first_index; - cba = max(cba, vba); - cea = first_index + nr; - cea = min(cea, vea); - - start_addr = ((cba - vba) << PAGE_SHIFT) + vma->vm_start; - end_addr = ((cea - vba) << PAGE_SHIFT) + vma->vm_start; - if (start_addr >= end_addr) + last_index) { + /* Clip native file indices to the VMA's exact byte interval. */ + start_addr = first_index <= vma->vm_pgoff ? vma->vm_start : + vma_address_at_offset(vma, (struct vm_page_offset) { first_index, 0 }); + end_addr = last_index >= vma_last_pgoff(vma) ? vma->vm_end : + vma_address_at_offset(vma, (struct vm_page_offset) { last_index + 1, 0 }); + if (IS_ERR_VALUE(start_addr) || IS_ERR_VALUE(end_addr) || start_addr >= end_addr) continue; walk.vma = vma; From a17e1b52418c7fa60b2358b06804aa31f5212c0e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 06:02:48 -0400 Subject: [PATCH 0876/2680] mm: round up mapping lengths when checking I/O resource bounds A mapping can be shorter than a native page when its MM uses a smaller granule. Count a partial final native page in __simple_ioremap_prep() instead of truncating it away. Native-aligned mappings retain their existing calculation, and the quotient/remainder form avoids a rounding addition overflow. Both 16K/64K KASAN kernels build and pass the existing PFN-remap suite; the x86 memory object also builds. No device-boundary reproducer was run. This bounds correction does not qualify every device offset interface. Signed-off-by: Diljit Singh (cherry picked from commit d4e50b58b49627c95920d2f106da600ead1254ff) --- mm/memory.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/memory.c b/mm/memory.c index e3b1c17dcf91a..3b32187dc0267 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3697,7 +3697,7 @@ static int __simple_ioremap_prep(unsigned long vm_len, pgoff_t vm_pgoff, pages -= vm_pgoff; /* Can we fit all of the mapping? */ - if ((vm_len >> PAGE_SHIFT) > pages) + if ((vm_len >> PAGE_SHIFT) + !!offset_in_page(vm_len) > pages) return -EINVAL; *pfnp = pfn; From f07860340e82f91e29b7504920c0d28e2a1fcbf1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 06:07:24 -0400 Subject: [PATCH 0877/2680] arm64: retain exact fragments for remote MTE tag access The legacy remote-page interface cannot represent smaller-granule backing. Use the stable-VMA fragment interface for alternative MMs, take the actual physical offset from the returned record, and bound each transfer by its length. Release the same retained fragment and dirty its ownership record on writes. Preserve the native page path and existing ptrace permission and partial-transfer semantics. Reused the local core-file fixture to validate own-child ptrace tag reads, writes and restoration across three differently tagged mappings, including a read-only mapping. Native/4K cases pass in a 16K KASAN guest and native/4K/16K cases pass in a 64K guest; resulting core files also pass. The feature-disabled native MTE object builds. No additional submitted test program is introduced, and physical MTE hardware remains untested. Signed-off-by: Diljit Singh (cherry picked from commit f41afd84696d9b7381b25f105f58caa2c0590365) --- arch/arm64/kernel/mte.c | 44 +++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c index 8c73cf2336313..9032b307a06e2 100644 --- a/arch/arm64/kernel/mte.c +++ b/arch/arm64/kernel/mte.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -464,11 +465,24 @@ static int __access_remote_tags(struct mm_struct *mm, unsigned long addr, while (len) { struct vm_area_struct *vma; - unsigned long tags, offset; + unsigned long tags = 0, offset; void *maddr; - struct page *page = get_user_page_vma_remote(mm, addr, - gup_flags, &vma); + struct page *page; struct folio *folio; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment = {}; + + if (mm_page_size(mm) != PAGE_SIZE) { + unsigned long bytes = min(len, PAGE_SIZE / MTE_GRANULE_SIZE) * + MTE_GRANULE_SIZE; + int got = get_user_fragment_vma_remote(mm, addr, bytes, + gup_flags, &fragment, &vma); + + page = got > 0 ? folio_page(fragment.folio, + fragment.offset >> PAGE_SHIFT) : ERR_PTR(got ?: -EFAULT); + } else +#endif + page = get_user_page_vma_remote(mm, addr, gup_flags, &vma); if (IS_ERR(page)) { err = PTR_ERR(page); @@ -484,8 +498,7 @@ static int __access_remote_tags(struct mm_struct *mm, unsigned long addr, */ if (!(vma->vm_flags & VM_MTE)) { err = -EIO; - put_page(page); - break; + goto release; } folio = page_folio(page); @@ -498,18 +511,33 @@ static int __access_remote_tags(struct mm_struct *mm, unsigned long addr, /* limit access to the end of the page */ offset = offset_in_page(addr); tags = min(len, (PAGE_SIZE - offset) / MTE_GRANULE_SIZE); +#ifdef CONFIG_MM_SUBPAGE + if (fragment.folio) { + offset = offset_in_page(fragment.offset); + tags = fragment.length / MTE_GRANULE_SIZE; + } +#endif maddr = page_address(page); if (write) { tags = mte_copy_tags_from_user(maddr + offset, buf, tags); - set_page_dirty_lock(page); +#ifdef CONFIG_MM_SUBPAGE + if (!fragment.folio) +#endif + set_page_dirty_lock(page); } else { tags = mte_copy_tags_to_user(buf, maddr + offset, tags); } - put_page(page); +release: +#ifdef CONFIG_MM_SUBPAGE + if (fragment.folio) + release_user_fragments(&fragment, 1, write && !err); + else +#endif + put_page(page); /* error accessing the tracer's buffer */ - if (!tags) + if (err || !tags) break; len -= tags; From 041fcbd13cb9cc4ab784c61b4f53b2b998de1d8f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 06:18:24 -0400 Subject: [PATCH 0878/2680] iommu/arm-smmu-v3: use the bound mm granule for SVA Build the SVA context descriptor and invalidation geometry from the bound MM rather than PAGE_SIZE. Check the target device's granule support at allocation and attachment, and record the same granule in the domain so range invalidation commands agree with the CPU page-table geometry. Extend the existing SVA descriptor transition test to cover the enabled alternative granules. The existing 25-case SMMUv3 suite passes in both 16K and 64K KASAN guests; native feature-disabled SVA and test objects also compile. Actual SVA device DMA and invalidation delivery remain untested. The driver's existing global native-granule capability prerequisite is preserved. Signed-off-by: Diljit Singh (cherry picked from commit 4ccea57e713c22800c059cad4662abef494dd054) --- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 16 ++++++++----- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c | 23 +++++++++++++++---- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 4cd60d3c450ac..1b929c115c6f3 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -36,13 +36,13 @@ arm_smmu_update_s1_domain_cd_entry(struct arm_smmu_domain *smmu_domain) spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); } -static u64 page_size_to_cd(void) +static u64 page_size_to_cd(unsigned long page_size) { static_assert(PAGE_SIZE == SZ_4K || PAGE_SIZE == SZ_16K || PAGE_SIZE == SZ_64K); - if (PAGE_SIZE == SZ_64K) + if (page_size == SZ_64K) return ARM_LPAE_TCR_TG0_64K; - if (PAGE_SIZE == SZ_16K) + if (page_size == SZ_16K) return ARM_LPAE_TCR_TG0_16K; return ARM_LPAE_TCR_TG0_4K; } @@ -83,7 +83,7 @@ void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, target->data[0] |= cpu_to_le64( FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, 64ULL - vabits_actual) | - FIELD_PREP(CTXDESC_CD_0_TCR_TG0, page_size_to_cd()) | + FIELD_PREP(CTXDESC_CD_0_TCR_TG0, page_size_to_cd(mm_page_size(mm))) | FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, ARM_LPAE_TCR_RGN_WBWA) | FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, @@ -149,7 +149,7 @@ static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, */ size = end - start; - arm_smmu_domain_inv_range(smmu_domain, start, size, PAGE_SIZE, false); + arm_smmu_domain_inv_range(smmu_domain, start, size, mm_page_size(mm), false); } static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) @@ -270,6 +270,8 @@ static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain, if (!(master->smmu->features & ARM_SMMU_FEAT_SVA)) return -EOPNOTSUPP; + if (!(master->smmu->pgsize_bitmap & mm_page_size(domain->mm))) + return -EOPNOTSUPP; /* Prevent arm_smmu_mm_release from being called while we are attaching */ if (!mmget_not_zero(domain->mm)) @@ -326,6 +328,8 @@ struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev, if (!(master->smmu->features & ARM_SMMU_FEAT_SVA)) return ERR_PTR(-EOPNOTSUPP); + if (!(smmu->pgsize_bitmap & mm_page_size(mm))) + return ERR_PTR(-EOPNOTSUPP); smmu_domain = arm_smmu_domain_alloc(); if (IS_ERR(smmu_domain)) @@ -337,7 +341,7 @@ struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev, * Choose page_size as the leaf page size for invalidation when * ARM_SMMU_FEAT_RANGE_INV is present */ - smmu_domain->domain.pgsize_bitmap = PAGE_SIZE; + smmu_domain->domain.pgsize_bitmap = mm_page_size(mm); smmu_domain->stage = ARM_SMMU_DOMAIN_SVA; smmu_domain->smmu = smmu; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c index add671363c828..fa1c7b03394d6 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c @@ -616,12 +616,25 @@ static void arm_smmu_v3_write_cd_test_sva_clear(struct kunit *test) { struct arm_smmu_cd cd = {}; struct arm_smmu_cd cd_2; + const unsigned long flags[] = { 0, MMCF_USER_4K, MMCF_USER_16K }; + /* TG0 encodings: 4K = 0, 64K = 1, 16K = 2. */ + const unsigned int tg[] = { PAGE_SHIFT == 16 ? 1 : PAGE_SHIFT == 14 ? 2 : 0, 0, 2 }; + unsigned int i; - arm_smmu_test_make_sva_cd(&cd_2, 1997); - arm_smmu_v3_test_cd_expect_non_hitless_transition( - test, &cd, &cd_2, NUM_EXPECTED_SYNCS(2)); - arm_smmu_v3_test_cd_expect_non_hitless_transition( - test, &cd_2, &cd, NUM_EXPECTED_SYNCS(2)); + for (i = 0; i < ARRAY_SIZE(flags); i++) { + if (i && (!IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) || + (i == 2 && PAGE_SHIFT <= 14))) + continue; + sva_mm.context.flags = flags[i]; + arm_smmu_test_make_sva_cd(&cd_2, 1997); + KUNIT_EXPECT_EQ(test, FIELD_GET(CTXDESC_CD_0_TCR_TG0, + le64_to_cpu(cd_2.data[0])), tg[i]); + arm_smmu_v3_test_cd_expect_non_hitless_transition( + test, &cd, &cd_2, NUM_EXPECTED_SYNCS(2)); + arm_smmu_v3_test_cd_expect_non_hitless_transition( + test, &cd_2, &cd, NUM_EXPECTED_SYNCS(2)); + } + sva_mm.context.flags = 0; } static void arm_smmu_v3_write_cd_test_sva_release(struct kunit *test) From 21abc4ee129b46c894f6c2704d4fb963b5c7aef5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 06:21:46 -0400 Subject: [PATCH 0879/2680] ocfs2: use truncate_pagecache after shrinking an extent-backed file After ocfs2_orphan_for_truncate() updates i_size, use the generic pagecache truncation helper. Its invalidation covers smaller userspace leaves in the last native page and repeats after pagecache truncation to remove COW mappings created during the operation. Keep the inline-data path's ordering unchanged. OCFS2 blocks are at most 4K, so inline data cannot span multiple leaves of the exposed alternative userspace configurations. The changed OCFS2 file object builds for 16K and 64K arm64 configurations. A mounted OCFS2 filesystem has not been runtime-tested for this change. Signed-off-by: Diljit Singh (cherry picked from commit a6c5a0042f88694ebedb8b622cd0ca17896ba474) --- fs/ocfs2/file.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c index d6e977ba65656..d12e9a964f7cd 100644 --- a/fs/ocfs2/file.c +++ b/fs/ocfs2/file.c @@ -509,8 +509,7 @@ int ocfs2_truncate_file(struct inode *inode, goto bail_unlock_sem; } - unmap_mapping_range(inode->i_mapping, new_i_size + PAGE_SIZE - 1, 0, 1); - truncate_inode_pages(inode->i_mapping, new_i_size); + truncate_pagecache(inode, new_i_size); status = ocfs2_commit_truncate(osb, inode, di_bh); if (status < 0) { From 285806769d1b5ded637e098b19dab2742924db48 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 06:35:39 -0400 Subject: [PATCH 0880/2680] dma-buf: retain subpage offsets in CMA and udmabuf faults MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CMA and udmabuf fault handlers identify the native backing page but lose the smaller leaf's byte offset. Pass the explicit file subpage offset to PFN insertion. In udmabuf, use the same translation for demand faults and prefaults, advancing by the target MM granule so prefault coverage remains complete. Validated CPU mappings with the existing dma-buf heap and udmabuf selftests in 16K/64K KASAN guests for native, 4K and supported 16K processes. The local udmabuf test uses native-aligned buffer sizes and distinct per-leaf bytes; its two hugetlb cases are explicitly skipped without a reserved pool. Both CMA heap names and vgem import pass. No additional test source is submitted and physical device DMA remains unqualified. [ryan: adapted to 7.2.10 — minimal; for DJ: udmabuf keeps a page array in 7.2 (5bf888673e0d), so the fault helper takes the PFN from ubuf->pages[] instead of folios[] plus offsets[]] Signed-off-by: Diljit Singh (cherry picked from commit ac963beadc4754a08163d643e26b23f0d1c31bd5) --- drivers/dma-buf/heaps/cma_heap.c | 5 ++++- drivers/dma-buf/udmabuf.c | 35 ++++++++++++++++---------------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/drivers/dma-buf/heaps/cma_heap.c b/drivers/dma-buf/heaps/cma_heap.c index 3fb4b946c91a6..1d7c0a05b3fba 100644 --- a/drivers/dma-buf/heaps/cma_heap.c +++ b/drivers/dma-buf/heaps/cma_heap.c @@ -174,7 +174,10 @@ static vm_fault_t cma_heap_vm_fault(struct vm_fault *vmf) if (vmf->pgoff >= buffer->pagecount) return VM_FAULT_SIGBUS; - return vmf_insert_pfn(vma, vmf->address, page_to_pfn(buffer->pages[vmf->pgoff])); + return vmf_insert_pfn_prot_mkwrite_offset(vma, vmf->address, + page_to_pfn(buffer->pages[vmf->pgoff]), + vma_page_offset_at(vma, vmf->address).offset, + vma->vm_page_prot, false); } static const struct vm_operations_struct dma_heap_vm_ops = { diff --git a/drivers/dma-buf/udmabuf.c b/drivers/dma-buf/udmabuf.c index 08f57bc1294df..a67c160dcf87f 100644 --- a/drivers/dma-buf/udmabuf.c +++ b/drivers/dma-buf/udmabuf.c @@ -44,43 +44,44 @@ struct udmabuf { struct miscdevice *device; }; -static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf) +static vm_fault_t udmabuf_insert_pfn(struct vm_area_struct *vma, unsigned long addr) { - struct vm_area_struct *vma = vmf->vma; struct udmabuf *ubuf = vma->vm_private_data; - pgoff_t pgoff = vmf->pgoff; - unsigned long addr, pfn; - vm_fault_t ret; + struct vm_page_offset pos = vma_page_offset_at(vma, addr); + unsigned long pfn; - if (pgoff >= ubuf->pagecount) + if (pos.index >= ubuf->pagecount) return VM_FAULT_SIGBUS; - pfn = page_to_pfn(ubuf->pages[pgoff]); + pfn = page_to_pfn(ubuf->pages[pos.index]); + return vmf_insert_pfn_prot_mkwrite_offset(vma, addr, pfn, pos.offset, + vma->vm_page_prot, false); +} + +static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + unsigned long addr; + vm_fault_t ret; - ret = vmf_insert_pfn(vma, vmf->address, pfn); + ret = udmabuf_insert_pfn(vma, vmf->address); if (ret & VM_FAULT_ERROR) return ret; /* pre fault */ - pgoff = vma->vm_pgoff; addr = vma->vm_start; - for (; addr < vma->vm_end; pgoff++, addr += PAGE_SIZE) { + for (; addr < vma->vm_end; addr += mm_page_size(vma->vm_mm)) { if (addr == vmf->address) continue; - if (WARN_ON(pgoff >= ubuf->pagecount)) - break; - - pfn = page_to_pfn(ubuf->pages[pgoff]); - /** - * If the below vmf_insert_pfn() fails, we do not return an + * If the below insertion fails, we do not return an * error here during this pre-fault step. However, an error * will be returned if the failure occurs when the addr is * truly accessed. */ - if (vmf_insert_pfn(vma, addr, pfn) & VM_FAULT_ERROR) + if (udmabuf_insert_pfn(vma, addr) & VM_FAULT_ERROR) break; } From 04576c421ff8fbe29976a0bbcccaceb78c9e6724 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 06:51:16 -0400 Subject: [PATCH 0881/2680] arm64/mm: preserve neighboring subpage dirty state when clearing PTEs Contiguous-PTE access/dirty clearing can expand to the full group because native MM tracks those bits per folio. Smaller userspace leaves can be reclaimed independently, so extending a dirty clear can erase evidence belonging to a different retained fragment. Unfold partially covered groups for smaller-granule dirty clearing and operate on the requested entries only. Keep native handling, full-group operations and the existing folding policy. Update the existing contiguous-PTE expectations for fragment isolation. The lazyfree migration case accepts dirty retention because migration can refold a group and conservatively aggregate a neighbor's dirty state; it still rejects a leftover migration entry or clean retained slot. The existing contiguous-PTE and lazyfree suites pass on 16K/64K KASAN kernels. No data-loss reproducer was run. Signed-off-by: Diljit Singh (cherry picked from commit 425488825ee3b258100181b75295215cf50397fe) --- arch/arm64/mm/contpte.c | 12 +++++++++--- arch/arm64/mm/user4k-test.c | 20 ++++++++++++++++---- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c index e78e0d4265218..b3fcca267148b 100644 --- a/arch/arm64/mm/contpte.c +++ b/arch/arm64/mm/contpte.c @@ -614,6 +614,15 @@ void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, unsigned int nr, cydp_t flags) { struct mm_struct *mm = vma->vm_mm; + unsigned long start = addr; + unsigned long end = start + nr * mm_page_size(mm); + + /* Smaller leaves may be discarded independently within the folio. */ + if (mm_page_size(mm) < PAGE_SIZE && (flags & CYDP_CLEAR_DIRTY)) { + contpte_try_unfold_partial(mm, addr, ptep, nr); + __clear_young_dirty_ptes(vma, addr, ptep, nr, flags); + return; + } /* * We can safely clear access/dirty without needing to unfold from @@ -625,9 +634,6 @@ void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, * when it is covered by a single folio, we can get away with * clearing access/dirty for the whole block. */ - unsigned long start = addr; - unsigned long end = start + nr * mm_page_size(mm); - ptep = contpte_align_addr_ptep(vma->vm_mm, &start, &end, ptep, nr); __clear_young_dirty_ptes(vma, start, ptep, (end - start) >> mm_page_shift(mm), flags); } diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index e3a9946183e8e..68c7f9dad43d4 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -7773,14 +7773,19 @@ static void test_contpte_granule(struct kunit *test, unsigned int shift) CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY); for (i = 0; i < f->nr; i++) { observed = __ptep_get(pte + i); - ok.aged &= !pte_young(observed) && !pte_dirty(observed) && pte_cont(observed); + if (size < PAGE_SIZE) + ok.aged &= !pte_cont(observed) && + pte_young(observed) == (i != 3) && pte_dirty(observed) == (i != 3); + else + ok.aged &= !pte_young(observed) && !pte_dirty(observed) && + pte_cont(observed); } observed = pte_mkdirty(pte_mkyoung(__ptep_get(pte + 3))); access_same_ret = ptep_set_access_flags(&vma, TEST_VA + 3 * size, pte + 3, observed, 1); for (i = 0; i < f->nr; i++) { observed = __ptep_get(pte + i); - ok.access_all &= pte_cont(observed) && pte_young(observed) && + ok.access_all &= pte_cont(observed) == (size >= PAGE_SIZE) && pte_young(observed) && pte_dirty(observed) && !pte_write(observed); } access_noop_ret = ptep_set_access_flags(&vma, TEST_VA + 3 * size, @@ -9611,9 +9616,16 @@ static void user4k_lazyfree_test(struct kunit *test) folio_put(folio); for (i = 0; i < 4; i++) { pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); - KUNIT_EXPECT_EQ(test, pte_none(pte), + /* Migration may refold dirty neighbors and retain an advised slot. */ + if (mode == 5 && !i && pte_present(pte)) { + KUNIT_EXPECT_TRUE(test, pte_dirty(pte)); + continue; + } + KUNIT_EXPECT_EQ_MSG(test, pte_none(pte), (mode == 4 || mode == 9 || mode == 10) || - (i == 0 && (mode == 0 || mode == 5))); + (i == 0 && (mode == 0 || mode == 5)), + "mode=%u slot=%u pte=%llx", mode, i, + (unsigned long long)pte_val(pte)); } if (mode == 2 || mode == 6 || mode == 7) release_user_fragments(&held, 1, false); From 0012dd31e60915389b76b8bc38195f370d781239 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 06:58:30 -0400 Subject: [PATCH 0882/2680] mm: determine swap allocation requirements before taking the local lock The minimum anonymous folio order lookup temporarily holds an anon-vma reference. Dropping its last reference may sleep, so eligibility checks must not repeat that lookup under the per-CPU swap-cluster local lock. Reuse the mandatory-allocation decision already calculated by folio_alloc_swap() and pass it through the fast and slow allocation paths. The required fallback uses the same decision; hibernation has no folio. Document the lookup's sleepable context. 16K/64K KASAN builds and the existing swap/swapoff and UFFD move contract checks pass. The x86 swapfile object compiles. These checks do not reproduce the destructor lifetime race or qualify regular-file swap transport. Signed-off-by: Diljit Singh (cherry picked from commit 5f06041cbb3a178674d5aae5eb40483980175fca) --- mm/rmap.c | 5 ++++- mm/swapfile.c | 29 ++++++++++++++++------------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/mm/rmap.c b/mm/rmap.c index 05f041419c3a5..de8bed44b5f20 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -633,7 +633,10 @@ struct anon_vma *folio_get_anon_vma(const struct folio *folio) } #ifdef CONFIG_MM_SUBPAGE -/* The caller retains the folio; splitting rechecks the order under its lock. */ +/* + * The caller retains the folio; splitting rechecks the order under its lock. + * May sleep when dropping the temporary anon-vma reference. + */ unsigned int folio_anon_min_order(struct folio *folio) { struct anon_vma *anon_vma = NULL; diff --git a/mm/swapfile.c b/mm/swapfile.c index 7ab5fe05a0e2f..7b5f9caabbb36 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1102,7 +1102,8 @@ static void swap_reclaim_work(struct work_struct *work) swap_reclaim_full_clusters(si, true); } -static bool swap_folio_alloc_allowed(struct swap_info_struct *si, struct folio *folio) +static bool swap_folio_alloc_allowed(struct swap_info_struct *si, struct folio *folio, + bool mandatory) { unsigned int flags; @@ -1112,7 +1113,7 @@ static bool swap_folio_alloc_allowed(struct swap_info_struct *si, struct folio * if (flags & SWP_BLKDEV) return true; /* Filesystem swap_rw implementations retain their existing contract. */ - return si->bdev && !(flags & SWP_FS_OPS) && folio_anon_min_order(folio); + return si->bdev && !(flags & SWP_FS_OPS) && mandatory; } /* @@ -1120,13 +1121,13 @@ static bool swap_folio_alloc_allowed(struct swap_info_struct *si, struct folio * * cluster for current CPU too. */ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, - struct folio *folio) + struct folio *folio, bool mandatory) { struct swap_cluster_info *ci; unsigned int order = likely(folio) ? folio_order(folio) : 0; unsigned int offset = SWAP_ENTRY_INVALID, found = SWAP_ENTRY_INVALID; - if (!swap_folio_alloc_allowed(si, folio)) + if (!swap_folio_alloc_allowed(si, folio, mandatory)) return 0; if (!(si->flags & SWP_SOLIDSTATE)) { @@ -1384,7 +1385,7 @@ static bool get_swap_device_info(struct swap_info_struct *si) * Fast path try to get swap entries with specified order from current * CPU's swap entry pool (a cluster). */ -static bool swap_alloc_fast(struct folio *folio) +static bool swap_alloc_fast(struct folio *folio, bool mandatory) { unsigned int order = folio_order(folio); struct swap_cluster_info *ci; @@ -1399,7 +1400,7 @@ static bool swap_alloc_fast(struct folio *folio) offset = this_cpu_read(percpu_swap_cluster.offset[order]); if (!si || !offset || !get_swap_device_info(si)) return false; - if (!swap_folio_alloc_allowed(si, folio)) { + if (!swap_folio_alloc_allowed(si, folio, mandatory)) { put_swap_device(si); return false; } @@ -1418,7 +1419,7 @@ static bool swap_alloc_fast(struct folio *folio) } /* Rotate the device and switch to a new cluster */ -static void swap_alloc_slow(struct folio *folio) +static void swap_alloc_slow(struct folio *folio, bool mandatory) { struct swap_info_struct *si, *next; @@ -1429,7 +1430,7 @@ static void swap_alloc_slow(struct folio *folio) plist_requeue(&si->avail_list, &swap_avail_head); spin_unlock(&swap_avail_lock); if (get_swap_device_info(si)) { - cluster_alloc_swap_entry(si, folio); + cluster_alloc_swap_entry(si, folio, mandatory); put_swap_device(si); if (folio_test_swapcache(folio)) return; @@ -1847,7 +1848,7 @@ static void swap_alloc_required_from_device(struct swap_info_struct *si, unsigned int order = folio_order(folio), nr = folio_nr_pages(folio); unsigned long start; - if (!swap_folio_alloc_allowed(si, folio)) + if (!swap_folio_alloc_allowed(si, folio, true)) return; for (start = 0; start < si->max; start += SWAPFILE_CLUSTER) { struct swap_cluster_info *ci = __swap_offset_to_cluster(si, start); @@ -1925,10 +1926,12 @@ int folio_alloc_swap(struct folio *folio) { unsigned int order = folio_order(folio); unsigned int size = 1 << order; - bool mandatory = order && folio_anon_min_order(folio); + bool mandatory; VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); VM_BUG_ON_FOLIO(!folio_test_uptodate(folio), folio); + /* The temporary anon-vma reference may require sleepable final release. */ + mandatory = order && folio_anon_min_order(folio); if (order) { /* @@ -1950,8 +1953,8 @@ int folio_alloc_swap(struct folio *folio) again: local_lock(&percpu_swap_cluster.lock); - if (!swap_alloc_fast(folio)) - swap_alloc_slow(folio); + if (!swap_alloc_fast(folio, mandatory)) + swap_alloc_slow(folio, mandatory); local_unlock(&percpu_swap_cluster.lock); if (mandatory && unlikely(!folio_test_swapcache(folio))) @@ -2385,7 +2388,7 @@ swp_entry_t swap_alloc_hibernation_slot(int type) swap_cluster_unlock(ci); } if (!offset) - offset = cluster_alloc_swap_entry(si, NULL); + offset = cluster_alloc_swap_entry(si, NULL, false); local_unlock(&percpu_swap_cluster.lock); if (offset) entry = swp_entry(si->type, offset); From 1e14bace9e50835e963b6340e151e9f2bdc417d5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:17:39 -0400 Subject: [PATCH 0883/2680] mm: return the mapped fragment address from page_mapped_in_vma Supply the native page offset when bounding the reverse-map walk. For a smaller-granule MM, return the matching leaf found by the walk instead of the initial address, which can refer to an unmapped leading fragment. Keep the byte address within a covering leaf for native and coarser MMs. Extend the existing boundary case with a nonzero file offset and an absent leading fragment. Reverse-map tests pass in 16K/64K KASAN guests; native x86 objects build with CONFIG_MEMORY_FAILURE enabled. Signed-off-by: Diljit Singh (cherry picked from commit c11e970bc8c123095f2deb62b40abe29cb8d129d) --- arch/arm64/mm/user4k-test.c | 16 ++++++++++++++++ mm/page_vma_mapped.c | 8 ++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 68c7f9dad43d4..1fe2070e711ce 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -1507,11 +1507,14 @@ static void user4k_rmap_boundary_test(struct kunit *test) KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); vma->vm_start = SZ_2M - SZ_4K; vma->vm_end = vma->vm_start + PAGE_SIZE; + vma->vm_pgoff = 37; + page_folio(pfn_to_page(ctx->phys[0] >> PAGE_SHIFT))->index = vma->vm_pgoff; for (i = 0; i < 3; i++) KUNIT_ASSERT_EQ(test, test_map_alias(ctx, 4 + i, vma->vm_start + i * SZ_4K, ctx->phys[i]), 0); walk = (struct page_vma_mapped_walk) { .pfn = ctx->phys[0] >> PAGE_SHIFT, + .pgoff = vma->vm_pgoff, .nr_pages = 1, .vma = vma, .address = vma->vm_start, @@ -1529,6 +1532,19 @@ static void user4k_rmap_boundary_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, valid); KUNIT_EXPECT_EQ(test, count, 3U); KUNIT_EXPECT_TRUE(test, walk.flags & PVMW_PGTABLE_CROSSED); +#ifdef CONFIG_MEMORY_FAILURE + { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(ctx->mm, vma->vm_start, &ptl); + + KUNIT_ASSERT_NOT_NULL(test, pte); + ptep_get_and_clear(ctx->mm, vma->vm_start, pte); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, + page_mapped_in_vma(pfn_to_page(ctx->phys[0] >> PAGE_SHIFT), vma), + vma->vm_start + SZ_4K); + } +#endif } static void native_rmap_walk_test(struct kunit *test) diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index 272e7f98d1d2a..048d08d3a5b97 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -389,20 +389,24 @@ unsigned long page_mapped_in_vma(const struct page *page, unsigned long address; struct page_vma_mapped_walk pvmw = { .pfn = page_to_pfn(page), + .pgoff = page_pgoff(folio, page), .nr_pages = 1, .vma = vma, .flags = PVMW_SYNC, }; - address = vma_address(vma, page_pgoff(folio, page), 1); + address = vma_address(vma, pvmw.pgoff, 1); pvmw.address = address; if (pvmw.address == -EFAULT) goto out; if (!page_vma_mapped_walk(&pvmw)) return -EFAULT; + /* A smaller leaf can map the page after an unmapped leading fragment. */ + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + address = pvmw.address; page_vma_mapped_walk_done(&pvmw); out: - /* Report this native page's byte address, not the covering leaf start. */ + /* Coarser leaves retain this native page's byte address within the leaf. */ return address; } #endif From 85fee8b57ab6e2b0c4337a2a399843b954294ac3 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:17:39 -0400 Subject: [PATCH 0884/2680] mm: clear poison entries on subpage MADV_FREE Match the native MADV_FREE handling of hardware-poison and software poison markers. Clearing these non-swap entries must not change swap accounting. Extend the existing userfaultfd marker case to check MADV_FREE recovery and neighboring data. The suite passes for native/4K processes on 16K and native/4K/16K processes on 64K KASAN kernels. Hardware memory errors were not injected. Signed-off-by: Diljit Singh (cherry picked from commit f86041e8396472c60b6a4adf063c4d93b99082d5) --- mm/madvise.c | 3 +++ tools/testing/selftests/arm64/user4k/uffd-wp-init.c | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/mm/madvise.c b/mm/madvise.c index 86210ebeb9fea..8e7dee152389b 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -698,6 +698,9 @@ static int madvise_free_subpage_range(pmd_t *pmd, unsigned long addr, pte_swp_subpage_offset(pte), true); pte_clear_not_present_full(mm, addr, ptep, tlb->fullmm); swapped--; + } else if (softleaf_is_hwpoison(entry) || + softleaf_is_poison_marker(entry)) { + pte_clear_not_present_full(mm, addr, ptep, tlb->fullmm); } continue; } diff --git a/tools/testing/selftests/arm64/user4k/uffd-wp-init.c b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c index 19747755c4c6c..f34ab8328721d 100644 --- a/tools/testing/selftests/arm64/user4k/uffd-wp-init.c +++ b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c @@ -432,8 +432,12 @@ static void markers(size_t ps) CHECK(!sigaction(SIGBUS, &old, NULL)); p[17] = 0x81; p[2 * ps + 17] = 0x82; - CHECK(!close(fd) && !munmap(p, 3 * ps)); - printf("ok - uffd ABI %zuK poisoned leaf signals SIGBUS and preserves neighbours\n", + CHECK(!madvise(p + ps, ps, MADV_FREE)); + CHECK(!close(fd)); + CHECK(p[ps + 17] == 0); + CHECK(p[17] == 0x81 && p[2 * ps + 17] == 0x82); + CHECK(!munmap(p, 3 * ps)); + printf("ok - uffd ABI %zuK poison: SIGBUS, MADV_FREE recovery, neighbours\n", ps / 1024); } struct pager_request { From 2815e6cffa9a0dd018498abfab0d98103ce53ec0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:17:40 -0400 Subject: [PATCH 0885/2680] arm64: retain the fragment pin test workers through cleanup A pin worker can exit on an error before cleanup calls kthread_stop(). Create it stopped, acquire a task reference, register cleanup, and then wake it. Release the reference after kthread_stop(), including the cleanup registration failure path. The existing retained-pin test passes on 16K and 64K KASAN kernels. No additional test fixture is introduced. Signed-off-by: Diljit Singh (cherry picked from commit e7d10fd228c70c65e932a002296d07a48a0bcdfd) --- arch/arm64/mm/user4k-test.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 1fe2070e711ce..f5a0796d6dd04 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -10408,6 +10408,7 @@ static void fragment_pin_worker_stop(void *arg) if (worker->task) { kthread_stop(worker->task); + put_task_struct(worker->task); worker->task = NULL; } } @@ -10435,13 +10436,16 @@ static void user4k_fragment_pin_race_test(struct kunit *test) for (i = 0; i < 2; i++) { workers[i].mm = vma->vm_mm; init_completion(&workers[i].ready); - workers[i].task = kthread_run(fragment_pin_worker_run, &workers[i], "user4k-pin-%u", i); + workers[i].task = kthread_create(fragment_pin_worker_run, &workers[i], + "user4k-pin-%u", i); if (IS_ERR(workers[i].task)) { workers[i].task = NULL; KUNIT_FAIL(test, "pin worker creation failed"); return; } + get_task_struct(workers[i].task); KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_pin_worker_stop, &workers[i]), 0); + wake_up_process(workers[i].task); KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].ready, 10 * HZ), 0UL); } for (round = 0; round < 32; round++) { From 45994aa5536ac1b9c6e53c5deb1f2625c0e50c9d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:39:15 -0400 Subject: [PATCH 0886/2680] mm: preserve physical subpage offsets in batched PFN remaps Add an explicit physical offset to the PFN range mapping path. Carry it through the existing page-table batching and error cleanup, record it in COW mapping offsets, and track the complete covering native-page range. The existing remap_pfn_range() interface selects offset zero. Use the offset-aware range interface in the system dma-buf heap. Shorten the first scatterlist extent by the starting offset and reset that offset when advancing to the next extent. Physical identity remains independent of the target virtual address. Extend the existing PFN mapping test with noncongruent physical/virtual offsets. 16K/64K KASAN builds and tests pass. Locally adapted existing dma-buf heap tests also verify offset aliases across backing segments for native/4K/16K processes, including coherent writes. Native x86 PFN tracking and system-heap objects build. Physical DMA hardware remains unqualified. Signed-off-by: Diljit Singh (cherry picked from commit eadcabe164613222b98854a43e75b1b35d8c8e0a) --- arch/arm64/mm/user4k-test.c | 15 ++++-- drivers/dma-buf/heaps/system_heap.c | 10 ++-- include/linux/mm.h | 2 + mm/memory.c | 72 ++++++++++++++++++++--------- 4 files changed, 70 insertions(+), 29 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index f5a0796d6dd04..5cddc2deaf578 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -5650,6 +5650,7 @@ static void user4k_remap_pfn_range_test(struct kunit *test) for (unsigned int shape = 0; shape < ARRAY_SIZE(boundaries); shape++) { unsigned long start = boundaries[shape] - size; unsigned long length = PAGE_SIZE + size; + unsigned int physical_offset = native || !shape ? 0 : size; struct vm_area_struct *vma; int ret; @@ -5661,8 +5662,12 @@ static void user4k_remap_pfn_range_test(struct kunit *test) KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); mmap_write_lock(mm); - ret = remap_pfn_range(vma, start, folio_pfn(folio), length, - vma->vm_page_prot); + if (physical_offset) + ret = remap_pfn_range_offset(vma, start, folio_pfn(folio), + physical_offset, length, vma->vm_page_prot); + else + ret = remap_pfn_range(vma, start, folio_pfn(folio), length, + vma->vm_page_prot); mmap_write_unlock(mm); KUNIT_ASSERT_EQ(test, ret, 0); for (unsigned long offset = 0; offset < length; offset += size) { @@ -5671,13 +5676,15 @@ static void user4k_remap_pfn_range_test(struct kunit *test) struct follow_pfnmap_args args = { .vma = vma, .address = start + offset + 31, }; - phys_addr_t expected = PFN_PHYS(folio_pfn(folio)) + offset + 31; + phys_addr_t expected = PFN_PHYS(folio_pfn(folio)) + + physical_offset + offset + 31; int ret; KUNIT_EXPECT_TRUE(test, pte_present(pte)); KUNIT_EXPECT_TRUE(test, pte_special(pte)); KUNIT_EXPECT_TRUE(test, pte_write(pte)); KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), - PFN_PHYS(folio_pfn(folio)) + offset); + PFN_PHYS(folio_pfn(folio)) + + physical_offset + offset); mmap_read_lock(mm); ret = follow_pfnmap_start(&args); diff --git a/drivers/dma-buf/heaps/system_heap.c b/drivers/dma-buf/heaps/system_heap.c index c8959eadc71d1..8effbd8e25139 100644 --- a/drivers/dma-buf/heaps/system_heap.c +++ b/drivers/dma-buf/heaps/system_heap.c @@ -231,6 +231,7 @@ static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) struct sg_table *table = &buffer->sg_table; unsigned long addr = vma->vm_start; unsigned long pgoff = vma->vm_pgoff; + unsigned int offset = vma_subpage_offset(vma); struct scatterlist *sg; pgprot_t prot; int i, ret; @@ -250,17 +251,18 @@ static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) for (; sg && addr < vma->vm_end; sg = sg_next(sg)) { unsigned long n = (sg->length >> PAGE_SHIFT) - pgoff; struct page *page = sg_page(sg) + pgoff; - unsigned long size = n << PAGE_SHIFT; + unsigned long size = (n << PAGE_SHIFT) - offset; - if (addr + size > vma->vm_end) - size = vma->vm_end - addr; + size = min(size, vma->vm_end - addr); - ret = remap_pfn_range(vma, addr, page_to_pfn(page), size, prot); + ret = remap_pfn_range_offset(vma, addr, page_to_pfn(page), offset, + size, prot); if (ret) return ret; addr += size; pgoff = 0; + offset = 0; } return 0; diff --git a/include/linux/mm.h b/include/linux/mm.h index 362fea150df79..dd060a1be187f 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4586,6 +4586,8 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *, unsigned long addr); int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t pgprot); +int remap_pfn_range_offset(struct vm_area_struct *vma, unsigned long addr, + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot); int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *); int vm_insert_pages_range(struct vm_area_struct *vma, unsigned long addr, diff --git a/mm/memory.c b/mm/memory.c index 3b32187dc0267..47c98f072ad67 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3467,13 +3467,13 @@ static int get_remap_pgoff(bool is_cow, unsigned long addr, } static int remap_pfn_range_internal(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, unsigned long size, pgprot_t prot) + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot) { pgd_t *pgd; unsigned long next; struct mm_struct *mm = vma->vm_mm; unsigned long end = addr + mm_page_align(mm, size); - phys_addr_t phys = PFN_PHYS(pfn); + phys_addr_t phys = PFN_PHYS(pfn) + offset; int err; if (WARN_ON_ONCE(!IS_ALIGNED(addr, mm_page_size(mm)))) @@ -3501,9 +3501,9 @@ static int remap_pfn_range_internal(struct vm_area_struct *vma, unsigned long ad * must have pre-validated the caching bits of the pgprot_t. */ static int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, unsigned long size, pgprot_t prot) + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot) { - int error = remap_pfn_range_internal(vma, addr, pfn, size, prot); + int error = remap_pfn_range_internal(vma, addr, pfn, offset, size, prot); if (!error) return 0; @@ -3547,12 +3547,17 @@ void pfnmap_track_ctx_release(struct kref *ref) } static int remap_pfn_range_track(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, unsigned long size, pgprot_t prot) + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot) { struct pfnmap_track_ctx *ctx = NULL; + unsigned long track_size; int err; size = mm_page_align(vma->vm_mm, size); + if (check_add_overflow(size, (unsigned long)offset, &track_size) || + track_size > ULONG_MAX - (PAGE_SIZE - 1)) + return -EINVAL; + track_size = PAGE_ALIGN(track_size); /* * If we cover the full VMA, we'll perform actual tracking, and @@ -3566,14 +3571,14 @@ static int remap_pfn_range_track(struct vm_area_struct *vma, unsigned long addr, if (addr == vma->vm_start && addr + size == vma->vm_end) { if (vma->pfnmap_track_ctx) return -EINVAL; - ctx = pfnmap_track_ctx_alloc(pfn, size, &prot); + ctx = pfnmap_track_ctx_alloc(pfn, track_size, &prot); if (IS_ERR(ctx)) return PTR_ERR(ctx); - } else if (pfnmap_setup_cachemode(pfn, size, &prot)) { + } else if (pfnmap_setup_cachemode(pfn, track_size, &prot)) { return -EINVAL; } - err = remap_pfn_range_notrack(vma, addr, pfn, size, prot); + err = remap_pfn_range_notrack(vma, addr, pfn, offset, size, prot); if (ctx) { if (err) kref_put(&ctx->kref, pfnmap_track_ctx_release); @@ -3584,15 +3589,15 @@ static int remap_pfn_range_track(struct vm_area_struct *vma, unsigned long addr, } static int do_remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, unsigned long size, pgprot_t prot) + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot) { - return remap_pfn_range_track(vma, addr, pfn, size, prot); + return remap_pfn_range_track(vma, addr, pfn, offset, size, prot); } #else static int do_remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, unsigned long size, pgprot_t prot) + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot) { - return remap_pfn_range_notrack(vma, addr, pfn, size, prot); + return remap_pfn_range_notrack(vma, addr, pfn, offset, size, prot); } #endif @@ -3619,7 +3624,7 @@ int remap_pfn_range_prepare(struct vm_area_desc *desc) static int remap_pfn_range_prepare_vma(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, - unsigned long size) + unsigned int offset, unsigned long size) { const unsigned long end = addr + mm_page_align(vma->vm_mm, size); const bool is_cow = is_cow_mapping(vma->vm_flags); @@ -3630,10 +3635,41 @@ static int remap_pfn_range_prepare_vma(struct vm_area_struct *vma, if (err) return err; + if (is_cow) + vma_set_page_offset(vma, (struct vm_page_offset) { pfn, offset }); vma_set_flags_mask(vma, VMA_REMAP_FLAGS); return 0; } +/** + * remap_pfn_range_offset - remap a physical range starting within a native page + * @vma: user VMA to map to + * @addr: target address, aligned to the VMA's mapping granule + * @pfn: native page frame containing the first byte + * @offset: byte offset within @pfn, aligned to the mapping granule + * @size: size of the mapping, in bytes + * @prot: page protection flags + * + * The caller must hold the mmap lock. The physical offset is independent of + * the target virtual address. References and error cleanup match remap_pfn_range(). + * + * Return: 0 on success, a negative error code otherwise. + */ +int remap_pfn_range_offset(struct vm_area_struct *vma, unsigned long addr, + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot) +{ + int err; + + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, mm_page_size(vma->vm_mm))) + return -EINVAL; + err = remap_pfn_range_prepare_vma(vma, addr, pfn, offset, size); + if (err) + return err; + + return do_remap_pfn_range(vma, addr, pfn, offset, size, prot); +} +EXPORT_SYMBOL(remap_pfn_range_offset); + /** * remap_pfn_range - remap kernel memory to userspace * @vma: user vma to map to @@ -3649,13 +3685,7 @@ static int remap_pfn_range_prepare_vma(struct vm_area_struct *vma, int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t prot) { - int err; - - err = remap_pfn_range_prepare_vma(vma, addr, pfn, size); - if (err) - return err; - - return do_remap_pfn_range(vma, addr, pfn, size, prot); + return remap_pfn_range_offset(vma, addr, pfn, 0, size, prot); } EXPORT_SYMBOL(remap_pfn_range); @@ -3667,7 +3697,7 @@ int remap_pfn_range_complete(struct vm_area_struct *vma, const unsigned long size = action->remap.size; const pgprot_t prot = action->remap.pgprot; - return do_remap_pfn_range(vma, start, pfn, size, prot); + return do_remap_pfn_range(vma, start, pfn, 0, size, prot); } static int __simple_ioremap_prep(unsigned long vm_len, pgoff_t vm_pgoff, From 31047b0a4f09973539db826bfbc51fed1d2c8cad Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 08:00:36 -0400 Subject: [PATCH 0887/2680] mm: retain physical offsets through I/O mapping preparation Round resource coverage in the target MM's granule and combine the resource start with the VMA's complete byte offset. Validate lengths and offset arithmetic before publishing mappings. Native-page rounding is unchanged. Carry the resulting physical offset through both vm_iomap_memory() and mmap_prepare actions. Keep the native architecture PFN conversion and protection hooks, include the offset in their range, and preserve COW physical identity. The action field is absent without CONFIG_MM_SUBPAGE. Extend the existing PFN mapping test across native, 4K and supported 16K process geometries. Direct and descriptor I/O mappings check noncongruent offsets and reject insufficient resource ranges. 16K/64K KASAN builds and tests pass; native x86 mapping/action objects compile. Physical MMIO device access is not qualified by these RAM-backed fixtures. Signed-off-by: Diljit Singh (cherry picked from commit fa6b5cff34588782b063a10d40df4a4ecf882db8) --- arch/arm64/mm/user4k-test.c | 35 +++++++++++++++---- include/linux/mm.h | 30 +++++++++++++--- include/linux/mm_types.h | 3 ++ mm/internal.h | 6 +++- mm/memory.c | 70 +++++++++++++++++++++++-------------- 5 files changed, 106 insertions(+), 38 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 5cddc2deaf578..b3be0270c4638 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -5635,18 +5635,17 @@ static void user4k_remap_pfn_range_test(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, folio); KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); - for (unsigned int native = 0; native < 2; native++) { - struct mm_struct *mm = native ? mm_alloc() : test_vma_mm(test); - unsigned long size, boundaries[3]; + for (unsigned int shift = 12; shift <= PAGE_SHIFT; shift += 2) { + struct mm_struct *mm = test_vma_mm_granule(test, shift); + bool native = shift == PAGE_SHIFT; + unsigned long size, boundaries[4]; KUNIT_ASSERT_NOT_NULL(test, mm); - if (native) - KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, - user4k_vma_mm_free, mm), 0); size = mm_page_size(mm); boundaries[0] = pmd_size_mm(mm); boundaries[1] = pud_size_mm(mm); boundaries[2] = pgd_size_mm(mm); + boundaries[3] = 2 * pmd_size_mm(mm); for (unsigned int shape = 0; shape < ARRAY_SIZE(boundaries); shape++) { unsigned long start = boundaries[shape] - size; unsigned long length = PAGE_SIZE + size; @@ -5662,7 +5661,29 @@ static void user4k_remap_pfn_range_test(struct kunit *test) KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); mmap_write_lock(mm); - if (physical_offset) + if (shape >= 2) { + phys_addr_t base = PFN_PHYS(folio_pfn(folio)) + physical_offset; + unsigned long available = folio_size(folio) - physical_offset; + + vma_set_page_offset(vma, (struct vm_page_offset) { + .offset = physical_offset, + }); + if (shape == 2) { + KUNIT_EXPECT_EQ(test, + vm_iomap_memory(vma, base, PAGE_SIZE), -EINVAL); + ret = vm_iomap_memory(vma, base, available); + } else { + struct vm_area_desc desc; + + compat_set_desc_from_vma(&desc, NULL, vma); + mmap_action_simple_ioremap(&desc, base, PAGE_SIZE); + KUNIT_EXPECT_EQ(test, + __compat_vma_mmap(&desc, vma), -EINVAL); + mmap_action_simple_ioremap(&desc, base, available); + ret = __compat_vma_mmap(&desc, vma); + } + physical_offset *= 2; + } else if (physical_offset) ret = remap_pfn_range_offset(vma, start, folio_pfn(folio), physical_offset, length, vma->vm_page_prot); else diff --git a/include/linux/mm.h b/include/linux/mm.h index dd060a1be187f..2e1a4d6e05147 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4348,6 +4348,15 @@ static inline unsigned long vma_desc_pages(const struct vm_area_desc *desc) return vma_desc_size(desc) >> PAGE_SHIFT; } +static inline unsigned int mmap_action_remap_offset(const struct mmap_action *action) +{ +#ifdef CONFIG_MM_SUBPAGE + return action->remap.offset; +#else + return 0; +#endif +} + /** * mmap_action_remap - helper for mmap_prepare hook to specify that a pure PFN * remap is required. @@ -4373,6 +4382,9 @@ static inline void mmap_action_remap(struct vm_area_desc *desc, action->remap.start_pfn = start_pfn; action->remap.size = size; action->remap.pgprot = desc->page_prot; +#ifdef CONFIG_MM_SUBPAGE + action->remap.offset = 0; +#endif } /** @@ -4723,14 +4735,24 @@ static inline unsigned long io_remap_pfn_range_pfn(unsigned long pfn, } #endif +static inline int io_remap_pfn_range_offset(struct vm_area_struct *vma, + unsigned long addr, unsigned long orig_pfn, unsigned int offset, + unsigned long size, pgprot_t orig_prot) +{ + const pgprot_t prot = pgprot_decrypted(orig_prot); + unsigned long pfn; + + if (size > ULONG_MAX - offset) + return -EINVAL; + pfn = io_remap_pfn_range_pfn(orig_pfn, size + offset); + return remap_pfn_range_offset(vma, addr, pfn, offset, size, prot); +} + static inline int io_remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long orig_pfn, unsigned long size, pgprot_t orig_prot) { - const unsigned long pfn = io_remap_pfn_range_pfn(orig_pfn, size); - const pgprot_t prot = pgprot_decrypted(orig_prot); - - return remap_pfn_range(vma, addr, pfn, size, prot); + return io_remap_pfn_range_offset(vma, addr, orig_pfn, 0, size, orig_prot); } static inline vm_fault_t vmf_error(int err) diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index a075ab8f48bea..5b35d0e4e3867 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -839,6 +839,9 @@ struct mmap_action { unsigned long start_pfn; unsigned long size; pgprot_t pgprot; +#ifdef CONFIG_MM_SUBPAGE + unsigned int offset; +#endif } remap; struct { phys_addr_t start_phys_addr; diff --git a/mm/internal.h b/mm/internal.h index 7e4c6d7bc9484..087a62eec671e 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1930,9 +1930,13 @@ static inline int io_remap_pfn_range_prepare(struct vm_area_desc *desc) const unsigned long orig_pfn = action->remap.start_pfn; const pgprot_t orig_pgprot = action->remap.pgprot; const unsigned long size = action->remap.size; - const unsigned long pfn = io_remap_pfn_range_pfn(orig_pfn, size); + const unsigned int offset = mmap_action_remap_offset(action); + unsigned long pfn; int err; + if (size > ULONG_MAX - offset) + return -EINVAL; + pfn = io_remap_pfn_range_pfn(orig_pfn, size + offset); action->remap.start_pfn = pfn; action->remap.pgprot = pgprot_decrypted(orig_pgprot); err = remap_pfn_range_prepare(desc); diff --git a/mm/memory.c b/mm/memory.c index 47c98f072ad67..20564a51a4465 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3607,16 +3607,23 @@ int remap_pfn_range_prepare(struct vm_area_desc *desc) const unsigned long start = action->remap.start; const unsigned long end = start + action->remap.size; const unsigned long pfn = action->remap.start_pfn; + const unsigned int offset = mmap_action_remap_offset(action); const bool is_cow = vma_desc_is_cow_mapping(desc); int err; if (!range_in_vma_desc(desc, start, end)) return -EFAULT; + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, mm_page_size(desc->mm))) + return -EINVAL; err = get_remap_pgoff(is_cow, start, end, desc->start, desc->end, pfn, &desc->pgoff); if (err) return err; +#ifdef CONFIG_MM_SUBPAGE + if (is_cow) + desc->subpage_offset = offset; +#endif vma_desc_set_flags_mask(desc, VMA_REMAP_FLAGS); return 0; @@ -3697,40 +3704,43 @@ int remap_pfn_range_complete(struct vm_area_struct *vma, const unsigned long size = action->remap.size; const pgprot_t prot = action->remap.pgprot; - return do_remap_pfn_range(vma, start, pfn, 0, size, prot); + return do_remap_pfn_range(vma, start, pfn, mmap_action_remap_offset(action), size, prot); } -static int __simple_ioremap_prep(unsigned long vm_len, pgoff_t vm_pgoff, +static int __simple_ioremap_prep(struct mm_struct *mm, unsigned long vm_len, + struct vm_page_offset pos, phys_addr_t start_phys, unsigned long size, - unsigned long *pfnp) + struct vm_page_offset *physical) { - unsigned long pfn, pages; + unsigned long granule = mm_page_size(mm); + unsigned long offset, bias = start_phys & (granule - 1); /* Check that the physical memory area passed in looks valid */ if (start_phys + size < start_phys) return -EINVAL; /* - * You *really* shouldn't map things that aren't page-aligned, - * but we've historically allowed it because IO memory might - * just have smaller alignment. + * Preserve the historical allowance for unaligned I/O resources, using + * the target MM's granule for the prefix and final partial mapping. */ - size += start_phys & ~PAGE_MASK; - pfn = start_phys >> PAGE_SHIFT; - pages = (size + ~PAGE_MASK) >> PAGE_SHIFT; - if (pfn + pages < pfn) + if (check_add_overflow(size, bias, &size) || size > ULONG_MAX - (granule - 1)) return -EINVAL; - - /* We start the mapping 'vm_pgoff' pages into the area */ - if (vm_pgoff > pages) + size = ALIGN(size, granule); + start_phys -= bias; + if (start_phys + size < start_phys) + return -EINVAL; + if (pos.offset >= PAGE_SIZE || !IS_ALIGNED(pos.offset, granule) || + check_mul_overflow(pos.index, PAGE_SIZE, &offset) || + check_add_overflow(offset, (unsigned long)pos.offset, &offset)) return -EINVAL; - pfn += vm_pgoff; - pages -= vm_pgoff; - /* Can we fit all of the mapping? */ - if ((vm_len >> PAGE_SHIFT) + !!offset_in_page(vm_len) > pages) + if (offset > size || vm_len > size - offset) return -EINVAL; - *pfnp = pfn; + start_phys += offset; + physical->index = PHYS_PFN(start_phys); + physical->offset = offset_in_page(start_phys); + if (PFN_PHYS(physical->index) + physical->offset != start_phys) + return -EINVAL; return 0; } @@ -3739,16 +3749,22 @@ int simple_ioremap_prepare(struct vm_area_desc *desc) struct mmap_action *action = &desc->action; const phys_addr_t start = action->simple_ioremap.start_phys_addr; const unsigned long size = action->simple_ioremap.size; - unsigned long pfn; + struct vm_page_offset physical, pos = { .index = desc->pgoff }; int err; - err = __simple_ioremap_prep(vma_desc_size(desc), desc->pgoff, - start, size, &pfn); +#ifdef CONFIG_MM_SUBPAGE + pos.offset = desc->subpage_offset; +#endif + err = __simple_ioremap_prep(desc->mm, vma_desc_size(desc), pos, + start, size, &physical); if (err) return err; /* The I/O remap logic does the heavy lifting. */ - mmap_action_ioremap_full(desc, pfn); + mmap_action_ioremap_full(desc, physical.index); +#ifdef CONFIG_MM_SUBPAGE + action->remap.offset = physical.offset; +#endif return io_remap_pfn_range_prepare(desc); } @@ -3772,15 +3788,17 @@ int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long const unsigned long vm_start = vma->vm_start; const unsigned long vm_end = vma->vm_end; const unsigned long vm_len = vm_end - vm_start; - unsigned long pfn; + struct vm_page_offset physical; int err; - err = __simple_ioremap_prep(vm_len, vma->vm_pgoff, start, len, &pfn); + err = __simple_ioremap_prep(vma->vm_mm, vm_len, vma_page_offset_at(vma, vm_start), + start, len, &physical); if (err) return err; /* Ok, let it rip */ - return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot); + return io_remap_pfn_range_offset(vma, vm_start, physical.index, physical.offset, + vm_len, vma->vm_page_prot); } EXPORT_SYMBOL(vm_iomap_memory); From a70aed7a14d4ad0b15491497d38369fe94f6470a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 08:05:14 -0400 Subject: [PATCH 0888/2680] mm: preserve the PFN remap interface on no-MMU kernels Provide a zero-offset wrapper for remap_pfn_range_offset() without an MMU. It retains the existing identity-mapping checks and rejects offsets that this configuration cannot represent. MMU kernels keep the range mapper. ARM MPS2 no-MMU builds of mm/nommu.o and the system heap pass; their symbols contain no unresolved offset-remap reference. Native x86 mapping and action objects also build. No no-MMU runtime test was performed. Signed-off-by: Diljit Singh (cherry picked from commit 207036cd20c2f5b43958146873fdba2d0d1d6b3b) --- include/linux/mm.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/linux/mm.h b/include/linux/mm.h index 2e1a4d6e05147..20854dcc94ee1 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4598,8 +4598,18 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *, unsigned long addr); int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t pgprot); +#ifdef CONFIG_MMU int remap_pfn_range_offset(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot); +#else +static inline int remap_pfn_range_offset(struct vm_area_struct *vma, unsigned long addr, + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot) +{ + if (offset) + return -EINVAL; + return remap_pfn_range(vma, addr, pfn, size, prot); +} +#endif int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *); int vm_insert_pages_range(struct vm_area_struct *vma, unsigned long addr, From 75de82828796a2cb1bd43afea50015e1518a46ab Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 08:13:02 -0400 Subject: [PATCH 0889/2680] userfaultfd: update published folio flags atomically Shared shmem backing and neighboring retained slots can already expose the folio while a fill operation updates it. Holding the folio lock and PTL does not exclude other atomic changes to the folio flags word. Use folio_mark_uptodate() for both published-backing paths, preserving the write barrier while avoiding a non-atomic flags update. Keep private allocation initialization unchanged. 16K/64K KASAN builds and the existing userfaultfd contract suite pass for native/4K and supported 16K processes. No new fixture was added and the specific flag-word race was not forced at runtime. Signed-off-by: Diljit Singh (cherry picked from commit f2928b082c7a0e6e5b7588180daebce0f1efbea8) --- mm/userfaultfd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 4edaf4dacf345..1fdba014467c7 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -454,7 +454,7 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, } kunmap_local(dst); flush_dcache_folio(folio); - __folio_mark_uptodate(folio); + folio_mark_uptodate(folio); folio_mark_dirty(folio); } #endif @@ -829,7 +829,7 @@ static int mfill_atomic_subpage(struct mfill_state *state) } } /* The allocator zeroed every byte, including unissued neighbours. */ - __folio_mark_uptodate(folio); + folio_mark_uptodate(folio); vma = state->vma; pte = pte_offset_map_lock(mm, state->pmd, state->dst_addr, &ptl); err = -EAGAIN; From 09d086b9650bfa5b35786737b39fb5a9b5fdd537 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 08:48:13 -0400 Subject: [PATCH 0890/2680] tomoyo: inspect exec arguments through virtual-byte access Argument staging can use a smaller userspace granule, where legacy page-array GUP cannot represent the backing. Read the requested virtual bytes through access_remote_vm(), retaining the existing native-page buffer and caller iteration units. Keep the no-MMU page-array path. Cache the first copied virtual position instead of a backing-page pointer. Refill on a backward access outside the cached tail and invalidate before a potentially partial copy. Multiple fragments may contribute to one cached native-sized virtual range. Local extensions of existing exec tests verify argument bytes and cache rewinds on 16K/64K KASAN kernels for all supported process granules. Native x86 and ARM no-MMU TOMOYO objects compile. The temporary test hooks are not submitted; full policy enforcement was not exercised. Signed-off-by: Diljit Singh (cherry picked from commit 3001cf4667e728ce38f3453aa98e5995af1ca098) --- security/tomoyo/common.h | 4 ++-- security/tomoyo/domain.c | 38 +++++++++++++++----------------------- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/security/tomoyo/common.h b/security/tomoyo/common.h index d098cf8aae610..b859f099ae9ea 100644 --- a/security/tomoyo/common.h +++ b/security/tomoyo/common.h @@ -575,8 +575,8 @@ struct tomoyo_mini_stat { /* Structure for dumping argv[] and envp[] of "struct linux_binprm". */ struct tomoyo_page_dump { - struct page *page; /* Previously dumped page. */ - char *data; /* Contents of "page". Size is PAGE_SIZE. */ + unsigned long pos; /* First cached virtual byte, or ULONG_MAX. */ + char *data; /* Buffer for one native page of virtual bytes. */ }; /* Structure for attribute checks in addition to pathname checks. */ diff --git a/security/tomoyo/domain.c b/security/tomoyo/domain.c index eeaad421f15ba..2ff8d783f54cc 100644 --- a/security/tomoyo/domain.c +++ b/security/tomoyo/domain.c @@ -906,45 +906,37 @@ int tomoyo_find_next_domain(struct linux_binprm *bprm) bool tomoyo_dump_page(struct linux_binprm *bprm, unsigned long pos, struct tomoyo_page_dump *dump) { - struct page *page; -#ifdef CONFIG_MMU - int ret; -#endif + const unsigned int offset = offset_in_page(pos); + const unsigned int length = PAGE_SIZE - offset; /* dump->data is released by tomoyo_find_next_domain(). */ if (!dump->data) { dump->data = kzalloc(PAGE_SIZE, GFP_NOFS); if (!dump->data) return false; + dump->pos = ULONG_MAX; } - /* Same with get_arg_page(bprm, pos, 0) in fs/exec.c */ + if (pos >= dump->pos && (pos & PAGE_MASK) == (dump->pos & PAGE_MASK)) + return true; + /* A failed partial copy must not leave the previous cache valid. */ + dump->pos = ULONG_MAX; #ifdef CONFIG_MMU /* - * This is called at execve() time in order to dig around - * in the argv/environment of the new process - * (represented by bprm). + * Inspect the nascent MM's virtual bytes. Smaller userspace mappings + * can use several independently retained fragments within this range. */ - mmap_read_lock(bprm->mm); - ret = get_user_pages_remote(bprm->mm, pos, 1, - FOLL_FORCE, &page, NULL); - mmap_read_unlock(bprm->mm); - if (ret <= 0) + if (access_remote_vm(bprm->mm, pos, dump->data + offset, length, + FOLL_FORCE) != length) return false; #else - page = bprm->page[pos / PAGE_SIZE]; -#endif - if (page != dump->page) { - const unsigned int offset = pos % PAGE_SIZE; + { + struct page *page = bprm->page[pos / PAGE_SIZE]; char *kaddr = kmap_local_page(page); - dump->page = page; - memcpy(dump->data + offset, kaddr + offset, - PAGE_SIZE - offset); + memcpy(dump->data + offset, kaddr + offset, length); kunmap_local(kaddr); } - /* Same with put_arg_page(page) in fs/exec.c */ -#ifdef CONFIG_MMU - put_page(page); #endif + dump->pos = pos; return true; } From d4b71b6e65fa6d0ebd38fbc47bf919af43344c0c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 09:37:17 -0400 Subject: [PATCH 0891/2680] block: estimate user vectors with the process fragment size Fragment pinning emits records in mm_user_fragment_size() intervals. Count the same intervals when reserving vectors, including unaligned segments and saturation at the caller's limit. Fixed 4K counting otherwise overestimates 16K userspace buffers on a 64K kernel, affecting allocation and direct-I/O path selection. Local extensions of the existing BIO tests cover native and smaller granules on 16K/64K KASAN kernels. The native x86 BIO object builds. Signed-off-by: Diljit Singh (cherry picked from commit e139642c1fac2cfce399d3b6cd8d229fbe7cf93c) --- block/bio.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/block/bio.c b/block/bio.c index e60b37c491333..b2ba5ac367374 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1282,10 +1282,12 @@ static bool bio_subpage_iter(const struct iov_iter *iter) int bio_iov_user_vecs(const struct iov_iter *iter, int max_vecs) { struct iov_iter walk = *iter; + unsigned long size; unsigned int count = 0; if (!bio_subpage_iter(iter)) return iov_iter_npages(iter, max_vecs); + size = mm_user_fragment_size(current->mm); while (iov_iter_count(&walk) && count < max_vecs) { size_t len, offset, nr; @@ -1293,9 +1295,8 @@ int bio_iov_user_vecs(const struct iov_iter *iter, int max_vecs) len = min(iter_iov_len(&walk), iov_iter_count(&walk)); if (WARN_ON_ONCE(!len)) return max_vecs; - offset = (unsigned long)iter_iov_addr(&walk) & (MM_SUBPAGE_SIZE - 1); - nr = (len >> MM_SUBPAGE_SHIFT) + - DIV_ROUND_UP(offset + (len & (MM_SUBPAGE_SIZE - 1)), MM_SUBPAGE_SIZE); + offset = (unsigned long)iter_iov_addr(&walk) & (size - 1); + nr = len / size + DIV_ROUND_UP(offset + (len & (size - 1)), size); if (nr >= max_vecs - count) return max_vecs; count += nr; From 605cddcb6f67bcf761715e331b273e4c26464801 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 09:37:17 -0400 Subject: [PATCH 0892/2680] block: preserve nofault when pinning user fragments Pass FOLL_NOFAULT when the iterator forbids faulting, matching ordinary iterator extraction. Filesystems set this flag while holding locks that must not be reacquired through a user-buffer fault. Keep existing partial-progress and cleanup behavior. Local extensions of the existing BIO tests check missing and resident mappings with nofault set on 16K/64K KASAN kernels. Filesystem-specific locking paths have not been exercised. Signed-off-by: Diljit Singh (cherry picked from commit 1de4d3c0c6bdd60c98f992aa259ab3ed1eb4c397) --- block/bio.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/block/bio.c b/block/bio.c index b2ba5ac367374..5d3aa1e8170b0 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1318,6 +1318,8 @@ static int bio_iov_iter_get_fragments(struct bio *bio, struct iov_iter *iter, if (WARN_ON_ONCE(bio->bi_vcnt || bio->bi_user_fragments || !capacity)) return -EINVAL; + if (iter->nofault) + flags |= FOLL_NOFAULT; bio->bi_user_fragments = kmalloc_array(capacity, sizeof(*bio->bi_user_fragments), GFP_KERNEL); if (!bio->bi_user_fragments) From 4cc5b374cfaccb4b4895abcd446df8e2478ba2b8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:02:10 -0400 Subject: [PATCH 0893/2680] net: allocate a new stream zerocopy owner after subpage splice A queued TCP skb can carry retained subpage references in its ubuf_info. A subsequent MSG_ZEROCOPY send cannot extend that owner as a message completion record. Allocate a fresh record for stream sockets and let the existing incompatible-owner handling select another skb, as it already does when a completion record reaches its size limit. Keep the datagram behavior unchanged. Mixed-granule pipe producers and native TCP consumers passed both send orders on IPv4/IPv6 loopback in 16K/64K KASAN guests, including completion notifications. Signed-off-by: Diljit Singh (cherry picked from commit 4043902ee814ee5c94e1bd9c97f35dd81493a743) --- net/core/skbuff.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 1c1a491e75834..2085fb912ab48 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1773,8 +1773,11 @@ struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size, u32 bytelen, next; /* there might be non MSG_ZEROCOPY users */ - if (uarg->ops != &msg_zerocopy_ubuf_ops) + if (uarg->ops != &msg_zerocopy_ubuf_ops) { + if (sk->sk_type == SOCK_STREAM) + goto new_alloc; return NULL; + } /* realloc only when socket is locked (TCP, UDP cork), * so uarg->len and sk_zckey access is serialized From bd7cba32240b70a327694e3e5145d6311e511c5c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:02:10 -0400 Subject: [PATCH 0894/2680] tcp: start a new segment for incompatible splice ownership Splicing a retained subpage into a queued MSG_ZEROCOPY skb needs its own fragment owner. Report this conflict as EEXIST and retry on a new TCP segment, matching the reverse zerocopy transition. Preserve the iterator position and the existing owner's completion references. IPv4/IPv6 loopback checks cover both operation orders, exact bytes and completion notifications on 16K/64K KASAN guests. Existing SKB clone, trim and retained-reference tests also pass. No NIC throughput claim is made by these checks. Signed-off-by: Diljit Singh (cherry picked from commit f22a44fa973b4b96923246686d8d277ef13f8785) --- net/core/skbuff.c | 5 +++-- net/ipv4/tcp.c | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 2085fb912ab48..b03a7b0b7495d 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -7480,7 +7480,7 @@ static int skb_splice_hold_subpages(struct sk_buff *skb, struct page *page, return 0; mm_subpage_put(probe); if (old && old->ops != &skb_subpage_ops) - return -EOPNOTSUPP; + return -EEXIST; refs = old ? container_of(old, struct skb_subpage_refs, ubuf) : NULL; if (!refs || refcount_read(&old->refcnt) != 1) { new = kzalloc(sizeof(*new), GFP_ATOMIC); @@ -7534,7 +7534,8 @@ static void skb_splice_csum_page(struct sk_buff *skb, struct page *page, * pages). * * Returns the amount of data spliced/copied or -EMSGSIZE if there's - * insufficient space in the buffer to transfer anything. + * insufficient space in the buffer to transfer anything, or -EEXIST if the + * buffer already has an incompatible fragment owner. */ ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter, ssize_t maxsize) diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 93efc948d2be7..19663713ef92a 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1355,7 +1355,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) err = skb_splice_from_iter(skb, &msg->msg_iter, copy); if (err < 0) { - if (err == -EMSGSIZE) { + if (err == -EMSGSIZE || err == -EEXIST) { tcp_mark_push(tp, skb); goto new_segment; } From 97696ca62e575aab22eac46ea035331bf3513f00 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:02:10 -0400 Subject: [PATCH 0895/2680] selftests/arm64: derive splice fixture sizes from runtime limits The splice fixture assumes a 16K native pipe page, a corked payload that fits below TCP's MSS, and a process page smaller than a UDP datagram. Those assumptions fail on a 64K kernel even for native processes. Discover the native pipe allocation unit, size the test mapping to hold it, and bound corked TCP and UDP payloads by their protocol limits. Retain the data, ownership, teardown and capacity assertions. The same existing fixture now passes its 148 checks on both 16K and 64K kernels. Signed-off-by: Diljit Singh (cherry picked from commit 260567108121b13af59df7430fb2137b6558e2cb) --- .../arm64/user4k/splice-fixture-init.c | 50 +++++++++++++------ 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/tools/testing/selftests/arm64/user4k/splice-fixture-init.c b/tools/testing/selftests/arm64/user4k/splice-fixture-init.c index 282ba6268b25d..bac72da15f40e 100644 --- a/tools/testing/selftests/arm64/user4k/splice-fixture-init.c +++ b/tools/testing/selftests/arm64/user4k/splice-fixture-init.c @@ -336,8 +336,12 @@ static void tls_checks(size_t ps, unsigned char *p, size_t len, static int exercise(void) { - const size_t ps = getauxval(AT_PAGESZ), len = 8 * ps; - unsigned char *p, *back = malloc(len); + const size_t ps = getauxval(AT_PAGESZ); + size_t len; + unsigned char *p, *back; + int native; + unsigned int mss = 0; + socklen_t mss_len = sizeof(mss); int fds[2], dup[2], file, sockets[2]; struct iovec v[3]; struct stat self, init; @@ -348,9 +352,17 @@ static int exercise(void) if (stat("/proc/self/exe", &self) || stat("/proc/1/exe", &init) || self.st_dev != init.st_dev || self.st_ino != init.st_ino) return 2; + if (pipe(fds) || pipe(dup)) + ksft_exit_fail_msg("pipe setup\n"); + /* A pipe has at least one native page, independently of this MM. */ + native = fcntl(fds[1], F_SETPIPE_SZ, 1); + if (native <= 0 || fcntl(fds[1], F_SETPIPE_SZ, 16 * native) < 0) + ksft_exit_fail_msg("pipe capacity setup\n"); + len = 8 * ps > (size_t)native ? 8 * ps : (size_t)native; + back = malloc(len); p = mmap((void *)0x300000000UL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); - if (p == MAP_FAILED || !back || pipe(fds) || pipe(dup)) + if (p == MAP_FAILED || !back) ksft_exit_fail_msg("setup\n"); ksft_print_header(); ksft_set_plan(use_tcp ? 22 : 10); @@ -370,21 +382,21 @@ static int exercise(void) ok = read_exact(fds[0], back, result) && check(back, ps - 17, 17) && check(back + ps - 17, 2 * ps - 31, 2 * ps + 31); report(ok, ps, "unaligned fragmented iovecs and empty segment"); - v[0] = (struct iovec){ p, 16384 }; - ok = fcntl(fds[1], F_SETPIPE_SZ, 16384) == 16384; + v[0] = (struct iovec){ p, native }; + ok = fcntl(fds[1], F_SETPIPE_SZ, native) == native; result = vmsplice(fds[1], v, 1, SPLICE_F_NONBLOCK); { int pending = 0; - ok &= result == 16384 && !ioctl(fds[0], FIONREAD, &pending) && - pending == 16384; + ok &= result == native && !ioctl(fds[0], FIONREAD, &pending) && + pending == native; } if (result > 0) ok &= read_exact(fds[0], back, result) && check(back, result, 0); report(ok, ps, - "contiguous quarters retain one-native-page pipe capacity"); - fcntl(fds[1], F_SETPIPE_SZ, 16 * 16384); + "contiguous fragments retain one-native-page pipe capacity"); + fcntl(fds[1], F_SETPIPE_SZ, 16 * native); v[0] = (struct iovec){ p + 11, 3 * ps - 11 }; result = vmsplice(fds[1], v, 1, 0); if (result != (ssize_t)(3 * ps - 11) || @@ -422,14 +434,22 @@ static int exercise(void) ksft_exit_fail_msg("socketpair errno=%d\n", errno); setsockopt(sockets[1], SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); + if (use_tcp && + (getsockopt(sockets[0], IPPROTO_TCP, TCP_MAXSEG, &mss, &mss_len) || mss <= 1)) + ksft_exit_fail_msg("TCP_MAXSEG setup\n"); for (unsigned int mode = 0; mode < 3; mode++) { - const size_t bytes = mode == 1 ? ps : + size_t bytes = mode == 1 ? ps : (use_udp && ps == 4096 ? 6 * ps - 256 : 2 * ps - 256); const size_t offset = mode == 1 ? 0 : ps + 29; int cork = 1; + /* Keep the corked payload below one segment; UDP has a byte limit. */ + if (use_tcp && mode < 2 && bytes >= mss) + bytes = mss - 1; + if (use_udp && bytes > 65507) + bytes = 65507; if ((use_tcp || use_udp) && mode < 2 && setsockopt(sockets[0], use_tcp ? IPPROTO_TCP : IPPROTO_UDP, use_tcp ? TCP_CORK : UDP_CORK, &cork, @@ -497,12 +517,14 @@ static int exercise(void) close(sockets[1]); ok = true; for (unsigned int round = 0; round < 128 && ok; round++) { + const size_t bytes = use_udp && ps > 65507 ? 65507 : ps; + fill(p, len); ok = !connected_pair(sockets); - v[0] = (struct iovec){ p + ps, ps }; - ok &= vmsplice(fds[1], v, 1, 0) == (ssize_t)ps && - splice(fds[0], NULL, sockets[0], NULL, ps, 0) == - (ssize_t)ps; + v[0] = (struct iovec){ p + ps, bytes }; + ok &= vmsplice(fds[1], v, 1, 0) == (ssize_t)bytes && + splice(fds[0], NULL, sockets[0], NULL, bytes, 0) == + (ssize_t)bytes; close(sockets[1]); close(sockets[0]); ok &= !madvise(p + ps, ps, MADV_DONTNEED); From 49a9215993766c45d42ad2287f503fd3642f5141 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 12:24:35 -0400 Subject: [PATCH 0896/2680] arm64: align mixed-granule KUnit cases with migration dependencies The migration guard accidentally encloses the VMA split and brk tests, but their registrations are unconditional. Keep those definitions available without migration and guard the reclaim cases which explicitly exercise migration. Guard the userfaultfd MOVE/migration fixture and its registration too; a migration-disabled build exposed its unguarded reference to the migration allocator. The existing KUnit object builds on 16K and 64K configurations with migration both enabled and disabled. No new tests are added. Signed-off-by: Diljit Singh (cherry picked from commit 76088e05aa1e391fdcf2d19944c2bf2f2412e3a7) --- arch/arm64/mm/user4k-test.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index b3be0270c4638..877a16842f2cb 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -9389,6 +9389,7 @@ static long test_madvise(unsigned long addr, unsigned long size, unsigned long b return __arm64_sys_madvise(®s); } +#endif static void user_granule_large_vma_split_test(struct kunit *test) { @@ -9477,6 +9478,7 @@ static void user_granule_brk_extent_test(struct kunit *test) } } +#ifdef CONFIG_MIGRATION static void user4k_droppable_reclaim_test(struct kunit *test) { unsigned int mode; @@ -11710,6 +11712,7 @@ static ssize_t test_uffd_move(struct vm_area_struct *dst, unsigned long dst_addr return ret; } +#ifdef CONFIG_MIGRATION struct move_migration_worker { struct userfaultfd_ctx ctx; struct vm_area_struct *vma[2]; @@ -11865,6 +11868,8 @@ static void user4k_move_migration_race_test(struct kunit *test) } } +#endif + static void user4k_move_swap_get_test(struct kunit *test) { unsigned int variant; @@ -13422,7 +13427,9 @@ static struct kunit_suite user4k_shmem_swap_suite = { static struct kunit_case user4k_nonlinear_swap_cases[] = { #ifdef CONFIG_USERFAULTFD KUNIT_CASE(user4k_move_swap_get_test), +#ifdef CONFIG_MIGRATION KUNIT_CASE(user4k_move_migration_race_test), +#endif #endif KUNIT_CASE(user4k_nonlinear_swap_cached_test), KUNIT_CASE(user4k_nonlinear_swap_disk_test), @@ -14216,8 +14223,10 @@ static struct kunit_case user4k_test_cases[] = { KUNIT_CASE(user4k_mlock_repair_test), KUNIT_CASE(user_granule_large_vma_split_test), KUNIT_CASE(user_granule_brk_extent_test), +#ifdef CONFIG_MIGRATION KUNIT_CASE(user4k_droppable_reclaim_test), KUNIT_CASE(user4k_lazyfree_test), +#endif KUNIT_CASE(user4k_zero_cow_test), KUNIT_CASE(user16k_zero_cow_test), KUNIT_CASE(user4k_shmem_swap_usage_test), From faca4bc48c62a62cc3f08bcfe8c640bfbca6e57e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 13:40:54 -0400 Subject: [PATCH 0897/2680] io_uring: retain shared fragment charges through buffer teardown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Track fragment-backed registrations in a refcounted account for each user/MM counter pair. Charge each backing folio once and release its charge with the final retained fragment, including cloned buffers. - Keep the account alive independently of the originating ring. Roll back only the references acquired by failed preparation. - Serialize account updates locally; cache the registry lookup per ring. Preserve existing ring-context member offsets and keep native backing owners in their separate accounting path. - Validate with 16K/64K KASAN and lockdep builds, 110 guest lifecycle checks, and an x86 io_uring build. Warmed emulated registration measurements do not reproduce the earlier native timing discrepancy; hardware overhead and total metadata costs remain unqualified. [ryan: adapted to 7.2.10 — minimal; for DJ: 7.2 replaced headpage_already_acct() with per-ring compound-page refs (df0a52537c0f), so that hunk is dropped; fragment-backed buffers now report no pages to 7.2's unaccount path and skip the clone hpage refs, and their charge is held by io_fragment_account] Signed-off-by: Diljit Singh (cherry picked from commit d90fab78cd1b2044701c39b8593891a9b514b05f) --- include/linux/io_uring_types.h | 3 + io_uring/rsrc.c | 164 +++++++++++++++++++++++++++++---- 2 files changed, 147 insertions(+), 20 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 20116462cf3d7..f83da0af60d1d 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -544,6 +544,9 @@ struct io_ring_ctx { struct io_mapped_region ring_region; /* used for optimised request parameter and wait argument passing */ struct io_mapped_region param_region; +#ifdef CONFIG_MM_SUBPAGE + struct io_fragment_account *fragment_account; +#endif }; /* diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 39d0d06ca38bc..25510f038e393 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -76,6 +77,10 @@ static bool hpage_acct_unref(struct io_ring_ctx *ctx, struct page *hpage) return false; } +#ifdef CONFIG_MM_SUBPAGE +static void io_fragment_account_put(struct io_fragment_account *account); +#endif + /* only define max */ #define IORING_MAX_FIXED_FILES (1U << 20) #define IORING_MAX_REG_BUFFERS (1U << 14) @@ -177,7 +182,6 @@ static void io_free_imu(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu) #ifdef CONFIG_MM_SUBPAGE static void io_release_fragment_buffer(void *priv); -static unsigned long io_fragment_buffer_charge(void *priv); #endif static unsigned long io_buffer_unaccount_pages(struct io_ring_ctx *ctx, @@ -188,9 +192,9 @@ static unsigned long io_buffer_unaccount_pages(struct io_ring_ctx *ctx, int i; #ifdef CONFIG_MM_SUBPAGE - /* Fragment-backed buffers carry their own charge, not compound refs. */ + /* Fragment-backed buffers release their own charge, not compound refs. */ if (imu->release == io_release_fragment_buffer) - return io_fragment_buffer_charge(imu->priv); + return 0; #endif if (imu->flags & IO_REGBUF_F_KBUF || !ctx->user) return 0; @@ -266,6 +270,10 @@ bool io_rsrc_cache_init(struct io_ring_ctx *ctx) void io_rsrc_cache_free(struct io_ring_ctx *ctx) { +#ifdef CONFIG_MM_SUBPAGE + io_fragment_account_put(ctx->fragment_account); + ctx->fragment_account = NULL; +#endif io_alloc_cache_free(&ctx->node_cache, kfree); io_alloc_cache_free(&ctx->imu_cache, kvfree); } @@ -864,20 +872,145 @@ bool io_check_coalesce_buffer(struct page **page_array, int nr_pages, } #ifdef CONFIG_MM_SUBPAGE +/* + * Shared by rings charging the same counters, including cloned buffers. + * uring_lock nests outside lock; the registry lock is never held with lock. + * Long-term pins keep each indexed folio and its size stable until release. + */ +struct io_fragment_account { + struct hlist_node node; + refcount_t refs; + struct mutex lock; + struct xarray folios; + struct user_struct *user; + struct mm_struct *mm; +}; + +static DEFINE_HASHTABLE(io_fragment_accounts, 6); +static DEFINE_MUTEX(io_fragment_accounts_lock); + +static void io_fragment_account_put(struct io_fragment_account *account) +{ + if (!account || !refcount_dec_and_mutex_lock(&account->refs, + &io_fragment_accounts_lock)) + return; + hash_del(&account->node); + mutex_unlock(&io_fragment_accounts_lock); + WARN_ON_ONCE(!xa_empty(&account->folios)); + xa_destroy(&account->folios); + if (account->mm) + mmdrop(account->mm); + if (account->user) + free_uid(account->user); + kfree(account); +} + +static struct io_fragment_account *io_fragment_account_get(struct io_ring_ctx *ctx) +{ + unsigned long key = (unsigned long)ctx->mm_account ^ (unsigned long)ctx->user; + struct io_fragment_account *account; + + lockdep_assert_held(&ctx->uring_lock); + account = ctx->fragment_account; + if (account) + goto found; + + mutex_lock(&io_fragment_accounts_lock); + hash_for_each_possible(io_fragment_accounts, account, node, key) { + if (account->user == ctx->user && account->mm == ctx->mm_account) { + refcount_inc(&account->refs); + goto install; + } + } + account = kzalloc(sizeof(*account), GFP_KERNEL_ACCOUNT); + if (!account) { + mutex_unlock(&io_fragment_accounts_lock); + return NULL; + } + refcount_set(&account->refs, 1); + mutex_init(&account->lock); + xa_init(&account->folios); + account->user = ctx->user ? get_uid(ctx->user) : NULL; + account->mm = ctx->mm_account; + if (account->mm) + mmgrab(account->mm); + hash_add(io_fragment_accounts, &account->node, key); +install: + ctx->fragment_account = account; + mutex_unlock(&io_fragment_accounts_lock); +found: + /* The buffer retains its account after the originating ring exits. */ + refcount_inc(&account->refs); + return account; +} + struct io_fragment_buffer { unsigned int count; - unsigned long charged; + unsigned int accounted; + struct io_fragment_account *account; struct user_page_fragment fragments[]; }; -static unsigned long io_fragment_buffer_charge(void *priv) +static int io_account_fragment_buffer(struct io_fragment_buffer *buffer) { - return ((struct io_fragment_buffer *)priv)->charged; + struct io_fragment_account *account = buffer->account; + unsigned int i; + int ret = 0; + + mutex_lock(&account->lock); + for (i = 0; i < buffer->count; i++) { + struct folio *folio = buffer->fragments[i].folio; + unsigned long pfn = folio_pfn(folio); + void *entry = xa_load(&account->folios, pfn); + unsigned long refs = entry ? xa_to_value(entry) : 0; + + if (refs == LONG_MAX) { + ret = -EOVERFLOW; + break; + } + ret = xa_err(xa_store(&account->folios, pfn, xa_mk_value(refs + 1), + GFP_KERNEL_ACCOUNT)); + if (ret) + break; + if (!refs) { + ret = io_account_mem(account->user, account->mm, + folio_nr_pages(folio)); + if (ret) { + xa_erase(&account->folios, pfn); + break; + } + } + buffer->accounted++; + } + mutex_unlock(&account->lock); + return ret; } static void io_release_fragment_buffer(void *priv) { struct io_fragment_buffer *buffer = priv; + struct io_fragment_account *account = buffer->account; + unsigned int i; + + if (account) { + mutex_lock(&account->lock); + for (i = 0; i < buffer->accounted; i++) { + struct folio *folio = buffer->fragments[i].folio; + unsigned long pfn = folio_pfn(folio); + unsigned long refs = xa_to_value(xa_load(&account->folios, pfn)); + + if (refs == 1) { + xa_erase(&account->folios, pfn); + io_unaccount_mem(account->user, account->mm, + folio_nr_pages(folio)); + } else { + xa_store(&account->folios, pfn, xa_mk_value(refs - 1), + GFP_NOWAIT); + } + } + mutex_unlock(&account->lock); + io_fragment_account_put(account); + } release_user_fragments(buffer->fragments, buffer->count, false); kvfree(buffer); @@ -892,9 +1025,7 @@ static struct io_rsrc_node *io_register_fragment_buffer(struct io_ring_ctx *ctx, struct io_fragment_buffer *buffer; struct io_mapped_ubuf *imu = NULL; struct io_rsrc_node *node; - unsigned long charged = 0; unsigned int i; - struct xarray seen; long nr; int ret = -ENOMEM; @@ -905,7 +1036,8 @@ static struct io_rsrc_node *io_register_fragment_buffer(struct io_ring_ctx *ctx, if (!buffer) goto free_node; buffer->count = 0; - buffer->charged = 0; + buffer->accounted = 0; + buffer->account = NULL; nr = pin_user_fragments_remote(current->mm, (unsigned long)iov->iov_base, iov->iov_len, FOLL_WRITE | FOLL_LONGTERM, buffer->fragments, count); if (nr > 0) @@ -918,26 +1050,18 @@ static struct io_rsrc_node *io_register_fragment_buffer(struct io_ring_ctx *ctx, if (!imu) goto free_buffer; imu->nr_bvecs = count; - /* Several user leaves can share native backing, including aliases. */ - xa_init(&seen); for (i = 0; i < count; i++) { struct user_page_fragment *f = &buffer->fragments[i]; - ret = xa_insert(&seen, folio_pfn(f->folio), xa_mk_value(1), GFP_KERNEL); - if (!ret) - charged += folio_nr_pages(f->folio); - else if (ret != -EBUSY) - break; bvec_set_page(&imu->bvec[i], folio_page(f->folio, f->offset >> PAGE_SHIFT), f->length, offset_in_page(f->offset)); } - xa_destroy(&seen); - if (ret && ret != -EBUSY) + buffer->account = io_fragment_account_get(ctx); + if (!buffer->account) goto free_imu; - ret = io_account_mem(ctx->user, ctx->mm_account, charged); + ret = io_account_fragment_buffer(buffer); if (ret) goto free_imu; - buffer->charged = charged; imu->ubuf = (unsigned long)iov->iov_base; imu->len = iov->iov_len; imu->folio_shift = mm_page_shift(current->mm); From 4e6f2f8a88e6363e7117f01e4f46d52a8379af72 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 13:46:31 -0400 Subject: [PATCH 0898/2680] mm: serialize subpage lazyfree cancellation with discard checks - Take the pool lock when cancelling lazyfree so discard eligibility cannot combine an earlier lazyfree state with a later reference count. - Keep cancellation ordered before releasing the caller's retained reference. Existing callers acquire the helper outside the pool lock. - Validate with 16K/64K KASAN and lockdep builds and existing lazyfree and ownership suites; no concurrent-race or hardware reproduction claimed. Signed-off-by: Diljit Singh (cherry picked from commit 808513ac0c8a339799e0eb8650d2e2b0fd388e38) --- include/linux/mm_subpage.h | 1 + mm/user-subpage.c | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h index 4a6224f8cc95a..81f70bed1f2c8 100644 --- a/include/linux/mm_subpage.h +++ b/include/linux/mm_subpage.h @@ -251,6 +251,7 @@ bool mm_subpage_mark_lazyfree(struct mm_subpage *subpage); bool mm_subpage_all_lazyfree(struct mm_subpage *subpage); bool mm_subpage_discardable(struct mm_subpage *subpage); bool mm_subpage_is_lazyfree(const struct mm_subpage *subpage); +/* Hold a slot reference through cancellation; serializes with discard checks. */ void mm_subpage_cancel_lazyfree(struct mm_subpage *subpage); /* * Caller holds a live slot reference and its PTE lock. Success takes an diff --git a/mm/user-subpage.c b/mm/user-subpage.c index cff26e5b5c803..c08a816b90b20 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -1884,7 +1884,12 @@ bool mm_subpage_is_lazyfree(const struct mm_subpage *subpage) void mm_subpage_cancel_lazyfree(struct mm_subpage *subpage) { + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + + spin_lock_irqsave(&pool->lock, flags); WRITE_ONCE(subpage->lazyfree, false); + spin_unlock_irqrestore(&pool->lock, flags); } bool mm_subpage_mark_lazyfree(struct mm_subpage *subpage) From 6e9d57f3f0bce2854adfacaa2644caa8124514fe Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 13:46:32 -0400 Subject: [PATCH 0899/2680] arm64: manage lazyfree test fragments through assertion cleanup - Keep retained fragments in KUnit-managed storage with a release action so a fatal assertion cannot strand the backing reference. - Use the already-retained folio when exercising reclaim; transient migration PTEs must not turn a successful GET into a failed lookup. - Keep the existing eleven test modes. Both 16K/64K lazyfree runs pass; no concurrent memory-offline experiment was performed. Signed-off-by: Diljit Singh (cherry picked from commit 5c9f905211e7abb99fe60c9eb21872938d1e67b2) --- arch/arm64/mm/user4k-test.c | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 877a16842f2cb..a8cc79960e48a 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -9576,6 +9576,11 @@ static void user4k_droppable_reclaim_test(struct kunit *test) } } +static void test_release_fragment(void *arg) +{ + release_user_fragments(arg, 1, false); +} + static void user4k_lazyfree_test(struct kunit *test) { unsigned int mode; @@ -9588,7 +9593,7 @@ static void user4k_lazyfree_test(struct kunit *test) for (mode = 0; mode < 11; mode++) { struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); struct mm_struct *mm; - struct user_page_fragment held = {}; + struct user_page_fragment *held; struct folio *folio; struct mm_subpage *slot; unsigned int i, succeeded = 0; @@ -9597,11 +9602,15 @@ static void user4k_lazyfree_test(struct kunit *test) LIST_HEAD(folios); KUNIT_ASSERT_NOT_NULL(test, vma); + held = kunit_kzalloc(test, sizeof(*held), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, held); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, test_release_fragment, held), 0); mm = vma->vm_mm; KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); lru_add_drain_all(); if (mode == 6) { - ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1); + ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, held, 1); KUNIT_ASSERT_EQ(test, ret, 1L); } kthread_use_mm(mm); @@ -9621,15 +9630,15 @@ static void user4k_lazyfree_test(struct kunit *test) if (mode == 2 || mode == 3 || mode == 8 || mode == 10) { ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, - mode == 3 ? FOLL_WRITE : 0, &held, 1); + mode == 3 ? FOLL_WRITE : 0, held, 1); KUNIT_ASSERT_EQ(test, ret, 1L); if (mode == 3 || mode == 8) { - memset(folio_address(held.folio) + held.offset, 0xd3, SZ_4K); - release_user_fragments(&held, 1, true); + memset(folio_address(held->folio) + held->offset, 0xd3, SZ_4K); + release_user_fragments(held, 1, true); } } if (mode == 7) { - ret = pin_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1); + ret = pin_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, held, 1); KUNIT_ASSERT_EQ(test, ret, 1L); } if (mode == 5 || mode == 9) { @@ -9646,7 +9655,12 @@ static void user4k_lazyfree_test(struct kunit *test) KUNIT_EXPECT_EQ(test, succeeded, 1U); } lru_add_drain_all(); - folio = test_fault_folio(mm, TEST_VA); + /* A retained GET can coexist with a temporary migration PTE. */ + folio = held->folio; + if (folio) + folio_get(folio); + else + folio = test_fault_folio(mm, TEST_VA); KUNIT_ASSERT_NOT_NULL(test, folio); folio_lock(folio); try_to_unmap(folio, TTU_SYNC); @@ -9655,7 +9669,7 @@ static void user4k_lazyfree_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio)); KUNIT_EXPECT_FALSE(test, folio_test_dirty(folio)); KUNIT_EXPECT_FALSE(test, pte_none(test_fault_entry(test, mm, TEST_VA))); - release_user_fragments(&held, 1, false); + release_user_fragments(held, 1, false); try_to_unmap(folio, TTU_SYNC); } folio_unlock(folio); @@ -9674,7 +9688,9 @@ static void user4k_lazyfree_test(struct kunit *test) (unsigned long long)pte_val(pte)); } if (mode == 2 || mode == 6 || mode == 7) - release_user_fragments(&held, 1, false); + release_user_fragments(held, 1, false); + kunit_release_action(test, test_release_fragment, held); + kunit_kfree(test, held); kunit_release_action(test, user4k_vma_mm_free, mm); } } From 137a1be41e0454e560ee4519b13b75e755c720c7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 14:22:28 -0400 Subject: [PATCH 0900/2680] xen/gntdev: retain typed fragments for mixed-granule grant copies Pin alternative-granule local ranges with the fragment API and derive the Xen frame from the retained backing offset. Keep native fast GUP and the existing hypercall batch size. Cache the fragment array with each batch, selecting its use from the calling MM on every ioctl. Track write intent per local pin so a later source-only segment cannot suppress dirty release for an earlier destination. Release adjacent pins with the same intent together, including on preparation failures. Signed-off-by: Diljit Singh (cherry picked from commit acebd9789c415af2f3e5492d839b8231e930671f) --- drivers/xen/gntdev.c | 82 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 68 insertions(+), 14 deletions(-) diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c index 1dcc4675580ed..c35d81d26e981 100644 --- a/drivers/xen/gntdev.c +++ b/drivers/xen/gntdev.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -64,7 +65,11 @@ struct gntdev_copy_batch { s16 __user *status[GNTDEV_COPY_BATCH]; unsigned int nr_ops; unsigned int nr_pages; - bool writeable; + unsigned long writeable; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment *fragments; + bool use_fragments; +#endif struct gntdev_copy_batch *next; }; @@ -631,6 +636,9 @@ static int gntdev_release(struct inode *inode, struct file *flip) while (priv->batch) { batch = priv->batch; priv->batch = batch->next; +#ifdef CONFIG_MM_SUBPAGE + kfree(batch->fragments); +#endif kfree(batch); } mutex_unlock(&priv->batch_lock); @@ -805,20 +813,45 @@ static long gntdev_ioctl_notify(struct gntdev_priv *priv, void __user *u) } static int gntdev_get_page(struct gntdev_copy_batch *batch, void __user *virt, - unsigned long *gfn) + size_t len, bool writeable, unsigned long *gfn) { unsigned long addr = (unsigned long)virt; struct page *page; unsigned long xen_pfn; int ret; - ret = pin_user_pages_fast(addr, 1, batch->writeable ? FOLL_WRITE : 0, &page); - if (ret < 0) - return ret; - - batch->pages[batch->nr_pages++] = page; +#ifdef CONFIG_MM_SUBPAGE + if (batch->use_fragments) { + struct user_page_fragment *fragment; - xen_pfn = page_to_xen_pfn(page) + XEN_PFN_DOWN(addr & ~PAGE_MASK); + if (!batch->fragments) { + batch->fragments = kcalloc(GNTDEV_COPY_BATCH, + sizeof(*fragment), GFP_KERNEL); + if (!batch->fragments) + return -ENOMEM; + } + fragment = &batch->fragments[batch->nr_pages]; + ret = pin_user_fragments_remote(current->mm, addr, len, + writeable ? FOLL_WRITE : 0, fragment, 1); + if (ret != 1) + return ret < 0 ? ret : -EFAULT; + if (fragment->length != len) { + release_user_fragments(fragment, 1, false); + return -EFAULT; + } + xen_pfn = XEN_PFN_DOWN(PFN_PHYS(folio_pfn(fragment->folio)) + fragment->offset); + } else +#endif + { + ret = pin_user_pages_fast(addr, 1, writeable ? FOLL_WRITE : 0, &page); + if (ret != 1) + return ret < 0 ? ret : -EFAULT; + batch->pages[batch->nr_pages] = page; + xen_pfn = page_to_xen_pfn(page) + XEN_PFN_DOWN(addr & ~PAGE_MASK); + } + if (writeable) + __set_bit(batch->nr_pages, &batch->writeable); + batch->nr_pages++; *gfn = pfn_to_gfn(xen_pfn); return 0; @@ -826,9 +859,24 @@ static int gntdev_get_page(struct gntdev_copy_batch *batch, void __user *virt, static void gntdev_put_pages(struct gntdev_copy_batch *batch) { - unpin_user_pages_dirty_lock(batch->pages, batch->nr_pages, batch->writeable); + unsigned int first = 0; + + while (first < batch->nr_pages) { + bool dirty = test_bit(first, &batch->writeable); + unsigned int next = dirty ? + find_next_zero_bit(&batch->writeable, batch->nr_pages, first) : + find_next_bit(&batch->writeable, batch->nr_pages, first); + +#ifdef CONFIG_MM_SUBPAGE + if (batch->use_fragments) + release_user_fragments(batch->fragments + first, next - first, dirty); + else +#endif + unpin_user_pages_dirty_lock(batch->pages + first, next - first, dirty); + first = next; + } batch->nr_pages = 0; - batch->writeable = false; + batch->writeable = 0; } static int gntdev_copy(struct gntdev_copy_batch *batch) @@ -917,9 +965,8 @@ static int gntdev_grant_copy_seg(struct gntdev_copy_batch *batch, virt = seg->source.virt + copied; off = (unsigned long)virt & ~XEN_PAGE_MASK; len = min(len, (size_t)XEN_PAGE_SIZE - off); - batch->writeable = false; - ret = gntdev_get_page(batch, virt, &gfn); + ret = gntdev_get_page(batch, virt, len, false, &gfn); if (ret < 0) return ret; @@ -937,9 +984,8 @@ static int gntdev_grant_copy_seg(struct gntdev_copy_batch *batch, virt = seg->dest.virt + copied; off = (unsigned long)virt & ~XEN_PAGE_MASK; len = min(len, (size_t)XEN_PAGE_SIZE - off); - batch->writeable = true; - ret = gntdev_get_page(batch, virt, &gfn); + ret = gntdev_get_page(batch, virt, len, true, &gfn); if (ret < 0) return ret; @@ -971,6 +1017,10 @@ static long gntdev_ioctl_grant_copy(struct gntdev_priv *priv, void __user *u) mutex_lock(&priv->batch_lock); if (!priv->batch) { batch = kmalloc_obj(*batch); +#ifdef CONFIG_MM_SUBPAGE + if (batch) + batch->fragments = NULL; +#endif } else { batch = priv->batch; priv->batch = batch->next; @@ -981,6 +1031,10 @@ static long gntdev_ioctl_grant_copy(struct gntdev_priv *priv, void __user *u) batch->nr_ops = 0; batch->nr_pages = 0; + batch->writeable = 0; +#ifdef CONFIG_MM_SUBPAGE + batch->use_fragments = mm_page_size(current->mm) < PAGE_SIZE; +#endif for (i = 0; i < copy.count; i++) { struct gntdev_grant_copy_segment seg; From 7ea2b61d0604c8f4bc80e104d8615214d3ecf3f7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 14:34:29 -0400 Subject: [PATCH 0901/2680] mm: retire poisoned subpage mappings and exclude poisoned backing - Replace poisoned anonymous subpage PTEs with hardware-poison entries under the existing folio/PTE locks and MMU notifier scope. - Complete invalidation before dropping mapping ownership, preserving exact slot references retained by pins or asynchronous users. - Reject poisoned pool supply and skip poisoned owners during allocation; unused neighboring slots share the same native hardware-error granule. - Preserve the existing swap-cache containment path when TTU_HWPOISON is cleared by unmap_poisoned_folio(). Signed-off-by: Diljit Singh (cherry picked from commit ab3d927968dd9fb0563b26db4cc5fb7fb40a3d34) --- mm/rmap.c | 18 +++++++++++++++++- mm/user-subpage.c | 6 ++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/mm/rmap.c b/mm/rmap.c index de8bed44b5f20..053164212efa6 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2161,11 +2161,27 @@ static bool try_to_unmap_subpage(struct folio *folio, struct vm_area_struct *vma swp_entry_t entry; bool exclusive, ret = false; - if (!pte_present(pte) || (flags & TTU_HWPOISON)) + if (!pte_present(pte)) return false; slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); if (WARN_ON_ONCE(!slot)) return false; + if (folio_test_hwpoison(folio) && (flags & TTU_HWPOISON)) { + flush_cache_range(vma, address, end); + pte = ptep_get_and_clear(mm, address, pvmw->pte); + /* Retained references keep their slot, but no CPU may keep its PTE. */ + flush_tlb_range(vma, address, end); + if (pte_dirty(pte)) + folio_mark_dirty(folio); + update_hiwater_rss(mm); + dec_mm_counter(mm, MM_ANONPAGES); + set_pte_at(mm, address, pvmw->pte, + swp_entry_to_pte(make_hwpoison_entry(&folio->page))); + mm_subpage_remove_anon_rmap(slot, vma); + mm_subpage_put(slot); /* Removed PTE reference, after invalidation. */ + ret = true; + goto out; + } if ((vma->vm_flags & VM_DROPPABLE) || mm_subpage_is_lazyfree(slot)) { flush_cache_range(vma, address, end); pte = ptep_get_and_clear(mm, address, pvmw->pte); diff --git a/mm/user-subpage.c b/mm/user-subpage.c index c08a816b90b20..bcb375cf53eca 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -1086,7 +1086,8 @@ static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, (!pool->cow_shared || !folio_test_locked(owner->folio)) && cow_owner_eligible(pool, owner->folio) && !folio_test_swapcache(owner->folio) && - !folio_test_writeback(owner->folio)) + !folio_test_writeback(owner->folio) && + !folio_test_hwpoison(owner->folio)) return owner; } return NULL; @@ -1103,7 +1104,8 @@ static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, /* Only an unexposed order-0 allocation can become a slot owner. */ if (folio_order(folio) || folio_ref_count(folio) != 1 || folio->mapping || folio_mapped(folio) || folio_test_lru(folio) || - folio_test_private(folio) || folio_test_swapcache(folio)) + folio_test_private(folio) || folio_test_swapcache(folio) || + folio_test_hwpoison(folio)) return -EINVAL; owner = kzalloc(sizeof(*owner), gfp | __GFP_ACCOUNT); if (!owner) From f4f2c9e695a8a177b3067dd96143ea032f3d0193 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 14:50:27 -0400 Subject: [PATCH 0902/2680] mm: reject non-native geometry in native PFN array interfaces - HMM results represent whole native pages without fragment offsets or per-leaf permissions. Reject non-native MMs before walking or changing the output PFN array. - Device VMA migration collects, owns and restores native pages. Reject non-native MMs before rounding endpoints or modifying mappings/arrays. - Return -EOPNOTSUPP as a non-retry error. Native MMs keep the existing paths; enabling alternatives requires converting the complete API and its consumers together. Signed-off-by: Diljit Singh (cherry picked from commit 39c50fbe0687b81bdc2664da3dd051b3ebc43a5a) --- mm/hmm.c | 4 ++++ mm/migrate_device.c | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/mm/hmm.c b/mm/hmm.c index c72c9ddfdb2f6..4f5fe14094f52 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -652,6 +652,7 @@ static const struct mm_walk_ops hmm_walk_ops = { * the invalidation to finish. * -EFAULT: A page was requested to be valid and could not be made valid * ie it has no backing VMA or it is illegal to access + * -EOPNOTSUPP: The MM uses a non-native mapping granule. * * This is similar to get_user_pages(), except that it can read the page tables * without mutating them (ie causing faults). @@ -666,6 +667,9 @@ int hmm_range_fault(struct hmm_range *range) int ret; mmap_assert_locked(mm); + /* Each output PFN describes a whole native page, without a byte offset. */ + if (mm_page_size(mm) != PAGE_SIZE) + return -EOPNOTSUPP; do { /* If range is no longer valid force retry. */ diff --git a/mm/migrate_device.c b/mm/migrate_device.c index aa9407397e5a8..69af794043b58 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -668,7 +668,7 @@ static void migrate_vma_unmap(struct migrate_vma *migrate) * @args: contains the vma, start, and pfns arrays for the migration * * Returns: negative errno on failures, 0 when 0 or more pages were migrated - * without an error. + * without an error. Non-native MM granules return -EOPNOTSUPP. * * Prepare to migrate a range of memory virtual address range by collecting all * the pages backing each virtual address in the range, saving them inside the @@ -731,6 +731,10 @@ int migrate_vma_setup(struct migrate_vma *args) { long nr_pages = (args->end - args->start) >> PAGE_SHIFT; + /* PFN arrays and migration ownership describe whole native pages. */ + if (args->vma && mm_page_size(args->vma->vm_mm) != PAGE_SIZE) + return -EOPNOTSUPP; + args->start &= PAGE_MASK; args->end &= PAGE_MASK; if (!args->vma || is_vm_hugetlb_page(args->vma) || From a8a4b92eec1b7fe0d3dff6c849d8db303a1fe75f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 11 Oct 2026 17:26:09 -0400 Subject: [PATCH 0903/2680] mm: exclude poisoned backing from COW pool retry and revalidation The published COW pool work added a pressure retry scan and folio-locked eligibility rechecks after the original poison-containment fix was prepared. Checking hwpoison only in find_available leaves those paths inconsistent: the pressure path can report unusable poisoned backing as retryable. Put the exclusion in the shared eligibility predicate, before the non-COW early return, so ordinary allocation, pressure retry and locked revalidation all use the same condition. Keep the fresh-supply rejection. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c404199c7213c475f36788bd9336e2f9ebb89651) --- mm/user-subpage.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mm/user-subpage.c b/mm/user-subpage.c index bcb375cf53eca..88f043bae28b2 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -988,6 +988,9 @@ void mm_subpage_pool_close(struct mm_subpage_pool *pool) static bool cow_owner_eligible(struct mm_subpage_pool *pool, struct folio *folio) { + /* All allocation and retry paths must exclude poisoned backing. */ + if (folio_test_hwpoison(folio)) + return false; if (!pool->cow_shared) return true; /* A previously unlocked owner can be mlocked or fully MADV_FREE'd @@ -1086,8 +1089,7 @@ static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, (!pool->cow_shared || !folio_test_locked(owner->folio)) && cow_owner_eligible(pool, owner->folio) && !folio_test_swapcache(owner->folio) && - !folio_test_writeback(owner->folio) && - !folio_test_hwpoison(owner->folio)) + !folio_test_writeback(owner->folio)) return owner; } return NULL; From d09ea4cb08370b64bf8ac1fa631174aafe4b26f2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 11 Oct 2026 17:32:45 -0400 Subject: [PATCH 0904/2680] mm: protect COW backing charge comparisons with RCU The remote fragment-pin/COW test reaches mm_subpage_cow_folio_matches() without an RCU read lock. With lockdep enabled, folio_memcg() warns in obj_cgroup_memcg() because the charge's memcg can be reparented. Hold RCU across the comparison as required by folio_memcg(). The pool's retained memcg reference does not protect a different memcg obtained from the folio. No returned memcg pointer escapes the read-side section. Fixes: 5469651a0899 ("mm: reuse backing folios for small-granule anonymous COW") Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 613963008f40faa6787e0a678afd7a008ca52554) --- mm/user-subpage.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mm/user-subpage.c b/mm/user-subpage.c index 88f043bae28b2..68e410e546302 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -510,7 +510,14 @@ struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *folio) { - return pool->cow_memcg == folio_memcg(folio) && pool->cow_nid == folio_nid(folio); + bool matches; + + /* The folio charge may be reparented while this pool remains alive. */ + rcu_read_lock(); + matches = pool->cow_memcg == folio_memcg(folio) && + pool->cow_nid == folio_nid(folio); + rcu_read_unlock(); + return matches; } void mm_subpage_cow_pool_exit(struct mm_struct *mm) From bbe153c9f7fa07f7298374cc89caf04bdb233ec1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 11 Oct 2026 17:38:01 -0400 Subject: [PATCH 0905/2680] arm64: validate COW tests against the mapped physical quarter COW backing can now use a physical quarter unrelated to the fault's virtual quarter. Update the zero, fork, file and fragment-permission fixtures to validate the actual PTE and slot rather than requiring the old fixed-offset allocation policy. Read zeroed bytes and write child-private bytes at the actual slot offset. Keep the checks for untouched neighboring PTEs, copied data, parent and file-cache isolation, mapping ownership and access permissions. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit ef7f72d427299dd14bfed784ba917d19967f0bb3) --- arch/arm64/mm/user4k-test.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index a8cc79960e48a..f5cd92058acce 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -7098,7 +7098,7 @@ static void user4k_file_cow_test(struct kunit *test) KUNIT_ASSERT_TRUE(test, pte_present(entry)); KUNIT_EXPECT_TRUE(test, pte_write(entry)); KUNIT_EXPECT_TRUE(test, folio_test_anon(page_folio(pte_page(entry)))); - KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, entry) & ~PAGE_MASK, 2UL * SZ_4K); + /* Private COW can pack into any physical quarter of its backing. */ KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(vma->vm_mm, entry)), 0x33, SZ_4K), NULL); memset(phys_to_virt(pte_phys_mm(vma->vm_mm, entry)), 0x91, SZ_4K); @@ -7316,11 +7316,12 @@ static void test_zero_cow_granule(struct kunit *test, unsigned int shift) KUNIT_ASSERT_TRUE(test, pte_write(after)); slot = mm_subpage_get_from_phys(pte_phys_mm(mm, after)); KUNIT_ASSERT_NOT_NULL(test, slot); - KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), (unsigned int)size); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), + (unsigned int)offset_in_page(pte_phys_mm(mm, after))); KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); KUNIT_EXPECT_EQ(test, folio_ref_count(mm_subpage_folio(slot)), 2); KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + - size, 0, size), NULL); + mm_subpage_offset(slot), 0, size), NULL); mm_subpage_put(slot); } KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1UL); @@ -7461,13 +7462,15 @@ static void test_fork_cow_granule(struct kunit *test, unsigned int shift) KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, after) != mm_subpage_phys(held[i]), changed); slot = mm_subpage_get_from_phys(pte_phys_mm(mm, after)); KUNIT_ASSERT_NOT_NULL(test, slot); - KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * (unsigned int)size); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), + (unsigned int)offset_in_page(pte_phys_mm(mm, after))); KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + mm_subpage_offset(slot), 0x31 + i, size), NULL); if (changed) { KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); KUNIT_EXPECT_EQ(test, folio_ref_count(mm_subpage_folio(slot)), 2); - memset(folio_address(mm_subpage_folio(slot)) + size, 0x77, size); + memset(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0x77, size); } mm_subpage_put(slot); } @@ -9227,7 +9230,9 @@ static void user4k_fragment_permissions_test(struct kunit *test) ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_WRITE, 8); KUNIT_ASSERT_EQ(test, ret, 1L); KUNIT_EXPECT_NOT_NULL(test, fragment->subpage); - KUNIT_EXPECT_EQ(test, fragment->offset, SZ_4K + 17UL); + /* COW backing may use a different quarter from the virtual address. */ + KUNIT_EXPECT_EQ(test, page_to_phys(&fragment->folio->page) + fragment->offset, + pte_phys_mm(mm, test_fault_entry(test, mm, start)) + 17); memset(folio_address(fragment->folio) + fragment->offset, 0xa4, fragment->length); fragment_test_unpin(pins); mmap_write_lock(mm); From b79a87d77447eba08ed208dc215a5d0d2be0a5bd Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 11 Oct 2026 19:36:01 -0400 Subject: [PATCH 0906/2680] mm: guard subpage COW pool charge references without MEMCG get_mem_cgroup_from_mm() returns NULL without CONFIG_MEMCG, but the COW pool still references the incomplete struct mem_cgroup in the css_get() expression. Guard the retained-reference operation so mixed-granule kernels build without memory cgroups. Keep the reference acquisition unchanged for enabled memory cgroups. --- mm/user-subpage.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/user-subpage.c b/mm/user-subpage.c index 68e410e546302..c9101990099df 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -466,8 +466,10 @@ struct mm_subpage_pool *mm_subpage_cow_pool_get_charged(struct mm_struct *mm, mm_subpage_pool_put(new); break; } +#ifdef CONFIG_MEMCG if (memcg) css_get(&memcg->css); +#endif new->cow_memcg = memcg; new->cow_nid = nid; new->cow_shared = true; From a61b6cb244ecef098a904cedeaafabcb2ba02da5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 11 Oct 2026 19:42:05 -0400 Subject: [PATCH 0907/2680] arm64: validate packed fragments through mapped physical offsets The shared COW pool may place a private fragment in another physical quarter of the same native folio. Sparse anonymous allocations likewise need not match their virtual-page index. Follow actual PTE/subpage offsets when checking pinned copies, live COW writes and restored swap data; record pre-migration and pre-swap offsets where the operation must preserve them. Verify tags at the actual copied slot and require child read pins and swap COW writes to retain distinct physical backing from the parent. Keep data, permissions, ownership and accounting assertions intact. Validated with 233 passing ARM64 KUnit tests (39 skips), all 17 disposable swap tests, and native16K/4K exec and thread VM fixtures. A separate memory-cgroup-enabled COW object build also passes. --- arch/arm64/mm/user4k-test.c | 72 +++++++++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index f5cd92058acce..eb17e0637d67f 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -3019,7 +3019,9 @@ static void test_anon_write_fault_granule(struct kunit *test, unsigned int shift if (!backing) backing = folio; identity = folio == backing && - mm_subpage_offset(slot) == i * size; + mm_subpage_phys(slot) == + pte_phys_mm(vma->vm_mm, ptep_get(pte)) && + (shape || mm_subpage_offset(slot) == i * size); zero = !memchr_inv(folio_address(folio) + mm_subpage_offset(slot), 0, size); mm_subpage_put(slot); @@ -8555,7 +8557,8 @@ static void test_swap_tags_set(struct folio *folio) #endif } -static bool test_swap_tags_match(struct folio *folio, unsigned int slot) +static bool test_swap_tags_match_at(struct folio *folio, unsigned int start, + unsigned int slot) { #ifdef CONFIG_ARM64_MTE unsigned int offset; @@ -8564,7 +8567,7 @@ static bool test_swap_tags_match(struct folio *folio, unsigned int slot) return true; if (!page_mte_tagged(&folio->page)) return false; - for (offset = slot * SZ_4K; offset < (slot + 1) * SZ_4K; offset += MTE_GRANULE_SIZE) { + for (offset = start; offset < start + SZ_4K; offset += MTE_GRANULE_SIZE) { unsigned long tag = 0; asm volatile(".arch_extension memtag\n\tldg %0, [%1]" @@ -8576,6 +8579,11 @@ static bool test_swap_tags_match(struct folio *folio, unsigned int slot) return true; } +static bool test_swap_tags_match(struct folio *folio, unsigned int slot) +{ + return test_swap_tags_match_at(folio, slot * SZ_4K, slot); +} + struct fragment_test_pins { struct user_page_fragment fragments[2 * TEST_SLOTS]; unsigned long count; @@ -8683,9 +8691,14 @@ static void user4k_fragment_pin_test(struct kunit *test) KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, 0x71 + i, f->length), NULL); if (!native) { + pte_t parent_pte = test_fault_entry(test, mm, TEST_VA + i * size); + KUNIT_EXPECT_TRUE(test, mm_subpage_is_exclusive(f->subpage)); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, pte), + pte_phys_mm(mm, parent_pte)); if (i) - KUNIT_EXPECT_PTR_NE(test, f->folio, pins->fragments[0].folio); + KUNIT_EXPECT_NE(test, mm_subpage_phys(f->subpage), + mm_subpage_phys(pins->fragments[0].subpage)); } } fragment_test_unpin(pins); @@ -9339,9 +9352,10 @@ static void user4k_slot_pin_test(struct kunit *test) copy = mm_subpage_get_from_phys(pte_phys_mm(child, pte)); KUNIT_ASSERT_NOT_NULL(test, copy); KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(copy), i == 1); - KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(copy)) + i * SZ_4K, + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(copy)) + mm_subpage_offset(copy), 0x61 + i, SZ_4K), NULL); - KUNIT_EXPECT_TRUE(test, test_swap_tags_match(mm_subpage_folio(copy), i)); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match_at(mm_subpage_folio(copy), + mm_subpage_offset(copy), i)); mm_subpage_put(copy); } /* A read pin on a shared slot must request unsharing, not pin shared COW data. */ @@ -10326,6 +10340,7 @@ static void user4k_migration_test(struct kunit *test) unsigned long start = edge ? boundaries[mode - 6] : TEST_VA + (sparse ? SZ_4K : 0); unsigned int nr = sparse ? 1 : 4, i, side, succeeded = 0; + unsigned int offsets[TEST_SLOTS]; struct vm_area_struct *vma = test_fault_vma(test, start, nr * SZ_4K); struct mm_struct *child = NULL; struct folio *folio; @@ -10352,6 +10367,9 @@ static void user4k_migration_test(struct kunit *test) for (i = 0; i < TEST_SLOTS; i++) memset(folio_address(folio) + i * SZ_4K, 0x41 + i + mode, SZ_4K); test_swap_tags_set(folio); + for (i = 0; i < nr; i++) + offsets[i] = offset_in_page(pte_phys_mm(vma->vm_mm, + test_fault_entry(test, vma->vm_mm, start + i * SZ_4K))); folio_put(folio); if (forked) { child = test_vma_mm(test); @@ -10381,7 +10399,7 @@ static void user4k_migration_test(struct kunit *test) struct mm_struct *mm = side ? child : vma->vm_mm; for (i = 0; i < nr; i++) { - unsigned int index = sparse ? offset_in_page(start) / SZ_4K : i; + unsigned int index = offsets[i] / SZ_4K; pte_t pte = test_fault_entry(test, mm, start + i * SZ_4K); struct mm_subpage *slot; @@ -10868,13 +10886,27 @@ static void test_user4k_swap_roundtrip(struct kunit *test, unsigned int mode, u8 KUNIT_EXPECT_TRUE(test, pte_write(pte)); folio = test_fault_folio(child, TEST_VA + SZ_4K); KUNIT_ASSERT_NOT_NULL(test, folio); - KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, base + 1, SZ_4K), NULL); - KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, 1)); - memset(folio_address(folio) + SZ_4K, 0xa7, SZ_4K); + { + unsigned int offset = pte_phys_mm(child, pte) - PFN_PHYS(folio_pfn(folio)); + + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + offset, + base + 1, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match_at(folio, offset, 1)); + memset(folio_address(folio) + offset, 0xa7, SZ_4K); + } folio_put(folio); folio = test_fault_folio(parent->vm_mm, TEST_VA + SZ_4K); KUNIT_ASSERT_NOT_NULL(test, folio); - KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, base + 1, SZ_4K), NULL); + { + pte_t parent_pte = test_fault_entry(test, parent->vm_mm, TEST_VA + SZ_4K); + unsigned int offset = pte_phys_mm(parent->vm_mm, parent_pte) - + PFN_PHYS(folio_pfn(folio)); + + KUNIT_EXPECT_NE(test, pte_phys_mm(child, pte), + pte_phys_mm(parent->vm_mm, parent_pte)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + offset, + base + 1, SZ_4K), NULL); + } folio_put(folio); KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 1UL); KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), TEST_SLOTS - 1UL); @@ -11249,12 +11281,13 @@ static void user4k_swapoff_boundaries_test(struct kunit *test) struct vm_area_struct *parent = test_fault_vma(test, start, 2 * SZ_4K); struct mm_struct *child; unsigned int i, side; + unsigned int offsets[2]; int ret; KUNIT_ASSERT_NOT_NULL(test, parent); for (i = 0; i < 2; i++) { unsigned long addr = start + i * SZ_4K; - unsigned int offset = vma_page_offset_at(parent, addr).offset; + unsigned int offset; struct folio *folio; vm_fault_t fault; swp_entry_t entry; @@ -11264,6 +11297,9 @@ static void user4k_swapoff_boundaries_test(struct kunit *test) fault = handle_mm_fault(parent, addr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); mmap_read_unlock(parent->vm_mm); KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + offset = offset_in_page(pte_phys_mm(parent->vm_mm, + test_fault_entry(test, parent->vm_mm, addr))); + offsets[i] = offset; lru_add_drain_all(); folio = test_fault_folio(parent->vm_mm, addr); KUNIT_ASSERT_NOT_NULL(test, folio); @@ -11295,7 +11331,7 @@ static void user4k_swapoff_boundaries_test(struct kunit *test) KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 2UL); for (i = 0; i < 2; i++) { unsigned long addr = start + i * SZ_4K; - unsigned int offset = vma_page_offset_at(parent, addr).offset; + unsigned int offset = offsets[i]; pte_t pte = test_fault_entry(test, mm, addr); struct folio *folio; @@ -11522,8 +11558,14 @@ static void test_live_fault_roundtrip(struct kunit *test, unsigned int mode) } folio = test_fault_folio(child, TEST_VA + SZ_4K); KUNIT_ASSERT_NOT_NULL(test, folio); - for (i = 0; i < 2; i++) - KUNIT_EXPECT_EQ(test, *((u8 *)folio_address(folio) + SZ_4K + 64 + i), (u8)(0xe0 + i)); + { + pte_t pte = test_fault_entry(test, child, TEST_VA + SZ_4K); + unsigned long offset = pte_phys_mm(child, pte) - PFN_PHYS(folio_pfn(folio)); + + for (i = 0; i < 2; i++) + KUNIT_EXPECT_EQ(test, *((u8 *)folio_address(folio) + offset + 64 + i), + (u8)(0xe0 + i)); + } folio_put(folio); mmap_read_lock(parent->vm_mm); fault = handle_mm_fault(parent, TEST_VA + SZ_4K, FAULT_FLAG_REMOTE, NULL); From c656b928388a1313f9996cd1d4cf978ad66e4c31 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:37:55 +1100 Subject: [PATCH 0908/2680] btrfs: keep the extent map referenced until a compressed read is submitted When btrfs_do_readpage() adds blocks of a compressed extent to a bio it drops its reference on the extent map, and btrfs_submit_compressed_read() looks the extent map up again in the inode's extent map tree, by the bio's file offset, to learn where the extent starts in the file and how it is compressed. If the lookup misses, the whole bio fails with -EIO. The lookup can miss. The bio is submitted when a later block does not fit it, or at the end of the read_folio/readahead call after the range has been unlocked, and the extent map shrinker can remove the extent map from the tree in between: apart from pinned extent maps it drops whatever it finds, regardless of locked io tree ranges or of references held by readers. Under memory pressure, in particular in a memory cgroup at its limit whose reclaim keeps invoking the fs shrinker, this shows up as spurious -EIO on reads of compressed files, and on buffered writes that have to read a partial folio first. Have the bio_ctrl hold a reference on the extent map of the compressed extent its bio reads, handed over by btrfs_do_readpage() once the block is in the bio and dropped once the bio is submitted, and pass it to btrfs_submit_compressed_read() instead of looking it up again. Fixes: 956a17d9d050 ("btrfs: add a shrinker for extent maps") Assisted-by: LLM Signed-off-by: Ryan Murray --- fs/btrfs/compression.c | 18 +++++++--------- fs/btrfs/compression.h | 3 ++- fs/btrfs/extent_io.c | 47 ++++++++++++++++++++++++++++++++++++++---- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index ffb6b52863a78..87606f92e340b 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -519,11 +519,10 @@ static noinline int add_ra_bio_folios(struct inode *inode, u64 compressed_end, * After the compressed pages are read, we copy the bytes into the * bio we were passed and then call the bio end_io calls */ -void btrfs_submit_compressed_read(struct btrfs_bio *bbio) +void btrfs_submit_compressed_read(struct btrfs_bio *bbio, struct extent_map *em) { struct btrfs_inode *inode = bbio->inode; struct btrfs_fs_info *fs_info = inode->root->fs_info; - struct extent_map_tree *em_tree = &inode->extent_tree; struct compressed_bio *cb; unsigned int compressed_len; const u32 min_folio_size = btrfs_min_folio_size(fs_info); @@ -531,7 +530,6 @@ void btrfs_submit_compressed_read(struct btrfs_bio *bbio) gfp_t gfp; u64 em_len; u64 em_start; - struct extent_map *em; unsigned long pflags; int memstall = 0; int ret; @@ -547,16 +545,18 @@ void btrfs_submit_compressed_read(struct btrfs_bio *bbio) else gfp = GFP_NOFS; - /* we need the actual starting offset of this extent in the file */ - read_lock(&em_tree->lock); - em = btrfs_lookup_extent_mapping(em_tree, file_offset, fs_info->sectorsize); - read_unlock(&em_tree->lock); - if (!em) { + /* + * We need the actual starting offset of this extent in the file. The + * caller holds a reference on the extent map the bio was built from; + * it may no longer be in the inode's extent map tree. + */ + if (WARN_ON_ONCE(!em)) { ret = -EIO; goto out; } ASSERT(btrfs_extent_map_is_compressed(em)); + ASSERT(em->start <= file_offset && file_offset < btrfs_extent_map_end(em)); compressed_len = em->disk_num_bytes; cb = alloc_compressed_bio(inode, file_offset, REQ_OP_READ, @@ -571,8 +571,6 @@ void btrfs_submit_compressed_read(struct btrfs_bio *bbio) cb->orig_bbio = bbio; cb->bbio.csum_search_commit_root = bbio->csum_search_commit_root; - btrfs_free_extent_map(em); - for (int i = 0; i * min_folio_size < compressed_len; i++) { struct folio *folio; u32 cur_len = min(compressed_len - i * min_folio_size, min_folio_size); diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h index 1022dc53ec51e..6ccbcfaf3280e 100644 --- a/fs/btrfs/compression.h +++ b/fs/btrfs/compression.h @@ -20,6 +20,7 @@ struct address_space; struct inode; struct btrfs_inode; struct btrfs_ordered_extent; +struct extent_map; /* * We want to make sure that amount of RAM required to uncompress an extent is @@ -94,7 +95,7 @@ struct compressed_bio *btrfs_alloc_compressed_write(struct btrfs_inode *inode, u64 start, u64 len); void btrfs_submit_compressed_write(struct btrfs_ordered_extent *ordered, struct compressed_bio *cb); -void btrfs_submit_compressed_read(struct btrfs_bio *bbio); +void btrfs_submit_compressed_read(struct btrfs_bio *bbio, struct extent_map *em); int btrfs_compress_str2level(unsigned int type, const char *str, int *level_ret); diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index ae6294fedd2d7..d199712531218 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -152,6 +152,16 @@ struct btrfs_bio_ctrl { * inside the same inode. */ u64 last_em_start; + + /* + * For a compressed read, the extent map of the compressed extent that + * @bbio reads, referenced until the bio is submitted. + * + * btrfs_submit_compressed_read() needs it, but must not look it up in + * the inode's extent map tree again: by then the extent map shrinker + * may have removed it from the tree. + */ + struct extent_map *compressed_em; }; /* @@ -178,8 +188,10 @@ static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl) { struct btrfs_bio *bbio = bio_ctrl->bbio; - if (!bbio) + if (!bbio) { + ASSERT(!bio_ctrl->compressed_em); return; + } /* Caller should ensure the bio has at least some range added */ ASSERT(bbio->bio.bi_iter.bi_size); @@ -188,12 +200,14 @@ static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl) if (btrfs_op(&bbio->bio) == BTRFS_MAP_READ && bio_ctrl->compress_type != BTRFS_COMPRESS_NONE) - btrfs_submit_compressed_read(bbio); + btrfs_submit_compressed_read(bbio, bio_ctrl->compressed_em); else btrfs_submit_bbio(bbio, 0); /* The bbio is owned by the end_io handler now */ bio_ctrl->bbio = NULL; + btrfs_free_extent_map(bio_ctrl->compressed_em); + bio_ctrl->compressed_em = NULL; /* * We used the generation to decide whether to lookup csums in the * commit_root or not when we called bio_set_csum_search_commit_root() @@ -1025,6 +1039,26 @@ static struct extent_map *get_extent_map(struct btrfs_inode *inode, return em; } +/* + * A block of the compressed extent mapped by @em has just been added to + * @bio_ctrl->bbio, which only ever reads blocks of one compressed extent. + * Take over the caller's reference on @em: bio_ctrl keeps one reference on + * the extent map of its bio until submit_one_bio() passes it on. + */ +static void bio_ctrl_hold_compressed_em(struct btrfs_bio_ctrl *bio_ctrl, + struct extent_map *em) +{ + struct extent_map *old = bio_ctrl->compressed_em; + + /* + * Any reference held so far is for an earlier block of the same bio, + * on the same extent map or on a copy of it loaded again after the + * shrinker dropped it: one is enough. + */ + bio_ctrl->compressed_em = em; + btrfs_free_extent_map(old); +} + static void btrfs_readahead_expand(struct readahead_control *ractl, const struct extent_map *em) { @@ -1184,17 +1218,17 @@ static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached, bio_ctrl->last_em_start = em->start; em_gen = em->generation; - btrfs_free_extent_map(em); - em = NULL; /* we've found a hole, just zero and go on */ if (block_start == EXTENT_MAP_HOLE) { + btrfs_free_extent_map(em); folio_zero_range(folio, pg_offset, blocksize); end_folio_read(vi, folio, true, cur, blocksize); continue; } /* the get_extent function already copied into the folio */ if (block_start == EXTENT_MAP_INLINE) { + btrfs_free_extent_map(em); end_folio_read(vi, folio, true, cur, blocksize); continue; } @@ -1210,6 +1244,11 @@ static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached, pg_offset, em_gen); /* Read submission should not fail. */ ASSERT(queued == blocksize); + + if (compress_type != BTRFS_COMPRESS_NONE) + bio_ctrl_hold_compressed_em(bio_ctrl, em); + else + btrfs_free_extent_map(em); } return 0; } From 1d1970be76d0ff5a8bd26992d47d6dc5c0ca8471 Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Fri, 25 Sep 2026 01:42:12 -0600 Subject: [PATCH 0909/2680] arm64: dts: apple: t600x: add spi2 and the Mesa fingerprint sensor Describe the SPI2 controller at 0x39b108000 and its Mesa fingerprint sensor child, both disabled. The interrupt (AIC 1108) comes from the J316s platform device tree (/arm-io/spi2, 0x454), which also gives SPI1 and SPI3 as 1107 and 1109, matching the existing nodes. The pin group was read back from J316s hardware. iBoot leaves pads 48-50 at function 1 with the same register values as spi3's signal pads 52-54, while pad 51 is unconfigured, the same as the unused pads 45-47. The platform device tree gives SPI2's chip select as "null", so the bus uses the controller's native chip select, as on t8103 and t8112. The clock parent is not in the platform device tree. On t8103 and t602x SPI2 shares its neighbour's parent, so this takes spi3's clkref. Assisted-by: Claude Opus 5.5 Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t600x-die0.dtsi | 28 +++++++++++++++++++ .../arm64/boot/dts/apple/t600x-gpio-pins.dtsi | 12 ++++++++ 2 files changed, 40 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index 83451df18ac25..aeef6d7f8b97c 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -663,6 +663,34 @@ status = "disabled"; }; + /* + * The Mesa fingerprint sensor's bus, from the J316s platform device tree + * (/arm-io/spi2: AIC 0x454). The clock follows spi3, as spi2 does on + * t8103 and t602x. Do not add cs-gpios: the sensor needs the + * controller's native chip select. + */ + spi2: spi@39b108000 { + compatible = "apple,t6000-spi", "apple,spi"; + reg = <0x3 0x9b108000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&clkref>; + pinctrl-0 = <&spi2_pins>; + pinctrl-names = "default"; + power-domains = <&ps_spi2>; + status = "disabled"; + + mesa: fingerprint@0 { + compatible = "apple,mesa-fingerprint"; + reg = <0>; + spi-max-frequency = <8000000>; + spi-cpol; + status = "disabled"; + }; + }; + spi3: spi@39b10c000 { compatible = "apple,t6000-spi", "apple,spi"; reg = <0x3 0x9b10c000 0x0 0x4000>; diff --git a/arch/arm64/boot/dts/apple/t600x-gpio-pins.dtsi b/arch/arm64/boot/dts/apple/t600x-gpio-pins.dtsi index 1a994c3c1b79f..476053d774365 100644 --- a/arch/arm64/boot/dts/apple/t600x-gpio-pins.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-gpio-pins.dtsi @@ -43,6 +43,18 @@ ; }; + /* + * Read back from J316s hardware: iBoot leaves pads 48-50 at function 1 + * with the same register values as spi3's signal pads 52-54, and pad 51 + * unconfigured. The platform device tree gives spi2's chip select as + * "null", so it uses the controller's native chip select. + */ + spi2_pins: spi2-pins { + pinmux = , + , + ; + }; + spi3_pins: spi3-pins { pinmux = , , From 5841557049a4b2ee5ad81154e8e174af0e9ac255 Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Fri, 25 Sep 2026 01:42:12 -0600 Subject: [PATCH 0910/2680] arm64: dts: apple: t6000-j316s: enable the SEP and the Mesa sensor Add the sep alias and enable the SEP node. m1n1's dt_set_sep() keys off the alias to add the sepfw region and the local-policy and iBoot manifests the driver needs to cold-boot SEPOS, as on t8103-j313. Enable SPI2 and the Mesa sensor. From the J316s platform device tree (/arm-io/spi2/mesa): the power line is gpio0 pin 196 (function-mesa_pwr, active high) and the data-ready interrupt is pin 195. The same reading of the J414s tree gives 122 and 121, the lines the driver hardcodes for that board. Assisted-by: Claude Opus 5.5 Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t6000-j316s.dts | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6000-j316s.dts b/arch/arm64/boot/dts/apple/t6000-j316s.dts index ddfc3c530923c..8ec8a0c91715c 100644 --- a/arch/arm64/boot/dts/apple/t6000-j316s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j316s.dts @@ -15,6 +15,41 @@ / { compatible = "apple,j316s", "apple,t6000", "apple,arm-platform"; model = "Apple MacBook Pro (16-inch, M1 Pro, 2021)"; + + aliases { + sep = &sep; + }; +}; + +/* + * m1n1 keys off the sep alias to add the sepfw region and the boot + * manifests the driver needs to cold-boot SEPOS. + */ +&sep { + status = "okay"; +}; + +/* + * From the J316s platform device tree (/arm-io/spi2/mesa): the power line + * is gpio0 pin 196 (function-mesa_pwr, active high) and the data-ready line + * is pin 195. J414s's lines, 122 and 121, read the same way from its tree. + */ +&spi2 { + status = "okay"; +}; + +&mesa { + status = "okay"; + enable-gpios = <&pinctrl_ap 196 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <195 IRQ_TYPE_EDGE_RISING>; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; + /* + * Per-device factory data, extracted from this machine's own macOS + * install; it is not redistributable. + */ + firmware-name = "apple/mesacal-j316s.bin"; }; &wifi0 { From c52844817927a9d2283dc3988a7b1e22c000e559 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Tue, 29 Sep 2026 16:41:26 -0400 Subject: [PATCH 0911/2680] arm64: dts: apple: t602x: add the Mesa fingerprint sensor on spi2 The Touch ID sensor sits on spi2, chip-select 0, on the T602x die as on T600x, but t602x-die0.dtsi had no node for it, so no M2 Pro/Max board could describe the sensor and the SEP driver's SPI side never bound. Add it under spi2 with the same constants as the t600x node: 8 MHz, SPI mode 2 (spi-cpol), compatible apple,mesa-fingerprint. It ships disabled; boards enable it with their own power and data-ready lines. Signed-off-by: Scott Jones Co-Authored-By: Claude Opus 5.5 (1M context) --- arch/arm64/boot/dts/apple/t602x-die0.dtsi | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t602x-die0.dtsi b/arch/arm64/boot/dts/apple/t602x-die0.dtsi index f928a122a432b..09756fdb29236 100644 --- a/arch/arm64/boot/dts/apple/t602x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-die0.dtsi @@ -846,6 +846,14 @@ pinctrl-names = "default"; power-domains = <&ps_spi2>; status = "disabled"; + + mesa: fingerprint@0 { + compatible = "apple,mesa-fingerprint"; + reg = <0>; + spi-max-frequency = <8000000>; + spi-cpol; + status = "disabled"; + }; }; spi4: spi@39b110000 { From eb26d875bd4d9233c1ac2fd718db6ac95d2fbe69 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Tue, 29 Sep 2026 16:41:27 -0400 Subject: [PATCH 0912/2680] arm64: dts: apple: t6021-j414c: enable spi2 and the Mesa fingerprint sensor From the J414c platform device tree (/arm-io/spi2/mesa): the sensor is on spi2 chip-select 0 at 8 MHz, its power line is gpio0 pin 122 (function-mesa_pwr, active high) and its data-ready line is pin 121 on the same controller (phandle 163, gpio,t8101 at 0x39b028000, which is pinctrl_ap). These are the lines J414s uses. Describe them in the node so the driver takes the power GPIO and the data-ready interrupt from the device tree rather than the J414s-only fallback. Tested on a MacBook Pro 14" M2 Max (J414c), OS firmware 13.5, with the m1n1 SEP warm-preservation guard: the sensor binds at 8 MHz in mode 2 with interrupt-driven capture, fprintd enrolment completes and fprintd-verify reports verify-match. Signed-off-by: Scott Jones Co-Authored-By: Claude Opus 5.5 (1M context) --- arch/arm64/boot/dts/apple/t6021-j414c.dts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6021-j414c.dts b/arch/arm64/boot/dts/apple/t6021-j414c.dts index ef90ffe8423f4..2521633009343 100644 --- a/arch/arm64/boot/dts/apple/t6021-j414c.dts +++ b/arch/arm64/boot/dts/apple/t6021-j414c.dts @@ -46,3 +46,21 @@ &mtp_mt { firmware-name = "apple/tpmtfw-j414c.bin"; }; + +/* + * From the J414c platform device tree (/arm-io/spi2/mesa): the power line is + * gpio0 pin 122 (function-mesa_pwr, active high) and the data-ready line is + * pin 121, the same lines as J414s. + */ +&spi2 { + status = "okay"; +}; + +&mesa { + status = "okay"; + enable-gpios = <&pinctrl_ap 122 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <121 IRQ_TYPE_EDGE_RISING>; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; +}; From 799b4ad3c83f455bbe0a2650b8c133297528a743 Mon Sep 17 00:00:00 2001 From: Vincent Fontaine Date: Thu, 1 Oct 2026 19:06:49 -0700 Subject: [PATCH 0913/2680] arm64: dts: apple: t8112-j415: enable the SEP and the Mesa fingerprint sensor Add the Mesa sensor under spi2 in t8112.dtsi, and keep the SEP DART off at boot. iBoot hands over a running SEP, and apple_sep sets the DMA range and enables the DART itself, as on t602x. Probing it at boot leaves it configured without the range, and the driver then refuses to attach. Wire the sensor on J415 from its ADT: power on AP GPIO 178, active high, 8 MHz, chip select 0, SPI mode 1. The data-ready line (GPIO 177) is left out, so the driver polls. The sep alias lets m1n1 hand over the local-policy and iBoot manifests that the warm registration needs. Tested on J415 with the same nodes applied as an overlay to the running device tree (sensor bound in mode 1 with hardware CS timing, enrol and verify passed); these board-file nodes enable spi2 and the sensor at boot, as j414c does, instead of on module load. Co-Authored-By: Claude Opus 5.5 --- arch/arm64/boot/dts/apple/t8112-j415.dts | 19 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8112.dtsi | 13 +++++++++++++ 2 files changed, 32 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112-j415.dts b/arch/arm64/boot/dts/apple/t8112-j415.dts index 7dfd891fd4c11..2e2b982c584e0 100644 --- a/arch/arm64/boot/dts/apple/t8112-j415.dts +++ b/arch/arm64/boot/dts/apple/t8112-j415.dts @@ -21,6 +21,7 @@ aliases { bluetooth0 = &bluetooth0; keyboard = &keyboard; + sep = &sep; wifi0 = &wifi0; }; @@ -283,6 +284,24 @@ }; #include "hwmon-laptop.dtsi" +/* + * Mesa fingerprint sensor on spi2 (J415 ADT /arm-io/spi2/mesa): power on AP + * GPIO 178, active high. The data-ready line (GPIO 177) is left out, so the + * driver polls until its trigger is confirmed. Native chip select only. + */ +&spi2 { + status = "okay"; +}; + +&mesa { + status = "okay"; + enable-gpios = <&pinctrl_ap 178 GPIO_ACTIVE_HIGH>; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; + /* Per-device factory data from this Mac's own iBoot system container. */ + firmware-name = "apple/mesacal-j415.bin"; +}; + #include "isp-imx558-cfg0.dtsi" &isp { diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 29980b6e35b09..a6beaffd11941 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -898,6 +898,14 @@ #address-cells = <1>; #size-cells = <0>; status = "disabled"; + + mesa: fingerprint@0 { + compatible = "apple,mesa-fingerprint"; + reg = <0>; + spi-max-frequency = <8000000>; + spi-cpha; + status = "disabled"; + }; }; spi3: spi@23510c000 { @@ -1668,12 +1676,17 @@ }; + /* + * iBoot hands over a running SEP. Keep Linux off its DART at boot; + * apple_sep sets the DMA range and enables it, as on t602x. + */ sep_dart: iommu@25d2c0000 { compatible = "apple,t8112-dart", "apple,t8110-dart"; reg = <0x2 0x5d2c0000 0x0 0x4000>; #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; + status = "disabled"; }; sep: sep@25e400000 { From 79c5332302197a01d8ec2018a567c2f6a8dc55b8 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:27:34 -0500 Subject: [PATCH 0914/2680] arm64: dts: apple: t8103: Add the ANE, its DARTs and power states Describe the M1 Neural Engine: the engine window at 0x26bc04000, the three ANE DARTs, and the ane_base and ane_set1..5 power states (ane_sys_cpu -> ane_base -> ane_set1..5). apple-dart owns the DARTs and the engine maps through their IOMMU domain. The ane driver binds these nodes. They equal, property for property with phandles compared by target path, the live device tree of a MacBook Pro 13" M1 (j293) on which the out-of-tree build of the same driver code was bound on 2026-09-30 (linux-asahi 7.1.13): 80 properties over the ane node, its DARTs and the eight ANE power states. Enabled: Linux starts no ANE firmware on T8103. Until the ane module loads, apple-dart binds the three DARTs and apple-pmgr-pwrstate registers the new states, as on that machine. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t8103-pmgr.dtsi | 54 +++++++++++++++++++++++ arch/arm64/boot/dts/apple/t8103.dtsi | 46 +++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi index 4a399bc8f0276..252c6914d690d 100644 --- a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi @@ -1003,6 +1003,60 @@ power-domains = <&ps_ane_sys>; }; + ps_ane_base: power-controller@c008 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_base"; + power-domains = <&ps_ane_sys_cpu>; + }; + + ps_ane_set1: power-controller@c010 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_set1"; + power-domains = <&ps_ane_base>; + }; + + ps_ane_set2: power-controller@c018 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_set2"; + power-domains = <&ps_ane_base>; + }; + + ps_ane_set3: power-controller@c020 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_set3"; + power-domains = <&ps_ane_base>; + }; + + ps_ane_set4: power-controller@c028 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_set4"; + power-domains = <&ps_ane_base>; + }; + + ps_ane_set5: power-controller@c030 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_set5"; + power-domains = <&ps_ane_base>; + }; + ps_disp0_cpu0: power-controller@10018 { compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x10018 4>; diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index d6afbdb483015..a61ebe8f29026 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -1509,6 +1509,52 @@ power-domains = <&ps_avd_sys>; }; + ane_dart0: iommu@26b800000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x6b800000 0x0 0x4000>; + #iommu-cells = <1>; + apple,dma-range = <0x0 0x0 0x0 0xe0000000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + status = "okay"; + }; + + ane_dart1: iommu@26b810000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x6b810000 0x0 0x4000>; + #iommu-cells = <1>; + apple,dma-range = <0x0 0x0 0x0 0xe0000000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + status = "okay"; + }; + + ane_dart2: iommu@26b820000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x6b820000 0x0 0x4000>; + #iommu-cells = <1>; + apple,dma-range = <0x0 0x0 0x0 0xe0000000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + status = "okay"; + }; + + ane: ane@26bc04000 { + compatible = "apple,t8103-ane"; + reg = <0x2 0x6bc04000 0x0 0x24000>; + reg-names = "engine"; + interrupt-parent = <&aic>; + interrupts = ; + interrupt-names = "ane"; + iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; + power-domains = <&ps_ane_set1>, <&ps_ane_set2>, + <&ps_ane_set3>, <&ps_ane_set4>, <&ps_ane_set5>; + status = "okay"; + }; + dispext0_dart: iommu@271304000 { compatible = "apple,t8103-dart"; reg = <0x2 0x71304000 0x0 0x4000>; From eae60e537c164b27334391835bb0486ea00d28da Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:27:34 -0500 Subject: [PATCH 0915/2680] arm64: dts: apple: t600x: Add the die 0 ANE, its DARTs and power states Describe the die 0 Neural Engine of the M1 Pro, Max and Ultra: the engine window at 0x285c04000, the three ANE DARTs, and the ane_set0, ane_base and ane_set1..5 power states (ane_sys_cpu -> ane_set0 -> ane_base -> ane_set1..5). ane_set5 is described but the ane node does not use it: it lies past the end of the ADT ANE pmgr range (0xc02c). The states are added on die 0 only; the M1 Ultra die 1 ANE stays undescribed. The pmgr window already spans 0x1c000 here, so it covers 0xc000..0xc030 without change. These nodes equal the packaged omarchy-ane device-tree overlay that booted on a MacBook Pro 16" M1 Max (j316c) on 2026-09-30 (linux-asahi 7.1.13): the out-of-tree build of the ane driver bound at 285c04000.ane and an inference smoke test matched its reference output hash. Property for property, with phandles compared by target path, the subtree is identical. One name differs: the overlay calls the node ane@284000000 (the ADT engine block base), which dtc W=1 flags (simple_bus_reg) because reg starts at 0x285c04000. It is ane@285c04000 here. The driver matches on the compatible. Enabled: Linux starts no ANE firmware on T6000. Until the ane module loads, apple-dart binds the three DARTs and apple-pmgr-pwrstate registers the new states, as on that machine. The M1 Pro and M1 Ultra share this die 0 block; only the M1 Max has run it. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t600x-die0.dtsi | 44 +++++++++++++++ arch/arm64/boot/dts/apple/t600x-pmgr.dtsi | 69 +++++++++++++++++++++++ 2 files changed, 113 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index aeef6d7f8b97c..5e29bcb13a92e 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -7,6 +7,50 @@ */ &SOC_NODE { + + ane_dart0: iommu@285800000 { + compatible = "apple,t6000-dart"; + reg = <0x2 0x85800000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + status = "okay"; + }; + + ane_dart1: iommu@285810000 { + compatible = "apple,t6000-dart"; + reg = <0x2 0x85810000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys_cpu>; + status = "okay"; + }; + + ane_dart2: iommu@285820000 { + compatible = "apple,t6000-dart"; + reg = <0x2 0x85820000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys_cpu>; + status = "okay"; + }; + + ane: ane@285c04000 { + compatible = "apple,t6000-ane"; + reg = <0x2 0x85c04000 0x0 0x24000>; + reg-names = "engine"; + interrupt-parent = <&aic>; + interrupts = ; + interrupt-names = "ane"; + iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; + power-domains = <&ps_ane_sys_cpu>, <&ps_ane_set1>, + <&ps_ane_set2>, <&ps_ane_set3>, <&ps_ane_set4>; + status = "okay"; + }; + nco: clock-controller@28e03c000 { compatible = "apple,t6000-nco", "apple,nco"; reg = <0x2 0x8e03c000 0x0 0x14000>; diff --git a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi index 8b2c2cacce8c1..fed2d79a8ba8a 100644 --- a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi @@ -675,6 +675,75 @@ #reset-cells = <0>; label = DIE_LABEL(trace_fab); }; + +#if DIE_NO == 0 + /* + * Only the die 0 ANE is described. The ane node leaves set5 out: + * it lies past the end of the ADT ANE pmgr range (0xc02c). + */ + DIE_NODE(ps_ane_set0): power-controller@c000 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_set0); + power-domains = <&DIE_NODE(ps_ane_sys_cpu)>; + }; + + DIE_NODE(ps_ane_base): power-controller@c008 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_base); + power-domains = <&DIE_NODE(ps_ane_set0)>; + }; + + DIE_NODE(ps_ane_set1): power-controller@c010 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_set1); + power-domains = <&DIE_NODE(ps_ane_base)>; + }; + + DIE_NODE(ps_ane_set2): power-controller@c018 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_set2); + power-domains = <&DIE_NODE(ps_ane_base)>; + }; + + DIE_NODE(ps_ane_set3): power-controller@c020 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_set3); + power-domains = <&DIE_NODE(ps_ane_base)>; + }; + + DIE_NODE(ps_ane_set4): power-controller@c028 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_set4); + power-domains = <&DIE_NODE(ps_ane_base)>; + }; + + DIE_NODE(ps_ane_set5): power-controller@c030 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_set5); + power-domains = <&DIE_NODE(ps_ane_base)>; + }; +#endif }; &DIE_NODE(pmgr_east) { From 50f858b0ed94fcf4ed693fb2eff2eb243f6b8303 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:27:34 -0500 Subject: [PATCH 0916/2680] arm64: dts: apple: t602x: Add the die 0 ANE Describe the die 0 Neural Engine of the M2 Pro, Max and Ultra: the whole 32 MiB engine range at 0x284000000 plus the PMGR state words and the SET window, IRQ 884, three ANE DARTs, the eight existing ANE power states with ane_cpu as the reset, the ANE coprocessor mailbox at 0x285408000, and an IOVA-only reservation for the firmware entry alias (16 MiB at 0x10000000000 in the ANE DART domain). The shared die 0 file uses apple,t6021-ane. t6020.dtsi and t6022.dtsi change it to apple,t6020-ane and apple,t6022-ane, because ane_t6021 has per-SoC data, for example the SoC revision it gives the firmware. The Apple device tree gives the ANE one interrupt line, so the mailbox names only recv-not-empty, and apple-mailbox polls for TX space. ane_dart0 is in the always-on pmp domain, as in the tree that booted; ane_sys was never booted for this DART. The nodes are enabled. By default ane_t6021 runs the firmware from its own memory (fw_alias_reserved=0), so it needs no reserved-memory node from the bootloader, and it does not start the mailbox (hello_wait_ms=0). The memory-region is an IOVA reservation only. Tested with the out-of-tree build of the same driver code: a MacBook Pro 14" M2 Max (j414c) ran linux-asahi 7.1.13 with these node values from an overlay and the packaged m1n1 1.6.1, and its test programs gave bit-exact results. That tree had two differences: the mailbox also named a placeholder send-empty line (AIC 1833), which the stock apple-mailbox needs, and the nodes came from an overlay. Earlier runs on the same machine used a one-interrupt mailbox node with a poll-TX apple-mailbox. Not tested: this kernel tree was not booted, and no M2 Pro or M2 Ultra ran these nodes. Their values are from the macOS 13.5 device trees of those machines, which give the same die 0 ANE values as the M2 Max. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t6020.dtsi | 4 ++ arch/arm64/boot/dts/apple/t6022.dtsi | 4 ++ arch/arm64/boot/dts/apple/t602x-common.dtsi | 5 ++ arch/arm64/boot/dts/apple/t602x-die0.dtsi | 59 +++++++++++++++++++++ 4 files changed, 72 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6020.dtsi b/arch/arm64/boot/dts/apple/t6020.dtsi index 500726e697a89..e9af24b25ab68 100644 --- a/arch/arm64/boot/dts/apple/t6020.dtsi +++ b/arch/arm64/boot/dts/apple/t6020.dtsi @@ -26,6 +26,10 @@ /delete-node/ &pmp_report_afnc4_ioa; /delete-node/ &pmp_report_afnc5_ioa; +&ane { + compatible = "apple,t6020-ane"; +}; + &gpu { compatible = "apple,agx-t6020", "apple,agx-g14x", "apple,agx-g14s"; diff --git a/arch/arm64/boot/dts/apple/t6022.dtsi b/arch/arm64/boot/dts/apple/t6022.dtsi index 1c6a67ddf0ac9..4d0c94ce67d5b 100644 --- a/arch/arm64/boot/dts/apple/t6022.dtsi +++ b/arch/arm64/boot/dts/apple/t6022.dtsi @@ -326,6 +326,10 @@ #undef DIE_NO #undef SOC_NODE +&ane { + compatible = "apple,t6022-ane"; +}; + &aic { affinities { e-core-pmu-affinity { diff --git a/arch/arm64/boot/dts/apple/t602x-common.dtsi b/arch/arm64/boot/dts/apple/t602x-common.dtsi index e370a2e12473d..e7b88dfff6bce 100644 --- a/arch/arm64/boot/dts/apple/t602x-common.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-common.dtsi @@ -627,5 +627,10 @@ uat_ttbs: uat-ttbs@30 { reg = <0 0x30 0 0>; }; + + /* IOVA-only: the ANE firmware entry alias, 16 MiB at 0x10000000000 */ + ane_alias_iova: ane-alias-iova { + iommu-addresses = <&ane 0x100 0x0 0x0 0x1000000>; + }; }; }; diff --git a/arch/arm64/boot/dts/apple/t602x-die0.dtsi b/arch/arm64/boot/dts/apple/t602x-die0.dtsi index 09756fdb29236..7003fc128f070 100644 --- a/arch/arm64/boot/dts/apple/t602x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-die0.dtsi @@ -7,6 +7,65 @@ */ &SOC_NODE { + ane_mbox: mailbox@285408000 { + compatible = "apple,t6021-ane-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x85408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + interrupt-names = "recv-not-empty"; + #mbox-cells = <0>; + status = "okay"; + }; + + ane_dart0: iommu@285800000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x2 0x85800000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_pmp>; + status = "okay"; + }; + + ane_dart1: iommu@285810000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x2 0x85810000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_cpu>; + status = "okay"; + }; + + ane_dart2: iommu@285820000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x2 0x85820000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_cpu>; + status = "okay"; + }; + + ane: ane@284000000 { + compatible = "apple,t6021-ane"; + reg = <0x2 0x84000000 0x0 0x2000000>, + <0x2 0x8e080000 0x0 0x4034>, + <0x2 0x8e08c000 0x0 0x4000>; + reg-names = "engine", "pmgr", "set"; + interrupt-parent = <&aic>; + interrupts = ; + interrupt-names = "ane"; + iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; + power-domains = <&ps_ane_sys_mpm>, <&ps_ane_td>, + <&ps_ane_base>, <&ps_ane_set1>, <&ps_ane_set2>, + <&ps_ane_set3>, <&ps_ane_set4>, <&ps_ane_cpu>; + resets = <&ps_ane_cpu>; + mboxes = <&ane_mbox>; + memory-region = <&ane_alias_iova>; + status = "okay"; + }; + nco: clock-controller@28e03c000 { compatible = "apple,t6020-nco", "apple,t8103-nco", "apple,nco"; reg = <0x2 0x8e03c000 0x0 0x14000>; From c4a7247038f9971070b61b595c665446fa707664 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:28:59 -0500 Subject: [PATCH 0917/2680] arm64: dts: apple: t8112: Add the ANE, disabled Describe the M2 Neural Engine: the whole 32 MiB engine range at 0x26a000000 plus the PMGR window, the SET window and the two eFuse words that give the SoC revision to the firmware, IRQ 520, three ANE DARTs (IRQ 521, in ane_sys), the ANE coprocessor mailbox at 0x26b408000, and the seven ANE power states ane_sys_cpu, ane_td, ane_base and ane_set1..4 (0xc008..0xc038). ane_sys_cpu is a child of ane_sys, and each later state is a child of the one before. The values are those of the omarchy-ane T8112 overlay. Its sources are the /arm-io/ane and /arm-io/dart-ane nodes of the macOS 13.5 and 27.0 device trees, which are the same on j413, j415, j473 and j493, and the power-on order of the macOS 13.5 ANE kext. The mailbox sits at the same offset in the ANE block as on T6021 and names only recv-not-empty (the ANE interrupt); the overlay adds a placeholder send-empty line, which this tree does not need because apple-mailbox polls for TX. The mailbox, the DARTs and the ane node stay disabled: no T8112 has run the ane_t6021 driver or booted these nodes. The other ANE nodes in this series have the values of trees that ran the driver on an M1, an M1 Max or an M2 Max. A board or overlay that tests the driver enables the five nodes. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t8112-pmgr.dtsi | 63 +++++++++++++++++++++++ arch/arm64/boot/dts/apple/t8112.dtsi | 58 +++++++++++++++++++++ 2 files changed, 121 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi index 128a560dbac73..0e8df1cfa21f6 100644 --- a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi @@ -1132,6 +1132,69 @@ power-domains = <&ps_venc_pipe5>, <&ps_venc_pipe4>; }; + ps_ane_sys_cpu: power-controller@c008 { + compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_sys_cpu"; + power-domains = <&ps_ane_sys>; + }; + + ps_ane_td: power-controller@c010 { + compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_td"; + power-domains = <&ps_ane_sys_cpu>; + }; + + ps_ane_base: power-controller@c018 { + compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_base"; + power-domains = <&ps_ane_td>; + }; + + ps_ane_set1: power-controller@c020 { + compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_set1"; + power-domains = <&ps_ane_base>; + }; + + ps_ane_set2: power-controller@c028 { + compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_set2"; + power-domains = <&ps_ane_set1>; + }; + + ps_ane_set3: power-controller@c030 { + compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_set3"; + power-domains = <&ps_ane_set2>; + }; + + ps_ane_set4: power-controller@c038 { + compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc038 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_set4"; + power-domains = <&ps_ane_set3>; + }; + ps_disp0_cpu0: power-controller@10000 { compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x10000 4>; diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index a6beaffd11941..88328b73555e9 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -1739,6 +1739,64 @@ power-domains = <&ps_avd_sys>; }; + ane_mbox: mailbox@26b408000 { + compatible = "apple,t8112-ane-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x6b408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + interrupt-names = "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + ane_dart0: iommu@26b800000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x6b800000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + + ane_dart1: iommu@26b810000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x6b810000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + + ane_dart2: iommu@26b820000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x6b820000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + + ane: ane@26a000000 { + compatible = "apple,t8112-ane"; + reg = <0x2 0x6a000000 0x0 0x2000000>, + <0x2 0x3b700000 0x0 0x18000>, + <0x2 0x3b724000 0x0 0x4000>, + <0x2 0x3d2c8060 0x0 0x8>; + reg-names = "engine", "pmgr", "set", "fuse"; + interrupt-parent = <&aic>; + interrupts = ; + interrupt-names = "ane"; + iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; + power-domains = <&ps_ane_sys_cpu>, <&ps_ane_td>, + <&ps_ane_base>, <&ps_ane_set1>, <&ps_ane_set2>, + <&ps_ane_set3>, <&ps_ane_set4>; + mboxes = <&ane_mbox>; + status = "disabled"; + }; + dispext0_dart: iommu@271304000 { compatible = "apple,t8112-dart", "apple,t8110-dart"; reg = <0x2 0x71304000 0x0 0x4000>; From d800e25e7d196083527c488ffab73e42c40ee945 Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 14:15:46 -0700 Subject: [PATCH 0918/2680] arm64: dts: apple: t600x: drop the ANE set5 power state The die 0 ANE power states that came with the ANE support in Joshua Warren's aurora-silicon/linux#155 include ps_ane_set5 at 0xc030, but the comment above them puts the end of the ANE's pmgr range at 0xc02c. The ane node does not use it either: it lists set1 to set4, and the driver takes its power domains from that list. The node was still live on every M1 Pro, Max and Ultra. The pmgr driver probes each enabled power-controller: it reads the register, sets the auto-PM enable bit if the state is active, and genpd turns unused domains off late in boot. Linux should not do any of that to a register outside the block it describes. Drop the node, and say in the comment why there is no set5. Found in a review of #155. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit 7280b1168af21b04adde3a38ae8fa07a1ab54f84) --- arch/arm64/boot/dts/apple/t600x-pmgr.dtsi | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi index fed2d79a8ba8a..a1c8748ee7cf9 100644 --- a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi @@ -678,8 +678,8 @@ #if DIE_NO == 0 /* - * Only the die 0 ANE is described. The ane node leaves set5 out: - * it lies past the end of the ADT ANE pmgr range (0xc02c). + * Only the die 0 ANE is described. Its pmgr range ends at 0xc02c, + * with set4, so no set5 is described at 0xc030. */ DIE_NODE(ps_ane_set0): power-controller@c000 { compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; @@ -734,15 +734,6 @@ label = DIE_LABEL(ane_set4); power-domains = <&DIE_NODE(ps_ane_base)>; }; - - DIE_NODE(ps_ane_set5): power-controller@c030 { - compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; - reg = <0xc030 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = DIE_LABEL(ane_set5); - power-domains = <&DIE_NODE(ps_ane_base)>; - }; #endif }; From 705924bf01abd7dbe34d1df1a2176fdb0992f8e7 Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 14:24:57 -0700 Subject: [PATCH 0919/2680] arm64: dts: apple: enable the ANE only on the M1 Max and M2 Max The ANE support from Joshua Warren's aurora-silicon/linux#155 enables the ANE on every chip that describes it except the M2 (T8112): the M1 (T8103) directly, and every M1 Pro/Max/Ultra (T6000/T6001/T6002) and M2 Pro/Max/Ultra (T6020/T6021/T6022) through the shared die 0 files. The driver has run on the M1 Max (T6001) and the M2 Max (T6021) only, and these device tree nodes have not booted on any machine yet. On an untested chip that means the ANE's DARTs, mailbox and power states probe and touch the hardware on every boot, and the driver binds, which on the M2 family starts the ANE firmware by default. Make the ANE node, its DARTs and mailbox, and the power states added for it default to disabled where they are described, and enable them per SoC: - t6001.dtsi enables them on the M1 Max: MacBook Pro j314c and j316c, and Mac Studio j375c. - t6021.dtsi enables them on the M2 Max: MacBook Pro j414c and j416c, and Mac Studio j475c. - t6000.dtsi and t6020.dtsi include those files, so they define APPLE_ANE_UNTESTED first and the M1 Pro and M2 Pro stay off. - t6002 and t6022 (Ultra) build die 0 from the shared files and never see an enable, so they stay off. - t8103.dtsi gets the same enable block behind a one-line APPLE_ANE_UNTESTED define, so enabling the M1 ANE after a supervised boot is a one-line deletion. Power states that predate #155 (ane_sys and ane_sys_cpu on t600x and t8103, and the ANE states in t602x-pmgr.dtsi) keep their status. Where the ANE stays off, each device tree is the pre-#155 one plus disabled nodes, apart from the inert ane-alias-iova reserved-memory entry on t602x, which has no reg or size and is only read through the ANE's memory-region. On the M1 Max and M2 Max the device tree is unchanged, except that the t600x ANE power states now carry an explicit status = "okay". Found in a review of #155. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit ecf86ec16ece1313dd16e135779bf7d93469f16f) --- arch/arm64/boot/dts/apple/t6000.dtsi | 3 ++ arch/arm64/boot/dts/apple/t6001.dtsi | 46 ++++++++++++++++++ arch/arm64/boot/dts/apple/t600x-die0.dtsi | 8 ++-- arch/arm64/boot/dts/apple/t600x-pmgr.dtsi | 10 +++- arch/arm64/boot/dts/apple/t6020.dtsi | 3 ++ arch/arm64/boot/dts/apple/t6021.dtsi | 27 +++++++++++ arch/arm64/boot/dts/apple/t602x-die0.dtsi | 10 ++-- arch/arm64/boot/dts/apple/t8103-pmgr.dtsi | 6 +++ arch/arm64/boot/dts/apple/t8103.dtsi | 57 +++++++++++++++++++++-- 9 files changed, 156 insertions(+), 14 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6000.dtsi b/arch/arm64/boot/dts/apple/t6000.dtsi index db7ea6eac5a6f..22cb23931e0b3 100644 --- a/arch/arm64/boot/dts/apple/t6000.dtsi +++ b/arch/arm64/boot/dts/apple/t6000.dtsi @@ -11,6 +11,9 @@ #define GPU_REPEAT(x) +/* The ANE has only run on the M1 Max so far: leave it off on the M1 Pro */ +#define APPLE_ANE_UNTESTED + #include "t6001.dtsi" / { diff --git a/arch/arm64/boot/dts/apple/t6001.dtsi b/arch/arm64/boot/dts/apple/t6001.dtsi index 33d8a06d75122..cf84f107a638e 100644 --- a/arch/arm64/boot/dts/apple/t6001.dtsi +++ b/arch/arm64/boot/dts/apple/t6001.dtsi @@ -93,3 +93,49 @@ }; #include "hwmon-common.dtsi" +/* + * The ANE has run on the M1 Max, so enable it here with its DARTs and the + * power states only it uses. t6000.dtsi defines APPLE_ANE_UNTESTED before + * including this file: the ANE has not run on the M1 Pro yet. + */ +#ifndef APPLE_ANE_UNTESTED +&ane { + status = "okay"; +}; + +&ane_dart0 { + status = "okay"; +}; + +&ane_dart1 { + status = "okay"; +}; + +&ane_dart2 { + status = "okay"; +}; + +&ps_ane_base { + status = "okay"; +}; + +&ps_ane_set0 { + status = "okay"; +}; + +&ps_ane_set1 { + status = "okay"; +}; + +&ps_ane_set2 { + status = "okay"; +}; + +&ps_ane_set3 { + status = "okay"; +}; + +&ps_ane_set4 { + status = "okay"; +}; +#endif diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index 5e29bcb13a92e..2cbe84a3d5625 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -15,7 +15,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; - status = "okay"; + status = "disabled"; }; ane_dart1: iommu@285810000 { @@ -25,7 +25,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys_cpu>; - status = "okay"; + status = "disabled"; }; ane_dart2: iommu@285820000 { @@ -35,7 +35,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys_cpu>; - status = "okay"; + status = "disabled"; }; ane: ane@285c04000 { @@ -48,7 +48,7 @@ iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; power-domains = <&ps_ane_sys_cpu>, <&ps_ane_set1>, <&ps_ane_set2>, <&ps_ane_set3>, <&ps_ane_set4>; - status = "okay"; + status = "disabled"; }; nco: clock-controller@28e03c000 { diff --git a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi index a1c8748ee7cf9..77469a62242a6 100644 --- a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi @@ -679,7 +679,9 @@ #if DIE_NO == 0 /* * Only the die 0 ANE is described. Its pmgr range ends at 0xc02c, - * with set4, so no set5 is described at 0xc030. + * with set4, so no set5 is described at 0xc030. These states serve + * only the ANE and are disabled with it; the SoC file that enables + * the ANE enables them too. */ DIE_NODE(ps_ane_set0): power-controller@c000 { compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; @@ -688,6 +690,7 @@ #reset-cells = <0>; label = DIE_LABEL(ane_set0); power-domains = <&DIE_NODE(ps_ane_sys_cpu)>; + status = "disabled"; }; DIE_NODE(ps_ane_base): power-controller@c008 { @@ -697,6 +700,7 @@ #reset-cells = <0>; label = DIE_LABEL(ane_base); power-domains = <&DIE_NODE(ps_ane_set0)>; + status = "disabled"; }; DIE_NODE(ps_ane_set1): power-controller@c010 { @@ -706,6 +710,7 @@ #reset-cells = <0>; label = DIE_LABEL(ane_set1); power-domains = <&DIE_NODE(ps_ane_base)>; + status = "disabled"; }; DIE_NODE(ps_ane_set2): power-controller@c018 { @@ -715,6 +720,7 @@ #reset-cells = <0>; label = DIE_LABEL(ane_set2); power-domains = <&DIE_NODE(ps_ane_base)>; + status = "disabled"; }; DIE_NODE(ps_ane_set3): power-controller@c020 { @@ -724,6 +730,7 @@ #reset-cells = <0>; label = DIE_LABEL(ane_set3); power-domains = <&DIE_NODE(ps_ane_base)>; + status = "disabled"; }; DIE_NODE(ps_ane_set4): power-controller@c028 { @@ -733,6 +740,7 @@ #reset-cells = <0>; label = DIE_LABEL(ane_set4); power-domains = <&DIE_NODE(ps_ane_base)>; + status = "disabled"; }; #endif }; diff --git a/arch/arm64/boot/dts/apple/t6020.dtsi b/arch/arm64/boot/dts/apple/t6020.dtsi index e9af24b25ab68..0fa3630e2849d 100644 --- a/arch/arm64/boot/dts/apple/t6020.dtsi +++ b/arch/arm64/boot/dts/apple/t6020.dtsi @@ -11,6 +11,9 @@ #define GPU_REPEAT(x) +/* The ANE has only run on the M2 Max so far: leave it off on the M2 Pro */ +#define APPLE_ANE_UNTESTED + #include "t6021.dtsi" / { diff --git a/arch/arm64/boot/dts/apple/t6021.dtsi b/arch/arm64/boot/dts/apple/t6021.dtsi index a541239222475..29190a96b8252 100644 --- a/arch/arm64/boot/dts/apple/t6021.dtsi +++ b/arch/arm64/boot/dts/apple/t6021.dtsi @@ -85,3 +85,30 @@ }; #include "hwmon-common.dtsi" + +/* + * The ANE has run on the M2 Max, so enable it here with its mailbox and + * DARTs. t6020.dtsi defines APPLE_ANE_UNTESTED before including this + * file: the ANE has not run on the M2 Pro yet. + */ +#ifndef APPLE_ANE_UNTESTED +&ane { + status = "okay"; +}; + +&ane_dart0 { + status = "okay"; +}; + +&ane_dart1 { + status = "okay"; +}; + +&ane_dart2 { + status = "okay"; +}; + +&ane_mbox { + status = "okay"; +}; +#endif diff --git a/arch/arm64/boot/dts/apple/t602x-die0.dtsi b/arch/arm64/boot/dts/apple/t602x-die0.dtsi index 7003fc128f070..6001004903632 100644 --- a/arch/arm64/boot/dts/apple/t602x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-die0.dtsi @@ -14,7 +14,7 @@ interrupts = ; interrupt-names = "recv-not-empty"; #mbox-cells = <0>; - status = "okay"; + status = "disabled"; }; ane_dart0: iommu@285800000 { @@ -24,7 +24,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_pmp>; - status = "okay"; + status = "disabled"; }; ane_dart1: iommu@285810000 { @@ -34,7 +34,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_cpu>; - status = "okay"; + status = "disabled"; }; ane_dart2: iommu@285820000 { @@ -44,7 +44,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_cpu>; - status = "okay"; + status = "disabled"; }; ane: ane@284000000 { @@ -63,7 +63,7 @@ resets = <&ps_ane_cpu>; mboxes = <&ane_mbox>; memory-region = <&ane_alias_iova>; - status = "okay"; + status = "disabled"; }; nco: clock-controller@28e03c000 { diff --git a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi index 252c6914d690d..c3230cd4bb81a 100644 --- a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi @@ -1010,6 +1010,7 @@ #reset-cells = <0>; label = "ane_base"; power-domains = <&ps_ane_sys_cpu>; + status = "disabled"; }; ps_ane_set1: power-controller@c010 { @@ -1019,6 +1020,7 @@ #reset-cells = <0>; label = "ane_set1"; power-domains = <&ps_ane_base>; + status = "disabled"; }; ps_ane_set2: power-controller@c018 { @@ -1028,6 +1030,7 @@ #reset-cells = <0>; label = "ane_set2"; power-domains = <&ps_ane_base>; + status = "disabled"; }; ps_ane_set3: power-controller@c020 { @@ -1037,6 +1040,7 @@ #reset-cells = <0>; label = "ane_set3"; power-domains = <&ps_ane_base>; + status = "disabled"; }; ps_ane_set4: power-controller@c028 { @@ -1046,6 +1050,7 @@ #reset-cells = <0>; label = "ane_set4"; power-domains = <&ps_ane_base>; + status = "disabled"; }; ps_ane_set5: power-controller@c030 { @@ -1055,6 +1060,7 @@ #reset-cells = <0>; label = "ane_set5"; power-domains = <&ps_ane_base>; + status = "disabled"; }; ps_disp0_cpu0: power-controller@10018 { diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index a61ebe8f29026..343ab91374398 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -1517,7 +1517,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; - status = "okay"; + status = "disabled"; }; ane_dart1: iommu@26b810000 { @@ -1528,7 +1528,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; - status = "okay"; + status = "disabled"; }; ane_dart2: iommu@26b820000 { @@ -1539,7 +1539,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; - status = "okay"; + status = "disabled"; }; ane: ane@26bc04000 { @@ -1552,7 +1552,7 @@ iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; power-domains = <&ps_ane_set1>, <&ps_ane_set2>, <&ps_ane_set3>, <&ps_ane_set4>, <&ps_ane_set5>; - status = "okay"; + status = "disabled"; }; dispext0_dart: iommu@271304000 { @@ -2250,3 +2250,52 @@ #include "t8103-pmgr.dtsi" #include "hwmon-common.dtsi" + +/* + * The ANE has not run on the M1 yet, so it stays off together with its + * DARTs and the power states only it uses. Drop this define to enable + * them once it has. + */ +#define APPLE_ANE_UNTESTED + +#ifndef APPLE_ANE_UNTESTED +&ane { + status = "okay"; +}; + +&ane_dart0 { + status = "okay"; +}; + +&ane_dart1 { + status = "okay"; +}; + +&ane_dart2 { + status = "okay"; +}; + +&ps_ane_base { + status = "okay"; +}; + +&ps_ane_set1 { + status = "okay"; +}; + +&ps_ane_set2 { + status = "okay"; +}; + +&ps_ane_set3 { + status = "okay"; +}; + +&ps_ane_set4 { + status = "okay"; +}; + +&ps_ane_set5 { + status = "okay"; +}; +#endif From 650bbf912f139e9402d7da3b39c51ca902dcb2f5 Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 14:24:57 -0700 Subject: [PATCH 0920/2680] arm64: dts: apple: t8112: disable the ANE power states with the ANE Joshua Warren's aurora-silicon/linux#155 adds the M2 (T8112) ANE with its node, mailbox and DARTs disabled, since it has not run there. The seven power states added for it (ane_sys_cpu, ane_td, ane_base and ane_set1 to ane_set4) have no status, though, so they are live on every boot of an M2 MacBook Air, 13" MacBook Pro or Mac mini. The pmgr driver probes each of them: it reads the register, sets the auto-PM enable bit if the state is active, and genpd turns unused domains off late in boot. Nothing consumes them while the ANE is disabled. Disable them as well, so the ANE and everything only it uses are off together, and say so above them. ane_sys predates #155 and keeps its status. Found in a review of #155. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit 72c7bd0c7a03d768daf7774e9481d9024825434c) --- arch/arm64/boot/dts/apple/t8112-pmgr.dtsi | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi index 0e8df1cfa21f6..26ab30e219f12 100644 --- a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi @@ -1132,6 +1132,10 @@ power-domains = <&ps_venc_pipe5>, <&ps_venc_pipe4>; }; + /* + * These states serve only the ANE. They stay disabled with the ane + * node in t8112.dtsi; enable them together with it. + */ ps_ane_sys_cpu: power-controller@c008 { compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xc008 4>; @@ -1139,6 +1143,7 @@ #reset-cells = <0>; label = "ane_sys_cpu"; power-domains = <&ps_ane_sys>; + status = "disabled"; }; ps_ane_td: power-controller@c010 { @@ -1148,6 +1153,7 @@ #reset-cells = <0>; label = "ane_td"; power-domains = <&ps_ane_sys_cpu>; + status = "disabled"; }; ps_ane_base: power-controller@c018 { @@ -1157,6 +1163,7 @@ #reset-cells = <0>; label = "ane_base"; power-domains = <&ps_ane_td>; + status = "disabled"; }; ps_ane_set1: power-controller@c020 { @@ -1166,6 +1173,7 @@ #reset-cells = <0>; label = "ane_set1"; power-domains = <&ps_ane_base>; + status = "disabled"; }; ps_ane_set2: power-controller@c028 { @@ -1175,6 +1183,7 @@ #reset-cells = <0>; label = "ane_set2"; power-domains = <&ps_ane_set1>; + status = "disabled"; }; ps_ane_set3: power-controller@c030 { @@ -1184,6 +1193,7 @@ #reset-cells = <0>; label = "ane_set3"; power-domains = <&ps_ane_set2>; + status = "disabled"; }; ps_ane_set4: power-controller@c038 { @@ -1193,6 +1203,7 @@ #reset-cells = <0>; label = "ane_set4"; power-domains = <&ps_ane_set3>; + status = "disabled"; }; ps_disp0_cpu0: power-controller@10000 { From e344d13037cb75eaa957d1363cbf18abb865b690 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:59:19 -0500 Subject: [PATCH 0921/2680] arm64: dts: apple: enable the ANE on t8103, t6000 and t6020 The ANE node, its DARTs, its mailbox and the power states only it uses are disabled on the M1 (T8103), the M1 Pro (T6000) and the M2 Pro (T6020) behind APPLE_ANE_UNTESTED, because the driver had run only on the M1 Max and the M2 Max. It has now run with this tree's own driver on all three. Each chip has a passing row in the public community dataset (promotion_check --remote: judged, driver_source=intree, 20 smoke calls that match the golden bit for bit, no ANE, DART or mailbox fault): M1 (T8103) row 3559e0f328ba M1 Pro (T6000) row 6eb94f49985b M2 Pro (T6020) row 3c9389040f51 Remove the define from t6000.dtsi, t6020.dtsi and t8103.dtsi. No file defines it after this, so also remove the #ifndef APPLE_ANE_UNTESTED guards in t6001.dtsi, t6021.dtsi and t8103.dtsi, and correct the comments that said the ANE had not run on the M1 Pro and the M2 Pro. The ANE is now enabled on the t8103 boards (j274, j293, j313, j456, j457), the t6000 boards (j314s, j316s) and the t6020 boards (j414s, j416s, j474s). The DTBs of the t6001, t6002, t6021, t6022 and t8112 boards do not change: t6002.dtsi and t6022.dtsi include neither t6001.dtsi nor t6021.dtsi, so the ANE stays disabled on the M1 Ultra (T6002), the M2 Ultra (T6022) and the M2 (T8112). Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t6000.dtsi | 3 --- arch/arm64/boot/dts/apple/t6001.dtsi | 7 ++----- arch/arm64/boot/dts/apple/t6020.dtsi | 3 --- arch/arm64/boot/dts/apple/t6021.dtsi | 7 ++----- arch/arm64/boot/dts/apple/t8103.dtsi | 9 ++------- 5 files changed, 6 insertions(+), 23 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6000.dtsi b/arch/arm64/boot/dts/apple/t6000.dtsi index 22cb23931e0b3..db7ea6eac5a6f 100644 --- a/arch/arm64/boot/dts/apple/t6000.dtsi +++ b/arch/arm64/boot/dts/apple/t6000.dtsi @@ -11,9 +11,6 @@ #define GPU_REPEAT(x) -/* The ANE has only run on the M1 Max so far: leave it off on the M1 Pro */ -#define APPLE_ANE_UNTESTED - #include "t6001.dtsi" / { diff --git a/arch/arm64/boot/dts/apple/t6001.dtsi b/arch/arm64/boot/dts/apple/t6001.dtsi index cf84f107a638e..5da33aa0db9df 100644 --- a/arch/arm64/boot/dts/apple/t6001.dtsi +++ b/arch/arm64/boot/dts/apple/t6001.dtsi @@ -94,11 +94,9 @@ #include "hwmon-common.dtsi" /* - * The ANE has run on the M1 Max, so enable it here with its DARTs and the - * power states only it uses. t6000.dtsi defines APPLE_ANE_UNTESTED before - * including this file: the ANE has not run on the M1 Pro yet. + * The ANE has run on the M1 Pro and the M1 Max, so enable it here with its + * DARTs and the power states only it uses. t6000.dtsi includes this file. */ -#ifndef APPLE_ANE_UNTESTED &ane { status = "okay"; }; @@ -138,4 +136,3 @@ &ps_ane_set4 { status = "okay"; }; -#endif diff --git a/arch/arm64/boot/dts/apple/t6020.dtsi b/arch/arm64/boot/dts/apple/t6020.dtsi index 0fa3630e2849d..e9af24b25ab68 100644 --- a/arch/arm64/boot/dts/apple/t6020.dtsi +++ b/arch/arm64/boot/dts/apple/t6020.dtsi @@ -11,9 +11,6 @@ #define GPU_REPEAT(x) -/* The ANE has only run on the M2 Max so far: leave it off on the M2 Pro */ -#define APPLE_ANE_UNTESTED - #include "t6021.dtsi" / { diff --git a/arch/arm64/boot/dts/apple/t6021.dtsi b/arch/arm64/boot/dts/apple/t6021.dtsi index 29190a96b8252..bb1436a1f2dd4 100644 --- a/arch/arm64/boot/dts/apple/t6021.dtsi +++ b/arch/arm64/boot/dts/apple/t6021.dtsi @@ -87,11 +87,9 @@ #include "hwmon-common.dtsi" /* - * The ANE has run on the M2 Max, so enable it here with its mailbox and - * DARTs. t6020.dtsi defines APPLE_ANE_UNTESTED before including this - * file: the ANE has not run on the M2 Pro yet. + * The ANE has run on the M2 Pro and the M2 Max, so enable it here with its + * mailbox and DARTs. t6020.dtsi includes this file. */ -#ifndef APPLE_ANE_UNTESTED &ane { status = "okay"; }; @@ -111,4 +109,3 @@ &ane_mbox { status = "okay"; }; -#endif diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 343ab91374398..6a9f4f00f1aae 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -2252,13 +2252,9 @@ #include "hwmon-common.dtsi" /* - * The ANE has not run on the M1 yet, so it stays off together with its - * DARTs and the power states only it uses. Drop this define to enable - * them once it has. + * The ANE has run on the M1, so enable it here with its DARTs and the + * power states only it uses. */ -#define APPLE_ANE_UNTESTED - -#ifndef APPLE_ANE_UNTESTED &ane { status = "okay"; }; @@ -2298,4 +2294,3 @@ &ps_ane_set5 { status = "okay"; }; -#endif From b150943a3dff9d61ff52889b260c75d276efeacc Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Tue, 29 Sep 2026 21:47:40 -0600 Subject: [PATCH 0922/2680] arm64: dts: apple: t6000-j314s: enable the SEP and the Mesa sensor Mirror t6000-j316s. Add the sep alias and enable the SEP node, so m1n1's dt_set_sep() adds the sepfw region and the boot manifests the driver needs to cold-boot SEPOS. Enable SPI2 and the Mesa sensor. The J314s platform device tree describes /arm-io/spi2/mesa like the J316s one: the power line is gpio0 pin 196 (function-mesa_pwr, active high), and the data-ready interrupt is pin 195. Tested on J314s (system firmware 26.3, m1n1 1.6.1). The compiled SEP, spi2 and mesa nodes are the same as in the DTB used for testing. SEPOS cold-boots and advertises all 12 endpoints, the sensor powers up and captures, and with the previous two commits Touch ID enrols and matches across reboots. [Chris Kearney: reworded where the pin values were read from] Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Christian Bartels Co-authored-by: Claude Opus 5.5 --- arch/arm64/boot/dts/apple/t6000-j314s.dts | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6000-j314s.dts b/arch/arm64/boot/dts/apple/t6000-j314s.dts index afa86668440f0..7bd2e4a0be65a 100644 --- a/arch/arm64/boot/dts/apple/t6000-j314s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j314s.dts @@ -15,6 +15,41 @@ / { compatible = "apple,j314s", "apple,t6000", "apple,arm-platform"; model = "Apple MacBook Pro (14-inch, M1 Pro, 2021)"; + + aliases { + sep = &sep; + }; +}; + +/* + * m1n1 keys off the sep alias to add the sepfw region and the boot + * manifests the driver needs to cold-boot SEPOS. + */ +&sep { + status = "okay"; +}; + +/* + * From the J314s platform device tree (/arm-io/spi2/mesa): the power line + * is gpio0 pin 196 (function-mesa_pwr, active high) and the data-ready line + * is pin 195, the same lines as J316s. + */ +&spi2 { + status = "okay"; +}; + +&mesa { + status = "okay"; + enable-gpios = <&pinctrl_ap 196 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <195 IRQ_TYPE_EDGE_RISING>; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; + /* + * Per-device factory data, extracted from this machine's own macOS + * install; it is not redistributable. + */ + firmware-name = "apple/mesacal-j314s.bin"; }; &wifi0 { From c9cbed03823d12f12fcb9d2b7cede3cc814e9877 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 30 Sep 2026 12:14:57 -0700 Subject: [PATCH 0923/2680] arm64: dts: apple: t8103-j293: run the Mesa sensor in SPI mode 2 t8103.dtsi describes the Mesa fingerprint sensor with spi-cpha alone, SPI mode 1. Since the driver started taking the SPI mode from the device tree, the j293 (MacBook Pro 13-inch, M1) sensor binds in mode 1 and nothing downstream works: sep sensor: bound, 8000000 Hz, mode 1 (CPOL=0 CPHA=1) sbio: identity count 0 after sensor serial registration matching unavailable this boot: ... template state is unproven In mode 2 (CPOL=1, CPHA=0) the device view populates and enrol and match work. Before the driver read the mode from the device tree, its old default of mode 2 hid this. Override the mode on j293 only. A J313 on the same dtsi has been reported working in mode 1, so t8103.dtsi is left alone. Reported and tested on a j293 by GitHub user 0800tim, with the same override applied to the booted DTB. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- arch/arm64/boot/dts/apple/t8103-j293.dts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103-j293.dts b/arch/arm64/boot/dts/apple/t8103-j293.dts index bf6d9f00bdc66..bc42ad5a57043 100644 --- a/arch/arm64/boot/dts/apple/t8103-j293.dts +++ b/arch/arm64/boot/dts/apple/t8103-j293.dts @@ -110,6 +110,13 @@ &mesa { status = "okay"; + /* + * This board's sensor answers in SPI mode 2 (CPOL=1, CPHA=0), not the + * mode 1 t8103.dtsi describes: in mode 1 it binds, but the enclave then + * lists no identities and matching stays off. + */ + /delete-property/ spi-cpha; + spi-cpol; enable-gpios = <&pinctrl_ap 108 GPIO_ACTIVE_HIGH>; spi-cs-setup-delay-ns = <20>; spi-cs-hold-delay-ns = <20>; From e0cee31045c478afac877659862caeb379a40f32 Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 09:52:18 -0700 Subject: [PATCH 0924/2680] arm64: dts: apple: t602x-j414-j416: enable the Mesa fingerprint sensor Only J414c described the Mesa fingerprint sensor's power and data-ready lines. J414s got them from a fallback in the SEP sensor shim that only applies to apple,j414s, and J416s and J416c had none: the SEP driver enables the sensor node at runtime, but with no power line the sensor never came up. The four M2 Pro/Max MacBook Pros share t602x-j414-j416.dtsi, which already describes the rest of their common wiring (speakers, headphone jack, USB-C PD controllers, MTP keyboard and trackpad). Move the spi2 and Mesa block there from t6021-j414c.dts, unchanged: power on AP GPIO 122 (active high), data-ready on AP GPIO 121 (rising edge), 20 ns chip-select setup and hold. These are the same lines on every board where they have been checked: J414c and J416c from their platform device trees, and J414s, whose fallback uses them. J416s is expected to match but has not been checked. Testing: J414c works end to end (fprintd enrolment and verify-match), and its DTB is byte-identical after this change. J414s has run on these lines through the fallback and now takes the device-node path instead. J416s and J416c have not been tested yet. Like J414c, the node names no firmware-name, so the driver loads its default calibration file. The J414c node is from Scott Jones' commit ee83202ee1e0 ("arm64: dts: apple: t6021-j414c: enable spi2 and the Mesa fingerprint sensor"). RJ Skerry-Ryan confirmed the J416c lines on his machine (rryan/linux 7a29ac2520ba, "arm64: dts: apple: t6021-j416c: describe the Mesa fingerprint sensor"). Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- arch/arm64/boot/dts/apple/t6021-j414c.dts | 18 ------------------ .../arm64/boot/dts/apple/t602x-j414-j416.dtsi | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6021-j414c.dts b/arch/arm64/boot/dts/apple/t6021-j414c.dts index 2521633009343..ef90ffe8423f4 100644 --- a/arch/arm64/boot/dts/apple/t6021-j414c.dts +++ b/arch/arm64/boot/dts/apple/t6021-j414c.dts @@ -46,21 +46,3 @@ &mtp_mt { firmware-name = "apple/tpmtfw-j414c.bin"; }; - -/* - * From the J414c platform device tree (/arm-io/spi2/mesa): the power line is - * gpio0 pin 122 (function-mesa_pwr, active high) and the data-ready line is - * pin 121, the same lines as J414s. - */ -&spi2 { - status = "okay"; -}; - -&mesa { - status = "okay"; - enable-gpios = <&pinctrl_ap 122 GPIO_ACTIVE_HIGH>; - interrupt-parent = <&pinctrl_ap>; - interrupts = <121 IRQ_TYPE_EDGE_RISING>; - spi-cs-setup-delay-ns = <20>; - spi-cs-hold-delay-ns = <20>; -}; diff --git a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi index 7affa3cb0aec4..38bd140754d7e 100644 --- a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi @@ -521,6 +521,25 @@ }; }; +/* + * Mesa fingerprint sensor. The power line is gpio0 pin 122 (function-mesa_pwr, + * active high) and the data-ready line is pin 121 on J414c and J416c (from + * their platform device trees, /arm-io/spi2/mesa) and on J414s. J416s shares + * the rest of this family's AP GPIO assignments and is expected to match. + */ +&spi2 { + status = "okay"; +}; + +&mesa { + status = "okay"; + enable-gpios = <&pinctrl_ap 122 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <121 IRQ_TYPE_EDGE_RISING>; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; +}; + &isp { apple,platform-id = <7>; /delete-node/ sensor-presets; /* Override j31[46] below */ From ef6046e648cc9656bed36c7054897bcca8c5445f Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 12:19:38 -0700 Subject: [PATCH 0925/2680] arm64: dts: apple: t6001-j314c/j316c: enable the SEP and the Mesa sensor Mirror t6000-j314s and t6000-j316s. Add the sep alias and enable the SEP node, so m1n1's dt_set_sep() adds the sepfw region and the local-policy and iBoot manifests the driver needs to cold-boot SEPOS. Enable SPI2 and the Mesa sensor with the M1 Pro lines: power on gpio0 pin 196 (function-mesa_pwr, active high) and the data-ready interrupt on pin 195, with 20 ns chip-select setup and hold. Brent Kearney read these from the J316s platform device tree in commit 4236dafcfd7f ("arm64: dts: apple: t6000-j316s: enable the SEP and the Mesa sensor"), and Christian Bartels found the J314s tree describes the sensor the same way, in commit b0070be282e8 ("arm64: dts: apple: t6000-j314s: enable the SEP and the Mesa sensor"). The M1 Max laptops share that family's board file, t600x-j314-j316.dtsi, and their spi2 controller, pins and power domain are the same nodes as on the M1 Pro. Each board names its own calibration file, apple/mesacal-j314c.bin and apple/mesacal-j316c.bin. The compiled j314c and j316c trees differ from before only by the sep alias, the sep, spi2 and mesa status, and the mesa properties above. Their spi2, mesa and SEP nodes, with every node those reference (pins, power domains, GPIO and interrupt providers, mailbox, DART), match t6000-j314s apart from firmware-name. Not checked or tested: nobody has yet read the J314c or J316c platform device tree to confirm pins 196 and 195, and neither machine has booted this. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- arch/arm64/boot/dts/apple/t6001-j314c.dts | 38 +++++++++++++++++++++++ arch/arm64/boot/dts/apple/t6001-j316c.dts | 38 +++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6001-j314c.dts b/arch/arm64/boot/dts/apple/t6001-j314c.dts index 245df6d03ee42..23be760ba6e49 100644 --- a/arch/arm64/boot/dts/apple/t6001-j314c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j314c.dts @@ -15,6 +15,44 @@ / { compatible = "apple,j314c", "apple,t6001", "apple,arm-platform"; model = "Apple MacBook Pro (14-inch, M1 Max, 2021)"; + + aliases { + sep = &sep; + }; +}; + +/* + * m1n1 keys off the sep alias to add the sepfw region and the boot + * manifests the driver needs to cold-boot SEPOS. + */ +&sep { + status = "okay"; +}; + +/* + * The J314s/J316s lines, read from those machines' platform device trees + * (/arm-io/spi2/mesa) and identical on the 14" and 16" M1 Pro: the power + * line is gpio0 pin 196 (function-mesa_pwr, active high) and the data-ready + * line is pin 195. The M1 Max boards share this family's board file, + * t600x-j314-j316.dtsi. J314c's own platform device tree has not been + * checked yet, and J314c is untested. + */ +&spi2 { + status = "okay"; +}; + +&mesa { + status = "okay"; + enable-gpios = <&pinctrl_ap 196 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <195 IRQ_TYPE_EDGE_RISING>; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; + /* + * Per-device factory data, extracted from this machine's own macOS + * install; it is not redistributable. + */ + firmware-name = "apple/mesacal-j314c.bin"; }; &wifi0 { diff --git a/arch/arm64/boot/dts/apple/t6001-j316c.dts b/arch/arm64/boot/dts/apple/t6001-j316c.dts index a000d497b705f..d171d9c1200b8 100644 --- a/arch/arm64/boot/dts/apple/t6001-j316c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j316c.dts @@ -15,6 +15,44 @@ / { compatible = "apple,j316c", "apple,t6001", "apple,arm-platform"; model = "Apple MacBook Pro (16-inch, M1 Max, 2021)"; + + aliases { + sep = &sep; + }; +}; + +/* + * m1n1 keys off the sep alias to add the sepfw region and the boot + * manifests the driver needs to cold-boot SEPOS. + */ +&sep { + status = "okay"; +}; + +/* + * The J314s/J316s lines, read from those machines' platform device trees + * (/arm-io/spi2/mesa) and identical on the 14" and 16" M1 Pro: the power + * line is gpio0 pin 196 (function-mesa_pwr, active high) and the data-ready + * line is pin 195. The M1 Max boards share this family's board file, + * t600x-j314-j316.dtsi. J316c's own platform device tree has not been + * checked yet, and J316c is untested. + */ +&spi2 { + status = "okay"; +}; + +&mesa { + status = "okay"; + enable-gpios = <&pinctrl_ap 196 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <195 IRQ_TYPE_EDGE_RISING>; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; + /* + * Per-device factory data, extracted from this machine's own macOS + * install; it is not redistributable. + */ + firmware-name = "apple/mesacal-j316c.bin"; }; &wifi0 { From 2db3ebfacda2d15076e82d30e247a7319356e699 Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 12:18:39 -0700 Subject: [PATCH 0926/2680] arm64: dts: apple: t8112-j413: enable the SEP and the Mesa fingerprint sensor Mirror t8112-j415, using ftechhelp's J415 block from commit a37935eca3cd ("arm64: dts: apple: t8112-j415: enable the SEP and the Mesa fingerprint sensor"), aurora-silicon/linux#152. Add the sep alias so m1n1 hands over the local-policy and iBoot manifests the warm registration needs. Enable spi2 and the Mesa sensor: power on AP GPIO 178, active high, with 20 ns chip-select setup and hold. t8112.dtsi already supplies 8 MHz, chip select 0 and SPI mode 1. As on J415, the data-ready line (GPIO 177) is left out, so the driver polls. The calibration file is apple/mesacal-j413.bin. The J413 platform device tree describes the sensor as J415's does: power 178, data-ready 177, 8 MHz, chip select 0. That matches what drivers/soc/apple/profile.rs records for T8112. As on J415, the SEP node and its DART stay disabled in the device tree. iBoot hands over a running SEP, and apple_sep enables both at module load after setting the DART's DMA range. The compiled j413 tree differs from before only by the sep alias, the spi2 and mesa status, and the mesa properties above. Its spi2, mesa and SEP nodes, with every node those reference, match t8112-j415 apart from firmware-name. Untested: no J413 has booted this. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- arch/arm64/boot/dts/apple/t8112-j413.dts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index 6a49b6727e3f6..eb03ebf173d76 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -21,6 +21,7 @@ aliases { bluetooth0 = &bluetooth0; keyboard = &keyboard; + sep = &sep; wifi0 = &wifi0; }; @@ -257,6 +258,26 @@ }; #include "hwmon-laptop.dtsi" +/* + * Mesa fingerprint sensor on spi2. The J413 platform device tree + * (/arm-io/spi2/mesa) describes it as J415's does: power on AP GPIO 178, + * active high, data-ready on GPIO 177. As on J415, the data-ready line is + * left out, so the driver polls until its trigger is confirmed. Native chip + * select only. J413 is untested. + */ +&spi2 { + status = "okay"; +}; + +&mesa { + status = "okay"; + enable-gpios = <&pinctrl_ap 178 GPIO_ACTIVE_HIGH>; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; + /* Per-device factory data from this Mac's own iBoot system container. */ + firmware-name = "apple/mesacal-j413.bin"; +}; + #include "isp-imx558-cfg0.dtsi" &isp { From d01143a0e92b3631f1d132ed5a0210e3a38c1fbb Mon Sep 17 00:00:00 2001 From: Wes Grimes Date: Fri, 2 Oct 2026 19:25:42 -0400 Subject: [PATCH 0927/2680] arm64: dts: apple: t602x: Add PCIe-C DART tunables The tunneled PCIe-C DART requires "apple,tunable". iBoot adds "dart-tunables-instance-0" to the ADT at boot, but m1n1 does not copy it for the dart-apciec* nodes. Without it the DART fails to probe and the xHCI behind the tunnel dies after its command ring times out. Add the values to the apciec0..2 DARTs of every T602x die. They were read from the ADTs of a J416c (T6021) and a J414s (T6020), and are identical on both machines and for all three ports; on j416s they bring up a Studio Display's xHCI. apciec3, present on the desktop machines, has no values yet, so pcie_apple.tunnel_kernel_init refuses that port. Signed-off-by: Wes Grimes --- arch/arm64/boot/dts/apple/t602x-dieX.dtsi | 3 +++ .../boot/dts/apple/t602x-pciec-dart-tunables.dtsi | 15 +++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t602x-pciec-dart-tunables.dtsi diff --git a/arch/arm64/boot/dts/apple/t602x-dieX.dtsi b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi index 41c924dc7c779..831e1dfa35dfa 100644 --- a/arch/arm64/boot/dts/apple/t602x-dieX.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi @@ -579,6 +579,7 @@ #iommu-cells = <1>; apple,dma-range = <0x0 0x80000000 0x0 0x80000000>; apple,dma-offset = <0x100 0x0>; +#include "t602x-pciec-dart-tunables.dtsi" }; DIE_NODE(apciec0): pcie@730000000 { @@ -805,6 +806,7 @@ #iommu-cells = <1>; apple,dma-range = <0x0 0x80000000 0x0 0x80000000>; apple,dma-offset = <0x100 0x0>; +#include "t602x-pciec-dart-tunables.dtsi" }; DIE_NODE(apciec1): pcie@b30000000 { @@ -1031,6 +1033,7 @@ #iommu-cells = <1>; apple,dma-range = <0x0 0x80000000 0x0 0x80000000>; apple,dma-offset = <0x100 0x0>; +#include "t602x-pciec-dart-tunables.dtsi" }; DIE_NODE(apciec2): pcie@f30000000 { diff --git a/arch/arm64/boot/dts/apple/t602x-pciec-dart-tunables.dtsi b/arch/arm64/boot/dts/apple/t602x-pciec-dart-tunables.dtsi new file mode 100644 index 0000000000000..8572cf187c17a --- /dev/null +++ b/arch/arm64/boot/dts/apple/t602x-pciec-dart-tunables.dtsi @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * T602x Thunderbolt PCIe (PCIe-C) DART tunables, included in each + * apciec[0-2] DART node. iBoot adds "dart-tunables-instance-0" to the ADT + * at boot, but m1n1 does not copy it for these DARTs, and the DART cannot + * probe without it. Values read from J416c and J414s ADTs, identical on + * both machines and for all three ports. Offsets are relative to the + * DART's reg. + */ +apple,tunable = <0x20c 0xff0000b7 0xe40000b7>, + <0x220 0x000f0f0f 0x000f0f0f>, + <0x224 0x00ffffff 0x00080808>, + <0x300 0x00001f31 0x00000001>, + <0x308 0x3ffffffc 0x10000000>, + <0x310 0x3ffffffc 0x3ffffffc>; From e749cd66e2157cf0e1f055e0dfa0da67ef58a7e1 Mon Sep 17 00:00:00 2001 From: Wes Grimes Date: Fri, 2 Oct 2026 19:49:28 -0400 Subject: [PATCH 0928/2680] arm64: dts: apple: t600x: Add PCIe-C DART tunables As on T602x, the tunneled PCIe-C DART requires "apple,tunable", which iBoot adds to the ADT at boot as "dart-tunables-instance-0" and m1n1 does not copy for the dart-apciec* nodes. Add the values to the apciec0..2 DARTs of T6000/T6001. They were read from a J314s ADT and are identical for all three ports. They differ from the T602x values. apciec3 has no values yet, so pcie_apple.tunnel_kernel_init refuses that port. Signed-off-by: Wes Grimes --- .../boot/dts/apple/t600x-pciec-dart-tunables.dtsi | 11 +++++++++++ arch/arm64/boot/dts/apple/t600x-pciec.dtsi | 3 +++ 2 files changed, 14 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t600x-pciec-dart-tunables.dtsi diff --git a/arch/arm64/boot/dts/apple/t600x-pciec-dart-tunables.dtsi b/arch/arm64/boot/dts/apple/t600x-pciec-dart-tunables.dtsi new file mode 100644 index 0000000000000..e72d1595f4de5 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t600x-pciec-dart-tunables.dtsi @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * T600x Thunderbolt PCIe (PCIe-C) DART tunables, included in each + * apciec[0-2] DART node. iBoot adds "dart-tunables-instance-0" to the ADT + * at boot, but m1n1 does not copy it for these DARTs, and the DART cannot + * probe without it. Values read from a J314s ADT, identical for all three + * ports. Offsets are relative to the DART's reg. + */ +apple,tunable = <0x20c 0xff000007 0xec000007>, + <0x220 0x000f0f0f 0x000f0f0f>, + <0x224 0x00ffffff 0x00080808>; diff --git a/arch/arm64/boot/dts/apple/t600x-pciec.dtsi b/arch/arm64/boot/dts/apple/t600x-pciec.dtsi index 86cc1ac550e6b..46fdce8e904f3 100644 --- a/arch/arm64/boot/dts/apple/t600x-pciec.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-pciec.dtsi @@ -16,6 +16,7 @@ interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; +#include "t600x-pciec-dart-tunables.dtsi" }; apciec0: pcie@730000000 { @@ -84,6 +85,7 @@ interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; +#include "t600x-pciec-dart-tunables.dtsi" }; apciec1: pcie@b30000000 { @@ -152,6 +154,7 @@ interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; +#include "t600x-pciec-dart-tunables.dtsi" }; apciec2: pcie@f30000000 { From e130cf844b28c57cc470feaa4fb2e006dbd17059 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 2 Oct 2026 23:21:03 -0700 Subject: [PATCH 0929/2680] arm64: dts: apple: t600x: Limit the PCIe-C DART window to 32 bits The driver configures a tunneled T8110 DART for 42-bit addresses and four-level page tables, and without an apple,dma-range the domain spans all 42 bits. The T600x PCIe-C DART walks three levels, so it reads the top-level table as the L1 table. Behind a Thunderbolt 3 dock on J314s, the dock's xHCI faulted on its first DMA and then died: apple-dart f21008000.iommu: translation fault: status:0x80000004 stream:0 code:0x4 (NO PMD FOR IOVA) at 0xffff0000 A 4 GiB window gives the domain three-level tables and leaves the four-level TCR bit clear. Tested on J314s with a CalDigit TS3 Plus: no DART faults, and a USB SSD on the dock's xHCI read 8 GiB without errors. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 9add89b380c0ea9864fc6b88ca4baebc77153a1d) --- arch/arm64/boot/dts/apple/t600x-pciec.dtsi | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-pciec.dtsi b/arch/arm64/boot/dts/apple/t600x-pciec.dtsi index 46fdce8e904f3..808048a7fcece 100644 --- a/arch/arm64/boot/dts/apple/t600x-pciec.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-pciec.dtsi @@ -16,6 +16,7 @@ interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; + apple,dma-range = <0x0 0x0 0x1 0x0>; #include "t600x-pciec-dart-tunables.dtsi" }; @@ -85,6 +86,7 @@ interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; + apple,dma-range = <0x0 0x0 0x1 0x0>; #include "t600x-pciec-dart-tunables.dtsi" }; @@ -154,6 +156,7 @@ interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; + apple,dma-range = <0x0 0x0 0x1 0x0>; #include "t600x-pciec-dart-tunables.dtsi" }; @@ -223,6 +226,7 @@ interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; + apple,dma-range = <0x0 0x0 0x1 0x0>; }; apciec3: pcie@1330000000 { From 8255d96b89bb0152ddec63611c864480b154c882 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 23:21:32 -0400 Subject: [PATCH 0930/2680] thunderbolt: apple: coordinate tunneled PCIe lifecycle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — Asahi exports the core symbols for the Apple driver with EXPORT_SYMBOL_FOR_MODULES(..., "thunderbolt_apple") and does not import the USB4 namespace; the new exports follow that] [ryan: adapted to 7.2.10 — split by topic: the device-tree hunks are carried by 000-devicetree] [ryan: adapted to 7.2.10 — Asahi documents these Thunderbolt functions with kerneldoc; our new code goes above those comments] Signed-off-by: DjDeveloperr --- drivers/thunderbolt/apple.c | 350 ++++++++++++++--------------------- drivers/thunderbolt/domain.c | 6 + 2 files changed, 140 insertions(+), 216 deletions(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index adbb3493d43ab..5ee113ad9a79a 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -88,6 +88,7 @@ #include #include #include +#include #include #include #include @@ -192,6 +193,7 @@ struct apple_cio { u32 target_cable_info; struct completion nhi_boot_completion; int nhi_boot_status; + struct platform_device *nhi_pdev; struct typec_thunderbolt_switch_dev *tbt_switch; /* Serializes PCIe-C population with cable teardown. */ @@ -201,37 +203,6 @@ struct apple_cio { bool pcie_tunnel_populated; }; -/* - * Temporary hardware bring-up gate. A non-zero value stops ACIO startup after - * the corresponding phase so each power/reset/MMIO transition can be tested - * independently on new SoCs without advancing into the next phase. - */ -static unsigned int apple_cio_debug_stage; -module_param_named(debug_stage, apple_cio_debug_stage, uint, 0644); -MODULE_PARM_DESC(debug_stage, "Stop ACIO startup after the selected debug phase"); - -static unsigned int apple_cio_pcie_tunnel_trace_delay_ms; -module_param_named(pcie_tunnel_trace_delay_ms, - apple_cio_pcie_tunnel_trace_delay_ms, uint, 0644); -MODULE_PARM_DESC(pcie_tunnel_trace_delay_ms, - "Delay before PCIe-C child population so its trace reaches persistent storage"); - -static bool apple_cio_pcie_intr2axi_prearmed; -module_param_named(pcie_intr2axi_prearmed, - apple_cio_pcie_intr2axi_prearmed, bool, 0644); -MODULE_PARM_DESC(pcie_intr2axi_prearmed, - "Diagnostic: consume a live userspace Intr2AXI pulse instead of pulsing again"); - -static bool apple_cio_defer_start; -module_param_named(defer_start, apple_cio_defer_start, bool, 0644); -MODULE_PARM_DESC(defer_start, - "Hold a detected USB4/TBT cable until ACIO is started through sysfs"); - -static int apple_cio_power_domain_limit = -1; -module_param_named(power_domain_limit, apple_cio_power_domain_limit, int, 0644); -MODULE_PARM_DESC(power_domain_limit, - "Enable this many traced ACIO power domains before returning (-1 = all)"); - static void apple_cio_of_node_put(void *data) { of_node_put(data); @@ -599,28 +570,14 @@ static int apple_cio_populate_pcie_tunnel(struct apple_cio *acio) * enabled both tunnel adapters and before either child can touch DART MMIO. * The bit self-clears after opening the bridge. */ - if (READ_ONCE(apple_cio_pcie_intr2axi_prearmed)) { - WRITE_ONCE(apple_cio_pcie_intr2axi_prearmed, false); - dev_info(acio->dev, - "PCIe-C Intr2AXI bridge accepted from live diagnostic handoff\n"); - } else { - writel(APPLE_CIO_PCIEC_INTR2AXI_ENABLE, - acio->pcie_intr2axi_base + APPLE_CIO_PCIEC_INTR2AXI_CTRL); - /* Complete the pulse without reading the transient register. */ - mb(); - dev_info(acio->dev, "PCIe-C Intr2AXI bridge enabled\n"); - } - if (apple_cio_debug_stage == 12) { - dev_info(acio->dev, - "ACIO debug stage 12: Intr2AXI pulsed; tunnel children held\n"); - return -EAGAIN; - } + writel(APPLE_CIO_PCIEC_INTR2AXI_ENABLE, + acio->pcie_intr2axi_base + APPLE_CIO_PCIEC_INTR2AXI_CTRL); + /* Complete the pulse without reading the transient register. */ + mb(); + dev_info(acio->dev, "PCIe-C Intr2AXI bridge enabled\n"); dev_info(acio->dev, "PCIe-C live handoff accepted; populating tunnel children\n"); - if (READ_ONCE(apple_cio_pcie_tunnel_trace_delay_ms)) - msleep(READ_ONCE(apple_cio_pcie_tunnel_trace_delay_ms)); - return of_platform_populate(acio->pcie_tunnel_np, NULL, NULL, acio->dev); } @@ -641,12 +598,6 @@ static int apple_cio_activate_pcie_tunnel_locked(struct apple_cio *acio) return dev_err_probe(acio->dev, -ENODEV, "pre-ACIO PCIe-C initialization is not active\n"); - if (apple_cio_debug_stage == 11) { - dev_info(acio->dev, - "ACIO debug stage 11: PCIe tunnel adapters held for manual population\n"); - return 0; - } - ret = apple_cio_populate_pcie_tunnel(acio); if (ret) return dev_err_probe(acio->dev, ret, @@ -677,73 +628,6 @@ static void apple_cio_pcie_tunnel_work(struct work_struct *work) dev_err(acio->dev, "deferred PCIe-C activation failed: %d\n", ret); } -static int apple_cio_start(struct apple_cio *acio); - -static ssize_t acio_start_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - struct apple_cio *acio = dev_get_drvdata(dev); - int ret; - - if (!sysfs_streq(buf, "1")) - return -EINVAL; - - ret = mutex_lock_interruptible(&acio->lock); - if (ret) - return ret; - - if (acio->current_cable_info) { - ret = -EALREADY; - goto out_unlock; - } - if (!acio->target_cable_info) { - ret = -ENODEV; - goto out_unlock; - } - - dev_info(acio->dev, - "starting deferred ACIO cable transition %#x\n", - acio->target_cable_info); - ret = apple_cio_start(acio); - -out_unlock: - mutex_unlock(&acio->lock); - return ret ?: count; -} -static DEVICE_ATTR_WO(acio_start); - -static ssize_t pcie_tunnel_populate_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - struct apple_cio *acio = dev_get_drvdata(dev); - int ret; - - if (!sysfs_streq(buf, "1")) - return -EINVAL; - - ret = mutex_lock_interruptible(&acio->pcie_tunnel_lock); - if (ret) - return ret; - - ret = apple_cio_activate_pcie_tunnel_locked(acio); - mutex_unlock(&acio->pcie_tunnel_lock); - - return ret ?: count; -} -static DEVICE_ATTR_WO(pcie_tunnel_populate); - -static struct attribute *apple_cio_attrs[] = { - &dev_attr_acio_start.attr, - &dev_attr_pcie_tunnel_populate.attr, - NULL, -}; - -static const struct attribute_group apple_cio_group = { - .attrs = apple_cio_attrs, -}; - static int apple_nhi_pci_tunnel_pre_activate(struct tb_nhi *nhi) { struct apple_nhi *anhi = nhi_to_anhi(nhi); @@ -761,21 +645,15 @@ static int apple_nhi_pci_tunnel_post_activate(struct tb_nhi *nhi) { struct apple_nhi *anhi = nhi_to_anhi(nhi); struct apple_cio *acio = anhi->acio; - int ret; /* - * This is the Linux equivalent of Apple's startUsingTunnel() boundary. - * Only now can releaseResetFlow() clear the tunneled PERST state before - * the root port enables LTSSM. + * This is the boundary at which the tunnel becomes usable. + * The USB4 configuration writes are asynchronous, so return to the + * Thunderbolt core before probing PCIe-C. The worker then waits on the + * hardware reset state without blocking completion of the tunnel paths. */ - mutex_lock(&acio->pcie_tunnel_lock); - ret = apple_cio_activate_pcie_tunnel_locked(acio); - mutex_unlock(&acio->pcie_tunnel_lock); - - /* Stage 12 intentionally leaves the active tunnel paths in place. */ - if (ret == -EAGAIN && apple_cio_debug_stage == 12) - return 0; - return ret; + mod_delayed_work(system_wq, &acio->pcie_tunnel_work, 0); + return 0; } static const struct tb_nhi_ops apple_nhi_ops = { @@ -930,6 +808,7 @@ static int apple_nhi_probe(struct platform_device *pdev) mutex_unlock(&anhi->tb->lock); + WRITE_ONCE(acio->nhi_pdev, pdev); acio->nhi_boot_status = 0; complete(&acio->nhi_boot_completion); return 0; @@ -948,10 +827,70 @@ static void apple_nhi_remove(struct platform_device *pdev) { struct apple_nhi *anhi = platform_get_drvdata(pdev); + WRITE_ONCE(anhi->acio->nhi_pdev, NULL); tb_domain_remove(anhi->tb); wait_for_completion(&anhi->nhi.domain_released); } +/* + * The Apple platform driver owns the NHI allocation and keeps an + * apple_nhi pointer in driver data. The generic NHI PM callbacks cannot be + * installed here because they expect driver data to contain struct tb. + */ +static int apple_nhi_suspend_noirq(struct device *dev) +{ + struct apple_nhi *anhi = dev_get_drvdata(dev); + + return tb_domain_suspend_noirq(anhi->tb); +} + +static int apple_nhi_resume_noirq(struct device *dev) +{ + struct apple_nhi *anhi = dev_get_drvdata(dev); + + return tb_domain_resume_noirq(anhi->tb); +} + +static int apple_nhi_freeze_noirq(struct device *dev) +{ + struct apple_nhi *anhi = dev_get_drvdata(dev); + + return tb_domain_freeze_noirq(anhi->tb); +} + +static int apple_nhi_thaw_noirq(struct device *dev) +{ + struct apple_nhi *anhi = dev_get_drvdata(dev); + + return tb_domain_thaw_noirq(anhi->tb); +} + +static int apple_nhi_suspend(struct device *dev) +{ + struct apple_nhi *anhi = dev_get_drvdata(dev); + + return tb_domain_suspend(anhi->tb); +} + +static void apple_nhi_complete(struct device *dev) +{ + struct apple_nhi *anhi = dev_get_drvdata(dev); + + tb_domain_complete(anhi->tb); +} + +static const struct dev_pm_ops apple_nhi_pm_ops = { + .suspend = apple_nhi_suspend, + .suspend_noirq = apple_nhi_suspend_noirq, + .resume_noirq = apple_nhi_resume_noirq, + .freeze_noirq = apple_nhi_freeze_noirq, + .thaw_noirq = apple_nhi_thaw_noirq, + .restore_noirq = apple_nhi_resume_noirq, + .poweroff = apple_nhi_suspend, + .poweroff_noirq = apple_nhi_suspend_noirq, + .complete = apple_nhi_complete, +}; + static const struct of_device_id apple_nhi_match[] = { { .compatible = "apple,t8103-usb4-nhi", @@ -964,11 +903,31 @@ static struct platform_driver apple_nhi_driver = { .driver = { .name = "thunderbolt-apple-nhi", .of_match_table = apple_nhi_match, + .pm = &apple_nhi_pm_ops, }, .probe = apple_nhi_probe, .remove = apple_nhi_remove, }; +static struct platform_device *apple_cio_find_pcie_tunnel(struct apple_cio *acio) +{ + struct platform_device *pdev = NULL; + + if (!acio->pcie_tunnel_np || !acio->pcie_tunnel_populated) + return NULL; + + for_each_available_child_of_node_scoped(acio->pcie_tunnel_np, child) { + if (!of_device_is_compatible(child, "apple,t8103-pciec") && + !of_device_is_compatible(child, "apple,t6000-pciec")) + continue; + + pdev = of_find_device_by_node(child); + break; + } + + return pdev; +} + /** * apple_cio_stop - Stop and power down the ACIO complex * @acio: ACIO complex to stop @@ -979,6 +938,7 @@ static struct platform_driver apple_nhi_driver = { */ static void apple_cio_stop(struct apple_cio *acio) { + struct platform_device *nhi_pdev, *pcie_pdev; int ret, i; lockdep_assert_held(&acio->lock); @@ -991,6 +951,35 @@ static void apple_cio_stop(struct apple_cio *acio) WRITE_ONCE(acio->pcie_tunnel_requested, false); cancel_delayed_work_sync(&acio->pcie_tunnel_work); mutex_lock(&acio->pcie_tunnel_lock); + + /* + * The PCIe-C tunnel reset handshake is carried by the NHI. Quiesce the + * remote PCI hierarchy and acknowledge tunneled PERST before removing + * that control plane. PCIe-C and its DART stay bound until the general + * child depopulation below. + */ + pcie_pdev = apple_cio_find_pcie_tunnel(acio); + if (pcie_pdev) { + ret = apple_pcie_tunnel_quiesce(&pcie_pdev->dev); + if (ret) + dev_warn(acio->dev, + "failed to quiesce PCIe-C before NHI shutdown: %d\n", + ret); + put_device(&pcie_pdev->dev); + } + + /* + * The NHI is the tunnel control plane. Stop it while the tunneled PCIe + * host and DART are still accessible, before depopulation tears down the + * data plane. Otherwise tb_domain_remove() walks the USB4 topology after + * PCIe-C has already reset its port and can wedge waiting on dead control + * traffic. The remaining children are still removed in reverse creation + * order below. + */ + nhi_pdev = READ_ONCE(acio->nhi_pdev); + if (nhi_pdev) + of_platform_device_destroy(&nhi_pdev->dev, NULL); + of_platform_depopulate(acio->dev); acio->pcie_tunnel_populated = false; mutex_unlock(&acio->pcie_tunnel_lock); @@ -1007,6 +996,7 @@ static void apple_cio_stop(struct apple_cio *acio) if (acio->pd_list->pd_links[i]) device_link_del(acio->pd_list->pd_links[i]); acio->pd_list->pd_links[i] = NULL; + dev_pm_syscore_device(acio->pd_list->pd_devs[i], false); } acio->current_cable_info = 0; @@ -1033,17 +1023,6 @@ static int apple_cio_start(struct apple_cio *acio) /* Create device links to the power domains in order to power them on */ for (i = 0; i < acio->pd_list->num_pds; i++) { - if (READ_ONCE(apple_cio_power_domain_limit) >= 0) { - dev_emerg(acio->dev, - "ACIO power-domain access %d armed: enable %s\n", - i + 1, dev_name(acio->pd_list->pd_devs[i])); - if (i >= READ_ONCE(apple_cio_power_domain_limit)) { - ret = -EAGAIN; - goto remove_links; - } - dev_emerg(acio->dev, - "ACIO power-domain access %d executing\n", i + 1); - } link = device_link_add(acio->dev, acio->pd_list->pd_devs[i], DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE); if (!link) { @@ -1051,20 +1030,14 @@ static int apple_cio_start(struct apple_cio *acio) goto remove_links; } acio->pd_list->pd_links[i] = link; - if (READ_ONCE(apple_cio_power_domain_limit) >= 0) - dev_emerg(acio->dev, - "ACIO power-domain access %d completed\n", i + 1); - dev_info(acio->dev, - "ACIO power domain %d (%s): runtime %s, usage %d\n", - i, dev_name(acio->pd_list->pd_devs[i]), - pm_runtime_active(acio->pd_list->pd_devs[i]) ? - "active" : "inactive", - atomic_read(&acio->pd_list->pd_devs[i]->power.usage_count)); - } - if (apple_cio_debug_stage == 1) { - dev_info(acio->dev, "ACIO debug stage 1: power domains enabled\n"); - ret = -EAGAIN; - goto remove_links; + /* + * ACIO owns these domains explicitly for the lifetime of the cable + * connection. The virtual genpd devices must not independently + * cycle them during system sleep: doing so destroys the live M3 and + * NHI state behind the still-bound child devices. Cable teardown + * removes the runtime-active links and gives system PM ownership back. + */ + dev_pm_syscore_device(acio->pd_list->pd_devs[i], true); } /* @@ -1076,11 +1049,6 @@ static int apple_cio_start(struct apple_cio *acio) dev_err(acio->dev, "ACIO block failed to start: %d\n", ret); goto remove_links; } - if (apple_cio_debug_stage == 2) { - dev_info(acio->dev, "ACIO debug stage 2: reset deasserted\n"); - ret = -EAGAIN; - goto remove_links; - } /* * Start and wait for the co-processor to boot. @@ -1091,39 +1059,19 @@ static int apple_cio_start(struct apple_cio *acio) * them with a bare store. */ val = readl(acio->rc_base + APPLE_CIO_M3_CTRL); - if (apple_cio_debug_stage == 3) { - dev_info(acio->dev, "ACIO debug stage 3: M3 control read as %#x\n", val); - ret = -EAGAIN; - goto remove_links; - } writel(val | APPLE_CIO_M3_CTRL_START, acio->rc_base + APPLE_CIO_M3_CTRL); - if (apple_cio_debug_stage == 4) { - dev_info(acio->dev, "ACIO debug stage 4: M3 start requested\n"); - ret = -EAGAIN; - goto remove_links; - } acio->rtk = apple_rtkit_init(acio->dev, acio, NULL, 0, &apple_cio_rtkit_ops); if (IS_ERR(acio->rtk)) { ret = PTR_ERR(acio->rtk); dev_err(acio->dev, "Failed to initialize RTKit: %d\n", ret); goto remove_links; } - if (apple_cio_debug_stage == 5) { - dev_info(acio->dev, "ACIO debug stage 5: RTKit initialized\n"); - ret = -EAGAIN; - goto err_free_rtkit; - } ret = apple_rtkit_boot(acio->rtk); if (ret) { dev_err(acio->dev, "M3 RTKit failed to boot: %d\n", ret); goto err_free_rtkit; } - if (apple_cio_debug_stage == 6) { - dev_info(acio->dev, "ACIO debug stage 6: RTKit booted\n"); - ret = -EAGAIN; - goto err_shutdown_rtkit; - } ret = readl_poll_timeout(acio->rc_base + APPLE_CIO_M3_STAT, state, state & APPLE_CIO_M3_STAT_STATE, 100, 500000); @@ -1131,18 +1079,8 @@ static int apple_cio_start(struct apple_cio *acio) dev_err(acio->dev, "M3 firmware failed to get ready: %d\n", ret); goto err_shutdown_rtkit; } - if (apple_cio_debug_stage == 7) { - dev_info(acio->dev, "ACIO debug stage 7: M3 ready state %#x\n", state); - ret = -EAGAIN; - goto err_shutdown_rtkit; - } apple_tunable_apply(acio->rc_base, acio->rc_tunable); - if (apple_cio_debug_stage == 8) { - dev_info(acio->dev, "ACIO debug stage 8: RC tunables applied\n"); - ret = -EAGAIN; - goto err_shutdown_rtkit; - } /* * Bring up devices which are part of ACIO and are now accessible by the main SoC @@ -1154,11 +1092,6 @@ static int apple_cio_start(struct apple_cio *acio) dev_err(acio->dev, "failed to populate children: %d\n", ret); goto err_depopulate; } - if (apple_cio_debug_stage == 9) { - dev_info(acio->dev, "ACIO debug stage 9: child devices populated\n"); - ret = -EAGAIN; - goto err_depopulate; - } if (!wait_for_completion_timeout(&acio->nhi_boot_completion, msecs_to_jiffies(APPLE_CIO_NHI_BOOT_TIMEOUT_MS))) { @@ -1170,11 +1103,6 @@ static int apple_cio_start(struct apple_cio *acio) ret = acio->nhi_boot_status; goto err_depopulate; } - if (apple_cio_debug_stage == 10) { - dev_info(acio->dev, "ACIO debug stage 10: NHI is ready\n"); - ret = -EAGAIN; - goto err_depopulate; - } acio->current_cable_info = acio->target_cable_info; return 0; @@ -1192,6 +1120,7 @@ static int apple_cio_start(struct apple_cio *acio) if (acio->pd_list->pd_links[i]) device_link_del(acio->pd_list->pd_links[i]); acio->pd_list->pd_links[i] = NULL; + dev_pm_syscore_device(acio->pd_list->pd_devs[i], false); } return ret; } @@ -1276,13 +1205,6 @@ static int apple_cio_tbt_switch_set(struct typec_thunderbolt_switch_dev *sw, * Bring up or power down the ACIO complex * current_cable_info will be updated in the start/stop functions */ - if (acio->target_cable_info && READ_ONCE(apple_cio_defer_start)) { - dev_info(acio->dev, - "USB4/TBT cable transition %#x held for controlled ACIO start\n", - acio->target_cable_info); - return 0; - } - if (acio->target_cable_info) return apple_cio_start(acio); @@ -1378,10 +1300,6 @@ static int apple_cio_probe(struct platform_device *pdev) else if (ret < 3) return dev_err_probe(dev, -EINVAL, "Not enough PM domains\n"); - ret = devm_device_add_group(dev, &apple_cio_group); - if (ret) - return ret; - /* And finally register the OOB notification for Thunderbolt/USB4 cables */ struct typec_thunderbolt_switch_desc desc = { .fwnode = pdev->dev.fwnode, diff --git a/drivers/thunderbolt/domain.c b/drivers/thunderbolt/domain.c index 65d0eca3c9a80..69a46c59b23e8 100644 --- a/drivers/thunderbolt/domain.c +++ b/drivers/thunderbolt/domain.c @@ -545,6 +545,7 @@ int tb_domain_suspend_noirq(struct tb *tb) return ret; } +EXPORT_SYMBOL_FOR_MODULES(tb_domain_suspend_noirq, "thunderbolt_apple"); /** * tb_domain_resume_noirq() - Resume a domain @@ -567,11 +568,13 @@ int tb_domain_resume_noirq(struct tb *tb) return ret; } +EXPORT_SYMBOL_FOR_MODULES(tb_domain_resume_noirq, "thunderbolt_apple"); int tb_domain_suspend(struct tb *tb) { return tb->cm_ops->suspend ? tb->cm_ops->suspend(tb) : 0; } +EXPORT_SYMBOL_FOR_MODULES(tb_domain_suspend, "thunderbolt_apple"); int tb_domain_freeze_noirq(struct tb *tb) { @@ -586,6 +589,7 @@ int tb_domain_freeze_noirq(struct tb *tb) return ret; } +EXPORT_SYMBOL_FOR_MODULES(tb_domain_freeze_noirq, "thunderbolt_apple"); int tb_domain_thaw_noirq(struct tb *tb) { @@ -599,12 +603,14 @@ int tb_domain_thaw_noirq(struct tb *tb) return ret; } +EXPORT_SYMBOL_FOR_MODULES(tb_domain_thaw_noirq, "thunderbolt_apple"); void tb_domain_complete(struct tb *tb) { if (tb->cm_ops->complete) tb->cm_ops->complete(tb); } +EXPORT_SYMBOL_FOR_MODULES(tb_domain_complete, "thunderbolt_apple"); int tb_domain_runtime_suspend(struct tb *tb) { From 63c70b5aebf9d4465aaa80b4c7b50e55a4d33703 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 23:22:16 -0400 Subject: [PATCH 0931/2680] usb: dwc3-apple: preserve live ports across system sleep Signed-off-by: DjDeveloperr --- drivers/usb/dwc3/dwc3-apple.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index 40c3ccfddb670..33d3b467f5380 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -301,6 +301,14 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st } appledwc->state = state; + /* + * A connected cable leaves the Apple DWC3 core and its PHY as one live, + * stateful unit. The glue driver tears that unit down explicitly on a + * role change or disconnect. Do not let generic system PM independently + * power-gate the DWC3 device in between: xHCI then observes a dead core on + * resume and cannot restore the root hubs without a full cable teardown. + */ + dev_pm_syscore_device(appledwc->dev, true); return 0; core_exit: @@ -325,9 +333,11 @@ static int dwc3_apple_exit(struct dwc3_apple *appledwc) /* Nothing to do if we're already off */ return 0; case DWC3_APPLE_DEVICE: + dev_pm_syscore_device(appledwc->dev, false); dwc3_gadget_exit(&appledwc->dwc); break; case DWC3_APPLE_HOST: + dev_pm_syscore_device(appledwc->dev, false); dwc3_host_exit(&appledwc->dwc); break; } From 9590aaec8b59523b7ef51129666bf3f57e3b382f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 23:23:12 -0400 Subject: [PATCH 0932/2680] usb: typec: cd321x: isolate hotplug state by port Signed-off-by: DjDeveloperr --- drivers/usb/typec/tipd/core.c | 55 +++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index b2c25bb05cecf..f6d53416e251c 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -685,6 +685,11 @@ static void cd321x_update_work(struct work_struct *work) bool usb_connection = st.data_status & (TPS_DATA_STATUS_USB2_CONNECTION | TPS_DATA_STATUS_USB3_CONNECTION); + bool dp_mode_changed = st.data_status_changed & + (TPS_DATA_STATUS_DP_CONNECTION | + TPS_DATA_STATUS_DP_PIN_ASSIGNMENT_MASK); + bool dp_was_connected = cd321x->state.alt == cd321x->port_altmode_dp; + bool dp_connected = st.data_status & TPS_DATA_STATUS_DP_CONNECTION; bool dp_hpd = st.data_status & CD321X_DATA_STATUS_HPD_LEVEL; bool dp_hpd_changed = st.data_status_changed & CD321X_DATA_STATUS_HPD_LEVEL; @@ -713,11 +718,31 @@ static void cd321x_update_work(struct work_struct *work) (is_pd && memcmp(&st.partner_identity, &cd321x->cur_partner_identity, sizeof(struct usb_pd_identity)))); - /* If we are switching from an active role, transition to USB_ROLE_NONE first */ - if (old_role != USB_ROLE_NONE && (new_role != old_role || was_disconnected)) + /* + * ACIO carries the tunneled PCIe reset handshake over the still-live + * Type-C data path. Close a disappearing USB4/TBT session before dropping + * the USB role; otherwise the role transition tears down the ATC lanes and + * PCIe-C cannot quiesce its hierarchy before NHI removal. + */ + if ((!new_connected || was_disconnected) && + cd321x->state.mode != TYPEC_STATE_SAFE) + typec_thunderbolt_switch_set(cd321x->tbt_switch, + &tbt_switch_data); + + /* If we are switching from an active role, transition to USB_ROLE_NONE. */ + if (old_role != USB_ROLE_NONE && + (new_role != old_role || was_disconnected || dp_mode_changed)) usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); - if (cd321x->connector_fwnode && (!dp_hpd || dp_hpd_changed)) { + /* + * Every Type-C port references the same external DCP connector. Do not + * report HPD-low for a port which neither had nor has a DP connection: + * USB4/TBT status changes on such a port would otherwise tear down a DP + * stream carried by a different Type-C port. + */ + if (cd321x->connector_fwnode && + (dp_was_connected || dp_connected) && + (!dp_hpd || dp_hpd_changed)) { drm_connector_oob_hotplug_event(cd321x->connector_fwnode, connector_status_disconnected); } @@ -730,7 +755,6 @@ static void cd321x_update_work(struct work_struct *work) /* If there was a disconnection, set PHY to off */ if (!new_connected || was_disconnected) { - typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); cd321x->state.alt = NULL; cd321x->state.mode = TYPEC_STATE_SAFE; cd321x->state.data = NULL; @@ -777,7 +801,7 @@ static void cd321x_update_work(struct work_struct *work) /* Launch the USB role switch */ usb_role_switch_set_role(tps->role_sw, new_role); - if (cd321x->connector_fwnode && dp_hpd) + if (cd321x->connector_fwnode && dp_connected && dp_hpd) drm_connector_oob_hotplug_event(cd321x->connector_fwnode, connector_status_connected); power_supply_changed(tps->psy); @@ -1695,8 +1719,29 @@ tps25750_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) static void cd321x_remove(struct tps6598x *tps) { struct cd321x *cd321x = container_of(tps, struct cd321x, tps); + struct typec_thunderbolt_switch_data tbt_switch_data = { + .state = TYPEC_THUNDERBOLT_SWITCH_OFF, + }; + int ret; cancel_delayed_work_sync(&cd321x->update_work); + + /* + * Driver teardown must close the same hardware sessions as a physical + * disconnect. Otherwise ACIO retains the old USB4 cable state while a + * reprobe enters a new session, which its firmware treats as an invalid + * live-to-live transition and resolves by resetting the SoC. + * + * Close ACIO first while the ATC lanes can still carry PCIe-C's tunneled + * reset handshake, then drop the USB role after NHI and PCIe-C teardown. + */ + ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, + &tbt_switch_data); + if (ret) + dev_warn(tps->dev, + "failed to close Thunderbolt/USB4 session on remove: %d\n", + ret); + usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); } int tipd_init(struct tps6598x *tps) From 63b339dcfe0737f33b6e0a8c7fa260347843df2b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 29 Aug 2026 03:48:04 -0400 Subject: [PATCH 0933/2680] usb: typec: cd321x: preserve events during debounce A reconnect IRQ can arrive after the update worker releases its lock but before the workqueue clears the running state. In that window cancel_delayed_work() does not cancel anything and schedule_delayed_work() refuses to queue another pass, leaving the accumulated cable state unprocessed.\n\nUse mod_delayed_work() so a later pass is guaranteed even when the previous update is still completing. Signed-off-by: DjDeveloperr --- drivers/usb/typec/tipd/core.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index f6d53416e251c..c1c384aa8c2f1 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -830,12 +830,13 @@ static int cd321x_connect(struct tps6598x *tps, u32 status) cd321x_queue_status(cd321x); /* - * Cancel pending work if not already running, then requeue after CD321X_DEBOUNCE_DELAY_MS - * regardless since the work function will check for any plug or altmodes changes since - * its last run anyway. + * Requeue even when the previous update is already running. A separate + * cancel_delayed_work() and schedule_delayed_work() pair can lose a cable + * event in that window because the latter sees the running work as busy. + * The worker consumes every accumulated status change on its next pass. */ - cancel_delayed_work(&cd321x->update_work); - schedule_delayed_work(&cd321x->update_work, msecs_to_jiffies(CD321X_DEBOUNCE_DELAY_MS)); + mod_delayed_work(system_wq, &cd321x->update_work, + msecs_to_jiffies(CD321X_DEBOUNCE_DELAY_MS)); return 0; } From 68bf5d1e02e1c918f6a73f70e9d06f55b365ee5e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 29 Aug 2026 04:55:14 -0400 Subject: [PATCH 0934/2680] usb: typec: cd321x: track the active display route Every CD321x port on these systems references the same external DCP connector. A second sink can negotiate DisplayPort Alt Mode while the DCP mux rejects its route with -EBUSY. The port nevertheless emitted shared-connector disconnect events, blanking the display already carried by another port. Track display-route ownership only after typec_mux_set() succeeds, propagate mux failures into that decision, and allow only the owner to emit disconnect events. This makes a rejected second display non-disruptive and keeps the software mux state retryable after an error. Fixes: 5a46b5cc44de ("usb: typec: cd321x: isolate hotplug state by port") Signed-off-by: DjDeveloperr --- drivers/usb/typec/tipd/core.c | 72 +++++++++++++++++++++---------- drivers/usb/typec/tipd/tps6598x.h | 1 + 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index c1c384aa8c2f1..6f76cc4f184d1 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -542,21 +542,24 @@ static void tps6598x_handle_plug_event(struct tps6598x *tps, u32 status) } } -static void cd321x_typec_update_mode(struct tps6598x *tps, struct cd321x_status *st) +static int cd321x_typec_update_mode(struct tps6598x *tps, + struct cd321x_status *st) { struct cd321x *cd321x = container_of(tps, struct cd321x, tps); + struct typec_mux_state old_state = cd321x->state; struct typec_thunderbolt_switch_data tbt_switch_data = { .state = TYPEC_THUNDERBOLT_SWITCH_OFF, }; + int ret = 0; if (!(st->data_status & TPS_DATA_STATUS_DATA_CONNECTION)) { if (cd321x->state.mode == TYPEC_STATE_SAFE) - return; + return 0; cd321x->state.alt = NULL; cd321x->state.mode = TYPEC_STATE_SAFE; cd321x->state.data = NULL; typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); - typec_mux_set(cd321x->mux, &cd321x->state); + ret = typec_mux_set(cd321x->mux, &cd321x->state); } else if (st->data_status & TPS_DATA_STATUS_DP_CONNECTION) { struct typec_displayport_data dp_data; unsigned long mode; @@ -582,12 +585,12 @@ static void cd321x_typec_update_mode(struct tps6598x *tps, struct cd321x_status break; default: dev_err(tps->dev, "Invalid DP pin assignment\n"); - return; + return -EINVAL; } if (cd321x->state.alt == cd321x->port_altmode_dp && cd321x->state.mode == mode) { - return; + return 0; } dp_data.status = le32_to_cpu(st->dp_sid_status.status_rx); @@ -596,13 +599,13 @@ static void cd321x_typec_update_mode(struct tps6598x *tps, struct cd321x_status cd321x->state.data = &dp_data; cd321x->state.mode = mode; typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); - typec_mux_set(cd321x->mux, &cd321x->state); + ret = typec_mux_set(cd321x->mux, &cd321x->state); } else if (st->data_status & TPS_DATA_STATUS_TBT_CONNECTION) { struct typec_thunderbolt_data tbt_data; if (cd321x->state.alt == cd321x->port_altmode_tbt && cd321x->state.mode == TYPEC_TBT_MODE) - return; + return 0; tbt_data.cable_mode = TBT_MODE | TBT_SET_CABLE_SPEED(TPS_DATA_STATUS_TBT_CABLE_SPEED(st->data_status)) | @@ -620,7 +623,9 @@ static void cd321x_typec_update_mode(struct tps6598x *tps, struct cd321x_status cd321x->state.alt = cd321x->port_altmode_tbt; cd321x->state.mode = TYPEC_TBT_MODE; cd321x->state.data = &tbt_data; - typec_mux_set(cd321x->mux, &cd321x->state); + ret = typec_mux_set(cd321x->mux, &cd321x->state); + if (ret) + goto out; tbt_switch_data.state = TYPEC_THUNDERBOLT_SWITCH_TBT; tbt_switch_data.tbt = tbt_data; @@ -632,7 +637,7 @@ static void cd321x_typec_update_mode(struct tps6598x *tps, struct cd321x_status struct enter_usb_data eusb_data; if (cd321x->state.alt == NULL && cd321x->state.mode == TYPEC_MODE_USB4) - return; + return 0; eusb_data.eudo = le32_to_cpu(st->usb4_status.eudo); eusb_data.active_link_training = @@ -641,7 +646,9 @@ static void cd321x_typec_update_mode(struct tps6598x *tps, struct cd321x_status cd321x->state.alt = NULL; cd321x->state.data = &eusb_data; cd321x->state.mode = TYPEC_MODE_USB4; - typec_mux_set(cd321x->mux, &cd321x->state); + ret = typec_mux_set(cd321x->mux, &cd321x->state); + if (ret) + goto out; tbt_switch_data.state = TYPEC_THUNDERBOLT_SWITCH_USB4; tbt_switch_data.usb4 = eusb_data; @@ -651,16 +658,23 @@ static void cd321x_typec_update_mode(struct tps6598x *tps, struct cd321x_status typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); } else { if (cd321x->state.alt == NULL && cd321x->state.mode == TYPEC_STATE_USB) - return; + return 0; cd321x->state.alt = NULL; cd321x->state.mode = TYPEC_STATE_USB; cd321x->state.data = NULL; typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); - typec_mux_set(cd321x->mux, &cd321x->state); + ret = typec_mux_set(cd321x->mux, &cd321x->state); } +out: /* Clear data since it's no longer used after typec_mux_set and points to the stack */ cd321x->state.data = NULL; + if (ret) { + cd321x->state = old_state; + dev_warn(tps->dev, "failed to configure Type-C mux: %d\n", ret); + } + + return ret; } static void cd321x_update_work(struct work_struct *work) @@ -688,7 +702,7 @@ static void cd321x_update_work(struct work_struct *work) bool dp_mode_changed = st.data_status_changed & (TPS_DATA_STATUS_DP_CONNECTION | TPS_DATA_STATUS_DP_PIN_ASSIGNMENT_MASK); - bool dp_was_connected = cd321x->state.alt == cd321x->port_altmode_dp; + bool dp_route_was_active = cd321x->display_route_active; bool dp_connected = st.data_status & TPS_DATA_STATUS_DP_CONNECTION; bool dp_hpd = st.data_status & CD321X_DATA_STATUS_HPD_LEVEL; @@ -735,14 +749,16 @@ static void cd321x_update_work(struct work_struct *work) usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); /* - * Every Type-C port references the same external DCP connector. Do not - * report HPD-low for a port which neither had nor has a DP connection: - * USB4/TBT status changes on such a port would otherwise tear down a DP - * stream carried by a different Type-C port. + * Every Type-C port references the same external DCP connector. Only the + * port which successfully acquired the shared display route may report a + * disconnect. A second DP sink can negotiate Alt Mode even though its mux + * request is rejected with -EBUSY; treating that sink as the connector + * owner would tear down the display carried by the first port. */ if (cd321x->connector_fwnode && - (dp_was_connected || dp_connected) && - (!dp_hpd || dp_hpd_changed)) { + dp_route_was_active && + (!new_connected || was_disconnected || !dp_connected || !dp_hpd || + dp_hpd_changed || dp_mode_changed)) { drm_connector_oob_hotplug_event(cd321x->connector_fwnode, connector_status_disconnected); } @@ -755,10 +771,21 @@ static void cd321x_update_work(struct work_struct *work) /* If there was a disconnection, set PHY to off */ if (!new_connected || was_disconnected) { + struct typec_mux_state old_state = cd321x->state; + int ret; + cd321x->state.alt = NULL; cd321x->state.mode = TYPEC_STATE_SAFE; cd321x->state.data = NULL; - typec_set_mode(tps->port, TYPEC_STATE_SAFE); + ret = typec_set_mode(tps->port, TYPEC_STATE_SAFE); + if (ret) { + dev_warn(tps->dev, + "failed to place Type-C mux in safe mode: %d\n", + ret); + cd321x->state = old_state; + } else { + cd321x->display_route_active = false; + } } /* Update Type-C properties */ @@ -796,12 +823,13 @@ static void cd321x_update_work(struct work_struct *work) } /* Update the TypeC MUX/PHY state */ - cd321x_typec_update_mode(tps, &st); + if (!cd321x_typec_update_mode(tps, &st)) + cd321x->display_route_active = dp_connected; /* Launch the USB role switch */ usb_role_switch_set_role(tps->role_sw, new_role); - if (cd321x->connector_fwnode && dp_connected && dp_hpd) + if (cd321x->connector_fwnode && cd321x->display_route_active && dp_hpd) drm_connector_oob_hotplug_event(cd321x->connector_fwnode, connector_status_connected); power_supply_changed(tps->psy); diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index d03b9ae7fa58e..c3ec241fd56f0 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -380,6 +380,7 @@ struct cd321x { struct typec_mux_state state; struct typec_thunderbolt_switch *tbt_switch; + bool display_route_active; struct cd321x_status update_status; struct delayed_work update_work; From 44737e8f663c475083dbf8e39f54a270ae145024 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 29 Aug 2026 19:30:15 -0400 Subject: [PATCH 0935/2680] usb: typec: cd321x: cache and replay DP alt-mode status cd321x only forwarded a DisplayPort mux state when the alternate mode or the pin assignment changed. A partner that re-asserts HPD, or that changes its DP status while staying in the same mode, produced no mux notification at all, so a DisplayPort sink could sit powered and enumerated with no display route ever being established. Cache the DP status and configuration alongside the mux state and resend the notification whenever either changes. Take the HPD level from the CD321x DATA_STATUS register, which reports it reliably, rather than from the status_rx VDO. Clear the cache when the port leaves DP so the next entry always replays a fresh state. Signed-off-by: DjDeveloperr --- drivers/usb/typec/tipd/core.c | 21 +++++++++++++++++---- drivers/usb/typec/tipd/tps6598x.h | 3 ++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 6f76cc4f184d1..8a217ae6e6b93 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -588,18 +588,28 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, return -EINVAL; } + dp_data.status = le32_to_cpu(st->dp_sid_status.status_rx); + dp_data.conf = le32_to_cpu(st->dp_sid_status.configure); + if (st->data_status & CD321X_DATA_STATUS_HPD_LEVEL) + dp_data.status |= DP_STATUS_HPD_STATE; + else + dp_data.status &= ~DP_STATUS_HPD_STATE; + if (cd321x->state.alt == cd321x->port_altmode_dp && - cd321x->state.mode == mode) { + cd321x->state.mode == mode && + cd321x->dp_status == dp_data.status && + cd321x->dp_conf == dp_data.conf) return 0; - } - dp_data.status = le32_to_cpu(st->dp_sid_status.status_rx); - dp_data.conf = le32_to_cpu(st->dp_sid_status.configure); cd321x->state.alt = cd321x->port_altmode_dp; cd321x->state.data = &dp_data; cd321x->state.mode = mode; typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); ret = typec_mux_set(cd321x->mux, &cd321x->state); + if (!ret) { + cd321x->dp_status = dp_data.status; + cd321x->dp_conf = dp_data.conf; + } } else if (st->data_status & TPS_DATA_STATUS_TBT_CONNECTION) { struct typec_thunderbolt_data tbt_data; @@ -672,6 +682,9 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, if (ret) { cd321x->state = old_state; dev_warn(tps->dev, "failed to configure Type-C mux: %d\n", ret); + } else if (cd321x->state.alt != cd321x->port_altmode_dp) { + cd321x->dp_status = 0; + cd321x->dp_conf = 0; } return ret; diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index c3ec241fd56f0..32f7bd5cf7093 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -378,7 +378,8 @@ struct cd321x { struct typec_mux *mux; struct typec_mux_state state; - + u32 dp_status; + u32 dp_conf; struct typec_thunderbolt_switch *tbt_switch; bool display_route_active; From b9b741b5131c5176b36fef2f069549b7c4dec7a4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 29 Aug 2026 00:57:28 -0400 Subject: [PATCH 0936/2680] drm: apple: retrain Type-C display after USB4 route changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — split by topic: only the drivers/usb hunks of this dcp-display commit; the rest is in 200-dcp] Signed-off-by: DjDeveloperr --- drivers/usb/typec/tipd/core.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 8a217ae6e6b93..44e17c13c262f 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -717,6 +717,9 @@ static void cd321x_update_work(struct work_struct *work) TPS_DATA_STATUS_DP_PIN_ASSIGNMENT_MASK); bool dp_route_was_active = cd321x->display_route_active; bool dp_connected = st.data_status & TPS_DATA_STATUS_DP_CONNECTION; + bool usb4_started = + (st.data_status_changed & CD321X_DATA_STATUS_USB4_CONNECTION) && + (st.data_status & CD321X_DATA_STATUS_USB4_CONNECTION); bool dp_hpd = st.data_status & CD321X_DATA_STATUS_HPD_LEVEL; bool dp_hpd_changed = st.data_status_changed & CD321X_DATA_STATUS_HPD_LEVEL; @@ -845,6 +848,16 @@ static void cd321x_update_work(struct work_struct *work) if (cd321x->connector_fwnode && cd321x->display_route_active && dp_hpd) drm_connector_oob_hotplug_event(cd321x->connector_fwnode, connector_status_connected); + /* + * Apple routes all Type-C ports through shared high-speed display fabric. + * Starting USB4 on one port can interrupt a DP stream on another without + * changing that display's HPD state. Ask the display driver to retrain the + * still-connected route after ACIO and the USB role are live. + */ + if (cd321x->connector_fwnode && usb4_started && !dp_connected) + drm_connector_oob_hotplug_event(cd321x->connector_fwnode, + connector_status_unknown); + power_supply_changed(tps->psy); } From 3b06bae5f62a70fd74600e17d5d8b685faf7d626 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 31 Aug 2026 16:18:53 -0400 Subject: [PATCH 0937/2680] thunderbolt: Add a PCI tunnel deactivate callback A host controller can need to react to a PCI tunnel going away, not just to one being established. On Apple silicon the tunneled PCIe hierarchy is only reachable through the tunnel, so it has to be torn down before the paths are disabled rather than left behind with nothing underneath it. Add pci_tunnel_deactivate to tb_nhi_ops as the counterpart to the existing pre- and post-activate callbacks, and call it at the start of tunnel deactivation while the hierarchy can still be reached. Signed-off-by: DjDeveloperr --- drivers/thunderbolt/nhi.h | 1 + drivers/thunderbolt/tunnel.c | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index 95c8ec724678a..7147217310763 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -103,6 +103,7 @@ struct tb_nhi_ops { void (*ring_configure)(struct tb_ring *ring, u32 flags, u32 e2e_flags); int (*pci_tunnel_pre_activate)(struct tb_nhi *nhi); int (*pci_tunnel_post_activate)(struct tb_nhi *nhi); + int (*pci_tunnel_deactivate)(struct tb_nhi *nhi); bool (*is_present)(struct tb_nhi *nhi); int (*init_interrupts)(struct tb_nhi *nhi); void (*reset_interface)(struct tb_nhi *nhi); diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index 09d6b1d3f2637..0be1fb7085112 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -382,10 +382,29 @@ static int tb_pci_post_activate(struct tb_tunnel *tunnel) return 0; } +static int tb_pci_deactivate(struct tb_tunnel *tunnel) +{ + const struct tb_nhi_ops *ops = tunnel->tb->nhi->ops; + + if (ops && ops->pci_tunnel_deactivate) + return ops->pci_tunnel_deactivate(tunnel->tb->nhi); + return 0; +} + static int tb_pci_activate(struct tb_tunnel *tunnel, bool activate) { int res; + /* + * Let the host drop its view of the tunnel before the paths go, while + * what is behind it can still be reached. + */ + if (!activate) { + res = tb_pci_deactivate(tunnel); + if (res) + return res; + } + if (activate) { res = tb_pci_set_ext_encapsulation(tunnel, activate); if (res) From 6e49ff35c394089b1b9246ca07c19dca31e13cf5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 31 Aug 2026 16:18:53 -0400 Subject: [PATCH 0938/2680] thunderbolt: apple: tear down PCIe-C when its tunnel goes away MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only an NHI shutdown quiesced the tunneled PCI hierarchy. A tunnel torn down while the router stays up left the endpoint enumerated with its data path gone, so every access to it blocked until the function driver's own timeout fired. For NVMe that is around a minute of hung I/O ending in nvme nvme1: controller is down; CSTS=0xffffffff, PCI_STATUS=0xffff nvme nvme1: Disabling device after reset failure: -19 and the device never came back when the tunnel did, because nothing had removed it for the host to enumerate again. Quiesce PCIe-C from the new deactivate callback, and restore it when the tunnel is activated again. Restoring restarts the ports the way resume does and rescans the bus. Quiescing now removes the hierarchy rather than only stopping it. Stopping unbinds the drivers but leaves the pci_dev objects behind, so a later rescan finds stale devices: the endpoint returns with an unbalanced runtime-PM count and its driver fails to probe. [ryan: adapted to 7.2.10 — split by topic: the PCIe controller hunks are carried by 080-radio] Signed-off-by: DjDeveloperr --- drivers/thunderbolt/apple.c | 59 ++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 5ee113ad9a79a..215b4adc55c02 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -560,6 +560,8 @@ static void apple_nhi_ring_configure(struct tb_ring *ring, u32 flags, u32 e2e_fl writel(flags | e2e_flags, options); } +static struct platform_device *apple_cio_find_pcie_tunnel(struct apple_cio *acio); + static int apple_cio_populate_pcie_tunnel(struct apple_cio *acio) { /* @@ -588,8 +590,27 @@ static int apple_cio_activate_pcie_tunnel_locked(struct apple_cio *acio) lockdep_assert_held(&acio->pcie_tunnel_lock); - if (!acio->pcie_tunnel_np || acio->pcie_tunnel_populated) + if (!acio->pcie_tunnel_np) return 0; + + /* + * The host survives a tunnel teardown, quiesced with its hierarchy + * removed. Bring that one back rather than populating a second. + */ + if (acio->pcie_tunnel_populated) { + struct platform_device *pcie_pdev; + + pcie_pdev = apple_cio_find_pcie_tunnel(acio); + if (pcie_pdev) { + ret = apple_pcie_tunnel_restore(&pcie_pdev->dev); + if (ret) + dev_warn(acio->dev, + "failed to restore PCIe-C after tunnel activation: %d\n", + ret); + put_device(&pcie_pdev->dev); + } + return 0; + } if (!acio->pcie_tunnel_preinitialized) return dev_err_probe(acio->dev, -ENODEV, "PCIe-C requires a successful m1n1 preinit handoff\n"); @@ -628,6 +649,41 @@ static void apple_cio_pcie_tunnel_work(struct work_struct *work) dev_err(acio->dev, "deferred PCIe-C activation failed: %d\n", ret); } +static int apple_nhi_pci_tunnel_deactivate(struct tb_nhi *nhi) +{ + struct apple_nhi *anhi = nhi_to_anhi(nhi); + struct apple_cio *acio = anhi->acio; + struct platform_device *pcie_pdev; + int ret; + + /* + * The tunnel is going away while the router and NHI stay up, so nothing + * else tears the tunneled PCI hierarchy down. Left alone the endpoint + * keeps its driver bound with no data path underneath it: every access + * blocks until the function driver's own timeout fires, and the host has + * nothing to enumerate onto when the tunnel comes back. + */ + mutex_lock(&acio->pcie_tunnel_lock); + if (!acio->pcie_tunnel_populated) { + mutex_unlock(&acio->pcie_tunnel_lock); + return 0; + } + + pcie_pdev = apple_cio_find_pcie_tunnel(acio); + if (pcie_pdev) { + ret = apple_pcie_tunnel_quiesce(&pcie_pdev->dev); + if (ret) + dev_warn(acio->dev, + "failed to quiesce PCIe-C on tunnel teardown: %d\n", + ret); + put_device(&pcie_pdev->dev); + } + + mutex_unlock(&acio->pcie_tunnel_lock); + + return 0; +} + static int apple_nhi_pci_tunnel_pre_activate(struct tb_nhi *nhi) { struct apple_nhi *anhi = nhi_to_anhi(nhi); @@ -664,6 +720,7 @@ static const struct tb_nhi_ops apple_nhi_ops = { .ring_configure = apple_nhi_ring_configure, .pci_tunnel_pre_activate = apple_nhi_pci_tunnel_pre_activate, .pci_tunnel_post_activate = apple_nhi_pci_tunnel_post_activate, + .pci_tunnel_deactivate = apple_nhi_pci_tunnel_deactivate, }; static const struct tb_nhi_ring_layout apple_nhi_ring_layout = { From bab97e811d85ec5524edc7dbb2045d3f94f1ec88 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 2 Sep 2026 19:13:33 -0400 Subject: [PATCH 0939/2680] thunderbolt: apple: accept cable speeds above Gen3 The cable information written to the switch only set the 20 Gbps bit when the Enter_USB4 data object reported exactly Gen3. That field is three bits wide and USB4 v2 defines Gen4 above Gen3, so a Thunderbolt 5 cable reports 4 and fails the equality test. The firmware is then told the cable cannot carry 20 Gbps and mistrains the link: thunderbolt-apple-acio 701ac0000.cio: RTKit: syslog message: high_speed_lane.c:289: Gen2/3 link error. lane=0, error=89 Those errors repeat until the port gives up. No router is ever found, so a dock on such a cable enumerates nothing at all, while a Gen3 cable on the same port works. A device that does link comes up on one lane and its PCIe tunnel fails to activate. The host supports nothing beyond Gen3, so any cable reporting Gen3 or faster can carry 20 Gbps. Compare accordingly. Signed-off-by: DjDeveloperr --- drivers/thunderbolt/apple.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 215b4adc55c02..784690e256a49 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -1232,7 +1232,7 @@ static int apple_cio_tbt_switch_set(struct typec_thunderbolt_switch_dev *sw, acio->target_cable_info = TB_VSE_CAP_APPLE_CABLE_INFO_PRESENT; if (FIELD_GET(EUDO_CABLE_TYPE_MASK, data->usb4.eudo) != EUDO_CABLE_TYPE_PASSIVE) acio->target_cable_info |= TB_VSE_CAP_APPLE_CABLE_INFO_ACTIVE_CABLE; - if (FIELD_GET(EUDO_CABLE_SPEED_MASK, data->usb4.eudo) == EUDO_CABLE_SPEED_USB4_GEN3) + if (FIELD_GET(EUDO_CABLE_SPEED_MASK, data->usb4.eudo) >= EUDO_CABLE_SPEED_USB4_GEN3) acio->target_cable_info |= TB_VSE_CAP_APPLE_CABLE_INFO_20_GBPS; if (data->orientation == TYPEC_ORIENTATION_REVERSE) acio->target_cable_info |= TB_VSE_CAP_APPLE_CABLE_INFO_ORIENTATION_REVERSE; From 4b31f5056ebcfa6e029b1f2928695dacf31e3524 Mon Sep 17 00:00:00 2001 From: Marcelo Date: Thu, 10 Sep 2026 09:37:44 +1000 Subject: [PATCH 0940/2680] usb: typec: tipd: fix partner identity use-after-free on CD321x cd321x_update_work() points desc.identity at st.partner_identity, where "st" is a stack local. typec_register_partner() stores that pointer in partner->identity rather than copying the structure, so once the work function returns the partner is left referencing a freed stack page. Any later read of the partner's PD identity then dereferences it. On a MacBook Pro (16-inch, M2 Max) reading /sys/class/typec/port0-partner/type as an unprivileged user is enough: Unable to handle kernel paging request at virtual address ffff800083a87d0c Internal error: Oops: 0000000096000007 [#1] SMP CPU: 10 UID: 1000 PID: 83017 Comm: cat pc : get_pd_product_type+0x54/0xa4 [typec] lr : type_show+0x14/0x3c [typec] get_pd_product_type() NULL-checks partner->identity and then reads id->id_header, which is the faulting access; the pointer is not NULL, just dangling. With VMAP_STACK the dead stack is unmapped, hence the vmalloc-range fault address. The driver already keeps a persistent copy in cd321x->cur_partner_identity, but only assigns it after registration. Fill it in first and register the partner against that instead. The generic tps6598x path already does the equivalent, pointing desc.identity at tps->partner_identity. Fixes: 82432bbfb9e8 ("usb: typec: tipd: Handle mode transitions for CD321x") Signed-off-by: Marcelo --- drivers/usb/typec/tipd/core.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 44e17c13c262f..c614e26273e79 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -823,8 +823,15 @@ static void cd321x_update_work(struct work_struct *work) desc.usb_pd = is_pd; desc.accessory = TYPEC_ACCESSORY_NONE; /* XXX: handle accessories */ - if (desc.usb_pd) - desc.identity = &st.partner_identity; + /* + * typec_register_partner() stores this pointer rather than + * copying what it points at, and the partner outlives this + * function, so it must not point into "st" on the stack. + */ + if (desc.usb_pd) { + cd321x->cur_partner_identity = st.partner_identity; + desc.identity = &cd321x->cur_partner_identity; + } tps->partner = typec_register_partner(tps->port, &desc); if (IS_ERR(tps->partner)) { @@ -832,10 +839,8 @@ static void cd321x_update_work(struct work_struct *work) return; } - if (desc.identity) { + if (desc.identity) typec_partner_set_identity(tps->partner); - cd321x->cur_partner_identity = st.partner_identity; - } } /* Update the TypeC MUX/PHY state */ From 6bdee6ba5db7bab3e9a397bb0de3213f352be40d Mon Sep 17 00:00:00 2001 From: Marcelo Date: Thu, 10 Sep 2026 10:03:40 +1000 Subject: [PATCH 0941/2680] usb: typec: fix sysfs_emit_at() offset underflow on an empty mode list usb_mode_show() and usb_capability_show() build a space-separated list and then overwrite the trailing space with a newline: sysfs_emit_at(buf, len - 1, "\n"); When the partner or port advertises none of USB2/USB3/USB4 the loop body never runs, len stays 0, and the call is made with at == -1: invalid sysfs_emit_at: buf:00000000ffaad2a3 at:-1 WARNING: fs/sysfs/file.c:785 at sysfs_emit_at+0x54/0xd4, CPU#1: cat/892 sysfs_emit_at() rejects the negative offset, so the attribute also reads back empty rather than as a newline like every other list attribute. Seen on a MacBook Pro (16-inch, M2 Max), where one of the Type-C partners registered by the CD321x has no USB capability bits set, by reading /sys/class/typec/port1-partner/usb_mode Return a bare newline when nothing was written. Fixes: 2140a952c4e9 ("usb: typec: Add attribute file showing the USB Modes of the partner") Signed-off-by: Marcelo --- drivers/usb/typec/class.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c index 0595e8cb83aa6..ed2c919e50287 100644 --- a/drivers/usb/typec/class.c +++ b/drivers/usb/typec/class.c @@ -753,6 +753,10 @@ usb_mode_show(struct device *dev, struct device_attribute *attr, char *buf) len += sysfs_emit_at(buf, len, "%s ", usb_modes[i]); } + /* Nothing was written, so there is no trailing separator to replace. */ + if (!len) + return sysfs_emit(buf, "\n"); + sysfs_emit_at(buf, len - 1, "\n"); return len; @@ -1542,6 +1546,10 @@ usb_capability_show(struct device *dev, struct device_attribute *attr, char *buf len += sysfs_emit_at(buf, len, "%s ", usb_modes[i]); } + /* Nothing was written, so there is no trailing separator to replace. */ + if (!len) + return sysfs_emit(buf, "\n"); + sysfs_emit_at(buf, len - 1, "\n"); return len; From 4aa5a23106d9f16fc1b46a37648fde725b818361 Mon Sep 17 00:00:00 2001 From: Marcelo Alcantara Date: Wed, 16 Sep 2026 11:33:22 +1000 Subject: [PATCH 0942/2680] mux: apple-display-crossbar: replace the T602x selector fields instead of OR-ing them apple_dpxbar_set_t602x() sets the two selector fields with dpxbar_set32(), which ORs, and the disconnect path never writes them at all, so a port stays routed to the pipeline selected before it: going from dispext2 (state 4) to dispext0 (state 0) leaves 4 in place. The DCP that asked for the port then drives nothing -- its DPTX calls time out and the firmware reports the device as not started, leaving the pipeline unusable until reboot. Signed-off-by: Marcelo Alcantara --- drivers/mux/apple-display-crossbar.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/mux/apple-display-crossbar.c b/drivers/mux/apple-display-crossbar.c index 9b17371d92c3b..d3cd6399ead66 100644 --- a/drivers/mux/apple-display-crossbar.c +++ b/drivers/mux/apple-display-crossbar.c @@ -184,8 +184,17 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) } if (enable) { - dpxbar_set32(dpxbar, T602X_REG_030, state << 20); - dpxbar_set32(dpxbar, T602X_REG_030, state << 8); + /* + * Both selector fields must be replaced, not OR-ed in: leaving a + * previous pipeline's bits set keeps the crossbar pointed at it, + * and selecting dispext0 (state 0) would change nothing at all. + * The port then never reaches the DCP that asked for it, whose + * DPTX calls time out with the firmware reporting the device as + * not started. + */ + dpxbar_mask32(dpxbar, T602X_REG_030, + GENMASK(23, 20) | GENMASK(11, 8), + (state << 20) | (state << 8)); udelay(10); dpxbar_clear32(dpxbar, T602X_FIFO_WR_N_CLK_EN, dispext_bit); From 2a70261a6884f350af197ebdc24f6ec9ca301e55 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:01:39 +1000 Subject: [PATCH 0943/2680] dt-bindings: phy: apple,atcphy: Add apple,t8140-atcphy The Type-C PHY of the T8140 (A18 Pro) differs from the T8122 PHY it is otherwise programmed like in two ways: there is no USB4/Thunderbolt controller behind it, and it carries a second eUSB2 PHY register bank, mapped separately from the usb2phy block, whose per-device defaults live in Apple's Device Tree and are copied into the node by the bootloader like the other tunables. Add a stand-alone apple,t8140-atcphy compatible, a sixth "usb2phy-reg" register region and apple,tunable-usb2phy-reg-dflt for those defaults. Both are limited to the T8140 compatible; the existing compatibles keep exactly five regions. The T8140 does not claim compatibility with apple,t8122-atcphy: a driver matching only the T8122 entry would program the PHY as USB4-capable and would not know about the eUSB2 bank. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/phy/apple,atcphy.yaml | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml index 0acac7e3ee67e..c42b9c5684296 100644 --- a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml @@ -26,8 +26,30 @@ description: > below by our bootloader m1n1 are required. Without these only USB2 operation is possible. + The T8140 variant has no USB4/Thunderbolt controller behind it and carries a + second eUSB2 PHY register bank whose per-device defaults are also provided by + the bootloader. + allOf: - $ref: /schemas/usb/usb-switch.yaml# + - if: + properties: + compatible: + contains: + const: apple,t8140-atcphy + then: + properties: + reg: + minItems: 6 + reg-names: + minItems: 6 + else: + properties: + reg: + maxItems: 5 + reg-names: + maxItems: 5 + apple,tunable-usb2phy-reg-dflt: false $defs: apple,tunable: @@ -55,22 +77,27 @@ properties: - apple,t8112-atcphy - const: apple,t8103-atcphy - const: apple,t8103-atcphy + - const: apple,t8140-atcphy reg: + minItems: 5 items: - description: Common controls for all PHYs (USB2/3/4, DisplayPort, TBT) - description: DisplayPort Alternate Mode PHY specific controls - description: Type-C PHY AXI to Apple Fabric interconnect controls - description: USB2 PHY specific controls - description: USB3 PIPE interface controls + - description: Secondary eUSB2 PHY register bank (T8140) reg-names: + minItems: 5 items: - const: core - const: lpdptx - const: axi2af - const: usb2phy - const: pipehandler + - const: usb2phy-reg "#phy-cells": const: 1 @@ -151,6 +178,13 @@ properties: Note that lane here refers to a USB RX and TX pair re-used for DisplayPort and not to an individual DisplayPort differential lane. + apple,tunable-usb2phy-reg-dflt: + $ref: "#/$defs/apple,tunable" + description: > + Per-device defaults for the secondary eUSB2 PHY register bank of the + T8140, copied from Apple's Device Tree by the bootloader. Unlike the + other tunables these are applied to the usb2phy-reg region. + required: - compatible - reg @@ -220,3 +254,44 @@ examples: }; }; }; + + - | + phy@b000000 { + compatible = "apple,t8140-atcphy"; + reg = <0x0b000000 0x4c000>, + <0x0b050000 0x4000>, + <0x08000000 0x8000>, + <0x0aa90000 0x4000>, + <0x0aa84000 0x4000>, + <0x0a300000 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler", "usb2phy-reg"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&ps_atc0_usb>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + endpoint { + remote-endpoint = <&typec0_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + + endpoint { + remote-endpoint = <&dwc3_0_ss>; + }; + }; + }; + }; From 80094c9ad77f9d9246ec002bc4eccb8534f0c203 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:03:33 +1000 Subject: [PATCH 0944/2680] dt-bindings: usb: Add the VIA Labs VL122 USB 2.0 hub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apple's J700 (MacBook Neo) has a VIA Labs VL122 two-port USB 2.0 hub permanently attached to the USB 2.0 root port of its Type-C USB controller. The hub's two downstream ports go to the high-speed pairs of the two Type-C connectors; only the rear connector also has SuperSpeed lanes, which bypass the hub. On Linux the hub enumerates as USB ID 2109:2122. Describe it with the generic USB hub binding, so that a board can wire the downstream ports to its connectors through the graph. The Type-C port controllers then find no USB role switch on their high-speed path, which is correct for a fixed hub: the controller behind it is a host regardless of what happens at the connectors. Add the schema to the T8140/J700 MAINTAINERS entry. [ryan: adapted to 7.2.10 — split by topic: the MAINTAINERS lines are carried by 900-config-tooling] Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/usb/vialab,vl122.yaml | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 Documentation/devicetree/bindings/usb/vialab,vl122.yaml diff --git a/Documentation/devicetree/bindings/usb/vialab,vl122.yaml b/Documentation/devicetree/bindings/usb/vialab,vl122.yaml new file mode 100644 index 0000000000000..86cc0999f7353 --- /dev/null +++ b/Documentation/devicetree/bindings/usb/vialab,vl122.yaml @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/usb/vialab,vl122.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: VIA Labs VL122 USB 2.0 hub controller + +maintainers: + - Ryan Murray + +description: + The VL122 is a two-port USB 2.0 hub controller. Apple's J700 has one + permanently attached to the USB 2.0 root port of a Type-C USB controller, + with the two downstream ports wired to the high-speed pairs of the two + Type-C connectors. + +allOf: + - $ref: usb-hub.yaml# + +properties: + compatible: + enum: + - usb2109,2122 + + reg: true + + ports: + $ref: /schemas/graph.yaml#/properties/ports + description: The two downstream facing ports. + + properties: + port@1: + $ref: /schemas/graph.yaml#/properties/port + description: Downstream port 1. + + port@2: + $ref: /schemas/graph.yaml#/properties/port + description: Downstream port 2. + +required: + - compatible + - reg + +unevaluatedProperties: false + +examples: + - | + usb { + #address-cells = <1>; + #size-cells = <0>; + + hub@1 { + compatible = "usb2109,2122"; + reg = <1>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + + hub_front: endpoint { + remote-endpoint = <&typec1_connector_hs>; + }; + }; + + port@2 { + reg = <2>; + + hub_rear: endpoint { + remote-endpoint = <&typec0_connector_hs>; + }; + }; + }; + }; + }; From fe3238242c900489e36c5e3845dca0effd25303e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:03:34 +1000 Subject: [PATCH 0945/2680] dt-bindings: usb: apple,dwc3: Add apple,t8140-dwc3 The T8140 (A18 Pro) has the same DWC3 controller and Apple-specific register region as the earlier SoCs, so its node is compatible with apple,t8103-dwc3. Add the SoC-specific entry to the list. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/usb/apple,dwc3.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/usb/apple,dwc3.yaml b/Documentation/devicetree/bindings/usb/apple,dwc3.yaml index f70c33f32c5d6..44e8b33576dda 100644 --- a/Documentation/devicetree/bindings/usb/apple,dwc3.yaml +++ b/Documentation/devicetree/bindings/usb/apple,dwc3.yaml @@ -24,6 +24,7 @@ properties: - apple,t6000-dwc3 - apple,t6020-dwc3 - apple,t8112-dwc3 + - apple,t8140-dwc3 - const: apple,t8103-dwc3 - const: apple,t8103-dwc3 From a3943ea8e67223d4482946d12301ac249815ae31 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:04:38 +1000 Subject: [PATCH 0946/2680] usb: dwc3: apple: Release the devres of a failed core probe The glue defers dwc3_core_probe() to the first cable event, so it runs from the USB role switch callback and not from the glue's own probe. When it fails there, the glue device stays bound, and everything that dwc3_core_probe() acquired through devres before failing stays attached to the device, the request of the core register region included. The next cable event retries dwc3_core_probe() on the same struct and its devm_ioremap_resource() fails with -EBUSY on the region that is still requested, so the port never recovers without a rebind. Open a devres group around dwc3_core_probe() and release it when the probe fails. Zero the embedded struct dwc3 as well: dwc3_core_probe() initialises its locks, work items and lists, and the retry expects the zeroed struct it was given the first time. Based on the failed-probe cleanup in Eryk Wieliczko's dwc3-apple work for the T8132 (aurora-silicon, 2026). Eryk Wieliczko's Signed-off-by is outstanding. Fixes: 0ec946d32ef7 ("usb: dwc3: Add Apple Silicon DWC3 glue layer driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/dwc3-apple.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index 33d3b467f5380..e5588a1481eae 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -171,6 +171,7 @@ static void dwc3_apple_set_ptrcap(struct dwc3_apple *appledwc, u32 mode) static int dwc3_apple_core_probe(struct dwc3_apple *appledwc) { struct dwc3_probe_data probe_data = {}; + void *group; int ret; lockdep_assert_held(&appledwc->lock); @@ -183,9 +184,30 @@ static int dwc3_apple_core_probe(struct dwc3_apple *appledwc) probe_data.skip_core_init_mode = true; probe_data.properties = DWC3_DEFAULT_PROPERTIES; + /* + * This runs from the role switch callback, so a failure leaves the glue + * device bound with everything dwc3_core_probe() acquired through devres + * still attached, the request of the core register region included. The + * next cable event retries dwc3_core_probe() on this struct and would fail + * with -EBUSY on that region. Group the devres of one attempt so that a + * failed attempt can be released. + */ + group = devres_open_group(appledwc->dev, NULL, GFP_KERNEL); + if (!group) + return -ENOMEM; + ret = dwc3_core_probe(&probe_data); - if (ret) + if (ret) { + devres_release_group(appledwc->dev, group); + /* + * dwc3_core_probe() initialised the locks, work items and lists of + * the embedded struct dwc3. The retry expects a zeroed struct, as + * on the first attempt. + */ + memset(&appledwc->dwc, 0, sizeof(appledwc->dwc)); return ret; + } + devres_remove_group(appledwc->dev, group); appledwc->state = DWC3_APPLE_NO_CABLE; return 0; From 2480209077e6f4d6de503adf62417f8cd40a47d4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:07:05 +1000 Subject: [PATCH 0947/2680] phy: apple: atc: Add the T8140 secondary eUSB2 register bank MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The T8140 (A18 Pro) Type-C PHY is programmed like the T8122 one, with three differences that need SoC data of their own: - it has a second eUSB2 register bank, mapped separately from the usb2phy block, whose per-device defaults come from the ADT through the bootloader (apple,tunable-usb2phy-reg-dflt); - there is no USB4/Thunderbolt controller behind it, so it has no CIO lane tunables and cannot enter the TBT or USB4 modes; - its ADT has no common-a tunable entries, and the bootloader may leave out the SuperSpeed tunables altogether, in which case only USB2 can be used. Describe those in struct atcphy_hw and match apple,t8140-atcphy with it. Map the "usb2phy-reg" region, require its defaults and apply them after every USB2 PHY power-on, since the power-off path asserts the PHY resets. The 5 ms interval between the reset release and starting the link is the one the T8140 bring-up used for the eUSB2 repeater behind the PHY; it has not been narrowed down. Missing SuperSpeed tunables are recorded instead of failing the probe on this SoC, and the Type-C mux then refuses every mode that would run the uncalibrated lanes, so the port stays usable at USB2. TBT and USB4 are refused on a PHY without a USB4 controller. The T8140 DWC3 core also does not complete its initialisation against a parked PIPE unless the dummy PHY behind the mux is enabled explicitly (PIPEHANDLER_DUMMY_PHY_EN). Set it whenever the PIPE is parked on the dummy backend, only on SoCs without USB4: the earlier SoCs come up and run USB2-only with the bit clear and are left as they are. The eUSB2 sequencing follows the T8140 bring-up by the original author, who saw the fixed hub enumerate and run sustained transfers with it on a J700. This patch is compile-tested only. [ryan: adapted to 7.2.10 — Asahi moved the post-tunable wake into per-SoC callbacks and limited the T8122 one to the USB3 pipehandler state; T8140 gets its own callback that keeps waking the common blocks in every mode (DisplayPort included), as on 7.1] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 162 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 153 insertions(+), 9 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index f6981b79e6e02..24b7009ead2f8 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -597,11 +597,26 @@ struct atcphy_mode_configuration { struct apple_atcphy; +/** + * struct atcphy_hw - SoC-specific PHY description + * @configure_post_tunable: Generation-specific setup after the tunables + * @gen: Register programming generation + * @aciophy_lane_mode: Lane mode register offset + * @aciophy_crossbar: Crossbar register offset + * @has_usb4: A USB4/Thunderbolt controller sits behind the PHY + * @has_usb2phy_reg: The PHY has the secondary eUSB2 register bank (T8140) + * @optional_tunables: The bootloader may leave out the common-a tunables (this + * generation has none) and the SuperSpeed tunables; USB2 + * still works without the latter + */ struct atcphy_hw { void (*configure_post_tunable)(struct apple_atcphy *, enum atcphy_mode); enum atcphy_generation gen; int aciophy_lane_mode; int aciophy_crossbar; + bool has_usb4; + bool has_usb2phy_reg; + bool optional_tunables; }; /** @@ -614,6 +629,9 @@ struct atcphy_hw { * @tunables.lane_usb3: USB3 lane-specific tunables * @tunables.lane_dp: DisplayPort lane-specific tunables * @tunables.lane_usb4: USB4 lane-specific tunables + * @tunables.usb2phy_reg_dflt: Defaults for the secondary eUSB2 register bank + * @hw: SoC-specific PHY description + * @ss_tunables: The complete SuperSpeed tunable set was supplied * @mode: Current PHY operating mode * @swap_lanes: True if lanes must be swapped due to cable orientation * @dp_link_rate: DisplayPort link rate @@ -622,11 +640,13 @@ struct atcphy_hw { * @regs.core: Core registers * @regs.axi2af: AXI to Apple Fabric interface registers * @regs.usb2phy: USB2 PHY registers + * @regs.usb2phy_reg: Secondary eUSB2 register bank (T8140) * @regs.pipehandler: USB3 PIPE interface ("pipehandler") registers * @regs.lpdptx: DisplayPort registers * @res: Resources for memory-mapped registers, used to verify that tunables aren't out of bounds * @res.core: Core register resource * @res.axi2af: AXI to Apple Fabric interface resource + * @res.usb2phy_reg: Secondary eUSB2 register bank resource * @phys: PHY instances * @phys.usb2: USB2 PHY instance * @phys.usb3: USB3 PHY instance @@ -647,9 +667,11 @@ struct apple_atcphy { struct apple_tunable *lane_usb3[2]; struct apple_tunable *lane_dp[2]; struct apple_tunable *lane_usb4[2]; + struct apple_tunable *usb2phy_reg_dflt; } tunables; const struct atcphy_hw *hw; + bool ss_tunables; enum atcphy_mode mode; int dp_link_rate; bool swap_lanes; @@ -659,6 +681,7 @@ struct apple_atcphy { void __iomem *core; void __iomem *axi2af; void __iomem *usb2phy; + void __iomem *usb2phy_reg; void __iomem *pipehandler; void __iomem *lpdptx; } regs; @@ -666,6 +689,7 @@ struct apple_atcphy { struct { struct resource *core; struct resource *axi2af; + struct resource *usb2phy_reg; } res; struct { @@ -1252,6 +1276,18 @@ static int atcphy_configure_pipehandler_usb4(struct apple_atcphy *atcphy) return 0; } +/* + * The T8140 DWC3 core does not complete its initialisation against a parked + * PIPE unless the dummy PHY behind the mux is enabled as well; selecting the + * dummy backend in PIPEHANDLER_MUX_CTRL is not enough. The earlier SoCs come + * up and run USB2-only with the bit clear, so they are left alone. + */ +static void atcphy_enable_dummy_phy(struct apple_atcphy *atcphy) +{ + set32(atcphy->regs.pipehandler + PIPEHANDLER_NONSELECTED_OVERRIDE, + PIPEHANDLER_DUMMY_PHY_EN); +} + static int atcphy_configure_pipehandler_dummy(struct apple_atcphy *atcphy) { int ret; @@ -1282,6 +1318,8 @@ static int atcphy_configure_pipehandler_dummy(struct apple_atcphy *atcphy) PIPEHANDLER_NATIVE_POWER_DOWN, FIELD_PREP(PIPEHANDLER_NATIVE_POWER_DOWN, 2)); set32(atcphy->regs.pipehandler + PIPEHANDLER_NONSELECTED_OVERRIDE, PIPEHANDLER_NATIVE_RESET); + if (!atcphy->hw->has_usb4) + atcphy_enable_dummy_phy(atcphy); return 0; } @@ -1316,6 +1354,8 @@ static void atcphy_setup_pipehandler(struct apple_atcphy *atcphy) atcphy_pipehandler_set_mux(atcphy, PIPEHANDLER_MUX_CTRL_DATA_DUMMY, PIPEHANDLER_MUX_CTRL_CLK_DUMMY); + if (!atcphy->hw->has_usb4) + atcphy_enable_dummy_phy(atcphy); } static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode mode) @@ -1862,8 +1902,23 @@ static void atcphy_usb2_power_on(struct apple_atcphy *atcphy) clear32(atcphy->regs.usb2phy + USB2PHY_MISCTUNE, USB2PHY_MISCTUNE_APBCLK_GATE_OFF); clear32(atcphy->regs.usb2phy + USB2PHY_MISCTUNE, USB2PHY_MISCTUNE_REFCLK_GATE_OFF); + /* + * Give the eUSB2 repeater behind the T8140 PHY time to settle between the + * reset release and the start of the link. The 5 ms interval is the one + * the T8140 bring-up used; it has not been narrowed down. + */ + if (atcphy->hw->has_usb2phy_reg) + fsleep(5000); + /* Enable the PHY */ writel(USB2PHY_USBCTL_RUN, atcphy->regs.usb2phy + USB2PHY_USBCTL); + + /* + * The T8140 keeps per-device defaults for its secondary eUSB2 register + * bank in the ADT. The power-off path asserts the PHY resets, so apply + * them after every power-on. + */ + apple_tunable_apply(atcphy->regs.usb2phy_reg, atcphy->tunables.usb2phy_reg_dflt); } static int atcphy_power_on(struct apple_atcphy *atcphy) @@ -1930,10 +1985,8 @@ static void atcphy_configure_post_tunable_t8103(struct apple_atcphy *atcphy, enu udelay(10); } -static void atcphy_configure_post_tunable_t8122(struct apple_atcphy *atcphy, enum atcphy_mode mode) +static void atcphy_wake_common_t8122(struct apple_atcphy *atcphy) { - if (atcphy_modes[mode].pipehandler_state != ATCPHY_PIPEHANDLER_STATE_USB3) - return; core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL); udelay(10); core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL_OV); @@ -1950,6 +2003,23 @@ static void atcphy_configure_post_tunable_t8122(struct apple_atcphy *atcphy, enu udelay(10); } +static void atcphy_configure_post_tunable_t8122(struct apple_atcphy *atcphy, enum atcphy_mode mode) +{ + if (atcphy_modes[mode].pipehandler_state != ATCPHY_PIPEHANDLER_STATE_USB3) + return; + atcphy_wake_common_t8122(atcphy); +} + +/* + * T8140 shares the T8122 register generation but has no USB4. The common + * blocks are woken after the tunables in every mode, DisplayPort included, + * as this PHY was brought up and qualified. + */ +static void atcphy_configure_post_tunable_t8140(struct apple_atcphy *atcphy, enum atcphy_mode mode) +{ + atcphy_wake_common_t8122(atcphy); +} + static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) { int ret = 0; @@ -2362,6 +2432,24 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta if (atcphy->mode == target_mode) return 0; + switch (target_mode) { + case APPLE_ATCPHY_MODE_TBT: + case APPLE_ATCPHY_MODE_USB4: + if (!atcphy->hw->has_usb4) + return -EOPNOTSUPP; + fallthrough; + case APPLE_ATCPHY_MODE_USB3: + case APPLE_ATCPHY_MODE_USB3_DP: + case APPLE_ATCPHY_MODE_DP: + /* Without the lane tunables the SuperSpeed lanes are not calibrated */ + if (!atcphy->ss_tunables) + return -EOPNOTSUPP; + break; + case APPLE_ATCPHY_MODE_OFF: + case APPLE_ATCPHY_MODE_USB2: + break; + } + /* * If the pipehandler is still/already up here there's a bug somewhere so make sure to * complain loudly. We can still try to switch modes and hope for the best though, @@ -2395,6 +2483,7 @@ static int atcphy_probe_mux(struct apple_atcphy *atcphy) static int atcphy_load_tunables(struct apple_atcphy *atcphy) { size_t tunable_count; + bool ss_missing = false; struct { const char *dt_name; struct apple_tunable **tunable; @@ -2418,15 +2507,47 @@ static int atcphy_load_tunables(struct apple_atcphy *atcphy) tunable_count = ARRAY_SIZE(tunables); for (size_t i = 0; i < tunable_count; i++) { - *tunables[i].tunable = devm_apple_tunable_parse( - atcphy->dev, atcphy->np, tunables[i].dt_name, tunables[i].res); - if (IS_ERR(*tunables[i].tunable)) { - dev_err(atcphy->dev, "Failed to read tunable %s: %ld\n", - tunables[i].dt_name, PTR_ERR(*tunables[i].tunable)); - return PTR_ERR(*tunables[i].tunable); + struct apple_tunable *tunable; + + if (!atcphy->hw->has_usb4 && + (tunables[i].tunable == &atcphy->tunables.lane_usb4[0] || + tunables[i].tunable == &atcphy->tunables.lane_usb4[1])) { + *tunables[i].tunable = NULL; + continue; } + + tunable = devm_apple_tunable_parse(atcphy->dev, atcphy->np, tunables[i].dt_name, + tunables[i].res); + if (IS_ERR(tunable)) { + if (PTR_ERR(tunable) != -ENOENT || !atcphy->hw->optional_tunables) { + dev_err(atcphy->dev, "Failed to read tunable %s: %ld\n", + tunables[i].dt_name, PTR_ERR(tunable)); + return PTR_ERR(tunable); + } + /* The common-a tunables do not exist on this generation */ + if (tunables[i].tunable != &atcphy->tunables.common[0]) + ss_missing = true; + tunable = NULL; + } + *tunables[i].tunable = tunable; + } + + if (atcphy->hw->has_usb2phy_reg) { + struct apple_tunable *tunable; + + tunable = devm_apple_tunable_parse(atcphy->dev, atcphy->np, + "apple,tunable-usb2phy-reg-dflt", + atcphy->res.usb2phy_reg); + if (IS_ERR(tunable)) + return dev_err_probe(atcphy->dev, PTR_ERR(tunable), + "Failed to read tunable apple,tunable-usb2phy-reg-dflt\n"); + atcphy->tunables.usb2phy_reg_dflt = tunable; } + atcphy->ss_tunables = !ss_missing; + if (ss_missing) + dev_warn(atcphy->dev, "SuperSpeed tunables missing, USB2 only\n"); + return 0; } @@ -2458,6 +2579,17 @@ static int atcphy_map_resources(struct platform_device *pdev, struct apple_atcph *resources[i].res = res; } + if (atcphy->hw->has_usb2phy_reg) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "usb2phy-reg"); + addr = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(addr)) + return dev_err_probe(atcphy->dev, PTR_ERR(addr), + "Unable to map usb2phy-reg regs"); + + atcphy->regs.usb2phy_reg = addr; + atcphy->res.usb2phy_reg = res; + } + return 0; } @@ -2528,6 +2660,7 @@ static const struct atcphy_hw atcphy_hw_t8103 = { .aciophy_lane_mode = ACIOPHY_LANE_MODE_T8103, .aciophy_crossbar = ACIOPHY_CROSSBAR_T8103, .configure_post_tunable = atcphy_configure_post_tunable_t8103, + .has_usb4 = true, }; static const struct atcphy_hw atcphy_hw_t8122 = { @@ -2535,11 +2668,22 @@ static const struct atcphy_hw atcphy_hw_t8122 = { .aciophy_lane_mode = ACIOPHY_LANE_MODE_T8122, .aciophy_crossbar = ACIOPHY_CROSSBAR_T8122, .configure_post_tunable = atcphy_configure_post_tunable_t8122, + .has_usb4 = true, +}; + +static const struct atcphy_hw atcphy_hw_t8140 = { + .configure_post_tunable = atcphy_configure_post_tunable_t8140, + .gen = ATCPHY_GENERATION_T8122, + .aciophy_lane_mode = ACIOPHY_LANE_MODE_T8122, + .aciophy_crossbar = ACIOPHY_CROSSBAR_T8122, + .has_usb2phy_reg = true, + .optional_tunables = true, }; static const struct of_device_id atcphy_match[] = { { .compatible = "apple,t8103-atcphy", .data = &atcphy_hw_t8103 }, { .compatible = "apple,t8122-atcphy", .data = &atcphy_hw_t8122 }, + { .compatible = "apple,t8140-atcphy", .data = &atcphy_hw_t8140 }, {}, }; MODULE_DEVICE_TABLE(of, atcphy_match); From 3d2a81fe0d631f80a0892f775ec7ebf41292105e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:09:47 +1000 Subject: [PATCH 0948/2680] phy: apple: atc: Track the PIPE mux state The driver keeps a single flag, pipehandler_up, to remember that the PIPE mux has been routed to the USB3 or USB4 backend. That is enough to configure the PIPE once per mode, but it cannot tell which backend the mux is on, and it makes the dummy state a special case: a set_mode call in a USB2 mode always re-runs the dummy sequence, including its lock handshake, although the PIPE is on the dummy backend already. Record the backend the mux was last routed to instead, as the pipehandler state the mode table already uses. set_mode then returns early whenever the PIPE is already on the backend the current mode needs, which covers the repeated calls from the dwc3 glue and the xHCI core for every backend, and the USB4 configuration is still reached when the mux is parked. As before, the state is recorded even when the sequence fails, so a failed attempt is not retried on the next call. Parking the PIPE on the dummy backend is factored into one helper for the USB3 power-off and the DWC3 reset paths, which skips the sequence when the PIPE is parked already. The check in the Type-C mux callback that no mode change arrives with the PIPE routed to a live backend is kept. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 60 +++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 24b7009ead2f8..d25daf8f2d5cc 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -635,7 +635,7 @@ struct atcphy_hw { * @mode: Current PHY operating mode * @swap_lanes: True if lanes must be swapped due to cable orientation * @dp_link_rate: DisplayPort link rate - * @pipehandler_up: True if the PIPE mux ("pipehandler") is set to USB3 or USB4 mode + * @pipe_state: Backend the PIPE mux ("pipehandler") is routed to * @regs: Memory-mapped registers * @regs.core: Core registers * @regs.axi2af: AXI to Apple Fabric interface registers @@ -675,7 +675,7 @@ struct apple_atcphy { enum atcphy_mode mode; int dp_link_rate; bool swap_lanes; - bool pipehandler_up; + enum atcphy_pipehandler_state pipe_state; struct { void __iomem *core; @@ -1324,36 +1324,57 @@ static int atcphy_configure_pipehandler_dummy(struct apple_atcphy *atcphy) return 0; } +/* + * Route the PIPE to the backend the current mode needs. The state is recorded + * whether or not the sequence succeeded: the PIPE is configured once per mode, + * and a failed attempt is not retried on the next set_mode call. + */ static int atcphy_configure_pipehandler(struct apple_atcphy *atcphy, bool host) { + enum atcphy_pipehandler_state state = atcphy_modes[atcphy->mode].pipehandler_state; int ret = -EINVAL; lockdep_assert_held(&atcphy->lock); - switch (atcphy_modes[atcphy->mode].pipehandler_state) { + switch (state) { case ATCPHY_PIPEHANDLER_STATE_USB3: ret = atcphy_configure_pipehandler_usb3(atcphy, host); - atcphy->pipehandler_up = true; break; case ATCPHY_PIPEHANDLER_STATE_USB4: ret = atcphy_configure_pipehandler_usb4(atcphy); - atcphy->pipehandler_up = true; break; case ATCPHY_PIPEHANDLER_STATE_DUMMY: ret = atcphy_configure_pipehandler_dummy(atcphy); - atcphy->pipehandler_up = false; break; } + atcphy->pipe_state = state; return ret; } +/* Route the PIPE back to the dummy backend unless it is there already */ +static void atcphy_park_pipehandler(struct apple_atcphy *atcphy) +{ + int ret; + + lockdep_assert_held(&atcphy->lock); + + if (atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY) + return; + + ret = atcphy_configure_pipehandler_dummy(atcphy); + if (ret) + dev_warn(atcphy->dev, "Failed to switch PIPE to dummy: %d\n", ret); + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; +} + static void atcphy_setup_pipehandler(struct apple_atcphy *atcphy) { lockdep_assert_held(&atcphy->lock); atcphy_pipehandler_set_mux(atcphy, PIPEHANDLER_MUX_CTRL_DATA_DUMMY, PIPEHANDLER_MUX_CTRL_CLK_DUMMY); + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; if (!atcphy->hw->has_usb4) atcphy_enable_dummy_phy(atcphy); } @@ -2101,15 +2122,10 @@ static const struct phy_ops apple_atc_usb2_phy_ops = { static int atcphy_usb3_power_off(struct phy *phy) { struct apple_atcphy *atcphy = phy_get_drvdata(phy); - int ret; guard(mutex)(&atcphy->lock); - ret = atcphy_configure_pipehandler_dummy(atcphy); - if (ret) - dev_warn(atcphy->dev, "Failed to switch pipe to dummy: %d", ret); - - atcphy->pipehandler_up = false; + atcphy_park_pipehandler(atcphy); if (atcphy->mode != APPLE_ATCPHY_MODE_OFF) atcphy_configure(atcphy, APPLE_ATCPHY_MODE_OFF); @@ -2126,9 +2142,10 @@ static int atcphy_usb3_set_mode(struct phy *phy, enum phy_mode mode, int submode /* * We may get multiple calls to set_mode (for host mode e.g. at least one from the dwc3 glue * driver and then another one from the generic xhci code) but must only configure the - * PIPE handler once. + * PIPE handler once. Nothing needs to be done either when the PIPE is already routed to + * the backend the current mode uses, which includes the dummy backend of the USB2 modes. */ - if (atcphy->pipehandler_up) + if (atcphy->pipe_state == atcphy_modes[atcphy->mode].pipehandler_state) return 0; switch (mode) { @@ -2276,20 +2293,11 @@ static void _atcphy_dwc3_reset_assert(struct apple_atcphy *atcphy) static int atcphy_dwc3_reset_assert(struct reset_controller_dev *rcdev, unsigned long id) { struct apple_atcphy *atcphy = container_of(rcdev, struct apple_atcphy, rcdev); - int ret; guard(mutex)(&atcphy->lock); _atcphy_dwc3_reset_assert(atcphy); - - if (atcphy->pipehandler_up) { - ret = atcphy_configure_pipehandler_dummy(atcphy); - if (ret) - dev_warn(atcphy->dev, "Failed to switch PIPE to dummy: %d\n", ret); - else - atcphy->pipehandler_up = false; - } - + atcphy_park_pipehandler(atcphy); atcphy_usb2_power_off(atcphy); return 0; @@ -2455,7 +2463,7 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta * complain loudly. We can still try to switch modes and hope for the best though, * in the worst case the hardware will fall back to USB2-only. */ - WARN_ON_ONCE(atcphy->pipehandler_up); + WARN_ON_ONCE(atcphy->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY); return atcphy_configure(atcphy, target_mode); } @@ -2650,7 +2658,7 @@ static int atcphy_probe(struct platform_device *pdev) return ret; atcphy->mode = APPLE_ATCPHY_MODE_OFF; - atcphy->pipehandler_up = false; + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; return atcphy_probe_finalize(atcphy); } From 003d1a6af559e3fa86d6ac12821a562ae89a72c4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:11:57 +1000 Subject: [PATCH 0949/2680] phy: apple: atc: Keep USB2 up when the port sits behind a fixed hub On the J700 the USB2 pairs of the Type-C USB controller do not go to a connector: they go to a VIA VL122 hub on the board, whose downstream ports serve both Type-C connectors. The controller is a host whatever happens at the connectors, and there is no cable event that would bring the PHY out of its powered-off state before the controller starts. Today the PHY is powered off at probe and only ever powered on by a Type-C mode change, so such a controller never gets a working USB2 PHY. Detect the arrangement from the graph: the port is behind a fixed hub when the USB controller its USB3 port is linked to has an available "hub" child. For such a port: - bring the block up in USB2 mode at probe, while dwc3 is still held in reset, so that the common block, its clamps and the crossbar are set up the way a mode change sets them up; powering only the USB2 PHY later is not equivalent. The block is powered off again when a later probe step fails; - power the USB2 PHY on from the PHY's init callback: dwc3 asserts the reset that powers the USB2 PHY off before it initialises its PHYs, and nothing else would power it on again. When the whole block was powered off, because the USB3 PHY was, bring it back in the mode the Type-C mux last asked for, USB2 until a cable event says otherwise, so that a SuperSpeed device is not left at USB2 after the controller was re-initialised; - keep the block in USB2 when the connector goes to its safe state instead of powering it off underneath the running hub, which is what an unplug at the rear connector would otherwise do; - leave the common and AXI2AF tunables unapplied while it runs USB2 with the lanes off, as the T8140 bring-up did; the first SuperSpeed mode change applies the complete set. Ports whose USB2 pairs go to a connector see no change. The J700 sequence follows the original author's bring-up, which saw the hub enumerate and run sustained transfers. The graph-based detection and the rear-unplug behaviour are new and compile-tested only; a rear unplug during front-hub traffic is one of the boot checks owed. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 117 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 109 insertions(+), 8 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index d25daf8f2d5cc..62c98791d968f 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -632,6 +633,10 @@ struct atcphy_hw { * @tunables.usb2phy_reg_dflt: Defaults for the secondary eUSB2 register bank * @hw: SoC-specific PHY description * @ss_tunables: The complete SuperSpeed tunable set was supplied + * @fixed_usb2: The USB2 pairs go to a fixed hub on the USB controller, which + * stays in host mode; the PHY provides USB2 from probe on + * @typec_mode: Mode the Type-C mux last asked for; a fixed-hub port returns to + * it when the PHY is brought back up after a power-off * @mode: Current PHY operating mode * @swap_lanes: True if lanes must be swapped due to cable orientation * @dp_link_rate: DisplayPort link rate @@ -672,6 +677,8 @@ struct apple_atcphy { const struct atcphy_hw *hw; bool ss_tunables; + bool fixed_usb2; + enum atcphy_mode typec_mode; enum atcphy_mode mode; int dp_link_rate; bool swap_lanes; @@ -952,6 +959,15 @@ static void atcphy_apply_tunables(struct apple_atcphy *atcphy, enum atcphy_mode const int lane0 = atcphy->swap_lanes ? 1 : 0; const int lane1 = atcphy->swap_lanes ? 0 : 1; + /* + * A port behind a fixed hub runs USB2 with the SuperSpeed lanes off from + * probe on. The T8140 bring-up left the common and AXI2AF tunables + * unapplied in that state; they are applied together with the lane + * tunables on the first SuperSpeed mode change. + */ + if (mode == APPLE_ATCPHY_MODE_USB2 && atcphy->fixed_usb2) + return; + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.common[0]); apple_tunable_apply(atcphy->regs.axi2af, atcphy->tunables.axi2af); apple_tunable_apply(atcphy->regs.core, atcphy->tunables.common[1]); @@ -2114,8 +2130,35 @@ static int atcphy_usb2_set_mode(struct phy *phy, enum phy_mode mode, int submode return 0; } +static int atcphy_usb2_init(struct phy *phy) +{ + struct apple_atcphy *atcphy = phy_get_drvdata(phy); + + if (!atcphy->fixed_usb2) + return 0; + + guard(mutex)(&atcphy->lock); + + /* + * dwc3 initialises its PHYs after releasing its reset, and asserting that + * reset powered the USB2 PHY off. A port behind a fixed hub gets no Type-C + * event that would power it up again before the controller starts, so do + * it here. The port is a host port by construction. When the USB3 PHY was + * powered off as well the whole block is off; bring it back in the mode + * the Type-C mux last asked for, USB2 until a cable event says otherwise. + */ + set32(atcphy->regs.usb2phy + USB2PHY_SIG, USB2PHY_SIG_HOST); + if (atcphy->mode == APPLE_ATCPHY_MODE_OFF) + return atcphy_configure(atcphy, atcphy->typec_mode); + + atcphy_usb2_power_on(atcphy); + + return 0; +} + static const struct phy_ops apple_atc_usb2_phy_ops = { .owner = THIS_MODULE, + .init = atcphy_usb2_init, .set_mode = atcphy_usb2_set_mode, }; @@ -2437,6 +2480,15 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta target_mode = APPLE_ATCPHY_MODE_OFF; } + /* + * A port behind a fixed hub keeps its USB controller in host mode, and + * the controller keeps using the USB2 PHY for the hub. Keep the block in + * USB2 when the connector goes to its safe state instead of powering it + * off underneath the hub. + */ + if (atcphy->fixed_usb2 && target_mode == APPLE_ATCPHY_MODE_OFF) + target_mode = APPLE_ATCPHY_MODE_USB2; + if (atcphy->mode == target_mode) return 0; @@ -2457,6 +2509,7 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta case APPLE_ATCPHY_MODE_USB2: break; } + atcphy->typec_mode = target_mode; /* * If the pipehandler is still/already up here there's a bug somewhere so make sure to @@ -2601,6 +2654,25 @@ static int atcphy_map_resources(struct platform_device *pdev, struct apple_atcph return 0; } +/* + * The USB2 pairs of a port can go to a fixed hub on the USB controller instead + * of straight to the connector. Such a board describes the hub as a child of + * the controller our USB3 port is linked to. There is then no cable event + * that would bring the PHY up, and the controller is a host whatever happens + * at the connectors, so the PHY has to provide USB2 from probe on. + */ +static bool atcphy_usb2_behind_fixed_hub(struct apple_atcphy *atcphy) +{ + struct device_node *ep __free(device_node) = + of_graph_get_endpoint_by_regs(atcphy->np, 1, -1); + struct device_node *ctrl __free(device_node) = + ep ? of_graph_get_remote_port_parent(ep) : NULL; + struct device_node *hub __free(device_node) = + ctrl ? of_get_available_child_by_name(ctrl, "hub") : NULL; + + return hub; +} + static int atcphy_probe_finalize(struct apple_atcphy *atcphy) { int ret; @@ -2615,20 +2687,47 @@ static int atcphy_probe_finalize(struct apple_atcphy *atcphy) atcphy_power_off(atcphy); atcphy_setup_pipehandler(atcphy); + /* + * Powering only the USB2 PHY when the controller initialises its PHYs is + * not enough for a port behind a fixed hub: the common block, its clamps + * and the crossbar are set up by the mode change a cable event would + * bring. Establish the complete USB2 state now, while dwc3 is still held + * in reset. + */ + if (atcphy->fixed_usb2) { + set32(atcphy->regs.usb2phy + USB2PHY_SIG, USB2PHY_SIG_HOST); + ret = atcphy_configure(atcphy, APPLE_ATCPHY_MODE_USB2); + if (ret) + return dev_err_probe(atcphy->dev, ret, "Failed to bring up USB2\n"); + } + ret = atcphy_probe_rcdev(atcphy); - if (ret) - return dev_err_probe(atcphy->dev, ret, "Probing rcdev failed"); + if (ret) { + ret = dev_err_probe(atcphy->dev, ret, "Probing rcdev failed"); + goto power_off; + } ret = atcphy_probe_mux(atcphy); - if (ret) - return dev_err_probe(atcphy->dev, ret, "Probing mux failed"); + if (ret) { + ret = dev_err_probe(atcphy->dev, ret, "Probing mux failed"); + goto power_off; + } ret = atcphy_probe_switch(atcphy); - if (ret) - return dev_err_probe(atcphy->dev, ret, "Probing switch failed"); + if (ret) { + ret = dev_err_probe(atcphy->dev, ret, "Probing switch failed"); + goto power_off; + } ret = atcphy_probe_phy(atcphy); - if (ret) - return dev_err_probe(atcphy->dev, ret, "Probing phy failed"); + if (ret) { + ret = dev_err_probe(atcphy->dev, ret, "Probing phy failed"); + goto power_off; + } return 0; + +power_off: + if (atcphy->fixed_usb2) + atcphy_configure(atcphy, APPLE_ATCPHY_MODE_OFF); + return ret; } static int atcphy_probe(struct platform_device *pdev) @@ -2659,6 +2758,8 @@ static int atcphy_probe(struct platform_device *pdev) atcphy->mode = APPLE_ATCPHY_MODE_OFF; atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + atcphy->fixed_usb2 = atcphy_usb2_behind_fixed_hub(atcphy); + atcphy->typec_mode = APPLE_ATCPHY_MODE_USB2; return atcphy_probe_finalize(atcphy); } From 9ae42a22f0c358b33e8cc65379b42f0cdeed043c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:13:41 +1000 Subject: [PATCH 0950/2680] phy: apple: atc: Route the PIPE from the Type-C mux on a fixed-hub port The PIPE mux is normally routed from the USB3 PHY's set_mode callback, which dwc3 calls once per bring-up, after the Type-C mux has selected the mode. That works because a Type-C port controller changes the USB role on every cable event, so dwc3 is torn down and brought up again around each mode change, with the PIPE parked in between. A port behind a fixed hub has no role changes: the controller stays in host mode and dwc3 stays up. A cable at the rear connector still moves the PHY between USB2 and the SuperSpeed modes, but nothing routes the PIPE to the USB3 backend when a SuperSpeed device arrives, and nothing parks it before the lanes are reprogrammed when the device goes away. Remember from set_mode whether dwc3 is up in host mode, and clear that when the USB3 PHY is powered off or the dwc3 reset is asserted. On a fixed-hub port, the Type-C mux callback then does the PIPE work itself: it parks the PIPE before changing to a mode without a USB3 backend, and after a successful mode change it runs the host PIPE sequence when the new mode needs a different backend. Ports with a role switch keep the existing check that no mode change arrives with the PIPE routed to a live backend. The original author drove the same PIPE transitions from a role switch callback and saw the rear port train at 10 Gb/s in both orientations with the front hub in use. Driving them from the mux callback removes the dependency on the Type-C driver's call order; it is compile-tested only. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 50 ++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 62c98791d968f..9204630b0287a 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -637,6 +637,8 @@ struct atcphy_hw { * stays in host mode; the PHY provides USB2 from probe on * @typec_mode: Mode the Type-C mux last asked for; a fixed-hub port returns to * it when the PHY is brought back up after a power-off + * @host_active: dwc3 has selected host mode on the USB3 PHY; a fixed-hub port + * then routes the PIPE itself on later Type-C mode changes * @mode: Current PHY operating mode * @swap_lanes: True if lanes must be swapped due to cable orientation * @dp_link_rate: DisplayPort link rate @@ -679,6 +681,7 @@ struct apple_atcphy { bool ss_tunables; bool fixed_usb2; enum atcphy_mode typec_mode; + bool host_active; enum atcphy_mode mode; int dp_link_rate; bool swap_lanes; @@ -2169,6 +2172,7 @@ static int atcphy_usb3_power_off(struct phy *phy) guard(mutex)(&atcphy->lock); atcphy_park_pipehandler(atcphy); + atcphy->host_active = false; if (atcphy->mode != APPLE_ATCPHY_MODE_OFF) atcphy_configure(atcphy, APPLE_ATCPHY_MODE_OFF); @@ -2182,6 +2186,17 @@ static int atcphy_usb3_set_mode(struct phy *phy, enum phy_mode mode, int submode guard(mutex)(&atcphy->lock); + switch (mode) { + case PHY_MODE_USB_HOST: + atcphy->host_active = true; + break; + case PHY_MODE_USB_DEVICE: + atcphy->host_active = false; + break; + default: + return -EINVAL; + } + /* * We may get multiple calls to set_mode (for host mode e.g. at least one from the dwc3 glue * driver and then another one from the generic xhci code) but must only configure the @@ -2191,14 +2206,7 @@ static int atcphy_usb3_set_mode(struct phy *phy, enum phy_mode mode, int submode if (atcphy->pipe_state == atcphy_modes[atcphy->mode].pipehandler_state) return 0; - switch (mode) { - case PHY_MODE_USB_HOST: - return atcphy_configure_pipehandler(atcphy, true); - case PHY_MODE_USB_DEVICE: - return atcphy_configure_pipehandler(atcphy, false); - default: - return -EINVAL; - } + return atcphy_configure_pipehandler(atcphy, mode == PHY_MODE_USB_HOST); } static const struct phy_ops apple_atc_usb3_phy_ops = { @@ -2341,6 +2349,7 @@ static int atcphy_dwc3_reset_assert(struct reset_controller_dev *rcdev, unsigned _atcphy_dwc3_reset_assert(atcphy); atcphy_park_pipehandler(atcphy); + atcphy->host_active = false; atcphy_usb2_power_off(atcphy); return 0; @@ -2427,6 +2436,7 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta { struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); enum atcphy_mode target_mode; + int ret; guard(mutex)(&atcphy->lock); @@ -2511,6 +2521,30 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta } atcphy->typec_mode = target_mode; + if (atcphy->fixed_usb2) { + enum atcphy_pipehandler_state pipe_state; + + pipe_state = atcphy_modes[target_mode].pipehandler_state; + /* + * The controller of a port behind a fixed hub stays up across mode + * changes, so the PIPE has to follow the mode from here rather than + * from a later set_mode call: park it before the lanes change to a + * mode without a USB3 backend, and route it to the new backend + * afterwards while dwc3 is up in host mode. + */ + if (pipe_state != ATCPHY_PIPEHANDLER_STATE_USB3) + atcphy_park_pipehandler(atcphy); + + ret = atcphy_configure(atcphy, target_mode); + if (ret) + return ret; + + if (atcphy->host_active && atcphy->pipe_state != pipe_state) + ret = atcphy_configure_pipehandler(atcphy, true); + + return ret; + } + /* * If the pipehandler is still/already up here there's a bug somewhere so make sure to * complain loudly. We can still try to switch modes and hope for the best though, From e63f8aa9cdadd5b900501d02af703e704e262a13 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:15:32 +1000 Subject: [PATCH 0951/2680] usb: dwc3: apple: Acquire the PHYs at probe The glue sets the USB2 PHY's host or device mode before it releases the dwc3 reset, because the PHY has to be configured while it is still powered off. It does so through the handles that dwc3_core_probe() stores in struct dwc3, but the core probe is deferred to the first cable event, so on that first event the handles are still NULL and the call is a no-op: the first bring-up after boot runs with the PHY in whatever mode it was left in. The same handles are used for the USB3 PHY afterwards, and both calls ignore their return value. Look the two PHYs up in the glue's probe and use those handles. The lookup defers the glue until the PHY driver is bound, which the first cable event needed anyway, and the Type-C port controllers defer on the glue's role switch in turn, so everything is still in place before the first cable event. Warn when a set_mode call fails and continue as before: a USB3 PHY that cannot be routed leaves the port at USB2 rather than dead. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/dwc3-apple.c | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index e5588a1481eae..c063564cabcb4 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -82,6 +83,8 @@ enum dwc3_apple_state { * @mmio_resource: Resource to be passed to dwc3_core_probe * @apple_regs: Apple-specific DWC3 registers * @reset: Reset control + * @usb2_phy: USB2 PHY, configured before the core is brought up + * @usb3_phy: USB3 PHY, configured after the core is brought up * @role_sw: USB role switch * @lock: Mutex for synchronizing access * @state: Current state of the controller, see documentation for the enum for details @@ -94,6 +97,8 @@ struct dwc3_apple { void __iomem *apple_regs; struct reset_control *reset; + struct phy *usb2_phy; + struct phy *usb3_phy; struct usb_role_switch *role_sw; struct mutex lock; @@ -252,18 +257,22 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st * will sometimes only take affect after the *next* time dwc3 is brought up which causes * the connected device to just not work. * The USB3 PHY must be configured later after dwc3 has already been initialized. + * Both PHYs were looked up at probe, so this also covers the first bring-up, before + * dwc3_core_probe() has looked them up itself. */ switch (state) { case DWC3_APPLE_HOST: - phy_set_mode(appledwc->dwc.usb2_generic_phy[0], PHY_MODE_USB_HOST); + ret = phy_set_mode(appledwc->usb2_phy, PHY_MODE_USB_HOST); break; case DWC3_APPLE_DEVICE: - phy_set_mode(appledwc->dwc.usb2_generic_phy[0], PHY_MODE_USB_DEVICE); + ret = phy_set_mode(appledwc->usb2_phy, PHY_MODE_USB_DEVICE); break; default: /* Unreachable unless there's a bug in this driver */ return -EINVAL; } + if (ret) + dev_warn(appledwc->dev, "Failed to set the USB2 PHY mode, err=%d\n", ret); ret = reset_control_deassert(appledwc->reset); if (ret) { @@ -291,7 +300,9 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st * has already been configured to the correct mode earlier. */ dwc3_enable_susphy(&appledwc->dwc, true); - phy_set_mode(appledwc->dwc.usb3_generic_phy[0], PHY_MODE_USB_HOST); + ret = phy_set_mode(appledwc->usb3_phy, PHY_MODE_USB_HOST); + if (ret) + dev_warn(appledwc->dev, "USB3 PHY setup failed, USB2 only, err=%d\n", ret); ret = dwc3_host_init(&appledwc->dwc); if (ret) { dev_err(appledwc->dev, "Failed to initialize host, ret=%d\n", ret); @@ -308,7 +319,9 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st * has already been configured to the correct mode earlier. */ dwc3_enable_susphy(&appledwc->dwc, true); - phy_set_mode(appledwc->dwc.usb3_generic_phy[0], PHY_MODE_USB_DEVICE); + ret = phy_set_mode(appledwc->usb3_phy, PHY_MODE_USB_DEVICE); + if (ret) + dev_warn(appledwc->dev, "USB3 PHY setup failed, USB2 only, err=%d\n", ret); ret = dwc3_gadget_init(&appledwc->dwc); if (ret) { dev_err(appledwc->dev, "Failed to initialize gadget, ret=%d\n", ret); @@ -482,6 +495,16 @@ static int dwc3_apple_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(appledwc->reset), "Failed to get reset control\n"); + appledwc->usb2_phy = devm_phy_get(dev, "usb2-phy"); + if (IS_ERR(appledwc->usb2_phy)) + return dev_err_probe(dev, PTR_ERR(appledwc->usb2_phy), + "Failed to get the USB2 PHY\n"); + + appledwc->usb3_phy = devm_phy_get(dev, "usb3-phy"); + if (IS_ERR(appledwc->usb3_phy)) + return dev_err_probe(dev, PTR_ERR(appledwc->usb3_phy), + "Failed to get the USB3 PHY\n"); + ret = reset_control_assert(appledwc->reset); if (ret) { dev_err(&pdev->dev, "Failed to assert reset, err=%d\n", ret); From 0a686234b1e65ccbb07b9b0fffd292e75b00f462 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:17:21 +1000 Subject: [PATCH 0952/2680] usb: dwc3: apple: Keep a controller with a fixed hub in host mode On the J700 the USB2 pairs of the Type-C USB controller go to a VIA VL122 hub on the board, whose two downstream ports serve the two Type-C connectors; only the rear connector also has SuperSpeed lanes, wired straight to the PHY. The device tree describes the hub as a "hub" child of the controller node with its ports linked to the connectors, so the Type-C port controllers find no USB role switch on their high-speed path and never drive the controller's role. Such a controller is a host whatever happens at the connectors, and the PHY provides USB2 for the hub from its own probe on. Bring the controller up in host mode at probe when it has an available hub child, instead of waiting for a cable event that never comes, and make the role switch callback a no-op for it. When the host bring-up fails after the core was probed, remove the core again, so that a later attempt does not start from an already probed core. The PHY follows the rear connector's Type-C mode changes on its own while the controller stays up, so a SuperSpeed device at the rear connector runs at SuperSpeed and a rear unplug leaves the front hub alone. Controllers without a hub child see no change. The original author ran the same fixed-hub host mode, keyed on a device tree property and with role-switch inputs for the connectors, and saw the hub and its devices enumerate on a J700. This version is compile-tested only. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/dwc3-apple.c | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index c063564cabcb4..f95af33859ae9 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -68,6 +68,12 @@ * in DWC3_APPLE_PROBE_PENDING. * Once a cable is connected we then keep track of the controller mode here by transitioning to * DWC3_APPLE_HOST or DWC3_APPLE_DEVICE. + * + * Some boards do not wire the controller's USB2 pairs to a connector but to a fixed USB2 hub on + * the board, whose downstream ports serve the Type-C connectors (Apple J700). Such a controller is + * a host whatever happens at the connectors and never sees a role change: it is brought up in + * DWC3_APPLE_HOST at probe and stays there. The Type-C port controllers only drive the PHY's + * Type-C mux for the SuperSpeed lanes, and the PHY follows those mode changes on its own. */ enum dwc3_apple_state { DWC3_APPLE_PROBE_PENDING, /* Before first cable connection, dwc3_core_probe not called */ @@ -86,6 +92,7 @@ enum dwc3_apple_state { * @usb2_phy: USB2 PHY, configured before the core is brought up * @usb3_phy: USB3 PHY, configured after the core is brought up * @role_sw: USB role switch + * @fixed_hub: A fixed hub sits on the controller's USB2 port, the controller stays in host mode * @lock: Mutex for synchronizing access * @state: Current state of the controller, see documentation for the enum for details */ @@ -100,6 +107,7 @@ struct dwc3_apple { struct phy *usb2_phy; struct phy *usb3_phy; struct usb_role_switch *role_sw; + bool fixed_hub; struct mutex lock; @@ -402,6 +410,10 @@ static int dwc3_usb_role_switch_set(struct usb_role_switch *sw, enum usb_role ro guard(mutex)(&appledwc->lock); + /* A controller behind a fixed hub is a host regardless of the connectors */ + if (appledwc->fixed_hub) + return 0; + /* * Skip role switches if appledwc is already in the desired state. The * USB-C port controller on M2 and M1/M2 Pro/Max/Ultra devices issues @@ -477,6 +489,14 @@ static int dwc3_apple_setup_role_switch(struct dwc3_apple *appledwc) return 0; } +static bool dwc3_apple_has_fixed_hub(struct device *dev) +{ + struct device_node *hub __free(device_node) = + of_get_available_child_by_name(dev->of_node, "hub"); + + return hub; +} + static int dwc3_apple_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -488,6 +508,7 @@ static int dwc3_apple_probe(struct platform_device *pdev) return -ENOMEM; appledwc->dev = &pdev->dev; + appledwc->fixed_hub = dwc3_apple_has_fixed_hub(dev); mutex_init(&appledwc->lock); appledwc->reset = devm_reset_control_get_exclusive(dev, NULL); @@ -534,6 +555,24 @@ static int dwc3_apple_probe(struct platform_device *pdev) if (ret) return dev_err_probe(&pdev->dev, ret, "Failed to setup role switch\n"); + if (!appledwc->fixed_hub) + return 0; + + /* + * The hub is the controller's permanent USB2 peer, and the PHY has kept + * its USB2 path up for it since its own probe, so there is nothing to + * wait for: bring the host up now. + */ + scoped_guard(mutex, &appledwc->lock) + ret = dwc3_apple_init(appledwc, DWC3_APPLE_HOST); + if (ret) { + /* dwc3_apple_init() has already undone the core initialisation */ + if (appledwc->state != DWC3_APPLE_PROBE_PENDING) + dwc3_core_remove(&appledwc->dwc); + usb_role_switch_unregister(appledwc->role_sw); + return dev_err_probe(dev, ret, "Failed to start host mode\n"); + } + return 0; } From bd8134643699b196c277f0331bae5d27e098c50b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:19:11 +1000 Subject: [PATCH 0953/2680] usb: dwc3: apple: Split the core start and the role setup out of init dwc3_apple_init() does three things in a row: it configures the USB2 PHY, releases the reset and starts the core; it selects the role in the core and routes the USB3 PHY; and it starts xhci or the gadget. The first two are also what a system resume of a controller that stays in host mode across sleep needs, without the third, so split them into dwc3_apple_core_start() and dwc3_apple_set_role(). No functional change. The error paths keep their order: a core start failure asserts the reset, a host or gadget start failure exits the core first. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/dwc3-apple.c | 74 ++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index f95af33859ae9..3d8465f451643 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -253,7 +253,11 @@ static int dwc3_apple_core_init(struct dwc3_apple *appledwc) return ret; } -static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state state) +/* + * Bring the core up for the given role: configure the USB2 PHY, release the reset and probe + * (on the first call) or initialise the core. + */ +static int dwc3_apple_core_start(struct dwc3_apple *appledwc, enum dwc3_apple_state state) { int ret, ret_reset; @@ -289,8 +293,12 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st } ret = dwc3_apple_core_init(appledwc); - if (ret) - goto reset_assert; + if (ret) { + ret_reset = reset_control_assert(appledwc->reset); + if (ret_reset) + dev_warn(appledwc->dev, "Failed to assert reset, err=%d\n", ret_reset); + return ret; + } /* * Now that the core is initialized and already went through dwc3_core_soft_reset we can @@ -298,38 +306,51 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st */ dwc3_apple_setup_cio(appledwc); + return 0; +} + +/* Select the role in the core and route the USB3 PHY, before xhci or the gadget is started */ +static void dwc3_apple_set_role(struct dwc3_apple *appledwc, enum dwc3_apple_state state) +{ + bool host = state == DWC3_APPLE_HOST; + int ret; + + lockdep_assert_held(&appledwc->lock); + + appledwc->dwc.dr_mode = host ? USB_DR_MODE_HOST : USB_DR_MODE_PERIPHERAL; + dwc3_apple_set_ptrcap(appledwc, host ? DWC3_GCTL_PRTCAP_HOST : DWC3_GCTL_PRTCAP_DEVICE); + /* + * This platform requires SUSPHY to be enabled here already in order to properly + * configure the PHY and switch dwc3's PIPE interface to USB3 PHY. The USB2 PHY + * has already been configured to the correct mode earlier. + */ + dwc3_enable_susphy(&appledwc->dwc, true); + ret = phy_set_mode(appledwc->usb3_phy, host ? PHY_MODE_USB_HOST : PHY_MODE_USB_DEVICE); + if (ret) + dev_warn(appledwc->dev, "USB3 PHY setup failed, USB2 only, err=%d\n", ret); +} + +static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state state) +{ + int ret, ret_reset; + + lockdep_assert_held(&appledwc->lock); + + ret = dwc3_apple_core_start(appledwc, state); + if (ret) + return ret; + + dwc3_apple_set_role(appledwc, state); + switch (state) { case DWC3_APPLE_HOST: - appledwc->dwc.dr_mode = USB_DR_MODE_HOST; - dwc3_apple_set_ptrcap(appledwc, DWC3_GCTL_PRTCAP_HOST); - /* - * This platform requires SUSPHY to be enabled here already in order to properly - * configure the PHY and switch dwc3's PIPE interface to USB3 PHY. The USB2 PHY - * has already been configured to the correct mode earlier. - */ - dwc3_enable_susphy(&appledwc->dwc, true); - ret = phy_set_mode(appledwc->usb3_phy, PHY_MODE_USB_HOST); - if (ret) - dev_warn(appledwc->dev, "USB3 PHY setup failed, USB2 only, err=%d\n", ret); ret = dwc3_host_init(&appledwc->dwc); if (ret) { dev_err(appledwc->dev, "Failed to initialize host, ret=%d\n", ret); goto core_exit; } - break; case DWC3_APPLE_DEVICE: - appledwc->dwc.dr_mode = USB_DR_MODE_PERIPHERAL; - dwc3_apple_set_ptrcap(appledwc, DWC3_GCTL_PRTCAP_DEVICE); - /* - * This platform requires SUSPHY to be enabled here already in order to properly - * configure the PHY and switch dwc3's PIPE interface to USB3 PHY. The USB2 PHY - * has already been configured to the correct mode earlier. - */ - dwc3_enable_susphy(&appledwc->dwc, true); - ret = phy_set_mode(appledwc->usb3_phy, PHY_MODE_USB_DEVICE); - if (ret) - dev_warn(appledwc->dev, "USB3 PHY setup failed, USB2 only, err=%d\n", ret); ret = dwc3_gadget_init(&appledwc->dwc); if (ret) { dev_err(appledwc->dev, "Failed to initialize gadget, ret=%d\n", ret); @@ -356,7 +377,6 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st core_exit: dwc3_core_exit(&appledwc->dwc); -reset_assert: ret_reset = reset_control_assert(appledwc->reset); if (ret_reset) dev_warn(appledwc->dev, "Failed to assert reset, err=%d\n", ret_reset); From 490d3eacbe0835132f020ec9a105229f96973d96 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:20:41 +1000 Subject: [PATCH 0954/2680] usb: dwc3: apple: Add system sleep support for a fixed-hub controller Controllers with a Type-C role switch are marked as syscore devices while a cable is connected, since commit 3b7f11e579c4 ("usb: dwc3-apple: preserve live ports across system sleep"): the glue owns the PHY and core as one unit and tears it down on the next role change, and system PM is kept away from it in between. A controller behind a fixed hub is always up and never sees a role change, and on the J700 its core does not keep its state across s2idle. Left untouched, it comes back with every xHCI command timing out ("Host halt failed, -110") and the whole USB tree disconnects; rebinding xhci-hcd fails the same way, which places the loss in the dwc3 core. Give the fixed-hub controller sleep callbacks instead of the syscore marking, following what dwc3_suspend_common() and dwc3_resume_common() do for the host role: suspend exits the core and asserts the reset; resume configures the USB2 PHY, releases the reset, initialises the core, restores the Apple-specific registers, PRTCAP and the USB3 PHY mode. The xHCI child stays registered throughout. It suspends before the glue and resumes after it, and then resets the re-initialised controller. Adding or removing the child from inside a PM callback is not an option, since that re-enters the PM core's device list. The sleeping core gets a state of its own, DWC3_APPLE_SUSPENDED, so that a role-switch query during sleep still reports host mode and a failed resume leaves the controller in a state that remove() can tear down without registering a second xHCI device. Because the PHY brings a fixed-hub port back in the mode the Type-C mux last selected, a SuperSpeed device that stayed plugged in at the rear connector is routed at SuperSpeed again after resume. No wakeup capability is declared: the AIC driver implements no irq_set_wake, so no interrupt on this SoC can be armed to end s2idle, and a wakeup path through a USB device cannot work yet. Once it can, the wake IRQ belongs on the xHCI device (dev_pm_set_wake_irq()) and a keep-powered branch like dwc3_suspend_common()'s belongs here. The observations above and the same exit-and-initialise sequence come from the J700 bring-up that preceded this series; this version is compile-tested only, and s2idle with a SuperSpeed device at the rear connector is one of the boot checks owed. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/dwc3-apple.c | 94 +++++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 3 deletions(-) diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index 3d8465f451643..1f86350a11dfc 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include "glue.h" @@ -74,12 +75,15 @@ * a host whatever happens at the connectors and never sees a role change: it is brought up in * DWC3_APPLE_HOST at probe and stays there. The Type-C port controllers only drive the PHY's * Type-C mux for the SuperSpeed lanes, and the PHY follows those mode changes on its own. + * Across system sleep such a controller goes through DWC3_APPLE_SUSPENDED: the core is exited + * and initialised again around the sleep while xhci stays registered, see dwc3_apple_suspend(). */ enum dwc3_apple_state { DWC3_APPLE_PROBE_PENDING, /* Before first cable connection, dwc3_core_probe not called */ DWC3_APPLE_NO_CABLE, /* No cable connected, dwc3 suspended after dwc3_core_exit */ DWC3_APPLE_HOST, /* Cable connected, dwc3 in host mode */ DWC3_APPLE_DEVICE, /* Cable connected, dwc3 in device mode */ + DWC3_APPLE_SUSPENDED, /* Fixed-hub host asleep: core exited, xhci still registered */ }; /** @@ -239,6 +243,7 @@ static int dwc3_apple_core_init(struct dwc3_apple *appledwc) dev_err(appledwc->dev, "Failed to probe DWC3 Core, err=%d\n", ret); break; case DWC3_APPLE_NO_CABLE: + case DWC3_APPLE_SUSPENDED: ret = dwc3_core_init(&appledwc->dwc); if (ret) dev_err(appledwc->dev, "Failed to initialize DWC3 Core, err=%d\n", ret); @@ -371,8 +376,11 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st * role change or disconnect. Do not let generic system PM independently * power-gate the DWC3 device in between: xHCI then observes a dead core on * resume and cannot restore the root hubs without a full cable teardown. + * A fixed-hub controller is never torn down by a role change and has + * sleep callbacks of its own instead, see dwc3_apple_suspend(). */ - dev_pm_syscore_device(appledwc->dev, true); + if (!appledwc->fixed_hub) + dev_pm_syscore_device(appledwc->dev, true); return 0; core_exit: @@ -396,13 +404,20 @@ static int dwc3_apple_exit(struct dwc3_apple *appledwc) /* Nothing to do if we're already off */ return 0; case DWC3_APPLE_DEVICE: - dev_pm_syscore_device(appledwc->dev, false); + if (!appledwc->fixed_hub) + dev_pm_syscore_device(appledwc->dev, false); dwc3_gadget_exit(&appledwc->dwc); break; case DWC3_APPLE_HOST: - dev_pm_syscore_device(appledwc->dev, false); + if (!appledwc->fixed_hub) + dev_pm_syscore_device(appledwc->dev, false); dwc3_host_exit(&appledwc->dwc); break; + case DWC3_APPLE_SUSPENDED: + /* The core is exited and the reset asserted already, see dwc3_apple_suspend() */ + dwc3_host_exit(&appledwc->dwc); + appledwc->state = DWC3_APPLE_NO_CABLE; + return 0; } /* @@ -481,6 +496,7 @@ static enum usb_role dwc3_usb_role_switch_get(struct usb_role_switch *sw) switch (appledwc->state) { case DWC3_APPLE_HOST: + case DWC3_APPLE_SUSPENDED: return USB_ROLE_HOST; case DWC3_APPLE_DEVICE: return USB_ROLE_DEVICE; @@ -621,12 +637,84 @@ static const struct of_device_id dwc3_apple_of_match[] = { }; MODULE_DEVICE_TABLE(of, dwc3_apple_of_match); +/* + * System sleep of a fixed-hub controller. + * + * A controller with a role switch is a syscore device while a cable is connected, see + * dwc3_apple_init(). A fixed-hub controller is always up, and its core does not keep its state + * across s2idle, so it gets what dwc3_suspend_common() and dwc3_resume_common() give a host: + * the core is exited on the way down and initialised again on the way up. The xhci child stays + * registered throughout. It suspends before this device and resumes after it, and then resets + * the re-initialised controller. Adding or removing the child from inside a PM callback is not + * an option, since that re-enters the PM core's device list. + * + * No wakeup path is offered: the interrupt controller implements no irq_set_wake, so no + * interrupt on this SoC can be armed to end s2idle, and a remote-wakeup capable USB device + * could not end it either. Once that changes, the wake IRQ belongs on the xhci device + * (dev_pm_set_wake_irq()) and a keep-powered branch like dwc3_suspend_common()'s belongs here. + */ +static int dwc3_apple_suspend(struct device *dev) +{ + struct dwc3 *dwc = dev_get_drvdata(dev); + struct dwc3_apple *appledwc; + int ret; + + if (!dwc) + return 0; + appledwc = to_dwc3_apple(dwc); + + guard(mutex)(&appledwc->lock); + + if (!appledwc->fixed_hub || appledwc->state != DWC3_APPLE_HOST) + return 0; + + /* SUSPHY has to be enabled for the PHY to power down properly, as in dwc3_apple_exit() */ + dwc3_enable_susphy(&appledwc->dwc, true); + dwc3_core_exit(&appledwc->dwc); + appledwc->state = DWC3_APPLE_SUSPENDED; + + ret = reset_control_assert(appledwc->reset); + if (ret) + dev_err(appledwc->dev, "Failed to assert reset, err=%d\n", ret); + + return ret; +} + +static int dwc3_apple_resume(struct device *dev) +{ + struct dwc3 *dwc = dev_get_drvdata(dev); + struct dwc3_apple *appledwc; + int ret; + + if (!dwc) + return 0; + appledwc = to_dwc3_apple(dwc); + + guard(mutex)(&appledwc->lock); + + if (appledwc->state != DWC3_APPLE_SUSPENDED) + return 0; + + /* A failure leaves the controller in DWC3_APPLE_SUSPENDED, which remove() can tear down */ + ret = dwc3_apple_core_start(appledwc, DWC3_APPLE_HOST); + if (ret) + return ret; + + dwc3_apple_set_role(appledwc, DWC3_APPLE_HOST); + appledwc->state = DWC3_APPLE_HOST; + + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(dwc3_apple_pm_ops, dwc3_apple_suspend, dwc3_apple_resume); + static struct platform_driver dwc3_apple_driver = { .probe = dwc3_apple_probe, .remove = dwc3_apple_remove, .driver = { .name = "dwc3-apple", .of_match_table = dwc3_apple_of_match, + .pm = pm_sleep_ptr(&dwc3_apple_pm_ops), }, }; From d1ed3de87c97c4660c1eaa64e9d1b40e4ab006b6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 05:21:56 +0000 Subject: [PATCH 0955/2680] usb: core: ignore remote wakeup on the VIA VL122 hub The MacBook Neo (J700) routes both Type-C connectors' USB 2.0 pairs through a VIA Labs VL122 two-port hub (2109:2122) on the board. The hub advertises remote wakeup in its configuration descriptor, so the core lets it autosuspend as soon as both connectors are empty, but it never drives resume signalling when a device is then attached downstream. Observed on the clean J700 kernel: with the hub runtime-suspended, the xHCI USB 2.0 root port sat at PORTSC 0x0c000e63 (U3, connected, WDE and WOE armed, no PLC or CSC) for 431 s while a dock was attached to one connector and then the other; the dock's USB 2.0 tree only enumerated once something resumed the hub from the host side (a lsusb descriptor read, or writing "on" to its power/control). The SuperSpeed branch on the other root port enumerated normally throughout. With usbcore.quirks=2109:2122:j applied at runtime and the hub re-enumerated, the core refuses to autosuspend it and downstream connects are seen immediately. Mark the hub with USB_QUIRK_IGNORE_REMOTE_WAKEUP so its wakeup capability is not trusted and it stays awake. Signed-off-by: Ryan Murray --- drivers/usb/core/quirks.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c index b5b577f0b931c..ae941ec487e44 100644 --- a/drivers/usb/core/quirks.c +++ b/drivers/usb/core/quirks.c @@ -583,6 +583,10 @@ static const struct usb_device_id usb_quirk_list[] = { /* VLI disk */ { USB_DEVICE(0x2109, 0x0711), .driver_info = USB_QUIRK_NO_LPM }, + /* VIA Labs, Inc. VL122 USB2.0 Hub: claims remote wakeup, never sends it */ + { USB_DEVICE(0x2109, 0x2122), .driver_info = + USB_QUIRK_IGNORE_REMOTE_WAKEUP }, + /* VIA Labs, Inc. USB2.0 Hub */ { USB_DEVICE(0x2109, 0x2817), .driver_info = USB_QUIRK_NO_LPM }, From 4660eb69583946a3a090af35fadc2b7c74a5fbe9 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Sun, 20 Sep 2026 10:30:18 +0200 Subject: [PATCH 0956/2680] usb: typec: tipd: prefer USB4/Thunderbolt over DP on CD321x Apple CD321x reports DP_CONNECTION together with USB4_CONNECTION on USB4 docks that also advertise DP Alt Mode (OWC Thunderbolt 5 Hub is one). cd321x_typec_update_mode() handled DP first and forced the thunderbolt switch OFF, so the Apple ACIO block never started and tunneled USB 3.x never appeared. Check USB4, then Thunderbolt, then native DP. Tested on MacBook Pro 16-inch M2 Pro (apple,j416s) with an OWC Thunderbolt 5 Hub; a Keychron Q4 on the hub USB-A port enumerates. Signed-off-by: Oliver Lukschander --- drivers/usb/typec/tipd/core.c | 112 +++++++++++++++++----------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index c614e26273e79..bb36451ba93dd 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -560,6 +560,62 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, cd321x->state.data = NULL; typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); ret = typec_mux_set(cd321x->mux, &cd321x->state); + } else if (st->data_status & CD321X_DATA_STATUS_USB4_CONNECTION) { + struct enter_usb_data eusb_data; + + if (cd321x->state.alt == NULL && cd321x->state.mode == TYPEC_MODE_USB4) + return 0; + + eusb_data.eudo = le32_to_cpu(st->usb4_status.eudo); + eusb_data.active_link_training = + !!(st->data_status & TPS_DATA_STATUS_ACTIVE_LINK_TRAIN); + + cd321x->state.alt = NULL; + cd321x->state.data = &eusb_data; + cd321x->state.mode = TYPEC_MODE_USB4; + ret = typec_mux_set(cd321x->mux, &cd321x->state); + if (ret) + goto out; + + tbt_switch_data.state = TYPEC_THUNDERBOLT_SWITCH_USB4; + tbt_switch_data.usb4 = eusb_data; + tbt_switch_data.orientation = TPS_STATUS_TO_UPSIDE_DOWN(st->status) ? + TYPEC_ORIENTATION_REVERSE : + TYPEC_ORIENTATION_NORMAL; + typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); + } else if (st->data_status & TPS_DATA_STATUS_TBT_CONNECTION) { + struct typec_thunderbolt_data tbt_data; + + if (cd321x->state.alt == cd321x->port_altmode_tbt && + cd321x->state.mode == TYPEC_TBT_MODE) + return 0; + + tbt_data.cable_mode = TBT_MODE | + TBT_SET_CABLE_SPEED(TPS_DATA_STATUS_TBT_CABLE_SPEED(st->data_status)) | + TBT_SET_CABLE_ROUNDED(TPS_DATA_STATUS_TBT_CABLE_GEN(st->data_status)); + if (st->data_status & TPS_DATA_STATUS_OPTICAL_CABLE) + tbt_data.cable_mode |= TBT_CABLE_OPTICAL; + if (st->data_status & TPS_DATA_STATUS_ACTIVE_LINK_TRAIN) + tbt_data.cable_mode |= TBT_CABLE_LINK_TRAINING; + if (st->data_status & CD321X_DATA_STATUS_ACTIVE_CABLE) + tbt_data.cable_mode |= TBT_CABLE_ACTIVE_PASSIVE; + tbt_data.device_mode = TBT_MODE | + (u32)le16_to_cpu(st->intel_vid_status.device_mode) << 16; + tbt_data.enter_vdo = + (u32)le16_to_cpu(st->intel_vid_status.enter_vdo) << 16; + cd321x->state.alt = cd321x->port_altmode_tbt; + cd321x->state.mode = TYPEC_TBT_MODE; + cd321x->state.data = &tbt_data; + ret = typec_mux_set(cd321x->mux, &cd321x->state); + if (ret) + goto out; + + tbt_switch_data.state = TYPEC_THUNDERBOLT_SWITCH_TBT; + tbt_switch_data.tbt = tbt_data; + tbt_switch_data.orientation = TPS_STATUS_TO_UPSIDE_DOWN(st->status) ? + TYPEC_ORIENTATION_REVERSE : + TYPEC_ORIENTATION_NORMAL; + typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); } else if (st->data_status & TPS_DATA_STATUS_DP_CONNECTION) { struct typec_displayport_data dp_data; unsigned long mode; @@ -610,62 +666,6 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, cd321x->dp_status = dp_data.status; cd321x->dp_conf = dp_data.conf; } - } else if (st->data_status & TPS_DATA_STATUS_TBT_CONNECTION) { - struct typec_thunderbolt_data tbt_data; - - if (cd321x->state.alt == cd321x->port_altmode_tbt && - cd321x->state.mode == TYPEC_TBT_MODE) - return 0; - - tbt_data.cable_mode = TBT_MODE | - TBT_SET_CABLE_SPEED(TPS_DATA_STATUS_TBT_CABLE_SPEED(st->data_status)) | - TBT_SET_CABLE_ROUNDED(TPS_DATA_STATUS_TBT_CABLE_GEN(st->data_status)); - if (st->data_status & TPS_DATA_STATUS_OPTICAL_CABLE) - tbt_data.cable_mode |= TBT_CABLE_OPTICAL; - if (st->data_status & TPS_DATA_STATUS_ACTIVE_LINK_TRAIN) - tbt_data.cable_mode |= TBT_CABLE_LINK_TRAINING; - if (st->data_status & CD321X_DATA_STATUS_ACTIVE_CABLE) - tbt_data.cable_mode |= TBT_CABLE_ACTIVE_PASSIVE; - tbt_data.device_mode = TBT_MODE | - (u32)le16_to_cpu(st->intel_vid_status.device_mode) << 16; - tbt_data.enter_vdo = - (u32)le16_to_cpu(st->intel_vid_status.enter_vdo) << 16; - cd321x->state.alt = cd321x->port_altmode_tbt; - cd321x->state.mode = TYPEC_TBT_MODE; - cd321x->state.data = &tbt_data; - ret = typec_mux_set(cd321x->mux, &cd321x->state); - if (ret) - goto out; - - tbt_switch_data.state = TYPEC_THUNDERBOLT_SWITCH_TBT; - tbt_switch_data.tbt = tbt_data; - tbt_switch_data.orientation = TPS_STATUS_TO_UPSIDE_DOWN(st->status) ? - TYPEC_ORIENTATION_REVERSE : - TYPEC_ORIENTATION_NORMAL; - typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); - } else if (st->data_status & CD321X_DATA_STATUS_USB4_CONNECTION) { - struct enter_usb_data eusb_data; - - if (cd321x->state.alt == NULL && cd321x->state.mode == TYPEC_MODE_USB4) - return 0; - - eusb_data.eudo = le32_to_cpu(st->usb4_status.eudo); - eusb_data.active_link_training = - !!(st->data_status & TPS_DATA_STATUS_ACTIVE_LINK_TRAIN); - - cd321x->state.alt = NULL; - cd321x->state.data = &eusb_data; - cd321x->state.mode = TYPEC_MODE_USB4; - ret = typec_mux_set(cd321x->mux, &cd321x->state); - if (ret) - goto out; - - tbt_switch_data.state = TYPEC_THUNDERBOLT_SWITCH_USB4; - tbt_switch_data.usb4 = eusb_data; - tbt_switch_data.orientation = TPS_STATUS_TO_UPSIDE_DOWN(st->status) ? - TYPEC_ORIENTATION_REVERSE : - TYPEC_ORIENTATION_NORMAL; - typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); } else { if (cd321x->state.alt == NULL && cd321x->state.mode == TYPEC_STATE_USB) return 0; From a024dfed20e51e5c012f6e1154afea94b3fa5dc6 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 21:03:28 -0400 Subject: [PATCH 0957/2680] usb: add verified M5 J813 USB3 host support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — split by topic: the device-tree hunks are carried by 000-devicetree; the config fragment hunks are carried by 900-config-tooling] [ryan: adapted to 7.2.10 — atc.c: Asahi factored the PIPE mux sequence into a different spot, so the existing helper is converted to the per-SoC field masks instead of adding a second copy; T8142 keeps waking the common blocks in every mode as on 7.1 (the T8140 post-tunable callback is renamed _all_modes for that). spmi-apple-controller.c: Asahi renamed spmi_raw_cmd buffers to buf_wr/len_wr/buf_rd/len_rd; kept their names with our spmi->hw register offsets. tipd/spmi.c: Asahi made the sleep-completion timeout non-fatal; kept ours (polled ports skip the wait, IRQ ports still fail suspend with -ETIMEDOUT) so the sn201202x path stays as on 7.1] --- .../devicetree/bindings/phy/apple,atcphy.yaml | 1 + .../devicetree/bindings/spmi/apple,spmi.yaml | 1 + .../devicetree/bindings/usb/apple,dwc3.yaml | 1 + .../bindings/usb/apple,sn201202x.yaml | 8 +- drivers/phy/apple/atc.c | 49 ++++++-- drivers/spmi/spmi-apple-controller.c | 60 ++++++--- drivers/usb/typec/tipd/core.c | 9 +- drivers/usb/typec/tipd/spmi.c | 114 ++++++++++++------ drivers/usb/typec/tipd/tps6598x.h | 3 + 9 files changed, 178 insertions(+), 68 deletions(-) diff --git a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml index c42b9c5684296..2484121870400 100644 --- a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml @@ -78,6 +78,7 @@ properties: - const: apple,t8103-atcphy - const: apple,t8103-atcphy - const: apple,t8140-atcphy + - const: apple,t8142-atcphy reg: minItems: 5 diff --git a/Documentation/devicetree/bindings/spmi/apple,spmi.yaml b/Documentation/devicetree/bindings/spmi/apple,spmi.yaml index 1d14025b44c5b..f4f701e31f65b 100644 --- a/Documentation/devicetree/bindings/spmi/apple,spmi.yaml +++ b/Documentation/devicetree/bindings/spmi/apple,spmi.yaml @@ -33,6 +33,7 @@ properties: - apple,t6000-spmi - apple,t8112-spmi - const: apple,spmi + - const: apple,t8142-spmi reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/usb/apple,dwc3.yaml b/Documentation/devicetree/bindings/usb/apple,dwc3.yaml index 44e8b33576dda..9dc0adc535bea 100644 --- a/Documentation/devicetree/bindings/usb/apple,dwc3.yaml +++ b/Documentation/devicetree/bindings/usb/apple,dwc3.yaml @@ -25,6 +25,7 @@ properties: - apple,t6020-dwc3 - apple,t8112-dwc3 - apple,t8140-dwc3 + - apple,t8142-dwc3 - const: apple,t8103-dwc3 - const: apple,t8103-dwc3 diff --git a/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml b/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml index 133a71bbedb7c..9b1a566fd1437 100644 --- a/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml +++ b/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml @@ -15,6 +15,8 @@ description: | A variant of tps6598x controller also known as Apple ACE3 present on hardware with Apple SoCs starting from the M3. + The controller can be polled when interrupt resources are not supplied. + properties: compatible: enum: @@ -43,8 +45,10 @@ properties: required: - compatible - reg - - interrupts - - interrupt-names + +dependentRequired: + interrupts: [interrupt-names] + interrupt-names: [interrupts] additionalProperties: false diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 9204630b0287a..d025fc4cac8cd 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -469,8 +469,10 @@ enum atcphy_lane_mode { #define PIPEHANDLER_OVERRIDE_VAL_PHY_STATUS BIT(4) #define PIPEHANDLER_MUX_CTRL 0x0c -#define PIPEHANDLER_MUX_CTRL_CLK GENMASK(5, 3) -#define PIPEHANDLER_MUX_CTRL_DATA GENMASK(2, 0) +#define PIPEHANDLER_MUX_CTRL_CLK_T8103 GENMASK(5, 3) +#define PIPEHANDLER_MUX_CTRL_DATA_T8103 GENMASK(2, 0) +#define PIPEHANDLER_MUX_CTRL_CLK_T8142 GENMASK(6, 4) +#define PIPEHANDLER_MUX_CTRL_DATA_T8142 GENMASK(1, 0) #define PIPEHANDLER_MUX_CTRL_CLK_OFF 0 #define PIPEHANDLER_MUX_CTRL_CLK_USB3 1 #define PIPEHANDLER_MUX_CTRL_CLK_USB4 2 @@ -605,6 +607,8 @@ struct apple_atcphy; * @aciophy_lane_mode: Lane mode register offset * @aciophy_crossbar: Crossbar register offset * @has_usb4: A USB4/Thunderbolt controller sits behind the PHY + * @pipe_mux_clk: PIPE clock selection field mask + * @pipe_mux_data: PIPE data selection field mask * @has_usb2phy_reg: The PHY has the secondary eUSB2 register bank (T8140) * @optional_tunables: The bootloader may leave out the common-a tunables (this * generation has none) and the SuperSpeed tunables; USB2 @@ -616,6 +620,8 @@ struct atcphy_hw { int aciophy_lane_mode; int aciophy_crossbar; bool has_usb4; + u32 pipe_mux_clk; + u32 pipe_mux_data; bool has_usb2phy_reg; bool optional_tunables; }; @@ -1191,14 +1197,14 @@ static void atcphy_configure_pipehandler_usb3_host_t8103(struct apple_atcphy *at static void atcphy_pipehandler_set_mux(struct apple_atcphy *atcphy, u32 data, u32 clk) { - mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_CLK, - FIELD_PREP(PIPEHANDLER_MUX_CTRL_CLK, PIPEHANDLER_MUX_CTRL_CLK_OFF)); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, atcphy->hw->pipe_mux_clk, + field_prep(atcphy->hw->pipe_mux_clk, PIPEHANDLER_MUX_CTRL_CLK_OFF)); udelay(10); - mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_DATA, - FIELD_PREP(PIPEHANDLER_MUX_CTRL_DATA, data)); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, atcphy->hw->pipe_mux_data, + field_prep(atcphy->hw->pipe_mux_data, data)); udelay(10); - mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_CLK, - FIELD_PREP(PIPEHANDLER_MUX_CTRL_CLK, clk)); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, atcphy->hw->pipe_mux_clk, + field_prep(atcphy->hw->pipe_mux_clk, clk)); udelay(10); } @@ -2051,11 +2057,11 @@ static void atcphy_configure_post_tunable_t8122(struct apple_atcphy *atcphy, enu } /* - * T8140 shares the T8122 register generation but has no USB4. The common - * blocks are woken after the tunables in every mode, DisplayPort included, - * as this PHY was brought up and qualified. + * T8140 and T8142 share the T8122 register generation. Their common blocks + * are woken after the tunables in every mode, DisplayPort included, as these + * PHYs were brought up and qualified. */ -static void atcphy_configure_post_tunable_t8140(struct apple_atcphy *atcphy, enum atcphy_mode mode) +static void atcphy_configure_post_tunable_all_modes(struct apple_atcphy *atcphy, enum atcphy_mode mode) { atcphy_wake_common_t8122(atcphy); } @@ -2803,6 +2809,8 @@ static const struct atcphy_hw atcphy_hw_t8103 = { .aciophy_lane_mode = ACIOPHY_LANE_MODE_T8103, .aciophy_crossbar = ACIOPHY_CROSSBAR_T8103, .configure_post_tunable = atcphy_configure_post_tunable_t8103, + .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, + .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, .has_usb4 = true, }; @@ -2811,21 +2819,36 @@ static const struct atcphy_hw atcphy_hw_t8122 = { .aciophy_lane_mode = ACIOPHY_LANE_MODE_T8122, .aciophy_crossbar = ACIOPHY_CROSSBAR_T8122, .configure_post_tunable = atcphy_configure_post_tunable_t8122, + .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, + .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, .has_usb4 = true, }; static const struct atcphy_hw atcphy_hw_t8140 = { - .configure_post_tunable = atcphy_configure_post_tunable_t8140, + .configure_post_tunable = atcphy_configure_post_tunable_all_modes, .gen = ATCPHY_GENERATION_T8122, .aciophy_lane_mode = ACIOPHY_LANE_MODE_T8122, .aciophy_crossbar = ACIOPHY_CROSSBAR_T8122, + .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, + .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, .has_usb2phy_reg = true, .optional_tunables = true, }; +static const struct atcphy_hw atcphy_hw_t8142 = { + .configure_post_tunable = atcphy_configure_post_tunable_all_modes, + .gen = ATCPHY_GENERATION_T8122, + .aciophy_lane_mode = ACIOPHY_LANE_MODE_T8122, + .aciophy_crossbar = ACIOPHY_CROSSBAR_T8122, + .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8142, + .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8142, + .has_usb4 = true, +}; + static const struct of_device_id atcphy_match[] = { { .compatible = "apple,t8103-atcphy", .data = &atcphy_hw_t8103 }, { .compatible = "apple,t8122-atcphy", .data = &atcphy_hw_t8122 }, + { .compatible = "apple,t8142-atcphy", .data = &atcphy_hw_t8142 }, { .compatible = "apple,t8140-atcphy", .data = &atcphy_hw_t8140 }, {}, }; diff --git a/drivers/spmi/spmi-apple-controller.c b/drivers/spmi/spmi-apple-controller.c index 2390174452ab4..172899d2bb219 100644 --- a/drivers/spmi/spmi-apple-controller.c +++ b/drivers/spmi/spmi-apple-controller.c @@ -51,8 +51,32 @@ #define REG_POLL_INTERVAL_US 10000 #define REG_POLL_TIMEOUT_US (REG_POLL_INTERVAL_US * 5) +struct apple_spmi_hw { + u32 status; + u32 command; + u32 reply; + u32 rx_empty; + bool legacy_irqs; +}; + +static const struct apple_spmi_hw apple_spmi_gen1 = { + .status = SPMI_STATUS_REG, + .command = SPMI_CMD_REG, + .reply = SPMI_RSP_REG, + .rx_empty = SPMI_RX_FIFO_EMPTY, + .legacy_irqs = true, +}; + +static const struct apple_spmi_hw apple_spmi_gen4 = { + .status = 0x200, + .command = 0x210, + .reply = 0x220, + .rx_empty = BIT(30), +}; + struct apple_spmi { void __iomem *regs; + const struct apple_spmi_hw *hw; struct mutex fifo_lock; struct completion fifo_rx; struct irq_domain *irqd; @@ -63,10 +87,6 @@ struct apple_spmi { bool prev_fail; }; -#define poll_reg(spmi, reg, val, cond) \ - readl_poll_timeout((spmi)->regs + (reg), (val), (cond), \ - REG_POLL_INTERVAL_US, REG_POLL_TIMEOUT_US) - static void apple_spmi_irq_ack_raw(struct apple_spmi *spmi, u32 irq) { u32 __iomem *reg = spmi->regs + SPMI_IRQ_ACK_BASE + (irq / 32) * 4; @@ -134,12 +154,14 @@ static int apple_spmi_wait_rx_not_empty(struct spmi_controller *ctrl) usecs_to_jiffies(REG_POLL_TIMEOUT_US)); if (!ret) ret = -ETIMEDOUT; - else if (readl(spmi->regs + SPMI_STATUS_REG) & SPMI_RX_FIFO_EMPTY) + else if (readl(spmi->regs + spmi->hw->status) & spmi->hw->rx_empty) ret = -EIO; else ret = 0; } else { - ret = poll_reg(spmi, SPMI_STATUS_REG, status, !(status & SPMI_RX_FIFO_EMPTY)); + ret = readl_poll_timeout(spmi->regs + spmi->hw->status, status, + !(status & spmi->hw->rx_empty), 10, + REG_POLL_TIMEOUT_US); } if (ret) { @@ -164,19 +186,22 @@ static int spmi_raw_cmd(struct spmi_controller *ctrl, u8 opc, u8 sid, u16 param, guard(mutex)(&spmi->fifo_lock); if (spmi->prev_fail) { + /* Do not consume a late reply as the reply to a new command. */ + if (!spmi->hw->legacy_irqs) + return -EIO; writel(SPMI_ACT_FIFO_FLUSH, spmi->regs + SPMI_ACT_REG); apple_spmi_irq_ack_raw(spmi, SPMI_IRQ_NOTIFY); spmi->prev_fail = false; } reinit_completion(&spmi->fifo_rx); - writel(spmi_cmd, spmi->regs + SPMI_CMD_REG); + writel(spmi_cmd, spmi->regs + spmi->hw->command); while (i < len_wr) { j = min_t(size_t, sizeof(spmi_cmd), len_wr - i); spmi_cmd = 0; memcpy(&spmi_cmd, buf_wr + i, j); - writel(spmi_cmd, spmi->regs + SPMI_CMD_REG); + writel(spmi_cmd, spmi->regs + spmi->hw->command); i += j; } @@ -184,24 +209,24 @@ static int spmi_raw_cmd(struct spmi_controller *ctrl, u8 opc, u8 sid, u16 param, if (ret) return ret; - reply = readl(spmi->regs + SPMI_RSP_REG); + reply = readl(spmi->regs + spmi->hw->reply); /* Read SPMI data reply */ i = 0; while (i < len_rd) { - if (readl(spmi->regs + SPMI_STATUS_REG) & SPMI_RX_FIFO_EMPTY) { + if (readl(spmi->regs + spmi->hw->status) & spmi->hw->rx_empty) { spmi->prev_fail = true; dev_err_ratelimited(&ctrl->dev, "FIFO lacks reply data, controller stuck?\n"); return -EIO; } - rsp = readl(spmi->regs + SPMI_RSP_REG); + rsp = readl(spmi->regs + spmi->hw->reply); j = min_t(size_t, sizeof(spmi_cmd), len_rd - i); memcpy(buf_rd + i, &rsp, j); i += j; } - if (!(readl(spmi->regs + SPMI_STATUS_REG) & SPMI_RX_FIFO_EMPTY)) { + if (!(readl(spmi->regs + spmi->hw->status) & spmi->hw->rx_empty)) { dev_warn(&ctrl->dev, "FIFO has extra data\n"); spmi->prev_fail = true; } @@ -443,6 +468,9 @@ static int apple_spmi_probe(struct platform_device *pdev) return -ENOMEM; spmi = spmi_controller_get_drvdata(ctrl); + spmi->hw = device_get_match_data(&pdev->dev); + if (!spmi->hw) + return -EINVAL; mutex_init(&spmi->fifo_lock); init_completion(&spmi->fifo_rx); platform_set_drvdata(pdev, spmi); @@ -461,6 +489,9 @@ static int apple_spmi_probe(struct platform_device *pdev) if (spmi->irq < 0 && spmi->irq != -ENXIO) return spmi->irq; if (spmi->irq >= 0) { + if (!spmi->hw->legacy_irqs) + return dev_err_probe(&pdev->dev, -EOPNOTSUPP, + "Generation 4 interrupt layout is not supported\n"); ret = apple_spmi_init_irq(pdev, spmi, spmi->irq); if (ret) return ret; @@ -475,8 +506,9 @@ static int apple_spmi_probe(struct platform_device *pdev) } static const struct of_device_id apple_spmi_match_table[] = { - { .compatible = "apple,t8103-spmi", }, - { .compatible = "apple,spmi", }, + { .compatible = "apple,t8142-spmi", .data = &apple_spmi_gen4 }, + { .compatible = "apple,t8103-spmi", .data = &apple_spmi_gen1 }, + { .compatible = "apple,spmi", .data = &apple_spmi_gen1 }, {} }; MODULE_DEVICE_TABLE(of, apple_spmi_match_table); diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index bb36451ba93dd..91b453b7cded1 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -28,7 +28,6 @@ /* Register offsets */ #define TPS_REG_VID 0x00 -#define TPS_REG_MODE 0x03 #define TPS_REG_CMD1 0x08 #define TPS_REG_DATA1 0x09 #define TPS_REG_VERSION 0x0F @@ -771,7 +770,7 @@ static void cd321x_update_work(struct work_struct *work) * request is rejected with -EBUSY; treating that sink as the connector * owner would tear down the display carried by the first port. */ - if (cd321x->connector_fwnode && + if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && dp_route_was_active && (!new_connected || was_disconnected || !dp_connected || !dp_hpd || dp_hpd_changed || dp_mode_changed)) { @@ -850,7 +849,8 @@ static void cd321x_update_work(struct work_struct *work) /* Launch the USB role switch */ usb_role_switch_set_role(tps->role_sw, new_role); - if (cd321x->connector_fwnode && cd321x->display_route_active && dp_hpd) + if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && + cd321x->display_route_active && dp_hpd) drm_connector_oob_hotplug_event(cd321x->connector_fwnode, connector_status_connected); /* @@ -859,7 +859,8 @@ static void cd321x_update_work(struct work_struct *work) * changing that display's HPD state. Ask the display driver to retrain the * still-connected route after ACIO and the USB role are live. */ - if (cd321x->connector_fwnode && usb4_started && !dp_connected) + if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && + usb4_started && !dp_connected) drm_connector_oob_hotplug_event(cd321x->connector_fwnode, connector_status_unknown); diff --git a/drivers/usb/typec/tipd/spmi.c b/drivers/usb/typec/tipd/spmi.c index 15976d15d9bce..2960646d1c47a 100644 --- a/drivers/usb/typec/tipd/spmi.c +++ b/drivers/usb/typec/tipd/spmi.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #include #include @@ -28,6 +29,15 @@ static int regmap_sn201202x_select_reg(struct spmi_device *sdev, u8 reg) if (err) return err; + if (sn->polled) { + int poll_err; + + poll_err = read_poll_timeout(spmi_register_read, err, + err || val == reg, 100, 100000, + false, sdev, 0, &val); + return err ?: poll_err; + } + if (!wait_for_completion_timeout(&sn->select_completion, msecs_to_jiffies(100))) return -ETIMEDOUT; @@ -161,6 +171,39 @@ static const struct of_device_id sn201202x_of_match[] = { {} }; +static int sn201202x_wake(struct sn201202x *sn) +{ + unsigned long deadline; + u8 size; + int ret; + + reinit_completion(&sn->wake_completion); + ret = spmi_command_wakeup(sn->sdev); + if (ret) + return ret; + if (!sn->polled) + return wait_for_completion_timeout(&sn->wake_completion, + msecs_to_jiffies(100)) ? 0 : -ETIMEDOUT; + + /* A sleeping ACE3 ACKs commands but returns zeros until it is awake. */ + deadline = jiffies + msecs_to_jiffies(500); + do { + ret = regmap_sn201202x_select_reg(sn->sdev, TPS_REG_MODE); + if (ret == -ETIMEDOUT) + continue; + if (ret) + return ret; + ret = spmi_register_read(sn->sdev, 0x1f, &size); + if (ret) + return ret; + if (size) + return 0; + usleep_range(1000, 2000); + } while (time_before(jiffies, deadline)); + + return -ETIMEDOUT; +} + static int sn201202x_probe(struct spmi_device *device) { const struct of_device_id *match; @@ -185,46 +228,48 @@ static int sn201202x_probe(struct spmi_device *device) tps->dev = &device->dev; tps->data = data; - tps->irq = of_irq_get_byname(device->dev.of_node, "irq"); - if (tps->irq < 0) - return tps->irq; - irq_select = of_irq_get_byname(device->dev.of_node, "select"); - if (irq_select < 0) - return irq_select; - irq_sleep = of_irq_get_byname(device->dev.of_node, "sleep"); - if (irq_sleep < 0) - return irq_sleep; - irq_wake = of_irq_get_byname(device->dev.of_node, "wake"); - if (irq_wake < 0) - return irq_wake; - init_completion(&sn->select_completion); init_completion(&sn->sleep_completion); init_completion(&sn->wake_completion); - ret = devm_request_irq(&device->dev, irq_select, sn201202x_irq, - 0, NULL, &sn->select_completion); - if (ret) - return ret; - ret = devm_request_irq(&device->dev, irq_sleep, sn201202x_irq, - 0, NULL, &sn->sleep_completion); - if (ret) - return ret; - ret = devm_request_irq(&device->dev, irq_wake, sn201202x_irq, - 0, NULL, &sn->wake_completion); - if (ret) - return ret; + sn->polled = !of_property_present(device->dev.of_node, "interrupts") && + !of_property_present(device->dev.of_node, "interrupts-extended"); + if (!sn->polled) { + tps->irq = of_irq_get_byname(device->dev.of_node, "irq"); + if (tps->irq < 0) + return tps->irq; + irq_select = of_irq_get_byname(device->dev.of_node, "select"); + if (irq_select < 0) + return irq_select; + irq_sleep = of_irq_get_byname(device->dev.of_node, "sleep"); + if (irq_sleep < 0) + return irq_sleep; + irq_wake = of_irq_get_byname(device->dev.of_node, "wake"); + if (irq_wake < 0) + return irq_wake; + + ret = devm_request_irq(&device->dev, irq_select, sn201202x_irq, + 0, NULL, &sn->select_completion); + if (ret) + return ret; + ret = devm_request_irq(&device->dev, irq_sleep, sn201202x_irq, + 0, NULL, &sn->sleep_completion); + if (ret) + return ret; + ret = devm_request_irq(&device->dev, irq_wake, sn201202x_irq, + 0, NULL, &sn->wake_completion); + if (ret) + return ret; + } spmi_device_set_drvdata(device, tps); tps->regmap = devm_regmap_init_sn201202x(device, &tps6598x_regmap_config); if (IS_ERR(tps->regmap)) return PTR_ERR(tps->regmap); - ret = spmi_command_wakeup(device); + ret = sn201202x_wake(sn); if (ret) return ret; - if (!wait_for_completion_timeout(&sn->wake_completion, msecs_to_jiffies(100))) - return -ETIMEDOUT; ret = tipd_init(tps); if (ret) @@ -247,12 +292,9 @@ static int __maybe_unused sn201202x_resume(struct device *dev) struct sn201202x *sn = tps_to_sn(tps); int err; - reinit_completion(&sn->wake_completion); - err = spmi_command_wakeup(sn->sdev); + err = sn201202x_wake(sn); if (err) return err; - if (!wait_for_completion_timeout(&sn->wake_completion, msecs_to_jiffies(100))) - return -ETIMEDOUT; return tipd_resume(tps); } @@ -269,9 +311,11 @@ static int __maybe_unused sn201202x_suspend(struct device *dev) err = spmi_command_sleep(sn->sdev); if (err) goto out_resume; - if (!wait_for_completion_timeout(&sn->sleep_completion, msecs_to_jiffies(100))) - dev_err(dev, "Waiting for sleep complete irq timed out"); - + if (!sn->polled && + !wait_for_completion_timeout(&sn->sleep_completion, msecs_to_jiffies(100))) { + err = -ETIMEDOUT; + goto out_resume; + } return 0; out_resume: diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 32f7bd5cf7093..cfbbe4e8d1d86 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -16,6 +16,8 @@ #ifndef __TPS6598X_H__ #define __TPS6598X_H__ +#define TPS_REG_MODE 0x03 + #define TPS_FIELD_GET(_mask, _reg) ((typeof(_mask))(((_reg) & (_mask)) >> __bf_shf(_mask))) /* TPS_REG_STATUS bits */ @@ -392,6 +394,7 @@ struct cd321x { struct sn201202x { struct cd321x cd; + bool polled; struct completion select_completion; struct completion sleep_completion; struct completion wake_completion; From 1f7543ee730c01c22910de1fc166a163fc8178ea Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 21:09:10 -0400 Subject: [PATCH 0958/2680] dt-bindings: thunderbolt: describe T8142 Type 7 NHI and CIO reset --- .../bindings/reset/apple,t8103-cio-reset.yaml | 6 ++++- .../thunderbolt/apple,t8103-usb4-acio.yaml | 1 + .../thunderbolt/apple,t8103-usb4-nhi.yaml | 24 ++++++++++++++++++- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml b/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml index 6f602669d2420..83f307d3fc178 100644 --- a/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml +++ b/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml @@ -19,13 +19,17 @@ description: | the PMGR MMIO region it is represented as a sub-node of the PMGR syscon with reg used as an offset into it. - In both cases the reset id is the CIO port index on the die. + On t8142, a standalone register page contains a shared request register + with alternating request and busy bits for four ports. + + In all cases the reset id is the CIO port index on the die. properties: compatible: oneOf: - enum: - apple,t8103-cio-reset + - apple,t8142-cio-reset - apple,t6000-cio-reset - items: - const: apple,t8112-cio-reset diff --git a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml index c83e00f446314..e673942941aa6 100644 --- a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml +++ b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml @@ -29,6 +29,7 @@ properties: compatible: oneOf: - const: apple,t8103-usb4-acio + - const: apple,t8142-usb4-acio - items: - enum: - apple,t6000-usb4-acio diff --git a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml index 4f95132cebf30..49b60aaa8deaf 100644 --- a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml +++ b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml @@ -21,6 +21,7 @@ properties: compatible: oneOf: - const: apple,t8103-usb4-nhi + - const: apple,t8142-usb4-nhi - items: - enum: - apple,t6000-usb4-nhi @@ -32,6 +33,7 @@ properties: - const: apple,t8103-usb4-nhi reg: + minItems: 1 items: - description: NHI registers with the transmit and receive rings - description: @@ -39,6 +41,7 @@ properties: accepted by each receive ring reg-names: + minItems: 1 items: - const: nhi - const: pdf @@ -85,7 +88,8 @@ properties: all other rings which are only used for XDomain connections to other hosts. - T6030, T6031 and T8122 use 24 entries. These are most likely one DART + T8142 uses one DART stream per ring: TX streams 0-11 and RX streams 16-27. + T6030, T6031 and T8122 also use 24 entries. These are most likely one DART stream for each of the 12 transmit and 12 receive rings. apple,thunderbolt-drom: @@ -115,6 +119,23 @@ required: - apple,thunderbolt-drom allOf: + - if: + properties: + compatible: + contains: + const: apple,t8142-usb4-nhi + then: + properties: + reg: + maxItems: 1 + reg-names: + maxItems: 1 + else: + properties: + reg: + minItems: 2 + reg-names: + minItems: 2 - if: properties: compatible: @@ -123,6 +144,7 @@ allOf: - apple,t6030-usb4-nhi - apple,t6031-usb4-nhi - apple,t8122-usb4-nhi + - apple,t8142-usb4-nhi then: properties: iommus: From 2638c88742981752bedb1b63f33dd40f8512aa9e Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 21:09:11 -0400 Subject: [PATCH 0959/2680] thunderbolt: prepare M5 Type 7 controller support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Handle the T8142 reset request layout, firmware ready state, and receive-ring PDF masks without the older external PDF aperture. Require loader tuning and matching ACIO/NHI generations. Add an opt-in build fragment; controller DT enablement and firmware startup remain separate qualification work. [ryan: adapted to 7.2.10 — split by topic: the config fragment hunks are carried by 900-config-tooling] --- drivers/reset/reset-apple-cio.c | 57 ++++++++++++++++++++++++++++----- drivers/thunderbolt/apple.c | 48 ++++++++++++++++++++++----- 2 files changed, 89 insertions(+), 16 deletions(-) diff --git a/drivers/reset/reset-apple-cio.c b/drivers/reset/reset-apple-cio.c index 39b20eadbc1e1..214a2aedce2e4 100644 --- a/drivers/reset/reset-apple-cio.c +++ b/drivers/reset/reset-apple-cio.c @@ -18,6 +18,7 @@ * t8103 0 0x84000 + 0x4000 * port BIT(0), rmw BIT(1) * t8112 0x63 0x8000 / 0x4000 BIT(0), rmw BIT(1) * t8122 0x62 0x8 BIT(slot) BIT(16 + slot) + * t8142 0x62 0x30 BIT(2 * port) BIT(2 * port + 1) * t6000 0 0x1a034 BIT(port) BIT(16 + port) * t6020 0 0xa02c BIT(port) BIT(16 + port) * t6030 0x62 0x8 BIT(slot) BIT(16 + slot) @@ -29,18 +30,18 @@ * "port = slot % 4", which the device tree expresses as one controller node * per die instead. The others index by slot directly. * - * "regmap" is an ApplePMGR::RegMap enum rather than an address. Only 0 is - * pinned down, as the PMGR block itself: t8103 resolves to 0x23b784000 and + * "regmap" is an ApplePMGR::RegMap enum rather than an address. RegMap 0 is + * the PMGR block itself: t8103 resolves to 0x23b784000 and * t6020 to the pmgr node plus 0xa02c, both of which match this driver. The - * non-zero selectors are unresolved, so t8122 and t6030 cannot be wired up + * t8122 and t6030 selectors are unresolved, so they cannot be wired up * yet even though their offset and bit layout are known. * - * t8132 and t8142 have a different shape again, indexing a per-port table out - * of a driver-private structure rather than forming an offset, and have not - * been decoded. + * On t8142, RegMap 0x62 is a separate MMIO region at 0x380780000. + * The t8132 layout has not been decoded. * - * Every implementation polls the busy bits, stores the request, then polls - * again, with a 192 ms budget. The first poll's result is discarded. + * Reference firmware polls the busy bits, stores the request, then polls + * again, with a 192 ms budget. It discards the first poll's result. The + * t8142 implementation below instead refuses to issue a request while busy. */ #include @@ -63,6 +64,11 @@ #define T6000_CIO_CTRL_INIT_REQ(port) BIT(port) #define T6000_CIO_CTRL_INIT_BUSY(port) BIT(16 + (port)) +#define T8142_CIO_CTRL 0x30 +#define T8142_CIO_CTRL_INIT_REQ(port) BIT(2 * (port)) +#define T8142_CIO_CTRL_INIT_BUSY(port) BIT(2 * (port) + 1) +#define T8142_CIO_CTRL_BUSY_MASK 0xaa + struct apple_cio_reset; struct apple_cio_reset_variant { @@ -159,6 +165,37 @@ static int t6000_cio_deassert(struct apple_cio_reset *priv, unsigned long id) APPLE_CIO_RESET_TIMEOUT_US); } +static int t8142_cio_deassert(struct apple_cio_reset *priv, unsigned long id) +{ + u32 val; + int ret; + + guard(mutex)(&priv->lock); + + /* Serialize all four ports sharing the request register. */ + ret = regmap_read_poll_timeout(priv->regmap, T8142_CIO_CTRL, val, + !(val & T8142_CIO_CTRL_BUSY_MASK), + APPLE_CIO_RESET_POLL_US, + APPLE_CIO_RESET_TIMEOUT_US); + if (ret) + return ret; + + ret = regmap_write(priv->regmap, T8142_CIO_CTRL, + T8142_CIO_CTRL_INIT_REQ(id)); + if (ret) + return ret; + + return regmap_read_poll_timeout(priv->regmap, T8142_CIO_CTRL, val, + !(val & T8142_CIO_CTRL_INIT_BUSY(id)), + APPLE_CIO_RESET_POLL_US, + APPLE_CIO_RESET_TIMEOUT_US); +} + +static const struct apple_cio_reset_variant apple_t8142_cio_reset = { + .nr_resets = 4, + .deassert = t8142_cio_deassert, +}; + static const struct apple_cio_reset_variant apple_t8103_cio_reset = { .nr_resets = 2, .deassert = t8103_cio_deassert, @@ -239,6 +276,10 @@ static int apple_cio_reset_probe(struct platform_device *pdev) } static const struct of_device_id apple_cio_reset_match[] = { + { + .compatible = "apple,t8142-cio-reset", + .data = &apple_t8142_cio_reset, + }, { .compatible = "apple,t8103-cio-reset", .data = &apple_t8103_cio_reset, diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 784690e256a49..18cb1c73cacd4 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -170,6 +170,7 @@ */ struct apple_cio { struct device *dev; + bool type7; struct device_node *np; struct device_node *pcie_tunnel_np; bool pcie_tunnel_preinitialized; @@ -550,7 +551,9 @@ static void apple_nhi_ring_configure(struct tb_ring *ring, u32 flags, u32 e2e_fl sof_eof_mask = ring->sof_mask << 16 | ring->eof_mask; writel(sof_eof_mask, options + 4); - writel(sof_eof_mask, anhi->pdf_base + ring->hop * APPLE_CIO_NHI_PDF_STRIDE); + if (anhi->pdf_base) + writel(sof_eof_mask, + anhi->pdf_base + ring->hop * APPLE_CIO_NHI_PDF_STRIDE); } /* @@ -742,6 +745,12 @@ static int apple_nhi_probe(struct platform_device *pdev) int cap_apple; int ret = 0; + if (acio->type7 != + of_device_is_compatible(pdev->dev.of_node, "apple,t8142-usb4-nhi")) { + ret = dev_err_probe(&pdev->dev, -EINVAL, "ACIO/NHI generation mismatch\n"); + goto err; + } + anhi = devm_kzalloc(&pdev->dev, sizeof(*anhi), GFP_KERNEL); if (!anhi) { ret = -ENOMEM; @@ -757,6 +766,12 @@ static int apple_nhi_probe(struct platform_device *pdev) ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(42)); if (ret) goto err; + if (acio->type7 && + of_property_count_u32_elems(anhi->np, "apple,tunable-nhi") <= 0) { + ret = dev_err_probe(anhi->dev, -EINVAL, + "Missing bootloader NHI tuning\n"); + goto err; + } res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "nhi"); anhi->nhi_base = devm_ioremap_resource(&pdev->dev, res); @@ -772,12 +787,15 @@ static int apple_nhi_probe(struct platform_device *pdev) } apple_tunable_apply(anhi->nhi_base, tunable); - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pdf"); - anhi->pdf_base = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(anhi->pdf_base)) { - ret = dev_err_probe(&pdev->dev, PTR_ERR(anhi->pdf_base), - "Unable to map PDF regs\n"); - goto err; + /* Type 7 keeps the PDF masks in the receive ring registers only. */ + if (!acio->type7) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pdf"); + anhi->pdf_base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(anhi->pdf_base)) { + ret = dev_err_probe(&pdev->dev, PTR_ERR(anhi->pdf_base), + "Unable to map PDF regs\n"); + goto err; + } } ret = apple_nhi_probe_irqs(anhi); @@ -949,6 +967,9 @@ static const struct dev_pm_ops apple_nhi_pm_ops = { }; static const struct of_device_id apple_nhi_match[] = { + { + .compatible = "apple,t8142-usb4-nhi", + }, { .compatible = "apple,t8103-usb4-nhi", }, @@ -1130,8 +1151,11 @@ static int apple_cio_start(struct apple_cio *acio) goto err_free_rtkit; } + /* Type 7 reports ready as state 1; other nonzero states are not ready. */ ret = readl_poll_timeout(acio->rc_base + APPLE_CIO_M3_STAT, state, - state & APPLE_CIO_M3_STAT_STATE, 100, 500000); + acio->type7 ? + (state & APPLE_CIO_M3_STAT_STATE) == BIT(24) : + (state & APPLE_CIO_M3_STAT_STATE), 100, 500000); if (ret < 0) { dev_err(acio->dev, "M3 firmware failed to get ready: %d\n", ret); goto err_shutdown_rtkit; @@ -1291,6 +1315,11 @@ static int apple_cio_probe(struct platform_device *pdev) init_completion(&acio->nhi_boot_completion); acio->dev = &pdev->dev; acio->np = dev->of_node; + acio->type7 = of_device_is_compatible(acio->np, "apple,t8142-usb4-acio"); + if (acio->type7 && + of_property_count_u32_elems(acio->np, "apple,tunable-rc") <= 0) + return dev_err_probe(dev, -EINVAL, "Missing bootloader RC tuning\n"); + acio->pcie_tunnel_np = of_parse_phandle(dev->of_node, "apple,pcie-tunnel", 0); if (acio->pcie_tunnel_np) { @@ -1402,6 +1431,9 @@ static const struct dev_pm_ops apple_cio_pm_ops = { }; static const struct of_device_id apple_acio_match[] = { + { + .compatible = "apple,t8142-usb4-acio", + }, { .compatible = "apple,t8103-usb4-acio", }, From 624059f02f2f0aa2d66743271067d77872b557a3 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 22 Sep 2026 23:17:56 -0700 Subject: [PATCH 0960/2680] phy: apple: atc: pixel clock for DP-over-Thunderbolt tunnels on t8103 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A DP tunnel from the host router's DP IN adapter needs a pixel clock from the ATC's AUSPLL. Type-C DP alt mode sets it up together with the lanes; a tunnel has no lanes to set up, and the PHY stays in Thunderbolt/USB4 mode. Export apple_atc_dp_tunnel_rate(). It keeps the PHY blocks awake, enables DPTX PCLK1 with a per-rate selector and programs a fixed AUSPLL descriptor, without touching the lane mux. Rate 0 stops the clock: PLL output drivers off, then an APB power-down command. The sleep/clamp overrides and TX_DP_CTRL0 are left as they are; the next start programs them again. A start is refused while PLL outputs are live or a PLL request is pending, and DP alt mode configuration is refused while the tunnel clock runs, so the two never share the PLL. The clock is stopped before any PHY mode change. Only t8103 is supported: the descriptor and selectors are for that SoC. Name the TX/RX sleep and clamp value fields next to their override fields, and add include/linux/soc/apple/dp-tunnel.h for the functions the Thunderbolt glue, appledrm, the ATC PHY and the display crossbar call on each other. They are looked up with symbol_get(), so none of these modules depends on another. Based on Oliver Lukschander's t6020 tunnel clock work. [ryan: adapted to 7.2.10 — the tunnel clock helpers go after the per-SoC post-tunable callbacks that Asahi now has in atc.c] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/phy/apple/atc.c | 206 ++++++++++++++++++++++++++++ include/linux/soc/apple/dp-tunnel.h | 43 ++++++ 2 files changed, 249 insertions(+) create mode 100644 include/linux/soc/apple/dp-tunnel.h diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index d025fc4cac8cd..dc8d008a5ae79 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -87,6 +88,9 @@ #define AUSPLL_CLKOUT_MASTER_PCLK_DRVR_EN BIT(2) #define AUSPLL_CLKOUT_MASTER_PCLK2_DRVR_EN BIT(4) #define AUSPLL_CLKOUT_MASTER_REFBUFCLK_DRVR_EN BIT(6) +#define AUSPLL_CLKOUT_MASTER_DRVR_EN \ + (AUSPLL_CLKOUT_MASTER_PCLK_DRVR_EN | AUSPLL_CLKOUT_MASTER_PCLK2_DRVR_EN | \ + AUSPLL_CLKOUT_MASTER_REFBUFCLK_DRVR_EN) #define AUSPLL_CLKOUT_DIV 0x2208 #define AUSPLL_CLKOUT_PLLA_REFBUFCLK_DI GENMASK(20, 16) @@ -141,8 +145,11 @@ #define ACIOPHY_CFG0_COMMON_SMALL_OV BIT(3) #define ACIOPHY_CFG0_COMMON_CLAMP BIT(4) #define ACIOPHY_CFG0_COMMON_CLAMP_OV BIT(5) +#define ACIOPHY_CFG0_RX_SMALL GENMASK(7, 6) #define ACIOPHY_CFG0_RX_SMALL_OV GENMASK(9, 8) +#define ACIOPHY_CFG0_RX_BIG GENMASK(11, 10) #define ACIOPHY_CFG0_RX_BIG_OV GENMASK(13, 12) +#define ACIOPHY_CFG0_RX_CLAMP GENMASK(15, 14) #define ACIOPHY_CFG0_RX_CLAMP_OV GENMASK(17, 16) #define ACIOPHY_CROSSBAR_T8103 0x4c @@ -215,8 +222,11 @@ enum atcphy_lane_mode { #define ACIOPHY_BIST_MAC_USBPHY_CFG_OV2_LN0_PWR_DOWN_OV 0x20 #define ACIOPHY_SLEEP_CTRL 0x1b0 +#define ACIOPHY_SLEEP_CTRL_TX_BIG GENMASK(1, 0) #define ACIOPHY_SLEEP_CTRL_TX_BIG_OV GENMASK(3, 2) +#define ACIOPHY_SLEEP_CTRL_TX_SMALL GENMASK(5, 4) #define ACIOPHY_SLEEP_CTRL_TX_SMALL_OV GENMASK(7, 6) +#define ACIOPHY_SLEEP_CTRL_TX_CLAMP GENMASK(9, 8) #define ACIOPHY_SLEEP_CTRL_TX_CLAMP_OV GENMASK(11, 10) #define ACIOPHY_PLL_PCTL_FSM_CTRL1 0x1014 @@ -649,6 +659,8 @@ struct atcphy_hw { * @swap_lanes: True if lanes must be swapped due to cable orientation * @dp_link_rate: DisplayPort link rate * @pipe_state: Backend the PIPE mux ("pipehandler") is routed to + * @tunnel_clock_on: True while the DisplayPort-over-Thunderbolt pixel clock runs + * @tunnel_rate: DP link rate code the tunnel pixel clock is set up for * @regs: Memory-mapped registers * @regs.core: Core registers * @regs.axi2af: AXI to Apple Fabric interface registers @@ -693,6 +705,9 @@ struct apple_atcphy { bool swap_lanes; enum atcphy_pipehandler_state pipe_state; + bool tunnel_clock_on; + u8 tunnel_rate; + struct { void __iomem *core; void __iomem *axi2af; @@ -1777,6 +1792,9 @@ static int atcphy_dp_configure(struct apple_atcphy *atcphy, enum atcphy_dp_link_ u32 reg; guard(mutex)(&atcphy->lock); + /* the AUSPLL is in use as the Thunderbolt DP tunnel pixel clock */ + if (atcphy->tunnel_clock_on) + return -EBUSY; mode_cfg = atcphy_get_mode_config(atcphy, atcphy->mode); cfg = &dp_lr_config[lr]; @@ -2066,12 +2084,170 @@ static void atcphy_configure_post_tunable_all_modes(struct apple_atcphy *atcphy, atcphy_wake_common_t8122(atcphy); } +/* + * DisplayPort over Thunderbolt/USB4: the host router's DP IN adapter needs a + * pixel clock from the ATC's AUSPLL. Ordinary Type-C DP sets it up together + * with the lanes; for a tunnel the PHY blocks are woken, DPTX PCLK1 is enabled + * with a per-rate selector and the AUSPLL gets a fixed descriptor + * ({0,0x21c,8,3,8,7,0,0,0,0,2,1,0,5,5,1,0,1,1,1}, APB commands 0 then 0x2000), + * without touching the lane mux. Based on Oliver Lukschander's t6020 tunnel + * clock patch (asahi-j416s-display). + */ +/* + * Stop the tunnel pixel clock: switch the AUSPLL output drivers off and send + * APB command 3. TX_DP_CTRL0, the sleep overrides and the + * descriptor are left as they are; the next start programs them again. + */ +static void atc_tunnel_stop(struct apple_atcphy *atcphy) +{ + lockdep_assert_held(&atcphy->lock); + if (!atcphy->tunnel_clock_on) + return; + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_DRVR_EN); + atcphy_auspll_apb_command(atcphy, 3); + atcphy->tunnel_clock_on = false; + atcphy->tunnel_rate = 0; + dev_dbg(atcphy->dev, "DP tunnel clock stopped\n"); +} + +static int atc_tunnel_start(struct apple_atcphy *atcphy, u8 rate) +{ + u32 selector, value; + int ret; + + lockdep_assert_held(&atcphy->lock); + switch (rate) { + case 0x06: /* RBR */ + selector = 4; + break; + case 0x0a: /* HBR */ + selector = 3; + break; + case 0x14: /* HBR2 */ + selector = 1; + break; + case 0x1e: /* HBR3 */ + selector = 0; + break; + default: + return -EINVAL; + } + if (atcphy->tunnel_clock_on) { + if (atcphy->tunnel_rate == rate) + return 0; + /* + * Rate change: DCP brackets it with WillChange/DidChange link + * configuration, so the crossbar is already down; reselect PCLK1. + */ + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_SELECT, + FIELD_PREP(DPTX_PCLK1_SELECT, selector)); + atcphy->tunnel_rate = rate; + return 0; + } + /* + * Don't take the PLL from another clock client or an in-flight command. + * Only live PLL outputs or a pending request mean that: the PCLK gates + * in TX_DP_CTRL0 read set when idle and AUSPLL_LOCK stays set after a + * stop, so neither says anything about another client. + */ + if (readl(atcphy->regs.core + AUSPLL_CLKOUT_MASTER) & AUSPLL_CLKOUT_MASTER_DRVR_EN || + readl(atcphy->regs.core + AUSPLL_APB_CMD_OVERRIDE) & AUSPLL_APB_CMD_OVERRIDE_REQ) + return -EBUSY; + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_CMN_SHM_STS_REG0, value, + value & ACIOPHY_CMN_SHM_STS_REG0_CMD_READY, 10, 10000); + if (ret) + return ret; + atcphy->tunnel_clock_on = true; + /* the PLL no longer holds a DP alt mode configuration */ + atcphy->dp_link_rate = -1; + + /* + * The PHY is up in USB4/TBT mode here. These overrides only force the + * already awake blocks to stay awake for the DP clock path; the order + * and the 2 us gaps are the ones this was validated with, the USB4 + * link stays up across them. + * + * Common block: override small/big sleep off and release the clamp. + */ + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL_OV); + udelay(2); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_BIG); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_BIG_OV); + udelay(2); + core_clear32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP_OV); + udelay(2); + /* same for the TX lanes */ + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_SMALL); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_SMALL_OV); + udelay(2); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_BIG); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_BIG_OV); + udelay(2); + core_clear32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_CLAMP); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_CLAMP_OV); + udelay(2); + /* and for the RX lanes */ + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG_OV); + udelay(2); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL_OV); + udelay(2); + core_clear32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP_OV); + udelay(2); + /* TX_DP_CTRL0: PMA lane reset release, PCLK1 enable + rate selector */ + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N_OV); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_SELECT, + FIELD_PREP(DPTX_PCLK1_SELECT, selector)); + + /* fixed AUSPLL descriptor */ + core_clear32(atcphy, AUSPLL_FREQ_CFG, AUSPLL_FREQ_REFCLK); + core_mask32(atcphy, AUSPLL_FREQ_DESC_A, U32_MAX, + FIELD_PREP(AUSPLL_FD_FREQ_COUNT_TARGET, 0x21c) | + FIELD_PREP(AUSPLL_FD_KI_MAN, 8) | FIELD_PREP(AUSPLL_FD_KI_EXP, 3) | + FIELD_PREP(AUSPLL_FD_KP_MAN, 8) | FIELD_PREP(AUSPLL_FD_KP_EXP, 7)); + core_clear32(atcphy, AUSPLL_FREQ_DESC_B, + AUSPLL_FD_FBDIVN_FRAC_DEN | AUSPLL_FD_FBDIVN_FRAC_NUM); + core_mask32(atcphy, AUSPLL_FREQ_DESC_C, + AUSPLL_FD_SDM_SSC_STEP | AUSPLL_FD_SDM_SSC_EN | AUSPLL_FD_PCLK_DIV_SEL | + AUSPLL_FD_LFSDM_DIV | AUSPLL_FD_LFCLK_CTRL | AUSPLL_FD_VCLK_OP_DIVN | + AUSPLL_FD_VCLK_PRE_DIVN, + FIELD_PREP(AUSPLL_FD_PCLK_DIV_SEL, 5) | FIELD_PREP(AUSPLL_FD_LFSDM_DIV, 1) | + FIELD_PREP(AUSPLL_FD_LFCLK_CTRL, 5) | FIELD_PREP(AUSPLL_FD_VCLK_OP_DIVN, 2) | + AUSPLL_FD_VCLK_PRE_DIVN); + core_mask32(atcphy, AUSPLL_CLKOUT_DIV, AUSPLL_CLKOUT_PLLA_REFBUFCLK_DI, + FIELD_PREP(AUSPLL_CLKOUT_PLLA_REFBUFCLK_DI, 1)); + core_set32(atcphy, AUSPLL_CLKOUT_DTC_VREG, AUSPLL_DTC_VREG_BYPASS); + core_set32(atcphy, AUSPLL_BGR, AUSPLL_BGR_CTRL_AVAIL); + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_PCLK_DRVR_EN); + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_PCLK2_DRVR_EN); + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_REFBUFCLK_DRVR_EN); + atcphy_auspll_apb_command(atcphy, 0); + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT, value, + value & ACIOPHY_AUSPLL_LOCK, 10, 10000); + if (ret) { + dev_err(atcphy->dev, "DP tunnel clock: AUSPLL did not lock\n"); + atc_tunnel_stop(atcphy); + return ret; + } + atcphy_auspll_apb_command(atcphy, 0x2000); + atcphy->tunnel_rate = rate; + return 0; +} + static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) { int ret = 0; u32 reg; lockdep_assert_held(&atcphy->lock); + /* the mode change below may power the PHY down: stop the tunnel clock first */ + atc_tunnel_stop(atcphy); if (mode == APPLE_ATCPHY_MODE_OFF) { ret = atcphy_power_off(atcphy); @@ -2299,6 +2475,36 @@ static const struct phy_ops apple_atc_dp_phy_ops = { .set_mode = atcphy_dpphy_set_mode, }; +/* + * Called by appledrm when DCP sets the link rate of a DPTX that feeds a + * Thunderbolt DP IN adapter (rate is the DP link rate code, 0 = stop). + */ +int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate) +{ + struct apple_atcphy *atcphy; + int ret; + + if (!phy || phy->ops != &apple_atc_dp_phy_ops) + return -EINVAL; + atcphy = phy_get_drvdata(phy); + /* the fixed PLL descriptor and PCLK1 selectors are for t8103 only */ + if (!of_machine_is_compatible("apple,t8103")) + return -EOPNOTSUPP; + guard(mutex)(&atcphy->lock); + if (!rate) { + atc_tunnel_stop(atcphy); + return 0; + } + if (atcphy->mode != APPLE_ATCPHY_MODE_USB4 && atcphy->mode != APPLE_ATCPHY_MODE_TBT) + return -EBUSY; + ret = atc_tunnel_start(atcphy, rate); + dev_dbg(atcphy->dev, "DP tunnel clock rate 0x%x: %d (TX_DP_CTRL0=%08x PCLK_STAT=%08x)\n", + rate, ret, readl(atcphy->regs.core + ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0), + readl(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT)); + return ret; +} +EXPORT_SYMBOL_GPL(apple_atc_dp_tunnel_rate); + static struct phy *atcphy_xlate(struct device *dev, const struct of_phandle_args *args) { struct apple_atcphy *atcphy = dev_get_drvdata(dev); diff --git a/include/linux/soc/apple/dp-tunnel.h b/include/linux/soc/apple/dp-tunnel.h new file mode 100644 index 0000000000000..b1bbbc3a6017d --- /dev/null +++ b/include/linux/soc/apple/dp-tunnel.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * DisplayPort over Thunderbolt on Apple silicon: the pieces that bring up a + * DP tunnel live in the Thunderbolt glue, appledrm, the ATC PHY and the + * display crossbar. They find each other with symbol_get(), so none of these + * modules has to be built or loaded for the others to work. + */ +#ifndef _LINUX_SOC_APPLE_DP_TUNNEL_H_ +#define _LINUX_SOC_APPLE_DP_TUNNEL_H_ + +#include + +struct device_node; +struct mux_control; +struct phy; + +/* + * appledrm: route a display pipeline to (active) or away from (!active) the + * crossbar output of DP IN adapter @dpin (0/1) of the router wired to + * @connector_np. @set_active(@ctx, active) runs the DP IN adapter's + * DPTX_INACTIVE handshake and is called back from DCP's link activation. + */ +int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, + bool active, int (*set_active)(void *ctx, bool active), + void *ctx); + +/* + * ATC PHY: start the pixel clock for a DP tunnel at DP link rate code @rate, + * or stop it (@rate == 0). The PHY must be in Thunderbolt/USB4 mode. + */ +int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate); + +/* + * Display crossbar: take the connection of an already selected output down or + * bring it back up (clock gates and enables) around a link reconfiguration. + * The mux selection is left alone; link_down() also leaves the ATC output + * enable set and link_up() re-asserts it. The caller keeps the output + * selected (holds the mux) around both. t8103-style crossbars only. + */ +int apple_dpxbar_link_down(struct mux_control *mux); +int apple_dpxbar_link_up(struct mux_control *mux); + +#endif From a2ad089dca28e73da5f61414669751807b38b215 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 22 Sep 2026 23:17:56 -0700 Subject: [PATCH 0961/2680] mux: apple-display-crossbar: link up/down helpers for tunnel outputs When DCP retrains a link through a Thunderbolt DP tunnel, the crossbar connection has to follow it. Add apple_dpxbar_link_down() and apple_dpxbar_link_up(). They gate and ungate the clocks and enables of an already selected output. The mux selection is left alone; link_down() also leaves the ATC output enable set, and link_up() re-asserts it. The caller keeps the output selected around both; the selection is read under the crossbar lock. Also let the clock gates release before enabling the clocks behind them, both when an output is selected and in link_up(), and warn if they do not. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/mux/apple-display-crossbar.c | 121 +++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/drivers/mux/apple-display-crossbar.c b/drivers/mux/apple-display-crossbar.c index d3cd6399ead66..52d20235a6ef0 100644 --- a/drivers/mux/apple-display-crossbar.c +++ b/drivers/mux/apple-display-crossbar.c @@ -17,6 +17,7 @@ #include #include #include +#include /* * T602x register interface is cleary different so most of the names below are @@ -230,6 +231,19 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) return ret; } +/* Warn if the clock gates of an output did not release. Diagnostic only. */ +static void apple_dpxbar_check_gates(struct apple_dpxbar *dpxbar, unsigned int index, + u32 dispext_bit, u32 atc_bit) +{ + u32 wr = readl(dpxbar->regs + FIFO_WR_N_CLK_EN_STAT); + u32 rd = readl(dpxbar->regs + FIFO_RD_N_CLK_EN_STAT); + u32 out = readl(dpxbar->regs + OUT_N_CLK_EN_STAT); + + if ((wr & dispext_bit) || (rd & dispext_bit) || (out & atc_bit)) + dev_warn(dpxbar->dev, "%s: clock gates still set (%08x %08x %08x)\n", + apple_dpxbar_names[index], wr, rd, out); +} + static int apple_dpxbar_set(struct mux_control *mux, int state) { struct apple_dpxbar *dpxbar = mux_chip_priv(mux->chip); @@ -329,6 +343,9 @@ static int apple_dpxbar_set(struct mux_control *mux, int state) dpxbar_clear32(dpxbar, FIFO_WR_N_CLK_EN, dispext_bit); dpxbar_clear32(dpxbar, FIFO_RD_N_CLK_EN, dispext_bit); dpxbar_clear32(dpxbar, OUT_N_CLK_EN, atc_bit); + /* let the gates release before enabling the clocks behind them */ + udelay(1); + apple_dpxbar_check_gates(dpxbar, index, dispext_bit, atc_bit); dpxbar_set32(dpxbar, FIFO_WR_UNK_EN, dispext_bit); dpxbar_set32(dpxbar, FIFO_RD_UNK_EN, dispext_bit_en); dpxbar_set32(dpxbar, OUT_UNK_EN, atc_bit); @@ -365,6 +382,110 @@ static int apple_dpxbar_set(struct mux_control *mux, int state) return ret; } +static const struct mux_control_ops apple_dpxbar_ops; + +static const u32 apple_dpxbar_atc_bits[MUX_MAX] = { ATC_DPPHY, ATC_DPIN0, ATC_DPIN1 }; + +/* + * The output's current selection; dpxbar->lock held. The caller keeps the + * output selected (holds the mux) around the link helpers. + */ +static int apple_dpxbar_link_bits(struct apple_dpxbar *dpxbar, unsigned int index, + u32 *dispext_bit, u32 *atc_bit) +{ + int state; + + lockdep_assert_held(&dpxbar->lock); + state = dpxbar->selected_dispext[index]; + if (state < 0) + return -ENODEV; + *dispext_bit = 1 << state; + *atc_bit = apple_dpxbar_atc_bits[index]; + return 0; +} + +static struct apple_dpxbar *apple_dpxbar_from_mux(struct mux_control *mux, + unsigned int *index) +{ + if (mux->chip->ops != &apple_dpxbar_ops) + return NULL; + *index = mux_control_get_index(mux); + if (*index >= MUX_MAX) + return NULL; + return mux_chip_priv(mux->chip); +} + +int apple_dpxbar_link_down(struct mux_control *mux) +{ + struct apple_dpxbar *dpxbar; + u32 dispext_bit, atc_bit; + unsigned long flags; + unsigned int index; + int ret; + + dpxbar = apple_dpxbar_from_mux(mux, &index); + if (!dpxbar) + return -EINVAL; + + spin_lock_irqsave(&dpxbar->lock, flags); + ret = apple_dpxbar_link_bits(dpxbar, index, &dispext_bit, &atc_bit); + if (!ret) { + dpxbar_clear32(dpxbar, CROSSBAR_DISPEXT_EN, dispext_bit); + dpxbar_clear32(dpxbar, FIFO_WR_DPTX_CLK_EN, dispext_bit); + dpxbar_clear32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + dpxbar_clear32(dpxbar, OUT_PCLK1_EN, atc_bit); + udelay(1); + dpxbar_set32(dpxbar, FIFO_WR_N_CLK_EN, dispext_bit); + dpxbar_set32(dpxbar, FIFO_RD_N_CLK_EN, dispext_bit); + dpxbar_set32(dpxbar, OUT_N_CLK_EN, atc_bit); + } + spin_unlock_irqrestore(&dpxbar->lock, flags); + return ret; +} +EXPORT_SYMBOL_GPL(apple_dpxbar_link_down); + +int apple_dpxbar_link_up(struct mux_control *mux) +{ + struct apple_dpxbar *dpxbar; + u32 dispext_bit, atc_bit; + unsigned long flags; + unsigned int index; + int ret; + + dpxbar = apple_dpxbar_from_mux(mux, &index); + if (!dpxbar) + return -EINVAL; + + spin_lock_irqsave(&dpxbar->lock, flags); + ret = apple_dpxbar_link_bits(dpxbar, index, &dispext_bit, &atc_bit); + if (!ret) { + dpxbar_clear32(dpxbar, FIFO_WR_N_CLK_EN, dispext_bit); + dpxbar_clear32(dpxbar, FIFO_RD_N_CLK_EN, dispext_bit); + dpxbar_clear32(dpxbar, OUT_N_CLK_EN, atc_bit); + udelay(1); + apple_dpxbar_check_gates(dpxbar, index, dispext_bit, atc_bit); + dpxbar_set32(dpxbar, FIFO_WR_DPTX_CLK_EN, dispext_bit); + dpxbar_set32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + dpxbar_set32(dpxbar, OUT_PCLK1_EN, atc_bit); + dpxbar_set32(dpxbar, CROSSBAR_ATC_EN, atc_bit); + dpxbar_set32(dpxbar, CROSSBAR_DISPEXT_EN, dispext_bit); + dpxbar_clear32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + udelay(10); + dpxbar_set32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + } + spin_unlock_irqrestore(&dpxbar->lock, flags); + if (ret) + return ret; + + dev_dbg(dpxbar->dev, "link up: ATC_EN=%08x STAT WR=%08x RD=%08x OUT=%08x\n", + readl(dpxbar->regs + CROSSBAR_ATC_EN), + readl(dpxbar->regs + FIFO_WR_DPTX_CLK_EN_STAT), + readl(dpxbar->regs + FIFO_RD_PCLK1_EN_STAT), + readl(dpxbar->regs + OUT_PCLK1_EN_STAT)); + return 0; +} +EXPORT_SYMBOL_GPL(apple_dpxbar_link_up); + static const struct mux_control_ops apple_dpxbar_ops = { .set = apple_dpxbar_set, }; From fccd47969e42460fb6dde05aa68ea8da64aaa2a8 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 22 Sep 2026 23:17:56 -0700 Subject: [PATCH 0962/2680] thunderbolt: DP tunnels from Apple silicon host DP IN adapters On Apple silicon the display engine is not wired to the host router's DP IN adapters; the NHI glue has to route it there. Add an NHI op, dp_tunnel_changed, called when a DP tunnel from a host DP IN adapter comes up, before one is torn down, and when the connection manager stops with such a tunnel in place. Every "down" is paired with an earlier "up", so a tunnel released twice (at tb_stop() and by a late DPRX failure) is only reported once. System and runtime suspend release all DP tunnels through the normal teardown, so the glue hears about them; they come back through plug events after resume, which take the full path again. Only hosts whose glue provides the op get the extra handling, and only for their own DP IN adapters: - after activation, pulse the DP IN adapter's HPD propagation bit (ADP_DP_CS_3 bit 10) and wait for ADP_DP_CS_2_HPD, at most 2 s under tb->lock; - the DP IN video hop gets 5 NFC credits, including on USB4 routers; - no LTTPR non-transparent mode on Titan Ridge DP OUT; the host DPTX trains the sink through the tunnel; - the DP OUT adapter must not start link training on its own (ADP_DP_CS_3 bit 8) while the tunnel is up; the bit is restored on teardown, also when disabling the adapter fails. The check is safe for routers without a domain, as built by the KUnit tests. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/thunderbolt/nhi.h | 1 + drivers/thunderbolt/switch.c | 3 +- drivers/thunderbolt/tb.c | 64 +++++++++++++++++++++++++++++++++++ drivers/thunderbolt/tb.h | 14 ++++++++ drivers/thunderbolt/tb_regs.h | 3 ++ drivers/thunderbolt/tunnel.c | 40 ++++++++++++++++++++-- drivers/thunderbolt/tunnel.h | 2 ++ 7 files changed, 124 insertions(+), 3 deletions(-) diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index 7147217310763..ac8e481704654 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -105,6 +105,7 @@ struct tb_nhi_ops { int (*pci_tunnel_post_activate)(struct tb_nhi *nhi); int (*pci_tunnel_deactivate)(struct tb_nhi *nhi); bool (*is_present)(struct tb_nhi *nhi); + void (*dp_tunnel_changed)(struct tb_nhi *nhi, u8 in_port, bool active); int (*init_interrupts)(struct tb_nhi *nhi); void (*reset_interface)(struct tb_nhi *nhi); }; diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c index 6e0f0eca87887..88bbb4f714721 100644 --- a/drivers/thunderbolt/switch.c +++ b/drivers/thunderbolt/switch.c @@ -575,7 +575,8 @@ int tb_port_add_nfc_credits(struct tb_port *port, int credits) * USB4 restricts programming NFC buffers to lane adapters only * so skip other ports. */ - if (tb_switch_is_usb4(port->sw) && !tb_port_is_null(port)) + if (tb_switch_is_usb4(port->sw) && !tb_port_is_null(port) && + !tb_port_is_apple_host_dpin(port)) return 0; nfc_credits = port->config.nfc_credits & ADP_CS_4_NFC_BUFFERS_MASK; diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index 2492fcb82fb9c..b523764342f93 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -1723,6 +1723,65 @@ static void tb_discover_tunnels(struct tb *tb) } } +/* + * On some hosts (Apple silicon) the display engine has to be routed to a + * DP IN adapter of the host router by hand. Tell the NHI glue when a DP + * tunnel starting at one of those adapters comes or goes. + */ +static void tb_dp_tunnel_notify(struct tb_tunnel *tunnel, bool active) +{ + struct tb_port *in = tunnel->src_port; + + /* + * Only undo what was announced: a tunnel can be released twice (at + * tb_stop() and by a late DPRX failure), the second time after the + * host router is gone. + */ + if (!active && !tunnel->host_dp_notified) + return; + if (!tb_tunnel_is_dp(tunnel) || !tb_port_is_apple_host_dpin(in)) + return; + /* + * After both DP adapters are enabled, pulse the DP IN adapter's HPD + * propagation bit for 10 ms and wait up to 2 s for its HPD status, so + * HPD has reached the host before the display side starts. This runs + * under tb->lock; HPD normally follows at once, 2 s is the worst case. + */ + if (active && in->cap_adap) { + int i, hpd = 0; + u32 v; + + if (tb_port_read(in, &v, TB_CFG_PORT, in->cap_adap + ADP_DP_CS_3, 1)) { + tb_port_warn(in, "cannot read DP adapter state, HPD not pulsed\n"); + } else { + /* HPDC may still be set from a previous tunnel's teardown */ + v &= ~ADP_DP_CS_3_HPDC; + v |= ADP_DP_CS_3_HPD_PROPAGATE; + if (tb_port_write(in, &v, TB_CFG_PORT, in->cap_adap + ADP_DP_CS_3, 1)) { + tb_port_warn(in, "cannot pulse HPD propagation\n"); + goto notify; + } + usleep_range(10000, 11000); + v &= ~ADP_DP_CS_3_HPD_PROPAGATE; + if (tb_port_write(in, &v, TB_CFG_PORT, in->cap_adap + ADP_DP_CS_3, 1)) + tb_port_warn(in, "cannot end HPD propagation pulse\n"); + for (i = 0; i < 200; i++) { + hpd = tb_dp_port_hpd_is_active(in); + if (hpd) + break; + usleep_range(10000, 11000); + } + if (hpd < 0) + tb_port_warn(in, "cannot read HPD status: %d\n", hpd); + else if (!hpd) + tb_port_warn(in, "HPD did not propagate\n"); + } + } +notify: + tunnel->host_dp_notified = active; + tunnel->tb->nhi->ops->dp_tunnel_changed(tunnel->tb->nhi, in->port, active); +} + static void tb_deactivate_and_free_tunnel(struct tb_tunnel *tunnel) { struct tb_port *src_port, *dst_port; @@ -1731,6 +1790,7 @@ static void tb_deactivate_and_free_tunnel(struct tb_tunnel *tunnel) if (!tunnel) return; + tb_dp_tunnel_notify(tunnel, false); tb_tunnel_deactivate(tunnel); list_del(&tunnel->list); @@ -2055,6 +2115,7 @@ static void tb_tunnel_one_dp(struct tb *tb, struct tb_port *in, goto err_free; } + tb_dp_tunnel_notify(tunnel, true); return; err_free: @@ -2969,6 +3030,9 @@ static void tb_stop(struct tb *tb) tb_tunnel_deactivate(tunnel); else if (tb_tunnel_is_dp(tunnel) && !tb_tunnel_is_active(tunnel)) tb_tunnel_deactivate(tunnel); + /* the host side of a DP tunnel goes away with us */ + else if (tb_tunnel_is_dp(tunnel)) + tb_dp_tunnel_notify(tunnel, false); tb_tunnel_put(tunnel); } tb_switch_remove(tb->root_switch); diff --git a/drivers/thunderbolt/tb.h b/drivers/thunderbolt/tb.h index 8f2ac506f9dd3..3e4cbedd55963 100644 --- a/drivers/thunderbolt/tb.h +++ b/drivers/thunderbolt/tb.h @@ -1130,6 +1130,20 @@ tb_port_path_direction_downstream(const struct tb_port *src, return src->sw->config.depth < dst->sw->config.depth; } +/* + * DP IN adapter of an Apple silicon host router (its NHI glue wants DP tunnel + * notifications). These need a few extra steps compared to other hosts. + */ +static inline bool tb_port_is_apple_host_dpin(const struct tb_port *port) +{ + const struct tb *tb = port->sw->tb; + + /* the KUnit tests build switches without a domain */ + if (!tb || !tb->nhi || tb_route(port->sw) || !tb_port_is_dpin(port)) + return false; + return tb->nhi->ops && tb->nhi->ops->dp_tunnel_changed; +} + static inline bool tb_port_use_credit_allocation(const struct tb_port *port) { return tb_port_is_null(port) && port->sw->credit_allocation; diff --git a/drivers/thunderbolt/tb_regs.h b/drivers/thunderbolt/tb_regs.h index 47a1a1e909b87..5ad19c5177041 100644 --- a/drivers/thunderbolt/tb_regs.h +++ b/drivers/thunderbolt/tb_regs.h @@ -428,6 +428,9 @@ struct tb_regs_port_header { #define ADP_DP_CS_2_ESTIMATED_BW_SHIFT 24 #define ADP_DP_CS_3 0x03 #define ADP_DP_CS_3_HPDC BIT(9) +/* Apple silicon host routers */ +#define ADP_DP_CS_3_NO_AUTO_LT BIT(8) /* DP OUT of the tunnel */ +#define ADP_DP_CS_3_HPD_PROPAGATE BIT(10) /* host DP IN */ #define DP_LOCAL_CAP 0x04 #define DP_REMOTE_CAP 0x05 /* For DP IN adapter */ diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index 0be1fb7085112..3e5c229aa42f5 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -947,8 +947,13 @@ static int tb_dp_xchg_caps(struct tb_tunnel *tunnel) * Titan Ridge does not disable AUX timers when it gets * SET_CONFIG with SET_LTTPR_MODE set. This causes problems with * DP tunneling. + * + * Not with an Apple host: its DPTX trains the sink through the + * tunnel itself, and the configuration validated on hardware leaves + * the capability alone. */ - if (tb_route(out->sw) && tb_switch_is_titan_ridge(out->sw)) { + if (tb_route(out->sw) && tb_switch_is_titan_ridge(out->sw) && + !tb_port_is_apple_host_dpin(in)) { out_dp_cap |= DP_COMMON_CAP_LTTPR_NS; tb_tunnel_dbg(tunnel, "disabling LTTPR\n"); } @@ -1227,7 +1232,32 @@ static int tb_dp_activate(struct tb_tunnel *tunnel, bool active) return ret; if (tb_port_is_dpout(tunnel->dst_port)) { - ret = tb_dp_port_enable(tunnel->dst_port, active); + struct tb_port *out = tunnel->dst_port; + + /* + * With an Apple host the sink link is trained by the host's + * DPTX through the tunnel: keep the DP OUT adapter from + * starting link training on its own, and hand it back once + * the tunnel is gone. + */ + bool apple = tb_port_is_apple_host_dpin(tunnel->src_port) && + out->cap_adap; + u32 v; + + if (active && apple && + !tb_port_read(out, &v, TB_CFG_PORT, out->cap_adap + ADP_DP_CS_3, 1)) { + v |= ADP_DP_CS_3_NO_AUTO_LT; + if (tb_port_write(out, &v, TB_CFG_PORT, out->cap_adap + ADP_DP_CS_3, 1)) + tb_port_warn(out, "cannot hold off DP link training\n"); + } + ret = tb_dp_port_enable(out, active); + /* hand link training back even if the disable failed */ + if (!active && apple && + !tb_port_read(out, &v, TB_CFG_PORT, out->cap_adap + ADP_DP_CS_3, 1)) { + v &= ~ADP_DP_CS_3_NO_AUTO_LT; + if (tb_port_write(out, &v, TB_CFG_PORT, out->cap_adap + ADP_DP_CS_3, 1)) + tb_port_warn(out, "cannot restore DP link training\n"); + } if (ret) return ret; } @@ -1538,6 +1568,12 @@ static int tb_dp_init_video_credits(struct tb_path_hop *hop) struct tb_port *port = hop->in_port; struct tb_switch *sw = port->sw; + /* Apple host DP IN adapters take 5 NFC credits for the video path. */ + if (tb_port_is_apple_host_dpin(port)) { + hop->nfc_credits = 5; + return 0; + } + if (tb_port_use_credit_allocation(port)) { unsigned int nfc_credits; size_t max_dp_streams; diff --git a/drivers/thunderbolt/tunnel.h b/drivers/thunderbolt/tunnel.h index 28f49f7e9f563..660666669240c 100644 --- a/drivers/thunderbolt/tunnel.h +++ b/drivers/thunderbolt/tunnel.h @@ -64,6 +64,7 @@ enum tb_tunnel_state { * determine consumed and allocated bandwidth * @dprx_started: DPRX negotiation was started (tb_dp_dprx_start() was called for it) * @dprx_canceled: Was DPRX capabilities read poll canceled + * @host_dp_notified: The NHI glue was told this DP tunnel is up (Apple hosts) * @dprx_timeout: If set DPRX capabilities read poll work will timeout after this passes * @dprx_work: Worker that is scheduled to poll completion of DPRX capabilities read * @callback: Callback called when DP tunnel is fully activated @@ -101,6 +102,7 @@ struct tb_tunnel { bool bw_mode; bool dprx_started; bool dprx_canceled; + bool host_dp_notified; ktime_t dprx_timeout; struct delayed_work dprx_work; void (*callback)(struct tb_tunnel *tunnel); From c8bef7d87d7cd70847cb9f7aaf7b946ab066c5ff Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 22 Sep 2026 23:17:56 -0700 Subject: [PATCH 0963/2680] thunderbolt: apple: bring up DP IN adapters for display tunnels (t8103) Implement dp_tunnel_changed. It is only registered when thunderbolt_apple.dp_display=1 is given and the machine is a t8103, so the connection manager's Apple DP handling stays off otherwise. When a DP tunnel from a host DP IN adapter comes up: 1. Map the adapter's register block non-posted; posted writes to it take an SError. 2. Acknowledge and enable its interrupts. 3. Ask appledrm to route a display pipeline to it. If appledrm is still probing (a dock present at boot), wait for it for up to 30 s. The adapter's DPTX_INACTIVE handshake is run from DCP's link activation (the adapter may only be woken while DCP drives the DPTX), within one 500 ms budget per call. Before a tunnel is torn down the adapter is put to sleep and masked synchronously, and nothing touches it after that. Each adapter has one work item on an ordered queue that brings the display side in line with the tunnel state, so nothing is allocated on the way down and whatever was handed to appledrm is always taken back. When the ACIO is stopped, the display side is released before it powers off. The parameter is read-only at runtime. Not covered yet: the DP IN block is found at a fixed offset from the ACIO "rc" region (there is no device tree node for it), its interrupts have no handler, and suspend with an active tunnel has not been validated. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- .../admin-guide/kernel-parameters.txt | 9 + drivers/thunderbolt/apple.c | 321 +++++++++++++++++- 2 files changed, 329 insertions(+), 1 deletion(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 4929f80027a78..5521eaea7fb26 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -7557,6 +7557,15 @@ Kernel parameters See Documentation/admin-guide/mm/transhuge.rst for more details. + thunderbolt_apple.dp_display= + [HW,ARM64] + Format: + Drive displays behind Thunderbolt DisplayPort tunnels + on Apple silicon (experimental, t8103 only): route a + display pipeline to the host router's DP IN adapter + whenever the connection manager sets up a DP tunnel. + Default: false + tlbi= [X86-64] Format: {ipi} ipi: switch to IPI-based TLB flushing diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 18cb1c73cacd4..01f74dcf14701 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -87,6 +87,7 @@ #include #include #include +#include #include #include #include @@ -94,6 +95,7 @@ #include #include #include +#include #include #include #include @@ -202,8 +204,246 @@ struct apple_cio { struct delayed_work pcie_tunnel_work; bool pcie_tunnel_requested; bool pcie_tunnel_populated; + + /* Type-C connector this router is wired to, for DP tunnel routing */ + struct device_node *connector_np; + struct workqueue_struct *dp_wq; + struct apple_dpin_ctx { + struct apple_cio *acio; + struct work_struct work; + struct mutex lock; /* protects regs and alive */ + void __iomem *regs; /* mapped while a DP tunnel uses this adapter */ + unsigned int idx; + bool alive; /* tunnel up; cleared before it is torn down */ + bool handed; /* appledrm has our callback; work only */ + } dpin[2]; }; +/* + * The ATC's DP IN adapter blocks ("atcN-dpin0/1" in the Apple device tree). + * Registers on t8103: + * 0x00 status, bit 2 = HPD level + * 0x04 interrupt status (write 1 to clear) + * 0x08 interrupt enable [1:0] + * 0x0c bit 0 DPTX_INACTIVE + * 0x10 bit 0 DPTX_INACTIVE_ACK + * The block sits 0x390000 after the ACIO "rc" region; dpin1 is 0x8000 after dpin0. + */ +#define APPLE_DPIN_OFFSET 0x390000 +#define APPLE_DPIN_STRIDE 0x8000 +#define APPLE_DPIN_SIZE 0x4000 +#define APPLE_DPIN_STATUS 0x00 +#define APPLE_DPIN_STATUS_HPD BIT(2) +#define APPLE_DPIN_IRQ_STATUS 0x04 +#define APPLE_DPIN_IRQ_ENABLE 0x08 +#define APPLE_DPIN_CTRL 0x0c +#define APPLE_DPIN_CTRL_INACTIVE BIT(0) +#define APPLE_DPIN_ACK 0x10 +#define APPLE_DPIN_ACK_INACTIVE BIT(0) + +static bool dp_display; +module_param(dp_display, bool, 0444); +MODULE_PARM_DESC(dp_display, "Drive displays behind Thunderbolt DP tunnels (experimental, t8103)"); + +/* DPTX_INACTIVE handshake: request (in)active, wait for the ACK */ +static int apple_dpin_set_active(struct apple_cio *acio, void __iomem *dpin, + unsigned int idx, bool active) +{ + u32 want = active ? 0 : APPLE_DPIN_ACK_INACTIVE; + unsigned long deadline = jiffies + msecs_to_jiffies(500); + u32 st = 0, ctrl, ack; + + /* one 500 ms budget for the whole handshake: DCP waits on this */ + if (active) { + while (!((st = readl(dpin + APPLE_DPIN_STATUS)) & APPLE_DPIN_STATUS_HPD)) { + if (time_after(jiffies, deadline)) { + dev_warn(acio->dev, "dpin%u: no HPD (status=%08x)\n", + idx, st); + return -ETIMEDOUT; + } + usleep_range(1000, 1200); + } + } + + ctrl = readl(dpin + APPLE_DPIN_CTRL); + ack = readl(dpin + APPLE_DPIN_ACK); + if (!!(ctrl & APPLE_DPIN_CTRL_INACTIVE) == !active && + (ack & APPLE_DPIN_ACK_INACTIVE) == want) + return 0; + + if (active) + ctrl &= ~APPLE_DPIN_CTRL_INACTIVE; + else + ctrl |= APPLE_DPIN_CTRL_INACTIVE; + writel(ctrl, dpin + APPLE_DPIN_CTRL); + + while (((ack = readl(dpin + APPLE_DPIN_ACK)) & APPLE_DPIN_ACK_INACTIVE) != want) { + if (time_after(jiffies, deadline)) { + dev_warn(acio->dev, "dpin%u: no DPTX_INACTIVE_ACK=%u (ack=%08x)\n", + idx, !active, ack); + return -ETIMEDOUT; + } + usleep_range(1000, 1200); + } + dev_dbg(acio->dev, "dpin%u: %s (status=%08x)\n", idx, + active ? "active" : "inactive", readl(dpin + APPLE_DPIN_STATUS)); + return 0; +} + +/* Called by appledrm from DCP's Activate/Deactivate calls. */ +static int apple_dpin_dcp_set_active(void *data, bool active) +{ + struct apple_dpin_ctx *c = data; + + guard(mutex)(&c->lock); + /* the tunnel is being torn down: the block may already be off */ + if (!c->alive || !c->regs) + return -ENODEV; + return apple_dpin_set_active(c->acio, c->regs, c->idx, active); +} + +/* appledrm may still be probing when a dock is present at boot: wait up to 30 s */ +#define APPLE_DP_CONNECT_TRIES 60 +#define APPLE_DP_CONNECT_WAIT_MS 500 + +static int apple_dpin_connect(struct apple_dpin_ctx *c, bool active) +{ + struct apple_cio *acio = c->acio; + typeof(&apple_dcp_tb_dp_tunnel) fn; + unsigned int tries; + bool alive; + int ret; + + for (tries = 1; ; tries++) { + fn = symbol_get(apple_dcp_tb_dp_tunnel); + if (fn) { + ret = fn(acio->connector_np, c->idx, active, + active ? apple_dpin_dcp_set_active : NULL, + active ? c : NULL); + symbol_put(apple_dcp_tb_dp_tunnel); + } else { + /* appledrm gone: it has dropped our callback with it */ + if (!active) + return 0; + ret = -ENODEV; + } + if (!active || ret != -ENODEV || tries >= APPLE_DP_CONNECT_TRIES) + return ret; + + scoped_guard(mutex, &c->lock) + alive = c->alive; + if (!alive) + return -ENODEV; + if (tries == 1) + dev_info(acio->dev, "dpin%u: waiting for the display driver\n", + c->idx); + msleep(APPLE_DP_CONNECT_WAIT_MS); + } +} + +static int apple_dpin_up(struct apple_dpin_ctx *c) +{ + struct apple_cio *acio = c->acio; + void __iomem *regs = c->regs; + u32 irq; + int ret; + + if (!regs) { + regs = ioremap_np(acio->rc_res->start + APPLE_DPIN_OFFSET + + c->idx * APPLE_DPIN_STRIDE, APPLE_DPIN_SIZE); + if (!regs) + return -ENOMEM; + } + scoped_guard(mutex, &c->lock) { + c->regs = regs; + /* unplugged meanwhile: the block may be off, leave it alone */ + if (!c->alive) + return -ENODEV; + /* + * Acknowledge what is pending and enable the DP IN interrupts: + * +0x4 is write-1-to-clear, and a plug event (bit 0) also + * latches status bits in +0x0 that are cleared by writing it + * back. Nothing handles these interrupts yet; they are enabled + * as part of bringing the adapter up. + */ + irq = readl(regs + APPLE_DPIN_IRQ_STATUS); + writel(irq, regs + APPLE_DPIN_IRQ_STATUS); + if (irq & BIT(0)) + writel(readl(regs + APPLE_DPIN_STATUS), regs + APPLE_DPIN_STATUS); + writel(readl(regs + APPLE_DPIN_IRQ_ENABLE) | 3, regs + APPLE_DPIN_IRQ_ENABLE); + } + + ret = apple_dpin_connect(c, true); + if (!ret) + c->handed = true; + return ret; +} + +static void apple_dpin_down(struct apple_dpin_ctx *c) +{ + void __iomem *regs; + int ret; + + if (c->handed) { + ret = apple_dpin_connect(c, false); + if (ret) + dev_warn(c->acio->dev, "dpin%u: display teardown failed: %d\n", + c->idx, ret); + c->handed = false; + } + /* the adapter was put to sleep before the tunnel went away */ + scoped_guard(mutex, &c->lock) { + regs = c->regs; + c->regs = NULL; + } + if (regs) + iounmap(regs); +} + +/* + * Bring the display side in line with the tunnel state. One work item per + * adapter on an ordered queue, so events are never lost and nothing has to be + * allocated on the way down: whatever was handed to appledrm is taken back. + */ +static void apple_dpin_work_fn(struct work_struct *work) +{ + struct apple_dpin_ctx *c = container_of(work, struct apple_dpin_ctx, work); + bool want; + int ret; + + for (;;) { + scoped_guard(mutex, &c->lock) + want = c->alive; + + if (!want) { + if (c->handed || c->regs) { + apple_dpin_down(c); + dev_dbg(c->acio->dev, "dpin%u: DP tunnel down\n", c->idx); + } + return; + } + if (c->handed) + return; + + ret = apple_dpin_up(c); + if (ret) { + scoped_guard(mutex, &c->lock) + want = c->alive; + if (!want) + continue; /* unplugged meanwhile: clean up */ + dev_warn(c->acio->dev, "dpin%u: DP tunnel setup failed: %d\n", + c->idx, ret); + return; + } + dev_dbg(c->acio->dev, "dpin%u: DP tunnel up\n", c->idx); + } +} + +static void apple_cio_destroy_wq(void *data) +{ + destroy_workqueue(data); +} + static void apple_cio_of_node_put(void *data) { of_node_put(data); @@ -300,6 +540,7 @@ struct apple_nhi { struct tb *tb; struct tb_nhi nhi; + struct tb_nhi_ops ops; void __iomem *nhi_base; void __iomem *pdf_base; @@ -715,6 +956,52 @@ static int apple_nhi_pci_tunnel_post_activate(struct tb_nhi *nhi) return 0; } +static void apple_nhi_dp_tunnel_changed(struct tb_nhi *nhi, u8 in_port, + bool active) +{ + struct apple_nhi *anhi = nhi_to_anhi(nhi); + struct apple_cio *acio = anhi->acio; + struct tb_port *port; + unsigned int dpin = 0; + bool found = false; + + /* The crossbar's dpin0/dpin1 follow the host router's DP IN adapters in order. */ + tb_switch_for_each_port(anhi->tb->root_switch, port) { + if (!tb_port_is_dpin(port)) + continue; + if (port->port == in_port) { + found = true; + break; + } + dpin++; + } + if (!found || dpin > 1) { + dev_warn(acio->dev, "DP tunnel from unexpected adapter %u\n", in_port); + return; + } + + /* + * Runs synchronously before the tunnel is torn down: from here on the + * DP IN block may lose power, so nothing may touch it any more. + */ + scoped_guard(mutex, &acio->dpin[dpin].lock) { + struct apple_dpin_ctx *c = &acio->dpin[dpin]; + + /* + * Going down: the block is still powered here (the tunnel is + * torn down after this returns), so put the adapter back to + * sleep and mask its interrupts now; nothing may touch it after. + */ + if (!active && c->alive && c->regs) { + apple_dpin_set_active(acio, c->regs, dpin, false); + writel(readl(c->regs + APPLE_DPIN_IRQ_ENABLE) & ~3, + c->regs + APPLE_DPIN_IRQ_ENABLE); + } + c->alive = active; + } + queue_work(acio->dp_wq, &acio->dpin[dpin].work); +} + static const struct tb_nhi_ops apple_nhi_ops = { .request_ring_irq = apple_nhi_request_irq, .release_ring_irq = apple_nhi_release_irq, @@ -804,7 +1091,14 @@ static int apple_nhi_probe(struct platform_device *pdev) spin_lock_init(&anhi->nhi.lock); anhi->nhi.iommu_dma_protection = true; - anhi->nhi.ops = &apple_nhi_ops; + /* + * DP tunnel display support (and with it the host DP IN handling in + * the connection manager) only where it is enabled and known to work. + */ + anhi->ops = apple_nhi_ops; + if (dp_display && acio->dp_wq && of_machine_is_compatible("apple,t8103")) + anhi->ops.dp_tunnel_changed = apple_nhi_dp_tunnel_changed; + anhi->nhi.ops = &anhi->ops; anhi->nhi.ring_layout = &apple_nhi_ring_layout; anhi->nhi.iobase = anhi->nhi_base; anhi->nhi.quirks = QUIRK_NO_DMA_PORT | QUIRK_NO_USB3_BW_ALLOC; @@ -1057,6 +1351,9 @@ static void apple_cio_stop(struct apple_cio *acio) nhi_pdev = READ_ONCE(acio->nhi_pdev); if (nhi_pdev) of_platform_device_destroy(&nhi_pdev->dev, NULL); + /* removing the NHI released its DP tunnels: let the display side finish */ + if (acio->dp_wq) + flush_workqueue(acio->dp_wq); of_platform_depopulate(acio->dev); acio->pcie_tunnel_populated = false; @@ -1386,6 +1683,28 @@ static int apple_cio_probe(struct platform_device *pdev) else if (ret < 3) return dev_err_probe(dev, -EINVAL, "Not enough PM domains\n"); + acio->connector_np = of_graph_get_remote_node(dev->of_node, 1, -1); + if (acio->connector_np) { + ret = devm_add_action_or_reset(dev, apple_cio_of_node_put, + acio->connector_np); + if (ret) + return ret; + for (unsigned int i = 0; i < ARRAY_SIZE(acio->dpin); i++) { + acio->dpin[i].acio = acio; + acio->dpin[i].idx = i; + INIT_WORK(&acio->dpin[i].work, apple_dpin_work_fn); + ret = devm_mutex_init(dev, &acio->dpin[i].lock); + if (ret) + return ret; + } + acio->dp_wq = alloc_ordered_workqueue("%s-dp", 0, dev_name(dev)); + if (!acio->dp_wq) + return -ENOMEM; + ret = devm_add_action_or_reset(dev, apple_cio_destroy_wq, acio->dp_wq); + if (ret) + return ret; + } + /* And finally register the OOB notification for Thunderbolt/USB4 cables */ struct typec_thunderbolt_switch_desc desc = { .fwnode = pdev->dev.fwnode, From 9b92840bf6abc5f5ee42c763210e9c22a9ceb737 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 23 Sep 2026 10:56:26 -0700 Subject: [PATCH 0964/2680] PCI: apple: enumerate Thunderbolt PCIe on t8103 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit t8103 boots with the Thunderbolt PCIe port held in reset. Bring the port up when the tunnel activates, and do it before the port's IOMMU probes: that IOMMU's invalidate command does not finish while the port clock is still gated. Program the port tunables and the reset table, release PERST, enable the port clock, then program the root-port configuration tunables. Link training stays in the host driver. If the port is already running, the host does not replay the reset table. Endpoint memory on t8103 is left posted. The PCIe-C IOMMU uses the 64-stream USB4 register block: 16 KiB pages, a 32-bit IOVA and a 36-bit physical address. The 42-bit four-level geometry remains on T8110, which is the hardware that walks it. An xHCI early handoff no longer faults when the capability length read from a BAR is shorter than 16 bytes or not dword-aligned. Tested on a MacBook Pro 13" M1 (j293) with a CalDigit TS3 Plus on USB-1. The tunnel enumerated the dock bridges, four USB host controllers and an I210. The USB devices and a gigabit link came up, and the display on the same port still comes up. [ryan: adapted to 7.2.10 — split by topic: only the drivers/thunderbolt, drivers/usb hunks of this radio commit; the rest is in 080-radio] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/thunderbolt/apple.c | 69 +++++++++++++++++++++++++++++------ drivers/usb/host/pci-quirks.c | 15 +++++++- 2 files changed, 72 insertions(+), 12 deletions(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 01f74dcf14701..e32905925af12 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -182,6 +182,8 @@ struct apple_cio { void __iomem *rc_base; struct resource *rc_res; struct apple_tunable *rc_tunable; + void __iomem *pcie_adapter_base; + struct apple_tunable *pcie_adapter_tunable; struct resource *sram_res; void __iomem *sram_base; @@ -459,9 +461,11 @@ apple_cio_pcie_tunnel_is_preinitialized(struct device_node *parent) !of_device_is_compatible(child, "apple,t6000-pciec")) continue; - return !of_property_read_u32(child, - "apple,pciec-preinit-status", - &status) && status == 1; + if (!of_property_read_u32(child, "apple,pciec-preinit-status", + &status) && status == 1) + return true; + /* t8103 brings the port up itself. */ + return of_property_read_bool(child, "apple,pciec-kernel-init"); } return false; @@ -484,7 +488,7 @@ static int apple_cio_map_pcie_intr2axi(struct apple_cio *acio) index = of_property_match_string(child, "reg-names", "intr2axi"); if (index < 0) - return index; + return 0; ret = of_address_to_resource(child, index, &res); if (ret) @@ -808,6 +812,8 @@ static struct platform_device *apple_cio_find_pcie_tunnel(struct apple_cio *acio static int apple_cio_populate_pcie_tunnel(struct apple_cio *acio) { + int ret; + /* * PCIe-C was cold-initialized by m1n1 before the ACIO M3 started, but * enabling ACIO's cable-powered PCIe domain closes the Intr2AXI bridge @@ -816,14 +822,24 @@ static int apple_cio_populate_pcie_tunnel(struct apple_cio *acio) * enabled both tunnel adapters and before either child can touch DART MMIO. * The bit self-clears after opening the bridge. */ - writel(APPLE_CIO_PCIEC_INTR2AXI_ENABLE, - acio->pcie_intr2axi_base + APPLE_CIO_PCIEC_INTR2AXI_CTRL); - /* Complete the pulse without reading the transient register. */ - mb(); - dev_info(acio->dev, "PCIe-C Intr2AXI bridge enabled\n"); + if (acio->pcie_intr2axi_base) { + writel(APPLE_CIO_PCIEC_INTR2AXI_ENABLE, + acio->pcie_intr2axi_base + APPLE_CIO_PCIEC_INTR2AXI_CTRL); + /* Complete the pulse without reading the transient register. */ + mb(); + dev_info(acio->dev, "PCIe-C Intr2AXI bridge enabled\n"); + } - dev_info(acio->dev, - "PCIe-C live handoff accepted; populating tunnel children\n"); + /* + * The DART is the first child and probes as soon as it is created. + * Its command engine stays busy until this port clock is running, so + * finish cold init before either child is populated. + */ + ret = apple_pcie_tunnel_prepare(acio->dev, acio->pcie_tunnel_np); + if (ret) + return ret; + + dev_info(acio->dev, "PCIe-C populating tunnel children\n"); return of_platform_populate(acio->pcie_tunnel_np, NULL, NULL, acio->dev); } @@ -1459,6 +1475,11 @@ static int apple_cio_start(struct apple_cio *acio) } apple_tunable_apply(acio->rc_base, acio->rc_tunable); + if (acio->pcie_adapter_base && acio->pcie_adapter_tunable) { + dev_info(acio->dev, "applying PCIe adapter tunable\n"); + apple_tunable_apply(acio->pcie_adapter_base, + acio->pcie_adapter_tunable); + } /* * Bring up devices which are part of ACIO and are now accessible by the main SoC @@ -1655,6 +1676,32 @@ static int apple_cio_probe(struct platform_device *pdev) if (IS_ERR(acio->rc_tunable)) return dev_err_probe(dev, PTR_ERR(acio->rc_tunable), "Unable to load rc tunable\n"); + { + struct resource *adapter; + + adapter = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "pcie-adapter"); + if (adapter) { + adapter->flags |= IORESOURCE_MEM_NONPOSTED; + acio->pcie_adapter_base = devm_ioremap_resource(dev, adapter); + if (IS_ERR(acio->pcie_adapter_base)) + return dev_err_probe(dev, PTR_ERR(acio->pcie_adapter_base), + "Unable to map PCIe adapter regs\n"); + acio->pcie_adapter_tunable = + devm_apple_tunable_parse(dev, acio->np, + "apple,tunable-pcie-adapter", + adapter); + if (IS_ERR(acio->pcie_adapter_tunable)) { + if (PTR_ERR(acio->pcie_adapter_tunable) == -ENOENT) + acio->pcie_adapter_tunable = NULL; + else + return dev_err_probe(dev, + PTR_ERR(acio->pcie_adapter_tunable), + "Unable to load PCIe adapter tunable\n"); + } + } + } + acio->reset = devm_reset_control_get_exclusive(dev, NULL); if (IS_ERR(acio->reset)) return dev_err_probe(dev, PTR_ERR(acio->reset), "Unable to get CIO reset\n"); diff --git a/drivers/usb/host/pci-quirks.c b/drivers/usb/host/pci-quirks.c index 0404489c2f6a9..26bba2d42b1bb 100644 --- a/drivers/usb/host/pci-quirks.c +++ b/drivers/usb/host/pci-quirks.c @@ -1224,7 +1224,20 @@ static void quirk_usb_handoff_xhci(struct pci_dev *pdev) if (pdev->vendor == PCI_VENDOR_ID_INTEL) usb_enable_intel_xhci_ports(pdev); - op_reg_base = base + XHCI_HC_LENGTH(readl(base)); + /* + * A dead or unprogrammed BAR reads as all-ones. The capability + * length is then 0xff, and the operational-register read below is + * not 4-byte aligned and faults. A real xHCI length is at least + * 0x10 and dword-aligned. + */ + val = readl(base); + if (XHCI_HC_LENGTH(val) < 0x10 || (XHCI_HC_LENGTH(val) & 3)) { + dev_warn(&pdev->dev, + "xHCI capability length %#x unusable, skipping handoff\n", + XHCI_HC_LENGTH(val)); + goto iounmap; + } + op_reg_base = base + XHCI_HC_LENGTH(val); /* Wait for the host controller to be ready before writing any * operational or runtime registers. Wait 5 seconds and no more. From 2778dd6e6374e5eacd1ba5a677ef296b32284942 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Thu, 24 Sep 2026 16:47:51 +0200 Subject: [PATCH 0965/2680] soc: apple: add DisplayPort tunnel routing interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the shared header that lets the Thunderbolt/ACIO driver, the DCP display driver, the ATC PHY and the display crossbar mux driver cooperate on routing a DisplayPort tunnel over USB4/Thunderbolt on Apple Silicon (starting with T602X/j416s). The four declared entry points cover the full path a tunneled DP stream takes: apple_dcp_tb_dp_tunnel() lets Thunderbolt hand a DP IN adapter's crossbar output to DCP (or take it back) and runs DCP's DPTX_INACTIVE handshake via a callback; apple_atc_dp_tunnel_rate() starts or stops the PHY's pixel clock at a given DP link rate once the PHY is in USB4 mode; and apple_dpxbar_link_down()/apple_dpxbar_link_up() let an already-selected crossbar output be quiesced and restored around a link reconfiguration without disturbing the mux selection. None of the four providers is a build-time dependency of the others: each caller resolves its counterpart with symbol_get()/symbol_put(), so the four drivers can each be loaded independently. This header keeps those calls type-checked against a single declaration. [ryan: adapted to 7.2.10 — split by topic: only the include/linux/soc/apple/dp-tunnel.h hunks of this soc-platform commit; the rest is in 010-soc] [ryan: adapted to 7.2.10 — add/add with 019279b11e71, which created the same header on the M1 branch; resolved as merge cf5f7d19399e did (that header kept, this commit only widens the crossbar note)] Co-Authored-By: Claude Sonnet 5 --- include/linux/soc/apple/dp-tunnel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/soc/apple/dp-tunnel.h b/include/linux/soc/apple/dp-tunnel.h index b1bbbc3a6017d..2767c16b127ff 100644 --- a/include/linux/soc/apple/dp-tunnel.h +++ b/include/linux/soc/apple/dp-tunnel.h @@ -35,7 +35,7 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate); * bring it back up (clock gates and enables) around a link reconfiguration. * The mux selection is left alone; link_down() also leaves the ATC output * enable set and link_up() re-asserts it. The caller keeps the output - * selected (holds the mux) around both. t8103-style crossbars only. + * selected (holds the mux) around both. Supports t8103 and T602X crossbars. */ int apple_dpxbar_link_down(struct mux_control *mux); int apple_dpxbar_link_up(struct mux_control *mux); From 00d334962f70abcfe2de33bbb777bb5efd84d68f Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Thu, 24 Sep 2026 16:47:52 +0200 Subject: [PATCH 0966/2680] thunderbolt: add NHI hooks and Apple host DP tunnel support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Give struct tb_nhi_ops three new optional callbacks -- dp_tunnel_pre_activate, dp_tunnel_post_activate and dp_tunnel_deactivate -- mirroring the existing pci_tunnel_* hooks, so a platform NHI driver can observe and participate in DP tunnel bring-up and teardown: pre_activate runs after DP hops are programmed but before VE/AE are enabled, post_activate runs once the adapters are enabled, and deactivate runs when the tunnel comes down. Export tb_cfg_read() into the USB4 symbol namespace alongside the already-exported tb_cfg_write(), since an out-of-tree platform NHI driver implementing these hooks needs tb_port_read()/tb_port_write(). Fix tb_dp_activate() to not abort the rest of its teardown sequence when disabling one adapter fails: once a port is marked unplugged, tb_port_read()/tb_port_write() short-circuit to -ENODEV, so a departing DP OUT adapter always makes the disable call "fail" during a hot-unplug teardown. Only bail out early when actually activating; keep going on deactivate regardless of an individual adapter's error, since tb_tunnel_deactivate() already discards this function's return value and callers rely on the rest of the teardown running. Add Apple silicon host DP IN/OUT adapter support, gated on the NHI being compatible with "apple,t8103-usb4-nhi" (the family also used by T6020/M2 Pro) so it has no effect on non-Apple hardware. Apple hosts need several deviations from the standard USB4 DP tunnel sequence: the DP IN adapter has no physical connector wired to its own hotplug-detect line, so pulse ADP_DP_CS_3_HPD_PROPAGATE after activation to make it propagate HPD itself; the sink is trained by the host's DPTX through the tunnel rather than by the DP OUT adapter's own link training, so hold ADP_DP_CS_3_NO_AUTO_LT set on DP OUT for the tunnel's lifetime; the host-side DP IN adapter needs a fixed 5 NFC credits on the video path rather than the standard credit-allocation formula; on some ports the DP IN adapter never raises a bandwidth-allocation request notification as the spec expects, so grant its already-computed non-reduced bandwidth immediately once capped at what the connection manager reserved; and because Apple's DPTX negotiates AUX capability through the DP IN adapter only after the tunnel's paths already exist, treat the tunnel as active as soon as its paths are programmed and keep it up across a DPRX capability-read timeout rather than tearing it down. Add an off-by-default dp_video_counter module parameter that turns on hardware packet counters on the DP video path for debugging, readable through the existing debugfs port counter files. [ryan: adapted to 7.2.10 — Asahi exports the core symbols for the Apple driver with EXPORT_SYMBOL_FOR_MODULES(..., "thunderbolt_apple") and does not import the USB4 namespace; the new exports follow that] [ryan: adapted to 7.2.10 — regions that overlap the parallel t8103 DP tunnel series are resolved as merge cf5f7d19399e did; tb_dp_dprx_start() follows 7.2, where the activation callback is mandatory, and keeps the Apple early-activation on top] Co-Authored-By: Claude Sonnet 5 --- drivers/thunderbolt/ctl.c | 1 + drivers/thunderbolt/nhi.h | 13 ++ drivers/thunderbolt/tb.c | 14 +- drivers/thunderbolt/tb_regs.h | 20 ++- drivers/thunderbolt/tunnel.c | 297 +++++++++++++++++++++++++++++++--- 5 files changed, 310 insertions(+), 35 deletions(-) diff --git a/drivers/thunderbolt/ctl.c b/drivers/thunderbolt/ctl.c index 41d554a5442c1..95896401571cc 100644 --- a/drivers/thunderbolt/ctl.c +++ b/drivers/thunderbolt/ctl.c @@ -1133,6 +1133,7 @@ int tb_cfg_read(struct tb_ctl *ctl, void *buffer, u64 route, u32 port, } return res.err; } +EXPORT_SYMBOL_FOR_MODULES(tb_cfg_read, "thunderbolt_apple"); int tb_cfg_write(struct tb_ctl *ctl, const void *buffer, u64 route, u32 port, enum tb_cfg_space space, u32 offset, u32 length) diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index ac8e481704654..dc85752d72d96 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -11,6 +11,8 @@ #include +struct tb_port; + enum nhi_fw_mode { NHI_FW_SAFE_MODE, NHI_FW_AUTH_MODE, @@ -83,6 +85,11 @@ struct tb_nhi_ring_layout { * been allocated but before its paths and adapters are enabled * @pci_tunnel_post_activate: NHI specific hook run after a PCIe tunnel's * paths and adapters have been enabled + * @dp_tunnel_pre_activate: NHI specific hook run after DP hops are programmed + * and before VE/AE. USB4 DPTX Discovery belongs here. + * @dp_tunnel_post_activate: NHI specific hook run after a DP tunnel's + * adapters have VE/AE enabled + * @dp_tunnel_deactivate: NHI specific hook run when a DP tunnel is torn down * @is_present: Whether the device is currently present on the parent bus * @init_interrupts: NHI specific interrupt initialization hook * @reset_interface: Resets the host interface @@ -104,6 +111,12 @@ struct tb_nhi_ops { int (*pci_tunnel_pre_activate)(struct tb_nhi *nhi); int (*pci_tunnel_post_activate)(struct tb_nhi *nhi); int (*pci_tunnel_deactivate)(struct tb_nhi *nhi); + int (*dp_tunnel_pre_activate)(struct tb_nhi *nhi, struct tb_port *in, + struct tb_port *out); + int (*dp_tunnel_post_activate)(struct tb_nhi *nhi, struct tb_port *in, + struct tb_port *out); + void (*dp_tunnel_deactivate)(struct tb_nhi *nhi, struct tb_port *in, + struct tb_port *out); bool (*is_present)(struct tb_nhi *nhi); void (*dp_tunnel_changed)(struct tb_nhi *nhi, u8 in_port, bool active); int (*init_interrupts)(struct tb_nhi *nhi); diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index b523764342f93..4e99275d00367 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -2027,17 +2028,10 @@ static void tb_dp_tunnel_active(struct tb_tunnel *tunnel) * happens either because there is no graphics driver * loaded or not all DP cables where connected to the * discrete router. - * - * In both cases we remove the DP IN adapter from the - * available resources as it is not usable. This will - * also tear down the tunnel and try to re-use the - * released DP OUT. - * - * It will be added back only if there is hotplug for - * the DP IN again. */ tb_tunnel_warn(tunnel, "not active, tearing down\n"); - tb_dp_resource_unavailable(tb, in, "DPRX negotiation failed"); + tb_dp_resource_unavailable(tb, in, + "DPRX negotiation failed"); } mutex_unlock(&tb->lock); } @@ -2963,6 +2957,8 @@ static void tb_handle_notification(struct tb *tb, u64 route, case TB_CFG_ERROR_PCIE_WAKE: case TB_CFG_ERROR_DP_CON_CHANGE: case TB_CFG_ERROR_DPTX_DISCOVERY: + tb_info(tb, "DPTX discovery notification route=%llx port=%u\n", + route, error->port); if (tb_cfg_ack_notification(tb->ctl, route, error)) tb_warn(tb, "could not ack notification on %llx\n", route); diff --git a/drivers/thunderbolt/tb_regs.h b/drivers/thunderbolt/tb_regs.h index 5ad19c5177041..84f22d937453e 100644 --- a/drivers/thunderbolt/tb_regs.h +++ b/drivers/thunderbolt/tb_regs.h @@ -428,9 +428,18 @@ struct tb_regs_port_header { #define ADP_DP_CS_2_ESTIMATED_BW_SHIFT 24 #define ADP_DP_CS_3 0x03 #define ADP_DP_CS_3_HPDC BIT(9) -/* Apple silicon host routers */ -#define ADP_DP_CS_3_NO_AUTO_LT BIT(8) /* DP OUT of the tunnel */ -#define ADP_DP_CS_3_HPD_PROPAGATE BIT(10) /* host DP IN */ +/* + * Apple silicon host DP IN adapter: pulsing this tells the adapter to + * propagate HPD to ADP_DP_CS_2_HPD. Ported from aurora-silicon/linux#8 + * (t8103, hardware-tested). + */ +#define ADP_DP_CS_3_HPD_PROPAGATE BIT(10) +/* + * Apple silicon host: hold the DP OUT (hub-side) adapter's own link + * training off while a tunnel is up -- the host's DPTX trains the sink + * through the tunnel itself. Same source as HPD_PROPAGATE above. + */ +#define ADP_DP_CS_3_NO_AUTO_LT BIT(8) #define DP_LOCAL_CAP 0x04 #define DP_REMOTE_CAP 0x05 /* For DP IN adapter */ @@ -473,6 +482,11 @@ struct tb_regs_port_header { #define ADP_DP_CS_8 0x08 #define ADP_DP_CS_8_DPME BIT(30) #define ADP_DP_CS_8_DR BIT(31) +#define ADP_DP_CS_13 0x0d +/* USB4 CM Guide 5.4.1.4: DPTX Discovery Mode. Bit position not in that + * guide; bit 0 is the enable-style field used on other DP CS words. + */ +#define ADP_DP_CS_13_DPTX_DISCOVERY_MODE BIT(0) /* PCIe adapter registers */ #define ADP_PCIE_CS_0 0x00 diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index 3e5c229aa42f5..f9eb7cd1aae31 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -7,9 +7,12 @@ */ #include +#include #include #include #include +#include +#include #include #include "tunnel.h" @@ -98,6 +101,27 @@ module_param(bw_alloc_mode, bool, 0444); MODULE_PARM_DESC(bw_alloc_mode, "enable bandwidth allocation mode if supported (default: true)"); +static bool dp_video_counter; +module_param(dp_video_counter, bool, 0444); +MODULE_PARM_DESC(dp_video_counter, + "diagnostic: count packets on the DP video path's DP IN hop and its downstream (hub-side) hop (Apple j416s right ACIO route only; read via debugfs port counters); default: false"); + +static void tb_dp_dump_apple(struct tb_tunnel *tunnel); +static int tb_apple_nhi_typec_index(struct tb_nhi *nhi); + +static bool tb_nhi_is_apple(const struct tb_nhi *nhi) +{ + struct device_node *np; + + if (!nhi || !nhi->dev || !nhi->ops || + !nhi->ops->dp_tunnel_post_activate) + return false; + np = nhi->dev->of_node; + if (!np && nhi->dev->parent) + np = nhi->dev->parent->of_node; + return np && of_device_is_compatible(np, "apple,t8103-usb4-nhi"); +} + static const char * const tb_tunnel_names[] = { "PCI", "DP", "DMA", "USB3" }; static const char * const tb_event_names[] = { @@ -947,10 +971,6 @@ static int tb_dp_xchg_caps(struct tb_tunnel *tunnel) * Titan Ridge does not disable AUX timers when it gets * SET_CONFIG with SET_LTTPR_MODE set. This causes problems with * DP tunneling. - * - * Not with an Apple host: its DPTX trains the sink through the - * tunnel itself, and the configuration validated on hardware leaves - * the capability alone. */ if (tb_route(out->sw) && tb_switch_is_titan_ridge(out->sw) && !tb_port_is_apple_host_dpin(in)) { @@ -962,9 +982,30 @@ static int tb_dp_xchg_caps(struct tb_tunnel *tunnel) in->cap_adap + DP_REMOTE_CAP, 1); } +/* Apple j416s right-hand ACIO DP IN adapter only; shared by the diagnostic + * packet counter and the bandwidth-grant workaround below. + */ +static bool tb_dp_is_apple_j416s_right_dpin(const struct tb_port *in) +{ + if (!tb_port_is_dpin(in)) + return false; + if (!in->sw->tb || !tb_nhi_is_apple(in->sw->tb->nhi)) + return false; + if (!of_machine_is_compatible("apple,j416s")) + return false; + + /* Right-hand USB-C ports only ("f01f" NHI); see tb_apple_nhi_typec_index(). */ + return tb_apple_nhi_typec_index(in->sw->tb->nhi) == 2; +} + +static bool tb_dp_apple_dpin_needs_bw_grant(const struct tb_port *in) +{ + return tb_dp_is_apple_j416s_right_dpin(in); +} + static int tb_dp_bandwidth_alloc_mode_enable(struct tb_tunnel *tunnel) { - int ret, estimated_bw, granularity, tmp; + int ret, estimated_bw, granularity, tmp, non_reduced_bw; struct tb_port *out = tunnel->dst_port; struct tb_port *in = tunnel->src_port; u32 out_dp_cap, out_rate, out_lanes; @@ -1004,6 +1045,7 @@ static int tb_dp_bandwidth_alloc_mode_enable(struct tb_tunnel *tunnel) rate = min(in_rate, out_rate); lanes = min(in_lanes, out_lanes); tmp = tb_dp_bandwidth(rate, lanes); + non_reduced_bw = tmp; tb_tunnel_dbg(tunnel, "non-reduced bandwidth %u Mb/s x%u = %u Mb/s\n", rate, lanes, tmp); @@ -1055,8 +1097,21 @@ static int tb_dp_bandwidth_alloc_mode_enable(struct tb_tunnel *tunnel) if (ret) return ret; - /* Initial allocation should be 0 according the spec */ - ret = usb4_dp_port_allocate_bandwidth(in, 0); + /* + * Initial allocation should be 0 according the spec, which relies + * on the DP IN adapter later raising it via a bandwidth request + * notification. The Apple j416s right-hand ACIO DP IN adapter has + * never been observed to generate that notification: link training + * and a full DCP frame complete with this field (DP_STATUS + * allocated-bandwidth) still reading 0 and no picture. + * On that one route only, grant the already-computed non-reduced + * bandwidth immediately, capped at what the connection manager + * already reserved for this tunnel (estimated_bw), instead of + * waiting indefinitely for a request that does not arrive. + */ + tmp = tb_dp_apple_dpin_needs_bw_grant(in) ? + min(non_reduced_bw, estimated_bw) : 0; + ret = usb4_dp_port_allocate_bandwidth(in, tmp); if (ret) return ret; @@ -1157,6 +1212,12 @@ static void tb_dp_dprx_work(struct work_struct *work) mutex_unlock(&tb->lock); return; } + if (tb_nhi_is_apple(tb->nhi)) { + tb_tunnel_warn(tunnel, + "Apple: DPRX timeout, keeping DP tunnel\n"); + tb_dp_dump_apple(tunnel); + tb_tunnel_set_active(tunnel, true); + } } else { tb_tunnel_set_active(tunnel, true); } @@ -1170,6 +1231,12 @@ static void tb_dp_dprx_work(struct work_struct *work) static int tb_dp_dprx_start(struct tb_tunnel *tunnel) { + if (tb_nhi_is_apple(tunnel->tb->nhi)) { + tb_tunnel_warn(tunnel, + "Apple: DP tunnel paths up, not waiting for DPRX\n"); + tb_tunnel_set_active(tunnel, true); + } + /* * Bump up the references to keep the tunnel and the domain around * until the work has run or has been canceled. @@ -1199,12 +1266,61 @@ static void tb_dp_dprx_stop(struct tb_tunnel *tunnel) } } +/* + * Apple silicon host DP IN adapter: this adapter has no real physical DP + * connector wired to it, so nothing ever sets its own ADP_DP_CS_2_HPD -- + * pulse ADP_DP_CS_3_HPD_PROPAGATE to tell it to propagate HPD itself, and + * wait for it to take, same as aurora-silicon/linux#8's hardware-tested + * t8103 implementation (three docks, multiple monitors). On the + * dcpext1/right-port path, the CS0-CS13 registers this pulse is meant to + * move have been observed to stay completely static across every + * failure, unlike the working dcpext0/left-port case. Logs and continues + * either way; a failed pulse is not treated as a fatal tunnel-activation + * error, matching the reference. + */ +static void tb_dp_apple_pulse_hpd(struct tb_port *in) +{ + int i, hpd = 0; + u32 v; + + if (!in->cap_adap) + return; + if (tb_port_read(in, &v, TB_CFG_PORT, in->cap_adap + ADP_DP_CS_3, 1)) { + tb_port_warn(in, "Apple: cannot read DP adapter state, HPD not pulsed\n"); + return; + } + /* HPDC may still be set from a previous tunnel's teardown */ + v &= ~ADP_DP_CS_3_HPDC; + v |= ADP_DP_CS_3_HPD_PROPAGATE; + if (tb_port_write(in, &v, TB_CFG_PORT, in->cap_adap + ADP_DP_CS_3, 1)) { + tb_port_warn(in, "Apple: cannot pulse HPD propagation\n"); + return; + } + usleep_range(10000, 11000); + v &= ~ADP_DP_CS_3_HPD_PROPAGATE; + if (tb_port_write(in, &v, TB_CFG_PORT, in->cap_adap + ADP_DP_CS_3, 1)) + tb_port_warn(in, "Apple: cannot end HPD propagation pulse\n"); + for (i = 0; i < 200; i++) { + hpd = tb_dp_port_hpd_is_active(in); + if (hpd) + break; + usleep_range(10000, 11000); + } + if (hpd < 0) + tb_port_warn(in, "Apple: cannot read HPD status: %d\n", hpd); + else if (!hpd) + tb_port_warn(in, "Apple: HPD did not propagate\n"); + else + tb_port_info(in, "Apple: HPD propagated\n"); +} + static int tb_dp_activate(struct tb_tunnel *tunnel, bool active) { int ret; if (active) { struct tb_path **paths; + const struct tb_nhi_ops *ops; int last; paths = tunnel->paths; @@ -1219,6 +1335,15 @@ static int tb_dp_activate(struct tb_tunnel *tunnel, bool active) paths[TB_DP_VIDEO_PATH_OUT]->hops[last].next_hop_index, paths[TB_DP_AUX_PATH_IN]->hops[0].in_hop_index, paths[TB_DP_AUX_PATH_OUT]->hops[last].next_hop_index); + + ops = tunnel->tb->nhi->ops; + if (ops && ops->dp_tunnel_pre_activate) { + ret = ops->dp_tunnel_pre_activate( + tunnel->tb->nhi, tunnel->src_port, + tunnel->dst_port); + if (ret) + return ret; + } } else { tb_dp_dprx_stop(tunnel); tb_dp_port_hpd_clear(tunnel->src_port); @@ -1228,40 +1353,79 @@ static int tb_dp_activate(struct tb_tunnel *tunnel, bool active) } ret = tb_dp_port_enable(tunnel->src_port, active); - if (ret) + /* + * On deactivate, a departing dst_port already marked unplugged + * makes tb_port_read/write short-circuit to -ENODEV with zero I/O + * (tb.h) -- deterministically, not just on hardware timing. Do not + * let that skip ops->dp_tunnel_deactivate() below: this tunnel.c is + * generic core code, but ops->dp_tunnel_deactivate is what apple.c + * relies on to clear its own apple_dpin_ctx latch -- without it, a + * physical unplug/replug can never re-arm a fresh connect. The caller + * (tb_tunnel_deactivate()) already discards this function's return + * value on the deactivate path, so relaxing it here changes nothing + * any caller observes; the activate (active=true) path is untouched. + */ + if (ret && active) return ret; if (tb_port_is_dpout(tunnel->dst_port)) { struct tb_port *out = tunnel->dst_port; + bool apple_dpin = (tb_nhi_is_apple(tunnel->tb->nhi) || + tb_port_is_apple_host_dpin(tunnel->src_port)) && + tb_port_is_dpin(tunnel->src_port) && + out->cap_adap; + u32 v; /* - * With an Apple host the sink link is trained by the host's - * DPTX through the tunnel: keep the DP OUT adapter from - * starting link training on its own, and hand it back once - * the tunnel is gone. + * Apple silicon host: the sink link is trained by the + * host's DPTX through the tunnel; keep this DP OUT adapter + * from starting link training on its own, and hand it back + * once the tunnel is gone. Ported from aurora-silicon/ + * linux#8. */ - bool apple = tb_port_is_apple_host_dpin(tunnel->src_port) && - out->cap_adap; - u32 v; - - if (active && apple && + if (active && apple_dpin && !tb_port_read(out, &v, TB_CFG_PORT, out->cap_adap + ADP_DP_CS_3, 1)) { v |= ADP_DP_CS_3_NO_AUTO_LT; if (tb_port_write(out, &v, TB_CFG_PORT, out->cap_adap + ADP_DP_CS_3, 1)) - tb_port_warn(out, "cannot hold off DP link training\n"); + tb_port_warn(out, "Apple: cannot hold off DP link training\n"); } ret = tb_dp_port_enable(out, active); /* hand link training back even if the disable failed */ - if (!active && apple && + if (!active && apple_dpin && !tb_port_read(out, &v, TB_CFG_PORT, out->cap_adap + ADP_DP_CS_3, 1)) { v &= ~ADP_DP_CS_3_NO_AUTO_LT; if (tb_port_write(out, &v, TB_CFG_PORT, out->cap_adap + ADP_DP_CS_3, 1)) - tb_port_warn(out, "cannot restore DP link training\n"); + tb_port_warn(out, "Apple: cannot restore DP link training\n"); } - if (ret) + if (ret && active) return ret; } + if (active && tb_nhi_is_apple(tunnel->tb->nhi)) { + if (tb_port_is_dpin(tunnel->src_port)) + tb_dp_apple_pulse_hpd(tunnel->src_port); + tb_dp_dump_apple(tunnel); + } + + if (active) { + const struct tb_nhi_ops *ops = tunnel->tb->nhi->ops; + + if (ops && ops->dp_tunnel_post_activate) { + ret = ops->dp_tunnel_post_activate(tunnel->tb->nhi, + tunnel->src_port, + tunnel->dst_port); + if (ret) + return ret; + } + } else { + const struct tb_nhi_ops *ops = tunnel->tb->nhi->ops; + + if (ops && ops->dp_tunnel_deactivate) + ops->dp_tunnel_deactivate(tunnel->tb->nhi, + tunnel->src_port, + tunnel->dst_port); + } + return active ? tb_dp_dprx_start(tunnel) : 0; } @@ -1568,8 +1732,13 @@ static int tb_dp_init_video_credits(struct tb_path_hop *hop) struct tb_port *port = hop->in_port; struct tb_switch *sw = port->sw; - /* Apple host DP IN adapters take 5 NFC credits for the video path. */ - if (tb_port_is_apple_host_dpin(port)) { + /* + * Apple silicon host DP IN adapters take 5 NFC credits for the + * video path -- same source as the HPD/NO_AUTO_LT handling above. + */ + if (tb_port_is_apple_host_dpin(port) || + (port->sw->tb && tb_nhi_is_apple(port->sw->tb->nhi) && + !tb_route(port->sw) && tb_port_is_dpin(port))) { hop->nfc_credits = 5; return 0; } @@ -1598,6 +1767,14 @@ static int tb_dp_init_video_credits(struct tb_path_hop *hop) return 0; } +static bool tb_dp_video_counter_wanted(const struct tb_path *path) +{ + if (!dp_video_counter || !path->path_length) + return false; + + return tb_dp_is_apple_j416s_right_dpin(path->hops[0].in_port); +} + static int tb_dp_init_video_path(struct tb_path *path, bool pm_support) { struct tb_path_hop *hop; @@ -1609,6 +1786,27 @@ static int tb_dp_init_video_path(struct tb_path *path, bool pm_support) path->priority = TB_DP_VIDEO_PRIORITY; path->weight = TB_DP_VIDEO_WEIGHT; + if (tb_dp_video_counter_wanted(path)) { + struct tb_path_hop *last = &path->hops[path->path_length - 1]; + + path->hops[0].in_counter_index = 0; + tb_port_dbg(path->hops[0].in_port, + "dp_video_counter: enabling counter 0 on DP IN hop\n"); + + /* + * Second checkpoint: the downstream router's ingress side of + * the same path (e.g. the hub's upstream link), so comparing + * its counter against the first checkpoint can show whether + * DP-IN traffic actually crosses into the far end, not just + * leaves the host. + */ + if (last != &path->hops[0]) { + last->in_counter_index = 0; + tb_port_dbg(last->in_port, + "dp_video_counter: enabling counter 0 on downstream hop\n"); + } + } + tb_path_for_each_hop(path, hop) { int ret; @@ -1622,6 +1820,59 @@ static int tb_dp_init_video_path(struct tb_path *path, bool pm_support) return 0; } +static void tb_dp_dump_apple_adapter(struct tb_port *port, const char *tag) +{ + u32 w[9]; + int i, ret; + + for (i = 0; i <= 8; i++) { + ret = tb_port_read(port, &w[i], TB_CFG_PORT, + port->cap_adap + i, 1); + if (ret) + w[i] = 0xffffffff; + } + tb_port_dbg(port, + "%s CS0=%08x CS1=%08x CS2=%08x CS3=%08x LOCAL=%08x REMOTE=%08x STAT=%08x COMMON=%08x CS8=%08x VE=%u AE=%u HPD=%u DPRX=%u\n", + tag, w[0], w[1], w[2], w[3], w[4], w[5], w[6], w[7], w[8], + !!(w[0] & ADP_DP_CS_0_VE), !!(w[0] & ADP_DP_CS_0_AE), + !!(w[2] & ADP_DP_CS_2_HPD), + !!(w[7] & DP_COMMON_CAP_DPRX_DONE)); +} + +static int tb_apple_nhi_typec_index(struct tb_nhi *nhi) +{ + const char *name; + + if (!nhi || !nhi->dev) + return -1; + name = dev_name(nhi->dev); + if (strstr(name, "701f")) + return 0; + if (strstr(name, "b01f")) + return 1; + if (strstr(name, "f01f")) + return 2; + return -1; +} + +static void tb_dp_dump_apple(struct tb_tunnel *tunnel) +{ + u32 cs2 = 0; + int hpd; + + tb_dp_dump_apple_adapter(tunnel->src_port, "DP IN"); + if (tb_port_is_dpout(tunnel->dst_port)) + tb_dp_dump_apple_adapter(tunnel->dst_port, "DP OUT"); + + if (!tb_port_read(tunnel->src_port, &cs2, TB_CFG_PORT, + tunnel->src_port->cap_adap + ADP_DP_CS_2, 1)) { + hpd = !!(cs2 & ADP_DP_CS_2_HPD); + tb_tunnel_dbg(tunnel, + "Apple: DP IN HPD=%d typec=%d (DPTX should use this adapter, not ATC)\n", + hpd, tb_apple_nhi_typec_index(tunnel->tb->nhi)); + } +} + static void tb_dp_dump(struct tb_tunnel *tunnel) { struct tb_port *in, *out; From c069e7846564ebb5c23737621206652c3d1e6971 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Thu, 24 Sep 2026 16:47:52 +0200 Subject: [PATCH 0967/2680] thunderbolt: apple: route Thunderbolt DP tunnels to a display pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire up the tb_nhi_ops dp_tunnel_pre_activate/post_activate/deactivate hooks so a DisplayPort tunnel programmed by the Thunderbolt connection manager -- whether the monitor hangs off the host router directly or off a Thunderbolt/USB4 hub -- gets a real display pipeline attached, instead of only working through direct HDMI or a USB-C alt-mode adapter as before. Add a small, self-contained helper (apple-dpin-handshake.h) that drives the ACIO DP IN adapter's DPTX_INACTIVE register handshake: HPD/CONTROL/ ACK plus the two link-rate mode fields that must be programmed before the adapter's crossbar output will pass a real display signal. The link-rate value is an empirically-determined estimate rather than a value confirmed against any spec or reference implementation; it is exposed as a module parameter (dpin_mode_value, default matching the value confirmed working on T602X/M2 Pro hardware) rather than hardcoded, in case other Apple silicon generations need a different value. apple_cio gains the Type-C connector node this router is wired to (found via the port@1 OF-graph endpoint to the PD controller) and a per-adapter context with its own lock, mapped-register pointer and lifetime flags, driven from a dedicated ordered workqueue so tunnel up/down events are serialized and never lost. When a tunnel activates, the driver maps the relevant DPIN handshake register block and calls into the DCP display driver's apple_dcp_tb_dp_tunnel() (resolved via symbol_get()/symbol_put(), so neither module hard-depends on the other being loaded) with a callback that runs the DPTX_INACTIVE handshake at the point DCP itself decides to bring the link up or down; a bounded retry loop covers DCP still probing when a dock is already present at boot. Deactivation reverses this in mirror order once the Thunderbolt core has already disabled both tunnel adapters. Give apple_cio's RTKit ops a recv_message callback that logs any message the ACIO M3 coprocessor sends unprompted, for visibility into ACIO's own asynchronous notifications. [ryan: adapted to 7.2.10 — this series and the parallel t8103 DP tunnel series were reconciled by Chris Kearney in merge cf5f7d19399e (per-SoC apple_dpin_set_active(), the DP tunnel hooks installed only on j416s, one copy of the handshake helpers); this port applies that merged apple.c, without the retry of 69893ec95743, which follows] Co-Authored-By: Claude Sonnet 5 --- drivers/thunderbolt/apple-dpin-handshake.h | 168 ++++++ drivers/thunderbolt/apple.c | 561 ++++++++++++++++++++- 2 files changed, 722 insertions(+), 7 deletions(-) create mode 100644 drivers/thunderbolt/apple-dpin-handshake.h diff --git a/drivers/thunderbolt/apple-dpin-handshake.h b/drivers/thunderbolt/apple-dpin-handshake.h new file mode 100644 index 0000000000000..4e492b2cc5802 --- /dev/null +++ b/drivers/thunderbolt/apple-dpin-handshake.h @@ -0,0 +1,168 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _APPLE_DPIN_HANDSHAKE_H +#define _APPLE_DPIN_HANDSHAKE_H + +#include + +#define APPLE_DPIN_HPD 0x00 +#define APPLE_DPIN_CONTROL 0x0c +#define APPLE_DPIN_ACK 0x10 +#define APPLE_DPIN_HPD_LEVEL (1U << 2) +#define APPLE_DPIN_INACTIVE 1U + +/* + * Set unconditionally by native AppleCIODPTX::bringConnectionUp on both the + * HPD and CONTROL registers of this exact DPIN0 block when bringing a + * connection up (single-lane-group case, which is what this port is). The + * HPD-register write is additionally gated there on a single/multi-stream + * check; every topology this driver drives through DPIN0 is a single + * external monitor (SST), so it is unconditional here too. Neither write + * clears any other bit (native masks both with 0, a pure OR). Not written + * on deactivate: native's teardown path was not traced, so this preserves + * the existing "do not invent resets" behavior there. + */ +#define APPLE_DPIN_CONNECTED (1U << 1) + +/* + * Two further offsets native AppleCIODPTX::bringConnectionUp writes on + * this exact DPIN0 block, in between the HPD and CONTROL writes above + * (native order: HPD, then MODE_B, then MODE_A, then CONTROL). Unlike + * CONNECTED, the exact values here are NOT confirmed from the native + * binary: native computes them from a per-connection attributes value + * (rate class and an unidentified secondary field) that is copied + * verbatim from this connection's negotiation and is not itself built + * anywhere in the traced kernel/kext code -- its origin could not be + * pinned down statically (most likely the separate DCP coprocessor + * firmware). The value used is therefore an informed, explicitly-labeled + * estimate, not a confirmed constant, and is passed in by the caller + * (mode_value below) rather than fixed at compile time, so the bounded + * formula's one free parameter can be swept across a live connection + * without reinstalling or rebooting between values: + * - bits 4-7 of the field select a rate class using the same RBR=0/ + * HBR=1/HBR2=2/HBR3=3 ordinal already used elsewhere in this driver + * (drivers/thunderbolt/tb_regs.h DP_COMMON_CAP_RATE_*); this link + * negotiates HBR2, so 2 -- high confidence, unchanged across the + * whole sweep. + * - a secondary bit, native-gated on lane_count>=2 (true here: 4) and + * on the same "which DPIN0 sub-instance" selector already confirmed + * unconditional-0 for this single, non-split port, is set from a + * nearby field that other native code also treats as a small, + * 3-valid-value enumeration (0, 1, or 2) -- this is the weak half of + * the formula and the only free parameter, swept across + * mode_value = rate_class(2) * lane_count(4) + secondary_bit = 8, 9 + * (both already tested via separate reboots, each a clean boot with + * no picture), or 10. + * + * mode_value is bounded to APPLE_DPIN_MODE_VALUE_MAX: this caps both the + * single bit MODE_A can set (1 << mode_value) and the field width MODE_B + * ORs in, which is also exactly the range the deactivate path below + * knows how to clear back to a clean baseline. Raising this bound is a + * new, reviewable change, not a runtime knob. + */ +#define APPLE_DPIN_MODE_A 0x14 +#define APPLE_DPIN_MODE_B 0x1c +#define APPLE_DPIN_MODE_VALUE_MAX 15U + +/* Caller owns powered register access and provides a bounded wait. */ +struct apple_dpin_io { + unsigned int (*read)(void *ctx, unsigned int offset); + void (*write)(void *ctx, unsigned int offset, unsigned int value); + int (*wait)(void *ctx); + void *ctx; +}; + +static inline int apple_dpin_handshake(const struct apple_dpin_io *io, + int active, unsigned int mode_value) +{ + unsigned int hpd, saved, value, ack; + int ret; + + if (active && mode_value > APPLE_DPIN_MODE_VALUE_MAX) + return -EINVAL; + + if (active) { + hpd = io->read(io->ctx, APPLE_DPIN_HPD); + if (hpd == ~0U) + return -EIO; + if (!(hpd & APPLE_DPIN_HPD_LEVEL)) + return -ENOLINK; + } + saved = io->read(io->ctx, APPLE_DPIN_CONTROL); + if (saved == ~0U) + return -EIO; + if (active) { + unsigned int mode_a, mode_b; + + io->write(io->ctx, APPLE_DPIN_HPD, hpd | APPLE_DPIN_CONNECTED); + + mode_b = io->read(io->ctx, APPLE_DPIN_MODE_B); + if (mode_b != ~0U) + io->write(io->ctx, APPLE_DPIN_MODE_B, + mode_b | (mode_value << 7)); + + mode_a = io->read(io->ctx, APPLE_DPIN_MODE_A); + if (mode_a != ~0U) + io->write(io->ctx, APPLE_DPIN_MODE_A, + (mode_a & ~0xffU) | + (1U << mode_value)); + } else { + unsigned int mode_a, mode_b; + + /* + * Not native teardown behavior (never traced; see the + * CONNECTED comment above) -- this exists solely so a later + * activate on this same boot starts from clean state and is + * a valid isolated test of a different mode_value. Bounded + * to exactly the bits any in-range mode_value write above + * could have set: MODE_B's OR'd field (bits 7..7+MAX's + * width) and MODE_A's cleared-low-byte-plus-one-set-bit + * (bits 0..MAX). Never touches bits outside that range. + */ + mode_b = io->read(io->ctx, APPLE_DPIN_MODE_B); + if (mode_b != ~0U) + io->write(io->ctx, APPLE_DPIN_MODE_B, + mode_b & ~(APPLE_DPIN_MODE_VALUE_MAX << 7)); + + mode_a = io->read(io->ctx, APPLE_DPIN_MODE_A); + if (mode_a != ~0U) + io->write(io->ctx, APPLE_DPIN_MODE_A, + mode_a & ~((1U << (APPLE_DPIN_MODE_VALUE_MAX + 1)) - 1)); + } + value = active ? (saved & ~APPLE_DPIN_INACTIVE) | APPLE_DPIN_CONNECTED : + saved | APPLE_DPIN_INACTIVE; + io->write(io->ctx, APPLE_DPIN_CONTROL, value); + for (;;) { + ack = io->read(io->ctx, APPLE_DPIN_ACK); + if (ack == ~0U) { + ret = -EIO; + break; + } + if ((ack & APPLE_DPIN_INACTIVE) == + (value & APPLE_DPIN_INACTIVE)) + return 0; + if (active) { + hpd = io->read(io->ctx, APPLE_DPIN_HPD); + if (hpd == ~0U || !(hpd & APPLE_DPIN_HPD_LEVEL)) { + ret = hpd == ~0U ? -EIO : -ENOLINK; + break; + } + } + ret = io->wait(io->ctx); + if (ret) + break; + } + /* + * Restore only the bits we own (INACTIVE, and CONNECTED now that we + * set it above); do not invent resets on a failed handshake. HPD is + * left as-is here: native's teardown path for it was not traced, so + * no rollback is invented for a register this driver otherwise never + * wrote before this change. + */ + ack = io->read(io->ctx, APPLE_DPIN_CONTROL); + if (ack != ~0U) + io->write(io->ctx, APPLE_DPIN_CONTROL, + (ack & ~(APPLE_DPIN_INACTIVE | APPLE_DPIN_CONNECTED)) | + (saved & (APPLE_DPIN_INACTIVE | APPLE_DPIN_CONNECTED))); + return ret; +} +#endif diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index e32905925af12..7b636bb36a673 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -85,6 +85,7 @@ #include #include #include +#include #include #include #include @@ -104,6 +105,7 @@ #include #include +#include "apple-dpin-handshake.h" #include "nhi.h" #include "tb.h" @@ -151,6 +153,11 @@ #define TB_VSE_CAP_APPLE_CABLE_INFO_LEGACY_ADAPTER BIT(9) #define TB_VSE_CAP_APPLE_CABLE_INFO_TBT2_3 BIT(10) +static unsigned int dpin_mode_value = 8; +module_param(dpin_mode_value, uint, 0644); +MODULE_PARM_DESC(dpin_mode_value, + "DPIN0 MODE_A/MODE_B lab value (0-" __stringify(APPLE_DPIN_MODE_VALUE_MAX) "); read at each activate"); + /** * struct apple_cio - Apple Converged I/O host router complex * @dev: ACIO device @@ -240,7 +247,6 @@ struct apple_cio { #define APPLE_DPIN_IRQ_ENABLE 0x08 #define APPLE_DPIN_CTRL 0x0c #define APPLE_DPIN_CTRL_INACTIVE BIT(0) -#define APPLE_DPIN_ACK 0x10 #define APPLE_DPIN_ACK_INACTIVE BIT(0) static bool dp_display; @@ -248,7 +254,57 @@ module_param(dp_display, bool, 0444); MODULE_PARM_DESC(dp_display, "Drive displays behind Thunderbolt DP tunnels (experimental, t8103)"); /* DPTX_INACTIVE handshake: request (in)active, wait for the ACK */ -static int apple_dpin_set_active(struct apple_cio *acio, void __iomem *dpin, +struct apple_dpin_poll { + void __iomem *base; + unsigned long deadline; +}; + +static unsigned int apple_dpin_read(void *ctx, unsigned int offset) +{ + struct apple_dpin_poll *poll = ctx; + + return readl(poll->base + offset); +} + +static void apple_dpin_write(void *ctx, unsigned int offset, unsigned int value) +{ + struct apple_dpin_poll *poll = ctx; + + writel(value, poll->base + offset); +} + +static int apple_dpin_wait(void *ctx) +{ + struct apple_dpin_poll *poll = ctx; + + if (time_after_eq(jiffies, poll->deadline)) + return -ETIMEDOUT; + usleep_range(1000, 2000); + return 0; +} + +static int apple_dpin_set_active_t602x(struct apple_cio *acio, void __iomem *regs, + unsigned int idx, bool active) +{ + struct apple_dpin_poll poll = { + .base = regs, + .deadline = jiffies + msecs_to_jiffies(1000), + }; + struct apple_dpin_io io = { + .read = apple_dpin_read, + .write = apple_dpin_write, + .wait = apple_dpin_wait, + .ctx = &poll, + }; + int ret; + + ret = apple_dpin_handshake(&io, active, dpin_mode_value); + dev_info(acio->dev, "dpin%u: %s handshake=%d\n", idx, + active ? "active" : "inactive", ret); + return ret; +} + +static int apple_dpin_set_active_t8103(struct apple_cio *acio, void __iomem *dpin, unsigned int idx, bool active) { u32 want = active ? 0 : APPLE_DPIN_ACK_INACTIVE; @@ -292,6 +348,14 @@ static int apple_dpin_set_active(struct apple_cio *acio, void __iomem *dpin, return 0; } +static int apple_dpin_set_active(struct apple_cio *acio, void __iomem *regs, + unsigned int idx, bool active) +{ + if (of_machine_is_compatible("apple,j416s")) + return apple_dpin_set_active_t602x(acio, regs, idx, active); + return apple_dpin_set_active_t8103(acio, regs, idx, active); +} + /* Called by appledrm from DCP's Activate/Deactivate calls. */ static int apple_dpin_dcp_set_active(void *data, bool active) { @@ -368,11 +432,13 @@ static int apple_dpin_up(struct apple_dpin_ctx *c) * back. Nothing handles these interrupts yet; they are enabled * as part of bringing the adapter up. */ - irq = readl(regs + APPLE_DPIN_IRQ_STATUS); - writel(irq, regs + APPLE_DPIN_IRQ_STATUS); - if (irq & BIT(0)) - writel(readl(regs + APPLE_DPIN_STATUS), regs + APPLE_DPIN_STATUS); - writel(readl(regs + APPLE_DPIN_IRQ_ENABLE) | 3, regs + APPLE_DPIN_IRQ_ENABLE); + if (of_machine_is_compatible("apple,t8103")) { + irq = readl(regs + APPLE_DPIN_IRQ_STATUS); + writel(irq, regs + APPLE_DPIN_IRQ_STATUS); + if (irq & BIT(0)) + writel(readl(regs + APPLE_DPIN_STATUS), regs + APPLE_DPIN_STATUS); + writel(readl(regs + APPLE_DPIN_IRQ_ENABLE) | 3, regs + APPLE_DPIN_IRQ_ENABLE); + } } ret = apple_dpin_connect(c, true); @@ -554,6 +620,14 @@ struct apple_nhi { const char **tx_irq_names; const char **rx_irq_names; size_t n_rings; + + /* DP IN analog/AUX: poll adapter CS after the tunnel is up. */ + struct delayed_work dp_aux_work; + u8 dp_in_port; + u32 analog_base; + u32 dp_in_cs[14]; + bool dp_aux_armed; + unsigned int dp_aux_polls; }; #define nhi_to_anhi(nhi_) container_of((nhi_), struct apple_nhi, nhi) @@ -972,6 +1046,469 @@ static int apple_nhi_pci_tunnel_post_activate(struct tb_nhi *nhi) return 0; } +#define APPLE_DP_AUX_POLL_MS 500 +#define APPLE_DP_AUX_POLL_MAX 24 + +/* + * apple,tunable-rc programs two analog PHY blocks inside the already-mapped + * ACIO RC window (0xc000). They match the two host DP IN adapters (0:5, 0:6). + * Dumping 0x00-0xff never saw them. Do not scan unmapped ACIO ranges. + */ +#define APPLE_CIO_DPIN0_ANALOG 0x4000 +#define APPLE_CIO_DPIN1_ANALOG 0x8000 +#define APPLE_CIO_DPIN_ANALOG_SIZE 0x80 +#define APPLE_CIO_DPIN_ANALOG_FSM 0x18 +#define APPLE_CIO_DPIN_ANALOG_HOLE 0x20 +#define APPLE_CIO_DPIN_ANALOG_EMPTY 0x80000000 +#define APPLE_CIO_DPIN_ANALOG_HOLE_VAL 0x40 + +/* + * Analog MMIO writes are closed: +0x00 / +0x18 / +0x20 do not stick. + * +0x18 is first-read status 0x1017 (read-to-clear). + * + * 0 = do not touch analog at tunnel-up; dump it only after DPRX timeout + * (default). Tests whether consuming +0x18 aborted a handshake. + * 1 = old pulse +0x00 / fill +0x20 (rollback) + */ +static int apple_dpin_aux; +static struct apple_nhi *apple_dpin_anhi; + +static int apple_dpin_aux_set(const char *val, const struct kernel_param *kp); +static const struct kernel_param_ops apple_dpin_aux_ops = { + .set = apple_dpin_aux_set, + .get = param_get_int, +}; +module_param_cb(dpin_aux, &apple_dpin_aux_ops, &apple_dpin_aux, 0644); +MODULE_PARM_DESC(dpin_aux, + "Apple DP IN analog: 0=hands-off until timeout, 1=pulse +0x00"); + +static void apple_dp_dump_hop(struct tb_port *port, unsigned int hopid) +{ + struct tb_regs_hop hop; + int ret; + + ret = tb_port_read(port, &hop, TB_CFG_HOPS, 2 * hopid, 2); + if (ret) { + tb_port_dbg(port, "DP IN hop %u read failed: %d\n", hopid, ret); + return; + } + tb_port_dbg(port, + "DP IN hop %u enable=%u out=%u next=%u credits=%u\n", + hopid, hop.enable, hop.out_port, hop.next_hop, + hop.initial_credits); +} + + + +static void apple_dp_dump_adapter(struct tb_port *port, const char *tag) +{ + u32 w[17], cs4 = 0xffffffff; + int i, ret; + + ret = tb_port_read(port, &w[0], TB_CFG_PORT, port->cap_adap, 1); + if (ret || w[0] == 0xffffffff) { + tb_port_dbg(port, "%s config space dead ret=%d (not DPRX)\n", + tag, ret); + return; + } + for (i = 1; i <= 16; i++) { + ret = tb_port_read(port, &w[i], TB_CFG_PORT, + port->cap_adap + i, 1); + if (ret) + w[i] = 0xffffffff; + } + tb_port_read(port, &cs4, TB_CFG_PORT, ADP_CS_4, 1); + tb_port_dbg(port, + "%s type=%06x cap=%d CS0=%08x CS1=%08x CS2=%08x CS3=%08x LOCAL=%08x REMOTE=%08x STAT=%08x COMMON=%08x CS8=%08x CS9=%08x CS10=%08x ADP_CS4=%08x VE=%u AE=%u HPD=%u DPRX=%u LCK=%u\n", + tag, port->config.type, port->cap_adap, + w[0], w[1], w[2], w[3], w[4], w[5], w[6], w[7], w[8], + w[9], w[10], cs4, + !!(w[0] & ADP_DP_CS_0_VE), !!(w[0] & ADP_DP_CS_0_AE), + !!(w[2] & ADP_DP_CS_2_HPD), + !!(w[7] & DP_COMMON_CAP_DPRX_DONE), + !!(cs4 & ADP_CS_4_LCK)); + tb_port_dbg(port, + "%s CS11=%08x CS12=%08x CS13=%08x CS14=%08x CS15=%08x CS16=%08x disc=%u\n", + tag, w[11], w[12], w[13], w[14], w[15], w[16], + !!(w[13] & ADP_DP_CS_13_DPTX_DISCOVERY_MODE)); +} + +static void apple_dp_dump_host_adapters(struct apple_nhi *anhi) +{ + struct tb_port *port; + + if (!anhi->tb || !anhi->tb->root_switch) + return; + + tb_switch_for_each_port(anhi->tb->root_switch, port) { + if (tb_port_is_dpin(port)) + apple_dp_dump_adapter(port, "host DP IN"); + else if (tb_port_is_dpout(port)) + apple_dp_dump_adapter(port, "host DP OUT"); + } +} + +static void apple_dp_dump_rc_range(struct apple_cio *acio, u32 base, u32 len, + const char *tag, bool skip_zero) +{ + char buf[320]; + int n = 0; + u32 off, size; + + if (!acio->rc_base || !acio->rc_res) + return; + size = resource_size(acio->rc_res); + if (base >= size) + return; + if (base + len > size) + len = size - base; + + for (off = 0; off < len; off += 4) { + u32 val = readl(acio->rc_base + base + off); + + if (skip_zero && !val) + continue; + n += scnprintf(buf + n, sizeof(buf) - n, " %03x=%08x", off, val); + if (n >= (int)sizeof(buf) - 24) { + dev_dbg(acio->dev, "ACIO RC %s 0x%x:%s\n", tag, base, buf); + n = 0; + buf[0] = '\0'; + } + } + if (n) + dev_dbg(acio->dev, "ACIO RC %s 0x%x:%s\n", tag, base, buf); +} + +static void apple_dp_dump_rc(struct apple_cio *acio) +{ + apple_dp_dump_rc_range(acio, 0, 0x100, "ctrl", true); +} + +static void apple_dp_dump_vse(struct tb_switch *sw) +{ + char buf[320]; + int cap, i, n = 0, ret; + u32 w; + + cap = tb_switch_find_vse_cap(sw, TB_VSE_CAP_APPLE); + if (cap < 0) { + tb_sw_dbg(sw, "Apple VSE cap missing: %d\n", cap); + return; + } + for (i = 0; i < 16; i++) { + ret = tb_sw_read(sw, &w, TB_CFG_SWITCH, cap + i, 1); + if (ret) + w = 0xffffffff; + n += scnprintf(buf + n, sizeof(buf) - n, " %02x=%08x", i, w); + if (n >= (int)sizeof(buf) - 20) { + tb_sw_dbg(sw, "Apple VSE cap=%d:%s\n", cap, buf); + n = 0; + buf[0] = '\0'; + } + } + if (n) + tb_sw_dbg(sw, "Apple VSE cap=%d:%s\n", cap, buf); +} + + + +static void apple_dp_dump_analog(struct apple_cio *acio, const char *tag) +{ + apple_dp_dump_rc_range(acio, APPLE_CIO_DPIN0_ANALOG, + APPLE_CIO_DPIN_ANALOG_SIZE, tag, false); + apple_dp_dump_rc_range(acio, APPLE_CIO_DPIN1_ANALOG, + APPLE_CIO_DPIN_ANALOG_SIZE, "dpin1 analog", false); +} + +static u32 apple_dp_in_analog_base(struct apple_nhi *anhi, struct tb_port *in) +{ + unsigned int idx = 0; + struct tb_port *port; + + if (!anhi->tb || !anhi->tb->root_switch) + return APPLE_CIO_DPIN0_ANALOG; + + tb_switch_for_each_port(anhi->tb->root_switch, port) { + if (!tb_port_is_dpin(port)) + continue; + if (port == in) + return idx ? APPLE_CIO_DPIN1_ANALOG : + APPLE_CIO_DPIN0_ANALOG; + idx++; + } + return APPLE_CIO_DPIN0_ANALOG; +} + +static void apple_dp_start_analog(struct apple_nhi *anhi, bool pulse) +{ + struct apple_cio *acio; + u32 block, ctrl, hole, fsm; + + if (!anhi || !anhi->acio || !anhi->acio->rc_base) + return; + acio = anhi->acio; + block = anhi->analog_base ?: APPLE_CIO_DPIN0_ANALOG; + if (block + APPLE_CIO_DPIN_ANALOG_HOLE + 4 > resource_size(acio->rc_res)) + return; + + ctrl = readl(acio->rc_base + block); + hole = readl(acio->rc_base + block + APPLE_CIO_DPIN_ANALOG_HOLE); + fsm = readl(acio->rc_base + block + APPLE_CIO_DPIN_ANALOG_FSM); + dev_info(acio->dev, + "DP IN analog +0x00=%08x +0x18=%08x +0x20=%08x pulse=%d\n", + ctrl, fsm, hole, pulse); + + if (!pulse) + return; + + /* +0x00 readback is empty status; still pulse start like PCIe-C. */ + writel(ctrl | BIT(0), acio->rc_base + block); + mb(); + ctrl = readl(acio->rc_base + block); + + if (hole == 0) { + writel(APPLE_CIO_DPIN_ANALOG_HOLE_VAL, + acio->rc_base + block + APPLE_CIO_DPIN_ANALOG_HOLE); + mb(); + hole = readl(acio->rc_base + block + APPLE_CIO_DPIN_ANALOG_HOLE); + } + + fsm = readl(acio->rc_base + block + APPLE_CIO_DPIN_ANALOG_FSM); + dev_info(acio->dev, + "DP IN analog after start +0x00=%08x +0x18=%08x +0x20=%08x\n", + ctrl, fsm, hole); + apple_dp_dump_rc_range(acio, block, APPLE_CIO_DPIN_ANALOG_SIZE, + "dpin analog after start", false); +} + +static int apple_dpin_aux_set(const char *val, const struct kernel_param *kp) +{ + struct apple_nhi *anhi; + int v, ret; + + ret = kstrtoint(val, 0, &v); + if (ret) + return ret; + apple_dpin_aux = v; + anhi = READ_ONCE(apple_dpin_anhi); + if (v >= 1 && anhi) + apple_dp_start_analog(anhi, true); + return 0; +} + +static int apple_dp_dptx_discover(struct tb_port *in) +{ + u32 cs0 = 0; + + tb_port_read(in, &cs0, TB_CFG_PORT, in->cap_adap + ADP_DP_CS_0, 1); + tb_port_warn(in, + "USB4 DPTX Discovery closed (CS13 writable, CS9 static); AE=%u VE=%u\n", + !!(cs0 & ADP_DP_CS_0_AE), !!(cs0 & ADP_DP_CS_0_VE)); + return 0; +} + +static void apple_dp_aux_work(struct work_struct *work) +{ + struct apple_nhi *anhi = + container_of(to_delayed_work(work), struct apple_nhi, + dp_aux_work); + struct tb_port *port; + u32 cs[14]; + int i, ret; + bool changed = false, dprx = false; + + if (!anhi->tb) + return; + + mutex_lock(&anhi->tb->lock); + if (!anhi->dp_aux_armed || !anhi->tb->root_switch) + goto out_unlock; + + port = &anhi->tb->root_switch->ports[anhi->dp_in_port]; + if (!tb_port_is_dpin(port)) + goto out_unlock; + + for (i = 0; i <= 13; i++) { + ret = tb_port_read(port, &cs[i], TB_CFG_PORT, + port->cap_adap + i, 1); + if (ret) + cs[i] = 0xffffffff; + if (cs[i] != anhi->dp_in_cs[i]) + changed = true; + } + if (changed) { + memcpy(anhi->dp_in_cs, cs, sizeof(cs)); + tb_port_dbg(port, + "DP IN CS changed CS0=%08x CS2=%08x CS9=%08x CS13=%08x COMMON=%08x VE=%u AE=%u HPD=%u DPRX=%u disc=%u\n", + cs[0], cs[2], cs[9], cs[13], cs[7], + !!(cs[0] & ADP_DP_CS_0_VE), + !!(cs[0] & ADP_DP_CS_0_AE), + !!(cs[2] & ADP_DP_CS_2_HPD), + !!(cs[7] & DP_COMMON_CAP_DPRX_DONE), + !!(cs[13] & ADP_DP_CS_13_DPTX_DISCOVERY_MODE)); + } + dprx = !!(cs[7] & DP_COMMON_CAP_DPRX_DONE); + if (dprx) + tb_port_dbg(port, "DP IN DPRX_DONE=1 (ACIO AUX completed)\n"); + + anhi->dp_aux_polls++; + /* + * Deliberately not dumping or reading the analog block on every + * poll here. +0x18 is an ordinary one-shot, read-to-clear status + * register -- already documented above (APPLE_CIO_DPIN_ANALOG_EMPTY, + * and the comment a few lines up: "+0x18 is first-read status + * 0x1017 (read-to-clear)"), not a stuck error FSM, so polling it + * repeatedly has no diagnostic value and risks consuming a status + * this project's own dpin_aux default-off comment already warns + * could abort a handshake if read too early. A write-1-to-clear + * ack against it has no effect either: 0x80000000 is the + * register's own idle/empty sentinel, not a latched error that + * needs acknowledging. Stick to a single post-mortem dump below, + * after the fact, once polling ends. + */ + if (!dprx && anhi->dp_aux_polls < APPLE_DP_AUX_POLL_MAX) { + mod_delayed_work(system_wq, &anhi->dp_aux_work, + msecs_to_jiffies(APPLE_DP_AUX_POLL_MS)); + } else if (anhi->acio) { + apple_dp_dump_analog(anhi->acio, + dprx ? "dpin0 analog DPRX done" : + "dpin0 analog at timeout"); + } + +out_unlock: + mutex_unlock(&anhi->tb->lock); +} + +/* + * Which DP IN adapter (0/1) on the host router this port is -- same + * counting order as apple_dp_in_analog_base() above, and as + * apple_nhi_dp_tunnel_changed()'s port-to-dpin mapping in + * aurora-silicon/linux#8. + */ +static int apple_dpin_index_for_port(struct apple_nhi *anhi, struct tb_port *in) +{ + unsigned int idx = 0; + struct tb_port *port; + + if (!anhi->tb || !anhi->tb->root_switch) + return -1; + + tb_switch_for_each_port(anhi->tb->root_switch, port) { + if (!tb_port_is_dpin(port)) + continue; + if (port == in) + return idx; + idx++; + } + return -1; +} + +static int apple_nhi_dp_tunnel_pre_activate(struct tb_nhi *nhi, + struct tb_port *in, + struct tb_port *out) +{ + if (tb_port_is_dpin(in)) + return apple_dp_dptx_discover(in); + return 0; +} + +static int apple_nhi_dp_tunnel_post_activate(struct tb_nhi *nhi, + struct tb_port *in, + struct tb_port *out) +{ + struct apple_nhi *anhi = nhi_to_anhi(nhi); + int i, idx; + + dev_info(anhi->dev, + "DP IN tunnel routing: tunnel %u:%u <-> %u:%u\n", + in->sw->config.depth, in->port, + out ? out->sw->config.depth : 0, + out ? out->port : 0); + + anhi->dp_in_port = in->port; + anhi->analog_base = apple_dp_in_analog_base(anhi, in); + anhi->dp_aux_polls = 0; + anhi->dp_aux_armed = true; + WRITE_ONCE(apple_dpin_anhi, anhi); + + apple_dp_dump_rc(anhi->acio); + apple_dp_dump_vse(in->sw); + + /* + * Route a real display pipeline to this tunnel, ported from + * aurora-silicon/linux#8 -- the dpin_aux "analog AUX serializer" + * mechanism previously here is confirmed ineffective on this + * hardware, including after everything else in the tunnel-up path + * was independently fixed. + */ + idx = apple_dpin_index_for_port(anhi, in); + if (idx < 0 || idx > 1 || !anhi->acio) { + dev_warn(anhi->dev, + "DP IN tunnel: could not map port %u to a dpin index\n", + in->port); + } else if (!anhi->acio->connector_np) { + dev_warn(anhi->dev, + "DP IN tunnel: no Type-C connector for dpin%d\n", idx); + } else if (!anhi->acio->dp_wq) { + dev_warn(anhi->dev, "DP IN tunnel: no work queue for dpin%d\n", idx); + } else { + struct apple_dpin_ctx *c = &anhi->acio->dpin[idx]; + + scoped_guard(mutex, &c->lock) + c->alive = true; + queue_work(anhi->acio->dp_wq, &c->work); + } + apple_dp_dump_host_adapters(anhi); + if (tb_port_is_dpin(in)) { + apple_dp_dump_hop(in, 8); + apple_dp_dump_hop(in, 9); + } + if (out && tb_port_is_dpout(out)) + apple_dp_dump_adapter(out, "hub DP OUT"); + + for (i = 0; i <= 13; i++) { + if (tb_port_read(in, &anhi->dp_in_cs[i], TB_CFG_PORT, + in->cap_adap + i, 1)) + anhi->dp_in_cs[i] = 0xffffffff; + } + mod_delayed_work(system_wq, &anhi->dp_aux_work, + msecs_to_jiffies(APPLE_DP_AUX_POLL_MS)); + return 0; +} + +static void apple_nhi_dp_tunnel_deactivate(struct tb_nhi *nhi, + struct tb_port *in, + struct tb_port *out) +{ + struct apple_nhi *anhi = nhi_to_anhi(nhi); + int idx; + + anhi->dp_aux_armed = false; + cancel_delayed_work(&anhi->dp_aux_work); + if (READ_ONCE(apple_dpin_anhi) == anhi) + WRITE_ONCE(apple_dpin_anhi, NULL); + + /* + * Runs after both adapters are already disabled (tb_dp_activate()'s + * teardown order) -- later than aurora-silicon/linux#8's own + * dp_tunnel_changed(active=false), which fires before deactivation + * while the block is still powered. apple_dpin_dcp_set_active()'s + * own "block may already be off" check (matching the reference + * implementation's identical defensive check) covers this gap; a + * spurious no-op register access here is not a functional blocker, + * only a less clean teardown than the ideal ordering. + */ + idx = apple_dpin_index_for_port(anhi, in); + if (idx >= 0 && idx <= 1 && anhi->acio && anhi->acio->dp_wq) { + struct apple_dpin_ctx *c = &anhi->acio->dpin[idx]; + + scoped_guard(mutex, &c->lock) + c->alive = false; + queue_work(anhi->acio->dp_wq, &c->work); + } + dev_info(anhi->dev, "DP IN tunnel routing: tunnel down\n"); +} + static void apple_nhi_dp_tunnel_changed(struct tb_nhi *nhi, u8 in_port, bool active) { @@ -1064,6 +1601,7 @@ static int apple_nhi_probe(struct platform_device *pdev) anhi->dev = &pdev->dev; anhi->np = pdev->dev.of_node; anhi->acio = acio; + INIT_DELAYED_WORK(&anhi->dp_aux_work, apple_dp_aux_work); platform_set_drvdata(pdev, anhi); ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(42)); @@ -1114,6 +1652,11 @@ static int apple_nhi_probe(struct platform_device *pdev) anhi->ops = apple_nhi_ops; if (dp_display && acio->dp_wq && of_machine_is_compatible("apple,t8103")) anhi->ops.dp_tunnel_changed = apple_nhi_dp_tunnel_changed; + if (acio->dp_wq && of_machine_is_compatible("apple,j416s")) { + anhi->ops.dp_tunnel_pre_activate = apple_nhi_dp_tunnel_pre_activate; + anhi->ops.dp_tunnel_post_activate = apple_nhi_dp_tunnel_post_activate; + anhi->ops.dp_tunnel_deactivate = apple_nhi_dp_tunnel_deactivate; + } anhi->nhi.ops = &anhi->ops; anhi->nhi.ring_layout = &apple_nhi_ring_layout; anhi->nhi.iobase = anhi->nhi_base; @@ -1212,6 +1755,10 @@ static void apple_nhi_remove(struct platform_device *pdev) { struct apple_nhi *anhi = platform_get_drvdata(pdev); + anhi->dp_aux_armed = false; + cancel_delayed_work_sync(&anhi->dp_aux_work); + if (READ_ONCE(apple_dpin_anhi) == anhi) + WRITE_ONCE(apple_dpin_anhi, NULL); WRITE_ONCE(anhi->acio->nhi_pdev, NULL); tb_domain_remove(anhi->tb); wait_for_completion(&anhi->nhi.domain_released); From 451cc5b42f16c592d7cb4c0232b8b63e815f9ddf Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Thu, 24 Sep 2026 16:48:39 +0200 Subject: [PATCH 0968/2680] mux: apple-display-crossbar: route T602X DPIN0/DPIN1 through the mux MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit apple_dpxbar_set_t602x() only ever programmed the DPPHY fields of the mux control register and its ATC bit, regardless of which of the three physical outputs (dpphy, dpin0, dpin1) was actually selected. Selecting dpin0 or dpin1 therefore only updated driver-internal bookkeeping without ever routing analog onto the ACIO DP IN pins, and the enable/disable paths never touched the enable bits the generic (non-T602X) crossbar path does. Add helpers to resolve the correct ATC bit and mux-select field for the index actually being switched, and use them in both the enable and disable sequences, including the enable-bit sequencing that was previously only done for the generic crossbar. Reject an out-of-range index up front rather than proceeding with an unresolved mux mask. Add apple_dpxbar_link_up()/apple_dpxbar_link_down(), exported for the DCP driver to reach through symbol_get()/symbol_put() so the crossbar driver does not need to be a hard module dependency of the display driver. A Thunderbolt DP tunnel re-link needs to bounce the crossbar's clock/FIFO gates on an output that is already selected, without reselecting the mux or dropping the enable bits -- a full teardown/ re-enable would drop the route entirely. Both functions operate on whichever output the mux already has recorded as selected. DPIN0's field masks and ATC bit are hardware-confirmed on T602X/M2 Pro; DPIN1's are filled in by the same per-index resolution by symmetry with DPPHY/DPIN0's layout, not by independent confirmation on real hardware -- flagging this here since getting a field encoding wrong on this hardware is silent (no error, just no picture). [ryan: adapted to 7.2.10 — the parallel t8103 link helpers (6f787d45a116) and these T602X ones were reconciled by Chris Kearney in merge cf5f7d19399e into per-SoC helpers behind apple_dpxbar_link_up()/_down(); this port applies that merged result] Co-Authored-By: Claude Sonnet 5 --- drivers/mux/apple-display-crossbar.c | 263 ++++++++++++++++++++++++--- 1 file changed, 233 insertions(+), 30 deletions(-) diff --git a/drivers/mux/apple-display-crossbar.c b/drivers/mux/apple-display-crossbar.c index 52d20235a6ef0..31788a927ea74 100644 --- a/drivers/mux/apple-display-crossbar.c +++ b/drivers/mux/apple-display-crossbar.c @@ -7,6 +7,7 @@ * Author: Sven Peter */ +#include #include #include #include @@ -119,6 +120,82 @@ static inline void dpxbar_clear32(struct apple_dpxbar *xbar, u32 reg, u32 clear) dpxbar_mask32(xbar, reg, clear, 0); } +static u32 t602x_atc_bit(unsigned int index) +{ + switch (index) { + case MUX_DPIN0: + return ATC_DPIN0; + case MUX_DPIN1: + return ATC_DPIN1; + case MUX_DPPHY: + default: + return ATC_DPPHY; + } +} + +static u32 t602x_mux_mask(unsigned int index) +{ + switch (index) { + case MUX_DPIN0: + return CROSSBAR_MUX_CTRL_DPIN0_SELECT0 | + CROSSBAR_MUX_CTRL_DPIN0_SELECT1; + case MUX_DPIN1: + return CROSSBAR_MUX_CTRL_DPIN1_SELECT0 | + CROSSBAR_MUX_CTRL_DPIN1_SELECT1; + case MUX_DPPHY: + default: + return CROSSBAR_MUX_CTRL_DPPHY_SELECT0 | + CROSSBAR_MUX_CTRL_DPPHY_SELECT1; + } +} + +static u32 t602x_mux_set(unsigned int index, unsigned int mux_state) +{ + switch (index) { + case MUX_DPIN0: + return FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN0_SELECT0, mux_state) | + FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN0_SELECT1, mux_state); + case MUX_DPIN1: + return FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN1_SELECT0, mux_state) | + FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN1_SELECT1, mux_state); + case MUX_DPPHY: + default: + return FIELD_PREP(CROSSBAR_MUX_CTRL_DPPHY_SELECT0, mux_state) | + FIELD_PREP(CROSSBAR_MUX_CTRL_DPPHY_SELECT1, mux_state); + } +} + +static void t602x_dump(struct apple_dpxbar *xbar, const char *tag) +{ + static const u32 offs[] = { + 0x000, 0x004, 0x008, 0x00c, 0x014, 0x018, 0x01c, 0x024, + 0x028, 0x02c, 0x030, 0x034, 0x040, 0x044, 0x048, 0x04c, + 0x050, 0x060, 0x070, + }; + char buf[320]; + int n = 0; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(offs); i++) { + u32 val = readl(xbar->regs + offs[i]); + + n += scnprintf(buf + n, sizeof(buf) - n, " %03x=%08x", + offs[i], val); + if (n >= (int)sizeof(buf) - 20) + break; + } + dev_dbg(xbar->dev, "t602x %s:%s\n", tag, buf); + dev_dbg(xbar->dev, + "t602x %s clk: 000=%08x 800=%08x 020=%08x 820=%08x 024=%08x 81c=%08x\n", + tag, + readl(xbar->regs + T602X_FIFO_WR_DPTX_CLK_EN), + readl(xbar->regs + FIFO_WR_DPTX_CLK_EN_STAT), + readl(xbar->regs + FIFO_RD_PCLK1_EN), + readl(xbar->regs + FIFO_RD_PCLK1_EN_STAT), + readl(xbar->regs + T602X_FIFO_RD_PCLK2_EN), + readl(xbar->regs + T602X_REG_81C_STAT)); +} + static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) { struct apple_dpxbar *dpxbar = mux_chip_priv(mux->chip); @@ -127,15 +204,14 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) unsigned int mux_state; unsigned int dispext_bit; unsigned int dispext_bit_en; + u32 atc_bit, mux_mask, mux_val; bool enable; int ret = 0; + if (index >= MUX_MAX) + return -EINVAL; + if (state == MUX_IDLE_DISCONNECT) { - /* - * Technically this will select dispext0,0 in the mux control - * register. Practically that doesn't matter since everything - * else is disabled. - */ mux_state = 0; enable = false; } else if (state >= 0 && state < 9) { @@ -147,6 +223,10 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) return -EINVAL; } + atc_bit = t602x_atc_bit(index); + mux_mask = t602x_mux_mask(index); + mux_val = t602x_mux_set(index, mux_state); + spin_lock_irqsave(&dpxbar->lock, flags); /* ensure the selected dispext isn't already used in this crossbar */ @@ -167,51 +247,57 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) dpxbar_clear32(dpxbar, T602X_FIFO_RD_UNK_EN, prev_dispext_bit); dpxbar_clear32(dpxbar, T602X_FIFO_WR_DPTX_CLK_EN, prev_dispext_bit); - dpxbar_clear32(dpxbar, T602X_REG_00C, prev_dispext_bit_en); + /* DPIN0 uses the same single-bit source gate on up and down. */ + dpxbar_clear32(dpxbar, T602X_REG_00C, + index == MUX_DPIN0 ? prev_dispext_bit : prev_dispext_bit_en); - dpxbar_clear32(dpxbar, T602X_REG_01C, 0x100); + dpxbar_clear32(dpxbar, T602X_REG_01C, atc_bit); dpxbar_clear32(dpxbar, T602X_FIFO_WR_UNK_EN, prev_dispext_bit); dpxbar_clear32(dpxbar, T602X_REG_018, prev_dispext_bit_en); - dpxbar_clear32(dpxbar, T602X_FIFO_RD_N_CLK_EN, 0x100); + dpxbar_clear32(dpxbar, T602X_FIFO_RD_N_CLK_EN, atc_bit); dpxbar_set32(dpxbar, T602X_FIFO_WR_N_CLK_EN, prev_dispext_bit); dpxbar_set32(dpxbar, T602X_REG_014, 0x4); - dpxbar_set32(dpxbar, FIFO_RD_PCLK1_EN, 0x100); + /* Native T602x DPIN0 teardown restores the read reset at +0x24. */ + dpxbar_set32(dpxbar, index == MUX_DPIN0 ? + T602X_FIFO_RD_PCLK2_EN : FIFO_RD_PCLK1_EN, atc_bit); + + dpxbar_clear32(dpxbar, T602X_REG_034, atc_bit); + dpxbar_clear32(dpxbar, CROSSBAR_ATC_EN, atc_bit); + dpxbar_clear32(dpxbar, CROSSBAR_DISPEXT_EN, prev_dispext_bit); + dpxbar_mask32(dpxbar, T602X_REG_030, mux_mask, 0); dpxbar->selected_dispext[index] = -1; } if (enable) { - /* - * Both selector fields must be replaced, not OR-ed in: leaving a - * previous pipeline's bits set keeps the crossbar pointed at it, - * and selecting dispext0 (state 0) would change nothing at all. - * The port then never reaches the DCP that asked for it, whose - * DPTX calls time out with the firmware reporting the device as - * not started. - */ - dpxbar_mask32(dpxbar, T602X_REG_030, - GENMASK(23, 20) | GENMASK(11, 8), - (state << 20) | (state << 8)); + dpxbar_mask32(dpxbar, T602X_REG_030, mux_mask, mux_val); udelay(10); dpxbar_clear32(dpxbar, T602X_FIFO_WR_N_CLK_EN, dispext_bit); dpxbar_clear32(dpxbar, T602X_REG_014, 0x4); - dpxbar_clear32(dpxbar, T602X_FIFO_RD_PCLK2_EN, 0x100); + dpxbar_clear32(dpxbar, T602X_FIFO_RD_PCLK2_EN, atc_bit); dpxbar_set32(dpxbar, T602X_FIFO_WR_UNK_EN, dispext_bit); dpxbar_set32(dpxbar, T602X_REG_018, dispext_bit_en); - dpxbar_set32(dpxbar, T602X_FIFO_RD_N_CLK_EN, 0x100); + dpxbar_set32(dpxbar, T602X_FIFO_RD_N_CLK_EN, atc_bit); dpxbar_set32(dpxbar, T602X_FIFO_WR_DPTX_CLK_EN, dispext_bit); dpxbar_set32(dpxbar, T602X_REG_00C, dispext_bit); - dpxbar_set32(dpxbar, T602X_REG_01C, 0x100); - dpxbar_set32(dpxbar, T602X_REG_034, 0x100); + dpxbar_set32(dpxbar, T602X_REG_01C, atc_bit); + dpxbar_set32(dpxbar, T602X_REG_034, atc_bit); + dpxbar_set32(dpxbar, CROSSBAR_ATC_EN, atc_bit); + /* + * t8103 enables the dispext source at 0x050. T602x left + * that register at 0, so dpin0 was selected with no + * source clock into the ACIO analog PHY. + */ + dpxbar_set32(dpxbar, CROSSBAR_DISPEXT_EN, dispext_bit); dpxbar_set32(dpxbar, T602X_FIFO_RD_UNK_EN, dispext_bit); @@ -220,13 +306,18 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) spin_unlock_irqrestore(&dpxbar->lock, flags); - if (enable) + if (enable) { dev_info(dpxbar->dev, "Switched %s to dispext%u,%u\n", apple_dpxbar_names[index], mux_state >> 1, mux_state & 1); - else + dev_dbg(dpxbar->dev, "t602x atc=0x%x mux=0x%x\n", atc_bit, + mux_val); + } else { dev_info(dpxbar->dev, "Switched %s to disconnected state\n", apple_dpxbar_names[index]); + } + + t602x_dump(dpxbar, enable ? apple_dpxbar_names[index] : "idle"); return ret; } @@ -415,7 +506,7 @@ static struct apple_dpxbar *apple_dpxbar_from_mux(struct mux_control *mux, return mux_chip_priv(mux->chip); } -int apple_dpxbar_link_down(struct mux_control *mux) +static int apple_dpxbar_t8103_link_down(struct mux_control *mux) { struct apple_dpxbar *dpxbar; u32 dispext_bit, atc_bit; @@ -442,9 +533,8 @@ int apple_dpxbar_link_down(struct mux_control *mux) spin_unlock_irqrestore(&dpxbar->lock, flags); return ret; } -EXPORT_SYMBOL_GPL(apple_dpxbar_link_down); -int apple_dpxbar_link_up(struct mux_control *mux) +static int apple_dpxbar_t8103_link_up(struct mux_control *mux) { struct apple_dpxbar *dpxbar; u32 dispext_bit, atc_bit; @@ -484,7 +574,6 @@ int apple_dpxbar_link_up(struct mux_control *mux) readl(dpxbar->regs + OUT_PCLK1_EN_STAT)); return 0; } -EXPORT_SYMBOL_GPL(apple_dpxbar_link_up); static const struct mux_control_ops apple_dpxbar_ops = { .set = apple_dpxbar_set, @@ -494,6 +583,120 @@ static const struct mux_control_ops apple_dpxbar_t602x_ops = { .set = apple_dpxbar_set_t602x, }; +static int apple_dpxbar_t602x_link_up(struct mux_control *mux) +{ + struct apple_dpxbar *xbar; + unsigned int index; + unsigned long flags; + u32 dispext_bit, atc_bit; + int state, ret = 0; + + if (!mux || mux->chip->ops != &apple_dpxbar_t602x_ops) + return -EINVAL; + index = mux_control_get_index(mux); + if (index >= MUX_MAX) + return -EINVAL; + atc_bit = t602x_atc_bit(index); + xbar = mux_chip_priv(mux->chip); + + spin_lock_irqsave(&xbar->lock, flags); + state = xbar->selected_dispext[index]; + if (state < 0) { + spin_unlock_irqrestore(&xbar->lock, flags); + return -ENODEV; + } + dispext_bit = 1 << state; + + dpxbar_clear32(xbar, T602X_FIFO_WR_N_CLK_EN, dispext_bit); + dpxbar_clear32(xbar, T602X_REG_014, dispext_bit); + dpxbar_clear32(xbar, index == MUX_DPIN0 ? T602X_FIFO_RD_PCLK2_EN : + FIFO_RD_PCLK1_EN, atc_bit); + udelay(1); + + dpxbar_set32(xbar, T602X_FIFO_WR_UNK_EN, dispext_bit); + dpxbar_mask32(xbar, T602X_REG_018, GENMASK(5, 4), BIT(4)); + /* + * The only hardware-verified value for this 2-bit field is dpin0's + * BIT(0), which is both ATC_DPIN0 and MUX_DPIN0's own enum value (1) + * coincidentally overlapping at bit 0 -- so this field's encoding + * for a second index was never independently confirmed on real + * hardware. FIELD_PREP(..., index), matching + * MUX_DPPHY=0/MUX_DPIN0=1/MUX_DPIN1=2, is an inferred generalization + * of that single known-good dpin0 data point, not a value verified + * against dpin1 hardware. If dpin1 misbehaves, this field is a + * prime suspect. + */ + dpxbar_mask32(xbar, T602X_FIFO_RD_N_CLK_EN, GENMASK(1, 0), + FIELD_PREP(GENMASK(1, 0), index)); + dpxbar_set32(xbar, T602X_FIFO_WR_DPTX_CLK_EN, dispext_bit); + dpxbar_set32(xbar, T602X_REG_00C, dispext_bit); + dpxbar_set32(xbar, T602X_REG_01C, atc_bit); + dpxbar_set32(xbar, T602X_REG_034, atc_bit); + dpxbar_set32(xbar, CROSSBAR_ATC_EN, atc_bit); + dpxbar_set32(xbar, CROSSBAR_DISPEXT_EN, dispext_bit); + dpxbar_set32(xbar, T602X_FIFO_RD_UNK_EN, dispext_bit); + spin_unlock_irqrestore(&xbar->lock, flags); + + dev_info(xbar->dev, "%s: crossbar link up (dispext=%d atc=0x%x)\n", + apple_dpxbar_names[index], state, atc_bit); + return ret; +} + +int apple_dpxbar_link_up(struct mux_control *mux) +{ + if (!mux) + return -EINVAL; + if (mux->chip->ops == &apple_dpxbar_t602x_ops) + return apple_dpxbar_t602x_link_up(mux); + return apple_dpxbar_t8103_link_up(mux); +} +EXPORT_SYMBOL_GPL(apple_dpxbar_link_up); + +static int apple_dpxbar_t602x_link_down(struct mux_control *mux) +{ + struct apple_dpxbar *xbar; + unsigned int index; + unsigned long flags; + u32 dispext_bit, atc_bit; + int state; + + if (!mux || mux->chip->ops != &apple_dpxbar_t602x_ops) + return -EINVAL; + index = mux_control_get_index(mux); + if (index >= MUX_MAX) + return -EINVAL; + atc_bit = t602x_atc_bit(index); + xbar = mux_chip_priv(mux->chip); + + spin_lock_irqsave(&xbar->lock, flags); + state = xbar->selected_dispext[index]; + if (state < 0) { + spin_unlock_irqrestore(&xbar->lock, flags); + return -ENODEV; + } + dispext_bit = 1 << state; + + dpxbar_set32(xbar, T602X_FIFO_WR_N_CLK_EN, dispext_bit); + dpxbar_set32(xbar, T602X_REG_014, dispext_bit); + dpxbar_set32(xbar, index == MUX_DPIN0 ? T602X_FIFO_RD_PCLK2_EN : + FIFO_RD_PCLK1_EN, atc_bit); + spin_unlock_irqrestore(&xbar->lock, flags); + + dev_info(xbar->dev, "%s: crossbar link down (dispext=%d atc=0x%x)\n", + apple_dpxbar_names[index], state, atc_bit); + return 0; +} + +int apple_dpxbar_link_down(struct mux_control *mux) +{ + if (!mux) + return -EINVAL; + if (mux->chip->ops == &apple_dpxbar_t602x_ops) + return apple_dpxbar_t602x_link_down(mux); + return apple_dpxbar_t8103_link_down(mux); +} +EXPORT_SYMBOL_GPL(apple_dpxbar_link_down); + static int apple_dpxbar_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; From 9469d94301b025ddc16c0b5e222204d068262fa6 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Thu, 24 Sep 2026 21:00:14 +0200 Subject: [PATCH 0969/2680] phy: apple: atc: add DisplayPort tunnel clock support for USB4/TBT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On T602X (M2 Pro, j416s) Type-C PHYs, a monitor connected directly via DP alt-mode or a USB-C-to-HDMI adapter already works, because that path drives the AUSPLL through the PHY's normal per-mode configuration table. A monitor connected through a USB4/Thunderbolt hub instead carries DisplayPort as a tunneled stream over the USB4 link, and the DCP display coprocessor firmware expects the PHY driver to hand it a dedicated tunnel pixel clock on request, outside the normal mode-configure path. Nothing implements that request today, so a tunneled display never gets a clock and stays dark even though the USB4 link itself comes up. Add the tunnel clock state machine: atc_tunnel_start() brings up the fixed AUSPLL frequency descriptor and PCLK1 output for one of the four standard DisplayPort link rates, after a preflight read of the gate/output/lock registers confirms the PHY is in the specific idle state this sequence expects (refusing with -EBUSY otherwise rather than clobbering live state). Before touching anything it snapshots the registers it is about to modify; atc_tunnel_restore() replays that snapshot in reverse to undo the sequence, and is called unconditionally at the top of atcphy_configure() so any mode change, power-down, or unplug leaves the tunnel registers exactly as it found them instead of stuck mid-sequence, and clears the request-guard state so a later tunnel request is not permanently refused by a stale in-progress flag. It also waits for ACIOPHY_AUSPLL_LOCK to clear before returning: a request landing right after a teardown -- a live replug is fast enough to do this, and so is DCP firmware's own retry cadence -- would otherwise see the PLL's own not-yet-settled lock bit and refuse itself as busy, indefinitely. Expose this through a new exported apple_atc_dp_tunnel_rate(phy, rate) -- the exact symbol name drivers/gpu/drm/apple/dcp.c resolves via symbol_get() to ask a PHY for a tunnel clock. It is gated to apple,j416s machines and to ATC cores that are both a t6020-atcphy and a recognized Type-C core address, so it is a correctly-typed no-op everywhere else, and only proceeds while the PHY is already in USB4 or TBT mode. [ryan: adapted to 7.2.10 — this commit and the parallel t8103 tunnel clock (019279b11e71) were reconciled by Chris Kearney in merge cf5f7d19399e (per-SoC atc_tunnel_start_t8103/_t602x); this port applies that merged result, so the commit also carries the merge resolution] Co-Authored-By: Claude Sonnet 5 --- drivers/phy/apple/atc.c | 239 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 230 insertions(+), 9 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index dc8d008a5ae79..bd10c7cd9856a 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -706,6 +706,8 @@ struct apple_atcphy { enum atcphy_pipehandler_state pipe_state; bool tunnel_clock_on; + bool tunnel_attempted, tunnel_saved; + u32 tunnel_saved_regs[12]; u8 tunnel_rate; struct { @@ -1793,7 +1795,7 @@ static int atcphy_dp_configure(struct apple_atcphy *atcphy, enum atcphy_dp_link_ guard(mutex)(&atcphy->lock); /* the AUSPLL is in use as the Thunderbolt DP tunnel pixel clock */ - if (atcphy->tunnel_clock_on) + if (atcphy->tunnel_clock_on || atcphy->tunnel_saved) return -EBUSY; mode_cfg = atcphy_get_mode_config(atcphy, atcphy->mode); cfg = &dp_lr_config[lr]; @@ -2098,7 +2100,7 @@ static void atcphy_configure_post_tunable_all_modes(struct apple_atcphy *atcphy, * APB command 3. TX_DP_CTRL0, the sleep overrides and the * descriptor are left as they are; the next start programs them again. */ -static void atc_tunnel_stop(struct apple_atcphy *atcphy) +static void atc_tunnel_stop_t8103(struct apple_atcphy *atcphy) { lockdep_assert_held(&atcphy->lock); if (!atcphy->tunnel_clock_on) @@ -2110,7 +2112,7 @@ static void atc_tunnel_stop(struct apple_atcphy *atcphy) dev_dbg(atcphy->dev, "DP tunnel clock stopped\n"); } -static int atc_tunnel_start(struct apple_atcphy *atcphy, u8 rate) +static int atc_tunnel_start_t8103(struct apple_atcphy *atcphy, u8 rate) { u32 selector, value; int ret; @@ -2232,7 +2234,7 @@ static int atc_tunnel_start(struct apple_atcphy *atcphy, u8 rate) value & ACIOPHY_AUSPLL_LOCK, 10, 10000); if (ret) { dev_err(atcphy->dev, "DP tunnel clock: AUSPLL did not lock\n"); - atc_tunnel_stop(atcphy); + atc_tunnel_stop_t8103(atcphy); return ret; } atcphy_auspll_apb_command(atcphy, 0x2000); @@ -2240,6 +2242,209 @@ static int atc_tunnel_start(struct apple_atcphy *atcphy, u8 rate) return 0; } +struct atc_tunnel_saved_reg { + u32 reg; + u32 mask; +}; + +static const struct atc_tunnel_saved_reg atc_tunnel_regs[] = { + { ACIOPHY_CFG0, 0x0003ffff }, + { ACIOPHY_SLEEP_CTRL, 0x00000fff }, + { ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, 0x0000207c }, + { AUSPLL_FREQ_CFG, 0x00000003 }, + { AUSPLL_FREQ_DESC_A, 0xffffffff }, + { AUSPLL_FREQ_DESC_B, 0x0fffffff }, + { AUSPLL_FREQ_DESC_C, 0x007fffff }, + { AUSPLL_CLKOUT_DIV, 0x001f0000 }, + { AUSPLL_CLKOUT_DTC_VREG, 0x00000080 }, + { AUSPLL_BGR, 0x00000001 }, + { AUSPLL_CLKOUT_MASTER, 0x00000054 }, + { AUSPLL_APB_CMD_OVERRIDE, 0x1ffffff9 }, +}; + +static int atc_tunnel_command(struct apple_atcphy *atcphy, u32 command) +{ + u32 value; + int ret; + + core_mask32(atcphy, AUSPLL_APB_CMD_OVERRIDE, + AUSPLL_APB_CMD_OVERRIDE_CMD, + FIELD_PREP(AUSPLL_APB_CMD_OVERRIDE_CMD, command) | + AUSPLL_APB_CMD_OVERRIDE_REQ | AUSPLL_APB_CMD_OVERRIDE_UNK28); + ret = readl_poll_timeout(atcphy->regs.core + AUSPLL_APB_CMD_OVERRIDE, + value, value & AUSPLL_APB_CMD_OVERRIDE_ACK, 10, 10000); + core_clear32(atcphy, AUSPLL_APB_CMD_OVERRIDE, AUSPLL_APB_CMD_OVERRIDE_REQ); + return ret; +} + +static void atc_tunnel_restore(struct apple_atcphy *atcphy) +{ + u32 value; + int i, ret; + + lockdep_assert_held(&atcphy->lock); + if (!atcphy->tunnel_saved) + return; + /* Stop our output before restoring the unused PLL descriptor. */ + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, 0x54); + for (i = ARRAY_SIZE(atc_tunnel_regs) - 1; i >= 0; i--) + core_mask32(atcphy, atc_tunnel_regs[i].reg, atc_tunnel_regs[i].mask, + atcphy->tunnel_saved_regs[i] & atc_tunnel_regs[i].mask); + /* + * atc_tunnel_start_t602x()'s preflight refuses to proceed while + * ACIOPHY_AUSPLL_LOCK is still set, to avoid clobbering a PLL a + * concurrent user actually has locked. Immediately after tearing our + * own lock down, that same bit can still read set for a brief window + * before the lock detector catches up -- and DCP firmware retries a + * failed tunnel clock request roughly once a second, which is faster + * than that window on a hot retrain loop. Wait for it to clear here + * so a request coming in right after this returns sees a genuinely + * idle PLL instead of refusing itself forever. + */ + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT, value, + !(value & ACIOPHY_AUSPLL_LOCK), 10, 10000); + if (ret) + dev_warn(atcphy->dev, + "USB4 tunnel clock: AUSPLL_LOCK did not clear after teardown (stat=%#x)\n", + value); + atcphy->tunnel_saved = false; + atcphy->tunnel_attempted = false; + atcphy->tunnel_rate = 0; +} + +static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy, u8 rate) +{ + u32 selector, value, gates, outputs, command, status; + unsigned int i; + int ret; + + lockdep_assert_held(&atcphy->lock); + switch (rate) { + case 0x06: + selector = 4; + break; + case 0x0a: + selector = 3; + break; + case 0x14: + selector = 1; + break; + case 0x1e: + selector = 0; + break; + default: + return -EINVAL; + } + if (atcphy->tunnel_saved) { + if (atcphy->tunnel_rate != rate) + dev_info(atcphy->dev, "USB4 tunnel clock busy: active rate=0x%x requested=0x%x\n", + atcphy->tunnel_rate, rate); + return atcphy->tunnel_rate == rate ? 0 : -EBUSY; + } + if (atcphy->tunnel_attempted) + return -EALREADY; + /* Read every guard before deciding, so a refusal records the whole state. */ + gates = readl(atcphy->regs.core + ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0); + outputs = readl(atcphy->regs.core + AUSPLL_CLKOUT_MASTER); + command = readl(atcphy->regs.core + AUSPLL_APB_CMD_OVERRIDE); + status = readl(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT); + dev_info(atcphy->dev, + "USB4 tunnel clock preflight: +7000=%08x +2200=%08x +2000=%08x +7044=%08x\n", + gates, outputs, command, status); + /* + * Before the first rate request, TX_DP_CTRL0 gate bits read 0xe001: + * the enable gates are set, the byte-clock reset is asserted, and the + * selector and reset-release bits are clear. Native configureDPTunnelMode + * tracks tunnel clients in software rather than through these gates, so + * this exact state is the only additional gate configuration accepted + * here, together with no enabled PLL output, no outstanding command, + * and no lock. Any other gate state is refused. + */ + if (outputs & 0x54 || command & AUSPLL_APB_CMD_OVERRIDE_REQ || + status & ACIOPHY_AUSPLL_LOCK) + return -EBUSY; + if ((gates & (DPTX_PCLK1_ENABLE | DPTX_PCLK2_ENABLE | DPRX_PCLK_ENABLE)) && + gates != 0x0000e001) + return -EBUSY; + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_CMN_SHM_STS_REG0, + value, value & ACIOPHY_CMN_SHM_STS_REG0_CMD_READY, + 10, 10000); + if (ret) + return ret; + atcphy->tunnel_attempted = true; + for (i = 0; i < ARRAY_SIZE(atc_tunnel_regs); i++) + atcphy->tunnel_saved_regs[i] = readl(atcphy->regs.core + atc_tunnel_regs[i].reg); + atcphy->tunnel_saved = true; + + /* Native configureDPTunnelMode, first clock client; no lane mux writes. */ + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL_OV); + udelay(2); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_BIG); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_BIG_OV); + udelay(2); + core_clear32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP_OV); + udelay(2); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0x30); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0xc0); + udelay(2); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0x03); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0x0c); + udelay(2); + core_clear32(atcphy, ACIOPHY_SLEEP_CTRL, 0x300); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0xc00); + udelay(2); + /* Additional override bit alongside RX_BIG_OV; exact meaning not independently confirmed. */ + core_set32(atcphy, ACIOPHY_CFG0, 0xc00); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG_OV); + udelay(2); + /* Additional override bit alongside RX_SMALL_OV; exact meaning not independently confirmed. */ + core_set32(atcphy, ACIOPHY_CFG0, 0xc0); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL_OV); + udelay(2); + /* Additional override bit alongside RX_CLAMP_OV; exact meaning not independently confirmed. */ + core_clear32(atcphy, ACIOPHY_CFG0, 0xc000); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP_OV); + udelay(2); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, BIT(2)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, BIT(3)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_SELECT, + FIELD_PREP(DPTX_PCLK1_SELECT, selector)); + + /* Native fixed descriptor at cache VA 0xfffffe0007582418. */ + core_clear32(atcphy, AUSPLL_FREQ_CFG, AUSPLL_FREQ_REFCLK); + core_mask32(atcphy, AUSPLL_FREQ_DESC_A, 0xffffffff, + 0x21c | (8 << 14) | (3 << 18) | (8 << 22) | (7 << 26)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_B, 0x0fffffff, 0); + core_mask32(atcphy, AUSPLL_FREQ_DESC_C, 0x007fffff, + (5 << 9) | (1 << 14) | (5 << 16) | (2 << 20) | BIT(22)); + core_mask32(atcphy, AUSPLL_CLKOUT_DIV, AUSPLL_CLKOUT_PLLA_REFBUFCLK_DI, + FIELD_PREP(AUSPLL_CLKOUT_PLLA_REFBUFCLK_DI, 1)); + core_set32(atcphy, AUSPLL_CLKOUT_DTC_VREG, AUSPLL_DTC_VREG_BYPASS); + core_set32(atcphy, AUSPLL_BGR, AUSPLL_BGR_CTRL_AVAIL); + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(2)); + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(4)); + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(6)); + ret = atc_tunnel_command(atcphy, 0); + if (ret) + goto restore; + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT, + value, value & ACIOPHY_AUSPLL_LOCK, 10, 10000); + if (ret) + goto restore; + ret = atc_tunnel_command(atcphy, 0x2000); + if (ret) + goto restore; + atcphy->tunnel_rate = rate; + return 0; +restore: + atc_tunnel_restore(atcphy); + return ret; +} + static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) { int ret = 0; @@ -2247,7 +2452,8 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) lockdep_assert_held(&atcphy->lock); /* the mode change below may power the PHY down: stop the tunnel clock first */ - atc_tunnel_stop(atcphy); + atc_tunnel_stop_t8103(atcphy); + atc_tunnel_restore(atcphy); if (mode == APPLE_ATCPHY_MODE_OFF) { ret = atcphy_power_off(atcphy); @@ -2479,6 +2685,12 @@ static const struct phy_ops apple_atc_dp_phy_ops = { * Called by appledrm when DCP sets the link rate of a DPTX that feeds a * Thunderbolt DP IN adapter (rate is the DP link rate code, 0 = stop). */ +static bool apple_atc_is_typec_core(u64 base) +{ + return base == 0x703000000ULL || base == 0xb03000000ULL || + base == 0xf03000000ULL; +} + int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate) { struct apple_atcphy *atcphy; @@ -2487,17 +2699,26 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate) if (!phy || phy->ops != &apple_atc_dp_phy_ops) return -EINVAL; atcphy = phy_get_drvdata(phy); - /* the fixed PLL descriptor and PCLK1 selectors are for t8103 only */ - if (!of_machine_is_compatible("apple,t8103")) + /* Keep each supported SoC on its qualified clock sequence. */ + if (!of_machine_is_compatible("apple,t8103") && + !of_machine_is_compatible("apple,j416s")) return -EOPNOTSUPP; + if (of_machine_is_compatible("apple,j416s") && + (!of_device_is_compatible(atcphy->np, "apple,t6020-atcphy") || + !apple_atc_is_typec_core(atcphy->res.core->start) || + resource_size(atcphy->res.core) < 0x7048)) + return -EINVAL; guard(mutex)(&atcphy->lock); if (!rate) { - atc_tunnel_stop(atcphy); + atc_tunnel_stop_t8103(atcphy); + atc_tunnel_restore(atcphy); return 0; } if (atcphy->mode != APPLE_ATCPHY_MODE_USB4 && atcphy->mode != APPLE_ATCPHY_MODE_TBT) return -EBUSY; - ret = atc_tunnel_start(atcphy, rate); + ret = of_machine_is_compatible("apple,t8103") ? + atc_tunnel_start_t8103(atcphy, rate) : + atc_tunnel_start_t602x(atcphy, rate); dev_dbg(atcphy->dev, "DP tunnel clock rate 0x%x: %d (TX_DP_CTRL0=%08x PCLK_STAT=%08x)\n", rate, ret, readl(atcphy->regs.core + ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0), readl(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT)); From 6b6dbf8d69eefccf9b092f6d17f2dd05d72557e5 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Fri, 25 Sep 2026 09:55:41 +0200 Subject: [PATCH 0970/2680] thunderbolt: apple: retry the ACIO reset handshake on resume-time contention MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit apple_cio_start()'s reset_control_deassert(acio->reset) is a request/ack handshake with the M3/PMGR firmware (100ms budget, no .assert op to fall back on). Confirmed via live journalctl/dmesg to time out on resume from suspend with an active USB4 tunnel connected, during the wider resume storm every other coprocessor on the SoC is also going through. Retry up to APPLE_CIO_START_RETRIES (5) times, 100ms apart, before giving up. No new code path, no suspend/resume PM hook added -- deliberately not attempting the more invasive fix (giving apple_cio_driver its own PM hooks), since apple_cio_tbt_switch_set() documents that an invalid direct cable-state transition can crash ACIO and trigger an SoC watchdog reset, and a second independent stop()/start() trigger around the same suspend/resume window as the Type-C mux's own decision risks exactly that. This only widens how persistently the existing, already-serialized call waits for the firmware to answer. [ryan: adapted to 7.2.10 — the long comment this added above the retry loop was dropped in merge cf5f7d19399e and is not on aurora-wip; only the retry is carried] Co-Authored-By: Claude Sonnet 5 --- drivers/thunderbolt/apple.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 7b636bb36a673..1ab70f4eef075 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -134,6 +134,9 @@ #define APPLE_CIO_NHI_BOOT_TIMEOUT_MS 10000 #define APPLE_CIO_PCIEC_READY_DELAY_MS 300 +#define APPLE_CIO_START_RETRIES 5 +#define APPLE_CIO_START_RETRY_DELAY_MS 100 + #define APPLE_CIO_PCIEC_INTR2AXI_CTRL 0x80 #define APPLE_CIO_PCIEC_INTR2AXI_ENABLE BIT(0) @@ -1982,7 +1985,17 @@ static int apple_cio_start(struct apple_cio *acio) * After the power domains are on we need to signal and wait for the ACIO block * to actually start before we can bring up the co-processor. */ - ret = reset_control_deassert(acio->reset); + for (i = 0; i < APPLE_CIO_START_RETRIES; i++) { + ret = reset_control_deassert(acio->reset); + if (!ret) + break; + if (i + 1 < APPLE_CIO_START_RETRIES) { + dev_warn(acio->dev, + "ACIO block failed to start (attempt %d/%d): %d, retrying\n", + i + 1, APPLE_CIO_START_RETRIES, ret); + msleep(APPLE_CIO_START_RETRY_DELAY_MS); + } + } if (ret) { dev_err(acio->dev, "ACIO block failed to start: %d\n", ret); goto remove_links; From d79bd65a2d3863e1cc97826afdc589ce4cd9299b Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Fri, 25 Sep 2026 09:55:56 +0200 Subject: [PATCH 0971/2680] usb: typec: tipd: auto-recover the display after standby, no replug needed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Building on the previous commit (ACIO/Thunderbolt itself now survives suspend): the display still didn't come back on its own after resume, only after a physical unplug/replug of the dock. tipd_resume() never re-verified the port's actual attach state -- it only checks firmware liveness and re-arms the IRQ. A cable that was never physically removed generates no fresh attach event, so nothing re-drives apple_cio_tbt_switch_set()/apple_cio_start(), even though thunderbolt-core's own tb_resume_noirq() -> tb_free_invalid_tunnels() had already silently torn down the live DP tunnel underneath it (confirmed generic thunderbolt-core behaviour, not Apple-specific). A naive "just re-verify and re-assert the same state" fix is a no-op: apple_cio_tbt_switch_set()'s first check is target_cable_info == current_cable_info, and current_cable_info is only ever written inside apple_cio_start()/apple_cio_stop() themselves -- re-detecting the same, unchanged cable computes the same target and hits that early return immediately. Added cd321x_resume_reverify(), wired only into the Apple-specific CD321x/sn201202x vtables (the plain TI TPS6598X/TPS25750 variants are unaffected). It reproduces a genuine unplug-then-replug entirely through the existing, already-hardware-validated connect() code path: calls the existing connect() callback twice -- once with TPS_STATUS_PLUG_PRESENT cleared (a synthetic disconnect), then once with the real, unmodified status. Both land on the same debounced cd321x_update_work(), coalescing into a single run with was_disconnected=true and the correct final state: the exact same typec_thunderbolt_switch_set() OFF-then-back-on sequence, through the exact same apple_cio_tbt_switch_set() guard, a real physical replug already exercises every time someone does it by hand. The synthetic disconnect always drives through a clean zero state first, the same shape apple_cio_tbt_switch_set()'s own watchdog-reset warning already treats as safe (a direct transition between two different nonzero cable-info states, with no shutdown in between, is what it warns against). Triggered from a PM_POST_SUSPEND notifier rather than directly from tipd_resume(): tipd_resume() is just one individual device's own .resume() callback, with no guarantee the rest of the system (ACIO/ Thunderbolt in particular) has finished its own resume work yet. PM_POST_SUSPEND is delivered strictly after dpm_resume_end(), i.e. after every device has completed its own ordinary resume, which matters here specifically: driving the reconnect too early can fire apple_cio_start()'s M3/PMGR reset-deassert handshake while that firmware is still busy servicing other subsystems' resume, and exhaust its retry budget outright. A real physical replug never hits this, because a human always does it seconds after the whole system has visibly settled. Confirmed on hardware: real lid-close/s2idle/lid-open cycle, hub never touched. dmesg shows the full automatic chain (PM_POST_SUSPEND -> synthetic disconnect and reconnect -> tunnel connect -> modeset) succeeding with zero ACIO start retries, and the picture returns with no replug and no manual intervention. [ryan: adapted to 7.2.10 — Asahi now keeps the variant data static behind tipd_hw_variants[]; the PM notifier goes in front of it; the result matches merge cf5f7d19399e] Co-Authored-By: Claude Sonnet 5 --- drivers/usb/typec/tipd/core.c | 78 +++++++++++++++++++++++++++++++ drivers/usb/typec/tipd/tps6598x.h | 3 ++ 2 files changed, 81 insertions(+) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 91b453b7cded1..18e43359159fe 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -22,6 +22,7 @@ #include #include #include +#include #include "tps6598x.h" #include "trace.h" @@ -901,6 +902,44 @@ static int cd321x_connect(struct tps6598x *tps, u32 status) return 0; } +/* + * A cable that was already attached before a suspend/resume cycle generates + * no fresh attach event on resume: the PD controller's own status hasn't + * changed, so cd321x_connect()'s diff against its last cached status sees + * nothing different and cd321x_update_work() takes its "nothing to do" + * path. Meanwhile thunderbolt-core's own noirq-phase resume handling + * (tb_free_invalid_tunnels()) has already torn down any USB4/TBT tunnel + * that didn't survive the sleep at the link level -- and nothing else + * re-establishes it. + * + * Reproduce a genuine unplug-then-replug in software instead of inventing + * a new code path: call the existing connect() callback once with the + * current status but PLUG_PRESENT cleared (a synthetic disconnect), then + * again with the real, current status. Both updates land on the same + * debounced work item, which coalesces them into a single run with + * was_disconnected=true and the correct final state -- exactly the same + * teardown-then-bring-up sequence, through the same already-validated + * apple_cio_tbt_switch_set() path, that a real physical replug drives. + */ +static void cd321x_resume_reverify(struct tps6598x *tps) +{ + u32 status; + + if (!tps6598x_read_status(tps, &status)) + return; + + if (!(status & TPS_STATUS_PLUG_PRESENT)) + return; + + if (!tps6598x_read_power_status(tps)) + return; + if (!tps->data->read_data_status(tps)) + return; + + tps->data->connect(tps, status & ~TPS_STATUS_PLUG_PRESENT); + tps->data->connect(tps, status); +} + static irqreturn_t cd321x_interrupt(int irq, void *data) { struct tps6598x *tps = data; @@ -1805,6 +1844,8 @@ static void cd321x_remove(struct tps6598x *tps) usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); } +static int tipd_pm_notify(struct notifier_block *nb, unsigned long action, void *data); + int tipd_init(struct tps6598x *tps) { struct fwnode_handle *fwnode; @@ -1912,6 +1953,11 @@ int tipd_init(struct tps6598x *tps) enable_irq_wake(tps->irq); } + if (tps->data->resume_reverify) { + tps->pm_nb.notifier_call = tipd_pm_notify; + register_pm_notifier(&tps->pm_nb); + } + return 0; err_disconnect: @@ -1935,6 +1981,9 @@ EXPORT_SYMBOL_GPL(tipd_init); void tipd_remove(struct tps6598x *tps) { + if (tps->data->resume_reverify) + unregister_pm_notifier(&tps->pm_nb); + if (!tps->irq) cancel_delayed_work_sync(&tps->wq_poll); else @@ -2001,6 +2050,33 @@ int tipd_resume(struct tps6598x *tps) } EXPORT_SYMBOL_GPL(tipd_resume); +/* + * tipd_resume() runs as an ordinary device .resume() callback, which can + * complete well before the rest of the system -- ACIO/Thunderbolt, DCP, + * the crossbar -- has finished its own resume work. Driving the reconnect + * from here directly (as originally tried) fires apple_cio_start()'s + * M3/PMGR reset-deassert handshake while that firmware is potentially + * still busy servicing other subsystems' resume, and can exhaust its + * retry budget outright -- confirmed on hardware, see + * notes/2026-09-24-0147-typec-resume-reverify.md. A real physical + * replug never hits this because a human always does it well after + * the whole system has settled. + * + * PM_POST_SUSPEND is delivered strictly after dpm_resume_end() -- + * i.e. after every device, including ACIO/Thunderbolt, has completed + * its own ordinary resume -- so triggering the reverify from here + * instead removes the race without needing any arbitrary delay. + */ +static int tipd_pm_notify(struct notifier_block *nb, unsigned long action, void *data) +{ + struct tps6598x *tps = container_of(nb, struct tps6598x, pm_nb); + + if (action == PM_POST_SUSPEND && tps->data->resume_reverify) + tps->data->resume_reverify(tps); + + return NOTIFY_DONE; +} + static const struct tipd_data tipd_cd321x_data = { .irq_handler = cd321x_interrupt, .irq_mask1 = APPLE_CD_REG_INT_POWER_STATUS_UPDATE | @@ -2018,6 +2094,7 @@ static const struct tipd_data tipd_cd321x_data = { .reset = cd321x_reset, .switch_power_state = cd321x_switch_power_state, .connect = cd321x_connect, + .resume_reverify = cd321x_resume_reverify, }; static const struct tipd_data tipd_tps6598x_data = { @@ -2073,6 +2150,7 @@ static const struct tipd_data tipd_sn201202x_data = { .reset = cd321x_reset, .switch_power_state = cd321x_switch_power_state, .connect = cd321x_connect, + .resume_reverify = cd321x_resume_reverify, }; const struct tipd_data *tipd_hw_variants[] = { diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index cfbbe4e8d1d86..a74b72e8329f7 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -303,6 +303,7 @@ struct tipd_data { bool (*read_data_status)(struct tps6598x *tps); int (*reset)(struct tps6598x *tps); int (*connect)(struct tps6598x *tps, u32 status); + void (*resume_reverify)(struct tps6598x *tps); }; struct tps6598x { @@ -329,6 +330,8 @@ struct tps6598x { u16 pwr_status; struct delayed_work wq_poll; + struct notifier_block pm_nb; + const struct tipd_data *data; }; From d00f8f1eb7f99deb957e363a7974437b9ef46079 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 20:24:03 -0700 Subject: [PATCH 0972/2680] thunderbolt: release host DP state before adapter and domain teardown Track whether the NHI post-activation hook has acquired display state, including partial setup. Release it once before disabling adapters and when the domain stops, while the host router is still accessible. Put the Apple DP IN adapter to sleep before marking its context dead. Add KUnit coverage for repeated cleanup after domain removal and cleanup without a prior host activation. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/thunderbolt/apple.c | 16 ++++------ drivers/thunderbolt/nhi.h | 3 +- drivers/thunderbolt/tb.c | 4 ++- drivers/thunderbolt/test.c | 57 ++++++++++++++++++++++++++++++++++++ drivers/thunderbolt/tunnel.c | 37 ++++++++++++----------- drivers/thunderbolt/tunnel.h | 3 ++ 6 files changed, 88 insertions(+), 32 deletions(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 1ab70f4eef075..4f87409c2ee7b 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -1491,22 +1491,16 @@ static void apple_nhi_dp_tunnel_deactivate(struct tb_nhi *nhi, if (READ_ONCE(apple_dpin_anhi) == anhi) WRITE_ONCE(apple_dpin_anhi, NULL); - /* - * Runs after both adapters are already disabled (tb_dp_activate()'s - * teardown order) -- later than aurora-silicon/linux#8's own - * dp_tunnel_changed(active=false), which fires before deactivation - * while the block is still powered. apple_dpin_dcp_set_active()'s - * own "block may already be off" check (matching the reference - * implementation's identical defensive check) covers this gap; a - * spurious no-op register access here is not a functional blocker, - * only a less clean teardown than the ideal ordering. - */ + /* Sleep the adapter before tunnel teardown can remove its power. */ idx = apple_dpin_index_for_port(anhi, in); if (idx >= 0 && idx <= 1 && anhi->acio && anhi->acio->dp_wq) { struct apple_dpin_ctx *c = &anhi->acio->dpin[idx]; - scoped_guard(mutex, &c->lock) + scoped_guard(mutex, &c->lock) { + if (c->alive && c->regs) + apple_dpin_set_active(c->acio, c->regs, c->idx, false); c->alive = false; + } queue_work(anhi->acio->dp_wq, &c->work); } dev_info(anhi->dev, "DP IN tunnel routing: tunnel down\n"); diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index dc85752d72d96..ea49aa02e1016 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -89,7 +89,8 @@ struct tb_nhi_ring_layout { * and before VE/AE. USB4 DPTX Discovery belongs here. * @dp_tunnel_post_activate: NHI specific hook run after a DP tunnel's * adapters have VE/AE enabled - * @dp_tunnel_deactivate: NHI specific hook run when a DP tunnel is torn down + * @dp_tunnel_deactivate: Releases state from an attempted post-activation hook, + * before adapters are disabled or the host router is removed * @is_present: Whether the device is currently present on the parent bus * @init_interrupts: NHI specific interrupt initialization hook * @reset_interface: Resets the host interface diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index 4e99275d00367..60f0248770bb0 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -3027,8 +3027,10 @@ static void tb_stop(struct tb *tb) else if (tb_tunnel_is_dp(tunnel) && !tb_tunnel_is_active(tunnel)) tb_tunnel_deactivate(tunnel); /* the host side of a DP tunnel goes away with us */ - else if (tb_tunnel_is_dp(tunnel)) + else if (tb_tunnel_is_dp(tunnel)) { tb_dp_tunnel_notify(tunnel, false); + tb_dp_tunnel_deactivate_host(tunnel); + } tb_tunnel_put(tunnel); } tb_switch_remove(tb->root_switch); diff --git a/drivers/thunderbolt/test.c b/drivers/thunderbolt/test.c index a1f9dbb669406..a7ac36549b846 100644 --- a/drivers/thunderbolt/test.c +++ b/drivers/thunderbolt/test.c @@ -3131,10 +3131,67 @@ static void tb_test_property_merge(struct kunit *test) tb_property_free_dir(dir1); } +struct tb_test_dp_host { + struct tb_nhi nhi; + unsigned int disconnects; +}; + +static void tb_test_dp_host_disconnect(struct tb_nhi *nhi, struct tb_port *in, + struct tb_port *out) +{ + struct tb_test_dp_host *host = container_of(nhi, struct tb_test_dp_host, nhi); + + host->disconnects++; +} + +static void tb_test_dp_host_teardown_once(struct kunit *test) +{ + static const struct tb_nhi_ops ops = { + .dp_tunnel_deactivate = tb_test_dp_host_disconnect, + }; + struct tb_test_dp_host *host; + struct tb_tunnel *tunnel; + struct tb *tb; + + host = kunit_kzalloc(test, sizeof(*host), GFP_KERNEL); + tunnel = kunit_kzalloc(test, sizeof(*tunnel), GFP_KERNEL); + tb = kunit_kzalloc(test, sizeof(*tb), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, host); + KUNIT_ASSERT_NOT_NULL(test, tunnel); + KUNIT_ASSERT_NOT_NULL(test, tb); + host->nhi.ops = &ops; + tb->nhi = &host->nhi; + tunnel->tb = tb; + tunnel->host_dp_activated = true; + + /* Domain removal must release the display even without adapter I/O. */ + tb_dp_tunnel_deactivate_host(tunnel); + KUNIT_EXPECT_EQ(test, host->disconnects, 1U); + KUNIT_EXPECT_FALSE(test, tunnel->host_dp_activated); + + /* A late tunnel cleanup must not access the already removed domain. */ + tunnel->tb = NULL; + tb_dp_tunnel_deactivate_host(tunnel); + KUNIT_EXPECT_EQ(test, host->disconnects, 1U); +} + +static void tb_test_dp_host_teardown_unannounced(struct kunit *test) +{ + struct tb_tunnel *tunnel; + + tunnel = kunit_kzalloc(test, sizeof(*tunnel), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tunnel); + /* A failed allocation or a domain-less test has no host state to undo. */ + tb_dp_tunnel_deactivate_host(tunnel); + KUNIT_EXPECT_FALSE(test, tunnel->host_dp_activated); +} + static struct kunit_case tb_test_cases[] = { KUNIT_CASE(tb_test_property_parse_u32_wrap), KUNIT_CASE(tb_test_property_parse_recursion), KUNIT_CASE(tb_test_property_parse_dir_len_underflow), + KUNIT_CASE(tb_test_dp_host_teardown_once), + KUNIT_CASE(tb_test_dp_host_teardown_unannounced), KUNIT_CASE(tb_test_path_basic), KUNIT_CASE(tb_test_path_not_connected_walk), KUNIT_CASE(tb_test_path_single_hop_walk), diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index f9eb7cd1aae31..e7debc0065073 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -1314,6 +1314,20 @@ static void tb_dp_apple_pulse_hpd(struct tb_port *in) tb_port_info(in, "Apple: HPD propagated\n"); } +/* Release display-side state while the host router is still accessible. */ +void tb_dp_tunnel_deactivate_host(struct tb_tunnel *tunnel) +{ + const struct tb_nhi_ops *ops; + + if (!tunnel->host_dp_activated) + return; + tunnel->host_dp_activated = false; + ops = tunnel->tb->nhi->ops; + if (ops && ops->dp_tunnel_deactivate) + ops->dp_tunnel_deactivate(tunnel->tb->nhi, tunnel->src_port, + tunnel->dst_port); +} + static int tb_dp_activate(struct tb_tunnel *tunnel, bool active) { int ret; @@ -1345,6 +1359,7 @@ static int tb_dp_activate(struct tb_tunnel *tunnel, bool active) return ret; } } else { + tb_dp_tunnel_deactivate_host(tunnel); tb_dp_dprx_stop(tunnel); tb_dp_port_hpd_clear(tunnel->src_port); tb_dp_port_set_hops(tunnel->src_port, 0, 0, 0); @@ -1353,18 +1368,7 @@ static int tb_dp_activate(struct tb_tunnel *tunnel, bool active) } ret = tb_dp_port_enable(tunnel->src_port, active); - /* - * On deactivate, a departing dst_port already marked unplugged - * makes tb_port_read/write short-circuit to -ENODEV with zero I/O - * (tb.h) -- deterministically, not just on hardware timing. Do not - * let that skip ops->dp_tunnel_deactivate() below: this tunnel.c is - * generic core code, but ops->dp_tunnel_deactivate is what apple.c - * relies on to clear its own apple_dpin_ctx latch -- without it, a - * physical unplug/replug can never re-arm a fresh connect. The caller - * (tb_tunnel_deactivate()) already discards this function's return - * value on the deactivate path, so relaxing it here changes nothing - * any caller observes; the activate (active=true) path is untouched. - */ + /* A departed adapter must not prevent disabling the remaining adapter. */ if (ret && active) return ret; @@ -1411,19 +1415,14 @@ static int tb_dp_activate(struct tb_tunnel *tunnel, bool active) const struct tb_nhi_ops *ops = tunnel->tb->nhi->ops; if (ops && ops->dp_tunnel_post_activate) { + /* Also unwind a hook that fails after partial setup. */ + tunnel->host_dp_activated = true; ret = ops->dp_tunnel_post_activate(tunnel->tb->nhi, tunnel->src_port, tunnel->dst_port); if (ret) return ret; } - } else { - const struct tb_nhi_ops *ops = tunnel->tb->nhi->ops; - - if (ops && ops->dp_tunnel_deactivate) - ops->dp_tunnel_deactivate(tunnel->tb->nhi, - tunnel->src_port, - tunnel->dst_port); } return active ? tb_dp_dprx_start(tunnel) : 0; diff --git a/drivers/thunderbolt/tunnel.h b/drivers/thunderbolt/tunnel.h index 660666669240c..b4e73d1142f4d 100644 --- a/drivers/thunderbolt/tunnel.h +++ b/drivers/thunderbolt/tunnel.h @@ -65,6 +65,7 @@ enum tb_tunnel_state { * @dprx_started: DPRX negotiation was started (tb_dp_dprx_start() was called for it) * @dprx_canceled: Was DPRX capabilities read poll canceled * @host_dp_notified: The NHI glue was told this DP tunnel is up (Apple hosts) + * @host_dp_activated: The NHI post-activation hook owns display-side state * @dprx_timeout: If set DPRX capabilities read poll work will timeout after this passes * @dprx_work: Worker that is scheduled to poll completion of DPRX capabilities read * @callback: Callback called when DP tunnel is fully activated @@ -103,6 +104,7 @@ struct tb_tunnel { bool dprx_started; bool dprx_canceled; bool host_dp_notified; + bool host_dp_activated; ktime_t dprx_timeout; struct delayed_work dprx_work; void (*callback)(struct tb_tunnel *tunnel); @@ -136,6 +138,7 @@ struct tb_tunnel *tb_tunnel_alloc_usb3(struct tb *tb, struct tb_port *up, int max_down); void tb_tunnel_put(struct tb_tunnel *tunnel); +void tb_dp_tunnel_deactivate_host(struct tb_tunnel *tunnel); int tb_tunnel_activate(struct tb_tunnel *tunnel); void tb_tunnel_deactivate(struct tb_tunnel *tunnel); From 41aded0b7c02a79761f08ba8e4c0bb4384b13a09 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 20:24:04 -0700 Subject: [PATCH 0973/2680] usb: typec: tipd: serialize post-suspend port revalidation Take the port mutex while reading status and scheduling the synthetic disconnect/reconnect so the PM notifier cannot race the IRQ and debounce workers. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/usb/typec/tipd/core.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 18e43359159fe..82d8d508ce2c1 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -2071,8 +2071,11 @@ static int tipd_pm_notify(struct notifier_block *nb, unsigned long action, void { struct tps6598x *tps = container_of(nb, struct tps6598x, pm_nb); - if (action == PM_POST_SUSPEND && tps->data->resume_reverify) + if (action == PM_POST_SUSPEND && tps->data->resume_reverify) { + mutex_lock(&tps->lock); tps->data->resume_reverify(tps); + mutex_unlock(&tps->lock); + } return NOTIFY_DONE; } From 5cae0c23677050e3df965c6755d1b3821a159c0b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 20:24:04 -0700 Subject: [PATCH 0974/2680] drm/apple: keep the j416s tunnel hotplug policy scoped to M2 Pro MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retain the M1 link-status handling when combining the two display series. Document the T602X PHY state carried alongside the M1 clock state. [ryan: adapted to 7.2.10 — split by topic: only the drivers/phy/apple hunks of this dcp-display commit; the rest is in 200-dcp] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/phy/apple/atc.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index bd10c7cd9856a..ce0b3af536aa0 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -660,6 +660,9 @@ struct atcphy_hw { * @dp_link_rate: DisplayPort link rate * @pipe_state: Backend the PIPE mux ("pipehandler") is routed to * @tunnel_clock_on: True while the DisplayPort-over-Thunderbolt pixel clock runs + * @tunnel_attempted: A T602X tunnel clock setup has been attempted + * @tunnel_saved: T602X PHY registers have been saved for tunnel teardown + * @tunnel_saved_regs: Original values of the T602X tunnel clock registers * @tunnel_rate: DP link rate code the tunnel pixel clock is set up for * @regs: Memory-mapped registers * @regs.core: Core registers From a14db997a9cb77cc6d14a794cce1d3f49aa00be7 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 21:00:22 -0700 Subject: [PATCH 0975/2680] thunderbolt: apple: protect diagnostics and apply host DP credits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sample the optional analog diagnostic at tunnel activation instead of dereferencing a global controller from sysfs. Disable AUX work before domain removal so racing activation cannot rearm it. Use an explicit host DP credit quirk for both allocation and programming, including j416s without the M1 notification hook. Cover the quirk with KUnit and document the diagnostic parameters. [ryan: adapted to 7.2.10 — Asahi took QUIRK bits 2-4 (RESET_DMA_ON_TEARDOWN, NO_DMA_PORT, NO_USB3_BW_ALLOC), so QUIRK_HOST_DP_NFC_CREDITS moves to BIT(5); the KUnit case uses the 7.2 tb_tunnel_alloc_dp() with its mandatory, data-less callback] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- .../admin-guide/kernel-parameters.txt | 25 +++++++++++ drivers/thunderbolt/apple.c | 42 +++++-------------- drivers/thunderbolt/nhi.h | 1 + drivers/thunderbolt/switch.c | 2 +- drivers/thunderbolt/tb.h | 8 ++++ drivers/thunderbolt/test.c | 38 +++++++++++++++++ drivers/thunderbolt/tunnel.c | 4 +- 7 files changed, 84 insertions(+), 36 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 5521eaea7fb26..0d0d644297c89 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -7557,6 +7557,14 @@ Kernel parameters See Documentation/admin-guide/mm/transhuge.rst for more details. + thunderbolt.dp_video_counter= + [HW,ARM64] + Format: + Enable diagnostic counter 0 on the DP video path's + host and downstream ingress hops on the Apple j416s + right-hand USB-C route. Read counters through Thunderbolt + debugfs. This is a load-time setting. Default: false. + thunderbolt_apple.dp_display= [HW,ARM64] Format: @@ -7566,6 +7574,23 @@ Kernel parameters whenever the connection manager sets up a DP tunnel. Default: false + thunderbolt_apple.dpin_aux= + [HW,ARM64] + Format: + Pulse the experimental DP IN analog start control on + Apple j416s at the next DP tunnel activation. Writes to + the module parameter do not access a live controller. + Default: false (leave analog registers alone until + DPRX negotiation finishes or times out). + + thunderbolt_apple.dpin_mode_value= + [HW,ARM64] + Format: (0-15) + Experimental DP IN MODE_A/MODE_B value on Apple j416s. + Sampled for each adapter activation; values above 15 + make activation fail. Default: 8. The value is empirical + and does not select the negotiated DisplayPort rate. + tlbi= [X86-64] Format: {ipi} ipi: switch to IPI-based TLB flushing diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 4f87409c2ee7b..86c9a71f07214 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -1073,17 +1073,10 @@ static int apple_nhi_pci_tunnel_post_activate(struct tb_nhi *nhi) * (default). Tests whether consuming +0x18 aborted a handshake. * 1 = old pulse +0x00 / fill +0x20 (rollback) */ -static int apple_dpin_aux; -static struct apple_nhi *apple_dpin_anhi; - -static int apple_dpin_aux_set(const char *val, const struct kernel_param *kp); -static const struct kernel_param_ops apple_dpin_aux_ops = { - .set = apple_dpin_aux_set, - .get = param_get_int, -}; -module_param_cb(dpin_aux, &apple_dpin_aux_ops, &apple_dpin_aux, 0644); +static bool apple_dpin_aux; +module_param_named(dpin_aux, apple_dpin_aux, bool, 0644); MODULE_PARM_DESC(dpin_aux, - "Apple DP IN analog: 0=hands-off until timeout, 1=pulse +0x00"); + "Apple DP IN analog: pulse +0x00 at the next tunnel activation (default: false)"); static void apple_dp_dump_hop(struct tb_port *port, unsigned int hopid) { @@ -1284,21 +1277,6 @@ static void apple_dp_start_analog(struct apple_nhi *anhi, bool pulse) "dpin analog after start", false); } -static int apple_dpin_aux_set(const char *val, const struct kernel_param *kp) -{ - struct apple_nhi *anhi; - int v, ret; - - ret = kstrtoint(val, 0, &v); - if (ret) - return ret; - apple_dpin_aux = v; - anhi = READ_ONCE(apple_dpin_anhi); - if (v >= 1 && anhi) - apple_dp_start_analog(anhi, true); - return 0; -} - static int apple_dp_dptx_discover(struct tb_port *in) { u32 cs0 = 0; @@ -1432,7 +1410,9 @@ static int apple_nhi_dp_tunnel_post_activate(struct tb_nhi *nhi, anhi->analog_base = apple_dp_in_analog_base(anhi, in); anhi->dp_aux_polls = 0; anhi->dp_aux_armed = true; - WRITE_ONCE(apple_dpin_anhi, anhi); + /* The domain lock keeps this optional MMIO access ahead of teardown. */ + if (READ_ONCE(apple_dpin_aux)) + apple_dp_start_analog(anhi, true); apple_dp_dump_rc(anhi->acio); apple_dp_dump_vse(in->sw); @@ -1488,8 +1468,6 @@ static void apple_nhi_dp_tunnel_deactivate(struct tb_nhi *nhi, anhi->dp_aux_armed = false; cancel_delayed_work(&anhi->dp_aux_work); - if (READ_ONCE(apple_dpin_anhi) == anhi) - WRITE_ONCE(apple_dpin_anhi, NULL); /* Sleep the adapter before tunnel teardown can remove its power. */ idx = apple_dpin_index_for_port(anhi, in); @@ -1658,6 +1636,8 @@ static int apple_nhi_probe(struct platform_device *pdev) anhi->nhi.ring_layout = &apple_nhi_ring_layout; anhi->nhi.iobase = anhi->nhi_base; anhi->nhi.quirks = QUIRK_NO_DMA_PORT | QUIRK_NO_USB3_BW_ALLOC; + if (anhi->ops.dp_tunnel_changed || anhi->ops.dp_tunnel_post_activate) + anhi->nhi.quirks |= QUIRK_HOST_DP_NFC_CREDITS; anhi->nhi.hop_count = readl(anhi->nhi_base + APPLE_CIO_NHI_HOP_COUNT) & APPLE_CIO_NHI_HOP_COUNT_MASK; @@ -1752,10 +1732,8 @@ static void apple_nhi_remove(struct platform_device *pdev) { struct apple_nhi *anhi = platform_get_drvdata(pdev); - anhi->dp_aux_armed = false; - cancel_delayed_work_sync(&anhi->dp_aux_work); - if (READ_ONCE(apple_dpin_anhi) == anhi) - WRITE_ONCE(apple_dpin_anhi, NULL); + /* Stop racing tunnel activation from rearming work during removal. */ + disable_delayed_work_sync(&anhi->dp_aux_work); WRITE_ONCE(anhi->acio->nhi_pdev, NULL); tb_domain_remove(anhi->tb); wait_for_completion(&anhi->nhi.domain_released); diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index ea49aa02e1016..b9587d8697cdc 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -192,6 +192,7 @@ struct tb_nhi_ops { #define QUIRK_RESET_DMA_ON_TEARDOWN BIT(2) #define QUIRK_NO_DMA_PORT BIT(3) #define QUIRK_NO_USB3_BW_ALLOC BIT(4) +#define QUIRK_HOST_DP_NFC_CREDITS BIT(5) /* * Minimal number of vectors when we use MSI-X. Two for control channel diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c index 88bbb4f714721..22f071430763a 100644 --- a/drivers/thunderbolt/switch.c +++ b/drivers/thunderbolt/switch.c @@ -576,7 +576,7 @@ int tb_port_add_nfc_credits(struct tb_port *port, int credits) * so skip other ports. */ if (tb_switch_is_usb4(port->sw) && !tb_port_is_null(port) && - !tb_port_is_apple_host_dpin(port)) + !tb_port_needs_host_dp_credits(port)) return 0; nfc_credits = port->config.nfc_credits & ADP_CS_4_NFC_BUFFERS_MASK; diff --git a/drivers/thunderbolt/tb.h b/drivers/thunderbolt/tb.h index 3e4cbedd55963..a02920275e30e 100644 --- a/drivers/thunderbolt/tb.h +++ b/drivers/thunderbolt/tb.h @@ -1144,6 +1144,14 @@ static inline bool tb_port_is_apple_host_dpin(const struct tb_port *port) return tb->nhi->ops && tb->nhi->ops->dp_tunnel_changed; } +static inline bool tb_port_needs_host_dp_credits(const struct tb_port *port) +{ + const struct tb *tb = port->sw->tb; + + return tb && tb->nhi && !tb_route(port->sw) && tb_port_is_dpin(port) && + (tb->nhi->quirks & QUIRK_HOST_DP_NFC_CREDITS); +} + static inline bool tb_port_use_credit_allocation(const struct tb_port *port) { return tb_port_is_null(port) && port->sw->credit_allocation; diff --git a/drivers/thunderbolt/test.c b/drivers/thunderbolt/test.c index a7ac36549b846..bb5bce1c7d3c2 100644 --- a/drivers/thunderbolt/test.c +++ b/drivers/thunderbolt/test.c @@ -3186,10 +3186,48 @@ static void tb_test_dp_host_teardown_unannounced(struct kunit *test) KUNIT_EXPECT_FALSE(test, tunnel->host_dp_activated); } +static void tb_test_tunnel_dp_host_credits(struct kunit *test) +{ + struct tb_switch *host, *dev; + struct tb_tunnel *tunnel; + struct tb_nhi *nhi; + struct tb *tb; + + host = alloc_host(test); + KUNIT_ASSERT_NOT_NULL(test, host); + dev = alloc_dev_with_dpin(test, host, 0x3, true); + KUNIT_ASSERT_NOT_NULL(test, dev); + tb = kunit_kzalloc(test, sizeof(*tb), GFP_KERNEL); + nhi = kunit_kzalloc(test, sizeof(*nhi), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tb); + KUNIT_ASSERT_NOT_NULL(test, nhi); + tb->nhi = nhi; + host->tb = tb; + dev->tb = tb; + + KUNIT_EXPECT_FALSE(test, tb_port_needs_host_dp_credits(&host->ports[5])); + /* j416s has no dp_tunnel_changed hook; the quirk must be sufficient. */ + nhi->quirks = QUIRK_HOST_DP_NFC_CREDITS; + KUNIT_EXPECT_TRUE(test, tb_port_needs_host_dp_credits(&host->ports[5])); + KUNIT_EXPECT_FALSE(test, tb_port_needs_host_dp_credits(&host->ports[1])); + KUNIT_EXPECT_FALSE(test, tb_port_needs_host_dp_credits(&dev->ports[13])); + + dev->ports[14].config.type = TB_TYPE_DP_HDMI_OUT; + tunnel = tb_tunnel_alloc_dp(NULL, &host->ports[5], &dev->ports[14], + 1, 0, 0, tb_test_dp_tunnel_active); + KUNIT_ASSERT_NOT_NULL(test, tunnel); + KUNIT_EXPECT_EQ(test, tunnel->paths[0]->hops[0].nfc_credits, 5U); + tb_tunnel_put(tunnel); + + host->tb = NULL; + KUNIT_EXPECT_FALSE(test, tb_port_needs_host_dp_credits(&host->ports[5])); +} + static struct kunit_case tb_test_cases[] = { KUNIT_CASE(tb_test_property_parse_u32_wrap), KUNIT_CASE(tb_test_property_parse_recursion), KUNIT_CASE(tb_test_property_parse_dir_len_underflow), + KUNIT_CASE(tb_test_tunnel_dp_host_credits), KUNIT_CASE(tb_test_dp_host_teardown_once), KUNIT_CASE(tb_test_dp_host_teardown_unannounced), KUNIT_CASE(tb_test_path_basic), diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index e7debc0065073..f495772bb20bb 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -1735,9 +1735,7 @@ static int tb_dp_init_video_credits(struct tb_path_hop *hop) * Apple silicon host DP IN adapters take 5 NFC credits for the * video path -- same source as the HPD/NO_AUTO_LT handling above. */ - if (tb_port_is_apple_host_dpin(port) || - (port->sw->tb && tb_nhi_is_apple(port->sw->tb->nhi) && - !tb_route(port->sw) && tb_port_is_dpin(port))) { + if (tb_port_needs_host_dp_credits(port)) { hop->nfc_credits = 5; return 0; } From 14345808a0a5f09944a3bccaba705c873dadc968 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 21:00:22 -0700 Subject: [PATCH 0976/2680] phy: apple: atc: allow owned T602X tunnel clocks to change rate DCP can request a new nonzero link rate during retraining. Restore the descriptor owned by this tunnel and start the requested rate through the guarded T602X sequence instead of permanently refusing it as busy. Keep rejecting clocks owned by another user. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/phy/apple/atc.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index ce0b3af536aa0..6a3ab34b02775 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2340,10 +2340,10 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy, u8 rate) return -EINVAL; } if (atcphy->tunnel_saved) { - if (atcphy->tunnel_rate != rate) - dev_info(atcphy->dev, "USB4 tunnel clock busy: active rate=0x%x requested=0x%x\n", - atcphy->tunnel_rate, rate); - return atcphy->tunnel_rate == rate ? 0 : -EBUSY; + if (atcphy->tunnel_rate == rate) + return 0; + /* A retrain may change rate without first requesting rate zero. */ + atc_tunnel_restore(atcphy); } if (atcphy->tunnel_attempted) return -EALREADY; From ca7ba9876e089022179d9ead827e9a6a32089c5e Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 23:51:53 -0700 Subject: [PATCH 0977/2680] thunderbolt: preserve PCIe host ownership across system resume On M1 with a TS4 and igc bound, resume discovers a temporary PCIe tunnel and deactivates it before restoring the saved tunnels. Its host hook removes the live PCI hierarchy while endpoint drivers are suspended, triggering an igc double-disable warning. Deferred activation can also rescan on system_wq before the PCI host and DART have resumed, producing a sleeping-parent warning and a DART translation fault. Track host ownership only after invoking the PCIe post-activation hook. Discovered tunnels cannot tear down another tunnel's host state, repeated teardown is harmless, and failed cleanup retains ownership for retry. Add a KUnit regression covering these cases; it fails before the fix. On Apple hosts, defer PCI removal during system PM until endpoint drivers have resumed, then process pending teardown before activation. Use a freezable workqueue and prepare/complete gating so topology changes cannot race noirq resume. Record activation only after the adapters are enabled, cancel obsolete requests on teardown, and propagate restore errors. All 46 DART/Thunderbolt KUnit tests and native W=1 Apple Thunderbolt and tunnel object builds pass. Hardware suspend/resume retesting is pending. Fixes: f6d4f47575f3 ("thunderbolt: Add a PCI tunnel deactivate callback") Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/thunderbolt/apple.c | 98 +++++++++++++++++++++++------------- drivers/thunderbolt/test.c | 58 +++++++++++++++++++++ drivers/thunderbolt/tunnel.c | 22 +++++--- drivers/thunderbolt/tunnel.h | 3 ++ 4 files changed, 139 insertions(+), 42 deletions(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 86c9a71f07214..34399e9d9b968 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -216,6 +216,8 @@ struct apple_cio { struct delayed_work pcie_tunnel_work; bool pcie_tunnel_requested; bool pcie_tunnel_populated; + bool pcie_pm_prepared; + bool pcie_quiesce_pending; /* Type-C connector this router is wired to, for DP tunnel routing */ struct device_node *connector_np; @@ -923,7 +925,7 @@ static int apple_cio_populate_pcie_tunnel(struct apple_cio *acio) static int apple_cio_activate_pcie_tunnel_locked(struct apple_cio *acio) { - int ret; + int ret = 0; lockdep_assert_held(&acio->pcie_tunnel_lock); @@ -946,7 +948,7 @@ static int apple_cio_activate_pcie_tunnel_locked(struct apple_cio *acio) ret); put_device(&pcie_pdev->dev); } - return 0; + return ret; } if (!acio->pcie_tunnel_preinitialized) return dev_err_probe(acio->dev, -ENODEV, @@ -967,31 +969,50 @@ static int apple_cio_activate_pcie_tunnel_locked(struct apple_cio *acio) return 0; } +static int apple_cio_quiesce_pcie_tunnel_locked(struct apple_cio *acio) +{ + struct platform_device *pcie_pdev; + int ret = 0; + + lockdep_assert_held(&acio->pcie_tunnel_lock); + pcie_pdev = apple_cio_find_pcie_tunnel(acio); + if (pcie_pdev) { + ret = apple_pcie_tunnel_quiesce(&pcie_pdev->dev); + put_device(&pcie_pdev->dev); + } + if (!ret) + acio->pcie_quiesce_pending = false; + return ret; +} + static void apple_cio_pcie_tunnel_work(struct work_struct *work) { struct apple_cio *acio = container_of(to_delayed_work(work), struct apple_cio, pcie_tunnel_work); - int ret; + int ret = 0; mutex_lock(&acio->pcie_tunnel_lock); - if (!READ_ONCE(acio->pcie_tunnel_requested)) { - mutex_unlock(&acio->pcie_tunnel_lock); - return; + if (acio->pcie_pm_prepared) + goto unlock; + if (acio->pcie_quiesce_pending) { + ret = apple_cio_quiesce_pcie_tunnel_locked(acio); + if (ret) + goto unlock; } - - ret = apple_cio_activate_pcie_tunnel_locked(acio); + if (READ_ONCE(acio->pcie_tunnel_requested)) + ret = apple_cio_activate_pcie_tunnel_locked(acio); +unlock: mutex_unlock(&acio->pcie_tunnel_lock); if (ret && ret != -EAGAIN) - dev_err(acio->dev, "deferred PCIe-C activation failed: %d\n", ret); + dev_err(acio->dev, "deferred PCIe-C transition failed: %d\n", ret); } static int apple_nhi_pci_tunnel_deactivate(struct tb_nhi *nhi) { struct apple_nhi *anhi = nhi_to_anhi(nhi); struct apple_cio *acio = anhi->acio; - struct platform_device *pcie_pdev; - int ret; + int ret = 0; /* * The tunnel is going away while the router and NHI stay up, so nothing @@ -1001,37 +1022,21 @@ static int apple_nhi_pci_tunnel_deactivate(struct tb_nhi *nhi) * nothing to enumerate onto when the tunnel comes back. */ mutex_lock(&acio->pcie_tunnel_lock); - if (!acio->pcie_tunnel_populated) { - mutex_unlock(&acio->pcie_tunnel_lock); - return 0; - } - - pcie_pdev = apple_cio_find_pcie_tunnel(acio); - if (pcie_pdev) { - ret = apple_pcie_tunnel_quiesce(&pcie_pdev->dev); + WRITE_ONCE(acio->pcie_tunnel_requested, false); + cancel_delayed_work(&acio->pcie_tunnel_work); + acio->pcie_quiesce_pending = true; + /* Endpoint drivers must resume before they can be unbound safely. */ + if (!acio->pcie_pm_prepared) { + ret = apple_cio_quiesce_pcie_tunnel_locked(acio); if (ret) dev_warn(acio->dev, "failed to quiesce PCIe-C on tunnel teardown: %d\n", ret); - put_device(&pcie_pdev->dev); } mutex_unlock(&acio->pcie_tunnel_lock); - return 0; -} - -static int apple_nhi_pci_tunnel_pre_activate(struct tb_nhi *nhi) -{ - struct apple_nhi *anhi = nhi_to_anhi(nhi); - struct apple_cio *acio = anhi->acio; - - /* - * Remember that the PCIe tunnel reached activation. PCIe-C must not be - * populated here: the tunnel paths and both adapters are still disabled. - */ - WRITE_ONCE(acio->pcie_tunnel_requested, true); - return 0; + return ret; } static int apple_nhi_pci_tunnel_post_activate(struct tb_nhi *nhi) @@ -1045,7 +1050,11 @@ static int apple_nhi_pci_tunnel_post_activate(struct tb_nhi *nhi) * Thunderbolt core before probing PCIe-C. The worker then waits on the * hardware reset state without blocking completion of the tunnel paths. */ - mod_delayed_work(system_wq, &acio->pcie_tunnel_work, 0); + mutex_lock(&acio->pcie_tunnel_lock); + WRITE_ONCE(acio->pcie_tunnel_requested, true); + /* No PCI removal or rescan until system resume has finished. */ + mod_delayed_work(system_freezable_wq, &acio->pcie_tunnel_work, 0); + mutex_unlock(&acio->pcie_tunnel_lock); return 0; } @@ -1536,7 +1545,6 @@ static const struct tb_nhi_ops apple_nhi_ops = { .ring_interrupt_active = apple_nhi_ring_interrupt_active, .ring_interrupt_mask = apple_nhi_ring_interrupt_mask, .ring_configure = apple_nhi_ring_configure, - .pci_tunnel_pre_activate = apple_nhi_pci_tunnel_pre_activate, .pci_tunnel_post_activate = apple_nhi_pci_tunnel_post_activate, .pci_tunnel_deactivate = apple_nhi_pci_tunnel_deactivate, }; @@ -1744,6 +1752,16 @@ static void apple_nhi_remove(struct platform_device *pdev) * apple_nhi pointer in driver data. The generic NHI PM callbacks cannot be * installed here because they expect driver data to contain struct tb. */ +static int apple_nhi_prepare(struct device *dev) +{ + struct apple_nhi *anhi = dev_get_drvdata(dev); + struct apple_cio *acio = anhi->acio; + + guard(mutex)(&acio->pcie_tunnel_lock); + acio->pcie_pm_prepared = true; + return 0; +} + static int apple_nhi_suspend_noirq(struct device *dev) { struct apple_nhi *anhi = dev_get_drvdata(dev); @@ -1782,11 +1800,18 @@ static int apple_nhi_suspend(struct device *dev) static void apple_nhi_complete(struct device *dev) { struct apple_nhi *anhi = dev_get_drvdata(dev); + struct apple_cio *acio = anhi->acio; tb_domain_complete(anhi->tb); + + guard(mutex)(&acio->pcie_tunnel_lock); + acio->pcie_pm_prepared = false; + if (acio->pcie_quiesce_pending || READ_ONCE(acio->pcie_tunnel_requested)) + mod_delayed_work(system_freezable_wq, &acio->pcie_tunnel_work, 0); } static const struct dev_pm_ops apple_nhi_pm_ops = { + .prepare = apple_nhi_prepare, .suspend = apple_nhi_suspend, .suspend_noirq = apple_nhi_suspend_noirq, .resume_noirq = apple_nhi_resume_noirq, @@ -1895,6 +1920,7 @@ static void apple_cio_stop(struct apple_cio *acio) of_platform_depopulate(acio->dev); acio->pcie_tunnel_populated = false; + acio->pcie_quiesce_pending = false; mutex_unlock(&acio->pcie_tunnel_lock); /* Try to shut down and power off the co-processor gracefully */ diff --git a/drivers/thunderbolt/test.c b/drivers/thunderbolt/test.c index bb5bce1c7d3c2..f710355f770d7 100644 --- a/drivers/thunderbolt/test.c +++ b/drivers/thunderbolt/test.c @@ -3131,6 +3131,63 @@ static void tb_test_property_merge(struct kunit *test) tb_property_free_dir(dir1); } +struct tb_test_pci_host { + struct tb_nhi nhi; + unsigned int disconnects; + int result; +}; + +static int tb_test_pci_host_disconnect(struct tb_nhi *nhi) +{ + struct tb_test_pci_host *host = container_of(nhi, struct tb_test_pci_host, nhi); + + host->disconnects++; + return host->result; +} + +static void tb_test_pci_host_teardown(struct kunit *test) +{ + static const struct tb_nhi_ops ops = { + .pci_tunnel_deactivate = tb_test_pci_host_disconnect, + }; + struct tb_test_pci_host *host; + struct tb_tunnel *tunnel; + struct tb *tb; + + host = kunit_kzalloc(test, sizeof(*host), GFP_KERNEL); + tunnel = kunit_kzalloc(test, sizeof(*tunnel), GFP_KERNEL); + tb = kunit_kzalloc(test, sizeof(*tb), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, host); + KUNIT_ASSERT_NOT_NULL(test, tunnel); + KUNIT_ASSERT_NOT_NULL(test, tb); + host->nhi.ops = &ops; + tb->nhi = &host->nhi; + tunnel->tb = tb; + + /* A discovered firmware tunnel does not own Linux's PCIe hierarchy. */ + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_deactivate_host(tunnel), 0); + KUNIT_EXPECT_EQ(test, host->disconnects, 0U); + + /* An activated tunnel must release the hierarchy exactly once. */ + tunnel->host_pci_activated = true; + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_deactivate_host(tunnel), 0); + KUNIT_EXPECT_EQ(test, host->disconnects, 1U); + KUNIT_EXPECT_FALSE(test, tunnel->host_pci_activated); + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_deactivate_host(tunnel), 0); + KUNIT_EXPECT_EQ(test, host->disconnects, 1U); + + /* Failed teardown keeps ownership so a later attempt can retry. */ + tunnel->host_pci_activated = true; + host->result = -EIO; + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_deactivate_host(tunnel), -EIO); + KUNIT_EXPECT_TRUE(test, tunnel->host_pci_activated); + KUNIT_EXPECT_EQ(test, host->disconnects, 2U); + host->result = 0; + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_deactivate_host(tunnel), 0); + KUNIT_EXPECT_FALSE(test, tunnel->host_pci_activated); + KUNIT_EXPECT_EQ(test, host->disconnects, 3U); +} + struct tb_test_dp_host { struct tb_nhi nhi; unsigned int disconnects; @@ -3227,6 +3284,7 @@ static struct kunit_case tb_test_cases[] = { KUNIT_CASE(tb_test_property_parse_u32_wrap), KUNIT_CASE(tb_test_property_parse_recursion), KUNIT_CASE(tb_test_property_parse_dir_len_underflow), + KUNIT_CASE(tb_test_pci_host_teardown), KUNIT_CASE(tb_test_tunnel_dp_host_credits), KUNIT_CASE(tb_test_dp_host_teardown_once), KUNIT_CASE(tb_test_dp_host_teardown_unannounced), diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index f495772bb20bb..f9d3b26814a7f 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -401,18 +401,28 @@ static int tb_pci_post_activate(struct tb_tunnel *tunnel) { const struct tb_nhi_ops *ops = tunnel->tb->nhi->ops; - if (ops && ops->pci_tunnel_post_activate) + if (ops && ops->pci_tunnel_post_activate) { + /* Also unwind a hook that fails after partial host setup. */ + tunnel->host_pci_activated = true; return ops->pci_tunnel_post_activate(tunnel->tb->nhi); + } return 0; } -static int tb_pci_deactivate(struct tb_tunnel *tunnel) +int tb_pci_tunnel_deactivate_host(struct tb_tunnel *tunnel) { - const struct tb_nhi_ops *ops = tunnel->tb->nhi->ops; + const struct tb_nhi_ops *ops; + int ret = 0; + /* Firmware tunnels discovered during resume do not own host state. */ + if (!tunnel->host_pci_activated) + return 0; + ops = tunnel->tb->nhi->ops; if (ops && ops->pci_tunnel_deactivate) - return ops->pci_tunnel_deactivate(tunnel->tb->nhi); - return 0; + ret = ops->pci_tunnel_deactivate(tunnel->tb->nhi); + if (!ret) + tunnel->host_pci_activated = false; + return ret; } static int tb_pci_activate(struct tb_tunnel *tunnel, bool activate) @@ -424,7 +434,7 @@ static int tb_pci_activate(struct tb_tunnel *tunnel, bool activate) * what is behind it can still be reached. */ if (!activate) { - res = tb_pci_deactivate(tunnel); + res = tb_pci_tunnel_deactivate_host(tunnel); if (res) return res; } diff --git a/drivers/thunderbolt/tunnel.h b/drivers/thunderbolt/tunnel.h index b4e73d1142f4d..3a3b89d0ab980 100644 --- a/drivers/thunderbolt/tunnel.h +++ b/drivers/thunderbolt/tunnel.h @@ -66,6 +66,7 @@ enum tb_tunnel_state { * @dprx_canceled: Was DPRX capabilities read poll canceled * @host_dp_notified: The NHI glue was told this DP tunnel is up (Apple hosts) * @host_dp_activated: The NHI post-activation hook owns display-side state + * @host_pci_activated: The NHI post-activation hook owns PCIe host state * @dprx_timeout: If set DPRX capabilities read poll work will timeout after this passes * @dprx_work: Worker that is scheduled to poll completion of DPRX capabilities read * @callback: Callback called when DP tunnel is fully activated @@ -105,6 +106,7 @@ struct tb_tunnel { bool dprx_canceled; bool host_dp_notified; bool host_dp_activated; + bool host_pci_activated; ktime_t dprx_timeout; struct delayed_work dprx_work; void (*callback)(struct tb_tunnel *tunnel); @@ -116,6 +118,7 @@ struct tb_tunnel *tb_tunnel_discover_pci(struct tb *tb, struct tb_port *down, bool alloc_hopid); struct tb_tunnel *tb_tunnel_alloc_pci(struct tb *tb, struct tb_port *up, struct tb_port *down); +int tb_pci_tunnel_deactivate_host(struct tb_tunnel *tunnel); bool tb_tunnel_reserved_pci(struct tb_port *port, int *reserved_up, int *reserved_down); struct tb_tunnel *tb_tunnel_discover_dp(struct tb *tb, struct tb_port *in, From 49a9a053504d60129ff59ae601f07bf08dddad38 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:06:29 -0700 Subject: [PATCH 0978/2680] thunderbolt: scope native PCIe ownership to host tunnels Only tunnels originating at the host router control its native PCIe port. Calling NHI hooks for a downstream daisy-chain tunnel can otherwise quiesce the first dock's entire PCI hierarchy when only the second dock disconnects. Adopt host state for enabled PCIe tunnels retained at domain startup so an NHI rebind can restore a previously quiesced host. Temporary tunnels discovered solely for resume cleanup remain unowned. Add a KUnit daisy-chain regression covering hook isolation, retained host ownership and partial activation failure cleanup. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/thunderbolt/nhi.h | 9 +++-- drivers/thunderbolt/tb.c | 3 ++ drivers/thunderbolt/test.c | 78 ++++++++++++++++++++++++++++++++++++ drivers/thunderbolt/tunnel.c | 9 ++++- drivers/thunderbolt/tunnel.h | 1 + 5 files changed, 95 insertions(+), 5 deletions(-) diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index b9587d8697cdc..e272166635c82 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -81,10 +81,13 @@ struct tb_nhi_ring_layout { * @ring_configure: NHI specific hook to program the ring options registers * and enable the ring with the given flags. If not set * the standard USB4 NHI registers are used. - * @pci_tunnel_pre_activate: NHI specific hook run after a PCIe tunnel has + * @pci_tunnel_pre_activate: NHI specific hook run after a host PCIe tunnel has * been allocated but before its paths and adapters are enabled - * @pci_tunnel_post_activate: NHI specific hook run after a PCIe tunnel's - * paths and adapters have been enabled + * @pci_tunnel_post_activate: NHI specific hook run after a host PCIe tunnel's + * paths and adapters have been enabled, including + * adoption of an enabled tunnel at domain startup + * @pci_tunnel_deactivate: Release native host state acquired by the + * PCIe post-activation hook * @dp_tunnel_pre_activate: NHI specific hook run after DP hops are programmed * and before VE/AE. USB4 DPTX Discovery belongs here. * @dp_tunnel_post_activate: NHI specific hook run after a DP tunnel's diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index 60f0248770bb0..58c61acf397c0 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -1707,6 +1707,9 @@ static void tb_discover_tunnels(struct tb *tb) if (tb_tunnel_is_pci(tunnel)) { struct tb_switch *parent = tunnel->dst_port->sw; + /* Adopt host state only for tunnels retained by this domain. */ + if (tb_pci_tunnel_activate_host(tunnel)) + tb_tunnel_warn(tunnel, "failed to adopt PCIe host state\n"); while (parent != tunnel->src_port->sw) { parent->boot = true; parent = tb_switch_parent(parent); diff --git a/drivers/thunderbolt/test.c b/drivers/thunderbolt/test.c index f710355f770d7..2bbc7deb57a5f 100644 --- a/drivers/thunderbolt/test.c +++ b/drivers/thunderbolt/test.c @@ -3133,10 +3133,28 @@ static void tb_test_property_merge(struct kunit *test) struct tb_test_pci_host { struct tb_nhi nhi; + unsigned int prepares; + unsigned int connects; unsigned int disconnects; int result; }; +static int tb_test_pci_host_prepare(struct tb_nhi *nhi) +{ + struct tb_test_pci_host *host = container_of(nhi, struct tb_test_pci_host, nhi); + + host->prepares++; + return host->result; +} + +static int tb_test_pci_host_connect(struct tb_nhi *nhi) +{ + struct tb_test_pci_host *host = container_of(nhi, struct tb_test_pci_host, nhi); + + host->connects++; + return host->result; +} + static int tb_test_pci_host_disconnect(struct tb_nhi *nhi) { struct tb_test_pci_host *host = container_of(nhi, struct tb_test_pci_host, nhi); @@ -3188,6 +3206,65 @@ static void tb_test_pci_host_teardown(struct kunit *test) KUNIT_EXPECT_EQ(test, host->disconnects, 3U); } +static void tb_test_pci_host_daisy_chain(struct kunit *test) +{ + static const struct tb_nhi_ops ops = { + .pci_tunnel_pre_activate = tb_test_pci_host_prepare, + .pci_tunnel_post_activate = tb_test_pci_host_connect, + .pci_tunnel_deactivate = tb_test_pci_host_disconnect, + }; + struct tb_tunnel *root_tunnel, *downstream_tunnel; + struct tb_switch *root, *dev1, *dev2; + struct tb_test_pci_host *host; + struct tb *tb; + + host = kunit_kzalloc(test, sizeof(*host), GFP_KERNEL); + tb = kunit_kzalloc(test, sizeof(*tb), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, host); + KUNIT_ASSERT_NOT_NULL(test, tb); + host->nhi.ops = &ops; + tb->nhi = &host->nhi; + root = alloc_host(test); + KUNIT_ASSERT_NOT_NULL(test, root); + dev1 = alloc_dev_default(test, root, 0x1, true); + KUNIT_ASSERT_NOT_NULL(test, dev1); + dev2 = alloc_dev_default(test, dev1, 0x501, true); + KUNIT_ASSERT_NOT_NULL(test, dev2); + root_tunnel = tb_tunnel_alloc_pci(tb, &dev1->ports[9], &root->ports[8]); + KUNIT_ASSERT_NOT_NULL(test, root_tunnel); + downstream_tunnel = tb_tunnel_alloc_pci(tb, &dev2->ports[9], &dev1->ports[10]); + KUNIT_ASSERT_NOT_NULL(test, downstream_tunnel); + + KUNIT_EXPECT_EQ(test, root_tunnel->pre_activate(root_tunnel), 0); + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_activate_host(root_tunnel), 0); + KUNIT_EXPECT_TRUE(test, root_tunnel->host_pci_activated); + KUNIT_EXPECT_EQ(test, host->prepares, 1U); + KUNIT_EXPECT_EQ(test, host->connects, 1U); + + /* A second dock must not acquire or tear down the host's PCIe port. */ + KUNIT_EXPECT_EQ(test, downstream_tunnel->pre_activate(downstream_tunnel), 0); + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_activate_host(downstream_tunnel), 0); + KUNIT_EXPECT_FALSE(test, downstream_tunnel->host_pci_activated); + KUNIT_EXPECT_EQ(test, host->prepares, 1U); + KUNIT_EXPECT_EQ(test, host->connects, 1U); + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_deactivate_host(downstream_tunnel), 0); + KUNIT_EXPECT_EQ(test, host->disconnects, 0U); + KUNIT_EXPECT_TRUE(test, root_tunnel->host_pci_activated); + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_deactivate_host(root_tunnel), 0); + KUNIT_EXPECT_EQ(test, host->disconnects, 1U); + + /* A failed host activation may still need to unwind partial setup. */ + host->result = -EIO; + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_activate_host(root_tunnel), -EIO); + KUNIT_EXPECT_TRUE(test, root_tunnel->host_pci_activated); + host->result = 0; + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_deactivate_host(root_tunnel), 0); + KUNIT_EXPECT_EQ(test, host->disconnects, 2U); + + tb_tunnel_put(downstream_tunnel); + tb_tunnel_put(root_tunnel); +} + struct tb_test_dp_host { struct tb_nhi nhi; unsigned int disconnects; @@ -3285,6 +3362,7 @@ static struct kunit_case tb_test_cases[] = { KUNIT_CASE(tb_test_property_parse_recursion), KUNIT_CASE(tb_test_property_parse_dir_len_underflow), KUNIT_CASE(tb_test_pci_host_teardown), + KUNIT_CASE(tb_test_pci_host_daisy_chain), KUNIT_CASE(tb_test_tunnel_dp_host_credits), KUNIT_CASE(tb_test_dp_host_teardown_once), KUNIT_CASE(tb_test_dp_host_teardown_unannounced), diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index f9d3b26814a7f..9c35b236b4a8b 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -392,15 +392,20 @@ static int tb_pci_host_pre_activate(struct tb_tunnel *tunnel) { const struct tb_nhi_ops *ops = tunnel->tb->nhi->ops; + /* Only the tunnel starting at the host owns its native PCIe port. */ + if (tb_route(tunnel->src_port->sw)) + return 0; if (ops && ops->pci_tunnel_pre_activate) return ops->pci_tunnel_pre_activate(tunnel->tb->nhi); return 0; } -static int tb_pci_post_activate(struct tb_tunnel *tunnel) +int tb_pci_tunnel_activate_host(struct tb_tunnel *tunnel) { const struct tb_nhi_ops *ops = tunnel->tb->nhi->ops; + if (tb_route(tunnel->src_port->sw)) + return 0; if (ops && ops->pci_tunnel_post_activate) { /* Also unwind a hook that fails after partial host setup. */ tunnel->host_pci_activated = true; @@ -457,7 +462,7 @@ static int tb_pci_activate(struct tb_tunnel *tunnel, bool activate) if (res) return res; - return tb_pci_post_activate(tunnel); + return tb_pci_tunnel_activate_host(tunnel); } else { /* Downstream router could be unplugged */ tb_pci_port_enable(tunnel->dst_port, activate); diff --git a/drivers/thunderbolt/tunnel.h b/drivers/thunderbolt/tunnel.h index 3a3b89d0ab980..65a99b871117b 100644 --- a/drivers/thunderbolt/tunnel.h +++ b/drivers/thunderbolt/tunnel.h @@ -118,6 +118,7 @@ struct tb_tunnel *tb_tunnel_discover_pci(struct tb *tb, struct tb_port *down, bool alloc_hopid); struct tb_tunnel *tb_tunnel_alloc_pci(struct tb *tb, struct tb_port *up, struct tb_port *down); +int tb_pci_tunnel_activate_host(struct tb_tunnel *tunnel); int tb_pci_tunnel_deactivate_host(struct tb_tunnel *tunnel); bool tb_tunnel_reserved_pci(struct tb_port *port, int *reserved_up, int *reserved_down); From ec2929ed4385c2b2868c9b5b8887eb4b8b6e94b9 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:06:30 -0700 Subject: [PATCH 0979/2680] thunderbolt: apple: drain PCIe work before removing the domain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cable teardown held pcie_tunnel_lock while domain removal acquired tb->lock, reversing the order used by tunnel hooks. An in-flight hotplug could deadlock teardown, and simply cancelling the deferred worker did not prevent new requests from rearming it. Gate requests before draining work, quiesce the PCIe hierarchy while its NHI remains live, and release the PCIe lock before domain removal. Use the same ordered child cleanup on startup failure, and drain NHI diagnostics on probe failure. A new NHI domain reopens requests after setup. [ryan: adapted to 7.2.10 — Asahi NHI probe unwinds through err_unlock_tb_domain (domain lock held); the drain and PCIe stop follow the unlock there; Asahi apple_cio_stop() kerneldoc moved with the function] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/thunderbolt/apple.c | 105 ++++++++++++++++++++++-------------- 1 file changed, 66 insertions(+), 39 deletions(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 34399e9d9b968..cfaf030b4cb8b 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -216,6 +216,7 @@ struct apple_cio { struct delayed_work pcie_tunnel_work; bool pcie_tunnel_requested; bool pcie_tunnel_populated; + bool pcie_tunnel_stopping; bool pcie_pm_prepared; bool pcie_quiesce_pending; @@ -993,7 +994,7 @@ static void apple_cio_pcie_tunnel_work(struct work_struct *work) int ret = 0; mutex_lock(&acio->pcie_tunnel_lock); - if (acio->pcie_pm_prepared) + if (acio->pcie_tunnel_stopping || acio->pcie_pm_prepared) goto unlock; if (acio->pcie_quiesce_pending) { ret = apple_cio_quiesce_pcie_tunnel_locked(acio); @@ -1022,6 +1023,8 @@ static int apple_nhi_pci_tunnel_deactivate(struct tb_nhi *nhi) * nothing to enumerate onto when the tunnel comes back. */ mutex_lock(&acio->pcie_tunnel_lock); + if (acio->pcie_tunnel_stopping) + goto unlock; WRITE_ONCE(acio->pcie_tunnel_requested, false); cancel_delayed_work(&acio->pcie_tunnel_work); acio->pcie_quiesce_pending = true; @@ -1034,6 +1037,7 @@ static int apple_nhi_pci_tunnel_deactivate(struct tb_nhi *nhi) ret); } +unlock: mutex_unlock(&acio->pcie_tunnel_lock); return ret; @@ -1051,6 +1055,10 @@ static int apple_nhi_pci_tunnel_post_activate(struct tb_nhi *nhi) * hardware reset state without blocking completion of the tunnel paths. */ mutex_lock(&acio->pcie_tunnel_lock); + if (acio->pcie_tunnel_stopping) { + mutex_unlock(&acio->pcie_tunnel_lock); + return -ESHUTDOWN; + } WRITE_ONCE(acio->pcie_tunnel_requested, true); /* No PCI removal or rescan until system resume has finished. */ mod_delayed_work(system_freezable_wq, &acio->pcie_tunnel_work, 0); @@ -1058,6 +1066,26 @@ static int apple_nhi_pci_tunnel_post_activate(struct tb_nhi *nhi) return 0; } +static void apple_cio_stop_pcie_tunnel(struct apple_cio *acio) +{ + int ret; + + /* Block new requests before draining work, without nesting tb->lock. */ + mutex_lock(&acio->pcie_tunnel_lock); + acio->pcie_tunnel_stopping = true; + WRITE_ONCE(acio->pcie_tunnel_requested, false); + mutex_unlock(&acio->pcie_tunnel_lock); + cancel_delayed_work_sync(&acio->pcie_tunnel_work); + + /* The NHI must remain alive to complete the tunneled reset handshake. */ + mutex_lock(&acio->pcie_tunnel_lock); + ret = apple_cio_quiesce_pcie_tunnel_locked(acio); + mutex_unlock(&acio->pcie_tunnel_lock); + if (ret) + dev_warn(acio->dev, + "failed to quiesce PCIe-C before NHI shutdown: %d\n", ret); +} + #define APPLE_DP_AUX_POLL_MS 500 #define APPLE_DP_AUX_POLL_MAX 24 @@ -1673,9 +1701,14 @@ static int apple_nhi_probe(struct platform_device *pdev) goto err; } + scoped_guard(mutex, &acio->pcie_tunnel_lock) { + acio->pcie_tunnel_stopping = false; + acio->pcie_pm_prepared = false; + } ret = tb_domain_add(anhi->tb, false); if (ret) { dev_err_probe(anhi->dev, ret, "failed to add TB domain\n"); + apple_cio_stop_pcie_tunnel(acio); tb_domain_put(anhi->tb); wait_for_completion(&anhi->nhi.domain_released); goto err; @@ -1728,6 +1761,8 @@ static int apple_nhi_probe(struct platform_device *pdev) err_unlock_tb_domain: mutex_unlock(&anhi->tb->lock); + disable_delayed_work_sync(&anhi->dp_aux_work); + apple_cio_stop_pcie_tunnel(acio); tb_domain_remove(anhi->tb); wait_for_completion(&anhi->nhi.domain_released); err: @@ -1742,6 +1777,7 @@ static void apple_nhi_remove(struct platform_device *pdev) /* Stop racing tunnel activation from rearming work during removal. */ disable_delayed_work_sync(&anhi->dp_aux_work); + apple_cio_stop_pcie_tunnel(anhi->acio); WRITE_ONCE(anhi->acio->nhi_pdev, NULL); tb_domain_remove(anhi->tb); wait_for_completion(&anhi->nhi.domain_released); @@ -1806,7 +1842,8 @@ static void apple_nhi_complete(struct device *dev) guard(mutex)(&acio->pcie_tunnel_lock); acio->pcie_pm_prepared = false; - if (acio->pcie_quiesce_pending || READ_ONCE(acio->pcie_tunnel_requested)) + if (!acio->pcie_tunnel_stopping && + (acio->pcie_quiesce_pending || READ_ONCE(acio->pcie_tunnel_requested))) mod_delayed_work(system_freezable_wq, &acio->pcie_tunnel_work, 0); } @@ -1863,45 +1900,18 @@ static struct platform_device *apple_cio_find_pcie_tunnel(struct apple_cio *acio return pdev; } -/** - * apple_cio_stop - Stop and power down the ACIO complex - * @acio: ACIO complex to stop - * - * Remove the NHI and DART children before shutting down the co-processor and - * dropping the power-domain links. The Type-C PHY must remain configured - * until this function returns. The caller must hold @acio->lock. - */ -static void apple_cio_stop(struct apple_cio *acio) +static void apple_cio_remove_children(struct apple_cio *acio) { - struct platform_device *nhi_pdev, *pcie_pdev; - int ret, i; + struct platform_device *nhi_pdev; lockdep_assert_held(&acio->lock); /* - * First, shutdown the blocks inside the ACIO complex, like the NHI and the IOMMU. - * After we shut down the ACIO co-processor we will no longer be able to access - * the MMIO space of these so make sure nothing tries to do just that. + * Stop requests before removing the NHI. Do not hold pcie_tunnel_lock + * across domain removal: it takes tb->lock, which tunnel callbacks hold + * when they acquire pcie_tunnel_lock. */ - WRITE_ONCE(acio->pcie_tunnel_requested, false); - cancel_delayed_work_sync(&acio->pcie_tunnel_work); - mutex_lock(&acio->pcie_tunnel_lock); - - /* - * The PCIe-C tunnel reset handshake is carried by the NHI. Quiesce the - * remote PCI hierarchy and acknowledge tunneled PERST before removing - * that control plane. PCIe-C and its DART stay bound until the general - * child depopulation below. - */ - pcie_pdev = apple_cio_find_pcie_tunnel(acio); - if (pcie_pdev) { - ret = apple_pcie_tunnel_quiesce(&pcie_pdev->dev); - if (ret) - dev_warn(acio->dev, - "failed to quiesce PCIe-C before NHI shutdown: %d\n", - ret); - put_device(&pcie_pdev->dev); - } + apple_cio_stop_pcie_tunnel(acio); /* * The NHI is the tunnel control plane. Stop it while the tunneled PCIe @@ -1919,9 +1929,26 @@ static void apple_cio_stop(struct apple_cio *acio) flush_workqueue(acio->dp_wq); of_platform_depopulate(acio->dev); - acio->pcie_tunnel_populated = false; - acio->pcie_quiesce_pending = false; - mutex_unlock(&acio->pcie_tunnel_lock); + scoped_guard(mutex, &acio->pcie_tunnel_lock) { + acio->pcie_tunnel_populated = false; + acio->pcie_quiesce_pending = false; + } +} + +/** + * apple_cio_stop - Stop and power down the ACIO complex + * @acio: ACIO complex to stop + * + * Remove the NHI and DART children before shutting down the co-processor and + * dropping the power-domain links. The Type-C PHY must remain configured + * until this function returns. The caller must hold @acio->lock. + */ +static void apple_cio_stop(struct apple_cio *acio) +{ + int ret, i; + + lockdep_assert_held(&acio->lock); + apple_cio_remove_children(acio); /* Try to shut down and power off the co-processor gracefully */ ret = apple_rtkit_poweroff(acio->rtk); @@ -2065,7 +2092,7 @@ static int apple_cio_start(struct apple_cio *acio) return 0; err_depopulate: - of_platform_depopulate(acio->dev); + apple_cio_remove_children(acio); err_shutdown_rtkit: /* Ignore errors here since we're about to cut power to the entire block anyway */ apple_rtkit_poweroff(acio->rtk); From b2e99b4feae94c22fd8637c0df086f84c8f50dff Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:08:44 -0700 Subject: [PATCH 0980/2680] usb: typec: tipd: preserve retry after USB4 switch failure USB4 and Thunderbolt switch activation can fail while booting ACIO or registering its children. Ignoring that return value caches the requested mode as successful, so later status updates skip the switch callback and cannot retry activation. Pass activation failures through state rollback and invalidate the mode cache. The PHY may already have changed before switch activation fails, so a later update must reconfigure even when it requests the old cached mode. Mark the cache valid only after successful configuration, including an explicit safe-mode transition. This does not add retries without a new status update. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/usb/typec/tipd/core.c | 25 +++++++++++++++---------- drivers/usb/typec/tipd/tps6598x.h | 1 + 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 82d8d508ce2c1..c68866c9655bb 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -553,7 +553,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, int ret = 0; if (!(st->data_status & TPS_DATA_STATUS_DATA_CONNECTION)) { - if (cd321x->state.mode == TYPEC_STATE_SAFE) + if (cd321x->state_valid && cd321x->state.mode == TYPEC_STATE_SAFE) return 0; cd321x->state.alt = NULL; cd321x->state.mode = TYPEC_STATE_SAFE; @@ -563,7 +563,8 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, } else if (st->data_status & CD321X_DATA_STATUS_USB4_CONNECTION) { struct enter_usb_data eusb_data; - if (cd321x->state.alt == NULL && cd321x->state.mode == TYPEC_MODE_USB4) + if (cd321x->state_valid && !cd321x->state.alt && + cd321x->state.mode == TYPEC_MODE_USB4) return 0; eusb_data.eudo = le32_to_cpu(st->usb4_status.eudo); @@ -582,11 +583,11 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, tbt_switch_data.orientation = TPS_STATUS_TO_UPSIDE_DOWN(st->status) ? TYPEC_ORIENTATION_REVERSE : TYPEC_ORIENTATION_NORMAL; - typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); + ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); } else if (st->data_status & TPS_DATA_STATUS_TBT_CONNECTION) { struct typec_thunderbolt_data tbt_data; - if (cd321x->state.alt == cd321x->port_altmode_tbt && + if (cd321x->state_valid && cd321x->state.alt == cd321x->port_altmode_tbt && cd321x->state.mode == TYPEC_TBT_MODE) return 0; @@ -615,7 +616,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, tbt_switch_data.orientation = TPS_STATUS_TO_UPSIDE_DOWN(st->status) ? TYPEC_ORIENTATION_REVERSE : TYPEC_ORIENTATION_NORMAL; - typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); + ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); } else if (st->data_status & TPS_DATA_STATUS_DP_CONNECTION) { struct typec_displayport_data dp_data; unsigned long mode; @@ -651,7 +652,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, else dp_data.status &= ~DP_STATUS_HPD_STATE; - if (cd321x->state.alt == cd321x->port_altmode_dp && + if (cd321x->state_valid && cd321x->state.alt == cd321x->port_altmode_dp && cd321x->state.mode == mode && cd321x->dp_status == dp_data.status && cd321x->dp_conf == dp_data.conf) @@ -667,7 +668,8 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, cd321x->dp_conf = dp_data.conf; } } else { - if (cd321x->state.alt == NULL && cd321x->state.mode == TYPEC_STATE_USB) + if (cd321x->state_valid && !cd321x->state.alt && + cd321x->state.mode == TYPEC_STATE_USB) return 0; cd321x->state.alt = NULL; cd321x->state.mode = TYPEC_STATE_USB; @@ -679,9 +681,11 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, out: /* Clear data since it's no longer used after typec_mux_set and points to the stack */ cd321x->state.data = NULL; + cd321x->state_valid = !ret; if (ret) { + /* The mux may have changed before the switch failed. */ cd321x->state = old_state; - dev_warn(tps->dev, "failed to configure Type-C mux: %d\n", ret); + dev_warn(tps->dev, "failed to configure Type-C connection: %d\n", ret); } else if (cd321x->state.alt != cd321x->port_altmode_dp) { cd321x->dp_status = 0; cd321x->dp_conf = 0; @@ -755,7 +759,7 @@ static void cd321x_update_work(struct work_struct *work) * PCIe-C cannot quiesce its hierarchy before NHI removal. */ if ((!new_connected || was_disconnected) && - cd321x->state.mode != TYPEC_STATE_SAFE) + (!cd321x->state_valid || cd321x->state.mode != TYPEC_STATE_SAFE)) typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); @@ -794,6 +798,7 @@ static void cd321x_update_work(struct work_struct *work) cd321x->state.mode = TYPEC_STATE_SAFE; cd321x->state.data = NULL; ret = typec_set_mode(tps->port, TYPEC_STATE_SAFE); + cd321x->state_valid = !ret; if (ret) { dev_warn(tps->dev, "failed to place Type-C mux in safe mode: %d\n", @@ -1371,7 +1376,7 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) cd321x->state.alt = NULL; cd321x->state.mode = TYPEC_STATE_SAFE; cd321x->state.data = NULL; - typec_set_mode(tps->port, TYPEC_STATE_SAFE); + cd321x->state_valid = !typec_set_mode(tps->port, TYPEC_STATE_SAFE); return 0; diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index a74b72e8329f7..0bb92e907fbbd 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -383,6 +383,7 @@ struct cd321x { struct typec_mux *mux; struct typec_mux_state state; + bool state_valid; u32 dp_status; u32 dp_conf; struct typec_thunderbolt_switch *tbt_switch; From 71767532236216c578f0e0c430522cf361cd67d1 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:11:48 -0700 Subject: [PATCH 0981/2680] phy: apple: atc: propagate pixel clock command failures The AUSPLL command helper reports an acknowledgment timeout but returns success. Its DP caller cannot act on the failure, and the M1 tunnel clock path ignores both command results before caching the requested rate. Return the polling error and use the tunnel stop path when either command fails. Leave the rate uncached so a subsequent activation can retry. The successful register sequence is unchanged. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/phy/apple/atc.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 6a3ab34b02775..c808ebe020446 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1786,7 +1786,7 @@ static int atcphy_auspll_apb_command(struct apple_atcphy *atcphy, u32 command) core_clear32(atcphy, AUSPLL_APB_CMD_OVERRIDE, AUSPLL_APB_CMD_OVERRIDE_REQ); - return 0; + return ret; } static int atcphy_dp_configure(struct apple_atcphy *atcphy, enum atcphy_dp_link_rate lr) @@ -2232,17 +2232,24 @@ static int atc_tunnel_start_t8103(struct apple_atcphy *atcphy, u8 rate) core_set32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_PCLK_DRVR_EN); core_set32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_PCLK2_DRVR_EN); core_set32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_REFBUFCLK_DRVR_EN); - atcphy_auspll_apb_command(atcphy, 0); + ret = atcphy_auspll_apb_command(atcphy, 0); + if (ret) + goto err_stop; ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT, value, value & ACIOPHY_AUSPLL_LOCK, 10, 10000); if (ret) { dev_err(atcphy->dev, "DP tunnel clock: AUSPLL did not lock\n"); - atc_tunnel_stop_t8103(atcphy); - return ret; + goto err_stop; } - atcphy_auspll_apb_command(atcphy, 0x2000); + ret = atcphy_auspll_apb_command(atcphy, 0x2000); + if (ret) + goto err_stop; atcphy->tunnel_rate = rate; return 0; + +err_stop: + atc_tunnel_stop_t8103(atcphy); + return ret; } struct atc_tunnel_saved_reg { From f8e27ca50010c09edf7edb1e398ec584e4432629 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:11:22 -0700 Subject: [PATCH 0982/2680] thunderbolt: describe register handling and ownership directly Keep comments focused on register fields, bounds and Linux driver behavior. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/thunderbolt/apple-dpin-handshake.h | 68 ++-------------------- drivers/thunderbolt/apple.c | 21 +++---- drivers/thunderbolt/tb_regs.h | 5 +- drivers/thunderbolt/tunnel.c | 19 ++---- 4 files changed, 20 insertions(+), 93 deletions(-) diff --git a/drivers/thunderbolt/apple-dpin-handshake.h b/drivers/thunderbolt/apple-dpin-handshake.h index 4e492b2cc5802..feaab365b6464 100644 --- a/drivers/thunderbolt/apple-dpin-handshake.h +++ b/drivers/thunderbolt/apple-dpin-handshake.h @@ -10,54 +10,13 @@ #define APPLE_DPIN_HPD_LEVEL (1U << 2) #define APPLE_DPIN_INACTIVE 1U -/* - * Set unconditionally by native AppleCIODPTX::bringConnectionUp on both the - * HPD and CONTROL registers of this exact DPIN0 block when bringing a - * connection up (single-lane-group case, which is what this port is). The - * HPD-register write is additionally gated there on a single/multi-stream - * check; every topology this driver drives through DPIN0 is a single - * external monitor (SST), so it is unconditional here too. Neither write - * clears any other bit (native masks both with 0, a pure OR). Not written - * on deactivate: native's teardown path was not traced, so this preserves - * the existing "do not invent resets" behavior there. - */ +/* Activation sets CONNECTED without clearing unrelated HPD or CONTROL bits. */ #define APPLE_DPIN_CONNECTED (1U << 1) /* - * Two further offsets native AppleCIODPTX::bringConnectionUp writes on - * this exact DPIN0 block, in between the HPD and CONTROL writes above - * (native order: HPD, then MODE_B, then MODE_A, then CONTROL). Unlike - * CONNECTED, the exact values here are NOT confirmed from the native - * binary: native computes them from a per-connection attributes value - * (rate class and an unidentified secondary field) that is copied - * verbatim from this connection's negotiation and is not itself built - * anywhere in the traced kernel/kext code -- its origin could not be - * pinned down statically (most likely the separate DCP coprocessor - * firmware). The value used is therefore an informed, explicitly-labeled - * estimate, not a confirmed constant, and is passed in by the caller - * (mode_value below) rather than fixed at compile time, so the bounded - * formula's one free parameter can be swept across a live connection - * without reinstalling or rebooting between values: - * - bits 4-7 of the field select a rate class using the same RBR=0/ - * HBR=1/HBR2=2/HBR3=3 ordinal already used elsewhere in this driver - * (drivers/thunderbolt/tb_regs.h DP_COMMON_CAP_RATE_*); this link - * negotiates HBR2, so 2 -- high confidence, unchanged across the - * whole sweep. - * - a secondary bit, native-gated on lane_count>=2 (true here: 4) and - * on the same "which DPIN0 sub-instance" selector already confirmed - * unconditional-0 for this single, non-split port, is set from a - * nearby field that other native code also treats as a small, - * 3-valid-value enumeration (0, 1, or 2) -- this is the weak half of - * the formula and the only free parameter, swept across - * mode_value = rate_class(2) * lane_count(4) + secondary_bit = 8, 9 - * (both already tested via separate reboots, each a clean boot with - * no picture), or 10. - * - * mode_value is bounded to APPLE_DPIN_MODE_VALUE_MAX: this caps both the - * single bit MODE_A can set (1 << mode_value) and the field width MODE_B - * ORs in, which is also exactly the range the deactivate path below - * knows how to clear back to a clean baseline. Raising this bound is a - * new, reviewable change, not a runtime knob. + * The caller supplies the experimental DPIN mode value. Bound it to the + * four-bit MODE_B field and the range of one-hot MODE_A bits that teardown + * clears, so changing the parameter cannot modify unrelated register bits. */ #define APPLE_DPIN_MODE_A 0x14 #define APPLE_DPIN_MODE_B 0x1c @@ -108,16 +67,7 @@ static inline int apple_dpin_handshake(const struct apple_dpin_io *io, } else { unsigned int mode_a, mode_b; - /* - * Not native teardown behavior (never traced; see the - * CONNECTED comment above) -- this exists solely so a later - * activate on this same boot starts from clean state and is - * a valid isolated test of a different mode_value. Bounded - * to exactly the bits any in-range mode_value write above - * could have set: MODE_B's OR'd field (bits 7..7+MAX's - * width) and MODE_A's cleared-low-byte-plus-one-set-bit - * (bits 0..MAX). Never touches bits outside that range. - */ + /* Clear only the fields that an in-range mode value can modify. */ mode_b = io->read(io->ctx, APPLE_DPIN_MODE_B); if (mode_b != ~0U) io->write(io->ctx, APPLE_DPIN_MODE_B, @@ -151,13 +101,7 @@ static inline int apple_dpin_handshake(const struct apple_dpin_io *io, if (ret) break; } - /* - * Restore only the bits we own (INACTIVE, and CONNECTED now that we - * set it above); do not invent resets on a failed handshake. HPD is - * left as-is here: native's teardown path for it was not traced, so - * no rollback is invented for a register this driver otherwise never - * wrote before this change. - */ + /* Restore the owned CONTROL bits after a failed handshake. */ ack = io->read(io->ctx, APPLE_DPIN_CONTROL); if (ack != ~0U) io->write(io->ctx, APPLE_DPIN_CONTROL, diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index cfaf030b4cb8b..bc855122af35f 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -862,8 +862,8 @@ static void apple_nhi_ring_configure(struct tb_ring *ring, u32 flags, u32 e2e_fl options += APPLE_CIO_NHI_TXRING_DESC_BASE; /* - * All TX rings share what macOS calls a shared buffer with 232 entries. This is how - * macOS splits it up, ring 0 only carries control packets and gets the minimum. + * Partition the 232 shared TX entries: two for control ring 0, + * forty each for rings 1-5, and five each for the remaining rings. */ if (ring->hop == 0) writel(2, options + 4); @@ -882,8 +882,8 @@ static void apple_nhi_ring_configure(struct tb_ring *ring, u32 flags, u32 e2e_fl } /* - * The firmware samples the ring configuration when the valid bit is set and E2E flow - * control never engages when configured afterwards. Write everything at once like macOS. + * The firmware samples E2E flow control when the valid bit is set. + * Program both in the same write. */ writel(flags | e2e_flags, options); } @@ -1399,9 +1399,8 @@ static void apple_dp_aux_work(struct work_struct *work) /* * Which DP IN adapter (0/1) on the host router this port is -- same - * counting order as apple_dp_in_analog_base() above, and as - * apple_nhi_dp_tunnel_changed()'s port-to-dpin mapping in - * aurora-silicon/linux#8. + * counting order as apple_dp_in_analog_base() above and the port-to-dpin + * mapping in apple_nhi_dp_tunnel_changed(). */ static int apple_dpin_index_for_port(struct apple_nhi *anhi, struct tb_port *in) { @@ -1454,13 +1453,7 @@ static int apple_nhi_dp_tunnel_post_activate(struct tb_nhi *nhi, apple_dp_dump_rc(anhi->acio); apple_dp_dump_vse(in->sw); - /* - * Route a real display pipeline to this tunnel, ported from - * aurora-silicon/linux#8 -- the dpin_aux "analog AUX serializer" - * mechanism previously here is confirmed ineffective on this - * hardware, including after everything else in the tunnel-up path - * was independently fixed. - */ + /* Route the display pipeline through the selected host DP IN adapter. */ idx = apple_dpin_index_for_port(anhi, in); if (idx < 0 || idx > 1 || !anhi->acio) { dev_warn(anhi->dev, diff --git a/drivers/thunderbolt/tb_regs.h b/drivers/thunderbolt/tb_regs.h index 84f22d937453e..54057a331a596 100644 --- a/drivers/thunderbolt/tb_regs.h +++ b/drivers/thunderbolt/tb_regs.h @@ -430,14 +430,13 @@ struct tb_regs_port_header { #define ADP_DP_CS_3_HPDC BIT(9) /* * Apple silicon host DP IN adapter: pulsing this tells the adapter to - * propagate HPD to ADP_DP_CS_2_HPD. Ported from aurora-silicon/linux#8 - * (t8103, hardware-tested). + * propagate HPD to ADP_DP_CS_2_HPD. */ #define ADP_DP_CS_3_HPD_PROPAGATE BIT(10) /* * Apple silicon host: hold the DP OUT (hub-side) adapter's own link * training off while a tunnel is up -- the host's DPTX trains the sink - * through the tunnel itself. Same source as HPD_PROPAGATE above. + * through the tunnel itself. */ #define ADP_DP_CS_3_NO_AUTO_LT BIT(8) #define DP_LOCAL_CAP 0x04 diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index 9c35b236b4a8b..340134d8990e9 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -1282,16 +1282,9 @@ static void tb_dp_dprx_stop(struct tb_tunnel *tunnel) } /* - * Apple silicon host DP IN adapter: this adapter has no real physical DP - * connector wired to it, so nothing ever sets its own ADP_DP_CS_2_HPD -- - * pulse ADP_DP_CS_3_HPD_PROPAGATE to tell it to propagate HPD itself, and - * wait for it to take, same as aurora-silicon/linux#8's hardware-tested - * t8103 implementation (three docks, multiple monitors). On the - * dcpext1/right-port path, the CS0-CS13 registers this pulse is meant to - * move have been observed to stay completely static across every - * failure, unlike the working dcpext0/left-port case. Logs and continues - * either way; a failed pulse is not treated as a fatal tunnel-activation - * error, matching the reference. + * The host DP IN adapter has no physical DP connector. Pulse HPD propagation + * after enabling the tunnel and allow time for the adapter to report HPD. + * Failure is diagnostic: leave the tunnel available for display recovery. */ static void tb_dp_apple_pulse_hpd(struct tb_port *in) { @@ -1399,8 +1392,7 @@ static int tb_dp_activate(struct tb_tunnel *tunnel, bool active) * Apple silicon host: the sink link is trained by the * host's DPTX through the tunnel; keep this DP OUT adapter * from starting link training on its own, and hand it back - * once the tunnel is gone. Ported from aurora-silicon/ - * linux#8. + * once the tunnel is gone. */ if (active && apple_dpin && !tb_port_read(out, &v, TB_CFG_PORT, out->cap_adap + ADP_DP_CS_3, 1)) { @@ -1747,8 +1739,7 @@ static int tb_dp_init_video_credits(struct tb_path_hop *hop) struct tb_switch *sw = port->sw; /* - * Apple silicon host DP IN adapters take 5 NFC credits for the - * video path -- same source as the HPD/NO_AUTO_LT handling above. + * Apple silicon host DP IN adapters require five NFC video credits. */ if (tb_port_needs_host_dp_credits(port)) { hop->nfc_credits = 5; From 5e80803a931cbff7fda51b92d12526b67d40f6b8 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:13:07 -0700 Subject: [PATCH 0983/2680] phy: apple: describe tunnel clock and crossbar state directly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document the accepted initial gate state and the clock and reset operations performed by these helpers. Remove unrelated implementation narratives. [ryan: adapted to 7.2.10 — Asahi now waits for the T8122 RCAL codes only when the PIPE goes to USB3; T8140/T8142 (woken in every mode) keep waiting in every mode as on 7.1] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/mux/apple-display-crossbar.c | 2 +- drivers/phy/apple/atc.c | 20 +++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/drivers/mux/apple-display-crossbar.c b/drivers/mux/apple-display-crossbar.c index 31788a927ea74..e7a8470aa7c79 100644 --- a/drivers/mux/apple-display-crossbar.c +++ b/drivers/mux/apple-display-crossbar.c @@ -261,7 +261,7 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) dpxbar_set32(dpxbar, T602X_FIFO_WR_N_CLK_EN, prev_dispext_bit); dpxbar_set32(dpxbar, T602X_REG_014, 0x4); - /* Native T602x DPIN0 teardown restores the read reset at +0x24. */ + /* Restore the selected input's FIFO read reset. */ dpxbar_set32(dpxbar, index == MUX_DPIN0 ? T602X_FIFO_RD_PCLK2_EN : FIFO_RD_PCLK1_EN, atc_bit); diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index c808ebe020446..f7499182a71a6 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2365,11 +2365,9 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy, u8 rate) /* * Before the first rate request, TX_DP_CTRL0 gate bits read 0xe001: * the enable gates are set, the byte-clock reset is asserted, and the - * selector and reset-release bits are clear. Native configureDPTunnelMode - * tracks tunnel clients in software rather than through these gates, so - * this exact state is the only additional gate configuration accepted - * here, together with no enabled PLL output, no outstanding command, - * and no lock. Any other gate state is refused. + * selector and reset-release bits are clear. Accept this initial gate + * state only with no enabled PLL output, no outstanding command and + * no lock. Any other active gate state is refused. */ if (outputs & 0x54 || command & AUSPLL_APB_CMD_OVERRIDE_REQ || status & ACIOPHY_AUSPLL_LOCK) @@ -2387,7 +2385,7 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy, u8 rate) atcphy->tunnel_saved_regs[i] = readl(atcphy->regs.core + atc_tunnel_regs[i].reg); atcphy->tunnel_saved = true; - /* Native configureDPTunnelMode, first clock client; no lane mux writes. */ + /* Wake the common clock blocks without changing the lane mux. */ core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL); core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL_OV); udelay(2); @@ -2424,7 +2422,7 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy, u8 rate) core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_SELECT, FIELD_PREP(DPTX_PCLK1_SELECT, selector)); - /* Native fixed descriptor at cache VA 0xfffffe0007582418. */ + /* Program the fixed tunnel clock descriptor. */ core_clear32(atcphy, AUSPLL_FREQ_CFG, AUSPLL_FREQ_REFCLK); core_mask32(atcphy, AUSPLL_FREQ_DESC_A, 0xffffffff, 0x21c | (8 << 14) | (3 << 18) | (8 << 22) | (7 << 26)); @@ -2492,12 +2490,12 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) /* Take the USB3 PHY out of reset */ core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_PHY_RESET_N); /* - * NOTE: Not what macos does, it takes the phy out of reset right before writing to ACIOPHY_CFG0_COMMON_SMALL - * does lane configuration with phy out of reset, and the poll is right after AUS_COMMON_SHIM_BLK_BIAS_REG - * write. Does not seem to matter, but if something does not work, consider re-ordering things to macos order. + * T8122 only calibrates for the USB3 PIPE; the T8140/T8142 PHYs, woken in + * every mode, wait for the calibration in every mode as on 7.1. */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8122 && - atcphy_modes[mode].pipehandler_state == ATCPHY_PIPEHANDLER_STATE_USB3) { + (atcphy_modes[mode].pipehandler_state == ATCPHY_PIPEHANDLER_STATE_USB3 || + atcphy->hw->configure_post_tunable == atcphy_configure_post_tunable_all_modes)) { ret = readl_poll_timeout(atcphy->regs.core + AUS_COMMON_DIG_RCAL1, reg, (reg & AUS_COMMON_DIG_RCAL1_ALL_CODES_DONE), 10, 100000); if (ret) { From b155e1205eb964df0f37911dfe84e706c492faa7 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:15:05 -0700 Subject: [PATCH 0984/2680] phy: apple: atc: invalidate the cached rate before reprogramming Once PLL programming starts, an acknowledgment or lock timeout leaves the old cached DP rate stale. A later request for that old rate then returns success without restoring its hardware configuration. Invalidate the rate after the command-ready check and before the first register change. Only a completed configuration may cache the new rate. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/phy/apple/atc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index f7499182a71a6..c2b82cc7ee57c 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1813,6 +1813,8 @@ static int atcphy_dp_configure(struct apple_atcphy *atcphy, enum atcphy_dp_link_ return ret; } + /* The old rate no longer describes hardware once reprogramming starts. */ + atcphy->dp_link_rate = -1; core_clear32(atcphy, AUSPLL_FREQ_CFG, AUSPLL_FREQ_REFCLK); core_mask32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_FREQ_COUNT_TARGET, From 6abdb3937f46e67d6c9b83b2d89c48379cbefce0 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:16:07 -0700 Subject: [PATCH 0985/2680] thunderbolt: apple: report rejected live cable transitions When a live cable transition requires shutting down ACIO, return an error instead of reporting the requested mode as active. This lets the Type-C caller invalidate its mode cache and retry a later update. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/thunderbolt/apple.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index bc855122af35f..c7dccf9f2db4b 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -2165,17 +2165,16 @@ static int apple_cio_tbt_switch_set(struct typec_thunderbolt_switch_dev *sw, return 0; /* - * Transitions between different cables without a shutdown inbetween are invalid and can - * only happen when there's a bug inside the Type-C PD driver. If we tried such a - * transition, ACIO would crash and then trigger some watchdog that would reset the entire - * SoC a few seconds later. Shutting down instead only makes the connected device not work - * but we should be able to recover once the next cable is plugged in. + * Changing live cable parameters requires an ACIO shutdown. Report the + * rejected transition so the caller does not cache the new mode as active. */ if (acio->current_cable_info && acio->target_cable_info) { dev_err(acio->dev, "Invalid cable transition from 0x%x to 0x%x, shutting down instead\n", acio->current_cable_info, acio->target_cable_info); acio->target_cable_info = 0; + apple_cio_stop(acio); + return -EINVAL; } /* From 5ec0600cdca47a8a152d2d72b27efae9a8d18f04 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 20:35:13 -0700 Subject: [PATCH 0986/2680] thunderbolt: restore M1 tunneled PCIe state and recover late link loss MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kensington USB devices can disappear after s2idle even when the initial port restart and cable health check succeed. A later PCIe link failure must trigger another fresh validation instead of waiting for a cable interrupt or manual replug. Restore the resettable M1 tunneled host and DART state before restarting the downstream link. Gate DART commands and fault handling around clock shutdown, preserve stream selection, and describe the reset and register initialization properties in the binding. Gate Type-C work across initialization, sleep and removal, read fresh cable metadata after resume, and preserve an unchanged healthy session. Propagate late link failures through an atomic switch notification into bounded process-context revalidation. Keep failure history sticky while allowing later notifications from a retained host. Failed reads or health checks request a bounded reconnect; successful checks preserve routing. Keep failed display-route acquisition retryable. Add state-machine and DART regression coverage. The native ARM64 build and 88 KUnit cases pass. Two M1/j293 Kensington USB-only suspend cycles exercised late link-loss recovery and restored usable input without a manual replug. The initiating xHCI resume fault remains unresolved. [ryan: adapted to 7.2.10 — split by topic: the PCIe binding hunks are carried by 080-radio; the device-tree hunks are carried by 000-devicetree; the drm/apple hunks are carried by 200-dcp; the apple-dart hunks are carried by 020-dart; the PCIe controller hunks are carried by 080-radio] [ryan: adapted to 7.2.10 — Asahi documents these functions with kerneldoc; our new code goes above those comments] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/thunderbolt/apple.c | 121 ++++++- drivers/usb/typec/mux.c | 50 +++ drivers/usb/typec/mux.h | 2 + drivers/usb/typec/tipd/Kconfig | 10 + drivers/usb/typec/tipd/Makefile | 2 + drivers/usb/typec/tipd/cd321x-pm-test.c | 399 ++++++++++++++++++++++++ drivers/usb/typec/tipd/cd321x-pm.h | 143 +++++++++ drivers/usb/typec/tipd/core.c | 301 +++++++++++++----- drivers/usb/typec/tipd/i2c.c | 2 + drivers/usb/typec/tipd/spmi.c | 2 + drivers/usb/typec/tipd/tps6598x.h | 16 +- include/linux/usb/typec_mux.h | 27 ++ 12 files changed, 994 insertions(+), 81 deletions(-) create mode 100644 drivers/usb/typec/tipd/cd321x-pm-test.c create mode 100644 drivers/usb/typec/tipd/cd321x-pm.h diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index c7dccf9f2db4b..ccec6f80409a3 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -90,6 +90,7 @@ #include #include #include +#include #include #include #include @@ -186,6 +187,7 @@ struct apple_cio { struct device_node *np; struct device_node *pcie_tunnel_np; bool pcie_tunnel_preinitialized; + struct reset_control *pcie_reset; void __iomem *pcie_intr2axi_base; struct apple_rtkit *rtk; @@ -211,6 +213,7 @@ struct apple_cio { struct platform_device *nhi_pdev; struct typec_thunderbolt_switch_dev *tbt_switch; + struct notifier_block pcie_notifier; /* Serializes PCIe-C population with cable teardown. */ struct mutex pcie_tunnel_lock; struct delayed_work pcie_tunnel_work; @@ -218,6 +221,7 @@ struct apple_cio { bool pcie_tunnel_populated; bool pcie_tunnel_stopping; bool pcie_pm_prepared; + bool pcie_resume_expected; bool pcie_quiesce_pending; /* Type-C connector this router is wired to, for DP tunnel routing */ @@ -543,6 +547,44 @@ apple_cio_pcie_tunnel_is_preinitialized(struct device_node *parent) return false; } +static void apple_cio_reset_control_put(void *data) +{ + reset_control_put(data); +} + +/* + * PCIe-C's reset covers the controller and the DART beside it. It is + * optional, since older device trees do not describe it, and only used where + * the kernel initializes the controller: resetting it would discard a boot + * loader's setup. + */ +static int apple_cio_get_pcie_reset(struct apple_cio *acio) +{ + struct reset_control *rst = NULL; + + for_each_available_child_of_node_scoped(acio->pcie_tunnel_np, child) { + if (!of_device_is_compatible(child, "apple,t8103-pciec") && + !of_device_is_compatible(child, "apple,t6000-pciec")) + continue; + if (!of_property_read_bool(child, "apple,pciec-kernel-init")) + break; + + rst = of_reset_control_get_optional_exclusive(child, NULL); + break; + } + if (IS_ERR(rst)) + return dev_err_probe(acio->dev, PTR_ERR(rst), + "Unable to get PCIe-C reset\n"); + if (!rst) + return 0; + + /* The bound PCIe host owns the line between ACIO startup attempts. */ + reset_control_release(rst); + acio->pcie_reset = rst; + return devm_add_action_or_reset(acio->dev, apple_cio_reset_control_put, + rst); +} + static void apple_cio_iounmap(void *data) { iounmap((__force void __iomem *)data); @@ -1697,6 +1739,7 @@ static int apple_nhi_probe(struct platform_device *pdev) scoped_guard(mutex, &acio->pcie_tunnel_lock) { acio->pcie_tunnel_stopping = false; acio->pcie_pm_prepared = false; + acio->pcie_resume_expected = false; } ret = tb_domain_add(anhi->tb, false); if (ret) { @@ -1787,6 +1830,8 @@ static int apple_nhi_prepare(struct device *dev) struct apple_cio *acio = anhi->acio; guard(mutex)(&acio->pcie_tunnel_lock); + /* Keep an unresolved expectation across another sleep before revalidation. */ + acio->pcie_resume_expected |= READ_ONCE(acio->pcie_tunnel_requested); acio->pcie_pm_prepared = true; return 0; } @@ -1980,6 +2025,27 @@ static int apple_cio_start(struct apple_cio *acio) lockdep_assert_held(&acio->lock); + /* + * PCIe-C's power domain stays on across cable teardown, so apart from + * this reset nothing returns the controller to a known state between + * tunnels. After system sleep with a Thunderbolt 3 dock, state left in + * it has kept every later link from training until reboot. Reset it + * while its host and DART are unbound and before ACIO's domains come + * up, in the same order as after boot. + */ + if (acio->pcie_reset) { + ret = reset_control_acquire(acio->pcie_reset); + if (ret) + return dev_err_probe(acio->dev, ret, + "PCIe-C reset is still in use\n"); + ret = reset_control_reset(acio->pcie_reset); + reset_control_release(acio->pcie_reset); + if (ret) + return dev_err_probe(acio->dev, ret, + "failed to reset PCIe-C\n"); + dev_info(acio->dev, "PCIe-C reset before ACIO start\n"); + } + /* Create device links to the power domains in order to power them on */ for (i = 0; i < acio->pd_list->num_pds; i++) { link = device_link_add(acio->dev, acio->pd_list->pd_devs[i], @@ -2102,6 +2168,34 @@ static int apple_cio_start(struct apple_cio *acio) return ret; } +static int apple_cio_check_connection(struct apple_cio *acio) +{ + struct platform_device *pdev; + int ret; + + lockdep_assert_held(&acio->lock); + guard(mutex)(&acio->pcie_tunnel_lock); + if (!READ_ONCE(acio->nhi_pdev) || acio->pcie_tunnel_stopping || + apple_rtkit_is_crashed(acio->rtk)) + return -ENODEV; + if (acio->pcie_pm_prepared || acio->pcie_quiesce_pending) + return -EAGAIN; + + /* A display-only or unauthorized session need not have a PCIe tunnel. */ + if (!READ_ONCE(acio->pcie_tunnel_requested)) + return acio->pcie_resume_expected ? -ENOLINK : 0; + if (!acio->pcie_tunnel_populated) + return -EAGAIN; + pdev = apple_cio_find_pcie_tunnel(acio); + if (!pdev) + return -ENODEV; + ret = apple_pcie_tunnel_check_state(&pdev->dev); + put_device(&pdev->dev); + if (!ret) + acio->pcie_resume_expected = false; + return ret; +} + /** * apple_cio_tbt_switch_set - Handle a USB4/Thunderbolt cable transition * @sw: USB4/Thunderbolt switch @@ -2162,7 +2256,7 @@ static int apple_cio_tbt_switch_set(struct typec_thunderbolt_switch_dev *sw, } if (acio->target_cable_info == acio->current_cable_info) - return 0; + return acio->current_cable_info ? apple_cio_check_connection(acio) : 0; /* * Changing live cable parameters requires an ACIO shutdown. Report the @@ -2188,6 +2282,18 @@ static int apple_cio_tbt_switch_set(struct typec_thunderbolt_switch_dev *sw, return 0; } +static int apple_cio_pcie_notify(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct apple_cio *acio = container_of(nb, struct apple_cio, pcie_notifier); + + if (event != APPLE_PCIE_TUNNEL_LINK_DOWN || data != acio->dev) + return NOTIFY_DONE; + + typec_thunderbolt_switch_notify(acio->tbt_switch); + return NOTIFY_OK; +} + static int apple_cio_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -2234,6 +2340,9 @@ static int apple_cio_probe(struct platform_device *pdev) if (ret) return dev_err_probe(dev, ret, "failed to map PCIe-C Intr2AXI registers\n"); + ret = apple_cio_get_pcie_reset(acio); + if (ret) + return ret; } } @@ -2335,11 +2444,20 @@ static int apple_cio_probe(struct platform_device *pdev) .set = apple_cio_tbt_switch_set, .drvdata = acio, }; + /* A consumer cannot start a host before its failure listener is ready. */ + guard(mutex)(&acio->lock); acio->tbt_switch = typec_thunderbolt_switch_register(dev, &desc); if (IS_ERR(acio->tbt_switch)) return dev_err_probe(dev, PTR_ERR(acio->tbt_switch), "Unable to register thunderbolt switch\n"); + acio->pcie_notifier.notifier_call = apple_cio_pcie_notify; + ret = apple_pcie_tunnel_register_notifier(&acio->pcie_notifier); + if (ret) { + typec_thunderbolt_switch_unregister(acio->tbt_switch); + return ret; + } + return 0; } @@ -2347,6 +2465,7 @@ static void apple_cio_remove(struct platform_device *pdev) { struct apple_cio *acio = platform_get_drvdata(pdev); + apple_pcie_tunnel_unregister_notifier(&acio->pcie_notifier); typec_thunderbolt_switch_unregister(acio->tbt_switch); cancel_delayed_work_sync(&acio->pcie_tunnel_work); diff --git a/drivers/usb/typec/mux.c b/drivers/usb/typec/mux.c index 60a120284ea82..2563531127344 100644 --- a/drivers/usb/typec/mux.c +++ b/drivers/usb/typec/mux.c @@ -597,6 +597,55 @@ int typec_thunderbolt_switch_set(struct typec_thunderbolt_switch *sw, } EXPORT_SYMBOL_GPL(typec_thunderbolt_switch_set); +/** + * typec_thunderbolt_switch_register_notifier - Watch provider health changes + * @sw: USB4/Thunderbolt switch held by the consumer + * @nb: Atomic notifier, which must not sleep or call the switch's set method + * + * Notifications request fresh state validation in consumer work context. + * Unregister the notifier before releasing @sw or its callback data. + */ +int typec_thunderbolt_switch_register_notifier(struct typec_thunderbolt_switch *sw, + struct notifier_block *nb) +{ + if (IS_ERR_OR_NULL(sw)) + return 0; + + return atomic_notifier_chain_register(&sw->sw_dev->notifiers, nb); +} +EXPORT_SYMBOL_GPL(typec_thunderbolt_switch_register_notifier); + +/** + * typec_thunderbolt_switch_unregister_notifier - Stop provider notifications + * @sw: USB4/Thunderbolt switch held by the consumer + * @nb: Previously registered notifier + * + * Drains callbacks. Call in process context, outside callback/worker locks, + * before cancelling work queued by the callback. + */ +void typec_thunderbolt_switch_unregister_notifier(struct typec_thunderbolt_switch *sw, + struct notifier_block *nb) +{ + if (!IS_ERR_OR_NULL(sw)) + atomic_notifier_chain_unregister(&sw->sw_dev->notifiers, nb); +} +EXPORT_SYMBOL_GPL(typec_thunderbolt_switch_unregister_notifier); + +/** + * typec_thunderbolt_switch_notify - Request consumer revalidation + * @sw: Registered provider switch + * + * May be called from interrupt context. No connection state is passed: a + * delayed notification must validate the current connection, not reset it + * using details from an earlier session. Stop notifications before unregister. + */ +void typec_thunderbolt_switch_notify(struct typec_thunderbolt_switch_dev *sw) +{ + atomic_notifier_call_chain(&sw->notifiers, + TYPEC_THUNDERBOLT_SWITCH_REVALIDATE, NULL); +} +EXPORT_SYMBOL_GPL(typec_thunderbolt_switch_notify); + static void typec_thunderbolt_switch_release(struct device *dev) { kfree(to_typec_thunderbolt_switch_dev(dev)); @@ -637,6 +686,7 @@ typec_thunderbolt_switch_register(struct device *parent, sw_dev->dev.fwnode = desc->fwnode; sw_dev->dev.class = &typec_mux_class; sw_dev->dev.type = &typec_thunderbolt_switch_dev_type; + ATOMIC_INIT_NOTIFIER_HEAD(&sw_dev->notifiers); sw_dev->dev.driver_data = desc->drvdata; ret = dev_set_name(&sw_dev->dev, "%s-thunderbolt-switch", desc->name ? desc->name : dev_name(parent)); diff --git a/drivers/usb/typec/mux.h b/drivers/usb/typec/mux.h index a2c92cab12114..12e1ac47a11f1 100644 --- a/drivers/usb/typec/mux.h +++ b/drivers/usb/typec/mux.h @@ -3,6 +3,7 @@ #ifndef __USB_TYPEC_MUX__ #define __USB_TYPEC_MUX__ +#include #include struct typec_switch_dev { @@ -29,6 +30,7 @@ extern const struct device_type typec_mux_dev_type; struct typec_thunderbolt_switch_dev { struct device dev; typec_thunderbolt_switch_set_fn_t set; + struct atomic_notifier_head notifiers; }; #define to_typec_thunderbolt_switch_dev(_dev_) \ diff --git a/drivers/usb/typec/tipd/Kconfig b/drivers/usb/typec/tipd/Kconfig index 8177f860f89de..2f73bb17ed352 100644 --- a/drivers/usb/typec/tipd/Kconfig +++ b/drivers/usb/typec/tipd/Kconfig @@ -26,3 +26,13 @@ config TYPEC_SN201202X If you choose to build this driver as a dynamically linked module, the module will be called sn201202x.ko. + +config TYPEC_TIPD_KUNIT_TEST + tristate "KUnit tests for CD321x resume state" if !KUNIT_ALL_TESTS + depends on KUNIT && TYPEC_TPS6598X_CORE + default KUNIT_ALL_TESTS + help + Test suspend gating, fresh cable-state revalidation, bounded retries + and removal during resume. The tests exercise the state transitions + shared with the driver, including aborted and repeated sleep cycles. + These tests run without accessing a physical Type-C controller. diff --git a/drivers/usb/typec/tipd/Makefile b/drivers/usb/typec/tipd/Makefile index 6f5ebc1e1a5fe..c0e850a56f289 100644 --- a/drivers/usb/typec/tipd/Makefile +++ b/drivers/usb/typec/tipd/Makefile @@ -10,3 +10,5 @@ tps6598x-y := i2c.o obj-$(CONFIG_TYPEC_SN201202X) += sn201202x.o sn201202x-y := spmi.o + +obj-$(CONFIG_TYPEC_TIPD_KUNIT_TEST) += cd321x-pm-test.o diff --git a/drivers/usb/typec/tipd/cd321x-pm-test.c b/drivers/usb/typec/tipd/cd321x-pm-test.c new file mode 100644 index 0000000000000..79b6797c73b9e --- /dev/null +++ b/drivers/usb/typec/tipd/cd321x-pm-test.c @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include + +#include "cd321x-pm.h" + +static void cd321x_resume_requires_snapshot_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + cd321x_pm_prepare(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + /* A queued cable update may not apply the pre-sleep snapshot. */ + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_TRUE(test, cd321x_pm_can_update(&pm)); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_RUNNING); +} + +static void cd321x_revalidation_errors_bounded_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int i; + + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + for (i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_EXPECT_EQ(test, cd321x_pm_retry(&pm), + i + 1 < CD321X_RESUME_ATTEMPTS); + } + /* Give up without permitting an update from the old cable snapshot. */ + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); +} + +static void cd321x_activation_failure_rereads_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + /* A PHY or ACIO error must not be treated as successful restoration. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_TRUE(test, cd321x_pm_can_update(&pm)); + KUNIT_EXPECT_EQ(test, pm.attempts_left, 0U); +} + +static void cd321x_abort_and_duplicate_completion_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + /* Prepare followed by completion, with no device suspend, is an abort. */ + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_resume(&pm)); + KUNIT_EXPECT_EQ(test, pm.attempts_left, CD321X_RESUME_ATTEMPTS - 1U); + cd321x_pm_snapshot_ready(&pm); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_TRUE(test, cd321x_pm_can_update(&pm)); +} + +static void cd321x_repeat_suspend_cancels_retry_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm)); + /* The next prepare wins over a delayed retry from the previous wake. */ + cd321x_pm_prepare(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_retry(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_EXPECT_EQ(test, pm.attempts_left, (unsigned int)CD321X_RESUME_ATTEMPTS); +} + +static void cd321x_suspend_before_apply_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + /* The debounce timer has not applied the freshly read state yet. */ + cd321x_pm_prepare(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_retry(&pm)); +} + +static void cd321x_remove_drains_resume_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + cd321x_pm_remove(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_retry(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + cd321x_pm_prepare(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_resume(&pm)); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_REMOVED); +} + +static void cd321x_read_and_apply_share_budget_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int i; + + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + for (i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + if (i & 1) + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_EQ(test, cd321x_pm_retry(&pm), + i + 1 < CD321X_RESUME_ATTEMPTS); + } + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); +} + +static void cd321x_status_event_after_exhaustion_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int i; + + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + for (i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_retry(&pm); + } + /* A power/plug-only interrupt has not read the data or cable registers. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_event(&pm)); + KUNIT_EXPECT_EQ(test, pm.attempts_left, (unsigned int)CD321X_RESUME_ATTEMPTS); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_TRUE(test, cd321x_pm_can_update(&pm)); + cd321x_pm_remove(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_event(&pm)); +} + +static void cd321x_runtime_failure_revalidates_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int i; + + /* A failed ordinary cable update has already consumed its IRQ changes. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + for (i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_EQ(test, cd321x_pm_retry(&pm), + i + 1 < CD321X_RESUME_ATTEMPTS); + } + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + /* A later physical event can retry, but cannot directly apply stale data. */ + KUNIT_EXPECT_TRUE(test, cd321x_pm_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); +} + +static void cd321x_healthy_resume_keeps_connection_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int i; + + /* A successful resume and an aborted suspend both revalidate in place. */ + for (i = 0; i < 2; i++) { + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_FALSE(test, pm.force_reconnect); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_FALSE(test, pm.force_reconnect); + } +} + +static void cd321x_failed_resume_forces_reconnect_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + /* Fresh cable data cannot override a provider's recorded resume failure. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm)); + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); + /* A new suspend must preserve recovery that has not been applied yet. */ + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_FALSE(test, pm.force_reconnect); +} + +static void cd321x_stale_recovery_keeps_reconnect_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int i; + + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm)); + for (i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_retry(&pm); + } + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); + KUNIT_ASSERT_TRUE(test, cd321x_pm_event(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_FALSE(test, pm.force_reconnect); +} + +static void cd321x_late_link_failure_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + cd321x_pm_complete(&pm); + /* The link drops after the final successful resume health check. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_link_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_EXPECT_FALSE(test, pm.force_reconnect); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + /* Only the provider's failed check requests a fresh forced reconnect. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm)); + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_EQ(test, pm.link_events_left, CD321X_LINK_EVENTS - 1U); +} + +static void cd321x_link_event_during_final_apply_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int i; + + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + for (i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + if (i + 1 < CD321X_RESUME_ATTEMPTS) + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm)); + } + cd321x_pm_snapshot_ready(&pm); + KUNIT_ASSERT_EQ(test, pm.attempts_left, 0U); + /* A queued APPLY must not hide an event with its exhausted read budget. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_link_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); +} + +static void cd321x_link_reprobe_storm_bounded_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int event, attempt; + + cd321x_pm_new_connection(&pm); + for (event = 0; event < CD321X_LINK_EVENTS; event++) { + KUNIT_ASSERT_TRUE(test, cd321x_pm_link_event(&pm)); + for (attempt = 0; attempt < CD321X_RESUME_ATTEMPTS; attempt++) { + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_EQ(test, cd321x_pm_retry(&pm), + attempt + 1 < CD321X_RESUME_ATTEMPTS); + } + } + KUNIT_EXPECT_FALSE(test, cd321x_pm_link_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); +} + +static void cd321x_success_does_not_refill_link_budget_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int i; + + cd321x_pm_new_connection(&pm); + for (i = 0; i < CD321X_LINK_EVENTS; i++) { + KUNIT_ASSERT_TRUE(test, cd321x_pm_link_event(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + /* An old host event can reach a now healthy session: keep it. */ + KUNIT_EXPECT_FALSE(test, pm.force_reconnect); + cd321x_pm_complete(&pm); + } + KUNIT_EXPECT_FALSE(test, cd321x_pm_link_event(&pm)); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_RUNNING); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); +} + +static void cd321x_link_event_init_and_remove_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + cd321x_pm_init(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_link_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_ready(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_ready(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_remove(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_link_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_ready(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); +} + +static void cd321x_link_event_suspend_and_rearm_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int i; + + cd321x_pm_new_connection(&pm); + for (i = 0; i < CD321X_LINK_EVENTS; i++) + KUNIT_ASSERT_TRUE(test, cd321x_pm_link_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_link_event(&pm)); + cd321x_pm_prepare(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_link_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_EXPECT_EQ(test, pm.link_events_left, (unsigned int)CD321X_LINK_EVENTS); + KUNIT_ASSERT_TRUE(test, cd321x_pm_link_event(&pm)); + /* An actual attachment change also permits a bounded new sequence. */ + cd321x_pm_new_connection(&pm); + KUNIT_EXPECT_EQ(test, pm.link_events_left, (unsigned int)CD321X_LINK_EVENTS); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); +} + +static struct kunit_case cd321x_pm_cases[] = { + KUNIT_CASE(cd321x_late_link_failure_test), + KUNIT_CASE(cd321x_link_event_during_final_apply_test), + KUNIT_CASE(cd321x_link_reprobe_storm_bounded_test), + KUNIT_CASE(cd321x_success_does_not_refill_link_budget_test), + KUNIT_CASE(cd321x_link_event_init_and_remove_test), + KUNIT_CASE(cd321x_link_event_suspend_and_rearm_test), + KUNIT_CASE(cd321x_healthy_resume_keeps_connection_test), + KUNIT_CASE(cd321x_failed_resume_forces_reconnect_test), + KUNIT_CASE(cd321x_stale_recovery_keeps_reconnect_test), + KUNIT_CASE(cd321x_resume_requires_snapshot_test), + KUNIT_CASE(cd321x_revalidation_errors_bounded_test), + KUNIT_CASE(cd321x_activation_failure_rereads_test), + KUNIT_CASE(cd321x_abort_and_duplicate_completion_test), + KUNIT_CASE(cd321x_repeat_suspend_cancels_retry_test), + KUNIT_CASE(cd321x_suspend_before_apply_test), + KUNIT_CASE(cd321x_remove_drains_resume_test), + KUNIT_CASE(cd321x_read_and_apply_share_budget_test), + KUNIT_CASE(cd321x_status_event_after_exhaustion_test), + KUNIT_CASE(cd321x_runtime_failure_revalidates_test), + {} +}; + +static struct kunit_suite cd321x_pm_suite = { + .name = "cd321x-pm", + .test_cases = cd321x_pm_cases, +}; + +kunit_test_suite(cd321x_pm_suite); +MODULE_LICENSE("GPL"); diff --git a/drivers/usb/typec/tipd/cd321x-pm.h b/drivers/usb/typec/tipd/cd321x-pm.h new file mode 100644 index 0000000000000..da764777ba36e --- /dev/null +++ b/drivers/usb/typec/tipd/cd321x-pm.h @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __CD321X_PM_H__ +#define __CD321X_PM_H__ + +#include + +#define CD321X_RESUME_ATTEMPTS 3 +#define CD321X_LINK_EVENTS 3 + +enum cd321x_pm_phase { + CD321X_PM_RUNNING, + CD321X_PM_PREPARED, + CD321X_PM_REVALIDATE, + CD321X_PM_APPLY, + CD321X_PM_STALE, + CD321X_PM_REMOVED, + CD321X_PM_INITIALIZING, +}; + +/* All transitions and the associated work scheduling hold tps->lock. */ +struct cd321x_pm_state { + enum cd321x_pm_phase phase; + unsigned int attempts_left; + unsigned int link_events_left; + bool force_reconnect; +}; + +static inline void cd321x_pm_new_connection(struct cd321x_pm_state *pm) +{ + pm->link_events_left = CD321X_LINK_EVENTS; +} + +static inline void cd321x_pm_init(struct cd321x_pm_state *pm) +{ + pm->phase = CD321X_PM_INITIALIZING; + cd321x_pm_new_connection(pm); +} + +static inline bool cd321x_pm_ready(struct cd321x_pm_state *pm) +{ + if (pm->phase != CD321X_PM_INITIALIZING) + return false; + pm->phase = CD321X_PM_REVALIDATE; + pm->attempts_left = CD321X_RESUME_ATTEMPTS; + return true; +} + +static inline bool cd321x_pm_can_update(const struct cd321x_pm_state *pm) +{ + return pm->phase == CD321X_PM_RUNNING || pm->phase == CD321X_PM_APPLY; +} + +static inline void cd321x_pm_prepare(struct cd321x_pm_state *pm) +{ + if (pm->phase != CD321X_PM_REMOVED) + pm->phase = CD321X_PM_PREPARED; + pm->attempts_left = 0; +} + +static inline bool cd321x_pm_resume(struct cd321x_pm_state *pm) +{ + if (pm->phase != CD321X_PM_PREPARED) + return false; + pm->phase = CD321X_PM_REVALIDATE; + pm->attempts_left = CD321X_RESUME_ATTEMPTS; + cd321x_pm_new_connection(pm); + return true; +} + +static inline bool cd321x_pm_link_event(struct cd321x_pm_state *pm) +{ + if (pm->phase == CD321X_PM_INITIALIZING || + pm->phase == CD321X_PM_PREPARED || pm->phase == CD321X_PM_REMOVED || + !pm->link_events_left) + return false; + + /* Host reprobes must not create an unlimited series of recovery runs. */ + pm->link_events_left--; + pm->phase = CD321X_PM_REVALIDATE; + pm->attempts_left = CD321X_RESUME_ATTEMPTS; + /* A stale notification does not itself prove the current session failed. */ + return true; +} + +static inline bool cd321x_pm_begin_read(struct cd321x_pm_state *pm) +{ + if (pm->phase != CD321X_PM_REVALIDATE || !pm->attempts_left) + return false; + pm->attempts_left--; + return true; +} + +static inline void cd321x_pm_snapshot_ready(struct cd321x_pm_state *pm) +{ + pm->phase = CD321X_PM_APPLY; +} + +static inline bool cd321x_pm_retry(struct cd321x_pm_state *pm) +{ + /* Runtime cable updates need the same bounded fresh-read recovery. */ + if (pm->phase == CD321X_PM_RUNNING) { + pm->force_reconnect = true; + pm->phase = CD321X_PM_REVALIDATE; + pm->attempts_left = CD321X_RESUME_ATTEMPTS; + return true; + } + if (pm->phase != CD321X_PM_REVALIDATE && pm->phase != CD321X_PM_APPLY) + return false; + pm->force_reconnect = true; + if (pm->attempts_left) { + pm->phase = CD321X_PM_REVALIDATE; + return true; + } + /* Only a fresh read, triggered by a later event, can reopen updates. */ + pm->phase = CD321X_PM_STALE; + return false; +} + +static inline bool cd321x_pm_event(struct cd321x_pm_state *pm) +{ + if (pm->phase != CD321X_PM_STALE) + return false; + pm->phase = CD321X_PM_REVALIDATE; + pm->attempts_left = CD321X_RESUME_ATTEMPTS; + return true; +} + +static inline void cd321x_pm_complete(struct cd321x_pm_state *pm) +{ + if (pm->phase == CD321X_PM_APPLY) { + pm->phase = CD321X_PM_RUNNING; + pm->attempts_left = 0; + pm->force_reconnect = false; + } +} + +static inline void cd321x_pm_remove(struct cd321x_pm_state *pm) +{ + pm->phase = CD321X_PM_REMOVED; + pm->attempts_left = 0; +} + +#endif /* __CD321X_PM_H__ */ diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index c68866c9655bb..a73434b5bca1b 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -22,7 +22,6 @@ #include #include #include -#include #include "tps6598x.h" #include "trace.h" @@ -543,7 +542,7 @@ static void tps6598x_handle_plug_event(struct tps6598x *tps, u32 status) } static int cd321x_typec_update_mode(struct tps6598x *tps, - struct cd321x_status *st) + struct cd321x_status *st, bool *display_busy) { struct cd321x *cd321x = container_of(tps, struct cd321x, tps); struct typec_mux_state old_state = cd321x->state; @@ -552,13 +551,16 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, }; int ret = 0; + *display_busy = false; if (!(st->data_status & TPS_DATA_STATUS_DATA_CONNECTION)) { if (cd321x->state_valid && cd321x->state.mode == TYPEC_STATE_SAFE) return 0; cd321x->state.alt = NULL; cd321x->state.mode = TYPEC_STATE_SAFE; cd321x->state.data = NULL; - typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); + ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); + if (ret) + goto out; ret = typec_mux_set(cd321x->mux, &cd321x->state); } else if (st->data_status & CD321X_DATA_STATUS_USB4_CONNECTION) { struct enter_usb_data eusb_data; @@ -661,8 +663,11 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, cd321x->state.alt = cd321x->port_altmode_dp; cd321x->state.data = &dp_data; cd321x->state.mode = mode; - typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); + ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); + if (ret) + goto out; ret = typec_mux_set(cd321x->mux, &cd321x->state); + *display_busy = ret == -EBUSY; if (!ret) { cd321x->dp_status = dp_data.status; cd321x->dp_conf = dp_data.conf; @@ -674,7 +679,9 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, cd321x->state.alt = NULL; cd321x->state.mode = TYPEC_STATE_USB; cd321x->state.data = NULL; - typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); + ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); + if (ret) + goto out; ret = typec_mux_set(cd321x->mux, &cd321x->state); } @@ -694,6 +701,24 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, return ret; } +static void cd321x_retry_revalidation(struct cd321x *cd321x) +{ + lockdep_assert_held(&cd321x->tps.lock); + if (cd321x->pm.phase != CD321X_PM_RUNNING && + cd321x->pm.phase != CD321X_PM_REVALIDATE && + cd321x->pm.phase != CD321X_PM_APPLY) + return; + + /* Keep a later physical event retryable if the automatic budget expires. */ + cd321x->state_valid = false; + if (cd321x_pm_retry(&cd321x->pm)) + mod_delayed_work(system_freezable_wq, &cd321x->resume_work, + msecs_to_jiffies(CD321X_DEBOUNCE_DELAY_MS)); + else + dev_err(cd321x->tps.dev, "cable revalidation failed after %u attempts\n", + CD321X_RESUME_ATTEMPTS); +} + static void cd321x_update_work(struct work_struct *work) { struct cd321x *cd321x = container_of(to_delayed_work(work), @@ -703,8 +728,12 @@ static void cd321x_update_work(struct work_struct *work) struct typec_thunderbolt_switch_data tbt_switch_data = { .state = TYPEC_THUNDERBOLT_SWITCH_OFF, }; + bool display_busy; + int ret; guard(mutex)(&tps->lock); + if (!cd321x_pm_can_update(&cd321x->pm)) + return; st = cd321x->update_status; cd321x->update_status.status_changed = 0; @@ -712,7 +741,10 @@ static void cd321x_update_work(struct work_struct *work) bool old_connected = !!tps->partner; bool new_connected = st.status & TPS_STATUS_PLUG_PRESENT; - bool was_disconnected = st.status_changed & TPS_STATUS_PLUG_PRESENT; + bool was_disconnected = cd321x->pm.force_reconnect || + (st.status_changed & (TPS_STATUS_PLUG_PRESENT | TPS_STATUS_PLUG_UPSIDE_DOWN)) || + (st.data_status_changed & (TPS_DATA_STATUS_TBT_CONNECTION | + CD321X_DATA_STATUS_USB4_CONNECTION)); bool usb_connection = st.data_status & (TPS_DATA_STATUS_USB2_CONNECTION | TPS_DATA_STATUS_USB3_CONNECTION); @@ -748,10 +780,13 @@ static void cd321x_update_work(struct work_struct *work) bool is_pd = pwr_opmode == TYPEC_PWR_MODE_PD; bool partner_changed = old_connected && new_connected && - (was_disconnected || + (was_disconnected || is_pd != cd321x->cur_partner_is_pd || (is_pd && memcmp(&st.partner_identity, &cd321x->cur_partner_identity, sizeof(struct usb_pd_identity)))); + /* A different dock must release the old routing before reconfiguration. */ + was_disconnected |= partner_changed; + /* * ACIO carries the tunneled PCIe reset handshake over the still-live * Type-C data path. Close a disappearing USB4/TBT session before dropping @@ -759,14 +794,20 @@ static void cd321x_update_work(struct work_struct *work) * PCIe-C cannot quiesce its hierarchy before NHI removal. */ if ((!new_connected || was_disconnected) && - (!cd321x->state_valid || cd321x->state.mode != TYPEC_STATE_SAFE)) - typec_thunderbolt_switch_set(cd321x->tbt_switch, - &tbt_switch_data); + (!cd321x->state_valid || cd321x->state.mode != TYPEC_STATE_SAFE)) { + ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, + &tbt_switch_data); + if (ret) + goto retry; + } /* If we are switching from an active role, transition to USB_ROLE_NONE. */ if (old_role != USB_ROLE_NONE && - (new_role != old_role || was_disconnected || dp_mode_changed)) - usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); + (new_role != old_role || was_disconnected || dp_mode_changed)) { + ret = usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); + if (ret) + goto retry; + } /* * Every Type-C port references the same external DCP connector. Only the @@ -787,12 +828,12 @@ static void cd321x_update_work(struct work_struct *work) if (!IS_ERR(tps->partner)) typec_unregister_partner(tps->partner); tps->partner = NULL; + cd321x->cur_partner_is_pd = false; } /* If there was a disconnection, set PHY to off */ if (!new_connected || was_disconnected) { struct typec_mux_state old_state = cd321x->state; - int ret; cd321x->state.alt = NULL; cd321x->state.mode = TYPEC_STATE_SAFE; @@ -804,6 +845,7 @@ static void cd321x_update_work(struct work_struct *work) "failed to place Type-C mux in safe mode: %d\n", ret); cd321x->state = old_state; + goto retry; } else { cd321x->display_route_active = false; } @@ -818,8 +860,10 @@ static void cd321x_update_work(struct work_struct *work) power_supply_changed(tps->psy); /* If the plug is disconnected, we are done */ - if (!new_connected) + if (!new_connected) { + cd321x_pm_complete(&cd321x->pm); return; + } /* Set up partner if we were previously disconnected (or changed). */ if (!tps->partner) { @@ -841,19 +885,26 @@ static void cd321x_update_work(struct work_struct *work) tps->partner = typec_register_partner(tps->port, &desc); if (IS_ERR(tps->partner)) { dev_warn(tps->dev, "%s: failed to register partner\n", __func__); - return; + tps->partner = NULL; + goto retry; } + cd321x->cur_partner_is_pd = is_pd; if (desc.identity) typec_partner_set_identity(tps->partner); } /* Update the TypeC MUX/PHY state */ - if (!cd321x_typec_update_mode(tps, &st)) - cd321x->display_route_active = dp_connected; + ret = cd321x_typec_update_mode(tps, &st, &display_busy); + if (ret && !display_busy) + goto retry; + /* A shared display route can be busy while the dock's USB path is usable. */ + cd321x->display_route_active = dp_connected && !ret; /* Launch the USB role switch */ - usb_role_switch_set_role(tps->role_sw, new_role); + ret = usb_role_switch_set_role(tps->role_sw, new_role); + if (ret) + goto retry; if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && cd321x->display_route_active && dp_hpd) @@ -871,6 +922,11 @@ static void cd321x_update_work(struct work_struct *work) connector_status_unknown); power_supply_changed(tps->psy); + cd321x_pm_complete(&cd321x->pm); + return; + +retry: + cd321x_retry_revalidation(cd321x); } static void cd321x_queue_status(struct cd321x *cd321x) @@ -892,8 +948,15 @@ static int cd321x_connect(struct tps6598x *tps, u32 status) { struct cd321x *cd321x = container_of(tps, struct cd321x, tps); + if ((tps->status ^ status) & TPS_STATUS_PLUG_PRESENT) + cd321x_pm_new_connection(&cd321x->pm); tps->status = status; cd321x_queue_status(cd321x); + /* A plug or power IRQ alone does not refresh the cached cable metadata. */ + if (cd321x_pm_event(&cd321x->pm)) + mod_delayed_work(system_freezable_wq, &cd321x->resume_work, 0); + if (!cd321x_pm_can_update(&cd321x->pm)) + return 0; /* * Requeue even when the previous update is already running. A separate @@ -901,48 +964,85 @@ static int cd321x_connect(struct tps6598x *tps, u32 status) * event in that window because the latter sees the running work as busy. * The worker consumes every accumulated status change on its next pass. */ - mod_delayed_work(system_wq, &cd321x->update_work, + mod_delayed_work(system_freezable_wq, &cd321x->update_work, msecs_to_jiffies(CD321X_DEBOUNCE_DELAY_MS)); return 0; } /* - * A cable that was already attached before a suspend/resume cycle generates - * no fresh attach event on resume: the PD controller's own status hasn't - * changed, so cd321x_connect()'s diff against its last cached status sees - * nothing different and cd321x_update_work() takes its "nothing to do" - * path. Meanwhile thunderbolt-core's own noirq-phase resume handling - * (tb_free_invalid_tunnels()) has already torn down any USB4/TBT tunnel - * that didn't survive the sleep at the link level -- and nothing else - * re-establishes it. - * - * Reproduce a genuine unplug-then-replug in software instead of inventing - * a new code path: call the existing connect() callback once with the - * current status but PLUG_PRESENT cleared (a synthetic disconnect), then - * again with the real, current status. Both updates land on the same - * debounced work item, which coalesces them into a single run with - * was_disconnected=true and the correct final state -- exactly the same - * teardown-then-bring-up sequence, through the same already-validated - * apple_cio_tbt_switch_set() path, that a real physical replug drives. + * An attached cable need not generate an IRQ after sleep. Read a fresh + * snapshot and reapply it with the mode cache invalidated, allowing the + * provider to validate the resumed session. Preserve a healthy connection; + * only a real cable change or a failed read/apply requests forced recovery. */ -static void cd321x_resume_reverify(struct tps6598x *tps) +static void cd321x_resume_work(struct work_struct *work) { + struct cd321x *cd321x = container_of(to_delayed_work(work), + struct cd321x, resume_work); + struct tps6598x *tps = &cd321x->tps; u32 status; - if (!tps6598x_read_status(tps, &status)) + guard(mutex)(&tps->lock); + if (cd321x->pm.phase == CD321X_PM_INITIALIZING || + cd321x->pm.phase == CD321X_PM_PREPARED || + cd321x->pm.phase == CD321X_PM_REMOVED) return; - if (!(status & TPS_STATUS_PLUG_PRESENT)) + if (atomic_xchg(&cd321x->link_event, 0)) { + if (cd321x_pm_link_event(&cd321x->pm)) { + cd321x->state_valid = false; + dev_info(tps->dev, "revalidating cable after tunnel link failure\n"); + } else { + dev_warn_ratelimited(tps->dev, + "tunnel link recovery budget exhausted\n"); + } + } + if (!cd321x_pm_begin_read(&cd321x->pm)) return; - if (!tps6598x_read_power_status(tps)) - return; - if (!tps->data->read_data_status(tps)) - return; + if (!tps6598x_read_status(tps, &status)) + goto retry; + + if (status & TPS_STATUS_PLUG_PRESENT) { + if (!tps6598x_read_power_status(tps)) + goto retry; + if (TPS_POWER_STATUS_PWROPMODE(tps->pwr_status) == TYPEC_PWR_MODE_PD && + tps6598x_read_partner_identity(tps)) + goto retry; + if (!tps->data->read_data_status(tps)) + goto retry; + } - tps->data->connect(tps, status & ~TPS_STATUS_PLUG_PRESENT); + /* Also report a cable removed during sleep, even if its IRQ was lost. */ + cd321x_pm_snapshot_ready(&cd321x->pm); tps->data->connect(tps, status); + return; + +retry: + cd321x_retry_revalidation(cd321x); +} + +static void cd321x_suspend_prepare(struct tps6598x *tps) +{ + struct cd321x *cd321x = container_of(tps, struct cd321x, tps); + + scoped_guard(mutex, &tps->lock) { + cd321x_pm_prepare(&cd321x->pm); + cd321x->state_valid = false; + } + /* Gate scheduling first; neither worker may be drained under its mutex. */ + cancel_delayed_work_sync(&cd321x->resume_work); + cancel_delayed_work_sync(&cd321x->update_work); +} + +static void cd321x_resume_reverify(struct tps6598x *tps) +{ + struct cd321x *cd321x = container_of(tps, struct cd321x, tps); + + guard(mutex)(&tps->lock); + if (cd321x_pm_resume(&cd321x->pm)) + mod_delayed_work(system_freezable_wq, &cd321x->resume_work, 0); } static irqreturn_t cd321x_interrupt(int irq, void *data) @@ -1340,6 +1440,47 @@ static int cd321x_register_port_altmodes(struct cd321x *cd321x) return 0; } +static int cd321x_tbt_notify(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct cd321x *cd321x = container_of(nb, struct cd321x, tbt_notifier); + + if (event != TYPEC_THUNDERBOLT_SWITCH_REVALIDATE) + return NOTIFY_DONE; + + /* Publish the event even if the worker is already reading or applying. */ + atomic_xchg(&cd321x->link_event, 1); + mod_delayed_work(system_freezable_wq, &cd321x->resume_work, 0); + return NOTIFY_OK; +} + +static void cd321x_ready(struct tps6598x *tps) +{ + struct cd321x *cd321x = container_of(tps, struct cd321x, tps); + + guard(mutex)(&tps->lock); + /* Include failures reported while the port was still being initialized. */ + if (cd321x_pm_ready(&cd321x->pm)) { + cd321x->state_valid = false; + mod_delayed_work(system_freezable_wq, &cd321x->resume_work, 0); + } +} + +static void cd321x_stop_work(struct cd321x *cd321x) +{ + scoped_guard(mutex, &cd321x->tps.lock) + cd321x_pm_remove(&cd321x->pm); + + /* Drain producers before cancelling work, including on probe rollback. */ + if (cd321x->tbt_notifier_registered) { + typec_thunderbolt_switch_unregister_notifier(cd321x->tbt_switch, + &cd321x->tbt_notifier); + cd321x->tbt_notifier_registered = false; + } + cancel_delayed_work_sync(&cd321x->resume_work); + cancel_delayed_work_sync(&cd321x->update_work); +} + static int cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) { @@ -1348,6 +1489,9 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) int ret; INIT_DELAYED_WORK(&cd321x->update_work, cd321x_update_work); + INIT_DELAYED_WORK(&cd321x->resume_work, cd321x_resume_work); + atomic_set(&cd321x->link_event, 0); + cd321x_pm_init(&cd321x->pm); ret = tps6598x_register_port(tps, fwnode); if (ret) @@ -1378,8 +1522,18 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) cd321x->state.data = NULL; cd321x->state_valid = !typec_set_mode(tps->port, TYPEC_STATE_SAFE); + cd321x->tbt_notifier.notifier_call = cd321x_tbt_notify; + ret = typec_thunderbolt_switch_register_notifier(cd321x->tbt_switch, + &cd321x->tbt_notifier); + if (ret) + goto err_put_switch; + cd321x->tbt_notifier_registered = true; + return 0; +err_put_switch: + typec_thunderbolt_switch_put(cd321x->tbt_switch); + cd321x->tbt_switch = NULL; err_unregister_mux: typec_mux_put(cd321x->mux); cd321x->mux = NULL; @@ -1404,6 +1558,7 @@ cd321x_unregister_port(struct tps6598x *tps) { struct cd321x *cd321x = container_of(tps, struct cd321x, tps); + cd321x_stop_work(cd321x); typec_thunderbolt_switch_put(cd321x->tbt_switch); cd321x->tbt_switch = NULL; typec_mux_put(cd321x->mux); @@ -1829,7 +1984,7 @@ static void cd321x_remove(struct tps6598x *tps) }; int ret; - cancel_delayed_work_sync(&cd321x->update_work); + cd321x_stop_work(cd321x); /* * Driver teardown must close the same hardware sessions as a physical @@ -1849,7 +2004,6 @@ static void cd321x_remove(struct tps6598x *tps) usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); } -static int tipd_pm_notify(struct notifier_block *nb, unsigned long action, void *data); int tipd_init(struct tps6598x *tps) { @@ -1958,10 +2112,8 @@ int tipd_init(struct tps6598x *tps) enable_irq_wake(tps->irq); } - if (tps->data->resume_reverify) { - tps->pm_nb.notifier_call = tipd_pm_notify; - register_pm_notifier(&tps->pm_nb); - } + if (tps->data->ready) + tps->data->ready(tps); return 0; @@ -1986,9 +2138,6 @@ EXPORT_SYMBOL_GPL(tipd_init); void tipd_remove(struct tps6598x *tps) { - if (tps->data->resume_reverify) - unregister_pm_notifier(&tps->pm_nb); - if (!tps->irq) cancel_delayed_work_sync(&tps->wq_poll); else @@ -2055,35 +2204,27 @@ int tipd_resume(struct tps6598x *tps) } EXPORT_SYMBOL_GPL(tipd_resume); -/* - * tipd_resume() runs as an ordinary device .resume() callback, which can - * complete well before the rest of the system -- ACIO/Thunderbolt, DCP, - * the crossbar -- has finished its own resume work. Driving the reconnect - * from here directly (as originally tried) fires apple_cio_start()'s - * M3/PMGR reset-deassert handshake while that firmware is potentially - * still busy servicing other subsystems' resume, and can exhaust its - * retry budget outright -- confirmed on hardware, see - * notes/2026-09-24-0147-typec-resume-reverify.md. A real physical - * replug never hits this because a human always does it well after - * the whole system has settled. - * - * PM_POST_SUSPEND is delivered strictly after dpm_resume_end() -- - * i.e. after every device, including ACIO/Thunderbolt, has completed - * its own ordinary resume -- so triggering the reverify from here - * instead removes the race without needing any arbitrary delay. - */ -static int tipd_pm_notify(struct notifier_block *nb, unsigned long action, void *data) +/* Freezable cable work has drained before device PM calls prepare. */ +int tipd_prepare(struct device *dev) { - struct tps6598x *tps = container_of(nb, struct tps6598x, pm_nb); + struct tps6598x *tps = dev_get_drvdata(dev); - if (action == PM_POST_SUSPEND && tps->data->resume_reverify) { - mutex_lock(&tps->lock); - tps->data->resume_reverify(tps); - mutex_unlock(&tps->lock); - } + if (tps->data->suspend_prepare) + tps->data->suspend_prepare(tps); - return NOTIFY_DONE; + return 0; +} +EXPORT_SYMBOL_GPL(tipd_prepare); + +/* Completion also runs after an aborted suspend; work starts after thaw. */ +void tipd_complete(struct device *dev) +{ + struct tps6598x *tps = dev_get_drvdata(dev); + + if (tps->data->resume_reverify) + tps->data->resume_reverify(tps); } +EXPORT_SYMBOL_GPL(tipd_complete); static const struct tipd_data tipd_cd321x_data = { .irq_handler = cd321x_interrupt, @@ -2094,6 +2235,7 @@ static const struct tipd_data tipd_cd321x_data = { .remove = cd321x_remove, .register_port = cd321x_register_port, .unregister_port = cd321x_unregister_port, + .ready = cd321x_ready, .trace_data_status = trace_cd321x_data_status, .trace_power_status = trace_tps6598x_power_status, .trace_status = trace_tps6598x_status, @@ -2102,6 +2244,7 @@ static const struct tipd_data tipd_cd321x_data = { .reset = cd321x_reset, .switch_power_state = cd321x_switch_power_state, .connect = cd321x_connect, + .suspend_prepare = cd321x_suspend_prepare, .resume_reverify = cd321x_resume_reverify, }; @@ -2150,6 +2293,7 @@ static const struct tipd_data tipd_sn201202x_data = { .remove = cd321x_remove, .register_port = cd321x_register_port, .unregister_port = cd321x_unregister_port, + .ready = cd321x_ready, .trace_data_status = trace_cd321x_data_status, .trace_power_status = trace_tps6598x_power_status, .trace_status = trace_tps6598x_status, @@ -2158,6 +2302,7 @@ static const struct tipd_data tipd_sn201202x_data = { .reset = cd321x_reset, .switch_power_state = cd321x_switch_power_state, .connect = cd321x_connect, + .suspend_prepare = cd321x_suspend_prepare, .resume_reverify = cd321x_resume_reverify, }; diff --git a/drivers/usb/typec/tipd/i2c.c b/drivers/usb/typec/tipd/i2c.c index f852cdc327f5d..4136cb9222361 100644 --- a/drivers/usb/typec/tipd/i2c.c +++ b/drivers/usb/typec/tipd/i2c.c @@ -73,6 +73,8 @@ static int __maybe_unused tps6598x_resume(struct device *dev) } static const struct dev_pm_ops tps6598x_pm_ops = { + .prepare = pm_sleep_ptr(tipd_prepare), + .complete = pm_sleep_ptr(tipd_complete), SET_SYSTEM_SLEEP_PM_OPS(tps6598x_suspend, tps6598x_resume) }; diff --git a/drivers/usb/typec/tipd/spmi.c b/drivers/usb/typec/tipd/spmi.c index 2960646d1c47a..59d563aebfff5 100644 --- a/drivers/usb/typec/tipd/spmi.c +++ b/drivers/usb/typec/tipd/spmi.c @@ -326,6 +326,8 @@ static int __maybe_unused sn201202x_suspend(struct device *dev) MODULE_DEVICE_TABLE(of, sn201202x_of_match); static const struct dev_pm_ops sn201202x_pm_ops = { + .prepare = pm_sleep_ptr(tipd_prepare), + .complete = pm_sleep_ptr(tipd_complete), SET_SYSTEM_SLEEP_PM_OPS(sn201202x_suspend, sn201202x_resume) }; diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 0bb92e907fbbd..e06a671b4e57e 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -6,9 +6,11 @@ * Author: Heikki Krogerus */ +#include #include #include #include +#include #include #include #include @@ -16,6 +18,8 @@ #ifndef __TPS6598X_H__ #define __TPS6598X_H__ +#include "cd321x-pm.h" + #define TPS_REG_MODE 0x03 #define TPS_FIELD_GET(_mask, _reg) ((typeof(_mask))(((_reg) & (_mask)) >> __bf_shf(_mask))) @@ -294,6 +298,7 @@ struct tipd_data { void (*remove)(struct tps6598x *tps); int (*register_port)(struct tps6598x *tps, struct fwnode_handle *node); void (*unregister_port)(struct tps6598x *tps); + void (*ready)(struct tps6598x *tps); void (*trace_data_status)(u32 status); void (*trace_power_status)(u16 status); void (*trace_status)(u32 status); @@ -303,6 +308,7 @@ struct tipd_data { bool (*read_data_status)(struct tps6598x *tps); int (*reset)(struct tps6598x *tps); int (*connect)(struct tps6598x *tps, u32 status); + void (*suspend_prepare)(struct tps6598x *tps); void (*resume_reverify)(struct tps6598x *tps); }; @@ -330,8 +336,6 @@ struct tps6598x { u16 pwr_status; struct delayed_work wq_poll; - struct notifier_block pm_nb; - const struct tipd_data *data; }; @@ -387,11 +391,17 @@ struct cd321x { u32 dp_status; u32 dp_conf; struct typec_thunderbolt_switch *tbt_switch; + struct notifier_block tbt_notifier; + bool tbt_notifier_registered; + atomic_t link_event; bool display_route_active; struct cd321x_status update_status; struct delayed_work update_work; + struct delayed_work resume_work; + struct cd321x_pm_state pm; struct usb_pd_identity cur_partner_identity; + bool cur_partner_is_pd; struct fwnode_handle *connector_fwnode; }; @@ -410,6 +420,8 @@ extern const struct regmap_config tps6598x_regmap_config; int tipd_init(struct tps6598x *tps); void tipd_remove(struct tps6598x *tps); +int tipd_prepare(struct device *dev); +void tipd_complete(struct device *dev); int tipd_suspend(struct tps6598x *tps); int tipd_resume(struct tps6598x *tps); diff --git a/include/linux/usb/typec_mux.h b/include/linux/usb/typec_mux.h index 0726407bb6240..d58d60fcae21f 100644 --- a/include/linux/usb/typec_mux.h +++ b/include/linux/usb/typec_mux.h @@ -15,6 +15,7 @@ struct typec_switch; struct typec_switch_dev; struct typec_altmode; struct fwnode_handle; +struct notifier_block; typedef int (*typec_switch_set_fn_t)(struct typec_switch_dev *sw, enum typec_orientation orientation); @@ -145,6 +146,9 @@ static inline struct typec_mux *typec_mux_get(struct device *dev) struct typec_thunderbolt_switch; struct typec_thunderbolt_switch_dev; +/* Recheck fresh cable data and provider health; not a request to reset. */ +#define TYPEC_THUNDERBOLT_SWITCH_REVALIDATE 1 + enum typec_thunderbolt_switch_state { TYPEC_THUNDERBOLT_SWITCH_OFF, TYPEC_THUNDERBOLT_SWITCH_TBT, @@ -192,6 +196,11 @@ fwnode_typec_thunderbolt_switch_get(struct fwnode_handle *fwnode); void typec_thunderbolt_switch_put(struct typec_thunderbolt_switch *sw); int typec_thunderbolt_switch_set(struct typec_thunderbolt_switch *sw, const struct typec_thunderbolt_switch_data *data); +int typec_thunderbolt_switch_register_notifier(struct typec_thunderbolt_switch *sw, + struct notifier_block *nb); +void typec_thunderbolt_switch_unregister_notifier(struct typec_thunderbolt_switch *sw, + struct notifier_block *nb); +void typec_thunderbolt_switch_notify(struct typec_thunderbolt_switch_dev *sw); struct typec_thunderbolt_switch_dev * typec_thunderbolt_switch_register(struct device *parent, @@ -230,6 +239,24 @@ typec_thunderbolt_switch_register(struct device *parent, return ERR_PTR(-EOPNOTSUPP); } +static inline int +typec_thunderbolt_switch_register_notifier(struct typec_thunderbolt_switch *sw, + struct notifier_block *nb) +{ + return 0; +} + +static inline void +typec_thunderbolt_switch_unregister_notifier(struct typec_thunderbolt_switch *sw, + struct notifier_block *nb) +{ +} + +static inline void +typec_thunderbolt_switch_notify(struct typec_thunderbolt_switch_dev *sw) +{ +} + static inline void typec_thunderbolt_switch_unregister(struct typec_thunderbolt_switch_dev *sw) { From 56e64cc49b6163e795f3e1dde2343955c516e7ff Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:39:09 -0700 Subject: [PATCH 0987/2680] thunderbolt: keep routers awake across system sleep on Apple hosts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Apple silicon the host router stays powered through suspend-to-idle and its links stay trained. Asking the routers to sleep anyway leaves the request pending with nothing to complete it, and the Thunderbolt link then fails at a fixed time after the request, whether the host is still asleep or has already resumed: thunderbolt-apple-acio ...: RTKit: syslog message: ... Gen2/3 link error pcie-apple 3b0000000.pcie: Link down on ... On a MacBook Pro 13" M1 (j293) with a Kensington SD5560T this happened 40.8 s after "PM: suspend entry" in each of ten cycles, on four kernels and both ports. A suspend longer than that loses the dock while asleep; a shorter one loses it after resume. A fresh link training before the deadline cancels it. Add QUIRK_NO_SYSTEM_SLEEP and skip the wake and sleep programming for system sleep when it is set. Plug events stay disabled as before and are re-enabled on resume. The Apple NHI sets the quirk unless the router_sleep test parameter asks for the previous behaviour. [ryan: adapted to 7.2.10 — Asahi took QUIRK bits 2-4, so QUIRK_NO_SYSTEM_SLEEP moves to BIT(6)] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PA1D59hehGmwnpSxrYcDbk --- drivers/thunderbolt/apple.c | 13 +++++++++++++ drivers/thunderbolt/nhi.h | 6 ++++++ drivers/thunderbolt/switch.c | 4 ++++ 3 files changed, 23 insertions(+) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index ccec6f80409a3..5b650e0e7e548 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -1836,10 +1836,23 @@ static int apple_nhi_prepare(struct device *dev) return 0; } +/* + * ACIO stays powered through suspend-to-idle and keeps its links trained, so + * nothing ever completes a router sleep request. Leave the routers awake. + */ +static bool router_sleep; +module_param(router_sleep, bool, 0644); +MODULE_PARM_DESC(router_sleep, "Ask routers to sleep on system suspend (test only)"); + static int apple_nhi_suspend_noirq(struct device *dev) { struct apple_nhi *anhi = dev_get_drvdata(dev); + if (READ_ONCE(router_sleep)) + anhi->nhi.quirks &= ~QUIRK_NO_SYSTEM_SLEEP; + else + anhi->nhi.quirks |= QUIRK_NO_SYSTEM_SLEEP; + return tb_domain_suspend_noirq(anhi->tb); } diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index e272166635c82..195030cc51bc5 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -196,6 +196,12 @@ struct tb_nhi_ops { #define QUIRK_NO_DMA_PORT BIT(3) #define QUIRK_NO_USB3_BW_ALLOC BIT(4) #define QUIRK_HOST_DP_NFC_CREDITS BIT(5) +/* + * The host router stays powered and its links stay up across system sleep, + * so routers must not be asked to enter sleep: a TBT3 device router that + * was told to sleep drops its link on its own some tens of seconds later. + */ +#define QUIRK_NO_SYSTEM_SLEEP BIT(6) /* * Minimal number of vectors when we use MSI-X. Two for control channel diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c index 22f071430763a..1ea18ceb274c3 100644 --- a/drivers/thunderbolt/switch.c +++ b/drivers/thunderbolt/switch.c @@ -3684,6 +3684,10 @@ void tb_switch_suspend(struct tb_switch *sw, bool runtime) flags |= TB_WAKE_ON_USB4 | TB_WAKE_ON_USB3 | TB_WAKE_ON_PCIE; } + /* The link stays up; a sleeping router would drop it by itself. */ + if (!runtime && (sw->tb->nhi->quirks & QUIRK_NO_SYSTEM_SLEEP)) + return; + tb_switch_set_wake(sw, flags, runtime); if (tb_switch_is_usb4(sw)) From e495e179a7498b5292ec293e9f0fff28188590c1 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:39:09 -0700 Subject: [PATCH 0988/2680] thunderbolt: restore DP resources after resume when routers stayed awake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Suspend releases every DP tunnel and resource, expecting plug events to announce them again after resume. Routers that were never asked to sleep send no such events, so the DP IN and DP OUT adapters are not paired up again and an external display behind a dock stays dark after resume. For hosts with QUIRK_NO_SYSTEM_SLEEP, re-add the host and device DP IN resources and any DP OUT adapter with HPD asserted from tb_complete(), then pair them. [ryan: adapted to 7.2.10 — 7.2 unregisters unplugged XDomains outside the domain lock; the DP resource restore takes the lock itself] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PA1D59hehGmwnpSxrYcDbk --- drivers/thunderbolt/tb.c | 47 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index 58c61acf397c0..e495060c15a19 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -3311,6 +3311,43 @@ static int tb_thaw_noirq(struct tb *tb) return 0; } +static bool tb_dp_resource_listed(struct tb *tb, struct tb_port *port) +{ + struct tb_cm *tcm = tb_priv(tb); + struct tb_port *p; + + list_for_each_entry(p, &tcm->dp_resources, list) { + if (p == port) + return true; + } + return false; +} + +static void tb_restore_dp_resources(struct tb_switch *sw) +{ + struct tb_cm *tcm = tb_priv(sw->tb); + struct tb_port *port; + + if (sw->is_unplugged) + return; + + tb_switch_for_each_port(sw, port) { + if (tb_port_has_remote(port)) { + tb_restore_dp_resources(port->remote->sw); + } else if (tb_port_is_dpin(port)) { + if (tb_dp_resource_listed(sw->tb, port) || + !tb_switch_query_dp_resource(sw, port)) + continue; + tb_port_dbg(port, "DP IN resource available after resume\n"); + list_add_tail(&port->list, &tcm->dp_resources); + } else if (tb_port_is_dpout(port) && + tb_dp_port_hpd_is_active(port) == 1 && + !tb_dp_port_is_enabled(port)) { + tb_dp_resource_available(sw->tb, port); + } + } +} + static void tb_complete(struct tb *tb) { /* @@ -3322,6 +3359,16 @@ static void tb_complete(struct tb *tb) scoped_guard(mutex, &tb->lock) tb_scan_switch(tb->root_switch); } + + /* + * Routers kept awake through system sleep send no plug events for + * the DP resources released at suspend, so pair them up again here. + */ + if (tb->nhi->quirks & QUIRK_NO_SYSTEM_SLEEP) { + guard(mutex)(&tb->lock); + tb_restore_dp_resources(tb->root_switch); + tb_tunnel_dp(tb); + } } static int tb_runtime_suspend(struct tb *tb) From 70440e3210e6db7e8accfe514952b44671f386e6 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:03:25 -0700 Subject: [PATCH 0989/2680] thunderbolt: keep M1 PCIe tunnels up through suspend-to-idle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In suspend-to-idle ACIO stays powered and, with the routers kept awake, the Thunderbolt link stays trained. Stopping the tunneled PCIe port anyway holds the dock hierarchy in reset for the whole sleep. After that, every function below it has to be restored and each USB device behind the dock's xHCI reset-resumed one after another, about 5 s before input works again. After a sleep of about 80 s, a Kensington SD5560T on a MacBook Pro 13" M1 (j293) stopped answering altogether shortly after resume: its link trained and its bridges were restored, then every request timed out. When the tunneled port is up and healthy at suspend-to-idle, leave it running. On resume, check that the link is still up and that no link-down event is pending; if so the hierarchy kept its state and resumes like any other PCIe device. If the link was lost while asleep, fall back to the stop, reset and restart path. The Thunderbolt connection manager then leaves the still-active tunnels alone instead of tearing them down and setting them up again. The pcie_apple s2idle_keep_link parameter restores the previous behaviour for comparison. [ryan: adapted to 7.2.10 — split by topic: the PCIe controller hunks are carried by 080-radio] [ryan: adapted to 7.2.10 — Asahi took QUIRK bits 2-4, so QUIRK_KEEP_TUNNELS moves to BIT(7)] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PA1D59hehGmwnpSxrYcDbk --- drivers/thunderbolt/apple.c | 23 ++++++++++++++++++++--- drivers/thunderbolt/nhi.h | 5 +++++ drivers/thunderbolt/tb.c | 10 ++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 5b650e0e7e548..d93cfdf3701d6 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -1844,14 +1844,31 @@ static bool router_sleep; module_param(router_sleep, bool, 0644); MODULE_PARM_DESC(router_sleep, "Ask routers to sleep on system suspend (test only)"); +static bool apple_nhi_pcie_link_kept(struct apple_cio *acio) +{ + struct platform_device *pcie_pdev; + bool kept = false; + + guard(mutex)(&acio->pcie_tunnel_lock); + pcie_pdev = apple_cio_find_pcie_tunnel(acio); + if (pcie_pdev) { + kept = apple_pcie_tunnel_link_kept(&pcie_pdev->dev); + put_device(&pcie_pdev->dev); + } + return kept; +} + static int apple_nhi_suspend_noirq(struct device *dev) { struct apple_nhi *anhi = dev_get_drvdata(dev); - if (READ_ONCE(router_sleep)) - anhi->nhi.quirks &= ~QUIRK_NO_SYSTEM_SLEEP; - else + anhi->nhi.quirks &= ~(QUIRK_NO_SYSTEM_SLEEP | QUIRK_KEEP_TUNNELS); + if (!READ_ONCE(router_sleep)) { anhi->nhi.quirks |= QUIRK_NO_SYSTEM_SLEEP; + /* PCIe-C suspends first; its tunnel must then survive resume. */ + if (apple_nhi_pcie_link_kept(anhi->acio)) + anhi->nhi.quirks |= QUIRK_KEEP_TUNNELS; + } return tb_domain_suspend_noirq(anhi->tb); } diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index 195030cc51bc5..9676dd8cd8acf 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -202,6 +202,11 @@ struct tb_nhi_ops { * was told to sleep drops its link on its own some tens of seconds later. */ #define QUIRK_NO_SYSTEM_SLEEP BIT(6) +/* + * Set for one system sleep when the routers stayed awake and the tunneled + * PCIe host kept its link up: the tunnels are still active on resume. + */ +#define QUIRK_KEEP_TUNNELS BIT(7) /* * Minimal number of vectors when we use MSI-X. Two for control channel diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index e495060c15a19..4c7954ba91cc1 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -3255,6 +3255,15 @@ static int tb_resume_noirq(struct tb *tb) tb_free_unplugged_xdomains(tb->root_switch); tb_restore_children(tb->root_switch); + /* + * Routers that stayed awake still carry our tunnels. Restarting them + * would take down links the hosts kept up through the sleep. + */ + if (tb->nhi->quirks & QUIRK_KEEP_TUNNELS) { + tb_dbg(tb, "tunnels kept across sleep\n"); + goto out; + } + /* * If we get here from suspend to disk the boot firmware or the * restore kernel might have created tunnels of its own. Since @@ -3287,6 +3296,7 @@ static int tb_resume_noirq(struct tb *tb) tb_dbg(tb, "tunnels restarted, sleeping for 100ms\n"); msleep(100); } +out: tb_switch_enter_redrive(tb->root_switch); /* Allow tb_handle_hotplug to progress events */ tcm->hotplug_active = true; From 5b797965fef9b01b3e9a6156204472031c76b193 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:31:47 -0700 Subject: [PATCH 0990/2680] thunderbolt: keep DP tunnels through suspend-to-idle with kept tunnels Suspend releases DP tunnels so plug events can rebuild them after resume. When the routers stay awake and the other tunnels are kept, that release is the only thing that disconnects a display behind a dock. The display driver reports it to userspace as an unplug. After resume the compositor then switches off the output it still believes is gone, racing the restored tunnel. With a CalDigit TS3 Plus on a MacBook Pro 13" M1 (j293) the monitor stayed dark after resume: the DPTX link trained, then the output was powered off and never re-enabled. Keep DP tunnels as well when tunnels are kept. The display is then powered down and up across the sleep, as for DPMS, and is never reported as unplugged. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PA1D59hehGmwnpSxrYcDbk --- drivers/thunderbolt/tb.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index 4c7954ba91cc1..5d87d6c7da63a 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -3174,7 +3174,12 @@ static int tb_suspend_noirq(struct tb *tb) struct tb_cm *tcm = tb_priv(tb); tb_dbg(tb, "suspending...\n"); - tb_disconnect_and_release_dp(tb); + /* + * Kept tunnels survive the sleep. Keep DP ones as well, so a display + * behind a dock is only powered down, not reported as unplugged. + */ + if (!(tb->nhi->quirks & QUIRK_KEEP_TUNNELS)) + tb_disconnect_and_release_dp(tb); tb_switch_exit_redrive(tb->root_switch); tb_switch_suspend(tb->root_switch, false); tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */ From 03b3ab1716b94986b55b5182c7e0677b9b27ac9b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:14:06 -0700 Subject: [PATCH 0991/2680] usb: dwc3: apple: leave the USB2 host role selection to xHCI Since "usb: dwc3: apple: Acquire the PHYs at probe", the glue selects the USB2 PHY role before it releases the dwc3 reset on every bring-up, including the first one after boot. Before that commit, the first bring-up's selection was a no-op, and mainline still behaves that way. For a host, the role was then selected once the core was out of reset: usb_add_hcd() asks the root hub PHYs for PHY_MODE_USB_HOST_SS, which the USB2 PHY rejects, and then for PHY_MODE_USB_HOST. On a Type-C port the USB2 PHY is already running when the glue makes its selection, because the Type-C mux starts it before the role switch reaches the glue. Switching the running PHY to host mode while the core is held in reset leaves the USB2 port unable to see a connection for the whole session. The SuperSpeed side is unaffected. Selecting host mode is harmless only when the PHY is in host mode already, and then the selection does nothing. On a t8103 (j293) with a CalDigit TS4 in USB4 mode, the first session on each port after boot never connected USB2. That was 7 of 7 sessions, across both ports, with the dock present at boot and hot-plugged. The xHCI USB2 root port stayed powered in RxDetect (PORTSC 0x0a0002a0). USB2-only devices behind the dock, such as keyboards and mice, never appeared, and only a replug on the same port brought them up. Later sessions connected, because the PHY kept host mode from the first one. Power-cycling only the USB2 PHY around the role change did not help. Leaving host mode to the root hub did: the dock's USB2 hub connected on the first session, 0.4 s after the dock, with its keyboard and mice. Select only device mode before the reset. Device mode has no later selector, since the glue writes PRTCAP itself and the gadget code sets no PHY mode, so it keeps its pre-reset selection. Host mode is always left to the root hub. Only the first host session after boot was tested. A later device-to-host change reaches the same PHY state: host mode is clear and the PHY is powered off at the end of the device session and restarted by the Type-C mux. It is expected to behave the same, but has not been tested. A fixed-hub controller is unaffected, because its PHY driver sets host mode for it. Fixes: 0ab0ac187140 ("usb: dwc3: apple: Acquire the PHYs at probe") Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/usb/dwc3/dwc3-apple.c | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index 1f86350a11dfc..2d0662ac77f72 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -93,7 +93,8 @@ enum dwc3_apple_state { * @mmio_resource: Resource to be passed to dwc3_core_probe * @apple_regs: Apple-specific DWC3 registers * @reset: Reset control - * @usb2_phy: USB2 PHY, configured before the core is brought up + * @usb2_phy: USB2 PHY, set to device mode before the core is brought up; the xHCI root hub + * sets host mode * @usb3_phy: USB3 PHY, configured after the core is brought up * @role_sw: USB role switch * @fixed_hub: A fixed hub sits on the controller's USB2 port, the controller stays in host mode @@ -269,17 +270,23 @@ static int dwc3_apple_core_start(struct dwc3_apple *appledwc, enum dwc3_apple_st lockdep_assert_held(&appledwc->lock); /* - * The USB2 PHY on this platform must be configured for host or device mode while it is - * still powered off and before dwc3 tries to access it. Otherwise, the new configuration - * will sometimes only take affect after the *next* time dwc3 is brought up which causes - * the connected device to just not work. + * The USB2 PHY on this platform must be configured for device mode before dwc3 tries to + * access it. Otherwise, the new configuration will sometimes only take affect after the + * *next* time dwc3 is brought up which causes the connected device to just not work. + * Host mode is left to the xHCI root hub, which selects it once the core is out of reset + * (usb_add_hcd() asks for PHY_MODE_USB_HOST_SS, which the PHY rejects, and then for + * PHY_MODE_USB_HOST). On a Type-C port the PHY is already running at this point, because + * the Type-C mux starts it before the role switch reaches this driver, and switching the + * running PHY to host mode while the core is still held in reset leaves the USB2 port + * without a connection for the whole session. A PHY that is in host mode already needs no + * selection here either. * The USB3 PHY must be configured later after dwc3 has already been initialized. - * Both PHYs were looked up at probe, so this also covers the first bring-up, before - * dwc3_core_probe() has looked them up itself. + * Both PHYs were looked up at probe, so device mode is also selected on the first + * bring-up, before dwc3_core_probe() has looked them up itself. */ switch (state) { case DWC3_APPLE_HOST: - ret = phy_set_mode(appledwc->usb2_phy, PHY_MODE_USB_HOST); + ret = 0; break; case DWC3_APPLE_DEVICE: ret = phy_set_mode(appledwc->usb2_phy, PHY_MODE_USB_DEVICE); @@ -326,8 +333,9 @@ static void dwc3_apple_set_role(struct dwc3_apple *appledwc, enum dwc3_apple_sta dwc3_apple_set_ptrcap(appledwc, host ? DWC3_GCTL_PRTCAP_HOST : DWC3_GCTL_PRTCAP_DEVICE); /* * This platform requires SUSPHY to be enabled here already in order to properly - * configure the PHY and switch dwc3's PIPE interface to USB3 PHY. The USB2 PHY - * has already been configured to the correct mode earlier. + * configure the PHY and switch dwc3's PIPE interface to USB3 PHY. The USB2 PHY's + * device mode has already been selected in dwc3_apple_core_start(); its host mode + * is selected by the xHCI root hub as it starts. */ dwc3_enable_susphy(&appledwc->dwc, true); ret = phy_set_mode(appledwc->usb3_phy, host ? PHY_MODE_USB_HOST : PHY_MODE_USB_DEVICE); From 8594732724b11638fc54212aa88c254d8294f37e Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:25:13 -0700 Subject: [PATCH 0992/2680] usb: dwc3: apple: test the USB2 PHY mode choice with KUnit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit selects the USB2 PHY's mode before dwc3 leaves reset for device mode only, and leaves host mode to the xHCI root hub. Selecting host mode there again would bring back sessions without USB2. Move that choice into dwc3_apple_usb2_mode_before_reset(), which also rejects states that are not roles, and move the controller state enum into dwc3-apple-state.h with it. dwc3_apple_core_start() takes both the validation and the mode from the helper, so the choice lives in one place. A KUnit suite pins the helper's results for every state. It covers the helper, not the hardware ordering behind it. [ryan: adapted to 7.2.10 — split by topic: the MAINTAINERS lines are carried by 900-config-tooling] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/usb/dwc3/Kconfig | 14 ++++++ drivers/usb/dwc3/Makefile | 1 + drivers/usb/dwc3/dwc3-apple-state-test.c | 56 ++++++++++++++++++++++++ drivers/usb/dwc3/dwc3-apple-state.h | 51 +++++++++++++++++++++ drivers/usb/dwc3/dwc3-apple.c | 32 ++++++-------- 5 files changed, 135 insertions(+), 19 deletions(-) create mode 100644 drivers/usb/dwc3/dwc3-apple-state-test.c create mode 100644 drivers/usb/dwc3/dwc3-apple-state.h diff --git a/drivers/usb/dwc3/Kconfig b/drivers/usb/dwc3/Kconfig index 18169727a413e..3e033ddc86d3d 100644 --- a/drivers/usb/dwc3/Kconfig +++ b/drivers/usb/dwc3/Kconfig @@ -223,6 +223,20 @@ config USB_DWC3_APPLE mode on these machines. Say 'Y' or 'M' if you have such device. +config USB_DWC3_APPLE_KUNIT_TEST + tristate "KUnit tests for the Apple Silicon DWC3 glue" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Test the helper with which the Apple Silicon DWC3 glue chooses + the USB2 PHY mode it selects before the core leaves reset: + device mode, never host mode, and nothing for other states. + The tests need no Apple hardware or platform support. They are + built under USB_DWC3, so UML without DMA emulation cannot run + them. + + If unsure, say N. + config USB_DWC3_GOOGLE tristate "Google Platform" help diff --git a/drivers/usb/dwc3/Makefile b/drivers/usb/dwc3/Makefile index f37971197203e..76578fc4a5da4 100644 --- a/drivers/usb/dwc3/Makefile +++ b/drivers/usb/dwc3/Makefile @@ -44,6 +44,7 @@ endif obj-$(CONFIG_USB_DWC3_AM62) += dwc3-am62.o obj-$(CONFIG_USB_DWC3_APPLE) += dwc3-apple.o +obj-$(CONFIG_USB_DWC3_APPLE_KUNIT_TEST) += dwc3-apple-state-test.o obj-$(CONFIG_USB_DWC3_OMAP) += dwc3-omap.o obj-$(CONFIG_USB_DWC3_EXYNOS) += dwc3-exynos.o obj-$(CONFIG_USB_DWC3_PCI) += dwc3-pci.o diff --git a/drivers/usb/dwc3/dwc3-apple-state-test.c b/drivers/usb/dwc3/dwc3-apple-state-test.c new file mode 100644 index 0000000000000..bf9a539a53e3f --- /dev/null +++ b/drivers/usb/dwc3/dwc3-apple-state-test.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +#include "dwc3-apple-state.h" + +/* Host mode is never selected before the reset; the xHCI root hub selects it. */ +static void dwc3_apple_host_left_to_xhci_test(struct kunit *test) +{ + enum phy_mode mode = PHY_MODE_USB_HOST; + + KUNIT_EXPECT_EQ(test, dwc3_apple_usb2_mode_before_reset(DWC3_APPLE_HOST, &mode), 0); + KUNIT_EXPECT_EQ(test, mode, PHY_MODE_INVALID); +} + +/* Device mode has no later selector and is selected before the reset. */ +static void dwc3_apple_device_selected_before_reset_test(struct kunit *test) +{ + enum phy_mode mode = PHY_MODE_INVALID; + + KUNIT_EXPECT_EQ(test, dwc3_apple_usb2_mode_before_reset(DWC3_APPLE_DEVICE, &mode), 0); + KUNIT_EXPECT_EQ(test, mode, PHY_MODE_USB_DEVICE); +} + +/* States that are not roles are rejected and select nothing. */ +static void dwc3_apple_non_role_test(struct kunit *test) +{ + static const enum dwc3_apple_state states[] = { + DWC3_APPLE_PROBE_PENDING, DWC3_APPLE_NO_CABLE, DWC3_APPLE_SUSPENDED, + }; + enum phy_mode mode; + int i; + + for (i = 0; i < ARRAY_SIZE(states); i++) { + mode = PHY_MODE_USB_HOST; + KUNIT_EXPECT_EQ(test, dwc3_apple_usb2_mode_before_reset(states[i], &mode), -EINVAL); + KUNIT_EXPECT_EQ(test, mode, PHY_MODE_INVALID); + } +} + +static struct kunit_case dwc3_apple_state_test_cases[] = { + KUNIT_CASE(dwc3_apple_host_left_to_xhci_test), + KUNIT_CASE(dwc3_apple_device_selected_before_reset_test), + KUNIT_CASE(dwc3_apple_non_role_test), + {} +}; + +static struct kunit_suite dwc3_apple_state_test_suite = { + .name = "dwc3_apple_state", + .test_cases = dwc3_apple_state_test_cases, +}; + +kunit_test_suite(dwc3_apple_state_test_suite); + +MODULE_DESCRIPTION("KUnit tests for the Apple Silicon DWC3 glue USB2 mode choice"); +MODULE_LICENSE("GPL"); diff --git a/drivers/usb/dwc3/dwc3-apple-state.h b/drivers/usb/dwc3/dwc3-apple-state.h new file mode 100644 index 0000000000000..89ce0b416bb40 --- /dev/null +++ b/drivers/usb/dwc3/dwc3-apple-state.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Apple Silicon DWC3 glue: controller states and the USB2 PHY role choice, + * kept in a header so that the choice can be tested without the hardware. + */ + +#ifndef __DWC3_APPLE_STATE_H +#define __DWC3_APPLE_STATE_H + +#include +#include + +/* See the comment at the top of dwc3-apple.c for how these states are used. */ +enum dwc3_apple_state { + DWC3_APPLE_PROBE_PENDING, /* Before first cable connection, dwc3_core_probe not called */ + DWC3_APPLE_NO_CABLE, /* No cable connected, dwc3 suspended after dwc3_core_exit */ + DWC3_APPLE_HOST, /* Cable connected, dwc3 in host mode */ + DWC3_APPLE_DEVICE, /* Cable connected, dwc3 in device mode */ + DWC3_APPLE_SUSPENDED, /* Fixed-hub host asleep: core exited, xhci still registered */ +}; + +/** + * dwc3_apple_usb2_mode_before_reset() - USB2 PHY mode to select before dwc3 leaves reset + * @target: the state the controller is brought up in + * @mode: set to the mode to select, or to PHY_MODE_INVALID to select none + * + * Device mode is selected before the reset is released, because nothing selects it + * later. Host mode is left to the xHCI root hub, which selects it once the core is out + * of reset: on a Type-C port the PHY is already running when the glue brings the core + * up, and switching it to host mode while the core is held in reset leaves the USB2 + * port without a connection for the whole session. + * + * Return: 0, or -EINVAL if @target is not DWC3_APPLE_HOST or DWC3_APPLE_DEVICE. + */ +static inline int dwc3_apple_usb2_mode_before_reset(enum dwc3_apple_state target, + enum phy_mode *mode) +{ + switch (target) { + case DWC3_APPLE_HOST: + *mode = PHY_MODE_INVALID; + return 0; + case DWC3_APPLE_DEVICE: + *mode = PHY_MODE_USB_DEVICE; + return 0; + default: + *mode = PHY_MODE_INVALID; + return -EINVAL; + } +} + +#endif /* __DWC3_APPLE_STATE_H */ diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index 2d0662ac77f72..da01a43877473 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -16,6 +16,7 @@ #include #include +#include "dwc3-apple-state.h" #include "glue.h" /* @@ -77,14 +78,9 @@ * Type-C mux for the SuperSpeed lanes, and the PHY follows those mode changes on its own. * Across system sleep such a controller goes through DWC3_APPLE_SUSPENDED: the core is exited * and initialised again around the sleep while xhci stays registered, see dwc3_apple_suspend(). + * + * enum dwc3_apple_state is defined in dwc3-apple-state.h. */ -enum dwc3_apple_state { - DWC3_APPLE_PROBE_PENDING, /* Before first cable connection, dwc3_core_probe not called */ - DWC3_APPLE_NO_CABLE, /* No cable connected, dwc3 suspended after dwc3_core_exit */ - DWC3_APPLE_HOST, /* Cable connected, dwc3 in host mode */ - DWC3_APPLE_DEVICE, /* Cable connected, dwc3 in device mode */ - DWC3_APPLE_SUSPENDED, /* Fixed-hub host asleep: core exited, xhci still registered */ -}; /** * struct dwc3_apple - Apple-specific DWC3 USB controller @@ -99,7 +95,7 @@ enum dwc3_apple_state { * @role_sw: USB role switch * @fixed_hub: A fixed hub sits on the controller's USB2 port, the controller stays in host mode * @lock: Mutex for synchronizing access - * @state: Current state of the controller, see documentation for the enum for details + * @state: Current state of the controller, see enum dwc3_apple_state and the comment above */ struct dwc3_apple { struct dwc3 dwc; @@ -265,6 +261,7 @@ static int dwc3_apple_core_init(struct dwc3_apple *appledwc) */ static int dwc3_apple_core_start(struct dwc3_apple *appledwc, enum dwc3_apple_state state) { + enum phy_mode mode; int ret, ret_reset; lockdep_assert_held(&appledwc->lock); @@ -284,19 +281,16 @@ static int dwc3_apple_core_start(struct dwc3_apple *appledwc, enum dwc3_apple_st * Both PHYs were looked up at probe, so device mode is also selected on the first * bring-up, before dwc3_core_probe() has looked them up itself. */ - switch (state) { - case DWC3_APPLE_HOST: - ret = 0; - break; - case DWC3_APPLE_DEVICE: - ret = phy_set_mode(appledwc->usb2_phy, PHY_MODE_USB_DEVICE); - break; - default: + ret = dwc3_apple_usb2_mode_before_reset(state, &mode); + if (ret) { /* Unreachable unless there's a bug in this driver */ - return -EINVAL; + return ret; + } + if (mode != PHY_MODE_INVALID) { + ret = phy_set_mode(appledwc->usb2_phy, mode); + if (ret) + dev_warn(appledwc->dev, "Failed to set the USB2 PHY mode, err=%d\n", ret); } - if (ret) - dev_warn(appledwc->dev, "Failed to set the USB2 PHY mode, err=%d\n", ret); ret = reset_control_deassert(appledwc->reset); if (ret) { From 536e9aac7fd0330952fbaffdb1f02fb30c4dc491 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Sun, 27 Sep 2026 12:40:33 +0200 Subject: [PATCH 0993/2680] phy: apple: atc: power the AUSPLL down when tearing the DP tunnel clock down Turning a USB4-tunneled display off and back on in the compositor left it in standby until a physical replug. atc_tunnel_restore() stopped PCLK1, cleared the PLL output drivers and replayed the saved descriptor registers, but never powered the AUSPLL down. The PLL kept running and ACIOPHY_AUSPLL_LOCK stayed set: the teardown logged "AUSPLL_LOCK did not clear after teardown (stat=0x8)", and the next atc_tunnel_start() preflight read +7044=00000008 and refused its own stale lock with -EBUSY. DCP then modeset into a tunnel with no pixel clock. A replug only recovered it because it resets the PHY. Clear the three PLL output drivers one at a time, then send APB command 3 through the same command/ACK handshake atc_tunnel_start() uses for commands 0 and 0x2000. This powers the AUSPLL down: the lock drops and the next preflight reads +7044=00000000. The saved descriptor registers are replayed afterwards, as before. The lock-clear poll stays as a check, since DCP retries a failed tunnel clock request quickly enough that the next request must not see a lock that is still dropping. Confirmed on hardware: M2 Pro (j416s), BenQ 1440p through a USB4 hub. At boot the preflight reads +7044=00000000 and the tunnel clock comes up (rate=0xa result=0). Disabling the output in Hyprland takes the crossbar link down with no "AUSPLL_LOCK did not clear" warning; enabling it 3s later preflights with +7044=00000000 again, gets rate=0xa result=0 and "crossbar link up", with no "AUSPLL power-down was not acked", and the picture comes back without a replug. Co-authored-by: Claude Opus 5.5 (cherry picked from commit dbaae5c0f6930405f1596deaef05198ad8d1642e) --- drivers/phy/apple/atc.c | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index c2b82cc7ee57c..1bcd2a210180f 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2297,22 +2297,28 @@ static void atc_tunnel_restore(struct apple_atcphy *atcphy) lockdep_assert_held(&atcphy->lock); if (!atcphy->tunnel_saved) return; - /* Stop our output before restoring the unused PLL descriptor. */ + /* + * Clearing the PLL output drivers alone leaves the AUSPLL running, + * with ACIOPHY_AUSPLL_LOCK still set. Clear the three drivers, then + * power the PLL down with APB command 3, after which the lock drops. + */ core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); - core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, 0x54); + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(2)); + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(4)); + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(6)); + ret = atc_tunnel_command(atcphy, 3); + if (ret) + dev_warn(atcphy->dev, "USB4 tunnel clock: AUSPLL power-down was not acked\n"); + /* Then restore the unused PLL descriptor. */ for (i = ARRAY_SIZE(atc_tunnel_regs) - 1; i >= 0; i--) core_mask32(atcphy, atc_tunnel_regs[i].reg, atc_tunnel_regs[i].mask, atcphy->tunnel_saved_regs[i] & atc_tunnel_regs[i].mask); /* * atc_tunnel_start_t602x()'s preflight refuses to proceed while - * ACIOPHY_AUSPLL_LOCK is still set, to avoid clobbering a PLL a - * concurrent user actually has locked. Immediately after tearing our - * own lock down, that same bit can still read set for a brief window - * before the lock detector catches up -- and DCP firmware retries a - * failed tunnel clock request roughly once a second, which is faster - * than that window on a hot retrain loop. Wait for it to clear here - * so a request coming in right after this returns sees a genuinely - * idle PLL instead of refusing itself forever. + * ACIOPHY_AUSPLL_LOCK is set, to avoid clobbering a PLL a concurrent + * user actually has locked. DCP firmware retries a failed tunnel + * clock request quickly, so confirm the lock has dropped after the + * power-down before a new request can come in. */ ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT, value, !(value & ACIOPHY_AUSPLL_LOCK), 10, 10000); From 6f6469f0541a66d5ff72c85ce3583f62e39a8fdf Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 29 Sep 2026 13:23:51 -0700 Subject: [PATCH 0994/2680] dt-bindings: thunderbolt: apple: describe the ACIO PCIe adapter The t8103 CIO nodes carry a third reg entry, reg-names "pcie-adapter", and apple,tunable-pcie-adapter, which the binding never described. Both controllers on j313 therefore failed validation four times over: reg and reg-names "too long", and the tunable rejected as undocumented. The third entry is t8103-only -- t8112 has two -- so reg and reg-names take minItems: 2 and the adapter is described as the optional third. While here, give thunderbolt-switch a type. It was declared as bare "true", which dtschema reports as a missing type definition; every user in tree spells it as a valueless boolean. Validated against a built t8103-j313.dtb: the eight acio errors and the thunderbolt-switch warning are gone, with no new diagnostics. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- .../thunderbolt/apple,t8103-usb4-acio.yaml | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml index e673942941aa6..33960878649c2 100644 --- a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml +++ b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml @@ -41,14 +41,21 @@ properties: - const: apple,t8103-usb4-acio reg: + minItems: 2 items: - description: Root complex and co-processor control registers - description: SRAM used for communication with the co-processor + - description: | + PCIe adapter configuration space of the USB4 router. Present where + the root complex exposes the adapter separately, so that the tunnel + can be configured before a PCIe-C link is brought up. reg-names: + minItems: 2 items: - const: rc - const: sram + - const: pcie-adapter mboxes: maxItems: 1 @@ -67,7 +74,11 @@ properties: apple,pcie-tunnel is present, the fourth domain powers its PCIe-C controller and is enabled only after the USB4 PCIe tunnel is active. - thunderbolt-switch: true + thunderbolt-switch: + type: boolean + description: + Marks this root complex as the USB4/Thunderbolt switch that the Type-C + PD controller hands cable information to, so the Type-C mux can find it. apple,pcie-tunnel: $ref: /schemas/types.yaml#/definitions/phandle @@ -87,6 +98,18 @@ properties: Tree, converted by the bootloader and used to configure the root complex after the co-processor has booted. + apple,tunable-pcie-adapter: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Mask of bits to clear + - description: Bits to set + description: | + As apple,tunable-rc, but applied to the PCIe adapter configuration space + named by the pcie-adapter reg entry. Only meaningful where that entry is + present. + '#address-cells': const: 1 From 77b11cbd82cca7079c51231ecbe41f1183b69ce8 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:40:24 -0700 Subject: [PATCH 0995/2680] thunderbolt: apple: drive t8103 Thunderbolt displays by default A display behind a Thunderbolt dock on t8103 stayed dark unless thunderbolt_apple.dp_display=1 was passed. The path it enables is the one this series has been tested on: on a 13" M1 MacBook Pro, CalDigit TS3 Plus, CalDigit TS4 and Kensington SD5560T docks drove 4K displays through hotplug, both ports and suspend-to-idle, and the aurora test builds have shipped it on by default. Default dp_display to true. Passing 0 still leaves the displays dark. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- Documentation/admin-guide/kernel-parameters.txt | 8 ++++---- drivers/thunderbolt/apple.c | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 0d0d644297c89..dd6af05c248f2 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -7569,10 +7569,10 @@ Kernel parameters [HW,ARM64] Format: Drive displays behind Thunderbolt DisplayPort tunnels - on Apple silicon (experimental, t8103 only): route a - display pipeline to the host router's DP IN adapter - whenever the connection manager sets up a DP tunnel. - Default: false + on Apple t8103 (M1): route a display pipeline to the + host router's DP IN adapter whenever the connection + manager sets up a DP tunnel. Set to 0 to leave displays + behind a dock dark. Default: true thunderbolt_apple.dpin_aux= [HW,ARM64] diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index d93cfdf3701d6..0097bb0bb84a3 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -259,9 +259,9 @@ struct apple_cio { #define APPLE_DPIN_CTRL_INACTIVE BIT(0) #define APPLE_DPIN_ACK_INACTIVE BIT(0) -static bool dp_display; +static bool dp_display = true; module_param(dp_display, bool, 0444); -MODULE_PARM_DESC(dp_display, "Drive displays behind Thunderbolt DP tunnels (experimental, t8103)"); +MODULE_PARM_DESC(dp_display, "Drive displays behind Thunderbolt DP tunnels on t8103 (default: true)"); /* DPTX_INACTIVE handshake: request (in)active, wait for the ACK */ struct apple_dpin_poll { From b373c49bf186f60d87d6535662083d71fcd46617 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:41:15 -0700 Subject: [PATCH 0996/2680] soc: apple: dp-tunnel: drive Thunderbolt displays on every M2 Pro and M2 Max laptop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The T602X display tunnel path ran only on the machine it was brought up on, the 16" M2 Pro (j416s): the DP IN handshake, the tunnel pixel clock, the longer link timeouts, the pipeline preference and the right-hand port's bandwidth grant were all keyed to "apple,j416s". The 14" M2 Pro (j414s), 14" M2 Max (j414c) and 16" M2 Max (j416c) use the same ATC PHYs ("apple,t6020-atcphy") and include the same t602x-j414-j416.dtsi, which carries the crossbar's typecN-usb4 tunnel routes and the USB-C port layout this code relies on. On those machines a display behind a dock stayed dark. Add apple_dp_tunnel_t602x() to the shared DP tunnel header, matching the four laptops, and use it for the ten j416s checks in appledrm, the ATC PHY and the Thunderbolt code. The M2 Pro and M2 Max desktops (j474s, j475c, j475d) are left out: their device tree has no crossbar tunnel routes yet. So far only j416s has run this path. [ryan: adapted to 7.2.10 — split by topic: the drm/apple hunks are carried by 200-dcp] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- .../admin-guide/kernel-parameters.txt | 10 ++++++---- drivers/phy/apple/atc.c | 5 ++--- drivers/thunderbolt/apple.c | 4 ++-- drivers/thunderbolt/tunnel.c | 18 ++++++++++-------- include/linux/soc/apple/dp-tunnel.h | 16 ++++++++++++++++ 5 files changed, 36 insertions(+), 17 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index dd6af05c248f2..5fa793939c975 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -7561,8 +7561,8 @@ Kernel parameters [HW,ARM64] Format: Enable diagnostic counter 0 on the DP video path's - host and downstream ingress hops on the Apple j416s - right-hand USB-C route. Read counters through Thunderbolt + host and downstream ingress hops on the right-hand + USB-C route of the Apple M2 Pro and M2 Max laptops. Read counters through Thunderbolt debugfs. This is a load-time setting. Default: false. thunderbolt_apple.dp_display= @@ -7578,7 +7578,8 @@ Kernel parameters [HW,ARM64] Format: Pulse the experimental DP IN analog start control on - Apple j416s at the next DP tunnel activation. Writes to + the Apple M2 Pro and M2 Max laptops at the next DP + tunnel activation. Writes to the module parameter do not access a live controller. Default: false (leave analog registers alone until DPRX negotiation finishes or times out). @@ -7586,7 +7587,8 @@ Kernel parameters thunderbolt_apple.dpin_mode_value= [HW,ARM64] Format: (0-15) - Experimental DP IN MODE_A/MODE_B value on Apple j416s. + Experimental DP IN MODE_A/MODE_B value on the Apple + M2 Pro and M2 Max laptops. Sampled for each adapter activation; values above 15 make activation fail. Default: 8. The value is empirical and does not select the negotiated DisplayPort rate. diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 1bcd2a210180f..81eac9b463885 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2716,10 +2716,9 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate) return -EINVAL; atcphy = phy_get_drvdata(phy); /* Keep each supported SoC on its qualified clock sequence. */ - if (!of_machine_is_compatible("apple,t8103") && - !of_machine_is_compatible("apple,j416s")) + if (!of_machine_is_compatible("apple,t8103") && !apple_dp_tunnel_t602x()) return -EOPNOTSUPP; - if (of_machine_is_compatible("apple,j416s") && + if (apple_dp_tunnel_t602x() && (!of_device_is_compatible(atcphy->np, "apple,t6020-atcphy") || !apple_atc_is_typec_core(atcphy->res.core->start) || resource_size(atcphy->res.core) < 0x7048)) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 0097bb0bb84a3..2e33b40a4e27d 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -361,7 +361,7 @@ static int apple_dpin_set_active_t8103(struct apple_cio *acio, void __iomem *dpi static int apple_dpin_set_active(struct apple_cio *acio, void __iomem *regs, unsigned int idx, bool active) { - if (of_machine_is_compatible("apple,j416s")) + if (apple_dp_tunnel_t602x()) return apple_dpin_set_active_t602x(acio, regs, idx, active); return apple_dpin_set_active_t8103(acio, regs, idx, active); } @@ -1698,7 +1698,7 @@ static int apple_nhi_probe(struct platform_device *pdev) anhi->ops = apple_nhi_ops; if (dp_display && acio->dp_wq && of_machine_is_compatible("apple,t8103")) anhi->ops.dp_tunnel_changed = apple_nhi_dp_tunnel_changed; - if (acio->dp_wq && of_machine_is_compatible("apple,j416s")) { + if (acio->dp_wq && apple_dp_tunnel_t602x()) { anhi->ops.dp_tunnel_pre_activate = apple_nhi_dp_tunnel_pre_activate; anhi->ops.dp_tunnel_post_activate = apple_nhi_dp_tunnel_post_activate; anhi->ops.dp_tunnel_deactivate = apple_nhi_dp_tunnel_deactivate; diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index 340134d8990e9..34bd70f92a3a0 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -14,6 +14,7 @@ #include #include #include +#include #include "tunnel.h" #include "tb.h" @@ -104,7 +105,7 @@ MODULE_PARM_DESC(bw_alloc_mode, static bool dp_video_counter; module_param(dp_video_counter, bool, 0444); MODULE_PARM_DESC(dp_video_counter, - "diagnostic: count packets on the DP video path's DP IN hop and its downstream (hub-side) hop (Apple j416s right ACIO route only; read via debugfs port counters); default: false"); + "diagnostic: count packets on the DP video path's DP IN hop and its downstream (hub-side) hop (Apple M2 Pro/Max laptop right ACIO route only; read via debugfs port counters); default: false"); static void tb_dp_dump_apple(struct tb_tunnel *tunnel); static int tb_apple_nhi_typec_index(struct tb_nhi *nhi); @@ -997,16 +998,17 @@ static int tb_dp_xchg_caps(struct tb_tunnel *tunnel) in->cap_adap + DP_REMOTE_CAP, 1); } -/* Apple j416s right-hand ACIO DP IN adapter only; shared by the diagnostic - * packet counter and the bandwidth-grant workaround below. +/* Right-hand ACIO DP IN adapter of the Apple M2 Pro and M2 Max laptops only; + * shared by the diagnostic packet counter and the bandwidth-grant workaround + * below. */ -static bool tb_dp_is_apple_j416s_right_dpin(const struct tb_port *in) +static bool tb_dp_is_apple_t602x_right_dpin(const struct tb_port *in) { if (!tb_port_is_dpin(in)) return false; if (!in->sw->tb || !tb_nhi_is_apple(in->sw->tb->nhi)) return false; - if (!of_machine_is_compatible("apple,j416s")) + if (!apple_dp_tunnel_t602x()) return false; /* Right-hand USB-C ports only ("f01f" NHI); see tb_apple_nhi_typec_index(). */ @@ -1015,7 +1017,7 @@ static bool tb_dp_is_apple_j416s_right_dpin(const struct tb_port *in) static bool tb_dp_apple_dpin_needs_bw_grant(const struct tb_port *in) { - return tb_dp_is_apple_j416s_right_dpin(in); + return tb_dp_is_apple_t602x_right_dpin(in); } static int tb_dp_bandwidth_alloc_mode_enable(struct tb_tunnel *tunnel) @@ -1115,7 +1117,7 @@ static int tb_dp_bandwidth_alloc_mode_enable(struct tb_tunnel *tunnel) /* * Initial allocation should be 0 according the spec, which relies * on the DP IN adapter later raising it via a bandwidth request - * notification. The Apple j416s right-hand ACIO DP IN adapter has + * notification. The Apple M2 Pro right-hand ACIO DP IN adapter has * never been observed to generate that notification: link training * and a full DCP frame complete with this field (DP_STATUS * allocated-bandwidth) still reading 0 and no picture. @@ -1775,7 +1777,7 @@ static bool tb_dp_video_counter_wanted(const struct tb_path *path) if (!dp_video_counter || !path->path_length) return false; - return tb_dp_is_apple_j416s_right_dpin(path->hops[0].in_port); + return tb_dp_is_apple_t602x_right_dpin(path->hops[0].in_port); } static int tb_dp_init_video_path(struct tb_path *path, bool pm_support) diff --git a/include/linux/soc/apple/dp-tunnel.h b/include/linux/soc/apple/dp-tunnel.h index 2767c16b127ff..8117ba0cea949 100644 --- a/include/linux/soc/apple/dp-tunnel.h +++ b/include/linux/soc/apple/dp-tunnel.h @@ -8,6 +8,7 @@ #ifndef _LINUX_SOC_APPLE_DP_TUNNEL_H_ #define _LINUX_SOC_APPLE_DP_TUNNEL_H_ +#include #include struct device_node; @@ -40,4 +41,19 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate); int apple_dpxbar_link_down(struct mux_control *mux); int apple_dpxbar_link_up(struct mux_control *mux); +/* + * The M2 Pro and M2 Max laptops. Their shared device tree routes the display + * crossbar to the Thunderbolt DP IN adapters, and they take the T602X tunnel + * path: DP IN handshake, tunnel pixel clock, and the longer link timeouts. + * The M2 Pro and M2 Max desktops have no such routes in their device tree. + */ +static inline bool apple_dp_tunnel_t602x(void) +{ + static const char *const machines[] = { + "apple,j414s", "apple,j414c", "apple,j416s", "apple,j416c", NULL, + }; + + return of_machine_compatible_match(machines); +} + #endif From 8b6c040d1eb6f445c3b2d34ab1e233c64e872163 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 24 Sep 2026 23:11:18 -0400 Subject: [PATCH 0997/2680] thunderbolt: apple: rearm the display route for a new DP tunnel Re-establish the DCP route when a new dock tunnel appears after a controller detach without a prior teardown callback: force the route through a down/up cycle instead of treating the old handoff as this one. Tested on J414s m2fix42: dock DP tunnel enabled USB-3 at 1920x1280@60, direct USB-2 at 3840x2160@120 and eDP at 3024x1964@120 with active CRTCs and zero failed units. Controller-detach disconnect reporting remains under investigation. [Chris Kearney: ported onto aurora-silicon/linux#8 from "arm64: enable J414s Thunderbolt DP tunnel and rearm display route". The J414s gates that commit added are covered by apple_dp_tunnel_t602x(), which also takes in j414c and j416c, so only the route rearm is carried here.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/thunderbolt/apple.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 2e33b40a4e27d..3660cc9f543b9 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -235,6 +235,7 @@ struct apple_cio { unsigned int idx; bool alive; /* tunnel up; cleared before it is torn down */ bool handed; /* appledrm has our callback; work only */ + bool rearm; /* a fresh tunnel needs a fresh DCP route */ } dpin[2]; }; @@ -486,12 +487,15 @@ static void apple_dpin_down(struct apple_dpin_ctx *c) static void apple_dpin_work_fn(struct work_struct *work) { struct apple_dpin_ctx *c = container_of(work, struct apple_dpin_ctx, work); - bool want; + bool want, rearm; int ret; for (;;) { - scoped_guard(mutex, &c->lock) + scoped_guard(mutex, &c->lock) { want = c->alive; + rearm = c->rearm; + c->rearm = false; + } if (!want) { if (c->handed || c->regs) { @@ -500,6 +504,13 @@ static void apple_dpin_work_fn(struct work_struct *work) } return; } + if (rearm && c->handed) { + dev_info(c->acio->dev, + "dpin%u: replacing stale display route for new tunnel\n", + c->idx); + apple_dpin_down(c); + continue; + } if (c->handed) return; @@ -1509,8 +1520,15 @@ static int apple_nhi_dp_tunnel_post_activate(struct tb_nhi *nhi, } else { struct apple_dpin_ctx *c = &anhi->acio->dpin[idx]; - scoped_guard(mutex, &c->lock) + /* + * A new tunnel can arrive without the old tunnel's teardown hook + * running on a controller detach. Force the DCP route through a + * down/up cycle instead of treating the old handoff as this one. + */ + scoped_guard(mutex, &c->lock) { c->alive = true; + c->rearm = true; + } queue_work(anhi->acio->dp_wq, &c->work); } apple_dp_dump_host_adapters(anhi); From 2092ada746acb0c14fffed44cfc316cedaa93921 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 26 Sep 2026 18:51:29 -0400 Subject: [PATCH 0998/2680] drm/apple: drive two dock DP tunnels independently MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Route DPIN1 through the native T602x crossbar fields and source selector, give the two tunnel streams separate DCP connector/route ownership, and keep both ATC pixel clocks alive at independent link rates. Wait for Type-C IOMFB hotplug after a display power-on. The dual-dock path was exercised on J414s/m2fix48 with Dell 3840x2160@120 and JTG 1920x1280@60, per-output DPMS and idle/wake; the user confirmed the displayed image. This commit removes a frame-complete callback that had no register side effect. The cleaned tree builds with the Asahi profile; the cleaned commit itself is not yet deployed. [Chris Kearney: ported onto aurora-silicon/linux#8 from "drm/apple: drive two J414s dock DP tunnels independently": - The J414s checks go through apple_dp_tunnel_t602x(), so all four M2 Pro/Max laptops get the second connector per port and the per-DP-IN pipeline preference (DPIN0 dcpext0, DPIN1 dcpext1). Oliver Lukschander ran two displays through one hub on j416s with the same gates. - The per-DP-IN ATC clock tracking replaces #8's single-rate T602X restart: a rate change reselects that DP IN's PCLK. A DPMS stop gates the outputs and keeps the PLL; a PHY mode change still powers the AUSPLL down in atc_tunnel_restore(), which now also forgets the users. The t8103 tunnel clock path is unchanged. - Only the T602X crossbar preselects a DP IN's source; t8103 tunnels still select it at link-up, so -EOPNOTSUPP from the selector is not an error.] [ryan: adapted to 7.2.10 — split by topic: only the drivers/mux, drivers/phy/apple, include/linux/soc/apple/dp-tunnel.h hunks of this dcp-display commit; the rest is in 200-dcp] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/mux/apple-display-crossbar.c | 131 +++++++++++++++------ drivers/phy/apple/atc.c | 170 +++++++++++++++++++++------ include/linux/soc/apple/dp-tunnel.h | 3 +- 3 files changed, 234 insertions(+), 70 deletions(-) diff --git a/drivers/mux/apple-display-crossbar.c b/drivers/mux/apple-display-crossbar.c index e7a8470aa7c79..9a50dc7eb70b9 100644 --- a/drivers/mux/apple-display-crossbar.c +++ b/drivers/mux/apple-display-crossbar.c @@ -29,6 +29,7 @@ #define T602X_FIFO_WR_N_CLK_EN 0x004 #define T602X_FIFO_WR_UNK_EN 0x008 #define T602X_REG_00C 0x00c +#define T602X_REG_010 0x010 #define T602X_REG_014 0x014 #define T602X_REG_018 0x018 #define T602X_REG_01C 0x01c @@ -247,23 +248,27 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) dpxbar_clear32(dpxbar, T602X_FIFO_RD_UNK_EN, prev_dispext_bit); dpxbar_clear32(dpxbar, T602X_FIFO_WR_DPTX_CLK_EN, prev_dispext_bit); - /* DPIN0 uses the same single-bit source gate on up and down. */ - dpxbar_clear32(dpxbar, T602X_REG_00C, - index == MUX_DPIN0 ? prev_dispext_bit : prev_dispext_bit_en); + dpxbar_clear32(dpxbar, + index == MUX_DPIN1 ? T602X_REG_010 : T602X_REG_00C, + prev_dispext_bit); - dpxbar_clear32(dpxbar, T602X_REG_01C, atc_bit); + dpxbar_clear32(dpxbar, + index == MUX_DPIN1 ? FIFO_RD_PCLK1_EN : T602X_REG_01C, + atc_bit); dpxbar_clear32(dpxbar, T602X_FIFO_WR_UNK_EN, prev_dispext_bit); - dpxbar_clear32(dpxbar, T602X_REG_018, prev_dispext_bit_en); + dpxbar_clear32(dpxbar, T602X_REG_018, + prev_dispext_bit_en * + (index == MUX_DPIN1 ? 3 : 1)); - dpxbar_clear32(dpxbar, T602X_FIFO_RD_N_CLK_EN, atc_bit); + dpxbar_clear32(dpxbar, T602X_FIFO_RD_N_CLK_EN, + atc_bit * (index == MUX_DPIN1 ? 3 : 1)); dpxbar_set32(dpxbar, T602X_FIFO_WR_N_CLK_EN, prev_dispext_bit); dpxbar_set32(dpxbar, T602X_REG_014, 0x4); - /* Restore the selected input's FIFO read reset. */ - dpxbar_set32(dpxbar, index == MUX_DPIN0 ? - T602X_FIFO_RD_PCLK2_EN : FIFO_RD_PCLK1_EN, atc_bit); + /* Native T602x DPIN0 teardown restores the read reset at +0x24. */ + dpxbar_set32(dpxbar, T602X_FIFO_RD_PCLK2_EN, atc_bit); dpxbar_clear32(dpxbar, T602X_REG_034, atc_bit); dpxbar_clear32(dpxbar, CROSSBAR_ATC_EN, atc_bit); @@ -280,16 +285,37 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) dpxbar_clear32(dpxbar, T602X_FIFO_WR_N_CLK_EN, dispext_bit); dpxbar_clear32(dpxbar, T602X_REG_014, 0x4); + /* Match the DPIN1 sequence used by link_up(): clear +0x24 + * before setting the separate +0x20 field below. + */ dpxbar_clear32(dpxbar, T602X_FIFO_RD_PCLK2_EN, atc_bit); + if (index == MUX_DPIN1) + dpxbar_clear32(dpxbar, FIFO_RD_PCLK1_EN, atc_bit); dpxbar_set32(dpxbar, T602X_FIFO_WR_UNK_EN, dispext_bit); - dpxbar_set32(dpxbar, T602X_REG_018, dispext_bit_en); - - dpxbar_set32(dpxbar, T602X_FIFO_RD_N_CLK_EN, atc_bit); + /* Native T602x encodes DPIN1's role as 3 in both two-bit + * source and sink fields, versus 1 for DPIN0. + */ + if (index == MUX_DPIN1) { + dpxbar_mask32(dpxbar, T602X_REG_018, + dispext_bit_en * 3, + dispext_bit_en * 3); + dpxbar_mask32(dpxbar, T602X_FIFO_RD_N_CLK_EN, + atc_bit * 3, atc_bit * 3); + } else { + dpxbar_set32(dpxbar, T602X_REG_018, + dispext_bit_en); + dpxbar_set32(dpxbar, T602X_FIFO_RD_N_CLK_EN, + atc_bit); + } dpxbar_set32(dpxbar, T602X_FIFO_WR_DPTX_CLK_EN, dispext_bit); - dpxbar_set32(dpxbar, T602X_REG_00C, dispext_bit); + dpxbar_set32(dpxbar, + index == MUX_DPIN1 ? T602X_REG_010 : T602X_REG_00C, + dispext_bit); - dpxbar_set32(dpxbar, T602X_REG_01C, atc_bit); + dpxbar_set32(dpxbar, + index == MUX_DPIN1 ? FIFO_RD_PCLK1_EN : T602X_REG_01C, + atc_bit); dpxbar_set32(dpxbar, T602X_REG_034, atc_bit); dpxbar_set32(dpxbar, CROSSBAR_ATC_EN, atc_bit); /* @@ -583,12 +609,48 @@ static const struct mux_control_ops apple_dpxbar_t602x_ops = { .set = apple_dpxbar_set_t602x, }; +/* + * Select the DCP source for a USB4 DP IN before DCP probes AUX. Do not + * enable any FIFO clock or ATC output here: full activation still waits for + * DCP's SetLinkRate and the tunnel pixel clock. dispext0 works with the + * reset selector value (0); dispext1 needs this explicit preselection. + */ +int apple_dpxbar_tunnel_select_source(struct mux_control *mux, int state) +{ + struct apple_dpxbar *xbar; + unsigned long flags; + unsigned int index; + int ret = 0; + + if (!mux || mux->chip->ops != &apple_dpxbar_t602x_ops) + return -EOPNOTSUPP; + index = mux_control_get_index(mux); + if (index != MUX_DPIN0 && index != MUX_DPIN1) + return -EINVAL; + if (state < -1 || state >= 9) + return -EINVAL; + + xbar = mux_chip_priv(mux->chip); + spin_lock_irqsave(&xbar->lock, flags); + if (xbar->selected_dispext[index] >= 0 && + xbar->selected_dispext[index] != state) { + ret = -EBUSY; + } else { + dpxbar_mask32(xbar, T602X_REG_030, t602x_mux_mask(index), + state < 0 ? 0 : t602x_mux_set(index, state)); + } + spin_unlock_irqrestore(&xbar->lock, flags); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_dpxbar_tunnel_select_source); + static int apple_dpxbar_t602x_link_up(struct mux_control *mux) { struct apple_dpxbar *xbar; unsigned int index; unsigned long flags; - u32 dispext_bit, atc_bit; + u32 dispext_bit, dispext_bit_en, atc_bit; int state, ret = 0; if (!mux || mux->chip->ops != &apple_dpxbar_t602x_ops) @@ -606,31 +668,32 @@ static int apple_dpxbar_t602x_link_up(struct mux_control *mux) return -ENODEV; } dispext_bit = 1 << state; + dispext_bit_en = 1 << (2 * state); dpxbar_clear32(xbar, T602X_FIFO_WR_N_CLK_EN, dispext_bit); dpxbar_clear32(xbar, T602X_REG_014, dispext_bit); - dpxbar_clear32(xbar, index == MUX_DPIN0 ? T602X_FIFO_RD_PCLK2_EN : - FIFO_RD_PCLK1_EN, atc_bit); + dpxbar_clear32(xbar, T602X_FIFO_RD_PCLK2_EN, atc_bit); udelay(1); dpxbar_set32(xbar, T602X_FIFO_WR_UNK_EN, dispext_bit); - dpxbar_mask32(xbar, T602X_REG_018, GENMASK(5, 4), BIT(4)); - /* - * The only hardware-verified value for this 2-bit field is dpin0's - * BIT(0), which is both ATC_DPIN0 and MUX_DPIN0's own enum value (1) - * coincidentally overlapping at bit 0 -- so this field's encoding - * for a second index was never independently confirmed on real - * hardware. FIELD_PREP(..., index), matching - * MUX_DPPHY=0/MUX_DPIN0=1/MUX_DPIN1=2, is an inferred generalization - * of that single known-good dpin0 data point, not a value verified - * against dpin1 hardware. If dpin1 misbehaves, this field is a - * prime suspect. + if (index == MUX_DPIN1) { + dpxbar_mask32(xbar, T602X_REG_018, + dispext_bit_en * 3, dispext_bit_en * 3); + dpxbar_mask32(xbar, T602X_FIFO_RD_N_CLK_EN, + atc_bit * 3, atc_bit * 3); + } else { + dpxbar_set32(xbar, T602X_REG_018, dispext_bit_en); + } + /* set_t602x() already enabled this output's N clock. In particular, + * do not rewrite the low bits here: DPIN1 would clear DPIN0's clock. */ - dpxbar_mask32(xbar, T602X_FIFO_RD_N_CLK_EN, GENMASK(1, 0), - FIELD_PREP(GENMASK(1, 0), index)); dpxbar_set32(xbar, T602X_FIFO_WR_DPTX_CLK_EN, dispext_bit); - dpxbar_set32(xbar, T602X_REG_00C, dispext_bit); - dpxbar_set32(xbar, T602X_REG_01C, atc_bit); + dpxbar_set32(xbar, + index == MUX_DPIN1 ? T602X_REG_010 : T602X_REG_00C, + dispext_bit); + dpxbar_set32(xbar, + index == MUX_DPIN1 ? FIFO_RD_PCLK1_EN : T602X_REG_01C, + atc_bit); dpxbar_set32(xbar, T602X_REG_034, atc_bit); dpxbar_set32(xbar, CROSSBAR_ATC_EN, atc_bit); dpxbar_set32(xbar, CROSSBAR_DISPEXT_EN, dispext_bit); @@ -675,11 +738,9 @@ static int apple_dpxbar_t602x_link_down(struct mux_control *mux) return -ENODEV; } dispext_bit = 1 << state; - dpxbar_set32(xbar, T602X_FIFO_WR_N_CLK_EN, dispext_bit); dpxbar_set32(xbar, T602X_REG_014, dispext_bit); - dpxbar_set32(xbar, index == MUX_DPIN0 ? T602X_FIFO_RD_PCLK2_EN : - FIFO_RD_PCLK1_EN, atc_bit); + dpxbar_set32(xbar, T602X_FIFO_RD_PCLK2_EN, atc_bit); spin_unlock_irqrestore(&xbar->lock, flags); dev_info(xbar->dev, "%s: crossbar link down (dispext=%d atc=0x%x)\n", diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 81eac9b463885..fdcc2427c992b 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -663,7 +663,9 @@ struct atcphy_hw { * @tunnel_attempted: A T602X tunnel clock setup has been attempted * @tunnel_saved: T602X PHY registers have been saved for tunnel teardown * @tunnel_saved_regs: Original values of the T602X tunnel clock registers - * @tunnel_rate: DP link rate code the tunnel pixel clock is set up for + * @tunnel_rate: DP link rate code the t8103 tunnel pixel clock is set up for + * @tunnel_users: T602X DP IN adapters (BIT(dpin)) whose tunnel pixel clock runs + * @tunnel_dpin_rate: DP link rate code each T602X DP IN adapter's clock runs at * @regs: Memory-mapped registers * @regs.core: Core registers * @regs.axi2af: AXI to Apple Fabric interface registers @@ -712,6 +714,8 @@ struct apple_atcphy { bool tunnel_attempted, tunnel_saved; u32 tunnel_saved_regs[12]; u8 tunnel_rate; + u8 tunnel_users; + u8 tunnel_dpin_rate[2]; struct { void __iomem *core; @@ -2262,7 +2266,8 @@ struct atc_tunnel_saved_reg { static const struct atc_tunnel_saved_reg atc_tunnel_regs[] = { { ACIOPHY_CFG0, 0x0003ffff }, { ACIOPHY_SLEEP_CTRL, 0x00000fff }, - { ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, 0x0000207c }, + { ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + 0x0000207c | DPTX_PCLK2_SELECT | DPTX_PCLK2_ENABLE }, { AUSPLL_FREQ_CFG, 0x00000003 }, { AUSPLL_FREQ_DESC_A, 0xffffffff }, { AUSPLL_FREQ_DESC_B, 0x0fffffff }, @@ -2295,6 +2300,9 @@ static void atc_tunnel_restore(struct apple_atcphy *atcphy) int i, ret; lockdep_assert_held(&atcphy->lock); + atcphy->tunnel_users = 0; + atcphy->tunnel_dpin_rate[0] = 0; + atcphy->tunnel_dpin_rate[1] = 0; if (!atcphy->tunnel_saved) return; /* @@ -2302,7 +2310,8 @@ static void atc_tunnel_restore(struct apple_atcphy *atcphy) * with ACIOPHY_AUSPLL_LOCK still set. Clear the three drivers, then * power the PLL down with APB command 3, after which the lock drops. */ - core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_ENABLE | DPTX_PCLK2_ENABLE); core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(2)); core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(4)); core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(6)); @@ -2331,35 +2340,36 @@ static void atc_tunnel_restore(struct apple_atcphy *atcphy) atcphy->tunnel_rate = 0; } -static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy, u8 rate) +static int atc_tunnel_rate_selector(u8 rate) { - u32 selector, value, gates, outputs, command, status; - unsigned int i; - int ret; - - lockdep_assert_held(&atcphy->lock); switch (rate) { case 0x06: - selector = 4; - break; + return 4; case 0x0a: - selector = 3; - break; + return 3; case 0x14: - selector = 1; - break; + return 1; case 0x1e: - selector = 0; - break; + return 0; default: return -EINVAL; } - if (atcphy->tunnel_saved) { - if (atcphy->tunnel_rate == rate) - return 0; - /* A retrain may change rate without first requesting rate zero. */ - atc_tunnel_restore(atcphy); - } +} + +/* + * Bring the T602X tunnel PLL up with its fixed descriptor. The rate is not + * part of it: each DP IN adapter selects its own rate on PCLK1 or PCLK2 in + * apple_atc_dp_tunnel_rate(). + */ +static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy) +{ + u32 value, gates, outputs, command, status; + unsigned int i; + int ret; + + lockdep_assert_held(&atcphy->lock); + if (atcphy->tunnel_saved) + return 0; if (atcphy->tunnel_attempted) return -EALREADY; /* Read every guard before deciding, so a refusal records the whole state. */ @@ -2426,9 +2436,6 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy, u8 rate) udelay(2); core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, BIT(2)); core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, BIT(3)); - core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); - core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_SELECT, - FIELD_PREP(DPTX_PCLK1_SELECT, selector)); /* Program the fixed tunnel clock descriptor. */ core_clear32(atcphy, AUSPLL_FREQ_CFG, AUSPLL_FREQ_REFCLK); @@ -2454,13 +2461,106 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy, u8 rate) ret = atc_tunnel_command(atcphy, 0x2000); if (ret) goto restore; - atcphy->tunnel_rate = rate; return 0; restore: atc_tunnel_restore(atcphy); return ret; } +/* Stop one T602X DP IN adapter's pixel clock; atcphy->lock held. */ +static void atc_tunnel_stop_t602x(struct apple_atcphy *atcphy, unsigned int dpin) +{ + u8 user = BIT(dpin); + + lockdep_assert_held(&atcphy->lock); + if (atcphy->tunnel_users & user) { + if (dpin == 0) { + /* + * DPIN1 still needs the PCLK1 gate. On J414s, clearing + * it stopped DPIN1's video packets even though PCLK2 + * remained enabled and locked. + */ + if (!(atcphy->tunnel_users & BIT(1))) + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_ENABLE); + } else { + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK2_ENABLE); + if (!(atcphy->tunnel_users & BIT(0))) + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_ENABLE); + } + } + atcphy->tunnel_users &= ~user; + atcphy->tunnel_dpin_rate[dpin] = 0; + /* + * Gate the PLL outputs but keep the descriptor across DPMS: the PLL + * keeps its lock with its outputs gated, and the next start re-enables + * them. A mode change still powers it down in atc_tunnel_restore(). + */ + if (!atcphy->tunnel_users && atcphy->tunnel_saved) + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, 0x54); +} + +/* Run one T602X DP IN adapter's pixel clock at @rate; atcphy->lock held. */ +static int atc_tunnel_set_t602x(struct apple_atcphy *atcphy, unsigned int dpin, + u8 rate) +{ + int selector, ret = 0; + u32 value; + + lockdep_assert_held(&atcphy->lock); + selector = atc_tunnel_rate_selector(rate); + if (selector < 0) + return selector; + if (atcphy->tunnel_dpin_rate[dpin] == rate) + return 0; + if (!atcphy->tunnel_users) { + if (!atcphy->tunnel_saved) { + ret = atc_tunnel_start_t602x(atcphy); + } else { + /* + * The PLL retains lock after its outputs are gated. Keep + * its descriptor until the USB4 PHY changes mode instead of + * reprogramming a still-locked PLL on every DPMS wake. + */ + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, 0x54); + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT, + value, value & ACIOPHY_AUSPLL_LOCK, + 10, 10000); + if (ret) + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, 0x54); + } + if (ret) + return ret; + } + if (dpin == 0) { + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_SELECT, FIELD_PREP(DPTX_PCLK1_SELECT, selector)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_ENABLE); + } else { + /* + * PCLK2 alone does not sustain a DPIN1 stream on J414s. Keep + * PCLK1 gated on as a shared prerequisite; if DPIN0 is inactive, + * select DPIN1's rate for that clock as well. + */ + if (!(atcphy->tunnel_users & BIT(0))) + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_SELECT, + FIELD_PREP(DPTX_PCLK1_SELECT, selector)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_ENABLE); + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK2_SELECT, FIELD_PREP(DPTX_PCLK2_SELECT, selector)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK2_ENABLE); + } + atcphy->tunnel_dpin_rate[dpin] = rate; + atcphy->tunnel_users |= BIT(dpin); + return 0; +} + static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) { int ret = 0; @@ -2698,8 +2798,8 @@ static const struct phy_ops apple_atc_dp_phy_ops = { }; /* - * Called by appledrm when DCP sets the link rate of a DPTX that feeds a - * Thunderbolt DP IN adapter (rate is the DP link rate code, 0 = stop). + * Called by appledrm when DCP sets the link rate of a DPTX that feeds + * Thunderbolt DP IN adapter @dpin (rate is the DP link rate code, 0 = stop). */ static bool apple_atc_is_typec_core(u64 base) { @@ -2707,12 +2807,12 @@ static bool apple_atc_is_typec_core(u64 base) base == 0xf03000000ULL; } -int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate) +int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate) { struct apple_atcphy *atcphy; int ret; - if (!phy || phy->ops != &apple_atc_dp_phy_ops) + if (!phy || phy->ops != &apple_atc_dp_phy_ops || dpin > 1) return -EINVAL; atcphy = phy_get_drvdata(phy); /* Keep each supported SoC on its qualified clock sequence. */ @@ -2725,15 +2825,17 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate) return -EINVAL; guard(mutex)(&atcphy->lock); if (!rate) { - atc_tunnel_stop_t8103(atcphy); - atc_tunnel_restore(atcphy); + if (of_machine_is_compatible("apple,t8103")) + atc_tunnel_stop_t8103(atcphy); + else + atc_tunnel_stop_t602x(atcphy, dpin); return 0; } if (atcphy->mode != APPLE_ATCPHY_MODE_USB4 && atcphy->mode != APPLE_ATCPHY_MODE_TBT) return -EBUSY; ret = of_machine_is_compatible("apple,t8103") ? atc_tunnel_start_t8103(atcphy, rate) : - atc_tunnel_start_t602x(atcphy, rate); + atc_tunnel_set_t602x(atcphy, dpin, rate); dev_dbg(atcphy->dev, "DP tunnel clock rate 0x%x: %d (TX_DP_CTRL0=%08x PCLK_STAT=%08x)\n", rate, ret, readl(atcphy->regs.core + ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0), readl(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT)); diff --git a/include/linux/soc/apple/dp-tunnel.h b/include/linux/soc/apple/dp-tunnel.h index 8117ba0cea949..4d2ab4eeb07da 100644 --- a/include/linux/soc/apple/dp-tunnel.h +++ b/include/linux/soc/apple/dp-tunnel.h @@ -29,7 +29,7 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, * ATC PHY: start the pixel clock for a DP tunnel at DP link rate code @rate, * or stop it (@rate == 0). The PHY must be in Thunderbolt/USB4 mode. */ -int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate); +int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate); /* * Display crossbar: take the connection of an already selected output down or @@ -40,6 +40,7 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate); */ int apple_dpxbar_link_down(struct mux_control *mux); int apple_dpxbar_link_up(struct mux_control *mux); +int apple_dpxbar_tunnel_select_source(struct mux_control *mux, int state); /* * The M2 Pro and M2 Max laptops. Their shared device tree routes the display From c103ecea33b99e167ef48bf2c43eb8d21f7ce6b3 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 26 Sep 2026 18:59:19 -0400 Subject: [PATCH 0999/2680] thunderbolt: account live USB3 use on Apple hosts without BW allocation Apple host USB3 adapters do not acknowledge bandwidth-allocation writes, so their software-only initial allocation should not be treated as consumed USB3 bandwidth when admitting DP tunnels. Read the downstream router USB3 UP adapter consumed-bandwidth counters instead, retaining 900 Mb/s minimum headroom for isochronous traffic. If those counters cannot be read, keep the existing conservative allocation estimate. The change does not cap USB3 bulk traffic or write the unsupported host registers. The m2fix48 dock test reached Dell 4K120 and JTG 60 with USB devices enumerated; a standalone throughput or isochronous stress test has not been performed. [Chris Kearney: applies to every Apple host, since apple.c sets QUIRK_NO_USB3_BW_ALLOC on all of them, the t8103 included.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/thunderbolt/tb.h | 2 ++ drivers/thunderbolt/tunnel.c | 31 +++++++++++++++++++++++++++++-- drivers/thunderbolt/usb4.c | 11 +++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/drivers/thunderbolt/tb.h b/drivers/thunderbolt/tb.h index a02920275e30e..976f77907181c 100644 --- a/drivers/thunderbolt/tb.h +++ b/drivers/thunderbolt/tb.h @@ -1484,6 +1484,8 @@ int usb4_port_retimer_nvm_read(struct tb_port *port, u8 index, unsigned int address, void *buf, size_t size); int usb4_usb3_port_max_link_rate(struct tb_port *port); +int usb4_usb3_port_consumed_bandwidth(struct tb_port *port, int *upstream_bw, + int *downstream_bw); int usb4_usb3_port_allocated_bandwidth(struct tb_port *port, int *upstream_bw, int *downstream_bw); int usb4_usb3_port_allocate_bandwidth(struct tb_port *port, int *upstream_bw, diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index 34bd70f92a3a0..7c4f39e051e33 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -2421,14 +2421,41 @@ static int tb_usb3_consumed_bandwidth(struct tb_tunnel *tunnel, { struct tb_port *port = tb_upstream_port(tunnel->dst_port->sw); int pcie_weight = tb_acpi_may_tunnel_pcie() ? TB_PCI_WEIGHT : 0; + int allocated_up = tunnel->allocated_up; + int allocated_down = tunnel->allocated_down; + int ret; + + if (tunnel->src_port->sw->no_usb3_bw_alloc) { + /* + * Apple's host USB3 adapter cannot acknowledge bandwidth + * allocation requests. Its software-only initial allocation is a + * maximum, not bandwidth actually occupied by USB3. The device + * router's USB3 UP adapter still reports consumed bandwidth; use + * that live value for DP admission without limiting USB3 bulk + * traffic or writing the unsupported host registers. + */ + ret = usb4_usb3_port_consumed_bandwidth(tunnel->dst_port, + &allocated_up, &allocated_down); + if (ret) { + /* Preserve the conservative allocation-based estimate when + * a downstream adapter lacks readable consumed counters. + */ + allocated_up = tunnel->allocated_up; + allocated_down = tunnel->allocated_down; + } else { + /* Leave headroom for new isochronous USB3 transfers. */ + allocated_up = max(allocated_up, 900); + allocated_down = max(allocated_down, 900); + } + } /* * PCIe tunneling, if enabled, affects the USB3 bandwidth so * take that into account here. */ - *consumed_up = tunnel->allocated_up * + *consumed_up = allocated_up * (TB_USB3_WEIGHT + pcie_weight) / TB_USB3_WEIGHT; - *consumed_down = tunnel->allocated_down * + *consumed_down = allocated_down * (TB_USB3_WEIGHT + pcie_weight) / TB_USB3_WEIGHT; if (tb_port_get_link_generation(port) >= 4) { diff --git a/drivers/thunderbolt/usb4.c b/drivers/thunderbolt/usb4.c index 5fd4fe070f258..dc5dc17498b86 100644 --- a/drivers/thunderbolt/usb4.c +++ b/drivers/thunderbolt/usb4.c @@ -2365,6 +2365,17 @@ static int usb4_usb3_port_read_consumed_bandwidth(struct tb_port *port, return 0; } +int usb4_usb3_port_consumed_bandwidth(struct tb_port *port, int *upstream_bw, + int *downstream_bw) +{ + if (!tb_port_is_usb3_up(port)) + return -EINVAL; + + /* Device-router USB3 UP adapters expose these counters without CMR. */ + return usb4_usb3_port_read_consumed_bandwidth(port, upstream_bw, + downstream_bw); +} + static int usb4_usb3_port_write_allocated_bandwidth(struct tb_port *port, int upstream_bw, int downstream_bw) From e6e1a5d27d56bb71ba612b48572de0f9d53ad8b3 Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Wed, 30 Sep 2026 09:54:14 -0600 Subject: [PATCH 1000/2680] mux: apple-display-crossbar: let a DP IN output be pointed at its source early A Thunderbolt DP tunnel on t8103 leaves the crossbar's DP IN output at its idle source, dispext0, until DCP reports DidChangeLinkConfig. That works there because dispext0 is the only external pipeline. On t600x the Type-C ports are driven by dcpext1 (mux index 2), and a pipeline whose output still points at dispext0 activates the link, gets no answer from the sink and gives up before it sets a link rate. Add apple_dpxbar_preselect(), which writes only the source select of a DP IN output that is not selected yet. The connection itself still comes up in the normal mux selection. MUX_IDLE_DISCONNECT puts the output back at its idle source. Also skip the FIFO_RD_PCLK1 cycle-slip toggle for DP IN on t6000, which runs without it, as found on j316c in omacom/linux#9. The DP PHY output keeps the toggle. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Christian Bartels Co-Authored-By: Claude Opus 5.5 [Chris Kearney: ported onto aurora-silicon/linux#8; comment and message reworded to describe the observed behaviour only.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/mux/apple-display-crossbar.c | 78 +++++++++++++++++++++++++--- include/linux/soc/apple/dp-tunnel.h | 8 +++ 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/drivers/mux/apple-display-crossbar.c b/drivers/mux/apple-display-crossbar.c index 9a50dc7eb70b9..f49977b846f04 100644 --- a/drivers/mux/apple-display-crossbar.c +++ b/drivers/mux/apple-display-crossbar.c @@ -93,10 +93,13 @@ struct apple_dpxbar_hw { unsigned int n_ufp; u32 tunable; const struct mux_control_ops *ops; + /* DP IN runs without the FIFO_RD_PCLK1 cycle-slip toggle (t600x) */ + bool dpin_no_cycle_slip; }; struct apple_dpxbar { struct device *dev; + const struct apple_dpxbar_hw *hw; void __iomem *regs; int selected_dispext[MUX_MAX]; spinlock_t lock; @@ -361,6 +364,12 @@ static void apple_dpxbar_check_gates(struct apple_dpxbar *dpxbar, unsigned int i apple_dpxbar_names[index], wr, rd, out); } +/* The RD_PCLK toggle below is needed for the PHY output, and for DP IN on some SoCs. */ +static bool apple_dpxbar_cycle_slip(struct apple_dpxbar *dpxbar, unsigned int index) +{ + return index == MUX_DPPHY || !dpxbar->hw->dpin_no_cycle_slip; +} + static int apple_dpxbar_set(struct mux_control *mux, int state) { struct apple_dpxbar *dpxbar = mux_chip_priv(mux->chip); @@ -479,9 +488,11 @@ static int apple_dpxbar_set(struct mux_control *mux, int state) * 5 usec is required which is doubled here to be on the * safe side. */ - dpxbar_clear32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); - udelay(10); - dpxbar_set32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + if (apple_dpxbar_cycle_slip(dpxbar, index)) { + dpxbar_clear32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + udelay(10); + dpxbar_set32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + } dpxbar->selected_dispext[index] = state; } @@ -585,9 +596,11 @@ static int apple_dpxbar_t8103_link_up(struct mux_control *mux) dpxbar_set32(dpxbar, OUT_PCLK1_EN, atc_bit); dpxbar_set32(dpxbar, CROSSBAR_ATC_EN, atc_bit); dpxbar_set32(dpxbar, CROSSBAR_DISPEXT_EN, dispext_bit); - dpxbar_clear32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); - udelay(10); - dpxbar_set32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + if (apple_dpxbar_cycle_slip(dpxbar, index)) { + dpxbar_clear32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + udelay(10); + dpxbar_set32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + } } spin_unlock_irqrestore(&dpxbar->lock, flags); if (ret) @@ -601,6 +614,57 @@ static int apple_dpxbar_t8103_link_up(struct mux_control *mux) return 0; } +int apple_dpxbar_preselect(struct mux_control *mux, int state) +{ + struct apple_dpxbar *dpxbar; + unsigned int index, mux_state; + unsigned long flags; + u32 mask, set; + int ret = 0; + + if (!mux) + return -EINVAL; + dpxbar = apple_dpxbar_from_mux(mux, &index); + if (!dpxbar) + return -EOPNOTSUPP; + if (state == MUX_IDLE_DISCONNECT) + mux_state = 0; + else if (state >= 0 && state < 9) + mux_state = state; + else + return -EINVAL; + + switch (index) { + case MUX_DPIN0: + mask = CROSSBAR_MUX_CTRL_DPIN0_SELECT0 | CROSSBAR_MUX_CTRL_DPIN0_SELECT1; + set = FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN0_SELECT0, mux_state) | + FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN0_SELECT1, mux_state); + break; + case MUX_DPIN1: + mask = CROSSBAR_MUX_CTRL_DPIN1_SELECT0 | CROSSBAR_MUX_CTRL_DPIN1_SELECT1; + set = FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN1_SELECT0, mux_state) | + FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN1_SELECT1, mux_state); + break; + default: + return -EOPNOTSUPP; + } + + spin_lock_irqsave(&dpxbar->lock, flags); + /* a selected output already points at its source */ + if (dpxbar->selected_dispext[index] >= 0) + ret = -EBUSY; + else + dpxbar_mask32(dpxbar, CROSSBAR_MUX_CTRL, mask, set); + spin_unlock_irqrestore(&dpxbar->lock, flags); + if (ret) + return ret; + + dev_dbg(dpxbar->dev, "%s: source preselected, MUX_CTRL=%08x\n", + apple_dpxbar_names[index], readl(dpxbar->regs + CROSSBAR_MUX_CTRL)); + return 0; +} +EXPORT_SYMBOL_GPL(apple_dpxbar_preselect); + static const struct mux_control_ops apple_dpxbar_ops = { .set = apple_dpxbar_set, }; @@ -772,6 +836,7 @@ static int apple_dpxbar_probe(struct platform_device *pdev) return PTR_ERR(mux_chip); dpxbar = mux_chip_priv(mux_chip); + dpxbar->hw = hw; mux_chip->ops = hw->ops; spin_lock_init(&dpxbar->lock); @@ -815,6 +880,7 @@ static const struct apple_dpxbar_hw apple_dpxbar_hw_t6000 = { .n_ufp = 9, .tunable = 5, .ops = &apple_dpxbar_ops, + .dpin_no_cycle_slip = true, }; static const struct apple_dpxbar_hw apple_dpxbar_hw_t6020 = { diff --git a/include/linux/soc/apple/dp-tunnel.h b/include/linux/soc/apple/dp-tunnel.h index 4d2ab4eeb07da..6156aed1f169c 100644 --- a/include/linux/soc/apple/dp-tunnel.h +++ b/include/linux/soc/apple/dp-tunnel.h @@ -57,4 +57,12 @@ static inline bool apple_dp_tunnel_t602x(void) return of_machine_compatible_match(machines); } +/* + * Display crossbar: point a DP IN output that is not selected yet at source + * @state without bringing the connection up, or back at its idle source + * (MUX_IDLE_DISCONNECT). The later mux selection does the rest. + * -EOPNOTSUPP on T602X crossbars. + */ +int apple_dpxbar_preselect(struct mux_control *mux, int state); + #endif From e078472ccca8c47fe33f0d4a385c9d1e0a29582d Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Wed, 30 Sep 2026 09:54:14 -0600 Subject: [PATCH 1001/2680] phy: apple: atc: run the t8103 tunnel clock sequence on t600x The M1 Pro/Max ATC is the t8103 generation, and the Thunderbolt DP tunnel pixel clock is set up the same way. Take t6000 and t6001 through atc_tunnel_start_t8103(). Unlike t8103, which feeds the tunnel from PCLK1 alone, t600x also runs PCLK2 and the DP IN (DPRX) clock with the same rate selector. Split the sleep overrides and the TX_DP_CTRL0 setup out of the start function and add apple_atc_dp_tunnel_open(), which applies both ahead of the first rate request. On t600x DCP does not set a link rate until the DP clock path is awake, so the display driver calls this before it tells DCP about the display. Other SoCs return -EOPNOTSUPP and keep doing all of it in the rate call. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Christian Bartels Co-Authored-By: Claude Opus 5.5 [Chris Kearney: ported onto aurora-silicon/linux#8 on top of the per-DP-IN T602X tunnel clock; t600x takes the t8103 path for both start and stop.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/phy/apple/atc.c | 156 +++++++++++++++++++--------- include/linux/soc/apple/dp-tunnel.h | 7 ++ 2 files changed, 114 insertions(+), 49 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index fdcc2427c992b..a0e055fb76137 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2104,6 +2104,13 @@ static void atcphy_configure_post_tunable_all_modes(struct apple_atcphy *atcphy, * without touching the lane mux. Based on Oliver Lukschander's t6020 tunnel * clock patch (asahi-j416s-display). */ +/* t600x (M1 Pro/Max) has the t8103 generation ATC. */ +static bool apple_atc_tunnel_is_t600x(void) +{ + return of_machine_is_compatible("apple,t6000") || + of_machine_is_compatible("apple,t6001"); +} + /* * Stop the tunnel pixel clock: switch the AUSPLL output drivers off and send * APB command 3. TX_DP_CTRL0, the sleep overrides and the @@ -2121,6 +2128,69 @@ static void atc_tunnel_stop_t8103(struct apple_atcphy *atcphy) dev_dbg(atcphy->dev, "DP tunnel clock stopped\n"); } +/* + * TX_DP_CTRL0: PMA lane reset release, pixel clock enable + rate selector. + * t8103 feeds the tunnel from PCLK1 alone; t600x also runs PCLK2 and the DP IN + * (DPRX) clock at the same rate. + */ +static void atc_tunnel_pclk_t8103(struct apple_atcphy *atcphy, u32 selector) +{ + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N_OV); + if (apple_atc_tunnel_is_t600x()) { + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPRX_PCLK_ENABLE); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK2_ENABLE); + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPRX_PCLK_SELECT, + FIELD_PREP(DPRX_PCLK_SELECT, selector)); + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK2_SELECT, + FIELD_PREP(DPTX_PCLK2_SELECT, selector)); + } + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_SELECT, + FIELD_PREP(DPTX_PCLK1_SELECT, selector)); +} + +static void atc_tunnel_wake_t8103(struct apple_atcphy *atcphy) +{ + /* + * The PHY is up in USB4/TBT mode here. These overrides only force the + * already awake blocks to stay awake for the DP clock path; the order + * and the 2 us gaps are the ones this was validated with, the USB4 + * link stays up across them. + * + * Common block: override small/big sleep off and release the clamp. + */ + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL_OV); + udelay(2); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_BIG); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_BIG_OV); + udelay(2); + core_clear32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP_OV); + udelay(2); + /* same for the TX lanes */ + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_SMALL); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_SMALL_OV); + udelay(2); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_BIG); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_BIG_OV); + udelay(2); + core_clear32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_CLAMP); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_CLAMP_OV); + udelay(2); + /* and for the RX lanes */ + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG_OV); + udelay(2); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL_OV); + udelay(2); + core_clear32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP_OV); + udelay(2); +} + static int atc_tunnel_start_t8103(struct apple_atcphy *atcphy, u8 rate) { u32 selector, value; @@ -2148,10 +2218,9 @@ static int atc_tunnel_start_t8103(struct apple_atcphy *atcphy, u8 rate) return 0; /* * Rate change: DCP brackets it with WillChange/DidChange link - * configuration, so the crossbar is already down; reselect PCLK1. + * configuration, so the crossbar is already down; reselect the clocks. */ - core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_SELECT, - FIELD_PREP(DPTX_PCLK1_SELECT, selector)); + atc_tunnel_pclk_t8103(atcphy, selector); atcphy->tunnel_rate = rate; return 0; } @@ -2172,49 +2241,8 @@ static int atc_tunnel_start_t8103(struct apple_atcphy *atcphy, u8 rate) /* the PLL no longer holds a DP alt mode configuration */ atcphy->dp_link_rate = -1; - /* - * The PHY is up in USB4/TBT mode here. These overrides only force the - * already awake blocks to stay awake for the DP clock path; the order - * and the 2 us gaps are the ones this was validated with, the USB4 - * link stays up across them. - * - * Common block: override small/big sleep off and release the clamp. - */ - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL); - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL_OV); - udelay(2); - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_BIG); - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_BIG_OV); - udelay(2); - core_clear32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP); - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP_OV); - udelay(2); - /* same for the TX lanes */ - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_SMALL); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_SMALL_OV); - udelay(2); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_BIG); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_BIG_OV); - udelay(2); - core_clear32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_CLAMP); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_CLAMP_OV); - udelay(2); - /* and for the RX lanes */ - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG); - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG_OV); - udelay(2); - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL); - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL_OV); - udelay(2); - core_clear32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP); - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP_OV); - udelay(2); - /* TX_DP_CTRL0: PMA lane reset release, PCLK1 enable + rate selector */ - core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N); - core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N_OV); - core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); - core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_SELECT, - FIELD_PREP(DPTX_PCLK1_SELECT, selector)); + atc_tunnel_wake_t8103(atcphy); + atc_tunnel_pclk_t8103(atcphy, selector); /* fixed AUSPLL descriptor */ core_clear32(atcphy, AUSPLL_FREQ_CFG, AUSPLL_FREQ_REFCLK); @@ -2807,6 +2835,12 @@ static bool apple_atc_is_typec_core(u64 base) base == 0xf03000000ULL; } +/* t600x (M1 Pro/Max) runs the t8103 tunnel clock sequence unchanged. */ +static bool apple_atc_tunnel_is_t8103_style(void) +{ + return of_machine_is_compatible("apple,t8103") || apple_atc_tunnel_is_t600x(); +} + int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate) { struct apple_atcphy *atcphy; @@ -2816,7 +2850,7 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate) return -EINVAL; atcphy = phy_get_drvdata(phy); /* Keep each supported SoC on its qualified clock sequence. */ - if (!of_machine_is_compatible("apple,t8103") && !apple_dp_tunnel_t602x()) + if (!apple_atc_tunnel_is_t8103_style() && !apple_dp_tunnel_t602x()) return -EOPNOTSUPP; if (apple_dp_tunnel_t602x() && (!of_device_is_compatible(atcphy->np, "apple,t6020-atcphy") || @@ -2825,7 +2859,7 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate) return -EINVAL; guard(mutex)(&atcphy->lock); if (!rate) { - if (of_machine_is_compatible("apple,t8103")) + if (apple_atc_tunnel_is_t8103_style()) atc_tunnel_stop_t8103(atcphy); else atc_tunnel_stop_t602x(atcphy, dpin); @@ -2833,7 +2867,7 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate) } if (atcphy->mode != APPLE_ATCPHY_MODE_USB4 && atcphy->mode != APPLE_ATCPHY_MODE_TBT) return -EBUSY; - ret = of_machine_is_compatible("apple,t8103") ? + ret = apple_atc_tunnel_is_t8103_style() ? atc_tunnel_start_t8103(atcphy, rate) : atc_tunnel_set_t602x(atcphy, dpin, rate); dev_dbg(atcphy->dev, "DP tunnel clock rate 0x%x: %d (TX_DP_CTRL0=%08x PCLK_STAT=%08x)\n", @@ -2843,6 +2877,30 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate) } EXPORT_SYMBOL_GPL(apple_atc_dp_tunnel_rate); +int apple_atc_dp_tunnel_open(struct phy *phy) +{ + struct apple_atcphy *atcphy; + + if (!phy || phy->ops != &apple_atc_dp_phy_ops) + return -EINVAL; + if (!apple_atc_tunnel_is_t600x()) + return -EOPNOTSUPP; + atcphy = phy_get_drvdata(phy); + guard(mutex)(&atcphy->lock); + if (atcphy->mode != APPLE_ATCPHY_MODE_USB4 && atcphy->mode != APPLE_ATCPHY_MODE_TBT) + return -EBUSY; + if (!atcphy->tunnel_clock_on) { + atc_tunnel_wake_t8103(atcphy); + atc_tunnel_pclk_t8103(atcphy, 0); + } + dev_dbg(atcphy->dev, "DP tunnel open (CFG0=%08x SLEEP_CTRL=%08x TX_DP_CTRL0=%08x)\n", + readl(atcphy->regs.core + ACIOPHY_CFG0), + readl(atcphy->regs.core + ACIOPHY_SLEEP_CTRL), + readl(atcphy->regs.core + ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0)); + return 0; +} +EXPORT_SYMBOL_GPL(apple_atc_dp_tunnel_open); + static struct phy *atcphy_xlate(struct device *dev, const struct of_phandle_args *args) { struct apple_atcphy *atcphy = dev_get_drvdata(dev); diff --git a/include/linux/soc/apple/dp-tunnel.h b/include/linux/soc/apple/dp-tunnel.h index 6156aed1f169c..174173d46f714 100644 --- a/include/linux/soc/apple/dp-tunnel.h +++ b/include/linux/soc/apple/dp-tunnel.h @@ -31,6 +31,13 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, */ int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate); +/* + * ATC PHY: wake the DP clock path of a PHY in Thunderbolt/USB4 mode ahead of + * apple_atc_dp_tunnel_rate(), before DCP is told about the display. + * -EOPNOTSUPP where the rate call does all of it (everything but t600x). + */ +int apple_atc_dp_tunnel_open(struct phy *phy); + /* * Display crossbar: take the connection of an already selected output down or * bring it back up (clock gates and enables) around a link reconfiguration. From 6af2c121c5580ccaf95a7b26a9d7e59de2063b7c Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Wed, 30 Sep 2026 09:54:14 -0600 Subject: [PATCH 1002/2680] thunderbolt: apple: drive DP tunnels on t6000 and t6001 The M1 Pro/Max ATC has the t8103 DP IN adapter registers, the same crossbar and the same tunnel pixel clock sequence. Install the DP tunnel hook there too, and clear the DP IN interrupt status the way t8103 does. Tested on a MacBook Pro 14" M1 Pro (j314s) with an LG UltraFine 5K on both left ports: 3840x2160@60 at cold boot, on hotplug and after suspend-to-idle. t6001 is untested. apple-display-crossbar 70304c000.mux: dpin0: source preselected, MUX_CTRL=00002002 phy-apple-atc 703000000.phy: DP tunnel open (CFG0=11833fef SLEEP_CTRL=15570cff TX_DP_CTRL0=0000e00d) apple-dcp 28cc00000.dcp: display routed to Thunderbolt DP tunnel dpin0 apple-dcp 28cc00000.dcp: DPTXPort: SET_LINK_RATE 0x14 apple-display-crossbar 70304c000.mux: Switched dpin0 to dispext1,0 apple-dcp 28cc00000.dcp: dcp_hotplug() connected:1 valid_mode:0 nr_modes:5 The second DP tunnel of the display is still refused ("port already routed"), so it runs as a single 4K stream, not as two 5K tiles. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Christian Bartels Co-Authored-By: Claude Opus 5.5 [Chris Kearney: ported onto aurora-silicon/linux#8, where dp_display defaults to on; t600x takes that default too.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- Documentation/admin-guide/kernel-parameters.txt | 9 +++++---- drivers/thunderbolt/apple.c | 17 ++++++++++++++--- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 5fa793939c975..a017cf8a20a65 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -7569,10 +7569,11 @@ Kernel parameters [HW,ARM64] Format: Drive displays behind Thunderbolt DisplayPort tunnels - on Apple t8103 (M1): route a display pipeline to the - host router's DP IN adapter whenever the connection - manager sets up a DP tunnel. Set to 0 to leave displays - behind a dock dark. Default: true + on Apple t8103 and t600x (M1, M1 Pro and M1 Max): + route a display pipeline to the host router's DP IN + adapter whenever the connection manager sets up a DP + tunnel. Set to 0 to leave displays behind a dock dark. + Default: true thunderbolt_apple.dpin_aux= [HW,ARM64] diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 3660cc9f543b9..c2969ae0043e0 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -262,7 +262,18 @@ struct apple_cio { static bool dp_display = true; module_param(dp_display, bool, 0444); -MODULE_PARM_DESC(dp_display, "Drive displays behind Thunderbolt DP tunnels on t8103 (default: true)"); +MODULE_PARM_DESC(dp_display, "Drive displays behind Thunderbolt DP tunnels on t8103 and t600x (default: true)"); + +/* + * The M1 Pro/Max ATC is the t8103 generation: same DP IN adapter registers, + * same crossbar and the same tunnel pixel clock sequence. + */ +static bool apple_cio_dp_is_t8103_style(void) +{ + return of_machine_is_compatible("apple,t8103") || + of_machine_is_compatible("apple,t6000") || + of_machine_is_compatible("apple,t6001"); +} /* DPTX_INACTIVE handshake: request (in)active, wait for the ACK */ struct apple_dpin_poll { @@ -443,7 +454,7 @@ static int apple_dpin_up(struct apple_dpin_ctx *c) * back. Nothing handles these interrupts yet; they are enabled * as part of bringing the adapter up. */ - if (of_machine_is_compatible("apple,t8103")) { + if (apple_cio_dp_is_t8103_style()) { irq = readl(regs + APPLE_DPIN_IRQ_STATUS); writel(irq, regs + APPLE_DPIN_IRQ_STATUS); if (irq & BIT(0)) @@ -1714,7 +1725,7 @@ static int apple_nhi_probe(struct platform_device *pdev) * the connection manager) only where it is enabled and known to work. */ anhi->ops = apple_nhi_ops; - if (dp_display && acio->dp_wq && of_machine_is_compatible("apple,t8103")) + if (dp_display && acio->dp_wq && apple_cio_dp_is_t8103_style()) anhi->ops.dp_tunnel_changed = apple_nhi_dp_tunnel_changed; if (acio->dp_wq && apple_dp_tunnel_t602x()) { anhi->ops.dp_tunnel_pre_activate = apple_nhi_dp_tunnel_pre_activate; From b5c82f853bbe952a9a3860972879cd18c605ef71 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Thu, 1 Oct 2026 16:04:13 +0000 Subject: [PATCH 1003/2680] usb: typec: tipd: accept Apple "DBMa" mode, add debugfs command access MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apple's CD321x and SN201202x firmware has a debug mode, "DBMa". It is entered with an unlock key (LOCK) followed by the DBMa command. In this mode the application firmware still handles the port. It also accepts VDMs, which sends Apple VDMs (reboot, DFU, serial, Debug USB) to a connected Mac, and DVEn, which enables the local end of a debug route. tuxvdmtool and macvdmtool use these commands to control another Mac over its DFU port. A boot loader that arms Apple Debug USB on a port leaves its controller in DBMa. tipd then fails to probe it with 'controller in unsupported mode "DBMa"' (-ENODEV). The port gets no Type-C device and no kernel interface can reach the controller. SPMI-attached SN201202x controllers (M3 and later) also have no /dev/i2c-N fallback for userspace. Accept DBMa at probe and resume, as the Asahi tree has done since commit 786523ac62f0 ("usb: typec: tipd: Add cd321x specific control commands"). Unlike that commit, keep rejecting BIST and DISC. Use the controller as found: - The switch to the S0 power state runs before the mode check, as before. It sends SSPS only if the controller is not already in S0. - The patch/init step still runs only in PTCH mode. - Everything else is register reads plus the INT_MASK1 write. Probing therefore sends no command that leaves DBMa or disturbs a Debug USB route set up before Linux. Record whether DBMa was active at probe. Add a root-only interface at /sys/kernel/debug/usb/tipd//: command Write a 4CC followed by 0..64 bytes of DATA1. The command runs as a whole under the driver lock through tps6598x_exec_cmd_tmo(). Reading returns DATA1 as left by the last completed command of this open file. Only LOCK, DBMa, VDMs (200 ms timeout), DVEn and DISC are accepted. regs Read-only. pread() at offset = register (0..0x7f) returns up to 64 bytes, read under the driver lock. The interface works over both the i2c and SPMI transports. Unlike raw /dev/i2c access, it serialises with the interrupt handler. When an open command file that put the controller into DBMa is released, or the device is unbound while such a file is open, the controller leaves DBMa again (DBMa 0, then LOCK 0000). This is skipped if the controller was already in DBMa at probe, since that mode belongs to the boot loader's Debug USB setup. Controllers that are not in DBMa probe exactly as before, and nothing changes while the debugfs files are unused. The interface exists only with CONFIG_DEBUG_FS. [ryan: adapted to 7.2.10 — Asahi split the tipd core from the bus glue; the task codes and TPS_MAX_LEN move to tps6598x.h next to the variant enum, and debugfs registration stays ahead of the variant ready hook] Link: https://github.com/AsahiLinux/linux/commit/786523ac62f0 Link: https://github.com/aurora-silicon/tuxvdmtool Co-Authored-By: Claude Opus 5.5 (1M context) --- drivers/usb/typec/tipd/Kconfig | 1 + drivers/usb/typec/tipd/Makefile | 1 + drivers/usb/typec/tipd/core.c | 51 ++++-- drivers/usb/typec/tipd/debugfs.c | 292 ++++++++++++++++++++++++++++++ drivers/usb/typec/tipd/tps6598x.h | 30 +++ 5 files changed, 359 insertions(+), 16 deletions(-) create mode 100644 drivers/usb/typec/tipd/debugfs.c diff --git a/drivers/usb/typec/tipd/Kconfig b/drivers/usb/typec/tipd/Kconfig index 2f73bb17ed352..a447a4622ea76 100644 --- a/drivers/usb/typec/tipd/Kconfig +++ b/drivers/usb/typec/tipd/Kconfig @@ -2,6 +2,7 @@ config TYPEC_TPS6598X_CORE tristate select USB_ROLE_SWITCH select POWER_SUPPLY + select USB_COMMON if DEBUG_FS config TYPEC_TPS6598X tristate "TI TPS6598x USB Power Delivery controller driver" diff --git a/drivers/usb/typec/tipd/Makefile b/drivers/usb/typec/tipd/Makefile index c0e850a56f289..1a7605e7e57ec 100644 --- a/drivers/usb/typec/tipd/Makefile +++ b/drivers/usb/typec/tipd/Makefile @@ -4,6 +4,7 @@ CFLAGS_trace.o := -I$(src) obj-$(CONFIG_TYPEC_TPS6598X_CORE) += tps6598x-core.o tps6598x-core-y := core.o tps6598x-core-$(CONFIG_TRACING) += trace.o +tps6598x-core-$(CONFIG_DEBUG_FS) += debugfs.o obj-$(CONFIG_TYPEC_TPS6598X) += tps6598x.o tps6598x-y := i2c.o diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index a73434b5bca1b..1f7d168c63b22 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -91,10 +91,6 @@ struct tps6598x_rx_identity_reg { struct usb_pd_identity identity; } __packed; -/* Standard Task return codes */ -#define TPS_TASK_TIMEOUT 1 -#define TPS_TASK_REJECTED 3 - /* Debounce delay for mode changes, in milliseconds */ #define CD321X_DEBOUNCE_DELAY_MS 500 @@ -104,6 +100,7 @@ enum { TPS_MODE_BIST, TPS_MODE_DISC, TPS_MODE_PTCH, + TPS_MODE_DBMA, }; static const char *const modes[] = { @@ -112,6 +109,7 @@ static const char *const modes[] = { [TPS_MODE_BIST] = "BIST", [TPS_MODE_DISC] = "DISC", [TPS_MODE_PTCH] = "PTCH", + [TPS_MODE_DBMA] = "DBMa", }; /* Unrecognized commands will be replaced with "!CMD" */ @@ -124,14 +122,7 @@ static enum power_supply_property tps6598x_psy_props[] = { static const char *tps6598x_psy_name_prefix = "tps6598x-source-psy-"; -/* - * Max data bytes for Data1, Data2, and other registers. See ch 1.3.2: - * https://www.ti.com/lit/ug/slvuan1a/slvuan1a.pdf - */ -#define TPS_MAX_LEN 64 - -static int -tps6598x_block_read(struct tps6598x *tps, u8 reg, void *val, size_t len) +int tps6598x_block_read(struct tps6598x *tps, u8 reg, void *val, size_t len) { u8 data[TPS_MAX_LEN + 1]; int ret; @@ -296,10 +287,10 @@ static void tps6598x_disconnect(struct tps6598x *tps, u32 status) power_supply_changed(tps->psy); } -static int tps6598x_exec_cmd_tmo(struct tps6598x *tps, const char *cmd, - size_t in_len, const u8 *in_data, - size_t out_len, u8 *out_data, - u32 cmd_timeout_ms, u32 res_delay_ms) +int tps6598x_exec_cmd_tmo(struct tps6598x *tps, const char *cmd, + size_t in_len, const u8 *in_data, + size_t out_len, u8 *out_data, + u32 cmd_timeout_ms, u32 res_delay_ms) { unsigned long timeout; u32 val; @@ -1249,6 +1240,14 @@ static int tps6598x_check_mode(struct tps6598x *tps) case TPS_MODE_BOOT: dev_warn(tps->dev, "dead-battery condition\n"); return ret; + case TPS_MODE_DBMA: + /* + * Apple firmware only: the application firmware runs in its + * debug mode (entered with LOCK + "DBMa"), in which it still + * handles the port and additionally accepts VDMs/DVEn. A boot + * loader arming Apple Debug USB on this port leaves it here. + */ + return ret; case TPS_MODE_BIST: case TPS_MODE_DISC: default: @@ -2044,6 +2043,17 @@ int tipd_init(struct tps6598x *tps) return ret; } + /* + * A controller found in "DBMa" is used as it is. Nothing below issues a + * 4CC command (the S0 power state switch above only does if the + * controller is not in S0 already), so probing neither leaves the debug + * mode nor disturbs a Debug USB or serial route set up before Linux. + * Remember it, so that the debugfs interface never tears that mode down. + */ + tps->dbma_at_probe = ret == TPS_MODE_DBMA; + if (tps->dbma_at_probe) + dev_info(tps->dev, "controller in debug mode \"DBMa\", leaving it there\n"); + ret = tps6598x_write64(tps, TPS_REG_INT_MASK1, tps->data->irq_mask1); if (ret) goto err_reset_controller; @@ -2112,6 +2122,7 @@ int tipd_init(struct tps6598x *tps) enable_irq_wake(tps->irq); } + tipd_debugfs_register(tps); if (tps->data->ready) tps->data->ready(tps); @@ -2138,6 +2149,8 @@ EXPORT_SYMBOL_GPL(tipd_init); void tipd_remove(struct tps6598x *tps) { + tipd_debugfs_unregister(tps); + if (!tps->irq) cancel_delayed_work_sync(&tps->wq_poll); else @@ -2314,6 +2327,12 @@ const struct tipd_data *tipd_hw_variants[] = { }; EXPORT_SYMBOL_GPL(tipd_hw_variants); +static void __exit tipd_core_exit(void) +{ + tipd_debugfs_exit(); +} +module_exit(tipd_core_exit); + MODULE_AUTHOR("Heikki Krogerus "); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("TI TPS6598x USB Power Delivery Controller Core Functions"); diff --git a/drivers/usb/typec/tipd/debugfs.c b/drivers/usb/typec/tipd/debugfs.c new file mode 100644 index 0000000000000..33dbc9dca411d --- /dev/null +++ b/drivers/usb/typec/tipd/debugfs.c @@ -0,0 +1,292 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * TI TPS6598x / Apple CD321x / SN201202x debugfs interface + * + * /sys/kernel/debug/usb/tipd//command + * write: 4CC command followed by 0..64 bytes for DATA1. The command is + * executed as a whole under the driver's lock. Only a small set + * of commands used to talk to a connected Apple device (as + * tuxvdmtool does) is accepted. + * read: the 64 bytes of DATA1 that the last completed command of this + * open file left behind. + * + * /sys/kernel/debug/usb/tipd//regs + * read-only register access: pread() with the register as offset + * (0x00..0x7f) and the length (1..64 bytes) as count. + * + * An open "command" file that put the controller into "DBMa" leaves that + * mode (DBMa 0, then LOCK 0000) when it is released, unless the controller + * was in "DBMa" already when the driver probed: that mode belongs to + * whoever set it up before Linux (e.g. a boot loader arming Debug USB). + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "tps6598x.h" + +#define TPS_REG_MAX 0x7f + +struct tipd_debugfs { + struct dentry *dentry; + /* Open "command" files that entered "DBMa"; protected by tps->lock */ + unsigned int dbma_holders; +}; + +struct tipd_debugfs_cmd { + char cmd[4]; + u8 min_len; + u8 max_len; + u32 timeout_ms; +}; + +static const struct tipd_debugfs_cmd tipd_debugfs_cmds[] = { + { "LOCK", 4, 4, 1000 }, /* unlock key, 0000 locks again */ + { "DBMa", 1, 1, 1000 }, /* 1: enter, 0: leave debug mode */ + { "VDMs", 5, 29, 200 }, /* SOP type/count + 1..7 VDOs */ + { "DVEn", 4, 28, 1000 }, /* local end of a serial/debug route */ + { "DISC", 0, 1, 1000 }, /* simulated disconnect */ +}; + +struct tipd_debugfs_file { + struct tps6598x *tps; + bool holds_dbma; + bool have_result; + u8 result[TPS_MAX_LEN]; +}; + +static struct dentry *tipd_debugfs_root; +static DEFINE_MUTEX(tipd_debugfs_root_lock); + +static const struct tipd_debugfs_cmd *tipd_debugfs_find_cmd(const u8 *cmd) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(tipd_debugfs_cmds); i++) + if (!memcmp(tipd_debugfs_cmds[i].cmd, cmd, 4)) + return &tipd_debugfs_cmds[i]; + + return NULL; +} + +/* Leave the debug mode entered through debugfs. Called with tps->lock held. */ +static void tipd_debugfs_leave_dbma(struct tps6598x *tps) +{ + static const u8 lock_key[4]; + u8 dbma = 0; + int ret; + + lockdep_assert_held(&tps->lock); + + if (tps->dbma_at_probe) + return; + + ret = tps6598x_exec_cmd_tmo(tps, "DBMa", sizeof(dbma), &dbma, 0, NULL, + 1000, 0); + if (ret) + dev_warn(tps->dev, "debugfs: failed to leave \"DBMa\": %d\n", ret); + + ret = tps6598x_exec_cmd_tmo(tps, "LOCK", sizeof(lock_key), lock_key, + 0, NULL, 1000, 0); + if (ret) + dev_warn(tps->dev, "debugfs: failed to lock: %d\n", ret); +} + +static int tipd_debugfs_command_open(struct inode *inode, struct file *file) +{ + struct tipd_debugfs_file *f; + + f = kzalloc_obj(*f); + if (!f) + return -ENOMEM; + + f->tps = inode->i_private; + file->private_data = f; + + return 0; +} + +static int tipd_debugfs_command_release(struct inode *inode, struct file *file) +{ + struct tipd_debugfs_file *f = file->private_data; + struct dentry *dentry = file->f_path.dentry; + struct tps6598x *tps = f->tps; + + /* + * debugfs calls ->release even after the file was removed, when tps + * may be gone. tipd_debugfs_unregister() has then left "DBMa" itself. + */ + if (f->holds_dbma && !debugfs_file_get(dentry)) { + mutex_lock(&tps->lock); + if (!--tps->debugfs->dbma_holders) + tipd_debugfs_leave_dbma(tps); + mutex_unlock(&tps->lock); + debugfs_file_put(dentry); + } + + kfree(f); + + return 0; +} + +static ssize_t tipd_debugfs_command_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct tipd_debugfs_file *f = file->private_data; + struct tps6598x *tps = f->tps; + const struct tipd_debugfs_cmd *cmd; + u8 buf[4 + TPS_MAX_LEN]; + u8 out[TPS_MAX_LEN] = { }; + size_t len; + int ret; + + if (count < 4 || count > sizeof(buf)) + return -EINVAL; + + if (copy_from_user(buf, ubuf, count)) + return -EFAULT; + + cmd = tipd_debugfs_find_cmd(buf); + if (!cmd) + return -EOPNOTSUPP; + + len = count - 4; + if (len < cmd->min_len || len > cmd->max_len) + return -EINVAL; + if (!memcmp(cmd->cmd, "VDMs", 4) && (len - 1) % 4) + return -EINVAL; + if (!memcmp(cmd->cmd, "DVEn", 4) && len % 4) + return -EINVAL; + + mutex_lock(&tps->lock); + + ret = tps6598x_exec_cmd_tmo(tps, cmd->cmd, len, len ? &buf[4] : NULL, + sizeof(out), out, cmd->timeout_ms, 0); + + /* DATA1 was read back if the command ran, even if its task failed */ + f->have_result = !ret || + (ret == -ETIMEDOUT && out[0] == TPS_TASK_TIMEOUT) || + (ret == -EPERM && out[0] == TPS_TASK_REJECTED); + if (f->have_result) + memcpy(f->result, out, sizeof(out)); + + if (!ret && !memcmp(cmd->cmd, "DBMa", 4)) { + if (buf[4] && !(out[0] & 0xf) && !f->holds_dbma) { + f->holds_dbma = true; + tps->debugfs->dbma_holders++; + } else if (!buf[4] && f->holds_dbma) { + f->holds_dbma = false; + tps->debugfs->dbma_holders--; + } + } + + mutex_unlock(&tps->lock); + + return ret ? ret : count; +} + +static ssize_t tipd_debugfs_command_read(struct file *file, char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct tipd_debugfs_file *f = file->private_data; + + if (!f->have_result) + return -ENODATA; + + return simple_read_from_buffer(ubuf, count, ppos, f->result, + sizeof(f->result)); +} + +static const struct file_operations tipd_debugfs_command_fops = { + .owner = THIS_MODULE, + .open = tipd_debugfs_command_open, + .release = tipd_debugfs_command_release, + .write = tipd_debugfs_command_write, + .read = tipd_debugfs_command_read, + .llseek = default_llseek, +}; + +static ssize_t tipd_debugfs_regs_read(struct file *file, char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct tps6598x *tps = file->private_data; + u8 buf[TPS_MAX_LEN]; + int ret; + + if (*ppos < 0 || *ppos > TPS_REG_MAX) + return -EINVAL; + if (!count) + return 0; + if (count > sizeof(buf)) + return -EINVAL; + + mutex_lock(&tps->lock); + ret = tps6598x_block_read(tps, *ppos, buf, count); + mutex_unlock(&tps->lock); + if (ret) + return ret; + + /* The offset selects a register; it does not advance. */ + if (copy_to_user(ubuf, buf, count)) + return -EFAULT; + + return count; +} + +static const struct file_operations tipd_debugfs_regs_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = tipd_debugfs_regs_read, + .llseek = default_llseek, +}; + +void tipd_debugfs_register(struct tps6598x *tps) +{ + struct tipd_debugfs *dbg; + + mutex_lock(&tipd_debugfs_root_lock); + if (!tipd_debugfs_root) + tipd_debugfs_root = debugfs_create_dir("tipd", usb_debug_root); + mutex_unlock(&tipd_debugfs_root_lock); + + dbg = kzalloc_obj(*dbg); + if (!dbg) + return; + + dbg->dentry = debugfs_create_dir(dev_name(tps->dev), tipd_debugfs_root); + tps->debugfs = dbg; + debugfs_create_file("command", 0600, dbg->dentry, tps, + &tipd_debugfs_command_fops); + debugfs_create_file("regs", 0400, dbg->dentry, tps, + &tipd_debugfs_regs_fops); +} + +void tipd_debugfs_unregister(struct tps6598x *tps) +{ + struct tipd_debugfs *dbg = tps->debugfs; + + if (!dbg) + return; + + /* Waits for running file operations, later ones fail with -EIO */ + debugfs_remove_recursive(dbg->dentry); + + mutex_lock(&tps->lock); + if (dbg->dbma_holders) + tipd_debugfs_leave_dbma(tps); + tps->debugfs = NULL; + mutex_unlock(&tps->lock); + + kfree(dbg); +} + +void tipd_debugfs_exit(void) +{ + debugfs_remove(tipd_debugfs_root); +} diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index e06a671b4e57e..e290f9d2d80df 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -280,6 +280,16 @@ #define TPS_65981_2_6_INTEVENT_LEN 8 #define TPS_65987_8_INTEVENT_LEN 11 +/* + * Max data bytes for Data1, Data2, and other registers. See ch 1.3.2: + * https://www.ti.com/lit/ug/slvuan1a/slvuan1a.pdf + */ +#define TPS_MAX_LEN 64 + +/* Standard Task return codes */ +#define TPS_TASK_TIMEOUT 1 +#define TPS_TASK_REJECTED 3 + enum tipd_hw_variant { TIPD_VARIANT_INVALID = 0, TIPD_VARIANT_CD321X, @@ -318,6 +328,7 @@ struct tps6598x { struct mutex lock; /* device lock */ int irq; u8 i2c_protocol:1; + u8 dbma_at_probe:1; /* Apple "DBMa" debug mode was active at probe */ struct gpio_desc *reset; struct typec_port *port; @@ -337,6 +348,8 @@ struct tps6598x { struct delayed_work wq_poll; const struct tipd_data *data; + + struct tipd_debugfs *debugfs; }; /* TPS_REG_USB4_STATUS */ @@ -425,4 +438,21 @@ void tipd_complete(struct device *dev); int tipd_suspend(struct tps6598x *tps); int tipd_resume(struct tps6598x *tps); +/* Internal to tps6598x-core */ +int tps6598x_block_read(struct tps6598x *tps, u8 reg, void *val, size_t len); +int tps6598x_exec_cmd_tmo(struct tps6598x *tps, const char *cmd, + size_t in_len, const u8 *in_data, + size_t out_len, u8 *out_data, + u32 cmd_timeout_ms, u32 res_delay_ms); + +#ifdef CONFIG_DEBUG_FS +void tipd_debugfs_register(struct tps6598x *tps); +void tipd_debugfs_unregister(struct tps6598x *tps); +void tipd_debugfs_exit(void); +#else +static inline void tipd_debugfs_register(struct tps6598x *tps) { } +static inline void tipd_debugfs_unregister(struct tps6598x *tps) { } +static inline void tipd_debugfs_exit(void) { } +#endif + #endif /* __TPS6598X_H__ */ From 6d6ae2f5d4fb413dc2686583734caae78abacb78 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:21:57 +1100 Subject: [PATCH 1004/2680] usb: typec: tipd: preserve shared and inherited debug mode Reject mode exit and relocking when they would invalidate another open file or the bootloader Debug USB route. Validate the DBMa boolean payload, and serialize command-result reads with writes on the controller lock. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/debugfs.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/drivers/usb/typec/tipd/debugfs.c b/drivers/usb/typec/tipd/debugfs.c index 33dbc9dca411d..5a8d576eff0dc 100644 --- a/drivers/usb/typec/tipd/debugfs.c +++ b/drivers/usb/typec/tipd/debugfs.c @@ -163,9 +163,21 @@ static ssize_t tipd_debugfs_command_write(struct file *file, return -EINVAL; if (!memcmp(cmd->cmd, "DVEn", 4) && len % 4) return -EINVAL; + if (!memcmp(cmd->cmd, "DBMa", 4) && buf[4] > 1) + return -EINVAL; mutex_lock(&tps->lock); + /* Preserve inherited mode and the leases held by other open files. */ + if ((!memcmp(cmd->cmd, "DBMa", 4) && !buf[4] && + (tps->dbma_at_probe || + tps->debugfs->dbma_holders > (unsigned int)f->holds_dbma)) || + (!memcmp(cmd->cmd, "LOCK", 4) && !memchr_inv(buf + 4, 0, 4) && + (tps->dbma_at_probe || tps->debugfs->dbma_holders))) { + ret = -EBUSY; + goto out; + } + ret = tps6598x_exec_cmd_tmo(tps, cmd->cmd, len, len ? &buf[4] : NULL, sizeof(out), out, cmd->timeout_ms, 0); @@ -186,6 +198,7 @@ static ssize_t tipd_debugfs_command_write(struct file *file, } } +out: mutex_unlock(&tps->lock); return ret ? ret : count; @@ -195,12 +208,17 @@ static ssize_t tipd_debugfs_command_read(struct file *file, char __user *ubuf, size_t count, loff_t *ppos) { struct tipd_debugfs_file *f = file->private_data; + ssize_t ret; + mutex_lock(&f->tps->lock); if (!f->have_result) - return -ENODATA; + ret = -ENODATA; + else + ret = simple_read_from_buffer(ubuf, count, ppos, f->result, + sizeof(f->result)); + mutex_unlock(&f->tps->lock); - return simple_read_from_buffer(ubuf, count, ppos, f->result, - sizeof(f->result)); + return ret; } static const struct file_operations tipd_debugfs_command_fops = { From a7755c690e7c017b0f0ae2819a3f169710d9095b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:40:33 +1100 Subject: [PATCH 1005/2680] dt-bindings: apple: Describe T8140 idle and display crossbar controllers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document the T8140 CPU idle selector and the existing Apple display crossbar interface. T8140 uses only the WFI clock-gated state; CPU and cluster power-down are not supported. The crossbar compatible follows the controller type reported by the ADT and the existing Asahi driver. Keep CS42L42 supply phandles mandatory while permitting the established CS42L83 board model with platform-managed supplies. Existing public Apple board descriptions already use that model. [ryan: adapted to 7.2.10 — split by topic: the cirrus,cs42l42 binding hunk is carried by 070-audio; the MAINTAINERS lines are carried by 900-config-tooling] Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/cpu/apple,t8140-cpuidle.yaml | 33 +++++++++++++ .../bindings/mux/apple,display-crossbar.yaml | 47 +++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml create mode 100644 Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml diff --git a/Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml b/Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml new file mode 100644 index 0000000000000..aa27ec55309c8 --- /dev/null +++ b/Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/cpu/apple,t8140-cpuidle.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 CPU idle controller + +maintainers: + - Ryan Murray + +description: + Selects the Apple CPU idle driver on T8140 systems. Only the WFI + clock-gated state is supported; CPU and cluster power-down states + are not enabled on this SoC. + +properties: + $nodename: + const: cpuidle + + compatible: + const: apple,t8140-cpuidle + +required: + - compatible + +additionalProperties: false + +examples: + - | + cpuidle { + compatible = "apple,t8140-cpuidle"; + }; diff --git a/Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml b/Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml new file mode 100644 index 0000000000000..eb80cf382a188 --- /dev/null +++ b/Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mux/apple,display-crossbar.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple display crossbar + +maintainers: + - Sven Peter + +description: + Routes display pipelines to the DisplayPort transmitters and inputs + of an Apple Type-C PHY. Each crossbar exposes three mux controls, + selected by the consumer's mux specifier. + +allOf: + - $ref: mux-controller.yaml# + +properties: + compatible: + enum: + - apple,t8103-display-crossbar + - apple,t8112-display-crossbar + - apple,t6000-display-crossbar + - apple,t6020-display-crossbar + + reg: + maxItems: 1 + + '#mux-control-cells': + const: 1 + +required: + - compatible + - reg + - '#mux-control-cells' + +unevaluatedProperties: false + +examples: + - | + mux-controller@b04c000 { + compatible = "apple,t6020-display-crossbar"; + reg = <0xb04c000 0x4000>; + #mux-control-cells = <1>; + }; From f09082993554142486c6911f7ed6ad428eaed18a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 11:12:21 +1100 Subject: [PATCH 1006/2680] dt-bindings: cpu: apple,t8140-cpuidle: Correct the idle description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The description calls the T8140 WFI state "clock-gated" and says the power-down states are "not enabled". On a J700, WFI already lets the hardware power-gate idle cores: the PMGR power-state registers of the idle E cores and of the second P core read as power-gated in 94% to 99.9% of 1 kHz samples. What T8140 lacks is a way for software to ask for power-down, so there is no separate state to enable. Describe that instead. No property changes. [ryan: adapted to 7.2.10 — split by topic: only the Documentation/devicetree/bindings/cpu hunks of this dt commit; the rest is in 000-devicetree] Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/cpu/apple,t8140-cpuidle.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml b/Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml index aa27ec55309c8..7d4f9cdbc9182 100644 --- a/Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml +++ b/Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml @@ -10,9 +10,10 @@ maintainers: - Ryan Murray description: - Selects the Apple CPU idle driver on T8140 systems. Only the WFI - clock-gated state is supported; CPU and cluster power-down states - are not enabled on this SoC. + Selects the Apple CPU idle driver on T8140 systems. On this SoC the + hardware decides when an idle core is power-gated after WFI; software + cannot request core or cluster power-down, so WFI is the only idle + state. properties: $nodename: From b6ddf3cac1a64a623ccc17b2edbef84bb98ef1e1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 20:59:06 +1100 Subject: [PATCH 1007/2680] usb: dwc3: apple: Keep a fixed-hub controller running across sleep Commit 11585d940629 ("usb: dwc3: apple: Add system sleep support for a fixed-hub controller") exits the core of a fixed-hub controller for system sleep and initialises it again on resume. Every resume then costs a full USB re-enumeration: xHCI finds its saved state gone, reports "xHC error in resume, USBSTS 0x411, Reinit", and resets the controller and every device behind the fixed hub. On the J700 that is most of the 2.8 s device resume, and the network behind the hub goes away meanwhile. On the way down the core exit also powers the USB3 PHY off, which routes the PIPE back to the dummy PHY while the SuperSpeed port is suspended; the pipehandler lock is then never acknowledged ("Pipehandler lock not acked", "Failed to lock pipehandler"). Left running, the core keeps its state across suspend-to-idle. Mark a fixed-hub controller as a syscore device while its core is up, as a Type-C controller is while a cable is connected since commit 3b7f11e579c4 ("usb: dwc3-apple: preserve live ports across system sleep"), and drop the sleep callbacks and the DWC3_APPLE_SUSPENDED state. xHCI saves its state on suspend and restores it on resume, and the devices behind the hub are resumed instead of reset. As a syscore device the core is also never counted as suspended in its power domain, so genpd cannot switch the domain off underneath it in the noirq phase. On the J700, ten 70 s suspend-to-idle cycles resumed their devices in 293-297 ms instead of 2.8 s, with no xHCI reinit, no pipehandler message and none of the devices behind the hub reset or re-enumerated. Fixes: 11585d940629 ("usb: dwc3: apple: Add system sleep support for a fixed-hub controller") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/dwc3-apple-state-test.c | 2 +- drivers/usb/dwc3/dwc3-apple-state.h | 1 - drivers/usb/dwc3/dwc3-apple.c | 100 +++-------------------- 3 files changed, 11 insertions(+), 92 deletions(-) diff --git a/drivers/usb/dwc3/dwc3-apple-state-test.c b/drivers/usb/dwc3/dwc3-apple-state-test.c index bf9a539a53e3f..7a04604ddd4a1 100644 --- a/drivers/usb/dwc3/dwc3-apple-state-test.c +++ b/drivers/usb/dwc3/dwc3-apple-state-test.c @@ -26,7 +26,7 @@ static void dwc3_apple_device_selected_before_reset_test(struct kunit *test) static void dwc3_apple_non_role_test(struct kunit *test) { static const enum dwc3_apple_state states[] = { - DWC3_APPLE_PROBE_PENDING, DWC3_APPLE_NO_CABLE, DWC3_APPLE_SUSPENDED, + DWC3_APPLE_PROBE_PENDING, DWC3_APPLE_NO_CABLE, }; enum phy_mode mode; int i; diff --git a/drivers/usb/dwc3/dwc3-apple-state.h b/drivers/usb/dwc3/dwc3-apple-state.h index 89ce0b416bb40..42c13d900428e 100644 --- a/drivers/usb/dwc3/dwc3-apple-state.h +++ b/drivers/usb/dwc3/dwc3-apple-state.h @@ -16,7 +16,6 @@ enum dwc3_apple_state { DWC3_APPLE_NO_CABLE, /* No cable connected, dwc3 suspended after dwc3_core_exit */ DWC3_APPLE_HOST, /* Cable connected, dwc3 in host mode */ DWC3_APPLE_DEVICE, /* Cable connected, dwc3 in device mode */ - DWC3_APPLE_SUSPENDED, /* Fixed-hub host asleep: core exited, xhci still registered */ }; /** diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index da01a43877473..232a4de95bb75 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include "dwc3-apple-state.h" @@ -76,8 +75,9 @@ * a host whatever happens at the connectors and never sees a role change: it is brought up in * DWC3_APPLE_HOST at probe and stays there. The Type-C port controllers only drive the PHY's * Type-C mux for the SuperSpeed lanes, and the PHY follows those mode changes on its own. - * Across system sleep such a controller goes through DWC3_APPLE_SUSPENDED: the core is exited - * and initialised again around the sleep while xhci stays registered, see dwc3_apple_suspend(). + * + * Whenever the core is up, for either kind of port, it is kept running across system sleep, see + * dwc3_apple_init(). * * enum dwc3_apple_state is defined in dwc3-apple-state.h. */ @@ -240,7 +240,6 @@ static int dwc3_apple_core_init(struct dwc3_apple *appledwc) dev_err(appledwc->dev, "Failed to probe DWC3 Core, err=%d\n", ret); break; case DWC3_APPLE_NO_CABLE: - case DWC3_APPLE_SUSPENDED: ret = dwc3_core_init(&appledwc->dwc); if (ret) dev_err(appledwc->dev, "Failed to initialize DWC3 Core, err=%d\n", ret); @@ -378,11 +377,12 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st * role change or disconnect. Do not let generic system PM independently * power-gate the DWC3 device in between: xHCI then observes a dead core on * resume and cannot restore the root hubs without a full cable teardown. - * A fixed-hub controller is never torn down by a role change and has - * sleep callbacks of its own instead, see dwc3_apple_suspend(). + * The same holds for a fixed-hub controller, whose core is up for as long + * as the driver is bound. As a syscore device the core is not counted as + * suspended in its power domain, so the domain stays on, and xHCI saves + * its state on suspend and restores it on resume. */ - if (!appledwc->fixed_hub) - dev_pm_syscore_device(appledwc->dev, true); + dev_pm_syscore_device(appledwc->dev, true); return 0; core_exit: @@ -406,20 +406,13 @@ static int dwc3_apple_exit(struct dwc3_apple *appledwc) /* Nothing to do if we're already off */ return 0; case DWC3_APPLE_DEVICE: - if (!appledwc->fixed_hub) - dev_pm_syscore_device(appledwc->dev, false); + dev_pm_syscore_device(appledwc->dev, false); dwc3_gadget_exit(&appledwc->dwc); break; case DWC3_APPLE_HOST: - if (!appledwc->fixed_hub) - dev_pm_syscore_device(appledwc->dev, false); + dev_pm_syscore_device(appledwc->dev, false); dwc3_host_exit(&appledwc->dwc); break; - case DWC3_APPLE_SUSPENDED: - /* The core is exited and the reset asserted already, see dwc3_apple_suspend() */ - dwc3_host_exit(&appledwc->dwc); - appledwc->state = DWC3_APPLE_NO_CABLE; - return 0; } /* @@ -498,7 +491,6 @@ static enum usb_role dwc3_usb_role_switch_get(struct usb_role_switch *sw) switch (appledwc->state) { case DWC3_APPLE_HOST: - case DWC3_APPLE_SUSPENDED: return USB_ROLE_HOST; case DWC3_APPLE_DEVICE: return USB_ROLE_DEVICE; @@ -639,84 +631,12 @@ static const struct of_device_id dwc3_apple_of_match[] = { }; MODULE_DEVICE_TABLE(of, dwc3_apple_of_match); -/* - * System sleep of a fixed-hub controller. - * - * A controller with a role switch is a syscore device while a cable is connected, see - * dwc3_apple_init(). A fixed-hub controller is always up, and its core does not keep its state - * across s2idle, so it gets what dwc3_suspend_common() and dwc3_resume_common() give a host: - * the core is exited on the way down and initialised again on the way up. The xhci child stays - * registered throughout. It suspends before this device and resumes after it, and then resets - * the re-initialised controller. Adding or removing the child from inside a PM callback is not - * an option, since that re-enters the PM core's device list. - * - * No wakeup path is offered: the interrupt controller implements no irq_set_wake, so no - * interrupt on this SoC can be armed to end s2idle, and a remote-wakeup capable USB device - * could not end it either. Once that changes, the wake IRQ belongs on the xhci device - * (dev_pm_set_wake_irq()) and a keep-powered branch like dwc3_suspend_common()'s belongs here. - */ -static int dwc3_apple_suspend(struct device *dev) -{ - struct dwc3 *dwc = dev_get_drvdata(dev); - struct dwc3_apple *appledwc; - int ret; - - if (!dwc) - return 0; - appledwc = to_dwc3_apple(dwc); - - guard(mutex)(&appledwc->lock); - - if (!appledwc->fixed_hub || appledwc->state != DWC3_APPLE_HOST) - return 0; - - /* SUSPHY has to be enabled for the PHY to power down properly, as in dwc3_apple_exit() */ - dwc3_enable_susphy(&appledwc->dwc, true); - dwc3_core_exit(&appledwc->dwc); - appledwc->state = DWC3_APPLE_SUSPENDED; - - ret = reset_control_assert(appledwc->reset); - if (ret) - dev_err(appledwc->dev, "Failed to assert reset, err=%d\n", ret); - - return ret; -} - -static int dwc3_apple_resume(struct device *dev) -{ - struct dwc3 *dwc = dev_get_drvdata(dev); - struct dwc3_apple *appledwc; - int ret; - - if (!dwc) - return 0; - appledwc = to_dwc3_apple(dwc); - - guard(mutex)(&appledwc->lock); - - if (appledwc->state != DWC3_APPLE_SUSPENDED) - return 0; - - /* A failure leaves the controller in DWC3_APPLE_SUSPENDED, which remove() can tear down */ - ret = dwc3_apple_core_start(appledwc, DWC3_APPLE_HOST); - if (ret) - return ret; - - dwc3_apple_set_role(appledwc, DWC3_APPLE_HOST); - appledwc->state = DWC3_APPLE_HOST; - - return 0; -} - -static DEFINE_SIMPLE_DEV_PM_OPS(dwc3_apple_pm_ops, dwc3_apple_suspend, dwc3_apple_resume); - static struct platform_driver dwc3_apple_driver = { .probe = dwc3_apple_probe, .remove = dwc3_apple_remove, .driver = { .name = "dwc3-apple", .of_match_table = dwc3_apple_of_match, - .pm = pm_sleep_ptr(&dwc3_apple_pm_ops), }, }; From ee7c994c6066fd8a845281ab14610ffddfa69785 Mon Sep 17 00:00:00 2001 From: Wes Grimes Date: Fri, 2 Oct 2026 19:25:42 -0400 Subject: [PATCH 1008/2680] thunderbolt: apple: Accept PCIe-C kernel cold init on T600x/T602x ACIO only populates the tunneled PCIe-C host after an m1n1 handoff or for "apple,pciec-kernel-init" (t8103) ports. Also accept ports that the PCIe driver will cold-initialize itself, which it does only when booted with pcie_apple.tunnel_kernel_init=1, and word the disabled-tunnel messages so that they no longer name the m1n1 handoff as the only way. Signed-off-by: Wes Grimes --- drivers/thunderbolt/apple.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index c2969ae0043e0..b12f293255f4b 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -563,7 +563,10 @@ apple_cio_pcie_tunnel_is_preinitialized(struct device_node *parent) &status) && status == 1) return true; /* t8103 brings the port up itself. */ - return of_property_read_bool(child, "apple,pciec-kernel-init"); + if (of_property_read_bool(child, "apple,pciec-kernel-init")) + return true; + /* T600x/T602x without a handoff, when opted in. */ + return apple_pcie_tunnel_needs_cold_init(child); } return false; @@ -1017,7 +1020,7 @@ static int apple_cio_activate_pcie_tunnel_locked(struct apple_cio *acio) } if (!acio->pcie_tunnel_preinitialized) return dev_err_probe(acio->dev, -ENODEV, - "PCIe-C requires a successful m1n1 preinit handoff\n"); + "PCIe-C is not initialized by m1n1 or the kernel\n"); if (acio->pd_list->num_pds < 4 || !acio->pd_list->pd_links[3]) return dev_err_probe(acio->dev, -ENODEV, @@ -2393,7 +2396,7 @@ static int apple_cio_probe(struct platform_device *pdev) acio->pcie_tunnel_np); if (!acio->pcie_tunnel_preinitialized) { dev_warn(dev, - "PCIe-C tunnel disabled: m1n1 handoff is not initialized\n"); + "PCIe-C tunnel disabled: not initialized by m1n1 or the kernel\n"); } else { ret = apple_cio_map_pcie_intr2axi(acio); if (ret) From aac344986bb336fb4f69a86b4ea7464a480045cf Mon Sep 17 00:00:00 2001 From: Wes Grimes Date: Fri, 2 Oct 2026 19:27:18 -0400 Subject: [PATCH 1009/2680] tools: aurora-tb: Add a Thunderbolt PCIe test report A read-only script for testers of pcie_apple.tunnel_kernel_init on T600x/T602x: it checks the machine and parameter, lists the DART tunables per tunnel and the PCI and USB devices behind the tunnels, collects the relevant kernel log, and writes a markdown report. When a tunnel has no DART tunables, it prints the macOS ioreg command that reads them. Signed-off-by: Wes Grimes --- tools/aurora-tb/README.md | 24 ++++++ tools/aurora-tb/tb-pcie-report | 142 +++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 tools/aurora-tb/README.md create mode 100755 tools/aurora-tb/tb-pcie-report diff --git a/tools/aurora-tb/README.md b/tools/aurora-tb/README.md new file mode 100644 index 0000000000000..6ca8f3fae8dbb --- /dev/null +++ b/tools/aurora-tb/README.md @@ -0,0 +1,24 @@ +# Thunderbolt PCIe (PCIe-C) test report + +On M1 Pro/Max (T600x) and M2 Pro/Max (T602x) Macs, PCIe over +Thunderbolt is off unless the kernel is booted with +`pcie_apple.tunnel_kernel_init=1`. With it, the PCIe devices inside +Thunderbolt 3 docks and displays come up: for an Apple Studio Display, its +USB-C ports, camera, speakers and microphone. + +To test it: + +1. Add `pcie_apple.tunnel_kernel_init=1` to the kernel command line and + reboot. In GRUB, press `e` on the boot entry to add it for one boot. +2. Plug in a Thunderbolt 3 dock or display. USB-C hubs and DisplayPort-only + monitors do not use the PCIe tunnel and do not test it. +3. Run `tools/aurora-tb/tb-pcie-report` (with `sudo` if your user cannot + read the kernel log) and post the report it saves. + +Worth adding to the report: whether the device still works after +unplugging and replugging it, after power-cycling it, and after +`systemctl suspend`. + +A tunnel whose DART has no tunables in the device tree stays off. The +report then prints a read-only `ioreg` command to run in macOS on the same +Mac; its output lets those values be added. diff --git a/tools/aurora-tb/tb-pcie-report b/tools/aurora-tb/tb-pcie-report new file mode 100755 index 0000000000000..8b6d659d57299 --- /dev/null +++ b/tools/aurora-tb/tb-pcie-report @@ -0,0 +1,142 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Report whether PCIe over Thunderbolt (PCIe-C) works on this Mac. +# +# Plug a Thunderbolt 3 dock or display (for example an Apple Studio Display) +# into the Mac, then run this. It only reads system state. Run it with sudo +# if your user cannot read the kernel log. +# +# tb-pcie-report # wait for you to plug the device in +# tb-pcie-report --no-wait # report the current state now +set -u + +dt=/proc/device-tree +param=/sys/module/pcie_apple/parameters/tunnel_kernel_init + +say() { printf '%s\n' "$*"; } + +compat=$(tr '\0' ' ' <"$dt/compatible" 2>/dev/null) +model=$(tr -d '\0' <"$dt/model" 2>/dev/null) +case "$compat" in +*apple,t6000*|*apple,t6001*) chip=T600x ;; +*apple,t6020*|*apple,t6021*) chip=T602x ;; +*apple,t8103*) chip=t8103 ;; +*) chip=other ;; +esac + +if [ "$chip" = other ]; then + say "This Mac ($compat) is not covered: only M1, M1 Pro/Max and" + say "M2 Pro/Max have PCIe-C support in this kernel." + exit 1 +fi + +enabled=no +[ "$(cat "$param" 2>/dev/null)" = Y ] && enabled=yes +# The parameter file only exists once pcie_apple is loaded. +grep -qE '(^| )pcie_apple\.tunnel_kernel_init=(1|y|Y)( |$)' /proc/cmdline && + enabled=yes +if [ "$chip" != t8103 ] && [ "$enabled" = no ]; then + say "pcie_apple.tunnel_kernel_init is not enabled, so the tunnel stays off." + say "Add pcie_apple.tunnel_kernel_init=1 to the kernel command line and" + say "reboot: press 'e' on the boot entry in GRUB for a single boot, or add" + say "it to GRUB_CMDLINE_LINUX_DEFAULT in /etc/default/grub and run" + say "'sudo grub-mkconfig -o /boot/grub/grub.cfg'." + exit 1 +fi + +if [ "${1:-}" != --no-wait ]; then + say "Plug the Thunderbolt dock or display in now, wait about 20 seconds," + printf '%s' "then press Enter. " + read -r _ +fi + +if klog=$(journalctl -k -b -o short-monotonic --no-pager 2>/dev/null) && + [ -n "$klog" ]; then + : +elif klog=$(dmesg 2>/dev/null) && [ -n "$klog" ]; then + : +else + say "Cannot read the kernel log; run again with sudo." + exit 1 +fi + +out=$(mktemp "${TMPDIR:-/tmp}/tb-pcie-report.XXXXXX") || exit 1 + +{ + say "# PCIe-C report: $model" + say + say "- Machine: \`$compat\` ($chip)" + say "- Kernel: \`$(uname -r)\`" + say "- tunnel_kernel_init: $enabled" + say "- Date: $(date -u +%Y-%m-%dT%H:%MZ)" + say + say "## DART tunables" + say + missing=no + for t in "$dt"/soc/usb4-pcie-tunnel-*; do + # m1n1 only fills in ports that this Mac has. + [ -e "$t"/pcie@*/apple,tunable-rc ] || continue + for d in "$t"/iommu@*; do + if [ -e "$d/apple,tunable" ]; then + say "- $(basename "$t"): present" + else + say "- $(basename "$t"): **missing**" + missing=yes + fi + done + done + say + say "## Thunderbolt PCIe devices" + say + say '```' + found=no + for p in /sys/bus/pci/devices/*; do + addr=$(basename "$p") + case "$addr" in 0000:*) continue ;; esac + found=yes + driver=$(readlink "$p/driver" 2>/dev/null) && driver=${driver##*/} + printf '%s %s:%s class %s driver %s runtime %s\n' "$addr" \ + "$(cut -c3- "$p/vendor")" "$(cut -c3- "$p/device")" \ + "$(cut -c3- "$p/class")" "${driver:-none}" \ + "$(cat "$p/power/runtime_status" 2>/dev/null)" + done + [ "$found" = yes ] || say "(none)" + say '```' + say + say "## USB devices behind the tunnel" + say + say '```' + found=no + for u in /sys/bus/usb/devices/*; do + [ -e "$u/idVendor" ] || continue + case "$(readlink -f "$u")" in + */pci000[1-9]:*|*/pci00[1-9]?:*) ;; + *) continue ;; + esac + found=yes + printf '%s %s:%s %s %s\n' "$(basename "$u")" "$(cat "$u/idVendor")" \ + "$(cat "$u/idProduct")" "$(cat "$u/manufacturer" 2>/dev/null)" \ + "$(cat "$u/product" 2>/dev/null)" + done + [ "$found" = yes ] || say "(none)" + say '```' + say + say "## Kernel log" + say + say '```' + printf '%s\n' "$klog" | grep -E \ + 'PCIe-C|usb4-pcie-tunnel|apple-dart [0-9a-f]+21008000|xhci_hcd 000[1-9]|tunnel reset' | + grep -v -e 'RTKit: syslog' -e 'dependency cycle' | tail -n 80 + say '```' +} >"$out" + +cat "$out" +say +say "Report saved to $out" +if [ "$missing" = yes ]; then + say + say "Some tunnels have no DART tunables, so they stay off. If you can boot" + say "macOS on this Mac, please run this there and attach the output too:" + say + say " { sysctl -n hw.model; sw_vers -productVersion; ioreg -p IODeviceTree -l -w0 | grep -E 'dart-apciec|dart-tunables-instance'; } > ~/Desktop/apciec-dart.txt" +fi From 2cf44365922a668bc5d4240f49d8396527f2d47e Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 00:03:45 -0700 Subject: [PATCH 1010/2680] PCI: apple: Cold-init T600x/T602x PCIe-C ports by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PCIe over Thunderbolt on M1 Pro/Max and M2 Pro/Max no longer needs pcie_apple.tunnel_kernel_init=1. The parameter stays as an off switch: pcie_apple.tunnel_kernel_init=0 keeps these tunnels off. Run on J416s (M2 Pro, Apple Studio Display) and J314s (M1 Pro, CalDigit TS3 Plus: USB keyboard and mouse, camera, audio, Ethernet and a USB SSD behind the dock). M1 Max and M2 Max share those device trees and the same code path but have not run it yet. tb-pcie-report and its README now treat the tunnel as on unless the command line turns it off. [ryan: adapted to 7.2.10 — split by topic: only the tools/aurora-tb hunks of this radio commit; the rest is in 080-radio] Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- tools/aurora-tb/README.md | 17 ++++++++--------- tools/aurora-tb/tb-pcie-report | 15 ++++++--------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/tools/aurora-tb/README.md b/tools/aurora-tb/README.md index 6ca8f3fae8dbb..39542f0045612 100644 --- a/tools/aurora-tb/README.md +++ b/tools/aurora-tb/README.md @@ -1,18 +1,17 @@ # Thunderbolt PCIe (PCIe-C) test report -On M1 Pro/Max (T600x) and M2 Pro/Max (T602x) Macs, PCIe over -Thunderbolt is off unless the kernel is booted with -`pcie_apple.tunnel_kernel_init=1`. With it, the PCIe devices inside -Thunderbolt 3 docks and displays come up: for an Apple Studio Display, its -USB-C ports, camera, speakers and microphone. +On M1 Pro/Max (T600x) and M2 Pro/Max (T602x) Macs, the kernel brings up +PCIe over Thunderbolt itself, so the PCIe devices inside Thunderbolt 3 docks +and displays come up: for an Apple Studio Display, its USB-C ports, camera, +speakers and microphone; for a CalDigit TS3 Plus, its USB ports, Ethernet +and audio. `pcie_apple.tunnel_kernel_init=0` on the kernel command line +turns it off. To test it: -1. Add `pcie_apple.tunnel_kernel_init=1` to the kernel command line and - reboot. In GRUB, press `e` on the boot entry to add it for one boot. -2. Plug in a Thunderbolt 3 dock or display. USB-C hubs and DisplayPort-only +1. Plug in a Thunderbolt 3 dock or display. USB-C hubs and DisplayPort-only monitors do not use the PCIe tunnel and do not test it. -3. Run `tools/aurora-tb/tb-pcie-report` (with `sudo` if your user cannot +2. Run `tools/aurora-tb/tb-pcie-report` (with `sudo` if your user cannot read the kernel log) and post the report it saves. Worth adding to the report: whether the device still works after diff --git a/tools/aurora-tb/tb-pcie-report b/tools/aurora-tb/tb-pcie-report index 8b6d659d57299..e13cb7ded6266 100755 --- a/tools/aurora-tb/tb-pcie-report +++ b/tools/aurora-tb/tb-pcie-report @@ -30,17 +30,14 @@ if [ "$chip" = other ]; then exit 1 fi -enabled=no -[ "$(cat "$param" 2>/dev/null)" = Y ] && enabled=yes +enabled=yes +[ "$(cat "$param" 2>/dev/null)" = N ] && enabled=no # The parameter file only exists once pcie_apple is loaded. -grep -qE '(^| )pcie_apple\.tunnel_kernel_init=(1|y|Y)( |$)' /proc/cmdline && - enabled=yes +grep -qE '(^| )pcie_apple\.tunnel_kernel_init=(0|n|N)( |$)' /proc/cmdline && + enabled=no if [ "$chip" != t8103 ] && [ "$enabled" = no ]; then - say "pcie_apple.tunnel_kernel_init is not enabled, so the tunnel stays off." - say "Add pcie_apple.tunnel_kernel_init=1 to the kernel command line and" - say "reboot: press 'e' on the boot entry in GRUB for a single boot, or add" - say "it to GRUB_CMDLINE_LINUX_DEFAULT in /etc/default/grub and run" - say "'sudo grub-mkconfig -o /boot/grub/grub.cfg'." + say "pcie_apple.tunnel_kernel_init=0 is on the kernel command line, so the" + say "tunnel stays off. Remove it from the command line and reboot." exit 1 fi From de8ace834a88afb2f9138f8b4ac2be65ef9e5814 Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Sat, 3 Oct 2026 16:06:12 -0400 Subject: [PATCH 1011/2680] drm/apple: keep data-only USB4 changes local to their port MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A USB4 transition on an unrelated Type-C port queues mode invalidation and retraining of every live DCP route. The CD321x path also reports an unknown connector status for data-only USB4 attachments. A right-side port reset can consequently reconnect a Samsung display on another port. Remove both blanket notifications and their delayed retrain work. Actual HPD changes, display-route ownership changes and per-display recovery continue to use their existing paths. This needs hardware testing for shared-fabric stalls as well as unrelated-port hotplug. [ryan: adapted to 7.2.10 — split by topic: only the drivers/usb hunks of this dcp-display commit; the rest is in 200-dcp] --- drivers/usb/typec/tipd/core.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 1f7d168c63b22..b61629af92c43 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -744,9 +744,6 @@ static void cd321x_update_work(struct work_struct *work) TPS_DATA_STATUS_DP_PIN_ASSIGNMENT_MASK); bool dp_route_was_active = cd321x->display_route_active; bool dp_connected = st.data_status & TPS_DATA_STATUS_DP_CONNECTION; - bool usb4_started = - (st.data_status_changed & CD321X_DATA_STATUS_USB4_CONNECTION) && - (st.data_status & CD321X_DATA_STATUS_USB4_CONNECTION); bool dp_hpd = st.data_status & CD321X_DATA_STATUS_HPD_LEVEL; bool dp_hpd_changed = st.data_status_changed & CD321X_DATA_STATUS_HPD_LEVEL; @@ -901,17 +898,6 @@ static void cd321x_update_work(struct work_struct *work) cd321x->display_route_active && dp_hpd) drm_connector_oob_hotplug_event(cd321x->connector_fwnode, connector_status_connected); - /* - * Apple routes all Type-C ports through shared high-speed display fabric. - * Starting USB4 on one port can interrupt a DP stream on another without - * changing that display's HPD state. Ask the display driver to retrain the - * still-connected route after ACIO and the USB role are live. - */ - if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && - usb4_started && !dp_connected) - drm_connector_oob_hotplug_event(cd321x->connector_fwnode, - connector_status_unknown); - power_supply_changed(tps->psy); cd321x_pm_complete(&cd321x->pm); return; From bd8ca81b6567dffdb63556920ebff7c2915c325c Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Sat, 3 Oct 2026 16:58:25 -0400 Subject: [PATCH 1012/2680] usb: typec: defer CD321x revalidation while PCIe activation is busy ACIO returns -EAGAIN while its PCIe host is being populated or quiesced. CD321x treats this as a failed cable and forces a complete disconnect on the next status snapshot, resetting an otherwise unchanged USB4 session. Keep the bounded fresh-read retry budget, but only request a forced reconnect for non-deferred errors. Preserve any earlier genuine failure. Notify the Type-C consumer after asynchronous PCIe activation succeeds so an exhausted busy retry can read the current cable state again. Return -EAGAIN when an allegedly populated host is not yet available, rather than reporting false activation success. Propagate partner-registration errors and add regression tests for busy retries, completion after budget exhaustion and a busy retry following a real failure. --- drivers/thunderbolt/apple.c | 10 ++- drivers/usb/typec/tipd/cd321x-pm-test.c | 85 ++++++++++++++++++++----- drivers/usb/typec/tipd/cd321x-pm.h | 6 +- drivers/usb/typec/tipd/core.c | 12 ++-- 4 files changed, 88 insertions(+), 25 deletions(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index b12f293255f4b..65fbf2a365ede 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -1015,6 +1015,8 @@ static int apple_cio_activate_pcie_tunnel_locked(struct apple_cio *acio) "failed to restore PCIe-C after tunnel activation: %d\n", ret); put_device(&pcie_pdev->dev); + } else { + ret = -EAGAIN; } return ret; } @@ -1058,6 +1060,7 @@ static void apple_cio_pcie_tunnel_work(struct work_struct *work) struct apple_cio *acio = container_of(to_delayed_work(work), struct apple_cio, pcie_tunnel_work); + bool activated = false; int ret = 0; mutex_lock(&acio->pcie_tunnel_lock); @@ -1068,10 +1071,15 @@ static void apple_cio_pcie_tunnel_work(struct work_struct *work) if (ret) goto unlock; } - if (READ_ONCE(acio->pcie_tunnel_requested)) + if (READ_ONCE(acio->pcie_tunnel_requested)) { ret = apple_cio_activate_pcie_tunnel_locked(acio); + activated = !ret; + } unlock: mutex_unlock(&acio->pcie_tunnel_lock); + /* Wake a deferred Type-C check after asynchronous PCIe setup finishes. */ + if (activated) + typec_thunderbolt_switch_notify(acio->tbt_switch); if (ret && ret != -EAGAIN) dev_err(acio->dev, "deferred PCIe-C transition failed: %d\n", ret); } diff --git a/drivers/usb/typec/tipd/cd321x-pm-test.c b/drivers/usb/typec/tipd/cd321x-pm-test.c index 79b6797c73b9e..de9db7437dce2 100644 --- a/drivers/usb/typec/tipd/cd321x-pm-test.c +++ b/drivers/usb/typec/tipd/cd321x-pm-test.c @@ -31,7 +31,7 @@ static void cd321x_revalidation_errors_bounded_test(struct kunit *test) for (i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); - KUNIT_EXPECT_EQ(test, cd321x_pm_retry(&pm), + KUNIT_EXPECT_EQ(test, cd321x_pm_retry(&pm, true), i + 1 < CD321X_RESUME_ATTEMPTS); } /* Give up without permitting an update from the old cable snapshot. */ @@ -48,7 +48,7 @@ static void cd321x_activation_failure_rereads_test(struct kunit *test) KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); cd321x_pm_snapshot_ready(&pm); /* A PHY or ACIO error must not be treated as successful restoration. */ - KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm, true)); KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); cd321x_pm_complete(&pm); KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); @@ -81,11 +81,11 @@ static void cd321x_repeat_suspend_cancels_retry_test(struct kunit *test) cd321x_pm_prepare(&pm); KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); - KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm, true)); /* The next prepare wins over a delayed retry from the previous wake. */ cd321x_pm_prepare(&pm); KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); - KUNIT_EXPECT_FALSE(test, cd321x_pm_retry(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_retry(&pm, true)); KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); KUNIT_EXPECT_EQ(test, pm.attempts_left, (unsigned int)CD321X_RESUME_ATTEMPTS); @@ -104,7 +104,7 @@ static void cd321x_suspend_before_apply_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); cd321x_pm_complete(&pm); KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); - KUNIT_EXPECT_FALSE(test, cd321x_pm_retry(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_retry(&pm, true)); } static void cd321x_remove_drains_resume_test(struct kunit *test) @@ -115,7 +115,7 @@ static void cd321x_remove_drains_resume_test(struct kunit *test) KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); cd321x_pm_remove(&pm); KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); - KUNIT_EXPECT_FALSE(test, cd321x_pm_retry(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_retry(&pm, true)); KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); cd321x_pm_prepare(&pm); KUNIT_EXPECT_FALSE(test, cd321x_pm_resume(&pm)); @@ -133,7 +133,7 @@ static void cd321x_read_and_apply_share_budget_test(struct kunit *test) KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); if (i & 1) cd321x_pm_snapshot_ready(&pm); - KUNIT_EXPECT_EQ(test, cd321x_pm_retry(&pm), + KUNIT_EXPECT_EQ(test, cd321x_pm_retry(&pm, true), i + 1 < CD321X_RESUME_ATTEMPTS); } KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); @@ -148,7 +148,7 @@ static void cd321x_status_event_after_exhaustion_test(struct kunit *test) KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); for (i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); - cd321x_pm_retry(&pm); + cd321x_pm_retry(&pm, true); } /* A power/plug-only interrupt has not read the data or cable registers. */ KUNIT_ASSERT_TRUE(test, cd321x_pm_event(&pm)); @@ -168,12 +168,12 @@ static void cd321x_runtime_failure_revalidates_test(struct kunit *test) unsigned int i; /* A failed ordinary cable update has already consumed its IRQ changes. */ - KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm, true)); KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); for (i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); cd321x_pm_snapshot_ready(&pm); - KUNIT_EXPECT_EQ(test, cd321x_pm_retry(&pm), + KUNIT_EXPECT_EQ(test, cd321x_pm_retry(&pm, true), i + 1 < CD321X_RESUME_ATTEMPTS); } KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); @@ -210,7 +210,7 @@ static void cd321x_failed_resume_forces_reconnect_test(struct kunit *test) KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); cd321x_pm_snapshot_ready(&pm); /* Fresh cable data cannot override a provider's recorded resume failure. */ - KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm, true)); KUNIT_EXPECT_TRUE(test, pm.force_reconnect); /* A new suspend must preserve recovery that has not been applied yet. */ cd321x_pm_prepare(&pm); @@ -227,10 +227,10 @@ static void cd321x_stale_recovery_keeps_reconnect_test(struct kunit *test) struct cd321x_pm_state pm = {}; unsigned int i; - KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm, true)); for (i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); - cd321x_pm_retry(&pm); + cd321x_pm_retry(&pm, true); } KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); KUNIT_EXPECT_TRUE(test, pm.force_reconnect); @@ -258,7 +258,7 @@ static void cd321x_late_link_failure_test(struct kunit *test) KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); cd321x_pm_snapshot_ready(&pm); /* Only the provider's failed check requests a fresh forced reconnect. */ - KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm, true)); KUNIT_EXPECT_TRUE(test, pm.force_reconnect); KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); cd321x_pm_snapshot_ready(&pm); @@ -276,7 +276,7 @@ static void cd321x_link_event_during_final_apply_test(struct kunit *test) for (i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); if (i + 1 < CD321X_RESUME_ATTEMPTS) - KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm, true)); } cd321x_pm_snapshot_ready(&pm); KUNIT_ASSERT_EQ(test, pm.attempts_left, 0U); @@ -300,7 +300,7 @@ static void cd321x_link_reprobe_storm_bounded_test(struct kunit *test) for (attempt = 0; attempt < CD321X_RESUME_ATTEMPTS; attempt++) { KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); cd321x_pm_snapshot_ready(&pm); - KUNIT_EXPECT_EQ(test, cd321x_pm_retry(&pm), + KUNIT_EXPECT_EQ(test, cd321x_pm_retry(&pm, true), attempt + 1 < CD321X_RESUME_ATTEMPTS); } } @@ -367,7 +367,60 @@ static void cd321x_link_event_suspend_and_rearm_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); } +static void cd321x_busy_provider_preserves_connection_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + cd321x_pm_new_connection(&pm); + /* PCIe activation runs asynchronously after the Type-C switch returns. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm, false)); + KUNIT_EXPECT_FALSE(test, pm.force_reconnect); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_RUNNING); + KUNIT_EXPECT_FALSE(test, pm.force_reconnect); +} + +static void cd321x_busy_provider_completion_reopens_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int i; + + cd321x_pm_new_connection(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm, false)); + for (i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + cd321x_pm_retry(&pm, false); + } + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); + KUNIT_EXPECT_FALSE(test, pm.force_reconnect); + /* The PCIe worker's completion notification permits a fresh check. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_link_event(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_RUNNING); + KUNIT_EXPECT_FALSE(test, pm.force_reconnect); +} + +static void cd321x_busy_provider_keeps_real_failure_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm, true)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + /* Busy retries must not erase recovery requested by a real failure. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm, false)); + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); +} + static struct kunit_case cd321x_pm_cases[] = { + KUNIT_CASE(cd321x_busy_provider_preserves_connection_test), + KUNIT_CASE(cd321x_busy_provider_completion_reopens_test), + KUNIT_CASE(cd321x_busy_provider_keeps_real_failure_test), KUNIT_CASE(cd321x_late_link_failure_test), KUNIT_CASE(cd321x_link_event_during_final_apply_test), KUNIT_CASE(cd321x_link_reprobe_storm_bounded_test), diff --git a/drivers/usb/typec/tipd/cd321x-pm.h b/drivers/usb/typec/tipd/cd321x-pm.h index da764777ba36e..a96392f88bee4 100644 --- a/drivers/usb/typec/tipd/cd321x-pm.h +++ b/drivers/usb/typec/tipd/cd321x-pm.h @@ -95,18 +95,18 @@ static inline void cd321x_pm_snapshot_ready(struct cd321x_pm_state *pm) pm->phase = CD321X_PM_APPLY; } -static inline bool cd321x_pm_retry(struct cd321x_pm_state *pm) +static inline bool cd321x_pm_retry(struct cd321x_pm_state *pm, bool reconnect) { /* Runtime cable updates need the same bounded fresh-read recovery. */ if (pm->phase == CD321X_PM_RUNNING) { - pm->force_reconnect = true; + pm->force_reconnect |= reconnect; pm->phase = CD321X_PM_REVALIDATE; pm->attempts_left = CD321X_RESUME_ATTEMPTS; return true; } if (pm->phase != CD321X_PM_REVALIDATE && pm->phase != CD321X_PM_APPLY) return false; - pm->force_reconnect = true; + pm->force_reconnect |= reconnect; if (pm->attempts_left) { pm->phase = CD321X_PM_REVALIDATE; return true; diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index b61629af92c43..084bdc679ce70 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -692,7 +692,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, return ret; } -static void cd321x_retry_revalidation(struct cd321x *cd321x) +static void cd321x_retry_revalidation(struct cd321x *cd321x, bool reconnect) { lockdep_assert_held(&cd321x->tps.lock); if (cd321x->pm.phase != CD321X_PM_RUNNING && @@ -702,7 +702,7 @@ static void cd321x_retry_revalidation(struct cd321x *cd321x) /* Keep a later physical event retryable if the automatic budget expires. */ cd321x->state_valid = false; - if (cd321x_pm_retry(&cd321x->pm)) + if (cd321x_pm_retry(&cd321x->pm, reconnect)) mod_delayed_work(system_freezable_wq, &cd321x->resume_work, msecs_to_jiffies(CD321X_DEBOUNCE_DELAY_MS)); else @@ -872,6 +872,7 @@ static void cd321x_update_work(struct work_struct *work) tps->partner = typec_register_partner(tps->port, &desc); if (IS_ERR(tps->partner)) { + ret = PTR_ERR(tps->partner); dev_warn(tps->dev, "%s: failed to register partner\n", __func__); tps->partner = NULL; goto retry; @@ -903,7 +904,8 @@ static void cd321x_update_work(struct work_struct *work) return; retry: - cd321x_retry_revalidation(cd321x); + /* A deferred PCIe activation is not evidence of a failed cable. */ + cd321x_retry_revalidation(cd321x, ret != -EAGAIN); } static void cd321x_queue_status(struct cd321x *cd321x) @@ -969,7 +971,7 @@ static void cd321x_resume_work(struct work_struct *work) if (atomic_xchg(&cd321x->link_event, 0)) { if (cd321x_pm_link_event(&cd321x->pm)) { cd321x->state_valid = false; - dev_info(tps->dev, "revalidating cable after tunnel link failure\n"); + dev_info(tps->dev, "revalidating cable after tunnel state change\n"); } else { dev_warn_ratelimited(tps->dev, "tunnel link recovery budget exhausted\n"); @@ -997,7 +999,7 @@ static void cd321x_resume_work(struct work_struct *work) return; retry: - cd321x_retry_revalidation(cd321x); + cd321x_retry_revalidation(cd321x, true); } static void cd321x_suspend_prepare(struct tps6598x *tps) From 3bad19bb36594b9066a1df0b7058f9d3831c3d69 Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:01:40 -0400 Subject: [PATCH 1013/2680] thunderbolt: apple: drain activation notifications before unregister The PCIe activation worker now notifies the Type-C switch on successful completion. Disable and drain that work before unregistering the switch, otherwise a pending completion can dereference the released switch. Hold the ACIO lock while disabling work and closing the provider so a concurrent cable update cannot restart activation during removal. Disabling rather than just cancelling also prevents late NHI events from requeuing the notification worker during teardown. --- drivers/thunderbolt/apple.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 65fbf2a365ede..1a5a97925b6c3 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -2536,10 +2536,11 @@ static void apple_cio_remove(struct platform_device *pdev) struct apple_cio *acio = platform_get_drvdata(pdev); apple_pcie_tunnel_unregister_notifier(&acio->pcie_notifier); - typec_thunderbolt_switch_unregister(acio->tbt_switch); - cancel_delayed_work_sync(&acio->pcie_tunnel_work); guard(mutex)(&acio->lock); + /* The worker can notify this switch; stop current and future enqueues. */ + disable_delayed_work_sync(&acio->pcie_tunnel_work); + typec_thunderbolt_switch_unregister(acio->tbt_switch); if (acio->current_cable_info) apple_cio_stop(acio); } From 7fff836696e547e92e5f95aeb3e7f66fc8dcba59 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:44:13 +1100 Subject: [PATCH 1014/2680] dt-bindings: phy: apple,atcphy: describe T8140 DP setup lists Document the three bounded calibration lists used for T8140 DisplayPort setup. Describe their register regions and ordering, and reject these properties on other PHY variants. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/phy/apple,atcphy.yaml | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml index 2484121870400..fef0605343e8f 100644 --- a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml @@ -50,6 +50,9 @@ allOf: reg-names: maxItems: 5 apple,tunable-usb2phy-reg-dflt: false + apple,tunable-dp-common-pre: false + apple,tunable-dp-lpdptx: false + apple,tunable-dp-common-post: false $defs: apple,tunable: @@ -179,6 +182,27 @@ properties: Note that lane here refers to a USB RX and TX pair re-used for DisplayPort and not to an individual DisplayPort differential lane. + apple,tunable-dp-common-pre: + $ref: "#/$defs/apple,tunable" + maxItems: 5461 + description: > + T8140 DisplayPort core setup applied before the LPDPTX list. The + bootloader supplies device-specific values for the core register region. + + apple,tunable-dp-lpdptx: + $ref: "#/$defs/apple,tunable" + maxItems: 5461 + description: > + T8140 DisplayPort setup for the lpdptx register region, applied between + the common-pre and common-post lists. + + apple,tunable-dp-common-post: + $ref: "#/$defs/apple,tunable" + maxItems: 5461 + description: > + T8140 DisplayPort core setup applied after the LPDPTX list. All three + DisplayPort lists are needed before selecting a DisplayPort PHY mode. + apple,tunable-usb2phy-reg-dflt: $ref: "#/$defs/apple,tunable" description: > From 8492f277ee3d20357bb80a604715446edc93e81c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:44:13 +1100 Subject: [PATCH 1015/2680] phy: apple: apply the T8140 DisplayPort setup lists The T8140 variant currently returns from AUX setup without configuring it. The bootloader supplies three ordered DisplayPort setup lists, but the PHY driver does not read or apply them. Validate every list against its mapped register resource and apply the complete bundle at AUX setup. Reject DP selection if the bundle is missing or invalid, while keeping ordinary USB operation independent. Other PHY generations keep their existing setup paths. Drive-level training programming remains unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-dp-tunables.h | 13 ++++++ drivers/phy/apple/atc.c | 68 ++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 drivers/phy/apple/atc-dp-tunables.h diff --git a/drivers/phy/apple/atc-dp-tunables.h b/drivers/phy/apple/atc-dp-tunables.h new file mode 100644 index 0000000000000..6d0cb00158c71 --- /dev/null +++ b/drivers/phy/apple/atc-dp-tunables.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* Copyright 2026 Aurora Silicon */ +#ifndef APPLE_ATC_DP_TUNABLES_H +#define APPLE_ATC_DP_TUNABLES_H + +#include + +static inline bool atc_dp_list_length_valid(int length) +{ + return length >= 0 && length <= 65536 && length % 12 == 0; +} + +#endif diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index a0e055fb76137..26100cdacb0ae 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -46,6 +46,8 @@ #include #include +#include "atc-dp-tunables.h" + #define AUSPLL_FSM_CTRL 0x1014 #define AUSPLL_APB_CMD_OVERRIDE 0x2000 @@ -620,6 +622,7 @@ struct apple_atcphy; * @pipe_mux_clk: PIPE clock selection field mask * @pipe_mux_data: PIPE data selection field mask * @has_usb2phy_reg: The PHY has the secondary eUSB2 register bank (T8140) + * @has_t8140_dp: Uses the T8140 DisplayPort tunable bundle * @optional_tunables: The bootloader may leave out the common-a tunables (this * generation has none) and the SuperSpeed tunables; USB2 * still works without the latter @@ -634,6 +637,7 @@ struct atcphy_hw { u32 pipe_mux_data; bool has_usb2phy_reg; bool optional_tunables; + bool has_t8140_dp; }; /** @@ -646,9 +650,13 @@ struct atcphy_hw { * @tunables.lane_usb3: USB3 lane-specific tunables * @tunables.lane_dp: DisplayPort lane-specific tunables * @tunables.lane_usb4: USB4 lane-specific tunables + * @tunables.dp_common_pre: T8140 DisplayPort common setup + * @tunables.dp_lpdptx: T8140 DisplayPort LPDPTX setup + * @tunables.dp_common_post: T8140 DisplayPort common completion * @tunables.usb2phy_reg_dflt: Defaults for the secondary eUSB2 register bank * @hw: SoC-specific PHY description * @ss_tunables: The complete SuperSpeed tunable set was supplied + * @dp_tunables: The complete T8140 DisplayPort bundle was supplied * @fixed_usb2: The USB2 pairs go to a fixed hub on the USB controller, which * stays in host mode; the PHY provides USB2 from probe on * @typec_mode: Mode the Type-C mux last asked for; a fixed-hub port returns to @@ -675,6 +683,7 @@ struct atcphy_hw { * @regs.lpdptx: DisplayPort registers * @res: Resources for memory-mapped registers, used to verify that tunables aren't out of bounds * @res.core: Core register resource + * @res.lpdptx: DisplayPort register resource * @res.axi2af: AXI to Apple Fabric interface resource * @res.usb2phy_reg: Secondary eUSB2 register bank resource * @phys: PHY instances @@ -698,10 +707,14 @@ struct apple_atcphy { struct apple_tunable *lane_dp[2]; struct apple_tunable *lane_usb4[2]; struct apple_tunable *usb2phy_reg_dflt; + struct apple_tunable *dp_common_pre; + struct apple_tunable *dp_lpdptx; + struct apple_tunable *dp_common_post; } tunables; const struct atcphy_hw *hw; bool ss_tunables; + bool dp_tunables; bool fixed_usb2; enum atcphy_mode typec_mode; bool host_active; @@ -728,6 +741,7 @@ struct apple_atcphy { struct { struct resource *core; + struct resource *lpdptx; struct resource *axi2af; struct resource *usb2phy_reg; } res; @@ -1486,6 +1500,13 @@ static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode static void atcphy_enable_dp_aux(struct apple_atcphy *atcphy) { + if (atcphy->hw->has_t8140_dp) { + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_pre); + apple_tunable_apply(atcphy->regs.lpdptx, atcphy->tunables.dp_lpdptx); + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_post); + return; + } + /* FIXME */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8122) return; @@ -2595,6 +2616,9 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) u32 reg; lockdep_assert_held(&atcphy->lock); + if (atcphy->hw->has_t8140_dp && atcphy_modes[mode].enable_dp_aux && + !atcphy->dp_tunables) + return -EOPNOTSUPP; /* the mode change below may power the PHY down: stop the tunnel clock first */ atc_tunnel_stop_t8103(atcphy); atc_tunnel_restore(atcphy); @@ -3119,6 +3143,9 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta case APPLE_ATCPHY_MODE_USB3: case APPLE_ATCPHY_MODE_USB3_DP: case APPLE_ATCPHY_MODE_DP: + if (atcphy->hw->has_t8140_dp && atcphy_modes[target_mode].enable_dp_aux && + !atcphy->dp_tunables) + return -EOPNOTSUPP; /* Without the lane tunables the SuperSpeed lanes are not calibrated */ if (!atcphy->ss_tunables) return -EOPNOTSUPP; @@ -3183,6 +3210,37 @@ static int atcphy_probe_mux(struct apple_atcphy *atcphy) return devm_add_action_or_reset(atcphy->dev, atcphy_typec_mux_unregister, mux); } +static int atcphy_load_t8140_dp(struct apple_atcphy *atcphy) +{ + struct { + const char *name; + struct resource *resource; + struct apple_tunable **out; + } lists[] = { + { "apple,tunable-dp-common-pre", atcphy->res.core, + &atcphy->tunables.dp_common_pre }, + { "apple,tunable-dp-lpdptx", atcphy->res.lpdptx, + &atcphy->tunables.dp_lpdptx }, + { "apple,tunable-dp-common-post", atcphy->res.core, + &atcphy->tunables.dp_common_post }, + }; + if (!of_machine_is_compatible("apple,j700")) + return -EOPNOTSUPP; + for (unsigned int i = 0; i < ARRAY_SIZE(lists); i++) { + struct property *property = of_find_property(atcphy->np, lists[i].name, NULL); + + if (!property || !atc_dp_list_length_valid(property->length)) + return -EINVAL; + *lists[i].out = devm_apple_tunable_parse(atcphy->dev, atcphy->np, + lists[i].name, lists[i].resource); + if (IS_ERR(*lists[i].out)) + return PTR_ERR(*lists[i].out); + } + + atcphy->dp_tunables = true; + return 0; +} + static int atcphy_load_tunables(struct apple_atcphy *atcphy) { size_t tunable_count; @@ -3247,6 +3305,13 @@ static int atcphy_load_tunables(struct apple_atcphy *atcphy) atcphy->tunables.usb2phy_reg_dflt = tunable; } + if (atcphy->hw->has_t8140_dp) { + int ret = atcphy_load_t8140_dp(atcphy); + + if (ret) + dev_warn(atcphy->dev, "DisplayPort tunable bundle unavailable: %d\n", ret); + } + atcphy->ss_tunables = !ss_missing; if (ss_missing) dev_warn(atcphy->dev, "SuperSpeed tunables missing, USB2 only\n"); @@ -3262,7 +3327,7 @@ static int atcphy_map_resources(struct platform_device *pdev, struct apple_atcph struct resource **res; } resources[] = { { "core", &atcphy->regs.core, &atcphy->res.core }, - { "lpdptx", &atcphy->regs.lpdptx, NULL }, + { "lpdptx", &atcphy->regs.lpdptx, &atcphy->res.lpdptx }, { "axi2af", &atcphy->regs.axi2af, &atcphy->res.axi2af }, { "usb2phy", &atcphy->regs.usb2phy, NULL }, { "pipehandler", &atcphy->regs.pipehandler, NULL }, @@ -3434,6 +3499,7 @@ static const struct atcphy_hw atcphy_hw_t8140 = { .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, .has_usb2phy_reg = true, + .has_t8140_dp = true, .optional_tunables = true, }; From 5fa916ca407bd7b74179f409f72b8e23516aa776 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:54:12 +1100 Subject: [PATCH 1016/2680] drm/apple: open the version-two DPTX controller interfaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The experimental external profile uses AFKV2 for its DPTX port, but the companion controller endpoint still uses the legacy path. Its interfaces are discovered without receiving the version-two OPEN handshake. Select AFKV2 for the controller before starting it, so controller publication performs the same OPEN-before-ready sequence as the port. Add a one-time PHY setup message to distinguish successful calibration application from controller activation during hardware qualification. [ryan: adapted to 7.2.10 — split by topic: only the drivers/phy/apple hunks of this dcp-display commit; the rest is in 200-dcp] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 26100cdacb0ae..5cdfefa74e085 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1504,6 +1504,7 @@ static void atcphy_enable_dp_aux(struct apple_atcphy *atcphy) apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_pre); apple_tunable_apply(atcphy->regs.lpdptx, atcphy->tunables.dp_lpdptx); apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_post); + dev_info_once(atcphy->dev, "Applied T8140 DisplayPort setup lists\n"); return; } From 9608b5f177357a71ac0d8dced42200add5e71143 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:12:31 +1100 Subject: [PATCH 1017/2680] phy: apple: sequence J700 AUX power separately from calibration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit J700 reaches DisplayPort mode with its setup-list values present, but readback shows AUX still clamped and both supply overrides unset. The calibration lists do not perform the AUX power transition. Request the small and large supplies in order, release the clamp, then release analog shutdown. Check each masked state with a bounded poll. Track incomplete startup so a later shutdown can unwind it, and refuse to stop the domain if AUX cleanup fails. Leave ordinary USB operation and other PHY generations on their existing paths. Add host tests of the production transition helpers with register mocks, covering phase ordering, unrelated bits and failure at every stage. These pass with AddressSanitizer and UBSan; hardware qualification is pending. [ryan: adapted to 7.2.10 — split by topic: the MAINTAINERS lines are carried by 900-config-tooling; the tools/diagnostics hunks are dropped: none of those files exists at aurora-wip] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 94 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 7 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 5cdfefa74e085..fa95fbab09653 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -657,6 +657,7 @@ struct atcphy_hw { * @hw: SoC-specific PHY description * @ss_tunables: The complete SuperSpeed tunable set was supplied * @dp_tunables: The complete T8140 DisplayPort bundle was supplied + * @aux_power_requested: AUX power-up began and needs matching shutdown * @fixed_usb2: The USB2 pairs go to a fixed hub on the USB controller, which * stays in host mode; the PHY provides USB2 from probe on * @typec_mode: Mode the Type-C mux last asked for; a fixed-hub port returns to @@ -715,6 +716,7 @@ struct apple_atcphy { const struct atcphy_hw *hw; bool ss_tunables; bool dp_tunables; + bool aux_power_requested; bool fixed_usb2; enum atcphy_mode typec_mode; bool host_active; @@ -1498,19 +1500,81 @@ static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode } } -static void atcphy_enable_dp_aux(struct apple_atcphy *atcphy) +#define J700_AUX_ANALOG_CONTROL 0x16000 +#define J700_AUX_SUPPLY_CONTROL 0x16400 +#define J700_AUX_ANALOG_OFF BIT(0) +#define J700_AUX_CLAMP_SELECT BIT(8) +#define J700_AUX_CLAMP_ENABLE BIT(9) + +static int j700_aux_update(struct apple_atcphy *atcphy, u32 offset, u32 mask, u32 value) +{ + u32 observed; + + core_mask32(atcphy, offset, mask, value); + usleep_range(10, 20); + return readl_poll_timeout_atomic(atcphy->regs.core + offset, observed, + (observed & mask) == value, 1, 100); +} + +static int j700_aux_enable(struct apple_atcphy *atcphy) +{ + u32 clamp = J700_AUX_CLAMP_SELECT | J700_AUX_CLAMP_ENABLE; + int ret; + + /* Even an incomplete power-up must be unwound before the domain is stopped. */ + atcphy->aux_power_requested = true; + for (unsigned int stage = 0; stage < 2; stage++) { + u32 fields = 3U << (2 * stage); + + ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, fields, fields); + if (ret) + return ret; + } + ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, clamp, J700_AUX_CLAMP_SELECT); + if (ret) + return ret; + + return j700_aux_update(atcphy, J700_AUX_ANALOG_CONTROL, J700_AUX_ANALOG_OFF, 0); +} + +static int j700_aux_disable(struct apple_atcphy *atcphy) +{ + u32 clamp = J700_AUX_CLAMP_SELECT | J700_AUX_CLAMP_ENABLE; + int ret; + + if (!atcphy->aux_power_requested) + return 0; + ret = j700_aux_update(atcphy, J700_AUX_ANALOG_CONTROL, + J700_AUX_ANALOG_OFF, J700_AUX_ANALOG_OFF); + if (ret) + return ret; + ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, clamp, clamp); + if (ret) + return ret; + for (unsigned int remaining = 2; remaining; remaining--) { + u32 override = BIT(2 * remaining - 1); + + ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, override, 0); + if (ret) + return ret; + } + atcphy->aux_power_requested = false; + return 0; +} + +static int atcphy_enable_dp_aux(struct apple_atcphy *atcphy) { if (atcphy->hw->has_t8140_dp) { apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_pre); apple_tunable_apply(atcphy->regs.lpdptx, atcphy->tunables.dp_lpdptx); apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_post); dev_info_once(atcphy->dev, "Applied T8140 DisplayPort setup lists\n"); - return; + return j700_aux_enable(atcphy); } /* FIXME */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8122) - return; + return 0; core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N); core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N_OV); @@ -1560,10 +1624,14 @@ static void atcphy_enable_dp_aux(struct apple_atcphy *atcphy) FIELD_PREP(LPDPTX_AUX_MARGIN_RCAL_TXSWING, 12)); atcphy->dp_link_rate = -1; + return 0; } -static void atcphy_disable_dp_aux(struct apple_atcphy *atcphy) +static int atcphy_disable_dp_aux(struct apple_atcphy *atcphy) { + if (atcphy->hw->has_t8140_dp) + return j700_aux_disable(atcphy); + /* FIXME */ if (atcphy->hw->gen != ATCPHY_GENERATION_T8122) { set32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_AUX_PWN_DOWN); @@ -1578,6 +1646,7 @@ static void atcphy_disable_dp_aux(struct apple_atcphy *atcphy) core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPRX_PCLK_ENABLE); core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK2_ENABLE); + return 0; } static int atcphy_dp_configure_lane(struct apple_atcphy *atcphy, enum atcphy_lane lane, @@ -1944,7 +2013,9 @@ static int atcphy_power_off(struct apple_atcphy *atcphy) u32 reg; int ret; - atcphy_disable_dp_aux(atcphy); + ret = atcphy_disable_dp_aux(atcphy); + if (ret) + return ret; /* Enable all reset lines */ core_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_PHY_RESET_N); @@ -2620,6 +2691,12 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) if (atcphy->hw->has_t8140_dp && atcphy_modes[mode].enable_dp_aux && !atcphy->dp_tunables) return -EOPNOTSUPP; + if (atcphy->hw->has_t8140_dp && !atcphy_modes[mode].enable_dp_aux && + atcphy->aux_power_requested) { + ret = j700_aux_disable(atcphy); + if (ret) + return ret; + } /* the mode change below may power the PHY down: stop the tunnel clock first */ atc_tunnel_stop_t8103(atcphy); atc_tunnel_restore(atcphy); @@ -2638,8 +2715,11 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) atcphy->hw->configure_post_tunable(atcphy, mode); /* Setup AUX channel if DP altmode is requested */ - if (atcphy_modes[mode].enable_dp_aux) - atcphy_enable_dp_aux(atcphy); + if (atcphy_modes[mode].enable_dp_aux) { + ret = atcphy_enable_dp_aux(atcphy); + if (ret) + return ret; + } /* Enable clocks and configure lanes */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8103) { From 0b53fcc577d23a49d8dd0ee7f6165ac225c83b5d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:19:18 +1100 Subject: [PATCH 1018/2680] phy: apple: park the J700 DP lane before link training MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The selected DP lane still reports the active power-control nibble after its protocol changes, with both receive overrides clear. The J700 path requires this lane to reach its quiescent state before link training. Assert only the selected lane's receive overrides, then sequence its power-control field into idle with bounded readback checks. Preserve the USB lane and unrelated bits. Return failures to the mux caller instead of publishing a completed mode transition. Host tests of the production helpers cover both lanes, invalid indices, unrelated-bit preservation and a failed write at every phase, under AddressSanitizer and UBSan. Hardware qualification remains pending. [ryan: adapted to 7.2.10 — split by topic: the tools/diagnostics hunks are dropped: none of those files exists at aurora-wip] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 88 ++++++++++++++++++++++++++++++++--------- 1 file changed, 69 insertions(+), 19 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index fa95fbab09653..d8d4951521bc2 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1444,7 +1444,50 @@ static void atcphy_setup_pipehandler(struct apple_atcphy *atcphy) atcphy_enable_dummy_phy(atcphy); } -static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode mode) +#define J700_PMA_RECEIVER_CONTROL 0x000c +#define J700_PMA_POWER_CONTROL 0x0200 + +static int j700_field_update(struct apple_atcphy *atcphy, u32 offset, u32 mask, u32 value) +{ + u32 observed; + + core_mask32(atcphy, offset, mask, value); + usleep_range(10, 20); + return readl_poll_timeout_atomic(atcphy->regs.core + offset, observed, + (observed & mask) == value, 1, 100); +} + +static int j700_pma_idle(struct apple_atcphy *atcphy, unsigned int lane) +{ + u32 field; + int ret; + + if (lane >= 2) + return -EINVAL; + for (unsigned int supply = 0; supply < 2; supply++) { + field = BIT(2 + lane + 4 * supply); + ret = j700_field_update(atcphy, J700_PMA_RECEIVER_CONTROL, field, field); + if (ret) + return ret; + } + field = BIT(4 * lane + 3); + ret = j700_field_update(atcphy, J700_PMA_POWER_CONTROL, field, 0); + if (ret) + return ret; + field = BIT(4 * lane); + ret = j700_field_update(atcphy, J700_PMA_POWER_CONTROL, field, field); + if (ret) + return ret; + for (unsigned int control = 1; control <= 2; control++) { + field = BIT(4 * lane + control); + ret = j700_field_update(atcphy, J700_PMA_POWER_CONTROL, field, 0); + if (ret) + return ret; + } + return 0; +} + +static int atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode mode) { const struct atcphy_mode_configuration *mode_cfg = atcphy_get_mode_config(atcphy, mode); @@ -1471,9 +1514,22 @@ static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode else core_clear32(atcphy, atcphy->hw->aciophy_crossbar, ACIOPHY_CROSSBAR_DP_BOTH_PMA); + if (atcphy->hw->has_t8140_dp) { + for (unsigned int lane = 0; lane < 2; lane++) { + int ret; + + if (!mode_cfg->dp_lane[lane]) + continue; + ret = j700_pma_idle(atcphy, lane); + if (ret) + return ret; + } + return 0; + } + /* unclear if we need the remainder for t8122 */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8122) - return; + return 0; if (mode_cfg->dp_lane[0]) { core_set32(atcphy, LN0_AUSPMA_RX_TOP + LN_AUSPMA_RX_TOP_PMAFSM, @@ -1498,6 +1554,7 @@ static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode LN_AUSPMA_RX_TOP_PMAFSM_PCS_OV); udelay(10); } + return 0; } #define J700_AUX_ANALOG_CONTROL 0x16000 @@ -1506,15 +1563,6 @@ static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode #define J700_AUX_CLAMP_SELECT BIT(8) #define J700_AUX_CLAMP_ENABLE BIT(9) -static int j700_aux_update(struct apple_atcphy *atcphy, u32 offset, u32 mask, u32 value) -{ - u32 observed; - - core_mask32(atcphy, offset, mask, value); - usleep_range(10, 20); - return readl_poll_timeout_atomic(atcphy->regs.core + offset, observed, - (observed & mask) == value, 1, 100); -} static int j700_aux_enable(struct apple_atcphy *atcphy) { @@ -1526,15 +1574,15 @@ static int j700_aux_enable(struct apple_atcphy *atcphy) for (unsigned int stage = 0; stage < 2; stage++) { u32 fields = 3U << (2 * stage); - ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, fields, fields); + ret = j700_field_update(atcphy, J700_AUX_SUPPLY_CONTROL, fields, fields); if (ret) return ret; } - ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, clamp, J700_AUX_CLAMP_SELECT); + ret = j700_field_update(atcphy, J700_AUX_SUPPLY_CONTROL, clamp, J700_AUX_CLAMP_SELECT); if (ret) return ret; - return j700_aux_update(atcphy, J700_AUX_ANALOG_CONTROL, J700_AUX_ANALOG_OFF, 0); + return j700_field_update(atcphy, J700_AUX_ANALOG_CONTROL, J700_AUX_ANALOG_OFF, 0); } static int j700_aux_disable(struct apple_atcphy *atcphy) @@ -1544,17 +1592,17 @@ static int j700_aux_disable(struct apple_atcphy *atcphy) if (!atcphy->aux_power_requested) return 0; - ret = j700_aux_update(atcphy, J700_AUX_ANALOG_CONTROL, - J700_AUX_ANALOG_OFF, J700_AUX_ANALOG_OFF); + ret = j700_field_update(atcphy, J700_AUX_ANALOG_CONTROL, + J700_AUX_ANALOG_OFF, J700_AUX_ANALOG_OFF); if (ret) return ret; - ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, clamp, clamp); + ret = j700_field_update(atcphy, J700_AUX_SUPPLY_CONTROL, clamp, clamp); if (ret) return ret; for (unsigned int remaining = 2; remaining; remaining--) { u32 override = BIT(2 * remaining - 1); - ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, override, 0); + ret = j700_field_update(atcphy, J700_AUX_SUPPLY_CONTROL, override, 0); if (ret) return ret; } @@ -2726,7 +2774,9 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) core_set32(atcphy, CIO3PLL_CLK_CTRL, CIO3PLL_CLK_PCLK_EN); core_set32(atcphy, CIO3PLL_CLK_CTRL, CIO3PLL_CLK_REFCLK_EN); } - atcphy_configure_lanes(atcphy, mode); + ret = atcphy_configure_lanes(atcphy, mode); + if (ret) + return ret; /* Take the USB3 PHY out of reset */ core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_PHY_RESET_N); From 005b99faaf89e012ffb1d9e9ca38932cd4d3c152 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:34:02 +1100 Subject: [PATCH 1019/2680] dt-bindings: phy: apple: describe T8140 DisplayPort drive presets Document the firmware-provided fallback and per-rate drive tables. Each has sixteen entries indexed by voltage swing and pre-emphasis; other PHYs do not accept these properties. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/phy/apple,atcphy.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml index fef0605343e8f..3e36244ef4d5b 100644 --- a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml @@ -53,6 +53,8 @@ allOf: apple,tunable-dp-common-pre: false apple,tunable-dp-lpdptx: false apple,tunable-dp-common-post: false + patternProperties: + "^apple,dp-training-table(-rbr|-hbr|-hbr2|-hbr3)?$": false $defs: apple,tunable: @@ -210,6 +212,16 @@ properties: T8140, copied from Apple's Device Tree by the bootloader. Unlike the other tunables these are applied to the usb2phy-reg region. +patternProperties: + "^apple,dp-training-table(-rbr|-hbr|-hbr2|-hbr3)?$": + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 16 + maxItems: 16 + description: > + Firmware-provided T8140 drive presets indexed by voltage swing times + four plus pre-emphasis. The unsuffixed table is the required fallback; + a link-rate suffix selects an optional rate-specific override. + required: - compatible - reg From de2f726294650a0e40974a17b3b9541d1925f5b5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:34:02 +1100 Subject: [PATCH 1020/2680] phy: apple: implement bounded J700 DisplayPort main-link training MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After correcting the crossbar route, J700 reaches SET_LINK_RATE but the legacy PHY path waits for the wrong command-ready register. Add an isolated T8140 main-link sequence, firmware drive presets, and cleanup ownership that survives failed setup or shutdown. Preserve the other PMA used for USB. Handle the version-two group-one drive command with complete bounds and range checks before changing the PHY. Cache training values only after a successful PHY operation and retain the legacy endpoint behavior. Hardware register facts were audited with explicit user authorization. The implementation and tests are independently written. The copy audit matched only three hardware-field expressions and standard C test headers. No private implementation patch was applied. Validation: 29 PHY state scenarios, 112 callback boundary cases, 100 group routing cases, 905 AFK assertions and production owned-buffer/AUX tests under ASan/UBSan; schema validation and strict checkpatch. Hardware qualification remains pending. [ryan: adapted to 7.2.10 — split by topic: the MAINTAINERS lines are carried by 900-config-tooling; the drm/apple hunks are carried by 200-dcp; the tools/diagnostics hunks are dropped: none of those files exists at aurora-wip] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-j700-dp.h | 333 ++++++++++++++++++++++++++++++++ drivers/phy/apple/atc.c | 45 ++++- 2 files changed, 376 insertions(+), 2 deletions(-) create mode 100644 drivers/phy/apple/atc-j700-dp.h diff --git a/drivers/phy/apple/atc-j700-dp.h b/drivers/phy/apple/atc-j700-dp.h new file mode 100644 index 0000000000000..185551e5a589e --- /dev/null +++ b/drivers/phy/apple/atc-j700-dp.h @@ -0,0 +1,333 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* Copyright 2026 Aurora Silicon */ +#ifndef APPLE_ATC_J700_DP_H +#define APPLE_ATC_J700_DP_H + +/* Register facts from the authorized J700 hardware audit. */ +struct j700_dp_step { + u32 offset, mask, value; + u16 delay_ns; +}; + +static void j700_dp_steps(struct apple_atcphy *p, u32 base, + const struct j700_dp_step *steps, size_t count) +{ + for (size_t i = 0; i < count; i++) { + core_mask32(p, base + steps[i].offset, steps[i].mask, steps[i].value); + if (steps[i].delay_ns) + ndelay(steps[i].delay_ns); + } +} + +static int j700_dp_wait(struct apple_atcphy *p, u32 offset, u32 mask, bool set) +{ + u32 value; + + return readl_poll_timeout_atomic(p->regs.core + offset, value, + !!(value & mask) == set, 1, 100); +} + +static void j700_dp_power(struct apple_atcphy *p, unsigned int lane) +{ + unsigned int shift = 4 * lane; + const u32 bits[] = { BIT(2), BIT(1), BIT(0), BIT(3) }; + + for (unsigned int i = 0; i < ARRAY_SIZE(bits); i++) { + u32 mask = bits[i] << shift; + + core_mask32(p, 0x200, mask, i == 2 ? 0 : mask); + udelay(1); + } + for (unsigned int bank = 0; bank < 2; bank++) { + core_clear32(p, 0xc, BIT(6 - 4 * bank + lane)); + udelay(1); + } +} + +static void j700_dp_output_power(struct apple_atcphy *p, unsigned int lane, bool tx) +{ + u32 reg = tx ? 0x10c : 0xc; + + core_set32(p, reg, (BIT(4) | BIT(6)) << lane); + udelay(1); + core_set32(p, reg, (BIT(0) | BIT(2)) << lane); + udelay(1); + if (!tx) { + core_set32(p, 0x8, (BIT(10) | BIT(12)) << lane); + udelay(1); + core_set32(p, 0x8, (BIT(6) | BIT(8)) << lane); + udelay(1); + } + reg = tx ? 0x10c : 0x8; + core_clear32(p, reg, BIT(tx ? 8 : 14) << lane); + core_set32(p, reg, BIT(tx ? 10 : 16) << lane); + udelay(1); +} + +static void j700_dp_equalize(struct apple_atcphy *p, unsigned int lane, + bool tx, u32 preset) +{ + u32 base = (tx ? 0xc000 : 0x9000) + lane * 0x7000; + u32 swing = ((preset & 3) << 14) | ((preset >> 2) & 15); + u32 taps = ((preset >> 6) & 0x3ffff) * 2 + 1; + + if (tx) + core_set32(p, base + 0xd0, BIT(20)); + core_mask32(p, base + (tx ? 0x50 : 0x210), 0xc00f, swing); + core_set32(p, base + (tx ? 0xd4 : 0x248), BIT(28)); + core_mask32(p, base + (tx ? 0xd0 : 0x244), 0x7ffff, taps); +} + +static void j700_dp_transmitter(struct apple_atcphy *p, unsigned int lane, int rate) +{ + static const struct j700_dp_step start[] = { + { 0x44, ~0U, 0x0aafb800, 500 }, + { 0x44, 0x1000000, 0x1000000, 250 }, + { 0x44, 0x4000000, 0x4000000, 250 }, + { 0x44, 0x7c0, 0x300, 250 }, + { 0x44, 0x6c000, 0, 250 }, + }; + static const struct j700_dp_step divide[] = { + { 0x44, 0x100000, 0x100000, 250 }, + { 0x44, 0x400000, 0x400000, 0 }, + }; + static const struct j700_dp_step finish[] = { + { 0x04, 0x33000000, 0x32000000, 0 }, + { 0x10, 0x600, 0x600, 0 }, + { 0x1c, 0xc00000, 0x800000, 0 }, + }; + u32 base = 0xd000 + lane * 0x7000; + + j700_dp_output_power(p, lane, true); + j700_dp_steps(p, base, start, ARRAY_SIZE(start)); + j700_dp_equalize(p, lane, true, 0); + if (rate != ATCPHY_DP_LINK_RATE_HBR3) + j700_dp_steps(p, base, divide, ARRAY_SIZE(divide)); + j700_dp_steps(p, base, finish, ARRAY_SIZE(finish)); + core_mask32(p, base + 0xc, 0x7000000, + rate == ATCPHY_DP_LINK_RATE_RBR ? 0x5000000 : 0x4000000); +} + +static void j700_dp_receiver(struct apple_atcphy *p, unsigned int lane, int rate) +{ + static const struct j700_dp_step setup[] = { + { 0xa4, 3, 2, 0 }, { 0xa8, 0x18000, 0x10000, 0 }, + { 0x6c, 0x18000, 0x10000, 0 }, + { 0xd4, 0x1f1f80, 0x121180, 0 }, { 0x64, 0xfc00, 0xb800, 0 }, + { 0xd8, 7, 4, 0 }, { 0xdc, 0x18, 0, 0 }, + { 0xd4, 0xe000, 0xc000, 100 }, + }; + static const struct j700_dp_step settle[] = { + { 0x6c, 0x600, 0x400, 100 }, { 0xd4, 0xd0000, 0x50000, 0 }, + { 0xd8, 3, 1, 500 }, { 0xd4, 0xc0000, 0, 200 }, + { 0x68, 0x180, 0x180, 200 }, { 0x68, 0x60, 0x60, 200 }, + { 0xd8, 3, 3, 1000 }, { 0xd8, 3, 2, 1000 }, + { 0xd8, 3, 0, 750 }, { 0xa8, 0x60000, 0x60000, 1000 }, + { 0x6c, 0x6000, 0x4000, 0 }, + }; + u32 top = 0x9000 + lane * 0x7000; + u32 base = top + 0x2000; + + core_set32(p, top + 0x160, 1); + core_set32(p, top + 0x1dc, 0x18000); + j700_dp_steps(p, base, setup, ARRAY_SIZE(setup)); + j700_dp_output_power(p, lane, false); + core_set32(p, base + 0x6c, BIT(11)); + core_mask32(p, top + 0x1d4, 3, 1); + ndelay(10); + core_set32(p, top + 0x1d4, 2); + j700_dp_steps(p, base, settle, ARRAY_SIZE(settle)); + j700_dp_equalize(p, lane, false, 0x3f000); + core_mask32(p, base + 0xa8, 0x380000, + rate == ATCPHY_DP_LINK_RATE_RBR ? 0x280000 : 0x200000); + core_mask32(p, base + 0xa8, 0xc00000, 0x800000); + ndelay(50); + core_set32(p, base + 0x30, 0xc00); + ndelay(100); +} + +static void j700_dp_outputs_off(struct apple_atcphy *p, unsigned int lane) +{ + static const struct j700_dp_step tx[] = { + { 0x1c, 0xc00000, 0xc00000, 0 }, { 0x10, 0x600, 0x400, 0 }, + { 0x04, 0x30000000, 0x20000000, 250 }, + { 0x44, 0x100000, 0, 250 }, { 0x44, 0x400000, 0, 250 }, + { 0x44, 0x4000000, 0, 250 }, { 0x44, 0x1000000, 0, 250 }, + { 0x44, 0x1000, 0, 0 }, + }; + static const struct j700_dp_step rx[] = { + { 0x30, 0xc00, 0x800, 50 }, { 0xa8, 0xc00000, 0xc00000, 500 }, + { 0xa8, 0x60000, 0x40000, 0 }, { 0x68, 0x60, 0x40, 200 }, + { 0x68, 0x180, 0x100, 200 }, { 0xd4, 0x10000, 0, 100 }, + { 0x6c, 0x600, 0x600, 0 }, + }; + const u32 tx_bits[] = { BIT(8), BIT(0), BIT(4) }; + const u32 rx_bits[] = { BIT(14), BIT(6), BIT(10) }; + + j700_dp_steps(p, 0xd000 + lane * 0x7000, tx, ARRAY_SIZE(tx)); + for (unsigned int i = 0; i < ARRAY_SIZE(tx_bits); i++) { + core_mask32(p, 0x10c, tx_bits[i] << lane, i ? 0 : tx_bits[i] << lane); + udelay(1); + } + j700_dp_steps(p, 0xb000 + lane * 0x7000, rx, ARRAY_SIZE(rx)); + core_mask32(p, 0x91d4 + lane * 0x7000, 3, 1); + core_clear32(p, 0xb06c + lane * 0x7000, BIT(11)); + for (unsigned int i = 0; i < ARRAY_SIZE(rx_bits); i++) { + core_mask32(p, 0x8, rx_bits[i] << lane, i ? 0 : rx_bits[i] << lane); + udelay(1); + } + for (unsigned int i = 0; i < 2; i++) { + core_clear32(p, 0xc, BIT(4 * i + lane)); + udelay(1); + } +} + +static int j700_dp_stop(struct apple_atcphy *p) +{ + int ret; + + if (!p->dp_owned_pmas) + return 0; + core_set32(p, 0x7000, BIT(0)); + for (unsigned int i = 0; i < 2; i++) + if (p->dp_owned_pmas & BIT(i)) + j700_dp_outputs_off(p, i); + core_mask32(p, 0x7000, 0xc, 8); + core_mask32(p, 0x2000, 0x1ffffff9, 0x10000001); + ret = j700_dp_wait(p, 0x2000, BIT(1), true); + if (ret) + return ret; + core_mask32(p, 0x2000, 0x1ffffff9, 0x10000018); + ret = j700_dp_wait(p, 0x2000, BIT(1), false); + if (!ret) + ret = j700_dp_wait(p, 0x7034, BIT(3), false); + if (ret) + return ret; + core_clear32(p, 0x2200, BIT(3)); + for (unsigned int i = 0; i < 2; i++) { + if (!(p->dp_owned_pmas & BIT(i))) + continue; + ret = j700_pma_idle(p, i); + if (ret) + return ret; + } + p->dp_owned_pmas = 0; + p->dp_link_rate = -1; + return 0; +} + +static int j700_dp_start(struct apple_atcphy *p, unsigned int pmas, int rate, bool ssc) +{ + static const u32 parameters[4][4] = { + { 0x1e0e021c, 0, 0x156600, 0x15671b }, + { 0x1e0e01c2, 0x07fffffe, 0x155200, 0x155316 }, + { 0x1e0e01c2, 0x07fffffe, 0x554800, 0x554916 }, + { 0x1e0e02a3, 0x0bff7ffc, 0x554810, 0x564922 }, + }; + int ret; + + if (!pmas || (pmas & ~3U) || rate < 0 || (unsigned int)rate >= ARRAY_SIZE(parameters)) + return -EINVAL; + /* Claim before the first write: failed setup must still be cleaned up. */ + p->dp_owned_pmas = pmas; + p->dp_link_rate = -1; + for (unsigned int i = 0; i < 2; i++) + if (pmas & BIT(i)) + j700_dp_power(p, i); + core_clear32(p, 0x2234, 3); + core_mask32(p, 0x7000, 0x1ff0, 0x490); + for (unsigned int i = 0; i < 3; i++) + writel(parameters[rate][i == 2 ? 2 + ssc : i], p->regs.core + 0x2080 + 4 * i); + core_mask32(p, 0x2208, 0x1f0000, 0x70000); + core_set32(p, 0x2218, 1); + core_set32(p, 0x2200, BIT(3)); + core_mask32(p, 0x2000, 0x1ffffff9, 0x10000001); + ret = j700_dp_wait(p, 0x2000, BIT(1), true); + if (!ret) + ret = j700_dp_wait(p, 0x7034, BIT(3), true); + if (ret) + return ret; + core_mask32(p, 0x2000, 0x1ffffff9, 0x10014000); + ret = j700_dp_wait(p, 0x2000, BIT(1), false); + if (ret) + return ret; + for (unsigned int i = 0; i < 2; i++) + if (pmas & BIT(i)) + core_mask32(p, 0xb064 + i * 0x7000, 0xc, 8); + core_set32(p, 0x7000, 0xc); + udelay(1); + for (unsigned int i = 0; i < 2; i++) { + if (!(pmas & BIT(i))) + continue; + j700_dp_transmitter(p, i, rate); + j700_dp_receiver(p, i, rate); + } + core_clear32(p, 0x7000, 1); + p->dp_link_rate = rate; + p->dp_spread_spectrum = ssc; + return 0; +} + +static int j700_dp_configure(struct apple_atcphy *p, const struct phy_configure_opts_dp *o) +{ + const struct atcphy_mode_configuration *mode; + unsigned int pmas, capacity, first, table; + int rate = -1, ret; + + guard(mutex)(&p->lock); + if ((o->set_rate && !o->link_rate) || (o->set_lanes && !o->lanes)) + return o->set_voltages ? -EINVAL : j700_dp_stop(p); + if (p->mode != APPLE_ATCPHY_MODE_DP && p->mode != APPLE_ATCPHY_MODE_USB3_DP) + return -EINVAL; + mode = atcphy_get_mode_config(p, p->mode); + pmas = !!mode->dp_lane[0] | (!!mode->dp_lane[1] << 1); + capacity = 2 * hweight32(pmas); + if (o->lanes > capacity || (o->set_lanes && o->lanes != 1 && + o->lanes != 2 && o->lanes != 4)) + return -EINVAL; + if (!p->dp_presets_valid[0]) + return -ENOENT; + if (o->set_voltages) { + if (!o->lanes) + return -EINVAL; + for (unsigned int i = 0; i < o->lanes; i++) + if (o->voltage[i] > 3 || o->pre[i] > 3) + return -EINVAL; + } + if (o->set_rate) { + const unsigned int rates[] = { 1620, 2700, 5400, 8100 }; + + for (unsigned int i = 0; i < ARRAY_SIZE(rates); i++) + if (o->link_rate == rates[i]) + rate = i; + if (rate < 0) + return -EINVAL; + if (p->dp_link_rate != rate || p->dp_owned_pmas != pmas || + p->dp_spread_spectrum != o->ssc) { + ret = j700_dp_stop(p); + if (ret) + return ret; + ret = j700_dp_start(p, pmas, rate, o->ssc); + if (ret) { + int cleanup = j700_dp_stop(p); + + dev_err(p->dev, "DP setup failed: %d, cleanup: %d\n", ret, cleanup); + return ret; + } + } + } + if (!o->set_voltages) + return 0; + if (!p->dp_owned_pmas || p->dp_link_rate < 0) + return -EINVAL; + table = p->dp_link_rate + 1; + if (!p->dp_presets_valid[table]) + table = 0; + first = pmas == 3 ? !p->swap_lanes : (pmas == 2); + for (unsigned int i = 0; i < o->lanes; i++) + j700_dp_equalize(p, first ^ (i / 2), i % 2, + p->dp_presets[table][4 * o->voltage[i] + o->pre[i]]); + return 0; +} + +#endif diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index d8d4951521bc2..9a75bb2f2ebd9 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -658,6 +658,10 @@ struct atcphy_hw { * @ss_tunables: The complete SuperSpeed tunable set was supplied * @dp_tunables: The complete T8140 DisplayPort bundle was supplied * @aux_power_requested: AUX power-up began and needs matching shutdown + * @dp_owned_pmas: J700 PMAs requiring main-link cleanup + * @dp_spread_spectrum: J700 main-link spread-spectrum state + * @dp_presets: Firmware-derived per-rate drive presets + * @dp_presets_valid: Firmware supplied each drive-preset table * @fixed_usb2: The USB2 pairs go to a fixed hub on the USB controller, which * stays in host mode; the PHY provides USB2 from probe on * @typec_mode: Mode the Type-C mux last asked for; a fixed-hub port returns to @@ -717,6 +721,10 @@ struct apple_atcphy { bool ss_tunables; bool dp_tunables; bool aux_power_requested; + u8 dp_owned_pmas; + bool dp_spread_spectrum; + u32 dp_presets[5][16]; + bool dp_presets_valid[5]; bool fixed_usb2; enum atcphy_mode typec_mode; bool host_active; @@ -1487,6 +1495,8 @@ static int j700_pma_idle(struct apple_atcphy *atcphy, unsigned int lane) return 0; } +#include "atc-j700-dp.h" + static int atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode mode) { const struct atcphy_mode_configuration *mode_cfg = atcphy_get_mode_config(atcphy, mode); @@ -1677,8 +1687,11 @@ static int atcphy_enable_dp_aux(struct apple_atcphy *atcphy) static int atcphy_disable_dp_aux(struct apple_atcphy *atcphy) { - if (atcphy->hw->has_t8140_dp) - return j700_aux_disable(atcphy); + if (atcphy->hw->has_t8140_dp) { + int ret = j700_dp_stop(atcphy); + + return ret ? ret : j700_aux_disable(atcphy); + } /* FIXME */ if (atcphy->hw->gen != ATCPHY_GENERATION_T8122) { @@ -2736,6 +2749,11 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) u32 reg; lockdep_assert_held(&atcphy->lock); + if (atcphy->hw->has_t8140_dp && atcphy->dp_owned_pmas && mode != atcphy->mode) { + ret = j700_dp_stop(atcphy); + if (ret) + return ret; + } if (atcphy->hw->has_t8140_dp && atcphy_modes[mode].enable_dp_aux && !atcphy->dp_tunables) return -EOPNOTSUPP; @@ -2941,6 +2959,9 @@ static int atcphy_dpphy_configure(struct phy *phy, union phy_configure_opts *opt struct apple_atcphy *atcphy = phy_get_drvdata(phy); enum atcphy_dp_link_rate link_rate; + if (atcphy->hw->has_t8140_dp) + return j700_dp_configure(atcphy, opts); + if (opts->set_voltages) return -EINVAL; if (opts->set_lanes) @@ -3368,6 +3389,26 @@ static int atcphy_load_t8140_dp(struct apple_atcphy *atcphy) return PTR_ERR(*lists[i].out); } + for (unsigned int i = 0; i < ARRAY_SIZE(atcphy->dp_presets); i++) { + static const char * const properties[] = { + "apple,dp-training-table", "apple,dp-training-table-rbr", + "apple,dp-training-table-hbr", "apple,dp-training-table-hbr2", + "apple,dp-training-table-hbr3", + }; + int count = of_property_count_u32_elems(atcphy->np, properties[i]); + int ret; + + if (i && !of_find_property(atcphy->np, properties[i], NULL)) + continue; + if (count != ARRAY_SIZE(atcphy->dp_presets[i])) + return -EINVAL; + ret = of_property_read_u32_array(atcphy->np, properties[i], + atcphy->dp_presets[i], count); + if (ret) + return ret; + atcphy->dp_presets_valid[i] = true; + } + atcphy->dp_tunables = true; return 0; } From 88ee9f2184994d914b5ef98a9d504af2b11ea1b7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:42:34 +1100 Subject: [PATCH 1021/2680] phy: apple: preserve J700 lane-routing state during DP setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the audited T8140 routing masks for USB3/DP and four-lane DP. The generic sequence changes the miscellaneous lane-swap control and clears routing state that this generation preserves. Update the low protocol fields in order and idle only the PMAs assigned to DisplayPort. Add opt-in callback and PHY completion diagnostics to distinguish firmware waiting from a blocked PHY operation. No calibration values are logged. The routing values are hardware facts from the user-authorized audit; this implementation is independently written. Copy audit clean. Host checks pass: 33 PHY routing/state cases, 112 training bounds, 100 group routing cases, 917 codec assertions, owned-copy and AUX transition tests. Strict checkpatch passes. External-display qualification remains pending. [ryan: adapted to 7.2.10 — split by topic: the drm/apple hunks are carried by 200-dcp; the tools/diagnostics hunks are dropped: none of those files exists at aurora-wip] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-j700-dp.h | 27 +++++++++++++++++++++++++++ drivers/phy/apple/atc.c | 17 +++++------------ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/drivers/phy/apple/atc-j700-dp.h b/drivers/phy/apple/atc-j700-dp.h index 185551e5a589e..5ce78d085159d 100644 --- a/drivers/phy/apple/atc-j700-dp.h +++ b/drivers/phy/apple/atc-j700-dp.h @@ -3,6 +3,29 @@ #ifndef APPLE_ATC_J700_DP_H #define APPLE_ATC_J700_DP_H +/* Preserve the upper routing state established by the platform firmware. */ +static int j700_dp_route(struct apple_atcphy *p, bool four_lanes) +{ + u32 protocols = four_lanes ? 0x492 : (p->swap_lanes ? 0x252 : 0x489); + int ret; + + core_mask32(p, 0x60, 0xfff, protocols); + if (four_lanes) { + core_mask32(p, 0x64, 0x1f, 0x14); + core_mask32(p, 0x64, 0x3ffe0, p->swap_lanes ? 0x100 : 0x22000); + } else { + core_mask32(p, 0x64, 0x1ffff, 0x110 | p->swap_lanes); + } + for (unsigned int lane = 0; lane < 2; lane++) { + if (!four_lanes && lane != !p->swap_lanes) + continue; + ret = j700_pma_idle(p, lane); + if (ret) + return ret; + } + return 0; +} + /* Register facts from the authorized J700 hardware audit. */ struct j700_dp_step { u32 offset, mask, value; @@ -213,6 +236,7 @@ static int j700_dp_stop(struct apple_atcphy *p) } p->dp_owned_pmas = 0; p->dp_link_rate = -1; + dev_dbg(p->dev, "J700 DP main stopped\n"); return 0; } @@ -265,6 +289,7 @@ static int j700_dp_start(struct apple_atcphy *p, unsigned int pmas, int rate, bo core_clear32(p, 0x7000, 1); p->dp_link_rate = rate; p->dp_spread_spectrum = ssc; + dev_dbg(p->dev, "J700 DP main ready: PMAs=%u rate=%d ssc=%u\n", pmas, rate, ssc); return 0; } @@ -274,7 +299,9 @@ static int j700_dp_configure(struct apple_atcphy *p, const struct phy_configure_ unsigned int pmas, capacity, first, table; int rate = -1, ret; + dev_dbg(p->dev, "J700 DP configure waiting for PHY lock\n"); guard(mutex)(&p->lock); + dev_dbg(p->dev, "J700 DP configure acquired PHY lock\n"); if ((o->set_rate && !o->link_rate) || (o->set_lanes && !o->lanes)) return o->set_voltages ? -EINVAL : j700_dp_stop(p); if (p->mode != APPLE_ATCPHY_MODE_DP && p->mode != APPLE_ATCPHY_MODE_USB3_DP) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 9a75bb2f2ebd9..0ab3c77380520 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1501,6 +1501,11 @@ static int atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode { const struct atcphy_mode_configuration *mode_cfg = atcphy_get_mode_config(atcphy, mode); + if (atcphy->hw->has_t8140_dp && + (mode == APPLE_ATCPHY_MODE_USB3 || mode == APPLE_ATCPHY_MODE_USB3_DP || + mode == APPLE_ATCPHY_MODE_DP)) + return j700_dp_route(atcphy, mode == APPLE_ATCPHY_MODE_DP); + core_mask32(atcphy, atcphy->hw->aciophy_lane_mode, ACIOPHY_LANE_MODE_RX0, FIELD_PREP(ACIOPHY_LANE_MODE_RX0, mode_cfg->lane_mode[0])); core_mask32(atcphy, atcphy->hw->aciophy_lane_mode, ACIOPHY_LANE_MODE_TX0, @@ -1524,18 +1529,6 @@ static int atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode else core_clear32(atcphy, atcphy->hw->aciophy_crossbar, ACIOPHY_CROSSBAR_DP_BOTH_PMA); - if (atcphy->hw->has_t8140_dp) { - for (unsigned int lane = 0; lane < 2; lane++) { - int ret; - - if (!mode_cfg->dp_lane[lane]) - continue; - ret = j700_pma_idle(atcphy, lane); - if (ret) - return ret; - } - return 0; - } /* unclear if we need the remainder for t8122 */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8122) From b68c2d625a1bd87c9299cb0f7671911290350229 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:45:36 +1100 Subject: [PATCH 1022/2680] mux: apple: retain the physical J700 crossbar gate sequence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep the newer T602x DP IN clock controls for existing users, but exclude them from the J700 physical DP output. That output uses the original T602x gate sequence and read-clock reset location. The previous generic path wrote the additional source/output gate registers on J700 too. This small condition is derived from audited hardware facts; no private implementation is imported. The production change has no copy-audit matches. Shared generic test-extractor/header boilerplate was reviewed. Validate the actual production mux function for every source on all three outputs, both with and without the J700 condition: 54 select/disconnect scenarios pass under ASan/UBSan. Existing PHY, training and AFK checks pass. Strict checkpatch passes; external hardware qualification remains pending. [ryan: adapted to 7.2.10 — split by topic: only the drivers/mux hunks of this tooling/selftests/other commit; the rest is in 900-config-tooling] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/mux/apple-display-crossbar.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/drivers/mux/apple-display-crossbar.c b/drivers/mux/apple-display-crossbar.c index f49977b846f04..a0a7ac57ccf00 100644 --- a/drivers/mux/apple-display-crossbar.c +++ b/drivers/mux/apple-display-crossbar.c @@ -210,6 +210,8 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) unsigned int dispext_bit_en; u32 atc_bit, mux_mask, mux_val; bool enable; + /* J700 physical DP retains the original T602x clock-gate sequence. */ + bool j700_physical = index == MUX_DPPHY && of_machine_is_compatible("apple,j700"); int ret = 0; if (index >= MUX_MAX) @@ -271,11 +273,14 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) dpxbar_set32(dpxbar, T602X_REG_014, 0x4); /* Native T602x DPIN0 teardown restores the read reset at +0x24. */ - dpxbar_set32(dpxbar, T602X_FIFO_RD_PCLK2_EN, atc_bit); + dpxbar_set32(dpxbar, j700_physical ? FIFO_RD_PCLK1_EN : + T602X_FIFO_RD_PCLK2_EN, atc_bit); dpxbar_clear32(dpxbar, T602X_REG_034, atc_bit); - dpxbar_clear32(dpxbar, CROSSBAR_ATC_EN, atc_bit); - dpxbar_clear32(dpxbar, CROSSBAR_DISPEXT_EN, prev_dispext_bit); + if (!j700_physical) { + dpxbar_clear32(dpxbar, CROSSBAR_ATC_EN, atc_bit); + dpxbar_clear32(dpxbar, CROSSBAR_DISPEXT_EN, prev_dispext_bit); + } dpxbar_mask32(dpxbar, T602X_REG_030, mux_mask, 0); dpxbar->selected_dispext[index] = -1; @@ -320,13 +325,15 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) index == MUX_DPIN1 ? FIFO_RD_PCLK1_EN : T602X_REG_01C, atc_bit); dpxbar_set32(dpxbar, T602X_REG_034, atc_bit); - dpxbar_set32(dpxbar, CROSSBAR_ATC_EN, atc_bit); + if (!j700_physical) + dpxbar_set32(dpxbar, CROSSBAR_ATC_EN, atc_bit); /* * t8103 enables the dispext source at 0x050. T602x left * that register at 0, so dpin0 was selected with no * source clock into the ACIO analog PHY. */ - dpxbar_set32(dpxbar, CROSSBAR_DISPEXT_EN, dispext_bit); + if (!j700_physical) + dpxbar_set32(dpxbar, CROSSBAR_DISPEXT_EN, dispext_bit); dpxbar_set32(dpxbar, T602X_FIFO_RD_UNK_EN, dispext_bit); From c9fb703926562bc5e0fda1d9f4968d3b834d3da3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:57:03 +1100 Subject: [PATCH 1023/2680] phy: apple: preserve J700 transmitter programming order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep the separate supply request/override writes and program the link-rate field before the final transmitter output controls. Combining those writes and moving the rate field after output setup did not preserve the audited hardware sequence. The host test now checks the actual production register transaction order, including supply writes. It rejects the previous implementation and passes the corrected sequence for both PMAs and every supported rate/SSC pair. Strict checkpatch and the expanded copy audit pass. [ryan: adapted to 7.2.10 — split by topic: the tools/diagnostics hunks are dropped: none of those files exists at aurora-wip] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-j700-dp.h | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/drivers/phy/apple/atc-j700-dp.h b/drivers/phy/apple/atc-j700-dp.h index 5ce78d085159d..11d7b32003748 100644 --- a/drivers/phy/apple/atc-j700-dp.h +++ b/drivers/phy/apple/atc-j700-dp.h @@ -71,16 +71,18 @@ static void j700_dp_output_power(struct apple_atcphy *p, unsigned int lane, bool { u32 reg = tx ? 0x10c : 0xc; - core_set32(p, reg, (BIT(4) | BIT(6)) << lane); - udelay(1); - core_set32(p, reg, (BIT(0) | BIT(2)) << lane); - udelay(1); - if (!tx) { - core_set32(p, 0x8, (BIT(10) | BIT(12)) << lane); - udelay(1); - core_set32(p, 0x8, (BIT(6) | BIT(8)) << lane); + for (unsigned int pair = 0; pair < 2; pair++) { + core_set32(p, reg, BIT(4 - 4 * pair + lane)); + core_set32(p, reg, BIT(6 - 4 * pair + lane)); udelay(1); } + if (!tx) { + for (unsigned int pair = 0; pair < 2; pair++) { + core_set32(p, 0x8, BIT(10 - 4 * pair + lane)); + core_set32(p, 0x8, BIT(12 - 4 * pair + lane)); + udelay(1); + } + } reg = tx ? 0x10c : 0x8; core_clear32(p, reg, BIT(tx ? 8 : 14) << lane); core_set32(p, reg, BIT(tx ? 10 : 16) << lane); @@ -115,7 +117,8 @@ static void j700_dp_transmitter(struct apple_atcphy *p, unsigned int lane, int r { 0x44, 0x400000, 0x400000, 0 }, }; static const struct j700_dp_step finish[] = { - { 0x04, 0x33000000, 0x32000000, 0 }, + { 0x04, 0x03000000, 0x02000000, 0 }, + { 0x04, 0x30000000, 0x30000000, 0 }, { 0x10, 0x600, 0x600, 0 }, { 0x1c, 0xc00000, 0x800000, 0 }, }; @@ -126,9 +129,10 @@ static void j700_dp_transmitter(struct apple_atcphy *p, unsigned int lane, int r j700_dp_equalize(p, lane, true, 0); if (rate != ATCPHY_DP_LINK_RATE_HBR3) j700_dp_steps(p, base, divide, ARRAY_SIZE(divide)); - j700_dp_steps(p, base, finish, ARRAY_SIZE(finish)); + j700_dp_steps(p, base, finish, 2); core_mask32(p, base + 0xc, 0x7000000, rate == ATCPHY_DP_LINK_RATE_RBR ? 0x5000000 : 0x4000000); + j700_dp_steps(p, base, finish + 2, ARRAY_SIZE(finish) - 2); } static void j700_dp_receiver(struct apple_atcphy *p, unsigned int lane, int rate) From 7bf485af126bab3011f415bb0424a7417fee911a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:02:26 +1100 Subject: [PATCH 1024/2680] phy: apple: apply J700 DP calibration before common power-up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the firmware DP calibration bundle into the tunable-application phase, before the common supply/clamp sequence. Applying it later from AUX enable preserved the final register values but not the required initialization order. Full DP uses its own common bundle; shared USB/DP keeps the USB lane calibration and then applies the DP bundle. The phase ordering follows the authorized hardware audit. Existing public tunable operations are reused; no private implementation is imported. Actual production-dispatch tests cover full/shared/reversed DP, the legacy path and fixed USB2. Existing PHY, codec and crossbar tests pass. Strict checkpatch passes; only generic test scaffolding matched in the reviewed copy audit. Hardware qualification remains pending. [ryan: adapted to 7.2.10 — split by topic: the MAINTAINERS lines are carried by 900-config-tooling; the tools/diagnostics hunks are dropped: none of those files exists at aurora-wip] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 0ab3c77380520..7ca8ddee63e77 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1011,6 +1011,14 @@ static const struct atcphy_mode_configuration *atcphy_get_mode_config(struct app return &atcphy_modes[mode].normal; } +static void j700_dp_tunables(struct apple_atcphy *atcphy) +{ + apple_tunable_apply(atcphy->regs.axi2af, atcphy->tunables.axi2af); + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_pre); + apple_tunable_apply(atcphy->regs.lpdptx, atcphy->tunables.dp_lpdptx); + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_post); +} + static void atcphy_apply_tunables(struct apple_atcphy *atcphy, enum atcphy_mode mode) { const int lane0 = atcphy->swap_lanes ? 1 : 0; @@ -1025,6 +1033,11 @@ static void atcphy_apply_tunables(struct apple_atcphy *atcphy, enum atcphy_mode if (mode == APPLE_ATCPHY_MODE_USB2 && atcphy->fixed_usb2) return; + if (atcphy->hw->has_t8140_dp && mode == APPLE_ATCPHY_MODE_DP) { + j700_dp_tunables(atcphy); + return; + } + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.common[0]); apple_tunable_apply(atcphy->regs.axi2af, atcphy->tunables.axi2af); apple_tunable_apply(atcphy->regs.core, atcphy->tunables.common[1]); @@ -1061,6 +1074,9 @@ static void atcphy_apply_tunables(struct apple_atcphy *atcphy, enum atcphy_mode case APPLE_ATCPHY_MODE_USB2: break; } + if (atcphy->hw->has_t8140_dp && mode == APPLE_ATCPHY_MODE_USB3_DP) + j700_dp_tunables(atcphy); + } static int atcphy_pipehandler_lock(struct apple_atcphy *atcphy) @@ -1616,9 +1632,6 @@ static int j700_aux_disable(struct apple_atcphy *atcphy) static int atcphy_enable_dp_aux(struct apple_atcphy *atcphy) { if (atcphy->hw->has_t8140_dp) { - apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_pre); - apple_tunable_apply(atcphy->regs.lpdptx, atcphy->tunables.dp_lpdptx); - apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_post); dev_info_once(atcphy->dev, "Applied T8140 DisplayPort setup lists\n"); return j700_aux_enable(atcphy); } From 3be8b1a033b2bc6a6d1d0c3fcc3c7769e2483174 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:45:26 +1100 Subject: [PATCH 1025/2680] drm/apple: remove temporary external bring-up diagnostics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop temporary protocol capture counters, PHY lock chatter, the global Type-C trace addition and unused link-rate override. Preserve standard tracepoints, failure diagnostics and the validated HBR3 behavior. Register the remaining focused tests and output-frame utility with MAINTAINERS. [ryan: adapted to 7.2.10 — split by topic: only the drivers/phy/apple, drivers/usb hunks of this dcp-display commit; the rest is in 200-dcp] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-j700-dp.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/phy/apple/atc-j700-dp.h b/drivers/phy/apple/atc-j700-dp.h index 11d7b32003748..9071587bd5e8d 100644 --- a/drivers/phy/apple/atc-j700-dp.h +++ b/drivers/phy/apple/atc-j700-dp.h @@ -303,9 +303,7 @@ static int j700_dp_configure(struct apple_atcphy *p, const struct phy_configure_ unsigned int pmas, capacity, first, table; int rate = -1, ret; - dev_dbg(p->dev, "J700 DP configure waiting for PHY lock\n"); guard(mutex)(&p->lock); - dev_dbg(p->dev, "J700 DP configure acquired PHY lock\n"); if ((o->set_rate && !o->link_rate) || (o->set_lanes && !o->lanes)) return o->set_voltages ? -EINVAL : j700_dp_stop(p); if (p->mode != APPLE_ATCPHY_MODE_DP && p->mode != APPLE_ATCPHY_MODE_USB3_DP) From 7f1e2f0cd2651132468b83dbe19699e40ccdf9b9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:51:31 +1100 Subject: [PATCH 1026/2680] drm/apple: close external admission and recovery races MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make explicitly enabled external providers participate in component matching before readiness, then validate prepared state after component binding. Reject unpublished or retired DPTX services and preserve stable ownership rather than reusing a published service generation. Retain failed J700 PHY transition state so SAFE retries cleanup, and reject normal training while that state is dirty. Add fault, admission/lifetime and provider-order regressions. A published DPTX service loss remains fail-closed until recovery; these changes do not claim physical hotplug qualification. [ryan: adapted to 7.2.10 — split by topic: only the drivers/phy/apple hunks of this dcp-display commit; the rest is in 200-dcp] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-j700-dp.h | 26 +++++++++++++++++++++----- drivers/phy/apple/atc.c | 28 ++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/drivers/phy/apple/atc-j700-dp.h b/drivers/phy/apple/atc-j700-dp.h index 9071587bd5e8d..ead9ea6be3f3a 100644 --- a/drivers/phy/apple/atc-j700-dp.h +++ b/drivers/phy/apple/atc-j700-dp.h @@ -3,11 +3,26 @@ #ifndef APPLE_ATC_J700_DP_H #define APPLE_ATC_J700_DP_H +/* Retry only the idle operations claimed by an incomplete route change. */ +static int j700_dp_idle_pending(struct apple_atcphy *p) +{ + int ret; + + for (unsigned int lane = 0; lane < 2; lane++) { + if (!(p->dp_idle_pmas & BIT(lane))) + continue; + ret = j700_pma_idle(p, lane); + if (ret) + return ret; + p->dp_idle_pmas &= ~BIT(lane); + } + return 0; +} + /* Preserve the upper routing state established by the platform firmware. */ static int j700_dp_route(struct apple_atcphy *p, bool four_lanes) { u32 protocols = four_lanes ? 0x492 : (p->swap_lanes ? 0x252 : 0x489); - int ret; core_mask32(p, 0x60, 0xfff, protocols); if (four_lanes) { @@ -19,11 +34,9 @@ static int j700_dp_route(struct apple_atcphy *p, bool four_lanes) for (unsigned int lane = 0; lane < 2; lane++) { if (!four_lanes && lane != !p->swap_lanes) continue; - ret = j700_pma_idle(p, lane); - if (ret) - return ret; + p->dp_idle_pmas |= BIT(lane); } - return 0; + return j700_dp_idle_pending(p); } /* Register facts from the authorized J700 hardware audit. */ @@ -306,6 +319,9 @@ static int j700_dp_configure(struct apple_atcphy *p, const struct phy_configure_ guard(mutex)(&p->lock); if ((o->set_rate && !o->link_rate) || (o->set_lanes && !o->lanes)) return o->set_voltages ? -EINVAL : j700_dp_stop(p); + /* The last completed mode cannot describe a partially changed lane route. */ + if (p->mode_dirty) + return -EBUSY; if (p->mode != APPLE_ATCPHY_MODE_DP && p->mode != APPLE_ATCPHY_MODE_USB3_DP) return -EINVAL; mode = atcphy_get_mode_config(p, p->mode); diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 7ca8ddee63e77..a8c7dca42a735 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -658,6 +658,8 @@ struct atcphy_hw { * @ss_tunables: The complete SuperSpeed tunable set was supplied * @dp_tunables: The complete T8140 DisplayPort bundle was supplied * @aux_power_requested: AUX power-up began and needs matching shutdown + * @mode_dirty: J700 mode transition needs cleanup before it can be retried + * @dp_idle_pmas: J700 PMAs with an incomplete routing idle sequence * @dp_owned_pmas: J700 PMAs requiring main-link cleanup * @dp_spread_spectrum: J700 main-link spread-spectrum state * @dp_presets: Firmware-derived per-rate drive presets @@ -721,6 +723,8 @@ struct apple_atcphy { bool ss_tunables; bool dp_tunables; bool aux_power_requested; + bool mode_dirty; + u8 dp_idle_pmas; u8 dp_owned_pmas; bool dp_spread_spectrum; u32 dp_presets[5][16]; @@ -2755,28 +2759,43 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) u32 reg; lockdep_assert_held(&atcphy->lock); - if (atcphy->hw->has_t8140_dp && atcphy->dp_owned_pmas && mode != atcphy->mode) { + if (atcphy->hw->has_t8140_dp && atcphy->dp_owned_pmas && + (mode != atcphy->mode || atcphy->mode_dirty)) { + atcphy->mode_dirty = true; ret = j700_dp_stop(atcphy); if (ret) return ret; } + if (atcphy->hw->has_t8140_dp && atcphy->mode_dirty) { + ret = j700_dp_idle_pending(atcphy); + if (ret) + return ret; + } if (atcphy->hw->has_t8140_dp && atcphy_modes[mode].enable_dp_aux && !atcphy->dp_tunables) return -EOPNOTSUPP; - if (atcphy->hw->has_t8140_dp && !atcphy_modes[mode].enable_dp_aux && + if (atcphy->hw->has_t8140_dp && + (!atcphy_modes[mode].enable_dp_aux || atcphy->mode_dirty) && atcphy->aux_power_requested) { + atcphy->mode_dirty = true; ret = j700_aux_disable(atcphy); if (ret) return ret; } + /* Keep failed transitions retryable even when the requested mode is unchanged. */ + if (atcphy->hw->has_t8140_dp) + atcphy->mode_dirty = true; /* the mode change below may power the PHY down: stop the tunnel clock first */ atc_tunnel_stop_t8103(atcphy); atc_tunnel_restore(atcphy); if (mode == APPLE_ATCPHY_MODE_OFF) { ret = atcphy_power_off(atcphy); + if (ret) + return ret; atcphy->mode = mode; - return ret; + atcphy->mode_dirty = false; + return 0; } ret = atcphy_power_on(atcphy); @@ -2820,6 +2839,7 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) } atcphy->mode = mode; + atcphy->mode_dirty = false; return 0; } @@ -3289,7 +3309,7 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta if (atcphy->fixed_usb2 && target_mode == APPLE_ATCPHY_MODE_OFF) target_mode = APPLE_ATCPHY_MODE_USB2; - if (atcphy->mode == target_mode) + if (atcphy->mode == target_mode && !atcphy->mode_dirty) return 0; switch (target_mode) { From 9ad9ca7c5356ae56af6bbeddd31afbb91a4c3e78 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 14 Oct 2026 01:22:15 +0000 Subject: [PATCH 1027/2680] usb: typec: serialize display teardown and preserve recovery ownership MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Quiesce the display before SAFE provider writes and role-clock shutdown. Serialize connector transitions across mux handles, retain failed SAFE ownership, and check finalization before admitting a new DP route. Keep SRCU reader accounting local and preserve module/provider lifetime. Track consumed mux release separately so retries cannot release twice. Refuse reset when provider acquisition cannot establish SAFE. Retain compact actual-source regression coverage for transaction ordering, error paths, reference lifetime and release accounting. Independent review passes all five retained failure witnesses. Complete ARM64 Image/modules/ DTB build passes. Target recovery qualification is pending; this is not a hardware hotplug or suspend/wake pass. [ryan: adapted to 7.2.10 — split by topic: the drm/apple hunks are carried by 200-dcp; the apple-dcp selftest hunks are carried by 200-dcp] [ryan: adapted to 7.2.10 — mainline (stable commit c5df31f10880 ("usb: typec: tipd: don't send GAID when probe fails in APP mode")) now resets only a controller whose patch was loaded at probe; both conditions apply (cd321x_reset() is a no-op, so Apple controllers are unaffected)] Signed-off-by: Ryan Murray Assisted-by: LLM --- drivers/usb/typec/mux.c | 228 ++++++++++++++++++++++++++++-- drivers/usb/typec/mux.h | 6 + drivers/usb/typec/tipd/core.c | 129 +++++++++++++---- drivers/usb/typec/tipd/tps6598x.h | 5 +- include/linux/usb/typec_mux.h | 32 +++++ 5 files changed, 364 insertions(+), 36 deletions(-) diff --git a/drivers/usb/typec/mux.c b/drivers/usb/typec/mux.c index 2563531127344..4170d238c01f7 100644 --- a/drivers/usb/typec/mux.c +++ b/drivers/usb/typec/mux.c @@ -13,6 +13,8 @@ #include #include #include +#include +#include #include "class.h" #include "mux.h" @@ -261,11 +263,60 @@ EXPORT_SYMBOL_GPL(typec_switch_get_drvdata); /* ------------------------------------------------------------------------- */ +/* Class, altmode and controller handles for one connector share this domain. */ +struct typec_mux_context { + struct list_head list; + struct fwnode_handle *fwnode; + struct mutex lock; + unsigned int users; +}; + +static LIST_HEAD(typec_mux_contexts); +static DEFINE_MUTEX(typec_mux_contexts_lock); +DEFINE_STATIC_SRCU(typec_mux_srcu); + struct typec_mux { struct typec_mux_dev *mux_devs[TYPEC_MUX_MAX_DEVS]; unsigned int num_mux_devs; + struct typec_mux_context *context; + struct typec_mux_state transaction; + struct task_struct *owner; + int srcu_index; + unsigned int prepared; + bool applied; }; +static struct typec_mux_context *typec_mux_context_get(struct fwnode_handle *fwnode) +{ + struct typec_mux_context *context; + + guard(mutex)(&typec_mux_contexts_lock); + list_for_each_entry(context, &typec_mux_contexts, list) { + if (context->fwnode == fwnode) { + context->users++; + return context; + } + } + context = kzalloc_obj(*context); + if (!context) + return ERR_PTR(-ENOMEM); + context->fwnode = fwnode_handle_get(fwnode); + context->users = 1; + mutex_init(&context->lock); + list_add_tail(&context->list, &typec_mux_contexts); + return context; +} + +static void typec_mux_context_put(struct typec_mux_context *context) +{ + guard(mutex)(&typec_mux_contexts_lock); + if (--context->users) + return; + list_del(&context->list); + fwnode_handle_put(context->fwnode); + kfree(context); +} + static int mux_fwnode_match(struct device *dev, const void *fwnode) { if (!is_typec_mux_dev(dev)) @@ -324,17 +375,21 @@ struct typec_mux *fwnode_typec_mux_get(struct fwnode_handle *fwnode) int count; int err; int i; + int index; mux = kzalloc_obj(*mux); if (!mux) return ERR_PTR(-ENOMEM); + /* Unregister must drain lookup and module-reference conversion too. */ + index = srcu_read_lock(&typec_mux_srcu); count = fwnode_connection_find_matches(fwnode, "mode-switch", (void **)mux_devs, typec_mux_match, (void **)mux_devs, ARRAY_SIZE(mux_devs)); if (count <= 0) { + srcu_read_unlock(&typec_mux_srcu, index); kfree(mux); return NULL; } @@ -347,12 +402,26 @@ struct typec_mux *fwnode_typec_mux_get(struct fwnode_handle *fwnode) } for (i = 0; i < count; i++) { - WARN_ON(!try_module_get(mux_devs[i]->dev.parent->driver->owner)); + if (READ_ONCE(mux_devs[i]->removed) || + !try_module_get(mux_devs[i]->owner)) { + err = -ENODEV; + while (i--) + module_put(mux_devs[i]->owner); + goto put_mux_devs; + } mux->mux_devs[i] = mux_devs[i]; } mux->num_mux_devs = count; + mux->context = typec_mux_context_get(fwnode); + if (IS_ERR(mux->context)) { + err = PTR_ERR(mux->context); + for (i = 0; i < count; i++) + module_put(mux_devs[i]->owner); + goto put_mux_devs; + } + srcu_read_unlock(&typec_mux_srcu, index); return mux; put_mux_devs: @@ -361,6 +430,7 @@ struct typec_mux *fwnode_typec_mux_get(struct fwnode_handle *fwnode) put_device(&mux_devs[i]->dev); } + srcu_read_unlock(&typec_mux_srcu, index); kfree(mux); return ERR_PTR(err); @@ -383,31 +453,162 @@ void typec_mux_put(struct typec_mux *mux) for (i = 0; i < mux->num_mux_devs; i++) { mux_dev = mux->mux_devs[i]; - module_put(mux_dev->dev.parent->driver->owner); + module_put(mux_dev->owner); put_device(&mux_dev->dev); } + typec_mux_context_put(mux->context); kfree(mux); } EXPORT_SYMBOL_GPL(typec_mux_put); -int typec_mux_set(struct typec_mux *mux, struct typec_mux_state *state) +static void typec_mux_unlock(struct typec_mux *mux) +{ + int index = mux->srcu_index; + + mux->prepared = 0; + mux->applied = false; + mux->owner = NULL; + mutex_unlock(&mux->context->lock); + srcu_read_unlock(&typec_mux_srcu, index); +} + +void typec_mux_abort(struct typec_mux *mux) { - struct typec_mux_dev *mux_dev; unsigned int i; - int ret; + + if (IS_ERR_OR_NULL(mux)) + return; + if (WARN_ON_ONCE(mux->owner != current)) + return; + for (i = mux->prepared; i > 0; i--) { + struct typec_mux_dev *dev = mux->mux_devs[i - 1]; + + if (dev->abort) + dev->abort(dev, &mux->transaction); + } + typec_mux_unlock(mux); +} +EXPORT_SYMBOL_GPL(typec_mux_abort); + +int typec_mux_begin(struct typec_mux *mux, struct typec_mux_state *state) +{ + unsigned int i; + int index, ret; if (IS_ERR_OR_NULL(mux)) return 0; + index = srcu_read_lock(&typec_mux_srcu); + /* Nonblocking acquisition also rejects same-domain forwarding recursion. */ + if (!mutex_trylock(&mux->context->lock)) { + srcu_read_unlock(&typec_mux_srcu, index); + return -EBUSY; + } + mux->srcu_index = index; + mux->owner = current; + mux->transaction = *state; + for (i = 0; i < mux->num_mux_devs; i++) { + struct typec_mux_dev *dev = mux->mux_devs[i]; + if (READ_ONCE(dev->removed)) { + typec_mux_unlock(mux); + return -ENODEV; + } + } for (i = 0; i < mux->num_mux_devs; i++) { - mux_dev = mux->mux_devs[i]; + struct typec_mux_dev *dev = mux->mux_devs[i]; + + mux->prepared = i + 1; + if (!dev->prepare) + continue; + ret = dev->prepare(dev, &mux->transaction); + if (ret) + goto abort; + } + return 0; +abort: + typec_mux_abort(mux); + return ret; +} +EXPORT_SYMBOL_GPL(typec_mux_begin); - ret = mux_dev->set(mux_dev, state); - if (ret && ret != -EOPNOTSUPP) +int typec_mux_apply(struct typec_mux *mux) +{ + unsigned int i; + int ret; + + if (IS_ERR_OR_NULL(mux)) + return 0; + if (WARN_ON_ONCE(mux->owner != current)) + return -EINVAL; + if (mux->applied) + return 0; + for (i = 0; i < mux->num_mux_devs; i++) { + struct typec_mux_dev *dev = mux->mux_devs[i]; + + ret = dev->set(dev, &mux->transaction); + if (ret && ret != -EOPNOTSUPP) { + typec_mux_abort(mux); return ret; + } + } + mux->applied = true; + return 0; +} +EXPORT_SYMBOL_GPL(typec_mux_apply); + +int typec_mux_end(struct typec_mux *mux) +{ + unsigned int i; + int ret; + + if (IS_ERR_OR_NULL(mux)) + return 0; + ret = typec_mux_apply(mux); + if (ret) + return ret; + /* Finalizers can fail, but must not release their software reservation. */ + for (i = 0; i < mux->num_mux_devs; i++) { + struct typec_mux_dev *dev = mux->mux_devs[i]; + + if (!dev->finish) + continue; + ret = dev->finish(dev, &mux->transaction); + if (ret) + goto abort; } + /* Only this no-fail publication phase may release owner reservations. */ + for (i = 0; i < mux->num_mux_devs; i++) { + struct typec_mux_dev *dev = mux->mux_devs[i]; + if (dev->commit) + dev->commit(dev, &mux->transaction); + } + typec_mux_unlock(mux); return 0; +abort: + typec_mux_abort(mux); + return ret; +} +EXPORT_SYMBOL_GPL(typec_mux_end); + +/* Preparation-only callers leave recovery ownership pending, never committed. */ +int typec_mux_prepare(struct typec_mux *mux, struct typec_mux_state *state) +{ + int ret = typec_mux_begin(mux, state); + + if (!ret) + typec_mux_abort(mux); + return ret; +} +EXPORT_SYMBOL_GPL(typec_mux_prepare); + +int typec_mux_set(struct typec_mux *mux, struct typec_mux_state *state) +{ + int ret = typec_mux_begin(mux, state); + + if (ret) + return ret; + return typec_mux_end(mux); } EXPORT_SYMBOL_GPL(typec_mux_set); @@ -444,7 +645,12 @@ typec_mux_register(struct device *parent, const struct typec_mux_desc *desc) if (!mux_dev) return ERR_PTR(-ENOMEM); + mux_dev->owner = parent->driver ? parent->driver->owner : NULL; mux_dev->set = desc->set; + mux_dev->prepare = desc->prepare; + mux_dev->finish = desc->finish; + mux_dev->commit = desc->commit; + mux_dev->abort = desc->abort; device_initialize(&mux_dev->dev); mux_dev->dev.parent = parent; @@ -477,8 +683,12 @@ EXPORT_SYMBOL_GPL(typec_mux_register); */ void typec_mux_unregister(struct typec_mux_dev *mux_dev) { - if (!IS_ERR_OR_NULL(mux_dev)) + if (!IS_ERR_OR_NULL(mux_dev)) { + WRITE_ONCE(mux_dev->removed, true); + /* Callbacks may nest through forwarders; SRCU permits that nesting. */ + synchronize_srcu(&typec_mux_srcu); device_unregister(&mux_dev->dev); + } } EXPORT_SYMBOL_GPL(typec_mux_unregister); diff --git a/drivers/usb/typec/mux.h b/drivers/usb/typec/mux.h index 12e1ac47a11f1..42e44211fd553 100644 --- a/drivers/usb/typec/mux.h +++ b/drivers/usb/typec/mux.h @@ -12,8 +12,14 @@ struct typec_switch_dev { }; struct typec_mux_dev { + struct module *owner; struct device dev; typec_mux_set_fn_t set; + typec_mux_set_fn_t prepare; + typec_mux_set_fn_t finish; + typec_mux_notify_fn_t commit; + typec_mux_notify_fn_t abort; + bool removed; }; #define to_typec_switch_dev(_dev_) container_of(_dev_, struct typec_switch_dev, dev) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 084bdc679ce70..8e4a2da2c5ff1 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -212,10 +212,11 @@ static int tps6598x_read_partner_identity(struct tps6598x *tps) return 0; } -static void tps6598x_set_data_role(struct tps6598x *tps, +static int tps6598x_set_data_role(struct tps6598x *tps, enum typec_data_role role, bool connected) { enum usb_role role_val; + int ret; if (role == TYPEC_HOST) role_val = USB_ROLE_HOST; @@ -225,8 +226,11 @@ static void tps6598x_set_data_role(struct tps6598x *tps, if (!connected) role_val = USB_ROLE_NONE; - usb_role_switch_set_role(tps->role_sw, role_val); + ret = usb_role_switch_set_role(tps->role_sw, role_val); + if (ret) + return ret; typec_set_data_role(tps->port, role); + return 0; } static int tps6598x_connect(struct tps6598x *tps, u32 status) @@ -258,7 +262,9 @@ static int tps6598x_connect(struct tps6598x *tps, u32 status) else typec_set_orientation(tps->port, TYPEC_ORIENTATION_NORMAL); typec_set_mode(tps->port, TYPEC_STATE_USB); - tps6598x_set_data_role(tps, TPS_STATUS_TO_TYPEC_DATAROLE(status), true); + ret = tps6598x_set_data_role(tps, TPS_STATUS_TO_TYPEC_DATAROLE(status), true); + if (ret) + return ret; tps->partner = typec_register_partner(tps->port, &desc); if (IS_ERR(tps->partner)) @@ -272,7 +278,7 @@ static int tps6598x_connect(struct tps6598x *tps, u32 status) return 0; } -static void tps6598x_disconnect(struct tps6598x *tps, u32 status) +static void tps6598x_drop_partner(struct tps6598x *tps, u32 status) { if (!IS_ERR(tps->partner)) typec_unregister_partner(tps->partner); @@ -280,11 +286,37 @@ static void tps6598x_disconnect(struct tps6598x *tps, u32 status) typec_set_pwr_opmode(tps->port, TYPEC_PWR_MODE_USB); typec_set_pwr_role(tps->port, TPS_STATUS_TO_TYPEC_PORTROLE(status)); typec_set_vconn_role(tps->port, TPS_STATUS_TO_TYPEC_VCONN(status)); - typec_set_orientation(tps->port, TYPEC_ORIENTATION_NONE); - typec_set_mode(tps->port, TYPEC_STATE_SAFE); - tps6598x_set_data_role(tps, TPS_STATUS_TO_TYPEC_DATAROLE(status), false); +} + +static int tps6598x_disconnect(struct tps6598x *tps, u32 status) +{ + struct typec_mux_state safe = { .mode = TYPEC_STATE_SAFE }; + bool prepared = tps->remove_safe_prepared; + int ret; + + tps->remove_safe_prepared = false; + tps6598x_drop_partner(tps, status); + if (!prepared) { + typec_set_orientation(tps->port, TYPEC_ORIENTATION_NONE); + ret = typec_mux_begin(tps->mux, &safe); + if (ret) + return ret; + /* Park while the host still supplies the PIPE backend clock. */ + ret = typec_mux_apply(tps->mux); + if (ret) + return ret; + } + ret = tps6598x_set_data_role(tps, TPS_STATUS_TO_TYPEC_DATAROLE(status), false); + if (ret) { + typec_mux_abort(tps->mux); + return ret; + } + ret = typec_mux_end(tps->mux); + if (ret) + return ret; power_supply_changed(tps->psy); + return 0; } int tps6598x_exec_cmd_tmo(struct tps6598x *tps, const char *cmd, @@ -528,7 +560,9 @@ static void tps6598x_handle_plug_event(struct tps6598x *tps, u32 status) if (ret) dev_err(tps->dev, "failed to register partner\n"); } else { - tps6598x_disconnect(tps, status); + ret = tps6598x_disconnect(tps, status); + if (ret) + dev_err(tps->dev, "hardware disconnect failed: %d\n", ret); } } @@ -1963,12 +1997,13 @@ tps25750_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) return 0; } -static void cd321x_remove(struct tps6598x *tps) +static int cd321x_remove(struct tps6598x *tps) { struct cd321x *cd321x = container_of(tps, struct cd321x, tps); struct typec_thunderbolt_switch_data tbt_switch_data = { .state = TYPEC_THUNDERBOLT_SWITCH_OFF, }; + struct typec_mux_state safe = { .mode = TYPEC_STATE_SAFE }; int ret; cd321x_stop_work(cd321x); @@ -1985,10 +2020,24 @@ static void cd321x_remove(struct tps6598x *tps) ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); if (ret) - dev_warn(tps->dev, - "failed to close Thunderbolt/USB4 session on remove: %d\n", - ret); - usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); + return ret; + /* Display quiescence must precede even an ordinary host-role shutdown. */ + typec_set_orientation(tps->port, TYPEC_ORIENTATION_NONE); + ret = typec_mux_begin(tps->mux, &safe); + if (ret) + return ret; + /* Park while the host still supplies the PIPE backend clock. */ + ret = typec_mux_apply(tps->mux); + if (ret) + return ret; + ret = usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); + if (ret) { + typec_mux_abort(tps->mux); + return ret; + } + /* Same task/handle completes without another post-shutdown set callback. */ + tps->remove_safe_prepared = true; + return 0; } @@ -1998,7 +2047,8 @@ int tipd_init(struct tps6598x *tps) bool patch_loaded = false; u32 status; u32 vid; - int ret; + int ret, cleanup_ret; + bool reset_allowed = true; tps->reset = devm_gpiod_get_optional(tps->dev, "reset", GPIOD_OUT_LOW); if (IS_ERR(tps->reset)) @@ -2072,19 +2122,32 @@ int tipd_init(struct tps6598x *tps) if (ret) goto err_role_put; - ret = tps->data->register_port(tps, fwnode); - if (ret) + tps->mux = fwnode_typec_mux_get(fwnode); + if (IS_ERR(tps->mux)) { + ret = PTR_ERR(tps->mux); + /* No provider reference means rollback cannot prove SAFE. */ + reset_allowed = false; goto err_role_put; + } + + ret = tps->data->register_port(tps, fwnode); + if (ret) { + /* No live port remains to prove SAFE on failed registration. */ + reset_allowed = false; + goto err_mux_put; + } if (status & TPS_STATUS_PLUG_PRESENT) { ret = -EINVAL; if (!tps6598x_read_power_status(tps)) - goto err_unregister_port; + goto err_disconnect; if (!tps->data->read_data_status(tps)) - goto err_unregister_port; + goto err_disconnect; ret = tps->data->connect(tps, status); - if (ret) + if (ret) { dev_err(tps->dev, "failed to register partner\n"); + goto err_disconnect; + } } if (tps->irq) { @@ -2117,9 +2180,14 @@ int tipd_init(struct tps6598x *tps) return 0; err_disconnect: - tps6598x_disconnect(tps, 0); -err_unregister_port: + cleanup_ret = tps6598x_disconnect(tps, 0); + if (cleanup_ret) { + dev_err(tps->dev, "probe hardware rollback failed: %d\n", cleanup_ret); + reset_allowed = false; + } tps->data->unregister_port(tps); +err_mux_put: + typec_mux_put(tps->mux); err_role_put: usb_role_switch_put(tps->role_sw); err_fwnode_put: @@ -2128,7 +2196,7 @@ int tipd_init(struct tps6598x *tps) tps6598x_write64(tps, TPS_REG_INT_MASK1, 0); err_reset_controller: /* Reset PD controller to remove any applied patch */ - if (patch_loaded) + if (patch_loaded && reset_allowed) tps->data->reset(tps); return ret; @@ -2137,6 +2205,8 @@ EXPORT_SYMBOL_GPL(tipd_init); void tipd_remove(struct tps6598x *tps) { + int ret = 0; + tipd_debugfs_unregister(tps); if (!tps->irq) @@ -2145,16 +2215,23 @@ void tipd_remove(struct tps6598x *tps) devm_free_irq(tps->dev, tps->irq, tps); if (tps->data->remove) - tps->data->remove(tps); + ret = tps->data->remove(tps); - tps6598x_disconnect(tps, 0); + if (!ret) + ret = tps6598x_disconnect(tps, 0); + else + tps6598x_drop_partner(tps, 0); + if (ret) + dev_err(tps->dev, "hardware teardown failed on remove: %d\n", ret); tps->data->unregister_port(tps); + typec_mux_put(tps->mux); usb_role_switch_put(tps->role_sw); /* Reset PD controller to remove any applied patch */ - tps->data->reset(tps); + if (!ret) + tps->data->reset(tps); - if (tps->reset) + if (!ret && tps->reset) gpiod_set_value_cansleep(tps->reset, 1); } EXPORT_SYMBOL_GPL(tipd_remove); diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index e290f9d2d80df..b241541715df0 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -305,7 +305,7 @@ struct tipd_data { irq_handler_t irq_handler; u64 irq_mask1; size_t tps_struct_size; - void (*remove)(struct tps6598x *tps); + int (*remove)(struct tps6598x *tps); int (*register_port)(struct tps6598x *tps, struct fwnode_handle *node); void (*unregister_port)(struct tps6598x *tps); void (*ready)(struct tps6598x *tps); @@ -323,6 +323,9 @@ struct tipd_data { }; struct tps6598x { + struct typec_mux *mux; + /* CD remove leaves one open SAFE transaction for generic completion. */ + bool remove_safe_prepared; struct device *dev; struct regmap *regmap; struct mutex lock; /* device lock */ diff --git a/include/linux/usb/typec_mux.h b/include/linux/usb/typec_mux.h index d58d60fcae21f..a18e4e0141adf 100644 --- a/include/linux/usb/typec_mux.h +++ b/include/linux/usb/typec_mux.h @@ -86,6 +86,9 @@ struct typec_mux_state { void *data; }; +typedef void (*typec_mux_notify_fn_t)(struct typec_mux_dev *mux, + struct typec_mux_state *state); + typedef int (*typec_mux_set_fn_t)(struct typec_mux_dev *mux, struct typec_mux_state *state); @@ -94,12 +97,26 @@ struct typec_mux_desc { typec_mux_set_fn_t set; const char *name; void *drvdata; + /* Quiesce consumers before any provider changes the shared route. */ + typec_mux_set_fn_t prepare; + /* Fallible finalization retains ownership; commit cannot fail. */ + typec_mux_set_fn_t finish; + typec_mux_notify_fn_t commit; + typec_mux_notify_fn_t abort; }; #if IS_ENABLED(CONFIG_TYPEC) struct typec_mux *fwnode_typec_mux_get(struct fwnode_handle *fwnode); void typec_mux_put(struct typec_mux *mux); +/* begin/end must run in the same task; failure retains provider recovery state. */ +/* Same-task begin/end; retain the handle and state pointees until end/abort. */ +int typec_mux_begin(struct typec_mux *mux, struct typec_mux_state *state); +/* Apply the providers while retaining the transaction and ownership. */ +int typec_mux_apply(struct typec_mux *mux); +int typec_mux_end(struct typec_mux *mux); +void typec_mux_abort(struct typec_mux *mux); +int typec_mux_prepare(struct typec_mux *mux, struct typec_mux_state *state); int typec_mux_set(struct typec_mux *mux, struct typec_mux_state *state); struct typec_mux_dev * @@ -118,6 +135,21 @@ static inline struct typec_mux *fwnode_typec_mux_get(struct fwnode_handle *fwnod static inline void typec_mux_put(struct typec_mux *mux) {} +static inline int typec_mux_begin(struct typec_mux *mux, struct typec_mux_state *state) +{ + return 0; +} + +static inline int typec_mux_apply(struct typec_mux *mux) { return 0; } + +static inline int typec_mux_end(struct typec_mux *mux) { return 0; } +static inline void typec_mux_abort(struct typec_mux *mux) {} + +static inline int typec_mux_prepare(struct typec_mux *mux, struct typec_mux_state *state) +{ + return 0; +} + static inline int typec_mux_set(struct typec_mux *mux, struct typec_mux_state *state) { return 0; From e94b88db43e2bd89af8219968e8056943779e756 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 19:57:16 +1100 Subject: [PATCH 1028/2680] phy: apple: atc: Own the fixed-host PIPE handoff and resume its root The J700 fixed-host USB lifecycle reached aurora-wip only as content of an integration merge; no commit carries it. This re-makes its USB part as a commit so the 7.2.10 port does not lose it. The merge is commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"). - atc: resume the matching SuperSpeed root across Type-C PHY transitions, retry a failed PIPE setup on an unchanged mode without touching the configured lanes, and keep failed handoff ownership instead of reporting an untrained link as usable. The J700 DisplayPort helpers in atc-j700-dp.h grow the matching admission state. PHY_APPLE_ATC now follows USB (depends on USB || !USB). - dwc3: split dwc3_core_exit() so the Apple glue can power the PHYs off itself, track every core-owned PHY reference and the controller reset, and finish a failed shutdown before the next start (dwc3_core_exit_after_phy_power_off(), dwc3_core_remove_no_shutdown()). - tipd: the CD321x/SN201202x side of the same ownership rules. The content is the delta of that merge over the automatic merge of its parents, applied unchanged except for one dynamic-debug line from the dropped 294891122c09. The drm/apple, SEP, ISP, MAINTAINERS and selftest parts of the merge belong to other topics. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/Kconfig | 1 + drivers/phy/apple/atc-j700-dp.h | 160 +++++++++++- drivers/phy/apple/atc.c | 445 ++++++++++++++++++++++++++++---- drivers/usb/dwc3/core.c | 31 ++- drivers/usb/dwc3/dwc3-apple.c | 114 +++++--- drivers/usb/dwc3/glue.h | 20 ++ drivers/usb/typec/tipd/core.c | 47 ++-- 7 files changed, 711 insertions(+), 107 deletions(-) diff --git a/drivers/phy/apple/Kconfig b/drivers/phy/apple/Kconfig index 8409b67f6d1ec..26cd09c05497c 100644 --- a/drivers/phy/apple/Kconfig +++ b/drivers/phy/apple/Kconfig @@ -3,6 +3,7 @@ config PHY_APPLE_ATC tristate "Apple Type-C PHY" depends on (ARM64 && ARCH_APPLE) || (COMPILE_TEST && !GENERIC_ATOMIC64) depends on TYPEC + depends on USB || !USB select GENERIC_PHY select APPLE_TUNABLE help diff --git a/drivers/phy/apple/atc-j700-dp.h b/drivers/phy/apple/atc-j700-dp.h index ead9ea6be3f3a..a07f1125c2c43 100644 --- a/drivers/phy/apple/atc-j700-dp.h +++ b/drivers/phy/apple/atc-j700-dp.h @@ -3,6 +3,159 @@ #ifndef APPLE_ATC_J700_DP_H #define APPLE_ATC_J700_DP_H +/* Masks are the union of writes in the public J700 DP start/voltage/stop paths. */ +struct j700_usb_field { + u32 offset, mask; +}; + +static const struct j700_usb_field j700_usb_fields[] = { + { 0x0008, 0x00015540 }, + { 0x000c, 0x00000055 }, + { 0x010c, 0x00000555 }, + { 0x0200, 0x0000000f }, + { 0x9160, 0x00000001 }, + { 0x91d4, 0x00000003 }, + { 0x91dc, 0x00018000 }, + { 0x9210, 0x0000c00f }, + { 0x9244, 0x0007ffff }, + { 0x9248, 0x10000000 }, + { 0xb030, 0x00000c00 }, + { 0xb064, 0x0000fc0c }, + { 0xb068, 0x000001e0 }, + { 0xb06c, 0x0001ee00 }, + { 0xb0a4, 0x00000003 }, + { 0xb0a8, 0x00ff8000 }, + { 0xb0d4, 0x001fff80 }, + { 0xb0d8, 0x00000007 }, + { 0xb0dc, 0x00000018 }, + { 0xc050, 0x0000c00f }, + { 0xc0d0, 0x0017ffff }, + { 0xc0d4, 0x10000000 }, + { 0xd004, 0x33000000 }, + { 0xd00c, 0x07000000 }, + { 0xd010, 0x00000600 }, + { 0xd01c, 0x00c00000 }, + { 0xd044, 0xffffffff }, +}; + +static_assert(ARRAY_SIZE(j700_usb_fields) == 27, "USB context storage size"); + +static void j700_usb_field(unsigned int index, unsigned int lane, u32 *offset, u32 *mask) +{ + *offset = j700_usb_fields[index].offset; + *mask = j700_usb_fields[index].mask; + if (index < 4) + *mask <<= index == 3 ? 4 * lane : lane; + else + *offset += 0x7000 * lane; +} + +/* Capture before the first DP tunable or idle write; never replace a retained context. */ +static int j700_usb_capture(struct apple_atcphy *p, enum atcphy_mode mode) +{ + const struct atcphy_mode_configuration *cfg = atcphy_get_mode_config(p, mode); + + if (!p->hw->has_t8140_dp || !p->fixed_usb2) + return 0; + if (p->usb_off_pending || + p->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY || p->pipe_pending) + return -EBUSY; + for (unsigned int lane = 0; lane < 2; lane++) { + if (!cfg->dp_lane[lane] || (p->dp_usb_context_valid & BIT(lane))) + continue; + for (unsigned int i = 0; i < ARRAY_SIZE(j700_usb_fields); i++) { + u32 offset, mask; + + j700_usb_field(i, lane, &offset, &mask); + p->dp_usb_context[lane][i] = core_read32(p, offset) & mask; + } + p->dp_usb_context_valid |= BIT(lane); + } + return 0; +} + +/* Same control write/delay/poll sequence, retaining its final sample for diagnostics. */ +static int j700_usb_field_update(struct apple_atcphy *p, u32 offset, u32 mask, + u32 value, u32 *observed) +{ + core_mask32(p, offset, mask, value); + usleep_range(10, 20); + return readl_poll_timeout_atomic(p->regs.core + offset, *observed, + (*observed & mask) == value, 1, 100); +} + +/* Restore only DP-owned bits, with the PIPE parked and the DP PMA idle. */ +static int j700_usb_restore(struct apple_atcphy *p, enum atcphy_mode mode) +{ + const struct atcphy_mode_configuration *cfg = atcphy_get_mode_config(p, mode); + const unsigned int power_order[] = { 2, 1, 0, 3 }; + const unsigned int receiver_order[] = { 4, 0, 6, 2 }; + int ret; + + if (!p->hw->has_t8140_dp || !p->fixed_usb2) + return 0; + if (p->usb_off_pending && mode != APPLE_ATCPHY_MODE_OFF) + return -EBUSY; + if (p->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY || p->pipe_pending) + return -EBUSY; + if (p->dp_owned_pmas || p->dp_idle_pmas) + return -EBUSY; + if (mode == APPLE_ATCPHY_MODE_OFF && p->usb_off_drained) + return 0; + for (unsigned int lane = 0; lane < 2; lane++) { + if ((mode != APPLE_ATCPHY_MODE_OFF && + cfg->lane_mode[lane] != ACIOPHY_LANE_MODE_USB3) || + !(p->dp_usb_context_valid & BIT(lane))) + continue; + if (p->dp_usb_context_dirty & BIT(lane)) { + for (unsigned int i = 0; i < ARRAY_SIZE(j700_usb_fields); i++) { + u32 offset, mask, observed, value = p->dp_usb_context[lane][i]; + + if (i == 1 || i == 3) + continue; + j700_usb_field(i, lane, &offset, &mask); + core_mask32(p, offset, mask, value); + observed = core_read32(p, offset) & mask; + if (observed != value) { + dev_dbg(p->dev, "USB restore stage=static pma=%u reg=%#x mask=%#x expected=%#x observed=%#x\n", + lane, offset, mask, value, observed); + return -EIO; + } + } + } + /* Preserve the public PMA power bit order, using saved values. */ + for (unsigned int i = 0; i < ARRAY_SIZE(power_order); i++) { + u32 mask = BIT(4 * lane + power_order[i]); + u32 value = p->dp_usb_context[lane][3] & mask, observed; + + ret = j700_usb_field_update(p, 0x200, mask, value, &observed); + if (ret) { + dev_dbg(p->dev, "USB restore stage=power pma=%u reg=0x200 mask=%#x expected=%#x observed=%#x ret=%d\n", + lane, mask, value, observed & mask, ret); + return ret; + } + } + /* Restore receiver enables, then release its two idle controls. */ + for (unsigned int i = 0; i < ARRAY_SIZE(receiver_order); i++) { + u32 mask = BIT(lane + receiver_order[i]); + u32 value = p->dp_usb_context[lane][1] & mask, observed; + + ret = j700_usb_field_update(p, 0xc, mask, value, &observed); + if (ret) { + dev_dbg(p->dev, "USB restore stage=receiver pma=%u reg=0xc mask=%#x expected=%#x observed=%#x ret=%d\n", + lane, mask, value, observed & mask, ret); + return ret; + } + } + /* OFF keeps the shadow until domain shutdown succeeds. */ + if (mode != APPLE_ATCPHY_MODE_OFF) { + p->dp_usb_context_dirty &= ~BIT(lane); + p->dp_usb_context_valid &= ~BIT(lane); + } + } + return 0; +} + /* Retry only the idle operations claimed by an incomplete route change. */ static int j700_dp_idle_pending(struct apple_atcphy *p) { @@ -269,6 +422,11 @@ static int j700_dp_start(struct apple_atcphy *p, unsigned int pmas, int rate, bo if (!pmas || (pmas & ~3U) || rate < 0 || (unsigned int)rate >= ARRAY_SIZE(parameters)) return -EINVAL; + if (p->hw->has_t8140_dp && p->fixed_usb2) { + if ((p->dp_usb_context_valid & pmas) != pmas) + return -EIO; + p->dp_usb_context_dirty |= pmas; + } /* Claim before the first write: failed setup must still be cleaned up. */ p->dp_owned_pmas = pmas; p->dp_link_rate = -1; @@ -320,7 +478,7 @@ static int j700_dp_configure(struct apple_atcphy *p, const struct phy_configure_ if ((o->set_rate && !o->link_rate) || (o->set_lanes && !o->lanes)) return o->set_voltages ? -EINVAL : j700_dp_stop(p); /* The last completed mode cannot describe a partially changed lane route. */ - if (p->mode_dirty) + if (p->mode_dirty || p->pipe_pending || p->usb_off_pending) return -EBUSY; if (p->mode != APPLE_ATCPHY_MODE_DP && p->mode != APPLE_ATCPHY_MODE_USB3_DP) return -EINVAL; diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index a8c7dca42a735..2c0ef71a8291f 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -33,12 +33,16 @@ #include #include #include +#include #include #include +#include #include #include #include #include +#include +#include #include #include #include @@ -658,9 +662,15 @@ struct atcphy_hw { * @ss_tunables: The complete SuperSpeed tunable set was supplied * @dp_tunables: The complete T8140 DisplayPort bundle was supplied * @aux_power_requested: AUX power-up began and needs matching shutdown - * @mode_dirty: J700 mode transition needs cleanup before it can be retried + * @mode_dirty: Mode transition needs cleanup before it can be retried * @dp_idle_pmas: J700 PMAs with an incomplete routing idle sequence * @dp_owned_pmas: J700 PMAs requiring main-link cleanup + * @dp_usb_context_valid: Physical PMAs retaining their pre-DP USB field values + * @dp_usb_context_dirty: PMAs whose static fields were programmed by DP + * @usb_off_pending: Failed full OFF must finish before a new powered epoch + * @usb_off_drained: Masked recovery data was discharged before OFF began + * @pipe_pending: PIPE handoff still requires acknowledged dummy parking + * @dp_usb_context: Saved bits from the public per-PMA DP write masks * @dp_spread_spectrum: J700 main-link spread-spectrum state * @dp_presets: Firmware-derived per-rate drive presets * @dp_presets_valid: Firmware supplied each drive-preset table @@ -726,6 +736,9 @@ struct apple_atcphy { bool mode_dirty; u8 dp_idle_pmas; u8 dp_owned_pmas; + u8 dp_usb_context_valid, dp_usb_context_dirty; + u32 dp_usb_context[2][27]; + bool usb_off_pending, usb_off_drained, pipe_pending; bool dp_spread_spectrum; u32 dp_presets[5][16]; bool dp_presets_valid[5]; @@ -1015,6 +1028,13 @@ static const struct atcphy_mode_configuration *atcphy_get_mode_config(struct app return &atcphy_modes[mode].normal; } +/* Readback only; callers hold the PHY mutex and keep the fixed host live. */ +static inline u32 core_read32(struct apple_atcphy *atcphy, u32 reg) +{ + return readl(atcphy->regs.core + reg); +} + +/* Static configuration fields already read/modified by the public DP path. */ static void j700_dp_tunables(struct apple_atcphy *atcphy) { apple_tunable_apply(atcphy->regs.axi2af, atcphy->tunables.axi2af); @@ -1087,17 +1107,22 @@ static int atcphy_pipehandler_lock(struct apple_atcphy *atcphy) { int ret; u32 reg; + u32 requested = readl(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ); - if (readl(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ) & PIPEHANDLER_LOCK_EN) { - dev_warn(atcphy->dev, "Pipehandler already locked\n"); - return 0; + dev_dbg(atcphy->dev, "PIPE request: req=%#x mode=%u pipe=%u host=%u dirty=%u\n", + requested, atcphy->mode, atcphy->pipe_state, + atcphy->host_active, atcphy->mode_dirty); + if (requested & PIPEHANDLER_LOCK_EN) { + dev_dbg(atcphy->dev, "Pipehandler lock already requested\n"); + } else { + set32(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ, PIPEHANDLER_LOCK_EN); } - set32(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ, PIPEHANDLER_LOCK_EN); - ret = readl_poll_timeout(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_ACK, reg, reg & PIPEHANDLER_LOCK_EN, 10, PIPEHANDLER_LOCK_ACK_TIMEOUT_US); if (ret) { + dev_dbg(atcphy->dev, "PIPE lock timeout: ack=%#x mode=%u pipe=%u dirty=%u\n", + reg, atcphy->mode, atcphy->pipe_state, atcphy->mode_dirty); clear32(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ, 1); dev_warn(atcphy->dev, "Pipehandler lock not acked.\n"); } @@ -1113,8 +1138,11 @@ static int atcphy_pipehandler_unlock(struct apple_atcphy *atcphy) clear32(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ, PIPEHANDLER_LOCK_EN); ret = readl_poll_timeout(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_ACK, reg, !(reg & PIPEHANDLER_LOCK_EN), 10, PIPEHANDLER_LOCK_ACK_TIMEOUT_US); - if (ret) + if (ret) { + dev_dbg(atcphy->dev, "PIPE unlock timeout: ack=%#x mode=%u pipe=%u dirty=%u\n", + reg, atcphy->mode, atcphy->pipe_state, atcphy->mode_dirty); dev_warn(atcphy->dev, "Pipehandler lock release not acked.\n"); + } return ret; } @@ -1147,9 +1175,9 @@ static void atcphy_clear_nonselected_phy_reset(struct apple_atcphy *atcphy) PIPEHANDLER_NATIVE_RESET); } -static void atcphy_configure_pipehandler_usb3_host_t8122(struct apple_atcphy *atcphy) +static int atcphy_configure_pipehandler_usb3_host_t8122(struct apple_atcphy *atcphy) { - int ret; + int ret, error = 0; u32 reg; core_set32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_CLOCK_EN); @@ -1162,15 +1190,25 @@ static void atcphy_configure_pipehandler_usb3_host_t8122(struct apple_atcphy *at core_set32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_USB_EN); ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_BIST_CFG1, reg, (reg & ACIOPHY_BIST_CFG1_USB_EN), 10, 10000); - if (ret) + if (ret) { + if (!error) + error = ret; + dev_dbg(atcphy->dev, "BIST USB_EN phase=1 value=%#x mode=%u pipe=%u\n", + reg, atcphy->mode, atcphy->pipe_state); dev_warn(atcphy->dev, "Timed out waiting for ACIOPHY_BIST_CFG1_USB_EN\n"); + } core_clear32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_USB_EN); ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_BIST_MAC_USBPHY_STATUS2, reg, !(reg & ACIOPHY_BIST_MAC_USBPHY_STATUS2_LN0_PHY_STATUS2), 10, 10000); - if (ret) + if (ret) { + if (!error) + error = ret; + dev_dbg(atcphy->dev, "BIST STATUS2 phase=1 value=%#x swap=%u mode=%u pipe=%u\n", + reg, atcphy->swap_lanes, atcphy->mode, atcphy->pipe_state); dev_warn(atcphy->dev, "Timed out waiting for ACIOPHY_BIST_MAC_USBPHY_STATUS2_LN0_PHY_STATUS2\n"); + } core_set32(atcphy, ACIOPHY_BIST_CFG2, ACIOPHY_BIST_CFG2_LN0_PHY_STATUS_RE); udelay(10); @@ -1183,15 +1221,25 @@ static void atcphy_configure_pipehandler_usb3_host_t8122(struct apple_atcphy *at core_set32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_USB_EN); ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_BIST_CFG1, reg, (reg & ACIOPHY_BIST_CFG1_USB_EN), 10, 10000); - if (ret) + if (ret) { + if (!error) + error = ret; + dev_dbg(atcphy->dev, "BIST USB_EN phase=2 value=%#x mode=%u pipe=%u\n", + reg, atcphy->mode, atcphy->pipe_state); dev_warn(atcphy->dev, "Timed out waiting for ACIOPHY_BIST_CFG1_USB_EN 2nd toggle\n"); + } core_clear32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_USB_EN); ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_BIST_MAC_USBPHY_STATUS2, reg, !(reg & ACIOPHY_BIST_MAC_USBPHY_STATUS2_LN0_PHY_STATUS2), 10, 10000); - if (ret) + if (ret) { + if (!error) + error = ret; + dev_dbg(atcphy->dev, "BIST STATUS2 phase=2 value=%#x swap=%u mode=%u pipe=%u\n", + reg, atcphy->swap_lanes, atcphy->mode, atcphy->pipe_state); dev_warn(atcphy->dev, "Timed out waiting for ACIOPHY_BIST_MAC_USBPHY_STATUS2_LN0_PHY_STATUS2 2nd time\n"); + } core_set32(atcphy, ACIOPHY_BIST_CFG2, ACIOPHY_BIST_CFG2_LN0_PHY_STATUS_RE); udelay(10); core_clear32(atcphy, ACIOPHY_BIST_CFG2, ACIOPHY_BIST_CFG2_LN0_PHY_STATUS_RE); @@ -1203,16 +1251,22 @@ static void atcphy_configure_pipehandler_usb3_host_t8122(struct apple_atcphy *at core_set32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_USB_EN); ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_BIST_CFG1, reg, (reg & ACIOPHY_BIST_CFG1_USB_EN), 10, 10000); - if (ret) + if (ret) { + if (!error) + error = ret; + dev_dbg(atcphy->dev, "BIST USB_EN phase=3 value=%#x mode=%u pipe=%u\n", + reg, atcphy->mode, atcphy->pipe_state); dev_warn(atcphy->dev, "Timed out waiting for ACIOPHY_BIST_CFG1_USB_EN 3rd toggle\n"); + } core_clear32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_USB_EN); core_clear32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_CLOCK_EN); + return error; } -static void atcphy_configure_pipehandler_usb3_host_t8103(struct apple_atcphy *atcphy) +static int atcphy_configure_pipehandler_usb3_host_t8103(struct apple_atcphy *atcphy) { - int ret; + int ret, error = 0; u32 reg; core_set32(atcphy, ACIOPHY_TOP_BIST_PHY_CFG0, @@ -1220,9 +1274,12 @@ static void atcphy_configure_pipehandler_usb3_host_t8103(struct apple_atcphy *at core_set32(atcphy, ACIOPHY_TOP_BIST_OV_CFG, ACIOPHY_TOP_BIST_OV_CFG_LN0_RESET_N_OV); ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_TOP_PHY_STAT, reg, !(reg & ACIOPHY_TOP_PHY_STAT_LN0_UNK23), 10, 10000); - if (ret) + if (ret) { + if (!error) + error = ret; dev_warn(atcphy->dev, "Timed out waiting for ACIOPHY_TOP_PHY_STAT_LN0_UNK23\n"); + } core_set32(atcphy, ACIOPHY_TOP_BIST_READ_CTRL, ACIOPHY_TOP_BIST_READ_CTRL_LN0_PHY_STATUS_RE); @@ -1243,15 +1300,21 @@ static void atcphy_configure_pipehandler_usb3_host_t8103(struct apple_atcphy *at ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_TOP_PHY_STAT, reg, (reg & ACIOPHY_TOP_PHY_STAT_LN0_UNK0), 10, 10000); - if (ret) + if (ret) { + if (!error) + error = ret; dev_warn(atcphy->dev, "timed out waiting for ACIOPHY_TOP_PHY_STAT_LN0_UNK0\n"); + } ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_TOP_PHY_STAT, reg, !(reg & ACIOPHY_TOP_PHY_STAT_LN0_UNK23), 10, 10000); - if (ret) + if (ret) { + if (!error) + error = ret; dev_warn(atcphy->dev, "timed out waiting for ACIOPHY_TOP_PHY_STAT_LN0_UNK23\n"); + } atcphy_clear_nonselected_phy_reset(atcphy); @@ -1261,6 +1324,7 @@ static void atcphy_configure_pipehandler_usb3_host_t8103(struct apple_atcphy *at ACIOPHY_TOP_BIST_CIOPHY_CFG1_CLK_EN); core_set32(atcphy, ACIOPHY_TOP_BIST_CIOPHY_CFG1, ACIOPHY_TOP_BIST_CIOPHY_CFG1_BIST_EN); + return error; } static void atcphy_pipehandler_set_mux(struct apple_atcphy *atcphy, u32 data, u32 clk) @@ -1305,14 +1369,28 @@ static int atcphy_configure_pipehandler_usb3(struct apple_atcphy *atcphy, bool h /* BIST dance */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8103) - atcphy_configure_pipehandler_usb3_host_t8103(atcphy); + ret = atcphy_configure_pipehandler_usb3_host_t8103(atcphy); else - atcphy_configure_pipehandler_usb3_host_t8122(atcphy); + ret = atcphy_configure_pipehandler_usb3_host_t8122(atcphy); + if (ret) { + int cleanup; + + clear32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, + PIPEHANDLER_OVERRIDE_RXVALID | PIPEHANDLER_OVERRIDE_RXDETECT); + cleanup = atcphy_pipehandler_unlock(atcphy); + if (cleanup) + dev_warn(atcphy->dev, "PIPE cleanup after BIST: %d\n", cleanup); + /* Native BIST changed state; a retry must rebuild after parking. */ + atcphy->mode_dirty = true; + return ret; + } } /* Configure PIPE mux to USB3 PHY */ atcphy_pipehandler_set_mux(atcphy, PIPEHANDLER_MUX_CTRL_DATA_USB3, PIPEHANDLER_MUX_CTRL_CLK_USB3); + /* Track the selected backend even if its completion ACK fails. */ + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_USB3; /* Remove link detection override */ clear32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, PIPEHANDLER_OVERRIDE_RXVALID); @@ -1321,8 +1399,11 @@ static int atcphy_configure_pipehandler_usb3(struct apple_atcphy *atcphy, bool h /* Pipehandler was only locked when the BIST sequence was applied for host mode */ if (host) { ret = atcphy_pipehandler_unlock(atcphy); - if (ret) + if (ret) { dev_warn(atcphy->dev, "Failed to unlock pipehandler"); + atcphy->mode_dirty = true; + return ret; + } } return 0; @@ -1383,11 +1464,18 @@ static void atcphy_enable_dummy_phy(struct apple_atcphy *atcphy) static int atcphy_configure_pipehandler_dummy(struct apple_atcphy *atcphy) { + bool live_usb3 = atcphy->pipe_pending || + (atcphy->hw->has_t8140_dp && atcphy->fixed_usb2 && atcphy->host_active && + atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3); int ret; ret = atcphy_pipehandler_check(atcphy); - if (ret) + if (ret) { + if (live_usb3) + if (atcphy->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY) + atcphy->mode_dirty = true; return ret; + } /* Force disable link detection */ clear32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE_VALUES, @@ -1396,16 +1484,30 @@ static int atcphy_configure_pipehandler_dummy(struct apple_atcphy *atcphy) set32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, PIPEHANDLER_OVERRIDE_RXDETECT); ret = atcphy_pipehandler_lock(atcphy); - if (ret) + if (ret) { + /* A live controller must acknowledge before its backend changes. */ + if (live_usb3) { + if (atcphy->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY) + atcphy->mode_dirty = true; + return ret; + } dev_warn(atcphy->dev, "Failed to lock pipehandler"); + } /* Switch to dummy PHY */ atcphy_pipehandler_set_mux(atcphy, PIPEHANDLER_MUX_CTRL_DATA_DUMMY, PIPEHANDLER_MUX_CTRL_CLK_DUMMY); ret = atcphy_pipehandler_unlock(atcphy); - if (ret) + if (ret) { + /* The mux changed, but the handoff has not completed. Retain ownership. */ + if (live_usb3) { + if (atcphy->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY) + atcphy->mode_dirty = true; + return ret; + } dev_warn(atcphy->dev, "Failed to unlock pipehandler"); + } mask32(atcphy->regs.pipehandler + PIPEHANDLER_NONSELECTED_OVERRIDE, PIPEHANDLER_NATIVE_POWER_DOWN, FIELD_PREP(PIPEHANDLER_NATIVE_POWER_DOWN, 2)); @@ -1419,8 +1521,7 @@ static int atcphy_configure_pipehandler_dummy(struct apple_atcphy *atcphy) /* * Route the PIPE to the backend the current mode needs. The state is recorded - * whether or not the sequence succeeded: the PIPE is configured once per mode, - * and a failed attempt is not retried on the next set_mode call. + * only after the sequence succeeds, so a failed attempt remains retryable. */ static int atcphy_configure_pipehandler(struct apple_atcphy *atcphy, bool host) { @@ -1431,6 +1532,7 @@ static int atcphy_configure_pipehandler(struct apple_atcphy *atcphy, bool host) switch (state) { case ATCPHY_PIPEHANDLER_STATE_USB3: + atcphy->pipe_pending = true; ret = atcphy_configure_pipehandler_usb3(atcphy, host); break; case ATCPHY_PIPEHANDLER_STATE_USB4: @@ -1440,25 +1542,35 @@ static int atcphy_configure_pipehandler(struct apple_atcphy *atcphy, bool host) ret = atcphy_configure_pipehandler_dummy(atcphy); break; } - atcphy->pipe_state = state; + if (!ret) { + atcphy->pipe_state = state; + atcphy->pipe_pending = false; + } return ret; } /* Route the PIPE back to the dummy backend unless it is there already */ -static void atcphy_park_pipehandler(struct apple_atcphy *atcphy) +static int atcphy_park_pipehandler(struct apple_atcphy *atcphy) { int ret; lockdep_assert_held(&atcphy->lock); - if (atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY) - return; + dev_dbg(atcphy->dev, "PIPE park: mode=%u pipe=%u host=%u dirty=%u owned=%u idle=%u\n", + atcphy->mode, atcphy->pipe_state, atcphy->host_active, atcphy->mode_dirty, + atcphy->dp_owned_pmas, atcphy->dp_idle_pmas); + if (atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY && !atcphy->pipe_pending) + return 0; ret = atcphy_configure_pipehandler_dummy(atcphy); - if (ret) + if (ret) { dev_warn(atcphy->dev, "Failed to switch PIPE to dummy: %d\n", ret); + return ret; + } atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + atcphy->pipe_pending = false; + return 0; } static void atcphy_setup_pipehandler(struct apple_atcphy *atcphy) @@ -1468,6 +1580,7 @@ static void atcphy_setup_pipehandler(struct apple_atcphy *atcphy) atcphy_pipehandler_set_mux(atcphy, PIPEHANDLER_MUX_CTRL_DATA_DUMMY, PIPEHANDLER_MUX_CTRL_CLK_DUMMY); atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + atcphy->pipe_pending = false; if (!atcphy->hw->has_usb4) atcphy_enable_dummy_phy(atcphy); } @@ -1963,6 +2076,9 @@ static int atcphy_dp_configure(struct apple_atcphy *atcphy, enum atcphy_dp_link_ u32 reg; guard(mutex)(&atcphy->lock); + /* A failed transition may have asserted the PHY resets already. */ + if (atcphy->mode_dirty || atcphy->pipe_pending) + return -EBUSY; /* the AUSPLL is in use as the Thunderbolt DP tunnel pixel clock */ if (atcphy->tunnel_clock_on || atcphy->tunnel_saved) return -EBUSY; @@ -2759,6 +2875,13 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) u32 reg; lockdep_assert_held(&atcphy->lock); + if (atcphy->pipe_pending) + return -EBUSY; + if (atcphy->usb_off_pending && mode != APPLE_ATCPHY_MODE_OFF) + return -EBUSY; + if (atcphy->hw->has_t8140_dp && atcphy->fixed_usb2 && + atcphy->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY) + return -EBUSY; if (atcphy->hw->has_t8140_dp && atcphy->dp_owned_pmas && (mode != atcphy->mode || atcphy->mode_dirty)) { atcphy->mode_dirty = true; @@ -2783,25 +2906,55 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) return ret; } /* Keep failed transitions retryable even when the requested mode is unchanged. */ - if (atcphy->hw->has_t8140_dp) - atcphy->mode_dirty = true; + atcphy->mode_dirty = true; /* the mode change below may power the PHY down: stop the tunnel clock first */ atc_tunnel_stop_t8103(atcphy); atc_tunnel_restore(atcphy); if (mode == APPLE_ATCPHY_MODE_OFF) { + if (atcphy->hw->has_t8140_dp && atcphy->fixed_usb2 && + (atcphy->dp_usb_context_valid || atcphy->usb_off_pending)) { + atcphy->usb_off_pending = true; + if (!atcphy->usb_off_drained) { + ret = j700_usb_restore(atcphy, APPLE_ATCPHY_MODE_OFF); + if (ret) + return ret; + atcphy->usb_off_drained = true; + } + } ret = atcphy_power_off(atcphy); if (ret) return ret; + /* Recovery data was discharged before OFF, not discarded while dirty. */ + atcphy->dp_usb_context_valid = 0; + atcphy->dp_usb_context_dirty = 0; + atcphy->usb_off_pending = false; + atcphy->usb_off_drained = false; atcphy->mode = mode; atcphy->mode_dirty = false; return 0; } + /* + * Fixed-hub SAFE keeps USB2 up rather than taking the block OFF. + * With PIPE parked and DP owners released, reinitialize USB3 before + * programming its lanes; the existing sequence releases this reset. + */ + if (atcphy->hw->has_t8140_dp && atcphy->fixed_usb2) + core_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_PHY_RESET_N); + ret = atcphy_power_on(atcphy); if (ret) return ret; + if (atcphy->hw->has_t8140_dp && atcphy->fixed_usb2) { + ret = j700_usb_capture(atcphy, mode); + if (!ret) + ret = j700_usb_restore(atcphy, mode); + if (ret) + return ret; + } + atcphy_apply_tunables(atcphy, mode); atcphy->hw->configure_post_tunable(atcphy, mode); @@ -2868,10 +3021,23 @@ static int atcphy_usb2_init(struct phy *phy) { struct apple_atcphy *atcphy = phy_get_drvdata(phy); - if (!atcphy->fixed_usb2) - return 0; - guard(mutex)(&atcphy->lock); + if (atcphy->usb_off_pending) + return -EBUSY; + if (atcphy->pipe_pending && + (atcphy->mode == APPLE_ATCPHY_MODE_OFF || atcphy->mode_dirty)) { + int ret = atcphy_park_pipehandler(atcphy); + + if (ret) + return ret; + } + + /* A delayed shutdown may have completed after the last Type-C event. */ + if (!atcphy->fixed_usb2) { + if (atcphy->mode == APPLE_ATCPHY_MODE_OFF || atcphy->mode_dirty) + return atcphy_configure(atcphy, atcphy->typec_mode); + return 0; + } /* * dwc3 initialises its PHYs after releasing its reset, and asserting that @@ -2882,7 +3048,7 @@ static int atcphy_usb2_init(struct phy *phy) * the Type-C mux last asked for, USB2 until a cable event says otherwise. */ set32(atcphy->regs.usb2phy + USB2PHY_SIG, USB2PHY_SIG_HOST); - if (atcphy->mode == APPLE_ATCPHY_MODE_OFF) + if (atcphy->mode == APPLE_ATCPHY_MODE_OFF || atcphy->mode_dirty) return atcphy_configure(atcphy, atcphy->typec_mode); atcphy_usb2_power_on(atcphy); @@ -2899,15 +3065,20 @@ static const struct phy_ops apple_atc_usb2_phy_ops = { static int atcphy_usb3_power_off(struct phy *phy) { struct apple_atcphy *atcphy = phy_get_drvdata(phy); + int ret; guard(mutex)(&atcphy->lock); - atcphy_park_pipehandler(atcphy); + ret = atcphy_park_pipehandler(atcphy); + if (ret) + return ret; + if (atcphy->mode != APPLE_ATCPHY_MODE_OFF || atcphy->mode_dirty) { + ret = atcphy_configure(atcphy, APPLE_ATCPHY_MODE_OFF); + if (ret) + return ret; + } atcphy->host_active = false; - if (atcphy->mode != APPLE_ATCPHY_MODE_OFF) - atcphy_configure(atcphy, APPLE_ATCPHY_MODE_OFF); - return 0; } @@ -2917,6 +3088,9 @@ static int atcphy_usb3_set_mode(struct phy *phy, enum phy_mode mode, int submode guard(mutex)(&atcphy->lock); + if (atcphy->usb_off_pending) + return -EBUSY; + switch (mode) { case PHY_MODE_USB_HOST: atcphy->host_active = true; @@ -2934,7 +3108,17 @@ static int atcphy_usb3_set_mode(struct phy *phy, enum phy_mode mode, int submode * PIPE handler once. Nothing needs to be done either when the PIPE is already routed to * the backend the current mode uses, which includes the dummy backend of the USB2 modes. */ - if (atcphy->pipe_state == atcphy_modes[atcphy->mode].pipehandler_state) + /* A failed lane/domain transition must be restored before routing PIPE. */ + if (atcphy->mode_dirty) + return -EBUSY; + if (atcphy->pipe_pending) { + int ret = atcphy_park_pipehandler(atcphy); + + if (ret) + return ret; + } + if (atcphy->pipe_state == atcphy_modes[atcphy->mode].pipehandler_state && + !atcphy->pipe_pending) return 0; return atcphy_configure_pipehandler(atcphy, mode == PHY_MODE_USB_HOST); @@ -3154,11 +3338,16 @@ static void _atcphy_dwc3_reset_assert(struct apple_atcphy *atcphy) static int atcphy_dwc3_reset_assert(struct reset_controller_dev *rcdev, unsigned long id) { struct apple_atcphy *atcphy = container_of(rcdev, struct apple_atcphy, rcdev); + int ret; guard(mutex)(&atcphy->lock); + /* Do not reset the controller or USB2 PHY if PIPE parking fails. */ + ret = atcphy_park_pipehandler(atcphy); + if (ret) + return ret; + _atcphy_dwc3_reset_assert(atcphy); - atcphy_park_pipehandler(atcphy); atcphy->host_active = false; atcphy_usb2_power_off(atcphy); @@ -3204,19 +3393,30 @@ static int atcphy_probe_rcdev(struct apple_atcphy *atcphy) static int atcphy_sw_set(struct typec_switch_dev *sw, enum typec_orientation orientation) { struct apple_atcphy *atcphy = typec_switch_get_drvdata(sw); + bool swap_lanes; guard(mutex)(&atcphy->lock); switch (orientation) { case TYPEC_ORIENTATION_NONE: - break; + return 0; case TYPEC_ORIENTATION_NORMAL: - atcphy->swap_lanes = false; + swap_lanes = false; break; case TYPEC_ORIENTATION_REVERSE: - atcphy->swap_lanes = true; + swap_lanes = true; break; + default: + return -EINVAL; + } + /* The mode alone cannot describe a route after its orientation changes. */ + if (atcphy->swap_lanes != swap_lanes) { + atcphy->swap_lanes = swap_lanes; + atcphy->mode_dirty = true; } + dev_dbg(atcphy->dev, "orientation=%u mode=%u dirty=%u pipe=%u host=%u\n", + orientation, atcphy->mode, atcphy->mode_dirty, + atcphy->pipe_state, atcphy->host_active); return 0; } @@ -3242,7 +3442,109 @@ static int atcphy_probe_switch(struct apple_atcphy *atcphy) return devm_add_action_or_reset(atcphy->dev, atcphy_typec_switch_unregister, sw); } -static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state) +/* + * The fixed USB2 hub keeps the host controller active, but its independent + * SuperSpeed root hub may autosuspend while PIPE still needs it awake. + * Own a bounded root-hub resume reference across a Type-C handoff. Resume + * precedes the ATC lock: USB PM can call back into the PHY on recovery. + */ +struct atcphy_usb3_pm { + struct platform_device *controller; + struct usb_device *root; +}; + +#if IS_REACHABLE(CONFIG_USB) +static int atcphy_usb3_pm_find(struct usb_device *udev, void *data) +{ + struct atcphy_usb3_pm *pm = data; + + if (udev->parent || udev->speed < USB_SPEED_SUPER || + udev->bus->sysdev != &pm->controller->dev) + return 0; + + pm->root = usb_get_dev(udev); + return 1; +} +#endif + +static int atcphy_usb3_pm_get(struct apple_atcphy *atcphy, struct atcphy_usb3_pm *pm) +{ +#if IS_REACHABLE(CONFIG_USB) + struct device_node *ep __free(device_node) = NULL; + struct device_node *ctrl __free(device_node) = NULL; + struct usb_hcd *hcd; + int ret; +#endif + + if (!atcphy->fixed_usb2 || !atcphy->hw->has_t8140_dp) + return 0; + +#if IS_REACHABLE(CONFIG_USB) + ep = of_graph_get_endpoint_by_regs(atcphy->np, 1, -1); + ctrl = ep ? of_graph_get_remote_port_parent(ep) : NULL; + if (!ctrl) + return -ENODEV; + pm->controller = of_find_device_by_node(ctrl); + if (!pm->controller) + return -EAGAIN; + + /* Firmware identity, not a bus number, binds this PHY to its root hub. */ + usb_for_each_dev(pm, atcphy_usb3_pm_find); + if (!pm->root) { + ret = -EAGAIN; + goto put_controller; + } + + /* Exclude root-hub teardown while its controller backs the handoff. */ + usb_lock_device(pm->root); + hcd = bus_to_hcd(pm->root->bus); + if (READ_ONCE(pm->root->state) == USB_STATE_NOTATTACHED || HCD_DEAD(hcd)) { + ret = -ENODEV; + goto unlock_root; + } + ret = pm_runtime_resume_and_get(&pm->root->dev); + if (ret < 0) + goto unlock_root; + if (READ_ONCE(pm->root->state) == USB_STATE_NOTATTACHED || + HCD_DEAD(hcd) || !HCD_HW_ACCESSIBLE(hcd)) { + pm_runtime_put_noidle(&pm->root->dev); + ret = -ENODEV; + goto unlock_root; + } + return 0; + +unlock_root: + usb_unlock_device(pm->root); + usb_put_dev(pm->root); + pm->root = NULL; +put_controller: + put_device(&pm->controller->dev); + pm->controller = NULL; + return ret; +#else + /* typec_mux_set() skips EOPNOTSUPP, which would report a false success. */ + return -ENODEV; +#endif +} + +static void atcphy_usb3_pm_put(struct atcphy_usb3_pm *pm) +{ +#if IS_REACHABLE(CONFIG_USB) + if (!pm->root) + return; + + /* Restore normal runtime policy after both success and failed parking. */ + usb_mark_last_busy(pm->root); + pm_runtime_put_autosuspend(&pm->root->dev); + usb_unlock_device(pm->root); + usb_put_dev(pm->root); + put_device(&pm->controller->dev); + pm->root = NULL; + pm->controller = NULL; +#endif +} + +static int atcphy_mux_set_locked(struct typec_mux_dev *mux, struct typec_mux_state *state) { struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); enum atcphy_mode target_mode; @@ -3309,8 +3611,20 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta if (atcphy->fixed_usb2 && target_mode == APPLE_ATCPHY_MODE_OFF) target_mode = APPLE_ATCPHY_MODE_USB2; - if (atcphy->mode == target_mode && !atcphy->mode_dirty) + if (atcphy->mode == target_mode && !atcphy->mode_dirty) { + /* Retry a failed PIPE setup without disturbing the configured lanes. */ + if (atcphy->fixed_usb2 && atcphy->host_active && + (atcphy->pipe_state != atcphy_modes[target_mode].pipehandler_state || + atcphy->pipe_pending)) { + if (atcphy->pipe_pending) { + ret = atcphy_park_pipehandler(atcphy); + if (ret) + return ret; + } + return atcphy_configure_pipehandler(atcphy, true); + } return 0; + } switch (target_mode) { case APPLE_ATCPHY_MODE_TBT: @@ -3341,23 +3655,32 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta /* * The controller of a port behind a fixed hub stays up across mode * changes, so the PIPE has to follow the mode from here rather than - * from a later set_mode call: park it before the lanes change to a - * mode without a USB3 backend, and route it to the new backend - * afterwards while dwc3 is up in host mode. + * from a later set_mode call. Even another USB3 mode can move the + * backend to the other PMA when orientation changes. Park before + * rebuilding the lanes, then reconnect while dwc3 is in host mode. */ - if (pipe_state != ATCPHY_PIPEHANDLER_STATE_USB3) - atcphy_park_pipehandler(atcphy); + ret = atcphy_park_pipehandler(atcphy); + if (ret) + return ret; ret = atcphy_configure(atcphy, target_mode); if (ret) return ret; + if (atcphy->host_active && atcphy->pipe_state != pipe_state) ret = atcphy_configure_pipehandler(atcphy, true); return ret; } + /* A failed USB3 handoff must finish parking before ordinary-port lane work. */ + if (atcphy->pipe_pending) { + ret = atcphy_park_pipehandler(atcphy); + if (ret) + return ret; + } + /* * If the pipehandler is still/already up here there's a bug somewhere so make sure to * complain loudly. We can still try to switch modes and hope for the best though, @@ -3367,6 +3690,20 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta return atcphy_configure(atcphy, target_mode); } +static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state) +{ + struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); + struct atcphy_usb3_pm pm = {}; + int ret; + + ret = atcphy_usb3_pm_get(atcphy, &pm); + if (ret) + return ret; + ret = atcphy_mux_set_locked(mux, state); + atcphy_usb3_pm_put(&pm); + return ret; +} + static void atcphy_typec_mux_unregister(void *data) { typec_mux_unregister(data); diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index ceb49f2f80041..34627ddeaaa1d 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -991,14 +991,20 @@ static void dwc3_clk_disable(struct dwc3 *dwc) clk_disable_unprepare(dwc->bus_clk); } -void dwc3_core_exit(struct dwc3 *dwc) +void dwc3_core_exit_after_phy_power_off(struct dwc3 *dwc) { - dwc3_event_buffers_cleanup(dwc); - dwc3_phy_power_off(dwc); dwc3_phy_exit(dwc); dwc3_clk_disable(dwc); reset_control_assert(dwc->reset); } +EXPORT_SYMBOL_GPL(dwc3_core_exit_after_phy_power_off); + +void dwc3_core_exit(struct dwc3 *dwc) +{ + dwc3_event_buffers_cleanup(dwc); + dwc3_phy_power_off(dwc); + dwc3_core_exit_after_phy_power_off(dwc); +} EXPORT_SYMBOL_GPL(dwc3_core_exit); static bool dwc3_core_is_valid(struct dwc3 *dwc) @@ -2406,14 +2412,16 @@ static int dwc3_probe(struct platform_device *pdev) return dwc3_core_probe(&probe_data); } -void dwc3_core_remove(struct dwc3 *dwc) +static void dwc3_core_remove_common(struct dwc3 *dwc, bool core_exited) { pm_runtime_get_sync(dwc->dev); - dwc3_core_exit_mode(dwc); + if (!core_exited) + dwc3_core_exit_mode(dwc); dwc3_debugfs_exit(dwc); - dwc3_core_exit(dwc); + if (!core_exited) + dwc3_core_exit(dwc); dwc3_ulpi_exit(dwc); pm_runtime_allow(dwc->dev); @@ -2434,8 +2442,19 @@ void dwc3_core_remove(struct dwc3 *dwc) power_supply_put(dwc->usb_psy); } } + +void dwc3_core_remove(struct dwc3 *dwc) +{ + dwc3_core_remove_common(dwc, false); +} EXPORT_SYMBOL_GPL(dwc3_core_remove); +void dwc3_core_remove_no_shutdown(struct dwc3 *dwc) +{ + dwc3_core_remove_common(dwc, true); +} +EXPORT_SYMBOL_GPL(dwc3_core_remove_no_shutdown); + static void dwc3_remove(struct platform_device *pdev) { dwc3_core_remove(platform_get_drvdata(pdev)); diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index 232a4de95bb75..e1d1104db98a3 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -14,6 +14,7 @@ #include #include #include +#include #include "dwc3-apple-state.h" #include "glue.h" @@ -96,6 +97,11 @@ * @fixed_hub: A fixed hub sits on the controller's USB2 port, the controller stays in host mode * @lock: Mutex for synchronizing access * @state: Current state of the controller, see enum dwc3_apple_state and the comment above + * @core_active: Core still owns initialized PHYs + * @core_exit_pending: Event buffers have been cleaned for this shutdown + * @reset_pending: Controller reset must complete before another start + * @usb2_power_off_pending: Core-owned USB2 references still to release + * @usb3_power_off_pending: Core-owned USB3 references still to release */ struct dwc3_apple { struct dwc3 dwc; @@ -113,6 +119,11 @@ struct dwc3_apple { struct mutex lock; enum dwc3_apple_state state; + bool core_active; + bool core_exit_pending; + bool reset_pending; + unsigned long usb2_power_off_pending; + unsigned long usb3_power_off_pending; }; #define to_dwc3_apple(d) container_of((d), struct dwc3_apple, dwc) @@ -254,6 +265,52 @@ static int dwc3_apple_core_init(struct dwc3_apple *appledwc) return ret; } +/* Keep failed releases owned by the glue, without repeating completed teardown. */ +static int dwc3_apple_core_stop(struct dwc3_apple *appledwc) +{ + struct dwc3 *dwc = &appledwc->dwc; + int i, ret; + + lockdep_assert_held(&appledwc->lock); + if (appledwc->core_active) { + if (!appledwc->core_exit_pending) { + dwc3_enable_susphy(dwc, true); + dwc3_event_buffers_cleanup(dwc); + appledwc->core_exit_pending = true; + } + for (i = 0; i < dwc->num_usb3_ports; i++) { + if (!(appledwc->usb3_power_off_pending & BIT(i))) + continue; + ret = phy_power_off(dwc->usb3_generic_phy[i]); + if (ret) + return ret; + appledwc->usb3_power_off_pending &= ~BIT(i); + } + for (i = 0; i < dwc->num_usb2_ports; i++) { + if (!(appledwc->usb2_power_off_pending & BIT(i))) + continue; + ret = phy_power_off(dwc->usb2_generic_phy[i]); + if (ret) + return ret; + appledwc->usb2_power_off_pending &= ~BIT(i); + } + usb_phy_set_suspend(dwc->usb3_phy, 1); + usb_phy_set_suspend(dwc->usb2_phy, 1); + dwc3_core_exit_after_phy_power_off(dwc); + appledwc->core_active = false; + appledwc->core_exit_pending = false; + } + if (appledwc->reset_pending) { + ret = reset_control_assert(appledwc->reset); + if (ret) + return ret; + appledwc->reset_pending = false; + } + dev_pm_syscore_device(appledwc->dev, false); + + return 0; +} + /* * Bring the core up for the given role: configure the USB2 PHY, release the reset and probe * (on the first call) or initialise the core. @@ -265,6 +322,11 @@ static int dwc3_apple_core_start(struct dwc3_apple *appledwc, enum dwc3_apple_st lockdep_assert_held(&appledwc->lock); + /* A failed exit must finish before any new PHY reference is acquired. */ + ret = dwc3_apple_core_stop(appledwc); + if (ret) + return ret; + /* * The USB2 PHY on this platform must be configured for device mode before dwc3 tries to * access it. Otherwise, the new configuration will sometimes only take affect after the @@ -297,11 +359,14 @@ static int dwc3_apple_core_start(struct dwc3_apple *appledwc, enum dwc3_apple_st return ret; } + appledwc->reset_pending = true; + /* Keep register access powered until both PHY shutdown and reset finish. */ + dev_pm_syscore_device(appledwc->dev, true); ret = dwc3_apple_core_init(appledwc); if (ret) { - ret_reset = reset_control_assert(appledwc->reset); + ret_reset = dwc3_apple_core_stop(appledwc); if (ret_reset) - dev_warn(appledwc->dev, "Failed to assert reset, err=%d\n", ret_reset); + dev_warn(appledwc->dev, "Failed to finish core shutdown, err=%d\n", ret_reset); return ret; } @@ -310,6 +375,9 @@ static int dwc3_apple_core_start(struct dwc3_apple *appledwc, enum dwc3_apple_st * configure some unknown Apple-specific settings and then bring up xhci or gadget mode. */ dwc3_apple_setup_cio(appledwc); + appledwc->core_active = true; + appledwc->usb2_power_off_pending = BIT(appledwc->dwc.num_usb2_ports) - 1; + appledwc->usb3_power_off_pending = BIT(appledwc->dwc.num_usb3_ports) - 1; return 0; } @@ -386,51 +454,32 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st return 0; core_exit: - dwc3_core_exit(&appledwc->dwc); - ret_reset = reset_control_assert(appledwc->reset); + ret_reset = dwc3_apple_core_stop(appledwc); if (ret_reset) - dev_warn(appledwc->dev, "Failed to assert reset, err=%d\n", ret_reset); + dev_warn(appledwc->dev, "Failed to finish core shutdown, err=%d\n", ret_reset); return ret; } static int dwc3_apple_exit(struct dwc3_apple *appledwc) { - int ret = 0; - lockdep_assert_held(&appledwc->lock); switch (appledwc->state) { case DWC3_APPLE_PROBE_PENDING: case DWC3_APPLE_NO_CABLE: - /* Nothing to do if we're already off */ - return 0; + break; case DWC3_APPLE_DEVICE: - dev_pm_syscore_device(appledwc->dev, false); dwc3_gadget_exit(&appledwc->dwc); + appledwc->state = DWC3_APPLE_NO_CABLE; break; case DWC3_APPLE_HOST: - dev_pm_syscore_device(appledwc->dev, false); dwc3_host_exit(&appledwc->dwc); + appledwc->state = DWC3_APPLE_NO_CABLE; break; } - /* - * This platform requires SUSPHY to be enabled in order to properly power down the PHY - * and switch dwc3's PIPE interface back to a dummy PHY (i.e. no USB3 support and USB2 via - * a different PHY connected through ULPI). - */ - dwc3_enable_susphy(&appledwc->dwc, true); - dwc3_core_exit(&appledwc->dwc); - appledwc->state = DWC3_APPLE_NO_CABLE; - - ret = reset_control_assert(appledwc->reset); - if (ret) { - dev_err(appledwc->dev, "Failed to assert reset, err=%d\n", ret); - return ret; - } - - return 0; + return dwc3_apple_core_stop(appledwc); } static int dwc3_usb_role_switch_set(struct usb_role_switch *sw, enum usb_role role) @@ -596,9 +645,11 @@ static int dwc3_apple_probe(struct platform_device *pdev) scoped_guard(mutex, &appledwc->lock) ret = dwc3_apple_init(appledwc, DWC3_APPLE_HOST); if (ret) { - /* dwc3_apple_init() has already undone the core initialisation */ + /* A failed cleanup remains visible even when probe cannot be retried. */ + if (appledwc->core_active || appledwc->reset_pending) + dev_err(dev, "Host startup left pending PHY/reset cleanup\n"); if (appledwc->state != DWC3_APPLE_PROBE_PENDING) - dwc3_core_remove(&appledwc->dwc); + dwc3_core_remove_no_shutdown(&appledwc->dwc); usb_role_switch_unregister(appledwc->role_sw); return dev_err_probe(dev, ret, "Failed to start host mode\n"); } @@ -620,9 +671,10 @@ static void dwc3_apple_remove(struct platform_device *pdev) * dwc3_core_probe was never called and there's hence no need to call dwc3_core_remove. * dwc3_apple_exit can be called unconditionally because it checks the state itself. */ - dwc3_apple_exit(appledwc); + if (dwc3_apple_exit(appledwc)) + dev_err(appledwc->dev, "Removal left pending PHY/reset cleanup\n"); if (appledwc->state != DWC3_APPLE_PROBE_PENDING) - dwc3_core_remove(&appledwc->dwc); + dwc3_core_remove_no_shutdown(&appledwc->dwc); } static const struct of_device_id dwc3_apple_of_match[] = { diff --git a/drivers/usb/dwc3/glue.h b/drivers/usb/dwc3/glue.h index d738e1739ae01..db5f32df712bf 100644 --- a/drivers/usb/dwc3/glue.h +++ b/drivers/usb/dwc3/glue.h @@ -108,6 +108,26 @@ int dwc3_core_init(struct dwc3 *dwc); */ void dwc3_core_exit(struct dwc3 *dwc); +/** + * dwc3_core_exit_after_phy_power_off - Finish a glue-owned core shutdown + * @dwc: Pointer to DWC3 controller context + * + * The glue must first stop host/gadget operation, clean up event buffers, + * release every core-owned generic PHY power reference and suspend legacy + * USB PHYs. This finishes PHY exit, clock disable and reset exactly once. + */ +void dwc3_core_exit_after_phy_power_off(struct dwc3 *dwc); + +/** + * dwc3_core_remove_no_shutdown - Release resources without a second shutdown + * @dwc: Pointer to DWC3 controller context + * + * The glue owns host/gadget and core shutdown, including reporting any + * failure to finish it. This releases software resources without repeating + * those operations or discarding another generic PHY reference. + */ +void dwc3_core_remove_no_shutdown(struct dwc3 *dwc); + /** * dwc3_host_init - Initialize host mode operation * @dwc: Pointer to DWC3 controller context diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 8e4a2da2c5ff1..12c0b5330611a 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -809,6 +809,23 @@ static void cd321x_update_work(struct work_struct *work) /* A different dock must release the old routing before reconfiguration. */ was_disconnected |= partner_changed; + /* + * Report sink absence before any fallible hardware transition. A failed + * shutdown retains the route/PHY owner, but cannot keep the sink present. + * Every Type-C port references the same external DCP connector. Only the + * port which successfully acquired the shared display route may report a + * disconnect. A second DP sink can negotiate Alt Mode even though its mux + * request is rejected with -EBUSY; treating that sink as the connector + * owner would tear down the display carried by the first port. + */ + if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && + dp_route_was_active && + (!new_connected || was_disconnected || !dp_connected || !dp_hpd || + dp_hpd_changed || dp_mode_changed)) { + drm_connector_oob_hotplug_event(cd321x->connector_fwnode, + connector_status_disconnected); + } + /* * ACIO carries the tunneled PCIe reset handshake over the still-live * Type-C data path. Close a disappearing USB4/TBT session before dropping @@ -831,20 +848,6 @@ static void cd321x_update_work(struct work_struct *work) goto retry; } - /* - * Every Type-C port references the same external DCP connector. Only the - * port which successfully acquired the shared display route may report a - * disconnect. A second DP sink can negotiate Alt Mode even though its mux - * request is rejected with -EBUSY; treating that sink as the connector - * owner would tear down the display carried by the first port. - */ - if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && - dp_route_was_active && - (!new_connected || was_disconnected || !dp_connected || !dp_hpd || - dp_hpd_changed || dp_mode_changed)) { - drm_connector_oob_hotplug_event(cd321x->connector_fwnode, connector_status_disconnected); - } - /* Process partner disconnection or change */ if (!new_connected || partner_changed) { if (!IS_ERR(tps->partner)) @@ -1541,7 +1544,17 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) cd321x->state.alt = NULL; cd321x->state.mode = TYPEC_STATE_SAFE; cd321x->state.data = NULL; - cd321x->state_valid = !typec_set_mode(tps->port, TYPEC_STATE_SAFE); + ret = typec_set_mode(tps->port, TYPEC_STATE_SAFE); + cd321x->state_valid = !ret; + /* + * A fixed host can register its SuperSpeed root after the mux provider. + * Wait for that consumer at probe instead of exhausting the cable retry + * budget before it exists. Host startup does not depend on this port. + */ + if (ret == -EAGAIN) { + ret = -EPROBE_DEFER; + goto err_put_switch; + } cd321x->tbt_notifier.notifier_call = cd321x_tbt_notify; ret = typec_thunderbolt_switch_register_notifier(cd321x->tbt_switch, @@ -1556,6 +1569,8 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) typec_thunderbolt_switch_put(cd321x->tbt_switch); cd321x->tbt_switch = NULL; err_unregister_mux: + fwnode_handle_put(cd321x->connector_fwnode); + cd321x->connector_fwnode = NULL; typec_mux_put(cd321x->mux); cd321x->mux = NULL; err_unregister_altmodes: @@ -1580,6 +1595,8 @@ cd321x_unregister_port(struct tps6598x *tps) struct cd321x *cd321x = container_of(tps, struct cd321x, tps); cd321x_stop_work(cd321x); + fwnode_handle_put(cd321x->connector_fwnode); + cd321x->connector_fwnode = NULL; typec_thunderbolt_switch_put(cd321x->tbt_switch); cd321x->tbt_switch = NULL; typec_mux_put(cd321x->mux); From d51e799068b6cf3183f4da51a2574244dc781630 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 19:57:31 +1100 Subject: [PATCH 1029/2680] phy: apple: atc: Own the attached USB3 link across the fixed-host handoff The USB3 part of the fixed-host PHY handoff reached aurora-wip only as content of an integration merge; no commit carries it. Re-make it as a commit so the 7.2.10 port does not lose it. The merge is commit 08e0d0a90be3 ("drm/apple: integrate reviewed fixed-host lifecycle and scoped boot fixes"). atcphy_usb3_pm_get() now takes ownership of the attached upstream USB3 link (the SuperSpeed root of the fixed hub) for the duration of the handoff and refuses a detached or quiescing root, and atcphy_usb3_pm_put() releases exactly what was taken. Together with the DWC3/xHCI removal ordering this keeps the root from being torn down under the PHY while the PIPE is switched. The content is the delta of that merge over the automatic merge of its parents, applied unchanged. The apple-dcp selftest (pm.py) that models this sequence lives in a directory the 200-dcp and 900 topics create, so its hunks are handed to them instead. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 101 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 2c0ef71a8291f..f3ed26db09234 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -3450,10 +3450,58 @@ static int atcphy_probe_switch(struct apple_atcphy *atcphy) */ struct atcphy_usb3_pm { struct platform_device *controller; + struct device *host; struct usb_device *root; + struct usb_device *link; }; #if IS_REACHABLE(CONFIG_USB) +/* The root's runtime state does not cover selective suspend of its port. */ +static int atcphy_usb3_link_get(struct atcphy_usb3_pm *pm) +{ + struct usb_device *link; + int ret; + + /* The graph identifies the controller, not a port in a multiport root. */ + if (pm->root->maxchild != 1) + return pm->root->maxchild ? -ENODEV : -EAGAIN; + link = usb_hub_find_child(pm->root, 1); + if (!link) + return 0; + pm->link = usb_get_dev(link); + usb_lock_device(pm->link); + if (READ_ONCE(pm->link->state) == USB_STATE_NOTATTACHED) { + ret = 0; + goto unlock_link; + } + ret = pm_runtime_resume_and_get(&pm->link->dev); + if (ret < 0) + goto unlock_link; + if (READ_ONCE(pm->link->state) == USB_STATE_NOTATTACHED) { + pm_runtime_put_noidle(&pm->link->dev); + ret = -ENODEV; + goto unlock_link; + } + return 0; + +unlock_link: + usb_unlock_device(pm->link); + usb_put_dev(pm->link); + pm->link = NULL; + return ret; +} + +static void atcphy_usb3_link_put(struct atcphy_usb3_pm *pm) +{ + if (!pm->link) + return; + usb_mark_last_busy(pm->link); + pm_runtime_put_autosuspend(&pm->link->dev); + usb_unlock_device(pm->link); + usb_put_dev(pm->link); + pm->link = NULL; +} + static int atcphy_usb3_pm_find(struct usb_device *udev, void *data) { struct atcphy_usb3_pm *pm = data; @@ -3488,6 +3536,17 @@ static int atcphy_usb3_pm_get(struct apple_atcphy *atcphy, struct atcphy_usb3_pm if (!pm->controller) return -EAGAIN; + /* Driver core takes this lock before DWC3 removal touches SUSPHY. */ + device_lock(&pm->controller->dev); + if (!pm->controller->dev.driver) { + ret = -EAGAIN; + goto put_controller; + } + if (strcmp(pm->controller->dev.driver->name, "dwc3-apple")) { + ret = -ENODEV; + goto put_controller; + } + /* Firmware identity, not a bus number, binds this PHY to its root hub. */ usb_for_each_dev(pm, atcphy_usb3_pm_find); if (!pm->root) { @@ -3495,7 +3554,20 @@ static int atcphy_usb3_pm_get(struct apple_atcphy *atcphy, struct atcphy_usb3_pm goto put_controller; } - /* Exclude root-hub teardown while its controller backs the handoff. */ + /* Also serialize direct xHCI unbind, before it marks the HCD removing. */ + pm->host = get_device(pm->root->bus->controller); + device_lock(pm->host); + if (!pm->host->driver) { + ret = -EAGAIN; + goto put_root; + } + if (pm->host->parent != &pm->controller->dev || + strcmp(pm->host->driver->name, "xhci-hcd")) { + ret = -ENODEV; + goto put_root; + } + + /* USB destruction follows the platform lifecycle locks, not vice versa. */ usb_lock_device(pm->root); hcd = bus_to_hcd(pm->root->bus); if (READ_ONCE(pm->root->state) == USB_STATE_NOTATTACHED || HCD_DEAD(hcd)) { @@ -3511,13 +3583,33 @@ static int atcphy_usb3_pm_get(struct apple_atcphy *atcphy, struct atcphy_usb3_pm ret = -ENODEV; goto unlock_root; } + /* Resume the attached upstream link before taking the ATC mutex. */ + ret = atcphy_usb3_link_get(pm); + if (ret) { + usb_mark_last_busy(pm->root); + pm_runtime_put_autosuspend(&pm->root->dev); + goto unlock_root; + } + if (READ_ONCE(pm->root->state) == USB_STATE_NOTATTACHED || + READ_ONCE(hcd->state) != HC_STATE_RUNNING || + HCD_DEAD(hcd) || !HCD_HW_ACCESSIBLE(hcd)) { + atcphy_usb3_link_put(pm); + pm_runtime_put_noidle(&pm->root->dev); + ret = -ENODEV; + goto unlock_root; + } return 0; unlock_root: usb_unlock_device(pm->root); +put_root: + device_unlock(pm->host); + put_device(pm->host); + pm->host = NULL; usb_put_dev(pm->root); pm->root = NULL; put_controller: + device_unlock(&pm->controller->dev); put_device(&pm->controller->dev); pm->controller = NULL; return ret; @@ -3533,12 +3625,19 @@ static void atcphy_usb3_pm_put(struct atcphy_usb3_pm *pm) if (!pm->root) return; + /* Release child ownership before the root: keep the lock order hierarchical. */ + atcphy_usb3_link_put(pm); + /* Restore normal runtime policy after both success and failed parking. */ usb_mark_last_busy(pm->root); pm_runtime_put_autosuspend(&pm->root->dev); usb_unlock_device(pm->root); usb_put_dev(pm->root); + device_unlock(pm->host); + put_device(pm->host); + device_unlock(&pm->controller->dev); put_device(&pm->controller->dev); + pm->host = NULL; pm->root = NULL; pm->controller = NULL; #endif From 88dd4bcb8b5bdb61102404cdab43e644fcb37a6d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 19:59:14 +1100 Subject: [PATCH 1030/2680] thunderbolt: apple: Only cold-init t8103 PCIe-C ports from the kernel-init flag apple_cio_pcie_tunnel_is_preinitialized() treats "apple,pciec-kernel-init" as "the PCIe driver brings this port up itself". That is only true for t8103 ports; T600x/T602x ports are cold-initialized only when the PCIe driver says so through apple_pcie_tunnel_needs_cold_init(). Check the compatible as well, so a T600x/T602x port with the flag set but cold init disabled is not treated as initialized. This is the resolution DjDeveloperr made in an integration merge when joining the two cold-init lines; no commit carries it, so it is re-made here to keep the 7.2.10 port equal to aurora-wip. The merge is commit 45da41c0cd1b ("Merge the running hotplug and mixed-ABI kernel into aurora-wip"). Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/thunderbolt/apple.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 1a5a97925b6c3..443c80e74c1f7 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -563,7 +563,8 @@ apple_cio_pcie_tunnel_is_preinitialized(struct device_node *parent) &status) && status == 1) return true; /* t8103 brings the port up itself. */ - if (of_property_read_bool(child, "apple,pciec-kernel-init")) + if (of_device_is_compatible(child, "apple,t8103-pciec") && + of_property_read_bool(child, "apple,pciec-kernel-init")) return true; /* T600x/T602x without a handoff, when opted in. */ return apple_pcie_tunnel_needs_cold_init(child); From 2e5235268de14e84aa0d8c5e0fcc71c372931d89 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 19:59:14 +1100 Subject: [PATCH 1031/2680] thunderbolt: apple: Allow system sleep with a link the routers keep up The Apple Silicon driver in the 7.2 base refuses system sleep from apple_cio_prepare() while a USB4/Thunderbolt connection is active, because a router that is told to sleep drops its link and the tunnels do not come back. Our resume support no longer asks the routers to sleep: unless the router_sleep test parameter is set, the NHI is marked QUIRK_NO_SYSTEM_SLEEP for the cycle, the tunneled PCIe-C host keeps its link (QUIRK_KEEP_TUNNELS) and the DP resources are paired up again after resume. With the refusal in place none of that can run and a docked M1 or M2 laptop could not suspend at all, which is a regression against 7.1, where the driver had no such check. Keep the refusal only for the router_sleep test mode, where the reason for it still holds. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/thunderbolt/apple.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 443c80e74c1f7..4a46b0b7637c9 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -2552,7 +2552,11 @@ static int apple_cio_prepare(struct device *dev) guard(mutex)(&acio->lock); - if (acio->current_cable_info) { + /* + * Routers are kept awake across system sleep unless router_sleep is + * set; only a router told to sleep loses an active connection. + */ + if (READ_ONCE(router_sleep) && acio->current_cable_info) { dev_err(dev, "unable to suspend while a USB4/Thunderbolt connection is active\n"); return -EBUSY; } From 01d99f648d7e7b83f3ec102dfbdebfae8dc9547d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 25 Aug 2026 14:16:23 -0400 Subject: [PATCH 1032/2680] drm: apple: support shared Type-C display routes Signed-off-by: DjDeveloperr --- drivers/gpu/drm/apple/dcp-internal.h | 16 +++ drivers/gpu/drm/apple/dcp.c | 153 ++++++++++++++++++++++++++- drivers/gpu/drm/apple/dptxep.c | 10 +- 3 files changed, 173 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index a2e992b808513..d3bbf3d63d4fc 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -23,10 +23,21 @@ #include "version_utils.h" #define DCP_MAX_PLANES 4 +#define DCP_MAX_TYPEC_ROUTES 4 struct apple_dcp; struct apple_dcp_afkep; +struct apple_dcp_typec_route { + struct apple_dcp *dcp; + struct phy *phy; + struct mux_control *xbar; + struct typec_mux_dev *typec_mux; + u32 dptx_phy; + u32 mux_index; + bool selected; +}; + struct dcpav_service_epic; enum { @@ -253,6 +264,11 @@ struct apple_dcp { struct phy *phy; struct mux_control *xbar; struct typec_mux *typec_mux; + struct apple_dcp_typec_route typec_routes[DCP_MAX_TYPEC_ROUTES]; + struct apple_dcp_typec_route *active_typec_route; + struct mutex typec_route_lock; /* serializes Type-C route ownership */ + u32 nr_typec_routes; + bool phy_managed_by_typec; struct gpio_desc *hdmi_hpd; struct gpio_desc *hdmi_pwren; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 7629cf080f6d2..1c293d0e90d50 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -58,6 +58,150 @@ static bool unstable_edid = true; module_param(unstable_edid, bool, 0644); MODULE_PARM_DESC(unstable_edid, "Enable unstable EDID retrival support"); +static bool dcp_typec_route_is_dp(const struct typec_mux_state *state) +{ + return state->alt && state->alt->svid == USB_TYPEC_DP_SID && + (state->mode == TYPEC_DP_STATE_C || + state->mode == TYPEC_DP_STATE_D || + state->mode == TYPEC_DP_STATE_E); +} + +static int dcp_typec_route_set(struct typec_mux_dev *mux, + struct typec_mux_state *state) +{ + struct apple_dcp_typec_route *route = typec_mux_get_drvdata(mux); + struct apple_dcp *dcp = route->dcp; + int ret = 0; + + guard(mutex)(&dcp->typec_route_lock); + + if (!dcp_typec_route_is_dp(state)) { + if (!route->selected) + return 0; + + ret = mux_control_deselect(route->xbar); + route->selected = false; + if (dcp->active_typec_route == route) + dcp->active_typec_route = NULL; + return ret; + } + + if (route->selected) + return 0; + + if (dcp->active_typec_route) { + dev_warn(dcp->dev, "DP route is already in use by another Type-C port\n"); + return -EBUSY; + } + + ret = mux_control_select(route->xbar, route->mux_index); + if (ret) + return ret; + + dcp->phy = route->phy; + dcp->dptx_phy = route->dptx_phy; + dcp->active_typec_route = route; + route->selected = true; + + dev_info(dcp->dev, "Routed DCP output to Type-C DPTX PHY %u\n", + route->dptx_phy); + return 0; +} + +static void dcp_typec_route_unregister(void *data) +{ + struct apple_dcp_typec_route *route = data; + struct apple_dcp *dcp = route->dcp; + + guard(mutex)(&dcp->typec_route_lock); + if (route->selected) { + mux_control_deselect(route->xbar); + route->selected = false; + if (dcp->active_typec_route == route) + dcp->active_typec_route = NULL; + } + typec_mux_unregister(route->typec_mux); +} + +static int dcp_register_typec_routes(struct apple_dcp *dcp) +{ + struct device_node *routes __free(device_node) = + of_get_child_by_name(dcp->dev->of_node, "typec-routes"); + struct device *dev = dcp->dev; + u32 route_index; + int ret; + + if (!routes) + return 0; + + for_each_available_child_of_node_scoped(routes, route_np) { + struct apple_dcp_typec_route *route; + struct typec_mux_desc desc = {}; + const char *name; + + if (dcp->nr_typec_routes == DCP_MAX_TYPEC_ROUTES) + return dev_err_probe(dev, -E2BIG, "Too many Type-C display routes\n"); + + ret = of_property_read_u32(route_np, "reg", &route_index); + if (ret) + return dev_err_probe(dev, ret, "%pOF: missing route index\n", route_np); + if (route_index >= DCP_MAX_TYPEC_ROUTES) + return dev_err_probe(dev, -EINVAL, "%pOF: invalid route index %u\n", + route_np, route_index); + + name = devm_kasprintf(dev, GFP_KERNEL, "typec%u", route_index); + if (!name) + return -ENOMEM; + + route = &dcp->typec_routes[dcp->nr_typec_routes]; + route->dcp = dcp; + route->phy = devm_phy_get(dev, name); + if (IS_ERR(route->phy)) + return dev_err_probe(dev, PTR_ERR(route->phy), + "%pOF: failed to get DP PHY\n", route_np); + + route->xbar = devm_mux_control_get(dev, name); + if (IS_ERR(route->xbar)) + return dev_err_probe(dev, PTR_ERR(route->xbar), + "%pOF: failed to get display crossbar\n", route_np); + + ret = of_property_read_u32_index(dev->of_node, "apple,typec-mux-indices", + route_index, &route->mux_index); + if (ret) + return dev_err_probe(dev, ret, "%pOF: missing crossbar state\n", + route_np); + + ret = of_property_read_u32_index(dev->of_node, "apple,typec-dptx-phys", + route_index, &route->dptx_phy); + if (ret) + return dev_err_probe(dev, ret, "%pOF: missing DPTX PHY index\n", + route_np); + + desc.fwnode = of_fwnode_handle(route_np); + desc.set = dcp_typec_route_set; + desc.name = name; + desc.drvdata = route; + route->typec_mux = typec_mux_register(dev, &desc); + if (IS_ERR(route->typec_mux)) + return dev_err_probe(dev, PTR_ERR(route->typec_mux), + "%pOF: failed to register Type-C route\n", route_np); + + ret = devm_add_action_or_reset(dev, dcp_typec_route_unregister, route); + if (ret) + return ret; + + if (!dcp->phy) + dcp->phy = route->phy; + dcp->nr_typec_routes++; + } + + if (!dcp->nr_typec_routes) + return dev_err_probe(dev, -EINVAL, "Type-C route container is empty\n"); + + dcp->phy_managed_by_typec = true; + return 0; +} + /* copied and simplified from drm_vblank.c */ static void send_vblank_event(struct drm_device *dev, struct drm_pending_vblank_event *e, @@ -1168,7 +1312,7 @@ static int dcp_platform_probe(struct platform_device *pdev) enum dcp_firmware_version fw_compat; struct device *dev = &pdev->dev; struct apple_dcp *dcp; - int surf, num_surfs; + int ret, surf, num_surfs; u32 surf_en; u32 mux_index; @@ -1191,6 +1335,7 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp->fw_compat = fw_compat; dcp->dev = dev; dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); + mutex_init(&dcp->typec_route_lock); platform_set_drvdata(pdev, dcp); @@ -1200,6 +1345,10 @@ static int dcp_platform_probe(struct platform_device *pdev) return PTR_ERR(dcp->phy); } + ret = dcp_register_typec_routes(dcp); + if (ret) + return ret; + bitmap_zero(dcp->iomfb_surfaces, DCP_MAX_PLANES); if (!of_property_present(dev->of_node, "apple,iomfb-surfaces")) num_surfs = 0; @@ -1295,6 +1444,8 @@ static int dcp_platform_probe(struct platform_device *pdev) }; int ret = typec_mux_set(dcp->typec_mux, &state); dev_info(dev, "typec_mux_set() returned: %d\n", ret); + if (!ret) + dcp->phy_managed_by_typec = true; } else { dev_info(dev, "fwnode_typec_mux_get() returned: %ld\n", IS_ERR(dcp->typec_mux) ? PTR_ERR(dcp->typec_mux) : 0); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 1ea00602d476d..fa47374304022 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -478,9 +478,8 @@ dptxport_call_activate(struct apple_epic_service *service, struct dptx_port *dptx = service->cookie; const struct apple_dcp *dcp = service->ep->dcp; - // TODO: hack, use phy_set_mode to select the correct DCP(EXT) input - // for standalone phy (i.e. not atc phy). - if (!dcp->typec_mux) + /* Standalone PHYs need DCP input selection here. Type-C owns ATC PHY mode. */ + if (!dcp->phy_managed_by_typec) phy_set_mode_ext(dptx->atcphy, PHY_MODE_DP, dcp->index); memcpy(reply, data, min(reply_size, data_size)); @@ -496,9 +495,10 @@ dptxport_call_deactivate(struct apple_epic_service *service, void *reply, size_t reply_size) { struct dptx_port *dptx = service->cookie; + const struct apple_dcp *dcp = service->ep->dcp; - /* deactivate phy */ - phy_set_mode_ext(dptx->atcphy, PHY_MODE_INVALID, 0); + if (!dcp->phy_managed_by_typec) + phy_set_mode_ext(dptx->atcphy, PHY_MODE_INVALID, 0); memcpy(reply, data, min(reply_size, data_size)); if (reply_size >= 4) From 59036cc66c7090fa6cb70bc0c6d8041786970355 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 26 Aug 2026 01:16:44 -0400 Subject: [PATCH 1033/2680] drm: apple: ignore legacy mux-index on Type-C DCPs Signed-off-by: DjDeveloperr --- drivers/gpu/drm/apple/dcp.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 1c293d0e90d50..54d25b149bf4e 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1418,7 +1418,16 @@ static int dcp_platform_probe(struct platform_device *pdev) if (IS_ERR(dcp->dp2hdmi_pwren)) return PTR_ERR(dcp->dp2hdmi_pwren); - ret = of_property_read_u32(dev->of_node, "mux-index", &mux_index); + /* + * Firmware or an inherited board DTS may leave the legacy mux-index + * property on external DCP nodes. Multi-route USB-C DCPs select their + * crossbar through the per-port routes registered above and do not + * have a single "dp-xbar" mux. Treat mux-index only as the legacy + * single-route binding; otherwise one stale property prevents every + * Type-C controller from probing while they wait for the DCP routes. + */ + ret = dcp->nr_typec_routes ? -ENODATA : + of_property_read_u32(dev->of_node, "mux-index", &mux_index); if (!ret) { dcp->xbar = devm_mux_control_get(dev, "dp-xbar"); if (IS_ERR(dcp->xbar)) { From c4db563b631c4f2ee983587bc616e4a1697bb8eb Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 29 Aug 2026 16:40:20 -0400 Subject: [PATCH 1034/2680] drm: apple: recover Type-C displays across link interruptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Type-C display that briefly loses its route came back as a powered but blank sink. DCP discards its display mode on link loss while the DRM CRTC stays active, so userspace keeps submitting plane-only commits that never retrain the link, and the swap for such a commit is never completed. Track whether the cable is present, re-establish the DPTX session from a freezable worker when DCP tears its synthetic HPD down with the cable still attached, and re-apply the active CRTC state once the link-config callback has run. Bounce plane-only commits that arrive without a valid mode to the vblank worker so they cannot sit in front of the recovery commit with an unsignalled flip event. Tell DCP whether the routed output supports HPD when connecting the DPTX port, report the real HPD state back from set_hpd instead of assuming success, and answer the tiled-display hint so DCP stops retrying it. [ryan: adapted to 7.2.10 — struct drm_atomic_state is struct drm_atomic_commit] Signed-off-by: DjDeveloperr --- drivers/gpu/drm/apple/apple_drv.c | 5 +- drivers/gpu/drm/apple/dcp-internal.h | 3 + drivers/gpu/drm/apple/dcp.c | 154 +++++++++++++++++++++++++-- drivers/gpu/drm/apple/dptxep.c | 28 ++++- drivers/gpu/drm/apple/dptxep.h | 3 +- drivers/gpu/drm/apple/iomfb.c | 51 ++++++++- 6 files changed, 227 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 85d8302d30c64..f8d8788afbf53 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -89,8 +89,6 @@ static void apple_connector_oob_hotplug(struct drm_connector *connector, { struct apple_connector *apple_connector = to_apple_connector(connector); - printk("#### oob_hotplug status:0x%x ####\n", (u32)status); - if (status == connector_status_connected) dcp_dptx_connect_oob(apple_connector->dcp, 0); else if (status == connector_status_disconnected) @@ -342,6 +340,9 @@ static int apple_probe_per_dcp(struct device *dev, dcp_get_connector_type(dcp)); if (ret) return ret; + ret = drm_connector_attach_vrr_capable_property(&connector->base); + if (ret) + return ret; connector->base.polled = DRM_CONNECTOR_POLL_HPD; connector->connected = false; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index d3bbf3d63d4fc..4c78f19f47b4d 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -269,6 +269,9 @@ struct apple_dcp { struct mutex typec_route_lock; /* serializes Type-C route ownership */ u32 nr_typec_routes; bool phy_managed_by_typec; + bool typec_cable_connected; + struct delayed_work typec_reconnect_wq; + u32 typec_reconnect_tries; struct gpio_desc *hdmi_hpd; struct gpio_desc *hdmi_pwren; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 54d25b149bf4e..8fdbb02f1f9c9 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -500,6 +500,8 @@ int dcp_get_connector_type(struct platform_device *pdev) } #define DPTX_CONNECT_TIMEOUT msecs_to_jiffies(2000) +#define DPTX_RECONNECT_DELAY msecs_to_jiffies(1000) +#define DPTX_RECONNECT_RETRIES 5 static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) { @@ -523,19 +525,57 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) reinit_completion(&dcp->dptxport[port].linkcfg_completion); dcp->dptxport[port].atcphy = dcp->phy; - dptxport_connect(dcp->dptxport[port].service, 0, dcp->dptx_phy, dcp->dptx_die); - dptxport_request_display(dcp->dptxport[port].service); + ret = dptxport_validate_connection(dcp->dptxport[port].service, 0, + dcp->dptx_phy, dcp->dptx_die); + if (ret) { + dev_err(dcp->dev, + "dcp_dptx_connect: failed to validate DPTX target %u:%u: %d\n", + dcp->dptx_die, dcp->dptx_phy, ret); + goto out_unlock; + } + + ret = dptxport_connect(dcp->dptxport[port].service, 0, + dcp->dptx_phy, dcp->dptx_die, + dcp->connector_type == DRM_MODE_CONNECTOR_USB); + if (ret) { + dev_err(dcp->dev, + "dcp_dptx_connect: failed to connect DPTX target %u:%u: %d\n", + dcp->dptx_die, dcp->dptx_phy, ret); + goto out_unlock; + } + + ret = dptxport_request_display(dcp->dptxport[port].service); + if (ret) { + dev_err(dcp->dev, + "dcp_dptx_connect: failed to request display: %d\n", + ret); + goto out_release; + } dcp->dptxport[port].connected = true; + if (dcp->connector_type == DRM_MODE_CONNECTOR_USB) { + ret = dptxport_set_hpd(dcp->dptxport[port].service, true); + if (ret) { + dev_err(dcp->dev, + "dcp_dptx_connect: failed to assert Type-C HPD: %d\n", + ret); + dcp->dptxport[port].connected = false; + goto out_release; + } + } mutex_unlock(&dcp->hpd_mutex); ret = wait_for_completion_timeout(&dcp->dptxport[port].linkcfg_completion, DPTX_CONNECT_TIMEOUT); - if (ret < 0) - dev_warn(dcp->dev, "dcp_dptx_connect: port %d link complete failed:%d\n", - port, ret); - else - dev_dbg(dcp->dev, "dcp_dptx_connect: waited %d ms for link\n", - jiffies_to_msecs(DPTX_CONNECT_TIMEOUT - ret)); + if (!ret) { + dev_err(dcp->dev, + "dcp_dptx_connect: timed out waiting for port %u link configuration\n", + port); + ret = -ETIMEDOUT; + goto out_disconnect; + } + + dev_dbg(dcp->dev, "dcp_dptx_connect: waited %d ms for link\n", + jiffies_to_msecs(DPTX_CONNECT_TIMEOUT - ret)); usleep_range(5, 10); @@ -547,11 +587,43 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) return 0; +out_disconnect: + mutex_lock(&dcp->hpd_mutex); + dcp->dptxport[port].connected = false; +out_release: + dptxport_release_display(dcp->dptxport[port].service); + out_unlock: mutex_unlock(&dcp->hpd_mutex); return ret; } +static void dcp_typec_reconnect_work(struct work_struct *work) +{ + struct apple_dcp *dcp = + container_of(to_delayed_work(work), struct apple_dcp, + typec_reconnect_wq); + int ret; + + if (!READ_ONCE(dcp->typec_cable_connected)) + return; + + ret = dcp_dptx_connect(dcp, 0); + if (!ret) { + dcp->typec_reconnect_tries = 0; + return; + } + + if (++dcp->typec_reconnect_tries < DPTX_RECONNECT_RETRIES) { + mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, + DPTX_RECONNECT_DELAY); + return; + } + + dev_err(dcp->dev, "Type-C DPTX reconnect failed after %u retries: %d\n", + dcp->typec_reconnect_tries, ret); +} + static void disconnected_hpd_event(struct apple_connector *con) { if (con && con->connected) { @@ -577,13 +649,31 @@ static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) { struct apple_dcp *dcp = platform_get_drvdata(pdev); - return dcp_dptx_connect(dcp, port); + int ret; + + if (dcp->connector_type == DRM_MODE_CONNECTOR_USB) { + WRITE_ONCE(dcp->typec_cable_connected, true); + dcp->typec_reconnect_tries = 0; + cancel_delayed_work(&dcp->typec_reconnect_wq); + } + + ret = dcp_dptx_connect(dcp, port); + if (ret && dcp->connector_type == DRM_MODE_CONNECTOR_USB) + mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, + DPTX_RECONNECT_DELAY); + + return ret; } int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) { struct apple_dcp *dcp = platform_get_drvdata(pdev); + if (dcp->connector_type == DRM_MODE_CONNECTOR_USB) { + WRITE_ONCE(dcp->typec_cable_connected, false); + cancel_delayed_work(&dcp->typec_reconnect_wq); + } + disconnected_hpd_event(dcp->connector); if (dcp->avep) @@ -799,6 +889,7 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) void dcp_poweron(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); + int ret; if (dcp->hdmi_hpd) { bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); @@ -806,6 +897,20 @@ void dcp_poweron(struct platform_device *pdev) if (connected) dcp_dptx_connect(dcp, 0); + } else if (dcp->connector_type == DRM_MODE_CONNECTOR_USB && + READ_ONCE(dcp->typec_cable_connected)) { + /* + * A Type-C CRTC disable releases its DPTX session. Re-establish + * that session synchronously before powering IOMFB back on, so the + * following modeset cannot race the delayed reconnect worker. + */ + cancel_delayed_work(&dcp->typec_reconnect_wq); + dcp->typec_reconnect_tries = 0; + ret = dcp_dptx_connect(dcp, 0); + if (ret) + mod_delayed_work(system_freezable_wq, + &dcp->typec_reconnect_wq, + DPTX_RECONNECT_DELAY); } switch (dcp->fw_compat) { @@ -830,6 +935,7 @@ void dcp_poweron(struct platform_device *pdev) void dcp_poweroff(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); + int ret; if (dcp->avep) av_service_disconnect(dcp); @@ -842,6 +948,33 @@ void dcp_poweroff(struct platform_device *pdev) disconnected_hpd_event(dcp->connector); dcp_dptx_disconnect(dcp, 0); } + } else if (dcp->connector_type == DRM_MODE_CONNECTOR_USB && + dcp->dptxport[0].enabled && dcp->dptxport[0].connected) { + /* + * Unlike a physical DP/HDMI HPD, the Type-C HPD exposed by the + * DPTX service belongs to this power-on session. Release it when + * the CRTC powers down so the next enable performs the same orderly + * connect/request/HPD sequence as a fresh cable connection. + */ + ret = dptxport_set_hpd(dcp->dptxport[0].service, false); + if (ret) + dev_warn(dcp->dev, + "failed to deassert Type-C DPTX HPD: %d\n", ret); + dcp_dptx_disconnect(dcp, 0); + + /* + * DCP tears down its synthetic HPD when the CRTC powers off, but + * the Type-C controller does not necessarily emit another cable + * connect edge. Recreate the DPTX session while the physical cable + * remains present. A freezable workqueue keeps this out of the + * suspend path; if suspend wins the race, reconnect after thaw. + */ + if (READ_ONCE(dcp->typec_cable_connected)) { + dcp->typec_reconnect_tries = 0; + mod_delayed_work(system_freezable_wq, + &dcp->typec_reconnect_wq, + DPTX_RECONNECT_DELAY); + } } } @@ -1296,6 +1429,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); } + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); cancel_work_sync(&dcp->vblank_wq); devm_clk_put(dev, dcp->clk); @@ -1336,6 +1470,8 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp->dev = dev; dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); mutex_init(&dcp->typec_route_lock); + INIT_DELAYED_WORK(&dcp->typec_reconnect_wq, + dcp_typec_reconnect_work); platform_set_drvdata(pdev, dcp); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index fa47374304022..3d286fc2c846c 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -69,6 +69,10 @@ struct dptxport_apcall_drive_settings { __le32 unk7; }; +struct dptxport_apcall_set_tiled { + __le32 retcode; +}; + int dptxport_validate_connection(struct apple_epic_service *service, u8 core, u8 atc, u8 die) { @@ -98,11 +102,11 @@ int dptxport_validate_connection(struct apple_epic_service *service, u8 core, } int dptxport_connect(struct apple_epic_service *service, u8 core, u8 atc, - u8 die) + u8 die, bool supports_hpd) { struct dptx_port *dptx = service->cookie; struct dcpdptx_connection_cmd cmd, resp; - u32 unk_field = 0x0; // seen as 0x100 under some conditions + u32 unk_field = supports_hpd ? DCPDPTX_REMOTE_PORT_SUPPORTS_HPD : 0; int ret; u32 target = FIELD_PREP(DCPDPTX_REMOTE_PORT_CORE, core) | FIELD_PREP(DCPDPTX_REMOTE_PORT_ATC, atc) | @@ -151,7 +155,7 @@ int dptxport_set_hpd(struct apple_epic_service *service, bool hpd) sizeof(resp), 12); if (ret) return ret; - if (le32_to_cpu(resp.unk) != 1) + if (le32_to_cpu(resp.unk) != hpd) return -EINVAL; return 0; } @@ -446,13 +450,14 @@ static int dptxport_call_set_link_rate(struct apple_epic_service *service, static int dptxport_call_get_supports_hpd(struct apple_epic_service *service, void *reply_, size_t reply_size) { + struct apple_dcp *dcp = service->ep->dcp; struct dptxport_apcall_get_support *reply = reply_; if (reply_size < sizeof(*reply)) return -EINVAL; reply->retcode = cpu_to_le32(0); - reply->supported = cpu_to_le32(0); + reply->supported = cpu_to_le32(dcp->connector_type == DRM_MODE_CONNECTOR_USB); return 0; } @@ -470,6 +475,18 @@ dptxport_call_get_supports_downspread(struct apple_epic_service *service, return 0; } +static int dptxport_call_set_tiled_display_hint(void *reply_, + size_t reply_size) +{ + struct dptxport_apcall_set_tiled *reply = reply_; + + if (reply_size < sizeof(*reply)) + return -EINVAL; + + reply->retcode = cpu_to_le32(1); + return 0; +} + static int dptxport_call_activate(struct apple_epic_service *service, const void *data, size_t data_size, @@ -541,6 +558,9 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, case DPTX_APCALL_GET_MAX_DRIVE_SETTINGS: return dptxport_call_get_max_drive_settings(service, reply, reply_size); + case DPTX_APCALL_SET_TILED_DISPLAY_HINTS: + memcpy(reply, data, min(reply_size, data_size)); + return dptxport_call_set_tiled_display_hint(reply, reply_size); case DPTX_APCALL_GET_DRIVE_SETTINGS: return dptxport_call_get_drive_settings(service, data, data_size, reply, reply_size); diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index 0bf2534054fd7..c8816a040f321 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -36,6 +36,7 @@ enum dptx_apcall { #define DCPDPTX_REMOTE_PORT_ATC GENMASK(7, 4) #define DCPDPTX_REMOTE_PORT_DIE GENMASK(11, 8) #define DCPDPTX_REMOTE_PORT_CONNECTED BIT(15) +#define DCPDPTX_REMOTE_PORT_SUPPORTS_HPD BIT(8) enum dptx_link_rate { LINK_RATE_RBR = 0x06, @@ -63,7 +64,7 @@ struct dptx_port { int dptxport_validate_connection(struct apple_epic_service *service, u8 core, u8 atc, u8 die); int dptxport_connect(struct apple_epic_service *service, u8 core, u8 atc, - u8 die); + u8 die, bool supports_hpd); int dptxport_request_display(struct apple_epic_service *service); int dptxport_release_display(struct apple_epic_service *service); int dptxport_set_hpd(struct apple_epic_service *service, bool hpd); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 5c6cca8eeca6f..c81a74a0f9351 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -16,11 +16,14 @@ #include #include +#include +#include #include #include #include #include #include +#include #include #include @@ -231,10 +234,31 @@ void dcp_ack(struct apple_dcp *dcp, enum dcp_context_id context) * waits for vblank (a DCP callback). That means we deadlock if we call from * the RTKit thread! Instead, move the call to another thread via a workqueue. */ +static int dcp_retrain_active_crtc(struct apple_connector *connector) +{ + struct drm_device *dev = connector->base.dev; + struct drm_modeset_acquire_ctx ctx; + struct drm_crtc *crtc; + int ret; + + DRM_MODESET_LOCK_ALL_BEGIN(dev, ctx, 0, ret); + + crtc = connector->base.state ? connector->base.state->crtc : NULL; + if (crtc && crtc->state && crtc->state->active) + ret = drm_atomic_helper_reset_crtc(crtc, &ctx); + else + ret = 0; + + DRM_MODESET_LOCK_ALL_END(dev, ctx, ret); + + return ret; +} + void dcp_hotplug(struct work_struct *work) { struct apple_connector *connector; struct apple_dcp *dcp; + int ret; connector = container_of(work, struct apple_connector, hotplug_wq); @@ -252,10 +276,24 @@ void dcp_hotplug(struct work_struct *work) * display modes from atomic_flush, so userspace needs to trigger a * flush, or the CRTC gets no signal. */ - if (connector->base.state && !dcp->valid_mode && connector->connected) + if (connector->base.state && !dcp->valid_mode && connector->connected) { drm_connector_set_link_status_property(&connector->base, DRM_MODE_LINK_STATUS_BAD); + /* + * A short Type-C route interruption can leave the DRM CRTC active + * while DCP has discarded its display mode. Userspace is then free + * to keep submitting plane-only commits, none of which retrains the + * link. Re-apply the active CRTC state once the DPTX link-config + * callback has completed. + */ + ret = dcp_retrain_active_crtc(connector); + if (ret) + dev_warn(dcp->dev, + "failed to retrain active CRTC after hotplug: %d\n", + ret); + } + drm_kms_helper_connector_hotplug_event(&connector->base); } @@ -466,6 +504,17 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit) struct platform_device *pdev = to_apple_crtc(crtc)->dcp; struct apple_dcp *dcp = platform_get_drvdata(pdev); + /* + * DCP does not complete swaps after a link loss. A plane-only commit + * arriving between the reconnect callback and the required modeset + * would otherwise leave an unsignalled flip event in front of the + * recovery commit. Modesets set valid_mode before reaching flush. + */ + if (!dcp->valid_mode) { + schedule_work(&dcp->vblank_wq); + return; + } + if (dcp_channel_busy(&dcp->ch_cmd)) { if (!dcp->ch_cmd.warned_busy) { From 7c7dfca757c5a687490222982e3d17b72cd0db5b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 29 Aug 2026 00:57:28 -0400 Subject: [PATCH 1035/2680] drm: apple: retrain Type-C display after USB4 route changes [ryan: split by topic: only the drm/apple part and its selftests are here; the PHY, Type-C, Thunderbolt and crossbar hunks are in 170-usb] Signed-off-by: DjDeveloperr --- drivers/gpu/drm/apple/apple_drv.c | 2 ++ drivers/gpu/drm/apple/connector.h | 1 + drivers/gpu/drm/apple/iomfb.c | 17 +++++++++++++++++ drivers/gpu/drm/drm_connector.c | 4 ++++ 4 files changed, 24 insertions(+) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index f8d8788afbf53..0e9029421cb07 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -93,6 +93,8 @@ static void apple_connector_oob_hotplug(struct drm_connector *connector, dcp_dptx_connect_oob(apple_connector->dcp, 0); else if (status == connector_status_disconnected) dcp_dptx_disconnect_oob(apple_connector->dcp, 0); + else if (status == connector_status_unknown) + dcp_retrain_oob(apple_connector); else dev_err(&apple_connector->dcp->dev, "unexpected connector status" ":0x%x in oob_hotplug event\n", (u32)status); diff --git a/drivers/gpu/drm/apple/connector.h b/drivers/gpu/drm/apple/connector.h index ef2c23737aac6..8c9d367eda94f 100644 --- a/drivers/gpu/drm/apple/connector.h +++ b/drivers/gpu/drm/apple/connector.h @@ -15,6 +15,7 @@ struct apple_connector; #include "dcp-internal.h" void dcp_hotplug(struct work_struct *work); +void dcp_retrain_oob(struct apple_connector *connector); struct apple_connector { struct drm_connector base; diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index c81a74a0f9351..0a883bbd31a5a 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -254,6 +254,23 @@ static int dcp_retrain_active_crtc(struct apple_connector *connector) return ret; } +void dcp_retrain_oob(struct apple_connector *connector) +{ + struct apple_dcp *dcp = platform_get_drvdata(connector->dcp); + + if (!READ_ONCE(connector->connected) || !READ_ONCE(dcp->valid_mode)) + return; + + /* + * Bringing up another high-speed Type-C route can disturb an active DPTX + * stream without changing its HPD state. Invalidate the IOMFB mode and + * use the normal hotplug worker to replay the active CRTC from process + * context; sending a synthetic disconnect would tear down the connector. + */ + WRITE_ONCE(dcp->valid_mode, false); + schedule_work(&connector->hotplug_wq); +} + void dcp_hotplug(struct work_struct *work) { struct apple_connector *connector; diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c index 11646453aaac9..e254206ffd9c8 100644 --- a/drivers/gpu/drm/drm_connector.c +++ b/drivers/gpu/drm/drm_connector.c @@ -3511,6 +3511,10 @@ struct drm_connector *drm_connector_find_by_fwnode(struct fwnode_handle *fwnode) * muxes the DisplayPort data and aux-lines but does not pass the altmode HPD * status bit to the GPU's DP HPD pin. * + * A caller may report &connector_status_unknown when a shared display route + * changed without a logical HPD transition and the connector should recheck or + * retrain its existing link. + * * This function can be used to report these out-of-band events after obtaining * a drm_connector reference through calling drm_connector_find_by_fwnode(). */ From b7d0e92cbca0cb8f445bf9cf19fd49312f370051 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 29 Aug 2026 04:55:06 -0400 Subject: [PATCH 1036/2680] drm: apple: refresh EDID after Type-C disconnect The out-of-band Type-C disconnect path marks the connector offline and emits a hotplug event before the DCP callback arrives. Since that callback then observes no connector state transition, the normal worker never releases the cached EDID. A replacement display can therefore inherit the previous monitor name even though DCP has discovered its new modes. Release the cached EDID in the synchronous out-of-band disconnect path before notifying DRM. The next mode probe will fetch the replacement display EDID from DCPAV. Fixes: 85dd82bc5967 ("drm/apple: support shared Type-C display routes") Signed-off-by: DjDeveloperr --- drivers/gpu/drm/apple/dcp.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8fdbb02f1f9c9..02918703e50ff 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -628,6 +628,8 @@ static void disconnected_hpd_event(struct apple_connector *con) { if (con && con->connected) { con->connected = 0; + drm_edid_free(con->drm_edid); + con->drm_edid = NULL; drm_kms_helper_connector_hotplug_event(&con->base); } } From 2314090e59f04bc557de17a0ab4d5e054b2d117e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 31 Aug 2026 01:09:14 -0400 Subject: [PATCH 1037/2680] drm: apple: route Type-C displays through a shared DCP fabric MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every Type-C port was bound to one fixed external DCP, so a second DP-alt-mode connection collided with the first and was rejected with "DP route is already in use by another Type-C port" even when another external DCP sat idle. The hardware does not work that way: a display crossbar sits between the DCPs and the Type-C PHYs, and any pipeline it can reach can drive any port wired to it. Build a display fabric from the OF graph. Candidate pipelines are grouped by the remote USB-C connector and the first notification to arrive for a port does the work, the rest falling through. There is deliberately no nominated coordinator: fwnode_typec_mux_get() caps the providers one connector may have and drops the remainder silently, so a nominated route might never be called and the port would never be routed at all. The candidate count comes from the device tree, so machines with one, two or more dcpext instances work without a per-board constant. A Type-C output is a physical port, not a pipeline, so give each port its own DRM connector. Userspace keys its per-monitor configuration on the connector name, so ports are enumerated in device-tree order and a port keeps its name whichever pipeline drives it. A pipeline whose only output is Type-C has no connector, and no encoder, of its own. Give each port a single encoder whose possible_crtcs covers every pipeline that can drive it, and narrow that mask to the pipeline actually routed once the fabric picks one. Both halves are needed. Userspace takes the CRTCs a connector can use to be what all of its encoders have in common, so one encoder per pipeline would leave nothing in common and the port would have no usable CRTC at all. Leaving the mask wide instead lets userspace pair the port with a pipeline holding another monitor's mode list, which the CRTC's mode check then rejects. The hotplug that follows a route change makes userspace re-read the mask. While a port has no pipeline it reports disconnected, so nothing probes an output whose ->dcp is NULL. A hybrid pipeline saves its fixed PHY, crossbar and DPTX target, and is only borrowed while its fixed output is idle. On release it parks its DPTX target on a Type-C PHY rather than the fixed one: re-targeting an idle fixed PHY leaves DCP unable to train a link on a later Type-C target, answering DEVICE_NOT_STARTED until every DPTX call times out. [ryan: adapted to 7.2.10 — struct drm_atomic_state is struct drm_atomic_commit] Signed-off-by: DjDeveloperr --- drivers/gpu/drm/apple/apple_drv.c | 228 ++++++-- drivers/gpu/drm/apple/connector.h | 19 + drivers/gpu/drm/apple/dcp-internal.h | 20 +- drivers/gpu/drm/apple/dcp.c | 743 ++++++++++++++++++++++----- drivers/gpu/drm/apple/dcp.h | 11 + drivers/gpu/drm/apple/dptxep.c | 11 +- drivers/gpu/drm/apple/iomfb.c | 13 +- 7 files changed, 888 insertions(+), 157 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 0e9029421cb07..d51e20a5e8dc9 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -44,7 +45,7 @@ #define DRIVER_NAME "apple" #define DRIVER_DESC "Apple display controller DRM driver" -#define MAX_COPROCESSORS 3 +#define MAX_COPROCESSORS 8 struct apple_drm_private { struct drm_device drm; @@ -250,9 +251,35 @@ static const struct drm_connector_funcs apple_connector_funcs = { .oob_hotplug_event = apple_connector_oob_hotplug, }; +/* + * A Type-C connector is attached to every pipeline that can drive its port, so + * the atomic helper cannot pick one on its own. Follow the fabric's choice. + */ +static struct drm_encoder * +apple_connector_atomic_best_encoder(struct drm_connector *conn, + struct drm_atomic_commit *state) +{ + struct apple_connector *apple_connector = to_apple_connector(conn); + struct drm_encoder *encoder; + + /* + * A Type-C port has one encoder, and its possible_crtcs already names + * the pipeline the fabric routed the port to, so it is the only answer + * there is. + */ + if (apple_connector->port_encoder) + return apple_connector->port_encoder; + + drm_connector_for_each_possible_encoder(conn, encoder) + return encoder; + + return NULL; +} + static const struct drm_connector_helper_funcs apple_connector_helper_funcs = { .get_modes = dcp_get_modes, .mode_valid = dcp_mode_valid, + .atomic_best_encoder = apple_connector_atomic_best_encoder, }; static const struct drm_crtc_helper_funcs apple_crtc_helper_funcs = { @@ -264,18 +291,69 @@ static const struct drm_crtc_helper_funcs apple_crtc_helper_funcs = { .mode_fixup = dcp_crtc_mode_fixup, }; +static int apple_connector_create(struct drm_device *drm, + struct platform_device *dcp, + struct apple_crtc *crtc, + struct apple_encoder *encoder, + int connector_type, bool attach_fwnode) +{ + struct apple_connector *connector; + int ret; + + connector = kzalloc_obj(*connector); + if (!connector) + return -ENOMEM; + + mutex_init(&connector->chunk_lock); + drm_connector_helper_add(&connector->base, + &apple_connector_helper_funcs); + if (attach_fwnode) + connector->base.fwnode = fwnode_handle_get(dcp->dev.fwnode); + + ret = drm_connector_init(drm, &connector->base, &apple_connector_funcs, + connector_type); + if (ret) + goto err_free; + ret = drm_connector_attach_vrr_capable_property(&connector->base); + if (ret) + goto err_cleanup; + + connector->base.polled = DRM_CONNECTOR_POLL_HPD; + connector->connected = false; + connector->dcp = dcp; + INIT_WORK(&connector->hotplug_wq, dcp_hotplug); + + ret = drm_connector_attach_encoder(&connector->base, &encoder->base); + if (ret) + goto err_cleanup; + dcp_link(dcp, crtc, connector); + + return 0; + +err_cleanup: + drm_connector_cleanup(&connector->base); + kfree(connector); + return ret; +err_free: + fwnode_handle_put(connector->base.fwnode); + kfree(connector); + return ret; +} + static int apple_probe_per_dcp(struct device *dev, struct drm_device *drm, struct platform_device *dcp, - int num, bool dcp_ext) + int num, bool dcp_ext, + struct drm_encoder **encoder, + u32 *crtc_mask) { struct apple_crtc *crtc; - struct apple_connector *connector; struct apple_encoder *enc; struct drm_plane *planes[DCP_MAX_PLANES]; unsigned long *iomfb_surfaces = dcp_get_iomfb_surfaces(dcp); unsigned long top_surf = find_last_bit(iomfb_surfaces, DCP_MAX_PLANES); int ret; + int connector_type; u32 surf; int zpos = 0; bool supports_l10r = !dcp_fw_compat_is_12_x(dcp); @@ -323,39 +401,113 @@ static int apple_probe_per_dcp(struct device *dev, drm_crtc_helper_add(&crtc->base, &apple_crtc_helper_funcs); drm_crtc_enable_color_mgmt(&crtc->base, 0, true, 0); + crtc->dcp = dcp; + *crtc_mask = drm_crtc_mask(&crtc->base); + connector_type = dcp_get_connector_type(dcp); + + /* + * Type-C outputs are physical ports, not pipelines: several DCPs may be + * able to drive one port, and which one does can change at runtime. + * Those connectors are created per port once every pipeline is known, + * so a pipeline whose only output is Type-C gets no connector here -- + * and no encoder either, because the port's encoder spans every + * pipeline that can drive it. + */ + if (connector_type == DRM_MODE_CONNECTOR_USB) { + dcp_link(dcp, crtc, NULL); + *encoder = NULL; + return 0; + } + enc = drmm_simple_encoder_alloc(drm, struct apple_encoder, base, DRM_MODE_ENCODER_TMDS); if (IS_ERR(enc)) - return PTR_ERR(enc); - enc->base.possible_crtcs = drm_crtc_mask(&crtc->base); + return PTR_ERR(enc); + enc->base.possible_crtcs = *crtc_mask; + *encoder = &enc->base; - connector = kzalloc(sizeof(*connector), GFP_KERNEL); - mutex_init(&connector->chunk_lock); - drm_connector_helper_add(&connector->base, - &apple_connector_helper_funcs); + return apple_connector_create(drm, dcp, crtc, enc, connector_type, + dcp_ext); +} - // HACK: - if (dcp_ext) - connector->base.fwnode = fwnode_handle_get(dcp->dev.fwnode); +/* + * Create one connector per physical Type-C port and attach it to every + * pipeline that can drive it. Keeping the connector tied to the port rather + * than to a pipeline is what gives userspace a stable name to hang its + * per-monitor configuration on. + */ +static int apple_probe_typec_ports(struct drm_device *drm, + struct platform_device **dcp, + u32 *crtc_mask, int num_dcp) +{ + unsigned int idx, nr_ports = dcp_typec_nr_ports(); + int i, ret; + + for (idx = 0; idx < nr_ports; idx++) { + struct apple_connector *connector; + struct apple_encoder *enc; + u32 mask = 0; + + connector = kzalloc_obj(*connector); + if (!connector) + return -ENOMEM; + + mutex_init(&connector->chunk_lock); + drm_connector_helper_add(&connector->base, + &apple_connector_helper_funcs); + + ret = drm_connector_init(drm, &connector->base, + &apple_connector_funcs, + DRM_MODE_CONNECTOR_USB); + if (ret) { + kfree(connector); + return ret; + } - ret = drm_connector_init(drm, &connector->base, &apple_connector_funcs, - dcp_get_connector_type(dcp)); - if (ret) - return ret; - ret = drm_connector_attach_vrr_capable_property(&connector->base); - if (ret) - return ret; + ret = drm_connector_attach_vrr_capable_property(&connector->base); + if (ret) + return ret; - connector->base.polled = DRM_CONNECTOR_POLL_HPD; - connector->connected = false; - connector->dcp = dcp; + connector->base.polled = DRM_CONNECTOR_POLL_HPD; + connector->connected = false; + connector->dcp = NULL; + INIT_WORK(&connector->hotplug_wq, dcp_hotplug); - INIT_WORK(&connector->hotplug_wq, dcp_hotplug); + for (i = 0; i < num_dcp; i++) { + if (dcp_typec_port_has_candidate(idx, dcp[i])) + mask |= crtc_mask[i]; + } - crtc->dcp = dcp; - dcp_link(dcp, crtc, connector); + if (!mask) { + drm_warn(drm, "Type-C port %u has no display pipeline\n", + idx); + return -ENODEV; + } - return drm_connector_attach_encoder(&connector->base, &enc->base); + /* + * One encoder for the port, spanning every pipeline that can + * drive it. Attaching one encoder per pipeline would describe + * the same hardware, but userspace takes the CRTCs a connector + * can use to be what all of its encoders have in common, and + * single-pipeline encoders have nothing in common -- the port + * ends up with no usable CRTC and the monitor is left dark. + */ + enc = drmm_simple_encoder_alloc(drm, struct apple_encoder, base, + DRM_MODE_ENCODER_TMDS); + if (IS_ERR(enc)) + return PTR_ERR(enc); + enc->base.possible_crtcs = mask; + + ret = drm_connector_attach_encoder(&connector->base, &enc->base); + if (ret) + return ret; + connector->port_encoder = &enc->base; + connector->candidate_crtcs = mask; + + dcp_typec_port_set_connector(idx, connector); + } + + return 0; } static int apple_get_fb_resource(struct device *dev, const char *name, @@ -400,6 +552,8 @@ static int apple_drm_init_dcp(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); struct platform_device *dcp[MAX_COPROCESSORS]; + struct drm_encoder *encoder[MAX_COPROCESSORS]; + u32 crtc_mask[MAX_COPROCESSORS] = {}; struct device_node *np; u64 timeout; int i, ret, num_dcp = 0; @@ -421,11 +575,8 @@ static int apple_drm_init_dcp(struct device *dev) device_link_add(dev, &dcp[num_dcp]->dev, DL_FLAG_AUTOREMOVE_SUPPLIER); ret = apple_probe_per_dcp(dev, &apple->drm, dcp[num_dcp], - num_dcp, dcp_ext); - if (ret) - continue; - - ret = dcp_start(dcp[num_dcp]); + num_dcp, dcp_ext, &encoder[num_dcp], + &crtc_mask[num_dcp]); if (ret) continue; @@ -435,6 +586,21 @@ static int apple_drm_init_dcp(struct device *dev) if (num_dcp < 1) return -ENODEV; + /* + * Build the Type-C port connectors before starting any pipeline: a + * pipeline can be handed a route as soon as the Type-C mux notifies, + * and it needs somewhere to report the display. + */ + ret = apple_probe_typec_ports(&apple->drm, dcp, crtc_mask, num_dcp); + if (ret) + return ret; + + for (i = 0; i < num_dcp; i++) { + ret = dcp_start(dcp[i]); + if (ret) + dev_warn(dev, "DCP[%d] failed to start: %d\n", i, ret); + } + /* * Starting DPTX might take some time. */ diff --git a/drivers/gpu/drm/apple/connector.h b/drivers/gpu/drm/apple/connector.h index 8c9d367eda94f..4cb58502c1ef8 100644 --- a/drivers/gpu/drm/apple/connector.h +++ b/drivers/gpu/drm/apple/connector.h @@ -17,12 +17,31 @@ struct apple_connector; void dcp_hotplug(struct work_struct *work); void dcp_retrain_oob(struct apple_connector *connector); +/* + * How many DCP pipelines may offer a route to one Type-C port. The Type-C mux + * class caps the mode-switch providers per connector (TYPEC_MUX_MAX_DEVS) and + * the ATC PHY already claims one of those slots, so this is generous. + */ struct apple_connector { struct drm_connector base; bool connected; + /* The pipeline currently driving this connector, NULL if unrouted. */ struct platform_device *dcp; + /* + * A Type-C connector is a physical port rather than a fixed pipeline, + * and it has one encoder covering every pipeline that can drive it. + * One encoder per pipeline would describe the same hardware, but + * userspace takes the CRTCs a connector can use to be what all of its + * encoders have in common, and single-pipeline encoders have nothing + * in common. + */ + struct drm_encoder *port_encoder; + + /* the CRTCs of every pipeline that can drive the port */ + u32 candidate_crtcs; + const struct drm_edid *drm_edid; /* Workqueue for sending hotplug events to the associated device */ diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 4c78f19f47b4d..7fc0e1f04c9e0 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -27,9 +28,12 @@ struct apple_dcp; struct apple_dcp_afkep; +struct apple_dcp_typec_port; struct apple_dcp_typec_route { struct apple_dcp *dcp; + struct apple_dcp_typec_port *port; + struct list_head port_link; struct phy *phy; struct mux_control *xbar; struct typec_mux_dev *typec_mux; @@ -38,6 +42,8 @@ struct apple_dcp_typec_route { bool selected; }; +bool dcp_is_typec_output(struct apple_dcp *dcp); + struct dcpav_service_epic; enum { @@ -191,11 +197,16 @@ struct apple_dcp { /* swap id of the last completed swap */ u32 last_swap_id; ktime_t swap_start; + u64 swap_submit_timestamp; /* Current display mode */ bool during_modeset; + /* hotplug reported while a modeset was in flight, applied afterwards */ + bool pending_hotplug; + bool pending_hotplug_connected; bool valid_mode; bool use_timestamps; + bool vrr_enabled; struct dcp_set_digital_out_mode_req mode; /* completion for active turning true */ @@ -219,6 +230,7 @@ struct apple_dcp { /* Attributes of the connector */ int connector_type; + int fixed_connector_type; /* Attributes of the connected display */ int width_mm, height_mm; @@ -262,15 +274,16 @@ struct apple_dcp { /* these fields are output port specific */ struct phy *phy; + struct phy *fixed_phy; struct mux_control *xbar; struct typec_mux *typec_mux; struct apple_dcp_typec_route typec_routes[DCP_MAX_TYPEC_ROUTES]; struct apple_dcp_typec_route *active_typec_route; - struct mutex typec_route_lock; /* serializes Type-C route ownership */ u32 nr_typec_routes; bool phy_managed_by_typec; bool typec_cable_connected; struct delayed_work typec_reconnect_wq; + struct delayed_work typec_fabric_retrain_wq; u32 typec_reconnect_tries; struct gpio_desc *hdmi_hpd; @@ -281,6 +294,11 @@ struct apple_dcp { u32 dptx_phy; u32 dptx_die; + u32 fixed_dptx_phy; + u32 fixed_mux_index; + bool fixed_route_selected; + struct apple_connector *fixed_connector; + struct apple_connector *typec_connector; int hdmi_hpd_irq; }; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 02918703e50ff..dd3060d6613c1 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -58,69 +59,481 @@ static bool unstable_edid = true; module_param(unstable_edid, bool, 0644); MODULE_PARM_DESC(unstable_edid, "Enable unstable EDID retrival support"); +struct apple_dcp_typec_port { + struct list_head link; + struct list_head routes; + struct device_node *connector_np; + struct apple_dcp_typec_route *owner; + /* DRM connector for this physical port, driven by whichever DCP owns it */ + struct apple_connector *connector; + /* last mux state acted on, to collapse the per-candidate notifications */ + struct typec_altmode *applied_alt; + unsigned long applied_mode; + u32 applied_status; + u32 applied_conf; + bool applied_valid; + bool hpd; +}; + +static DEFINE_MUTEX(dcp_typec_fabric_lock); +static LIST_HEAD(dcp_typec_ports); + +static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port); +static void disconnected_hpd_event(struct apple_connector *connector); + +bool dcp_is_typec_output(struct apple_dcp *dcp) +{ + return dcp->active_typec_route || + dcp->fixed_connector_type == DRM_MODE_CONNECTOR_USB; +} + static bool dcp_typec_route_is_dp(const struct typec_mux_state *state) { return state->alt && state->alt->svid == USB_TYPEC_DP_SID && - (state->mode == TYPEC_DP_STATE_C || - state->mode == TYPEC_DP_STATE_D || - state->mode == TYPEC_DP_STATE_E); + state->mode >= TYPEC_DP_STATE_A && + state->mode <= TYPEC_DP_STATE_F; } -static int dcp_typec_route_set(struct typec_mux_dev *mux, - struct typec_mux_state *state) +/* Keep a live fixed output on its own pipeline. */ +static bool dcp_typec_route_fixed_output_busy(struct apple_dcp_typec_route *route) { - struct apple_dcp_typec_route *route = typec_mux_get_drvdata(mux); struct apple_dcp *dcp = route->dcp; - int ret = 0; - guard(mutex)(&dcp->typec_route_lock); + if (dcp->fixed_connector_type == DRM_MODE_CONNECTOR_USB) + return false; + if (dcp->fixed_connector && dcp->fixed_connector->connected) + return true; + if (dcp->hdmi_hpd && gpiod_get_value_cansleep(dcp->hdmi_hpd)) + return true; - if (!dcp_typec_route_is_dp(state)) { - if (!route->selected) - return 0; + return false; +} - ret = mux_control_deselect(route->xbar); - route->selected = false; - if (dcp->active_typec_route == route) - dcp->active_typec_route = NULL; - return ret; - } +static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) +{ + return !route->dcp->active_typec_route && + !dcp_typec_route_fixed_output_busy(route); +} - if (route->selected) - return 0; +static int dcp_typec_route_activate(struct apple_dcp_typec_route *route); +static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); + +/* + * Pipelines are ranked by CRTC index so the fabric's choice is a pure function + * of the topology rather than of plug order. A pipeline whose fixed output is + * live is not a candidate at all, so a hybrid is only ever ranked here when it + * is genuinely free. + */ +static unsigned int dcp_typec_route_score(struct apple_dcp_typec_route *route) +{ + struct apple_dcp *dcp = route->dcp; - if (dcp->active_typec_route) { - dev_warn(dcp->dev, "DP route is already in use by another Type-C port\n"); - return -EBUSY; + if (!dcp->crtc) + return UINT_MAX - 1; + + return drm_crtc_index(&dcp->crtc->base); +} + +static int dcp_typec_route_activate(struct apple_dcp_typec_route *route) +{ + struct apple_dcp *dcp = route->dcp; + int ret; + + if (dcp->fixed_route_selected) { + ret = mux_control_deselect(dcp->xbar); + if (ret) + return ret; + dcp->fixed_route_selected = false; } ret = mux_control_select(route->xbar, route->mux_index); - if (ret) + if (ret) { + if (dcp->xbar) { + int restore_ret; + + restore_ret = mux_control_select(dcp->xbar, + dcp->fixed_mux_index); + if (!restore_ret) + dcp->fixed_route_selected = true; + else + dev_err(dcp->dev, + "failed to restore fixed display route: %d\n", + restore_ret); + } return ret; + } dcp->phy = route->phy; dcp->dptx_phy = route->dptx_phy; + dcp->connector_type = DRM_MODE_CONNECTOR_USB; + if (route->port->connector) { + route->port->connector->dcp = to_platform_device(dcp->dev); + dcp->typec_connector = route->port->connector; + dcp->connector = route->port->connector; + + /* + * Narrow the port to the pipeline now driving it. The encoder + * spans every pipeline that could, which is what lets userspace + * see the port as usable at all -- but only one of them is + * routed to the display, and userspace has no way to tell which. + * Offering it the choice makes it pair the port with a pipeline + * holding a different monitor's mode list, and the modeset is + * rejected with no way for it to recover. The hotplug that + * follows makes it re-read this. + */ + if (route->port->connector->port_encoder && dcp->crtc) + route->port->connector->port_encoder->possible_crtcs = + drm_crtc_mask(&dcp->crtc->base); + } dcp->active_typec_route = route; route->selected = true; - dev_info(dcp->dev, "Routed DCP output to Type-C DPTX PHY %u\n", - route->dptx_phy); + dev_info(dcp->dev, "allocated Type-C DPTX PHY %u\n", route->dptx_phy); return 0; } -static void dcp_typec_route_unregister(void *data) +static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) { - struct apple_dcp_typec_route *route = data; struct apple_dcp *dcp = route->dcp; + int ret; + + ret = mux_control_deselect(route->xbar); + if (ret) + return ret; + + route->selected = false; + if (dcp->active_typec_route == route) + dcp->active_typec_route = NULL; + + if (route->port->connector && + route->port->connector->dcp == to_platform_device(dcp->dev)) { + /* + * Until the port is activated again it has no pipeline behind + * it, and nothing can read modes or EDID from it. Report it + * disconnected for that window: leaving a connected connector + * whose ->dcp is NULL lets anything probing it in between -- + * a compositor starting up while the fabric is still settling + * -- see an output it cannot get a mode for, and give up on + * it. The new pipeline marks it connected again once the + * display has come back up on it. + */ + WRITE_ONCE(route->port->connector->connected, false); + route->port->connector->dcp = NULL; + + /* Unrouted: the port could go to any of its pipelines again. */ + if (route->port->connector->port_encoder) + route->port->connector->port_encoder->possible_crtcs = + route->port->connector->candidate_crtcs; + } + dcp->typec_connector = NULL; + dcp->connector = dcp->fixed_connector; + + if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_USB) { + dcp->connector_type = dcp->fixed_connector_type; + + /* + * Only hand the pipeline back to its fixed output if that output + * is live. Re-targeting the DPTX endpoint at the fixed PHY while + * nothing is attached there leaves DCP unable to train a link on a + * later Type-C target: it answers DEVICE_NOT_STARTED and every + * following DPTX call times out. Park on a Type-C PHY instead, + * for the same reason the USB-C-only case does below. + */ + if (dcp->hdmi_hpd && gpiod_get_value_cansleep(dcp->hdmi_hpd)) { + dcp->phy = dcp->fixed_phy; + dcp->dptx_phy = dcp->fixed_dptx_phy; + + if (dcp->xbar) { + ret = mux_control_select(dcp->xbar, + dcp->fixed_mux_index); + if (ret) + return ret; + dcp->fixed_route_selected = true; + } + } else { + dcp->phy = dcp->typec_routes[0].phy; + dcp->dptx_phy = dcp->typec_routes[0].dptx_phy; + } + } else { + /* Keep DPTX endpoint discovery working before a cable is attached. */ + dcp->phy = dcp->typec_routes[0].phy; + dcp->dptx_phy = dcp->typec_routes[0].dptx_phy; + dcp->connector_type = DRM_MODE_CONNECTOR_USB; + } + + return 0; +} + +static void dcp_typec_retrain_work(struct work_struct *work) +{ + struct apple_dcp *dcp = + container_of(to_delayed_work(work), struct apple_dcp, + typec_fabric_retrain_wq); + + if (READ_ONCE(dcp->active_typec_route) && dcp->typec_connector) + dcp_retrain_oob(dcp->typec_connector); +} + +static void dcp_typec_retrain_active_routes(void) +{ + struct apple_dcp_typec_port *port; + + list_for_each_entry(port, &dcp_typec_ports, link) { + if (!port->owner) + continue; + mod_delayed_work(system_freezable_wq, + &port->owner->dcp->typec_fabric_retrain_wq, + msecs_to_jiffies(200)); + } +} + +static int dcp_typec_route_set(struct typec_mux_dev *mux, + struct typec_mux_state *state) +{ + struct apple_dcp_typec_route *route = typec_mux_get_drvdata(mux); + struct apple_dcp_typec_port *port = route->port; + struct apple_dcp_typec_route *candidate, *best = NULL; + bool is_dp = dcp_typec_route_is_dp(state); + struct typec_displayport_data *dp_data = is_dp ? state->data : NULL; + u32 dp_status = dp_data ? dp_data->status : 0; + u32 dp_conf = dp_data ? dp_data->conf : 0; + unsigned int best_score = UINT_MAX; + bool hpd; + int ret = 0; + + guard(mutex)(&dcp_typec_fabric_lock); + + /* + * Every candidate route for this port is notified with the same state, + * so only the first to arrive does the work; the others return early. + * + * Deliberately not a nominated coordinator: fwnode_typec_mux_get() + * caps the providers one connector may have and drops the remainder + * without a word, so a nominated route might never be called at all -- + * and the port would then never be routed. + */ + if (port->applied_valid && port->applied_alt == state->alt && + port->applied_mode == state->mode && + port->applied_status == dp_status && port->applied_conf == dp_conf) + return 0; + + port->applied_alt = state->alt; + port->applied_mode = state->mode; + port->applied_status = dp_status; + port->applied_conf = dp_conf; + port->applied_valid = true; + + if (!is_dp) { + if (port->owner) { + struct apple_dcp *dcp = port->owner->dcp; + + if (port->hpd || dcp->typec_cable_connected || + (dcp->typec_connector && + dcp->typec_connector->connected)) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + port->hpd = false; + ret = dcp_typec_route_deactivate(port->owner); + if (ret) + return ret; + port->owner = NULL; + if (dcp->hdmi_hpd && dcp->active && + gpiod_get_value_cansleep(dcp->hdmi_hpd)) + dcp_dptx_connect(dcp, 0); + } + + if (state->mode == TYPEC_MODE_USB4) + dcp_typec_retrain_active_routes(); + return 0; + } + + if (!port->owner) { + list_for_each_entry(candidate, &port->routes, port_link) { + unsigned int score; + + if (!dcp_typec_route_available(candidate)) + continue; + score = dcp_typec_route_score(candidate); + if (score < best_score) { + best = candidate; + best_score = score; + } + } + + if (!best) + return -EBUSY; + + ret = dcp_typec_route_activate(best); + if (ret) + return ret; + port->owner = best; + } + + + hpd = dp_data && (dp_data->status & DP_STATUS_HPD_STATE); + if (!hpd && port->hpd) { + dcp_dptx_disconnect_oob(to_platform_device(port->owner->dcp->dev), 0); + } else if (hpd && !port->hpd) { + struct apple_dcp *dcp = port->owner->dcp; + + WRITE_ONCE(dcp->typec_cable_connected, true); + if (dcp->typec_connector) + dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); + } else if (hpd && dp_data && (dp_data->status & DP_STATUS_IRQ_HPD)) { + struct apple_dcp *dcp = port->owner->dcp; + + if (dcp->typec_connector) + dcp_retrain_oob(dcp->typec_connector); + } + port->hpd = hpd; + + return 0; +} + +static struct apple_dcp_typec_port * +dcp_typec_port_get(struct device_node *connector_np) +{ + struct apple_dcp_typec_port *port, *pos; + + lockdep_assert_held(&dcp_typec_fabric_lock); + + list_for_each_entry(port, &dcp_typec_ports, link) { + if (port->connector_np == connector_np) { + of_node_put(connector_np); + return port; + } + } + + port = kzalloc_obj(*port); + if (!port) { + of_node_put(connector_np); + return NULL; + } + + INIT_LIST_HEAD(&port->routes); + port->connector_np = connector_np; + + /* + * Insert in device-tree order rather than DCP probe order. The list + * index becomes the DRM connector index, and userspace keys its + * per-monitor configuration (scale, rotation, layout) on the connector + * name -- so it has to mean the same physical port on every boot. + */ + list_for_each_entry(pos, &dcp_typec_ports, link) + if (strcmp(of_node_full_name(connector_np), + of_node_full_name(pos->connector_np)) < 0) + break; + list_add_tail(&port->link, &pos->link); + return port; +} + +static struct apple_dcp_typec_port *dcp_typec_port_by_index(unsigned int idx) +{ + struct apple_dcp_typec_port *port; + unsigned int i = 0; + + lockdep_assert_held(&dcp_typec_fabric_lock); + + list_for_each_entry(port, &dcp_typec_ports, link) + if (i++ == idx) + return port; + + return NULL; +} + +unsigned int dcp_typec_nr_ports(void) +{ + struct apple_dcp_typec_port *port; + unsigned int n = 0; + + guard(mutex)(&dcp_typec_fabric_lock); + + list_for_each_entry(port, &dcp_typec_ports, link) + n++; + + return n; +} + +struct device_node *dcp_typec_port_of_node(unsigned int idx) +{ + struct apple_dcp_typec_port *port; + + guard(mutex)(&dcp_typec_fabric_lock); + + port = dcp_typec_port_by_index(idx); + + return port ? port->connector_np : NULL; +} + +bool dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev) +{ + struct apple_dcp_typec_port *port; + struct apple_dcp_typec_route *route; + + guard(mutex)(&dcp_typec_fabric_lock); + + port = dcp_typec_port_by_index(idx); + if (!port) + return false; + + list_for_each_entry(route, &port->routes, port_link) + if (route->dcp->dev == &pdev->dev) + return true; + + return false; +} + +void dcp_typec_port_set_connector(unsigned int idx, + struct apple_connector *connector) +{ + struct apple_dcp_typec_port *port; + + guard(mutex)(&dcp_typec_fabric_lock); - guard(mutex)(&dcp->typec_route_lock); - if (route->selected) { - mux_control_deselect(route->xbar); - route->selected = false; - if (dcp->active_typec_route == route) - dcp->active_typec_route = NULL; + port = dcp_typec_port_by_index(idx); + if (!port) + return; + + port->connector = connector; + + /* + * The port may already have been routed, either before DRM bound or + * while these connectors were being created. Adopt that owner now, + * otherwise its display would be reported on the pipeline's fixed + * connector instead of the port it is actually plugged into. + */ + if (port->owner) { + struct apple_dcp *dcp = port->owner->dcp; + + connector->dcp = to_platform_device(dcp->dev); + dcp->typec_connector = connector; + dcp->connector = connector; + dcp->connector_type = DRM_MODE_CONNECTOR_USB; } +} + +static void dcp_typec_route_unregister(void *data) +{ + struct apple_dcp_typec_route *route = data; + struct apple_dcp_typec_port *port = route->port; + typec_mux_unregister(route->typec_mux); + + guard(mutex)(&dcp_typec_fabric_lock); + if (port->owner == route) { + struct apple_dcp *dcp = route->dcp; + + if (port->hpd || dcp->typec_cable_connected) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + port->hpd = false; + dcp_typec_route_deactivate(route); + port->owner = NULL; + } + list_del(&route->port_link); + if (list_empty(&port->routes)) { + list_del(&port->link); + of_node_put(port->connector_np); + kfree(port); + } } static int dcp_register_typec_routes(struct apple_dcp *dcp) @@ -136,8 +549,10 @@ static int dcp_register_typec_routes(struct apple_dcp *dcp) for_each_available_child_of_node_scoped(routes, route_np) { struct apple_dcp_typec_route *route; + struct device_node *endpoint __free(device_node) = NULL; + struct device_node *connector_np; struct typec_mux_desc desc = {}; - const char *name; + const char *name, *mux_name; if (dcp->nr_typec_routes == DCP_MAX_TYPEC_ROUTES) return dev_err_probe(dev, -E2BIG, "Too many Type-C display routes\n"); @@ -153,8 +568,19 @@ static int dcp_register_typec_routes(struct apple_dcp *dcp) if (!name) return -ENOMEM; + /* + * The DT lookups above are per-DCP, but the typec_mux class is + * global. Several DCPs can offer a route to the same Type-C port, + * so the registered mux needs a name unique across all of them. + */ + mux_name = devm_kasprintf(dev, GFP_KERNEL, "%s-typec%u", + dev_name(dev), route_index); + if (!mux_name) + return -ENOMEM; + route = &dcp->typec_routes[dcp->nr_typec_routes]; route->dcp = dcp; + INIT_LIST_HEAD(&route->port_link); route->phy = devm_phy_get(dev, name); if (IS_ERR(route->phy)) return dev_err_probe(dev, PTR_ERR(route->phy), @@ -177,14 +603,42 @@ static int dcp_register_typec_routes(struct apple_dcp *dcp) return dev_err_probe(dev, ret, "%pOF: missing DPTX PHY index\n", route_np); + endpoint = of_graph_get_next_endpoint(route_np, NULL); + if (!endpoint) + return dev_err_probe(dev, -EINVAL, + "%pOF: missing Type-C graph endpoint\n", + route_np); + connector_np = of_graph_get_remote_port_parent(endpoint); + if (!connector_np) + return dev_err_probe(dev, -EINVAL, + "%pOF: missing Type-C connector\n", + route_np); + + mutex_lock(&dcp_typec_fabric_lock); + route->port = dcp_typec_port_get(connector_np); + if (route->port) + list_add_tail(&route->port_link, &route->port->routes); + mutex_unlock(&dcp_typec_fabric_lock); + if (!route->port) + return -ENOMEM; + desc.fwnode = of_fwnode_handle(route_np); desc.set = dcp_typec_route_set; - desc.name = name; + desc.name = mux_name; desc.drvdata = route; route->typec_mux = typec_mux_register(dev, &desc); - if (IS_ERR(route->typec_mux)) + if (IS_ERR(route->typec_mux)) { + mutex_lock(&dcp_typec_fabric_lock); + list_del(&route->port_link); + if (list_empty(&route->port->routes)) { + list_del(&route->port->link); + of_node_put(route->port->connector_np); + kfree(route->port); + } + mutex_unlock(&dcp_typec_fabric_lock); return dev_err_probe(dev, PTR_ERR(route->typec_mux), "%pOF: failed to register Type-C route\n", route_np); + } ret = devm_add_action_or_reset(dev, dcp_typec_route_unregister, route); if (ret) @@ -484,9 +938,16 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commi crtc_state = drm_atomic_get_new_crtc_state(commit, crtc); needs_modeset = drm_atomic_crtc_needs_modeset(crtc_state) || !dcp->valid_mode; - if (!needs_modeset && !dcp->connector->connected) { - dev_err(dcp->dev, "crtc_atomic_check: disconnected but no modeset\n"); - return -EINVAL; + if (!needs_modeset && (!dcp->connector || !dcp->connector->connected)) { + /* + * Resume restores the mode before the firmware reports the + * display back, so a plane-only commit lands here while the + * connector is still marked disconnected. Rejecting it makes + * the compositor fail every flip and give up on the output; + * dcp_flush() defers the commit until the link returns. + */ + dev_dbg(dcp->dev, + "crtc_atomic_check: deferring commit, link still down\n"); } return 0; @@ -496,7 +957,14 @@ int dcp_get_connector_type(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); - return (dcp->connector_type); + return dcp->fixed_connector_type; +} + +bool dcp_has_typec_routes(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return dcp->nr_typec_routes; } #define DPTX_CONNECT_TIMEOUT msecs_to_jiffies(2000) @@ -511,7 +979,12 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) dev_warn(dcp->dev, "dcp_dptx_connect: missing phy\n"); return -ENODEV; } - dev_info(dcp->dev, "%s(port=%d)\n", __func__, port); + dev_info(dcp->dev, + "%s(port=%d) target=%u:%u typec=%d route=%s conn_type=%d connected=%d\n", + __func__, port, dcp->dptx_die, dcp->dptx_phy, + dcp_is_typec_output(dcp), + dcp->active_typec_route ? "borrowed" : "fixed", + dcp->connector_type, dcp->dptxport[port].connected); mutex_lock(&dcp->hpd_mutex); if (!dcp->dptxport[port].enabled) { @@ -536,7 +1009,7 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) ret = dptxport_connect(dcp->dptxport[port].service, 0, dcp->dptx_phy, dcp->dptx_die, - dcp->connector_type == DRM_MODE_CONNECTOR_USB); + dcp_is_typec_output(dcp)); if (ret) { dev_err(dcp->dev, "dcp_dptx_connect: failed to connect DPTX target %u:%u: %d\n", @@ -552,7 +1025,7 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) goto out_release; } dcp->dptxport[port].connected = true; - if (dcp->connector_type == DRM_MODE_CONNECTOR_USB) { + if (dcp_is_typec_output(dcp)) { ret = dptxport_set_hpd(dcp->dptxport[port].service, true); if (ret) { dev_err(dcp->dev, @@ -653,14 +1126,14 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; - if (dcp->connector_type == DRM_MODE_CONNECTOR_USB) { + if (dcp_is_typec_output(dcp)) { WRITE_ONCE(dcp->typec_cable_connected, true); dcp->typec_reconnect_tries = 0; cancel_delayed_work(&dcp->typec_reconnect_wq); } ret = dcp_dptx_connect(dcp, port); - if (ret && dcp->connector_type == DRM_MODE_CONNECTOR_USB) + if (ret && dcp_is_typec_output(dcp)) mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, DPTX_RECONNECT_DELAY); @@ -671,7 +1144,7 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) { struct apple_dcp *dcp = platform_get_drvdata(pdev); - if (dcp->connector_type == DRM_MODE_CONNECTOR_USB) { + if (dcp_is_typec_output(dcp)) { WRITE_ONCE(dcp->typec_cable_connected, false); cancel_delayed_work(&dcp->typec_reconnect_wq); } @@ -690,7 +1163,13 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) static irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) { struct apple_dcp *dcp = data; - bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); + bool connected; + + guard(mutex)(&dcp_typec_fabric_lock); + + if (READ_ONCE(dcp->active_typec_route)) + return IRQ_HANDLED; + connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); /* do nothing on disconnect and trust that dcp detects it itself. * Parallel disconnect HPDs result drm disabling the CRTC even when it @@ -718,7 +1197,20 @@ void dcp_link(struct platform_device *pdev, struct apple_crtc *crtc, struct apple_dcp *dcp = platform_get_drvdata(pdev); dcp->crtc = crtc; - dcp->connector = connector; + + /* + * Type-C connectors belong to physical ports and are bound by the + * display fabric when a pipeline takes a route, so a pipeline with no + * fixed output simply has no connector until then. + */ + if (!connector) + return; + + dcp->fixed_connector = connector; + if (!dcp->active_typec_route) { + dcp->connector = connector; + dcp->connector_type = dcp->fixed_connector_type; + } } @@ -828,8 +1320,11 @@ static void _dcp_poweroff(struct apple_dcp *dcp) static int dcp_enable_dp2hdmi_hpd(struct apple_dcp *dcp) { - // check HPD state before enabling the edge triggered IRQ - if (dcp->hdmi_hpd) { + if (dcp_is_typec_output(dcp)) { + if (READ_ONCE(dcp->typec_cable_connected)) + dcp_dptx_connect(dcp, 0); + } else if (dcp->hdmi_hpd) { + /* Check HPD before enabling the edge-triggered IRQ. */ bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); dev_info(dcp->dev, "%s: DP2HDMI HPD connected:%d\n", __func__, connected); @@ -893,26 +1388,26 @@ void dcp_poweron(struct platform_device *pdev) struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; - if (dcp->hdmi_hpd) { + if (dcp_is_typec_output(dcp)) { + /* + * A Type-C CRTC disable releases its DPTX session. Re-establish it + * synchronously before IOMFB is powered back on. + */ + if (READ_ONCE(dcp->typec_cable_connected)) { + cancel_delayed_work(&dcp->typec_reconnect_wq); + dcp->typec_reconnect_tries = 0; + ret = dcp_dptx_connect(dcp, 0); + if (ret) + mod_delayed_work(system_freezable_wq, + &dcp->typec_reconnect_wq, + DPTX_RECONNECT_DELAY); + } + } else if (dcp->hdmi_hpd) { bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); dev_info(dcp->dev, "%s: DP2HDMI HPD connected:%d\n", __func__, connected); if (connected) dcp_dptx_connect(dcp, 0); - } else if (dcp->connector_type == DRM_MODE_CONNECTOR_USB && - READ_ONCE(dcp->typec_cable_connected)) { - /* - * A Type-C CRTC disable releases its DPTX session. Re-establish - * that session synchronously before powering IOMFB back on, so the - * following modeset cannot race the delayed reconnect worker. - */ - cancel_delayed_work(&dcp->typec_reconnect_wq); - dcp->typec_reconnect_tries = 0; - ret = dcp_dptx_connect(dcp, 0); - if (ret) - mod_delayed_work(system_freezable_wq, - &dcp->typec_reconnect_wq, - DPTX_RECONNECT_DELAY); } switch (dcp->fw_compat) { @@ -944,39 +1439,31 @@ void dcp_poweroff(struct platform_device *pdev) _dcp_poweroff(dcp); - if (dcp->hdmi_hpd) { + if (dcp_is_typec_output(dcp)) { + /* + * DCP owns a synthetic HPD for Type-C. Release it with the CRTC, + * then recreate the session while the cable remains present. + */ + if (dcp->dptxport[0].enabled && dcp->dptxport[0].connected) { + ret = dptxport_set_hpd(dcp->dptxport[0].service, false); + if (ret) + dev_warn(dcp->dev, + "failed to deassert Type-C DPTX HPD: %d\n", ret); + dcp_dptx_disconnect(dcp, 0); + + if (READ_ONCE(dcp->typec_cable_connected)) { + dcp->typec_reconnect_tries = 0; + mod_delayed_work(system_freezable_wq, + &dcp->typec_reconnect_wq, + DPTX_RECONNECT_DELAY); + } + } + } else if (dcp->hdmi_hpd) { bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); if (!connected) { disconnected_hpd_event(dcp->connector); dcp_dptx_disconnect(dcp, 0); } - } else if (dcp->connector_type == DRM_MODE_CONNECTOR_USB && - dcp->dptxport[0].enabled && dcp->dptxport[0].connected) { - /* - * Unlike a physical DP/HDMI HPD, the Type-C HPD exposed by the - * DPTX service belongs to this power-on session. Release it when - * the CRTC powers down so the next enable performs the same orderly - * connect/request/HPD sequence as a fresh cable connection. - */ - ret = dptxport_set_hpd(dcp->dptxport[0].service, false); - if (ret) - dev_warn(dcp->dev, - "failed to deassert Type-C DPTX HPD: %d\n", ret); - dcp_dptx_disconnect(dcp, 0); - - /* - * DCP tears down its synthetic HPD when the CRTC powers off, but - * the Type-C controller does not necessarily emit another cable - * connect edge. Recreate the DPTX session while the physical cable - * remains present. A freezable workqueue keeps this out of the - * suspend path; if suspend wins the race, reconnect after thaw. - */ - if (READ_ONCE(dcp->typec_cable_connected)) { - dcp->typec_reconnect_tries = 0; - mod_delayed_work(system_freezable_wq, - &dcp->typec_reconnect_wq, - DPTX_RECONNECT_DELAY); - } } } @@ -1262,6 +1749,18 @@ static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) return DCP_FIRMWARE_UNKNOWN; } +static int dcp_connector_type_from_dt(struct device_node *np) +{ + if (of_property_match_string(np, "apple,connector-type", "HDMI-A") >= 0) + return DRM_MODE_CONNECTOR_HDMIA; + if (of_property_match_string(np, "apple,connector-type", "DP") >= 0) + return DRM_MODE_CONNECTOR_DisplayPort; + if (of_property_match_string(np, "apple,connector-type", "USB-C") >= 0) + return DRM_MODE_CONNECTOR_USB; + + return DRM_MODE_CONNECTOR_Unknown; +} + static int dcp_comp_bind(struct device *dev, struct device *main, void *data) { struct device_node *panel_np; @@ -1277,12 +1776,6 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (IS_ERR(dcp->coproc_reg)) return PTR_ERR(dcp->coproc_reg); - of_property_read_u32(dev->of_node, "apple,dcp-index", - &dcp->index); - of_property_read_u32(dev->of_node, "apple,dptx-phy", - &dcp->dptx_phy); - of_property_read_u32(dev->of_node, "apple,dptx-die", - &dcp->dptx_die); if (dcp->index || dcp->dptx_phy || dcp->dptx_die) dev_info(dev, "DCP index:%u dptx target phy: %u dptx die: %u\n", dcp->index, dcp->dptx_phy, dcp->dptx_die); @@ -1321,18 +1814,12 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) &dcp->panel.height_mm); of_node_put(panel_np); + dcp->fixed_connector_type = DRM_MODE_CONNECTOR_eDP; dcp->connector_type = DRM_MODE_CONNECTOR_eDP; INIT_WORK(&dcp->bl_register_wq, dcp_work_register_backlight); mutex_init(&dcp->bl_register_mutex); INIT_WORK(&dcp->bl_update_wq, dcp_work_update_backlight); - } else if (of_property_match_string(dev->of_node, "apple,connector-type", "HDMI-A") >= 0) - dcp->connector_type = DRM_MODE_CONNECTOR_HDMIA; - else if (of_property_match_string(dev->of_node, "apple,connector-type", "DP") >= 0) - dcp->connector_type = DRM_MODE_CONNECTOR_DisplayPort; - else if (of_property_match_string(dev->of_node, "apple,connector-type", "USB-C") >= 0) - dcp->connector_type = DRM_MODE_CONNECTOR_USB; - else - dcp->connector_type = DRM_MODE_CONNECTOR_Unknown; + } ret = dcp_create_piodma_iommu_dev(dcp); if (ret || !dcp->iommu_dom) @@ -1432,6 +1919,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) cancel_work_sync(&dcp->bl_update_wq); } cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); cancel_work_sync(&dcp->vblank_wq); devm_clk_put(dev, dcp->clk); @@ -1471,9 +1959,16 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp->fw_compat = fw_compat; dcp->dev = dev; dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); - mutex_init(&dcp->typec_route_lock); + dcp->fixed_connector_type = dcp_connector_type_from_dt(dev->of_node); + dcp->connector_type = dcp->fixed_connector_type; + of_property_read_u32(dev->of_node, "apple,dcp-index", &dcp->index); + of_property_read_u32(dev->of_node, "apple,dptx-phy", &dcp->dptx_phy); + of_property_read_u32(dev->of_node, "apple,dptx-die", &dcp->dptx_die); + dcp->fixed_dptx_phy = dcp->dptx_phy; INIT_DELAYED_WORK(&dcp->typec_reconnect_wq, dcp_typec_reconnect_work); + INIT_DELAYED_WORK(&dcp->typec_fabric_retrain_wq, + dcp_typec_retrain_work); platform_set_drvdata(pdev, dcp); @@ -1482,10 +1977,7 @@ static int dcp_platform_probe(struct platform_device *pdev) dev_err(dev, "Failed to get dp-phy: %ld\n", PTR_ERR(dcp->phy)); return PTR_ERR(dcp->phy); } - - ret = dcp_register_typec_routes(dcp); - if (ret) - return ret; + dcp->fixed_phy = dcp->phy; bitmap_zero(dcp->iomfb_surfaces, DCP_MAX_PLANES); if (!of_property_present(dev->of_node, "apple,iomfb-surfaces")) @@ -1557,16 +2049,15 @@ static int dcp_platform_probe(struct platform_device *pdev) return PTR_ERR(dcp->dp2hdmi_pwren); /* - * Firmware or an inherited board DTS may leave the legacy mux-index - * property on external DCP nodes. Multi-route USB-C DCPs select their - * crossbar through the per-port routes registered above and do not - * have a single "dp-xbar" mux. Treat mux-index only as the legacy - * single-route binding; otherwise one stale property prevents every - * Type-C controller from probing while they wait for the DCP routes. + * A DCP may have both a fixed HDMI/DP route and allocatable Type-C + * routes. Keep the fixed route selected until the allocator borrows + * this otherwise-idle pipeline for a Type-C display. */ - ret = dcp->nr_typec_routes ? -ENODATA : - of_property_read_u32(dev->of_node, "mux-index", &mux_index); + ret = dcp->fixed_phy ? + of_property_read_u32(dev->of_node, "mux-index", &mux_index) : + -ENODATA; if (!ret) { + dcp->fixed_mux_index = mux_index; dcp->xbar = devm_mux_control_get(dev, "dp-xbar"); if (IS_ERR(dcp->xbar)) { dev_err(dev, "Failed to get dp-xbar: %ld\n", PTR_ERR(dcp->xbar)); @@ -1575,6 +2066,8 @@ static int dcp_platform_probe(struct platform_device *pdev) ret = mux_control_select(dcp->xbar, mux_index); if (ret) dev_warn(dev, "mux_control_select failed: %d\n", ret); + else + dcp->fixed_route_selected = true; /* * Switch atcphy to DP-only. should move to a Macbook Pro @@ -1601,6 +2094,10 @@ static int dcp_platform_probe(struct platform_device *pdev) } } + ret = dcp_register_typec_routes(dcp); + if (ret) + return ret; + return component_add(&pdev->dev, &dcp_comp_ops); } @@ -1623,8 +2120,10 @@ static int dcp_platform_suspend(struct device *dev) if (dcp->hdmi_hpd_irq) { disable_irq(dcp->hdmi_hpd_irq); - disconnected_hpd_event(dcp->connector); - dcp_dptx_disconnect(dcp, 0); + if (!dcp->active_typec_route) { + disconnected_hpd_event(dcp->connector); + dcp_dptx_disconnect(dcp, 0); + } } /* * Set the device as a wakeup device, which forces its power diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 086ac70b0cc3a..2466aa9d13571 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -34,6 +34,17 @@ void dcp_poweron(struct platform_device *pdev); int dcp_set_crc(struct drm_crtc *crtc, bool enabled); int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commit); int dcp_get_connector_type(struct platform_device *pdev); +bool dcp_has_typec_routes(struct platform_device *pdev); + +/* + * The Type-C display fabric. Ports are enumerated in device-tree order, not + * DCP probe order, so a given physical port keeps the same DRM connector index + * across boots -- userspace keys its per-monitor configuration on that name. + */ +unsigned int dcp_typec_nr_ports(void); +struct device_node *dcp_typec_port_of_node(unsigned int idx); +bool dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev); +void dcp_typec_port_set_connector(unsigned int idx, struct apple_connector *connector); bool dcp_fw_compat_is_12_x(struct platform_device *pdev); unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev); void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 3d286fc2c846c..525ba63744370 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -457,7 +457,7 @@ static int dptxport_call_get_supports_hpd(struct apple_epic_service *service, return -EINVAL; reply->retcode = cpu_to_le32(0); - reply->supported = cpu_to_le32(dcp->connector_type == DRM_MODE_CONNECTOR_USB); + reply->supported = cpu_to_le32(dcp_is_typec_output(dcp)); return 0; } @@ -558,6 +558,15 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, case DPTX_APCALL_GET_MAX_DRIVE_SETTINGS: return dptxport_call_get_max_drive_settings(service, reply, reply_size); + case DPTX_APCALL_DEVICE_NOT_RESPONDING: + case DPTX_APCALL_DEVICE_BUSY_TIMEOUT: + case DPTX_APCALL_DEVICE_NOT_STARTED: + dev_warn(service->ep->dcp->dev, + "DPTXPort: firmware reports link fault %u on target %u:%u\n", + idx, service->ep->dcp->dptx_die, + service->ep->dcp->dptx_phy); + memcpy(reply, data, min(reply_size, data_size)); + return 0; case DPTX_APCALL_SET_TILED_DISPLAY_HINTS: memcpy(reply, data, min(reply_size, data_size)); return dptxport_call_set_tiled_display_hint(reply, reply_size); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 0a883bbd31a5a..ba4cf8da6caf3 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -402,12 +402,17 @@ int dcp_get_modes(struct drm_connector *connector) { struct apple_connector *apple_connector = to_apple_connector(connector); struct platform_device *pdev = apple_connector->dcp; - struct apple_dcp *dcp = platform_get_drvdata(pdev); + struct apple_dcp *dcp; struct drm_device *dev = connector->dev; struct drm_display_mode *mode; int i; + /* A Type-C port has no pipeline while the fabric is moving it. */ + if (!pdev) + return 0; + dcp = platform_get_drvdata(pdev); + for (i = 0; i < dcp->nr_modes; ++i) { mode = drm_mode_duplicate(dev, &dcp->modes[i].mode); @@ -457,7 +462,11 @@ enum drm_mode_status dcp_mode_valid(struct drm_connector *connector, { struct apple_connector *apple_connector = to_apple_connector(connector); struct platform_device *pdev = apple_connector->dcp; - struct apple_dcp *dcp = platform_get_drvdata(pdev); + struct apple_dcp *dcp; + + if (!pdev) + return MODE_ERROR; + dcp = platform_get_drvdata(pdev); return lookup_mode(dcp, mode) ? MODE_OK : MODE_BAD; } From 98ef26e3d2aa57ff18ddf08160f9b0c81dbc942f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 29 Aug 2026 23:56:34 -0400 Subject: [PATCH 1038/2680] drm: apple: keep Type-C displays configured across suspend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A display attached over Type-C came back blank after resume and stayed that way: the compositor reported every page flip as EINVAL, gave up on the output and tore it down. DCP reports the display back while the resume modeset is still in flight, and cb_hotplug() discarded any hotplug seen during a modeset. Nothing re-sends it, so the connector stayed marked disconnected for good. Latch that state instead and replay it once the modeset completes, taking the same path it would have taken had it arrived a moment later. The connector is briefly disconnected either way, because resume marks the mode valid again before the firmware reports the display back. A plane-only commit landing in that window used to be failed outright, which is what the compositor saw; defer it in dcp_flush() as is already done while a mode is invalid, and let the check pass. [ryan: adapted to 7.2.10 — struct drm_atomic_state is struct drm_atomic_commit] Signed-off-by: DjDeveloperr --- drivers/gpu/drm/apple/iomfb.c | 7 +++++-- drivers/gpu/drm/apple/iomfb_template.c | 28 +++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index ba4cf8da6caf3..3f99d4fe19605 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -423,7 +423,6 @@ int dcp_get_modes(struct drm_connector *connector) drm_mode_probed_add(connector, mode); } - if (dcp->nr_modes && dcp->dcpavserv.enabled && !apple_connector->drm_edid) { const struct drm_edid *edid; @@ -535,8 +534,12 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit) * arriving between the reconnect callback and the required modeset * would otherwise leave an unsignalled flip event in front of the * recovery commit. Modesets set valid_mode before reaching flush. + * + * The same applies while a connector is still disconnected: resume + * re-runs the modeset, which marks the mode valid again, before the + * firmware has reported the display back. */ - if (!dcp->valid_mode) { + if (!dcp->valid_mode || !dcp->connector || !dcp->connector->connected) { schedule_work(&dcp->vblank_wq); return; } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 19dde2b978723..4c4f42b432b61 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1031,9 +1031,17 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) return; if (dcp->during_modeset) { + /* + * Remember it rather than dropping it. Resume re-runs the + * modeset and the firmware reports the display back while that + * is still in flight; discarding it leaves the connector marked + * disconnected forever, with no further event to correct it. + */ dev_info(dcp->dev, - "cb_hotplug() ignored during modeset connected:%llu\n", + "cb_hotplug() deferred during modeset connected:%llu\n", *connected); + dcp->pending_hotplug = true; + dcp->pending_hotplug_connected = !!(*connected); return; } @@ -1267,6 +1275,24 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, kref_put(&cookie->refcount, release_wait_cookie); dcp->during_modeset = false; + + if (dcp->pending_hotplug) { + bool connected = dcp->pending_hotplug_connected; + struct apple_connector *connector = dcp->connector; + + dcp->pending_hotplug = false; + dev_info(dcp->dev, "replaying deferred hotplug connected:%d\n", + connected); + + if (!connected) + dcp->valid_mode = false; + + if (connector && connector->connected != connected) { + connector->connected = connected; + dcp->valid_mode = false; + schedule_work(&connector->hotplug_wq); + } + } dev_info(dcp->dev, "set_digital_out_mode finished:%d\n", ret); if (ret == 0) { From 9bc0cb45027ca728b1f54b5ac3f3281d69d460bf Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 30 Aug 2026 18:38:37 -0400 Subject: [PATCH 1039/2680] drm: apple: don't use devres for DCP property chunks The DCP that receives a chunked transfer allocates the buffer and hands it to the connector, which frees the previous one when the next arrives. devres ties the buffer to the allocating DCP, but the free uses the connector's current DCP, and a Type-C port can move to another pipeline in between. devres_destroy() then fails to find the resource on the device it is given, so devm_kfree() warns and the buffer leaks, once per dictionary: WARNING: drivers/base/devres.c:1184 at devm_kfree+0x80/0x98 The lifetime is managed explicitly already. Use kzalloc() and kfree(), and release the dictionaries when the connector is destroyed. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/apple/apple_drv.c | 8 +++++++- drivers/gpu/drm/apple/connector.c | 9 +++++++-- drivers/gpu/drm/apple/iomfb_template.c | 2 +- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index d51e20a5e8dc9..75c1d9d113a67 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -236,8 +236,14 @@ static const struct drm_mode_config_helper_funcs apple_mode_config_helpers = { static void appledrm_connector_cleanup(struct drm_connector *connector) { + struct apple_connector *apple_connector = to_apple_connector(connector); + drm_connector_cleanup(connector); - kfree(to_apple_connector(connector)); + kfree(apple_connector->color_elements.data); + kfree(apple_connector->timing_elements.data); + kfree(apple_connector->display_attributes.data); + kfree(apple_connector->transport.data); + kfree(apple_connector); } static const struct drm_connector_funcs apple_connector_funcs = { diff --git a/drivers/gpu/drm/apple/connector.c b/drivers/gpu/drm/apple/connector.c index 15b3664d85631..c7a2d97087499 100644 --- a/drivers/gpu/drm/apple/connector.c +++ b/drivers/gpu/drm/apple/connector.c @@ -125,8 +125,13 @@ static void dcp_connector_set_dict(struct apple_connector *connector, struct dcp_chunks *dict, struct dcp_chunks *chunks) { - if (dict->data) - devm_kfree(&connector->dcp->dev, dict->data); + /* + * Not devm: the buffer is allocated by whichever DCP received the + * chunks, but a Type-C port can move to another pipeline before it is + * replaced, and freeing it against the connector's current DCP then + * fails devres_destroy() -- one WARN and one leak per dictionary. + */ + kfree(dict->data); *dict = *chunks; } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 4c4f42b432b61..b3329ca2c1218 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -517,7 +517,7 @@ static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) } dcp->chunks.length = *length; - dcp->chunks.data = devm_kzalloc(dcp->dev, *length, GFP_KERNEL); + dcp->chunks.data = kzalloc(*length, GFP_KERNEL); if (!dcp->chunks.data) { dev_warn(dcp->dev, "failed to allocate chunks\n"); From 614c8d0081b8964c8f9e0bf3b428ff0aa3c73a31 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 31 Aug 2026 01:15:06 -0400 Subject: [PATCH 1040/2680] drm: apple: support ProMotion variable refresh rate DCP exposes an adaptive-sync parameter taking a minimum refresh rate, a media target rate and a fractional rate. Program it during modesets and report the panel's range through the standard vrr_capable connector property, so userspace drives VRR through the atomic VRR_ENABLED CRTC property rather than a driver-specific knob. The swap path previously wrote a constant 120 into the three timestamp fields to hold the internal panel at its maximum rate. Replace that with architectural counter timestamps taken at swap submit and at present, which is what the firmware needs in order to vary the rate at all. An internal ProMotion panel carries no EDID or DisplayID, so DCP reports no adaptive-sync range for it. Assume the ProMotion floor of 24 Hz up to the rate the mode itself advertises. Key that off the connector type from the device tree rather than the main-display flag, which is filled in by an asynchronous response and is not reliably set by the time the timing elements are parsed. External displays keep the range DCP reports, and a reported range is only believed when its maximum exceeds its minimum. Fill in display_info.monitor_range as well. Nothing else populates it for a panel with no EDID, so without it the range is missing from debugfs and from anything else that reads it. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/apple/dcp.c | 1 + drivers/gpu/drm/apple/iomfb.c | 26 ++++++++ drivers/gpu/drm/apple/iomfb_template.c | 84 +++++++++++++++++++++----- drivers/gpu/drm/apple/iomfb_template.h | 10 +-- drivers/gpu/drm/apple/parser.c | 29 +++++---- drivers/gpu/drm/apple/parser.h | 3 +- drivers/gpu/drm/apple/trace.h | 21 +++++++ 7 files changed, 138 insertions(+), 36 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index dd3060d6613c1..0cebdbccea5b4 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -656,6 +656,7 @@ static int dcp_register_typec_routes(struct apple_dcp *dcp) return 0; } + /* copied and simplified from drm_vblank.c */ static void send_vblank_event(struct drm_device *dev, struct drm_pending_vblank_event *e, diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 3f99d4fe19605..5b78f77dc7401 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -406,6 +406,8 @@ int dcp_get_modes(struct drm_connector *connector) struct drm_device *dev = connector->dev; struct drm_display_mode *mode; + u16 min_vfreq = 0, max_vfreq = 0; + bool vrr_capable = false; int i; /* A Type-C port has no pipeline while the fabric is moving it. */ @@ -414,6 +416,16 @@ int dcp_get_modes(struct drm_connector *connector) dcp = platform_get_drvdata(pdev); for (i = 0; i < dcp->nr_modes; ++i) { + if (dcp->modes[i].vrr) { + u16 lo = dcp->modes[i].min_vrr >> 16; + u16 hi = dcp->modes[i].max_vrr >> 16; + + if (!min_vfreq || lo < min_vfreq) + min_vfreq = lo; + if (hi > max_vfreq) + max_vfreq = hi; + } + vrr_capable |= dcp->modes[i].vrr; mode = drm_mode_duplicate(dev, &dcp->modes[i].mode); if (!mode) { @@ -423,6 +435,8 @@ int dcp_get_modes(struct drm_connector *connector) drm_mode_probed_add(connector, mode); } + drm_connector_set_vrr_capable_property(connector, vrr_capable); + if (dcp->nr_modes && dcp->dcpavserv.enabled && !apple_connector->drm_edid) { const struct drm_edid *edid; @@ -437,6 +451,18 @@ int dcp_get_modes(struct drm_connector *connector) if (dcp->nr_modes && apple_connector->drm_edid) drm_edid_connector_update(connector, apple_connector->drm_edid); + /* + * An internal panel has no EDID, so nothing fills in the refresh range + * that userspace needs before it will drive VRR. Supply the range DCP + * reported for the mode. This has to follow the EDID update, which + * resets display_info. + */ + if (vrr_capable && max_vfreq && + !connector->display_info.monitor_range.max_vfreq) { + connector->display_info.monitor_range.min_vfreq = min_vfreq; + connector->display_info.monitor_range.max_vfreq = max_vfreq; + } + return dcp->nr_modes; } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index b3329ca2c1218..7bd1e5d062486 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -4,6 +4,7 @@ * Copyright The Asahi Linux Contributors */ +#include #include #include #include @@ -34,6 +35,9 @@ /* Register defines used in bandwidth setup structure */ #define REG_DOORBELL_BIT(idx) (2 + (idx)) +static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), timestamp[6]) == 0x30); +static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), flags1) == 0x40); + struct dcp_wait_cookie { struct kref refcount; struct completion done; @@ -568,7 +572,9 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, if (!strcmp(req->key, "TimingElements")) { dcp->modes = enumerate_modes(&ctx, &dcp->nr_modes, dcp->width_mm, dcp->height_mm, - dcp->notch_height); + dcp->notch_height, + dcp->fixed_connector_type == + DRM_MODE_CONNECTOR_eDP); if (IS_ERR(dcp->modes)) { dev_warn(dcp->dev, "failed to parse modes\n"); @@ -1162,6 +1168,7 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) return; } dcp->swap_start = ktime_get(); + dcp->swap_submit_timestamp = arch_timer_read_counter(); while (!list_empty(&dcp->swapped_out_fbs)) { struct dcp_fb_reference *entry; @@ -1208,6 +1215,35 @@ static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, } } +/* DCP applies Adaptive Sync changes when the display mode is reselected. */ +static void dcp_on_set_adaptive_sync(struct apple_dcp *dcp, void *out, + void *cookie) +{ + dcp_set_digital_out_mode(dcp, false, &dcp->mode, + complete_set_digital_out_mode, cookie); +} + +static void dcp_set_adaptive_sync(struct apple_dcp *dcp, u32 min_vrr, + void *cookie) +{ + struct dcp_set_parameter_dcp param = { + .param = IOMFBPARAM_ADAPTIVE_SYNC, + .value = { + min_vrr, /* minRR, 16.16 fixed-point Hz */ + 0, /* mediaTargetRate */ + 0, /* fractional rate */ + }, +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + .count = 3, +#else + .count = 1, +#endif + }; + + dcp_set_parameter_dcp(dcp, false, ¶m, dcp_on_set_adaptive_sync, + cookie); +} + int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, struct drm_crtc_state *crtc_state) { @@ -1247,8 +1283,8 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, .timing_mode_id = mode->timing_mode_id }; - /* Keep track of suspected vrr modes */ - dcp->use_timestamps = mode->vrr; + /* Built-in ProMotion panels require timestamps even in fixed-120 mode. */ + dcp->use_timestamps = mode->vrr && dcp->main_display; cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) { @@ -1261,9 +1297,15 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, kref_get(&cookie->refcount); dcp->during_modeset = true; + dcp->swap_submit_timestamp = 0; - dcp_set_digital_out_mode(dcp, false, &dcp->mode, - complete_set_digital_out_mode, cookie); + if (mode->vrr) + dcp_set_adaptive_sync(dcp, + crtc_state->vrr_enabled ? mode->min_vrr : 0, + cookie); + else + dcp_set_digital_out_mode(dcp, false, &dcp->mode, + complete_set_digital_out_mode, cookie); /* * The DCP firmware has an internal timeout of ~8 seconds for @@ -1273,7 +1315,6 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(8500)); - kref_put(&cookie->refcount, release_wait_cookie); dcp->during_modeset = false; if (dcp->pending_hotplug) { @@ -1297,18 +1338,21 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, if (ret == 0) { dev_info(dcp->dev, "set_digital_out_mode timed out\n"); + kref_put(&cookie->refcount, release_wait_cookie); return -EIO; } else if (ret < 0) { dev_info(dcp->dev, "waiting on set_digital_out_mode failed:%d\n", ret); + kref_put(&cookie->refcount, release_wait_cookie); return -EIO; - - } else if (ret > 0) { + } else { dev_dbg(dcp->dev, "set_digital_out_mode finished with %d to spare\n", jiffies_to_msecs(ret)); } + kref_put(&cookie->refcount, release_wait_cookie); dcp->valid_mode = true; + dcp->vrr_enabled = mode->vrr && crtc_state->vrr_enabled; return 0; } @@ -1417,14 +1461,26 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru req->clear = 1; } - if (has_surface && dcp->use_timestamps) { + if (has_surface && (dcp->use_timestamps || dcp->vrr_enabled)) { + u64 submit_timestamp = dcp->swap_submit_timestamp; + u64 timestamp = arch_timer_read_counter(); + /* - * Fake timstamps to get 120hz refresh rate. It looks - * like the actual value does not matter, as long as it is non zero. + * IOMobileFramebuffer uses Mach continuous-time values here. On + * Apple Silicon that is the ARM architectural counter. Empirical + * testing shows that using the current submission time together + * with the previous accepted swap makes DCP follow swap pacing. */ - req->swap.ts1 = 120; - req->swap.ts2 = 120; - req->swap.ts3 = 120; + if (!submit_timestamp) + submit_timestamp = timestamp; + + /* Firmware 12.x/13.x requires timestamp types 1, 2, and 7. */ + req->swap.timestamp[0] = timestamp; + req->swap.timestamp[1] = submit_timestamp; + req->swap.timestamp[6] = timestamp; + + trace_iomfb_vrr_timestamps(dcp, dcp->vrr_enabled, timestamp, + submit_timestamp); } /* These fields should be set together */ diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index 8aea39d6664c9..31f0f76a02300 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -18,14 +18,8 @@ #include "version_utils.h" struct DCP_FW_NAME(dcp_swap) { - u64 ts1; - u64 ts2; - - u64 unk_10; - u64 unk_18; - u64 ts64_unk; - u64 unk_28; - u64 ts3; + /* IOMobileFramebuffer timestamp types 1 through 7, in wire order. */ + u64 timestamp[7]; u64 unk_38; u64 flags1; diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 3dc6fe8f0e7d8..b1bacb57e1cca 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -443,7 +443,7 @@ static u32 calculate_clock(struct dimension *horiz, struct dimension *vert) static int parse_mode(struct dcp_parse_ctx *handle, struct dcp_display_mode *out, s64 *score, int width_mm, - int height_mm, unsigned notch_height) + int height_mm, unsigned notch_height, bool internal) { int ret = 0; struct iterator it; @@ -522,20 +522,21 @@ static int parse_mode(struct dcp_parse_ctx *handle, return -EINVAL; /* - * HACK: - * Mark the 120 Hz mode on j314/j316 (identified by resolution) as vrr. - * We still do not know how to drive VRR but at least seetinng timestamps - * in the the swap_surface message to non-zero values drives the display - * at 120 fps. - */ - if (vert.precise_sync_rate >> 16 == 120 && - ((horiz.active == 3024 && vert.active == 1964) || - (horiz.active == 3456 && vert.active == 2234))) + * An internal ProMotion panel carries no EDID or DisplayID, so DCP + * reports no adaptive-sync range for it. Assume the ProMotion floor of + * 24 Hz up to whatever rate the mode itself advertises. + */ + if (internal && vert.precise_sync_rate >> 16 == 120) { + out->min_vrr = 24 << 16; + out->max_vrr = vert.precise_sync_rate; out->vrr = true; + } - if (min_vrr && max_vrr) { + /* Refresh rates are reported by DCP as 16.16 fixed-point Hz. */ + if (min_vrr && max_vrr > min_vrr) { out->min_vrr = min_vrr; out->max_vrr = max_vrr; + out->vrr = true; } vert.active -= notch_height; @@ -575,7 +576,8 @@ static int parse_mode(struct dcp_parse_ctx *handle, struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, - int height_mm, unsigned notch_height) + int height_mm, unsigned notch_height, + bool internal) { struct iterator it; int ret; @@ -597,7 +599,8 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, for (; it.idx < it.len; ++it.idx) { mode = &modes[*count]; - ret = parse_mode(it.handle, mode, &score, width_mm, height_mm, notch_height); + ret = parse_mode(it.handle, mode, &score, width_mm, height_mm, + notch_height, internal); /* Errors for a single mode are recoverable -- just skip it. */ if (ret) diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 52c4de46d2f79..35f4e81e76212 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -104,7 +104,8 @@ struct dimension { int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx); struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, - int height_mm, unsigned notch_height); + int height_mm, unsigned notch_height, + bool internal); int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, int *height_mm); int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, diff --git a/drivers/gpu/drm/apple/trace.h b/drivers/gpu/drm/apple/trace.h index a13dd34fb7aab..c0acac6ee690a 100644 --- a/drivers/gpu/drm/apple/trace.h +++ b/drivers/gpu/drm/apple/trace.h @@ -263,6 +263,27 @@ TRACE_EVENT(iomfb_swap_submit, __entry->swap_id) ); +TRACE_EVENT(iomfb_vrr_timestamps, + TP_PROTO(struct apple_dcp *dcp, bool enabled, u64 timestamp, + u64 submit_timestamp), + TP_ARGS(dcp, enabled, timestamp, submit_timestamp), + TP_STRUCT__entry( + __field(u64, dcp) + __field(bool, enabled) + __field(u64, timestamp) + __field(u64, submit_timestamp) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->enabled = enabled; + __entry->timestamp = timestamp; + __entry->submit_timestamp = submit_timestamp; + ), + TP_printk("dcp=%llx, enabled=%u, timestamp=%llu, previous=%llu", + __entry->dcp, __entry->enabled, __entry->timestamp, + __entry->submit_timestamp) +); + TRACE_EVENT(iomfb_swap_complete, TP_PROTO(struct apple_dcp *dcp, u32 swap_id), TP_ARGS(dcp, swap_id), From e5d990c528438ba34278b99915f7e93951587d20 Mon Sep 17 00:00:00 2001 From: Marcelo Alcantara Date: Wed, 16 Sep 2026 08:06:37 +1000 Subject: [PATCH 1041/2680] drm/apple: release a stale DPTX link before borrowing a Type-C route DCP's DP2HDMI HPD handler leaves disconnects to the firmware, so after the HDMI display on a hybrid pipeline is unplugged dptxport[0].connected stays true. When the fabric then borrows that pipeline for a Type-C port, dcp_dptx_connect() sees the stale flag, returns early and never trains the link, leaving the Type-C display dark. Release the leftover link when a route is activated; route availability already excludes a pipeline whose fixed output is live. Signed-off-by: Marcelo Alcantara --- drivers/gpu/drm/apple/dcp.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 0cebdbccea5b4..1b664892c3ae8 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -117,6 +117,7 @@ static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) static int dcp_typec_route_activate(struct apple_dcp_typec_route *route); static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); +static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port); /* * Pipelines are ranked by CRTC index so the fabric's choice is a pure function @@ -139,6 +140,13 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route) struct apple_dcp *dcp = route->dcp; int ret; + /* + * The fixed output's HPD handler leaves disconnects to DCP, so the port + * can still be marked connected to a display that is gone. Release it + * (a no-op otherwise), or connecting the borrowed route returns early. + */ + dcp_dptx_disconnect(dcp, 0); + if (dcp->fixed_route_selected) { ret = mux_control_deselect(dcp->xbar); if (ret) From 3fdb5c2255f2fff83f2362fe855a5ed00c5cce81 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Sun, 27 Sep 2026 14:29:45 +0200 Subject: [PATCH 1042/2680] drm/apple: afk: reuse torn-down service slots DCP tears its EPIC services down when an external display disconnects and announces them again on fresh channels when it reconnects; it never reuses a channel. On the j416s DPAVSERV endpoint (0x28), every compositor off/on or replug of the display tears down dcpav-service-epic (the service EDID is read through) and dcpdp-service-epic, and announces both again: channels 1/3 at boot, then 5/7, 9/11 and so on. afk_recv_handle_init() handed out slots append-only (ch_idx = num_channels++), and teardown only disables a service, so its slot was never given back. After boot plus seven reconnects all AFK_MAX_CHANNEL (16) slots were used, and the next reconnect failed: AFK[ep:28]: too many enabled services! AFK[ep:28]: expected announce but got 0xc0 on channel 33 The re-announced dcpav-service-epic then had no slot, so dcpavserv stayed disabled and dcp_get_modes() skipped the EDID read without logging anything. The connector's EDID stayed empty for the rest of the boot, and a physical replug did not bring it back, since the endpoint is never restarted. Reuse a slot whose service was torn down, is disabled and has no command pending. Pending commands keep their buffers in the slot until the firmware acks them, so such a slot is left alone. The slot lock is initialised once, when the slot is first used. Add debug messages for teardown and slot reuse, since a normal teardown was not logged at all. Confirmed on hardware: M2 Pro (j416s), BenQ 1440p through a USB4 hub, fresh boot, 10 compositor off/on toggles. Each cycle logs teardown of dcpdp-service-epic and dcpav-service-epic, then "reusing slot 0" and "reusing slot 1" as the two services come back, with channels reaching 43 and no "too many enabled services". The EDID read back as 256 bytes after every toggle and Hyprland kept identifying the display. Co-authored-by: Claude Opus 5.5 Signed-off-by: Oliver Lukschander --- drivers/gpu/drm/apple/afk.c | 54 +++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index b7f1629882692..52054b6845490 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -238,12 +238,49 @@ static struct apple_epic_service *afk_epic_find_service(struct apple_dcp_afkep * return NULL; } +/* + * DCP announces a service again on a fresh channel each time a display + * reconnects; it never reuses the old channel. Reuse the slot of a + * service that was torn down and disabled and has no command pending, + * otherwise repeated reconnects fill the table and later services, such + * as the one EDID is read through, cannot be registered. + */ +static int afk_alloc_service_slot(struct apple_dcp_afkep *ep) +{ + struct apple_epic_service *service; + unsigned long flags; + bool idle; + u32 i; + + for (i = 0; i < ep->num_channels; i++) { + service = &ep->services[i]; + if (service->enabled || !service->torndown) + continue; + spin_lock_irqsave(&service->lock, flags); + idle = bitmap_empty(service->cmd_map, MAX_PENDING_CMDS); + spin_unlock_irqrestore(&service->lock, flags); + if (idle) { + dev_dbg(ep->dcp->dev, + "AFK[ep:%02x]: reusing slot %u of torn-down channel %u\n", + ep->endpoint, i, service->channel); + return i; + } + } + + if (ep->num_channels >= AFK_MAX_CHANNEL) + return -ENOSPC; + + spin_lock_init(&ep->services[ep->num_channels].lock); + return ep->num_channels++; +} + static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, u8 *payload, size_t payload_size) { char name[32]; s64 epic_unit = -1; u32 ch_idx; + int slot; const char *service_name = name; const char *epic_name = NULL, *epic_class = NULL; const struct apple_epic_service_ops *ops; @@ -259,12 +296,6 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, return; } - if (ep->num_channels >= AFK_MAX_CHANNEL) { - dev_err(ep->dcp->dev, "AFK[ep:%02x]: too many enabled services!\n", - ep->endpoint); - return; - } - strscpy(name, payload, sizeof(name)); /* @@ -305,8 +336,13 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, goto free; } - ch_idx = ep->num_channels++; - spin_lock_init(&ep->services[ch_idx].lock); + slot = afk_alloc_service_slot(ep); + if (slot < 0) { + dev_err(ep->dcp->dev, "AFK[ep:%02x]: too many enabled services!\n", + ep->endpoint); + goto free; + } + ch_idx = slot; ep->services[ch_idx].enabled = true; ep->services[ch_idx].torndown = false; ep->services[ch_idx].ops = ops; @@ -337,6 +373,8 @@ static void afk_recv_handle_teardown(struct apple_dcp_afkep *ep, u32 channel) return; } + dev_dbg(ep->dcp->dev, "AFK[ep:%02x]: teardown of %s on channel %u\n", + ep->endpoint, service->ops->name, channel); afk_remove_service_debugfs(service); // TODO: think through what locking is necessary From 848730ea064a745f7e82b9fbe0470b1c79d736a2 Mon Sep 17 00:00:00 2001 From: Marcelo Alcantara Date: Sat, 26 Sep 2026 20:10:00 +1000 Subject: [PATCH 1043/2680] drm/apple: keep a Type-C display connected while its CRTC is off Powering a Type-C CRTC off drops DCP's synthetic HPD, and the firmware answers with a hotplug_gated(0) that dcpep_cb_hotplug() passes on as an unplug. dcp_poweroff() then recreates the DPTX session a second later and the firmware reports the display back. Userspace sees the connector go away and return, and compositors enable a display that appears, so DPMS off on a Type-C display is undone about 1.3 s later. Each blank also moves the display's workspaces and focus to another output and back. Ignore that unplug while the CRTC is off and the Type-C mux still reports the cable, as is already done for the unplug integrated panels report after SetPowerState(). Cable removal reaches DRM through dcp_dptx_disconnect_oob() regardless. Drop the reconnect from dcp_poweroff(): with the connector still connected, userspace enables the CRTC again and dcp_poweron() re-establishes the DPTX session synchronously, as it already does. Fixes: 5c29016e370b ("drm: apple: recover Type-C displays across link interruptions") Signed-off-by: Marcelo Alcantara --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/dcp.c | 27 +++++++++++++++----------- drivers/gpu/drm/apple/iomfb_template.c | 12 ++++++++++++ 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 7fc0e1f04c9e0..501dd96fd7c38 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -282,6 +282,8 @@ struct apple_dcp { u32 nr_typec_routes; bool phy_managed_by_typec; bool typec_cable_connected; + /* CRTC powered off while the Type-C cable stays attached */ + bool typec_crtc_off; struct delayed_work typec_reconnect_wq; struct delayed_work typec_fabric_retrain_wq; u32 typec_reconnect_tries; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 1b664892c3ae8..d9d164edd87b1 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1155,6 +1155,7 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) if (dcp_is_typec_output(dcp)) { WRITE_ONCE(dcp->typec_cable_connected, false); + WRITE_ONCE(dcp->typec_crtc_off, false); cancel_delayed_work(&dcp->typec_reconnect_wq); } @@ -1398,6 +1399,8 @@ void dcp_poweron(struct platform_device *pdev) int ret; if (dcp_is_typec_output(dcp)) { + WRITE_ONCE(dcp->typec_crtc_off, false); + /* * A Type-C CRTC disable releases its DPTX session. Re-establish it * synchronously before IOMFB is powered back on. @@ -1446,26 +1449,28 @@ void dcp_poweroff(struct platform_device *pdev) if (dcp->avep) av_service_disconnect(dcp); + /* + * Powering a Type-C CRTC off drops DCP's synthetic HPD, and the firmware + * reports that as an unplug. The display is still attached: keep the + * connector connected (see dcpep_cb_hotplug()) and let dcp_poweron() + * re-establish the DPTX session. Recreating it here instead makes the + * display vanish and come back, and compositors light a returning + * display, so DPMS off never sticks. Cable removal is reported through + * the Type-C mux. + */ + if (dcp_is_typec_output(dcp) && READ_ONCE(dcp->typec_cable_connected)) + WRITE_ONCE(dcp->typec_crtc_off, true); + _dcp_poweroff(dcp); if (dcp_is_typec_output(dcp)) { - /* - * DCP owns a synthetic HPD for Type-C. Release it with the CRTC, - * then recreate the session while the cable remains present. - */ + /* DCP owns a synthetic HPD for Type-C. Release it with the CRTC. */ if (dcp->dptxport[0].enabled && dcp->dptxport[0].connected) { ret = dptxport_set_hpd(dcp->dptxport[0].service, false); if (ret) dev_warn(dcp->dev, "failed to deassert Type-C DPTX HPD: %d\n", ret); dcp_dptx_disconnect(dcp, 0); - - if (READ_ONCE(dcp->typec_cable_connected)) { - dcp->typec_reconnect_tries = 0; - mod_delayed_work(system_freezable_wq, - &dcp->typec_reconnect_wq, - DPTX_RECONNECT_DELAY); - } } } else if (dcp->hdmi_hpd) { bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 7bd1e5d062486..64e0e0f00df58 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1036,6 +1036,18 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) if (dcp->main_display) return; + /* + * Same for the unplug a Type-C output reports after its CRTC was + * powered off with the cable still attached (see dcp_poweroff()). + */ + if (!(*connected) && READ_ONCE(dcp->typec_crtc_off) && + READ_ONCE(dcp->typec_cable_connected)) { + dev_dbg(dcp->dev, "cb_hotplug() ignoring unplug of powered-off Type-C output\n"); + dcp->valid_mode = false; + schedule_work(&dcp->vblank_wq); + return; + } + if (dcp->during_modeset) { /* * Remember it rather than dropping it. Resume re-runs the From 7e6fae3a99a604fb9c516b94960032780f04d169 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 23:34:50 -0400 Subject: [PATCH 1044/2680] drm: apple: parser: zero display modes before parsing parse_mode() only ever sets vrr (and min_vrr/max_vrr) to true, never clears it, but parse_display_modes() allocates the mode array with kmalloc_array(). When the firmware reports the display again, for example after every DPMS power-on, a mode can inherit a stale vrr flag from the reused allocation. The modeset then enables timestamped VRR swaps the display does not support, and page flips stop completing. Allocate the array zeroed. Fixes: 38c14f6107df ("drm: apple: support ProMotion variable refresh rate") --- drivers/gpu/drm/apple/parser.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index b1bacb57e1cca..a44f9d547e413 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -590,8 +590,8 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, if (ret) return ERR_PTR(ret); - /* Start with a worst case allocation */ - modes = kmalloc_array(it.len, sizeof(*modes), GFP_KERNEL); + /* Start with a worst case allocation; parse_mode() only sets vrr. */ + modes = kcalloc(it.len, sizeof(*modes), GFP_KERNEL); *count = 0; if (!modes) From db4e7fb6012db42a54272ba8133b0e8b356143b9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 21:07:31 +1100 Subject: [PATCH 1045/2680] drm/apple: iomfb: Keep the 14.7 swap layout off H17-generation firmware The 14.7 firmware ABI changes the swap record: a 0x48-byte gap before the swap ID, a wider block before the backlight fields, a u32 bl_update and bl_unk, an extra tail, and padding in the submit request. These are gated on DCP_FW_VER >= 14.7, which also matches the H17-generation firmware (26.0) that the next commit adds. That firmware keeps the earlier record and extends it with its own fields, including its own 0x48-byte gap before the swap ID. The H17P present and backlight paths were validated on that layout. Name the 14.7 layout DCP_FW_LAYOUT_14_7() and stop it at 26.0, so both layouts can be selected per firmware version. Plane IDs are also passed as IOSurface IDs in the swap and in each surface. The H17P present path was validated with those IDs left at zero, so keep them at zero for 26.0 and later until that is tested on hardware. This adapts the H17-generation support to the 7.2.10 base. It is not a change to the 14.7 support. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_template.c | 9 +++++++-- drivers/gpu/drm/apple/iomfb_template.h | 12 ++++++------ drivers/gpu/drm/apple/version_utils.h | 8 ++++++++ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 64e0e0f00df58..4d5d09607f710 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -921,7 +921,7 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) * brightness. */ if (dcp_has_panel(dcp)) { -#if DCP_FW_VER < DCP_FW_VERSION(14, 7, 0) +#if !DCP_FW_LAYOUT_14_7(DCP_FW_VER) swap->swap.bl_unk = 1; #else swap->swap.bl_update = 1; @@ -1442,13 +1442,18 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru req->swap.src_rect[l] = apple_state->src_rect; req->swap.dst_rect[l] = apple_state->dst_rect; +#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) req->swap.surf_ids[apl_plane->iomfb_surf] = plane->base.id; +#endif if (dcp->notch_height > 0) req->swap.dst_rect[l].y += dcp->notch_height; req->surf_iova[l] = apple_state->iova; req->surf[l].base = apple_state->surf; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + req->surf[l].base.surface_id = 0; +#endif /* Use sRGB colorspace only for internal panels. External * displays are expected to have EDID and user space can use @@ -1500,7 +1505,7 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru /* update brightness if changed */ if (dcp_has_panel(dcp) && dcp->brightness.update) { -#if DCP_FW_VER >= DCP_FW_VERSION(14, 7, 0) +#if DCP_FW_LAYOUT_14_7(DCP_FW_VER) req->swap.bl_update = 1; #else req->swap.bl_unk = 1; diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index 31f0f76a02300..33302f78d5ad1 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -24,7 +24,7 @@ struct DCP_FW_NAME(dcp_swap) { u64 flags1; u64 flags2; -#if DCP_FW_VER >= DCP_FW_VERSION(14, 7, 0) +#if DCP_FW_LAYOUT_14_7(DCP_FW_VER) u8 unk_v14_7[0x48]; #endif u32 swap_id; @@ -43,12 +43,12 @@ struct DCP_FW_NAME(dcp_swap) { struct dcp_rect active_regions[SWAP_SURFACES]; u8 unk_190[0x138]; u32 unk_2c8; -#if DCP_FW_VER < DCP_FW_VERSION(14, 7, 0) +#if !DCP_FW_LAYOUT_14_7(DCP_FW_VER) u8 unk_2cc[0x14]; #else u8 unk_2cc[0x40]; #endif -#if DCP_FW_VER < DCP_FW_VERSION(14, 7, 0) +#if !DCP_FW_LAYOUT_14_7(DCP_FW_VER) u32 unk_2e0; #else u32 bl_update; @@ -58,7 +58,7 @@ struct DCP_FW_NAME(dcp_swap) { #else u8 unk_2e2[3]; #endif -#if DCP_FW_VER < DCP_FW_VERSION(14, 7 ,0) +#if !DCP_FW_LAYOUT_14_7(DCP_FW_VER) u64 bl_unk; #else u32 bl_unk; @@ -68,7 +68,7 @@ struct DCP_FW_NAME(dcp_swap) { u8 unk_2f3[0x2d]; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) u8 unk_320[0x13f]; -#if DCP_FW_VER >= DCP_FW_VERSION(14, 7, 0) +#if DCP_FW_LAYOUT_14_7(DCP_FW_VER) u8 unk_14_7_2[0x30]; #endif u32 unk_flags; @@ -107,7 +107,7 @@ struct DCP_FW_NAME(dcp_swap_submit_req) { #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) u32 unkU32Ptr; #endif -#if DCP_FW_VER >= DCP_FW_VERSION(14, 7, 0) +#if DCP_FW_LAYOUT_14_7(DCP_FW_VER) u8 padding_14_7[0x234]; #endif u8 swap_null; diff --git a/drivers/gpu/drm/apple/version_utils.h b/drivers/gpu/drm/apple/version_utils.h index 242c156c75518..6d395f77e9212 100644 --- a/drivers/gpu/drm/apple/version_utils.h +++ b/drivers/gpu/drm/apple/version_utils.h @@ -12,6 +12,14 @@ #define DCP_FW_NAME(name) CONCATENATE(name, DCP_FW_SUFFIX) #define DCP_FW_VERSION(x, y, z) ( ((x) << 16) | ((y) << 8) | (z) ) +/* + * The swap record layout introduced with the 14.7 firmware. The + * H17-generation firmware (26.0 and later) keeps the earlier layout and + * extends it with its own fields. + */ +#define DCP_FW_LAYOUT_14_7(ver) \ + ((ver) >= DCP_FW_VERSION(14, 7, 0) && (ver) < DCP_FW_VERSION(26, 0, 0)) + enum dcp_firmware_version { DCP_FIRMWARE_UNKNOWN, DCP_FIRMWARE_V_12_3, From 89119a64303b556982953a4fa9a16608afdd26ed Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 23:34:50 -0400 Subject: [PATCH 1046/2680] drm: apple: support H17-generation DCP firmware on T8142 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit iBoot leaves the T8142 (M5) display coprocessor running with macOS 26 ("H17") firmware, mapped through locked DARTs. Add support for that firmware generation and attach to the live session: - Match apple,t8142-dcp and pin the H17 interface. Attach to the running coprocessor with the standard RTKit wake handshake; never stop, reset or reload it. - Add the H17 IOMFB method and callback tables, with T8142 method aliases, the nested-call reply handling, the AFK receive layout and the service-init keys this firmware uses, and the DCP bandwidth and clock requests answered from DT resources. - Take the panel dimensions from the firmware at startup and keep the mode the boot loader programmed. Only that mode is offered. - Skip teardown calls whose H17 method numbers are not established, and don't send AFK requests on H17, whose send format is unknown. Linux loads no firmware here, and no panel timing, geometry, power model or other calibration data is included. Optional properties the firmware asks for are answered "not found". Changes to shared code that alter behaviour (ACK reply buffers, framebuffer release, EPIC service-init keys) are limited to H17 firmware; v12.3 and v13.5 keep their existing paths. [ryan: adapted to 7.2.10 — DCP_FIRMWARE_H17P joins the firmware enum, which 7.2 moved to version_utils.h; the H17P cases sit next to Asahi's 14.7 ones; iomfb_h17p.c and iomfb_v14_7.c include each other's per-version headers; struct drm_atomic_state is struct drm_atomic_commit] Co-authored-by: Dj <43033058+DjDeveloperr@users.noreply.github.com> --- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/afk.c | 121 +++- drivers/gpu/drm/apple/afk.h | 46 +- drivers/gpu/drm/apple/dcp-internal.h | 35 + drivers/gpu/drm/apple/dcp-link.h | 95 +++ drivers/gpu/drm/apple/dcp.c | 220 ++++++- drivers/gpu/drm/apple/iomfb.c | 125 +++- drivers/gpu/drm/apple/iomfb.h | 57 +- drivers/gpu/drm/apple/iomfb_h17p.c | 240 +++++++ drivers/gpu/drm/apple/iomfb_h17p.h | 22 + drivers/gpu/drm/apple/iomfb_template.c | 869 +++++++++++++++++++++++-- drivers/gpu/drm/apple/iomfb_template.h | 127 +++- drivers/gpu/drm/apple/iomfb_v12_3.c | 1 + drivers/gpu/drm/apple/iomfb_v13_3.c | 1 + drivers/gpu/drm/apple/iomfb_v14_7.c | 1 + drivers/gpu/drm/apple/parser.c | 25 +- drivers/gpu/drm/apple/parser.h | 2 +- drivers/gpu/drm/apple/version_utils.h | 1 + 18 files changed, 1848 insertions(+), 141 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp-link.h create mode 100644 drivers/gpu/drm/apple/iomfb_h17p.c create mode 100644 drivers/gpu/drm/apple/iomfb_h17p.h diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index b0a7dbfbcaca8..27d7586db902c 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -11,6 +11,7 @@ appledrm-y += ibootep.o appledrm-y += iomfb_v12_3.o appledrm-y += iomfb_v13_3.o appledrm-y += iomfb_v14_7.o +appledrm-y += iomfb_h17p.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 52054b6845490..4a4215e0af2df 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -104,7 +104,9 @@ int afk_start(struct apple_dcp_afkep *ep) int ret; reinit_completion(&ep->started); - apple_rtkit_start_ep(ep->dcp->rtk, ep->endpoint); + ret = apple_rtkit_start_ep(ep->dcp->rtk, ep->endpoint); + if (ret) + return ret; afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_INIT)); ret = wait_for_completion_timeout(&ep->started, msecs_to_jiffies(1000)); @@ -116,7 +118,7 @@ int afk_start(struct apple_dcp_afkep *ep) static void afk_getbuf(struct apple_dcp_afkep *ep, u64 message) { - u16 size = FIELD_GET(GETBUF_SIZE, message) << BLOCK_SHIFT; + u32 size = FIELD_GET(GETBUF_SIZE, message) << BLOCK_SHIFT; u16 tag = FIELD_GET(GETBUF_TAG, message); u64 reply; @@ -147,10 +149,10 @@ static void afk_getbuf(struct apple_dcp_afkep *ep, u64 message) static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, struct afk_ringbuffer *bfr) { - u16 base = FIELD_GET(INITRB_OFFSET, message) << BLOCK_SHIFT; - u16 size = FIELD_GET(INITRB_SIZE, message) << BLOCK_SHIFT; + u32 base = FIELD_GET(INITRB_OFFSET, message) << BLOCK_SHIFT; + u32 size = FIELD_GET(INITRB_SIZE, message) << BLOCK_SHIFT; u16 tag = FIELD_GET(INITRB_TAG, message); - u32 bufsz, end; + u32 bufsz, hdrsz, end, block; if (tag != ep->bfr_tag) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: expected tag 0x%x but got 0x%x\n", @@ -164,7 +166,7 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, return; } - if (base >= ep->bfr_size) { + if (!ep->bfr || size < sizeof(__le32) || base >= ep->bfr_size) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: requested base 0x%x >= max size 0x%lx\n", ep->endpoint, base, ep->bfr_size); @@ -180,15 +182,21 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, } bfr->hdr = ep->bfr + base; - bufsz = le32_to_cpu(bfr->hdr->bufsz); - if (bufsz + sizeof(*bfr->hdr) != size) { + dma_rmb(); + bufsz = le32_to_cpu(READ_ONCE(*(__le32 *)bfr->hdr)); + hdrsz = size - bufsz; + block = hdrsz / AFK_RB_BLOCKS; + if (bufsz >= size || hdrsz % AFK_RB_BLOCKS || + block < AFK_RB_BLOCK_MIN || !is_power_of_2(block) || + !IS_ALIGNED(bufsz, block) || bufsz < 2 * block) { dev_err(ep->dcp->dev, - "AFK[ep:%02x]: ring buffer size 0x%x != expected 0x%lx\n", - ep->endpoint, bufsz, sizeof(*bfr->hdr)); + "AFK[ep:%02x]: ring bufsz %#x with size %#x gives no valid header (base %#x, msg %#llx)\n", + ep->endpoint, bufsz, size, base, message); return; } + bfr->block = block; - bfr->buf = bfr->hdr + 1; + bfr->buf = (u8 *)bfr->hdr + hdrsz; bfr->bufsz = bufsz; bfr->ready = true; @@ -312,7 +320,8 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, ep->endpoint, name); return; } - ret = parse_epic_service_init(&ctx, &epic_name, &epic_class, &epic_unit); + ret = parse_epic_service_init(&ctx, &epic_name, &epic_class, &epic_unit, + ep->dcp->fw_compat == DCP_FIRMWARE_H17P); if (ret) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: failed to extract init props: %d\n", @@ -553,16 +562,51 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, struct epic_hdr *ehdr = (struct epic_hdr *)data; struct epic_sub_hdr *eshdr = (struct epic_sub_hdr *)(data + sizeof(*ehdr)); - u16 subtype = le16_to_cpu(eshdr->type); - u8 *payload = data + sizeof(*ehdr) + sizeof(*eshdr); + struct epic_sub_hdr h17p_eshdr; + size_t hdr_len = sizeof(*ehdr) + sizeof(*eshdr); + u16 subtype; + u8 *payload; size_t payload_size; - if (data_size < sizeof(*ehdr) + sizeof(*eshdr)) { - dev_err(ep->dcp->dev, "AFK[ep:%02x]: payload too small: %lx\n", - ep->endpoint, data_size); - return; + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P) { + const struct epic_sub_hdr_h17p *c = + (const struct epic_sub_hdr_h17p *)(data + sizeof(*ehdr)); + + hdr_len = sizeof(*ehdr) + sizeof(*c); + if (data_size < hdr_len) + goto too_small; + + /* + * H17P multiplexes every service of an endpoint onto queue + * channel 0 and carries the real channel in the EPIC header + * instead: byte 0 is a sequence counter, byte 1 a flags byte + * and bytes 2-3 the channel. ep:20 announces "system" as + * channel 1 and "powerlog-service" as channel 3, then sends the + * mNits/uAmps/iDAC backlight reports on channel 3. Taking the + * queue channel at face value binds "system" to 0 and makes + * every later announce and report look like a duplicate. + */ + channel = le16_to_cpup((__le16 *)(data + 2)); + + /* + * Normalise into the wide sub-header the rest of this file + * expects, mapping H17P's announce subtype onto the canonical + * one so no downstream check has to know the difference. + */ + memset(&h17p_eshdr, 0, sizeof(h17p_eshdr)); + h17p_eshdr.category = c->category; + h17p_eshdr.type = cpu_to_le16(c->type == EPIC_SUBTYPE_ANNOUNCE_H17P ? + EPIC_SUBTYPE_ANNOUNCE : c->type); + h17p_eshdr.tag = cpu_to_le16(le32_to_cpu(c->tag)); + eshdr = &h17p_eshdr; } - payload_size = data_size - sizeof(*ehdr) - sizeof(*eshdr); + + if (data_size < hdr_len) + goto too_small; + + subtype = le16_to_cpu(eshdr->type); + payload = data + hdr_len; + payload_size = data_size - hdr_len; trace_afk_recv_handle(ep, channel, type, data_size, ehdr, eshdr); @@ -620,6 +664,11 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, "(type %x subtype %x)\n", ep->endpoint, channel, type, subtype); print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, payload, payload_size, true); + return; + +too_small: + dev_err(ep->dcp->dev, "AFK[ep:%02x]: payload too small: %lx\n", + ep->endpoint, data_size); } static bool afk_recv(struct apple_dcp_afkep *ep) @@ -634,8 +683,8 @@ static bool afk_recv(struct apple_dcp_afkep *ep) return false; } - rptr = le32_to_cpu(ep->rxbfr.hdr->rptr); - wptr = le32_to_cpu(ep->rxbfr.hdr->wptr); + rptr = le32_to_cpu(*afk_rb_rptr(&ep->rxbfr)); + wptr = le32_to_cpu(*afk_rb_wptr(&ep->rxbfr)); trace_afk_recv_rwptr_pre(ep, rptr, wptr); if (rptr == wptr) @@ -680,7 +729,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) return false; } - ep->rxbfr.hdr->rptr = cpu_to_le32(rptr); + *afk_rb_rptr(&ep->rxbfr) = cpu_to_le32(rptr); } if (rptr + size + sizeof(*hdr) > ep->rxbfr.bufsz) { @@ -693,7 +742,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) channel = le32_to_cpu(hdr->channel); type = le32_to_cpu(hdr->type); - rptr = ALIGN(rptr + sizeof(*hdr) + size, 1 << BLOCK_SHIFT); + rptr = ALIGN(rptr + sizeof(*hdr) + size, ep->rxbfr.block); if (WARN_ON(rptr > ep->rxbfr.bufsz)) rptr = 0; if (rptr == ep->rxbfr.bufsz) @@ -701,7 +750,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) dma_mb(); - ep->rxbfr.hdr->rptr = cpu_to_le32(rptr); + *afk_rb_rptr(&ep->rxbfr) = cpu_to_le32(rptr); trace_afk_recv_rwptr_post(ep, rptr, wptr); /* @@ -790,11 +839,27 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, size_t total_epic_size, total_size; int ret; + /* + * H17P frames EPIC messages differently (see afk_recv_handle()): an + * 8-byte sub-header, with the service channel carried in the EPIC + * header on queue channel 0. Only the receive side of that format is + * known, and the 24-byte framing below would be misparsed, so send + * nothing. The display path needs no AP-initiated EPIC traffic: the + * callers are optional (verbose firmware logging, EDID copy, audio, + * DPTX, debugfs) or replies to calls only DPTX services make. + */ + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P) { + dev_dbg_once(ep->dcp->dev, + "AFK[ep:%02x]: not sending EPIC message on H17P\n", + ep->endpoint); + return -EOPNOTSUPP; + } + spin_lock_irqsave(&ep->lock, flags); dma_rmb(); - rptr = le32_to_cpu(ep->txbfr.hdr->rptr); - wptr = le32_to_cpu(ep->txbfr.hdr->wptr); + rptr = le32_to_cpu(*afk_rb_rptr(&ep->txbfr)); + wptr = le32_to_cpu(*afk_rb_wptr(&ep->txbfr)); trace_afk_send_rwptr_pre(ep, rptr, wptr); total_epic_size = sizeof(*ehdr) + sizeof(*eshdr) + payload_len; total_size = sizeof(*hdr) + total_epic_size; @@ -907,12 +972,12 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, memcpy(ep->txbfr.buf + wptr, payload, payload_len); wptr += payload_len; - wptr = ALIGN(wptr, 1 << BLOCK_SHIFT); + wptr = ALIGN(wptr, ep->txbfr.block); if (wptr == ep->txbfr.bufsz) wptr = 0; trace_afk_send_rwptr_post(ep, rptr, wptr); - ep->txbfr.hdr->wptr = cpu_to_le32(wptr); + *afk_rb_wptr(&ep->txbfr) = cpu_to_le32(wptr); afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_SEND) | FIELD_PREP(SEND_WPTR, wptr)); ret = 0; diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index a339c00a2a013..6ba47202e23a9 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -71,15 +71,17 @@ struct apple_epic_service_ops { void (*teardown)(struct apple_epic_service *service); }; -struct afk_ringbuffer_header { - __le32 bufsz; - u32 unk; - u32 _pad1[14]; - __le32 rptr; - u32 _pad2[15]; - __le32 wptr; - u32 _pad3[15]; -}; +/* + * The ring header is three control blocks -- bufsz, rptr, wptr -- each padded + * out to the firmware's control-block size. That block is 0x40 bytes up to + * v13.5 but 0x80 on H17P, making the header 0xc0 or 0x180. The size is + * self-describing: the INITRB message gives the total ring size and the first + * word of block 0 gives the payload size, so the stride is (size - bufsz) / 3 + * and no firmware-version test is needed. Address the fields by stride rather + * than through a fixed struct. + */ +#define AFK_RB_BLOCK_MIN 0x40 +#define AFK_RB_BLOCKS 3 struct afk_qe { #define QE_MAGIC 0x20504f49 // ' POI' @@ -109,6 +111,21 @@ struct epic_sub_hdr { __le32 inline_len; } __attribute__((packed)); +/* + * H17P compresses the EPIC sub-header from 24 bytes to 8 and drops the + * separate length/timestamp: the announce for the "system" service puts its + * name at data + 0x18, not data + 0x28. An announce carries type 0x11 and a + * standard-service report carries the usual 0xc0. + */ +struct epic_sub_hdr_h17p { + u8 type; + u8 category; + __le16 flags; + __le32 tag; +} __packed; + +#define EPIC_SUBTYPE_ANNOUNCE_H17P 0x11 + struct epic_cmd { __le32 retcode; __le64 rxbuf; @@ -152,12 +169,21 @@ enum epic_subtype { struct afk_ringbuffer { bool ready; - struct afk_ringbuffer_header *hdr; + void *hdr; + u32 block; /* control-block stride, 0x40 or 0x80 */ u32 rptr; void *buf; size_t bufsz; }; +static inline __le32 *afk_rb_field(struct afk_ringbuffer *bfr, unsigned int idx) +{ + return (__le32 *)((u8 *)bfr->hdr + idx * bfr->block); +} + +#define afk_rb_rptr(bfr) afk_rb_field(bfr, 1) +#define afk_rb_wptr(bfr) afk_rb_field(bfr, 2) + struct apple_dcp_afkep { struct apple_dcp *dcp; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 501dd96fd7c38..7f80c1810e2ed 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -17,6 +18,7 @@ #include "dptxep.h" #include "iomfb.h" +#include "iomfb_h17p.h" #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" #include "iomfb_v14_7.h" @@ -74,6 +76,8 @@ struct dcp_mem_descriptor { dma_addr_t dva; struct sg_table map; u64 reg; + /* live iommu mappings installed by dcpep_cb_map_piodma() */ + bool piodma_mapped; }; /* Limit on call stack depth (arbitrary). Some nesting is required */ @@ -96,7 +100,14 @@ struct dcp_channel { struct dcp_fb_reference { struct list_head head; struct drm_framebuffer *fb; + /* + * Id of the swap that unbinds @fb, i.e. the one that puts its + * replacement on screen. Only meaningful once @armed is set: the + * firmware assigns the id in the swap_start reply, which happens after + * the atomic commit that displaced @fb has already queued this entry. + */ u32 swap_id; + bool armed; }; #define MAX_NOTCH_HEIGHT 160 @@ -122,8 +133,20 @@ struct dcp_panel { bool has_mini_led; }; +enum dcp_iomfb_method_profile { + DCP_IOMFB_METHODS_DEFAULT, + DCP_IOMFB_METHODS_H17G, +}; + struct apple_dcp_hw_data { u32 num_dptx_ports; + enum dcp_iomfb_method_profile iomfb_method_profile; + /* + * The bootloader leaves the coprocessor running. Attach to it with a + * standard RTKit wake instead of starting or restarting the ASC. + */ + bool adopt_live_session; + enum dcp_firmware_version firmware_compat; }; /* TODO: move IOMFB members to its own struct */ @@ -131,6 +154,8 @@ struct apple_dcp { struct device *dev; struct platform_device *piodma; struct iommu_domain *iommu_dom; + /* which of the nine cumulative A031 notify-client states to send next */ + unsigned int a031_step; struct apple_rtkit *rtk; struct apple_crtc *crtc; struct apple_connector *connector; @@ -155,6 +180,7 @@ struct apple_dcp { /* clock rate request by dcp in */ struct clk *clk; + struct clk *clk_194; /* DCP shared memory */ void *shmem; @@ -192,10 +218,13 @@ struct apple_dcp { struct dcp_swap_submit_req_v12_3 v12_3; struct dcp_swap_submit_req_v13_3 v13_3; struct dcp_swap_submit_req_v14_7_0 v14_7_0; + struct dcp_swap_submit_req_h17p h17p; } swap; /* swap id of the last completed swap */ u32 last_swap_id; + /* last_swap_id is only meaningful after the first swap completes */ + bool have_swap_complete; ktime_t swap_start; u64 swap_submit_timestamp; @@ -244,6 +273,12 @@ struct apple_dcp { * on the next successfully completed swap. */ struct list_head swapped_out_fbs; + /* + * Protects swapped_out_fbs, which is appended to from the DRM atomic + * commit (dcp_flush -> .atomic_flush) and armed/drained from the RTKit + * workqueue that runs the DCP callbacks. + */ + spinlock_t swapped_out_lock; struct dcp_brightness brightness; /* Workqueue for updating the initial brightness */ diff --git a/drivers/gpu/drm/apple/dcp-link.h b/drivers/gpu/drm/apple/dcp-link.h new file mode 100644 index 0000000000000..ae1fcabdfc0b2 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-link.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright The Asahi Linux Contributors */ + +#ifndef __APPLE_DCP_LINK_H__ +#define __APPLE_DCP_LINK_H__ + +#include +#include +#include + +#include + +/* + * H17-generation DCP firmware uses the ordinary IOMFB transport once the + * INIT/SetShmem handshake is done: the same six fixed channels (CMD 0x00000, + * OOBCMD 0x08000, ASYNC 0x40000, OOBASYNC 0x48000, CB 0x60000, + * OOBCB 0x68000), selected by the context field of the message. Records are + * allocated on each channel's stack at + * off_next = off_cur + align_up(12 + in_len + out_len, 0x40) + * which is the rule dcp_packet_start() and DCP_PACKET_ALIGNMENT implement in + * iomfb.c. + * + * This header only describes the callback payloads whose layout differs from + * older firmware. + */ + +#define APPLE_DCP_H17P_CLOCK_ID_154 0x154 +#define APPLE_DCP_H17P_CLOCK_ID_194 0x194 + +struct apple_dcp_h17p_clock_request { + __le32 service; + __le32 clock_id; +} __packed; + +struct apple_dcp_h17p_rt_bw_config { + __le64 scratch; + __le64 clock_request; +} __packed; + +struct apple_dcp_h17p_rt_bw_request { + __le32 config_null; +} __packed; + +struct apple_dcp_h17p_rt_bw_reply { + struct apple_dcp_h17p_rt_bw_config config; + __le32 status; +} __packed; + +static_assert(sizeof(struct apple_dcp_h17p_clock_request) == 0x08); +static_assert(sizeof(struct apple_dcp_h17p_rt_bw_config) == 0x10); +static_assert(sizeof(struct apple_dcp_h17p_rt_bw_request) == 0x04); +static_assert(sizeof(struct apple_dcp_h17p_rt_bw_reply) == 0x14); + +static inline int +apple_dcp_h17p_clock_rate(const struct apple_dcp_h17p_clock_request *request, + u64 rate_154, u64 rate_194, u64 *rate) +{ + if (!request || !rate) + return -EINVAL; + + switch (le32_to_cpu(request->clock_id)) { + case APPLE_DCP_H17P_CLOCK_ID_154: + *rate = rate_154; + return 0; + case APPLE_DCP_H17P_CLOCK_ID_194: + *rate = rate_194; + return 0; + default: + return -ENOENT; + } +} + +static inline void +apple_dcp_h17p_rt_bw_encode(struct apple_dcp_h17p_rt_bw_config *config, + u64 scratch, u64 clock_request) +{ + config->scratch = cpu_to_le64(scratch); + config->clock_request = cpu_to_le64(clock_request); +} + +/* D003: fill in the bandwidth config unless the firmware passed it as null */ +static inline int +apple_dcp_h17p_rt_bw_encode_reply(const struct apple_dcp_h17p_rt_bw_request *request, + struct apple_dcp_h17p_rt_bw_reply *reply, + u64 scratch, u64 clock_request) +{ + if (le32_to_cpu(request->config_null) & 1) + return -EINVAL; + + apple_dcp_h17p_rt_bw_encode(&reply->config, scratch, clock_request); + reply->status = cpu_to_le32(0); + return 0; +} + +#endif /* __APPLE_DCP_LINK_H__ */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index d9d164edd87b1..b6cf87eb97126 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -826,12 +827,42 @@ static void dcp_delayed_vblank(struct work_struct *work) dcp_drm_crtc_vblank(dcp->crtc); } +static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) +{ + switch (endpoint) { + case AV_ENDPOINT: + return dcp->avep; + case SYSTEM_ENDPOINT: + return dcp->systemep; + case DISP0_ENDPOINT: + return dcp->ibootep; + case DPAVSERV_ENDPOINT: + return dcp->dcpavservep; + case DPTX_ENDPOINT: + return dcp->dptxep; + default: + return NULL; + } +} + static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) { struct apple_dcp *dcp = cookie; trace_dcp_recv_msg(dcp, endpoint, message); + /* + * H17P starts endpoints that have no AFK instance behind them (the + * remote allocator endpoint, for one). Log their messages rather than + * dereferencing a NULL afkep or warning about an unknown endpoint. + */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P && endpoint != IOMFB_ENDPOINT && + !dcp_afkep(dcp, endpoint)) { + dev_dbg_ratelimited(dcp->dev, "ep %#04x: %#llx\n", endpoint, + message); + return; + } + switch (endpoint) { case IOMFB_ENDPOINT: return iomfb_recv_msg(dcp, message); @@ -870,9 +901,32 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ complete(&dcp->start_done); } +/* + * Check that every IOMMU page of [iova, iova + size) translates to the same + * offset from @phys as it has from @iova. + */ +static int dcp_check_iova_contiguous(struct iommu_domain *domain, + dma_addr_t iova, size_t size, + phys_addr_t phys) +{ + unsigned long pgsize = domain->pgsize_bitmap ? + 1UL << __ffs(domain->pgsize_bitmap) : PAGE_SIZE; + dma_addr_t end = iova + size; + dma_addr_t addr = iova; + + while (addr < end) { + if (iommu_iova_to_phys(domain, addr) != phys + (addr - iova)) + return -EINVAL; + addr = ALIGN_DOWN(addr, pgsize) + pgsize; + } + + return 0; +} + static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) { struct apple_dcp *dcp = cookie; + int ret; if (bfr->iova) { struct iommu_domain *domain = @@ -887,6 +941,16 @@ static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) if (!phy_addr) return -ENOMEM; + /* memremap() below needs the whole buffer physically contiguous */ + ret = dcp_check_iova_contiguous(domain, bfr->iova, bfr->size, + phy_addr); + if (ret) { + dev_err(dcp->dev, + "shmem_setup: iova %pad (%#zx bytes) is not physically contiguous\n", + &bfr->iova, bfr->size); + return ret; + } + // TODO: verify phy_addr, cache attribute bfr->buffer = memremap(phy_addr, bfr->size, MEMREMAP_WB); if (!bfr->buffer) @@ -1245,6 +1309,18 @@ int dcp_start(struct platform_device *pdev) init_completion(&dcp->start_done); + /* + * The coprocessor start can fail (or be recovered from) without tearing + * the DRM side down, so this can be reached with no RTKit instance at + * all. Starting an endpoint then dereferences NULL inside + * apple_rtkit_start_ep() and takes the machine with it. + */ + if (!dcp->rtk || IS_ERR(dcp->rtk)) { + dev_err(dcp->dev, + "cannot start endpoints: RTKit is not initialised\n"); + return -ENODEV; + } + /* start RTKit endpoints */ ret = systemep_init(dcp); if (ret) @@ -1322,6 +1398,9 @@ static void _dcp_poweroff(struct apple_dcp *dcp) case DCP_FIRMWARE_V_14_7: iomfb_poweroff_v14_7_0(dcp); break; + case DCP_FIRMWARE_H17P: + iomfb_poweroff_h17p(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -1387,6 +1466,9 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) case DCP_FIRMWARE_V_14_7: iomfb_sleep_v14_7_0(dcp); break; + case DCP_FIRMWARE_H17P: + iomfb_sleep_h17p(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -1432,6 +1514,9 @@ void dcp_poweron(struct platform_device *pdev) case DCP_FIRMWARE_V_14_7: iomfb_poweron_v14_7_0(dcp); break; + case DCP_FIRMWARE_H17P: + iomfb_poweron_h17p(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -1522,8 +1607,30 @@ static int dcp_create_piodma_iommu_dev(struct apple_dcp *dcp) "Failed to get piodma child DT node\n"); dcp->piodma = of_platform_device_create(node, NULL, dcp->dev); - if (!dcp->piodma) - return dev_err_probe(dcp->dev, -ENODEV, "Failed to create piodma pdev for %pOF\n", node); + if (!dcp->piodma) { + /* + * of_platform_device_create() returns NULL when the node already + * has a platform device, which happens whenever a previous bind + * attempt created it and then failed further down (the caller's + * later error paths do not destroy it). Probe is retried - a + * deferral, or a component re-bind - and would then fail here + * forever with -ENODEV even though nothing is actually wrong. + * Adopt the existing device instead of giving up. + */ + dcp->piodma = of_find_device_by_node(node); + if (!dcp->piodma) + return dev_err_probe(dcp->dev, -ENODEV, + "Failed to create piodma pdev for %pOF\n", + node); + /* + * Drop the lookup reference: the device stays registered until + * of_platform_device_destroy(), exactly like one created above, + * and that call only releases the registration reference. + */ + put_device(&dcp->piodma->dev); + dev_info(dcp->dev, "reusing existing piodma pdev for %pOF\n", + node); + } ret = dma_set_mask_and_coherent(&dcp->piodma->dev, DMA_BIT_MASK(42)); if (ret) @@ -1648,18 +1755,67 @@ static int dcp_get_bw_doorbell_reg(struct apple_dcp *dcp, u32 expected) return ret; } +/* + * Count the "disp-*" entries in reg-names. + * + * The positional form below assumes the reg layout is [coproc, disp-0 ...], + * i.e. exactly one non-disp register, and derives the count as + * num_resources - 1. DCPs with H17-generation firmware list a trailing + * "iop-vbar" register after the display apertures, which would inflate that + * count, bind the last disp_registers[] entry to iop-vbar and make the + * "apple,bw-scratch" disp_reg index check reject a valid device tree. + * + * Returns a negative errno when reg-names is absent, so callers fall back. + */ +static int dcp_count_disp_regs(struct device *dev) +{ + int n = of_property_count_strings(dev->of_node, "reg-names"); + const char *name; + int i, count = 0; + + if (n <= 0) + return -EINVAL; + + for (i = 0; i < n; ++i) { + if (of_property_read_string_index(dev->of_node, "reg-names", i, + &name)) + return -EINVAL; + if (!strncmp(name, "disp-", 5)) + count++; + } + + return count ? count : -EINVAL; +} + static int dcp_get_disp_regs(struct apple_dcp *dcp) { struct platform_device *pdev = to_platform_device(dcp->dev); - int count = pdev->num_resources - 1; + int count = dcp_count_disp_regs(dcp->dev); + bool by_name = count > 0; int i, ret; + /* Device trees without reg-names keep the historical positional form. */ + if (!by_name) + count = pdev->num_resources - 1; + if (count <= 0 || count > MAX_DISP_REGISTERS) return -EINVAL; for (i = 0; i < count; ++i) { - dcp->disp_registers[i] = - platform_get_resource(pdev, IORESOURCE_MEM, 1 + i); + if (by_name) { + char name[8]; + + snprintf(name, sizeof(name), "disp-%d", i); + dcp->disp_registers[i] = platform_get_resource_byname( + pdev, IORESOURCE_MEM, name); + } else { + dcp->disp_registers[i] = platform_get_resource( + pdev, IORESOURCE_MEM, 1 + i); + } + if (!dcp->disp_registers[i]) { + dev_err(dcp->dev, "missing display register %d\n", i); + return -EINVAL; + } } /* load pmgr bandwidth scratch resource and offset */ @@ -1726,10 +1882,18 @@ static int dcp_read_fw_version(struct device *dev, const char *name, static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) { + const struct apple_dcp_hw_data *hw = of_device_get_match_data(dev); char compat_str[DCP_FW_VERSION_STR_LEN]; char fw_str[DCP_FW_VERSION_STR_LEN]; int ret; + /* + * SoCs introduced with H17-generation firmware pin its interface; the + * loader may not recognise their boot firmware version. + */ + if (hw->firmware_compat != DCP_FIRMWARE_UNKNOWN) + return hw->firmware_compat; + /* firmware version is just informative */ dcp_read_fw_version(dev, "apple,firmware-version", fw_str); @@ -1790,6 +1954,20 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (IS_ERR(dcp->coproc_reg)) return PTR_ERR(dcp->coproc_reg); + /* + * Display coprocessors running H17-generation firmware are left running + * by the bootloader with their firmware mapped through a locked DART. + * Linux attaches to that session with a standard RTKit INIT/HELLO + * exchange and must not stop or restart the ASC. + */ + if (dcp->hw.adopt_live_session) { + cpu_ctrl = readl_relaxed(dcp->coproc_reg + + APPLE_DCP_COPROC_CPU_CONTROL); + if (!(cpu_ctrl & APPLE_DCP_COPROC_CPU_CONTROL_RUN)) + return dev_err_probe(dev, -EOPNOTSUPP, + "DCP was not left running by the bootloader\n"); + } + if (dcp->index || dcp->dptx_phy || dcp->dptx_die) dev_info(dev, "DCP index:%u dptx target phy: %u dptx die: %u\n", dcp->index, dcp->dptx_phy, dcp->dptx_die); @@ -1850,6 +2028,10 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (IS_ERR(dcp->clk)) return dev_err_probe(dev, PTR_ERR(dcp->clk), "Unable to find clock\n"); + dcp->clk_194 = devm_clk_get_optional(dev, "clock-194"); + if (IS_ERR(dcp->clk_194)) + return dev_err_probe(dev, PTR_ERR(dcp->clk_194), + "Unable to find clock 0x194\n"); bitmap_zero(dcp->memdesc_map, DCP_MAX_MAPPINGS); // TDOD: mem_desc IDs start at 1, for simplicity just skip '0' entry @@ -1859,17 +2041,23 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) dcp->swapped_out_fbs = (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); + spin_lock_init(&dcp->swapped_out_lock); - cpu_ctrl = - readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); - writel_relaxed(cpu_ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, - dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + if (!dcp->hw.adopt_live_session) { + cpu_ctrl = + readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + writel_relaxed(cpu_ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, + dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + } dcp->rtk = devm_apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); if (IS_ERR(dcp->rtk)) return dev_err_probe(dev, PTR_ERR(dcp->rtk), "Failed to initialize RTKit\n"); + if (dcp->hw.adopt_live_session) + dev_info(dev, "negotiating RTKit with the running DCP\n"); + ret = apple_rtkit_wake(dcp->rtk); if (ret) return dev_err_probe(dev, ret, @@ -1938,6 +2126,8 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) devm_clk_put(dev, dcp->clk); dcp->clk = NULL; + /* optional and possibly NULL, released with the bind's devres group */ + dcp->clk_194 = NULL; } static const struct component_ops dcp_comp_ops = { @@ -2174,6 +2364,17 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8112 = { .num_dptx_ports = 2, }; +/* + * M5 (T8142) runs H17-generation DCP firmware with the H17G method numbering + * and is left running by the bootloader. + */ +static const struct apple_dcp_hw_data apple_dcp_hw_t8142 = { + .num_dptx_ports = 0, + .iomfb_method_profile = DCP_IOMFB_METHODS_H17G, + .adopt_live_session = true, + .firmware_compat = DCP_FIRMWARE_H17P, +}; + static const struct apple_dcp_hw_data apple_dcp_hw_dcp = { .num_dptx_ports = 0, }; @@ -2185,6 +2386,7 @@ static const struct apple_dcp_hw_data apple_dcp_hw_dcpext = { static const struct of_device_id of_match[] = { { .compatible = "apple,t6020-dcp", .data = &apple_dcp_hw_t6020, }, { .compatible = "apple,t8112-dcp", .data = &apple_dcp_hw_t8112, }, + { .compatible = "apple,t8142-dcp", .data = &apple_dcp_hw_t8142, }, { .compatible = "apple,dcp", .data = &apple_dcp_hw_dcp, }, { .compatible = "apple,dcpext", .data = &apple_dcp_hw_dcpext, }, {} diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 5b78f77dc7401..6c2fd76b8ba02 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -161,6 +161,13 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca u32 in_len, u32 out_len, void *data, dcp_callback_t cb, void *cookie) { + struct dcp_method_entry resolved = *call; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G && + call->tag_h17g[0]) + memcpy(resolved.tag, call->tag_h17g, sizeof(resolved.tag)); + call = &resolved; + enum dcp_context_id context = dcp_call_context(dcp, oob); struct dcp_channel *ch = dcp_get_channel(dcp, context); @@ -324,15 +331,30 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, struct dcp_channel *ch = dcp_get_channel(dcp, context); u8 depth; - if (tag < 0 || tag >= IOMFB_MAX_CB || !dcp->cb_handlers || !dcp->cb_handlers[tag]) { + in = data + sizeof(*hdr); + out = in + hdr->in_len; + + if (tag < 0 || tag >= IOMFB_MAX_CB || !dcp->cb_handlers || + !dcp->cb_handlers[tag]) { dev_warn(dev, "received unknown callback %c%c%c%c\n", hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); + /* + * Leaving a callback unanswered wedges the coprocessor: it + * waits for the ack forever and the outer call never returns. + * On H17P answer it with a zeroed output instead. + */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + if (hdr->out_len) + memset(out, 0, hdr->out_len); + depth = dcp_push_depth(&ch->depth); + ch->output[depth] = out; + ch->end[depth] = offset + + ALIGN(length, DCP_PACKET_ALIGNMENT); + dcp_ack(dcp, context); + } return; } - in = data + sizeof(*hdr); - out = in + hdr->in_len; - // TODO: verify that in_len and out_len match our prototypes // for now just clear the out data to have at least consistent results if (hdr->out_len) @@ -351,7 +373,8 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, { struct dcp_packet_header *header = data; struct dcp_channel *ch = dcp_get_channel(dcp, context); - void *cookie; + bool h17p = dcp->fw_compat == DCP_FIRMWARE_H17P; + void *cookie, *out; dcp_callback_t cb; if (!ch) { @@ -359,6 +382,11 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, return; } + if (h17p && !ch->depth) { + dev_warn(dcp->dev, "ignoring ack on idle context %X\n", context); + return; + } + dcp_pop_depth(&ch->depth); cb = ch->callbacks[ch->depth]; @@ -367,8 +395,21 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, ch->callbacks[ch->depth] = NULL; ch->cookies[ch->depth] = NULL; + if (h17p) { + /* + * H17P acks carry no payload and offset zero, including for + * nested calls, so the output cannot be located from the ack + * message. It is in the original AP command record, whose + * address dcp_push() saved. + */ + out = ch->output[ch->depth]; + ch->output[ch->depth] = NULL; + } else { + out = data + sizeof(*header) + header->in_len; + } + if (cb) - cb(dcp, data + sizeof(*header) + header->in_len, cookie); + cb(dcp, out, cookie); } static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) @@ -437,7 +478,9 @@ int dcp_get_modes(struct drm_connector *connector) } drm_connector_set_vrr_capable_property(connector, vrr_capable); + /* H17P sends no EPIC commands; see afk_send_epic(). */ if (dcp->nr_modes && dcp->dcpavserv.enabled && + dcp->fw_compat != DCP_FIRMWARE_H17P && !apple_connector->drm_edid) { const struct drm_edid *edid; edid = dcpavserv_copy_edid(dcp->dcpavserv.service); @@ -482,6 +525,27 @@ struct dcp_display_mode *lookup_mode(struct apple_dcp *dcp, return NULL; } +/* + * H17P keeps the mode the bootloader programmed and never sends + * set_digital_out_mode (see DCP_INHERIT_BOOT_MODE in iomfb_template.c), so a + * modeset to any other mode would silently not be applied. The bootloader + * brings the panel up in its native timing, which is the mode the firmware + * scores highest and enumerate_modes() marks preferred; offer only that one. + */ +static bool dcp_mode_settable(struct apple_dcp *dcp, + const struct drm_display_mode *mode) +{ + struct dcp_display_mode *dcp_mode = lookup_mode(dcp, mode); + + if (!dcp_mode) + return false; + + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_mode->mode.type & DRM_MODE_TYPE_PREFERRED; + + return true; +} + enum drm_mode_status dcp_mode_valid(struct drm_connector *connector, const struct drm_display_mode *mode) { @@ -493,7 +557,7 @@ enum drm_mode_status dcp_mode_valid(struct drm_connector *connector, return MODE_ERROR; dcp = platform_get_drvdata(pdev); - return lookup_mode(dcp, mode) ? MODE_OK : MODE_BAD; + return dcp_mode_settable(dcp, mode) ? MODE_OK : MODE_BAD; } int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, @@ -528,6 +592,9 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, case DCP_FIRMWARE_V_14_7: ret = iomfb_modeset_v14_7_0(dcp, crtc_state); break; + case DCP_FIRMWARE_H17P: + ret = iomfb_modeset_h17p(dcp, crtc_state); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); @@ -546,7 +613,7 @@ bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, struct apple_dcp *dcp = platform_get_drvdata(pdev); /* TODO: support synthesized modes through scaling */ - return lookup_mode(dcp, mode) != NULL; + return dcp_mode_settable(dcp, mode); } @@ -595,6 +662,9 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit) case DCP_FIRMWARE_V_14_7: iomfb_flush_v14_7_0(dcp, crtc, commit); break; + case DCP_FIRMWARE_H17P: + iomfb_flush_h17p(dcp, crtc, commit); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -613,6 +683,9 @@ static void iomfb_start(struct apple_dcp *dcp) case DCP_FIRMWARE_V_14_7: iomfb_start_v14_7_0(dcp); break; + case DCP_FIRMWARE_H17P: + iomfb_start_h17p(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -630,9 +703,19 @@ void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) { enum dcpep_type type = FIELD_GET(IOMFB_MESSAGE_TYPE, message); - if (type == IOMFB_MESSAGE_TYPE_INITIALIZED) + if (type == IOMFB_MESSAGE_TYPE_INITIALIZED) { + /* + * H17P reports its interface version (bits 63:48) and a + * firmware hash in the high half of the InitComplete word. + * The transport below it is unchanged. + */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + dev_info(dcp->dev, + "IOMFB: init complete %#llx (version %llu)\n", + message, + FIELD_GET(GENMASK_ULL(63, 48), message)); iomfb_start(dcp); - else if (type == IOMFB_MESSAGE_TYPE_MSG) + } else if (type == IOMFB_MESSAGE_TYPE_MSG) dcpep_got_msg(dcp, message); else dev_warn(dcp->dev, "Ignoring unknown message %llx\n", message); @@ -641,10 +724,27 @@ void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) int iomfb_start_rtkit(struct apple_dcp *dcp) { dma_addr_t shmem_iova; - apple_rtkit_start_ep(dcp->rtk, IOMFB_ENDPOINT); + int ret; + + /* + * H17P firmware expects the remote allocator endpoint to be started + * before IOMFB. A firmware subsystem whose endpoint was never started + * has nothing answering it on the AP side, and without this one the + * display DART faults. + */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + ret = apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); + if (ret) + return ret; + } + ret = apple_rtkit_start_ep(dcp->rtk, IOMFB_ENDPOINT); + if (ret) + return ret; dcp->shmem = dma_alloc_coherent(dcp->dev, DCP_SHMEM_SIZE, &shmem_iova, GFP_KERNEL); + if (!dcp->shmem) + return -ENOMEM; dcp_send_message(dcp, IOMFB_ENDPOINT, dcpep_set_shmem(shmem_iova)); @@ -667,6 +767,9 @@ void iomfb_shutdown(struct apple_dcp *dcp) case DCP_FIRMWARE_V_14_7: iomfb_shutdown_v14_7_0(dcp); break; + case DCP_FIRMWARE_H17P: + iomfb_shutdown_h17p(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 531089dab6313..b816c75e2a855 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -156,6 +156,7 @@ enum dcpep_method { dcpep_flush_supports_power, dcpep_set_power_state, dcpep_first_client_open, + dcpep_register_dfb_surface, dcpep_set_parameter_dcp, dcpep_enable_disable_video_power_savings, dcpep_is_main_display, @@ -166,14 +167,43 @@ enum dcpep_method { iomfbep_last_client_close, iomfbep_abort_swaps_dcp, iomfbep_set_matrix, + /* + * H17P default-framebuffer queries, issued from inside + * start_hardware_boot (D121) between create_default_fb and + * setup_video_limits. + */ + dcpep_get_dfb_compression_info, + dcpep_get_dfb_info, + dcpep_get_dfb_layout, + dcpep_get_dfb_state, + dcpep_set_dfb_dimensions, + dcpep_commit_dfb_info, + dcpep_dfb_query, + dcpep_dfb_ready_query, + /* + * H17P pipe-configuration calls made before the first swap. Most + * names are unknown; they are identified by method number. + */ + dcpep_pipe_cfg_415, + dcpep_pipe_cfg_031, + dcpep_pipe_cfg_414, + dcpep_pipe_query_478, + dcpep_pipe_query_474, + dcpep_pipe_set_476, + dcpep_pipe_set_428, dcpep_num_methods }; #define IOMFB_METHOD(tag, name) [name] = { #name, { tag[0], tag[1], tag[2], tag[3] } } +#define IOMFB_METHOD_H17(tag, h17g, name) [name] = { #name, \ + { tag[0], tag[1], tag[2], tag[3] }, \ + { h17g[0], h17g[1], h17g[2], h17g[3] } } struct dcp_method_entry { const char *name; char tag[4]; + /* H17G shares the H17P transport but changes selected method IDs. */ + char tag_h17g[4]; }; #define IOMFB_MAX_CB (1000) @@ -196,12 +226,6 @@ struct dcp_map_buf_req { u8 dva_null; } __packed; -struct dcp_map_buf_resp { - u64 vaddr; - u64 dva; - u32 ret; -} __packed; - struct dcp_unmap_buf_resp { u64 buffer; u64 vaddr; @@ -242,20 +266,6 @@ struct dcp_map_physical_resp { u32 mem_desc_id; } __packed; -struct dcp_swap_start_req { - u32 swap_id; - struct dcp_iouserclient client; - u8 swap_id_null; - u8 client_null; - u8 padding[2]; -} __packed; - -struct dcp_swap_start_resp { - u32 swap_id; - struct dcp_iouserclient client; - u32 ret; -} __packed; - struct dcp_get_uint_prop_req { char obj[4]; char key[0x40]; @@ -284,13 +294,6 @@ struct iomfb_set_fx_prop_req { u32 value; } __packed; -struct dcp_set_power_state_req { - u64 unklong; - u8 unkbool; - u8 unkint_null; - u8 padding[2]; -} __packed; - struct dcp_set_power_state_resp { u32 unkint; u32 ret; diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c new file mode 100644 index 0000000000000..633f09c2be9d8 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#include "dcp-link.h" +#include "iomfb_h17p.h" +#include "iomfb_v12_3.h" +#include "iomfb_v13_3.h" +#include "iomfb_v14_7.h" +#include "version_utils.h" + +struct dcp_h17p_hotplug_request { + __le64 connected; + u8 tiled_display[0x4c]; + u8 tiled_display_null; + u8 padding[3]; +} __packed; + +struct dcp_h17p_swap_info_request { + u8 swap_info[0xe0]; + u8 swap_info_null; + u8 padding[3]; +} __packed; + +static_assert(sizeof(struct dcp_h17p_hotplug_request) == 0x58); +static_assert(sizeof(struct dcp_h17p_swap_info_request) == 0xe4); +/* D590 swap_complete_ap_gated is 0x730 bytes on the wire. */ +static_assert(sizeof(struct dc_swap_complete_resp_h17p) == 0x730); +static_assert(offsetof(struct dc_swap_complete_resp_h17p, swap_id) == 0); + +static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { + IOMFB_METHOD("A000", dcpep_late_init_signal), + IOMFB_METHOD_H17("A025", "A029", dcpep_setup_video_limits), /* (0,0) */ + IOMFB_METHOD("A131", iomfbep_a131_pmu_service_matched), /* nested in D206 */ + IOMFB_METHOD("A132", iomfbep_a132_backlight_service_matched), /* nested in D207 */ + IOMFB_METHOD("A385", dcpep_set_create_dfb), /* first call inside D121; (0,0) */ + IOMFB_METHOD("A386", iomfbep_a358_vi_set_temperature_hint), /* nested in D100 */ + IOMFB_METHOD("A401", dcpep_start_signal), + IOMFB_METHOD("A406", dcpep_swap_start), + /* + * swap_start moved A407 -> A406 on H17P, but swap_submit did not move + * with it: every swap is A406 followed by A408, A407 is unused, and the + * A408 response is 0x0c bytes (dcp_swap_submit_resp_h17p). + */ + IOMFB_METHOD("A408", dcpep_swap_submit), + IOMFB_METHOD("A411", dcpep_set_display_device), + IOMFB_METHOD("A412", dcpep_is_main_display), + IOMFB_METHOD("A413", dcpep_set_digital_out_mode), + IOMFB_METHOD("A423", iomfbep_set_matrix), + IOMFB_METHOD("A427", iomfbep_get_color_remap_mode), + IOMFB_METHOD("A442", dcpep_set_parameter_dcp), /* (0x28, 0x4) */ + IOMFB_METHOD("A446", dcpep_create_default_fb), /* follows A385 in D121 */ + /* + * The default-framebuffer block the firmware expects inside D121, + * straight after A446 and before A025. Sizes are (in, out). + */ + IOMFB_METHOD_H17("A035", "A039", dcpep_get_dfb_compression_info), /* (0x8, 0x8) */ + IOMFB_METHOD_H17("A034", "A038", dcpep_get_dfb_info), /* (0x2c, 0x2c) */ + IOMFB_METHOD("A103", dcpep_get_dfb_layout), /* (0xc, 0x8) */ + IOMFB_METHOD("A445", dcpep_get_dfb_state), /* (0x4, 0x8) */ + IOMFB_METHOD("A104", dcpep_set_dfb_dimensions), /* (0x8, 0) w,h */ + IOMFB_METHOD("A105", dcpep_commit_dfb_info), /* (0, 0) */ + IOMFB_METHOD_H17("A033", "A037", dcpep_dfb_query), /* (0, 0x4) */ + IOMFB_METHOD("A380", dcpep_dfb_ready_query), /* (0, 0x4) */ + IOMFB_METHOD("A415", dcpep_pipe_cfg_415), /* (0xc, 0xc) zeros */ + /* update_notify_clients_dcp */ + IOMFB_METHOD_H17("A031", "A035", dcpep_pipe_cfg_031), /* (0x6c, 0) */ + IOMFB_METHOD("A414", dcpep_pipe_cfg_414), /* (0x8, 0x8) zeros */ + IOMFB_METHOD("A478", dcpep_pipe_query_478), /* (0, 0x4) */ + IOMFB_METHOD("A474", dcpep_pipe_query_474), /* (0, 0x4) */ + IOMFB_METHOD("A476", dcpep_pipe_set_476), /* (0x4, 0) in 1 */ + IOMFB_METHOD("A428", dcpep_pipe_set_428), /* (0x4, 0x4) 0x10000 */ + IOMFB_METHOD("A450", dcpep_enable_disable_video_power_savings), /* in = u32 0 */ + IOMFB_METHOD("A457", dcpep_first_client_open), /* (0, 0) */ + /* D561 (displayMinRefreshInterval) arrives while A465 is in flight */ + IOMFB_METHOD("A465", dcpep_set_display_refresh_properties), + IOMFB_METHOD("A468", dcpep_flush_supports_power), /* follows A025 in D121, in = 1 */ + /* + * iomfbep_last_client_close and iomfbep_abort_swaps_dcp are deliberately + * absent: their H17P numbers are unverified, so the teardown paths skip + * them (DCP_HAS_CLIENT_TEARDOWN in iomfb_template.c). + */ + IOMFB_METHOD("A473", dcpep_set_power_state), /* out-of-band; out 0x8 */ + IOMFB_METHOD("A472", dcpep_register_dfb_surface), /* default FB surface, nested in D582 */ +}; + +#define DCP_FW h17p +#define DCP_FW_VER DCP_FW_VERSION(26, 6, 0) + +#include "iomfb_template.c" + +static bool trampoline_rt_bandwidth_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + /* + * The bandwidth scratch and doorbell come from the DCP's DT node; the + * apple,bw-doorbell binding has no offset cell. + */ + u64 scratch = dcp->disp_bw_scratch_res.start + + dcp->disp_bw_scratch_offset; + u64 clock_request = dcp->disp_bw_doorbell_res.start; + + trace_iomfb_callback(dcp, tag, __func__); + if (apple_dcp_h17p_rt_bw_encode_reply(in, out, scratch, clock_request)) + dev_warn(dcp->dev, "D003 bandwidth request is malformed\n"); + return true; +} + +static bool trampoline_get_frequency_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct apple_dcp_h17p_clock_request *request = in; + u64 rate; + int ret; + + trace_iomfb_callback(dcp, tag, __func__); + ret = apple_dcp_h17p_clock_rate(request, clk_get_rate(dcp->clk), + clk_get_rate(dcp->clk_194), &rate); + if (ret) { + dev_warn(dcp->dev, "unknown display clock ID %#x\n", + le32_to_cpu(request->clock_id)); + rate = 0; + } + + *(__le64 *)out = cpu_to_le64(rate); + return true; +} + +static bool trampoline_hotplug_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_hotplug_request *request = in; + + if (!(request->tiled_display_null & 1)) + memcpy(out, request->tiled_display, + sizeof(request->tiled_display)); + return trampoline_hotplug(dcp, tag, out, in); +} + +static bool trampoline_swap_info_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_swap_info_request *request = in; + + trace_iomfb_callback(dcp, tag, __func__); + if (!(request->swap_info_null & 1)) + memcpy(out, request->swap_info, sizeof(request->swap_info)); + return true; +} + +/* + * D400 (get_property) is left unhandled: the zeroed reply means "no such + * property". The firmware then runs without a panel power model, as it does + * for its other optional power properties. + */ + +/* + * H17P callback numbering. It is not a uniform shift of the v13.5 table: the + * service-creation block is D108..D113 exactly as on v13.5, and the swap + * completion is D590. A callback with no entry here is answered with a + * zeroed reply by dcpep_handle_cb(). + */ +static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { + [0] = dcpep_cb_d000_h17p, /* acked after a nested A033 */ + [1] = trampoline_true, + [2] = trampoline_nop, + [3] = trampoline_rt_bandwidth_h17p, + [6] = trampoline_set_frame_sync_props_h17p, /* 0x54/0x50, identity scale */ + [100] = iomfbep_cb_match_pmu_service, /* match_pmu_service */ + [101] = trampoline_zero, + [102] = trampoline_nop, /* set_number_property */ + [104] = trampoline_nop, /* set_boolean_property */ + [107] = trampoline_nop, + [108] = trampoline_true, /* create_provider_service */ + [109] = trampoline_true, /* create_product_service */ + [110] = trampoline_true, /* create_PMU_service */ + [111] = trampoline_true, /* create_iomfb_service */ + [112] = trampoline_create_backlight_service, /* create_backlight_service */ + [113] = trampoline_true, /* create_nvram_service */ + [114] = trampoline_zero, /* CoreAnalyticsSendEvent */ + [117] = trampoline_nop, /* set_idle_caching_state_ap */ + [118] = trampoline_zero, /* upload_trace_start */ + [119] = trampoline_zero, /* upload_trace_chunk */ + [120] = trampoline_zero, /* upload_trace_end */ + [121] = dcpep_cb_boot_1, /* start_hardware_boot */ + [122] = trampoline_false, /* is_dark_boot */ + [123] = trampoline_false, /* is_waking_from_hibernate */ + [124] = trampoline_zero, /* detect_fastsim */ + [125] = trampoline_read_edt_data, /* read_edt_data */ + [127] = trampoline_prop_start, /* setDCPAVPropStart */ + [128] = trampoline_prop_chunk, /* setDCPAVPropChunk */ + [129] = trampoline_prop_end, /* setDCPAVPropEnd */ + [130] = trampoline_allocate_bandwidth, /* allocate_bandwidth */ + [201] = trampoline_map_piodma, /* map_buf */ + [202] = trampoline_unmap_piodma, /* unmap_buf */ + [206] = iomfbep_cb_match_pmu_service_2, /* match_pmu_service */ + [207] = iomfbep_cb_match_backlight_service, /* match_backlight_service */ + [208] = trampoline_nop, /* update_backlight_factor_prop */ + [209] = trampoline_get_time, /* get_calendar_time_ms */ + [300] = trampoline_pr_publish, + [401] = trampoline_get_uint_prop, /* get_uint_prop */ + [404] = trampoline_nop, /* set_uint_prop */ + [406] = trampoline_set_fx_prop, /* set_fx_prop */ + [408] = trampoline_get_frequency_h17p, /* getClockFrequency */ + [411] = trampoline_map_reg, /* mapDeviceMemoryWithIndex */ + [413] = trampoline_true, /* setProperty */ + [414] = trampoline_sr_set_property_int, /* setProperty */ + [415] = trampoline_true, /* setProperty */ + [451] = trampoline_allocate_buffer, /* allocate_buffer */ + [452] = trampoline_map_physical, /* prepare */ + [454] = trampoline_release_mem_desc, /* release_descriptor */ + [552] = trampoline_true, + [561] = trampoline_true, + [563] = trampoline_true, + [565] = trampoline_true, + [567] = trampoline_true, + [572] = trampoline_zero, /* powerUpDART */ + [575] = trampoline_hotplug_h17p, + [576] = trampoline_nop, + [582] = iomfbep_cb_create_dfb_surface, /* create_default_fb_surface */ + [583] = trampoline_nop, /* clear_default_surface */ + [584] = trampoline_nop, /* swap_notify_gated */ + [585] = trampoline_swap_info_h17p, /* swap_info_notify_dispatch */ + [590] = trampoline_swap_complete, /* swap_complete_ap_gated */ + [592] = trampoline_swap_complete_intent_gated, /* swap_complete_intent_gated */ + [593] = trampoline_abort_swap_ap_gated, /* abort_swap_ap_gated */ + [594] = trampoline_enable_backlight_message_ap_gated, + [595] = trampoline_nop, /* setSystemConsoleMode */ + [597] = trampoline_false, /* isDFBAllocated */ + [598] = trampoline_false, + [599] = trampoline_nop, /* find_swap_function_gated */ +}; + +void iomfb_start_h17p(struct apple_dcp *dcp) +{ + dcp->cb_handlers = cb_handlers; + dcp_start_signal(dcp, false, dcp_started, NULL); +} + +#undef DCP_FW_VER +#undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h new file mode 100644 index 0000000000000..97c0da9e05b94 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright The Asahi Linux Contributors */ + +#ifndef __APPLE_IOMFB_H17P_H__ +#define __APPLE_IOMFB_H17P_H__ + +#include "version_utils.h" + +#define DCP_FW h17p +#define DCP_FW_VER DCP_FW_VERSION(26, 6, 0) + +#include "iomfb_template.h" + +static_assert(sizeof(struct dcp_swap_h17p) == 0x58a); +static_assert(sizeof(struct dcp_surface_h17p) == 0x22c); +static_assert(sizeof(struct dcp_swap_submit_req_h17p) == 0xe9c); +static_assert(sizeof(struct dcp_swap_submit_resp_h17p) == 0x0c); + +#undef DCP_FW_VER +#undef DCP_FW + +#endif /* __APPLE_IOMFB_H17P_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 4d5d09607f710..5e4f2099c368f 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -64,13 +64,50 @@ DCP_THUNK_INOUT(dcp_swap_submit, dcpep_swap_submit, struct DCP_FW_NAME(dcp_swap_submit_req), struct DCP_FW_NAME(dcp_swap_submit_resp)); -DCP_THUNK_INOUT(dcp_swap_start, dcpep_swap_start, struct dcp_swap_start_req, - struct dcp_swap_start_resp); +DCP_THUNK_INOUT(dcp_swap_start, dcpep_swap_start, struct DCP_FW_NAME(dcp_swap_start_req), + struct DCP_FW_NAME(dcp_swap_start_resp)); DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, - struct dcp_set_power_state_req, + struct DCP_FW_NAME(dcp_set_power_state_req), struct dcp_set_power_state_resp); +/* + * H17P firmware expects set_power_state (A473) on the out-of-band command + * channel. Sent on the ordinary command channel it is never answered, which + * strands the power transition and every later call behind it. + */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +#define DCP_POWER_OOB true +#else +#define DCP_POWER_OOB false +#endif + +/* + * H17P firmware is attached to a display the bootloader has already brought + * up, and it does not accept set_digital_out_mode (A413) for that display: + * the call fails (0x8000000b) and the timing generator never starts, so there + * is no vblank and no swap completion. Keep the mode the bootloader + * programmed, which is the panel's native mode, instead of requesting one. + */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +#define DCP_INHERIT_BOOT_MODE true +#else +#define DCP_INHERIT_BOOT_MODE false +#endif + +/* + * abort_swaps_dcp and last_client_close have no verified H17P method numbers, + * and the H17P table leaves them out. Rather than send the firmware a record + * it would decode as some other call, power-off and sleep skip both and go + * straight to set_power_state, whose number and sizes are known. Power-off + * still sends its clear swap first. + */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +#define DCP_HAS_CLIENT_TEARDOWN false +#else +#define DCP_HAS_CLIENT_TEARDOWN true +#endif + DCP_THUNK_INOUT(dcp_set_digital_out_mode, dcpep_set_digital_out_mode, struct dcp_set_digital_out_mode_req, u32); @@ -89,6 +126,68 @@ DCP_THUNK_OUT(dcp_create_default_fb, dcpep_create_default_fb, u32); DCP_THUNK_OUT(dcp_start_signal, dcpep_start_signal, u32); DCP_THUNK_VOID(dcp_setup_video_limits, dcpep_setup_video_limits); DCP_THUNK_VOID(dcp_set_create_dfb, dcpep_set_create_dfb); + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * Default-framebuffer queries issued from inside start_hardware_boot (D121). + * The getters take a zeroed input of the expected size; the firmware only + * checks the size. + */ +struct dcp_dfb_info_h17p { + u8 data[0x2c]; +} __packed; + +struct dcp_dfb_layout_req_h17p { + u8 data[0xc]; +} __packed; + +struct dcp_dfb_dimensions_h17p { + u32 width; + u32 height; +} __packed; + +DCP_THUNK_INOUT(dcp_get_dfb_compression_info, dcpep_get_dfb_compression_info, + u64, u64); +DCP_THUNK_INOUT(dcp_get_dfb_info, dcpep_get_dfb_info, + struct dcp_dfb_info_h17p, struct dcp_dfb_info_h17p); +DCP_THUNK_INOUT(dcp_get_dfb_layout, dcpep_get_dfb_layout, + struct dcp_dfb_layout_req_h17p, u64); +DCP_THUNK_INOUT(dcp_get_dfb_state, dcpep_get_dfb_state, u32, u64); +DCP_THUNK_IN(dcp_set_dfb_dimensions, dcpep_set_dfb_dimensions, + struct dcp_dfb_dimensions_h17p); +DCP_THUNK_VOID(dcp_commit_dfb_info, dcpep_commit_dfb_info); +DCP_THUNK_OUT(dcp_dfb_query, dcpep_dfb_query, u32); +DCP_THUNK_OUT(dcp_dfb_ready_query, dcpep_dfb_ready_query, u32); + +/* + * Pipe-configuration calls the firmware expects between the boot chain and + * the first swap. Most of them have no known name; they are identified by + * method number and wire size. + */ +struct dcp_pipe_cfg_415 { + u8 data[0xc]; +} __packed; + +/* + * A031 is update_notify_clients_dcp(u32[27]): an array of the DCP + * notification clients the AP wants delivered. Clients that are not + * registered here never send the callbacks they gate. + */ +struct dcp_pipe_cfg_031 { + __le32 clients[27]; +} __packed; +static_assert(sizeof(struct dcp_pipe_cfg_031) == 0x6c); + +DCP_THUNK_INOUT(dcp_pipe_cfg_415, dcpep_pipe_cfg_415, + struct dcp_pipe_cfg_415, struct dcp_pipe_cfg_415); +DCP_THUNK_IN(dcp_pipe_cfg_031, dcpep_pipe_cfg_031, struct dcp_pipe_cfg_031); +DCP_THUNK_INOUT(dcp_pipe_cfg_414, dcpep_pipe_cfg_414, u64, u64); +DCP_THUNK_OUT(dcp_pipe_query_478, dcpep_pipe_query_478, u32); +DCP_THUNK_OUT(dcp_pipe_query_474, dcpep_pipe_query_474, u32); +DCP_THUNK_IN(dcp_pipe_set_476, dcpep_pipe_set_476, u32); +DCP_THUNK_INOUT(dcp_pipe_set_428, dcpep_pipe_set_428, u32, u32); +#endif + DCP_THUNK_VOID(dcp_first_client_open, dcpep_first_client_open); DCP_THUNK_INOUT(dcp_set_parameter_dcp, dcpep_set_parameter_dcp, @@ -120,6 +219,96 @@ static u32 dcpep_cb_zero(struct apple_dcp *dcp) return 0; } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * Bind every framebuffer displaced by the commit that produced this swap to + * that swap's id. + * + * The id only exists once the firmware answers swap_start, after the atomic + * commit has queued its entries, so they are queued unarmed and stamped here. + * Swap-start requests are serialized by the DCP command stack. + */ +static void dcp_arm_swapped_out_fbs(struct apple_dcp *dcp, u32 swap_id) +{ + struct dcp_fb_reference *entry; + unsigned long flags; + + spin_lock_irqsave(&dcp->swapped_out_lock, flags); + list_for_each_entry(entry, &dcp->swapped_out_fbs, head) { + if (entry->armed) + continue; + entry->swap_id = swap_id; + entry->armed = true; + } + spin_unlock_irqrestore(&dcp->swapped_out_lock, flags); +} + +/* + * Drop framebuffers whose unbinding swap has completed. + * + * Entries must be keyed to the swap that unbinds them, rather than the last + * swap that happened to have completed when the atomic commit ran. A later + * successful completion also retires entries whose own completion was lost; + * the signed difference keeps that comparison valid across a 32-bit wrap. + */ +static void dcp_release_swapped_out_fbs(struct apple_dcp *dcp) +{ + struct dcp_fb_reference *entry, *tmp; + unsigned long flags; + LIST_HEAD(done); + + /* + * last_swap_id / have_swap_complete are only ever touched by the RTKit + * worker, which is also the only caller of this function, so the lock + * is here for the list alone. + */ + spin_lock_irqsave(&dcp->swapped_out_lock, flags); + while (!list_empty(&dcp->swapped_out_fbs)) { + entry = list_first_entry(&dcp->swapped_out_fbs, + struct dcp_fb_reference, head); + + /* Newest entries; their swap has not been started yet. */ + if (!entry->armed) + break; + /* last_swap_id is 0 from kzalloc until a swap really lands. */ + if (!dcp->have_swap_complete) + break; + /* Hold the old framebuffer for one additional completed swap. */ + if ((s32)(dcp->last_swap_id - entry->swap_id) <= 0) + break; + + list_move_tail(&entry->head, &done); + } + spin_unlock_irqrestore(&dcp->swapped_out_lock, flags); + + /* drm_framebuffer_put() can sleep, so never call it under the lock. */ + list_for_each_entry_safe(entry, tmp, &done, head) { + if (entry->fb) + drm_framebuffer_put(entry->fb); + list_del(&entry->head); + kfree(entry); + } +} +#else +static void dcp_arm_swapped_out_fbs(struct apple_dcp *dcp, u32 swap_id) { } + +static void dcp_release_swapped_out_fbs(struct apple_dcp *dcp) +{ + while (!list_empty(&dcp->swapped_out_fbs)) { + struct dcp_fb_reference *entry; + + entry = list_first_entry(&dcp->swapped_out_fbs, + struct dcp_fb_reference, head); + if (entry->swap_id == dcp->last_swap_id) + break; + if (entry->fb) + drm_framebuffer_put(entry->fb); + list_del(&entry->head); + kfree(entry); + } +} +#endif + static void dcpep_cb_swap_complete(struct apple_dcp *dcp, struct DCP_FW_NAME(dc_swap_complete_resp) *resp) { @@ -128,6 +317,10 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, dcp->last_swap_id = resp->swap_id; dcp_drm_crtc_page_flip(dcp, now); +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + dcp->have_swap_complete = true; + dcp_release_swapped_out_fbs(dcp); +#endif if (dcp->crc_enabled) { u32 crc32 = 0; drm_crtc_add_crc_entry(&dcp->crtc->base, true, resp->swap_id, &crc32); @@ -152,6 +345,94 @@ static bool iomfbep_cb_match_pmu_service(struct apple_dcp *dcp, int tag, void *o return false; } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * H17P asks the AP to make the default framebuffer surface (D582). The + * firmware expects the AP to register a surface from inside that callback, + * with a nested A472 carrying a 0x22c-byte surface descriptor, and only then + * to report success. Without it the firmware has no surface, programs the + * display engine from a null base and the piodma DART faults. + */ +struct dcp_h17p_dfb_surface_req { + u8 surface[0x22c]; + u8 surface_null; + u8 padding[3]; +} __packed; +static_assert(sizeof(struct dcp_h17p_dfb_surface_req) == 0x230); + +/* + * A472 has a 4-byte output; with out_len 0 the firmware has nowhere to write + * its result and stops answering. + */ +DCP_THUNK_INOUT(dcp_register_dfb_surface, dcpep_register_dfb_surface, + struct dcp_h17p_dfb_surface_req, u32); + +static void dcp_ack_dfb_surface(struct apple_dcp *dcp) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + + *succ = true; + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +/* + * Fill the default surface descriptor. Fields not set here are zero. The + * format at +0x0b is the fourcc 'b3a8', stored byte-reversed; +0x35 is the + * default surface ID. The bytes at +0x29..+0x2c are an opaque word whose + * meaning is not known. + */ +static void dcp_fill_dfb_surface(struct apple_dcp *dcp, u8 *s) +{ + s[0x002] = 0x01; + memcpy(s + 0x00b, "8a3b", 4); + s[0x019] = 0x01; + s[0x01b] = 0x01; + s[0x01c] = 0x01; + s[0x02b] = 0x4d; + s[0x02c] = 0x01; + s[0x035] = 0x02; + s[0x051] = 0x01; + s[0x149] = 0x01; +} + +/* + * A471 (update_dfb) is never sent: on H17P it makes the coprocessor take an + * AXI read error. Scanout does not need it, since every swap carries its own + * surface. Just acknowledge the callback. + */ +static void complete_dfb_surface(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_ack_dfb_surface(dcp); +} + +static bool iomfbep_cb_create_dfb_surface(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + struct dcp_h17p_dfb_surface_req *req; + + trace_iomfb_callback(dcp, tag, __func__); + + /* + * The request is too large for the stack this deep in the RTKit + * receive work (rx work -> iomfb_recv_msg -> dcpep_handle_cb -> here -> + * dcp_push). dcp_push() copies the payload into shared memory before + * returning, so a short-lived allocation is enough. + */ + req = kzalloc_obj(*req); + if (!req) + return true; + + dcp_fill_dfb_surface(dcp, req->surface); + + dcp_register_dfb_surface(dcp, false, req, complete_dfb_surface, NULL); + kfree(req); + + /* deferred ACK: the A472 completion reports success for us */ + return false; +} +#endif + static void complete_pmu_service_matched(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_channel *ch = &dcp->ch_cb; @@ -266,7 +547,7 @@ static void iomfbep_cb_set_fx_prop(struct apple_dcp *dcp, struct iomfb_set_fx_pr * PIODMA is separate from the main DCP and uses own IOVA space on a dedicated * stream of the display DART, rather than the expected DCP DART. */ -static struct dcp_map_buf_resp dcpep_cb_map_piodma(struct apple_dcp *dcp, +static struct DCP_FW_NAME(dcp_map_buf_resp) dcpep_cb_map_piodma(struct apple_dcp *dcp, struct dcp_map_buf_req *req) { struct dcp_mem_descriptor *memdesc; @@ -294,12 +575,30 @@ static struct dcp_map_buf_resp dcpep_cb_map_piodma(struct apple_dcp *dcp, goto reject; } - return (struct dcp_map_buf_resp){ .dva = memdesc->dva }; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* remembered so release_mem_desc() can tear a stale mapping down */ + memdesc->piodma_mapped = true; + + /* the H17P reply echoes the buffer index */ + return (struct DCP_FW_NAME(dcp_map_buf_resp)){ + .buffer = req->buffer, + .dva = memdesc->dva, + }; +#else + return (struct DCP_FW_NAME(dcp_map_buf_resp)){ .dva = memdesc->dva }; +#endif reject: dev_err(dcp->dev, "denying map of invalid buffer %llx for piodma\n", req->buffer); - return (struct dcp_map_buf_resp){ .ret = EINVAL }; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + return (struct DCP_FW_NAME(dcp_map_buf_resp)){ + .buffer = req->buffer, + .dva = 0, + }; +#else + return (struct DCP_FW_NAME(dcp_map_buf_resp)){ .ret = EINVAL }; +#endif } static void dcpep_cb_unmap_piodma(struct apple_dcp *dcp, @@ -334,6 +633,7 @@ static void dcpep_cb_unmap_piodma(struct apple_dcp *dcp, * full pages */ iommu_unmap(dcp->iommu_dom, memdesc->dva, ALIGN(memdesc->size, SZ_16K)); + memdesc->piodma_mapped = false; } /* @@ -365,6 +665,7 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, memdesc = &dcp->memdesc[id]; + memdesc->piodma_mapped = false; memdesc->size = resp.dva_size; /* HACK: align size to 16K since the iommu API only maps full pages */ size = ALIGN(resp.dva_size, SZ_16K); @@ -375,6 +676,16 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, size); resp.dva = memdesc->dva; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P firmware also needs the physical address: it loads the APT M3 + * microcode through it, and with a zero paddr the timing sequencer + * comes up and is powered straight back down. + */ + if (memdesc->map.sgl) + resp.paddr = sg_phys(memdesc->map.sgl); +#endif + return resp; } @@ -398,6 +709,23 @@ static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) memdesc = &dcp->memdesc[id]; size = ALIGN(memdesc->size, SZ_16K); + + /* + * Tear down any translation still standing before the pages go back to + * the allocator. The firmware is expected to unmap piodma first, but + * if it releases a descriptor without doing so we would hand the pages + * back while the DART still resolves them -- the display engine would + * then read whoever owns them next. Only the H17P map_piodma handler + * sets piodma_mapped, so older firmware is unaffected. + */ + if (memdesc->piodma_mapped) { + dev_warn(dcp->dev, + "releasing buffer %u while still mapped for piodma; unmapping first\n", + id); + iommu_unmap(dcp->iommu_dom, memdesc->dva, size); + memdesc->piodma_mapped = false; + } + if (memdesc->buf) { dma_free_coherent(dcp->dev, size, memdesc->buf, memdesc->dva); memdesc->buf = NULL; @@ -475,7 +803,15 @@ static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *d return (struct DCP_FW_NAME(dcp_map_reg_resp)){ .ret = 1 }; } else { struct resource *rsrc = dcp->disp_registers[req->index]; -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P reaches these apertures physically, not through its + * DART, and expects dva == addr in the reply. Trying to + * dma_map_resource() them instead fails and hands the firmware + * DMA_MAPPING_ERROR as a device address. + */ + dma_addr_t dva = rsrc->start; +#elif DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) dma_addr_t dva = dma_map_resource(dcp->dev, rsrc->start, resource_size(rsrc), DMA_BIDIRECTIONAL, 0); WARN_ON(dva == DMA_MAPPING_ERROR); @@ -509,7 +845,9 @@ static void iomfbep_cb_enable_backlight_message_ap_gated(struct apple_dcp *dcp, * syslog: "[BrightnessLCD.cpp:743][AFK]nitsToDBV: iDAC out of range" */ dcp->brightness.update = true; - schedule_work(&dcp->bl_update_wq); + /* Backlight handling is only set up for panels described in the DT. */ + if (dcp_has_panel(dcp)) + schedule_work(&dcp->bl_update_wq); } /* Chunked data transfer for property dictionaries */ @@ -520,6 +858,19 @@ static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) return false; } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * This length comes straight off the wire. A bogus value would turn + * the allocation below into a multi-gigabyte GFP_KERNEL request. + * Property blobs are a few KiB, so bound it. + */ + if (*length == 0 || *length > SZ_1M) { + dev_warn(dcp->dev, "rejecting %u byte property transfer\n", + *length); + return false; + } +#endif + dcp->chunks.length = *length; dcp->chunks.data = kzalloc(*length, GFP_KERNEL); @@ -623,12 +974,22 @@ static void boot_done(struct apple_dcp *dcp, void *out, void *cookie) dcp_ack(dcp, DCP_CONTEXT_CB); } -static void boot_5(struct apple_dcp *dcp, void *out, void *cookie) +static void boot_5b(struct apple_dcp *dcp, void *out, void *cookie) { dcp_set_display_refresh_properties(dcp, false, boot_done, NULL); } -static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) +static void boot_5(struct apple_dcp *dcp, void *out, void *cookie) +{ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* H17P expects A380 right before set_display_refresh_properties */ + dcp_dfb_ready_query(dcp, false, boot_5b, NULL); +#else + boot_5b(dcp, out, cookie); +#endif +} + +static void boot_4b(struct apple_dcp *dcp, void *out, void *cookie) { #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) u32 v_true = 1; @@ -638,6 +999,16 @@ static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) #endif } +static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) +{ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* H17P expects A033 between flush_supports_power and late_init_signal */ + dcp_dfb_query(dcp, false, boot_4b, NULL); +#else + boot_4b(dcp, out, cookie); +#endif +} + static void boot_3(struct apple_dcp *dcp, void *out, void *cookie) { u32 v_true = true; @@ -650,11 +1021,103 @@ static void boot_2(struct apple_dcp *dcp, void *out, void *cookie) dcp_setup_video_limits(dcp, false, boot_3, NULL); } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * H17P expects the default framebuffer to be described between + * create_default_fb and setup_video_limits: + * get_dfb_compression_info -> get_dfb_info -> get_dfb_layout -> + * get_dfb_state -> set_dfb_dimensions(w, h) -> commit_dfb_info + * Without it the firmware never finishes its own default framebuffer setup. + */ +static void boot_1_11(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_commit_dfb_info(dcp, false, boot_2, NULL); +} + +static void boot_1_10(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_dfb_dimensions_h17p dim; + + /* + * get_dfb_state returns the default framebuffer dimensions, which are + * passed back unchanged to set_dfb_dimensions. + */ + memcpy(&dim, out, sizeof(dim)); + if (!dim.width || !dim.height || dim.width > 16384 || dim.height > 16384) { + struct dcp_channel *ch = &dcp->ch_cb; + + dev_err(dcp->dev, "invalid default framebuffer dimensions %ux%u\n", + dim.width, dim.height); + *(u8 *)ch->output[ch->depth - 1] = false; + dcp_ack(dcp, DCP_CONTEXT_CB); + return; + } + dev_dbg(dcp->dev, "default framebuffer dimensions %ux%u\n", + dim.width, dim.height); + + dcp_set_dfb_dimensions(dcp, false, &dim, boot_1_11, NULL); +} + +static void boot_1_9(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 zero = 0; + + dcp_get_dfb_state(dcp, false, &zero, boot_1_10, NULL); +} + +static void boot_1_8(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_dfb_layout_req_h17p req = {}; + + dcp_get_dfb_layout(dcp, false, &req, boot_1_9, NULL); +} + +static void boot_1_7(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_dfb_info_h17p info = {}; + + dcp_get_dfb_info(dcp, false, &info, boot_1_8, NULL); +} + +static void boot_1_6(struct apple_dcp *dcp, void *out, void *cookie) +{ + u64 zero = 0; + + dcp_get_dfb_compression_info(dcp, false, &zero, boot_1_7, NULL); +} +#endif + static void boot_1_5(struct apple_dcp *dcp, void *out, void *cookie) { +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + dcp_create_default_fb(dcp, false, boot_1_6, NULL); +#else dcp_create_default_fb(dcp, false, boot_2, NULL); +#endif } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +static void d000_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + + *succ = true; + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +/* + * The firmware expects D000 to be answered only after a nested A033 + * (dfb_query), and then with 1. Defer the ACK until that call completes. + */ +static bool dcpep_cb_d000_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + dcp_dfb_query(dcp, false, d000_done, NULL); + return false; +} +#endif + /* Use special function signature to defer the ACK */ static bool dcpep_cb_boot_1(struct apple_dcp *dcp, int tag, void *out, void *in) { @@ -702,6 +1165,46 @@ static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) return rt_bw; } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * H17P's D006 is 0x54 in / 0x50 out, far wider than the 28-byte + * frame_sync_props of older firmware, so the fields that matter sit past + * anything the old struct could describe. + * + * The firmware expects zeros except for three u32 at 0x24, 0x28 and 0x2c, + * each 0x10000 -- 1.0 in 16.16 fixed point, the identity scale. An all-zero + * reply hands the pipe a zero scale, and it then programs a null surface + * base: the display DART faults at DVA 0 and the coprocessor takes an AXI + * error. + */ +#define DCP_FRAME_SYNC_IDENTITY 0x10000 + +struct dcp_set_frame_sync_props_req_h17p { + u8 unk[0x54]; +} __packed; + +struct dcp_set_frame_sync_props_resp_h17p { + u8 unk[0x50]; +} __packed; + +static struct dcp_set_frame_sync_props_resp_h17p +dcpep_cb_set_frame_sync_props_h17p(struct apple_dcp *dcp, + struct dcp_set_frame_sync_props_req_h17p *req) +{ + struct dcp_set_frame_sync_props_resp_h17p resp = {}; + static const u16 identity_at[] = { 0x24, 0x28, 0x2c }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(identity_at); i++) { + u32 v = DCP_FRAME_SYNC_IDENTITY; + + memcpy(&resp.unk[identity_at[i]], &v, sizeof(v)); + } + + return resp; +} +#endif + static struct dcp_set_frame_sync_props_resp dcpep_cb_set_frame_sync_props(struct apple_dcp *dcp, struct dcp_set_frame_sync_props_req *req) @@ -729,6 +1232,71 @@ static void release_swap_cookie(struct kref *ref) kfree(cookie); } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +#define DCP_SWAP_FLAGS1 0x34865202ULL +#define DCP_SWAP_FLAGS2 0x4ULL + +/* + * H17P's swap surface has no is_tearing_allowed and no is_premultiplied: + * plane_cnt sits at +1, plane_cnt2 at +5 and format at +9, two bytes earlier + * than on every other firmware. With the older layout the firmware reads the + * format two bytes late and its surface check rejects the swap as an + * unsupported format. + * + * Shift each surface's tail down by two rather than fork the shared struct. + */ +static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) +{ + struct DCP_FW_NAME(dcp_swap_submit_req) *req = &DCP_FW_UNION(dcp->swap); + unsigned int i; + + /* + * The firmware expects flags1 = 0x34865202 and flags2 = 4 (at +0x40 + * and +0x48) in every swap. + */ + req->swap.flags1 = DCP_SWAP_FLAGS1; + req->swap.flags2 = DCP_SWAP_FLAGS2; + + /* + * The H17P gap before swap_id carries two small counters the firmware + * expects to be 2 (at +0x50) and 3 (at +0x78). The gap starts at + * +0x50, so these are indices 0 and 0x28 of the blob. + */ + req->swap.h17p_pre_swap_id[0x00] = 2; + req->swap.h17p_pre_swap_id[0x28] = 3; + + for (i = 0; i < SWAP_SURFACES; i++) { + u8 *p = (u8 *)&req->surf[i]; + size_t n = sizeof(req->surf[i]); + + memmove(p + 1, p + 3, n - 3); + memset(p + n - 2, 0, 2); + } + + /* + * Everything from surf_iova onward also sits two bytes earlier on + * H17P: surf_iova[0] is at +0xe38 of the 0xe9c-byte record, not + * +0xe3a. Left in place, the firmware reads every IOVA two bytes off + * and the trailing null-flag block misaligned, so it faults on a bogus + * address or verifies layers that are not present. Shift the whole + * block down. + */ + { + u8 *r = (u8 *)req; + size_t n = sizeof(*req); + size_t off = offsetof(typeof(*req), surf_iova); + + static_assert(offsetof(typeof(*req), surf_iova) == 0xe3a, + "H17P source IOVA offset"); + static_assert(sizeof(*req) == 0xe9c, "H17P request size"); + memmove(r + off - 2, r + off, n - off); + memset(r + n - 2, 0, 2); + } +} +#else +static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) { } +#endif + static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; @@ -745,30 +1313,22 @@ static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) return; } - while (!list_empty(&dcp->swapped_out_fbs)) { - struct dcp_fb_reference *entry; - entry = list_first_entry(&dcp->swapped_out_fbs, - struct dcp_fb_reference, head); - if (entry->swap_id == dcp->last_swap_id) - break; - if (entry->fb) - drm_framebuffer_put(entry->fb); - list_del(&entry->head); - kfree(entry); - } + dcp_release_swapped_out_fbs(dcp); } static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, void *cookie) { - struct dcp_swap_start_resp *resp = data; + struct DCP_FW_NAME(dcp_swap_start_resp) *resp = data; DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; + dcp_arm_swapped_out_fbs(dcp, resp->swap_id); if (cookie) { struct dcp_swap_cookie *info = cookie; info->swap_id = resp->swap_id; } + dcp_h17p_fix_swap_surfaces(dcp); dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swap_cleared, cookie); } @@ -784,11 +1344,11 @@ static void dcp_on_final(struct apple_dcp *dcp, void *out, void *cookie) static void dcp_on_set_power_state(struct apple_dcp *dcp, void *out, void *cookie) { - struct dcp_set_power_state_req req = { + struct DCP_FW_NAME(dcp_set_power_state_req) req = { .unklong = 1, }; - dcp_set_power_state(dcp, false, &req, dcp_on_final, cookie); + dcp_set_power_state(dcp, DCP_POWER_OOB, &req, dcp_on_final, cookie); } static void dcp_on_set_parameter(struct apple_dcp *dcp, void *out, void *cookie) @@ -823,9 +1383,19 @@ void DCP_FW_NAME(iomfb_poweron)(struct apple_dcp *dcp) kref_get(&cookie->refcount); if (dcp->main_display) { +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P takes set_display_device (A411) once, in the + * client-open sequence. Repeating it after the client is open + * is a call the firmware stops answering, which wedges the + * command channel and with it every later swap. + */ + dcp_on_set_power_state(dcp, NULL, cookie); +#else handle = 0; dcp_set_display_device(dcp, false, &handle, dcp_on_set_power_state, cookie); +#endif } else { handle = 2; dcp_set_display_device(dcp, false, &handle, @@ -866,10 +1436,10 @@ static void complete_set_powerstate(struct apple_dcp *dcp, void *out, static void last_client_closed_poff(struct apple_dcp *dcp, void *out, void *cookie) { - struct dcp_set_power_state_req power_req = { + struct DCP_FW_NAME(dcp_set_power_state_req) power_req = { .unklong = 0, }; - dcp_set_power_state(dcp, false, &power_req, complete_set_powerstate, + dcp_set_power_state(dcp, DCP_POWER_OOB, &power_req, complete_set_powerstate, cookie); } @@ -895,7 +1465,7 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) }; struct dcp_swap_cookie *cookie; struct dcp_wait_cookie *poff_cookie; - struct dcp_swap_start_req swap_req = { 0 }; + struct DCP_FW_NAME(dcp_swap_start_req) swap_req = { 0 }; struct DCP_FW_NAME(dcp_swap_submit_req) *swap = &DCP_FW_UNION(dcp->swap); cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); @@ -934,8 +1504,10 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) for (int l = 0; l < SWAP_SURFACES; l++) swap->surf_null[l] = true; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) +#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) for (int l = 0; l < 5; l++) swap->surf2_null[l] = true; +#endif swap->unkU32Ptr_null = true; swap->unkU32out_null = true; #endif @@ -960,8 +1532,14 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) /* increase refcount to ensure the receiver has a reference */ kref_get(&poff_cookie->refcount); - iomfb_abort_swaps_dcp(dcp, false, &abort_req, - aborted_swaps_dcp_poff, poff_cookie); + if (DCP_HAS_CLIENT_TEARDOWN) { + iomfb_abort_swaps_dcp(dcp, false, &abort_req, + aborted_swaps_dcp_poff, poff_cookie); + } else { + dev_dbg_once(dcp->dev, + "power-off: skipping abort_swaps/last_client_close\n"); + last_client_closed_poff(dcp, NULL, poff_cookie); + } ret = wait_for_completion_timeout(&poff_cookie->done, msecs_to_jiffies(1000)); @@ -979,10 +1557,10 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) static void last_client_closed_sleep(struct apple_dcp *dcp, void *out, void *cookie) { - struct dcp_set_power_state_req power_req = { + struct DCP_FW_NAME(dcp_set_power_state_req) power_req = { .unklong = 0, }; - dcp_set_power_state(dcp, false, &power_req, complete_set_powerstate, cookie); + dcp_set_power_state(dcp, DCP_POWER_OOB, &power_req, complete_set_powerstate, cookie); } static void aborted_swaps_dcp_sleep(struct apple_dcp *dcp, void *out, void *cookie) @@ -1010,8 +1588,14 @@ void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp) /* increase refcount to ensure the receiver has a reference */ kref_get(&cookie->refcount); - iomfb_abort_swaps_dcp(dcp, false, &req, aborted_swaps_dcp_sleep, - cookie); + if (DCP_HAS_CLIENT_TEARDOWN) { + iomfb_abort_swaps_dcp(dcp, false, &req, aborted_swaps_dcp_sleep, + cookie); + } else { + dev_dbg_once(dcp->dev, + "sleep: skipping abort_swaps/last_client_close\n"); + last_client_closed_sleep(dcp, NULL, cookie); + } ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(1000)); @@ -1107,7 +1691,19 @@ dcpep_cb_get_tiling_state(struct apple_dcp *dcp, static u8 dcpep_cb_create_backlight_service(struct apple_dcp *dcp) { +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P asks during start_ios_gated, before any mode is known, but + * dcp_has_panel() only becomes true once the panel dimensions have been + * parsed. An internal panel always has a backlight, so answer for the + * connector; answering false makes the firmware report that it could + * not find the backlight service. + */ + return dcp_has_panel(dcp) || + dcp->connector_type == DRM_MODE_CONNECTOR_eDP; +#else return dcp_has_panel(dcp); +#endif } TRAMPOLINE_VOID(trampoline_nop, dcpep_cb_nop); @@ -1121,7 +1717,7 @@ TRAMPOLINE_INOUT(trampoline_get_uint_prop, dcpep_cb_get_uint_prop, TRAMPOLINE_IN(trampoline_set_fx_prop, iomfbep_cb_set_fx_prop, struct iomfb_set_fx_prop_req) TRAMPOLINE_INOUT(trampoline_map_piodma, dcpep_cb_map_piodma, - struct dcp_map_buf_req, struct dcp_map_buf_resp); + struct dcp_map_buf_req, struct DCP_FW_NAME(dcp_map_buf_resp)); TRAMPOLINE_IN(trampoline_unmap_piodma, dcpep_cb_unmap_piodma, struct dcp_unmap_buf_resp); TRAMPOLINE_INOUT(trampoline_sr_set_property_int, iomfbep_cb_sr_set_property_int, @@ -1150,6 +1746,12 @@ TRAMPOLINE_OUT(trampoline_rt_bandwidth, dcpep_cb_rt_bandwidth, TRAMPOLINE_INOUT(trampoline_set_frame_sync_props, dcpep_cb_set_frame_sync_props, struct dcp_set_frame_sync_props_req, struct dcp_set_frame_sync_props_resp); +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +TRAMPOLINE_INOUT(trampoline_set_frame_sync_props_h17p, + dcpep_cb_set_frame_sync_props_h17p, + struct dcp_set_frame_sync_props_req_h17p, + struct dcp_set_frame_sync_props_resp_h17p); +#endif TRAMPOLINE_OUT(trampoline_get_frequency, dcpep_cb_get_frequency, u64); TRAMPOLINE_OUT(trampoline_get_time, dcpep_cb_get_time, u64); TRAMPOLINE_IN(trampoline_hotplug, dcpep_cb_hotplug, u64); @@ -1182,33 +1784,25 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) dcp->swap_start = ktime_get(); dcp->swap_submit_timestamp = arch_timer_read_counter(); - while (!list_empty(&dcp->swapped_out_fbs)) { - struct dcp_fb_reference *entry; - entry = list_first_entry(&dcp->swapped_out_fbs, - struct dcp_fb_reference, head); - if (entry->swap_id == dcp->last_swap_id) - break; - if (entry->fb) - drm_framebuffer_put(entry->fb); - list_del(&entry->head); - kfree(entry); - } + dcp_release_swapped_out_fbs(dcp); } static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) { - struct dcp_swap_start_resp *resp = data; + struct DCP_FW_NAME(dcp_swap_start_resp) *resp = data; DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; + dcp_arm_swapped_out_fbs(dcp, resp->swap_id); trace_iomfb_swap_submit(dcp, resp->swap_id); + dcp_h17p_fix_swap_surfaces(dcp); dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swapped, NULL); } /* Helpers to modeset and swap, used to flush */ static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) { - struct dcp_swap_start_req start_req = { 0 }; + struct DCP_FW_NAME(dcp_swap_start_req) start_req = { 0 }; if (dcp->connector && dcp->connector->connected) dcp_swap_start(dcp, false, &start_req, dcp_swap_started, NULL); @@ -1216,6 +1810,19 @@ static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) dcp_drm_crtc_vblank(dcp->crtc); } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * H17P expects A428 between set_matrix and swap_start, with 0x10000 (1.0 in + * 16.16 fixed point), the same identity value the D006 reply carries. + */ +static void dcp_pre_swap_a428(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 identity = 0x10000; + + dcp_pipe_set_428(dcp, false, &identity, do_swap, cookie); +} +#endif + static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, void *cookie) { @@ -1311,7 +1918,10 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, dcp->during_modeset = true; dcp->swap_submit_timestamp = 0; - if (mode->vrr) + if (DCP_INHERIT_BOOT_MODE) { + dev_dbg(dcp->dev, "inheriting the bootloader's display mode\n"); + complete_set_digital_out_mode(dcp, NULL, cookie); + } else if (mode->vrr) dcp_set_adaptive_sync(dcp, crtc_state->vrr_enabled ? mode->min_vrr : 0, cookie); @@ -1385,8 +1995,10 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru for (l = 0; l < SWAP_SURFACES; l++) req->surf_null[l] = true; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) +#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) for (l = 0; l < 5; l++) req->surf2_null[l] = true; +#endif req->unkU32Ptr_null = true; req->unkU32out_null = true; #endif @@ -1425,10 +2037,27 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru struct dcp_fb_reference *entry = kzalloc(sizeof(*entry), GFP_KERNEL); if (entry) { +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + unsigned long flags; + + entry->fb = old_state->fb; + /* + * Left unarmed: the id of the swap that + * unbinds this framebuffer is not known until + * the firmware answers swap_start. + */ + spin_lock_irqsave(&dcp->swapped_out_lock, + flags); + list_add_tail(&entry->head, + &dcp->swapped_out_fbs); + spin_unlock_irqrestore(&dcp->swapped_out_lock, + flags); +#else entry->fb = old_state->fb; entry->swap_id = dcp->last_swap_id; list_add_tail(&entry->head, &dcp->swapped_out_fbs); +#endif } drm_framebuffer_get(old_state->fb); } @@ -1527,7 +2156,11 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru mat.matrix[0] = mat.matrix[4] = mat.matrix[8] = 1LLU << 32; } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + iomfb_set_matrix(dcp, false, &mat, dcp_pre_swap_a428, NULL); +#else iomfb_set_matrix(dcp, false, &mat, do_swap, NULL); +#endif } else do_swap(dcp, NULL, NULL); } @@ -1555,6 +2188,141 @@ static void init_3(struct apple_dcp *dcp, void *out, void *cookie) dcp_is_main_display(dcp, false, res_is_main_display, NULL); } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * The client-open sequence H17P firmware expects before the first swap, in + * H17P method numbers (dcp_push() applies the H17G aliases): + * + * A445 A411 A445 A415 A450 A031 x9 A414 A478 A457 A474 A478 A476 A478 A412 + * + * A412 is not in the table: its result feeds res_is_main_display(), so it + * stays on the existing path. + */ +enum dcp_client_step_kind { + DCP_STEP_A445, DCP_STEP_A411, DCP_STEP_A415, DCP_STEP_A450, + DCP_STEP_A031, DCP_STEP_A414, DCP_STEP_A478, DCP_STEP_A457, + DCP_STEP_A474, DCP_STEP_A476, +}; + +static const u8 dcp_client_open_seq[] = { + DCP_STEP_A445, DCP_STEP_A411, DCP_STEP_A445, DCP_STEP_A415, + DCP_STEP_A450, + DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, + DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, + DCP_STEP_A031, + DCP_STEP_A414, DCP_STEP_A478, DCP_STEP_A457, DCP_STEP_A474, + DCP_STEP_A478, DCP_STEP_A476, DCP_STEP_A478, +}; + +static void dcp_client_open_run(struct apple_dcp *dcp, void *out, void *cookie) +{ + unsigned long idx = (unsigned long)cookie; + dcp_callback_t next = dcp_client_open_run; + void *ncookie = (void *)(idx + 1); + + if (idx >= ARRAY_SIZE(dcp_client_open_seq)) { + init_3(dcp, NULL, NULL); + return; + } + + switch (dcp_client_open_seq[idx]) { + case DCP_STEP_A445: { + u32 zero = 0; + + dcp_get_dfb_state(dcp, false, &zero, next, ncookie); + break; + } + case DCP_STEP_A411: { + u32 handle = 0; + + dcp_set_display_device(dcp, false, &handle, next, ncookie); + break; + } + case DCP_STEP_A415: { + struct dcp_pipe_cfg_415 cfg = {}; + + dcp_pipe_cfg_415(dcp, false, &cfg, next, ncookie); + break; + } + case DCP_STEP_A450: { + u32 val = 0; + + dcp_enable_disable_video_power_savings(dcp, false, &val, next, + ncookie); + break; + } + case DCP_STEP_A031: { + /* + * Nine cumulative states: each call repeats the previous set + * of notification clients and adds one, so state n lists n + * clients. + */ + static const u8 notify_clients[9][8] = { + {}, + { 6 }, + { 6, 7 }, + { 6, 7, 8 }, + { 6, 7, 8, 10 }, + { 6, 7, 8, 10, 11 }, + { 6, 7, 8, 10, 11, 12 }, + { 6, 7, 8, 10, 11, 12, 25 }, + { 6, 7, 8, 10, 11, 12, 25, 26 }, + }; + struct dcp_pipe_cfg_031 cfg = {}; + unsigned int step = dcp->a031_step; + unsigned int i; + + if (step >= ARRAY_SIZE(notify_clients)) + step = ARRAY_SIZE(notify_clients) - 1; + for (i = 0; i < step; i++) + cfg.clients[i] = cpu_to_le32(notify_clients[step][i]); + dcp->a031_step++; + + dcp_pipe_cfg_031(dcp, false, &cfg, next, ncookie); + break; + } + case DCP_STEP_A414: { + u64 val = 0; + + dcp_pipe_cfg_414(dcp, false, &val, next, ncookie); + break; + } + case DCP_STEP_A478: + dcp_pipe_query_478(dcp, false, next, ncookie); + break; + case DCP_STEP_A457: + dcp_first_client_open(dcp, false, next, ncookie); + break; + case DCP_STEP_A474: + dcp_pipe_query_474(dcp, false, next, ncookie); + break; + case DCP_STEP_A476: { + /* + * A476 takes a single u32, which the firmware expects to be 1. + * Its name is not known for certain: the shape (in 4, out 0) + * fits both set_dcp_clamshellstate() and + * set_has_frame_swap_function(). + */ + u32 state = 1; + + dcp_pipe_set_476(dcp, false, &state, next, ncookie); + break; + } + default: + dev_warn(dcp->dev, "unknown client-open step %u\n", + dcp_client_open_seq[idx]); + init_3(dcp, NULL, NULL); + break; + } +} + +static void init_1(struct apple_dcp *dcp, void *out, void *cookie) +{ + /* every client-open sequence starts from the empty A031 state */ + dcp->a031_step = 0; + dcp_client_open_run(dcp, NULL, (void *)0UL); +} +#else static void init_2(struct apple_dcp *dcp, void *out, void *cookie) { dcp_first_client_open(dcp, false, init_3, NULL); @@ -1565,6 +2333,7 @@ static void init_1(struct apple_dcp *dcp, void *out, void *cookie) u32 val = 0; dcp_enable_disable_video_power_savings(dcp, false, &val, init_2, NULL); } +#endif static void dcp_started(struct apple_dcp *dcp, void *data, void *cookie) { @@ -1580,9 +2349,9 @@ static void dcp_started(struct apple_dcp *dcp, void *data, void *cookie) void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp) { - struct dcp_set_power_state_req req = { + struct DCP_FW_NAME(dcp_set_power_state_req) req = { /* defaults are ok */ }; - dcp_set_power_state(dcp, false, &req, NULL, NULL); + dcp_set_power_state(dcp, DCP_POWER_OOB, &req, NULL, NULL); } diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index 33302f78d5ad1..06584b0291964 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -27,6 +27,18 @@ struct DCP_FW_NAME(dcp_swap) { #if DCP_FW_LAYOUT_14_7(DCP_FW_VER) u8 unk_v14_7[0x48]; #endif + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P inserts 0x48 bytes between flags2 and the swap ID, which moves + * to +0x98. Every following field shifts with it: surf_ids +0x9c, + * src_rect +0xac, surf_flags +0xec, dst_rect +0x10c, + * swap_enabled/completed +0x14c/+0x150, bg_color +0x154. The gap holds + * two small u32 counters (at +0x50 and +0x78) that are not modelled. + */ + u8 h17p_pre_swap_id[0x48]; +#endif + u32 swap_id; u32 surf_ids[SWAP_SURFACES]; @@ -74,6 +86,13 @@ struct DCP_FW_NAME(dcp_swap) { u32 unk_flags; u32 unk_flags2; #endif +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * The H17P A408 request is 0xe9c bytes; with a 0x22c-byte surface, + * SWAP_SURFACES = 4 and no surf2 block, that leaves this tail. + */ + u8 h17p_tail[0xda]; +#endif } __packed; /* Information describing a surface */ @@ -94,8 +113,14 @@ struct DCP_FW_NAME(dcp_swap_submit_req) { u64 surf_iova[SWAP_SURFACES]; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) u64 unk_u64_a[SWAP_SURFACES]; + /* + * H17P carries no surf2 block: its trailing u8 null-flag run is 8 bytes + * (swap_null + surf_null[4] + three tail bools). + */ +#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) struct DCP_FW_NAME(dcp_surface) surf2[5]; u64 surf2_iova[5]; +#endif #endif u8 unkbool; u64 unkdouble; @@ -113,14 +138,18 @@ struct DCP_FW_NAME(dcp_swap_submit_req) { u8 swap_null; u8 surf_null[SWAP_SURFACES]; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) +#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) u8 surf2_null[5]; +#endif #endif u8 unkoutbool_null; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) u8 unkU32Ptr_null; u8 unkU32out_null; #endif +#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) u8 padding[1]; +#endif } __packed; struct DCP_FW_NAME(dcp_swap_submit_resp) { @@ -132,9 +161,104 @@ struct DCP_FW_NAME(dcp_swap_submit_resp) { u8 padding[3]; } __packed; -struct DCP_FW_NAME(dc_swap_complete_resp) { +/* + * A406 swap_start / A473 set_power_state. H17P shrank both requests: the + * 0x10-byte struct dcp_iouserclient became a bare u64 handle and the + * swap_start response no longer echoes it. swap_start is 0x10 in / 0x8 out, + * set_power_state 0x10 in. + */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +struct DCP_FW_NAME(dcp_swap_start_req) { + u32 swap_id; + u64 client; /* IOUserClient kernel VA */ + u8 client_null; + u8 padding[3]; +} __packed; + +struct DCP_FW_NAME(dcp_swap_start_resp) { + u32 swap_id; + u32 ret; +} __packed; + +struct DCP_FW_NAME(dcp_set_power_state_req) { + u64 unklong; + u32 unkint; /* widened from u8 on H17P */ + u8 unkint_null; + u8 padding[3]; +} __packed; + +static_assert(sizeof(struct DCP_FW_NAME(dcp_swap_start_req)) == 0x10); +static_assert(sizeof(struct DCP_FW_NAME(dcp_swap_start_resp)) == 0x8); +static_assert(sizeof(struct DCP_FW_NAME(dcp_set_power_state_req)) == 0x10); +#else +struct DCP_FW_NAME(dcp_swap_start_req) { + u32 swap_id; + struct dcp_iouserclient client; + u8 swap_id_null; + u8 client_null; + u8 padding[2]; +} __packed; + +struct DCP_FW_NAME(dcp_swap_start_resp) { u32 swap_id; + struct dcp_iouserclient client; + u32 ret; +} __packed; + +struct DCP_FW_NAME(dcp_set_power_state_req) { + u64 unklong; u8 unkbool; + u8 unkint_null; + u8 padding[2]; +} __packed; +#endif + +/* + * D201 map_piodma reply. H17P declares a 16-byte output, not 20, laid out as + * { u32 buffer index, u64 dva, u32 pad } rather than { u64 vaddr, u64 dva, + * u32 ret }. With the old layout the DVA lands four bytes late and the + * display engine DMAs to a bogus address. + */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +struct DCP_FW_NAME(dcp_map_buf_resp) { + u32 buffer; + u64 dva; + u32 unk; +} __packed; +static_assert(sizeof(struct DCP_FW_NAME(dcp_map_buf_resp)) == 0x10); +#else +struct DCP_FW_NAME(dcp_map_buf_resp) { + u64 vaddr; + u64 dva; + u32 ret; +} __packed; +#endif + +struct DCP_FW_NAME(dc_swap_complete_resp) { + u32 swap_id; /* +0x000 */ + u8 unkbool; /* +0x004 */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P: swap_complete_ap_gated(u32, bool, SwapCompleteData const *, + * SwapInfoBlob const *, u32, bool) is 0x730 bytes on the wire, 89 more + * than the pre-H17P record: the SwapCompleteData grew from 8 to 0x22, + * swap_info became eight 0xe0-byte SwapInfoBlobs rather than one + * 0x6c5 blob, and the tail gained a flag and three pad bytes. Only + * swap_info_count of those blobs are populated; the rest are + * uninitialised DCP buffer memory. + */ + u32 hol_swap_ids[4]; /* +0x005 */ + u32 hol_arg; /* +0x015 */ + u32 hol_count; /* +0x019 */ + u8 hol_valid; /* +0x01d */ + u64 displayed_data; /* +0x01e, unaligned */ + u8 notify_displayed; /* +0x026 */ + u8 swap_info[8][0xe0]; /* +0x027, blob[0]+0x38 == swap_id */ + u32 swap_info_count; /* +0x727, max 8 */ + u8 completion_flag; /* +0x72b */ + u8 swap_data_null; /* +0x72c, 0 => data at +0x005 valid */ + u8 padding[3]; /* +0x72d, to sizeof == 0x730 */ +#else u64 swap_data; #if DCP_FW_VER < DCP_FW_VERSION(13, 2, 0) u8 swap_info[0x6c4]; @@ -143,6 +267,7 @@ struct DCP_FW_NAME(dc_swap_complete_resp) { #endif u32 unkint; u8 swap_info_null; +#endif } __packed; struct DCP_FW_NAME(dcp_map_reg_req) { diff --git a/drivers/gpu/drm/apple/iomfb_v12_3.c b/drivers/gpu/drm/apple/iomfb_v12_3.c index 4d7492e76961f..9b87a35148231 100644 --- a/drivers/gpu/drm/apple/iomfb_v12_3.c +++ b/drivers/gpu/drm/apple/iomfb_v12_3.c @@ -5,6 +5,7 @@ #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" #include "iomfb_v14_7.h" +#include "iomfb_h17p.h" #include "version_utils.h" static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { diff --git a/drivers/gpu/drm/apple/iomfb_v13_3.c b/drivers/gpu/drm/apple/iomfb_v13_3.c index b3ceabb0d610a..574c957f1643b 100644 --- a/drivers/gpu/drm/apple/iomfb_v13_3.c +++ b/drivers/gpu/drm/apple/iomfb_v13_3.c @@ -5,6 +5,7 @@ #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" #include "iomfb_v14_7.h" +#include "iomfb_h17p.h" #include "version_utils.h" static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index f5145e2669c05..ccf3310267604 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -5,6 +5,7 @@ #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" #include "iomfb_v14_7.h" +#include "iomfb_h17p.h" #include "version_utils.h" static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index a44f9d547e413..7982aae874a3f 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -655,7 +655,7 @@ int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, } int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, - const char **class, s64 *unit) + const char **class, s64 *unit, bool h17p_keys) { int ret = 0; struct iterator it; @@ -674,19 +674,36 @@ int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, break; } - if (!strcmp(key, "EPICName")) { + /* + * H17P advertises its system service with lower-case keys -- + * "compartment", "name", "interface-id" -- where earlier + * firmware uses EPICName/EPICProviderClass/EPICUnit. Without + * them the service never registers, the firmware has no system + * service to ask and idles the panel, so no vblank and no swap + * ever completes. Only H17P callers accept them. If a + * dictionary carries both spellings, the later value wins and + * the earlier string is freed. + */ + if (!strcmp(key, "EPICName") || + (h17p_keys && !strcmp(key, "compartment"))) { + if (parsed_name) + kfree(*name); *name = parse_string(it.handle); if (IS_ERR(*name)) ret = PTR_ERR(*name); else parsed_name = true; - } else if (!strcmp(key, "EPICProviderClass")) { + } else if (!strcmp(key, "EPICProviderClass") || + (h17p_keys && !strcmp(key, "name"))) { + if (parsed_class) + kfree(*class); *class = parse_string(it.handle); if (IS_ERR(*class)) ret = PTR_ERR(*class); else parsed_class = true; - } else if (!strcmp(key, "EPICUnit")) { + } else if (!strcmp(key, "EPICUnit") || + (h17p_keys && !strcmp(key, "interface-id"))) { ret = parse_int(it.handle, unit); if (!ret) parsed_unit = true; diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 35f4e81e76212..bfb0a06c8760a 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -109,7 +109,7 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, int *height_mm); int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, - const char **class, s64 *unit); + const char **class, s64 *unit, bool h17p_keys); struct dcp_sound_format_mask { u64 formats; /* SNDRV_PCM_FMTBIT_* */ diff --git a/drivers/gpu/drm/apple/version_utils.h b/drivers/gpu/drm/apple/version_utils.h index 6d395f77e9212..5fd801dadc259 100644 --- a/drivers/gpu/drm/apple/version_utils.h +++ b/drivers/gpu/drm/apple/version_utils.h @@ -25,6 +25,7 @@ enum dcp_firmware_version { DCP_FIRMWARE_V_12_3, DCP_FIRMWARE_V_13_5, DCP_FIRMWARE_V_14_7, + DCP_FIRMWARE_H17P, }; #endif /*__APPLE_VERSION_UTILS_H__*/ From 039adcd762a56a5440af60717407e2b6bc408283 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:15:47 +1000 Subject: [PATCH 1047/2680] drm: apple: balance piodma child ownership across bind attempts Adopt the registered piodma child rather than creating a duplicate during component rebind. Keep its lookup reference until unbind and release it on every later unsuccessful bind; failed component binds have no unbind call. A missing IOMMU domain defers bind. One cleanup helper distinguishes an adopted registration from a child created by this bind and clears stale child/domain pointers. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 99 +++++++++++++++------------- 2 files changed, 54 insertions(+), 46 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 7f80c1810e2ed..ff9426486717f 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -153,6 +153,7 @@ struct apple_dcp_hw_data { struct apple_dcp { struct device *dev; struct platform_device *piodma; + bool piodma_created; struct iommu_domain *iommu_dom; /* which of the nine cumulative A031 notify-client states to send next */ unsigned int a031_step; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index b6cf87eb97126..b20b0a0e79561 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1597,6 +1597,19 @@ static void dcp_work_update_backlight(struct work_struct *work) dcp_backlight_update(dcp); } +static void dcp_release_piodma_iommu_dev(struct apple_dcp *dcp) +{ + if (dcp->piodma) { + if (dcp->piodma_created) + of_platform_device_destroy(&dcp->piodma->dev, NULL); + else + put_device(&dcp->piodma->dev); + } + dcp->piodma = NULL; + dcp->iommu_dom = NULL; + dcp->piodma_created = false; +} + static int dcp_create_piodma_iommu_dev(struct apple_dcp *dcp) { int ret; @@ -1606,31 +1619,13 @@ static int dcp_create_piodma_iommu_dev(struct apple_dcp *dcp) return dev_err_probe(dcp->dev, -ENODEV, "Failed to get piodma child DT node\n"); - dcp->piodma = of_platform_device_create(node, NULL, dcp->dev); - if (!dcp->piodma) { - /* - * of_platform_device_create() returns NULL when the node already - * has a platform device, which happens whenever a previous bind - * attempt created it and then failed further down (the caller's - * later error paths do not destroy it). Probe is retried - a - * deferral, or a component re-bind - and would then fail here - * forever with -ENODEV even though nothing is actually wrong. - * Adopt the existing device instead of giving up. - */ - dcp->piodma = of_find_device_by_node(node); - if (!dcp->piodma) - return dev_err_probe(dcp->dev, -ENODEV, - "Failed to create piodma pdev for %pOF\n", - node); - /* - * Drop the lookup reference: the device stays registered until - * of_platform_device_destroy(), exactly like one created above, - * and that call only releases the registration reference. - */ - put_device(&dcp->piodma->dev); - dev_info(dcp->dev, "reusing existing piodma pdev for %pOF\n", - node); - } + dcp->piodma = of_find_device_by_node(node); + dcp->piodma_created = !dcp->piodma; + if (dcp->piodma_created) + dcp->piodma = of_platform_device_create(node, NULL, dcp->dev); + if (!dcp->piodma) + return dev_err_probe(dcp->dev, -ENODEV, + "Failed to create piodma pdev for %pOF\n", node); ret = dma_set_mask_and_coherent(&dcp->piodma->dev, DMA_BIT_MASK(42)); if (ret) @@ -1654,7 +1649,7 @@ static int dcp_create_piodma_iommu_dev(struct apple_dcp *dcp) return 0; err_destroy_pdev: - of_platform_device_destroy(&dcp->piodma->dev, NULL); + dcp_release_piodma_iommu_dev(dcp); return ret; } @@ -2014,24 +2009,33 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) } ret = dcp_create_piodma_iommu_dev(dcp); - if (ret || !dcp->iommu_dom) + if (ret) return dev_err_probe(dev, ret, - "Failed to created PIODMA iommu child device"); + "Failed to create PIODMA iommu child device\n"); + if (!dcp->iommu_dom) { + ret = dev_err_probe(dev, -EPROBE_DEFER, + "PIODMA iommu domain is unavailable\n"); + goto err_piodma; + } ret = dcp_get_disp_regs(dcp); if (ret) { dev_err(dev, "failed to find display registers\n"); - return ret; + goto err_piodma; } dcp->clk = devm_clk_get(dev, NULL); - if (IS_ERR(dcp->clk)) - return dev_err_probe(dev, PTR_ERR(dcp->clk), - "Unable to find clock\n"); + if (IS_ERR(dcp->clk)) { + ret = dev_err_probe(dev, PTR_ERR(dcp->clk), + "Unable to find clock\n"); + goto err_piodma; + } dcp->clk_194 = devm_clk_get_optional(dev, "clock-194"); - if (IS_ERR(dcp->clk_194)) - return dev_err_probe(dev, PTR_ERR(dcp->clk_194), - "Unable to find clock 0x194\n"); + if (IS_ERR(dcp->clk_194)) { + ret = dev_err_probe(dev, PTR_ERR(dcp->clk_194), + "Unable to find clock 0x194\n"); + goto err_piodma; + } bitmap_zero(dcp->memdesc_map, DCP_MAX_MAPPINGS); // TDOD: mem_desc IDs start at 1, for simplicity just skip '0' entry @@ -2051,17 +2055,24 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) } dcp->rtk = devm_apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); - if (IS_ERR(dcp->rtk)) - return dev_err_probe(dev, PTR_ERR(dcp->rtk), - "Failed to initialize RTKit\n"); + if (IS_ERR(dcp->rtk)) { + ret = dev_err_probe(dev, PTR_ERR(dcp->rtk), + "Failed to initialize RTKit\n"); + goto err_piodma; + } if (dcp->hw.adopt_live_session) dev_info(dev, "negotiating RTKit with the running DCP\n"); ret = apple_rtkit_wake(dcp->rtk); - if (ret) - return dev_err_probe(dev, ret, - "Failed to boot RTKit: %d\n", ret); + if (ret) { + ret = dev_err_probe(dev, ret, "Failed to boot RTKit: %d\n", ret); + goto err_piodma; + } + return 0; + +err_piodma: + dcp_release_piodma_iommu_dev(dcp); return ret; } @@ -2110,11 +2121,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) if (dcp->shmem) iomfb_shutdown(dcp); - if (dcp->piodma) { - dcp->iommu_dom = NULL; - of_platform_device_destroy(&dcp->piodma->dev, NULL); - dcp->piodma = NULL; - } + dcp_release_piodma_iommu_dev(dcp); if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { cancel_work_sync(&dcp->bl_register_wq); From c53656cea00498f44e0b8df272c786e9287436d2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:16:49 +1000 Subject: [PATCH 1048/2680] drm: apple: look up display register windows by reg-names The number of display windows differs between DCP instances. Count the disp-* names and fetch each resource by name so unrelated registers do not shift the bandwidth window indexes. Retain the positional lookup for device trees without reg-names. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 70 +++++++++++++------------------------ 1 file changed, 24 insertions(+), 46 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index b20b0a0e79561..4f50d0037723c 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1750,67 +1750,45 @@ static int dcp_get_bw_doorbell_reg(struct apple_dcp *dcp, u32 expected) return ret; } -/* - * Count the "disp-*" entries in reg-names. - * - * The positional form below assumes the reg layout is [coproc, disp-0 ...], - * i.e. exactly one non-disp register, and derives the count as - * num_resources - 1. DCPs with H17-generation firmware list a trailing - * "iop-vbar" register after the display apertures, which would inflate that - * count, bind the last disp_registers[] entry to iop-vbar and make the - * "apple,bw-scratch" disp_reg index check reject a valid device tree. - * - * Returns a negative errno when reg-names is absent, so callers fall back. - */ -static int dcp_count_disp_regs(struct device *dev) -{ - int n = of_property_count_strings(dev->of_node, "reg-names"); - const char *name; - int i, count = 0; - - if (n <= 0) - return -EINVAL; - - for (i = 0; i < n; ++i) { - if (of_property_read_string_index(dev->of_node, "reg-names", i, - &name)) - return -EINVAL; - if (!strncmp(name, "disp-", 5)) - count++; - } - - return count ? count : -EINVAL; -} - static int dcp_get_disp_regs(struct apple_dcp *dcp) { struct platform_device *pdev = to_platform_device(dcp->dev); - int count = dcp_count_disp_regs(dcp->dev); - bool by_name = count > 0; + int count = 0; int i, ret; + char name[16]; + const char *reg_name; + struct resource *res; + + if (of_property_present(dcp->dev->of_node, "reg-names")) { + ret = of_property_count_strings(dcp->dev->of_node, "reg-names"); + if (ret < 0) + return ret; - /* Device trees without reg-names keep the historical positional form. */ - if (!by_name) + for (i = 0; i < ret; i++) { + if (of_property_read_string_index(dcp->dev->of_node, + "reg-names", i, ®_name)) + return -EINVAL; + if (!strncmp(reg_name, "disp-", 5)) + count++; + } + } else { count = pdev->num_resources - 1; + } if (count <= 0 || count > MAX_DISP_REGISTERS) return -EINVAL; for (i = 0; i < count; ++i) { - if (by_name) { - char name[8]; - + if (of_property_present(dcp->dev->of_node, "reg-names")) { snprintf(name, sizeof(name), "disp-%d", i); - dcp->disp_registers[i] = platform_get_resource_byname( - pdev, IORESOURCE_MEM, name); + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + name); } else { - dcp->disp_registers[i] = platform_get_resource( - pdev, IORESOURCE_MEM, 1 + i); + res = platform_get_resource(pdev, IORESOURCE_MEM, 1 + i); } - if (!dcp->disp_registers[i]) { - dev_err(dcp->dev, "missing display register %d\n", i); + if (!res) return -EINVAL; - } + dcp->disp_registers[i] = res; } /* load pmgr bandwidth scratch resource and offset */ From d84452d46028cf40f8a83868e15cdc0f2677d556 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:22:27 +1000 Subject: [PATCH 1049/2680] drm: apple: map firmware-provided RTKit buffers write-combined The DCP writes its RTKit buffers without CPU cache maintenance. A write-back mapping can read stale crash-log contents after the firmware has filled the buffer. Use a write-combined mapping for firmware-provided physical pages. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 4f50d0037723c..d7dde4b4ec65d 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -951,8 +951,8 @@ static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) return ret; } - // TODO: verify phy_addr, cache attribute - bfr->buffer = memremap(phy_addr, bfr->size, MEMREMAP_WB); + /* Firmware writes without CPU cache maintenance. */ + bfr->buffer = memremap(phy_addr, bfr->size, MEMREMAP_WC); if (!bfr->buffer) return -ENOMEM; From 118daf1c0c99a8678bb8937e2a655c795f4a4bf6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:36:09 +1000 Subject: [PATCH 1050/2680] drm: apple: afk: derive the ring header stride from INITRB AFK rings describe their payload size in the first word. Derive the three header blocks from the INITRB total instead of assuming 64 bytes per block. Keep the offset and size fields wide enough for the full wire range, and align queue entries to the derived stride. Synthetic KUnit cases cover the older and larger ring headers and malformed layouts (AFK-1). Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Kconfig | 10 ++++++ drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/afk.c | 55 +++++++++++++++----------------- drivers/gpu/drm/apple/afk.h | 49 ++++++++++++++++------------ drivers/gpu/drm/apple/afk_test.c | 44 +++++++++++++++++++++++++ drivers/gpu/drm/apple/trace.h | 4 +-- 6 files changed, 110 insertions(+), 53 deletions(-) create mode 100644 drivers/gpu/drm/apple/afk_test.c diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index ecf7afd76ceda..c4eff8fd0c5b3 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -28,3 +28,13 @@ config DRM_APPLE_DEBUG bool "Enable additional driver debugging" depends on DRM_APPLE depends on EXPERT # only for developers + +config DRM_APPLE_KUNIT_TEST + tristate "KUnit tests for Apple display controller" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Build tests for the Apple display driver's AFK ring parser. + The tests cover the header strides used by older firmware and + larger rings, as well as malformed ring size combinations. + This option is intended for developer test kernels. diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 27d7586db902c..63b4ea7cdca24 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -19,3 +19,4 @@ appledrm-y += plane.o appledrm-$(CONFIG_TRACING) += trace.o obj-$(CONFIG_DRM_APPLE) += appledrm.o +obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk_test.o diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 4a4215e0af2df..daa27e1ad7238 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -152,7 +152,8 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, u32 base = FIELD_GET(INITRB_OFFSET, message) << BLOCK_SHIFT; u32 size = FIELD_GET(INITRB_SIZE, message) << BLOCK_SHIFT; u16 tag = FIELD_GET(INITRB_TAG, message); - u32 bufsz, hdrsz, end, block; + u32 bufsz, end, stride; + u8 *hdr; if (tag != ep->bfr_tag) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: expected tag 0x%x but got 0x%x\n", @@ -166,10 +167,10 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, return; } - if (!ep->bfr || size < sizeof(__le32) || base >= ep->bfr_size) { + if (!ep->bfr || base >= ep->bfr_size || size < sizeof(u32)) { dev_err(ep->dcp->dev, - "AFK[ep:%02x]: requested base 0x%x >= max size 0x%lx\n", - ep->endpoint, base, ep->bfr_size); + "AFK[ep:%02x]: invalid ring base 0x%x or size 0x%x\n", + ep->endpoint, base, size); return; } @@ -181,23 +182,21 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, return; } - bfr->hdr = ep->bfr + base; + hdr = (u8 *)ep->bfr + base; dma_rmb(); - bufsz = le32_to_cpu(READ_ONCE(*(__le32 *)bfr->hdr)); - hdrsz = size - bufsz; - block = hdrsz / AFK_RB_BLOCKS; - if (bufsz >= size || hdrsz % AFK_RB_BLOCKS || - block < AFK_RB_BLOCK_MIN || !is_power_of_2(block) || - !IS_ALIGNED(bufsz, block) || bufsz < 2 * block) { + bufsz = le32_to_cpu(READ_ONCE(*(__le32 *)hdr)); + if (afk_ring_stride(size, bufsz, &stride)) { dev_err(ep->dcp->dev, - "AFK[ep:%02x]: ring bufsz %#x with size %#x gives no valid header (base %#x, msg %#llx)\n", - ep->endpoint, bufsz, size, base, message); + "AFK[ep:%02x]: invalid ring payload size 0x%x of 0x%x\n", + ep->endpoint, bufsz, size); return; } - bfr->block = block; - bfr->buf = (u8 *)bfr->hdr + hdrsz; + bfr->rptr = (__le32 *)(hdr + stride); + bfr->wptr = (__le32 *)(hdr + 2 * stride); + bfr->buf = hdr + 3 * stride; bfr->bufsz = bufsz; + bfr->stride = stride; bfr->ready = true; if (ep->rxbfr.ready && ep->txbfr.ready) @@ -683,8 +682,8 @@ static bool afk_recv(struct apple_dcp_afkep *ep) return false; } - rptr = le32_to_cpu(*afk_rb_rptr(&ep->rxbfr)); - wptr = le32_to_cpu(*afk_rb_wptr(&ep->rxbfr)); + rptr = le32_to_cpu(*ep->rxbfr.rptr); + wptr = le32_to_cpu(*ep->rxbfr.wptr); trace_afk_recv_rwptr_pre(ep, rptr, wptr); if (rptr == wptr) @@ -729,7 +728,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) return false; } - *afk_rb_rptr(&ep->rxbfr) = cpu_to_le32(rptr); + *ep->rxbfr.rptr = cpu_to_le32(rptr); } if (rptr + size + sizeof(*hdr) > ep->rxbfr.bufsz) { @@ -742,15 +741,12 @@ static bool afk_recv(struct apple_dcp_afkep *ep) channel = le32_to_cpu(hdr->channel); type = le32_to_cpu(hdr->type); - rptr = ALIGN(rptr + sizeof(*hdr) + size, ep->rxbfr.block); - if (WARN_ON(rptr > ep->rxbfr.bufsz)) - rptr = 0; - if (rptr == ep->rxbfr.bufsz) - rptr = 0; + rptr = afk_ring_advance(rptr, sizeof(*hdr) + size, + ep->rxbfr.bufsz, ep->rxbfr.stride); dma_mb(); - *afk_rb_rptr(&ep->rxbfr) = cpu_to_le32(rptr); + *ep->rxbfr.rptr = cpu_to_le32(rptr); trace_afk_recv_rwptr_post(ep, rptr, wptr); /* @@ -858,8 +854,8 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, spin_lock_irqsave(&ep->lock, flags); dma_rmb(); - rptr = le32_to_cpu(*afk_rb_rptr(&ep->txbfr)); - wptr = le32_to_cpu(*afk_rb_wptr(&ep->txbfr)); + rptr = le32_to_cpu(*ep->txbfr.rptr); + wptr = le32_to_cpu(*ep->txbfr.wptr); trace_afk_send_rwptr_pre(ep, rptr, wptr); total_epic_size = sizeof(*ehdr) + sizeof(*eshdr) + payload_len; total_size = sizeof(*hdr) + total_epic_size; @@ -972,12 +968,11 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, memcpy(ep->txbfr.buf + wptr, payload, payload_len); wptr += payload_len; - wptr = ALIGN(wptr, ep->txbfr.block); - if (wptr == ep->txbfr.bufsz) - wptr = 0; + wptr = afk_ring_advance(wptr, 0, ep->txbfr.bufsz, + ep->txbfr.stride); trace_afk_send_rwptr_post(ep, rptr, wptr); - *afk_rb_wptr(&ep->txbfr) = cpu_to_le32(wptr); + *ep->txbfr.wptr = cpu_to_le32(wptr); afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_SEND) | FIELD_PREP(SEND_WPTR, wptr)); ret = 0; diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 6ba47202e23a9..1e691ec47c85e 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -8,7 +8,9 @@ #define _DRM_APPLE_DCP_AFK_H #include +#include #include +#include #include #include "dcp.h" @@ -71,17 +73,29 @@ struct apple_epic_service_ops { void (*teardown)(struct apple_epic_service *service); }; -/* - * The ring header is three control blocks -- bufsz, rptr, wptr -- each padded - * out to the firmware's control-block size. That block is 0x40 bytes up to - * v13.5 but 0x80 on H17P, making the header 0xc0 or 0x180. The size is - * self-describing: the INITRB message gives the total ring size and the first - * word of block 0 gives the payload size, so the stride is (size - bufsz) / 3 - * and no firmware-version test is needed. Address the fields by stride rather - * than through a fixed struct. - */ -#define AFK_RB_BLOCK_MIN 0x40 -#define AFK_RB_BLOCKS 3 +static inline int afk_ring_stride(u32 total, u32 bufsz, u32 *stride) +{ + u32 header, block; + + if (bufsz >= total) + return -EINVAL; + header = total - bufsz; + block = header / 3; + if (header % 3 || block < 0x40 || block % 0x40 || + bufsz < block || bufsz % block) + return -EINVAL; + + *stride = block; + return 0; +} + +static inline u32 afk_ring_advance(u32 ptr, u32 bytes, u32 bufsz, + u32 stride) +{ + u32 next = roundup(ptr + bytes, stride); + + return next == bufsz ? 0 : next; +} struct afk_qe { #define QE_MAGIC 0x20504f49 // ' POI' @@ -169,20 +183,13 @@ enum epic_subtype { struct afk_ringbuffer { bool ready; - void *hdr; - u32 block; /* control-block stride, 0x40 or 0x80 */ - u32 rptr; + __le32 *rptr; + __le32 *wptr; void *buf; size_t bufsz; + u32 stride; }; -static inline __le32 *afk_rb_field(struct afk_ringbuffer *bfr, unsigned int idx) -{ - return (__le32 *)((u8 *)bfr->hdr + idx * bfr->block); -} - -#define afk_rb_rptr(bfr) afk_rb_field(bfr, 1) -#define afk_rb_wptr(bfr) afk_rb_field(bfr, 2) struct apple_dcp_afkep { struct apple_dcp *dcp; diff --git a/drivers/gpu/drm/apple/afk_test.c b/drivers/gpu/drm/apple/afk_test.c new file mode 100644 index 0000000000000..e58a952642045 --- /dev/null +++ b/drivers/gpu/drm/apple/afk_test.c @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include + +#include "afk.h" + +static void afk_ring_stride_test(struct kunit *test) +{ + u32 stride; + + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x10c0, 0x1000, &stride), 0); + KUNIT_EXPECT_EQ(test, stride, 0x40U); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x1180, 0x1000, &stride), 0); + KUNIT_EXPECT_EQ(test, stride, 0x80U); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x1000, 0x1000, &stride), -EINVAL); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x10bd, 0x1000, &stride), -EINVAL); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x10c1, 0x1000, &stride), -EINVAL); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0xc0, 0, &stride), -EINVAL); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x10c3, 0x1000, &stride), -EINVAL); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x10c0, 0x1001, &stride), -EINVAL); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x11c0, 0x1040, &stride), -EINVAL); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x1180, 0x1000, &stride), 0); + KUNIT_EXPECT_EQ(test, afk_ring_advance(0xf80, 0x80, 0x1000, stride), 0U); + KUNIT_EXPECT_EQ(test, afk_ring_advance(0xe80, 0x80, 0x1000, stride), + 0xf00U); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x10c0, 0x1000, &stride), 0); + KUNIT_EXPECT_EQ(test, afk_ring_advance(0xfc0, 0x40, 0x1000, stride), 0U); +} + +static struct kunit_case afk_ring_cases[] = { + KUNIT_CASE(afk_ring_stride_test), + {} +}; + +static struct kunit_suite afk_ring_suite = { + .name = "apple-afk-ring", + .test_cases = afk_ring_cases, +}; + +kunit_test_suite(afk_ring_suite); + +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/trace.h b/drivers/gpu/drm/apple/trace.h index c0acac6ee690a..d3c5106e5d66d 100644 --- a/drivers/gpu/drm/apple/trace.h +++ b/drivers/gpu/drm/apple/trace.h @@ -109,11 +109,11 @@ TRACE_EVENT(dcp_send_msg, show_dcp_endpoint(__entry->endpoint), __entry->message)); TRACE_EVENT( - afk_getbuf, TP_PROTO(struct apple_dcp_afkep *ep, u16 size, u16 tag), + afk_getbuf, TP_PROTO(struct apple_dcp_afkep *ep, u32 size, u16 tag), TP_ARGS(ep, size, tag), TP_STRUCT__entry(__string(devname, dev_name(ep->dcp->dev)) - __field(u8, endpoint) __field(u16, size) + __field(u8, endpoint) __field(u32, size) __field(u16, tag)), TP_fast_assign(__assign_str(devname); From 807129721fa839fe88240b8a19405db915398597 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:40:44 +1000 Subject: [PATCH 1051/2680] drm: apple: drop messages for endpoints without a handler Firmware may send on an endpoint that this DCP instance did not start. Log the endpoint with rate limiting and drop the message instead of calling AFK through a NULL pointer or WARNing. Refuse to start the DCP when RTKit is absent (RTK-2). Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index d7dde4b4ec65d..2f132d6ebcd84 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -848,6 +848,7 @@ static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) { struct apple_dcp *dcp = cookie; + struct apple_dcp_afkep *ep; trace_dcp_recv_msg(dcp, endpoint, message); @@ -867,23 +868,33 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) case IOMFB_ENDPOINT: return iomfb_recv_msg(dcp, message); case AV_ENDPOINT: - afk_receive_message(dcp->avep, message); - return; + ep = dcp->avep; + break; case SYSTEM_ENDPOINT: - afk_receive_message(dcp->systemep, message); - return; + ep = dcp->systemep; + break; case DISP0_ENDPOINT: - afk_receive_message(dcp->ibootep, message); - return; + ep = dcp->ibootep; + break; case DPAVSERV_ENDPOINT: - afk_receive_message(dcp->dcpavservep, message); - return; + ep = dcp->dcpavservep; + break; case DPTX_ENDPOINT: - afk_receive_message(dcp->dptxep, message); - return; + ep = dcp->dptxep; + break; default: - WARN(endpoint, "unknown DCP endpoint %hhu\n", endpoint); + ep = NULL; + break; + } + + if (!ep) { + dev_warn_ratelimited(dcp->dev, + "dropping message for unhandled endpoint %#x\n", + endpoint); + return; } + + afk_receive_message(ep, message); } static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_size) @@ -1307,6 +1318,9 @@ int dcp_start(struct platform_device *pdev) struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; + if (!dcp->rtk) + return -ENODEV; + init_completion(&dcp->start_done); /* From 6f8290e7133f63fd7d78bcc479bc5d54d3769559 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 6 Oct 2026 21:29:18 +1100 Subject: [PATCH 1052/2680] drm: apple: duplicate cached scanout geometry with plane state An inactive CRTC update can duplicate plane state without invoking the plane update hook. Copy its cached rectangles and IOVA along with the surface descriptor so a later activation presents the same scanout. DRM's atomic plane helpers can skip updates on inactive CRTCs. Keep the cached scanout fields consistent with the duplicated framebuffer state. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/plane.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index bd16e533cfae6..01243a39bb653 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -458,6 +458,9 @@ apple_plane_duplicate_state(struct drm_plane *plane) __drm_atomic_helper_plane_duplicate_state(plane, &apple_plane_state->base); apple_plane_state->surf = old_apple_plane_state->surf; + apple_plane_state->src_rect = old_apple_plane_state->src_rect; + apple_plane_state->dst_rect = old_apple_plane_state->dst_rect; + apple_plane_state->iova = old_apple_plane_state->iova; return &apple_plane_state->base; } From 4de9c3000640c911134ecbad88a7cf124b66428e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:47:31 +1000 Subject: [PATCH 1053/2680] drm: apple: retain displaced framebuffers until confirmed completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allocate and reference displaced scanout during plane preparation, where allocation failure can still abort the atomic commit. Transfer prepared entries before disabling or updating CRTCs, including inactive commits. Cleanup and state destruction release only unconsumed reservations. Retain transferred references until a successful successor swap completes or a successful power response confirms scanout stop on existing firmware variants. Rejected swap starts never arm retirement; rejected, missing or malformed power responses, timeouts and late replies cannot release live scanout. H17P stop semantics remain unqualified, so its power replies do not retire references. Initialize command output to an invalid status and validate legacy reply lengths before dispatching responses. Initialize retention ownership once at probe and reject rebind while a previous session's references remain. This preserves quarantine ownership instead of losing list links after a timed-out shutdown. [ryan: adapted to 7.2.10 — struct drm_atomic_state is struct drm_atomic_commit] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 26 ++- drivers/gpu/drm/apple/dcp-internal.h | 14 +- drivers/gpu/drm/apple/dcp.c | 86 +++++++++- drivers/gpu/drm/apple/dcp.h | 6 + drivers/gpu/drm/apple/iomfb.c | 11 ++ drivers/gpu/drm/apple/iomfb_template.c | 226 ++++++++----------------- drivers/gpu/drm/apple/plane.c | 63 ++++++- drivers/gpu/drm/apple/plane.h | 3 + 8 files changed, 262 insertions(+), 173 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 75c1d9d113a67..2244568705e76 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -122,10 +122,11 @@ static void apple_crtc_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_commit *commit) { struct drm_crtc_state *crtc_state; + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + crtc_state = drm_atomic_get_new_crtc_state(commit, crtc); if (crtc_state->active_changed && !crtc_state->active) { - struct apple_crtc *apple_crtc = to_apple_crtc(crtc); dcp_poweroff(apple_crtc->dcp); } @@ -230,8 +231,29 @@ static const struct drm_mode_config_funcs apple_mode_config_funcs = { .fb_create = drm_gem_fb_create, }; +static void apple_atomic_commit_tail(struct drm_atomic_commit *state) +{ + struct drm_plane *plane; + struct drm_plane_state *old, *new; + int i; + + /* Retain displaced scanout even when the CRTC stays inactive. */ + for_each_oldnew_plane_in_state(state, plane, old, new, i) { + struct apple_plane_state *apple_state = to_apple_plane_state(new); + struct dcp_fb_reference *entry = apple_state->retirement; + + if (!entry) + continue; + + apple_state->retirement = NULL; + dcp_retain_framebuffer(to_apple_crtc(old->crtc)->dcp, entry); + } + + drm_atomic_helper_commit_tail_rpm(state); +} + static const struct drm_mode_config_helper_funcs apple_mode_config_helpers = { - .atomic_commit_tail = drm_atomic_helper_commit_tail_rpm, + .atomic_commit_tail = apple_atomic_commit_tail, }; static void appledrm_connector_cleanup(struct drm_connector *connector) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index ff9426486717f..c7081bfd150c0 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -89,6 +89,8 @@ struct dcp_channel { dcp_callback_t callbacks[DCP_MAX_CALL_DEPTH]; void *cookies[DCP_MAX_CALL_DEPTH]; void *output[DCP_MAX_CALL_DEPTH]; + u32 in_len[DCP_MAX_CALL_DEPTH]; + u32 out_len[DCP_MAX_CALL_DEPTH]; u16 end[DCP_MAX_CALL_DEPTH]; /* Current depth of the call stack. Less than DCP_MAX_CALL_DEPTH */ @@ -270,16 +272,12 @@ struct apple_dcp { /* Workqueue for sending vblank events when a dcp swap is not possible */ struct work_struct vblank_wq; - /* List of referenced drm_framebuffers which can be unreferenced - * on the next successfully completed swap. + /* List of referenced framebuffers pending a completed replacement swap. + * The commit and RTKit work queues share it under this lock. */ struct list_head swapped_out_fbs; - /* - * Protects swapped_out_fbs, which is appended to from the DRM atomic - * commit (dcp_flush -> .atomic_flush) and armed/drained from the RTKit - * workqueue that runs the DCP callbacks. - */ - spinlock_t swapped_out_lock; + /* Protects swapped_out_fbs. */ + struct mutex swapped_out_fbs_lock; struct dcp_brightness brightness; /* Workqueue for updating the initial brightness */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 2f132d6ebcd84..4e0977f304f33 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1624,6 +1624,77 @@ static void dcp_release_piodma_iommu_dev(struct apple_dcp *dcp) dcp->piodma_created = false; } +void dcp_retain_framebuffer(struct platform_device *pdev, + struct dcp_fb_reference *entry) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + mutex_lock(&dcp->swapped_out_fbs_lock); + list_add_tail(&entry->head, &dcp->swapped_out_fbs); + mutex_unlock(&dcp->swapped_out_fbs_lock); +} + +void dcp_arm_retained_framebuffers(struct apple_dcp *dcp, u32 swap_id) +{ + struct dcp_fb_reference *entry; + + mutex_lock(&dcp->swapped_out_fbs_lock); + list_for_each_entry(entry, &dcp->swapped_out_fbs, head) { + if (entry->armed) + continue; + entry->swap_id = swap_id; + entry->armed = true; + } + mutex_unlock(&dcp->swapped_out_fbs_lock); +} + +void dcp_unarm_retained_framebuffers(struct apple_dcp *dcp, u32 swap_id) +{ + struct dcp_fb_reference *entry; + + mutex_lock(&dcp->swapped_out_fbs_lock); + list_for_each_entry(entry, &dcp->swapped_out_fbs, head) + if (entry->armed && entry->swap_id == swap_id) + entry->armed = false; + mutex_unlock(&dcp->swapped_out_fbs_lock); +} + +void dcp_release_retained_framebuffers(struct apple_dcp *dcp, u32 swap_id) +{ + struct dcp_fb_reference *entry, *tmp; + LIST_HEAD(completed); + + mutex_lock(&dcp->swapped_out_fbs_lock); + list_for_each_entry_safe(entry, tmp, &dcp->swapped_out_fbs, head) { + if (!entry->armed || entry->swap_id != swap_id) + continue; + list_move_tail(&entry->head, &completed); + } + mutex_unlock(&dcp->swapped_out_fbs_lock); + + list_for_each_entry_safe(entry, tmp, &completed, head) { + list_del(&entry->head); + drm_framebuffer_put(entry->fb); + kfree(entry); + } +} + +void dcp_release_all_retained_framebuffers(struct apple_dcp *dcp) +{ + struct dcp_fb_reference *entry, *tmp; + LIST_HEAD(completed); + + mutex_lock(&dcp->swapped_out_fbs_lock); + list_splice_init(&dcp->swapped_out_fbs, &completed); + mutex_unlock(&dcp->swapped_out_fbs_lock); + + list_for_each_entry_safe(entry, tmp, &completed, head) { + list_del(&entry->head); + drm_framebuffer_put(entry->fb); + kfree(entry); + } +} + static int dcp_create_piodma_iommu_dev(struct apple_dcp *dcp) { int ret; @@ -1933,6 +2004,14 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) u32 cpu_ctrl; int ret; + /* A timed-out prior session may still scan these mappings. */ + mutex_lock(&dcp->swapped_out_fbs_lock); + ret = list_empty(&dcp->swapped_out_fbs) ? 0 : -EBUSY; + mutex_unlock(&dcp->swapped_out_fbs_lock); + if (ret) + return dev_err_probe(dev, ret, + "Previous scanout has not been stopped\n"); + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); if (ret) return ret; @@ -2035,10 +2114,6 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); - dcp->swapped_out_fbs = - (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); - spin_lock_init(&dcp->swapped_out_lock); - if (!dcp->hw.adopt_live_session) { cpu_ctrl = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); @@ -2159,6 +2234,9 @@ static int dcp_platform_probe(struct platform_device *pdev) if (!dcp) return -ENOMEM; + INIT_LIST_HEAD(&dcp->swapped_out_fbs); + mutex_init(&dcp->swapped_out_fbs_lock); + dcp->fw_compat = fw_compat; dcp->dev = dev; dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 2466aa9d13571..89047832b018d 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -52,6 +52,12 @@ void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, int dcp_start(struct platform_device *pdev); int dcp_wait_ready(struct platform_device *pdev, u64 timeout); void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit); +void dcp_retain_framebuffer(struct platform_device *pdev, + struct dcp_fb_reference *entry); +void dcp_arm_retained_framebuffers(struct apple_dcp *dcp, u32 swap_id); +void dcp_unarm_retained_framebuffers(struct apple_dcp *dcp, u32 swap_id); +void dcp_release_retained_framebuffers(struct apple_dcp *dcp, u32 swap_id); +void dcp_release_all_retained_framebuffers(struct apple_dcp *dcp); bool dcp_is_initialized(struct platform_device *pdev); void apple_crtc_vblank(struct apple_crtc *apple); void dcp_drm_crtc_vblank(struct apple_crtc *crtc); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 6c2fd76b8ba02..d72539fb65699 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -193,12 +193,17 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca if (in_len > 0) memcpy(out_data, data, in_len); + /* An unwritten status must not look like a successful response. */ + if (out_len) + memset(out_data + in_len, 0xff, out_len); trace_iomfb_push(dcp, call, context, offset, depth); ch->callbacks[depth] = cb; ch->cookies[depth] = cookie; ch->output[depth] = out + sizeof(header) + in_len; + ch->in_len[depth] = in_len; + ch->out_len[depth] = out_len; ch->end[depth] = offset + ALIGN(data_len, DCP_PACKET_ALIGNMENT); dcp_send_message(dcp, IOMFB_ENDPOINT, @@ -405,6 +410,12 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, out = ch->output[ch->depth]; ch->output[ch->depth] = NULL; } else { + if (header->in_len != ch->in_len[ch->depth] || + header->out_len != ch->out_len[ch->depth]) { + dev_err(dcp->dev, "invalid command response lengths\n"); + dcp->crashed = true; + return; + } out = data + sizeof(*header) + header->in_len; } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 5e4f2099c368f..1fc32fdca9c95 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -41,6 +41,7 @@ static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), flags1) == 0x40); struct dcp_wait_cookie { struct kref refcount; struct completion done; + u32 status; }; static void release_wait_cookie(struct kref *ref) @@ -219,108 +220,15 @@ static u32 dcpep_cb_zero(struct apple_dcp *dcp) return 0; } -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -/* - * Bind every framebuffer displaced by the commit that produced this swap to - * that swap's id. - * - * The id only exists once the firmware answers swap_start, after the atomic - * commit has queued its entries, so they are queued unarmed and stamped here. - * Swap-start requests are serialized by the DCP command stack. - */ -static void dcp_arm_swapped_out_fbs(struct apple_dcp *dcp, u32 swap_id) -{ - struct dcp_fb_reference *entry; - unsigned long flags; - - spin_lock_irqsave(&dcp->swapped_out_lock, flags); - list_for_each_entry(entry, &dcp->swapped_out_fbs, head) { - if (entry->armed) - continue; - entry->swap_id = swap_id; - entry->armed = true; - } - spin_unlock_irqrestore(&dcp->swapped_out_lock, flags); -} - -/* - * Drop framebuffers whose unbinding swap has completed. - * - * Entries must be keyed to the swap that unbinds them, rather than the last - * swap that happened to have completed when the atomic commit ran. A later - * successful completion also retires entries whose own completion was lost; - * the signed difference keeps that comparison valid across a 32-bit wrap. - */ -static void dcp_release_swapped_out_fbs(struct apple_dcp *dcp) -{ - struct dcp_fb_reference *entry, *tmp; - unsigned long flags; - LIST_HEAD(done); - - /* - * last_swap_id / have_swap_complete are only ever touched by the RTKit - * worker, which is also the only caller of this function, so the lock - * is here for the list alone. - */ - spin_lock_irqsave(&dcp->swapped_out_lock, flags); - while (!list_empty(&dcp->swapped_out_fbs)) { - entry = list_first_entry(&dcp->swapped_out_fbs, - struct dcp_fb_reference, head); - - /* Newest entries; their swap has not been started yet. */ - if (!entry->armed) - break; - /* last_swap_id is 0 from kzalloc until a swap really lands. */ - if (!dcp->have_swap_complete) - break; - /* Hold the old framebuffer for one additional completed swap. */ - if ((s32)(dcp->last_swap_id - entry->swap_id) <= 0) - break; - - list_move_tail(&entry->head, &done); - } - spin_unlock_irqrestore(&dcp->swapped_out_lock, flags); - - /* drm_framebuffer_put() can sleep, so never call it under the lock. */ - list_for_each_entry_safe(entry, tmp, &done, head) { - if (entry->fb) - drm_framebuffer_put(entry->fb); - list_del(&entry->head); - kfree(entry); - } -} -#else -static void dcp_arm_swapped_out_fbs(struct apple_dcp *dcp, u32 swap_id) { } - -static void dcp_release_swapped_out_fbs(struct apple_dcp *dcp) -{ - while (!list_empty(&dcp->swapped_out_fbs)) { - struct dcp_fb_reference *entry; - - entry = list_first_entry(&dcp->swapped_out_fbs, - struct dcp_fb_reference, head); - if (entry->swap_id == dcp->last_swap_id) - break; - if (entry->fb) - drm_framebuffer_put(entry->fb); - list_del(&entry->head); - kfree(entry); - } -} -#endif - static void dcpep_cb_swap_complete(struct apple_dcp *dcp, struct DCP_FW_NAME(dc_swap_complete_resp) *resp) { ktime_t now = ktime_get(); trace_iomfb_swap_complete(dcp, resp->swap_id); dcp->last_swap_id = resp->swap_id; + dcp_release_retained_framebuffers(dcp, resp->swap_id); dcp_drm_crtc_page_flip(dcp, now); -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - dcp->have_swap_complete = true; - dcp_release_swapped_out_fbs(dcp); -#endif if (dcp->crc_enabled) { u32 crc32 = 0; drm_crtc_add_crc_entry(&dcp->crtc->base, true, resp->swap_id, &crc32); @@ -1221,6 +1129,7 @@ static u64 dcpep_cb_get_time(struct apple_dcp *dcp) struct dcp_swap_cookie { struct kref refcount; struct completion done; + u32 status; u32 swap_id; }; @@ -1300,33 +1209,42 @@ static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) { } static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; + struct dcp_swap_cookie *info = cookie; + u32 swap_id = DCP_FW_UNION(dcp->swap).swap.swap_id; + u32 status = resp ? resp->ret : ~0U; - if (cookie) { - struct dcp_swap_cookie *info = cookie; + if (status) { + dev_err(dcp->dev, "swap_clear failed! status %u\n", status); + dcp_unarm_retained_framebuffers(dcp, swap_id); + dcp_drm_crtc_vblank(dcp->crtc); + } + if (info) { + WRITE_ONCE(info->status, status); complete(&info->done); kref_put(&info->refcount, release_swap_cookie); } - - if (resp->ret) { - dev_err(dcp->dev, "swap_clear failed! status %u\n", resp->ret); - dcp_drm_crtc_vblank(dcp->crtc); - return; - } - - dcp_release_swapped_out_fbs(dcp); } static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_start_resp) *resp = data; + struct dcp_swap_cookie *info = cookie; + + if (!resp || resp->ret) { + if (info) { + WRITE_ONCE(info->status, resp ? resp->ret : ~0U); + complete(&info->done); + kref_put(&info->refcount, release_swap_cookie); + } + dcp_drm_crtc_vblank(dcp->crtc); + return; + } DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; - dcp_arm_swapped_out_fbs(dcp, resp->swap_id); + dcp_arm_retained_framebuffers(dcp, resp->swap_id); - if (cookie) { - struct dcp_swap_cookie *info = cookie; + if (info) info->swap_id = resp->swap_id; - } dcp_h17p_fix_swap_surfaces(dcp); dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swap_cleared, cookie); @@ -1377,6 +1295,7 @@ void DCP_FW_NAME(iomfb_poweron)(struct apple_dcp *dcp) if (!cookie) return; + cookie->status = ~0U; init_completion(&cookie->done); kref_init(&cookie->refcount); /* increase refcount to ensure the receiver has a reference */ @@ -1426,14 +1345,26 @@ void DCP_FW_NAME(iomfb_poweron)(struct apple_dcp *dcp) static void complete_set_powerstate(struct apple_dcp *dcp, void *out, void *cookie) { + struct dcp_set_power_state_resp *resp = out; struct dcp_wait_cookie *wait = cookie; if (wait) { + WRITE_ONCE(wait->status, resp ? resp->ret : ~0U); complete(&wait->done); kref_put(&wait->refcount, release_wait_cookie); } } +static bool dcp_power_stop_confirmed(struct dcp_wait_cookie *cookie) +{ + /* H17P stop semantics still require hardware qualification. */ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + return false; +#else + return READ_ONCE(cookie->status) == 0; +#endif +} + static void last_client_closed_poff(struct apple_dcp *dcp, void *out, void *cookie) { struct DCP_FW_NAME(dcp_set_power_state_req) power_req = { @@ -1471,6 +1402,7 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) return; + cookie->status = ~0U; init_completion(&cookie->done); kref_init(&cookie->refcount); /* increase refcount to ensure the receiver has a reference */ @@ -1516,6 +1448,8 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(50)); swap_id = cookie->swap_id; + if (READ_ONCE(cookie->status)) + ret = 0; kref_put(&cookie->refcount, release_swap_cookie); if (ret <= 0) { dcp->crashed = true; @@ -1527,6 +1461,7 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) poff_cookie = kzalloc(sizeof(*poff_cookie), GFP_KERNEL); if (!poff_cookie) return; + poff_cookie->status = ~0U; init_completion(&poff_cookie->done); kref_init(&poff_cookie->refcount); /* increase refcount to ensure the receiver has a reference */ @@ -1545,10 +1480,15 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) if (ret == 0) dev_warn(dcp->dev, "setPowerState(0) timeout %u ms\n", 1000); - else if (ret > 0) + else if (ret > 0) { dev_dbg(dcp->dev, "setPowerState(0) finished with %d ms to spare", jiffies_to_msecs(ret)); + if (dcp_power_stop_confirmed(poff_cookie)) + dcp_release_all_retained_framebuffers(dcp); + else + dev_warn(dcp->dev, "scanout stop was not confirmed\n"); + } kref_put(&poff_cookie->refcount, release_wait_cookie); @@ -1583,6 +1523,7 @@ void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp) cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) return; + cookie->status = ~0U; init_completion(&cookie->done); kref_init(&cookie->refcount); /* increase refcount to ensure the receiver has a reference */ @@ -1775,24 +1716,29 @@ TRAMPOLINE_OUT(trampoline_create_backlight_service, dcpep_cb_create_backlight_se static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; + u32 swap_id = DCP_FW_UNION(dcp->swap).swap.swap_id; if (resp->ret) { dev_err(dcp->dev, "swap failed! status %u\n", resp->ret); + dcp_unarm_retained_framebuffers(dcp, swap_id); dcp_drm_crtc_vblank(dcp->crtc); return; } dcp->swap_start = ktime_get(); dcp->swap_submit_timestamp = arch_timer_read_counter(); - - dcp_release_swapped_out_fbs(dcp); } static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_start_resp) *resp = data; + if (!resp || resp->ret) { + dev_err(dcp->dev, "swap_start was rejected\n"); + dcp_drm_crtc_vblank(dcp->crtc); + return; + } DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; - dcp_arm_swapped_out_fbs(dcp, resp->swap_id); + dcp_arm_retained_framebuffers(dcp, resp->swap_id); trace_iomfb_swap_submit(dcp, resp->swap_id); dcp_h17p_fix_swap_surfaces(dcp); @@ -1910,6 +1856,7 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, return -ENOMEM; } + cookie->status = ~0U; init_completion(&cookie->done); kref_init(&cookie->refcount); /* increase refcount to ensure the receiver has a reference */ @@ -2024,44 +1971,6 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru l = apl_plane->iomfb_surf; req->swap.swap_enabled |= BIT(l); - if (old_state->fb && new_state->fb != old_state->fb) { - /* - * Race condition between a framebuffer unbind getting - * swapped out and GEM unreferencing a framebuffer. If - * we lose the race, the display gets IOVA faults and - * the DCP crashes. We need to extend the lifetime of - * the drm_framebuffer (and hence the GEM object) until - * after we get a swap complete for the swap unbinding - * it. - */ - struct dcp_fb_reference *entry = - kzalloc(sizeof(*entry), GFP_KERNEL); - if (entry) { -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - unsigned long flags; - - entry->fb = old_state->fb; - /* - * Left unarmed: the id of the swap that - * unbinds this framebuffer is not known until - * the firmware answers swap_start. - */ - spin_lock_irqsave(&dcp->swapped_out_lock, - flags); - list_add_tail(&entry->head, - &dcp->swapped_out_fbs); - spin_unlock_irqrestore(&dcp->swapped_out_lock, - flags); -#else - entry->fb = old_state->fb; - entry->swap_id = dcp->last_swap_id; - list_add_tail(&entry->head, - &dcp->swapped_out_fbs); -#endif - } - drm_framebuffer_get(old_state->fb); - } - if (!new_state->fb || !new_state->visible) { continue; } @@ -2352,6 +2261,21 @@ void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp) struct DCP_FW_NAME(dcp_set_power_state_req) req = { /* defaults are ok */ }; + struct dcp_wait_cookie *cookie; + int ret; - dcp_set_power_state(dcp, DCP_POWER_OOB, &req, NULL, NULL); + cookie = kzalloc_obj(*cookie); + if (!cookie) + return; + cookie->status = ~0U; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + kref_get(&cookie->refcount); + dcp_set_power_state(dcp, DCP_POWER_OOB, &req, complete_set_powerstate, cookie); + ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(1000)); + if (ret > 0 && dcp_power_stop_confirmed(cookie)) + dcp_release_all_retained_framebuffers(dcp); + else + dev_warn(dcp->dev, "shutdown scanout stop was not confirmed\n"); + kref_put(&cookie->refcount, release_wait_cookie); } diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index 01243a39bb653..d068cf4c07298 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -5,6 +5,7 @@ #include "plane.h" +#include "dcp-internal.h" #include "iomfb_internal.h" #include "iomfb_plane.h" @@ -14,6 +15,7 @@ #include #include #include +#include #include #include @@ -417,13 +419,56 @@ static void apple_plane_atomic_update(struct drm_plane *plane, new_state->iova = obj->dma_addr + base->fb->offsets[0]; } +static void apple_plane_cleanup_fb(struct drm_plane *plane, + struct drm_plane_state *state) +{ + struct apple_plane_state *apple_state = to_apple_plane_state(state); + struct dcp_fb_reference *entry = apple_state->retirement; + + if (!entry) + return; + + apple_state->retirement = NULL; + drm_framebuffer_put(entry->fb); + kfree(entry); +} + +static int apple_plane_prepare_fb(struct drm_plane *plane, + struct drm_plane_state *state) +{ + struct drm_plane_state *old; + struct dcp_fb_reference *entry; + int ret; + + ret = drm_gem_plane_helper_prepare_fb(plane, state); + if (ret) + return ret; + + old = drm_atomic_get_old_plane_state(state->state, plane); + if (!old || !old->crtc || !old->fb || old->fb == state->fb) + return 0; + + entry = kzalloc_obj(*entry); + if (!entry) + return -ENOMEM; + + drm_framebuffer_get(old->fb); + entry->fb = old->fb; + to_apple_plane_state(state)->retirement = entry; + return 0; +} + static const struct drm_plane_helper_funcs apple_primary_plane_helper_funcs = { + .prepare_fb = apple_plane_prepare_fb, + .cleanup_fb = apple_plane_cleanup_fb, .atomic_check = apple_plane_atomic_check, .atomic_update = apple_plane_atomic_update, .get_scanout_buffer = drm_fb_dma_get_scanout_buffer, }; static const struct drm_plane_helper_funcs apple_plane_helper_funcs = { + .prepare_fb = apple_plane_prepare_fb, + .cleanup_fb = apple_plane_cleanup_fb, .atomic_check = apple_plane_atomic_check, .atomic_update = apple_plane_atomic_update, }; @@ -432,8 +477,10 @@ static const struct drm_plane_helper_funcs apple_plane_helper_funcs = { static void apple_plane_reset(struct drm_plane *plane) { struct apple_plane_state *state = to_apple_plane_state(plane->state); - if (state) + if (state) { + apple_plane_cleanup_fb(plane, &state->base); __drm_atomic_helper_plane_destroy_state(&state->base); + } kfree(state); plane->state = NULL; @@ -465,19 +512,19 @@ apple_plane_duplicate_state(struct drm_plane *plane) return &apple_plane_state->base; } -// void apple_plane_destroy_state(struct drm_plane *plane, -// struct drm_plane_state *state) -// { -// drm_atomic_helper_plane_destroy_state(plane, state); -// } +static void apple_plane_destroy_state(struct drm_plane *plane, + struct drm_plane_state *state) +{ + apple_plane_cleanup_fb(plane, state); + drm_atomic_helper_plane_destroy_state(plane, state); +} static const struct drm_plane_funcs apple_plane_funcs = { .update_plane = drm_atomic_helper_update_plane, .disable_plane = drm_atomic_helper_disable_plane, .reset = apple_plane_reset, .atomic_duplicate_state = apple_plane_duplicate_state, - // .atomic_destroy_state = apple_plane_destroy_state, - .atomic_destroy_state = drm_atomic_helper_plane_destroy_state, + .atomic_destroy_state = apple_plane_destroy_state, }; /* diff --git a/drivers/gpu/drm/apple/plane.h b/drivers/gpu/drm/apple/plane.h index 1a467a3d15675..5d6b25912b8ce 100644 --- a/drivers/gpu/drm/apple/plane.h +++ b/drivers/gpu/drm/apple/plane.h @@ -19,12 +19,15 @@ struct apple_plane { #define to_apple_plane(x) container_of(x, struct apple_plane, base) +struct dcp_fb_reference; + struct apple_plane_state { struct drm_plane_state base; struct dcp_surface surf; struct dcp_rect src_rect; struct dcp_rect dst_rect; u64 iova; + struct dcp_fb_reference *retirement; }; #define to_apple_plane_state(x) container_of(x, struct apple_plane_state, base) From ecff7f5db94aafff6df4c8e2031aa447aa6d22f2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 01:07:37 +1000 Subject: [PATCH 1054/2680] dt-bindings: display: apple,dcp: document the DCP and display subsystem Describe the existing Asahi display nodes and the T8140 internal and external DCP compatibles. The piodma child and panel properties are included, and apple,iomfb-surfaces retains its per-slot enable meaning. T8140 DCPs must not specify a reset, because resetting the inherited display CPU can hang the machine (DT-5, DT-6). [ryan: split by topic: only the drm/apple part and its selftests are here; the MAINTAINERS lines go to 900-config-tooling] Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/display/apple,dcp.yaml | 256 ++++++++++++++++++ .../display/apple,display-subsystem.yaml | 29 ++ 2 files changed, 285 insertions(+) create mode 100644 Documentation/devicetree/bindings/display/apple,dcp.yaml create mode 100644 Documentation/devicetree/bindings/display/apple,display-subsystem.yaml diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml new file mode 100644 index 0000000000000..47610c176cfd2 --- /dev/null +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -0,0 +1,256 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/apple,dcp.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple display coprocessor + +maintainers: + - Ryan Murray + +description: + Apple's Display Co-Processor (DCP) drives internal and external displays + through an RTKit mailbox. Its piodma child uses a separate display DART + stream from the coprocessor itself. + +properties: + compatible: + oneOf: + - items: + - enum: + - apple,t8103-dcp + - apple,t8112-dcp + - apple,t6000-dcp + - apple,t6020-dcp + - apple,t8140-dcp + - const: apple,dcp + - items: + - enum: + - apple,t8103-dcpext + - apple,t8112-dcpext + - apple,t6000-dcpext + - apple,t6020-dcpext + - const: apple,dcpext + - items: + - const: apple,t8140-dcpext + - const: apple,dcp + + reg: + minItems: 2 + maxItems: 6 + + reg-names: + minItems: 2 + items: + - const: coproc + - const: disp-0 + - const: disp-1 + - const: disp-2 + - const: disp-3 + - const: disp-4 + + mboxes: + maxItems: 1 + + mbox-names: + items: + - const: mbox + + iommus: + maxItems: 1 + + power-domains: + maxItems: 1 + + resets: + maxItems: 1 + + clocks: + minItems: 1 + maxItems: 2 + + phys: + minItems: 1 + maxItems: 8 + + phy-names: + minItems: 1 + maxItems: 8 + items: + pattern: '^(dp-phy|typec[0-9]+)$' + + apple,bw-scratch: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: PMGR bandwidth scratch resource, display index and offset. + + apple,bw-doorbell: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: Optional PMGR bandwidth doorbell resource and display index. + + apple,iomfb-surfaces: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 4 + maxItems: 4 + items: + enum: [0, 1] + description: Per-slot enable flags, in slot order. + + apple,firmware-version: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 3 + maxItems: 5 + description: Version reported by the loaded display firmware. + + apple,firmware-compat: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 3 + maxItems: 5 + description: Loader-provided firmware ABI version used for IOMFB selection. + + apple,connector-type: + enum: [HDMI-A, DP, USB-C] + + apple,dcp-index: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Display coprocessor index on multi-DCP systems. + + apple,dptx-phy: + $ref: /schemas/types.yaml#/definitions/uint32 + description: DisplayPort transmitter PHY selector. + + apple,dptx-die: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Die containing the DisplayPort transmitter. + + apple,notch-height: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Height in pixels reserved for the display notch. + + apple,typec-mux-indices: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 4 + description: Crossbar state for each Type-C display route. + + apple,typec-dptx-phys: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 4 + description: DisplayPort PHY index for each Type-C display route. + + mux-controls: + $ref: /schemas/types.yaml#/definitions/phandle-array + minItems: 1 + maxItems: 8 + + mux-control-names: + minItems: 1 + maxItems: 8 + items: + pattern: '^(dp-xbar|typec[0-9]+)$' + + mux-index: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Crossbar state for a fixed output. + + hdmi-hpd-gpios: + maxItems: 1 + + hdmi-pwren-gpios: + maxItems: 1 + + dp2hdmi-pwren-gpios: + maxItems: 1 + + '#address-cells': + const: 2 + + '#size-cells': + const: 2 + + piodma: + type: object + properties: + iommus: + maxItems: 1 + required: + - iommus + additionalProperties: false + + panel: + type: object + properties: + compatible: + oneOf: + - const: apple,panel + - items: + - pattern: '^apple,panel-j[0-9]+$' + - const: apple,panel + - items: + - pattern: '^apple,panel-j[0-9]+$' + - const: apple,panel-mini-led + - const: apple,panel + width-mm: + description: Physical panel width. + height-mm: + description: Physical panel height. + adj-height-mm: + description: Visible height below the notch. + apple,max-brightness: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Maximum firmware backlight level. + required: + - compatible + additionalProperties: false + + typec-routes: + type: object + properties: + '#address-cells': + const: 1 + '#size-cells': + const: 0 + patternProperties: + '^route@[0-3]$': + type: object + properties: + reg: + maxItems: 1 + mode-switch: + type: boolean + port: + $ref: /schemas/graph.yaml#/properties/port + required: + - reg + - mode-switch + - port + additionalProperties: false + required: + - '#address-cells' + - '#size-cells' + additionalProperties: false + + ports: + $ref: /schemas/graph.yaml#/properties/ports + +required: + - compatible + - reg + - reg-names + - mboxes + - iommus + - apple,bw-scratch + +allOf: + - if: + properties: + compatible: + contains: + enum: ['apple,t8140-dcp', 'apple,t8140-dcpext'] + then: + properties: + resets: false + +unevaluatedProperties: false +... diff --git a/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml b/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml new file mode 100644 index 0000000000000..429e68b29d864 --- /dev/null +++ b/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/apple,display-subsystem.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple display subsystem + +maintainers: + - Ryan Murray + +description: + Component master for one or more Apple display coprocessors. Each IOMMU + entry describes a scanout DART stream owned by a DCP instance. + +properties: + compatible: + const: apple,display-subsystem + + iommus: + minItems: 1 + maxItems: 8 + +required: + - compatible + - iommus + +unevaluatedProperties: false +... From a7a341738281904c0e8b3b2cb5392af9efdad1cd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 6 Oct 2026 23:02:48 +1100 Subject: [PATCH 1055/2680] dt-bindings: display: apple,dcp: describe named handoff resources Describe the H17-generation iop-vbar window independently of the numbered display windows and include the existing T8142 compatible. Document the clock names consumed by the driver and the loader's reserved-memory lists. Permit partial surface-enable arrays: the public driver reads one flag per slot and accepts up to four entries. This preserves its existing meaning instead of requiring every device tree to provide all four slots. The named resources follow the public Apple DRM driver and the Linux-side J700 hardware-description observation. No reset is added to a device tree. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/display/apple,dcp.yaml | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index 47610c176cfd2..ac2286832ba7d 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -24,6 +24,7 @@ properties: - apple,t6000-dcp - apple,t6020-dcp - apple,t8140-dcp + - apple,t8142-dcp - const: apple,dcp - items: - enum: @@ -38,17 +39,18 @@ properties: reg: minItems: 2 - maxItems: 6 + maxItems: 7 reg-names: minItems: 2 items: - const: coproc - const: disp-0 - - const: disp-1 - - const: disp-2 - - const: disp-3 - - const: disp-4 + - enum: [disp-1, iop-vbar] + - enum: [disp-2, iop-vbar] + - enum: [disp-3, iop-vbar] + - enum: [disp-4, iop-vbar] + - const: iop-vbar mboxes: maxItems: 1 @@ -70,6 +72,20 @@ properties: minItems: 1 maxItems: 2 + clock-names: + minItems: 1 + items: + - const: clock-154 + - const: clock-194 + + memory-region: + minItems: 1 + maxItems: 64 + + memory-region-names: + minItems: 1 + maxItems: 64 + phys: minItems: 1 maxItems: 8 @@ -90,7 +106,7 @@ properties: apple,iomfb-surfaces: $ref: /schemas/types.yaml#/definitions/uint32-array - minItems: 4 + minItems: 1 maxItems: 4 items: enum: [0, 1] From a1d1a8bb461f5e06f89a6f29f9942d828bfd0e73 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 7 Oct 2026 14:14:32 +1100 Subject: [PATCH 1056/2680] drm: apple: preserve scanout in brightness-only commits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The brightness update creates a CRTC-only atomic state and marks its colour state changed. The atomic helpers do not add unchanged planes for this non-modesetting transaction, but the DCP present builder clears all surfaces and repopulates them only from planes in the atomic state. Add the CRTC's affected planes before committing so a static-screen brightness update re-presents the current scanout instead of a request without surfaces. Use the existing cached plane-state duplication and framebuffer references. Plane-state allocation and locking errors follow the same atomic backoff and cleanup path as the CRTC state. This uses the public DRM atomic-state contract and the existing Asahi present path for all firmware variants, without changing record layouts. [ryan: adapted to 7.2.10 — applied to Asahi's reworked drm_crtc_set_brightness(), which already balances the modeset locking that our commit a88aa68f9fd7 ("drm: apple: balance backlight modeset locking") fixed] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp_backlight.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 8531b51ac550a..6474888cab781 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -164,6 +164,11 @@ static int drm_crtc_set_brightness(struct apple_dcp *dcp) goto out; } + /* Re-present the current scanout when brightness is the only change. */ + ret = drm_atomic_add_affected_planes(commit, crtc); + if (ret) + goto out; + crtc_state->color_mgmt_changed |= true; ret = drm_atomic_commit(commit); From 08bb429ead4009d155955e883365a2801deedda8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 7 Oct 2026 16:59:37 +1100 Subject: [PATCH 1057/2680] drm: apple: initialise each parsed display mode A timing entry can populate its colour modes before being rejected as virtual. enumerate_modes() then reuses that output slot for the next entry. Clearing only the original allocation lets a later entry without ColorModes inherit those colours. Even the first such entry sees zero scores as valid, rather than reporting that no colour was selected. Reset the output and score for each timing entry and initialise the intermediate timing attributes. Initialise every colour entry with invalid field values so a partial dictionary cannot reuse stack data from a preceding colour entry. Complete dictionaries keep their existing colour selection and preferred-mode ranking. This follows the public TimingElements parser and the DRM-3 requirement that rejected entries cannot leak fields into subsequent modes. Sixteen own dictionary fixtures exercise the actual parser, including missing colour arrays, rejected entries, partial colours and preferred ranking. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/parser.c | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 7982aae874a3f..1a13196388b8b 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -370,7 +370,15 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, dcp_parse_foreach_in_array(handle, outer_it) { struct iterator it; bool is_virtual = true; - struct color_mode cmode; + struct color_mode cmode = { + .colorimetry = -1, + .depth = -1, + .dynamic_range = -1, + .eotf = -1, + .id = -1, + .pixel_encoding = -1, + .score = -1, + }; dcp_parse_foreach_in_dict(handle, it) { char *key = parse_string(it.handle); @@ -447,7 +455,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, { int ret = 0; struct iterator it; - struct dimension horiz, vert; + struct dimension horiz = {}, vert = {}; s64 min_vrr = 0, max_vrr = 0; s64 id = -1; s64 best_color_mode = -1; @@ -455,6 +463,14 @@ static int parse_mode(struct dcp_parse_ctx *handle, bool is_preferred = false; struct drm_display_mode *mode = &out->mode; + *out = (struct dcp_display_mode) { + .sdr_rgb.score = -1, + .sdr_444.score = -1, + .sdr.score = -1, + .best.score = -1, + }; + *score = -1; + dcp_parse_foreach_in_dict(handle, it) { char *key = parse_string(it.handle); @@ -590,7 +606,7 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, if (ret) return ERR_PTR(ret); - /* Start with a worst case allocation; parse_mode() only sets vrr. */ + /* Start with a worst case allocation. */ modes = kcalloc(it.len, sizeof(*modes), GFP_KERNEL); *count = 0; From 2a62eec0740f7a229a2e21f60c3e8d4cade8f06a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 7 Oct 2026 17:09:51 +1100 Subject: [PATCH 1058/2680] drm: apple: skip failed timing entries before parsing the next enumerate_modes() treats a failure to parse one timing entry as recoverable, but parse_mode() can return before consuming the rest of that entry. The next iteration then starts inside the rejected dictionary instead of at the next mode. Save the entry's starting position and skip the complete entry after a parse failure. If even the generic dictionary walker cannot skip it, return that error instead of continuing from an unknown boundary. Own public-format dictionary fixtures reproduce the failure with a wrongly typed timing score or colour ID followed by a valid mode. The following modes are now retained, including when a valid mode precedes the rejected one. Complete dictionaries keep the existing parse path. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/parser.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 1a13196388b8b..3cb9c4df72de1 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -614,13 +614,23 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, return ERR_PTR(-ENOMEM); for (; it.idx < it.len; ++it.idx) { + struct dcp_parse_ctx entry = *it.handle; + mode = &modes[*count]; ret = parse_mode(it.handle, mode, &score, width_mm, height_mm, notch_height, internal); /* Errors for a single mode are recoverable -- just skip it. */ - if (ret) + if (ret) { + /* A failed parse can stop in the middle of the dictionary. */ + *it.handle = entry; + ret = skip(it.handle); + if (ret) { + kfree(modes); + return ERR_PTR(ret); + } continue; + } /* Process a successful mode */ (*count)++; From e3d181ff4348b78643dfe759b38e80974cbad0e2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 7 Oct 2026 17:21:48 +1100 Subject: [PATCH 1059/2680] drm: apple: avoid aligned pointers to packed dictionary integers Dictionary tags are aligned to four bytes, so the following eight-byte integer need not be naturally aligned. parse_int_bound() already copies that value into aligned storage, but its source pointer still has an s64 type. Keep the source as an untyped byte pointer for memcpy(). Own public-format dictionary fixtures place integers at both four- and eight-byte boundaries. The actual parser now passes the alignment sanitizer as well as the mode enumeration checks without changing any wire bytes or integer bounds. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/parser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 3cb9c4df72de1..f597e5c5e9aec 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -181,7 +181,7 @@ static char *parse_string(struct dcp_parse_ctx *handle) static int parse_int_bound(struct dcp_parse_ctx *handle, s64 *value, s64 min, s64 max) { const void *tag = parse_tag_of_type(handle, DCP_TYPE_INT64); - const s64 *in; + const void *in; if (IS_ERR(tag)) return PTR_ERR(tag); From 401bee58e82f7b0569a11bbddf32b9f6f508fea9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 00:12:50 +1100 Subject: [PATCH 1060/2680] drm: apple: add present-coupled H17P backlight policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement the clean display SPEC BACKLIGHT-1..4 policy in the existing Asahi backlight path. Keep commanded nits and completed levels separate, preserve takeover, and cache requests while blanked or suspended. Internal H17P DPMS keeps the pipe and surface alive. Admit range, takeover and wire encoding through a narrow interface to the serialized present queue. No H17P brightness bytes or default nits are inferred. Keep policy storage at the end of the device and gate template changes at compile time so older firmware keeps its existing fields and generated code. [ryan: adapted to 7.2.10 — dcp_set_brightness() keeps Asahi's lock_single_interruptible() locking after the new H17P path] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 9 ++ drivers/gpu/drm/apple/dcp.c | 18 +++ drivers/gpu/drm/apple/dcp_backlight.c | 159 +++++++++++++++++++++++- drivers/gpu/drm/apple/dcp_backlight.h | 165 +++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 12 +- drivers/gpu/drm/apple/systemep.c | 6 + 6 files changed, 366 insertions(+), 3 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp_backlight.h diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index c7081bfd150c0..70ba67c793666 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -17,6 +17,7 @@ #include #include "dptxep.h" +#include "dcp_backlight.h" #include "iomfb.h" #include "iomfb_h17p.h" #include "iomfb_v12_3.h" @@ -336,6 +337,14 @@ struct apple_dcp { struct apple_connector *fixed_connector; struct apple_connector *typec_connector; int hdmi_hpd_irq; + + /* H17P policy shared by backlight, commit, and reply workers. */ + struct { + /* Protects state and callback installation. */ + spinlock_t lock; + struct dcp_backlight_state state; + void (*kick)(struct apple_dcp *dcp); + } backlight; }; void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 4e0977f304f33..349e254e4658d 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1494,6 +1494,13 @@ void dcp_poweron(struct platform_device *pdev) struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; + if (dcp_backlight_active(dcp) && dcp_has_panel(dcp)) { + ret = dcp_backlight_dpms(dcp, true); + if (ret) + dev_warn(dcp->dev, "backlight restore unavailable: %d\n", ret); + return; + } + if (dcp_is_typec_output(dcp)) { WRITE_ONCE(dcp->typec_crtc_off, false); @@ -1545,6 +1552,14 @@ void dcp_poweroff(struct platform_device *pdev) struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; + /* Internal H17P DPMS is a brightness present, never a pipe stop. */ + if (dcp_backlight_active(dcp) && dcp_has_panel(dcp)) { + ret = dcp_backlight_dpms(dcp, false); + if (ret) + dev_warn(dcp->dev, "backlight blank unavailable: %d\n", ret); + return; + } + if (dcp->avep) av_service_disconnect(dcp); @@ -1593,6 +1608,8 @@ static void dcp_work_register_backlight(struct work_struct *work) /* try to register backlight device, */ ret = dcp_backlight_register(dcp); + if (ret == -ENODATA) + goto out_unlock; if (ret) { dev_err(dcp->dev, "Unable to register backlight device\n"); dcp->brightness.maximum = 0; @@ -2236,6 +2253,7 @@ static int dcp_platform_probe(struct platform_device *pdev) INIT_LIST_HEAD(&dcp->swapped_out_fbs); mutex_init(&dcp->swapped_out_fbs_lock); + spin_lock_init(&dcp->backlight.lock); dcp->fw_compat = fw_compat; dcp->dev = dev; diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 6474888cab781..a3065e63f3c7d 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -82,13 +82,125 @@ static u32 brightness_part2[] = { }; +bool dcp_backlight_active(struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_H17P && + READ_ONCE(dcp->backlight.state.ready); +} + static int dcp_get_brightness(struct backlight_device *bd) { struct apple_dcp *dcp = bl_get_data(bd); + unsigned long flags; + u32 actual; + + if (dcp_backlight_active(dcp)) { + spin_lock_irqsave(&dcp->backlight.lock, flags); + actual = dcp->backlight.state.actual; + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return actual; + } return dcp->brightness.nits; } +int dcp_backlight_configure(struct apple_dcp *dcp, u32 maximum, + bool inherited_valid, u32 inherited, + bool default_valid, u32 default_nits, + void (*kick)(struct apple_dcp *dcp)) +{ + unsigned long flags; + int ret; + + if (dcp->fw_compat != DCP_FIRMWARE_H17P || !dcp_has_panel(dcp) || + !dcp->crtc || !kick) + return -EINVAL; + + mutex_lock(&dcp->bl_register_mutex); + spin_lock_irqsave(&dcp->backlight.lock, flags); + if (dcp->backlight.state.ready || dcp->brightness.bl_dev) { + ret = -EBUSY; + } else { + ret = dcp_bl_init(&dcp->backlight.state, maximum, + inherited_valid, inherited, + default_valid, default_nits); + if (!ret) + dcp->backlight.kick = kick; + } + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + mutex_unlock(&dcp->bl_register_mutex); + if (!ret) + schedule_work(&dcp->bl_register_wq); + return ret; +} + +bool dcp_backlight_seed(struct apple_dcp *dcp, u32 nits) +{ + unsigned long flags; + bool seeded; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + seeded = dcp_bl_seed(&dcp->backlight.state, nits); + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return seeded; +} + +static void dcp_backlight_kick(struct apple_dcp *dcp) +{ + void (*kick)(struct apple_dcp *dcp) = READ_ONCE(dcp->backlight.kick); + + if (kick) + kick(dcp); +} + +int dcp_backlight_dpms(struct apple_dcp *dcp, bool on) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + ret = dcp_bl_dpms(&dcp->backlight.state, on); + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + if (!ret) + dcp_backlight_kick(dcp); + return ret; +} + +int dcp_backlight_prepare(struct apple_dcp *dcp, bool have_surface, + struct dcp_backlight_present *present) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + ret = dcp_bl_prepare(&dcp->backlight.state, have_surface, present); + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return ret; +} + +bool dcp_backlight_complete(struct apple_dcp *dcp, u64 sequence, bool accepted) +{ + unsigned long flags; + bool completed; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + completed = dcp_bl_complete(&dcp->backlight.state, sequence, accepted); + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + /* Queue retries after the central queue has finished this transaction. */ + return completed; +} + +bool dcp_backlight_pending(struct apple_dcp *dcp) +{ + unsigned long flags; + bool pending; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + pending = dcp->backlight.state.dirty && !dcp->backlight.state.in_flight; + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return pending; +} + #define SCALE_FACTOR (1 << 10) static u32 interpolate(int val, int min, int max, u32 *tbl, size_t tbl_size) @@ -189,6 +301,11 @@ static int drm_crtc_set_brightness(struct apple_dcp *dcp) int dcp_backlight_update(struct apple_dcp *dcp) { + if (dcp_backlight_active(dcp)) { + dcp_backlight_kick(dcp); + return 0; + } + /* * Do not actively try to change brightness if no mode is set. * TODO: should this be reflected the in backlight's power property? @@ -207,9 +324,23 @@ int dcp_backlight_update(struct apple_dcp *dcp) static int dcp_set_brightness(struct backlight_device *bd) { + int ret = 0; struct apple_dcp *dcp = bl_get_data(bd); int brightness = backlight_get_brightness(bd); - int ret; + unsigned long flags; + + if (dcp_backlight_active(dcp)) { + /* Preserve the requested level even while the core forces zero. */ + spin_lock_irqsave(&dcp->backlight.lock, flags); + ret = dcp_bl_request(&dcp->backlight.state, + bd->props.brightness, + backlight_is_blank(bd), + bd->props.state & BL_CORE_SUSPENDED); + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + if (!ret) + dcp_backlight_kick(dcp); + return ret; + } ret = drm_modeset_lock_single_interruptible(&dcp->crtc->base.mutex); if (ret) @@ -223,22 +354,45 @@ static int dcp_set_brightness(struct backlight_device *bd) return dcp_backlight_update(dcp); } +static bool dcp_backlight_controls_device(struct backlight_device *bd, + struct device *display_dev) +{ + struct apple_dcp *dcp = bl_get_data(bd); + + return !display_dev || display_dev == dcp->dev || + (dcp->crtc && display_dev == dcp->crtc->base.dev->dev); +} + static const struct backlight_ops dcp_backlight_ops = { .options = BL_CORE_SUSPENDRESUME, .get_brightness = dcp_get_brightness, .update_status = dcp_set_brightness, + .controls_device = dcp_backlight_controls_device, }; int dcp_backlight_register(struct apple_dcp *dcp) { struct device *dev = dcp->dev; struct backlight_device *bl_dev; + unsigned long flags; struct backlight_properties props = { .type = BACKLIGHT_PLATFORM, .brightness = dcp->brightness.nits, .scale = BACKLIGHT_SCALE_LINEAR, }; props.max_brightness = min(dcp->brightness.maximum, MAX_BRIGHTNESS_PART2 - 1); + if (dcp_backlight_active(dcp)) { + spin_lock_irqsave(&dcp->backlight.lock, flags); + if (!dcp->backlight.state.ready) { + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return -ENODATA; + } + /* Freeze takeover before userspace can discover the device. */ + dcp->backlight.state.controlled = true; + props.brightness = dcp->backlight.state.target; + props.max_brightness = dcp->backlight.state.maximum; + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + } bl_dev = devm_backlight_device_register(dev, "apple-panel-bl", dev, dcp, &dcp_backlight_ops, &props); @@ -246,7 +400,8 @@ int dcp_backlight_register(struct apple_dcp *dcp) return PTR_ERR(bl_dev); dcp->brightness.bl_dev = bl_dev; - dcp->brightness.dac = calculate_dac(dcp, dcp->brightness.nits); + if (!dcp_backlight_active(dcp)) + dcp->brightness.dac = calculate_dac(dcp, dcp->brightness.nits); return 0; } diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h new file mode 100644 index 0000000000000..6d52a8f9eb7fe --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -0,0 +1,165 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Ryan Murray */ + +#ifndef __APPLE_DCP_BACKLIGHT_H__ +#define __APPLE_DCP_BACKLIGHT_H__ + +#include +#include +#include + +/* Commanded nits, independent of firmware encoding. No fade policy. */ +struct dcp_backlight_state { + u32 maximum; + u32 target; + u32 actual; + u32 sent_nits; + u64 sequence; + bool ready; + bool controlled; + bool dpms_off; + bool core_blank; + bool suspended; + bool dirty; + bool in_flight; +}; + +struct dcp_backlight_present { + u64 sequence; + u32 nits; +}; + +/* Inputs must come from admitted panel/loader records, never a guess. */ +static inline int dcp_bl_init(struct dcp_backlight_state *state, u32 maximum, + bool inherited_valid, u32 inherited, + bool default_valid, u32 default_nits) +{ + u32 nits; + + if (!maximum || maximum > INT_MAX) + return -EINVAL; + if (!inherited_valid && !default_valid) + return -ENODATA; + nits = inherited_valid ? inherited : default_nits; + if (nits > maximum) + return -ERANGE; + + *state = (struct dcp_backlight_state) { + .maximum = maximum, + .target = nits, + .actual = nits, + .ready = true, + }; + return 0; +} + +/* A decoded powerlog sample can seed takeover only before Linux control. */ +static inline bool dcp_bl_seed(struct dcp_backlight_state *state, u32 nits) +{ + if (!state->ready || state->controlled || nits > state->maximum) + return false; + state->target = nits; + state->actual = nits; + return true; +} + +static inline u32 dcp_bl_effective(const struct dcp_backlight_state *state) +{ + if (state->dpms_off || state->core_blank || state->suspended) + return 0; + return state->target; +} + +static inline int dcp_bl_request(struct dcp_backlight_state *state, u32 nits, + bool core_blank, bool suspended) +{ + if (!state->ready) + return -ENODATA; + if (nits > state->maximum) + return -ERANGE; + state->controlled = true; + if (state->target != nits || state->core_blank != core_blank || + state->suspended != suspended) { + state->target = nits; + state->core_blank = core_blank; + state->suspended = suspended; + state->dirty = true; + } + return 0; +} + +static inline int dcp_bl_dpms(struct dcp_backlight_state *state, bool on) +{ + if (!state->ready) + return -ENODATA; + state->controlled = true; + if (state->dpms_off != !on) { + state->dpms_off = !on; + state->dirty = true; + } + return 0; +} + +/* + * Called by the single outbound queue, with a pinned, accepted scanout or a + * prepared replacement. Soft-off never releases it. A brightness-only present + * must use the last accepted surface, not a rejected replacement's DRM state. + */ +static inline int dcp_bl_prepare(struct dcp_backlight_state *state, + bool have_surface, + struct dcp_backlight_present *present) +{ + if (!state->ready || !have_surface) + return -ENODATA; + if (state->in_flight) + return -EBUSY; + if (!state->dirty) + return -EALREADY; + + state->controlled = true; + state->in_flight = true; + state->dirty = false; + state->sent_nits = dcp_bl_effective(state); + present->nits = state->sent_nits; + present->sequence = ++state->sequence; + return 0; +} + +/* + * accepted means both accepted submit and its matching completed present. + * Rejection/serialization failure cancels the reservation without changing + * actual. This result is NEVER a framebuffer retirement authorization. + */ +static inline bool dcp_bl_complete(struct dcp_backlight_state *state, + u64 sequence, bool accepted) +{ + if (!state->in_flight || sequence != state->sequence) + return false; + state->in_flight = false; + if (accepted) + state->actual = state->sent_nits; + else + state->dirty = true; + return true; +} + +struct apple_dcp; + +/* + * kick only queues work; it must not submit a top-level RPC from the caller. + * The central IOMFB queue owns surface references, retries, and wire encoding. + * Configure before registration and keep the callback valid until teardown. + */ +bool dcp_backlight_active(struct apple_dcp *dcp); +int dcp_backlight_configure(struct apple_dcp *dcp, u32 maximum, + bool inherited_valid, u32 inherited, + bool default_valid, u32 default_nits, + void (*kick)(struct apple_dcp *dcp)); +bool dcp_backlight_seed(struct apple_dcp *dcp, u32 nits); +int dcp_backlight_dpms(struct apple_dcp *dcp, bool on); +int dcp_backlight_prepare(struct apple_dcp *dcp, bool have_surface, + struct dcp_backlight_present *present); +bool dcp_backlight_complete(struct apple_dcp *dcp, u64 sequence, bool accepted); +bool dcp_backlight_pending(struct apple_dcp *dcp); + +#endif /* __APPLE_DCP_BACKLIGHT_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 1fc32fdca9c95..3322391efca41 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -396,6 +396,11 @@ static void iomfb_cb_pr_publish(struct apple_dcp *dcp, struct iomfb_property *pr case IOMFB_PROPERTY_NITS_V13_3: case IOMFB_PROPERTY_NITS_V14_7: { +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + /* Configured H17P takeover uses only the admitted sample interface. */ + if (dcp_backlight_active(dcp)) + break; +#endif if (dcp_has_panel(dcp)) { dcp->brightness.nits = prop->value / dcp->brightness.scale; /* notify backlight device of the initial brightness */ @@ -1934,6 +1939,7 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru struct DCP_FW_NAME(dcp_swap_submit_req) *req = &DCP_FW_UNION(dcp->swap); int plane_idx, l; int has_surface = 0; + bool update_brightness; crtc_state = drm_atomic_get_new_crtc_state(commit, crtc); @@ -2042,7 +2048,11 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru req->swap.swap_completed = req->swap.swap_enabled; /* update brightness if changed */ - if (dcp_has_panel(dcp) && dcp->brightness.update) { + update_brightness = dcp_has_panel(dcp) && dcp->brightness.update; +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + update_brightness = update_brightness && !dcp_backlight_active(dcp); +#endif + if (update_brightness) { #if DCP_FW_LAYOUT_14_7(DCP_FW_VER) req->swap.bl_update = 1; #else diff --git a/drivers/gpu/drm/apple/systemep.c b/drivers/gpu/drm/apple/systemep.c index 9fe7a0ce495aa..3133cd539b7f1 100644 --- a/drivers/gpu/drm/apple/systemep.c +++ b/drivers/gpu/drm/apple/systemep.c @@ -75,6 +75,12 @@ static int powerlog_report(struct apple_epic_service *service, enum epic_subtype struct apple_dcp *dcp = service->ep->dcp; int ret; + /* H17P takeover decoding requires an admitted record. Later zero reports + * must not overwrite the cached target or completed level. + */ + if (dcp_backlight_active(dcp)) + return 0; + dev_dbg(dcp->dev, "systemep[ch:%u]: report type:%02x len:%zu\n", service->channel, type, data_size); From f4212cf62791902d01995cec062d3c4366e38266 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 00:12:50 +1100 Subject: [PATCH 1061/2680] drm: apple: test H17P backlight takeover and DPMS policy Exercise the clean SPEC takeover, cached-level restore and suspend ordering requirements. Verify that missing surfaces, busy transactions and rejected or stale completions do not change the completed level or lose pending requests. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Kconfig | 3 +- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/dcp_backlight_test.c | 208 +++++++++++++++++++++ 3 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/apple/dcp_backlight_test.c diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index c4eff8fd0c5b3..c225708dff19b 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -34,7 +34,8 @@ config DRM_APPLE_KUNIT_TEST depends on KUNIT default KUNIT_ALL_TESTS help - Build tests for the Apple display driver's AFK ring parser. + Build tests for the Apple display driver's AFK ring parser and + backlight policy (takeover, DPMS, suspend and present completion). The tests cover the header strides used by older firmware and larger rings, as well as malformed ring size combinations. This option is intended for developer test kernels. diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 63b4ea7cdca24..786e648244af0 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -20,3 +20,4 @@ appledrm-$(CONFIG_TRACING) += trace.o obj-$(CONFIG_DRM_APPLE) += appledrm.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk_test.o +obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_backlight_test.o diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c new file mode 100644 index 0000000000000..e990ea3e0bf06 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include + +#include "dcp_backlight.h" + +static void backlight_takeover_test(struct kunit *test) +{ + struct dcp_backlight_state state = {}; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 140, true, 36), 0); + KUNIT_EXPECT_EQ(test, state.target, 140U); + KUNIT_EXPECT_EQ(test, state.actual, 140U); + KUNIT_EXPECT_FALSE(test, state.dirty); + KUNIT_EXPECT_TRUE(test, dcp_bl_seed(&state, 400)); + KUNIT_EXPECT_EQ(test, state.target, 400U); + KUNIT_EXPECT_FALSE(test, dcp_bl_seed(&state, 601)); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, false, false), 0); + KUNIT_EXPECT_FALSE(test, dcp_bl_seed(&state, 0)); + KUNIT_EXPECT_EQ(test, state.target, 400U); + KUNIT_EXPECT_EQ(test, state.actual, 400U); + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 0, true, 36), 0); + KUNIT_EXPECT_EQ(test, state.target, 0U); + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, false, 0, true, 36), 0); + KUNIT_EXPECT_EQ(test, state.target, 36U); +} + +static void backlight_range_test(struct kunit *test) +{ + struct dcp_backlight_state state = {}; + + KUNIT_EXPECT_EQ(test, dcp_bl_init(&state, 600, false, 0, false, 0), -ENODATA); + KUNIT_EXPECT_FALSE(test, state.ready); + KUNIT_EXPECT_EQ(test, dcp_bl_request(&state, 140, false, false), -ENODATA); + KUNIT_EXPECT_EQ(test, dcp_bl_dpms(&state, false), -ENODATA); + KUNIT_EXPECT_EQ(test, dcp_bl_init(&state, 0, true, 0, false, 0), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_bl_init(&state, U32_MAX, true, 0, false, 0), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_bl_init(&state, 600, true, 601, false, 0), -ERANGE); + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 140, false, 0), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_request(&state, 601, false, false), -ERANGE); + KUNIT_EXPECT_EQ(test, state.target, 140U); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 600, false, false), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 600U); +} + +static void backlight_dpms_cycles_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + int i; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 400, false, 0), 0); + for (i = 0; i < 20; i++) { + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_EQ(test, state.actual, 0U); + KUNIT_EXPECT_EQ(test, state.target, 400U); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 400U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_EQ(test, state.actual, 400U); + } + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_EXPECT_EQ(test, state.target, 400U); +} + +static void backlight_blanked_target_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 400, false, 0), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 140, false, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_EQ(test, state.target, 140U); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 140U); + KUNIT_EXPECT_EQ(test, state.actual, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_EQ(test, state.actual, 140U); +} + +static void backlight_suspend_test(struct kunit *test) +{ + struct dcp_backlight_state state; + int off_first; + + for (off_first = 0; off_first < 2; off_first++) { + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 140, false, 0), 0); + if (off_first) + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 140, true, true), 0); + if (!off_first) + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 36, true, true), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 36, false, false), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 36U); + KUNIT_EXPECT_EQ(test, state.target, 36U); + } + /* A core blank also keeps the panel dark independently of DRM DPMS. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, true, false), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, false, false), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 400U); +} + +static void backlight_surface_and_busy_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present first, second; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 140, false, 0), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_prepare(&state, true, &first), -EALREADY); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 36, false, false), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_prepare(&state, false, &first), -ENODATA); + KUNIT_EXPECT_TRUE(test, state.dirty); + KUNIT_EXPECT_FALSE(test, state.in_flight); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &first), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_prepare(&state, true, &second), -EBUSY); + KUNIT_EXPECT_EQ(test, state.actual, 140U); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, false, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, first.sequence, true)); + KUNIT_EXPECT_EQ(test, state.actual, 36U); + KUNIT_EXPECT_EQ(test, state.target, 400U); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &second), 0); + KUNIT_EXPECT_EQ(test, second.nits, 400U); + KUNIT_EXPECT_FALSE(test, dcp_bl_complete(&state, first.sequence, true)); + KUNIT_EXPECT_TRUE(test, state.in_flight); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, second.sequence, true)); + KUNIT_EXPECT_EQ(test, state.actual, 400U); +} + +static void backlight_reject_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 140, false, 0), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, false, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, false)); + KUNIT_EXPECT_EQ(test, state.actual, 140U); + KUNIT_EXPECT_TRUE(test, state.dirty); + KUNIT_EXPECT_FALSE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_EQ(test, state.target, 400U); + KUNIT_EXPECT_EQ(test, state.actual, 0U); +} + +static void backlight_inflight_dpms_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 140, false, 0), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_TRUE(test, state.dirty); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 140U); +} + +static struct kunit_case backlight_cases[] = { + KUNIT_CASE(backlight_takeover_test), + KUNIT_CASE(backlight_range_test), + KUNIT_CASE(backlight_dpms_cycles_test), + KUNIT_CASE(backlight_blanked_target_test), + KUNIT_CASE(backlight_suspend_test), + KUNIT_CASE(backlight_surface_and_busy_test), + KUNIT_CASE(backlight_reject_test), + KUNIT_CASE(backlight_inflight_dpms_test), + {} +}; + +static struct kunit_suite backlight_suite = { + .name = "apple-dcp-backlight", + .test_cases = backlight_cases, +}; + +kunit_test_suite(backlight_suite); + +MODULE_LICENSE("Dual MIT/GPL"); From 9443e561e8439a1b9ea846411dbf211acc96df15 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 00:12:51 +1100 Subject: [PATCH 1062/2680] docs: gpu: describe Apple H17P backlight admission contract Document the clean SPEC backlight policy, central queue and surface ownership interface, missing admission records, and hardware gates. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/gpu/apple-dcp-backlight.rst | 129 ++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 Documentation/gpu/apple-dcp-backlight.rst diff --git a/Documentation/gpu/apple-dcp-backlight.rst b/Documentation/gpu/apple-dcp-backlight.rst new file mode 100644 index 0000000000000..e47e7f3b4dd4b --- /dev/null +++ b/Documentation/gpu/apple-dcp-backlight.rst @@ -0,0 +1,129 @@ +.. SPDX-License-Identifier: GPL-2.0-only OR MIT + +Apple DCP H17P backlight integration +=================================== + +The backlight policy implements the clean display specification's BACKLIGHT-1 +through BACKLIGHT-4, with IOMFB-3 and LIFETIME-1 through LIFETIME-3 constraints. +It extends the existing Asahi backlight driver. It contains no H17P brightness +wire layout, calibration table, property command, or guessed initial level. +Policy activation requires explicit successful configuration. Existing +unconfigured H17P sessions, including T8142, keep their established DAC, +property/powerlog and DPMS paths. Older firmware remains on its existing path. +Neo startup must configure the policy before exposing the backlight or starting +its DPMS/brightness traffic; a firmware-version match alone does not activate +this policy. + +Admission and startup +--------------------- + +The central display worker must call ``dcp_backlight_configure()`` from process +context after linking the CRTC and admitting all required records. Its arguments +are the validated maximum commanded nits, an optional inherited level, an +optional documented ADT default, and a callback that schedules the central +outbound queue. Maximum must fit the backlight core's signed integer range. +An inherited zero is valid and takes precedence over the default. With neither +level available, configuration returns ``-ENODATA``. The policy backlight is registered only after configuration succeeds. +Configuration must precede any existing backlight registration; it returns +``-EBUSY`` if a device has already been published. Configuration does not send a command. + +The device exposes ``apple-panel-bl`` with platform type, linear scale and nits +units. Its maximum is the supplied ceiling; the older 509-nit DAC cap does not +apply to configured H17P policy. Consumer matching accepts the DCP device and its DRM device. +Registration freezes takeover before publishing the backlight. A decoded +powerlog value may seed the state through ``dcp_backlight_seed()`` before that +point. All samples after Linux takes control, including per-frame zeros, are +ignored. The older property and powerlog parsers do not seed an active policy implicitly. +Gather a sample before configuration and pass it as the inherited argument if +asynchronous registration could otherwise win the race. + +``brightness`` is the requested restore target. ``actual_brightness`` is the +last completed commanded level, initially the inherited/default estimate. This +is not a photometric measurement. Updates step immediately; no kernel fade or +initial-brightness/blank-level override is introduced. + +Queue and lifetime contract +--------------------------- + +The callback only schedules/coalesces work. It must not submit an RPC directly, +take DRM modeset locks, or run a present inline in a receive/commit callback. +It remains valid until the central queue and backlight workers are drained at +teardown. Policy access is protected by the brightness spinlock; the callback +is called after dropping it. Configuration also takes the registration mutex. + +The single outbound queue handles normal and brightness-only presents. A static +or inactive screen uses the last *accepted* surface, including its cached +geometry and IOVA. It must hold that surface's framebuffer reference and mapping +before calling ``dcp_backlight_prepare()`` with ``have_surface = true``. A DRM +plane state containing a rejected replacement is not an accepted surface. +The policy's boolean is a caller assertion, not a framebuffer reference. +Allocate any retirement bookkeeping during atomic prepare, never here. + +Prepare reserves a sequence and returns commanded nits. ``-ENODATA`` means no +configuration or surface; ``-EBUSY`` means a prior brightness transaction is +outstanding; ``-EALREADY`` means no brightness change is pending. These results +do not clear pending work. The serializer places the returned level only in +admitted fields of the next valid present. A normal present without a brightness +change keeps the inherited/completed level; its clean layout must support that. +The sequence is a software cookie, not a wire present ID. + +Associate the cookie with the central transaction and its strict wire ID. Call +``dcp_backlight_complete(sequence, true)`` only after both accepted submission +and matching completion. On serialization/submission rejection call it with +``false``; actual remains unchanged and the update is pending again. Wrong or +duplicate sequence callbacks have no effect. On a firmware crash retain the +surface and stop new submissions; there is no local reset/retry ladder. +After a transaction drains, inspect ``dcp_backlight_pending()`` and schedule the +next update or defer until the pipe becomes usable. Requests received during a +transaction remain pending. The central queue must bound static-screen latency +to 200 ms; this still requires a hardware timing test. + +A brightness-only completion never authorizes framebuffer retirement, including +retirement associated with a rejected replacement. Existing generic retention +helpers must be called only for the corresponding accepted replacement. This +policy never arms, releases, or unmaps a framebuffer. + +DPMS and suspend +---------------- + +Internal-panel H17P ``dcp_poweroff()`` and ``dcp_poweron()`` change the policy and +schedule the retained-surface queue. They bypass the legacy pipe-off/on path, +including surface clearing and final release. Until configuration, they warn +and leave the pipe intact; they cannot claim a completed hardware blank. +Startup must independently establish/adopt the live pipe before enabling the +policy. External outputs retain their existing power handling. + +DPMS off, a backlight core blank, or backlight core suspend forces effective nits +to zero while preserving the user target. Brightness changes during any blank +update the target. DPMS on restores it only when core blank/suspend is cleared. +The backlight core's suspend/resume callbacks feed this state; the central PM +worker still owns queue draining, domain lifetime and resume order. Do not call +unqualified H17P sleep/pipe-off methods as a suspend fallback. Keep the last +accepted surface mapped across soft-off and resume. + +Required records and qualification +---------------------------------- + +The following SPEC admission records must supply facts before production use: + +* R-ADT: panel ceiling/calibration in commanded nits and a documented default + if no inherited sample is available. No expected 525-nit ceiling is hardcoded. +* R-BACKLIGHT: takeover sample source, units, validity and timing; decoded + powerlog semantics; commanded-nits conversion, valid update/enable fields, + field offsets, widths and byte order in the present. +* R-SWAP: complete surface layout and successful submit/completion correlation + for normal and retained-surface presents, including inactive CRTC handling. +* R-DPMS: which admitted brightness/property actions blank and restore the live + pipe without clearing its surface, plus wake ordering. No new numeric property + or guessed DPMS payload is emitted by this policy. +* R-SERIALIZE: ordered brightness/present/property/diagnostic timeline and busy + deferral through the central queue; static-screen update latency. +* R-TEARDOWN: safe final release and suspend/resume boundary. Soft-off supplies + no such release boundary. + +Run ``apple-dcp-backlight`` KUnit tests with ``CONFIG_DRM_APPLE_KUNIT_TEST``. +They cover takeover/default/zero handling, range validation, twenty DPMS cycles, +cached restore, both suspend/DPMS orders, static-surface deferral, busy sequencing, +rejection and stale completion. They do not qualify firmware fields or GNOME. +Hardware acceptance requires O-BACKLIGHT, O-BACKLIGHT-TAKEOVER (10 Hz PBwo with +no jump above 0.05 W), O-BACKLIGHT-CRC, O-DPMS (twenty cycles), and O-SUSPEND. From 0392c491bfdb6c375205bde527c2b37a457d7318 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 00:53:23 +1100 Subject: [PATCH 1063/2680] drm: apple: reject truncated firmware dictionary values Dictionary skips must validate payload bounds just like parsed values. Stop at the first error, bound recursive nesting, and propagate invalid container and skipped-value errors to each caller. Reject length truncation and cursor overflow, and preserve the audio look-ahead cursor until a complete key matches. Handle string allocation failure and failed event strings explicitly. These checks follow the existing public dictionary format and do not change valid payloads. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/parser.c | 199 ++++++++++++++++++++++----------- 1 file changed, 136 insertions(+), 63 deletions(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index f597e5c5e9aec..6673500886cca 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -35,11 +35,12 @@ struct dcp_parse_tag { static const void *parse_bytes(struct dcp_parse_ctx *ctx, size_t count) { - const void *ptr = ctx->blob + ctx->pos; + const void *ptr; - if (ctx->pos + count > ctx->len) + if (ctx->pos > ctx->len || count > ctx->len - ctx->pos) return ERR_PTR(-EINVAL); + ptr = ctx->blob + ctx->pos; ctx->pos += count; return ptr; } @@ -54,6 +55,8 @@ static const struct dcp_parse_tag *parse_tag(struct dcp_parse_ctx *ctx) const struct dcp_parse_tag *tag; /* Align to 32-bits */ + if (ctx->pos > U32_MAX - 3) + return ERR_PTR(-EINVAL); ctx->pos = round_up(ctx->pos, 4); tag = parse_bytes(ctx, sizeof(struct dcp_parse_tag)); @@ -81,38 +84,45 @@ static const struct dcp_parse_tag *parse_tag_of_type(struct dcp_parse_ctx *ctx, return tag; } -static int skip(struct dcp_parse_ctx *handle) +/* Bound stack use when ignoring nested firmware dictionaries. */ +#define DCP_PARSE_MAX_DEPTH 32 + +static int skip_value(struct dcp_parse_ctx *handle, unsigned int depth) { - const struct dcp_parse_tag *tag = parse_tag(handle); - int ret = 0; - int i; + const struct dcp_parse_tag *tag; + const void *data; + unsigned int count, i; + int ret; + if (depth >= DCP_PARSE_MAX_DEPTH) + return -EINVAL; + + tag = parse_tag(handle); if (IS_ERR(tag)) return PTR_ERR(tag); switch (tag->type) { case DCP_TYPE_DICTIONARY: - for (i = 0; i < tag->size; ++i) { - ret |= skip(handle); /* key */ - ret |= skip(handle); /* value */ - } - - return ret; - case DCP_TYPE_ARRAY: - for (i = 0; i < tag->size; ++i) - ret |= skip(handle); + count = tag->size; + if (tag->type == DCP_TYPE_DICTIONARY) + count *= 2; - return ret; + for (i = 0; i < count; ++i) { + ret = skip_value(handle, depth + 1); + if (ret) + return ret; + } + return 0; case DCP_TYPE_INT64: - handle->pos += sizeof(s64); - return 0; + data = parse_bytes(handle, sizeof(s64)); + break; case DCP_TYPE_STRING: case DCP_TYPE_BLOB: - handle->pos += tag->size; - return 0; + data = parse_bytes(handle, tag->size); + break; case DCP_TYPE_BOOL: return 0; @@ -120,6 +130,13 @@ static int skip(struct dcp_parse_ctx *handle) default: return -EINVAL; } + + return IS_ERR(data) ? PTR_ERR(data) : 0; +} + +static int skip(struct dcp_parse_ctx *handle) +{ + return skip_value(handle, 0); } #if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) @@ -136,23 +153,19 @@ static int skip_pair(struct dcp_parse_ctx *handle) static bool consume_string(struct dcp_parse_ctx *ctx, const char *specimen) { + struct dcp_parse_ctx probe = *ctx; const struct dcp_parse_tag *tag; const char *key; - ctx->pos = round_up(ctx->pos, 4); - if (ctx->pos + sizeof(*tag) + strlen(specimen) - 1 > ctx->len) - return false; - tag = ctx->blob + ctx->pos; - key = ctx->blob + ctx->pos + sizeof(*tag); - if (tag->padding) + tag = parse_tag_of_type(&probe, DCP_TYPE_STRING); + if (IS_ERR(tag) || tag->size != strlen(specimen)) return false; - if (tag->type != DCP_TYPE_STRING || - tag->size != strlen(specimen) || - strncmp(key, specimen, tag->size)) + key = parse_bytes(&probe, tag->size); + if (IS_ERR(key) || memcmp(key, specimen, tag->size)) return false; - skip(ctx); + *ctx = probe; return true; } #endif @@ -172,6 +185,8 @@ static char *parse_string(struct dcp_parse_ctx *handle) return (void *)in; out = kmalloc(tag->size + 1, GFP_KERNEL); + if (!out) + return ERR_PTR(-ENOMEM); memcpy(out, in, tag->size); out[tag->size] = '\0'; @@ -261,15 +276,13 @@ static int iterator_begin(struct dcp_parse_ctx *handle, struct iterator *it, return 0; } -#define dcp_parse_foreach_in_array(handle, it) \ - for (iterator_begin(handle, &it, false); it.idx < it.len; ++it.idx) -#define dcp_parse_foreach_in_dict(handle, it) \ - for (iterator_begin(handle, &it, true); it.idx < it.len; ++it.idx) - int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx) { const u32 *header; + if (size > U32_MAX) + return -EINVAL; + *ctx = (struct dcp_parse_ctx) { .blob = blob, .len = size, @@ -291,7 +304,11 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) struct iterator it; int ret = 0; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); if (IS_ERR(key)) @@ -307,7 +324,7 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) else if (!strcmp(key, "PreciseSyncRate")) ret = parse_int(it.handle, &dim->precise_sync_rate); else - skip(it.handle); + ret = skip(it.handle); if (!IS_ERR_OR_NULL(key)) kfree(key); @@ -367,7 +384,11 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, out->sdr.score = -1; out->best.score = -1; - dcp_parse_foreach_in_array(handle, outer_it) { + ret = iterator_begin(handle, &outer_it, false); + if (ret) + return ret; + + for (; outer_it.idx < outer_it.len; ++outer_it.idx) { struct iterator it; bool is_virtual = true; struct color_mode cmode = { @@ -380,7 +401,11 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, .score = -1, }; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); if (IS_ERR(key)) @@ -402,7 +427,7 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, else if (!strcmp(key, "Score")) ret = parse_int(it.handle, &cmode.score); else - skip(it.handle); + ret = skip(it.handle); if (!IS_ERR_OR_NULL(key)) kfree(key); @@ -471,13 +496,17 @@ static int parse_mode(struct dcp_parse_ctx *handle, }; *score = -1; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); if (IS_ERR(key)) ret = PTR_ERR(key); else if (is_virtual) - skip(it.handle); + ret = skip(it.handle); else if (!strcmp(key, "HorizontalAttributes")) ret = parse_dimension(it.handle, &horiz); else if (!strcmp(key, "VerticalAttributes")) @@ -497,7 +526,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, else if (!strcmp(key, "Score")) ret = parse_int(it.handle, score); else - skip(it.handle); + ret = skip(it.handle); if (!IS_ERR_OR_NULL(key)) kfree(key); @@ -654,7 +683,11 @@ int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, struct iterator it; s64 width_cm = 0, height_cm = 0; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); if (IS_ERR(key)) @@ -664,7 +697,7 @@ int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, else if (!strcmp(key, "MaxVerticalImageSize")) ret = parse_int(it.handle, &height_cm); else - skip(it.handle); + ret = skip(it.handle); if (!IS_ERR_OR_NULL(key)) kfree(key); @@ -692,7 +725,11 @@ int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, *name = ERR_PTR(-ENOENT); *class = ERR_PTR(-ENOENT); - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); if (IS_ERR(key)) { @@ -734,7 +771,7 @@ int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, if (!ret) parsed_unit = true; } else { - skip(it.handle); + ret = skip(it.handle); } kfree(key); @@ -847,14 +884,16 @@ static int parse_chmap(struct dcp_parse_ctx *handle, struct snd_pcm_chmap_elem * struct iterator it; int i, ret; - if (!chmap) { - skip(handle); - return 0; - } + if (!chmap) + return skip(handle); chmap->channels = 0; - dcp_parse_foreach_in_array(handle, it) { + ret = iterator_begin(handle, &it, false); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { for (i = 0; i < ARRAY_SIZE(chan_position_names); i++) if (consume_string(it.handle, chan_position_names[i].label)) break; @@ -882,7 +921,11 @@ static int parse_chan_layout_element(struct dcp_parse_ctx *handle, int ret; s64 nchans = 0; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { if (consume_string(it.handle, "ActiveChannelCount")) ret = parse_int(it.handle, &nchans); else if (consume_string(it.handle, "ChannelLayout")) @@ -907,7 +950,11 @@ static int parse_nchans_mask(struct dcp_parse_ctx *handle, unsigned int *mask) *mask = 0; - dcp_parse_foreach_in_array(handle, it) { + ret = iterator_begin(handle, &it, false); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { int nchans; ret = parse_chan_layout_element(it.handle, &nchans, NULL); @@ -927,7 +974,11 @@ static int parse_avep_element(struct dcp_parse_ctx *handle, struct iterator it; int ret; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { if (consume_string(handle, "StreamSampleRate")) ret = parse_sample_rate_bit(it.handle, &mask.rates); else if (consume_string(handle, "SampleSize")) @@ -965,12 +1016,20 @@ static int parse_mode_in_avep_element(struct dcp_parse_ctx *handle, struct dcp_parse_ctx save_handle; int ret; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { if (consume_string(it.handle, "AudioChannelLayoutElements")) { struct iterator inner_it; int nchans; - dcp_parse_foreach_in_array(it.handle, inner_it) { + ret = iterator_begin(it.handle, &inner_it, false); + if (ret) + return ret; + + for (; inner_it.idx < inner_it.len; ++inner_it.idx) { save_handle = *it.handle; ret = parse_chan_layout_element(inner_it.handle, &nchans, NULL); @@ -1023,7 +1082,11 @@ int parse_sound_constraints(struct dcp_parse_ctx *handle, hits->nchans = 0; } - dcp_parse_foreach_in_array(handle, it) { + ret = iterator_begin(handle, &it, false); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { ret = parse_avep_element(it.handle, sieve, hits); if (ret < 0) @@ -1042,7 +1105,11 @@ int parse_sound_mode(struct dcp_parse_ctx *handle, struct iterator it; int ret; - dcp_parse_foreach_in_array(handle, it) { + ret = iterator_begin(handle, &it, false); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { save_handle = *it.handle; ret = parse_avep_element(it.handle, sieve, NULL); @@ -1072,7 +1139,11 @@ int parse_system_log_mnits(struct dcp_parse_ctx *handle, struct dcp_system_ev_mn s64 timestamp = -1; bool type_match = false; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); if (IS_ERR(key)) { ret = PTR_ERR(key); @@ -1081,15 +1152,17 @@ int parse_system_log_mnits(struct dcp_parse_ctx *handle, struct dcp_system_ev_mn } else if (!strcmp(key, "iDAC")) { ret = parse_int(it.handle, &idac); } else if (!strcmp(key, "logEvent")) { - const char * value = parse_string(it.handle); - if (!IS_ERR_OR_NULL(value)) { + const char *value = parse_string(it.handle); + + ret = IS_ERR(value) ? PTR_ERR(value) : 0; + if (!ret) { type_match = strcmp(value, "Display (Event Forward)") == 0; kfree(value); } } else if (!strcmp(key, "timestamp")) { ret = parse_int(it.handle, ×tamp); } else { - skip(it.handle); + ret = skip(it.handle); } if (!IS_ERR_OR_NULL(key)) From c8fd7e1155be5d5da84a36fc879a94fb2a6d5b52 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 00:55:19 +1100 Subject: [PATCH 1064/2680] drm: apple: validate firmware timings before creating modes Initialize timing fields as absent and require an active area, total, porches, vertical refresh rate, timing ID and score. Validate each dimension before calculating the clock or narrowing geometry to DRM mode fields. Reject notch underflow, unrepresentable IDs and clocks, and sync intervals beyond the advertised total. The existing public dictionary parser supplies all inputs. Invalid timing entries remain recoverable so a later valid mode can still be advertised; explicitly zero porches remain valid. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/parser.c | 45 +++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 6673500886cca..6bd98e01a0791 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -304,6 +304,14 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) struct iterator it; int ret = 0; + *dim = (struct dimension) { + .active = -1, + .total = -1, + .front_porch = -1, + .sync_width = -1, + .precise_sync_rate = -1, + }; + ret = iterator_begin(handle, &it, true); if (ret) return ret; @@ -314,15 +322,15 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) if (IS_ERR(key)) ret = PTR_ERR(key); else if (!strcmp(key, "Active")) - ret = parse_int(it.handle, &dim->active); + ret = parse_int_bound(it.handle, &dim->active, 0, U16_MAX); else if (!strcmp(key, "Total")) - ret = parse_int(it.handle, &dim->total); + ret = parse_int_bound(it.handle, &dim->total, 0, U16_MAX); else if (!strcmp(key, "FrontPorch")) - ret = parse_int(it.handle, &dim->front_porch); + ret = parse_int_bound(it.handle, &dim->front_porch, 0, U16_MAX); else if (!strcmp(key, "SyncWidth")) - ret = parse_int(it.handle, &dim->sync_width); + ret = parse_int_bound(it.handle, &dim->sync_width, 0, U16_MAX); else if (!strcmp(key, "PreciseSyncRate")) - ret = parse_int(it.handle, &dim->precise_sync_rate); + ret = parse_int_bound(it.handle, &dim->precise_sync_rate, 1, U32_MAX); else ret = skip(it.handle); @@ -333,6 +341,11 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) return ret; } + if (dim->active <= 0 || dim->total < dim->active || + dim->front_porch < 0 || dim->sync_width < 0 || + dim->front_porch + dim->sync_width > dim->total - dim->active) + return -EINVAL; + return 0; } @@ -349,6 +362,9 @@ struct color_mode { static int fill_color_mode(struct dcp_color_mode *color, struct color_mode *cmode) { + if (cmode->id < 0 || cmode->id > U32_MAX) + return -EINVAL; + if (color->score >= cmode->score) return 0; @@ -466,7 +482,7 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, * specifies the clock in kHz. The intermediate result may overflow a u32, so * use a u64 where required. */ -static u32 calculate_clock(struct dimension *horiz, struct dimension *vert) +static u64 calculate_clock(struct dimension *horiz, struct dimension *vert) { u32 pixels = horiz->total * vert->total; u64 clock = mul_u32_u32(pixels, vert->precise_sync_rate); @@ -480,8 +496,9 @@ static int parse_mode(struct dcp_parse_ctx *handle, { int ret = 0; struct iterator it; - struct dimension horiz = {}, vert = {}; + struct dimension horiz = { .active = -1 }, vert = { .active = -1 }; s64 min_vrr = 0, max_vrr = 0; + u64 clock; s64 id = -1; s64 best_color_mode = -1; bool is_virtual = false; @@ -518,13 +535,13 @@ static int parse_mode(struct dcp_parse_ctx *handle, else if (!strcmp(key, "ColorModes")) ret = parse_color_modes(it.handle, out); else if (!strcmp(key, "ID")) - ret = parse_int(it.handle, &id); + ret = parse_int_bound(it.handle, &id, 0, U32_MAX); else if (!strcmp(key, "IsVirtual")) ret = parse_bool(it.handle, &is_virtual); else if (!strcmp(key, "IsPreferred")) ret = parse_bool(it.handle, &is_preferred); else if (!strcmp(key, "Score")) - ret = parse_int(it.handle, score); + ret = parse_int_bound(it.handle, score, 0, S64_MAX); else ret = skip(it.handle); @@ -555,7 +572,9 @@ static int parse_mode(struct dcp_parse_ctx *handle, /* * Reject modes without valid color mode. */ - if (best_color_mode < 0) + if (best_color_mode < 0 || id < 0 || *score < 0 || + horiz.active <= 0 || vert.active <= notch_height || + vert.precise_sync_rate <= 0) return -EINVAL; /* @@ -584,13 +603,17 @@ static int parse_mode(struct dcp_parse_ctx *handle, out->vrr = true; } + clock = calculate_clock(&horiz, &vert); + if (!clock || clock > INT_MAX) + return -EINVAL; + vert.active -= notch_height; vert.sync_width += notch_height; /* From here we must succeed. Start filling out the mode. */ *mode = (struct drm_display_mode) { .type = DRM_MODE_TYPE_DRIVER, - .clock = calculate_clock(&horiz, &vert), + .clock = clock, .vdisplay = vert.active, .vsync_start = vert.active + vert.front_porch, From 56b3fe1799690d15e92ededd5d9a5faaaac29a9f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 00:57:28 +1100 Subject: [PATCH 1065/2680] dt-bindings: display: apple,dcp: describe bandwidth argument tuples Constrain each bandwidth property to a single provider reference with its resource index, display register index and, for scratch registers, byte offset. These are the arguments consumed by the existing public DCP driver and supplied by the Apple device trees. Add an example of the existing T8103 internal display resources and piodma child so the binding checks also exercise the argument tuples. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/display/apple,dcp.yaml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index ac2286832ba7d..33f1bf6e44359 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -98,10 +98,21 @@ properties: apple,bw-scratch: $ref: /schemas/types.yaml#/definitions/phandle-array + items: + - items: + - description: Phandle of the PMGR bandwidth register provider. + - description: Index into the provider's reg property. + - description: Index in the firmware display register table. + - description: Byte offset of the scratch register within the resource. description: PMGR bandwidth scratch resource, display index and offset. apple,bw-doorbell: $ref: /schemas/types.yaml#/definitions/phandle-array + items: + - items: + - description: Phandle of the PMGR bandwidth register provider. + - description: Index into the provider's reg property. + - description: Index in the firmware display register table. description: Optional PMGR bandwidth doorbell resource and display index. apple,iomfb-surfaces: @@ -269,4 +280,32 @@ allOf: resets: false unevaluatedProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + dcp@231c00000 { + compatible = "apple,t8103-dcp", "apple,dcp"; + reg = <0x2 0x31c00000 0x0 0x4000>, + <0x2 0x30000000 0x0 0x3e8000>, + <0x2 0x31320000 0x0 0x4000>, + <0x2 0x31344000 0x0 0x4000>, + <0x2 0x31800000 0x0 0x800000>, + <0x2 0x3b3d0000 0x0 0x4000>; + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 0>; + apple,bw-scratch = <&pmgr_dcp 0 5 0x14>; + apple,bw-doorbell = <&pmgr_dcp 1 6>; + + piodma { + iommus = <&disp_dart 0>; + }; + }; + }; ... From c06d588bd1f25891fa2da0231729adacb640db73 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 03:39:03 +1100 Subject: [PATCH 1066/2680] drm: apple: use the measured H17P late-init input The Linux-side startup record R-IOMFB-startup-run21 observes A000 with four zero input bytes. The public template instead sends one, as required by older firmware. Use zero for H17P while keeping the older variants' late-init requests unchanged. Keep the existing H17G input as well: the record admits only the H17P method profile. This corrects one measured input in the existing Asahi boot chain; it does not qualify the rest of startup or enable another device. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_template.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 3322391efca41..2ae79aa6ad56c 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -905,8 +905,12 @@ static void boot_5(struct apple_dcp *dcp, void *out, void *cookie) static void boot_4b(struct apple_dcp *dcp, void *out, void *cookie) { #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - u32 v_true = 1; - dcp_late_init_signal(dcp, false, &v_true, boot_5, NULL); +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + u32 value = dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G; +#else + u32 value = 1; +#endif + dcp_late_init_signal(dcp, false, &value, boot_5, NULL); #else dcp_late_init_signal(dcp, false, boot_5, NULL); #endif From b0a0af2de38af374f0a3d50426c0050eef18ce89 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 03:43:41 +1100 Subject: [PATCH 1067/2680] drm: apple: validate measured H17P callback sizes H17P dispatch currently trusts the firmware's input and output lengths before running typed callback handlers. A short record can make a handler read outside its input or write beyond its reply. Check the envelope and channel stack bounds before reading the header, then require a recorded callback tuple before clearing or dispatching its output. R-IOMFB-startup-run21 supplies the startup and present tuples; R-ANALYTICS supplies the D114 tuple. Stop the H17P interface on a mismatch rather than advancing its callback stack or sending subsequent calls. Older firmware and the existing H17G tuple dispatch stay unchanged. Tuple admission establishes sizes, not callback semantics or full device support. H17P callbacks absent from the measured set fail closed until an interoperability record admits them. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 27 ++++++++++++- drivers/gpu/drm/apple/iomfb.h | 2 + drivers/gpu/drm/apple/iomfb_h17p.c | 62 ++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index d72539fb65699..5bc14218a2b11 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -163,6 +163,9 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca { struct dcp_method_entry resolved = *call; + if (dcp->fw_compat == DCP_FIRMWARE_H17P && READ_ONCE(dcp->crashed)) + return; + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G && call->tag_h17g[0]) memcpy(resolved.tag, call->tag_h17g, sizeof(resolved.tag)); @@ -332,10 +335,29 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, struct device *dev = dcp->dev; struct dcp_packet_header *hdr = data; void *in, *out; - int tag = dcp_parse_tag(hdr->tag); + int tag; struct dcp_channel *ch = dcp_get_channel(dcp, context); u8 depth; + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + (length < sizeof(*hdr) || !ch || ch->depth >= DCP_MAX_CALL_DEPTH || + offset >= 0x8000 || length > 0x8000 - offset)) { + dev_err(dev, "invalid IOMFB callback envelope\n"); + dcp->crashed = true; + return; + } + + tag = dcp_parse_tag(hdr->tag); + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + ((u64)sizeof(*hdr) + hdr->in_len + hdr->out_len != length || + (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !iomfb_validate_callback_h17p(tag, hdr->in_len, hdr->out_len)))) { + dev_err(dev, "unqualified IOMFB callback %d (%u, %u)\n", + tag, hdr->in_len, hdr->out_len); + dcp->crashed = true; + return; + } + in = data + sizeof(*hdr); out = in + hdr->in_len; @@ -714,6 +736,9 @@ void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) { enum dcpep_type type = FIELD_GET(IOMFB_MESSAGE_TYPE, message); + if (dcp->fw_compat == DCP_FIRMWARE_H17P && READ_ONCE(dcp->crashed)) + return; + if (type == IOMFB_MESSAGE_TYPE_INITIALIZED) { /* * H17P reports its interface version (bits 63:48) and a diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index b816c75e2a855..1a33b1f474554 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -207,6 +207,8 @@ struct dcp_method_entry { }; #define IOMFB_MAX_CB (1000) + +bool iomfb_validate_callback_h17p(int tag, u32 in_len, u32 out_len); struct apple_dcp; typedef bool (*iomfb_cb_handler)(struct apple_dcp *, int, void *, void *); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 633f09c2be9d8..8692757b52bda 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -8,6 +8,68 @@ #include "iomfb_v14_7.h" #include "version_utils.h" +struct dcp_callback_size { + u32 in_len; + u32 out_len; + bool valid; +}; + +/* R-IOMFB-startup-run21 and R-ANALYTICS Linux boundary observations. */ +static const struct dcp_callback_size callback_sizes[IOMFB_MAX_CB] = { + [0] = { 0x0, 0x4, true }, + [1] = { 0x0, 0x4, true }, + [3] = { 0x4, 0x14, true }, + [6] = { 0x54, 0x50, true }, + [100] = { 0x0, 0x0, true }, + [101] = { 0x0, 0x4, true }, + [102] = { 0x44, 0x0, true }, + [104] = { 0x44, 0x0, true }, + [108] = { 0x0, 0x4, true }, + [109] = { 0x0, 0x4, true }, + [110] = { 0x0, 0x4, true }, + [111] = { 0x0, 0x4, true }, + [112] = { 0x0, 0x4, true }, + [113] = { 0x0, 0x4, true }, + [114] = { 0x1044, 0x1004, true }, + [121] = { 0x0, 0x4, true }, + [123] = { 0x0, 0x4, true }, + [125] = { 0x64, 0x24, true }, + [127] = { 0x4, 0x4, true }, + [128] = { 0x1008, 0x4, true }, + [129] = { 0x40, 0x4, true }, + [201] = { 0xc, 0x10, true }, + [206] = { 0x0, 0x4, true }, + [207] = { 0x0, 0x4, true }, + [300] = { 0x10, 0x0, true }, + [400] = { 0x4c, 0xc04, true }, + [401] = { 0x50, 0xc, true }, + [406] = { 0x48, 0x0, true }, + [411] = { 0x10, 0x1c, true }, + [413] = { 0x1048, 0x4, true }, + [414] = { 0x50, 0x4, true }, + [415] = { 0x4c, 0x4, true }, + [451] = { 0x14, 0x1c, true }, + [552] = { 0x1044, 0x4, true }, + [561] = { 0x1044, 0x4, true }, + [563] = { 0x4c, 0x4, true }, + [565] = { 0x48, 0x4, true }, + [574] = { 0x4, 0x8, true }, + [575] = { 0x58, 0x4c, true }, + [582] = { 0x8, 0x4, true }, + [590] = { 0x730, 0x0, true }, + [599] = { 0x0, 0x0, true }, +}; + +bool iomfb_validate_callback_h17p(int tag, u32 in_len, u32 out_len) +{ + const struct dcp_callback_size *size; + + if (tag < 0 || tag >= ARRAY_SIZE(callback_sizes)) + return false; + size = &callback_sizes[tag]; + return size->valid && size->in_len == in_len && size->out_len == out_len; +} + struct dcp_h17p_hotplug_request { __le64 connected; u8 tiled_display[0x4c]; From 918aca4ad5f6b74aa7622d6a7564066f8168e1bd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 03:44:54 +1100 Subject: [PATCH 1068/2680] drm: apple: answer H17P analytics with an empty dictionary R-ANALYTICS records four complete Linux-side D114 exchanges with a 0x1044-byte request and a 0x1004-byte reply. Every reply begins with the OSDictionary tag 'd' and has zero remaining bytes; the public Asahi OSDictionary decoder accepts it as an empty dictionary. Encode that reply for the measured H17P profile after callback-size validation. Keep the existing H17G response, whose dictionary contract was not measured. This extends the existing Asahi callback table without introducing an arbitrary-call interface or a recovery retry. Byte equivalence to the recorded replies is checked separately from the required two-hour survival gate, which remains to be run on the device. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 8692757b52bda..2246dd617527d 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -187,6 +187,19 @@ static bool trampoline_get_frequency_h17p(struct apple_dcp *dcp, int tag, return true; } +static bool trampoline_analytics_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return trampoline_zero(dcp, tag, out, in); + + trace_iomfb_callback(dcp, tag, __func__); + /* The dispatcher has validated the measured D114 reply size. */ + memset(out, 0, 0x1004); + *(u8 *)out = 'd'; + return true; +} + static bool trampoline_hotplug_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) { @@ -238,7 +251,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [111] = trampoline_true, /* create_iomfb_service */ [112] = trampoline_create_backlight_service, /* create_backlight_service */ [113] = trampoline_true, /* create_nvram_service */ - [114] = trampoline_zero, /* CoreAnalyticsSendEvent */ + [114] = trampoline_analytics_h17p, /* CoreAnalyticsSendEvent */ [117] = trampoline_nop, /* set_idle_caching_state_ap */ [118] = trampoline_zero, /* upload_trace_start */ [119] = trampoline_zero, /* upload_trace_chunk */ From f7b27af2206c7410deda1d3a792b05c6a5c2c897 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 03:49:42 +1100 Subject: [PATCH 1069/2680] drm: apple: serialize H17P presents at the measured boundary The public H17 template starts its surfaces after a 0x58a-byte C swap record and repairs their fields in place. SPEC PRESENT-2 requires a 0x588-byte wire boundary. R-SWAP-controlled-surface-run21 independently locates format, width, height, stride and per-plane dimensions/byte count at offsets matching Asahi's existing surface record at that boundary. Serialize H17P into separate device-owned staging storage, copying the swap prefix, unchanged public surfaces and trailing arguments to their wire positions. Keep the total A408 input at 0xe9c bytes, zero the final two bytes and preserve the native atomic input for repeated encoding. H17G keeps its existing public packing path. Static size/offset checks and randomized byte maps cover every byte of the transfer. Nine populated fields match the controlled Linux capture. This does not establish unmeasured field semantics or qualify resource ownership, formats, backlight or a functional display boot. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/iomfb_h17p.c | 18 +++++++++++++++ drivers/gpu/drm/apple/iomfb_h17p.h | 14 +++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 32 ++++++++++++++++++++++---- 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 70ba67c793666..ef3daf2e57cab 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -224,6 +224,8 @@ struct apple_dcp { struct dcp_swap_submit_req_v14_7_0 v14_7_0; struct dcp_swap_submit_req_h17p h17p; } swap; + /* Staging wire record; serialization never mutates the atomic inputs. */ + struct dcp_present_h17p present_h17p; /* swap id of the last completed swap */ u32 last_swap_id; diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 2246dd617527d..2b922b326e27b 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -89,6 +89,24 @@ static_assert(sizeof(struct dcp_h17p_swap_info_request) == 0xe4); static_assert(sizeof(struct dc_swap_complete_resp_h17p) == 0x730); static_assert(offsetof(struct dc_swap_complete_resp_h17p, swap_id) == 0); +void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, + const struct dcp_swap_submit_req_h17p *request) +{ + size_t tail_offset = offsetof(struct dcp_swap_submit_req_h17p, surf_iova); + size_t tail_size = sizeof(*request) - tail_offset; + + static_assert(sizeof(request->swap) == sizeof(wire->swap) + 2); + static_assert(sizeof(request->surf) == sizeof(wire->surf)); + static_assert(sizeof(*request) - + offsetof(struct dcp_swap_submit_req_h17p, surf_iova) == + sizeof(wire->tail) - 2); + + memcpy(wire->swap, &request->swap, sizeof(wire->swap)); + memcpy(wire->surf, request->surf, sizeof(wire->surf)); + memcpy(wire->tail, (const u8 *)request + tail_offset, tail_size); + memset(wire->tail + tail_size, 0, sizeof(wire->tail) - tail_size); +} + static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { IOMFB_METHOD("A000", dcpep_late_init_signal), IOMFB_METHOD_H17("A025", "A029", dcpep_setup_video_limits), /* (0,0) */ diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 97c0da9e05b94..e9f4dfb6dd855 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -16,6 +16,20 @@ static_assert(sizeof(struct dcp_surface_h17p) == 0x22c); static_assert(sizeof(struct dcp_swap_submit_req_h17p) == 0xe9c); static_assert(sizeof(struct dcp_swap_submit_resp_h17p) == 0x0c); +/* The H17P swap boundary is independent of the template's native layout. */ +struct dcp_present_h17p { + u8 swap[0x588]; + struct dcp_surface_h17p surf[SWAP_SURFACES]; + u8 tail[0x64]; +} __packed; + +static_assert(sizeof(struct dcp_present_h17p) == 0xe9c); +static_assert(offsetof(struct dcp_present_h17p, surf) == 0x588); +static_assert(offsetof(struct dcp_present_h17p, tail) == 0xe38); + +void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, + const struct dcp_swap_submit_req_h17p *request); + #undef DCP_FW_VER #undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 2ae79aa6ad56c..a5a9cb9074374 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -61,9 +61,26 @@ IOMFB_THUNK_INOUT(get_color_remap_mode); IOMFB_THUNK_INOUT(last_client_close); IOMFB_THUNK_INOUT(abort_swaps_dcp); +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER +static void dcp_swap_submit(struct apple_dcp *dcp, bool oob, + struct dcp_swap_submit_req_h17p *request, + dcp_callback_t cb, void *cookie) +{ + const void *data = request; + + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + iomfb_serialize_present_h17p(&dcp->present_h17p, request); + data = &dcp->present_h17p; + } + + dcp_push(dcp, oob, &dcp_methods[dcpep_swap_submit], sizeof(*request), + sizeof(struct dcp_swap_submit_resp_h17p), (void *)data, cb, cookie); +} +#else DCP_THUNK_INOUT(dcp_swap_submit, dcpep_swap_submit, struct DCP_FW_NAME(dcp_swap_submit_req), struct DCP_FW_NAME(dcp_swap_submit_resp)); +#endif DCP_THUNK_INOUT(dcp_swap_start, dcpep_swap_start, struct DCP_FW_NAME(dcp_swap_start_req), struct DCP_FW_NAME(dcp_swap_start_resp)); @@ -1161,9 +1178,10 @@ static void release_swap_cookie(struct kref *ref) * format two bytes late and its surface check rejects the swap as an * unsupported format. * - * Shift each surface's tail down by two rather than fork the shared struct. + * H17G retains this public packing path. The measured H17P profile uses + * a separate wire record with the SPEC's 0x588 swap boundary. */ -static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) +static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) { struct DCP_FW_NAME(dcp_swap_submit_req) *req = &DCP_FW_UNION(dcp->swap); unsigned int i; @@ -1183,6 +1201,10 @@ static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) req->swap.h17p_pre_swap_id[0x00] = 2; req->swap.h17p_pre_swap_id[0x28] = 3; + /* The measured H17P layout is serialized without shifting its surfaces. */ + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + return; + for (i = 0; i < SWAP_SURFACES; i++) { u8 *p = (u8 *)&req->surf[i]; size_t n = sizeof(req->surf[i]); @@ -1212,7 +1234,7 @@ static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) } } #else -static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) { } +static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) { } #endif static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) @@ -1255,7 +1277,7 @@ static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, if (info) info->swap_id = resp->swap_id; - dcp_h17p_fix_swap_surfaces(dcp); + dcp_h17p_prepare_swap(dcp); dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swap_cleared, cookie); } @@ -1750,7 +1772,7 @@ static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) dcp_arm_retained_framebuffers(dcp, resp->swap_id); trace_iomfb_swap_submit(dcp, resp->swap_id); - dcp_h17p_fix_swap_surfaces(dcp); + dcp_h17p_prepare_swap(dcp); dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swapped, NULL); } From 89bc1566fd9002b5e89f1743f070ea1bae88c188 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 03:56:32 +1100 Subject: [PATCH 1070/2680] drm: apple: require accepted H17P present IDs before retirement R-SWAP's controlled captures correlate the A406-assigned ID, A408 input and D590 completion. Every retained run21 completion follows its matching submit acknowledgment. The existing completion handler retires buffers and signals a flip without checking whether that ID was accepted. Track one started and accepted H17P present in the RTKit receive thread. Reject overlapping starts, inconsistent submit state, early completion, wrong IDs and duplicate completion before touching framebuffer retirement or DRM flip state. Stop the interface on protocol inconsistency. Rejected submits cancel the reservation while the existing retention code unarms their references; a later unsolicited completion cannot retire them. Apply the same policy to ordinary and clear swaps, preserving older firmware and H17G behavior. This is a lifecycle guard, not a substitute for the outbound transaction queue or hardware lifetime qualification. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 6 ++- drivers/gpu/drm/apple/iomfb_h17p.h | 40 ++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 63 +++++++++++++++++++++++++- 3 files changed, 105 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index ef3daf2e57cab..8ef850dab23e1 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -224,8 +224,6 @@ struct apple_dcp { struct dcp_swap_submit_req_v14_7_0 v14_7_0; struct dcp_swap_submit_req_h17p h17p; } swap; - /* Staging wire record; serialization never mutates the atomic inputs. */ - struct dcp_present_h17p present_h17p; /* swap id of the last completed swap */ u32 last_swap_id; @@ -347,6 +345,10 @@ struct apple_dcp { struct dcp_backlight_state state; void (*kick)(struct apple_dcp *dcp); } backlight; + + /* Staging wire record; serialization never mutates the atomic inputs. */ + struct dcp_present_h17p present_h17p; + struct dcp_present_state_h17p present_state_h17p; }; void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index e9f4dfb6dd855..2b260ebb2c86b 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -16,6 +16,46 @@ static_assert(sizeof(struct dcp_surface_h17p) == 0x22c); static_assert(sizeof(struct dcp_swap_submit_req_h17p) == 0xe9c); static_assert(sizeof(struct dcp_swap_submit_resp_h17p) == 0x0c); +/* Owned by the RTKit receive thread, independently of DRM state lifetime. */ +struct dcp_present_state_h17p { + u32 swap_id; + bool pending; + bool accepted; +}; + +static inline bool +dcp_present_begin_h17p(struct dcp_present_state_h17p *state, u32 swap_id) +{ + if (state->pending) + return false; + state->swap_id = swap_id; + state->pending = true; + state->accepted = false; + return true; +} + +static inline bool +dcp_present_submit_h17p(struct dcp_present_state_h17p *state, u32 swap_id, + bool accepted) +{ + if (!state->pending || state->accepted || state->swap_id != swap_id) + return false; + state->accepted = accepted; + if (!accepted) + state->pending = false; + return true; +} + +static inline bool +dcp_present_complete_h17p(struct dcp_present_state_h17p *state, u32 swap_id) +{ + if (!state->pending || !state->accepted || state->swap_id != swap_id) + return false; + state->pending = false; + state->accepted = false; + return true; +} + /* The H17P swap boundary is independent of the template's native layout. */ struct dcp_present_h17p { u8 swap[0x588]; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index a5a9cb9074374..d59e16b40be04 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -241,6 +241,15 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, struct DCP_FW_NAME(dc_swap_complete_resp) *resp) { ktime_t now = ktime_get(); + +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !dcp_present_complete_h17p(&dcp->present_state_h17p, resp->swap_id)) { + dev_err(dcp->dev, "unexpected present completion %u\n", resp->swap_id); + dcp->crashed = true; + return; + } +#endif trace_iomfb_swap_complete(dcp, resp->swap_id); dcp->last_swap_id = resp->swap_id; dcp_release_retained_framebuffers(dcp, resp->swap_id); @@ -1237,6 +1246,32 @@ static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) { } #endif +static bool dcp_present_begin(struct apple_dcp *dcp, u32 swap_id) +{ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !dcp_present_begin_h17p(&dcp->present_state_h17p, swap_id)) { + dev_err(dcp->dev, "overlapping present %u\n", swap_id); + dcp->crashed = true; + return false; + } +#endif + return true; +} + +static bool dcp_present_submit(struct apple_dcp *dcp, u32 swap_id, bool accepted) +{ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !dcp_present_submit_h17p(&dcp->present_state_h17p, swap_id, accepted)) { + dev_err(dcp->dev, "unexpected present submission %u\n", swap_id); + dcp->crashed = true; + return false; + } +#endif + return true; +} + static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; @@ -1244,6 +1279,9 @@ static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) u32 swap_id = DCP_FW_UNION(dcp->swap).swap.swap_id; u32 status = resp ? resp->ret : ~0U; + if (!dcp_present_submit(dcp, swap_id, !status)) + status = ~0U; + if (status) { dev_err(dcp->dev, "swap_clear failed! status %u\n", status); dcp_unarm_retained_framebuffers(dcp, swap_id); @@ -1271,6 +1309,15 @@ static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, dcp_drm_crtc_vblank(dcp->crtc); return; } + if (!dcp_present_begin(dcp, resp->swap_id)) { + if (info) { + WRITE_ONCE(info->status, ~0U); + complete(&info->done); + kref_put(&info->refcount, release_swap_cookie); + } + dcp_drm_crtc_vblank(dcp->crtc); + return; + } DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; dcp_arm_retained_framebuffers(dcp, resp->swap_id); @@ -1748,9 +1795,17 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; u32 swap_id = DCP_FW_UNION(dcp->swap).swap.swap_id; +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + u32 status = resp ? resp->ret : ~0U; +#else + u32 status = resp->ret; +#endif - if (resp->ret) { - dev_err(dcp->dev, "swap failed! status %u\n", resp->ret); + if (!dcp_present_submit(dcp, swap_id, !status)) + status = ~0U; + + if (status) { + dev_err(dcp->dev, "swap failed! status %u\n", status); dcp_unarm_retained_framebuffers(dcp, swap_id); dcp_drm_crtc_vblank(dcp->crtc); return; @@ -1768,6 +1823,10 @@ static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) dcp_drm_crtc_vblank(dcp->crtc); return; } + if (!dcp_present_begin(dcp, resp->swap_id)) { + dcp_drm_crtc_vblank(dcp->crtc); + return; + } DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; dcp_arm_retained_framebuffers(dcp, resp->swap_id); From 041c85559f3688b9025fc4002b6872a2c9f28e51 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 04:23:57 +1100 Subject: [PATCH 1071/2680] drm: apple: serialize H17P command and present transactions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RTKit replies can issue nested calls while atomic commits and power requests arrive from other threads. Channel depth alone cannot distinguish those callers, and preparing a second present can overwrite the request still awaiting the first submit reply. Queue H17P top-level operations in FIFO order and serialize transmit preparation with the receive thread. Copy standalone command inputs and hold atomic state through the reply chain and matching present completion. Keep nested calls in their originating transaction. A submit ACK alone does not let the next operation reuse the present record. Bound the pending queue and stop new operations on a ten-second timeout. Keep the active operation pinned on failure because a timeout does not prove firmware has stopped using it. An expired watchdog from a previous operation checks the new deadline before reporting a timeout. Older firmware and the H17G method profile keep their existing transport. [ryan: adapted to 7.2.10 — struct drm_atomic_state is struct drm_atomic_commit] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 25 +++ drivers/gpu/drm/apple/dcp.c | 2 + drivers/gpu/drm/apple/iomfb.c | 295 ++++++++++++++++++++++++++- 3 files changed, 319 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 8ef850dab23e1..57afa3a2aa1ac 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -31,6 +31,13 @@ struct apple_dcp; struct apple_dcp_afkep; + +/* One operation, including all of its nested replies and present completion. */ +struct iomfb_transaction { + struct list_head link; + void (*start)(struct apple_dcp *dcp, struct iomfb_transaction *transaction); + void (*release)(struct iomfb_transaction *transaction); +}; struct apple_dcp_typec_port; struct apple_dcp_typec_route { @@ -349,8 +356,26 @@ struct apple_dcp { /* Staging wire record; serialization never mutates the atomic inputs. */ struct dcp_present_h17p present_h17p; struct dcp_present_state_h17p present_state_h17p; + + /* Serializes H17P transmit preparation with RTKit receive callbacks. */ + struct { + /* Protects queue, channel stacks, and the current owner. */ + struct mutex lock; + struct list_head pending; + struct iomfb_transaction *active; + struct task_struct *owner; + struct work_struct work; + struct delayed_work timeout; + unsigned long deadline; + unsigned int queued; + bool stopped; + } iomfb; }; +void iomfb_queue_init(struct apple_dcp *dcp); +void iomfb_queue_stop(struct apple_dcp *dcp); +int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction); + void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); int dcp_backlight_register(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 349e254e4658d..0e20c26c627f0 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2205,6 +2205,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) if (dcp->shmem) iomfb_shutdown(dcp); + iomfb_queue_stop(dcp); dcp_release_piodma_iommu_dev(dcp); if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { @@ -2254,6 +2255,7 @@ static int dcp_platform_probe(struct platform_device *pdev) INIT_LIST_HEAD(&dcp->swapped_out_fbs); mutex_init(&dcp->swapped_out_fbs_lock); spin_lock_init(&dcp->backlight.lock); + iomfb_queue_init(dcp); dcp->fw_compat = fw_compat; dcp->dev = dev; diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 5bc14218a2b11..38be05617e9f2 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -156,6 +157,195 @@ static u8 dcp_pop_depth(u8 *depth) return --(*depth); } +/* Older firmware and the H17G method profile retain their existing transport. */ +static bool iomfb_uses_queue(struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G; +} + +static bool iomfb_channels_idle(struct apple_dcp *dcp) +{ + return !dcp->ch_cmd.depth && !dcp->ch_cb.depth && + !dcp->ch_oobcmd.depth && !dcp->ch_oobcb.depth && + !dcp->ch_async.depth && !dcp->ch_oobasync.depth; +} + +static void iomfb_discard_pending(struct apple_dcp *dcp) +{ + struct iomfb_transaction *transaction, *next; + + lockdep_assert_held(&dcp->iomfb.lock); + list_for_each_entry_safe(transaction, next, &dcp->iomfb.pending, link) { + list_del(&transaction->link); + dcp->iomfb.queued--; + transaction->release(transaction); + } +} + +static void iomfb_queue_advance(struct apple_dcp *dcp) +{ + struct iomfb_transaction *transaction = dcp->iomfb.active; + + lockdep_assert_held(&dcp->iomfb.lock); + if (READ_ONCE(dcp->crashed) || dcp->iomfb.stopped) { + iomfb_discard_pending(dcp); + /* The active operation may still be visible to firmware. */ + return; + } + if (!iomfb_channels_idle(dcp) || dcp->present_state_h17p.pending) + return; + + if (transaction) { + dcp->iomfb.active = NULL; + cancel_delayed_work(&dcp->iomfb.timeout); + transaction->release(transaction); + } + if (!list_empty(&dcp->iomfb.pending)) + schedule_work(&dcp->iomfb.work); +} + +static void iomfb_queue_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(work, struct apple_dcp, iomfb.work); + struct iomfb_transaction *transaction; + + mutex_lock(&dcp->iomfb.lock); + if (READ_ONCE(dcp->crashed) || dcp->iomfb.stopped) { + iomfb_discard_pending(dcp); + goto unlock; + } + if (dcp->iomfb.active || !iomfb_channels_idle(dcp) || + dcp->present_state_h17p.pending || list_empty(&dcp->iomfb.pending)) + goto unlock; + + transaction = list_first_entry(&dcp->iomfb.pending, + struct iomfb_transaction, link); + list_del(&transaction->link); + dcp->iomfb.queued--; + dcp->iomfb.active = transaction; + WRITE_ONCE(dcp->iomfb.owner, current); + dcp->iomfb.deadline = jiffies + msecs_to_jiffies(10000); + mod_delayed_work(system_wq, &dcp->iomfb.timeout, msecs_to_jiffies(10000)); + transaction->start(dcp, transaction); + WRITE_ONCE(dcp->iomfb.owner, NULL); + iomfb_queue_advance(dcp); +unlock: + mutex_unlock(&dcp->iomfb.lock); +} + +static void iomfb_queue_timeout(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), + struct apple_dcp, iomfb.timeout); + + mutex_lock(&dcp->iomfb.lock); + if (dcp->iomfb.active && time_before(jiffies, dcp->iomfb.deadline)) { + mod_delayed_work(system_wq, &dcp->iomfb.timeout, + dcp->iomfb.deadline - jiffies); + } else if (dcp->iomfb.active) { + WRITE_ONCE(dcp->crashed, true); + dev_err(dcp->dev, "IOMFB transaction timed out\n"); + iomfb_discard_pending(dcp); + schedule_work(&dcp->vblank_wq); + } + mutex_unlock(&dcp->iomfb.lock); +} + +void iomfb_queue_init(struct apple_dcp *dcp) +{ + mutex_init(&dcp->iomfb.lock); + INIT_LIST_HEAD(&dcp->iomfb.pending); + INIT_WORK(&dcp->iomfb.work, iomfb_queue_work); + INIT_DELAYED_WORK(&dcp->iomfb.timeout, iomfb_queue_timeout); +} + +void iomfb_queue_stop(struct apple_dcp *dcp) +{ + if (!iomfb_uses_queue(dcp)) + return; + + mutex_lock(&dcp->iomfb.lock); + dcp->iomfb.stopped = true; + iomfb_discard_pending(dcp); + mutex_unlock(&dcp->iomfb.lock); + cancel_work_sync(&dcp->iomfb.work); + cancel_delayed_work_sync(&dcp->iomfb.timeout); +} + +int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction) +{ + int ret = 0; + + mutex_lock(&dcp->iomfb.lock); + if (READ_ONCE(dcp->crashed) || dcp->iomfb.stopped) { + ret = -EIO; + } else if (dcp->iomfb.queued >= 32) { + ret = -EBUSY; + } else { + dcp->iomfb.queued++; + list_add_tail(&transaction->link, &dcp->iomfb.pending); + schedule_work(&dcp->iomfb.work); + } + mutex_unlock(&dcp->iomfb.lock); + return ret; +} + +struct iomfb_command { + struct iomfb_transaction transaction; + struct dcp_method_entry method; + dcp_callback_t callback; + void *cookie; + u32 in_len; + u32 out_len; + bool oob; + u8 data[]; +}; + +static void iomfb_command_start(struct apple_dcp *dcp, + struct iomfb_transaction *transaction) +{ + struct iomfb_command *command = container_of(transaction, + struct iomfb_command, transaction); + + dcp_push(dcp, command->oob, &command->method, command->in_len, + command->out_len, command->data, command->callback, command->cookie); +} + +static void iomfb_command_release(struct iomfb_transaction *transaction) +{ + kfree(container_of(transaction, struct iomfb_command, transaction)); +} + +static int iomfb_queue_command(struct apple_dcp *dcp, bool oob, + const struct dcp_method_entry *method, + u32 in_len, u32 out_len, void *data, + dcp_callback_t callback, void *cookie) +{ + struct iomfb_command *command; + int ret; + + if ((u64)sizeof(struct dcp_packet_header) + in_len + out_len > 0x8000) + return -EMSGSIZE; + command = kzalloc(struct_size(command, data, in_len), GFP_KERNEL); + if (!command) + return -ENOMEM; + command->transaction.start = iomfb_command_start; + command->transaction.release = iomfb_command_release; + command->method = *method; + command->callback = callback; + command->cookie = cookie; + command->in_len = in_len; + command->out_len = out_len; + command->oob = oob; + if (in_len) + memcpy(command->data, data, in_len); + ret = iomfb_queue(dcp, &command->transaction); + if (ret) + iomfb_command_release(&command->transaction); + return ret; +} + /* Call a DCP function given by a tag */ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *call, u32 in_len, u32 out_len, void *data, dcp_callback_t cb, @@ -163,6 +353,14 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca { struct dcp_method_entry resolved = *call; + /* Reply chains stay on the serialized receiver; callers enqueue copies. */ + if (iomfb_uses_queue(dcp) && READ_ONCE(dcp->iomfb.owner) != current) { + if (iomfb_queue_command(dcp, oob, call, in_len, out_len, + data, cb, cookie)) + WRITE_ONCE(dcp->crashed, true); + return; + } + if (dcp->fw_compat == DCP_FIRMWARE_H17P && READ_ONCE(dcp->crashed)) return; @@ -185,8 +383,21 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca .tag[3] = call->tag[0], }; - u8 depth = dcp_push_depth(&ch->depth); - u16 offset = dcp_packet_start(ch, depth); + u8 depth; + u16 offset; + + if (iomfb_uses_queue(dcp) && + (!ch || ch->depth >= DCP_MAX_CALL_DEPTH || + (u64)sizeof(header) + in_len + out_len > 0x8000 || + dcp_packet_start(ch, ch->depth) > + 0x8000 - ALIGN(sizeof(header) + in_len + out_len, + DCP_PACKET_ALIGNMENT))) { + dev_err(dcp->dev, "invalid IOMFB command envelope\n"); + WRITE_ONCE(dcp->crashed, true); + return; + } + depth = dcp_push_depth(&ch->depth); + offset = dcp_packet_start(ch, depth); void *out = dcp->shmem + dcp_tx_offset(context) + offset; void *out_data = out + sizeof(header); @@ -650,11 +861,63 @@ bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, } +struct iomfb_atomic_transaction { + struct iomfb_transaction transaction; + struct drm_atomic_commit *state; + struct drm_crtc *crtc; +}; + +static void iomfb_atomic_start(struct apple_dcp *dcp, + struct iomfb_transaction *transaction) +{ + struct iomfb_atomic_transaction *atomic = container_of(transaction, + struct iomfb_atomic_transaction, transaction); + + if (dcp->valid_mode && dcp->connector && dcp->connector->connected) + iomfb_flush_h17p(dcp, atomic->crtc, atomic->state); + else + schedule_work(&dcp->vblank_wq); +} + +static void iomfb_atomic_release(struct iomfb_transaction *transaction) +{ + struct iomfb_atomic_transaction *atomic = container_of(transaction, + struct iomfb_atomic_transaction, transaction); + + drm_atomic_commit_put(atomic->state); + kfree(atomic); +} + +static void iomfb_queue_atomic(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *state) +{ + struct iomfb_atomic_transaction *atomic; + + atomic = kzalloc_obj(*atomic); + if (!atomic) + goto failed; + atomic->transaction.start = iomfb_atomic_start; + atomic->transaction.release = iomfb_atomic_release; + atomic->state = drm_atomic_commit_get(state); + atomic->crtc = crtc; + if (!iomfb_queue(dcp, &atomic->transaction)) + return; + iomfb_atomic_release(&atomic->transaction); +failed: + WRITE_ONCE(dcp->crashed, true); + schedule_work(&dcp->vblank_wq); +} + void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit) { struct platform_device *pdev = to_apple_crtc(crtc)->dcp; struct apple_dcp *dcp = platform_get_drvdata(pdev); + if (iomfb_uses_queue(dcp)) { + iomfb_queue_atomic(dcp, crtc, commit); + return; + } + /* * DCP does not complete swaps after a link loss. A plane-only commit * arriving between the reconnect callback and the required modeset @@ -732,7 +995,7 @@ bool dcp_is_initialized(struct platform_device *pdev) return dcp->active; } -void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) +static void iomfb_recv_message(struct apple_dcp *dcp, u64 message) { enum dcpep_type type = FIELD_GET(IOMFB_MESSAGE_TYPE, message); @@ -757,11 +1020,37 @@ void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) dev_warn(dcp->dev, "Ignoring unknown message %llx\n", message); } +void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) +{ + if (!iomfb_uses_queue(dcp)) { + iomfb_recv_message(dcp, message); + return; + } + + mutex_lock(&dcp->iomfb.lock); + WRITE_ONCE(dcp->iomfb.owner, current); + if (!dcp->iomfb.stopped) + iomfb_recv_message(dcp, message); + WRITE_ONCE(dcp->iomfb.owner, NULL); + iomfb_queue_advance(dcp); + mutex_unlock(&dcp->iomfb.lock); +} + int iomfb_start_rtkit(struct apple_dcp *dcp) { dma_addr_t shmem_iova; int ret; + if (iomfb_uses_queue(dcp)) { + mutex_lock(&dcp->iomfb.lock); + if (dcp->iomfb.active) { + mutex_unlock(&dcp->iomfb.lock); + return -EBUSY; + } + dcp->iomfb.stopped = false; + mutex_unlock(&dcp->iomfb.lock); + } + /* * H17P firmware expects the remote allocator endpoint to be started * before IOMFB. A firmware subsystem whose endpoint was never started From 1effd253bac793621d7eaa55a88af7411e9e1979 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 04:33:56 +1100 Subject: [PATCH 1072/2680] drm: apple: pin the last completed H17P scanout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A rejected atomic replacement can leave DRM state referring to a surface that the display never accepted. Re-presenting that state for a backlight update would bypass the previous submit failure and retire the wrong framebuffers. Keep an independent snapshot with framebuffer references for each accepted surface. Stage replacements from the atomic operation and publish them only after the accepted submit's matching completion. Merge unchanged planes and preserve the background on partial updates. Rejected operations release only their staged snapshot, leaving the completed scanout pinned. Retain the snapshot on soft-off or firmware failure. It supplies no proof of a final DMA stop, and rebinding an unconfirmed session is refused. Older firmware and the H17G method profile retain their existing path. [ryan: adapted to 7.2.10 — struct drm_atomic_state is struct drm_atomic_commit] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 9 ++ drivers/gpu/drm/apple/iomfb.c | 117 ++++++++++++++++++++++++- drivers/gpu/drm/apple/iomfb_template.c | 2 + 3 files changed, 124 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 57afa3a2aa1ac..a6db905f9d8ca 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -32,6 +32,12 @@ struct apple_dcp; struct apple_dcp_afkep; +/* Snapshot of a completed present, independent of the latest DRM state. */ +struct iomfb_scanout_h17p { + struct dcp_swap_submit_req_h17p request; + struct drm_framebuffer *fb[SWAP_SURFACES]; +}; + /* One operation, including all of its nested replies and present completion. */ struct iomfb_transaction { struct list_head link; @@ -369,9 +375,12 @@ struct apple_dcp { unsigned long deadline; unsigned int queued; bool stopped; + struct iomfb_scanout_h17p *scanout; + struct iomfb_scanout_h17p *next_scanout; } iomfb; }; +void iomfb_scanout_complete_h17p(struct apple_dcp *dcp); void iomfb_queue_init(struct apple_dcp *dcp); void iomfb_queue_stop(struct apple_dcp *dcp); int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 38be05617e9f2..ca2cb85cf2fff 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -861,10 +861,105 @@ bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, } +static void iomfb_scanout_release_h17p(struct iomfb_scanout_h17p *scanout) +{ + unsigned int i; + + if (!scanout) + return; + for (i = 0; i < SWAP_SURFACES; i++) + if (scanout->fb[i]) + drm_framebuffer_put(scanout->fb[i]); + kfree(scanout); +} + +static struct iomfb_scanout_h17p * +iomfb_scanout_prepare_h17p(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *state) +{ + struct iomfb_scanout_h17p *scanout; + struct drm_plane *plane; + struct drm_plane_state *old_state, *new_state; + unsigned int i, slot; + int index; + + lockdep_assert_held(&dcp->iomfb.lock); + scanout = kzalloc_obj(*scanout); + if (!scanout) + return NULL; + if (dcp->iomfb.scanout) { + scanout->request = dcp->iomfb.scanout->request; + for (i = 0; i < SWAP_SURFACES; i++) { + scanout->fb[i] = dcp->iomfb.scanout->fb[i]; + if (scanout->fb[i]) + drm_framebuffer_get(scanout->fb[i]); + } + } + + for_each_oldnew_plane_in_state(state, plane, old_state, new_state, index) { + if (old_state->crtc != crtc && new_state->crtc != crtc) + continue; + slot = to_apple_plane(plane)->iomfb_surf; + if (slot >= SWAP_SURFACES) { + iomfb_scanout_release_h17p(scanout); + return NULL; + } + if (scanout->fb[slot]) + drm_framebuffer_put(scanout->fb[slot]); + scanout->fb[slot] = NULL; + if (new_state->crtc == crtc && new_state->visible && new_state->fb) { + scanout->fb[slot] = new_state->fb; + drm_framebuffer_get(new_state->fb); + } + } + return scanout; +} + +void iomfb_scanout_complete_h17p(struct apple_dcp *dcp) +{ + struct iomfb_scanout_h17p *scanout = dcp->iomfb.next_scanout; + struct iomfb_scanout_h17p *previous = dcp->iomfb.scanout; + struct dcp_swap_submit_req_h17p *request = &dcp->swap.h17p; + unsigned int i; + + lockdep_assert_held(&dcp->iomfb.lock); + if (!scanout) + return; + + /* Keep common present arguments, including the null output pointers. */ + scanout->request = *request; + /* Preserve unchanged planes when only a subset was presented. */ + for (i = 0; i < SWAP_SURFACES; i++) { + if (!(request->swap.swap_enabled & BIT(i)) && previous) { + scanout->request.surf[i] = previous->request.surf[i]; + scanout->request.surf_iova[i] = previous->request.surf_iova[i]; + scanout->request.swap.src_rect[i] = previous->request.swap.src_rect[i]; + scanout->request.swap.dst_rect[i] = previous->request.swap.dst_rect[i]; + scanout->request.swap.surf_ids[i] = previous->request.swap.surf_ids[i]; + scanout->request.swap.surf_flags[i] = previous->request.swap.surf_flags[i]; + scanout->request.swap.surf_unk[i] = previous->request.swap.surf_unk[i]; + } + scanout->request.surf_null[i] = !scanout->fb[i]; + } + /* Brightness re-presents all pinned surfaces with a fresh swap ID. */ + scanout->request.swap.swap_enabled = IOMFB_SET_BACKGROUND; + for (i = 0; i < SWAP_SURFACES; i++) + if (scanout->fb[i]) + scanout->request.swap.swap_enabled |= BIT(i); + scanout->request.swap.swap_completed = scanout->request.swap.swap_enabled; + if (!(request->swap.swap_enabled & IOMFB_SET_BACKGROUND) && previous) + scanout->request.swap.bg_color = previous->request.swap.bg_color; + dcp->iomfb.scanout = scanout; + dcp->iomfb.next_scanout = NULL; + iomfb_scanout_release_h17p(previous); +} + struct iomfb_atomic_transaction { struct iomfb_transaction transaction; struct drm_atomic_commit *state; struct drm_crtc *crtc; + struct apple_dcp *dcp; + struct iomfb_scanout_h17p *scanout; }; static void iomfb_atomic_start(struct apple_dcp *dcp, @@ -873,10 +968,18 @@ static void iomfb_atomic_start(struct apple_dcp *dcp, struct iomfb_atomic_transaction *atomic = container_of(transaction, struct iomfb_atomic_transaction, transaction); - if (dcp->valid_mode && dcp->connector && dcp->connector->connected) - iomfb_flush_h17p(dcp, atomic->crtc, atomic->state); - else + if (!dcp->valid_mode || !dcp->connector || !dcp->connector->connected) { schedule_work(&dcp->vblank_wq); + return; + } + atomic->scanout = iomfb_scanout_prepare_h17p(dcp, atomic->crtc, atomic->state); + if (!atomic->scanout) { + WRITE_ONCE(dcp->crashed, true); + schedule_work(&dcp->vblank_wq); + return; + } + dcp->iomfb.next_scanout = atomic->scanout; + iomfb_flush_h17p(dcp, atomic->crtc, atomic->state); } static void iomfb_atomic_release(struct iomfb_transaction *transaction) @@ -884,6 +987,11 @@ static void iomfb_atomic_release(struct iomfb_transaction *transaction) struct iomfb_atomic_transaction *atomic = container_of(transaction, struct iomfb_atomic_transaction, transaction); + if (atomic->scanout && atomic->dcp->iomfb.scanout != atomic->scanout) { + if (atomic->dcp->iomfb.next_scanout == atomic->scanout) + atomic->dcp->iomfb.next_scanout = NULL; + iomfb_scanout_release_h17p(atomic->scanout); + } drm_atomic_commit_put(atomic->state); kfree(atomic); } @@ -900,6 +1008,7 @@ static void iomfb_queue_atomic(struct apple_dcp *dcp, struct drm_crtc *crtc, atomic->transaction.release = iomfb_atomic_release; atomic->state = drm_atomic_commit_get(state); atomic->crtc = crtc; + atomic->dcp = dcp; if (!iomfb_queue(dcp, &atomic->transaction)) return; iomfb_atomic_release(&atomic->transaction); @@ -1043,7 +1152,7 @@ int iomfb_start_rtkit(struct apple_dcp *dcp) if (iomfb_uses_queue(dcp)) { mutex_lock(&dcp->iomfb.lock); - if (dcp->iomfb.active) { + if (dcp->iomfb.active || dcp->iomfb.scanout) { mutex_unlock(&dcp->iomfb.lock); return -EBUSY; } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index d59e16b40be04..d1b7d4243cea8 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -249,6 +249,8 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, dcp->crashed = true; return; } + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + iomfb_scanout_complete_h17p(dcp); #endif trace_iomfb_swap_complete(dcp, resp->swap_id); dcp->last_swap_id = resp->swap_id; From fcd5cf6a35e6854247e1c3302ec730bb011bb047 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 04:37:53 +1100 Subject: [PATCH 1073/2680] drm: apple: encode H17P commanded nits without floating point Linux-side captures during ordinary backlight sysfs changes show integer commanded levels in a little-endian binary64 field at offset 0x35e of the present record. Controlled levels of 2, 36, 140 and 400 nits produce those exact bytes while a retained framebuffer's blend CRC remains constant. Encode integer nits by constructing the exponent and mantissa, avoiding floating-point operations in the kernel. Write only the measured eight bytes in the staging record. This helper does not enable the backlight or assign meaning to surrounding calibration and control fields. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 18 ++++++++++++++++++ drivers/gpu/drm/apple/iomfb_h17p.h | 2 ++ 2 files changed, 20 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 2b922b326e27b..cb6b95f446d8b 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -1,6 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright The Asahi Linux Contributors */ +#include +#include + #include "dcp-link.h" #include "iomfb_h17p.h" #include "iomfb_v12_3.h" @@ -107,6 +110,21 @@ void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, memset(wire->tail + tail_size, 0, sizeof(wire->tail) - tail_size); } +/* Integer commanded nits are exact binary64 values; no kernel FP is needed. */ +void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits) +{ + u64 value = 0; + unsigned int exponent; + + if (nits) { + exponent = fls(nits) - 1; + value = (u64)(exponent + 1023) << 52; + value |= ((u64)nits << (52 - exponent)) & GENMASK_ULL(51, 0); + } + /* Measured from Linux presents during ordinary sysfs brightness changes. */ + put_unaligned_le64(value, wire->swap + 0x35e); +} + static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { IOMFB_METHOD("A000", dcpep_late_init_signal), IOMFB_METHOD_H17("A025", "A029", dcpep_setup_video_limits), /* (0,0) */ diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 2b260ebb2c86b..0121d334c214a 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -67,6 +67,8 @@ static_assert(sizeof(struct dcp_present_h17p) == 0xe9c); static_assert(offsetof(struct dcp_present_h17p, surf) == 0x588); static_assert(offsetof(struct dcp_present_h17p, tail) == 0xe38); +void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits); + void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, const struct dcp_swap_submit_req_h17p *request); From 7f9c5d2afada68c95bdcfb3a24044e6bdba0cfa9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 04:46:59 +1100 Subject: [PATCH 1074/2680] drm: apple: couple H17P backlight updates to completed presents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backlight updates need a valid surface even on a static screen or while DRM considers the internal CRTC inactive. Use the transaction queue to re-present the pinned, completed scanout. Coalesce brightness requests and reserve the policy sequence when preparing the wire record. Update cached actual nits only after accepted submission and its matching completion. Rejection cancels the reservation without updating actual. Brightness-only completions neither arm retirement nor release displaced framebuffers or signal another commit's flip event. Requests made during a successful present remain pending for the next operation; rejection does not create an automatic firmware retry loop. Expose a configuration hook requiring admitted range and takeover inputs; do not enable it with guessed values. Internal H17P DPMS preserves the pipe and requests zero or the retained target, reporting unavailable policy until configured. Preserve older firmware and H17G handling. Prepare any external H17P clear record inside the serialized reply chain, so a power request cannot overwrite a present still in flight. [ryan: adapted to 7.2.10 — dcp_prepare_clear_swap() keeps Asahi's 14.7 bl_update selection from the code it replaces; struct drm_atomic_state is struct drm_atomic_commit] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 15 +++ drivers/gpu/drm/apple/dcp.c | 12 +- drivers/gpu/drm/apple/dcp_backlight.c | 3 + drivers/gpu/drm/apple/iomfb.c | 120 +++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 153 +++++++++++++++++-------- drivers/gpu/drm/apple/iomfb_template.h | 4 +- 6 files changed, 255 insertions(+), 52 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index a6db905f9d8ca..846cd4d459ca5 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -43,6 +43,10 @@ struct iomfb_transaction { struct list_head link; void (*start)(struct apple_dcp *dcp, struct iomfb_transaction *transaction); void (*release)(struct iomfb_transaction *transaction); + struct dcp_backlight_present backlight; + bool backlight_reserved; + bool brightness_only; + bool completed; }; struct apple_dcp_typec_port; @@ -375,12 +379,23 @@ struct apple_dcp { unsigned long deadline; unsigned int queued; bool stopped; + bool backlight_queued; struct iomfb_scanout_h17p *scanout; struct iomfb_scanout_h17p *next_scanout; } iomfb; }; void iomfb_scanout_complete_h17p(struct apple_dcp *dcp); +void iomfb_present_backlight_h17p(struct apple_dcp *dcp); +bool iomfb_present_brightness_only_h17p(struct apple_dcp *dcp); +bool iomfb_present_complete_h17p(struct apple_dcp *dcp); +void iomfb_present_failed_h17p(struct apple_dcp *dcp); +bool iomfb_apply_backlight_h17p(struct apple_dcp *dcp, + const struct dcp_swap_submit_req_h17p *request, + struct dcp_present_h17p *wire); +int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum, + bool inherited_valid, u32 inherited, + bool default_valid, u32 default_nits); void iomfb_queue_init(struct apple_dcp *dcp); void iomfb_queue_stop(struct apple_dcp *dcp); int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 0e20c26c627f0..e67d766bfdaa2 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1489,12 +1489,20 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) } } +static bool dcp_uses_soft_dpms(struct apple_dcp *dcp) +{ + return (dcp_backlight_active(dcp) && dcp_has_panel(dcp)) || + (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + dcp->connector_type == DRM_MODE_CONNECTOR_eDP); +} + void dcp_poweron(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; - if (dcp_backlight_active(dcp) && dcp_has_panel(dcp)) { + if (dcp_uses_soft_dpms(dcp)) { ret = dcp_backlight_dpms(dcp, true); if (ret) dev_warn(dcp->dev, "backlight restore unavailable: %d\n", ret); @@ -1553,7 +1561,7 @@ void dcp_poweroff(struct platform_device *pdev) int ret; /* Internal H17P DPMS is a brightness present, never a pipe stop. */ - if (dcp_backlight_active(dcp) && dcp_has_panel(dcp)) { + if (dcp_uses_soft_dpms(dcp)) { ret = dcp_backlight_dpms(dcp, false); if (ret) dev_warn(dcp->dev, "backlight blank unavailable: %d\n", ret); diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index a3065e63f3c7d..298d4e4380a06 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -174,6 +174,9 @@ int dcp_backlight_prepare(struct apple_dcp *dcp, bool have_surface, spin_lock_irqsave(&dcp->backlight.lock, flags); ret = dcp_bl_prepare(&dcp->backlight.state, have_surface, present); + /* No reservation: carry the last completed level on an ordinary present. */ + if (ret == -EALREADY) + present->nits = dcp->backlight.state.actual; spin_unlock_irqrestore(&dcp->backlight.lock, flags); return ret; } diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index ca2cb85cf2fff..575869f599b5e 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -179,6 +179,8 @@ static void iomfb_discard_pending(struct apple_dcp *dcp) list_for_each_entry_safe(transaction, next, &dcp->iomfb.pending, link) { list_del(&transaction->link); dcp->iomfb.queued--; + if (transaction->brightness_only) + dcp->iomfb.backlight_queued = false; transaction->release(transaction); } } @@ -199,6 +201,11 @@ static void iomfb_queue_advance(struct apple_dcp *dcp) if (transaction) { dcp->iomfb.active = NULL; cancel_delayed_work(&dcp->iomfb.timeout); + if (transaction->brightness_only) + dcp->iomfb.backlight_queued = false; + /* A new level requested during a successful present stays pending. */ + if (transaction->completed && dcp_backlight_pending(dcp)) + schedule_work(&dcp->bl_update_wq); transaction->release(transaction); } if (!list_empty(&dcp->iomfb.pending)) @@ -280,9 +287,13 @@ int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction) mutex_lock(&dcp->iomfb.lock); if (READ_ONCE(dcp->crashed) || dcp->iomfb.stopped) { ret = -EIO; + } else if (transaction->brightness_only && dcp->iomfb.backlight_queued) { + ret = -EALREADY; } else if (dcp->iomfb.queued >= 32) { ret = -EBUSY; } else { + if (transaction->brightness_only) + dcp->iomfb.backlight_queued = true; dcp->iomfb.queued++; list_add_tail(&transaction->link, &dcp->iomfb.pending); schedule_work(&dcp->iomfb.work); @@ -954,6 +965,115 @@ void iomfb_scanout_complete_h17p(struct apple_dcp *dcp) iomfb_scanout_release_h17p(previous); } +bool iomfb_present_brightness_only_h17p(struct apple_dcp *dcp) +{ + return dcp->iomfb.active && dcp->iomfb.active->brightness_only; +} + +bool iomfb_apply_backlight_h17p(struct apple_dcp *dcp, + const struct dcp_swap_submit_req_h17p *request, + struct dcp_present_h17p *wire) +{ + struct iomfb_transaction *transaction = dcp->iomfb.active; + struct dcp_backlight_present present; + bool have_surface = false; + unsigned int i; + int ret; + + lockdep_assert_held(&dcp->iomfb.lock); + if (!dcp_backlight_active(dcp)) + return true; + if (!transaction) + return false; + for (i = 0; i < SWAP_SURFACES; i++) + have_surface |= !request->surf_null[i]; + ret = dcp_backlight_prepare(dcp, have_surface, &present); + if (ret && ret != -EALREADY) + return false; + if (!ret) { + transaction->backlight = present; + transaction->backlight_reserved = true; + } + iomfb_encode_backlight_h17p(wire, present.nits); + return true; +} + +void iomfb_present_failed_h17p(struct apple_dcp *dcp) +{ + struct iomfb_transaction *transaction = dcp->iomfb.active; + + if (transaction && transaction->backlight_reserved) { + dcp_backlight_complete(dcp, transaction->backlight.sequence, false); + transaction->backlight_reserved = false; + } +} + +bool iomfb_present_complete_h17p(struct apple_dcp *dcp) +{ + struct iomfb_transaction *transaction = dcp->iomfb.active; + + lockdep_assert_held(&dcp->iomfb.lock); + if (transaction) { + transaction->completed = true; + if (transaction->backlight_reserved) { + dcp_backlight_complete(dcp, transaction->backlight.sequence, true); + transaction->backlight_reserved = false; + } + if (transaction->brightness_only) + return false; + } + iomfb_scanout_complete_h17p(dcp); + return true; +} + +static void iomfb_backlight_start(struct apple_dcp *dcp, + struct iomfb_transaction *transaction) +{ + struct iomfb_scanout_h17p *scanout = dcp->iomfb.scanout; + bool have_surface = false; + unsigned int i; + + if (!dcp_backlight_pending(dcp) || !dcp->valid_mode || !scanout || + !dcp->connector || !dcp->connector->connected) + return; + for (i = 0; i < SWAP_SURFACES; i++) + have_surface |= !!scanout->fb[i]; + if (!have_surface) + return; + + dcp->swap.h17p = scanout->request; + iomfb_present_backlight_h17p(dcp); +} + +static void iomfb_backlight_release(struct iomfb_transaction *transaction) +{ + kfree(transaction); +} + +static void iomfb_backlight_kick(struct apple_dcp *dcp) +{ + struct iomfb_transaction *transaction; + + transaction = kzalloc_obj(*transaction); + if (!transaction) + return; + transaction->start = iomfb_backlight_start; + transaction->release = iomfb_backlight_release; + transaction->brightness_only = true; + if (iomfb_queue(dcp, transaction)) + kfree(transaction); +} + +int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum, + bool inherited_valid, u32 inherited, + bool default_valid, u32 default_nits) +{ + if (!iomfb_uses_queue(dcp)) + return -EINVAL; + return dcp_backlight_configure(dcp, maximum, inherited_valid, inherited, + default_valid, default_nits, iomfb_backlight_kick); +} + struct iomfb_atomic_transaction { struct iomfb_transaction transaction; struct drm_atomic_commit *state; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index d1b7d4243cea8..9aba956c4afe6 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -70,6 +70,11 @@ static void dcp_swap_submit(struct apple_dcp *dcp, bool oob, if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { iomfb_serialize_present_h17p(&dcp->present_h17p, request); + if (!iomfb_apply_backlight_h17p(dcp, request, &dcp->present_h17p)) { + if (cb) + cb(dcp, NULL, cookie); + return; + } data = &dcp->present_h17p; } @@ -241,6 +246,7 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, struct DCP_FW_NAME(dc_swap_complete_resp) *resp) { ktime_t now = ktime_get(); + bool retire = true; #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && @@ -250,13 +256,14 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, return; } if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) - iomfb_scanout_complete_h17p(dcp); + retire = iomfb_present_complete_h17p(dcp); #endif trace_iomfb_swap_complete(dcp, resp->swap_id); dcp->last_swap_id = resp->swap_id; - dcp_release_retained_framebuffers(dcp, resp->swap_id); - - dcp_drm_crtc_page_flip(dcp, now); + if (retire) { + dcp_release_retained_framebuffers(dcp, resp->swap_id); + dcp_drm_crtc_page_flip(dcp, now); + } if (dcp->crc_enabled) { u32 crc32 = 0; drm_crtc_add_crc_entry(&dcp->crtc->base, true, resp->swap_id, &crc32); @@ -1248,6 +1255,23 @@ static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) { } #endif +static bool dcp_present_retires(struct apple_dcp *dcp) +{ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + return !iomfb_present_brightness_only_h17p(dcp); +#endif + return true; +} + +static void dcp_present_failed(struct apple_dcp *dcp) +{ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + iomfb_present_failed_h17p(dcp); +#endif +} + static bool dcp_present_begin(struct apple_dcp *dcp, u32 swap_id) { #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER @@ -1274,6 +1298,47 @@ static bool dcp_present_submit(struct apple_dcp *dcp, u32 swap_id, bool accepted return true; } +static void dcp_prepare_clear_swap(struct apple_dcp *dcp, void *request) +{ + typeof(DCP_FW_UNION(dcp->swap)) *swap = request; + + /* Clear surfaces. */ + memset(swap, 0, sizeof(*swap)); + + swap->swap.swap_enabled = + swap->swap.swap_completed = IOMFB_SET_BACKGROUND | 0x7; + swap->swap.bg_color = 0xFF000000; + + /* + * Turn off the backlight. This matters because the DCP's idea of + * backlight brightness gets desynced after a power change, and it + * needs to be told it's going to turn off so it will consider the + * subsequent update on poweron an actual change and restore the + * brightness. + */ + if (dcp_has_panel(dcp)) { +#if !DCP_FW_LAYOUT_14_7(DCP_FW_VER) + swap->swap.bl_unk = 1; +#else + swap->swap.bl_update = 1; +#endif + swap->swap.bl_value = 0; + swap->swap.bl_power = 0; + } + + /* Null all surfaces */ + for (int l = 0; l < SWAP_SURFACES; l++) + swap->surf_null[l] = true; +#if DCP_FW_VERSION(13, 2, 0) <= DCP_FW_VER +#if DCP_FW_VERSION(26, 0, 0) > DCP_FW_VER + for (int l = 0; l < 5; l++) + swap->surf2_null[l] = true; +#endif + swap->unknown_pointer_null = true; + swap->unknown_output_null = true; +#endif +} + static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; @@ -1286,8 +1351,11 @@ static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) if (status) { dev_err(dcp->dev, "swap_clear failed! status %u\n", status); - dcp_unarm_retained_framebuffers(dcp, swap_id); - dcp_drm_crtc_vblank(dcp->crtc); + dcp_present_failed(dcp); + if (dcp_present_retires(dcp)) { + dcp_unarm_retained_framebuffers(dcp, swap_id); + dcp_drm_crtc_vblank(dcp->crtc); + } } if (info) { WRITE_ONCE(info->status, status); @@ -1320,8 +1388,13 @@ static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, dcp_drm_crtc_vblank(dcp->crtc); return; } +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + dcp_prepare_clear_swap(dcp, &DCP_FW_UNION(dcp->swap)); +#endif DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; - dcp_arm_retained_framebuffers(dcp, resp->swap_id); + if (dcp_present_retires(dcp)) + dcp_arm_retained_framebuffers(dcp, resp->swap_id); if (info) info->swap_id = resp->swap_id; @@ -1477,7 +1550,7 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) struct dcp_swap_cookie *cookie; struct dcp_wait_cookie *poff_cookie; struct DCP_FW_NAME(dcp_swap_start_req) swap_req = { 0 }; - struct DCP_FW_NAME(dcp_swap_submit_req) *swap = &DCP_FW_UNION(dcp->swap); + typeof(DCP_FW_UNION(dcp->swap)) *swap = &DCP_FW_UNION(dcp->swap); cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) @@ -1488,40 +1561,11 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) /* increase refcount to ensure the receiver has a reference */ kref_get(&cookie->refcount); - // clear surfaces - memset(swap, 0, sizeof(*swap)); - - swap->swap.swap_enabled = - swap->swap.swap_completed = IOMFB_SET_BACKGROUND | 0x7; - swap->swap.bg_color = 0xFF000000; - - /* - * Turn off the backlight. This matters because the DCP's idea of - * backlight brightness gets desynced after a power change, and it - * needs to be told it's going to turn off so it will consider the - * subsequent update on poweron an actual change and restore the - * brightness. - */ - if (dcp_has_panel(dcp)) { -#if !DCP_FW_LAYOUT_14_7(DCP_FW_VER) - swap->swap.bl_unk = 1; +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + dcp_prepare_clear_swap(dcp, swap); #else - swap->swap.bl_update = 1; -#endif - swap->swap.bl_value = 0; - swap->swap.bl_power = 0; - } - - /* Null all surfaces */ - for (int l = 0; l < SWAP_SURFACES; l++) - swap->surf_null[l] = true; -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) -#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) - for (int l = 0; l < 5; l++) - swap->surf2_null[l] = true; -#endif - swap->unkU32Ptr_null = true; - swap->unkU32out_null = true; + dcp_prepare_clear_swap(dcp, swap); #endif dcp_swap_start(dcp, false, &swap_req, dcp_swap_clear_started, cookie); @@ -1808,8 +1852,11 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) if (status) { dev_err(dcp->dev, "swap failed! status %u\n", status); - dcp_unarm_retained_framebuffers(dcp, swap_id); - dcp_drm_crtc_vblank(dcp->crtc); + dcp_present_failed(dcp); + if (dcp_present_retires(dcp)) { + dcp_unarm_retained_framebuffers(dcp, swap_id); + dcp_drm_crtc_vblank(dcp->crtc); + } return; } dcp->swap_start = ktime_get(); @@ -1822,15 +1869,18 @@ static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) if (!resp || resp->ret) { dev_err(dcp->dev, "swap_start was rejected\n"); - dcp_drm_crtc_vblank(dcp->crtc); + if (dcp_present_retires(dcp)) + dcp_drm_crtc_vblank(dcp->crtc); return; } if (!dcp_present_begin(dcp, resp->swap_id)) { - dcp_drm_crtc_vblank(dcp->crtc); + if (dcp_present_retires(dcp)) + dcp_drm_crtc_vblank(dcp->crtc); return; } DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; - dcp_arm_retained_framebuffers(dcp, resp->swap_id); + if (dcp_present_retires(dcp)) + dcp_arm_retained_framebuffers(dcp, resp->swap_id); trace_iomfb_swap_submit(dcp, resp->swap_id); dcp_h17p_prepare_swap(dcp); @@ -1853,6 +1903,13 @@ static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) * H17P expects A428 between set_matrix and swap_start, with 0x10000 (1.0 in * 16.16 fixed point), the same identity value the D006 reply carries. */ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER +void iomfb_present_backlight_h17p(struct apple_dcp *dcp) +{ + do_swap(dcp, NULL, NULL); +} +#endif + static void dcp_pre_swap_a428(struct apple_dcp *dcp, void *out, void *cookie) { u32 identity = 0x10000; @@ -2039,8 +2096,8 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru for (l = 0; l < 5; l++) req->surf2_null[l] = true; #endif - req->unkU32Ptr_null = true; - req->unkU32out_null = true; + req->unknown_pointer_null = true; + req->unknown_output_null = true; #endif /* diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index 06584b0291964..b987ddfbd8ce7 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -144,8 +144,8 @@ struct DCP_FW_NAME(dcp_swap_submit_req) { #endif u8 unkoutbool_null; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - u8 unkU32Ptr_null; - u8 unkU32out_null; + u8 unknown_pointer_null; + u8 unknown_output_null; #endif #if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) u8 padding[1]; From 387f3c766b17ca424f158366b08eb20459240757 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 04:49:40 +1100 Subject: [PATCH 1075/2680] drm: apple: stop on H17P callbacks without reply semantics A measured callback length establishes buffer bounds but does not make an all-zero reply valid. The H17P dispatch table still lacks a handler for the measured property request; treating that omission as a successful empty reply would hide an incomplete protocol admission. Stop the measured H17P interface before modifying output or acknowledging an unhandled callback. Keep the active transaction and its surface references pinned for recovery by reboot. Explicitly supported handlers still receive zero-initialized output, and the existing H17G fallback is preserved. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 5 +++++ drivers/gpu/drm/apple/iomfb_h17p.c | 13 ++++--------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 575869f599b5e..fc4b1e87fee3a 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -587,6 +587,11 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, !dcp->cb_handlers[tag]) { dev_warn(dev, "received unknown callback %c%c%c%c\n", hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); + if (iomfb_uses_queue(dcp)) { + /* A measured size does not establish a safe reply payload. */ + WRITE_ONCE(dcp->crashed, true); + return; + } /* * Leaving a callback unanswered wedges the coprocessor: it * waits for the ack forever and the outer call never returns. diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index cb6b95f446d8b..dc5f0e0749665 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -259,17 +259,12 @@ static bool trampoline_swap_info_h17p(struct apple_dcp *dcp, int tag, } /* - * D400 (get_property) is left unhandled: the zeroed reply means "no such - * property". The firmware then runs without a panel power model, as it does - * for its other optional power properties. + * D400 has a measured size but no admitted property-reply semantics. It stays + * unhandled, so the measured H17P profile stops instead of guessing a reply. + * H17G retains its existing missing-property response. */ -/* - * H17P callback numbering. It is not a uniform shift of the v13.5 table: the - * service-creation block is D108..D113 exactly as on v13.5, and the swap - * completion is D590. A callback with no entry here is answered with a - * zeroed reply by dcpep_handle_cb(). - */ +/* H17P callback numbering is not a uniform shift of the v13.5 table. */ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [0] = dcpep_cb_d000_h17p, /* acked after a nested A033 */ [1] = trampoline_true, From 43f533c36e420431cf13f610b715183842dd7019 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 05:36:15 +1100 Subject: [PATCH 1076/2680] drm: apple: preserve unknown H17P background on replay The first completed present need not set the background. Always adding a background update to its cached replay would then send a value that the firmware had never accepted, potentially changing alpha composition during a brightness-only update. Keep the background-update bit only when this or a previous completed present supplied the background. Continue re-presenting the pinned surfaces without changing an unknown inherited background. An explicit accepted background remains available across partial-plane updates. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index fc4b1e87fee3a..8f6bd148f862b 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -958,7 +958,12 @@ void iomfb_scanout_complete_h17p(struct apple_dcp *dcp) scanout->request.surf_null[i] = !scanout->fb[i]; } /* Brightness re-presents all pinned surfaces with a fresh swap ID. */ - scanout->request.swap.swap_enabled = IOMFB_SET_BACKGROUND; + /* Replay only a background established by a completed present. */ + scanout->request.swap.swap_enabled = + request->swap.swap_enabled & IOMFB_SET_BACKGROUND; + if (previous) + scanout->request.swap.swap_enabled |= + previous->request.swap.swap_enabled & IOMFB_SET_BACKGROUND; for (i = 0; i < SWAP_SURFACES; i++) if (scanout->fb[i]) scanout->request.swap.swap_enabled |= BIT(i); From fb11c71fc458c365844040883203057c3629b494 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 05:52:14 +1100 Subject: [PATCH 1077/2680] dt-bindings: display: apple,dcp: describe firmware power tables Linux startup callbacks request five power lookup table properties from the display provider. Their replies reproduce the corresponding disp0 ADT bytes: three four-byte values and two 52-byte arrays. The table indices include the firmware's power model axis, which does not define Linux's backlight range. Describe bootloader-supplied byte arrays so the driver can transfer these per-device firmware inputs without embedding panel data or interpreting the power model. Limit the properties to the measured T8140 internal DCP and document their little-endian representation. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/display/apple,dcp.yaml | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index 33f1bf6e44359..aefb570de5983 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -123,6 +123,49 @@ properties: enum: [0, 1] description: Per-slot enable flags, in slot order. + apple,power-lut-data-x: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 4 + maxItems: 4 + description: | + First dimension of the display firmware power lookup table. The + bootloader copies the disp0 ADT property verbatim, as a little-endian + 32-bit word, for transfer to the display firmware. + + apple,power-lut-data-y: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 4 + maxItems: 4 + description: | + Second dimension of the display firmware power lookup table, copied + verbatim from disp0 in the ADT as a little-endian 32-bit word. + + apple,power-lut-data-xindex: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 52 + maxItems: 52 + description: | + First-axis indices of the display firmware power lookup table, copied + verbatim from disp0 in the ADT as thirteen little-endian 32-bit words. + These are firmware table inputs, not a Linux backlight range. + + apple,power-lut-data-yindex: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 4 + maxItems: 4 + description: | + Second-axis index of the display firmware power lookup table, copied + verbatim from disp0 in the ADT as a little-endian 32-bit word. + + apple,power-lut-data-lut: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 52 + maxItems: 52 + description: | + Display firmware power lookup table, copied verbatim from disp0 in + the ADT as thirteen little-endian 32-bit words. Linux passes these + values to the firmware without interpreting or changing them. + apple,firmware-version: $ref: /schemas/types.yaml#/definitions/uint32-array minItems: 3 @@ -270,6 +313,19 @@ required: - apple,bw-scratch allOf: + - if: + not: + properties: + compatible: + contains: + const: apple,t8140-dcp + then: + properties: + apple,power-lut-data-x: false + apple,power-lut-data-y: false + apple,power-lut-data-xindex: false + apple,power-lut-data-yindex: false + apple,power-lut-data-lut: false - if: properties: compatible: From 6174d1684a77ea109b1e314709cf08b8c43dbbb7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 05:54:56 +1100 Subject: [PATCH 1078/2680] drm: apple: transfer measured H17P provider power tables During Linux startup the display firmware requests five provider power table properties. The returned bytes match disp0 in the ADT, followed by zero padding and a little-endian byte count. A sixth request refers to a property absent from that node and receives an empty reply. Firmware leaves bytes following the key's first NUL uninitialized. Handle these transfers using bootloader-supplied DT data. Validate the observed scope, capacity, trailing bytes, terminated key and exact data sizes before replying. Missing required data, unknown keys and unmeasured request variants stop the transaction instead of inventing a property. Keep the existing empty-reply behaviour for the H17G method profile. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 100 +++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index dc5f0e0749665..2baf3d7f29f77 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -258,11 +258,100 @@ static bool trampoline_swap_info_h17p(struct apple_dcp *dcp, int tag, return true; } -/* - * D400 has a measured size but no admitted property-reply semantics. It stays - * unhandled, so the measured H17P profile stops instead of guessing a reply. - * H17G retains its existing missing-property response. - */ +struct dcp_h17p_provider_request { + u8 scope[4]; + char name[64]; + __le32 capacity; + u8 reserved[4]; +} __packed; + +struct dcp_h17p_provider_reply { + u8 data[0xc00]; + __le32 length; +} __packed; + +struct dcp_h17p_provider_property { + const char *name; + const char *dt_name; + u32 length; +}; + +static_assert(sizeof(struct dcp_h17p_provider_request) == 0x4c); +static_assert(sizeof(struct dcp_h17p_provider_reply) == 0xc04); + +/* Only these provider transfers have been measured against the disp0 ADT. */ +static const struct dcp_h17p_provider_property provider_properties[] = { + { "power-lut-data-x", "apple,power-lut-data-x", 4 }, + { "power-lut-data-y", "apple,power-lut-data-y", 4 }, + { "power-lut-data-xindex", "apple,power-lut-data-xindex", 52 }, + { "power-lut-data-yindex", "apple,power-lut-data-yindex", 4 }, + { "power-lut-data-lut", "apple,power-lut-data-lut", 52 }, + { "power-lut-vbatt-cur-nominal", "apple,power-lut-vbatt-cur-nominal", 0 }, +}; + +static const struct dcp_h17p_provider_property * +dcp_provider_property_h17p(const struct dcp_h17p_provider_request *request) +{ + unsigned int i; + + if (memcmp(request->scope, "VORP", sizeof(request->scope)) || + le32_to_cpu(request->capacity) != + sizeof_field(struct dcp_h17p_provider_reply, data) || + memchr_inv(request->reserved, 0, sizeof(request->reserved)) || + strnlen(request->name, sizeof(request->name)) == sizeof(request->name)) + return NULL; + + /* Bytes following the first NUL were not initialized by the firmware. */ + for (i = 0; i < ARRAY_SIZE(provider_properties); i++) + if (!strcmp(request->name, provider_properties[i].name)) + return &provider_properties[i]; + return NULL; +} + +static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_provider_property *property; + struct dcp_h17p_provider_reply *reply = out; + struct device_node *node = dcp->dev->of_node; + int length, ret; + + /* The dispatcher already cleared the exact H17G output length. */ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return true; + + trace_iomfb_callback(dcp, tag, __func__); + property = dcp_provider_property_h17p(in); + if (!property) { + dev_err(dcp->dev, "unqualified D400 provider property request\n"); + goto fail; + } + + length = of_property_count_u8_elems(node, property->dt_name); + if (!property->length) { + /* Only this absent property has a measured empty reply. */ + if (of_property_present(node, property->dt_name)) + goto fail; + } else if (length != (int)property->length) { + dev_err(dcp->dev, "invalid provider property %s: %d bytes\n", + property->dt_name, length); + goto fail; + } + + memset(reply, 0, sizeof(*reply)); + if (property->length) { + ret = of_property_read_u8_array(node, property->dt_name, + reply->data, property->length); + if (ret) + goto fail; + } + reply->length = cpu_to_le32(property->length); + return true; + +fail: + WRITE_ONCE(dcp->crashed, true); + return false; +} /* H17P callback numbering is not a uniform shift of the v13.5 table. */ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { @@ -303,6 +392,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [208] = trampoline_nop, /* update_backlight_factor_prop */ [209] = trampoline_get_time, /* get_calendar_time_ms */ [300] = trampoline_pr_publish, + [400] = trampoline_provider_property_h17p, [401] = trampoline_get_uint_prop, /* get_uint_prop */ [404] = trampoline_nop, /* set_uint_prop */ [406] = trampoline_set_fx_prop, /* set_fx_prop */ From 729055102faf0378327843b6bc2471fd526086cc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:11:18 +1100 Subject: [PATCH 1079/2680] drm: apple: unwind failed firmware buffer allocations The firmware buffer callback reserves a descriptor before allocating its backing memory and ignores failures from both dma_alloc_coherent() and dma_get_sgtable(). A failed allocation can therefore reach scatterlist creation with a NULL buffer, or leave an occupied descriptor without a usable mapping. Release also discards the scatterlist without freeing it. Reject zero-sized and overflowing requests before either page-rounding operation. Build the descriptor locally and publish it only after both DMA operations succeed. Free the coherent allocation if scatterlist creation fails, and release the scatterlist together with its backing allocation. Successful requests retain the existing wire-size rounding and reply fields. Fault injection exercises both failure points, descriptor exhaustion, rounding boundaries, retry after failure and release for the older firmware and H17P template profiles. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_template.c | 56 ++++++++++++++++---------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 9aba956c4afe6..684d0258d1383 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -594,35 +595,45 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, struct dcp_allocate_buffer_req *req) { struct dcp_allocate_buffer_resp resp = { 0 }; - struct dcp_mem_descriptor *memdesc; - size_t size; + struct dcp_mem_descriptor allocated = { 0 }; + size_t size, rounded; u32 id; + int ret; + + /* Validate both wire-size and IOMMU-page rounding before allocating. */ + if (!req->size || req->size > SIZE_MAX || + check_add_overflow((size_t)req->size, (size_t)4095, &rounded)) + return resp; + allocated.size = round_down(rounded, 4096); + if (check_add_overflow(allocated.size, (size_t)SZ_16K - 1, &rounded)) + return resp; + size = round_down(rounded, SZ_16K); - resp.dva_size = ALIGN(req->size, 4096); - resp.mem_desc_id = - find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); + id = find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); - if (resp.mem_desc_id >= DCP_MAX_MAPPINGS) { + if (id >= DCP_MAX_MAPPINGS) { dev_warn(dcp->dev, "DCP overflowed mapping table, ignoring\n"); - resp.dva_size = 0; - resp.mem_desc_id = 0; return resp; } - id = resp.mem_desc_id; - set_bit(id, dcp->memdesc_map); - memdesc = &dcp->memdesc[id]; + allocated.buf = dma_alloc_coherent(dcp->dev, size, &allocated.dva, + GFP_KERNEL); + if (!allocated.buf) + return resp; - memdesc->piodma_mapped = false; - memdesc->size = resp.dva_size; - /* HACK: align size to 16K since the iommu API only maps full pages */ - size = ALIGN(resp.dva_size, SZ_16K); - memdesc->buf = dma_alloc_coherent(dcp->dev, size, - &memdesc->dva, GFP_KERNEL); + ret = dma_get_sgtable(dcp->dev, &allocated.map, allocated.buf, + allocated.dva, size); + if (ret) { + dma_free_coherent(dcp->dev, size, allocated.buf, allocated.dva); + return resp; + } - dma_get_sgtable(dcp->dev, &memdesc->map, memdesc->buf, memdesc->dva, - size); - resp.dva = memdesc->dva; + /* Callbacks are serialized; publish only a fully initialized descriptor. */ + dcp->memdesc[id] = allocated; + set_bit(id, dcp->memdesc_map); + resp.mem_desc_id = id; + resp.dva_size = allocated.size; + resp.dva = allocated.dva; #if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) /* @@ -630,8 +641,8 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, * microcode through it, and with a zero paddr the timing sequencer * comes up and is powered straight back down. */ - if (memdesc->map.sgl) - resp.paddr = sg_phys(memdesc->map.sgl); + if (allocated.map.sgl) + resp.paddr = sg_phys(allocated.map.sgl); #endif return resp; @@ -675,6 +686,7 @@ static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) } if (memdesc->buf) { + sg_free_table(&memdesc->map); dma_free_coherent(dcp->dev, size, memdesc->buf, memdesc->dva); memdesc->buf = NULL; memset(&memdesc->map, 0, sizeof(memdesc->map)); From a0e75faed33ade41ba491475081c85c2521ce0d3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:13:59 +1100 Subject: [PATCH 1080/2680] drm: apple: qualify H17P firmware buffer allocation replies Linux startup measurements correlate seven firmware allocation replies with the core DMA allocation and IOMMU mapping events. The first reply address matches the first physical scatterlist page, the second matches the device DMA address, and the reported length is rounded to 4 KiB. Two allocations have physically noncontiguous backing pages. Route H17P allocation requests through a bounded callback that admits the two observed option combinations and zero trailing/null bytes. Reuse the existing Asahi allocator and encode its successful reply explicitly in little endian. Stop the transaction when validation or allocation fails, since no H17P error reply has been measured. The H17G profile keeps its existing callback path. Describe the first address as a measured scatterlist correlation rather than assigning an unmeasured firmware use to it. This does not establish secondary-domain ownership or enable unqualified teardown callbacks. Replay all seven complete replies against independent core trace inputs, and exercise option mutations, both DMA failure points, overflow and mapping-table exhaustion. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 45 +++++++++++++++++++++++++- drivers/gpu/drm/apple/iomfb_template.c | 5 ++- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 2baf3d7f29f77..f98fe6459d537 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -353,6 +353,49 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, return false; } +static bool trampoline_allocate_buffer_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_allocate_buffer_req *wire = in; + struct dcp_allocate_buffer_req request = { 0 }; + struct dcp_allocate_buffer_resp reply; + u32 options; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return trampoline_allocate_buffer(dcp, tag, out, in); + + static_assert(sizeof(*wire) == 0x14); + static_assert(sizeof(reply) == 0x1c); + static_assert(offsetof(struct dcp_allocate_buffer_req, size) == 4); + static_assert(offsetof(struct dcp_allocate_buffer_req, unk2) == 12); + static_assert(offsetof(struct dcp_allocate_buffer_req, paddr_null) == 16); + + trace_iomfb_callback(dcp, tag, __func__); + options = get_unaligned_le32(&wire->unk2); + /* Only these option combinations have observed successful replies. */ + if (get_unaligned_le32(&wire->unk0) != 0x703 || + (options != 4 && options != SZ_16K) || + memchr_inv(&wire->paddr_null, 0, 4)) + goto fail; + + request.size = get_unaligned_le64(&wire->size); + reply = dcpep_cb_allocate_buffer(dcp, &request); + if (!reply.dva_size) + goto fail; + + put_unaligned_le64(reply.paddr, out); + put_unaligned_le64(reply.dva, (u8 *)out + 8); + put_unaligned_le64(reply.dva_size, (u8 *)out + 16); + put_unaligned_le32(reply.mem_desc_id, (u8 *)out + 24); + return true; + +fail: + /* Allocation failures have no measured H17P reply encoding. */ + dev_err(dcp->dev, "unqualified or failed D451 buffer allocation\n"); + WRITE_ONCE(dcp->crashed, true); + return false; +} + /* H17P callback numbering is not a uniform shift of the v13.5 table. */ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [0] = dcpep_cb_d000_h17p, /* acked after a nested A033 */ @@ -401,7 +444,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [413] = trampoline_true, /* setProperty */ [414] = trampoline_sr_set_property_int, /* setProperty */ [415] = trampoline_true, /* setProperty */ - [451] = trampoline_allocate_buffer, /* allocate_buffer */ + [451] = trampoline_allocate_buffer_h17p, /* allocate_buffer */ [452] = trampoline_map_physical, /* prepare */ [454] = trampoline_release_mem_desc, /* release_descriptor */ [552] = trampoline_true, diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 684d0258d1383..a31006a3e3c09 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -637,9 +637,8 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, #if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) /* - * H17P firmware also needs the physical address: it loads the APT M3 - * microcode through it, and with a zero paddr the timing sequencer - * comes up and is powered straight back down. + * Linux allocation traces correlate the first H17P reply address with + * the first SG page. The remaining pages need not be contiguous. */ if (allocated.map.sgl) resp.paddr = sg_phys(allocated.map.sgl); From 83979d2ce07fd4bc097da1403c970bfb2274a7b6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:35:59 +1100 Subject: [PATCH 1081/2680] drm: apple: validate compact H17P receive message lengths Linux startup capture on the internal display endpoint records seventeen compact messages. Their header length counts all bytes after the first eight bytes, and the service channel is carried at offset two. Announce and report payloads start after the compact 24-byte header. Check the recorded length against the actual queue-entry size before normalizing a measured H17P receive header or dispatching its payload. Keep the existing H17G framing path unchanged. No outgoing compact message encoding is introduced by this receive-side check. Captured messages, unaligned buffers, truncated headers and length-bit mutations exercise the actual validator under address/undefined checks. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index daa27e1ad7238..9440662d55788 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -554,6 +555,14 @@ static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, payload_size, true); } +static bool afk_validate_h17p_header(const u8 *data, size_t size) +{ + /* The measured length counts bytes after the first eight header bytes. */ + return size >= sizeof(struct epic_hdr) + + sizeof(struct epic_sub_hdr_h17p) && + get_unaligned_le32(data + 4) == size - 8; +} + static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, u8 *data, size_t data_size) { @@ -574,6 +583,13 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, hdr_len = sizeof(*ehdr) + sizeof(*c); if (data_size < hdr_len) goto too_small; + if (ep->dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !afk_validate_h17p_header(data, data_size)) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: invalid compact message length\n", + ep->endpoint); + return; + } /* * H17P multiplexes every service of an endpoint onto queue From de677e4b017701adfb5328644ddfb27182e8064c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:37:26 +1100 Subject: [PATCH 1082/2680] drm: apple: answer the measured H17P startup integer query Three Linux startup transactions query the PMU Temperature property and receive the same 64-bit value 3029 with a successful boolean result. This is the existing Asahi firmware reply value, but the generic handler only returns it for panels marked as mini-LED. The internal H17P panel also makes this startup query. Use the observed reply for the measured H17P property request, validating the object, bounded property name and zero null/trailing bytes. Bytes after the name terminator and the input's previous value are ignored. Stop on unmeasured queries rather than returning an invented zero value. The H17G profile retains the generic panel-dependent behavior. The scalar is a firmware interoperability value; its temperature units and any relationship to a physical sensor have not been established. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index f98fe6459d537..eed7625b4d3a7 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -353,6 +353,30 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, return false; } +static bool trampoline_get_uint_prop_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_get_uint_prop_req *request = in; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return trampoline_get_uint_prop(dcp, tag, out, in); + + trace_iomfb_callback(dcp, tag, __func__); + /* The observed startup query uses Asahi's existing Temperature reply. */ + if (memcmp(request->obj, "SUMP", sizeof(request->obj)) || + strnlen(request->key, sizeof(request->key)) == sizeof(request->key) || + strcmp(request->key, "Temperature") || request->value_null || + memchr_inv(request->padding, 0, sizeof(request->padding))) { + dev_err(dcp->dev, "unqualified D401 integer property query\n"); + WRITE_ONCE(dcp->crashed, true); + return false; + } + + put_unaligned_le64(3029, out); + *((u8 *)out + offsetof(struct dcp_get_uint_prop_resp, ret)) = true; + return true; +} + static bool trampoline_allocate_buffer_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) { @@ -436,7 +460,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [209] = trampoline_get_time, /* get_calendar_time_ms */ [300] = trampoline_pr_publish, [400] = trampoline_provider_property_h17p, - [401] = trampoline_get_uint_prop, /* get_uint_prop */ + [401] = trampoline_get_uint_prop_h17p, /* get_uint_prop */ [404] = trampoline_nop, /* set_uint_prop */ [406] = trampoline_set_fx_prop, /* set_fx_prop */ [408] = trampoline_get_frequency_h17p, /* getClockFrequency */ From 5df3c2979f558c8ad50393ef6688935070312ba2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:39:36 +1100 Subject: [PATCH 1083/2680] drm: apple: select the internal T8140 H17P method profile Linux startup measurements on T8140 identify the internal display's H17P IOMFB interface and a live bootloader-owned RTKit session. Falling back to the generic DCP match cannot select that firmware profile and would attempt to start the coprocessor instead of adopting its session. Match the internal T8140 DCP explicitly and reuse the existing H17 live-session adoption with the measured method profile. The match advertises no external display ports; external DCP qualification remains separate. The public DT still leaves display resources disabled. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index e67d766bfdaa2..cbb8f414de36f 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2469,6 +2469,13 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8112 = { .num_dptx_ports = 2, }; +/* The internal T8140 endpoint uses the measured H17P method profile. */ +static const struct apple_dcp_hw_data apple_dcp_hw_t8140 = { + .num_dptx_ports = 0, + .adopt_live_session = true, + .firmware_compat = DCP_FIRMWARE_H17P, +}; + /* * M5 (T8142) runs H17-generation DCP firmware with the H17G method numbering * and is left running by the bootloader. @@ -2491,6 +2498,7 @@ static const struct apple_dcp_hw_data apple_dcp_hw_dcpext = { static const struct of_device_id of_match[] = { { .compatible = "apple,t6020-dcp", .data = &apple_dcp_hw_t6020, }, { .compatible = "apple,t8112-dcp", .data = &apple_dcp_hw_t8112, }, + { .compatible = "apple,t8140-dcp", .data = &apple_dcp_hw_t8140, }, { .compatible = "apple,t8142-dcp", .data = &apple_dcp_hw_t8142, }, { .compatible = "apple,dcp", .data = &apple_dcp_hw_dcp, }, { .compatible = "apple,dcpext", .data = &apple_dcp_hw_dcpext, }, From f93aa3bc0a1c7195c39f9e4e3dfcfbd7c82f22dd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:52:22 +1100 Subject: [PATCH 1084/2680] drm: apple: answer the H17P luma-range startup callback The internal H17P display requests an eight-byte luma-range reply during client open. Selector zero receives the binary32 encoding of 219.0 and a zero trailing word. Leaving this measured callback unbound stops startup; a zero range also cannot provide the firmware's required colour scale. Provide the recorded reply for selector zero without kernel floating point. Reject unqualified selectors, and preserve the H17G profile's existing zero-filled unknown-callback result, including short output envelopes. The actual dispatcher reproduces the captured reply and rejects selector mutations without acknowledging them. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index eed7625b4d3a7..2dab823031b51 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -353,6 +353,25 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, return false; } +static bool trampoline_luma_span_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + /* Preserve the older profile's already-cleared unknown-callback reply. */ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return true; + + trace_iomfb_callback(dcp, tag, __func__); + if (get_unaligned_le32(in)) { + dev_err(dcp->dev, "unqualified D574 range selector\n"); + WRITE_ONCE(dcp->crashed, true); + return false; + } + + /* Selector zero receives binary32 219.0 followed by a zero word. */ + put_unaligned_le64(0x435b0000, out); + return true; +} + static bool trampoline_get_uint_prop_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) { @@ -477,6 +496,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [565] = trampoline_true, [567] = trampoline_true, [572] = trampoline_zero, /* powerUpDART */ + [574] = trampoline_luma_span_h17p, [575] = trampoline_hotplug_h17p, [576] = trampoline_nop, [582] = iomfbep_cb_create_dfb_surface, /* create_default_fb_surface */ From 5d0bdfb892bf3c22dabbc4656f83d036920aac30 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:57:31 +1100 Subject: [PATCH 1085/2680] drm: apple: register the measured internal H17P default surface The internal H17P default-framebuffer registration uses an ARGB format word, an opaque layout word and reserved surface handle four. The existing template sends the older profile's different values. Comparing the entire registered descriptor against a Linux startup capture exposes the mismatch before the firmware receives it. Override those three fields for the measured H17P profile using explicit byte/unaligned encoding. Preserve the established H17G descriptor and leave presentation surface construction on its separate path. The full 560-byte registration request now reproduces the retained Linux input. Hardware registration and presentation remain separate checks. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_template.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index a31006a3e3c09..e917e210a9da1 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -338,6 +338,13 @@ static void dcp_fill_dfb_surface(struct apple_dcp *dcp, u8 *s) s[0x035] = 0x02; s[0x051] = 0x01; s[0x149] = 0x01; + + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + /* Internal H17P default registration differs from the older profile. */ + memcpy(s + 11, "ARGB", 4); + put_unaligned_le32(0x00e44000, s + 41); + s[53] = BIT(2); + } } /* From ce888330d20aa50ca3a9a9b2d4d3c30d78729e21 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 08:15:17 +1100 Subject: [PATCH 1086/2680] drm: apple: acknowledge bounded unknown H17P callbacks H17P waits for every firmware callback acknowledgment. Stopping on an unknown callback leaves the outer command outstanding and blocks all later presentation. Its generic unsupported-callback response is the zero-filled output area followed by the normal acknowledgment. Restore that bounded fallback for callbacks without a handler. Validate the header, tag range, stack depth, channel extent and complete record length before touching output. Typed handlers still require their known input/output sizes. Record unknown tags once per device, preserving the older firmware profiles' established dispatch behavior. Dispatcher fault injection covers malformed envelopes, output canaries, missing handlers and the complete measured startup callback set. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/iomfb.c | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 846cd4d459ca5..1ea630119eda1 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -366,6 +366,7 @@ struct apple_dcp { /* Staging wire record; serialization never mutates the atomic inputs. */ struct dcp_present_h17p present_h17p; struct dcp_present_state_h17p present_state_h17p; + DECLARE_BITMAP(unknown_callbacks, IOMFB_MAX_CB); /* Serializes H17P transmit preparation with RTKit receive callbacks. */ struct { diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 8f6bd148f862b..6a87a98fbe337 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -560,6 +560,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, int tag; struct dcp_channel *ch = dcp_get_channel(dcp, context); u8 depth; + bool handled; if (dcp->fw_compat == DCP_FIRMWARE_H17P && (length < sizeof(*hdr) || !ch || ch->depth >= DCP_MAX_CALL_DEPTH || @@ -570,9 +571,13 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, } tag = dcp_parse_tag(hdr->tag); + handled = tag >= 0 && tag < IOMFB_MAX_CB && dcp->cb_handlers && + dcp->cb_handlers[tag]; if (dcp->fw_compat == DCP_FIRMWARE_H17P && - ((u64)sizeof(*hdr) + hdr->in_len + hdr->out_len != length || + (tag < 0 || tag >= IOMFB_MAX_CB || + (u64)sizeof(*hdr) + hdr->in_len + hdr->out_len != length || (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + handled && !iomfb_validate_callback_h17p(tag, hdr->in_len, hdr->out_len)))) { dev_err(dev, "unqualified IOMFB callback %d (%u, %u)\n", tag, hdr->in_len, hdr->out_len); @@ -583,15 +588,11 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, in = data + sizeof(*hdr); out = in + hdr->in_len; - if (tag < 0 || tag >= IOMFB_MAX_CB || !dcp->cb_handlers || - !dcp->cb_handlers[tag]) { - dev_warn(dev, "received unknown callback %c%c%c%c\n", - hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); - if (iomfb_uses_queue(dcp)) { - /* A measured size does not establish a safe reply payload. */ - WRITE_ONCE(dcp->crashed, true); - return; - } + if (!handled) { + if (!iomfb_uses_queue(dcp) || + !test_and_set_bit(tag, dcp->unknown_callbacks)) + dev_warn(dev, "received unknown callback %c%c%c%c\n", + hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); /* * Leaving a callback unanswered wedges the coprocessor: it * waits for the ack forever and the outer call never returns. From 7408e46eb97ca1f9942a4664edec72cf4d94b4e5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 08:18:56 +1100 Subject: [PATCH 1087/2680] drm: apple: describe T8140 firmware clock aperture in the hardware profile The internal T8140 firmware exposes a sixth register aperture at 0x302800000 with size 0xbc000. Its bandwidth callback names the scratch at offset 0x20000 and request doorbell at offset 0x68000. Using the AP PMGR window from an older loader tree supplies different addresses and breaks the display firmware's clock setup. Describe this firmware aperture in the T8140 hardware profile. Append it only after the five required display windows, validate both offsets and use the same resource for the firmware register and bandwidth replies. This does not map, prime or write the clock window from the AP. Older profiles retain their DT-provided bandwidth resources and probe checks. The resource matches the independently retained disp0 ADT register and Linux register reply. Resource tests cover the two bandwidth addresses, missing windows, bad offsets and the older DT path. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 4 ++++ drivers/gpu/drm/apple/dcp.c | 28 +++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 1ea630119eda1..8dc07352b5093 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -167,6 +167,10 @@ struct apple_dcp_hw_data { */ bool adopt_live_session; enum dcp_firmware_version firmware_compat; + /* Firmware-visible clock aperture, independent of the AP PMGR window. */ + const struct resource *firmware_clock; + u32 firmware_scratch; + u32 firmware_request; }; /* TODO: move IOMFB members to its own struct */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index cbb8f414de36f..8841074cd9164 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1901,6 +1901,24 @@ static int dcp_get_disp_regs(struct apple_dcp *dcp) dcp->disp_registers[i] = res; } + if (dcp->hw.firmware_clock) { + /* The internal firmware's last aperture follows five display windows. */ + if (count != 5 || resource_size(dcp->hw.firmware_clock) < 4 || + dcp->hw.firmware_scratch > resource_size(dcp->hw.firmware_clock) - 4 || + dcp->hw.firmware_request > resource_size(dcp->hw.firmware_clock) - 4) + return -EINVAL; + + dcp->disp_bw_scratch_res = *dcp->hw.firmware_clock; + dcp->disp_bw_scratch_index = count; + dcp->disp_bw_scratch_offset = dcp->hw.firmware_scratch; + dcp->disp_bw_doorbell_res = dcp->disp_bw_scratch_res; + dcp->disp_bw_doorbell_res.start += dcp->hw.firmware_request; + dcp->disp_bw_doorbell_res.end = dcp->disp_bw_doorbell_res.start + 3; + dcp->disp_registers[count] = &dcp->disp_bw_scratch_res; + dcp->nr_disp_registers = count + 1; + return 0; + } + /* load pmgr bandwidth scratch resource and offset */ ret = dcp_get_bw_scratch_reg(dcp, count); if (ret < 0) @@ -2237,6 +2255,7 @@ static const struct component_ops dcp_comp_ops = { static int dcp_platform_probe(struct platform_device *pdev) { + const struct apple_dcp_hw_data *hw = of_device_get_match_data(&pdev->dev); enum dcp_firmware_version fw_compat; struct device *dev = &pdev->dev; struct apple_dcp *dcp; @@ -2252,7 +2271,8 @@ static int dcp_platform_probe(struct platform_device *pdev) * device trees. This prevents replacing simpledrm and ending up without * display. */ - if (!of_property_present(dev->of_node, "apple,bw-scratch")) + if (!of_property_present(dev->of_node, "apple,bw-scratch") && + !hw->firmware_clock) return dev_err_probe(dev, -ENODEV, "Incompatible devicetree! " "Use devicetree matching this kernel.\n"); @@ -2470,10 +2490,16 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8112 = { }; /* The internal T8140 endpoint uses the measured H17P method profile. */ +static const struct resource t8140_firmware_clock = + DEFINE_RES_MEM(0x302800000, 0xbc000); + static const struct apple_dcp_hw_data apple_dcp_hw_t8140 = { .num_dptx_ports = 0, .adopt_live_session = true, .firmware_compat = DCP_FIRMWARE_H17P, + .firmware_clock = &t8140_firmware_clock, + .firmware_scratch = 0x20000, + .firmware_request = 0x68000, }; /* From 1fcdce06379c66411c553827734c93b2620cda5d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:46:10 +1100 Subject: [PATCH 1088/2680] drm: apple: latch bounded H17P backlight takeover hints Linux powerlog observations contain millinits brightness hints before presentation. Later reports can differ from the first commanded level, so use the first valid panel-bounded hint and ignore subsequent reports once the policy is ready. Keep the existing panel maximum and 509-nit software clamp without inventing a fallback or calibration. Connect the public report parser to the queued backlight policy. Reject millinits overflow before narrowing, and keep registration and legacy brightness submission unavailable until the measured profile has a hint. Preserve the H17G and older firmware paths. Physical inherited-level continuity remains a hardware qualification gate. Add KUnit coverage for conversion bounds, zero hints and protection from late reports after control registration. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp_backlight.c | 24 ++++++++++++++++++ drivers/gpu/drm/apple/dcp_backlight.h | 12 +++++++++ drivers/gpu/drm/apple/dcp_backlight_test.c | 29 ++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 4 +-- drivers/gpu/drm/apple/parser.c | 2 ++ drivers/gpu/drm/apple/systemep.c | 13 +++++++--- 6 files changed, 79 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 298d4e4380a06..fbf2fef59dd8e 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -85,9 +85,26 @@ static u32 brightness_part2[] = { bool dcp_backlight_active(struct apple_dcp *dcp) { return dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && READ_ONCE(dcp->backlight.state.ready); } +int dcp_backlight_takeover(struct apple_dcp *dcp, u32 millinits) +{ + u32 maximum, nits; + int ret; + + if (dcp_backlight_active(dcp)) + return 0; + maximum = min(dcp->brightness.maximum, MAX_BRIGHTNESS_PART2 - 1); + ret = dcp_bl_takeover_nits(maximum, millinits, &nits); + if (ret) + return ret; + + /* Keep the first bounded hint. Later reports are not user requests. */ + return iomfb_configure_backlight_h17p(dcp, maximum, true, nits, false, 0); +} + static int dcp_get_brightness(struct backlight_device *bd) { struct apple_dcp *dcp = bl_get_data(bd); @@ -308,6 +325,9 @@ int dcp_backlight_update(struct apple_dcp *dcp) dcp_backlight_kick(dcp); return 0; } + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + return -ENODATA; /* * Do not actively try to change brightness if no mode is set. @@ -384,6 +404,10 @@ int dcp_backlight_register(struct apple_dcp *dcp) .scale = BACKLIGHT_SCALE_LINEAR, }; props.max_brightness = min(dcp->brightness.maximum, MAX_BRIGHTNESS_PART2 - 1); + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !dcp_backlight_active(dcp)) + return -ENODATA; if (dcp_backlight_active(dcp)) { spin_lock_irqsave(&dcp->backlight.lock, flags); if (!dcp->backlight.state.ready) { diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index 6d52a8f9eb7fe..e93ee7511c626 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -29,6 +29,17 @@ struct dcp_backlight_present { u32 nits; }; +/* Public powerlog reports use millinits; validate before rounding to nits. */ +static inline int dcp_bl_takeover_nits(u32 maximum, u32 millinits, u32 *nits) +{ + if (!maximum || maximum > INT_MAX) + return -EINVAL; + if ((u64)millinits > (u64)maximum * 1000) + return -ERANGE; + *nits = millinits / 1000; + return 0; +} + /* Inputs must come from admitted panel/loader records, never a guess. */ static inline int dcp_bl_init(struct dcp_backlight_state *state, u32 maximum, bool inherited_valid, u32 inherited, @@ -151,6 +162,7 @@ struct apple_dcp; * Configure before registration and keep the callback valid until teardown. */ bool dcp_backlight_active(struct apple_dcp *dcp); +int dcp_backlight_takeover(struct apple_dcp *dcp, u32 millinits); int dcp_backlight_configure(struct apple_dcp *dcp, u32 maximum, bool inherited_valid, u32 inherited, bool default_valid, u32 default_nits, diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index e990ea3e0bf06..a130f6f414712 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -6,6 +6,34 @@ #include "dcp_backlight.h" +static void backlight_millinits_takeover_test(struct kunit *test) +{ + struct dcp_backlight_state state; + u32 nits = 123; + + KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(0, 100000, &nits), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(U32_MAX, 100000, &nits), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(509, 509001, &nits), -ERANGE); + KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(509, U32_MAX, &nits), -ERANGE); + KUNIT_EXPECT_EQ(test, nits, 123U); + KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 100999, &nits), 0); + KUNIT_EXPECT_EQ(test, nits, 100U); + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 509, true, nits, false, 0), 0); + KUNIT_EXPECT_EQ(test, state.actual, 100U); + KUNIT_EXPECT_FALSE(test, state.dirty); + /* Registration freezes the hint before exposing the controls. */ + state.controlled = true; + KUNIT_EXPECT_FALSE(test, dcp_bl_seed(&state, 400)); + KUNIT_EXPECT_FALSE(test, dcp_bl_seed(&state, 0)); + KUNIT_EXPECT_EQ(test, state.target, 100U); + KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 0, &nits), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 509, true, nits, false, 0), 0); + KUNIT_EXPECT_EQ(test, state.target, 0U); + KUNIT_EXPECT_FALSE(test, state.dirty); + KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 509000, &nits), 0); + KUNIT_EXPECT_EQ(test, nits, 509U); +} + static void backlight_takeover_test(struct kunit *test) { struct dcp_backlight_state state = {}; @@ -187,6 +215,7 @@ static void backlight_inflight_dpms_test(struct kunit *test) } static struct kunit_case backlight_cases[] = { + KUNIT_CASE(backlight_millinits_takeover_test), KUNIT_CASE(backlight_takeover_test), KUNIT_CASE(backlight_range_test), KUNIT_CASE(backlight_dpms_cycles_test), diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index e917e210a9da1..a39032152d3ea 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -440,8 +440,8 @@ static void iomfb_cb_pr_publish(struct apple_dcp *dcp, struct iomfb_property *pr case IOMFB_PROPERTY_NITS_V14_7: { #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - /* Configured H17P takeover uses only the admitted sample interface. */ - if (dcp_backlight_active(dcp)) + /* Measured H17P takeover comes from the powerlog hint interface. */ + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) break; #endif if (dcp_has_panel(dcp)) { diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 6bd98e01a0791..415617d4875db 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -1199,6 +1199,8 @@ int parse_system_log_mnits(struct dcp_parse_ctx *handle, struct dcp_system_ev_mn if (!type_match || mnits < 0 || idac < 0 || timestamp < 0) return -EINVAL; + if (mnits > U32_MAX) + return -ERANGE; entry->millinits = mnits; entry->idac = idac; diff --git a/drivers/gpu/drm/apple/systemep.c b/drivers/gpu/drm/apple/systemep.c index 3133cd539b7f1..a32f0a0bcf6a0 100644 --- a/drivers/gpu/drm/apple/systemep.c +++ b/drivers/gpu/drm/apple/systemep.c @@ -75,9 +75,7 @@ static int powerlog_report(struct apple_epic_service *service, enum epic_subtype struct apple_dcp *dcp = service->ep->dcp; int ret; - /* H17P takeover decoding requires an admitted record. Later zero reports - * must not overwrite the cached target or completed level. - */ + /* Later reports must not overwrite a latched hint or Linux's target. */ if (dcp_backlight_active(dcp)) return 0; @@ -102,6 +100,15 @@ static int powerlog_report(struct apple_epic_service *service, enum epic_subtype dev_dbg(dcp->dev, "systemep: mNits event: Nits: %u.%03u, iDAC: %u\n", mnits.millinits / 1000, mnits.millinits % 1000, mnits.idac); + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + if (!dcp_has_panel(dcp)) + return 0; + ret = dcp_backlight_takeover(dcp, mnits.millinits); + if (ret && ret != -EBUSY) + dev_dbg(dcp->dev, "backlight takeover hint unavailable: %d\n", ret); + return 0; + } dcp->brightness.nits = mnits.millinits / 1000; From 37309ea6f90fbf499b903e58e071c8b904b91c66 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:53:14 +1100 Subject: [PATCH 1089/2680] drm: apple: encode H17P backlight activation and panel ceilings Linux brightness changes and soft DPMS presents use the same activation bytes and integer binary64 nits field. Encode the complete brightness block through the existing queued reservation, preserving the panel ceiling pair on every frame, including idle and pre-takeover frames. Refuse missing ceilings instead of submitting a zero calibration pair. An accepted matching completion updates the completed commanded level. Idle presents leave activation disabled; zero-nit enabled presents blank the retained scanout and unblank restores the user's target. Keep the wire ceiling separate from the existing exposed software brightness cap. Add a native actual-function fixture with captured packet fragments, full-range enabled/disabled replay, registration and reservation checks, rejection/coalescing and twenty software DPMS cycles. Document its mocked transport limits and the remaining integrated hardware qualification. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/gpu/apple-dcp-backlight.rst | 56 +++- drivers/gpu/drm/apple/iomfb.c | 14 +- drivers/gpu/drm/apple/iomfb_h17p.c | 31 ++- drivers/gpu/drm/apple/iomfb_h17p.h | 3 +- drivers/gpu/drm/apple/iomfb_template.c | 3 +- tools/testing/apple/backlight-golden.json | 297 ++++++++++++++++++++++ tools/testing/apple/check-backlight.py | 279 ++++++++++++++++++++ 7 files changed, 664 insertions(+), 19 deletions(-) create mode 100644 tools/testing/apple/backlight-golden.json create mode 100644 tools/testing/apple/check-backlight.py diff --git a/Documentation/gpu/apple-dcp-backlight.rst b/Documentation/gpu/apple-dcp-backlight.rst index e47e7f3b4dd4b..e486fcf4840df 100644 --- a/Documentation/gpu/apple-dcp-backlight.rst +++ b/Documentation/gpu/apple-dcp-backlight.rst @@ -5,20 +5,19 @@ Apple DCP H17P backlight integration The backlight policy implements the clean display specification's BACKLIGHT-1 through BACKLIGHT-4, with IOMFB-3 and LIFETIME-1 through LIFETIME-3 constraints. -It extends the existing Asahi backlight driver. It contains no H17P brightness -wire layout, calibration table, property command, or guessed initial level. -Policy activation requires explicit successful configuration. Existing -unconfigured H17P sessions, including T8142, keep their established DAC, -property/powerlog and DPMS paths. Older firmware remains on its existing path. -Neo startup must configure the policy before exposing the backlight or starting -its DPMS/brightness traffic; a firmware-version match alone does not activate -this policy. +It extends the existing Asahi backlight driver. The measured internal H17P profile carries brightness activation through the +central present queue. It latches the first bounded powerlog hint without +sending a brightness update and leaves registration unavailable if there is +no hint. H17G and older firmware retain their existing paths. Physical loader +level continuity remains a measurement gate. Admission and startup --------------------- -The central display worker must call ``dcp_backlight_configure()`` from process -context after linking the CRTC and admitting all required records. Its arguments +The measured profile connects the public powerlog parser to +``iomfb_configure_backlight_h17p()`` after linking the CRTC. Generic users of +``dcp_backlight_configure()`` must still supply validated inputs from process +context. Its arguments are the validated maximum commanded nits, an optional inherited level, an optional documented ADT default, and a callback that schedules the central outbound queue. Maximum must fit the backlight core's signed integer range. @@ -64,7 +63,8 @@ configuration or surface; ``-EBUSY`` means a prior brightness transaction is outstanding; ``-EALREADY`` means no brightness change is pending. These results do not clear pending work. The serializer places the returned level only in admitted fields of the next valid present. A normal present without a brightness -change keeps the inherited/completed level; its clean layout must support that. +change disables the measured activation fragment, preserving the existing +physical level without resending a level. The sequence is a software cookie, not a wire present ID. Associate the cookie with the central transaction and its strict wire ID. Call @@ -127,3 +127,37 @@ cached restore, both suspend/DPMS orders, static-surface deferral, busy sequenci rejection and stale completion. They do not qualify firmware fields or GNOME. Hardware acceptance requires O-BACKLIGHT, O-BACKLIGHT-TAKEOVER (10 Hz PBwo with no jump above 0.05 W), O-BACKLIGHT-CRC, O-DPMS (twenty cycles), and O-SUSPEND. + +Measured activation and software replay +-------------------------------------- + +Linux-boundary measurements on the default J700 driver establish an 18-byte +fragment at A408 offsets 0x354 through 0x365. Brightness-enabled presents use +``01 00 00 00 01 01 01 00 01 00`` followed by integer little-endian binary64 +nits. Ordinary presents disable the activation and nits fragment. Its individual control +bytes have no separately inferred semantics. The two binary64 ceiling fields +at 0x36e and 0x376 carry the raw panel maximum on every internal present, +including ordinary frames and frames before a takeover hint arrives. Enabled +updates additionally carry unity at 0x37e and 0x3e6; other scalars are zero. +The complete owned block ends at 0x3ed. The observed 525-nit ceiling is distinct +from the 509-nit exposed software range. Missing ceilings refuse serialization. +The older DAC/power words at 0x32f through 0x33b stay zero on this profile, +including before a takeover hint arrives. Enabled zero nits blanks the +panel while keeping the pipe and retained surface alive. Requests while +blanked change the restore target; unblank restores it through the same queue. + +The measurements include sysfs changes, two GNOME Mutter SetBacklight API +steps and one PowerSaveMode off/on cycle. The latter reaches near-zero panel +power while display-pipe power stays at 605 mW. Cached sysfs actual brightness +on the observed driver is not physical readback while blanked. These records +do not establish clean-driver firmware acceptance, physical takeover continuity, +static-screen latency, suspend/resume or a twenty-cycle hardware gate. + +The native fixture extracts the actual registration, takeover and queue +reservation functions and replays captured fragments under AddressSanitizer +and UndefinedBehaviorSanitizer:: + + python3 tools/testing/apple/check-backlight.py --output /path/to/scratch + +It mocks locks, scheduling, DRM registration and transport. The fragment corpus +retains hashes of clean Linux observations, with runtime addresses omitted. diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 6a87a98fbe337..319039320f297 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -992,9 +992,16 @@ bool iomfb_apply_backlight_h17p(struct apple_dcp *dcp, int ret; lockdep_assert_held(&dcp->iomfb.lock); - if (!dcp_backlight_active(dcp)) + if (!dcp_backlight_active(dcp)) { + if (!dcp_has_panel(dcp)) + return true; + if (!dcp->brightness.maximum) + return false; + iomfb_encode_backlight_h17p(wire, 0, + dcp->brightness.maximum, false); return true; - if (!transaction) + } + if (!transaction || !dcp->brightness.maximum) return false; for (i = 0; i < SWAP_SURFACES; i++) have_surface |= !request->surf_null[i]; @@ -1005,7 +1012,8 @@ bool iomfb_apply_backlight_h17p(struct apple_dcp *dcp, transaction->backlight = present; transaction->backlight_reserved = true; } - iomfb_encode_backlight_h17p(wire, present.nits); + iomfb_encode_backlight_h17p(wire, present.nits, + dcp->brightness.maximum, !ret); return true; } diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 2dab823031b51..54d7dd6c7c1d1 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -111,7 +111,7 @@ void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, } /* Integer commanded nits are exact binary64 values; no kernel FP is needed. */ -void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits) +static u64 iomfb_nits_binary64_h17p(u32 nits) { u64 value = 0; unsigned int exponent; @@ -121,8 +121,33 @@ void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits) value = (u64)(exponent + 1023) << 52; value |= ((u64)nits << (52 - exponent)) & GENMASK_ULL(51, 0); } - /* Measured from Linux presents during ordinary sysfs brightness changes. */ - put_unaligned_le64(value, wire->swap + 0x35e); + return value; +} + +void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits, + u32 maximum, bool update) +{ + u64 ceiling = iomfb_nits_binary64_h17p(maximum); + u64 unity = iomfb_nits_binary64_h17p(1); + + /* This profile does not use the older DAC/power words. */ + memset(wire->swap + 0x32f, 0, 0xd); + memset(wire->swap + 0x354, 0, 0x9a); + /* The ceiling pair must remain valid even on a brightness-idle frame. */ + put_unaligned_le64(ceiling, wire->swap + 0x36e); + put_unaligned_le64(ceiling, wire->swap + 0x376); + if (!update) + return; + + /* Linux brightness changes and soft DPMS use the same activation bytes. */ + put_unaligned_le32(1, wire->swap + 0x354); + wire->swap[0x358] = 1; + wire->swap[0x359] = 1; + wire->swap[0x35a] = 1; + put_unaligned_le16(1, wire->swap + 0x35c); + put_unaligned_le64(iomfb_nits_binary64_h17p(nits), wire->swap + 0x35e); + put_unaligned_le64(unity, wire->swap + 0x37e); + put_unaligned_le64(unity, wire->swap + 0x3e6); } static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 0121d334c214a..e4eb5be2b0143 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -67,7 +67,8 @@ static_assert(sizeof(struct dcp_present_h17p) == 0xe9c); static_assert(offsetof(struct dcp_present_h17p, surf) == 0x588); static_assert(offsetof(struct dcp_present_h17p, tail) == 0xe38); -void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits); +void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits, + u32 maximum, bool update); void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, const struct dcp_swap_submit_req_h17p *request); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index a39032152d3ea..e1bbef0984c1d 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -2212,7 +2212,8 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru /* update brightness if changed */ update_brightness = dcp_has_panel(dcp) && dcp->brightness.update; #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - update_brightness = update_brightness && !dcp_backlight_active(dcp); + update_brightness = update_brightness && + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G; #endif if (update_brightness) { #if DCP_FW_LAYOUT_14_7(DCP_FW_VER) diff --git a/tools/testing/apple/backlight-golden.json b/tools/testing/apple/backlight-golden.json new file mode 100644 index 0000000000000..d1a8af2ab2011 --- /dev/null +++ b/tools/testing/apple/backlight-golden.json @@ -0,0 +1,297 @@ +[ + { + "nits": 197, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000a06840000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "59bc5374f6e05d6bcf2839b9522bdb9456074f80c7d542c0ecdc7e90cf7c1d16" + }, + { + "nits": 188, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000806740000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "05c226553a34876237f2eef41246a75d013cd29115e68437df03f594f2459941" + }, + { + "nits": 179, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000606640000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "e74693443c8a029a060cf3612556f93837a82cc3f3936975dcd4848432186067" + }, + { + "nits": 170, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000406540000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "67521cee14ce162f0a733a815b1f059f3fe394c0c5e87bfe178f14b3acceb52c" + }, + { + "nits": 161, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000206440000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "f2996dbaff64ac3d653cd4b420df919c418697fcf8dc4f995f562e5839efcef2" + }, + { + "nits": 152, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000006340000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "b134853efd740adc11f258a206b4deb098df1765191c073178e033df83d17148" + }, + { + "nits": 143, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000e06140000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "1488c635d8d55ab49f73f9d9698a4b3e0f8fe95fcf925b73c21f4846fa7188f2" + }, + { + "nits": 134, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000c06040000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "132f2aba31bdc58731510361e6b934284ac81baeed5346dc235b615a43f6d798" + }, + { + "nits": 125, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000405f40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "a4f3e35516ba00acbed9c6c04ebb564334dab983b1a016f570dfe739ea2a9ae3" + }, + { + "nits": 116, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000005d40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "837951003a3632bb5c063e3ee70606583af2f843ef722864cbf461ef01fc5a81" + }, + { + "nits": 106, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000805a40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "a68f25f1ce7f63c335781888df47a8eb12f863c37795b2377d69c270dc063cac" + }, + { + "nits": 97, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000405840000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "6c5108c9aadd93c72b2f0d5b72adb09f55ca63159f909d0876714bbf62f80ea5" + }, + { + "nits": 88, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000005640000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "b2db893e953fda0c249ecf7c674acfd2018d1267985cc09d842b2f38910b2189" + }, + { + "nits": 79, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000c05340000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "321da86969fcfdb11f7d6e508a627bb0f9146defd48c4fa0d0cd76d7b63fd49b" + }, + { + "nits": 70, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000805140000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "66443759f999549d7d9b076c338cc9f5ba7b2606ec5f764f14a17f8a23f28acb" + }, + { + "nits": 61, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000804e40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "4c3447c9290bdda6fc9f38d024f8c8482fc7692b61243680cf6dc2012dd8d8a1" + }, + { + "nits": 52, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000004a40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "ce727d2eb98cbdeba8be5215577c2486ad8d5ad1b7ca1db2358d7c27f0cc2f1f" + }, + { + "nits": 43, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000804540000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "00f94f04f7e431494dddf154cf8e68a1e3fe91afa3ab18d41c52223d582112ae" + }, + { + "nits": 36, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000004240000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "4ae2c36b9d9a773bcfaf68c74b81e3761f4b0da68ffff4f7a55194207b313828" + }, + { + "nits": 59, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000804d40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "aa653a1c3d85ad9cf3a487aebdfafd0332c6d81335e9bc051292cab4610997de" + }, + { + "nits": 92, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000005740000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "a57f482514132d36aa5c46026b7c579bd74b1d21b84e5ed67e4123eab72e1054" + }, + { + "nits": 125, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000405f40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "6260e7be2d29193f51a0fe60337efcdc9c6e73b469f527afb9f0ff83765467ec" + }, + { + "nits": 140, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000806140000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "0eab7f3ebd304ddb5f40d2c8bc1ecd2110b166faf70f89b98a2521e0b6c7ac95" + }, + { + "nits": 200, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000006940000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "98df21464117081715a48b0076304aad0f2b8307ee682c20a7feafd9a1be9202" + }, + { + "nits": 287, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000f07140000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "7f5df7ed6f804a3340c9782698f5292e3b4117f2eb9943778010936a6b2746a4" + }, + { + "nits": 375, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000707740000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "994697bd5bb64697f4d2bbbf2311a7c383f356161184e2a9aeea610515dda25e" + }, + { + "nits": 400, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000007940000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "a9fbdac983180e10843468b14c2cade1ee7d0e8711e02fce56489112c91b38e0" + }, + { + "nits": 335, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000f07440000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "4cf3e7680e17a183e40e54b69d54693f43f3643346de050beb5c9cb73214b8f8" + }, + { + "nits": 249, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000206f40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "9b174a609f7fda9e379923c5e1fba03952b6f44d3ad031dfa8660a015326725e" + }, + { + "nits": 163, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000606440000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "bea9a384a1ceb723ddda8416748e20057e67627d768e9d1a81207d9f1a21d812" + }, + { + "nits": 131, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000606040000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "2445a2a32d0cbf27f6eb394bc48ff08c58ca88a32e6a6f9cbe633e983ab05256" + }, + { + "nits": 185, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000206740000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "1a23772d33bedf58b4c566ae84267a0631f7ced0d881ac9470667ec2eab477bc" + }, + { + "nits": 264, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000807040000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "3bd83da635cd09c608e2124c75e9aaa9ed9337622e0a818c500ddf83d8146d33" + }, + { + "nits": 337, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000107540000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "d44097a52f6cef9ca7c4ddf39897bf67a9915b520b154d50d5210595997f0495" + }, + { + "nits": 357, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000507640000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "51f19fe9d95e6d95b8b81b158a3d5c2db391973a66695858610ae24ade44c0d8" + }, + { + "nits": 0, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000000000000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "3f3d1f6ae52991efa63ac64d74e3e9f28e8b000791732102a69a5c05452d2512" + }, + { + "nits": 140, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000806140000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "5283a0beb8119ca257f5c00ae267598427c2bb01a472395c83b325f8c7a4b81a" + }, + { + "nits": 212, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000806a40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "d1bdce12296d366b8315c73d850fc80e92720748aff8ce3ff28a136e6104f0c8" + }, + { + "nits": 313, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000907340000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "a86f277bb5b34a2b13301d2c3464db76325d0bacd6009b5e46329fe97c82df60" + }, + { + "nits": 413, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000d07940000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "bca47e8efde3063740bd65f174ccc0b502b5ef2350cc821397679724ecf7d2d6" + }, + { + "nits": 454, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000607c40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "17c4a25be2d6cefb43cb282859f500415e3902be3d9d9ef5ab9045e6e9c6af53" + }, + { + "nits": 100, + "maximum": 525, + "enabled": false, + "fragment": "00000000000000000000000000000000000000000000000000000000000000688040000000000068804000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "payload_sha256": "7de62e12dc678b02f3b47b68ab369efc651cc05d765d2e024652583e410461da", + "provenance": "run24 clean ordinary startup presents" + } +] diff --git a/tools/testing/apple/check-backlight.py b/tools/testing/apple/check-backlight.py new file mode 100644 index 0000000000000..e1df3dc3458de --- /dev/null +++ b/tools/testing/apple/check-backlight.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Native replay of clean brightness fragments and actual policy/queue functions. + +Run with --output on a scratch volume. This mocks kernel locks, DRM registration +and queue transport; it does not claim firmware or KUnit boot acceptance. +""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import subprocess + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--output', required=True, type=Path) +parser.add_argument('--cc', default='clang') +args = parser.parse_args() +root = Path(__file__).resolve().parents[3] +src = root / 'drivers/gpu/drm/apple' +out = args.output.resolve() +out.mkdir(parents=True, exist_ok=True) + + +def function(file, name): + text = (src / file).read_text() + match = re.search(r'^\w[\w\s*]*?\b' + name + r'\(', text, re.M) + assert match, name + start = text.index('{', match.end()) + depth = 1 + pos = start + 1 + while depth: + depth += (text[pos] == '{') - (text[pos] == '}') + pos += 1 + return text[match.start():pos] + + +include = out / 'include/linux' +include.mkdir(parents=True, exist_ok=True) +(include / 'types.h').write_text('#include \n#include \ntypedef uint8_t u8;typedef uint32_t u32;typedef uint64_t u64;\n') +(include / 'limits.h').write_text('#include \n#define U32_MAX UINT32_MAX\n') +(include / 'errno.h').write_text('#include \n#ifndef ENODATA\n#define ENODATA 61\n#endif\n') +prelude = r''' +#include +#include +#include +#include +#include +#include +#include "dcp_backlight.h" +#define DCP_FIRMWARE_H17P 17 +#define DCP_IOMFB_METHODS_H17G 1 +#define MAX_BRIGHTNESS_PART2 510U +#define SWAP_SURFACES 4 +#define READ_ONCE(x) (x) +#define min(a,b) ((a)<(b)?(a):(b)) +#define lockdep_assert_held(x) ((void)(x)) +#define mutex_lock(x) ((void)(x)) +#define mutex_unlock(x) ((void)(x)) +#define spin_lock_irqsave(x,f) do { (void)(x);(f)=0; } while(0) +#define spin_unlock_irqrestore(x,f) do { (void)(x);(void)(f); } while(0) +#define GENMASK_ULL(h,l) ((~UINT64_C(0) >> (63-(h))) & (~UINT64_C(0) << (l))) +#define fls(x) (32-__builtin_clz(x)) +#define BACKLIGHT_PLATFORM 1 +#define BACKLIGHT_SCALE_LINEAR 2 +struct device { int unused; }; +struct backlight_device { int unused; }; +struct backlight_properties { int type;u32 brightness,max_brightness;int scale; }; +struct dcp_present_h17p { u8 swap[0x588];u8 guard[32]; }; +struct dcp_swap_submit_req_h17p { u8 surf_null[SWAP_SURFACES]; }; +struct iomfb_transaction { + struct dcp_backlight_present backlight; + bool backlight_reserved,brightness_only,completed; +}; +struct apple_dcp { + int fw_compat;struct { int iomfb_method_profile; } hw; + bool panel;void *crtc;struct device *dev; + int bl_register_mutex,bl_register_wq; + struct { u32 maximum;u32 nits;struct backlight_device *bl_dev;u32 dac; } brightness; + struct { int lock;struct dcp_backlight_state state;void (*kick)(struct apple_dcp *); } backlight; + struct { int lock;struct iomfb_transaction *active; } iomfb; +}; +static int schedule_count,register_count,retire_count,dac_count; +static struct backlight_properties registered; +static const int dcp_backlight_ops; +static bool dcp_has_panel(struct apple_dcp *dcp) { return dcp->panel; } +static void schedule_work(int *work) { (void)work;schedule_count++; } +static void iomfb_backlight_kick(struct apple_dcp *dcp) { (void)dcp; } +static void iomfb_scanout_complete_h17p(struct apple_dcp *dcp) { (void)dcp;retire_count++; } +static u32 calculate_dac(struct apple_dcp *dcp,u32 nits) { (void)dcp;dac_count++;return nits; } +static struct backlight_device *devm_backlight_device_register(struct device *dev,const char *name, + struct device *parent,struct apple_dcp *dcp,const int *ops,const struct backlight_properties *props) { + static struct backlight_device device; + (void)dev;(void)parent;(void)dcp;(void)ops; + assert(!strcmp(name,"apple-panel-bl"));registered=*props;register_count++;return &device; +} +#define IS_ERR(p) (!(p)) +#define PTR_ERR(p) ((void)(p),-ENOMEM) +static void put_unaligned_le64(u64 v,void *p) { for(int i=0;i<8;i++)((u8 *)p)[i]=v>>(8*i); } +static void put_unaligned_le32(u32 v,void *p) { for(int i=0;i<4;i++)((u8 *)p)[i]=v>>(8*i); } +static void put_unaligned_le16(uint16_t v,void *p) { for(int i=0;i<2;i++)((u8 *)p)[i]=v>>(8*i); } +int iomfb_configure_backlight_h17p(struct apple_dcp *,u32,bool,u32,bool,u32); +''' +functions = [ + ('iomfb_h17p.c', 'iomfb_nits_binary64_h17p'), + ('iomfb_h17p.c', 'iomfb_encode_backlight_h17p'), + ('iomfb.c', 'iomfb_uses_queue'), + ('dcp_backlight.c', 'dcp_backlight_active'), + ('dcp_backlight.c', 'dcp_backlight_configure'), + ('iomfb.c', 'iomfb_configure_backlight_h17p'), + ('dcp_backlight.c', 'dcp_backlight_takeover'), + ('dcp_backlight.c', 'dcp_backlight_register'), + ('dcp_backlight.c', 'dcp_backlight_prepare'), + ('dcp_backlight.c', 'dcp_backlight_complete'), + ('iomfb.c', 'iomfb_apply_backlight_h17p'), + ('iomfb.c', 'iomfb_present_failed_h17p'), + ('iomfb.c', 'iomfb_present_complete_h17p'), +] +main = r''' +static struct apple_dcp fixture(void) { + return (struct apple_dcp){.fw_compat=DCP_FIRMWARE_H17P,.panel=true, + .crtc=(void *)1,.brightness.maximum=525}; +} +static u64 level_bits(const struct dcp_present_h17p *wire) { + u64 value=0;for(int i=0;i<8;i++)value|=(u64)wire->swap[0x35e + i]<<(i*8);return value; +} +static double level(const struct dcp_present_h17p *wire) { + u64 bits=level_bits(wire);double value;memcpy(&value,&bits,sizeof(value));return value; +} +static void outside_unchanged(const struct dcp_present_h17p *wire) { + const u8 *data=(const u8 *)wire; + for(size_t i=0;i=0x32f && i<0x33c)assert(!data[i]); + else if(i<0x354||i>=0x3ee)assert(data[i]==0xa5); + } +} +int main(int argc,char **argv) { + struct dcp_present_h17p wire; + if(argc==3) { + memset(&wire,0xa5,sizeof(wire)); + iomfb_encode_backlight_h17p(&wire,strtoul(argv[1],NULL,0),525,atoi(argv[2])); + outside_unchanged(&wire); + for(int i=0x354;i<0x3ee;i++)printf("%02x",wire.swap[i]);puts("");return 0; + } + if(argc==2) { + for(u32 nits=0;nits<510;nits++)for(int update=0;update<2;update++) { + memset(&wire,0xa5,sizeof(wire)); + iomfb_encode_backlight_h17p(&wire,nits,525,update); + outside_unchanged(&wire); + printf("%u %d ",nits,update); + for(int i=0x354;i<0x3ee;i++)printf("%02x",wire.swap[i]);puts(""); + } + return 0; + } + struct apple_dcp dcp=fixture(); + assert(dcp_backlight_register(&dcp)==-ENODATA); + assert(!register_count); + assert(dcp_backlight_takeover(&dcp,509001)==-ERANGE); + assert(!dcp_backlight_active(&dcp)); + assert(dcp_backlight_takeover(&dcp,100000)==0); + assert(dcp.backlight.state.target==100 && dcp.backlight.state.actual==100); + assert(!dcp.backlight.state.dirty && schedule_count==1); + assert(dcp_backlight_takeover(&dcp,400000)==0); + assert(dcp_backlight_takeover(&dcp,0)==0); + assert(dcp.backlight.state.target==100 && schedule_count==1); + assert(dcp_backlight_register(&dcp)==0 && register_count==1); + assert(registered.brightness==100 && registered.max_brightness==509 && !dac_count); + assert(dcp.backlight.state.controlled); + struct iomfb_transaction transaction={.brightness_only=true}; + struct dcp_swap_submit_req_h17p request={.surf_null={0,1,1,1}}; + dcp.iomfb.active=&transaction; + struct apple_dcp unready=fixture(); + memset(&wire,0xa5,sizeof(wire)); + assert(iomfb_apply_backlight_h17p(&unready,&request,&wire)); + assert(!wire.swap[0x354] && !level_bits(&wire)); + assert(wire.swap[0x374] || wire.swap[0x375]); + unready.brightness.maximum=0; + memset(&wire,0xa5,sizeof(wire)); + assert(!iomfb_apply_backlight_h17p(&unready,&request,&wire)); + assert(wire.swap[0x354]==0xa5); + dcp.brightness.maximum=0; + assert(!iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + dcp.brightness.maximum=525; + memset(&wire,0xa5,sizeof(wire)); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + for(int i=0x354;i<0x366;i++)assert(!wire.swap[i]); + assert(!transaction.backlight_reserved); + assert(dcp_bl_request(&dcp.backlight.state,400,false,false)==0); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(level(&wire)==400 && dcp.backlight.state.actual==100); + assert(transaction.backlight_reserved); + assert(!dcp_backlight_complete(&dcp,transaction.backlight.sequence+1,true)); + assert(dcp_bl_request(&dcp.backlight.state,140,false,false)==0); + assert(!iomfb_present_complete_h17p(&dcp)); + assert(dcp.backlight.state.actual==400 && dcp.backlight.state.dirty && !retire_count); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(level(&wire)==140); + iomfb_present_failed_h17p(&dcp); + assert(dcp.backlight.state.actual==400 && dcp.backlight.state.dirty); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(!iomfb_present_complete_h17p(&dcp)); + assert(dcp.backlight.state.actual==140 && !retire_count); + for(int i=0;i<20;i++) { + assert(dcp_bl_dpms(&dcp.backlight.state,false)==0); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(level(&wire)==0 && wire.swap[0x354]==1); + assert(!iomfb_present_complete_h17p(&dcp)); + assert(dcp.backlight.state.actual==0 && dcp.backlight.state.target==140); + assert(dcp_bl_dpms(&dcp.backlight.state,true)==0); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(level(&wire)==140); + assert(!iomfb_present_complete_h17p(&dcp)); + } + assert(!retire_count); + assert(dcp_bl_dpms(&dcp.backlight.state,false)==0); + assert(dcp_bl_request(&dcp.backlight.state,400,false,false)==0); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(level(&wire)==0 && dcp.backlight.state.target==400); + assert(!iomfb_present_complete_h17p(&dcp)); + assert(dcp_bl_dpms(&dcp.backlight.state,true)==0); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(level(&wire)==400); + assert(!iomfb_present_complete_h17p(&dcp)); + assert(dcp_bl_request(&dcp.backlight.state,36,false,false)==0); + memset(request.surf_null,1,sizeof(request.surf_null)); + memset(&wire,0xa5,sizeof(wire)); + assert(!iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(wire.swap[0x354]==0xa5 && dcp.backlight.state.dirty); + struct apple_dcp h17g=fixture();h17g.hw.iomfb_method_profile=DCP_IOMFB_METHODS_H17G; + assert(dcp_backlight_takeover(&h17g,100000)==-EINVAL); + assert(!h17g.backlight.state.ready); + struct apple_dcp external=fixture();external.panel=false; + assert(dcp_backlight_takeover(&external,100000)==-EINVAL); + struct apple_dcp unlinked=fixture();unlinked.crtc=NULL; + assert(dcp_backlight_takeover(&unlinked,100000)==-EINVAL); + struct apple_dcp zero=fixture(); + assert(dcp_backlight_takeover(&zero,0)==0); + assert(zero.backlight.state.target==0 && !zero.backlight.state.dirty); + puts("Actual takeover/registration/reservation/accepted completion/rejection/20 DPMS cycles passed"); + return 0; +} +''' +source = prelude + '\n'.join(function(f, n) for f, n in functions) + main +(out / 'fixture.c').write_text(source) +argv = [args.cc, '-std=gnu11', '-Wall', '-Wextra', '-Werror', '-fsanitize=address,undefined', + '-I' + str(out / 'include'), '-I' + str(src), str(out / 'fixture.c'), '-o', str(out / 'fixture')] +build = subprocess.run(argv, text=True, capture_output=True) +(out / 'compile.log').write_text(build.stdout + build.stderr) +assert not build.returncode, build.stderr +run = subprocess.run([str(out / 'fixture')], text=True, capture_output=True) +(out / 'run.log').write_text(run.stdout + run.stderr) +assert not run.returncode, run.stdout + run.stderr +# All measured brightness fragments, including DPMS zero and ordinary startup. +golden = json.loads(Path(__file__).with_name('backlight-golden.json').read_text()) +for case in golden: + fragment = subprocess.check_output([str(out / 'fixture'), str(case['nits']), str(int(case['enabled']))], text=True).strip() + assert fragment == case['fragment'], (case, fragment) +# Exact integer binary64 throughout the exposed range, plus disabled updates. +import struct +range_output = subprocess.check_output([str(out / 'fixture'), 'range'], text=True) +for line in range_output.splitlines(): + n, enabled, encoded = line.split() + nits, enabled = int(n), bool(int(enabled)) + expected = bytearray(0x9a) + expected[0x1a:0x22] = expected[0x22:0x2a] = struct.pack(' Date: Thu, 8 Oct 2026 08:34:48 +1100 Subject: [PATCH 1090/2680] drm: apple: enable the internal H17P pipe once before soft DPMS Internal H17P startup requires one out-of-band power-on request after client open. The soft-DPMS shortcut currently bypasses that first enable, although it is appropriate for later brightness-only transitions. Latch the matching successful enable reply and perform the initial enable before entering soft DPMS. Later unblank operations restore brightness without another power or display-device request. A missing or malformed enable reply stops presentation instead of treating the pipe as ready. Keep the measured H17P pipe live through off, sleep and shutdown entry points: surface-free presents and firmware power-down commands can cause null-base DMA faults or firmware NMI. Stop the software queue on shutdown and retain scanout ownership without claiming a final hardware stop. The older firmware profiles keep their established lifecycle paths. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/dcp.c | 10 ++++++++++ drivers/gpu/drm/apple/iomfb_h17p.c | 1 + drivers/gpu/drm/apple/iomfb_template.c | 24 ++++++++++++++++++++++++ 4 files changed, 37 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 8dc07352b5093..8b39935aa9d62 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -268,6 +268,8 @@ struct apple_dcp { /* Is the DCP booted? */ bool active; + /* A successful initial enable keeps the H17P pipe running across DPMS. */ + bool pipe_enabled_h17p; /* eDP display without DP-HDMI conversion */ bool main_display; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8841074cd9164..66a982538b813 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1503,6 +1503,16 @@ void dcp_poweron(struct platform_device *pdev) int ret; if (dcp_uses_soft_dpms(dcp)) { + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !READ_ONCE(dcp->pipe_enabled_h17p)) { + iomfb_poweron_h17p(dcp); + if (!READ_ONCE(dcp->pipe_enabled_h17p)) { + dev_err(dcp->dev, "initial display pipe enable failed\n"); + WRITE_ONCE(dcp->crashed, true); + return; + } + } ret = dcp_backlight_dpms(dcp, true); if (ret) dev_warn(dcp->dev, "backlight restore unavailable: %d\n", ret); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 54d7dd6c7c1d1..1e5aba452536f 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -540,6 +540,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { void iomfb_start_h17p(struct apple_dcp *dcp) { + WRITE_ONCE(dcp->pipe_enabled_h17p, false); dcp->cb_handlers = cb_handlers; dcp_start_signal(dcp, false, dcp_started, NULL); } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index e1bbef0984c1d..4a610000490b0 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1425,6 +1425,12 @@ static void dcp_on_final(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_wait_cookie *wait = cookie; +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + out && get_unaligned_le64(out) == 1) + WRITE_ONCE(dcp->pipe_enabled_h17p, true); +#endif + if (wait) { complete(&wait->done); kref_put(&wait->refcount, release_wait_cookie); @@ -1570,6 +1576,12 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) struct DCP_FW_NAME(dcp_swap_start_req) swap_req = { 0 }; typeof(DCP_FW_UNION(dcp->swap)) *swap = &DCP_FW_UNION(dcp->swap); +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + /* The measured profile has no safe pipe-off or surface-free transition. */ + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + return; +#endif + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) return; @@ -1662,6 +1674,11 @@ void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp) struct dcp_wait_cookie *cookie; +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + return; +#endif + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) return; @@ -2437,6 +2454,13 @@ void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp) struct dcp_wait_cookie *cookie; int ret; +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + iomfb_queue_stop(dcp); + return; + } +#endif + cookie = kzalloc_obj(*cookie); if (!cookie) return; From 0ecda622fcd887ddc0c318099826e31f27e25bd2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:55:15 +1100 Subject: [PATCH 1091/2680] drm: apple: use ARGB for H17P fbdev H17P applies the stored fourth byte as straight alpha even when the framebuffer uses an X format. A depth-24 fbdev console therefore produces transparent pixels and appears black. Request depth 32 from the DMA fbdev helper only for the H17P internal-panel profile. Older firmware retains its existing XRGB console. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 2244568705e76..e56d6f7503a43 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -65,9 +65,30 @@ static int apple_drm_gem_dumb_create(struct drm_file *file_priv, return drm_gem_dma_dumb_create_internal(file_priv, drm, args); } +static int apple_drm_fbdev_probe(struct drm_fb_helper *helper, + struct drm_fb_helper_surface_size *sizes) +{ + struct drm_crtc *crtc; + + drm_for_each_crtc(crtc, helper->dev) { + struct apple_dcp *dcp; + + dcp = platform_get_drvdata(to_apple_crtc(crtc)->dcp); + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + /* H17P applies the stored fourth byte as straight alpha. */ + sizes->surface_bpp = 32; + sizes->surface_depth = 32; + break; + } + } + + return drm_fbdev_dma_driver_fbdev_probe(helper, sizes); +} + static const struct drm_driver apple_drm_driver = { DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(apple_drm_gem_dumb_create), - DRM_FBDEV_DMA_DRIVER_OPS, + .fbdev_probe = apple_drm_fbdev_probe, .name = DRIVER_NAME, .desc = DRIVER_DESC, .major = 1, From d9620fba93e2b91db5db769ac98d38c0c64613d5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:56:15 +1100 Subject: [PATCH 1092/2680] drm: apple: configure opaque X formats on H17P H17P blends the stored X byte as alpha. Linux boundary measurements show that property 0x49 set to zero and the existing premultiplied surface marker make 8-bit XRGB scanout opaque without changing ARGB alpha. Describe the measured A352 request and send it as a priority item on the serialized H17P queue after the first completed present of each full power-on. Keep unqualified 10-bit XRGB off the H17P primary plane, and cover the admitted surface offsets, IOVA tail and property bytes with KUnit fixtures. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/apple_drv.c | 9 ++- drivers/gpu/drm/apple/dcp-internal.h | 4 ++ drivers/gpu/drm/apple/iomfb.c | 84 +++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_h17p.c | 13 ++++ drivers/gpu/drm/apple/iomfb_h17p.h | 15 +++++ drivers/gpu/drm/apple/iomfb_h17p_test.c | 84 +++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 5 ++ drivers/gpu/drm/apple/plane.c | 23 ++++++- drivers/gpu/drm/apple/plane.h | 1 + tools/testing/apple/check-backlight.py | 13 +++- 11 files changed, 249 insertions(+), 3 deletions(-) create mode 100644 drivers/gpu/drm/apple/iomfb_h17p_test.c diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 786e648244af0..44ead545d61c5 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -12,6 +12,7 @@ appledrm-y += iomfb_v12_3.o appledrm-y += iomfb_v13_3.o appledrm-y += iomfb_v14_7.o appledrm-y += iomfb_h17p.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += iomfb_h17p_test.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index e56d6f7503a43..c950ccd53d9f3 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -406,9 +406,14 @@ static int apple_probe_per_dcp(struct device *dev, u32 surf; int zpos = 0; bool supports_l10r = !dcp_fw_compat_is_12_x(dcp); + struct apple_dcp *dcp_data = platform_get_drvdata(dcp); + bool supports_xrgb2101010; enum drm_plane_type plane_type; bool interchange; + supports_xrgb2101010 = dcp_data->fw_compat != DCP_FIRMWARE_H17P || + dcp_data->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G; + /* * BUG: * Some DCPs support more than two surfaces. For some reason, these @@ -422,7 +427,9 @@ static int apple_probe_per_dcp(struct device *dev, plane_type = (zpos == 0) ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY; interchange = (zpos <= 1 || surf != top_surf); - planes[zpos] = apple_plane_init(drm, 1U << num, surf, supports_l10r, + planes[zpos] = apple_plane_init(drm, 1U << num, surf, + supports_l10r, + supports_xrgb2101010, plane_type, interchange); if (IS_ERR(planes[zpos])) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 8b39935aa9d62..299c909c70acb 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -387,6 +387,7 @@ struct apple_dcp { unsigned int queued; bool stopped; bool backlight_queued; + atomic_t opaque_x_state; struct iomfb_scanout_h17p *scanout; struct iomfb_scanout_h17p *next_scanout; } iomfb; @@ -400,6 +401,9 @@ void iomfb_present_failed_h17p(struct apple_dcp *dcp); bool iomfb_apply_backlight_h17p(struct apple_dcp *dcp, const struct dcp_swap_submit_req_h17p *request, struct dcp_present_h17p *wire); +void iomfb_apply_opaque_x_h17p(struct apple_dcp *dcp, dcp_callback_t callback, + void *cookie); +void iomfb_opaque_x_reset_h17p(struct apple_dcp *dcp); int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum, bool inherited_valid, u32 inherited, bool default_valid, u32 default_nits); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 319039320f297..ec628f6b67fac 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -171,6 +171,46 @@ static bool iomfb_channels_idle(struct apple_dcp *dcp) !dcp->ch_async.depth && !dcp->ch_oobasync.depth; } +enum iomfb_opaque_x_state { + IOMFB_OPAQUE_X_WAITING, + IOMFB_OPAQUE_X_NEEDED, + IOMFB_OPAQUE_X_QUEUED, + IOMFB_OPAQUE_X_READY, +}; + +struct iomfb_opaque_x_transaction { + struct iomfb_transaction transaction; +}; + +static void iomfb_opaque_x_start(struct apple_dcp *dcp, + struct iomfb_transaction *transaction); +static void iomfb_opaque_x_release(struct iomfb_transaction *transaction); + +static int iomfb_enqueue_opaque_x(struct apple_dcp *dcp) +{ + struct iomfb_opaque_x_transaction *opaque; + + lockdep_assert_held(&dcp->iomfb.lock); + if (atomic_read(&dcp->iomfb.opaque_x_state) != IOMFB_OPAQUE_X_NEEDED) + return 0; + if (dcp->iomfb.queued >= 32) + return -EBUSY; + opaque = kzalloc_obj(*opaque); + if (!opaque) + return -ENOMEM; + if (atomic_cmpxchg(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_NEEDED, + IOMFB_OPAQUE_X_QUEUED) != IOMFB_OPAQUE_X_NEEDED) { + kfree(opaque); + return 0; + } + + opaque->transaction.start = iomfb_opaque_x_start; + opaque->transaction.release = iomfb_opaque_x_release; + dcp->iomfb.queued++; + list_add(&opaque->transaction.link, &dcp->iomfb.pending); + return 0; +} + static void iomfb_discard_pending(struct apple_dcp *dcp) { struct iomfb_transaction *transaction, *next; @@ -208,6 +248,12 @@ static void iomfb_queue_advance(struct apple_dcp *dcp) schedule_work(&dcp->bl_update_wq); transaction->release(transaction); } + if (iomfb_enqueue_opaque_x(dcp)) { + WRITE_ONCE(dcp->crashed, true); + iomfb_discard_pending(dcp); + schedule_work(&dcp->vblank_wq); + return; + } if (!list_empty(&dcp->iomfb.pending)) schedule_work(&dcp->iomfb.work); } @@ -259,12 +305,48 @@ static void iomfb_queue_timeout(struct work_struct *work) mutex_unlock(&dcp->iomfb.lock); } +static void iomfb_opaque_x_complete(struct apple_dcp *dcp, void *out, + void *cookie) +{ + u32 status = out ? *(u32 *)out : ~0U; + + (void)cookie; + + if (status) { + dev_err(dcp->dev, "opaque X property failed: %u\n", status); + WRITE_ONCE(dcp->crashed, true); + schedule_work(&dcp->vblank_wq); + return; + } + + atomic_set(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_READY); +} + +static void iomfb_opaque_x_start(struct apple_dcp *dcp, + struct iomfb_transaction *transaction) +{ + (void)transaction; + iomfb_apply_opaque_x_h17p(dcp, iomfb_opaque_x_complete, NULL); +} + +static void iomfb_opaque_x_release(struct iomfb_transaction *transaction) +{ + kfree(container_of(transaction, struct iomfb_opaque_x_transaction, + transaction)); +} + +void iomfb_opaque_x_reset_h17p(struct apple_dcp *dcp) +{ + atomic_set(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING); +} + void iomfb_queue_init(struct apple_dcp *dcp) { mutex_init(&dcp->iomfb.lock); INIT_LIST_HEAD(&dcp->iomfb.pending); INIT_WORK(&dcp->iomfb.work, iomfb_queue_work); INIT_DELAYED_WORK(&dcp->iomfb.timeout, iomfb_queue_timeout); + atomic_set(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING); } void iomfb_queue_stop(struct apple_dcp *dcp) @@ -1041,6 +1123,8 @@ bool iomfb_present_complete_h17p(struct apple_dcp *dcp) if (transaction->brightness_only) return false; } + atomic_cmpxchg(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING, + IOMFB_OPAQUE_X_NEEDED); iomfb_scanout_complete_h17p(dcp); return true; } diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 1e5aba452536f..c2617ee0b36e0 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -211,6 +211,19 @@ static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { #include "iomfb_template.c" +void iomfb_apply_opaque_x_h17p(struct apple_dcp *dcp, dcp_callback_t callback, + void *cookie) +{ + static const struct dcp_method_entry method = { + .name = "apply_property", + .tag = { 'A', '3', '5', '2' }, + }; + struct dcp_apply_property_h17p property = dcp_opaque_x_property_h17p(); + + dcp_push(dcp, false, &method, sizeof(property), sizeof(u32), &property, + callback, cookie); +} + static bool trampoline_rt_bandwidth_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) { diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index e4eb5be2b0143..9473940250297 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -67,6 +67,21 @@ static_assert(sizeof(struct dcp_present_h17p) == 0xe9c); static_assert(offsetof(struct dcp_present_h17p, surf) == 0x588); static_assert(offsetof(struct dcp_present_h17p, tail) == 0xe38); +struct dcp_apply_property_h17p { + u32 property; + u32 value; +} __packed; + +static_assert(sizeof(struct dcp_apply_property_h17p) == 0x8); + +static inline struct dcp_apply_property_h17p dcp_opaque_x_property_h17p(void) +{ + return (struct dcp_apply_property_h17p) { + .property = 0x49, + .value = 0, + }; +} + void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits, u32 maximum, bool update); diff --git a/drivers/gpu/drm/apple/iomfb_h17p_test.c b/drivers/gpu/drm/apple/iomfb_h17p_test.c new file mode 100644 index 0000000000000..c2a71ffa8cdbf --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_h17p_test.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include +#include + +#include "iomfb_h17p.h" + +static void h17p_surface_wire_test(struct kunit *test) +{ + struct dcp_swap_submit_req_h17p *request; + struct dcp_present_h17p *wire; + struct dcp_surface *surface; + u64 iova = 0x123456789aULL; + + request = kunit_kzalloc(test, sizeof(*request), GFP_KERNEL); + wire = kunit_kmalloc(test, sizeof(*wire), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, request); + KUNIT_ASSERT_NOT_NULL(test, wire); + surface = &request->surf[0].base; + memset(wire, 0xa5, sizeof(*wire)); + surface->is_premultiplied = 1; + surface->plane_cnt = 1; + surface->plane_cnt2 = 1; + surface->format = DCP_FORMAT_BGRA; + surface->xfer_func = DCP_XFER_FUNC_SDR; + surface->colorspace = DCP_COLORSPACE_NATIVE; + surface->stride = 10176; + surface->width = 2408; + surface->height = 1506; + surface->buf_size = 10176 * 1506; + surface->planes[0].width = 2408; + surface->planes[0].height = 1506; + surface->planes[0].stride = 10176; + surface->planes[0].size = 10176 * 1506; + request->surf_iova[0] = iova; + request->surf_null[0] = false; + request->surf_null[1] = true; + request->surf_null[2] = true; + request->surf_null[3] = true; + + iomfb_serialize_present_h17p(wire, request); + + KUNIT_EXPECT_MEMEQ(test, wire->swap, &request->swap, sizeof(wire->swap)); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x593), + DCP_FORMAT_BGRA); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x59d), 10176U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5a9), 2408U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5ad), 1506U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5b1), + 10176U * 1506U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5e1), 2408U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5e5), 1506U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5f1), 10176U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5f5), + 10176U * 1506U); + KUNIT_EXPECT_EQ(test, get_unaligned_le64((u8 *)wire + 0xe38), iova); + KUNIT_EXPECT_EQ(test, wire->tail[sizeof(wire->tail) - 2], 0U); + KUNIT_EXPECT_EQ(test, wire->tail[sizeof(wire->tail) - 1], 0U); +} + +static void h17p_opaque_x_parameter_test(struct kunit *test) +{ + struct dcp_apply_property_h17p property = dcp_opaque_x_property_h17p(); + + KUNIT_EXPECT_EQ(test, property.property, 0x49U); + KUNIT_EXPECT_EQ(test, property.value, 0U); +} + +static struct kunit_case h17p_present_cases[] = { + KUNIT_CASE(h17p_surface_wire_test), + KUNIT_CASE(h17p_opaque_x_parameter_test), + {} +}; + +static struct kunit_suite h17p_present_suite = { + .name = "apple-dcp-h17p-present", + .test_cases = h17p_present_cases, +}; + +kunit_test_suite(h17p_present_suite); + +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 4a610000490b0..c86d8d848365e 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1468,6 +1468,11 @@ void DCP_FW_NAME(iomfb_poweron)(struct apple_dcp *dcp) u32 handle; dev_info(dcp->dev, "dcp_poweron() starting\n"); +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + iomfb_opaque_x_reset_h17p(dcp); +#endif + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) return; diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index d068cf4c07298..d5a27bc05f268 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -561,6 +561,23 @@ static const u32 dcp_primary_formats[] = { #endif }; +/* H17P 10-bit XRGB alpha semantics have not been qualified. */ +static const u32 dcp_primary_formats_h17p[] = { + DRM_FORMAT_ARGB2101010, + DRM_FORMAT_XRGB8888, + DRM_FORMAT_ARGB8888, + DRM_FORMAT_XBGR8888, + DRM_FORMAT_ABGR8888, + DRM_FORMAT_NV12, + DRM_FORMAT_NV16, + DRM_FORMAT_NV24, + DRM_FORMAT_P010, + DRM_FORMAT_P210, +#if defined(DRM_FORMAT_P410) + DRM_FORMAT_P410, +#endif +}; + static const u32 dcp_overlay_formats[] = { DRM_FORMAT_ABGR16161616F, DRM_FORMAT_ARGB2101010, @@ -627,6 +644,7 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, unsigned long possible_crtcs, u32 iomfb_surf, bool supports_l10r, + bool supports_xrgb2101010, enum drm_plane_type type, bool interchange) { @@ -636,9 +654,12 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, switch (type) { case DRM_PLANE_TYPE_PRIMARY: - if (supports_l10r) { + if (supports_l10r && supports_xrgb2101010) { fmts = dcp_primary_formats; num_fmts = ARRAY_SIZE(dcp_primary_formats); + } else if (supports_l10r) { + fmts = dcp_primary_formats_h17p; + num_fmts = ARRAY_SIZE(dcp_primary_formats_h17p); } else { fmts = dcp_primary_formats_12_x; num_fmts = ARRAY_SIZE(dcp_primary_formats_12_x); diff --git a/drivers/gpu/drm/apple/plane.h b/drivers/gpu/drm/apple/plane.h index 5d6b25912b8ce..d07cb52e12c9c 100644 --- a/drivers/gpu/drm/apple/plane.h +++ b/drivers/gpu/drm/apple/plane.h @@ -36,6 +36,7 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, unsigned long possible_crtcs, u32 iomfb_surf, bool supports_l10r, + bool supports_xrgb2101010, enum drm_plane_type type, bool interchange); diff --git a/tools/testing/apple/check-backlight.py b/tools/testing/apple/check-backlight.py index e1df3dc3458de..2a1f6412c9639 100644 --- a/tools/testing/apple/check-backlight.py +++ b/tools/testing/apple/check-backlight.py @@ -79,7 +79,7 @@ def function(file, name): int bl_register_mutex,bl_register_wq; struct { u32 maximum;u32 nits;struct backlight_device *bl_dev;u32 dac; } brightness; struct { int lock;struct dcp_backlight_state state;void (*kick)(struct apple_dcp *); } backlight; - struct { int lock;struct iomfb_transaction *active; } iomfb; + struct { int lock;struct iomfb_transaction *active;int opaque_x_state; } iomfb; }; static int schedule_count,register_count,retire_count,dac_count; static struct backlight_properties registered; @@ -100,6 +100,10 @@ def function(file, name): static void put_unaligned_le64(u64 v,void *p) { for(int i=0;i<8;i++)((u8 *)p)[i]=v>>(8*i); } static void put_unaligned_le32(u32 v,void *p) { for(int i=0;i<4;i++)((u8 *)p)[i]=v>>(8*i); } static void put_unaligned_le16(uint16_t v,void *p) { for(int i=0;i<2;i++)((u8 *)p)[i]=v>>(8*i); } +enum { IOMFB_OPAQUE_X_WAITING, IOMFB_OPAQUE_X_NEEDED }; +static int atomic_cmpxchg(int *value,int expected,int desired) { + int old=*value;if(old==expected)*value=desired;return old; +} int iomfb_configure_backlight_h17p(struct apple_dcp *,u32,bool,u32,bool,u32); ''' functions = [ @@ -194,6 +198,7 @@ def function(file, name): assert(dcp_bl_request(&dcp.backlight.state,140,false,false)==0); assert(!iomfb_present_complete_h17p(&dcp)); assert(dcp.backlight.state.actual==400 && dcp.backlight.state.dirty && !retire_count); + assert(dcp.iomfb.opaque_x_state==IOMFB_OPAQUE_X_WAITING); assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); assert(level(&wire)==140); iomfb_present_failed_h17p(&dcp); @@ -237,6 +242,12 @@ def function(file, name): struct apple_dcp zero=fixture(); assert(dcp_backlight_takeover(&zero,0)==0); assert(zero.backlight.state.target==0 && !zero.backlight.state.dirty); + transaction = (struct iomfb_transaction){.brightness_only=false}; + dcp.iomfb.active = &transaction; + dcp.iomfb.opaque_x_state = IOMFB_OPAQUE_X_WAITING; + assert(iomfb_present_complete_h17p(&dcp)); + assert(dcp.iomfb.opaque_x_state == IOMFB_OPAQUE_X_NEEDED); + puts("Actual takeover/registration/reservation/accepted completion/rejection/20 DPMS cycles passed"); return 0; } From b8173416de4aa7e51fcf79caf23579dfd1730af1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:55:15 +1100 Subject: [PATCH 1093/2680] drm: apple: bound H17P firmware buffer mappings H17P allocates firmware buffers through the DCP device DMA domain, then requests the same scatterlist and IOVA in the piodma child domain. The secondary mapping is read/write. Allocation failures receive an all-zero reply; rejected maps echo the descriptor with a zero device address; normal lifetime unmaps the secondary translation before releasing backing pages. Decode and encode the packed H17P records explicitly in little endian. Admit the unmap and release envelopes, reject stale and duplicate descriptors, and roll back short map results. Keep ownership when unmap is short, and make release remove any live secondary translation before freeing the scatterlist and coherent allocation. This bounds descriptor exhaustion, allocation and scatterlist failures, partial maps and unmaps, duplicate operations, invalid IDs, and release without an earlier unmap while retaining the existing older-firmware paths. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 99 ++++++++++++++++---- drivers/gpu/drm/apple/iomfb_template.c | 124 +++++++++++++++++-------- 2 files changed, 164 insertions(+), 59 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index c2617ee0b36e0..679be29081060 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -41,6 +41,7 @@ static const struct dcp_callback_size callback_sizes[IOMFB_MAX_CB] = { [128] = { 0x1008, 0x4, true }, [129] = { 0x40, 0x4, true }, [201] = { 0xc, 0x10, true }, + [202] = { 0x1a, 0x0, true }, [206] = { 0x0, 0x4, true }, [207] = { 0x0, 0x4, true }, [300] = { 0x10, 0x0, true }, @@ -52,6 +53,7 @@ static const struct dcp_callback_size callback_sizes[IOMFB_MAX_CB] = { [414] = { 0x50, 0x4, true }, [415] = { 0x4c, 0x4, true }, [451] = { 0x14, 0x1c, true }, + [454] = { 0x4, 0x1, true }, [552] = { 0x1044, 0x4, true }, [561] = { 0x1044, 0x4, true }, [563] = { 0x4c, 0x4, true }, @@ -440,7 +442,6 @@ static bool trampoline_allocate_buffer_h17p(struct apple_dcp *dcp, int tag, const struct dcp_allocate_buffer_req *wire = in; struct dcp_allocate_buffer_req request = { 0 }; struct dcp_allocate_buffer_resp reply; - u32 options; if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) return trampoline_allocate_buffer(dcp, tag, out, in); @@ -452,29 +453,91 @@ static bool trampoline_allocate_buffer_h17p(struct apple_dcp *dcp, int tag, static_assert(offsetof(struct dcp_allocate_buffer_req, paddr_null) == 16); trace_iomfb_callback(dcp, tag, __func__); - options = get_unaligned_le32(&wire->unk2); - /* Only these option combinations have observed successful replies. */ - if (get_unaligned_le32(&wire->unk0) != 0x703 || - (options != 4 && options != SZ_16K) || - memchr_inv(&wire->paddr_null, 0, 4)) - goto fail; - request.size = get_unaligned_le64(&wire->size); reply = dcpep_cb_allocate_buffer(dcp, &request); - if (!reply.dva_size) - goto fail; + if (!reply.dva_size) { + /* Exhaustion and allocation errors use the all-zero reply. */ + memset(out, 0, sizeof(reply)); + return true; + } put_unaligned_le64(reply.paddr, out); put_unaligned_le64(reply.dva, (u8 *)out + 8); put_unaligned_le64(reply.dva_size, (u8 *)out + 16); put_unaligned_le32(reply.mem_desc_id, (u8 *)out + 24); return true; +} -fail: - /* Allocation failures have no measured H17P reply encoding. */ - dev_err(dcp->dev, "unqualified or failed D451 buffer allocation\n"); - WRITE_ONCE(dcp->crashed, true); - return false; +static bool trampoline_map_piodma_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_map_buf_req *wire = in; + struct dcp_map_buf_req request = { 0 }; + struct dcp_map_buf_resp_h17p reply = { 0 }; + u64 buffer; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return trampoline_map_piodma(dcp, tag, out, in); + + static_assert(sizeof(*wire) == 0xc); + static_assert(sizeof(reply) == 0x10); + static_assert(offsetof(struct dcp_map_buf_req, unk) == 8); + static_assert(offsetof(struct dcp_map_buf_req, dva_null) == 11); + + trace_iomfb_callback(dcp, tag, __func__); + buffer = get_unaligned_le64(&wire->buffer); + request.buffer = buffer; + if (!dcpep_map_piodma(dcp, &request, &reply)) { + /* A rejected mapping echoes the descriptor with a zero DVA. */ + put_unaligned_le32(buffer, out); + put_unaligned_le64(0, (u8 *)out + 4); + put_unaligned_le32(0, (u8 *)out + 12); + return true; + } + + put_unaligned_le32(reply.buffer, out); + put_unaligned_le64(reply.dva, (u8 *)out + 4); + put_unaligned_le32(reply.unk, (u8 *)out + 12); + return true; +} + +static bool trampoline_unmap_piodma_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_unmap_buf_resp *wire = in; + struct dcp_unmap_buf_resp request = { 0 }; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return trampoline_unmap_piodma(dcp, tag, out, in); + + static_assert(sizeof(*wire) == 0x1a); + static_assert(offsetof(struct dcp_unmap_buf_resp, vaddr) == 8); + static_assert(offsetof(struct dcp_unmap_buf_resp, dva) == 16); + static_assert(offsetof(struct dcp_unmap_buf_resp, buf_null) == 25); + + trace_iomfb_callback(dcp, tag, __func__); + request.buffer = get_unaligned_le64(&wire->buffer); + request.vaddr = get_unaligned_le64(&wire->vaddr); + request.dva = get_unaligned_le64(&wire->dva); + request.unk = wire->unk; + request.buf_null = wire->buf_null; + dcpep_unmap_piodma(dcp, &request); + return true; +} + +static bool +trampoline_release_mem_desc_h17p(struct apple_dcp *dcp, int tag, void *out, + void *in) +{ + u32 id; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return trampoline_release_mem_desc(dcp, tag, out, in); + + trace_iomfb_callback(dcp, tag, __func__); + id = get_unaligned_le32(in); + *(u8 *)out = dcpep_cb_release_mem_desc(dcp, &id); + return true; } /* H17P callback numbering is not a uniform shift of the v13.5 table. */ @@ -509,8 +572,8 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [128] = trampoline_prop_chunk, /* setDCPAVPropChunk */ [129] = trampoline_prop_end, /* setDCPAVPropEnd */ [130] = trampoline_allocate_bandwidth, /* allocate_bandwidth */ - [201] = trampoline_map_piodma, /* map_buf */ - [202] = trampoline_unmap_piodma, /* unmap_buf */ + [201] = trampoline_map_piodma_h17p, /* map_buf */ + [202] = trampoline_unmap_piodma_h17p, /* unmap_buf */ [206] = iomfbep_cb_match_pmu_service_2, /* match_pmu_service */ [207] = iomfbep_cb_match_backlight_service, /* match_backlight_service */ [208] = trampoline_nop, /* update_backlight_factor_prop */ @@ -527,7 +590,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [415] = trampoline_true, /* setProperty */ [451] = trampoline_allocate_buffer_h17p, /* allocate_buffer */ [452] = trampoline_map_physical, /* prepare */ - [454] = trampoline_release_mem_desc, /* release_descriptor */ + [454] = trampoline_release_mem_desc_h17p, /* release_descriptor */ [552] = trampoline_true, [561] = trampoline_true, [563] = trampoline_true, diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index c86d8d848365e..2bd76dccb95be 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -100,7 +100,7 @@ DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, * channel. Sent on the ordinary command channel it is never answered, which * strands the power transition and every later call behind it. */ -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER #define DCP_POWER_OOB true #else #define DCP_POWER_OOB false @@ -503,11 +503,13 @@ static void iomfbep_cb_set_fx_prop(struct apple_dcp *dcp, struct iomfb_set_fx_pr * PIODMA is separate from the main DCP and uses own IOVA space on a dedicated * stream of the display DART, rather than the expected DCP DART. */ -static struct DCP_FW_NAME(dcp_map_buf_resp) dcpep_cb_map_piodma(struct apple_dcp *dcp, - struct dcp_map_buf_req *req) +static bool dcpep_map_piodma(struct apple_dcp *dcp, + struct dcp_map_buf_req *req, + struct DCP_FW_NAME(dcp_map_buf_resp) * resp) { struct dcp_mem_descriptor *memdesc; struct sg_table *map; + size_t size; ssize_t ret; if (req->buffer >= ARRAY_SIZE(dcp->memdesc)) @@ -515,81 +517,112 @@ static struct DCP_FW_NAME(dcp_map_buf_resp) dcpep_cb_map_piodma(struct apple_dcp memdesc = &dcp->memdesc[req->buffer]; map = &memdesc->map; + size = ALIGN(memdesc->size, SZ_16K); - if (!map->sgl) + if (!test_bit(req->buffer, dcp->memdesc_map) || !map->sgl || + memdesc->piodma_mapped || !dcp->iommu_dom) goto reject; /* use the piodma iommu domain to map against the right IOMMU */ ret = iommu_map_sgtable(dcp->iommu_dom, memdesc->dva, map, IOMMU_READ | IOMMU_WRITE); - /* HACK: expect size to be 16K aligned since the iommu API only maps - * full pages - */ - if (ret < 0 || ret != ALIGN(memdesc->size, SZ_16K)) { - dev_err(dcp->dev, "iommu_map_sgtable() returned %zd instead of expected buffer size of %zu\n", ret, memdesc->size); + if (ret != (ssize_t)size) { + dev_err(dcp->dev, + "iommu_map_sgtable() returned %zd instead of %zu\n", + ret, size); + if (ret > 0) + iommu_unmap(dcp->iommu_dom, memdesc->dva, ret); goto reject; } #if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* remembered so release_mem_desc() can tear a stale mapping down */ memdesc->piodma_mapped = true; /* the H17P reply echoes the buffer index */ - return (struct DCP_FW_NAME(dcp_map_buf_resp)){ + *resp = (struct DCP_FW_NAME(dcp_map_buf_resp)) { .buffer = req->buffer, .dva = memdesc->dva, }; #else - return (struct DCP_FW_NAME(dcp_map_buf_resp)){ .dva = memdesc->dva }; + *resp = (struct DCP_FW_NAME(dcp_map_buf_resp)) { .dva = memdesc->dva }; #endif + return true; reject: dev_err(dcp->dev, "denying map of invalid buffer %llx for piodma\n", req->buffer); + return false; +} + +static struct DCP_FW_NAME(dcp_map_buf_resp) dcpep_cb_map_piodma(struct apple_dcp *dcp, + struct dcp_map_buf_req *req) +{ + struct DCP_FW_NAME(dcp_map_buf_resp) resp = { 0 }; + + if (dcpep_map_piodma(dcp, req, &resp)) + return resp; + #if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - return (struct DCP_FW_NAME(dcp_map_buf_resp)){ - .buffer = req->buffer, - .dva = 0, - }; + resp.buffer = req->buffer; #else - return (struct DCP_FW_NAME(dcp_map_buf_resp)){ .ret = EINVAL }; + resp.ret = EINVAL; #endif + return resp; } -static void dcpep_cb_unmap_piodma(struct apple_dcp *dcp, - struct dcp_unmap_buf_resp *resp) +static bool dcpep_unmap_piodma(struct apple_dcp *dcp, + struct dcp_unmap_buf_resp *request) { struct dcp_mem_descriptor *memdesc; + size_t size, unmapped; - if (resp->buffer >= ARRAY_SIZE(dcp->memdesc)) { + if (request->buffer >= ARRAY_SIZE(dcp->memdesc)) { dev_warn(dcp->dev, "unmap request for out of range buffer %llu\n", - resp->buffer); - return; + request->buffer); + return false; } - memdesc = &dcp->memdesc[resp->buffer]; + memdesc = &dcp->memdesc[request->buffer]; - if (!memdesc->buf) { - dev_warn(dcp->dev, - "unmap for non-mapped buffer %llu iova:0x%08llx\n", - resp->buffer, resp->dva); - return; - } + if (!test_bit(request->buffer, dcp->memdesc_map) || !memdesc->buf || + !dcp->iommu_dom) + goto not_mapped; +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (!memdesc->piodma_mapped) + goto not_mapped; +#endif - if (memdesc->dva != resp->dva) { - dev_warn(dcp->dev, "unmap buffer %llu address mismatch " - "memdesc.dva:%llx dva:%llx\n", resp->buffer, - memdesc->dva, resp->dva); - return; + if (memdesc->dva != request->dva) { + dev_warn(dcp->dev, + "unmap buffer %llu address mismatch memdesc.dva:%llx dva:%llx\n", + request->buffer, + memdesc->dva, request->dva); + return false; } /* use the piodma iommu domain to unmap from the right IOMMU */ - /* HACK: expect size to be 16K aligned since the iommu API only maps - * full pages - */ - iommu_unmap(dcp->iommu_dom, memdesc->dva, ALIGN(memdesc->size, SZ_16K)); + size = ALIGN(memdesc->size, SZ_16K); + unmapped = iommu_unmap(dcp->iommu_dom, memdesc->dva, size); + if (unmapped != size) { + dev_err(dcp->dev, "iommu_unmap() returned %zu instead of %zu\n", + unmapped, size); + return false; + } memdesc->piodma_mapped = false; + return true; + +not_mapped: + dev_warn(dcp->dev, + "unmap for non-mapped buffer %llu iova:0x%08llx\n", + request->buffer, request->dva); + return false; +} + +static void dcpep_cb_unmap_piodma(struct apple_dcp *dcp, + struct dcp_unmap_buf_resp *request) +{ + dcpep_unmap_piodma(dcp, request); } /* @@ -657,7 +690,7 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) { struct dcp_mem_descriptor *memdesc; - size_t size; + size_t size, unmapped; u32 id = *mem_desc_id; if (id >= DCP_MAX_MAPPINGS) { @@ -666,7 +699,7 @@ static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) return 0; } - if (!test_and_clear_bit(id, dcp->memdesc_map)) { + if (!test_bit(id, dcp->memdesc_map)) { dev_warn(dcp->dev, "unmap request for unused mem_desc_id %u\n", id); return 0; @@ -687,9 +720,18 @@ static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) dev_warn(dcp->dev, "releasing buffer %u while still mapped for piodma; unmapping first\n", id); - iommu_unmap(dcp->iommu_dom, memdesc->dva, size); + if (!dcp->iommu_dom) + return 0; + unmapped = iommu_unmap(dcp->iommu_dom, memdesc->dva, size); + if (unmapped != size) { + dev_err(dcp->dev, + "failed to unmap buffer %u before release: %zu of %zu\n", + id, unmapped, size); + return 0; + } memdesc->piodma_mapped = false; } + clear_bit(id, dcp->memdesc_map); if (memdesc->buf) { sg_free_table(&memdesc->map); From a4dd20d5077d41f4b4d3d8a58c291820f524bef5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 08:46:32 +1100 Subject: [PATCH 1094/2680] drm: apple: retain J700 provider replies with older loaders The working J700 loader predates the display power-table DT properties. Requiring only those new properties prevents the firmware's default framebuffer setup before a compatible loader can be installed. Provide the independently observed J700 table bytes when the internal T8140 endpoint lacks a property and the machine is specifically J700. Explicit per-device DT data always wins; malformed present properties still fail validation. Other boards keep the loader-data requirement. The byte arrays match both the retained disp0 ADT and firmware replies; no physical brightness range or power-model interpretation is added. New loaders can forward all five properties verbatim using the existing DT interface, removing the need for this compatibility path. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 51 ++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 679be29081060..d02c26d04671a 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -314,19 +314,49 @@ struct dcp_h17p_provider_property { const char *name; const char *dt_name; u32 length; + const u8 *legacy_data; }; static_assert(sizeof(struct dcp_h17p_provider_request) == 0x4c); static_assert(sizeof(struct dcp_h17p_provider_reply) == 0xc04); +/* Compatibility data matches the independently decoded J700 disp0 ADT. */ +static const u8 j700_provider_data_0[] = { + 0x0d, 0x00, 0x00, 0x00 +}; + +static const u8 j700_provider_data_1[] = { + 0x01, 0x00, 0x00, 0x00 +}; + +static const u8 j700_provider_data_2[] = { + 0x02, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, + 0x64, 0x00, 0x00, 0x00, 0x96, 0x00, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, + 0xfa, 0x00, 0x00, 0x00, 0x2c, 0x01, 0x00, 0x00, 0x5e, 0x01, 0x00, 0x00, + 0x90, 0x01, 0x00, 0x00, 0xc2, 0x01, 0x00, 0x00, 0xf4, 0x01, 0x00, 0x00, + 0x0d, 0x02, 0x00, 0x00 +}; + +static const u8 j700_provider_data_3[] = { + 0x64, 0x00, 0x00, 0x00 +}; + +static const u8 j700_provider_data_4[] = { + 0x34, 0x03, 0x00, 0x00, 0x84, 0x03, 0x00, 0x00, 0x24, 0x05, 0x00, 0x00, + 0x52, 0x07, 0x00, 0x00, 0x11, 0x09, 0x00, 0x00, 0xde, 0x0a, 0x00, 0x00, + 0xd2, 0x0c, 0x00, 0x00, 0xe2, 0x0e, 0x00, 0x00, 0xfb, 0x10, 0x00, 0x00, + 0x64, 0x13, 0x00, 0x00, 0xd5, 0x15, 0x00, 0x00, 0x68, 0x18, 0x00, 0x00, + 0xa2, 0x19, 0x00, 0x00 +}; + /* Only these provider transfers have been measured against the disp0 ADT. */ static const struct dcp_h17p_provider_property provider_properties[] = { - { "power-lut-data-x", "apple,power-lut-data-x", 4 }, - { "power-lut-data-y", "apple,power-lut-data-y", 4 }, - { "power-lut-data-xindex", "apple,power-lut-data-xindex", 52 }, - { "power-lut-data-yindex", "apple,power-lut-data-yindex", 4 }, - { "power-lut-data-lut", "apple,power-lut-data-lut", 52 }, - { "power-lut-vbatt-cur-nominal", "apple,power-lut-vbatt-cur-nominal", 0 }, + { "power-lut-data-x", "apple,power-lut-data-x", 4, j700_provider_data_0 }, + { "power-lut-data-y", "apple,power-lut-data-y", 4, j700_provider_data_1 }, + { "power-lut-data-xindex", "apple,power-lut-data-xindex", 52, j700_provider_data_2 }, + { "power-lut-data-yindex", "apple,power-lut-data-yindex", 4, j700_provider_data_3 }, + { "power-lut-data-lut", "apple,power-lut-data-lut", 52, j700_provider_data_4 }, + { "power-lut-vbatt-cur-nominal", "apple,power-lut-vbatt-cur-nominal", 0, NULL }, }; static const struct dcp_h17p_provider_property * @@ -372,6 +402,15 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, /* Only this absent property has a measured empty reply. */ if (of_property_present(node, property->dt_name)) goto fail; + } else if (!of_property_present(node, property->dt_name) && + of_machine_is_compatible("apple,j700") && + of_device_is_compatible(node, "apple,t8140-dcp")) { + /* Qualified older loaders lack the newly described table properties. */ + memset(reply, 0, sizeof(*reply)); + memcpy(reply->data, property->legacy_data, property->length); + reply->length = cpu_to_le32(property->length); + dev_warn_once(dcp->dev, "using J700 provider data for an older loader\n"); + return true; } else if (length != (int)property->length) { dev_err(dcp->dev, "invalid provider property %s: %d bytes\n", property->dt_name, length); From 4fa948c905d2b546c0e1f8b528289532ef66019e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 10:37:24 +1100 Subject: [PATCH 1095/2680] drm: apple: accept the legacy J700 primary surface selection The qualified J700 loader passes one zero in apple,iomfb-surfaces to select firmware surface zero. The per-slot flag parser instead leaves the bitmap empty, which registers a CRTC without a primary plane and crashes fbdev initialization. Recognize this exact legacy encoding only on J700 internal T8140 DCP, and reject empty surface masks before CRTC creation on every profile. Also check the CRTC allocation before use. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 4 ++++ drivers/gpu/drm/apple/dcp.c | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index c950ccd53d9f3..1166963c6591f 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -413,6 +413,8 @@ static int apple_probe_per_dcp(struct device *dev, supports_xrgb2101010 = dcp_data->fw_compat != DCP_FIRMWARE_H17P || dcp_data->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G; + if (bitmap_empty(iomfb_surfaces, DCP_MAX_PLANES)) + return dev_err_probe(dev, -EINVAL, "No usable display surfaces\n"); /* * BUG: @@ -449,6 +451,8 @@ static int apple_probe_per_dcp(struct device *dev, * knows what to do with overlays. */ crtc = kzalloc(sizeof(*crtc), GFP_KERNEL); + if (!crtc) + return -ENOMEM; ret = drm_crtc_init_with_planes(drm, &crtc->base, planes[0], NULL, &apple_crtc_funcs, NULL); if (ret) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 66a982538b813..7159eff4d89bb 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2343,6 +2343,13 @@ static int dcp_platform_probe(struct platform_device *pdev) set_bit(surf, dcp->iomfb_surfaces); surf++; } + /* The qualified J700 loader names surface zero rather than enabling it. */ + if (of_machine_is_compatible("apple,j700") && + of_device_is_compatible(dev->of_node, "apple,t8140-dcp") && + num_surfs == 1 && + !of_property_read_u32(dev->of_node, "apple,iomfb-surfaces", &surf_en) && + surf_en == 0) + set_bit(0, dcp->iomfb_surfaces); if (dcp->phy) { int ret; From 6bbaa0b6505919f3572f2a49e78e2e62f7d9e5fb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 10:37:25 +1100 Subject: [PATCH 1096/2680] dt-bindings: display: apple: document legacy J700 surface selection The qualified J700 bootloader uses one zero to select the primary surface, rather than the per-slot enable flags used by other loaders. Document that existing compatibility encoding. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/display/apple,dcp.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index aefb570de5983..bba5d64f50451 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -121,7 +121,9 @@ properties: maxItems: 4 items: enum: [0, 1] - description: Per-slot enable flags, in slot order. + description: | + Per-slot enable flags, in slot order. The legacy J700 bootloader + supplies a single zero to select surface zero instead. apple,power-lut-data-x: $ref: /schemas/types.yaml#/definitions/uint8-array From 6de4ce8a04e669316f707c51e4eac444017d0635 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 11:54:21 +1100 Subject: [PATCH 1097/2680] drm: apple: retain adopted H17P DMA resources through reboot Internal H17P keeps its display pipe running when the panel is blanked. Its shutdown path stops the outbound queue without a firmware-acknowledged scanout stop. Deleting the component nevertheless releases the binding's devres group, including RTKit shared buffers, and destroys a host-created piodma device. The running firmware can still access those resources. During system shutdown, stop host submissions and backlight work while keeping the adopted component and its DMA resources allocated until reset. Keep the existing component teardown for the older firmware profiles. This does not qualify component removal or a firmware client-close method. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 7159eff4d89bb..a38edf8604f17 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2454,6 +2454,24 @@ static void dcp_platform_remove(struct platform_device *pdev) static void dcp_platform_shutdown(struct platform_device *pdev) { + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + if (dcp && dcp->hw.adopt_live_session && + dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + /* + * This firmware keeps scanning across soft DPMS and has no + * qualified stop sequence. Component unbind releases the RTKit + * devres group and the piodma domain while firmware may still + * access them. Stop host submissions but keep the component and + * its DMA resources alive until the system resets. + */ + iomfb_queue_stop(dcp); + cancel_work_sync(&dcp->bl_register_wq); + cancel_work_sync(&dcp->bl_update_wq); + return; + } + component_del(&pdev->dev, &dcp_comp_ops); } From 4a96a7711f683203b2815ad865af8fbd3ca1fdf9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:12:00 +1100 Subject: [PATCH 1098/2680] drm: apple: retain live H17P resources across component teardown An adopted H17P session keeps scanning after soft DPMS. Component unbind and bind-error unwind release the RTKit devres group even though firmware has not acknowledged a DMA stop. Driver removal can also drop its default IOMMU-domain usage and release AFK rings and consumer contexts. Export the existing default-domain usage helpers for module callers. Own that usage in the DCP driver and retain it, RTKit shared buffers, AFK rings, PIODMA and the referenced DRM objects until reset once negotiation with the live firmware starts. Quiesce host callbacks and AFK work without sending an unqualified firmware stop. Keep their contexts valid through managed cleanup and unplug the DRM device before teardown. Pin the DMA device, IOMMU group and module for that firmware lifetime, reject rebinding the same firmware node with an explicit reboot-required message, and keep the existing resource releases for profiles with a normal stop path. Pre-negotiation probe failures still release their resources. Add a native regression exercising the actual component/master unbind and resource-release functions through simulated component and driver devres cleanup, including partial initialization and legacy release behavior. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 43 +++++- drivers/gpu/drm/apple/afk.h | 1 + drivers/gpu/drm/apple/apple_drv.c | 2 +- drivers/gpu/drm/apple/dcp-internal.h | 4 + drivers/gpu/drm/apple/dcp.c | 149 ++++++++++++++++++- drivers/gpu/drm/apple/dcp_backlight.c | 8 ++ drivers/iommu/iommu.c | 8 +- tools/testing/apple/check-lifetime.py | 199 ++++++++++++++++++++++++++ 8 files changed, 402 insertions(+), 12 deletions(-) create mode 100644 tools/testing/apple/check-lifetime.py diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 9440662d55788..27857ab068467 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -54,13 +55,22 @@ static void afk_send(struct apple_dcp_afkep *ep, u64 message) dcp_send_message(ep->dcp, ep->endpoint, message); } +static void afk_release_context(void *data) +{ + struct apple_dcp_afkep *afkep = data; + + afk_quiesce(afkep); + if (!afkep->dcp->retain_dma) + kfree(afkep); +} + struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, const struct apple_epic_service_ops *ops) { struct apple_dcp_afkep *afkep; int ret; - afkep = devm_kzalloc(dcp->dev, sizeof(*afkep), GFP_KERNEL); + afkep = kzalloc_obj(*afkep); if (!afkep) return ERR_PTR(-ENOMEM); @@ -74,19 +84,32 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, goto out_free_afkep; } - // TODO: devm_ for wq - init_completion(&afkep->started); init_completion(&afkep->stopped); spin_lock_init(&afkep->lock); + ret = devm_add_action_or_reset(dcp->dev, afk_release_context, afkep); + if (ret) + return ERR_PTR(ret); return afkep; out_free_afkep: - devm_kfree(dcp->dev, afkep); + kfree(afkep); return ERR_PTR(ret); } +void afk_quiesce(struct apple_dcp_afkep *afkep) +{ + if (IS_ERR_OR_NULL(afkep)) + return; + if (afkep->wq) { + destroy_workqueue(afkep->wq); + afkep->wq = NULL; + } + debugfs_remove_recursive(afkep->debugfs_entry); + afkep->debugfs_entry = NULL; +} + void afk_shutdown(struct apple_dcp_afkep *afkep) { afk_send(afkep, FIELD_PREP(RBEP_TYPE, RBEP_SHUTDOWN)); @@ -97,7 +120,7 @@ void afk_shutdown(struct apple_dcp_afkep *afkep) dev_err(afkep->dcp->dev, "Timed out shutting down AFK endpoint %02x", afkep->endpoint); } - destroy_workqueue(afkep->wq); + afk_quiesce(afkep); } int afk_start(struct apple_dcp_afkep *ep) @@ -131,8 +154,11 @@ static void afk_getbuf(struct apple_dcp_afkep *ep, u64 message) return; } - ep->bfr = dmam_alloc_coherent(ep->dcp->dev, size, &ep->bfr_dma, - GFP_KERNEL); + /* An unstopped adopted session owns this ring until the machine resets. */ + if (READ_ONCE(ep->dcp->retain_dma)) + ep->bfr = dma_alloc_coherent(ep->dcp->dev, size, &ep->bfr_dma, GFP_KERNEL); + else + ep->bfr = dmam_alloc_coherent(ep->dcp->dev, size, &ep->bfr_dma, GFP_KERNEL); if (!ep->bfr) { dev_err(ep->dcp->dev, "Failed to allocate %d bytes buffer\n", size); @@ -1011,6 +1037,9 @@ int afk_send_command(struct apple_epic_service *service, u8 type, struct apple_dcp_afkep *ep = service->ep; DECLARE_COMPLETION_ONSTACK(completion); + if (READ_ONCE(ep->dcp->quiescing)) + return -ENODEV; + rxbuf = dma_alloc_coherent(ep->dcp->dev, output_len, &rxbuf_dma, GFP_KERNEL); if (!rxbuf) diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 1e691ec47c85e..85085a3cc411c 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -224,6 +224,7 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, const struct apple_epic_service_ops *ops); int afk_start(struct apple_dcp_afkep *ep); void afk_shutdown(struct apple_dcp_afkep *ep); +void afk_quiesce(struct apple_dcp_afkep *ep); int afk_receive_message(struct apple_dcp_afkep *ep, u64 message); int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, enum epic_type etype, enum epic_category ecat, u8 stype, diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 1166963c6591f..d33b73de74e58 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -766,7 +766,7 @@ static void apple_drm_uninit(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); - drm_dev_unregister(&apple->drm); + drm_dev_unplug(&apple->drm); drm_atomic_helper_shutdown(&apple->drm); component_unbind_all(dev, NULL); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 299c909c70acb..f794cece99a0d 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -175,6 +175,10 @@ struct apple_dcp_hw_data { /* TODO: move IOMFB members to its own struct */ struct apple_dcp { + /* An adopted session has no confirmed firmware DMA stop boundary. */ + bool retain_dma; + bool quiescing; + bool drm_retained; struct device *dev; struct platform_device *piodma; bool piodma_created; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a38edf8604f17..1ef15011a69c8 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,7 @@ #include #include +#include #include #include #include @@ -46,6 +48,83 @@ #define APPLE_DCP_COPROC_CPU_CONTROL 0x44 #define APPLE_DCP_COPROC_CPU_CONTROL_RUN BIT(4) +struct dcp_session_ref { + struct list_head link; + struct device *dev; + struct iommu_group *group; +}; + +static LIST_HEAD(dcp_retained_sessions); +static DEFINE_MUTEX(dcp_retained_sessions_lock); + +static bool dcp_session_retained(struct device *dev) +{ + struct dcp_session_ref *session; + bool retained = false; + + mutex_lock(&dcp_retained_sessions_lock); + list_for_each_entry(session, &dcp_retained_sessions, link) { + if (dev_fwnode(session->dev) == dev_fwnode(dev)) { + retained = true; + break; + } + } + mutex_unlock(&dcp_retained_sessions_lock); + return retained; +} + +static int dcp_pin_live_session(struct apple_dcp *dcp) +{ + struct dcp_session_ref *session; + + if (!dcp->hw.adopt_live_session || dcp->fw_compat != DCP_FIRMWARE_H17P || + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G || dcp->retain_dma) + return 0; + session = kzalloc_obj(*session); + if (!session) + return -ENOMEM; + + /* Keep the DMA device, domain usage and driver code valid until reset. */ + session->dev = get_device(dcp->dev); + session->group = iommu_group_get(dcp->dev); + __module_get(THIS_MODULE); + WRITE_ONCE(dcp->retain_dma, true); + mutex_lock(&dcp_retained_sessions_lock); + list_add_tail(&session->link, &dcp_retained_sessions); + mutex_unlock(&dcp_retained_sessions_lock); + dev_info(dcp->dev, "live H17P DMA resources retained until reboot; rebind unsupported\n"); + return 0; +} + +static void dcp_release_dma_domain(void *data) +{ + struct apple_dcp *dcp = data; + + if (!dcp->retain_dma) + iommu_device_unuse_default_domain(dcp->dev); +} + +static void dcp_release_context(void *data) +{ + struct apple_dcp *dcp = data; + + if (!dcp->retain_dma) + kfree(dcp); +} + +static void dcp_release_rtkit(void *data) +{ + struct apple_dcp *dcp = data; + + if (dcp->retain_dma) { + WRITE_ONCE(dcp->quiescing, true); + apple_rtkit_free_retaining_buffers(dcp->rtk); + } else { + apple_rtkit_free(dcp->rtk); + } + dcp->rtk = NULL; +} + #define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) static bool show_notch; @@ -850,6 +929,9 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) struct apple_dcp *dcp = cookie; struct apple_dcp_afkep *ep; + if (READ_ONCE(dcp->quiescing)) + return; + trace_dcp_recv_msg(dcp, endpoint, message); /* @@ -901,6 +983,9 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ { struct apple_dcp *dcp = cookie; + if (READ_ONCE(dcp->quiescing)) + return; + dcp->crashed = true; dev_err(dcp->dev, "DCP has crashed\n"); if (dcp->connector) { @@ -1004,6 +1089,8 @@ static struct apple_rtkit_ops rtkit_ops = { void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message) { + if (READ_ONCE(dcp->quiescing)) + return; trace_dcp_send_msg(dcp, endpoint, message); apple_rtkit_send_message(dcp->rtk, endpoint, message, NULL, true); @@ -1648,6 +1735,10 @@ static void dcp_work_update_backlight(struct work_struct *work) static void dcp_release_piodma_iommu_dev(struct apple_dcp *dcp) { + if (dcp->retain_dma) { + WRITE_ONCE(dcp->quiescing, true); + return; + } if (dcp->piodma) { if (dcp->piodma_created) of_platform_device_destroy(&dcp->piodma->dev, NULL); @@ -2057,6 +2148,10 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) u32 cpu_ctrl; int ret; + if (READ_ONCE(dcp->quiescing)) + return dev_err_probe(dev, -EBUSY, + "Live DCP session cannot be rebound; reboot required\n"); + /* A timed-out prior session may still scan these mappings. */ mutex_lock(&dcp->swapped_out_fbs_lock); ret = list_empty(&dcp->swapped_out_fbs) ? 0 : -EBUSY; @@ -2174,12 +2269,19 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); } - dcp->rtk = devm_apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); + /* Registering the mailbox can already expose RTKit buffer requests. */ + ret = dcp_pin_live_session(dcp); + if (ret) + goto err_piodma; + dcp->rtk = apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); if (IS_ERR(dcp->rtk)) { ret = dev_err_probe(dev, PTR_ERR(dcp->rtk), "Failed to initialize RTKit\n"); goto err_piodma; } + ret = devm_add_action_or_reset(dev, dcp_release_rtkit, dcp); + if (ret) + goto err_piodma; if (dcp->hw.adopt_live_session) dev_info(dev, "negotiating RTKit with the running DCP\n"); @@ -2207,6 +2309,34 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) if (!dcp) return; + if (dcp->retain_dma) { + /* No firmware stop is acknowledged: detach host users, never DMA. */ + WRITE_ONCE(dcp->quiescing, true); + if (dcp->crtc && !dcp->drm_retained) { + drm_dev_get(dcp->crtc->base.dev); + dcp->drm_retained = true; + } + iomfb_queue_stop(dcp); + afk_quiesce(dcp->avep); + afk_quiesce(dcp->dptxep); + afk_quiesce(dcp->ibootep); + afk_quiesce(dcp->systemep); + afk_quiesce(dcp->dcpavservep); + if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { + cancel_work_sync(&dcp->bl_register_wq); + cancel_work_sync(&dcp->bl_update_wq); + } + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); + cancel_work_sync(&dcp->vblank_wq); + if (dcp->connector) + cancel_work_sync(&dcp->connector->hotplug_wq); + if (dcp->typec_connector && dcp->typec_connector != dcp->connector) + cancel_work_sync(&dcp->typec_connector->hotplug_wq); + dev_warn(dev, "retaining live DCP DMA resources after unbind; reboot required\n"); + return; + } + if (dcp->hdmi_hpd_irq) disable_irq(dcp->hdmi_hpd_irq); @@ -2273,6 +2403,9 @@ static int dcp_platform_probe(struct platform_device *pdev) u32 surf_en; u32 mux_index; + if (dcp_session_retained(dev)) + return dev_err_probe(dev, -EBUSY, "Previous live DCP session requires a reboot\n"); + fw_compat = dcp_check_firmware_version(dev); if (fw_compat == DCP_FIRMWARE_UNKNOWN) return -ENODEV; @@ -2286,9 +2419,12 @@ static int dcp_platform_probe(struct platform_device *pdev) return dev_err_probe(dev, -ENODEV, "Incompatible devicetree! " "Use devicetree matching this kernel.\n"); - dcp = devm_kzalloc(dev, sizeof(*dcp), GFP_KERNEL); + dcp = kzalloc_obj(*dcp); if (!dcp) return -ENOMEM; + ret = devm_add_action_or_reset(dev, dcp_release_context, dcp); + if (ret) + return ret; INIT_LIST_HEAD(&dcp->swapped_out_fbs); mutex_init(&dcp->swapped_out_fbs_lock); @@ -2298,6 +2434,14 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp->fw_compat = fw_compat; dcp->dev = dev; dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); + + /* The domain use must survive driver-core DMA cleanup for a live session. */ + ret = iommu_device_use_default_domain(dev); + if (ret) + return ret; + ret = devm_add_action_or_reset(dev, dcp_release_dma_domain, dcp); + if (ret) + return ret; dcp->fixed_connector_type = dcp_connector_type_from_dt(dev->of_node); dcp->connector_type = dcp->fixed_connector_type; of_property_read_u32(dev->of_node, "apple,dcp-index", &dcp->index); @@ -2568,6 +2712,7 @@ static const struct of_device_id of_match[] = { MODULE_DEVICE_TABLE(of, of_match); static struct platform_driver apple_platform_driver = { + .driver_managed_dma = true, .probe = dcp_platform_probe, .remove = dcp_platform_remove, .shutdown = dcp_platform_shutdown, diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index fbf2fef59dd8e..c2a902de07773 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -111,6 +111,9 @@ static int dcp_get_brightness(struct backlight_device *bd) unsigned long flags; u32 actual; + if (READ_ONCE(dcp->quiescing)) + return -ENODEV; + if (dcp_backlight_active(dcp)) { spin_lock_irqsave(&dcp->backlight.lock, flags); actual = dcp->backlight.state.actual; @@ -321,6 +324,8 @@ static int drm_crtc_set_brightness(struct apple_dcp *dcp) int dcp_backlight_update(struct apple_dcp *dcp) { + if (READ_ONCE(dcp->quiescing)) + return -ENODEV; if (dcp_backlight_active(dcp)) { dcp_backlight_kick(dcp); return 0; @@ -352,6 +357,9 @@ static int dcp_set_brightness(struct backlight_device *bd) int brightness = backlight_get_brightness(bd); unsigned long flags; + if (READ_ONCE(dcp->quiescing)) + return -ENODEV; + if (dcp_backlight_active(dcp)) { /* Preserve the requested level even while the core forces zero. */ spin_lock_irqsave(&dcp->backlight.lock, flags); diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index aa4b60914c4a0..54a0e4a4e0993 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -3349,10 +3349,12 @@ static ssize_t iommu_group_store_type(struct iommu_group *group, * * The device driver about to bind @dev wants to do DMA through the kernel * DMA API. Return 0 if it is allowed, otherwise an error. + * Drivers with driver_managed_dma may manage this usage themselves. The + * device lock must serialize probe and removal while holding the usage. */ int iommu_device_use_default_domain(struct device *dev) { - /* Caller is the driver core during the pre-probe path */ + /* Called with the device lock held during probe. */ struct iommu_group *group = dev->iommu_group; int ret = 0; @@ -3379,6 +3381,7 @@ int iommu_device_use_default_domain(struct device *dev) mutex_unlock(&group->mutex); return ret; } +EXPORT_SYMBOL_GPL(iommu_device_use_default_domain); /** * iommu_device_unuse_default_domain() - Device driver stops handling device @@ -3390,7 +3393,7 @@ int iommu_device_use_default_domain(struct device *dev) */ void iommu_device_unuse_default_domain(struct device *dev) { - /* Caller is the driver core during the post-probe path */ + /* Called with the device lock held during removal. */ struct iommu_group *group = dev->iommu_group; if (!group) @@ -3402,6 +3405,7 @@ void iommu_device_unuse_default_domain(struct device *dev) mutex_unlock(&group->mutex); } +EXPORT_SYMBOL_GPL(iommu_device_unuse_default_domain); static int __iommu_group_alloc_blocking_domain(struct iommu_group *group) { diff --git a/tools/testing/apple/check-lifetime.py b/tools/testing/apple/check-lifetime.py new file mode 100644 index 0000000000000..d7f82f276dc84 --- /dev/null +++ b/tools/testing/apple/check-lifetime.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Native regression for component/devres teardown of an unstopped session. + +Mocks kernel resource releases and the component framework. It executes the +actual unbind, PIODMA release, RTKit release, domain release and ring allocator. +No firmware calls or hardware acceptance are claimed. +""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import subprocess + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument('--output', type=Path, required=True) +p.add_argument('--cc', default='clang') +a = p.parse_args() +root = Path(__file__).resolve().parents[3] +src = root / 'drivers/gpu/drm/apple' +a.output.mkdir(parents=True, exist_ok=True) + + +def function(file, name): + text = (src / file).read_text() + match = re.search(r'^\w[\w\s*]*?\b' + name + r'\(', text, re.M) + assert match, name + start = text.index('{', match.end()) + pos, depth = start + 1, 1 + while depth: + depth += (text[pos] == '{') - (text[pos] == '}') + pos += 1 + return text[match.start():pos] + + +# This is the driver-core switch that transfers default-domain lifetime to us. +text = (src / 'dcp.c').read_text() +assert re.search(r'static struct platform_driver apple_platform_driver = \{\s*' + r'\.driver_managed_dma = true,', text) +functions = '\n'.join(function(f, n) for f, n in ( + ('dcp.c', 'dcp_release_dma_domain'), ('dcp.c', 'dcp_release_rtkit'), + ('dcp.c', 'dcp_release_piodma_iommu_dev'), ('afk.c', 'afk_quiesce'), + ('afk.c', 'afk_release_context'), ('dcp.c', 'dcp_release_context'), + ('afk.c', 'afk_getbuf'), ('dcp.c', 'dcp_comp_unbind'), + ('apple_drv.c', 'apple_drm_uninit'))) +prefix = r''' +#include +#include +#include +#include +#include +#include +typedef uint8_t u8;typedef uint16_t u16;typedef uint32_t u32;typedef uint64_t u64; +#define GFP_KERNEL 0 +#define BLOCK_SHIFT 6 +#define GETBUF_SIZE 0xffffU +#define GETBUF_TAG 0xffff0000U +#define RBEP_TYPE 0 +#define RBEP_GETBUF_ACK 0 +#define GETBUF_ACK_DVA 0 +#define FIELD_GET(mask,x) ((x)&(mask)) +#define FIELD_PREP(mask,x) ((void)(mask),(u64)(x)) +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define IS_ERR_OR_NULL(x) (!(x)||(uintptr_t)(x)>(uintptr_t)-4096) +#define DRM_MODE_CONNECTOR_eDP 14 +struct work_struct {int id;};struct delayed_work {struct work_struct work;}; +struct drm_device {int refs;bool unplugged;}; +struct apple_drm_private {struct drm_device drm;}; +struct device {void *data;int domain_users;}; +struct platform_device {struct device dev;}; +struct apple_crtc {struct {struct drm_device *dev;} base;}; +struct apple_connector {struct work_struct hotplug_wq;}; +struct apple_rtkit {bool rx_stopped;}; +struct apple_dcp; +struct apple_dcp_afkep { + struct apple_dcp *dcp;void *wq;void *bfr;void *debugfs_entry; + u64 bfr_dma;u32 bfr_size;u16 bfr_tag; +}; +struct apple_dcp { + bool retain_dma,quiescing,drm_retained,piodma_created;int connector_type,hdmi_hpd_irq; + struct device *dev;struct apple_crtc *crtc; + struct apple_connector *connector,*typec_connector; + struct platform_device *piodma;void *iommu_dom,*typec_mux,*shmem,*clk,*clk_194; + struct apple_rtkit *rtk; + struct apple_dcp_afkep *avep,*dptxep,*ibootep,*systemep,*dcpavservep; + struct work_struct bl_register_wq,bl_update_wq,vblank_wq; + struct delayed_work typec_reconnect_wq,typec_fabric_retrain_wq; +}; +static int dma_frees,domain_drops,pio_destroys,pio_puts,firmware_stops,queue_stops; +static int raw_rings,managed_rings,work_drains; +static int context_frees; +static void *ring; +static struct device *component; +static void dcp_comp_unbind(struct device *,struct device *,void *); +static void kfree(void *p) {assert(p);context_frees++;} +static void debugfs_remove_recursive(void *p) {(void)p;} +static void dev_warn(struct device *d,const char *fmt) {(void)d;(void)fmt;} +static void dev_err(struct device *d,const char *fmt,...) {(void)d;(void)fmt;} +static void *dev_get_drvdata(struct device *d) {return d->data;} +static void dev_set_drvdata(struct device *d,void *p) {d->data=p;} +static void drm_dev_unplug(struct drm_device *d) {d->unplugged=true;} +static void drm_atomic_helper_shutdown(struct drm_device *d) {assert(d->unplugged);} +static void component_unbind_all(struct device *m,void *p) {dcp_comp_unbind(component,m,p);} +static void iommu_device_unuse_default_domain(struct device *d) {domain_drops++;d->domain_users--;} +static void apple_rtkit_free(struct apple_rtkit *r) {r->rx_stopped=true;dma_frees++;} +static void apple_rtkit_free_retaining_buffers(struct apple_rtkit *r) {r->rx_stopped=true;} +static void of_platform_device_destroy(struct device *d,void *p) {(void)d;(void)p;pio_destroys++;} +static void put_device(struct device *d) {(void)d;pio_puts++;} +static void drm_dev_get(struct drm_device *d) {d->refs++;} +static void iomfb_queue_stop(struct apple_dcp *d) {assert(d->quiescing||!d->retain_dma);queue_stops++;} +static void destroy_workqueue(void *w) {assert(w);work_drains++;} +static void cancel_work_sync(struct work_struct *w) {(void)w;} +static void cancel_delayed_work_sync(struct delayed_work *w) {(void)w;} +static void disable_irq(int i) {(void)i;} +static void typec_mux_put(void *p) {(void)p;} +static void av_service_disconnect(struct apple_dcp *d) {(void)d;firmware_stops++;} +static void afk_shutdown(struct apple_dcp_afkep *e) {(void)e;firmware_stops++;} +static void iomfb_shutdown(struct apple_dcp *d) {(void)d;firmware_stops++;} +static void devm_clk_put(struct device *d,void *c) {(void)d;(void)c;} +static void trace_afk_getbuf(struct apple_dcp_afkep *e,u32 size,u16 tag) {(void)e;(void)size;(void)tag;} +static void afk_send(struct apple_dcp_afkep *e,u64 msg) {(void)e;(void)msg;} +static void *dma_alloc_coherent(struct device *d,u32 size,u64 *addr,int flags) { + (void)d;(void)flags;raw_rings++;ring=calloc(1,size);assert(ring);*addr=123;return ring; +} +static void *dmam_alloc_coherent(struct device *d,u32 size,u64 *addr,int flags) { + (void)d;(void)flags;managed_rings++;ring=calloc(1,size);assert(ring);*addr=123;return ring; +} +''' +main = r''' +int main(void) { + unsigned cases=0; + for(int live=0;live<2;live++)for(int created=0;created<2;created++)for(int linked=0;linked<2;linked++)for(int master=0;master<2;master++) { + struct device dev={.domain_users=1};struct platform_device pio={0}; + struct apple_drm_private apple={.drm={.refs=1}}; + struct device main={.data=&apple};struct apple_crtc crtc={.base={&apple.drm}}; + struct apple_rtkit rtk={0};struct apple_connector con={0}; + struct apple_dcp d={.retain_dma=live,.dev=&dev,.crtc=linked?&crtc:NULL, + .connector=&con,.piodma=&pio,.piodma_created=created,.iommu_dom=&dev, + .rtk=&rtk,.shmem=&dev,.connector_type=DRM_MODE_CONNECTOR_eDP}; + struct apple_dcp_afkep ep[5]={0}; + for(int i=0;i<5;i++)ep[i]=(struct apple_dcp_afkep){.dcp=&d,.wq=&dev}; + d.avep=&ep[0];d.dptxep=&ep[1];d.ibootep=&ep[2];d.systemep=&ep[3];d.dcpavservep=&ep[4];dev.data=&d; + dma_frees=domain_drops=pio_destroys=pio_puts=firmware_stops=queue_stops=0; + raw_rings=managed_rings=work_drains=context_frees=0; + afk_getbuf(&ep[0],1);assert(ep[0].bfr_size==64); + afk_getbuf(&ep[0],1);assert(raw_rings+managed_rings==1); + component=&dev; + if(master) {apple_drm_uninit(&main);assert(!main.data&&apple.drm.unplugged);} + else dcp_comp_unbind(&dev,&dev,NULL); + /* Component devres release runs even after an early unbind return. */ + dcp_release_rtkit(&d);assert(rtk.rx_stopped&&!d.rtk); + /* Driver removal also releases probe resources and its domain action. */ + dcp_release_dma_domain(&d); + if(live) { + assert(d.quiescing&&!dma_frees&&!domain_drops&&dev.domain_users==1); + assert(d.piodma==&pio&&d.iommu_dom==&dev&&!pio_destroys&&!pio_puts); + assert(raw_rings==1&&!managed_rings&&!firmware_stops&&work_drains==5); + assert(apple.drm.refs==(linked?2:1)); + dcp_comp_unbind(&dev,&dev,NULL);assert(apple.drm.refs==(linked?2:1)); + dcp_release_piodma_iommu_dev(&d);assert(!pio_destroys&&!pio_puts); + } else { + assert(dma_frees==1&&domain_drops==1&&dev.domain_users==0); + assert(!d.piodma&&!d.iommu_dom&&managed_rings==1&&!raw_rings); + assert(pio_destroys==created&&pio_puts==!created&&firmware_stops>0); + assert(apple.drm.refs==1); + } + for(int i=0;i<5;i++)afk_release_context(&ep[i]); + dcp_release_context(&d);assert(context_frees==(live?0:6)); + afk_quiesce(NULL);afk_quiesce((struct apple_dcp_afkep *)(intptr_t)-12); + /* Simulated machine reset: test memory can now be reclaimed. */ + free(ring);cases++; + } + /* A failed bind has no unbind callback: resource cleanup must close rebind. */ + { + struct device dev={0};struct platform_device pio={0};struct apple_rtkit rtk={0}; + struct apple_dcp d={.dev=&dev,.retain_dma=true,.piodma=&pio,.rtk=&rtk}; + assert(!d.quiescing);dcp_release_piodma_iommu_dev(&d); + assert(d.quiescing&&d.piodma==&pio);cases++; + d.quiescing=false;dcp_release_rtkit(&d); + assert(d.quiescing&&rtk.rx_stopped&&!d.rtk);cases++; + } + puts("PASS: 18 actual-source component, master-unwind, driver-resource-release and failed-bind cases; live DMA/domain/rings retained; legacy releases preserved"); + assert(cases==18);return 0; +} +''' +code = a.output / 'lifetime.c' +code.write_text(prefix + functions + main) +command = [a.cc, '-std=gnu11', '-O1', '-g', '-Wall', '-Wextra', '-Werror', + '-Wno-unused-parameter', '-fsanitize=address,undefined', str(code), + '-o', str(a.output / 'lifetime')] +subprocess.run(command, check=True) +result = subprocess.check_output([str(a.output / 'lifetime')], text=True) +(a.output / 'receipt.json').write_text(json.dumps(dict(command=command, result=result, + functions_sha256=hashlib.sha256(functions.encode()).hexdigest(), + scope='Native resource-release simulation; not a live firmware unbind test'), indent=2) + '\n') +print(result) From 0bda3e162630713d182162198c5d32a6aff7bbc9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:32:00 +1100 Subject: [PATCH 1099/2680] drm: apple: retry rejected H17P brightness presents with a bound A rejected brightness present cancels its reservation and leaves the policy dirty, but it receives no successful completion. The queue's completion-only reschedule condition therefore drops the update on a static display until another user request or frame arrives. Track a rejected backlight reservation separately from a completed present. After its channel drains, schedule up to three deferred retries at 100, 200 and 400 ms through the existing outbound queue. Preserve the last accepted surface, leave an exhausted target dirty, reset the budget on a new user request or successful completion, and cancel delayed work when the queue stops. Crashed and stopped queues never retry. Add a native regression that executes the actual failure, queue-advance, retry-work and completion functions: one rejected present is retried and accepted without another user request. Check the finite retry budget, new-target rearming and stopped, crashed and undrained-channel guards. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 2 + drivers/gpu/drm/apple/dcp_backlight.c | 11 ++ drivers/gpu/drm/apple/dcp_backlight.h | 21 ++- drivers/gpu/drm/apple/iomfb.c | 29 +++- tools/testing/apple/check-backlight-retry.py | 162 +++++++++++++++++++ tools/testing/apple/check-backlight.py | 2 +- 6 files changed, 222 insertions(+), 5 deletions(-) create mode 100644 tools/testing/apple/check-backlight-retry.py diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index f794cece99a0d..de7fe04ba880d 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -45,6 +45,7 @@ struct iomfb_transaction { void (*release)(struct iomfb_transaction *transaction); struct dcp_backlight_present backlight; bool backlight_reserved; + bool backlight_failed; bool brightness_only; bool completed; }; @@ -387,6 +388,7 @@ struct apple_dcp { struct task_struct *owner; struct work_struct work; struct delayed_work timeout; + struct delayed_work backlight_retry; unsigned long deadline; unsigned int queued; bool stopped; diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index c2a902de07773..0e4784509717f 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -224,6 +224,17 @@ bool dcp_backlight_pending(struct apple_dcp *dcp) return pending; } +unsigned int dcp_backlight_retry_delay(struct apple_dcp *dcp) +{ + unsigned long flags; + unsigned int delay; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + delay = dcp_bl_retry_delay(&dcp->backlight.state); + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return delay; +} + #define SCALE_FACTOR (1 << 10) static u32 interpolate(int val, int min, int max, u32 *tbl, size_t tbl_size) diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index e93ee7511c626..2ad4abb4acd37 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -15,6 +15,7 @@ struct dcp_backlight_state { u32 actual; u32 sent_nits; u64 sequence; + u8 retries; bool ready; bool controlled; bool dpms_off; @@ -89,6 +90,7 @@ static inline int dcp_bl_request(struct dcp_backlight_state *state, u32 nits, if (nits > state->maximum) return -ERANGE; state->controlled = true; + state->retries = 0; if (state->target != nits || state->core_blank != core_blank || state->suspended != suspended) { state->target = nits; @@ -104,6 +106,7 @@ static inline int dcp_bl_dpms(struct dcp_backlight_state *state, bool on) if (!state->ready) return -ENODATA; state->controlled = true; + state->retries = 0; if (state->dpms_off != !on) { state->dpms_off = !on; state->dirty = true; @@ -147,13 +150,26 @@ static inline bool dcp_bl_complete(struct dcp_backlight_state *state, if (!state->in_flight || sequence != state->sequence) return false; state->in_flight = false; - if (accepted) + if (accepted) { state->actual = state->sent_nits; - else + state->retries = 0; + } else { state->dirty = true; + if (state->retries < 4) + state->retries++; + } return true; } +/* Three deferred retries per user request; a permanent rejection stays dirty. */ +static inline unsigned int dcp_bl_retry_delay(const struct dcp_backlight_state *state) +{ + if (!state->ready || !state->dirty || state->in_flight || + !state->retries || state->retries > 3) + return 0; + return 100U << (state->retries - 1); +} + struct apple_dcp; /* @@ -173,5 +189,6 @@ int dcp_backlight_prepare(struct apple_dcp *dcp, bool have_surface, struct dcp_backlight_present *present); bool dcp_backlight_complete(struct apple_dcp *dcp, u64 sequence, bool accepted); bool dcp_backlight_pending(struct apple_dcp *dcp); +unsigned int dcp_backlight_retry_delay(struct apple_dcp *dcp); #endif /* __APPLE_DCP_BACKLIGHT_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index ec628f6b67fac..cc493903b240e 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -244,8 +244,20 @@ static void iomfb_queue_advance(struct apple_dcp *dcp) if (transaction->brightness_only) dcp->iomfb.backlight_queued = false; /* A new level requested during a successful present stays pending. */ - if (transaction->completed && dcp_backlight_pending(dcp)) - schedule_work(&dcp->bl_update_wq); + if (dcp_backlight_pending(dcp)) { + if (transaction->completed) { + schedule_work(&dcp->bl_update_wq); + } else if (transaction->backlight_failed) { + unsigned int delay = dcp_backlight_retry_delay(dcp); + + if (delay) + mod_delayed_work(system_wq, &dcp->iomfb.backlight_retry, + msecs_to_jiffies(delay)); + else + dev_warn_ratelimited(dcp->dev, + "backlight retry limit reached\n"); + } + } transaction->release(transaction); } if (iomfb_enqueue_opaque_x(dcp)) { @@ -340,12 +352,23 @@ void iomfb_opaque_x_reset_h17p(struct apple_dcp *dcp) atomic_set(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING); } +static void iomfb_backlight_retry(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, + iomfb.backlight_retry); + + if (!READ_ONCE(dcp->crashed) && !READ_ONCE(dcp->iomfb.stopped) && + dcp_backlight_pending(dcp)) + schedule_work(&dcp->bl_update_wq); +} + void iomfb_queue_init(struct apple_dcp *dcp) { mutex_init(&dcp->iomfb.lock); INIT_LIST_HEAD(&dcp->iomfb.pending); INIT_WORK(&dcp->iomfb.work, iomfb_queue_work); INIT_DELAYED_WORK(&dcp->iomfb.timeout, iomfb_queue_timeout); + INIT_DELAYED_WORK(&dcp->iomfb.backlight_retry, iomfb_backlight_retry); atomic_set(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING); } @@ -360,6 +383,7 @@ void iomfb_queue_stop(struct apple_dcp *dcp) mutex_unlock(&dcp->iomfb.lock); cancel_work_sync(&dcp->iomfb.work); cancel_delayed_work_sync(&dcp->iomfb.timeout); + cancel_delayed_work_sync(&dcp->iomfb.backlight_retry); } int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction) @@ -1106,6 +1130,7 @@ void iomfb_present_failed_h17p(struct apple_dcp *dcp) if (transaction && transaction->backlight_reserved) { dcp_backlight_complete(dcp, transaction->backlight.sequence, false); transaction->backlight_reserved = false; + transaction->backlight_failed = true; } } diff --git a/tools/testing/apple/check-backlight-retry.py b/tools/testing/apple/check-backlight-retry.py new file mode 100644 index 0000000000000..32d1fc40580f2 --- /dev/null +++ b/tools/testing/apple/check-backlight-retry.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Native regression for rejected brightness transactions and bounded retries. + +Executes actual reservation, failure, queue-drain, retry-work and completion +functions with mocked scheduling and transport. Does not submit firmware RPCs. +""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import subprocess + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument('--output', type=Path, required=True) +p.add_argument('--cc', default='clang') +a = p.parse_args() +root = Path(__file__).resolve().parents[3] +src = root / 'drivers/gpu/drm/apple' +a.output.mkdir(parents=True, exist_ok=True) + + +def function(file, name): + text = (src / file).read_text() + match = re.search(r'^\w[\w\s*]*?\b' + name + r'\(', text, re.M) + assert match, name + start = text.index('{', match.end()) + pos, depth = start + 1, 1 + while depth: + depth += (text[pos] == '{') - (text[pos] == '}') + pos += 1 + return text[match.start():pos] + + +policy = re.sub(r'^#include ]+>\n', '', + (src / 'dcp_backlight.h').read_text(), flags=re.M) +functions = '\n'.join(function(f, n) for f, n in ( + ('dcp_backlight.c', 'dcp_backlight_complete'), + ('dcp_backlight.c', 'dcp_backlight_pending'), + ('dcp_backlight.c', 'dcp_backlight_retry_delay'), + ('iomfb.c', 'iomfb_present_failed_h17p'), + ('iomfb.c', 'iomfb_present_complete_h17p'), + ('iomfb.c', 'iomfb_queue_advance'), ('iomfb.c', 'iomfb_backlight_retry'))) +prefix = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8;typedef uint32_t u32;typedef uint64_t u64; +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define lockdep_assert_held(x) ((void)(x)) +#define spin_lock_irqsave(x,f) do { (void)(x);(f)=0; } while(0) +#define spin_unlock_irqrestore(x,f) do { (void)(x);(void)(f); } while(0) +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +#define to_delayed_work(p) container_of(p,struct delayed_work,work) +#define msecs_to_jiffies(x) (x) +#define IOMFB_OPAQUE_X_WAITING 0 +#define IOMFB_OPAQUE_X_NEEDED 1 +struct work_struct {int id;};struct delayed_work {struct work_struct work;}; +struct list_head {int empty;}; +struct iomfb_transaction { + struct dcp_backlight_present backlight; + bool backlight_reserved,backlight_failed,brightness_only,completed; + void (*release)(struct iomfb_transaction *); +}; +struct apple_dcp { + bool crashed;void *dev;struct work_struct bl_update_wq,vblank_wq; + struct {int lock;struct dcp_backlight_state state;} backlight; + struct {bool pending;} present_state_h17p; + struct {int lock;bool stopped,backlight_queued;struct iomfb_transaction *active; + struct delayed_work timeout,backlight_retry;struct work_struct work; + struct list_head pending;int opaque_x_state; + } iomfb; +}; +static unsigned delay,scheduled,released,discarded;static bool idle=true; +static void *system_wq; +static bool iomfb_channels_idle(struct apple_dcp *d) {(void)d;return idle;} +static int iomfb_enqueue_opaque_x(struct apple_dcp *d) {(void)d;return 0;} +static void iomfb_discard_pending(struct apple_dcp *d) {(void)d;discarded++;} +static bool list_empty(struct list_head *l) {return l->empty;} +static void cancel_delayed_work(struct delayed_work *w) {(void)w;} +static void schedule_work(struct work_struct *w) {assert(w->id==1);scheduled++;} +static void mod_delayed_work(void *q,struct delayed_work *w,unsigned ms) { + (void)q;assert(w->work.id==2);delay=ms; +} +static void dev_warn_ratelimited(void *d,const char *msg) {(void)d;(void)msg;} +static void iomfb_scanout_complete_h17p(struct apple_dcp *d) {(void)d;} +static void atomic_cmpxchg(int *p,int old,int value) {if(*p==old)*p=value;} +static void release(struct iomfb_transaction *t) {(void)t;released++;} +''' +# The policy header defines structs before the fixture needs them. +prefix = prefix[:prefix.index('struct work_struct')] + policy + prefix[prefix.index('struct work_struct'):] +main = r''' +static void init(struct apple_dcp *d) { + u32 inherited; + memset(d,0,sizeof(*d));d->bl_update_wq.id=1;d->iomfb.backlight_retry.work.id=2; + assert(!dcp_bl_takeover_nits(509,36000,&inherited)&&inherited==36); + d->iomfb.pending.empty=1;assert(!dcp_bl_init(&d->backlight.state,509,true,inherited,false,0)); + assert(dcp_bl_seed(&d->backlight.state,36));assert(!dcp_bl_dpms(&d->backlight.state,true)); + assert(!dcp_bl_request(&d->backlight.state,140,false,false)); + delay=scheduled=released=discarded=0;idle=true; +} +static void reserve(struct apple_dcp *d,struct iomfb_transaction *t) { + memset(t,0,sizeof(*t));t->brightness_only=true;t->release=release; + assert(!dcp_bl_prepare(&d->backlight.state,true,&t->backlight)); + t->backlight_reserved=true;d->iomfb.active=t;d->iomfb.backlight_queued=true; +} +static void reject(struct apple_dcp *d) { + iomfb_present_failed_h17p(d);d->present_state_h17p.pending=false; + iomfb_queue_advance(d); +} +int main(void) { + struct apple_dcp d;struct iomfb_transaction t; + init(&d);reserve(&d,&t);reject(&d); + assert(delay==100&&!d.iomfb.active&&!d.iomfb.backlight_queued&&released==1); + assert(d.backlight.state.actual==36&&d.backlight.state.dirty&&!t.completed); + /* The deferred worker retries on retained pixels, with NO new user request. */ + iomfb_backlight_retry(&d.iomfb.backlight_retry.work);assert(scheduled==1); + reserve(&d,&t);assert(!iomfb_present_complete_h17p(&d));iomfb_queue_advance(&d); + assert(d.backlight.state.actual==140&&!d.backlight.state.dirty&&!d.backlight.state.retries); + init(&d); + for(unsigned i=0;i<4;i++) { + reserve(&d,&t);delay=0;reject(&d);assert(delay==(i<3?(100U< Date: Tue, 22 Sep 2026 23:17:56 -0700 Subject: [PATCH 1100/2680] drm/apple: drive Thunderbolt DP tunnels through DCP (t8103) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add apple_dcp_tb_dp_tunnel() for the Thunderbolt glue. It picks a free display pipeline for the port and routes it to the crossbar's dpin0/dpin1 output. It then out-of-band connects DPTX with the DFP port (1/2) and the DP IN role (attribute bit 0), and reports the display as attached. During link setup: - SetLinkRate starts or stops the ATC tunnel pixel clock. - After DidChangeLinkConfiguration the crossbar connection is brought up and DP IN is activated. The first time this is the mux selection; after that only the clocks are brought back. - Before WillChangeLinkConfiguration DP IN goes inactive and the crossbar clocks go down. Failures are logged and keep the crossbar down, so no stream is started without a clock. Tunnel state, the clock and the crossbar are handled under one lock, so teardown cannot race a link change, and DCP calls never block on the mux. The lock (and hpd_mutex, which the tunnel path also reaches before the DRM device binds) is initialized at probe. While a tunnel owns a Type-C port, alt mode state for it is not acted on; the tunnel teardown drops the recorded state and reconnects a fixed HDMI output if one is live. The Type-C DP alt mode path is unchanged: DFP port 0, role 0 and a strict check of DCP's connection reply. Tested on a MacBook Pro 13" M1 (j293) with a CalDigit TS3 Plus and a Kensington SD5560T: 3840x2160 at 60 Hz over HBR2 and HBR3, unplug and replug, monitor cable replug behind the dock, and a dock present at boot. [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/dcp-internal.h | 20 ++ drivers/gpu/drm/apple/dcp.c | 330 ++++++++++++++++++++++++++- drivers/gpu/drm/apple/dcp.h | 8 + drivers/gpu/drm/apple/dptxep.c | 46 +++- 4 files changed, 387 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 7fc0e1f04c9e0..85b49c1e1e40c 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -40,6 +40,12 @@ struct apple_dcp_typec_route { u32 dptx_phy; u32 mux_index; bool selected; + /* crossbar output actually selected: xbar (dpphy) or a Thunderbolt dpin */ + unsigned int tunnel_dpin; + struct mux_control *active_xbar; + bool tunnel; + /* tunnel: crossbar brought up (at DidChangeLinkConfiguration) */ + bool xbar_up; }; bool dcp_is_typec_output(struct apple_dcp *dcp); @@ -282,6 +288,20 @@ struct apple_dcp { u32 nr_typec_routes; bool phy_managed_by_typec; bool typec_cable_connected; + /* DPTX feeds a Thunderbolt DP IN adapter, not the Type-C PHY lanes */ + bool dptx_tunnel; + /* DFP port in the DPTX target: 0 = dpphy, 1 = dpin0, 2 = dpin1 */ + u8 dptx_dfp_port; + /* wakes/sleeps the Thunderbolt DP IN adapter from DCP Activate/Deactivate */ + int (*tb_dpin_set_active)(void *ctx, bool active); + void *tb_dpin_ctx; + /* + * Serializes the Thunderbolt DP IN callback and tunnel crossbar state + * between DCP apcalls and tunnel teardown; never held while waiting + * for DCP. + */ + struct mutex tb_lock; + bool tb_clock_ok; struct delayed_work typec_reconnect_wq; struct delayed_work typec_fabric_retrain_wq; u32 typec_reconnect_tries; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 1b664892c3ae8..be03495721fd8 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include #include #include @@ -115,7 +117,8 @@ static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) !dcp_typec_route_fixed_output_busy(route); } -static int dcp_typec_route_activate(struct apple_dcp_typec_route *route); +static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, + struct mux_control *xbar); static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port); @@ -135,7 +138,8 @@ static unsigned int dcp_typec_route_score(struct apple_dcp_typec_route *route) return drm_crtc_index(&dcp->crtc->base); } -static int dcp_typec_route_activate(struct apple_dcp_typec_route *route) +static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, + struct mux_control *xbar) { struct apple_dcp *dcp = route->dcp; int ret; @@ -154,7 +158,12 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route) dcp->fixed_route_selected = false; } - ret = mux_control_select(route->xbar, route->mux_index); + /* + * Thunderbolt DP IN: the crossbar connection may only be brought up + * once DCP has configured the link and the tunnel pixel clock runs, + * see dcp_tunnel_crossbar_up(). Just remember the output here. + */ + ret = xbar == route->xbar ? mux_control_select(xbar, route->mux_index) : 0; if (ret) { if (dcp->xbar) { int restore_ret; @@ -194,6 +203,15 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route) drm_crtc_mask(&dcp->crtc->base); } dcp->active_typec_route = route; + scoped_guard(mutex, &dcp->tb_lock) { + route->active_xbar = xbar; + route->tunnel = xbar != route->xbar; + route->xbar_up = !route->tunnel; + dcp->dptx_tunnel = route->tunnel; + /* crossbar controls are dpphy, dpin0, dpin1: same order as the DFP port */ + dcp->dptx_dfp_port = route->tunnel ? xbar - &route->xbar->chip->mux[0] : 0; + dcp->tb_clock_ok = false; + } route->selected = true; dev_info(dcp->dev, "allocated Type-C DPTX PHY %u\n", route->dptx_phy); @@ -203,12 +221,37 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route) static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) { struct apple_dcp *dcp = route->dcp; - int ret; + int ret = 0; - ret = mux_control_deselect(route->xbar); - if (ret) - return ret; + /* + * Under tb_lock so a DCP link (re)configuration can neither select the + * crossbar nor restart the tunnel pixel clock behind our back. + */ + scoped_guard(mutex, &dcp->tb_lock) { + if (!route->tunnel || route->xbar_up) + ret = mux_control_deselect(route->active_xbar ?: route->xbar); + if (ret) + return ret; + route->xbar_up = false; + + if (route->tunnel && dcp->phy) { + /* the tunnel pixel clock must not outlive the tunnel */ + typeof(&apple_atc_dp_tunnel_rate) stop = + symbol_get(apple_atc_dp_tunnel_rate); + if (stop) { + stop(dcp->phy, 0, 0); + symbol_put(apple_atc_dp_tunnel_rate); + } + } + route->active_xbar = NULL; + route->tunnel = false; + dcp->dptx_tunnel = false; + dcp->dptx_dfp_port = 0; + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + dcp->tb_clock_ok = false; + } route->selected = false; if (dcp->active_typec_route == route) dcp->active_typec_route = NULL; @@ -332,6 +375,9 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, port->applied_valid = true; if (!is_dp) { + /* a Thunderbolt/USB4 DP tunnel is torn down by its own path */ + if (port->owner && port->owner->tunnel) + return 0; if (port->owner) { struct apple_dcp *dcp = port->owner->dcp; @@ -354,6 +400,16 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, return 0; } + /* + * A Thunderbolt DP tunnel still owns the port (its teardown is on the + * way): don't act on or remember this state; the tunnel teardown drops + * what was recorded so the next update is applied. + */ + if (port->owner && port->owner->tunnel) { + port->applied_valid = false; + return 0; + } + if (!port->owner) { list_for_each_entry(candidate, &port->routes, port_link) { unsigned int score; @@ -370,7 +426,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (!best) return -EBUSY; - ret = dcp_typec_route_activate(best); + ret = dcp_typec_route_activate(best, best->xbar); if (ret) return ret; port->owner = best; @@ -397,6 +453,251 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, return 0; } +/* + * Crossbar connection up/down, looked up at runtime so appledrm does not + * require the crossbar driver to be built. + */ +static int dcp_dpxbar_link(struct mux_control *mux, bool up) +{ + typeof(&apple_dpxbar_link_up) fn; + int ret; + + fn = up ? symbol_get(apple_dpxbar_link_up) : + symbol_get(apple_dpxbar_link_down); + if (!fn) + return -ENOENT; + ret = fn(mux); + if (up) + symbol_put(apple_dpxbar_link_up); + else + symbol_put(apple_dpxbar_link_down); + return ret; +} + +/* DP IN adapter handshake through the thunderbolt glue; tb_lock held */ +static int dcp_tunnel_dpin_locked(struct apple_dcp *dcp, bool active) +{ + lockdep_assert_held(&dcp->tb_lock); + if (!dcp->tb_dpin_set_active) + return -ENODEV; + return dcp->tb_dpin_set_active(dcp->tb_dpin_ctx, active); +} + +/* + * Thunderbolt DP IN, from DCP's DidChangeLinkConfiguration once a link rate + * is set: bring the crossbar connection up (FIFO/PCLK/ATC enables) now that + * the tunnel pixel clock runs, and re-assert the DP IN adapter's + * DPTX_INACTIVE=0 afterwards. The first time this is the mux selection; after + * a re-link only the clocks are brought back (the mux selection and the ATC + * output enable are kept, see dcp_tunnel_crossbar_down()). + * Runs inside a DCP apcall: only tb_lock, which nobody holds across a DCP call. + */ +int dcp_tunnel_crossbar_up(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp->tb_lock); + route = dcp->active_typec_route; + if (!route || !route->tunnel || !route->active_xbar) + return -ENODEV; + if (!dcp->tb_clock_ok) { + dev_warn(dcp->dev, "no DP tunnel pixel clock, crossbar left down\n"); + return -EIO; + } + if (!route->xbar_up) { + /* never block a DCP call on the mux semaphore */ + ret = mux_control_try_select(route->active_xbar, route->mux_index); + if (!ret) + route->xbar_up = true; + } else { + ret = dcp_dpxbar_link(route->active_xbar, true); + } + if (ret) { + dev_warn(dcp->dev, "DP tunnel crossbar up failed: %d\n", ret); + return ret; + } + return dcp_tunnel_dpin_locked(dcp, true); +} + +/* + * Thunderbolt DP IN, from DCP's WillChangeLinkConfiguration on an established + * link: DP IN inactive, crossbar clocks down (mux selection and ATC output + * enable kept). DP IN goes active again in dcp_tunnel_crossbar_up(). + */ +int dcp_tunnel_crossbar_down(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + + guard(mutex)(&dcp->tb_lock); + route = dcp->active_typec_route; + if (!route || !route->tunnel || !route->xbar_up) + return 0; + dcp_tunnel_dpin_locked(dcp, false); + return dcp_dpxbar_link(route->active_xbar, false); +} + +/* + * Thunderbolt DP IN, from DCP's SetLinkRate: start (rate != 0) or stop the + * tunnel pixel clock. A stopped clock also takes the crossbar connection down. + */ +int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate) +{ + typeof(&apple_atc_dp_tunnel_rate) fn; + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp->tb_lock); + if (!dcp->dptx_tunnel) + return -ENODEV; + fn = symbol_get(apple_atc_dp_tunnel_rate); + if (!fn) { + dev_err(dcp->dev, "phy-apple-atc not loaded, no DP tunnel clock\n"); + return -ENOENT; + } + route = dcp->active_typec_route; + if (!link_rate && route && route->xbar_up) + dcp_dpxbar_link(route->active_xbar, false); + ret = fn(phy, 0, link_rate); + symbol_put(apple_atc_dp_tunnel_rate); + dcp->tb_clock_ok = !ret && link_rate; + if (ret) + dev_warn(dcp->dev, "DP tunnel pixel clock (rate 0x%x) failed: %d\n", + link_rate, ret); + return ret; +} + +/* Thunderbolt DP IN: DCP Activate/Deactivate */ +int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active) +{ + guard(mutex)(&dcp->tb_lock); + if (!dcp->dptx_tunnel) + return 0; + return dcp_tunnel_dpin_locked(dcp, active); +} + +/* + * Thunderbolt DP tunnels: the host router's DP IN adapters sit behind the + * crossbar's dpin0/dpin1 outputs of the port's ATC. When the Thunderbolt + * connection manager has set up a tunnel from one of them, route a free + * display pipeline there and tell DCP a display is attached, so it trains + * the link (and completes DPRX) through the tunnel. + */ +int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, + bool active, int (*set_active)(void *ctx, bool active), + void *ctx) +{ + struct apple_dcp_typec_port *port = NULL, *pos; + struct apple_dcp_typec_route *candidate, *best = NULL; + unsigned int best_score = UINT_MAX; + struct mux_control *ctl; + struct apple_dcp *dcp; + int ret; + + if (!connector_np || dpin > 1) + return -EINVAL; + + guard(mutex)(&dcp_typec_fabric_lock); + + list_for_each_entry(pos, &dcp_typec_ports, link) { + if (pos->connector_np == connector_np) { + port = pos; + break; + } + } + if (!port) + return -ENODEV; + + if (!active) { + if (!port->owner || !port->owner->tunnel || + port->owner->tunnel_dpin != dpin) + return 0; + dcp = port->owner->dcp; + if (port->hpd || dcp->typec_cable_connected) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + port->hpd = false; + ret = dcp_typec_route_deactivate(port->owner); + if (ret) { + /* the caller's context is going away regardless */ + scoped_guard(mutex, &dcp->tb_lock) { + dcp->dptx_tunnel = false; + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + } + return ret; + } + port->owner = NULL; + /* re-apply the next Type-C mux state in full */ + port->applied_valid = false; + if (dcp->hdmi_hpd && dcp->active && + gpiod_get_value_cansleep(dcp->hdmi_hpd)) + dcp_dptx_connect(dcp, 0); + return 0; + } + + if (port->owner) { + if (port->owner->tunnel && port->owner->tunnel_dpin == dpin) + return 0; + dev_warn(port->owner->dcp->dev, + "port already routed, not taking DP tunnel dpin%u\n", dpin); + return -EBUSY; + } + + list_for_each_entry(candidate, &port->routes, port_link) { + unsigned int score; + + if (!dcp_typec_route_available(candidate)) + continue; + score = dcp_typec_route_score(candidate); + if (score < best_score) { + best = candidate; + best_score = score; + } + } + if (!best) + return -EBUSY; + + /* The route's crossbar control is dpphy (0); dpin0/dpin1 are 1/2. */ + if (best->xbar != &best->xbar->chip->mux[0] || + best->xbar->chip->controllers < 3) + return -EOPNOTSUPP; + ctl = &best->xbar->chip->mux[1 + dpin]; + + dcp = best->dcp; + scoped_guard(mutex, &dcp->tb_lock) { + dcp->tb_dpin_set_active = set_active; + dcp->tb_dpin_ctx = ctx; + } + best->tunnel_dpin = dpin; + ret = dcp_typec_route_activate(best, ctl); + if (ret) { + scoped_guard(mutex, &dcp->tb_lock) { + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + } + return ret; + } + port->owner = best; + + dev_info(dcp->dev, "display routed to Thunderbolt DP tunnel dpin%u\n", dpin); + + /* + * The DP IN adapter may only be woken (DPTX_INACTIVE=0) while DCP + * drives the DPTX, i.e. from DCP's Activate call; waking it earlier + * hangs the machine. dptxep calls set_active back from Activate and + * Deactivate (set above, before the route became a tunnel). + */ + if (!dcp->typec_connector) + dev_warn(dcp->dev, "no Type-C connector for the DP tunnel\n"); + WRITE_ONCE(dcp->typec_cable_connected, true); + port->hpd = true; + if (dcp->typec_connector) + dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); + + return 0; +} +EXPORT_SYMBOL_GPL(apple_dcp_tb_dp_tunnel); + static struct apple_dcp_typec_port * dcp_typec_port_get(struct device_node *connector_np) { @@ -1007,7 +1308,8 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) reinit_completion(&dcp->dptxport[port].linkcfg_completion); dcp->dptxport[port].atcphy = dcp->phy; - ret = dptxport_validate_connection(dcp->dptxport[port].service, 0, + ret = dptxport_validate_connection(dcp->dptxport[port].service, + dcp->dptx_dfp_port, dcp->dptx_phy, dcp->dptx_die); if (ret) { dev_err(dcp->dev, @@ -1016,7 +1318,8 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) goto out_unlock; } - ret = dptxport_connect(dcp->dptxport[port].service, 0, + ret = dptxport_connect(dcp->dptxport[port].service, + dcp->dptx_dfp_port, dcp->dptx_phy, dcp->dptx_die, dcp_is_typec_output(dcp)); if (ret) { @@ -1788,7 +2091,6 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (dcp->index || dcp->dptx_phy || dcp->dptx_die) dev_info(dev, "DCP index:%u dptx target phy: %u dptx die: %u\n", dcp->index, dcp->dptx_phy, dcp->dptx_die); - mutex_init(&dcp->hpd_mutex); if (!show_notch) ret = of_property_read_u32(dev->of_node, "apple,notch-height", @@ -1967,6 +2269,12 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp->fw_compat = fw_compat; dcp->dev = dev; + /* + * Type-C and Thunderbolt routes can be activated as soon as they are + * registered below, before the DRM device binds. + */ + mutex_init(&dcp->hpd_mutex); + mutex_init(&dcp->tb_lock); dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); dcp->fixed_connector_type = dcp_connector_type_from_dt(dev->of_node); dcp->connector_type = dcp->fixed_connector_type; diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 2466aa9d13571..c42bc5cf015f5 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -69,6 +69,14 @@ void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message); int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port); int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); +struct apple_dcp; +struct phy; +/* Thunderbolt DP tunnels, called from the DPTX endpoint */ +int dcp_tunnel_crossbar_up(struct apple_dcp *dcp); +int dcp_tunnel_crossbar_down(struct apple_dcp *dcp); +int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate); +int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active); + int iomfb_start_rtkit(struct apple_dcp *dcp); void iomfb_shutdown(struct apple_dcp *dcp); /* rtkit message handler for IOMFB messages */ diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 525ba63744370..ee13005202786 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -86,8 +86,11 @@ int dptxport_validate_connection(struct apple_epic_service *service, u8 core, trace_dptxport_validate_connection(dptx, core, atc, die); + /* attributes: role (0 = direct PHY, 1 = Thunderbolt DP IN) | supportsHPD << 8 */ + u32 attrs = 0x100 | (service->ep->dcp->dptx_tunnel ? 1 : 0); + cmd.target = cpu_to_le32(target); - cmd.unk = cpu_to_le32(0x100); + cmd.unk = cpu_to_le32(attrs); ret = afk_service_call(service, 0, 12, &cmd, sizeof(cmd), 40, &resp, sizeof(resp), 40); if (ret) @@ -95,8 +98,14 @@ int dptxport_validate_connection(struct apple_epic_service *service, u8 core, if (le32_to_cpu(resp.target) != target) return -EINVAL; - if (le32_to_cpu(resp.unk) != 0x100) - return -EINVAL; + if (le32_to_cpu(resp.unk) != attrs) { + /* only the Thunderbolt DP IN role is allowed to differ */ + if (!service->ep->dcp->dptx_tunnel) + return -EINVAL; + dev_dbg(service->ep->dcp->dev, + "validate_connection: attrs reply 0x%x (sent 0x%x)\n", + le32_to_cpu(resp.unk), attrs); + } return 0; } @@ -106,7 +115,8 @@ int dptxport_connect(struct apple_epic_service *service, u8 core, u8 atc, { struct dptx_port *dptx = service->cookie; struct dcpdptx_connection_cmd cmd, resp; - u32 unk_field = supports_hpd ? DCPDPTX_REMOTE_PORT_SUPPORTS_HPD : 0; + u32 unk_field = (supports_hpd ? DCPDPTX_REMOTE_PORT_SUPPORTS_HPD : 0) | + (service->ep->dcp->dptx_tunnel ? 1 : 0); int ret; u32 target = FIELD_PREP(DCPDPTX_REMOTE_PORT_CORE, core) | FIELD_PREP(DCPDPTX_REMOTE_PORT_ATC, atc) | @@ -429,7 +439,17 @@ static int dptxport_call_set_link_rate(struct apple_epic_service *service, dptx->phy_ops.dp.link_rate = phy_link_rate; dptx->phy_ops.dp.set_rate = 1; - if (dptx->atcphy) { + if (dptx->atcphy && service->ep->dcp->dptx_tunnel) { + /* + * Thunderbolt DP IN: no lane setup, but the ATC must + * provide the tunnel pixel clock for this rate. A + * failure is logged and keeps the crossbar down (see + * dcp_tunnel_crossbar_up()), so no stream is started + * without a clock. + */ + dcp_tunnel_set_rate(service->ep->dcp, dptx->atcphy, + link_rate); + } else if (dptx->atcphy) { ret = phy_configure(dptx->atcphy, &dptx->phy_ops); if (ret) return ret; @@ -499,6 +519,9 @@ dptxport_call_activate(struct apple_epic_service *service, if (!dcp->phy_managed_by_typec) phy_set_mode_ext(dptx->atcphy, PHY_MODE_DP, dcp->index); + if (dcp->dptx_tunnel) + dcp_tunnel_dpin_activate(service->ep->dcp, true); + memcpy(reply, data, min(reply_size, data_size)); if (reply_size >= 4) memset(reply, 0, 4); @@ -517,6 +540,9 @@ dptxport_call_deactivate(struct apple_epic_service *service, if (!dcp->phy_managed_by_typec) phy_set_mode_ext(dptx->atcphy, PHY_MODE_INVALID, 0); + if (dcp->dptx_tunnel) + dcp_tunnel_dpin_activate(service->ep->dcp, false); + memcpy(reply, data, min(reply_size, data_size)); if (reply_size >= 4) memset(reply, 0, 4); @@ -529,13 +555,21 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, size_t reply_size) { struct dptx_port *dptx = service->cookie; + int ret; + trace_dptxport_apcall(dptx, idx, data_size); switch (idx) { case DPTX_APCALL_WILL_CHANGE_LINKG_CONFIG: + /* re-link on an established tunnel: take the crossbar connection down first */ + if (service->ep->dcp->dptx_tunnel && dptx->link_rate) + dcp_tunnel_crossbar_down(service->ep->dcp); return dptxport_call_will_change_link_config(service); case DPTX_APCALL_DID_CHANGE_LINK_CONFIG: - return dptxport_call_did_change_link_config(service); + ret = dptxport_call_did_change_link_config(service); + if (!ret && service->ep->dcp->dptx_tunnel && dptx->link_rate) + dcp_tunnel_crossbar_up(service->ep->dcp); + return ret; case DPTX_APCALL_GET_MAX_LINK_RATE: return dptxport_call_get_max_link_rate(service, reply, reply_size); From 74197cf2ab4cba070fa95522bdf9e991e2953984 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 19:53:55 +1100 Subject: [PATCH 1101/2680] drm: apple: guard the custom fbdev probe with its configuration Apple DRM can be enabled without DRM_FBDEV_EMULATION. In that configuration the DMA fbdev helper does not declare its probe function, but the custom Apple probe still calls it and fails to compile. Compile the custom probe and select it only when fbdev emulation is enabled. Otherwise use the DMA helper driver-ops macro, which supplies a NULL probe. The H17P ARGB console policy is unchanged when enabled. Fixes: 544eb67a1a17 ("drm: apple: use ARGB for H17P fbdev") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index d33b73de74e58..00dce5c373159 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -65,6 +65,7 @@ static int apple_drm_gem_dumb_create(struct drm_file *file_priv, return drm_gem_dma_dumb_create_internal(file_priv, drm, args); } +#ifdef CONFIG_DRM_FBDEV_EMULATION static int apple_drm_fbdev_probe(struct drm_fb_helper *helper, struct drm_fb_helper_surface_size *sizes) { @@ -85,10 +86,15 @@ static int apple_drm_fbdev_probe(struct drm_fb_helper *helper, return drm_fbdev_dma_driver_fbdev_probe(helper, sizes); } +#endif static const struct drm_driver apple_drm_driver = { DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(apple_drm_gem_dumb_create), +#ifdef CONFIG_DRM_FBDEV_EMULATION .fbdev_probe = apple_drm_fbdev_probe, +#else + DRM_FBDEV_DMA_DRIVER_OPS, +#endif .name = DRIVER_NAME, .desc = DRIVER_DESC, .major = 1, From dc98a7d11d1399c628bbd8d6ff67d380dfea84b3 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Thu, 24 Sep 2026 16:48:00 +0200 Subject: [PATCH 1102/2680] drm/apple: add timeout-aware AFK command variants Add afk_send_command_timeout()/afk_service_call_timeout(), which take an explicit reply timeout instead of the existing functions' hardcoded one second, and rework afk_send_command()/afk_service_call() as thin wrappers over them at the previous default. A Thunderbolt DP tunnel's AUX proxying round-trip for an HPD-assert reply can take longer than a direct-PHY connection's one-second budget, and the caller needs to widen just that one call rather than every AFK command in the driver. Add dev_warn() logging on the failure paths (handler error, timeout, non-zero retcode, reply magic/group/command mismatch) and a dev_dbg() on success, so a failed AFK round trip is diagnosable without needing to instrument individual call sites. Co-Authored-By: Claude Sonnet 5 --- drivers/gpu/drm/apple/afk.c | 57 ++++++++++++++++++++++++++++++++++--- drivers/gpu/drm/apple/afk.h | 8 ++++++ 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index b7f1629882692..9124f9e46647b 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -481,6 +481,10 @@ static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, payload + sizeof(*call), call_size, reply + sizeof(*call), call_size); if (ret) { + dev_warn(ep->dcp->dev, + "AFK[ep:%02x]: %s (chan:%u) apcall %u handler returned %d, sending NO reply\n", + ep->endpoint, service->ops->name, channel, + le32_to_cpu(call->type), ret); kfree(reply); return; } @@ -887,6 +891,16 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, int afk_send_command(struct apple_epic_service *service, u8 type, const void *payload, size_t payload_len, void *output, size_t output_len, u32 *retcode) +{ + return afk_send_command_timeout(service, type, payload, payload_len, + output, output_len, retcode, + MSEC_PER_SEC); +} + +int afk_send_command_timeout(struct apple_epic_service *service, u8 type, + const void *payload, size_t payload_len, + void *output, size_t output_len, u32 *retcode, + unsigned int timeout_ms) { struct epic_cmd cmd; void *rxbuf, *txbuf; @@ -949,7 +963,7 @@ int afk_send_command(struct apple_epic_service *service, u8 type, goto err_free_cmd; ret = wait_for_completion_timeout(&completion, - msecs_to_jiffies(MSEC_PER_SEC)); + msecs_to_jiffies(timeout_ms)); if (ret <= 0) { spin_lock_irqsave(&service->lock, flags); @@ -962,6 +976,10 @@ int afk_send_command(struct apple_epic_service *service, u8 type, service->cmds[idx].completion = NULL; service->cmds[idx].free_on_ack = true; spin_unlock_irqrestore(&service->lock, flags); + dev_warn(ep->dcp->dev, + "AFK[ep:%02x]: %s (chan:%u) command type 0x%x tag 0x%04x timed out after %u ms\n", + ep->endpoint, service->ops->name, service->channel, + type, tag, timeout_ms); return -ETIMEDOUT; } spin_unlock_irqrestore(&service->lock, flags); @@ -987,6 +1005,16 @@ int afk_send_command(struct apple_epic_service *service, u8 type, int afk_service_call(struct apple_epic_service *service, u16 group, u32 command, const void *data, size_t data_len, size_t data_pad, void *output, size_t output_len, size_t output_pad) +{ + return afk_service_call_timeout(service, group, command, data, data_len, + data_pad, output, output_len, + output_pad, MSEC_PER_SEC); +} + +int afk_service_call_timeout(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, unsigned int timeout_ms) { struct epic_service_call *call; void *bfr; @@ -1010,20 +1038,41 @@ int afk_service_call(struct apple_epic_service *service, u16 group, u32 command, memcpy(bfr + sizeof(*call), data, data_len); - ret = afk_send_command(service, EPIC_SUBTYPE_STD_SERVICE, bfr, bfr_len, - bfr, bfr_len, &retcode); - if (ret) + ret = afk_send_command_timeout(service, EPIC_SUBTYPE_STD_SERVICE, bfr, + bfr_len, bfr, bfr_len, &retcode, + timeout_ms); + if (ret) { + dev_warn(service->ep->dcp->dev, + "AFK[ep:%02x]: %s (chan:%u) service call group %u cmd %u failed to complete: %d\n", + service->ep->endpoint, service->ops->name, + service->channel, group, command, ret); goto out; + } if (retcode) { + dev_warn(service->ep->dcp->dev, + "AFK[ep:%02x]: %s (chan:%u) service call group %u cmd %u returned retcode 0x%08x (reply data_len %u)\n", + service->ep->endpoint, service->ops->name, + service->channel, group, command, retcode, + le32_to_cpu(call->data_len)); ret = -EINVAL; goto out; } if (le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || le16_to_cpu(call->group) != group || le32_to_cpu(call->command) != command) { + dev_warn(service->ep->dcp->dev, + "AFK[ep:%02x]: %s (chan:%u) service call reply mismatch: sent magic 0x%08x group %u cmd %u, got magic 0x%08x group %u cmd %u\n", + service->ep->endpoint, service->ops->name, + service->channel, EPIC_SERVICE_CALL_MAGIC, group, + command, le32_to_cpu(call->magic), + le16_to_cpu(call->group), le32_to_cpu(call->command)); ret = -EINVAL; goto out; } + dev_dbg(service->ep->dcp->dev, + "AFK[ep:%02x]: %s (chan:%u) service call group %u cmd %u ok (reply data_len %u)\n", + service->ep->endpoint, service->ops->name, service->channel, + group, command, le32_to_cpu(call->data_len)); retlen = le32_to_cpu(call->data_len); if (output_len < retlen) diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index a339c00a2a013..d63d26b79d457 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -198,7 +198,15 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, int afk_send_command(struct apple_epic_service *service, u8 type, const void *payload, size_t payload_len, void *output, size_t output_len, u32 *retcode); +int afk_send_command_timeout(struct apple_epic_service *service, u8 type, + const void *payload, size_t payload_len, + void *output, size_t output_len, u32 *retcode, + unsigned int timeout_ms); int afk_service_call(struct apple_epic_service *service, u16 group, u32 command, const void *data, size_t data_len, size_t data_pad, void *output, size_t output_len, size_t output_pad); +int afk_service_call_timeout(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, unsigned int timeout_ms); #endif From 92f3d9cf44366066bdaa6bea7abf568b66fd7d66 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 23 Sep 2026 11:38:28 -0700 Subject: [PATCH 1103/2680] drm/apple: re-read a placeholder Type-C EDID once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some USB-C display adapters answer the first hotplug with a 128-byte EDID whose product name is "Non-PnP" and whose preferred mode is 1024x768. The panel timings, including 3840x2160 at 120 Hz, show up only after HPD drops and returns. macOS ends up with the panel mode. The first Linux modeset was keeping the placeholder. When a Type-C output copies that EDID, pulse HPD once. A second placeholder is left as-is, so a display that really is that mode does not loop. The pulse is cancelled when the cable goes away. [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/dcp-internal.h | 5 ++ drivers/gpu/drm/apple/dcp.c | 77 ++++++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb.c | 4 +- 3 files changed, 85 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 85b49c1e1e40c..5b8de19b98821 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -49,6 +49,8 @@ struct apple_dcp_typec_route { }; bool dcp_is_typec_output(struct apple_dcp *dcp); +void dcp_retry_placeholder_edid(struct apple_dcp *dcp, + const struct drm_edid *drm_edid); struct dcpav_service_epic; @@ -211,6 +213,9 @@ struct apple_dcp { bool pending_hotplug; bool pending_hotplug_connected; bool valid_mode; + /* One HPD pulse after a placeholder EDID, per Type-C connection. */ + bool placeholder_retried; + struct delayed_work placeholder_edid_wq; bool use_timestamps; bool vrr_enabled; struct dcp_set_digital_out_mode_req mode; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index be03495721fd8..1d173559ae4b1 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -29,6 +29,7 @@ #include #include +#include #include #include #include @@ -1383,6 +1384,76 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) return ret; } +static bool dcp_edid_is_placeholder(const struct drm_edid *drm_edid) +{ + const u8 *raw = (const u8 *)drm_edid_raw(drm_edid); + static const u8 name[] = "Non-PnP"; + unsigned int i; + + if (!raw) + return false; + + /* EDID contains interior NUL bytes, so this cannot use strnstr(). */ + for (i = 0; i + sizeof(name) - 1 <= sizeof(struct edid); i++) { + if (!memcmp(raw + i, name, sizeof(name) - 1)) + return true; + } + + return false; +} + +void dcp_retry_placeholder_edid(struct apple_dcp *dcp, + const struct drm_edid *drm_edid) +{ + if (!dcp_is_typec_output(dcp) || dcp->placeholder_retried) + return; + if (!dcp_edid_is_placeholder(drm_edid)) + return; + + dcp->placeholder_retried = true; + schedule_delayed_work(&dcp->placeholder_edid_wq, msecs_to_jiffies(300)); +} + +static void dcp_placeholder_edid_work(struct work_struct *work) +{ + struct apple_dcp *dcp = + container_of(to_delayed_work(work), struct apple_dcp, + placeholder_edid_wq); + struct apple_epic_service *service; + int ret; + + if (!READ_ONCE(dcp->typec_cable_connected)) + return; + + mutex_lock(&dcp->hpd_mutex); + service = dcp->dptxport[0].connected ? dcp->dptxport[0].service : NULL; + mutex_unlock(&dcp->hpd_mutex); + if (!service) + return; + + /* + * Some adapters answer the first connection with a 1024x768 + * placeholder and publish the panel EDID only after HPD drops + * and returns. One pulse; a second placeholder is left alone. + */ + ret = dptxport_set_hpd(service, false); + if (ret) { + dev_info(dcp->dev, "placeholder EDID: HPD drop failed: %d\n", + ret); + return; + } + + msleep(1000); + + if (!READ_ONCE(dcp->typec_cable_connected)) + return; + + ret = dptxport_set_hpd(service, true); + if (ret) + dev_info(dcp->dev, "placeholder EDID: HPD assert failed: %d\n", + ret); +} + static void dcp_typec_reconnect_work(struct work_struct *work) { struct apple_dcp *dcp = @@ -1441,7 +1512,9 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) if (dcp_is_typec_output(dcp)) { WRITE_ONCE(dcp->typec_cable_connected, true); dcp->typec_reconnect_tries = 0; + dcp->placeholder_retried = false; cancel_delayed_work(&dcp->typec_reconnect_wq); + cancel_delayed_work(&dcp->placeholder_edid_wq); } ret = dcp_dptx_connect(dcp, port); @@ -1459,6 +1532,7 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) if (dcp_is_typec_output(dcp)) { WRITE_ONCE(dcp->typec_cable_connected, false); cancel_delayed_work(&dcp->typec_reconnect_wq); + cancel_delayed_work(&dcp->placeholder_edid_wq); } disconnected_hpd_event(dcp->connector); @@ -2230,6 +2304,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) cancel_work_sync(&dcp->bl_update_wq); } cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); cancel_work_sync(&dcp->vblank_wq); @@ -2284,6 +2359,8 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp->fixed_dptx_phy = dcp->dptx_phy; INIT_DELAYED_WORK(&dcp->typec_reconnect_wq, dcp_typec_reconnect_work); + INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, + dcp_placeholder_edid_work); INIT_DELAYED_WORK(&dcp->typec_fabric_retrain_wq, dcp_typec_retrain_work); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 5b78f77dc7401..5d255adcdd243 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -448,8 +448,10 @@ int dcp_get_modes(struct drm_connector *connector) apple_connector->drm_edid = edid; } } - if (dcp->nr_modes && apple_connector->drm_edid) + if (dcp->nr_modes && apple_connector->drm_edid) { drm_edid_connector_update(connector, apple_connector->drm_edid); + dcp_retry_placeholder_edid(dcp, apple_connector->drm_edid); + } /* * An internal panel has no EDID, so nothing fills in the refresh range From 55832c2e3dd800b3af474d8ffcc313461d1fca10 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Thu, 24 Sep 2026 16:48:11 +0200 Subject: [PATCH 1104/2680] drm/apple: route Thunderbolt DP tunnels through the Type-C crossbar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add core routing support for a USB4/Thunderbolt DisplayPort tunnel arriving behind a Thunderbolt hub, as opposed to a direct Type-C DP alt-mode connection. struct apple_dcp_typec_route grows a tunnel flag, an active_xbar pointer distinct from the route's own crossbar control, a tunnel_dpin index and an xbar_up flag, so dcp_typec_route_activate()/ deactivate() can be handed the crossbar control to actually use: for a tunnel this is a dpin0/dpin1 controller resolved at tunnel-creation time rather than the route's own dpphy control, and the mux selection for it is deferred until the tunnel pixel clock is confirmed running rather than issued immediately. Add apple_dcp_tb_dp_tunnel(), the entry point Thunderbolt calls once it has stood up a DP IN tunnel: it finds a free Type-C route on the target port and activates it as a tunnel. A Type-C-only pipeline (no fixed output of its own) never completes link training for a tunneled target on this hardware -- DCP firmware accepts request_display but never issues the follow-up apcall, despite byte-identical connect parameters, a firmware-internal decision on that coprocessor instance with no driver-source asymmetry to explain it. A pipeline with a fixed output of its own reliably reaches a full AUX/DPCD link, so route scoring prefers it unconditionally for a tunnel route on this hardware. Add dcp_tunnel_crossbar_up()/down()/set_rate()/dpin_activate(), the hooks the DPTX endpoint uses to keep the crossbar clocks and the DP IN adapter's active state in step with DCP's link (re)configuration, serialized by a new per-DCP tb_lock. Make the Type-C alt-mode mux callback leave a tunnel-owned port alone -- a tunnel is torn down by apple_dcp_tb_dp_tunnel() itself, not by an alt-mode notification, and reapplying a stale alt-mode state on top of one would fight it. Thread the DFP port selected by route activation through dcp_dptx_connect() into the DPTX validate/connect calls, which previously always targeted DFP port 0 -- this is what makes a tunneled connection train against the Thunderbolt DP IN adapter DCP was actually told to use, rather than the direct PHY. Widen DPTX_CONNECT_TIMEOUT from 2s to 8s and the Type-C HPD-assert wait to 8s, and drop DPTX_RECONNECT_RETRIES from 5 to 1: a tunneled connection's handshake round-trip through the DP IN adapter can outlast a direct-PHY connection's budget. Add dptxport_set_hpd_timeout() for that HPD-assert call, so a caller can widen the AFK reply timeout, since a tunnel's AUX proxying round-trip can be slower than a direct PHY's. [Chris Kearney: moved dptxport_set_hpd_timeout() here from the next commit, which added it after its first caller, so each commit builds] [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Co-Authored-By: Claude Sonnet 5 --- drivers/gpu/drm/apple/dcp-internal.h | 34 ++ drivers/gpu/drm/apple/dcp.c | 451 +++++++++++++++++++++++++-- drivers/gpu/drm/apple/dcp.h | 8 + drivers/gpu/drm/apple/dptxep.c | 18 +- drivers/gpu/drm/apple/dptxep.h | 3 + 5 files changed, 489 insertions(+), 25 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 7fc0e1f04c9e0..a8db856f8f901 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -39,10 +39,26 @@ struct apple_dcp_typec_route { struct typec_mux_dev *typec_mux; u32 dptx_phy; u32 mux_index; + u32 typec_index; bool selected; + /* Ported from aurora-silicon/linux#8: this route currently feeds a + * Thunderbolt DP IN adapter rather than the crossbar's dpphy output. + */ + bool tunnel; + /* Which crossbar output is actually selected: route->xbar (dpphy) or + * a Thunderbolt dpin controller computed at tunnel-creation time. + */ + struct mux_control *active_xbar; + /* Which DP IN adapter (0/1) a tunnel on this route is bound to. */ + unsigned int tunnel_dpin; + /* Tunnel only: crossbar connection brought up (at + * DidChangeLinkConfiguration, once the tunnel pixel clock runs). + */ + bool xbar_up; }; bool dcp_is_typec_output(struct apple_dcp *dcp); +bool dcp_is_usb4_output(struct apple_dcp *dcp); struct dcpav_service_epic; @@ -286,6 +302,24 @@ struct apple_dcp { struct delayed_work typec_fabric_retrain_wq; u32 typec_reconnect_tries; + /* Ported from aurora-silicon/linux#8. */ + /* DPTX feeds a Thunderbolt DP IN adapter, not the Type-C PHY lanes. */ + bool dptx_tunnel; + /* DFP port in the DPTX target: 0 = dpphy, 1 = dpin0, 2 = dpin1. */ + u8 dptx_dfp_port; + /* Wakes/sleeps the Thunderbolt DP IN adapter from DCP Activate/ + * Deactivate. Set by apple_dcp_tb_dp_tunnel(), cleared by + * dcp_typec_route_deactivate(). + */ + int (*tb_dpin_set_active)(void *ctx, bool active); + void *tb_dpin_ctx; + /* Serializes the Thunderbolt DP IN callback and tunnel crossbar state + * between DCP apcalls and tunnel teardown; never held while waiting + * for DCP. + */ + struct mutex tb_lock; + bool tb_clock_ok; + struct gpio_desc *hdmi_hpd; struct gpio_desc *hdmi_pwren; struct gpio_desc *dp2hdmi_pwren; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 1b664892c3ae8..6bb664ef3ce2b 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -9,27 +9,38 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include #include #include #include +#include #include #include +#include #include +#include +#include +#include +#include +#include #include #include #include #include +#include #include #include #include +#include #include #include #include @@ -48,6 +59,7 @@ #define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) static bool show_notch; + module_param(show_notch, bool, 0644); MODULE_PARM_DESC(show_notch, "Use the full display height and shows the notch"); @@ -87,6 +99,11 @@ bool dcp_is_typec_output(struct apple_dcp *dcp) dcp->fixed_connector_type == DRM_MODE_CONNECTOR_USB; } +bool dcp_is_usb4_output(struct apple_dcp *dcp) +{ + return dcp->active_typec_route && dcp->active_typec_route->tunnel; +} + static bool dcp_typec_route_is_dp(const struct typec_mux_state *state) { return state->alt && state->alt->svid == USB_TYPEC_DP_SID && @@ -115,7 +132,8 @@ static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) !dcp_typec_route_fixed_output_busy(route); } -static int dcp_typec_route_activate(struct apple_dcp_typec_route *route); +static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, + struct mux_control *xbar); static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port); @@ -135,7 +153,8 @@ static unsigned int dcp_typec_route_score(struct apple_dcp_typec_route *route) return drm_crtc_index(&dcp->crtc->base); } -static int dcp_typec_route_activate(struct apple_dcp_typec_route *route) +static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, + struct mux_control *xbar) { struct apple_dcp *dcp = route->dcp; int ret; @@ -154,7 +173,13 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route) dcp->fixed_route_selected = false; } - ret = mux_control_select(route->xbar, route->mux_index); + /* + * Thunderbolt DP IN: the crossbar connection may only be brought up + * once DCP has configured the link and the tunnel pixel clock runs + * (see dcp_tunnel_crossbar_up()). Just remember the output here. + * Ported from aurora-silicon/linux#8. + */ + ret = xbar == route->xbar ? mux_control_select(xbar, route->mux_index) : 0; if (ret) { if (dcp->xbar) { int restore_ret; @@ -194,6 +219,17 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route) drm_crtc_mask(&dcp->crtc->base); } dcp->active_typec_route = route; + scoped_guard(mutex, &dcp->tb_lock) { + route->active_xbar = xbar; + route->tunnel = xbar != route->xbar; + route->xbar_up = !route->tunnel; + dcp->dptx_tunnel = route->tunnel; + /* Crossbar controls are dpphy, dpin0, dpin1: same order as + * the DFP port. */ + dcp->dptx_dfp_port = route->tunnel ? + xbar - &route->xbar->chip->mux[0] : 0; + dcp->tb_clock_ok = false; + } route->selected = true; dev_info(dcp->dev, "allocated Type-C DPTX PHY %u\n", route->dptx_phy); @@ -203,11 +239,38 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route) static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) { struct apple_dcp *dcp = route->dcp; - int ret; + int ret = 0; - ret = mux_control_deselect(route->xbar); - if (ret) - return ret; + /* + * Under tb_lock so a DCP link (re)configuration can neither select + * the crossbar nor restart the tunnel pixel clock behind our back. + * Ported from aurora-silicon/linux#8. + */ + scoped_guard(mutex, &dcp->tb_lock) { + if (!route->tunnel || route->xbar_up) + ret = mux_control_deselect(route->active_xbar ?: route->xbar); + if (ret) + return ret; + route->xbar_up = false; + + if (route->tunnel && dcp->phy) { + /* The tunnel pixel clock must not outlive the tunnel. */ + typeof(&apple_atc_dp_tunnel_rate) stop = + symbol_get(apple_atc_dp_tunnel_rate); + + if (stop) { + stop(dcp->phy, 0, 0); + symbol_put(apple_atc_dp_tunnel_rate); + } + } + route->active_xbar = NULL; + route->tunnel = false; + dcp->dptx_tunnel = false; + dcp->dptx_dfp_port = 0; + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + dcp->tb_clock_ok = false; + } route->selected = false; if (dcp->active_typec_route == route) @@ -295,6 +358,275 @@ static void dcp_typec_retrain_active_routes(void) } } +/* + * Thunderbolt DP tunnels, ported from aurora-silicon/linux#8. Crossbar + * connection up/down, looked up at runtime so appledrm does not require the + * crossbar driver to be built. + */ +static int dcp_dpxbar_link(struct mux_control *mux, bool up) +{ + typeof(&apple_dpxbar_link_up) fn; + int ret; + + fn = up ? symbol_get(apple_dpxbar_link_up) : + symbol_get(apple_dpxbar_link_down); + if (!fn) + return -ENOENT; + ret = fn(mux); + if (up) + symbol_put(apple_dpxbar_link_up); + else + symbol_put(apple_dpxbar_link_down); + return ret; +} + +/* DP IN adapter handshake through the thunderbolt glue; tb_lock held. */ +static int dcp_tunnel_dpin_locked(struct apple_dcp *dcp, bool active) +{ + lockdep_assert_held(&dcp->tb_lock); + if (!dcp->tb_dpin_set_active) + return -ENODEV; + return dcp->tb_dpin_set_active(dcp->tb_dpin_ctx, active); +} + +/* + * Thunderbolt DP IN, from DCP's DidChangeLinkConfiguration once a link rate + * is set: bring the crossbar connection up (FIFO/PCLK/ATC enables) now that + * the tunnel pixel clock runs, and re-assert the DP IN adapter's + * DPTX_INACTIVE=0 afterwards. The first time this is the mux selection; + * after a re-link only the clocks are brought back (the mux selection and + * the ATC output enable are kept, see dcp_tunnel_crossbar_down()). Runs + * inside a DCP apcall: only tb_lock, which nobody holds across a DCP call. + */ +int dcp_tunnel_crossbar_up(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp->tb_lock); + route = dcp->active_typec_route; + if (!route || !route->tunnel || !route->active_xbar) + return -ENODEV; + if (!dcp->tb_clock_ok) { + dev_warn(dcp->dev, "no DP tunnel pixel clock, crossbar left down\n"); + return -EIO; + } + if (!route->xbar_up) { + /* Never block a DCP call on the mux semaphore. */ + ret = mux_control_try_select(route->active_xbar, route->mux_index); + if (!ret) + route->xbar_up = true; + } else { + ret = dcp_dpxbar_link(route->active_xbar, true); + } + if (ret) { + dev_warn(dcp->dev, "DP tunnel crossbar up failed: %d\n", ret); + return ret; + } + return dcp_tunnel_dpin_locked(dcp, true); +} + +/* + * Thunderbolt DP IN, from DCP's WillChangeLinkConfiguration on an + * established link: DP IN inactive, crossbar clocks down (mux selection and + * ATC output enable kept). DP IN goes active again in + * dcp_tunnel_crossbar_up(). + */ +int dcp_tunnel_crossbar_down(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + + guard(mutex)(&dcp->tb_lock); + route = dcp->active_typec_route; + if (!route || !route->tunnel || !route->xbar_up) + return 0; + dcp_tunnel_dpin_locked(dcp, false); + return dcp_dpxbar_link(route->active_xbar, false); +} + +/* + * Thunderbolt DP IN, from DCP's SetLinkRate: start (rate != 0) or stop the + * tunnel pixel clock. A stopped clock also takes the crossbar connection + * down. + */ +int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate) +{ + typeof(&apple_atc_dp_tunnel_rate) fn; + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp->tb_lock); + if (!dcp->dptx_tunnel) + return -ENODEV; + fn = symbol_get(apple_atc_dp_tunnel_rate); + if (!fn) { + dev_err(dcp->dev, "phy-apple-atc not loaded, no DP tunnel clock\n"); + return -ENOENT; + } + route = dcp->active_typec_route; + if (!link_rate && route && route->xbar_up) + dcp_dpxbar_link(route->active_xbar, false); + ret = fn(phy, 0, link_rate); + symbol_put(apple_atc_dp_tunnel_rate); + dcp->tb_clock_ok = !ret && link_rate; + if (ret) + dev_warn(dcp->dev, "DP tunnel pixel clock (rate 0x%x) failed: %d\n", + link_rate, ret); + return ret; +} + +/* Thunderbolt DP IN: DCP Activate/Deactivate. */ +int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active) +{ + guard(mutex)(&dcp->tb_lock); + if (!dcp->dptx_tunnel) + return 0; + return dcp_tunnel_dpin_locked(dcp, active); +} + +/* + * Thunderbolt DP tunnels: the host router's DP IN adapters sit behind the + * crossbar's dpin0/dpin1 outputs of the port's ATC. When the Thunderbolt + * connection manager has set up a tunnel from one of them, route a free + * display pipeline there and tell DCP a display is attached, so it trains + * the link (and completes DPRX) through the tunnel. + */ +int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, + bool active, int (*set_active)(void *ctx, bool active), + void *ctx) +{ + struct apple_dcp_typec_port *port = NULL, *pos; + struct apple_dcp_typec_route *candidate, *best = NULL; + unsigned int best_score = UINT_MAX; + struct mux_control *ctl; + struct apple_dcp *dcp; + int ret; + + if (!connector_np || dpin > 1) + return -EINVAL; + + guard(mutex)(&dcp_typec_fabric_lock); + + list_for_each_entry(pos, &dcp_typec_ports, link) { + if (pos->connector_np == connector_np) { + port = pos; + break; + } + } + if (!port) + return -ENODEV; + + if (!active) { + if (!port->owner || !port->owner->tunnel || + port->owner->tunnel_dpin != dpin) + return 0; + dcp = port->owner->dcp; + if (port->hpd || dcp->typec_cable_connected) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + port->hpd = false; + ret = dcp_typec_route_deactivate(port->owner); + if (ret) { + /* The caller's context is going away regardless. */ + scoped_guard(mutex, &dcp->tb_lock) { + dcp->dptx_tunnel = false; + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + } + return ret; + } + port->owner = NULL; + /* Re-apply the next Type-C mux state in full. */ + port->applied_valid = false; + if (dcp->hdmi_hpd && dcp->active && + gpiod_get_value_cansleep(dcp->hdmi_hpd)) + dcp_dptx_connect(dcp, 0); + return 0; + } + + if (port->owner) { + if (port->owner->tunnel && port->owner->tunnel_dpin == dpin) + return 0; + dev_warn(port->owner->dcp->dev, + "port already routed, not taking DP tunnel dpin%u\n", dpin); + return -EBUSY; + } + + list_for_each_entry(candidate, &port->routes, port_link) { + unsigned int score; + + if (!dcp_typec_route_available(candidate)) + continue; + score = dcp_typec_route_score(candidate); + /* + * j416s-specific, not in the reference (t8103 has a single + * dcpext): prefer a pipeline with a fixed output of its own + * (dcpext0) for a Type-C tunnel route. dcpext1 (Type-C only, + * no fixed output of its own) never completes link training + * for a tunneled target on this hardware: DCP firmware + * accepts request_display but never issues another apcall, + * on every attempt, even though the driver-issued connect + * parameters are byte-identical between the two pipelines. + * Nothing in this driver's source explains the difference, + * so this is a firmware-internal decision on the dcpext1 + * coprocessor instance, not something fixable here. dcpext0, + * forced onto the same physical port and tunnel, reaches a + * full AUX/DPCD link (DPRX_DONE=1) and a working picture, so + * it is preferred unconditionally for a tunnel route on this + * hardware. + */ + if (!candidate->dcp->fixed_phy) + score += 100; + if (score < best_score) { + best = candidate; + best_score = score; + } + } + if (!best) + return -EBUSY; + + /* The route's crossbar control is dpphy (0); dpin0/dpin1 are 1/2. */ + if (best->xbar != &best->xbar->chip->mux[0] || + best->xbar->chip->controllers < 3) + return -EOPNOTSUPP; + ctl = &best->xbar->chip->mux[1 + dpin]; + + dcp = best->dcp; + scoped_guard(mutex, &dcp->tb_lock) { + dcp->tb_dpin_set_active = set_active; + dcp->tb_dpin_ctx = ctx; + } + best->tunnel_dpin = dpin; + ret = dcp_typec_route_activate(best, ctl); + if (ret) { + scoped_guard(mutex, &dcp->tb_lock) { + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + } + return ret; + } + port->owner = best; + + dev_info(dcp->dev, "display routed to Thunderbolt DP tunnel dpin%u\n", dpin); + + /* + * The DP IN adapter may only be woken (DPTX_INACTIVE=0) while DCP + * drives the DPTX, i.e. from DCP's Activate call; waking it earlier + * hangs the machine (confirmed independently by this project's own + * AFK ordered-workqueue deadlock finding). dptxep calls set_active + * back from Activate and Deactivate (set above, before the route + * became a tunnel). + */ + if (!dcp->typec_connector) + dev_warn(dcp->dev, "no Type-C connector for the DP tunnel\n"); + WRITE_ONCE(dcp->typec_cable_connected, true); + port->hpd = true; + if (dcp->typec_connector) + dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); + + return 0; +} +EXPORT_SYMBOL_GPL(apple_dcp_tb_dp_tunnel); + static int dcp_typec_route_set(struct typec_mux_dev *mux, struct typec_mux_state *state) { @@ -331,7 +663,22 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, port->applied_conf = dp_conf; port->applied_valid = true; + dev_info(route->dcp->dev, + "typec mux set typec%u alt=%p svid=%04x mode=%lu dp=%d usb4=%d tbt=%d\n", + route->typec_index, state->alt, + state->alt ? state->alt->svid : 0, state->mode, is_dp, + state->mode == TYPEC_MODE_USB4, + state->alt && state->alt->svid == USB_TYPEC_TBT_SID); + if (!is_dp) { + /* + * A Thunderbolt/USB4 DP tunnel is torn down by its own path + * (apple_dcp_tb_dp_tunnel()), not by a Type-C mux notification. + * Ported from aurora-silicon/linux#8. + */ + if (port->owner && port->owner->tunnel) + return 0; + if (port->owner) { struct apple_dcp *dcp = port->owner->dcp; @@ -348,9 +695,18 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, gpiod_get_value_cansleep(dcp->hdmi_hpd)) dcp_dptx_connect(dcp, 0); } + dcp_typec_retrain_active_routes(); + return 0; + } - if (state->mode == TYPEC_MODE_USB4) - dcp_typec_retrain_active_routes(); + /* + * A Thunderbolt DP tunnel still owns the port (its teardown is on the + * way): don't act on or remember this state; the tunnel teardown + * drops what was recorded so the next update is applied. Ported from + * aurora-silicon/linux#8. + */ + if (port->owner && port->owner->tunnel) { + port->applied_valid = false; return 0; } @@ -370,7 +726,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (!best) return -EBUSY; - ret = dcp_typec_route_activate(best); + ret = dcp_typec_route_activate(best, best->xbar); if (ret) return ret; port->owner = best; @@ -588,6 +944,7 @@ static int dcp_register_typec_routes(struct apple_dcp *dcp) route = &dcp->typec_routes[dcp->nr_typec_routes]; route->dcp = dcp; + route->typec_index = route_index; INIT_LIST_HEAD(&route->port_link); route->phy = devm_phy_get(dev, name); if (IS_ERR(route->phy)) @@ -599,6 +956,13 @@ static int dcp_register_typec_routes(struct apple_dcp *dcp) return dev_err_probe(dev, PTR_ERR(route->xbar), "%pOF: failed to get display crossbar\n", route_np); + /* + * No per-route USB4/dpin crossbar lookup here any more: + * apple_dcp_tb_dp_tunnel() computes the dpin0/dpin1 crossbar + * controller dynamically from route->xbar->chip->mux[1+dpin] + * at tunnel-creation time. Ported from aurora-silicon/linux#8. + */ + ret = of_property_read_u32_index(dev->of_node, "apple,typec-mux-indices", route_index, &route->mux_index); if (ret) @@ -976,9 +1340,17 @@ bool dcp_has_typec_routes(struct platform_device *pdev) return dcp->nr_typec_routes; } -#define DPTX_CONNECT_TIMEOUT msecs_to_jiffies(2000) +/* + * DCP's tunnel handshake round-trip through the Thunderbolt DP IN + * adapter can take longer than a direct-PHY connection, so both the + * connect wait and the reconnect policy need more headroom than a + * direct connection does. An 8s connect timeout with a single + * reconnect retry is the tested, working configuration for a + * tunneled connection. + */ +#define DPTX_CONNECT_TIMEOUT msecs_to_jiffies(8000) #define DPTX_RECONNECT_DELAY msecs_to_jiffies(1000) -#define DPTX_RECONNECT_RETRIES 5 +#define DPTX_RECONNECT_RETRIES 1 static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) { @@ -989,9 +1361,9 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) return -ENODEV; } dev_info(dcp->dev, - "%s(port=%d) target=%u:%u typec=%d route=%s conn_type=%d connected=%d\n", + "%s(port=%d) target=%u:%u dfp=%u typec=%d route=%s conn_type=%d connected=%d\n", __func__, port, dcp->dptx_die, dcp->dptx_phy, - dcp_is_typec_output(dcp), + dcp->dptx_dfp_port, dcp_is_typec_output(dcp), dcp->active_typec_route ? "borrowed" : "fixed", dcp->connector_type, dcp->dptxport[port].connected); @@ -1006,8 +1378,16 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) goto out_unlock; reinit_completion(&dcp->dptxport[port].linkcfg_completion); + /* + * Ported from aurora-silicon/linux#8: a Thunderbolt DP tunnel uses + * the exact same connect path as a direct alt-mode PHY. The only + * differences are dcp->dptx_dfp_port (0 = dpphy, 1/2 = dpin0/dpin1, + * set by dcp_typec_route_activate()) and dcp->phy already being the + * route's own ATC PHY rather than a separate shared lpdptxphy. + */ dcp->dptxport[port].atcphy = dcp->phy; - ret = dptxport_validate_connection(dcp->dptxport[port].service, 0, + ret = dptxport_validate_connection(dcp->dptxport[port].service, + dcp->dptx_dfp_port, dcp->dptx_phy, dcp->dptx_die); if (ret) { dev_err(dcp->dev, @@ -1016,9 +1396,10 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) goto out_unlock; } - ret = dptxport_connect(dcp->dptxport[port].service, 0, + ret = dptxport_connect(dcp->dptxport[port].service, + dcp->dptx_dfp_port, dcp->dptx_phy, dcp->dptx_die, - dcp_is_typec_output(dcp)); + dcp_is_typec_output(dcp)); if (ret) { dev_err(dcp->dev, "dcp_dptx_connect: failed to connect DPTX target %u:%u: %d\n", @@ -1035,7 +1416,17 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) } dcp->dptxport[port].connected = true; if (dcp_is_typec_output(dcp)) { - ret = dptxport_set_hpd(dcp->dptxport[port].service, true); + /* + * DCP's reply to this call can take longer than the normal + * 1000ms budget when the connection is a Thunderbolt DP + * tunnel, so widen the timeout to 8000ms for a Type-C/ + * tunneled output. This is independent of + * tb_dp_wait_dprx()'s own 12000ms poll for DPRX (generic + * thunderbolt/tunnel.c, started at tunnel-up), which is + * unaffected by this call either way. + */ + ret = dptxport_set_hpd_timeout(dcp->dptxport[port].service, + true, 8000); if (ret) { dev_err(dcp->dev, "dcp_dptx_connect: failed to assert Type-C HPD: %d\n", @@ -1046,8 +1437,9 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) } mutex_unlock(&dcp->hpd_mutex); + ret = wait_for_completion_timeout(&dcp->dptxport[port].linkcfg_completion, - DPTX_CONNECT_TIMEOUT); + DPTX_CONNECT_TIMEOUT); if (!ret) { dev_err(dcp->dev, "dcp_dptx_connect: timed out waiting for port %u link configuration\n", @@ -1089,7 +1481,6 @@ static void dcp_typec_reconnect_work(struct work_struct *work) if (!READ_ONCE(dcp->typec_cable_connected)) return; - ret = dcp_dptx_connect(dcp, 0); if (!ret) { dcp->typec_reconnect_tries = 0; @@ -1788,7 +2179,6 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (dcp->index || dcp->dptx_phy || dcp->dptx_die) dev_info(dev, "DCP index:%u dptx target phy: %u dptx die: %u\n", dcp->index, dcp->dptx_phy, dcp->dptx_die); - mutex_init(&dcp->hpd_mutex); if (!show_notch) ret = of_property_read_u32(dev->of_node, "apple,notch-height", @@ -1978,6 +2368,14 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp_typec_reconnect_work); INIT_DELAYED_WORK(&dcp->typec_fabric_retrain_wq, dcp_typec_retrain_work); + /* + * Type-C and Thunderbolt routes can be activated as soon as they are + * registered below (dcp_register_typec_routes()), before the DRM + * device binds and dcp_comp_bind() would otherwise init this. Ported + * from aurora-silicon/linux#8. + */ + mutex_init(&dcp->hpd_mutex); + mutex_init(&dcp->tb_lock); platform_set_drvdata(pdev, dcp); @@ -2117,6 +2515,13 @@ static void dcp_platform_remove(struct platform_device *pdev) static void dcp_platform_shutdown(struct platform_device *pdev) { + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + if (dcp) { + WRITE_ONCE(dcp->typec_cable_connected, false); + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); + } component_del(&pdev->dev, &dcp_comp_ops); } @@ -2124,6 +2529,10 @@ static int dcp_platform_suspend(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); + WRITE_ONCE(dcp->typec_cable_connected, false); + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); + if (dcp->avep) av_service_disconnect(dcp); diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 2466aa9d13571..55fb42f154077 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -69,6 +69,14 @@ void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message); int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port); int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); +/* Thunderbolt DP tunnels, called from the DPTX endpoint (dptxep.c). Ported + * from aurora-silicon/linux#8. + */ +int dcp_tunnel_crossbar_up(struct apple_dcp *dcp); +int dcp_tunnel_crossbar_down(struct apple_dcp *dcp); +int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate); +int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active); + int iomfb_start_rtkit(struct apple_dcp *dcp); void iomfb_shutdown(struct apple_dcp *dcp); /* rtkit message handler for IOMFB messages */ diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 525ba63744370..6ce49d64dc4e0 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -141,7 +141,8 @@ int dptxport_release_display(struct apple_epic_service *service) return afk_service_call(service, 0, 7, NULL, 0, 16, NULL, 0, 16); } -int dptxport_set_hpd(struct apple_epic_service *service, bool hpd) +int dptxport_set_hpd_timeout(struct apple_epic_service *service, bool hpd, + unsigned int timeout_ms) { struct dcpdptx_hotplug_cmd cmd, resp; int ret; @@ -151,15 +152,24 @@ int dptxport_set_hpd(struct apple_epic_service *service, bool hpd) if (hpd) cmd.unk = cpu_to_le32(1); - ret = afk_service_call(service, 8, 8, &cmd, sizeof(cmd), 12, &resp, - sizeof(resp), 12); + ret = afk_service_call_timeout(service, 8, 8, &cmd, sizeof(cmd), 12, + &resp, sizeof(resp), 12, timeout_ms); if (ret) return ret; - if (le32_to_cpu(resp.unk) != hpd) + if (le32_to_cpu(resp.unk) != hpd) { + dev_warn(service->ep->dcp->dev, + "set_hpd: unk reply 0x%x (sent hpd=%d)\n", + le32_to_cpu(resp.unk), hpd); return -EINVAL; + } return 0; } +int dptxport_set_hpd(struct apple_epic_service *service, bool hpd) +{ + return dptxport_set_hpd_timeout(service, hpd, MSEC_PER_SEC); +} + static int dptxport_call_get_max_drive_settings(struct apple_epic_service *service, void *reply_, size_t reply_size) diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index c8816a040f321..47539fd4bcc71 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -68,4 +68,7 @@ int dptxport_connect(struct apple_epic_service *service, u8 core, u8 atc, int dptxport_request_display(struct apple_epic_service *service); int dptxport_release_display(struct apple_epic_service *service); int dptxport_set_hpd(struct apple_epic_service *service, bool hpd); +int dptxport_set_hpd_timeout(struct apple_epic_service *service, bool hpd, + unsigned int timeout_ms); + #endif From 227d96744d0e88824ea33bd2d3041c6cf77a5f2b Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Thu, 24 Sep 2026 16:48:24 +0200 Subject: [PATCH 1105/2680] drm/apple: teach the DPTX endpoint about USB4 tunnel routes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Thunderbolt/USB4 DisplayPort tunnel reaches the same DCP DPTX AFK/EPIC endpoint as a direct HDMI or USB-C alt-mode PHY, but several request/reply shapes and side effects differ: the endpoint has no local ATC DisplayPort PHY to configure, the pixel clock and display crossbar connection are owned by the Thunderbolt tunnel infrastructure instead of being driven directly, and firmware's link-training sequencing includes calls (INACTIVE_SINK_DETECTED, FORCE_HOTPLUG_DETECT, GET_ACTIVE_LANE_COUNT) that the direct-PHY path never exercises. Factor VALIDATE_CONNECTION/CONNECT's remote-port target computation into dptxport_remote_target(), and add a tunnel "role" attribute bit that lets firmware distinguish a Thunderbolt DP-IN route from a direct PHY; tolerate the resulting difference in firmware's attrs reply only for a tunnel route, still rejecting a mismatch for the direct-PHY case. Treat a NULL dptx->atcphy (the USB4 DP-IN service has no ATC PHY of its own) as "report a fixed 4 lanes, skip phy_configure()" in GET_MAX_LANE_COUNT/SET_ACTIVE_LANE_COUNT. Route SET_LINK_RATE, ACTIVATE and DEACTIVATE through the dp-tunnel.h API for a tunnel connection instead of touching the ATC PHY's DP mode or rate registers directly -- the PHY must stay in USB4/Thunderbolt mode for the life of the tunnel. Bring the tunnel's display crossbar connection down before a re-link and back up once it completes, and handle GET_ACTIVE_LANE_COUNT, FORCE_HOTPLUG_DETECT and INACTIVE_SINK_DETECTED explicitly rather than relying on the generic unhandled-call ACK. dcp_hotplug()'s forced link-status retrain recovers a connector that reports connected without a valid mode set yet, for a Type-C route switch that can drop the active mode without changing HPD state. A USB4 tunnel connector's link comes up through the DPTX endpoint's own crossbar/clock sequencing rather than a Type-C route switch, so restrict that recovery path to non-tunnel connectors. Trace the read_edt_data and property-dictionary DCP callbacks' key, size and result for any connector other than the fixed eDP panel, and include the firmware-reported mode count in the existing hotplug log line, to make the boot-property and mode-list exchange with firmware observable for a tunnel connector. [Chris Kearney: dptxport_set_hpd_timeout() moved to the previous commit, its first caller] [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Co-Authored-By: Claude Sonnet 5 --- drivers/gpu/drm/apple/dptxep.c | 269 +++++++++++++++++++++---- drivers/gpu/drm/apple/dptxep.h | 7 + drivers/gpu/drm/apple/iomfb.c | 3 +- drivers/gpu/drm/apple/iomfb_template.c | 24 ++- 4 files changed, 265 insertions(+), 38 deletions(-) diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 6ce49d64dc4e0..11091084f05e2 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -3,8 +3,13 @@ #include #include +#include #include #include +#include +#include + +#include #include "afk.h" #include "dcp.h" @@ -73,30 +78,79 @@ struct dptxport_apcall_set_tiled { __le32 retcode; }; +/* + * Ported from aurora-silicon/linux#8: a Thunderbolt DP tunnel uses the same + * plain CORE|ATC|DIE|CONNECTED target as a direct alt-mode PHY. CORE is the + * DFP port (0 = dpphy, 1/2 = dpin0/dpin1, dcp->dptx_dfp_port), ATC is the + * route's own ATC index -- no separate DPIN field. + */ +static u32 dptxport_remote_target(struct apple_dcp *dcp, u8 core, u8 atc, + u8 die) +{ + return FIELD_PREP(DCPDPTX_REMOTE_PORT_CORE, core) | + FIELD_PREP(DCPDPTX_REMOTE_PORT_ATC, atc) | + FIELD_PREP(DCPDPTX_REMOTE_PORT_DIE, die) | + DCPDPTX_REMOTE_PORT_CONNECTED; +} + int dptxport_validate_connection(struct apple_epic_service *service, u8 core, u8 atc, u8 die) { struct dptx_port *dptx = service->cookie; struct dcpdptx_connection_cmd cmd, resp; int ret; - u32 target = FIELD_PREP(DCPDPTX_REMOTE_PORT_CORE, core) | - FIELD_PREP(DCPDPTX_REMOTE_PORT_ATC, atc) | - FIELD_PREP(DCPDPTX_REMOTE_PORT_DIE, die) | - DCPDPTX_REMOTE_PORT_CONNECTED; + u32 target = dptxport_remote_target(service->ep->dcp, core, atc, die); + + /* + * attributes: role (0 = direct PHY, 1 = Thunderbolt/USB4 DP IN) | + * supportsHPD << 8. A real, hardware-validated reference + * implementation of the equivalent Thunderbolt DP tunnel mechanism + * on a different SoC (aurora-silicon/linux#8) adds this exact role + * bit for its tunnel routes; our own analog-DPIN path has never set + * it, always sending a plain direct-PHY attributes value even + * though this is a genuinely USB4-tunneled connection. DCP firmware + * appears to treat a failed AUX probe as fatal for role=0 but keeps + * training regardless for role=1 (a Thunderbolt tunnel's AUX + * proxying may not respond as fast as a direct PHY's): repeated + * test runs with substantially different Activate/crossbar ordering + * all reached the identical INACTIVE_SINK_DETECTED/DPRX-timeout + * outcome, which points at a signal we are not sending at all + * rather than a timing issue fixable by reordering. + */ + u32 attrs = 0x100 | (dcp_is_usb4_output(service->ep->dcp) ? 1 : 0); trace_dptxport_validate_connection(dptx, core, atc, die); + dptx->validate_calls++; + dev_dbg(service->ep->dcp->dev, + "DPTX validate: call #%u this boot target=0x%x core=%u atc=%u die=%u attrs=0x%x caller=%pS\n", + dptx->validate_calls, target, core, atc, die, + attrs, __builtin_return_address(0)); cmd.target = cpu_to_le32(target); - cmd.unk = cpu_to_le32(0x100); + cmd.unk = cpu_to_le32(attrs); ret = afk_service_call(service, 0, 12, &cmd, sizeof(cmd), 40, &resp, sizeof(resp), 40); if (ret) return ret; - if (le32_to_cpu(resp.target) != target) - return -EINVAL; - if (le32_to_cpu(resp.unk) != 0x100) + if (le32_to_cpu(resp.target) != target) { + dev_warn(service->ep->dcp->dev, + "validate_connection: target reply 0x%x (sent 0x%x)\n", + le32_to_cpu(resp.target), target); return -EINVAL; + } + if (le32_to_cpu(resp.unk) != attrs) { + /* only the Thunderbolt DP IN role is allowed to differ */ + if (!dcp_is_usb4_output(service->ep->dcp)) { + dev_warn(service->ep->dcp->dev, + "validate_connection: attrs reply 0x%x (sent 0x%x), rejecting\n", + le32_to_cpu(resp.unk), attrs); + return -EINVAL; + } + dev_info(service->ep->dcp->dev, + "validate_connection: attrs reply 0x%x (sent 0x%x)\n", + le32_to_cpu(resp.unk), attrs); + } return 0; } @@ -106,14 +160,18 @@ int dptxport_connect(struct apple_epic_service *service, u8 core, u8 atc, { struct dptx_port *dptx = service->cookie; struct dcpdptx_connection_cmd cmd, resp; - u32 unk_field = supports_hpd ? DCPDPTX_REMOTE_PORT_SUPPORTS_HPD : 0; + /* same role bit as dptxport_validate_connection() above */ + u32 unk_field = (supports_hpd ? DCPDPTX_REMOTE_PORT_SUPPORTS_HPD : 0) | + (dcp_is_usb4_output(service->ep->dcp) ? 1 : 0); int ret; - u32 target = FIELD_PREP(DCPDPTX_REMOTE_PORT_CORE, core) | - FIELD_PREP(DCPDPTX_REMOTE_PORT_ATC, atc) | - FIELD_PREP(DCPDPTX_REMOTE_PORT_DIE, die) | - DCPDPTX_REMOTE_PORT_CONNECTED; + u32 target = dptxport_remote_target(service->ep->dcp, core, atc, die); trace_dptxport_connect(dptx, core, atc, die); + dptx->connect_calls++; + dev_dbg(service->ep->dcp->dev, + "DPTX connect: call #%u this boot target=0x%x unk=0x%x caller=%pS\n", + dptx->connect_calls, target, unk_field, + __builtin_return_address(0)); cmd.target = cpu_to_le32(target); cmd.unk = cpu_to_le32(unk_field); @@ -122,8 +180,12 @@ int dptxport_connect(struct apple_epic_service *service, u8 core, u8 atc, if (ret) return ret; - if (le32_to_cpu(resp.target) != target) + if (le32_to_cpu(resp.target) != target) { + dev_warn(service->ep->dcp->dev, + "connect: target reply 0x%x (sent 0x%x)\n", + le32_to_cpu(resp.target), target); return -EINVAL; + } if (le32_to_cpu(resp.unk) != unk_field) dev_notice(service->ep->dcp->dev, "unexpected unk field in reply: 0x%x (0x%x)\n", le32_to_cpu(resp.unk), unk_field); @@ -133,12 +195,34 @@ int dptxport_connect(struct apple_epic_service *service, u8 core, u8 atc, int dptxport_request_display(struct apple_epic_service *service) { - return afk_service_call(service, 0, 6, NULL, 0, 16, NULL, 0, 16); + struct dptx_port *dptx = service->cookie; + int ret; + + dptx->request_calls++; + dev_dbg(service->ep->dcp->dev, + "DPTX request_display: call #%u this boot caller=%pS\n", + dptx->request_calls, __builtin_return_address(0)); + ret = afk_service_call(service, 0, 6, NULL, 0, 16, NULL, 0, 16); + dev_dbg(service->ep->dcp->dev, + "DPTX request_display: call #%u result=%d\n", + dptx->request_calls, ret); + return ret; } int dptxport_release_display(struct apple_epic_service *service) { - return afk_service_call(service, 0, 7, NULL, 0, 16, NULL, 0, 16); + struct dptx_port *dptx = service->cookie; + int ret; + + dptx->release_calls++; + dev_dbg(service->ep->dcp->dev, + "DPTX release_display: call #%u this boot caller=%pS\n", + dptx->release_calls, __builtin_return_address(0)); + ret = afk_service_call(service, 0, 7, NULL, 0, 16, NULL, 0, 16); + dev_dbg(service->ep->dcp->dev, + "DPTX release_display: call #%u result=%d\n", + dptx->release_calls, ret); + return ret; } int dptxport_set_hpd_timeout(struct apple_epic_service *service, bool hpd, @@ -271,6 +355,15 @@ static int dptxport_call_get_max_lane_count(struct apple_epic_service *service, if (reply_size < sizeof(*reply)) return -EINVAL; + if (!dptx->atcphy) { + /* USB4 DP IN: no ATC DP PHY to validate. */ + dptx->lane_count = 4; + reply->retcode = cpu_to_le32(0); + reply->lane_count = cpu_to_le64(4); + dev_info(dcp->dev, "get_max_lane_count: USB4 DP IN, 4 lanes\n"); + return 0; + } + ret = phy_validate(dptx->atcphy, PHY_MODE_DP, 0, &phy_ops); if (ret < 0) { dev_err(dcp->dev, "phy_validate failed: %d\n", ret); @@ -320,9 +413,9 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic case 0 ... 2: case 4: dptx->phy_ops.dp.lanes = lane_count; - // Use dptx phy index > 3 as indication for dptx-phy or - // lpdptx-phy and configure the number of lanes for those - dptx->phy_ops.dp.set_lanes = (dcp->dptx_phy > 3); + /* USB4 DP IN has no ATC PHY; still accept the lane count. */ + dptx->phy_ops.dp.set_lanes = + dcp_is_usb4_output(dcp) || (dcp->dptx_phy > 3); break; default: dev_err(dcp->dev, "set_active_lane_count: invalid lane count:%llu\n", lane_count); @@ -332,7 +425,7 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic } if (dptx->phy_ops.dp.set_lanes) { - if (dptx->atcphy) { + if (dptx->atcphy && !dcp_is_usb4_output(dcp)) { ret = phy_configure(dptx->atcphy, &dptx->phy_ops); if (ret) return ret; @@ -344,8 +437,30 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic reply->retcode = cpu_to_le32(retcode); reply->lane_count = cpu_to_le64(lane_count); - if (lane_count > 0) + if (lane_count > 0) { + dev_info(dcp->dev, "USB4/DPTX: SET_ACTIVE_LANE_COUNT %llu\n", + lane_count); + /* + * dcp_usb4_drm_allowed() (usb4_force_dptx) used to gate this + * for the old manual-training sysfs knob (module_param_cb + * usb4_dptx_train), removed wholesale in 0dc9f50 when + * apple_dcp_tb_dp_tunnel() replaced it with automatic tunnel + * detection -- but usb4_force_dptx itself, and this gate, + * were left behind with no way left to ever set it true. + * dcp_dptx_connect()'s single wait_for_completion_timeout() + * on linkcfg_completion is now shared by both the alt-mode + * and USB4-tunnel paths (also part of 0dc9f50's unification), + * so gating it here meant every tunneled connect timed out + * after a fully successful DPRX/AUX handshake and lane + * negotiation -- firmware does not send FORCE_HOTPLUG_DETECT + * (the only other completion site) for a tunnel. The + * reference (aurora-silicon/linux#8) completes + * linkcfg_completion here unconditionally; usb4_lane_completion + * has no consumer anywhere in this tree, so this drops both the + * dead gate and the dead completion and matches the reference. + */ complete(&dptx->linkcfg_completion); + } return ret; } @@ -380,6 +495,14 @@ dptxport_call_will_change_link_config(struct apple_epic_service *service) static int dptxport_call_did_change_link_config(struct apple_epic_service *service) { + /* + * Ported from aurora-silicon/linux#8: bringing the tunnel crossbar + * connection up (dcp_tunnel_crossbar_up()) now happens in + * dptxport_call(), after this succeeds, gated on + * dptx_tunnel && link_rate -- not inside this handler, and its + * result is never propagated back to DCP as a call failure. + */ + /* assume the link config did change and wait a little bit */ mdelay(10); @@ -404,6 +527,8 @@ static int dptxport_call_set_link_rate(struct apple_epic_service *service, link_rate = le32_to_cpu(request->link_rate); trace_dptxport_call_set_link_rate(dptx, link_rate); + dev_info(service->ep->dcp->dev, "DPTXPort: SET_LINK_RATE 0x%x\n", + link_rate); switch (link_rate) { case LINK_RATE_RBR: @@ -436,18 +561,25 @@ static int dptxport_call_set_link_rate(struct apple_epic_service *service, } if (phy_set_rate) { - dptx->phy_ops.dp.link_rate = phy_link_rate; - dptx->phy_ops.dp.set_rate = 1; - - if (dptx->atcphy) { + /* + * Ported from aurora-silicon/linux#8: a Thunderbolt DP tunnel + * starts/stops its own pixel clock instead of configuring the + * PHY directly, and never fails the apcall over it -- a + * failed clock just keeps the crossbar connection down + * (dcp->tb_clock_ok), which dcp_tunnel_crossbar_up() checks. + */ + if (dptx->atcphy && service->ep->dcp->dptx_tunnel) { + dcp_tunnel_set_rate(service->ep->dcp, dptx->atcphy, + link_rate); + } else if (dptx->atcphy) { + dptx->phy_ops.dp.link_rate = phy_link_rate; + dptx->phy_ops.dp.set_rate = 1; ret = phy_configure(dptx->atcphy, &dptx->phy_ops); if (ret) return ret; } - //if (dptx->phy_ops.dp.set_rate) dptx->link_rate = dptx->pending_link_rate = link_rate; - } //dptx->pending_link_rate = link_rate; @@ -467,7 +599,15 @@ static int dptxport_call_get_supports_hpd(struct apple_epic_service *service, return -EINVAL; reply->retcode = cpu_to_le32(0); - reply->supported = cpu_to_le32(dcp_is_typec_output(dcp)); + /* + * Analog DPIN CORE=1 already uses AFK set_hpd (returns 0). + * Denying HPD here made request_display ACTIVATE then 22/24 + * DEVICE_NOT_STARTED (~5.5s). Advertise HPD so firmware uses + * that path instead of waiting for a PHY start. + */ + reply->supported = cpu_to_le32(dcp_is_typec_output(dcp) ? 1 : 0); + dev_info(dcp->dev, "DPTXPort: GET_SUPPORTS_HPD %u usb4=%d\n", + le32_to_cpu(reply->supported), dcp_is_usb4_output(dcp)); return 0; } @@ -503,11 +643,20 @@ dptxport_call_activate(struct apple_epic_service *service, void *reply, size_t reply_size) { struct dptx_port *dptx = service->cookie; - const struct apple_dcp *dcp = service->ep->dcp; + struct apple_dcp *dcp = service->ep->dcp; - /* Standalone PHYs need DCP input selection here. Type-C owns ATC PHY mode. */ - if (!dcp->phy_managed_by_typec) + /* + * Ported from aurora-silicon/linux#8: no crossbar, and no PHY mode + * change for a tunnel (dptx->atcphy must stay in USB4/TBT mode the + * whole connection, see dcp_tunnel_set_rate()). The DP IN adapter is + * only woken (DPTX_INACTIVE=0) here, via dcp_tunnel_dpin_activate(); + * waking it earlier hangs the machine. Activate always replies + * success to DCP. + */ + if (dptx->atcphy && !dcp->phy_managed_by_typec) phy_set_mode_ext(dptx->atcphy, PHY_MODE_DP, dcp->index); + if (dcp->dptx_tunnel) + dcp_tunnel_dpin_activate(dcp, true); memcpy(reply, data, min(reply_size, data_size)); if (reply_size >= 4) @@ -522,9 +671,12 @@ dptxport_call_deactivate(struct apple_epic_service *service, void *reply, size_t reply_size) { struct dptx_port *dptx = service->cookie; - const struct apple_dcp *dcp = service->ep->dcp; + struct apple_dcp *dcp = service->ep->dcp; - if (!dcp->phy_managed_by_typec) + dev_info(dcp->dev, "DPTXPort: DEACTIVATE\n"); + if (dcp->dptx_tunnel) + dcp_tunnel_dpin_activate(dcp, false); + if (dptx->atcphy && !dcp->phy_managed_by_typec) phy_set_mode_ext(dptx->atcphy, PHY_MODE_INVALID, 0); memcpy(reply, data, min(reply_size, data_size)); @@ -540,12 +692,30 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, { struct dptx_port *dptx = service->cookie; trace_dptxport_apcall(dptx, idx, data_size); + dev_dbg(service->ep->dcp->dev, "DPTXPort: APCALL %u (%zu bytes)\n", + idx, data_size); + if (data_size) + print_hex_dump_debug("DPTXPort: apcall data: ", + DUMP_PREFIX_OFFSET, 16, 1, data, + min(data_size, (size_t)64), true); switch (idx) { case DPTX_APCALL_WILL_CHANGE_LINKG_CONFIG: + /* + * Ported from aurora-silicon/linux#8: a re-link on an + * established tunnel takes the crossbar connection down + * first. + */ + if (service->ep->dcp->dptx_tunnel && dptx->link_rate) + dcp_tunnel_crossbar_down(service->ep->dcp); return dptxport_call_will_change_link_config(service); - case DPTX_APCALL_DID_CHANGE_LINK_CONFIG: - return dptxport_call_did_change_link_config(service); + case DPTX_APCALL_DID_CHANGE_LINK_CONFIG: { + int ret = dptxport_call_did_change_link_config(service); + + if (!ret && service->ep->dcp->dptx_tunnel && dptx->link_rate) + dcp_tunnel_crossbar_up(service->ep->dcp); + return ret; + } case DPTX_APCALL_GET_MAX_LINK_RATE: return dptxport_call_get_max_link_rate(service, reply, reply_size); @@ -556,6 +726,15 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, reply, reply_size); case DPTX_APCALL_GET_MAX_LANE_COUNT: return dptxport_call_get_max_lane_count(service, reply, reply_size); + case DPTX_APCALL_GET_ACTIVE_LANE_COUNT: { + struct dptxport_apcall_lane_count *lc = reply; + + if (reply_size < sizeof(*lc)) + return -EINVAL; + lc->retcode = cpu_to_le32(0); + lc->lane_count = cpu_to_le64(dptx->lane_count); + return 0; + } case DPTX_APCALL_SET_ACTIVE_LANE_COUNT: return dptxport_call_set_active_lane_count(service, data, data_size, reply, reply_size); @@ -592,6 +771,26 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, case DPTX_APCALL_DEACTIVATE: return dptxport_call_deactivate(service, data, data_size, reply, reply_size); + case DPTX_APCALL_FORCE_HOTPLUG_DETECT: + dev_info(service->ep->dcp->dev, + "DPTXPort: FORCE_HOTPLUG_DETECT\n"); + complete(&dptx->linkcfg_completion); + memcpy(reply, data, min(reply_size, data_size)); + if (reply_size >= 4) + memset(reply, 0, 4); + return 0; + case DPTX_APCALL_INACTIVE_SINK_DETECTED: + /* + * Normal prelude to link training on USB4. Ack and wait for + * SET_ACTIVE_LANE_COUNT. Treating this as failure made + * firmware DEACTIVATE after it had already set lanes/rate. + */ + dev_info(service->ep->dcp->dev, + "DPTXPort: INACTIVE_SINK_DETECTED (keep waiting for lanes)\n"); + memcpy(reply, data, min(reply_size, data_size)); + if (reply_size >= 4) + memset(reply, 0, 4); + return 0; default: /* just try to ACK and hope for the best... */ dev_info(service->ep->dcp->dev, "DPTXPort: acking unhandled call %u\n", diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index 47539fd4bcc71..885e3675f9db5 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -59,6 +59,13 @@ struct dptx_port { u32 lane_count; u32 link_rate, pending_link_rate; u32 drive_settings[2]; + /* + * Per-boot invocation counters for the validate/connect/request/ + * release AFK calls, used together with the caller return address + * logged alongside them to correlate call ordering when diagnosing + * link-training sequencing issues. + */ + u32 validate_calls, connect_calls, request_calls, release_calls; }; int dptxport_validate_connection(struct apple_epic_service *service, u8 core, diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 5b78f77dc7401..9fde318f5fa61 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -293,7 +293,8 @@ void dcp_hotplug(struct work_struct *work) * display modes from atomic_flush, so userspace needs to trigger a * flush, or the CRTC gets no signal. */ - if (connector->base.state && !dcp->valid_mode && connector->connected) { + if (connector->base.state && !dcp->valid_mode && connector->connected && + !dcp_is_usb4_output(dcp)) { drm_connector_set_link_status_property(&connector->base, DRM_MODE_LINK_STATUS_BAD); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 7bd1e5d062486..0c62d01fa1d6c 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -494,6 +494,13 @@ static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *d static struct dcp_read_edt_data_resp dcpep_cb_read_edt_data(struct apple_dcp *dcp, struct dcp_read_edt_data_req *req) { + /* Observe boot-property requests without inventing firmware timings. */ + if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_eDP) + dev_info(dcp->dev, + "read_edt_data key=%.*s count=%u default0=%#x ret=0\n", + (int)sizeof(req->key), req->key, req->count, + req->value[0]); + return (struct dcp_read_edt_data_resp){ .value[0] = req->value[0], .ret = 0, @@ -604,6 +611,12 @@ static u8 dcpep_cb_prop_end(struct apple_dcp *dcp, { u8 resp = dcpep_process_chunks(dcp, req); + if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_eDP) + dev_info(dcp->dev, + "DCP property key=%.*s bytes=%zu accepted=%u nr_modes=%u\n", + (int)sizeof(req->key), req->key, dcp->chunks.length, + resp, dcp->nr_modes); + /* move chunked data to connector to provide it via debugfs */ dcp_connector_update_dict(dcp->connector, req->key, &dcp->chunks); dcp->chunks.data = NULL; @@ -1035,6 +1048,13 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) */ if (dcp->main_display) return; + /* + * Report firmware hotplug independently of the USB4 PHY experiment. + * Reassigning lpdptxphy blanked eDP even with these callbacks ignored; + * suppressing connector notifications does not protect the panel. + * Mode probing still uses this DCP's firmware modes, and mode_valid + * rejects modes absent from that list. Do not synthesize a mode here. + */ if (dcp->during_modeset) { /* @@ -1051,8 +1071,8 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) return; } - dev_info(dcp->dev, "cb_hotplug() connected:%llu, valid_mode:%d\n", - *connected, dcp->valid_mode); + dev_info(dcp->dev, "cb_hotplug() connected:%llu, valid_mode:%d nr_modes:%u\n", + *connected, dcp->valid_mode, dcp->nr_modes); /* Hotplug invalidates mode. DRM doesn't always handle this. */ if (!(*connected)) { From ee2dec28de55160961eb8eb590398eb253e0ea23 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 20:24:04 -0700 Subject: [PATCH 1106/2680] drm/apple: keep the j416s tunnel hotplug policy scoped to M2 Pro MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retain the M1 link-status handling when combining the two display series. Document the T602X PHY state carried alongside the M1 clock state. [ryan: split by topic: only the drm/apple part and its selftests are here; the PHY, Type-C, Thunderbolt and crossbar hunks are in 170-usb] [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/iomfb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 58c176b344bfc..4fe1565d2d446 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -294,7 +294,7 @@ void dcp_hotplug(struct work_struct *work) * flush, or the CRTC gets no signal. */ if (connector->base.state && !dcp->valid_mode && connector->connected && - !dcp_is_usb4_output(dcp)) { + !(dcp_is_usb4_output(dcp) && of_machine_is_compatible("apple,j416s"))) { drm_connector_set_link_status_property(&connector->base, DRM_MODE_LINK_STATUS_BAD); From 360d2b814dbc9d522841b11ca956520d64c5a295 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 21:00:22 -0700 Subject: [PATCH 1107/2680] drm/apple: drain display retries before stopping firmware endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Identify each Type-C connection when queuing a placeholder EDID retry and serialize HPD pulses with disconnect. Drain stale retries before reusing a route. Disable delayed display work before endpoint shutdown or suspend, then balance the enable on bind and resume. Keep work disabled before the component binds and retry a cable already routed at boot. Use the selected connection timeout when reporting elapsed time. [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/dcp-internal.h | 2 + drivers/gpu/drm/apple/dcp.c | 96 ++++++++++++++++++---------- 2 files changed, 64 insertions(+), 34 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index cb50dd974125a..be668e0c648d8 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -216,6 +216,8 @@ struct apple_dcp { bool valid_mode; /* One HPD pulse after a placeholder EDID, per Type-C connection. */ bool placeholder_retried; + u64 typec_generation; /* hpd_mutex: identifies the current connection */ + u64 placeholder_generation; struct delayed_work placeholder_edid_wq; bool use_timestamps; bool vrr_enabled; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index f350c4c160aee..c57726b143bbe 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1294,6 +1294,7 @@ bool dcp_has_typec_routes(struct platform_device *pdev) static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) { + unsigned long timeout; int ret = 0; if (!dcp->phy) { @@ -1365,10 +1366,10 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) } mutex_unlock(&dcp->hpd_mutex); + timeout = dcp_is_usb4_output(dcp) && of_machine_is_compatible("apple,j416s") ? + DPTX_TUNNEL_CONNECT_TIMEOUT : DPTX_CONNECT_TIMEOUT; ret = wait_for_completion_timeout(&dcp->dptxport[port].linkcfg_completion, - dcp_is_usb4_output(dcp) && - of_machine_is_compatible("apple,j416s") ? - DPTX_TUNNEL_CONNECT_TIMEOUT : DPTX_CONNECT_TIMEOUT); + timeout); if (!ret) { dev_err(dcp->dev, "dcp_dptx_connect: timed out waiting for port %u link configuration\n", @@ -1378,7 +1379,7 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) } dev_dbg(dcp->dev, "dcp_dptx_connect: waited %d ms for link\n", - jiffies_to_msecs(DPTX_CONNECT_TIMEOUT - ret)); + jiffies_to_msecs(timeout - ret)); usleep_range(5, 10); @@ -1422,12 +1423,15 @@ static bool dcp_edid_is_placeholder(const struct drm_edid *drm_edid) void dcp_retry_placeholder_edid(struct apple_dcp *dcp, const struct drm_edid *drm_edid) { - if (!dcp_is_typec_output(dcp) || dcp->placeholder_retried) + guard(mutex)(&dcp->hpd_mutex); + if (!dcp_is_typec_output(dcp) || !dcp->typec_cable_connected || + dcp->placeholder_retried) return; if (!dcp_edid_is_placeholder(drm_edid)) return; dcp->placeholder_retried = true; + dcp->placeholder_generation = dcp->typec_generation; schedule_delayed_work(&dcp->placeholder_edid_wq, msecs_to_jiffies(300)); } @@ -1437,16 +1441,15 @@ static void dcp_placeholder_edid_work(struct work_struct *work) container_of(to_delayed_work(work), struct apple_dcp, placeholder_edid_wq); struct apple_epic_service *service; + u64 generation; int ret; - if (!READ_ONCE(dcp->typec_cable_connected)) - return; - mutex_lock(&dcp->hpd_mutex); - service = dcp->dptxport[0].connected ? dcp->dptxport[0].service : NULL; - mutex_unlock(&dcp->hpd_mutex); - if (!service) - return; + generation = dcp->placeholder_generation; + if (!dcp->typec_cable_connected || !dcp->dptxport[0].connected || + !dcp->dptxport[0].enabled || generation != dcp->typec_generation) + goto out_unlock; + service = dcp->dptxport[0].service; /* * Some adapters answer the first connection with a 1024x768 @@ -1457,18 +1460,23 @@ static void dcp_placeholder_edid_work(struct work_struct *work) if (ret) { dev_info(dcp->dev, "placeholder EDID: HPD drop failed: %d\n", ret); - return; + goto out_unlock; } + mutex_unlock(&dcp->hpd_mutex); msleep(1000); - if (!READ_ONCE(dcp->typec_cable_connected)) - return; + mutex_lock(&dcp->hpd_mutex); + if (!dcp->typec_cable_connected || !dcp->dptxport[0].connected || + generation != dcp->typec_generation) + goto out_unlock; ret = dptxport_set_hpd(service, true); if (ret) dev_info(dcp->dev, "placeholder EDID: HPD assert failed: %d\n", ret); +out_unlock: + mutex_unlock(&dcp->hpd_mutex); } static void dcp_typec_reconnect_work(struct work_struct *work) @@ -1529,11 +1537,13 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) int ret; if (dcp_is_typec_output(dcp)) { + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + guard(mutex)(&dcp->hpd_mutex); + dcp->typec_generation++; WRITE_ONCE(dcp->typec_cable_connected, true); dcp->typec_reconnect_tries = 0; dcp->placeholder_retried = false; cancel_delayed_work(&dcp->typec_reconnect_wq); - cancel_delayed_work(&dcp->placeholder_edid_wq); } ret = dcp_dptx_connect(dcp, port); @@ -1549,10 +1559,13 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) struct apple_dcp *dcp = platform_get_drvdata(pdev); if (dcp_is_typec_output(dcp)) { - WRITE_ONCE(dcp->typec_cable_connected, false); + scoped_guard(mutex, &dcp->hpd_mutex) { + WRITE_ONCE(dcp->typec_cable_connected, false); + dcp->typec_generation++; + } WRITE_ONCE(dcp->typec_crtc_off, false); - cancel_delayed_work(&dcp->typec_reconnect_wq); - cancel_delayed_work(&dcp->placeholder_edid_wq); + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); } disconnected_hpd_event(dcp->connector); @@ -2171,6 +2184,28 @@ static int dcp_connector_type_from_dt(struct device_node *np) return DRM_MODE_CONNECTOR_Unknown; } +static void dcp_disable_typec_work(struct apple_dcp *dcp) +{ + scoped_guard(mutex, &dcp->hpd_mutex) { + WRITE_ONCE(dcp->typec_cable_connected, false); + dcp->typec_generation++; + } + /* Block new enqueues as well as draining users of the AFK endpoints. */ + disable_delayed_work_sync(&dcp->typec_reconnect_wq); + disable_delayed_work_sync(&dcp->placeholder_edid_wq); + disable_delayed_work_sync(&dcp->typec_fabric_retrain_wq); +} + +static void dcp_enable_typec_work(struct apple_dcp *dcp) +{ + enable_delayed_work(&dcp->typec_reconnect_wq); + enable_delayed_work(&dcp->placeholder_edid_wq); + enable_delayed_work(&dcp->typec_fabric_retrain_wq); + /* A cable can be routed before the DRM component binds. */ + if (READ_ONCE(dcp->typec_cable_connected)) + mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); +} + static int dcp_comp_bind(struct device *dev, struct device *main, void *data) { struct device_node *panel_np; @@ -2269,6 +2304,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (ret) return dev_err_probe(dev, ret, "Failed to boot RTKit: %d\n", ret); + dcp_enable_typec_work(dcp); return ret; } @@ -2286,6 +2322,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) if (dcp->hdmi_hpd_irq) disable_irq(dcp->hdmi_hpd_irq); + dcp_disable_typec_work(dcp); typec_mux_put(dcp->typec_mux); if (dcp->avep) { @@ -2327,9 +2364,6 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); } - cancel_delayed_work_sync(&dcp->typec_reconnect_wq); - cancel_delayed_work_sync(&dcp->placeholder_edid_wq); - cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); cancel_work_sync(&dcp->vblank_wq); devm_clk_put(dev, dcp->clk); @@ -2387,6 +2421,10 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp_placeholder_edid_work); INIT_DELAYED_WORK(&dcp->typec_fabric_retrain_wq, dcp_typec_retrain_work); + /* Balanced by enable at successful component bind. */ + disable_delayed_work(&dcp->typec_reconnect_wq); + disable_delayed_work(&dcp->placeholder_edid_wq); + disable_delayed_work(&dcp->typec_fabric_retrain_wq); platform_set_drvdata(pdev, dcp); @@ -2526,14 +2564,6 @@ static void dcp_platform_remove(struct platform_device *pdev) static void dcp_platform_shutdown(struct platform_device *pdev) { - struct apple_dcp *dcp = platform_get_drvdata(pdev); - - if (dcp) { - WRITE_ONCE(dcp->typec_cable_connected, false); - cancel_delayed_work_sync(&dcp->typec_reconnect_wq); - cancel_delayed_work_sync(&dcp->placeholder_edid_wq); - cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); - } component_del(&pdev->dev, &dcp_comp_ops); } @@ -2541,10 +2571,7 @@ static int dcp_platform_suspend(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); - WRITE_ONCE(dcp->typec_cable_connected, false); - cancel_delayed_work_sync(&dcp->typec_reconnect_wq); - cancel_delayed_work_sync(&dcp->placeholder_edid_wq); - cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); + dcp_disable_typec_work(dcp); if (dcp->avep) av_service_disconnect(dcp); @@ -2570,6 +2597,7 @@ static int dcp_platform_resume(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); + dcp_enable_typec_work(dcp); if (dcp->hdmi_hpd_irq) enable_irq(dcp->hdmi_hpd_irq); From 11986ea5927b497f0b4b956e5ee3386bc01ceb2d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 21:04:12 -0700 Subject: [PATCH 1108/2680] drm/apple: preserve unbound pipeline ranking for j416s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Type-C routes are registered before the DRM component binds. Their score is UINT_MAX-1 until a CRTC exists; adding the j416s preference penalty wraps it to 98 and can select an unbound, nonpreferred pipeline first. Keep the sentinel score intact when applying the preference. [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/dcp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index c57726b143bbe..8343d00ebfd6f 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -657,7 +657,7 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, score = dcp_typec_route_score(candidate); /* j416s dcpext0 completes tunneled link training; prefer it. */ if (of_machine_is_compatible("apple,j416s") && - !candidate->dcp->fixed_phy) + !candidate->dcp->fixed_phy && score < UINT_MAX - 100) score += 100; if (score < best_score) { best = candidate; From c57255ba2c9491e19a09a028e7ddc4d82b0e6fb6 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:09:23 -0700 Subject: [PATCH 1109/2680] drm/apple: retain AFK receive payload until dispatch completes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copy each ring entry before publishing the consumer position. The producer can otherwise reuse bytes while a report or command handler still reads them. Also defer reading the subtype until the header bounds check has succeeded. [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/afk.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 9124f9e46647b..3fbc5fe4d601c 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -519,7 +519,7 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, struct epic_hdr *ehdr = (struct epic_hdr *)data; struct epic_sub_hdr *eshdr = (struct epic_sub_hdr *)(data + sizeof(*ehdr)); - u16 subtype = le16_to_cpu(eshdr->type); + u16 subtype; u8 *payload = data + sizeof(*ehdr) + sizeof(*eshdr); size_t payload_size; @@ -529,6 +529,7 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, return; } payload_size = data_size - sizeof(*ehdr) - sizeof(*eshdr); + subtype = le16_to_cpu(eshdr->type); trace_afk_recv_handle(ep, channel, type, data_size, ehdr, eshdr); @@ -591,6 +592,7 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, static bool afk_recv(struct apple_dcp_afkep *ep) { struct afk_qe *hdr; + u8 *data; u32 rptr, wptr; u32 magic, size, channel, type; @@ -645,8 +647,6 @@ static bool afk_recv(struct apple_dcp_afkep *ep) ep->endpoint, magic); return false; } - - ep->rxbfr.hdr->rptr = cpu_to_le32(rptr); } if (rptr + size + sizeof(*hdr) > ep->rxbfr.bufsz) { @@ -658,6 +658,14 @@ static bool afk_recv(struct apple_dcp_afkep *ep) channel = le32_to_cpu(hdr->channel); type = le32_to_cpu(hdr->type); + /* + * Publishing rptr releases this entry to the firmware. Handlers can + * sleep and send replies, so retain a private copy before allowing + * the producer to reuse the ring storage. + */ + data = kmemdup(hdr->data, size, GFP_KERNEL); + if (!data) + return false; rptr = ALIGN(rptr + sizeof(*hdr) + size, 1 << BLOCK_SHIFT); if (WARN_ON(rptr > ep->rxbfr.bufsz)) @@ -670,14 +678,8 @@ static bool afk_recv(struct apple_dcp_afkep *ep) ep->rxbfr.hdr->rptr = cpu_to_le32(rptr); trace_afk_recv_rwptr_post(ep, rptr, wptr); - /* - * TODO: this is theoretically unsafe since DCP could overwrite data - * after the read pointer was updated above. Do it anyway since - * it avoids 2 problems in the DCP tracer: - * 1. the tracer sees replies before the notifies from dcp - * 2. the tracer tries to read buffers after they are unmapped. - */ - afk_recv_handle(ep, channel, type, hdr->data, size); + afk_recv_handle(ep, channel, type, data, size); + kfree(data); return true; } From 8aaaee7c51504d1f9d0790a7ca9b53d39fad9af0 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:11:52 -0700 Subject: [PATCH 1110/2680] drm/apple: preserve mode invalidation across hotplug and modeset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Serialize mode validity and deferred connector changes between the atomic commit worker and callback thread. A completed modeset must not overwrite a disconnect or other link invalidation received while it was waiting. Invalidate before starting the operation so lookup, allocation and timeout failures cannot leave an old mode marked valid. Recheck validity after an asynchronous color-matrix operation before beginning a swap. Add seven KUnit scenarios for failure, deferred connect/disconnect, transient link loss and invalidation on either side of completion. [ryan: adapted to 7.2.10 — struct drm_atomic_state is struct drm_atomic_commit] [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/Makefile | 1 + drivers/gpu/drm/apple/Kconfig | 10 ++ drivers/gpu/drm/apple/dcp-internal.h | 8 +- drivers/gpu/drm/apple/dcp.c | 4 +- drivers/gpu/drm/apple/dcp_backlight.c | 2 +- drivers/gpu/drm/apple/iomfb-state-test.c | 127 +++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb-state.h | 110 ++++++++++++++++++++ drivers/gpu/drm/apple/iomfb.c | 29 ++++-- drivers/gpu/drm/apple/iomfb_template.c | 61 ++--------- 9 files changed, 284 insertions(+), 68 deletions(-) create mode 100644 drivers/gpu/drm/apple/iomfb-state-test.c create mode 100644 drivers/gpu/drm/apple/iomfb-state.h diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index c62687d651942..8bc9a05484f21 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -228,6 +228,7 @@ obj-$(CONFIG_DRM_PANFROST) += panfrost/ obj-$(CONFIG_DRM_PANTHOR) += panthor/ obj-$(CONFIG_DRM_TYR) += tyr/ obj-$(CONFIG_DRM_APPLE) += apple/ +obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple/iomfb-state-test.o obj-$(CONFIG_DRM_ASPEED_GFX) += aspeed/ obj-$(CONFIG_DRM_MCDE) += mcde/ obj-$(CONFIG_DRM_TIDSS) += tidss/ diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index ecf7afd76ceda..65c7fb2256344 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -28,3 +28,13 @@ config DRM_APPLE_DEBUG bool "Enable additional driver debugging" depends on DRM_APPLE depends on EXPERT # only for developers + +config DRM_APPLE_KUNIT_TEST + tristate "KUnit tests for Apple display state" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Test modeset and hotplug state transitions without display hardware, + including link changes that race with a modeset and recovery after + failed modesets. These tests can run on other architectures and do + not require the Apple display driver to be enabled. diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index be668e0c648d8..3de4748e53d88 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -17,6 +17,7 @@ #include "dptxep.h" #include "iomfb.h" +#include "iomfb-state.h" #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" #include "iomfb_v14_7.h" @@ -209,11 +210,7 @@ struct apple_dcp { u64 swap_submit_timestamp; /* Current display mode */ - bool during_modeset; - /* hotplug reported while a modeset was in flight, applied afterwards */ - bool pending_hotplug; - bool pending_hotplug_connected; - bool valid_mode; + struct dcp_mode_state mode_state; /* One HPD pulse after a placeholder EDID, per Type-C connection. */ bool placeholder_retried; u64 typec_generation; /* hpd_mutex: identifies the current connection */ @@ -333,6 +330,7 @@ struct apple_dcp { }; void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); +void dcp_handle_hotplug_actions(struct apple_dcp *dcp, unsigned int action); int dcp_backlight_register(struct apple_dcp *dcp); int dcp_backlight_update(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8343d00ebfd6f..439adf0d6e057 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1257,7 +1257,8 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commi crtc_state = drm_atomic_get_new_crtc_state(commit, crtc); - needs_modeset = drm_atomic_crtc_needs_modeset(crtc_state) || !dcp->valid_mode; + needs_modeset = drm_atomic_crtc_needs_modeset(crtc_state) || + !READ_ONCE(dcp->mode_state.valid); if (!needs_modeset && (!dcp->connector || !dcp->connector->connected)) { /* * Resume restores the mode before the firmware reports the @@ -2408,6 +2409,7 @@ static int dcp_platform_probe(struct platform_device *pdev) */ mutex_init(&dcp->hpd_mutex); mutex_init(&dcp->tb_lock); + spin_lock_init(&dcp->mode_state.lock); dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); dcp->fixed_connector_type = dcp_connector_type_from_dt(dev->of_node); dcp->connector_type = dcp->fixed_connector_type; diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 8531b51ac550a..45fa08ec7d1b5 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -190,7 +190,7 @@ int dcp_backlight_update(struct apple_dcp *dcp) * defer this hopefully until it becomes irrelevant due to proper * drm integrated backlight handling */ - if (!dcp->valid_mode) + if (!READ_ONCE(dcp->mode_state.valid)) return 0; /* Wait 1 vblank cycle in the hope an atomic swap has already updated diff --git a/drivers/gpu/drm/apple/iomfb-state-test.c b/drivers/gpu/drm/apple/iomfb-state-test.c new file mode 100644 index 0000000000000..d751625a94058 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-state-test.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "iomfb-state.h" + +static void dcp_mode_success_test(struct kunit *test) +{ + struct dcp_mode_state state = { }; + bool connected = true; + + spin_lock_init(&state.lock); + dcp_mode_begin(&state); + KUNIT_EXPECT_FALSE(test, state.valid); + KUNIT_EXPECT_EQ(test, dcp_mode_finish(&state, true, &connected), 0U); + KUNIT_EXPECT_TRUE(test, state.valid); + KUNIT_EXPECT_FALSE(test, state.changing); +} + +static void dcp_mode_failure_test(struct kunit *test) +{ + struct dcp_mode_state state = { .valid = true }; + bool connected = true; + + spin_lock_init(&state.lock); + dcp_mode_begin(&state); + dcp_mode_finish(&state, false, &connected); + KUNIT_EXPECT_FALSE(test, state.valid); + KUNIT_EXPECT_FALSE(test, state.changing); +} + +static void dcp_mode_deferred_disconnect_test(struct kunit *test) +{ + struct dcp_mode_state state = { }; + bool connected = true; + + spin_lock_init(&state.lock); + dcp_mode_begin(&state); + KUNIT_EXPECT_EQ(test, dcp_mode_hotplug(&state, false, &connected), 0U); + KUNIT_EXPECT_TRUE(test, connected); + KUNIT_EXPECT_EQ(test, dcp_mode_finish(&state, true, &connected), + (unsigned int)(DCP_HOTPLUG_VBLANK | DCP_HOTPLUG_NOTIFY)); + KUNIT_EXPECT_FALSE(test, connected); + KUNIT_EXPECT_FALSE(test, state.valid); + KUNIT_EXPECT_FALSE(test, state.hotplug_pending); +} + +static void dcp_mode_deferred_connect_test(struct kunit *test) +{ + struct dcp_mode_state state = { }; + bool connected = false; + + spin_lock_init(&state.lock); + dcp_mode_begin(&state); + dcp_mode_hotplug(&state, true, &connected); + KUNIT_EXPECT_EQ(test, dcp_mode_finish(&state, true, &connected), + (unsigned int)DCP_HOTPLUG_NOTIFY); + KUNIT_EXPECT_TRUE(test, connected); + KUNIT_EXPECT_FALSE(test, state.valid); + /* A recovery modeset without another link change can now succeed. */ + dcp_mode_begin(&state); + dcp_mode_finish(&state, true, &connected); + KUNIT_EXPECT_TRUE(test, state.valid); +} + +static void dcp_mode_transient_disconnect_test(struct kunit *test) +{ + struct dcp_mode_state state = { }; + bool connected = true; + + spin_lock_init(&state.lock); + dcp_mode_begin(&state); + dcp_mode_hotplug(&state, false, &connected); + dcp_mode_hotplug(&state, true, &connected); + KUNIT_EXPECT_EQ(test, dcp_mode_finish(&state, true, &connected), + (unsigned int)DCP_HOTPLUG_NOTIFY); + KUNIT_EXPECT_TRUE(test, connected); + KUNIT_EXPECT_FALSE(test, state.valid); +} + +static void dcp_mode_invalidation_order_test(struct kunit *test) +{ + struct dcp_mode_state state = { }; + bool connected = true; + + spin_lock_init(&state.lock); + /* Invalidation just before completion must win. */ + dcp_mode_begin(&state); + dcp_mode_invalidate(&state); + dcp_mode_finish(&state, true, &connected); + KUNIT_EXPECT_FALSE(test, state.valid); + /* So must invalidation just after completion. */ + dcp_mode_begin(&state); + dcp_mode_finish(&state, true, &connected); + dcp_mode_invalidate(&state); + KUNIT_EXPECT_FALSE(test, state.valid); +} + +static void dcp_mode_disconnected_test(struct kunit *test) +{ + struct dcp_mode_state state = { }; + bool connected = false; + + spin_lock_init(&state.lock); + dcp_mode_begin(&state); + dcp_mode_finish(&state, true, &connected); + KUNIT_EXPECT_FALSE(test, state.valid); +} + +static struct kunit_case dcp_mode_cases[] = { + KUNIT_CASE(dcp_mode_success_test), + KUNIT_CASE(dcp_mode_failure_test), + KUNIT_CASE(dcp_mode_deferred_disconnect_test), + KUNIT_CASE(dcp_mode_deferred_connect_test), + KUNIT_CASE(dcp_mode_transient_disconnect_test), + KUNIT_CASE(dcp_mode_invalidation_order_test), + KUNIT_CASE(dcp_mode_disconnected_test), + { } +}; + +static struct kunit_suite dcp_mode_suite = { + .name = "apple-iomfb-mode-state", + .test_cases = dcp_mode_cases, +}; + +kunit_test_suite(dcp_mode_suite); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/iomfb-state.h b/drivers/gpu/drm/apple/iomfb-state.h new file mode 100644 index 0000000000000..a9ada00a8528b --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-state.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __APPLE_IOMFB_STATE_H__ +#define __APPLE_IOMFB_STATE_H__ + +#include + +/* Shared by the atomic commit worker and the firmware callback thread. */ +struct dcp_mode_state { + spinlock_t lock; /* Protects all state and hotplug connector updates. */ + bool valid; + bool changing; + bool invalidated; + bool hotplug_pending; + bool connected_pending; +}; + +enum dcp_hotplug_action { + DCP_HOTPLUG_NONE = 0, + DCP_HOTPLUG_VBLANK = BIT(0), + DCP_HOTPLUG_NOTIFY = BIT(1), +}; + +static inline void dcp_mode_begin(struct dcp_mode_state *state) +{ + unsigned long flags; + + spin_lock_irqsave(&state->lock, flags); + state->valid = false; + state->changing = true; + state->invalidated = false; + state->hotplug_pending = false; + spin_unlock_irqrestore(&state->lock, flags); +} + +static inline void dcp_mode_invalidate(struct dcp_mode_state *state) +{ + unsigned long flags; + + spin_lock_irqsave(&state->lock, flags); + state->valid = false; + state->invalidated = true; + spin_unlock_irqrestore(&state->lock, flags); +} + +/* The caller holds state->lock. */ +static inline unsigned int +dcp_mode_apply_hotplug(struct dcp_mode_state *state, bool connected, + bool *connector_connected) +{ + unsigned int action = DCP_HOTPLUG_NONE; + + if (!connected) { + state->valid = false; + action |= DCP_HOTPLUG_VBLANK; + } + if (connector_connected && *connector_connected != connected) { + *connector_connected = connected; + state->valid = false; + action |= DCP_HOTPLUG_NOTIFY; + } + return action; +} + +static inline unsigned int +dcp_mode_hotplug(struct dcp_mode_state *state, bool connected, + bool *connector_connected) +{ + unsigned long flags; + unsigned int action = DCP_HOTPLUG_NONE; + + spin_lock_irqsave(&state->lock, flags); + if (state->changing) { + state->hotplug_pending = true; + state->connected_pending = connected; + /* A later connect must not erase a link loss during this call. */ + if (!connected) + state->invalidated = true; + } else { + action = dcp_mode_apply_hotplug(state, connected, + connector_connected); + } + spin_unlock_irqrestore(&state->lock, flags); + return action; +} + +static inline unsigned int +dcp_mode_finish(struct dcp_mode_state *state, bool completed, + bool *connector_connected) +{ + unsigned long flags; + unsigned int action = DCP_HOTPLUG_NONE; + + spin_lock_irqsave(&state->lock, flags); + state->valid = completed && !state->invalidated; + state->changing = false; + if (state->hotplug_pending) { + state->hotplug_pending = false; + action = dcp_mode_apply_hotplug(state, state->connected_pending, + connector_connected); + } + if (connector_connected && !*connector_connected) + state->valid = false; + /* A transient unplug also needs recovery if the final state is up. */ + if (state->invalidated && connector_connected && *connector_connected) + action |= DCP_HOTPLUG_NOTIFY; + spin_unlock_irqrestore(&state->lock, flags); + return action; +} + +#endif diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 4fe1565d2d446..5ab5961042ff0 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -254,11 +254,19 @@ static int dcp_retrain_active_crtc(struct apple_connector *connector) return ret; } +void dcp_handle_hotplug_actions(struct apple_dcp *dcp, unsigned int action) +{ + if (action & DCP_HOTPLUG_VBLANK) + schedule_work(&dcp->vblank_wq); + if ((action & DCP_HOTPLUG_NOTIFY) && dcp->connector) + schedule_work(&dcp->connector->hotplug_wq); +} + void dcp_retrain_oob(struct apple_connector *connector) { struct apple_dcp *dcp = platform_get_drvdata(connector->dcp); - if (!READ_ONCE(connector->connected) || !READ_ONCE(dcp->valid_mode)) + if (!READ_ONCE(connector->connected)) return; /* @@ -267,7 +275,7 @@ void dcp_retrain_oob(struct apple_connector *connector) * use the normal hotplug worker to replay the active CRTC from process * context; sending a synthetic disconnect would tear down the connector. */ - WRITE_ONCE(dcp->valid_mode, false); + dcp_mode_invalidate(&dcp->mode_state); schedule_work(&connector->hotplug_wq); } @@ -281,7 +289,7 @@ void dcp_hotplug(struct work_struct *work) dcp = platform_get_drvdata(connector->dcp); dev_info(dcp->dev, "%s() connected:%d valid_mode:%d nr_modes:%u\n", __func__, - connector->connected, dcp->valid_mode, dcp->nr_modes); + connector->connected, READ_ONCE(dcp->mode_state.valid), dcp->nr_modes); if (!connector->connected) { drm_edid_free(connector->drm_edid); @@ -293,7 +301,7 @@ void dcp_hotplug(struct work_struct *work) * display modes from atomic_flush, so userspace needs to trigger a * flush, or the CRTC gets no signal. */ - if (connector->base.state && !dcp->valid_mode && connector->connected && + if (connector->base.state && !READ_ONCE(dcp->mode_state.valid) && connector->connected && !(dcp_is_usb4_output(dcp) && of_machine_is_compatible("apple,j416s"))) { drm_connector_set_link_status_property(&connector->base, DRM_MODE_LINK_STATUS_BAD); @@ -505,6 +513,7 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, struct apple_crtc *apple_crtc = to_apple_crtc(crtc); struct apple_dcp *dcp = platform_get_drvdata(apple_crtc->dcp); struct drm_crtc_state *crtc_state; + unsigned int action; int ret = -EIO; bool modeset; @@ -512,7 +521,8 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, if (!crtc_state) return 0; - modeset = drm_atomic_crtc_needs_modeset(crtc_state) || !dcp->valid_mode; + modeset = drm_atomic_crtc_needs_modeset(crtc_state) || + !READ_ONCE(dcp->mode_state.valid); if (!modeset) return 0; @@ -521,6 +531,7 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, if (crtc_state->mode.hdisplay == 0 && crtc_state->mode.vdisplay == 0) return 0; + dcp_mode_begin(&dcp->mode_state); switch (dcp->fw_compat) { case DCP_FIRMWARE_V_12_3: ret = iomfb_modeset_v12_3(dcp, crtc_state); @@ -537,6 +548,10 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, break; } + action = dcp_mode_finish(&dcp->mode_state, !ret, dcp->connector ? + &dcp->connector->connected : NULL); + dcp_handle_hotplug_actions(dcp, action); + return ret; } @@ -568,7 +583,7 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit) * re-runs the modeset, which marks the mode valid again, before the * firmware has reported the display back. */ - if (!dcp->valid_mode || !dcp->connector || !dcp->connector->connected) { + if (!READ_ONCE(dcp->mode_state.valid) || !dcp->connector || !dcp->connector->connected) { schedule_work(&dcp->vblank_wq); return; } @@ -658,7 +673,7 @@ void iomfb_shutdown(struct apple_dcp *dcp) { /* We're going down */ dcp->active = false; - dcp->valid_mode = false; + dcp_mode_invalidate(&dcp->mode_state); switch (dcp->fw_compat) { case DCP_FIRMWARE_V_12_3: diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index fab39b5499b73..44723bdb95420 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1038,6 +1038,7 @@ void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp) static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) { struct apple_connector *connector = dcp->connector; + unsigned int action; /* DCP issues hotplug_gated callbacks after SetPowerState() calls on * devices with display (macbooks, imacs). This must not result in @@ -1063,42 +1064,14 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) if (!(*connected) && READ_ONCE(dcp->typec_crtc_off) && READ_ONCE(dcp->typec_cable_connected)) { dev_dbg(dcp->dev, "cb_hotplug() ignoring unplug of powered-off Type-C output\n"); - dcp->valid_mode = false; + dcp_mode_invalidate(&dcp->mode_state); schedule_work(&dcp->vblank_wq); return; } - if (dcp->during_modeset) { - /* - * Remember it rather than dropping it. Resume re-runs the - * modeset and the firmware reports the display back while that - * is still in flight; discarding it leaves the connector marked - * disconnected forever, with no further event to correct it. - */ - dev_info(dcp->dev, - "cb_hotplug() deferred during modeset connected:%llu\n", - *connected); - dcp->pending_hotplug = true; - dcp->pending_hotplug_connected = !!(*connected); - return; - } - - dev_info(dcp->dev, "cb_hotplug() connected:%llu, valid_mode:%d nr_modes:%u\n", - *connected, dcp->valid_mode, dcp->nr_modes); - - /* Hotplug invalidates mode. DRM doesn't always handle this. */ - if (!(*connected)) { - dcp->valid_mode = false; - /* after unplug swap will not complete until the next - * set_digital_out_mode */ - schedule_work(&dcp->vblank_wq); - } - - if (connector && connector->connected != !!(*connected)) { - connector->connected = !!(*connected); - dcp->valid_mode = false; - schedule_work(&connector->hotplug_wq); - } + action = dcp_mode_hotplug(&dcp->mode_state, !!(*connected), + connector ? &connector->connected : NULL); + dcp_handle_hotplug_actions(dcp, action); } static void @@ -1230,7 +1203,8 @@ static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) { struct dcp_swap_start_req start_req = { 0 }; - if (dcp->connector && dcp->connector->connected) + if (READ_ONCE(dcp->mode_state.valid) && dcp->connector && + READ_ONCE(dcp->connector->connected)) dcp_swap_start(dcp, false, &start_req, dcp_swap_started, NULL); else dcp_drm_crtc_vblank(dcp->crtc); @@ -1328,7 +1302,6 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, /* increase refcount to ensure the receiver has a reference */ kref_get(&cookie->refcount); - dcp->during_modeset = true; dcp->swap_submit_timestamp = 0; if (mode->vrr) @@ -1347,25 +1320,6 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(8500)); - dcp->during_modeset = false; - - if (dcp->pending_hotplug) { - bool connected = dcp->pending_hotplug_connected; - struct apple_connector *connector = dcp->connector; - - dcp->pending_hotplug = false; - dev_info(dcp->dev, "replaying deferred hotplug connected:%d\n", - connected); - - if (!connected) - dcp->valid_mode = false; - - if (connector && connector->connected != connected) { - connector->connected = connected; - dcp->valid_mode = false; - schedule_work(&connector->hotplug_wq); - } - } dev_info(dcp->dev, "set_digital_out_mode finished:%d\n", ret); if (ret == 0) { @@ -1383,7 +1337,6 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, jiffies_to_msecs(ret)); } kref_put(&cookie->refcount, release_wait_cookie); - dcp->valid_mode = true; dcp->vrr_enabled = mode->vrr && crtc_state->vrr_enabled; return 0; From ebc6194c47a69f34d2072931f3973a54eaaf340b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:21:51 -0700 Subject: [PATCH 1111/2680] drm/apple: pin DP service owners before recycling retired AFK slots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Repeated DP service announcements consumed the fixed service table even when earlier services had been torn down. Reuse converted service slots only after owner pins and pending commands have drained, and retain retired channels for late command replies. Keep the command tag sequence when reclaiming a slot. Pin the EDID service while accessing it, serialize owner lookup with teardown, and discard a result if the service changed during the read. Keep services with unconverted owners nonrecyclable. Gate receive work against queue destruction and clean up failed DP endpoint startup. Add six KUnit cases covering repeated generations, reader pins, pending and timed-out commands, completed-but-unconsumed replies, tag continuity and owners that have not opted in to reuse. [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/Makefile | 1 + drivers/gpu/drm/apple/Kconfig | 3 +- drivers/gpu/drm/apple/afk-service-test.c | 142 +++++++++++++++++++++++ drivers/gpu/drm/apple/afk.c | 118 +++++++++++++------ drivers/gpu/drm/apple/afk.h | 90 ++++++++++++++ drivers/gpu/drm/apple/av.c | 4 +- drivers/gpu/drm/apple/dcp.c | 2 + drivers/gpu/drm/apple/epic/dpavservep.c | 78 +++++++++++-- drivers/gpu/drm/apple/epic/dpavservep.h | 6 +- drivers/gpu/drm/apple/iomfb.c | 4 +- 10 files changed, 396 insertions(+), 52 deletions(-) create mode 100644 drivers/gpu/drm/apple/afk-service-test.c diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index 8bc9a05484f21..200ff122fee5c 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -229,6 +229,7 @@ obj-$(CONFIG_DRM_PANTHOR) += panthor/ obj-$(CONFIG_DRM_TYR) += tyr/ obj-$(CONFIG_DRM_APPLE) += apple/ obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple/iomfb-state-test.o +obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple/afk-service-test.o obj-$(CONFIG_DRM_ASPEED_GFX) += aspeed/ obj-$(CONFIG_DRM_MCDE) += mcde/ obj-$(CONFIG_DRM_TIDSS) += tidss/ diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 65c7fb2256344..a0a597dc0355e 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -37,4 +37,5 @@ config DRM_APPLE_KUNIT_TEST Test modeset and hotplug state transitions without display hardware, including link changes that race with a modeset and recovery after failed modesets. These tests can run on other architectures and do - not require the Apple display driver to be enabled. + not require the Apple display driver to be enabled. Also test service + ownership and pending commands across disconnect and reannouncement. diff --git a/drivers/gpu/drm/apple/afk-service-test.c b/drivers/gpu/drm/apple/afk-service-test.c new file mode 100644 index 0000000000000..4a5f07fa9b15f --- /dev/null +++ b/drivers/gpu/drm/apple/afk-service-test.c @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "afk.h" + +static const struct apple_epic_service_ops reusable_ops = { + .name = "reusable", + .reusable = true, +}; + +static const struct apple_epic_service_ops permanent_ops = { + .name = "permanent", +}; + +static int afk_service_test_init(struct kunit *test) +{ + struct apple_epic_service *service; + + service = kunit_kzalloc(test, sizeof(*service), GFP_KERNEL); + if (!service) + return -ENOMEM; + spin_lock_init(&service->lock); + test->priv = service; + return 0; +} + +static void afk_test_retire(struct apple_epic_service *service) +{ + unsigned long flags; + + spin_lock_irqsave(&service->lock, flags); + service->enabled = false; + service->torndown = true; + spin_unlock_irqrestore(&service->lock, flags); +} + +static void afk_service_many_generations_test(struct kunit *test) +{ + struct apple_epic_service *service = test->priv; + unsigned int i; + + /* Repeated unplug/replug must not consume an unbounded slot budget. */ + for (i = 0; i < 4 * AFK_MAX_CHANNEL; i++) { + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, i)); + KUNIT_EXPECT_EQ(test, service->channel, i); + KUNIT_EXPECT_TRUE(test, afk_service_matches(service, i)); + afk_test_retire(service); + } +} + +static void afk_service_reader_pin_test(struct kunit *test) +{ + struct apple_epic_service *service = test->priv; + + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 1)); + KUNIT_ASSERT_PTR_EQ(test, afk_service_get(service), service); /* owner */ + KUNIT_ASSERT_PTR_EQ(test, afk_service_get(service), service); /* reader */ + afk_test_retire(service); + afk_service_put(service); /* owner detaches while EDID is being read */ + KUNIT_EXPECT_FALSE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); + KUNIT_EXPECT_EQ(test, service->channel, 1U); + KUNIT_EXPECT_PTR_EQ(test, afk_service_get(service), NULL); + afk_service_put(service); + KUNIT_EXPECT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); +} + +static void afk_service_pending_reply_test(struct kunit *test) +{ + struct apple_epic_service *service = test->priv; + + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 1)); + set_bit(3, service->cmd_map); + afk_test_retire(service); + KUNIT_EXPECT_FALSE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); + KUNIT_EXPECT_TRUE(test, afk_service_matches(service, 1)); + KUNIT_EXPECT_FALSE(test, afk_service_matches(service, 2)); + /* A timed-out command retains its DMA and slot until the late reply. */ + service->cmds[3].free_on_ack = true; + KUNIT_EXPECT_FALSE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); + clear_bit(3, service->cmd_map); + KUNIT_EXPECT_FALSE(test, afk_service_matches(service, 1)); + KUNIT_EXPECT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); +} + +static void afk_service_completed_reader_test(struct kunit *test) +{ + struct apple_epic_service *service = test->priv; + + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 1)); + KUNIT_ASSERT_PTR_EQ(test, afk_service_get(service), service); + set_bit(0, service->cmd_map); + afk_test_retire(service); + /* Reply arrived, but the waiting thread has not consumed it yet. */ + service->cmds[0].done = true; + KUNIT_EXPECT_FALSE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); + clear_bit(0, service->cmd_map); + KUNIT_EXPECT_FALSE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); + afk_service_put(service); + KUNIT_EXPECT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); +} + +static void afk_service_tag_sequence_test(struct kunit *test) +{ + struct apple_epic_service *service = test->priv; + + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 1)); + service->cmd_tag = 0xa9; + service->cookie = test; + afk_test_retire(service); + KUNIT_EXPECT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); + KUNIT_EXPECT_EQ(test, service->cmd_tag, (u8)0xa9); + KUNIT_EXPECT_PTR_EQ(test, service->cookie, NULL); +} + +static void afk_service_unconverted_owner_test(struct kunit *test) +{ + struct apple_epic_service *service = test->priv; + + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &permanent_ops, 1)); + afk_test_retire(service); + KUNIT_EXPECT_FALSE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); +} + +static struct kunit_case afk_service_cases[] = { + KUNIT_CASE(afk_service_many_generations_test), + KUNIT_CASE(afk_service_reader_pin_test), + KUNIT_CASE(afk_service_pending_reply_test), + KUNIT_CASE(afk_service_completed_reader_test), + KUNIT_CASE(afk_service_tag_sequence_test), + KUNIT_CASE(afk_service_unconverted_owner_test), + { } +}; + +static struct kunit_suite afk_service_suite = { + .name = "apple-afk-service-lifetime", + .init = afk_service_test_init, + .test_cases = afk_service_cases, +}; + +kunit_test_suite(afk_service_suite); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 3fbc5fe4d601c..de2a83a9fcb60 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -57,7 +57,7 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, const struct apple_epic_service_ops *ops) { struct apple_dcp_afkep *afkep; - int ret; + int ret, i; afkep = devm_kzalloc(dcp->dev, sizeof(*afkep), GFP_KERNEL); if (!afkep) @@ -78,6 +78,8 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, init_completion(&afkep->started); init_completion(&afkep->stopped); spin_lock_init(&afkep->lock); + for (i = 0; i < AFK_MAX_CHANNEL; i++) + spin_lock_init(&afkep->services[i].lock); return afkep; @@ -88,6 +90,8 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, void afk_shutdown(struct apple_dcp_afkep *afkep) { + unsigned long flags; + afk_send(afkep, FIELD_PREP(RBEP_TYPE, RBEP_SHUTDOWN)); int ret; @@ -96,6 +100,9 @@ void afk_shutdown(struct apple_dcp_afkep *afkep) dev_err(afkep->dcp->dev, "Timed out shutting down AFK endpoint %02x", afkep->endpoint); } + spin_lock_irqsave(&afkep->lock, flags); + afkep->stopping = true; + spin_unlock_irqrestore(&afkep->lock, flags); destroy_workqueue(afkep->wq); } @@ -213,7 +220,7 @@ afk_match_service(struct apple_dcp_afkep *ep, const char *name) { const struct apple_epic_service_ops *ops; - if (!name[0]) + if (!name || !name[0]) return NULL; if (!ep->ops) return NULL; @@ -229,13 +236,18 @@ afk_match_service(struct apple_dcp_afkep *ep, const char *name) } static struct apple_epic_service *afk_epic_find_service(struct apple_dcp_afkep *ep, - u32 channel) + u32 channel) { - for (u32 i = 0; i < ep->num_channels; i++) - if (ep->services[i].enabled && ep->services[i].channel == channel) - return &ep->services[i]; + struct apple_epic_service *service; + u32 i; + + for (i = 0; i < ep->num_channels; i++) { + service = &ep->services[i]; + if (afk_service_matches(service, channel)) + return service; + } - return NULL; + return NULL; } static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, @@ -243,6 +255,7 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, { char name[32]; s64 epic_unit = -1; + struct apple_epic_service *service = NULL; u32 ch_idx; const char *service_name = name; const char *epic_name = NULL, *epic_class = NULL; @@ -251,7 +264,8 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, u8 *props = payload + sizeof(name); size_t props_size = payload_size - sizeof(name); - WARN_ON(afk_epic_find_service(ep, channel)); + if (afk_epic_find_service(ep, channel)) + return; if (payload_size < sizeof(name)) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: payload too small: %lx\n", @@ -259,12 +273,6 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, return; } - if (ep->num_channels >= AFK_MAX_CHANNEL) { - dev_err(ep->dcp->dev, "AFK[ep:%02x]: too many enabled services!\n", - ep->endpoint); - return; - } - strscpy(name, payload, sizeof(name)); /* @@ -305,19 +313,25 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, goto free; } - ch_idx = ep->num_channels++; - spin_lock_init(&ep->services[ch_idx].lock); - ep->services[ch_idx].enabled = true; - ep->services[ch_idx].torndown = false; - ep->services[ch_idx].ops = ops; - ep->services[ch_idx].ep = ep; - ep->services[ch_idx].channel = channel; - ep->services[ch_idx].cmd_tag = 0; - ops->init(&ep->services[ch_idx], epic_name, epic_class, epic_unit); + for (ch_idx = 0; ch_idx < AFK_MAX_CHANNEL; ch_idx++) { + struct apple_epic_service *candidate = &ep->services[ch_idx]; + + if (afk_service_reinit(candidate, ep, ops, channel)) { + service = candidate; + break; + } + } + if (!service) { + dev_err(ep->dcp->dev, "AFK[ep:%02x]: no reusable service slots\n", + ep->endpoint); + goto free; + } + ep->num_channels = max(ep->num_channels, ch_idx + 1); + ops->init(service, epic_name, epic_class, epic_unit); dev_info(ep->dcp->dev, "AFK[ep:%02x]: new service %s on channel %d\n", ep->endpoint, service_name, channel); - afk_populate_service_debugfs(&ep->services[ch_idx]); + afk_populate_service_debugfs(service); free: kfree(epic_name); @@ -339,14 +353,14 @@ static void afk_recv_handle_teardown(struct apple_dcp_afkep *ep, u32 channel) afk_remove_service_debugfs(service); - // TODO: think through what locking is necessary spin_lock_irqsave(&service->lock, flags); - /* - * teardown must not disable the service since since it may be sent as - * side effect of a COMMAND which for which a reply is expected. - * Seen with DCP's "av" endpoint during the close afk_service_call. - */ + if (service->torndown) { + spin_unlock_irqrestore(&service->lock, flags); + return; + } + /* Outstanding commands remain discoverable until their replies arrive. */ service->torndown = true; + service->enabled = false; ops = service->ops; spin_unlock_irqrestore(&service->lock, flags); @@ -387,7 +401,7 @@ static void afk_recv_handle_reply(struct apple_dcp_afkep *ep, u32 channel, } spin_lock_irqsave(&service->lock, flags); - if (service->cmds[idx].done) { + if (!test_bit(idx, service->cmd_map) || service->cmds[idx].done) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: command reply on channel %d already handled\n", ep->endpoint, channel); @@ -732,6 +746,10 @@ static void afk_receive_message_worker(struct work_struct *work_) int afk_receive_message(struct apple_dcp_afkep *ep, u64 message) { struct afk_receive_message_work *work; + unsigned long flags; + + if (!ep) + return -ENODEV; // TODO: comment why decoupling from rtkit thread is required here work = kzalloc(sizeof(*work), GFP_KERNEL); @@ -741,7 +759,14 @@ int afk_receive_message(struct apple_dcp_afkep *ep, u64 message) work->ep = ep; work->message = message; INIT_WORK(&work->work, afk_receive_message_worker); + spin_lock_irqsave(&ep->lock, flags); + if (ep->stopping) { + spin_unlock_irqrestore(&ep->lock, flags); + kfree(work); + return -ESHUTDOWN; + } queue_work(ep->wq, &work->work); + spin_unlock_irqrestore(&ep->lock, flags); return 0; } @@ -759,6 +784,10 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, int ret; spin_lock_irqsave(&ep->lock, flags); + if (ep->stopping) { + ret = -ESHUTDOWN; + goto out; + } dma_rmb(); rptr = le32_to_cpu(ep->txbfr.hdr->rptr); @@ -910,13 +939,20 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, unsigned long flags; int ret, idx; u16 tag; - struct apple_dcp_afkep *ep = service->ep; + struct apple_dcp_afkep *ep; DECLARE_COMPLETION_ONSTACK(completion); + service = afk_service_get(service); + if (!service) + return -ENODEV; + ep = service->ep; + rxbuf = dma_alloc_coherent(ep->dcp->dev, output_len, &rxbuf_dma, GFP_KERNEL); - if (!rxbuf) - return -ENOMEM; + if (!rxbuf) { + ret = -ENOMEM; + goto err_put_service; + } txbuf = dma_alloc_coherent(ep->dcp->dev, payload_len, &txbuf_dma, GFP_KERNEL); if (!txbuf) { @@ -934,6 +970,10 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, cmd.txlen = cpu_to_le32(payload_len); spin_lock_irqsave(&service->lock, flags); + if (service->torndown || !service->enabled) { + ret = -ENODEV; + goto err_unlock; + } idx = bitmap_find_free_region(service->cmd_map, MAX_PENDING_CMDS, 0); if (idx < 0) { ret = -ENOSPC; @@ -956,11 +996,10 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, service->cmds[idx].completion = &completion; init_completion(&completion); - spin_unlock_irqrestore(&service->lock, flags); - ret = afk_send_epic(service->ep, service->channel, tag, EPIC_TYPE_COMMAND, EPIC_CAT_COMMAND, type, &cmd, sizeof(cmd)); + spin_unlock_irqrestore(&service->lock, flags); if (ret) goto err_free_cmd; @@ -982,6 +1021,7 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, "AFK[ep:%02x]: %s (chan:%u) command type 0x%x tag 0x%04x timed out after %u ms\n", ep->endpoint, service->ops->name, service->channel, type, tag, timeout_ms); + afk_service_put(service); return -ETIMEDOUT; } spin_unlock_irqrestore(&service->lock, flags); @@ -995,12 +1035,15 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, err_free_cmd: spin_lock_irqsave(&service->lock, flags); + service->cmds[idx].completion = NULL; bitmap_release_region(service->cmd_map, idx, 0); err_unlock: spin_unlock_irqrestore(&service->lock, flags); dma_free_coherent(ep->dcp->dev, payload_len, txbuf, txbuf_dma); err_free_rxbuf: dma_free_coherent(ep->dcp->dev, output_len, rxbuf, rxbuf_dma); +err_put_service: + afk_service_put(service); return ret; } @@ -1026,6 +1069,9 @@ int afk_service_call_timeout(struct apple_epic_service *service, u16 group, u32 retcode; u32 retlen; + if (!service) + return -ENODEV; + bfr = kzalloc(bfr_len, GFP_KERNEL); if (!bfr) return -ENOMEM; diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index d63d26b79d457..880987cbb6ef4 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -7,7 +7,9 @@ #ifndef _DRM_APPLE_DCP_AFK_H #define _DRM_APPLE_DCP_AFK_H +#include #include +#include #include #include @@ -43,6 +45,8 @@ struct apple_epic_service { DECLARE_BITMAP(cmd_map, MAX_PENDING_CMDS); u8 cmd_tag; spinlock_t lock; + /* Pins held by owners and synchronous callers; protected by lock. */ + unsigned int users; u32 channel; bool enabled; @@ -60,6 +64,8 @@ enum epic_subtype; struct apple_epic_service_ops { const char name[32]; + /* All retained pointers must be pinned before enabling slot reuse. */ + bool reusable; void (*init)(struct apple_epic_service *service, const char *name, const char *class, s64 unit); @@ -71,6 +77,89 @@ struct apple_epic_service_ops { void (*teardown)(struct apple_epic_service *service); }; +static inline struct apple_epic_service *afk_service_get(struct apple_epic_service *service) +{ + unsigned long flags; + bool available; + + if (!service) + return NULL; + + spin_lock_irqsave(&service->lock, flags); + available = service->enabled && !service->torndown; + if (available) + service->users++; + spin_unlock_irqrestore(&service->lock, flags); + + return available ? service : NULL; +} + +static inline void afk_service_put(struct apple_epic_service *service) +{ + unsigned long flags; + + spin_lock_irqsave(&service->lock, flags); + if (!WARN_ON(!service->users)) + service->users--; + spin_unlock_irqrestore(&service->lock, flags); +} + +static inline void afk_service_disable(struct apple_epic_service *service) +{ + unsigned long flags; + + spin_lock_irqsave(&service->lock, flags); + service->enabled = false; + spin_unlock_irqrestore(&service->lock, flags); +} + +static inline bool afk_service_matches(struct apple_epic_service *service, + u32 channel) +{ + unsigned long flags; + bool found; + + spin_lock_irqsave(&service->lock, flags); + /* Teardown does not cancel commands or relinquish their DMA. */ + found = service->channel == channel && + (service->enabled || + !bitmap_empty(service->cmd_map, MAX_PENDING_CMDS)); + spin_unlock_irqrestore(&service->lock, flags); + return found; +} + +/* The caller holds service->lock. */ +static inline bool afk_service_reclaimable(struct apple_epic_service *service) +{ + return !service->ops || + (service->ops->reusable && service->torndown && + !service->enabled && !service->users && + bitmap_empty(service->cmd_map, MAX_PENDING_CMDS)); +} + +static inline bool +afk_service_reinit(struct apple_epic_service *service, + struct apple_dcp_afkep *ep, + const struct apple_epic_service_ops *ops, u32 channel) +{ + unsigned long flags; + bool reusable; + + spin_lock_irqsave(&service->lock, flags); + reusable = afk_service_reclaimable(service); + if (reusable) { + service->enabled = true; + service->torndown = false; + service->ops = ops; + service->ep = ep; + service->channel = channel; + service->cookie = NULL; + /* Keep the command-tag sequence across service generations. */ + } + spin_unlock_irqrestore(&service->lock, flags); + return reusable; +} + struct afk_ringbuffer_header { __le32 bufsz; u32 unk; @@ -177,6 +266,7 @@ struct apple_dcp_afkep { spinlock_t lock; u16 qe_seq; + bool stopping; /* lock: no new receive work after shutdown. */ const struct apple_epic_service_ops *ops; struct apple_epic_service services[AFK_MAX_CHANNEL]; diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c index ca4612df8696f..13fdd068daa67 100644 --- a/drivers/gpu/drm/apple/av.c +++ b/drivers/gpu/drm/apple/av.c @@ -79,7 +79,7 @@ static void av_interface_teardown(struct apple_epic_service *service) struct apple_dcp *dcp = service->ep->dcp; struct audiosrv_data *asrv = dcp->audiosrv; - service->enabled = false; + afk_service_disable(service); mutex_lock(&asrv->plug_lock); @@ -351,7 +351,7 @@ void av_service_disconnect(struct apple_dcp *dcp) dev_err(dcp->dev, "error closing audio service: %d\n", ret); } if (service->torndown) - service->enabled = false; + afk_service_disable(service); asrv->is_open = false; } diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 439adf0d6e057..03e2688f0d14f 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2349,6 +2349,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) if (dcp->dcpavservep) { afk_shutdown(dcp->dcpavservep); + dpavservep_detach(dcp); dcp->dcpavservep = NULL; } @@ -2410,6 +2411,7 @@ static int dcp_platform_probe(struct platform_device *pdev) mutex_init(&dcp->hpd_mutex); mutex_init(&dcp->tb_lock); spin_lock_init(&dcp->mode_state.lock); + spin_lock_init(&dcp->dcpavserv.lock); dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); dcp->fixed_connector_type = dcp_connector_type_from_dt(dev->of_node); dcp->connector_type = dcp->fixed_connector_type; diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 2de9d2fe4c24a..cfcf3e184f5a7 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -18,31 +18,57 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { struct apple_dcp *dcp = service->ep->dcp; + unsigned long flags; + trace_dcpavserv_init(dcp, unit); if (unit == 0 && name && !strcmp(name, "dcpav-service-epic")) { + spin_lock_irqsave(&dcp->dcpavserv.lock, flags); if (dcp->dcpavserv.enabled) { + spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); dev_err(dcp->dev, "DCPAVSERV: unit %lld already exists\n", unit); return; } - dcp->dcpavserv.service = service; + dcp->dcpavserv.service = afk_service_get(service); dcp->dcpavserv.enabled = true; service->cookie = &dcp->dcpavserv; complete(&dcp->dcpavserv.enable_completion); + spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); } } static void dcpavserv_teardown(struct apple_epic_service *service) { struct apple_dcp *dcp = service->ep->dcp; - service->enabled = false; + unsigned long flags; + bool owned = false; - if (dcp->dcpavserv.enabled) { + spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + if (dcp->dcpavserv.service == service) { dcp->dcpavserv.enabled = false; dcp->dcpavserv.service = NULL; - service->cookie = NULL; reinit_completion(&dcp->dcpavserv.enable_completion); + owned = true; + } + spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + if (owned) + afk_service_put(service); +} + +void dpavservep_detach(struct apple_dcp *dcp) +{ + struct apple_epic_service *service; + unsigned long flags; + + spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + service = dcp->dcpavserv.service; + dcp->dcpavserv.service = NULL; + dcp->dcpavserv.enabled = false; + spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + if (service) { + afk_service_disable(service); + afk_service_put(service); } } @@ -53,7 +79,7 @@ static void dcpdpserv_init(struct apple_epic_service *service, const char *name, static void dcpdpserv_teardown(struct apple_epic_service *service) { - service->enabled = false; + afk_service_disable(service); } struct dcpavserv_status_report { @@ -155,7 +181,7 @@ static int dcpdpserv_report(struct apple_epic_service *service, return parse_report(service, type, data, data_size); } -const struct drm_edid *dcpavserv_copy_edid(struct apple_epic_service *service) +static const struct drm_edid *dcpavserv_read_edid(struct apple_epic_service *service) { struct dpavserv_copy_edid_cmd cmd; struct dpavserv_copy_edid_resp *resp __free(kfree) = NULL; @@ -192,15 +218,41 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_epic_service *service) data_size - EDID_LEADING_DATA_SIZE); } +const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp) +{ + struct apple_epic_service *service; + const struct drm_edid *edid; + unsigned long flags; + + spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + service = afk_service_get(dcp->dcpavserv.service); + spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + if (!service) + return ERR_PTR(-ENODEV); + + edid = dcpavserv_read_edid(service); + spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + if (dcp->dcpavserv.service != service) { + if (!IS_ERR(edid)) + drm_edid_free(edid); + edid = ERR_PTR(-ENODEV); + } + spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + afk_service_put(service); + return edid; +} + static const struct apple_epic_service_ops dpavservep_ops[] = { { .name = "dcpav-service-epic", + .reusable = true, .init = dcpavserv_init, .teardown = dcpavserv_teardown, .report = dcpavserv_report, }, { .name = "dcpdp-service-epic", + .reusable = true, .init = dcpdpserv_init, .teardown = dcpdpserv_teardown, .report = dcpdpserv_report, @@ -210,19 +262,25 @@ static const struct apple_epic_service_ops dpavservep_ops[] = { int dpavservep_init(struct apple_dcp *dcp) { + struct apple_dcp_afkep *ep; int ret; init_completion(&dcp->dcpavserv.enable_completion); - dcp->dcpavservep = afk_init(dcp, DPAVSERV_ENDPOINT, dpavservep_ops); - if (IS_ERR(dcp->dcpavservep)) - return PTR_ERR(dcp->dcpavservep); + ep = afk_init(dcp, DPAVSERV_ENDPOINT, dpavservep_ops); + if (IS_ERR(ep)) + return PTR_ERR(ep); + dcp->dcpavservep = ep; dcp->dcpavservep->match_epic_name = true; ret = afk_start(dcp->dcpavservep); - if (ret) + if (ret) { + afk_shutdown(ep); + dpavservep_detach(dcp); + dcp->dcpavservep = NULL; return ret; + } ret = wait_for_completion_timeout(&dcp->dcpavserv.enable_completion, msecs_to_jiffies(1000)); diff --git a/drivers/gpu/drm/apple/epic/dpavservep.h b/drivers/gpu/drm/apple/epic/dpavservep.h index 858ff14b0bd7b..e9ab17a7302c1 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.h +++ b/drivers/gpu/drm/apple/epic/dpavservep.h @@ -5,18 +5,22 @@ #define _DRM_APPLE_EPIC_DPAVSERV_H #include +#include #include struct drm_edid; struct apple_epic_service; +struct apple_dcp; struct dcpavserv { + spinlock_t lock; /* Protects the service pointer and its owner pin. */ bool enabled; struct completion enable_completion; u32 unit; struct apple_epic_service *service; }; -const struct drm_edid *dcpavserv_copy_edid(struct apple_epic_service *service); +const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp); +void dpavservep_detach(struct apple_dcp *dcp); #endif /* _DRM_APPLE_EPIC_DPAVSERV_H */ diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 5ab5961042ff0..24a3cbb471e12 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -446,10 +446,10 @@ int dcp_get_modes(struct drm_connector *connector) } drm_connector_set_vrr_capable_property(connector, vrr_capable); - if (dcp->nr_modes && dcp->dcpavserv.enabled && + if (dcp->nr_modes && !apple_connector->drm_edid) { const struct drm_edid *edid; - edid = dcpavserv_copy_edid(dcp->dcpavserv.service); + edid = dcpavserv_copy_edid(dcp); if (IS_ERR_OR_NULL(edid)) { dev_info(dcp->dev, "copy_edid failed: %pe\n", edid); } else { From 0c8ca4ab35754126b96af9098e3a8dc474b81acb Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:24:07 -0700 Subject: [PATCH 1112/2680] drm/apple: preallocate the AFK receive dispatch buffer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allocate receive scratch space while initializing the ring so dispatch cannot fail allocation after a receive notification has been consumed. Each endpoint has an ordered worker and dispatch completes synchronously before the next entry is copied. Free scratch only after receive admission has closed and the worker queue has drained. Reject rings smaller than an aligned entry before marking them ready. [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/afk.c | 23 +++++++++++++++++------ drivers/gpu/drm/apple/afk.h | 2 ++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index de2a83a9fcb60..404ff9cf6e7ce 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -104,6 +104,10 @@ void afk_shutdown(struct apple_dcp_afkep *afkep) afkep->stopping = true; spin_unlock_irqrestore(&afkep->lock, flags); destroy_workqueue(afkep->wq); + if (afkep->rx_scratch) { + devm_kfree(afkep->dcp->dev, afkep->rx_scratch); + afkep->rx_scratch = NULL; + } } int afk_start(struct apple_dcp_afkep *ep) @@ -194,6 +198,17 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, ep->endpoint, bufsz, sizeof(*bfr->hdr)); return; } + if (bufsz < BIT(BLOCK_SHIFT)) { + dev_err(ep->dcp->dev, "AFK[ep:%02x]: ring cannot hold an entry\n", + ep->endpoint); + return; + } + if (bfr == &ep->rxbfr) { + /* Receive dispatch cannot recurse on the ordered endpoint queue. */ + ep->rx_scratch = devm_kmalloc(ep->dcp->dev, bufsz, GFP_KERNEL); + if (!ep->rx_scratch) + return; + } bfr->buf = bfr->hdr + 1; bfr->bufsz = bufsz; @@ -606,7 +621,6 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, static bool afk_recv(struct apple_dcp_afkep *ep) { struct afk_qe *hdr; - u8 *data; u32 rptr, wptr; u32 magic, size, channel, type; @@ -677,9 +691,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) * sleep and send replies, so retain a private copy before allowing * the producer to reuse the ring storage. */ - data = kmemdup(hdr->data, size, GFP_KERNEL); - if (!data) - return false; + memcpy(ep->rx_scratch, hdr->data, size); rptr = ALIGN(rptr + sizeof(*hdr) + size, 1 << BLOCK_SHIFT); if (WARN_ON(rptr > ep->rxbfr.bufsz)) @@ -692,8 +704,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) ep->rxbfr.hdr->rptr = cpu_to_le32(rptr); trace_afk_recv_rwptr_post(ep, rptr, wptr); - afk_recv_handle(ep, channel, type, data, size); - kfree(data); + afk_recv_handle(ep, channel, type, ep->rx_scratch, size); return true; } diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 880987cbb6ef4..2713372d48f60 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -263,6 +263,8 @@ struct apple_dcp_afkep { struct afk_ringbuffer txbfr; struct afk_ringbuffer rxbfr; + /* Private receive entry, owned by the ordered endpoint worker. */ + void *rx_scratch; spinlock_t lock; u16 qe_seq; From 2bcc23302271ff556980938883fa238feb4de49f Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:24:58 -0700 Subject: [PATCH 1113/2680] drm/apple: order AFK entries before publishing the producer position MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Place a DMA write barrier between filling the entry and updating the shared producer position. A consumer already draining an earlier notification must not observe the new position before its payload. The existing send barrier still orders the position before notification. [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/afk.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 404ff9cf6e7ce..04f31f3661797 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -920,6 +920,8 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, wptr = 0; trace_afk_send_rwptr_post(ep, rptr, wptr); + /* Publish the completed entry before allowing the consumer to read it. */ + dma_wmb(); ep->txbfr.hdr->wptr = cpu_to_le32(wptr); afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_SEND) | FIELD_PREP(SEND_WPTR, wptr)); From 8e29ea2aa2e4fd98d3a499d5c9084f8a1141b281 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:26:19 -0700 Subject: [PATCH 1114/2680] drm/apple: copy the complete DP service EDID response MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The EDID reply includes a 48-byte header before the advertised data area. Use that structure size for both allocation and the output copy instead of using the pointer size for the latter, which truncated the largest reply by 40 bytes on 64-bit systems. Reject reported data lengths beyond the advertised capacity before examining the EDID blocks. [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/epic/dpavservep.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index cfcf3e184f5a7..b9f7c619dd4ed 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -113,6 +113,7 @@ struct dpavserv_copy_edid_resp { u8 _pad2[8]; u8 data[]; } __packed; +static_assert(sizeof(struct dpavserv_copy_edid_resp) == 48); static int parse_report(struct apple_epic_service *service, enum epic_subtype type, const void *data, size_t data_size) @@ -185,18 +186,19 @@ static const struct drm_edid *dcpavserv_read_edid(struct apple_epic_service *ser { struct dpavserv_copy_edid_cmd cmd; struct dpavserv_copy_edid_resp *resp __free(kfree) = NULL; + size_t resp_size = sizeof(*resp) + EDID_BUF_SIZE; int num_blocks; u64 data_size; int ret; memset(&cmd, 0, sizeof(cmd)); cmd.max_size = cpu_to_le64(EDID_BUF_SIZE); - resp = kzalloc(sizeof(*resp) + EDID_BUF_SIZE, GFP_KERNEL); + resp = kzalloc(resp_size, GFP_KERNEL); if (!resp) return ERR_PTR(-ENOMEM); ret = afk_service_call(service, 1, 7, &cmd, sizeof(cmd), EDID_BUF_SIZE, resp, - sizeof(resp) + EDID_BUF_SIZE, 0); + resp_size, 0); if (ret < 0) return ERR_PTR(ret); @@ -207,7 +209,8 @@ static const struct drm_edid *dcpavserv_read_edid(struct apple_epic_service *ser // 16, 1, resp, 192, true); data_size = le64_to_cpu(resp->used_size); - if (data_size < EDID_LEADING_DATA_SIZE + EDID_BLOCK_SIZE) + if (data_size < EDID_LEADING_DATA_SIZE + EDID_BLOCK_SIZE || + data_size > EDID_BUF_SIZE) return ERR_PTR(-EIO); num_blocks = resp->data[EDID_LEADING_DATA_SIZE + EDID_EXT_BLOCK_COUNT_OFFSET]; From b34c978eba7b36419b4f2c0a150eb7c62dd26a68 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:27:40 -0700 Subject: [PATCH 1115/2680] drm/apple: bound AFK debug requests and reject partial user copies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reject raw requests larger than the shared 8192-byte request/reply buffer before copying user data. Treat every nonzero copy_from_user result as a fault so partially copied command headers or payloads cannot be sent. Handle empty raw writes without submitting a command. [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/afk.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 04f31f3661797..4845811ea9fb2 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -1172,8 +1172,7 @@ static ssize_t service_call_write_file(struct file *file, const char __user *use return -ENOMEM; } - ret = copy_from_user(&call_info, user_buf, sizeof(call_info)); - if (ret == sizeof(call_info)) + if (copy_from_user(&call_info, user_buf, sizeof(call_info))) return -EFAULT; user_buf += sizeof(call_info); count -= sizeof(call_info); @@ -1181,8 +1180,7 @@ static ssize_t service_call_write_file(struct file *file, const char __user *use buf = kmalloc(count, GFP_KERNEL); if (!buf) return -ENOMEM; - ret = copy_from_user(buf, user_buf, count); - if (ret == count) { + if (copy_from_user(buf, user_buf, count)) { kfree(buf); return -EFAULT; } @@ -1225,6 +1223,11 @@ static ssize_t service_raw_call_write_file(struct file *file, const char __user u32 retcode; int ret; + if (!count) + return 0; + if (count > AFK_DEBUGFS_MAX_REPLY) + return -E2BIG; + if (!srv->debugfs.scratch) { srv->debugfs.scratch = \ devm_kzalloc(srv->ep->dcp->dev, AFK_DEBUGFS_MAX_REPLY, GFP_KERNEL); @@ -1233,8 +1236,7 @@ static ssize_t service_raw_call_write_file(struct file *file, const char __user } memset(srv->debugfs.scratch, 0, AFK_DEBUGFS_MAX_REPLY); - ret = copy_from_user(srv->debugfs.scratch, user_buf, count); - if (ret == count) + if (copy_from_user(srv->debugfs.scratch, user_buf, count)) return -EFAULT; ret = afk_send_command(srv, EPIC_SUBTYPE_STD_SERVICE, srv->debugfs.scratch, count, From 00fc733e4cc5f177becec743b3f978f2a4910aff Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:27:58 -0700 Subject: [PATCH 1116/2680] drm/apple: free the temporary audio object reply buffer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Release the temporary audio service request/reply allocation on both success and error returns. Copying the result to the caller does not transfer ownership of that allocation. [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/av.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c index 13fdd068daa67..ed6f05e401199 100644 --- a/drivers/gpu/drm/apple/av.c +++ b/drivers/gpu/drm/apple/av.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -196,7 +197,7 @@ dcp_audiosrv_osobject_call(struct apple_epic_service *service, u16 group, } __attribute__((packed)) *hdr; static_assert(sizeof(*hdr) == 48); size_t bfr_len = output_maxsize + sizeof(*hdr); - void *bfr; + void *bfr __free(kfree) = NULL; int ret; bfr = kzalloc(bfr_len, GFP_KERNEL); From 8a450538f3f4f56861f990d8c2f93e3b10f61254 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:30:36 -0700 Subject: [PATCH 1117/2680] drm/apple: report audio query failures without reading unset sizes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The warning guards were inverted, suppressing the first error and leaving the warned flags false. That error also took the success debug branch, although the output size is only assigned by a successful query. Report the first failed query and log the returned size only on success. [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/av.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c index ed6f05e401199..049fcc5dae086 100644 --- a/drivers/gpu/drm/apple/av.c +++ b/drivers/gpu/drm/apple/av.c @@ -232,10 +232,10 @@ int dcp_audiosrv_get_elements(struct device *dev, void *elements, size_t maxsize elements, maxsize, &size); up_write(&asrv->srv_rwsem); - if (ret && asrv->warned_get_elements) { + if (ret && !asrv->warned_get_elements) { dev_err(dev, "audiosrv: error getting elements: %d\n", ret); asrv->warned_get_elements = true; - } else { + } else if (!ret) { dev_dbg(dev, "audiosrv: got %zd bytes worth of elements\n", size); } @@ -255,10 +255,10 @@ int dcp_audiosrv_get_product_attrs(struct device *dev, void *attrs, size_t maxsi maxsize, &size); up_write(&asrv->srv_rwsem); - if (ret && asrv->warned_get_product_attrs) { + if (ret && !asrv->warned_get_product_attrs) { dev_err(dev, "audiosrv: error getting product attributes: %d\n", ret); asrv->warned_get_product_attrs = true; - } else { + } else if (!ret) { dev_dbg(dev, "audiosrv: got %zd bytes worth of product attributes\n", size); } From faa33761d0c0a62eeb0b3260c7ba5aa425baa457 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 20:35:13 -0700 Subject: [PATCH 1118/2680] thunderbolt: restore M1 tunneled PCIe state and recover late link loss MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kensington USB devices can disappear after s2idle even when the initial port restart and cable health check succeed. A later PCIe link failure must trigger another fresh validation instead of waiting for a cable interrupt or manual replug. Restore the resettable M1 tunneled host and DART state before restarting the downstream link. Gate DART commands and fault handling around clock shutdown, preserve stream selection, and describe the reset and register initialization properties in the binding. Gate Type-C work across initialization, sleep and removal, read fresh cable metadata after resume, and preserve an unchanged healthy session. Propagate late link failures through an atomic switch notification into bounded process-context revalidation. Keep failure history sticky while allowing later notifications from a retained host. Failed reads or health checks request a bounded reconnect; successful checks preserve routing. Keep failed display-route acquisition retryable. Add state-machine and DART regression coverage. The native ARM64 build and 88 KUnit cases pass. Two M1/j293 Kensington USB-only suspend cycles exercised late link-loss recovery and restored usable input without a manual replug. The initiating xHCI resume fault remains unresolved. [ryan: split by topic: only the drm/apple part and its selftests are here; the Documentation/devicetree/bindings/pci/apple,pcie.yaml hunk belongs to its own topic; the DT hunks are in 000-devicetree; the drivers/iommu/apple-dart-test.c hunk belongs to its own topic; the drivers/iommu/apple-dart.c hunk belongs to its own topic; the drivers/pci/controller/pcie-apple.c hunk belongs to its own topic; the PHY, Type-C, Thunderbolt and crossbar hunks are in 170-usb; the include/linux/pci-apple.h hunk belongs to its own topic; the include/linux/soc/apple/dart.h hunk belongs to its own topic] [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/dcp.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 03e2688f0d14f..bf3303807425b 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -378,12 +378,15 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, port->applied_mode = state->mode; port->applied_status = dp_status; port->applied_conf = dp_conf; - port->applied_valid = true; + /* Failed route acquisition must remain retryable on the next update. */ + port->applied_valid = false; if (!is_dp) { /* a Thunderbolt/USB4 DP tunnel is torn down by its own path */ - if (port->owner && port->owner->tunnel) + if (port->owner && port->owner->tunnel) { + port->applied_valid = true; return 0; + } if (port->owner) { struct apple_dcp *dcp = port->owner->dcp; @@ -403,6 +406,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (state->mode == TYPEC_MODE_USB4) dcp_typec_retrain_active_routes(); + port->applied_valid = true; return 0; } @@ -455,6 +459,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, dcp_retrain_oob(dcp->typec_connector); } port->hpd = hpd; + port->applied_valid = true; return 0; } From 33dbc586243ca0ad66775df2e6cd168c2c3e5499 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:41:25 -0700 Subject: [PATCH 1119/2680] drm/apple: keep the Type-C cable state across system sleep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Suspend cleared typec_cable_connected, relying on the Thunderbolt DP tunnel being released at suspend and set up again at resume, which reports the cable once more. When the tunnel is kept through suspend-to-idle nothing reports it again. dcp_poweron() then skips the DPTX connect, the modeset fails, and every later flip on that CRTC times out: apple-drm soc:display-subsystem: [drm] *ERROR* flip_done timed out apple-drm soc:display-subsystem: [drm] *ERROR* [CRTC:59:crtc-1] commit wait timed out The compositor stalls until the dock is unplugged. The route already reports cable removal through dcp_dptx_disconnect_oob(). Leave the cable state alone on suspend and clear it only on unbind, so a kept tunnel resumes like DPMS on. [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PA1D59hehGmwnpSxrYcDbk --- drivers/gpu/drm/apple/dcp.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index bf3303807425b..74cd657d235af 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2190,10 +2190,11 @@ static int dcp_connector_type_from_dt(struct device_node *np) return DRM_MODE_CONNECTOR_Unknown; } -static void dcp_disable_typec_work(struct apple_dcp *dcp) +static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) { scoped_guard(mutex, &dcp->hpd_mutex) { - WRITE_ONCE(dcp->typec_cable_connected, false); + if (release_cable) + WRITE_ONCE(dcp->typec_cable_connected, false); dcp->typec_generation++; } /* Block new enqueues as well as draining users of the AFK endpoints. */ @@ -2328,7 +2329,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) if (dcp->hdmi_hpd_irq) disable_irq(dcp->hdmi_hpd_irq); - dcp_disable_typec_work(dcp); + dcp_disable_typec_work(dcp, true); typec_mux_put(dcp->typec_mux); if (dcp->avep) { @@ -2580,7 +2581,13 @@ static int dcp_platform_suspend(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); - dcp_disable_typec_work(dcp); + /* + * The Type-C route reports cable removal through + * dcp_dptx_disconnect_oob(). A DP tunnel kept through the sleep stays + * connected, and resume powers the CRTC back up through dcp_poweron() + * as after DPMS off. + */ + dcp_disable_typec_work(dcp, false); if (dcp->avep) av_service_disconnect(dcp); From f0447d5a2bdf5519d6e42c8a0df22718ff6b9e8d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:02:44 -0700 Subject: [PATCH 1120/2680] drm/apple: recover Type-C outputs whose swaps DCP drops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DCP accepts a modeset on an external Type-C output even when the pipe cannot be enabled yet, and then drops every swap without completing it. That happens on resume from suspend-to-idle when the sink is slower than the link: a TV behind the DP-to-HDMI converter of a Kensington SD5560T dock asserted HPD only after the first modeset. Each flip then waits for the full commit timeout, and the compositor stalls with it: apple-drm soc:display-subsystem: [drm] *ERROR* flip_done timed out apple-drm soc:display-subsystem: [drm] *ERROR* [CRTC:59:crtc-1] commit wait timed out Arm a watchdog when a swap is submitted on a Type-C output. If DCP has not completed the swap within a second, complete the flip, mark the mode invalid so further commits do not wait for DCP, and let the hotplug worker re-apply the active CRTC, a bounded number of times. When the sink asserts HPD after a failed modeset, re-apply the mode as well. [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PA1D59hehGmwnpSxrYcDbk --- drivers/gpu/drm/apple/dcp-internal.h | 6 ++++ drivers/gpu/drm/apple/dcp.c | 42 ++++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 13 ++++++++ 3 files changed, 61 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 3de4748e53d88..88d95205e955b 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -50,6 +50,8 @@ struct apple_dcp_typec_route { }; bool dcp_is_typec_output(struct apple_dcp *dcp); +void dcp_swap_watchdog_arm(struct apple_dcp *dcp); +void dcp_swap_watchdog_complete(struct apple_dcp *dcp); bool dcp_is_usb4_output(struct apple_dcp *dcp); void dcp_retry_placeholder_edid(struct apple_dcp *dcp, const struct drm_edid *drm_edid); @@ -251,6 +253,10 @@ struct apple_dcp { /* Workqueue for sending vblank events when a dcp swap is not possible */ struct work_struct vblank_wq; + /* Completes a Type-C swap that DCP dropped, and recovers the pipe. */ + struct delayed_work swap_watchdog_wq; + unsigned int swap_watchdog_retrains; + /* List of referenced drm_framebuffers which can be unreferenced * on the next successfully completed swap. */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 74cd657d235af..71f0c021f29b8 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1142,6 +1142,44 @@ static void dcp_delayed_vblank(struct work_struct *work) dcp_drm_crtc_vblank(dcp->crtc); } +#define DCP_SWAP_WATCHDOG_MS 1000 +#define DCP_SWAP_WATCHDOG_RETRAINS 5 + +/* + * DCP drops swaps without completing them while an external pipe is not + * enabled, for instance after a modeset that raced a Type-C sink which had + * not asserted HPD yet. Userspace would then wait for the flip until the + * commit times out, stalling every output it drives. Complete the flip, + * mark the mode invalid so later commits do not wait for DCP, and let the + * hotplug worker re-apply the active CRTC. + */ +static void dcp_swap_watchdog(struct work_struct *work) +{ + struct apple_dcp *dcp = + container_of(to_delayed_work(work), struct apple_dcp, + swap_watchdog_wq); + + dev_warn(dcp->dev, "swap not completed, retraining the display\n"); + dcp_mode_invalidate(&dcp->mode_state); + dcp_drm_crtc_vblank(dcp->crtc); + if (dcp->connector && + dcp->swap_watchdog_retrains++ < DCP_SWAP_WATCHDOG_RETRAINS) + schedule_work(&dcp->connector->hotplug_wq); +} + +void dcp_swap_watchdog_arm(struct apple_dcp *dcp) +{ + if (dcp_is_typec_output(dcp)) + mod_delayed_work(system_wq, &dcp->swap_watchdog_wq, + msecs_to_jiffies(DCP_SWAP_WATCHDOG_MS)); +} + +void dcp_swap_watchdog_complete(struct apple_dcp *dcp) +{ + cancel_delayed_work(&dcp->swap_watchdog_wq); + dcp->swap_watchdog_retrains = 0; +} + static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) { struct apple_dcp *dcp = cookie; @@ -1861,6 +1899,7 @@ void dcp_poweroff(struct platform_device *pdev) struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; + cancel_delayed_work(&dcp->swap_watchdog_wq); if (dcp->avep) av_service_disconnect(dcp); @@ -2293,6 +2332,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) set_bit(0, dcp->memdesc_map); INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); + INIT_DELAYED_WORK(&dcp->swap_watchdog_wq, dcp_swap_watchdog); dcp->swapped_out_fbs = (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); @@ -2372,6 +2412,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); } + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); cancel_work_sync(&dcp->vblank_wq); devm_clk_put(dev, dcp->clk); @@ -2588,6 +2629,7 @@ static int dcp_platform_suspend(struct device *dev) * as after DPMS off. */ dcp_disable_typec_work(dcp, false); + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); if (dcp->avep) av_service_disconnect(dcp); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 44723bdb95420..2b3586d735145 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -126,6 +126,7 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, ktime_t now = ktime_get(); trace_iomfb_swap_complete(dcp, resp->swap_id); dcp->last_swap_id = resp->swap_id; + dcp_swap_watchdog_complete(dcp); dcp_drm_crtc_page_flip(dcp, now); if (dcp->crc_enabled) { @@ -1071,6 +1072,17 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) action = dcp_mode_hotplug(&dcp->mode_state, !!(*connected), connector ? &connector->connected : NULL); + /* + * A Type-C sink can assert HPD only after a modeset has already failed, + * as a TV behind a DP-to-HDMI converter does when it wakes from standby. + * The connector state does not change then; re-apply the mode anyway. + */ + if (*connected && dcp_is_typec_output(dcp) && + !READ_ONCE(dcp->mode_state.valid) && + !READ_ONCE(dcp->mode_state.changing)) { + dcp->swap_watchdog_retrains = 0; + action |= DCP_HOTPLUG_NOTIFY; + } dcp_handle_hotplug_actions(dcp, action); } @@ -1174,6 +1186,7 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) } dcp->swap_start = ktime_get(); dcp->swap_submit_timestamp = arch_timer_read_counter(); + dcp_swap_watchdog_arm(dcp); while (!list_empty(&dcp->swapped_out_fbs)) { struct dcp_fb_reference *entry; From 4da0f99ba2e341e9cd2f60e18e68d364118e1286 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:41:15 -0700 Subject: [PATCH 1121/2680] soc: apple: dp-tunnel: drive Thunderbolt displays on every M2 Pro and M2 Max laptop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The T602X display tunnel path ran only on the machine it was brought up on, the 16" M2 Pro (j416s): the DP IN handshake, the tunnel pixel clock, the longer link timeouts, the pipeline preference and the right-hand port's bandwidth grant were all keyed to "apple,j416s". The 14" M2 Pro (j414s), 14" M2 Max (j414c) and 16" M2 Max (j416c) use the same ATC PHYs ("apple,t6020-atcphy") and include the same t602x-j414-j416.dtsi, which carries the crossbar's typecN-usb4 tunnel routes and the USB-C port layout this code relies on. On those machines a display behind a dock stayed dark. Add apple_dp_tunnel_t602x() to the shared DP tunnel header, matching the four laptops, and use it for the ten j416s checks in appledrm, the ATC PHY and the Thunderbolt code. The M2 Pro and M2 Max desktops (j474s, j475c, j475d) are left out: their device tree has no crossbar tunnel routes yet. So far only j416s has run this path. [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i [ryan: split by topic: only the drm/apple part and its selftests are here; the Documentation/admin-guide/kernel-parameters.txt hunk belongs to its own topic; the PHY, Type-C, Thunderbolt and crossbar hunks are in 170-usb] --- drivers/gpu/drm/apple/dcp.c | 11 +++++------ drivers/gpu/drm/apple/iomfb.c | 3 ++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 71f0c021f29b8..8f082b44816fc 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -660,8 +660,8 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, if (!dcp_typec_route_available(candidate)) continue; score = dcp_typec_route_score(candidate); - /* j416s dcpext0 completes tunneled link training; prefer it. */ - if (of_machine_is_compatible("apple,j416s") && + /* dcpext0 completes tunneled link training here; prefer it. */ + if (apple_dp_tunnel_t602x() && !candidate->dcp->fixed_phy && score < UINT_MAX - 100) score += 100; if (score < best_score) { @@ -1394,8 +1394,7 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) } dcp->dptxport[port].connected = true; if (dcp_is_typec_output(dcp)) { - if (dcp_is_usb4_output(dcp) && - of_machine_is_compatible("apple,j416s")) + if (dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x()) ret = dptxport_set_hpd_timeout(dcp->dptxport[port].service, true, 8000); else @@ -1410,7 +1409,7 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) } mutex_unlock(&dcp->hpd_mutex); - timeout = dcp_is_usb4_output(dcp) && of_machine_is_compatible("apple,j416s") ? + timeout = dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x() ? DPTX_TUNNEL_CONNECT_TIMEOUT : DPTX_CONNECT_TIMEOUT; ret = wait_for_completion_timeout(&dcp->dptxport[port].linkcfg_completion, timeout); @@ -1540,7 +1539,7 @@ static void dcp_typec_reconnect_work(struct work_struct *work) } if (++dcp->typec_reconnect_tries < - (dcp_is_usb4_output(dcp) && of_machine_is_compatible("apple,j416s") ? + (dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x() ? 1 : DPTX_RECONNECT_RETRIES)) { mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, DPTX_RECONNECT_DELAY); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 24a3cbb471e12..40253e8ddfb27 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -302,7 +303,7 @@ void dcp_hotplug(struct work_struct *work) * flush, or the CRTC gets no signal. */ if (connector->base.state && !READ_ONCE(dcp->mode_state.valid) && connector->connected && - !(dcp_is_usb4_output(dcp) && of_machine_is_compatible("apple,j416s"))) { + !(dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x())) { drm_connector_set_link_status_property(&connector->base, DRM_MODE_LINK_STATUS_BAD); From 82b876eafda7e2c23afc55ccc63ce9062cc3db1f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 00:57:11 -0400 Subject: [PATCH 1122/2680] drm/apple: preserve Type-C routes across blanking and detach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retain the last available DCP route and avoid retraining unrelated outputs during the immediate DP-to-USB4 teardown sequence. Keep synthetic DPMS disconnects separate from physical cable removal, while allowing positive hotplug events through after a replug. Let the RTKit crash callback, rather than a power-off clear-swap timeout, establish a fatal firmware crash. [Chris Kearney: ported onto aurora-silicon/linux#8. The DPMS handling (typec_crtc_off, no reconnect from dcp_poweroff(), ignoring the unplug of a powered-off output) is already in aurora-wip as ab7cad0652d5, so this keeps that version and adds the route preference, the detach retrain guard, the reconnect cancel and the clear-swap change.] [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/dcp.c | 62 +++++++++++++++++++------- drivers/gpu/drm/apple/iomfb_template.c | 6 ++- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8f082b44816fc..6a5437eac9b60 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -67,6 +67,10 @@ struct apple_dcp_typec_port { struct list_head routes; struct device_node *connector_np; struct apple_dcp_typec_route *owner; + /* Keep a port on its last DCP while that pipeline remains free. */ + struct apple_dcp_typec_route *preferred_route; + /* Ignore the USB4 fallback immediately following this port's DP teardown. */ + unsigned long dp_release_deadline; /* DRM connector for this physical port, driven by whichever DCP owns it */ struct apple_connector *connector; /* last mux state acted on, to collapse the per-candidate notifications */ @@ -129,10 +133,9 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port); /* - * Pipelines are ranked by CRTC index so the fabric's choice is a pure function - * of the topology rather than of plug order. A pipeline whose fixed output is - * live is not a candidate at all, so a hybrid is only ever ranked here when it - * is genuinely free. + * For a port without a prior owner, rank pipelines by CRTC index. A pipeline + * whose fixed output is live is not a candidate at all, so a hybrid is only + * ever ranked here when it is genuinely free. */ static unsigned int dcp_typec_route_score(struct apple_dcp_typec_route *route) { @@ -390,6 +393,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (port->owner) { struct apple_dcp *dcp = port->owner->dcp; + port->preferred_route = port->owner; if (port->hpd || dcp->typec_cable_connected || (dcp->typec_connector && dcp->typec_connector->connected)) @@ -399,13 +403,24 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (ret) return ret; port->owner = NULL; + port->dp_release_deadline = jiffies + msecs_to_jiffies(10000); if (dcp->hdmi_hpd && dcp->active && gpiod_get_value_cansleep(dcp->hdmi_hpd)) dcp_dptx_connect(dcp, 0); } - if (state->mode == TYPEC_MODE_USB4) - dcp_typec_retrain_active_routes(); + /* + * A port leaving DP can report SAFE/NONE before falling back to USB4. + * Resetting every other live CRTC for that same cable removal blanks + * unaffected displays. Keep the guard across the Type-C state sequence; + * a later, independent USB4 attach still gets recovery. + */ + if (state->mode == TYPEC_MODE_USB4) { + if (!port->dp_release_deadline || + time_after_eq(jiffies, port->dp_release_deadline)) + dcp_typec_retrain_active_routes(); + port->dp_release_deadline = 0; + } port->applied_valid = true; return 0; } @@ -421,15 +436,21 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, } if (!port->owner) { - list_for_each_entry(candidate, &port->routes, port_link) { - unsigned int score; - - if (!dcp_typec_route_available(candidate)) - continue; - score = dcp_typec_route_score(candidate); - if (score < best_score) { - best = candidate; - best_score = score; + if (port->preferred_route && + dcp_typec_route_available(port->preferred_route)) + best = port->preferred_route; + + if (!best) { + list_for_each_entry(candidate, &port->routes, port_link) { + unsigned int score; + + if (!dcp_typec_route_available(candidate)) + continue; + score = dcp_typec_route_score(candidate); + if (score < best_score) { + best = candidate; + best_score = score; + } } } @@ -440,6 +461,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (ret) return ret; port->owner = best; + port->dp_release_deadline = 0; } @@ -843,6 +865,8 @@ static void dcp_typec_route_unregister(void *data) typec_mux_unregister(route->typec_mux); guard(mutex)(&dcp_typec_fabric_lock); + if (port->preferred_route == route) + port->preferred_route = NULL; if (port->owner == route) { struct apple_dcp *dcp = route->dcp; @@ -1911,8 +1935,12 @@ void dcp_poweroff(struct platform_device *pdev) * display, so DPMS off never sticks. Cable removal is reported through * the Type-C mux. */ - if (dcp_is_typec_output(dcp) && READ_ONCE(dcp->typec_cable_connected)) - WRITE_ONCE(dcp->typec_crtc_off, true); + if (dcp_is_typec_output(dcp)) { + if (READ_ONCE(dcp->typec_cable_connected)) + WRITE_ONCE(dcp->typec_crtc_off, true); + /* dcp_poweron() reconnects the link on DPMS wake. */ + cancel_delayed_work(&dcp->typec_reconnect_wq); + } _dcp_poweroff(dcp); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 2b3586d735145..c3cf81082ed96 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -960,7 +960,11 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) swap_id = cookie->swap_id; kref_put(&cookie->refcount, release_swap_cookie); if (ret <= 0) { - dcp->crashed = true; + /* A disconnected external display can power down before it ACKs this + * clear swap. The deadline alone does not mean RTKit crashed; its + * crash callback owns the fatal flag. + */ + dev_warn(dcp->dev, "clear swap timed out during power-off\n"); return; } From ac7ca0600b413368d8728b5ba70ec9fb4fcb8bfc Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 24 Sep 2026 23:39:53 -0400 Subject: [PATCH 1123/2680] drm/apple: reserve the hybrid DCP for a Thunderbolt DP tunnel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Route an available direct Type-C DP-alt-mode display to dcpext1 first so dcpext0 remains available for a dock DP tunnel. The tunnel still prefers dcpext0, which is the pipeline that completes link training on T6020. m2fix45 cold-booted with both displays attached: direct 3840x2160@120 on dcpext1 and dock 1920x1280@60 on dcpext0 came up without port resets, alongside eDP. SEP identity, haptic FF ioctl, and AVD H.264 decode remained functional. [Chris Kearney: ported onto aurora-silicon/linux#8; the J414s check goes through apple_dp_tunnel_t602x(), so j414c, j416s and j416c take the same pipeline preference, as tested on j416s by Oliver Lukschander and proposed for j416c in #50.] [ryan: adapted to 7.2.10 — 170-usb carries the final three-argument apple_atc_dp_tunnel_rate(); pass DP IN 0, whose pixel clock the two-argument call drove, until "drm/apple: drive two dock DP tunnels independently" passes the route's DP IN] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/dcp.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 6a5437eac9b60..009cae69c7755 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -447,6 +447,14 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (!dcp_typec_route_available(candidate)) continue; score = dcp_typec_route_score(candidate); + /* + * On the M2 Pro/Max laptops, leave dcpext0 free for + * a dock's DP tunnel when a direct DP-alt-mode + * display can use the Type-C-only dcpext1 pipeline. + */ + if (apple_dp_tunnel_t602x() && + candidate->dcp->fixed_phy) + score += 100; if (score < best_score) { best = candidate; best_score = score; From a3a964f263a9fbd63c08506ee3629d41ff55dc72 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 26 Sep 2026 18:51:29 -0400 Subject: [PATCH 1124/2680] drm/apple: drive two dock DP tunnels independently Route DPIN1 through the native T602x crossbar fields and source selector, give the two tunnel streams separate DCP connector/route ownership, and keep both ATC pixel clocks alive at independent link rates. Wait for Type-C IOMFB hotplug after a display power-on. The dual-dock path was exercised on J414s/m2fix48 with Dell 3840x2160@120 and JTG 1920x1280@60, per-output DPMS and idle/wake; the user confirmed the displayed image. This commit removes a frame-complete callback that had no register side effect. The cleaned tree builds with the Asahi profile; the cleaned commit itself is not yet deployed. [Chris Kearney: ported onto aurora-silicon/linux#8 from "drm/apple: drive two J414s dock DP tunnels independently": - The J414s checks go through apple_dp_tunnel_t602x(), so all four M2 Pro/Max laptops get the second connector per port and the per-DP-IN pipeline preference (DPIN0 dcpext0, DPIN1 dcpext1). Oliver Lukschander ran two displays through one hub on j416s with the same gates. - The per-DP-IN ATC clock tracking replaces #8's single-rate T602X restart: a rate change reselects that DP IN's PCLK. A DPMS stop gates the outputs and keeps the PLL; a PHY mode change still powers the AUSPLL down in atc_tunnel_restore(), which now also forgets the users. The t8103 tunnel clock path is unchanged. - Only the T602X crossbar preselects a DP IN's source; t8103 tunnels still select it at link-up, so -EOPNOTSUPP from the selector is not an error.] [ryan: split by topic: only the drm/apple part and its selftests are here; the PHY, Type-C, Thunderbolt and crossbar hunks are in 170-usb] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/apple_drv.c | 23 ++-- drivers/gpu/drm/apple/dcp-internal.h | 2 + drivers/gpu/drm/apple/dcp.c | 166 +++++++++++++++++++------ drivers/gpu/drm/apple/dcp.h | 3 +- drivers/gpu/drm/apple/iomfb_template.c | 2 + 5 files changed, 149 insertions(+), 47 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 75c1d9d113a67..c42604c875c54 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -437,10 +438,10 @@ static int apple_probe_per_dcp(struct device *dev, } /* - * Create one connector per physical Type-C port and attach it to every - * pipeline that can drive it. Keeping the connector tied to the port rather - * than to a pipeline is what gives userspace a stable name to hang its - * per-monitor configuration on. + * Create a connector per physical Type-C port and attach it to every + * pipeline that can drive it. The M2 Pro/Max laptops append a second + * connector per port for a dock's second DP tunnel, preserving primary + * connector names. */ static int apple_probe_typec_ports(struct drm_device *drm, struct platform_device **dcp, @@ -449,9 +450,15 @@ static int apple_probe_typec_ports(struct drm_device *drm, unsigned int idx, nr_ports = dcp_typec_nr_ports(); int i, ret; - for (idx = 0; idx < nr_ports; idx++) { + /* Keep the existing physical-port connector numbers stable. On the + * M2 Pro/Max laptops, append one more connector per port for a second + * USB4 DP tunnel. + */ + for (idx = 0; idx < nr_ports * (apple_dp_tunnel_t602x() ? 2 : 1); idx++) { struct apple_connector *connector; struct apple_encoder *enc; + unsigned int port_idx = idx % nr_ports; + bool secondary = idx >= nr_ports; u32 mask = 0; connector = kzalloc_obj(*connector); @@ -480,13 +487,13 @@ static int apple_probe_typec_ports(struct drm_device *drm, INIT_WORK(&connector->hotplug_wq, dcp_hotplug); for (i = 0; i < num_dcp; i++) { - if (dcp_typec_port_has_candidate(idx, dcp[i])) + if (dcp_typec_port_has_candidate(port_idx, dcp[i])) mask |= crtc_mask[i]; } if (!mask) { drm_warn(drm, "Type-C port %u has no display pipeline\n", - idx); + port_idx); return -ENODEV; } @@ -510,7 +517,7 @@ static int apple_probe_typec_ports(struct drm_device *drm, connector->port_encoder = &enc->base; connector->candidate_crtcs = mask; - dcp_typec_port_set_connector(idx, connector); + dcp_typec_port_set_connector(port_idx, secondary, connector); } return 0; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 88d95205e955b..e38d3573a4b5a 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -315,6 +315,8 @@ struct apple_dcp { bool tb_clock_ok; /* CRTC powered off while the Type-C cable stays attached */ bool typec_crtc_off; + /* IOMFB reports its video interface ready after DPTX link training. */ + struct completion typec_iomfb_hpd_ready; struct delayed_work typec_reconnect_wq; struct delayed_work typec_fabric_retrain_wq; u32 typec_reconnect_tries; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 009cae69c7755..4c836fbb84c5a 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -67,12 +67,15 @@ struct apple_dcp_typec_port { struct list_head routes; struct device_node *connector_np; struct apple_dcp_typec_route *owner; + struct apple_dcp_typec_route *secondary_owner; /* Keep a port on its last DCP while that pipeline remains free. */ struct apple_dcp_typec_route *preferred_route; /* Ignore the USB4 fallback immediately following this port's DP teardown. */ unsigned long dp_release_deadline; /* DRM connector for this physical port, driven by whichever DCP owns it */ struct apple_connector *connector; + /* A second logical stream through this port's USB4 dock. */ + struct apple_connector *secondary_connector; /* last mux state acted on, to collapse the per-candidate notifications */ struct typec_altmode *applied_alt; unsigned long applied_mode; @@ -132,6 +135,19 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port); +static int dcp_dpxbar_tunnel_select_source(struct mux_control *mux, int state) +{ + typeof(&apple_dpxbar_tunnel_select_source) select = + symbol_get(apple_dpxbar_tunnel_select_source); + int ret; + + if (!select) + return -ENOENT; + ret = select(mux, state); + symbol_put(apple_dpxbar_tunnel_select_source); + return ret; +} + /* * For a port without a prior owner, rank pipelines by CRTC index. A pipeline * whose fixed output is live is not a candidate at all, so a hybrid is only @@ -151,6 +167,9 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, struct mux_control *xbar) { struct apple_dcp *dcp = route->dcp; + struct apple_connector *connector = + xbar != route->xbar && route->tunnel_dpin == 1 ? + route->port->secondary_connector : route->port->connector; int ret; /* @@ -172,7 +191,19 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, * once DCP has configured the link and the tunnel pixel clock runs, * see dcp_tunnel_crossbar_up(). Just remember the output here. */ - ret = xbar == route->xbar ? mux_control_select(xbar, route->mux_index) : 0; + if (xbar != route->xbar && route->mux_index) { + /* + * The T602X crossbar must point a DP IN at its pipeline before + * DCP probes AUX; other crossbars select it at link-up and + * report -EOPNOTSUPP here. + */ + ret = dcp_dpxbar_tunnel_select_source(xbar, route->mux_index); + if (ret == -EOPNOTSUPP) + ret = 0; + } else { + ret = xbar == route->xbar ? + mux_control_select(xbar, route->mux_index) : 0; + } if (ret) { if (dcp->xbar) { int restore_ret; @@ -192,10 +223,10 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, dcp->phy = route->phy; dcp->dptx_phy = route->dptx_phy; dcp->connector_type = DRM_MODE_CONNECTOR_USB; - if (route->port->connector) { - route->port->connector->dcp = to_platform_device(dcp->dev); - dcp->typec_connector = route->port->connector; - dcp->connector = route->port->connector; + if (connector) { + connector->dcp = to_platform_device(dcp->dev); + dcp->typec_connector = connector; + dcp->connector = connector; /* * Narrow the port to the pipeline now driving it. The encoder @@ -207,8 +238,8 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, * rejected with no way for it to recover. The hotplug that * follows makes it re-read this. */ - if (route->port->connector->port_encoder && dcp->crtc) - route->port->connector->port_encoder->possible_crtcs = + if (connector->port_encoder && dcp->crtc) + connector->port_encoder->possible_crtcs = drm_crtc_mask(&dcp->crtc->base); } dcp->active_typec_route = route; @@ -230,6 +261,9 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) { struct apple_dcp *dcp = route->dcp; + struct apple_connector *connector = dcp->typec_connector; + struct mux_control *active_xbar = route->active_xbar; + bool was_tunnel = route->tunnel; int ret = 0; /* @@ -249,7 +283,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) symbol_get(apple_atc_dp_tunnel_rate); if (stop) { - stop(dcp->phy, 0, 0); + stop(dcp->phy, route->tunnel_dpin, 0); symbol_put(apple_atc_dp_tunnel_rate); } } @@ -261,12 +295,17 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) dcp->tb_dpin_ctx = NULL; dcp->tb_clock_ok = false; } + if (was_tunnel && route->mux_index) { + int sel = dcp_dpxbar_tunnel_select_source(active_xbar, -1); + + if (sel && sel != -EOPNOTSUPP) + dev_warn(dcp->dev, "DP tunnel source reset failed: %d\n", sel); + } route->selected = false; if (dcp->active_typec_route == route) dcp->active_typec_route = NULL; - if (route->port->connector && - route->port->connector->dcp == to_platform_device(dcp->dev)) { + if (connector && connector->dcp == to_platform_device(dcp->dev)) { /* * Until the port is activated again it has no pipeline behind * it, and nothing can read modes or EDID from it. Report it @@ -277,13 +316,13 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) * it. The new pipeline marks it connected again once the * display has come back up on it. */ - WRITE_ONCE(route->port->connector->connected, false); - route->port->connector->dcp = NULL; + WRITE_ONCE(connector->connected, false); + connector->dcp = NULL; /* Unrouted: the port could go to any of its pipelines again. */ - if (route->port->connector->port_encoder) - route->port->connector->port_encoder->possible_crtcs = - route->port->connector->candidate_crtcs; + if (connector->port_encoder) + connector->port_encoder->possible_crtcs = + connector->candidate_crtcs; } dcp->typec_connector = NULL; dcp->connector = dcp->fixed_connector; @@ -339,11 +378,14 @@ static void dcp_typec_retrain_active_routes(void) struct apple_dcp_typec_port *port; list_for_each_entry(port, &dcp_typec_ports, link) { - if (!port->owner) - continue; - mod_delayed_work(system_freezable_wq, + if (port->owner) + mod_delayed_work(system_freezable_wq, &port->owner->dcp->typec_fabric_retrain_wq, msecs_to_jiffies(200)); + if (port->secondary_owner) + mod_delayed_work(system_freezable_wq, + &port->secondary_owner->dcp->typec_fabric_retrain_wq, + msecs_to_jiffies(200)); } } @@ -597,9 +639,13 @@ int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate) return -ENOENT; } route = dcp->active_typec_route; - if (!link_rate && route && route->xbar_up) + if (!route) { + symbol_put(apple_atc_dp_tunnel_rate); + return -ENODEV; + } + if (!link_rate && route->xbar_up) dcp_dpxbar_link(route->active_xbar, false); - ret = fn(phy, 0, link_rate); + ret = fn(phy, route->tunnel_dpin, link_rate); symbol_put(apple_atc_dp_tunnel_rate); dcp->tb_clock_ok = !ret && link_rate; if (ret) @@ -630,6 +676,7 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, { struct apple_dcp_typec_port *port = NULL, *pos; struct apple_dcp_typec_route *candidate, *best = NULL; + struct apple_dcp_typec_route **slot; unsigned int best_score = UINT_MAX; struct mux_control *ctl; struct apple_dcp *dcp; @@ -648,16 +695,16 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, } if (!port) return -ENODEV; + slot = dpin ? &port->secondary_owner : &port->owner; if (!active) { - if (!port->owner || !port->owner->tunnel || - port->owner->tunnel_dpin != dpin) + if (!*slot || !(*slot)->tunnel || + (*slot)->tunnel_dpin != dpin) return 0; - dcp = port->owner->dcp; + dcp = (*slot)->dcp; if (port->hpd || dcp->typec_cable_connected) dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - port->hpd = false; - ret = dcp_typec_route_deactivate(port->owner); + ret = dcp_typec_route_deactivate(*slot); if (ret) { /* the caller's context is going away regardless */ scoped_guard(mutex, &dcp->tb_lock) { @@ -667,7 +714,8 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, } return ret; } - port->owner = NULL; + *slot = NULL; + port->hpd = !!(port->owner || port->secondary_owner); /* re-apply the next Type-C mux state in full */ port->applied_valid = false; if (dcp->hdmi_hpd && dcp->active && @@ -676,13 +724,15 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, return 0; } - if (port->owner) { - if (port->owner->tunnel && port->owner->tunnel_dpin == dpin) + if (*slot) { + if ((*slot)->tunnel && (*slot)->tunnel_dpin == dpin) return 0; - dev_warn(port->owner->dcp->dev, + dev_warn((*slot)->dcp->dev, "port already routed, not taking DP tunnel dpin%u\n", dpin); return -EBUSY; } + if (port->owner && !port->owner->tunnel) + return -EBUSY; list_for_each_entry(candidate, &port->routes, port_link) { unsigned int score; @@ -690,9 +740,15 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, if (!dcp_typec_route_available(candidate)) continue; score = dcp_typec_route_score(candidate); - /* dcpext0 completes tunneled link training here; prefer it. */ - if (apple_dp_tunnel_t602x() && - !candidate->dcp->fixed_phy && score < UINT_MAX - 100) + /* + * On the M2 Pro/Max laptops DPIN0 prefers the hybrid dcpext0, + * which completes tunneled link training, and DPIN1 prefers + * dcpext1, so the two pipelines drive independent streams + * through one dock. + */ + if (apple_dp_tunnel_t602x() && score < UINT_MAX - 100 && + ((dpin == 0 && !candidate->dcp->fixed_phy) || + (dpin == 1 && candidate->dcp->fixed_phy))) score += 100; if (score < best_score) { best = candidate; @@ -722,7 +778,7 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, } return ret; } - port->owner = best; + *slot = best; dev_info(dcp->dev, "display routed to Thunderbolt DP tunnel dpin%u\n", dpin); @@ -836,10 +892,11 @@ bool dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev return false; } -void dcp_typec_port_set_connector(unsigned int idx, +void dcp_typec_port_set_connector(unsigned int idx, bool secondary, struct apple_connector *connector) { struct apple_dcp_typec_port *port; + struct apple_dcp_typec_route *owner; guard(mutex)(&dcp_typec_fabric_lock); @@ -847,7 +904,13 @@ void dcp_typec_port_set_connector(unsigned int idx, if (!port) return; - port->connector = connector; + if (secondary) { + port->secondary_connector = connector; + owner = port->secondary_owner; + } else { + port->connector = connector; + owner = port->owner; + } /* * The port may already have been routed, either before DRM bound or @@ -855,8 +918,12 @@ void dcp_typec_port_set_connector(unsigned int idx, * otherwise its display would be reported on the pipeline's fixed * connector instead of the port it is actually plugged into. */ - if (port->owner) { - struct apple_dcp *dcp = port->owner->dcp; + if (owner) { + struct apple_dcp *dcp = owner->dcp; + + if (dcp->crtc && connector->port_encoder) + connector->port_encoder->possible_crtcs = + drm_crtc_mask(&dcp->crtc->base); connector->dcp = to_platform_device(dcp->dev); dcp->typec_connector = connector; @@ -884,6 +951,15 @@ static void dcp_typec_route_unregister(void *data) dcp_typec_route_deactivate(route); port->owner = NULL; } + if (port->secondary_owner == route) { + struct apple_dcp *dcp = route->dcp; + + if (port->hpd || dcp->typec_cable_connected) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + dcp_typec_route_deactivate(route); + port->secondary_owner = NULL; + } + port->hpd = !!(port->owner || port->secondary_owner); list_del(&route->port_link); if (list_empty(&port->routes)) { list_del(&port->link); @@ -1639,6 +1715,7 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) dcp->typec_generation++; } WRITE_ONCE(dcp->typec_crtc_off, false); + reinit_completion(&dcp->typec_iomfb_hpd_ready); cancel_delayed_work_sync(&dcp->typec_reconnect_wq); cancel_delayed_work_sync(&dcp->placeholder_edid_wq); } @@ -1880,9 +1957,13 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) void dcp_poweron(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); + bool wait_for_typec_hpd = false; + unsigned long remaining; int ret; if (dcp_is_typec_output(dcp)) { + wait_for_typec_hpd = READ_ONCE(dcp->typec_crtc_off) && + READ_ONCE(dcp->typec_cable_connected); WRITE_ONCE(dcp->typec_crtc_off, false); /* @@ -1897,6 +1978,14 @@ void dcp_poweron(struct platform_device *pdev) mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, DPTX_RECONNECT_DELAY); + else if (wait_for_typec_hpd) { + remaining = wait_for_completion_timeout( + &dcp->typec_iomfb_hpd_ready, + msecs_to_jiffies(3000)); + if (!remaining) + dev_warn(dcp->dev, + "Type-C IOMFB hotplug not ready on wake\n"); + } } } else if (dcp->hdmi_hpd) { bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); @@ -1920,7 +2009,6 @@ void dcp_poweron(struct platform_device *pdev) WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; } - if (dcp->avep) av_service_connect(dcp); } @@ -1944,6 +2032,7 @@ void dcp_poweroff(struct platform_device *pdev) * the Type-C mux. */ if (dcp_is_typec_output(dcp)) { + reinit_completion(&dcp->typec_iomfb_hpd_ready); if (READ_ONCE(dcp->typec_cable_connected)) WRITE_ONCE(dcp->typec_crtc_off, true); /* dcp_poweron() reconnects the link on DPMS wake. */ @@ -2501,6 +2590,7 @@ static int dcp_platform_probe(struct platform_device *pdev) of_property_read_u32(dev->of_node, "apple,dptx-phy", &dcp->dptx_phy); of_property_read_u32(dev->of_node, "apple,dptx-die", &dcp->dptx_die); dcp->fixed_dptx_phy = dcp->dptx_phy; + init_completion(&dcp->typec_iomfb_hpd_ready); INIT_DELAYED_WORK(&dcp->typec_reconnect_wq, dcp_typec_reconnect_work); INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index c42bc5cf015f5..ef89adad21849 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -44,7 +44,8 @@ bool dcp_has_typec_routes(struct platform_device *pdev); unsigned int dcp_typec_nr_ports(void); struct device_node *dcp_typec_port_of_node(unsigned int idx); bool dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev); -void dcp_typec_port_set_connector(unsigned int idx, struct apple_connector *connector); +void dcp_typec_port_set_connector(unsigned int idx, bool secondary, + struct apple_connector *connector); bool dcp_fw_compat_is_12_x(struct platform_device *pdev); unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev); void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index c3cf81082ed96..c88b8238d72e7 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1073,6 +1073,8 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) schedule_work(&dcp->vblank_wq); return; } + if (dcp_is_typec_output(dcp) && *connected && dcp->nr_modes) + complete_all(&dcp->typec_iomfb_hpd_ready); action = dcp_mode_hotplug(&dcp->mode_state, !!(*connected), connector ? &connector->connected : NULL); From fe6e4eed439673a3aae53955dcca240d0afd8021 Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Wed, 30 Sep 2026 09:54:14 -0600 Subject: [PATCH 1125/2680] drm/apple: prepare crossbar and PHY before DCP hears about a tunneled display With only the SoC checks widened, a Thunderbolt display on the MacBook Pro 14" M1 Pro (j314s) got as far as an active DP IN adapter. DCP then stayed silent for 5 s after Activate, every DPTX call timed out, and it reported link faults 22 and 24 without ever sending SET_LINK_RATE. All three Type-C routes of j314s use dcpext1 (apple,typec-mux-indices = <2 2 2>), while the DP IN output idles at dispext0. Point the output at the pipeline that takes the tunnel and wake the ATC's DP clock path before the DPTX connect, and put the output back at its idle source when the route goes away without the crossbar having come up. Both helpers return -EOPNOTSUPP where they do not apply, so t8103 and T602X behave as before. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Christian Bartels Co-Authored-By: Claude Opus 5.5 [Chris Kearney: ported onto aurora-silicon/linux#8 on top of the dual DP tunnel series; the tunnel is prepared once its slot is taken.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/dcp.c | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 4c836fbb84c5a..de3e986bbabae 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -133,6 +133,7 @@ static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, struct mux_control *xbar); static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); +static int dcp_dpxbar_preselect(struct mux_control *mux, int state); static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port); static int dcp_dpxbar_tunnel_select_source(struct mux_control *mux, int state) @@ -273,6 +274,8 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) scoped_guard(mutex, &dcp->tb_lock) { if (!route->tunnel || route->xbar_up) ret = mux_control_deselect(route->active_xbar ?: route->xbar); + else + dcp_dpxbar_preselect(route->active_xbar, MUX_IDLE_DISCONNECT); if (ret) return ret; route->xbar_up = false; @@ -557,6 +560,46 @@ static int dcp_dpxbar_link(struct mux_control *mux, bool up) return ret; } +static int dcp_dpxbar_preselect(struct mux_control *mux, int state) +{ + typeof(&apple_dpxbar_preselect) fn = symbol_get(apple_dpxbar_preselect); + int ret; + + if (!fn) + return -ENOENT; + ret = fn(mux, state); + symbol_put(apple_dpxbar_preselect); + return ret; +} + +/* + * Thunderbolt DP IN, before DCP hears about the display: point the DP IN + * output at this pipeline and wake the ATC's DP clock path. The crossbar + * connection itself still only comes up in dcp_tunnel_crossbar_up(), but an + * output left at its idle source (dispext0) only suits the pipeline that is + * dispext0: any other one activates the link, gets no answer from the sink + * and gives up with DEVICE_NOT_RESPONDING before it ever sets a link rate. + */ +static void dcp_tunnel_prepare(struct apple_dcp_typec_route *route, + struct mux_control *xbar) +{ + typeof(&apple_atc_dp_tunnel_open) open; + struct apple_dcp *dcp = route->dcp; + int ret; + + ret = dcp_dpxbar_preselect(xbar, route->mux_index); + if (ret && ret != -EOPNOTSUPP) + dev_warn(dcp->dev, "DP tunnel crossbar preselect failed: %d\n", ret); + + open = symbol_get(apple_atc_dp_tunnel_open); + if (!open) + return; + ret = open(dcp->phy); + symbol_put(apple_atc_dp_tunnel_open); + if (ret && ret != -EOPNOTSUPP) + dev_warn(dcp->dev, "DP tunnel PHY open failed: %d\n", ret); +} + /* DP IN adapter handshake through the thunderbolt glue; tb_lock held */ static int dcp_tunnel_dpin_locked(struct apple_dcp *dcp, bool active) { @@ -779,6 +822,7 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, return ret; } *slot = best; + dcp_tunnel_prepare(best, ctl); dev_info(dcp->dev, "display routed to Thunderbolt DP tunnel dpin%u\n", dpin); From 31d400ad28448aef878a1265aa941835d44d30ad Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Wed, 30 Sep 2026 09:54:14 -0600 Subject: [PATCH 1126/2680] drm/apple: a late clear swap at power-off is not a DCP crash iomfb_poweroff() waits 50 ms for the clear swap and sets dcp->crashed when that times out. When a Thunderbolt display is unplugged, the firmware is busy powering the external pipe down on its own and answers the swap a little later. Nothing logs the timeout, but from then on dcp_crtc_atomic_check() fails every commit with -EINVAL, so a replugged display stays dark until reboot: [CRTC:73] atomic driver check failed A real firmware crash is reported through the RTKit crash callback. Wait 500 ms, warn on a timeout and carry on with the power-off. Seen on j314s with an LG UltraFine 5K: unplug, then replug. With this change the unplug logs "dcp_poweroff() done" and the display comes back on replug. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Christian Bartels Co-Authored-By: Claude Opus 5.5 [Chris Kearney: ported onto aurora-silicon/linux#8, where DjDeveloperr's "preserve Type-C routes across blanking and detach" already stopped setting crashed but returned early; this carries on with the power-off. This is also the DCP unplug wedge Oliver Lukschander reported on #8.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/iomfb_template.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index c88b8238d72e7..5f66d731b49fe 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -956,19 +956,21 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) dcp_swap_start(dcp, false, &swap_req, dcp_swap_clear_started, cookie); - ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(50)); + /* + * On unplug the firmware powers the external pipe down on its own and + * can take tens of milliseconds before it answers (and swallows) the + * clear swap. That is not a crash: a real one is reported through the + * RTKit crash callback. Wait longer and carry on with the power-off + * either way, otherwise every later modeset fails with -EINVAL. + */ + ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(500)); swap_id = cookie->swap_id; kref_put(&cookie->refcount, release_swap_cookie); - if (ret <= 0) { - /* A disconnected external display can power down before it ACKs this - * clear swap. The deadline alone does not mean RTKit crashed; its - * crash callback owns the fatal flag. - */ - dev_warn(dcp->dev, "clear swap timed out during power-off\n"); - return; - } - - dev_dbg(dcp->dev, "%s: clear swap submitted: %u\n", __func__, swap_id); + if (ret <= 0) + dev_warn(dcp->dev, "%s: clear swap timed out\n", __func__); + else + dev_dbg(dcp->dev, "%s: clear swap submitted: %u\n", __func__, + swap_id); poff_cookie = kzalloc(sizeof(*poff_cookie), GFP_KERNEL); if (!poff_cookie) From c661a00bd3a3e4339446d13090af0f5247810fc8 Mon Sep 17 00:00:00 2001 From: RJ Skerry-Ryan Date: Sun, 27 Sep 2026 10:58:17 -0700 Subject: [PATCH 1127/2680] drm/apple: expose Type-C display connectors as DisplayPort The per-port Type-C connectors carry DisplayPort (DP alt mode or a Thunderbolt/USB4 DP tunnel), but were registered as DRM_MODE_CONNECTOR_USB, which DRM reserves for USB display adapters (gud, appletbdrm). Userspace keys on the type: systemd-logind only counts VGA/DVI/DP/HDMI/TV connectors as external displays, so a docked lid close suspended the machine instead of honoring HandleLidSwitchDocked=ignore. Register them as DisplayPort. The driver's internal "is Type-C" checks use dcp->connector_type / fixed_connector_type and are unaffected. Connectors are now named DP-N instead of USB-N. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: RJ Skerry-Ryan --- drivers/gpu/drm/apple/apple_drv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index c42604c875c54..6e1a806b5bbc3 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -471,7 +471,7 @@ static int apple_probe_typec_ports(struct drm_device *drm, ret = drm_connector_init(drm, &connector->base, &apple_connector_funcs, - DRM_MODE_CONNECTOR_USB); + DRM_MODE_CONNECTOR_DisplayPort); if (ret) { kfree(connector); return ret; From da74d02c56953ff5ff56d95bda77b36f467c7add Mon Sep 17 00:00:00 2001 From: RJ Skerry-Ryan Date: Sun, 27 Sep 2026 22:53:20 -0700 Subject: [PATCH 1128/2680] drm/apple: Keep dual-stream Type-C possible_crtcs fixed A Type-C port's encoder spans every pipeline that can drive the port, and routing narrows its possible_crtcs to the pipeline actually chosen, relying on userspace to re-read them on the following hotplug. Compositors that read possible_crtcs once when the connector appears (aquamarine, used by Hyprland) never see that. With a USB4 dock carrying two streams, whichever connector connects first takes the lowest free CRTC. When that is the DPIN1 connector, it gets dcpext0's CRTC while the stream is routed to dcpext1, every modeset fails the atomic check with -EINVAL, and both dock displays stay dark until the dock is replugged and happens to connect in the other order. possible_crtcs is meant to be static. On the dual-stream machines (J414s, J416c): - Fix the DPIN1 connector to the lowest-indexed Type-C-only pipeline (dcpext1), where DPIN1 is always routed, and route DPIN1 only within the connector's possible_crtcs. - Leave the primary connector spanning all its pipelines and stop narrowing or restoring possible_crtcs at runtime. - Route direct DP-alt displays to the lowest free CRTC, which is what a compositor picks from the port's fixed possible_crtcs, instead of steering them away from dcpext0. A direct DP-alt display now takes dcpext0 when it is free, so a single-stream dock on another port lands on the next free pipeline. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: RJ Skerry-Ryan [Chris Kearney: ported onto aurora-silicon/linux#8. dcp_typec_dual_stream() is apple_dp_tunnel_t602x() here, so j414c and j416s are dual-stream too; they share the j414/j416 device tree and the dcpext0/dcpext1 layout.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/apple_drv.c | 28 +++++++++--- drivers/gpu/drm/apple/dcp.c | 73 +++++++++++++++++++++++-------- drivers/gpu/drm/apple/dcp.h | 2 + 3 files changed, 79 insertions(+), 24 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 6e1a806b5bbc3..7166321b30004 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -270,9 +270,10 @@ apple_connector_atomic_best_encoder(struct drm_connector *conn, struct drm_encoder *encoder; /* - * A Type-C port has one encoder, and its possible_crtcs already names - * the pipeline the fabric routed the port to, so it is the only answer - * there is. + * A Type-C port has one encoder, whose possible_crtcs either already + * names the pipeline the fabric routed the port to or, on dual-stream + * machines, the fixed set the fabric only routes within, so it is the + * only answer there is. */ if (apple_connector->port_encoder) return apple_connector->port_encoder; @@ -454,7 +455,8 @@ static int apple_probe_typec_ports(struct drm_device *drm, * M2 Pro/Max laptops, append one more connector per port for a second * USB4 DP tunnel. */ - for (idx = 0; idx < nr_ports * (apple_dp_tunnel_t602x() ? 2 : 1); idx++) { + for (idx = 0; idx < nr_ports * (dcp_typec_dual_stream() ? 2 : 1); + idx++) { struct apple_connector *connector; struct apple_encoder *enc; unsigned int port_idx = idx % nr_ports; @@ -487,8 +489,22 @@ static int apple_probe_typec_ports(struct drm_device *drm, INIT_WORK(&connector->hotplug_wq, dcp_hotplug); for (i = 0; i < num_dcp; i++) { - if (dcp_typec_port_has_candidate(port_idx, dcp[i])) - mask |= crtc_mask[i]; + if (!dcp_typec_port_has_candidate(port_idx, dcp[i])) + continue; + /* + * A dock's second stream (DPIN1) always runs on the + * lowest-indexed Type-C-only pipeline (dcpext1). Fix the + * connector to it: compositors read possible_crtcs once + * and would otherwise pair it with a pipeline the stream + * can never be routed to. + */ + if (secondary) { + if (dcp_is_typec_only(dcp[i]) && + (!mask || crtc_mask[i] < mask)) + mask = crtc_mask[i]; + continue; + } + mask |= crtc_mask[i]; } if (!mask) { diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index de3e986bbabae..bc6ec5fdfadb5 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -130,6 +130,39 @@ static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) !dcp_typec_route_fixed_output_busy(route); } +/* + * Machines whose USB4 docks carry two independent DP streams through one + * port: DPIN0 drives the hybrid dcpext0, DPIN1 the Type-C-only dcpext1, and + * each port has a second connector for the DPIN1 stream. + */ +bool dcp_typec_dual_stream(void) +{ + return apple_dp_tunnel_t602x(); +} + +bool dcp_is_typec_only(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return !dcp->fixed_phy; +} + +/* + * Can this route feed the given connector without changing its + * possible_crtcs? On dual-stream machines those are fixed at probe, since + * compositors read them once when the connector appears and pair the + * connector with a CRTC before this fabric has routed it. + */ +static bool dcp_typec_route_fits(struct apple_dcp_typec_route *route, + struct apple_connector *connector) +{ + struct apple_dcp *dcp = route->dcp; + + if (!dcp_typec_dual_stream() || !connector || !dcp->crtc) + return true; + return connector->candidate_crtcs & drm_crtc_mask(&dcp->crtc->base); +} + static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, struct mux_control *xbar); static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); @@ -237,9 +270,12 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, * Offering it the choice makes it pair the port with a pipeline * holding a different monitor's mode list, and the modeset is * rejected with no way for it to recover. The hotplug that - * follows makes it re-read this. + * follows makes it re-read this. Dual-stream machines keep + * fixed possible_crtcs instead: compositors that read them once + * (e.g. aquamarine/Hyprland) never see the narrowing. */ - if (connector->port_encoder && dcp->crtc) + if (connector->port_encoder && dcp->crtc && + !dcp_typec_dual_stream()) connector->port_encoder->possible_crtcs = drm_crtc_mask(&dcp->crtc->base); } @@ -323,7 +359,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) connector->dcp = NULL; /* Unrouted: the port could go to any of its pipelines again. */ - if (connector->port_encoder) + if (connector->port_encoder && !dcp_typec_dual_stream()) connector->port_encoder->possible_crtcs = connector->candidate_crtcs; } @@ -491,15 +527,12 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (!dcp_typec_route_available(candidate)) continue; - score = dcp_typec_route_score(candidate); /* - * On the M2 Pro/Max laptops, leave dcpext0 free for - * a dock's DP tunnel when a direct DP-alt-mode - * display can use the Type-C-only dcpext1 pipeline. + * Lowest free CRTC index first: on dual-stream + * machines that is what a compositor picks from + * the port's fixed possible_crtcs. */ - if (apple_dp_tunnel_t602x() && - candidate->dcp->fixed_phy) - score += 100; + score = dcp_typec_route_score(candidate); if (score < best_score) { best = candidate; best_score = score; @@ -782,16 +815,19 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, if (!dcp_typec_route_available(candidate)) continue; + if (!dcp_typec_route_fits(candidate, dpin ? + port->secondary_connector : + port->connector)) + continue; score = dcp_typec_route_score(candidate); /* - * On the M2 Pro/Max laptops DPIN0 prefers the hybrid dcpext0, - * which completes tunneled link training, and DPIN1 prefers - * dcpext1, so the two pipelines drive independent streams - * through one dock. + * DPIN0 prefers the hybrid dcpext0, which completes tunneled + * link training. On dual-stream machines DPIN1 is confined to + * dcpext1 by its connector's possible_crtcs, so both pipelines + * drive independent streams through one dock. */ - if (apple_dp_tunnel_t602x() && score < UINT_MAX - 100 && - ((dpin == 0 && !candidate->dcp->fixed_phy) || - (dpin == 1 && candidate->dcp->fixed_phy))) + if (dcp_typec_dual_stream() && dpin == 0 && + !candidate->dcp->fixed_phy && score < UINT_MAX - 100) score += 100; if (score < best_score) { best = candidate; @@ -965,7 +1001,8 @@ void dcp_typec_port_set_connector(unsigned int idx, bool secondary, if (owner) { struct apple_dcp *dcp = owner->dcp; - if (dcp->crtc && connector->port_encoder) + if (dcp->crtc && connector->port_encoder && + !dcp_typec_dual_stream()) connector->port_encoder->possible_crtcs = drm_crtc_mask(&dcp->crtc->base); diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index ef89adad21849..0e21184fd6d75 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -46,6 +46,8 @@ struct device_node *dcp_typec_port_of_node(unsigned int idx); bool dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev); void dcp_typec_port_set_connector(unsigned int idx, bool secondary, struct apple_connector *connector); +bool dcp_typec_dual_stream(void); +bool dcp_is_typec_only(struct platform_device *pdev); bool dcp_fw_compat_is_12_x(struct platform_device *pdev); unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev); void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, From cd578da7cc2dfc457fb2aab19c3d698fe314142c Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 10:14:44 -0700 Subject: [PATCH 1129/2680] drm/apple: route Type-C displays in compositor pairing order On the M2 Pro/Max 14" and 16" laptops (J414s/c, J416s/c) the Type-C ports share the external pipelines: dcpext0 (which also drives HDMI) and dcpext1, plus dcpext2/3 on the M2 Max boards. Since "drm/apple: Keep dual-stream Type-C possible_crtcs fixed", every port's connector offers all of them. Compositors read possible_crtcs once and pair connectors with CRTCs themselves: aquamarine (Hyprland) walks the CRTCs in index order and gives each to the first connected connector, in connector order, that can use it. The fabric instead hands out pipelines in whatever order the Type-C and Thunderbolt events arrive. That is a regression from the commit above on a J416c with two direct DP-alt monitors at boot: when port 1's event was handled first, port 1 took dcpext0 and port 0 dcpext1, and Hyprland paired them the other way round. Each connector's modes were then checked against the other monitor's list, native modes failed the atomic check with -EINVAL, and only modes both monitors share could be set. fbdev, which pairs every connected display afresh on each hotplug, and a dock's DPIN0 stream attaching after a direct display cross the same way. Until a compositor owns the display, keep the routes equal to that pairing computed from scratch: pipelines in CRTC index order, each to the first stream that wants one and has it in its possible_crtcs, with the primary connectors in port order ahead of the DPIN1 ones. A hybrid pipeline whose HDMI output is live is left to it, unless a tunnel holds it. A direct DP-alt stream wants a pipeline while its sink asserts HPD, as only then can its connector read connected. Only direct routes move, each to exactly its planned pipeline. A Thunderbolt tunnel never moves once set up. A direct stream planned onto a pipeline a tunnel holds stays unrouted and reads disconnected, since putting it anywhere else would have the compositor cross it with another display; it is then left out and the pass re-run, so the streams after it are not planned one pipeline off. A move is a disconnect and reconnect: every route that moves is taken down before any comes up, so two never share a pipeline, HPD is replayed on the new pipeline, and the hotplugs make fbdev and userspace re-probe. The pass runs on every DP-alt and tunnel attach, detach and HPD change, when HDMI is plugged in while a Type-C route has its pipeline, once DRM is registered, and from postclose when a file that was DRM master is closed; drm_release() has taken that file off the file list by then. A compositor owns the display while any open file has been DRM master: it pairs its connectors when it starts and keeps that pairing across a VT switch, where it drops master but keeps the device open. Until the last such file is closed routing stays as it was, the lowest free pipeline on DP entry with preferred_route honoured, and nothing moves. The check walks the file list under filelist_mutex, taken inside the fabric lock; nothing takes the two the other way round. cd321x does not report an unchanged DP state again, so a port left without a pipeline used to stay dark for the session. When a route lets its pipeline go while a compositor owns the display, such a port now goes back to the pipeline it last had if that is free, as the compositor keeps a reconnected connector's CRTC, and otherwise takes the lowest free one, the CRTC it gives a newly connected connector. Hotplug and retrain work queued for a Type-C connector can run after its route is gone and connector->dcp is NULL; check for that instead of dereferencing it. possible_crtcs stay static, since compositors read them only once, and the DPIN1 connector stays fixed to dcpext1. Single-stream machines take none of the new paths. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- drivers/gpu/drm/apple/apple_drv.c | 15 + drivers/gpu/drm/apple/dcp.c | 525 ++++++++++++++++++++++++++++-- drivers/gpu/drm/apple/dcp.h | 1 + drivers/gpu/drm/apple/iomfb.c | 19 +- 4 files changed, 522 insertions(+), 38 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 7166321b30004..ef6f8a956ff17 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -66,9 +66,21 @@ static int apple_drm_gem_dumb_create(struct drm_file *file_priv, return drm_gem_dma_dumb_create_internal(file_priv, drm, args); } +/* + * A compositor or boot splash is closing the device. drm_release() has + * already taken its file off the file list, so the Type-C fabric can tell + * whether any other one still has the device open. + */ +static void apple_drm_postclose(struct drm_device *drm, struct drm_file *file) +{ + if (file->was_master) + dcp_typec_reorder(); +} + static const struct drm_driver apple_drm_driver = { DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(apple_drm_gem_dumb_create), DRM_FBDEV_DMA_DRIVER_OPS, + .postclose = apple_drm_postclose, .name = DRIVER_NAME, .desc = DRIVER_DESC, .major = 1, @@ -722,6 +734,9 @@ static int apple_drm_init(struct device *dev) if (ret) goto err_unbind; + /* the fabric keeps order only once registered: catch up on probe */ + dcp_typec_reorder(); + drm_client_setup_with_fourcc(&apple->drm, DRM_FORMAT_XRGB8888); return 0; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index bc6ec5fdfadb5..2d57a5f8312d2 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -83,6 +83,14 @@ struct apple_dcp_typec_port { u32 applied_conf; bool applied_valid; bool hpd; + /* Direct DP-alt: the port is in DP mode, routed or not, and its HPD. */ + bool dp_wanted; + bool dp_hpd; + /* dcp_typec_rebalance_locked()'s plan for the port's two streams */ + struct apple_dcp_typec_route *target; + struct apple_dcp_typec_route *secondary_target; + /* left out of the plan: its planned pipeline is held by a tunnel */ + bool plan_dark; }; static DEFINE_MUTEX(dcp_typec_fabric_lock); @@ -258,7 +266,7 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, dcp->dptx_phy = route->dptx_phy; dcp->connector_type = DRM_MODE_CONNECTOR_USB; if (connector) { - connector->dcp = to_platform_device(dcp->dev); + WRITE_ONCE(connector->dcp, to_platform_device(dcp->dev)); dcp->typec_connector = connector; dcp->connector = connector; @@ -356,7 +364,8 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) * display has come back up on it. */ WRITE_ONCE(connector->connected, false); - connector->dcp = NULL; + /* hotplug work queued before this checks for it */ + WRITE_ONCE(connector->dcp, NULL); /* Unrouted: the port could go to any of its pipelines again. */ if (connector->port_encoder && !dcp_typec_dual_stream()) @@ -408,8 +417,10 @@ static void dcp_typec_retrain_work(struct work_struct *work) container_of(to_delayed_work(work), struct apple_dcp, typec_fabric_retrain_wq); - if (READ_ONCE(dcp->active_typec_route) && dcp->typec_connector) - dcp_retrain_oob(dcp->typec_connector); + struct apple_connector *connector = READ_ONCE(dcp->typec_connector); + + if (READ_ONCE(dcp->active_typec_route) && connector) + dcp_retrain_oob(connector); } static void dcp_typec_retrain_active_routes(void) @@ -428,18 +439,406 @@ static void dcp_typec_retrain_active_routes(void) } } +static struct apple_dcp_typec_route * +dcp_typec_port_route(struct apple_dcp_typec_port *port, struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + + list_for_each_entry(route, &port->routes, port_link) + if (route->dcp == dcp) + return route; + + return NULL; +} + +/* + * The DRM device once it has bound, that is once every pipeline has its + * CRTC and every port its connectors; NULL before. + */ +static struct drm_device *dcp_typec_drm(void) +{ + struct apple_dcp_typec_port *port; + struct apple_dcp_typec_route *route; + struct drm_device *drm = NULL; + + list_for_each_entry(port, &dcp_typec_ports, link) { + if (!port->connector || !port->secondary_connector) + return NULL; + list_for_each_entry(route, &port->routes, port_link) { + if (!route->dcp->crtc) + return NULL; + drm = route->dcp->crtc->base.dev; + } + } + + return drm; +} + +/* + * Has a compositor (or boot splash) taken the display? It paired its + * connectors with CRTCs when it started and keeps that pairing, also + * across a VT switch, where it drops DRM master but keeps the device + * open; moving routes under it would hand connectors to pipelines driving + * other displays. So the routes are frozen while any open file has ever + * been master, and thaw when the last such file is closed. + * + * Taken under dcp_typec_fabric_lock, next to the routing it decides, so + * the order is the fabric lock, then filelist_mutex. Nothing nests them + * the other way: filelist_mutex only covers list edits and walks, and + * drm_release() drops it before drm_file_free() calls postclose, which is + * where a close takes the fabric lock. drm_open() makes a file master + * before adding it here, but its owner cannot have paired anything before + * open() returns, and it re-probes on the hotplugs the moves send. + */ +static bool dcp_typec_frozen(struct drm_device *drm) +{ + struct drm_file *file; + + guard(mutex)(&drm->filelist_mutex); + list_for_each_entry(file, &drm->filelist, lhead) { + /* set under master_mutex, and only ever from false to true */ + if (READ_ONCE(file->was_master)) + return true; + } + + return false; +} + +/* Are the routes kept in compositor pairing order right now? */ +static bool dcp_typec_keep_order(void) +{ + struct drm_device *drm; + + if (!dcp_typec_dual_stream()) + return false; + drm = dcp_typec_drm(); + + return drm && READ_ONCE(drm->registered) && !dcp_typec_frozen(drm); +} + +/* A Thunderbolt tunnel holds @dcp's pipeline: it never moves. */ +static bool dcp_typec_tunnel_held(struct apple_dcp *dcp) +{ + return dcp->active_typec_route && dcp->active_typec_route->tunnel; +} + +/* + * Give @dcp's pipeline to the first stream, in connector order, that + * wants one, has none yet and has it in its possible_crtcs. A tunnel + * stream wants one once it is set up, or as @arriving/@dpin. A direct + * DP-alt stream wants one once its sink asserts HPD: only then can its + * connector read connected, and a compositor pairs only those. + */ +static void dcp_typec_plan_pipeline(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct apple_dcp_typec_port *arriving, + unsigned int dpin) +{ + struct apple_dcp_typec_port *port; + bool secondary = false; + + /* primary connectors in port order, then the DPIN1 ones */ + do { + list_for_each_entry(port, &dcp_typec_ports, link) { + struct apple_dcp_typec_route **slot = secondary ? + &port->secondary_target : &port->target; + struct apple_connector *connector = secondary ? + port->secondary_connector : port->connector; + struct apple_dcp_typec_route *route = + dcp_typec_port_route(port, dcp); + bool wants; + + if (secondary) + wants = port->secondary_owner || + (port == arriving && dpin == 1); + else + wants = (port->owner && port->owner->tunnel) || + (port->dp_wanted && port->dp_hpd && + !port->plan_dark) || + (port == arriving && dpin == 0); + + if (!wants || *slot || !route || + !(connector->candidate_crtcs & drm_crtc_mask(crtc))) + continue; + + *slot = route; + return; + } + secondary = !secondary; + } while (secondary); +} + +/* + * The pairing a compositor starting now would make, wherever the streams + * sit at the moment: pipelines in CRTC index order, each to the first + * stream that wants it. A pipeline whose fixed output is live is left to + * that output, unless a tunnel holds it: the output cannot have it back + * then, and its connector reads disconnected. + * + * Tunnels are planned in connector order like any other stream, not + * seated on the pipelines they hold: the compositor pairs their connectors + * by order too, and seating them would plan the direct streams around + * pairings it never makes. A direct stream planned onto a pipeline a + * tunnel holds cannot have it and stays dark, so it is left out and the + * pass re-run, or every stream after it would be planned one pipeline + * off. Streams only ever drop out, so that settles within one pass per + * port. It also brings a tunnel's plan onto the pipeline it holds + * wherever direct streams ahead of it were in the way. A tunnel still + * planned elsewhere cannot be brought there by any direct stream dropping + * out, and the compositor pairs its connector wrongly whatever they do. + */ +static void dcp_typec_plan(struct drm_device *drm, + struct apple_dcp_typec_port *arriving, unsigned int dpin) +{ + struct apple_dcp_typec_port *port; + struct drm_crtc *crtc; + bool again; + + list_for_each_entry(port, &dcp_typec_ports, link) + port->plan_dark = false; + + do { + list_for_each_entry(port, &dcp_typec_ports, link) { + port->target = NULL; + port->secondary_target = NULL; + } + + /* CRTCs are listed in index order */ + drm_for_each_crtc(crtc, drm) { + struct apple_dcp *dcp = + platform_get_drvdata(to_apple_crtc(crtc)->dcp); + + if (dcp->nr_typec_routes && + (dcp_typec_tunnel_held(dcp) || + !dcp_typec_route_fixed_output_busy(&dcp->typec_routes[0]))) + dcp_typec_plan_pipeline(dcp, crtc, arriving, dpin); + } + + again = false; + list_for_each_entry(port, &dcp_typec_ports, link) { + if (port == arriving || !port->target || + (port->owner && port->owner->tunnel) || + !dcp_typec_tunnel_held(port->target->dcp)) + continue; + port->plan_dark = true; + again = true; + } + } while (again); +} + +/* Replay HPD to the pipeline a direct DP-alt port has just been given. */ +static void dcp_typec_port_attach(struct apple_dcp_typec_port *port) +{ + struct apple_dcp *dcp = port->owner->dcp; + + port->hpd = port->dp_hpd; + if (!port->hpd) + return; + + WRITE_ONCE(dcp->typec_cable_connected, true); + if (dcp->typec_connector) + dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); +} + +static struct apple_dcp_typec_route * +dcp_typec_lowest_free(struct apple_dcp_typec_port *port) +{ + struct apple_dcp_typec_route *candidate, *best = NULL; + unsigned int best_score = UINT_MAX; + + list_for_each_entry(candidate, &port->routes, port_link) { + unsigned int score; + + if (!dcp_typec_route_available(candidate)) + continue; + /* + * Lowest free CRTC index first: on dual-stream machines that + * is what a compositor picks from the port's fixed + * possible_crtcs. + */ + score = dcp_typec_route_score(candidate); + if (score < best_score) { + best = candidate; + best_score = score; + } + } + + return best; +} + +/* + * Route the direct DP-alt ports left waiting for a pipeline. Their DP + * state is not reported again while it stays the same, so nothing else + * would. Each goes back to the pipeline it last had if that is free, as + * a compositor keeps a reconnected connector's CRTC, and otherwise takes + * the lowest free one, the CRTC a compositor gives a new connector. + */ +static void dcp_typec_route_waiting(void) +{ + struct apple_dcp_typec_port *port; + struct apple_dcp_typec_route *route; + + list_for_each_entry(port, &dcp_typec_ports, link) { + if (port->owner || !port->dp_wanted || !port->dp_hpd) + continue; + + route = port->preferred_route; + if (!route || !dcp_typec_route_available(route)) + route = dcp_typec_lowest_free(port); + if (!route || dcp_typec_route_activate(route, route->xbar)) + continue; + port->owner = route; + port->dp_release_deadline = 0; + dcp_typec_port_attach(port); + } +} + +/* + * Keep the Type-C routes where a compositor starting now expects them. + * Dual-stream machines keep possible_crtcs fixed, and compositors read + * them once and pair connectors with CRTCs themselves: aquamarine + * (Hyprland) walks the CRTCs in index order and gives each to the first + * connected connector, in connector order, that can use it. A connector + * paired with a pipeline other than the one routed to its display has its + * modes checked against the other display's list, so its modesets fail. + * The Type-C and Thunderbolt events that route the ports come in no + * particular order, so until a compositor owns the display (see + * dcp_typec_frozen()) every route change re-runs that pairing from + * scratch (dcp_typec_plan()) and follows it. + * + * Only direct DP-alt routes move: each goes to exactly its planned + * pipeline. A Thunderbolt tunnel never moves once set up, so where one + * sits on a pipeline planned for a direct stream, or the plan has nothing + * for it, that stream stays unrouted and its connector disconnected. + * Placing it on some other pipeline would have the compositor cross both + * displays; dark is the better failure. @arriving/@dpin is a tunnel + * stream asking for a pipeline, planned like any other; its route is + * returned for apple_dcp_tb_dp_tunnel() to set up, or NULL if the plan + * has none for it or another tunnel holds that one. + * + * A move is an unplug and replug. Every route that moves is taken down + * first, so that two never share a pipeline midway, then each goes up on + * its new pipeline with its HPD replayed; the hotplugs this sends make + * fbdev and userspace re-probe. + */ +static struct apple_dcp_typec_route * +dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, + unsigned int dpin) +{ + struct drm_device *drm = dcp_typec_drm(); + struct apple_dcp_typec_route *planned = NULL; + struct apple_dcp_typec_port *port; + + lockdep_assert_held(&dcp_typec_fabric_lock); + + dcp_typec_plan(drm, arriving, dpin); + if (arriving) { + planned = dpin ? arriving->secondary_target : arriving->target; + /* it will be refused: plan as if it had not asked */ + if (planned && dcp_typec_tunnel_held(planned->dcp)) { + planned = NULL; + dcp_typec_plan(drm, NULL, 0); + } + } + + list_for_each_entry(port, &dcp_typec_ports, link) { + struct apple_dcp_typec_route *owner = port->owner; + struct apple_dcp *dcp; + + if (!owner || owner->tunnel || owner == port->target) + continue; + + dcp = owner->dcp; + dev_info(dcp->dev, "re-routing %pOF from %s to %s for Type-C connector order\n", + port->connector_np, dev_name(dcp->dev), + port->target ? dev_name(port->target->dcp->dev) : "none"); + if (port->hpd || dcp->typec_cable_connected || + (dcp->typec_connector && dcp->typec_connector->connected)) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + port->hpd = false; + if (dcp_typec_route_deactivate(owner) && owner->selected) { + /* still routed: leave the display where it is */ + dcp_typec_port_attach(port); + continue; + } + port->owner = NULL; + + /* as after a DP exit, hand the hybrid back to a live HDMI */ + if (dcp->hdmi_hpd && dcp->active && + gpiod_get_value_cansleep(dcp->hdmi_hpd)) + dcp_dptx_connect(dcp, 0); + } + + list_for_each_entry(port, &dcp_typec_ports, link) { + struct apple_dcp_typec_route *route = port->target; + + if (port->owner || !port->dp_wanted || !port->dp_hpd || + port == arriving) + continue; + + if (!route || !dcp_typec_route_available(route) || + dcp_typec_route_activate(route, route->xbar)) { + /* + * Dark for now. Its unchanged DP state is not reported + * again, so it is placed when the plan next runs, or + * once a compositor owns the display, when a pipeline + * is freed (dcp_typec_route_waiting()). + */ + port->applied_valid = false; + continue; + } + port->owner = route; + port->preferred_route = route; + port->dp_release_deadline = 0; + dcp_typec_port_attach(port); + } + + return planned; +} + +/* + * A route has let its pipeline go. Until a compositor owns the display + * the plan places everything anew; after that the pipeline goes to a port + * left waiting for one. + */ +static void dcp_typec_pipeline_freed(void) +{ + if (!dcp_typec_dual_stream()) + return; + + if (dcp_typec_keep_order()) + dcp_typec_rebalance_locked(NULL, 0); + else + dcp_typec_route_waiting(); +} + +/* + * Re-run the pairing pass from outside the fabric: once DRM is registered, + * as ports routed before that could not follow it, and when the last + * compositor or boot splash has closed the device, as the next one pairs + * the connectors from scratch. + */ +void dcp_typec_reorder(void) +{ + if (!dcp_typec_dual_stream()) + return; + + guard(mutex)(&dcp_typec_fabric_lock); + + if (dcp_typec_keep_order()) + dcp_typec_rebalance_locked(NULL, 0); +} + static int dcp_typec_route_set(struct typec_mux_dev *mux, struct typec_mux_state *state) { struct apple_dcp_typec_route *route = typec_mux_get_drvdata(mux); struct apple_dcp_typec_port *port = route->port; - struct apple_dcp_typec_route *candidate, *best = NULL; + struct apple_dcp_typec_route *best = NULL; bool is_dp = dcp_typec_route_is_dp(state); struct typec_displayport_data *dp_data = is_dp ? state->data : NULL; u32 dp_status = dp_data ? dp_data->status : 0; u32 dp_conf = dp_data ? dp_data->conf : 0; - unsigned int best_score = UINT_MAX; - bool hpd; + bool hpd, was_counted; int ret = 0; guard(mutex)(&dcp_typec_fabric_lock); @@ -465,7 +864,13 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, /* Failed route acquisition must remain retryable on the next update. */ port->applied_valid = false; + /* did the pairing pass count this port's direct stream so far? */ + was_counted = port->dp_wanted && port->dp_hpd; + if (!is_dp) { + port->dp_wanted = false; + port->dp_hpd = false; + /* a Thunderbolt/USB4 DP tunnel is torn down by its own path */ if (port->owner && port->owner->tunnel) { port->applied_valid = true; @@ -488,6 +893,10 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (dcp->hdmi_hpd && dcp->active && gpiod_get_value_cansleep(dcp->hdmi_hpd)) dcp_dptx_connect(dcp, 0); + dcp_typec_pipeline_freed(); + } else if (was_counted && dcp_typec_keep_order()) { + /* a stream the plan left dark is gone: plan the rest */ + dcp_typec_rebalance_locked(NULL, 0); } /* @@ -516,29 +925,43 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, return 0; } + hpd = dp_data && (dp_data->status & DP_STATUS_HPD_STATE); + port->dp_wanted = true; + port->dp_hpd = hpd; + + /* + * Until a compositor owns the display, a direct DP-alt stream has a + * pipeline only while its sink asserts HPD, and the pairing pass + * places it: HPD coming or going is the stream connecting or + * disconnecting as far as a compositor can tell. Otherwise the port + * takes the lowest free pipeline on DP entry, as a compositor does. + */ + if (dcp_typec_keep_order()) { + struct apple_dcp_typec_route *owner = port->owner; + + /* it connects or disconnects, or is not routed as it should be */ + if (hpd != was_counted || !owner != !hpd) + dcp_typec_rebalance_locked(NULL, 0); + if (!port->owner) { + if (hpd) + return -EBUSY; + port->applied_valid = true; + return 0; + } + /* just attached, its HPD replayed: nothing left to apply */ + if (port->owner != owner) { + port->applied_valid = true; + return 0; + } + } + if (!port->owner) { if (port->preferred_route && dcp_typec_route_available(port->preferred_route)) best = port->preferred_route; - if (!best) { - list_for_each_entry(candidate, &port->routes, port_link) { - unsigned int score; - - if (!dcp_typec_route_available(candidate)) - continue; - /* - * Lowest free CRTC index first: on dual-stream - * machines that is what a compositor picks from - * the port's fixed possible_crtcs. - */ - score = dcp_typec_route_score(candidate); - if (score < best_score) { - best = candidate; - best_score = score; - } - } - } + if (!best) + best = dcp_typec_lowest_free(port); if (!best) return -EBUSY; @@ -551,7 +974,6 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, } - hpd = dp_data && (dp_data->status & DP_STATUS_HPD_STATE); if (!hpd && port->hpd) { dcp_dptx_disconnect_oob(to_platform_device(port->owner->dcp->dev), 0); } else if (hpd && !port->hpd) { @@ -751,11 +1173,12 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, void *ctx) { struct apple_dcp_typec_port *port = NULL, *pos; - struct apple_dcp_typec_route *candidate, *best = NULL; + struct apple_dcp_typec_route *candidate, *best = NULL, *planned = NULL; struct apple_dcp_typec_route **slot; unsigned int best_score = UINT_MAX; struct mux_control *ctl; struct apple_dcp *dcp; + bool ordered; int ret; if (!connector_np || dpin > 1) @@ -797,6 +1220,7 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, if (dcp->hdmi_hpd && dcp->active && gpiod_get_value_cansleep(dcp->hdmi_hpd)) dcp_dptx_connect(dcp, 0); + dcp_typec_pipeline_freed(); return 0; } @@ -810,9 +1234,20 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, if (port->owner && !port->owner->tunnel) return -EBUSY; + /* + * Until a compositor owns the display, the pairing pass decides where + * the stream goes, moving direct DP-alt routes out of its way, and + * whether it gets a pipeline at all. + */ + ordered = dcp_typec_keep_order(); + if (ordered) + planned = dcp_typec_rebalance_locked(port, dpin); + list_for_each_entry(candidate, &port->routes, port_link) { unsigned int score; + if (ordered && candidate != planned) + continue; if (!dcp_typec_route_available(candidate)) continue; if (!dcp_typec_route_fits(candidate, dpin ? @@ -834,13 +1269,17 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, best_score = score; } } - if (!best) - return -EBUSY; + if (!best) { + ret = -EBUSY; + goto err_reorder; + } /* The route's crossbar control is dpphy (0); dpin0/dpin1 are 1/2. */ if (best->xbar != &best->xbar->chip->mux[0] || - best->xbar->chip->controllers < 3) - return -EOPNOTSUPP; + best->xbar->chip->controllers < 3) { + ret = -EOPNOTSUPP; + goto err_reorder; + } ctl = &best->xbar->chip->mux[1 + dpin]; dcp = best->dcp; @@ -855,9 +1294,11 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, dcp->tb_dpin_set_active = NULL; dcp->tb_dpin_ctx = NULL; } - return ret; + goto err_reorder; } *slot = best; + /* the port is in USB4 mode, not DP-alt */ + port->dp_wanted = false; dcp_tunnel_prepare(best, ctl); dev_info(dcp->dev, "display routed to Thunderbolt DP tunnel dpin%u\n", dpin); @@ -876,6 +1317,12 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); return 0; + +err_reorder: + /* the pass kept a pipeline for this stream: give it to the others */ + if (planned) + dcp_typec_rebalance_locked(NULL, 0); + return ret; } EXPORT_SYMBOL_GPL(apple_dcp_tb_dp_tunnel); @@ -1819,8 +2266,20 @@ static irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) guard(mutex)(&dcp_typec_fabric_lock); - if (READ_ONCE(dcp->active_typec_route)) + if (READ_ONCE(dcp->active_typec_route)) { + /* + * Until a compositor owns the display, a live HDMI output + * takes its pipeline back from a direct DP-alt route: the + * compositor pairs the HDMI connector with it first. + */ + if (dcp_typec_keep_order() && + gpiod_get_value_cansleep(dcp->hdmi_hpd)) { + msleep(500); + if (gpiod_get_value_cansleep(dcp->hdmi_hpd)) + dcp_typec_rebalance_locked(NULL, 0); + } return IRQ_HANDLED; + } connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); /* do nothing on disconnect and trust that dcp detects it itself. diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 0e21184fd6d75..efae9081072f1 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -47,6 +47,7 @@ bool dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev void dcp_typec_port_set_connector(unsigned int idx, bool secondary, struct apple_connector *connector); bool dcp_typec_dual_stream(void); +void dcp_typec_reorder(void); bool dcp_is_typec_only(struct platform_device *pdev); bool dcp_fw_compat_is_12_x(struct platform_device *pdev); unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 40253e8ddfb27..0862f2306604a 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -265,10 +265,13 @@ void dcp_handle_hotplug_actions(struct apple_dcp *dcp, unsigned int action) void dcp_retrain_oob(struct apple_connector *connector) { - struct apple_dcp *dcp = platform_get_drvdata(connector->dcp); + struct platform_device *pdev = READ_ONCE(connector->dcp); + struct apple_dcp *dcp; - if (!READ_ONCE(connector->connected)) + /* a Type-C port with no pipeline behind it has nothing to retrain */ + if (!pdev || !READ_ONCE(connector->connected)) return; + dcp = platform_get_drvdata(pdev); /* * Bringing up another high-speed Type-C route can disturb an active DPTX @@ -283,12 +286,18 @@ void dcp_retrain_oob(struct apple_connector *connector) void dcp_hotplug(struct work_struct *work) { struct apple_connector *connector; + struct platform_device *pdev; struct apple_dcp *dcp; int ret; connector = container_of(work, struct apple_connector, hotplug_wq); - dcp = platform_get_drvdata(connector->dcp); + pdev = READ_ONCE(connector->dcp); + if (!pdev) { /* a Type-C port unrouted after this was queued */ + drm_kms_helper_connector_hotplug_event(&connector->base); + return; + } + dcp = platform_get_drvdata(pdev); dev_info(dcp->dev, "%s() connected:%d valid_mode:%d nr_modes:%u\n", __func__, connector->connected, READ_ONCE(dcp->mode_state.valid), dcp->nr_modes); @@ -411,7 +420,7 @@ static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) int dcp_get_modes(struct drm_connector *connector) { struct apple_connector *apple_connector = to_apple_connector(connector); - struct platform_device *pdev = apple_connector->dcp; + struct platform_device *pdev = READ_ONCE(apple_connector->dcp); struct apple_dcp *dcp; struct drm_device *dev = connector->dev; @@ -498,7 +507,7 @@ enum drm_mode_status dcp_mode_valid(struct drm_connector *connector, const struct drm_display_mode *mode) { struct apple_connector *apple_connector = to_apple_connector(connector); - struct platform_device *pdev = apple_connector->dcp; + struct platform_device *pdev = READ_ONCE(apple_connector->dcp); struct apple_dcp *dcp; if (!pdev) From 16e49598690a72c51a321cfa82c5cc5ea8fab8a2 Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Wed, 30 Sep 2026 12:26:34 -0600 Subject: [PATCH 1130/2680] drm/apple: accept an EDID longer than its base block announces dcpavserv_copy_edid() rejects an EDID unless the payload is exactly (1 + extension count) blocks. An HDMI 2.x sink that uses the HDMI Forum EDID Extension Override Data Block (HF-EEODB) sets the extension count in the base block to 1 and puts the real count in the HF-EEODB, so the DCP returns more blocks than the base block says. The EDID is dropped with -EIO, the connector has no EDID, and userspace sees no make or model. Accept any whole number of blocks at least as long as the base block announces and let drm_edid, which handles HF-EEODB, decide how many blocks to use. Also bound used_size by the buffer size, which the exact match used to do implicitly. Seen on j314s (M1 Pro) with a Dell S3225QS on the HDMI port: used_size 392, base block announces 1 extension (256 bytes), payload 384 bytes. With this change the connector gets the EDID. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Christian Bartels Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/apple/epic/dpavservep.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index b9f7c619dd4ed..4ddce69860379 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -213,8 +213,16 @@ static const struct drm_edid *dcpavserv_read_edid(struct apple_epic_service *ser data_size > EDID_BUF_SIZE) return ERR_PTR(-EIO); + /* + * An HDMI 2.x sink using the HF-EEODB announces fewer extension blocks + * in the base block than it sends, so the payload may be longer than + * the base block says. Accept any whole number of blocks at least that + * long and leave the block count to drm_edid, which knows about + * HF-EEODB. + */ num_blocks = resp->data[EDID_LEADING_DATA_SIZE + EDID_EXT_BLOCK_COUNT_OFFSET]; - if ((1 + num_blocks) * EDID_BLOCK_SIZE != data_size - EDID_LEADING_DATA_SIZE) + if ((1 + num_blocks) * EDID_BLOCK_SIZE > data_size - EDID_LEADING_DATA_SIZE || + (data_size - EDID_LEADING_DATA_SIZE) % EDID_BLOCK_SIZE) return ERR_PTR(-EIO); return drm_edid_alloc(resp->data + EDID_LEADING_DATA_SIZE, From 13b2abedf0b53fde55184cbe018afd19bf6f7ca4 Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Sun, 4 Oct 2026 10:00:39 -0600 Subject: [PATCH 1131/2680] drm/apple: audio: release data_lock when DMA setup fails on open dcp_pcm_open() takes data_lock before dcpaud_init_dma() and returns its error with the lock still held. The next open, the hotplug notifications (dcpaud_connect()/dcpaud_disconnect()) and hw_params/prepare (through dcpaud_connection_up()) all take data_lock and then block forever. Unlock before returning. Signed-off-by: Christian Bartels --- drivers/gpu/drm/apple/audio.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index 38718e2f56117..5446038f70097 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -251,8 +251,10 @@ static int dcp_pcm_open(struct snd_pcm_substream *substream) mutex_lock(&dcpaud->data_lock); ret = dcpaud_init_dma(dcpaud); - if (ret < 0) + if (ret < 0) { + mutex_unlock(&dcpaud->data_lock); return ret; + } if (!dcpaud->connected) { mutex_unlock(&dcpaud->data_lock); From c92f63d63895c8dac69c30d80a4e7f9abf72970a Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Sun, 4 Oct 2026 10:01:03 -0600 Subject: [PATCH 1132/2680] drm/apple: audio: allow opening the PCM without a sink dcp_pcm_open() fails with -ENXIO while no display is connected, because the format and rate constraints come from the sink's audio elements. PipeWire and PulseAudio probe every PCM of a UCM profile once, when the card appears: they open it, set hw_params and prepare it. If that fails, the profile is marked unsupported for good, and udev "change" events for a card that has already been seen are ignored. So on a Mac booted without a monitor on HDMI or USB-C, a display attached later never gets an audio output until the sound server is restarted. Let the PCM open without a sink, with a plain stereo fallback (2 channels; 32, 44.1 or 48 kHz; S16, S24 or S32), as HDMI PCMs on other platforms do. hw_params and prepare succeed as no-ops for such an open, since alsa-lib prepares straight after hw_params. trigger still fails until a sink is connected, so nothing is sent to a missing display. Once a display is attached, the next open gets the sink's real constraints. Tested on a MacBook Pro 14" M1 Pro (j314s) with a 4K monitor on the built-in HDMI port: booted without a monitor on HDMI, the HDMI output appears when the monitor is plugged in, without restarting WirePlumber, and plays. Audio over DP alt mode on USB-C was not tested. Signed-off-by: Christian Bartels --- drivers/gpu/drm/apple/audio.c | 46 ++++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index 5446038f70097..ac55f1f30f6ac 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -43,6 +43,7 @@ struct dcp_audio { struct snd_jack *jack; struct snd_pcm_substream *substream; unsigned int open_cookie; + bool open_unplugged; struct mutex data_lock; bool dcp_connected; /// dcp status keep for delayed initialization @@ -74,6 +75,32 @@ static const struct snd_pcm_hardware dcp_pcm_hw = { .periods_max = UINT_MAX, }; +/* + * Without a sink the DCP has no audio elements to derive constraints from. + * Offer a plain stereo format so that the PCM can still be opened and + * configured, as HDMI PCMs on other platforms can. Userspace (PipeWire, + * PulseAudio) probes the PCM once when the card appears and drops the + * output for good if that fails, so a monitor attached later would never + * show up. Starting the stream still fails until a sink is connected. + */ +static const struct snd_pcm_hardware dcp_pcm_hw_unplugged = { + .info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | + SNDRV_PCM_INFO_INTERLEAVED, + .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S24_LE | + SNDRV_PCM_FMTBIT_S32_LE, + .rates = SNDRV_PCM_RATE_32000 | SNDRV_PCM_RATE_44100 | + SNDRV_PCM_RATE_48000, + .rate_min = 32000, + .rate_max = 48000, + .channels_min = 2, + .channels_max = 2, + .buffer_bytes_max = SIZE_MAX, + .period_bytes_min = 4096, + .period_bytes_max = SIZE_MAX, + .periods_min = 2, + .periods_max = UINT_MAX, +}; + static int dcpaud_read_remote_info(struct dcp_audio *dcpaud) { int ret; @@ -256,13 +283,15 @@ static int dcp_pcm_open(struct snd_pcm_substream *substream) return ret; } - if (!dcpaud->connected) { - mutex_unlock(&dcpaud->data_lock); - return -ENXIO; - } + dcpaud->open_unplugged = !dcpaud->connected; dcpaud->open_cookie = dcpaud->connection_cookie; mutex_unlock(&dcpaud->data_lock); + if (dcpaud->open_unplugged) { + hw = dcp_pcm_hw_unplugged; + goto refine; + } + ret = dcpaud_read_remote_info(dcpaud); if (ret < 0) return ret; @@ -278,6 +307,7 @@ static int dcp_pcm_open(struct snd_pcm_substream *substream) SNDRV_PCM_HW_PARAM_FORMAT, SNDRV_PCM_HW_PARAM_CHANNELS, -1); hw = dcp_pcm_hw; +refine: hw.info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | SNDRV_PCM_INFO_INTERLEAVED; hw.periods_min = 2; @@ -299,6 +329,7 @@ static int dcp_pcm_close(struct snd_pcm_substream *substream) { struct dcp_audio *dcpaud = substream->pcm->private_data; dcpaud->selected_chmap.channels = 0; + dcpaud->open_unplugged = false; return snd_dmaengine_pcm_close(substream); } @@ -321,6 +352,9 @@ static int dcp_pcm_hw_params(struct snd_pcm_substream *substream, struct dma_chan *chan = snd_dmaengine_pcm_get_chan(substream); int ret; + if (dcpaud->open_unplugged) + return 0; + if (!dcpaud_connection_up(dcpaud)) return -ENXIO; @@ -362,6 +396,10 @@ static int dcp_pcm_prepare(struct snd_pcm_substream *substream) { struct dcp_audio *dcpaud = substream->pcm->private_data; + /* alsa-lib prepares right after hw_params; see dcp_pcm_hw_unplugged */ + if (dcpaud->open_unplugged) + return 0; + if (!dcpaud_connection_up(dcpaud)) return -ENXIO; From 3ed0c2bd27c458c379d50d9f0be92e7935f536ca Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 20:47:57 +1100 Subject: [PATCH 1133/2680] drm/apple: retain the guarded fbdev probe in the display composition Keep the probe and driver callback guarded by CONFIG_DRM_FBDEV_EMULATION when composing the older dual-stream branches with the newer framebuffer helper fix. Enabled and disabled configurations retain the intended DRM helper behavior. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 676703d1f935c..f961f87dc1b40 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -66,6 +66,7 @@ static int apple_drm_gem_dumb_create(struct drm_file *file_priv, return drm_gem_dma_dumb_create_internal(file_priv, drm, args); } +#ifdef CONFIG_DRM_FBDEV_EMULATION static int apple_drm_fbdev_probe(struct drm_fb_helper *helper, struct drm_fb_helper_surface_size *sizes) { @@ -86,6 +87,7 @@ static int apple_drm_fbdev_probe(struct drm_fb_helper *helper, return drm_fbdev_dma_driver_fbdev_probe(helper, sizes); } +#endif /* * A compositor or boot splash is closing the device. drm_release() has @@ -100,7 +102,11 @@ static void apple_drm_postclose(struct drm_device *drm, struct drm_file *file) static const struct drm_driver apple_drm_driver = { DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(apple_drm_gem_dumb_create), +#ifdef CONFIG_DRM_FBDEV_EMULATION .fbdev_probe = apple_drm_fbdev_probe, +#else + DRM_FBDEV_DMA_DRIVER_OPS, +#endif .postclose = apple_drm_postclose, .name = DRIVER_NAME, .desc = DRIVER_DESC, From ac94db9f50e700451be2000b4d5a864177c0eca1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 20:32:03 +1100 Subject: [PATCH 1134/2680] drm: apple: read H17P blend CRCs through the outbound queue Linux measurements show that the H17P blend CRC command takes no input and returns one little-endian 32-bit value. The existing completion path instead supplies a constant zero to the DRM CRC interface. Queue one read for each validated present completion while CRC capture is enabled. Put it before the next pending present so the sample refers to the completed frame, and send it only after the callback is acknowledged and the transport is idle. Use the existing transaction timeout and crash handling, reject malformed replies, and discard queued reads on shutdown. Disabling capture suppresses both unsent requests and returned samples. Older firmware and the H17G method profile retain their current behavior. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 2 +- drivers/gpu/drm/apple/iomfb.c | 69 ++++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 8 ++- 4 files changed, 78 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index ae52da4a2cd90..0e72bdcf97baf 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -449,6 +449,7 @@ int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum, void iomfb_queue_init(struct apple_dcp *dcp); void iomfb_queue_stop(struct apple_dcp *dcp); int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction); +void iomfb_queue_crc_h17p(struct apple_dcp *dcp, u32 swap_id); void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); void dcp_handle_hotplug_actions(struct apple_dcp *dcp, unsigned int action); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index ae81b59597743..f2c687eecaf60 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1829,7 +1829,7 @@ int dcp_set_crc(struct drm_crtc *crtc, bool enabled) struct apple_crtc *ac = to_apple_crtc(crtc); struct apple_dcp *dcp = platform_get_drvdata(ac->dcp); - dcp->crc_enabled = enabled; + WRITE_ONCE(dcp->crc_enabled, enabled); return 0; } diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 782fbe56df9d3..c5d6a4e6f7731 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -409,6 +410,74 @@ int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction) return ret; } +struct iomfb_crc_transaction { + struct iomfb_transaction transaction; + u32 swap_id; +}; + +static void iomfb_crc_complete(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct iomfb_crc_transaction *crc = cookie; + struct dcp_packet_header *header; + u32 value; + + /* A106 has no input and a single little-endian blend-output CRC. */ + if (!out) + goto invalid; + header = out - sizeof(*header); + if (header->in_len || header->out_len != sizeof(__le32)) + goto invalid; + value = get_unaligned_le32(out); + if (READ_ONCE(dcp->crc_enabled)) + drm_crtc_add_crc_entry(&dcp->crtc->base, true, crc->swap_id, &value); + return; + +invalid: + dev_err(dcp->dev, "invalid blend CRC response\n"); + WRITE_ONCE(dcp->crashed, true); +} + +static void iomfb_crc_start(struct apple_dcp *dcp, + struct iomfb_transaction *transaction) +{ + static const struct dcp_method_entry method = { + .tag = { 'A', '1', '0', '6' }, + .name = "read_blend_crc", + }; + struct iomfb_crc_transaction *crc = container_of(transaction, + struct iomfb_crc_transaction, + transaction); + + if (READ_ONCE(dcp->crc_enabled)) + dcp_push(dcp, false, &method, 0, sizeof(__le32), NULL, + iomfb_crc_complete, crc); +} + +static void iomfb_crc_release(struct iomfb_transaction *transaction) +{ + kfree(container_of(transaction, struct iomfb_crc_transaction, transaction)); +} + +void iomfb_queue_crc_h17p(struct apple_dcp *dcp, u32 swap_id) +{ + struct iomfb_crc_transaction *crc; + + lockdep_assert_held(&dcp->iomfb.lock); + if (!READ_ONCE(dcp->crc_enabled) || READ_ONCE(dcp->crashed) || + dcp->iomfb.stopped || dcp->iomfb.queued >= 32) + return; + crc = kzalloc_obj(*crc); + if (!crc) + return; + crc->swap_id = swap_id; + crc->transaction.start = iomfb_crc_start; + crc->transaction.release = iomfb_crc_release; + /* Read the completed frame before another queued present changes it. */ + list_add(&crc->transaction.link, &dcp->iomfb.pending); + dcp->iomfb.queued++; + /* The receiver advances the queue only after acknowledging completion. */ +} + struct iomfb_command { struct iomfb_transaction transaction; struct dcp_method_entry method; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 089a3b2ddebad..4b3bc389f78f4 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -266,7 +266,13 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, dcp_release_retained_framebuffers(dcp, resp->swap_id); dcp_drm_crtc_page_flip(dcp, now); } - if (dcp->crc_enabled) { + if (READ_ONCE(dcp->crc_enabled)) { +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + iomfb_queue_crc_h17p(dcp, resp->swap_id); + return; + } +#endif u32 crc32 = 0; drm_crtc_add_crc_entry(&dcp->crtc->base, true, resp->swap_id, &crc32); } From 6d2c33b3b5e0e6aecfcb82904af5eb9d89c5b9e4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 23:48:39 +1100 Subject: [PATCH 1135/2680] drm: apple: locate legacy command replies in the saved record DCP firmware answers an AP command by writing the reply in place into the command record, then sending an ack whose offset and length fields are zero. This holds for H17P and, as traced on t8112 and t6000 with 13.5 firmware, for older firmware too. The legacy path instead located the reply through the ack: it read a packet header at offset zero of the ack context's area. That only finds the right record for an unnested command, which dcp_push() places at offset zero of the command area. A command sent while a callback is being handled goes after the callback in the command area, but its ack arrives on the callback context, whose offset zero holds the callback record. Since commit c529eadaf40a ("drm: apple: retain displaced framebuffers until confirmed completion") the lengths in that header are compared with the command's, so the first command the boot sequence sends from a callback fails the check. The DCP is then marked crashed and the internal panel is left without modes. Use the record address saved by dcp_push() on all firmware, as H17P already does. On older firmware, keep validating the reply before it is dispatched by requiring the record header to still match the one that was sent. Ignore an ack on an idle context on all firmware instead of underflowing the call depth. Fixes: c529eadaf40a ("drm: apple: retain displaced framebuffers until confirmed completion") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 4 +-- drivers/gpu/drm/apple/iomfb.c | 51 +++++++++++++++------------- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 0e72bdcf97baf..d30a99330b541 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -120,8 +120,8 @@ struct dcp_channel { dcp_callback_t callbacks[DCP_MAX_CALL_DEPTH]; void *cookies[DCP_MAX_CALL_DEPTH]; void *output[DCP_MAX_CALL_DEPTH]; - u32 in_len[DCP_MAX_CALL_DEPTH]; - u32 out_len[DCP_MAX_CALL_DEPTH]; + /* Header of each pending AP command record, as dcp_push() wrote it */ + struct dcp_packet_header header[DCP_MAX_CALL_DEPTH]; u16 end[DCP_MAX_CALL_DEPTH]; /* Current depth of the call stack. Less than DCP_MAX_CALL_DEPTH */ diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index c5d6a4e6f7731..1cee1b1cd746e 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -603,8 +603,7 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca ch->callbacks[depth] = cb; ch->cookies[depth] = cookie; ch->output[depth] = out + sizeof(header) + in_len; - ch->in_len[depth] = in_len; - ch->out_len[depth] = out_len; + ch->header[depth] = header; ch->end[depth] = offset + ALIGN(data_len, DCP_PACKET_ALIGNMENT); dcp_send_message(dcp, IOMFB_ENDPOINT, @@ -817,12 +816,10 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, dcp_ack(dcp, context); } -static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, - void *data, u32 length) +static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context) { - struct dcp_packet_header *header = data; struct dcp_channel *ch = dcp_get_channel(dcp, context); - bool h17p = dcp->fw_compat == DCP_FIRMWARE_H17P; + const struct dcp_packet_header *record, *sent; void *cookie, *out; dcp_callback_t cb; @@ -831,7 +828,7 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, return; } - if (h17p && !ch->depth) { + if (!ch->depth) { dev_warn(dcp->dev, "ignoring ack on idle context %X\n", context); return; } @@ -840,27 +837,35 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, cb = ch->callbacks[ch->depth]; cookie = ch->cookies[ch->depth]; + out = ch->output[ch->depth]; + sent = &ch->header[ch->depth]; ch->callbacks[ch->depth] = NULL; ch->cookies[ch->depth] = NULL; + ch->output[ch->depth] = NULL; - if (h17p) { - /* - * H17P acks carry no payload and offset zero, including for - * nested calls, so the output cannot be located from the ack - * message. It is in the original AP command record, whose - * address dcp_push() saved. - */ - out = ch->output[ch->depth]; - ch->output[ch->depth] = NULL; - } else { - if (header->in_len != ch->in_len[ch->depth] || - header->out_len != ch->out_len[ch->depth]) { - dev_err(dcp->dev, "invalid command response lengths\n"); - dcp->crashed = true; + /* + * An ack completes the innermost pending command on its context. The + * firmware writes the reply in place into that AP command record, whose + * address dcp_push() saved; the ack itself carries neither offset nor + * length on any firmware. A command sent from within a callback is + * acked on the callback context, whose receive area holds the callback + * record rather than the reply, so the ack cannot be used to find it. + * + * Older firmware leaves the record header untouched. If it no longer + * matches what was sent, the record was overwritten and its output + * cannot be trusted. + */ + if (dcp->fw_compat != DCP_FIRMWARE_H17P) { + record = out - sent->in_len - sizeof(*record); + if (memcmp(record, sent, sizeof(*record))) { + dev_err(dcp->dev, + "corrupt %c%c%c%c command record on context %X\n", + sent->tag[3], sent->tag[2], sent->tag[1], + sent->tag[0], context); + WRITE_ONCE(dcp->crashed, true); return; } - out = data + sizeof(*header) + header->in_len; } if (cb) @@ -889,7 +894,7 @@ static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) data = dcp->shmem + channel_offset + offset; if (FIELD_GET(IOMFB_MSG_ACK, message)) - dcpep_handle_ack(dcp, ctx_id, data, length); + dcpep_handle_ack(dcp, ctx_id); else dcpep_handle_cb(dcp, ctx_id, data, length, offset); } From 18e084526dbbc6e2035acffb353ea7e5ad24c0da Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 01:51:05 +1100 Subject: [PATCH 1136/2680] drm: apple: read the main display reply as a boolean The firmware answers is_main_display by writing a single byte into the 32-bit reply slot. Since commit c529eadaf40a ("drm: apple: retain displaced framebuffers until confirmed completion") dcp_push() fills reply slots with 0xff so that an unwritten status cannot pass for success, and the three bytes the firmware does not write stay set. An integrated panel's DCP now reads back -255, and an external display DCP, which answers 0, reads back -256 and is treated as the main display. Its hotplug callbacks are then ignored and power-on takes the integrated panel path. Read only the byte the firmware writes. Fixes: c529eadaf40a ("drm: apple: retain displaced framebuffers until confirmed completion") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_template.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 4b3bc389f78f4..120e9ae4c5304 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -2314,10 +2314,15 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru static void res_is_main_display(struct apple_dcp *dcp, void *out, void *cookie) { struct apple_connector *connector; - int result = *(int *)out; + /* + * The reply is a one-byte boolean. The firmware leaves the rest of + * the 32-bit slot alone, and dcp_push() fills that with 0xff. + */ + bool result = *(u8 *)out; + dev_info(dcp->dev, "DCP is_main_display: %d\n", result); - dcp->main_display = result != 0; + dcp->main_display = result; connector = dcp->connector; if (connector) { From d52ada94924898fb26679a651ca5a76a88165855 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:04:57 +1100 Subject: [PATCH 1137/2680] drm: apple: accept H17P runtime callbacks within their reply bounds The H17P callback dispatcher only accepts callbacks whose record lengths appear in the startup observations. Every other callback that has a handler is treated as unqualified and marks the coprocessor as crashed, which stops all presents until the next reboot. Twenty-six handled callbacks are missing from those observations because they arrive only after startup: present notifications, swap aborts, the backlight re-send request, the wall-clock and clock-rate queries, and several acknowledgements. A long session that sees any of them loses the internal panel even though the driver knows how to answer. Record, for each of them, the input its handler reads and the reply it writes, and accept any record at least that large. The dispatcher already zeroes the whole reply before the handler runs, so a longer reply carries zeroes beyond the handler's fields. The measured startup tuples stay exact, and tags without a handler are still answered with a zeroed reply as before. Add a KUnit case that every installed handler has bounds and that the exact and bounded rules hold. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 57 ++++++++++++++++++++++++- drivers/gpu/drm/apple/iomfb_h17p.h | 4 ++ drivers/gpu/drm/apple/iomfb_h17p_test.c | 28 ++++++++++++ 3 files changed, 88 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index d02c26d04671a..423a86289f2a0 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -15,18 +15,32 @@ struct dcp_callback_size { u32 in_len; u32 out_len; bool valid; + /* Lengths bound what the handler reads and writes; no capture exists. */ + bool minimum; }; +/* + * Callbacks that only arrive after startup: present notifications, swap + * aborts, the backlight re-send request, wall-clock and clock-rate queries. + * Their record lengths are not in the startup observations, so accept any + * record that holds what the handler reads and the reply it writes. The + * dispatcher zeroes the rest of a longer reply. + */ +#define DCP_CB_AT_LEAST(in, out) \ + { .in_len = (in), .out_len = (out), .valid = true, .minimum = true } + /* R-IOMFB-startup-run21 and R-ANALYTICS Linux boundary observations. */ static const struct dcp_callback_size callback_sizes[IOMFB_MAX_CB] = { [0] = { 0x0, 0x4, true }, [1] = { 0x0, 0x4, true }, + [2] = DCP_CB_AT_LEAST(0x0, 0x0), [3] = { 0x4, 0x14, true }, [6] = { 0x54, 0x50, true }, [100] = { 0x0, 0x0, true }, [101] = { 0x0, 0x4, true }, [102] = { 0x44, 0x0, true }, [104] = { 0x44, 0x0, true }, + [107] = DCP_CB_AT_LEAST(0x0, 0x0), [108] = { 0x0, 0x4, true }, [109] = { 0x0, 0x4, true }, [110] = { 0x0, 0x4, true }, @@ -34,34 +48,58 @@ static const struct dcp_callback_size callback_sizes[IOMFB_MAX_CB] = { [112] = { 0x0, 0x4, true }, [113] = { 0x0, 0x4, true }, [114] = { 0x1044, 0x1004, true }, + [117] = DCP_CB_AT_LEAST(0x0, 0x0), + [118] = DCP_CB_AT_LEAST(0x0, 0x4), + [119] = DCP_CB_AT_LEAST(0x0, 0x4), + [120] = DCP_CB_AT_LEAST(0x0, 0x4), [121] = { 0x0, 0x4, true }, + [122] = DCP_CB_AT_LEAST(0x0, 0x1), [123] = { 0x0, 0x4, true }, + [124] = DCP_CB_AT_LEAST(0x0, 0x4), [125] = { 0x64, 0x24, true }, [127] = { 0x4, 0x4, true }, [128] = { 0x1008, 0x4, true }, [129] = { 0x40, 0x4, true }, + [130] = DCP_CB_AT_LEAST(0x22, 0x14), [201] = { 0xc, 0x10, true }, [202] = { 0x1a, 0x0, true }, [206] = { 0x0, 0x4, true }, [207] = { 0x0, 0x4, true }, + [208] = DCP_CB_AT_LEAST(0x0, 0x0), + [209] = DCP_CB_AT_LEAST(0x0, 0x8), [300] = { 0x10, 0x0, true }, [400] = { 0x4c, 0xc04, true }, [401] = { 0x50, 0xc, true }, + [404] = DCP_CB_AT_LEAST(0x0, 0x0), [406] = { 0x48, 0x0, true }, + [408] = DCP_CB_AT_LEAST(0x8, 0x8), [411] = { 0x10, 0x1c, true }, [413] = { 0x1048, 0x4, true }, [414] = { 0x50, 0x4, true }, [415] = { 0x4c, 0x4, true }, [451] = { 0x14, 0x1c, true }, + [452] = DCP_CB_AT_LEAST(0x18, 0x14), [454] = { 0x4, 0x1, true }, [552] = { 0x1044, 0x4, true }, [561] = { 0x1044, 0x4, true }, [563] = { 0x4c, 0x4, true }, [565] = { 0x48, 0x4, true }, + [567] = DCP_CB_AT_LEAST(0x0, 0x1), + [572] = DCP_CB_AT_LEAST(0x0, 0x4), [574] = { 0x4, 0x8, true }, [575] = { 0x58, 0x4c, true }, + [576] = DCP_CB_AT_LEAST(0x0, 0x0), [582] = { 0x8, 0x4, true }, + [583] = DCP_CB_AT_LEAST(0x0, 0x0), + [584] = DCP_CB_AT_LEAST(0x0, 0x0), + [585] = DCP_CB_AT_LEAST(0xe4, 0xe0), [590] = { 0x730, 0x0, true }, + [592] = DCP_CB_AT_LEAST(0x11, 0x0), + [593] = DCP_CB_AT_LEAST(0x4, 0x0), + [594] = DCP_CB_AT_LEAST(0x1, 0x0), + [595] = DCP_CB_AT_LEAST(0x0, 0x0), + [597] = DCP_CB_AT_LEAST(0x0, 0x1), + [598] = DCP_CB_AT_LEAST(0x0, 0x1), [599] = { 0x0, 0x0, true }, }; @@ -72,7 +110,11 @@ bool iomfb_validate_callback_h17p(int tag, u32 in_len, u32 out_len) if (tag < 0 || tag >= ARRAY_SIZE(callback_sizes)) return false; size = &callback_sizes[tag]; - return size->valid && size->in_len == in_len && size->out_len == out_len; + if (!size->valid) + return false; + if (size->minimum) + return in_len >= size->in_len && out_len >= size->out_len; + return size->in_len == in_len && size->out_len == out_len; } struct dcp_h17p_hotplug_request { @@ -660,5 +702,18 @@ void iomfb_start_h17p(struct apple_dcp *dcp) dcp_start_signal(dcp, false, dcp_started, NULL); } +#if IS_ENABLED(CONFIG_DRM_APPLE_KUNIT_TEST) +/* Returns the first handled tag without reply bounds, or -1. */ +int iomfb_h17p_first_unbounded_callback(void) +{ + int tag; + + for (tag = 0; tag < IOMFB_MAX_CB; tag++) + if (cb_handlers[tag] && !callback_sizes[tag].valid) + return tag; + return -1; +} +#endif + #undef DCP_FW_VER #undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 9473940250297..44e524abd368a 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -88,6 +88,10 @@ void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits, void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, const struct dcp_swap_submit_req_h17p *request); +#if IS_ENABLED(CONFIG_DRM_APPLE_KUNIT_TEST) +int iomfb_h17p_first_unbounded_callback(void); +#endif + #undef DCP_FW_VER #undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_h17p_test.c b/drivers/gpu/drm/apple/iomfb_h17p_test.c index c2a71ffa8cdbf..dcd26fb1a9ed3 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p_test.c +++ b/drivers/gpu/drm/apple/iomfb_h17p_test.c @@ -68,9 +68,37 @@ static void h17p_opaque_x_parameter_test(struct kunit *test) KUNIT_EXPECT_EQ(test, property.value, 0U); } +static void h17p_callback_bounds_test(struct kunit *test) +{ + /* Every installed handler is reachable without failing the session. */ + KUNIT_EXPECT_EQ(test, iomfb_h17p_first_unbounded_callback(), -1); + + /* Startup observations stay exact. */ + KUNIT_EXPECT_TRUE(test, iomfb_validate_callback_h17p(114, 0x1044, 0x1004)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(114, 0x1044, 0x1008)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(590, 0x734, 0x0)); + + /* Runtime callbacks must hold what their handlers read and write. */ + KUNIT_EXPECT_TRUE(test, iomfb_validate_callback_h17p(209, 0x0, 0x8)); + KUNIT_EXPECT_TRUE(test, iomfb_validate_callback_h17p(209, 0x4, 0xc)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(209, 0x0, 0x4)); + KUNIT_EXPECT_TRUE(test, iomfb_validate_callback_h17p(593, 0x4, 0x0)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(593, 0x2, 0x0)); + KUNIT_EXPECT_TRUE(test, iomfb_validate_callback_h17p(594, 0x4, 0x0)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(594, 0x0, 0x0)); + KUNIT_EXPECT_TRUE(test, iomfb_validate_callback_h17p(585, 0xe4, 0xe0)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(585, 0xe0, 0xe0)); + + /* Tags without a handler or out of range never validate. */ + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(5, 0x0, 0x0)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(-1, 0x0, 0x0)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(IOMFB_MAX_CB, 0x0, 0x0)); +} + static struct kunit_case h17p_present_cases[] = { KUNIT_CASE(h17p_surface_wire_test), KUNIT_CASE(h17p_opaque_x_parameter_test), + KUNIT_CASE(h17p_callback_bounds_test), {} }; From c2801d149463dad3e5635b953b62cbdb83cd2941 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:06:36 +1100 Subject: [PATCH 1138/2680] drm: apple: bound firmware register mapping requests The prepare callback maps a physical display register range for the coprocessor. Its size arrives from the firmware as a 64-bit value but is rounded into an int, so a large size wraps negative. The window check then compares a wrapped end address, and dma_map_resource() receives the sign-extended size. The descriptor ID is also taken from find_first_zero_bit() without checking for a full table, so a mapping requested while all 128 descriptors are in use writes past the array. A failed DMA mapping is only warned about and returned as the device address. Keep the size in 64 bits, refuse a zero size or a range that wraps, refuse when no descriptor is free, and refuse a failed DMA mapping before claiming a descriptor. Each refusal is the existing all-zero reply already used for addresses outside the display windows. A valid request is mapped and answered exactly as before. H17P accepts this callback at runtime from now on, which makes the bounds matter there too. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_template.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 120e9ae4c5304..0e1de7d1abe13 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -778,24 +778,37 @@ static bool is_disp_register(struct apple_dcp *dcp, u64 start, u64 end) static struct dcp_map_physical_resp dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) { - int size = ALIGN(req->size, 4096); + u64 size, end; dma_addr_t dva; u32 id; - if (!is_disp_register(dcp, req->paddr, req->paddr + size - 1)) { + /* Both values come from the coprocessor; reject wrapping ranges. */ + if (!req->size || check_add_overflow(req->size, 4095ULL, &size) || + check_add_overflow(req->paddr, round_down(size, 4096) - 1, &end) || + !is_disp_register(dcp, req->paddr, end)) { dev_err(dcp->dev, "refusing to map phys address %llx size %llx\n", req->paddr, req->size); return (struct dcp_map_physical_resp){}; } + size = round_down(size, 4096); id = find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); + if (id >= DCP_MAX_MAPPINGS) { + dev_warn(dcp->dev, "DCP overflowed mapping table, ignoring\n"); + return (struct dcp_map_physical_resp){}; + } + + dva = dma_map_resource(dcp->dev, req->paddr, size, DMA_BIDIRECTIONAL, 0); + if (dma_mapping_error(dcp->dev, dva)) { + dev_err(dcp->dev, "failed to map phys address %llx size %llx\n", + req->paddr, size); + return (struct dcp_map_physical_resp){}; + } + set_bit(id, dcp->memdesc_map); dcp->memdesc[id].size = size; dcp->memdesc[id].reg = req->paddr; - dva = dma_map_resource(dcp->dev, req->paddr, size, DMA_BIDIRECTIONAL, 0); - WARN_ON(dva == DMA_MAPPING_ERROR); - return (struct dcp_map_physical_resp){ .dva_size = size, .mem_desc_id = id, From 4de2e65e470d9ee76b411c5ac0b939a1de07f2ff Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:09:58 +1100 Subject: [PATCH 1139/2680] drm: apple: finish H17P presents that the firmware aborts H17P reports an aborted present through its abort callback and never sends a completion for it. The driver only traced the callback, so the present stayed pending: the outbound queue waited until its ten-second timeout declared the coprocessor crashed, and the page-flip event was never signalled. Track the abort in the present state. After the submit reply, an abort ends the present like a rejected submission: the brightness reservation is retried, framebuffers displaced by the present are unarmed and stay retained for the next completed present, and the DRM event is signalled. An abort that arrives before the submit reply is recorded, and the reply then fails the present instead of waiting for a completion. An abort naming a present that is not in flight is only logged. Older firmware still only traces the callback. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.h | 23 +++++++++++++++- drivers/gpu/drm/apple/iomfb_h17p_test.c | 33 +++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 36 +++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 44e524abd368a..94ac90be4247a 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -21,6 +21,8 @@ struct dcp_present_state_h17p { u32 swap_id; bool pending; bool accepted; + /* The firmware aborted the present before its submit reply. */ + bool aborted; }; static inline bool @@ -31,6 +33,7 @@ dcp_present_begin_h17p(struct dcp_present_state_h17p *state, u32 swap_id) state->swap_id = swap_id; state->pending = true; state->accepted = false; + state->aborted = false; return true; } @@ -41,11 +44,29 @@ dcp_present_submit_h17p(struct dcp_present_state_h17p *state, u32 swap_id, if (!state->pending || state->accepted || state->swap_id != swap_id) return false; state->accepted = accepted; - if (!accepted) + if (!accepted || state->aborted) state->pending = false; return true; } +/* + * An aborted present never completes. Before its submit reply, the abort is + * recorded and the reply ends the present instead. + */ +static inline bool +dcp_present_abort_h17p(struct dcp_present_state_h17p *state, u32 swap_id) +{ + if (!state->pending || state->swap_id != swap_id) + return false; + if (state->accepted) { + state->pending = false; + state->accepted = false; + } else { + state->aborted = true; + } + return true; +} + static inline bool dcp_present_complete_h17p(struct dcp_present_state_h17p *state, u32 swap_id) { diff --git a/drivers/gpu/drm/apple/iomfb_h17p_test.c b/drivers/gpu/drm/apple/iomfb_h17p_test.c index dcd26fb1a9ed3..70fecfe52c0ff 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p_test.c +++ b/drivers/gpu/drm/apple/iomfb_h17p_test.c @@ -95,10 +95,43 @@ static void h17p_callback_bounds_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(IOMFB_MAX_CB, 0x0, 0x0)); } +static void h17p_present_abort_test(struct kunit *test) +{ + struct dcp_present_state_h17p state = {}; + + /* An abort after acceptance ends the present without a completion. */ + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(&state, 7)); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 7, true)); + KUNIT_EXPECT_FALSE(test, dcp_present_abort_h17p(&state, 6)); + KUNIT_EXPECT_TRUE(test, state.pending); + KUNIT_EXPECT_TRUE(test, dcp_present_abort_h17p(&state, 7)); + KUNIT_EXPECT_FALSE(test, state.pending); + KUNIT_EXPECT_FALSE(test, dcp_present_complete_h17p(&state, 7)); + KUNIT_EXPECT_FALSE(test, dcp_present_abort_h17p(&state, 7)); + + /* An abort before the submit reply leaves the reply to finish it. */ + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(&state, 8)); + KUNIT_EXPECT_TRUE(test, dcp_present_abort_h17p(&state, 8)); + KUNIT_EXPECT_TRUE(test, state.pending); + KUNIT_EXPECT_TRUE(test, state.aborted); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 8, true)); + KUNIT_EXPECT_FALSE(test, state.pending); + KUNIT_EXPECT_FALSE(test, dcp_present_complete_h17p(&state, 8)); + + /* The next present starts clean and completes normally. */ + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(&state, 9)); + KUNIT_EXPECT_FALSE(test, state.aborted); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 9, true)); + KUNIT_EXPECT_TRUE(test, state.pending); + KUNIT_EXPECT_TRUE(test, dcp_present_complete_h17p(&state, 9)); + KUNIT_EXPECT_FALSE(test, state.pending); +} + static struct kunit_case h17p_present_cases[] = { KUNIT_CASE(h17p_surface_wire_test), KUNIT_CASE(h17p_opaque_x_parameter_test), KUNIT_CASE(h17p_callback_bounds_test), + KUNIT_CASE(h17p_present_abort_test), {} }; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 0e1de7d1abe13..31e89b15b5a8f 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1387,6 +1387,12 @@ static bool dcp_present_submit(struct apple_dcp *dcp, u32 swap_id, bool accepted dcp->crashed = true; return false; } + /* An abort received before this reply turns acceptance into failure. */ + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + accepted && dcp->present_state_h17p.aborted) { + dev_warn(dcp->dev, "firmware aborted present %u\n", swap_id); + return false; + } #endif return true; } @@ -1852,10 +1858,40 @@ dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, info->width, info->height); } +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER +/* + * An aborted H17P present never completes. Finish it like a rejected one: + * keep the displaced framebuffers for the next completed present and signal + * the DRM event, so neither the queue nor userspace waits for a completion. + */ +static void dcp_present_aborted(struct apple_dcp *dcp, u32 swap_id) +{ + if (!dcp_present_abort_h17p(&dcp->present_state_h17p, swap_id)) { + dev_warn(dcp->dev, "abort for present %u, which is not in flight\n", + swap_id); + return; + } + /* The submit reply has not arrived; it finishes the present. */ + if (dcp->present_state_h17p.pending) + return; + + dev_warn(dcp->dev, "firmware aborted present %u\n", swap_id); + dcp_present_failed(dcp); + if (dcp_present_retires(dcp)) { + dcp_unarm_retained_framebuffers(dcp, swap_id); + dcp_drm_crtc_vblank(dcp->crtc); + } +} +#endif + static void dcpep_cb_abort_swap_ap_gated(struct apple_dcp *dcp, u32 *swap_id) { trace_iomfb_abort_swap_ap_gated(dcp, *swap_id); +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + dcp_present_aborted(dcp, *swap_id); +#endif } static struct dcpep_get_tiling_state_resp From ea78fc4380cf0950b7fa20bfc15b90e26f1748af Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:11:50 +1100 Subject: [PATCH 1140/2680] drm: apple: re-send the H17P brightness when the firmware asks The enable-backlight-message callback asks the AP to send the panel level again. On H17P the level travels only inside a present, so the existing handler, which sets the older firmware's update flag, sends nothing: the brightness policy sees no pending change and the queued brightness transaction returns without presenting. On the measured H17P profile, mark the effective level dirty and let the update worker queue a brightness present, which re-presents the retained scanout as for any other level change. A blanked panel is re-sent at zero and keeps its restore target. Nothing is sent before Linux controls the level, while a present already carries it, or more than once a second, so a request repeated after every brightness present cannot keep the queue busy. The receive path only schedules the worker; it never queues a transaction itself. Older firmware keeps its existing handling. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 3 +++ drivers/gpu/drm/apple/dcp_backlight.c | 24 +++++++++++++++++ drivers/gpu/drm/apple/dcp_backlight.h | 14 ++++++++++ drivers/gpu/drm/apple/dcp_backlight_test.c | 31 ++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 9 +++++++ 5 files changed, 81 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index d30a99330b541..5b24f51856c6a 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -405,6 +405,9 @@ struct apple_dcp { spinlock_t lock; struct dcp_backlight_state state; void (*kick)(struct apple_dcp *dcp); + /* Rate limit for firmware re-send requests. */ + unsigned long resent_at; + bool resent; } backlight; /* Staging wire record; serialization never mutates the atomic inputs. */ diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 0fda6a3eb906d..8c2dfae24f51b 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -224,6 +224,30 @@ bool dcp_backlight_pending(struct apple_dcp *dcp) return pending; } +/* + * Called from the receive path, which must not queue a transaction itself; + * the caller schedules the update worker when this returns true. Requests + * are honoured at most once per second, so a firmware that repeats the + * request after every brightness present cannot keep the queue busy. + */ +bool dcp_backlight_resend(struct apple_dcp *dcp) +{ + unsigned long flags; + bool resend = false; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + if (!dcp->backlight.resent || + time_after(jiffies, dcp->backlight.resent_at + HZ)) { + resend = dcp_bl_resend(&dcp->backlight.state); + if (resend) { + dcp->backlight.resent = true; + dcp->backlight.resent_at = jiffies; + } + } + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return resend; +} + unsigned int dcp_backlight_retry_delay(struct apple_dcp *dcp) { unsigned long flags; diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index 2ad4abb4acd37..cb6dafd6bdf42 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -114,6 +114,19 @@ static inline int dcp_bl_dpms(struct dcp_backlight_state *state, bool on) return 0; } +/* + * The firmware asked for the level again. Re-send the effective level once + * Linux controls it, unless a present already carries or will carry it. + */ +static inline bool dcp_bl_resend(struct dcp_backlight_state *state) +{ + if (!state->ready || !state->controlled || state->dirty || + state->in_flight) + return false; + state->dirty = true; + return true; +} + /* * Called by the single outbound queue, with a pinned, accepted scanout or a * prepared replacement. Soft-off never releases it. A brightness-only present @@ -189,6 +202,7 @@ int dcp_backlight_prepare(struct apple_dcp *dcp, bool have_surface, struct dcp_backlight_present *present); bool dcp_backlight_complete(struct apple_dcp *dcp, u64 sequence, bool accepted); bool dcp_backlight_pending(struct apple_dcp *dcp); +bool dcp_backlight_resend(struct apple_dcp *dcp); unsigned int dcp_backlight_retry_delay(struct apple_dcp *dcp); #endif /* __APPLE_DCP_BACKLIGHT_H__ */ diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index a130f6f414712..441410bd5a84a 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -214,6 +214,36 @@ static void backlight_inflight_dpms_test(struct kunit *test) KUNIT_EXPECT_EQ(test, present.nits, 140U); } +static void backlight_resend_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + /* Nothing is re-sent before Linux controls the level. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, true, 140, false, 0), 0); + KUNIT_EXPECT_FALSE(test, dcp_bl_resend(&state)); + KUNIT_EXPECT_FALSE(test, state.dirty); + + state.controlled = true; + KUNIT_EXPECT_TRUE(test, dcp_bl_resend(&state)); + KUNIT_EXPECT_FALSE(test, dcp_bl_resend(&state)); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 140U); + /* The present in flight already carries the level. */ + KUNIT_EXPECT_FALSE(test, dcp_bl_resend(&state)); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_FALSE(test, state.dirty); + + /* A blanked panel is re-sent dark, keeping the restore target. */ + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_TRUE(test, dcp_bl_resend(&state)); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 0U); + KUNIT_EXPECT_EQ(test, state.target, 140U); +} + static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_millinits_takeover_test), KUNIT_CASE(backlight_takeover_test), @@ -224,6 +254,7 @@ static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_surface_and_busy_test), KUNIT_CASE(backlight_reject_test), KUNIT_CASE(backlight_inflight_dpms_test), + KUNIT_CASE(backlight_resend_test), {} }; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 31e89b15b5a8f..de7e8ee3f0539 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -874,6 +874,15 @@ dcpep_cb_read_edt_data(struct apple_dcp *dcp, struct dcp_read_edt_data_req *req) static void iomfbep_cb_enable_backlight_message_ap_gated(struct apple_dcp *dcp, u8 *enabled) { +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + /* H17P carries the level only in presents; ask for one more. */ + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + if (dcp_has_panel(dcp) && dcp_backlight_active(dcp) && + dcp_backlight_resend(dcp)) + schedule_work(&dcp->bl_update_wq); + return; + } +#endif /* * update backlight brightness on next swap, on non mini-LED displays * DCP seems to set an invalid iDAC value after coming out of DPMS. From 90fb4e4f7dc6714872e3d539a0047565244be12b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:12:30 +1100 Subject: [PATCH 1141/2680] drm: apple: expose the full H17P panel brightness range The H17P backlight is commanded in nits, and the firmware converts them with its own tables up to the panel ceiling from the device tree, 525 nits on J700. Takeover still clamps the exposed range to 509 nits, the end of the older firmware's DAC interpolation table, which H17P never uses. The top of the panel's range is therefore unreachable, and a loader level reported above 509 nits rejects the takeover hint, which leaves the backlight unregistered until a later report falls below it. Use the panel ceiling as the H17P maximum, and clamp a powerlog report above it to the ceiling instead of rejecting it. The present encoding already writes the same ceiling into every present. Older firmware keeps its existing range. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp_backlight.c | 3 ++- drivers/gpu/drm/apple/dcp_backlight.h | 10 ++++++---- drivers/gpu/drm/apple/dcp_backlight_test.c | 9 +++++++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 8c2dfae24f51b..492be0390a74d 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -96,7 +96,8 @@ int dcp_backlight_takeover(struct apple_dcp *dcp, u32 millinits) if (dcp_backlight_active(dcp)) return 0; - maximum = min(dcp->brightness.maximum, MAX_BRIGHTNESS_PART2 - 1); + /* H17P takes nits up to the panel ceiling, not the older DAC table's. */ + maximum = dcp->brightness.maximum; ret = dcp_bl_takeover_nits(maximum, millinits, &nits); if (ret) return ret; diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index cb6dafd6bdf42..75242111bd1de 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -6,6 +6,7 @@ #include #include +#include #include /* Commanded nits, independent of firmware encoding. No fade policy. */ @@ -30,14 +31,15 @@ struct dcp_backlight_present { u32 nits; }; -/* Public powerlog reports use millinits; validate before rounding to nits. */ +/* + * Public powerlog reports use millinits. A report above the panel ceiling + * describes the brightest level the panel can show, so clamp it. + */ static inline int dcp_bl_takeover_nits(u32 maximum, u32 millinits, u32 *nits) { if (!maximum || maximum > INT_MAX) return -EINVAL; - if ((u64)millinits > (u64)maximum * 1000) - return -ERANGE; - *nits = millinits / 1000; + *nits = min(millinits / 1000, maximum); return 0; } diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index 441410bd5a84a..12e039d5ce0a1 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -13,9 +13,14 @@ static void backlight_millinits_takeover_test(struct kunit *test) KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(0, 100000, &nits), -EINVAL); KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(U32_MAX, 100000, &nits), -EINVAL); - KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(509, 509001, &nits), -ERANGE); - KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(509, U32_MAX, &nits), -ERANGE); KUNIT_EXPECT_EQ(test, nits, 123U); + /* Reports above the panel ceiling clamp to it. */ + KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(525, 525999, &nits), 0); + KUNIT_EXPECT_EQ(test, nits, 525U); + KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 509001, &nits), 0); + KUNIT_EXPECT_EQ(test, nits, 509U); + KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, U32_MAX, &nits), 0); + KUNIT_EXPECT_EQ(test, nits, 509U); KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 100999, &nits), 0); KUNIT_EXPECT_EQ(test, nits, 100U); KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 509, true, nits, false, 0), 0); From 9d65ba0c207ef38be36eea61f9fb4c97cec9af85 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:14:54 +1100 Subject: [PATCH 1142/2680] drm: apple: register the H17P backlight without a loader level H17P never publishes the panel level. The driver registers its backlight only when the system endpoint forwards a powerlog report with the loader's level. That report is optional: when it does not arrive, or arrives in a form the parser does not accept, no backlight device is ever registered and desktops offer no brightness control. Two seconds after the first pipe enable, register the backlight anyway if no report has configured it. Expose the middle of the panel range as the current level, but present nothing: the policy starts clean, so the panel keeps the loader's level until userspace writes one. A report that arrives later is ignored, as are reports after a hint-based registration. The worker is cancelled with the other backlight work on unbind and shutdown. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/dcp.c | 38 ++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 5b24f51856c6a..2d768397ee994 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -331,6 +331,8 @@ struct apple_dcp { struct mutex bl_register_mutex; /* Workqueue for updating the brightness */ struct work_struct bl_update_wq; + /* Registers the H17P backlight when the loader reports no level. */ + struct delayed_work bl_fallback_wq; /* integrated panel if present */ struct dcp_panel panel; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index f2c687eecaf60..39c23eba2d0f8 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2667,6 +2667,8 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) } } +#define DCP_BL_FALLBACK_DELAY msecs_to_jiffies(2000) + static bool dcp_uses_soft_dpms(struct apple_dcp *dcp) { return (dcp_backlight_active(dcp) && dcp_has_panel(dcp)) || @@ -2692,6 +2694,10 @@ void dcp_poweron(struct platform_device *pdev) WRITE_ONCE(dcp->crashed, true); return; } + /* The powerlog takeover report normally precedes this. */ + if (dcp_has_panel(dcp)) + schedule_delayed_work(&dcp->bl_fallback_wq, + DCP_BL_FALLBACK_DELAY); } ret = dcp_backlight_dpms(dcp, true); if (ret) @@ -2842,6 +2848,34 @@ static void dcp_work_update_backlight(struct work_struct *work) dcp_backlight_update(dcp); } +/* + * H17P never publishes the panel level, and the loader's level arrives only + * as an optional powerlog report. Without one, register the backlight at + * the middle of the panel range. That value is only reported: no level is + * presented until userspace writes one, so the panel keeps the loader's + * level. + */ +static void dcp_work_backlight_fallback(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), + struct apple_dcp, bl_fallback_wq); + u32 nits = dcp->brightness.maximum / 2; + int ret; + + if (READ_ONCE(dcp->quiescing) || READ_ONCE(dcp->crashed) || + dcp_backlight_active(dcp)) + return; + + ret = iomfb_configure_backlight_h17p(dcp, dcp->brightness.maximum, + false, 0, true, nits); + if (!ret) + dev_info(dcp->dev, + "no loader brightness reported; backlight starts at %u nits without a panel change\n", + nits); + else if (ret != -EBUSY) + dev_warn(dcp->dev, "backlight registration failed: %d\n", ret); +} + static void dcp_release_piodma_iommu_dev(struct apple_dcp *dcp) { if (dcp->retain_dma) { @@ -3455,6 +3489,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) afk_quiesce(dcp->ibootep); afk_quiesce(dcp->systemep); afk_quiesce(dcp->dcpavservep); + cancel_delayed_work_sync(&dcp->bl_fallback_wq); if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); @@ -3509,6 +3544,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) iomfb_queue_stop(dcp); dcp_release_piodma_iommu_dev(dcp); + cancel_delayed_work_sync(&dcp->bl_fallback_wq); if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); @@ -3597,6 +3633,7 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp_placeholder_edid_work); INIT_DELAYED_WORK(&dcp->typec_fabric_retrain_wq, dcp_typec_retrain_work); + INIT_DELAYED_WORK(&dcp->bl_fallback_wq, dcp_work_backlight_fallback); /* Balanced by enable at successful component bind. */ disable_delayed_work(&dcp->typec_reconnect_wq); disable_delayed_work(&dcp->placeholder_edid_wq); @@ -3760,6 +3797,7 @@ static void dcp_platform_shutdown(struct platform_device *pdev) * its DMA resources alive until the system resets. */ iomfb_queue_stop(dcp); + cancel_delayed_work_sync(&dcp->bl_fallback_wq); cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); return; From 92c57b9f34999feb93d08e788af1060ad4dff5f6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:15:31 +1100 Subject: [PATCH 1143/2680] drm: apple: drain H17P display updates before system sleep H17P keeps its pipe and display domains running across system sleep, and blanks the panel with a brightness present instead. The DRM and backlight suspend handlers only queue that present: the serialized IOMFB queue sends it, and finishes it when the firmware reports the completion. The DCP suspend handler returned without waiting, so the blanking present could still be in flight when interrupts are suspended, leaving the panel lit through the sleep and the transaction stalled until resume. Before the DCP suspends, flush the brightness worker and wait up to half a second for the queue to go idle, twice, since a completed present can schedule one more brightness present. A crashed or stopped queue counts as idle. If the queue does not drain, log it and continue suspending. Older firmware does not use the queue and is unaffected. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 25 +++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb.c | 26 ++++++++++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 2d768397ee994..b77c8b24e8325 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -453,6 +453,7 @@ int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum, bool default_valid, u32 default_nits); void iomfb_queue_init(struct apple_dcp *dcp); void iomfb_queue_stop(struct apple_dcp *dcp); +bool iomfb_queue_drain(struct apple_dcp *dcp, unsigned long timeout); int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction); void iomfb_queue_crc_h17p(struct apple_dcp *dcp, u32 swap_id); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 39c23eba2d0f8..a54afa5594905 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3806,10 +3806,35 @@ static void dcp_platform_shutdown(struct platform_device *pdev) component_del(&pdev->dev, &dcp_comp_ops); } +/* + * The display stays powered across system sleep, and the blanking present + * queued by the DRM and backlight suspend handlers must reach the firmware + * before its mailbox interrupt is suspended. + */ +static void dcp_drain_for_sleep(struct apple_dcp *dcp) +{ + int pass; + + if (!dcp->hw.adopt_live_session || dcp->fw_compat != DCP_FIRMWARE_H17P || + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return; + + /* A completed present can schedule one more brightness present. */ + for (pass = 0; pass < 2; pass++) { + if (dcp_has_panel(dcp)) + flush_work(&dcp->bl_update_wq); + if (!iomfb_queue_drain(dcp, msecs_to_jiffies(500))) { + dev_warn(dcp->dev, "display updates still pending at suspend\n"); + return; + } + } +} + static int dcp_platform_suspend(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); + dcp_drain_for_sleep(dcp); /* * The Type-C route reports cable removal through * dcp_dptx_disconnect_oob(). A DP tunnel kept through the sleep stays diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 1cee1b1cd746e..eae3fa91566b4 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -388,6 +388,32 @@ void iomfb_queue_stop(struct apple_dcp *dcp) cancel_delayed_work_sync(&dcp->iomfb.backlight_retry); } +/* + * Wait until every queued transaction, including a present's completion, has + * finished. A stopped or crashed queue has nothing left to deliver. + */ +bool iomfb_queue_drain(struct apple_dcp *dcp, unsigned long timeout) +{ + unsigned long deadline = jiffies + timeout; + bool idle; + + if (!iomfb_uses_queue(dcp)) + return true; + + for (;;) { + mutex_lock(&dcp->iomfb.lock); + idle = READ_ONCE(dcp->crashed) || dcp->iomfb.stopped || + (!dcp->iomfb.active && list_empty(&dcp->iomfb.pending) && + !dcp->present_state_h17p.pending); + mutex_unlock(&dcp->iomfb.lock); + if (idle) + return true; + if (time_after(jiffies, deadline)) + return false; + usleep_range(2000, 4000); + } +} + int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction) { int ret = 0; From a19faf45b1b08399ed0738274b437e0c0b67e29e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:15:58 +1100 Subject: [PATCH 1144/2680] drm: apple: signal pending DRM events when H17P firmware crashes When the H17P firmware crashes, the driver marks the coprocessor failed and reports the connector disconnected. A present in flight at that moment never completes, so its page-flip event is only released by the DRM core's flip timeout, and the compositor stalls on it first. Schedule the existing delayed vblank work from the crash handler on H17P so the pending event is signalled. Framebuffers stay retained, since the display engine may still be reading the last surface. H17P has no in-place restart, so also log that a reboot is needed to restore the display. Older firmware keeps its existing crash handling. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a54afa5594905..8cdc510378a59 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1968,6 +1968,12 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ dcp->crashed = true; dev_err(dcp->dev, "DCP has crashed\n"); + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + /* No in-place restart exists; the last scanout stays retained. */ + dev_err(dcp->dev, "a reboot is required to restore the display\n"); + /* Nothing will complete a present that was in flight. */ + schedule_work(&dcp->vblank_wq); + } if (dcp->connector) { dcp->connector->connected = 0; drm_edid_free(dcp->connector->drm_edid); From 0012639bcb38108648af5f69f149736fb9908b2e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:16:49 +1100 Subject: [PATCH 1145/2680] drm: apple: keep the internal panel when the T8140 external DCP is enabled The display subsystem builds its component match from every available DCP node and binds only when all of them have bound. The T8140 external DCP runs firmware this driver cannot drive yet: its node falls back to the generic DCP match, which has no firmware profile for it. Its probe then either fails, leaving the component master waiting forever, or selects an older firmware interface from the loader's properties. Enabling that node in a device tree would therefore take the internal panel down with it. Leave T8140 external DCP nodes out of the component match and the pipeline scan, and refuse them in the DCP probe with an informational message, so they are never touched. Other SoCs are unaffected. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 15 +++++++++++++-- drivers/gpu/drm/apple/dcp.c | 6 ++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index f961f87dc1b40..cab19d90e201a 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -654,6 +654,17 @@ static const struct of_device_id apple_dcp_id_tbl[] = { {}, }; +/* + * The T8140 external display coprocessor needs firmware support this driver + * does not have yet. Leave it out of the display subsystem, so that an + * enabled node cannot hold back the internal panel's component master. + */ +static bool apple_dcp_usable(struct device_node *np) +{ + return of_device_is_available(np) && + !of_device_is_compatible(np, "apple,t8140-dcpext"); +} + static int apple_drm_init_dcp(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); @@ -666,7 +677,7 @@ static int apple_drm_init_dcp(struct device *dev) for_each_matching_node(np, apple_dcp_id_tbl) { bool dcp_ext; - if (!of_device_is_available(np)) { + if (!apple_dcp_usable(np)) { of_node_put(np); continue; } @@ -845,7 +856,7 @@ static int add_dcp_components(struct device *dev, int num = 0; for_each_matching_node(np, apple_dcp_id_tbl) { - if (of_device_is_available(np)) { + if (apple_dcp_usable(np)) { drm_of_component_match_add(dev, matchptr, component_compare_of, np); num++; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8cdc510378a59..b27ec8e28c851 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3582,6 +3582,12 @@ static int dcp_platform_probe(struct platform_device *pdev) if (dcp_session_retained(dev)) return dev_err_probe(dev, -EBUSY, "Previous live DCP session requires a reboot\n"); + /* Not part of the display subsystem yet; see apple_dcp_usable(). */ + if (of_device_is_compatible(dev->of_node, "apple,t8140-dcpext")) { + dev_info(dev, "external display coprocessor not supported yet\n"); + return -ENODEV; + } + fw_compat = dcp_check_firmware_version(dev); if (fw_compat == DCP_FIRMWARE_UNKNOWN) return -ENODEV; From c221f31f260d18d8a58bd54d78f1cb2d389b0bb2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:20:12 +1100 Subject: [PATCH 1146/2680] drm: apple: only signal crash events on a linked H17P pipeline The H17P crash handler schedules the delayed vblank work, so that a present in flight at the time of the crash completes its page-flip event. That work dereferences dcp->crtc, which is set only when the display subsystem links the pipeline. RTKit is already running before that: component binding wakes the coprocessor before the DRM device creates its CRTCs, and a pipeline whose CRTC setup failed stays bound with no CRTC at all. A firmware crash in either state then oopses the workqueue on a NULL CRTC. Schedule the work from the crash handler only once a CRTC is linked, and make the delayed vblank work itself return early without one, which also covers its other callers. Publish the CRTC with WRITE_ONCE() to match the lockless readers. Also limit the crash-time completion to the measured H17P profile, as every other H17P change in this series is. The H17G profile (T8142) keeps its existing crash handling. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index b27ec8e28c851..85f5c84725866 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1843,9 +1843,15 @@ static void dcp_delayed_vblank(struct work_struct *work) { struct apple_dcp *dcp; + struct apple_crtc *crtc; + dcp = container_of(work, struct apple_dcp, vblank_wq); + /* RTKit can report a crash before the pipeline has a CRTC. */ + crtc = READ_ONCE(dcp->crtc); + if (!crtc) + return; mdelay(5); - dcp_drm_crtc_vblank(dcp->crtc); + dcp_drm_crtc_vblank(crtc); } static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) @@ -1968,11 +1974,16 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ dcp->crashed = true; dev_err(dcp->dev, "DCP has crashed\n"); - if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { /* No in-place restart exists; the last scanout stays retained. */ dev_err(dcp->dev, "a reboot is required to restore the display\n"); - /* Nothing will complete a present that was in flight. */ - schedule_work(&dcp->vblank_wq); + /* + * Nothing will complete a present that was in flight. Before + * the pipeline is linked there is no event to complete. + */ + if (READ_ONCE(dcp->crtc)) + schedule_work(&dcp->vblank_wq); } if (dcp->connector) { dcp->connector->connected = 0; @@ -2465,7 +2476,7 @@ void dcp_link(struct platform_device *pdev, struct apple_crtc *crtc, { struct apple_dcp *dcp = platform_get_drvdata(pdev); - dcp->crtc = crtc; + WRITE_ONCE(dcp->crtc, crtc); /* * Type-C connectors belong to physical ports and are bound by the From f0a16cf8874e7a227222528c865dbaf5f7dc1b70 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:20:22 +1100 Subject: [PATCH 1147/2680] drm: apple: keep the H17P backlight policy when registration fails When the backlight class device cannot be registered, the worker zeroes the panel maximum so that older firmware stops encoding brightness. On H17P the brightness policy is already active by then, and the present path needs that maximum for the ceiling it writes into every present: with it zeroed, every present is refused and the panel freezes on its last frame. Registration is attempted on every H17P boot now, so this error path is always reachable. Keep the maximum while the H17P policy is active. Presents keep their ceiling and the policy keeps applying DPMS and the firmware's requests; only the sysfs control is missing. Log the error code in either case. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 85f5c84725866..5cd811ae4810c 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2848,8 +2848,14 @@ static void dcp_work_register_backlight(struct work_struct *work) if (ret == -ENODATA) goto out_unlock; if (ret) { - dev_err(dcp->dev, "Unable to register backlight device\n"); - dcp->brightness.maximum = 0; + dev_err(dcp->dev, "Unable to register backlight device: %d\n", ret); + /* + * The H17P policy encodes the panel ceiling into every present + * and keeps working without a class device; only older + * firmware stops sending brightness here. + */ + if (!dcp_backlight_active(dcp)) + dcp->brightness.maximum = 0; } out_unlock: From a366871a13d83fd716c67351155426ece359c727 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:21:34 +1100 Subject: [PATCH 1148/2680] drm: apple: never re-send a guessed H17P backlight level Without a loader brightness report, the H17P backlight is registered with a mid-range default that is only reported to userspace; the panel keeps the loader's level. Registration also marks the policy as controlled, which is the only condition the firmware's re-send request checked. A re-send request before userspace writes a level therefore presented the default and moved the panel from the loader's level to mid-range, contrary to the intent of the re-send and fallback registration changes. Track whether the level the panel shows is known: the loader's reported level, or one a completed present carried. Require it for a re-send, so a re-send request now leaves a panel showing an unknown level alone. A rejected present does not make a level known. A DPMS blank and wake, or a suspend and resume, after a fallback registration still has to light the panel at some level and uses the reported default. The panel keeps the loader's level only until the first such transition or userspace write. Also reject powerlog reports above twice the panel ceiling instead of clamping them, so a corrupt sample cannot become a known level. Add a KUnit case for the fallback registration, the re-send request, rejected and completed presents, and the DPMS wake level. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp_backlight.h | 29 ++++++++++---- drivers/gpu/drm/apple/dcp_backlight_test.c | 46 +++++++++++++++++++++- 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index 75242111bd1de..73683cb0322a3 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -24,6 +24,12 @@ struct dcp_backlight_state { bool suspended; bool dirty; bool in_flight; + /* + * The panel shows a level Linux knows: the loader's reported level, or + * one a completed present carried. A default used for registration + * without a loader report is not known until a present carries a level. + */ + bool level_known; }; struct dcp_backlight_present { @@ -32,18 +38,24 @@ struct dcp_backlight_present { }; /* - * Public powerlog reports use millinits. A report above the panel ceiling - * describes the brightest level the panel can show, so clamp it. + * Public powerlog reports use millinits. A report slightly above the panel + * ceiling describes the brightest level the panel can show, so clamp it; one + * more than twice the ceiling is not a credible panel level. */ static inline int dcp_bl_takeover_nits(u32 maximum, u32 millinits, u32 *nits) { - if (!maximum || maximum > INT_MAX) + if (!maximum || maximum > INT_MAX / 2000) return -EINVAL; + if (millinits / 1000 > 2 * maximum) + return -ERANGE; *nits = min(millinits / 1000, maximum); return 0; } -/* Inputs must come from admitted panel/loader records, never a guess. */ +/* + * An inherited level comes from a loader report. A default level is only + * reported to userspace; it is never presented on its own (see level_known). + */ static inline int dcp_bl_init(struct dcp_backlight_state *state, u32 maximum, bool inherited_valid, u32 inherited, bool default_valid, u32 default_nits) @@ -63,6 +75,7 @@ static inline int dcp_bl_init(struct dcp_backlight_state *state, u32 maximum, .target = nits, .actual = nits, .ready = true, + .level_known = inherited_valid, }; return 0; } @@ -118,12 +131,13 @@ static inline int dcp_bl_dpms(struct dcp_backlight_state *state, bool on) /* * The firmware asked for the level again. Re-send the effective level once - * Linux controls it, unless a present already carries or will carry it. + * Linux controls it and knows it, unless a present already carries or will + * carry it. A default level is never sent this way. */ static inline bool dcp_bl_resend(struct dcp_backlight_state *state) { - if (!state->ready || !state->controlled || state->dirty || - state->in_flight) + if (!state->ready || !state->controlled || !state->level_known || + state->dirty || state->in_flight) return false; state->dirty = true; return true; @@ -168,6 +182,7 @@ static inline bool dcp_bl_complete(struct dcp_backlight_state *state, if (accepted) { state->actual = state->sent_nits; state->retries = 0; + state->level_known = true; } else { state->dirty = true; if (state->retries < 4) diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index 12e039d5ce0a1..0b94b41b41482 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -19,8 +19,13 @@ static void backlight_millinits_takeover_test(struct kunit *test) KUNIT_EXPECT_EQ(test, nits, 525U); KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 509001, &nits), 0); KUNIT_EXPECT_EQ(test, nits, 509U); - KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, U32_MAX, &nits), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 1018999, &nits), 0); KUNIT_EXPECT_EQ(test, nits, 509U); + /* More than twice the ceiling is not a panel level. */ + nits = 123; + KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(509, 1019000, &nits), -ERANGE); + KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(509, U32_MAX, &nits), -ERANGE); + KUNIT_EXPECT_EQ(test, nits, 123U); KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 100999, &nits), 0); KUNIT_EXPECT_EQ(test, nits, 100U); KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 509, true, nits, false, 0), 0); @@ -249,6 +254,44 @@ static void backlight_resend_test(struct kunit *test) KUNIT_EXPECT_EQ(test, state.target, 140U); } +static void backlight_fallback_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + /* A registration default is reported, never presented on its own. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, false, 0, true, 262), 0); + KUNIT_EXPECT_FALSE(test, state.level_known); + state.controlled = true; + KUNIT_EXPECT_FALSE(test, dcp_bl_resend(&state)); + KUNIT_EXPECT_FALSE(test, state.dirty); + KUNIT_EXPECT_EQ(test, dcp_bl_prepare(&state, true, &present), -EALREADY); + + /* A rejected userspace level does not become known. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 300, false, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 300U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, false)); + KUNIT_EXPECT_FALSE(test, state.level_known); + + /* Once a present carrying it completes, the level is re-sendable. */ + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_TRUE(test, state.level_known); + KUNIT_EXPECT_TRUE(test, dcp_bl_resend(&state)); + + /* DPMS has to light the panel at some level: wake uses the default. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, false, 0, true, 262), 0); + state.controlled = true; + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 262U); +} + static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_millinits_takeover_test), KUNIT_CASE(backlight_takeover_test), @@ -260,6 +303,7 @@ static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_reject_test), KUNIT_CASE(backlight_inflight_dpms_test), KUNIT_CASE(backlight_resend_test), + KUNIT_CASE(backlight_fallback_test), {} }; From 99202d3f60e9d304dbeae1e1a24355fb6d6cd20d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:22:15 +1100 Subject: [PATCH 1149/2680] drm: apple: tolerate late H17P completions of aborted presents The H17P abort callback is handled as "this present never completes", but no capture confirms that. If the firmware still completes an aborted present, the completion matches no present in flight and the driver declares the coprocessor crashed, turning the first real abort into a dead display. Remember the most recently aborted present. A completion for it is logged and otherwise ignored: its displaced framebuffers were already unarmed and its DRM event signalled when it was aborted, and it does not disturb a later present in flight. Also settle the swap watchdog when an accepted present is aborted. Otherwise a Type-C output would report a missing completion and retrain one second later. Rate-limit the abort messages, so that a firmware that aborts every frame cannot flood the log. Extend the KUnit present-state case with stale completions. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.h | 13 +++++++++++++ drivers/gpu/drm/apple/iomfb_h17p_test.c | 9 +++++++++ drivers/gpu/drm/apple/iomfb_template.c | 24 ++++++++++++++++++++---- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 94ac90be4247a..75856d1921562 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -23,6 +23,9 @@ struct dcp_present_state_h17p { bool accepted; /* The firmware aborted the present before its submit reply. */ bool aborted; + /* The most recent aborted present, whose late completion is ignored. */ + bool has_last_aborted; + u32 last_aborted; }; static inline bool @@ -64,9 +67,19 @@ dcp_present_abort_h17p(struct dcp_present_state_h17p *state, u32 swap_id) } else { state->aborted = true; } + state->has_last_aborted = true; + state->last_aborted = swap_id; return true; } +/* Whether the firmware aborted @swap_id; a completion for it is stale. */ +static inline bool +dcp_present_was_aborted_h17p(const struct dcp_present_state_h17p *state, + u32 swap_id) +{ + return state->has_last_aborted && state->last_aborted == swap_id; +} + static inline bool dcp_present_complete_h17p(struct dcp_present_state_h17p *state, u32 swap_id) { diff --git a/drivers/gpu/drm/apple/iomfb_h17p_test.c b/drivers/gpu/drm/apple/iomfb_h17p_test.c index 70fecfe52c0ff..c017651945a4e 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p_test.c +++ b/drivers/gpu/drm/apple/iomfb_h17p_test.c @@ -104,9 +104,13 @@ static void h17p_present_abort_test(struct kunit *test) KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 7, true)); KUNIT_EXPECT_FALSE(test, dcp_present_abort_h17p(&state, 6)); KUNIT_EXPECT_TRUE(test, state.pending); + KUNIT_EXPECT_FALSE(test, dcp_present_was_aborted_h17p(&state, 7)); KUNIT_EXPECT_TRUE(test, dcp_present_abort_h17p(&state, 7)); KUNIT_EXPECT_FALSE(test, state.pending); KUNIT_EXPECT_FALSE(test, dcp_present_complete_h17p(&state, 7)); + /* A late completion for it is recognisable as stale, not unexpected. */ + KUNIT_EXPECT_TRUE(test, dcp_present_was_aborted_h17p(&state, 7)); + KUNIT_EXPECT_FALSE(test, dcp_present_was_aborted_h17p(&state, 6)); KUNIT_EXPECT_FALSE(test, dcp_present_abort_h17p(&state, 7)); /* An abort before the submit reply leaves the reply to finish it. */ @@ -123,8 +127,13 @@ static void h17p_present_abort_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, state.aborted); KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 9, true)); KUNIT_EXPECT_TRUE(test, state.pending); + /* A stale completion for 8 neither matches nor disturbs present 9. */ + KUNIT_EXPECT_FALSE(test, dcp_present_complete_h17p(&state, 8)); + KUNIT_EXPECT_TRUE(test, dcp_present_was_aborted_h17p(&state, 8)); + KUNIT_EXPECT_TRUE(test, state.pending); KUNIT_EXPECT_TRUE(test, dcp_present_complete_h17p(&state, 9)); KUNIT_EXPECT_FALSE(test, state.pending); + KUNIT_EXPECT_FALSE(test, dcp_present_was_aborted_h17p(&state, 9)); } static struct kunit_case h17p_present_cases[] = { diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index de7e8ee3f0539..ea993ebb5e01f 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -252,6 +252,18 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && !dcp_present_complete_h17p(&dcp->present_state_h17p, resp->swap_id)) { + /* + * Abort semantics are not captured on H17P. If an aborted + * present completes after all, its framebuffers were already + * unarmed and its event signalled; record it and carry on. + */ + if (dcp_present_was_aborted_h17p(&dcp->present_state_h17p, + resp->swap_id)) { + dev_warn_ratelimited(dcp->dev, + "completion for aborted present %u ignored\n", + resp->swap_id); + return; + } dev_err(dcp->dev, "unexpected present completion %u\n", resp->swap_id); dcp->crashed = true; return; @@ -1399,7 +1411,8 @@ static bool dcp_present_submit(struct apple_dcp *dcp, u32 swap_id, bool accepted /* An abort received before this reply turns acceptance into failure. */ if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && accepted && dcp->present_state_h17p.aborted) { - dev_warn(dcp->dev, "firmware aborted present %u\n", swap_id); + dev_warn_ratelimited(dcp->dev, "firmware aborted present %u\n", + swap_id); return false; } #endif @@ -1876,15 +1889,18 @@ dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, static void dcp_present_aborted(struct apple_dcp *dcp, u32 swap_id) { if (!dcp_present_abort_h17p(&dcp->present_state_h17p, swap_id)) { - dev_warn(dcp->dev, "abort for present %u, which is not in flight\n", - swap_id); + dev_warn_ratelimited(dcp->dev, + "abort for present %u, which is not in flight\n", + swap_id); return; } /* The submit reply has not arrived; it finishes the present. */ if (dcp->present_state_h17p.pending) return; - dev_warn(dcp->dev, "firmware aborted present %u\n", swap_id); + dev_warn_ratelimited(dcp->dev, "firmware aborted present %u\n", swap_id); + /* No completion follows, so the swap watchdog must not wait for one. */ + dcp_swap_watchdog_complete(dcp); dcp_present_failed(dcp); if (dcp_present_retires(dcp)) { dcp_unarm_retained_framebuffers(dcp, swap_id); From 5dba5f191fecc5b294322d5b7e164766e0e2586f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:22:39 +1100 Subject: [PATCH 1150/2680] drm: apple: settle H17P brightness retries before system sleep The sleep drain waited for the IOMFB queue to go idle, but a rejected blanking present leaves the queue idle with a delayed retry scheduled. The drain then returned, and the retry ran after the coprocessor's mailbox had suspended, leaving the panel lit through sleep. Run a pending brightness retry immediately in each drain pass, and keep draining while the brightness policy still has a level to send. Four passes cover the first attempt and the policy's three retries, each bounded as before; a level that is still pending after them is logged. Also queue the fallback backlight registration on the freezable system workqueue, so that it cannot register a class device in the middle of a suspend that starts within two seconds of the first pipe enable. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 5cd811ae4810c..8abd5fec0bbb9 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2713,8 +2713,9 @@ void dcp_poweron(struct platform_device *pdev) } /* The powerlog takeover report normally precedes this. */ if (dcp_has_panel(dcp)) - schedule_delayed_work(&dcp->bl_fallback_wq, - DCP_BL_FALLBACK_DELAY); + queue_delayed_work(system_freezable_wq, + &dcp->bl_fallback_wq, + DCP_BL_FALLBACK_DELAY); } ret = dcp_backlight_dpms(dcp, true); if (ret) @@ -3848,15 +3849,24 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) return; - /* A completed present can schedule one more brightness present. */ - for (pass = 0; pass < 2; pass++) { + /* + * A completed present can schedule one more brightness present, and a + * rejected one a delayed retry; run a pending retry now instead of + * letting it fire after the mailbox has suspended. Four passes cover + * the first attempt and the policy's three retries. + */ + for (pass = 0; pass < 4; pass++) { + flush_delayed_work(&dcp->iomfb.backlight_retry); if (dcp_has_panel(dcp)) flush_work(&dcp->bl_update_wq); if (!iomfb_queue_drain(dcp, msecs_to_jiffies(500))) { dev_warn(dcp->dev, "display updates still pending at suspend\n"); return; } + if (!dcp_backlight_active(dcp) || !dcp_backlight_pending(dcp)) + return; } + dev_warn(dcp->dev, "brightness change still pending at suspend\n"); } static int dcp_platform_suspend(struct device *dev) From 8085b3744c4158677f0e8bf456e9406d53127a6c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:23:14 +1100 Subject: [PATCH 1151/2680] drm: apple: report H17P runtime callback sizes that exceed their bounds The runtime callbacks accepted within reply bounds are validated against what their handlers read and write, because no capture of their record lengths exists. A longer record is accepted silently, so a layout that differs from the handler's assumption, for instance a status word at another offset, would go unnoticed. The first time each of these callbacks arrives with lengths other than its handler's, log the lengths seen next to the handler's. Each tag is reported at most once per boot, and the measured startup callbacks are unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/iomfb.c | 2 +- drivers/gpu/drm/apple/iomfb.h | 2 ++ drivers/gpu/drm/apple/iomfb_h17p.c | 22 ++++++++++++++++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index b77c8b24e8325..8b487104ae55e 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -416,6 +416,8 @@ struct apple_dcp { struct dcp_present_h17p present_h17p; struct dcp_present_state_h17p present_state_h17p; DECLARE_BITMAP(unknown_callbacks, IOMFB_MAX_CB); + /* Runtime callbacks whose record size has been reported once. */ + DECLARE_BITMAP(sized_callbacks, IOMFB_MAX_CB); /* Serializes H17P transmit preparation with RTKit receive callbacks. */ struct { diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index eae3fa91566b4..3f91f13841154 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -797,7 +797,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, (u64)sizeof(*hdr) + hdr->in_len + hdr->out_len != length || (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && handled && - !iomfb_validate_callback_h17p(tag, hdr->in_len, hdr->out_len)))) { + !iomfb_check_callback_h17p(dcp, tag, hdr->in_len, hdr->out_len)))) { dev_err(dev, "unqualified IOMFB callback %d (%u, %u)\n", tag, hdr->in_len, hdr->out_len); dcp->crashed = true; diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 1a33b1f474554..25c2e0254183f 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -210,6 +210,8 @@ struct dcp_method_entry { bool iomfb_validate_callback_h17p(int tag, u32 in_len, u32 out_len); struct apple_dcp; +bool iomfb_check_callback_h17p(struct apple_dcp *dcp, int tag, u32 in_len, + u32 out_len); typedef bool (*iomfb_cb_handler)(struct apple_dcp *, int, void *, void *); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 423a86289f2a0..7e6dcb263d6d8 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -695,6 +695,28 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [599] = trampoline_nop, /* find_swap_function_gated */ }; +/* + * Validate a callback record and, the first time each runtime callback whose + * length is only bounded arrives with a different length, report the length + * actually seen. Those reports are the first capture of these records. + */ +bool iomfb_check_callback_h17p(struct apple_dcp *dcp, int tag, u32 in_len, + u32 out_len) +{ + const struct dcp_callback_size *size; + + if (!iomfb_validate_callback_h17p(tag, in_len, out_len)) + return false; + size = &callback_sizes[tag]; + if (size->minimum && + (in_len != size->in_len || out_len != size->out_len) && + !test_and_set_bit(tag, dcp->sized_callbacks)) + dev_info(dcp->dev, + "callback D%03d record is %#x/%#x bytes; its handler uses %#x/%#x\n", + tag, in_len, out_len, size->in_len, size->out_len); + return true; +} + void iomfb_start_h17p(struct apple_dcp *dcp) { WRITE_ONCE(dcp->pipe_enabled_h17p, false); From 9ead2d2e0068eff21821dcb9f699a4909832ed02 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:07:06 +1100 Subject: [PATCH 1152/2680] drm: apple: skip the H17P sleep drain on a crashed or stopped DCP Before system sleep, the H17P drain runs up to four passes to settle a pending brightness change. A crashed or stopped DCP rejects every present, and the queue drain reports it idle at once, so a level that was dirty when it stopped stays pending: all four passes run and the suspend logs "brightness change still pending at suspend", although there is nothing left that could be sent. Return from the drain as soon as the DCP has crashed or its queue has stopped. The crash itself is already reported. Fixes: 6d4dc0a94dc9 ("drm: apple: settle H17P brightness retries before system sleep") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8abd5fec0bbb9..7b41d835e99a7 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3856,6 +3856,12 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) * the first attempt and the policy's three retries. */ for (pass = 0; pass < 4; pass++) { + /* + * A crashed or stopped DCP takes no more presents, so a level + * still pending there can never be sent and is not waited for. + */ + if (READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.stopped)) + return; flush_delayed_work(&dcp->iomfb.backlight_retry); if (dcp_has_panel(dcp)) flush_work(&dcp->bl_update_wq); From 6f3e3f34dc10f89aab89118c1ecb88766e91b5bd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:07:35 +1100 Subject: [PATCH 1153/2680] drm: apple: treat a seeded H17P backlight level as known The H17P backlight policy re-sends a level on the firmware's request only when Linux knows what the panel shows: the loader's reported level or one that a completed present carried. dcp_bl_seed() puts a decoded powerlog sample, which is the loader's level, into the state before Linux takes control, but it does not mark the level as known. A seed over a registration default therefore leaves a real level that cannot be re-sent, and the state disagrees with its own definition. The seeding helper has no caller in the driver yet, so nothing goes wrong today, but wiring it up later, for example to let a late powerlog report correct a registration default, would hit this. Mark a seeded level as known and cover both the accepted and the refused seed in KUnit. Fixes: 73dd5dedcc77 ("drm: apple: never re-send a guessed H17P backlight level") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp_backlight.h | 6 ++++- drivers/gpu/drm/apple/dcp_backlight_test.c | 26 ++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index 73683cb0322a3..2c9a0a7ab97be 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -80,13 +80,17 @@ static inline int dcp_bl_init(struct dcp_backlight_state *state, u32 maximum, return 0; } -/* A decoded powerlog sample can seed takeover only before Linux control. */ +/* + * A decoded powerlog sample can seed takeover only before Linux control. + * It reports the level the panel shows, so the level is then known. + */ static inline bool dcp_bl_seed(struct dcp_backlight_state *state, u32 nits) { if (!state->ready || state->controlled || nits > state->maximum) return false; state->target = nits; state->actual = nits; + state->level_known = true; return true; } diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index 0b94b41b41482..a0aaad3ddd232 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -292,6 +292,31 @@ static void backlight_fallback_test(struct kunit *test) KUNIT_EXPECT_EQ(test, present.nits, 262U); } +static void backlight_seed_known_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + /* A loader sample seeded over a registration default is known. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, false, 0, true, 262), 0); + KUNIT_EXPECT_FALSE(test, state.level_known); + KUNIT_EXPECT_TRUE(test, dcp_bl_seed(&state, 180)); + KUNIT_EXPECT_TRUE(test, state.level_known); + state.controlled = true; + KUNIT_EXPECT_TRUE(test, dcp_bl_resend(&state)); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 180U); + + /* A refused sample leaves the default unknown. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, false, 0, true, 262), 0); + KUNIT_EXPECT_FALSE(test, dcp_bl_seed(&state, 526)); + KUNIT_EXPECT_FALSE(test, state.level_known); + state.controlled = true; + KUNIT_EXPECT_FALSE(test, dcp_bl_seed(&state, 180)); + KUNIT_EXPECT_FALSE(test, state.level_known); + KUNIT_EXPECT_FALSE(test, dcp_bl_resend(&state)); +} + static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_millinits_takeover_test), KUNIT_CASE(backlight_takeover_test), @@ -304,6 +329,7 @@ static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_inflight_dpms_test), KUNIT_CASE(backlight_resend_test), KUNIT_CASE(backlight_fallback_test), + KUNIT_CASE(backlight_seed_known_test), {} }; From 00082b6678122bc7b84c4e5ef400e0d19b527efc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:08:20 +1100 Subject: [PATCH 1154/2680] drm: apple: stop dropping DCP node references inside the match loops for_each_matching_node() hands each node to the loop body with a reference held and drops that reference itself when it moves on to the next node. apple_drm_init_dcp() also calls of_node_put() on every node it visits, and add_dcp_components() does the same, so every probe of the display subsystem drops one reference more than it took on each DCP node, and binding it drops another. The nodes are static and their platform devices keep further references, so nothing has been freed, but the counts are wrong, and repeated probe and bind attempts walk them towards a "Bad of_node_put()" error with CONFIG_OF_DYNAMIC. Leave the references to the iterator. Fixes: f2bc92ff49a9 ("drm/apple: Add DCP display driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index cab19d90e201a..694dc92603695 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -677,15 +677,12 @@ static int apple_drm_init_dcp(struct device *dev) for_each_matching_node(np, apple_dcp_id_tbl) { bool dcp_ext; - if (!apple_dcp_usable(np)) { - of_node_put(np); + if (!apple_dcp_usable(np)) continue; - } dcp_ext = of_device_is_compatible(np, "apple,dcpext") || of_property_present(np, "phys"); dcp[num_dcp] = of_find_device_by_node(np); - of_node_put(np); if (!dcp[num_dcp]) continue; @@ -889,7 +886,6 @@ static int add_dcp_components(struct device *dev, of_node_put(port); } } - of_node_put(np); } return num; From f65ace2035cee4ba0407282b8dd2abb9585bc1a6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:08:39 +1100 Subject: [PATCH 1155/2680] drm: apple: do not log a missing EDID source for internal panels dcp_get_modes() used to ask for an EDID only once the DCP had announced its AV service. Since the EDID read pins the service itself, it is asked on every mode probe and returns -ENODEV when there is no service. Internal panels never have one, so every mode probe on a laptop now logs "copy_edid failed: -ENODEV" at info level. Treat -ENODEV on a DCP that drives an internal panel as the expected "no EDID" answer and log it at debug level. External outputs, and any other failure, are reported as before, and the EDID read itself is unchanged. Fixes: 5837706b10e3 ("drm/apple: pin DP service owners before recycling retired AFK slots") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 3f91f13841154..a5126d1973583 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -970,7 +970,14 @@ int dcp_get_modes(struct drm_connector *connector) const struct drm_edid *edid; edid = dcpavserv_copy_edid(dcp); if (IS_ERR_OR_NULL(edid)) { - dev_info(dcp->dev, "copy_edid failed: %pe\n", edid); + /* + * An internal panel has no AV service and so no EDID; + * do not report that on every mode probe. + */ + if (dcp_has_panel(dcp) && PTR_ERR(edid) == -ENODEV) + dev_dbg(dcp->dev, "no EDID source for the panel\n"); + else + dev_info(dcp->dev, "copy_edid failed: %pe\n", edid); } else { drm_edid_free(apple_connector->drm_edid); apple_connector->drm_edid = edid; From 2a3d4bdf747ed194378dee83a511454dbfe61130 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:07:58 +1100 Subject: [PATCH 1156/2680] drm: apple: check only the lengths of a legacy reply record Commit 2b8d51be4f0a ("drm: apple: locate legacy command replies in the saved record") validates a reply on firmware older than H17P by comparing the whole record header, tag included, with the one that was sent. The lengths are what the check before it compared; requiring the firmware to leave the tag alone as well is a new assumption, and it was only confirmed by traces of 13.5 firmware on t8112 and t6000. The same path serves the 12.3 and 13.3 firmware interfaces. If either rewrote the tag when writing its reply, every ack would fail the check, the DCP would be marked crashed during start-up, and the display would stay dark. Compare only the in and out lengths, which is enough to tell that the record was overwritten. Fixes: 2b8d51be4f0a ("drm: apple: locate legacy command replies in the saved record") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index a5126d1973583..f8e18ac90d5e2 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -878,13 +878,15 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context) * acked on the callback context, whose receive area holds the callback * record rather than the reply, so the ack cannot be used to find it. * - * Older firmware leaves the record header untouched. If it no longer - * matches what was sent, the record was overwritten and its output - * cannot be trusted. + * On older firmware the lengths in the record header must still be + * the ones that were sent; otherwise the record was overwritten and + * its output cannot be trusted. The tag is left out of the check: it + * has only been seen preserved on 13.5 firmware. */ if (dcp->fw_compat != DCP_FIRMWARE_H17P) { record = out - sent->in_len - sizeof(*record); - if (memcmp(record, sent, sizeof(*record))) { + if (record->in_len != sent->in_len || + record->out_len != sent->out_len) { dev_err(dcp->dev, "corrupt %c%c%c%c command record on context %X\n", sent->tag[3], sent->tag[2], sent->tag[1], From 6a19127dea26265432d889eac671739af701c86c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:08:33 +1100 Subject: [PATCH 1157/2680] drm: apple: complete a legacy command whose reply record is corrupt When the reply record of a command on firmware older than H17P fails validation, dcpep_handle_ack() marks the DCP crashed and returns without calling the command's callback. The callback owns a reference to its cookie: for power and modeset commands that is a wait cookie whose caller then waits out its full timeout (1 to 10 seconds), and the reference is never dropped, so each such command leaks its cookie. The start-up wait in dcp_wait_ready() likewise runs to its timeout when the is_main_display reply is the one rejected. Call the callback with no output instead. The callbacks used on this firmware either ignore their output or already treat a missing one as a failed command, except two that dereferenced it unconditionally: make dcp_swapped() report a failed swap, as it already does on H17P, and make res_is_main_display() end the start-up wait, which then returns -ENODEV for the crashed DCP. Fixes: c529eadaf40a ("drm: apple: retain displaced framebuffers until confirmed completion") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 7 ++++++- drivers/gpu/drm/apple/iomfb_template.c | 14 +++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index f8e18ac90d5e2..acea7991630b3 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -892,7 +892,12 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context) sent->tag[3], sent->tag[2], sent->tag[1], sent->tag[0], context); WRITE_ONCE(dcp->crashed, true); - return; + /* + * Still complete the command, without an output, so + * that its callback drops the cookie reference and + * wakes the waiter instead of leaving it to time out. + */ + out = NULL; } } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index ea993ebb5e01f..b456e5fe339bc 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -2016,11 +2016,7 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; u32 swap_id = DCP_FW_UNION(dcp->swap).swap.swap_id; -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER u32 status = resp ? resp->ret : ~0U; -#else - u32 status = resp->ret; -#endif if (!dcp_present_submit(dcp, swap_id, !status)) status = ~0U; @@ -2388,11 +2384,19 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru static void res_is_main_display(struct apple_dcp *dcp, void *out, void *cookie) { struct apple_connector *connector; + bool result; + + /* No reply: the DCP is marked crashed, so only end the start-up wait. */ + if (!out) { + complete(&dcp->start_done); + return; + } + /* * The reply is a one-byte boolean. The firmware leaves the rest of * the 32-bit slot alone, and dcp_push() fills that with 0xff. */ - bool result = *(u8 *)out; + result = *(u8 *)out; dev_info(dcp->dev, "DCP is_main_display: %d\n", result); From 1c703d467e86c875de2e06d28748d5f6d416a91c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:12:11 +1100 Subject: [PATCH 1158/2680] drm: apple: unmap a released DCP register range The DCP firmware asks for display register ranges to be mapped through map_physical and gives them back through release_mem_desc. The address mapped with dma_map_resource() is returned to the firmware but never stored, and releasing the descriptor only clears it, so every released range leaves its mapping in place: the IOVA space stays allocated and the translation onto the registers stays live. A firmware that maps and releases a range repeatedly grows the leak for as long as the system runs. H17P firmware has been able to reach this path since its map_physical callback was admitted. Record the mapped address and unmap the range when its descriptor is released. The reserved descriptor 0 has no range and is left alone. Fixes: f2bc92ff49a9 ("drm/apple: Add DCP display driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_template.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index b456e5fe339bc..e08211a196582 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -757,9 +757,13 @@ static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) dma_free_coherent(dcp->dev, size, memdesc->buf, memdesc->dva); memdesc->buf = NULL; memset(&memdesc->map, 0, sizeof(memdesc->map)); - } else { + } else if (memdesc->size) { + /* A register range mapped by map_physical */ + dma_unmap_resource(dcp->dev, memdesc->dva, memdesc->size, + DMA_BIDIRECTIONAL, 0); memdesc->reg = 0; } + memdesc->dva = 0; memdesc->size = 0; @@ -820,6 +824,7 @@ dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) set_bit(id, dcp->memdesc_map); dcp->memdesc[id].size = size; dcp->memdesc[id].reg = req->paddr; + dcp->memdesc[id].dva = dva; return (struct dcp_map_physical_resp){ .dva_size = size, From 87dd6158617e568ba02021b89d0d948e94732766 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:14:46 +1100 Subject: [PATCH 1159/2680] drm: apple: present a backlight write while the H17P level is unknown Without a loader brightness report the H17P backlight registers with the middle of the panel range as its reported level, while the panel keeps the loader's level until a present carries one. dcp_bl_request() only marks a new request dirty when it differs from the target, so a write of exactly the reported level, as systemd-backlight makes when it restores a level saved on an earlier boot without a report, presents nothing. The panel then stays at the loader's level, the reported level stays wrong, and the first DPMS or suspend cycle jumps to it. Also mark a request dirty while the level is not known, so the first write is always presented and makes the level known. Add a KUnit case. Fixes: 34cd8f3032b3 ("drm: apple: register the H17P backlight without a loader level") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp_backlight.h | 6 +++- drivers/gpu/drm/apple/dcp_backlight_test.c | 36 ++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index 2c9a0a7ab97be..1232ff40a84ba 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -110,8 +110,12 @@ static inline int dcp_bl_request(struct dcp_backlight_state *state, u32 nits, return -ERANGE; state->controlled = true; state->retries = 0; + /* + * Until a present carries a level, the panel does not show the target, + * so even a request for the reported default has to be presented. + */ if (state->target != nits || state->core_blank != core_blank || - state->suspended != suspended) { + state->suspended != suspended || !state->level_known) { state->target = nits; state->core_blank = core_blank; state->suspended = suspended; diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index a0aaad3ddd232..41ac118f26cee 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -292,6 +292,41 @@ static void backlight_fallback_test(struct kunit *test) KUNIT_EXPECT_EQ(test, present.nits, 262U); } +static void backlight_fallback_same_level_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + /* A userspace write of the reported default is still presented. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, false, 0, true, 262), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 262, false, false), 0); + KUNIT_EXPECT_TRUE(test, state.dirty); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 262U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_TRUE(test, state.level_known); + + /* Once the level is known, repeating it presents nothing. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 262, false, false), 0); + KUNIT_EXPECT_FALSE(test, state.dirty); + KUNIT_EXPECT_EQ(test, dcp_bl_prepare(&state, true, &present), -EALREADY); + + /* A rejected present leaves it unknown, so the same write is retried. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, false, 0, true, 262), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 262, false, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, false)); + KUNIT_EXPECT_FALSE(test, state.level_known); + state.dirty = false; + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 262, false, false), 0); + KUNIT_EXPECT_TRUE(test, state.dirty); + + /* An inherited level is known from the start. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, true, 180, false, 0), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 180, false, false), 0); + KUNIT_EXPECT_FALSE(test, state.dirty); +} + static void backlight_seed_known_test(struct kunit *test) { struct dcp_backlight_state state; @@ -329,6 +364,7 @@ static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_inflight_dpms_test), KUNIT_CASE(backlight_resend_test), KUNIT_CASE(backlight_fallback_test), + KUNIT_CASE(backlight_fallback_same_level_test), KUNIT_CASE(backlight_seed_known_test), {} }; From 53fa12a88968fbc126023c9fb59d5978548cd550 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:18:29 +1100 Subject: [PATCH 1160/2680] drm: apple: do not report an aborted H17P present as a failed swap When H17P firmware aborts a present before its submit reply arrives, dcp_present_submit() logs the abort, rate-limited, and the reply handler then turns the accepted reply into a failure and logs it again, without a rate limit, as "swap failed! status 4294967295". A firmware that aborted every frame would log that error once per frame, with a status the firmware never returned. Let dcp_present_submit() return why a present does not stand, and skip the failure message for an abort that has already been reported. The present is still finished as a failure. Fixes: 239607cb12c3 ("drm: apple: finish H17P presents that the firmware aborts") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_template.c | 29 +++++++++++++++++++------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index e08211a196582..45c65732aed03 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1404,24 +1404,29 @@ static bool dcp_present_begin(struct apple_dcp *dcp, u32 swap_id) return true; } -static bool dcp_present_submit(struct apple_dcp *dcp, u32 swap_id, bool accepted) +/* + * Returns 0 if the submitted present stands, -ECANCELED if the firmware + * aborted it before replying (already reported), or -EPROTO if no present + * was expecting this reply. + */ +static int dcp_present_submit(struct apple_dcp *dcp, u32 swap_id, bool accepted) { #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && !dcp_present_submit_h17p(&dcp->present_state_h17p, swap_id, accepted)) { dev_err(dcp->dev, "unexpected present submission %u\n", swap_id); dcp->crashed = true; - return false; + return -EPROTO; } /* An abort received before this reply turns acceptance into failure. */ if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && accepted && dcp->present_state_h17p.aborted) { dev_warn_ratelimited(dcp->dev, "firmware aborted present %u\n", swap_id); - return false; + return -ECANCELED; } #endif - return true; + return 0; } static void dcp_prepare_clear_swap(struct apple_dcp *dcp, void *request) @@ -1471,12 +1476,16 @@ static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) struct dcp_swap_cookie *info = cookie; u32 swap_id = DCP_FW_UNION(dcp->swap).swap.swap_id; u32 status = resp ? resp->ret : ~0U; + int ret; - if (!dcp_present_submit(dcp, swap_id, !status)) + ret = dcp_present_submit(dcp, swap_id, !status); + if (ret) status = ~0U; if (status) { - dev_err(dcp->dev, "swap_clear failed! status %u\n", status); + /* An aborted present has already been reported. */ + if (ret != -ECANCELED) + dev_err(dcp->dev, "swap_clear failed! status %u\n", status); dcp_present_failed(dcp); if (dcp_present_retires(dcp)) { dcp_unarm_retained_framebuffers(dcp, swap_id); @@ -2019,15 +2028,19 @@ TRAMPOLINE_OUT(trampoline_create_backlight_service, dcpep_cb_create_backlight_se static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) { + int ret; struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; u32 swap_id = DCP_FW_UNION(dcp->swap).swap.swap_id; u32 status = resp ? resp->ret : ~0U; - if (!dcp_present_submit(dcp, swap_id, !status)) + ret = dcp_present_submit(dcp, swap_id, !status); + if (ret) status = ~0U; if (status) { - dev_err(dcp->dev, "swap failed! status %u\n", status); + /* An aborted present has already been reported. */ + if (ret != -ECANCELED) + dev_err(dcp->dev, "swap failed! status %u\n", status); dcp_present_failed(dcp); if (dcp_present_retires(dcp)) { dcp_unarm_retained_framebuffers(dcp, swap_id); From df62955f9200730d02ebae2c51bea18fc769ac7a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:18:52 +1100 Subject: [PATCH 1161/2680] drm: apple: check for a dead DCP after each H17P sleep drain pass Commit 744c2c7abc15 ("drm: apple: skip the H17P sleep drain on a crashed or stopped DCP") checks for a crashed or stopped DCP at the start of each drain pass. A crash during the last pass is not seen: the queue drain then reports the DCP idle at once, the level is still pending, and the loop ends with the "brightness change still pending at suspend" warning that commit set out to avoid. Returning before the first pass also skips the queue drain, whose lock orders the suspend after a queued transaction that is still being started. Check after the queue drain instead, before the pending test. A crashed or stopped DCP now always goes through the drain, which returns at once for it. Fixes: 744c2c7abc15 ("drm: apple: skip the H17P sleep drain on a crashed or stopped DCP") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 7b41d835e99a7..dc0dac9ff4c56 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3856,12 +3856,6 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) * the first attempt and the policy's three retries. */ for (pass = 0; pass < 4; pass++) { - /* - * A crashed or stopped DCP takes no more presents, so a level - * still pending there can never be sent and is not waited for. - */ - if (READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.stopped)) - return; flush_delayed_work(&dcp->iomfb.backlight_retry); if (dcp_has_panel(dcp)) flush_work(&dcp->bl_update_wq); @@ -3869,6 +3863,14 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) dev_warn(dcp->dev, "display updates still pending at suspend\n"); return; } + /* + * A crashed or stopped DCP takes no more presents, so a level + * still pending there can never be sent and is not waited for. + * Checking after the drain, which returns at once for such a + * DCP, also covers a crash during the pass. + */ + if (READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.stopped)) + return; if (!dcp_backlight_active(dcp) || !dcp_backlight_pending(dcp)) return; } From 38fa35a9f3aafa0c8e794da1f99878942526f03b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:20:16 +1100 Subject: [PATCH 1162/2680] drm: apple: fade between H17P backlight levels The H17P backlight presents every new level in one step, so each press of a brightness key and every change from an ambient light policy shows as a visible jump on the panel. Fade between levels instead. The policy now keeps the level it presents separate from the requested target: a new target for a panel that stays lit at a known level starts a linear fade from the level presented last, and a delayed work item moves it every 16 ms for 250 ms, the last step being the target itself. Each step reuses the existing brightness-only present, so steps that come faster than presents complete merge into the next one, and a step does not bypass the backoff of a rejected present. A new level during a fade restarts it from where it is. Blanking, unblanking, suspend and a level the panel may not show yet still jump as before, and a fade still running at system sleep ends at its target before the queue is drained. The KUnit cases cover the fade steps, restarts, merging, overflow and the cases that still jump. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Kconfig | 3 +- drivers/gpu/drm/apple/dcp-internal.h | 6 + drivers/gpu/drm/apple/dcp.c | 12 +- drivers/gpu/drm/apple/dcp_backlight.c | 94 ++++++++++++- drivers/gpu/drm/apple/dcp_backlight.h | 76 ++++++++++- drivers/gpu/drm/apple/dcp_backlight_test.c | 152 +++++++++++++++++++++ 6 files changed, 330 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 4416c3e6bd729..d74b2dfdb0f88 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -35,7 +35,8 @@ config DRM_APPLE_KUNIT_TEST default KUNIT_ALL_TESTS help Build tests for the Apple display driver's AFK ring parser and - backlight policy (takeover, DPMS, suspend and present completion). + backlight policy (takeover, DPMS, suspend, fades and present + completion). The tests cover the header strides used by older firmware and larger rings, as well as malformed ring size combinations. This option is intended for developer test kernels. diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 8b487104ae55e..b5ffa71339522 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -407,6 +407,9 @@ struct apple_dcp { spinlock_t lock; struct dcp_backlight_state state; void (*kick)(struct apple_dcp *dcp); + /* Steps a running fade; fade_start is its start in jiffies. */ + struct delayed_work fade_work; + unsigned long fade_start; /* Rate limit for firmware re-send requests. */ unsigned long resent_at; bool resent; @@ -462,6 +465,9 @@ void iomfb_queue_crc_h17p(struct apple_dcp *dcp, u32 swap_id); void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); void dcp_handle_hotplug_actions(struct apple_dcp *dcp, unsigned int action); +int dcp_backlight_init(struct apple_dcp *dcp, struct device *dev); +void dcp_backlight_stop(struct apple_dcp *dcp); +void dcp_backlight_finish_fade(struct apple_dcp *dcp); int dcp_backlight_register(struct apple_dcp *dcp); int dcp_backlight_update(struct apple_dcp *dcp); bool dcp_has_panel(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index dc0dac9ff4c56..790fbbc1f9ae2 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3514,6 +3514,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) afk_quiesce(dcp->systemep); afk_quiesce(dcp->dcpavservep); cancel_delayed_work_sync(&dcp->bl_fallback_wq); + dcp_backlight_stop(dcp); if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); @@ -3569,6 +3570,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) dcp_release_piodma_iommu_dev(dcp); cancel_delayed_work_sync(&dcp->bl_fallback_wq); + dcp_backlight_stop(dcp); if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); @@ -3628,7 +3630,9 @@ static int dcp_platform_probe(struct platform_device *pdev) INIT_LIST_HEAD(&dcp->swapped_out_fbs); mutex_init(&dcp->swapped_out_fbs_lock); - spin_lock_init(&dcp->backlight.lock); + ret = dcp_backlight_init(dcp, dev); + if (ret) + return ret; iomfb_queue_init(dcp); dcp->fw_compat = fw_compat; @@ -3828,6 +3832,7 @@ static void dcp_platform_shutdown(struct platform_device *pdev) */ iomfb_queue_stop(dcp); cancel_delayed_work_sync(&dcp->bl_fallback_wq); + dcp_backlight_stop(dcp); cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); return; @@ -3853,8 +3858,11 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) * A completed present can schedule one more brightness present, and a * rejected one a delayed retry; run a pending retry now instead of * letting it fire after the mailbox has suspended. Four passes cover - * the first attempt and the policy's three retries. + * the first attempt and the policy's three retries. A fade still + * running ends at its target first. */ + if (dcp_has_panel(dcp)) + dcp_backlight_finish_fade(dcp); for (pass = 0; pass < 4; pass++) { flush_delayed_work(&dcp->iomfb.backlight_retry); if (dcp_has_panel(dcp)) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 492be0390a74d..1552bb7ca9384 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -82,6 +82,14 @@ static u32 brightness_part2[] = { }; +/* + * A new level fades in over DCP_BL_FADE_MS, one brightness-only present per + * step at most. Steps that come faster than presents complete merge, so the + * step only bounds how smooth the fade can be. + */ +#define DCP_BL_FADE_MS 250U +#define DCP_BL_FADE_STEP_MS 16U + bool dcp_backlight_active(struct apple_dcp *dcp) { return dcp->fw_compat == DCP_FIRMWARE_H17P && @@ -145,8 +153,10 @@ int dcp_backlight_configure(struct apple_dcp *dcp, u32 maximum, ret = dcp_bl_init(&dcp->backlight.state, maximum, inherited_valid, inherited, default_valid, default_nits); - if (!ret) + if (!ret) { + dcp->backlight.state.fade = true; dcp->backlight.kick = kick; + } } spin_unlock_irqrestore(&dcp->backlight.lock, flags); mutex_unlock(&dcp->bl_register_mutex); @@ -174,6 +184,71 @@ static void dcp_backlight_kick(struct apple_dcp *dcp) kick(dcp); } +static void dcp_backlight_fade_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), + struct apple_dcp, + backlight.fade_work); + unsigned long flags; + bool running, kick; + u32 elapsed; + + if (READ_ONCE(dcp->quiescing) || READ_ONCE(dcp->crashed) || + READ_ONCE(dcp->iomfb.stopped)) + return; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + elapsed = jiffies_to_msecs(jiffies - dcp->backlight.fade_start); + running = dcp_bl_fade_step(&dcp->backlight.state, elapsed, + DCP_BL_FADE_MS); + /* A rejected present is retried with backoff, carrying the new level. */ + kick = dcp->backlight.state.dirty && !dcp->backlight.state.retries; + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + + if (kick) + dcp_backlight_kick(dcp); + if (running) + queue_delayed_work(system_wq, &dcp->backlight.fade_work, + msecs_to_jiffies(DCP_BL_FADE_STEP_MS)); +} + +void dcp_backlight_stop(struct apple_dcp *dcp) +{ + cancel_delayed_work_sync(&dcp->backlight.fade_work); +} + +static void dcp_backlight_release(void *data) +{ + dcp_backlight_stop(data); +} + +/* + * The backlight device is a devres of @dev, registered after probe, so it is + * unregistered before this action runs, and no request can start a fade + * afterwards. + */ +int dcp_backlight_init(struct apple_dcp *dcp, struct device *dev) +{ + spin_lock_init(&dcp->backlight.lock); + INIT_DELAYED_WORK(&dcp->backlight.fade_work, dcp_backlight_fade_work); + return devm_add_action_or_reset(dev, dcp_backlight_release, dcp); +} + +/* Present the target of a running fade at once, e.g. before system sleep. */ +void dcp_backlight_finish_fade(struct apple_dcp *dcp) +{ + unsigned long flags; + bool kick; + + cancel_delayed_work_sync(&dcp->backlight.fade_work); + spin_lock_irqsave(&dcp->backlight.lock, flags); + dcp_bl_fade_step(&dcp->backlight.state, DCP_BL_FADE_MS, DCP_BL_FADE_MS); + kick = dcp->backlight.state.dirty; + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + if (kick) + dcp_backlight_kick(dcp); +} + int dcp_backlight_dpms(struct apple_dcp *dcp, bool on) { unsigned long flags; @@ -397,14 +472,25 @@ static int dcp_set_brightness(struct backlight_device *bd) return -ENODEV; if (dcp_backlight_active(dcp)) { + struct dcp_backlight_state *state = &dcp->backlight.state; + bool fade; + u32 old; + /* Preserve the requested level even while the core forces zero. */ spin_lock_irqsave(&dcp->backlight.lock, flags); - ret = dcp_bl_request(&dcp->backlight.state, - bd->props.brightness, + old = state->target; + ret = dcp_bl_request(state, bd->props.brightness, backlight_is_blank(bd), bd->props.state & BL_CORE_SUSPENDED); + /* A new level restarts the fade from the level presented last. */ + fade = !ret && state->fading && state->target != old; + if (fade) + dcp->backlight.fade_start = jiffies; spin_unlock_irqrestore(&dcp->backlight.lock, flags); - if (!ret) + if (fade) + queue_delayed_work(system_wq, &dcp->backlight.fade_work, + msecs_to_jiffies(DCP_BL_FADE_STEP_MS)); + else if (!ret) dcp_backlight_kick(dcp); return ret; } diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index 1232ff40a84ba..92ac07b2e9d9d 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -6,13 +6,20 @@ #include #include +#include #include #include -/* Commanded nits, independent of firmware encoding. No fade policy. */ +/* + * Commanded nits, independent of firmware encoding. target is the requested + * level and level the one presented while the panel is lit. They differ only + * while a fade runs; the caller supplies the fade's clock. + */ struct dcp_backlight_state { u32 maximum; u32 target; + u32 level; + u32 fade_from; u32 actual; u32 sent_nits; u64 sequence; @@ -24,6 +31,10 @@ struct dcp_backlight_state { bool suspended; bool dirty; bool in_flight; + /* Fade between lit levels instead of jumping to them. */ + bool fade; + /* A fade from fade_from to target is running. */ + bool fading; /* * The panel shows a level Linux knows: the loader's reported level, or * one a completed present carried. A default used for registration @@ -73,6 +84,7 @@ static inline int dcp_bl_init(struct dcp_backlight_state *state, u32 maximum, *state = (struct dcp_backlight_state) { .maximum = maximum, .target = nits, + .level = nits, .actual = nits, .ready = true, .level_known = inherited_valid, @@ -89,21 +101,40 @@ static inline bool dcp_bl_seed(struct dcp_backlight_state *state, u32 nits) if (!state->ready || state->controlled || nits > state->maximum) return false; state->target = nits; + state->level = nits; state->actual = nits; state->level_known = true; return true; } +static inline bool dcp_bl_lit(const struct dcp_backlight_state *state) +{ + return !state->dpms_off && !state->core_blank && !state->suspended; +} + static inline u32 dcp_bl_effective(const struct dcp_backlight_state *state) { - if (state->dpms_off || state->core_blank || state->suspended) - return 0; - return state->target; + return dcp_bl_lit(state) ? state->level : 0; } +/* Stop a running fade and present the target directly. */ +static inline void dcp_bl_jump(struct dcp_backlight_state *state) +{ + state->fading = false; + state->level = state->target; + state->dirty = true; +} + +/* + * With fades enabled, a new level for a panel that stays lit at a known level + * starts a fade from the level presented last; dcp_bl_fade_step() moves it. + * Blanking, unblanking and a level the panel may not show still jump. + */ static inline int dcp_bl_request(struct dcp_backlight_state *state, u32 nits, bool core_blank, bool suspended) { + bool was_lit = dcp_bl_lit(state); + if (!state->ready) return -ENODATA; if (nits > state->maximum) @@ -119,11 +150,44 @@ static inline int dcp_bl_request(struct dcp_backlight_state *state, u32 nits, state->target = nits; state->core_blank = core_blank; state->suspended = suspended; - state->dirty = true; + if (state->fade && state->level_known && was_lit && + dcp_bl_lit(state) && state->level != nits) { + state->fade_from = state->level; + state->fading = true; + } else { + dcp_bl_jump(state); + } } return 0; } +/* + * Move a running fade to @elapsed of @duration, in any one unit. The level + * follows a straight line from the level presented when the fade started, + * and the last step is the target itself. Returns true while the fade runs. + */ +static inline bool dcp_bl_fade_step(struct dcp_backlight_state *state, + u32 elapsed, u32 duration) +{ + u32 from = state->fade_from, to = state->target, level; + + if (!state->fading) + return false; + if (elapsed >= duration) { + level = to; + state->fading = false; + } else if (to >= from) { + level = from + div_u64(mul_u32_u32(to - from, elapsed), duration); + } else { + level = from - div_u64(mul_u32_u32(from - to, elapsed), duration); + } + if (state->level != level) { + state->level = level; + state->dirty = true; + } + return state->fading; +} + static inline int dcp_bl_dpms(struct dcp_backlight_state *state, bool on) { if (!state->ready) @@ -132,7 +196,7 @@ static inline int dcp_bl_dpms(struct dcp_backlight_state *state, bool on) state->retries = 0; if (state->dpms_off != !on) { state->dpms_off = !on; - state->dirty = true; + dcp_bl_jump(state); } return 0; } diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index 41ac118f26cee..a6fb45958b168 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -352,6 +352,155 @@ static void backlight_seed_known_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, dcp_bl_resend(&state)); } +static void backlight_fade_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 100, false, 0), 0); + state.fade = true; + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, false, false), 0); + KUNIT_EXPECT_TRUE(test, state.fading); + KUNIT_EXPECT_FALSE(test, state.dirty); + KUNIT_EXPECT_EQ(test, state.target, 400U); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 100U); + + /* The start of the fade is the level already shown. */ + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 0, 250)); + KUNIT_EXPECT_FALSE(test, state.dirty); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 125, 250)); + KUNIT_EXPECT_TRUE(test, state.dirty); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 250U); + + /* Steps during a present merge into the next one. */ + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 150, 250)); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 200, 250)); + KUNIT_EXPECT_EQ(test, dcp_bl_prepare(&state, true, &present), -EBUSY); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_EQ(test, state.actual, 250U); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 340U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + + /* The last step is exactly the target, however late it runs. */ + KUNIT_EXPECT_FALSE(test, dcp_bl_fade_step(&state, 1000, 250)); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 400U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_EQ(test, state.actual, 400U); + KUNIT_EXPECT_FALSE(test, dcp_bl_fade_step(&state, 250, 250)); + KUNIT_EXPECT_FALSE(test, state.dirty); + + /* Fading down reaches the target too, including zero. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 0, false, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 249, 250)); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 2U); + KUNIT_EXPECT_FALSE(test, dcp_bl_fade_step(&state, 250, 250)); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); +} + +static void backlight_fade_restart_test(struct kunit *test) +{ + struct dcp_backlight_state state; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 100, false, 0), 0); + state.fade = true; + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 500, false, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 100, 250)); + KUNIT_EXPECT_EQ(test, state.level, 260U); + + /* Repeating the target keeps the running fade. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 500, false, false), 0); + KUNIT_EXPECT_TRUE(test, state.fading); + KUNIT_EXPECT_EQ(test, state.fade_from, 100U); + + /* A new level replaces it, starting from the level presented last. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 60, false, false), 0); + KUNIT_EXPECT_TRUE(test, state.fading); + KUNIT_EXPECT_EQ(test, state.fade_from, 260U); + KUNIT_EXPECT_EQ(test, state.level, 260U); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 125, 250)); + KUNIT_EXPECT_EQ(test, state.level, 160U); + KUNIT_EXPECT_FALSE(test, dcp_bl_fade_step(&state, 250, 250)); + KUNIT_EXPECT_EQ(test, state.level, 60U); + + /* A new level equal to the one presented ends the fade there. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 300, false, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 125, 250)); + KUNIT_EXPECT_EQ(test, state.level, 180U); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 180, false, false), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 180U); + + /* No overflow over the whole range a panel ceiling may have. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, INT_MAX, true, 0, false, 0), 0); + state.fade = true; + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, INT_MAX, false, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 249, 250)); + KUNIT_EXPECT_EQ(test, state.level, (u32)div_u64((u64)INT_MAX * 249, 250)); + KUNIT_EXPECT_FALSE(test, dcp_bl_fade_step(&state, 250, 250)); + KUNIT_EXPECT_EQ(test, state.level, (u32)INT_MAX); +} + +static void backlight_fade_blank_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + /* DPMS off and on during a fade jump, as without fades. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 100, false, 0), 0); + state.fade = true; + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, false, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 125, 250)); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_EXPECT_TRUE(test, state.dirty); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_EXPECT_FALSE(test, dcp_bl_fade_step(&state, 200, 250)); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 400U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + + /* A level set while blanked is shown at once on unblank. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, true, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 200, true, false), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 200, false, false), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 200U); + + /* Suspend during a fade blanks at once; resume shows the target. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 500, false, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 50, 250)); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 500, true, true), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 500, false, false), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 500U); + + /* A level the panel may not show yet is presented directly. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, false, 0, true, 262), 0); + state.fade = true; + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 100, false, false), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 100U); + + /* Without fades a new level is presented directly. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 100, false, 0), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, false, false), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 400U); +} + static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_millinits_takeover_test), KUNIT_CASE(backlight_takeover_test), @@ -366,6 +515,9 @@ static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_fallback_test), KUNIT_CASE(backlight_fallback_same_level_test), KUNIT_CASE(backlight_seed_known_test), + KUNIT_CASE(backlight_fade_test), + KUNIT_CASE(backlight_fade_restart_test), + KUNIT_CASE(backlight_fade_blank_test), {} }; From a2760b59de5c1b68ba005424fdb08051e974483f Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:41:05 -0700 Subject: [PATCH 1163/2680] video/logo: boot with the Omarchy logo Our kernels showed the stock Tux logo on any boot without "quiet", while m1n1 and Plymouth already show Omarchy's marks. Replace it with the Omarchy icon (basecamp/omarchy icon.png): the same 15x15 glyph in #9ece6a that Omarchy's m1n1 boot logos carry, drawn at 10 px per cell with a 20 px black margin so the rounded display corners don't clip it. Make it the Kconfig default for LOGO_LINUX_CLUT224_FILE, and show one logo instead of one per online CPU (8 to 12 copies of a brand mark in a row on these machines). fbcon=logo-count:-1 brings the row back. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- Documentation/fb/fbcon.rst | 2 +- drivers/video/fbdev/core/fb_logo.c | 3 +- drivers/video/logo/Kconfig | 4 +- drivers/video/logo/logo_omarchy_clut224.ppm | 3950 +++++++++++++++++++ 4 files changed, 3955 insertions(+), 4 deletions(-) create mode 100644 drivers/video/logo/logo_omarchy_clut224.ppm diff --git a/Documentation/fb/fbcon.rst b/Documentation/fb/fbcon.rst index a98a5cb0b0d8b..5a0d2706b927d 100644 --- a/Documentation/fb/fbcon.rst +++ b/Documentation/fb/fbcon.rst @@ -175,7 +175,7 @@ C. Boot options 8. fbcon=logo-count: The value 'n' overrides the number of bootup logos. 0 disables the - logo, and -1 gives the default which is the number of online CPUs. + logo, and -1 shows one logo per online CPU. The default is 1. D. Attaching, Detaching and Unloading ===================================== diff --git a/drivers/video/fbdev/core/fb_logo.c b/drivers/video/fbdev/core/fb_logo.c index 0bab8352b684a..05f1c47280f52 100644 --- a/drivers/video/fbdev/core/fb_logo.c +++ b/drivers/video/fbdev/core/fb_logo.c @@ -6,7 +6,8 @@ #include "fb_internal.h" bool fb_center_logo __read_mostly; -int fb_logo_count __read_mostly = -1; +/* One Omarchy logo, not one per CPU; fbcon=logo-count:-1 restores the row. */ +int fb_logo_count __read_mostly = 1; static inline unsigned int safe_shift(unsigned int d, int n) { diff --git a/drivers/video/logo/Kconfig b/drivers/video/logo/Kconfig index cda15b95891ed..3ca5585451cbf 100644 --- a/drivers/video/logo/Kconfig +++ b/drivers/video/logo/Kconfig @@ -66,10 +66,10 @@ config LOGO_LINUX_CLUT224_FILE default "drivers/video/logo/logo_sgi_clut224.ppm" if SGI_IP22 || SGI_IP27 || SGI_IP32 default "drivers/video/logo/logo_sun_clut224.ppm" if SPARC default "drivers/video/logo/logo_superh_clut224.ppm" if SUPERH - default "drivers/video/logo/logo_linux_clut224.ppm" + default "drivers/video/logo/logo_omarchy_clut224.ppm" help Takes a path to a 224-color logo in the portable pixmap file - format (.ppm). This defaults to the Tux penguin. + format (.ppm). This defaults to the Omarchy logo. For example, the below ImageMagick command can be used to reduce an image palette to 224 colors and convert it into a ppm file: diff --git a/drivers/video/logo/logo_omarchy_clut224.ppm b/drivers/video/logo/logo_omarchy_clut224.ppm new file mode 100644 index 0000000000000..f18d8f0d15352 --- /dev/null +++ b/drivers/video/logo/logo_omarchy_clut224.ppm @@ -0,0 +1,3950 @@ +P3 +# Omarchy logo (basecamp/omarchy icon.png), 15x15 cells of 10 px +190 190 +255 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 +206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 +158 206 106 158 206 106 158 206 106 158 206 106 158 206 106 158 206 +106 158 206 106 158 206 106 158 206 106 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 From d30315f01f747de47c175f65c5b359820e7561ff Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:15:50 +1100 Subject: [PATCH 1164/2680] drm: apple: isolate an experimental external H17P transport Add a separate external H17P method profile and compact service serializer for bounded trials. Keep the existing internal H17P send rejection and boot-mode inheritance unchanged. Restrict the new serializer to the existing DisplayPort and EDID service calls, validate ring occupancy, and reject unknown callbacks without fabricating success. The compact transmit header and existing command descriptor are protocol hypotheses derived from public interfaces. No hardware compatibility is claimed. This commit does not enable or match an external controller. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 130 +++++++++++++++++++------ drivers/gpu/drm/apple/afk.h | 2 + drivers/gpu/drm/apple/dcp-internal.h | 3 + drivers/gpu/drm/apple/dcp.c | 10 ++ drivers/gpu/drm/apple/dcp.h | 2 + drivers/gpu/drm/apple/dptxep.c | 6 +- drivers/gpu/drm/apple/iomfb.c | 19 +++- drivers/gpu/drm/apple/iomfb_template.c | 8 +- 8 files changed, 148 insertions(+), 32 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index ff1f0f57afd9f..8161db7eb440f 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -572,8 +573,10 @@ static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, dev_err(ep->dcp->dev, "AFK[ep:%02x]: channel %d received unhandled standard service message: %x / %x\n", ep->endpoint, channel, type, eshdr->category); - print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, payload, - payload_size, true); + if (ep->dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, + payload, payload_size, true); } static bool afk_validate_h17p_header(const u8 *data, size_t size) @@ -645,6 +648,16 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, payload_size = data_size - hdr_len; trace_afk_recv_handle(ep, channel, type, data_size, ehdr, eshdr); + if (ep->dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + ep->experimental_rx_logs < 32) { + ep->experimental_rx_logs++; + dev_info(ep->dcp->dev, + "external AFK ep=%u channel=%u type=%u subtype=%u category=%u bytes=%zu\n", + ep->endpoint, channel, type, subtype, eshdr->category, + data_size); + } + service = afk_epic_find_service(ep, channel); @@ -698,8 +711,10 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, dev_err(ep->dcp->dev, "AFK[ep:%02x]: channel %d received unhandled message " "(type %x subtype %x)\n", ep->endpoint, channel, type, subtype); - print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, payload, - payload_size, true); + if (ep->dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, + payload, payload_size, true); return; too_small: @@ -878,6 +893,8 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, struct epic_sub_hdr *eshdr; unsigned long flags; size_t total_epic_size, total_size; + bool compact = ep->dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; int ret; /* @@ -885,30 +902,73 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, * 8-byte sub-header, with the service channel carried in the EPIC * header on queue channel 0. Only the receive side of that format is * known, and the 24-byte framing below would be misparsed, so send - * nothing. The display path needs no AP-initiated EPIC traffic: the + * nothing for the established internal profile. The external-only + * experiment below tests receive-layout symmetry. Internal callers are: the * callers are optional (verbose firmware logging, EDID copy, audio, * DPTX, debugfs) or replies to calls only DPTX services make. */ - if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P) { + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P && !compact) { dev_dbg_once(ep->dcp->dev, "AFK[ep:%02x]: not sending EPIC message on H17P\n", ep->endpoint); return -EOPNOTSUPP; } + /* Only existing DPTX/EDID service commands participate in this trial. */ + if (compact && + (channel > U16_MAX || payload_len > SZ_1M || + (payload_len && !payload) || stype != EPIC_SUBTYPE_STD_SERVICE || + (ep->endpoint != DPTX_ENDPOINT && ep->endpoint != DPAVSERV_ENDPOINT) || + !((ecat == EPIC_CAT_COMMAND && etype == EPIC_TYPE_COMMAND && + payload_len == sizeof(struct epic_cmd)) || + (ecat == EPIC_CAT_REPLY && + (etype == EPIC_TYPE_REPLY || etype == EPIC_TYPE_NOTIFY_ACK))))) + return -EOPNOTSUPP; + spin_lock_irqsave(&ep->lock, flags); if (ep->stopping) { ret = -ESHUTDOWN; goto out; } + if (compact && (!ep->txbfr.ready || !ep->txbfr.rptr || + !ep->txbfr.wptr || !ep->txbfr.buf)) { + ret = -ESHUTDOWN; + goto out; + } + dma_rmb(); rptr = le32_to_cpu(*ep->txbfr.rptr); wptr = le32_to_cpu(*ep->txbfr.wptr); trace_afk_send_rwptr_pre(ep, rptr, wptr); - total_epic_size = sizeof(*ehdr) + sizeof(*eshdr) + payload_len; + total_epic_size = sizeof(*ehdr) + + (compact ? sizeof(struct epic_sub_hdr_h17p) : sizeof(*eshdr)) + + payload_len; total_size = sizeof(*hdr) + total_epic_size; + if (compact) { + u32 available, needed; + + if (!ep->txbfr.ready || !ep->txbfr.stride || + (ep->txbfr.stride & (ep->txbfr.stride - 1)) || + rptr >= ep->txbfr.bufsz || wptr >= ep->txbfr.bufsz || + !IS_ALIGNED(rptr, ep->txbfr.stride) || + !IS_ALIGNED(wptr, ep->txbfr.stride) || + total_size >= ep->txbfr.bufsz) { + ret = -EPROTO; + goto out; + } + available = rptr > wptr ? rptr - wptr : ep->txbfr.bufsz - wptr + rptr; + needed = ALIGN(total_size, ep->txbfr.stride); + if (total_size > ep->txbfr.bufsz - wptr) + needed += ep->txbfr.bufsz - wptr; + /* Retain an empty slot; equal read/write pointers mean empty. */ + if (needed >= available) { + ret = -ENOSPC; + goto out; + } + } + hdr = hdr2 = NULL; /* @@ -989,31 +1049,45 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, hdr->magic = cpu_to_le32(QE_MAGIC); hdr->size = cpu_to_le32(total_epic_size); - hdr->channel = cpu_to_le32(channel); + hdr->channel = cpu_to_le32(compact ? 0 : channel); hdr->type = cpu_to_le32(etype); if (hdr2) memcpy(hdr2, hdr, sizeof(*hdr)); - ehdr = ep->txbfr.buf + wptr; - memset(ehdr, 0, sizeof(*ehdr)); - ehdr->version = 2; - ehdr->seq = cpu_to_le16(ep->qe_seq++); - ehdr->timestamp = cpu_to_le64(0); - wptr += sizeof(*ehdr); - - eshdr = ep->txbfr.buf + wptr; - memset(eshdr, 0, sizeof(*eshdr)); - eshdr->length = cpu_to_le32(payload_len); - eshdr->version = 4; - eshdr->category = ecat; - eshdr->type = cpu_to_le16(stype); - eshdr->timestamp = cpu_to_le64(0); - eshdr->tag = cpu_to_le16(tag); - if (ecat == EPIC_CAT_REPLY) - eshdr->inline_len = cpu_to_le32(payload_len - 4); - else - eshdr->inline_len = cpu_to_le32(0); - wptr += sizeof(*eshdr); + if (compact) { + u8 *wire = ep->txbfr.buf + wptr; + + /* Experimental symmetry with the published H17P receive layout. */ + memset(wire, 0, sizeof(*ehdr) + sizeof(struct epic_sub_hdr_h17p)); + wire[0] = ep->qe_seq++; + put_unaligned_le16(channel, wire + 2); + put_unaligned_le32(total_epic_size - 8, wire + 4); + wire[16] = stype; + wire[17] = ecat; + put_unaligned_le32(tag, wire + 20); + wptr += sizeof(*ehdr) + sizeof(struct epic_sub_hdr_h17p); + } else { + ehdr = ep->txbfr.buf + wptr; + memset(ehdr, 0, sizeof(*ehdr)); + ehdr->version = 2; + ehdr->seq = cpu_to_le16(ep->qe_seq++); + ehdr->timestamp = cpu_to_le64(0); + wptr += sizeof(*ehdr); + + eshdr = ep->txbfr.buf + wptr; + memset(eshdr, 0, sizeof(*eshdr)); + eshdr->length = cpu_to_le32(payload_len); + eshdr->version = 4; + eshdr->category = ecat; + eshdr->type = cpu_to_le16(stype); + eshdr->timestamp = cpu_to_le64(0); + eshdr->tag = cpu_to_le16(tag); + if (ecat == EPIC_CAT_REPLY) + eshdr->inline_len = cpu_to_le32(payload_len - 4); + else + eshdr->inline_len = cpu_to_le32(0); + wptr += sizeof(*eshdr); + } memcpy(ep->txbfr.buf + wptr, payload, payload_len); wptr += payload_len; diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 0cef14fa6dfe8..8eb797e12ef07 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -291,6 +291,8 @@ struct afk_ringbuffer { struct apple_dcp_afkep { struct apple_dcp *dcp; + /* Bounded external protocol observations, owned by the RX worker. */ + u8 experimental_rx_logs; u32 endpoint; struct workqueue_struct *wq; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 5c081e0873eea..ac9776e5fef6a 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -169,6 +169,7 @@ struct dcp_panel { enum dcp_iomfb_method_profile { DCP_IOMFB_METHODS_DEFAULT, DCP_IOMFB_METHODS_H17G, + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL, }; struct apple_dcp_hw_data { @@ -203,6 +204,8 @@ struct apple_dcp { struct apple_connector *connector; struct apple_dcp_hw_data hw; + /* Protected by the H17P IOMFB receive mutex. */ + u8 experimental_callback_logs; /* firmware version and compatible firmware version */ enum dcp_firmware_version fw_compat; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 7514f2b13dde5..9791df970a43b 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -130,6 +130,16 @@ static void dcp_release_rtkit(void *data) #define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) +static bool experimental_h17p_external; +module_param(experimental_h17p_external, bool, 0444); +MODULE_PARM_DESC(experimental_h17p_external, + "Enable experimental T8140 external display protocol"); + +bool dcp_h17p_external_enabled(void) +{ + return experimental_h17p_external; +} + static bool show_notch; module_param(show_notch, bool, 0644); MODULE_PARM_DESC(show_notch, "Use the full display height and shows the notch"); diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 74d4b04095c94..d906c81865a2a 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -12,6 +12,8 @@ #include "dcp-internal.h" #include "parser.h" +bool dcp_h17p_external_enabled(void); + struct apple_crtc { struct drm_crtc base; struct drm_pending_vblank_event *event; diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 11091084f05e2..ec190a7461df9 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -694,7 +694,8 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, trace_dptxport_apcall(dptx, idx, data_size); dev_dbg(service->ep->dcp->dev, "DPTXPort: APCALL %u (%zu bytes)\n", idx, data_size); - if (data_size) + if (data_size && service->ep->dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) print_hex_dump_debug("DPTXPort: apcall data: ", DUMP_PREFIX_OFFSET, 16, 1, data, min(data_size, (size_t)64), true); @@ -792,6 +793,9 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, memset(reply, 0, 4); return 0; default: + if (service->ep->dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + return -EOPNOTSUPP; /* just try to ACK and hope for the best... */ dev_info(service->ep->dcp->dev, "DPTXPort: acking unhandled call %u\n", idx); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index acea7991630b3..418e0dd896d25 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -790,6 +790,14 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, } tag = dcp_parse_tag(hdr->tag); + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + dcp->experimental_callback_logs < 64) { + dcp->experimental_callback_logs++; + dev_info(dev, "external IOMFB callback=%d in=%u out=%u context=%u\n", + tag, hdr->in_len, hdr->out_len, context); + } + handled = tag >= 0 && tag < IOMFB_MAX_CB && dcp->cb_handlers && dcp->cb_handlers[tag]; if (dcp->fw_compat == DCP_FIRMWARE_H17P && @@ -812,6 +820,13 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, !test_and_set_bit(tag, dcp->unknown_callbacks)) dev_warn(dev, "received unknown callback %c%c%c%c\n", hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + /* Retain the failed session without inventing a successful reply. */ + dcp->crashed = true; + complete(&dcp->start_done); + return; + } /* * Leaving a callback unanswered wedges the coprocessor: it * waits for the ack forever and the outer call never returns. @@ -1041,7 +1056,9 @@ static bool dcp_mode_settable(struct apple_dcp *dcp, if (!dcp_mode) return false; - if (dcp->fw_compat == DCP_FIRMWARE_H17P) + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) return dcp_mode->mode.type & DRM_MODE_TYPE_PREFERRED; return true; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 45c65732aed03..4c5a2a3924e2e 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -2204,10 +2204,14 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, dcp->swap_submit_timestamp = 0; - if (DCP_INHERIT_BOOT_MODE) { + if (DCP_INHERIT_BOOT_MODE && + dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { dev_dbg(dcp->dev, "inheriting the bootloader's display mode\n"); complete_set_digital_out_mode(dcp, NULL, cookie); - } else if (mode->vrr) + } else if (mode->vrr && + dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) dcp_set_adaptive_sync(dcp, crtc_state->vrr_enabled ? mode->min_vrr : 0, cookie); From 06a58beca05d118e992451d4eb67bbdfb670f6ef Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Sat, 3 Oct 2026 16:06:12 -0400 Subject: [PATCH 1165/2680] drm/apple: keep data-only USB4 changes local to their port A USB4 transition on an unrelated Type-C port queues mode invalidation and retraining of every live DCP route. The CD321x path also reports an unknown connector status for data-only USB4 attachments. A right-side port reset can consequently reconnect a Samsung display on another port. Remove both blanket notifications and their delayed retrain work. Actual HPD changes, display-route ownership changes and per-display recovery continue to use their existing paths. This needs hardware testing for shared-fabric stalls as well as unrelated-port hotplug. [ryan: linearised onto the H17P series; the backlight fallback work initialisation next to the removed retrain work stays] [ryan: also takes the part of merge 45da41c0cd1b (DjDeveloperr) that completes this commit on the H17P series: the unbind no longer cancels the removed retrain work; four blank lines follow the merge] --- drivers/gpu/drm/apple/dcp-internal.h | 1 - drivers/gpu/drm/apple/dcp.c | 57 +++------------------------- 2 files changed, 5 insertions(+), 53 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index b5ffa71339522..5c081e0873eea 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -383,7 +383,6 @@ struct apple_dcp { /* IOMFB reports its video interface ready after DPTX link training. */ struct completion typec_iomfb_hpd_ready; struct delayed_work typec_reconnect_wq; - struct delayed_work typec_fabric_retrain_wq; u32 typec_reconnect_tries; struct gpio_desc *hdmi_hpd; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 790fbbc1f9ae2..7514f2b13dde5 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -150,8 +150,6 @@ struct apple_dcp_typec_port { struct apple_dcp_typec_route *secondary_owner; /* Keep a port on its last DCP while that pipeline remains free. */ struct apple_dcp_typec_route *preferred_route; - /* Ignore the USB4 fallback immediately following this port's DP teardown. */ - unsigned long dp_release_deadline; /* DRM connector for this physical port, driven by whichever DCP owns it */ struct apple_connector *connector; /* A second logical stream through this port's USB4 dock. */ @@ -489,34 +487,6 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) return 0; } -static void dcp_typec_retrain_work(struct work_struct *work) -{ - struct apple_dcp *dcp = - container_of(to_delayed_work(work), struct apple_dcp, - typec_fabric_retrain_wq); - - struct apple_connector *connector = READ_ONCE(dcp->typec_connector); - - if (READ_ONCE(dcp->active_typec_route) && connector) - dcp_retrain_oob(connector); -} - -static void dcp_typec_retrain_active_routes(void) -{ - struct apple_dcp_typec_port *port; - - list_for_each_entry(port, &dcp_typec_ports, link) { - if (port->owner) - mod_delayed_work(system_freezable_wq, - &port->owner->dcp->typec_fabric_retrain_wq, - msecs_to_jiffies(200)); - if (port->secondary_owner) - mod_delayed_work(system_freezable_wq, - &port->secondary_owner->dcp->typec_fabric_retrain_wq, - msecs_to_jiffies(200)); - } -} - static struct apple_dcp_typec_route * dcp_typec_port_route(struct apple_dcp_typec_port *port, struct apple_dcp *dcp) { @@ -763,7 +733,7 @@ static void dcp_typec_route_waiting(void) if (!route || dcp_typec_route_activate(route, route->xbar)) continue; port->owner = route; - port->dp_release_deadline = 0; + dcp_typec_port_attach(port); } } @@ -859,7 +829,7 @@ dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, } port->owner = route; port->preferred_route = route; - port->dp_release_deadline = 0; + dcp_typec_port_attach(port); } @@ -960,7 +930,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (ret) return ret; port->owner = NULL; - port->dp_release_deadline = jiffies + msecs_to_jiffies(10000); + if (dcp->hdmi_hpd && dcp->active && gpiod_get_value_cansleep(dcp->hdmi_hpd)) dcp_dptx_connect(dcp, 0); @@ -970,18 +940,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, dcp_typec_rebalance_locked(NULL, 0); } - /* - * A port leaving DP can report SAFE/NONE before falling back to USB4. - * Resetting every other live CRTC for that same cable removal blanks - * unaffected displays. Keep the guard across the Type-C state sequence; - * a later, independent USB4 attach still gets recovery. - */ - if (state->mode == TYPEC_MODE_USB4) { - if (!port->dp_release_deadline || - time_after_eq(jiffies, port->dp_release_deadline)) - dcp_typec_retrain_active_routes(); - port->dp_release_deadline = 0; - } + /* Data-only USB4 changes do not invalidate other display routes. */ port->applied_valid = true; return 0; } @@ -1041,7 +1000,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (ret) return ret; port->owner = best; - port->dp_release_deadline = 0; + } @@ -3318,14 +3277,12 @@ static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) /* Block new enqueues as well as draining users of the AFK endpoints. */ disable_delayed_work_sync(&dcp->typec_reconnect_wq); disable_delayed_work_sync(&dcp->placeholder_edid_wq); - disable_delayed_work_sync(&dcp->typec_fabric_retrain_wq); } static void dcp_enable_typec_work(struct apple_dcp *dcp) { enable_delayed_work(&dcp->typec_reconnect_wq); enable_delayed_work(&dcp->placeholder_edid_wq); - enable_delayed_work(&dcp->typec_fabric_retrain_wq); /* A cable can be routed before the DRM component binds. */ if (READ_ONCE(dcp->typec_cable_connected)) mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); @@ -3520,7 +3477,6 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) cancel_work_sync(&dcp->bl_update_wq); } cancel_delayed_work_sync(&dcp->typec_reconnect_wq); - cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); cancel_work_sync(&dcp->vblank_wq); if (dcp->connector) cancel_work_sync(&dcp->connector->hotplug_wq); @@ -3665,13 +3621,10 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp_typec_reconnect_work); INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, dcp_placeholder_edid_work); - INIT_DELAYED_WORK(&dcp->typec_fabric_retrain_wq, - dcp_typec_retrain_work); INIT_DELAYED_WORK(&dcp->bl_fallback_wq, dcp_work_backlight_fallback); /* Balanced by enable at successful component bind. */ disable_delayed_work(&dcp->typec_reconnect_wq); disable_delayed_work(&dcp->placeholder_edid_wq); - disable_delayed_work(&dcp->typec_fabric_retrain_wq); platform_set_drvdata(pdev, dcp); From 3c39da5b276356274fae207de21be60e894e8cec Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:15:50 +1100 Subject: [PATCH 1166/2680] drm: apple: prepare optional external H17P before component matching Add a default-off T8140 external-controller experiment. Require the inherited START, STOPPED and IDLE state before the standard start/wake path and pin its DMA lifetime before any startup write. Complete this optional preparation during platform probe so failed or deferred controllers do not become prerequisites for the internal DRM master. Only successfully bound, prepared controllers are included. Preserve started-session mappings on failure and quiesce retained external work during shutdown. Internal H17P lifecycle remains unchanged. Hardware qualification and validated external resource mappings remain required before this experiment can be used as a supported display path. [ryan: squashed with commit c67346da36d6 ("drm: apple: drain the work items present in the pinned base"), its immediate fix: the two disable_delayed_work_sync() calls on typec_fabric_retrain_wq, a work item this series does not have, broke the build in between; dcp_disable_typec_work() already drains both Type-C work items] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 3 +- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 98 +++++++++++++++++++++++++++- drivers/gpu/drm/apple/dcp.h | 1 + 4 files changed, 99 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 694dc92603695..8fdbd1f367061 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -662,7 +662,8 @@ static const struct of_device_id apple_dcp_id_tbl[] = { static bool apple_dcp_usable(struct device_node *np) { return of_device_is_available(np) && - !of_device_is_compatible(np, "apple,t8140-dcpext"); + (!of_device_is_compatible(np, "apple,t8140-dcpext") || + dcp_h17p_external_ready(np)); } static int apple_drm_init_dcp(struct device *dev) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index ac9776e5fef6a..75aeed23b1cb6 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -206,6 +206,7 @@ struct apple_dcp { struct apple_dcp_hw_data hw; /* Protected by the H17P IOMFB receive mutex. */ u8 experimental_callback_logs; + bool external_prepared; /* firmware version and compatible firmware version */ enum dcp_firmware_version fw_compat; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 9791df970a43b..2fc98b4cb3dc9 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -50,6 +51,9 @@ #define APPLE_DCP_COPROC_CPU_CONTROL 0x44 #define APPLE_DCP_COPROC_CPU_CONTROL_RUN BIT(4) +#define APPLE_DCP_COPROC_CPU_STATUS 0x48 +#define APPLE_DCP_COPROC_CPU_STATUS_STOPPED BIT(1) +#define APPLE_DCP_COPROC_CPU_STATUS_IDLE BIT(5) struct dcp_session_ref { struct list_head link; @@ -80,7 +84,10 @@ static int dcp_pin_live_session(struct apple_dcp *dcp) { struct dcp_session_ref *session; - if (!dcp->hw.adopt_live_session || dcp->fw_compat != DCP_FIRMWARE_H17P || + if ((!dcp->hw.adopt_live_session && + dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) || + dcp->fw_compat != DCP_FIRMWARE_H17P || dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G || dcp->retain_dma) return 0; session = kzalloc_obj(*session); @@ -140,6 +147,27 @@ bool dcp_h17p_external_enabled(void) return experimental_h17p_external; } +bool dcp_h17p_external_ready(struct device_node *np) +{ + struct platform_device *pdev; + struct apple_dcp *dcp; + bool ready = false; + + if (!dcp_h17p_external_enabled()) + return false; + pdev = of_find_device_by_node(np); + if (!pdev) + return false; + /* Failed and deferred probes must never become required components. */ + if (device_is_bound(&pdev->dev)) { + dcp = platform_get_drvdata(pdev); + ready = dcp && READ_ONCE(dcp->external_prepared) && + !READ_ONCE(dcp->quiescing) && !READ_ONCE(dcp->crashed); + } + put_device(&pdev->dev); + return ready; +} + static bool show_notch; module_param(show_notch, bool, 0644); MODULE_PARM_DESC(show_notch, "Use the full display height and shows the notch"); @@ -3298,6 +3326,17 @@ static void dcp_enable_typec_work(struct apple_dcp *dcp) mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); } +static bool dcp_external_is_idle(struct apple_dcp *dcp) +{ + u32 control = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + u32 status = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS); + + /* Same externally inherited idle state accepted by the public loader. */ + return (control & APPLE_DCP_COPROC_CPU_CONTROL_RUN) && + (status & APPLE_DCP_COPROC_CPU_STATUS_STOPPED) && + (status & APPLE_DCP_COPROC_CPU_STATUS_IDLE); +} + static int dcp_comp_bind(struct device *dev, struct device *main, void *data) { struct device_node *panel_np; @@ -3309,6 +3348,9 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) return dev_err_probe(dev, -EBUSY, "Live DCP session cannot be rebound; reboot required\n"); + if (dcp->external_prepared) + return 0; + /* A timed-out prior session may still scan these mappings. */ mutex_lock(&dcp->swapped_out_fbs_lock); ret = list_empty(&dcp->swapped_out_fbs) ? 0 : -EBUSY; @@ -3339,6 +3381,12 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) "DCP was not left running by the bootloader\n"); } + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + !dcp_external_is_idle(dcp)) + return dev_err_probe(dev, -EBUSY, + "External DCP is not in the inherited idle state\n"); + if (dcp->index || dcp->dptx_phy || dcp->dptx_die) dev_info(dev, "DCP index:%u dptx target phy: %u dptx die: %u\n", dcp->index, dcp->dptx_phy, dcp->dptx_die); @@ -3419,6 +3467,19 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); INIT_DELAYED_WORK(&dcp->swap_watchdog_wq, dcp_swap_watchdog); + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + /* Pin before the first write or mailbox wake can expose DMA. */ + ret = dcp_pin_live_session(dcp); + if (ret) + goto err_piodma; + if (!dcp_external_is_idle(dcp)) { + ret = dev_err_probe(dev, -EBUSY, + "External DCP changed state before startup\n"); + goto err_piodma; + } + } + if (!dcp->hw.adopt_live_session) { cpu_ctrl = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); @@ -3569,7 +3630,8 @@ static int dcp_platform_probe(struct platform_device *pdev) return dev_err_probe(dev, -EBUSY, "Previous live DCP session requires a reboot\n"); /* Not part of the display subsystem yet; see apple_dcp_usable(). */ - if (of_device_is_compatible(dev->of_node, "apple,t8140-dcpext")) { + if (of_device_is_compatible(dev->of_node, "apple,t8140-dcpext") && + !dcp_h17p_external_enabled()) { dev_info(dev, "external display coprocessor not supported yet\n"); return -ENODEV; } @@ -3771,7 +3833,22 @@ static int dcp_platform_probe(struct platform_device *pdev) if (ret) return ret; - return component_add(&pdev->dev, &dcp_comp_ops); + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + /* Complete optional startup before the DRM master can depend on it. */ + ret = dcp_comp_bind(dev, NULL, NULL); + if (ret) { + dcp_disable_typec_work(dcp, true); + return ret; + } + WRITE_ONCE(dcp->external_prepared, true); + } + + ret = component_add(&pdev->dev, &dcp_comp_ops); + if (ret && dcp->external_prepared) { + dcp_disable_typec_work(dcp, true); + } + return ret; } static void dcp_platform_remove(struct platform_device *pdev) @@ -3783,6 +3860,12 @@ static void dcp_platform_shutdown(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); + if (dcp && dcp->retain_dma && dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + dcp_comp_unbind(&pdev->dev, NULL, NULL); + return; + } + if (dcp && dcp->hw.adopt_live_session && dcp->fw_compat == DCP_FIRMWARE_H17P && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { @@ -3921,6 +4004,14 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8140 = { .firmware_request = 0x68000, }; +/* Separate from the established internal-panel H17P profile. */ +static const struct apple_dcp_hw_data apple_dcp_hw_t8140_external = { + .num_dptx_ports = 1, + .iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL, + .adopt_live_session = false, + .firmware_compat = DCP_FIRMWARE_H17P, +}; + /* * M5 (T8142) runs H17-generation DCP firmware with the H17G method numbering * and is left running by the bootloader. @@ -3944,6 +4035,7 @@ static const struct of_device_id of_match[] = { { .compatible = "apple,t6020-dcp", .data = &apple_dcp_hw_t6020, }, { .compatible = "apple,t8112-dcp", .data = &apple_dcp_hw_t8112, }, { .compatible = "apple,t8140-dcp", .data = &apple_dcp_hw_t8140, }, + { .compatible = "apple,t8140-dcpext", .data = &apple_dcp_hw_t8140_external, }, { .compatible = "apple,t8142-dcp", .data = &apple_dcp_hw_t8142, }, { .compatible = "apple,dcp", .data = &apple_dcp_hw_dcp, }, { .compatible = "apple,dcpext", .data = &apple_dcp_hw_dcpext, }, diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index d906c81865a2a..f5f3d29e39d04 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -13,6 +13,7 @@ #include "parser.h" bool dcp_h17p_external_enabled(void); +bool dcp_h17p_external_ready(struct device_node *np); struct apple_crtc { struct drm_crtc base; From 3e2f864745bda4b79f52495f7c4777c1ad8cc74d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:30:45 +1100 Subject: [PATCH 1167/2680] drm: apple: recognize the T8140 external binding in the display master Use the generic DCP fallback required by the published T8140 external binding and explicitly classify that specific compatible as an external pipeline. Otherwise the display master never discovers a successfully prepared external node. The device and route remain disabled by default. [ryan: split by topic: only the drm/apple part and its selftests are here; the DT hunks are in 000-devicetree] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 8fdbd1f367061..e6e449d9654f3 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -681,6 +681,7 @@ static int apple_drm_init_dcp(struct device *dev) if (!apple_dcp_usable(np)) continue; dcp_ext = of_device_is_compatible(np, "apple,dcpext") || + of_device_is_compatible(np, "apple,t8140-dcpext") || of_property_present(np, "phys"); dcp[num_dcp] = of_find_device_by_node(np); From 3d777eb7ed7d2e2bd25714c94a2093114f0a2529 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:40:04 +1100 Subject: [PATCH 1168/2680] drm: apple: validate external display clock and bandwidth resources Separate the number of display windows from the shared firmware clock aperture. Preserve the internal five-window profile and validate the external four-window profile against its existing final clock resource. Require explicit read-only experimental bandwidth offsets for the external profile, with alignment, distinctness and aperture bounds checks before startup. Keep both offsets unset by default and reject missing external clock rates rather than guessing them. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-external-bw.h | 24 ++++++++ drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 56 ++++++++++++++++++- .../drivers/gpu/test_t8140_external_bw.py | 45 +++++++++++++++ 4 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp-external-bw.h create mode 100644 tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py diff --git a/drivers/gpu/drm/apple/dcp-external-bw.h b/drivers/gpu/drm/apple/dcp-external-bw.h new file mode 100644 index 0000000000000..439db008b5084 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-external-bw.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Aurora Silicon */ +#ifndef APPLE_DCP_EXTERNAL_BW_H +#define APPLE_DCP_EXTERNAL_BW_H + +#include + +static inline bool apple_dcp_external_bw_valid(u64 size, int scratch, int request) +{ + if (size < sizeof(u32) || scratch < 0 || request < 0) + return false; + if ((scratch & 3) || (request & 3) || scratch == request) + return false; + return (u64)scratch <= size - sizeof(u32) && + (u64)request <= size - sizeof(u32); +} + +static inline bool apple_dcp_external_clock_window(u64 start, u64 size, + u64 expected_start, u64 expected_size) +{ + return start == expected_start && size == expected_size; +} + +#endif diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 75aeed23b1cb6..d28025e31a487 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -183,6 +183,7 @@ struct apple_dcp_hw_data { enum dcp_firmware_version firmware_compat; /* Firmware-visible clock aperture, independent of the AP PMGR window. */ const struct resource *firmware_clock; + u32 firmware_display_windows; u32 firmware_scratch; u32 firmware_request; }; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 2fc98b4cb3dc9..b20950fd9cb01 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -45,6 +45,7 @@ #include "av.h" #include "dcp.h" #include "dcp-internal.h" +#include "dcp-external-bw.h" #include "iomfb.h" #include "parser.h" #include "trace.h" @@ -142,6 +143,16 @@ module_param(experimental_h17p_external, bool, 0444); MODULE_PARM_DESC(experimental_h17p_external, "Enable experimental T8140 external display protocol"); +static int experimental_h17p_external_bw_scratch = -1; +module_param(experimental_h17p_external_bw_scratch, int, 0444); +MODULE_PARM_DESC(experimental_h17p_external_bw_scratch, + "Experimental external bandwidth scratch offset; unset by default"); + +static int experimental_h17p_external_bw_request = -1; +module_param(experimental_h17p_external_bw_request, int, 0444); +MODULE_PARM_DESC(experimental_h17p_external_bw_request, + "Experimental external bandwidth request offset; unset by default"); + bool dcp_h17p_external_enabled(void) { return experimental_h17p_external; @@ -3167,8 +3178,25 @@ static int dcp_get_disp_regs(struct apple_dcp *dcp) } if (dcp->hw.firmware_clock) { - /* The internal firmware's last aperture follows five display windows. */ - if (count != 5 || resource_size(dcp->hw.firmware_clock) < 4 || + unsigned int windows = dcp->hw.firmware_display_windows; + bool external = dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + + if (!windows || windows >= MAX_DISP_REGISTERS) + return -EINVAL; + if (external) { + /* The external clock aperture is already described by disp-4. */ + if (count != windows + 1) + return -EINVAL; + res = dcp->disp_registers[windows]; + if (!apple_dcp_external_clock_window(res->start, + resource_size(res), + dcp->hw.firmware_clock->start, + resource_size(dcp->hw.firmware_clock))) + return -EINVAL; + count = windows; + } + if (count != windows || resource_size(dcp->hw.firmware_clock) < 4 || dcp->hw.firmware_scratch > resource_size(dcp->hw.firmware_clock) - 4 || dcp->hw.firmware_request > resource_size(dcp->hw.firmware_clock) - 4) return -EINVAL; @@ -3460,6 +3488,15 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) goto err_piodma; } + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + (!dcp->clk_194 || !clk_get_rate(dcp->clk) || + !clk_get_rate(dcp->clk_194))) { + ret = dev_err_probe(dev, -EINVAL, + "External display clocks are unavailable\n"); + goto err_piodma; + } + bitmap_zero(dcp->memdesc_map, DCP_MAX_MAPPINGS); // TDOD: mem_desc IDs start at 1, for simplicity just skip '0' entry set_bit(0, dcp->memdesc_map); @@ -3674,6 +3711,18 @@ static int dcp_platform_probe(struct platform_device *pdev) spin_lock_init(&dcp->mode_state.lock); spin_lock_init(&dcp->dcpavserv.lock); dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + if (!dcp->hw.firmware_clock || + !apple_dcp_external_bw_valid(resource_size(dcp->hw.firmware_clock), + experimental_h17p_external_bw_scratch, + experimental_h17p_external_bw_request)) + return dev_err_probe(dev, -EINVAL, + "External bandwidth offsets are unset or invalid\n"); + dcp->hw.firmware_scratch = experimental_h17p_external_bw_scratch; + dcp->hw.firmware_request = experimental_h17p_external_bw_request; + } + /* The domain use must survive driver-core DMA cleanup for a live session. */ ret = iommu_device_use_default_domain(dev); @@ -4000,6 +4049,7 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8140 = { .adopt_live_session = true, .firmware_compat = DCP_FIRMWARE_H17P, .firmware_clock = &t8140_firmware_clock, + .firmware_display_windows = 5, .firmware_scratch = 0x20000, .firmware_request = 0x68000, }; @@ -4010,6 +4060,8 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8140_external = { .iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL, .adopt_live_session = false, .firmware_compat = DCP_FIRMWARE_H17P, + .firmware_clock = &t8140_firmware_clock, + .firmware_display_windows = 4, }; /* diff --git a/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py b/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py new file mode 100644 index 0000000000000..73f0dba8d4d22 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: MIT +import pathlib +import subprocess +import tempfile +import unittest + + +class BandwidthBounds(unittest.TestCase): + def test_shared_header_boundaries(self): + root = pathlib.Path(__file__).resolve().parents[5] + code = r""" +#include +#include +#include "dcp-external-bw.h" +int main(void) { + const u64 n=0xbc000; + assert(!apple_dcp_external_bw_valid(n,-1,-1)); + assert(!apple_dcp_external_bw_valid(n,0,-1)); + assert(!apple_dcp_external_bw_valid(n,-1,0)); + assert(!apple_dcp_external_bw_valid(n,4,4)); + assert(!apple_dcp_external_bw_valid(n,2,4)); + assert(!apple_dcp_external_bw_valid(n,4,6)); + assert(!apple_dcp_external_bw_valid(n,n,4)); + assert(!apple_dcp_external_bw_valid(n,0,n)); + assert(!apple_dcp_external_bw_valid(3,0,4)); + assert(!apple_dcp_external_bw_valid(n,INT_MAX,0)); + assert(apple_dcp_external_bw_valid(n,0,4)); + assert(apple_dcp_external_bw_valid(n,n-4,0)); + assert(apple_dcp_external_bw_valid(n,0,n-4)); + assert(apple_dcp_external_clock_window(0x302800000ULL,n,0x302800000ULL,n)); + assert(!apple_dcp_external_clock_window(0x302800000ULL,n-4,0x302800000ULL,n)); + assert(!apple_dcp_external_clock_window(0x302800004ULL,n,0x302800000ULL,n)); + return 0; +} +""" + with tempfile.TemporaryDirectory() as directory: + tmp=pathlib.Path(directory);(tmp/'linux').mkdir() + (tmp/'linux/types.h').write_text('#include \n#include \ntypedef uint64_t u64; typedef uint32_t u32;\n') + (tmp/'test.c').write_text(code) + subprocess.run(['clang','-std=c11','-Wall','-Wextra','-Werror','-pedantic','-I'+str(tmp),'-I'+str(root/'drivers/gpu/drm/apple'),str(tmp/'test.c'),'-o',str(tmp/'test')],check=True) + subprocess.run([str(tmp/'test')],check=True) + + +if __name__=='__main__': + unittest.main() From e4329a29d595876067328aadfba4a212fd00490b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:47:51 +1100 Subject: [PATCH 1169/2680] drm/apple: announce AP readiness early for experimental external H17P The external H17P trial completes endpoint setup but its normal RTKit wake sequence times out. Use the existing early-AP wake helper for this experimental profile, which announces host readiness before waiting for IOP readiness. Keep the normal wake sequence for other profiles. This ordering is a hardware qualification candidate. The previous trial retained the internal panel but did not expose an external connector. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index b20950fd9cb01..1a1d2d15af956 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3541,7 +3541,12 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (dcp->hw.adopt_live_session) dev_info(dev, "negotiating RTKit with the running DCP\n"); - ret = apple_rtkit_wake(dcp->rtk); + /* Announce host readiness before awaiting external IOP readiness. */ + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + ret = apple_rtkit_wake_early_ap(dcp->rtk); + else + ret = apple_rtkit_wake(dcp->rtk); if (ret) { ret = dev_err_probe(dev, ret, "Failed to boot RTKit: %d\n", ret); goto err_piodma; From b1406ce94415043f5b743727ad8e3bc47cbc7449 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:54:13 +1100 Subject: [PATCH 1170/2680] drm/apple: acknowledge firmware-owned external H17P buffers The external H17P startup requests a shared buffer at a firmware-provided IOVA. Its setup callback maps that buffer, but the standard RTKit ops do not acknowledge mapped buffers. Select the existing co-processor-owned buffer protocol for this experimental profile so RTKit acknowledges the request and starts the crashlog endpoint before its endpoint-map reply. Keep other display profiles on their existing ops. Restore the normal wake ordering after the earlier AP-readiness trial also timed out, while retaining diagnostics for subsequent qualification. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 1a1d2d15af956..98d8b2ddff417 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2092,6 +2092,14 @@ static struct apple_rtkit_ops rtkit_ops = { .shmem_destroy = dcp_rtk_shmem_destroy, }; +static const struct apple_rtkit_ops external_h17p_rtkit_ops = { + .flags = APPLE_RTKIT_COPROC_PLACES_BUFFERS, + .crashed = dcp_rtk_crashed, + .recv_message = dcp_recv_msg, + .shmem_setup = dcp_rtk_shmem_setup, + .shmem_destroy = dcp_rtk_shmem_destroy, +}; + void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message) { if (READ_ONCE(dcp->quiescing)) @@ -3528,7 +3536,13 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) ret = dcp_pin_live_session(dcp); if (ret) goto err_piodma; - dcp->rtk = apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); + /* External H17P supplies shared-buffer addresses during RTKit startup. */ + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + dcp->rtk = apple_rtkit_init(dev, dcp, "mbox", 0, + &external_h17p_rtkit_ops); + else + dcp->rtk = apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); if (IS_ERR(dcp->rtk)) { ret = dev_err_probe(dev, PTR_ERR(dcp->rtk), "Failed to initialize RTKit\n"); @@ -3541,12 +3555,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (dcp->hw.adopt_live_session) dev_info(dev, "negotiating RTKit with the running DCP\n"); - /* Announce host readiness before awaiting external IOP readiness. */ - if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) - ret = apple_rtkit_wake_early_ap(dcp->rtk); - else - ret = apple_rtkit_wake(dcp->rtk); + ret = apple_rtkit_wake(dcp->rtk); if (ret) { ret = dev_err_probe(dev, ret, "Failed to boot RTKit: %d\n", ret); goto err_piodma; From 521a0ab247624ec0b0df664fb3d68de5d88fd713 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:00:10 +1100 Subject: [PATCH 1171/2680] drm/apple: pair external buffer acknowledgments with early AP readiness The early-AP trial still lacked acknowledgments for firmware-owned shared buffers. The buffer-protocol trial supplied those acknowledgments but kept waiting for IOP readiness before announcing AP readiness. Neither trial established that one change alone could complete the external handshake. Combine the existing early-AP helper with the corrected buffer protocol for the experimental external profile. Keep normal display profiles and all timeout and lifetime rules unchanged. Hardware qualification of the combined sequence remains required. [ryan: the inventory listed this as a duplicate of commit 5da6455cdd51 ("drm/apple: announce AP readiness early for experimental external H17P"); it re-applies that change after commit e00ebb9cde3c ("drm/apple: acknowledge firmware-owned external H17P buffers") removed it, so it is carried] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 98d8b2ddff417..b2d905549e912 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3555,7 +3555,12 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (dcp->hw.adopt_live_session) dev_info(dev, "negotiating RTKit with the running DCP\n"); - ret = apple_rtkit_wake(dcp->rtk); + /* Announce host readiness before awaiting external IOP readiness. */ + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + ret = apple_rtkit_wake_early_ap(dcp->rtk); + else + ret = apple_rtkit_wake(dcp->rtk); if (ret) { ret = dev_err_probe(dev, ret, "Failed to boot RTKit: %d\n", ret); goto err_piodma; From 3f36d056687fbc56a99b8f038ca44cbaeae21d92 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:07:14 +1100 Subject: [PATCH 1172/2680] drm/apple: recognize the external H17P display service The external H17P firmware announces dispext-service on the existing IBOOT endpoint. Match that measured service name for the experimental external profile, keeping the normal disp0-service table unchanged. Propagate endpoint allocation and startup failures to the caller. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/ibootep.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/ibootep.c b/drivers/gpu/drm/apple/ibootep.c index ae4bc8a69f2a8..79d5dc19569c3 100644 --- a/drivers/gpu/drm/apple/ibootep.c +++ b/drivers/gpu/drm/apple/ibootep.c @@ -20,10 +20,23 @@ static const struct apple_epic_service_ops ibootep_ops[] = { {} }; +static const struct apple_epic_service_ops external_ibootep_ops[] = { + { + .name = "dispext-service", + .init = disp_service_init, + }, + {} +}; + int ibootep_init(struct apple_dcp *dcp) { - dcp->ibootep = afk_init(dcp, DISP0_ENDPOINT, ibootep_ops); - afk_start(dcp->ibootep); + const struct apple_epic_service_ops *ops = ibootep_ops; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + ops = external_ibootep_ops; + dcp->ibootep = afk_init(dcp, DISP0_ENDPOINT, ops); + if (IS_ERR(dcp->ibootep)) + return PTR_ERR(dcp->ibootep); - return 0; + return afk_start(dcp->ibootep); } From bf7b1feb52377bb1a4c38e4e125179a488b7f465 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:07:15 +1100 Subject: [PATCH 1173/2680] drm/apple: report external AFK startup progress Report external-profile control-message types and ring readiness when startup times out. Exclude normal receive notifications and rate-limit control diagnostics; no payload contents or addresses are logged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 8161db7eb440f..c7e2baf56a93a 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -146,6 +146,11 @@ int afk_start(struct apple_dcp_afkep *ep) afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_INIT)); ret = wait_for_completion_timeout(&ep->started, msecs_to_jiffies(1000)); + if (ret <= 0 && ep->dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + dev_warn(ep->dcp->dev, + "external AFK startup ep=%#x tx_ready=%u rx_ready=%u bytes=%zu\n", + ep->endpoint, ep->txbfr.ready, ep->rxbfr.ready, ep->bfr_size); if (ret <= 0) return -ETIMEDOUT; else @@ -818,6 +823,11 @@ static void afk_receive_message_worker(struct work_struct *work_) work = container_of(work_, struct afk_receive_message_work, work); type = FIELD_GET(RBEP_TYPE, work->message); + if (work->ep->dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && type != RBEP_RECV) + dev_info_ratelimited(work->ep->dcp->dev, + "external AFK control ep=%#x type=%#x\n", + work->ep->endpoint, type); switch (type) { case RBEP_INIT_ACK: break; From 0121ab6e27a11e28a50e15497bc3794d6cc71466 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:11:52 +1100 Subject: [PATCH 1174/2680] drm/apple: start the external H17P allocator before service discovery External H17P reaches AFK transport readiness but does not announce the DPTX port service. Its existing allocator endpoint is currently started only later with IOMFB. Make that known endpoint available before external service discovery, and skip its duplicate start during IOMFB startup. Keep normal H17P ordering unchanged. This external-only ordering change requires hardware validation; no new endpoint or message is introduced. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 7 +++++++ drivers/gpu/drm/apple/iomfb.c | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index b2d905549e912..ac91e9196cbcb 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2546,6 +2546,13 @@ int dcp_start(struct platform_device *pdev) return -ENODEV; } + /* Make the H17P allocator available before external service discovery. */ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + ret = apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); + if (ret) + return dev_err_probe(dcp->dev, ret, "Failed to start remote allocator\n"); + } + /* start RTKit endpoints */ ret = systemep_init(dcp); if (ret) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 418e0dd896d25..20c0ad6711347 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1572,7 +1572,8 @@ int iomfb_start_rtkit(struct apple_dcp *dcp) * has nothing answering it on the AP side, and without this one the * display DART faults. */ - if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { ret = apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); if (ret) return ret; From ac9e66906050fb211f0af435c1952f69846ef80a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:11:52 +1100 Subject: [PATCH 1175/2680] drm/apple: isolate DPTX discovery diagnostics Limit external AFK control diagnostics to DPTX so unrelated endpoint startup cannot consume their rate limit. Distinguish a ready transport with no announced port service from a transport-start timeout. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 3 ++- drivers/gpu/drm/apple/dptxep.c | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index c7e2baf56a93a..c8a6d1426171a 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -824,7 +824,8 @@ static void afk_receive_message_worker(struct work_struct *work_) type = FIELD_GET(RBEP_TYPE, work->message); if (work->ep->dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && type != RBEP_RECV) + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + work->ep->endpoint == DPTX_ENDPOINT && type != RBEP_RECV) dev_info_ratelimited(work->ep->dcp->dev, "external AFK control ep=%#x type=%#x\n", work->ep->endpoint, type); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index ec190a7461df9..571f6f1b03d3e 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -871,10 +871,13 @@ int dptxep_init(struct apple_dcp *dcp) for (port = 0; port < dcp->hw.num_dptx_ports; port++) { ret = wait_for_completion_timeout(&dcp->dptxport[port].enable_completion, timeout); - if (!ret) + if (!ret) { + dev_warn(dcp->dev, "DPTX transport ready but port %u service not announced\n", + port); return -ETIMEDOUT; - else if (ret < 0) + } else if (ret < 0) { return ret; + } timeout = ret; } From 502986c8bde0ffef162a4d75e1dfbfb1bd9d9f55 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:17:13 +1100 Subject: [PATCH 1176/2680] drm/apple: initialize the external DPTX controller before port discovery The public m1n1 display startup initializes the DPTX controller endpoint before the port endpoint. Add that existing AFK controller interface to the experimental external H17P path, wait for its service announcement, and then continue port discovery. Send no controller service commands. Use a dedicated endpoint pointer and route its incoming messages through the existing AFK worker. Cover both retained-session quiescing and normal teardown, and never publish an allocation error as an endpoint pointer. Normal display profiles retain their existing startup sequence. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 3 +++ drivers/gpu/drm/apple/dcp.c | 19 +++++++++++++++ drivers/gpu/drm/apple/dcp.h | 1 + drivers/gpu/drm/apple/dptxep.c | 35 ++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index d28025e31a487..410cb41457a8c 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -84,6 +84,7 @@ enum { TEST_ENDPOINT = 0x21, DCP_EXPERT_ENDPOINT = 0x22, DISP0_ENDPOINT = 0x23, + DPTX_CONTROLLER_ENDPOINT = 0x24, DPAVSERV_ENDPOINT = 0x28, AV_ENDPOINT = 0x29, DPTX_ENDPOINT = 0x2a, @@ -353,6 +354,8 @@ struct apple_dcp { struct audiosrv_data *audiosrv; struct apple_dcp_afkep *dptxep; + struct apple_dcp_afkep *dptx_controller_ep; + struct completion dptx_controller_ready; struct dptx_port dptxport[2]; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index ac91e9196cbcb..8b6502817e6b4 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1875,6 +1875,8 @@ static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) return dcp->dcpavservep; case DPTX_ENDPOINT: return dcp->dptxep; + case DPTX_CONTROLLER_ENDPOINT: + return dcp->dptx_controller_ep; default: return NULL; } @@ -1958,6 +1960,9 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) case DPTX_ENDPOINT: ep = dcp->dptxep; break; + case DPTX_CONTROLLER_ENDPOINT: + ep = dcp->dptx_controller_ep; + break; default: ep = NULL; break; @@ -2571,6 +2576,14 @@ int dcp_start(struct platform_device *pdev) dev_warn(dcp->dev, "Failed to start IBOOT endpoint: %d\n", ret); + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + ret = dptx_controller_init(dcp); + if (ret) + return dev_err_probe(dcp->dev, ret, + "Failed to start DPTX controller\n"); + } + ret = dptxep_init(dcp); if (ret) { dev_warn(dcp->dev, "Failed to start DPTX endpoint: %d\n", @@ -3601,6 +3614,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) iomfb_queue_stop(dcp); afk_quiesce(dcp->avep); afk_quiesce(dcp->dptxep); + afk_quiesce(dcp->dptx_controller_ep); afk_quiesce(dcp->ibootep); afk_quiesce(dcp->systemep); afk_quiesce(dcp->dcpavservep); @@ -3637,6 +3651,11 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) dcp->dptxep = NULL; } + if (!IS_ERR_OR_NULL(dcp->dptx_controller_ep)) { + afk_shutdown(dcp->dptx_controller_ep); + dcp->dptx_controller_ep = NULL; + } + if (dcp->ibootep) { afk_shutdown(dcp->ibootep); dcp->ibootep = NULL; diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index f5f3d29e39d04..f1afe81e475d2 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -97,6 +97,7 @@ void iomfb_recv_msg(struct apple_dcp *dcp, u64 message); int systemep_init(struct apple_dcp *dcp); int dptxep_init(struct apple_dcp *dcp); +int dptx_controller_init(struct apple_dcp *dcp); int ibootep_init(struct apple_dcp *dcp); int dpavservep_init(struct apple_dcp *dcp); int avep_init(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 571f6f1b03d3e..e1eeb8aa6bd47 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -849,6 +849,41 @@ static const struct apple_epic_service_ops dptxep_ops[] = { {} }; +static void dptx_controller_service_init(struct apple_epic_service *service, + const char *name, const char *class, s64 unit) +{ + complete(&service->ep->dcp->dptx_controller_ready); +} + +static const struct apple_epic_service_ops dptx_controller_ops[] = { + { + .name = "AppleDCPDPTXController", + .init = dptx_controller_service_init, + }, + {} +}; + +int dptx_controller_init(struct apple_dcp *dcp) +{ + struct apple_dcp_afkep *ep; + int ret; + + init_completion(&dcp->dptx_controller_ready); + ep = afk_init(dcp, DPTX_CONTROLLER_ENDPOINT, dptx_controller_ops); + if (IS_ERR(ep)) + return PTR_ERR(ep); + dcp->dptx_controller_ep = ep; + + ret = afk_start(ep); + if (ret) + return ret; + if (!wait_for_completion_timeout(&dcp->dptx_controller_ready, + msecs_to_jiffies(1000))) + return -ETIMEDOUT; + + return 0; +} + int dptxep_init(struct apple_dcp *dcp) { int ret; From 331807ec9b560a7bbd71a5751618f6aec5467803 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:27:24 +1100 Subject: [PATCH 1177/2680] drm/apple: discover the version-two external DPTX service The external H17P test reaches transport readiness on the legacy DPTX endpoint without receiving a port announcement. The public version-two firmware implementation uses endpoint 0x2c, which this firmware advertises. Use that endpoint for an isolated service-discovery test. Keep a separate endpoint object with an initialization-only service table, RX routing and lifecycle handling. This does not enable operational port callbacks or transmit service commands; the discovered identity and framing must be validated before extending the command path. Normal profiles retain the legacy DPTX endpoint. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 3 +- drivers/gpu/drm/apple/dcp-internal.h | 3 ++ drivers/gpu/drm/apple/dcp.c | 12 ++++++++ drivers/gpu/drm/apple/dptxep.c | 41 ++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index c8a6d1426171a..546774c00f45c 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -825,7 +825,8 @@ static void afk_receive_message_worker(struct work_struct *work_) type = FIELD_GET(RBEP_TYPE, work->message); if (work->ep->dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && - work->ep->endpoint == DPTX_ENDPOINT && type != RBEP_RECV) + (work->ep->endpoint == DPTX_ENDPOINT || + work->ep->endpoint == DPTX_ENDPOINT_V2) && type != RBEP_RECV) dev_info_ratelimited(work->ep->dcp->dev, "external AFK control ep=%#x type=%#x\n", work->ep->endpoint, type); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 410cb41457a8c..3db1a80c1087f 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -88,6 +88,7 @@ enum { DPAVSERV_ENDPOINT = 0x28, AV_ENDPOINT = 0x29, DPTX_ENDPOINT = 0x2a, + DPTX_ENDPOINT_V2 = 0x2c, HDCP_ENDPOINT = 0x2b, REMOTE_ALLOC_ENDPOINT = 0x2d, IOMFB_ENDPOINT = 0x37, @@ -356,6 +357,8 @@ struct apple_dcp { struct apple_dcp_afkep *dptxep; struct apple_dcp_afkep *dptx_controller_ep; struct completion dptx_controller_ready; + struct apple_dcp_afkep *dptx_v2_probe_ep; + struct completion dptx_v2_probe_ready; struct dptx_port dptxport[2]; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8b6502817e6b4..7feafd092851b 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1875,6 +1875,9 @@ static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) return dcp->dcpavservep; case DPTX_ENDPOINT: return dcp->dptxep; + case DPTX_ENDPOINT_V2: + return dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL ? dcp->dptx_v2_probe_ep : NULL; case DPTX_CONTROLLER_ENDPOINT: return dcp->dptx_controller_ep; default: @@ -1960,6 +1963,9 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) case DPTX_ENDPOINT: ep = dcp->dptxep; break; + case DPTX_ENDPOINT_V2: + ep = dcp->dptx_v2_probe_ep; + break; case DPTX_CONTROLLER_ENDPOINT: ep = dcp->dptx_controller_ep; break; @@ -3615,6 +3621,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) afk_quiesce(dcp->avep); afk_quiesce(dcp->dptxep); afk_quiesce(dcp->dptx_controller_ep); + afk_quiesce(dcp->dptx_v2_probe_ep); afk_quiesce(dcp->ibootep); afk_quiesce(dcp->systemep); afk_quiesce(dcp->dcpavservep); @@ -3651,6 +3658,11 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) dcp->dptxep = NULL; } + if (!IS_ERR_OR_NULL(dcp->dptx_v2_probe_ep)) { + afk_shutdown(dcp->dptx_v2_probe_ep); + dcp->dptx_v2_probe_ep = NULL; + } + if (!IS_ERR_OR_NULL(dcp->dptx_controller_ep)) { afk_shutdown(dcp->dptx_controller_ep); dcp->dptx_controller_ep = NULL; diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index e1eeb8aa6bd47..e00c4cbcea171 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -884,12 +884,53 @@ int dptx_controller_init(struct apple_dcp *dcp) return 0; } +static void dptx_v2_probe_service(struct apple_epic_service *service, + const char *name, const char *class, s64 unit) +{ + dev_info(service->ep->dcp->dev, + "DPTX v2 discovery name=%s class=%s unit=%lld channel=%u\n", + name ?: "(none)", class ?: "(none)", unit, service->channel); + complete(&service->ep->dcp->dptx_v2_probe_ready); +} + +static const struct apple_epic_service_ops dptx_v2_probe_ops[] = { + { + .name = "AppleDCPDPTXRemotePort", + .init = dptx_v2_probe_service, + }, + {} +}; + +static int dptx_v2_discover(struct apple_dcp *dcp) +{ + struct apple_dcp_afkep *ep; + int ret; + + init_completion(&dcp->dptx_v2_probe_ready); + ep = afk_init(dcp, DPTX_ENDPOINT_V2, dptx_v2_probe_ops); + if (IS_ERR(ep)) + return PTR_ERR(ep); + dcp->dptx_v2_probe_ep = ep; + ret = afk_start(ep); + if (ret) + return ret; + if (!wait_for_completion_timeout(&dcp->dptx_v2_probe_ready, + msecs_to_jiffies(1000))) + return -ETIMEDOUT; + + /* Discovery does not enable port callbacks or transmit service commands. */ + return -EOPNOTSUPP; +} + int dptxep_init(struct apple_dcp *dcp) { int ret; u32 port; unsigned long timeout = msecs_to_jiffies(1000); + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + return dptx_v2_discover(dcp); + init_completion(&dcp->dptxport[0].enable_completion); init_completion(&dcp->dptxport[1].enable_completion); init_completion(&dcp->dptxport[0].linkcfg_completion); From ec099ac10ff021e9e0b4553363986ab7958a2780 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:39:11 +1100 Subject: [PATCH 1178/2680] drm/apple: operate the external DPTX service over AFKV2 The external T8140 controller announces its port on endpoint 0x2c using version-two frames. Select that transport per endpoint, open announced interfaces and map the single physical DCP unit to logical port zero. Encode bounded command descriptors and validate replies against pending owned buffers before completing them. Preserve callback tokens and limit peer-buffer access to active allocations of the same service. Keep the legacy transport for other display profiles. Add host selftests for frame encoding, malformed records, response correlation, command identifiers and the production owned-copy helper. These pass 905 codec assertions and owned-copy cases with sanitizers; hardware qualification of the operational transport remains pending. [ryan: split by topic: only the drm/apple part and its selftests are here; the MAINTAINERS lines go to 900-config-tooling] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 355 +++++++++++++++++- drivers/gpu/drm/apple/afk.h | 6 + drivers/gpu/drm/apple/afkv2-codec.h | 270 +++++++++++++ drivers/gpu/drm/apple/dcp-internal.h | 2 - drivers/gpu/drm/apple/dcp.c | 10 +- drivers/gpu/drm/apple/dptxep.c | 81 ++-- .../drivers/gpu/apple-afkv2/Makefile | 9 + .../selftests/drivers/gpu/apple-afkv2/README | 24 ++ .../selftests/drivers/gpu/apple-afkv2/codec.c | 183 +++++++++ .../drivers/gpu/apple-afkv2/extract-owned.py | 28 ++ .../apple-afkv2/host-include/linux/errno.h | 5 + .../apple-afkv2/host-include/linux/string.h | 2 + .../apple-afkv2/host-include/linux/types.h | 8 + .../drivers/gpu/apple-afkv2/owned-prefix.c | 50 +++ .../drivers/gpu/apple-afkv2/owned-suffix.c | 56 +++ .../gpu/apple-afkv2/run-afkv2-tests.sh | 40 ++ 16 files changed, 1049 insertions(+), 80 deletions(-) create mode 100644 drivers/gpu/drm/apple/afkv2-codec.h create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/README create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/extract-owned.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/string.h create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/types.h create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/owned-prefix.c create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/owned-suffix.c create mode 100755 tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 546774c00f45c..e7c76e2d8a610 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -14,6 +14,7 @@ #include #include "afk.h" +#include "afkv2-codec.h" #include "trace.h" struct afk_receive_message_work { @@ -305,6 +306,10 @@ static struct apple_epic_service *afk_epic_find_service(struct apple_dcp_afkep * return NULL; } +static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, + enum epic_type etype, enum epic_category ecat, u8 stype, + const void *payload, size_t payload_len, u64 token); + static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, u8 *payload, size_t payload_size) { @@ -383,6 +388,22 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, goto free; } ep->num_channels = max(ep->num_channels, ch_idx + 1); + if (ep->wire_v2) { + int ret = afk_send_epic(ep, channel, 0, EPIC_TYPE_NOTIFY, + EPIC_CAT_REPORT, AFKV2_OPEN, NULL, 0); + + if (ret) { + unsigned long flags; + + spin_lock_irqsave(&service->lock, flags); + service->enabled = false; + service->torndown = true; + spin_unlock_irqrestore(&service->lock, flags); + dev_err(ep->dcp->dev, "AFKV2 interface %u OPEN failed: %d\n", + channel, ret); + goto free; + } + } ops->init(service, epic_name, epic_class, epic_unit); dev_info(ep->dcp->dev, "AFK[ep:%02x]: new service %s on channel %d\n", ep->endpoint, service_name, channel); @@ -430,7 +451,7 @@ static void afk_recv_handle_reply(struct apple_dcp_afkep *ep, u32 channel, struct epic_cmd *cmd = payload; struct apple_epic_service *service; unsigned long flags; - u8 idx = tag & 0xff; + u8 idx = tag & (ep->wire_v2 ? 0xf : 0xff); void *rxbuf, *txbuf; dma_addr_t rxbuf_dma, txbuf_dma; size_t rxlen, txlen; @@ -592,6 +613,233 @@ static bool afk_validate_h17p_header(const u8 *data, size_t size) get_unaligned_le32(data + 4) == size - 8; } +static void afk_v2_status(struct apple_dcp_afkep *ep, + const struct afkv2_frame *frame, + const struct afkv2_command *command, u32 status) +{ + u8 record[AFKV2_HEADER_SIZE + AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE]; + size_t size; + int ret; + + ret = afkv2_encode_error_reply(record, sizeof(record), 0, frame, command, + status, &size); + if (!ret) + ret = afk_send_epic_token(ep, frame->interface, command->id, + EPIC_TYPE_REPLY, EPIC_CAT_REPLY, frame->type, + record + AFKV2_HEADER_SIZE, + size - AFKV2_HEADER_SIZE, frame->token); + if (ret) + dev_warn(ep->dcp->dev, "AFKV2 error reply failed: %d\n", ret); +} + +static int afk_v2_access_owned(struct apple_epic_service *service, u64 dva, + void *buffer, size_t size, bool to_peer, bool copy) +{ + unsigned long flags; + unsigned int i; + int ret = -EACCES; + + if (!service || !service->ep->wire_v2 || size > SZ_1M || (copy && size && !buffer)) + return -EINVAL; + if (!size) + return 0; + spin_lock_irqsave(&service->lock, flags); + for (i = 0; i < MAX_PENDING_CMDS; i++) { + struct epic_cmd_info *cmd = &service->cmds[i]; + void *owned = NULL; + u64 offset; + + if (!test_bit(i, service->cmd_map) || cmd->done) + continue; + if (cmd->rxbuf && afkv2_owned_span(cmd->rxbuf_dma, cmd->rxlen, dva, size)) { + offset = dva - cmd->rxbuf_dma; + owned = cmd->rxbuf + offset; + } else if (cmd->txbuf && afkv2_owned_span(cmd->txbuf_dma, cmd->txlen, dva, size)) { + offset = dva - cmd->txbuf_dma; + owned = cmd->txbuf + offset; + } + if (!owned) + continue; + if (!copy) { + ret = 0; + break; + } + if (to_peer) { + memcpy(owned, buffer, size); + dma_wmb(); + } else { + dma_rmb(); + memcpy(buffer, owned, size); + } + ret = 0; + break; + } + spin_unlock_irqrestore(&service->lock, flags); + if (ret) + dev_warn_ratelimited(service->ep->dcp->dev, + "AFKV2 unowned buffer: ep=%u interface=%u size=%zu write=%u\n", + service->ep->endpoint, service->channel, size, to_peer); + return ret; +} + +int afk_v2_copy_owned(struct apple_epic_service *service, u64 dva, + void *buffer, size_t size, bool to_peer) +{ + return afk_v2_access_owned(service, dva, buffer, size, to_peer, true); +} + +static void afk_v2_callback(struct apple_dcp_afkep *ep, + const struct afkv2_frame *frame, + const struct afkv2_command *command) +{ + struct apple_epic_service *service; + const struct epic_service_call *call; + const u8 *request = command->inline_data; + u8 *owned_request = NULL, *reply = NULL; + size_t request_size = command->oob ? command->tx_size : command->inline_size; + size_t body_size, reply_size; + u32 status = 0xe00002c7; + int ret; + + if (frame->type != EPIC_SUBTYPE_STD_SERVICE || + request_size < sizeof(*call) + sizeof(u32) || + request_size > SZ_1M - AFKV2_COMMAND_SIZE || + request_size > command->value || + (command->oob && request_size > command->rx_size)) + goto reject; + service = afk_service_get(afk_epic_find_service(ep, frame->interface)); + if (!service || READ_ONCE(service->torndown) || !service->ops->call) { + if (service) + afk_service_put(service); + goto reject; + } + if (command->oob) { + /* Reject an unowned response destination before any callback side effect. */ + ret = afk_v2_access_owned(service, command->rx_address, NULL, + request_size, true, false); + if (ret) + goto release; + owned_request = kmalloc(request_size, GFP_KERNEL); + if (!owned_request) { + status = 0xe00002bd; + goto release; + } + ret = afk_v2_copy_owned(service, command->tx_address, + owned_request, request_size, false); + if (ret) + goto release; + request = owned_request; + } + call = (const void *)request; + body_size = request_size - sizeof(*call); + if (le16_to_cpu(call->group) || + le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || + le32_to_cpu(call->data_len) != body_size) + goto release; + reply = kzalloc(AFKV2_COMMAND_SIZE + request_size, GFP_KERNEL); + if (!reply) { + status = 0xe00002bd; + goto release; + } + reply[1] = command->id; + memcpy(reply + AFKV2_COMMAND_SIZE, call, sizeof(*call)); + ret = service->ops->call(service, le32_to_cpu(call->command), + request + sizeof(*call), body_size, + reply + AFKV2_COMMAND_SIZE + sizeof(*call), body_size); + if (ret) + goto release; + reply_size = AFKV2_COMMAND_SIZE + request_size; + if (command->oob) { + ret = afk_v2_copy_owned(service, command->rx_address, + reply + AFKV2_COMMAND_SIZE, request_size, true); + if (ret) + goto release; + ret = afkv2_encode_oob_command(reply, reply_size, command->id, + command->rx_address, request_size, + command->tx_address, command->tx_size); + if (ret) + goto release; + afkv2_put32(reply + 4, 0); + reply_size = AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE; + } + ret = afk_send_epic_token(ep, frame->interface, command->id, EPIC_TYPE_REPLY, + EPIC_CAT_REPLY, frame->type, reply, reply_size, frame->token); + kfree(reply); + kfree(owned_request); + afk_service_put(service); + if (ret) + dev_warn(ep->dcp->dev, "AFKV2 callback reply failed: %d\n", ret); + return; +release: + kfree(reply); + kfree(owned_request); + afk_service_put(service); +reject: + afk_v2_status(ep, frame, command, status); +} + +static void afk_v2_receive(struct apple_dcp_afkep *ep, u32 channel, u32 type, + const void *data, size_t size) +{ + struct afkv2_frame frame; + struct afkv2_command response; + struct apple_epic_service *service; + struct epic_cmd legacy = {}; + unsigned long flags; + unsigned int slot; + int ret; + + if (channel || type || afkv2_decode_frame(data, size, false, &frame) || + !frame.interface) + return; + if (frame.category == AFKV2_REPORT) { + if (frame.type == EPIC_SUBTYPE_ANNOUNCE_H17P) + afk_recv_handle_init(ep, frame.interface, (void *)frame.payload, + frame.payload_size); + else if (frame.type == AFKV2_CLOSE && frame.interface_flags == 1) + afk_recv_handle_teardown(ep, frame.interface); + return; + } + if (afkv2_decode_command(frame.payload, frame.payload_size, &response)) + return; + if (frame.category == AFKV2_COMMAND) { + afk_v2_callback(ep, &frame, &response); + return; + } + service = afk_epic_find_service(ep, frame.interface); + if (!service) + return; + spin_lock_irqsave(&service->lock, flags); + if (bitmap_empty(service->cmd_map, MAX_PENDING_CMDS)) { + spin_unlock_irqrestore(&service->lock, flags); + return; + } + service->users++; + spin_unlock_irqrestore(&service->lock, flags); + slot = response.id & 0xf; + spin_lock_irqsave(&service->lock, flags); + ret = -EPROTO; + if (slot < MAX_PENDING_CMDS && test_bit(slot, service->cmd_map) && + !service->cmds[slot].done && response.id == service->cmds[slot].tag) + ret = afkv2_check_response(&frame, &response, service->channel, + service->cmds[slot].type, service->cmds[slot].tag, + service->cmds[slot].rxbuf_dma, + service->cmds[slot].rxlen, + service->cmds[slot].txbuf_dma, + service->cmds[slot].txlen); + if (!ret) + service->cmds[slot].reply_length = response.value ? 0 : response.rx_size; + spin_unlock_irqrestore(&service->lock, flags); + if (!ret) { + legacy.retcode = cpu_to_le32(response.value); + afk_recv_handle_reply(ep, frame.interface, response.id, + &legacy, sizeof(legacy)); + } else { + dev_warn(ep->dcp->dev, "AFKV2 reply rejected: %d\n", ret); + } + afk_service_put(service); +} + static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, u8 *data, size_t data_size) { @@ -605,6 +853,11 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, u8 *payload; size_t payload_size; + if (ep->wire_v2) { + afk_v2_receive(ep, channel, type, data, data_size); + return; + } + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P) { const struct epic_sub_hdr_h17p *c = (const struct epic_sub_hdr_h17p *)(data + sizeof(*ehdr)); @@ -895,9 +1148,9 @@ int afk_receive_message(struct apple_dcp_afkep *ep, u64 message) return 0; } -int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, - enum epic_type etype, enum epic_category ecat, u8 stype, - const void *payload, size_t payload_len) +static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, + enum epic_type etype, enum epic_category ecat, u8 stype, + const void *payload, size_t payload_len, u64 token) { u32 rptr, wptr; struct afk_qe *hdr, *hdr2; @@ -905,7 +1158,10 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, struct epic_sub_hdr *eshdr; unsigned long flags; size_t total_epic_size, total_size; - bool compact = ep->dcp->hw.iomfb_method_profile == + u8 v2_command[AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE]; + u8 v2_category = 0; + u16 v2_flags = 0; + bool compact = !ep->wire_v2 && ep->dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; int ret; @@ -919,7 +1175,7 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, * callers are optional (verbose firmware logging, EDID copy, audio, * DPTX, debugfs) or replies to calls only DPTX services make. */ - if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P && !compact) { + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P && !compact && !ep->wire_v2) { dev_dbg_once(ep->dcp->dev, "AFK[ep:%02x]: not sending EPIC message on H17P\n", ep->endpoint); @@ -937,14 +1193,44 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, (etype == EPIC_TYPE_REPLY || etype == EPIC_TYPE_NOTIFY_ACK))))) return -EOPNOTSUPP; + if (ep->wire_v2) { + if (!channel || channel > U16_MAX || tag > U8_MAX || payload_len > SZ_1M) + return -EINVAL; + if (ecat == EPIC_CAT_REPORT) { + if ((stype != AFKV2_OPEN && stype != AFKV2_CLOSE) || payload_len) + return -EOPNOTSUPP; + v2_flags = 1; + } else if (ecat == EPIC_CAT_COMMAND) { + const struct epic_cmd *cmd = payload; + + if (stype != EPIC_SUBTYPE_STD_SERVICE || !cmd || + payload_len != sizeof(*cmd)) + return -EINVAL; + ret = afkv2_encode_oob_command(v2_command, sizeof(v2_command), tag, + le64_to_cpu(cmd->rxbuf), + le32_to_cpu(cmd->rxlen), + le64_to_cpu(cmd->txbuf), + le32_to_cpu(cmd->txlen)); + if (ret) + return ret; + payload = v2_command; + payload_len = sizeof(v2_command); + v2_category = AFKV2_COMMAND; + } else if (ecat == EPIC_CAT_REPLY) { + v2_category = AFKV2_RESPONSE; + } else { + return -EOPNOTSUPP; + } + } + spin_lock_irqsave(&ep->lock, flags); if (ep->stopping) { ret = -ESHUTDOWN; goto out; } - if (compact && (!ep->txbfr.ready || !ep->txbfr.rptr || - !ep->txbfr.wptr || !ep->txbfr.buf)) { + if ((compact || ep->wire_v2) && + (!ep->txbfr.ready || !ep->txbfr.rptr || !ep->txbfr.wptr || !ep->txbfr.buf)) { ret = -ESHUTDOWN; goto out; } @@ -954,11 +1240,11 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, wptr = le32_to_cpu(*ep->txbfr.wptr); trace_afk_send_rwptr_pre(ep, rptr, wptr); total_epic_size = sizeof(*ehdr) + - (compact ? sizeof(struct epic_sub_hdr_h17p) : sizeof(*eshdr)) + + ((compact || ep->wire_v2) ? sizeof(struct epic_sub_hdr_h17p) : sizeof(*eshdr)) + payload_len; total_size = sizeof(*hdr) + total_epic_size; - if (compact) { + if (compact || ep->wire_v2) { u32 available, needed; if (!ep->txbfr.ready || !ep->txbfr.stride || @@ -1061,12 +1347,21 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, hdr->magic = cpu_to_le32(QE_MAGIC); hdr->size = cpu_to_le32(total_epic_size); - hdr->channel = cpu_to_le32(compact ? 0 : channel); - hdr->type = cpu_to_le32(etype); + hdr->channel = cpu_to_le32((compact || ep->wire_v2) ? 0 : channel); + hdr->type = cpu_to_le32(ep->wire_v2 ? 0 : etype); if (hdr2) memcpy(hdr2, hdr, sizeof(*hdr)); - if (compact) { + if (ep->wire_v2) { + size_t written; + + ret = afkv2_encode_frame(ep->txbfr.buf + wptr, total_epic_size, + ep->qe_seq++, channel, token, stype, v2_category, + v2_flags, payload, payload_len, &written); + if (ret) + goto out; + wptr += written; + } else if (compact) { u8 *wire = ep->txbfr.buf + wptr; /* Experimental symmetry with the published H17P receive layout. */ @@ -1101,8 +1396,10 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, wptr += sizeof(*eshdr); } - memcpy(ep->txbfr.buf + wptr, payload, payload_len); - wptr += payload_len; + if (!ep->wire_v2) { + memcpy(ep->txbfr.buf + wptr, payload, payload_len); + wptr += payload_len; + } wptr = afk_ring_advance(wptr, 0, ep->txbfr.bufsz, ep->txbfr.stride); trace_afk_send_rwptr_post(ep, rptr, wptr); @@ -1119,6 +1416,14 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, return ret; } +int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, + enum epic_type etype, enum epic_category ecat, u8 stype, + const void *payload, size_t payload_len) +{ + return afk_send_epic_token(ep, channel, tag, etype, ecat, stype, + payload, payload_len, 0); +} + int afk_send_command(struct apple_epic_service *service, u8 type, const void *payload, size_t payload_len, void *output, size_t output_len, u32 *retcode) @@ -1151,6 +1456,12 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, goto err_put_service; } + if (ep->wire_v2 && (payload_len > SZ_1M || output_len > SZ_1M || + (payload_len && !payload))) { + ret = -EINVAL; + goto err_put_service; + } + rxbuf = dma_alloc_coherent(ep->dcp->dev, output_len, &rxbuf_dma, GFP_KERNEL); if (!rxbuf) { @@ -1184,11 +1495,13 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, goto err_unlock; } - tag = (service->cmd_tag & 0xff) << 8; - tag |= idx & 0xff; + tag = ep->wire_v2 ? afkv2_command_id(service->cmd_tag, idx) : + ((service->cmd_tag & 0xff) << 8) | idx; service->cmd_tag++; service->cmds[idx].tag = tag; + service->cmds[idx].type = type; + service->cmds[idx].reply_length = 0; service->cmds[idx].rxbuf = rxbuf; service->cmds[idx].txbuf = txbuf; service->cmds[idx].rxbuf_dma = rxbuf_dma; @@ -1234,8 +1547,14 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, ret = 0; if (retcode) *retcode = service->cmds[idx].retcode; - if (output && output_len) + if (ep->wire_v2 && !service->cmds[idx].retcode && + service->cmds[idx].reply_length < output_len) + ret = -EMSGSIZE; + else if (output && output_len && (!ep->wire_v2 || !service->cmds[idx].retcode)) { + if (ep->wire_v2) + dma_rmb(); memcpy(output, rxbuf, output_len); + } err_free_cmd: spin_lock_irqsave(&service->lock, flags); diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 8eb797e12ef07..a4de9090681c2 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -25,6 +25,8 @@ struct apple_dcp_afkep; struct epic_cmd_info { u16 tag; + u8 type; + size_t reply_length; void *rxbuf; void *txbuf; @@ -295,6 +297,8 @@ struct apple_dcp_afkep { u8 experimental_rx_logs; u32 endpoint; + /* Set before afk_start; protocol selection is per endpoint. */ + bool wire_v2; struct workqueue_struct *wq; struct completion started; @@ -325,6 +329,8 @@ struct apple_dcp_afkep { struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, const struct apple_epic_service_ops *ops); +int afk_v2_copy_owned(struct apple_epic_service *service, u64 dva, + void *buffer, size_t size, bool to_peer); int afk_start(struct apple_dcp_afkep *ep); void afk_shutdown(struct apple_dcp_afkep *ep); void afk_quiesce(struct apple_dcp_afkep *ep); diff --git a/drivers/gpu/drm/apple/afkv2-codec.h b/drivers/gpu/drm/apple/afkv2-codec.h new file mode 100644 index 0000000000000..cdd5d5ad20b22 --- /dev/null +++ b/drivers/gpu/drm/apple/afkv2-codec.h @@ -0,0 +1,270 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Aurora Silicon */ +/* Wire layout: public Linux PR254 and m1n1 AFKV2 implementations. */ +#ifndef AURORA_AFKV2_CODEC_H +#define AURORA_AFKV2_CODEC_H + +#include +#include +#include + +#define AFKV2_HEADER_SIZE 24U +#define AFKV2_COMMAND_SIZE 8U +#define AFKV2_OOB_SIZE 24U +#define AFKV2_MAX_PAYLOAD 1048576U +#define AFKV2_REPORT 0U +#define AFKV2_COMMAND 1U +#define AFKV2_RESPONSE 2U +#define AFKV2_OPEN 0x12U +#define AFKV2_CLOSE 0x13U + +struct afkv2_frame { + u8 sequence; + u8 transport_flags; + u16 interface; + u64 token; + u8 type; + u8 category; + u16 interface_flags; + u32 reserved; + const u8 *payload; + size_t payload_size; + size_t padding_size; +}; + +struct afkv2_command { + u8 id; + u32 value; + bool oob; + u64 rx_address; + u64 tx_address; + u32 rx_size; + u32 tx_size; + const u8 *inline_data; + size_t inline_size; +}; + +static inline u16 afkv2_get16(const u8 *p) +{ + return (u16)p[0] | (u16)p[1] << 8; +} + +static inline u32 afkv2_get32(const u8 *p) +{ + return (u32)p[0] | (u32)p[1] << 8 | (u32)p[2] << 16 | (u32)p[3] << 24; +} + +static inline u64 afkv2_get64(const u8 *p) +{ + return (u64)afkv2_get32(p) | (u64)afkv2_get32(p + 4) << 32; +} + +static inline void afkv2_put16(u8 *p, u16 value) +{ + p[0] = value; + p[1] = value >> 8; +} + +static inline void afkv2_put32(u8 *p, u32 value) +{ + p[0] = value; + p[1] = value >> 8; + p[2] = value >> 16; + p[3] = value >> 24; +} + +static inline void afkv2_put64(u8 *p, u64 value) +{ + afkv2_put32(p, value); + afkv2_put32(p + 4, value >> 32); +} + +/* Decode metadata only. Never follow an address supplied by the peer. */ +static inline int afkv2_decode_frame(const void *record, size_t size, + bool allow_padding, struct afkv2_frame *out) +{ + const u8 *p = record; + u32 length; + struct afkv2_frame result = {0}; + + if (!record || !out || size < AFKV2_HEADER_SIZE) + return -EINVAL; + if (size > AFKV2_HEADER_SIZE + AFKV2_MAX_PAYLOAD) + return -E2BIG; + length = afkv2_get32(p + 4); + if (length < 16 || length - 16 > AFKV2_MAX_PAYLOAD || length > size - 8) + return -EINVAL; + if (!allow_padding && length != size - 8) + return -EINVAL; + if (p[17] > AFKV2_RESPONSE) + return -EPROTO; + result.sequence = p[0]; + result.transport_flags = p[1]; + result.interface = afkv2_get16(p + 2); + result.token = afkv2_get64(p + 8); + result.type = p[16]; + result.category = p[17]; + result.interface_flags = afkv2_get16(p + 18); + result.reserved = afkv2_get32(p + 20); + result.payload = p + AFKV2_HEADER_SIZE; + result.payload_size = length - 16; + result.padding_size = size - 8 - length; + *out = result; + return 0; +} + +static inline int afkv2_encode_frame(void *record, size_t capacity, u8 sequence, + u16 interface, u64 token, u8 type, u8 category, + u16 flags, const void *payload, size_t payload_size, + size_t *written) +{ + u8 *p = record; + + if (!record || !written || !interface || category > AFKV2_RESPONSE || + (payload_size && !payload)) + return -EINVAL; + if (payload_size > AFKV2_MAX_PAYLOAD) + return -E2BIG; + if (capacity < AFKV2_HEADER_SIZE || payload_size > capacity - AFKV2_HEADER_SIZE) + return -ENOSPC; + /* Move first so an in-buffer payload is safe even when it overlaps. */ + if (payload_size) + memmove(p + AFKV2_HEADER_SIZE, payload, payload_size); + memset(p, 0, AFKV2_HEADER_SIZE); + p[0] = sequence; + afkv2_put16(p + 2, interface); + afkv2_put32(p + 4, payload_size + 16); + afkv2_put64(p + 8, token); + p[16] = type; + p[17] = category; + afkv2_put16(p + 18, flags); + *written = AFKV2_HEADER_SIZE + payload_size; + return 0; +} + +static inline int afkv2_encode_open(void *record, size_t capacity, u8 sequence, + u16 interface, size_t *written) +{ + return afkv2_encode_frame(record, capacity, sequence, interface, 0, + AFKV2_OPEN, AFKV2_REPORT, 1, NULL, 0, written); +} + +static inline bool afkv2_valid_span(u64 address, u32 size) +{ + if (!size) + return !address; + return address && size <= AFKV2_MAX_PAYLOAD && + address <= ~(u64)0 - (size - 1); +} + +static inline int afkv2_encode_oob_command(void *payload, size_t capacity, + u8 id, u64 rx_address, u32 rx_size, + u64 tx_address, u32 tx_size) +{ + u8 *p = payload; + + if (!payload || !afkv2_valid_span(rx_address, rx_size) || + !afkv2_valid_span(tx_address, tx_size)) + return -EINVAL; + if (capacity < AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE) + return -ENOSPC; + memset(p, 0, AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE); + p[0] = 1; + p[1] = id; + afkv2_put32(p + 4, rx_size); + afkv2_put64(p + 8, rx_address); + afkv2_put64(p + 16, tx_address); + afkv2_put32(p + 24, rx_size); + afkv2_put32(p + 28, tx_size); + return 0; +} + +static inline int afkv2_decode_command(const void *payload, size_t size, + struct afkv2_command *out) +{ + const u8 *p = payload; + struct afkv2_command result = {0}; + size_t header = AFKV2_COMMAND_SIZE; + + if (!payload || !out || size < header || p[0] & ~1U || afkv2_get16(p + 2)) + return -EINVAL; + result.id = p[1]; + result.value = afkv2_get32(p + 4); + result.oob = p[0] & 1; + if (result.oob) { + header += AFKV2_OOB_SIZE; + if (size < header) + return -EINVAL; + result.rx_address = afkv2_get64(p + 8); + result.tx_address = afkv2_get64(p + 16); + result.rx_size = afkv2_get32(p + 24); + result.tx_size = afkv2_get32(p + 28); + /* A failed response may retain addresses with zero lengths. */ + if ((result.rx_size && !afkv2_valid_span(result.rx_address, result.rx_size)) || + (result.tx_size && !afkv2_valid_span(result.tx_address, result.tx_size))) + return -EINVAL; + } + result.inline_data = p + header; + result.inline_size = size - header; + if (result.inline_size > AFKV2_MAX_PAYLOAD) + return -E2BIG; + *out = result; + return 0; +} + +/* Correlate only; no buffer is read, written or freed by this helper. */ +static inline int afkv2_check_response(const struct afkv2_frame *frame, + const struct afkv2_command *response, + u16 interface, u8 type, u8 id, + u64 rx_address, u32 rx_capacity, + u64 tx_address, u32 tx_capacity) +{ + if (!frame || !response || frame->category != AFKV2_RESPONSE || + frame->interface != interface || frame->type != type || response->id != id) + return -EPROTO; + if (response->value) + return 0; + if (!response->oob || response->inline_size || + response->rx_address != rx_address || response->tx_address != tx_address || + response->rx_size > rx_capacity || response->tx_size > tx_capacity) + return -EPROTO; + return 0; +} + +/* Reject a peer command without touching its advertised DMA buffers. */ +static inline int afkv2_encode_error_reply(void *record, size_t capacity, u8 sequence, + const struct afkv2_frame *request, + const struct afkv2_command *command, + u32 status, size_t *written) +{ + u8 payload[AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE] = {0}; + size_t size = AFKV2_COMMAND_SIZE; + + if (!request || !command || request->category != AFKV2_COMMAND || !status) + return -EINVAL; + payload[0] = command->oob ? 1 : 0; + payload[1] = command->id; + afkv2_put32(payload + 4, status); + if (command->oob) { + afkv2_put64(payload + 8, command->rx_address); + afkv2_put64(payload + 16, command->tx_address); + size += AFKV2_OOB_SIZE; + } + return afkv2_encode_frame(record, capacity, sequence, request->interface, + request->token, request->type, AFKV2_RESPONSE, 0, + payload, size, written); +} + +static inline u8 afkv2_command_id(u8 generation, u8 slot) +{ + return ((generation & 15) << 4) | (slot & 15); +} + +static inline bool afkv2_owned_span(u64 base, size_t length, u64 address, size_t size) +{ + if (!length || base > ~(u64)0 - (length - 1)) + return false; + return address >= base && size <= length && address - base <= length - size; +} + +#endif diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 3db1a80c1087f..f1d64e225d438 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -357,8 +357,6 @@ struct apple_dcp { struct apple_dcp_afkep *dptxep; struct apple_dcp_afkep *dptx_controller_ep; struct completion dptx_controller_ready; - struct apple_dcp_afkep *dptx_v2_probe_ep; - struct completion dptx_v2_probe_ready; struct dptx_port dptxport[2]; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 7feafd092851b..198e44e98e86d 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1877,7 +1877,7 @@ static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) return dcp->dptxep; case DPTX_ENDPOINT_V2: return dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL ? dcp->dptx_v2_probe_ep : NULL; + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL ? dcp->dptxep : NULL; case DPTX_CONTROLLER_ENDPOINT: return dcp->dptx_controller_ep; default: @@ -1964,7 +1964,7 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) ep = dcp->dptxep; break; case DPTX_ENDPOINT_V2: - ep = dcp->dptx_v2_probe_ep; + ep = dcp->dptxep; break; case DPTX_CONTROLLER_ENDPOINT: ep = dcp->dptx_controller_ep; @@ -3621,7 +3621,6 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) afk_quiesce(dcp->avep); afk_quiesce(dcp->dptxep); afk_quiesce(dcp->dptx_controller_ep); - afk_quiesce(dcp->dptx_v2_probe_ep); afk_quiesce(dcp->ibootep); afk_quiesce(dcp->systemep); afk_quiesce(dcp->dcpavservep); @@ -3658,11 +3657,6 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) dcp->dptxep = NULL; } - if (!IS_ERR_OR_NULL(dcp->dptx_v2_probe_ep)) { - afk_shutdown(dcp->dptx_v2_probe_ep); - dcp->dptx_v2_probe_ep = NULL; - } - if (!IS_ERR_OR_NULL(dcp->dptx_controller_ep)) { afk_shutdown(dcp->dptx_controller_ep); dcp->dptx_controller_ep = NULL; diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index e00c4cbcea171..8d605beb7faeb 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -812,27 +812,40 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, static void dptxport_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { + struct apple_dcp *dcp = service->ep->dcp; + s64 port = unit; + if (!name || !class) + return; if (strcmp(name, "dcpdptx-port-epic")) return; if (strcmp(class, "AppleDCPDPTXRemotePort")) return; - trace_dptxport_init(service->ep->dcp, unit); + trace_dptxport_init(dcp, unit); + if (service->ep->wire_v2) { + /* The single external service reports its physical DCP unit. */ + if (dcp->hw.num_dptx_ports != 1 || unit != dcp->index) { + dev_err(dcp->dev, "Unexpected external DPTX unit %lld\n", unit); + return; + } + port = 0; + dev_info(dcp->dev, "DPTX physical unit %lld maps to logical port 0\n", unit); + } - switch (unit) { + switch (port) { case 0: case 1: - if (service->ep->dcp->dptxport[unit].enabled) { + if (service->ep->dcp->dptxport[port].enabled) { dev_err(service->ep->dcp->dev, "DPTXPort: unit %lld already exists\n", unit); return; } - service->ep->dcp->dptxport[unit].unit = unit; - service->ep->dcp->dptxport[unit].service = service; - service->ep->dcp->dptxport[unit].enabled = true; - service->cookie = (void *)&service->ep->dcp->dptxport[unit]; - complete(&service->ep->dcp->dptxport[unit].enable_completion); + service->ep->dcp->dptxport[port].unit = unit; + service->ep->dcp->dptxport[port].service = service; + service->ep->dcp->dptxport[port].enabled = true; + service->cookie = (void *)&service->ep->dcp->dptxport[port]; + complete(&service->ep->dcp->dptxport[port].enable_completion); break; default: dev_err(service->ep->dcp->dev, "DPTXPort: invalid unit %lld\n", @@ -884,61 +897,25 @@ int dptx_controller_init(struct apple_dcp *dcp) return 0; } -static void dptx_v2_probe_service(struct apple_epic_service *service, - const char *name, const char *class, s64 unit) -{ - dev_info(service->ep->dcp->dev, - "DPTX v2 discovery name=%s class=%s unit=%lld channel=%u\n", - name ?: "(none)", class ?: "(none)", unit, service->channel); - complete(&service->ep->dcp->dptx_v2_probe_ready); -} - -static const struct apple_epic_service_ops dptx_v2_probe_ops[] = { - { - .name = "AppleDCPDPTXRemotePort", - .init = dptx_v2_probe_service, - }, - {} -}; - -static int dptx_v2_discover(struct apple_dcp *dcp) -{ - struct apple_dcp_afkep *ep; - int ret; - - init_completion(&dcp->dptx_v2_probe_ready); - ep = afk_init(dcp, DPTX_ENDPOINT_V2, dptx_v2_probe_ops); - if (IS_ERR(ep)) - return PTR_ERR(ep); - dcp->dptx_v2_probe_ep = ep; - ret = afk_start(ep); - if (ret) - return ret; - if (!wait_for_completion_timeout(&dcp->dptx_v2_probe_ready, - msecs_to_jiffies(1000))) - return -ETIMEDOUT; - - /* Discovery does not enable port callbacks or transmit service commands. */ - return -EOPNOTSUPP; -} - int dptxep_init(struct apple_dcp *dcp) { + struct apple_dcp_afkep *ep; + bool wire_v2 = dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; int ret; u32 port; unsigned long timeout = msecs_to_jiffies(1000); - if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) - return dptx_v2_discover(dcp); - init_completion(&dcp->dptxport[0].enable_completion); init_completion(&dcp->dptxport[1].enable_completion); init_completion(&dcp->dptxport[0].linkcfg_completion); init_completion(&dcp->dptxport[1].linkcfg_completion); - dcp->dptxep = afk_init(dcp, DPTX_ENDPOINT, dptxep_ops); - if (IS_ERR(dcp->dptxep)) - return PTR_ERR(dcp->dptxep); + ep = afk_init(dcp, wire_v2 ? DPTX_ENDPOINT_V2 : DPTX_ENDPOINT, dptxep_ops); + if (IS_ERR(ep)) + return PTR_ERR(ep); + ep->wire_v2 = wire_v2; + dcp->dptxep = ep; ret = afk_start(dcp->dptxep); if (ret) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile b/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile new file mode 100644 index 0000000000000..65780f3d5003f --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +HOSTCC ?= cc +export HOSTCC +export AFKV2_SOURCE_ROOT := $(abspath ../../../../../..) + +TEST_PROGS := run-afkv2-tests.sh +TEST_FILES := codec.c owned-prefix.c owned-suffix.c extract-owned.py host-include + +include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/README b/tools/testing/selftests/drivers/gpu/apple-afkv2/README new file mode 100644 index 0000000000000..42eb152d716a2 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/README @@ -0,0 +1,24 @@ +AFKV2 host selftests +=================== + +Run ./run-afkv2-tests.sh from this directory, or use make run_tests in a +complete kernel source checkout. HOSTCC, CFLAGS and PYTHON select host tools +(standalone CC is also accepted). +AFKV2_SANITIZERS=1 additionally enables AddressSanitizer and UBSan. +AFKV2_SOURCE_ROOT can name the source checkout when running a copied test +package; the Makefile exports it automatically for separate output builds. + +The codec tests include the production header directly and compare invented, +independently specified golden bytes. They cover truncation, category ranges, +length/padding/overflow, unaligned input, OOB descriptor ownership, response +correlation, token preservation and command ID packing. + +The owned-copy test extracts the actual production function from afk.c, prints +its SHA256, and compiles it with synthetic command slots and CPU buffers. It +checks active/inactive/done slots, high device addresses, boundaries and copies. +Locks and DMA barriers are mocked: this test does not qualify concurrency, +DMA coherency, firmware, a graphics device or any hardware display behavior. + +All build products are temporary. No device node is opened and no GPU, firmware +or target machine is contacted. The extractor fails if production function +boundaries change, requiring review rather than silently testing a stale copy. diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c new file mode 100644 index 0000000000000..de49360bdabea --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Aurora Silicon */ +#undef NDEBUG +#include +#include +#include +#include +#include "afkv2-codec.h" + +static unsigned int assertions; +#define CHECK(x) \ + do { \ + assertions++; \ + assert(x); \ + } while (0) + +/* Invented fixtures with independently specified wire bytes; no device data. */ +static const u8 open_fixture[24] = { 0x7b, 0, 0x34, 0x12, 0x10, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0x12, 0, 1, 0, 0, 0, 0, 0 }; +static const u8 descriptor_fixture[32] = { + 1, 0xab, 0, 0, 0x40, 0, 0, 0, 0x88, 0x77, 0x66, + 0x55, 0x44, 0x33, 0x22, 0x11, 8, 7, 6, 5, 4, 3, + 2, 1, 0x40, 0, 0, 0, 0x10, 0, 0, 0 +}; + +int main(void) +{ + u8 data[128], command[32], unaligned[129], saved[128]; + struct afkv2_frame frame, reply; + struct afkv2_command parsed, response; + size_t written = 999; + int ret; + + memset(data, 0xa5, sizeof(data)); + CHECK(!afkv2_encode_open(data, sizeof(data), 0x7b, 0x1234, &written)); + CHECK(written == sizeof(open_fixture)); + CHECK(!memcmp(data, open_fixture, sizeof(open_fixture))); + CHECK(data[24] == 0xa5); + CHECK(!afkv2_decode_frame(data, written, false, &frame)); + CHECK(frame.interface == 0x1234 && frame.type == AFKV2_OPEN); + CHECK(frame.category == AFKV2_REPORT && frame.interface_flags == 1); + CHECK(frame.payload_size == 0 && frame.padding_size == 0); + memcpy(unaligned + 1, open_fixture, sizeof(open_fixture)); + CHECK(!afkv2_decode_frame(unaligned + 1, sizeof(open_fixture), false, + &frame)); + CHECK(afkv2_decode_frame(data, SIZE_MAX, true, &frame) == -E2BIG); + CHECK(afkv2_decode_frame(NULL, 24, false, &frame) == -EINVAL); + CHECK(afkv2_decode_frame(data, 24, false, NULL) == -EINVAL); + for (size_t i = 0; i < 24; i++) + CHECK(afkv2_decode_frame(data, i, false, &frame) == -EINVAL); + CHECK(afkv2_decode_frame(data, 32, false, &frame) == -EINVAL); + CHECK(!afkv2_decode_frame(data, 32, true, &frame) && + frame.padding_size == 8); + for (u32 i = 0; i < 16; i++) { + afkv2_put32(data + 4, i); + CHECK(afkv2_decode_frame(data, 24, false, &frame) == -EINVAL); + } + afkv2_put32(data + 4, 17); + CHECK(afkv2_decode_frame(data, 24, false, &frame) == -EINVAL); + afkv2_put32(data + 4, UINT32_MAX); + CHECK(afkv2_decode_frame(data, 24, false, &frame) == -EINVAL); + memcpy(data, open_fixture, sizeof(open_fixture)); + for (unsigned int category = 0; category < 256; category++) { + data[17] = category; + ret = afkv2_decode_frame(data, 24, false, &frame); + CHECK(category <= 2 ? ret == 0 : ret == -EPROTO); + } + CHECK(!afkv2_encode_oob_command(command, sizeof(command), 0xab, + 0x1122334455667788ULL, 64, + 0x0102030405060708ULL, 16)); + CHECK(!memcmp(command, descriptor_fixture, sizeof(command))); + CHECK(!afkv2_decode_command(command, sizeof(command), &parsed)); + CHECK(parsed.oob && parsed.id == 0xab && parsed.value == 64); + CHECK(parsed.rx_address == 0x1122334455667788ULL && + parsed.tx_size == 16); + CHECK(parsed.inline_size == 0); + for (size_t i = 0; i < sizeof(command); i++) + CHECK(afkv2_decode_command(command, i, &response) == -EINVAL); + command[0] = 2; + CHECK(afkv2_decode_command(command, sizeof(command), &response) == + -EINVAL); + memcpy(command, descriptor_fixture, sizeof(command)); + command[2] = 1; + CHECK(afkv2_decode_command(command, sizeof(command), &response) == + -EINVAL); + memcpy(command, descriptor_fixture, sizeof(command)); + CHECK(afkv2_encode_oob_command(command, 31, 0, 1, 1, 2, 1) == -ENOSPC); + CHECK(afkv2_encode_oob_command(command, 32, 0, UINT64_MAX, 2, 0, 0) == + -EINVAL); + CHECK(afkv2_encode_oob_command(command, 32, 0, 1, AFKV2_MAX_PAYLOAD + 1, + 0, 0) == -EINVAL); + CHECK(afkv2_encode_oob_command(command, 32, 0, 1, 0, 0, 0) == -EINVAL); + memcpy(command, descriptor_fixture, sizeof(command)); + CHECK(!afkv2_encode_frame(data, sizeof(data), 1, 0x1234, + 0xaabbccddeeff0011ULL, 0xc0, AFKV2_COMMAND, 0, + command, sizeof(command), &written)); + CHECK(written == 56 && afkv2_get32(data + 4) == 48); + CHECK(!afkv2_decode_frame(data, written, false, &frame)); + CHECK(frame.token == 0xaabbccddeeff0011ULL && + frame.category == AFKV2_COMMAND); + CHECK(!afkv2_encode_error_reply(unaligned, sizeof(unaligned), 2, &frame, + &parsed, 0xe00002c7U, &written)); + CHECK(!afkv2_decode_frame(unaligned, written, false, &reply)); + CHECK(reply.token == frame.token && reply.interface == frame.interface); + CHECK(reply.type == frame.type && reply.category == AFKV2_RESPONSE); + CHECK(!afkv2_decode_command(reply.payload, reply.payload_size, + &response)); + CHECK(response.value == 0xe00002c7U && response.rx_size == 0 && + response.tx_size == 0); + CHECK(response.rx_address == parsed.rx_address && + response.tx_address == parsed.tx_address); + CHECK(!afkv2_check_response(&reply, &response, 0x1234, 0xc0, 0xab, + parsed.rx_address, 64, parsed.tx_address, + 16)); + response.value = 0; + response.rx_size = 64; + response.tx_size = 16; + CHECK(!afkv2_check_response(&reply, &response, 0x1234, 0xc0, 0xab, + parsed.rx_address, 64, parsed.tx_address, + 16)); + response.rx_address++; + CHECK(afkv2_check_response(&reply, &response, 0x1234, 0xc0, 0xab, + parsed.rx_address, 64, parsed.tx_address, + 16) == -EPROTO); + response.rx_address--; + response.rx_size++; + CHECK(afkv2_check_response(&reply, &response, 0x1234, 0xc0, 0xab, + parsed.rx_address, 64, parsed.tx_address, + 16) == -EPROTO); + response.rx_size--; + CHECK(afkv2_check_response(&reply, &response, 0x1235, 0xc0, 0xab, + parsed.rx_address, 64, parsed.tx_address, + 16) == -EPROTO); + CHECK(afkv2_check_response(&reply, &response, 0x1234, 0xc1, 0xab, + parsed.rx_address, 64, parsed.tx_address, + 16) == -EPROTO); + CHECK(afkv2_check_response(&reply, &response, 0x1234, 0xc0, 0xaa, + parsed.rx_address, 64, parsed.tx_address, + 16) == -EPROTO); + memset(data, 0xa5, sizeof(data)); + memcpy(saved, data, sizeof(data)); + CHECK(afkv2_encode_open(data, 23, 0, 1, &written) == -ENOSPC); + CHECK(!memcmp(data, saved, sizeof(data))); + CHECK(afkv2_encode_open(data, sizeof(data), 0, 0, &written) == -EINVAL); + CHECK(!memcmp(data, saved, sizeof(data))); + CHECK(afkv2_encode_frame(data, sizeof(data), 0, 1, 0, 0, 0, 0, data, + SIZE_MAX, &written) == -E2BIG); + CHECK(!memcmp(data, saved, sizeof(data))); + u8 *large = malloc(AFKV2_HEADER_SIZE + AFKV2_MAX_PAYLOAD); + + CHECK(large); + memset(large, 0x5a, AFKV2_HEADER_SIZE + AFKV2_MAX_PAYLOAD); + CHECK(!afkv2_encode_frame(large, AFKV2_HEADER_SIZE + AFKV2_MAX_PAYLOAD, + 0, 1, 0, 0xc0, 1, 0, large, AFKV2_MAX_PAYLOAD, + &written)); + CHECK(!afkv2_decode_frame(large, written, false, &frame)); + CHECK(frame.payload_size == AFKV2_MAX_PAYLOAD && + frame.payload[0] == 0x5a); + free(large); + bool ids[256] = { 0 }; + + for (unsigned int generation = 0; generation < 16; generation++) { + for (unsigned int slot = 0; slot < 16; slot++) { + u8 id = afkv2_command_id(generation, slot); + + CHECK(!ids[id]); + ids[id] = true; + CHECK((id & 15) == slot); + } + } + CHECK(afkv2_command_id(16, 3) == afkv2_command_id(0, 3)); + CHECK(afkv2_owned_span(0x1000, 32, 0x1000, 32)); + CHECK(afkv2_owned_span(0x1000, 32, 0x101c, 4)); + CHECK(!afkv2_owned_span(0x1000, 32, 0x0fff, 1)); + CHECK(!afkv2_owned_span(0x1000, 32, 0x101d, 4)); + CHECK(!afkv2_owned_span(0x1000, 32, 0x1020, 1)); + CHECK(!afkv2_owned_span(0x1000, 0, 0x1000, 0)); + CHECK(afkv2_owned_span(UINT64_MAX - 15, 16, UINT64_MAX, 1)); + CHECK(!afkv2_owned_span(UINT64_MAX - 15, 17, UINT64_MAX, 1)); + printf("PASS %u assertions\n", assertions); + return 0; +} diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-owned.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-owned.py new file mode 100644 index 0000000000000..21fe63b6d3823 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-owned.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Compile the production ownership checker with host-only synchronization mocks.""" +import hashlib +from pathlib import Path +import sys + + +def main(): + source = Path(sys.argv[1]).read_text(encoding="utf-8") + output = Path(sys.argv[2]) + first = "static int afk_v2_access_owned(" + last = "static void afk_v2_callback(" + if source.count(first) != 1 or source.count(last) != 1: + raise SystemExit("FAIL: ownership function boundaries changed; review extractor") + start = source.index(first) + end = source.index(last, start) + body = source[start:end] + if "int afk_v2_copy_owned(" not in body: + raise SystemExit("FAIL: production ownership wrapper missing") + directory = Path(__file__).resolve().parent + output.write_text((directory / "owned-prefix.c").read_text() + body + + (directory / "owned-suffix.c").read_text(), encoding="utf-8") + print("Production ownership function SHA256:", hashlib.sha256(body.encode()).hexdigest()) + + +if __name__ == "__main__": + main() diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h new file mode 100644 index 0000000000000..a5c3a3f44a591 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#define EINVAL 22 +#define ENOSPC 28 +#define E2BIG 7 +#define EPROTO 71 diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/string.h b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/string.h new file mode 100644 index 0000000000000..e74214a66ed83 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/string.h @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/types.h b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/types.h new file mode 100644 index 0000000000000..4d490ef828ccd --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/types.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-prefix.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-prefix.c new file mode 100644 index 0000000000000..8ff88ec1ec5b0 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-prefix.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Aurora Silicon */ +#undef NDEBUG +#include +#include +#include "afkv2-codec.h" +#define MAX_PENDING_CMDS 16 +#define SZ_1M 1048576U +#define EACCES 13 +#define spin_lock_irqsave(lock, flags) \ + do { \ + (void)(lock); \ + (flags) = 0; \ + } while (0) +#define spin_unlock_irqrestore(lock, flags) \ + do { \ + (void)(lock); \ + (void)(flags); \ + } while (0) +#define dma_rmb() \ + do { \ + } while (0) +#define dma_wmb() \ + do { \ + } while (0) +#define dev_warn_ratelimited(...) \ + do { \ + } while (0) +static int test_bit(unsigned int bit, const unsigned long *map) +{ + return !!(*map & (1UL << bit)); +} + +struct epic_cmd_info { + bool done; + void *rxbuf, *txbuf; + u64 rxbuf_dma, txbuf_dma; + size_t rxlen, txlen; +}; + +struct apple_dcp_afkep { + bool wire_v2; +}; + +struct apple_epic_service { + int lock; + struct apple_dcp_afkep *ep; + unsigned long cmd_map[1]; + struct epic_cmd_info cmds[MAX_PENDING_CMDS]; +}; diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-suffix.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-suffix.c new file mode 100644 index 0000000000000..770b16bc2fa94 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-suffix.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Aurora Silicon */ +int main(void) +{ + u8 rx[16], tx[16], output[16]; + struct apple_dcp_afkep ep = { .wire_v2 = true }; + struct apple_epic_service service = { .ep = &ep }; + struct epic_cmd_info *cmd = &service.cmds[0]; + + memset(rx, 0x12, sizeof(rx)); + memset(tx, 0x34, sizeof(tx)); + memset(output, 0x56, sizeof(output)); + cmd->rxbuf = rx; + cmd->rxbuf_dma = 0x10000001000ULL; + cmd->rxlen = sizeof(rx); + cmd->txbuf = tx; + cmd->txbuf_dma = 0x10000002000ULL; + cmd->txlen = sizeof(tx); + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma, output, 4, false) == + -EACCES); + service.cmd_map[0] = 1; + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma + 12, output, 4, + false) == 0); + assert(output[0] == 0x12 && output[3] == 0x12 && output[4] == 0x56); + assert(afk_v2_copy_owned(&service, cmd->txbuf_dma + 4, output, 4, + false) == 0); + assert(output[0] == 0x34 && output[4] == 0x56); + memset(output, 0x78, 4); + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma + 8, output, 4, + true) == 0); + assert(rx[7] == 0x12 && rx[8] == 0x78 && rx[11] == 0x78 && + rx[12] == 0x12); + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma - 1, output, 1, + true) == -EACCES); + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma + 13, output, 4, + true) == -EACCES); + assert(afk_v2_copy_owned(&service, cmd->txbuf_dma + 16, output, 1, + true) == -EACCES); + cmd->done = true; + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma, output, 1, false) == + -EACCES); + cmd->done = false; + ep.wire_v2 = false; + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma, output, 1, false) == + -EINVAL); + ep.wire_v2 = true; + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma, NULL, 1, false) == + -EINVAL); + assert(afk_v2_copy_owned(&service, 0, NULL, 0, false) == 0); + assert(afk_v2_access_owned(&service, cmd->rxbuf_dma + 1, NULL, 15, true, + false) == 0); + assert(afk_v2_access_owned(&service, cmd->rxbuf_dma + 1, NULL, 16, true, + false) == -EACCES); + puts("PASS owned-copy boundaries (actual extracted function; lock/barrier mocks)"); + return 0; +} diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh b/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh new file mode 100755 index 0000000000000..4e13f6d05d986 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu + +afkv2_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +afkv2_root=${AFKV2_SOURCE_ROOT:-$(CDPATH= cd -- "$afkv2_dir/../../../../../.." && pwd)} +afkv2_cc=${HOSTCC:-${CC:-cc}} +afkv2_cc_command=${afkv2_cc%% *} +afkv2_python=${PYTHON:-python3} +if ! command -v "$afkv2_cc_command" >/dev/null 2>&1 || + ! command -v "$afkv2_python" >/dev/null 2>&1; then + echo "SKIP: a host C compiler and Python 3 are required" + exit 4 +fi +if [ ! -f "$afkv2_root/drivers/gpu/drm/apple/afkv2-codec.h" ]; then + echo "SKIP: run from a source tree containing the AFKV2 codec" + exit 4 +fi +afkv2_tmp=$(mktemp -d "${TMPDIR:-/tmp}/apple-afkv2.XXXXXX") +trap 'rm -rf -- "$afkv2_tmp"' EXIT +trap 'exit 1' HUP INT TERM +afkv2_sanitizers= +if [ "${AFKV2_SANITIZERS:-0}" = 1 ]; then + afkv2_sanitizers="-fsanitize=address,undefined -fno-omit-frame-pointer" +fi + +# Compiler arguments, CFLAGS and sanitizers use normal compiler word splitting. +# shellcheck disable=SC2086 +$afkv2_cc -std=c11 -Wall -Wextra -Werror -pedantic ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_dir/codec.c" -o "$afkv2_tmp/codec" +"$afkv2_tmp/codec" +"$afkv2_python" "$afkv2_dir/extract-owned.py" \ + "$afkv2_root/drivers/gpu/drm/apple/afk.c" "$afkv2_tmp/owned.c" +# The extracted kernel function uses GNU C void-pointer arithmetic. +# shellcheck disable=SC2086 +$afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_tmp/owned.c" -o "$afkv2_tmp/owned" +"$afkv2_tmp/owned" From 70a42da00b3e1b803052a2d104129d9bb22fc0aa Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:54:12 +1100 Subject: [PATCH 1179/2680] drm/apple: open the version-two DPTX controller interfaces The experimental external profile uses AFKV2 for its DPTX port, but the companion controller endpoint still uses the legacy path. Its interfaces are discovered without receiving the version-two OPEN handshake. Select AFKV2 for the controller before starting it, so controller publication performs the same OPEN-before-ready sequence as the port. Add a one-time PHY setup message to distinguish successful calibration application from controller activation during hardware qualification. [ryan: split by topic: only the drm/apple part and its selftests are here; the PHY, Type-C, Thunderbolt and crossbar hunks are in 170-usb] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dptxep.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 8d605beb7faeb..856a9a75a389d 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -885,6 +885,9 @@ int dptx_controller_init(struct apple_dcp *dcp) ep = afk_init(dcp, DPTX_CONTROLLER_ENDPOINT, dptx_controller_ops); if (IS_ERR(ep)) return PTR_ERR(ep); + /* Controller interfaces require the same OPEN handshake as the port. */ + ep->wire_v2 = dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; dcp->dptx_controller_ep = ep; ret = afk_start(ep); From fd24edd706377bc626395e73a6c446302a1d0da0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:34:02 +1100 Subject: [PATCH 1180/2680] phy: apple: implement bounded J700 DisplayPort main-link training After correcting the crossbar route, J700 reaches SET_LINK_RATE but the legacy PHY path waits for the wrong command-ready register. Add an isolated T8140 main-link sequence, firmware drive presets, and cleanup ownership that survives failed setup or shutdown. Preserve the other PMA used for USB. Handle the version-two group-one drive command with complete bounds and range checks before changing the PHY. Cache training values only after a successful PHY operation and retain the legacy endpoint behavior. Hardware register facts were audited with explicit user authorization. The implementation and tests are independently written. The copy audit matched only three hardware-field expressions and standard C test headers. No private implementation patch was applied. Validation: 29 PHY state scenarios, 112 callback boundary cases, 100 group routing cases, 905 AFK assertions and production owned-buffer/AUX tests under ASan/UBSan; schema validation and strict checkpatch. Hardware qualification remains pending. [ryan: split by topic: only the drm/apple part and its selftests are here; the MAINTAINERS lines go to 900-config-tooling; the PHY, Type-C, Thunderbolt and crossbar hunks are in 170-usb; the tools/diagnostics scripts are dropped (aurora-wip deleted them in commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"))] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 17 +++++-- drivers/gpu/drm/apple/afk.h | 2 + drivers/gpu/drm/apple/dptx-j700-training.h | 54 ++++++++++++++++++++++ drivers/gpu/drm/apple/dptxep.c | 30 +++++++++++- drivers/gpu/drm/apple/dptxep.h | 1 + 5 files changed, 98 insertions(+), 6 deletions(-) create mode 100644 drivers/gpu/drm/apple/dptx-j700-training.h diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index e7c76e2d8a610..b7c3905303eab 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -708,7 +708,8 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, (command->oob && request_size > command->rx_size)) goto reject; service = afk_service_get(afk_epic_find_service(ep, frame->interface)); - if (!service || READ_ONCE(service->torndown) || !service->ops->call) { + if (!service || READ_ONCE(service->torndown) || + (!service->ops->call && !service->ops->call_v2)) { if (service) afk_service_put(service); goto reject; @@ -732,7 +733,7 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, } call = (const void *)request; body_size = request_size - sizeof(*call); - if (le16_to_cpu(call->group) || + if ((le16_to_cpu(call->group) && !service->ops->call_v2) || le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || le32_to_cpu(call->data_len) != body_size) goto release; @@ -743,9 +744,15 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, } reply[1] = command->id; memcpy(reply + AFKV2_COMMAND_SIZE, call, sizeof(*call)); - ret = service->ops->call(service, le32_to_cpu(call->command), - request + sizeof(*call), body_size, - reply + AFKV2_COMMAND_SIZE + sizeof(*call), body_size); + if (service->ops->call_v2) + ret = service->ops->call_v2(service, le16_to_cpu(call->group), + le32_to_cpu(call->command), + request + sizeof(*call), body_size, + reply + AFKV2_COMMAND_SIZE + sizeof(*call), body_size); + else + ret = service->ops->call(service, le32_to_cpu(call->command), + request + sizeof(*call), body_size, + reply + AFKV2_COMMAND_SIZE + sizeof(*call), body_size); if (ret) goto release; reply_size = AFKV2_COMMAND_SIZE + request_size; diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index a4de9090681c2..3d1f975d798a8 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -76,6 +76,8 @@ struct apple_epic_service_ops { int (*call)(struct apple_epic_service *service, u32 idx, const void *data, size_t data_size, void *reply, size_t reply_size); + int (*call_v2)(struct apple_epic_service *service, u16 group, u32 command, + const void *data, size_t size, void *reply, size_t room); int (*report)(struct apple_epic_service *service, enum epic_subtype type, const void *data, size_t data_size); void (*teardown)(struct apple_epic_service *service); diff --git a/drivers/gpu/drm/apple/dptx-j700-training.h b/drivers/gpu/drm/apple/dptx-j700-training.h new file mode 100644 index 0000000000000..09a5d1018a7fb --- /dev/null +++ b/drivers/gpu/drm/apple/dptx-j700-training.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Aurora Silicon */ +#ifndef APPLE_DPTX_J700_TRAINING_H +#define APPLE_DPTX_J700_TRAINING_H + +static int dptx_j700_training(struct apple_epic_service *service, bool write, + const void *data, size_t size, void *reply, size_t room) +{ + struct dptx_port *port = service->cookie; + union phy_configure_opts options = {}; + u32 count; + int ret; + + if (!port || !port->atcphy) + return -ENODEV; + if (size < 32 || room < 32) + return -EINVAL; + count = get_unaligned_le32(data + (write ? 16 : 0)); + if (count > 4 || (write && (!count || count == 3)) || room < 32 + 12 * count || + (write && size < 32 + 12 * count)) + return -EINVAL; + if (write) { + ret = phy_validate(port->atcphy, PHY_MODE_DP, 0, &options); + if (ret) + return ret; + if (count > options.dp.lanes) + return -EINVAL; + for (unsigned int i = 0; i < count; i++) { + const u8 *entry = data + 32 + 12 * i; + + if (get_unaligned_le32(entry) || get_unaligned_le32(entry + 4) > 3 || + get_unaligned_le32(entry + 8) > 3) + return -EINVAL; + options.dp.voltage[i] = get_unaligned_le32(entry + 4); + options.dp.pre[i] = get_unaligned_le32(entry + 8); + } + options.dp.lanes = count; + options.dp.set_voltages = true; + ret = phy_configure(port->atcphy, &options); + if (ret) + return ret; + memcpy(port->trained_voltage, options.dp.voltage, sizeof(port->trained_voltage)); + memcpy(port->trained_pre, options.dp.pre, sizeof(port->trained_pre)); + } + memset(reply, 0, room); + put_unaligned_le32(count, reply + (write ? 16 : 0)); + for (unsigned int i = 0; i < count; i++) { + put_unaligned_le32(port->trained_voltage[i], reply + 36 + 12 * i); + put_unaligned_le32(port->trained_pre[i], reply + 40 + 12 * i); + } + return 0; +} + +#endif diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 856a9a75a389d..d2ab05ef9e57a 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -17,6 +18,8 @@ #include "parser.h" #include "trace.h" +#include "dptx-j700-training.h" + struct dcpdptx_connection_cmd { __le32 unk; __le32 target; @@ -264,6 +267,13 @@ dptxport_call_get_max_drive_settings(struct apple_epic_service *service, return -EINVAL; reply->retcode = cpu_to_le32(0); + if (service->ep->wire_v2) { + memset(reply, 0, reply_size); + put_unaligned_le32(3, reply_ + 20); + put_unaligned_le32(3, reply_ + 24); + return 0; + } + reply->max_drive_settings[0] = cpu_to_le32(0x3); reply->max_drive_settings[1] = cpu_to_le32(0x3); @@ -405,6 +415,9 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic u64 lane_count = le64_to_cpu(request->lane_count); + if (service->ep->wire_v2 && lane_count > dptx->lane_count) + return -EINVAL; + if (dptx->lane_count < lane_count) dev_err(dcp->dev, "set_active_lane_count: unexpected lane " "count:%llu phy: %d\n", lane_count, dptx->lane_count); @@ -415,7 +428,7 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic dptx->phy_ops.dp.lanes = lane_count; /* USB4 DP IN has no ATC PHY; still accept the lane count. */ dptx->phy_ops.dp.set_lanes = - dcp_is_usb4_output(dcp) || (dcp->dptx_phy > 3); + service->ep->wire_v2 || dcp_is_usb4_output(dcp) || (dcp->dptx_phy > 3); break; default: dev_err(dcp->dev, "set_active_lane_count: invalid lane count:%llu\n", lane_count); @@ -853,11 +866,26 @@ static void dptxport_init(struct apple_epic_service *service, const char *name, } } +static int dptxport_group_call(struct apple_epic_service *service, u16 group, u32 command, + const void *data, size_t size, void *reply, size_t room) +{ + if (group == 1 && command == DPTX_APCALL_SET_DRIVE_SETTINGS) + return dptx_j700_training(service, true, data, size, reply, room); + if (group) + return -EOPNOTSUPP; + if (command == DPTX_APCALL_SET_DRIVE_SETTINGS) + return -EOPNOTSUPP; + if (command == DPTX_APCALL_GET_DRIVE_SETTINGS) + return dptx_j700_training(service, false, data, size, reply, room); + return dptxport_call(service, command, data, size, reply, room); +} + static const struct apple_epic_service_ops dptxep_ops[] = { { .name = "AppleDCPDPTXRemotePort", .init = dptxport_init, .call = dptxport_call, + .call_v2 = dptxport_group_call, }, {} }; diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index 885e3675f9db5..3eb71cf00f9ba 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -59,6 +59,7 @@ struct dptx_port { u32 lane_count; u32 link_rate, pending_link_rate; u32 drive_settings[2]; + u32 trained_voltage[4], trained_pre[4]; /* * Per-boot invocation counters for the validate/connect/request/ * release AFK calls, used together with the caller return address From 930dfb66955811497aa133605dad16de08755efd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:36:12 +1100 Subject: [PATCH 1181/2680] drm: apple: honor separate AFK V2 callback reply capacity A drive-settings read has a short request and a larger response. Validate both owned-buffer lengths before dispatch, allocate the response capacity, and report its body length without extending the input read. Keep the full response range ownership check before any callback side effect. The 917 codec assertions now include unequal request/response lengths, undersized buffers, limits, and invalid output pointers. Production owned copy tests and all J700 PHY/training/group tests pass under sanitizers. Strict checkpatch and the opaque copy audit are clean. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 27 ++++++++++--------- drivers/gpu/drm/apple/afkv2-codec.h | 19 +++++++++++++ .../selftests/drivers/gpu/apple-afkv2/codec.c | 24 +++++++++++++++++ 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index b7c3905303eab..8bcdb8ec24033 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -696,16 +696,15 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, const struct epic_service_call *call; const u8 *request = command->inline_data; u8 *owned_request = NULL, *reply = NULL; - size_t request_size = command->oob ? command->tx_size : command->inline_size; + size_t request_size, response_size, response_body; size_t body_size, reply_size; u32 status = 0xe00002c7; int ret; if (frame->type != EPIC_SUBTYPE_STD_SERVICE || - request_size < sizeof(*call) + sizeof(u32) || - request_size > SZ_1M - AFKV2_COMMAND_SIZE || - request_size > command->value || - (command->oob && request_size > command->rx_size)) + afkv2_callback_lengths(command, sizeof(*call) + sizeof(u32), + SZ_1M - AFKV2_COMMAND_SIZE, + &request_size, &response_size)) goto reject; service = afk_service_get(afk_epic_find_service(ep, frame->interface)); if (!service || READ_ONCE(service->torndown) || @@ -717,7 +716,7 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, if (command->oob) { /* Reject an unowned response destination before any callback side effect. */ ret = afk_v2_access_owned(service, command->rx_address, NULL, - request_size, true, false); + response_size, true, false); if (ret) goto release; owned_request = kmalloc(request_size, GFP_KERNEL); @@ -737,32 +736,36 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || le32_to_cpu(call->data_len) != body_size) goto release; - reply = kzalloc(AFKV2_COMMAND_SIZE + request_size, GFP_KERNEL); + response_body = response_size - sizeof(*call); + reply = kzalloc(AFKV2_COMMAND_SIZE + response_size, GFP_KERNEL); if (!reply) { status = 0xe00002bd; goto release; } reply[1] = command->id; memcpy(reply + AFKV2_COMMAND_SIZE, call, sizeof(*call)); + ((struct epic_service_call *)(reply + AFKV2_COMMAND_SIZE))->data_len = + cpu_to_le32(response_body); if (service->ops->call_v2) ret = service->ops->call_v2(service, le16_to_cpu(call->group), le32_to_cpu(call->command), request + sizeof(*call), body_size, - reply + AFKV2_COMMAND_SIZE + sizeof(*call), body_size); + reply + AFKV2_COMMAND_SIZE + sizeof(*call), + response_body); else ret = service->ops->call(service, le32_to_cpu(call->command), request + sizeof(*call), body_size, - reply + AFKV2_COMMAND_SIZE + sizeof(*call), body_size); + reply + AFKV2_COMMAND_SIZE + sizeof(*call), response_body); if (ret) goto release; - reply_size = AFKV2_COMMAND_SIZE + request_size; + reply_size = AFKV2_COMMAND_SIZE + response_size; if (command->oob) { ret = afk_v2_copy_owned(service, command->rx_address, - reply + AFKV2_COMMAND_SIZE, request_size, true); + reply + AFKV2_COMMAND_SIZE, response_size, true); if (ret) goto release; ret = afkv2_encode_oob_command(reply, reply_size, command->id, - command->rx_address, request_size, + command->rx_address, response_size, command->tx_address, command->tx_size); if (ret) goto release; diff --git a/drivers/gpu/drm/apple/afkv2-codec.h b/drivers/gpu/drm/apple/afkv2-codec.h index cdd5d5ad20b22..8990182275354 100644 --- a/drivers/gpu/drm/apple/afkv2-codec.h +++ b/drivers/gpu/drm/apple/afkv2-codec.h @@ -212,6 +212,25 @@ static inline int afkv2_decode_command(const void *payload, size_t size, return 0; } +/* Requests and responses may have different owned-buffer capacities. */ +static inline int afkv2_callback_lengths(const struct afkv2_command *command, + size_t minimum, size_t limit, + size_t *request, size_t *response) +{ + size_t in, out; + + if (!command || !request || !response || minimum > limit) + return -EINVAL; + in = command->oob ? command->tx_size : command->inline_size; + out = command->oob ? command->rx_size : command->inline_size; + if (in < minimum || out < minimum || in > limit || out > limit || + in > command->value || out > command->value) + return -EINVAL; + *request = in; + *response = out; + return 0; +} + /* Correlate only; no buffer is read, written or freed by this helper. */ static inline int afkv2_check_response(const struct afkv2_frame *frame, const struct afkv2_command *response, diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c index de49360bdabea..79598033c084a 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c @@ -178,6 +178,30 @@ int main(void) CHECK(!afkv2_owned_span(0x1000, 0, 0x1000, 0)); CHECK(afkv2_owned_span(UINT64_MAX - 15, 16, UINT64_MAX, 1)); CHECK(!afkv2_owned_span(UINT64_MAX - 15, 17, UINT64_MAX, 1)); + struct afkv2_command callback = { .oob = true, .tx_size = 96, + .rx_size = 144, .value = 144 }; + size_t input = 0, output = 0; + + CHECK(!afkv2_callback_lengths(&callback, 68, 1024, &input, &output)); + CHECK(input == 96 && output == 144); + callback.value = 143; + CHECK(afkv2_callback_lengths(&callback, 68, 1024, &input, &output)); + callback.value = 2048; + callback.rx_size = 2048; + CHECK(afkv2_callback_lengths(&callback, 68, 1024, &input, &output)); + callback.rx_size = 67; + CHECK(afkv2_callback_lengths(&callback, 68, 1024, &input, &output)); + callback.rx_size = 144; + callback.tx_size = 67; + CHECK(afkv2_callback_lengths(&callback, 68, 1024, &input, &output)); + callback.oob = false; + callback.inline_size = 96; + CHECK(!afkv2_callback_lengths(&callback, 68, 1024, &input, &output)); + CHECK(input == 96 && output == 96); + CHECK(afkv2_callback_lengths(NULL, 68, 1024, &input, &output)); + CHECK(afkv2_callback_lengths(&callback, 68, 1024, NULL, &output)); + CHECK(afkv2_callback_lengths(&callback, 68, 1024, &input, NULL)); + CHECK(afkv2_callback_lengths(&callback, 1025, 1024, &input, &output)); printf("PASS %u assertions\n", assertions); return 0; } From 1f0cbc19bf5a198f03bcacca0b7201eaa600e484 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:42:34 +1100 Subject: [PATCH 1182/2680] phy: apple: preserve J700 lane-routing state during DP setup Use the audited T8140 routing masks for USB3/DP and four-lane DP. The generic sequence changes the miscellaneous lane-swap control and clears routing state that this generation preserves. Update the low protocol fields in order and idle only the PMAs assigned to DisplayPort. Add opt-in callback and PHY completion diagnostics to distinguish firmware waiting from a blocked PHY operation. No calibration values are logged. The routing values are hardware facts from the user-authorized audit; this implementation is independently written. Copy audit clean. Host checks pass: 33 PHY routing/state cases, 112 training bounds, 100 group routing cases, 917 codec assertions, owned-copy and AUX transition tests. Strict checkpatch passes. External-display qualification remains pending. [ryan: split by topic: only the drm/apple part and its selftests are here; the PHY, Type-C, Thunderbolt and crossbar hunks are in 170-usb; the tools/diagnostics scripts are dropped (aurora-wip deleted them in commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"))] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 8bcdb8ec24033..38b44c2d6d785 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -701,6 +701,8 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, u32 status = 0xe00002c7; int ret; + dev_dbg(ep->dcp->dev, "AFKV2 callback sizes: tx=%u rx=%u value=%u inline=%zu\n", + command->tx_size, command->rx_size, command->value, command->inline_size); if (frame->type != EPIC_SUBTYPE_STD_SERVICE || afkv2_callback_lengths(command, sizeof(*call) + sizeof(u32), SZ_1M - AFKV2_COMMAND_SIZE, @@ -746,6 +748,8 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, memcpy(reply + AFKV2_COMMAND_SIZE, call, sizeof(*call)); ((struct epic_service_call *)(reply + AFKV2_COMMAND_SIZE))->data_len = cpu_to_le32(response_body); + dev_dbg(ep->dcp->dev, "AFKV2 callback begin: group=%u command=%u in=%zu out=%zu\n", + le16_to_cpu(call->group), le32_to_cpu(call->command), body_size, response_body); if (service->ops->call_v2) ret = service->ops->call_v2(service, le16_to_cpu(call->group), le32_to_cpu(call->command), @@ -756,6 +760,8 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, ret = service->ops->call(service, le32_to_cpu(call->command), request + sizeof(*call), body_size, reply + AFKV2_COMMAND_SIZE + sizeof(*call), response_body); + dev_dbg(ep->dcp->dev, "AFKV2 callback end: group=%u command=%u result=%d\n", + le16_to_cpu(call->group), le32_to_cpu(call->command), ret); if (ret) goto release; reply_size = AFKV2_COMMAND_SIZE + response_size; From dbee6469df4802932566da037bf516374589d056 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:50:55 +1100 Subject: [PATCH 1183/2680] drm: apple: validate J700 active lanes against PHY capacity Firmware resets the active lane count to zero before setting its link rate. Comparing the following two-lane request against that cached zero rejected a valid activation with EINVAL and prevented link completion. Query the PHY capacity for each request instead. Validate impossible counts before changing state, and propagate validation failures. Cover the zero-to-two reset sequence, unsupported counts and missing PHY. Validation: J700 PHY/training tests, 917 AFK assertions and owned-buffer checks pass under sanitizers; strict checkpatch and copy audit clean. [ryan: split by topic: only the drm/apple part and its selftests are here; the tools/diagnostics scripts are dropped (aurora-wip deleted them in commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"))] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dptx-j700-training.h | 16 ++++++++++++++++ drivers/gpu/drm/apple/dptxep.c | 7 +++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dptx-j700-training.h b/drivers/gpu/drm/apple/dptx-j700-training.h index 09a5d1018a7fb..a4632cd363abd 100644 --- a/drivers/gpu/drm/apple/dptx-j700-training.h +++ b/drivers/gpu/drm/apple/dptx-j700-training.h @@ -3,6 +3,22 @@ #ifndef APPLE_DPTX_J700_TRAINING_H #define APPLE_DPTX_J700_TRAINING_H +/* Active lanes may be zero during the firmware's training reset. */ +static int dptx_j700_validate_lanes(struct dptx_port *port, u64 requested) +{ + union phy_configure_opts capacity = {}; + int ret; + + if (!port || !port->atcphy) + return -ENODEV; + if (requested > 4 || requested == 3) + return -EINVAL; + ret = phy_validate(port->atcphy, PHY_MODE_DP, 0, &capacity); + if (ret) + return ret; + return requested <= capacity.dp.lanes ? 0 : -EINVAL; +} + static int dptx_j700_training(struct apple_epic_service *service, bool write, const void *data, size_t size, void *reply, size_t room) { diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index d2ab05ef9e57a..eaa3a3b26bf61 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -415,8 +415,11 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic u64 lane_count = le64_to_cpu(request->lane_count); - if (service->ep->wire_v2 && lane_count > dptx->lane_count) - return -EINVAL; + if (service->ep->wire_v2) { + ret = dptx_j700_validate_lanes(dptx, lane_count); + if (ret) + return ret; + } if (dptx->lane_count < lane_count) dev_err(dcp->dev, "set_active_lane_count: unexpected lane " From 20143385b775a4bbb5760eb9e7ef00d5c034ee5b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:52:49 +1100 Subject: [PATCH 1184/2680] drm: apple: avoid stale active-lane warnings after V2 validation The V2 request has already passed the physical-capacity check. Restrict the older cached-count warning to the legacy path so a valid zero-to-two transition does not emit an incorrect error. No state transition changes. Validation: J700 host regression tests, strict checkpatch and copy audit. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dptxep.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index eaa3a3b26bf61..4935539bd2920 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -419,9 +419,7 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic ret = dptx_j700_validate_lanes(dptx, lane_count); if (ret) return ret; - } - - if (dptx->lane_count < lane_count) + } else if (dptx->lane_count < lane_count) dev_err(dcp->dev, "set_active_lane_count: unexpected lane " "count:%llu phy: %d\n", lane_count, dptx->lane_count); From b1b82f30c78d6fcf54a6d403f60039e1cc7b5031 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:57:03 +1100 Subject: [PATCH 1185/2680] drm: apple: answer the V2 physical-device mode query The external firmware asks whether the connection is a virtual display. Return a bounded, zero-initialized physical-device answer instead of unsupported, matching the public m1n1 V2 DPTX protocol. Keep other unknown commands rejected and retain the callback group restrictions. Validation: query size/group checks, J700 training tests, 54 crossbar scenarios, 917 codec assertions and owned-buffer tests; checkpatch and copy audit pass. Hardware qualification remains pending. [ryan: split by topic: only the drm/apple part and its selftests are here; the tools/diagnostics scripts are dropped (aurora-wip deleted them in commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"))] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dptxep.c | 6 ++++++ drivers/gpu/drm/apple/dptxep.h | 1 + 2 files changed, 7 insertions(+) diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 4935539bd2920..bae72b7dc7dfb 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -876,6 +876,12 @@ static int dptxport_group_call(struct apple_epic_service *service, u16 group, u3 return -EOPNOTSUPP; if (command == DPTX_APCALL_SET_DRIVE_SETTINGS) return -EOPNOTSUPP; + if (command == DPTX_APCALL_GET_VIRTUAL_DEVICE_MODE) { + if (room < sizeof(struct dptxport_apcall_get_support)) + return -EINVAL; + memset(reply, 0, room); + return 0; + } if (command == DPTX_APCALL_GET_DRIVE_SETTINGS) return dptx_j700_training(service, false, data, size, reply, room); return dptxport_call(service, command, data, size, reply, room); diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index 3eb71cf00f9ba..14530c99567b1 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -30,6 +30,7 @@ enum dptx_apcall { DPTX_APCALL_DEVICE_NOT_RESPONDING = 22, DPTX_APCALL_DEVICE_BUSY_TIMEOUT = 23, DPTX_APCALL_DEVICE_NOT_STARTED = 24, + DPTX_APCALL_GET_VIRTUAL_DEVICE_MODE = 26, }; #define DCPDPTX_REMOTE_PORT_CORE GENMASK(3, 0) From 1ea5a713af84dea1e9785b5c84578c1e916d0ada Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:12:35 +1100 Subject: [PATCH 1186/2680] drm: apple: allow the qualified J700 provider fallback on DCPEXT Missing power-LUT properties on the external controller took the provider failure path and set dcp->crashed before display modes could be published. The existing J700 compatibility payloads were accepted only on the internal controller. Allow those same validated payloads for the explicitly experimental J700 external profile. Keep all request, capacity and property-length checks, and keep malformed properties and other machines/profiles fail-closed. The five existing public payloads match the audited working handoff; no new calibration data is added. Validation: actual provider-callback host tests cover the six known properties, internal compatibility, experimental external compatibility, other profiles/machines, malformed lengths and unknown requests under ASan/UBSan. The old implementation rejects the external regression case. Strict checkpatch and copy audit pass; hardware qualification is pending. [ryan: split by topic: only the drm/apple part and its selftests are here; the MAINTAINERS lines go to 900-config-tooling; the tools/diagnostics scripts are dropped (aurora-wip deleted them in commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"))] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 7e6dcb263d6d8..fd8cee0ccaf7d 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -446,7 +446,9 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, goto fail; } else if (!of_property_present(node, property->dt_name) && of_machine_is_compatible("apple,j700") && - of_device_is_compatible(node, "apple,t8140-dcp")) { + (of_device_is_compatible(node, "apple,t8140-dcp") || + (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + of_device_is_compatible(node, "apple,t8140-dcpext")))) { /* Qualified older loaders lack the newly described table properties. */ memset(reply, 0, sizeof(*reply)); memcpy(reply->data, property->legacy_data, property->length); From 85fd4c69f5d4994d550d4470a2fd232d7e04adbe Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:20:51 +1100 Subject: [PATCH 1187/2680] drm/apple: report unknown callback bounds for diagnosis Include validated input/output lengths and channel context in the existing warning. Retain the external fail-closed behavior while identifying the D134 callback observed after link setup. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 20c0ad6711347..2445639ff5aac 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -818,8 +818,9 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, if (!handled) { if (!iomfb_uses_queue(dcp) || !test_and_set_bit(tag, dcp->unknown_callbacks)) - dev_warn(dev, "received unknown callback %c%c%c%c\n", - hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); + dev_warn(dev, "received unknown callback %c%c%c%c (%u, %u) context=%u\n", + hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0], + hdr->in_len, hdr->out_len, context); if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { /* Retain the failed session without inventing a successful reply. */ From 75d4a128df3ae2b14873079b2138ef19d50c38a8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:25:59 +1100 Subject: [PATCH 1188/2680] drm/apple: answer the measured J700 external D134 record The external controller reaches video-interface publication, then sends D134 on the async channel with no input and a four-byte output. Its unanswered callback marks the strict external session crashed. Permit the existing zero reply only for that exact record on J700 T8140 DCPEXT. Retain fail-closed behavior for every other unknown external callback and preserve legacy handling. Test the production dispatcher under ASan/UBSan across 15552 tag, size, context and device/profile combinations, including output bounds and unchanged legacy replies. [ryan: split by topic: only the drm/apple part and its selftests are here; the tools/diagnostics scripts are dropped (aurora-wip deleted them in commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"))] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 2445639ff5aac..2894795fc8ad1 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -770,6 +770,23 @@ void dcp_hotplug(struct work_struct *work) drm_kms_helper_connector_hotplug_event(&connector->base); } +/* + * J700/25G83 asks D134 after publishing its external video interface. + * The observed async record has no input and a four-byte result. Use the + * existing zero reply for this record only; its feature remains unsupported. + */ +static bool dcp_external_zero_reply(struct apple_dcp *dcp, int tag, + enum dcp_context_id context, + u32 in_len, u32 out_len) +{ + return tag == 134 && context == DCP_CONTEXT_ASYNC && + in_len == 0 && out_len == sizeof(u32) && + dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + of_machine_is_compatible("apple,j700") && + of_device_is_compatible(dcp->dev->of_node, "apple,t8140-dcpext"); +} + static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, void *data, u32 length, u16 offset) { @@ -779,7 +796,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, int tag; struct dcp_channel *ch = dcp_get_channel(dcp, context); u8 depth; - bool handled; + bool handled, zero_reply; if (dcp->fw_compat == DCP_FIRMWARE_H17P && (length < sizeof(*hdr) || !ch || ch->depth >= DCP_MAX_CALL_DEPTH || @@ -816,13 +833,16 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, out = in + hdr->in_len; if (!handled) { - if (!iomfb_uses_queue(dcp) || - !test_and_set_bit(tag, dcp->unknown_callbacks)) + zero_reply = dcp_external_zero_reply(dcp, tag, context, + hdr->in_len, hdr->out_len); + if (!zero_reply && + (!iomfb_uses_queue(dcp) || + !test_and_set_bit(tag, dcp->unknown_callbacks))) dev_warn(dev, "received unknown callback %c%c%c%c (%u, %u) context=%u\n", hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0], hdr->in_len, hdr->out_len, context); if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && !zero_reply) { /* Retain the failed session without inventing a successful reply. */ dcp->crashed = true; complete(&dcp->start_done); From 38a814922f03005a4b21106d0ff8eae6dcadc0ec Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:30:46 +1100 Subject: [PATCH 1189/2680] drm/apple: route native training readback during modeset R24 reaches external mode export and a visible but frozen image. Firmware uses group 1 command 4 to read training settings during modeset; route it to the bounded native readback helper instead of rejecting it. Test returned cached lane values, no PHY writes and short-reply rejection. Include pending command and present-state information in the existing transaction timeout diagnostic. [ryan: split by topic: only the drm/apple part and its selftests are here; the tools/diagnostics scripts are dropped (aurora-wip deleted them in commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"))] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dptxep.c | 2 ++ drivers/gpu/drm/apple/iomfb.c | 14 +++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index bae72b7dc7dfb..e3061f5ef4067 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -872,6 +872,8 @@ static int dptxport_group_call(struct apple_epic_service *service, u16 group, u3 { if (group == 1 && command == DPTX_APCALL_SET_DRIVE_SETTINGS) return dptx_j700_training(service, true, data, size, reply, room); + if (group == 1 && command == DPTX_APCALL_GET_DRIVE_SETTINGS) + return dptx_j700_training(service, false, data, size, reply, room); if (group) return -EOPNOTSUPP; if (command == DPTX_APCALL_SET_DRIVE_SETTINGS) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 2894795fc8ad1..fb160068be939 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -312,7 +312,19 @@ static void iomfb_queue_timeout(struct work_struct *work) dcp->iomfb.deadline - jiffies); } else if (dcp->iomfb.active) { WRITE_ONCE(dcp->crashed, true); - dev_err(dcp->dev, "IOMFB transaction timed out\n"); + dev_err(dcp->dev, + "IOMFB transaction timed out: cmd=%u oob=%u present=%u accepted=%u swap=%u\n", + dcp->ch_cmd.depth, dcp->ch_oobcmd.depth, + dcp->present_state_h17p.pending, dcp->present_state_h17p.accepted, + dcp->present_state_h17p.swap_id); + if (dcp->ch_cmd.depth && dcp->ch_cmd.depth <= DCP_MAX_CALL_DEPTH) { + const struct dcp_packet_header *h = + &dcp->ch_cmd.header[dcp->ch_cmd.depth - 1]; + + dev_err(dcp->dev, "pending IOMFB %c%c%c%c in=%u out=%u\n", + h->tag[3], h->tag[2], h->tag[1], h->tag[0], + h->in_len, h->out_len); + } iomfb_discard_pending(dcp); schedule_work(&dcp->vblank_wq); } From 7adac73ebda75cd0910065596968513b2f834adc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:35:59 +1100 Subject: [PATCH 1190/2680] drm/apple: keep opaque X property off experimental DCPEXT The external controller rejects the post-first-present enableGPAlphaDiv property with status 4. Treating that panel workaround as required marks the external session crashed and freezes a visible image. Leave the experimental external controller on its native presentation path while preserving the existing internal-panel property queue. Cover 396 production-queue scope/state/capacity cases and allocation failure under ASan/UBSan. [ryan: split by topic: only the drm/apple part and its selftests are here; the tools/diagnostics scripts are dropped (aurora-wip deleted them in commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"))] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index fb160068be939..5034c3cc28b90 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -193,6 +193,10 @@ static int iomfb_enqueue_opaque_x(struct apple_dcp *dcp) struct iomfb_opaque_x_transaction *opaque; lockdep_assert_held(&dcp->iomfb.lock); + /* DCPEXT rejects enableGPAlphaDiv; retain the panel-only workaround. */ + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + return 0; if (atomic_read(&dcp->iomfb.opaque_x_state) != IOMFB_OPAQUE_X_NEEDED) return 0; if (dcp->iomfb.queued >= 32) From efe82156147c136ca805a9402c88aa3f3d875465 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:45:26 +1100 Subject: [PATCH 1191/2680] drm/apple: remove temporary external bring-up diagnostics Drop temporary protocol capture counters, PHY lock chatter, the global Type-C trace addition and unused link-rate override. Preserve standard tracepoints, failure diagnostics and the validated HBR3 behavior. Register the remaining focused tests and output-frame utility with MAINTAINERS. [ryan: without removing the J700 link-rate override: commit e8fe156f2d3d ("drm/apple: add scoped J700 link-rate diagnostic ceiling"), which added it, is not carried] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 17 +---------------- drivers/gpu/drm/apple/afk.h | 2 -- drivers/gpu/drm/apple/dcp-internal.h | 2 -- drivers/gpu/drm/apple/dptxep.c | 2 +- drivers/gpu/drm/apple/iomfb.c | 8 +------- 5 files changed, 3 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 38b44c2d6d785..b45695ddf9836 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -922,15 +922,6 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, payload_size = data_size - hdr_len; trace_afk_recv_handle(ep, channel, type, data_size, ehdr, eshdr); - if (ep->dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && - ep->experimental_rx_logs < 32) { - ep->experimental_rx_logs++; - dev_info(ep->dcp->dev, - "external AFK ep=%u channel=%u type=%u subtype=%u category=%u bytes=%zu\n", - ep->endpoint, channel, type, subtype, eshdr->category, - data_size); - } service = afk_epic_find_service(ep, channel); @@ -1092,13 +1083,7 @@ static void afk_receive_message_worker(struct work_struct *work_) work = container_of(work_, struct afk_receive_message_work, work); type = FIELD_GET(RBEP_TYPE, work->message); - if (work->ep->dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && - (work->ep->endpoint == DPTX_ENDPOINT || - work->ep->endpoint == DPTX_ENDPOINT_V2) && type != RBEP_RECV) - dev_info_ratelimited(work->ep->dcp->dev, - "external AFK control ep=%#x type=%#x\n", - work->ep->endpoint, type); + switch (type) { case RBEP_INIT_ACK: break; diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 3d1f975d798a8..c6a34fa67399e 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -295,8 +295,6 @@ struct afk_ringbuffer { struct apple_dcp_afkep { struct apple_dcp *dcp; - /* Bounded external protocol observations, owned by the RX worker. */ - u8 experimental_rx_logs; u32 endpoint; /* Set before afk_start; protocol selection is per endpoint. */ diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index f1d64e225d438..95fe377b69106 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -207,8 +207,6 @@ struct apple_dcp { struct apple_connector *connector; struct apple_dcp_hw_data hw; - /* Protected by the H17P IOMFB receive mutex. */ - u8 experimental_callback_logs; bool external_prepared; /* firmware version and compatible firmware version */ diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index e3061f5ef4067..6c4012d6ad752 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -844,7 +844,7 @@ static void dptxport_init(struct apple_epic_service *service, const char *name, return; } port = 0; - dev_info(dcp->dev, "DPTX physical unit %lld maps to logical port 0\n", unit); + dev_dbg(dcp->dev, "DPTX physical unit %lld maps to logical port 0\n", unit); } switch (port) { diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 5034c3cc28b90..1bc3eff4943fe 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -823,13 +823,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, } tag = dcp_parse_tag(hdr->tag); - if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && - dcp->experimental_callback_logs < 64) { - dcp->experimental_callback_logs++; - dev_info(dev, "external IOMFB callback=%d in=%u out=%u context=%u\n", - tag, hdr->in_len, hdr->out_len, context); - } + handled = tag >= 0 && tag < IOMFB_MAX_CB && dcp->cb_handlers && dcp->cb_handlers[tag]; From 776dd465751fc1f1b12036ddfdb41909f4cd5181 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:51:31 +1100 Subject: [PATCH 1192/2680] drm/apple: close external admission and recovery races Make explicitly enabled external providers participate in component matching before readiness, then validate prepared state after component binding. Reject unpublished or retired DPTX services and preserve stable ownership rather than reusing a published service generation. Retain failed J700 PHY transition state so SAFE retries cleanup, and reject normal training while that state is dirty. Add fault, admission/lifetime and provider-order regressions. A published DPTX service loss remains fail-closed until recovery; these changes do not claim physical hotplug qualification. [ryan: split by topic: only the drm/apple part and its selftests are here; the MAINTAINERS lines go to 900-config-tooling; the PHY, Type-C, Thunderbolt and crossbar hunks are in 170-usb; the tools/diagnostics scripts are dropped (aurora-wip deleted them in commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"))] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Makefile | 2 + drivers/gpu/drm/apple/afk-service-test.c | 99 ++++++++++++++++++++++++ drivers/gpu/drm/apple/afk.c | 27 +++++-- drivers/gpu/drm/apple/afk.h | 32 +++++++- drivers/gpu/drm/apple/apple_drv.c | 23 ++++-- drivers/gpu/drm/apple/dcp.c | 27 ++++--- drivers/gpu/drm/apple/dptx-service.h | 63 +++++++++++++++ drivers/gpu/drm/apple/dptxep.c | 55 +++++-------- 8 files changed, 266 insertions(+), 62 deletions(-) create mode 100644 drivers/gpu/drm/apple/dptx-service.h diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 44ead545d61c5..8b4a53838eefe 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -22,3 +22,5 @@ appledrm-$(CONFIG_TRACING) += trace.o obj-$(CONFIG_DRM_APPLE) += appledrm.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk_test.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_backlight_test.o + +obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk-service-test.o diff --git a/drivers/gpu/drm/apple/afk-service-test.c b/drivers/gpu/drm/apple/afk-service-test.c index 4a5f07fa9b15f..a86df389deb17 100644 --- a/drivers/gpu/drm/apple/afk-service-test.c +++ b/drivers/gpu/drm/apple/afk-service-test.c @@ -3,6 +3,7 @@ #include #include "afk.h" +#include "dptx-service.h" static const struct apple_epic_service_ops reusable_ops = { .name = "reusable", @@ -122,7 +123,105 @@ static void afk_service_unconverted_owner_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); } +static int afk_test_prepare(struct apple_epic_service *service, + const char *name, const char *class, s64 unit) +{ + return 0; +} + +static const struct apple_epic_service_ops prepared_ops = { + .name = "prepared", + .prepare = afk_test_prepare, + .teardown = dptx_service_unbind, +}; + +static void afk_service_admission_test(struct kunit *test) +{ + struct apple_epic_service *service = test->priv; + + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &prepared_ops, 1)); + /* Neither callback dispatch nor command callers can observe preparation. */ + KUNIT_EXPECT_FALSE(test, afk_service_matches(service, 1)); + KUNIT_EXPECT_PTR_EQ(test, afk_service_get(service), NULL); + /* A rejected announcement or failed OPEN consumes no service slot. */ + afk_service_discard(service); + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &prepared_ops, 2)); + /* Cookie-less services are valid: admission does not test cookie != NULL. */ + afk_service_publish(service); + KUNIT_EXPECT_TRUE(test, afk_service_matches(service, 2)); + KUNIT_ASSERT_PTR_EQ(test, afk_service_get(service), service); + afk_service_put(service); +} + +static void dptx_service_identity_test(struct kunit *test) +{ + const char *name = "dcpdptx-port-epic"; + const char *class = "AppleDCPDPTXRemotePort"; + + KUNIT_EXPECT_EQ(test, dptx_service_port(name, class, 3, true, 3, 1), 0); + KUNIT_EXPECT_EQ(test, dptx_service_port(name, class, 0, true, 3, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, dptx_service_port(name, class, -1, true, 3, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, dptx_service_port(name, class, 3, true, 3, 2), -EINVAL); + KUNIT_EXPECT_EQ(test, dptx_service_port(NULL, class, 3, true, 3, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, dptx_service_port(name, NULL, 3, true, 3, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, dptx_service_port("wrong", class, 3, true, 3, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, dptx_service_port(name, "wrong", 3, true, 3, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, dptx_service_port(name, class, 1, false, 3, 2), 1); + KUNIT_EXPECT_EQ(test, dptx_service_port(name, class, 2, false, 3, 2), -EINVAL); +} + +static void dptx_service_duplicate_close_test(struct kunit *test) +{ + struct apple_epic_service *first = test->priv; + struct apple_epic_service *next; + struct dptx_port port = {}; + + next = kunit_kzalloc(test, sizeof(*next), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, next); + spin_lock_init(&next->lock); + init_completion(&port.enable_completion); + KUNIT_ASSERT_TRUE(test, afk_service_reinit(first, NULL, &prepared_ops, 1)); + KUNIT_ASSERT_EQ(test, dptx_service_bind(first, &port, 3), 0); + /* An observer during preparation/OPEN sees no available port or owner. */ + KUNIT_EXPECT_FALSE(test, dptx_service_available(&port)); + KUNIT_EXPECT_PTR_EQ(test, port.service, NULL); + KUNIT_EXPECT_PTR_EQ(test, first->cookie, &port); + /* Failed OPEN does not publish anything and permits a fresh attempt. */ + afk_service_discard(first); + KUNIT_EXPECT_FALSE(test, port.enabled); + KUNIT_EXPECT_PTR_EQ(test, port.service, NULL); + KUNIT_ASSERT_TRUE(test, afk_service_reinit(first, NULL, &prepared_ops, 1)); + KUNIT_ASSERT_EQ(test, dptx_service_bind(first, &port, 3), 0); + afk_service_publish(first); + KUNIT_EXPECT_FALSE(test, port.enabled); + dptx_service_publish(first); + KUNIT_EXPECT_TRUE(test, dptx_service_available(&port)); + KUNIT_EXPECT_PTR_EQ(test, port.service->cookie, &port); + KUNIT_ASSERT_TRUE(test, afk_service_reinit(next, NULL, &prepared_ops, 2)); + KUNIT_EXPECT_EQ(test, dptx_service_bind(next, &port, 3), -EBUSY); + afk_service_discard(next); + KUNIT_EXPECT_TRUE(test, port.enabled); + KUNIT_EXPECT_PTR_EQ(test, port.service, first); + /* CLOSE invalidates a connected session without publishing a replacement. */ + port.connected = true; + afk_test_retire(first); + dptx_service_unbind(first); + KUNIT_EXPECT_FALSE(test, port.enabled); + KUNIT_EXPECT_FALSE(test, port.connected); + KUNIT_EXPECT_PTR_EQ(test, first->cookie, &port); + KUNIT_ASSERT_TRUE(test, afk_service_reinit(next, NULL, &prepared_ops, 3)); + KUNIT_EXPECT_EQ(test, dptx_service_bind(next, &port, 3), -EBUSY); + afk_service_discard(next); + /* Old work between validate/connect/request still names the old session. */ + KUNIT_EXPECT_PTR_EQ(test, port.service, first); + KUNIT_EXPECT_PTR_EQ(test, afk_service_get(port.service), NULL); + KUNIT_EXPECT_FALSE(test, port.enabled); +} + static struct kunit_case afk_service_cases[] = { + KUNIT_CASE(afk_service_admission_test), + KUNIT_CASE(dptx_service_identity_test), + KUNIT_CASE(dptx_service_duplicate_close_test), KUNIT_CASE(afk_service_many_generations_test), KUNIT_CASE(afk_service_reader_pin_test), KUNIT_CASE(afk_service_pending_reply_test), diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index b45695ddf9836..3fecf8d931db5 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -388,6 +388,16 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, goto free; } ep->num_channels = max(ep->num_channels, ch_idx + 1); + if (ops->prepare) { + int ret = ops->prepare(service, epic_name, epic_class, epic_unit); + + if (ret) { + dev_err(ep->dcp->dev, "AFK service %s admission failed: %d\n", + service_name, ret); + afk_service_discard(service); + goto free; + } + } if (ep->wire_v2) { int ret = afk_send_epic(ep, channel, 0, EPIC_TYPE_NOTIFY, EPIC_CAT_REPORT, AFKV2_OPEN, NULL, 0); @@ -395,16 +405,23 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, if (ret) { unsigned long flags; - spin_lock_irqsave(&service->lock, flags); - service->enabled = false; - service->torndown = true; - spin_unlock_irqrestore(&service->lock, flags); + if (ops->prepare) { + afk_service_discard(service); + } else { + spin_lock_irqsave(&service->lock, flags); + service->enabled = false; + service->torndown = true; + spin_unlock_irqrestore(&service->lock, flags); + } dev_err(ep->dcp->dev, "AFKV2 interface %u OPEN failed: %d\n", channel, ret); goto free; } } - ops->init(service, epic_name, epic_class, epic_unit); + if (ops->prepare) + afk_service_publish(service); + if (ops->init) + ops->init(service, epic_name, epic_class, epic_unit); dev_info(ep->dcp->dev, "AFK[ep:%02x]: new service %s on channel %d\n", ep->endpoint, service_name, channel); diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index c6a34fa67399e..b8c889da5c356 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -71,6 +71,12 @@ struct apple_epic_service_ops { /* All retained pointers must be pinned before enabling slot reuse. */ bool reusable; + /* + * Optional admission, before OPEN/publication. Must not retain service + * on failure; teardown must accept a failed/partial preparation. + */ + int (*prepare)(struct apple_epic_service *service, const char *name, + const char *class, s64 unit); void (*init)(struct apple_epic_service *service, const char *name, const char *class, s64 unit); int (*call)(struct apple_epic_service *service, u32 idx, @@ -177,7 +183,7 @@ afk_service_reinit(struct apple_epic_service *service, spin_lock_irqsave(&service->lock, flags); reusable = afk_service_reclaimable(service); if (reusable) { - service->enabled = true; + service->enabled = !ops->prepare; service->torndown = false; service->ops = ops; service->ep = ep; @@ -189,6 +195,30 @@ afk_service_reinit(struct apple_epic_service *service, return reusable; } +/* Only used before a prepared service has been published to callers. */ +static inline void afk_service_discard(struct apple_epic_service *service) +{ + unsigned long flags; + + if (service->ops->teardown) + service->ops->teardown(service); + spin_lock_irqsave(&service->lock, flags); + service->enabled = false; + service->torndown = true; + service->cookie = NULL; + service->ops = NULL; + spin_unlock_irqrestore(&service->lock, flags); +} + +static inline void afk_service_publish(struct apple_epic_service *service) +{ + unsigned long flags; + + spin_lock_irqsave(&service->lock, flags); + service->enabled = true; + spin_unlock_irqrestore(&service->lock, flags); +} + struct afk_ringbuffer_header { __le32 bufsz; u32 unk; diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index e6e449d9654f3..e102b48cf1f57 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -654,16 +654,12 @@ static const struct of_device_id apple_dcp_id_tbl[] = { {}, }; -/* - * The T8140 external display coprocessor needs firmware support this driver - * does not have yet. Leave it out of the display subsystem, so that an - * enabled node cannot hold back the internal panel's component master. - */ +/* An explicitly enabled external provider participates in component matching. */ static bool apple_dcp_usable(struct device_node *np) { return of_device_is_available(np) && (!of_device_is_compatible(np, "apple,t8140-dcpext") || - dcp_h17p_external_ready(np)); + dcp_h17p_external_enabled()); } static int apple_drm_init_dcp(struct device *dev) @@ -684,6 +680,13 @@ static int apple_drm_init_dcp(struct device *dev) of_device_is_compatible(np, "apple,t8140-dcpext") || of_property_present(np, "phys"); + /* component_bind_all() established the matched provider lifetime. */ + if (of_device_is_compatible(np, "apple,t8140-dcpext") && + !dcp_h17p_external_ready(np)) { + of_node_put(np); + return -ENODEV; + } + dcp[num_dcp] = of_find_device_by_node(np); if (!dcp[num_dcp]) continue; @@ -693,8 +696,14 @@ static int apple_drm_init_dcp(struct device *dev) ret = apple_probe_per_dcp(dev, &apple->drm, dcp[num_dcp], num_dcp, dcp_ext, &encoder[num_dcp], &crtc_mask[num_dcp]); - if (ret) + if (ret) { + if (of_device_is_compatible(np, "apple,t8140-dcpext")) { + put_device(&dcp[num_dcp]->dev); + of_node_put(np); + return ret; + } continue; + } num_dcp++; } diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 198e44e98e86d..7d7118304f039 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -46,6 +46,7 @@ #include "dcp.h" #include "dcp-internal.h" #include "dcp-external-bw.h" +#include "dptx-service.h" #include "iomfb.h" #include "parser.h" #include "trace.h" @@ -169,12 +170,14 @@ bool dcp_h17p_external_ready(struct device_node *np) pdev = of_find_device_by_node(np); if (!pdev) return false; - /* Failed and deferred probes must never become required components. */ - if (device_is_bound(&pdev->dev)) { - dcp = platform_get_drvdata(pdev); - ready = dcp && READ_ONCE(dcp->external_prepared) && - !READ_ONCE(dcp->quiescing) && !READ_ONCE(dcp->crashed); - } + /* + * Called after component binding. component_add() may synchronously + * bind the master before the platform probe itself returns, so the + * driver core's bound flag is not a readiness condition here. + */ + dcp = platform_get_drvdata(pdev); + ready = dcp && READ_ONCE(dcp->external_prepared) && + !READ_ONCE(dcp->quiescing) && !READ_ONCE(dcp->crashed); put_device(&pdev->dev); return ready; } @@ -2185,7 +2188,7 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) dcp->connector_type, dcp->dptxport[port].connected); mutex_lock(&dcp->hpd_mutex); - if (!dcp->dptxport[port].enabled) { + if (!dptx_service_available(&dcp->dptxport[port])) { dev_warn(dcp->dev, "dcp_dptx_connect: dptx service for port %d not enabled\n", port); ret = -ENODEV; goto out_unlock; @@ -2322,7 +2325,7 @@ static void dcp_placeholder_edid_work(struct work_struct *work) mutex_lock(&dcp->hpd_mutex); generation = dcp->placeholder_generation; if (!dcp->typec_cable_connected || !dcp->dptxport[0].connected || - !dcp->dptxport[0].enabled || generation != dcp->typec_generation) + !dptx_service_available(&dcp->dptxport[0]) || generation != dcp->typec_generation) goto out_unlock; service = dcp->dptxport[0].service; @@ -2397,7 +2400,7 @@ static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) dev_info(dcp->dev, "%s(port=%d)\n", __func__, port); mutex_lock(&dcp->hpd_mutex); - if (dcp->dptxport[port].enabled && dcp->dptxport[port].connected) { + if (dptx_service_available(&dcp->dptxport[port]) && dcp->dptxport[port].connected) { dptxport_release_display(dcp->dptxport[port].service); dcp->dptxport[port].connected = false; } @@ -2449,7 +2452,7 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) if (dcp->avep) av_service_disconnect(dcp); - if (dcp->dptxport[port].enabled) + if (dptx_service_available(&dcp->dptxport[port])) dptxport_set_hpd(dcp->dptxport[port].service, false); return dcp_dptx_disconnect(dcp, port); @@ -2601,7 +2604,7 @@ int dcp_start(struct platform_device *pdev) * Since dcp0 is not used at the moment let's avoid this * since it is possibly the cause for startup issues. */ - } else if (dcp->dptxport[0].enabled) { + } else if (dptx_service_available(&dcp->dptxport[0])) { bool connected; /* force disconnect on start - necessary if the display * is already up from m1n1 @@ -2859,7 +2862,7 @@ void dcp_poweroff(struct platform_device *pdev) if (dcp_is_typec_output(dcp)) { /* DCP owns a synthetic HPD for Type-C. Release it with the CRTC. */ - if (dcp->dptxport[0].enabled && dcp->dptxport[0].connected) { + if (dptx_service_available(&dcp->dptxport[0]) && dcp->dptxport[0].connected) { ret = dptxport_set_hpd(dcp->dptxport[0].service, false); if (ret) dev_warn(dcp->dev, diff --git a/drivers/gpu/drm/apple/dptx-service.h b/drivers/gpu/drm/apple/dptx-service.h new file mode 100644 index 0000000000000..f806e3253999a --- /dev/null +++ b/drivers/gpu/drm/apple/dptx-service.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DPTX_SERVICE_H +#define APPLE_DPTX_SERVICE_H + +#include "afk.h" +#include "dptxep.h" + +static inline int dptx_service_port(const char *name, const char *class, + s64 unit, bool wire_v2, u32 index, + u32 num_ports) +{ + if (!name || !class || strcmp(name, "dcpdptx-port-epic") || + strcmp(class, "AppleDCPDPTXRemotePort")) + return -EINVAL; + if (wire_v2) + return num_ports == 1 && unit == index ? 0 : -EINVAL; + return unit >= 0 && unit < 2 ? unit : -EINVAL; +} + +static inline bool dptx_service_available(struct dptx_port *port) +{ + /* Pairs with publication before readers use the service and its cookie. */ + return smp_load_acquire(&port->enabled); +} + +static inline int dptx_service_bind(struct apple_epic_service *service, + struct dptx_port *port, s64 unit) +{ + if (port->service) + return -EBUSY; + port->unit = unit; + /* No owner is published until OPEN and AFK publication succeed. */ + service->cookie = port; + return 0; +} + +static inline void dptx_service_publish(struct apple_epic_service *service) +{ + struct dptx_port *port = service->cookie; + + port->service = service; + port->connected = false; + /* Pair with dcp.c readers before they dereference the service cookie. */ + smp_store_release(&port->enabled, true); +} + +static inline void dptx_service_unbind(struct apple_epic_service *service) +{ + struct dptx_port *port = service->cookie; + + if (port && port->service == service) { + WRITE_ONCE(port->enabled, false); + WRITE_ONCE(port->connected, false); + reinit_completion(&port->enable_completion); + } + /* + * Keep the published owner and cookie valid for retained callers. Once + * this session closes, bind fails closed until reboot: + * an in-flight connect must never switch to a new firmware session. + */ +} + +#endif diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 6c4012d6ad752..451820389f91c 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -15,6 +15,7 @@ #include "afk.h" #include "dcp.h" #include "dptxep.h" +#include "dptx-service.h" #include "parser.h" #include "trace.h" @@ -823,48 +824,26 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, } } -static void dptxport_init(struct apple_epic_service *service, const char *name, - const char *class, s64 unit) +static int dptxport_prepare(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) { struct apple_dcp *dcp = service->ep->dcp; - s64 port = unit; - - if (!name || !class) - return; - if (strcmp(name, "dcpdptx-port-epic")) - return; - if (strcmp(class, "AppleDCPDPTXRemotePort")) - return; + int port = dptx_service_port(name, class, unit, service->ep->wire_v2, + dcp->index, dcp->hw.num_dptx_ports); + if (port < 0) + return port; trace_dptxport_init(dcp, unit); - if (service->ep->wire_v2) { - /* The single external service reports its physical DCP unit. */ - if (dcp->hw.num_dptx_ports != 1 || unit != dcp->index) { - dev_err(dcp->dev, "Unexpected external DPTX unit %lld\n", unit); - return; - } - port = 0; - dev_dbg(dcp->dev, "DPTX physical unit %lld maps to logical port 0\n", unit); - } + return dptx_service_bind(service, &dcp->dptxport[port], unit); +} - switch (port) { - case 0: - case 1: - if (service->ep->dcp->dptxport[port].enabled) { - dev_err(service->ep->dcp->dev, - "DPTXPort: unit %lld already exists\n", unit); - return; - } - service->ep->dcp->dptxport[port].unit = unit; - service->ep->dcp->dptxport[port].service = service; - service->ep->dcp->dptxport[port].enabled = true; - service->cookie = (void *)&service->ep->dcp->dptxport[port]; - complete(&service->ep->dcp->dptxport[port].enable_completion); - break; - default: - dev_err(service->ep->dcp->dev, "DPTXPort: invalid unit %lld\n", - unit); - } +static void dptxport_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + struct dptx_port *port = service->cookie; + + dptx_service_publish(service); + complete(&port->enable_completion); } static int dptxport_group_call(struct apple_epic_service *service, u16 group, u32 command, @@ -892,6 +871,8 @@ static int dptxport_group_call(struct apple_epic_service *service, u16 group, u3 static const struct apple_epic_service_ops dptxep_ops[] = { { .name = "AppleDCPDPTXRemotePort", + .prepare = dptxport_prepare, + .teardown = dptx_service_unbind, .init = dptxport_init, .call = dptxport_call, .call_v2 = dptxport_group_call, From 7963fa566aa1d27d0e9b65b87942d855ecb3f1dc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:59:19 +1100 Subject: [PATCH 1193/2680] drm/apple: add bounded external stop qualification path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pause new presentation work while callbacks and control RPCs remain available; settle the active queue, serialize external lifecycle operations and retain resources on failure. Preserve first-start preparation and generation-aware CRTC recovery. OFF replies are captured but deliberately return EOPNOTSUPP until their contract is qualified; no successful production teardown or DMA retirement is claimed. Host tests exercise admission, drain, exact response bounds, timeout ownership and lifecycle ordering. A hypothetical qualified-OFF adapter exists only inside the host test. Astra reviewed this revision for one initialized DPMS-OFF capture followed by reboot, excluding unplug/suspend/restart qualification. [ryan: adapted to 7.2.10 — struct drm_atomic_state is struct drm_atomic_commit] [ryan: split by topic: only the drm/apple part and its selftests are here; the MAINTAINERS lines go to 900-config-tooling; the tools/diagnostics scripts are dropped (aurora-wip deleted them in commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"))] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 3 + drivers/gpu/drm/apple/dcp-internal.h | 25 +++ drivers/gpu/drm/apple/dcp.c | 281 ++++++++++++++++++++++++- drivers/gpu/drm/apple/dcp.h | 1 + drivers/gpu/drm/apple/iomfb.c | 64 +++++- drivers/gpu/drm/apple/iomfb_h17p.c | 99 +++++++++ drivers/gpu/drm/apple/iomfb_template.c | 3 + 7 files changed, 469 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index e102b48cf1f57..c13bc5643a179 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -152,6 +152,9 @@ static void apple_crtc_atomic_enable(struct drm_crtc *crtc, dcp_poweron(apple_crtc->dcp); /* Force the CTM to be set on first swap */ crtc_state->color_mgmt_changed = true; + } else if (crtc_state->active && drm_atomic_crtc_needs_modeset(crtc_state)) { + /* A cable generation can change while the CRTC stays active. */ + dcp_external_recover(to_apple_crtc(crtc)->dcp); } if (crtc_state->active) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 95fe377b69106..280d13ab07321 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -48,6 +48,7 @@ struct iomfb_transaction { bool backlight_reserved; bool backlight_failed; bool brightness_only; + bool presentation; bool completed; }; struct apple_dcp_typec_port; @@ -208,6 +209,25 @@ struct apple_dcp { struct apple_dcp_hw_data hw; bool external_prepared; + /* Serializes external connection, power, and CRTC generation changes. */ + struct mutex external_lock; + u64 external_generation; + u64 external_crtc_generation; + /* iomfb.lock: epoch used by receive-side deferred stop requests. */ + u64 external_rx_generation; + u64 external_stop_generation; + bool external_cable; + bool external_running; + bool external_power_attempted; + /* iomfb.lock: replay the latest callback after activation succeeds. */ + bool external_hotplug_pending; + bool external_hotplug_connected; + bool external_stopped; + int external_error; + u64 external_power_reply; + bool external_power_reply_valid; + struct work_struct external_stop_work; + /* firmware version and compatible firmware version */ enum dcp_firmware_version fw_compat; @@ -441,6 +461,7 @@ struct apple_dcp { unsigned long deadline; unsigned int queued; bool stopped; + bool presentation_paused; bool backlight_queued; atomic_t opaque_x_state; struct iomfb_scanout_h17p *scanout; @@ -465,11 +486,15 @@ int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum, void iomfb_queue_init(struct apple_dcp *dcp); void iomfb_queue_stop(struct apple_dcp *dcp); bool iomfb_queue_drain(struct apple_dcp *dcp, unsigned long timeout); +int iomfb_queue_settle(struct apple_dcp *dcp, unsigned long timeout); +void iomfb_pause_presentation(struct apple_dcp *dcp, bool pause); +int iomfb_external_power_h17p(struct apple_dcp *dcp, bool on); int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction); void iomfb_queue_crc_h17p(struct apple_dcp *dcp, u32 swap_id); void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); void dcp_handle_hotplug_actions(struct apple_dcp *dcp, unsigned int action); +bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected); int dcp_backlight_init(struct apple_dcp *dcp, struct device *dev); void dcp_backlight_stop(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 7d7118304f039..f6213cdec97f4 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -301,6 +301,14 @@ static bool dcp_typec_route_fits(struct apple_dcp_typec_route *route, return connector->candidate_crtcs & drm_crtc_mask(&dcp->crtc->base); } +static bool dcp_is_external_h17p(struct apple_dcp *dcp) +{ + return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; +} + +static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation); +static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical); + static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, struct mux_control *xbar); static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); @@ -349,7 +357,9 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, * can still be marked connected to a display that is gone. Release it * (a no-op otherwise), or connecting the borrowed route returns early. */ - dcp_dptx_disconnect(dcp, 0); + ret = dcp_dptx_disconnect(dcp, 0); + if (ret) + return ret; if (dcp->fixed_route_selected) { ret = mux_control_deselect(dcp->xbar); @@ -439,6 +449,11 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) bool was_tunnel = route->tunnel; int ret = 0; + guard(mutex)(&dcp->external_lock); + if (dcp_is_external_h17p(dcp) && + (dcp->external_error || (dcp->external_power_attempted && !dcp->external_stopped))) + return dcp->external_error ?: -EBUSY; + /* * Under tb_lock so a DCP link (re)configuration can neither select the * crossbar nor restart the tunnel pixel clock behind our back. @@ -1557,7 +1572,8 @@ static void dcp_typec_route_unregister(void *data) if (port->hpd || dcp->typec_cable_connected) dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); port->hpd = false; - dcp_typec_route_deactivate(route); + if (dcp_typec_route_deactivate(route) && dcp_is_external_h17p(dcp)) + return; port->owner = NULL; } if (port->secondary_owner == route) { @@ -1565,7 +1581,8 @@ static void dcp_typec_route_unregister(void *data) if (port->hpd || dcp->typec_cable_connected) dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - dcp_typec_route_deactivate(route); + if (dcp_typec_route_deactivate(route) && dcp_is_external_h17p(dcp)) + return; port->secondary_owner = NULL; } port->hpd = !!(port->owner || port->secondary_owner); @@ -2171,7 +2188,7 @@ bool dcp_has_typec_routes(struct platform_device *pdev) #define DPTX_RECONNECT_DELAY msecs_to_jiffies(1000) #define DPTX_RECONNECT_RETRIES 5 -static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) +static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port) { unsigned long timeout; int ret = 0; @@ -2280,6 +2297,186 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) return ret; } +static void dcp_external_report_disconnected(struct apple_dcp *dcp) +{ + unsigned int action; + + guard(mutex)(&dcp->iomfb.lock); + action = dcp_mode_hotplug(&dcp->mode_state, false, + dcp->connector ? &dcp->connector->connected : NULL); + /* The existing worker delivers DRM notifications outside our locks. */ + dcp_handle_hotplug_actions(dcp, action); +} + +static void dcp_external_publish_generation(struct apple_dcp *dcp) +{ + lockdep_assert_held(&dcp->external_lock); + guard(mutex)(&dcp->iomfb.lock); + dcp->external_rx_generation = dcp->external_generation; +} + +/* external_lock is never taken by the serialized IOMFB receive callbacks. */ +static int dcp_external_link_locked(struct apple_dcp *dcp) +{ + int ret; + + lockdep_assert_held(&dcp->external_lock); + if (!dcp->external_prepared) + return -EAGAIN; + if (dcp->external_error) + return dcp->external_error; + if (!dcp->external_cable) + return -ENODEV; + ret = dcp_dptx_connect_raw(dcp, 0); + if (ret) + dcp->external_error = ret; + return ret; +} + +static int dcp_external_start_locked(struct apple_dcp *dcp) +{ + int ret; + + lockdep_assert_held(&dcp->external_lock); + ret = dcp_external_link_locked(dcp); + if (ret || dcp->external_running) + return ret; + iomfb_pause_presentation(dcp, true); + /* Even a timed-out first activation can have changed firmware state. */ + dcp->external_power_attempted = true; + ret = iomfb_external_power_h17p(dcp, true); + if (ret) { + dcp->external_error = ret; + return ret; + } + dcp->external_running = true; + dcp->external_stopped = false; + WRITE_ONCE(dcp->pipe_enabled_h17p, true); + WRITE_ONCE(dcp->typec_crtc_off, false); + iomfb_pause_presentation(dcp, false); + return 0; +} + +static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) +{ + if (!dcp_is_external_h17p(dcp)) + return dcp_dptx_connect_raw(dcp, port); + guard(mutex)(&dcp->external_lock); + return dcp_external_link_locked(dcp); +} + +/* The generation token belongs to the CRTC that was actually enabled. */ +static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation) +{ + int ret; + + lockdep_assert_held(&dcp->external_lock); + if (generation != dcp->external_generation) + return 0; + if (dcp->external_error) + return dcp->external_error; + if (dcp->external_stopped) + return 0; + iomfb_pause_presentation(dcp, true); + dcp->external_running = false; + /* Route setup can call this before IOMFB or its shared memory exists. */ + if (!dcp->external_power_attempted) + goto release_link; + ret = iomfb_queue_settle(dcp, msecs_to_jiffies(500)); + if (!ret) + ret = iomfb_external_power_h17p(dcp, false); + if (!ret) + ret = iomfb_queue_settle(dcp, msecs_to_jiffies(500)); + if (ret) + goto failed; + /* This is a transport acknowledgment, not permission to free scanout. */ + WRITE_ONCE(dcp->pipe_enabled_h17p, false); +release_link: + scoped_guard(mutex, &dcp->hpd_mutex) { + if (dcp->dptxport[0].connected) { + if (!dptx_service_available(&dcp->dptxport[0])) { + ret = -ENODEV; + goto failed; + } + ret = dptxport_set_hpd(dcp->dptxport[0].service, false); + if (!ret) + ret = dptxport_release_display(dcp->dptxport[0].service); + if (ret) + goto failed; + dcp->dptxport[0].connected = false; + } + } + dcp->external_stopped = true; + WRITE_ONCE(dcp->typec_crtc_off, true); + if (dcp->avep) + av_service_disconnect(dcp); + return 0; +failed: + /* A retry could race a late reply. Keep this session blocked and owned. */ + dcp->external_error = ret; + return ret; +} + +static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical) +{ + int ret; + + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + guard(mutex)(&dcp->external_lock); + ret = dcp_external_stop_locked(dcp, dcp->external_generation); + if (physical) { + dcp->external_cable = false; + dcp->external_generation++; + dcp_external_publish_generation(dcp); + scoped_guard(mutex, &dcp->hpd_mutex) { + WRITE_ONCE(dcp->typec_cable_connected, false); + dcp->typec_generation++; + } + /* A missing cable is reported even when its hardware remains owned. */ + dcp_external_report_disconnected(dcp); + } + return ret; +} + +static void dcp_external_stop_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(work, struct apple_dcp, external_stop_work); + + u64 generation; + + scoped_guard(mutex, &dcp->iomfb.lock) + generation = dcp->external_stop_generation; + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + guard(mutex)(&dcp->external_lock); + if (generation == dcp->external_generation) { + dcp_external_stop_locked(dcp, generation); + dcp_external_report_disconnected(dcp); + } +} + +bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected) +{ + if (!dcp_is_external_h17p(dcp)) + return false; + /* No lifecycle or HPD mutex may be acquired from the RX callback. */ + lockdep_assert_held(&dcp->iomfb.lock); + if (READ_ONCE(dcp->quiescing)) + return true; + if (dcp->iomfb.presentation_paused) { + dcp->external_hotplug_pending = true; + dcp->external_hotplug_connected = connected; + return true; + } + if (!connected && !READ_ONCE(dcp->iomfb.presentation_paused)) { + dcp->external_stop_generation = dcp->external_rx_generation; + schedule_work(&dcp->external_stop_work); + return true; + } + return READ_ONCE(dcp->iomfb.presentation_paused); +} + static bool dcp_edid_is_placeholder(const struct drm_edid *drm_edid) { const u8 *raw = (const u8 *)drm_edid_raw(drm_edid); @@ -2397,6 +2594,9 @@ static void disconnected_hpd_event(struct apple_connector *con) static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) { + if (dcp_is_external_h17p(dcp)) + return dcp_external_disconnect(dcp, false); + dev_info(dcp->dev, "%s(port=%d)\n", __func__, port); mutex_lock(&dcp->hpd_mutex); @@ -2414,6 +2614,23 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; + if (dcp_is_external_h17p(dcp)) { + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + cancel_work_sync(&dcp->external_stop_work); + guard(mutex)(&dcp->external_lock); + if (!dcp->external_cable) + dcp->external_generation++; + dcp->external_cable = true; + dcp_external_publish_generation(dcp); + scoped_guard(mutex, &dcp->hpd_mutex) { + WRITE_ONCE(dcp->typec_cable_connected, true); + dcp->typec_generation++; + dcp->placeholder_retried = false; + } + return dcp_external_link_locked(dcp); + } + if (dcp_is_typec_output(dcp)) { cancel_delayed_work_sync(&dcp->placeholder_edid_wq); guard(mutex)(&dcp->hpd_mutex); @@ -2436,6 +2653,9 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) { struct apple_dcp *dcp = platform_get_drvdata(pdev); + if (dcp_is_external_h17p(dcp)) + return dcp_external_disconnect(dcp, true); + if (dcp_is_typec_output(dcp)) { scoped_guard(mutex, &dcp->hpd_mutex) { WRITE_ONCE(dcp->typec_cable_connected, false); @@ -2746,6 +2966,19 @@ void dcp_poweron(struct platform_device *pdev) unsigned long remaining; int ret; + if (dcp_is_external_h17p(dcp)) { + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + cancel_work_sync(&dcp->external_stop_work); + guard(mutex)(&dcp->external_lock); + ret = dcp_external_start_locked(dcp); + if (!ret) + dcp->external_crtc_generation = dcp->external_generation; + else + dev_err(dcp->dev, "external display activation failed: %d\n", ret); + return; + } + if (dcp_uses_soft_dpms(dcp)) { if (dcp->fw_compat == DCP_FIRMWARE_H17P && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && @@ -2823,11 +3056,30 @@ void dcp_poweron(struct platform_device *pdev) av_service_connect(dcp); } +/* Same-CRTC helper dependencies order this token refresh before later disables. */ +void dcp_external_recover(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + if (dcp_is_external_h17p(dcp)) + dcp_poweron(pdev); +} + void dcp_poweroff(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; + if (dcp_is_external_h17p(dcp)) { + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + guard(mutex)(&dcp->external_lock); + ret = dcp_external_stop_locked(dcp, dcp->external_crtc_generation); + if (ret) + dev_err(dcp->dev, "external display stop failed: %d\n", ret); + return; + } + cancel_delayed_work(&dcp->swap_watchdog_wq); /* Internal H17P DPMS is a brightness present, never a pipe stop. */ @@ -3380,6 +3632,7 @@ static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) /* Block new enqueues as well as draining users of the AFK endpoints. */ disable_delayed_work_sync(&dcp->typec_reconnect_wq); disable_delayed_work_sync(&dcp->placeholder_edid_wq); + cancel_work_sync(&dcp->external_stop_work); } static void dcp_enable_typec_work(struct apple_dcp *dcp) @@ -3615,7 +3868,9 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) if (dcp->retain_dma) { /* No firmware stop is acknowledged: detach host users, never DMA. */ - WRITE_ONCE(dcp->quiescing, true); + scoped_guard(mutex, &dcp->iomfb.lock) + WRITE_ONCE(dcp->quiescing, true); + cancel_work_sync(&dcp->external_stop_work); if (dcp->crtc && !dcp->drm_retained) { drm_dev_get(dcp->crtc->base.dev); dcp->drm_retained = true; @@ -3761,6 +4016,8 @@ static int dcp_platform_probe(struct platform_device *pdev) * registered below, before the DRM device binds. */ mutex_init(&dcp->hpd_mutex); + mutex_init(&dcp->external_lock); + INIT_WORK(&dcp->external_stop_work, dcp_external_stop_work); mutex_init(&dcp->tb_lock); spin_lock_init(&dcp->mode_state.lock); spin_lock_init(&dcp->dcpavserv.lock); @@ -3773,6 +4030,7 @@ static int dcp_platform_probe(struct platform_device *pdev) experimental_h17p_external_bw_request)) return dev_err_probe(dev, -EINVAL, "External bandwidth offsets are unset or invalid\n"); + iomfb_pause_presentation(dcp, true); dcp->hw.firmware_scratch = experimental_h17p_external_bw_scratch; dcp->hw.firmware_request = experimental_h17p_external_bw_request; } @@ -4038,6 +4296,19 @@ static int dcp_platform_suspend(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); + if (dcp_is_external_h17p(dcp)) { + int ret; + + dcp_disable_typec_work(dcp, false); + ret = dcp_external_disconnect(dcp, false); + if (ret) { + dcp_enable_typec_work(dcp); + return ret; + } + device_set_wakeup_path(dev); + return 0; + } + dcp_drain_for_sleep(dcp); /* * The Type-C route reports cable removal through diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index f1afe81e475d2..9bef4759e8dcb 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -34,6 +34,7 @@ struct apple_encoder { void dcp_poweroff(struct platform_device *pdev); void dcp_poweron(struct platform_device *pdev); +void dcp_external_recover(struct platform_device *pdev); int dcp_set_crc(struct drm_crtc *crtc, bool enabled); int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commit); int dcp_get_connector_type(struct platform_device *pdev); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 1bc3eff4943fe..7568d126c29c2 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -430,6 +430,45 @@ bool iomfb_queue_drain(struct apple_dcp *dcp, unsigned long timeout) } } +/* Admission is independent of transport: replies and control RPCs remain live. */ +void iomfb_pause_presentation(struct apple_dcp *dcp, bool pause) +{ + guard(mutex)(&dcp->iomfb.lock); + dcp->iomfb.presentation_paused = pause; + if (!pause && dcp->external_hotplug_pending) { + unsigned int action; + + dcp->external_hotplug_pending = false; + action = dcp_mode_hotplug(&dcp->mode_state, dcp->external_hotplug_connected, + dcp->connector ? &dcp->connector->connected : NULL); + dcp_handle_hotplug_actions(dcp, action); + } +} + +/* A dead transport is not proof that DMA or a pending present has stopped. */ +int iomfb_queue_settle(struct apple_dcp *dcp, unsigned long timeout) +{ + unsigned long deadline = jiffies + timeout; + bool idle; + + for (;;) { + mutex_lock(&dcp->iomfb.lock); + if (READ_ONCE(dcp->crashed) || dcp->iomfb.stopped) { + mutex_unlock(&dcp->iomfb.lock); + return -EIO; + } + idle = !dcp->iomfb.active && !dcp->iomfb.queued && + list_empty(&dcp->iomfb.pending) && + !dcp->present_state_h17p.pending && iomfb_channels_idle(dcp); + mutex_unlock(&dcp->iomfb.lock); + if (idle) + return 0; + if (time_after_eq(jiffies, deadline)) + return -ETIMEDOUT; + usleep_range(2000, 4000); + } +} + int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction) { int ret = 0; @@ -437,6 +476,8 @@ int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction) mutex_lock(&dcp->iomfb.lock); if (READ_ONCE(dcp->crashed) || dcp->iomfb.stopped) { ret = -EIO; + } else if (transaction->presentation && dcp->iomfb.presentation_paused) { + ret = -EAGAIN; } else if (transaction->brightness_only && dcp->iomfb.backlight_queued) { ret = -EALREADY; } else if (dcp->iomfb.queued >= 32) { @@ -506,7 +547,7 @@ void iomfb_queue_crc_h17p(struct apple_dcp *dcp, u32 swap_id) lockdep_assert_held(&dcp->iomfb.lock); if (!READ_ONCE(dcp->crc_enabled) || READ_ONCE(dcp->crashed) || - dcp->iomfb.stopped || dcp->iomfb.queued >= 32) + dcp->iomfb.stopped || dcp->iomfb.presentation_paused || dcp->iomfb.queued >= 32) return; crc = kzalloc_obj(*crc); if (!crc) @@ -1119,6 +1160,13 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, int ret = -EIO; bool modeset; + /* Disconnect cannot pass the admission check before these RPCs finish. */ + guard(mutex)(&dcp->external_lock); + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + (!dcp->external_running || dcp->external_error || READ_ONCE(dcp->quiescing) || + READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.presentation_paused))) + return -EPIPE; + crtc_state = drm_atomic_get_new_crtc_state(commit, crtc); if (!crtc_state) return 0; @@ -1377,6 +1425,7 @@ static void iomfb_backlight_kick(struct apple_dcp *dcp) transaction->start = iomfb_backlight_start; transaction->release = iomfb_backlight_release; transaction->brightness_only = true; + transaction->presentation = true; if (iomfb_queue(dcp, transaction)) kfree(transaction); } @@ -1437,18 +1486,29 @@ static void iomfb_queue_atomic(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_commit *state) { struct iomfb_atomic_transaction *atomic; + int ret; + if (READ_ONCE(dcp->iomfb.presentation_paused)) { + schedule_work(&dcp->vblank_wq); + return; + } atomic = kzalloc_obj(*atomic); if (!atomic) goto failed; atomic->transaction.start = iomfb_atomic_start; + atomic->transaction.presentation = true; atomic->transaction.release = iomfb_atomic_release; atomic->state = drm_atomic_commit_get(state); atomic->crtc = crtc; atomic->dcp = dcp; - if (!iomfb_queue(dcp, &atomic->transaction)) + ret = iomfb_queue(dcp, &atomic->transaction); + if (!ret) return; iomfb_atomic_release(&atomic->transaction); + if (ret == -EAGAIN) { + schedule_work(&dcp->vblank_wq); + return; + } failed: WRITE_ONCE(dcp->crashed, true); schedule_work(&dcp->vblank_wq); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index fd8cee0ccaf7d..dbfe025445f4e 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -719,6 +719,105 @@ bool iomfb_check_callback_h17p(struct apple_dcp *dcp, int tag, u32 in_len, return true; } +static_assert(sizeof(struct dcp_set_power_state_resp) == 8); + +struct dcp_external_power_cookie { + struct dcp_wait_cookie wait; + bool on; +}; + +/* A473's envelope is known; no OFF reply has been qualified as a stop. */ +static void iomfb_external_power_complete(struct apple_dcp *dcp, void *out, void *data) +{ + struct dcp_external_power_cookie *cookie = data; + const struct dcp_packet_header *header; + int ret = -EPROTO; + + if (out) { + header = out - sizeof(struct dcp_set_power_state_req_h17p) - sizeof(*header); + if (header->in_len == sizeof(struct dcp_set_power_state_req_h17p) && + header->out_len == sizeof(struct dcp_set_power_state_resp)) { + u64 reply = get_unaligned_le64(out); + + WRITE_ONCE(dcp->external_power_reply, reply); + WRITE_ONCE(dcp->external_power_reply_valid, true); + dev_info(dcp->dev, "A473 power=%u reply=%#llx%s\n", + cookie->on, reply, cookie->on ? "" : " (stop unqualified)"); + ret = cookie->on ? (reply == 1 ? 0 : -EREMOTEIO) : -EOPNOTSUPP; + } + } + WRITE_ONCE(cookie->wait.status, ret); + complete(&cookie->wait.done); + kref_put(&cookie->wait.refcount, release_wait_cookie); +} + +static bool iomfb_external_reply_size(void *out, u32 in_len, u32 out_len) +{ + const struct dcp_packet_header *header; + + if (!out) + return false; + header = out - in_len - sizeof(*header); + return header->in_len == in_len && header->out_len == out_len; +} + +static void iomfb_external_parameter_complete(struct apple_dcp *dcp, void *out, void *data) +{ + struct dcp_set_power_state_req_h17p request = { .unklong = 1 }; + + if (!iomfb_external_reply_size(out, sizeof(struct dcp_set_parameter_dcp), sizeof(u32))) { + iomfb_external_power_complete(dcp, NULL, data); + return; + } + dcp_set_power_state(dcp, true, &request, iomfb_external_power_complete, data); +} + +static void iomfb_external_device_complete(struct apple_dcp *dcp, void *out, void *data) +{ + struct dcp_set_parameter_dcp parameter = { + .param = IOMFBPARAM_ADAPTIVE_SYNC, + .count = 3, + }; + + if (!iomfb_external_reply_size(out, sizeof(u32), sizeof(u32))) { + iomfb_external_power_complete(dcp, NULL, data); + return; + } + /* Preserve the existing external power-on preparation and payload semantics. */ + dcp_set_parameter_dcp(dcp, false, ¶meter, iomfb_external_parameter_complete, data); +} + +int iomfb_external_power_h17p(struct apple_dcp *dcp, bool on) +{ + struct dcp_set_power_state_req_h17p request = { .unklong = on }; + struct dcp_external_power_cookie *cookie; + int ret; + + if (READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.stopped)) + return -EIO; + cookie = kzalloc_obj(*cookie); + if (!cookie) + return -ENOMEM; + cookie->on = on; + WRITE_ONCE(dcp->external_power_reply_valid, false); + cookie->wait.status = -ETIMEDOUT; + init_completion(&cookie->wait.done); + kref_init(&cookie->wait.refcount); + kref_get(&cookie->wait.refcount); + if (on) { + u32 handle = 2; + + iomfb_opaque_x_reset_h17p(dcp); + dcp_set_display_device(dcp, false, &handle, iomfb_external_device_complete, cookie); + } else { + dcp_set_power_state(dcp, true, &request, iomfb_external_power_complete, cookie); + } + ret = wait_for_completion_timeout(&cookie->wait.done, msecs_to_jiffies(500)); + ret = ret ? (int)READ_ONCE(cookie->wait.status) : -ETIMEDOUT; + kref_put(&cookie->wait.refcount, release_wait_cookie); + return ret; +} + void iomfb_start_h17p(struct apple_dcp *dcp) { WRITE_ONCE(dcp->pipe_enabled_h17p, false); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 4c5a2a3924e2e..efa81eb705cd9 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1870,6 +1870,9 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) if (dcp_is_typec_output(dcp) && *connected && dcp->nr_modes) complete_all(&dcp->typec_iomfb_hpd_ready); + if (dcp_external_hotplug(dcp, !!(*connected))) + return; + action = dcp_mode_hotplug(&dcp->mode_state, !!(*connected), connector ? &connector->connected : NULL); /* From 3ee78daa2ab5700415d445225ac5b5fe39b6131a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:13:34 +1100 Subject: [PATCH 1194/2680] drm/apple: retry initial external DPTX service discovery Distinguish a service not yet published from terminal session closure before issuing firmware commands. Keep initial publication retryable, preserve sticky failures after real connection commands, and schedule asynchronous reconnect when the service arrives late. The R28 hardware trial exposed the startup ordering regression. Extracted production tests cover initial and partial publication, closure, late arrival, and retained firmware failures. [ryan: split by topic: only the drm/apple part and its selftests are here; the MAINTAINERS lines go to 900-config-tooling; the tools/diagnostics scripts are dropped (aurora-wip deleted them in commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"))] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 16 ++++++++++++---- drivers/gpu/drm/apple/dptx-service.h | 4 +++- drivers/gpu/drm/apple/dptxep.c | 8 ++++++++ drivers/gpu/drm/apple/dptxep.h | 2 ++ 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index f6213cdec97f4..369012abb5481 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2188,11 +2188,13 @@ bool dcp_has_typec_routes(struct platform_device *pdev) #define DPTX_RECONNECT_DELAY msecs_to_jiffies(1000) #define DPTX_RECONNECT_RETRIES 5 -static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port) +static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *waiting_service) { unsigned long timeout; int ret = 0; + if (waiting_service) + *waiting_service = false; if (!dcp->phy) { dev_warn(dcp->dev, "dcp_dptx_connect: missing phy\n"); return -ENODEV; @@ -2208,6 +2210,11 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port) if (!dptx_service_available(&dcp->dptxport[port])) { dev_warn(dcp->dev, "dcp_dptx_connect: dptx service for port %d not enabled\n", port); ret = -ENODEV; + /* No firmware request has been made; the first publication can retry. */ + if (waiting_service && !READ_ONCE(dcp->dptxport[port].closed)) { + *waiting_service = true; + ret = -EAGAIN; + } goto out_unlock; } @@ -2318,6 +2325,7 @@ static void dcp_external_publish_generation(struct apple_dcp *dcp) /* external_lock is never taken by the serialized IOMFB receive callbacks. */ static int dcp_external_link_locked(struct apple_dcp *dcp) { + bool waiting_service; int ret; lockdep_assert_held(&dcp->external_lock); @@ -2327,8 +2335,8 @@ static int dcp_external_link_locked(struct apple_dcp *dcp) return dcp->external_error; if (!dcp->external_cable) return -ENODEV; - ret = dcp_dptx_connect_raw(dcp, 0); - if (ret) + ret = dcp_dptx_connect_raw(dcp, 0, &waiting_service); + if (ret && !waiting_service) dcp->external_error = ret; return ret; } @@ -2360,7 +2368,7 @@ static int dcp_external_start_locked(struct apple_dcp *dcp) static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) { if (!dcp_is_external_h17p(dcp)) - return dcp_dptx_connect_raw(dcp, port); + return dcp_dptx_connect_raw(dcp, port, NULL); guard(mutex)(&dcp->external_lock); return dcp_external_link_locked(dcp); } diff --git a/drivers/gpu/drm/apple/dptx-service.h b/drivers/gpu/drm/apple/dptx-service.h index f806e3253999a..3d990a9ca1ea1 100644 --- a/drivers/gpu/drm/apple/dptx-service.h +++ b/drivers/gpu/drm/apple/dptx-service.h @@ -49,7 +49,9 @@ static inline void dptx_service_unbind(struct apple_epic_service *service) struct dptx_port *port = service->cookie; if (port && port->service == service) { - WRITE_ONCE(port->enabled, false); + WRITE_ONCE(port->closed, true); + /* Make terminal state visible to readers of availability. */ + smp_store_release(&port->enabled, false); WRITE_ONCE(port->connected, false); reinit_completion(&port->enable_completion); } diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 451820389f91c..e1d9dfb34dca5 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -844,6 +844,14 @@ static void dptxport_init(struct apple_epic_service *service, const char *name, dptx_service_publish(service); complete(&port->enable_completion); + /* Publication may follow the startup wait or the bounded retry window. */ + if (service->ep->dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + READ_ONCE(service->ep->dcp->typec_cable_connected) && + !READ_ONCE(service->ep->dcp->quiescing) && + !READ_ONCE(service->ep->dcp->external_error)) + mod_delayed_work(system_freezable_wq, + &service->ep->dcp->typec_reconnect_wq, 0); } static int dptxport_group_call(struct apple_epic_service *service, u16 group, u32 command, diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index 14530c99567b1..e731604fb6768 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -50,6 +50,8 @@ struct apple_epic_service; struct dptx_port { bool enabled, connected; + /* A published session closed; unlike initial discovery, this is terminal. */ + bool closed; struct completion enable_completion; struct completion linkcfg_completion; u32 unit; From 87065d869e51b660bc9b4ac0ce3084a73bdeeb1c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:14:49 +1100 Subject: [PATCH 1195/2680] drm/apple: select compatible SDR color modes for external H17P Choose advertised full-range 8/10-bit SDR RGB modes for the experimental external H17P profile. Prefer sRGB or BT.709 over default RGB, then use firmware scores. Reject timings that only offer incompatible color modes instead of selecting BT.2020 for native SDR scanout. Leave internal and legacy policy unchanged. The working J700 selected BT.2020 RGB while displaying an SDR desktop. Production-parser tests cover 65 valid, malformed, legacy and high-refresh cases. Hardware color appearance still requires qualification. [ryan: split by topic: only the drm/apple part and its selftests are here; the MAINTAINERS lines go to 900-config-tooling; the tools/diagnostics scripts are dropped (aurora-wip deleted them in commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"))] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_template.c | 4 ++- drivers/gpu/drm/apple/parser.c | 44 +++++++++++++++++++++++--- drivers/gpu/drm/apple/parser.h | 2 +- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index efa81eb705cd9..b6c522f030a43 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -986,7 +986,9 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, dcp->width_mm, dcp->height_mm, dcp->notch_height, dcp->fixed_connector_type == - DRM_MODE_CONNECTOR_eDP); + DRM_MODE_CONNECTOR_eDP, + dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL); if (IS_ERR(dcp->modes)) { dev_warn(dcp->dev, "failed to parse modes\n"); diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 415617d4875db..b8d89a913317c 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -390,10 +390,32 @@ static int fill_color_mode(struct dcp_color_mode *color, return 0; } +/* External SDR scanout must not select a wide-gamut mode by score alone. */ +static unsigned int external_sdr_rank(const struct color_mode *color) +{ + if (color->id < 0 || color->id > U32_MAX || + color->eotf != DCP_EOTF_SDR_GAMMA || + color->pixel_encoding != DCP_COLOR_FORMAT_RGB || + color->dynamic_range != DCP_COLOR_YCBCR_RANGE_FULL || + (color->depth != 8 && color->depth != 10)) + return 0; + + switch (color->colorimetry) { + case DCP_COLORIMETRY_SRGB: + case DCP_COLORIMETRY_BT709: + return 2; + case DCP_COLORIMETRY_RGB: + return 1; + default: + return 0; + } +} + static int parse_color_modes(struct dcp_parse_ctx *handle, - struct dcp_display_mode *out) + struct dcp_display_mode *out, bool external_sdr) { struct iterator outer_it; + unsigned int best_sdr_rank = 0; int ret = 0; out->sdr_444.score = -1; out->sdr_rgb.score = -1; @@ -461,6 +483,18 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, cmode.eotf, cmode.dynamic_range, cmode.pixel_encoding); + if (external_sdr) { + unsigned int rank = external_sdr_rank(&cmode); + + if (!rank || rank < best_sdr_rank) + continue; + if (rank > best_sdr_rank) + out->sdr_rgb.score = -1; + fill_color_mode(&out->sdr_rgb, &cmode); + best_sdr_rank = rank; + continue; + } + if (cmode.eotf == DCP_EOTF_SDR_GAMMA) { if (cmode.pixel_encoding == DCP_COLOR_FORMAT_RGB && cmode.depth <= 10) @@ -492,7 +526,7 @@ static u64 calculate_clock(struct dimension *horiz, struct dimension *vert) static int parse_mode(struct dcp_parse_ctx *handle, struct dcp_display_mode *out, s64 *score, int width_mm, - int height_mm, unsigned notch_height, bool internal) + int height_mm, unsigned notch_height, bool internal, bool external_sdr) { int ret = 0; struct iterator it; @@ -533,7 +567,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, else if (!strcmp(key, "MaximumVariableRefreshRate")) ret = parse_int_bound(it.handle, &max_vrr, 0, U32_MAX); else if (!strcmp(key, "ColorModes")) - ret = parse_color_modes(it.handle, out); + ret = parse_color_modes(it.handle, out, external_sdr); else if (!strcmp(key, "ID")) ret = parse_int_bound(it.handle, &id, 0, U32_MAX); else if (!strcmp(key, "IsVirtual")) @@ -645,7 +679,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, int height_mm, unsigned notch_height, - bool internal) + bool internal, bool external_sdr) { struct iterator it; int ret; @@ -670,7 +704,7 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, mode = &modes[*count]; ret = parse_mode(it.handle, mode, &score, width_mm, height_mm, - notch_height, internal); + notch_height, internal, external_sdr); /* Errors for a single mode are recoverable -- just skip it. */ if (ret) { diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index bfb0a06c8760a..1f3b8782b7bfc 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -105,7 +105,7 @@ int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx); struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, int height_mm, unsigned notch_height, - bool internal); + bool internal, bool external_sdr); int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, int *height_mm); int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, From 4c54bc0d97f405bda33d01bf300d278ecdfb43eb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:18:06 +1100 Subject: [PATCH 1196/2680] drm/apple: retrieve external H17P EDID through bounded AFKV2 inline calls Use the external DPAV endpoint's inline framing with independent request length and response capacity. Validate actual service and EDID lengths before reading them, retain timeout ownership, and queue a connector reprobe when the service appears late. Preserve the legacy padded OOB path and EDID reader, and keep internal H17P excluded. Host tests exercise the production sender, service envelope, EDID reader, ring admission and late notification with malformed replies and timeout cases. Hardware identity and color qualification remain pending. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 168 ++++++++++++++---- drivers/gpu/drm/apple/afk.h | 7 + drivers/gpu/drm/apple/afkv2-codec.h | 52 ++++++ drivers/gpu/drm/apple/epic/dpavserv-edid.h | 48 +++++ drivers/gpu/drm/apple/epic/dpavservep.c | 74 ++++++-- drivers/gpu/drm/apple/iomfb.c | 7 +- .../drivers/gpu/apple-afkv2/Makefile | 2 +- .../selftests/drivers/gpu/apple-afkv2/README | 16 ++ .../drivers/gpu/apple-afkv2/extract-edid.py | 79 ++++++++ .../drivers/gpu/apple-afkv2/extract-notify.py | 50 ++++++ .../drivers/gpu/apple-afkv2/extract-ring.py | 50 ++++++ .../drivers/gpu/apple-afkv2/extract-send.py | 118 ++++++++++++ .../gpu/apple-afkv2/extract-service.py | 86 +++++++++ .../apple-afkv2/host-include/linux/errno.h | 5 + .../drivers/gpu/apple-afkv2/inline-edid.c | 113 ++++++++++++ .../gpu/apple-afkv2/run-afkv2-tests.sh | 42 +++++ 16 files changed, 869 insertions(+), 48 deletions(-) create mode 100644 drivers/gpu/drm/apple/epic/dpavserv-edid.h create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/extract-ring.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 3fecf8d931db5..2d7fe02b99ab1 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -853,15 +853,23 @@ static void afk_v2_receive(struct apple_dcp_afkep *ep, u32 channel, u32 type, spin_lock_irqsave(&service->lock, flags); ret = -EPROTO; if (slot < MAX_PENDING_CMDS && test_bit(slot, service->cmd_map) && - !service->cmds[slot].done && response.id == service->cmds[slot].tag) - ret = afkv2_check_response(&frame, &response, service->channel, - service->cmds[slot].type, service->cmds[slot].tag, - service->cmds[slot].rxbuf_dma, - service->cmds[slot].rxlen, - service->cmds[slot].txbuf_dma, - service->cmds[slot].txlen); - if (!ret) - service->cmds[slot].reply_length = response.value ? 0 : response.rx_size; + !service->cmds[slot].done && response.id == service->cmds[slot].tag) { + struct epic_cmd_info *cmd = &service->cmds[slot]; + + if (ep->inline_v2) + ret = afkv2_check_inline_response(&frame, &response, service->channel, + cmd->type, cmd->tag, cmd->rxlen); + else + ret = afkv2_check_response(&frame, &response, service->channel, + cmd->type, cmd->tag, cmd->rxbuf_dma, + cmd->rxlen, cmd->txbuf_dma, cmd->txlen); + if (!ret) { + cmd->reply_length = response.value ? 0 : + (ep->inline_v2 ? response.inline_size : response.rx_size); + if (ep->inline_v2 && !response.value && cmd->reply_length) + memcpy(cmd->rxbuf, response.inline_data, cmd->reply_length); + } + } spin_unlock_irqrestore(&service->lock, flags); if (!ret) { legacy.retcode = cpu_to_le32(response.value); @@ -1219,20 +1227,29 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, return -EOPNOTSUPP; v2_flags = 1; } else if (ecat == EPIC_CAT_COMMAND) { - const struct epic_cmd *cmd = payload; - - if (stype != EPIC_SUBTYPE_STD_SERVICE || !cmd || - payload_len != sizeof(*cmd)) - return -EINVAL; - ret = afkv2_encode_oob_command(v2_command, sizeof(v2_command), tag, - le64_to_cpu(cmd->rxbuf), - le32_to_cpu(cmd->rxlen), - le64_to_cpu(cmd->txbuf), - le32_to_cpu(cmd->txlen)); - if (ret) - return ret; - payload = v2_command; - payload_len = sizeof(v2_command); + if (ep->inline_v2) { + struct afkv2_command command; + + if (stype != EPIC_SUBTYPE_STD_SERVICE || + afkv2_decode_command(payload, payload_len, &command) || + command.oob || command.id != tag) + return -EINVAL; + } else { + const struct epic_cmd *cmd = payload; + + if (stype != EPIC_SUBTYPE_STD_SERVICE || !cmd || + payload_len != sizeof(*cmd)) + return -EINVAL; + ret = afkv2_encode_oob_command(v2_command, sizeof(v2_command), tag, + le64_to_cpu(cmd->rxbuf), + le32_to_cpu(cmd->rxlen), + le64_to_cpu(cmd->txbuf), + le32_to_cpu(cmd->txlen)); + if (ret) + return ret; + payload = v2_command; + payload_len = sizeof(v2_command); + } v2_category = AFKV2_COMMAND; } else if (ecat == EPIC_CAT_REPLY) { v2_category = AFKV2_RESPONSE; @@ -1451,12 +1468,28 @@ int afk_send_command(struct apple_epic_service *service, u8 type, MSEC_PER_SEC); } +static int afk_send_command_timeout_len(struct apple_epic_service *service, u8 type, + const void *payload, size_t payload_len, + void *output, size_t output_len, u32 *retcode, + unsigned int timeout_ms, size_t *received); + int afk_send_command_timeout(struct apple_epic_service *service, u8 type, const void *payload, size_t payload_len, void *output, size_t output_len, u32 *retcode, unsigned int timeout_ms) +{ + return afk_send_command_timeout_len(service, type, payload, payload_len, + output, output_len, retcode, timeout_ms, NULL); +} + +static int afk_send_command_timeout_len(struct apple_epic_service *service, u8 type, + const void *payload, size_t payload_len, + void *output, size_t output_len, u32 *retcode, + unsigned int timeout_ms, size_t *received) { struct epic_cmd cmd; + void *inline_command = NULL; + size_t copy_len = output_len; void *rxbuf, *txbuf; dma_addr_t rxbuf_dma, txbuf_dma; unsigned long flags; @@ -1465,6 +1498,8 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, struct apple_dcp_afkep *ep; DECLARE_COMPLETION_ONSTACK(completion); + if (received) + *received = 0; service = afk_service_get(service); if (!service) return -ENODEV; @@ -1494,6 +1529,17 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, } memcpy(txbuf, payload, payload_len); + if (ep->inline_v2) { + if (payload_len > AFKV2_MAX_PAYLOAD - AFKV2_COMMAND_SIZE) { + ret = -E2BIG; + goto err_free_txbuf; + } + inline_command = kmalloc(AFKV2_COMMAND_SIZE + payload_len, GFP_KERNEL); + if (!inline_command) { + ret = -ENOMEM; + goto err_free_txbuf; + } + } memset(&cmd, 0, sizeof(cmd)); cmd.retcode = cpu_to_le32(0); @@ -1531,9 +1577,21 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, service->cmds[idx].completion = &completion; init_completion(&completion); - ret = afk_send_epic(service->ep, service->channel, tag, - EPIC_TYPE_COMMAND, EPIC_CAT_COMMAND, type, &cmd, - sizeof(cmd)); + if (ep->inline_v2) { + ret = afkv2_encode_inline_command(inline_command, + AFKV2_COMMAND_SIZE + payload_len, + tag, payload, payload_len, output_len); + if (!ret) + ret = afk_send_epic(ep, service->channel, tag, EPIC_TYPE_COMMAND, + EPIC_CAT_COMMAND, type, inline_command, + AFKV2_COMMAND_SIZE + payload_len); + } else { + ret = afk_send_epic(service->ep, service->channel, tag, + EPIC_TYPE_COMMAND, EPIC_CAT_COMMAND, type, &cmd, + sizeof(cmd)); + } + kfree(inline_command); + inline_command = NULL; spin_unlock_irqrestore(&service->lock, flags); if (ret) goto err_free_cmd; @@ -1565,13 +1623,17 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, ret = 0; if (retcode) *retcode = service->cmds[idx].retcode; + if (ep->inline_v2 && received) + copy_len = service->cmds[idx].reply_length; if (ep->wire_v2 && !service->cmds[idx].retcode && - service->cmds[idx].reply_length < output_len) + service->cmds[idx].reply_length < copy_len) ret = -EMSGSIZE; else if (output && output_len && (!ep->wire_v2 || !service->cmds[idx].retcode)) { if (ep->wire_v2) dma_rmb(); - memcpy(output, rxbuf, output_len); + memcpy(output, rxbuf, copy_len); + if (received) + *received = copy_len; } err_free_cmd: @@ -1580,6 +1642,8 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, bitmap_release_region(service->cmd_map, idx, 0); err_unlock: spin_unlock_irqrestore(&service->lock, flags); +err_free_txbuf: + kfree(inline_command); dma_free_coherent(ep->dcp->dev, payload_len, txbuf, txbuf_dma); err_free_rxbuf: dma_free_coherent(ep->dcp->dev, output_len, rxbuf, rxbuf_dma); @@ -1597,12 +1661,40 @@ int afk_service_call(struct apple_epic_service *service, u16 group, u32 command, output_pad, MSEC_PER_SEC); } +static int afk_service_call_timeout_len(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, unsigned int timeout_ms, + size_t *received); + +int afk_service_call_len(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, size_t *received) +{ + return afk_service_call_timeout_len(service, group, command, data, data_len, + data_pad, output, output_len, output_pad, + MSEC_PER_SEC, received); +} + int afk_service_call_timeout(struct apple_epic_service *service, u16 group, u32 command, const void *data, size_t data_len, size_t data_pad, void *output, size_t output_len, size_t output_pad, unsigned int timeout_ms) +{ + return afk_service_call_timeout_len(service, group, command, data, data_len, + data_pad, output, output_len, output_pad, + timeout_ms, NULL); +} + +static int afk_service_call_timeout_len(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, unsigned int timeout_ms, + size_t *received) { struct epic_service_call *call; + size_t wire_received = 0, result_size; void *bfr; size_t bfr_len = max(data_len + data_pad, output_len + output_pad) + sizeof(*call); @@ -1610,6 +1702,8 @@ int afk_service_call_timeout(struct apple_epic_service *service, u16 group, u32 retcode; u32 retlen; + if (received) + *received = 0; if (!service) return -ENODEV; @@ -1622,14 +1716,17 @@ int afk_service_call_timeout(struct apple_epic_service *service, u16 group, memset(call, 0, sizeof(*call)); call->group = cpu_to_le16(group); call->command = cpu_to_le32(command); - call->data_len = cpu_to_le32(data_len + data_pad); + call->data_len = cpu_to_le32(service->ep->inline_v2 ? data_len : + data_len + data_pad); call->magic = cpu_to_le32(EPIC_SERVICE_CALL_MAGIC); memcpy(bfr + sizeof(*call), data, data_len); - ret = afk_send_command_timeout(service, EPIC_SUBTYPE_STD_SERVICE, bfr, - bfr_len, bfr, bfr_len, &retcode, - timeout_ms); + /* Inline carries only the request; its response capacity is independent. */ + ret = afk_send_command_timeout_len(service, EPIC_SUBTYPE_STD_SERVICE, bfr, + service->ep->inline_v2 ? sizeof(*call) + data_len : + bfr_len, bfr, bfr_len, &retcode, timeout_ms, + received ? &wire_received : NULL); if (ret) { dev_warn(service->ep->dcp->dev, "AFK[ep:%02x]: %s (chan:%u) service call group %u cmd %u failed to complete: %d\n", @@ -1646,6 +1743,13 @@ int afk_service_call_timeout(struct apple_epic_service *service, u16 group, ret = -EINVAL; goto out; } + if (received) { + ret = afkv2_service_reply_size(bfr, wire_received, group, command, + output_len, &result_size); + if (ret) + goto out; + *received = result_size; + } if (le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || le16_to_cpu(call->group) != group || le32_to_cpu(call->command) != command) { diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index b8c889da5c356..45fe9db64d30f 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -329,6 +329,8 @@ struct apple_dcp_afkep { u32 endpoint; /* Set before afk_start; protocol selection is per endpoint. */ bool wire_v2; + /* External H17P DPAV only; other endpoints retain owned-DMA commands. */ + bool inline_v2; struct workqueue_struct *wq; struct completion started; @@ -382,4 +384,9 @@ int afk_service_call_timeout(struct apple_epic_service *service, u16 group, u32 command, const void *data, size_t data_len, size_t data_pad, void *output, size_t output_len, size_t output_pad, unsigned int timeout_ms); +int afk_service_call_len(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, size_t *received); + #endif diff --git a/drivers/gpu/drm/apple/afkv2-codec.h b/drivers/gpu/drm/apple/afkv2-codec.h index 8990182275354..9ef52b2e70ea2 100644 --- a/drivers/gpu/drm/apple/afkv2-codec.h +++ b/drivers/gpu/drm/apple/afkv2-codec.h @@ -179,6 +179,26 @@ static inline int afkv2_encode_oob_command(void *payload, size_t capacity, return 0; } +/* Inline requests carry a response capacity separately from their body. */ +static inline int afkv2_encode_inline_command(void *payload, size_t capacity, + u8 id, const void *body, + size_t size, u32 response_capacity) +{ + u8 *p = payload; + + if (!payload || (size && !body) || size > AFKV2_MAX_PAYLOAD - AFKV2_COMMAND_SIZE || + response_capacity > AFKV2_MAX_PAYLOAD) + return -EINVAL; + if (capacity < AFKV2_COMMAND_SIZE || size > capacity - AFKV2_COMMAND_SIZE) + return -ENOSPC; + if (size) + memmove(p + AFKV2_COMMAND_SIZE, body, size); + memset(p, 0, AFKV2_COMMAND_SIZE); + p[1] = id; + afkv2_put32(p + 4, response_capacity); + return 0; +} + static inline int afkv2_decode_command(const void *payload, size_t size, struct afkv2_command *out) { @@ -250,6 +270,38 @@ static inline int afkv2_check_response(const struct afkv2_frame *frame, return 0; } +/* Kept separate so DPAV inline replies cannot loosen owned-DMA checks. */ +static inline int afkv2_check_inline_response(const struct afkv2_frame *frame, + const struct afkv2_command *response, + u16 interface, u8 type, u8 id, + size_t capacity) +{ + if (!frame || !response || frame->category != AFKV2_RESPONSE || + frame->interface != interface || frame->type != type || response->id != id || + frame->token || frame->transport_flags || frame->interface_flags || + frame->reserved || response->oob || response->inline_size > capacity) + return -EPROTO; + return 0; +} + +/* The standard service envelope precedes the actual, possibly short reply. */ +static inline int afkv2_service_reply_size(const void *body, size_t received, + u16 group, u32 command, size_t capacity, + size_t *size) +{ + const u8 *p = body; + u32 length; + + if (!body || !size || received < 64 || afkv2_get32(p + 12) != 0x69706378 || + afkv2_get16(p + 2) != group || afkv2_get32(p + 4) != command) + return -EPROTO; + length = afkv2_get32(p + 8); + if (length > received - 64 || length > capacity) + return -EMSGSIZE; + *size = length; + return 0; +} + /* Reject a peer command without touching its advertised DMA buffers. */ static inline int afkv2_encode_error_reply(void *record, size_t capacity, u8 sequence, const struct afkv2_frame *request, diff --git a/drivers/gpu/drm/apple/epic/dpavserv-edid.h b/drivers/gpu/drm/apple/epic/dpavserv-edid.h new file mode 100644 index 0000000000000..31baaaea7070d --- /dev/null +++ b/drivers/gpu/drm/apple/epic/dpavserv-edid.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DPAVSERV_EDID_H +#define APPLE_DPAVSERV_EDID_H + +#include "../afkv2-codec.h" + +#define DPAV_EDID_HEADER_SIZE 48U +#define EDID_LEADING_DATA_SIZE 8U +#define EDID_BLOCK_SIZE 128U +#define EDID_MAX_SIZE 32768U +#define EDID_BUF_SIZE (EDID_LEADING_DATA_SIZE + EDID_MAX_SIZE) + +/* Validate only bytes actually returned by firmware, never zero-fill padding. */ +static inline int dpavserv_edid_size(const void *response, size_t received, + size_t *edid_size) +{ + static const u8 header[] = { 0, 255, 255, 255, 255, 255, 255, 0 }; + const u8 *p = response; + u64 used; + size_t length, offset, i; + + if (!response || !edid_size || received < DPAV_EDID_HEADER_SIZE) + return -EIO; + if (afkv2_get64(p) != EDID_BUF_SIZE) + return -EIO; + used = afkv2_get64(p + 32); + if (used < EDID_LEADING_DATA_SIZE + EDID_BLOCK_SIZE || used > EDID_BUF_SIZE || + used > received - DPAV_EDID_HEADER_SIZE) + return -EIO; + length = used - EDID_LEADING_DATA_SIZE; + p += DPAV_EDID_HEADER_SIZE + EDID_LEADING_DATA_SIZE; + if (length % EDID_BLOCK_SIZE || memcmp(p, header, sizeof(header)) || + ((size_t)p[126] + 1) * EDID_BLOCK_SIZE > length) + return -EIO; + /* Permit extra whole blocks for HF-EEODB, as drm_edid supports it. */ + for (offset = 0; offset < length; offset += EDID_BLOCK_SIZE) { + u8 sum = 0; + + for (i = 0; i < EDID_BLOCK_SIZE; i++) + sum += p[offset + i]; + if (sum) + return -EIO; + } + *edid_size = length; + return 0; +} + +#endif diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 4ddce69860379..1b4019cc159f9 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -2,6 +2,9 @@ /* Copyright The Asahi Linux Contributors */ #include "dpavservep.h" +#include "dpavserv-edid.h" + +#define EDID_EXT_BLOCK_COUNT_OFFSET 0x7E #include @@ -12,8 +15,23 @@ #include "../afk.h" #include "../dcp.h" #include "../dcp-internal.h" +#include "../connector.h" #include "../trace.h" +static void dcpavserv_notify_edid(struct apple_epic_service *service) +{ + struct apple_dcp *dcp = service->ep->dcp; + struct apple_connector *connector = READ_ONCE(dcp->connector); + + if (!service->ep->inline_v2 || READ_ONCE(dcp->quiescing) || !connector || + !READ_ONCE(dcp->nr_modes) || !READ_ONCE(connector->connected) || + READ_ONCE(connector->base.registration_state) != DRM_CONNECTOR_REGISTERED || + READ_ONCE(connector->dcp) != to_platform_device(dcp->dev)) + return; + /* get_modes may already have run before this service was announced. */ + schedule_work(&connector->hotplug_wq); +} + static void dcpavserv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { @@ -35,6 +53,11 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, service->cookie = &dcp->dcpavserv; complete(&dcp->dcpavserv.enable_completion); spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + if (service->ep->inline_v2) { + dev_info(dcp->dev, "DCPAVSERV: inline EDID service ready on channel %u\n", + service->channel); + dcpavserv_notify_edid(service); + } } } @@ -100,12 +123,6 @@ struct dpavserv_copy_edid_cmd { u8 _pad2[8]; } __packed; -#define EDID_LEADING_DATA_SIZE 8 -#define EDID_BLOCK_SIZE 128 -#define EDID_EXT_BLOCK_COUNT_OFFSET 0x7E -#define EDID_MAX_SIZE SZ_32K -#define EDID_BUF_SIZE (EDID_LEADING_DATA_SIZE + EDID_MAX_SIZE) - struct dpavserv_copy_edid_resp { __le64 max_size; u8 _pad1[24]; @@ -182,7 +199,7 @@ static int dcpdpserv_report(struct apple_epic_service *service, return parse_report(service, type, data, data_size); } -static const struct drm_edid *dcpavserv_read_edid(struct apple_epic_service *service) +static const struct drm_edid *dcpavserv_read_edid_legacy(struct apple_epic_service *service) { struct dpavserv_copy_edid_cmd cmd; struct dpavserv_copy_edid_resp *resp __free(kfree) = NULL; @@ -229,6 +246,32 @@ static const struct drm_edid *dcpavserv_read_edid(struct apple_epic_service *ser data_size - EDID_LEADING_DATA_SIZE); } +static const struct drm_edid *dcpavserv_read_edid_inline(struct apple_epic_service *service) +{ + struct dpavserv_copy_edid_cmd cmd; + struct dpavserv_copy_edid_resp *resp __free(kfree) = NULL; + size_t resp_size = sizeof(*resp) + EDID_BUF_SIZE; + size_t received, edid_size; + int ret; + + memset(&cmd, 0, sizeof(cmd)); + cmd.max_size = cpu_to_le64(EDID_BUF_SIZE); + resp = kzalloc(resp_size, GFP_KERNEL); + if (!resp) + return ERR_PTR(-ENOMEM); + + ret = afk_service_call_len(service, 1, 7, &cmd, sizeof(cmd), EDID_BUF_SIZE, + resp, resp_size, 0, &received); + if (ret < 0) + return ERR_PTR(ret); + + ret = dpavserv_edid_size(resp, received, &edid_size); + if (ret) + return ERR_PTR(ret); + + return drm_edid_alloc(resp->data + EDID_LEADING_DATA_SIZE, edid_size); +} + const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp) { struct apple_epic_service *service; @@ -236,12 +279,13 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp) unsigned long flags; spin_lock_irqsave(&dcp->dcpavserv.lock, flags); - service = afk_service_get(dcp->dcpavserv.service); + service = dcp->dcpavserv.enabled ? afk_service_get(dcp->dcpavserv.service) : NULL; spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); if (!service) return ERR_PTR(-ENODEV); - edid = dcpavserv_read_edid(service); + edid = service->ep->inline_v2 ? dcpavserv_read_edid_inline(service) : + dcpavserv_read_edid_legacy(service); spin_lock_irqsave(&dcp->dcpavserv.lock, flags); if (dcp->dcpavserv.service != service) { if (!IS_ERR(edid)) @@ -281,6 +325,10 @@ int dpavservep_init(struct apple_dcp *dcp) ep = afk_init(dcp, DPAVSERV_ENDPOINT, dpavservep_ops); if (IS_ERR(ep)) return PTR_ERR(ep); + /* Only the external H17P DPAV endpoint uses inline AFKV2 service calls. */ + ep->wire_v2 = dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + ep->inline_v2 = ep->wire_v2; dcp->dcpavservep = ep; dcp->dcpavservep->match_epic_name = true; @@ -295,8 +343,10 @@ int dpavservep_init(struct apple_dcp *dcp) ret = wait_for_completion_timeout(&dcp->dcpavserv.enable_completion, msecs_to_jiffies(1000)); - if (ret >= 0) - return 0; + if (!ret) { + dev_warn(dcp->dev, "DCPAVSERV: timed out waiting for EDID service announcement\n"); + return -ETIMEDOUT; + } - return ret; + return 0; } diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 7568d126c29c2..b1e0c4b65776e 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1058,9 +1058,10 @@ int dcp_get_modes(struct drm_connector *connector) } drm_connector_set_vrr_capable_property(connector, vrr_capable); - /* H17P sends no EPIC commands; see afk_send_epic(). */ - if (dcp->nr_modes && dcp->fw_compat != DCP_FIRMWARE_H17P && - !apple_connector->drm_edid) { + /* Internal H17P still has no supported EDID command transport. */ + if (dcp->nr_modes && !apple_connector->drm_edid && + (dcp->fw_compat != DCP_FIRMWARE_H17P || + (dcp->dcpavservep && dcp->dcpavservep->inline_v2))) { const struct drm_edid *edid; edid = dcpavserv_copy_edid(dcp); if (IS_ERR_OR_NULL(edid)) { diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile b/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile index 65780f3d5003f..df1b033d9cb32 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile @@ -4,6 +4,6 @@ export HOSTCC export AFKV2_SOURCE_ROOT := $(abspath ../../../../../..) TEST_PROGS := run-afkv2-tests.sh -TEST_FILES := codec.c owned-prefix.c owned-suffix.c extract-owned.py host-include +TEST_FILES := codec.c owned-prefix.c owned-suffix.c extract-owned.py inline-edid.c extract-service.py extract-send.py extract-edid.py extract-ring.py extract-notify.py host-include include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/README b/tools/testing/selftests/drivers/gpu/apple-afkv2/README index 42eb152d716a2..064cdcf018595 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/README +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/README @@ -22,3 +22,19 @@ DMA coherency, firmware, a graphics device or any hardware display behavior. All build products are temporary. No device node is opened and no GPU, firmware or target machine is contacted. The extractor fails if production function boundaries change, requiring review rather than silently testing a stale copy. + +The inline EDID tests cover response context, status, declared/actual lengths, +EDID header/checksum/block limits, and continued rejection of inline responses +by the OOB matcher. The actual service-call body is extracted for short reply, +error status and timeout checks; its wire sender is mocked. + +The real sender is also extracted to test a 128-byte mocked command payload +with a 32 KiB response capacity, OOB behavior, send failures and timeout +retention. The two production EDID readers are tested independently to +prove legacy checksum tolerance remains unchanged. + +The complete encoded EDID request is 160 bytes including the queue entry, +consumes 192 bytes at a 64-byte stride, and needs a 256-byte empty ring. The +ring-admission block is extracted from the real transmitter to check this. +The late-publication notifier is extracted to test asynchronous hotplug +queueing and connector-registration, quiesce, routing and legacy guards. diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py new file mode 100644 index 0000000000000..0ab0f5eda8cac --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Verify legacy EDID tolerance and stricter external-inline parsing separately.""" +import pathlib +import re +import sys + +source = pathlib.Path(sys.argv[1]).read_text() + +def function(name): + match = re.search(r"static const struct drm_edid \*" + name + r"\([^;]*?\)\n\{", source) + if not match: + raise RuntimeError("production reader missing: " + name) + end, depth = match.end(), 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[match.start():end] + +assert "service->ep->inline_v2 ? dcpavserv_read_edid_inline(service)" in source +prefix = r''' +#undef NDEBUG +#include +#include +#include +#include +#include "epic/dpavserv-edid.h" +#define GFP_KERNEL 0 +#define kzalloc(n,f) calloc(1,n) +#define cpu_to_le64(x) (x) +#define le64_to_cpu(x) (x) +#define EDID_EXT_BLOCK_COUNT_OFFSET 0x7e +#define ERR_PTR(x) ((void*)(intptr_t)(x)) +#define IS_ERR(p) ((uintptr_t)(p)>(uintptr_t)-4096) +static void cleanup_free(void *p) { free(*(void**)p); } +#define __free(x) __attribute__((cleanup(cleanup_free))) +struct apple_epic_service { int unused; }; +struct drm_edid { size_t size; }; +struct dpavserv_copy_edid_cmd { u64 max_size;u8 pad1[24];u64 used_size;u8 pad2[8]; }; +struct dpavserv_copy_edid_resp { u64 max_size;u8 pad1[24];u64 used_size;u8 pad2[8];u8 data[]; }; +static u8 fixture[48+8+256]; +static bool truncated; +static const struct drm_edid *drm_edid_alloc(const void *p,size_t n) +{ struct drm_edid *e=malloc(sizeof(*e));(void)p;e->size=n;return e; } +static int afk_service_call(struct apple_epic_service *s,u16 group,u32 command, + const void *data,size_t data_len,size_t data_pad,void *output,size_t output_len,size_t output_pad) +{ + const struct dpavserv_copy_edid_cmd *cmd=data; + (void)s;(void)data_pad;(void)output_pad; + assert(group==1 && command==7 && data_len==48 && cmd->max_size==EDID_BUF_SIZE); + assert(output_len>=sizeof(fixture));memcpy(output,fixture,sizeof(fixture));return 0; +} +static int afk_service_call_len(struct apple_epic_service *s,u16 group,u32 command, + const void *data,size_t data_len,size_t data_pad,void *output,size_t output_len, + size_t output_pad,size_t *received) +{ int ret=afk_service_call(s,group,command,data,data_len,data_pad,output,output_len,output_pad); + *received=sizeof(fixture)-(truncated?1:0);return ret; } +static void fix_checksum(u8 *p) +{ unsigned int i;u8 sum=0;p[127]=0;for(i=0;i<127;i++)sum+=p[i];p[127]=-sum; } +''' +suffix = r''' +int main(void) +{ + struct apple_epic_service service={0};const struct drm_edid *e; + u8 *base=fixture+56,*extension=base+128; + afkv2_put64(fixture,EDID_BUF_SIZE);afkv2_put64(fixture+32,8+256); + memset(base+1,255,6);base[126]=1;fix_checksum(base); + extension[0]=2; /* deliberately invalid extension checksum */ + e=dcpavserv_read_edid_legacy(&service);assert(!IS_ERR(e) && e->size==256);free((void*)e); + e=dcpavserv_read_edid_inline(&service);assert(IS_ERR(e)); + fix_checksum(extension); + e=dcpavserv_read_edid_inline(&service);assert(!IS_ERR(e) && e->size==256);free((void*)e); + truncated=true; + e=dcpavserv_read_edid_inline(&service);assert(IS_ERR(e)); + e=dcpavserv_read_edid_legacy(&service);assert(!IS_ERR(e));free((void*)e); + puts("PASS: actual legacy EDID checksum tolerance preserved; inline checksum/actual-length validation enforced"); +} +''' +pathlib.Path(sys.argv[2]).write_text(prefix + function("dcpavserv_read_edid_legacy") + function("dcpavserv_read_edid_inline") + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py new file mode 100644 index 0000000000000..26579f8399cb1 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise actual late-EDID publication notification guards with mock work.""" +import pathlib +import re +import sys +source = pathlib.Path(sys.argv[1]).read_text() +match = re.search(r"static void dcpavserv_notify_edid\([^;]*?\)\n\{", source) +if not match: + raise RuntimeError("EDID notification helper missing") +end, depth = match.end(), 1 +while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 +prefix = r''' +#undef NDEBUG +#include +#include +#include +#include +#define READ_ONCE(x) (x) +#define DRM_CONNECTOR_REGISTERED 1 +struct base { int registration_state; }; +struct apple_connector { bool connected;struct base base;void *dcp;int hotplug_wq; }; +struct apple_dcp { struct apple_connector *connector;bool quiescing;unsigned int nr_modes;void *dev; }; +struct ep { struct apple_dcp *dcp;bool inline_v2; }; +struct apple_epic_service { struct ep *ep; }; +static unsigned int queued; +static void *to_platform_device(void *p) { return p; } +static void schedule_work(int *w) { (*w)++;queued++; } +''' +suffix = r''' +int main(void) +{ + int device,other;struct apple_dcp dcp={.dev=&device}; + struct ep ep={.dcp=&dcp,.inline_v2=true};struct apple_epic_service service={.ep=&ep}; + struct apple_connector connector={.dcp=&device}; + dcpavserv_notify_edid(&service);assert(queued==0); + dcp.connector=&connector;dcpavserv_notify_edid(&service);assert(queued==0); + dcp.nr_modes=1;connector.connected=true;dcpavserv_notify_edid(&service);assert(queued==0); + connector.base.registration_state=DRM_CONNECTOR_REGISTERED; + dcpavserv_notify_edid(&service);assert(queued==1 && connector.hotplug_wq==1); + dcp.quiescing=true;dcpavserv_notify_edid(&service);assert(queued==1); + dcp.quiescing=false;connector.dcp=&other;dcpavserv_notify_edid(&service);assert(queued==1); + connector.dcp=&device;ep.inline_v2=false;dcpavserv_notify_edid(&service);assert(queued==1); + ep.inline_v2=true;connector.connected=false;dcpavserv_notify_edid(&service);assert(queued==1); + puts("PASS: actual late-EDID notification queues ready connector only; legacy/quiesce/unrouted/unregistered guarded"); +} +''' +pathlib.Path(sys.argv[2]).write_text(prefix + source[match.start():end] + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-ring.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-ring.py new file mode 100644 index 0000000000000..65d6fdeca7a54 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-ring.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Check full AFKV2 frame size against the production ring-admission block.""" +import pathlib +import sys + +source = pathlib.Path(sys.argv[1]).read_text() +start = source.index("\ttotal_epic_size = sizeof(*ehdr) +", source.index("static int afk_send_epic_token", source.index("int afk_receive_message"))) +end = source.index("\n\thdr = hdr2 = NULL;", start) +block = source[start:end] +prefix = r''' +#undef NDEBUG +#include +#include +#include "afkv2-codec.h" +#define IS_ALIGNED(x,a) (!((x)&((a)-1))) +#define ALIGN(x,a) (((x)+(a)-1)&~((a)-1)) +struct epic_hdr { u8 data[16]; }; +struct epic_sub_hdr { u8 data[24]; }; +struct epic_sub_hdr_h17p { u8 data[8]; }; +struct afk_qe { u8 data[16]; }; +struct ring { bool ready; u32 stride,bufsz; }; +struct ep { bool wire_v2; struct ring txbfr; }; +static int check_ring(struct ep *ep,u32 rptr,u32 wptr,size_t payload_len,size_t *total) +{ + struct epic_hdr *ehdr;struct epic_sub_hdr *eshdr;struct afk_qe *hdr; + bool compact=false;size_t total_epic_size,total_size;int ret=0; +''' +suffix = r''' + *total=total_size; +out: + return ret; +} +int main(void) +{ + u8 request[112]={0},command[120],frame[144];size_t written,total; + struct ep ep={.wire_v2=true,.txbfr={.ready=true,.stride=64,.bufsz=256}}; + assert(!afkv2_encode_inline_command(command,sizeof(command),1,request,sizeof(request),32888)); + assert(!afkv2_encode_frame(frame,sizeof(frame),0,1,0,0xc0,AFKV2_COMMAND,0,command,sizeof(command),&written)); + assert(written==144 && written+sizeof(struct afk_qe)==160); + assert(!check_ring(&ep,0,0,sizeof(command),&total)); + assert(total==160 && ALIGN(total,64)==192); + ep.txbfr.bufsz=192;assert(check_ring(&ep,0,0,sizeof(command),&total)==-ENOSPC); + ep.txbfr.bufsz=128;assert(check_ring(&ep,0,0,sizeof(command),&total)==-EPROTO); + ep.txbfr.bufsz=256;assert(check_ring(&ep,192,0,sizeof(command),&total)==-ENOSPC); + assert(check_ring(&ep,0,1,sizeof(command),&total)==-EPROTO); + puts("PASS: full request 160 bytes / 192-byte aligned ring consumption; 256-byte empty ring accepted, smaller capacity rejected"); +} +''' +pathlib.Path(sys.argv[2]).write_text(prefix + block + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py new file mode 100644 index 0000000000000..92ca8857cc104 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile the real command sender against synthetic ring and DMA operations.""" +import pathlib +import re +import sys + +source = pathlib.Path(sys.argv[1]).read_text() +match = re.search(r"static int afk_send_command_timeout_len\([^;]*?\)\n\{", source) +if not match: + raise RuntimeError("production command sender not found") +start, end, depth = match.start(), match.end(), 1 +while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 +prefix = r''' +#undef NDEBUG +#include +#include +#include +#include +#include "afkv2-codec.h" +#define SZ_1M 1048576U +#define GFP_KERNEL 0 +#define MAX_PENDING_CMDS 16 +#define cpu_to_le32(x) (x) +#define cpu_to_le64(x) (x) +#define kmalloc(n,f) malloc(n) +#define kfree(p) free(p) +#define READ_ONCE(x) (x) +#define spin_lock_irqsave(l,f) ((void)(l),(f)=0) +#define spin_unlock_irqrestore(l,f) ((void)(l),(void)(f)) +#define dma_rmb() do {} while(0) +#define dev_warn(...) do {} while(0) +#define EPIC_TYPE_COMMAND 3 +#define EPIC_CAT_COMMAND 0x30 +#define DECLARE_COMPLETION_ONSTACK(n) int n=0 +#define init_completion(p) (*(p)=0) +#define msecs_to_jiffies(x) (x) +#define wait_for_completion_timeout(p,t) ((void)(t),*(p)) +typedef u64 dma_addr_t; +struct epic_cmd { u32 retcode; u64 rxbuf,txbuf; u32 rxlen,txlen; u8 rxcookie,txcookie; }; +struct cmd_info { + u16 tag; u8 type; size_t reply_length; void *rxbuf,*txbuf; + dma_addr_t rxbuf_dma,txbuf_dma; size_t rxlen,txlen; u32 retcode; + bool done,free_on_ack; int *completion; +}; +struct dcp { void *dev; bool quiescing; }; +struct apple_dcp_afkep { struct dcp *dcp; bool wire_v2,inline_v2; int endpoint; }; +struct ops { const char *name; }; +struct apple_epic_service { + struct apple_dcp_afkep *ep; struct ops *ops; unsigned int channel; unsigned long cmd_map[1]; + int lock; bool enabled,torndown; u8 cmd_tag; struct cmd_info cmds[16]; +}; +static struct apple_epic_service *active; +static unsigned int frees, sends; +static int wire_fail, timeout, reply_status; +static struct apple_epic_service *afk_service_get(struct apple_epic_service *s) { return s->enabled?s:NULL; } +static void afk_service_put(struct apple_epic_service *s) { (void)s; } +static void *dma_alloc_coherent(void *dev,size_t n,dma_addr_t *address,int flags) +{ void *p=calloc(1,n);(void)dev;(void)flags;*address=(uintptr_t)p;return p; } +static void dma_free_coherent(void *dev,size_t n,void *p,dma_addr_t a) +{ (void)dev;(void)n;assert((uintptr_t)p==a);free(p);frees++; } +static int bitmap_find_free_region(unsigned long *map,unsigned int n,int order) +{ (void)n;(void)order;assert(!*map);*map=1;return 0; } +static void bitmap_release_region(unsigned long *map,int i,int order) +{ (void)i;(void)order;*map=0; } +static int afk_send_epic(struct apple_dcp_afkep *ep,u32 channel,u16 tag,int type,int category,u8 subtype,const void *payload,size_t length) +{ + struct cmd_info *c=&active->cmds[0];struct afkv2_command parsed; + assert(channel==4 && type==3 && category==0x30 && subtype==0xc0); + sends++; + /* The payload-envelope mock bounds the command separately from its response. */ + assert(length<=128); + if(ep->inline_v2) { + assert(length==120); + assert(!afkv2_decode_command(payload,length,&parsed)); + assert(!parsed.oob && parsed.id==tag && parsed.value==32888 && parsed.inline_size==112); + assert(parsed.inline_data[0]==0x5a); + } else { + const struct epic_cmd *legacy=payload; + assert(length==sizeof(*legacy) && legacy->txlen==112 && legacy->rxlen==32888); + } + if(wire_fail)return -ENOSPC; + if(timeout)return 0; + c->reply_length=ep->inline_v2?4:32888;c->retcode=reply_status; + memset(c->rxbuf,0x77,c->reply_length);c->done=true;*c->completion=1; + return 0; +} +''' +suffix = r''' +int main(void) +{ + struct dcp dcp={0};struct apple_dcp_afkep ep={.dcp=&dcp,.wire_v2=true,.inline_v2=true}; + struct ops ops={.name="test"};struct apple_epic_service s={.ep=&ep,.ops=&ops,.channel=4,.enabled=true}; + u8 request[112]={0x5a},*output=calloc(1,32888);size_t received;u32 status=99; + active=&s; + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(received==4 && status==0 && output[0]==0x77 && frees==2 && sends==1 && !s.cmd_map[0]); + /* Existing strict callers still reject a short inline result. */ + assert(afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,NULL)==-EMSGSIZE); + reply_status=5;received=99; + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(status==5 && received==0);reply_status=0; + wire_fail=1; + assert(afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)==-ENOSPC && !s.cmd_map[0]); + wire_fail=0;ep.inline_v2=false; + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(received==32888 && status==0); + ep.inline_v2=true;timeout=1; + assert(afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)==-ETIMEDOUT); + assert(s.cmd_map[0] && s.cmds[0].free_on_ack && !s.cmds[0].completion); + /* Mock the late-reply owner's release, avoiding leaks in this host test. */ + free(s.cmds[0].rxbuf);free(s.cmds[0].txbuf);free(output); + puts("PASS: extracted production sender, 128-byte command envelope / 32888-byte reply, strict OOB and timeout ownership"); +} +''' +pathlib.Path(sys.argv[2]).write_text(prefix + source[start:end] + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py new file mode 100644 index 0000000000000..68e3f5e6ba980 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the production service-call body with a mocked wire reply.""" +import pathlib +import re +import sys + +source = pathlib.Path(sys.argv[1]).read_text() +match = re.search(r"static int afk_service_call_timeout_len\([^;]*?\)\n\{", source) +if not match: + raise RuntimeError("production service-call definition not found") +start = match.start() +end = match.end() +depth = 1 +while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 +prefix = r''' +#undef NDEBUG +#include +#include +#include +#include "afkv2-codec.h" +#define GFP_KERNEL 0 +#define kzalloc(n,f) calloc(1,n) +#define kfree(p) free(p) +#define max(a,b) ((a)>(b)?(a):(b)) +#define cpu_to_le16(x) (x) +#define cpu_to_le32(x) (x) +#define le16_to_cpu(x) (x) +#define le32_to_cpu(x) (x) +#define EPIC_SERVICE_CALL_MAGIC 0x69706378 +#define EPIC_SUBTYPE_STD_SERVICE 0xc0 +#define dev_warn(...) do {} while(0) +#define dev_dbg(...) do {} while(0) +struct epic_service_call { + u8 pad0[2]; u16 group; u32 command, data_len, magic; u8 pad1[48]; +}; +struct dcp { void *dev; }; +struct ops { const char *name; }; +struct ep { struct dcp *dcp; int endpoint; bool inline_v2; }; +struct apple_epic_service { struct ep *ep; struct ops *ops; u32 channel; }; +static int error_status, malformed, short_header, too_long, wire_error; +static int afk_send_command_timeout_len(struct apple_epic_service *s, u8 type, + const void *payload, size_t payload_len, void *output, size_t output_len, + u32 *retcode, unsigned int timeout_ms, size_t *received) +{ + struct epic_service_call *call=output; + (void)payload; + assert(payload_len==(s->ep->inline_v2?68:output_len)); + assert(call->data_len==(s->ep->inline_v2?4:4100)); + assert(type==0xc0 && timeout_ms==1000 && output_len>=68); + if(wire_error) return -ETIMEDOUT; + call->group=1;call->command=7;call->magic=EPIC_SERVICE_CALL_MAGIC; + call->data_len=too_long?5:4; + ((u8*)output)[64]=0xde;((u8*)output)[65]=0xad; + ((u8*)output)[66]=0xbe;((u8*)output)[67]=0xef; + if(malformed)call->command=8; + *retcode=error_status; + if(received)*received=short_header?63:68; + return 0; +} +''' +suffix = r''' +int main(void) +{ + struct ep ep={.inline_v2=true};struct apple_epic_service service={.ep=&ep}; + u8 input[4]={0}, output[128]={0}; size_t received=999; + assert(!afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)); + assert(received==4 && output[0]==0xde && output[3]==0xef && output[4]==0); + error_status=5;received=999; + assert(afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)==-EINVAL && received==0); + error_status=0;malformed=1; + assert(afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)); + malformed=0;short_header=1; + assert(afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)); + short_header=0;too_long=1; + assert(afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)); + too_long=0;wire_error=1; + assert(afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)==-ETIMEDOUT); + wire_error=0; ep.inline_v2=false; + assert(!afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,NULL)); + puts("PASS: extracted production service-call actual request / legacy padding / short reply / status / context / length tests"); +} +''' +pathlib.Path(sys.argv[2]).write_text(prefix + source[start:end] + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h index a5c3a3f44a591..23e11801f1e76 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h @@ -3,3 +3,8 @@ #define ENOSPC 28 #define E2BIG 7 #define EPROTO 71 +#define EIO 5 +#define EMSGSIZE 90 +#define ENODEV 19 +#define ETIMEDOUT 110 +#define ENOMEM 12 diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c new file mode 100644 index 0000000000000..7ce16fd2b6a8b --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#undef NDEBUG +#include +#include +#include "epic/dpavserv-edid.h" + +static unsigned int assertions; +#define CHECK(x) do { assertions++; assert(x); } while (0) + +static void checksum(u8 *block) +{ + u8 sum = 0; + unsigned int i; + + block[127] = 0; + for (i = 0; i < 127; i++) + sum += block[i]; + block[127] = -sum; +} + +int main(void) +{ + u8 command[192], packet[216], body[136] = {0}; + u8 edid[DPAV_EDID_HEADER_SIZE + 8 + 3 * 128] = {0}; + struct afkv2_frame frame; + struct afkv2_command reply; + size_t written, length = 0; + u8 *base = edid + DPAV_EDID_HEADER_SIZE + 8; + + CHECK(!afkv2_encode_inline_command(command, sizeof(command), 0x91, + body, sizeof(body), 400)); + CHECK(command[0] == 0 && command[1] == 0x91); + CHECK(afkv2_get32(command + 4) == 400); + CHECK(!afkv2_encode_frame(packet, sizeof(packet), 5, 4, 0, 0xc0, + AFKV2_RESPONSE, 0, command, 8 + sizeof(body), &written)); + /* Response uses status in the request-capacity field. */ + afkv2_put32(packet + AFKV2_HEADER_SIZE + 4, 0); + CHECK(!afkv2_decode_frame(packet, written, false, &frame)); + CHECK(!afkv2_decode_command(frame.payload, frame.payload_size, &reply)); + CHECK(!afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 135)); + CHECK(afkv2_check_inline_response(&frame, &reply, 5, 0xc0, 0x91, 136)); + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x92, 136)); + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc1, 0x91, 136)); + frame.category = AFKV2_COMMAND; + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + frame.category = AFKV2_RESPONSE; + frame.token = 1; + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + frame.token = 0; + frame.reserved = 1; + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + frame.reserved = 0; + reply.oob = true; + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + reply.oob = false; + /* The original OOB matcher must continue rejecting inline replies. */ + CHECK(afkv2_check_response(&frame, &reply, 4, 0xc0, 0x91, 0, 136, 0, 0)); + reply.value = 7; + CHECK(!afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + CHECK(reply.value != 0); /* Error status is preserved for the caller. */ + CHECK(afkv2_encode_inline_command(command, 7, 1, body, 1, 1)); + CHECK(afkv2_encode_inline_command(command, sizeof(command), 1, NULL, 1, 1)); + CHECK(afkv2_encode_inline_command(command, sizeof(command), 1, body, 1, + AFKV2_MAX_PAYLOAD + 1)); + + afkv2_put16(body + 2, 1); + afkv2_put32(body + 4, 7); + afkv2_put32(body + 8, 72); + afkv2_put32(body + 12, 0x69706378); + CHECK(!afkv2_service_reply_size(body, 136, 1, 7, 72, &length) && length == 72); + CHECK(afkv2_service_reply_size(body, 135, 1, 7, 72, &length)); + CHECK(afkv2_service_reply_size(body, 136, 1, 7, 71, &length)); + CHECK(afkv2_service_reply_size(body, 63, 1, 7, 72, &length)); + CHECK(afkv2_service_reply_size(body, 136, 2, 7, 72, &length)); + CHECK(afkv2_service_reply_size(body, 136, 1, 8, 72, &length)); + body[12] ^= 1; + CHECK(afkv2_service_reply_size(body, 136, 1, 7, 72, &length)); + + afkv2_put64(edid, EDID_BUF_SIZE); + afkv2_put64(edid + 32, 8 + 128); + memset(base + 1, 255, 6); + base[18] = 1; + base[19] = 4; + checksum(base); + CHECK(!dpavserv_edid_size(edid, 48 + 8 + 128, &length) && length == 128); + CHECK(dpavserv_edid_size(edid, 47, &length)); + CHECK(dpavserv_edid_size(edid, 48 + 8 + 127, &length)); + base[0] = 1; + CHECK(dpavserv_edid_size(edid, sizeof(edid), &length)); + base[0] = 0; + base[24] ^= 1; + CHECK(dpavserv_edid_size(edid, sizeof(edid), &length)); + base[24] ^= 1; + base[126] = 1; + checksum(base); + CHECK(dpavserv_edid_size(edid, sizeof(edid), &length)); + afkv2_put64(edid + 32, 8 + 256); + CHECK(!dpavserv_edid_size(edid, 48 + 8 + 256, &length) && length == 256); + /* HF-EEODB can supply more complete blocks than base[126] declares. */ + base[126] = 0; + checksum(base); + CHECK(!dpavserv_edid_size(edid, sizeof(edid), &length) && length == 256); + afkv2_put64(edid + 32, 8 + 255); + CHECK(dpavserv_edid_size(edid, sizeof(edid), &length)); + afkv2_put64(edid + 32, EDID_BUF_SIZE + 1); + CHECK(dpavserv_edid_size(edid, sizeof(edid), &length)); + afkv2_put64(edid + 32, 8 + 128); + afkv2_put64(edid, EDID_BUF_SIZE - 1); + CHECK(dpavserv_edid_size(edid, sizeof(edid), &length)); + printf("PASS: inline AFKV2 and EDID validation (%u checks)\n", assertions); + return 0; +} diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh b/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh index 4e13f6d05d986..4348eebc78246 100755 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh @@ -38,3 +38,45 @@ $afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ "$afkv2_tmp/owned.c" -o "$afkv2_tmp/owned" "$afkv2_tmp/owned" + +# Inline DPAV transport and actual-length EDID response validation. +# shellcheck disable=SC2086 +$afkv2_cc -std=c11 -Wall -Wextra -Werror -pedantic ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_dir/inline-edid.c" -o "$afkv2_tmp/inline-edid" +"$afkv2_tmp/inline-edid" +"$afkv2_python" "$afkv2_dir/extract-service.py" \ + "$afkv2_root/drivers/gpu/drm/apple/afk.c" "$afkv2_tmp/service.c" +# shellcheck disable=SC2086 +$afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_tmp/service.c" -o "$afkv2_tmp/service" +"$afkv2_tmp/service" + +"$afkv2_python" "$afkv2_dir/extract-send.py" \ + "$afkv2_root/drivers/gpu/drm/apple/afk.c" "$afkv2_tmp/send.c" +# shellcheck disable=SC2086 +$afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_tmp/send.c" -o "$afkv2_tmp/send" +"$afkv2_tmp/send" + +"$afkv2_python" "$afkv2_dir/extract-edid.py" \ + "$afkv2_root/drivers/gpu/drm/apple/epic/dpavservep.c" "$afkv2_tmp/edid.c" +# shellcheck disable=SC2086 +$afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_tmp/edid.c" -o "$afkv2_tmp/edid" +"$afkv2_tmp/edid" + +"$afkv2_python" "$afkv2_dir/extract-ring.py" \ + "$afkv2_root/drivers/gpu/drm/apple/afk.c" "$afkv2_tmp/ring.c" +"$afkv2_python" "$afkv2_dir/extract-notify.py" \ + "$afkv2_root/drivers/gpu/drm/apple/epic/dpavservep.c" "$afkv2_tmp/notify.c" +for afkv2_test in ring notify; do + # shellcheck disable=SC2086 + $afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_tmp/$afkv2_test.c" -o "$afkv2_tmp/$afkv2_test" + "$afkv2_tmp/$afkv2_test" +done From 6a960317369b2540468c392820d607844c29a256 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:39:37 +1100 Subject: [PATCH 1197/2680] drm/apple: refine inline reply validation and trace silent DPAV rings Treat the transport/interface/reserved header fields as opaque, matching captured firmware responses and the owned-DMA validator, while retaining identity, type, tag, category and payload-bound checks. Add an optional read-only ring snapshot at the DPAV announcement timeout; it does not consume data or send another command. Host tests preserve response mismatch rejection, short-reply checks and timeout ownership, and verify the snapshot's enable, readiness, stop and pointer guards. [ryan: carried without the DPAV ring-snapshot tracepoint, which aurora-wip removed again with the other external qualification tracepoints in merge commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate")] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afkv2-codec.h | 7 ++++--- .../selftests/drivers/gpu/apple-afkv2/inline-edid.c | 12 ++++++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/afkv2-codec.h b/drivers/gpu/drm/apple/afkv2-codec.h index 9ef52b2e70ea2..a06a1efd5fc7e 100644 --- a/drivers/gpu/drm/apple/afkv2-codec.h +++ b/drivers/gpu/drm/apple/afkv2-codec.h @@ -270,7 +270,9 @@ static inline int afkv2_check_response(const struct afkv2_frame *frame, return 0; } -/* Kept separate so DPAV inline replies cannot loosen owned-DMA checks. */ +/* Kept separate so DPAV inline replies cannot loosen owned-DMA checks. + * Transport/interface flags and the reserved word are opaque firmware data. + */ static inline int afkv2_check_inline_response(const struct afkv2_frame *frame, const struct afkv2_command *response, u16 interface, u8 type, u8 id, @@ -278,8 +280,7 @@ static inline int afkv2_check_inline_response(const struct afkv2_frame *frame, { if (!frame || !response || frame->category != AFKV2_RESPONSE || frame->interface != interface || frame->type != type || response->id != id || - frame->token || frame->transport_flags || frame->interface_flags || - frame->reserved || response->oob || response->inline_size > capacity) + frame->token || response->oob || response->inline_size > capacity) return -EPROTO; return 0; } diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c index 7ce16fd2b6a8b..b711adf804794 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c @@ -48,8 +48,16 @@ int main(void) frame.token = 1; CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); frame.token = 0; - frame.reserved = 1; - CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + /* R30 responses carry these opaque nonzero header fields. */ + frame.transport_flags = 0x18; + frame.interface_flags = 0x0400; + frame.reserved = 0x3607ebe0; + CHECK(!afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + /* Context and bounds remain mandatory even with opaque header values. */ + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x92, 136)); + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 135)); + frame.transport_flags = 0; + frame.interface_flags = 0; frame.reserved = 0; reply.oob = true; CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); From cb24595826987e9fb30e30172cc553a4ea467246 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:42:08 +1100 Subject: [PATCH 1198/2680] drm/apple: honor external timing color exclusions before SDR selection Exclude firmware-listed unsafe and DSC-required color IDs before choosing an external H17P SDR candidate. Read restrictions independently of property order and reject malformed or duplicate lists instead of silently treating them as empty. Preserve internal and legacy parsing. The captured dock route permits 60 Hz at 10-bit and 120 Hz at 8-bit; its 144/180 Hz timings have no eligible nonvirtual SDR candidate after exclusions. This is metadata admission, not a new physical-link qualification or a global refresh cap. [ryan: split by topic: only the drm/apple part and its selftests are here; the tools/diagnostics scripts are dropped (aurora-wip deleted them in commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"))] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/parser.c | 104 ++++++++++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index b8d89a913317c..cc97258307427 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -411,8 +411,92 @@ static unsigned int external_sdr_rank(const struct color_mode *color) } } +/* Firmware lists color element IDs which this timing cannot safely use. */ +#define DCP_TIMING_COLOR_ID_LIMIT 64 + +struct timing_color_ids { + u32 ids[DCP_TIMING_COLOR_ID_LIMIT]; + unsigned int count; +}; + +struct timing_color_restrictions { + struct timing_color_ids unsafe; + struct timing_color_ids dsc_required; +}; + +static bool timing_color_contains(const struct timing_color_ids *ids, u32 id) +{ + unsigned int i; + + for (i = 0; i < ids->count; i++) + if (ids->ids[i] == id) + return true; + return false; +} + +static int parse_timing_color_ids(struct dcp_parse_ctx *handle, + struct timing_color_ids *ids) +{ + struct iterator it; + int ret; + + ret = iterator_begin(handle, &it, false); + if (ret) + return ret; + if (it.len > DCP_TIMING_COLOR_ID_LIMIT) + return -E2BIG; + for (; it.idx < it.len; it.idx++) { + s64 id; + + ret = parse_int_bound(handle, &id, 0, U32_MAX); + if (ret) + return ret; + if (timing_color_contains(ids, id)) + return -EINVAL; + ids->ids[ids->count++] = id; + } + return 0; +} + +static int parse_timing_color_restrictions(struct dcp_parse_ctx *handle, + struct timing_color_restrictions *limits) +{ + struct iterator it; + bool unsafe_seen = false, dsc_seen = false, colors_seen = false; + int ret; + + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + for (; it.idx < it.len; it.idx++) { + char *key = parse_string(handle); + + if (IS_ERR(key)) + return PTR_ERR(key); + if (!strcmp(key, "UnsafeColorElementIDs")) { + ret = unsafe_seen ? -EINVAL : + parse_timing_color_ids(handle, &limits->unsafe); + unsafe_seen = true; + } else if (!strcmp(key, "DSCRequiredColorElementIDs")) { + ret = dsc_seen ? -EINVAL : + parse_timing_color_ids(handle, &limits->dsc_required); + dsc_seen = true; + } else if (!strcmp(key, "ColorModes")) { + ret = colors_seen ? -EINVAL : skip(handle); + colors_seen = true; + } else { + ret = skip(handle); + } + kfree(key); + if (ret) + return ret; + } + return 0; +} + static int parse_color_modes(struct dcp_parse_ctx *handle, - struct dcp_display_mode *out, bool external_sdr) + struct dcp_display_mode *out, bool external_sdr, + const struct timing_color_restrictions *limits) { struct iterator outer_it; unsigned int best_sdr_rank = 0; @@ -484,8 +568,13 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, cmode.pixel_encoding); if (external_sdr) { - unsigned int rank = external_sdr_rank(&cmode); + unsigned int rank; + if (cmode.id <= U32_MAX && + (timing_color_contains(&limits->unsafe, cmode.id) || + timing_color_contains(&limits->dsc_required, cmode.id))) + continue; + rank = external_sdr_rank(&cmode); if (!rank || rank < best_sdr_rank) continue; if (rank > best_sdr_rank) @@ -531,6 +620,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, int ret = 0; struct iterator it; struct dimension horiz = { .active = -1 }, vert = { .active = -1 }; + struct timing_color_restrictions limits = {}; s64 min_vrr = 0, max_vrr = 0; u64 clock; s64 id = -1; @@ -547,6 +637,14 @@ static int parse_mode(struct dcp_parse_ctx *handle, }; *score = -1; + if (external_sdr) { + struct dcp_parse_ctx scan = *handle; + + /* ColorModes can precede either list: collect restrictions first. */ + ret = parse_timing_color_restrictions(&scan, &limits); + if (ret) + return ret; + } ret = iterator_begin(handle, &it, true); if (ret) return ret; @@ -567,7 +665,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, else if (!strcmp(key, "MaximumVariableRefreshRate")) ret = parse_int_bound(it.handle, &max_vrr, 0, U32_MAX); else if (!strcmp(key, "ColorModes")) - ret = parse_color_modes(it.handle, out, external_sdr); + ret = parse_color_modes(it.handle, out, external_sdr, &limits); else if (!strcmp(key, "ID")) ret = parse_int_bound(it.handle, &id, 0, U32_MAX); else if (!strcmp(key, "IsVirtual")) From 5becfc767b8b5ba7258d9610c136ababd16905b7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:49:18 +1100 Subject: [PATCH 1199/2680] drm/apple: derive bounded external monitor metadata for initial attachment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expose the firmware-reported identity, primaries and preferred timing while raw DPAV EDID is unavailable on the experimental H17P external path. Keep this synthetic base block out of the raw cache, require complete metadata from the same initial connection, and invalidate it on route or cable loss. Do not synthesize extra timings, audio, HDR, VRR or link capabilities. Serialize EDID property and cache changes with connector probing. Defer commit-context disconnect cleanup through a persistent invalidation flag, so rapid reconnection cannot lose it or invert DRM modeset lock ordering. Include bounded-parser, independent EDID decoding and concurrent lifetime tests. Hardware qualification remains pending. [ryan: adapted to 7.2.10 — Asahi's IsPreferred score bonus stays for modes outside the external SDR path; the conflicts with the lifecycle line are resolved as merge commit 9f742ddfa628 ("Merge qualified output metadata into the external lifecycle candidate") resolved them] [ryan: split by topic: only the drm/apple part and its selftests are here; the MAINTAINERS lines go to 900-config-tooling; the tools/diagnostics scripts are dropped (aurora-wip deleted them in commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"))] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Makefile | 2 +- drivers/gpu/drm/apple/connector.h | 3 + drivers/gpu/drm/apple/dcp-display-metadata.c | 204 ++++++++++++++ drivers/gpu/drm/apple/dcp-display-metadata.h | 24 ++ drivers/gpu/drm/apple/dcp-internal.h | 23 ++ drivers/gpu/drm/apple/dcp-metadata-epoch.h | 48 ++++ drivers/gpu/drm/apple/dcp.c | 183 +++++++++++- drivers/gpu/drm/apple/iomfb.c | 66 ++++- drivers/gpu/drm/apple/iomfb_template.c | 20 +- drivers/gpu/drm/apple/parser.c | 276 ++++++++++++++++++- drivers/gpu/drm/apple/parser.h | 6 + 11 files changed, 838 insertions(+), 17 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp-display-metadata.c create mode 100644 drivers/gpu/drm/apple/dcp-display-metadata.h create mode 100644 drivers/gpu/drm/apple/dcp-metadata-epoch.h diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 8b4a53838eefe..0acdc4a7b127f 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -6,7 +6,7 @@ appledrm-y := apple_drv.o appledrm-y += afk.o dcp.o dcp_backlight.o dptxep.o iomfb.o parser.o systemep.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += audio.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += av.o -appledrm-y += connector.o +appledrm-y += connector.o dcp-display-metadata.o appledrm-y += ibootep.o appledrm-y += iomfb_v12_3.o appledrm-y += iomfb_v13_3.o diff --git a/drivers/gpu/drm/apple/connector.h b/drivers/gpu/drm/apple/connector.h index 4cb58502c1ef8..6f73e975ebdc1 100644 --- a/drivers/gpu/drm/apple/connector.h +++ b/drivers/gpu/drm/apple/connector.h @@ -43,6 +43,9 @@ struct apple_connector { u32 candidate_crtcs; const struct drm_edid *drm_edid; + bool drm_edid_firmware_derived; + bool edid_invalidation_pending; + u64 drm_edid_generation; /* Workqueue for sending hotplug events to the associated device */ struct work_struct hotplug_wq; diff --git a/drivers/gpu/drm/apple/dcp-display-metadata.c b/drivers/gpu/drm/apple/dcp-display-metadata.c new file mode 100644 index 0000000000000..cc4c69736d098 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-display-metadata.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include + +#include "dcp-display-metadata.h" + +static int dcp_metadata_timing(struct detailed_timing *timing, + const struct drm_display_mode *mode, + const struct dcp_display_metadata *metadata) +{ + struct detailed_pixel_timing *pixel = &timing->data.pixel_data; + u32 hblank, vblank, hoff, voff, hsync, vsync; + u32 width = metadata->width_cm * 10, height = metadata->height_cm * 10; + u32 hflags = mode->flags & (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC); + u32 vflags = mode->flags & (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC); + + if (mode->clock <= 0 || mode->clock > 655350 || mode->clock % 10 || + mode->hdisplay <= 0 || mode->vdisplay <= 0 || + mode->hdisplay > 4095 || mode->vdisplay > 4095 || + mode->htotal <= mode->hdisplay || mode->vtotal <= mode->vdisplay || + mode->hsync_start < mode->hdisplay || mode->hsync_end <= mode->hsync_start || + mode->hsync_end > mode->htotal || mode->vsync_start < mode->vdisplay || + mode->vsync_end <= mode->vsync_start || mode->vsync_end > mode->vtotal || + (hflags != DRM_MODE_FLAG_PHSYNC && hflags != DRM_MODE_FLAG_NHSYNC) || + (vflags != DRM_MODE_FLAG_PVSYNC && vflags != DRM_MODE_FLAG_NVSYNC) || + (mode->flags & ~(DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC | + DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC))) + return -EINVAL; + hblank = mode->htotal - mode->hdisplay; + vblank = mode->vtotal - mode->vdisplay; + hoff = mode->hsync_start - mode->hdisplay; + voff = mode->vsync_start - mode->vdisplay; + hsync = mode->hsync_end - mode->hsync_start; + vsync = mode->vsync_end - mode->vsync_start; + if (hblank > 4095 || vblank > 4095 || hoff > 1023 || hsync > 1023 || + voff > 63 || vsync > 63 || width > 4095 || height > 4095) + return -EINVAL; + + timing->pixel_clock = cpu_to_le16(mode->clock / 10); + pixel->hactive_lo = mode->hdisplay; + pixel->hblank_lo = hblank; + pixel->hactive_hblank_hi = (mode->hdisplay >> 8) << 4 | hblank >> 8; + pixel->vactive_lo = mode->vdisplay; + pixel->vblank_lo = vblank; + pixel->vactive_vblank_hi = (mode->vdisplay >> 8) << 4 | vblank >> 8; + pixel->hsync_offset_lo = hoff; + pixel->hsync_pulse_width_lo = hsync; + pixel->vsync_offset_pulse_width_lo = (voff & 15) << 4 | (vsync & 15); + pixel->hsync_vsync_offset_pulse_width_hi = (hoff >> 8) << 6 | (hsync >> 8) << 4 | + (voff >> 4) << 2 | vsync >> 4; + pixel->width_mm_lo = width; + pixel->height_mm_lo = height; + pixel->width_height_mm_hi = (width >> 8) << 4 | height >> 8; + pixel->misc = DRM_EDID_PT_SEPARATE_SYNC; + if (hflags == DRM_MODE_FLAG_PHSYNC) + pixel->misc |= DRM_EDID_PT_HSYNC_POSITIVE; + if (vflags == DRM_MODE_FLAG_PVSYNC) + pixel->misc |= DRM_EDID_PT_VSYNC_POSITIVE; + return 0; +} + +static int dcp_metadata_text(struct detailed_timing *timing, u8 type, const char *text) +{ + size_t length = strnlen(text, 14); + u8 *out = timing->data.other_data.data.str.str; + + if (!length || length > 13) + return -EINVAL; + for (size_t i = 0; i < length; i++) + if (text[i] < 0x20 || text[i] > 0x7e) + return -EINVAL; + timing->data.other_data.type = type; + memset(out, ' ', 13); + memcpy(out, text, length); + if (length < 13) + out[length] = '\n'; + return 0; +} + +static s64 dcp_metadata_side(const u32 *a, const u32 *b, const u32 *point) +{ + return ((s64)b[0] - a[0]) * ((s64)point[1] - a[1]) - + ((s64)b[1] - a[1]) * ((s64)point[0] - a[0]); +} + +/* Build only a base block: the authoritative firmware mode list stays separate. */ +static int dcp_metadata_block(struct edid *edid, const struct dcp_display_metadata *metadata, + const struct drm_display_mode *mode) +{ + u32 coordinate[8], manufacturer = 0, gamma; + u8 checksum = 0; + int ret; + + if (!metadata->width_cm || metadata->width_cm > 255 || + !metadata->height_cm || metadata->height_cm > 255 || + metadata->product > U16_MAX || metadata->week > 53 || + metadata->year < 1990 || metadata->year > 2245 || + metadata->gamma < 65536 || metadata->gamma > (354U << 16) / 100 || + strnlen(metadata->manufacturer, 4) != 3) + return -EINVAL; + for (unsigned int i = 0; i < 3; i++) { + if (metadata->manufacturer[i] < 'A' || metadata->manufacturer[i] > 'Z') + return -EINVAL; + manufacturer = manufacturer << 5 | (metadata->manufacturer[i] - '@'); + } + for (unsigned int i = 0; i < 8; i++) { + u32 value = i < 6 ? metadata->chromaticity[i / 2][i % 2] : metadata->white[i - 6]; + + if (!value || value >= 65536) + return -EINVAL; + coordinate[i] = DIV_ROUND_CLOSEST(value, 64); + if (!coordinate[i] || coordinate[i] > 1023) + return -EINVAL; + if ((i & 1) && coordinate[i - 1] + coordinate[i] > 1024) + return -EINVAL; + } + /* Degenerate primaries or a white point outside their triangle are not usable. */ + { + s64 area = dcp_metadata_side(coordinate, coordinate + 2, coordinate + 4); + + if (!area) + return -EINVAL; + for (unsigned int i = 0; i < 3; i++) { + s64 side = dcp_metadata_side(coordinate + 2 * i, + coordinate + 2 * ((i + 1) % 3), coordinate + 6); + + if ((area > 0 && side < 0) || (area < 0 && side > 0)) + return -EINVAL; + } + } + gamma = DIV_ROUND_CLOSEST(metadata->gamma * 100, 65536) - 100; + if (gamma > 254) + return -EINVAL; + + memset(edid, 0, sizeof(*edid)); + memcpy(edid->header, "\x00\xff\xff\xff\xff\xff\xff\x00", 8); + edid->product_id.manufacturer_name = cpu_to_be16(manufacturer); + edid->product_id.product_code = cpu_to_le16(metadata->product); + edid->product_id.serial_number = cpu_to_le32(metadata->serial); + edid->product_id.week_of_manufacture = metadata->week; + edid->product_id.year_of_manufacture = metadata->year - 1990; + edid->version = 1; + edid->revision = 4; + /* No claim about connector interface, bit depth, HDR, audio or VRR. */ + edid->input = DRM_EDID_INPUT_DIGITAL; + edid->width_cm = metadata->width_cm; + edid->height_cm = metadata->height_cm; + edid->gamma = gamma; + edid->features = DRM_EDID_FEATURE_PREFERRED_TIMING | DRM_EDID_FEATURE_RGB; + edid->red_green_lo = (coordinate[0] & 3) << 6 | (coordinate[1] & 3) << 4 | + (coordinate[2] & 3) << 2 | (coordinate[3] & 3); + edid->blue_white_lo = (coordinate[4] & 3) << 6 | (coordinate[5] & 3) << 4 | + (coordinate[6] & 3) << 2 | (coordinate[7] & 3); + edid->red_x = coordinate[0] >> 2; + edid->red_y = coordinate[1] >> 2; + edid->green_x = coordinate[2] >> 2; + edid->green_y = coordinate[3] >> 2; + edid->blue_x = coordinate[4] >> 2; + edid->blue_y = coordinate[5] >> 2; + edid->white_x = coordinate[6] >> 2; + edid->white_y = coordinate[7] >> 2; + memset(edid->standard_timings, 1, sizeof(edid->standard_timings)); + ret = dcp_metadata_timing(&edid->detailed_timings[0], mode, metadata); + if (!ret) + ret = dcp_metadata_text(&edid->detailed_timings[1], EDID_DETAIL_MONITOR_NAME, + metadata->name); + if (!ret) + ret = dcp_metadata_text(&edid->detailed_timings[2], EDID_DETAIL_MONITOR_STRING, + "DCP metadata"); + if (!ret) + ret = dcp_metadata_text(&edid->detailed_timings[3], + metadata->serial_text[0] ? EDID_DETAIL_MONITOR_SERIAL : + EDID_DETAIL_MONITOR_STRING, + metadata->serial_text[0] ? + metadata->serial_text : "FW derived"); + if (ret) + return ret; + for (unsigned int i = 0; i < sizeof(*edid) - 1; i++) + checksum += ((const u8 *)edid)[i]; + edid->checksum = -checksum; + return 0; +} + +const struct drm_edid *dcp_metadata_edid(const struct dcp_display_metadata *metadata, + const struct drm_display_mode *mode) +{ + struct edid block; + const struct drm_edid *edid; + int ret = dcp_metadata_block(&block, metadata, mode); + + if (ret) + return ERR_PTR(ret); + edid = drm_edid_alloc(&block, sizeof(block)); + if (!edid) + return ERR_PTR(-ENOMEM); + if (!drm_edid_valid(edid)) { + drm_edid_free(edid); + return ERR_PTR(-EINVAL); + } + return edid; +} diff --git a/drivers/gpu/drm/apple/dcp-display-metadata.h b/drivers/gpu/drm/apple/dcp-display-metadata.h new file mode 100644 index 0000000000000..537994420ebd6 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-display-metadata.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DCP_DISPLAY_METADATA_H +#define APPLE_DCP_DISPLAY_METADATA_H + +#include + +/* Values reported by DisplayAttributes, not a copy of the sink's EDID. */ +struct dcp_display_metadata { + u32 width_cm, height_cm; + u32 chromaticity[3][2]; + u32 white[2]; + u32 gamma; + u32 product, serial, week, year; + char manufacturer[4]; + char name[14]; + char serial_text[14]; +}; + +struct drm_edid; +struct drm_display_mode; +const struct drm_edid *dcp_metadata_edid(const struct dcp_display_metadata *metadata, + const struct drm_display_mode *mode); + +#endif diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 280d13ab07321..6be10edfc5daf 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -18,6 +18,8 @@ #include "dptxep.h" #include "dcp_backlight.h" +#include "dcp-display-metadata.h" +#include "dcp-metadata-epoch.h" #include "iomfb.h" #include "iomfb_h17p.h" #include "iomfb-state.h" @@ -99,6 +101,9 @@ enum { struct dcp_chunks { size_t length; void *data; + size_t received; + bool contiguous; + struct dcp_metadata_ticket metadata_ticket; }; #define DCP_MAX_MAPPINGS (128) /* should be enough */ @@ -279,6 +284,15 @@ struct apple_dcp { /* Active chunked transfer. There can only be one at a time. */ struct dcp_chunks chunks; + struct { + /* Protects connection tickets and cached metadata snapshots. */ + spinlock_t lock; + struct dcp_metadata_epoch epoch; + struct dcp_display_metadata attributes; + struct drm_display_mode preferred; + struct dcp_metadata_ticket attributes_ticket, timing_ticket; + bool attributes_valid, timing_valid, hpd_seen; + } metadata; /* Queued swap. Owned by the DCP to avoid per-swap memory allocation */ union { @@ -469,6 +483,15 @@ struct apple_dcp { } iomfb; }; +void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, bool connected); +void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected); +void dcp_firmware_metadata_begin(struct apple_dcp *dcp); +void dcp_firmware_metadata_attributes(struct apple_dcp *dcp, + const struct dcp_display_metadata *attributes); +void dcp_firmware_metadata_timing(struct apple_dcp *dcp); +bool dcp_firmware_metadata_current(struct apple_dcp *dcp, u64 generation); +const struct drm_edid *dcp_firmware_edid(struct apple_dcp *dcp, u64 *generation); + void iomfb_scanout_complete_h17p(struct apple_dcp *dcp); void iomfb_present_backlight_h17p(struct apple_dcp *dcp); bool iomfb_present_brightness_only_h17p(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp-metadata-epoch.h b/drivers/gpu/drm/apple/dcp-metadata-epoch.h new file mode 100644 index 0000000000000..c2ab0c22a74e0 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-metadata-epoch.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DCP_METADATA_EPOCH_H +#define APPLE_DCP_METADATA_EPOCH_H + +#include + +/* Caller serializes these fields with the metadata snapshot lock. */ +struct dcp_metadata_epoch { + u64 generation; + bool connected; + bool associated_once; + bool invalidated; +}; + +struct dcp_metadata_ticket { + u64 generation; + bool associated; +}; + +static inline void dcp_metadata_epoch_connection(struct dcp_metadata_epoch *epoch, + u64 generation, bool connected) +{ + if (epoch->associated_once && + (!connected || generation != epoch->generation)) + epoch->invalidated = true; + if (connected) + epoch->associated_once = true; + epoch->generation = generation; + epoch->connected = connected; +} + +static inline struct dcp_metadata_ticket +dcp_metadata_begin(const struct dcp_metadata_epoch *epoch) +{ + return (struct dcp_metadata_ticket) { + .generation = epoch->generation, + .associated = epoch->connected && !epoch->invalidated, + }; +} + +static inline bool dcp_metadata_matches(const struct dcp_metadata_epoch *epoch, + struct dcp_metadata_ticket ticket) +{ + return ticket.associated && epoch->connected && !epoch->invalidated && + ticket.generation == epoch->generation; +} + +#endif diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 369012abb5481..49b2fe65a773f 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -352,6 +352,8 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, route->port->secondary_connector : route->port->connector; int ret; + dcp_firmware_metadata_connection(dcp, READ_ONCE(dcp->typec_generation), false); + /* * The fixed output's HPD handler leaves disconnects to DCP, so the port * can still be marked connected to a display that is gone. Release it @@ -454,6 +456,8 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) (dcp->external_error || (dcp->external_power_attempted && !dcp->external_stopped))) return dcp->external_error ?: -EBUSY; + dcp_firmware_metadata_connection(dcp, READ_ONCE(dcp->typec_generation), false); + /* * Under tb_lock so a DCP link (re)configuration can neither select the * crossbar nor restart the tunnel pixel clock behind our back. @@ -1815,6 +1819,168 @@ void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now) spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); } +static bool dcp_firmware_metadata_enabled(struct apple_dcp *dcp) +{ + return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; +} + +void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, bool connected) +{ + unsigned long flags; + + if (!dcp_firmware_metadata_enabled(dcp)) + return; + spin_lock_irqsave(&dcp->metadata.lock, flags); + if (generation != dcp->metadata.epoch.generation || !connected) { + dcp->metadata.attributes_valid = false; + dcp->metadata.timing_valid = false; + dcp->metadata.hpd_seen = false; + } + dcp_metadata_epoch_connection(&dcp->metadata.epoch, generation, connected); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); +} + +void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected) +{ + unsigned long flags; + + if (!dcp_firmware_metadata_enabled(dcp)) + return; + spin_lock_irqsave(&dcp->metadata.lock, flags); + if (connected) { + dcp->metadata.hpd_seen = true; + } else if (dcp->metadata.hpd_seen) { + dcp_metadata_epoch_connection(&dcp->metadata.epoch, + dcp->metadata.epoch.generation, false); + dcp->metadata.attributes_valid = false; + dcp->metadata.timing_valid = false; + } + spin_unlock_irqrestore(&dcp->metadata.lock, flags); +} + +void dcp_firmware_metadata_begin(struct apple_dcp *dcp) +{ + unsigned long flags; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + dcp->chunks.metadata_ticket = dcp_metadata_begin(&dcp->metadata.epoch); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + dcp->chunks.received = 0; + dcp->chunks.contiguous = true; +} + +static void dcp_firmware_metadata_notify(struct apple_dcp *dcp) +{ + unsigned long flags; + bool ready; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + ready = dcp->metadata.attributes_valid && dcp->metadata.timing_valid && + dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.attributes_ticket) && + dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.timing_ticket); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (ready && dcp->connector && READ_ONCE(dcp->connector->connected)) + schedule_work(&dcp->connector->hotplug_wq); +} + +static bool dcp_firmware_metadata_complete(struct apple_dcp *dcp) +{ + return dcp->chunks.contiguous && dcp->chunks.received == dcp->chunks.length && + dcp_metadata_matches(&dcp->metadata.epoch, dcp->chunks.metadata_ticket); +} + +void dcp_firmware_metadata_attributes(struct apple_dcp *dcp, + const struct dcp_display_metadata *attributes) +{ + unsigned long flags; + + if (!dcp_firmware_metadata_enabled(dcp)) + return; + spin_lock_irqsave(&dcp->metadata.lock, flags); + if (dcp_metadata_matches(&dcp->metadata.epoch, dcp->chunks.metadata_ticket)) { + dcp->metadata.attributes_valid = attributes && dcp_firmware_metadata_complete(dcp); + if (dcp->metadata.attributes_valid) { + dcp->metadata.attributes = *attributes; + dcp->metadata.attributes_ticket = dcp->chunks.metadata_ticket; + } + } + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + dcp_firmware_metadata_notify(dcp); +} + +void dcp_firmware_metadata_timing(struct apple_dcp *dcp) +{ + unsigned long flags; + unsigned int i; + + if (!dcp_firmware_metadata_enabled(dcp)) + return; + spin_lock_irqsave(&dcp->metadata.lock, flags); + if (!dcp_metadata_matches(&dcp->metadata.epoch, dcp->chunks.metadata_ticket)) + goto unlock; + dcp->metadata.timing_valid = false; + if (!dcp_firmware_metadata_complete(dcp)) + goto unlock; + for (i = 0; i < dcp->nr_modes; i++) { + if (!(dcp->modes[i].mode.type & DRM_MODE_TYPE_PREFERRED) || + !dcp->modes[i].metadata_timing_valid) + continue; + dcp->metadata.preferred = dcp->modes[i].mode; + dcp->metadata.timing_ticket = dcp->chunks.metadata_ticket; + dcp->metadata.timing_valid = true; + break; + } +unlock: + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + dcp_firmware_metadata_notify(dcp); +} + +bool dcp_firmware_metadata_current(struct apple_dcp *dcp, u64 generation) +{ + unsigned long flags; + bool valid; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + valid = !dcp->metadata.epoch.invalidated && dcp->metadata.epoch.connected && + generation == dcp->metadata.epoch.generation; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + return valid; +} + +const struct drm_edid *dcp_firmware_edid(struct apple_dcp *dcp, u64 *generation) +{ + struct dcp_display_metadata attributes; + struct drm_display_mode preferred; + const struct drm_edid *edid; + unsigned long flags; + bool valid; + + if (!dcp_firmware_metadata_enabled(dcp)) + return ERR_PTR(-ENODEV); + spin_lock_irqsave(&dcp->metadata.lock, flags); + valid = dcp->metadata.attributes_valid && dcp->metadata.timing_valid && + dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.attributes_ticket) && + dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.timing_ticket); + if (valid) { + attributes = dcp->metadata.attributes; + preferred = dcp->metadata.preferred; + *generation = dcp->metadata.epoch.generation; + } + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (!valid) + return ERR_PTR(-ENODATA); + edid = dcp_metadata_edid(&attributes, &preferred); + spin_lock_irqsave(&dcp->metadata.lock, flags); + valid = !dcp->metadata.epoch.invalidated && dcp->metadata.epoch.connected && + *generation == dcp->metadata.epoch.generation; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (!valid && !IS_ERR(edid)) { + drm_edid_free(edid); + return ERR_PTR(-ESTALE); + } + return edid; +} + void dcp_set_dimensions(struct apple_dcp *dcp) { int i; @@ -2592,12 +2758,13 @@ static void dcp_typec_reconnect_work(struct work_struct *work) static void disconnected_hpd_event(struct apple_connector *con) { - if (con && con->connected) { - con->connected = 0; - drm_edid_free(con->drm_edid); - con->drm_edid = NULL; - drm_kms_helper_connector_hotplug_event(&con->base); - } + if (!con) + return; + + /* May run with DRM modeset locks held; defer property/cache cleanup. */ + WRITE_ONCE(con->connected, false); + WRITE_ONCE(con->edid_invalidation_pending, true); + schedule_work(&con->hotplug_wq); } static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) @@ -2644,6 +2811,7 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) guard(mutex)(&dcp->hpd_mutex); dcp->typec_generation++; WRITE_ONCE(dcp->typec_cable_connected, true); + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, true); dcp->typec_reconnect_tries = 0; dcp->placeholder_retried = false; cancel_delayed_work(&dcp->typec_reconnect_wq); @@ -2668,6 +2836,7 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) scoped_guard(mutex, &dcp->hpd_mutex) { WRITE_ONCE(dcp->typec_cable_connected, false); dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); } WRITE_ONCE(dcp->typec_crtc_off, false); reinit_completion(&dcp->typec_iomfb_hpd_ready); @@ -3636,6 +3805,7 @@ static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) if (release_cable) WRITE_ONCE(dcp->typec_cable_connected, false); dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); } /* Block new enqueues as well as draining users of the AFK endpoints. */ disable_delayed_work_sync(&dcp->typec_reconnect_wq); @@ -4028,6 +4198,7 @@ static int dcp_platform_probe(struct platform_device *pdev) INIT_WORK(&dcp->external_stop_work, dcp_external_stop_work); mutex_init(&dcp->tb_lock); spin_lock_init(&dcp->mode_state.lock); + spin_lock_init(&dcp->metadata.lock); spin_lock_init(&dcp->dcpavserv.lock); dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); if (dcp->hw.iomfb_method_profile == diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index b1e0c4b65776e..fa12af320ca82 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -777,6 +777,23 @@ void dcp_retrain_oob(struct apple_connector *connector) schedule_work(&connector->hotplug_wq); } +static void dcp_clear_connector_edid(struct apple_connector *connector) +{ + lockdep_assert_held(&connector->base.dev->mode_config.mutex); + drm_edid_connector_update(&connector->base, NULL); + drm_edid_free(connector->drm_edid); + connector->drm_edid = NULL; + connector->drm_edid_firmware_derived = false; +} + +static void dcp_consume_edid_invalidation(struct apple_connector *connector) +{ + lockdep_assert_held(&connector->base.dev->mode_config.mutex); + /* Preserve a disconnect even if the port reconnected before probing. */ + if (xchg(&connector->edid_invalidation_pending, false)) + dcp_clear_connector_edid(connector); +} + void dcp_hotplug(struct work_struct *work) { struct apple_connector *connector; @@ -785,9 +802,17 @@ void dcp_hotplug(struct work_struct *work) int ret; connector = container_of(work, struct apple_connector, hotplug_wq); + scoped_guard(mutex, &connector->base.dev->mode_config.mutex) + dcp_consume_edid_invalidation(connector); pdev = READ_ONCE(connector->dcp); if (!pdev) { /* a Type-C port unrouted after this was queued */ + scoped_guard(mutex, &connector->base.dev->mode_config.mutex) { + if (!READ_ONCE(connector->dcp) && connector->drm_edid_firmware_derived) { + drm_edid_connector_update(&connector->base, NULL); + connector->drm_edid_firmware_derived = false; + } + } drm_kms_helper_connector_hotplug_event(&connector->base); return; } @@ -795,9 +820,9 @@ void dcp_hotplug(struct work_struct *work) dev_info(dcp->dev, "%s() connected:%d valid_mode:%d nr_modes:%u\n", __func__, connector->connected, READ_ONCE(dcp->mode_state.valid), dcp->nr_modes); - if (!connector->connected) { - drm_edid_free(connector->drm_edid); - connector->drm_edid = NULL; + scoped_guard(mutex, &connector->base.dev->mode_config.mutex) { + if (!READ_ONCE(connector->connected)) + dcp_clear_connector_edid(connector); } /* @@ -1031,9 +1056,16 @@ int dcp_get_modes(struct drm_connector *connector) bool vrr_capable = false; int i; + dcp_consume_edid_invalidation(apple_connector); + /* A Type-C port has no pipeline while the fabric is moving it. */ - if (!pdev) + if (!pdev) { + if (apple_connector->drm_edid_firmware_derived) { + drm_edid_connector_update(connector, NULL); + apple_connector->drm_edid_firmware_derived = false; + } return 0; + } dcp = platform_get_drvdata(pdev); for (i = 0; i < dcp->nr_modes; ++i) { @@ -1079,8 +1111,34 @@ int dcp_get_modes(struct drm_connector *connector) } } if (dcp->nr_modes && apple_connector->drm_edid) { + apple_connector->drm_edid_firmware_derived = false; drm_edid_connector_update(connector, apple_connector->drm_edid); dcp_retry_placeholder_edid(dcp, apple_connector->drm_edid); + } else { + u64 generation = 0; + const struct drm_edid *derived = dcp->nr_modes ? + dcp_firmware_edid(dcp, &generation) : ERR_PTR(-ENODATA); + + if (!IS_ERR(derived)) { + bool log = !apple_connector->drm_edid_firmware_derived || + apple_connector->drm_edid_generation != generation; + int update; + + apple_connector->drm_edid_firmware_derived = true; + apple_connector->drm_edid_generation = generation; + update = drm_edid_connector_update(connector, derived); + drm_edid_free(derived); + if (update || !dcp_firmware_metadata_current(dcp, generation)) { + drm_edid_connector_update(connector, NULL); + apple_connector->drm_edid_firmware_derived = false; + } else if (log) { + dev_info(dcp->dev, "using firmware-derived EDID metadata for connection %llu\n", + generation); + } + } else if (apple_connector->drm_edid_firmware_derived) { + drm_edid_connector_update(connector, NULL); + apple_connector->drm_edid_firmware_derived = false; + } } /* diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index b6c522f030a43..759f60ec6b245 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -933,6 +933,7 @@ static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) #endif dcp->chunks.length = *length; + dcp_firmware_metadata_begin(dcp); dcp->chunks.data = kzalloc(*length, GFP_KERNEL); if (!dcp->chunks.data) { @@ -951,11 +952,17 @@ static u8 dcpep_cb_prop_chunk(struct apple_dcp *dcp, return false; } - if (req->offset + req->length > dcp->chunks.length) { + if (req->offset > dcp->chunks.length || req->length > sizeof(req->data) || + req->length > dcp->chunks.length - req->offset) { + dcp->chunks.contiguous = false; dev_warn(dcp->dev, "ignoring overflowing chunk\n"); return false; } + if (req->offset != dcp->chunks.received) + dcp->chunks.contiguous = false; + if (dcp->chunks.contiguous) + dcp->chunks.received += req->length; memcpy(dcp->chunks.data + req->offset, req->data, req->length); return true; } @@ -994,11 +1001,15 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, dev_warn(dcp->dev, "failed to parse modes\n"); dcp->modes = NULL; dcp->nr_modes = 0; + dcp_firmware_metadata_timing(dcp); return false; } if (dcp->nr_modes == 0) dev_warn(dcp->dev, "TimingElements without valid modes!\n"); + dcp_firmware_metadata_timing(dcp); } else if (!strcmp(req->key, "DisplayAttributes")) { + struct dcp_parse_ctx metadata_ctx = ctx; + ret = parse_display_attributes(&ctx, &dcp->width_mm, &dcp->height_mm); @@ -1008,6 +1019,12 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, } dcp_set_dimensions(dcp); + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + struct dcp_display_metadata metadata; + + ret = parse_display_metadata(&metadata_ctx, &metadata); + dcp_firmware_metadata_attributes(dcp, ret ? NULL : &metadata); + } } return true; @@ -1875,6 +1892,7 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) if (dcp_external_hotplug(dcp, !!(*connected))) return; + dcp_firmware_metadata_hotplug(dcp, !!(*connected)); action = dcp_mode_hotplug(&dcp->mode_state, !!(*connected), connector ? &connector->connected : NULL); /* diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index cc97258307427..01e1ac1d28ff1 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -299,7 +299,7 @@ int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx) return 0; } -static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) +static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim, bool polarity) { struct iterator it; int ret = 0; @@ -310,6 +310,8 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) .front_porch = -1, .sync_width = -1, .precise_sync_rate = -1, + .sync_polarity = -1, + .pixel_repetition = -1, }; ret = iterator_begin(handle, &it, true); @@ -331,6 +333,18 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) ret = parse_int_bound(it.handle, &dim->sync_width, 0, U16_MAX); else if (!strcmp(key, "PreciseSyncRate")) ret = parse_int_bound(it.handle, &dim->precise_sync_rate, 1, U32_MAX); + else if (polarity && !strcmp(key, "SyncPolarity")) { + if (dim->sync_polarity >= 0) + ret = -EINVAL; + else + ret = parse_int_bound(it.handle, &dim->sync_polarity, 0, 1); + } else if (polarity && !strcmp(key, "PixelRepetition")) { + if (dim->pixel_repetition >= 0) + ret = -EINVAL; + else + ret = parse_int_bound(it.handle, &dim->pixel_repetition, + 0, U32_MAX); + } else ret = skip(it.handle); @@ -626,6 +640,8 @@ static int parse_mode(struct dcp_parse_ctx *handle, s64 id = -1; s64 best_color_mode = -1; bool is_virtual = false; + bool preferred = false, interlaced = false; + bool have_preferred = false, have_interlaced = false; bool is_preferred = false; struct drm_display_mode *mode = &out->mode; @@ -657,9 +673,9 @@ static int parse_mode(struct dcp_parse_ctx *handle, else if (is_virtual) ret = skip(it.handle); else if (!strcmp(key, "HorizontalAttributes")) - ret = parse_dimension(it.handle, &horiz); + ret = parse_dimension(it.handle, &horiz, external_sdr); else if (!strcmp(key, "VerticalAttributes")) - ret = parse_dimension(it.handle, &vert); + ret = parse_dimension(it.handle, &vert, external_sdr); else if (!strcmp(key, "MinimumVariableRefreshRate")) ret = parse_int_bound(it.handle, &min_vrr, 0, U32_MAX); else if (!strcmp(key, "MaximumVariableRefreshRate")) @@ -670,9 +686,21 @@ static int parse_mode(struct dcp_parse_ctx *handle, ret = parse_int_bound(it.handle, &id, 0, U32_MAX); else if (!strcmp(key, "IsVirtual")) ret = parse_bool(it.handle, &is_virtual); - else if (!strcmp(key, "IsPreferred")) + else if (external_sdr && !strcmp(key, "IsPreferred")) { + if (have_preferred) + ret = -EINVAL; + else + ret = parse_bool(it.handle, &preferred); + have_preferred = true; + } else if (external_sdr && !strcmp(key, "IsInterlaced")) { + if (have_interlaced) + ret = -EINVAL; + else + ret = parse_bool(it.handle, &interlaced); + have_interlaced = true; + } else if (!strcmp(key, "IsPreferred")) { ret = parse_bool(it.handle, &is_preferred); - else if (!strcmp(key, "Score")) + } else if (!strcmp(key, "Score")) ret = parse_int_bound(it.handle, score, 0, S64_MAX); else ret = skip(it.handle); @@ -762,6 +790,12 @@ static int parse_mode(struct dcp_parse_ctx *handle, .height_mm = height_mm, }; + if (external_sdr && horiz.sync_polarity >= 0 && vert.sync_polarity >= 0) + mode->flags = (horiz.sync_polarity ? DRM_MODE_FLAG_PHSYNC : DRM_MODE_FLAG_NHSYNC) | + (vert.sync_polarity ? DRM_MODE_FLAG_PVSYNC : DRM_MODE_FLAG_NVSYNC); + out->metadata_timing_valid = external_sdr && have_preferred && preferred && + have_interlaced && !interlaced && horiz.pixel_repetition == 0 && + vert.pixel_repetition == 0 && horiz.sync_polarity >= 0 && vert.sync_polarity >= 0; drm_mode_set_name(mode); out->timing_mode_id = id; @@ -868,6 +902,238 @@ int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, return 0; } +/* Metadata strings must fit an EDID text descriptor without hidden suffixes. */ +static int parse_metadata_string(struct dcp_parse_ctx *handle, char *out, size_t size) +{ + const struct dcp_parse_tag *tag = parse_tag_of_type(handle, DCP_TYPE_STRING); + const u8 *value; + + if (IS_ERR(tag)) + return PTR_ERR(tag); + if (!tag->size || tag->size >= size) + return -EINVAL; + value = parse_bytes(handle, tag->size); + if (IS_ERR(value)) + return PTR_ERR(value); + for (unsigned int i = 0; i < tag->size; i++) + if (value[i] < 0x20 || value[i] > 0x7e) + return -EINVAL; + memcpy(out, value, tag->size); + out[tag->size] = 0; + return 0; +} + +static int parse_metadata_number(struct dcp_parse_ctx *handle, u32 *out, u32 min, u32 max) +{ + s64 value; + int ret = parse_int_bound(handle, &value, min, max); + + if (!ret) + *out = value; + return ret; +} + +static int parse_metadata_point(struct dcp_parse_ctx *handle, u32 point[2], u32 *gamma) +{ + struct iterator it; + unsigned int seen = 0; + int ret = iterator_begin(handle, &it, true); + + if (ret) + return ret; + for (; it.idx < it.len; it.idx++) { + char key[64]; + + ret = parse_metadata_string(handle, key, sizeof(key)); + if (ret) + return ret; + if (!strcmp(key, "X")) { + if (seen & 1) + return -EINVAL; + seen |= 1; + ret = parse_metadata_number(handle, &point[0], 1, 65535); + } else if (!strcmp(key, "Y")) { + if (seen & 2) + return -EINVAL; + seen |= 2; + ret = parse_metadata_number(handle, &point[1], 1, 65535); + } else if (gamma && !strcmp(key, "Gamma")) { + if (seen & 4) + return -EINVAL; + seen |= 4; + ret = parse_metadata_number(handle, gamma, 65536, (354U << 16) / 100); + } else { + ret = skip(handle); + } + if (ret) + return ret; + } + if (seen != (gamma ? 7 : 3) || point[0] + point[1] > 65536) + return -EINVAL; + return 0; +} + +static int parse_metadata_chromaticity(struct dcp_parse_ctx *handle, + struct dcp_display_metadata *out) +{ + static const char * const names[] = { "Red", "Green", "Blue" }; + struct iterator it; + unsigned int seen = 0; + int ret = iterator_begin(handle, &it, true); + + if (ret) + return ret; + for (; it.idx < it.len; it.idx++) { + char key[64]; + unsigned int color; + + ret = parse_metadata_string(handle, key, sizeof(key)); + if (ret) + return ret; + for (color = 0; color < ARRAY_SIZE(names); color++) + if (!strcmp(key, names[color])) + break; + if (color < ARRAY_SIZE(names)) { + if (seen & 1U << color) + return -EINVAL; + seen |= 1U << color; + ret = parse_metadata_point(handle, out->chromaticity[color], NULL); + } else { + ret = skip(handle); + } + if (ret) + return ret; + } + return seen == 7 ? 0 : -EINVAL; +} + +static int parse_metadata_product(struct dcp_parse_ctx *handle, struct dcp_display_metadata *out) +{ + struct iterator it; + unsigned int seen = 0; + u32 legacy = 0; + int ret = iterator_begin(handle, &it, true); + + if (ret) + return ret; + for (; it.idx < it.len; it.idx++) { + char key[64]; + + ret = parse_metadata_string(handle, key, sizeof(key)); + if (ret) + return ret; + if (!strcmp(key, "ManufacturerID")) { + if (seen & 1) + return -EINVAL; + seen |= 1; + ret = parse_metadata_string(handle, out->manufacturer, + sizeof(out->manufacturer)); + } else if (!strcmp(key, "ProductName")) { + if (seen & 2) + return -EINVAL; + seen |= 2; + ret = parse_metadata_string(handle, out->name, sizeof(out->name)); + } else if (!strcmp(key, "ProductID")) { + if (seen & 4) + return -EINVAL; + seen |= 4; + ret = parse_metadata_number(handle, &out->product, 0, U16_MAX); + } else if (!strcmp(key, "WeekOfManufacture")) { + if (seen & 8) + return -EINVAL; + seen |= 8; + ret = parse_metadata_number(handle, &out->week, 0, 53); + } else if (!strcmp(key, "YearOfManufacture")) { + if (seen & 16) + return -EINVAL; + seen |= 16; + ret = parse_metadata_number(handle, &out->year, 1990, 2245); + } else if (!strcmp(key, "SerialNumber")) { + if (seen & 64) + return -EINVAL; + seen |= 64; + ret = parse_metadata_number(handle, &out->serial, 0, U32_MAX); + } else if (!strcmp(key, "AlphanumericSerialNumber")) { + if (seen & 128) + return -EINVAL; + seen |= 128; + ret = parse_metadata_string(handle, out->serial_text, + sizeof(out->serial_text)); + } else if (!strcmp(key, "LegacyManufacturerID")) { + if (seen & 32) + return -EINVAL; + seen |= 32; + ret = parse_metadata_number(handle, &legacy, 0, 0x7fff); + } else { + ret = skip(handle); + } + if (ret) + return ret; + } + if ((seen & 31) != 31 || strlen(out->manufacturer) != 3) + return -EINVAL; + for (unsigned int i = 0; i < 3; i++) + if (out->manufacturer[i] < 'A' || out->manufacturer[i] > 'Z') + return -EINVAL; + if ((seen & 32) && legacy != ((out->manufacturer[0] - '@') << 10 | + (out->manufacturer[1] - '@') << 5 | + (out->manufacturer[2] - '@'))) + return -EINVAL; + return 0; +} + +int parse_display_metadata(struct dcp_parse_ctx *handle, struct dcp_display_metadata *out) +{ + struct dcp_display_metadata parsed = {}; + struct iterator it; + unsigned int seen = 0; + int ret = iterator_begin(handle, &it, true); + + if (ret) + return ret; + for (; it.idx < it.len; it.idx++) { + char key[64]; + + ret = parse_metadata_string(handle, key, sizeof(key)); + if (ret) + return ret; + if (!strcmp(key, "MaxHorizontalImageSize")) { + if (seen & 1) + return -EINVAL; + seen |= 1; + ret = parse_metadata_number(handle, &parsed.width_cm, 1, 255); + } else if (!strcmp(key, "MaxVerticalImageSize")) { + if (seen & 2) + return -EINVAL; + seen |= 2; + ret = parse_metadata_number(handle, &parsed.height_cm, 1, 255); + } else if (!strcmp(key, "Chromaticity")) { + if (seen & 4) + return -EINVAL; + seen |= 4; + ret = parse_metadata_chromaticity(handle, &parsed); + } else if (!strcmp(key, "DefaultWhitePoint")) { + if (seen & 8) + return -EINVAL; + seen |= 8; + ret = parse_metadata_point(handle, parsed.white, &parsed.gamma); + } else if (!strcmp(key, "ProductAttributes")) { + if (seen & 16) + return -EINVAL; + seen |= 16; + ret = parse_metadata_product(handle, &parsed); + } else { + ret = skip(handle); + } + if (ret) + return ret; + } + if (seen != 31) + return -EINVAL; + *out = parsed; + return 0; +} + int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, const char **class, s64 *unit, bool h17p_keys) { diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 1f3b8782b7bfc..50e0580f98080 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -7,6 +7,8 @@ /* For mode parsing */ #include +#include "dcp-display-metadata.h" + struct apple_dcp; struct dcp_parse_ctx { @@ -94,11 +96,14 @@ struct dcp_display_mode { struct dcp_color_mode sdr; struct dcp_color_mode best; bool vrr; + bool metadata_timing_valid; }; struct dimension { s64 total, front_porch, sync_width, active; s64 precise_sync_rate; + s64 sync_polarity; + s64 pixel_repetition; }; int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx); @@ -106,6 +111,7 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, int height_mm, unsigned notch_height, bool internal, bool external_sdr); +int parse_display_metadata(struct dcp_parse_ctx *handle, struct dcp_display_metadata *out); int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, int *height_mm); int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, From 41e82d0e4a28c213ddef2cd7fabdd23101c18bd6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:54:54 +1100 Subject: [PATCH 1200/2680] drm/apple: trace external metadata admission state Record bounded connection tickets and validity flags while holding the metadata lock. Trace initial association, property receipt and probing to distinguish an ineligible connection from incomplete or rejected metadata without logging monitor contents or weakening admission. The tracepoint is disabled by default. Exercise the observation path in the existing metadata lifetime test. [ryan: carried without the tracepoint, which aurora-wip removed again in merge commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"); what remains is the wider metadata lock in dcp_firmware_metadata_begin(), which aurora-wip keeps] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 49b2fe65a773f..08c4de13986bc 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1864,9 +1864,9 @@ void dcp_firmware_metadata_begin(struct apple_dcp *dcp) spin_lock_irqsave(&dcp->metadata.lock, flags); dcp->chunks.metadata_ticket = dcp_metadata_begin(&dcp->metadata.epoch); - spin_unlock_irqrestore(&dcp->metadata.lock, flags); dcp->chunks.received = 0; dcp->chunks.contiguous = true; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); } static void dcp_firmware_metadata_notify(struct apple_dcp *dcp) From 758d99a41380d145bcbca66e528c87308b285dd3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:59:53 +1100 Subject: [PATCH 1201/2680] drm/apple: validate delivered property extent before metadata admission H17P advertises one extra byte after the word-aligned serialized property, but does not send that byte. Requiring received == advertised discarded complete same-connection TimingElements and DisplayAttributes. Validate the complete serialized object against bytes actually received. Accept exact lengths or the observed one-byte framing convention only; reject gaps, excess trailing content and truncated values. Perform the bounded structural walk outside the metadata spinlock, then retain the existing connection-ticket checks. No absent byte is used as payload. Exercise captured property extents, truncation mutations and a missing final string byte alongside metadata lifetime and decoder tests. [ryan: without the payload_complete bit of the dropped metadata tracepoint] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 3 ++- drivers/gpu/drm/apple/iomfb_template.c | 6 ++++++ drivers/gpu/drm/apple/parser.c | 16 ++++++++++++++++ drivers/gpu/drm/apple/parser.h | 1 + 5 files changed, 26 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 6be10edfc5daf..b931c345a0769 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -103,6 +103,7 @@ struct dcp_chunks { void *data; size_t received; bool contiguous; + bool payload_complete; struct dcp_metadata_ticket metadata_ticket; }; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 08c4de13986bc..601edd1e29838 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1866,6 +1866,7 @@ void dcp_firmware_metadata_begin(struct apple_dcp *dcp) dcp->chunks.metadata_ticket = dcp_metadata_begin(&dcp->metadata.epoch); dcp->chunks.received = 0; dcp->chunks.contiguous = true; + dcp->chunks.payload_complete = false; spin_unlock_irqrestore(&dcp->metadata.lock, flags); } @@ -1885,7 +1886,7 @@ static void dcp_firmware_metadata_notify(struct apple_dcp *dcp) static bool dcp_firmware_metadata_complete(struct apple_dcp *dcp) { - return dcp->chunks.contiguous && dcp->chunks.received == dcp->chunks.length && + return dcp->chunks.contiguous && dcp->chunks.payload_complete && dcp_metadata_matches(&dcp->metadata.epoch, dcp->chunks.metadata_ticket); } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 759f60ec6b245..0a0c3d7b43994 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -978,6 +978,12 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, return false; } + /* Validate serialized extent outside the metadata spinlock. */ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + dcp->chunks.payload_complete = dcp->chunks.contiguous && + dcp_property_complete(dcp->chunks.data, dcp->chunks.received, + dcp->chunks.length); + /* used just as opaque pointer for tracing */ ctx.dcp = dcp; diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 01e1ac1d28ff1..77edffc6b294c 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -299,6 +299,22 @@ int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx) return 0; } +bool dcp_property_complete(const void *blob, size_t received, size_t advertised) +{ + struct dcp_parse_ctx ctx; + + if (!blob || !received || received > U32_MAX - 3 || + received > advertised || advertised - received > 1) + return false; + /* H17P may count one unsent byte after the aligned serialized object. */ + if (advertised != received && (received & 3)) + return false; + /* Bound decoding to bytes actually delivered, never allocation padding. */ + if (parse(blob, received, &ctx) || skip(&ctx)) + return false; + return ctx.pos == received || round_up(ctx.pos, 4) == received; +} + static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim, bool polarity) { struct iterator it; diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 50e0580f98080..0392b09751ac9 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -107,6 +107,7 @@ struct dimension { }; int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx); +bool dcp_property_complete(const void *blob, size_t received, size_t advertised); struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, int height_mm, unsigned notch_height, From 788675731ba549e4ea5779be20e885236eeaa604 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 21:49:16 +1100 Subject: [PATCH 1202/2680] drm/apple: publish initial external discovery while presents remain paused The lifecycle gate cached the first connected HPD while waiting for DRM activation to unpause presentation. DRM could not activate a connector that had never been published, leaving the R35 external output absent even with a connected DPTX link and valid mode metadata. Publish cable-backed initial discovery before the first power attempt, while retaining the presentation pause. Sticky failures, absent cables and OFF/restart callbacks remain withheld. Receive callbacks still acquire no lifecycle mutex or issue firmware control requests. The regression test fails on the prior production function and passes with this change; it verifies that discovery does not power or unpause the pipe. ASan/UBSan lifecycle, metadata and delayed-service tests pass. Hardware activation and OFF-response qualification remain pending. [ryan: split by topic: only the drm/apple part and its selftests are here; the tools/diagnostics scripts are dropped (aurora-wip deleted them in commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"))] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 601edd1e29838..e66d5c5e8629d 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2642,6 +2642,15 @@ bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected) if (dcp->iomfb.presentation_paused) { dcp->external_hotplug_pending = true; dcp->external_hotplug_connected = connected; + /* Publish discovery so DRM can activate the first CRTC; keep presents paused. */ + if (connected && READ_ONCE(dcp->external_cable) && + !READ_ONCE(dcp->external_power_attempted) && !READ_ONCE(dcp->external_error)) { + unsigned int action = dcp_mode_hotplug(&dcp->mode_state, true, + dcp->connector ? + &dcp->connector->connected : NULL); + + dcp_handle_hotplug_actions(dcp, action); + } return true; } if (!connected && !READ_ONCE(dcp->iomfb.presentation_paused)) { From 862ca1d5d86e5596635633fd837734bfcc5a0789 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 21:15:03 +1100 Subject: [PATCH 1203/2680] drm/apple: Tie the external metadata epoch to the display lifecycle The external H17P lifecycle and the bounded monitor metadata were developed side by side and joined by merge commit 9f742ddfa628 ("Merge qualified output metadata into the external lifecycle candidate"). The merge resolution carries code that no commit has: - A reported external disconnect marks the connector's EDID for invalidation and notifies the hotplug path. - An external disconnect closes the metadata connection, and a newly attached cable opens one for its new Type-C generation. Re-made from that merge resolution for the 7.2.10 port. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index e66d5c5e8629d..56d7d0e61beed 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2478,6 +2478,10 @@ static void dcp_external_report_disconnected(struct apple_dcp *dcp) guard(mutex)(&dcp->iomfb.lock); action = dcp_mode_hotplug(&dcp->mode_state, false, dcp->connector ? &dcp->connector->connected : NULL); + if (dcp->connector) { + WRITE_ONCE(dcp->connector->edid_invalidation_pending, true); + action |= DCP_HOTPLUG_NOTIFY; + } /* The existing worker delivers DRM notifications outside our locks. */ dcp_handle_hotplug_actions(dcp, action); } @@ -2607,6 +2611,7 @@ static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical) scoped_guard(mutex, &dcp->hpd_mutex) { WRITE_ONCE(dcp->typec_cable_connected, false); dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); } /* A missing cable is reported even when its hardware remains owned. */ dcp_external_report_disconnected(dcp); @@ -2800,17 +2805,23 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) int ret; if (dcp_is_external_h17p(dcp)) { + bool new_cable; + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); cancel_delayed_work_sync(&dcp->placeholder_edid_wq); cancel_work_sync(&dcp->external_stop_work); guard(mutex)(&dcp->external_lock); - if (!dcp->external_cable) + new_cable = !dcp->external_cable; + if (new_cable) dcp->external_generation++; dcp->external_cable = true; dcp_external_publish_generation(dcp); scoped_guard(mutex, &dcp->hpd_mutex) { WRITE_ONCE(dcp->typec_cable_connected, true); - dcp->typec_generation++; + if (new_cable) { + dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, true); + } dcp->placeholder_retried = false; } return dcp_external_link_locked(dcp); From 83270f1412ff92abbf5922c76e535ce6f04fadde Mon Sep 17 00:00:00 2001 From: nekolyanich Date: Mon, 12 Oct 2026 22:46:12 +0300 Subject: [PATCH 1204/2680] drm: apple: answer the DPTX port's group 1 calls as macOS does On t6020 with the 13.5 firmware, DCP's DPTX port sends AP calls of group 1, commands 3 and 4, around link (re)configuration, audio prepare included. The AFK v1 notify path dropped the group, so they reached the group 0 handlers of the same numbers, SET/GET_DRIVE_SETTINGS: DCP addresses from the payload were stored as drive settings. Behind a USB-C to HDMI adapter DCP then refused the audio start ("Capabilities changed") or never confirmed the stream. Pass the group through a call_v2 op, with the signature of call_v2 on aurora's J700 test branch (85d81b76, not in aurora-wip), and keep .call for group 0 only. A group call to a service without call_v2 gets a warning and no reply: AFK has no handler and does not guess one. The DPTX port answers group 1 commands 3 and 4 the way macOS 13 does in 42 of 42 traced pairs (TV through the adapter, Dell over Thunderbolt): an echo of the request with +0x00..0x03 cleared for (1, 3), and +0x10..0x13 and +0x20 to the end cleared for (1, 4). The lane count sits at +0x10 in (1, 3) and at +0x00 in (1, 4); macOS keeps it in all 42 and clears the other slot. The J700 code reads (1, 3) the same way, with per-lane levels from +0x20, which macOS does not clear there; it refuses (1, 4), and its group 0 GET (0, 4) has the count at +0x00. Other groups are echoed with a rate-limited warning. Testing: MacBook Pro 16" M2 Pro (j416s), DCP firmware 13.5, a TV on a USB-C to HDMI adapter and a Dell over Thunderbolt, 8 audio starts on the TV per boot. Cold boots under the m1n1 hypervisor, aurora-wip with PRs #275-#277, the AFK slots series and a bench DT change, interleaved: with this commit the first start was accepted in 5 of 5 boots, with 0 refusals and 8 of 8 starts played; without it, 3 of 3 boots refused the first start, 2 refusals each, 4 of 8 played. The Dell had no refusals in any boot, and no group warning was logged. Natively, on a downstream tree with the same reply rule, the boot connect and 3 of 8 TV replugs reached DCP (the rest failed earlier, in the PHY): 0 refusals in 4 connects; before, 10 refusals in 5 connects. Not tested: other SoCs, other firmware, AFK v2. Assisted-by: Claude:claude-opus-5-5 checkpatch Signed-off-by: nekolyanich --- drivers/gpu/drm/apple/afk.c | 48 ++++++++++++++++++++------- drivers/gpu/drm/apple/afk.h | 2 ++ drivers/gpu/drm/apple/dptxep.c | 60 +++++++++++++++++++++++++++++++++- drivers/gpu/drm/apple/trace.h | 17 ++++++---- 4 files changed, 107 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index ff1f0f57afd9f..18e85187660f0 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -493,14 +493,12 @@ static void afk_recv_handle_reply(struct apple_dcp_afkep *ep, u32 channel, dma_free_coherent(ep->dcp->dev, txlen, txbuf, txbuf_dma); } +/* A call from DCP: one word ahead of the call header AP sends. */ struct epic_std_service_ap_call { __le32 unk0; - __le32 unk1; - __le32 type; - __le32 len; - __le32 magic; - u8 _unk[48]; + struct epic_service_call call; } __attribute__((packed)); +static_assert(sizeof(struct epic_std_service_ap_call) == 68); static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, u32 type, struct epic_hdr *ehdr, @@ -525,30 +523,56 @@ static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, if (type == EPIC_TYPE_NOTIFY && eshdr->category == EPIC_CAT_NOTIFY) { struct epic_std_service_ap_call *call = payload; size_t call_size; + u32 command; void *reply; + u16 group; int ret; if (payload_size < sizeof(*call)) return; - call_size = le32_to_cpu(call->len); + call_size = le32_to_cpu(call->call.data_len); if (payload_size < sizeof(*call) + call_size) return; - if (!service->ops->call) + group = le16_to_cpu(call->call.group); + command = le32_to_cpu(call->call.command); + + /* + * Command numbers are per group: a group-less .call handler + * would take another group's call for a group 0 one. Without + * call_v2 there is no handler for it, and AFK cannot guess a + * reply, so it gets none, as a call without any handler. + */ + if (!service->ops->call_v2 && (group || !service->ops->call)) { + if (group) + dev_warn_ratelimited(ep->dcp->dev, + "AFK[ep:%02x]: %s (chan:%u) apcall %u:%u has no handler, sending NO reply\n", + ep->endpoint, service->ops->name, + channel, group, command); return; + } reply = kzalloc(payload_size, GFP_KERNEL); if (!reply) return; - ret = service->ops->call(service, le32_to_cpu(call->type), - payload + sizeof(*call), call_size, - reply + sizeof(*call), call_size); + if (service->ops->call_v2) + ret = service->ops->call_v2(service, group, command, + payload + sizeof(*call), + call_size, + reply + sizeof(*call), + call_size); + else + ret = service->ops->call(service, command, + payload + sizeof(*call), + call_size, + reply + sizeof(*call), + call_size); if (ret) { dev_warn(ep->dcp->dev, - "AFK[ep:%02x]: %s (chan:%u) apcall %u handler returned %d, sending NO reply\n", + "AFK[ep:%02x]: %s (chan:%u) apcall %u:%u handler returned %d, sending NO reply\n", ep->endpoint, service->ops->name, channel, - le32_to_cpu(call->type), ret); + group, command, ret); kfree(reply); return; } diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 0cef14fa6dfe8..65696d6ac689e 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -74,6 +74,8 @@ struct apple_epic_service_ops { int (*call)(struct apple_epic_service *service, u32 idx, const void *data, size_t data_size, void *reply, size_t reply_size); + int (*call_v2)(struct apple_epic_service *service, u16 group, u32 command, + const void *data, size_t size, void *reply, size_t room); int (*report)(struct apple_epic_service *service, enum epic_subtype type, const void *data, size_t data_size); void (*teardown)(struct apple_epic_service *service); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 11091084f05e2..946c195520cbc 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -691,7 +691,7 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, size_t reply_size) { struct dptx_port *dptx = service->cookie; - trace_dptxport_apcall(dptx, idx, data_size); + trace_dptxport_apcall(dptx, 0, idx, data_size); dev_dbg(service->ep->dcp->dev, "DPTXPort: APCALL %u (%zu bytes)\n", idx, data_size); if (data_size) @@ -836,11 +836,69 @@ static void dptxport_init(struct apple_epic_service *service, const char *name, } } +/* + * Group 1 calls 3 and 4 come around link (re)configuration, audio prepare + * included. They carry a lane count at +0x10 in (1, 3) and at +0x00 in + * (1, 4); the J700 code reads (1, 3) as drive settings with levels per + * lane from +0x20, and its group 0 GET (0, 4) has the count at +0x00. + * macOS 13 acks both with the request, (1, 3) with +0x00..0x03 + * cleared and (1, 4) with +0x10..0x13 and +0x20 to the end cleared, so + * what DCP left there does not go back (42 of 42 traced pairs on t6020, + * 13.5 firmware: TV through a USB-C to HDMI adapter, Dell over TB). + */ +static int dptxport_call_group1(u32 command, const void *data, + size_t data_size, void *reply, size_t reply_size) +{ + size_t size = min(reply_size, data_size); + + memcpy(reply, data, size); + if (command == 3) { + memset(reply, 0, min_t(size_t, size, 0x4)); + } else if (size > 0x10) { + memset(reply + 0x10, 0, min_t(size_t, size - 0x10, 0x4)); + if (size > 0x20) + memset(reply + 0x20, 0, size - 0x20); + } + return 0; +} + +static int dptxport_group_call(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t size, + void *reply, size_t room) +{ + struct dptx_port *dptx = service->cookie; + + if (!group) + return dptxport_call(service, command, data, size, reply, room); + + trace_dptxport_apcall(dptx, group, command, size); + dev_dbg(service->ep->dcp->dev, "DPTXPort: APCALL %u:%u (%zu bytes)\n", + group, command, size); + if (size) + print_hex_dump_debug("DPTXPort: apcall data: ", + DUMP_PREFIX_OFFSET, 16, 1, data, + min(size, (size_t)64), true); + + if (group == 1 && (command == 3 || command == 4)) + return dptxport_call_group1(command, data, size, reply, room); + + /* + * Echo it, as group 0 acks an unhandled call: no reply might stall + * DCP's caller. + */ + dev_warn_ratelimited(service->ep->dcp->dev, + "DPTXPort: unknown AP call %u:%u, echoing it\n", + group, command); + memcpy(reply, data, min(room, size)); + return 0; +} + static const struct apple_epic_service_ops dptxep_ops[] = { { .name = "AppleDCPDPTXRemotePort", .init = dptxport_init, .call = dptxport_call, + .call_v2 = dptxport_group_call, }, {} }; diff --git a/drivers/gpu/drm/apple/trace.h b/drivers/gpu/drm/apple/trace.h index d3c5106e5d66d..cbb3f4080fd57 100644 --- a/drivers/gpu/drm/apple/trace.h +++ b/drivers/gpu/drm/apple/trace.h @@ -398,18 +398,21 @@ TRACE_EVENT(dptxport_init, TP_PROTO(struct apple_dcp *dcp, u64 unit), TRACE_EVENT( dptxport_apcall, - TP_PROTO(struct dptx_port *dptx, int idx, size_t len), - TP_ARGS(dptx, idx, len), + TP_PROTO(struct dptx_port *dptx, u16 group, int idx, size_t len), + TP_ARGS(dptx, group, idx, len), TP_STRUCT__entry(__string(devname, dev_name(dptx->service->ep->dcp->dev)) - __field(u32, unit) __field(int, idx) __field(size_t, len)), + __field(u32, unit) __field(u16, group) __field(int, idx) + __field(size_t, len)), TP_fast_assign(__assign_str(devname); - __entry->unit = dptx->unit; __entry->idx = idx; __entry->len = len;), + __entry->unit = dptx->unit; __entry->group = group; + __entry->idx = idx; __entry->len = len;), - TP_printk("%s: dptx%d: AP Call %d (%s) with len %lu", __get_str(devname), - __entry->unit, - __entry->idx, show_dptxport_apcall(__entry->idx), __entry->len)); + TP_printk("%s: dptx%d: AP Call %u:%d (%s) with len %lu", __get_str(devname), + __entry->unit, __entry->group, __entry->idx, + __entry->group ? "-" : show_dptxport_apcall(__entry->idx), + __entry->len)); TRACE_EVENT( dptxport_validate_connection, From 1a2b6d6e3a943654af2a075f168dfb402943eac3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 10:24:00 +1100 Subject: [PATCH 1205/2680] drm/apple: accept exact H17P power replies and republish reconnected displays Live R36 evidence records OFF state zero with zero status and a stopped external pipeline. Validate that exact echo while retaining malformed reply, timeout, error, queue drain and ownership guards. Publish discovery only for a cleanly stopped new physical cable epoch, and release a reattached link if unplugged before CRTC activation. Exercise production reply decoding and reconnect paths without the previous test-only OFF policy shim. [ryan: split by topic: only the drm/apple part and its selftests are here; the tools/diagnostics scripts are dropped (aurora-wip deleted them in commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"))] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 13 +++++++++---- drivers/gpu/drm/apple/iomfb_h17p.c | 8 ++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 56d7d0e61beed..0de1f2ed2cedc 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2554,12 +2554,12 @@ static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation) return 0; if (dcp->external_error) return dcp->external_error; - if (dcp->external_stopped) + if (dcp->external_stopped && !READ_ONCE(dcp->dptxport[0].connected)) return 0; iomfb_pause_presentation(dcp, true); dcp->external_running = false; /* Route setup can call this before IOMFB or its shared memory exists. */ - if (!dcp->external_power_attempted) + if (dcp->external_stopped || !dcp->external_power_attempted) goto release_link; ret = iomfb_queue_settle(dcp, msecs_to_jiffies(500)); if (!ret) @@ -2647,9 +2647,14 @@ bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected) if (dcp->iomfb.presentation_paused) { dcp->external_hotplug_pending = true; dcp->external_hotplug_connected = connected; - /* Publish discovery so DRM can activate the first CRTC; keep presents paused. */ + /* Publish initial/new-cable discovery, never same-cable DPMS wake. */ if (connected && READ_ONCE(dcp->external_cable) && - !READ_ONCE(dcp->external_power_attempted) && !READ_ONCE(dcp->external_error)) { + !READ_ONCE(dcp->external_error) && + (!READ_ONCE(dcp->external_power_attempted) || + (READ_ONCE(dcp->external_stopped) && + READ_ONCE(dcp->external_rx_generation) != + READ_ONCE(dcp->external_crtc_generation) && + READ_ONCE(dcp->dptxport[0].connected)))) { unsigned int action = dcp_mode_hotplug(&dcp->mode_state, true, dcp->connector ? &dcp->connector->connected : NULL); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index dbfe025445f4e..a86675d485a17 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -726,7 +726,7 @@ struct dcp_external_power_cookie { bool on; }; -/* A473's envelope is known; no OFF reply has been qualified as a stop. */ +/* A473 echoes the accepted state with zero status; DMA ownership is unchanged. */ static void iomfb_external_power_complete(struct apple_dcp *dcp, void *out, void *data) { struct dcp_external_power_cookie *cookie = data; @@ -741,9 +741,9 @@ static void iomfb_external_power_complete(struct apple_dcp *dcp, void *out, void WRITE_ONCE(dcp->external_power_reply, reply); WRITE_ONCE(dcp->external_power_reply_valid, true); - dev_info(dcp->dev, "A473 power=%u reply=%#llx%s\n", - cookie->on, reply, cookie->on ? "" : " (stop unqualified)"); - ret = cookie->on ? (reply == 1 ? 0 : -EREMOTEIO) : -EOPNOTSUPP; + dev_info(dcp->dev, "A473 power=%u reply=%#llx\n", + cookie->on, reply); + ret = reply == (u64)cookie->on ? 0 : -EREMOTEIO; } } WRITE_ONCE(cookie->wait.status, ret); From a1b787aabf4e725fd33ce134973cb92350e3f355 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 11:59:20 +1100 Subject: [PATCH 1206/2680] drm/apple: tag H17P external SDR RGB sources as sRGB Native-tagged RGB surfaces produce washed-out external output on J700 H17P despite correct KMS pixel values. The same MSI and cable render normally in macOS and M2 Fairydust. A controlled J700 trial changing only the source tag to sRGB restores normal colours. Apply that tag to native RGB surfaces on the external H17P profile. Preserve transfer, pixel format, output modes and other profiles. No diagnostic switch is needed. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p_test.c | 8 ++++++++ drivers/gpu/drm/apple/iomfb_template.c | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb_h17p_test.c b/drivers/gpu/drm/apple/iomfb_h17p_test.c index c017651945a4e..c5955836700ca 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p_test.c +++ b/drivers/gpu/drm/apple/iomfb_h17p_test.c @@ -45,6 +45,8 @@ static void h17p_surface_wire_test(struct kunit *test) KUNIT_EXPECT_MEMEQ(test, wire->swap, &request->swap, sizeof(wire->swap)); KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x593), DCP_FORMAT_BGRA); + KUNIT_EXPECT_EQ(test, ((u8 *)wire)[0x59b], (u8)DCP_XFER_FUNC_SDR); + KUNIT_EXPECT_EQ(test, ((u8 *)wire)[0x59c], (u8)DCP_COLORSPACE_NATIVE); KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x59d), 10176U); KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5a9), 2408U); KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5ad), 1506U); @@ -58,6 +60,12 @@ static void h17p_surface_wire_test(struct kunit *test) KUNIT_EXPECT_EQ(test, get_unaligned_le64((u8 *)wire + 0xe38), iova); KUNIT_EXPECT_EQ(test, wire->tail[sizeof(wire->tail) - 2], 0U); KUNIT_EXPECT_EQ(test, wire->tail[sizeof(wire->tail) - 1], 0U); + + /* Source-tag changes must reach the wire without changing transfer. */ + surface->colorspace = DCP_COLORSPACE_BG_SRGB; + iomfb_serialize_present_h17p(wire, request); + KUNIT_EXPECT_EQ(test, ((u8 *)wire)[0x59b], (u8)DCP_XFER_FUNC_SDR); + KUNIT_EXPECT_EQ(test, ((u8 *)wire)[0x59c], (u8)DCP_COLORSPACE_BG_SRGB); } static void h17p_opaque_x_parameter_test(struct kunit *test) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 0a0c3d7b43994..b9a6eaacdb74f 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -2354,6 +2354,18 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP && req->surf[l].base.colorspace == DCP_COLORSPACE_BG_SRGB) req->surf[l].base.colorspace = DCP_COLORSPACE_NATIVE; + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* H17P external SDR RGB needs an sRGB source tag. Native + * tagging washes out the same pixel values on this profile. + * Keep YUV, other source spaces and other profiles unchanged. + */ + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + !new_state->fb->format->is_yuv && + req->surf[l].base.colorspace == DCP_COLORSPACE_NATIVE) + req->surf[l].base.colorspace = DCP_COLORSPACE_BG_SRGB; +#endif } if (!has_surface && !crtc_state->color_mgmt_changed) { From 1a6cc36f63fb8e8cdb2f01fbd170758dfe5c50c3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 13:47:16 +1100 Subject: [PATCH 1207/2680] drm/apple: use the observed H17P DPAV service for fresh EDID The external H17P DPAV service is on endpoint 0x2a with the identity dcpav-service-epic / AppleDCPDP2HDMI / unit 1. Select that endpoint only for the external profile and route AFK receives by the instantiated DPAV and DPTX endpoints, preserving legacy 0x28 DPAV and 0x2a DPTX routing and native H17P DPTX at 0x2c. Admit the exact service before the existing AFKV2 OPEN. Keep the bounded read-only group 1 command 7 EDID request and its response validation. Reject in-flight EDID and cached/publication results when the physical Type-C generation changes. Invalidate that generation before waiting for firmware disconnect teardown. Release hpd_mutex during EDID requests and hold it for publication under the existing mode_config.mutex ordering. Keep the conservative firmware metadata epoch guard and service owner/reader pins. Host tests exercise actual production routing, admission, reader ownership and publication with synthetic disconnected/delayed responses. Firmware wire behavior still requires owner qualification. Complete existing kernel-doc parameter descriptions for a clean W=1 compile without warning suppression. [ryan: split by topic: only the drm/apple part and its selftests are here; the tools/diagnostics scripts are dropped (aurora-wip deleted them in commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"))] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 57 ++++++++--------------- drivers/gpu/drm/apple/epic/dpavservep.c | 60 +++++++++++++++++++++++-- drivers/gpu/drm/apple/epic/dpavservep.h | 3 +- drivers/gpu/drm/apple/iomfb.c | 40 ++++++++++++++++- 5 files changed, 117 insertions(+), 44 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index b931c345a0769..6e6d0d7b6ff50 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -89,6 +89,7 @@ enum { DISP0_ENDPOINT = 0x23, DPTX_CONTROLLER_ENDPOINT = 0x24, DPAVSERV_ENDPOINT = 0x28, + DPAVSERV_ENDPOINT_V2 = 0x2a, AV_ENDPOINT = 0x29, DPTX_ENDPOINT = 0x2a, DPTX_ENDPOINT_V2 = 0x2c, diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 0de1f2ed2cedc..c1b5cf96cdc6e 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1751,6 +1751,10 @@ static void send_vblank_event(struct drm_device *dev, /** * dcp_crtc_send_page_flip_event - helper to send vblank event after pageflip + * @crtc: CRTC whose flip completed + * @e: pending vblank event + * @now: swap completion timestamp + * @start: swap submission timestamp * * Compensate for unknown slack between page flip and arrival of the * swap_complete callback. Minimal observed duration on DCP with HDMI output @@ -2051,6 +2055,11 @@ static void dcp_delayed_vblank(struct work_struct *work) static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) { + /* DPAV 0x2a on H17P overlaps the legacy DPTX endpoint number. */ + if (dcp->dcpavservep && endpoint == dcp->dcpavservep->endpoint) + return dcp->dcpavservep; + if (dcp->dptxep && endpoint == dcp->dptxep->endpoint) + return dcp->dptxep; switch (endpoint) { case AV_ENDPOINT: return dcp->avep; @@ -2058,13 +2067,6 @@ static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) return dcp->systemep; case DISP0_ENDPOINT: return dcp->ibootep; - case DPAVSERV_ENDPOINT: - return dcp->dcpavservep; - case DPTX_ENDPOINT: - return dcp->dptxep; - case DPTX_ENDPOINT_V2: - return dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL ? dcp->dptxep : NULL; case DPTX_CONTROLLER_ENDPOINT: return dcp->dptx_controller_ep; default: @@ -2132,34 +2134,10 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) return; } - switch (endpoint) { - case IOMFB_ENDPOINT: + if (endpoint == IOMFB_ENDPOINT) return iomfb_recv_msg(dcp, message); - case AV_ENDPOINT: - ep = dcp->avep; - break; - case SYSTEM_ENDPOINT: - ep = dcp->systemep; - break; - case DISP0_ENDPOINT: - ep = dcp->ibootep; - break; - case DPAVSERV_ENDPOINT: - ep = dcp->dcpavservep; - break; - case DPTX_ENDPOINT: - ep = dcp->dptxep; - break; - case DPTX_ENDPOINT_V2: - ep = dcp->dptxep; - break; - case DPTX_CONTROLLER_ENDPOINT: - ep = dcp->dptx_controller_ep; - break; - default: - ep = NULL; - break; - } + + ep = dcp_afkep(dcp, endpoint); if (!ep) { dev_warn_ratelimited(dcp->dev, @@ -2603,16 +2581,19 @@ static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical) cancel_delayed_work_sync(&dcp->typec_reconnect_wq); cancel_delayed_work_sync(&dcp->placeholder_edid_wq); guard(mutex)(&dcp->external_lock); - ret = dcp_external_stop_locked(dcp, dcp->external_generation); + /* Invalidate physical EDID before waiting for firmware power/link teardown. */ if (physical) { - dcp->external_cable = false; - dcp->external_generation++; - dcp_external_publish_generation(dcp); scoped_guard(mutex, &dcp->hpd_mutex) { WRITE_ONCE(dcp->typec_cable_connected, false); dcp->typec_generation++; dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); } + } + ret = dcp_external_stop_locked(dcp, dcp->external_generation); + if (physical) { + dcp->external_cable = false; + dcp->external_generation++; + dcp_external_publish_generation(dcp); /* A missing cable is reported even when its hardware remains owned. */ dcp_external_report_disconnected(dcp); } diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 1b4019cc159f9..eaba2ea945714 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -32,6 +32,29 @@ static void dcpavserv_notify_edid(struct apple_epic_service *service) schedule_work(&connector->hotplug_wq); } +/* Admission runs before AFKV2 OPEN and before readers can pin the service. */ +static int dcpavserv_prepare(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + struct apple_dcp *dcp = service->ep->dcp; + unsigned long flags; + int ret; + + if (!name || strcmp(name, "dcpav-service-epic")) + return -ENODEV; + if (service->ep->inline_v2) { + if (service->ep->endpoint != DPAVSERV_ENDPOINT_V2 || !class || + strcmp(class, "AppleDCPDP2HDMI") || unit != 1) + return -ENODEV; + } else if (unit != 0) { + return -ENODEV; + } + spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + ret = dcp->dcpavserv.enabled ? -EBUSY : 0; + spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + return ret; +} + static void dcpavserv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { @@ -40,7 +63,7 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, trace_dcpavserv_init(dcp, unit); - if (unit == 0 && name && !strcmp(name, "dcpav-service-epic")) { + if (!dcpavserv_prepare(service, name, class, unit)) { spin_lock_irqsave(&dcp->dcpavserv.lock, flags); if (dcp->dcpavserv.enabled) { spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); @@ -95,6 +118,13 @@ void dpavservep_detach(struct apple_dcp *dcp) } } +static int dcpdpserv_prepare(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + /* No production command transport is needed for this H17P service. */ + return service->ep->inline_v2 ? -ENODEV : 0; +} + static void dcpdpserv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { @@ -272,12 +302,25 @@ static const struct drm_edid *dcpavserv_read_edid_inline(struct apple_epic_servi return drm_edid_alloc(resp->data + EDID_LEADING_DATA_SIZE, edid_size); } -const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp) +bool dcpavserv_edid_current(struct apple_dcp *dcp, u64 generation) +{ + lockdep_assert_held(&dcp->hpd_mutex); + return dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL || + (dcp->typec_cable_connected && generation == dcp->typec_generation && + !READ_ONCE(dcp->quiescing)); +} + +const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation) { struct apple_epic_service *service; const struct drm_edid *edid; unsigned long flags; + scoped_guard(mutex, &dcp->hpd_mutex) { + if (!dcpavserv_edid_current(dcp, generation)) + return ERR_PTR(-ESTALE); + } spin_lock_irqsave(&dcp->dcpavserv.lock, flags); service = dcp->dcpavserv.enabled ? afk_service_get(dcp->dcpavserv.service) : NULL; spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); @@ -294,6 +337,13 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp) } spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); afk_service_put(service); + scoped_guard(mutex, &dcp->hpd_mutex) { + if (!dcpavserv_edid_current(dcp, generation)) { + if (!IS_ERR(edid)) + drm_edid_free(edid); + edid = ERR_PTR(-ESTALE); + } + } return edid; } @@ -301,6 +351,7 @@ static const struct apple_epic_service_ops dpavservep_ops[] = { { .name = "dcpav-service-epic", .reusable = true, + .prepare = dcpavserv_prepare, .init = dcpavserv_init, .teardown = dcpavserv_teardown, .report = dcpavserv_report, @@ -308,6 +359,7 @@ static const struct apple_epic_service_ops dpavservep_ops[] = { { .name = "dcpdp-service-epic", .reusable = true, + .prepare = dcpdpserv_prepare, .init = dcpdpserv_init, .teardown = dcpdpserv_teardown, .report = dcpdpserv_report, @@ -322,7 +374,9 @@ int dpavservep_init(struct apple_dcp *dcp) init_completion(&dcp->dcpavserv.enable_completion); - ep = afk_init(dcp, DPAVSERV_ENDPOINT, dpavservep_ops); + ep = afk_init(dcp, dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL ? + DPAVSERV_ENDPOINT_V2 : DPAVSERV_ENDPOINT, dpavservep_ops); if (IS_ERR(ep)) return PTR_ERR(ep); /* Only the external H17P DPAV endpoint uses inline AFKV2 service calls. */ diff --git a/drivers/gpu/drm/apple/epic/dpavservep.h b/drivers/gpu/drm/apple/epic/dpavservep.h index e9ab17a7302c1..7e2972984c101 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.h +++ b/drivers/gpu/drm/apple/epic/dpavservep.h @@ -20,7 +20,8 @@ struct dcpavserv { struct apple_epic_service *service; }; -const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp); +bool dcpavserv_edid_current(struct apple_dcp *dcp, u64 generation); +const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation); void dpavservep_detach(struct apple_dcp *dcp); #endif /* _DRM_APPLE_EPIC_DPAVSERV_H */ diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index fa12af320ca82..2b6416d2b008f 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1054,6 +1054,8 @@ int dcp_get_modes(struct drm_connector *connector) struct drm_display_mode *mode; u16 min_vfreq = 0, max_vfreq = 0; bool vrr_capable = false; + bool physical_edid; + u64 edid_generation = 0; int i; dcp_consume_edid_invalidation(apple_connector); @@ -1090,12 +1092,40 @@ int dcp_get_modes(struct drm_connector *connector) } drm_connector_set_vrr_capable_property(connector, vrr_capable); + physical_edid = dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + /* get_modes holds mode_config.mutex; HPD paths never acquire it. */ + if (physical_edid) { + mutex_lock(&dcp->hpd_mutex); + edid_generation = dcp->typec_generation; + if (apple_connector->drm_edid && + apple_connector->drm_edid_generation != edid_generation) + dcp_clear_connector_edid(apple_connector); + if (!dcpavserv_edid_current(dcp, edid_generation)) { + dcp_clear_connector_edid(apple_connector); + goto edid_done; + } + } + /* Internal H17P still has no supported EDID command transport. */ if (dcp->nr_modes && !apple_connector->drm_edid && (dcp->fw_compat != DCP_FIRMWARE_H17P || (dcp->dcpavservep && dcp->dcpavservep->inline_v2))) { const struct drm_edid *edid; - edid = dcpavserv_copy_edid(dcp); + /* Let physical disconnect advance the generation during the request. */ + if (physical_edid) + mutex_unlock(&dcp->hpd_mutex); + edid = dcpavserv_copy_edid(dcp, edid_generation); + if (physical_edid) { + mutex_lock(&dcp->hpd_mutex); + if (!dcpavserv_edid_current(dcp, edid_generation) || + READ_ONCE(apple_connector->dcp) != pdev) { + if (!IS_ERR_OR_NULL(edid)) + drm_edid_free(edid); + dcp_clear_connector_edid(apple_connector); + goto edid_done; + } + } if (IS_ERR_OR_NULL(edid)) { /* * An internal panel has no AV service and so no EDID; @@ -1108,12 +1138,12 @@ int dcp_get_modes(struct drm_connector *connector) } else { drm_edid_free(apple_connector->drm_edid); apple_connector->drm_edid = edid; + apple_connector->drm_edid_generation = edid_generation; } } if (dcp->nr_modes && apple_connector->drm_edid) { apple_connector->drm_edid_firmware_derived = false; drm_edid_connector_update(connector, apple_connector->drm_edid); - dcp_retry_placeholder_edid(dcp, apple_connector->drm_edid); } else { u64 generation = 0; const struct drm_edid *derived = dcp->nr_modes ? @@ -1141,6 +1171,12 @@ int dcp_get_modes(struct drm_connector *connector) } } +edid_done: + if (physical_edid) + mutex_unlock(&dcp->hpd_mutex); + if (apple_connector->drm_edid) + dcp_retry_placeholder_edid(dcp, apple_connector->drm_edid); + /* * An internal panel has no EDID, so nothing fills in the refresh range * that userspace needs before it will drive VRR. Supply the range DCP From efcad5eacbcb715d747a4f11031ad7bf083a3c2c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 13:55:51 +1100 Subject: [PATCH 1208/2680] drm/apple: fence EDID publication by service lifetime and unblock RX A service CLOSE/replacement can leave a raw EDID cache valid even when the physical Type-C generation does not change. Replacement after the reader's last pointer check can also publish data from the retired service. Give DPAV an epoch that advances on service publication and retirement. Carry it from the pinned reader into the raw connector cache and validate it again while serializing publication against teardown. Use a process context mutex for the DPAV owner state, nesting below the DRM and HPD mutexes and releasing it during firmware waits. Defer CLOSE cache/property cleanup through the existing hotplug worker. Preserve the physical connection checks, owner/reader pins and firmware metadata epoch guard. DisplayAttributes is handled synchronously on RTKit's ordered RX worker. Taking mode_config.mutex there can block dispatch of the AFK reply that a mode probe is waiting for while it holds that mutex. Defer connector physical-dimension updates to the existing hotplug worker instead. Add a threaded host regression through the actual RTKit, IOMFB receive, dimensions and AFK enqueue functions. It fails on the parent with RX blocked before the AFK reply, and passes with the deferred update. Extend service and publication tests for same-slot CLOSE/re-OPEN, unchanged physical generations and replacement after the reader's last check. No command ABI or activation-readiness assumption is changed. [ryan: split by topic: only the drm/apple part and its selftests are here; the tools/diagnostics scripts are dropped (aurora-wip deleted them in commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"))] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/connector.h | 1 + drivers/gpu/drm/apple/dcp.c | 33 +++++++++---- drivers/gpu/drm/apple/dcp.h | 2 + drivers/gpu/drm/apple/epic/dpavservep.c | 66 +++++++++++++++++-------- drivers/gpu/drm/apple/epic/dpavservep.h | 12 +++-- drivers/gpu/drm/apple/iomfb.c | 32 ++++++++++-- 6 files changed, 108 insertions(+), 38 deletions(-) diff --git a/drivers/gpu/drm/apple/connector.h b/drivers/gpu/drm/apple/connector.h index 6f73e975ebdc1..e04bc2e119a5e 100644 --- a/drivers/gpu/drm/apple/connector.h +++ b/drivers/gpu/drm/apple/connector.h @@ -46,6 +46,7 @@ struct apple_connector { bool drm_edid_firmware_derived; bool edid_invalidation_pending; u64 drm_edid_generation; + u64 drm_edid_service_epoch; /* Workqueue for sending hotplug events to the associated device */ struct work_struct hotplug_wq; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index c1b5cf96cdc6e..463f0721ade1c 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1997,15 +1997,13 @@ void dcp_set_dimensions(struct apple_dcp *dcp) height_mm = dcp->panel.height_mm; } - /* Set the connector info */ - if (dcp->connector) { - struct drm_connector *connector = &dcp->connector->base; - - mutex_lock(&connector->dev->mode_config.mutex); - connector->display_info.width_mm = width_mm; - connector->display_info.height_mm = height_mm; - mutex_unlock(&connector->dev->mode_config.mutex); - } + /* + * This callback runs on the ordered RTKit RX worker. A mode probe + * can hold mode_config.mutex while waiting for an AFK reply which + * that same worker still has to dispatch. Defer the DRM update. + */ + if (dcp->connector && !READ_ONCE(dcp->quiescing)) + schedule_work(&dcp->connector->hotplug_wq); /* * Fix up any probed modes. Modes are created when parsing @@ -2018,6 +2016,21 @@ void dcp_set_dimensions(struct apple_dcp *dcp) } } +void dcp_update_connector_dimensions(struct apple_dcp *dcp, + struct drm_connector *connector) +{ + int width_mm = READ_ONCE(dcp->width_mm); + int height_mm = READ_ONCE(dcp->height_mm); + + lockdep_assert_held(&connector->dev->mode_config.mutex); + if (!width_mm || !height_mm) { + width_mm = dcp->panel.width_mm; + height_mm = dcp->panel.height_mm; + } + connector->display_info.width_mm = width_mm; + connector->display_info.height_mm = height_mm; +} + bool dcp_has_panel(struct apple_dcp *dcp) { return dcp->panel.width_mm > 0; @@ -4206,7 +4219,7 @@ static int dcp_platform_probe(struct platform_device *pdev) mutex_init(&dcp->tb_lock); spin_lock_init(&dcp->mode_state.lock); spin_lock_init(&dcp->metadata.lock); - spin_lock_init(&dcp->dcpavserv.lock); + mutex_init(&dcp->dcpavserv.lock); dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 9bef4759e8dcb..6e8d3632ae2fa 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -78,6 +78,8 @@ bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, const struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode); void dcp_set_dimensions(struct apple_dcp *dcp); +void dcp_update_connector_dimensions(struct apple_dcp *dcp, + struct drm_connector *connector); void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message); int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port); diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index eaba2ea945714..f7a54eefcf8ea 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -37,7 +37,6 @@ static int dcpavserv_prepare(struct apple_epic_service *service, const char *nam const char *class, s64 unit) { struct apple_dcp *dcp = service->ep->dcp; - unsigned long flags; int ret; if (!name || strcmp(name, "dcpav-service-epic")) @@ -49,9 +48,9 @@ static int dcpavserv_prepare(struct apple_epic_service *service, const char *nam } else if (unit != 0) { return -ENODEV; } - spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + mutex_lock(&dcp->dcpavserv.lock); ret = dcp->dcpavserv.enabled ? -EBUSY : 0; - spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + mutex_unlock(&dcp->dcpavserv.lock); return ret; } @@ -59,23 +58,23 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { struct apple_dcp *dcp = service->ep->dcp; - unsigned long flags; trace_dcpavserv_init(dcp, unit); if (!dcpavserv_prepare(service, name, class, unit)) { - spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + mutex_lock(&dcp->dcpavserv.lock); if (dcp->dcpavserv.enabled) { - spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + mutex_unlock(&dcp->dcpavserv.lock); dev_err(dcp->dev, "DCPAVSERV: unit %lld already exists\n", unit); return; } dcp->dcpavserv.service = afk_service_get(service); dcp->dcpavserv.enabled = true; + dcp->dcpavserv.epoch++; service->cookie = &dcp->dcpavserv; complete(&dcp->dcpavserv.enable_completion); - spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + mutex_unlock(&dcp->dcpavserv.lock); if (service->ep->inline_v2) { dev_info(dcp->dev, "DCPAVSERV: inline EDID service ready on channel %u\n", service->channel); @@ -84,20 +83,35 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, } } +/* Defer DRM cleanup: AFK callbacks must not take mode_config.mutex. */ +static void dcpavserv_invalidate_edid(struct apple_dcp *dcp) +{ + struct apple_connector *connector = READ_ONCE(dcp->connector); + + lockdep_assert_held(&dcp->dcpavserv.lock); + if (!connector || READ_ONCE(connector->dcp) != to_platform_device(dcp->dev)) + return; + WRITE_ONCE(connector->edid_invalidation_pending, true); + if (!READ_ONCE(dcp->quiescing) && + READ_ONCE(connector->base.registration_state) == DRM_CONNECTOR_REGISTERED) + schedule_work(&connector->hotplug_wq); +} + static void dcpavserv_teardown(struct apple_epic_service *service) { struct apple_dcp *dcp = service->ep->dcp; - unsigned long flags; bool owned = false; - spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + mutex_lock(&dcp->dcpavserv.lock); if (dcp->dcpavserv.service == service) { dcp->dcpavserv.enabled = false; dcp->dcpavserv.service = NULL; + dcp->dcpavserv.epoch++; + dcpavserv_invalidate_edid(dcp); reinit_completion(&dcp->dcpavserv.enable_completion); owned = true; } - spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + mutex_unlock(&dcp->dcpavserv.lock); if (owned) afk_service_put(service); } @@ -105,13 +119,16 @@ static void dcpavserv_teardown(struct apple_epic_service *service) void dpavservep_detach(struct apple_dcp *dcp) { struct apple_epic_service *service; - unsigned long flags; - spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + mutex_lock(&dcp->dcpavserv.lock); service = dcp->dcpavserv.service; dcp->dcpavserv.service = NULL; dcp->dcpavserv.enabled = false; - spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + if (service) { + dcp->dcpavserv.epoch++; + dcpavserv_invalidate_edid(dcp); + } + mutex_unlock(&dcp->dcpavserv.lock); if (service) { afk_service_disable(service); afk_service_put(service); @@ -311,31 +328,40 @@ bool dcpavserv_edid_current(struct apple_dcp *dcp, u64 generation) !READ_ONCE(dcp->quiescing)); } -const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation) +bool dcpavserv_edid_service_current(struct apple_dcp *dcp, u64 epoch) +{ + lockdep_assert_held(&dcp->dcpavserv.lock); + return dcp->dcpavserv.enabled && dcp->dcpavserv.service && + epoch == dcp->dcpavserv.epoch; +} + +const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation, + u64 *epoch) { struct apple_epic_service *service; const struct drm_edid *edid; - unsigned long flags; scoped_guard(mutex, &dcp->hpd_mutex) { if (!dcpavserv_edid_current(dcp, generation)) return ERR_PTR(-ESTALE); } - spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + mutex_lock(&dcp->dcpavserv.lock); service = dcp->dcpavserv.enabled ? afk_service_get(dcp->dcpavserv.service) : NULL; - spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + *epoch = dcp->dcpavserv.epoch; + mutex_unlock(&dcp->dcpavserv.lock); if (!service) return ERR_PTR(-ENODEV); edid = service->ep->inline_v2 ? dcpavserv_read_edid_inline(service) : dcpavserv_read_edid_legacy(service); - spin_lock_irqsave(&dcp->dcpavserv.lock, flags); - if (dcp->dcpavserv.service != service) { + mutex_lock(&dcp->dcpavserv.lock); + if (dcp->dcpavserv.service != service || + !dcpavserv_edid_service_current(dcp, *epoch)) { if (!IS_ERR(edid)) drm_edid_free(edid); edid = ERR_PTR(-ENODEV); } - spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + mutex_unlock(&dcp->dcpavserv.lock); afk_service_put(service); scoped_guard(mutex, &dcp->hpd_mutex) { if (!dcpavserv_edid_current(dcp, generation)) { diff --git a/drivers/gpu/drm/apple/epic/dpavservep.h b/drivers/gpu/drm/apple/epic/dpavservep.h index 7e2972984c101..b824a59918ea3 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.h +++ b/drivers/gpu/drm/apple/epic/dpavservep.h @@ -5,7 +5,7 @@ #define _DRM_APPLE_EPIC_DPAVSERV_H #include -#include +#include #include struct drm_edid; @@ -13,7 +13,11 @@ struct apple_epic_service; struct apple_dcp; struct dcpavserv { - spinlock_t lock; /* Protects the service pointer and its owner pin. */ + /* Process context only: serializes retirement and EDID publication. + * Nest below mode_config.mutex and hpd_mutex; never wait for firmware. + */ + struct mutex lock; + u64 epoch; /* Changes even when the same service slot is reused. */ bool enabled; struct completion enable_completion; u32 unit; @@ -21,7 +25,9 @@ struct dcpavserv { }; bool dcpavserv_edid_current(struct apple_dcp *dcp, u64 generation); -const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation); +bool dcpavserv_edid_service_current(struct apple_dcp *dcp, u64 epoch); +const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation, + u64 *epoch); void dpavservep_detach(struct apple_dcp *dcp); #endif /* _DRM_APPLE_EPIC_DPAVSERV_H */ diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 2b6416d2b008f..a2e19ea25e245 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -821,6 +821,8 @@ void dcp_hotplug(struct work_struct *work) connector->connected, READ_ONCE(dcp->mode_state.valid), dcp->nr_modes); scoped_guard(mutex, &connector->base.dev->mode_config.mutex) { + if (READ_ONCE(connector->dcp) == pdev) + dcp_update_connector_dimensions(dcp, &connector->base); if (!READ_ONCE(connector->connected)) dcp_clear_connector_edid(connector); } @@ -1056,6 +1058,7 @@ int dcp_get_modes(struct drm_connector *connector) bool vrr_capable = false; bool physical_edid; u64 edid_generation = 0; + u64 service_epoch = 0; int i; dcp_consume_edid_invalidation(apple_connector); @@ -1098,9 +1101,15 @@ int dcp_get_modes(struct drm_connector *connector) if (physical_edid) { mutex_lock(&dcp->hpd_mutex); edid_generation = dcp->typec_generation; - if (apple_connector->drm_edid && - apple_connector->drm_edid_generation != edid_generation) - dcp_clear_connector_edid(apple_connector); + } + mutex_lock(&dcp->dcpavserv.lock); + /* Consume CLOSE invalidation under the publication/retirement lock. */ + dcp_consume_edid_invalidation(apple_connector); + if (apple_connector->drm_edid && + (!dcpavserv_edid_service_current(dcp, apple_connector->drm_edid_service_epoch) || + (physical_edid && apple_connector->drm_edid_generation != edid_generation))) + dcp_clear_connector_edid(apple_connector); + if (physical_edid) { if (!dcpavserv_edid_current(dcp, edid_generation)) { dcp_clear_connector_edid(apple_connector); goto edid_done; @@ -1113,11 +1122,14 @@ int dcp_get_modes(struct drm_connector *connector) (dcp->dcpavservep && dcp->dcpavservep->inline_v2))) { const struct drm_edid *edid; /* Let physical disconnect advance the generation during the request. */ + mutex_unlock(&dcp->dcpavserv.lock); if (physical_edid) mutex_unlock(&dcp->hpd_mutex); - edid = dcpavserv_copy_edid(dcp, edid_generation); - if (physical_edid) { + edid = dcpavserv_copy_edid(dcp, edid_generation, &service_epoch); + if (physical_edid) mutex_lock(&dcp->hpd_mutex); + mutex_lock(&dcp->dcpavserv.lock); + if (physical_edid) { if (!dcpavserv_edid_current(dcp, edid_generation) || READ_ONCE(apple_connector->dcp) != pdev) { if (!IS_ERR_OR_NULL(edid)) @@ -1126,6 +1138,14 @@ int dcp_get_modes(struct drm_connector *connector) goto edid_done; } } + if (!IS_ERR_OR_NULL(edid) && + !dcpavserv_edid_service_current(dcp, service_epoch)) { + drm_edid_free(edid); + dcp_clear_connector_edid(apple_connector); + goto edid_done; + } + /* Retirement may have queued cleanup while this request waited. */ + dcp_consume_edid_invalidation(apple_connector); if (IS_ERR_OR_NULL(edid)) { /* * An internal panel has no AV service and so no EDID; @@ -1139,6 +1159,7 @@ int dcp_get_modes(struct drm_connector *connector) drm_edid_free(apple_connector->drm_edid); apple_connector->drm_edid = edid; apple_connector->drm_edid_generation = edid_generation; + apple_connector->drm_edid_service_epoch = service_epoch; } } if (dcp->nr_modes && apple_connector->drm_edid) { @@ -1172,6 +1193,7 @@ int dcp_get_modes(struct drm_connector *connector) } edid_done: + mutex_unlock(&dcp->dcpavserv.lock); if (physical_edid) mutex_unlock(&dcp->hpd_mutex); if (apple_connector->drm_edid) From 05312298cb39211ca5d0aff1ba2291ad576bb6c9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 14:32:53 +1100 Subject: [PATCH 1209/2680] drm/apple: read external H17P EDID through owned AFKV2 buffers Use the existing host-owned AFKV2 OOB command transport for the observed external H17P DPAV service. Select service admission, notification and the strict length-aware EDID reader by AFKV2 framing rather than inline payload placement. Keep legacy profiles on their existing endpoint and reader. Length-aware AFKV2 callers must copy only the validated actual reply length for both inline and owned OOB replies. Keep strict callers' full-length contract and non-AFKV2 behavior unchanged. Preserve exact original DMA address/bounds matching, command slots, timeout retention and late-ACK release, without following peer-provided pointers. Apply the public owned transport correction from 821e79667f4d935fb50e1c6bc95beb4710f316cd on the service-lifetime/RX-progress base. Retain physical generation and service epoch checks, strict EDID header/length/block/checksum validation, and the conservative firmware metadata guard. No one-shot diagnostic permission, enlarged trace capture, timeout snapshot or refresh-rate admission is included. Extend actual sender/service extraction fixtures for short owned OOB responses, strict and legacy contracts, and the real late-ACK completer. A mismatched or duplicate reply cannot release retained DMA; the matching late reply retires the pending slot even after service retirement. Adapt lifetime/notification fixtures to the framing guard. Validation: full AFKV2 suite with ASan/UBSan (917 codec assertions), EDID lifetime, threaded RX-progress and metadata regressions pass. Targeted ARM64 W=1 module compile/modpost/link passes without warnings. Initial boot-time reads and physical hotplug remain owner qualification work. [ryan: without the change to the dropped DPAV ring-snapshot tracepoint] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 2 +- drivers/gpu/drm/apple/epic/dpavservep.c | 16 ++--- drivers/gpu/drm/apple/iomfb.c | 2 +- .../drivers/gpu/apple-afkv2/extract-edid.py | 2 +- .../drivers/gpu/apple-afkv2/extract-notify.py | 8 +-- .../drivers/gpu/apple-afkv2/extract-send.py | 60 +++++++++++++++---- .../gpu/apple-afkv2/extract-service.py | 5 +- 7 files changed, 69 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index ff11b173edd69..6ab0182871d8b 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -1647,7 +1647,7 @@ static int afk_send_command_timeout_len(struct apple_epic_service *service, u8 t ret = 0; if (retcode) *retcode = service->cmds[idx].retcode; - if (ep->inline_v2 && received) + if (ep->wire_v2 && received) copy_len = service->cmds[idx].reply_length; if (ep->wire_v2 && !service->cmds[idx].retcode && service->cmds[idx].reply_length < copy_len) diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index f7a54eefcf8ea..75846d78302fa 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -23,7 +23,7 @@ static void dcpavserv_notify_edid(struct apple_epic_service *service) struct apple_dcp *dcp = service->ep->dcp; struct apple_connector *connector = READ_ONCE(dcp->connector); - if (!service->ep->inline_v2 || READ_ONCE(dcp->quiescing) || !connector || + if (!service->ep->wire_v2 || READ_ONCE(dcp->quiescing) || !connector || !READ_ONCE(dcp->nr_modes) || !READ_ONCE(connector->connected) || READ_ONCE(connector->base.registration_state) != DRM_CONNECTOR_REGISTERED || READ_ONCE(connector->dcp) != to_platform_device(dcp->dev)) @@ -41,7 +41,7 @@ static int dcpavserv_prepare(struct apple_epic_service *service, const char *nam if (!name || strcmp(name, "dcpav-service-epic")) return -ENODEV; - if (service->ep->inline_v2) { + if (service->ep->wire_v2) { if (service->ep->endpoint != DPAVSERV_ENDPOINT_V2 || !class || strcmp(class, "AppleDCPDP2HDMI") || unit != 1) return -ENODEV; @@ -75,8 +75,8 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, service->cookie = &dcp->dcpavserv; complete(&dcp->dcpavserv.enable_completion); mutex_unlock(&dcp->dcpavserv.lock); - if (service->ep->inline_v2) { - dev_info(dcp->dev, "DCPAVSERV: inline EDID service ready on channel %u\n", + if (service->ep->wire_v2) { + dev_info(dcp->dev, "DCPAVSERV: AFKV2 EDID service ready on channel %u\n", service->channel); dcpavserv_notify_edid(service); } @@ -139,7 +139,7 @@ static int dcpdpserv_prepare(struct apple_epic_service *service, const char *nam const char *class, s64 unit) { /* No production command transport is needed for this H17P service. */ - return service->ep->inline_v2 ? -ENODEV : 0; + return service->ep->wire_v2 ? -ENODEV : 0; } static void dcpdpserv_init(struct apple_epic_service *service, const char *name, @@ -352,7 +352,7 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation if (!service) return ERR_PTR(-ENODEV); - edid = service->ep->inline_v2 ? dcpavserv_read_edid_inline(service) : + edid = service->ep->wire_v2 ? dcpavserv_read_edid_inline(service) : dcpavserv_read_edid_legacy(service); mutex_lock(&dcp->dcpavserv.lock); if (dcp->dcpavserv.service != service || @@ -405,10 +405,10 @@ int dpavservep_init(struct apple_dcp *dcp) DPAVSERV_ENDPOINT_V2 : DPAVSERV_ENDPOINT, dpavservep_ops); if (IS_ERR(ep)) return PTR_ERR(ep); - /* Only the external H17P DPAV endpoint uses inline AFKV2 service calls. */ + /* External H17P uses AFKV2 service calls with host-owned OOB buffers. */ ep->wire_v2 = dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; - ep->inline_v2 = ep->wire_v2; + ep->inline_v2 = false; dcp->dcpavservep = ep; dcp->dcpavservep->match_epic_name = true; diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index a2e19ea25e245..b779323d73228 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1119,7 +1119,7 @@ int dcp_get_modes(struct drm_connector *connector) /* Internal H17P still has no supported EDID command transport. */ if (dcp->nr_modes && !apple_connector->drm_edid && (dcp->fw_compat != DCP_FIRMWARE_H17P || - (dcp->dcpavservep && dcp->dcpavservep->inline_v2))) { + (dcp->dcpavservep && dcp->dcpavservep->wire_v2))) { const struct drm_edid *edid; /* Let physical disconnect advance the generation during the request. */ mutex_unlock(&dcp->dcpavserv.lock); diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py index 0ab0f5eda8cac..0d554d1c82b5d 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py @@ -17,7 +17,7 @@ def function(name): end += 1 return source[match.start():end] -assert "service->ep->inline_v2 ? dcpavserv_read_edid_inline(service)" in source +assert "service->ep->wire_v2 ? dcpavserv_read_edid_inline(service)" in source prefix = r''' #undef NDEBUG #include diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py index 26579f8399cb1..481aac7d0246a 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py @@ -23,7 +23,7 @@ struct base { int registration_state; }; struct apple_connector { bool connected;struct base base;void *dcp;int hotplug_wq; }; struct apple_dcp { struct apple_connector *connector;bool quiescing;unsigned int nr_modes;void *dev; }; -struct ep { struct apple_dcp *dcp;bool inline_v2; }; +struct ep { struct apple_dcp *dcp;bool wire_v2; }; struct apple_epic_service { struct ep *ep; }; static unsigned int queued; static void *to_platform_device(void *p) { return p; } @@ -33,7 +33,7 @@ int main(void) { int device,other;struct apple_dcp dcp={.dev=&device}; - struct ep ep={.dcp=&dcp,.inline_v2=true};struct apple_epic_service service={.ep=&ep}; + struct ep ep={.dcp=&dcp,.wire_v2=true};struct apple_epic_service service={.ep=&ep}; struct apple_connector connector={.dcp=&device}; dcpavserv_notify_edid(&service);assert(queued==0); dcp.connector=&connector;dcpavserv_notify_edid(&service);assert(queued==0); @@ -42,8 +42,8 @@ dcpavserv_notify_edid(&service);assert(queued==1 && connector.hotplug_wq==1); dcp.quiescing=true;dcpavserv_notify_edid(&service);assert(queued==1); dcp.quiescing=false;connector.dcp=&other;dcpavserv_notify_edid(&service);assert(queued==1); - connector.dcp=&device;ep.inline_v2=false;dcpavserv_notify_edid(&service);assert(queued==1); - ep.inline_v2=true;connector.connected=false;dcpavserv_notify_edid(&service);assert(queued==1); + connector.dcp=&device;ep.wire_v2=false;dcpavserv_notify_edid(&service);assert(queued==1); + ep.wire_v2=true;connector.connected=false;dcpavserv_notify_edid(&service);assert(queued==1); puts("PASS: actual late-EDID notification queues ready connector only; legacy/quiesce/unrouted/unregistered guarded"); } ''' diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py index 92ca8857cc104..0d80a729a3342 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py @@ -13,6 +13,13 @@ while depth: depth += (source[end] == "{") - (source[end] == "}") end += 1 +reply_match = re.search(r"static void afk_recv_handle_reply\([^;]*?\)\n\{", source) +if not reply_match: + raise RuntimeError("production reply completer not found") +reply_end, reply_depth = reply_match.end(), 1 +while reply_depth: + reply_depth += (source[reply_end] == "{") - (source[reply_end] == "}") + reply_end += 1 prefix = r''' #undef NDEBUG #include @@ -24,6 +31,7 @@ #define GFP_KERNEL 0 #define MAX_PENDING_CMDS 16 #define cpu_to_le32(x) (x) +#define le32_to_cpu(x) (x) #define cpu_to_le64(x) (x) #define kmalloc(n,f) malloc(n) #define kfree(p) free(p) @@ -32,6 +40,8 @@ #define spin_unlock_irqrestore(l,f) ((void)(l),(void)(f)) #define dma_rmb() do {} while(0) #define dev_warn(...) do {} while(0) +#define dev_err(...) do {} while(0) +#define complete(p) (*(p)=1) #define EPIC_TYPE_COMMAND 3 #define EPIC_CAT_COMMAND 0x30 #define DECLARE_COMPLETION_ONSTACK(n) int n=0 @@ -50,13 +60,18 @@ struct ops { const char *name; }; struct apple_epic_service { struct apple_dcp_afkep *ep; struct ops *ops; unsigned int channel; unsigned long cmd_map[1]; - int lock; bool enabled,torndown; u8 cmd_tag; struct cmd_info cmds[16]; + int lock; unsigned int users; bool enabled,torndown; u8 cmd_tag; struct cmd_info cmds[16]; }; static struct apple_epic_service *active; static unsigned int frees, sends; static int wire_fail, timeout, reply_status; -static struct apple_epic_service *afk_service_get(struct apple_epic_service *s) { return s->enabled?s:NULL; } -static void afk_service_put(struct apple_epic_service *s) { (void)s; } +static size_t owned_reply_size=32888; +static struct apple_epic_service *afk_service_get(struct apple_epic_service *s) +{ if(!s->enabled) return NULL; s->users++; return s; } +static void afk_service_put(struct apple_epic_service *s) { assert(s->users); s->users--; } +static struct apple_epic_service *afk_epic_find_service(struct apple_dcp_afkep *ep,u32 channel) +{ return active->ep==ep && active->channel==channel && (active->enabled || active->cmd_map[0]) ? active : NULL; } +static int test_bit(unsigned int bit,const unsigned long *map) { return !!(*map & (1UL<reply_length=ep->inline_v2?4:32888;c->retcode=reply_status; + c->reply_length=ep->inline_v2?4:owned_reply_size;c->retcode=reply_status; memset(c->rxbuf,0x77,c->reply_length);c->done=true;*c->completion=1; return 0; } @@ -92,7 +107,7 @@ int main(void) { struct dcp dcp={0};struct apple_dcp_afkep ep={.dcp=&dcp,.wire_v2=true,.inline_v2=true}; - struct ops ops={.name="test"};struct apple_epic_service s={.ep=&ep,.ops=&ops,.channel=4,.enabled=true}; + struct ops ops={.name="test"};struct apple_epic_service s={.ep=&ep,.ops=&ops,.channel=4,.enabled=true,.users=1}; u8 request[112]={0x5a},*output=calloc(1,32888);size_t received;u32 status=99; active=&s; assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); @@ -107,12 +122,37 @@ wire_fail=0;ep.inline_v2=false; assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); assert(received==32888 && status==0); - ep.inline_v2=true;timeout=1; + /* Length-aware owned OOB callers copy only the bounded actual reply. */ + owned_reply_size=4;memset(output,0x19,32888); + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(received==4 && output[0]==0x77 && output[3]==0x77 && output[4]==0x19 && s.users==1); + memset(output,0x19,32888); + assert(afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,NULL)==-EMSGSIZE); + assert(output[0]==0x19 && !s.cmd_map[0]); + owned_reply_size=0;received=99; + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(received==0 && output[0]==0x19); + /* Legacy framing retains its full-buffer copy contract. */ + ep.wire_v2=false;owned_reply_size=4; + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(received==32888 && s.users==1); + ep.wire_v2=true;owned_reply_size=32888;timeout=1; assert(afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)==-ETIMEDOUT); assert(s.cmd_map[0] && s.cmds[0].free_on_ack && !s.cmds[0].completion); - /* Mock the late-reply owner's release, avoiding leaks in this host test. */ - free(s.cmds[0].rxbuf);free(s.cmds[0].txbuf);free(output); - puts("PASS: extracted production sender, 128-byte command envelope / 32888-byte reply, strict OOB and timeout ownership"); + /* Execute the real late-ACK completer: mismatch/duplicate cannot free DMA. */ + unsigned int before_late=frees;u16 retained_tag=s.cmds[0].tag; + struct epic_cmd reply={.retcode=0}; + s.enabled=false;s.torndown=true; /* Retired service still owns the pending slot. */ + afk_recv_handle_reply(&ep,4,retained_tag^0x10,&reply,sizeof(reply)); + assert(frees==before_late && s.cmd_map[0] && !s.cmds[0].done && s.users==1); + afk_recv_handle_reply(&ep,4,retained_tag,&reply,sizeof(reply)); + assert(frees==before_late+2 && !s.cmd_map[0] && s.cmds[0].done && s.users==1); + afk_recv_handle_reply(&ep,4,retained_tag,&reply,sizeof(reply)); + assert(frees==before_late+2); + s.enabled=true;s.torndown=false;timeout=0; + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(!s.cmd_map[0] && s.users==1);free(output); + puts("PASS: extracted production sender, 128-byte command envelope / 32888-byte reply, short OOB, strict/legacy callers and actual late-ACK ownership"); } ''' -pathlib.Path(sys.argv[2]).write_text(prefix + source[start:end] + suffix) +pathlib.Path(sys.argv[2]).write_text(prefix + source[start:end] + source[reply_match.start():reply_end] + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py index 68e3f5e6ba980..ab95f5c422710 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py @@ -79,8 +79,11 @@ too_long=0;wire_error=1; assert(afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)==-ETIMEDOUT); wire_error=0; ep.inline_v2=false; + received=999; + assert(!afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)); + assert(received==4 && output[0]==0xde && output[3]==0xef && output[4]==0); assert(!afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,NULL)); - puts("PASS: extracted production service-call actual request / legacy padding / short reply / status / context / length tests"); + puts("PASS: extracted production service-call actual request / OOB padding and short reply / legacy / status / context / length tests"); } ''' pathlib.Path(sys.argv[2]).write_text(prefix + source[start:end] + suffix) From a14495980a7bb384d00200f28fa9e94fb78ba7c5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 14:48:51 +1100 Subject: [PATCH 1210/2680] drm/apple: admit external AFKV2 EDID services asynchronously External H17P service announcements can arrive after the AFK endpoint starts. Waiting synchronously for the EDID service can time out just before exact admission completes, leaving a misleading startup failure despite a usable service and delaying other endpoint initialization. After a successful afk_start, return for the external AFKV2 profile and let the existing guarded admission and late connector notification publish the service. Do not claim a missing service is enabled. Retain the legacy completion wait, timeout behavior and AFK-start failure cleanup unchanged. Add host tests executing actual endpoint startup, admission, notification and retirement for early, late, permanently missing and wrong-identity announcements, plus legacy waits and endpoint-start failure. No protocol command, service identity, EDID validation, generation or lifetime fence is changed. Persistent Offline replies after physical reconnect remain separate session/attachment qualification work. [ryan: split by topic: only the drm/apple part and its selftests are here; the tools/diagnostics scripts are dropped (aurora-wip deleted them in commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"))] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/epic/dpavservep.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 75846d78302fa..131571bc6be08 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -421,6 +421,10 @@ int dpavservep_init(struct apple_dcp *dcp) return ret; } + /* Admission may arrive after startup; exact guards and late notify own it. */ + if (ep->wire_v2) + return 0; + ret = wait_for_completion_timeout(&dcp->dcpavserv.enable_completion, msecs_to_jiffies(1000)); if (!ret) { From cc2c20f5f53d12b712ecabda2ae35267a0869ea8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 15:15:51 +1100 Subject: [PATCH 1211/2680] drm/apple: replace H17P AV bindings across physical reconnect H17P parse_epic_service_init maps interface-id into the variable named unit. That value is a dynamic AFKV2 interface identity, not a physical-port number. Requiring unit 1 rejects fresh AV announcements after reconnect while retaining the old offline interface. Require the exact DPAV endpoint/compartment/class and a nonzero bounded interface-id equal to the validated service channel. Preserve legacy EPICUnit 0 admission. Retire the current AV registry binding on physical disconnect, current-generation deferred disconnect, route withdrawal and cable-release teardown, outside hpd_mutex. Ordinary DPMS poweroff does not withdraw it. Carry physical-generation and registry-epoch tickets from prepare through OPEN/init so a disconnect or retirement during admission cannot publish stale work. Local retirement disables the service and marks its slot torn down; reader pins and outstanding commands still prevent reuse, and timed-out DMA remains owned until a matching late ACK. Old CLOSE/ACK handling cannot revoke a fresh binding. Recognize the observed DPAV CLOSED report (0x14, zero token, low flags byte 1, one zero payload byte) only on the instantiated DPAV endpoint. Keep the existing 0x13 report handling and other endpoints unchanged. No new host opcode or reopen of an offline interface is introduced. Host regressions execute the production parser, ownership helpers, prepare/ init, retirement, receive dispatch and late-ACK completer for dynamic interfaces 1/5/9/13, forged identities, independent generation/epoch changes, old pins/CLOSED/replies and legacy unit 0. Full AFKV2 ASan/UBSan suite, startup, EDID lifetime, external lifecycle, metadata and RX-progress tests pass. Targeted ARM64 W=1 module compile/modpost/link passes without warnings. [ryan: split by topic: only the drm/apple part and its selftests are here; the tools/diagnostics scripts are dropped (aurora-wip deleted them in commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"))] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 3 +- drivers/gpu/drm/apple/afk.h | 13 +++ drivers/gpu/drm/apple/afkv2-codec.h | 10 +++ drivers/gpu/drm/apple/dcp.c | 8 ++ drivers/gpu/drm/apple/epic/dpavservep.c | 101 ++++++++++++++++-------- 5 files changed, 102 insertions(+), 33 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 6ab0182871d8b..13d954dc062d9 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -853,7 +853,8 @@ static void afk_v2_receive(struct apple_dcp_afkep *ep, u32 channel, u32 type, if (frame.type == EPIC_SUBTYPE_ANNOUNCE_H17P) afk_recv_handle_init(ep, frame.interface, (void *)frame.payload, frame.payload_size); - else if (frame.type == AFKV2_CLOSE && frame.interface_flags == 1) + else if ((frame.type == AFKV2_CLOSE && frame.interface_flags == 1) || + (ep == ep->dcp->dcpavservep && afkv2_closed_report(&frame))) afk_recv_handle_teardown(ep, frame.interface); return; } diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 45fe9db64d30f..ed0e3443a2969 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -56,6 +56,8 @@ struct apple_epic_service { bool enabled; bool torndown; + /* Provider-owned prepare/init ticket; AFK core never interprets it. */ + struct { u64 generation, epoch; } admission; void *cookie; struct { @@ -148,6 +150,17 @@ static inline void afk_service_disable(struct apple_epic_service *service) spin_unlock_irqrestore(&service->lock, flags); } +/* Retire local ownership while keeping timed-out commands discoverable. */ +static inline void afk_service_retire(struct apple_epic_service *service) +{ + unsigned long flags; + + spin_lock_irqsave(&service->lock, flags); + service->enabled = false; + service->torndown = true; + spin_unlock_irqrestore(&service->lock, flags); +} + static inline bool afk_service_matches(struct apple_epic_service *service, u32 channel) { diff --git a/drivers/gpu/drm/apple/afkv2-codec.h b/drivers/gpu/drm/apple/afkv2-codec.h index a06a1efd5fc7e..c42788a1b2a74 100644 --- a/drivers/gpu/drm/apple/afkv2-codec.h +++ b/drivers/gpu/drm/apple/afkv2-codec.h @@ -17,6 +17,7 @@ #define AFKV2_RESPONSE 2U #define AFKV2_OPEN 0x12U #define AFKV2_CLOSE 0x13U +#define AFKV2_CLOSED 0x14U struct afkv2_frame { u8 sequence; @@ -113,6 +114,15 @@ static inline int afkv2_decode_frame(const void *record, size_t size, return 0; } +/* Closed notification: flags occupy the low byte; the high byte is opaque. */ +static inline bool afkv2_closed_report(const struct afkv2_frame *frame) +{ + return frame && !frame->token && frame->category == AFKV2_REPORT && + frame->type == AFKV2_CLOSED && + (frame->interface_flags & 0xff) == 1 && frame->payload_size == 1 && + frame->payload && !frame->payload[0]; +} + static inline int afkv2_encode_frame(void *record, size_t capacity, u8 sequence, u16 interface, u64 token, u8 type, u8 category, u16 flags, const void *payload, size_t payload_size, diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 463f0721ade1c..7688e8212bfe4 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -495,6 +495,8 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) if (sel && sel != -EOPNOTSUPP) dev_warn(dcp->dev, "DP tunnel source reset failed: %d\n", sel); } + if (dcp_is_external_h17p(dcp)) + dpavservep_detach(dcp); route->selected = false; if (dcp->active_typec_route == route) dcp->active_typec_route = NULL; @@ -2601,6 +2603,8 @@ static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical) dcp->typec_generation++; dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); } + /* HPD is fenced; retire the AV owner after releasing hpd_mutex. */ + dpavservep_detach(dcp); } ret = dcp_external_stop_locked(dcp, dcp->external_generation); if (physical) { @@ -2625,6 +2629,8 @@ static void dcp_external_stop_work(struct work_struct *work) cancel_delayed_work_sync(&dcp->placeholder_edid_wq); guard(mutex)(&dcp->external_lock); if (generation == dcp->external_generation) { + /* The deferred disconnect also covers missing out-of-band HPD. */ + dpavservep_detach(dcp); dcp_external_stop_locked(dcp, generation); dcp_external_report_disconnected(dcp); } @@ -3827,6 +3833,8 @@ static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) dcp->typec_generation++; dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); } + if (release_cable && dcp_is_external_h17p(dcp)) + dpavservep_detach(dcp); /* Block new enqueues as well as draining users of the AFK endpoints. */ disable_delayed_work_sync(&dcp->typec_reconnect_wq); disable_delayed_work_sync(&dcp->placeholder_edid_wq); diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 131571bc6be08..9231c913f5767 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -32,25 +32,51 @@ static void dcpavserv_notify_edid(struct apple_epic_service *service) schedule_work(&connector->hotplug_wq); } +static bool dcpavserv_identity(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + if (!name || strcmp(name, "dcpav-service-epic")) + return false; + if (!service->ep->wire_v2) + return unit == 0; + /* H17P's lower-case interface-id is dynamic, not a physical-port unit. */ + return service->ep->endpoint == DPAVSERV_ENDPOINT_V2 && class && + !strcmp(class, "AppleDCPDP2HDMI") && service->channel && + service->channel <= U16_MAX && unit == service->channel; +} + +static bool dcpavserv_admission_current(struct apple_dcp *dcp, u64 generation) +{ + lockdep_assert_held(&dcp->hpd_mutex); + /* Initial service discovery may precede the first Type-C attachment. */ + return !READ_ONCE(dcp->quiescing) && generation == dcp->typec_generation && + (!generation || dcp->typec_cable_connected); +} + /* Admission runs before AFKV2 OPEN and before readers can pin the service. */ static int dcpavserv_prepare(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { struct apple_dcp *dcp = service->ep->dcp; - int ret; + bool v2 = service->ep->wire_v2; + int ret = 0; - if (!name || strcmp(name, "dcpav-service-epic")) - return -ENODEV; - if (service->ep->wire_v2) { - if (service->ep->endpoint != DPAVSERV_ENDPOINT_V2 || !class || - strcmp(class, "AppleDCPDP2HDMI") || unit != 1) - return -ENODEV; - } else if (unit != 0) { + if (!dcpavserv_identity(service, name, class, unit)) return -ENODEV; - } + if (v2) + mutex_lock(&dcp->hpd_mutex); mutex_lock(&dcp->dcpavserv.lock); - ret = dcp->dcpavserv.enabled ? -EBUSY : 0; + if (dcp->dcpavserv.enabled) + ret = -EBUSY; + else if (v2 && !dcpavserv_admission_current(dcp, dcp->typec_generation)) + ret = -ENODEV; + else { + service->admission.generation = v2 ? dcp->typec_generation : 0; + service->admission.epoch = dcp->dcpavserv.epoch; + } mutex_unlock(&dcp->dcpavserv.lock); + if (v2) + mutex_unlock(&dcp->hpd_mutex); return ret; } @@ -58,29 +84,39 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { struct apple_dcp *dcp = service->ep->dcp; + bool v2 = service->ep->wire_v2; + bool published = false; trace_dcpavserv_init(dcp, unit); - - if (!dcpavserv_prepare(service, name, class, unit)) { - mutex_lock(&dcp->dcpavserv.lock); - if (dcp->dcpavserv.enabled) { - mutex_unlock(&dcp->dcpavserv.lock); - dev_err(dcp->dev, - "DCPAVSERV: unit %lld already exists\n", unit); - return; - } + if (v2) + mutex_lock(&dcp->hpd_mutex); + mutex_lock(&dcp->dcpavserv.lock); + if (dcpavserv_identity(service, name, class, unit) && + !dcp->dcpavserv.enabled && + service->admission.epoch == dcp->dcpavserv.epoch && + (!v2 || dcpavserv_admission_current(dcp, service->admission.generation))) { dcp->dcpavserv.service = afk_service_get(service); - dcp->dcpavserv.enabled = true; - dcp->dcpavserv.epoch++; - service->cookie = &dcp->dcpavserv; - complete(&dcp->dcpavserv.enable_completion); - mutex_unlock(&dcp->dcpavserv.lock); - if (service->ep->wire_v2) { - dev_info(dcp->dev, "DCPAVSERV: AFKV2 EDID service ready on channel %u\n", - service->channel); - dcpavserv_notify_edid(service); + if (dcp->dcpavserv.service) { + dcp->dcpavserv.enabled = true; + dcp->dcpavserv.epoch++; + service->cookie = &dcp->dcpavserv; + complete(&dcp->dcpavserv.enable_completion); + published = true; } } + mutex_unlock(&dcp->dcpavserv.lock); + if (v2) + mutex_unlock(&dcp->hpd_mutex); + if (!published) { + /* A generation/registry change during OPEN cannot enable stale work. */ + afk_service_retire(service); + return; + } + if (v2) { + dev_info(dcp->dev, "DCPAVSERV: AFKV2 EDID service ready on channel %u\n", + service->channel); + dcpavserv_notify_edid(service); + } } /* Defer DRM cleanup: AFK callbacks must not take mode_config.mutex. */ @@ -124,15 +160,16 @@ void dpavservep_detach(struct apple_dcp *dcp) service = dcp->dcpavserv.service; dcp->dcpavserv.service = NULL; dcp->dcpavserv.enabled = false; + /* Also reject an in-flight prepare/OPEN with no published owner yet. */ + dcp->dcpavserv.epoch++; + reinit_completion(&dcp->dcpavserv.enable_completion); if (service) { - dcp->dcpavserv.epoch++; + afk_service_retire(service); dcpavserv_invalidate_edid(dcp); } mutex_unlock(&dcp->dcpavserv.lock); - if (service) { - afk_service_disable(service); + if (service) afk_service_put(service); - } } static int dcpdpserv_prepare(struct apple_epic_service *service, const char *name, From bf490aedc2d588f923afa2bb2250cb0e0550bbac Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 13 Oct 2026 07:38:47 -0400 Subject: [PATCH 1212/2680] drm/apple: enable the validated J700 external display profile by default Use native PMGR bandwidth scratch and clock-request offsets from the T8140 hardware profile. Keep diagnostic overrides bounded and retain the disable switch and loader firmware validation. Put the existing tested external route and scanout window in the normal J700 tree so users need no manual command line or alternate DTB. [ryan: split by topic: only the drm/apple part and its selftests are here; the DT hunks are in 000-devicetree] --- drivers/gpu/drm/apple/dcp-external-bw.h | 15 ++++++++++++ drivers/gpu/drm/apple/dcp.c | 23 +++++++++++-------- .../drivers/gpu/test_t8140_external_bw.py | 12 ++++++++++ 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-external-bw.h b/drivers/gpu/drm/apple/dcp-external-bw.h index 439db008b5084..a3a799ddefbd7 100644 --- a/drivers/gpu/drm/apple/dcp-external-bw.h +++ b/drivers/gpu/drm/apple/dcp-external-bw.h @@ -21,4 +21,19 @@ static inline bool apple_dcp_external_clock_window(u64 start, u64 size, return start == expected_start && size == expected_size; } +/* Resolve optional diagnostic overrides without changing the SoC defaults. */ +static inline bool apple_dcp_external_bw_resolve(u64 size, int default_scratch, + int default_request, int override_scratch, + int override_request, u32 *scratch, u32 *request) +{ + int s = override_scratch == -1 ? default_scratch : override_scratch; + int r = override_request == -1 ? default_request : override_request; + + if (!apple_dcp_external_bw_valid(size, s, r)) + return false; + *scratch = s; + *request = r; + return true; +} + #endif diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 7688e8212bfe4..f30e1283c9d5e 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -139,20 +139,20 @@ static void dcp_release_rtkit(void *data) #define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) -static bool experimental_h17p_external; +static bool experimental_h17p_external = true; module_param(experimental_h17p_external, bool, 0444); MODULE_PARM_DESC(experimental_h17p_external, - "Enable experimental T8140 external display protocol"); + "Enable T8140 external display protocol (default: enabled)"); static int experimental_h17p_external_bw_scratch = -1; module_param(experimental_h17p_external_bw_scratch, int, 0444); MODULE_PARM_DESC(experimental_h17p_external_bw_scratch, - "Experimental external bandwidth scratch offset; unset by default"); + "Override external bandwidth scratch offset (-1 uses the hardware profile)"); static int experimental_h17p_external_bw_request = -1; module_param(experimental_h17p_external_bw_request, int, 0444); MODULE_PARM_DESC(experimental_h17p_external_bw_request, - "Experimental external bandwidth request offset; unset by default"); + "Override external bandwidth request offset (-1 uses the hardware profile)"); bool dcp_h17p_external_enabled(void) { @@ -4232,14 +4232,16 @@ static int dcp_platform_probe(struct platform_device *pdev) if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { if (!dcp->hw.firmware_clock || - !apple_dcp_external_bw_valid(resource_size(dcp->hw.firmware_clock), + !apple_dcp_external_bw_resolve(resource_size(dcp->hw.firmware_clock), + dcp->hw.firmware_scratch, + dcp->hw.firmware_request, experimental_h17p_external_bw_scratch, - experimental_h17p_external_bw_request)) + experimental_h17p_external_bw_request, + &dcp->hw.firmware_scratch, + &dcp->hw.firmware_request)) return dev_err_probe(dev, -EINVAL, - "External bandwidth offsets are unset or invalid\n"); + "External bandwidth offsets are invalid\n"); iomfb_pause_presentation(dcp, true); - dcp->hw.firmware_scratch = experimental_h17p_external_bw_scratch; - dcp->hw.firmware_request = experimental_h17p_external_bw_request; } @@ -4594,6 +4596,9 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8140_external = { .firmware_compat = DCP_FIRMWARE_H17P, .firmware_clock = &t8140_firmware_clock, .firmware_display_windows = 4, + /* Native PMGR BWR agent 2 and boost agent 3 for DISPEXT0. */ + .firmware_scratch = 0x28000, + .firmware_request = 0x70000, }; /* diff --git a/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py b/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py index 73f0dba8d4d22..f55107c0ddaa6 100644 --- a/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py +++ b/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py @@ -14,6 +14,18 @@ def test_shared_header_boundaries(self): #include "dcp-external-bw.h" int main(void) { const u64 n=0xbc000; + u32 scratch=1, request=2; + assert(apple_dcp_external_bw_resolve(n,0x28000,0x70000,-1,-1,&scratch,&request)); + assert(scratch==0x28000 && request==0x70000); + assert(apple_dcp_external_bw_resolve(n,0x28000,0x70000,0x30000,-1,&scratch,&request)); + assert(scratch==0x30000 && request==0x70000); + assert(apple_dcp_external_bw_resolve(n,0x28000,0x70000,-1,0x74000,&scratch,&request)); + assert(scratch==0x28000 && request==0x74000); + assert(!apple_dcp_external_bw_resolve(n,0x28000,0x70000,-2,-1,&scratch,&request)); + assert(!apple_dcp_external_bw_resolve(n,0x28000,0x70000,-1,0x28000,&scratch,&request)); + assert(!apple_dcp_external_bw_resolve(n,0x28000,0x70000,0x28002,-1,&scratch,&request)); + assert(!apple_dcp_external_bw_resolve(n,0x28000,0x70000,-1,n,&scratch,&request)); + assert(scratch==0x28000 && request==0x74000); assert(!apple_dcp_external_bw_valid(n,-1,-1)); assert(!apple_dcp_external_bw_valid(n,0,-1)); assert(!apple_dcp_external_bw_valid(n,-1,0)); From a0bb16ce286351625bc9371762a6332a31dc057b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 21:51:16 +1100 Subject: [PATCH 1213/2680] drm/apple: admit the native DP EDID provider in the guarded AV service H17P can announce DCPDP13Service in the EDID AV compartment after an attachment change. The HDMI-only class check rejects that otherwise valid identity before OPEN, leaving the connector without an EDID owner. Accept that exact native class alongside AppleDCPDP2HDMI under the existing AFKV2 AV compartment, endpoint, channel and unit constraints. Keep legacy handling, auxiliary-service policy and all attachment, generation, epoch, ownership and retirement fences unchanged. Exercise the actual announcement and admission functions with both providers over 64 flip/revalidation/return cycles, including same-generation revalidation. Retain forged identity, stale OPEN, owner, auxiliary, reader/pending-command and legacy controls. The exact parent fails the first legitimate native publication; the correction passes. Independently authored from authorized hardware observations of provider identifiers and the existing licensed source. No private implementation, firmware control flow or raw trace content was copied. [ryan: split by topic: only the drm/apple part and its selftests are here; the tools/diagnostics scripts are dropped (aurora-wip deleted them in commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"))] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/epic/dpavservep.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 9231c913f5767..b3858de153c8c 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -41,7 +41,8 @@ static bool dcpavserv_identity(struct apple_epic_service *service, const char *n return unit == 0; /* H17P's lower-case interface-id is dynamic, not a physical-port unit. */ return service->ep->endpoint == DPAVSERV_ENDPOINT_V2 && class && - !strcmp(class, "AppleDCPDP2HDMI") && service->channel && + (!strcmp(class, "AppleDCPDP2HDMI") || + !strcmp(class, "DCPDP13Service")) && service->channel && service->channel <= U16_MAX && unit == service->channel; } From 5cf73676100aaa67a51bce685bf1977f18c6c1a7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 15:52:10 +0000 Subject: [PATCH 1214/2680] drm/apple: Compose the qualified J700 external display path Merge commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate") replaced the experimental external H17P switches with the validated profile. It carries this drm/apple code in its own resolution; no commit has it: - The external profile is DCP_IOMFB_METHODS_H17P_EXTERNAL, no longer "experimental". It is admitted when the DCPEXT node describes the supported firmware (26.6.2), not through a module parameter. The bandwidth override parameters are gone. - AFK sends no EPIC messages from the internal H17P profile. The compact send experiment is removed: external services use AFKv2. - Bounded 1920x1080 at 180 Hz admission (dcp-180hz.c, dcp-180hz-policy.h). The RGB8 candidate is published only while the physical link rate and lane facts proven for the current Type-C generation, route, PHY and service stay current. The proof is retried a bounded number of times. - Mode catalogs carry the AV service epoch and the metadata tickets of the receipt they came from. A replacement catalog is published once, under its physical generation, with removals notified as well. - A physical-absence tombstone for the external receive path. - The one-off capture diagnostics are removed: callback size reports, A473 reply logging and the stored power reply. - The Apple DCP selftest suite (tools/testing/selftests/drivers/gpu/ apple-dcp) that exercises admission, publication, PHY, ownership, retry and the legacy paths, and the matching apple-afkv2 and bandwidth test updates. Re-made from that merge resolution for the 7.2.10 port. The merge's other paths (PHY, USB, SEP, ISP, the selftests Makefile TARGETS line and the removed tools/diagnostics) belong to their own topics. [ryan: unified assembly: the DWC3 lifecycle fixtures under dwc3/ are reformatted to kernel style (clang-format with the tree's .clang-format) here and in every later commit that touches them, which clears the 139 checkpatch errors the 7.2.10 rebase introduced; the C the host harness compiles is unchanged in meaning] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Makefile | 2 +- drivers/gpu/drm/apple/afk.c | 52 +- drivers/gpu/drm/apple/apple_drv.c | 4 +- drivers/gpu/drm/apple/dcp-180hz-policy.h | 101 + drivers/gpu/drm/apple/dcp-180hz.c | 260 ++ drivers/gpu/drm/apple/dcp-180hz.h | 34 + drivers/gpu/drm/apple/dcp-external-bw.h | 27 +- drivers/gpu/drm/apple/dcp-internal.h | 16 +- drivers/gpu/drm/apple/dcp.c | 115 +- drivers/gpu/drm/apple/dcp.h | 2 +- drivers/gpu/drm/apple/dptxep.c | 65 +- drivers/gpu/drm/apple/epic/dpavservep.c | 6 +- drivers/gpu/drm/apple/ibootep.c | 2 +- drivers/gpu/drm/apple/iomfb.c | 163 +- drivers/gpu/drm/apple/iomfb_h17p.c | 25 +- drivers/gpu/drm/apple/iomfb_template.c | 99 +- drivers/gpu/drm/apple/parser.c | 226 +- drivers/gpu/drm/apple/parser.h | 11 + .../drivers/gpu/apple-afkv2/extract-ring.py | 2 +- .../selftests/drivers/gpu/apple-dcp/Makefile | 4 + .../selftests/drivers/gpu/apple-dcp/README | 56 + .../selftests/drivers/gpu/apple-dcp/common.py | 45 + .../drivers/gpu/apple-dcp/dwc3/cases.c | 511 ++++ .../drivers/gpu/apple-dcp/dwc3/mocks.c | 307 +++ .../drivers/gpu/apple-dcp/dwc3/prefix.c | 96 + .../selftests/drivers/gpu/apple-dcp/edid.py | 789 ++++++ .../selftests/drivers/gpu/apple-dcp/idle.py | 185 ++ .../drivers/gpu/apple-dcp/lifecycle.py | 1194 ++++++++ .../selftests/drivers/gpu/apple-dcp/phy.py | 2399 +++++++++++++++++ .../selftests/drivers/gpu/apple-dcp/pm.py | 453 ++++ .../selftests/drivers/gpu/apple-dcp/rate.py | 1125 ++++++++ .../selftests/drivers/gpu/apple-dcp/run.py | 112 + .../selftests/drivers/gpu/apple-dcp/run.sh | 4 + .../drivers/gpu/test_t8140_external_bw.py | 13 +- 34 files changed, 8275 insertions(+), 230 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp-180hz-policy.h create mode 100644 drivers/gpu/drm/apple/dcp-180hz.c create mode 100644 drivers/gpu/drm/apple/dcp-180hz.h create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/Makefile create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/README create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/common.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/cases.c create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/prefix.c create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/edid.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/idle.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/phy.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/pm.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/rate.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/run.py create mode 100755 tools/testing/selftests/drivers/gpu/apple-dcp/run.sh diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 0acdc4a7b127f..849d187c39385 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -6,7 +6,7 @@ appledrm-y := apple_drv.o appledrm-y += afk.o dcp.o dcp_backlight.o dptxep.o iomfb.o parser.o systemep.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += audio.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += av.o -appledrm-y += connector.o dcp-display-metadata.o +appledrm-y += connector.o dcp-display-metadata.o dcp-180hz.o appledrm-y += ibootep.o appledrm-y += iomfb_v12_3.o appledrm-y += iomfb_v13_3.o diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 13d954dc062d9..8a42985a091bb 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -148,7 +148,7 @@ int afk_start(struct apple_dcp_afkep *ep) ret = wait_for_completion_timeout(&ep->started, msecs_to_jiffies(1000)); if (ret <= 0 && ep->dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + DCP_IOMFB_METHODS_H17P_EXTERNAL) dev_warn(ep->dcp->dev, "external AFK startup ep=%#x tx_ready=%u rx_ready=%u bytes=%zu\n", ep->endpoint, ep->txbfr.ready, ep->rxbfr.ready, ep->bfr_size); @@ -641,7 +641,7 @@ static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, "AFK[ep:%02x]: channel %d received unhandled standard service message: %x / %x\n", ep->endpoint, channel, type, eshdr->category); if (ep->dcp->hw.iomfb_method_profile != - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + DCP_IOMFB_METHODS_H17P_EXTERNAL) print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, payload, payload_size, true); } @@ -1027,7 +1027,7 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, dev_err(ep->dcp->dev, "AFK[ep:%02x]: channel %d received unhandled message " "(type %x subtype %x)\n", ep->endpoint, channel, type, subtype); if (ep->dcp->hw.iomfb_method_profile != - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + DCP_IOMFB_METHODS_H17P_EXTERNAL) print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, payload, payload_size, true); return; @@ -1212,38 +1212,16 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, u8 v2_command[AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE]; u8 v2_category = 0; u16 v2_flags = 0; - bool compact = !ep->wire_v2 && ep->dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; int ret; - /* - * H17P frames EPIC messages differently (see afk_recv_handle()): an - * 8-byte sub-header, with the service channel carried in the EPIC - * header on queue channel 0. Only the receive side of that format is - * known, and the 24-byte framing below would be misparsed, so send - * nothing for the established internal profile. The external-only - * experiment below tests receive-layout symmetry. Internal callers are: the - * callers are optional (verbose firmware logging, EDID copy, audio, - * DPTX, debugfs) or replies to calls only DPTX services make. - */ - if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P && !compact && !ep->wire_v2) { + /* External H17P services use AFKV2; the internal profile has no EPIC sender. */ + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P && !ep->wire_v2) { dev_dbg_once(ep->dcp->dev, "AFK[ep:%02x]: not sending EPIC message on H17P\n", ep->endpoint); return -EOPNOTSUPP; } - /* Only existing DPTX/EDID service commands participate in this trial. */ - if (compact && - (channel > U16_MAX || payload_len > SZ_1M || - (payload_len && !payload) || stype != EPIC_SUBTYPE_STD_SERVICE || - (ep->endpoint != DPTX_ENDPOINT && ep->endpoint != DPAVSERV_ENDPOINT) || - !((ecat == EPIC_CAT_COMMAND && etype == EPIC_TYPE_COMMAND && - payload_len == sizeof(struct epic_cmd)) || - (ecat == EPIC_CAT_REPLY && - (etype == EPIC_TYPE_REPLY || etype == EPIC_TYPE_NOTIFY_ACK))))) - return -EOPNOTSUPP; - if (ep->wire_v2) { if (!channel || channel > U16_MAX || tag > U8_MAX || payload_len > SZ_1M) return -EINVAL; @@ -1289,7 +1267,7 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, goto out; } - if ((compact || ep->wire_v2) && + if (ep->wire_v2 && (!ep->txbfr.ready || !ep->txbfr.rptr || !ep->txbfr.wptr || !ep->txbfr.buf)) { ret = -ESHUTDOWN; goto out; @@ -1300,11 +1278,11 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, wptr = le32_to_cpu(*ep->txbfr.wptr); trace_afk_send_rwptr_pre(ep, rptr, wptr); total_epic_size = sizeof(*ehdr) + - ((compact || ep->wire_v2) ? sizeof(struct epic_sub_hdr_h17p) : sizeof(*eshdr)) + + (ep->wire_v2 ? sizeof(struct epic_sub_hdr_h17p) : sizeof(*eshdr)) + payload_len; total_size = sizeof(*hdr) + total_epic_size; - if (compact || ep->wire_v2) { + if (ep->wire_v2) { u32 available, needed; if (!ep->txbfr.ready || !ep->txbfr.stride || @@ -1407,7 +1385,7 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, hdr->magic = cpu_to_le32(QE_MAGIC); hdr->size = cpu_to_le32(total_epic_size); - hdr->channel = cpu_to_le32((compact || ep->wire_v2) ? 0 : channel); + hdr->channel = cpu_to_le32(ep->wire_v2 ? 0 : channel); hdr->type = cpu_to_le32(ep->wire_v2 ? 0 : etype); if (hdr2) memcpy(hdr2, hdr, sizeof(*hdr)); @@ -1421,18 +1399,6 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, if (ret) goto out; wptr += written; - } else if (compact) { - u8 *wire = ep->txbfr.buf + wptr; - - /* Experimental symmetry with the published H17P receive layout. */ - memset(wire, 0, sizeof(*ehdr) + sizeof(struct epic_sub_hdr_h17p)); - wire[0] = ep->qe_seq++; - put_unaligned_le16(channel, wire + 2); - put_unaligned_le32(total_epic_size - 8, wire + 4); - wire[16] = stype; - wire[17] = ecat; - put_unaligned_le32(tag, wire + 20); - wptr += sizeof(*ehdr) + sizeof(struct epic_sub_hdr_h17p); } else { ehdr = ep->txbfr.buf + wptr; memset(ehdr, 0, sizeof(*ehdr)); diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index c13bc5643a179..d6ea627cd3ca4 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -657,12 +657,12 @@ static const struct of_device_id apple_dcp_id_tbl[] = { {}, }; -/* An explicitly enabled external provider participates in component matching. */ +/* A supported firmware-described external provider participates in component matching. */ static bool apple_dcp_usable(struct device_node *np) { return of_device_is_available(np) && (!of_device_is_compatible(np, "apple,t8140-dcpext") || - dcp_h17p_external_enabled()); + dcp_h17p_external_supported(np)); } static int apple_drm_init_dcp(struct device *dev) diff --git a/drivers/gpu/drm/apple/dcp-180hz-policy.h b/drivers/gpu/drm/apple/dcp-180hz-policy.h new file mode 100644 index 0000000000000..a221c0e4a0773 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-180hz-policy.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DCP_180HZ_POLICY_H +#define APPLE_DCP_180HZ_POLICY_H + +#include +#include +#include + +/* Only this qualified uncompressed configuration is admitted. */ +static inline bool dcp_180hz_payload(u32 rate, u32 lanes, u32 clock) +{ + return rate == 0x1e && lanes == 2 && clock == 414090 && + (u64)clock * 24 <= (u64)8100000 * lanes * 8 / 10; +} + +static inline bool dcp_180hz_dtd(const u8 *d, const struct drm_display_mode *m) +{ + int ha = d[2] | (d[4] & 0xf0) << 4; + int hb = d[3] | (d[4] & 0x0f) << 8; + int va = d[5] | (d[7] & 0xf0) << 4; + int vb = d[6] | (d[7] & 0x0f) << 8; + int hf = d[8] | (d[11] & 0xc0) << 2; + int hs = d[9] | (d[11] & 0x30) << 4; + int vf = (d[10] >> 4) | (d[11] & 0x0c) << 2; + int vs = (d[10] & 0x0f) | (d[11] & 0x03) << 4; + u32 flags = (d[17] & 2 ? DRM_MODE_FLAG_PHSYNC : DRM_MODE_FLAG_NHSYNC) | + (d[17] & 4 ? DRM_MODE_FLAG_PVSYNC : DRM_MODE_FLAG_NVSYNC); + + return ((d[0] | d[1] << 8) * 10 == m->clock) && m->clock == 414090 && + (d[17] & 0x98) == 0x18 && ha == 1920 && va == 1080 && + ha + hb == 2080 && va + vb == 1106 && + m->hdisplay == ha && m->htotal == ha + hb && + m->hsync_start == ha + hf && m->hsync_end == ha + hf + hs && + m->vdisplay == va && m->vtotal == va + vb && + m->vsync_start == va + vf && m->vsync_end == va + vf + vs && + m->flags == flags; +} + +/* Bounded actual EDID only; no serial field is consulted. */ +static inline bool dcp_180hz_edid(const u8 *p, size_t len, + const struct drm_display_mode *mode) +{ + static const u8 header[] = { 0, 255, 255, 255, 255, 255, 255, 0 }; + bool name = false, timing = false, forum = false; + unsigned int names = 0, i, off, end; + + if (!p || !mode || len != 256 || memcmp(p, header, 8) || p[126] != 1 || + p[8] != 0x04 || p[9] != 0x72 || p[10] != 0x4f || p[11] != 0x0b || + p[18] != 1 || p[19] < 3 || p[19] > 4 || !(p[20] & 0x80)) + return false; + for (off = 0; off < len; off += 128) { + u8 sum = 0; + + for (i = 0; i < 128; i++) + sum += p[off + i]; + if (sum) + return false; + } + for (off = 54; off + 18 <= 126; off += 18) { + const u8 *d = p + off; + + if (d[0] || d[1]) { + timing |= dcp_180hz_dtd(d, mode); + } else if (!d[2] && d[3] == 0xfc && !d[4]) { + unsigned int n = 13; + + names++; + while (n && (d[5 + n - 1] == ' ' || d[5 + n - 1] == '\n' || + d[5 + n - 1] == 0)) + n--; + name = (n == 8 && !memcmp(d + 5, "QG270 S3", 8)) || + (n == 7 && !memcmp(d + 5, "QG270S3", 7)); + } + } + p += 128; + if (p[0] != 2 || p[1] < 3 || p[2] < 4 || p[2] > 127) + return false; + end = p[2]; + for (off = 4; off < end;) { + const u8 *db = p + off; + unsigned int size = (db[0] & 31) + 1; + + if (size > end - off) + return false; + if ((db[0] >> 5) == 3 && size >= 4 && + db[1] == 0xd8 && db[2] == 0x5d && db[3] == 0xc4) { + /* HF-VSDB overrides the older HDMI VSDB clock, as drm_edid does. */ + if (forum || size < 8 || db[4] != 1 || !(db[6] & 0x80) || + (u32)db[5] * 5000 < (u32)mode->clock) + return false; + forum = true; + } + off += size; + } + for (off = end; off + 18 <= 127; off += 18) + if (p[off] || p[off + 1]) + timing |= dcp_180hz_dtd(p + off, mode); + return name && names == 1 && timing && forum; +} + +#endif diff --git a/drivers/gpu/drm/apple/dcp-180hz.c b/drivers/gpu/drm/apple/dcp-180hz.c new file mode 100644 index 0000000000000..8eb4166aa6dbb --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-180hz.c @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include "connector.h" +#include "afk.h" +#include "dcp.h" +#include "parser.h" +#include "dcp-180hz.h" +#include "dcp-180hz-policy.h" + +static bool dcp_180hz_direct_port(struct apple_epic_service *service) +{ + struct apple_dcp *dcp; + struct dptx_port *port; + + if (!service || !service->ep || !service->cookie) + return false; + dcp = service->ep->dcp; + port = service->cookie; + return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && + service->ep->endpoint == DPTX_ENDPOINT_V2 && service->ep->wire_v2 && + port == &dcp->dptxport[0] && READ_ONCE(port->enabled) && + !READ_ONCE(port->closed) && READ_ONCE(port->service) == service && + READ_ONCE(service->enabled) && port->atcphy && port->atcphy == dcp->phy && + dcp->active_typec_route && !dcp->dptx_tunnel && !dcp_is_usb4_output(dcp); +} + +void dcp_180hz_invalidate(struct apple_dcp *dcp) +{ + unsigned long flags; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + dcp->rate_proof.sequence++; + dcp->rate_proof.rate.valid = false; + dcp->rate_proof.lanes.valid = false; + dcp->rate_proof.retries = 0; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); +} + +struct dcp_180hz_token dcp_180hz_config_begin(struct apple_epic_service *service, bool lanes) +{ + struct apple_dcp *dcp = service->ep->dcp; + struct dcp_180hz_token token = {}; + unsigned long flags; + + /* RX callbacks never acquire lifecycle/HPD/AV mutexes. */ + spin_lock_irqsave(&dcp->metadata.lock, flags); + token.generation = READ_ONCE(dcp->typec_generation); + token.sequence = dcp->rate_proof.sequence; + token.connection_sequence = dcp->rate_proof.connection_sequence; + token.phy = dcp->phy; + token.route = dcp->active_typec_route; + token.service = service; + token.valid = !READ_ONCE(dcp->quiescing) && dcp_180hz_direct_port(service) && + READ_ONCE(dcp->typec_cable_connected) && + token.generation == dcp->metadata.epoch.generation; + if (lanes) + dcp->rate_proof.lanes.valid = false; + else + dcp->rate_proof.rate.valid = false; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + return token; +} + +/* Preserve the receipt-start registry ticket even when it awaited OPEN. */ +static bool dcp_180hz_timing_epoch(const struct dcp_display_mode *mode, u64 epoch) +{ + if (mode->automatic_service_pending) + return mode->unsafe_rgb8_service_epoch != U64_MAX && + epoch == mode->unsafe_rgb8_service_epoch + 1; + return epoch && mode->unsafe_rgb8_service_epoch == epoch; +} + +void dcp_180hz_config_complete(struct apple_epic_service *service, bool lanes, + struct dcp_180hz_token token, u32 value, bool success) +{ + struct apple_dcp *dcp = service->ep->dcp; + struct dcp_180hz_fact *fact; + bool notify = false; + unsigned long flags; + + if (!success) { + dcp_180hz_invalidate(dcp); + return; + } + spin_lock_irqsave(&dcp->metadata.lock, flags); + if (token.valid && token.service == service && + token.sequence == dcp->rate_proof.sequence && + token.generation == READ_ONCE(dcp->typec_generation) && + token.connection_sequence == dcp->rate_proof.connection_sequence && + READ_ONCE(dcp->typec_cable_connected) && + token.phy == dcp->phy && token.route == dcp->active_typec_route && + dcp_180hz_direct_port(service) && !READ_ONCE(dcp->quiescing)) { + fact = lanes ? &dcp->rate_proof.lanes : &dcp->rate_proof.rate; + *fact = (struct dcp_180hz_fact) { .token = token, .value = value, .valid = true }; + notify = dcp->rate_proof.rate.valid && dcp->rate_proof.lanes.valid; + dcp->rate_proof.retries = 0; + } + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (notify) + dcp_180hz_notify(dcp); +} + +static bool dcp_180hz_fact_current(struct apple_dcp *dcp, const struct dcp_180hz_fact *fact) +{ + const struct dcp_180hz_token *t = &fact->token; + + return fact->valid && t->generation == dcp->typec_generation && + t->sequence == dcp->rate_proof.sequence && + t->connection_sequence == dcp->rate_proof.connection_sequence && t->phy == dcp->phy && + t->route == dcp->active_typec_route && t->service == dcp->dptxport[0].service; +} + +/* Catalog withdrawal needs a DRM refresh even when no exception is ready. */ +void dcp_180hz_catalog_changed(struct apple_dcp *dcp) +{ + struct apple_connector *con = READ_ONCE(dcp->connector); + + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL || + !con || READ_ONCE(dcp->quiescing) || !READ_ONCE(con->connected) || + READ_ONCE(con->dcp) != to_platform_device(dcp->dev) || + READ_ONCE(con->base.registration_state) != DRM_CONNECTOR_REGISTERED) + return; + schedule_work(&con->hotplug_wq); +} + +/* Timing and configuration can arrive in either order, before AV admission. */ +void dcp_180hz_notify(struct apple_dcp *dcp) +{ + struct apple_connector *con = READ_ONCE(dcp->connector); + unsigned long flags; + bool ready; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + ready = dcp->metadata.epoch.connected && READ_ONCE(dcp->typec_cable_connected) && + !READ_ONCE(dcp->quiescing) && READ_ONCE(dcp->nr_modes) && + dcp_180hz_direct_port(dcp->dptxport[0].service) && + dcp_180hz_fact_current(dcp, &dcp->rate_proof.rate) && + dcp_180hz_fact_current(dcp, &dcp->rate_proof.lanes) && + dcp_180hz_payload(dcp->rate_proof.rate.value, dcp->rate_proof.lanes.value, 414090); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (ready && con && READ_ONCE(con->connected) && READ_ONCE(dcp->dptxport[0].connected) && + READ_ONCE(con->dcp) == to_platform_device(dcp->dev) && + READ_ONCE(con->base.registration_state) == DRM_CONNECTOR_REGISTERED) + schedule_work(&con->hotplug_wq); +} + +bool dcp_180hz_mode_current(struct apple_dcp *dcp, const struct dcp_display_mode *mode) +{ + struct apple_connector *con = dcp->connector; + const struct edid *raw; + + lockdep_assert_held(&dcp->hpd_mutex); + lockdep_assert_held(&dcp->dcpavserv.lock); + lockdep_assert_held(&dcp->metadata.lock); + if (!con || dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL || + !dcp_180hz_direct_port(dcp->dptxport[0].service) || + !mode->unsafe_rgb8_ticket.associated || + mode->unsafe_rgb8_ticket.generation != dcp->typec_generation || + mode->automatic_connection_sequence != dcp->rate_proof.connection_sequence || + mode->automatic_catalog_sequence != dcp->rate_proof.catalog_sequence || + !dcp->metadata.epoch.connected || !dcp->typec_cable_connected || dcp->quiescing || + READ_ONCE(con->dcp) != to_platform_device(dcp->dev) || + !dcpavserv_edid_service_current(dcp, con->drm_edid_service_epoch) || + !dcp_180hz_timing_epoch(mode, dcp->dcpavserv.epoch) || + !con->drm_edid || con->drm_edid_firmware_derived || + READ_ONCE(con->edid_invalidation_pending) || + con->drm_edid_generation != dcp->typec_generation || + !dcp->dptxport[0].connected || !dcp->dptxport[0].enabled || dcp->dptxport[0].closed || + !dcp_180hz_fact_current(dcp, &dcp->rate_proof.rate) || + !dcp_180hz_fact_current(dcp, &dcp->rate_proof.lanes) || + !dcp_180hz_payload(dcp->rate_proof.rate.value, dcp->rate_proof.lanes.value, mode->mode.clock) || + mode->sdr_rgb.depth != 8 || mode->sdr_rgb.format != DCP_COLOR_FORMAT_RGB || + mode->sdr_rgb.range != DCP_COLOR_YCBCR_RANGE_FULL || + mode->sdr_rgb.eotf != DCP_EOTF_SDR_GAMMA || mode->sdr_rgb.colorimetry != DCP_COLORIMETRY_SRGB) + return false; + raw = drm_edid_raw(con->drm_edid); + /* The transport validates extent/checksums; this quirk accepts only its 256-byte form. */ + return raw && raw->extensions == 1 && + dcp_180hz_edid((const u8 *)raw, 256, &mode->mode); +} + +bool dcp_180hz_pending(struct apple_dcp *dcp, const struct dcp_display_mode *mode) +{ + struct apple_connector *con = dcp->connector; + const struct edid *raw; + + if (!con || !dcp->active_typec_route || dcp_is_usb4_output(dcp) || dcp->dptx_tunnel || + !mode->unsafe_rgb8_ticket.associated || + mode->unsafe_rgb8_ticket.generation != dcp->typec_generation || + mode->automatic_connection_sequence != dcp->rate_proof.connection_sequence || + mode->automatic_catalog_sequence != dcp->rate_proof.catalog_sequence || + (!dcp_180hz_timing_epoch(mode, dcp->dcpavserv.epoch) && + !(mode->automatic_service_pending && !dcp->dcpavserv.enabled && + mode->unsafe_rgb8_service_epoch == dcp->dcpavserv.epoch))) + return false; + if (!con->drm_edid) + return true; + raw = drm_edid_raw(con->drm_edid); + return raw && raw->extensions == 1 && dcp_180hz_edid((const u8 *)raw, 256, &mode->mode); +} + +static void dcp_180hz_retry_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, rate_retry_wq); + unsigned long flags; + bool current_generation; + + guard(mutex)(&dcp->hpd_mutex); + spin_lock_irqsave(&dcp->metadata.lock, flags); + current_generation = dcp->typec_cable_connected && !dcp->quiescing && + dcp->rate_proof.retry_generation == dcp->typec_generation; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (current_generation && dcp->connector && + READ_ONCE(dcp->connector->dcp) == to_platform_device(dcp->dev)) + schedule_work(&dcp->connector->hotplug_wq); +} + +void dcp_180hz_retry(struct apple_dcp *dcp) +{ + unsigned long flags; + bool retry = false; + + if (!dcp->rate_retry_initialized || READ_ONCE(dcp->quiescing)) + return; + spin_lock_irqsave(&dcp->metadata.lock, flags); + if (dcp->rate_proof.retry_generation != READ_ONCE(dcp->typec_generation)) { + dcp->rate_proof.retry_generation = READ_ONCE(dcp->typec_generation); + dcp->rate_proof.retries = 0; + } + if (READ_ONCE(dcp->typec_cable_connected) && dcp->rate_proof.retries < 8) { + dcp->rate_proof.retries++; + retry = true; + } + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (retry) + mod_delayed_work(system_freezable_wq, &dcp->rate_retry_wq, msecs_to_jiffies(300)); +} + +static void dcp_180hz_cancel(void *data) +{ + struct apple_dcp *dcp = data; + + WRITE_ONCE(dcp->rate_retry_initialized, false); + cancel_delayed_work_sync(&dcp->rate_retry_wq); +} + +int dcp_180hz_init(struct apple_dcp *dcp) +{ + int ret; + + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL) + return 0; + INIT_DELAYED_WORK(&dcp->rate_retry_wq, dcp_180hz_retry_work); + ret = devm_add_action_or_reset(dcp->dev, dcp_180hz_cancel, dcp); + if (!ret) + dcp->rate_retry_initialized = true; + return ret; +} diff --git a/drivers/gpu/drm/apple/dcp-180hz.h b/drivers/gpu/drm/apple/dcp-180hz.h new file mode 100644 index 0000000000000..4995dae4cf648 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-180hz.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DCP_180HZ_H +#define APPLE_DCP_180HZ_H +#include +struct apple_dcp; +struct apple_epic_service; +struct dcp_display_mode; +struct dcp_180hz_token { + /* Successful physical configuration is independent of AV discovery. */ + u64 generation, sequence, connection_sequence; + const void *phy, *route, *service; + bool valid; +}; +struct dcp_180hz_fact { + struct dcp_180hz_token token; + u32 value; + bool valid; +}; +struct dcp_180hz_proof { + u64 sequence, connection_sequence, catalog_sequence, retry_generation; + unsigned int retries; + struct dcp_180hz_fact rate, lanes; +}; +struct dcp_180hz_token dcp_180hz_config_begin(struct apple_epic_service *service, bool lanes); +void dcp_180hz_config_complete(struct apple_epic_service *service, bool lanes, + struct dcp_180hz_token token, u32 value, bool success); +void dcp_180hz_invalidate(struct apple_dcp *dcp); +void dcp_180hz_notify(struct apple_dcp *dcp); +void dcp_180hz_catalog_changed(struct apple_dcp *dcp); +bool dcp_180hz_mode_current(struct apple_dcp *dcp, const struct dcp_display_mode *mode); +bool dcp_180hz_pending(struct apple_dcp *dcp, const struct dcp_display_mode *mode); +void dcp_180hz_retry(struct apple_dcp *dcp); +int dcp_180hz_init(struct apple_dcp *dcp); +#endif diff --git a/drivers/gpu/drm/apple/dcp-external-bw.h b/drivers/gpu/drm/apple/dcp-external-bw.h index a3a799ddefbd7..99430d3e4d883 100644 --- a/drivers/gpu/drm/apple/dcp-external-bw.h +++ b/drivers/gpu/drm/apple/dcp-external-bw.h @@ -5,6 +5,19 @@ #include +#define DCP_EXTERNAL_FW_CELLS 3 + +/* The external H17P wire contract is supported only for this firmware release. */ +static inline bool +apple_dcp_external_firmware_supported(const u32 *firmware, int firmware_cells, + const u32 *compat, int compat_cells) +{ + return firmware && compat && firmware_cells == DCP_EXTERNAL_FW_CELLS && + compat_cells == DCP_EXTERNAL_FW_CELLS && + firmware[0] == 26 && firmware[1] == 6 && firmware[2] == 2 && + compat[0] == 26 && compat[1] == 6 && compat[2] == 2; +} + static inline bool apple_dcp_external_bw_valid(u64 size, int scratch, int request) { if (size < sizeof(u32) || scratch < 0 || request < 0) @@ -21,19 +34,5 @@ static inline bool apple_dcp_external_clock_window(u64 start, u64 size, return start == expected_start && size == expected_size; } -/* Resolve optional diagnostic overrides without changing the SoC defaults. */ -static inline bool apple_dcp_external_bw_resolve(u64 size, int default_scratch, - int default_request, int override_scratch, - int override_request, u32 *scratch, u32 *request) -{ - int s = override_scratch == -1 ? default_scratch : override_scratch; - int r = override_request == -1 ? default_request : override_request; - - if (!apple_dcp_external_bw_valid(size, s, r)) - return false; - *scratch = s; - *request = r; - return true; -} #endif diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 6e6d0d7b6ff50..d6a37a0240788 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -20,6 +20,7 @@ #include "dcp_backlight.h" #include "dcp-display-metadata.h" #include "dcp-metadata-epoch.h" +#include "dcp-180hz.h" #include "iomfb.h" #include "iomfb_h17p.h" #include "iomfb-state.h" @@ -106,6 +107,10 @@ struct dcp_chunks { bool contiguous; bool payload_complete; struct dcp_metadata_ticket metadata_ticket; + u64 av_service_epoch; /* Captured at property start, never at publication. */ + struct dcp_metadata_ticket mode_ticket; + u64 automatic_connection_sequence; + bool automatic_service_pending; }; #define DCP_MAX_MAPPINGS (128) /* should be enough */ @@ -179,7 +184,7 @@ struct dcp_panel { enum dcp_iomfb_method_profile { DCP_IOMFB_METHODS_DEFAULT, DCP_IOMFB_METHODS_H17G, - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL, + DCP_IOMFB_METHODS_H17P_EXTERNAL, }; struct apple_dcp_hw_data { @@ -224,6 +229,8 @@ struct apple_dcp { u64 external_rx_generation; u64 external_stop_generation; bool external_cable; + /* RX tombstone: initially unknown; only physical OOB events change it. */ + bool external_physically_absent; bool external_running; bool external_power_attempted; /* iomfb.lock: replay the latest callback after activation succeeds. */ @@ -231,8 +238,6 @@ struct apple_dcp { bool external_hotplug_connected; bool external_stopped; int external_error; - u64 external_power_reply; - bool external_power_reply_valid; struct work_struct external_stop_work; @@ -286,6 +291,9 @@ struct apple_dcp { /* Active chunked transfer. There can only be one at a time. */ struct dcp_chunks chunks; + struct dcp_180hz_proof rate_proof; + struct delayed_work rate_retry_wq; + bool rate_retry_initialized; struct { /* Protects connection tickets and cached metadata snapshots. */ spinlock_t lock; @@ -461,8 +469,6 @@ struct apple_dcp { struct dcp_present_h17p present_h17p; struct dcp_present_state_h17p present_state_h17p; DECLARE_BITMAP(unknown_callbacks, IOMFB_MAX_CB); - /* Runtime callbacks whose record size has been reported once. */ - DECLARE_BITMAP(sized_callbacks, IOMFB_MAX_CB); /* Serializes H17P transmit preparation with RTKit receive callbacks. */ struct { diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index f30e1283c9d5e..e18b6e6274c82 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -88,7 +88,7 @@ static int dcp_pin_live_session(struct apple_dcp *dcp) if ((!dcp->hw.adopt_live_session && dcp->hw.iomfb_method_profile != - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) || + DCP_IOMFB_METHODS_H17P_EXTERNAL) || dcp->fw_compat != DCP_FIRMWARE_H17P || dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G || dcp->retain_dma) return 0; @@ -139,24 +139,22 @@ static void dcp_release_rtkit(void *data) #define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) -static bool experimental_h17p_external = true; -module_param(experimental_h17p_external, bool, 0444); -MODULE_PARM_DESC(experimental_h17p_external, - "Enable T8140 external display protocol (default: enabled)"); - -static int experimental_h17p_external_bw_scratch = -1; -module_param(experimental_h17p_external_bw_scratch, int, 0444); -MODULE_PARM_DESC(experimental_h17p_external_bw_scratch, - "Override external bandwidth scratch offset (-1 uses the hardware profile)"); - -static int experimental_h17p_external_bw_request = -1; -module_param(experimental_h17p_external_bw_request, int, 0444); -MODULE_PARM_DESC(experimental_h17p_external_bw_request, - "Override external bandwidth request offset (-1 uses the hardware profile)"); - -bool dcp_h17p_external_enabled(void) +bool dcp_h17p_external_supported(struct device_node *np) { - return experimental_h17p_external; + u32 firmware[DCP_EXTERNAL_FW_CELLS], compat[DCP_EXTERNAL_FW_CELLS]; + int firmware_cells, compat_cells; + + if (!of_machine_is_compatible("apple,j700") || + !of_device_is_compatible(np, "apple,t8140-dcpext")) + return false; + firmware_cells = of_property_read_variable_u32_array( + np, "apple,firmware-version", firmware, + DCP_EXTERNAL_FW_CELLS, DCP_EXTERNAL_FW_CELLS); + compat_cells = of_property_read_variable_u32_array( + np, "apple,firmware-compat", compat, + DCP_EXTERNAL_FW_CELLS, DCP_EXTERNAL_FW_CELLS); + return apple_dcp_external_firmware_supported(firmware, firmware_cells, + compat, compat_cells); } bool dcp_h17p_external_ready(struct device_node *np) @@ -165,7 +163,7 @@ bool dcp_h17p_external_ready(struct device_node *np) struct apple_dcp *dcp; bool ready = false; - if (!dcp_h17p_external_enabled()) + if (!dcp_h17p_external_supported(np)) return false; pdev = of_find_device_by_node(np); if (!pdev) @@ -303,7 +301,7 @@ static bool dcp_typec_route_fits(struct apple_dcp_typec_route *route, static bool dcp_is_external_h17p(struct apple_dcp *dcp) { - return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL; } static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation); @@ -1827,7 +1825,7 @@ void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now) static bool dcp_firmware_metadata_enabled(struct apple_dcp *dcp) { - return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL; } void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, bool connected) @@ -1838,6 +1836,10 @@ void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, boo return; spin_lock_irqsave(&dcp->metadata.lock, flags); if (generation != dcp->metadata.epoch.generation || !connected) { + dcp->rate_proof.connection_sequence++; + dcp->rate_proof.sequence++; + dcp->rate_proof.rate.valid = false; + dcp->rate_proof.lanes.valid = false; dcp->metadata.attributes_valid = false; dcp->metadata.timing_valid = false; dcp->metadata.hpd_seen = false; @@ -1848,6 +1850,7 @@ void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, boo void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected) { + struct apple_connector *con = READ_ONCE(dcp->connector); unsigned long flags; if (!dcp_firmware_metadata_enabled(dcp)) @@ -1855,13 +1858,29 @@ void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected) spin_lock_irqsave(&dcp->metadata.lock, flags); if (connected) { dcp->metadata.hpd_seen = true; + /* + * Fresh proof may precede HPD assertion after a same-cable loss. + * Keep invalidated firmware metadata and old receipt tickets revoked. + */ + if (READ_ONCE(dcp->typec_cable_connected) && + dcp->metadata.epoch.generation == READ_ONCE(dcp->typec_generation)) + dcp->metadata.epoch.connected = true; } else if (dcp->metadata.hpd_seen) { + dcp->metadata.hpd_seen = false; + dcp->rate_proof.connection_sequence++; + dcp->rate_proof.sequence++; + dcp->rate_proof.rate.valid = false; + dcp->rate_proof.lanes.valid = false; + if (con && READ_ONCE(con->dcp) == to_platform_device(dcp->dev)) + WRITE_ONCE(con->edid_invalidation_pending, true); dcp_metadata_epoch_connection(&dcp->metadata.epoch, dcp->metadata.epoch.generation, false); dcp->metadata.attributes_valid = false; dcp->metadata.timing_valid = false; } spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (connected) + dcp_180hz_notify(dcp); } void dcp_firmware_metadata_begin(struct apple_dcp *dcp) @@ -2312,6 +2331,12 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commi crtc_state = drm_atomic_get_new_crtc_state(commit, crtc); + /* External H17P does not program adaptive sync. */ + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL && + crtc_state->vrr_enabled) + return -EINVAL; + needs_modeset = drm_atomic_crtc_needs_modeset(crtc_state) || !READ_ONCE(dcp->mode_state.valid); if (!needs_modeset && (!dcp->connector || !dcp->connector->connected)) { @@ -2469,6 +2494,8 @@ static void dcp_external_report_disconnected(struct apple_dcp *dcp) unsigned int action; guard(mutex)(&dcp->iomfb.lock); + /* A paused callback from the old sink must not be replayed on wake. */ + dcp->external_hotplug_pending = false; action = dcp_mode_hotplug(&dcp->mode_state, false, dcp->connector ? &dcp->connector->connected : NULL); if (dcp->connector) { @@ -2598,6 +2625,7 @@ static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical) guard(mutex)(&dcp->external_lock); /* Invalidate physical EDID before waiting for firmware power/link teardown. */ if (physical) { + WRITE_ONCE(dcp->external_physically_absent, true); scoped_guard(mutex, &dcp->hpd_mutex) { WRITE_ONCE(dcp->typec_cable_connected, false); dcp->typec_generation++; @@ -2642,7 +2670,7 @@ bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected) return false; /* No lifecycle or HPD mutex may be acquired from the RX callback. */ lockdep_assert_held(&dcp->iomfb.lock); - if (READ_ONCE(dcp->quiescing)) + if (READ_ONCE(dcp->quiescing) || READ_ONCE(dcp->external_physically_absent)) return true; if (dcp->iomfb.presentation_paused) { dcp->external_hotplug_pending = true; @@ -2664,6 +2692,8 @@ bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected) return true; } if (!connected && !READ_ONCE(dcp->iomfb.presentation_paused)) { + /* Fence admission in RX before a busy actor can delay teardown. */ + dcp_firmware_metadata_hotplug(dcp, false); dcp->external_stop_generation = dcp->external_rx_generation; schedule_work(&dcp->external_stop_work); return true; @@ -2820,6 +2850,7 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) if (new_cable) dcp->external_generation++; dcp->external_cable = true; + WRITE_ONCE(dcp->external_physically_absent, false); dcp_external_publish_generation(dcp); scoped_guard(mutex, &dcp->hpd_mutex) { WRITE_ONCE(dcp->typec_cable_connected, true); @@ -2984,7 +3015,7 @@ int dcp_start(struct platform_device *pdev) } /* Make the H17P allocator available before external service discovery. */ - if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { ret = apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); if (ret) return dev_err_probe(dcp->dev, ret, "Failed to start remote allocator\n"); @@ -3009,7 +3040,7 @@ int dcp_start(struct platform_device *pdev) ret); if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + DCP_IOMFB_METHODS_H17P_EXTERNAL) { ret = dptx_controller_init(dcp); if (ret) return dev_err_probe(dcp->dev, ret, @@ -3672,7 +3703,7 @@ static int dcp_get_disp_regs(struct apple_dcp *dcp) if (dcp->hw.firmware_clock) { unsigned int windows = dcp->hw.firmware_display_windows; bool external = dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + DCP_IOMFB_METHODS_H17P_EXTERNAL; if (!windows || windows >= MAX_DISP_REGISTERS) return -EINVAL; @@ -3906,7 +3937,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) } if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + DCP_IOMFB_METHODS_H17P_EXTERNAL && !dcp_external_is_idle(dcp)) return dev_err_probe(dev, -EBUSY, "External DCP is not in the inherited idle state\n"); @@ -3985,7 +4016,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) } if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + DCP_IOMFB_METHODS_H17P_EXTERNAL && (!dcp->clk_194 || !clk_get_rate(dcp->clk) || !clk_get_rate(dcp->clk_194))) { ret = dev_err_probe(dev, -EINVAL, @@ -4001,7 +4032,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) INIT_DELAYED_WORK(&dcp->swap_watchdog_wq, dcp_swap_watchdog); if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + DCP_IOMFB_METHODS_H17P_EXTERNAL) { /* Pin before the first write or mailbox wake can expose DMA. */ ret = dcp_pin_live_session(dcp); if (ret) @@ -4026,7 +4057,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) goto err_piodma; /* External H17P supplies shared-buffer addresses during RTKit startup. */ if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + DCP_IOMFB_METHODS_H17P_EXTERNAL) dcp->rtk = apple_rtkit_init(dev, dcp, "mbox", 0, &external_h17p_rtkit_ops); else @@ -4045,7 +4076,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) /* Announce host readiness before awaiting external IOP readiness. */ if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + DCP_IOMFB_METHODS_H17P_EXTERNAL) ret = apple_rtkit_wake_early_ap(dcp->rtk); else ret = apple_rtkit_wake(dcp->rtk); @@ -4181,10 +4212,10 @@ static int dcp_platform_probe(struct platform_device *pdev) if (dcp_session_retained(dev)) return dev_err_probe(dev, -EBUSY, "Previous live DCP session requires a reboot\n"); - /* Not part of the display subsystem yet; see apple_dcp_usable(). */ + /* Reject unsupported external firmware before replacing the boot display. */ if (of_device_is_compatible(dev->of_node, "apple,t8140-dcpext") && - !dcp_h17p_external_enabled()) { - dev_info(dev, "external display coprocessor not supported yet\n"); + !dcp_h17p_external_supported(dev->of_node)) { + dev_err(dev, "Unsupported external display hardware or firmware handoff\n"); return -ENODEV; } @@ -4230,17 +4261,13 @@ static int dcp_platform_probe(struct platform_device *pdev) mutex_init(&dcp->dcpavserv.lock); dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + DCP_IOMFB_METHODS_H17P_EXTERNAL) { if (!dcp->hw.firmware_clock || - !apple_dcp_external_bw_resolve(resource_size(dcp->hw.firmware_clock), + !apple_dcp_external_bw_valid(resource_size(dcp->hw.firmware_clock), dcp->hw.firmware_scratch, - dcp->hw.firmware_request, - experimental_h17p_external_bw_scratch, - experimental_h17p_external_bw_request, - &dcp->hw.firmware_scratch, - &dcp->hw.firmware_request)) + dcp->hw.firmware_request)) return dev_err_probe(dev, -EINVAL, - "External bandwidth offsets are invalid\n"); + "Invalid external profile bandwidth resources\n"); iomfb_pause_presentation(dcp, true); } @@ -4404,7 +4431,7 @@ static int dcp_platform_probe(struct platform_device *pdev) return ret; if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + DCP_IOMFB_METHODS_H17P_EXTERNAL) { /* Complete optional startup before the DRM master can depend on it. */ ret = dcp_comp_bind(dev, NULL, NULL); if (ret) { @@ -4431,7 +4458,7 @@ static void dcp_platform_shutdown(struct platform_device *pdev) struct apple_dcp *dcp = platform_get_drvdata(pdev); if (dcp && dcp->retain_dma && dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + DCP_IOMFB_METHODS_H17P_EXTERNAL) { dcp_comp_unbind(&pdev->dev, NULL, NULL); return; } @@ -4591,7 +4618,7 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8140 = { /* Separate from the established internal-panel H17P profile. */ static const struct apple_dcp_hw_data apple_dcp_hw_t8140_external = { .num_dptx_ports = 1, - .iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL, + .iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL, .adopt_live_session = false, .firmware_compat = DCP_FIRMWARE_H17P, .firmware_clock = &t8140_firmware_clock, diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 6e8d3632ae2fa..d57c533b63822 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -12,7 +12,7 @@ #include "dcp-internal.h" #include "parser.h" -bool dcp_h17p_external_enabled(void); +bool dcp_h17p_external_supported(struct device_node *np); bool dcp_h17p_external_ready(struct device_node *np); struct apple_crtc { diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 92b4a6e390722..7bc55bfffe213 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -15,6 +15,7 @@ #include "afk.h" #include "dcp.h" #include "dptxep.h" +#include "dcp-180hz.h" #include "dptx-service.h" #include "parser.h" #include "trace.h" @@ -218,6 +219,7 @@ int dptxport_release_display(struct apple_epic_service *service) struct dptx_port *dptx = service->cookie; int ret; + dcp_180hz_invalidate(service->ep->dcp); dptx->release_calls++; dev_dbg(service->ep->dcp->dev, "DPTX release_display: call #%u this boot caller=%pS\n", @@ -500,6 +502,7 @@ dptxport_call_will_change_link_config(struct apple_epic_service *service) { struct dptx_port *dptx = service->cookie; + dcp_180hz_invalidate(service->ep->dcp); dptx->phy_ops.dp.set_lanes = 0; dptx->phy_ops.dp.set_rate = 0; dptx->phy_ops.dp.set_voltages = 0; @@ -688,6 +691,7 @@ dptxport_call_deactivate(struct apple_epic_service *service, struct dptx_port *dptx = service->cookie; struct apple_dcp *dcp = service->ep->dcp; + dcp_180hz_invalidate(dcp); dev_info(dcp->dev, "DPTXPort: DEACTIVATE\n"); if (dcp->dptx_tunnel) dcp_tunnel_dpin_activate(dcp, false); @@ -701,6 +705,40 @@ dptxport_call_deactivate(struct apple_epic_service *service, return 0; } +static int dptxport_call_qualified_config(struct apple_epic_service *service, bool lanes, + const void *data, size_t size, void *reply, size_t room) +{ + struct dcp_180hz_token token = dcp_180hz_config_begin(service, lanes); + struct dptx_port *port = service->cookie; + bool success = false; + u32 value = 0; + int ret; + + if (lanes) { + ret = dptxport_call_set_active_lane_count(service, data, size, reply, room); + if (!ret && size >= sizeof(struct dptxport_apcall_set_active_lane_count) && + room >= sizeof(struct dptxport_apcall_set_active_lane_count)) { + const struct dptxport_apcall_set_active_lane_count *request = data; + const struct dptxport_apcall_set_active_lane_count *response = reply; + + value = le64_to_cpu(request->lane_count); + success = !le32_to_cpu(response->retcode) && value == 2 && port->lane_count == 2; + } + } else { + ret = dptxport_call_set_link_rate(service, data, size, reply, room); + if (!ret && size >= sizeof(struct dptxport_apcall_link_rate) && + room >= sizeof(struct dptxport_apcall_link_rate)) { + const struct dptxport_apcall_link_rate *request = data; + const struct dptxport_apcall_link_rate *response = reply; + + value = le32_to_cpu(request->link_rate); + success = !le32_to_cpu(response->retcode) && value == LINK_RATE_HBR3 && port->link_rate == value; + } + } + dcp_180hz_config_complete(service, lanes, token, value, success); + return ret; +} + static int dptxport_call(struct apple_epic_service *service, u32 idx, const void *data, size_t data_size, void *reply, size_t reply_size) @@ -710,7 +748,7 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, dev_dbg(service->ep->dcp->dev, "DPTXPort: APCALL %u (%zu bytes)\n", idx, data_size); if (data_size && service->ep->dcp->hw.iomfb_method_profile != - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + DCP_IOMFB_METHODS_H17P_EXTERNAL) print_hex_dump_debug("DPTXPort: apcall data: ", DUMP_PREFIX_OFFSET, 16, 1, data, min(data_size, (size_t)64), true); @@ -738,8 +776,7 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, case DPTX_APCALL_GET_LINK_RATE: return dptxport_call_get_link_rate(service, reply, reply_size); case DPTX_APCALL_SET_LINK_RATE: - return dptxport_call_set_link_rate(service, data, data_size, - reply, reply_size); + return dptxport_call_qualified_config(service, false, data, data_size, reply, reply_size); case DPTX_APCALL_GET_MAX_LANE_COUNT: return dptxport_call_get_max_lane_count(service, reply, reply_size); case DPTX_APCALL_GET_ACTIVE_LANE_COUNT: { @@ -752,8 +789,7 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, return 0; } case DPTX_APCALL_SET_ACTIVE_LANE_COUNT: - return dptxport_call_set_active_lane_count(service, data, data_size, - reply, reply_size); + return dptxport_call_qualified_config(service, true, data, data_size, reply, reply_size); case DPTX_APCALL_GET_SUPPORTS_HPD: return dptxport_call_get_supports_hpd(service, reply, reply_size); @@ -766,6 +802,7 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, case DPTX_APCALL_DEVICE_NOT_RESPONDING: case DPTX_APCALL_DEVICE_BUSY_TIMEOUT: case DPTX_APCALL_DEVICE_NOT_STARTED: + dcp_180hz_invalidate(service->ep->dcp); dev_warn(service->ep->dcp->dev, "DPTXPort: firmware reports link fault %u on target %u:%u\n", idx, service->ep->dcp->dptx_die, @@ -809,7 +846,7 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, return 0; default: if (service->ep->dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + DCP_IOMFB_METHODS_H17P_EXTERNAL) return -EOPNOTSUPP; /* just try to ACK and hope for the best... */ dev_info(service->ep->dcp->dev, "DPTXPort: acking unhandled call %u\n", @@ -846,7 +883,7 @@ static void dptxport_init(struct apple_epic_service *service, const char *name, complete(&port->enable_completion); /* Publication may follow the startup wait or the bounded retry window. */ if (service->ep->dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + DCP_IOMFB_METHODS_H17P_EXTERNAL && READ_ONCE(service->ep->dcp->typec_cable_connected) && !READ_ONCE(service->ep->dcp->quiescing) && !READ_ONCE(service->ep->dcp->external_error)) @@ -938,11 +975,17 @@ static int dptxport_group_call(struct apple_epic_service *service, u16 group, return 0; } +static void dptxport_qualified_teardown(struct apple_epic_service *service) +{ + dcp_180hz_invalidate(service->ep->dcp); + dptx_service_unbind(service); +} + static const struct apple_epic_service_ops dptxep_ops[] = { { .name = "AppleDCPDPTXRemotePort", .prepare = dptxport_prepare, - .teardown = dptx_service_unbind, + .teardown = dptxport_qualified_teardown, .init = dptxport_init, .call = dptxport_call, .call_v2 = dptxport_group_call, @@ -975,7 +1018,7 @@ int dptx_controller_init(struct apple_dcp *dcp) return PTR_ERR(ep); /* Controller interfaces require the same OPEN handshake as the port. */ ep->wire_v2 = dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + DCP_IOMFB_METHODS_H17P_EXTERNAL; dcp->dptx_controller_ep = ep; ret = afk_start(ep); @@ -992,7 +1035,7 @@ int dptxep_init(struct apple_dcp *dcp) { struct apple_dcp_afkep *ep; bool wire_v2 = dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + DCP_IOMFB_METHODS_H17P_EXTERNAL; int ret; u32 port; unsigned long timeout = msecs_to_jiffies(1000); @@ -1025,5 +1068,5 @@ int dptxep_init(struct apple_dcp *dcp) timeout = ret; } - return 0; + return dcp_180hz_init(dcp); } diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index b3858de153c8c..2fb18d1eb512c 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -361,7 +361,7 @@ bool dcpavserv_edid_current(struct apple_dcp *dcp, u64 generation) { lockdep_assert_held(&dcp->hpd_mutex); return dcp->hw.iomfb_method_profile != - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL || + DCP_IOMFB_METHODS_H17P_EXTERNAL || (dcp->typec_cable_connected && generation == dcp->typec_generation && !READ_ONCE(dcp->quiescing)); } @@ -439,13 +439,13 @@ int dpavservep_init(struct apple_dcp *dcp) init_completion(&dcp->dcpavserv.enable_completion); ep = afk_init(dcp, dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL ? + DCP_IOMFB_METHODS_H17P_EXTERNAL ? DPAVSERV_ENDPOINT_V2 : DPAVSERV_ENDPOINT, dpavservep_ops); if (IS_ERR(ep)) return PTR_ERR(ep); /* External H17P uses AFKV2 service calls with host-owned OOB buffers. */ ep->wire_v2 = dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + DCP_IOMFB_METHODS_H17P_EXTERNAL; ep->inline_v2 = false; dcp->dcpavservep = ep; diff --git a/drivers/gpu/drm/apple/ibootep.c b/drivers/gpu/drm/apple/ibootep.c index 79d5dc19569c3..3cada415c249a 100644 --- a/drivers/gpu/drm/apple/ibootep.c +++ b/drivers/gpu/drm/apple/ibootep.c @@ -32,7 +32,7 @@ int ibootep_init(struct apple_dcp *dcp) { const struct apple_epic_service_ops *ops = ibootep_ops; - if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) ops = external_ibootep_ops; dcp->ibootep = afk_init(dcp, DISP0_ENDPOINT, ops); if (IS_ERR(dcp->ibootep)) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index b779323d73228..6a3b1cc76209f 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -195,7 +195,7 @@ static int iomfb_enqueue_opaque_x(struct apple_dcp *dcp) lockdep_assert_held(&dcp->iomfb.lock); /* DCPEXT rejects enableGPAlphaDiv; retain the panel-only workaround. */ if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + DCP_IOMFB_METHODS_H17P_EXTERNAL) return 0; if (atomic_read(&dcp->iomfb.opaque_x_state) != IOMFB_OPAQUE_X_NEEDED) return 0; @@ -866,7 +866,7 @@ static bool dcp_external_zero_reply(struct apple_dcp *dcp, int tag, return tag == 134 && context == DCP_CONTEXT_ASYNC && in_len == 0 && out_len == sizeof(u32) && dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + DCP_IOMFB_METHODS_H17P_EXTERNAL && of_machine_is_compatible("apple,j700") && of_device_is_compatible(dcp->dev->of_node, "apple,t8140-dcpext"); } @@ -920,7 +920,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0], hdr->in_len, hdr->out_len, context); if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && !zero_reply) { + DCP_IOMFB_METHODS_H17P_EXTERNAL && !zero_reply) { /* Retain the failed session without inventing a successful reply. */ dcp->crashed = true; complete(&dcp->start_done); @@ -1046,6 +1046,55 @@ static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) dcpep_handle_cb(dcp, ctx_id, data, length, offset); } +/* Retained arrays are not freed while mode readers can still refer to them. */ +static struct dcp_display_mode *dcp_modes_snapshot(struct apple_dcp *dcp, + unsigned int *count) +{ + struct dcp_display_mode *modes; + unsigned long flags; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + modes = dcp->modes; + *count = dcp->nr_modes; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + return modes; +} + +/* Caller holds hpd_mutex -> dcpavserv.lock -> metadata.lock. */ +static bool dcp_exception_mode_current_locked(struct apple_dcp *dcp, + const struct dcp_display_mode *mode) +{ + if (!mode->automatic_rgb8_quirk) + return true; + return dcp_180hz_mode_current(dcp, mode); +} + +static bool dcp_publish_mode(struct apple_dcp *dcp, struct drm_connector *connector, + const struct dcp_display_mode *source, + struct drm_display_mode *copy) +{ + unsigned long flags; + bool eligible; + + /* Ordinary modes keep their existing publication policy. */ + if (!source->automatic_rgb8_quirk) { + drm_mode_probed_add(connector, copy); + return true; + } + /* External get_modes holds hpd_mutex after its final EDID generation check. */ + /* Allocation happened outside metadata.lock; recheck before linking. */ + spin_lock_irqsave(&dcp->metadata.lock, flags); + eligible = dcp->connector && &dcp->connector->base == connector && + dcp_exception_mode_current_locked(dcp, source); + if (eligible) + /* drm_mode_probed_add only links a list under mode_config.mutex. */ + drm_mode_probed_add(connector, copy); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (!eligible) + drm_mode_destroy(connector->dev, copy); + return eligible; +} + int dcp_get_modes(struct drm_connector *connector) { struct apple_connector *apple_connector = to_apple_connector(connector); @@ -1054,12 +1103,15 @@ int dcp_get_modes(struct drm_connector *connector) struct drm_device *dev = connector->dev; struct drm_display_mode *mode; + struct dcp_display_mode *modes; + unsigned int nr_modes, published = 0; u16 min_vfreq = 0, max_vfreq = 0; bool vrr_capable = false; bool physical_edid; + bool pending_180hz = false, published_180hz = false; u64 edid_generation = 0; u64 service_epoch = 0; - int i; + unsigned int i; dcp_consume_edid_invalidation(apple_connector); @@ -1072,31 +1124,10 @@ int dcp_get_modes(struct drm_connector *connector) return 0; } dcp = platform_get_drvdata(pdev); - - for (i = 0; i < dcp->nr_modes; ++i) { - if (dcp->modes[i].vrr) { - u16 lo = dcp->modes[i].min_vrr >> 16; - u16 hi = dcp->modes[i].max_vrr >> 16; - - if (!min_vfreq || lo < min_vfreq) - min_vfreq = lo; - if (hi > max_vfreq) - max_vfreq = hi; - } - vrr_capable |= dcp->modes[i].vrr; - mode = drm_mode_duplicate(dev, &dcp->modes[i].mode); - - if (!mode) { - dev_err(dev->dev, "Failed to duplicate display mode\n"); - return 0; - } - - drm_mode_probed_add(connector, mode); - } - drm_connector_set_vrr_capable_property(connector, vrr_capable); + modes = dcp_modes_snapshot(dcp, &nr_modes); physical_edid = dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + DCP_IOMFB_METHODS_H17P_EXTERNAL; /* get_modes holds mode_config.mutex; HPD paths never acquire it. */ if (physical_edid) { mutex_lock(&dcp->hpd_mutex); @@ -1117,7 +1148,7 @@ int dcp_get_modes(struct drm_connector *connector) } /* Internal H17P still has no supported EDID command transport. */ - if (dcp->nr_modes && !apple_connector->drm_edid && + if (nr_modes && !apple_connector->drm_edid && (dcp->fw_compat != DCP_FIRMWARE_H17P || (dcp->dcpavservep && dcp->dcpavservep->wire_v2))) { const struct drm_edid *edid; @@ -1162,12 +1193,18 @@ int dcp_get_modes(struct drm_connector *connector) apple_connector->drm_edid_service_epoch = service_epoch; } } - if (dcp->nr_modes && apple_connector->drm_edid) { - apple_connector->drm_edid_firmware_derived = false; - drm_edid_connector_update(connector, apple_connector->drm_edid); + if (nr_modes && apple_connector->drm_edid) { + int update = drm_edid_connector_update(connector, apple_connector->drm_edid); + + if (update) { + dev_info(dcp->dev, "raw EDID publication failed: %d\n", update); + dcp_clear_connector_edid(apple_connector); + } else { + apple_connector->drm_edid_firmware_derived = false; + } } else { u64 generation = 0; - const struct drm_edid *derived = dcp->nr_modes ? + const struct drm_edid *derived = nr_modes ? dcp_firmware_edid(dcp, &generation) : ERR_PTR(-ENODATA); if (!IS_ERR(derived)) { @@ -1193,9 +1230,38 @@ int dcp_get_modes(struct drm_connector *connector) } edid_done: + /* Publish after a possibly delayed raw read, under its physical generation. */ + for (i = 0; i < nr_modes; ++i) { + mode = drm_mode_duplicate(dev, &modes[i].mode); + if (!mode) { + dev_err(dev->dev, "Failed to duplicate display mode\n"); + published = 0; + goto modes_done; + } + pending_180hz |= modes[i].automatic_rgb8_quirk && + dcp_180hz_pending(dcp, &modes[i]); + if (!dcp_publish_mode(dcp, connector, &modes[i], mode)) + continue; + published_180hz |= modes[i].automatic_rgb8_quirk; + published++; + if (modes[i].vrr) { + u16 lo = modes[i].min_vrr >> 16; + u16 hi = modes[i].max_vrr >> 16; + + if (!min_vfreq || lo < min_vfreq) + min_vfreq = lo; + if (hi > max_vfreq) + max_vfreq = hi; + } + vrr_capable |= modes[i].vrr; + } + drm_connector_set_vrr_capable_property(connector, vrr_capable); +modes_done: mutex_unlock(&dcp->dcpavserv.lock); if (physical_edid) mutex_unlock(&dcp->hpd_mutex); + if (pending_180hz && !published_180hz) + dcp_180hz_retry(dcp); if (apple_connector->drm_edid) dcp_retry_placeholder_edid(dcp, apple_connector->drm_edid); @@ -1211,20 +1277,35 @@ int dcp_get_modes(struct drm_connector *connector) connector->display_info.monitor_range.max_vfreq = max_vfreq; } - return dcp->nr_modes; + return published; } /* The user may own drm_display_mode, so we need to search for our copy */ struct dcp_display_mode *lookup_mode(struct apple_dcp *dcp, const struct drm_display_mode *mode) { - int i; + struct dcp_display_mode *modes; + unsigned int count, i; + unsigned long flags; + bool eligible; - for (i = 0; i < dcp->nr_modes; ++i) { - if (drm_mode_match(mode, &dcp->modes[i].mode, + modes = dcp_modes_snapshot(dcp, &count); + for (i = 0; i < count; ++i) { + if (drm_mode_match(mode, &modes[i].mode, DRM_MODE_MATCH_TIMINGS | - DRM_MODE_MATCH_CLOCK)) - return &dcp->modes[i]; + DRM_MODE_MATCH_CLOCK)) { + if (!modes[i].automatic_rgb8_quirk) + return &modes[i]; + mutex_lock(&dcp->hpd_mutex); + mutex_lock(&dcp->dcpavserv.lock); + spin_lock_irqsave(&dcp->metadata.lock, flags); + eligible = dcp_exception_mode_current_locked(dcp, &modes[i]); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + mutex_unlock(&dcp->dcpavserv.lock); + mutex_unlock(&dcp->hpd_mutex); + if (eligible) + return &modes[i]; + } } return NULL; @@ -1247,7 +1328,7 @@ static bool dcp_mode_settable(struct apple_dcp *dcp, if (dcp->fw_compat == DCP_FIRMWARE_H17P && dcp->hw.iomfb_method_profile != - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + DCP_IOMFB_METHODS_H17P_EXTERNAL) return dcp_mode->mode.type & DRM_MODE_TYPE_PREFERRED; return true; @@ -1279,7 +1360,7 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, /* Disconnect cannot pass the admission check before these RPCs finish. */ guard(mutex)(&dcp->external_lock); - if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && (!dcp->external_running || dcp->external_error || READ_ONCE(dcp->quiescing) || READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.presentation_paused))) return -EPIPE; @@ -1781,7 +1862,7 @@ int iomfb_start_rtkit(struct apple_dcp *dcp) * display DART faults. */ if (dcp->fw_compat == DCP_FIRMWARE_H17P && - dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL) { ret = apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); if (ret) return ret; diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index a86675d485a17..869d6635deccb 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -447,7 +447,7 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, } else if (!of_property_present(node, property->dt_name) && of_machine_is_compatible("apple,j700") && (of_device_is_compatible(node, "apple,t8140-dcp") || - (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && of_device_is_compatible(node, "apple,t8140-dcpext")))) { /* Qualified older loaders lack the newly described table properties. */ memset(reply, 0, sizeof(*reply)); @@ -697,26 +697,10 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [599] = trampoline_nop, /* find_swap_function_gated */ }; -/* - * Validate a callback record and, the first time each runtime callback whose - * length is only bounded arrives with a different length, report the length - * actually seen. Those reports are the first capture of these records. - */ bool iomfb_check_callback_h17p(struct apple_dcp *dcp, int tag, u32 in_len, u32 out_len) { - const struct dcp_callback_size *size; - - if (!iomfb_validate_callback_h17p(tag, in_len, out_len)) - return false; - size = &callback_sizes[tag]; - if (size->minimum && - (in_len != size->in_len || out_len != size->out_len) && - !test_and_set_bit(tag, dcp->sized_callbacks)) - dev_info(dcp->dev, - "callback D%03d record is %#x/%#x bytes; its handler uses %#x/%#x\n", - tag, in_len, out_len, size->in_len, size->out_len); - return true; + return iomfb_validate_callback_h17p(tag, in_len, out_len); } static_assert(sizeof(struct dcp_set_power_state_resp) == 8); @@ -739,10 +723,6 @@ static void iomfb_external_power_complete(struct apple_dcp *dcp, void *out, void header->out_len == sizeof(struct dcp_set_power_state_resp)) { u64 reply = get_unaligned_le64(out); - WRITE_ONCE(dcp->external_power_reply, reply); - WRITE_ONCE(dcp->external_power_reply_valid, true); - dev_info(dcp->dev, "A473 power=%u reply=%#llx\n", - cookie->on, reply); ret = reply == (u64)cookie->on ? 0 : -EREMOTEIO; } } @@ -799,7 +779,6 @@ int iomfb_external_power_h17p(struct apple_dcp *dcp, bool on) if (!cookie) return -ENOMEM; cookie->on = on; - WRITE_ONCE(dcp->external_power_reply_valid, false); cookie->wait.status = -ETIMEDOUT; init_completion(&cookie->wait.done); kref_init(&cookie->wait.refcount); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index b9a6eaacdb74f..40f1733280aa6 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -933,7 +934,30 @@ static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) #endif dcp->chunks.length = *length; - dcp_firmware_metadata_begin(dcp); + dcp->chunks.av_service_epoch = 0; + dcp->chunks.automatic_service_pending = false; + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { + /* Match the receipt's start, not a replacement service at its end. */ + mutex_lock(&dcp->dcpavserv.lock); + dcp->chunks.av_service_epoch = dcp->dcpavserv.epoch; + dcp->chunks.automatic_service_pending = + !dcp->dcpavserv.enabled && !dcp->dcpavserv.service; + dcp_firmware_metadata_begin(dcp); + { + unsigned long flags; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + dcp->chunks.automatic_connection_sequence = dcp->rate_proof.connection_sequence; + dcp->chunks.mode_ticket = dcp_metadata_begin(&dcp->metadata.epoch); + dcp->chunks.mode_ticket.associated = + dcp->metadata.epoch.generation == READ_ONCE(dcp->typec_generation) && + READ_ONCE(dcp->typec_cable_connected); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + } + mutex_unlock(&dcp->dcpavserv.lock); + } else { + dcp_firmware_metadata_begin(dcp); + } dcp->chunks.data = kzalloc(*length, GFP_KERNEL); if (!dcp->chunks.data) { @@ -979,7 +1003,7 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, } /* Validate serialized extent outside the metadata spinlock. */ - if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) dcp->chunks.payload_complete = dcp->chunks.contiguous && dcp_property_complete(dcp->chunks.data, dcp->chunks.received, dcp->chunks.length); @@ -991,27 +1015,71 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, if (ret) { dev_warn(dcp->dev, "bad header on dcpav props\n"); + /* A failed replacement cannot keep an unsafe catalog admitted. */ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && + !strcmp(req->key, "TimingElements")) { + unsigned long flags; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + dcp->rate_proof.catalog_sequence++; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + dcp_180hz_catalog_changed(dcp); + } return false; } if (!strcmp(req->key, "TimingElements")) { - dcp->modes = enumerate_modes(&ctx, &dcp->nr_modes, + struct dcp_display_mode *modes; + struct dcp_metadata_ticket rate_ticket = {}; + u64 rate_service_epoch = 0; + unsigned int nr_modes = 0, i; + unsigned long flags; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && + dcp->chunks.payload_complete && dcp->chunks.mode_ticket.associated) { + ctx.automatic_1080p180_rgb8 = true; + rate_ticket = dcp->chunks.mode_ticket; + rate_service_epoch = dcp->chunks.av_service_epoch; + } + modes = enumerate_modes(&ctx, &nr_modes, dcp->width_mm, dcp->height_mm, dcp->notch_height, dcp->fixed_connector_type == DRM_MODE_CONNECTOR_eDP, dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL); + DCP_IOMFB_METHODS_H17P_EXTERNAL); - if (IS_ERR(dcp->modes)) { + if (IS_ERR(modes)) { dev_warn(dcp->dev, "failed to parse modes\n"); - dcp->modes = NULL; - dcp->nr_modes = 0; + modes = NULL; + nr_modes = 0; + } else { + for (i = 0; i < nr_modes; i++) + if (modes[i].automatic_rgb8_quirk) { + modes[i].unsafe_rgb8_ticket = rate_ticket; + modes[i].unsafe_rgb8_service_epoch = rate_service_epoch; + modes[i].automatic_connection_sequence = dcp->chunks.automatic_connection_sequence; + modes[i].automatic_service_pending = dcp->chunks.automatic_service_pending; + } + } + /* Publish one catalog revision with its immutable receipt tickets. */ + spin_lock_irqsave(&dcp->metadata.lock, flags); + dcp->rate_proof.catalog_sequence++; + for (i = 0; i < nr_modes; i++) + if (modes[i].automatic_rgb8_quirk) + modes[i].automatic_catalog_sequence = dcp->rate_proof.catalog_sequence; + dcp->modes = modes; + dcp->nr_modes = nr_modes; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + /* Notify removals as well as newly eligible modes, outside the lock. */ + dcp_180hz_catalog_changed(dcp); + if (!modes) { dcp_firmware_metadata_timing(dcp); return false; } - if (dcp->nr_modes == 0) + if (nr_modes == 0) dev_warn(dcp->dev, "TimingElements without valid modes!\n"); + dcp_180hz_notify(dcp); dcp_firmware_metadata_timing(dcp); } else if (!strcmp(req->key, "DisplayAttributes")) { struct dcp_parse_ctx metadata_ctx = ctx; @@ -1025,7 +1093,7 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, } dcp_set_dimensions(dcp); - if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { struct dcp_display_metadata metadata; ret = parse_display_metadata(&metadata_ctx, &metadata); @@ -2130,11 +2198,16 @@ void iomfb_present_backlight_h17p(struct apple_dcp *dcp) } #endif +static void dcp_post_swap_a428(struct apple_dcp *dcp, void *out, void *cookie) +{ + do_swap(dcp, out, cookie); +} + static void dcp_pre_swap_a428(struct apple_dcp *dcp, void *out, void *cookie) { u32 identity = 0x10000; - dcp_pipe_set_428(dcp, false, &identity, do_swap, cookie); + dcp_pipe_set_428(dcp, false, &identity, dcp_post_swap_a428, cookie); } #endif @@ -2235,12 +2308,12 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, if (DCP_INHERIT_BOOT_MODE && dcp->hw.iomfb_method_profile != - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + DCP_IOMFB_METHODS_H17P_EXTERNAL) { dev_dbg(dcp->dev, "inheriting the bootloader's display mode\n"); complete_set_digital_out_mode(dcp, NULL, cookie); } else if (mode->vrr && dcp->hw.iomfb_method_profile != - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + DCP_IOMFB_METHODS_H17P_EXTERNAL) dcp_set_adaptive_sync(dcp, crtc_state->vrr_enabled ? mode->min_vrr : 0, cookie); @@ -2361,7 +2434,7 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru * Keep YUV, other source spaces and other profiles unchanged. */ if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + DCP_IOMFB_METHODS_H17P_EXTERNAL && !new_state->fb->format->is_yuv && req->surf[l].base.colorspace == DCP_COLORSPACE_NATIVE) req->surf[l].base.colorspace = DCP_COLORSPACE_BG_SRGB; diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 77edffc6b294c..73e143bf0f3a5 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -317,6 +317,8 @@ bool dcp_property_complete(const void *blob, size_t received, size_t advertised) static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim, bool polarity) { + static const char * const fields[] = { "Active", "Total", "FrontPorch", "SyncWidth", "PreciseSyncRate" }; + unsigned int seen = 0; struct iterator it; int ret = 0; @@ -336,6 +338,14 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim, for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); + unsigned int field; + + if (!IS_ERR(key) && handle->automatic_1080p180_rgb8) + for (field = 0; field < 5; field++) + if (!strcmp(key, fields[field])) { + handle->ambiguous_180hz |= !!(seen & (1U << field)); + seen |= 1U << field; + } if (IS_ERR(key)) ret = PTR_ERR(key); @@ -526,10 +536,11 @@ static int parse_timing_color_restrictions(struct dcp_parse_ctx *handle, static int parse_color_modes(struct dcp_parse_ctx *handle, struct dcp_display_mode *out, bool external_sdr, - const struct timing_color_restrictions *limits) + const struct timing_color_restrictions *limits, + struct dcp_color_mode *unsafe_rgb8) { struct iterator outer_it; - unsigned int best_sdr_rank = 0; + unsigned int best_sdr_rank = 0, best_unsafe_rank = 0; int ret = 0; out->sdr_444.score = -1; out->sdr_rgb.score = -1; @@ -542,6 +553,8 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, for (; outer_it.idx < outer_it.len; ++outer_it.idx) { struct iterator it; + static const char * const fields[] = { "Colorimetry", "Depth", "DynamicRange", "EOTF", "ID", "PixelEncoding", "Score", "IsVirtual" }; + unsigned int seen = 0; bool is_virtual = true; struct color_mode cmode = { .colorimetry = -1, @@ -559,6 +572,14 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); + unsigned int field; + + if (!IS_ERR(key) && handle->automatic_1080p180_rgb8) + for (field = 0; field < 8; field++) + if (!strcmp(key, fields[field])) { + handle->ambiguous_180hz |= !!(seen & (1U << field)); + seen |= 1U << field; + } if (IS_ERR(key)) ret = PTR_ERR(key); @@ -601,10 +622,25 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, unsigned int rank; if (cmode.id <= U32_MAX && - (timing_color_contains(&limits->unsafe, cmode.id) || - timing_color_contains(&limits->dsc_required, cmode.id))) + timing_color_contains(&limits->dsc_required, cmode.id)) continue; rank = external_sdr_rank(&cmode); + if (cmode.id <= U32_MAX && + timing_color_contains(&limits->unsafe, cmode.id)) { + /* + * Collect only a timing-local bounded RGB8 candidate. + * Admission still requires current EDID and link proof. + */ + if (unsafe_rgb8 && cmode.depth == 8 && rank && + cmode.colorimetry == DCP_COLORIMETRY_SRGB && + rank >= best_unsafe_rank) { + if (rank > best_unsafe_rank) + unsafe_rgb8->score = -1; + fill_color_mode(unsafe_rgb8, &cmode); + best_unsafe_rank = rank; + } + continue; + } if (!rank || rank < best_sdr_rank) continue; if (rank > best_sdr_rank) @@ -647,10 +683,17 @@ static int parse_mode(struct dcp_parse_ctx *handle, struct dcp_display_mode *out, s64 *score, int width_mm, int height_mm, unsigned notch_height, bool internal, bool external_sdr) { + static const char * const fields[] = { + "ID", "Score", "MinimumVariableRefreshRate", "MaximumVariableRefreshRate", + }; + unsigned int seen = 0; int ret = 0; struct iterator it; struct dimension horiz = { .active = -1 }, vert = { .active = -1 }; struct timing_color_restrictions limits = {}; + struct dcp_color_mode unsafe_rgb8 = { .score = -1 }; + bool have_horiz = false, have_vert = false, have_virtual = false; + bool duplicate_scope = false; s64 min_vrr = 0, max_vrr = 0; u64 clock; s64 id = -1; @@ -668,6 +711,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, .best.score = -1, }; *score = -1; + handle->ambiguous_180hz = false; if (external_sdr) { struct dcp_parse_ctx scan = *handle; @@ -683,26 +727,42 @@ static int parse_mode(struct dcp_parse_ctx *handle, for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); + unsigned int field; + + if (!IS_ERR(key) && handle->automatic_1080p180_rgb8) + for (field = 0; field < ARRAY_SIZE(fields); field++) + if (!strcmp(key, fields[field])) { + handle->ambiguous_180hz |= !!(seen & (1U << field)); + seen |= 1U << field; + } if (IS_ERR(key)) ret = PTR_ERR(key); else if (is_virtual) ret = skip(it.handle); - else if (!strcmp(key, "HorizontalAttributes")) + else if (!strcmp(key, "HorizontalAttributes")) { + duplicate_scope |= have_horiz; + have_horiz = true; ret = parse_dimension(it.handle, &horiz, external_sdr); - else if (!strcmp(key, "VerticalAttributes")) + } else if (!strcmp(key, "VerticalAttributes")) { + duplicate_scope |= have_vert; + have_vert = true; ret = parse_dimension(it.handle, &vert, external_sdr); - else if (!strcmp(key, "MinimumVariableRefreshRate")) + } else if (!strcmp(key, "MinimumVariableRefreshRate")) ret = parse_int_bound(it.handle, &min_vrr, 0, U32_MAX); else if (!strcmp(key, "MaximumVariableRefreshRate")) ret = parse_int_bound(it.handle, &max_vrr, 0, U32_MAX); else if (!strcmp(key, "ColorModes")) - ret = parse_color_modes(it.handle, out, external_sdr, &limits); + ret = parse_color_modes(it.handle, out, external_sdr, &limits, + handle->automatic_1080p180_rgb8 ? + &unsafe_rgb8 : NULL); else if (!strcmp(key, "ID")) ret = parse_int_bound(it.handle, &id, 0, U32_MAX); - else if (!strcmp(key, "IsVirtual")) + else if (!strcmp(key, "IsVirtual")) { + duplicate_scope |= have_virtual; + have_virtual = true; ret = parse_bool(it.handle, &is_virtual); - else if (external_sdr && !strcmp(key, "IsPreferred")) { + } else if (external_sdr && !strcmp(key, "IsPreferred")) { if (have_preferred) ret = -EINVAL; else @@ -729,6 +789,27 @@ static int parse_mode(struct dcp_parse_ctx *handle, return ret; } } + /* + * Never replace a normal candidate or create a timing/color pairing. + * Only the bounded automatic policy may publish or use this candidate. + * Its EDID identity and successful link callbacks must remain current. + * The observed 2080x1106 total and 11796583/65536 Hz are deliberate: + * do not widen this to arbitrary nominal-180 Hz blanking or rates. + */ + if (external_sdr && !internal && + handle->automatic_1080p180_rgb8 && + !handle->ambiguous_180hz && + out->sdr_rgb.score < 0 && unsafe_rgb8.score >= 0 && + !duplicate_scope && have_horiz && have_vert && have_virtual && !is_virtual && + have_interlaced && !interlaced && !notch_height && + horiz.active == 1920 && vert.active == 1080 && + horiz.total == 2080 && vert.total == 1106 && + vert.precise_sync_rate == 11796583 && + horiz.pixel_repetition == 0 && vert.pixel_repetition == 0 && + horiz.sync_polarity >= 0 && vert.sync_polarity >= 0) { + out->sdr_rgb = unsafe_rgb8; + out->automatic_rgb8_quirk = handle->automatic_1080p180_rgb8; + } if (is_preferred) *score |= (1L << 32); @@ -772,8 +853,12 @@ static int parse_mode(struct dcp_parse_ctx *handle, out->vrr = true; } - /* Refresh rates are reported by DCP as 16.16 fixed-point Hz. */ - if (min_vrr && max_vrr > min_vrr) { + /* + * Refresh rates are reported by DCP as 16.16 fixed-point Hz. + * External H17P skips adaptive-sync programming: keep the fixed timing + * available, but do not advertise its sink VRR range as driver support. + */ + if (!external_sdr && min_vrr && max_vrr > min_vrr) { out->min_vrr = min_vrr; out->max_vrr = max_vrr; out->vrr = true; @@ -809,7 +894,8 @@ static int parse_mode(struct dcp_parse_ctx *handle, if (external_sdr && horiz.sync_polarity >= 0 && vert.sync_polarity >= 0) mode->flags = (horiz.sync_polarity ? DRM_MODE_FLAG_PHSYNC : DRM_MODE_FLAG_NHSYNC) | (vert.sync_polarity ? DRM_MODE_FLAG_PVSYNC : DRM_MODE_FLAG_NVSYNC); - out->metadata_timing_valid = external_sdr && have_preferred && preferred && + out->metadata_timing_valid = external_sdr && !out->automatic_rgb8_quirk && + have_preferred && preferred && have_interlaced && !interlaced && horiz.pixel_repetition == 0 && vert.pixel_repetition == 0 && horiz.sync_polarity >= 0 && vert.sync_polarity >= 0; drm_mode_set_name(mode); @@ -824,11 +910,110 @@ static int parse_mode(struct dcp_parse_ctx *handle, return 0; } +/* + * Inspect command identities before filtering can discard conflicting siblings. + * Only automatic exceptions use this conservative receipt-wide check. + */ +static bool dcp_180hz_color_ids_current(struct dcp_parse_ctx *handle, u32 id) +{ + static const char * const fields[] = { + "ID", "Depth", "DynamicRange", "EOTF", "PixelEncoding", "Colorimetry", + }; + struct iterator colors; + + if (iterator_begin(handle, &colors, false)) + return false; + for (; colors.idx < colors.len; colors.idx++) { + struct color_mode color = { .id = -1, .depth = -1, .dynamic_range = -1, + .eotf = -1, .pixel_encoding = -1, .colorimetry = -1 }; + s64 *values[] = { &color.id, &color.depth, &color.dynamic_range, + &color.eotf, &color.pixel_encoding, &color.colorimetry }; + bool is_virtual = true, virtual_seen = false; + unsigned int seen = 0; + struct iterator entry; + + if (iterator_begin(handle, &entry, true)) + return false; + for (; entry.idx < entry.len; entry.idx++) { + char *key = parse_string(handle); + unsigned int field; + int ret; + + if (IS_ERR(key)) + return false; + for (field = 0; field < ARRAY_SIZE(fields); field++) + if (!strcmp(key, fields[field])) + break; + if (field < ARRAY_SIZE(fields)) { + ret = seen & (1U << field) ? -EINVAL : + parse_int_bound(handle, values[field], 0, U32_MAX); + seen |= 1U << field; + } else if (!strcmp(key, "IsVirtual")) { + ret = virtual_seen ? -EINVAL : parse_bool(handle, &is_virtual); + virtual_seen = true; + } else { + ret = skip(handle); + } + kfree(key); + if (ret) + return false; + } + if (color.id == id && + (is_virtual || !virtual_seen || color.depth != 8 || + color.dynamic_range != DCP_COLOR_YCBCR_RANGE_FULL || + color.eotf != DCP_EOTF_SDR_GAMMA || + color.pixel_encoding != DCP_COLOR_FORMAT_RGB || + color.colorimetry != DCP_COLORIMETRY_SRGB)) + return false; + } + return true; +} + +static bool dcp_180hz_ids_current(struct dcp_parse_ctx handle, + const struct dcp_display_mode *mode) +{ + struct iterator timings; + unsigned int matches = 0; + + if (iterator_begin(&handle, &timings, false)) + return false; + for (; timings.idx < timings.len; timings.idx++) { + struct iterator entry; + + if (iterator_begin(&handle, &entry, true)) + return false; + for (; entry.idx < entry.len; entry.idx++) { + char *key = parse_string(&handle); + int ret = 0; + + if (IS_ERR(key)) + return false; + if (!strcmp(key, "ID")) { + s64 id; + + ret = parse_int_bound(&handle, &id, 0, U32_MAX); + if (!ret && id == mode->timing_mode_id) + matches++; + } else if (!strcmp(key, "ColorModes")) { + if (!dcp_180hz_color_ids_current(&handle, mode->color_mode_id)) + ret = -EINVAL; + } else { + ret = skip(&handle); + } + kfree(key); + if (ret || matches > 1) + return false; + } + } + return matches == 1; +} + struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, int height_mm, unsigned notch_height, bool internal, bool external_sdr) { + struct dcp_parse_ctx catalog = *handle; struct iterator it; int ret; struct dcp_display_mode *mode, *modes; @@ -869,7 +1054,7 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, /* Process a successful mode */ (*count)++; - if (score > best_score) { + if (!mode->automatic_rgb8_quirk && score > best_score) { best_score = score; best_mode = mode; } @@ -878,6 +1063,17 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, if (best_mode != NULL) best_mode->mode.type |= DRM_MODE_TYPE_PREFERRED; + if (handle->automatic_1080p180_rgb8) { + unsigned int i, kept = 0; + + for (i = 0; i < *count; i++) { + if (modes[i].automatic_rgb8_quirk && + !dcp_180hz_ids_current(catalog, &modes[i])) + continue; + modes[kept++] = modes[i]; + } + *count = kept; + } return modes; } @@ -1091,7 +1287,7 @@ static int parse_metadata_product(struct dcp_parse_ctx *handle, struct dcp_displ for (unsigned int i = 0; i < 3; i++) if (out->manufacturer[i] < 'A' || out->manufacturer[i] > 'Z') return -EINVAL; - if ((seen & 32) && legacy != ((out->manufacturer[0] - '@') << 10 | + if ((seen & 32) && legacy != (u32)((out->manufacturer[0] - '@') << 10 | (out->manufacturer[1] - '@') << 5 | (out->manufacturer[2] - '@'))) return -EINVAL; diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 0392b09751ac9..3b21263154754 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -8,6 +8,7 @@ #include #include "dcp-display-metadata.h" +#include "dcp-metadata-epoch.h" struct apple_dcp; @@ -15,6 +16,9 @@ struct dcp_parse_ctx { struct apple_dcp *dcp; const void *blob; u32 pos, len; + /* External H17P candidates require separate publication/lookup proof. */ + bool automatic_1080p180_rgb8; + bool ambiguous_180hz; }; enum dcp_color_eotf { @@ -97,6 +101,13 @@ struct dcp_display_mode { struct dcp_color_mode best; bool vrr; bool metadata_timing_valid; + bool automatic_rgb8_quirk; + u64 automatic_connection_sequence; + u64 automatic_catalog_sequence; + bool automatic_service_pending; + /* Immutable receipt provenance; never retag a cached exception. */ + struct dcp_metadata_ticket unsafe_rgb8_ticket; + u64 unsafe_rgb8_service_epoch; }; struct dimension { diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-ring.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-ring.py index 65d6fdeca7a54..b55eaea38c8d7 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-ring.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-ring.py @@ -24,7 +24,7 @@ static int check_ring(struct ep *ep,u32 rptr,u32 wptr,size_t payload_len,size_t *total) { struct epic_hdr *ehdr;struct epic_sub_hdr *eshdr;struct afk_qe *hdr; - bool compact=false;size_t total_epic_size,total_size;int ret=0; + size_t total_epic_size,total_size;int ret=0; ''' suffix = r''' *total=total_size; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile new file mode 100644 index 0000000000000..0d6d406ef5b9d --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +TEST_PROGS := run.sh +TEST_FILES := run.py common.py edid.py rate.py phy.py lifecycle.py pm.py idle.py dwc3 README +include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/README b/tools/testing/selftests/drivers/gpu/apple-dcp/README new file mode 100644 index 0000000000000..a61543d1168f8 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/README @@ -0,0 +1,56 @@ +Apple external display host regressions +====================================== + +Run from a kernel source checkout: + + make -C tools/testing/selftests TARGETS=drivers/gpu/apple-dcp run_tests + +Or keep translation units, commands, failure logs and source hashes: + + tools/testing/selftests/drivers/gpu/apple-dcp/run.sh --output /tmp/apple-dcp-results + +An installed copy needs --source /path/to/kernel. --group edid, rate, phy or +lifecycle selects one group. HOSTCC selects the host compiler (default clang). +Python 3 and a compiler supporting AddressSanitizer/UndefinedBehaviorSanitizer +are required. No target access, firmware command, captured fixture or kernel +configuration is required. + +One TAP entry point uses shared production extraction and compiler handling. +The modules contain public synthetic host boundaries and retained assertions: + +* edid: actual AFK admission and OPEN/CLOSE handling; native/HDMI providers + alternating over 64 epochs; foreign owners, stale replies, reader/command + pins, ordered receive progress and legacy service behavior. +* rate: actual binary parser, EDID/capacity policy, receipt ingress, getters + and lookup; all arrival orders, conflicting/discarded siblings, rejected + replacement catalogs, HPD retirement and recovery; malformed/duplicate + metadata and independent EDID decoding; eight bounded delayed retries. +* phy: actual parking/mux/init/reset/configure callbacks, masked PMA context, + faulted ACK/BIST cleanup, dirty-route retry, OFF/new-calibration epochs, + warm reset and legacy T8103/T8122/T8140/T8142 route controls. +* lifecycle: actual physical-disconnect authority and ownership despite failed + teardown; DWC3/core/PHY PM and reset-reference recovery; exact firmware + tuple/resource admission, unsupported VRR rejection and power reply bounds; + 15,552 D134 dispatcher scope/length/context and legacy reply cases. + +Transport, MMIO, kernel locks and work scheduling are host mocks. These tests +establish source behavior; electrical readiness, firmware/FDT ownership, +concurrency and system suspend require separate kernel/hardware qualification. +Kernel KUnit tests and apple-afkv2 remain separate tests in their existing +standard layouts. No archived diagnostic script is a dependency of this suite. + +The runner preserves host artifacts on failure as well as success. Existing +physical-disconnect fixture warnings for unused mock boundaries and signed +host/kernel ABI comparisons are handled only by those fixtures' original +compiler options; production source compilation uses the normal Kbuild policy. + +The bounded-PM group shares the PHY fixture and executes the actual PM get/put +and mux wrapper, usb_hc_died and typec_mux_set. The probe group executes real +CD321x register/unregister/rollback and retry-state functions. USB n/y/m uses +the actual optional dependency and a real Kconfig producer (KCONFIG_CONF may +select an existing producer). Legacy lane/teardown groups keep executing the +same locked mux body; they do not pretend their old USB mocks own runtime PM. +The cpuidle group executes the actual Apple table/probe, generic registration +and s2idle selector/entry, and tick_freeze/unfreeze. It checks source behavior +only; RTC, elapsed suspend time and simultaneous hardware execution remain +separate qualification gates. The original human cpuidle patch is preserved. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py new file mode 100644 index 0000000000000..1a61b02dac07e --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright 2026 Ryan Murray +"""Shared public-source extraction and strict host compilation.""" +from pathlib import Path +import hashlib, json, os, re, shlex, subprocess + +def block(source, marker): + start = source.index(marker) if isinstance(marker, str) else marker + opening = source.index('{', start) + end, depth = opening + 1, 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + +def source_defines(source): + lines = iter(source.splitlines()) + definitions = [] + for line in lines: + if not line.startswith('#define '): + continue + definition = [line] + while definition[-1].endswith(chr(92)): + definition.append(next(lines)) + definitions.append('\n'.join(definition)) + return definitions + +def run_unit(name, unit, output, cases=((),), includes=(), extra=()): + work = output / name + work.mkdir(parents=True, exist_ok=True) + source = work / 'host.c' + source.write_text(unit) + command = shlex.split(os.environ.get('HOSTCC', 'clang')) + [ + '-std=gnu11', '-Wall', '-Werror', '-O1', '-g', + '-fsanitize=address,undefined', '-fno-omit-frame-pointer', + ] + list(extra) + ['-I' + str(p) for p in includes] + [str(source), '-o', str(work / 'host')] + (work / 'command.json').write_text(json.dumps(command, indent=2) + '\n') + result = subprocess.run(command, capture_output=True, text=True, timeout=120) + (work / 'compile.log').write_text(result.stdout + result.stderr) + result.check_returncode() + for index, case in enumerate(cases): + result = subprocess.run([str(work / 'host'), *case], capture_output=True, text=True, timeout=120) + (work / ('case-' + str(index) + '.log')).write_text(result.stdout + result.stderr) + result.check_returncode() + return hashlib.sha256(unit.encode()).hexdigest() diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/cases.c b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/cases.c new file mode 100644 index 0000000000000..3e92ba747494f --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/cases.c @@ -0,0 +1,511 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +static struct atcphy_hw hardware; +static int counted_off(struct phy *p) +{ + callback_calls++; + return atcphy_usb3_power_off(p); +} +static int other_off(struct phy *p) +{ + (void)p; + return other_off_error; +} +static const struct phy_ops usb3_ops = { .power_off = counted_off }; +static const struct phy_ops usb2_ops = { .init = atcphy_usb2_init, + .power_off = other_off }; +static const struct phy_ops plain_ops = { .power_off = other_off }; +static void dwc3_host_exit(struct dwc3 *d) +{ + host_exits++; + if (root_refs) { + assert(phy_power_off(d->usb3_generic_phy[0]) == 0); + assert(phy_exit(d->usb3_generic_phy[0]) == 0); + root_refs = 0; + } +} +/* Boundary mocks acquire references through the production generic PHY API. */ +static int dwc3_core_init(struct dwc3 *d) +{ + starts++; + if (core_init_error) + return core_init_error; + for (int i = 0; i < d->num_usb2_ports; i++) + assert(phy_init(d->usb2_generic_phy[i]) == 0); + for (int i = 0; i < d->num_usb3_ports; i++) + assert(phy_init(d->usb3_generic_phy[i]) == 0); + return dwc3_phy_power_on(d); +} +static int dwc3_apple_core_probe(struct dwc3_apple *a) +{ + int ret = dwc3_core_init(&a->dwc); + if (!ret) + a->state = DWC3_APPLE_NO_CABLE; + return ret; +} +static void dwc3_apple_set_role(struct dwc3_apple *a, enum dwc3_apple_state s) +{ + assert(atcphy_usb3_set_mode(a->usb3_phy, + s == DWC3_APPLE_HOST ? PHY_MODE_USB_HOST : + PHY_MODE_USB_DEVICE, + 0) == 0); +} +struct fixture { + struct apple_atcphy atc; + struct phy p2, p3, extra; + struct dwc3_apple apple; +}; +static void initial(struct fixture *f, unsigned int gen, bool fixed, bool pm) +{ + memset(f, 0, sizeof(*f)); + memset(core_regs, 0, sizeof(core_regs)); + memset(pipe_regs, 0, sizeof(pipe_regs)); + memset(usb2_regs, 0, sizeof(usb2_regs)); + hardware = (struct atcphy_hw){ + .gen = gen == 0 ? ATCPHY_GENERATION_T8103 : + ATCPHY_GENERATION_T8122, + .has_t8140_dp = gen == 2, + .has_usb4 = gen != 2, + .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, + .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, + }; + f->atc = (struct apple_atcphy){ + .hw = &hardware, + .regs.core = (unsigned char *)core_regs, + .regs.lpdptx = (unsigned char *)core_regs, + .regs.pipehandler = (unsigned char *)pipe_regs, + .regs.usb2phy = (unsigned char *)usb2_regs, + .mode = APPLE_ATCPHY_MODE_USB3, + .typec_mode = APPLE_ATCPHY_MODE_USB3, + .pipe_state = ATCPHY_PIPEHANDLER_STATE_USB3, + .host_active = true, + .fixed_usb2 = fixed, + .ss_tunables = true, + .dp_tunables = true, + }; + f->p3 = (struct phy){ .data = &f->atc, + .ops = &usb3_ops, + .power_count = 1, + .init_count = 1, + .pm_enabled = pm, + .pm_refs = pm ? 1 : 0 }; + f->p2 = (struct phy){ .data = &f->atc, + .ops = &usb2_ops, + .power_count = 1, + .init_count = 1, + .pm_enabled = pm, + .pm_refs = pm ? 1 : 0 }; + f->extra = (struct phy){ .ops = &plain_ops, + .power_count = 1, + .init_count = 1, + .pm_enabled = pm, + .pm_refs = pm ? 1 : 0 }; + f->apple = (struct dwc3_apple){ + .state = DWC3_APPLE_HOST, + .reset = &f->atc.rcdev, + .usb2_phy = &f->p2, + .usb3_phy = &f->p3, + .core_active = true, + .reset_pending = true, + .usb3_power_off_pending = 1, + .usb2_power_off_pending = 1, + .dwc = { .num_usb3_ports = 1, + .num_usb2_ports = 1, + .usb3_generic_phy = { &f->p3 }, + .usb2_generic_phy = { &f->p2 } }, + }; + core_regs[ATCPHY_POWER_CTRL / 4] = + ATCPHY_POWER_PHY_RESET_N | ATCPHY_POWER_SLEEP_BIG | + ATCPHY_POWER_SLEEP_SMALL | ATCPHY_POWER_APB_RESET_N; + core_regs[ATCPHY_POWER_STAT / 4] = core_regs[ATCPHY_POWER_CTRL / 4]; + core_regs[AUS_COMMON_DIG_RCAL1 / 4] = + AUS_COMMON_DIG_RCAL1_ALL_CODES_DONE; + core_regs[ACIOPHY_TOP_PHY_STAT / 4] = ACIOPHY_TOP_PHY_STAT_LN0_UNK0; + pipe_regs[PIPEHANDLER_AON_GEN / 4] = PIPEHANDLER_AON_GEN_DWC3_RESET_N; + polls = fail_poll = reset_calls = reset_writes = core_writes = + warnings = 0; + generic_warnings = 0; + host_exits = clocks_disabled = callback_calls = gadget_exits = + event_cleanups = 0; + legacy_suspends = core_mode_exits = free_buffers = starts = deasserts = + setup_cio = root_refs = 0; + host_init_error = gadget_init_error = core_init_error = reset_error = + other_off_error = 0; + init_cleanup_fault = false; + syscore = true; + stuck_ack = false; +} +static void recover(void) +{ + fail_poll = 0; + stuck_ack = false; + pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = 0; +} +static int exit_apple(struct fixture *f) +{ + mutex_lock(&f->apple.lock); + int ret = dwc3_apple_exit(&f->apple); + mutex_unlock(&f->apple.lock); + return ret; +} +static int role(struct fixture *f, enum usb_role r) +{ + struct usb_role_switch sw = { &f->apple }; + return dwc3_usb_role_switch_set(&sw, r); +} +static void clean(struct fixture *f) +{ + assert(!f->p3.power_count && !f->p2.power_count && !f->p3.init_count && + !f->p2.init_count); + assert(!f->p3.pm_refs && !f->p2.pm_refs && !f->apple.core_active && + !f->apple.reset_pending); + assert(!f->apple.usb3_power_off_pending && + !f->apple.usb2_power_off_pending); + assert(!f->atc.host_active && !f->atc.mode_dirty && + f->atc.mode == APPLE_ATCPHY_MODE_OFF); + assert(!syscore); + assert(!f->atc.lock && !f->p3.mutex && !f->p2.mutex && !f->apple.lock); + assert(!(pipe_regs[PIPEHANDLER_AON_GEN / 4] & + PIPEHANDLER_AON_GEN_DWC3_RESET_N)); +} +static void cycles(void) +{ + unsigned int cases = 0; + for (unsigned int gen = 0; gen < 3; gen++) + for (unsigned int fixed = 0; fixed < 2; fixed++) + for (unsigned int pm = 0; pm < 2; pm++) + for (unsigned int fault = 0; fault < 4; + fault++) { + struct fixture f; + initial(&f, gen, fixed, pm); + if (fault == 1) { + stuck_ack = true; + pipe_regs[PIPEHANDLER_LOCK_ACK / + 4] = + PIPEHANDLER_LOCK_EN; + } else if (fault > 1) { + f.atc.pipe_state = + ATCPHY_PIPEHANDLER_STATE_DUMMY; + fail_poll = fault - 1; + } + int ret = exit_apple(&f); + printf("CYCLE gen=%u fixed=%u pm=%u fault=%u exit=%d refs=%d init=%d\n", + gen, fixed, pm, fault, ret, + f.p3.power_count, + f.p3.init_count); + fflush(stdout); + if (fault) { + assert(ret == -ETIMEDOUT && + f.p3.power_count == 1 && + f.p3.init_count == 1); + assert(f.apple.core_active && + clocks_disabled == 0 && + event_cleanups == 1 && + host_exits == 1); + assert(syscore); + if (fault > 1) + assert(f.atc.mode_dirty); + if (fault == 1) { + assert(role(&f, + USB_ROLE_HOST) == + -ETIMEDOUT && + starts == 0 && + deasserts == 0); + assert(host_exits == + 1 && + event_cleanups == + 1 && + f.p3.power_count == + 1); + } + recover(); + assert(exit_apple(&f) == 0); + } else + assert(ret == 0); + clean(&f); + assert(host_exits == 1 && + clocks_disabled == 1 && + event_cleanups == 1); + assert(exit_apple(&f) == 0 && + host_exits == 1 && + clocks_disabled == 1); + assert(role(&f, USB_ROLE_HOST) == 0 && + f.p3.power_count == 1 && + f.p3.init_count == 1); + assert(f.atc.mode == + APPLE_ATCPHY_MODE_USB3 && + !f.atc.mode_dirty); + assert(core_regs[ATCPHY_POWER_CTRL / 4] & + ATCPHY_POWER_PHY_RESET_N); + assert(role(&f, USB_ROLE_NONE) == 0); + clean(&f); + assert(!generic_warnings); + if (!fault) + assert(!warnings); + cases++; + } + printf("PASS: %u normal/parking/domain shutdown and restart scenarios\n", + cases); +} +static void pending_reset(void) +{ + for (unsigned int gen = 0; gen < 3; gen++) { + struct fixture f; + initial(&f, gen, false, true); + reset_error = -ETIMEDOUT; + assert(exit_apple(&f) == -ETIMEDOUT); + assert(!f.apple.core_active && f.apple.reset_pending && + !f.p3.power_count && !f.p3.init_count); + assert(syscore); + assert(role(&f, USB_ROLE_HOST) == -ETIMEDOUT && !starts && + !deasserts); + assert(host_exits == 1 && clocks_disabled == 1 && + event_cleanups == 1 && callback_calls == 1); + reset_error = 0; + assert(exit_apple(&f) == 0); + clean(&f); + assert(host_exits == 1 && clocks_disabled == 1 && + callback_calls == 1); + assert(reset_calls == 3); + } + puts("PASS: pending reset retry does not repeat host, PHY or core teardown"); +} +static void partial_release(void) +{ + struct fixture f; + initial(&f, 1, false, true); + other_off_error = -EIO; + assert(exit_apple(&f) == -EIO); + assert(!f.p3.power_count && f.p3.init_count == 1 && + f.p2.power_count == 1); + assert(!f.apple.usb3_power_off_pending && + f.apple.usb2_power_off_pending == 1); + assert(callback_calls == 1 && !clocks_disabled); + other_off_error = 0; + assert(exit_apple(&f) == 0); + clean(&f); + assert(callback_calls == 1 && event_cleanups == 1 && + clocks_disabled == 1); + initial(&f, 1, false, false); + f.apple.dwc.num_usb3_ports = 2; + f.apple.dwc.usb3_generic_phy[1] = &f.extra; + f.apple.usb3_power_off_pending = 3; + other_off_error = -EIO; + assert(exit_apple(&f) == -EIO && !f.p3.power_count && + f.extra.power_count == 1); + other_off_error = 0; + assert(exit_apple(&f) == 0 && !f.extra.power_count && + !f.extra.init_count); + clean(&f); + assert(callback_calls == 1); + puts("PASS: successful USB3 references are not released twice after another PHY fails"); +} +static void init_failures(void) +{ + for (unsigned int device = 0; device < 2; device++) { + struct fixture f; + initial(&f, 2, false, true); + assert(exit_apple(&f) == 0); + if (device) + gadget_init_error = -EIO; + else + host_init_error = -EIO; + reset_error = -ETIMEDOUT; + assert(role(&f, device ? USB_ROLE_DEVICE : USB_ROLE_HOST) == + -EIO); + assert(f.apple.reset_pending && !f.apple.core_active && + !f.p3.power_count); + assert(syscore); + host_init_error = gadget_init_error = reset_error = 0; + assert(role(&f, device ? USB_ROLE_DEVICE : USB_ROLE_HOST) == 0); + assert(f.p3.power_count == 1 && f.p3.init_count == 1); + assert(role(&f, USB_ROLE_NONE) == 0); + clean(&f); + } + struct fixture f; + initial(&f, 1, false, false); + assert(exit_apple(&f) == 0); + core_init_error = -EIO; + reset_error = -ETIMEDOUT; + assert(role(&f, USB_ROLE_HOST) == -EIO && f.apple.reset_pending && + !f.apple.core_active); + core_init_error = reset_error = 0; + assert(role(&f, USB_ROLE_HOST) == 0); + assert(role(&f, USB_ROLE_NONE) == 0); + clean(&f); + initial(&f, 1, false, true); + assert(exit_apple(&f) == 0); + host_init_error = -EIO; + init_cleanup_fault = true; + assert(role(&f, USB_ROLE_HOST) == -EIO && f.apple.core_active); + assert(syscore); + assert(f.p3.power_count == 1 && f.p3.init_count == 1); + recover(); + host_init_error = 0; + init_cleanup_fault = false; + assert(role(&f, USB_ROLE_HOST) == 0 && f.p3.power_count == 1 && + f.p3.init_count == 1); + assert(role(&f, USB_ROLE_NONE) == 0); + clean(&f); + puts("PASS: host/gadget/core startup failure reset obligations drain before retry"); +} +static void pending_restart(void) +{ + unsigned int cases = 0; + for (unsigned int gen = 0; gen < 3; gen++) + for (unsigned int fixed = 0; fixed < 2; fixed++) + for (unsigned int pm = 0; pm < 2; pm++) + for (unsigned int fault = 1; fault < 4; + fault++) { + struct fixture f; + initial(&f, gen, fixed, pm); + if (fault == 1) { + stuck_ack = true; + pipe_regs[PIPEHANDLER_LOCK_ACK / + 4] = + PIPEHANDLER_LOCK_EN; + } else { + f.atc.pipe_state = + ATCPHY_PIPEHANDLER_STATE_DUMMY; + fail_poll = fault - 1; + } + assert(exit_apple(&f) == -ETIMEDOUT); + recover(); + assert(role(&f, USB_ROLE_HOST) == 0 && + f.p3.power_count == 1 && + f.p3.init_count == 1); + assert(f.atc.mode == + APPLE_ATCPHY_MODE_USB3 && + !f.atc.mode_dirty); + assert(core_regs[ATCPHY_POWER_CTRL / 4] & + ATCPHY_POWER_PHY_RESET_N); + assert(!generic_warnings); + assert(role(&f, USB_ROLE_NONE) == 0); + clean(&f); + cases++; + } + struct fixture f; + initial(&f, 2, true, true); + f.apple.fixed_hub = true; + assert(role(&f, USB_ROLE_NONE) == 0 && + f.apple.state == DWC3_APPLE_HOST && f.p3.power_count == 1); + assert(exit_apple(&f) == 0); + mutex_lock(&f.apple.lock); + assert(dwc3_apple_init(&f.apple, DWC3_APPLE_HOST) == 0); + mutex_unlock(&f.apple.lock); + assert(f.p3.power_count == 1 && exit_apple(&f) == 0); + clean(&f); + printf("PASS: %u pending-failure to HOST transitions and actual fixed-hub role no-op\n", + cases); +} +static void negative_power(void) +{ + struct fixture f; + initial(&f, 0, false, false); + f.atc.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + fail_poll = 1; + assert(exit_apple(&f) == -ETIMEDOUT && f.p3.power_count == 1 && + f.p3.init_count == 1); + recover(); + assert(role(&f, USB_ROLE_HOST) == 0 && f.p3.power_count == 1); +} +static void negative_reset(void) +{ + struct fixture f; + initial(&f, 0, false, false); + reset_error = -ETIMEDOUT; + assert(exit_apple(&f) == -ETIMEDOUT && reset_calls == 1); + reset_error = 0; + assert(exit_apple(&f) == 0 && reset_calls == 2); + assert(!(pipe_regs[PIPEHANDLER_AON_GEN / 4] & + PIPEHANDLER_AON_GEN_DWC3_RESET_N)); +} +static void negative_legacy(void) +{ + struct fixture f; + initial(&f, 0, false, false); + f.atc.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + fail_poll = 1; + assert(phy_power_off(&f.p3) == -ETIMEDOUT && f.atc.mode_dirty); + recover(); + struct typec_mux_dev mux = { &f.atc }; + struct typec_mux_state state = { .mode = TYPEC_STATE_USB }; + assert(atcphy_mux_set(&mux, &state) == 0); + assert(core_regs[ATCPHY_POWER_CTRL / 4] & ATCPHY_POWER_PHY_RESET_N); +} +static void legacy_restore(void) +{ + for (unsigned int gen = 0; gen < 3; gen++) { + struct fixture f; + initial(&f, gen, false, false); + f.atc.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + fail_poll = 1; + assert(phy_power_off(&f.p3) == -ETIMEDOUT && f.atc.mode_dirty); + unsigned int before = core_writes; + assert(atcphy_usb3_set_mode(&f.p3, PHY_MODE_USB_HOST, 0) == + -EBUSY && + core_writes == before); + if (gen < 2) { + f.atc.dp_link_rate = ATCPHY_DP_LINK_RATE_RBR; + assert(atcphy_dp_configure(&f.atc, + ATCPHY_DP_LINK_RATE_RBR) == + -EBUSY && + core_writes == before); + } + recover(); + struct typec_mux_dev mux = { &f.atc }; + struct typec_mux_state state = { .mode = TYPEC_STATE_USB }; + assert(atcphy_mux_set(&mux, &state) == 0 && !f.atc.mode_dirty && + core_writes > before); + assert(core_regs[ATCPHY_POWER_CTRL / 4] & + ATCPHY_POWER_PHY_RESET_N); + assert(phy_power_off(&f.p3) == 0 && !f.p3.power_count); + } + puts("PASS: M1/M2/J700 aborted shutdown blocks PIPE and restores saved lane mode"); +} +static void removal_and_generic(void) +{ + struct fixture f; + initial(&f, 0, false, false); + root_refs = 1; + assert(phy_init(&f.p3) == 0 && phy_power_on(&f.p3) == 0); + assert(exit_apple(&f) == 0); + clean(&f); + unsigned int callbacks = callback_calls, clocks = clocks_disabled, + events = event_cleanups; + dwc3_core_remove_no_shutdown(&f.apple.dwc); + assert(callbacks == callback_calls && clocks == clocks_disabled && + events == event_cleanups); + assert(!core_mode_exits && free_buffers == 1 && !f.p3.power_count && + !f.p3.init_count); + initial(&f, 1, false, false); + dwc3_core_exit(&f.apple.dwc); + assert(!f.p3.power_count && !f.p3.init_count && !f.p2.power_count && + !f.p2.init_count); + assert(event_cleanups == 1 && clocks_disabled == 1 && + callback_calls == 1); + puts("PASS: root-hub references, already-exited removal and ordinary generic core exit"); +} +int main(int argc, char **argv) +{ + if (argc == 2 && !strcmp(argv[1], "power")) { + negative_power(); + return 0; + } + if (argc == 2 && !strcmp(argv[1], "reset")) { + negative_reset(); + return 0; + } + if (argc == 2 && !strcmp(argv[1], "legacy")) { + negative_legacy(); + return 0; + } + cycles(); + pending_restart(); + pending_reset(); + partial_release(); + init_failures(); + legacy_restore(); + removal_and_generic(); + puts("PASS: Apple teardown actual-function qualification"); + return 0; +} diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c new file mode 100644 index 0000000000000..777e12dc36135 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c @@ -0,0 +1,307 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +struct apple_atcphy { + const struct atcphy_hw *hw; + struct { + unsigned char *core, *lpdptx, *pipehandler, *usb2phy; + } regs; + struct reset_controller_dev rcdev; + void *dev; + int lock, dp_link_rate; + bool swap_lanes, tunnel_clock_on, tunnel_saved; + bool fixed_usb2, host_active, mode_dirty, ss_tunables, dp_tunables, + aux_power_requested; + unsigned int dp_owned_pmas, dp_idle_pmas; + unsigned int dp_usb_context_valid, dp_usb_context_dirty; + u32 dp_usb_context[2][27]; + bool pipe_pending, usb_off_pending, usb_off_drained; + enum atcphy_mode mode, typec_mode; + enum atcphy_pipehandler_state pipe_state; +}; +struct phy; +struct phy_ops { + int (*init)(struct phy *); + int (*exit)(struct phy *); + int (*power_on)(struct phy *); + int (*power_off)(struct phy *); +}; +struct phy { + struct apple_atcphy *data; + const struct phy_ops *ops; + int mutex, power_count, init_count, pm_refs, pwr, dev; + bool pm_enabled; +}; +#define phy_get_drvdata(p) ((p)->data) +static int phy_pm_runtime_get_sync(struct phy *p) +{ + if (!p->pm_enabled) + return -ENOTSUPP; + p->pm_refs++; + return 0; +} +static void phy_pm_runtime_put(struct phy *p) +{ + if (p->pm_enabled) + p->pm_refs--; +} +static void phy_pm_runtime_put_sync(struct phy *p) +{ + phy_pm_runtime_put(p); +} +static int regulator_enable(int p) +{ + assert(!p); + return 0; +} +static void regulator_disable(int p) +{ + assert(!p); +} +struct dwc3 { + struct phy *usb2_generic_phy[3], *usb3_generic_phy[3]; + int num_usb2_ports, num_usb3_ports, usb2_phy, usb3_phy; + struct reset_controller_dev *reset; + int dev, usb_psy, vbus_draw_work; +}; +struct dwc3_apple { + struct dwc3 dwc; + enum dwc3_apple_state state; + struct reset_controller_dev *reset; + int lock; + void *dev; + struct phy *usb2_phy, *usb3_phy; + bool fixed_hub, core_active, core_exit_pending, reset_pending; + unsigned long usb2_power_off_pending, usb3_power_off_pending; +}; +static unsigned int polls, fail_poll, reset_calls, reset_writes, core_writes; +static unsigned int host_exits, clocks_disabled, callback_calls; +static unsigned int gadget_exits, event_cleanups, legacy_suspends, + core_mode_exits, free_buffers; +static unsigned int starts, deasserts, setup_cio, root_refs; +static int host_init_error, gadget_init_error, core_init_error, reset_error, + other_off_error; +static bool init_cleanup_fault; +static bool syscore; +static bool stuck_ack; +static u32 core_regs[0x30000 / 4], pipe_regs[0x1000 / 4], usb2_regs[0x1000 / 4]; +static u32 readl(void *p) +{ + return *(u32 *)p; +} +static void mask32(void *p, u32 m, u32 v) +{ + *(u32 *)p = (*(u32 *)p & ~m) | v; + if (p == &pipe_regs[PIPEHANDLER_LOCK_REQ / 4] && !stuck_ack) + pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = *(u32 *)p & + PIPEHANDLER_LOCK_EN; + if (p == &pipe_regs[PIPEHANDLER_AON_GEN / 4]) + reset_writes++; + if ((uintptr_t)p >= (uintptr_t)core_regs && + (uintptr_t)p < (uintptr_t)core_regs + sizeof(core_regs)) + core_writes++; +} +static void writel(u32 v, void *p) +{ + mask32(p, ~0U, v); +} +static void set32(void *p, u32 v) +{ + mask32(p, 0, v); +} +static void clear32(void *p, u32 v) +{ + mask32(p, v, 0); +} +static void core_mask32(struct apple_atcphy *p, u32 o, u32 m, u32 v) +{ + assert(o < sizeof(core_regs)); + mask32(p->regs.core + o, m, v); + if (o == ATCPHY_POWER_CTRL) + core_regs[ATCPHY_POWER_STAT / 4] = core_regs[o / 4]; +} +static void core_set32(struct apple_atcphy *p, u32 o, u32 v) +{ + core_mask32(p, o, 0, v); +} +static void core_clear32(struct apple_atcphy *p, u32 o, u32 v) +{ + core_mask32(p, o, v, 0); +} +#define readl_poll_timeout(a, v, c, i, t) \ + ({ \ + (v) = readl(a); \ + ++polls; \ + polls == fail_poll || !(c) ? -ETIMEDOUT : 0; \ + }) +#define readl_poll_timeout_atomic readl_poll_timeout +/* These dependencies are outside the cross-layer test; no DP ownership is seeded. */ +static int j700_dp_stop(struct apple_atcphy *p) +{ + assert(!p->dp_owned_pmas); + return 0; +} +static int j700_dp_idle_pending(struct apple_atcphy *p) +{ + (void)p; + return 0; +} +static void atc_tunnel_stop_t8103(struct apple_atcphy *p) +{ + (void)p; +} +static void atc_tunnel_restore(struct apple_atcphy *p) +{ + (void)p; +} +/* USB2 power-on is outside this teardown/recovery control. */ +static void atcphy_usb2_power_on(struct apple_atcphy *p) +{ + (void)p; +} +static void atcphy_apply_tunables(struct apple_atcphy *p, enum atcphy_mode m) +{ + (void)p; + (void)m; +} +static int atcphy_configure_lanes(struct apple_atcphy *p, enum atcphy_mode m) +{ + (void)p; + (void)m; + return 0; +} +static int atcphy_enable_dp_aux(struct apple_atcphy *p) +{ + (void)p; + return 0; +} +static int atcphy_auspll_apb_command(struct apple_atcphy *p, int command) +{ + (void)p; + (void)command; + return 0; +} +static int +atcphy_dp_configure_lane(struct apple_atcphy *p, int lane, + const struct atcphy_dp_link_rate_configuration *config) +{ + (void)p; + (void)lane; + (void)config; + return 0; +} +static void usb_phy_set_suspend(int p, int s) +{ + (void)p; + (void)s; + legacy_suspends++; +} +static void usb_phy_shutdown(int p) +{ + (void)p; +} +static void dwc3_event_buffers_cleanup(struct dwc3 *d) +{ + (void)d; + event_cleanups++; +} +static void dwc3_clk_disable(struct dwc3 *d) +{ + (void)d; + clocks_disabled++; +} +static void dwc3_host_exit(struct dwc3 *d); +static void dwc3_gadget_exit(struct dwc3 *d) +{ + (void)d; + gadget_exits++; +} +static void dev_pm_syscore_device(void *d, bool s) +{ + (void)d; + syscore = s; +} +static void dwc3_enable_susphy(struct dwc3 *d, bool s) +{ + (void)d; + (void)s; +} +static int atcphy_dwc3_reset_assert(struct reset_controller_dev *, + unsigned long); +static int reset_control_assert(struct reset_controller_dev *r) +{ + if (!r) + return 0; + reset_calls++; + if (reset_error) + return reset_error; + return atcphy_dwc3_reset_assert(r, 0); +} +static int atcphy_dwc3_reset_deassert(struct reset_controller_dev *, + unsigned long); +static int reset_control_deassert(struct reset_controller_dev *r) +{ + deasserts++; + return atcphy_dwc3_reset_deassert(r, 0); +} +static int phy_set_mode(struct phy *p, int m) +{ + (void)p; + (void)m; + return 0; +} +static int dwc3_apple_core_probe(struct dwc3_apple *a); +static int dwc3_core_init(struct dwc3 *d); +static void dwc3_apple_setup_cio(struct dwc3_apple *a) +{ + (void)a; + setup_cio++; +} +static int dwc3_apple_usb2_mode_before_reset(enum dwc3_apple_state s, + enum phy_mode *m) +{ + if (s != DWC3_APPLE_HOST && s != DWC3_APPLE_DEVICE) + return -EINVAL; + *m = s == DWC3_APPLE_HOST ? PHY_MODE_INVALID : PHY_MODE_USB_DEVICE; + return 0; +} +static void dwc3_apple_set_role(struct dwc3_apple *a, enum dwc3_apple_state s); +static int dwc3_host_init(struct dwc3 *d) +{ + (void)d; + if (init_cleanup_fault) { + stuck_ack = true; + pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = PIPEHANDLER_LOCK_EN; + } + return host_init_error; +} +static int dwc3_gadget_init(struct dwc3 *d) +{ + (void)d; + return gadget_init_error; +} +#define pm_runtime_get_sync(...) ((void)0) +#define pm_runtime_allow(...) ((void)0) +#define pm_runtime_disable(...) ((void)0) +#define pm_runtime_dont_use_autosuspend(...) ((void)0) +#define pm_runtime_put_noidle(...) ((void)0) +#define pm_runtime_set_suspended(...) ((void)0) +#define dev_set_drvdata(...) ((void)0) +#define cancel_work_sync(...) ((void)0) +#define power_supply_put(...) ((void)0) +static void dwc3_core_exit_mode(struct dwc3 *d) +{ + (void)d; + core_mode_exits++; +} +static void dwc3_debugfs_exit(struct dwc3 *d) +{ + (void)d; +} +static void dwc3_ulpi_exit(struct dwc3 *d) +{ + (void)d; +} +static void dwc3_free_event_buffers(struct dwc3 *d) +{ + (void)d; + free_buffers++; +} diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/prefix.c b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/prefix.c new file mode 100644 index 0000000000000..4e3c13cb25496 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/prefix.c @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint16_t u16; +typedef uint8_t u8; +#define ENOTSUPP 524 +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#define BIT(n) (1U << (n)) +#define GENMASK(h, l) ((~0U << (l)) & (~0U >> (31 - (h)))) +#define FIELD_PREP(m, v) (((u32)(v) << __builtin_ctz(m)) & (m)) +#define FIELD_GET(m, v) (((v) & (m)) >> __builtin_ctz(m)) +#define field_prep FIELD_PREP +#define __iomem +#define udelay(n) ((void)(n)) +#define usleep_range(a, b) ((void)(a), (void)(b)) +#define fallthrough __attribute__((fallthrough)) +static unsigned int warnings, generic_warnings; +static void warning(const char *message) +{ + warnings++; + if (strstr(message, "phy_power_on was called before phy_init")) + generic_warnings++; +} +#define dev_warn(d, message, ...) warning(message) +#define dev_err(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define lockdep_assert_held(p) assert(*(p)) +#define WARN_ON_ONCE(v) assert(!(v)) +static void mutex_lock(int *m) +{ + assert(!*m); + *m = 1; +} +static void mutex_unlock(int *m) +{ + assert(*m); + *m = 0; +} +struct held { + int *m; +}; +static struct held hold(int *m) +{ + mutex_lock(m); + return (struct held){ m }; +} +static void release(struct held *h) +{ + mutex_unlock(h->m); +} +#define guard(kind) \ + struct held held_guard __attribute__((cleanup(release))) = hold +enum phy_mode { PHY_MODE_USB_HOST, PHY_MODE_USB_DEVICE }; +enum { APPLE_ATCPHY_LANE_0, APPLE_ATCPHY_LANE_1 }; +enum usb_role { USB_ROLE_NONE, USB_ROLE_HOST, USB_ROLE_DEVICE }; +struct usb_role_switch { + struct dwc3_apple *data; +}; +#define usb_role_switch_get_drvdata(s) ((s)->data) +#define PHY_MODE_INVALID (-1) +enum { + TYPEC_STATE_SAFE, + TYPEC_STATE_USB, + TYPEC_MODE_USB4, + TYPEC_DP_STATE_C, + TYPEC_DP_STATE_D, + TYPEC_DP_STATE_E +}; +enum { USB_TYPEC_TBT_SID = 0x8087, USB_TYPEC_DP_SID = 0xff01 }; +#define EUDO_USB_MODE_MASK 3U +enum { EUDO_USB_MODE_USB2, EUDO_USB_MODE_USB3, EUDO_USB_MODE_USB4 }; +struct enter_usb_data { + u32 eudo; +}; +struct typec_altmode { + unsigned int svid; +}; +struct typec_mux_state { + unsigned long mode; + struct typec_altmode *alt; + void *data; +}; +struct typec_mux_dev { + struct apple_atcphy *data; +}; +#define typec_mux_get_drvdata(mux) ((mux)->data) +struct reset_controller_dev { + int unused; +}; +#define container_of(p, t, m) ((t *)((char *)(p) - offsetof(t, m))) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py new file mode 100644 index 0000000000000..55a16678b97be --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py @@ -0,0 +1,789 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright 2026 Ryan Murray +"""Actual native EDID admission, service epochs, stale replies and RX progress.""" +from pathlib import Path +from common import block, source_defines + +def rebind(root): + base = root / 'drivers/gpu/drm/apple' + def function(file,signature): + text=(base/file).read_text();start=text.index(signature);opening=text.index('{',start) + while ';' in text[start:opening]: + start=text.index(signature,start+1);opening=text.index('{',start) + end,depth=opening+1,1 + while depth: + depth+=(text[end]=='{')-(text[end]=='}');end+=1 + return text[start:end] + prefix=r''' +#undef NDEBUG +#include +#include +#include +#include +#include +#include "afkv2-codec.h" +typedef int64_t s64; +typedef u64 dma_addr_t; +#define U16_MAX UINT16_MAX +#define EBUSY 16 +#define ENOENT 2 +#define MAX_PENDING_CMDS 16 +#define AFK_MAX_CHANNEL 16 +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define WARN_ON(x) (x) +#define bitmap_empty(p,n) (!*(p)) +#define spin_lock_irqsave(p,f) do { assert(!*(p));*(p)=true;(f)=0; } while(0) +#define spin_unlock_irqrestore(p,f) do { (void)(f);assert(*(p));*(p)=false; } while(0) +static int test_bit(unsigned int bit,const unsigned long *p) { return !!(*p & (1UL<(b)?(a):(b)) +#define strscpy(dst,src,n) snprintf(dst,n,"%s",(char *)(src)) +#define trace_dcpavserv_init(...) ((void)0) +#define dma_rmb() ((void)0) +#define DRM_CONNECTOR_REGISTERED 1 +#define DPAVSERV_ENDPOINT_V2 0x2a +#define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 +#define EPIC_SUBTYPE_STD_SERVICE 0xc0 +#define EPIC_SUBTYPE_ANNOUNCE_H17P 0x11 +#define to_platform_device(x) (x) +#define ERR_PTR(x) ((void *)(intptr_t)(x)) +#define IS_ERR(x) ((uintptr_t)(x)>(uintptr_t)-4096) +#define lockdep_assert_held(p) assert(*(p)) +static bool *hpd_lock,*av_lock; +static void mutex_lock(bool *p) { if(p==hpd_lock) assert(!*av_lock);assert(!*p);*p=true; } +static void mutex_unlock(bool *p) { assert(*p);*p=false; } +static void complete(int *p) { (*p)++; } +static void reinit_completion(int *p) { *p=0; } +static void schedule_work(unsigned int *p) { (*p)++; } +struct apple_dcp;struct apple_dcp_afkep;struct apple_epic_service; +struct apple_epic_service_ops { char name[32];bool reusable;void (*init)(struct apple_epic_service *,const char *,const char *,s64);int (*prepare)(struct apple_epic_service *,const char *,const char *,s64);void (*teardown)(struct apple_epic_service *); }; +struct epic_cmd_info { u16 tag;u8 type;size_t reply_length;void *rxbuf,*txbuf;dma_addr_t rxbuf_dma,txbuf_dma;size_t rxlen,txlen;u32 retcode;bool done,free_on_ack;int *completion; }; +struct apple_epic_service { + struct apple_dcp_afkep *ep;const struct apple_epic_service_ops *ops; + bool lock,enabled,torndown;unsigned int users;u32 channel;u8 cmd_tag; + unsigned long cmd_map[1];struct epic_cmd_info cmds[MAX_PENDING_CMDS]; + struct { u64 generation,epoch; } admission;void *cookie; +}; +struct apple_dcp_afkep { struct apple_dcp *dcp;const struct apple_epic_service_ops *ops;bool wire_v2,inline_v2,match_epic_name;u32 endpoint;unsigned int num_channels;struct apple_epic_service services[AFK_MAX_CHANNEL]; }; +struct apple_connector { void *dcp;bool connected,edid_invalidation_pending;struct { int registration_state; } base;unsigned int hotplug_wq; }; +struct apple_dcp { + void *dev;int fw_compat;struct apple_dcp_afkep *dcpavservep;struct apple_connector *connector; + bool hpd_mutex,quiescing,typec_cable_connected;u64 typec_generation;unsigned int nr_modes; + struct { int iomfb_method_profile; } hw; + struct { bool lock,enabled;u64 epoch;int enable_completion;struct apple_epic_service *service; } dcpavserv; +}; +struct epic_cmd { u32 retcode;u64 rxbuf,txbuf;u32 rxlen,txlen; }; +static unsigned int dma_frees,opens; +static int open_error,open_action; +void dpavservep_detach(struct apple_dcp *dcp); +static void inject_owner(struct apple_dcp_afkep *ep,u32 pending_channel); +static void dma_free_coherent(void *dev,size_t size,void *p,dma_addr_t address) +{ (void)dev;assert(size && (uintptr_t)p==address);free(p);dma_frees++; } +static void afk_remove_service_debugfs(struct apple_epic_service *s) { (void)s; } +static void afk_populate_service_debugfs(struct apple_epic_service *s) { (void)s; } +static int afk_send_epic(struct apple_dcp_afkep *e,u32 channel,u16 tag,int type,int category,u8 subtype,const void *p,size_t n) +{ + assert(channel && !tag && type==EPIC_TYPE_NOTIFY && category==EPIC_CAT_REPORT && subtype==AFKV2_OPEN && !p && !n); + opens++; + assert(!e->dcp->dcpavserv.enabled && !e->dcp->dcpavserv.service); + for(unsigned int i=0;inum_channels;i++) { + struct apple_epic_service *s=&e->services[i]; + if(s->channel==channel && !s->torndown) assert(!s->enabled && !s->users); + } + if(open_action==1) e->dcp->typec_generation++; + if(open_action==2) dpavservep_detach(e->dcp); + if(open_action==4) inject_owner(e,channel); + if(open_action==5) e->dcp->typec_cable_connected=false; + if(open_action==6) e->dcp->quiescing=true; + if(open_action==3) { + for(unsigned int i=0;inum_channels;i++) + if(e->services[i].channel==channel && !e->services[i].torndown) e->services[i].channel++; + } + return open_error; +} +static void afk_v2_callback(struct apple_dcp_afkep *e,const struct afkv2_frame *f,const struct afkv2_command *c) { (void)e;(void)f;(void)c;assert(!"unexpected callback"); } + +struct item { const char *key,*value; s64 integer; }; +struct dcp_parse_ctx { struct item *items;unsigned int count,index;bool key; }; +struct iterator { struct dcp_parse_ctx *handle;unsigned int idx,len; }; +#define kfree(p) free((void *)(p)) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +static struct item announcement_items[3]; +static int parse(u8 *p,size_t n,struct dcp_parse_ctx *ctx) +{ (void)p;assert(n==37);*ctx=(struct dcp_parse_ctx){.items=announcement_items,.count=3};return 0; } +static int iterator_begin(struct dcp_parse_ctx *h,struct iterator *it,bool dict) +{ assert(dict);h->index=0;h->key=true;*it=(struct iterator){.handle=h,.len=h->count};return 0; } +static char *parse_string(struct dcp_parse_ctx *h) +{ + struct item *i=&h->items[h->index];const char *p; + if(h->key) { p=i->key;h->key=false; } else { p=i->value;h->index++;h->key=true; } + if(!p) return ERR_PTR(-EINVAL);char *copy=malloc(strlen(p)+1);assert(copy);strcpy(copy,p);return copy; +} +static int parse_int(struct dcp_parse_ctx *h,s64 *out) +{ assert(!h->key);*out=h->items[h->index++].integer;h->key=true;return 0; } +static int skip(struct dcp_parse_ctx *h) { assert(!h->key);h->index++;h->key=true;return 0; } +''' + parts=[function('parser.c','int parse_epic_service_init('),function('afk.h','static inline struct apple_epic_service *afk_service_get('), + function('afk.h','static inline void afk_service_put('),function('afk.h','static inline void afk_service_retire('),function('afk.h','static inline void afk_service_disable('), + function('afk.h','static inline bool afk_service_matches('),function('afk.h','static inline bool afk_service_reclaimable('), + function('afk.h','static inline bool\nafk_service_reinit('),function('afk.h','static inline void afk_service_discard('), + function('afk.h','static inline void afk_service_publish('), + function('epic/dpavservep.c','static void dcpavserv_notify_edid('),function('epic/dpavservep.c','static bool dcpavserv_identity('), + function('epic/dpavservep.c','static bool dcpavserv_admission_current('),function('epic/dpavservep.c','static int dcpavserv_prepare('), + function('epic/dpavservep.c','static void dcpavserv_init('),function('epic/dpavservep.c','static void dcpavserv_invalidate_edid('), + function('epic/dpavservep.c','static void dcpavserv_teardown('),function('epic/dpavservep.c','void dpavservep_detach('), + function('afk.c','static const struct apple_epic_service_ops *\nafk_match_service('), + function('afk.c','static struct apple_epic_service *afk_epic_find_service('), + function('afk.c','static void afk_recv_handle_init('), + function('epic/dpavservep.c','static int dcpdpserv_prepare('), + function('epic/dpavservep.c','static void dcpdpserv_teardown('),function('afk.c','static void afk_recv_handle_teardown('), + function('afk.c','static void afk_recv_handle_reply('),function('afk.c','static void afk_v2_receive(')] + suffix=r''' +static struct apple_dcp d; +static struct apple_dcp_afkep ep; +static struct apple_connector con; +static const struct apple_epic_service_ops ops={.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.init=dcpavserv_init,.teardown=dcpavserv_teardown}; +static const struct apple_epic_service_ops announcement_ops[]={ + {.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.init=dcpavserv_init,.teardown=dcpavserv_teardown}, + {.name="dcpdp-service-epic",.reusable=true,.prepare=dcpdpserv_prepare,.teardown=dcpdpserv_teardown}, + {0} +}; +static const char *name="dcpav-service-epic",*class="AppleDCPDP2HDMI"; +static const char *native_class="DCPDP13Service"; +static struct apple_epic_service *prepare(u32 channel,s64 unit,const char *n,const char *c) +{ + for(unsigned int i=0;iusers==1); +} +static void repeated_announcements(void) +{ + ep.ops=announcement_ops;ep.match_epic_name=true;ep.num_channels=0;d.fw_compat=DCP_FIRMWARE_H17P; + /* More than the entire slot budget; wrap channel identity as well. */ + for(unsigned int i=0;i<4*AFK_MAX_CHANNEL;i++) { + u16 channel=1+4*(i%4);unsigned int before=opens; + /* Flip, same-attachment revalidation, return flip; swap both providers. */ + bool native=((i%4==1 || i%4==2) != !!((i/4)%2)); + const char *provider=native?native_class:class; + announce(channel,name,provider,channel); + struct apple_epic_service *owner=d.dcpavserv.service; + if(native) assert(owner && "native DP EDID provider must publish"); + assert(owner->ops==&announcement_ops[0]); + assert(owner && owner->channel==channel && owner->users==1 && opens==before+1); + assert(ep.num_channels<=2); /* Real allocator reuses, no slot exhaustion. */ + u64 epoch=d.dcpavserv.epoch; + announce(channel,name,provider,channel); /* Active duplicate: no extra OPEN/pin. */ + assert(opens==before+1 && owner->users==1 && d.dcpavserv.epoch==epoch); + announce(channel+2,"dcpdp-service-epic",provider,channel+2); + assert(opens==before+1 && d.dcpavserv.service==owner && d.dcpavserv.epoch==epoch); + if(i%4==1) { + u64 generation=d.typec_generation; + closed(channel,0x6a01,0); /* Revalidation retires service, not attachment. */ + assert(!d.dcpavserv.service && d.typec_generation==generation && d.dcpavserv.epoch>epoch); + } else { + disconnect();connect(); + } + } + unsigned int missing_ops_before=opens; + assert(afk_match_service(&ep,"missing-service-epic")==NULL); + assert(afk_match_service(&ep,"dcpav-service-epic")==&announcement_ops[0]); + announce(13,"missing-service-epic",native_class,13); + announce(13,"dcpav-service-epic-extra",native_class,13); + assert(opens==missing_ops_before && !d.dcpavserv.service); + assert(ep.num_channels==2); /* Auxiliary rejection leaves a reusable slot. */ + unsigned int before=opens; + announce(13,name,"synthetic-wrong-class",13);assert(opens==before && !d.dcpavserv.service); + char long_class[161];memset(long_class,'x',160);long_class[160]=0; + announce(13,name,long_class,13);assert(opens==before && !d.dcpavserv.service); + for(unsigned int variant=0;variant<2;variant++) { + const char *provider=variant?native_class:class; + assert(!prepare(0,0,name,provider));assert(!prepare(65536,65536,name,provider)); + assert(!prepare(13,-1,name,provider));assert(!prepare(13,0,name,provider)); + assert(!prepare(13,1,name,provider));assert(!prepare(13,13,NULL,provider)); + assert(!prepare(13,13,"dcpdp-service-epic",provider)); + announce(13,"wrong-compartment",provider,13);assert(opens==before && !d.dcpavserv.service); + ep.endpoint=0x2c;announce(13,name,provider,13);assert(opens==before && !d.dcpavserv.service);ep.endpoint=0x2a; + d.quiescing=true;announce(13,name,provider,13);assert(opens==before && !d.dcpavserv.service);d.quiescing=false; + disconnect();announce(13,name,provider,13);assert(opens==before && !d.dcpavserv.service);connect(); + } + for(unsigned int variant=0;variant<4;variant++) { + const char *forged[]={"DCPDP13ServiceExtra","dcpdp13service","AppleDCPDP2HDMIExtra","unrelated-provider"}; + announce(13,name,forged[variant],13);assert(opens==before && !d.dcpavserv.service); + } + assert(!prepare(13,13,name,NULL)); + open_error=-EIO;announce(13,name,native_class,13);assert(opens==++before && !d.dcpavserv.service);open_error=0; + for(unsigned int window=0;window<5;window++) { + const int actions[]={1,2,3,5,6};open_action=actions[window]; + announce(13,name,native_class,13); + assert(opens==++before && !d.dcpavserv.service && !d.dcpavserv.enabled); + d.typec_cable_connected=true;d.quiescing=false; + } + open_action=4;announce(13,name,native_class,13);assert(opens==++before && d.dcpavserv.service==injected_owner); + assert(injected_owner->users==1);dpavservep_detach(&d);assert(!injected_owner->users); + open_action=0;announce(13,name,native_class,13);assert(opens==++before && d.dcpavserv.service); + struct apple_epic_service *old=d.dcpavserv.service; + assert(afk_service_get(old)==old);old->cmd_map[0]=1; + disconnect();connect(); + announce(13,name,native_class,13); /* Same channel still owns a pending reply. */ + assert(opens==before && !d.dcpavserv.service && old->users==1 && old->channel==13); + announce(1,name,class,1);assert(opens==++before && d.dcpavserv.service!=old); + struct apple_epic_service *fresh=d.dcpavserv.service;u64 epoch=d.dcpavserv.epoch; + dcpavserv_teardown(old);assert(d.dcpavserv.service==fresh && d.dcpavserv.epoch==epoch); + old->cmd_map[0]=0;afk_service_put(old);disconnect();connect(); + announce(13,name,native_class,13);assert(opens==++before && d.dcpavserv.service); + struct apple_epic_service *owner=d.dcpavserv.service; + u64 owner_epoch=d.dcpavserv.epoch; + announce(5,name,class,5); /* A second admitted identity cannot displace the owner. */ + assert(opens==before && d.dcpavserv.service==owner && d.dcpavserv.epoch==owner_epoch); + dpavservep_detach(&d); + /* All slots held by retired readers: fail safely, then recover after puts. */ + for(unsigned int i=0;itorndown); + legacy=prepare(13,0,name,native_class);assert(legacy); + d.typec_generation++;afk_service_publish(legacy);dcpavserv_init(legacy,name,native_class,0); + assert(d.dcpavserv.service==legacy && legacy->users==1);dpavservep_detach(&d); + assert(dcpdpserv_prepare(legacy,"dcpdp-service-epic",native_class,0)==0); + ep.wire_v2=true;ep.endpoint=0x2a; + announce(U16_MAX,name,native_class,U16_MAX);assert(d.dcpavserv.service && d.dcpavserv.service->channel==U16_MAX); + dpavservep_detach(&d); + puts("PASS: actual AFKV2 HDMI/native DP alternating over 64 cycles: flip/revalidation/return, strict identity, stale OPEN generation/epoch/owner, auxiliary isolation, reader/command pins and legacy controls"); +} +int main(void) +{ + d.hw.iomfb_method_profile=2;d.typec_cable_connected=true;d.typec_generation=1;d.nr_modes=1;d.connector=&con;d.dev=&d; + con.dcp=&d;con.connected=true;con.base.registration_state=1; + ep.dcp=&d;ep.wire_v2=true;ep.endpoint=0x2a;ep.num_channels=AFK_MAX_CHANNEL;d.dcpavservep=&ep; + hpd_lock=&d.hpd_mutex;av_lock=&d.dcpavserv.lock; + struct item h17p[]={{"compartment","dcpav-service-epic",0},{"name","AppleDCPDP2HDMI",0},{"interface-id",NULL,13}}; + struct dcp_parse_ctx parse_ctx={.items=h17p,.count=3};const char *parsed_name,*parsed_class;s64 parsed_unit=-1; + assert(!parse_epic_service_init(&parse_ctx,&parsed_name,&parsed_class,&parsed_unit,true)); + assert(parsed_unit==13 && !strcmp(parsed_name,name) && !strcmp(parsed_class,class)); + struct apple_epic_service *proof=prepare(13,parsed_unit,parsed_name,parsed_class);assert(proof);afk_service_discard(proof); + free((void *)parsed_name);free((void *)parsed_class); + struct item legacy[]={{"EPICName","dcpav-service-epic",0},{"EPICProviderClass","legacy-class",0},{"EPICUnit",NULL,0}}; + parse_ctx=(struct dcp_parse_ctx){.items=legacy,.count=3}; + assert(!parse_epic_service_init(&parse_ctx,&parsed_name,&parsed_class,&parsed_unit,false));assert(parsed_unit==0); + free((void *)parsed_name);free((void *)parsed_class); + struct apple_epic_service *old=prepare(1,1,name,class);assert(old);publish(old,1); + assert(d.dcpavserv.service==old && old->users==1); + assert(afk_service_get(old)==old); /* Delayed EDID reader pins the retired slot. */ + old->cmds[0]=(struct epic_cmd_info){.tag=0,.type=0xc0,.rxlen=32,.txlen=32,.free_on_ack=true}; + old->cmds[0].rxbuf=calloc(1,32);old->cmds[0].txbuf=calloc(1,32);assert(old->cmds[0].rxbuf && old->cmds[0].txbuf); + old->cmds[0].rxbuf_dma=(uintptr_t)old->cmds[0].rxbuf;old->cmds[0].txbuf_dma=(uintptr_t)old->cmds[0].txbuf;old->cmd_map[0]=1; + disconnect();assert(!d.dcpavserv.service && !d.dcpavserv.enabled && old->users==1 && old->torndown && !old->enabled); + assert(con.edid_invalidation_pending && !afk_service_get(old)); + assert(!afk_service_reinit(old,&ep,&ops,5)); + assert(!prepare(5,5,name,class)); /* No admission while physically disconnected. */ + connect();struct apple_epic_service *fresh=prepare(5,5,name,class);assert(fresh && fresh!=old);publish(fresh,5); + assert(d.dcpavserv.service==fresh && fresh->users==1); + u64 epoch=d.dcpavserv.epoch; + dcpavserv_teardown(old);assert(d.dcpavserv.service==fresh && d.dcpavserv.epoch==epoch); + closed(1,0x6a01,0);assert(d.dcpavserv.service==fresh && d.dcpavserv.epoch==epoch && !dma_frees); + u8 oob[32],reply[56];size_t written; + assert(!afkv2_encode_oob_command(oob,sizeof(oob),0,old->cmds[0].rxbuf_dma,32,old->cmds[0].txbuf_dma,32));afkv2_put32(oob+4,0); + assert(!afkv2_encode_frame(reply,sizeof(reply),1,1,0,0xc0,AFKV2_RESPONSE,0,oob,sizeof(oob),&written)); + afk_v2_receive(&ep,0,0,reply,written); + assert(dma_frees==2 && !old->cmd_map[0] && old->users==1 && d.dcpavserv.service==fresh && d.dcpavserv.epoch==epoch); + assert(!afk_service_reinit(old,&ep,&ops,9));afk_service_put(old); + afk_v2_receive(&ep,0,0,reply,written);assert(dma_frees==2); /* Duplicate late ACK. */ + disconnect();connect();struct apple_epic_service *next=prepare(9,9,name,class);assert(next);publish(next,9); + assert(d.dcpavserv.service==next); + closed(1,0x6a01,0);closed(5,0x6a01,0);assert(d.dcpavserv.service==next); + disconnect();connect();next=prepare(13,13,name,class);assert(next);publish(next,13);assert(d.dcpavserv.service==next); + closed(13,0x6a01,1);assert(d.dcpavserv.service==next); /* Nonzero result rejected. */ + closed(13,0x6a00,0);assert(d.dcpavserv.service==next); /* Wrong low flags rejected. */ + d.dcpavservep=NULL;closed(13,0x6a01,0);assert(d.dcpavserv.service==next);d.dcpavservep=&ep; + closed(13,0x6a01,0);assert(!d.dcpavserv.service && !d.dcpavserv.enabled); + u8 result_byte=0;struct afkv2_frame notification={.category=AFKV2_REPORT,.type=AFKV2_CLOSED,.interface_flags=1,.payload=&result_byte,.payload_size=1}; + assert(afkv2_closed_report(¬ification));notification.token=1;assert(!afkv2_closed_report(¬ification)); + notification.token=0;notification.payload_size=0;assert(!afkv2_closed_report(¬ification)); + assert(!prepare(0,0,name,class));assert(!prepare(65536,65536,name,class)); + assert(!prepare(9,1,name,class));assert(!prepare(9,9,"wrong",class));assert(!prepare(9,9,name,"wrong")); + struct apple_epic_service *pending=prepare(9,9,name,class);assert(pending); + disconnect();connect();publish(pending,9); + assert(!d.dcpavserv.service && pending->torndown && !pending->enabled && !pending->users); + pending=prepare(9,9,name,class);assert(pending); + mutex_lock(&d.hpd_mutex);d.typec_generation++;mutex_unlock(&d.hpd_mutex); + publish(pending,9); /* Generation change alone, without registry retirement. */ + assert(!d.dcpavserv.service && pending->torndown && !pending->enabled); + pending=prepare(13,13,name,class);assert(pending); + dpavservep_detach(&d);publish(pending,13); /* Registry retirement with no physical change. */ + assert(!d.dcpavserv.service && pending->torndown && !pending->enabled); + ep.wire_v2=false;ep.endpoint=0x28;d.hw.iomfb_method_profile=0; + assert(!prepare(7,7,name,class));pending=prepare(7,0,name,NULL);assert(pending);publish(pending,0); + assert(d.dcpavserv.service==pending && pending->users==1);dpavservep_detach(&d); + assert(!pending->users && !d.hpd_mutex && !d.dcpavserv.lock); + memset(&ep.services,0,sizeof(ep.services));ep.wire_v2=true;ep.endpoint=0x2a;d.hw.iomfb_method_profile=2; + repeated_announcements(); + puts("PASS: actual dynamic 1/5/9/13 binding, forged identities, prepare/init epochs, physical retirement, reader pins, real old CLOSED/late ACK isolation and legacy unit 0"); +} +''' + return prefix + '\n'.join(parts) + suffix + +def admission(root): + + base = root / 'drivers/gpu/drm/apple' + + def function(file, signature): + source = (base / file).read_text() + start = source.index(signature) + opening = source.index('{', start) + while ';' in source[start:opening]: + start = source.index(signature, start + 1) + opening = source.index('{', start) + end, depth = opening + 1, 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + av = (base / 'epic/dpavservep.c').read_text() + af = (base / 'afk.c').read_text() + announce = af[af.index('static void afk_recv_handle_init('):af.index('static void afk_recv_handle_init(') + 6500] + assert announce.index('ops->prepare(service,') < announce.index('EPIC_CAT_REPORT, AFKV2_OPEN') + assert '.prepare = dcpavserv_prepare,' in av and '.prepare = dcpdpserv_prepare,' in av + prefix = r''' +#undef NDEBUG +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint64_t u64; +typedef int64_t s64; +#define U16_MAX UINT16_MAX +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define ERR_PTR(x) ((void *)(intptr_t)(x)) +#define PTR_ERR(x) ((int)(intptr_t)(x)) +#define IS_ERR(x) ((uintptr_t)(x) > (uintptr_t)-4096) +#define spin_lock_irqsave(p,f) do { assert(!*(p)); *(p)=true; (f)=0; } while(0) +#define spin_unlock_irqrestore(p,f) do { (void)(f); assert(*(p)); *(p)=false; } while(0) +#define lockdep_assert_held(p) assert(*(p)) +static bool *acquire(bool *p) { assert(!*p); *p=true; return p; } +static void release(bool **p) { if(*p) { assert(**p); **p=false; } } +static bool *once(bool *p) { assert(*p); *p=false; return NULL; } +#define scoped_guard(k,p) for(bool *_lock __attribute__((cleanup(release)))=acquire(p); _lock; _lock=once(_lock)) +static void mutex_lock(bool *p) { (void)acquire(p); } +static void mutex_unlock(bool *p) { (void)once(p); } +#define dev_info(...) ((void)0) +#define dev_err(...) ((void)0) +#define trace_dcpavserv_init(...) ((void)0) +#define DRM_CONNECTOR_REGISTERED 1 +#define to_platform_device(x) (x) +static void reinit_completion(int *p) { *p=0; } +static void complete(int *p) { *p=1; } +static void schedule_work(int *p) { (*p)++; } +#define dev_dbg_ratelimited(...) ((void)0) +#define dev_warn_ratelimited(...) ((void)0) +#define trace_dcp_recv_msg(...) ((void)0) +#define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 +#define DCP_FIRMWARE_H17P 2 +#define AV_ENDPOINT 0x29 +#define SYSTEM_ENDPOINT 0x20 +#define DISP0_ENDPOINT 0x23 +#define DPAVSERV_ENDPOINT 0x28 +#define DPAVSERV_ENDPOINT_V2 0x2a +#define DPTX_ENDPOINT 0x2a +#define DPTX_ENDPOINT_V2 0x2c +#define DPTX_CONTROLLER_ENDPOINT 0x24 +#define IOMFB_ENDPOINT 0x37 +struct apple_dcp; +struct apple_connector { void *dcp; struct { int registration_state; } base; bool connected, edid_invalidation_pending; int hotplug_wq; }; +struct apple_dcp_afkep { struct apple_dcp *dcp; u8 endpoint; bool wire_v2; }; +struct apple_epic_service { struct apple_dcp_afkep *ep; unsigned int users; bool enabled,torndown; struct { u64 generation,epoch; } admission; void *cookie; unsigned int channel; }; +struct apple_dcp { + struct apple_dcp_afkep *dcpavservep, *dptxep, *avep, *systemep, *ibootep, *dptx_controller_ep; + struct { bool lock, enabled; u64 epoch; int enable_completion; struct apple_epic_service *service; } dcpavserv; + struct apple_connector *connector; unsigned int nr_modes; + struct { int iomfb_method_profile; } hw; + bool hpd_mutex, typec_cable_connected, quiescing; + u64 typec_generation; + int fw_compat; + void *dev; +}; +struct drm_edid { int identity; }; +static unsigned int reads, allocations, legacy_reads, receives; +static unsigned int event; +static struct apple_dcp_afkep *received_ep; +static struct apple_epic_service *afk_service_get(struct apple_epic_service *s) +{ if(!s || !s->enabled || s->torndown) return NULL; s->users++; return s; } +static void afk_service_put(struct apple_epic_service *s) +{ assert(s->users); s->users--; } +static void afk_service_retire(struct apple_epic_service *s) { s->enabled=false;s->torndown=true; } +static void drm_edid_free(const struct drm_edid *e) +{ if(e) { assert(allocations); allocations--; free((void *)e); } } +static void dcpavserv_teardown(struct apple_epic_service *s); +static const struct drm_edid *dcpavserv_read_edid_inline(struct apple_epic_service *s) +{ + struct apple_dcp *d=s->ep->dcp; + assert(!d->hpd_mutex && !d->dcpavserv.lock && s->users==2); + reads++; + if(event==1 || event==2) { d->typec_generation++; d->typec_cable_connected=(event==2); } + if(event==3) { s->enabled=false; dcpavserv_teardown(s); } + struct drm_edid *e=malloc(sizeof(*e)); assert(e); e->identity=42; allocations++; return e; +} +static const struct drm_edid *dcpavserv_read_edid_legacy(struct apple_epic_service *s) +{ legacy_reads++; return dcpavserv_read_edid_inline(s); } +static void iomfb_recv_msg(struct apple_dcp *d, u64 message) { (void)d; (void)message; } +static void afk_receive_message(struct apple_dcp_afkep *ep, u64 message) +{ (void)message; received_ep=ep; receives++; } +''' + parts = [function('epic/dpavservep.c', 'static void dcpavserv_notify_edid('), + function('epic/dpavservep.c', 'static void dcpavserv_invalidate_edid('), + function('epic/dpavservep.c', 'static void dcpavserv_teardown('), + function('dcp.c', 'static struct apple_dcp_afkep *dcp_afkep('), + function('dcp.c', 'static void dcp_recv_msg('), + function('epic/dpavservep.c', 'static bool dcpavserv_identity('), + function('epic/dpavservep.c', 'static bool dcpavserv_admission_current('), + function('epic/dpavservep.c', 'static int dcpavserv_prepare('), + function('epic/dpavservep.c', 'static int dcpdpserv_prepare('), + function('epic/dpavservep.c', 'static void dcpavserv_init('), + function('epic/dpavservep.c', 'bool dcpavserv_edid_current('), + function('epic/dpavservep.c', 'bool dcpavserv_edid_service_current('), + function('epic/dpavservep.c', 'const struct drm_edid *dcpavserv_copy_edid(')] + suffix = r''' +int main(void) +{ + u64 epoch=0; + int device; + struct apple_connector connector={.dcp=&device,.base.registration_state=DRM_CONNECTOR_REGISTERED,.connected=true}; + struct apple_dcp d={.fw_compat=DCP_FIRMWARE_H17P,.dev=&device,.connector=&connector,.nr_modes=1}; + struct apple_dcp_afkep av={.dcp=&d,.endpoint=0x28}, tx={.dcp=&d,.endpoint=0x2a}; + d.dcpavservep=&av; d.dptxep=&tx; + assert(dcp_afkep(&d,0x28)==&av && dcp_afkep(&d,0x2a)==&tx && !dcp_afkep(&d,0x2c)); + dcp_recv_msg(&d,0x28,1); assert(received_ep==&av && receives==1); + dcp_recv_msg(&d,0x2a,2); assert(received_ep==&tx && receives==2); + d.hw.iomfb_method_profile=DCP_IOMFB_METHODS_H17P_EXTERNAL; + av.endpoint=0x2a; av.wire_v2=true; tx.endpoint=0x2c; + assert(!dcp_afkep(&d,0x28) && dcp_afkep(&d,0x2a)==&av && dcp_afkep(&d,0x2c)==&tx); + dcp_recv_msg(&d,0x28,3); assert(receives==2); + dcp_recv_msg(&d,0x2a,4); assert(received_ep==&av && receives==3); + dcp_recv_msg(&d,0x2c,5); assert(received_ep==&tx && receives==4); + d.dcpavservep=NULL; dcp_recv_msg(&d,0x2a,6); assert(receives==4); d.dcpavservep=&av; + struct apple_epic_service s={.ep=&av,.users=1,.enabled=true,.channel=1}; + const char *name="dcpav-service-epic", *class="AppleDCPDP2HDMI"; + assert(!dcpavserv_prepare(&s,name,class,1)); + assert(dcpavserv_prepare(&s,name,class,0)==-ENODEV); + assert(dcpavserv_prepare(&s,name,class,3)==-ENODEV); + assert(dcpavserv_prepare(&s,name,class,-1)==-ENODEV); + assert(dcpavserv_prepare(&s,"dcpdp-service-epic",class,1)==-ENODEV); + assert(dcpavserv_prepare(&s,name,"wrong",1)==-ENODEV); + assert(dcpavserv_prepare(&s,NULL,class,1)==-ENODEV); + assert(dcpavserv_prepare(&s,name,NULL,1)==-ENODEV); + av.endpoint=0x28; assert(dcpavserv_prepare(&s,name,class,1)==-ENODEV); av.endpoint=0x2a; + assert(dcpdpserv_prepare(&s,"dcpdp-service-epic",class,3)==-ENODEV); + d.dcpavserv.enabled=true; assert(dcpavserv_prepare(&s,name,class,1)==-EBUSY); + d.dcpavserv.service=&s; d.dcpavserv.epoch=1; d.typec_generation=1; d.typec_cable_connected=true; + const struct drm_edid *e=dcpavserv_copy_edid(&d,1,&epoch); + assert(!IS_ERR(e) && s.users==1 && reads==1); drm_edid_free(e); + d.typec_cable_connected=false; + assert(PTR_ERR(dcpavserv_copy_edid(&d,1,&epoch))==-ESTALE && reads==1); + d.typec_cable_connected=true; + assert(PTR_ERR(dcpavserv_copy_edid(&d,0,&epoch))==-ESTALE && reads==1); + event=1; assert(PTR_ERR(dcpavserv_copy_edid(&d,1,&epoch))==-ESTALE && !allocations && s.users==1); + event=2; d.typec_cable_connected=true; + assert(PTR_ERR(dcpavserv_copy_edid(&d,2,&epoch))==-ESTALE && !allocations && s.users==1); + event=0; e=dcpavserv_copy_edid(&d,3,&epoch); assert(!IS_ERR(e)); drm_edid_free(e); + event=3; assert(PTR_ERR(dcpavserv_copy_edid(&d,3,&epoch))==-ENODEV && !allocations && !s.users); + assert(PTR_ERR(dcpavserv_copy_edid(&d,3,&epoch))==-ENODEV); + assert(connector.edid_invalidation_pending && connector.hotplug_wq && d.dcpavserv.epoch==2); + /* Same slot and physical generation: CLOSE/re-OPEN gets a fresh ticket. */ + s.enabled=true; assert(!dcpavserv_prepare(&s,name,class,1)); dcpavserv_init(&s,name,class,1); + assert(d.dcpavserv.service==&s && s.users==1 && d.dcpavserv.epoch==3); + mutex_lock(&d.dcpavserv.lock); + assert(!dcpavserv_edid_service_current(&d,1) && dcpavserv_edid_service_current(&d,3)); + mutex_unlock(&d.dcpavserv.lock); + event=0; e=dcpavserv_copy_edid(&d,3,&epoch); assert(!IS_ERR(e) && epoch==3); + /* Simulate retirement in the final validation-to-publication gap. */ + s.enabled=false; dcpavserv_teardown(&s); s.enabled=true; assert(!dcpavserv_prepare(&s,name,class,1)); dcpavserv_init(&s,name,class,1); + mutex_lock(&d.dcpavserv.lock); assert(!dcpavserv_edid_service_current(&d,epoch)); mutex_unlock(&d.dcpavserv.lock); + drm_edid_free(e); assert(d.typec_generation==3 && d.dcpavserv.epoch==5 && s.users==1); + s.enabled=false; dcpavserv_teardown(&s); + av.wire_v2=false; av.endpoint=0x28; d.hw.iomfb_method_profile=0; + d.dcpavserv.enabled=false; assert(!dcpavserv_prepare(&s,name,NULL,0)); + assert(dcpavserv_prepare(&s,name,class,1)==-ENODEV); + assert(!dcpdpserv_prepare(&s,NULL,NULL,3)); + d.dcpavserv.service=&s; d.dcpavserv.enabled=true; s.enabled=true; s.users=1; event=0; + d.typec_cable_connected=false; e=dcpavserv_copy_edid(&d,0,&epoch); + assert(!IS_ERR(e) && legacy_reads==1 && s.users==1); drm_edid_free(e); + assert(!allocations && !d.hpd_mutex && !d.dcpavserv.lock); + puts("PASS: production old/new lookup and receive, exact service admission before OPEN, delayed disconnect/replug, service detach pins, legacy transport"); +} +''' + return prefix + '\n'.join(parts) + suffix + +def rx_progress(root): + + def source(path): + return (root / path).read_text() + + def function(text, signature): + start = text.index(signature) + opening = text.index('{', start) + while ';' in text[start:opening]: + start = text.index(signature, start+1) + opening = text.index('{', start) + end, depth = opening+1, 1 + while depth: + depth += (text[end]=='{')-(text[end]=='}') + end += 1 + return text[start:end] + + dcp=source('drivers/gpu/drm/apple/dcp.c') + iomfb=source('drivers/gpu/drm/apple/iomfb.c') + af=source('drivers/gpu/drm/apple/afk.c') + rtkit=source('drivers/soc/apple/rtkit.c') + template=source('drivers/gpu/drm/apple/iomfb_template.c') + assert 'alloc_ordered_workqueue("rtkit-%s"' in rtkit + assert '.recv_message = dcp_recv_msg' in dcp + assert 'dcp_set_dimensions(dcp);' in template[template.index('"DisplayAttributes"'):] + assert 'queue_work(ep->wq, &work->work)' in af + prefix=r''' +#undef NDEBUG +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint32_t u32; +typedef uint64_t u64; +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define GFP_KERNEL 0 +#define kzalloc(n,f) calloc(1,n) +#define kfree(p) free(p) +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +#define INIT_WORK(p,f) ((p)->fn=(f)) +#define dev_warn(...) ((void)0) +#define DCP_FW_VER 26 +#define DCP_FW_VERSION(a,b,c) (a) +#define SZ_1M (1024*1024) +#define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 +#define dev_warn_ratelimited(...) ((void)0) +#define dev_dbg_ratelimited(...) ((void)0) +#define trace_dcp_recv_msg(...) ((void)0) +#define DCP_FIRMWARE_H17P 2 +#define IOMFB_ENDPOINT 0x37 +#define AV_ENDPOINT 0x29 +#define SYSTEM_ENDPOINT 0x20 +#define DISP0_ENDPOINT 0x23 +#define DPTX_CONTROLLER_ENDPOINT 0x24 +#define APPLE_RTKIT_EP_MGMT 0 +#define APPLE_RTKIT_EP_CRASHLOG 1 +#define APPLE_RTKIT_EP_SYSLOG 2 +#define APPLE_RTKIT_EP_IOREPORT 4 +#define APPLE_RTKIT_EP_OSLOG 8 +#define APPLE_RTKIT_APP_ENDPOINT_START 0x20 +struct mutex { pthread_mutex_t native; }; +static _Thread_local struct mutex *locks[4]; +static _Thread_local unsigned int depth; +static bool held(struct mutex *m) { for(unsigned int i=0;inative)); locks[depth++]=m; } +static void mutex_unlock(struct mutex *m) { assert(depth && locks[depth-1]==m); depth--; assert(!pthread_mutex_unlock(&m->native)); } +#define lockdep_assert_held(p) assert(held(p)) +#define spin_lock_irqsave(p,f) do { mutex_lock(p); (f)=0; } while(0) +#define spin_unlock_irqrestore(p,f) do { (void)(f); mutex_unlock(p); } while(0) +struct work_struct { void (*fn)(struct work_struct *); }; +struct workqueue_struct { struct work_struct *pending; pthread_cond_t ready; pthread_mutex_t lock; }; +struct drm_device { struct { struct mutex mutex; } mode_config; }; +struct drm_connector { struct drm_device *dev; struct { int width_mm,height_mm; } display_info; }; +struct apple_connector { struct drm_connector base; struct work_struct hotplug_wq; }; +struct apple_dcp; +struct apple_dcp_afkep { struct apple_dcp *dcp; u8 endpoint; struct mutex lock; bool stopping; struct workqueue_struct *wq; }; +struct test_mode_ticket { u64 generation; bool associated; }; +struct apple_dcp { + bool quiescing; int fw_compat; void *dev; + struct { int iomfb_method_profile; } hw; + struct { struct mutex lock; u64 epoch; bool enabled; void *service; } dcpavserv; + struct test_mode_ticket dummy_ticket; + struct { u64 connection_sequence; } rate_proof; + struct { struct mutex lock; struct { u64 generation; bool connected; } epoch; } metadata; + u64 typec_generation; bool typec_cable_connected; + struct { size_t length; void *data; u64 av_service_epoch, automatic_connection_sequence; bool automatic_service_pending; struct test_mode_ticket mode_ticket; } chunks; + int width_mm,height_mm; struct { int width_mm,height_mm; } panel; + unsigned int nr_modes; struct { struct { int width_mm,height_mm; } mode; } modes[1]; + struct apple_connector *connector; + struct apple_dcp_afkep *dcpavservep,*dptxep,*avep,*systemep,*ibootep,*dptx_controller_ep; + struct { struct mutex lock; bool stopped; void *owner; } iomfb; +}; +struct afk_receive_message_work { struct apple_dcp_afkep *ep; u64 message; struct work_struct work; }; +struct apple_rtkit_ops { void (*recv_message)(void *,u8,u64); }; +struct apple_rtkit { bool shutting_down; const struct apple_rtkit_ops *ops; void *cookie,*dev; }; +struct apple_rtkit_rx_work { struct apple_rtkit *rtk; u8 ep; u64 msg; struct work_struct work; }; +#define current ((void *)1) +static pthread_mutex_t reply_lock=PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t reply_ready=PTHREAD_COND_INITIALIZER; +static bool reply_done; +static unsigned int advances, dimension_updates; +static struct workqueue_struct dimensions_queue={.ready=PTHREAD_COND_INITIALIZER,.lock=PTHREAD_MUTEX_INITIALIZER}; +static struct workqueue_struct afk_queue={.ready=PTHREAD_COND_INITIALIZER,.lock=PTHREAD_MUTEX_INITIALIZER}; +static void queue_work(struct workqueue_struct *q,struct work_struct *w) +{ assert(!pthread_mutex_lock(&q->lock)); assert(!q->pending); q->pending=w; pthread_cond_signal(&q->ready); assert(!pthread_mutex_unlock(&q->lock)); } +static void schedule_work(struct work_struct *w) { queue_work(&dimensions_queue,w); } +static void afk_receive_message_worker(struct work_struct *w) +{ + struct afk_receive_message_work *a=container_of(w,struct afk_receive_message_work,work); + assert(a->ep->endpoint==0x2a && a->message==7); + assert(!pthread_mutex_lock(&reply_lock)); reply_done=true; + pthread_cond_signal(&reply_ready); assert(!pthread_mutex_unlock(&reply_lock)); free(a); +} +static bool iomfb_uses_queue(struct apple_dcp *d) { (void)d; return true; } +static void iomfb_queue_advance(struct apple_dcp *d) { assert(held(&d->iomfb.lock)); advances++; } +static void dcp_set_dimensions(struct apple_dcp *d); +static u8 dcpep_cb_prop_start(struct apple_dcp *d,u32 *length); +static void dcp_firmware_metadata_begin(struct apple_dcp *d) { (void)d; } +#define dcp_metadata_begin(e) ((struct test_mode_ticket){.generation=(e)->generation,.associated=(e)->connected}) +static void iomfb_recv_message(struct apple_dcp *d,u64 message) +{ + assert(message==1 && held(&d->iomfb.lock)); dcp_set_dimensions(d); + u32 length=64; + assert(dcpep_cb_prop_start(d,&length)); +} +#define rx_unused(name) static void name(struct apple_rtkit *r,u64 m) { (void)r;(void)m; } +rx_unused(apple_rtkit_management_rx) +rx_unused(apple_rtkit_crashlog_rx) +rx_unused(apple_rtkit_syslog_rx) +rx_unused(apple_rtkit_ioreport_rx) +rx_unused(apple_rtkit_oslog_rx) +''' + parts=[function(dcp,'void dcp_set_dimensions('), function(template,'static u8 dcpep_cb_prop_start(')] + if 'void dcp_update_connector_dimensions(' in dcp: + parts.append(function(dcp,'void dcp_update_connector_dimensions(')) + else: + parts.append('static void dcp_update_connector_dimensions(struct apple_dcp *d, struct drm_connector *c) { (void)d; (void)c; }') + parts += [function(iomfb,'void iomfb_recv_msg('),function(af,'int afk_receive_message('), + function(dcp,'static struct apple_dcp_afkep *dcp_afkep('), + function(dcp,'static void dcp_recv_msg('),function(rtkit,'static void apple_rtkit_rx_work(')] + suffix=r''' +static struct drm_device drm={.mode_config.mutex.native=PTHREAD_MUTEX_INITIALIZER}; +static struct apple_connector connector={.base.dev=&drm}; +static struct apple_dcp d={.connector=&connector,.width_mm=600,.height_mm=340,.nr_modes=1,.fw_compat=DCP_FIRMWARE_H17P,.iomfb.lock.native=PTHREAD_MUTEX_INITIALIZER}; +static struct apple_dcp_afkep av={.dcp=&d,.endpoint=0x2a,.lock.native=PTHREAD_MUTEX_INITIALIZER,.wq=&afk_queue}; +static void dimension_work(struct work_struct *w) +{ + (void)w; + mutex_lock(&drm.mode_config.mutex); + dcp_update_connector_dimensions(&d,&connector.base); dimension_updates++; + mutex_unlock(&drm.mode_config.mutex); +} +static void *worker(void *arg) +{ + struct workqueue_struct *q=arg; + assert(!pthread_mutex_lock(&q->lock)); + while(!q->pending) assert(!pthread_cond_wait(&q->ready,&q->lock)); + struct work_struct *w=q->pending; q->pending=NULL; + assert(!pthread_mutex_unlock(&q->lock)); w->fn(w); return NULL; +} +static const struct apple_rtkit_ops ops={.recv_message=dcp_recv_msg}; +static struct apple_rtkit rtk={.ops=&ops,.cookie=&d}; +static void *ordered_rx(void *arg) +{ + (void)arg; + /* One ordered RX worker must complete DisplayAttributes before AFK. */ + struct apple_rtkit_rx_work *a=calloc(1,sizeof(*a)); assert(a); + a->rtk=&rtk; a->ep=IOMFB_ENDPOINT; a->msg=1; apple_rtkit_rx_work(&a->work); + a=calloc(1,sizeof(*a)); assert(a); + a->rtk=&rtk; a->ep=0x2a; a->msg=7; apple_rtkit_rx_work(&a->work); + return NULL; +} +int main(void) +{ + pthread_t rx,afk,dimensions; + (void)schedule_work; /* The negative baseline does not defer dimensions. */ + d.dcpavservep=&av; connector.hotplug_wq.fn=dimension_work; + d.hw.iomfb_method_profile=DCP_IOMFB_METHODS_H17P_EXTERNAL; + assert(!pthread_mutex_init(&d.dcpavserv.lock.native,NULL)); d.dcpavserv.epoch=1; + assert(!pthread_mutex_init(&d.metadata.lock.native,NULL)); d.metadata.epoch.generation=1; + d.metadata.epoch.connected=true; d.typec_generation=1; d.typec_cable_connected=true; + mutex_lock(&drm.mode_config.mutex); /* DRM get_modes holds this while waiting. */ + assert(!pthread_create(&afk,NULL,worker,&afk_queue)); + assert(!pthread_create(&dimensions,NULL,worker,&dimensions_queue)); + assert(!pthread_mutex_lock(&reply_lock)); + assert(!pthread_create(&rx,NULL,ordered_rx,NULL)); + struct timespec deadline; assert(!clock_gettime(CLOCK_REALTIME,&deadline)); deadline.tv_sec+=2; + int rc=0; + while(!reply_done && !rc) rc=pthread_cond_timedwait(&reply_ready,&reply_lock,&deadline); + if(!reply_done) { fputs("FAIL: ordered RTKit RX blocked before AFK reply while mode_config.mutex held\n",stderr); fflush(stderr); _Exit(1); } + assert(!pthread_mutex_unlock(&reply_lock)); + assert(!dimension_updates); /* Deferred worker cannot take the probe's mutex. */ + mutex_unlock(&drm.mode_config.mutex); + assert(!pthread_join(rx,NULL) && !pthread_join(afk,NULL) && !pthread_join(dimensions,NULL)); + assert(advances==1 && dimension_updates==1 && connector.base.display_info.width_mm==600 && connector.base.display_info.height_mm==340); + assert(d.modes[0].mode.width_mm==600 && d.modes[0].mode.height_mm==340); + assert(d.chunks.length==64 && d.chunks.data); free(d.chunks.data); + assert(!pthread_mutex_destroy(&d.dcpavserv.lock.native)); + assert(!pthread_mutex_destroy(&d.metadata.lock.native)); + puts("PASS: actual ordered RTKit RX -> synchronous IOMFB dimensions/property-start callbacks -> AFK enqueue makes reply progress while DRM mutex held; deferred dimensions update completes after release"); +} +''' + return prefix + '\n'.join(parts) + suffix diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/idle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/idle.py new file mode 100644 index 0000000000000..7a0666fbe92ef --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/idle.py @@ -0,0 +1,185 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Actual cpuidle driver/registration/selector and tick suspension source replay.""" +import re +def function(text, name): + match=re.search(r'^((?:static\s+)?(?:inline\s+)?(?:__cpuidle\s+|noinstr\s+)?(?:int|void)\s+'+re.escape(name)+r'\([^;{}]*?\))\s*\{',text,re.M) + if not match: raise ValueError(name) + start=match.start(); pos=match.end(); depth=1 + while depth: + if text[pos]=='{': depth+=1 + elif text[pos]=='}': depth-=1 + pos+=1 + return text[start:pos]+'\n' + +MOCKS = r''' +#include +#include +#include +#include +#include +#include +typedef uint64_t u64; typedef int64_t s64; typedef int64_t ktime_t; +#define CONFIG_SUSPEND 1 +#define CPUIDLE_NAME_LEN 16 +#define CPUIDLE_DESC_LEN 32 +#define CPUIDLE_STATE_MAX 10 +#define BIT(n) (1U<<(n)) +#define NSEC_PER_USEC 1000 +#define __cpuidle +#define noinstr +#define THIS_MODULE NULL +#define SYSTEM_SUSPEND 1 +#define SYSTEM_RUNNING 0 +#define TPS(x) x +#define instrumentation_begin() ((void)0) +#define instrumentation_end() ((void)0) +#define raw_spin_lock(x) ((void)0) +#define raw_spin_unlock(x) ((void)0) +#define lock_map_acquire_try(x) ((void)0) +#define lock_map_release(x) ((void)0) +#define trace_suspend_resume(...) ((void)0) +#define WARN_ON(x) ((x)?(warnings++,1):0) +#define WARN_ON_ONCE(x) WARN_ON(x) +#define pr_err(...) ((void)0) +#define for_each_cpu(cpu,mask) for(cpu=0;cpu<(mask)->count;cpu++) +#define per_cpu(v,cpu) v[cpu] +struct platform_device {int dummy;}; +struct module; struct cpumask {int count;}; +struct cpuidle_device; struct cpuidle_driver; +struct cpuidle_device {int cpu; struct cpuidle_state_usage states_usage[10];}; +static struct cpumask possible={6}; static const struct cpumask *cpu_possible_mask=&possible; +static struct cpuidle_device cpuidle_dev[6]; +static int current, irq[6], warnings, wfi, deep, pm, pm_fail, reg_fail=-1, drv_fail; +static int registered[6], unregistered, rcu[6], frozen, stopped[6], online=6; +static int tick_freeze_depth, tick_freeze_lock, tick_freeze_map, system_state; +static int tk_suspend, tk_resume, local_suspend, local_resume, sched_suspend, sched_resume; +static int fanout, faulty_irq, checks; +static uint64_t clock_ns; +static void check(bool b) {checks++; assert(b);} +static int num_online_cpus(void){return online;} +static void sched_clock_suspend(void){sched_suspend++;} +static void sched_clock_resume(void){sched_resume++;} +static void timekeeping_suspend(void){check(tick_freeze_depth==online);tk_suspend++;frozen=1;} +static void timekeeping_resume(void){check(frozen);tk_resume++;frozen=0;} +static void tick_suspend_local(void){local_suspend++;} +static void tick_resume_local(void){local_resume++;} +static void touch_softlockup_watchdog(void){} +static void ct_cpuidle_enter(void){check(irq[current]);rcu[current]++;} +static void ct_cpuidle_exit(void){check(irq[current]);rcu[current]--;} +static bool irqs_disabled(void){return irq[current];} +static void raw_local_irq_disable(void){irq[current]=1;} +static void local_irq_enable(void){check(!frozen);irq[current]=0;} +static void stop_critical_timings(void){stopped[current]++;} +static void start_critical_timings(void){stopped[current]--;} +static ktime_t local_clock_noinstr(void){return clock_ns+=1000;} +static ktime_t ns_to_ktime(ktime_t t){return t;} +static ktime_t ktime_us_delta(ktime_t a,ktime_t b){return (a-b)/1000;} +static u64 div_u64(u64 a,u64 b){return a/b;} +static u64 mul_u32_u32(unsigned a,unsigned b){return (u64)a*b;} +static bool t8140; +static bool of_machine_is_compatible(const char *s){return t8140&&!strcmp(s,"apple,t8140");} +static int cpu_pm_enter(void){pm++;return pm_fail;} +static void cpu_pm_exit(void){pm--;} +static void apple_cpu_deep_wfi(void){deep++;} +static void cpu_do_idle(void); +static int cpuidle_register_driver(struct cpuidle_driver *drv); +static int cpuidle_register_device(struct cpuidle_device *dev){if(dev->cpu==reg_fail)return -5; registered[dev->cpu]=1;return 0;} +static void cpuidle_unregister(struct cpuidle_driver *drv){(void)drv;memset(registered,0,sizeof(registered));unregistered++;} +''' +MAIN = r''' +static int cpuidle_register_driver(struct cpuidle_driver *drv){if(drv_fail)return -5;__cpuidle_driver_init(drv);return 0;} +static void cpu_do_idle(void){ + check(irq[current]);wfi++; + if(fanout && current<5){int previous=current;current++;irq[current]=1; + cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[current],100000);current=previous;} + if(faulty_irq)irq[current]=0; +} +static struct cpuidle_driver original; +static void reset(bool is_t8140){ + apple_idle_driver=original;t8140=is_t8140; current=0; + memset(cpuidle_dev,0,sizeof(cpuidle_dev)); memset(registered,0,sizeof(registered)); + for(int i=0;i<6;i++){irq[i]=1;rcu[i]=stopped[i]=0;} + wfi=deep=pm=pm_fail=warnings=unregistered=0;reg_fail=-1;drv_fail=0; + tick_freeze_depth=frozen=tk_suspend=tk_resume=local_suspend=local_resume=sched_suspend=sched_resume=0; + fanout=faulty_irq=0; +} +int main(void){ + original=apple_idle_driver; + reset(true);check(!apple_cpuidle_probe(NULL)); + check(apple_idle_driver.cpumask==cpu_possible_mask);check(!apple_idle_driver.bctimer); + for(int i=0;i<6;i++)check(registered[i]&&cpuidle_dev[i].cpu==i); + int expected=EXPECT_FIXED?1:0; + check(apple_idle_driver.state_count==(EXPECT_FIXED?2:1)); + check(find_deepest_state(&apple_idle_driver,&cpuidle_dev[0],100000,0,true)==expected); + check(cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],100000)==expected); + check(tick_freeze_depth==0&&!frozen);check(deep==0&&pm==0); + check(local_suspend==expected&&local_resume==expected); + if(EXPECT_FIXED){ + check(apple_idle_driver.states[1].enter==apple_enter_wfi); + check(apple_idle_driver.states[1].enter_s2idle==apple_enter_wfi); + check(apple_idle_driver.states[1].flags==0); + check(apple_idle_driver.states[1].exit_latency_ns==1000); + check(apple_idle_driver.states[1].target_residency_ns==1000); + check(apple_idle_driver.states[1].power_usage==apple_idle_driver.states[0].power_usage); + reset(true);apple_cpuidle_probe(NULL);fanout=1; + check(cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],100000)==1); + check(wfi==6&&deep==0&&pm==0);check(tk_suspend==1&&tk_resume==1); + check(sched_suspend==1&&sched_resume==1&&local_suspend==5&&local_resume==5); + check(tick_freeze_depth==0&&!frozen); + for(int i=0;i<6;i++)check(cpuidle_dev[i].states_usage[1].s2idle_usage==1&&rcu[i]==0&&stopped[i]==0); + reset(true);apple_cpuidle_probe(NULL);fanout=1;cpuidle_dev[5].states_usage[1].disable=1; + cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],100000); + check(wfi==5&&tk_suspend==0&&tk_resume==0&&tick_freeze_depth==0); + reset(true);apple_cpuidle_probe(NULL);cpuidle_dev[0].states_usage[1].disable=1; + check(cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],100000)==0&&wfi==0); + cpuidle_dev[0].states_usage[1].disable=0;apple_idle_driver.states[1].enter_s2idle=NULL; + check(cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],100000)==0&&wfi==0); + reset(true);apple_cpuidle_probe(NULL); + check(cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],999)==0); + check(cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],1000)==1); + reset(true);apple_cpuidle_probe(NULL);apple_idle_driver.states[1].exit_latency_ns=0; + check(find_deepest_state(&apple_idle_driver,&cpuidle_dev[0],100000,0,true)==0); + reset(true);apple_cpuidle_probe(NULL);apple_idle_driver.states[1].flags=CPUIDLE_FLAG_TIMER_STOP; + check(find_deepest_state(&apple_idle_driver,&cpuidle_dev[0],100000,CPUIDLE_FLAG_TIMER_STOP,false)==0); + reset(true);apple_cpuidle_probe(NULL);faulty_irq=1; + check(cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],100000)==1); + check(warnings==1&&tick_freeze_depth==0&&rcu[0]==0&&stopped[0]==0); + } + reset(true);apple_cpuidle_probe(NULL); + for(int i=0;i', '') + core = 'drivers/usb/typec/tipd/core.c' + dcp = 'drivers/gpu/drm/apple/dcp.c' + funcs = '\n'.join(function(f, n) for f, n in ( + ('drivers/gpu/drm/apple/iomfb.c', 'iomfb_pause_presentation'), + (dcp, 'dcp_external_report_disconnected'), + (dcp, 'dcp_external_publish_generation'), + (dcp, 'dcp_external_stop_locked'), + (dcp, 'dcp_external_disconnect'), + (dcp, 'dcp_external_stop_work'), + (dcp, 'dcp_external_hotplug'), + (dcp, 'dcp_external_link_locked'), + (dcp, 'dcp_dptx_connect_oob'), + (core, 'cd321x_retry_revalidation'), + (core, 'cd321x_update_work'))) + prefix = r''' +#include +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; typedef uint64_t u64; +#define guard(x) fake_guard +#define fake_guard(x) ((void)(x)) +#define scoped_guard(x,y) for (int once = ((void)(y), 1); once; once = 0) +#define lockdep_assert_held(x) ((void)(x)) +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x) = (v)) +#define container_of(p,t,m) ((t *)((char *)(p) - offsetof(t,m))) +#define to_delayed_work(w) container_of(w,struct delayed_work,work) +#define msecs_to_jiffies(x) (x) +#define IS_REACHABLE(x) 1 +#define CONFIG_DRM 1 +#define IS_ERR(x) false +#define PTR_ERR(x) (-EINVAL) +#define dev_warn(...) ((void)0) +#define dev_err(...) ((void)0) +#define CD321X_DEBOUNCE_DELAY_MS 500 +#define TPS_STATUS_PLUG_PRESENT (1U << 0) +#define TPS_STATUS_PLUG_UPSIDE_DOWN (1U << 1) +#define TPS_DATA_STATUS_USB2_CONNECTION (1U << 0) +#define TPS_DATA_STATUS_USB3_CONNECTION (1U << 1) +#define TPS_DATA_STATUS_TBT_CONNECTION (1U << 2) +#define TPS_DATA_STATUS_DP_CONNECTION (1U << 3) +#define TPS_DATA_STATUS_DP_PIN_ASSIGNMENT_MASK (3U << 4) +#define TPS_DATA_STATUS_USB_DATA_ROLE (1U << 6) +#define CD321X_DATA_STATUS_USB4_CONNECTION (1U << 7) +#define CD321X_DATA_STATUS_HPD_LEVEL (1U << 8) +#define TPS_POWER_STATUS_PWROPMODE(x) (x) +#define TPS_STATUS_TO_UPSIDE_DOWN(x) ((x)&TPS_STATUS_PLUG_UPSIDE_DOWN) +#define TPS_STATUS_TO_TYPEC_PORTROLE(x) 0 +#define TPS_STATUS_TO_TYPEC_VCONN(x) 0 +#define TPS_STATUS_TO_TYPEC_DATAROLE(x) 0 +#define TYPEC_STATE_SAFE 0 +#define TYPEC_STATE_USB 1 +#define TYPEC_THUNDERBOLT_SWITCH_OFF 0 +#define TYPEC_ACCESSORY_NONE 0 +#define DCP_HOTPLUG_NOTIFY 1 +struct work_struct { bool pending; }; +struct delayed_work { struct work_struct work; }; +static int system_freezable_wq; +static int schedule_work(struct work_struct *w) { w->pending = true; return 1; } +static void cancel_work_sync(struct work_struct *w) { w->pending = false; } +static void cancel_delayed_work_sync(struct delayed_work *w) { w->work.pending = false; } +#define cancel_delayed_work cancel_delayed_work_sync +static void mod_delayed_work(int q, struct delayed_work *w, int t) +{ (void)q; (void)t; w->work.pending = true; } +struct apple_connector { bool connected, edid_invalidation_pending; }; +struct apple_dcp { + int external_lock, hpd_mutex; + struct { int lock; bool presentation_paused; } iomfb; + struct { bool connected; void *service; } dptxport[1]; + struct apple_connector *connector; + struct delayed_work typec_reconnect_wq, placeholder_edid_wq; + struct work_struct external_stop_work; + bool external_running, external_stopped, external_power_attempted; + bool external_cable, typec_cable_connected, typec_crtc_off, pipe_enabled_h17p; + bool external_physically_absent, external_prepared, placeholder_retried; + int typec_reconnect_tries; + bool external_hotplug_pending, external_hotplug_connected, quiescing; + u64 external_generation, external_rx_generation, external_crtc_generation; + u64 external_stop_generation, typec_generation; + int external_error, mode_state; void *avep; +}; +struct platform_device { struct apple_dcp *data; }; +static struct apple_dcp *platform_get_drvdata(struct platform_device *p) { return p->data; } +static int raw_error, raw_links; +static int dcp_dptx_connect_raw(struct apple_dcp *d, u32 port, bool *waiting) +{ assert(port == 0); *waiting = false; raw_links++; + if (!raw_error) d->dptxport[0].connected = true; + return raw_error; } +static bool dcp_is_typec_output(struct apple_dcp *d) { (void)d; return true; } +static int dcp_dptx_connect(struct apple_dcp *d, u32 port) { (void)d; (void)port; return 0; } +#define DPTX_RECONNECT_DELAY 500 +static int settle_error, power_error, hpd_error, release_error; +static int settles, powers, releases, detaches, notices; +static bool dcp_is_external_h17p(struct apple_dcp *d) { (void)d; return true; } +static unsigned dcp_mode_hotplug(int *mode, bool connected, bool *sink) +{ (void)mode; if (sink) *sink = connected; return 1; } +static void dcp_handle_hotplug_actions(struct apple_dcp *d, unsigned act) +{ (void)d; notices += !!act; } +static int iomfb_queue_settle(struct apple_dcp *d, int t) +{ (void)d; (void)t; settles++; return settle_error; } +static int iomfb_external_power_h17p(struct apple_dcp *d, bool on) +{ (void)d; assert(!on); powers++; return power_error; } +static bool dptx_service_available(void *p) { (void)p; return true; } +static int dptxport_set_hpd(void *p, bool on) +{ (void)p; assert(!on); return hpd_error; } +static int dptxport_release_display(void *p) +{ (void)p; releases++; return release_error; } +static void av_service_disconnect(struct apple_dcp *d) { (void)d; } +static void dpavservep_detach(struct apple_dcp *d) { (void)d; detaches++; } +static void dcp_firmware_metadata_hotplug(struct apple_dcp *d, bool c) { (void)d; (void)c; } +static void dcp_firmware_metadata_connection(struct apple_dcp *d, u64 g, bool c) +{ (void)d; (void)g; (void)c; } +static int dcp_external_disconnect(struct apple_dcp *, bool); +static struct apple_dcp display; +static void drm_connector_oob_hotplug_event(void *node, int status) +{ assert(node); if (!status) dcp_external_disconnect(&display, true); } +#define connector_status_disconnected 0 +#define connector_status_connected 1 +''' + shim = r''' +enum usb_role { USB_ROLE_NONE, USB_ROLE_HOST, USB_ROLE_DEVICE }; +enum typec_pwr_opmode { TYPEC_PWR_MODE_USB, TYPEC_PWR_MODE_PD }; +enum typec_orientation { TYPEC_ORIENTATION_NONE, TYPEC_ORIENTATION_NORMAL, TYPEC_ORIENTATION_REVERSE }; +struct usb_pd_identity { int id; }; +struct typec_mux_state { void *alt, *data; int mode; }; +struct typec_thunderbolt_switch_data { int state; }; +struct typec_partner_desc { bool usb_pd; int accessory; struct usb_pd_identity *identity; }; +struct cd321x_status { u32 status, data_status, status_changed, data_status_changed, pwr_status; + struct usb_pd_identity partner_identity; }; +struct tps6598x { void *partner, *dev, *port, *psy; int lock; u32 data_status; enum usb_role *role_sw; }; +struct cd321x { struct tps6598x tps; struct cd321x_status update_status; + struct cd321x_pm_state pm; struct delayed_work update_work, resume_work; + struct typec_mux_state state; bool state_valid, display_route_active, cur_partner_is_pd; + struct usb_pd_identity cur_partner_identity; void *connector_fwnode, *tbt_switch; }; +static int tbt_error, role_error, safe_error, mode_error; +static int tbts, roles, safes; +static enum usb_role usb_role_switch_get_role(enum usb_role *r) { return *r; } +static int usb_role_switch_set_role(enum usb_role *r, enum usb_role n) +{ roles++; if (role_error) return role_error; *r = n; return 0; } +static int typec_thunderbolt_switch_set(void *p, void *s) +{ (void)p; (void)s; tbts++; return tbt_error; } +static int typec_set_mode(void *p, int s) +{ (void)p; assert(s == TYPEC_STATE_SAFE); safes++; return safe_error; } +static int cd321x_typec_update_mode(struct tps6598x *t, struct cd321x_status *s, bool *busy) +{ (void)t; (void)s; *busy = mode_error == -EBUSY; return mode_error; } +static void typec_unregister_partner(void *p) { (void)p; } +static void *typec_register_partner(void *p, struct typec_partner_desc *d) +{ (void)p; (void)d; return (void *)1; } +static void typec_partner_set_identity(void *p) { (void)p; } +#define typec_set_pwr_opmode(...) ((void)0) +#define typec_set_pwr_role(...) ((void)0) +#define typec_set_vconn_role(...) ((void)0) +#define typec_set_orientation(...) ((void)0) +#define typec_set_data_role(...) ((void)0) +#define power_supply_changed(...) ((void)0) +''' + tests = r''' +static int failures, checks; +#define CHECK(x) do { checks++; if (!(x)) { printf("FAIL line %d: %s\n", __LINE__, #x); failures++; } } while (0) +static struct apple_connector sink; +static enum usb_role role; +static struct cd321x port; +static void init(void) +{ + memset(&display,0,sizeof(display)); memset(&sink,0,sizeof(sink)); memset(&port,0,sizeof(port)); + sink.connected = true; display.connector = &sink; display.external_prepared = true; + raw_error = raw_links = 0; + display.external_running = display.external_power_attempted = true; + display.external_cable = display.typec_cable_connected = true; + display.dptxport[0].connected = display.pipe_enabled_h17p = true; + display.external_generation = display.external_rx_generation = display.external_crtc_generation = 7; + display.typec_generation = 4; + port.tps.partner = (void *)1; port.tps.role_sw = &role; role = USB_ROLE_HOST; + port.pm.phase = CD321X_PM_RUNNING; port.state_valid = true; port.state.mode = 2; + port.display_route_active = true; port.connector_fwnode = (void *)1; + settle_error = power_error = hpd_error = release_error = 0; + tbt_error = role_error = safe_error = mode_error = 0; + settles = powers = releases = detaches = notices = tbts = roles = safes = 0; +} +static void callback(bool connected) +{ + if (!dcp_external_hotplug(&display,connected)) + dcp_mode_hotplug(&display.mode_state,connected,&sink.connected); +} +int main(void) +{ + /* Unplug at each fallible transition with failing DCP power teardown. */ + for (int which = 0; which < 3; which++) { + init(); power_error = -ETIMEDOUT; + if (which == 0) tbt_error = -ETIMEDOUT; + if (which == 1) role_error = -EIO; + if (which == 2) safe_error = -ETIMEDOUT; + cd321x_update_work(&port.update_work.work); + CHECK(!sink.connected); CHECK(!display.external_cable); + CHECK(display.external_error == -ETIMEDOUT); + CHECK(display.dptxport[0].connected && display.pipe_enabled_h17p); + CHECK(!display.external_stopped && !display.typec_crtc_off); + CHECK(port.display_route_active && !port.state_valid); + CHECK(port.pm.phase == CD321X_PM_REVALIDATE && port.pm.attempts_left == CD321X_RESUME_ATTEMPTS); + CHECK(notices > 0 && releases == 0); + callback(true); CHECK(!sink.connected); + printf("case transition-%d complete\n",which); + } + /* A rejected/nonowner port cannot disconnect the shared owner's sink. */ + for (int which = 0; which < 3; which++) { + init(); port.display_route_active = false; + if (which == 0) tbt_error = -EIO; + if (which == 1) role_error = -EIO; + if (which == 2) safe_error = -EIO; + cd321x_update_work(&port.update_work.work); + CHECK(sink.connected && display.external_cable && !notices); + } + init(); port.display_route_active = false; role = USB_ROLE_NONE; + port.update_status.status = TPS_STATUS_PLUG_PRESENT; + port.update_status.data_status = TPS_DATA_STATUS_DP_CONNECTION | CD321X_DATA_STATUS_HPD_LEVEL; + mode_error = -EBUSY; cd321x_update_work(&port.update_work.work); + CHECK(sink.connected && !notices && !port.display_route_active); + /* Two ports share the connector: busy-port unplug leaves its owner live. */ + init(); + struct cd321x second = port; + enum usb_role second_role = USB_ROLE_NONE; + second.tps.role_sw = &second_role; second.display_route_active = false; + second.update_status.status = TPS_STATUS_PLUG_PRESENT; + second.update_status.data_status = TPS_DATA_STATUS_DP_CONNECTION | CD321X_DATA_STATUS_HPD_LEVEL; + mode_error = -EBUSY; cd321x_update_work(&second.update_work.work); + CHECK(!second.display_route_active && sink.connected && !notices); + second.update_status.status = second.update_status.data_status = 0; + tbt_error = -EIO; cd321x_update_work(&second.update_work.work); + CHECK(port.display_route_active && sink.connected && !notices); + power_error = -EIO; cd321x_update_work(&port.update_work.work); + CHECK(!sink.connected && port.display_route_active); + CHECK(display.dptxport[0].connected && display.pipe_enabled_h17p); + /* Revalidation is bounded even though the failed hardware remains owned. */ + init(); tbt_error = -EIO; + cd321x_update_work(&port.update_work.work); + for (int i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { + CHECK(cd321x_pm_begin_read(&port.pm)); cd321x_pm_snapshot_ready(&port.pm); + cd321x_update_work(&port.update_work.work); + } + CHECK(port.pm.phase == CD321X_PM_STALE && !port.pm.attempts_left); + CHECK(!sink.connected && port.display_route_active); + /* Stop failure stages preserve all state whose shutdown was not acked. */ + for (int which = 0; which < 4; which++) { + init(); + if (which == 0) settle_error = -EIO; + if (which == 1) power_error = -EIO; + if (which == 2) hpd_error = -EIO; + if (which == 3) release_error = -EIO; + CHECK(dcp_external_disconnect(&display,true) == -EIO); + CHECK(!sink.connected && !display.typec_cable_connected); + CHECK(display.external_generation == 8 && display.external_rx_generation == 8); + CHECK(display.external_error == -EIO && display.dptxport[0].connected); + CHECK(!display.external_stopped && !display.typec_crtc_off); + CHECK(display.pipe_enabled_h17p == (which < 2)); + callback(true); CHECK(!sink.connected); + } + /* An existing error returns before pause: late RX still cannot resurrect. */ + init(); display.external_error = -EIO; + CHECK(dcp_external_disconnect(&display,true) == -EIO); + CHECK(!display.iomfb.presentation_paused); + callback(true); CHECK(!sink.connected); + CHECK(display.dptxport[0].connected && display.pipe_enabled_h17p); + /* A queued callback from the old sink cannot be replayed after unplug. */ + init(); display.external_error = -EIO; + display.iomfb.presentation_paused = true; callback(true); + CHECK(display.external_hotplug_pending); + CHECK(dcp_external_disconnect(&display,true) == -EIO); + iomfb_pause_presentation(&display,false); CHECK(!sink.connected); + /* An old stop generation cannot touch a later session. */ + init(); display.external_stop_generation = 6; + dcp_external_stop_work(&display.external_stop_work); + CHECK(sink.connected && settles == 0 && powers == 0 && releases == 0); + CHECK(dcp_external_stop_locked(&display,6) == 0 && sink.connected); + /* Same-cable CRTC off remains distinct from physical absence. */ + init(); CHECK(dcp_external_disconnect(&display,false) == 0); + CHECK(display.external_cable && display.typec_cable_connected); + CHECK(display.external_generation == 7); + /* Genuine discovery and the paused DPMS generation gate still work. */ + init(); sink.connected = false; callback(true); CHECK(sink.connected); + init(); sink.connected = false; display.iomfb.presentation_paused = true; + display.external_power_attempted = false; callback(true); CHECK(sink.connected); + init(); sink.connected = false; display.iomfb.presentation_paused = true; + display.external_stopped = true; display.external_rx_generation = 8; + callback(true); CHECK(sink.connected); + init(); sink.connected = false; display.iomfb.presentation_paused = true; + display.external_stopped = true; callback(true); CHECK(!sink.connected); + init(); sink.connected = false; display.iomfb.presentation_paused = true; + display.external_error = -EIO; callback(true); CHECK(!sink.connected); + /* Fresh firmware discovery precedes OOB: no second RX is required. */ + init(); sink.connected = false; + display.external_cable = display.typec_cable_connected = false; + display.external_generation = display.external_rx_generation = 37; + display.external_power_attempted = false; display.external_stopped = true; + display.dptxport[0].connected = false; + callback(true); CHECK(sink.connected); CHECK(!display.external_physically_absent); + struct platform_device device = { .data = &display }; + CHECK(dcp_dptx_connect_oob(&device,0) == 0); + CHECK(raw_links == 1 && sink.connected); + CHECK(display.external_generation == 38); + /* Initially paused discovery keeps the original pending-replay behavior. */ + init(); sink.connected = false; display.external_cable = false; + display.external_power_attempted = false; display.iomfb.presentation_paused = true; + callback(true); CHECK(display.external_hotplug_pending); CHECK(!sink.connected); + CHECK(dcp_dptx_connect_oob(&device,0) == 0 && raw_links == 1); + iomfb_pause_presentation(&display,false); CHECK(sink.connected); + /* Physical absence consumes late RX and clears an already queued replay. */ + init(); display.iomfb.presentation_paused = true; callback(true); + CHECK(display.external_hotplug_pending); + CHECK(dcp_external_disconnect(&display,true) == 0); + CHECK(display.external_physically_absent && !display.external_hotplug_pending); + callback(true); CHECK(!sink.connected && !display.external_hotplug_pending); + /* A real next OOB connect removes the tombstone and permits discovery. */ + CHECK(dcp_dptx_connect_oob(&device,0) == 0); + CHECK(!display.external_physically_absent && display.external_cable); + CHECK(display.external_generation == 9 && raw_links == 1); + callback(true); CHECK(sink.connected); + /* Same-cable DPMS does not create a physical-absence tombstone. */ + init(); CHECK(dcp_external_disconnect(&display,false) == 0); + CHECK(!display.external_physically_absent && display.external_cable); + sink.connected = false; callback(true); CHECK(!sink.connected); + CHECK(display.external_hotplug_pending); + iomfb_pause_presentation(&display,false); CHECK(sink.connected); + /* A failed hardware stop remains owned, even after a real OOB reconnect. */ + init(); display.external_error = -EIO; + CHECK(dcp_external_disconnect(&display,true) == -EIO); + CHECK(display.external_physically_absent); callback(true); CHECK(!sink.connected); + CHECK(dcp_dptx_connect_oob(&device,0) == -EIO); + CHECK(!display.external_physically_absent && display.external_error == -EIO); + CHECK(raw_links == 0 && display.dptxport[0].connected && display.pipe_enabled_h17p); + /* Successful physical teardown may release ownership normally. */ + init(); CHECK(dcp_external_disconnect(&display,true) == 0); + CHECK(!sink.connected && !display.dptxport[0].connected); + CHECK(display.external_stopped && display.typec_crtc_off && !display.pipe_enabled_h17p); + printf("%d assertions, %d failures\n", checks, failures); return !!failures; +} +''' + source = prefix + pm + shim + funcs + tests + return source + +def teardown(root): + + files = {n: (root / f).read_text() for n, f in { + 'atc': 'drivers/phy/apple/atc.c', + 'dp': 'drivers/phy/apple/atc-j700-dp.h', + 'phy': 'drivers/phy/phy-core.c', + 'core': 'drivers/usb/dwc3/core.c', + 'apple': 'drivers/usb/dwc3/dwc3-apple.c', + 'state': 'drivers/usb/dwc3/dwc3-apple-state.h', + }.items()} + + + def fn(n, name): + s = files[n] + normalized = n == 'atc' and name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in s + if normalized: + name = 'atcphy_mux_set_locked' + m = re.search(r'^(?:static [^\n]*\b|(?:int|void) )' + name + r'\(', s, re.M) + assert m, (n, name) + result = block(s, m.start()) + return result.replace('atcphy_mux_set_locked(', 'atcphy_mux_set(', 1) if normalized else result + + decls = [] + for name in ('atcphy_generation', 'atcphy_lane_mode', 'atcphy_pipehandler_state', 'atcphy_mode', 'atcphy_dp_link_rate'): + decls.append(block(files['atc'], files['atc'].index('enum ' + name + ' {')) + ';') + for name in ('atcphy_mode_configuration', 'atcphy_hw', 'atcphy_dp_link_rate_configuration'): + decls.append(block(files['atc'], files['atc'].index('struct ' + name + ' {')) + ';') + decls.append(block(files['state'], files['state'].index('enum dwc3_apple_state {')) + ';') + start = files['atc'].index('static const struct {\n\tconst struct atcphy_mode_configuration normal;') + end = files['atc'].index('\n};', start) + 3 + decls.append(files['atc'][start:end]) + start = files['atc'].index('static const struct atcphy_dp_link_rate_configuration dp_lr_config[]') + end = files['atc'].index('\n};', start) + 3 + decls.append(files['atc'][start:end]) + + names = { + 'atc': ('atcphy_pipehandler_lock', 'atcphy_pipehandler_unlock', + 'atcphy_pipehandler_check', 'atcphy_pipehandler_set_mux', + 'atcphy_clear_nonselected_phy_reset', + 'atcphy_configure_pipehandler_usb3_host_t8103', + 'atcphy_configure_pipehandler_usb3_host_t8122', + 'atcphy_configure_pipehandler_usb3', 'atcphy_configure_pipehandler_usb4', + 'atcphy_enable_dummy_phy', 'atcphy_configure_pipehandler_dummy', + 'atcphy_configure_pipehandler', 'atcphy_park_pipehandler', + 'atcphy_get_mode_config', 'atcphy_dp_configure', + 'j700_field_update', 'j700_aux_disable', 'atcphy_disable_dp_aux', + 'atcphy_power_off', 'atcphy_power_on', 'atcphy_configure', 'atcphy_usb2_init', 'atcphy_usb3_power_off', + 'atcphy_usb3_set_mode', 'atcphy_usb2_power_off', + '_atcphy_dwc3_reset_assert', 'atcphy_dwc3_reset_assert', 'atcphy_mux_set'), + 'phy': ('phy_init', 'phy_exit', 'phy_power_on', 'phy_power_off'), + 'core': ('dwc3_phy_power_on', 'dwc3_phy_power_off', 'dwc3_phy_exit', 'dwc3_core_exit'), + 'apple': ('dwc3_apple_core_stop', 'dwc3_apple_core_init', 'dwc3_apple_core_start', + 'dwc3_apple_init', 'dwc3_apple_exit', 'dwc3_usb_role_switch_set'), + } + names['core'] = ('dwc3_phy_power_on', 'dwc3_phy_power_off', 'dwc3_phy_exit', + 'dwc3_core_exit_after_phy_power_off', 'dwc3_core_exit', + 'dwc3_core_remove_common', 'dwc3_core_remove', 'dwc3_core_remove_no_shutdown') + names['atc'] += ('atcphy_dwc3_reset_deassert',) + names['atc'] = tuple(name for name in names['atc'] if name not in ('atcphy_get_mode_config', 'j700_field_update')) + names = {n: tuple(name for name in group if re.search(r'^(?:static [^\n]*\b|(?:int|void) )' + name + r'\(', files[n], re.M)) + for n, group in names.items()} + context = (fn('atc', 'core_read32') + '\n' + fn('atc', 'atcphy_get_mode_config') + '\n' + fn('atc', 'j700_field_update') + '\n' + files['dp'][files['dp'].index('/* Masks are the union'):files['dp'].index('/* Retry only the idle operations')]) + functions = '\n\n'.join(fn(n, name) for n, group in names.items() for name in group) + definitions = [] + lines = iter(files['atc'].splitlines()) + for line in lines: + if not line.startswith('#define '): + continue + definition = [line] + while definition[-1].endswith('\\'): + definition.append(next(lines)) + definitions.append('\n'.join(definition)) + definitions = '\n'.join(definitions) + here = Path(__file__).resolve().parent + fallback = '' if 'dwc3_core_remove_no_shutdown' in names['core'] else '#define dwc3_core_remove_no_shutdown dwc3_core_remove\n' + unit = ((here / 'dwc3/prefix.c').read_text() + '\n' + definitions + '\n' + + '\n'.join(decls) + '\n' + (here / 'dwc3/mocks.c').read_text() + '\n' + + context + '\n' + functions + '\n' + fallback + (here / 'dwc3/cases.c').read_text()) + return unit + +def profile(root): + + base = root / "drivers/gpu/drm/apple" + source = (base / "dcp.c").read_text() + + + def block(marker): + start = source.index(marker) + opening = source.index("{", start) + end, depth = opening + 1, 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + + + clock = source[source.index("static const struct resource t8140_firmware_clock"): + source.index("static const struct apple_dcp_hw_data apple_dcp_hw_t8140 =")] + profiles = clock + "\n" + "\n".join(block(marker) + ";" for marker in ( + "static const struct apple_dcp_hw_data apple_dcp_hw_t8140 =", + "static const struct apple_dcp_hw_data apple_dcp_hw_t8140_external =", + )) + admission = block("static int dcp_get_disp_regs(") + headers = "\n".join(line for name in ("dcp-external-bw.h", "dcp-link.h") + for line in (base / name).read_text().splitlines() + if not line.startswith("#include")) + prefix = r''' +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; typedef uint64_t u64; +typedef uint32_t __le32; typedef uint64_t __le64; +#define __packed __attribute__((packed)) +#undef static_assert +#define static_assert(condition, ...) _Static_assert(condition, "wire layout") +#define le32_to_cpu(v) (v) +#define cpu_to_le32(v) (v) +#define cpu_to_le64(v) (v) +#define IORESOURCE_MEM 1 +#define MAX_DISP_REGISTERS 7 +#define DCP_FIRMWARE_H17P 3 +#define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 +#define DEFINE_RES_MEM(s,n) { .start=(s), .end=(s)+(n)-1 } +#define resource_size(r) ((r)->end-(r)->start+1) +struct resource { u64 start,end; }; +struct device_node { bool named; unsigned count; }; +struct device { struct device_node *of_node; }; +struct platform_device { struct device dev; int num_resources; struct resource resources[10]; }; +#define to_platform_device(d) ((struct platform_device *)(d)) +struct apple_dcp_hw_data { + unsigned num_dptx_ports; bool adopt_live_session; + int iomfb_method_profile,firmware_compat; + const struct resource *firmware_clock; + u32 firmware_display_windows,firmware_scratch,firmware_request; +}; +struct apple_dcp { + struct device *dev; struct apple_dcp_hw_data hw; + struct resource *disp_registers[10],disp_bw_scratch_res,disp_bw_doorbell_res; + u32 disp_bw_scratch_index,disp_bw_scratch_offset,disp_bw_doorbell_index,nr_disp_registers; +}; +static bool of_property_present(struct device_node *n,const char *name) +{ return !strcmp(name,"reg-names") && n->named; } +static int of_property_count_strings(struct device_node *n,const char *name) +{ assert(!strcmp(name,"reg-names"));return n->count+1; } +static int of_property_read_string_index(struct device_node *n,const char *name,int i,const char **v) +{ + static const char *names[]={"coproc","disp-0","disp-1","disp-2","disp-3","disp-4","disp-5"}; + assert(!strcmp(name,"reg-names"));if(i<0 || (unsigned)i>n->count)return -EINVAL; + *v=names[i];return 0; +} +static struct resource *platform_get_resource(struct platform_device *p,int type,int i) +{ assert(type==IORESOURCE_MEM);return i>=1 && inum_resources?&p->resources[i-1]:NULL; } +static struct resource *platform_get_resource_byname(struct platform_device *p,int type,const char *name) +{ int i=-1;assert(sscanf(name,"disp-%d",&i)==1);return platform_get_resource(p,type,i+1); } +static int dcp_get_bw_scratch_reg(struct apple_dcp *p,u32 count) { (void)p;(void)count;return -ENOENT; } +static int dcp_get_bw_doorbell_reg(struct apple_dcp *p,u32 count) { (void)p;(void)count;return -ENOENT; } +''' + tests = r''' +static struct platform_device provider; +static struct device_node node; +static struct apple_dcp d; +static void reset(bool external,bool named) +{ + node=(struct device_node){.named=named,.count=5}; + provider=(struct platform_device){.dev.of_node=&node,.num_resources=6}; + for(unsigned i=0;i<5;i++)provider.resources[i]=(struct resource)DEFINE_RES_MEM(0x1000ULL*i,0x1000); + provider.resources[4]=t8140_firmware_clock; + d=(struct apple_dcp){.dev=&provider.dev,.hw=external?apple_dcp_hw_t8140_external:apple_dcp_hw_t8140}; +} +int main(void) +{ + const u32 fw[]={26,6,2},unknown[]={0,0,0}; + assert(apple_dcp_external_bw_valid(0xbc000,0x28000,0x70000)); + assert(!apple_dcp_external_bw_valid(0xbc000,0x28002,0x70000)); + struct apple_dcp_h17p_clock_request clock={.clock_id=APPLE_DCP_H17P_CLOCK_ID_154};u64 rate; + assert(!apple_dcp_h17p_clock_rate(&clock,123,456,&rate) && rate==123); + assert(apple_dcp_external_firmware_supported(fw,3,fw,3)); + assert(!apple_dcp_external_firmware_supported(NULL,3,fw,3)); + assert(!apple_dcp_external_firmware_supported(fw,3,NULL,3)); + assert(!apple_dcp_external_firmware_supported(unknown,3,fw,3)); + assert(apple_dcp_hw_t8140.num_dptx_ports==0 && apple_dcp_hw_t8140.adopt_live_session); + assert(apple_dcp_hw_t8140.firmware_display_windows==5); + assert(apple_dcp_hw_t8140.firmware_scratch==0x20000 && apple_dcp_hw_t8140.firmware_request==0x68000); + assert(apple_dcp_hw_t8140_external.num_dptx_ports==1 && !apple_dcp_hw_t8140_external.adopt_live_session); + assert(apple_dcp_hw_t8140_external.firmware_display_windows==4); + assert(apple_dcp_hw_t8140_external.firmware_scratch==0x28000 && apple_dcp_hw_t8140_external.firmware_request==0x70000); + for(unsigned named=0;named<2;named++) { + reset(true,named);assert(dcp_get_disp_regs(&d)==0); + assert(d.nr_disp_registers==5 && d.disp_bw_scratch_index==4); + assert(d.disp_bw_scratch_res.start+d.disp_bw_scratch_offset==0x302828000ULL); + assert(d.disp_bw_doorbell_res.start==0x302870000ULL && resource_size(&d.disp_bw_doorbell_res)==4); + struct apple_dcp_h17p_rt_bw_request request={0};struct apple_dcp_h17p_rt_bw_reply reply; + assert(!apple_dcp_h17p_rt_bw_encode_reply(&request,&reply,d.disp_bw_scratch_res.start+d.disp_bw_scratch_offset,d.disp_bw_doorbell_res.start)); + assert(reply.config.scratch==0x302828000ULL && reply.config.clock_request==0x302870000ULL && !reply.status); + request.config_null=1;assert(apple_dcp_h17p_rt_bw_encode_reply(&request,&reply,0,0)==-EINVAL); + reset(false,named);assert(dcp_get_disp_regs(&d)==0 && d.nr_disp_registers==6); + assert(d.disp_bw_scratch_offset==0x20000 && d.disp_bw_doorbell_res.start==0x302868000ULL); + reset(true,named);provider.resources[4].start+=4;assert(dcp_get_disp_regs(&d)==-EINVAL); + reset(true,named);provider.resources[4].end-=4;assert(dcp_get_disp_regs(&d)==-EINVAL); + reset(true,named);node.count=4;provider.num_resources=5;assert(dcp_get_disp_regs(&d)==-EINVAL); + reset(true,named);d.hw.firmware_scratch=0xbc000;assert(dcp_get_disp_regs(&d)==-EINVAL); + reset(true,named);d.hw.firmware_request=0xbc000;assert(dcp_get_disp_regs(&d)==-EINVAL); + reset(true,named);d.hw.firmware_display_windows=0;assert(dcp_get_disp_regs(&d)==-EINVAL); + } + puts("PASS: actual internal/external profiles, resource admission and D003 bandwidth reply"); +} +''' + return prefix + headers + "\n" + profiles + admission + tests + +def firmware(root): + + source = (root / "drivers/gpu/drm/apple/iomfb_h17p.c").read_text() + body = source[source.index("struct dcp_h17p_provider_request {"): + source.index("static bool trampoline_luma_span_h17p")] + prefix = r''' +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint32_t u32; +typedef uint32_t __le32; +#define __packed __attribute__((packed)) +#undef static_assert +#define static_assert(cond) _Static_assert(cond, "") +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#define sizeof_field(t, f) sizeof(((t *)0)->f) +#define le32_to_cpu(x) (x) +#define cpu_to_le32(x) (x) +#define WRITE_ONCE(x, v) ((x) = (v)) +#define DCP_IOMFB_METHODS_H17G 1 +#define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 +#define trace_iomfb_callback(d, t, n) ((void)(d), (void)(t)) +#define dev_err(...) ((void)0) +#define dev_warn_once(...) ((void)0) +struct device_node { bool internal, external, present; int length; }; +struct device { struct device_node *of_node; }; +struct apple_dcp { struct { int iomfb_method_profile; } hw; struct device *dev; bool crashed; }; +static bool j700 = true; +static bool of_machine_is_compatible(const char *name) +{ + assert(!strcmp(name, "apple,j700")); + return j700; +} +static bool of_device_is_compatible(struct device_node *n, const char *name) +{ + return (!strcmp(name, "apple,t8140-dcp") && n->internal) || + (!strcmp(name, "apple,t8140-dcpext") && n->external); +} +static bool of_property_present(struct device_node *n, const char *name) +{ + (void)name; + return n->present; +} +static int of_property_count_u8_elems(struct device_node *n, const char *name) +{ + (void)name; + return n->present ? n->length : -EINVAL; +} +static int of_property_read_u8_array(struct device_node *n, const char *name, void *p, unsigned int len) +{ + (void)n; (void)name; + memset(p, 0x5a, len); + return 0; +} +static const void *memchr_inv(const void *p, int value, size_t size) +{ + const u8 *b = p; + for (size_t i = 0; i < size; i++) + if (b[i] != value) + return b + i; + return NULL; +} +''' + suffix = r''' +int main(void) +{ + struct device_node node = { .external = true }; + struct device dev = { .of_node = &node }; + struct apple_dcp dcp = { .dev = &dev, .hw.iomfb_method_profile = 2 }; + struct dcp_h17p_provider_request req = { .scope = { 'V', 'O', 'R', 'P' }, .capacity = 0xc00 }; + struct dcp_h17p_provider_reply reply; + unsigned int cases = 0; + for (unsigned int i = 0; i < ARRAY_SIZE(provider_properties); i++) { + memset(&reply, 0xff, sizeof(reply)); + strcpy(req.name, provider_properties[i].name); + assert(trampoline_provider_property_h17p(&dcp, 400, &reply, &req)); + assert(!dcp.crashed && reply.length == provider_properties[i].length); + if (reply.length) + assert(!memcmp(reply.data, provider_properties[i].legacy_data, reply.length)); + cases++; + } + strcpy(req.name, "power-lut-data-x"); + j700 = false; + assert(!trampoline_provider_property_h17p(&dcp, 400, &reply, &req) && dcp.crashed); + j700 = true; dcp.crashed = false; dcp.hw.iomfb_method_profile = 3; + assert(!trampoline_provider_property_h17p(&dcp, 400, &reply, &req) && dcp.crashed); + node.external = false; node.internal = true; dcp.crashed = false; + assert(trampoline_provider_property_h17p(&dcp, 400, &reply, &req) && !dcp.crashed); + node.internal = false; node.external = true; node.present = true; node.length = 3; + dcp.hw.iomfb_method_profile = 2; dcp.crashed = false; + assert(!trampoline_provider_property_h17p(&dcp, 400, &reply, &req) && dcp.crashed); + node.length = 4; dcp.crashed = false; + assert(trampoline_provider_property_h17p(&dcp, 400, &reply, &req) && !dcp.crashed); + assert(reply.length == 4 && reply.data[0] == 0x5a); + strcpy(req.name, "unqualified-property"); + assert(!trampoline_provider_property_h17p(&dcp, 400, &reply, &req)); + printf("PASS %u provider payloads and scope/length rejection cases\n", cases); + return 0; +} +''' + return prefix + body + suffix + +def readiness(root): + + dcp = (root / 'drivers/gpu/drm/apple/dcp.c').read_text() + drm = (root / 'drivers/gpu/drm/apple/apple_drv.c').read_text() + + + def function(source, marker): + start = source.index(marker) + opening = source.index('{', start) + end, depth = opening + 1, 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + + # This slice is the production probe's publication boundary, including its + # failure exits and prepared-before-component_add ordering, not a copied model. + probe = function(dcp, 'static int dcp_platform_probe(') + publish = probe[probe.rindex('\tif (dcp->hw.iomfb_method_profile ==\n\t DCP_IOMFB_METHODS_H17P_EXTERNAL) {'):] + production = '\n'.join(( + function(dcp, 'bool dcp_h17p_external_supported('), + function(dcp, 'bool dcp_h17p_external_ready('), + function(drm, 'static bool apple_dcp_usable('), + function(drm, 'static int add_dcp_components('), + function(drm, 'static int apple_drm_init_dcp('), + 'static int publish_provider(struct platform_device *pdev, struct apple_dcp *dcp)\n' + '{\n\tstruct device *dev = &pdev->dev;\n\tint ret;\n' + publish, + )) + prefix = r''' +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint64_t u64; +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x, v) ((x) = (v)) +#define IS_ENABLED(x) 0 +#define MAX_COPROCESSORS 4 +#define DL_FLAG_AUTOREMOVE_SUPPLIER 0 +#define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 +#define dev_warn(...) ((void)0) +#define msleep(x) ((void)(x)) +#define component_compare_of 0 +#define apple_dcp_id_tbl 0 +static int dcp_comp_ops; +struct device_node { bool available, external, phys; unsigned int index; u32 firmware[3], compat[3]; int firmware_cells, compat_cells; }; +struct device { struct device_node *of_node; void *data; }; +struct platform_device { struct device dev; bool bound, registered; }; +struct apple_dcp { + bool external_prepared, quiescing, crashed; + struct { int iomfb_method_profile; } hw; +}; +struct drm_device { int unused; }; +struct drm_encoder { int unused; }; +struct apple_drm_private { struct drm_device drm; }; +struct component_match { bool includes[2]; }; +static struct device_node nodes[2]; +static struct platform_device devices[2]; +static struct apple_dcp dcps[2]; +static struct apple_drm_private priv; +static struct device master = { .data = &priv }; +static struct component_match matches; +static bool machine_supported, absent_external; +static unsigned int references, released, node_puts, probes[2], bind_attempts, disables; +static int preflight_error, external_probe_error, bind_result; +static void try_bind(void); +static bool of_machine_is_compatible(const char *s) +{ assert(!strcmp(s,"apple,j700")); return machine_supported; } +static int of_property_read_variable_u32_array(struct device_node *n,const char *s,u32 *v,int min,int max) +{ + assert(min==3 && max==3); + bool fw=!strcmp(s,"apple,firmware-version"); + assert(fw || !strcmp(s,"apple,firmware-compat")); + int count=fw?n->firmware_cells:n->compat_cells; + if(count!=3) return -EINVAL; + memcpy(v,fw?n->firmware:n->compat,3*sizeof(*v)); return count; +} +static bool of_device_is_available(struct device_node *n) { return n->available; } +static bool of_device_is_compatible(struct device_node *n, const char *s) +{ + return n->external && (!strcmp(s, "apple,t8140-dcpext") || !strcmp(s, "apple,dcpext")); +} +static bool of_property_present(struct device_node *n, const char *s) +{ assert(!strcmp(s, "phys")); return n->phys; } +#define for_each_matching_node(np, table) \ + for (unsigned int node_i = 0; node_i < 2 && ((np) = &nodes[node_i]); node_i++) +#define for_each_endpoint_of_node(np, endpoint) \ + for ((endpoint) = NULL; (endpoint) != NULL; (endpoint) = NULL) +static struct device_node *of_graph_get_remote_port_parent(struct device_node *n) +{ return n; } +static void of_node_put(struct device_node *n) { (void)n; node_puts++; } +static struct platform_device *of_find_device_by_node(struct device_node *n) +{ + if (n->external && absent_external) + return NULL; + references++; + return &devices[n->index]; +} +static void *platform_get_drvdata(struct platform_device *p) { return p->dev.data; } +static void *dev_get_drvdata(struct device *d) { return d->data; } +static void put_device(struct device *d) { (void)d; released++; } +/* Deliberately provided: using this during synchronous bind is a regression. */ +#define device_is_bound(dev) (devices[(dev)->of_node->index].bound) +static void drm_of_component_match_add(struct device *d, struct component_match **m, + int compare, struct device_node *n) +{ (void)d; (void)compare; *m = &matches; matches.includes[n->index] = true; } +static void device_link_add(struct device *a, struct device *b, int flags) +{ (void)a; (void)b; (void)flags; } +static int apple_probe_per_dcp(struct device *d, struct drm_device *drm, + struct platform_device *p, int index, bool external, + struct drm_encoder **encoder, u32 *mask) +{ + (void)d; (void)drm; (void)index; (void)encoder; (void)mask; + assert(external == p->dev.of_node->external); + probes[p->dev.of_node->index]++; + return external ? external_probe_error : 0; +} +static int apple_probe_typec_ports(struct drm_device *d, struct platform_device **p, + u32 *masks, int count) +{ (void)d; (void)p; (void)masks; (void)count; return 0; } +static int dcp_start(struct platform_device *p) { (void)p; return 0; } +static int dcp_wait_ready(struct platform_device *p, u64 timeout) +{ (void)p; (void)timeout; return 0; } +static u64 get_jiffies_64(void) { return 0; } +static u64 msecs_to_jiffies(unsigned int ms) { return ms; } +static bool time_after_eq64(u64 a, u64 b) { return a >= b; } +static int dcp_comp_bind(struct device *d, void *master_data, void *data) +{ (void)d; (void)master_data; (void)data; return preflight_error; } +static void dcp_disable_typec_work(struct apple_dcp *d, bool quiesce) +{ disables++; d->quiescing = quiesce; } +static int component_add(struct device *d, const void *ops) +{ + (void)ops; + struct platform_device *p = &devices[d->of_node->index]; + assert(!p->bound); + p->registered = true; + try_bind(); /* Linux component_add may bind the master before probe returns. */ + return 0; +} +''' + suffix = r''' +static void try_bind(void) +{ + for (unsigned int i = 0; i < 2; i++) + if (matches.includes[i] && !devices[i].registered) + return; + bind_attempts++; + bind_result = apple_drm_init_dcp(&master); +} +static void initial(bool supported_firmware) +{ + memset(nodes, 0, sizeof(nodes)); + memset(devices, 0, sizeof(devices)); + memset(dcps, 0, sizeof(dcps)); + memset(&matches, 0, sizeof(matches)); + memset(probes, 0, sizeof(probes)); + for (unsigned int i = 0; i < 2; i++) { + nodes[i] = (struct device_node){ .available = true, .external = i == 1, .index = i }; + devices[i].dev = (struct device){ .of_node = &nodes[i], .data = &dcps[i] }; + } + dcps[1].hw.iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL; + devices[0].registered = devices[0].bound = true; + machine_supported = true; + nodes[1].firmware_cells=nodes[1].compat_cells=3; + if(supported_firmware) { + u32 version[]={26,6,2}; + memcpy(nodes[1].firmware,version,sizeof(version)); + memcpy(nodes[1].compat,version,sizeof(version)); + } + absent_external = false; + references = released = node_puts = bind_attempts = disables = 0; + preflight_error = external_probe_error = bind_result = 0; +} +static int match_providers(void) +{ + struct component_match *match = NULL; + return add_dcp_components(&master, &match); +} +int main(void) +{ + assert(apple_dcp_external_bw_valid(0xbc000,0x28000,0x70000)); + assert(apple_dcp_external_clock_window(0x302800000ULL,0xbc000,0x302800000ULL,0xbc000)); + /* No command line state: exact J700 firmware selects the external profile. */ + initial(true); assert(dcp_h17p_external_supported(&nodes[1])); + machine_supported=false; + assert(!dcp_h17p_external_supported(&nodes[1]) && match_providers()==1); + for(unsigned i=0;i<3;i++) { + initial(true); nodes[1].firmware[i]++; + assert(!dcp_h17p_external_supported(&nodes[1]) && match_providers()==1); + initial(true); nodes[1].compat[i]++; + assert(!dcp_h17p_external_supported(&nodes[1]) && match_providers()==1); + } + for(int count=-1;count<=5;count++) if(count!=3) { + initial(true); nodes[1].firmware_cells=count; + assert(!dcp_h17p_external_supported(&nodes[1])); + initial(true); nodes[1].compat_cells=count; + assert(!dcp_h17p_external_supported(&nodes[1])); + } + + /* Unknown external firmware preserves the normal internal provider. */ + initial(false); + assert(match_providers() == 1 && matches.includes[0] && !matches.includes[1]); + assert(!dcp_h17p_external_ready(&nodes[1]) && references == 0); + try_bind(); + assert(bind_attempts == 1 && bind_result == 0 && probes[0] == 1 && !probes[1]); + + /* Delayed external provider remains in the match; the master must wait. */ + initial(true); + assert(match_providers() == 2 && matches.includes[1]); + try_bind(); + assert(bind_attempts == 0 && !probes[0] && !probes[1]); + /* Last provider is prepared, but driver core bound is still false here. */ + assert(publish_provider(&devices[1], &dcps[1]) == 0); + assert(dcps[1].external_prepared && !devices[1].bound); + assert(bind_attempts == 1 && bind_result == 0 && probes[0] == 1 && probes[1] == 1); + devices[1].bound = true; + + /* Permanent preflight failure cannot silently drop a requested provider. */ + initial(true); + assert(match_providers() == 2); + preflight_error = -EIO; + assert(publish_provider(&devices[1], &dcps[1]) == -EIO); + assert(!devices[1].registered && !dcps[1].external_prepared && disables == 1); + try_bind(); + assert(matches.includes[1] && bind_attempts == 0 && !probes[0]); + + /* An unavailable DT node is excluded; a matched but absent device is fatal. */ + initial(true); + nodes[1].available = false; + assert(!apple_dcp_usable(&nodes[1]) && match_providers() == 1); + initial(true); + absent_external = true; + assert(match_providers() == 2); + assert(apple_drm_init_dcp(&master) == -ENODEV && !probes[1]); + + /* Readiness rejects every unavailable provider state and releases lookup refs. */ + for (unsigned int state = 0; state < 4; state++) { + initial(true); + dcps[1].external_prepared = true; + if (state == 0) devices[1].dev.data = NULL; + if (state == 1) dcps[1].external_prepared = false; + if (state == 2) dcps[1].quiescing = true; + if (state == 3) dcps[1].crashed = true; + assert(!dcp_h17p_external_ready(&nodes[1])); + assert(references == 1 && released == 1); + assert(apple_drm_init_dcp(&master) == -ENODEV && !probes[1]); + } + initial(true); + dcps[1].external_prepared = true; + external_probe_error = -EIO; + assert(apple_drm_init_dcp(&master) == -EIO); + assert(probes[1] == 1 && node_puts == 1); + puts("PASS: automatic hardware/firmware matching, delayed/preflight-failed providers, synchronous publication, readiness rejection"); + return 0; +} +''' + + header = (root / 'drivers/gpu/drm/apple/dcp-external-bw.h').read_text().replace('#include ', '') + return prefix + header + production + suffix + +def vrr(root): + + source = (root / 'drivers/gpu/drm/apple/dcp.c').read_text() + start = source.index('int dcp_crtc_atomic_check(') + opening = source.index('{', start) + end, depth = opening + 1, 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + check = source[start:end] + prefix = r""" +#undef NDEBUG +#include +#include +#include +#include +#define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 +#define READ_ONCE(x) (x) +#define dev_dbg(...) ((void)0) +struct apple_connector { bool connected; }; +struct apple_dcp { + bool crashed; + struct { int iomfb_method_profile; } hw; + struct { bool valid; } mode_state; + struct apple_connector *connector; + void *dev; +}; +struct platform_device { struct apple_dcp *data; }; +struct drm_crtc { struct platform_device *dcp; }; +struct drm_crtc_state { bool vrr_enabled, modeset; }; +struct drm_atomic_state { struct drm_crtc_state crtc; }; +#define to_apple_crtc(x) (x) +#define platform_get_drvdata(x) ((x)->data) +static struct drm_crtc_state *drm_atomic_get_new_crtc_state( + struct drm_atomic_state *s, struct drm_crtc *c) +{ (void)c; return &s->crtc; } +static bool drm_atomic_crtc_needs_modeset(struct drm_crtc_state *s) +{ return s->modeset; } +""" + suffix = r""" +int main(void) +{ + struct apple_connector connector = { .connected = true }; + struct apple_dcp dcp = { .connector = &connector }; + struct platform_device pdev = { .data = &dcp }; + struct drm_crtc crtc = { .dcp = &pdev }; + struct drm_atomic_state state = {0}; + unsigned int cases = 0; + /* Check initial modesets, plane-only commits and link recovery states. */ + for (unsigned int profile = 0; profile < 3; profile++) + for (unsigned int valid = 0; valid < 2; valid++) + for (unsigned int modeset = 0; modeset < 2; modeset++) + for (unsigned int link = 0; link < 3; link++) + for (unsigned int vrr = 0; vrr < 2; vrr++) { + dcp.hw.iomfb_method_profile = profile; + dcp.mode_state.valid = valid; + state.crtc.modeset = modeset; + state.crtc.vrr_enabled = vrr; + dcp.connector = link == 2 ? NULL : &connector; + connector.connected = link == 1; + int expected = profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && vrr ? -EINVAL : 0; + int actual = dcp_crtc_atomic_check(&crtc, &state); + if (actual != expected) { + fprintf(stderr, "profile=%u valid=%u modeset=%u link=%u vrr=%u: got %d expected %d\n", + profile, valid, modeset, link, vrr, actual, expected); + return 1; + } + cases++; + } + dcp.crashed = true; + state.crtc.vrr_enabled = false; + assert(dcp_crtc_atomic_check(&crtc, &state) == -EINVAL); + printf("PASS: %u production atomic VRR admission cases and crashed-device rejection\n", cases); + return 0; +} +""" + return prefix + check + suffix + + +def power_reply(root): + """Actual external power callback validation with host completion/refcount mocks.""" + source = (root / 'drivers/gpu/drm/apple/iomfb_h17p.c').read_text() + production = block(source, 'struct dcp_external_power_cookie {') + ';\n' + production += block(source, 'static void iomfb_external_power_complete(') + production += block(source, 'static bool iomfb_external_reply_size(') + prefix = r""" +#include +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; typedef uint64_t u64; +#define WRITE_ONCE(p,v) ((p)=(v)) +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +struct completion { bool done; }; +struct kref { unsigned count; }; +struct dcp_wait_cookie { struct kref refcount; struct completion done; u32 status; }; +struct dcp_packet_header { char tag[4]; u32 in_len,out_len; }; +struct dcp_set_power_state_req_h17p { u64 unklong; u32 unkint; unsigned char null,padding[3]; } __attribute__((packed)); +struct dcp_set_power_state_resp { u32 unknown,ret; } __attribute__((packed)); +struct apple_dcp { int unused; }; +static unsigned released; +static u64 get_unaligned_le64(const void *p) { u64 v; memcpy(&v,p,sizeof(v)); return v; } +static void complete(struct completion *c) { assert(!c->done); c->done=true; } +static void release_wait_cookie(struct kref *r) { assert(!r->count); released++; } +static void kref_put(struct kref *r,void (*release)(struct kref *)) +{ assert(r->count); if (!--r->count) release(r); } +""" + controls = r""" +int main(void) +{ + struct apple_dcp d={}; + struct { struct dcp_packet_header h; struct dcp_set_power_state_req_h17p q; unsigned char reply[8]; } packet={}; + for(unsigned on=0;on<2;on++) { + const u64 replies[]={0,1,0x123400000000ULL}; + for(unsigned i=0;i<3;i++) { + packet.h.in_len=sizeof(packet.q);packet.h.out_len=sizeof(packet.reply);memcpy(packet.reply,&replies[i],sizeof(packet.reply)); + struct dcp_external_power_cookie c={.on=on,.wait.refcount.count=2}; + assert(iomfb_external_reply_size(&packet.reply,sizeof(packet.q),sizeof(packet.reply))); + iomfb_external_power_complete(&d,&packet.reply,&c); + assert(c.wait.done.done && c.wait.refcount.count==1); + assert((int)c.wait.status==(replies[i]==on?0:-EREMOTEIO)); + kref_put(&c.wait.refcount,release_wait_cookie); + } + } + for(unsigned fault=0;fault<3;fault++) { + packet.h.in_len=sizeof(packet.q);packet.h.out_len=sizeof(packet.reply); + void *out=&packet.reply; + if(fault==0) out=NULL; + if(fault==1) packet.h.in_len--; + if(fault==2) packet.h.out_len--; + struct dcp_external_power_cookie c={.wait.refcount.count=2}; + assert(!iomfb_external_reply_size(out,sizeof(packet.q),sizeof(packet.reply))); + iomfb_external_power_complete(&d,out,&c); + assert(c.wait.done.done && c.wait.refcount.count==1 && (int)c.wait.status==-EPROTO); + kref_put(&c.wait.refcount,release_wait_cookie); + } + assert(released==9); + puts("PASS: actual exact ON/OFF echoes, nonzero status and null/truncated reply rejection preserve callback completion and references"); +} +""" + return prefix + production + controls + + +def dispatcher(root): + source = (root / "drivers/gpu/drm/apple/iomfb.c").read_text() + start = source.index("static bool dcp_external_zero_reply(") + end = source.index("static void dcpep_handle_ack(", start) + prefix = r""" +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; +#define DCP_FIRMWARE_H17P 1 +#define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 +#define DCP_IOMFB_METHODS_H17G 3 +#define DCP_MAX_CALL_DEPTH 8 +#define IOMFB_MAX_CB 600 +#define DCP_PACKET_ALIGNMENT 64 +#define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1)) +#define dev_err(...) ((void)dev) +#define dev_info(...) ((void)dev) +#define dev_warn(...) ((void)dev) +enum dcp_context_id { DCP_CONTEXT_CB = 0, DCP_CONTEXT_ASYNC = 3 }; +struct device_node { bool external; }; +struct device { struct device_node *of_node; }; +struct dcp_packet_header { char tag[4]; u32 in_len, out_len; }; +struct dcp_channel { unsigned depth; void *output[8]; unsigned end[8]; }; +struct apple_dcp { + struct { int iomfb_method_profile; } hw; + struct device *dev; + int fw_compat; + bool crashed; + bool (**cb_handlers)(struct apple_dcp *, int, void *, void *); + unsigned long unknown_callbacks[16]; + int start_done; + struct dcp_channel channel; +}; +static bool machine; +static unsigned acks, case_tag; +static struct dcp_channel *dcp_get_channel(struct apple_dcp *d, enum dcp_context_id c) +{ (void)c; return &d->channel; } +static int dcp_parse_tag(char *p) { (void)p; return case_tag; } +static bool of_machine_is_compatible(const char *n) { (void)n; return machine; } +static bool of_device_is_compatible(struct device_node *n, const char *s) +{ (void)s; return n->external; } +static bool iomfb_check_callback_h17p(struct apple_dcp *d, int t, u32 i, u32 o) +{ (void)d; (void)t; (void)i; (void)o; return true; } +static bool iomfb_uses_queue(struct apple_dcp *d) { (void)d; return false; } +static bool test_and_set_bit(int t, unsigned long *b) { (void)t; (void)b; return false; } +static void complete(int *p) { *p = 1; } +static u8 dcp_push_depth(unsigned *d) { return (*d)++; } +static void dcp_ack(struct apple_dcp *d, enum dcp_context_id c) +{ (void)c; acks++; d->channel.depth--; } +""" + suffix = r""" +int main(void) +{ + struct device_node node; + struct device dev = { &node }; + bool (*handlers[IOMFB_MAX_CB])(struct apple_dcp *, int, void *, void *) = {}; + unsigned cases = 0; + for (unsigned scope = 0; scope < 8; scope++) + for (unsigned tag = 133; tag <= 135; tag++) + for (unsigned context = 0; context < 8; context++) + for (unsigned in = 0; in <= 8; in++) + for (unsigned out = 0; out <= 8; out++) { + struct apple_dcp d = { + .dev = &dev, .fw_compat = 1, + .cb_handlers = handlers, .hw.iomfb_method_profile = 2, + }; + union { uint64_t align; unsigned char data[64]; } packet; + struct dcp_packet_header *h = (void *)packet.data; + memset(&packet, 0xa5, sizeof(packet)); + h->in_len = in; h->out_len = out; + machine = scope & 1; node.external = scope & 2; + if (!(scope & 4)) + d.hw.iomfb_method_profile = 0; + case_tag = tag; acks = 0; + dcpep_handle_cb(&d, context, h, sizeof(*h) + in + out, 0); + bool accepted = scope == 7 && tag == 134 && context == 3 && !in && out == 4; + if (scope & 4) { + assert(acks == accepted && d.crashed != accepted); + for (unsigned i = 0; i < out; i++) + assert(packet.data[sizeof(*h) + in + i] == (accepted ? 0 : 0xa5)); + } else { + assert(acks == 1 && !d.crashed); /* Existing legacy behavior. */ + } + assert(packet.data[sizeof(*h) + in + out] == 0xa5); + cases++; + } + return cases != 15552; +} +""" + + return prefix + source[start:end] + suffix diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py new file mode 100644 index 0000000000000..fbb1951c0f0b1 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py @@ -0,0 +1,2399 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Actual PHY parking, failure/retry, masked context and OFF controls.""" +from pathlib import Path +import re +from common import block, source_defines + +def parking(root): + + atc = (root / 'drivers/phy/apple/atc.c').read_text() + dp = (root / 'drivers/phy/apple/atc-j700-dp.h').read_text() + + + def function(source, name): + if name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in source: + marker = re.search(r'^static [^\n]+\batcphy_mux_set_locked\(', source, re.M).group() + return block(source, marker).replace('atcphy_mux_set_locked(', 'atcphy_mux_set(', 1) + return block(source, re.search(r'^static [^\n]+\b' + name + r'\(', source, re.M).group()) + + + # Use the production mode table and wrappers too, so SAFE mapping and fast paths + # are tested rather than mirrored by a second implementation. + parts = source_defines(atc) + for name in ('atcphy_generation', 'atcphy_lane_mode', 'atcphy_pipehandler_state', 'atcphy_mode', + 'atcphy_dp_link_rate'): + parts.append(block(atc, 'enum ' + name + ' {') + ';') + for name in ('atcphy_mode_configuration', 'atcphy_hw'): + parts.append(block(atc, 'struct ' + name + ' {') + ';') + parts.append(r''' +struct apple_atcphy { + const struct atcphy_hw *hw; + struct { unsigned char *core, *lpdptx, *pipehandler, *usb2phy; } regs; + struct reset_controller_dev rcdev; + void *dev; + int lock, dp_link_rate; + bool fixed_usb2, host_active, swap_lanes, ss_tunables, dp_tunables; + bool aux_power_requested, mode_dirty, dp_spread_spectrum, dp_presets_valid[5]; + u32 dp_presets[5][16]; + unsigned int dp_idle_pmas, dp_owned_pmas; + unsigned int dp_usb_context_valid, dp_usb_context_dirty; + u32 dp_usb_context[2][27]; + bool pipe_pending, usb_off_pending, usb_off_drained; + enum atcphy_mode mode, typec_mode; + enum atcphy_pipehandler_state pipe_state; +}; +''') + start = atc.index('static const struct {\n\tconst struct atcphy_mode_configuration normal;') + parts.append(atc[start:atc.index('\n};', start) + 3]) + parts.append(r''' +struct phy_configure_opts_dp { + unsigned int link_rate, lanes, voltage[4], pre[4]; + bool set_rate, set_lanes, set_voltages, ssc; +}; +static u32 registers[0x30000 / 4]; +static u32 pipe_regs[0x1000 / 4], usb2_regs[0x1000 / 4]; +static unsigned int polls, fail_poll, writes, usb2_on, usb2_writes, reset_writes, mux_writes; +static bool stuck_ack; +static bool hold_unlock_ack; +static const char *poll_caller[256], *fail_caller; +static bool bist_status_stuck; +static unsigned int bist_faults; +static bool require_parked, expected_j700_swap; +static u32 readl(void *address) { return *(u32 *)address; } +static void core_mask32(struct apple_atcphy *p, u32 offset, u32 mask, u32 value) +{ + (void)p; + assert(offset < sizeof(registers) && offset % 4 == 0); + if (require_parked) { + assert(p->pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); + assert(FIELD_GET(p->hw->pipe_mux_data, pipe_regs[PIPEHANDLER_MUX_CTRL / 4]) == + PIPEHANDLER_MUX_CTRL_DATA_DUMMY); + assert(FIELD_GET(p->hw->pipe_mux_clk, pipe_regs[PIPEHANDLER_MUX_CTRL / 4]) == + PIPEHANDLER_MUX_CTRL_CLK_DUMMY); + } + registers[offset / 4] = (registers[offset / 4] & ~mask) | value; + writes++; + if (offset == ATCPHY_POWER_CTRL) + registers[ATCPHY_POWER_STAT / 4] = registers[offset / 4]; + if (offset == 0x2000) { + if (value == 0x10000001) { + registers[offset / 4] |= BIT(1); + registers[0x7034 / 4] |= BIT(3); + } else { + registers[offset / 4] &= ~BIT(1); + registers[0x7034 / 4] &= ~BIT(3); + } + } +} +static void core_set32(struct apple_atcphy *p, u32 o, u32 v) { core_mask32(p, o, 0, v); } +static void core_clear32(struct apple_atcphy *p, u32 o, u32 v) { core_mask32(p, o, v, 0); } +static void mask32(void *p, u32 m, u32 v) +{ + *(u32 *)p = (*(u32 *)p & ~m) | v; + if (p == (void *)&pipe_regs[PIPEHANDLER_LOCK_REQ / 4] && !stuck_ack) + pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = hold_unlock_ack && + !(*(u32 *)p & PIPEHANDLER_LOCK_EN) ? PIPEHANDLER_LOCK_EN : + *(u32 *)p & PIPEHANDLER_LOCK_EN; + if (p == (void *)&pipe_regs[PIPEHANDLER_AON_GEN / 4]) reset_writes++; + if (p == (void *)&pipe_regs[PIPEHANDLER_MUX_CTRL / 4]) mux_writes++; + if ((uintptr_t)p >= (uintptr_t)usb2_regs && + (uintptr_t)p < (uintptr_t)usb2_regs + sizeof(usb2_regs)) usb2_writes++; +} +static void writel(u32 value, void *address) { mask32(address, ~0U, value); } +static void set32(void *p, u32 v) { mask32(p, 0, v); } +static void clear32(void *p, u32 v) { mask32(p, v, 0); } +#define readl_poll_timeout_atomic(address, value, condition, interval, timeout) \ + ({ (value) = readl(address); assert(polls < ARRAY_SIZE(poll_caller)); \ + poll_caller[polls++] = __func__; \ + bool bist_fault = bist_status_stuck && (void *)(address) == \ + (void *)®isters[ACIOPHY_BIST_MAC_USBPHY_STATUS2 / 4]; \ + bist_faults += bist_fault; \ + polls == fail_poll || bist_fault || \ + (fail_caller && !strcmp(fail_caller, __func__)) || !(condition) ? -ETIMEDOUT : 0; }) +#define readl_poll_timeout readl_poll_timeout_atomic +/* Observation is read-only and separately tested; retain production caller control flow. */ +static void atcphy_usb2_power_on(struct apple_atcphy *p) { (void)p; usb2_on++; } +static void atcphy_apply_tunables(struct apple_atcphy *p, enum atcphy_mode mode) +{ (void)p; (void)mode; } +static void atc_tunnel_stop_t8103(struct apple_atcphy *p) { (void)p; } +static void atc_tunnel_restore(struct apple_atcphy *p) { (void)p; } +''') + for name in ('atcphy_pipehandler_lock', 'atcphy_pipehandler_unlock', 'atcphy_pipehandler_check', + 'atcphy_pipehandler_set_mux', 'atcphy_clear_nonselected_phy_reset', + 'atcphy_configure_pipehandler_usb3_host_t8122', + 'atcphy_configure_pipehandler_usb3_host_t8103', + 'atcphy_configure_pipehandler_usb3', 'atcphy_configure_pipehandler_usb4', + 'atcphy_enable_dummy_phy', 'atcphy_configure_pipehandler_dummy', + 'atcphy_configure_pipehandler', 'atcphy_park_pipehandler', + 'atcphy_usb2_power_off', '_atcphy_dwc3_reset_assert', 'atcphy_dwc3_reset_assert'): + parts.append(function(atc, name)) + for name in ('core_read32', 'atcphy_get_mode_config', 'j700_field_update', 'j700_pma_idle'): + parts.append(function(atc, name)) + parts.append(dp) + for name in ('atcphy_configure_lanes', 'j700_aux_enable', 'j700_aux_disable', + 'atcphy_enable_dp_aux', 'atcphy_disable_dp_aux', 'atcphy_power_off', + 'atcphy_power_on', 'atcphy_configure', 'atcphy_sw_set', 'atcphy_mux_set', + 'atcphy_usb3_set_mode', 'atcphy_usb3_power_off'): + parts.append(function(atc, name)) + production = '\n'.join(parts) + if 'static int j700_usb_capture(' in dp: + production = '#define HAS_USB_CONTEXT\n' + production + prefix = r''' +#include +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint16_t u16; +#define BIT(n) (1U << (n)) +#define GENMASK(h, l) ((~0U << (l)) & (~0U >> (31 - (h)))) +#define FIELD_PREP(m, v) (((u32)(v) << __builtin_ctz(m)) & (m)) +#define FIELD_GET(m, v) (((v) & (m)) >> __builtin_ctz(m)) +#define field_prep FIELD_PREP +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#define hweight32(v) __builtin_popcount(v) +#define dev_info_once(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define dev_warn(...) ((void)0) +static void report(void *dev, const char *fmt, ...) { (void)dev; (void)fmt; } +#define dev_err report +#define lockdep_assert_held(...) ((void)0) +#define WARN_ON_ONCE(v) assert(!(v)) +#define guard(kind) (void) +#define fallthrough __attribute__((fallthrough)) +#define ndelay(n) ((void)(n)) +#define udelay(n) ((void)(n)) +#define usleep_range(a, b) ((void)(a), (void)(b)) +enum { TYPEC_STATE_SAFE, TYPEC_STATE_USB, TYPEC_MODE_USB4, + TYPEC_DP_STATE_C, TYPEC_DP_STATE_D, TYPEC_DP_STATE_E }; +enum { USB_TYPEC_TBT_SID = 0x8087, USB_TYPEC_DP_SID = 0xff01 }; +#define EUDO_USB_MODE_MASK 3U +enum { EUDO_USB_MODE_USB2, EUDO_USB_MODE_USB3, EUDO_USB_MODE_USB4 }; +enum typec_orientation { TYPEC_ORIENTATION_NONE, TYPEC_ORIENTATION_NORMAL, + TYPEC_ORIENTATION_REVERSE }; +enum phy_mode { PHY_MODE_USB_HOST, PHY_MODE_USB_DEVICE }; +struct enter_usb_data { u32 eudo; }; +struct typec_altmode { unsigned int svid; }; +struct typec_mux_state { unsigned long mode; struct typec_altmode *alt; void *data; }; +struct typec_mux_dev { struct apple_atcphy *data; }; +struct typec_switch_dev { struct apple_atcphy *data; }; +#define typec_mux_get_drvdata(mux) ((mux)->data) +#define typec_switch_get_drvdata(sw) ((sw)->data) +struct phy { struct apple_atcphy *data; }; +#define phy_get_drvdata(phy) ((phy)->data) +struct reset_controller_dev { int unused; }; +#define container_of(p, t, m) ((t *)((char *)(p) - offsetof(t, m))) +''' + suffix = r''' +static struct atcphy_hw hw = { + .has_t8140_dp = true, .gen = ATCPHY_GENERATION_T8122, + .aciophy_lane_mode = ACIOPHY_LANE_MODE_T8122, + .aciophy_crossbar = ACIOPHY_CROSSBAR_T8122, + .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, + .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, +}; +static void cold_initial(struct apple_atcphy *p, bool fixed, bool swapped) +{ + memset(registers, 0, sizeof(registers)); + memset(pipe_regs, 0, sizeof(pipe_regs)); + memset(usb2_regs, 0, sizeof(usb2_regs)); + registers[0x200 / 4] = 0xee; /* public PMA powered state, synthetic cold fixture */ + registers[AUS_COMMON_DIG_RCAL1 / 4] = AUS_COMMON_DIG_RCAL1_ALL_CODES_DONE; + pipe_regs[PIPEHANDLER_AON_GEN / 4] = PIPEHANDLER_AON_GEN_DWC3_RESET_N; + usb2_regs[USB2PHY_USBCTL / 4] = USB2PHY_USBCTL_RUN; + *p = (struct apple_atcphy) { + .hw = &hw, .regs.core = (unsigned char *)registers, + .regs.lpdptx = (unsigned char *)registers, + .regs.pipehandler = (unsigned char *)pipe_regs, + .regs.usb2phy = (unsigned char *)usb2_regs, + .fixed_usb2 = fixed, .host_active = true, .swap_lanes = swapped, + .ss_tunables = true, .dp_tunables = true, .dp_presets_valid = { true }, + .mode = APPLE_ATCPHY_MODE_USB3_DP, .typec_mode = APPLE_ATCPHY_MODE_USB3_DP, + .pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY, .mode_dirty = false, + .dp_owned_pmas = 0, .dp_idle_pmas = 0, + .aux_power_requested = false, + }; + polls = fail_poll = writes = usb2_on = usb2_writes = reset_writes = mux_writes = 0; + stuck_ack = bist_status_stuck = false; + hold_unlock_ack = false; + fail_caller = NULL; + bist_faults = 0; + require_parked = false; +} +static void initial(struct apple_atcphy *p, bool fixed, bool dirty, bool swapped) +{ + cold_initial(p, fixed, swapped); +#ifdef HAS_USB_CONTEXT + if (fixed && hw.has_t8140_dp) { + /* Two-PMA cleanup stress setup: capture before ownership and actual DP writes. */ + assert(j700_usb_capture(p, APPLE_ATCPHY_MODE_DP) == 0); + assert(atcphy_configure_pipehandler(p, true) == 0); + assert(j700_dp_start(p, 3, 3, false) == 0); + } else +#endif + { + p->pipe_state = ATCPHY_PIPEHANDLER_STATE_USB3; + p->dp_owned_pmas = 3; /* legacy synthetic cleanup ownership, no J700 context path */ + } + p->mode_dirty = dirty; + p->dp_idle_pmas = dirty ? 1 : 0; + p->aux_power_requested = true; + polls = fail_poll = writes = usb2_on = usb2_writes = reset_writes = mux_writes = 0; +} + +static void inject_parking(unsigned int fault) +{ + pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = PIPEHANDLER_LOCK_EN; + if (fault) + fail_poll = 1; + else + stuck_ack = true; +} +static void recover(void) +{ + stuck_ack = bist_status_stuck = false; + hold_unlock_ack = false; + fail_caller = NULL; + fail_poll = polls = 0; +} +static int request(struct apple_atcphy *p, bool safe) +{ + struct typec_mux_dev mux = { p }; + struct typec_altmode dp = { .svid = USB_TYPEC_DP_SID }; + struct typec_mux_state state = { + .mode = safe ? TYPEC_STATE_SAFE : TYPEC_DP_STATE_C, + .alt = safe ? NULL : &dp, + }; + return atcphy_mux_set(&mux, &state); +} +static void unchanged(struct apple_atcphy *p, bool dirty, bool host) +{ + bool expected_dirty = dirty || (p->hw->has_t8140_dp && p->fixed_usb2 && host); + assert(p->mode == APPLE_ATCPHY_MODE_USB3_DP && p->mode_dirty == expected_dirty); + assert(p->pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3 && p->host_active == host); + assert(p->dp_owned_pmas == 3 && p->dp_idle_pmas == (dirty ? 1U : 0U)); + assert(p->aux_power_requested); + assert(writes == 0 && usb2_on == 0 && usb2_writes == 0 && reset_writes == 0); + assert(mux_writes == 0); + assert(pipe_regs[PIPEHANDLER_AON_GEN / 4] == PIPEHANDLER_AON_GEN_DWC3_RESET_N); + assert(usb2_regs[USB2PHY_USBCTL / 4] == USB2PHY_USBCTL_RUN); + if (expected_dirty) { + struct phy_configure_opts_dp training = { + .set_rate = true, .set_lanes = true, .link_rate = 8100, .lanes = 2, + }; + assert(j700_dp_configure(p, &training) == -EBUSY && writes == 0); + } +} +static void observed(const char *caller, int result) +{ + printf("%s: result=%d core_writes=%u usb2_writes=%u reset_writes=%u\n", + caller, result, writes, usb2_writes, reset_writes); + fflush(stdout); +} +static void mux_parking(void) +{ + struct apple_atcphy p; + unsigned int cases = 0; + for (unsigned int dirty = 0; dirty < 2; dirty++) + for (unsigned int swapped = 0; swapped < 2; swapped++) + for (unsigned int host = 0; host < 2; host++) + for (unsigned int safe = 0; safe < 2; safe++) + for (unsigned int fault = 0; fault < 2; fault++) { + initial(&p, true, dirty, swapped); + p.host_active = host; + inject_parking(fault); + int ret = request(&p, safe); + observed("mux parking failure", ret); + assert(ret == -ETIMEDOUT); + unchanged(&p, dirty, host); + if (!fault) { + assert(request(&p, safe) == -ETIMEDOUT); + unchanged(&p, dirty, host); + } + recover(); + assert(request(&p, safe) == 0); + assert(p.mode == (safe ? APPLE_ATCPHY_MODE_USB2 : APPLE_ATCPHY_MODE_DP)); + assert(!p.mode_dirty && p.host_active == host); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY && !p.dp_owned_pmas); + assert(!p.dp_idle_pmas && p.aux_power_requested == !safe); + assert(usb2_on == 1 && usb2_writes == 0 && reset_writes == 0); + assert(usb2_regs[USB2PHY_USBCTL / 4] == USB2PHY_USBCTL_RUN); + unsigned int before = writes; + assert(request(&p, safe) == 0 && writes == before && usb2_on == 1); + cases++; + } + printf("PASS: %u actual mux parking/retry scenarios preserve fixed USB2 and ownership\n", cases); +} +static void power_parking(void) +{ + struct apple_atcphy p; + unsigned int cases = 0; + for (unsigned int fixed = 0; fixed < 2; fixed++) + for (unsigned int dirty = 0; dirty < 2; dirty++) + for (unsigned int swapped = 0; swapped < 2; swapped++) + for (unsigned int fault = 0; fault < 2; fault++) { + initial(&p, fixed, dirty, swapped); + struct phy phy = { &p }; + inject_parking(fault); + int ret = atcphy_usb3_power_off(&phy); + observed("power-off parking failure", ret); + assert(ret == -ETIMEDOUT); + unchanged(&p, dirty, true); + if (!fault) { + assert(atcphy_usb3_power_off(&phy) == -ETIMEDOUT); + unchanged(&p, dirty, true); + } + recover(); + assert(atcphy_usb3_power_off(&phy) == 0); + assert(p.mode == APPLE_ATCPHY_MODE_OFF && !p.mode_dirty && !p.host_active); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); + assert(!p.dp_owned_pmas && !p.dp_idle_pmas && !p.aux_power_requested); + assert(usb2_writes == 0 && reset_writes == 0); + unsigned int before = writes; + assert(atcphy_usb3_power_off(&phy) == 0 && writes == before); + cases++; + } + printf("PASS: %u actual power-off parking/retry scenarios\n", cases); +} +static void reset_parking(void) +{ + struct apple_atcphy p; + unsigned int cases = 0; + for (unsigned int fixed = 0; fixed < 2; fixed++) + for (unsigned int dirty = 0; dirty < 2; dirty++) + for (unsigned int fault = 0; fault < 2; fault++) { + initial(&p, fixed, dirty, false); + inject_parking(fault); + int ret = atcphy_dwc3_reset_assert(&p.rcdev, 0); + observed("reset parking failure", ret); + assert(ret == -ETIMEDOUT); + unchanged(&p, dirty, true); + if (!fault) { + assert(atcphy_dwc3_reset_assert(&p.rcdev, 0) == -ETIMEDOUT); + unchanged(&p, dirty, true); + } + recover(); + assert(atcphy_dwc3_reset_assert(&p.rcdev, 0) == 0); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY && !p.host_active); + assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && p.mode_dirty == (dirty || fixed)); + assert(p.dp_owned_pmas == 3 && p.dp_idle_pmas == (dirty ? 1U : 0U)); + assert(p.aux_power_requested && writes == 0); + assert(reset_writes == 2 && usb2_writes > 0); + assert(!(pipe_regs[PIPEHANDLER_AON_GEN / 4] & PIPEHANDLER_AON_GEN_DWC3_RESET_N)); + assert(pipe_regs[PIPEHANDLER_AON_GEN / 4] & PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); + assert(!(usb2_regs[USB2PHY_USBCTL / 4] & USB2PHY_USBCTL_RUN)); + cases++; + } + printf("PASS: %u actual reset parking/retry scenarios retain DP ownership\n", cases); +} +static void power_cleanup(void) +{ + struct apple_atcphy p; + unsigned int phases = 0; + for (unsigned int fixed = 0; fixed < 2; fixed++) { + initial(&p, fixed, false, false); + p.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + struct phy phy = { &p }; + fail_poll = 1; /* The first DP stop ACK cannot complete. */ + int ret = atcphy_usb3_power_off(&phy); + observed("power-off DP cleanup failure", ret); + assert(ret == -ETIMEDOUT && p.mode_dirty && p.host_active); + assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && p.dp_owned_pmas == 3); + assert(p.aux_power_requested && usb2_writes == 0 && reset_writes == 0); + struct phy_configure_opts_dp training = { + .set_rate = true, .set_lanes = true, .link_rate = 8100, .lanes = 2, + }; + unsigned int before = writes; + assert(j700_dp_configure(&p, &training) == -EBUSY && writes == before); + recover(); + assert(atcphy_usb3_power_off(&phy) == 0); + assert(!p.mode_dirty && !p.host_active && !p.dp_owned_pmas && !p.aux_power_requested); + /* A failed initial transition can leave OFF recorded with dirty resources. */ + initial(&p, fixed, true, false); + p.mode = APPLE_ATCPHY_MODE_OFF; + p.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + fail_poll = 1; + assert(atcphy_usb3_power_off(&phy) == -ETIMEDOUT); + assert(p.mode_dirty && p.host_active && p.dp_owned_pmas == 3); + recover(); + assert(atcphy_usb3_power_off(&phy) == 0); + assert(p.mode == APPLE_ATCPHY_MODE_OFF && !p.mode_dirty && !p.host_active); + assert(!p.dp_owned_pmas && !p.dp_idle_pmas && !p.aux_power_requested); + for (unsigned int fault = 1; fault <= 2; fault++) { + initial(&p, fixed, false, false); + p.mode = APPLE_ATCPHY_MODE_USB2; + p.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + p.dp_owned_pmas = p.dp_idle_pmas = 0; + p.aux_power_requested = false; + fail_poll = fault; /* BIG/SMALL sleep ACK. */ + assert(atcphy_usb3_power_off(&phy) == -ETIMEDOUT); + assert(p.mode == APPLE_ATCPHY_MODE_USB2 && p.mode_dirty && p.host_active); + recover(); + assert(atcphy_usb3_power_off(&phy) == 0 && !p.mode_dirty && !p.host_active); + } + /* Inject every poll in DP stop, pending PMA idle, AUX and domain sleep. */ + initial(&p, fixed, true, false); + p.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + assert(atcphy_usb3_power_off(&phy) == 0); + unsigned int total = polls; + for (unsigned int fault = 1; fault <= total; fault++) { + initial(&p, fixed, true, false); + p.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + fail_poll = fault; + assert(atcphy_usb3_power_off(&phy) == -ETIMEDOUT); + assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && p.mode_dirty && p.host_active); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); + assert(usb2_writes == 0 && reset_writes == 0); + unsigned int owned = p.dp_owned_pmas, idle = p.dp_idle_pmas; + bool aux = p.aux_power_requested; + unsigned int before = writes; + assert(j700_dp_configure(&p, &training) == -EBUSY && writes == before); + assert(p.dp_owned_pmas == owned && p.dp_idle_pmas == idle); + assert(p.aux_power_requested == aux); + recover(); + assert(atcphy_usb3_power_off(&phy) == 0); + assert(p.mode == APPLE_ATCPHY_MODE_OFF && !p.mode_dirty && !p.host_active); + assert(!p.dp_owned_pmas && !p.dp_idle_pmas && !p.aux_power_requested); + phases++; + } + } + printf("PASS: %u cleanup poll faults retain retry/ownership; dirty OFF retries cleanup\n", phases); +} +static int select_mode(struct apple_atcphy *p, enum atcphy_mode mode) +{ + struct typec_mux_dev mux = { p }; + struct typec_altmode dp = { .svid = USB_TYPEC_DP_SID }; + struct typec_mux_state state = { .mode = TYPEC_STATE_SAFE }; + + if (mode == APPLE_ATCPHY_MODE_USB3) + state.mode = TYPEC_STATE_USB; + else if (mode == APPLE_ATCPHY_MODE_USB3_DP || mode == APPLE_ATCPHY_MODE_DP) { + state.alt = &dp; + state.mode = mode == APPLE_ATCPHY_MODE_DP ? TYPEC_DP_STATE_C : TYPEC_DP_STATE_D; + } + return atcphy_mux_set(&mux, &state); +} +static void orient(struct apple_atcphy *p, bool reverse) +{ + struct typec_switch_dev sw = { p }; + assert(atcphy_sw_set(&sw, reverse ? TYPEC_ORIENTATION_REVERSE : + TYPEC_ORIENTATION_NORMAL) == 0); +} +static void orientation_initial(struct apple_atcphy *p, bool fixed) +{ + cold_initial(p, fixed, false); + p->mode = p->typec_mode = fixed ? APPLE_ATCPHY_MODE_USB2 : APPLE_ATCPHY_MODE_OFF; + p->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + p->dp_owned_pmas = p->dp_idle_pmas = 0; + p->aux_power_requested = false; + p->dp_link_rate = -1; + p->host_active = fixed; + registers[ACIOPHY_TOP_PHY_STAT / 4] = ACIOPHY_TOP_PHY_STAT_LN0_UNK0; + registers[ATCPHY_MISC / 4] = BIT(7) | ATCPHY_MISC_LANE_SWAP; + expected_j700_swap = true; + registers[hw.aciophy_lane_mode / 4] = 0xa5000000; + registers[hw.aciophy_crossbar / 4] = 0x80000000; + pipe_regs[PIPEHANDLER_MUX_CTRL / 4] = + FIELD_PREP(hw.pipe_mux_data, PIPEHANDLER_MUX_CTRL_DATA_DUMMY) | + FIELD_PREP(hw.pipe_mux_clk, PIPEHANDLER_MUX_CTRL_CLK_DUMMY); +} +static void check_route(struct apple_atcphy *p, enum atcphy_mode mode, bool reverse) +{ + assert(p->mode == mode && p->swap_lanes == reverse && !p->mode_dirty); + assert(registers[ATCPHY_MISC / 4] & BIT(7)); + /* Fixed public route expectations, independent of the production mode table. */ + assert(mode == APPLE_ATCPHY_MODE_USB3 || mode == APPLE_ATCPHY_MODE_USB3_DP || + mode == APPLE_ATCPHY_MODE_DP); + u32 lane = registers[hw.aciophy_lane_mode / 4]; + u32 crossbar = registers[hw.aciophy_crossbar / 4]; + u32 mask = hw.has_t8140_dp && mode != APPLE_ATCPHY_MODE_DP ? 0x1ffff : 0x3ffff; + assert((lane & 0xfff) == (mode == APPLE_ATCPHY_MODE_DP ? 0x492 : + (reverse ? 0x252 : 0x489))); + assert((lane & 0xfffff000) == 0xa5000000); + assert((crossbar & mask) == (mode == APPLE_ATCPHY_MODE_DP ? + (reverse ? 0x114 : 0x22014) : (0x110 | reverse))); + assert(crossbar & 0x80000000); + /* J700 preserves the boot swap bit; generic routes apply their explicit swap. */ + assert(!!(registers[ATCPHY_MISC / 4] & ATCPHY_MISC_LANE_SWAP) == + (hw.has_t8140_dp ? expected_j700_swap : + (mode != APPLE_ATCPHY_MODE_DP && reverse))); + assert(p->pipe_state == (p->fixed_usb2 && mode != APPLE_ATCPHY_MODE_DP ? + ATCPHY_PIPEHANDLER_STATE_USB3 : ATCPHY_PIPEHANDLER_STATE_DUMMY)); +} +static void orientation_roundtrip(void) +{ + struct apple_atcphy p; + const enum atcphy_mode modes[] = { + APPLE_ATCPHY_MODE_USB3, APPLE_ATCPHY_MODE_USB3_DP, APPLE_ATCPHY_MODE_DP, + }; + unsigned int cases = 0; + + for (unsigned int generation = 0; generation < 4; generation++) { + hw.gen = generation == 0 ? ATCPHY_GENERATION_T8103 : ATCPHY_GENERATION_T8122; + hw.has_t8140_dp = generation == 2; + hw.has_usb4 = !hw.has_t8140_dp; + hw.aciophy_lane_mode = generation == 0 ? ACIOPHY_LANE_MODE_T8103 : + ACIOPHY_LANE_MODE_T8122; + hw.aciophy_crossbar = generation == 0 ? ACIOPHY_CROSSBAR_T8103 : + ACIOPHY_CROSSBAR_T8122; + hw.pipe_mux_clk = generation == 3 ? PIPEHANDLER_MUX_CTRL_CLK_T8142 : + PIPEHANDLER_MUX_CTRL_CLK_T8103; + hw.pipe_mux_data = generation == 3 ? PIPEHANDLER_MUX_CTRL_DATA_T8142 : + PIPEHANDLER_MUX_CTRL_DATA_T8103; + for (unsigned int fixed = 0; fixed < 2; fixed++) { + for (unsigned int m = 0; m < ARRAY_SIZE(modes); m++) { + for (unsigned int first = 0; first < 2; first++) { + orientation_initial(&p, fixed); + for (unsigned int turn = 0; turn < 3; turn++) { + bool reverse = first ^ (turn & 1); + orient(&p, reverse); + if (turn) { + assert(p.mode_dirty); + struct phy phy = { &p }; + unsigned int before = writes; + assert(atcphy_usb3_set_mode(&phy, PHY_MODE_USB_HOST, 0) == -EBUSY); + assert(writes == before); + } + require_parked = true; + assert(select_mode(&p, modes[m]) == 0); + require_parked = false; + check_route(&p, modes[m], reverse); + unsigned int before = writes, before_mux = mux_writes; + orient(&p, reverse); + assert(select_mode(&p, modes[m]) == 0); + assert(writes == before && mux_writes == before_mux); + struct typec_switch_dev sw = { &p }; + assert(atcphy_sw_set(&sw, TYPEC_ORIENTATION_NONE) == 0); + assert(!p.mode_dirty && p.swap_lanes == reverse); + cases++; + } + /* A USB3 protocol change also rebuilds the live backend. */ + if (m == 0) { + require_parked = true; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + require_parked = false; + check_route(&p, APPLE_ATCPHY_MODE_USB3_DP, first); + } + assert(select_mode(&p, APPLE_ATCPHY_MODE_OFF) == 0); + assert(p.mode == (fixed ? APPLE_ATCPHY_MODE_USB2 : APPLE_ATCPHY_MODE_OFF)); + assert(!p.mode_dirty && !p.aux_power_requested && !p.dp_idle_pmas); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); + } + } + } + } + printf("PASS: %u production orientation routes across T8103/T8122/T8140/T8142, fixed/ordinary ports, both flip roundtrips\n", cases); +} +static void orientation_faults(void) +{ + struct apple_atcphy p; + unsigned int cases = 0; + + for (unsigned int fixed = 0; fixed < 2; fixed++) { + for (unsigned int first = 0; first < 2; first++) { + orientation_initial(&p, fixed); + orient(&p, first); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + struct phy_configure_opts_dp rate = { + .set_rate = true, .set_lanes = true, .link_rate = 8100, .lanes = 2, + }; + assert(j700_dp_configure(&p, &rate) == 0); + assert(p.dp_owned_pmas == BIT(!first)); + orient(&p, !first); + unsigned int before = writes; + struct phy_configure_opts_dp training = { .set_voltages = true, .lanes = 2 }; + assert(j700_dp_configure(&p, &training) == -EBUSY && writes == before); + assert(j700_dp_configure(&p, &rate) == -EBUSY && writes == before); + /* Derive all actual poll phases, then inject each in a fresh transition. */ + polls = 0; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + unsigned int total = polls; + bool advisory[256] = { false }; + for (unsigned int i = 0; i < total; i++) { + const char *caller = poll_caller[i]; + /* Every observed BIST and USB3 handoff failure is now fatal. */ + if (fixed && i + 1 == total) { + assert(!strcmp(caller, "atcphy_pipehandler_unlock")); + advisory[i] = false; + } else if (!strcmp(caller, "atcphy_configure_pipehandler_usb3_host_t8122")) { + assert(fixed); + advisory[i] = false; + } else { + assert(!strcmp(caller, "j700_dp_wait") || + !strcmp(caller, "j700_field_update") || + !strcmp(caller, "j700_usb_field_update") || + !strcmp(caller, "atcphy_power_on") || + !strcmp(caller, "atcphy_configure") || + (fixed && (!strcmp(caller, "atcphy_pipehandler_lock") || + !strcmp(caller, "atcphy_pipehandler_unlock")))); + } + } + for (unsigned int fault = 1; fault <= total; fault++) { + orientation_initial(&p, fixed); + orient(&p, first); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(j700_dp_configure(&p, &rate) == 0); + unsigned int owned = p.dp_owned_pmas; + orient(&p, !first); + polls = 0; + fail_poll = fault; + int ret = select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP); + /* A fatal phase must never be accepted as advisory success. */ + assert((ret == 0) == advisory[fault - 1]); + if (!ret) { + check_route(&p, APPLE_ATCPHY_MODE_USB3_DP, !first); + assert(!p.dp_owned_pmas && !p.dp_idle_pmas); + cases++; + continue; + } + assert(ret == -ETIMEDOUT); + if (p.mode_dirty || p.pipe_pending) { + before = writes; + assert(j700_dp_configure(&p, &training) == -EBUSY && writes == before); + assert(j700_dp_configure(&p, &rate) == -EBUSY && writes == before); + if (fault == 1 && !fixed) + assert(p.dp_owned_pmas == owned && p.aux_power_requested); + } else { + assert(fixed && p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); + } + recover(); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + check_route(&p, APPLE_ATCPHY_MODE_USB3_DP, !first); + assert(!p.dp_owned_pmas && !p.dp_idle_pmas); + assert(j700_dp_configure(&p, &rate) == 0); + assert(p.dp_owned_pmas == BIT(first)); + assert(j700_dp_configure(&p, &training) == 0); + cases++; + } + /* A fatal parking check preserves the old owner and all lane words. */ + if (fixed) { + orientation_initial(&p, true); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(j700_dp_configure(&p, &rate) == 0); + orient(&p, true); + before = writes; + inject_parking(0); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == -ETIMEDOUT); + assert(writes == before && p.dp_owned_pmas == 2 && p.mode_dirty); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3 && p.host_active); + recover(); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + check_route(&p, APPLE_ATCPHY_MODE_USB3_DP, true); + } + /* Reversing a pending flip cannot erase cleanup obligations. */ + orientation_initial(&p, fixed); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(j700_dp_configure(&p, &rate) == 0); + orient(&p, true); + orient(&p, false); + assert(p.mode_dirty && p.dp_owned_pmas == 2); + p.ss_tunables = false; + before = writes; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == -EOPNOTSUPP); + assert(writes == before && p.mode_dirty && p.dp_owned_pmas == 2); + p.ss_tunables = true; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + check_route(&p, APPLE_ATCPHY_MODE_USB3_DP, false); + assert(!p.dp_owned_pmas && !p.dp_idle_pmas); + } + } + printf("PASS: %u orientation transition poll injections, dirty training guard, ownership and retry controls\n", cases); +} +static void backend_change(void) +{ + struct apple_atcphy p; + + orientation_initial(&p, true); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3) == 0); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3); + require_parked = true; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + require_parked = false; + check_route(&p, APPLE_ATCPHY_MODE_USB3_DP, false); + puts("PASS: a live fixed-hub USB3 backend parks before same-orientation protocol changes"); +} + + +static void failed_flip_return(void) +{ + struct apple_atcphy p; + unsigned int cases = 0; + struct phy_configure_opts_dp rate = { + .set_rate = true, .set_lanes = true, .link_rate = 8100, .lanes = 2, + }; + + for (unsigned int fixed = 0; fixed < 2; fixed++) { + for (unsigned int first = 0; first < 2; first++) { + orientation_initial(&p, fixed); + orient(&p, first); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(j700_dp_configure(&p, &rate) == 0); + orient(&p, !first); + polls = 0; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + unsigned int route_fault = 0; + for (unsigned int i = 0; i < polls; i++) + if (!strcmp(poll_caller[i], "j700_field_update")) route_fault = i + 1; + assert(route_fault); + + orientation_initial(&p, fixed); + orient(&p, first); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(j700_dp_configure(&p, &rate) == 0); + orient(&p, !first); + polls = 0; + fail_poll = route_fault; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == -ETIMEDOUT); + /* The route changed before PMA idle completed; completed mode is still old. */ + assert((registers[0x60 / 4] & 0xfff) == (first ? 0x489 : 0x252)); + assert(p.mode_dirty && p.dp_idle_pmas == BIT(first) && !p.dp_owned_pmas); + orient(&p, first); + assert(p.mode_dirty && p.dp_idle_pmas == BIT(first)); + unsigned int before = writes; + assert(j700_dp_configure(&p, &rate) == -EBUSY && writes == before); + recover(); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + check_route(&p, APPLE_ATCPHY_MODE_USB3_DP, first); + assert(!p.dp_idle_pmas && !p.dp_owned_pmas); + assert(j700_dp_configure(&p, &rate) == 0); + assert(p.dp_owned_pmas == BIT(!first)); + cases++; + } + } + printf("PASS: %u partially applied flip cancellations preserve pending PMA ownership\n", cases); +} + +static void safe_first_recovery(void) +{ + struct apple_atcphy p; + unsigned int cases = 0; + + for (unsigned int first = 0; first < 2; first++) { + orientation_initial(&p, true); + orient(&p, first); + bist_status_stuck = true; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3) == -ETIMEDOUT); + assert(bist_faults == 2 && p.mode_dirty && p.pipe_pending); + bist_status_stuck = false; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3) == 0); + check_route(&p, APPLE_ATCPHY_MODE_USB3, first); + polls = 0; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + unsigned int host_lock_poll = 0; + for (unsigned int i = 0; i < polls; i++) + if (!strcmp(poll_caller[i], "atcphy_pipehandler_lock")) host_lock_poll = i + 1; + assert(host_lock_poll > 1); /* Inject the host lock, after acknowledged parking. */ + orientation_initial(&p, true); + orient(&p, first); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3) == 0); + polls = 0; + fail_poll = host_lock_poll; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == -ETIMEDOUT); + assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && !p.mode_dirty); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY && p.host_active); + assert(p.aux_power_requested); + + /* Persistent AUX cleanup failure blocks SAFE before orientation publication. */ + fail_caller = "j700_field_update"; + u32 route = registers[0x60 / 4], crossbar = registers[0x64 / 4]; + for (unsigned int attempt = 0; attempt < 3; attempt++) { + assert(select_mode(&p, APPLE_ATCPHY_MODE_OFF) == -ETIMEDOUT); + assert(p.swap_lanes == first && p.mode_dirty && p.aux_power_requested); + assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && p.host_active); + assert(registers[0x60 / 4] == route && registers[0x64 / 4] == crossbar); + assert(usb2_regs[USB2PHY_USBCTL / 4] == USB2PHY_USBCTL_RUN); + } + /* Returning the plug cannot clear an unavailable cleanup handshake. */ + assert(select_mode(&p, APPLE_ATCPHY_MODE_OFF) == -ETIMEDOUT); + assert(p.swap_lanes == first && p.mode_dirty); + recover(); /* Model hardware readiness returning; production performs the retry. */ + assert(select_mode(&p, APPLE_ATCPHY_MODE_OFF) == 0); + assert(p.mode == APPLE_ATCPHY_MODE_USB2 && !p.mode_dirty && !p.aux_power_requested); + /* SAFE uses the existing generic USB2 route and clears its swap control. */ + assert(!(registers[ATCPHY_MISC / 4] & ATCPHY_MISC_LANE_SWAP)); + expected_j700_swap = false; + orient(&p, !first); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + check_route(&p, APPLE_ATCPHY_MODE_USB3_DP, !first); + + /* Real reconnect order: SAFE must succeed before the new orientation is published. */ + struct phy_configure_opts_dp rate = { + .set_rate = true, .set_lanes = true, .link_rate = 8100, .lanes = 2, + }; + assert(j700_dp_configure(&p, &rate) == 0); + assert(p.dp_owned_pmas == BIT(first)); + assert(select_mode(&p, APPLE_ATCPHY_MODE_OFF) == 0); + assert(!p.dp_owned_pmas && !p.dp_idle_pmas && !p.aux_power_requested); + orient(&p, first); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + check_route(&p, APPLE_ATCPHY_MODE_USB3_DP, first); + assert(j700_dp_configure(&p, &rate) == 0); + assert(p.dp_owned_pmas == BIT(!first)); + unsigned int before = writes, before_mux = mux_writes; + orient(&p, first); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(writes == before && mux_writes == before_mux); + cases++; + } + printf("PASS: %u chained BIST/host-ACK failure, persistent SAFE failure and SAFE-first roundtrip controls\n", cases); +} + +static void live_parking_handshake(void) +{ + struct apple_atcphy p; + struct phy_configure_opts_dp rate = { + .set_rate = true, .set_lanes = true, .link_rate = 8100, .lanes = 2, + }; + struct phy_configure_opts_dp training = { .set_voltages = true, .lanes = 2 }; + unsigned int cases = 0; + for (unsigned int reverse = 0; reverse < 2; reverse++) { + for (unsigned int fault = 0; fault < 3; fault++) { + orientation_initial(&p, true); + orient(&p, reverse); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(j700_dp_configure(&p, &rate) == 0); + u32 lanes = registers[0x60 / 4], route = registers[0x64 / 4]; + u32 native = pipe_regs[PIPEHANDLER_NONSELECTED_OVERRIDE / 4]; + unsigned int before = writes, owned = p.dp_owned_pmas; + if (fault == 2) { + /* A previous host unlock left ACK asserted before parking. */ + pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = PIPEHANDLER_LOCK_EN; + hold_unlock_ack = true; + } else if (fault == 1) { + hold_unlock_ack = true; + } else { + stuck_ack = true; + } + for (unsigned int retry = 0; retry < 3; retry++) { + unsigned int before_mux = mux_writes; + assert(select_mode(&p, APPLE_ATCPHY_MODE_OFF) == -ETIMEDOUT); + assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && p.mode_dirty); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3 && p.host_active); + assert(p.dp_owned_pmas == owned && p.aux_power_requested); + assert(writes == before && registers[0x60 / 4] == lanes && registers[0x64 / 4] == route); + assert(pipe_regs[PIPEHANDLER_NONSELECTED_OVERRIDE / 4] == native); + if (fault != 1) assert(mux_writes == before_mux); + /* A failed SAFE cannot be forgotten by returning to the cached mode. */ + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == -ETIMEDOUT); + assert(j700_dp_configure(&p, &training) == -EBUSY && writes == before); + assert(!usb2_writes && !reset_writes); + } + recover(); + assert(select_mode(&p, APPLE_ATCPHY_MODE_OFF) == 0); + assert(p.mode == APPLE_ATCPHY_MODE_USB2 && !p.mode_dirty && p.host_active); + assert(!p.dp_owned_pmas && !p.dp_idle_pmas && !p.aux_power_requested); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); + expected_j700_swap = false; + orient(&p, !reverse); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + check_route(&p, APPLE_ATCPHY_MODE_USB3_DP, !reverse); + assert(j700_dp_configure(&p, &rate) == 0 && p.dp_owned_pmas == BIT(reverse)); + cases++; + } + } + printf("PASS: %u live parking entry-ACK/LOCK/UNLOCK failure, ownership, write ordering and retry roundtrips\n", cases); +} + +int main(int argc, char **argv) +{ + assert(argc == 2); + if (!strcmp(argv[1], "mux")) mux_parking(); + else if (!strcmp(argv[1], "power")) power_parking(); + else if (!strcmp(argv[1], "reset")) reset_parking(); + else if (!strcmp(argv[1], "cleanup")) power_cleanup(); + else if (!strcmp(argv[1], "orientation")) orientation_roundtrip(); + else if (!strcmp(argv[1], "orientation-faults")) orientation_faults(); + else if (!strcmp(argv[1], "backend-change")) backend_change(); + else if (!strcmp(argv[1], "safe-first")) safe_first_recovery(); + else if (!strcmp(argv[1], "live-park")) live_parking_handshake(); + else if (!strcmp(argv[1], "failed-flip-return")) failed_flip_return(); + else assert(false); + return 0; +} +''' + return {**locals(), "block": block} + +def adverse(root): + + # Preserve continued source macros; the parking fixture does not use the + # continued PLL clock macro exercised by the legacy rate helper here. + ns = parking(root) + production = ns['production'] + '\n' + ns['function'](ns['atc'], 'atcphy_usb2_init') + production = production.replace('bool pipe_pending, usb_off_pending, usb_off_drained;', + 'bool pipe_pending, usb_off_pending, usb_off_drained, tunnel_clock_on, tunnel_saved;', 1) + production += '\n' + ns['block'](ns['atc'], 'enum atcphy_lane {') + ';' + production += '\n' + ns['block'](ns['atc'], 'struct atcphy_dp_link_rate_configuration {') + ';' + production += '\n' + ns['block'](ns['atc'], 'static const struct atcphy_dp_link_rate_configuration dp_lr_config[] = {') + ';' + # These leaf operations are outside the pending/cache gate under test. + production += r''' +static int atcphy_auspll_apb_command(struct apple_atcphy *p, u32 command) +{ (void)p; (void)command; assert(false); return -EIO; } +static int atcphy_dp_configure_lane(struct apple_atcphy *p, enum atcphy_lane lane, + const struct atcphy_dp_link_rate_configuration *cfg) +{ (void)p; (void)lane; (void)cfg; assert(false); return -EIO; } +''' + production += '\n' + ns['function'](ns['atc'], 'atcphy_dp_configure') + for profile in ('t8103', 't8122', 't8140', 't8142'): + production += '\n' + ns['block'](ns['atc'], + 'static const struct atcphy_hw atcphy_hw_' + profile + ' = {') + ';' + production = production.replace('static unsigned int polls,', 'static int poll_errors[256];\nstatic unsigned int polls,', 1) + production = production.replace('polls == fail_poll || bist_fault ||', + 'poll_errors[polls] ? poll_errors[polls] : polls == fail_poll || bist_fault ||', 1) + production = production.replace('static u32 readl(void *address) { return *(u32 *)address; }', + '''static unsigned int mmio_reads; +static bool only_power_mmio; +static u32 readl(void *address) +{ + mmio_reads++; + if (only_power_mmio && (uintptr_t)address >= (uintptr_t)registers && + (uintptr_t)address < (uintptr_t)registers + sizeof(registers)) { + uintptr_t offset=(uintptr_t)address-(uintptr_t)registers; + assert(offset==ATCPHY_POWER_CTRL || offset==ATCPHY_POWER_STAT || offset==ATCPHY_MISC); + } + return *(u32 *)address; +}''', 1) + production = production.replace(' (void)p;\n assert(offset < sizeof(registers)', + ' (void)p;\n if (only_power_mmio) assert(offset==ATCPHY_POWER_CTRL || offset==ATCPHY_MISC);\n assert(offset < sizeof(registers)', 1) + controls = r''' +static void legacy_setup(struct apple_atcphy *p, unsigned int gen, bool fixed) +{ + hw = gen==0 ? atcphy_hw_t8103 : gen==1 ? atcphy_hw_t8122 : + gen==2 ? atcphy_hw_t8140 : atcphy_hw_t8142; + cold_initial(p, fixed, false); + p->mode = p->typec_mode = APPLE_ATCPHY_MODE_USB3; + registers[ACIOPHY_TOP_PHY_STAT/4] = ACIOPHY_TOP_PHY_STAT_LN0_UNK0; + memset(poll_errors, 0, sizeof(poll_errors)); +} +static void shared_bist(void) +{ + for (unsigned int gen=0; gen<2; gen++) { + unsigned int count=gen ? 5 : 3; + for (unsigned int fault=1; fault<=count; fault++) { + struct apple_atcphy p; + legacy_setup(&p,gen,false); + poll_errors[fault]=-EIO; + if (fault1 && !strcmp(argv[1],"epoch")) { epoch_controls(); return 0; } + if(argc>1 && !strcmp(argv[1],"bist")) { bist_controls(); return 0; } + if(argc>1)return original_fixture_main(argc,argv); + epoch_controls();bist_controls();return 0; +} +''' + return ns['prefix'] + production + ns['suffix'].replace('int main(', 'int original_fixture_main(', 1) + controls + +def warm_reset(root): + + ns = parking(root) + production = ns['production'] + reset_block = '''\t/* +\t * Fixed-hub SAFE keeps USB2 up rather than taking the block OFF. +\t * With PIPE parked and DP owners released, reinitialize USB3 before +\t * programming its lanes; the existing sequence releases this reset. +\t */ +\tif (atcphy->hw->has_t8140_dp && atcphy->fixed_usb2) +\t\tcore_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_PHY_RESET_N); + +''' + assert production.count(reset_block) == 1 + production = production.replace('static u32 registers[', '''static unsigned int warm_reset_edges, warm_unreset_lanes, warm_ownership_errors; +static u32 registers[''', 1) + production = production.replace(' registers[offset / 4] = (registers[offset / 4] & ~mask) | value;', ''' if (offset == ATCPHY_POWER_CTRL && (mask & ATCPHY_POWER_PHY_RESET_N) && + !(value & ATCPHY_POWER_PHY_RESET_N)) { + warm_reset_edges++; + if (p->fixed_usb2 && p->hw->has_t8140_dp && + (p->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY || p->dp_owned_pmas || + p->dp_idle_pmas || p->aux_power_requested)) warm_ownership_errors++; + } + if (offset == p->hw->aciophy_lane_mode && p->fixed_usb2 && p->hw->has_t8140_dp && + (registers[ATCPHY_POWER_CTRL / 4] & ATCPHY_POWER_PHY_RESET_N)) warm_unreset_lanes++; + registers[offset / 4] = (registers[offset / 4] & ~mask) | value;''', 1) + controls = r''' +static void warm_initial(struct apple_atcphy *p) +{ + orientation_initial(p, true); + registers[ATCPHY_POWER_CTRL / 4] |= ATCPHY_POWER_PHY_RESET_N; + warm_reset_edges = warm_unreset_lanes = warm_ownership_errors = 0; +} +static void warm_roundtrip(void) +{ + bool failed = false; + for (unsigned int first = 0; first < 2; first++) { + struct apple_atcphy p; + const struct phy_configure_opts_dp rate = { + .set_rate = true, .set_lanes = true, .link_rate = 8100, .lanes = 2, + }; + warm_initial(&p); + for (unsigned int attach = 0; attach < 3; attach++) { + orient(&p, first ^ (attach & 1)); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3 && !p.mode_dirty); + assert(j700_dp_configure(&p, &rate) == 0); + unsigned int edges = warm_reset_edges; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(warm_reset_edges == edges); /* cached same-mode must not reset */ + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB2) == 0); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY && p.host_active); + assert(!p.dp_owned_pmas && !p.dp_idle_pmas && !p.aux_power_requested); + assert(registers[ATCPHY_POWER_CTRL / 4] & ATCPHY_POWER_PHY_RESET_N); + assert(!reset_writes && !usb2_writes); + } + assert(usb2_on == 6); /* no extra USB2 power-on call introduced */ + printf("first=%u reset_assertions=%u unreset_lane_writes=%u ownership_errors=%u\n", + first, warm_reset_edges, warm_unreset_lanes, warm_ownership_errors); + failed |= warm_reset_edges != 6 || warm_unreset_lanes || warm_ownership_errors; + } + fflush(stdout); + assert(!failed); +} +static void warm_failure_controls(void) +{ + struct apple_atcphy p; + const struct phy_configure_opts_dp rate = { + .set_rate = true, .set_lanes = true, .link_rate = 8100, .lanes = 2, + }; + warm_initial(&p); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(j700_dp_configure(&p, &rate) == 0); + unsigned int edges = warm_reset_edges, owned = p.dp_owned_pmas; + stuck_ack = true; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB2) == -ETIMEDOUT); + assert(warm_reset_edges == edges && p.dp_owned_pmas == owned && p.mode_dirty); + assert(!usb2_writes && !reset_writes); + + warm_initial(&p); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(j700_dp_configure(&p, &rate) == 0); + edges = warm_reset_edges; owned = p.dp_owned_pmas; + fail_caller = "j700_field_update"; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB2) == -ETIMEDOUT); + assert(warm_reset_edges == edges && p.dp_owned_pmas == owned && p.mode_dirty); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); + assert(!usb2_writes && !reset_writes); + + warm_initial(&p); + fail_caller = "atcphy_configure"; /* existing RCAL poll, after reset release */ + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == -ETIMEDOUT); + assert(warm_reset_edges == 1 && !warm_ownership_errors && p.mode_dirty); + assert(registers[ATCPHY_POWER_CTRL / 4] & ATCPHY_POWER_PHY_RESET_N); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); + struct phy phy = { &p }; + unsigned int before = writes; + assert(atcphy_usb3_set_mode(&phy, PHY_MODE_USB_HOST, 0) == -EBUSY); + assert(j700_dp_configure(&p, &rate) == -EBUSY && writes == before); + fail_caller = NULL; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(!p.mode_dirty && !warm_ownership_errors); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3); + assert(!usb2_writes && !reset_writes); + puts("PASS: park/DP-cleanup failures precede reset; released-reset RCAL failure stays dirty and retryable"); +} +static void warm_scope_controls(void) +{ + struct apple_atcphy p; + for (unsigned int fixed = 0; fixed < 2; fixed++) { + for (unsigned int t8140 = 0; t8140 < 2; t8140++) { + orientation_initial(&p, fixed); + hw.has_t8140_dp = t8140; + registers[ATCPHY_POWER_CTRL / 4] |= ATCPHY_POWER_PHY_RESET_N; + warm_reset_edges = warm_unreset_lanes = warm_ownership_errors = 0; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(warm_reset_edges == (fixed && t8140)); + assert(!warm_ownership_errors && !usb2_writes && !reset_writes); + } + } + hw.has_t8140_dp = true; + puts("PASS: reset scope requires both fixed hub and T8140; ordinary/legacy paths unchanged"); +} +int main(int argc, char **argv) +{ + if (argc > 1) return original_fixture_main(argc, argv); + warm_roundtrip(); warm_failure_controls(); warm_scope_controls(); + puts("PASS: warm reset source ordering; no electrical recovery is modeled"); + return 0; +} +''' + suffix = ns['suffix'].replace('int main(', 'int original_fixture_main(', 1) + controls + return ns['prefix'] + production + suffix + +def usb_context(root): + ns = parking(root) + production=ns['production'].replace('static u32 registers[', 'static u32 blocked_offset;\nstatic unsigned int mmio_reads;\nstatic bool track_dp_writes;\nstatic u32 dp_write_masks[0x30000 / 4];\nstatic u32 registers[', 1) + production=production.replace('static u32 readl(void *address) { return *(u32 *)address; }', 'static u32 readl(void *address) { mmio_reads++; return *(u32 *)address; }', 1) + production=production.replace(' registers[offset / 4] = (registers[offset / 4] & ~mask) | value;', ' if (track_dp_writes) dp_write_masks[offset / 4] |= mask | value;\n if (blocked_offset && offset == blocked_offset) { writes++; return; }\n registers[offset / 4] = (registers[offset / 4] & ~mask) | value;', 1) + controls = r''' +static void seed_usb(struct apple_atcphy *p, unsigned int first) +{ + orientation_initial(p, true); + registers[0x200 / 4] = 0xee; + registers[0xc / 4] = 0; + /* Synthetic USB context, never a private factory table or captured register image. */ + for (unsigned int offset = 0x9000; offset < 0x15000; offset += 4) + registers[offset / 4] = 0x5a5a1234 ^ offset; + registers[0x8 / 4] = 0xa5000000; + registers[0x10c / 4] = 0xa5000000; + orient(p, first); + assert(select_mode(p, APPLE_ATCPHY_MODE_USB3_DP) == 0); +} + +static void mask_coverage(void) +{ + struct apple_atcphy p; + memset(dp_write_masks, 0, sizeof(dp_write_masks)); + seed_usb(&p, 1); + for (unsigned int rate = 0; rate < 4; rate++) { + track_dp_writes = true; + assert(j700_dp_start(&p, 1, rate, false) == 0); + j700_dp_equalize(&p, 0, true, 0x123); + j700_dp_equalize(&p, 0, false, 0x123); + assert(j700_dp_stop(&p) == 0); + track_dp_writes = false; + } + unsigned int entries = 0; + for (unsigned int offset = 0; offset < sizeof(dp_write_masks); offset += 4) { + if (!((offset >= 0x9000 && offset <= 0xdfff) || offset == 8 || + offset == 0xc || offset == 0x10c || offset == 0x200)) continue; + u32 expected = 0; + for (unsigned int i = 0; i < ARRAY_SIZE(j700_usb_fields); i++) + if (j700_usb_fields[i].offset == offset) expected = j700_usb_fields[i].mask; + assert(dp_write_masks[offset / 4] == expected); + entries += !!expected; + } + assert(entries == 27); + puts("PASS: 27 masks equal the actual public DP start/voltage/stop write union at every rate"); +} +static void masked_roundtrip(void) +{ + const struct phy_configure_opts_dp rate = { + .set_rate=true, .set_lanes=true, .link_rate=8100, .lanes=2, + }; + bool failure=false; + for (unsigned int first=0; first<2; first++) { + struct apple_atcphy p; + seed_usb(&p, first); + unsigned int old_dp=!first; +#ifdef PROPOSAL + assert(p.dp_usb_context_valid & BIT(old_dp)); + assert(p.dp_usb_context[old_dp][1] == 0); + assert(p.dp_usb_context[old_dp][3] == (0xeU << (4 * old_dp))); + assert(((registers[0x200 / 4] >> (4 * old_dp)) & 0xf) == 1); +#endif + u32 saved[27], outside[27]; + for (unsigned int i=0;i<27;i++) { + u32 offset,mask; + j700_usb_field(i,old_dp,&offset,&mask); + saved[i]=core_read32(&p,offset)&mask; + } + /* Power/receiver were saved before route idle, not at this post-route sample. */ + saved[1]=0; saved[3]=0xeU<<(4*old_dp); + assert(j700_dp_configure(&p,&rate)==0); + unsigned int edges_before=writes; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==0); + assert(writes==edges_before); /* same mode never resets or recaptures */ + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB2)==0); + for(unsigned int i=0;i<27;i++) { + u32 offset,mask;j700_usb_field(i,old_dp,&offset,&mask); + u32 other_offset,other_mask; j700_usb_field(i,!old_dp,&other_offset,&other_mask); + outside[i]=registers[offset/4]&~(mask | (i<4 ? other_mask : 0)); + } + orient(&p,!first); + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==0); + unsigned int mismatch=0; + for(unsigned int i=0;i<27;i++) { + u32 offset,mask;j700_usb_field(i,old_dp,&offset,&mask); + if((registers[offset/4]&mask)!=saved[i]) mismatch++; + u32 other_offset,other_mask; j700_usb_field(i,!old_dp,&other_offset,&other_mask); + assert((registers[offset/4]&~(mask | (i<4 ? other_mask : 0)))==outside[i]); + } + printf("first=%u former_dp_pma=%u masked_mismatches=%u receiver=%#x power=%#x\n", + first,old_dp,mismatch,registers[0xc/4],registers[0x200/4]); + failure |= mismatch!=0; + assert(!usb2_writes&&!reset_writes); + assert(j700_dp_configure(&p,&rate)==0); + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB2)==0); + orient(&p,first); + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==0); + assert(!p.mode_dirty&&p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3); + } + fflush(stdout);assert(!failure); +} +static void restore_controls(void) +{ +#ifdef PROPOSAL + struct apple_atcphy p; + const struct phy_configure_opts_dp rate={.set_rate=true,.set_lanes=true,.link_rate=8100,.lanes=2}; + seed_usb(&p,1); + assert(j700_dp_configure(&p,&rate)==0); + unsigned int valid=p.dp_usb_context_valid,dirty=p.dp_usb_context_dirty; + u32 snapshot[2][27];memcpy(snapshot,p.dp_usb_context,sizeof(snapshot)); + stuck_ack=true; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB2)==-ETIMEDOUT); + assert(p.dp_owned_pmas==1&&p.dp_usb_context_valid==valid&&p.dp_usb_context_dirty==dirty); + assert(!memcmp(snapshot,p.dp_usb_context,sizeof(snapshot))); + recover(); + fail_caller="j700_field_update"; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB2)==-ETIMEDOUT); + assert(p.dp_owned_pmas==1&&p.dp_usb_context_valid==valid); + assert(!memcmp(snapshot,p.dp_usb_context,sizeof(snapshot))); + recover();assert(select_mode(&p,APPLE_ATCPHY_MODE_USB2)==0); + orient(&p,0); + assert(j700_usb_capture(&p,APPLE_ATCPHY_MODE_USB3_DP)==0); + p.dp_owned_pmas=1; + unsigned int before=writes; + assert(j700_usb_restore(&p,APPLE_ATCPHY_MODE_USB3_DP)==-EBUSY&&writes==before); + p.dp_owned_pmas=0;p.dp_idle_pmas=1; + assert(j700_usb_restore(&p,APPLE_ATCPHY_MODE_USB3_DP)==-EBUSY&&writes==before); + p.dp_idle_pmas=0;p.pipe_state=ATCPHY_PIPEHANDLER_STATE_USB3; + assert(j700_usb_restore(&p,APPLE_ATCPHY_MODE_USB3_DP)==-EBUSY&&writes==before); + p.pipe_state=ATCPHY_PIPEHANDLER_STATE_DUMMY; + fail_caller="j700_usb_field_update"; + assert(j700_usb_restore(&p,APPLE_ATCPHY_MODE_USB3_DP)==-ETIMEDOUT); + assert(p.dp_usb_context_valid&1); + assert(p.dp_usb_context_dirty&1); + recover();assert(j700_usb_restore(&p,APPLE_ATCPHY_MODE_USB3_DP)==0); + assert(!(p.dp_usb_context_valid&1)&&!(p.dp_usb_context_dirty&1)); + seed_usb(&p,1); + assert(j700_dp_configure(&p,&rate)==0); + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB2)==0); + orient(&p,0); + u32 old_context[27];memcpy(old_context,p.dp_usb_context[0],sizeof(old_context)); + fail_caller="j700_usb_field_update"; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==-ETIMEDOUT); + assert(p.mode_dirty&&p.pipe_state==ATCPHY_PIPEHANDLER_STATE_DUMMY); + assert((p.dp_usb_context_valid&1)&&(p.dp_usb_context_dirty&1)); + assert(!memcmp(old_context,p.dp_usb_context[0],sizeof(old_context))); + before=writes; + assert(j700_dp_configure(&p,&rate)==-EBUSY&&writes==before); + recover();assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==0); + assert(!p.mode_dirty&&!(p.dp_usb_context_valid&1)); + seed_usb(&p,1); + assert(j700_dp_configure(&p,&rate)==0); + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB2)==0); + orient(&p,0); + blocked_offset=0x9244; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==-EIO); + assert(p.mode_dirty&&p.pipe_state==ATCPHY_PIPEHANDLER_STATE_DUMMY); + assert((p.dp_usb_context_valid&1)&&(p.dp_usb_context_dirty&1)); + before=writes; + assert(j700_dp_configure(&p,&rate)==-EBUSY&&writes==before); + blocked_offset=0; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==0&&!p.mode_dirty); + puts("PASS: actual mux static/power restore failures keep dirty route/context, block training and retry"); + seed_usb(&p,1); + assert(j700_dp_configure(&p,&rate)==0); + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB2)==0); + orient(&p,0); + registers[ACIOPHY_BIST_MAC_USBPHY_STATUS2/4]=BIT(7); + bist_status_stuck=true; bist_faults=0; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==-ETIMEDOUT); + assert(p.mode_dirty && p.pipe_pending); + assert(bist_faults==2 && (registers[ACIOPHY_BIST_MAC_USBPHY_STATUS2/4]&BIT(7))); + assert((registers[0x200/4]&0xf)==0xe && !(registers[0xc/4]&0x44)); + stuck_ack=true; + before=writes; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB2)==-ETIMEDOUT && writes==before); + puts("PASS: restoration never clears simulated BIST/ACK faults or forces readiness"); + for(unsigned int fixed=0;fixed<2;fixed++) { + seed_usb(&p,0);p.fixed_usb2=fixed;hw.has_t8140_dp=false; + before=writes; + assert(j700_usb_capture(&p,APPLE_ATCPHY_MODE_USB3_DP)==0); + assert(j700_usb_restore(&p,APPLE_ATCPHY_MODE_USB3_DP)==0&&writes==before); + hw.has_t8140_dp=true; + } + puts("PASS: park/DP-idle failure retention, immutable snapshot, live/pending-owner restore guards, power readback failure/retry, legacy exclusions"); +#endif +} + +static void inactive_host_gates(void) +{ +#ifdef PROPOSAL + const struct phy_configure_opts_dp rate = { + .set_rate=true, .set_lanes=true, .link_rate=8100, .lanes=2, + }; + unsigned int violations = 0; + for (unsigned int backend = ATCPHY_PIPEHANDLER_STATE_USB3; + backend <= ATCPHY_PIPEHANDLER_STATE_USB4; backend++) { + for (unsigned int entry = 0; entry < 3; entry++) { + struct apple_atcphy p; + seed_usb(&p, 1); + assert(j700_dp_configure(&p, &rate) == 0); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB2) == 0); + orient(&p, 0); + p.host_active = false; + p.pipe_state = backend; + unsigned int before_writes = writes, before_reads = mmio_reads; + unsigned int valid = p.dp_usb_context_valid, dirty = p.dp_usb_context_dirty; + u32 snapshot[2][27]; memcpy(snapshot, p.dp_usb_context, sizeof(snapshot)); + int ret = entry == 0 ? j700_usb_capture(&p, APPLE_ATCPHY_MODE_USB3_DP) : + entry == 1 ? j700_usb_restore(&p, APPLE_ATCPHY_MODE_USB3_DP) : + atcphy_configure(&p, APPLE_ATCPHY_MODE_USB3_DP); + bool blocked = ret == -EBUSY && writes == before_writes && + mmio_reads == before_reads && valid == p.dp_usb_context_valid && + dirty == p.dp_usb_context_dirty && + !memcmp(snapshot, p.dp_usb_context, sizeof(snapshot)); + printf("inactive-host backend=%u entry=%u ret=%d new_writes=%u new_reads=%u blocked=%u\n", + backend, entry, ret, writes-before_writes, mmio_reads-before_reads, blocked); + violations += !blocked; + } + } + fflush(stdout); assert(!violations); + puts("PASS: six inactive-host/live-PIPE entry gates retain snapshots and perform zero MMIO"); +#endif +} +static void failure_diagnostics(void) +{ +#ifdef PROPOSAL + struct apple_atcphy p; + const struct phy_configure_opts_dp rate={.set_rate=true,.set_lanes=true,.link_rate=8100,.lanes=2}; + const char *stages[] = { "stage=static", "stage=power", "stage=receiver" }; + for (unsigned int stage=0; stage<3; stage++) { + seed_usb(&p,1); assert(j700_dp_configure(&p,&rate)==0); + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB2)==0); orient(&p,0); + diagnostic_records=0; last_diagnostic[0]=0; + blocked_offset=stage==0 ? 0x9244 : stage==1 ? 0x200 : 0xc; + int ret=select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP); + assert(ret==(stage==0 ? -EIO : -ETIMEDOUT)); + assert(diagnostic_records==1 && strstr(last_diagnostic,stages[stage])); + assert(strstr(last_diagnostic,"pma=0") && strstr(last_diagnostic,"reg=")); + assert(strstr(last_diagnostic,"mask=") && strstr(last_diagnostic,"expected=")); + assert(strstr(last_diagnostic,"observed=")); + assert(p.mode_dirty && (p.dp_usb_context_valid&1) && (p.dp_usb_context_dirty&1)); + blocked_offset=0; + } + puts("PASS: static/power/receiver errors preserve state and report stage/PMA/register/mask/expected/final observed"); +#endif +} +int main(int argc,char **argv) +{ + if(argc>1 && !strcmp(argv[1],"inactive-host")) { inactive_host_gates(); return 0; } + if(argc>1)return original_fixture_main(argc,argv); + mask_coverage();masked_roundtrip();restore_controls();inactive_host_gates();failure_diagnostics(); + puts("PASS: masked restoration roundtrips; no BIST readiness or USB speed is modeled"); + return 0; +} +''' + logger = r''' +#include +static unsigned int diagnostic_records; +static char last_diagnostic[512]; +static void record_debug(const char *format, ...) +{ + if (!strstr(format, "USB restore stage=")) return; + va_list args; va_start(args,format); + vsnprintf(last_diagnostic,sizeof(last_diagnostic),format,args); + va_end(args); diagnostic_records++; + fputs(last_diagnostic,stdout); +} +#undef dev_dbg +#define dev_dbg(dev, format, ...) record_debug(format, ##__VA_ARGS__) +''' + return ns["prefix"] + logger + "#define PROPOSAL\n" + production + ns["suffix"].replace("int main(", "int original_fixture_main(", 1) + controls + +def pipe(root): + + source = (root / 'drivers/phy/apple/atc.c').read_text() + dp = (root / 'drivers/phy/apple/atc-j700-dp.h').read_text() + + + def block(marker): + start = source.index(marker) + opening = source.index('{', start) + end, depth = opening + 1, 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + + def function(name): + if name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in source: + marker = re.search(r'^static [^\n]+\batcphy_mux_set_locked\(', source, re.M).group() + return block(marker).replace('atcphy_mux_set_locked(', 'atcphy_mux_set(', 1) + return block(re.search(r'^static [^\n]+\b' + name + r'\(', source, re.M).group()) + + + parts = re.findall(r'^#define [^\n]*(?:\\\n[^\n]*)*', source, re.M) + for name in ('atcphy_generation', 'atcphy_lane_mode', 'atcphy_pipehandler_state', 'atcphy_mode'): + parts.append(block('enum ' + name + ' {') + ';') + for name in ('atcphy_mode_configuration', 'atcphy_hw'): + parts.append(block('struct ' + name + ' {') + ';') + parts.append(r''' +struct apple_atcphy { + const struct atcphy_hw *hw; + struct { unsigned char *core, *pipehandler; } regs; + void *dev; + int lock; + bool fixed_usb2, host_active, ss_tunables, dp_tunables, mode_dirty; + unsigned int dp_owned_pmas, dp_idle_pmas; + unsigned int dp_usb_context_valid, dp_usb_context_dirty; + u32 dp_usb_context[2][27]; + bool pipe_pending, usb_off_pending, usb_off_drained; + bool swap_lanes; + enum atcphy_mode mode, typec_mode; + enum atcphy_pipehandler_state pipe_state; +}; +''') + start = source.index('static const struct {\n\tconst struct atcphy_mode_configuration normal;') + parts.append(source[start:source.index('\n};', start) + 3]) + parts.append(r''' +static u32 pipe_regs[0x1000 / 4], core_regs[0x30000 / 4]; +static unsigned int polls, fail_poll, mux_writes, lane_calls; +static bool stuck_ack; +static u32 readl(void *address) { return *(u32 *)address; } +static void mask32(void *address, u32 mask, u32 value) +{ + *(u32 *)address = (*(u32 *)address & ~mask) | value; + if (address == (void *)&pipe_regs[PIPEHANDLER_LOCK_REQ / 4] && !stuck_ack) + pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = *(u32 *)address & PIPEHANDLER_LOCK_EN; + if (address == (void *)&pipe_regs[PIPEHANDLER_MUX_CTRL / 4]) + mux_writes++; +} +static void set32(void *p, u32 v) { mask32(p, 0, v); } +static void clear32(void *p, u32 v) { mask32(p, v, 0); } +static void writel(u32 v, void *p) { mask32(p, ~0U, v); } +static void core_mask32(struct apple_atcphy *p, u32 o, u32 m, u32 v) +{ assert(o < sizeof(core_regs) && o % 4 == 0); mask32(p->regs.core + o, m, v); } +static void core_set32(struct apple_atcphy *p, u32 o, u32 v) { core_mask32(p, o, 0, v); } +static void core_clear32(struct apple_atcphy *p, u32 o, u32 v) { core_mask32(p, o, v, 0); } +#define readl_poll_timeout_atomic readl_poll_timeout +#define readl_poll_timeout(address, value, condition, interval, timeout) \ + ({ (value) = readl(address); polls++; \ + polls == fail_poll || !(condition) ? -ETIMEDOUT : 0; }) +/* Lane configuration is outside this test: count calls and commit its result. */ +/* Observation is read-only and separately tested; retain production caller control flow. */ +static int atcphy_configure(struct apple_atcphy *p, enum atcphy_mode mode); +''') + for name in ('core_read32', 'atcphy_get_mode_config'): + parts.append(function(name)) + # Actual restoration functions and gates; analog lane setup stays outside this PIPE unit. + parts.append(dp[dp.index('/* Masks are the union'):dp.index('/* Retry only the idle operations')]) + parts.append(r''' +static int atcphy_configure(struct apple_atcphy *p, enum atcphy_mode mode) +{ + int ret; + lane_calls++; p->mode_dirty = true; + ret = j700_usb_capture(p, mode); + if (!ret) ret = j700_usb_restore(p, mode); + if (ret) return ret; + p->mode = mode; p->mode_dirty = false; return 0; +} +''') + for name in ('atcphy_pipehandler_lock', 'atcphy_pipehandler_unlock', 'atcphy_pipehandler_check', + 'atcphy_pipehandler_set_mux', 'atcphy_clear_nonselected_phy_reset', + 'atcphy_configure_pipehandler_usb3_host_t8122', + 'atcphy_configure_pipehandler_usb3_host_t8103', + 'atcphy_configure_pipehandler_usb3', 'atcphy_configure_pipehandler_usb4', + 'atcphy_enable_dummy_phy', 'atcphy_configure_pipehandler_dummy', + 'atcphy_configure_pipehandler', 'atcphy_park_pipehandler', + 'atcphy_usb3_set_mode', 'atcphy_mux_set'): + parts.append(function(name)) + production = '\n'.join(parts) + prefix = r''' +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint16_t u16; +#define BIT(n) (1U << (n)) +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#define GENMASK(h, l) ((~0U << (l)) & (~0U >> (31 - (h)))) +#define FIELD_PREP(m, v) (((u32)(v) << __builtin_ctz(m)) & (m)) +#define FIELD_GET(m, v) (((v) & (m)) >> __builtin_ctz(m)) +#define field_prep FIELD_PREP +#define dev_dbg(...) ((void)0) +#define dev_warn(...) ((void)0) +#define dev_err(...) ((void)0) +#define lockdep_assert_held(...) ((void)0) +#define WARN_ON_ONCE(v) assert(!(v)) +#define guard(kind) (void) +#define fallthrough __attribute__((fallthrough)) +#define usleep_range(...) ((void)0) +#define udelay(n) ((void)(n)) +enum { TYPEC_STATE_SAFE, TYPEC_STATE_USB, TYPEC_MODE_USB4, + TYPEC_DP_STATE_C, TYPEC_DP_STATE_D, TYPEC_DP_STATE_E }; +enum { USB_TYPEC_TBT_SID = 0x8087, USB_TYPEC_DP_SID = 0xff01 }; +#define EUDO_USB_MODE_MASK 3U +enum { EUDO_USB_MODE_USB2, EUDO_USB_MODE_USB3, EUDO_USB_MODE_USB4 }; +enum phy_mode { PHY_MODE_USB_HOST, PHY_MODE_USB_DEVICE, PHY_MODE_INVALID }; +struct enter_usb_data { u32 eudo; }; +struct typec_altmode { unsigned int svid; }; +struct typec_mux_state { unsigned long mode; struct typec_altmode *alt; void *data; }; +struct typec_mux_dev { struct apple_atcphy *data; }; +struct phy { struct apple_atcphy *data; }; +#define typec_mux_get_drvdata(mux) ((mux)->data) +#define phy_get_drvdata(phy) ((phy)->data) +''' + suffix = r''' +static struct atcphy_hw hw = { + .gen = ATCPHY_GENERATION_T8122, .has_t8140_dp = true, + .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, + .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, +}; +static void initial(struct apple_atcphy *p, enum atcphy_mode mode, + enum atcphy_pipehandler_state backend) +{ + memset(pipe_regs, 0, sizeof(pipe_regs)); + memset(core_regs, 0, sizeof(core_regs)); + core_regs[ACIOPHY_TOP_PHY_STAT / 4] = ACIOPHY_TOP_PHY_STAT_LN0_UNK0; + *p = (struct apple_atcphy) { + .hw = &hw, .regs.core = (unsigned char *)core_regs, + .regs.pipehandler = (unsigned char *)pipe_regs, + .mode = mode, .typec_mode = mode, .pipe_state = backend, + .ss_tunables = true, .dp_tunables = true, + }; + polls = fail_poll = mux_writes = lane_calls = 0; + stuck_ack = false; +} +static int request(struct apple_atcphy *p, enum atcphy_mode mode) +{ + struct typec_mux_dev mux = { p }; + struct typec_altmode dp = { .svid = USB_TYPEC_DP_SID }; + struct enter_usb_data data = { .eudo = EUDO_USB_MODE_USB4 }; + struct typec_mux_state state = { .mode = TYPEC_STATE_SAFE }; + if (mode == APPLE_ATCPHY_MODE_USB3) + state.mode = TYPEC_STATE_USB; + else if (mode == APPLE_ATCPHY_MODE_USB3_DP || mode == APPLE_ATCPHY_MODE_DP) { + state.mode = mode == APPLE_ATCPHY_MODE_DP ? TYPEC_DP_STATE_C : TYPEC_DP_STATE_D; + state.alt = &dp; + } else if (mode == APPLE_ATCPHY_MODE_USB4) { + state.mode = TYPEC_MODE_USB4; + state.data = &data; + } + return atcphy_mux_set(&mux, &state); +} +static void backend_failure(void) +{ + struct apple_atcphy p; + unsigned int cases = 0; + for (enum atcphy_mode mode = APPLE_ATCPHY_MODE_OFF; mode <= APPLE_ATCPHY_MODE_DP; mode++) { + enum atcphy_pipehandler_state target = atcphy_modes[mode].pipehandler_state; + enum atcphy_pipehandler_state old = target == ATCPHY_PIPEHANDLER_STATE_DUMMY ? + ATCPHY_PIPEHANDLER_STATE_USB3 : ATCPHY_PIPEHANDLER_STATE_DUMMY; + for (unsigned int host = 0; host < 2; host++) { + initial(&p, mode, old); + /* A previous lock ACK cannot be cleared: fatal before mux writes. */ + pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = PIPEHANDLER_LOCK_EN; + stuck_ack = true; + assert(atcphy_configure_pipehandler(&p, host) == -ETIMEDOUT); + assert(p.pipe_state == old && mux_writes == 0); + stuck_ack = false; + assert(atcphy_configure_pipehandler(&p, host) == 0); + assert(p.pipe_state == target && mux_writes == 3); + const unsigned int data[] = { PIPEHANDLER_MUX_CTRL_DATA_DUMMY, + PIPEHANDLER_MUX_CTRL_DATA_USB3, PIPEHANDLER_MUX_CTRL_DATA_USB4 }; + const unsigned int clk[] = { PIPEHANDLER_MUX_CTRL_CLK_DUMMY, + PIPEHANDLER_MUX_CTRL_CLK_USB3, PIPEHANDLER_MUX_CTRL_CLK_USB4 }; + u32 mux = pipe_regs[PIPEHANDLER_MUX_CTRL / 4]; + assert(FIELD_GET(hw.pipe_mux_data, mux) == data[target]); + assert(FIELD_GET(hw.pipe_mux_clk, mux) == clk[target]); + cases++; + } + } + printf("PASS: %u backend/role check-failure and retry cases\n", cases); +} +static void host_lock_failure(void) +{ + struct apple_atcphy p; + for (unsigned int gen = 0; gen < 2; gen++) { + hw.gen = gen ? ATCPHY_GENERATION_T8122 : ATCPHY_GENERATION_T8103; + initial(&p, APPLE_ATCPHY_MODE_USB3, ATCPHY_PIPEHANDLER_STATE_DUMMY); + struct phy phy = { &p }; + stuck_ack = true; + assert(atcphy_usb3_set_mode(&phy, PHY_MODE_USB_HOST, 0) == -ETIMEDOUT); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY && mux_writes == 0); + assert(!(pipe_regs[PIPEHANDLER_LOCK_REQ / 4] & PIPEHANDLER_LOCK_EN)); + stuck_ack = false; + assert(atcphy_usb3_set_mode(&phy, PHY_MODE_USB_HOST, 0) == 0); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3 && mux_writes == 6); + unsigned int before = polls; + assert(atcphy_usb3_set_mode(&phy, PHY_MODE_USB_HOST, 0) == 0); + assert(polls == before && mux_writes == 6); + } + puts("PASS: both USB3 host BIST generations retain fatal lock ACK and retry"); +} +static void mux_retry(void) +{ + struct apple_atcphy p; + const enum atcphy_mode modes[] = { APPLE_ATCPHY_MODE_USB3, APPLE_ATCPHY_MODE_USB3_DP }; + for (unsigned int m = 0; m < 2; m++) { + initial(&p, APPLE_ATCPHY_MODE_USB2, ATCPHY_PIPEHANDLER_STATE_DUMMY); + p.fixed_usb2 = p.host_active = true; + stuck_ack = true; + assert(request(&p, modes[m]) == -ETIMEDOUT); + assert(p.mode == modes[m] && !p.mode_dirty && lane_calls == 1); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY && mux_writes == 0); + p.dp_owned_pmas = 3; /* PIPE retry must not reconfigure an owned DP mode. */ + assert(request(&p, modes[m]) == -ETIMEDOUT); + assert(lane_calls == 1 && p.dp_owned_pmas == 3 && mux_writes == 0); + stuck_ack = false; + assert(request(&p, modes[m]) == 0); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3 && lane_calls == 1); + assert(p.dp_owned_pmas == 3 && mux_writes == 6); + unsigned int before = polls; + assert(request(&p, modes[m]) == 0 && polls == before && lane_calls == 1); + } + puts("PASS: fixed-hub same-mode retry preserves lanes and DP ownership"); +} +static void same_mode_pending(void) +{ + struct apple_atcphy p; + initial(&p, APPLE_ATCPHY_MODE_USB3, ATCPHY_PIPEHANDLER_STATE_DUMMY); + p.fixed_usb2 = p.host_active = true; + stuck_ack = true; + assert(request(&p, APPLE_ATCPHY_MODE_USB3) == -ETIMEDOUT); + assert(lane_calls == 0 && mux_writes == 0); + stuck_ack = false; + assert(request(&p, APPLE_ATCPHY_MODE_USB3) == 0); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3 && lane_calls == 0); + puts("PASS: same-mode pending PIPE setup independently retries"); +} +static void parking_and_guards(void) +{ + struct apple_atcphy p; + initial(&p, APPLE_ATCPHY_MODE_USB3, ATCPHY_PIPEHANDLER_STATE_USB3); + pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = PIPEHANDLER_LOCK_EN; + stuck_ack = true; + assert(atcphy_park_pipehandler(&p) == -ETIMEDOUT); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3 && mux_writes == 0); + stuck_ack = false; + assert(atcphy_park_pipehandler(&p) == 0); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY && mux_writes == 3); + assert(atcphy_park_pipehandler(&p) == 0); + assert(mux_writes == 3); + initial(&p, APPLE_ATCPHY_MODE_USB2, ATCPHY_PIPEHANDLER_STATE_DUMMY); + p.fixed_usb2 = p.host_active = true; + assert(request(&p, APPLE_ATCPHY_MODE_OFF) == 0); + assert(p.mode == APPLE_ATCPHY_MODE_USB2 && lane_calls == 0 && mux_writes == 0); + p.mode_dirty = true; + assert(request(&p, APPLE_ATCPHY_MODE_OFF) == 0 && lane_calls == 1 && !p.mode_dirty); + /* Inactive hosts and ordinary ports retain their deferred PIPE setup. */ + initial(&p, APPLE_ATCPHY_MODE_USB3, ATCPHY_PIPEHANDLER_STATE_DUMMY); + p.fixed_usb2 = true; + assert(request(&p, APPLE_ATCPHY_MODE_USB3) == 0 && mux_writes == 0); + p.fixed_usb2 = false; + p.host_active = true; + assert(request(&p, APPLE_ATCPHY_MODE_USB3) == 0 && mux_writes == 0); + /* Capability rejection must still precede mode configuration. */ + initial(&p, APPLE_ATCPHY_MODE_USB2, ATCPHY_PIPEHANDLER_STATE_DUMMY); + p.fixed_usb2 = p.host_active = true; + p.ss_tunables = false; + assert(request(&p, APPLE_ATCPHY_MODE_USB3) == -EOPNOTSUPP && lane_calls == 0); + p.ss_tunables = true; + p.dp_tunables = false; + assert(request(&p, APPLE_ATCPHY_MODE_USB3_DP) == -EOPNOTSUPP && lane_calls == 0); + assert(request(&p, APPLE_ATCPHY_MODE_USB4) == -EOPNOTSUPP && lane_calls == 0); + puts("PASS: parking retry, SAFE fixed USB2, dirty-mode and capability guards"); +} +static void requested_lock(void) +{ + struct apple_atcphy p; + initial(&p, APPLE_ATCPHY_MODE_USB3, ATCPHY_PIPEHANDLER_STATE_DUMMY); + pipe_regs[PIPEHANDLER_LOCK_REQ / 4] = PIPEHANDLER_LOCK_EN; + stuck_ack = true; + assert(atcphy_pipehandler_lock(&p) == -ETIMEDOUT && polls == 1); + assert(!(pipe_regs[PIPEHANDLER_LOCK_REQ / 4] & PIPEHANDLER_LOCK_EN)); + pipe_regs[PIPEHANDLER_LOCK_REQ / 4] = PIPEHANDLER_LOCK_EN; + pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = PIPEHANDLER_LOCK_EN; + assert(atcphy_pipehandler_lock(&p) == 0 && polls == 2); + puts("PASS: an existing LOCK request must still have actual ACK before success"); +} +static void live_policy_scope(void) +{ + struct apple_atcphy p; + for (unsigned int t8140 = 0; t8140 < 2; t8140++) { + for (unsigned int fixed = 0; fixed < 2; fixed++) { + for (unsigned int active = 0; active < 2; active++) { + hw.has_t8140_dp = t8140; + initial(&p, APPLE_ATCPHY_MODE_USB3, ATCPHY_PIPEHANDLER_STATE_USB3); + p.fixed_usb2 = fixed; + p.host_active = active; + stuck_ack = true; /* Request exists, but ACK never asserts. */ + int ret = atcphy_park_pipehandler(&p); + if (t8140 && fixed && active) { + assert(ret == -ETIMEDOUT && p.mode_dirty && mux_writes == 0); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3); + assert(!pipe_regs[PIPEHANDLER_NONSELECTED_OVERRIDE / 4]); + } else { + assert(ret == 0 && !p.mode_dirty && mux_writes == 3); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); + } + /* Entry ACK release is fatal everywhere, but dirties only live T8140 parking. */ + initial(&p, APPLE_ATCPHY_MODE_USB3, ATCPHY_PIPEHANDLER_STATE_USB3); + p.fixed_usb2 = fixed; + p.host_active = active; + pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = PIPEHANDLER_LOCK_EN; + stuck_ack = true; + assert(atcphy_park_pipehandler(&p) == -ETIMEDOUT); + assert(p.mode_dirty == (t8140 && fixed && active)); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3 && mux_writes == 0); + } + } + } + puts("PASS: eight live/inactive, fixed/ordinary and T8140/legacy LOCK and entry-ACK policy controls"); +} +int main(int argc, char **argv) +{ + assert(argc == 2); + if (!strcmp(argv[1], "backend")) backend_failure(); + else if (!strcmp(argv[1], "host")) host_lock_failure(); + else if (!strcmp(argv[1], "mux")) mux_retry(); + else if (!strcmp(argv[1], "same")) same_mode_pending(); + else if (!strcmp(argv[1], "park")) parking_and_guards(); + else if (!strcmp(argv[1], "requested")) requested_lock(); + else if (!strcmp(argv[1], "scope")) live_policy_scope(); + else assert(false); + return 0; +} +''' + return prefix + production + suffix + +def unwind(root): + + atc = (root / 'drivers/phy/apple/atc.c').read_text() + dp = (root / 'drivers/phy/apple/atc-j700-dp.h').read_text() + + + def function(source, name): + if name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in source: + marker = re.search(r'^static [^\n]+\batcphy_mux_set_locked\(', source, re.M).group() + return block(source, marker).replace('atcphy_mux_set_locked(', 'atcphy_mux_set(', 1) + return block(source, re.search(r'^static [^\n]+\b' + name + r'\(', source, re.M).group()) + + + # Use the production mode table and wrappers too, so SAFE mapping and fast paths + # are tested rather than mirrored by a second implementation. + parts = re.findall(r'^#define [^\n]*(?:\\\n[^\n]*)*', atc, re.M) + for name in ('atcphy_generation', 'atcphy_lane_mode', 'atcphy_pipehandler_state', 'atcphy_mode', + 'atcphy_dp_link_rate'): + parts.append(block(atc, 'enum ' + name + ' {') + ';') + for name in ('atcphy_mode_configuration', 'atcphy_hw'): + parts.append(block(atc, 'struct ' + name + ' {') + ';') + parts.append(r''' +struct apple_atcphy { + const struct atcphy_hw *hw; + struct { unsigned char *core, *lpdptx; } regs; + void *dev; + int lock, dp_link_rate; + bool fixed_usb2, host_active, swap_lanes, ss_tunables, dp_tunables; + bool aux_power_requested, mode_dirty, dp_spread_spectrum, dp_presets_valid[5]; + u32 dp_presets[5][16]; + unsigned int dp_idle_pmas, dp_owned_pmas; + unsigned int dp_usb_context_valid, dp_usb_context_dirty; + u32 dp_usb_context[2][27]; + bool pipe_pending, usb_off_pending, usb_off_drained; + enum atcphy_mode mode, typec_mode; + enum atcphy_pipehandler_state pipe_state; +}; +''') + start = atc.index('static const struct {\n\tconst struct atcphy_mode_configuration normal;') + parts.append(atc[start:atc.index('\n};', start) + 3]) + parts.append(r''' +struct phy_configure_opts_dp { + unsigned int link_rate, lanes, voltage[4], pre[4]; + bool set_rate, set_lanes, set_voltages, ssc; +}; +static u32 registers[0x30000 / 4]; +static unsigned int polls, fail_poll, writes, usb2_on; +static u32 readl(void *address) { return *(u32 *)address; } +static void core_mask32(struct apple_atcphy *p, u32 offset, u32 mask, u32 value) +{ + (void)p; + assert(offset < sizeof(registers) && offset % 4 == 0); + registers[offset / 4] = (registers[offset / 4] & ~mask) | value; + writes++; + if (offset == ATCPHY_POWER_CTRL) + registers[ATCPHY_POWER_STAT / 4] = registers[offset / 4]; + if (offset == 0x2000) { + if (value == 0x10000001) + registers[offset / 4] |= BIT(1); + else { + registers[offset / 4] &= ~BIT(1); + registers[0x7034 / 4] &= ~BIT(3); + } + } +} +static void core_set32(struct apple_atcphy *p, u32 o, u32 v) { core_mask32(p, o, 0, v); } +static void core_clear32(struct apple_atcphy *p, u32 o, u32 v) { core_mask32(p, o, v, 0); } +static void writel(u32 value, void *address) { *(u32 *)address = value; } +static void mask32(void *p, u32 m, u32 v) { * (u32 *)p = (*(u32 *)p & ~m) | v; } +static void set32(void *p, u32 v) { mask32(p, 0, v); } +static void clear32(void *p, u32 v) { mask32(p, v, 0); } +#define readl_poll_timeout_atomic(address, value, condition, interval, timeout) \ + ({ (value) = readl(address); polls++; \ + polls == fail_poll || !(condition) ? -ETIMEDOUT : 0; }) +#define readl_poll_timeout readl_poll_timeout_atomic +static void atcphy_usb2_power_on(struct apple_atcphy *p) { (void)p; usb2_on++; } +static void atcphy_apply_tunables(struct apple_atcphy *p, enum atcphy_mode mode) +{ (void)p; (void)mode; } +static void atc_tunnel_stop_t8103(struct apple_atcphy *p) { (void)p; } +static void atc_tunnel_restore(struct apple_atcphy *p) { (void)p; } +static int atcphy_park_pipehandler(struct apple_atcphy *p) +{ p->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; return 0; } +/* Observation is read-only and separately tested; retain production caller control flow. */ +static int atcphy_configure_pipehandler(struct apple_atcphy *p, bool host) +{ (void)host; p->pipe_state = atcphy_modes[p->mode].pipehandler_state; return 0; } +''') + for name in ('core_read32', 'atcphy_get_mode_config', 'j700_field_update', 'j700_pma_idle'): + parts.append(function(atc, name)) + parts.append(dp) + for name in ('atcphy_configure_lanes', 'j700_aux_enable', 'j700_aux_disable', + 'atcphy_enable_dp_aux', 'atcphy_disable_dp_aux', 'atcphy_power_off', + 'atcphy_power_on', 'atcphy_configure', 'atcphy_mux_set'): + parts.append(function(atc, name)) + production = '\n'.join(parts) + prefix = r''' +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint16_t u16; +#define BIT(n) (1U << (n)) +#define GENMASK(h, l) ((~0U << (l)) & (~0U >> (31 - (h)))) +#define FIELD_PREP(m, v) (((u32)(v) << __builtin_ctz(m)) & (m)) +#define FIELD_GET(m, v) (((v) & (m)) >> __builtin_ctz(m)) +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#define hweight32(v) __builtin_popcount(v) +#define dev_info_once(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define dev_warn(...) ((void)0) +static void report(void *dev, const char *fmt, ...) { (void)dev; (void)fmt; } +#define dev_err report +#define lockdep_assert_held(...) ((void)0) +#define WARN_ON_ONCE(v) assert(!(v)) +#define guard(kind) (void) +#define fallthrough __attribute__((fallthrough)) +#define ndelay(n) ((void)(n)) +#define udelay(n) ((void)(n)) +#define usleep_range(a, b) ((void)(a), (void)(b)) +enum { TYPEC_STATE_SAFE, TYPEC_STATE_USB, TYPEC_MODE_USB4, + TYPEC_DP_STATE_C, TYPEC_DP_STATE_D, TYPEC_DP_STATE_E }; +enum { USB_TYPEC_TBT_SID = 0x8087, USB_TYPEC_DP_SID = 0xff01 }; +#define EUDO_USB_MODE_MASK 3U +enum { EUDO_USB_MODE_USB2, EUDO_USB_MODE_USB3, EUDO_USB_MODE_USB4 }; +struct enter_usb_data { u32 eudo; }; +struct typec_altmode { unsigned int svid; }; +struct typec_mux_state { unsigned long mode; struct typec_altmode *alt; void *data; }; +struct typec_mux_dev { struct apple_atcphy *data; }; +#define typec_mux_get_drvdata(mux) ((mux)->data) +''' + suffix = r''' +static struct atcphy_hw hw = { + .has_t8140_dp = true, .gen = ATCPHY_GENERATION_T8122, + .aciophy_lane_mode = 0x60, .aciophy_crossbar = 0x64, +}; +static void initial(struct apple_atcphy *p, bool fixed, bool swapped) +{ + memset(registers, 0, sizeof(registers)); + registers[AUS_COMMON_DIG_RCAL1 / 4] = AUS_COMMON_DIG_RCAL1_ALL_CODES_DONE; + *p = (struct apple_atcphy) { + .hw = &hw, .regs.core = (unsigned char *)registers, + .regs.lpdptx = (unsigned char *)registers, + .fixed_usb2 = fixed, .swap_lanes = swapped, + .ss_tunables = true, .dp_tunables = true, .dp_presets_valid = { true }, + .mode = fixed ? APPLE_ATCPHY_MODE_USB2 : APPLE_ATCPHY_MODE_OFF, + .pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY, + }; + polls = fail_poll = writes = usb2_on = 0; +} +static int safe(struct apple_atcphy *p) +{ + struct typec_mux_dev mux = { p }; + struct typec_mux_state state = { .mode = TYPEC_STATE_SAFE }; + return atcphy_mux_set(&mux, &state); +} +int main(void) +{ + struct apple_atcphy p; + unsigned int cases = 0; + for (unsigned int fixed = 0; fixed < 2; fixed++) { + for (unsigned int swapped = 0; swapped < 2; swapped++) { + const enum atcphy_mode modes[] = { + APPLE_ATCPHY_MODE_DP, APPLE_ATCPHY_MODE_USB3_DP, APPLE_ATCPHY_MODE_USB3, + }; + for (unsigned int m = 0; m < ARRAY_SIZE(modes); m++) { + unsigned int aux = modes[m] != APPLE_ATCPHY_MODE_USB3 ? 4 : 0; + unsigned int idle = modes[m] == APPLE_ATCPHY_MODE_DP ? 12 : 6; + for (unsigned int fault = 3; fault <= 2 + aux + idle; fault++) { + enum atcphy_mode old = fixed ? APPLE_ATCPHY_MODE_USB2 : APPLE_ATCPHY_MODE_OFF; + initial(&p, fixed, swapped); + fail_poll = fault; + assert(atcphy_configure(&p, modes[m]) == -ETIMEDOUT); + assert(p.mode == old && p.mode_dirty); + if (fault <= 2 + aux) + assert(p.aux_power_requested && !p.dp_idle_pmas); + else + assert(p.dp_idle_pmas); + bool owned_aux = p.aux_power_requested; + unsigned int owned_idle = p.dp_idle_pmas; + polls = 0; + fail_poll = 1; + usb2_on = 0; + assert(safe(&p) == -ETIMEDOUT); + assert(p.mode == old && p.mode_dirty); + assert(p.aux_power_requested == owned_aux && p.dp_idle_pmas == owned_idle); + assert(usb2_on == 0); /* Cleanup failure must not continue into power changes. */ + polls = fail_poll = 0; + assert(safe(&p) == 0); + assert(p.mode == old && !p.mode_dirty); + assert(!p.aux_power_requested && !p.dp_idle_pmas && !p.dp_owned_pmas); + assert(usb2_on == fixed); /* Fixed-hub SAFE restores USB2, never powers it off. */ + polls = writes = 0; + assert(safe(&p) == 0 && polls == 0 && writes == 0); + cases++; + } + } + } + } + /* A failed route change must not expose the old DP mode to link training. */ + initial(&p, false, false); + p.mode = APPLE_ATCPHY_MODE_DP; + p.aux_power_requested = true; + p.dp_owned_pmas = 3; + assert(atcphy_configure(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + unsigned int last_idle_poll = polls - 1; /* The final poll is RCAL, after route idle. */ + initial(&p, false, false); + p.mode = APPLE_ATCPHY_MODE_DP; + p.aux_power_requested = true; + p.dp_owned_pmas = 3; + fail_poll = last_idle_poll; + assert(atcphy_configure(&p, APPLE_ATCPHY_MODE_USB3_DP) == -ETIMEDOUT); + assert(p.mode == APPLE_ATCPHY_MODE_DP && p.mode_dirty && p.dp_idle_pmas); + struct phy_configure_opts_dp rate = { + .set_rate = true, .set_lanes = true, .link_rate = 8100, .lanes = 4, + }; + struct phy_configure_opts_dp training = { .set_voltages = true, .lanes = 4 }; + unsigned int writes_before = writes, idle_before = p.dp_idle_pmas; + unsigned int owned_before = p.dp_owned_pmas; + assert(j700_dp_configure(&p, &rate) == -EBUSY); + assert(j700_dp_configure(&p, &training) == -EBUSY); + assert(writes == writes_before && p.dp_idle_pmas == idle_before); + assert(p.dp_owned_pmas == owned_before && p.aux_power_requested && p.mode_dirty); + /* Explicit stop is allowed, but must leave the route cleanup obligation. */ + struct phy_configure_opts_dp stop = { .set_rate = true }; + assert(j700_dp_configure(&p, &stop) == 0); + assert(p.dp_idle_pmas == idle_before && p.mode_dirty); + fail_poll = polls = 0; + assert(safe(&p) == 0 && !p.mode_dirty && !p.dp_idle_pmas && !p.aux_power_requested); + /* If teardown itself fails, reject training without dropping live ownership. */ + initial(&p, false, false); + p.mode = APPLE_ATCPHY_MODE_DP; + p.aux_power_requested = true; + p.dp_owned_pmas = 3; + fail_poll = 1; + assert(atcphy_configure(&p, APPLE_ATCPHY_MODE_USB3_DP) == -ETIMEDOUT); + assert(p.mode_dirty && p.dp_owned_pmas == 3); + writes_before = writes; + assert(j700_dp_configure(&p, &rate) == -EBUSY); + assert(j700_dp_configure(&p, &training) == -EBUSY); + assert(writes == writes_before && p.dp_owned_pmas == 3 && p.aux_power_requested); + fail_poll = polls = 0; + assert(safe(&p) == 0 && !p.mode_dirty && !p.dp_owned_pmas && !p.aux_power_requested); + /* A cleanup failure also dirties an initially successful mode transition. */ + initial(&p, false, false); + p.mode = APPLE_ATCPHY_MODE_DP; + p.aux_power_requested = true; + fail_poll = 1; + assert(safe(&p) == -ETIMEDOUT && p.mode_dirty && p.aux_power_requested); + fail_poll = polls = 0; + assert(safe(&p) == 0 && !p.mode_dirty && !p.aux_power_requested); + /* Existing main-link ownership survives a failed cleanup of a dirty mode. */ + initial(&p, true, false); + p.mode_dirty = p.aux_power_requested = true; + p.dp_owned_pmas = 3; + fail_poll = 1; + assert(safe(&p) == -ETIMEDOUT); + assert(p.mode_dirty && p.aux_power_requested && p.dp_owned_pmas == 3); + fail_poll = polls = 0; + assert(safe(&p) == 0 && !p.mode_dirty && !p.aux_power_requested && !p.dp_owned_pmas); + /* OFF is not committed when domain sleep fails, and SAFE retries it. */ + initial(&p, false, false); + p.mode = APPLE_ATCPHY_MODE_USB2; + fail_poll = 1; + assert(safe(&p) == -ETIMEDOUT && p.mode == APPLE_ATCPHY_MODE_USB2 && p.mode_dirty); + fail_poll = polls = 0; + assert(safe(&p) == 0 && p.mode == APPLE_ATCPHY_MODE_OFF && !p.mode_dirty); + printf("PASS: %u AUX/PMA fault scenarios, failed cleanup, dirty training guard, SAFE retry, fixed USB2, OFF commit\n", cases); + return 0; +} +''' + return prefix + production + suffix + +def aux(root): + + source = root / 'drivers/phy/apple/atc.c' + text = source.read_text() + parts = [line for line in text.splitlines() if line.startswith(('#define J700_AUX_', '#define J700_PMA_'))] + for name in ('j700_field_update', 'j700_pma_idle', 'j700_aux_enable', 'j700_aux_disable'): + start = text.index('static int ' + name + '(') + begin = text.index('{', start) + depth = 1 + end = begin + 1 + while depth: + depth += (text[end] == '{') - (text[end] == '}') + end += 1 + parts.append(text[start:end]) + production = '\n'.join(parts) + prefix = r''' +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +#define BIT(n) (1U << (n)) +struct apple_atcphy { struct { unsigned char *core; } regs; bool aux_power_requested; }; +struct event { u32 offset, mask, value; } events[32]; +static u32 registers[0x18000 / 4]; +static unsigned int writes, reject; +static void core_mask32(struct apple_atcphy *phy, u32 offset, u32 mask, u32 value) +{ + assert(offset < sizeof(registers) && offset % 4 == 0 && writes < 32); + assert(!(value & ~mask)); + events[writes] = (struct event){offset, mask, value}; + writes++; + if (writes != reject) + registers[offset / 4] = (registers[offset / 4] & ~mask) | value; + (void)phy; +} +static void usleep_range(unsigned int low, unsigned int high) +{ + assert(low && high >= low); +} +#define readl_poll_timeout_atomic(address, value, condition, interval, timeout) \ + ({ (value) = *(u32 *)(address); (condition) ? 0 : -110; }) +''' + suffix = r''' +static void initial(struct apple_atcphy *phy) +{ + memset(registers, 0, sizeof(registers)); + memset(events, 0, sizeof(events)); + registers[J700_AUX_SUPPLY_CONTROL / 4] = 0x80000200; + registers[J700_AUX_ANALOG_CONTROL / 4] = 0x40000001; + *phy = (struct apple_atcphy){ .regs.core = (unsigned char *)registers }; + writes = reject = 0; +} +int main(void) +{ + struct apple_atcphy phy; + initial(&phy); + assert(j700_aux_disable(&phy) == 0 && writes == 0); + assert(j700_aux_enable(&phy) == 0 && phy.aux_power_requested && writes == 4); + assert(registers[J700_AUX_SUPPLY_CONTROL / 4] == 0x8000010f); + assert(registers[J700_AUX_ANALOG_CONTROL / 4] == 0x40000000); + assert(events[0].mask == 3 && events[1].mask == 12); + assert(events[2].mask == 0x300 && events[2].value == 0x100); + assert(events[3].offset == J700_AUX_ANALOG_CONTROL); + writes = 0; + assert(j700_aux_disable(&phy) == 0 && !phy.aux_power_requested && writes == 4); + assert(events[0].offset == J700_AUX_ANALOG_CONTROL); + assert(events[1].mask == 0x300 && events[2].mask == 8 && events[3].mask == 2); + assert(registers[J700_AUX_SUPPLY_CONTROL / 4] == 0x80000305); + assert(registers[J700_AUX_ANALOG_CONTROL / 4] == 0x40000001); + for (unsigned int fault = 1; fault <= 4; fault++) { + initial(&phy); + reject = fault; + assert(j700_aux_enable(&phy) == -110 && phy.aux_power_requested); + reject = 0; + writes = 0; + assert(j700_aux_disable(&phy) == 0 && !phy.aux_power_requested); + initial(&phy); + assert(j700_aux_enable(&phy) == 0); + writes = 0; + reject = fault; + assert(j700_aux_disable(&phy) == -110 && phy.aux_power_requested); + } + initial(&phy); + assert(j700_pma_idle(&phy, 2) == -EINVAL && writes == 0); + for (unsigned int lane = 0; lane < 2; lane++) { + initial(&phy); + registers[J700_PMA_POWER_CONTROL / 4] = 0x800000ee; + registers[J700_PMA_RECEIVER_CONTROL / 4] = 0x5000; + assert(j700_pma_idle(&phy, lane) == 0 && writes == 6); + assert(((registers[J700_PMA_POWER_CONTROL / 4] >> (4 * lane)) & 15) == 1); + assert(((registers[J700_PMA_POWER_CONTROL / 4] >> (4 * (1 - lane))) & 15) == 14); + assert(registers[J700_PMA_POWER_CONTROL / 4] & 0x80000000); + assert(registers[J700_PMA_RECEIVER_CONTROL / 4] == (0x5000U | (0x44U << lane))); + for (unsigned int fault = 1; fault <= 6; fault++) { + initial(&phy); + registers[J700_PMA_POWER_CONTROL / 4] = 0xee; + reject = fault; + assert(j700_pma_idle(&phy, lane) == -110); + } + } + puts("PASS: sequencing, unrelated bits, every failed phase, retained cleanup ownership"); + return 0; +} +''' + return prefix + production + suffix diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py new file mode 100644 index 0000000000000..66c58178be9d4 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -0,0 +1,453 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Actual PM/HCD/mux/probe boundaries using the shared PHY extraction fixture.""" +from pathlib import Path +import os, re, subprocess +import phy +from common import block + +def ownership(root, usb_unreachable=False): + ns=phy.parking(root) + s=ns['atc'];has_pm='static int atcphy_usb3_pm_get(' in s + prod=ns['production'].replace('struct apple_atcphy {','struct apple_atcphy {\n struct device_node *np;',1) + if has_pm: prod=prod.replace('static int atcphy_mux_set(', 'static int atcphy_mux_set_locked(',1) + # Non-mux fixture entry points continue to exercise actual source. Only the new + # wrapper owns PM; the fixture's original helpers call it through this declaration. + prefix=ns['prefix']+'\nstruct device_node;\nstatic int atcphy_mux_set(struct typec_mux_dev *, struct typec_mux_state *);\n' + prefix=prefix.replace('struct typec_mux_dev { struct apple_atcphy *data; };', + '''struct typec_mux_dev { struct apple_atcphy *data; + int (*set)(struct typec_mux_dev *, struct typec_mux_state *); }; + struct typec_mux { unsigned int num_mux_devs; struct typec_mux_dev **mux_devs; }; + #define IS_ERR_OR_NULL(p) (!(p))''') + prefix=prefix.replace('#define guard(kind) (void)',r''' +struct lock_guard { int *lock; }; +static struct lock_guard lock_take(int *lock) { assert(!*lock); *lock=1; return (struct lock_guard){lock}; } +static void lock_release(struct lock_guard *g) { assert(*g->lock==1); *g->lock=0; } +#define guard(kind) struct lock_guard __attribute__((cleanup(lock_release))) _guard = lock_take +''') + # Basic USB/device APIs are modeled, but get/put/wrapper and all parking/ + # restoration bodies are extracted verbatim from the chosen public source. + pmfixture=r''' +#define CONFIG_USB 1 +#define READ_ONCE(x) (x) +#define IS_REACHABLE(x) USB_REACHABLE +struct device_node { int id; }; +struct device { int refs; }; +struct platform_device { struct device dev; }; +struct usb_device; +struct usb_hcd; +struct usb_bus { struct device *sysdev, *controller; struct usb_hcd *hcd; struct usb_device *root_hub; }; +struct usb_hcd { unsigned int rh_registered:1; + struct usb_bus self; unsigned long flags; + struct usb_hcd *shared_hcd,*primary_hcd; struct {bool pending;} died_work; }; +enum { HCD_FLAG_RH_RUNNING,HCD_FLAG_DEAD,HCD_FLAG_POLL_RH,HCD_FLAG_HW_ACCESSIBLE }; +static int __attribute__((unused)) hcd_root_hub_lock; +#define spin_lock_irqsave(lock, flags) do { assert(!*(lock));*(lock)=1;(flags)=0; } while(0) +#define spin_unlock_irqrestore(lock, flags) do { assert(*(lock));*(lock)=0;(void)(flags); } while(0) +#define clear_bit(bit, p) (*(p)&=~(1UL<<(bit))) +#define set_bit(bit, p) (*(p)|=1UL<<(bit)) +#define usb_hcd_is_primary_hcd(hcd) (!(hcd)->primary_hcd||(hcd)->primary_hcd==(hcd)) +#define usb_set_device_state(u, s) ((u)->state=(s)) +#define usb_kick_hub_wq(u) ((void)(u)) +#define schedule_work(w) ((w)->pending=true) +#define bus_to_hcd(bus) ((bus)->hcd) +#define HCD_DEAD(hcd) (!!((hcd)->flags&(1UL<flags&(1UL<refs>0);d->refs--; } +static struct usb_device *usb_get_dev(struct usb_device *u) { u->refs++;return u; } +static void usb_put_dev(struct usb_device *u) { assert(u->refs>0);u->refs--; } +static int usb_for_each_dev(void *data,int (*fn)(struct usb_device *,void *)) { if(root_missing)return 0;for(unsigned int i=0;i<4;i++){int r=fn(&roots[i],data);if(r)return r;}return 0; } +static void usb_lock_device(struct usb_device *u) { assert(u==&roots[3]&&!root_locked);root_locked=true;if(detach_on_lock)u->state=0; } +static void usb_unlock_device(struct usb_device *u) { assert(u==&roots[3]&&root_locked);root_locked=false; } +static int pm_runtime_resume_and_get(struct device *d) { + assert(d==&roots[3].dev&&root_locked&&!pm_phy->lock);resumes++; + /* A re-entrant PHY lock acquisition during resume must remain possible. */ + struct lock_guard g=lock_take(&pm_phy->lock);lock_release(&g); + if(resume_error)return resume_error; + pm_refs++;stuck_ack=false;if(detach_on_resume)roots[3].state=0; + if(inaccessible_on_resume)clear_bit(HCD_FLAG_HW_ACCESSIBLE,&hcd.flags); + if(dead_on_resume)set_bit(HCD_FLAG_DEAD,&hcd.flags); + return 0; +} +static void pm_runtime_put_noidle(struct device *d) { assert(d==&roots[3].dev&&pm_refs==1);pm_refs--; } +static void usb_mark_last_busy(struct usb_device *u) { assert(u==&roots[3]&&root_locked&&!pm_phy->lock); } +static void pm_runtime_put_autosuspend(struct device *d) { assert(d==&roots[3].dev&&root_locked&&!pm_phy->lock&&pm_refs==1);pm_refs--; } +''' + suffix=ns['suffix'].replace('int main(', 'int __attribute__((unused)) fixture_main(',1) + controls=r''' +static void prepare(struct apple_atcphy *p,bool fixed,bool t8140,bool swap) +{ + hw.has_t8140_dp=t8140;initial(p,fixed,false,swap); + p->np=&ctrl_node;pm_phy=p; + hcd=(struct usb_hcd){.rh_registered=true,.flags=1UL<lock&&!root_locked&&!pm_refs&&!node_refs&&!controller.dev.refs); + for(unsigned int i=0;i<4;i++)assert(!roots[i].refs); +} +int main(void) +{ + struct apple_atcphy p; +#if !USB_REACHABLE + { prepare(&p,true,true,false);unsigned int before=writes; + struct typec_mux_dev provider={.data=&p,.set=atcphy_mux_set}; + struct typec_mux_dev *providers[]={&provider};struct typec_mux mux={1,providers}; + struct typec_mux_state state={.mode=TYPEC_STATE_SAFE}; + assert(typec_mux_set(&mux,&state)==-ENODEV&&writes==before);balanced(&p); + puts("PASS USB-unreachable fixed route blocks before MMIO");return 0; } +#endif + /* HC death before rh_registered is set need not mark the root NOTATTACHED. */ + prepare(&p,true,true,false);hcd.rh_registered=false;usb_hc_died(&hcd); + assert(roots[3].state!=USB_STATE_NOTATTACHED&&HCD_DEAD(&hcd));unsigned int before_hcd=writes; + int hcd_ret=safe(&p);fprintf(stderr,"dead HCD admission returned %d, PM resumes %d\n",hcd_ret,resumes); + assert(hcd_ret==-ENODEV&&writes==before_hcd&&resumes==0);balanced(&p); + if(HCD_DEAD_ONLY){puts("PASS dead HCD rejected before PM/MMIO");return 0;} + for(unsigned int gate=0;gate<2;gate++){ + prepare(&p,true,true,false);before_hcd=writes; + if(gate==0)inaccessible_on_resume=1; + if(gate==1)dead_on_resume=1; + assert(safe(&p)==-ENODEV&&writes==before_hcd);balanced(&p); + } + for(unsigned int swap=0;swap<2;swap++){ + prepare(&p,true,true,swap);stuck_ack=true; + assert(safe(&p)==0);assert(resumes==1);balanced(&p); + } + /* Resume failure must return before ANY lane/core programming. */ + prepare(&p,true,true,false);unsigned int before=writes;resume_error=-EIO; + unsigned int valid=p.dp_usb_context_valid,dirty=p.dp_usb_context_dirty,owned=p.dp_owned_pmas; + assert(safe(&p)==-EIO&&writes==before&&p.dp_usb_context_valid==valid&&p.dp_usb_context_dirty==dirty&&p.dp_owned_pmas==owned);balanced(&p); + resume_error=0;assert(safe(&p)==0);balanced(&p); + /* PM resume is NOT proof of ACK. Real parking timeout stays fatal. */ + prepare(&p,true,true,false);fail_caller="atcphy_pipehandler_lock";before=writes; + assert(safe(&p)==-ETIMEDOUT&&writes==before&&p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3&&p.mode_dirty);balanced(&p); + fail_caller=NULL;assert(safe(&p)==0);balanced(&p); + for(unsigned int missing=0;missing<5;missing++){ + prepare(&p,true,true,false);before=writes; + if(missing==0)graph_missing=true; + if(missing==1)controller_missing=true; + if(missing==2)root_missing=true; + if(missing==3)detach_on_lock=1; + if(missing==4)detach_on_resume=1; + assert(safe(&p)==(missing==0||missing>=3?-ENODEV:-EAGAIN)&&writes==before);balanced(&p); + } + /* Root appears after independent fixed-host probe: missing root is retryable. */ + prepare(&p,true,true,false);root_missing=true;before=writes; + assert(safe(&p)==-EAGAIN&&writes==before);balanced(&p); + root_missing=false;assert(safe(&p)==0);balanced(&p); + /* BIST faults are independent of the runtime clock wake; never force readiness. */ + for(unsigned int swap=0;swap<2;swap++){ + prepare(&p,true,true,swap);assert(safe(&p)==0);balanced(&p); + orient(&p,!swap);bist_status_stuck=true; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==-ETIMEDOUT&&p.mode_dirty&&p.pipe_pending);balanced(&p); + bist_status_stuck=false;recover();assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==0);balanced(&p); + } + /* Scope and clean same-mode preserve MMIO guards; fixed no-op briefly owns PM. */ + struct typec_altmode alt={.svid=USB_TYPEC_DP_SID}; + struct typec_mux_dev mux={&p};struct typec_mux_state same={.mode=TYPEC_DP_STATE_D,.alt=&alt}; + prepare(&p,false,true,false);before=writes;assert(atcphy_mux_set(&mux,&same)==0&&resumes==0&&writes==before);balanced(&p); + prepare(&p,true,false,false);before=writes;assert(atcphy_mux_set(&mux,&same)==0&&resumes==0&&writes==before);balanced(&p); + prepare(&p,true,true,false);before=writes; + assert(atcphy_mux_set(&mux,&same)==0&&writes==before&&resumes==1);balanced(&p); + puts("PASS actual PM wrapper + actual parking/restore: both PMAs, gated clock, PM error/retry, fatal ACK/retry, graph/controller/root/removal gates, legacy/ordinary and cached mode"); + return 0; +} +''' + helpers='' + if has_pm: + helpers=ns['block'](s,'struct atcphy_usb3_pm {')+';\n' + helpers+='#if IS_REACHABLE(CONFIG_USB)\n'+ns['function'](s,'atcphy_usb3_pm_find')+'\n#endif\n' + helpers+=ns['function'](s,'atcphy_usb3_pm_get')+'\n'+ns['function'](s,'atcphy_usb3_pm_put')+'\n' + # The fixture extractor adapts the locked body; extract the wrapper directly. + marker=re.search(r'^static int atcphy_mux_set\(',s,re.M).group() + helpers+=ns['block'](s,marker)+'\n' + else: + # Parent runs its ACTUAL mux callback from production, without a substituted wrapper. + pmfixture=pmfixture.replace('static ', 'static __attribute__((unused)) ') + if usb_unreachable: + pmfixture=pmfixture.replace('static ', 'static __attribute__((unused)) ') + typec_source=(root/'drivers/usb/typec/mux.c').read_text() + typec_caller=ns['block'](typec_source,'int typec_mux_set(')+'\n' + hcd_source=(root/'drivers/usb/core/hcd.c').read_text() + hcd_death=ns['block'](hcd_source,'void usb_hc_died (')+'\n' + text=prefix+prod+'\n#define USB_REACHABLE '+str(int(not usb_unreachable))+'\n#define HCD_DEAD_ONLY '+str(int(False))+'\n'+pmfixture+helpers+typec_caller+hcd_death+suffix+controls + return text + +def probe(root): + def source(path): + return (root/path).read_text() + + + def function(text, name): + match = re.search(r'^static (?:int|void)\s*\n?' + name + r'\(', text, re.M) + assert match, name + start = match.start() + opening = text.index('{', start) + depth = 1 + end = opening + 1 + while depth: + depth += (text[end] == '{') - (text[end] == '}') + end += 1 + return text[start:end] + + + prefix = r''' +#include +#include +#include +#include +#include +#include +#include +#define EPROBE_DEFER 517 +#define IS_ERR(p) ((intptr_t)(p)<0) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +#define IS_ERR_OR_NULL(p) (!(p)||IS_ERR(p)) +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +#define guard(kind) (void) +#define TYPEC_STATE_SAFE 0 +#define INIT_DELAYED_WORK(p,fn) ((p)->pending=false) +#define atomic_set(p,n) (*(p)=(n)) +struct work { bool pending; }; +struct fwnode_handle { int refs; }; +struct typec_mux_state { void *alt,*data; unsigned long mode; }; +struct tps6598x { int lock; void *port; }; +struct notifier_block { int (*notifier_call)(struct notifier_block *,unsigned long,void *); }; +''' + pm = source('drivers/usb/typec/tipd/cd321x-pm.h').replace('#include ', '') + mocks = r''' +struct cd321x { + struct tps6598x tps; + struct work update_work,resume_work; + int link_event; + struct cd321x_pm_state pm; + void *mux,*tbt_switch,*port_altmode_dp,*port_altmode_tbt; + struct fwnode_handle *connector_fwnode; + struct typec_mux_state state; + struct notifier_block tbt_notifier; + bool state_valid,tbt_notifier_registered; +}; +static struct fwnode_handle connector; +static int mux_error, notifier_error, switch_error; +static int ports,altmodes,muxes,switches,notifiers,stop_calls; +static bool displayport=true; +static int tps6598x_register_port(struct tps6598x *t,struct fwnode_handle *f) { + (void)f;t->port=t;ports++;return 0; +} +static int cd321x_register_port_altmodes(struct cd321x *c) { + c->port_altmode_dp=c;c->port_altmode_tbt=c;altmodes+=2;return 0; +} +static void *fwnode_typec_mux_get(struct fwnode_handle *f) { (void)f;muxes++;return &muxes; } +static bool fwnode_property_present(struct fwnode_handle *f,const char *s) { (void)f;(void)s;return displayport; } +static struct fwnode_handle *fwnode_find_reference(struct fwnode_handle *f,const char *s,int n) { + (void)f;(void)s;(void)n;connector.refs++;return &connector; +} +static void __attribute__((unused)) fwnode_handle_put(struct fwnode_handle *f) { + if(f){assert(f->refs>0);f->refs--;} +} +static void *fwnode_typec_thunderbolt_switch_get(struct fwnode_handle *f) { + (void)f;if(switch_error)return (void *)(intptr_t)switch_error;switches++;return &switches; +} +static int typec_set_mode(void *p,unsigned long m) { (void)p;assert(m==TYPEC_STATE_SAFE);return mux_error; } +static int cd321x_tbt_notify(struct notifier_block *n,unsigned long e,void *d) { (void)n;(void)e;(void)d;return 0; } +static int typec_thunderbolt_switch_register_notifier(void *s,struct notifier_block *n) { + (void)s;(void)n;if(notifier_error)return notifier_error;notifiers++;return 0; +} +static void typec_thunderbolt_switch_put(void *s) { assert(s&&switches>0);switches--; } +static void typec_mux_put(void *m) { assert(m&&muxes>0);muxes--; } +static void typec_unregister_altmode(void *a) { if(a){assert(altmodes>0);altmodes--;} } +static void typec_unregister_port(void *p) { assert(p&&ports>0);ports--; } +static void cd321x_stop_work(struct cd321x *c) { + stop_calls++;if(c->tbt_notifier_registered){notifiers--;c->tbt_notifier_registered=false;} + cd321x_pm_remove(&c->pm);c->resume_work.pending=c->update_work.pending=false; +} +''' + core = source('drivers/usb/typec/tipd/core.c') + production = function(core, 'cd321x_register_port') + '\n' + function(core, 'cd321x_unregister_port') + controls = r''' +static void reset(struct cd321x *c) { + memset(c,0,sizeof(*c));connector.refs=0; + mux_error=notifier_error=switch_error=0;stop_calls=0; + assert(!ports&&!altmodes&&!muxes&&!switches&&!notifiers); +} +static void balanced(void) { + assert(!ports&&!altmodes&&!muxes&&!switches&&!notifiers&&!connector.refs); +} +int main(void) { + struct cd321x c; + struct cd321x_pm_state budget; + cd321x_pm_init(&budget);assert(cd321x_pm_ready(&budget)); + for(unsigned int i=0;iport=t;ports++;return 0;', '(void)f;t->port=&probe_port;ports++;return 0;') + # Real class dispatch requires the port's mux; its allocator remains a mock. + declarations = 'struct typec_port { struct typec_mux *mux; };\nstatic struct typec_port probe_port;\n' + dispatch = block((root / 'drivers/usb/typec/class.c').read_text(), 'int typec_set_mode(') + marker = port.index('static int tps6598x_register_port(') + port = port[:marker] + declarations + dispatch + '\n' + port[marker:] + import re + # Only helpers used at this boundary belong in this translation unit; + # the separate probe group exercises the complete retry state machine. + used = {'cd321x_pm_new_connection', 'cd321x_pm_init', 'cd321x_pm_remove', 'cd321x_pm_ready', + 'cd321x_pm_begin_read', 'cd321x_pm_snapshot_ready', + 'cd321x_pm_complete', 'cd321x_pm_can_update'} + for match in list(re.finditer(r'^static inline (?:void|bool) (cd321x_\w+)\(', port, re.M))[::-1]: + if match.group(1) not in used: + body = block(port, match.group()) + port = port[:match.start()] + port[match.start() + len(body):] + port = port.replace('static int mux_error, notifier_error, switch_error;', + 'static int notifier_error, switch_error;') + existing = set(re.findall(r'^\s*#define (\w+)', unit, re.M)) + port = '\n'.join(line for line in port.splitlines() + if not (line.startswith('#define ') and line.split()[1].split('(')[0] in existing)) + controls = r""" +static void port_balance(void) +{ + assert(!ports&&!altmodes&&!muxes&&!switches&&!notifiers&&!connector.refs); +} +int main(void) +{ + struct apple_atcphy p; + struct cd321x c={}; + struct typec_mux_dev provider={.data=&p,.set=atcphy_mux_set}; + struct typec_mux_dev *providers[]={&provider}; + struct typec_mux route={1,providers};probe_port.mux=&route; + prepare(&p,true,true,false);root_missing=true;unsigned int before=writes; + assert(cd321x_register_port(&c.tps,&connector)==-EPROBE_DEFER); + assert(!c.state_valid&&!resumes&&writes==before);balanced(&p);port_balance(); + memset(&c,0,sizeof(c));root_missing=false; + assert(cd321x_register_port(&c.tps,&connector)==0 && c.state_valid && resumes==1); + assert(cd321x_pm_ready(&c.pm) && cd321x_pm_begin_read(&c.pm)); + cd321x_pm_snapshot_ready(&c.pm);cd321x_pm_complete(&c.pm); + assert(cd321x_pm_can_update(&c.pm)); + balanced(&p);cd321x_unregister_port(&c.tps);port_balance(); + memset(&c,0,sizeof(c));prepare(&p,true,true,false);notifier_error=-ENOMEM; + assert(cd321x_register_port(&c.tps,&connector)==-ENOMEM); + balanced(&p);port_balance();notifier_error=0; + memset(&c,0,sizeof(c));prepare(&p,true,true,false);resume_error=-EIO;before=writes; + assert(cd321x_register_port(&c.tps,&connector)==0 && !c.state_valid && writes==before); + balanced(&p);cd321x_unregister_port(&c.tps);port_balance(); + puts("PASS: actual CD321x probe -> typec_set_mode -> typec_mux_set -> bounded PM wrapper; delayed root reprobe and rollback references"); +} +""" + return unit + '\n' + port + '\n' + controls diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py new file mode 100644 index 0000000000000..32585f1755845 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py @@ -0,0 +1,1125 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Actual parser, bounded rate admission, catalog replacement and HPD.""" +from pathlib import Path +import copy, struct +from common import block, source_defines + +def parser_fixture(root): + + base = root / 'drivers/gpu/drm/apple' + fixture = {'MaxHorizontalImageSize': 60, 'MaxVerticalImageSize': 33, 'Chromaticity': {'Red': {'X': 44608, 'Y': 20288}, 'Green': {'X': 16960, 'Y': 40448}, 'Blue': {'X': 9344, 'Y': 5184}}, 'DefaultWhitePoint': {'X': 20544, 'Y': 21568, 'Gamma': 144179}, 'WhitePoints': [{'X': 20544, 'Y': 21568, 'Gamma': 144179}], 'DefaultColorSpaceIsSRGB': True, 'SupportsPQEOTF': True, 'ProductAttributes': {'LegacyManufacturerID': 1138, 'ProductID': 2895, 'SerialNumber': 305419896, 'WeekOfManufacture': 7, 'YearOfManufacture': 2013, 'AlphanumericSerialNumber': 'ANON000000001', 'ProductName': 'QG270 S3', 'ManufacturerID': 'ACR'}} + + + class ObjectPairs(list): + pass + + + def wire(value, out): + out += b'\0' * (-len(out) % 4) + if isinstance(value, bool): + out += struct.pack('', '') + timing_fixture = [{ + "HorizontalAttributes": {"Total": 2200, "FrontPorch": 88, "SyncWidth": 44, + "Active": 1920, "PreciseSyncRate": 4423680, "SyncPolarity": 1, "PixelRepetition": 0}, + "VerticalAttributes": {"Total": 1125, "FrontPorch": 4, "SyncWidth": 5, + "Active": 1080, "PreciseSyncRate": 3932160, "SyncPolarity": 1, "PixelRepetition": 0}, + "ColorModes": [{"ID": 85, "Score": 1, "IsVirtual": False, "PixelEncoding": 0, + "Colorimetry": 10, "Depth": 10, "DynamicRange": 0, "EOTF": 0}], + "ID": 62, "Score": 16224, "IsPreferred": True, "IsInterlaced": False, "IsVirtual": False, + }] + timing_raw = bytearray(struct.pack(' +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; +typedef int64_t s64; +typedef u16 __le16; +typedef u16 __be16; +typedef u32 __le32; +#define __packed __attribute__((packed)) +#undef static_assert +#define static_assert(c) _Static_assert(c, #c) +#define IS_ENABLED(x) 0 +#define U32_MAX UINT32_MAX +#define U16_MAX UINT16_MAX +#define S64_MAX INT64_MAX +#define S64_MIN INT64_MIN +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#define round_up(n, a) (((n) + (a) - 1) & ~((a) - 1)) +#define ERR_PTR(n) ((void *)(intptr_t)(n)) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +#define IS_ERR(p) ((uintptr_t)(p) > (uintptr_t)-4096) +#define IS_ERR_OR_NULL(p) (!(p) || IS_ERR(p)) +#define GFP_KERNEL 0 +#define kmalloc(n, f) malloc(n) +#define kcalloc(n, s, f) calloc(n,s) +#define kfree(p) free(p) +#define mul_u32_u32(a, b) ((u64)(u32)(a) * (u32)(b)) +#define DIV_ROUND_CLOSEST_ULL(n, d) (((n) + (d) / 2) / (d)) +#define DIV_ROUND_CLOSEST(n, d) (((n) + (d) / 2) / (d)) +#define trace_iomfb_color_mode(...) ((void)0) +#define trace_iomfb_parse_mode_fail(...) ((void)0) +#define trace_iomfb_parse_mode_success(...) ((void)0) +#define trace_iomfb_timing_mode(...) ((void)0) +#define DRM_MODE_TYPE_DRIVER 1 +#define DRM_MODE_TYPE_PREFERRED 2 +#define DRM_MODE_FLAG_PHSYNC 1 +#define DRM_MODE_FLAG_NHSYNC 2 +#define DRM_MODE_FLAG_PVSYNC 4 +#define DRM_MODE_FLAG_NVSYNC 8 +#define cpu_to_le16(v) (v) +#define cpu_to_le32(v) (v) +#define cpu_to_be16(v) __builtin_bswap16(v) +struct drm_display_mode { + unsigned int type, flags; + int clock, vdisplay, vsync_start, vsync_end, vtotal; + int hdisplay, hsync_start, hsync_end, htotal, width_mm, height_mm; +}; +static void drm_mode_set_name(struct drm_display_mode *mode) { (void)mode; } +''' + stubs = r''' +struct drm_edid { struct edid raw; }; +static const struct drm_edid *drm_edid_alloc(const void *data, size_t size) +{ + assert(size == 128); + struct drm_edid *result = malloc(sizeof(*result)); + memcpy(&result->raw, data, size); + return result; +} +static void drm_edid_free(const struct drm_edid *edid) { free((void *)edid); } +static bool drm_edid_valid(const struct drm_edid *edid) +{ + const u8 *raw = (const u8 *)&edid->raw; + unsigned int sum = 0; + for (unsigned int i = 0; i < 128; i++) sum += raw[i]; + return !(sum & 255) && !raw[126] && !memcmp(raw, "\0\xff\xff\xff\xff\xff\xff\0", 8); +} +''' + suffix = r''' +static void epochs(void) +{ + struct dcp_metadata_epoch e = {0}; + struct dcp_metadata_ticket before = dcp_metadata_begin(&e); + assert(!dcp_metadata_matches(&e, before)); + dcp_metadata_epoch_connection(&e, 1, true); + struct dcp_metadata_ticket current = dcp_metadata_begin(&e); + assert(dcp_metadata_matches(&e, current)); + assert(!dcp_metadata_matches(&e, before)); /* Never retag pre-association metadata. */ + dcp_metadata_epoch_connection(&e, 1, true); + assert(dcp_metadata_matches(&e, current)); + dcp_metadata_epoch_connection(&e, 2, false); + assert(!dcp_metadata_matches(&e, current)); + dcp_metadata_epoch_connection(&e, 3, true); + assert(!dcp_metadata_matches(&e, dcp_metadata_begin(&e))); /* Later attachment ineligible. */ + e = (struct dcp_metadata_epoch){0}; + dcp_metadata_epoch_connection(&e, 9, true); + current = dcp_metadata_begin(&e); + dcp_metadata_epoch_connection(&e, 10, true); /* A route change also invalidates. */ + assert(!dcp_metadata_matches(&e, current)); +} +int main(int argc, char **argv) +{ + if (argc == 3 && !strcmp(argv[1], "--property")) { + FILE *f=fopen(argv[2],"rb"); assert(f && !fseek(f,0,SEEK_END)); + long n=ftell(f); assert(n>20 && !fseek(f,0,SEEK_SET)); + unsigned char *b=malloc(n); assert(fread(b,n,1,f)==1); fclose(f); + assert(dcp_property_complete(b,n-1,n)); + assert(dcp_property_complete(b,n-1,n-1)); + for(unsigned int cut=1;cut<=16;cut++) + assert(!dcp_property_complete(b,n-1-cut,n-cut)); + assert(!dcp_property_complete(b,n-1,n+1)); + free(b); return 0; + } + assert(argc == 4 || argc == 5); + /* A missing final string byte must not be supplied by zero allocation. */ + unsigned char incomplete[]={0xd3,0,0,0,1,0,0,0x82,1,0,0,0x89,0x42}; + assert(dcp_property_complete(incomplete,sizeof(incomplete),sizeof(incomplete))); + assert(!dcp_property_complete(incomplete,12,13)); + assert(!dcp_property_complete(NULL,0,0)); + epochs(); + timing_proof(); + FILE *f = fopen(argv[1], "rb"); + assert(f && !fseek(f, 0, SEEK_END)); + long length = ftell(f); + assert(length > 0 && !fseek(f, 0, SEEK_SET)); + void *data = malloc(length); + assert(fread(data, length, 1, f) == 1); + fclose(f); + struct dcp_parse_ctx ctx; + struct dcp_display_metadata metadata, unchanged; + memset(&metadata, 0xa5, sizeof(metadata)); unchanged = metadata; + assert(!parse(data, length, &ctx)); + int ret = parse_display_metadata(&ctx, &metadata); + if (atoi(argv[2])) { + assert(ret == -EINVAL && !memcmp(&metadata, &unchanged, sizeof(metadata))); + free(data); return 0; + } + assert(!ret); + assert(dcp_property_complete(data,length,length)); + size_t aligned=round_up((size_t)length,4); + unsigned char *padded=malloc(aligned+4); + memset(padded,0xa5,aligned+4); memcpy(padded,data,length); + assert(dcp_property_complete(padded,aligned,aligned+1)); + assert(!dcp_property_complete(data,length-1,length)); + assert(!dcp_property_complete(padded,aligned,aligned+2)); + assert(!dcp_property_complete(padded,aligned+4,aligned+5)); + assert(!dcp_property_complete(padded,aligned-1,aligned+1)); + free(padded); + struct dcp_parse_ctx timing_ctx; + unsigned int count = 0; + assert(!parse(timing_blob, sizeof(timing_blob), &timing_ctx)); + struct dcp_display_mode *timings = enumerate_modes(&timing_ctx, &count, + metadata.width_cm * 10, metadata.height_cm * 10, 0, false, true); + assert(!IS_ERR(timings) && count == 1 && timings[0].metadata_timing_valid); + struct drm_display_mode mode = timings[0].mode; + assert(mode.flags == (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC)); + assert(mode.clock == 148500 && mode.hdisplay == 1920 && mode.vdisplay == 1080); + free(timings); + const struct drm_edid *edid = dcp_metadata_edid(&metadata, &mode); + assert(!IS_ERR(edid)); + f = fopen(argv[3], "wb"); assert(f); + assert(fwrite(&edid->raw, 128, 1, f) == 1); fclose(f); + drm_edid_free(edid); + mode.flags = 0; + assert(IS_ERR(dcp_metadata_edid(&metadata, &mode))); + mode.flags = DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC; + mode.clock++; + assert(IS_ERR(dcp_metadata_edid(&metadata, &mode))); + free(data); + if (argc == 5) { + f=fopen(argv[4],"rb"); assert(f && !fseek(f,0,SEEK_END)); + length=ftell(f); assert(length>0 && !fseek(f,0,SEEK_SET)); + data=malloc(length); assert(fread(data,length,1,f)==1); fclose(f); + assert(!parse(data,length,&timing_ctx)); count=0; + timings=enumerate_modes(&timing_ctx,&count,600,330,0,false,true); + assert(!IS_ERR(timings)); bool found60=false, found120=false; + for(unsigned int i=0;iusb4; } +static void schedule_work(struct work_struct *w) { w->scheduled++; } +static const struct edid *drm_edid_raw(const struct drm_edid *e) { return (const void *)e->data; } +static bool dcpavserv_edid_service_current(struct apple_dcp *d, u64 e) { + assert(d->dcpavserv.lock); return e==d->dcpavserv.epoch && d->dcpavserv.enabled && d->dcpavserv.service; +} +''' + callback_stubs=r''' +#define le64_to_cpu(v) (v) +#define le32_to_cpu(v) (v) +#define LINK_RATE_HBR3 0x1e +static int configure_error; +static int dptxport_call_set_active_lane_count(struct apple_epic_service *s, const void *data, size_t size, void *reply, size_t room) { + if (configure_error) return configure_error; + if(sizecookie)->lane_count=((const struct dptxport_apcall_set_active_lane_count *)data)->lane_count; return 0; +} +static int dptxport_call_set_link_rate(struct apple_epic_service *s, const void *data, size_t size, void *reply, size_t room) { + if(configure_error) return configure_error; + if(sizecookie)->link_rate=((const struct dptxport_apcall_link_rate *)data)->link_rate; return 0; +} +static bool dcp_firmware_metadata_enabled(struct apple_dcp *d) { return d->hw.iomfb_method_profile==2; } +#define dcp_trace_metadata(d,p,v) ((void)0) +''' + callback_types='typedef u64 __le64;\n'+callback_types + test=r''' +static void checksum(u8 *p) { unsigned sum=0; for(unsigned i=0;i<127;i++) sum+=p[i]; p[127]=-sum; } +static void qualify(struct apple_dcp *d, struct apple_epic_service *s) { + struct dptxport_apcall_link_rate rate={.link_rate=0x1e}, rate_reply={}; + struct dptxport_apcall_set_active_lane_count lanes={.lane_count=2}, lanes_reply={}; + assert(!dptxport_call_qualified_config(s,false,&rate,sizeof(rate),&rate_reply,sizeof(rate_reply))); + assert(!dptxport_call_qualified_config(s,true,&lanes,sizeof(lanes),&lanes_reply,sizeof(lanes_reply))); + (void)d; +} +static bool current(struct apple_dcp *d,const struct dcp_display_mode *m) { + d->hpd_mutex=d->dcpavserv.lock=d->metadata.lock=true; + bool result=dcp_180hz_mode_current(d,m); + d->hpd_mutex=d->dcpavserv.lock=d->metadata.lock=false; + return result; +} +int main(int argc,char **argv) { + struct dcp_metadata_epoch epoch={}; + dcp_metadata_epoch_connection(&epoch,7,true); + assert(dcp_metadata_matches(&epoch,dcp_metadata_begin(&epoch))); + PARSER_CHECKS + struct drm_edid raw={}; u8 *p=raw.data; + struct dcp_display_mode m={}; + if(argc==2) { FILE *f=fopen(argv[1],"rb"); assert(f); assert(fread(p,1,256,f)==256 && fgetc(f)==EOF); fclose(f); } + else { + memcpy(p,"\0\xff\xff\xff\xff\xff\xff\0",8); + p[8]=4;p[9]=0x72;p[10]=0x4f;p[11]=0x0b;p[18]=1;p[19]=4;p[20]=0xa0;p[126]=1; + u8 *d=p+54; unsigned clock=41409;d[0]=clock;d[1]=clock>>8; + d[2]=1920&255;d[3]=160;d[4]=0x70;d[5]=1080&255;d[6]=26;d[7]=0x40; + d[8]=48;d[9]=32;d[10]=0x35;d[17]=0x1e; + p[75]=0xfc;memcpy(p+77,"QG270 S3\n ",13); + u8 *e=p+128;e[0]=2;e[1]=3;e[2]=12;e[4]=0x67;e[5]=0xd8;e[6]=0x5d;e[7]=0xc4;e[8]=1;e[9]=120;e[10]=0x80; + checksum(p);checksum(e); + } + /* Get the advertised DTD's porch/polarity, without copying identity data. */ + const u8 *dt=NULL; + for(unsigned off=54;off+18<=126;off+=18) if((p[off]|p[off+1]<<8)==41409)dt=p+off; + for(unsigned off=p[130];off+18<=127;off+=18) if((p[128+off]|p[129+off]<<8)==41409)dt=p+128+off; + assert(dt); + m.mode=(struct drm_display_mode){.clock=414090,.hdisplay=1920,.htotal=2080,.vdisplay=1080,.vtotal=1106}; + m.mode.hsync_start=1920+(dt[8]|(dt[11]&0xc0)<<2);m.mode.hsync_end=m.mode.hsync_start+(dt[9]|(dt[11]&0x30)<<4); + m.mode.vsync_start=1080+(dt[10]>>4|(dt[11]&0xc)<<2);m.mode.vsync_end=m.mode.vsync_start+((dt[10]&15)|(dt[11]&3)<<4); + m.mode.flags=(dt[17]&2?DRM_MODE_FLAG_PHSYNC:DRM_MODE_FLAG_NHSYNC)|(dt[17]&4?DRM_MODE_FLAG_PVSYNC:DRM_MODE_FLAG_NVSYNC); + assert(dcp_180hz_edid(p,256,&m.mode)); + assert(!dcp_180hz_edid(p,255,&m.mode)); + u8 saved[256];memcpy(saved,p,256); + p[10]^=1;checksum(p);assert(!dcp_180hz_edid(p,256,&m.mode));memcpy(p,saved,256); + p[8]^=1;checksum(p);assert(!dcp_180hz_edid(p,256,&m.mode));memcpy(p,saved,256); + unsigned hf=0;for(unsigned off=132;off<128+p[130];off+=(p[off]&31)+1) if(p[off+1]==0xd8 && p[off+2]==0x5d && p[off+3]==0xc4)hf=off; + assert(hf);p[hf+5]=82;checksum(p+128);assert(!dcp_180hz_edid(p,256,&m.mode));memcpy(p,saved,256); + p[127]^=1;assert(!dcp_180hz_edid(p,256,&m.mode));memcpy(p,saved,256); + /* Serial changes do not participate in identity matching. */ + p[12]^=0xff;checksum(p);assert(dcp_180hz_edid(p,256,&m.mode));memcpy(p,saved,256); + assert(dcp_180hz_payload(0x1e,2,414090));assert(!dcp_180hz_payload(0x14,2,414090));assert(!dcp_180hz_payload(0x1e,1,414090)); + struct apple_dcp d={.dev=&d,.phy=&raw,.active_typec_route=&m,.typec_cable_connected=true,.typec_generation=7,.nr_modes=1}; + struct apple_connector con={.dcp=&d,.connected=true,.drm_edid=&raw,.drm_edid_generation=7,.drm_edid_service_epoch=1,.base.registration_state=1}; + struct apple_epic ep={.dcp=&d,.endpoint=0x2c,.wire_v2=true}; + struct apple_epic_service s={.ep=&ep,.cookie=&d.dptxport[0],.enabled=true}; + d.connector=&con;d.hw.iomfb_method_profile=2;d.metadata.epoch.connected=true;d.metadata.epoch.generation=7; + d.dcpavserv.enabled=true;d.dcpavserv.service=&raw;d.dcpavserv.epoch=1; + d.dptxport[0]=(struct dptx_port){.enabled=true,.connected=true,.atcphy=d.phy,.service=&s}; + m.unsafe_rgb8_ticket=(struct dcp_metadata_ticket){.associated=true,.generation=7};m.unsafe_rgb8_service_epoch=1; + m.sdr_rgb=(struct dcp_color_mode){.depth=8,.format=0,.range=0,.eotf=0,.colorimetry=10}; + assert(!current(&d,&m));qualify(&d,&s);assert(current(&d,&m));assert(con.hotplug_wq.scheduled==1); + #define DENY(field,value) do { __typeof__(field) old=field;field=value;assert(!current(&d,&m));field=old;assert(current(&d,&m)); } while(0) + DENY(d.typec_generation,8);DENY(con.drm_edid_generation,6);DENY(d.dcpavserv.epoch,3);DENY(con.drm_edid_service_epoch,3); + /* Timings may precede the very first AV announcement. Only fresh epoch-1 + * EDID plus epoch-1 callback proof may admit this initial receipt. */ + m.unsafe_rgb8_service_epoch=0;m.automatic_service_pending=true;assert(current(&d,&m)); + d.dcpavserv.epoch=3;con.drm_edid_service_epoch=3;qualify(&d,&s); + assert(!current(&d,&m)); /* Initial receipt never survives retirement. */ + m.unsafe_rgb8_service_epoch=3;m.automatic_service_pending=false;assert(current(&d,&m)); + d.dcpavserv.epoch=1;con.drm_edid_service_epoch=1;m.unsafe_rgb8_service_epoch=1;qualify(&d,&s); + DENY(con.drm_edid_firmware_derived,true);DENY(d.usb4,true);DENY(d.dptx_tunnel,true); + DENY(d.quiescing,true);DENY(d.metadata.epoch.connected,false);DENY(d.rate_proof.connection_sequence,1); + DENY(d.rate_proof.rate.value,0x14);DENY(d.rate_proof.lanes.value,1);DENY(ep.endpoint,0x28); + DENY(m.sdr_rgb.depth,10);DENY(m.sdr_rgb.colorimetry,1);DENY(m.sdr_rgb.range,1);DENY(m.mode.clock,414091); + struct dptxport_apcall_link_rate rate={.link_rate=0x1e}, response={}; + configure_error=-EIO; + assert(dptxport_call_qualified_config(&s,false,&rate,sizeof(rate),&response,sizeof(response))==-EIO); + assert(!current(&d,&m));configure_error=0;qualify(&d,&s); + assert(dptxport_call_qualified_config(&s,false,&rate,sizeof(rate)-1,&response,sizeof(response))==-EINVAL); + assert(!current(&d,&m));qualify(&d,&s); + dcp_firmware_metadata_connection(&d,7,false);dcp_firmware_metadata_connection(&d,7,true); + assert(!current(&d,&m));qualify(&d,&s);assert(!current(&d,&m)); + m.automatic_connection_sequence=d.rate_proof.connection_sequence;assert(current(&d,&m)); + dcp_firmware_metadata_hotplug(&d,true);dcp_firmware_metadata_hotplug(&d,false); + assert(!current(&d,&m)); + dcp_firmware_metadata_connection(&d,7,true);m.automatic_connection_sequence=d.rate_proof.connection_sequence;con.edid_invalidation_pending=false;qualify(&d,&s);assert(current(&d,&m)); + struct dcp_180hz_token old=dcp_180hz_config_begin(&s,false);d.typec_generation++; + dcp_180hz_config_complete(&s,false,old,0x1e,true);assert(!d.rate_proof.rate.valid);d.typec_generation--;qualify(&d,&s); + old=dcp_180hz_config_begin(&s,true);d.dcpavserv.epoch+=2; + dcp_180hz_config_complete(&s,true,old,2,true);assert(d.rate_proof.lanes.valid);assert(!current(&d,&m));d.dcpavserv.epoch-=2;qualify(&d,&s); + old=dcp_180hz_config_begin(&s,false);dcp_180hz_invalidate(&d); + dcp_180hz_config_complete(&s,false,old,0x1e,true);assert(!current(&d,&m));qualify(&d,&s); + old=dcp_180hz_config_begin(&s,false);dcp_180hz_config_complete(&s,false,old,0x1e,false);assert(!current(&d,&m)); + d.dcpavserv.epoch=0;old=dcp_180hz_config_begin(&s,false);d.dcpavserv.epoch=1; + dcp_180hz_config_complete(&s,false,old,0x1e,true);assert(d.rate_proof.rate.valid);qualify(&d,&s);assert(current(&d,&m)); + puts("PASS: actual binary parser, Acer EDID, capacity, callback success, late-proof event, generation/AV lifetime and foreign-sink controls"); +} +'''.replace('PARSER_CHECKS', checks) + unit=namespace['prefix']+namespace['metadata']+namespace['epoch']+namespace['header']+namespace['edid_header']+namespace['parser']+blobs+policy+proof+state+source+callback_types+callback_stubs+callback+connection+test + return {**locals(), "block": block} + +def publication(root): + + n = policy(root) + n.update({key:n['namespace'][key] for key in ('prefix','metadata','epoch','header','edid_header','parser')}) + base = n['base'] + block = n['block'] + av = (base/'epic/dpavservep.c').read_text() + iomfb = (base/'iomfb.c').read_text() + template = (base/'iomfb_template.c').read_text() + + state = n['state'] + state = state.replace('struct apple_connector {', '''struct drm_device { void *dev; struct { bool mutex; } mode_config; }; +struct drm_connector { struct drm_device *dev; int registration_state; void *state; + struct { struct { unsigned min_vfreq,max_vfreq; } monitor_range; } display_info; +}; +struct apple_connector {''') + state = state.replace(' struct { int registration_state; } base;', ' struct drm_connector base;') + state = state.replace('struct apple_epic_service { struct apple_epic *ep; void *cookie; bool enabled; };', '''struct apple_epic_service { struct apple_epic *ep; void *cookie; bool enabled,torndown; + unsigned users,channel; struct { u64 generation,epoch; } admission; +};''') + state = state.replace('struct { bool lock, enabled; void *service; u64 epoch; } dcpavserv;', 'struct { bool lock, enabled; void *service; u64 epoch; int enable_completion; } dcpavserv;') + state = state.replace('u64 typec_generation; unsigned int nr_modes;', '''u64 typec_generation; unsigned int nr_modes; + struct dcp_display_mode *modes; + struct apple_epic *dcpavservep; + int fw_compat,width_mm,height_mm,fixed_connector_type; unsigned notch_height; + struct { size_t length,received; void *data; bool contiguous,payload_complete; + u64 av_service_epoch,automatic_connection_sequence; bool automatic_service_pending; + struct dcp_metadata_ticket metadata_ticket,mode_ticket; + } chunks;''') + + start = state.index('static bool dcpavserv_edid_service_current(') + state = state[:start] + + stubs = r''' +#define WRITE_ONCE(p,v) ((p)=(v)) +#define platform_device apple_dcp +#define platform_get_drvdata(p) ((struct apple_dcp *)(p)) +#define to_apple_connector(p) ((struct apple_connector *)((char *)(p)-offsetof(struct apple_connector,base))) +#define xchg(p,v) __extension__ ({ __typeof__(*(p)) old=*(p); *(p)=(v); old; }) +#define DRM_MODE_MATCH_TIMINGS 1 +#define DRM_MODE_MATCH_CLOCK 2 +#define DCP_FIRMWARE_H17P 2 +#define DPAVSERV_ENDPOINT_V2 0x2a +#define DRM_CONNECTOR_eDP 14 +#define DRM_MODE_CONNECTOR_eDP 14 +#define DCP_FW_VER 26 +#define DCP_FW_VERSION(a,b,c) (a) +#define SZ_1M (1024*1024) +#define kzalloc(n,f) calloc(1,n) +#define dev_err(...) ((void)0) +#define dev_warn(...) ((void)0) +#define dev_info(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define trace_dcpavserv_init(...) ((void)0) +#include +static bool *hpd_lock,*av_lock; +static void mutex_lock(bool *p) { assert(!*p); if(p==hpd_lock) assert(!*av_lock); *p=true; } +static void mutex_unlock(bool *p) { assert(*p); *p=false; } +static void complete(int *p) { (*p)++; } +static void reinit_completion(int *p) { *p=0; } +static struct apple_epic_service *afk_service_get(struct apple_epic_service *s) { if(!s->enabled) return NULL; s->users++; return s; } +static void afk_service_put(struct apple_epic_service *s) { assert(s->users); s->users--; } +static void afk_service_retire(struct apple_epic_service *s) { assert(!*hpd_lock); s->enabled=false; s->torndown=true; } +static void dpavservep_detach(struct apple_dcp *d); +static struct apple_dcp *probing; +static struct drm_edid transport_edid; +static unsigned raw_allocations,published,reads,retirement_during_read; +static bool fail_update,invalidate_during_duplicate,replace_catalog_during_duplicate; +static void conflicting_catalog_arrived(void); +static const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *d,u64 generation,u64 *epoch) { + assert(!d->hpd_mutex && !d->dcpavserv.lock && !d->metadata.lock); + if(!d->dcpavserv.enabled || !d->dcpavserv.service || generation!=d->typec_generation) return ERR_PTR(-ENODEV); + *epoch=d->dcpavserv.epoch;reads++; + struct drm_edid *e=malloc(sizeof(*e)); assert(e); *e=transport_edid;raw_allocations++; + if(retirement_during_read) { retirement_during_read--;dpavservep_detach(d); } + return e; +} +static void drm_edid_free(const struct drm_edid *e) { if(e) { assert(raw_allocations);raw_allocations--;free((void *)e); } } +static int drm_edid_connector_update(struct drm_connector *c,const struct drm_edid *e) { + assert(c->dev->mode_config.mutex); (void)e; return fail_update?-EIO:0; +} +static const struct drm_edid *dcp_firmware_edid(struct apple_dcp *d,u64 *generation) { (void)d; (void)generation; return ERR_PTR(-ENODATA); } +static bool dcp_firmware_metadata_current(struct apple_dcp *d,u64 generation) { (void)d; (void)generation; return false; } +static bool dcp_has_panel(struct apple_dcp *d) { return d->fixed_connector_type==DRM_CONNECTOR_eDP; } +static struct drm_display_mode *drm_mode_duplicate(struct drm_device *dev,const struct drm_display_mode *m) { + assert(dev->mode_config.mutex);struct drm_display_mode *copy=malloc(sizeof(*copy));assert(copy);*copy=*m; + if(invalidate_during_duplicate) { invalidate_during_duplicate=false;dcp_180hz_invalidate(probing); } + if(replace_catalog_during_duplicate) { replace_catalog_during_duplicate=false;conflicting_catalog_arrived(); } + return copy; +} +static void drm_mode_probed_add(struct drm_connector *c,struct drm_display_mode *m) { + assert(c->dev->mode_config.mutex && probing->hpd_mutex && probing->dcpavserv.lock && probing->metadata.lock); + published++;free(m); +} +static void drm_mode_destroy(struct drm_device *dev,struct drm_display_mode *m) { (void)dev; free(m); } +static bool drm_mode_match(const struct drm_display_mode *a,const struct drm_display_mode *b,unsigned flags) { + (void)flags;return a->clock==b->clock && a->hdisplay==b->hdisplay && a->vdisplay==b->vdisplay && a->htotal==b->htotal && a->vtotal==b->vtotal && a->hsync_start==b->hsync_start && a->hsync_end==b->hsync_end && a->vsync_start==b->vsync_start && a->vsync_end==b->vsync_end && a->flags==b->flags; +} +static void drm_connector_set_vrr_capable_property(struct drm_connector *c,bool capable) { (void)c; (void)capable; } +static void dcp_retry_placeholder_edid(struct apple_dcp *d,const struct drm_edid *e) { (void)d; (void)e; } +static unsigned pending_probes; +void dcp_180hz_retry(struct apple_dcp *d) { (void)d;pending_probes++; } +static void dcp_firmware_metadata_begin(struct apple_dcp *d) { + unsigned long flags;spin_lock_irqsave(&d->metadata.lock,flags); + d->chunks.metadata_ticket=dcp_metadata_begin(&d->metadata.epoch);spin_unlock_irqrestore(&d->metadata.lock,flags); +} +static void dcp_firmware_metadata_timing(struct apple_dcp *d) { (void)d; } +static void dcp_firmware_metadata_attributes(struct apple_dcp *d,const struct dcp_display_metadata *m) { (void)d; (void)m; } +static void dcp_set_dimensions(struct apple_dcp *d) { (void)d; } +struct dcp_set_dcpav_prop_end_req { char key[64]; }; +''' + transport = '\n'.join(block(av,marker) for marker in [ + 'static void dcpavserv_notify_edid(', 'static bool dcpavserv_identity(', + 'static bool dcpavserv_admission_current(', 'static int dcpavserv_prepare(', + 'static void dcpavserv_init(', 'static void dcpavserv_invalidate_edid(', + 'static void dcpavserv_teardown(', 'void dpavservep_detach(', + 'bool dcpavserv_edid_current(', 'bool dcpavserv_edid_service_current(', + ]) + mode_helpers = iomfb[iomfb.index('static struct dcp_display_mode *dcp_modes_snapshot('):iomfb.index('int dcp_get_modes(')] + publication = block(iomfb,'static void dcp_clear_connector_edid(')+'\n'+block(iomfb,'static void dcp_consume_edid_invalidation(')+mode_helpers+block(iomfb,'int dcp_get_modes(')+block(iomfb,'struct dcp_display_mode *lookup_mode(') + # Drive malformed replacements through the same chunk/end ingress as firmware. + # The dictionary cache is a boundary substitute and takes ownership of the blob. + receipt = r''' +struct dcp_set_dcpav_prop_chunk_req { u32 offset,length; u8 data[4096]; }; +#define dcp_connector_update_dict(con,key,chunks) do { (void)(con); (void)(key); free((chunks)->data); } while (0) +''' + '\n'.join(block(template,marker) for marker in [ + 'static u8 dcpep_cb_prop_start(', 'static u8 dcpep_cb_prop_chunk(', + 'static bool dcpep_process_chunks(', 'static u8 dcpep_cb_prop_end(']) + + main_prefix = n['test'].split('int main(int argc,char **argv) {',1)[1].split(' struct apple_dcp d={',1)[0] + helpers = n['test'].split('int main(int argc,char **argv) {',1)[0] + helpers = helpers.replace(block(helpers, 'static void qualify('), '') + fixture = r''' +static struct apple_dcp d; +static struct apple_connector con; +static struct drm_device drm; +static struct apple_epic ep,av_ep; +static struct apple_epic_service s,av_services[3]; +static void reset(void) { + assert(!raw_allocations); + d=(struct apple_dcp){.dev=&d,.phy=&transport_edid,.active_typec_route=&con,.typec_cable_connected=true,.typec_generation=7,.fw_compat=2,.width_mm=600,.height_mm=330}; + drm=(struct drm_device){}; + con=(struct apple_connector){.dcp=&d,.connected=true,.base={.dev=&drm,.registration_state=1}}; + ep=(struct apple_epic){.dcp=&d,.endpoint=0x2c,.wire_v2=true}; + av_ep=(struct apple_epic){.dcp=&d,.endpoint=0x2a,.wire_v2=true}; + s=(struct apple_epic_service){.ep=&ep,.cookie=&d.dptxport[0],.enabled=true}; + for(unsigned i=0;i<3;i++) av_services[i]=(struct apple_epic_service){.ep=&av_ep,.enabled=true,.channel=1+4*i}; + d.connector=&con;d.hw.iomfb_method_profile=2;d.metadata.epoch.connected=true;d.metadata.epoch.generation=7;dcp_metadata_epoch_connection(&d.metadata.epoch,7,true); + d.dptxport[0]=(struct dptx_port){.enabled=true,.connected=true,.atcphy=d.phy,.service=&s}; + d.dcpavservep=&av_ep;hpd_lock=&d.hpd_mutex;av_lock=&d.dcpavserv.lock;probing=&d; + published=reads=pending_probes=0;fail_update=invalidate_during_duplicate=false; +} +static void admit(unsigned index) { + struct apple_epic_service *a=&av_services[index]; + assert(!dcpavserv_prepare(a,"dcpav-service-epic","AppleDCPDP2HDMI",a->channel)); + dcpavserv_init(a,"dcpav-service-epic","AppleDCPDP2HDMI",a->channel); + assert(d.dcpavserv.enabled && d.dcpavserv.service==a); +} +static void start_receipt(void) { + u32 size=sizeof(input_0);assert(dcpep_cb_prop_start(&d,&size)); + memcpy(d.chunks.data,input_0,size);d.chunks.received=size;d.chunks.contiguous=true; +} +static void finish_receipt(void) { + struct dcp_set_dcpav_prop_end_req request={.key="TimingElements"}; + assert(dcpep_process_chunks(&d,&request));assert(d.nr_modes==1 && d.modes[0].automatic_rgb8_quirk); + free(d.chunks.data);d.chunks.data=NULL; +} +static void timings(void) { start_receipt();finish_receipt(); } +static void callbacks(bool lanes) { + if(lanes) { struct dptxport_apcall_set_active_lane_count a={.lane_count=2},b={}; + assert(!dptxport_call_qualified_config(&s,true,&a,sizeof(a),&b,sizeof(b))); } + else { struct dptxport_apcall_link_rate a={.link_rate=0x1e},b={}; + assert(!dptxport_call_qualified_config(&s,false,&a,sizeof(a),&b,sizeof(b))); } +} +static void probe(bool expected) { + drm.mode_config.mutex=true;unsigned before=published; + int count=dcp_get_modes(&con.base);drm.mode_config.mutex=false; + assert(count==(int)expected && published==before+(unsigned)expected); + if(d.nr_modes) assert(!!lookup_mode(&d,&d.modes[0].mode)==expected); + assert(!d.hpd_mutex && !d.dcpavserv.lock && !d.metadata.lock); +} +static void cleanup(void) { + drm_edid_free(con.drm_edid);con.drm_edid=NULL;free(d.modes);d.modes=NULL; + free(d.chunks.data);d.chunks.data=NULL; + if(d.dcpavserv.service)dpavservep_detach(&d); + assert(!raw_allocations); +} +static void permutations(void) { + unsigned runs=0; + for(unsigned a=0;a<4;a++) for(unsigned b=0;b<4;b++) for(unsigned c=0;c<4;c++) for(unsigned e=0;e<4;e++) { + if(a==b || a==c || a==e || b==c || b==e || c==e)continue; + reset();unsigned order[]={a,b,c,e};unsigned seen=0; + for(unsigned i=0;i<4;i++) { + unsigned notifications=con.hotplug_wq.scheduled; + switch(order[i]) { case 0:callbacks(false);break;case 1:callbacks(true);break;case 2:timings();break;case 3:admit(0);break; } + seen|=1u<notifications); + } + assert(current(&d,&d.modes[0]));cleanup();runs++; + } + assert(runs==24); +} +''' + main=r''' + transport_edid=raw; + /* Runtime EDID porches are encoded into the timing fixture by this + * harness, without embedding identity bytes or fabricating tickets. */ + permutations(); + reset();callbacks(false);callbacks(true);timings();probe(false);admit(0);probe(true); + assert(d.rate_proof.rate.valid && d.rate_proof.lanes.valid); /* No second callback. */ + struct dcp_display_mode old=d.modes[0]; + dcpavserv_teardown(&av_services[0]);probe(false); + admit(1);probe(false);assert(!memcmp(&old,d.modes,sizeof(old))); /* No retagging. */ + free(d.modes);d.modes=NULL;timings();probe(true); /* Existing physical config survives AV-only CLOSE. */ + dcpavserv_teardown(&av_services[0]);probe(true); /* Old CLOSE cannot revoke the new owner. */ + cleanup(); + + reset();callbacks(false);callbacks(true);timings();admit(0);probe(true); + dcp_firmware_metadata_connection(&d,8,false);d.typec_generation=8;d.typec_cable_connected=false;dpavservep_detach(&d);probe(false); + d.typec_generation=9;d.typec_cable_connected=true;dcp_firmware_metadata_connection(&d,9,true); + free(d.modes);d.modes=NULL;timings();admit(1);probe(false); /* Fresh EDID cannot rescue old PHY configuration. */ + callbacks(false);probe(false);callbacks(true);probe(true);cleanup(); + + reset();callbacks(false);callbacks(true);start_receipt();dpavservep_detach(&d);admit(0);finish_receipt();probe(false);cleanup(); + reset();callbacks(false);callbacks(true);admit(0);start_receipt();dcpavserv_teardown(&av_services[0]);admit(1);finish_receipt();probe(false);cleanup(); + reset();callbacks(false);callbacks(true);dpavservep_detach(&d);timings();admit(0);probe(true);cleanup(); /* Legitimate receipt awaiting post-retirement admission. */ + + reset();callbacks(false);callbacks(true);timings();admit(0);retirement_during_read=1;probe(false);cleanup(); + reset();callbacks(false);callbacks(true);timings();admit(0);invalidate_during_duplicate=true;probe(false);cleanup(); + reset();callbacks(false);callbacks(true);timings();admit(0);fail_update=true;probe(false);assert(!con.drm_edid);cleanup(); + reset();callbacks(false);callbacks(true);timings();admit(0);transport_edid.data[10]^=1;checksum(transport_edid.data);probe(false);cleanup();transport_edid=raw; + reset();callbacks(false);callbacks(true);timings();admit(0);configure_error=-EIO; + struct dptxport_apcall_link_rate req={.link_rate=0x1e},resp={};assert(dptxport_call_qualified_config(&s,false,&req,sizeof(req),&resp,sizeof(resp))==-EIO); + configure_error=0;probe(false);cleanup(); + puts("PASS: 24 callback/timing/AV arrival permutations and real get_modes/lookup publication; early configuration needs no second callback; CLOSE, physical reconnect, receipt epochs and publication races deny stale proof"); +} +''' + unit=n['prefix']+n['metadata']+n['epoch']+n['header']+n['edid_header']+n['parser']+n['blobs']+n['policy']+n['proof']+state+stubs+transport+n['source']+n['callback_types']+n['callback_stubs']+n['callback']+n['connection']+publication+receipt+helpers+fixture+'int main(int argc,char **argv) {'+main_prefix+main + # Conflicts are scanned across the full binary receipt, including filtered + # siblings. Exercise both orders, replacement receipts and equivalent aliases. + base_fixture=copy.deepcopy(n['fixture']) + conflict_blobs='' + conflict_checks='' + for kind in ('color', 'timing', 'other-color'): + for reverse in (False, True): + candidate=copy.deepcopy(base_fixture) + other=copy.deepcopy(candidate[0]) + if kind=='color': + other_color=copy.deepcopy(other['ColorModes'][0]) + other_color.update(Depth=10,EOTF=2,Score=999999) + candidate[0]['ColorModes'].insert(0 if reverse else 1,other_color) + elif kind=='other-color': + other.update(ID=79,DSCRequiredColorElementIDs=[80]) + other['ColorModes'][0].update(Depth=10,EOTF=2) + candidate.insert(0 if reverse else 1,other) + else: + other['HorizontalAttributes'].update(Total=2200,FrontPorch=88,SyncWidth=44) + other['VerticalAttributes'].update(Total=1125,FrontPorch=4,SyncWidth=5,PreciseSyncRate=60<<16) + other.update(DSCRequiredColorElementIDs=[80]) + candidate.insert(0 if reverse else 1,other) + raw_blob=bytearray(struct.pack('before);assert(!lookup_mode(&d,&saved));free(old); } + cleanup(); + if(late) { + reset();callbacks(false);callbacks(true);admit(0);timings();probe(true);old=d.modes;saved=old[0].mode; + u32 size=len;assert(dcpep_cb_prop_start(&d,&size));memcpy(d.chunks.data,blob,size); + d.chunks.received=size;d.chunks.contiguous=true; + before=con.hotplug_wq.scheduled; + replace_catalog_during_duplicate=true;probe(false); + assert(con.hotplug_wq.scheduled>before); + assert(!current(&d,old) && !lookup_mode(&d,&saved));free(old);cleanup(); + } +} +static void conflicting_receipts(void) { + CONFLICT_CHECKS + reset();callbacks(false);callbacks(true);admit(0);receipt_blob(equivalent_ids,sizeof(equivalent_ids));probe(true);cleanup(); + puts("PASS: conflicting color/timing IDs, discarded siblings, both receipt orders and late conflicts deny getter/lookup; equivalent color aliases remain eligible"); +} +""".replace('CONFLICT_CHECKS',conflict_checks) + unit=unit.replace('int main(int argc,char **argv) {',conflict_blobs+conflict_helper+'int main(int argc,char **argv) {') + unit=unit.replace(' permutations();',' permutations();conflicting_receipts();') + # Execute the production metadata notifier so a manual probe cannot hide a + # missing connector refresh after the last eligible mode is withdrawn. + unit=unit.replace('bool lock, attributes_valid, timing_valid, hpd_seen; struct dcp_metadata_epoch epoch;', + 'bool lock, attributes_valid, timing_valid, hpd_seen; struct dcp_metadata_ticket attributes_ticket,timing_ticket; struct drm_display_mode preferred; struct dcp_metadata_epoch epoch;') + dcp_source=(base/'dcp.c').read_text() + metadata_notifier='\n'.join(block(dcp_source,marker) for marker in ( + 'static void dcp_firmware_metadata_notify(', + 'static bool dcp_firmware_metadata_complete(', + 'void dcp_firmware_metadata_timing(')) + unit=unit.replace(block(unit,'static void dcp_firmware_metadata_timing('), + '#define dcp_trace_metadata(d,p,v) ((void)0)\nstatic bool dcp_firmware_metadata_enabled(struct apple_dcp *d);\n'+metadata_notifier) + + failed_receipts = r''' +static bool malformed_receipt(const char *key,unsigned fault) { + u32 size=fault==1 ? 3 : fault==2 ? 4 : sizeof(input_0); + assert(dcpep_cb_prop_start(&d,&size)); + struct dcp_set_dcpav_prop_chunk_req chunk={.length=size}; + assert(size<=sizeof(chunk.data));memcpy(chunk.data,input_0,size); + if(fault==0)chunk.data[0]^=1; + assert(dcpep_cb_prop_chunk(&d,&chunk)); + struct dcp_set_dcpav_prop_end_req request={}; + strcpy(request.key,key); + bool accepted=dcpep_cb_prop_end(&d,&request); + assert(!d.chunks.data && !d.chunks.length); + return accepted; +} +static void failed_replacements(void) { + for(unsigned fault=0;fault<3;fault++) { + reset();callbacks(false);callbacks(true);timings();admit(0);probe(true); + struct dcp_display_mode *old=d.modes; + struct drm_display_mode saved=old[0].mode; + unsigned before=con.hotplug_wq.scheduled; + assert(!malformed_receipt("TimingElements",fault)); + assert(!current(&d,old) && !lookup_mode(&d,&saved)); + assert(con.hotplug_wq.scheduled>before);probe(false); + /* Fresh catalog can recover using still-current EDID and link proof. */ + timings();probe(true);free(old);cleanup(); + } + reset();callbacks(false);callbacks(true);timings();admit(0);probe(true); + assert(!malformed_receipt("DisplayAttributes",0));probe(true);cleanup(); + puts("PASS: actual chunk/end callbacks withdraw malformed replacement exceptions and notify DRM; fresh catalogs recover; unrelated bad properties preserve admission"); +} +''' + unit=unit.replace('int main(int argc,char **argv) {',failed_receipts+'int main(int argc,char **argv) {') + unit=unit.replace(' permutations();conflicting_receipts();',' permutations();conflicting_receipts();failed_replacements();') + return {**locals(), "block": block} + +def hotplug(root): + n = publication(root) + unit = n['unit'] + block = n['block'] + base = root / 'drivers/gpu/drm/apple' + dptx = (base / 'dptxep.c').read_text() + training = (base / 'dptx-j700-training.h').read_text() + unit = unit.replace('struct dptx_port { u32 lane_count, link_rate;', ''' +union phy_configure_opts { struct { unsigned lanes,link_rate; bool set_lanes,set_rate,set_voltages; } dp; }; +struct dptx_port { union phy_configure_opts phy_ops; int linkcfg_completion; u32 pending_link_rate; +u32 lane_count, link_rate;''') + unit = unit.replace('int fw_compat,width_mm,height_mm,fixed_connector_type;', 'unsigned dptx_phy; int fw_compat,width_mm,height_mm,fixed_connector_type;') + real_handlers = r''' +#define cpu_to_le32(v) (v) +#define cpu_to_le64(v) (v) +#define LINK_RATE_RBR 0x06 +#define LINK_RATE_HBR 0x0a +#define LINK_RATE_HBR2 0x14 +#define PHY_MODE_DP 1 +#define trace_dptxport_call_set_link_rate(...) ((void)0) +static unsigned phy_capacity=2, phy_calls, hardware_rate, hardware_lanes; +static int validate_error; +static int phy_validate(void *phy,int mode,int submode,union phy_configure_opts *o) { + assert(phy);(void)mode;(void)submode; + if(validate_error)return validate_error; + o->dp.lanes=phy_capacity;return 0; +} +static int phy_configure(void *phy,union phy_configure_opts *o) { + assert(phy);phy_calls++; + if(configure_error)return configure_error; + if(o->dp.set_lanes)hardware_lanes=o->dp.lanes; + if(o->dp.set_rate)hardware_rate=o->dp.link_rate; + return 0; +} +static void dcp_tunnel_set_rate(struct apple_dcp *d,void *phy,unsigned rate) { + (void)d;(void)phy;(void)rate;assert(false); +} +''' + real_handlers += block(training, 'static int dptx_j700_validate_lanes(') + '\n' + real_handlers += block(dptx, 'static int dptxport_call_set_active_lane_count(') + '\n' + real_handlers += block(dptx, 'static int dptxport_call_set_link_rate(') + '\n' + for marker in ['static int dptxport_call_set_active_lane_count(', 'static int dptxport_call_set_link_rate(']: + unit = unit.replace(block(unit, marker), '') + unit = unit.replace('static int configure_error;', 'static int configure_error;\n' + real_handlers) + unit = unit.replace('published=reads=pending_probes=0;', 'phy_capacity=2;phy_calls=hardware_rate=hardware_lanes=0;validate_error=0;published=reads=pending_probes=0;') + unit = unit.replace('assert(current(&d,&d.modes[0]));cleanup();runs++;', 'assert(current(&d,&d.modes[0]));assert(hardware_rate==8100 && hardware_lanes==2 && phy_calls==2);cleanup();runs++;') + unit = unit.replace('struct apple_dcp {', '''struct apple_dcp { + struct { bool lock,presentation_paused; } iomfb; + struct { bool valid,changing; } mode_state; + struct work_struct external_stop_work,vblank_wq; + bool external_physically_absent,main_display,typec_crtc_off,external_cable,external_error,external_power_attempted,external_stopped; + bool external_hotplug_pending,external_hotplug_connected; + u64 external_stop_generation,external_rx_generation,external_crtc_generation; + unsigned swap_watchdog_retrains;int typec_iomfb_hpd_ready;''') + stubs = r''' +#define DCP_HOTPLUG_NOTIFY 1 +static bool dcp_is_external_h17p(struct apple_dcp *d) { return d->hw.iomfb_method_profile==2; } +static bool dcp_is_typec_output(struct apple_dcp *d) { (void)d;return true; } +static void complete_all(int *p) { (*p)++; } +#define dcp_mode_invalidate(s) ((s)->valid=false) +#define dcp_mode_hotplug(s,c,p) __extension__ ({ (void)(s);if(p)*(p)=(c);1u; }) +static void dcp_handle_hotplug_actions(struct apple_dcp *d,unsigned action) { (void)d;(void)action; } +''' + dcp = (root / 'drivers/gpu/drm/apple/dcp.c').read_text() + template = (root / 'drivers/gpu/drm/apple/iomfb_template.c').read_text() + functions = stubs + block(dcp, 'bool dcp_external_hotplug(') + '\n' + block(template, 'static void dcpep_cb_hotplug(') + + main=r"""int main(int argc,char **argv) { MAIN_PREFIX + transport_edid=raw;permutations();conflicting_receipts();failed_replacements(); + unsigned runs=0; + for(unsigned a=0;a<4;a++)for(unsigned b=0;b<4;b++)for(unsigned c=0;c<4;c++)for(unsigned e=0;e<4;e++) { + if(a==b||a==c||a==e||b==c||b==e||c==e)continue; + reset();callbacks(false);callbacks(true);timings();admit(0);probe(true); + d.external_cable=true;d.external_power_attempted=true;d.external_rx_generation=7; + d.iomfb.lock=true;u64 connected=1;dcpep_cb_hotplug(&d,&connected);d.iomfb.lock=false; + struct dcp_display_mode *old=d.modes;struct drm_display_mode saved=old[0].mode; + u64 sequence=d.rate_proof.sequence,connection=d.rate_proof.connection_sequence; + d.iomfb.lock=true;connected=0;dcpep_cb_hotplug(&d,&connected);d.iomfb.lock=false; + assert(d.external_stop_work.scheduled==1 && d.external_stop_generation==7); + assert(sequence!=d.rate_proof.sequence && connection!=d.rate_proof.connection_sequence); + assert(!d.rate_proof.rate.valid && !d.rate_proof.lanes.valid); + assert(!current(&d,old) && !lookup_mode(&d,&saved)); + assert(con.edid_invalidation_pending);probe(false); /* Work deliberately not executed. */ + dpavservep_detach(&d);dcp_180hz_invalidate(&d); /* Deferred retirement/release. */ + unsigned order[]={a,b,c,e}; + for(unsigned i=0;i<4;i++) { + switch(order[i]) { case 0:callbacks(false);break;case 1:callbacks(true);break;case 2:timings();break;case 3:admit(1);break; } + probe(false);assert(!current(&d,old)); + } + assert(d.rate_proof.rate.valid && d.rate_proof.lanes.valid); + /* Repeated low reports before reassertion are not a new HPD loss. */ + sequence=d.rate_proof.sequence;d.iomfb.lock=true;connected=0; + dcpep_cb_hotplug(&d,&connected);d.iomfb.lock=false; + assert(d.rate_proof.sequence==sequence && d.rate_proof.rate.valid && d.rate_proof.lanes.valid); + d.iomfb.lock=true;connected=1;dcpep_cb_hotplug(&d,&connected);d.iomfb.lock=false; + assert(d.metadata.epoch.connected && d.metadata.epoch.invalidated); + probe(true);assert(!current(&d,old));assert(phy_calls==4 && hardware_rate==8100 && hardware_lanes==2); + free(old);cleanup();runs++; + } + assert(runs==24); + /* Initial bootstrap false and power/paused suppression keep proof intact. */ + reset();callbacks(false);callbacks(true);timings();admit(0);probe(true); + d.iomfb.lock=true;u64 connected=0;u64 sequence=d.rate_proof.sequence; + dcpep_cb_hotplug(&d,&connected);assert(d.rate_proof.sequence==sequence && !con.edid_invalidation_pending); + connected=1;dcpep_cb_hotplug(&d,&connected); + connected=0;d.typec_crtc_off=true;dcpep_cb_hotplug(&d,&connected);assert(d.rate_proof.sequence==sequence); + d.typec_crtc_off=false;d.iomfb.presentation_paused=true;dcpep_cb_hotplug(&d,&connected); + assert(d.rate_proof.sequence==sequence && !con.edid_invalidation_pending);d.iomfb.lock=false;probe(true); + dcpavserv_teardown(&av_services[0]);cleanup(); + puts("PASS: actual RX HPD loss immediately denies getter/lookup before deferred work; 24 fresh-proof recovery orders need only one rate/lane callback; bootstrap/power/paused suppression preserved"); +}""".replace('MAIN_PREFIX',n['main_prefix']) + unit=unit.replace(block(unit,'int main(int argc,char **argv) {'),functions+'\n'+main) + return unit + + +def metadata(root, output): + """Keep malformed/duplicate metadata cases with synthetic bounded wire input.""" + n = parser_fixture(root) + fixture = n['fixture'] + cases = [(fixture, False)] + for path, value in [ + (('MaxHorizontalImageSize',), 256), (('MaxVerticalImageSize',), '33'), + (('Chromaticity', 'Red', 'X'), True), (('Chromaticity', 'Blue', 'Y'), 65536), + (('Chromaticity', 'Green', 'Y'), 64000), (('DefaultWhitePoint', 'Gamma'), 65535), + (('DefaultWhitePoint', 'Gamma'), 240000), (('ProductAttributes', 'ManufacturerID'), 'AC'), + (('ProductAttributes', 'ManufacturerID'), 'A\0R'), (('ProductAttributes', 'LegacyManufacturerID'), 1), + (('ProductAttributes', 'ProductName'), 'x' * 14), (('ProductAttributes', 'ProductID'), 65536), + (('ProductAttributes', 'YearOfManufacture'), 1989), (('ProductAttributes', 'WeekOfManufacture'), 54), + ]: + bad = copy.deepcopy(fixture) + target = bad + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + cases.append((bad, True)) + for key in ('Chromaticity', 'DefaultWhitePoint', 'ProductAttributes'): + bad = copy.deepcopy(fixture) + del bad[key] + cases.append((bad, True)) + second = copy.deepcopy(fixture[key]) + cases.append((n['ObjectPairs'](list(fixture.items()) + [(key, second)]), True)) + for path, field in [(('Chromaticity', 'Red'), 'X'), (('ProductAttributes',), 'ProductName')]: + bad = copy.deepcopy(fixture) + target = bad + for key in path[:-1]: + target = target[key] + value = target[path[-1]] + target[path[-1]] = n['ObjectPairs'](list(value.items()) + [(field, value[field])]) + cases.append((bad, True)) + inputs = output / 'metadata-inputs' + inputs.mkdir(parents=True, exist_ok=True) + arguments = [] + for index, (value, bad) in enumerate(cases): + data = bytearray(struct.pack('H', 1138) + assert raw[21:24] == bytes([60, 33, 120]) + assert raw[38:54] == b'\x01' * 16 and raw[35:38] == b'\0' * 3 + assert struct.unpack_from('> 4) << 8) == 1920 + assert raw[57] + ((raw[58] & 15) << 8) == 280 + assert raw[59] + ((raw[61] >> 4) << 8) == 1080 + assert raw[60] + ((raw[61] & 15) << 8) == 45 + assert raw[62:65] == bytes([88, 44, 0x45]) and raw[71] == 0x1e + chroma = [raw[27] * 4 + ((raw[25] >> 6) & 3), raw[28] * 4 + ((raw[25] >> 4) & 3), + raw[29] * 4 + ((raw[25] >> 2) & 3), raw[30] * 4 + (raw[25] & 3), + raw[31] * 4 + ((raw[26] >> 6) & 3), raw[32] * 4 + ((raw[26] >> 4) & 3), + raw[33] * 4 + ((raw[26] >> 2) & 3), raw[34] * 4 + (raw[26] & 3)] + assert chroma == [697, 317, 265, 632, 146, 81, 321, 337] + assert raw[72:77] == bytes.fromhex('000000fc00') + + +def retry(root): + source = (root / 'drivers/gpu/drm/apple/dcp-180hz.c').read_text() + production = '\n'.join(block(source, marker) for marker in ( + 'static void dcp_180hz_retry_work(', 'void dcp_180hz_retry(', 'static void dcp_180hz_cancel(')) + prefix = r""" +#include +#include +#include +#include +#include +typedef uint64_t u64; +#define READ_ONCE(x) (x) +#define WRITE_ONCE(p,v) ((p)=(v)) +#define guard(kind) (void) +#define spin_lock_irqsave(p,f) do { assert(!*(p)); *(p)=true; (f)=0; } while (0) +#define spin_unlock_irqrestore(p,f) do { (void)(f); assert(*(p)); *(p)=false; } while (0) +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +#define to_delayed_work(p) container_of(p,struct delayed_work,work) +#define to_platform_device(p) (p) +#define msecs_to_jiffies(x) (x) +struct work_struct { unsigned scheduled; }; +struct delayed_work { struct work_struct work; unsigned delay; bool pending; }; +struct apple_connector { void *dcp; struct work_struct hotplug_wq; }; +struct apple_dcp { + bool hpd_mutex, typec_cable_connected, quiescing, rate_retry_initialized; + struct { bool lock; } metadata; + struct { u64 retry_generation; unsigned retries; } rate_proof; + u64 typec_generation; + struct apple_connector *connector; + void *dev; + struct delayed_work rate_retry_wq; +}; +static int system_freezable_wq; +static unsigned submissions, cancellations; +static void schedule_work(struct work_struct *w) { w->scheduled++; } +static void mod_delayed_work(int q,struct delayed_work *w,unsigned delay) +{ (void)q; submissions++; w->pending=true; w->delay=delay; } +static void cancel_delayed_work_sync(struct delayed_work *w) +{ cancellations++; w->pending=false; } +""" + controls = r""" +int main(void) +{ + struct apple_dcp d={.typec_cable_connected=true,.typec_generation=7}; + struct apple_connector c={.dcp=&d}; d.connector=&c; d.dev=&d; + dcp_180hz_retry(&d); assert(!submissions); + d.rate_retry_initialized=true; + for(unsigned i=0;i<100;i++) dcp_180hz_retry(&d); + assert(submissions==8 && d.rate_proof.retries==8 && d.rate_retry_wq.delay==300); + dcp_180hz_retry_work(&d.rate_retry_wq.work); assert(c.hotplug_wq.scheduled==1); + d.typec_generation++; dcp_180hz_retry_work(&d.rate_retry_wq.work); + assert(c.hotplug_wq.scheduled==1); + d.typec_cable_connected=false; dcp_180hz_retry(&d); assert(submissions==8); + d.typec_cable_connected=true; dcp_180hz_retry(&d); assert(submissions==9 && d.rate_proof.retries==1); + d.quiescing=true; dcp_180hz_retry(&d); dcp_180hz_retry_work(&d.rate_retry_wq.work); + assert(submissions==9 && c.hotplug_wq.scheduled==1); + d.quiescing=false; c.dcp=NULL; dcp_180hz_retry_work(&d.rate_retry_wq.work); + assert(c.hotplug_wq.scheduled==1); + c.dcp=&d; dcp_180hz_retry_work(&d.rate_retry_wq.work); assert(c.hotplug_wq.scheduled==2); + dcp_180hz_cancel(&d); assert(!d.rate_retry_initialized && !d.rate_retry_wq.pending && cancellations==1); + dcp_180hz_retry(&d); assert(submissions==9); + puts("PASS: actual eight-retry bound, 300ms delay, stale-generation/disconnect/quiesce/ownership gates and cancellation"); +} +""" + return prefix + production + controls diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py new file mode 100644 index 0000000000000..fe62df8d78129 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright 2026 Ryan Murray +"""Host regressions for Apple external display ownership and recovery. + +Only public synthetic fixtures and extracted production functions are used. +The transport, MMIO, locks and work queues are mocks, not hardware validation. +""" +import argparse +import copy +import hashlib +import json +import os +from pathlib import Path +import struct +import tempfile +import traceback + +from common import run_unit +import edid +import lifecycle +import phy +import pm +import idle +import rate + + +def parking_unit(root): + fixture = phy.parking(root) + return ''.join(fixture[k] for k in ('prefix', 'production', 'suffix')) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--source', type=Path, default=Path(__file__).resolve().parents[6]) + parser.add_argument('--output', type=Path, help='retain generated C, commands and results here') + parser.add_argument('--group', choices=('all', 'edid', 'rate', 'phy', 'lifecycle'), default='all') + args = parser.parse_args() + root = args.source.resolve() + output = args.output or Path(tempfile.mkdtemp(prefix='apple-dcp-selftest-')) + output.mkdir(parents=True, exist_ok=True) + apple = root / 'drivers/gpu/drm/apple' + includes = (apple, root / 'tools/testing/selftests/drivers/gpu/apple-afkv2/host-include') + parking_cases = tuple((c,) for c in ('mux', 'power', 'reset', 'cleanup', 'orientation', + 'orientation-faults', 'backend-change', 'safe-first', + 'failed-flip-return', 'live-park')) + tests = [ + ('edid', 'service-rebind', edid.rebind, ((),), ()), + ('edid', 'provider-admission', edid.admission, ((),), ()), + ('edid', 'ordered-rx', edid.rx_progress, ((),), ('-pthread',)), + ('rate', 'parser-policy', lambda r: rate.policy(r)['unit'], ((),), ()), + ('rate', 'publication-hpd', rate.hotplug, ((),), ()), + ('rate', 'metadata-decoder', lambda r: rate.metadata(r, output), ((),), ('-Wno-sign-compare',)), + ('rate', 'bounded-retry', rate.retry, ((),), ()), + ('phy', 'pm-ownership', pm.ownership, ((),), ()), + ('phy', 'usb-unreachable', lambda r: pm.ownership(r, True), ((),), ()), + ('lifecycle', 'cpuidle-s2idle', idle.s2idle, ((),), ('-Wno-unused-parameter', '-Wno-unused-variable')), + ('lifecycle', 'typec-probe', pm.probe, ((),), ()), + ('lifecycle', 'probe-mux-pm-dispatch', pm.probe_dispatch, ((),), ()), + ('phy', 'parking', lambda r: parking_unit(r), parking_cases, ()), + ('phy', 'adverse-retry', phy.adverse, tuple((c,) for c in ('bist', 'init', 'mux', 'gate', 'dp-cache', 'off', 'success')), ()), + ('phy', 'masked-context', phy.usb_context, ((),), ()), + ('phy', 'off-hardening', phy.hardening, ((),), ()), + ('phy', 'warm-reset', phy.warm_reset, ((),), ()), + ('phy', 'pipe', phy.pipe, tuple((c,) for c in ('backend', 'host', 'mux', 'same', 'park', 'requested', 'scope')), ()), + ('phy', 'unwind', phy.unwind, ((),), ()), + ('phy', 'aux', phy.aux, ((),), ()), + ('lifecycle', 'physical-disconnect', lifecycle.disconnect, ((),), + ('-Wno-unused-variable', '-Wno-unused-function', '-Wno-unused-but-set-variable')), + ('lifecycle', 'dwc3-teardown', lifecycle.teardown, (('all',),), ()), + ('lifecycle', 'resource-profile', lifecycle.profile, ((),), ('-Wno-sign-compare',)), + ('lifecycle', 'firmware-provider', lifecycle.firmware, ((),), ()), + ('lifecycle', 'readiness', lifecycle.readiness, ((),), ()), + ('lifecycle', 'vrr-rejection', lifecycle.vrr, ((),), ()), + ('lifecycle', 'power-reply', lifecycle.power_reply, ((),), ()), + ('lifecycle', 'dispatcher-scope', lifecycle.dispatcher, ((),), ()), + ] + tests.append(('phy', 'usb-kconfig', lambda r: pm.kconfig(r, output), (), ())) + tests = [t for t in tests if args.group in ('all', t[0])] + print('TAP version 13', flush=True) + print('1..' + str(len(tests)), flush=True) + results = [] + for index, (group, name, build, cases, flags) in enumerate(tests, 1): + result = {'name': name, 'group': group, 'passed': False} + try: + unit = build(root) + if unit is None: + result['producer_passed'] = True + elif isinstance(unit, tuple): + unit, cases = unit + if unit is not None: + result['translation_unit_sha256'] = run_unit(name, unit, output, cases, includes if name == 'service-rebind' else (), flags) + if name == 'metadata-decoder': + rate.check_metadata(output) + result['passed'] = True + except Exception: + result['failure'] = traceback.format_exc() + (output / (name + '-failure.log')).write_text(result['failure']) + print(('ok ' if result['passed'] else 'not ok ') + str(index) + ' - ' + name, flush=True) + results.append(result) + sources = {} + for area in ('drivers/gpu/drm/apple', 'drivers/phy/apple', 'drivers/usb/dwc3', 'drivers/usb/typec/tipd'): + for path in sorted((root / area).rglob('*')): + if path.is_file() and path.suffix in ('.c', '.h'): + sources[str(path.relative_to(root))] = hashlib.sha256(path.read_bytes()).hexdigest() + (output / 'results.json').write_text(json.dumps({'source': str(root), 'sources': sources, 'results': results}, indent=2) + '\n') + print('# Artifacts: ' + str(output), flush=True) + return int(any(not r['passed'] for r in results)) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.sh b/tools/testing/selftests/drivers/gpu/apple-dcp/run.sh new file mode 100755 index 0000000000000..f4052ba6632f1 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only OR MIT +PYTHONDONTWRITEBYTECODE=1; export PYTHONDONTWRITEBYTECODE +exec python3 "$(dirname "$0")/run.py" "$@" diff --git a/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py b/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py index f55107c0ddaa6..5be56f76a6859 100644 --- a/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py +++ b/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py @@ -14,18 +14,7 @@ def test_shared_header_boundaries(self): #include "dcp-external-bw.h" int main(void) { const u64 n=0xbc000; - u32 scratch=1, request=2; - assert(apple_dcp_external_bw_resolve(n,0x28000,0x70000,-1,-1,&scratch,&request)); - assert(scratch==0x28000 && request==0x70000); - assert(apple_dcp_external_bw_resolve(n,0x28000,0x70000,0x30000,-1,&scratch,&request)); - assert(scratch==0x30000 && request==0x70000); - assert(apple_dcp_external_bw_resolve(n,0x28000,0x70000,-1,0x74000,&scratch,&request)); - assert(scratch==0x28000 && request==0x74000); - assert(!apple_dcp_external_bw_resolve(n,0x28000,0x70000,-2,-1,&scratch,&request)); - assert(!apple_dcp_external_bw_resolve(n,0x28000,0x70000,-1,0x28000,&scratch,&request)); - assert(!apple_dcp_external_bw_resolve(n,0x28000,0x70000,0x28002,-1,&scratch,&request)); - assert(!apple_dcp_external_bw_resolve(n,0x28000,0x70000,-1,n,&scratch,&request)); - assert(scratch==0x28000 && request==0x74000); + assert(apple_dcp_external_bw_valid(n,0x28000,0x70000)); assert(!apple_dcp_external_bw_valid(n,-1,-1)); assert(!apple_dcp_external_bw_valid(n,0,-1)); assert(!apple_dcp_external_bw_valid(n,-1,0)); From 7d103123bc2b4d5447a898411933c440b4e1b0cf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 19:16:00 +0000 Subject: [PATCH 1215/2680] drm/apple: scope deferred DPAV recovery to current interfaces Retain admitted reannouncements while old commands drain, cancel pending OPEN when CLOSED supersedes it, and preserve interface tag history across reader-pinned slot transitions. Fence explicit physical absence during admission, publication and EDID-current validation. Preserve old command buffers, reader pins and existing endpoint teardown ownership. Keep actual-function regressions in the compact standard selftests. Independent review verified corrected CLOSED/history/absence/fixture controls and strict ARM objects. Exact source tree f14f599948a8ccae7362c4d328dddea623442e54 passed the full strict R66 Image/modules/dtbs build. This is a functional qualification candidate. The inherited four-bit command identity still cannot distinguish a firmware-republished consumed reply when all correlation values recur; retained command-17 counterexamples remain unresolved. No new peer-token semantics, packet experiment, cached EDID override or production diagnostic knob is introduced. No release or hardware qualification is claimed for this new candidate. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 165 ++++++- drivers/gpu/drm/apple/afk.h | 18 +- drivers/gpu/drm/apple/epic/dpavservep.c | 30 +- .../drivers/gpu/apple-afkv2/extract-send.py | 16 +- .../selftests/drivers/gpu/apple-dcp/Makefile | 2 +- .../selftests/drivers/gpu/apple-dcp/README | 14 + .../selftests/drivers/gpu/apple-dcp/edid.py | 135 +++++- .../drivers/gpu/apple-dcp/recovery.py | 439 ++++++++++++++++++ .../selftests/drivers/gpu/apple-dcp/run.py | 3 + 9 files changed, 795 insertions(+), 27 deletions(-) create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 8a42985a091bb..a617d35fd49f2 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -52,6 +52,21 @@ enum rbep_msg_type { #define SEND_WPTR GENMASK(31, 0) +static void afk_service_worker(struct work_struct *work); +static void afk_discard_pending_open(struct apple_epic_service *service); + +void afk_service_retry(struct apple_dcp_afkep *ep) +{ + unsigned long flags; + + if (!ep || !ep->wire_v2 || ep != ep->dcp->dcpavservep) + return; + spin_lock_irqsave(&ep->lock, flags); + if (!ep->stopping) + queue_work(ep->wq, &ep->service_work); + spin_unlock_irqrestore(&ep->lock, flags); +} + static void afk_send(struct apple_dcp_afkep *ep, u64 message) { dcp_send_message(ep->dcp, ep->endpoint, message); @@ -62,8 +77,10 @@ static void afk_release_context(void *data) struct apple_dcp_afkep *afkep = data; afk_quiesce(afkep); - if (!afkep->dcp->retain_dma) + if (!afkep->dcp->retain_dma) { + kvfree(afkep->command_tags); kfree(afkep); + } } struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, @@ -86,6 +103,7 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, goto out_free_afkep; } + INIT_WORK(&afkep->service_work, afk_service_worker); init_completion(&afkep->started); init_completion(&afkep->stopped); spin_lock_init(&afkep->lock); @@ -104,12 +122,21 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, void afk_quiesce(struct apple_dcp_afkep *afkep) { + unsigned long flags; + u32 i; + if (IS_ERR_OR_NULL(afkep)) return; + spin_lock_irqsave(&afkep->lock, flags); + afkep->stopping = true; + spin_unlock_irqrestore(&afkep->lock, flags); if (afkep->wq) { destroy_workqueue(afkep->wq); afkep->wq = NULL; } + for (i = 0; i < afkep->num_channels; i++) + if (afkep->services[i].pending_open) + afk_discard_pending_open(&afkep->services[i]); debugfs_remove_recursive(afkep->debugfs_entry); afkep->debugfs_entry = NULL; } @@ -140,6 +167,14 @@ int afk_start(struct apple_dcp_afkep *ep) { int ret; + /* Allocate before firmware can announce services or issue commands. */ + if (ep->wire_v2 && !ep->command_tags) { + ep->command_tags = kvcalloc(AFKV2_INTERFACES, sizeof(*ep->command_tags), + GFP_KERNEL); + if (!ep->command_tags) + return -ENOMEM; + } + reinit_completion(&ep->started); ret = apple_rtkit_start_ep(ep->dcp->rtk, ep->endpoint); if (ret) @@ -310,12 +345,74 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, enum epic_type etype, enum epic_category ecat, u8 stype, const void *payload, size_t payload_len, u64 token); +/* Pending identities are owned only by the ordered endpoint workqueue. */ +static void afk_discard_pending_open(struct apple_epic_service *service) +{ + service->pending_open = false; + kfree(service->pending_name); + kfree(service->pending_class); + service->pending_name = NULL; + service->pending_class = NULL; + afk_service_discard(service); +} + +static void afk_retry_service_init(struct apple_dcp_afkep *ep) +{ + u32 i; + + for (i = 0; i < ep->num_channels; i++) { + struct apple_epic_service *service = &ep->services[i]; + const struct apple_epic_service_ops *ops = service->ops; + int ret; + + if (!service->pending_open) + continue; + ret = ops->revalidate(service, service->pending_name, + service->pending_class, service->pending_unit); + if (ret) { + afk_discard_pending_open(service); + continue; + } + /* Old replies keep their original slot, tag and buffers until drained. */ + if (afk_epic_find_service(ep, service->channel)) + continue; + ret = afk_send_epic(ep, service->channel, 0, EPIC_TYPE_NOTIFY, + EPIC_CAT_REPORT, AFKV2_OPEN, NULL, 0); + if (ret) { + dev_err(ep->dcp->dev, "AFKV2 interface %u deferred OPEN failed: %d\n", + service->channel, ret); + afk_discard_pending_open(service); + continue; + } + service->pending_open = false; + afk_service_publish(service); + ops->init(service, service->pending_name, service->pending_class, + service->pending_unit); + if (service->enabled) + afk_populate_service_debugfs(service); + kfree(service->pending_name); + kfree(service->pending_class); + service->pending_name = NULL; + service->pending_class = NULL; + } +} + +static void afk_service_worker(struct work_struct *work) +{ + struct apple_dcp_afkep *ep = container_of(work, struct apple_dcp_afkep, service_work); + + if (!READ_ONCE(ep->stopping)) + afk_retry_service_init(ep); +} + static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, u8 *payload, size_t payload_size) { char name[32]; s64 epic_unit = -1; - struct apple_epic_service *service = NULL; + struct apple_epic_service *service = NULL, *previous; + unsigned long flags; + bool retired = false; u32 ch_idx; const char *service_name = name; const char *epic_name = NULL, *epic_class = NULL; @@ -324,8 +421,25 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, u8 *props = payload + sizeof(name); size_t props_size = payload_size - sizeof(name); - if (afk_epic_find_service(ep, channel)) - return; + previous = afk_epic_find_service(ep, channel); + if (previous) { + spin_lock_irqsave(&previous->lock, flags); + retired = previous->torndown; + spin_unlock_irqrestore(&previous->lock, flags); + if (!ep->wire_v2 || ep != ep->dcp->dcpavservep || + !retired || !previous->ops->revalidate) + return; + } + for (ch_idx = 0; ch_idx < ep->num_channels; ch_idx++) { + struct apple_epic_service *pending = &ep->services[ch_idx]; + + if (!pending->pending_open || pending->channel != channel) + continue; + if (!pending->ops->revalidate(pending, pending->pending_name, + pending->pending_class, pending->pending_unit)) + return; + afk_discard_pending_open(pending); + } if (payload_size < sizeof(name)) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: payload too small: %lx\n", @@ -398,6 +512,15 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, goto free; } } + if (previous && retired && ops->revalidate) { + /* Preserve the admitted identity and its original provider ticket. */ + service->pending_open = true; + service->pending_name = epic_name; + service->pending_class = epic_class; + service->pending_unit = epic_unit; + afk_service_retry(ep); + return; + } if (ep->wire_v2) { int ret = afk_send_epic(ep, channel, 0, EPIC_TYPE_NOTIFY, EPIC_CAT_REPORT, AFKV2_OPEN, NULL, 0); @@ -438,6 +561,14 @@ static void afk_recv_handle_teardown(struct apple_dcp_afkep *ep, u32 channel) const struct apple_epic_service_ops *ops; unsigned long flags; + /* Accepted CLOSE/CLOSED supersedes unpublished recovery for this interface. */ + for (u32 i = 0; i < ep->num_channels; i++) { + struct apple_epic_service *pending = &ep->services[i]; + + if (pending->pending_open && pending->channel == channel) + afk_discard_pending_open(pending); + } + service = afk_epic_find_service(ep, channel); if (!service) { dev_warn(ep->dcp->dev, "AFK[ep:%02x]: teardown for disabled channel %u\n", @@ -1459,6 +1590,23 @@ int afk_send_command(struct apple_epic_service *service, u8 type, MSEC_PER_SEC); } +static u16 afk_command_tag(struct apple_epic_service *service, u8 slot) +{ + struct apple_dcp_afkep *ep = service->ep; + unsigned long flags; + u16 tag; + + lockdep_assert_held(&service->lock); + if (!ep->wire_v2) + return ((service->cmd_tag++ & 0xff) << 8) | slot; + + /* Send already nests the endpoint lock below the service lock. */ + spin_lock_irqsave(&ep->lock, flags); + tag = afkv2_command_id(ep->command_tags[service->channel]++, slot); + spin_unlock_irqrestore(&ep->lock, flags); + return tag; +} + static int afk_send_command_timeout_len(struct apple_epic_service *service, u8 type, const void *payload, size_t payload_len, void *output, size_t output_len, u32 *retcode, @@ -1495,6 +1643,11 @@ static int afk_send_command_timeout_len(struct apple_epic_service *service, u8 t if (!service) return -ENODEV; ep = service->ep; + if (ep->wire_v2 && (!service->channel || service->channel >= AFKV2_INTERFACES || + !ep->command_tags)) { + ret = -ENODEV; + goto err_put_service; + } if (READ_ONCE(ep->dcp->quiescing)) { ret = -ENODEV; goto err_put_service; @@ -1550,9 +1703,7 @@ static int afk_send_command_timeout_len(struct apple_epic_service *service, u8 t goto err_unlock; } - tag = ep->wire_v2 ? afkv2_command_id(service->cmd_tag, idx) : - ((service->cmd_tag & 0xff) << 8) | idx; - service->cmd_tag++; + tag = afk_command_tag(service, idx); service->cmds[idx].tag = tag; service->cmds[idx].type = type; diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index ed0e3443a2969..b51ac5ca8b9a4 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -19,10 +19,13 @@ #define AFK_MAX_CHANNEL 16 #define MAX_PENDING_CMDS 16 +#define AFKV2_INTERFACES BIT(16) struct apple_epic_service_ops; struct apple_dcp_afkep; +void afk_service_retry(struct apple_dcp_afkep *ep); + struct epic_cmd_info { u16 tag; u8 type; @@ -47,6 +50,7 @@ struct apple_epic_service { struct epic_cmd_info cmds[MAX_PENDING_CMDS]; DECLARE_BITMAP(cmd_map, MAX_PENDING_CMDS); + /* Legacy EPIC sequence; AFKV2 history belongs to the interface. */ u8 cmd_tag; spinlock_t lock; /* Pins held by owners and synchronous callers; protected by lock. */ @@ -60,6 +64,11 @@ struct apple_epic_service { struct { u64 generation, epoch; } admission; void *cookie; + /* Admitted reannouncement waiting for the previous channel to drain. */ + bool pending_open; + const char *pending_name, *pending_class; + s64 pending_unit; + struct { struct dentry *entry; u8 *scratch; @@ -79,6 +88,9 @@ struct apple_epic_service_ops { */ int (*prepare)(struct apple_epic_service *service, const char *name, const char *class, s64 unit); + /* Check the original prepare ticket before a deferred OPEN. */ + int (*revalidate)(struct apple_epic_service *service, const char *name, + const char *class, s64 unit); void (*init)(struct apple_epic_service *service, const char *name, const char *class, s64 unit); int (*call)(struct apple_epic_service *service, u32 idx, @@ -139,6 +151,7 @@ static inline void afk_service_put(struct apple_epic_service *service) if (!WARN_ON(!service->users)) service->users--; spin_unlock_irqrestore(&service->lock, flags); + afk_service_retry(service->ep); } static inline void afk_service_disable(struct apple_epic_service *service) @@ -202,7 +215,7 @@ afk_service_reinit(struct apple_epic_service *service, service->ep = ep; service->channel = channel; service->cookie = NULL; - /* Keep the command-tag sequence across service generations. */ + /* Keep legacy slot history; AFKV2 history belongs to the interface. */ } spin_unlock_irqrestore(&service->lock, flags); return reusable; @@ -344,7 +357,10 @@ struct apple_dcp_afkep { bool wire_v2; /* External H17P DPAV only; other endpoints retain owned-DMA commands. */ bool inline_v2; + /* Fixed 16-bit interface domain; survives slot and channel reuse. */ + u8 *command_tags; struct workqueue_struct *wq; + struct work_struct service_work; struct completion started; struct completion stopped; diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 2fb18d1eb512c..037a6388ffea2 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -50,7 +50,8 @@ static bool dcpavserv_admission_current(struct apple_dcp *dcp, u64 generation) { lockdep_assert_held(&dcp->hpd_mutex); /* Initial service discovery may precede the first Type-C attachment. */ - return !READ_ONCE(dcp->quiescing) && generation == dcp->typec_generation && + return !READ_ONCE(dcp->quiescing) && !READ_ONCE(dcp->external_physically_absent) && + generation == dcp->typec_generation && (!generation || dcp->typec_cable_connected); } @@ -81,6 +82,26 @@ static int dcpavserv_prepare(struct apple_epic_service *service, const char *nam return ret; } +static int dcpavserv_revalidate(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + struct apple_dcp *dcp = service->ep->dcp; + int ret = 0; + + if (!dcpavserv_identity(service, name, class, unit)) + return -ENODEV; + mutex_lock(&dcp->hpd_mutex); + mutex_lock(&dcp->dcpavserv.lock); + if (dcp->dcpavserv.enabled) + ret = -EBUSY; + else if (service->admission.epoch != dcp->dcpavserv.epoch || + !dcpavserv_admission_current(dcp, service->admission.generation)) + ret = -ENODEV; + mutex_unlock(&dcp->dcpavserv.lock); + mutex_unlock(&dcp->hpd_mutex); + return ret; +} + static void dcpavserv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { @@ -363,14 +384,16 @@ bool dcpavserv_edid_current(struct apple_dcp *dcp, u64 generation) return dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL || (dcp->typec_cable_connected && generation == dcp->typec_generation && - !READ_ONCE(dcp->quiescing)); + !READ_ONCE(dcp->quiescing) && !READ_ONCE(dcp->external_physically_absent)); } bool dcpavserv_edid_service_current(struct apple_dcp *dcp, u64 epoch) { lockdep_assert_held(&dcp->dcpavserv.lock); return dcp->dcpavserv.enabled && dcp->dcpavserv.service && - epoch == dcp->dcpavserv.epoch; + epoch == dcp->dcpavserv.epoch && + (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL || + !READ_ONCE(dcp->external_physically_absent)); } const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation, @@ -416,6 +439,7 @@ static const struct apple_epic_service_ops dpavservep_ops[] = { .name = "dcpav-service-epic", .reusable = true, .prepare = dcpavserv_prepare, + .revalidate = dcpavserv_revalidate, .init = dcpavserv_init, .teardown = dcpavserv_teardown, .report = dcpavserv_report, diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py index 0d80a729a3342..70dbd5d3a69f4 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py @@ -30,6 +30,8 @@ #define SZ_1M 1048576U #define GFP_KERNEL 0 #define MAX_PENDING_CMDS 16 +#define AFKV2_INTERFACES (1U << 16) +#define lockdep_assert_held(l) ((void)(l)) #define cpu_to_le32(x) (x) #define le32_to_cpu(x) (x) #define cpu_to_le64(x) (x) @@ -56,7 +58,7 @@ bool done,free_on_ack; int *completion; }; struct dcp { void *dev; bool quiescing; }; -struct apple_dcp_afkep { struct dcp *dcp; bool wire_v2,inline_v2; int endpoint; }; +struct apple_dcp_afkep { struct dcp *dcp; bool wire_v2,inline_v2; int endpoint,lock;u8 *command_tags; }; struct ops { const char *name; }; struct apple_epic_service { struct apple_dcp_afkep *ep; struct ops *ops; unsigned int channel; unsigned long cmd_map[1]; @@ -106,7 +108,8 @@ suffix = r''' int main(void) { - struct dcp dcp={0};struct apple_dcp_afkep ep={.dcp=&dcp,.wire_v2=true,.inline_v2=true}; + u8 tags[AFKV2_INTERFACES]={0}; + struct dcp dcp={0};struct apple_dcp_afkep ep={.dcp=&dcp,.wire_v2=true,.inline_v2=true,.command_tags=tags}; struct ops ops={.name="test"};struct apple_epic_service s={.ep=&ep,.ops=&ops,.channel=4,.enabled=true,.users=1}; u8 request[112]={0x5a},*output=calloc(1,32888);size_t received;u32 status=99; active=&s; @@ -155,4 +158,11 @@ puts("PASS: extracted production sender, 128-byte command envelope / 32888-byte reply, short OOB, strict/legacy callers and actual late-ACK ownership"); } ''' -pathlib.Path(sys.argv[2]).write_text(prefix + source[start:end] + source[reply_match.start():reply_end] + suffix) +tag_match = re.search(r"static u16 afk_command_tag\([^;]*?\)\n\{", source) +if not tag_match: + raise RuntimeError("production interface tag allocator not found") +tag_end, tag_depth = tag_match.end(), 1 +while tag_depth: + tag_depth += (source[tag_end] == "{") - (source[tag_end] == "}") + tag_end += 1 +pathlib.Path(sys.argv[2]).write_text(prefix + source[tag_match.start():tag_end] + source[start:end] + source[reply_match.start():reply_end] + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile index 0d6d406ef5b9d..cdc821ce4be25 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT TEST_PROGS := run.sh -TEST_FILES := run.py common.py edid.py rate.py phy.py lifecycle.py pm.py idle.py dwc3 README +TEST_FILES := run.py common.py edid.py recovery.py rate.py phy.py lifecycle.py pm.py idle.py dwc3 README include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/README b/tools/testing/selftests/drivers/gpu/apple-dcp/README index a61543d1168f8..e9bf9bdabcb67 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/README +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/README @@ -54,3 +54,17 @@ The cpuidle group executes the actual Apple table/probe, generic registration and s2idle selector/entry, and tick_freeze/unfreeze. It checks source behavior only; RTC, elapsed suspend time and simultaneous hardware execution remain separate qualification gates. The original human cpuidle patch is preserved. + +The service-rebind group also runs retained reviewer controls through the +actual sender, receive worker, recovery, quiesce and EDID wrapper. It covers +CLOSED cancellation before lookup, first late ACK versus already consumed +duplicates, reader-pinned cross-slot/channel reuse, and the explicit physical +absence window. Per-interface sequence history has a fixed 64-KiB bound per +AFKV2 endpoint; reply routing still ignores reader-only retired services. + +Two wrap controls intentionally retain the command-17 replay assertion +failure, for OOB and inline. The group verifies that these counterexamples +still reproduce, alongside nonvacuous mutations and passing corrected safety +controls. A passing host suite does not establish a replay-exclusion contract +or admit a hardware trial. Token/sequence semantics remain a separate protocol +fact to establish; the tests do not manufacture a firmware echo guarantee. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py index 55a16678b97be..7d892663ff675 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py @@ -21,6 +21,8 @@ def function(file,signature): #include #include #include +#include +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) #include "afkv2-codec.h" typedef int64_t s64; typedef u64 dma_addr_t; @@ -29,6 +31,7 @@ def function(file,signature): #define ENOENT 2 #define MAX_PENDING_CMDS 16 #define AFK_MAX_CHANNEL 16 +#define AFKV2_INTERFACES (1U << 16) #define READ_ONCE(x) (x) #define WRITE_ONCE(x,v) ((x)=(v)) #define WARN_ON(x) (x) @@ -65,25 +68,30 @@ def function(file,signature): static void complete(int *p) { (*p)++; } static void reinit_completion(int *p) { *p=0; } static void schedule_work(unsigned int *p) { (*p)++; } +struct work_struct { bool pending; }; +static bool queue_work(void *wq,struct work_struct *w) { assert(wq);bool fresh=!w->pending;w->pending=true;return fresh; } struct apple_dcp;struct apple_dcp_afkep;struct apple_epic_service; -struct apple_epic_service_ops { char name[32];bool reusable;void (*init)(struct apple_epic_service *,const char *,const char *,s64);int (*prepare)(struct apple_epic_service *,const char *,const char *,s64);void (*teardown)(struct apple_epic_service *); }; +void afk_service_retry(struct apple_dcp_afkep *ep); +struct apple_epic_service_ops { char name[32];bool reusable;void (*init)(struct apple_epic_service *,const char *,const char *,s64);int (*prepare)(struct apple_epic_service *,const char *,const char *,s64);int (*revalidate)(struct apple_epic_service *,const char *,const char *,s64);void (*teardown)(struct apple_epic_service *); }; struct epic_cmd_info { u16 tag;u8 type;size_t reply_length;void *rxbuf,*txbuf;dma_addr_t rxbuf_dma,txbuf_dma;size_t rxlen,txlen;u32 retcode;bool done,free_on_ack;int *completion; }; struct apple_epic_service { struct apple_dcp_afkep *ep;const struct apple_epic_service_ops *ops; bool lock,enabled,torndown;unsigned int users;u32 channel;u8 cmd_tag; unsigned long cmd_map[1];struct epic_cmd_info cmds[MAX_PENDING_CMDS]; struct { u64 generation,epoch; } admission;void *cookie; + bool pending_open;const char *pending_name,*pending_class;s64 pending_unit; }; -struct apple_dcp_afkep { struct apple_dcp *dcp;const struct apple_epic_service_ops *ops;bool wire_v2,inline_v2,match_epic_name;u32 endpoint;unsigned int num_channels;struct apple_epic_service services[AFK_MAX_CHANNEL]; }; +struct apple_dcp_afkep { u8 *command_tags;bool lock,stopping;void *wq;struct work_struct service_work;struct apple_dcp *dcp;const struct apple_epic_service_ops *ops;bool wire_v2,inline_v2,match_epic_name;u32 endpoint;unsigned int num_channels;struct apple_epic_service services[AFK_MAX_CHANNEL]; }; struct apple_connector { void *dcp;bool connected,edid_invalidation_pending;struct { int registration_state; } base;unsigned int hotplug_wq; }; struct apple_dcp { void *dev;int fw_compat;struct apple_dcp_afkep *dcpavservep;struct apple_connector *connector; - bool hpd_mutex,quiescing,typec_cable_connected;u64 typec_generation;unsigned int nr_modes; + bool external_physically_absent,hpd_mutex,quiescing,typec_cable_connected;u64 typec_generation;unsigned int nr_modes; struct { int iomfb_method_profile; } hw; struct { bool lock,enabled;u64 epoch;int enable_completion;struct apple_epic_service *service; } dcpavserv; }; struct epic_cmd { u32 retcode;u64 rxbuf,txbuf;u32 rxlen,txlen; }; static unsigned int dma_frees,opens; + static int open_error,open_action; void dpavservep_detach(struct apple_dcp *dcp); static void inject_owner(struct apple_dcp_afkep *ep,u32 pending_channel); @@ -105,6 +113,7 @@ def function(file,signature): if(open_action==4) inject_owner(e,channel); if(open_action==5) e->dcp->typec_cable_connected=false; if(open_action==6) e->dcp->quiescing=true; + if(open_action==7) e->dcp->external_physically_absent=true; if(open_action==3) { for(unsigned int i=0;inum_channels;i++) if(e->services[i].channel==channel && !e->services[i].torndown) e->services[i].channel++; @@ -133,18 +142,18 @@ def function(file,signature): { assert(!h->key);*out=h->items[h->index++].integer;h->key=true;return 0; } static int skip(struct dcp_parse_ctx *h) { assert(!h->key);h->index++;h->key=true;return 0; } ''' - parts=[function('parser.c','int parse_epic_service_init('),function('afk.h','static inline struct apple_epic_service *afk_service_get('), + parts=[function('afk.c','static u16 afk_command_tag('),function('afk.c','void afk_service_retry('),function('parser.c','int parse_epic_service_init('),function('afk.h','static inline struct apple_epic_service *afk_service_get('), function('afk.h','static inline void afk_service_put('),function('afk.h','static inline void afk_service_retire('),function('afk.h','static inline void afk_service_disable('), function('afk.h','static inline bool afk_service_matches('),function('afk.h','static inline bool afk_service_reclaimable('), function('afk.h','static inline bool\nafk_service_reinit('),function('afk.h','static inline void afk_service_discard('), function('afk.h','static inline void afk_service_publish('), function('epic/dpavservep.c','static void dcpavserv_notify_edid('),function('epic/dpavservep.c','static bool dcpavserv_identity('), function('epic/dpavservep.c','static bool dcpavserv_admission_current('),function('epic/dpavservep.c','static int dcpavserv_prepare('), - function('epic/dpavservep.c','static void dcpavserv_init('),function('epic/dpavservep.c','static void dcpavserv_invalidate_edid('), + function('epic/dpavservep.c','static int dcpavserv_revalidate('),function('epic/dpavservep.c','static void dcpavserv_init('),function('epic/dpavservep.c','static void dcpavserv_invalidate_edid('), function('epic/dpavservep.c','static void dcpavserv_teardown('),function('epic/dpavservep.c','void dpavservep_detach('), function('afk.c','static const struct apple_epic_service_ops *\nafk_match_service('), function('afk.c','static struct apple_epic_service *afk_epic_find_service('), - function('afk.c','static void afk_recv_handle_init('), + function('afk.c','static void afk_discard_pending_open('),function('afk.c','static void afk_retry_service_init('),function('afk.c','static void afk_service_worker('),function('afk.c','static void afk_recv_handle_init('), function('epic/dpavservep.c','static int dcpdpserv_prepare('), function('epic/dpavservep.c','static void dcpdpserv_teardown('),function('afk.c','static void afk_recv_handle_teardown('), function('afk.c','static void afk_recv_handle_reply('),function('afk.c','static void afk_v2_receive(')] @@ -152,9 +161,10 @@ def function(file,signature): static struct apple_dcp d; static struct apple_dcp_afkep ep; static struct apple_connector con; -static const struct apple_epic_service_ops ops={.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.init=dcpavserv_init,.teardown=dcpavserv_teardown}; +static void run_retries(void) { assert(!ep.lock);if(ep.service_work.pending) { ep.service_work.pending=false;afk_service_worker(&ep.service_work); } } +static const struct apple_epic_service_ops ops={.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.revalidate=dcpavserv_revalidate,.init=dcpavserv_init,.teardown=dcpavserv_teardown}; static const struct apple_epic_service_ops announcement_ops[]={ - {.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.init=dcpavserv_init,.teardown=dcpavserv_teardown}, + {.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.revalidate=dcpavserv_revalidate,.init=dcpavserv_init,.teardown=dcpavserv_teardown}, {.name="dcpdp-service-epic",.reusable=true,.prepare=dcpdpserv_prepare,.teardown=dcpdpserv_teardown}, {0} }; @@ -268,14 +278,39 @@ def function(file,signature): assert(injected_owner->users==1);dpavservep_detach(&d);assert(!injected_owner->users); open_action=0;announce(13,name,native_class,13);assert(opens==++before && d.dcpavserv.service); struct apple_epic_service *old=d.dcpavserv.service; - assert(afk_service_get(old)==old);old->cmd_map[0]=1; + assert(afk_service_get(old)==old);ep.command_tags[13]=1;old->cmd_map[0]=1; disconnect();connect(); announce(13,name,native_class,13); /* Same channel still owns a pending reply. */ assert(opens==before && !d.dcpavserv.service && old->users==1 && old->channel==13); + /* The firmware need not announce again after the old reply drains. */ + old->cmds[0]=(struct epic_cmd_info){.tag=0,.type=0xc0,.rxlen=32,.txlen=32,.free_on_ack=true}; + old->cmds[0].rxbuf=calloc(1,32);old->cmds[0].txbuf=calloc(1,32); + old->cmds[0].rxbuf_dma=(uintptr_t)old->cmds[0].rxbuf; + old->cmds[0].txbuf_dma=(uintptr_t)old->cmds[0].txbuf; + u8 drain_oob[32],drain_reply[56];size_t drain_written; + assert(!afkv2_encode_oob_command(drain_oob,sizeof(drain_oob),0,old->cmds[0].rxbuf_dma,32,old->cmds[0].txbuf_dma,32)); + afkv2_put32(drain_oob+4,0); + assert(!afkv2_encode_frame(drain_reply,sizeof(drain_reply),1,13,0,0xc0,AFKV2_RESPONSE,0,drain_oob,sizeof(drain_oob),&drain_written)); + afk_v2_receive(&ep,0,0,drain_reply,drain_written); + run_retries(); + assert(d.dcpavserv.service && "a blocked reannouncement must recover after the old ACK without another announcement"); + assert(d.dcpavserv.service!=old && opens==++before && old->users==1 && !old->cmd_map[0]); + struct apple_epic_service *resumed=d.dcpavserv.service; + resumed->cmd_map[0]=1; + /* The DMA allocator can reuse both old addresses for the new command. */ + resumed->lock=true; + resumed->cmds[0]=(struct epic_cmd_info){.tag=afk_command_tag(resumed,0),.type=0xc0, + .rxbuf_dma=old->cmds[0].rxbuf_dma,.txbuf_dma=old->cmds[0].txbuf_dma,.rxlen=32,.txlen=32}; + resumed->lock=false; + afk_v2_receive(&ep,0,0,drain_reply,drain_written); + assert(!resumed->cmds[0].done && resumed->cmd_map[0] && "old ACK must not complete the next owner's command"); + resumed->cmd_map[0]=0; + dpavservep_detach(&d); announce(1,name,class,1);assert(opens==++before && d.dcpavserv.service!=old); struct apple_epic_service *fresh=d.dcpavserv.service;u64 epoch=d.dcpavserv.epoch; dcpavserv_teardown(old);assert(d.dcpavserv.service==fresh && d.dcpavserv.epoch==epoch); old->cmd_map[0]=0;afk_service_put(old);disconnect();connect(); + run_retries(); announce(13,name,native_class,13);assert(opens==++before && d.dcpavserv.service); struct apple_epic_service *owner=d.dcpavserv.service; u64 owner_epoch=d.dcpavserv.epoch; @@ -294,7 +329,7 @@ def function(file,signature): dpavservep_detach(&d); assert(!d.hpd_mutex && !d.dcpavserv.lock); /* Legacy remains unit-zero and class-agnostic, with its original epoch fence. */ - ep.wire_v2=false;ep.endpoint=0x28; + ep.wire_v2=false;ep.endpoint=0x28;d.external_physically_absent=true; assert(!prepare(13,13,name,native_class)); struct apple_epic_service *legacy=prepare(13,0,name,native_class);assert(legacy); dpavservep_detach(&d);afk_service_publish(legacy);dcpavserv_init(legacy,name,native_class,0); @@ -303,11 +338,82 @@ def function(file,signature): d.typec_generation++;afk_service_publish(legacy);dcpavserv_init(legacy,name,native_class,0); assert(d.dcpavserv.service==legacy && legacy->users==1);dpavservep_detach(&d); assert(dcpdpserv_prepare(legacy,"dcpdp-service-epic",native_class,0)==0); - ep.wire_v2=true;ep.endpoint=0x2a; + ep.wire_v2=true;ep.endpoint=0x2a;d.external_physically_absent=false; announce(U16_MAX,name,native_class,U16_MAX);assert(d.dcpavserv.service && d.dcpavserv.service->channel==U16_MAX); dpavservep_detach(&d); puts("PASS: actual AFKV2 HDMI/native DP alternating over 64 cycles: flip/revalidation/return, strict identity, stale OPEN generation/epoch/owner, auxiliary isolation, reader/command pins and legacy controls"); } + +static void deferred_announcements(void) +{ + for(unsigned int cycle=0;cycle<64;cycle++) { + unsigned int event=cycle%10,before=opens,freed=dma_frees; + u16 channel=1+4*(cycle%4); + const char *provider=(cycle%2)?native_class:class; + announce(channel,name,provider,channel); + struct apple_epic_service *old=d.dcpavserv.service; + assert(old && opens==++before && afk_service_get(old)==old); + old->cmd_map[0]=1; + old->cmds[0]=(struct epic_cmd_info){.tag=0,.type=0xc0,.rxlen=32,.txlen=32,.free_on_ack=true}; + old->cmds[0].rxbuf=calloc(1,32);old->cmds[0].txbuf=calloc(1,32); + old->cmds[0].rxbuf_dma=(uintptr_t)old->cmds[0].rxbuf; + old->cmds[0].txbuf_dma=(uintptr_t)old->cmds[0].txbuf; + u8 oob[32],reply[56];size_t written; + assert(!afkv2_encode_oob_command(oob,sizeof(oob),0,old->cmds[0].rxbuf_dma,32,old->cmds[0].txbuf_dma,32)); + afkv2_put32(oob+4,0); + assert(!afkv2_encode_frame(reply,sizeof(reply),1,channel,0,0xc0,AFKV2_RESPONSE,0,oob,sizeof(oob),&written)); + u64 generation=d.typec_generation; + closed(channel,0x6a01,0); + assert(!d.dcpavserv.service && old->users==1 && old->torndown && d.typec_generation==generation); + u64 epoch=d.dcpavserv.epoch; + announce(channel,name,provider,channel); + run_retries(); + assert(opens==before && !d.dcpavserv.service && dma_frees==freed); + for(unsigned int i=0;ipending_open) continue; + assert(dcpavserv_revalidate(pending,pending->pending_name,pending->pending_class,pending->pending_unit)==0); + d.dcpavserv.enabled=true; + assert(dcpavserv_revalidate(pending,pending->pending_name,pending->pending_class,pending->pending_unit)==-EBUSY); + d.dcpavserv.enabled=false; + } + if(event==1) d.typec_generation++; + if(event==2) dpavservep_detach(&d); + if(event==3) d.typec_cable_connected=false; + if(event==4) d.quiescing=true; + struct apple_epic_service *other=NULL; + if(event==5) { other=prepare(channel+2,channel+2,name,class);publish(other,channel+2); } + if(event==6) assert(!ep.stopping); + if(event==7) open_error=-EIO; + if(event==8) announce(channel,name,provider,channel); + if(event==9) { d.typec_generation++;announce(channel,name,provider,channel); } + afk_v2_receive(&ep,0,0,reply,written); + if(event==6) { assert(ep.service_work.pending);ep.stopping=true; } + run_retries(); + assert(dma_frees==freed+2 && old->users==1 && !old->cmd_map[0]); + if(event==0 || event==8 || event==9) { + assert(d.dcpavserv.service && d.dcpavserv.service!=old); + assert(d.dcpavserv.service->channel==channel && opens==++before); + assert(d.dcpavserv.epoch>epoch && d.dcpavserv.service->users==1); + } else if(event==5) { + assert(d.dcpavserv.service==other && opens==before); + } else { + assert(!d.dcpavserv.service && opens==before+(event==7)); + } + afk_v2_receive(&ep,0,0,reply,written); /* Duplicate late ACK cannot touch the new owner. */ + assert(dma_frees==freed+2); + afk_service_put(old);run_retries(); + assert(!old->users); + if(event==6) { + for(unsigned int i=0;iusers==1); assert(afk_service_get(old)==old); /* Delayed EDID reader pins the retired slot. */ @@ -375,6 +482,8 @@ def function(file,signature): assert(!pending->users && !d.hpd_mutex && !d.dcpavserv.lock); memset(&ep.services,0,sizeof(ep.services));ep.wire_v2=true;ep.endpoint=0x2a;d.hw.iomfb_method_profile=2; repeated_announcements(); + deferred_announcements(); + free(ep.command_tags);ep.command_tags=NULL; puts("PASS: actual dynamic 1/5/9/13 binding, forged identities, prepare/init epochs, physical retirement, reader pins, real old CLOSED/late ACK isolation and legacy unit 0"); } ''' @@ -461,7 +570,7 @@ def function(file, signature): struct { bool lock, enabled; u64 epoch; int enable_completion; struct apple_epic_service *service; } dcpavserv; struct apple_connector *connector; unsigned int nr_modes; struct { int iomfb_method_profile; } hw; - bool hpd_mutex, typec_cable_connected, quiescing; + bool external_physically_absent,hpd_mutex, typec_cable_connected, quiescing; u64 typec_generation; int fw_compat; void *dev; @@ -571,6 +680,8 @@ def function(file, signature): d.dcpavserv.service=&s; d.dcpavserv.enabled=true; s.enabled=true; s.users=1; event=0; d.typec_cable_connected=false; e=dcpavserv_copy_edid(&d,0,&epoch); assert(!IS_ERR(e) && legacy_reads==1 && s.users==1); drm_edid_free(e); + d.hw.iomfb_method_profile=0;av.wire_v2=false;d.external_physically_absent=true; + mutex_lock(&d.hpd_mutex);assert(dcpavserv_edid_current(&d,0));mutex_unlock(&d.hpd_mutex); assert(!allocations && !d.hpd_mutex && !d.dcpavserv.lock); puts("PASS: production old/new lookup and receive, exact service admission before OPEN, delayed disconnect/replug, service detach pins, legacy transport"); } diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py b/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py new file mode 100644 index 0000000000000..310af408b0f96 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py @@ -0,0 +1,439 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright 2026 Ryan Murray +"""Retained reviewer controls: actual AFK sender/receive, queue and EDID functions. + +The wrap controls deliberately retain an expected replay assertion failure. +They document an unresolved peer contract, not successful replay protection. +""" +from pathlib import Path +import json, hashlib, subprocess + +def check(root, output): + import edid + out = output / 'service-recovery-controls' + def extract(text,signature): + start=text.index(signature);opening=text.index('{',start) + while ';' in text[start:opening]: + start=text.index(signature,start+1);opening=text.index('{',start) + end,depth=opening+1,1 + while depth: + depth+=(text[end]=='{')-(text[end]=='}');end+=1 + return text[start:end] + + unit=edid.rebind(root) + # Retain the original exact fixture separately. Correct only its causal ordering. + corrected=unit.replace('assert(afk_service_get(old)==old);old->cmd_map[0]=1;', + 'assert(afk_service_get(old)==old);old->cmd_tag=1;old->cmd_map[0]=1;') + corrected=corrected.replace(' old->cmd_tag=1;\n','') + + sender=extract((root/'drivers/gpu/drm/apple/afk.c').read_text(), + 'static int afk_send_command_timeout_len(') + unit=unit[:unit.index('int main(void)')] + unit=unit.replace('static unsigned int dma_frees,opens;',r''' + static unsigned int dma_frees,opens; + #define SZ_1M (1U<<20) + #define GFP_KERNEL 0 + #define DECLARE_COMPLETION_ONSTACK(c) int c=0 + #define init_completion(c) (*(c)=0) + #define msecs_to_jiffies(x) (x) + #define cpu_to_le64(x) (x) + #define kmalloc(n,g) malloc(n) + #define EPIC_TYPE_COMMAND 3 + #define EPIC_CAT_COMMAND 3 + static int bitmap_find_free_region(unsigned long *p,int n,int order) { + (void)order;for(int i=0;iinline_v2) { + struct afkv2_command decoded;u8 response[40]={0}; + assert(!afkv2_decode_command(p,n,&decoded) && !decoded.oob && decoded.value==32 && decoded.id==tag); + response[1]=decoded.id; + assert(!afkv2_encode_frame(current_reply,sizeof(current_reply),1,channel,0,subtype,AFKV2_RESPONSE,0,response,sizeof(response),¤t_size)); + current_channel=channel;current_tag=tag;return 0; + } + assert(n==sizeof(struct epic_cmd)); + const struct epic_cmd *cmd=p;u8 oob[32]; + assert(!afkv2_encode_oob_command(oob,sizeof(oob),tag,cmd->rxbuf,cmd->rxlen,cmd->txbuf,cmd->txlen)); + afkv2_put32(oob+4,0); + assert(!afkv2_encode_frame(current_reply,sizeof(current_reply),1,channel,0,subtype,AFKV2_RESPONSE,0,oob,sizeof(oob),¤t_size)); + current_channel=channel;current_tag=tag;return 0; + } + '''+marker) + + tests=r''' + static void setup(void) { + d.hw.iomfb_method_profile=2;d.typec_cable_connected=true;d.typec_generation=1; + d.nr_modes=1;d.connector=&con;d.dev=&d; + con.dcp=&d;con.connected=true;con.base.registration_state=1; + ep.dcp=&d;ep.wire_v2=true;ep.endpoint=0x2a;ep.wq=&ep;ep.command_tags=calloc(AFKV2_INTERFACES,1);assert(ep.command_tags); + d.dcpavservep=&ep;ep.ops=announcement_ops;ep.match_epic_name=true; + d.fw_compat=DCP_FIRMWARE_H17P;hpd_lock=&d.hpd_mutex;av_lock=&d.dcpavserv.lock;active_ep=&ep; + } + static int send_actual(struct apple_epic_service *s,bool ack,bool late) { + u8 payload[32]={0},output[32]={0};u32 code; + ack_current=ack;inject_late=late; + int result=afk_send_command_timeout_len(s,0xc0,payload,32,output,32,&code,1,NULL); + inject_late=false;return result; + } + static void save_ack(void) { memcpy(saved_reply,current_reply,current_size);saved_size=current_size; } + static void cleanup(void) { + dpavservep_detach(&d);run_retries(); + for(unsigned int i=0;ichannel==5); + assert(send_actual(middle,true,true)==0); + closed(5,0x6a01,0);announce(13,name,native_class,13);run_retries(); + assert(d.dcpavserv.service && d.dcpavserv.service==middle && d.dcpavserv.service!=old); + assert(send_actual(d.dcpavserv.service,true,true)==0); + afk_service_put(old); + } else if(!strcmp(argv[1],"multiple-first-late-acks")) { + reuse_dma=false;assert(afk_service_get(old)==old); + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack(); + u8 first[56];size_t first_size=saved_size;memcpy(first,saved_reply,first_size); + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack(); + assert(old->cmd_map[0]==3); + closed(13,0x6a01,0);announce(13,name,native_class,13);run_retries();assert(!d.dcpavserv.service); + afk_v2_receive(&ep,0,0,first,first_size);run_retries(); + assert(!d.dcpavserv.service && old->cmd_map[0]==2 && old->users==1); + unsigned int freed=dma_frees; + afk_v2_receive(&ep,0,0,first,first_size);run_retries(); + assert(dma_frees==freed && old->cmd_map[0]==2 && !d.dcpavserv.service); + afk_v2_receive(&ep,0,0,saved_reply,saved_size);run_retries(); + assert(d.dcpavserv.service && d.dcpavserv.service!=old && !old->cmd_map[0]); + afk_service_put(old); + } else if(!strcmp(argv[1],"tag-wrap") || !strcmp(argv[1],"inline-tag-wrap")) { + assert(send_actual(old,true,false)==0);save_ack(); + for(unsigned int i=0;i<15;i++) assert(send_actual(old,true,true)==0); + assert(send_actual(old,true,true)==0); + } else if(!strcmp(argv[1],"pending-closed")) { + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack(); + closed(13,0x6a01,0);announce(13,name,native_class,13);run_retries(); + assert(!d.dcpavserv.service); + closed(13,0x6a01,0); + afk_v2_receive(&ep,0,0,saved_reply,saved_size);run_retries(); + assert(!d.dcpavserv.service && "CLOSED during pending recovery must invalidate the pending OPEN"); + } else if(!strcmp(argv[1],"actual-tag-before-announcement")) { + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack();assert(ep.command_tags[13]==1); + closed(13,0x6a01,0);announce(13,name,native_class,13); + afk_v2_receive(&ep,0,0,saved_reply,saved_size);run_retries(); + assert(d.dcpavserv.service && send_actual(d.dcpavserv.service,true,true)==0); + } else if(!strncmp(argv[1],"open-race-",10)) { + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack(); + closed(13,0x6a01,0);announce(13,name,native_class,13);run_retries(); + if(!strcmp(argv[1],"open-race-generation")) open_action=1; + else if(!strcmp(argv[1],"open-race-epoch")) open_action=2; + else if(!strcmp(argv[1],"open-race-owner")) open_action=4; + else if(!strcmp(argv[1],"open-race-cable")) open_action=5; + else if(!strcmp(argv[1],"open-race-quiescing")) open_action=6; + else if(!strcmp(argv[1],"open-race-absence")) open_action=7; + else assert(!"unknown OPEN race"); + afk_v2_receive(&ep,0,0,saved_reply,saved_size);run_retries(); + if(open_action==4) assert(d.dcpavserv.service==injected_owner); + else assert(!d.dcpavserv.service); + open_action=0;d.quiescing=false;d.typec_cable_connected=true;d.external_physically_absent=false; + } else if(!strcmp(argv[1],"physical-absence")) { + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack(); + closed(13,0x6a01,0);announce(13,name,native_class,13);run_retries(); + d.external_physically_absent=true; + afk_v2_receive(&ep,0,0,saved_reply,saved_size);run_retries(); + assert(!d.dcpavserv.service && "physical absence authority must fence deferred OPEN before cable/generation update"); + } else if(!strncmp(argv[1],"fence-",6)) { + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack(); + closed(13,0x6a01,0);announce(13,name,native_class,13);run_retries(); + unsigned int before=opens; + if(!strcmp(argv[1],"fence-epoch")) dpavservep_detach(&d); + else if(!strcmp(argv[1],"fence-generation")) d.typec_generation++; + else if(!strcmp(argv[1],"fence-cable")) d.typec_cable_connected=false; + else if(!strcmp(argv[1],"fence-quiescing")) d.quiescing=true; + else if(!strcmp(argv[1],"fence-owner")) inject_owner(&ep,13); + else if(!strcmp(argv[1],"fence-open-failure")) open_error=-EIO; + else assert(!"unknown fence"); + afk_v2_receive(&ep,0,0,saved_reply,saved_size);run_retries(); + assert(opens==before+(!strcmp(argv[1],"fence-open-failure"))); + if(!strcmp(argv[1],"fence-owner")) assert(d.dcpavserv.service==injected_owner); + else assert(!d.dcpavserv.service); + for(unsigned int i=0;ifn=(f);(w)->pending=false;(w)->next=NULL; } while(0) + static struct work_struct *fifo_head,*fifo_tail;static bool destroyed,rx_running; + ''') + queued=queued.replace('static bool queue_work(void *wq,struct work_struct *w) { assert(wq);bool fresh=!w->pending;w->pending=true;return fresh; }',r''' + static bool queue_work(void *wq,struct work_struct *w) { + assert(wq && !destroyed);if(w->pending) return false; + w->pending=true;w->next=NULL;if(fifo_tail) fifo_tail->next=w;else fifo_head=w;fifo_tail=w;return true; + } + static void run_one_work(void) { + assert(fifo_head);struct work_struct *w=fifo_head;fifo_head=w->next;if(!fifo_head) fifo_tail=NULL; + w->pending=false;w->next=NULL;assert(w->fn);w->fn(w); + } + static void destroy_workqueue(void *wq) { assert(wq && !destroyed);while(fifo_head) run_one_work();destroyed=true; } + #define IS_ERR_OR_NULL(p) (!(p) || IS_ERR(p)) + #define debugfs_remove_recursive(p) ((void)(p)) + #define kzalloc(n,g) calloc(1,n) + #define kvfree(p) free(p) + #define ESHUTDOWN 108 + #define RBEP_TYPE 0xffff + #define FIELD_GET(mask,val) ((val)&(mask)) + #define RBEP_INIT_ACK 0xa0 + #define RBEP_START_ACK 0x86 + #define RBEP_SHUTDOWN_ACK 0xc1 + #define RBEP_GETBUF 0x89 + #define RBEP_INIT_TX 0x8a + #define RBEP_INIT_RX 0x8b + #define RBEP_RECV 0x85 + #define complete_all(p) complete(p) + ''') + queued=queued.replace('struct apple_dcp_afkep { u8 *command_tags;bool lock,stopping;', + 'struct apple_dcp_afkep { u8 *command_tags;int started,stopped,txbfr,rxbfr;void *debugfs_entry;bool lock,stopping;') + queued=queued.replace('bool external_physically_absent,hpd_mutex,quiescing,typec_cable_connected;', + 'bool retain_dma,external_physically_absent,hpd_mutex,quiescing,typec_cable_connected;') + queued=queued.replace('static void run_retries(void) { assert(!ep.lock);if(ep.service_work.pending) { ep.service_work.pending=false;afk_service_worker(&ep.service_work); } }', + 'static void run_retries(void) { assert(!ep.lock);while(fifo_head) run_one_work(); }') + queued=queued.replace(' if(type==EPIC_TYPE_COMMAND) {', + ' if(e->stopping) return -ESHUTDOWN;\n if(type==EPIC_TYPE_COMMAND) {') + queued=queued[:queued.index('int main(int argc')] + queued+=r''' + struct afk_receive_message_work { struct apple_dcp_afkep *ep;u64 message;struct work_struct work; }; + static void afk_getbuf(struct apple_dcp_afkep *e,u64 m) { (void)e;(void)m;assert(!"unexpected GETBUF"); } + static void afk_init_rxtx(struct apple_dcp_afkep *e,u64 m,int *r) { (void)e;(void)m;(void)r;assert(!"unexpected INIT"); } + static unsigned int rx_step;static bool close_pending; + static bool afk_recv(struct apple_dcp_afkep *e) { + assert(e==&ep);rx_running=true; + if(rx_step==0) closed(13,0x6a01,0); + else if(rx_step==1) announce(13,name,native_class,13); + else if(rx_step==2) afk_v2_receive(e,0,0,saved_reply,saved_size); + else if(rx_step==3 && close_pending) closed(13,0x6a01,0); + else { assert(!d.dcpavserv.service && "recovery must wait for the RX drain loop to finish");rx_running=false;return false; } + assert(!d.dcpavserv.service && "recovery must remain behind RX on the ordered queue");rx_step++;return true; + } + ''' + af=(root/'drivers/gpu/drm/apple/afk.c').read_text() + for signature in ['static void afk_receive_message_worker(', 'int afk_receive_message(', 'void afk_quiesce(', 'static void afk_release_context(']: + queued+=extract(af,signature)+'\n' + queued+=r''' + int main(int argc,char **argv) { + assert(argc==2);setup();reuse_dma=true;INIT_WORK(&ep.service_work,afk_service_worker); + announce(13,name,native_class,13);struct apple_epic_service *old=d.dcpavserv.service;assert(old); + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack();run_retries(); + unsigned int before=opens; + if(!strncmp(argv[1],"ordered-",8)) { + close_pending=!strcmp(argv[1],"ordered-pending-closed"); + assert(!afk_receive_message(&ep,RBEP_RECV));assert(fifo_head); + run_one_work();assert(!rx_running && opens==before && !d.dcpavserv.service); + assert(fifo_head==&ep.service_work);run_retries(); + if(close_pending) assert(!d.dcpavserv.service && "ordered CLOSED must cancel pending OPEN"); + else { assert(d.dcpavserv.service && opens==before+1);assert(send_actual(d.dcpavserv.service,true,true)==0); } + cleanup(); + } else { + assert(!strcmp(argv[1],"quiesce-pending"));assert(afk_service_get(old)==old); + closed(13,0x6a01,0);announce(13,name,native_class,13); + afk_v2_receive(&ep,0,0,saved_reply,saved_size);assert(ep.service_work.pending); + afk_quiesce(&ep);assert(destroyed && !ep.wq && ep.stopping && !fifo_head); + assert(opens==before && !d.dcpavserv.service && old->users==1); + for(unsigned int i=0;iusers); + assert(afk_receive_message(&ep,RBEP_RECV)==-ESHUTDOWN);assert(!fifo_head); + afk_quiesce(&ep);assert(!fifo_head); + struct apple_dcp_afkep *empty=calloc(1,sizeof(*empty));assert(empty);empty->dcp=&d; + afk_release_context(empty); + cleanup(); + } + printf("PASS actual RX worker/quiesce: %s\n",argv[1]); + } + ''' + run('actual-queue-quiesce',queued,[[c] for c in ['ordered-recovery','ordered-pending-closed','quiesce-pending']]) + + # Actual EDID wrapper exercises its owner/reader pins, epoch and generation exits. + reader=unit[:unit.index('int main(int argc')] + reader+=r''' + #define ESTALE 116 + struct drm_edid { int marker; }; + static unsigned int edid_allocations,edid_frees;static int read_action; + struct reader_guard { bool *lock;bool once; }; + static struct reader_guard lock_reader_guard(bool *lock) { mutex_lock(lock);return (struct reader_guard){lock,true}; } + static void unlock_reader_guard(struct reader_guard *guard) { mutex_unlock(guard->lock); } + #define scoped_guard(kind,p) for(struct reader_guard _guard __attribute__((cleanup(unlock_reader_guard)))=lock_reader_guard(p);_guard.once;_guard.once=false) + static void drm_edid_free(const struct drm_edid *edid) { assert(edid && !IS_ERR(edid));free((void *)edid);edid_frees++; } + static const struct drm_edid *dcpavserv_read_edid_inline(struct apple_epic_service *service) { + assert(service->users==2 && "owner plus actual EDID wrapper reader pin"); + assert(send_actual(service,true,false)==0); + if(read_action==1) { closed(service->channel,0x6a01,0);announce(5,name,class,5);assert(d.dcpavserv.service!=service); } + if(read_action==2) d.typec_generation++; + if(read_action==3) d.external_physically_absent=true; + if(read_action==4) { dpavservep_detach(&d);d.quiescing=true; } + struct drm_edid *edid=malloc(sizeof(*edid));assert(edid);edid->marker=1;edid_allocations++;return edid; + } + static const struct drm_edid *dcpavserv_read_edid_legacy(struct apple_epic_service *service) { return dcpavserv_read_edid_inline(service); } + ''' + dpav=(root/'drivers/gpu/drm/apple/epic/dpavservep.c').read_text() + for signature in ['bool dcpavserv_edid_current(', 'bool dcpavserv_edid_service_current(', 'const struct drm_edid *dcpavserv_copy_edid(']: + reader+=extract(dpav,signature)+'\n' + reader+=r''' + int main(int argc,char **argv) { + assert(argc==2);setup();reuse_dma=true;announce(13,name,class,13); + struct apple_epic_service *old=d.dcpavserv.service;assert(old); + if(!strcmp(argv[1],"reader-owner")) read_action=1; + else if(!strcmp(argv[1],"reader-generation")) read_action=2; + else if(!strcmp(argv[1],"reader-physical-absence")) read_action=3; + else if(!strcmp(argv[1],"reader-quiescing")) read_action=4; + else assert(!strcmp(argv[1],"reader-success")); + u64 epoch;const struct drm_edid *edid=dcpavserv_copy_edid(&d,1,&epoch); + if(!read_action) { assert(!IS_ERR(edid));drm_edid_free(edid); } + else { assert(IS_ERR(edid) && "actual EDID wrapper must reject invalidated ownership/physical state"); } + if(read_action==1 || read_action==4) assert(!old->users);else assert(old->users==1); + assert(edid_allocations==edid_frees && !d.hpd_mutex && !d.dcpavserv.lock); + d.quiescing=false;cleanup();printf("PASS actual copy_edid: %s\n",argv[1]); + } + ''' + run('actual-edid-reader',reader,[[c] for c in ['reader-success','reader-owner','reader-generation','reader-quiescing','reader-physical-absence']]) + mutations=[ + ('restart-tags','afkv2_command_id(ep->command_tags[service->channel]++, slot)','afkv2_command_id(0, slot)'), + ('no-drain-retry','\tafk_service_retry(service->ep);',''), + ('ignore-old-command','\t\tif (afk_epic_find_service(ep, service->channel))\n\t\t\tcontinue;',''), + ('ignore-epoch','service->admission.epoch != dcp->dcpavserv.epoch ||\n\t\t ','false ||\n\t\t '), + ('refresh-generation','!dcpavserv_admission_current(dcp, service->admission.generation))\n\t\tret = -ENODEV;','!dcpavserv_admission_current(dcp, dcp->typec_generation))\n\t\tret = -ENODEV;'), + ('ignore-owner','if (dcp->dcpavserv.enabled)\n\t\tret = -EBUSY;\n\telse if (service->admission.epoch','if (false)\n\t\tret = -EBUSY;\n\telse if (service->admission.epoch'), + ('ignore-stop','if (!READ_ONCE(ep->stopping))\n\t\tafk_retry_service_init(ep);','if (true)\n\t\tafk_retry_service_init(ep);'), + ] + + for name,text,case in [ + ('cancel-closed',unit.replace('if (pending->pending_open && pending->channel == channel)','if (false && pending->pending_open && pending->channel == channel)'), 'pending-closed'), + ('reader-lookup',unit.replace('(service->enabled ||','(service->users || service->enabled ||'), 'drained-reader-rebind'), + ('reset-interface-history',unit.replace('service->cookie = NULL;', 'service->cookie = NULL; if(ep->wire_v2 && ep->command_tags) ep->command_tags[channel]=0;',1), 'channel-reuse'), + ('absence-admission',unit.replace('!READ_ONCE(dcp->external_physically_absent) &&','true &&',1), 'initial-absence'), + ('absence-publication',unit.replace('!READ_ONCE(dcp->external_physically_absent) &&','true &&',1), 'initial-open-absence'), + ('absence-reader',reader.replace('!READ_ONCE(dcp->external_physically_absent)', 'true'), 'reader-physical-absence'), + ]: + run('mutation-'+name,text,[[case]]) + + for name,old,new in mutations: + assert corrected.count(old)==1,(name,corrected.count(old)) + run('mutation-'+name,corrected.replace(old,new),[[]]) + negative={ + 'actual-sender':{'tag-wrap','inline-tag-wrap'}, + 'actual-queue-quiesce':set(), + 'actual-edid-reader':set(), + } + for result in results: + case=result.get('case',[]) + fail_expected=bool('case' in result and (result['name'].startswith('mutation-') or (case and case[0] in negative.get(result['name'],set())))) + result['assertion_failure_expected']=fail_expected + result['expectation_met']=result['exit']==(-6 if fail_expected else 0) + if fail_expected: result['expectation_met'] = result['expectation_met'] and 'Assertion' in result.get('output','') + if case and case[0] in {'tag-wrap','inline-tag-wrap'}: + result['expectation_met'] = result['expectation_met'] and 'late ACK completed an actual new production command' in result.get('output','') + (out/'results.json').write_text(json.dumps(results,indent=2)+'\n') + + assert all(r['expectation_met'] for r in results), "retained recovery control expectation failed; see results.json" diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py index fe62df8d78129..94349823fa03c 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -18,6 +18,7 @@ from common import run_unit import edid +import recovery import lifecycle import phy import pm @@ -90,6 +91,8 @@ def main(): unit, cases = unit if unit is not None: result['translation_unit_sha256'] = run_unit(name, unit, output, cases, includes if name == 'service-rebind' else (), flags) + if name == 'service-rebind': + recovery.check(root, output) if name == 'metadata-decoder': rate.check_metadata(output) result['passed'] = True From 4419114c830d30b3ab58a97faa74517da4b6b3d0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 22:11:04 +0000 Subject: [PATCH 1216/2680] drm/apple: Report an intentionally unsupported AFK service quietly Merge commit 08e0d0a90be3 ("drm/apple: integrate reviewed fixed-host lifecycle and scoped boot fixes") carries these drm/apple changes in its own resolution; no commit has them: - An AFK service op can be marked optional. When its prepare hook returns -EOPNOTSUPP, the service is discarded with an info message rather than an admission error. Required services still log an error. - The dcpdp-service-epic role needs no command transport on the external H17P. It is optional and returns -EOPNOTSUPP there. - The J700 provider-data fallback for older loaders names the missing property and its size. - The apple-dcp selftests follow: the atcphy PM model in pm.py, the updated Makefile, README, run.py and edid.py, and the new boot.py and teardown_pm.py. Re-made from the merge's drm/apple and apple-dcp selftest hunks for the 7.2.10 port. Its atc.c and rtkit.c hunks belong to 170-usb and 010-soc. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 9 +- drivers/gpu/drm/apple/afk.h | 2 + drivers/gpu/drm/apple/epic/dpavservep.c | 3 +- drivers/gpu/drm/apple/iomfb_h17p.c | 4 +- .../selftests/drivers/gpu/apple-dcp/Makefile | 2 +- .../selftests/drivers/gpu/apple-dcp/README | 9 +- .../selftests/drivers/gpu/apple-dcp/boot.py | 56 +++++ .../selftests/drivers/gpu/apple-dcp/edid.py | 48 +++- .../selftests/drivers/gpu/apple-dcp/pm.py | 223 ++++++++++++++++-- .../selftests/drivers/gpu/apple-dcp/run.py | 7 +- .../drivers/gpu/apple-dcp/teardown_pm.py | 156 ++++++++++++ 11 files changed, 486 insertions(+), 33 deletions(-) create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/boot.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index a617d35fd49f2..0d598980c96e4 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -506,8 +506,13 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, int ret = ops->prepare(service, epic_name, epic_class, epic_unit); if (ret) { - dev_err(ep->dcp->dev, "AFK service %s admission failed: %d\n", - service_name, ret); + if (ops->optional && ret == -EOPNOTSUPP) + dev_info(ep->dcp->dev, + "AFK optional %s (%s) is unsupported on channel %u\n", + ops->name, service_name, channel); + else + dev_err(ep->dcp->dev, "AFK service %s admission failed: %d\n", + service_name, ret); afk_service_discard(service); goto free; } diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index b51ac5ca8b9a4..0304410bd6e82 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -81,6 +81,8 @@ struct apple_epic_service_ops { const char name[32]; /* All retained pointers must be pinned before enabling slot reuse. */ bool reusable; + /* EOPNOTSUPP from prepare means intentional absence only for this role. */ + bool optional; /* * Optional admission, before OPEN/publication. Must not retain service diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 037a6388ffea2..a61d28e5e2edb 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -198,7 +198,7 @@ static int dcpdpserv_prepare(struct apple_epic_service *service, const char *nam const char *class, s64 unit) { /* No production command transport is needed for this H17P service. */ - return service->ep->wire_v2 ? -ENODEV : 0; + return service->ep->wire_v2 ? -EOPNOTSUPP : 0; } static void dcpdpserv_init(struct apple_epic_service *service, const char *name, @@ -447,6 +447,7 @@ static const struct apple_epic_service_ops dpavservep_ops[] = { { .name = "dcpdp-service-epic", .reusable = true, + .optional = true, .prepare = dcpdpserv_prepare, .init = dcpdpserv_init, .teardown = dcpdpserv_teardown, diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 869d6635deccb..dcce1e22d1575 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -453,7 +453,9 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, memset(reply, 0, sizeof(*reply)); memcpy(reply->data, property->legacy_data, property->length); reply->length = cpu_to_le32(property->length); - dev_warn_once(dcp->dev, "using J700 provider data for an older loader\n"); + dev_warn_once(dcp->dev, + "missing provider property %s (%u bytes); using qualified J700 compatibility data\n", + property->dt_name, property->length); return true; } else if (length != (int)property->length) { dev_err(dcp->dev, "invalid provider property %s: %d bytes\n", diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile index cdc821ce4be25..bbb39cada55bf 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT TEST_PROGS := run.sh -TEST_FILES := run.py common.py edid.py recovery.py rate.py phy.py lifecycle.py pm.py idle.py dwc3 README +TEST_FILES := run.py common.py edid.py recovery.py rate.py phy.py lifecycle.py pm.py idle.py boot.py dwc3 README include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/README b/tools/testing/selftests/drivers/gpu/apple-dcp/README index e9bf9bdabcb67..b04b6b9934193 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/README +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/README @@ -10,7 +10,7 @@ Or keep translation units, commands, failure logs and source hashes: tools/testing/selftests/drivers/gpu/apple-dcp/run.sh --output /tmp/apple-dcp-results An installed copy needs --source /path/to/kernel. --group edid, rate, phy or -lifecycle selects one group. HOSTCC selects the host compiler (default clang). +lifecycle or boot selects one group. HOSTCC selects the host compiler (default clang). Python 3 and a compiler supporting AddressSanitizer/UndefinedBehaviorSanitizer are required. No target access, firmware command, captured fixture or kernel configuration is required. @@ -68,3 +68,10 @@ still reproduce, alongside nonvacuous mutations and passing corrected safety controls. A passing host suite does not establish a replay-exclusion contract or admit a hardware trial. Token/sequence semantics remain a separate protocol fact to establish; the tests do not manufacture a firmware echo guarantee. + +The boot group executes the actual OSLOG dispatcher with synthetic fixed-width +words. Known opaque event types remain visible, unknown types retain warnings, +and only the existing buffer-request path can allocate/ACK buffers. The EDID +service-rebind group also distinguishes intentional optional EOPNOTSUPP from +required admission failures and genuine optional-service errors, and proves +refused services never OPEN or publish. No captured firmware values are used. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py b/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py new file mode 100644 index 0000000000000..bf95df845443d --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright 2026 Ryan Murray +"""Opaque RTKit event dispatch: no firmware addresses or captured payloads.""" +from common import block + +def oslog(root): + source = (root / 'drivers/soc/apple/rtkit.c').read_text() + mask = next(line for line in source.splitlines() + if line.startswith('#define APPLE_RTKIT_OSLOG_TYPE ')) + return r''' +#include +#include +#include +typedef uint64_t u64; +typedef uint8_t u8; +#define GENMASK_ULL(h,l) ((~0ULL << (l)) & (~0ULL >> (63-(h)))) +#define FIELD_GET(mask,value) (((value) & (mask)) >> __builtin_ctzll(mask)) +#define APPLE_RTKIT_OSLOG_BUFFER_REQUEST 1 +#define APPLE_RTKIT_EP_OSLOG 8 +struct buffer { unsigned sentinel; }; +struct apple_rtkit { void *dev; struct buffer oslog_buffer; }; +static unsigned warnings, information, requests; +static u64 observed; +static u8 observed_type; +#define dev_warn(dev,fmt,msg) do { (void)(dev); (void)(fmt); warnings++; observed=(msg); } while (0) +#define dev_info(dev,fmt,type,msg) do { (void)(dev); (void)(fmt); information++; observed_type=(type); observed=(msg); } while (0) +static void apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, struct buffer *b, u8 ep, u64 msg) +{ assert(b==&rtk->oslog_buffer && ep==8); requests++; observed=msg; } +''' + mask + '\n' + block(source, 'static void apple_rtkit_oslog_rx(') + r''' +static void reset(void) { warnings=information=requests=0; observed=0; observed_type=0; } +int main(void) +{ + struct apple_rtkit rtk={.oslog_buffer={.sentinel=0x12345678}}; + for (u8 type=3; type<=5; type++) { + for (unsigned edge=0; edge<2; edge++) { + u64 msg=((u64)type<<56) | (edge ? ((1ULL<<56)-1) : 0); + reset(); apple_rtkit_oslog_rx(&rtk,msg); + assert(!warnings && information==1 && !requests); + assert(observed==msg && observed_type==type); + assert(rtk.oslog_buffer.sentinel==0x12345678); + } + } + puts("PASS recognized opaque types preserve both payload bounds without buffer/ACK operations"); + const u8 unknown[]={0,2,6,0xff}; + for (unsigned i=0;ipending;w->pending=true;return fresh; } struct apple_dcp;struct apple_dcp_afkep;struct apple_epic_service; void afk_service_retry(struct apple_dcp_afkep *ep); -struct apple_epic_service_ops { char name[32];bool reusable;void (*init)(struct apple_epic_service *,const char *,const char *,s64);int (*prepare)(struct apple_epic_service *,const char *,const char *,s64);int (*revalidate)(struct apple_epic_service *,const char *,const char *,s64);void (*teardown)(struct apple_epic_service *); }; +struct apple_epic_service_ops { char name[32];bool reusable,optional;void (*init)(struct apple_epic_service *,const char *,const char *,s64);int (*prepare)(struct apple_epic_service *,const char *,const char *,s64);int (*revalidate)(struct apple_epic_service *,const char *,const char *,s64);void (*teardown)(struct apple_epic_service *); }; struct epic_cmd_info { u16 tag;u8 type;size_t reply_length;void *rxbuf,*txbuf;dma_addr_t rxbuf_dma,txbuf_dma;size_t rxlen,txlen;u32 retcode;bool done,free_on_ack;int *completion; }; struct apple_epic_service { struct apple_dcp_afkep *ep;const struct apple_epic_service_ops *ops; @@ -136,7 +138,9 @@ def function(file,signature): { struct item *i=&h->items[h->index];const char *p; if(h->key) { p=i->key;h->key=false; } else { p=i->value;h->index++;h->key=true; } - if(!p) return ERR_PTR(-EINVAL);char *copy=malloc(strlen(p)+1);assert(copy);strcpy(copy,p);return copy; + if(!p) + return ERR_PTR(-EINVAL); + char *copy=malloc(strlen(p)+1);assert(copy);strcpy(copy,p);return copy; } static int parse_int(struct dcp_parse_ctx *h,s64 *out) { assert(!h->key);*out=h->items[h->index++].integer;h->key=true;return 0; } @@ -165,8 +169,8 @@ def function(file,signature): static const struct apple_epic_service_ops ops={.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.revalidate=dcpavserv_revalidate,.init=dcpavserv_init,.teardown=dcpavserv_teardown}; static const struct apple_epic_service_ops announcement_ops[]={ {.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.revalidate=dcpavserv_revalidate,.init=dcpavserv_init,.teardown=dcpavserv_teardown}, - {.name="dcpdp-service-epic",.reusable=true,.prepare=dcpdpserv_prepare,.teardown=dcpdpserv_teardown}, - {0} + {.name="dcpdp-service-epic",.reusable=true,.optional=true,.prepare=dcpdpserv_prepare,.teardown=dcpdpserv_teardown}, + {.name=""} }; static const char *name="dcpav-service-epic",*class="AppleDCPDP2HDMI"; static const char *native_class="DCPDP13Service"; @@ -414,12 +418,42 @@ def function(file,signature): } puts("PASS: 64 actual CLOSED/reannounce/late-ACK cycles with deferred OPEN, unchanged attachment, duplicate announcement/reply, stale generation/epoch, cable/quiesce/owner/stop and OPEN-error controls"); } +static int injected_prepare_error; +static int failed_prepare(struct apple_epic_service *s,const char *n,const char *c,s64 u) +{ (void)s;(void)n;(void)c;(void)u;return injected_prepare_error; } +static void optional_admission_controls(void) +{ + ep.ops=announcement_ops;ep.match_epic_name=true;ep.num_channels=0;d.fw_compat=DCP_FIRMWARE_H17P; + diagnostic_errors=diagnostic_information=0; + unsigned before=opens; + announce(3,"dcpdp-service-epic",class,3); + announce(5,"dcpdp-service-epic",native_class,5); + assert(!diagnostic_errors && diagnostic_information==2 && opens==before); + assert(!d.dcpavserv.enabled && !d.dcpavserv.service); + for(unsigned i=0;ihcd) #define HCD_DEAD(hcd) (!!((hcd)->flags&(1UL<flags&(1UL<refs++;return d; } +static void device_lock(struct device *d) { + if(d==&controller.dev){assert(!controller_locked&&!host_locked&&!root_locked);controller_locked=true;} + else {assert(d==&host_device&&controller_locked&&!host_locked&&!root_locked);host_locked=true;} +} +static void device_unlock(struct device *d) { + if(d==&host_device){assert(host_locked&&!root_locked);host_locked=false;} + else {assert(d==&controller.dev&&controller_locked&&!host_locked&&!root_locked);controller_locked=false;} +} static void put_device(struct device *d) { assert(d->refs>0);d->refs--; } static struct usb_device *usb_get_dev(struct usb_device *u) { u->refs++;return u; } static void usb_put_dev(struct usb_device *u) { assert(u->refs>0);u->refs--; } static int usb_for_each_dev(void *data,int (*fn)(struct usb_device *,void *)) { if(root_missing)return 0;for(unsigned int i=0;i<4;i++){int r=fn(&roots[i],data);if(r)return r;}return 0; } -static void usb_lock_device(struct usb_device *u) { assert(u==&roots[3]&&!root_locked);root_locked=true;if(detach_on_lock)u->state=0; } -static void usb_unlock_device(struct usb_device *u) { assert(u==&roots[3]&&root_locked);root_locked=false; } +static struct usb_device *usb_hub_find_child(struct usb_device *u, int port) { assert(u==&roots[3]&&root_locked&&port==1); + if(detach_at_lookup)usb_set_device_state(u,USB_STATE_NOTATTACHED); + if(quiesce_at_lookup)hcd.state=HC_STATE_QUIESCING; + return link_present?&link_device:NULL; } +static void usb_lock_device(struct usb_device *u) { + if(u==&roots[3]){assert(!root_locked);if(REQUIRES_PLATFORM_LOCKS)assert(controller_locked&&host_locked);root_locked=true;if(detach_on_lock)u->state=0;} + else {assert(u==&link_device&&root_locked&&!link_locked);link_locked=true;} +} +static void usb_unlock_device(struct usb_device *u) { + if(u==&roots[3]){assert(root_locked&&!link_locked);root_locked=false;} + else {assert(u==&link_device&&link_locked&&root_locked);link_locked=false;} +} static int pm_runtime_resume_and_get(struct device *d) { - assert(d==&roots[3].dev&&root_locked&&!pm_phy->lock);resumes++; + assert(root_locked&&!pm_phy->lock); + if(d==&link_device.dev){ + assert(link_locked);link_resumes++; + struct lock_guard g=lock_take(&pm_phy->lock);lock_release(&g); + if(link_error)return link_error; + link_pm_refs++;link_suspended=false;stuck_ack=false; + if(link_detach_on_resume)link_device.state=0; + if(link_dead_on_resume)set_bit(HCD_FLAG_DEAD,&hcd.flags); + return 0; + } + assert(d==&roots[3].dev);resumes++; /* A re-entrant PHY lock acquisition during resume must remain possible. */ struct lock_guard g=lock_take(&pm_phy->lock);lock_release(&g); if(resume_error)return resume_error; - pm_refs++;stuck_ack=false;if(detach_on_resume)roots[3].state=0; + pm_refs++;stuck_ack=link_present&&link_device.state!=USB_STATE_NOTATTACHED&&link_suspended;if(detach_on_resume)roots[3].state=0; if(inaccessible_on_resume)clear_bit(HCD_FLAG_HW_ACCESSIBLE,&hcd.flags); if(dead_on_resume)set_bit(HCD_FLAG_DEAD,&hcd.flags); return 0; } -static void pm_runtime_put_noidle(struct device *d) { assert(d==&roots[3].dev&&pm_refs==1);pm_refs--; } -static void usb_mark_last_busy(struct usb_device *u) { assert(u==&roots[3]&&root_locked&&!pm_phy->lock); } -static void pm_runtime_put_autosuspend(struct device *d) { assert(d==&roots[3].dev&&root_locked&&!pm_phy->lock&&pm_refs==1);pm_refs--; } +static void pm_runtime_put_noidle(struct device *d) { + if(d==&link_device.dev){assert(link_pm_refs==1);link_pm_refs--;} + else {assert(d==&roots[3].dev&&pm_refs==1);pm_refs--;} +} +static void usb_mark_last_busy(struct usb_device *u) { + assert(root_locked&&!pm_phy->lock); + if(u==&link_device)assert(link_locked);else assert(u==&roots[3]&&!link_locked); +} +static void pm_runtime_put_autosuspend(struct device *d) { + assert(root_locked&&!pm_phy->lock); + if(d==&link_device.dev){assert(link_locked&&link_pm_refs==1);link_pm_refs--;} + else {assert(d==&roots[3].dev&&!link_locked&&pm_refs==1);pm_refs--;} +} ''' suffix=ns['suffix'].replace('int main(', 'int __attribute__((unused)) fixture_main(',1) controls=r''' @@ -96,16 +143,23 @@ def ownership(root, usb_unreachable=False): { hw.has_t8140_dp=t8140;initial(p,fixed,false,swap); p->np=&ctrl_node;pm_phy=p; - hcd=(struct usb_hcd){.rh_registered=true,.flags=1UL<lock&&!root_locked&&!pm_refs&&!node_refs&&!controller.dev.refs); + assert(!controller_locked&&!host_locked&&!host_device.refs&&!p->lock&&!root_locked&&!link_locked&&!link_pm_refs&&!link_device.refs&&!pm_refs&&!node_refs&&!controller.dev.refs); for(unsigned int i=0;i<4;i++)assert(!roots[i].refs); } int main(void) @@ -180,6 +234,49 @@ def ownership(root, usb_unreachable=False): prepare(&p,true,false,false);before=writes;assert(atcphy_mux_set(&mux,&same)==0&&resumes==0&&writes==before);balanced(&p); prepare(&p,true,true,false);before=writes; assert(atcphy_mux_set(&mux,&same)==0&&writes==before&&resumes==1);balanced(&p); + /* Binding/identity failure must not reach USB PM or PHY programming. */ + for(unsigned int missing=0;missing<3;missing++){ + prepare(&p,true,true,false);before=writes; + if(missing==0)controller.dev.driver=NULL; + if(missing==1)host_device.driver=NULL; + if(missing==2)host_device.parent=&wrong_controller; + assert(safe(&p)==(missing==2?-ENODEV:-EAGAIN)&&writes==before&&resumes==0);balanced(&p); + } + /* Root state can change during lookup despite USB device locks. */ + for(unsigned int event=0;event<2;event++){ + prepare(&p,true,true,false);before=writes; + if(event==0)detach_at_lookup=1;else quiesce_at_lookup=1; + int ret=safe(&p);fprintf(stderr,"post-child admission event %u returned %d writes %u\n",event,ret,writes-before); + assert(ret==-ENODEV&&writes==before);balanced(&p); + } + /* Selectively suspended upstream link is not resumed by root ownership alone. */ + for(unsigned int swap=0;swap<2;swap++){ + prepare(&p,true,true,swap);link_present=link_suspended=stuck_ack=true; + int link_ret=safe(&p);fprintf(stderr,"selective link SAFE returned %d, root resumes %d, link resumes %d\n",link_ret,resumes,link_resumes); + assert(link_ret==0&&resumes==1&&link_resumes==1&&!link_suspended);balanced(&p); + } + for(unsigned int fault=0;fault<3;fault++){ + prepare(&p,true,true,false);link_present=link_suspended=true;before=writes; + unsigned int retained=p.dp_usb_context_valid,owners=p.dp_owned_pmas; + if(fault==0)link_error=-EIO; + if(fault==1)link_detach_on_resume=1; + if(fault==2)link_dead_on_resume=1; + assert(safe(&p)==(fault?-ENODEV:-EIO)&&writes==before&&p.dp_usb_context_valid==retained&&p.dp_owned_pmas==owners);balanced(&p); + } + prepare(&p,true,true,false);link_present=link_suspended=true;link_error=-EIO; + assert(safe(&p)==-EIO);balanced(&p);link_error=0;assert(safe(&p)==0);balanced(&p); + prepare(&p,true,true,false);link_present=link_suspended=true;fail_caller="atcphy_pipehandler_lock";before=writes; + assert(safe(&p)==-ETIMEDOUT&&writes==before&&p.mode_dirty&&p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3);balanced(&p); + fail_caller=NULL;assert(safe(&p)==0);balanced(&p); + for(unsigned int count=0;count<2;count++){ + prepare(&p,true,true,false);roots[3].maxchild=count?2:0;before=writes; + assert(safe(&p)==(count?-ENODEV:-EAGAIN)&&writes==before);balanced(&p); + } + prepare(&p,true,true,false);link_present=true;link_device.state=USB_STATE_NOTATTACHED; + assert(safe(&p)==0&&link_resumes==0);balanced(&p); + /* Startup with no old child is allowed; no fake dock or link is required. */ + prepare(&p,true,true,false);assert(safe(&p)==0&&link_resumes==0);balanced(&p); + puts("PASS actual upstream-link ownership: both PMAs, selective suspend, resume/removal/death/ACK failures, retry, unknown port map and no-child startup"); puts("PASS actual PM wrapper + actual parking/restore: both PMAs, gated clock, PM error/retry, fatal ACK/retry, graph/controller/root/removal gates, legacy/ordinary and cached mode"); return 0; } @@ -187,7 +284,10 @@ def ownership(root, usb_unreachable=False): helpers='' if has_pm: helpers=ns['block'](s,'struct atcphy_usb3_pm {')+';\n' - helpers+='#if IS_REACHABLE(CONFIG_USB)\n'+ns['function'](s,'atcphy_usb3_pm_find')+'\n#endif\n' + helpers+='#if IS_REACHABLE(CONFIG_USB)\n' + if 'static int atcphy_usb3_link_get(' in s: + helpers+=ns['function'](s,'atcphy_usb3_link_get')+'\n'+ns['function'](s,'atcphy_usb3_link_put')+'\n' + helpers+=ns['function'](s,'atcphy_usb3_pm_find')+'\n#endif\n' helpers+=ns['function'](s,'atcphy_usb3_pm_get')+'\n'+ns['function'](s,'atcphy_usb3_pm_put')+'\n' # The fixture extractor adapts the locked body; extract the wrapper directly. marker=re.search(r'^static int atcphy_mux_set\(',s,re.M).group() @@ -201,7 +301,7 @@ def ownership(root, usb_unreachable=False): typec_caller=ns['block'](typec_source,'int typec_mux_set(')+'\n' hcd_source=(root/'drivers/usb/core/hcd.c').read_text() hcd_death=ns['block'](hcd_source,'void usb_hc_died (')+'\n' - text=prefix+prod+'\n#define USB_REACHABLE '+str(int(not usb_unreachable))+'\n#define HCD_DEAD_ONLY '+str(int(False))+'\n'+pmfixture+helpers+typec_caller+hcd_death+suffix+controls + text=prefix+prod+'\n#define REQUIRES_PLATFORM_LOCKS '+str(int('device_lock(&pm->controller->dev)' in s))+'\n#define USB_REACHABLE '+str(int(not usb_unreachable))+'\n#define HCD_DEAD_ONLY '+str(int(False))+'\n'+pmfixture+helpers+typec_caller+hcd_death+suffix+controls return text def probe(root): @@ -368,7 +468,7 @@ def kconfig(root, output): compiler = Path(os.environ.get('KCONFIG_CONF', root / 'scripts/kconfig/conf')) if not compiler.is_file(): subprocess.run(['make', '-C', str(root), 'O=' + str(directory / 'producer'), - 'ARCH=arm64', 'LLVM=1', 'scripts/kconfig/conf'], check=True) + 'ARCH=arm64', 'LLVM=1', 'allnoconfig'], check=True) compiler = directory / 'producer/scripts/kconfig/conf' for usb, phy_setting in [('n', 'y'), ('y', 'y'), ('m', 'y'), ('m', 'm')]: case = directory / (usb + '-' + phy_setting) @@ -451,3 +551,88 @@ def probe_dispatch(root): } """ return unit + '\n' + port + '\n' + controls + + +def upstream_resume_boundary(root): + """Actual usbcore dispatch: root bus resume does not resume its child.""" + source = (root / 'drivers/usb/core/generic.c').read_text() + function = block(source, 'int usb_generic_driver_resume(') + prefix = r""" +#include +#include +#include +#include +typedef int pm_message_t; +struct usb_device { struct usb_device *parent; }; +static int bus_resumes, port_resumes, notifications, port_error; +static bool child_suspended = true; +static int hcd_bus_resume(struct usb_device *u, pm_message_t m) +{ (void)m; assert(!u->parent); bus_resumes++; return 0; } +static int usb_port_resume(struct usb_device *u, pm_message_t m) +{ (void)m; assert(u->parent); port_resumes++; if(port_error)return port_error; child_suspended=false; return 0; } +static void usbfs_notify_resume(struct usb_device *u) +{ (void)u; notifications++; } +""" + controls = r""" +int main(void) { + struct usb_device root={0}, child={.parent=&root}; + assert(usb_generic_driver_resume(&root,1)==0); + assert(bus_resumes==1&&port_resumes==0&&child_suspended&¬ifications==1); + assert(usb_generic_driver_resume(&child,1)==0); + assert(bus_resumes==1&&port_resumes==1&&!child_suspended&¬ifications==2); + child_suspended=true;port_error=-EIO; + assert(usb_generic_driver_resume(&child,1)==-EIO&&child_suspended&¬ifications==2); + puts("PASS actual usbcore root/child resume dispatch and failed child resume"); + return 0; +} +""" + return prefix + function + controls + + +def controller_lifecycle(root): + from teardown_pm import unit_for + return unit_for(root) + + +def asynchronous_root_state(root): + """Actual USB state API does not use the USB device mutex as its fence.""" + source = (root / 'drivers/usb/core/hub.c').read_text() + prefix = r""" +#include +#include +#include +enum usb_device_state { USB_STATE_NOTATTACHED, USB_STATE_CONFIGURED, USB_STATE_SUSPENDED }; +struct usb_device; +struct usb_port { struct usb_device *child; }; +struct usb_hub { struct usb_port **ports; }; +struct device { int unused; }; +struct config { struct { unsigned bmAttributes; } desc; }; +struct usb_device { struct device dev; struct usb_device *parent; struct usb_hub *hub; enum usb_device_state state; int maxchild; unsigned quirks; struct config *actconfig; unsigned long active_duration; bool device_locked; }; +static unsigned long jiffies=10; +static int device_state_lock; +#define USB_QUIRK_IGNORE_REMOTE_WAKEUP 1 +#define USB_CONFIG_ATT_WAKEUP 2 +#define spin_lock_irqsave(l,f) do { assert(!*(l));*(l)=1;(f)=0; } while(0) +#define spin_unlock_irqrestore(l,f) do { assert(*(l));*(l)=0;(void)(f); } while(0) +static struct usb_hub *usb_hub_to_struct_hub(struct usb_device *u) { return u->hub; } +static void update_port_device_state(struct usb_device *u) { (void)u; } +static void trace_usb_set_device_state(struct usb_device *u) { (void)u; } +static void device_set_wakeup_capable(struct device *d,int w) { (void)d;(void)w; } +""" + controls = r""" +int main(void) { + struct usb_device child={.state=USB_STATE_CONFIGURED,.device_locked=true}; + struct usb_port port={.child=&child};struct usb_port *ports[]={&port}; + struct usb_hub hub={.ports=ports}; + struct usb_device root={.state=USB_STATE_CONFIGURED,.maxchild=1,.hub=&hub,.device_locked=true}; + child.parent=&root; + usb_set_device_state(&root,USB_STATE_NOTATTACHED); + assert(root.state==USB_STATE_NOTATTACHED&&child.state==USB_STATE_NOTATTACHED); + assert(root.device_locked&&child.device_locked&&!device_state_lock); + usb_set_device_state(&child,USB_STATE_CONFIGURED); + assert(child.state==USB_STATE_NOTATTACHED); + puts("PASS actual asynchronous USB detach while device locks are held; no resurrection"); + return 0; +} +""" + return prefix + block(source, 'static void update_usb_device_state(') + block(source, 'static void recursively_mark_NOTATTACHED(') + block(source, 'void usb_set_device_state(') + controls diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py index 94349823fa03c..3365394b513e4 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -24,6 +24,7 @@ import pm import idle import rate +import boot def parking_unit(root): @@ -35,7 +36,7 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--source', type=Path, default=Path(__file__).resolve().parents[6]) parser.add_argument('--output', type=Path, help='retain generated C, commands and results here') - parser.add_argument('--group', choices=('all', 'edid', 'rate', 'phy', 'lifecycle'), default='all') + parser.add_argument('--group', choices=('all', 'edid', 'rate', 'phy', 'lifecycle', 'boot'), default='all') args = parser.parse_args() root = args.source.resolve() output = args.output or Path(tempfile.mkdtemp(prefix='apple-dcp-selftest-')) @@ -46,6 +47,7 @@ def main(): 'orientation-faults', 'backend-change', 'safe-first', 'failed-flip-return', 'live-park')) tests = [ + ('boot', 'oslog-events', boot.oslog, ((),), ()), ('edid', 'service-rebind', edid.rebind, ((),), ()), ('edid', 'provider-admission', edid.admission, ((),), ()), ('edid', 'ordered-rx', edid.rx_progress, ((),), ('-pthread',)), @@ -54,6 +56,9 @@ def main(): ('rate', 'metadata-decoder', lambda r: rate.metadata(r, output), ((),), ('-Wno-sign-compare',)), ('rate', 'bounded-retry', rate.retry, ((),), ()), ('phy', 'pm-ownership', pm.ownership, ((),), ()), + ('phy', 'controller-lifecycle', pm.controller_lifecycle, ((),), ('-pthread',)), + ('phy', 'asynchronous-root-state', pm.asynchronous_root_state, ((),), ()), + ('phy', 'upstream-resume-boundary', pm.upstream_resume_boundary, ((),), ()), ('phy', 'usb-unreachable', lambda r: pm.ownership(r, True), ((),), ()), ('lifecycle', 'cpuidle-s2idle', idle.s2idle, ((),), ('-Wno-unused-parameter', '-Wno-unused-variable')), ('lifecycle', 'typec-probe', pm.probe, ((),), ()), diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py new file mode 100644 index 0000000000000..1f17261136a9c --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py @@ -0,0 +1,156 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Actual wrapper/driver-core/DWC3/xHCI teardown order with modeled scheduling. + +Device mutex APIs use pthread mutexes. Clock, unregister, PM and hardware leaves +are mocks; this is not lockdep, system-PM or electrical atomicity qualification. +""" +import pm +from common import block + +def unit_for(source): + unit=pm.ownership(source) + unit=unit.replace('int main(void)','int __attribute__((unused)) ownership_main(void)',1) + unit=unit.replace('struct device_driver { const char *name; };', 'struct device;\nstruct device_driver { const char *name; void (*remove)(struct device *); void *dev_groups; };') + unit=unit.replace('struct device { int refs;', 'struct bus_type { bool need_parent_lock; void (*remove)(struct device *); void (*dma_cleanup)(struct device *); };\nstruct device_private { int knode_driver; };\nstruct device { struct bus_type *bus; struct device_private *p; int kobj; int refs;') + unit=unit.replace('static struct device host_device;', 'static struct platform_device host_platform_device;\n#define host_device host_platform_device.dev') + unit=unit.replace('static bool controller_locked, host_locked;', 'static _Atomic bool controller_locked, host_locked;') + pre=r''' +#include +#include +static pthread_mutex_t controller_mutex=PTHREAD_MUTEX_INITIALIZER, host_mutex=PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t schedule_mutex=PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t schedule_cond=PTHREAD_COND_INITIALIZER; +static _Thread_local bool remover; +static atomic_bool handoff_active, early_write; +static bool removal_started, blocked_attempt, release_remove, inject_remove; +static int remove_target, susphy_calls, host_remove_calls; +static pthread_t remove_thread; +static void start_remove(void); +static void removal_lock_attempt(bool held) { + if(!remover||!held)return; + pthread_mutex_lock(&schedule_mutex);blocked_attempt=true; + pthread_cond_broadcast(&schedule_cond);pthread_mutex_unlock(&schedule_mutex); +} +static void observed_controller_write(void) { + if(atomic_load(&handoff_active))atomic_store(&early_write,true); + pthread_mutex_lock(&schedule_mutex); + removal_started=true;pthread_cond_broadcast(&schedule_cond); + while(atomic_load(&handoff_active)&&!release_remove)pthread_cond_wait(&schedule_cond,&schedule_mutex); + pthread_mutex_unlock(&schedule_mutex); +} +''' + unit=unit.replace('#define CONFIG_USB 1',pre+'\n#define CONFIG_USB 1',1) + old=block(unit,'static void device_lock(') + new=r'''static void device_lock(struct device *d) { + if(d==&controller.dev){ + removal_lock_attempt(atomic_load(&controller_locked)); + pthread_mutex_lock(&controller_mutex);controller_locked=true; + }else{ + assert(d==&host_device);removal_lock_attempt(atomic_load(&host_locked)); + pthread_mutex_lock(&host_mutex);host_locked=true; + } +}''' + unit=unit.replace(old,new) + old=block(unit,'static void device_unlock(') + new=r'''static void device_unlock(struct device *d) { + if(d==&host_device){assert(host_locked);host_locked=false;pthread_mutex_unlock(&host_mutex);} + else {assert(d==&controller.dev&&controller_locked);if(!remover)atomic_store(&handoff_active,false);controller_locked=false;pthread_mutex_unlock(&controller_mutex);} +}''' + unit=unit.replace(old,new) + # The scheduler injection is a child-lookup interleaving, as in the review. + unit=unit.replace('if(detach_at_lookup)usb_set_device_state(u,USB_STATE_NOTATTACHED);','if(inject_remove){inject_remove=false;start_remove();}\n if(detach_at_lookup)usb_set_device_state(u,USB_STATE_NOTATTACHED);',1) + unit=unit.replace('static void pm_runtime_put_noidle(struct device *d) {','static void pm_runtime_put_noidle(struct device *d) {\n if(remover&&d==&host_device)return;',1) + # Old source lacks platform-lock helpers. Keep fixture-only API definitions + # available without claiming they are invoked by the parent. + if 'device_lock(&pm->controller->dev)' not in (source/'drivers/phy/apple/atc.c').read_text(): + unit=unit.replace('static struct device *get_device(', 'static __attribute__((unused)) struct device *get_device(') + prefix=r''' +struct clk { int unused; }; struct reset_control { int unused; }; +struct xhci_hcd { atomic_int xhc_state; struct clk *clk,*reg_clk; struct reset_control *reset; struct usb_hcd *shared_hcd; }; +static struct xhci_hcd xhci_context; +#define XHCI_STATE_REMOVING 1 +#define hcd_to_xhci(h) (&xhci_context) +struct dwc3 { bool sys_wakeup; struct platform_device *xhci; }; +enum { DWC3_APPLE_PROBE_PENDING, DWC3_APPLE_NO_CABLE, DWC3_APPLE_HOST, DWC3_APPLE_DEVICE }; +struct dwc3_apple { struct dwc3 dwc; struct device *dev; int lock,state; void *role_sw; }; +static struct dwc3_apple apple_context; +#define to_dwc3_apple(d) container_of(d,struct dwc3_apple,dwc) +static bool inside_host_remove; +static void *platform_get_drvdata(struct platform_device *d) { return d==&controller?(void *)&apple_context.dwc:(void *)&hcd; } +static void dwc3_enable_susphy(struct dwc3 *d,bool v) { (void)d;assert(v);susphy_calls++;observed_controller_write(); } +static void device_init_wakeup(struct device *d,bool b) { (void)d;(void)b; } +static void platform_device_unregister(struct platform_device *d); +static void dwc3_gadget_exit(struct dwc3 *d) { (void)d;assert(0); } +static int dwc3_apple_core_stop(struct dwc3_apple *d) { (void)d;return 0; } +static void usb_role_switch_unregister(void *s) { (void)s; } +static void dwc3_core_remove_no_shutdown(struct dwc3 *d) { (void)d; } +static void usb_remove_hcd(struct usb_hcd *h) { host_remove_calls++;h->state=HC_STATE_QUIESCING;usb_set_device_state(h->self.root_hub,USB_STATE_NOTATTACHED); } +static void usb_phy_shutdown(void *p) { (void)p; } +static void usb_put_hcd(struct usb_hcd *h) { (void)h; } +static void clk_disable_unprepare(struct clk *c) { (void)c; } +static void reset_control_assert(struct reset_control *r) { (void)r; } +static void pm_runtime_disable(struct device *d) { (void)d; } +static void pm_runtime_set_suspended(struct device *d) { (void)d; } +static int pm_runtime_get_sync(struct device *d) { if(inside_host_remove&&d==&host_device){assert(xhci_context.xhc_state&XHCI_STATE_REMOVING);observed_controller_write();}return 0; } +static void pm_runtime_put_sync(struct device *d) { (void)d; } +static void pm_runtime_put(struct device *d) { (void)d; } +#define dev_attr_state_synced 0 +#define BUS_NOTIFY_UNBIND_DRIVER 0 +#define BUS_NOTIFY_UNBOUND_DRIVER 1 +#define KOBJ_UNBIND 0 +static bool device_links_busy(struct device *d) { (void)d;return false; } +static void device_links_unbind_consumers(struct device *d) { (void)d;assert(0); } +static void driver_sysfs_remove(struct device *d) { (void)d; } +static void bus_notify(struct device *d,int n) { (void)d;(void)n; } +static void device_remove_file(struct device *d,void *a) { (void)d;(void)a; } +static void device_remove_groups(struct device *d,void *a) { (void)d;(void)a; } +static void device_unbind_cleanup(struct device *d) { d->driver=NULL; } +static void device_links_driver_cleanup(struct device *d) { (void)d; } +static void klist_remove(void *k) { (void)k; } +static void device_pm_check_callbacks(struct device *d) { (void)d; } +static void kobject_uevent(void *k,int n) { (void)k;(void)n; } +''' + prefix=prefix.replace('#define dev_attr_state_synced 0','static int dev_attr_state_synced;') + unit=unit.replace('struct usb_hcd { int state;', 'struct usb_hcd { void *usb_phy; int state;',1) + # Extract actual callback order; clock/USB/reset leaves remain mocks. + host=(source/'drivers/usb/dwc3/host.c').read_text() + glue=(source/'drivers/usb/dwc3/dwc3-apple.c').read_text() + plat=(source/'drivers/usb/host/xhci-plat.c').read_text() + dd=(source/'drivers/base/dd.c').read_text() + functions=block(host,'void dwc3_host_exit(')+'\n'+block(glue,'static int dwc3_apple_exit(')+'\n'+block(glue,'static void dwc3_apple_remove(')+'\n'+block(plat,'void xhci_plat_remove(')+'\n' + functions+='\n'.join(block(dd,x) for x in ['static void __device_driver_lock(', 'static void __device_driver_unlock(', 'static void device_remove(', 'static void __device_release_driver(', 'void device_release_driver_internal(']) + controls=r''' +static void controller_remove(struct device *d) { assert(d==&controller.dev);dwc3_apple_remove(&controller); } +static void host_remove(struct device *d) { assert(d==&host_device);inside_host_remove=true;xhci_plat_remove(&host_platform_device);inside_host_remove=false; } +static void platform_device_unregister(struct platform_device *d) { assert(d==&host_platform_device);device_release_driver_internal(&d->dev,NULL,NULL); } +static void *run_remove(void *unused) { (void)unused;remover=true;device_release_driver_internal(remove_target?&host_device:&controller.dev,NULL,NULL);return NULL; } +static void start_remove(void) { + pthread_mutex_lock(&schedule_mutex); + assert(pthread_create(&remove_thread,NULL,run_remove,NULL)==0); + while(!blocked_attempt&&!removal_started)pthread_cond_wait(&schedule_cond,&schedule_mutex); + pthread_mutex_unlock(&schedule_mutex); +} +int main(void) { + for(remove_target=0;remove_target<2;remove_target++) { + struct apple_atcphy p;prepare(&p,true,true,false); + struct device_private private={0};controller.dev.p=&private;host_device.p=&private; + controller_driver.remove=controller_remove;host_driver.remove=host_remove; + apple_context=(struct dwc3_apple){.dwc={.xhci=&host_platform_device},.dev=&controller.dev,.state=DWC3_APPLE_HOST}; + xhci_context=(struct xhci_hcd){0}; + inject_remove=true;removal_started=blocked_attempt=release_remove=false; + atomic_store(&early_write,false);atomic_store(&handoff_active,true); + susphy_calls=host_remove_calls=0; + int ret=safe(&p); + atomic_store(&handoff_active,false); + pthread_mutex_lock(&schedule_mutex);release_remove=true;pthread_cond_broadcast(&schedule_cond);pthread_mutex_unlock(&schedule_mutex); + assert(pthread_join(remove_thread,NULL)==0); + fprintf(stderr,"connected teardown target=%d ret=%d early_write=%d susphy=%d host_remove=%d\n",remove_target,ret,atomic_load(&early_write),susphy_calls,host_remove_calls); + assert(ret==0&&!atomic_load(&early_write)&&host_remove_calls==1); + assert(susphy_calls==(remove_target?0:1));balanced(&p); + } + puts("PASS actual PM wrapper connected to driver-core/DWC3/xHCI removal; synchronous teardown waits until owned transaction ends"); + return 0; +} +''' + return unit+prefix+functions+controls From fd1751016e9877f47696f7cff8fdacd991b16ff6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 23:21:04 +0000 Subject: [PATCH 1217/2680] dt-bindings: display: apple: allow T8140 provider data on DCPEXT The shared H17P provider callback consumes the same five opaque, exact-size table inputs for T8140 DCP and DCPEXT. Permit them on both controllers; keep the existing byte lengths and restrictions on every other compatible. No requiredness or runtime consumer change is made. Independent validation reproduces five DCPEXT errors on the parent and passes the correction, 113 contract controls and dt-doc validation. Live firmware presence and loader/FDT qualification remain separate. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/display/apple,dcp.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index bba5d64f50451..0fe31267f99e1 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -320,7 +320,7 @@ allOf: properties: compatible: contains: - const: apple,t8140-dcp + enum: ['apple,t8140-dcp', 'apple,t8140-dcpext'] then: properties: apple,power-lut-data-x: false From 59f879c928639d1dadbe0fca022898206eb134a8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 14 Oct 2026 01:22:15 +0000 Subject: [PATCH 1218/2680] usb: typec: serialize display teardown and preserve recovery ownership Quiesce the display before SAFE provider writes and role-clock shutdown. Serialize connector transitions across mux handles, retain failed SAFE ownership, and check finalization before admitting a new DP route. Keep SRCU reader accounting local and preserve module/provider lifetime. Track consumed mux release separately so retries cannot release twice. Refuse reset when provider acquisition cannot establish SAFE. Retain compact actual-source regression coverage for transaction ordering, error paths, reference lifetime and release accounting. Independent review passes all five retained failure witnesses. Complete ARM64 Image/modules/ DTB build passes. Target recovery qualification is pending; this is not a hardware hotplug or suspend/wake pass. [ryan: split by topic: only the drm/apple part and its selftests are here; the PHY, Type-C, Thunderbolt and crossbar hunks are in 170-usb] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 3 + drivers/gpu/drm/apple/dcp.c | 133 +++++- .../drivers/gpu/apple-dcp/accounting.py | 318 ++++++++++++++ .../selftests/drivers/gpu/apple-dcp/pm.py | 24 +- .../drivers/gpu/apple-dcp/quiesce.py | 147 +++++++ .../selftests/drivers/gpu/apple-dcp/run.py | 11 +- .../drivers/gpu/apple-dcp/transaction.py | 406 ++++++++++++++++++ 7 files changed, 1025 insertions(+), 17 deletions(-) create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/accounting.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index d6a37a0240788..0d26b7182ac68 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -72,6 +72,9 @@ struct apple_dcp_typec_route { bool tunnel; /* tunnel: crossbar brought up (at DidChangeLinkConfiguration) */ bool xbar_up; + /* Lower mux release is consumed even when idle programming fails. */ + bool deactivate_xbar_done; + int deactivate_error; }; bool dcp_is_typec_output(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index e18b6e6274c82..20544653f6167 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -210,6 +210,9 @@ struct apple_dcp_typec_port { u32 applied_status; u32 applied_conf; bool applied_valid; + /* SAFE preparation retains this owner until the set phase completes. */ + bool safe_pending; + bool safe_finish_ready; bool hpd; /* Direct DP-alt: the port is in DP mode, routed or not, and its HPD. */ bool dp_wanted; @@ -262,6 +265,13 @@ static bool dcp_typec_route_fixed_output_busy(struct apple_dcp_typec_route *rout static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) { + struct apple_dcp_typec_port *port; + + list_for_each_entry(port, &dcp_typec_ports, link) { + if (port->safe_pending && port->owner && + port->owner->dcp == route->dcp) + return false; + } return !route->dcp->active_typec_route && !dcp_typec_route_fixed_output_busy(route); } @@ -363,9 +373,9 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, if (dcp->fixed_route_selected) { ret = mux_control_deselect(dcp->xbar); + dcp->fixed_route_selected = false; if (ret) return ret; - dcp->fixed_route_selected = false; } /* @@ -427,6 +437,8 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, } dcp->active_typec_route = route; scoped_guard(mutex, &dcp->tb_lock) { + route->deactivate_xbar_done = false; + route->deactivate_error = 0; route->active_xbar = xbar; route->tunnel = xbar != route->xbar; route->xbar_up = !route->tunnel; @@ -461,13 +473,22 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) * crossbar nor restart the tunnel pixel clock behind our back. */ scoped_guard(mutex, &dcp->tb_lock) { - if (!route->tunnel || route->xbar_up) - ret = mux_control_deselect(route->active_xbar ?: route->xbar); - else - dcp_dpxbar_preselect(route->active_xbar, MUX_IDLE_DISCONNECT); - if (ret) - return ret; - route->xbar_up = false; + if (route->deactivate_error) + return route->deactivate_error; + if (!route->deactivate_xbar_done) { + if (!route->tunnel || route->xbar_up) + ret = mux_control_deselect(route->active_xbar ?: route->xbar); + else + ret = dcp_dpxbar_preselect(route->active_xbar, MUX_IDLE_DISCONNECT); + /* Deselect always unlocks: never consume that selection twice. */ + route->deactivate_xbar_done = true; + route->xbar_up = false; + if (ret) { + /* Idle is unproven; retain the owner and the real error. */ + route->deactivate_error = ret; + return ret; + } + } if (route->tunnel && dcp->phy) { /* the tunnel pixel clock must not outlive the tunnel */ @@ -537,7 +558,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) dcp->phy = dcp->fixed_phy; dcp->dptx_phy = dcp->fixed_dptx_phy; - if (dcp->xbar) { + if (dcp->xbar && !dcp->fixed_route_selected) { ret = mux_control_select(dcp->xbar, dcp->fixed_mux_index); if (ret) @@ -856,6 +877,8 @@ dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, struct apple_dcp_typec_route *owner = port->owner; struct apple_dcp *dcp; + if (port->safe_pending) + continue; if (!owner || owner->tunnel || owner == port->target) continue; @@ -940,6 +963,86 @@ void dcp_typec_reorder(void) dcp_typec_rebalance_locked(NULL, 0); } +static int dcp_typec_route_prepare(struct typec_mux_dev *mux, + struct typec_mux_state *state) +{ + struct apple_dcp_typec_route *route = typec_mux_get_drvdata(mux); + struct apple_dcp_typec_port *port = route->port; + struct apple_dcp *dcp; + int ret; + + guard(mutex)(&dcp_typec_fabric_lock); + if (state->mode != TYPEC_STATE_SAFE) + return port->safe_pending ? -EBUSY : 0; + if (!port->owner || port->owner->tunnel) + return 0; + dcp = port->owner->dcp; + /* Stop new admission, but retain the owner until the set phase succeeds. */ + if (!port->safe_pending) + port->safe_finish_ready = false; + port->safe_pending = true; + port->applied_valid = false; + port->dp_wanted = false; + port->dp_hpd = false; + /* HPD absence alone is not proof of a completed firmware shutdown. */ + if (dcp_is_external_h17p(dcp)) { + scoped_guard(mutex, &dcp->external_lock) { + if (dcp->external_error) + return dcp->external_error; + if (!dcp->external_cable && dcp->external_stopped && + !READ_ONCE(dcp->dptxport[0].connected)) { + port->hpd = false; + return 0; + } + } + } + ret = dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + if (ret) + return ret; + port->hpd = false; + return 0; +} + +static int dcp_typec_route_finish(struct typec_mux_dev *mux, + struct typec_mux_state *state) +{ + struct apple_dcp_typec_port *port = + ((struct apple_dcp_typec_route *)typec_mux_get_drvdata(mux))->port; + int ret; + + guard(mutex)(&dcp_typec_fabric_lock); + if (state->mode != TYPEC_STATE_SAFE || !port->safe_pending || + port->safe_finish_ready) + return 0; + ret = port->owner ? dcp_typec_route_deactivate(port->owner) : 0; + if (ret) + return ret; + port->safe_finish_ready = true; + return 0; +} + +static void dcp_typec_route_commit(struct typec_mux_dev *mux, + struct typec_mux_state *state) +{ + struct apple_dcp_typec_port *port = + ((struct apple_dcp_typec_route *)typec_mux_get_drvdata(mux))->port; + + guard(mutex)(&dcp_typec_fabric_lock); + if (state->mode != TYPEC_STATE_SAFE || !port->safe_pending) + return; + if (port->owner) + port->preferred_route = port->owner; + port->owner = NULL; + port->safe_pending = false; + port->safe_finish_ready = false; + port->applied_alt = state->alt; + port->applied_mode = state->mode; + port->applied_status = 0; + port->applied_conf = 0; + port->applied_valid = true; + dcp_typec_pipeline_freed(); +} + static int dcp_typec_route_set(struct typec_mux_dev *mux, struct typec_mux_state *state) { @@ -955,6 +1058,11 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, guard(mutex)(&dcp_typec_fabric_lock); + if (state->mode == TYPEC_STATE_SAFE && port->safe_pending) + return 0; + if (port->safe_pending) + return -EBUSY; + /* * Every candidate route for this port is notified with the same state, * so only the first to arrive does the work; the others return early. @@ -1011,6 +1119,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, dcp_typec_rebalance_locked(NULL, 0); } + port->safe_pending = false; /* Data-only USB4 changes do not invalidate other display routes. */ port->applied_valid = true; return 0; @@ -1181,7 +1290,8 @@ int dcp_tunnel_crossbar_up(struct apple_dcp *dcp) guard(mutex)(&dcp->tb_lock); route = dcp->active_typec_route; - if (!route || !route->tunnel || !route->active_xbar) + if (!route || !route->tunnel || !route->active_xbar || + route->deactivate_xbar_done) return -ENODEV; if (!dcp->tb_clock_ok) { dev_warn(dcp->dev, "no DP tunnel pixel clock, crossbar left down\n"); @@ -1686,6 +1796,9 @@ static int dcp_register_typec_routes(struct apple_dcp *dcp) desc.fwnode = of_fwnode_handle(route_np); desc.set = dcp_typec_route_set; + desc.prepare = dcp_typec_route_prepare; + desc.finish = dcp_typec_route_finish; + desc.commit = dcp_typec_route_commit; desc.name = mux_name; desc.drvdata = route; route->typec_mux = typec_mux_register(dev, &desc); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/accounting.py b/tools/testing/selftests/drivers/gpu/apple-dcp/accounting.py new file mode 100644 index 0000000000000..ce2791cc3a379 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/accounting.py @@ -0,0 +1,318 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Source-connected accounting regressions retaining adverse-review assertions. + +SRCU, module and hardware leaves are models; the transaction, DCP and mux core +functions are extracted from the selected public source. No device is accessed. +""" +import quiesce +import transaction +from common import block + +def srcu_index(ROOT): + unit=quiesce.make_unit(ROOT) + main=block(unit,'int main(') + unit=unit.replace(main,main.replace('int main(','int __attribute__((unused)) prior_main(',1),1) + old_lock=block(unit,'static int srcu_read_lock(') + old_unlock=block(unit,'static void srcu_read_unlock(') + old_mutex_unlock=block(unit,'static void mutex_unlock(') + leaves=r''' + static pthread_mutex_t schedule_mutex=PTHREAD_MUTEX_INITIALIZER; + static pthread_cond_t schedule_cond=PTHREAD_COND_INITIALIZER; + static int srcu_bank,bank_readers[2],released_index=-1; + static bool pause_unlock,pause_seen,resume_unlock; + static _Thread_local int expected_index=-1; + static int srcu_read_lock(int *s) { + (void)s;pthread_mutex_lock(&schedule_mutex);int i=srcu_bank;bank_readers[i]++;expected_index=i;pthread_mutex_unlock(&schedule_mutex);return i; + } + static void srcu_read_unlock(int *s,int i) { + (void)s;pthread_mutex_lock(&schedule_mutex);released_index=i; + fprintf(stderr,"SRCU unlock: acquired=%d released=%d bank0=%d bank1=%d\n",expected_index,i,bank_readers[0],bank_readers[1]); + assert(i==expected_index&&bank_readers[i]>0);bank_readers[i]--;expected_index=-1; + pthread_cond_broadcast(&schedule_cond);pthread_mutex_unlock(&schedule_mutex); + } + static void mutex_unlock(struct mutex *m) { + assert(!pthread_mutex_unlock(&m->value)); + if(m==&shared_context.lock&&pause_unlock) { + pthread_mutex_lock(&schedule_mutex);pause_seen=true;pause_unlock=false;pthread_cond_broadcast(&schedule_cond); + while(!resume_unlock)pthread_cond_wait(&schedule_cond,&schedule_mutex); + pthread_mutex_unlock(&schedule_mutex); + } + } + ''' + unit=unit.replace(old_lock,'',1).replace(old_unlock,'',1).replace(old_mutex_unlock,leaves,1) + unit+=r''' + static void *first_actor(void *p) { + struct typec_mux *m=p;struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE};assert(typec_mux_set(m,&safe)==0);return NULL; + } + int main(void) { + struct cd321x c;initial(&c);mux.num_mux_devs=0;pause_unlock=true; + pthread_t thread;assert(!pthread_create(&thread,NULL,first_actor,&mux)); + pthread_mutex_lock(&schedule_mutex);while(!pause_seen)pthread_cond_wait(&schedule_cond,&schedule_mutex); + srcu_bank=1;pthread_mutex_unlock(&schedule_mutex); + struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE};assert(typec_mux_begin(&mux,&safe)==0); + assert(mux.srcu_index==1&&bank_readers[0]==1&&bank_readers[1]==1); + pthread_mutex_lock(&schedule_mutex);resume_unlock=true;pthread_cond_broadcast(&schedule_cond);pthread_mutex_unlock(&schedule_mutex); + assert(!pthread_join(thread,NULL));typec_mux_abort(&mux);assert(bank_readers[0]==0&&bank_readers[1]==0); + puts("PASS same-handle SRCU read-index balance across context unlock"); + } + ''' + return unit + +def resources(ROOT): + actual=next(u for n,u,c,f in transaction.controls(ROOT) if n=='actual-dcp-normal-safe') + main=block(actual,'int main(') + actual=actual.replace(main,main.replace('int main(','int __attribute__((unused)) prior_route_main(',1),1) + actual=actual.replace('struct mux_control { int unused; };',r''' + struct mux_control; + struct mux_ops { int (*set)(struct mux_control *,int); }; + struct mux_chip { const struct mux_ops *ops; }; + struct mux_control { int lock,idle_state,cached_state; unsigned last_change;struct mux_chip *chip; }; + ''',1) + muxcore=(ROOT/'drivers/mux/core.c').read_text() + resource=r''' + #define MUX_IDLE_AS_IS -1 + #define MUX_CACHE_UNKNOWN -2 + static int hw_calls,hw_failures,fixed_restore_calls,fixed_restore_failures; + static unsigned ktime_get(void) { return 0; } + static void up(int *s) { (*s)++; } + static int hw_set(struct mux_control *m,int state) { (void)m;(void)state;hw_calls++;return hw_calls<=hw_failures?-EIO:0; } + ''' + resource+=block(muxcore,'static int mux_control_set(')+'\n'+block(muxcore,'int mux_control_deselect(')+'\n' + actual=actual.replace(block(actual,'static int mux_control_deselect('),resource,1) + actual=actual.replace(block(actual,'static int mux_control_select('),r'''static int mux_control_select(struct mux_control *m,unsigned n) { (void)m;(void)n;fixed_restore_calls++;return fixed_restore_calls<=fixed_restore_failures?-EIO:0; }''',1) + actual=actual.replace('static int gpiod_get_value_cansleep(void *p) { (void)p;return 0; }', 'static int gpio_level;\nstatic int gpiod_get_value_cansleep(void *p) { (void)p;return gpio_level; }',1) + actual+=r''' + static int peer_error; + static int peer_finish(struct typec_mux_dev *p,struct typec_mux_state *s) { (void)p;(void)s;return peer_error; } + int main(int argc,char **argv) { + (void)argc;struct cd321x c;initial(&c); + struct mux_ops ops={.set=hw_set};struct mux_chip chip={.ops=&ops}; + struct mux_control xbar={.lock=0,.idle_state=0,.cached_state=1,.chip=&chip}; + struct mux_control fixed_xbar={.lock=1,.idle_state=0,.cached_state=0,.chip=&chip}; + route.xbar=route.active_xbar=&xbar; + struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE}; + if(argv[1][0]=='n') { + assert(typec_mux_set(&mux,&safe)==0&&xbar.lock==1&&!port.owner&&!port.safe_pending); + puts("PASS actual DCP finalizer + real mux deselect resource balance");return 0; + } + if(argv[1][0]=='l') { + phy_provider.finish=peer_finish;mux.mux_devs[0]=&display_provider;mux.mux_devs[1]=&phy_provider;peer_error=-EIO; + assert(typec_mux_set(&mux,&safe)==-EIO&&xbar.lock==1&&port.owner==&route&&port.safe_pending&&port.safe_finish_ready); + peer_error=0;assert(typec_mux_set(&mux,&safe)==0&&xbar.lock==1&&!port.owner&&!port.safe_pending); + puts("PASS successful DCP finalizer is not repeated after a later provider failure");return 0; + } + if(argv[1][0]=='h')hw_failures=1; + if(argv[1][0]=='f') {display.fixed_connector_type=2;display.hdmi_hpd=(void *)1;gpio_level=1;display.xbar=&fixed_xbar;fixed_restore_failures=1;} + assert(typec_mux_set(&mux,&safe)==-EIO&&xbar.lock==1&&port.owner==&route&&port.safe_pending&&!port.safe_finish_ready); + bool selected_after_error=route.selected; + int ret=typec_mux_set(&mux,&safe); + fprintf(stderr,"finalizer retry: kind=%c ret=%d permits=%d selected_after_error=%d owner=%p pending=%d hw_calls=%d fixed_calls=%d\n",argv[1][0],ret,xbar.lock,selected_after_error,(void *)port.owner,port.safe_pending,hw_calls,fixed_restore_calls); + assert(xbar.lock==1); + if(argv[1][0]=='h')assert(ret==-EIO&&hw_calls==1&&port.owner==&route&&port.safe_pending); + if(argv[1][0]=='f')assert(ret==0&&!port.owner&&!port.safe_pending&&fixed_restore_calls==2); + return 0; + } + ''' + return actual + +def getter_unit(ROOT): + getter=transaction.lifetime(ROOT) + main=block(getter,'int main(') + renamed=main.replace('int main(','int __attribute__((unused)) prior_lifetime_main(',1) + renamed=renamed[:-1]+'return 0;\n}' + getter=getter.replace(main,renamed,1) + getter=getter.replace('struct typec_mux_dev {', 'struct module { unsigned refs;bool going; };\nstruct device { unsigned refs; };\nstruct typec_mux_dev { struct module *owner;struct device dev;',1) + getter=getter.replace(block(getter,'static void typec_mux_put('),'void typec_mux_put(struct typec_mux *mux);',1) + getter=getter.replace('#include ',r''' + #include + static int allocation_count,allocation_failure; + static void *review_alloc(size_t size) { if(++allocation_count==allocation_failure)return NULL;return calloc(1,size); } + ''',1) + getter=getter.replace('#define kzalloc_obj(x) calloc(1,sizeof(x))','#define kzalloc_obj(x) review_alloc(sizeof(x))',1) + getter+=r''' + #undef IS_ERR + #define IS_ERR(p) ((uintptr_t)(p)>=(uintptr_t)-4095) + #define PTR_ERR(p) ((int)(intptr_t)(p)) + #define TYPEC_MUX_MAX_DEVS 3 + #define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0])) + static struct typec_mux_dev graph_providers[2]; + static struct module modules[2]; + static bool graph_second_error; + static void *typec_mux_match(const struct fwnode_handle *f,const char *id,void *data) { (void)f;(void)id;(void)data;return NULL; } + static int fwnode_connection_find_matches(struct fwnode_handle *f,const char *id,void *data,void *(*fn)(const struct fwnode_handle *,const char *,void *),void **matches,unsigned max) { + (void)f;(void)id;(void)data;(void)fn;assert(max>=2); + graph_providers[0].dev.refs++;matches[0]=&graph_providers[0]; + if(graph_second_error)matches[1]=ERR_PTR(-EPROBE_DEFER); + else {graph_providers[1].dev.refs++;matches[1]=&graph_providers[1];} + return 2; + } + static bool try_module_get(struct module *m) { if(!m)return true;if(m->going)return false;m->refs++;return true; } + static void module_put(struct module *m) { if(m){assert(m->refs);m->refs--;} } + static void put_device(struct device *d) { assert(d->refs);d->refs--; } + ''' + getter='#define EPROBE_DEFER 517\n'+getter + mux=(ROOT/'drivers/usb/typec/mux.c').read_text() + getter+='\n'+block(mux,'struct typec_mux *fwnode_typec_mux_get(')+'\n'+block(mux,'void typec_mux_put(')+'\n' + getter+=r''' + static void graph_setup(void) { + graph_second_error=false;allocation_count=allocation_failure=0; + for(unsigned i=0;i<2;i++){modules[i]=(struct module){0};graph_providers[i]=(struct typec_mux_dev){.owner=&modules[i],.set=phy_set};} + } + static void graph_balanced(struct fwnode_handle *f) { + assert(!f->refs&&typec_mux_contexts.next==&typec_mux_contexts); + for(unsigned i=0;i<2;i++)assert(!modules[i].refs&&!graph_providers[i].dev.refs); + } + int main(void) { + struct fwnode_handle f={0};graph_setup(); + struct typec_mux *a=fwnode_typec_mux_get(&f),*b=fwnode_typec_mux_get(&f); + assert(!IS_ERR(a)&&!IS_ERR(b)&&a->context==b->context&&a->context->users==2&&f.refs==1); + for(unsigned i=0;i<2;i++)assert(modules[i].refs==2&&graph_providers[i].dev.refs==2); + typec_mux_put(a);typec_mux_put(b);graph_balanced(&f); + graph_setup();modules[1].going=true;a=fwnode_typec_mux_get(&f);assert(PTR_ERR(a)==-ENODEV);graph_balanced(&f); + graph_setup();graph_second_error=true;a=fwnode_typec_mux_get(&f);assert(PTR_ERR(a)==-EPROBE_DEFER);graph_balanced(&f); + graph_setup();allocation_failure=2;a=fwnode_typec_mux_get(&f);assert(PTR_ERR(a)==-ENOMEM);graph_balanced(&f); + graph_setup();a=fwnode_typec_mux_get(&f);graph_providers[0].removed=true; + struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE};unsigned before=phy_calls; + assert(typec_mux_set(a,&safe)==-ENODEV&&phy_calls==before);typec_mux_put(a);graph_balanced(&f); + puts("PASS actual getter/context/module/device balance: same fwnode, failed module conversion, deferred provider, context allocation and stale-provider handle"); + return 0; + } + ''' + return getter + +def controls(root): + clock = transaction.clock_order(root).replace('if(parked)return 0;if(!backend_clock)', 'if(!backend_clock)', 1) + clock = clock.replace('assert(!port.owner&&!port.safe_pending);', 'assert(!port.owner&&!port.safe_pending&&phy_calls==1&&role_calls==2);', 1) + return [ + ('same-handle-srcu-index', srcu_index(root), ((),), ('-pthread',)), + ('mux-release-stages', resources(root), (('normal',), ('late',), ('hardware',), ('fixed',)), ('-pthread',)), + ('single-safe-before-role-exit', clock, ((),), ('-pthread',)), + ('mux-get-put-balance', getter_unit(root), ((),), ('-pthread',)), + ('lookup-unregister-module-lifetime', lookup_lifetime(root), ((),), ('-pthread',)), + ('kernel-module-conversion-balance', module_conversion(root), ((),), ('-pthread',)), + ] + +def lookup_lifetime(ROOT): + getter = getter_unit(ROOT) + mux = (ROOT / "drivers/usb/typec/mux.c").read_text() + module_unit=getter.replace(block(getter,'int main('),block(getter,'int main(').replace('int main(','int __attribute__((unused)) prior_getter_main(',1),1) + module_unit=module_unit.replace('struct module { unsigned refs;bool going; };','struct module { unsigned refs,refcnt;bool going; };',1) + module_unit=module_unit.replace('struct device { unsigned refs; };','struct device { unsigned refs;bool unregistered; };',1) + sync=r''' + static pthread_mutex_t conversion_mutex=PTHREAD_MUTEX_INITIALIZER; + static pthread_cond_t conversion_cond=PTHREAD_COND_INITIALIZER; + static bool lookup_ready,lookup_resume; + static unsigned active_readers; + static void synchronize_srcu(int *s) { (void)s;pthread_mutex_lock(&conversion_mutex);assert(active_readers==0);pthread_mutex_unlock(&conversion_mutex); } + static void device_unregister(struct device *d) { d->unregistered=true;assert(d->refs);d->refs--; } + static bool module_is_live(struct module *m) { return !m->going; } + static bool atomic_inc_not_zero(unsigned *p) { if(!*p)return false;(*p)++;return true; } + #define likely(x) (x) + #define trace_module_get(...) ((void)0) + #define _RET_IP_ 0 + ''' + pos=module_unit.index('static bool try_module_get(') + module_unit=module_unit[:pos]+sync+module_unit[pos:] + modulemain=(ROOT/'kernel/module/main.c').read_text() + module_unit=module_unit.replace(block(module_unit,'static bool try_module_get('),block(modulemain,'bool try_module_get('),1) + needle='return 2;\n }' + assert module_unit.count(needle)==1 + module_unit=module_unit.replace(needle,r''' + pthread_mutex_lock(&conversion_mutex);lookup_ready=true;pthread_cond_broadcast(&conversion_cond); + while(!lookup_resume)pthread_cond_wait(&conversion_cond,&conversion_mutex); + pthread_mutex_unlock(&conversion_mutex);return 2; + }''',1) + + declarations = r''' +static pthread_mutex_t conversion_mutex=PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t conversion_cond=PTHREAD_COND_INITIALIZER; +static bool lookup_ready,lookup_resume,draining; +static unsigned active_readers; +#define WRITE_ONCE(x,v) ((x)=(v)) +''' + start = module_unit.index('static pthread_mutex_t conversion_mutex=') + end = module_unit.index('static void synchronize_srcu(', start) + module_unit = module_unit[:start] + module_unit[end:] + pos = module_unit.index('static int srcu_read_lock(') + module_unit = module_unit[:pos] + declarations + module_unit[pos:] + module_unit = module_unit.replace(block(module_unit, 'static int srcu_read_lock('), r''' +static int srcu_read_lock(int *s) { (void)s;pthread_mutex_lock(&conversion_mutex);active_readers++;pthread_mutex_unlock(&conversion_mutex);return 0; } +''', 1) + module_unit = module_unit.replace(block(module_unit, 'static void srcu_read_unlock('), r''' +static void srcu_read_unlock(int *s,int index) { (void)s;assert(index==0);pthread_mutex_lock(&conversion_mutex);assert(active_readers);active_readers--;pthread_cond_broadcast(&conversion_cond);pthread_mutex_unlock(&conversion_mutex); } +''', 1) + module_unit = module_unit.replace(block(module_unit, 'static void synchronize_srcu('), r''' +static void synchronize_srcu(int *s) { (void)s;pthread_mutex_lock(&conversion_mutex);draining=true;pthread_cond_broadcast(&conversion_cond);while(active_readers)pthread_cond_wait(&conversion_cond,&conversion_mutex);pthread_mutex_unlock(&conversion_mutex); } +''', 1) + module_unit += '\n' + block(mux, 'void typec_mux_unregister(') + r''' +static struct typec_mux *lookup_result; +static void *getter_actor(void *node) { lookup_result=fwnode_typec_mux_get(node);return NULL; } +static void *unregister_actor(void *provider) { typec_mux_unregister(provider);return NULL; } +int main(void) { + struct fwnode_handle f={0};graph_setup(); + struct module *departing=calloc(1,sizeof(*departing));departing->going=true;graph_providers[0].owner=departing;graph_providers[0].dev.refs=1; + pthread_t getter_thread,remove_thread;assert(!pthread_create(&getter_thread,NULL,getter_actor,&f)); + pthread_mutex_lock(&conversion_mutex);while(!lookup_ready)pthread_cond_wait(&conversion_cond,&conversion_mutex); + assert(active_readers==1&&graph_providers[0].dev.refs==2);pthread_mutex_unlock(&conversion_mutex); + assert(!pthread_create(&remove_thread,NULL,unregister_actor,&graph_providers[0])); + pthread_mutex_lock(&conversion_mutex);while(!draining)pthread_cond_wait(&conversion_cond,&conversion_mutex); + assert(active_readers==1&&!graph_providers[0].dev.unregistered); + lookup_resume=true;pthread_cond_broadcast(&conversion_cond);pthread_mutex_unlock(&conversion_mutex); + assert(!pthread_join(getter_thread,NULL));assert(!pthread_join(remove_thread,NULL)); + assert(PTR_ERR(lookup_result)==-ENODEV&&graph_providers[0].dev.unregistered&&graph_providers[0].dev.refs==0&&active_readers==0); + assert(graph_providers[1].dev.refs==0&&!f.refs); + free(departing); + puts("PASS lookup-to-module conversion holds a reader until removed-provider rejection and balanced cleanup"); + return 0; +} +''' + return module_unit + + +def module_conversion(ROOT): + unit = getter_unit(ROOT) + main = block(unit, 'int main(') + unit = unit.replace(main, main.replace('int main(', 'int __attribute__((unused)) prior_getter_main(', 1), 1) + unit = unit.replace('struct module { unsigned refs;bool going; };', 'struct module { unsigned refs,refcnt;bool going; };', 1) + source = (ROOT / 'kernel/module/main.c').read_text() + pos = unit.index('static bool try_module_get(') + leaves = r""" +static bool module_is_live(struct module *m) { return !m->going; } +static bool atomic_inc_not_zero(unsigned *p) { if(!*p)return false;(*p)++;return true; } +static int atomic_dec_if_positive(unsigned *p) { if(!*p)return -1;return --(*p); } +#define likely(x) (x) +#define trace_module_get(...) ((void)0) +#define trace_module_put(...) ((void)0) +#define preempt_disable() ((void)0) +#define preempt_enable() ((void)0) +#define WARN_ON(v) assert(!(v)) +#define _RET_IP_ 0 +""" + unit = unit[:pos] + leaves + unit[pos:] + unit = unit.replace(block(unit, 'static bool try_module_get('), block(source, 'bool try_module_get('), 1) + unit = unit.replace(block(unit, 'static void module_put('), block(source, 'void module_put('), 1) + old = block(unit, 'static int srcu_read_lock(') + '\n' + block(unit, 'static void srcu_read_unlock(') + unit = unit.replace(old, r""" +static unsigned conversion_readers; +static int srcu_read_lock(int *s) { (void)s;conversion_readers++;return 0; } +static void srcu_read_unlock(int *s,int i) { (void)s;assert(i==0&&conversion_readers);conversion_readers--; } +""", 1) + unit += r""" +static void live_graph(void) { graph_setup();for(unsigned i=0;i<2;i++)modules[i].refcnt=1; } +static void module_balance(struct fwnode_handle *f) { graph_balanced(f);assert(!conversion_readers);for(unsigned i=0;i<2;i++)assert(modules[i].refcnt==1); } +int main(void) { + struct fwnode_handle f={0};live_graph(); + struct typec_mux *a=fwnode_typec_mux_get(&f),*b=fwnode_typec_mux_get(&f); + assert(!IS_ERR(a)&&!IS_ERR(b)&&a->context==b->context&&!conversion_readers); + for(unsigned i=0;i<2;i++)assert(modules[i].refcnt==3&&graph_providers[i].dev.refs==2); + typec_mux_put(a);typec_mux_put(b);module_balance(&f); + live_graph();modules[1].going=true;a=fwnode_typec_mux_get(&f);assert(PTR_ERR(a)==-ENODEV);module_balance(&f); + live_graph();allocation_failure=2;a=fwnode_typec_mux_get(&f);assert(PTR_ERR(a)==-ENOMEM);module_balance(&f); + live_graph();graph_second_error=true;a=fwnode_typec_mux_get(&f);assert(PTR_ERR(a)==-EPROBE_DEFER);module_balance(&f); + live_graph();graph_providers[0].removed=true;a=fwnode_typec_mux_get(&f);assert(PTR_ERR(a)==-ENODEV);module_balance(&f); + puts("PASS actual kernel module get/put and mux get/put: shared handles and every failed conversion unwind"); + return 0; +} +""" + return unit diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index 3c2913281ba5e..6125dbf312b8f 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -16,9 +16,22 @@ def ownership(root, usb_unreachable=False): prefix=ns['prefix']+'\nstruct device_node;\nstatic int atcphy_mux_set(struct typec_mux_dev *, struct typec_mux_state *);\n' prefix=prefix.replace('struct typec_mux_dev { struct apple_atcphy *data; };', '''struct typec_mux_dev { struct apple_atcphy *data; - int (*set)(struct typec_mux_dev *, struct typec_mux_state *); }; - struct typec_mux { unsigned int num_mux_devs; struct typec_mux_dev **mux_devs; }; + int (*set)(struct typec_mux_dev *, struct typec_mux_state *); + int (*prepare)(struct typec_mux_dev *, struct typec_mux_state *); + int (*finish)(struct typec_mux_dev *, struct typec_mux_state *); + void (*commit)(struct typec_mux_dev *, struct typec_mux_state *); + void (*abort)(struct typec_mux_dev *, struct typec_mux_state *);bool removed; }; + struct typec_mux_context { struct { bool held; } lock; }; + struct task_struct { int unused; };static struct task_struct test_task; + #define current (&test_task) + static int typec_mux_srcu; + static int srcu_read_lock(int *s) { (void)s;return 0; } + static void srcu_read_unlock(int *s,int i) { (void)s;(void)i; } + #define mutex_trylock(m) ((m)->held ? 0 : ((m)->held=true)) + #define mutex_unlock(m) ((m)->held=false) + struct typec_mux { unsigned int num_mux_devs; struct typec_mux_dev **mux_devs; struct typec_mux_context *context; struct typec_mux_state transaction; struct task_struct *owner; int srcu_index; unsigned prepared; bool applied; }; #define IS_ERR_OR_NULL(p) (!(p))''') + prefix=prefix.replace('#define WARN_ON_ONCE(v) assert(!(v))', 'static bool checked_warn(bool v) { assert(!v);return v; }\n#define WARN_ON_ONCE(v) checked_warn(v)') prefix=prefix.replace('#define guard(kind) (void)',r''' struct lock_guard { int *lock; }; static struct lock_guard lock_take(int *lock) { assert(!*lock); *lock=1; return (struct lock_guard){lock}; } @@ -177,7 +190,7 @@ def ownership(root, usb_unreachable=False): #if !USB_REACHABLE { prepare(&p,true,true,false);unsigned int before=writes; struct typec_mux_dev provider={.data=&p,.set=atcphy_mux_set}; - struct typec_mux_dev *providers[]={&provider};struct typec_mux mux={1,providers}; + struct typec_mux_dev *providers[]={&provider};struct typec_mux_context typec_test_context={0};struct typec_mux mux={.num_mux_devs=1,.mux_devs=providers,.context=&typec_test_context}; struct typec_mux_state state={.mode=TYPEC_STATE_SAFE}; assert(typec_mux_set(&mux,&state)==-ENODEV&&writes==before);balanced(&p); puts("PASS USB-unreachable fixed route blocks before MMIO");return 0; } @@ -298,7 +311,8 @@ def ownership(root, usb_unreachable=False): if usb_unreachable: pmfixture=pmfixture.replace('static ', 'static __attribute__((unused)) ') typec_source=(root/'drivers/usb/typec/mux.c').read_text() - typec_caller=ns['block'](typec_source,'int typec_mux_set(')+'\n' + typec_caller='\n'.join(ns['block'](typec_source, marker) for marker in + ('static void typec_mux_unlock(', 'void typec_mux_abort(', 'int typec_mux_begin(', 'int typec_mux_apply(', 'int typec_mux_end(', 'int typec_mux_set('))+'\n' hcd_source=(root/'drivers/usb/core/hcd.c').read_text() hcd_death=ns['block'](hcd_source,'void usb_hc_died (')+'\n' text=prefix+prod+'\n#define REQUIRES_PLATFORM_LOCKS '+str(int('device_lock(&pm->controller->dev)' in s))+'\n#define USB_REACHABLE '+str(int(not usb_unreachable))+'\n#define HCD_DEAD_ONLY '+str(int(False))+'\n'+pmfixture+helpers+typec_caller+hcd_death+suffix+controls @@ -531,7 +545,7 @@ def probe_dispatch(root): struct cd321x c={}; struct typec_mux_dev provider={.data=&p,.set=atcphy_mux_set}; struct typec_mux_dev *providers[]={&provider}; - struct typec_mux route={1,providers};probe_port.mux=&route; + struct typec_mux_context typec_test_context={0};struct typec_mux route={.num_mux_devs=1,.mux_devs=providers,.context=&typec_test_context};probe_port.mux=&route; prepare(&p,true,true,false);root_missing=true;unsigned int before=writes; assert(cd321x_register_port(&c.tps,&connector)==-EPROBE_DEFER); assert(!c.state_valid&&!resumes&&writes==before);balanced(&p);port_balance(); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py new file mode 100644 index 0000000000000..6e7b9f69b2a25 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py @@ -0,0 +1,147 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Actual mux prepare/set and TPS removal ordering; firmware/PHY leaves are mocks.""" +from common import block + +def make_unit(source, parent=False, reference=None): + reference=source if reference is None else reference + core=(source/'drivers/usb/typec/tipd/core.c').read_text();mux=(source/'drivers/usb/typec/mux.c').read_text() + dcp=(reference/'drivers/gpu/drm/apple/dcp.c').read_text() + prefix=r''' +#include +#include +#include +#include +#include +#include +typedef unsigned u32; +#define IS_ERR(p) false +#define IS_ERR_OR_NULL(p) (!(p)) +#define EXPORT_SYMBOL_GPL(x) +#define guard(k) (void) +#define scoped_guard(k,l) for(int once=((void)(l),1);once;once=0) +#define READ_ONCE(x) (x) +#define WARN_ON_ONCE(x) (x) +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +#define to_platform_device(p) (p) +#define typec_mux_get_drvdata(m) ((m)->data) +#define dev_warn(...) ((void)0) +#define dev_info(...) ((void)0) +#define dev_name(p) "device" +#define list_for_each_entry(p,h,m) for((void)(h),(p)=sole_port();(p);(p)=NULL) +#define lockdep_assert_held(p) ((void)(p)) +#define dev_err(dev,fmt,ret) do { (void)(dev);last_error=(ret); } while(0) +enum {TYPEC_STATE_SAFE,TYPEC_STATE_USB,TYPEC_ORIENTATION_NONE,TYPEC_PWR_MODE_USB,TYPEC_THUNDERBOLT_SWITCH_OFF}; +enum usb_role {USB_ROLE_NONE,USB_ROLE_HOST,USB_ROLE_DEVICE}; +enum typec_data_role {TYPEC_HOST,TYPEC_DEVICE}; +#define TPS_STATUS_TO_TYPEC_PORTROLE(s) (s) +#define TPS_STATUS_TO_TYPEC_VCONN(s) (s) +#define TPS_STATUS_TO_TYPEC_DATAROLE(s) (s) +struct typec_mux_state { void *alt; unsigned long mode; void *data; }; +struct typec_mux_dev { bool removed; void (*commit)(struct typec_mux_dev *,struct typec_mux_state *); void (*abort)(struct typec_mux_dev *,struct typec_mux_state *); int (*finish)(struct typec_mux_dev *,struct typec_mux_state *); void *data; int (*set)(struct typec_mux_dev *,struct typec_mux_state *); int (*prepare)(struct typec_mux_dev *,struct typec_mux_state *); }; +struct task_struct { int unused; }; +static _Thread_local struct task_struct this_task; +#define current (&this_task) +struct mutex { pthread_mutex_t value; }; +struct typec_mux_context { struct mutex lock; }; +static struct typec_mux_context shared_context={.lock={.value=PTHREAD_MUTEX_INITIALIZER}}; +static int typec_mux_srcu; +static int srcu_read_lock(int *s) { (void)s;return 0; } +static void srcu_read_unlock(int *s,int i) { (void)s;(void)i; } +static int mutex_trylock(struct mutex *m) { return pthread_mutex_trylock(&m->value)==0; } +static void mutex_unlock(struct mutex *m) { assert(!pthread_mutex_unlock(&m->value)); } +struct typec_mux { struct typec_mux_context *context; struct typec_mux_state transaction; struct task_struct *owner; int srcu_index; unsigned prepared; bool applied; unsigned num_mux_devs; struct typec_mux_dev *mux_devs[3]; }; +struct connector { bool connected; }; +struct apple_dcp { int external_lock,external_error; bool external_cable,external_stopped; struct { bool connected; } dptxport[1]; void *hdmi_hpd; bool active; bool external,typec_cable_connected; void *dev; struct connector *typec_connector; }; +struct apple_dcp_typec_route; +struct apple_dcp_typec_port { int link; void *connector_np; struct apple_dcp_typec_route *owner,*target,*secondary_target,*preferred_route; void *applied_alt; unsigned long applied_mode; unsigned applied_status,applied_conf; bool applied_valid,safe_pending,safe_finish_ready,dp_wanted,dp_hpd,hpd; }; +struct apple_dcp_typec_route { struct apple_dcp_typec_port *port; struct apple_dcp *dcp; void *xbar; bool deactivate_xbar_done; int deactivate_error; bool selected,tunnel; }; +static int dcp_typec_fabric_lock,dcp_typec_ports,deactivate_calls; +struct drm_device { int unused; }; +static struct apple_dcp display; +static struct connector connector; +static struct apple_dcp_typec_port port; +static struct apple_dcp_typec_route route; +static struct apple_dcp_typec_port *sole_port(void) { return &port; } +static char events[100];static unsigned count; +static int power_error,phy_error,tbt_error,role_error,last_error,phy_calls,reset_calls,gpio_calls,role_calls,stops,unregisters; +static void event(char c) { assert(count+1external; } +static int dcp_dptx_disconnect_oob(void *dev,u32 index) { (void)dev;assert(index==0);event('Q');stops++;if(power_error){display.external_error=power_error;return power_error;}display.external_cable=false;display.external_stopped=true;display.dptxport[0].connected=false;display.typec_cable_connected=false;connector.connected=false;return 0; } +static struct drm_device *dcp_typec_drm(void) { return NULL; } +static void dcp_typec_plan(struct drm_device *d,struct apple_dcp_typec_port *p,unsigned n) { (void)d;(void)p;(void)n; } +static bool dcp_typec_tunnel_held(struct apple_dcp *d) { (void)d;return false; } +static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *r) { deactivate_calls++;r->selected=false;return 0; } +static void dcp_typec_port_attach(struct apple_dcp_typec_port *p) { (void)p; } +static int gpiod_get_value_cansleep(void *p) { (void)p;return 0; } +static int dcp_dptx_connect(struct apple_dcp *d,unsigned n) { (void)d;(void)n;return 0; } +static bool dcp_typec_route_available(struct apple_dcp_typec_route *r) { return !r->selected; } +static int dcp_typec_route_activate(struct apple_dcp_typec_route *r,void *x) { (void)x;r->selected=true;return 0; } +struct tps6598x; +struct tipd_data { REMOVE_RETURN (*remove)(struct tps6598x *); void (*unregister_port)(struct tps6598x *); int (*reset)(struct tps6598x *); }; +struct typec_port { struct typec_mux *mux; }; +struct tps6598x { bool remove_safe_prepared; struct typec_mux *mux; void *partner,*dev,*psy,*role_sw,*reset; int irq,wq_poll; struct typec_port *port; struct tipd_data *data; }; +struct cd321x { struct tps6598x tps; struct typec_mux *mux; void *tbt_switch; }; +struct typec_thunderbolt_switch_data { int state; }; +static void cd321x_stop_work(struct cd321x *c) { (void)c;event('W'); } +static int typec_thunderbolt_switch_set(void *p,void *s) { (void)p;(void)s;event('B');return tbt_error; } +static int usb_role_switch_set_role(void *p,int r) { (void)p;assert(r==USB_ROLE_NONE);event('R');role_calls++;return role_error; } +static void typec_unregister_partner(void *p) { (void)p;event('P'); } +#define typec_set_pwr_opmode(...) ((void)0) +#define typec_set_pwr_role(...) ((void)0) +#define typec_set_vconn_role(...) ((void)0) +static int typec_set_orientation(void *p,int o) { (void)p;assert(o==TYPEC_ORIENTATION_NONE);return 0; } +static void typec_set_data_role(struct typec_port *p,enum typec_data_role r) { (void)p;(void)r;event('D'); } +static void dcp_typec_pipeline_freed(void) { event('F'); } +static void power_supply_changed(void *p) { (void)p; } +static void tipd_debugfs_unregister(struct tps6598x *t) { (void)t; } +static void cancel_delayed_work_sync(void *w) { (void)w; } +static void devm_free_irq(void *d,int i,void *t) { (void)d;(void)i;(void)t; } +static void typec_mux_put(struct typec_mux *m) { (void)m; } +static void usb_role_switch_put(void *p) { (void)p;event('V'); } +static void gpiod_set_value_cansleep(void *p,int v) { (void)p;assert(v==1);gpio_calls++;event('G'); } + +'''.replace('REMOVE_RETURN','void' if parent else 'int') + functions='\n'.join(block(mux, marker) for marker in ('static void typec_mux_unlock(', 'void typec_mux_abort(', 'int typec_mux_begin(', 'int typec_mux_apply(', 'int typec_mux_end(', 'int typec_mux_prepare(')) if not parent else block((reference/'drivers/usb/typec/mux.c').read_text(),'int typec_mux_prepare(')+'\n' + functions+=block(mux,'int typec_mux_set(')+'\n'+block(dcp,'static int dcp_typec_route_prepare(')+'\n'+block(dcp,'static int dcp_typec_route_finish(')+'\n'+block(dcp,'static void dcp_typec_route_commit(')+'\n' + functions+=block(dcp,'static struct apple_dcp_typec_route *\ndcp_typec_rebalance_locked(')+'\n' + if not parent:functions+=block(core,'static void tps6598x_drop_partner(')+'\n'+block(core,'static int tps6598x_set_data_role(')+'\n' + functions+=block(core,'static '+('void' if parent else 'int')+' tps6598x_disconnect(')+'\n'+block(core,'static '+('void' if parent else 'int')+' cd321x_remove(')+'\n'+block(core,'void tipd_remove(')+'\n' + controls=r''' +static struct typec_mux_dev phy_provider,display_provider; +static struct typec_mux mux; +static struct typec_port class_port; +static struct tipd_data data; + +static int phy_set(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;assert(s->mode==TYPEC_STATE_SAFE);event('L');phy_calls++;return phy_error; } +static int display_set(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;(void)s;event('S');if(port.owner&&display.external&&connector.connected) {int r=dcp_dptx_disconnect_oob(display.dev,0);if(r)return r;}if(port.safe_pending)return 0;port.owner=NULL;return 0; } +static void unregister_port(struct tps6598x *t) { (void)t;unregisters++;event('U'); } +static int reset(struct tps6598x *t) { (void)t;reset_calls++;event('X');return 0; } +static void initial(struct cd321x *c) { + count=0;events[0]=0;power_error=phy_error=tbt_error=role_error=last_error=phy_calls=reset_calls=gpio_calls=role_calls=stops=unregisters=deactivate_calls=0; + connector=(struct connector){.connected=true};display=(struct apple_dcp){.external=true,.external_cable=true,.dptxport={{.connected=true}},.typec_cable_connected=true,.typec_connector=&connector}; + port=(struct apple_dcp_typec_port){.owner=&route,.applied_valid=true,.dp_wanted=true,.dp_hpd=true,.hpd=true};route=(struct apple_dcp_typec_route){.port=&port,.dcp=&display,.selected=true}; + phy_provider=(struct typec_mux_dev){.set=phy_set};display_provider=(struct typec_mux_dev){.data=&route,.set=display_set,.prepare=dcp_typec_route_prepare,.finish=dcp_typec_route_finish,.commit=dcp_typec_route_commit}; + mux=(struct typec_mux){.context=&shared_context,.num_mux_devs=2,.mux_devs={&phy_provider,&display_provider}};class_port.mux=&mux; + data=(struct tipd_data){.remove=cd321x_remove,.unregister_port=unregister_port,.reset=reset}; + *c=(struct cd321x){.tps={.mux=&mux,.partner=(void *)1,.port=&class_port,.data=&data,.reset=(void *)1},.mux=&mux}; +} +static int position(char c) { for(unsigned i=0;i=0&&position('Q')mode==TYPEC_STATE_SAFE);event('L');", "(void)m;(void)s;event('L');",1) + actual=actual.replace('connector=(struct connector){.connected=true};','connector=(struct connector){.connected=true};',1) + actual=actual.replace('phy_provider=(struct typec_mux_dev){.set=phy_set};','display.typec_routes=&route;display.fixed_connector_type=DRM_MODE_CONNECTOR_USB;display.active_typec_route=&route;\n phy_provider=(struct typec_mux_dev){.set=phy_set};',1) + controls=r''' +static int late_finish_error(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;(void)s;return -EIO; } + int main(int argc,char **argv) { + (void)argc;struct cd321x c;initial(&c);struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE}; + if(argv[1][0]=='o') { + assert(typec_mux_set(&mux,&safe)==0); + assert(stops==1&&!port.owner&&!port.safe_pending&&!route.selected); + puts("PASS actual DCP prepare + route_set + route_deactivate + framework normal SAFE"); + } + if(argv[1][0]=='r') { + mux.mux_devs[0]=&display_provider;mux.mux_devs[1]=&phy_provider;phy_error=-ETIMEDOUT; + assert(typec_mux_set(&mux,&safe)==-ETIMEDOUT); + fprintf(stderr,"display-first failed SAFE: owner=%p lease=%d selected=%d events=%s\n",(void *)port.owner,port.safe_pending,route.selected,events); + assert(port.owner==&route&&port.safe_pending&&route.selected); + } + if(argv[1][0]=='d') { + phy_error=-ETIMEDOUT;assert(typec_mux_set(&mux,&safe)==-ETIMEDOUT&&port.safe_pending&&port.owner==&route); + phy_error=0;struct typec_altmode alt={.svid=USB_TYPEC_DP_SID}; + struct typec_displayport_data data={.status=DP_STATUS_HPD_STATE}; + struct typec_mux_state dp={.alt=&alt,.mode=TYPEC_DP_STATE_A,.data=&data}; + int ret=typec_mux_set(&mux,&dp); + fprintf(stderr,"DP after failed SAFE: ret=%d connects=%d pending=%d wanted=%d hpd=%d\n",ret,connect_calls,port.safe_pending,port.dp_wanted,port.hpd); + assert(ret<0&&connect_calls==0&&!port.dp_wanted); + } + if(argv[1][0]=='a') { + struct altmode alt={.mux=&mux,.adev={.svid=USB_TYPEC_DP_SID}};phy_error=-ETIMEDOUT; + int ret=typec_altmode_set_mux(&alt,TYPEC_STATE_SAFE,NULL); + fprintf(stderr,"altmode SAFE: ret=%d stops=%d phy_calls=%d lease=%d\n",ret,stops,phy_calls,port.safe_pending); + assert(ret==-ETIMEDOUT&&stops==1&&port.safe_pending); + } + if(argv[1][0]=='x') { + xbar_error=-EIO;assert(typec_mux_set(&mux,&safe)==-EIO); + assert(port.owner==&route&&port.safe_pending&&!port.safe_finish_ready&&route.selected&&route.deactivate_xbar_done&&route.deactivate_error==-EIO); + xbar_error=0;assert(typec_mux_set(&mux,&safe)==-EIO&&port.owner==&route&&port.safe_pending&&xbar_calls==1); + } + if(argv[1][0]=='f') { + mux.mux_devs[0]=&display_provider;mux.mux_devs[1]=&phy_provider;phy_provider.finish=late_finish_error; + assert(typec_mux_set(&mux,&safe)==-EIO); + assert(port.owner==&route&&port.safe_pending&&port.safe_finish_ready&&!route.selected&&xbar_calls==1); + assert(!dcp_typec_route_available(&route)); + struct apple_dcp peer={0};struct apple_dcp_typec_route peer_route={.dcp=&peer}; + assert(dcp_typec_route_available(&peer_route)); + phy_provider.finish=NULL; + assert(typec_mux_set(&mux,&safe)==0&&!port.owner&&!port.safe_pending&&xbar_calls==1); + } + return 0; + } + ''' + bus=(ROOT/'drivers/usb/typec/bus.c').read_text() + actual+='\nstruct altmode { struct typec_mux *mux; struct typec_altmode adev; };\n'+block(bus,'static inline int\ntypec_altmode_set_mux(')+'\n'+controls + check('actual-dcp-normal-safe',actual,(('ordered',),)) + check('actual-dcp-finalizer-recovery',actual,(('xbar-error',),('finalizer',))) + check('provider-order-releases-failed-lease',actual,(('release',),)) + check('failed-safe-allows-new-dp-admission',actual,(('dp',),)) + check('altmode-safe-skips-quiescence',actual,(('altmode',),)) + + # Real pthread fabric/consumer mutexes. Pause the SAFE actor after its actual + # DCP prepare has returned and released its locks, before any set callback. + # A second handle can reference the same providers (class/altmode/CD handles + # are separately acquired in the real framework). + concurrent=actual.replace(block(actual,'int main('),block(actual,'int main(').replace('int main(','int __attribute__((unused)) route_controls_main(',1),1) + concurrent=concurrent.replace('#define guard(k) (void)','') + concurrent=concurrent.replace('#define scoped_guard(k,l) for(int once=((void)(l),1);once;once=0)','') + locks=r''' + #include + static pthread_mutex_t fabric_mutex=PTHREAD_MUTEX_INITIALIZER,external_mutex=PTHREAD_MUTEX_INITIALIZER,tb_mutex=PTHREAD_MUTEX_INITIALIZER; + struct real_guard { pthread_mutex_t *mutex; bool held; }; + static struct real_guard take_lock(int *key) { + pthread_mutex_t *m=key==&dcp_typec_fabric_lock?&fabric_mutex:key==&display.external_lock?&external_mutex:&tb_mutex; + assert(pthread_mutex_lock(m)==0);return (struct real_guard){m,true}; + } + static void drop_lock(struct real_guard *g) { if(g->held){g->held=false;assert(pthread_mutex_unlock(g->mutex)==0);} } + #define guard(k) struct real_guard __attribute__((cleanup(drop_lock))) _guard = take_lock + #define scoped_guard(k,l) for(struct real_guard __attribute__((cleanup(drop_lock))) _scope=take_lock(l);_scope.held;drop_lock(&_scope)) + static bool observed_unquiesced_safe; + ''' + pos=concurrent.index('static int dcp_typec_route_deactivate(') + concurrent=concurrent[:pos]+locks+concurrent[pos:] + concurrent=concurrent.replace("(void)m;(void)s;event('L');phy_calls++;", "(void)m;if(s->mode==TYPEC_STATE_SAFE&&display.dptxport[0].connected)observed_unquiesced_safe=true;event('L');phy_calls++;",1) + concurrent+=r''' + static pthread_mutex_t schedule_lock=PTHREAD_MUTEX_INITIALIZER; + static pthread_cond_t schedule_cond=PTHREAD_COND_INITIALIZER; + static bool prepared,continue_safe; + static int safe_ret,dp_ret; + static int scheduled_prepare(struct typec_mux_dev *m,struct typec_mux_state *s) { + int ret=dcp_typec_route_prepare(m,s); + if(s->mode==TYPEC_STATE_SAFE&&!s->alt) { + pthread_mutex_lock(&schedule_lock);prepared=true;pthread_cond_broadcast(&schedule_cond); + while(!continue_safe)pthread_cond_wait(&schedule_cond,&schedule_lock); + pthread_mutex_unlock(&schedule_lock); + } + return ret; + } + static void *safe_actor(void *unused) { + (void)unused;struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE};safe_ret=typec_mux_set(&mux,&safe);return NULL; + } + int main(void) { + struct cd321x c;initial(&c);display_provider.prepare=scheduled_prepare; + pthread_t thread;assert(pthread_create(&thread,NULL,safe_actor,NULL)==0); + pthread_mutex_lock(&schedule_lock);while(!prepared)pthread_cond_wait(&schedule_cond,&schedule_lock);pthread_mutex_unlock(&schedule_lock); + struct typec_mux second_handle=mux; + struct typec_altmode alt={.svid=USB_TYPEC_DP_SID};struct typec_displayport_data data={.status=DP_STATUS_HPD_STATE}; + struct typec_mux_state dp={.alt=&alt,.mode=TYPEC_DP_STATE_A,.data=&data}; + dp_ret=typec_mux_set(&second_handle,&dp); + pthread_mutex_lock(&schedule_lock);continue_safe=true;pthread_cond_broadcast(&schedule_cond);pthread_mutex_unlock(&schedule_lock); + assert(pthread_join(thread,NULL)==0); + fprintf(stderr,"overlapping handles: dp_ret=%d safe_ret=%d connected_at_SAFE_PHY=%d events=%s\n",dp_ret,safe_ret,observed_unquiesced_safe,events); + assert(!observed_unquiesced_safe); + } + ''' + check('concurrent-prepare-set-stale-quiescence',concurrent,flags=('-pthread',)) + check('connector-context-lifetime', lifetime(ROOT), flags=('-pthread',)) + check('probe-transaction-rollback', probe(ROOT)) + check('provider-unregister-drain', unregister(ROOT), flags=('-pthread',)) + check('park-before-host-role-exit', clock_order(ROOT)) + return tests + + +def lifetime(root): + mux = (root / 'drivers/usb/typec/mux.c').read_text() + unit = quiesce.make_unit(root) + unit = unit.replace(block(unit, 'int main('), + block(unit, 'int main(').replace('int main(', 'int __attribute__((unused)) prior_main(', 1), 1) + unit = unit.replace('struct typec_mux_context { struct mutex lock; };', r""" +struct list_head { struct list_head *next,*prev; }; +struct fwnode_handle { unsigned refs; }; +struct typec_mux_context { struct list_head list; struct fwnode_handle *fwnode; struct mutex lock; unsigned users; }; +""") + helpers = r""" +#include +#include +#undef guard +#undef list_for_each_entry +static struct mutex typec_mux_contexts_lock={.value=PTHREAD_MUTEX_INITIALIZER}; +static struct list_head typec_mux_contexts={&typec_mux_contexts,&typec_mux_contexts}; +static struct fwnode_handle *fwnode_handle_get(struct fwnode_handle *f) { f->refs++;return f; } +static void fwnode_handle_put(struct fwnode_handle *f) { assert(f->refs);f->refs--; } +static void mutex_init(struct mutex *m) { assert(!pthread_mutex_init(&m->value,NULL)); } +static struct mutex *registry_lock(struct mutex *m) { assert(!pthread_mutex_lock(&m->value));return m; } +static void registry_unlock(struct mutex **m) { mutex_unlock(*m); } +#define guard(k) struct mutex * __attribute__((cleanup(registry_unlock))) held=registry_lock +#define kzalloc_obj(x) calloc(1,sizeof(x)) +#define kfree(x) free(x) +#define ERR_PTR(e) ((void *)(intptr_t)(e)) +#define list_for_each_entry(p,h,m) for(p=container_of((h)->next,__typeof__(*p),m); &(p)->m!=(h);p=container_of((p)->m.next,__typeof__(*p),m)) +static void list_add_tail(struct list_head *n,struct list_head *h) { n->prev=h->prev;n->next=h;h->prev->next=n;h->prev=n; } +static void list_del(struct list_head *n) { n->prev->next=n->next;n->next->prev=n->prev; } +""" + unit += helpers + block(mux, 'static struct typec_mux_context *typec_mux_context_get(') + '\n' + unit += block(mux, 'static void typec_mux_context_put(') + '\n' + unit += r""" +static struct typec_mux downstream; +static int forwarded,commits,aborts,finish_error; +static int forward_set(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;forwarded++;return typec_mux_set(&downstream,s); } +static int finish(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;(void)s;return finish_error; } +static void commit(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;(void)s;commits++; } +static void abort_callback(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;(void)s;aborts++; } +int main(void) { + struct fwnode_handle a={0},b={0}; + struct typec_mux_context *one=typec_mux_context_get(&a),*two=typec_mux_context_get(&a),*other=typec_mux_context_get(&b); + assert(one==two&&one!=other&&one->users==2&&a.refs==1&&b.refs==1); + struct cd321x c;initial(&c);struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE}; + struct typec_mux second=mux; mux.context=one;second.context=two; + assert(typec_mux_begin(&mux,&safe)==0); + unsigned before=phy_calls;assert(typec_mux_set(&second,&safe)==-EBUSY&&phy_calls==before); + typec_mux_abort(&mux);assert(port.safe_pending&&port.owner==&route); + phy_error=-ETIMEDOUT;assert(typec_mux_set(&second,&safe)==-ETIMEDOUT&&port.owner==&route); + phy_error=0;assert(typec_mux_set(&second,&safe)==0&&!port.owner&&!port.safe_pending); + struct typec_mux_dev leaf={.set=phy_set,.finish=finish,.commit=commit,.abort=abort_callback}; + downstream=(struct typec_mux){.context=other,.num_mux_devs=1,.mux_devs={&leaf}}; + struct typec_mux_dev forward={.set=forward_set}; + mux=(struct typec_mux){.context=one,.num_mux_devs=1,.mux_devs={&forward}}; + assert(typec_mux_set(&mux,&safe)==0&&forwarded==1&&commits==1); + finish_error=-EIO;assert(typec_mux_set(&downstream,&safe)==-EIO&&commits==1&&aborts==1); + finish_error=0;assert(typec_mux_set(&downstream,&safe)==0&&commits==2); + downstream.context=one;before=phy_calls; + assert(typec_mux_set(&mux,&safe)==-EBUSY&&phy_calls==before); // cyclic same-connector forwarding + leaf.removed=true;downstream.context=other;before=phy_calls; + assert(typec_mux_set(&downstream,&safe)==-ENODEV&&phy_calls==before&&aborts==1); + typec_mux_context_put(one);assert(a.refs==1&&two->users==1); + typec_mux_context_put(two);assert(a.refs==0); + typec_mux_context_put(other);assert(b.refs==0&&typec_mux_contexts.next==&typec_mux_contexts); + puts("PASS actual connector registry, shared handles, forwarding, finalize/abort and removed-provider guards"); +} +""" + return unit + + +def probe(root): + core = (root / 'drivers/usb/typec/tipd/core.c').read_text() + unit = quiesce.make_unit(root) + unit = unit.replace(block(unit, 'int main('), block(unit, 'int main(').replace('int main(', 'int __attribute__((unused)) prior_main(', 1), 1) + unit = unit.replace('struct tipd_data {', 'struct fwnode_handle;\nstruct tipd_data { unsigned irq_mask1; int (*init)(struct tps6598x *); int (*switch_power_state)(struct tps6598x *, int); int (*register_port)(struct tps6598x *, struct fwnode_handle *); bool (*read_data_status)(struct tps6598x *); int (*connect)(struct tps6598x *,u32); void (*ready)(struct tps6598x *); void *irq_handler;') + unit = unit.replace('struct tps6598x {', 'struct tps6598x { bool dbma_at_probe,wakeup;') + helpers = r""" +#undef dev_err +#define dev_err(...) ((void)0) +#define TPS_SETUP_MS 1000 +#define TPS_SYSTEM_POWER_STATE_S0 0 +#define TPS_MODE_PTCH 2 +#define TPS_MODE_DBMA 3 +#define TPS_REG_VID 0 +#define TPS_REG_INT_MASK1 1 +#define TPS_STATUS_PLUG_PRESENT 1 +#define GPIOD_OUT_LOW 0 +#include +#undef IS_ERR +#define IS_ERR(p) ((uintptr_t)(p)>=(uintptr_t)-4095) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +#define IRQF_SHARED 0 +#define IRQF_ONESHOT 0 +#define POLL_INTERVAL 1000 +#define INIT_DELAYED_WORK(...) ((void)0) +#define tps6598x_poll_work 0 +#define devm_device_init_wakeup(...) ((void)0) +#define enable_irq_wake(...) ((void)0) +#define dev_err_probe(...) (-EINVAL) +struct fwnode_handle { int unused; }; +static int connect_error,irq_error,register_error,node_puts,mux_get_error; +static void *devm_gpiod_get_optional(void *d,const char *s,int v) { (void)d;(void)s;(void)v;return NULL; } +static void msleep(unsigned ms) { (void)ms; } +static bool device_is_compatible(void *d,const char *s) { (void)d;(void)s;return false; } +static int tps6598x_read32(struct tps6598x *t,int reg,u32 *v) { (void)t;(void)reg;*v=1;return 0; } +static int tps6598x_check_mode(struct tps6598x *t) { (void)t;return 0; } +static int tps6598x_write64(struct tps6598x *t,int reg,unsigned v) { (void)t;(void)reg;(void)v;return 0; } +static bool tps6598x_read_status(struct tps6598x *t,u32 *s) { (void)t;*s=TPS_STATUS_PLUG_PRESENT;return true; } +static bool tps6598x_read_power_status(struct tps6598x *t) { (void)t;return true; } +static struct fwnode_handle node; +static struct fwnode_handle *device_get_named_child_node(void *d,const char *s) { (void)d;(void)s;return &node; } +static void fw_devlink_purge_absent_suppliers(struct fwnode_handle *f) { (void)f; } +static void *fwnode_usb_role_switch_get(struct fwnode_handle *f) { (void)f;return (void *)1; } +static struct typec_mux *fwnode_typec_mux_get(struct fwnode_handle *f) { (void)f;return mux_get_error?(struct typec_mux *)(intptr_t)mux_get_error:&mux; } +static void fwnode_handle_put(struct fwnode_handle *f) { (void)f;node_puts++; } +static int devm_tps6598_psy_register(struct tps6598x *t) { (void)t;return 0; } +static int devm_request_threaded_irq(void *d,int irq,void *h,void *th,int flags,const char *name,void *arg) { (void)d;(void)irq;(void)h;(void)th;(void)flags;(void)name;(void)arg;return irq_error; } +static int system_power_efficient_wq; +static unsigned msecs_to_jiffies(unsigned ms) { return ms; } +static void queue_delayed_work(int wq,int *work,unsigned delay) { (void)wq;(void)work;(void)delay; } +static bool device_property_read_bool(void *d,const char *p) { (void)d;(void)p;return false; } +static void tipd_debugfs_register(struct tps6598x *t) { (void)t; } +static int register_port(struct tps6598x *t,struct fwnode_handle *f) { (void)t;(void)f;return register_error; } +static bool read_data_status(struct tps6598x *t) { (void)t;return true; } +static int connect(struct tps6598x *t,u32 status) { (void)t;(void)status;return connect_error; } +""" + unit += helpers + block(core, 'int tipd_init(') + r""" +static void probe_initial(struct cd321x *c) { + initial(c);c->tps.irq=1;data.register_port=register_port;data.read_data_status=read_data_status;data.connect=connect; + connect_error=irq_error=register_error=node_puts=mux_get_error=0; +} +int main(void) { + struct cd321x c; + probe_initial(&c);irq_error=-ENOSPC;phy_error=-ETIMEDOUT; + assert(tipd_init(&c.tps)==-ENOSPC&&reset_calls==0&&gpio_calls==0&&port.owner==&route&&port.safe_pending&&unregisters==1&&node_puts==1); + probe_initial(&c);connect_error=-EAGAIN;role_error=-EBUSY; + assert(tipd_init(&c.tps)==-EAGAIN&&reset_calls==0&&gpio_calls==0&&phy_calls==1&&port.owner==&route&&port.safe_pending); + probe_initial(&c);irq_error=-ENOSPC; + assert(tipd_init(&c.tps)==-ENOSPC&&reset_calls==1&&!port.owner&&!port.safe_pending&&unregisters==1); + probe_initial(&c);mux_get_error=-ENODEV; + assert(tipd_init(&c.tps)==-ENODEV&&reset_calls==0&&stops==0&&phy_calls==0&&unregisters==0&&node_puts==1); + probe_initial(&c);register_error=-EIO; + assert(tipd_init(&c.tps)==-EIO&&reset_calls==0&&stops==0&&phy_calls==0&&unregisters==0); + puts("PASS actual probe rollback preserves primary errors and blocks reset after SAFE/role/registration errors"); +} +""" + return unit + + +def unregister(root): + """Actual unregister/begin/end with sleeping SRCU leaves and two threads.""" + source = (root / 'drivers/usb/typec/mux.c').read_text() + unit = quiesce.make_unit(root) + unit = unit.replace(block(unit, 'int main('), block(unit, 'int main(').replace('int main(', 'int __attribute__((unused)) prior_main(', 1), 1) + unit = unit.replace('struct typec_mux_dev {', 'struct device { bool unregistered; };\nstruct typec_mux_dev { struct device dev;') + old = block(unit, 'static int srcu_read_lock(') + '\n' + block(unit, 'static void srcu_read_unlock(') + leaves = r""" +static pthread_mutex_t lifetime_lock=PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t lifetime_cond=PTHREAD_COND_INITIALIZER; +static unsigned readers; +static bool draining; +static int srcu_read_lock(int *s) { (void)s;pthread_mutex_lock(&lifetime_lock);readers++;pthread_mutex_unlock(&lifetime_lock);return 0; } +static void srcu_read_unlock(int *s,int i) { (void)s;(void)i;pthread_mutex_lock(&lifetime_lock);assert(readers);readers--;pthread_cond_broadcast(&lifetime_cond);pthread_mutex_unlock(&lifetime_lock); } +static void synchronize_srcu(int *s) { (void)s;pthread_mutex_lock(&lifetime_lock);draining=true;pthread_cond_broadcast(&lifetime_cond);while(readers)pthread_cond_wait(&lifetime_cond,&lifetime_lock);pthread_mutex_unlock(&lifetime_lock); } +static void device_unregister(struct device *d) { assert(!readers);d->unregistered=true; } +#define WRITE_ONCE(x,v) ((x)=(v)) +""" + unit = unit.replace(old, leaves, 1) + unit += block(source, 'void typec_mux_unregister(') + r""" +static void *remove_provider(void *arg) { typec_mux_unregister(arg);return NULL; } +int main(void) { + struct cd321x c;initial(&c);struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE}; + assert(typec_mux_begin(&mux,&safe)==0&&readers==1); + pthread_t thread;assert(!pthread_create(&thread,NULL,remove_provider,&phy_provider)); + pthread_mutex_lock(&lifetime_lock);while(!draining)pthread_cond_wait(&lifetime_cond,&lifetime_lock);pthread_mutex_unlock(&lifetime_lock); + assert(phy_provider.removed&&!phy_provider.dev.unregistered&&readers==1); + assert(typec_mux_end(&mux)==0); // pre-existing transaction finishes before removal returns + assert(!pthread_join(thread,NULL)&&phy_provider.dev.unregistered&&readers==0); + unsigned before=phy_calls; + assert(typec_mux_set(&mux,&safe)==-ENODEV&&phy_calls==before&&readers==0); + puts("PASS actual unregister drains the open transaction and tombstone blocks stale handles"); +} +""" + return unit + + +def clock_order(root): + unit = quiesce.make_unit(root) + unit = unit.replace(block(unit, 'int main('), block(unit, 'int main(').replace('int main(', 'int __attribute__((unused)) prior_main(', 1), 1) + unit = unit.replace('static int power_error,', 'static bool backend_clock=true,parked;\nstatic int power_error,') + unit = unit.replace("event('R');role_calls++;return role_error;", "event('R');role_calls++;if(!role_error)backend_clock=false;return role_error;", 1) + unit = unit.replace("event('L');phy_calls++;return phy_error;", "event('L');phy_calls++;if(parked)return 0;if(!backend_clock)return -ENODEV;if(!phy_error)parked=true;return phy_error;", 1) + unit += r""" +int main(void) { + (void)current;struct cd321x c;initial(&c); + tipd_remove(&c.tps); + assert(!last_error&&reset_calls==1&&gpio_calls==1&&parked&&!backend_clock); + assert(position('Q') Date: Thu, 15 Oct 2026 11:55:26 +1100 Subject: [PATCH 1219/2680] drm/apple: Resume a CRTC whose mode needs a new link disabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every resume with the external display at 1920x1080@180 logged [drm:drm_mode_config_helper_resume] *ERROR* Failed to resume (-22) The 180 Hz mode is admitted only while the DPTX link training that proves its bandwidth is current. System sleep releases the link, so when drm_atomic_helper_resume() commits the saved state, dcp_crtc_mode_fixup() rejects the mode and the whole commit fails with -EINVAL. Neither CRTC is restored; both outputs come back only when the compositor reacts to the hotplug that follows the relink. Before resuming, disable each CRTC of the saved state whose mode no longer passes the mode fixup, detaching its planes and connectors. The rest of the state is restored as before, and the DCP reports the external display again once its link is back, which is when userspace re-enables it, as after a replug. [ryan: adapted to 7.2.10 — struct drm_atomic_state is struct drm_atomic_commit] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 57 ++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index d6ea627cd3ca4..678b33ef56965 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -941,12 +942,66 @@ static int apple_platform_suspend(struct device *dev) return 0; } +/* + * A mode can be admissible only while the link it was proven on is up: the + * external H17P 180 Hz mode needs the current DPTX link training, and system + * sleep releases that link. Restoring such a CRTC as it was fails the whole + * resume commit in its mode fixup, which leaves every output dark until + * userspace reacts to the next hotplug. Restore that CRTC disabled instead. + * The DCP reports the display again once its link is back, and userspace + * re-enables it on that hotplug, as it does after a cable is replugged. + * + * The suspend state holds every CRTC, plane and connector already, so it is + * edited directly rather than through helpers that need the modeset locks. + */ +static void apple_drm_drop_stale_crtcs(struct drm_atomic_commit *state) +{ + struct drm_connector_state *conn_state; + struct drm_plane_state *plane_state; + struct drm_crtc_state *crtc_state; + struct drm_connector *connector; + struct drm_plane *plane; + struct drm_crtc *crtc; + int i, j; + + for_each_new_crtc_in_state(state, crtc, crtc_state, i) { + if (!crtc_state->enable || + dcp_crtc_mode_fixup(crtc, &crtc_state->mode, + &crtc_state->adjusted_mode)) + continue; + + drm_dbg_kms(crtc->dev, + "[CRTC:%d:%s] " DRM_MODE_FMT " needs a new link, resuming disabled\n", + crtc->base.id, crtc->name, DRM_MODE_ARG(&crtc_state->mode)); + + for_each_new_plane_in_state(state, plane, plane_state, j) { + if (plane_state->crtc != crtc) + continue; + crtc_state->plane_mask &= ~drm_plane_mask(plane); + plane_state->crtc = NULL; + drm_atomic_set_fb_for_plane(plane_state, NULL); + } + for_each_new_connector_in_state(state, connector, conn_state, j) { + /* Detaching from a CRTC already in the state cannot fail. */ + if (conn_state->crtc == crtc && + WARN_ON(drm_atomic_set_crtc_for_connector(conn_state, NULL))) + return; + } + crtc_state->active = false; + if (WARN_ON(drm_atomic_set_mode_for_crtc(crtc_state, NULL))) + return; + } +} + static int apple_platform_resume(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); - if (apple) + if (apple) { + if (apple->drm.mode_config.suspend_state) + apple_drm_drop_stale_crtcs(apple->drm.mode_config.suspend_state); drm_mode_config_helper_resume(&apple->drm); + } return 0; } From 899af79bd7dcd7e9bca474f196e94722c3d23e0d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:13:05 +1100 Subject: [PATCH 1220/2680] drm/apple: Test the resume of a CRTC whose mode no longer fixes up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resume restores a CRTC whose mode needs a link that system sleep released by bringing it back disabled. That edit of the suspend state detaches planes, connectors and the framebuffer by hand, outside the helpers, and nothing exercises it. Move it into its own file and have it ask the CRTC's own mode fixup, which for every apple-drm CRTC is dcp_crtc_mode_fixup(), so that KUnit can run it against a mock device. Add tests that take that device through drm_mode_config_helper_suspend() and _resume(): an inadmissible mode fails the resume when it is restored as is, and resumes disabled, with its planes, connector and framebuffer reference released, when it goes through the filter; an admissible or DPMS-off CRTC is handled the same way as before. No functional change. [ryan: adapted to 7.2.10 — struct drm_atomic_state is struct drm_atomic_commit] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Kconfig | 4 +- drivers/gpu/drm/apple/Makefile | 3 +- drivers/gpu/drm/apple/apple_drv.c | 53 +---- drivers/gpu/drm/apple/apple_resume.c | 66 ++++++ drivers/gpu/drm/apple/apple_resume.h | 11 + drivers/gpu/drm/apple/apple_resume_test.c | 261 ++++++++++++++++++++++ 6 files changed, 344 insertions(+), 54 deletions(-) create mode 100644 drivers/gpu/drm/apple/apple_resume.c create mode 100644 drivers/gpu/drm/apple/apple_resume.h create mode 100644 drivers/gpu/drm/apple/apple_resume_test.c diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index d74b2dfdb0f88..9c1ce96bc0dbb 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -32,6 +32,7 @@ config DRM_APPLE_DEBUG config DRM_APPLE_KUNIT_TEST tristate "KUnit tests for Apple display controller" if !KUNIT_ALL_TESTS depends on KUNIT + select DRM_KUNIT_TEST_HELPERS if DRM_APPLE default KUNIT_ALL_TESTS help Build tests for the Apple display driver's AFK ring parser and @@ -40,4 +41,5 @@ config DRM_APPLE_KUNIT_TEST The tests cover the header strides used by older firmware and larger rings, as well as malformed ring size combinations. This option is intended for developer test kernels. - Also test the Type-C modeset, hotplug and service ownership state. + Also test the Type-C modeset, hotplug and service ownership state, + and the system resume of a CRTC whose mode is no longer admissible. diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 849d187c39385..b282f798dfcdb 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -2,7 +2,7 @@ CFLAGS_trace.o = -I$(src) -appledrm-y := apple_drv.o +appledrm-y := apple_drv.o apple_resume.o appledrm-y += afk.o dcp.o dcp_backlight.o dptxep.o iomfb.o parser.o systemep.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += audio.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += av.o @@ -13,6 +13,7 @@ appledrm-y += iomfb_v13_3.o appledrm-y += iomfb_v14_7.o appledrm-y += iomfb_h17p.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += iomfb_h17p_test.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple_resume_test.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 678b33ef56965..5474d6dcf2114 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -21,7 +21,6 @@ #include #include -#include #include #include #include @@ -41,6 +40,7 @@ #include #include +#include "apple_resume.h" #include "dcp.h" #include "plane.h" @@ -942,57 +942,6 @@ static int apple_platform_suspend(struct device *dev) return 0; } -/* - * A mode can be admissible only while the link it was proven on is up: the - * external H17P 180 Hz mode needs the current DPTX link training, and system - * sleep releases that link. Restoring such a CRTC as it was fails the whole - * resume commit in its mode fixup, which leaves every output dark until - * userspace reacts to the next hotplug. Restore that CRTC disabled instead. - * The DCP reports the display again once its link is back, and userspace - * re-enables it on that hotplug, as it does after a cable is replugged. - * - * The suspend state holds every CRTC, plane and connector already, so it is - * edited directly rather than through helpers that need the modeset locks. - */ -static void apple_drm_drop_stale_crtcs(struct drm_atomic_commit *state) -{ - struct drm_connector_state *conn_state; - struct drm_plane_state *plane_state; - struct drm_crtc_state *crtc_state; - struct drm_connector *connector; - struct drm_plane *plane; - struct drm_crtc *crtc; - int i, j; - - for_each_new_crtc_in_state(state, crtc, crtc_state, i) { - if (!crtc_state->enable || - dcp_crtc_mode_fixup(crtc, &crtc_state->mode, - &crtc_state->adjusted_mode)) - continue; - - drm_dbg_kms(crtc->dev, - "[CRTC:%d:%s] " DRM_MODE_FMT " needs a new link, resuming disabled\n", - crtc->base.id, crtc->name, DRM_MODE_ARG(&crtc_state->mode)); - - for_each_new_plane_in_state(state, plane, plane_state, j) { - if (plane_state->crtc != crtc) - continue; - crtc_state->plane_mask &= ~drm_plane_mask(plane); - plane_state->crtc = NULL; - drm_atomic_set_fb_for_plane(plane_state, NULL); - } - for_each_new_connector_in_state(state, connector, conn_state, j) { - /* Detaching from a CRTC already in the state cannot fail. */ - if (conn_state->crtc == crtc && - WARN_ON(drm_atomic_set_crtc_for_connector(conn_state, NULL))) - return; - } - crtc_state->active = false; - if (WARN_ON(drm_atomic_set_mode_for_crtc(crtc_state, NULL))) - return; - } -} - static int apple_platform_resume(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); diff --git a/drivers/gpu/drm/apple/apple_resume.c b/drivers/gpu/drm/apple/apple_resume.c new file mode 100644 index 0000000000000..f03c1a55e4527 --- /dev/null +++ b/drivers/gpu/drm/apple/apple_resume.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include + +#include +#include +#include +#include +#include +#include + +#include "apple_resume.h" + +/* + * A mode can be admissible only while the link it was proven on is up: the + * external H17P 180 Hz mode needs the current DPTX link training, and system + * sleep releases that link. Restoring such a CRTC as it was fails the whole + * resume commit in its mode fixup, which leaves every output dark until + * userspace reacts to the next hotplug. Restore that CRTC disabled instead. + * The DCP reports the display again once its link is back, and userspace + * re-enables it on that hotplug, as it does after a cable is replugged. + * + * The suspend state holds every CRTC, plane and connector already, so it is + * edited directly rather than through helpers that need the modeset locks. + */ +void apple_drm_drop_stale_crtcs(struct drm_atomic_commit *state) +{ + const struct drm_crtc_helper_funcs *funcs; + struct drm_connector_state *conn_state; + struct drm_plane_state *plane_state; + struct drm_crtc_state *crtc_state; + struct drm_connector *connector; + struct drm_plane *plane; + struct drm_crtc *crtc; + int i, j; + + for_each_new_crtc_in_state(state, crtc, crtc_state, i) { + funcs = crtc->helper_private; + if (!crtc_state->enable || !funcs || !funcs->mode_fixup || + funcs->mode_fixup(crtc, &crtc_state->mode, + &crtc_state->adjusted_mode)) + continue; + + drm_dbg_kms(crtc->dev, + "[CRTC:%d:%s] " DRM_MODE_FMT " needs a new link, resuming disabled\n", + crtc->base.id, crtc->name, DRM_MODE_ARG(&crtc_state->mode)); + + for_each_new_plane_in_state(state, plane, plane_state, j) { + if (plane_state->crtc != crtc) + continue; + crtc_state->plane_mask &= ~drm_plane_mask(plane); + plane_state->crtc = NULL; + drm_atomic_set_fb_for_plane(plane_state, NULL); + } + for_each_new_connector_in_state(state, connector, conn_state, j) { + /* Detaching from a CRTC already in the state cannot fail. */ + if (conn_state->crtc == crtc && + WARN_ON(drm_atomic_set_crtc_for_connector(conn_state, NULL))) + return; + } + crtc_state->active = false; + if (WARN_ON(drm_atomic_set_mode_for_crtc(crtc_state, NULL))) + return; + } +} diff --git a/drivers/gpu/drm/apple/apple_resume.h b/drivers/gpu/drm/apple/apple_resume.h new file mode 100644 index 0000000000000..48d443afc029e --- /dev/null +++ b/drivers/gpu/drm/apple/apple_resume.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Ryan Murray */ + +#ifndef __APPLE_RESUME_H__ +#define __APPLE_RESUME_H__ + +struct drm_atomic_commit; + +void apple_drm_drop_stale_crtcs(struct drm_atomic_commit *state); + +#endif /* __APPLE_RESUME_H__ */ diff --git a/drivers/gpu/drm/apple/apple_resume_test.c b/drivers/gpu/drm/apple/apple_resume_test.c new file mode 100644 index 0000000000000..93c59f4ec1762 --- /dev/null +++ b/drivers/gpu/drm/apple/apple_resume_test.c @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "apple_resume.h" + +/* One pipe: a primary plane scanning a framebuffer out to one connector. */ +struct resume_test { + struct drm_device drm; + struct drm_plane *plane; + struct drm_crtc *crtc; + struct drm_encoder encoder; + struct drm_connector connector; + struct drm_framebuffer fb; + /* Whether the mode fixup admits the mode, as a current link would. */ + bool admissible; +}; + +static struct resume_test *to_resume_test(struct drm_device *drm) +{ + return container_of(drm, struct resume_test, drm); +} + +static bool resume_test_mode_fixup(struct drm_crtc *crtc, + const struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode) +{ + return to_resume_test(crtc->dev)->admissible; +} + +static const struct drm_crtc_helper_funcs resume_test_crtc_helpers = { + .mode_fixup = resume_test_mode_fixup, +}; + +/* Scanout itself is not under test; the plane helpers require an update. */ +static void resume_test_plane_update(struct drm_plane *plane, + struct drm_atomic_commit *state) +{ +} + +static const struct drm_plane_helper_funcs resume_test_plane_helpers = { + .atomic_update = resume_test_plane_update, +}; + +static const struct drm_connector_helper_funcs resume_test_connector_helpers = { +}; + +static const struct drm_connector_funcs resume_test_connector_funcs = { + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, + .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .reset = drm_atomic_helper_connector_reset, +}; + +static void resume_test_fb_destroy(struct drm_framebuffer *fb) +{ + drm_framebuffer_cleanup(fb); +} + +static const struct drm_framebuffer_funcs resume_test_fb_funcs = { + .destroy = resume_test_fb_destroy, +}; + +static void resume_test_teardown(void *data) +{ + struct resume_test *priv = data; + + /* Release the scanout reference before the device goes away. */ + drm_atomic_helper_shutdown(&priv->drm); + drm_framebuffer_put(&priv->fb); +} + +static struct resume_test *resume_test_init(struct kunit *test) +{ + struct drm_modeset_acquire_ctx ctx; + struct drm_connector_state *conn_state; + struct drm_plane_state *plane_state; + struct drm_crtc_state *crtc_state; + struct drm_atomic_commit *state; + struct drm_display_mode *mode; + struct resume_test *priv; + struct drm_device *drm; + struct device *dev; + int ret; + + dev = drm_kunit_helper_alloc_device(test); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + priv = drm_kunit_helper_alloc_drm_device(test, dev, struct resume_test, drm, + DRIVER_MODESET | DRIVER_ATOMIC); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv); + drm = &priv->drm; + priv->admissible = true; + + priv->plane = drm_kunit_helper_create_primary_plane(test, drm, NULL, + &resume_test_plane_helpers, + NULL, 0, NULL); + priv->crtc = drm_kunit_helper_create_crtc(test, drm, priv->plane, NULL, NULL, + &resume_test_crtc_helpers); + ret = drmm_encoder_init(drm, &priv->encoder, NULL, DRM_MODE_ENCODER_TMDS, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + priv->encoder.possible_crtcs = drm_crtc_mask(priv->crtc); + ret = drmm_connector_init(drm, &priv->connector, &resume_test_connector_funcs, + DRM_MODE_CONNECTOR_USB, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + drm_connector_helper_add(&priv->connector, &resume_test_connector_helpers); + ret = drm_connector_attach_encoder(&priv->connector, &priv->encoder); + KUNIT_ASSERT_EQ(test, ret, 0); + drm_mode_config_reset(drm); + + priv->fb.dev = drm; + priv->fb.format = drm_format_info(DRM_FORMAT_XRGB8888); + priv->fb.width = 1920; + priv->fb.height = 1080; + priv->fb.pitches[0] = 1920 * 4; + priv->fb.modifier = DRM_FORMAT_MOD_LINEAR; + ret = drm_framebuffer_init(drm, &priv->fb, &resume_test_fb_funcs); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = kunit_add_action_or_reset(test, resume_test_teardown, priv); + KUNIT_ASSERT_EQ(test, ret, 0); + + mode = drm_kunit_display_mode_from_cea_vic(test, drm, 16); + KUNIT_ASSERT_NOT_NULL(test, mode); + + /* Put by hand: a state kept to the end of the test pins old scanout. */ + drm_modeset_acquire_init(&ctx, 0); + state = drm_atomic_commit_alloc(drm); + KUNIT_ASSERT_NOT_NULL(test, state); + state->acquire_ctx = &ctx; + conn_state = drm_atomic_get_connector_state(state, &priv->connector); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state); + KUNIT_ASSERT_EQ(test, drm_atomic_set_crtc_for_connector(conn_state, priv->crtc), 0); + crtc_state = drm_atomic_get_crtc_state(state, priv->crtc); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state); + KUNIT_ASSERT_EQ(test, drm_atomic_set_mode_for_crtc(crtc_state, mode), 0); + crtc_state->active = true; + plane_state = drm_atomic_get_plane_state(state, priv->plane); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, plane_state); + KUNIT_ASSERT_EQ(test, drm_atomic_set_crtc_for_plane(plane_state, priv->crtc), 0); + drm_atomic_set_fb_for_plane(plane_state, &priv->fb); + plane_state->crtc_w = 1920; + plane_state->crtc_h = 1080; + plane_state->src_w = 1920 << 16; + plane_state->src_h = 1080 << 16; + ret = drm_atomic_commit(state); + drm_atomic_commit_put(state); + drm_modeset_drop_locks(&ctx); + drm_modeset_acquire_fini(&ctx); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_TRUE(test, priv->crtc->state->active); + + return priv; +} + +/* The failure the stale CRTC filter exists for: one bad mode fails resume. */ +static void resume_stale_mode_fails_unfiltered_test(struct kunit *test) +{ + struct resume_test *priv = resume_test_init(test); + + KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); + priv->admissible = false; + KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), -EINVAL); + KUNIT_EXPECT_FALSE(test, priv->crtc->state->enable); +} + +static void resume_stale_crtc_disabled_test(struct kunit *test) +{ + struct resume_test *priv = resume_test_init(test); + + KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); + priv->admissible = false; + apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state); + KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); + + KUNIT_EXPECT_FALSE(test, priv->crtc->state->enable); + KUNIT_EXPECT_FALSE(test, priv->crtc->state->active); + KUNIT_EXPECT_EQ(test, priv->crtc->state->plane_mask, 0U); + KUNIT_EXPECT_EQ(test, priv->crtc->state->connector_mask, 0U); + KUNIT_EXPECT_NULL(test, priv->plane->state->crtc); + KUNIT_EXPECT_NULL(test, priv->plane->state->fb); + KUNIT_EXPECT_NULL(test, priv->connector.state->crtc); + /* Only the creator's reference is left: the saved scanout was dropped. */ + KUNIT_EXPECT_EQ(test, drm_framebuffer_read_refcount(&priv->fb), 1U); +} + +static void resume_admissible_crtc_restored_test(struct kunit *test) +{ + struct resume_test *priv = resume_test_init(test); + + KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); + apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state); + KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); + + KUNIT_EXPECT_TRUE(test, priv->crtc->state->enable); + KUNIT_EXPECT_TRUE(test, priv->crtc->state->active); + KUNIT_EXPECT_PTR_EQ(test, priv->plane->state->crtc, priv->crtc); + KUNIT_EXPECT_PTR_EQ(test, priv->plane->state->fb, &priv->fb); + KUNIT_EXPECT_PTR_EQ(test, priv->connector.state->crtc, priv->crtc); + KUNIT_EXPECT_EQ(test, priv->connector.state->link_status, + DRM_LINK_STATUS_GOOD); + KUNIT_EXPECT_EQ(test, drm_framebuffer_read_refcount(&priv->fb), 2U); +} + +/* A DPMS-off CRTC keeps its mode, so it is checked like an active one. */ +static void resume_stale_dpms_off_crtc_disabled_test(struct kunit *test) +{ + struct resume_test *priv = resume_test_init(test); + struct drm_modeset_acquire_ctx ctx; + struct drm_crtc_state *crtc_state; + struct drm_atomic_commit *state; + int ret; + + drm_modeset_acquire_init(&ctx, 0); + state = drm_atomic_commit_alloc(&priv->drm); + KUNIT_ASSERT_NOT_NULL(test, state); + state->acquire_ctx = &ctx; + crtc_state = drm_atomic_get_crtc_state(state, priv->crtc); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state); + crtc_state->active = false; + ret = drm_atomic_commit(state); + drm_atomic_commit_put(state); + drm_modeset_drop_locks(&ctx); + drm_modeset_acquire_fini(&ctx); + KUNIT_ASSERT_EQ(test, ret, 0); + + KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); + priv->admissible = false; + apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state); + KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); + KUNIT_EXPECT_FALSE(test, priv->crtc->state->enable); + KUNIT_EXPECT_NULL(test, priv->connector.state->crtc); + KUNIT_EXPECT_EQ(test, drm_framebuffer_read_refcount(&priv->fb), 1U); +} + +static struct kunit_case resume_cases[] = { + KUNIT_CASE(resume_stale_mode_fails_unfiltered_test), + KUNIT_CASE(resume_stale_crtc_disabled_test), + KUNIT_CASE(resume_admissible_crtc_restored_test), + KUNIT_CASE(resume_stale_dpms_off_crtc_disabled_test), + {} +}; + +static struct kunit_suite resume_suite = { + .name = "apple-drm-resume", + .test_cases = resume_cases, +}; + +kunit_test_suite(resume_suite); From bc230530c842e234ae35e067c9311c586987a5e3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:22:06 +1100 Subject: [PATCH 1221/2680] drm/apple: Release the encoder of a connector detached for resume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A CRTC restored disabled after system sleep has its connectors detached from it in the suspend state, but each keeps the encoder it was routed through. The commit that restores the state then finds a connector with an encoder and no CRTC, which the atomic helpers only expect from a broken duplicate, and every such resume logs WARNING: drivers/gpu/drm/drm_atomic_helper.c:238 at set_best_encoder The CRTC also keeps the encoder in its encoder mask. Release the encoder together with the CRTC, as the helpers do when they route a connector to no CRTC. The KUnit test now checks the edited suspend state. [ryan: adapted to 7.2.10 — struct drm_atomic_state is struct drm_atomic_commit] Fixes: 229d38cd44fd ("drm/apple: Resume a CRTC whose mode needs a new link disabled") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_resume.c | 14 ++++++++++++-- drivers/gpu/drm/apple/apple_resume_test.c | 15 ++++++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_resume.c b/drivers/gpu/drm/apple/apple_resume.c index f03c1a55e4527..ebb78883861bf 100644 --- a/drivers/gpu/drm/apple/apple_resume.c +++ b/drivers/gpu/drm/apple/apple_resume.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -54,10 +55,19 @@ void apple_drm_drop_stale_crtcs(struct drm_atomic_commit *state) drm_atomic_set_fb_for_plane(plane_state, NULL); } for_each_new_connector_in_state(state, connector, conn_state, j) { + if (conn_state->crtc != crtc) + continue; /* Detaching from a CRTC already in the state cannot fail. */ - if (conn_state->crtc == crtc && - WARN_ON(drm_atomic_set_crtc_for_connector(conn_state, NULL))) + if (WARN_ON(drm_atomic_set_crtc_for_connector(conn_state, NULL))) return; + /* + * Nor does a connector without a CRTC keep an encoder: + * the resume check warns about one left routed. + */ + if (conn_state->best_encoder) + crtc_state->encoder_mask &= + ~drm_encoder_mask(conn_state->best_encoder); + conn_state->best_encoder = NULL; } crtc_state->active = false; if (WARN_ON(drm_atomic_set_mode_for_crtc(crtc_state, NULL))) diff --git a/drivers/gpu/drm/apple/apple_resume_test.c b/drivers/gpu/drm/apple/apple_resume_test.c index 93c59f4ec1762..7c3370e263b62 100644 --- a/drivers/gpu/drm/apple/apple_resume_test.c +++ b/drivers/gpu/drm/apple/apple_resume_test.c @@ -179,10 +179,23 @@ static void resume_stale_mode_fails_unfiltered_test(struct kunit *test) static void resume_stale_crtc_disabled_test(struct kunit *test) { struct resume_test *priv = resume_test_init(test); + struct drm_connector_state *conn_state; + struct drm_crtc_state *crtc_state; + struct drm_atomic_commit *state; KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); priv->admissible = false; - apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state); + state = priv->drm.mode_config.suspend_state; + apple_drm_drop_stale_crtcs(state); + + /* The resume check warns about an encoder routed without a CRTC. */ + conn_state = drm_atomic_get_new_connector_state(state, &priv->connector); + crtc_state = drm_atomic_get_new_crtc_state(state, priv->crtc); + KUNIT_ASSERT_NOT_NULL(test, conn_state); + KUNIT_ASSERT_NOT_NULL(test, crtc_state); + KUNIT_EXPECT_NULL(test, conn_state->best_encoder); + KUNIT_EXPECT_EQ(test, crtc_state->encoder_mask, 0U); + KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); KUNIT_EXPECT_FALSE(test, priv->crtc->state->enable); From daaa62acc4537eb83ae1e7f29fce4384f7d32ffe Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:23:56 +1100 Subject: [PATCH 1222/2680] drm/apple: Have userspace restore a CRTC resumed disabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A CRTC whose mode needs a link that system sleep released is restored disabled, on the expectation that the hotplug following the relink makes userspace enable it again. Nothing guarantees that hotplug: the DCP reports a display that comes back on the same cable only for a new cable or the first attachment, and the 180 Hz mode is announced again only if its rate proof is renewed. Even when a hotplug arrives, the connector is still connected with the same modes, so a compositor that tracks its own state sees no change, believes the output is still lit, and has every flip rejected on the disabled CRTC. Use the link-status contract instead: mark the link of each connector detached from such a CRTC bad in the state being restored, and send a hotplug event once resume has committed it. Userspace then sets the mode again, with whatever modes the connector offers at that point, as it does after a failed link retraining. [ryan: adapted to 7.2.10 — struct drm_atomic_state is struct drm_atomic_commit] Fixes: 229d38cd44fd ("drm/apple: Resume a CRTC whose mode needs a new link disabled") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 6 +++++- drivers/gpu/drm/apple/apple_resume.c | 19 ++++++++++++++----- drivers/gpu/drm/apple/apple_resume.h | 4 +++- drivers/gpu/drm/apple/apple_resume_test.c | 14 +++++++++++--- 4 files changed, 33 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 5474d6dcf2114..aa80399b3be40 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -945,11 +945,15 @@ static int apple_platform_suspend(struct device *dev) static int apple_platform_resume(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); + bool dropped = false; if (apple) { if (apple->drm.mode_config.suspend_state) - apple_drm_drop_stale_crtcs(apple->drm.mode_config.suspend_state); + dropped = apple_drm_drop_stale_crtcs(apple->drm.mode_config.suspend_state); drm_mode_config_helper_resume(&apple->drm); + /* Have userspace set the mode of a CRTC resumed disabled again. */ + if (dropped) + drm_kms_helper_hotplug_event(&apple->drm); } return 0; diff --git a/drivers/gpu/drm/apple/apple_resume.c b/drivers/gpu/drm/apple/apple_resume.c index ebb78883861bf..caf1c814c5b98 100644 --- a/drivers/gpu/drm/apple/apple_resume.c +++ b/drivers/gpu/drm/apple/apple_resume.c @@ -19,13 +19,17 @@ * sleep releases that link. Restoring such a CRTC as it was fails the whole * resume commit in its mode fixup, which leaves every output dark until * userspace reacts to the next hotplug. Restore that CRTC disabled instead. - * The DCP reports the display again once its link is back, and userspace - * re-enables it on that hotplug, as it does after a cable is replugged. + * Userspace still believes the CRTC is on, and the DCP need not report the + * same display again when its link comes back, so mark the link of each of + * its connectors bad: userspace then sets the mode again, on the hotplug + * event the caller sends, as after a failed link retraining. * * The suspend state holds every CRTC, plane and connector already, so it is * edited directly rather than through helpers that need the modeset locks. + * + * Returns true if a CRTC is to be resumed disabled. */ -void apple_drm_drop_stale_crtcs(struct drm_atomic_commit *state) +bool apple_drm_drop_stale_crtcs(struct drm_atomic_commit *state) { const struct drm_crtc_helper_funcs *funcs; struct drm_connector_state *conn_state; @@ -34,6 +38,7 @@ void apple_drm_drop_stale_crtcs(struct drm_atomic_commit *state) struct drm_connector *connector; struct drm_plane *plane; struct drm_crtc *crtc; + bool dropped = false; int i, j; for_each_new_crtc_in_state(state, crtc, crtc_state, i) { @@ -59,7 +64,7 @@ void apple_drm_drop_stale_crtcs(struct drm_atomic_commit *state) continue; /* Detaching from a CRTC already in the state cannot fail. */ if (WARN_ON(drm_atomic_set_crtc_for_connector(conn_state, NULL))) - return; + return true; /* * Nor does a connector without a CRTC keep an encoder: * the resume check warns about one left routed. @@ -68,9 +73,13 @@ void apple_drm_drop_stale_crtcs(struct drm_atomic_commit *state) crtc_state->encoder_mask &= ~drm_encoder_mask(conn_state->best_encoder); conn_state->best_encoder = NULL; + conn_state->link_status = DRM_LINK_STATUS_BAD; } crtc_state->active = false; + dropped = true; if (WARN_ON(drm_atomic_set_mode_for_crtc(crtc_state, NULL))) - return; + break; } + + return dropped; } diff --git a/drivers/gpu/drm/apple/apple_resume.h b/drivers/gpu/drm/apple/apple_resume.h index 48d443afc029e..20b4c2951463f 100644 --- a/drivers/gpu/drm/apple/apple_resume.h +++ b/drivers/gpu/drm/apple/apple_resume.h @@ -4,8 +4,10 @@ #ifndef __APPLE_RESUME_H__ #define __APPLE_RESUME_H__ +#include + struct drm_atomic_commit; -void apple_drm_drop_stale_crtcs(struct drm_atomic_commit *state); +bool apple_drm_drop_stale_crtcs(struct drm_atomic_commit *state); #endif /* __APPLE_RESUME_H__ */ diff --git a/drivers/gpu/drm/apple/apple_resume_test.c b/drivers/gpu/drm/apple/apple_resume_test.c index 7c3370e263b62..f9144fa02babb 100644 --- a/drivers/gpu/drm/apple/apple_resume_test.c +++ b/drivers/gpu/drm/apple/apple_resume_test.c @@ -186,7 +186,8 @@ static void resume_stale_crtc_disabled_test(struct kunit *test) KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); priv->admissible = false; state = priv->drm.mode_config.suspend_state; - apple_drm_drop_stale_crtcs(state); + /* The caller tells userspace about the CRTC it had and lost. */ + KUNIT_EXPECT_TRUE(test, apple_drm_drop_stale_crtcs(state)); /* The resume check warns about an encoder routed without a CRTC. */ conn_state = drm_atomic_get_new_connector_state(state, &priv->connector); @@ -205,6 +206,9 @@ static void resume_stale_crtc_disabled_test(struct kunit *test) KUNIT_EXPECT_NULL(test, priv->plane->state->crtc); KUNIT_EXPECT_NULL(test, priv->plane->state->fb); KUNIT_EXPECT_NULL(test, priv->connector.state->crtc); + /* Userspace must modeset again rather than flip onto a dead CRTC. */ + KUNIT_EXPECT_EQ(test, priv->connector.state->link_status, + DRM_LINK_STATUS_BAD); /* Only the creator's reference is left: the saved scanout was dropped. */ KUNIT_EXPECT_EQ(test, drm_framebuffer_read_refcount(&priv->fb), 1U); } @@ -214,7 +218,8 @@ static void resume_admissible_crtc_restored_test(struct kunit *test) struct resume_test *priv = resume_test_init(test); KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); - apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state); + KUNIT_EXPECT_FALSE(test, + apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state)); KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); KUNIT_EXPECT_TRUE(test, priv->crtc->state->enable); @@ -251,10 +256,13 @@ static void resume_stale_dpms_off_crtc_disabled_test(struct kunit *test) KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); priv->admissible = false; - apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state); + KUNIT_EXPECT_TRUE(test, + apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state)); KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); KUNIT_EXPECT_FALSE(test, priv->crtc->state->enable); KUNIT_EXPECT_NULL(test, priv->connector.state->crtc); + KUNIT_EXPECT_EQ(test, priv->connector.state->link_status, + DRM_LINK_STATUS_BAD); KUNIT_EXPECT_EQ(test, drm_framebuffer_read_refcount(&priv->fb), 1U); } From 12dca82fda369bf60cb893fa126a17feb8e545a3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 15:15:37 +1100 Subject: [PATCH 1223/2680] drm/apple: Hold internal DCP messages during system sleep DPMS off on the internal H17P panel is a brightness present and leaves the pipe scanning out. While it scans out, the firmware publishes a property to the AP on every frame (D300 pr_publish, id 108, about 60 per second), and the DCP mailbox interrupt is IRQF_NO_SUSPEND, so each of them wakes a CPU out of suspend-to-idle: a 60 s RTC sleep took about 3650 interrupts from 412e08000.mailbox-recv, a 360 s one 21676. Hold the coprocessor's messages in the mailbox as the last step of the DCP suspend handler, after presentation is paused and the command queue has settled, and process them first thing on resume. A 360 s sleep now takes 97 interrupts, all of them in the transitions, and the panel and the firmware carry on normally after it. apple-drm is a device-link consumer of each DCP, so its suspend handler has blanked the panel before this runs and its resume handler restores the CRTC after the messages are processed. Powering the pipe off with A473 or idling the coprocessor through RTKit instead are not usable: the first left the firmware crashed after the power on, and the firmware does not acknowledge the second. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/dcp.c | 37 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 0d26b7182ac68..314e9861f756c 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -340,6 +340,8 @@ struct apple_dcp { bool active; /* A successful initial enable keeps the H17P pipe running across DPMS. */ bool pipe_enabled_h17p; + /* Messages from the coprocessor are held in the mailbox for sleep. */ + bool rx_held; /* eDP display without DP-HDMI conversion */ bool main_display; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 20544653f6167..c0730659de90e 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -4641,6 +4641,40 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) dev_warn(dcp->dev, "brightness change still pending at suspend\n"); } +/* + * DPMS off leaves the internal H17P pipe scanning out (see dcp_poweroff()), + * and while it scans out the firmware publishes a property to the AP on every + * frame. The mailbox interrupt is not suspended, so each of those messages + * wakes a CPU out of suspend-to-idle about 60 times a second. Hold the + * messages in the mailbox for system sleep and process them at resume. + * apple-drm is a consumer of this device, so its suspend handler has blanked + * the panel before this runs and its resume handler restores the CRTC after. + */ +static void dcp_hold_for_sleep(struct apple_dcp *dcp) +{ + if (!dcp_uses_soft_dpms(dcp) || dcp->fw_compat != DCP_FIRMWARE_H17P || + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G || + READ_ONCE(dcp->crashed)) + return; + + iomfb_pause_presentation(dcp, true); + if (iomfb_queue_settle(dcp, msecs_to_jiffies(500))) { + iomfb_pause_presentation(dcp, false); + return; + } + apple_rtkit_hold_rx(dcp->rtk); + dcp->rx_held = true; +} + +static void dcp_release_after_sleep(struct apple_dcp *dcp) +{ + if (!dcp->rx_held) + return; + dcp->rx_held = false; + apple_rtkit_resume_rx(dcp->rtk); + iomfb_pause_presentation(dcp, false); +} + static int dcp_platform_suspend(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); @@ -4684,6 +4718,8 @@ static int dcp_platform_suspend(struct device *dev) * shutdown properly yet. */ device_set_wakeup_path(dev); + /* Last: the steps above still talk to the coprocessor. */ + dcp_hold_for_sleep(dcp); return 0; } @@ -4692,6 +4728,7 @@ static int dcp_platform_resume(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); + dcp_release_after_sleep(dcp); dcp_enable_typec_work(dcp); if (dcp->hdmi_hpd_irq) enable_irq(dcp->hdmi_hpd_irq); From d65530990cf06732964c349162728eee17152a55 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 15:32:20 +1100 Subject: [PATCH 1224/2680] drm/apple, usb: typec: tipd: Retry a display mux that is only busy The CD321x driver treats -EBUSY from typec_mux_set() as "the shared external display is driven by the other port": it sets up the port's USB path without the display and does not retry. -EBUSY has other sources on the same mux chain, though. The Apple Type-C PHY returns it while a mode or pipe transition is pending, and the DCP route returned it while the previous SAFE transition was still running and while the external DCP had not finished stopping. Those conditions clear on their own, but the port was left without its display until the next cable event. Have the DCP route report the lasting condition, every pipeline driving another port's display, as -ENOSPC, and the transient ones as -EAGAIN. The CD321x driver now skips the retry only for -ENOSPC, through a helper with a KUnit case; -EBUSY and -EAGAIN are retried like any other failure, -EAGAIN without counting against the port. [ryan: split by topic: only the drm/apple part and its selftests are here; the PHY, Type-C, Thunderbolt and crossbar hunks are in 170-usb] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index c0730659de90e..449b905ee9ca9 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -464,7 +464,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) guard(mutex)(&dcp->external_lock); if (dcp_is_external_h17p(dcp) && (dcp->external_error || (dcp->external_power_attempted && !dcp->external_stopped))) - return dcp->external_error ?: -EBUSY; + return dcp->external_error ?: -EAGAIN; dcp_firmware_metadata_connection(dcp, READ_ONCE(dcp->typec_generation), false); @@ -973,7 +973,7 @@ static int dcp_typec_route_prepare(struct typec_mux_dev *mux, guard(mutex)(&dcp_typec_fabric_lock); if (state->mode != TYPEC_STATE_SAFE) - return port->safe_pending ? -EBUSY : 0; + return port->safe_pending ? -EAGAIN : 0; if (!port->owner || port->owner->tunnel) return 0; dcp = port->owner->dcp; @@ -1060,8 +1060,9 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (state->mode == TYPEC_STATE_SAFE && port->safe_pending) return 0; + /* The previous SAFE transition has not finished; retry. */ if (port->safe_pending) - return -EBUSY; + return -EAGAIN; /* * Every candidate route for this port is notified with the same state, @@ -1154,7 +1155,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, dcp_typec_rebalance_locked(NULL, 0); if (!port->owner) { if (hpd) - return -EBUSY; + return -ENOSPC; port->applied_valid = true; return 0; } @@ -1173,8 +1174,9 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (!best) best = dcp_typec_lowest_free(port); + /* Every pipeline is driving another port's display. */ if (!best) - return -EBUSY; + return -ENOSPC; ret = dcp_typec_route_activate(best, best->xbar); if (ret) From 545013cf611b3d731fcadef05d8d9b15871002ed Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 15:34:20 +1100 Subject: [PATCH 1225/2680] drm/apple: Log routine DPTX and EDID service events at debug level Every hotplug, link training and resume of the external display logged a dozen info lines: each DPTX port call (SET_LINK_RATE, GET_SUPPORTS_HPD, DEACTIVATE, FORCE_HOTPLUG_DETECT, INACTIVE_SINK_DETECTED, the lane count calls and the validate_connection attributes), the AV service announcing its EDID channel, every connect attempt, every boot property and read_edt_data request, and "copy_edid failed: -ENODEV" whenever the EDID was probed while the link was down. None of them is a problem; they are the normal protocol flow. Log them with dev_dbg() instead, so dynamic debug still shows them. -ENODEV from the EDID copy now means no EDID source for any connector, not only the internal panel. A copy or publication failure with any other error is a real problem and is now a warning. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 12 +++++------ drivers/gpu/drm/apple/dptxep.c | 27 ++++++++++++------------- drivers/gpu/drm/apple/epic/dpavservep.c | 4 ++-- drivers/gpu/drm/apple/iomfb.c | 11 +++++----- drivers/gpu/drm/apple/iomfb_template.c | 8 ++++---- 5 files changed, 31 insertions(+), 31 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 449b905ee9ca9..35e3d026a0100 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2499,12 +2499,12 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *waiting_s dev_warn(dcp->dev, "dcp_dptx_connect: missing phy\n"); return -ENODEV; } - dev_info(dcp->dev, - "%s(port=%d) target=%u:%u typec=%d route=%s conn_type=%d connected=%d\n", - __func__, port, dcp->dptx_die, dcp->dptx_phy, - dcp_is_typec_output(dcp), - dcp->active_typec_route ? "borrowed" : "fixed", - dcp->connector_type, dcp->dptxport[port].connected); + dev_dbg(dcp->dev, + "%s(port=%d) target=%u:%u typec=%d route=%s conn_type=%d connected=%d\n", + __func__, port, dcp->dptx_die, dcp->dptx_phy, + dcp_is_typec_output(dcp), + dcp->active_typec_route ? "borrowed" : "fixed", + dcp->connector_type, dcp->dptxport[port].connected); mutex_lock(&dcp->hpd_mutex); if (!dptx_service_available(&dcp->dptxport[port])) { diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 7bc55bfffe213..1e06d9185fa5e 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -152,8 +152,8 @@ int dptxport_validate_connection(struct apple_epic_service *service, u8 core, le32_to_cpu(resp.unk), attrs); return -EINVAL; } - dev_info(service->ep->dcp->dev, - "validate_connection: attrs reply 0x%x (sent 0x%x)\n", + dev_dbg(service->ep->dcp->dev, + "validate_connection: attrs reply 0x%x (sent 0x%x)\n", le32_to_cpu(resp.unk), attrs); } @@ -373,7 +373,7 @@ static int dptxport_call_get_max_lane_count(struct apple_epic_service *service, dptx->lane_count = 4; reply->retcode = cpu_to_le32(0); reply->lane_count = cpu_to_le64(4); - dev_info(dcp->dev, "get_max_lane_count: USB4 DP IN, 4 lanes\n"); + dev_dbg(dcp->dev, "get_max_lane_count: USB4 DP IN, 4 lanes\n"); return 0; } @@ -455,8 +455,8 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic reply->lane_count = cpu_to_le64(lane_count); if (lane_count > 0) { - dev_info(dcp->dev, "USB4/DPTX: SET_ACTIVE_LANE_COUNT %llu\n", - lane_count); + dev_dbg(dcp->dev, "USB4/DPTX: SET_ACTIVE_LANE_COUNT %llu\n", + lane_count); /* * dcp_usb4_drm_allowed() (usb4_force_dptx) used to gate this * for the old manual-training sysfs knob (module_param_cb @@ -545,8 +545,8 @@ static int dptxport_call_set_link_rate(struct apple_epic_service *service, link_rate = le32_to_cpu(request->link_rate); trace_dptxport_call_set_link_rate(dptx, link_rate); - dev_info(service->ep->dcp->dev, "DPTXPort: SET_LINK_RATE 0x%x\n", - link_rate); + dev_dbg(service->ep->dcp->dev, "DPTXPort: SET_LINK_RATE 0x%x\n", + link_rate); switch (link_rate) { case LINK_RATE_RBR: @@ -624,8 +624,8 @@ static int dptxport_call_get_supports_hpd(struct apple_epic_service *service, * that path instead of waiting for a PHY start. */ reply->supported = cpu_to_le32(dcp_is_typec_output(dcp) ? 1 : 0); - dev_info(dcp->dev, "DPTXPort: GET_SUPPORTS_HPD %u usb4=%d\n", - le32_to_cpu(reply->supported), dcp_is_usb4_output(dcp)); + dev_dbg(dcp->dev, "DPTXPort: GET_SUPPORTS_HPD %u usb4=%d\n", + le32_to_cpu(reply->supported), dcp_is_usb4_output(dcp)); return 0; } @@ -692,7 +692,7 @@ dptxport_call_deactivate(struct apple_epic_service *service, struct apple_dcp *dcp = service->ep->dcp; dcp_180hz_invalidate(dcp); - dev_info(dcp->dev, "DPTXPort: DEACTIVATE\n"); + dev_dbg(dcp->dev, "DPTXPort: DEACTIVATE\n"); if (dcp->dptx_tunnel) dcp_tunnel_dpin_activate(dcp, false); if (dptx->atcphy && !dcp->phy_managed_by_typec) @@ -825,8 +825,7 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, return dptxport_call_deactivate(service, data, data_size, reply, reply_size); case DPTX_APCALL_FORCE_HOTPLUG_DETECT: - dev_info(service->ep->dcp->dev, - "DPTXPort: FORCE_HOTPLUG_DETECT\n"); + dev_dbg(service->ep->dcp->dev, "DPTXPort: FORCE_HOTPLUG_DETECT\n"); complete(&dptx->linkcfg_completion); memcpy(reply, data, min(reply_size, data_size)); if (reply_size >= 4) @@ -838,8 +837,8 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, * SET_ACTIVE_LANE_COUNT. Treating this as failure made * firmware DEACTIVATE after it had already set lanes/rate. */ - dev_info(service->ep->dcp->dev, - "DPTXPort: INACTIVE_SINK_DETECTED (keep waiting for lanes)\n"); + dev_dbg(service->ep->dcp->dev, + "DPTXPort: INACTIVE_SINK_DETECTED (keep waiting for lanes)\n"); memcpy(reply, data, min(reply_size, data_size)); if (reply_size >= 4) memset(reply, 0, 4); diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index a61d28e5e2edb..2476209719c08 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -135,8 +135,8 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, return; } if (v2) { - dev_info(dcp->dev, "DCPAVSERV: AFKV2 EDID service ready on channel %u\n", - service->channel); + dev_dbg(dcp->dev, "DCPAVSERV: AFKV2 EDID service ready on channel %u\n", + service->channel); dcpavserv_notify_edid(service); } } diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 6a3b1cc76209f..b11f658df5a8e 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1179,13 +1179,14 @@ int dcp_get_modes(struct drm_connector *connector) dcp_consume_edid_invalidation(apple_connector); if (IS_ERR_OR_NULL(edid)) { /* - * An internal panel has no AV service and so no EDID; + * There is no AV service for an internal panel, nor + * for an external display while its link is down; * do not report that on every mode probe. */ - if (dcp_has_panel(dcp) && PTR_ERR(edid) == -ENODEV) - dev_dbg(dcp->dev, "no EDID source for the panel\n"); + if (PTR_ERR(edid) == -ENODEV) + dev_dbg(dcp->dev, "no EDID source\n"); else - dev_info(dcp->dev, "copy_edid failed: %pe\n", edid); + dev_warn(dcp->dev, "copy_edid failed: %pe\n", edid); } else { drm_edid_free(apple_connector->drm_edid); apple_connector->drm_edid = edid; @@ -1197,7 +1198,7 @@ int dcp_get_modes(struct drm_connector *connector) int update = drm_edid_connector_update(connector, apple_connector->drm_edid); if (update) { - dev_info(dcp->dev, "raw EDID publication failed: %d\n", update); + dev_warn(dcp->dev, "raw EDID publication failed: %d\n", update); dcp_clear_connector_edid(apple_connector); } else { apple_connector->drm_edid_firmware_derived = false; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 40f1733280aa6..a4cce328cb337 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -878,8 +878,8 @@ dcpep_cb_read_edt_data(struct apple_dcp *dcp, struct dcp_read_edt_data_req *req) { /* Observe boot-property requests without inventing firmware timings. */ if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_eDP) - dev_info(dcp->dev, - "read_edt_data key=%.*s count=%u default0=%#x ret=0\n", + dev_dbg(dcp->dev, + "read_edt_data key=%.*s count=%u default0=%#x ret=0\n", (int)sizeof(req->key), req->key, req->count, req->value[0]); @@ -1110,8 +1110,8 @@ static u8 dcpep_cb_prop_end(struct apple_dcp *dcp, u8 resp = dcpep_process_chunks(dcp, req); if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_eDP) - dev_info(dcp->dev, - "DCP property key=%.*s bytes=%zu accepted=%u nr_modes=%u\n", + dev_dbg(dcp->dev, + "DCP property key=%.*s bytes=%zu accepted=%u nr_modes=%u\n", (int)sizeof(req->key), req->key, dcp->chunks.length, resp, dcp->nr_modes); From 3a5edb0f707d1cba04ac6bc780343277f92c4c7d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 15:35:16 +1100 Subject: [PATCH 1226/2680] drm/apple: Describe the H17P support without bring-up terms Comments, function names and a few messages in the H17P support still use the language of its bring-up: "measured" profiles, "qualified" loaders and sequences, an "unqualified" callback, the "USB4 PHY experiment". They tell a reader nothing beyond the code next to them, and "unqualified" in an error suggests a state the driver could get out of rather than an unsupported request. Say what is meant instead: the H17P profile, a known or unknown sequence, an unsupported callback or request, the J700 defaults. Rename dptxport_call_qualified_config() and dptxport_qualified_teardown() after what they do. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-180hz-policy.h | 2 +- drivers/gpu/drm/apple/dcp.c | 6 +++--- drivers/gpu/drm/apple/dptxep.c | 14 ++++++++------ drivers/gpu/drm/apple/iomfb.c | 2 +- drivers/gpu/drm/apple/iomfb_h17p.c | 16 ++++++++-------- drivers/gpu/drm/apple/iomfb_template.c | 12 ++++++------ drivers/gpu/drm/apple/plane.c | 2 +- 7 files changed, 28 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-180hz-policy.h b/drivers/gpu/drm/apple/dcp-180hz-policy.h index a221c0e4a0773..d23aa57c685e2 100644 --- a/drivers/gpu/drm/apple/dcp-180hz-policy.h +++ b/drivers/gpu/drm/apple/dcp-180hz-policy.h @@ -6,7 +6,7 @@ #include #include -/* Only this qualified uncompressed configuration is admitted. */ +/* Only this verified uncompressed configuration is admitted. */ static inline bool dcp_180hz_payload(u32 rate, u32 lanes, u32 clock) { return rate == 0x1e && lanes == 2 && clock == 414090 && diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 35e3d026a0100..047c653433262 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -4444,7 +4444,7 @@ static int dcp_platform_probe(struct platform_device *pdev) set_bit(surf, dcp->iomfb_surfaces); surf++; } - /* The qualified J700 loader names surface zero rather than enabling it. */ + /* The J700 loader names surface zero rather than enabling it. */ if (of_machine_is_compatible("apple,j700") && of_device_is_compatible(dev->of_node, "apple,t8140-dcp") && num_surfs == 1 && @@ -4583,7 +4583,7 @@ static void dcp_platform_shutdown(struct platform_device *pdev) dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { /* * This firmware keeps scanning across soft DPMS and has no - * qualified stop sequence. Component unbind releases the RTKit + * known stop sequence. Component unbind releases the RTKit * devres group and the piodma domain while firmware may still * access them. Stop host submissions but keep the component and * its DMA resources alive until the system resets. @@ -4753,7 +4753,7 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8112 = { .num_dptx_ports = 2, }; -/* The internal T8140 endpoint uses the measured H17P method profile. */ +/* The internal T8140 endpoint uses the H17P method profile. */ static const struct resource t8140_firmware_clock = DEFINE_RES_MEM(0x302800000, 0xbc000); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 1e06d9185fa5e..b9d4729407ed6 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -705,8 +705,8 @@ dptxport_call_deactivate(struct apple_epic_service *service, return 0; } -static int dptxport_call_qualified_config(struct apple_epic_service *service, bool lanes, - const void *data, size_t size, void *reply, size_t room) +static int dptxport_call_link_config(struct apple_epic_service *service, bool lanes, + const void *data, size_t size, void *reply, size_t room) { struct dcp_180hz_token token = dcp_180hz_config_begin(service, lanes); struct dptx_port *port = service->cookie; @@ -776,7 +776,8 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, case DPTX_APCALL_GET_LINK_RATE: return dptxport_call_get_link_rate(service, reply, reply_size); case DPTX_APCALL_SET_LINK_RATE: - return dptxport_call_qualified_config(service, false, data, data_size, reply, reply_size); + return dptxport_call_link_config(service, false, data, data_size, + reply, reply_size); case DPTX_APCALL_GET_MAX_LANE_COUNT: return dptxport_call_get_max_lane_count(service, reply, reply_size); case DPTX_APCALL_GET_ACTIVE_LANE_COUNT: { @@ -789,7 +790,8 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, return 0; } case DPTX_APCALL_SET_ACTIVE_LANE_COUNT: - return dptxport_call_qualified_config(service, true, data, data_size, reply, reply_size); + return dptxport_call_link_config(service, true, data, data_size, + reply, reply_size); case DPTX_APCALL_GET_SUPPORTS_HPD: return dptxport_call_get_supports_hpd(service, reply, reply_size); @@ -974,7 +976,7 @@ static int dptxport_group_call(struct apple_epic_service *service, u16 group, return 0; } -static void dptxport_qualified_teardown(struct apple_epic_service *service) +static void dptxport_teardown(struct apple_epic_service *service) { dcp_180hz_invalidate(service->ep->dcp); dptx_service_unbind(service); @@ -984,7 +986,7 @@ static const struct apple_epic_service_ops dptxep_ops[] = { { .name = "AppleDCPDPTXRemotePort", .prepare = dptxport_prepare, - .teardown = dptxport_qualified_teardown, + .teardown = dptxport_teardown, .init = dptxport_init, .call = dptxport_call, .call_v2 = dptxport_group_call, diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index b11f658df5a8e..00856206b8c70 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -901,7 +901,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && handled && !iomfb_check_callback_h17p(dcp, tag, hdr->in_len, hdr->out_len)))) { - dev_err(dev, "unqualified IOMFB callback %d (%u, %u)\n", + dev_err(dev, "unsupported IOMFB callback %d (%u, %u)\n", tag, hdr->in_len, hdr->out_len); dcp->crashed = true; return; diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index dcce1e22d1575..a77de05c538c1 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -312,7 +312,7 @@ static bool trampoline_analytics_h17p(struct apple_dcp *dcp, int tag, return trampoline_zero(dcp, tag, out, in); trace_iomfb_callback(dcp, tag, __func__); - /* The dispatcher has validated the measured D114 reply size. */ + /* The dispatcher has validated the D114 reply size. */ memset(out, 0, 0x1004); *(u8 *)out = 'd'; return true; @@ -391,7 +391,7 @@ static const u8 j700_provider_data_4[] = { 0xa2, 0x19, 0x00, 0x00 }; -/* Only these provider transfers have been measured against the disp0 ADT. */ +/* Only these provider transfers are known to match the disp0 ADT. */ static const struct dcp_h17p_provider_property provider_properties[] = { { "power-lut-data-x", "apple,power-lut-data-x", 4, j700_provider_data_0 }, { "power-lut-data-y", "apple,power-lut-data-y", 4, j700_provider_data_1 }, @@ -435,13 +435,13 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, trace_iomfb_callback(dcp, tag, __func__); property = dcp_provider_property_h17p(in); if (!property) { - dev_err(dcp->dev, "unqualified D400 provider property request\n"); + dev_err(dcp->dev, "unsupported D400 provider property request\n"); goto fail; } length = of_property_count_u8_elems(node, property->dt_name); if (!property->length) { - /* Only this absent property has a measured empty reply. */ + /* Only this absent property has a known empty reply. */ if (of_property_present(node, property->dt_name)) goto fail; } else if (!of_property_present(node, property->dt_name) && @@ -449,12 +449,12 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, (of_device_is_compatible(node, "apple,t8140-dcp") || (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && of_device_is_compatible(node, "apple,t8140-dcpext")))) { - /* Qualified older loaders lack the newly described table properties. */ + /* Older loaders lack the newly described table properties. */ memset(reply, 0, sizeof(*reply)); memcpy(reply->data, property->legacy_data, property->length); reply->length = cpu_to_le32(property->length); dev_warn_once(dcp->dev, - "missing provider property %s (%u bytes); using qualified J700 compatibility data\n", + "missing provider property %s (%u bytes); using the J700 defaults\n", property->dt_name, property->length); return true; } else if (length != (int)property->length) { @@ -487,7 +487,7 @@ static bool trampoline_luma_span_h17p(struct apple_dcp *dcp, int tag, trace_iomfb_callback(dcp, tag, __func__); if (get_unaligned_le32(in)) { - dev_err(dcp->dev, "unqualified D574 range selector\n"); + dev_err(dcp->dev, "unsupported D574 range selector\n"); WRITE_ONCE(dcp->crashed, true); return false; } @@ -511,7 +511,7 @@ static bool trampoline_get_uint_prop_h17p(struct apple_dcp *dcp, int tag, strnlen(request->key, sizeof(request->key)) == sizeof(request->key) || strcmp(request->key, "Temperature") || request->value_null || memchr_inv(request->padding, 0, sizeof(request->padding))) { - dev_err(dcp->dev, "unqualified D401 integer property query\n"); + dev_err(dcp->dev, "unsupported D401 integer property query\n"); WRITE_ONCE(dcp->crashed, true); return false; } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index a4cce328cb337..7bf13ff8f12d0 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -460,7 +460,7 @@ static void iomfb_cb_pr_publish(struct apple_dcp *dcp, struct iomfb_property *pr case IOMFB_PROPERTY_NITS_V14_7: { #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - /* Measured H17P takeover comes from the powerlog hint interface. */ + /* The H17P takeover level comes from the powerlog hint interface. */ if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) break; #endif @@ -1408,7 +1408,7 @@ static void release_swap_cookie(struct kref *ref) * format two bytes late and its surface check rejects the swap as an * unsupported format. * - * H17G retains this public packing path. The measured H17P profile uses + * H17G retains this public packing path. The H17P profile uses * a separate wire record with the SPEC's 0x588 swap boundary. */ static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) @@ -1431,7 +1431,7 @@ static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) req->swap.h17p_pre_swap_id[0x00] = 2; req->swap.h17p_pre_swap_id[0x28] = 3; - /* The measured H17P layout is serialized without shifting its surfaces. */ + /* The H17P layout is serialized without shifting its surfaces. */ if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) return; @@ -1749,7 +1749,7 @@ static void complete_set_powerstate(struct apple_dcp *dcp, void *out, static bool dcp_power_stop_confirmed(struct dcp_wait_cookie *cookie) { - /* H17P stop semantics still require hardware qualification. */ + /* H17P does not confirm that scanout has stopped. */ #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER return false; #else @@ -1792,7 +1792,7 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) typeof(DCP_FW_UNION(dcp->swap)) *swap = &DCP_FW_UNION(dcp->swap); #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - /* The measured profile has no safe pipe-off or surface-free transition. */ + /* H17P has no known safe pipe-off or surface-free transition. */ if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) return; #endif @@ -1942,7 +1942,7 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) if (dcp->main_display) return; /* - * Report firmware hotplug independently of the USB4 PHY experiment. + * Report firmware hotplug independently of the USB4 PHY. * Reassigning lpdptxphy blanked eDP even with these callbacks ignored; * suppressing connector notifications does not protect the panel. * Mode probing still uses this DCP's firmware modes, and mode_valid diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index d5a27bc05f268..22729e810391f 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -561,7 +561,7 @@ static const u32 dcp_primary_formats[] = { #endif }; -/* H17P 10-bit XRGB alpha semantics have not been qualified. */ +/* The H17P alpha semantics of 10-bit XRGB are unknown. */ static const u32 dcp_primary_formats_h17p[] = { DRM_FORMAT_ARGB2101010, DRM_FORMAT_XRGB8888, From 4fb18c5b55941f3d30f8f20354df3d978bd845ca Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 08:25:37 -0500 Subject: [PATCH 1227/2680] drm/asahi: Add stats_export module parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Opt-out toggle for the AGX firmware stats sysfs export. When 1 (the default), StatsChannel::poll retains the values it already decodes into the StatsSnapshot exposed via /sys/class/drm/card*/device/agx_stats. When 0, the export is suppressed and the file is not created — matching today's behaviour exactly (the channel is still polled, the messages are still debug-logged). [ryan: squashed f19bcf1a1e37 ("drm/asahi: stats: Add StatsSnapshot and firmware-stats decoder"), 72d21d654534 ("drm/asahi: Wire StatsSnapshot into StatsChannel and GpuManager"), e8c789f9f77b ("drm/asahi: Add sysfs shim for agx_stats"), 1b02b0b40af3 ("drm/asahi: Register agx_stats sysfs file on probe and unregister on drop"), c86f87f33104 ("Documentation/ABI: Document sysfs-driver-asahi-agx-stats"), 742f54c4855d ("drm/asahi: count completed submissions in agx_stats jobs"), aebd7f529942 ("Documentation/ABI: make sysfs-driver-asahi-agx-stats match the output"), bbfb96f309c8 ("drm/asahi: link the sysfs shim into the driver module correctly"), 01d19b8a4f55 ("drm/asahi: propagate the stats snapshot allocation result"), 91c24dc11b09 ("drm/asahi: decode the stats messages where StatsMsg is nameable"), dbc8b48aeb06 ("drm/asahi: qualify the versioned StatsMsg variant patterns"), 8ca5289bec3a ("drm/asahi: connect the snapshot pointer to the C sysfs static"), 4189501b643f ("drm/asahi: fix the agx_stats readout layout and the off-arm string"), 0b923fb29c84 ("drm/asahi: integrate utilization-weighted time into agx_stats busy_ns"), 9250872b1f33 ("drm/asahi: convert stats timestamps from base-clock ticks to ns"), 993288b37ec4 ("drm/asahi: split the tick window to avoid 128-bit division") into this commit; the agx_stats series does not build commit by commit (asahi.rs declares stats and sysfs_exports before they exist, StatsMsg is not nameable until 91c24dc11b09); adapted to 7.2.10 — minimal; for DJ/Joshua: the channel constructors take the owned 7.2 device handle, the probe registration lives in the 7.2 platform driver data (Send/Sync by hand), new_queue() passes the snapshot through the moved Gpu::new_queue(), and the sysfs FFI prototypes take c_void because rustc 1.97 rejects struct device in extern blocks (improper_ctypes)] Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- .../ABI/testing/sysfs-driver-asahi-agx-stats | 66 ++++++++ drivers/gpu/drm/asahi/Makefile | 1 + .../gpu/drm/asahi/{asahi.rs => asahi_core.rs} | 7 + drivers/gpu/drm/asahi/channel.rs | 93 ++++++++++- drivers/gpu/drm/asahi/driver.rs | 22 +++ drivers/gpu/drm/asahi/gpu.rs | 18 ++- drivers/gpu/drm/asahi/queue/mod.rs | 9 ++ drivers/gpu/drm/asahi/stats.rs | 79 ++++++++++ drivers/gpu/drm/asahi/sysfs.c | 148 ++++++++++++++++++ drivers/gpu/drm/asahi/sysfs.h | 12 ++ drivers/gpu/drm/asahi/sysfs_exports.rs | 46 ++++++ 11 files changed, 498 insertions(+), 3 deletions(-) create mode 100644 Documentation/ABI/testing/sysfs-driver-asahi-agx-stats rename drivers/gpu/drm/asahi/{asahi.rs => asahi_core.rs} (90%) create mode 100644 drivers/gpu/drm/asahi/stats.rs create mode 100644 drivers/gpu/drm/asahi/sysfs.c create mode 100644 drivers/gpu/drm/asahi/sysfs.h create mode 100644 drivers/gpu/drm/asahi/sysfs_exports.rs diff --git a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats new file mode 100644 index 0000000000000..278099b0570b8 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats @@ -0,0 +1,66 @@ +What: /sys/class/drm/card*/device/agx_stats +Date: October 2026 +Contact: Asahi Linux GPU driver maintainers +Description: + Read-only dump of the AGX firmware statistics decoded by the + drm/asahi driver. The driver already polls the firmware statistics + ring (see `drivers/gpu/drm/asahi/channel.rs::StatsChannel::poll`, + `drivers/gpu/drm/asahi/fw/channels.rs::StatsMsg`) and exposes the + latest values via this file. + + The file is owned by the platform device + `/sys/devices/platform/<...>.gpu/`, which is symlinked from the + `device` entry of the matching DRM card under `/sys/class/drm/`. + + One `key value` pair per line. Lines are ASCII integer values + unless noted. Unknown keys may be appended in future firmware + versions and must be ignored by readers. + + Module parameter `asahi.stats_export` (default 1) controls + whether the driver retains the values. When 0, the file shows + `unsupported` (the firmware messages are still polled but + discarded). + + Access: Read-only. Owner-readable (mode 0444). + + busy_ns cumulative u64; utilization-weighted busy time in + nanoseconds since boot: each Utilization message window + contributes its duration scaled by the busiest subqueue + percentage. Firmware timestamps are base-clock ticks + (HwConfig.base_clock_hz, 24 MHz on every supported SoC) + and are converted to ns by the driver. Monotonic; + bounded by elapsed wall time. MEASURED on T6021 + (M2 Max, 13.5): idle desktop fraction ~0.01, saturated + 4096^2 fp16 matmul fraction ~1.0 (util1..4 = 100). + T6001 shares the message format and the 24 MHz clock + (fw 13.5); the same idle-vs-matmul check must be run + there before the magnitude claim is repeated. + + jobs u64; cumulative completed submissions since boot. + Counted on the host at submission completion (when + the last command of a submission completes and its + fence signals), not from a firmware counter. + + pstate u32; current performance state index (PowerState.pstate). + + power_mw u32; latest average power reading in milliwatts + (AvgPower.avg_power). May be 0 if the firmware has not + yet reported AvgPower. + + util1..util4 u32; raw `Utilization` fields from the firmware, + unscaled. Semantics (firmware tick vs % vs MHz) are + validated against a controlled MLX load and may + differ per SoC revision. Treat as raw until a per-chip + doc note lands. + + temperature_raw, temperature_scale u32; latest Temperature + sample and scale (or 0 if Temperature has not yet + arrived). + + Field meanings (units, window, saturation) are documented in + `drivers/gpu/drm/asahi/stats.rs` and may evolve with firmware + revisions. busy_ns is monotonic non-decreasing while the device + is bound (it only accumulates). The snapshot struct is + `#[repr(C)]` on the Rust side and its offsets are asserted at + build time in `sysfs.c`; readers must parse `key value` lines and + must not assume binary layout of this file. \ No newline at end of file diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index e672486679876..a3aff32eea160 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += asahi.o +asahi-y := asahi_core.o sysfs.o diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi_core.rs similarity index 90% rename from drivers/gpu/drm/asahi/asahi.rs rename to drivers/gpu/drm/asahi/asahi_core.rs index 84d5d92e7dce8..ad789db094ecb 100644 --- a/drivers/gpu/drm/asahi/asahi.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -26,6 +26,8 @@ mod pgtable; mod queue; mod regs; mod slotalloc; +mod stats; +mod sysfs_exports; mod util; #[cfg(CONFIG_DRM_ASAHI_MAPLE_TREE)] mod vm; @@ -80,5 +82,10 @@ module! { // permissions: 0o644, description: "Fully isolate GPU contexts (limits performance)", }, + stats_export: u8 { + default: 1, + // permissions: 0, + description: "Export AGX firmware stats to sysfs (1 = on, 0 = off)", + }, }, } diff --git a/drivers/gpu/drm/asahi/channel.rs b/drivers/gpu/drm/asahi/channel.rs index 6b32cbd6dfb8d..01fd1d9f2d0bc 100644 --- a/drivers/gpu/drm/asahi/channel.rs +++ b/drivers/gpu/drm/asahi/channel.rs @@ -23,8 +23,10 @@ use crate::{ buffer, event, gpu, - mem, // + mem, + module_parameters, // }; +use core::sync::atomic::Ordering; use kernel::{ c_str, prelude::*, @@ -578,11 +580,21 @@ impl KTraceChannel { } /// Statistics channel, reporting power-related statistics to the driver. -/// Not really implemented other than debug logs yet... +/// Debug-logs the messages and, when the `stats_export` module parameter +/// is enabled, also updates the shared `StatsSnapshot` exported via +/// `/sys/class/drm/card*/device/agx_stats`. #[versions(AGX)] pub(crate) struct StatsChannel { dev: AsahiDevRef, ch: RxChannel, + snap: Arc, + /// Rate of firmware stats-message timestamps (Hz). All stats timestamps + /// run on the chip base clock (`HwConfig::base_clock_hz`, 24 MHz on every + /// supported SoC), not nanoseconds. + ts_hz: u64, + /// Last `Utilization` message timestamp (raw base-clock ticks), for + /// utilization-weighted `busy_ns` integration. + last_util_ts: u64, } #[versions(AGX)] @@ -591,10 +603,15 @@ impl StatsChannel::ver { pub(crate) fn new( dev: AsahiDevice, alloc: &mut gpu::KernelAllocators, + snap: Arc, + ts_hz: u64, ) -> Result { Ok(StatsChannel::ver { dev: dev.clone(), ch: RxChannel::::new(alloc, 0x100)?, + snap, + ts_hz, + last_util_ts: 0, }) } @@ -614,6 +631,78 @@ impl StatsChannel::ver { // accessing the enum view is valid. let msg = unsafe { msg.msg }; cls_dev_dbg!(StatsCh, self.dev, "Stats: {:?}\n", msg); + if *module_parameters::stats_export.value() != 0 { + // Retain the latest values for /sys/class/drm/card*/ + // device/agx_stats. Read-only with respect to + // scheduling or power behaviour: the firmware already + // sent the message, we only keep what it said. + match &msg { + StatsMsg::ver::Utilization { + timestamp, + util1, + util2, + util3, + util4, + } => { + self.snap.util1.store(*util1, Ordering::Relaxed); + self.snap.util2.store(*util2, Ordering::Relaxed); + self.snap.util3.store(*util3, Ordering::Relaxed); + self.snap.util4.store(*util4, Ordering::Relaxed); + // busy_ns: utilization-weighted busy time in + // nanoseconds. Each Utilization window + // contributes its duration scaled by the + // busiest subqueue's percentage. MEASURED on + // T6021 (round-3 window): util1..4 read 100 + // across a saturated 30 s matmul and 0-2 on + // an idle desktop, while FwBusy timestamp + // deltas (the prior producer) covered 0.17% + // of that matmul. The timestamps are base- + // clock ticks (24 MHz), so convert to ns + // once here. The kernel runtime has no + // 128-bit division, so split whole seconds + // from the tick remainder: whole seconds + // contribute util * 1e7 ns each (1e9 / 100 + // exactly), the remainder contributes + // rem * util * 1e9 / (100 * ts_hz), which + // cannot overflow u64 (rem < ts_hz <= 24e6 + // on supported SoCs). + let util = [*util1, *util2, *util3, *util4] + .into_iter() + .max() + .unwrap_or(0) + .min(100) as u64; + let ts = timestamp.0; + if self.last_util_ts != 0 && ts > self.last_util_ts { + let delta = ts - self.last_util_ts; + let whole_s = delta / self.ts_hz; + let rem = delta % self.ts_hz; + let busy = whole_s + .saturating_mul(util) + .saturating_mul(10_000_000) + .saturating_add( + rem * util * 1_000_000_000 / (100 * self.ts_hz), + ); + self.snap.busy_ns.fetch_add(busy, Ordering::Relaxed); + } + self.last_util_ts = ts; + } + StatsMsg::ver::PowerState { pstate, .. } => { + self.snap.pstate.store(*pstate, Ordering::Relaxed); + } + StatsMsg::ver::AvgPower { avg_power, .. } => { + self.snap.avg_power_mw.store(*avg_power, Ordering::Relaxed); + } + StatsMsg::ver::Temperature { + raw_value, scale, .. + } => { + self.snap + .temperature_raw + .store(*raw_value, Ordering::Relaxed); + self.snap.temperature_scale.store(*scale, Ordering::Relaxed); + } + _ => {} + } + } } _ => { // SAFETY: The raw view is always valid for all bit patterns. diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index d95dca73eb78e..176fe49488c5d 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -2,6 +2,8 @@ //! Top-level GPU driver implementation. +use crate::module_parameters; +use kernel::bindings; use kernel::{ device::{ Core, @@ -47,8 +49,16 @@ pub(crate) struct AsahiPlatformDriverData<'bound> { _reg: drm::Registration<'bound, AsahiDrmDriver>, /// The G17 GPU, whose coprocessors must be stopped when the driver is unbound. g17: Option>, + /// Raw `struct device *` of the platform device. Stored once at probe + /// time so the `Drop` impl can find the sysfs file on unregister. + raw_dev: *mut bindings::device, } +// SAFETY: `raw_dev` is only passed back to the sysfs shim, which serializes its own state. +unsafe impl Send for AsahiPlatformDriverData<'_> {} +// SAFETY: As above. +unsafe impl Sync for AsahiPlatformDriverData<'_> {} + /// Holds a reference to the top-level GPU object. #[pin_data] pub(crate) struct AsahiDrmRegistrationData<'drm> { @@ -249,10 +259,16 @@ impl platform::Driver for AsahiPlatformDriver { // unbound; it is never forgotten. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; + // Register the sysfs file on the platform device. Must happen after + // the DRM device is registered so the device is fully bound. + let raw_dev = pdev.as_ref().as_raw(); + crate::sysfs_exports::register(raw_dev, *module_parameters::stats_export.value() != 0)?; + let driver = AsahiPlatformDriverData { _device: reg.device().into(), _reg: reg, g17: None, + raw_dev, }; Ok(driver) @@ -288,12 +304,18 @@ impl AsahiPlatformDriver { _device: reg.device().into(), _reg: reg, g17: Some(gpu), + raw_dev: core::ptr::null_mut(), }) } } impl Drop for AsahiPlatformDriverData<'_> { fn drop(&mut self) { + // SAFETY: `self.raw_dev` was set in `probe` from the platform + // device's `as_raw()`, valid for the lifetime of the bound driver. + if !self.raw_dev.is_null() { + crate::sysfs_exports::unregister(self.raw_dev); + } if let Some(gpu) = self.g17.as_ref() { // Also covers failure to allocate platform driver data after the probe body returned. gpu.shutdown(); diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index f4ab618693163..ccdcfcee2d163 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -83,6 +83,8 @@ use crate::{ mmu, queue, regs, + stats, + sysfs_exports, workqueue, // }; @@ -256,6 +258,7 @@ pub(crate) struct GpuManager { fwctl_channel: Mutex, pipes: PipeChannels::ver, event_manager: Arc, + stats: Arc, buffer_mgr: buffer::BufferManager::ver, ids: SequenceIDs, #[allow(clippy::vec_box)] @@ -761,6 +764,12 @@ impl GpuManager::ver { let buffer_mgr = buffer::BufferManager::ver::new()?; let event_manager_clone = event_manager.clone(); let buffer_mgr_clone = buffer_mgr.clone(); + let stats_snap = Arc::new(stats::StatsSnapshot::default(), GFP_KERNEL)?; + // Publish the snapshot pointer to the C sysfs shim. It is cleared + // (by the same shim) on registration removal; until then readers see + // a stable pointer and AtomicU* races that govern. + sysfs_exports::set_snapshot_ptr(Arc::as_ptr(&stats_snap)); + let stats_snap_clone = stats_snap.clone(); let alloc_ref = &mut alloc; let rx_channels = KBox::init( try_init!(RxChannels::ver { @@ -772,7 +781,12 @@ impl GpuManager::ver { )?, fw_log: channel::FwLogChannel::new(dev.into(), alloc_ref)?, ktrace: channel::KTraceChannel::new(dev.into(), alloc_ref)?, - stats: channel::StatsChannel::ver::new(dev.into(), alloc_ref)?, + stats: channel::StatsChannel::ver::new( + dev.into(), + alloc_ref, + stats_snap_clone, + u64::from(cfg.base_clock_hz), + )?, }), GFP_KERNEL, )?; @@ -797,6 +811,7 @@ impl GpuManager::ver { rtkit <- new_mutex!(None, "rtkit"), crashed: AtomicBool::new(false), event_manager, + stats: stats_snap.clone(), alloc <- new_mutex!(alloc, "alloc"), fwctl_channel <- new_mutex!(fwctl_channel, "fwctl_channel"), rx_channels <- new_mutex!(KBox::::into_inner(rx_channels), "rx_channels"), @@ -1348,6 +1363,7 @@ impl Gpu for GpuManager::ver { ualloc, ualloc_priv, self.event_manager.clone(), + self.stats.clone(), &self.buffer_mgr, id, priority, diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 0e327ffef2e82..34fc306b66b4f 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -33,6 +33,7 @@ use crate::gpu::{ use crate::inner_weak_ptr; use crate::microseq; use crate::module_parameters; +use crate::stats; use crate::util::{ AnyBitPattern, Reader, // @@ -133,6 +134,7 @@ pub(crate) struct Queue { q_comp: Option, fence_ctx: FenceContexts, inner: QueueInner::ver, + stats: Arc, } #[versions(AGX)] @@ -154,6 +156,7 @@ pub(crate) struct QueueInner { pub(crate) struct JobFence { id: u64, pending: AtomicU64, + stats: Option>, } #[versions(AGX)] @@ -170,6 +173,9 @@ impl JobFence::ver { remain ); if remain == 0 { + if let Some(stats) = &self.stats { + stats.note_job(); + } mod_pr_debug!("JobFence[{}]: Signaling\n", self.id); self.signal(); } @@ -465,6 +471,7 @@ impl Queue::ver { ualloc: Arc>, ualloc_priv: Arc>, event_manager: Arc, + stats: Arc, mgr: &buffer::BufferManager::ver, id: u64, priority: u32, @@ -521,6 +528,7 @@ impl Queue::ver { q_frag: None, q_comp: None, fence_ctx: FenceContexts::new(1, QUEUE_NAME, QUEUE_CLASS_KEY)?, + stats, inner: QueueInner::ver { dev: dev.clone(), ualloc, @@ -707,6 +715,7 @@ impl Queue for Queue::ver { JobFence::ver { id, pending: Default::default(), + stats: Some(self.stats.clone()), }, )? .into(); diff --git a/drivers/gpu/drm/asahi/stats.rs b/drivers/gpu/drm/asahi/stats.rs new file mode 100644 index 0000000000000..e3160e1a5c632 --- /dev/null +++ b/drivers/gpu/drm/asahi/stats.rs @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Snapshot of AGX firmware statistics. +//! +//! [`StatsSnapshot`] accumulates the values decoded by `StatsChannel::poll` +//! (see `fw::channels.rs::StatsMsg`) and exposes them to the sysfs file +//! `agx_stats` (registered from `sysfs.c`). +//! +//! The snapshot is process-local and process-private. The firmware-stat +//! decoder that feeds it lives in `StatsChannel::poll` (`channel.rs`), inside +//! the versioned context where `StatsMsg` is nameable; `note_job()` is called +//! from the `recv_message` rtkit callback (so the firmware mailbox IRQ +//! context); the sysfs `show` callback runs in arbitrary process context. All +//! fields use `AtomicU*` so neither side takes a lock. +//! +//! Field semantics (firmware units, late-validated): +//! - `util1..util4`: raw `Utilization` u32s from the firmware. Meaning +//! (firmware tick or %?) is validated against a controlled MLX load; we +//! publish them unscaled per the producer contract. +//! - `pstate`: current performance state index from `PowerState`. +//! - `avg_power_mw`: average power in milliwatts from `AvgPower` (firmware +//! field is u32 — units not validated for every chip rev; treat as raw). +//! - `temperature`: raw `Temperature` value, scale, tmin, tmax. +//! - `busy_ns`: cumulative utilization-weighted busy time in nanoseconds, +//! derived from successive `Utilization` windows (busiest subqueue +//! percentage x window duration; the firmware timestamps are base-clock +//! ticks, 24 MHz, converted with `HwConfig::base_clock_hz`). Monotonic and +//! bounded by elapsed time. Saturates at u64::MAX. +//! - `jobs`: completed submissions since boot, counted at submission +//! completion (`JobFence::command_complete`). +//! +//! This module is read-only with respect to scheduling or power behaviour: +//! the firmware already sends the messages, the channel already polls them, +//! we only retain the latest values. + +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; + +/// Process-wide snapshot of AGX firmware stats. +/// +/// Lives in the `GpuManager`; the C static `asahi_stats_snapshot_ptr` (see +/// `sysfs.c`) holds its address for the C `show` callback, so the field +/// layout is a cross-language contract: `#[repr(C)]` makes the declaration +/// order binding and `sysfs.c` asserts the offsets at build time. +#[derive(Default)] +#[repr(C)] +pub(crate) struct StatsSnapshot { + pub(crate) util1: AtomicU32, + pub(crate) util2: AtomicU32, + pub(crate) util3: AtomicU32, + pub(crate) util4: AtomicU32, + pub(crate) pstate: AtomicU32, + pub(crate) avg_power_mw: AtomicU32, + pub(crate) temperature_raw: AtomicU32, + pub(crate) temperature_scale: AtomicU32, + pub(crate) temperature_tmin: AtomicU32, + pub(crate) temperature_tmax: AtomicU32, + /// Cumulative utilization-weighted busy time. + pub(crate) busy_ns: AtomicU64, + /// Completed submissions since boot, counted at fence signal in + /// `JobFence::command_complete` when the last command of a submission + /// completes. + pub(crate) jobs: AtomicU64, +} + +/// Pointer to the live snapshot: the C static `asahi_stats_snapshot_ptr` +/// (see `sysfs.c`) is the single storage location, written through +/// `sysfs_exports::set_snapshot_ptr` and read by the C `show` callback via +/// `READ_ONCE`. The pointer is established before the DRM device registers +/// and removed on unregister; the pointed-to fields are `AtomicU*` and carry +/// the data races themselves. + +impl StatsSnapshot { + /// Bump the completed-submission counter (called from the queue + /// completion path in `JobFence::command_complete`, NOT from the stats + /// channel). + pub(crate) fn note_job(&self) { + self.jobs.fetch_add(1, Ordering::Relaxed); + } +} diff --git a/drivers/gpu/drm/asahi/sysfs.c b/drivers/gpu/drm/asahi/sysfs.c new file mode 100644 index 0000000000000..923e4013804b7 --- /dev/null +++ b/drivers/gpu/drm/asahi/sysfs.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// +// Sysfs shim for the AGX firmware stats export. +// +// The Rust side keeps the stats state in `crate::stats::StatsSnapshot` and +// publishes its raw pointer to the static `asahi_stats_snapshot_ptr` below +// (an atomic `u64` so the writer and reader do not need locking). This file +// owns the actual `device_attribute` and the formatted output, because +// `device_create_file` and the `device_attribute` macros are not in the +// Rust bindgen bindings for this kernel tree. +// +// The contract is one read-only file: +// /sys/class/drm/cardX/device/agx_stats +// Format: `key value\n` per line, ASCII integers, owner-readable. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sysfs.h" + +/* + * Mirror of `#[repr(C)] crate::stats::StatsSnapshot` field layout. Rust sets + * `asahi_stats_snapshot_ptr` (an atomic u64) to the heap address of one of + * these, and clears it on unregister. Field reads are all `Relaxed` atomic + * u32 / u64 (the C side uses READ_ONCE). + * + * The Rust struct carries `#[repr(C)]` so this declaration order is binding; + * the BUILD_BUG_ONs in asahi_sysfs_register() turn any drift (a repr(Rust) + * struct is silently reordered by rustc, which once cross-aligned this + * readout: busy_ns bounced in 2^32 steps and jobs read 0 while incrementing) + * into a compile error. + */ +struct asahi_stats_snapshot { + u32 util1; + u32 util2; + u32 util3; + u32 util4; + u32 pstate; + u32 avg_power_mw; + u32 temperature_raw; + u32 temperature_scale; + u32 temperature_tmin; + u32 temperature_tmax; + u64 busy_ns; + u64 jobs; +}; + +static int asahi_stats_export_enabled; + +/* + * Set by Rust through asahi_stats_set_snapshot_ptr(); read via READ_ONCE. + * A NULL (0) pointer means the device has not yet exposed stats (or has been + * unregistered). The Rust side owns the lifetime of the pointed-to struct. + */ +unsigned long long asahi_stats_snapshot_ptr; + +void asahi_stats_set_snapshot_ptr(unsigned long long p) +{ + WRITE_ONCE(asahi_stats_snapshot_ptr, p); +} + +static ssize_t agx_stats_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct asahi_stats_snapshot __rcu *snap; + ssize_t n = 0; + + (void)attr; + + rcu_read_lock(); + snap = (struct asahi_stats_snapshot __rcu *) + READ_ONCE(asahi_stats_snapshot_ptr); + if (!snap || !READ_ONCE(asahi_stats_export_enabled)) { + rcu_read_unlock(); + return scnprintf(buf, PAGE_SIZE, "unsupported\n"); + } + n += scnprintf(buf + n, PAGE_SIZE - n, "busy_ns %llu\n", + (unsigned long long)READ_ONCE(snap->busy_ns)); + n += scnprintf(buf + n, PAGE_SIZE - n, "jobs %llu\n", + (unsigned long long)READ_ONCE(snap->jobs)); + n += scnprintf(buf + n, PAGE_SIZE - n, "pstate %u\n", + (u32)READ_ONCE(snap->pstate)); + n += scnprintf(buf + n, PAGE_SIZE - n, "power_mw %u\n", + (u32)READ_ONCE(snap->avg_power_mw)); + n += scnprintf(buf + n, PAGE_SIZE - n, "util1 %u\n", + (u32)READ_ONCE(snap->util1)); + n += scnprintf(buf + n, PAGE_SIZE - n, "util2 %u\n", + (u32)READ_ONCE(snap->util2)); + n += scnprintf(buf + n, PAGE_SIZE - n, "util3 %u\n", + (u32)READ_ONCE(snap->util3)); + n += scnprintf(buf + n, PAGE_SIZE - n, "util4 %u\n", + (u32)READ_ONCE(snap->util4)); + n += scnprintf(buf + n, PAGE_SIZE - n, "temperature_raw %u\n", + (u32)READ_ONCE(snap->temperature_raw)); + n += scnprintf(buf + n, PAGE_SIZE - n, "temperature_scale %u\n", + (u32)READ_ONCE(snap->temperature_scale)); + rcu_read_unlock(); + + return n; +} + +static DEVICE_ATTR_RO(agx_stats); + +/* + * Called from Rust's `AsahiDriver::probe` after the DRM device is + * registered. Returns 0 on success, or a negative errno. + */ +int asahi_sysfs_register(struct device *dev, int export_enabled) +{ + int ret; + + if (!dev) + return -ENODEV; + + BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, pstate) != 16); + BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, busy_ns) != 40); + BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, jobs) != 48); + BUILD_BUG_ON(sizeof(struct asahi_stats_snapshot) != 56); + + WRITE_ONCE(asahi_stats_export_enabled, export_enabled); + + ret = device_create_file(dev, &dev_attr_agx_stats); + if (ret) + return ret; + + return 0; +} +EXPORT_SYMBOL_GPL(asahi_sysfs_register); + +void asahi_sysfs_unregister(struct device *dev) +{ + if (!dev) + return; + + device_remove_file(dev, &dev_attr_agx_stats); + WRITE_ONCE(asahi_stats_snapshot_ptr, 0); +} +EXPORT_SYMBOL_GPL(asahi_sysfs_unregister); + +MODULE_AUTHOR("AGX driver maintainer"); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Asahi AGX firmware stats sysfs shim"); diff --git a/drivers/gpu/drm/asahi/sysfs.h b/drivers/gpu/drm/asahi/sysfs.h new file mode 100644 index 0000000000000..ed96fbd3afe50 --- /dev/null +++ b/drivers/gpu/drm/asahi/sysfs.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ + +#ifndef _ASAHI_SYSFS_H +#define _ASAHI_SYSFS_H + +struct device; + +int asahi_sysfs_register(struct device *dev, int export_enabled); +void asahi_sysfs_unregister(struct device *dev); +void asahi_stats_set_snapshot_ptr(unsigned long long p); + +#endif /* _ASAHI_SYSFS_H */ diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs new file mode 100644 index 0000000000000..214aa3436b052 --- /dev/null +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! FFI declarations for the C `sysfs.c` shim. +//! +//! The Rust driver keeps the firmware stats in [`crate::stats::StatsSnapshot`] +//! and uses these shims to register / unregister the sysfs file and to publish +//! the snapshot pointer for the C-side `show` callback. + +use core::ffi::{c_int, c_ulonglong, c_void}; + +extern "C" { + /// Register the `agx_stats` sysfs file on `dev`; `export_enabled` 0 + /// makes the file print `unsupported`. Returns 0 on success. + fn asahi_sysfs_register(dev: *mut c_void, export_enabled: c_int) -> c_int; + + /// Unregister the `agx_stats` sysfs file and clear the snapshot pointer. + fn asahi_sysfs_unregister(dev: *mut c_void); + + /// Publish the snapshot pointer for the C-side `show` callback (the C + /// static `asahi_stats_snapshot_ptr` is the single storage location). + fn asahi_stats_set_snapshot_ptr(p: c_ulonglong); +} + +/// Register the sysfs file. Safe to call from `AsahiDriver::probe` after the +/// DRM device has been registered. +pub(crate) fn register( + dev: *mut kernel::bindings::device, + export_enabled: bool, +) -> kernel::error::Result { + let ret = unsafe { asahi_sysfs_register(dev.cast(), export_enabled as c_int) }; + if ret < 0 { + Err(kernel::error::Error::from_errno(ret)) + } else { + Ok(()) + } +} + +/// Unregister the sysfs file. Safe to call from the device release path. +pub(crate) fn unregister(dev: *mut kernel::bindings::device) { + unsafe { asahi_sysfs_unregister(dev.cast()) }; +} + +/// Publish the address of the live `StatsSnapshot` to the C shim. +pub(crate) fn set_snapshot_ptr(ptr: *const crate::stats::StatsSnapshot) { + unsafe { asahi_stats_set_snapshot_ptr(ptr as c_ulonglong) }; +} From dd4f38930a092665a5b004ebf327f3b995d6881d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:25:52 +1100 Subject: [PATCH 1228/2680] drm/asahi: tie stats registration to a per-device snapshot owner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep a snapshot Arc in the successfully bound driver and pass its address to a per-device C attribute wrapper. Register after GPU initialization and drain sysfs readers before dropping the owner. Remove early global pointer publication so competing or failing probes cannot replace another device snapshot. [ryan: adapted to 7.2.10 — the per-device snapshot owner lives in the 7.2 platform driver data next to the registration handle; stats_snapshot() is on the DRM-facing Gpu trait and returns an Option, and the G17 probe registers agx_stats without a snapshot, as aurora-wip resolves it in merge commit 527179cf0856 ("Merge pull request #168 from joshuaswarren/agent/jw16-agx-stats4"): sysfs.c accepts a NULL snapshot when export is disabled, and the ABI doc carries the two G17 lines] Assisted-by: LLM Signed-off-by: Ryan Murray --- .../ABI/testing/sysfs-driver-asahi-agx-stats | 4 +- drivers/gpu/drm/asahi/driver.rs | 29 +++++- drivers/gpu/drm/asahi/g17/mod.rs | 4 + drivers/gpu/drm/asahi/gpu.rs | 12 ++- drivers/gpu/drm/asahi/stats.rs | 23 +++-- drivers/gpu/drm/asahi/sysfs.c | 99 ++++++++----------- drivers/gpu/drm/asahi/sysfs.h | 6 +- drivers/gpu/drm/asahi/sysfs_exports.rs | 49 ++++----- 8 files changed, 115 insertions(+), 111 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats index 278099b0570b8..11a964e6f17d8 100644 --- a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats +++ b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats @@ -63,4 +63,6 @@ Description: is bound (it only accumulates). The snapshot struct is `#[repr(C)]` on the Rust side and its offsets are asserted at build time in `sysfs.c`; readers must parse `key value` lines and - must not assume binary layout of this file. \ No newline at end of file + must not assume binary layout of this file. + G17 firmware has no supported decoder for this snapshot format; + its file reports unsupported rather than fabricated zero counters. diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 176fe49488c5d..13d5d72dcab32 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -52,6 +52,9 @@ pub(crate) struct AsahiPlatformDriverData<'bound> { /// Raw `struct device *` of the platform device. Stored once at probe /// time so the `Drop` impl can find the sysfs file on unregister. raw_dev: *mut bindings::device, + raw_stats: *mut core::ffi::c_void, + // Own the allocation until unregister has drained all sysfs readers. + stats: Option>, } // SAFETY: `raw_dev` is only passed back to the sysfs shim, which serializes its own state. @@ -253,6 +256,7 @@ impl platform::Driver for AsahiPlatformDriver { }; gpu.init()?; + let stats = gpu.stats_snapshot(); let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); // SAFETY: `reg` is stored in `AsahiPlatformDriverData` and dropped when the driver is @@ -262,13 +266,20 @@ impl platform::Driver for AsahiPlatformDriver { // Register the sysfs file on the platform device. Must happen after // the DRM device is registered so the device is fully bound. let raw_dev = pdev.as_ref().as_raw(); - crate::sysfs_exports::register(raw_dev, *module_parameters::stats_export.value() != 0)?; + let snapshot = stats.as_ref().map_or(core::ptr::null(), Arc::as_ptr); + let raw_stats = crate::sysfs_exports::register( + raw_dev, + snapshot, + stats.is_some() && *module_parameters::stats_export.value() != 0, + )?; let driver = AsahiPlatformDriverData { _device: reg.device().into(), _reg: reg, g17: None, raw_dev, + raw_stats, + stats, }; Ok(driver) @@ -300,11 +311,22 @@ impl AsahiPlatformDriver { return Err(error); } }; + // G17's firmware channel format has no supported stats decoder yet. + let raw_dev = pdev.as_ref().as_raw(); + let raw_stats = match crate::sysfs_exports::register(raw_dev, core::ptr::null(), false) { + Ok(handle) => handle, + Err(error) => { + gpu.shutdown(); + return Err(error); + } + }; Ok(AsahiPlatformDriverData { _device: reg.device().into(), _reg: reg, g17: Some(gpu), - raw_dev: core::ptr::null_mut(), + raw_dev, + raw_stats, + stats: None, }) } } @@ -314,8 +336,9 @@ impl Drop for AsahiPlatformDriverData<'_> { // SAFETY: `self.raw_dev` was set in `probe` from the platform // device's `as_raw()`, valid for the lifetime of the bound driver. if !self.raw_dev.is_null() { - crate::sysfs_exports::unregister(self.raw_dev); + crate::sysfs_exports::unregister(self.raw_dev, self.raw_stats); } + drop(self.stats.take()); if let Some(gpu) = self.g17.as_ref() { // Also covers failure to allocate platform driver data after the probe body returned. gpu.shutdown(); diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 21e244185bd57..380eba6a76c33 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1054,6 +1054,10 @@ impl gpu::Gpu for Gpu { self as Arc } + fn stats_snapshot(&self) -> Option> { + None + } + fn manager(&self) -> Option<&dyn gpu::GpuManager> { None } diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index ccdcfcee2d163..ed54c350d7dbc 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -84,7 +84,6 @@ use crate::{ queue, regs, stats, - sysfs_exports, workqueue, // }; @@ -268,6 +267,9 @@ pub(crate) struct GpuManager { /// Operations the DRM driver performs on a GPU, independent of its firmware interface. pub(crate) trait Gpu: Send + Sync { + /// The firmware statistics snapshot of this GPU, if its firmware reports any. + fn stats_snapshot(&self) -> Option>; + /// Cast as an Any type. fn as_any(&self) -> &dyn Any; /// Cast Arc as an Any type. @@ -765,10 +767,6 @@ impl GpuManager::ver { let event_manager_clone = event_manager.clone(); let buffer_mgr_clone = buffer_mgr.clone(); let stats_snap = Arc::new(stats::StatsSnapshot::default(), GFP_KERNEL)?; - // Publish the snapshot pointer to the C sysfs shim. It is cleared - // (by the same shim) on registration removal; until then readers see - // a stable pointer and AtomicU* races that govern. - sysfs_exports::set_snapshot_ptr(Arc::as_ptr(&stats_snap)); let stats_snap_clone = stats_snap.clone(); let alloc_ref = &mut alloc; let rx_channels = KBox::init( @@ -1269,6 +1267,10 @@ impl GpuManager::ver { #[versions(AGX)] impl Gpu for GpuManager::ver { + fn stats_snapshot(&self) -> Option> { + Some(self.stats.clone()) + } + fn as_any(&self) -> &dyn Any { self } diff --git a/drivers/gpu/drm/asahi/stats.rs b/drivers/gpu/drm/asahi/stats.rs index e3160e1a5c632..772d969d9baff 100644 --- a/drivers/gpu/drm/asahi/stats.rs +++ b/drivers/gpu/drm/asahi/stats.rs @@ -6,7 +6,7 @@ //! (see `fw::channels.rs::StatsMsg`) and exposes them to the sysfs file //! `agx_stats` (registered from `sysfs.c`). //! -//! The snapshot is process-local and process-private. The firmware-stat +//! The snapshot belongs to one GPU manager. The firmware-stat //! decoder that feeds it lives in `StatsChannel::poll` (`channel.rs`), inside //! the versioned context where `StatsMsg` is nameable; `note_job()` is called //! from the `recv_message` rtkit callback (so the firmware mailbox IRQ @@ -35,12 +35,10 @@ use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; -/// Process-wide snapshot of AGX firmware stats. +/// Per-device snapshot of AGX firmware stats. /// -/// Lives in the `GpuManager`; the C static `asahi_stats_snapshot_ptr` (see -/// `sysfs.c`) holds its address for the C `show` callback, so the field -/// layout is a cross-language contract: `#[repr(C)]` makes the declaration -/// order binding and `sysfs.c` asserts the offsets at build time. +/// The manager and bound driver retain Arcs. The driver's Arc outlives the +/// per-device C sysfs registration; repr(C) establishes the shared layout. #[derive(Default)] #[repr(C)] pub(crate) struct StatsSnapshot { @@ -62,12 +60,13 @@ pub(crate) struct StatsSnapshot { pub(crate) jobs: AtomicU64, } -/// Pointer to the live snapshot: the C static `asahi_stats_snapshot_ptr` -/// (see `sysfs.c`) is the single storage location, written through -/// `sysfs_exports::set_snapshot_ptr` and read by the C `show` callback via -/// `READ_ONCE`. The pointer is established before the DRM device registers -/// and removed on unregister; the pointed-to fields are `AtomicU*` and carry -/// the data races themselves. +// Keep the Rust half of the C sysfs layout contract checked as well. +const _: () = { + assert!(core::mem::offset_of!(StatsSnapshot, pstate) == 16); + assert!(core::mem::offset_of!(StatsSnapshot, busy_ns) == 40); + assert!(core::mem::offset_of!(StatsSnapshot, jobs) == 48); + assert!(core::mem::size_of::() == 56); +}; impl StatsSnapshot { /// Bump the completed-submission counter (called from the queue diff --git a/drivers/gpu/drm/asahi/sysfs.c b/drivers/gpu/drm/asahi/sysfs.c index 923e4013804b7..9f0baa4350d93 100644 --- a/drivers/gpu/drm/asahi/sysfs.c +++ b/drivers/gpu/drm/asahi/sysfs.c @@ -1,22 +1,14 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // -// Sysfs shim for the AGX firmware stats export. -// -// The Rust side keeps the stats state in `crate::stats::StatsSnapshot` and -// publishes its raw pointer to the static `asahi_stats_snapshot_ptr` below -// (an atomic `u64` so the writer and reader do not need locking). This file -// owns the actual `device_attribute` and the formatted output, because -// `device_create_file` and the `device_attribute` macros are not in the -// Rust bindgen bindings for this kernel tree. -// -// The contract is one read-only file: -// /sys/class/drm/cardX/device/agx_stats -// Format: `key value\n` per line, ASCII integers, owner-readable. +// Per-device sysfs shim. Rust owns the snapshot through an Arc in the bound +// driver. The attribute wrapper is removed and its readers drained before +// that Arc is dropped. Atomic fields may reflect different sample instants. #include #include #include #include +#include #include #include #include @@ -24,18 +16,7 @@ #include "sysfs.h" -/* - * Mirror of `#[repr(C)] crate::stats::StatsSnapshot` field layout. Rust sets - * `asahi_stats_snapshot_ptr` (an atomic u64) to the heap address of one of - * these, and clears it on unregister. Field reads are all `Relaxed` atomic - * u32 / u64 (the C side uses READ_ONCE). - * - * The Rust struct carries `#[repr(C)]` so this declaration order is binding; - * the BUILD_BUG_ONs in asahi_sysfs_register() turn any drift (a repr(Rust) - * struct is silently reordered by rustc, which once cross-aligned this - * readout: busy_ns bounced in 2^32 steps and jobs read 0 while incrementing) - * into a compile error. - */ +/* Mirror of repr(C) Rust atomics; READ_ONCE accesses the current values. */ struct asahi_stats_snapshot { u32 util1; u32 util2; @@ -51,35 +32,22 @@ struct asahi_stats_snapshot { u64 jobs; }; -static int asahi_stats_export_enabled; - -/* - * Set by Rust through asahi_stats_set_snapshot_ptr(); read via READ_ONCE. - * A NULL (0) pointer means the device has not yet exposed stats (or has been - * unregistered). The Rust side owns the lifetime of the pointed-to struct. - */ -unsigned long long asahi_stats_snapshot_ptr; - -void asahi_stats_set_snapshot_ptr(unsigned long long p) -{ - WRITE_ONCE(asahi_stats_snapshot_ptr, p); -} +struct asahi_stats_attribute { + struct device_attribute attr; + const struct asahi_stats_snapshot *snapshot; + bool export_enabled; +}; static ssize_t agx_stats_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct asahi_stats_snapshot __rcu *snap; + struct asahi_stats_attribute *stats = + container_of(attr, struct asahi_stats_attribute, attr); + const struct asahi_stats_snapshot *snap = stats->snapshot; ssize_t n = 0; - (void)attr; - - rcu_read_lock(); - snap = (struct asahi_stats_snapshot __rcu *) - READ_ONCE(asahi_stats_snapshot_ptr); - if (!snap || !READ_ONCE(asahi_stats_export_enabled)) { - rcu_read_unlock(); - return scnprintf(buf, PAGE_SIZE, "unsupported\n"); - } + if (!stats->export_enabled) + return sysfs_emit(buf, "unsupported\n"); n += scnprintf(buf + n, PAGE_SIZE - n, "busy_ns %llu\n", (unsigned long long)READ_ONCE(snap->busy_ns)); n += scnprintf(buf + n, PAGE_SIZE - n, "jobs %llu\n", @@ -100,22 +68,22 @@ static ssize_t agx_stats_show(struct device *dev, (u32)READ_ONCE(snap->temperature_raw)); n += scnprintf(buf + n, PAGE_SIZE - n, "temperature_scale %u\n", (u32)READ_ONCE(snap->temperature_scale)); - rcu_read_unlock(); return n; } -static DEVICE_ATTR_RO(agx_stats); /* * Called from Rust's `AsahiDriver::probe` after the DRM device is * registered. Returns 0 on success, or a negative errno. */ -int asahi_sysfs_register(struct device *dev, int export_enabled) +int asahi_sysfs_register(struct device *dev, const void *snapshot, + int export_enabled, void **handle) { + struct asahi_stats_attribute *stats; int ret; - if (!dev) + if (!dev || !handle || (export_enabled && !snapshot)) return -ENODEV; BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, pstate) != 16); @@ -123,23 +91,38 @@ int asahi_sysfs_register(struct device *dev, int export_enabled) BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, jobs) != 48); BUILD_BUG_ON(sizeof(struct asahi_stats_snapshot) != 56); - WRITE_ONCE(asahi_stats_export_enabled, export_enabled); + stats = kzalloc_obj(*stats); + if (!stats) + return -ENOMEM; + + sysfs_attr_init(&stats->attr.attr); + stats->attr.attr.name = "agx_stats"; + stats->attr.attr.mode = 0444; + stats->attr.show = agx_stats_show; + stats->snapshot = snapshot; + stats->export_enabled = export_enabled; - ret = device_create_file(dev, &dev_attr_agx_stats); - if (ret) + ret = device_create_file(dev, &stats->attr); + if (ret) { + kfree(stats); return ret; + } + *handle = stats; return 0; } EXPORT_SYMBOL_GPL(asahi_sysfs_register); -void asahi_sysfs_unregister(struct device *dev) +void asahi_sysfs_unregister(struct device *dev, void *handle) { - if (!dev) + struct asahi_stats_attribute *stats = handle; + + if (!dev || !stats) return; - device_remove_file(dev, &dev_attr_agx_stats); - WRITE_ONCE(asahi_stats_snapshot_ptr, 0); + /* Removal drains active show callbacks before either allocation dies. */ + device_remove_file(dev, &stats->attr); + kfree(stats); } EXPORT_SYMBOL_GPL(asahi_sysfs_unregister); diff --git a/drivers/gpu/drm/asahi/sysfs.h b/drivers/gpu/drm/asahi/sysfs.h index ed96fbd3afe50..5969a1718925f 100644 --- a/drivers/gpu/drm/asahi/sysfs.h +++ b/drivers/gpu/drm/asahi/sysfs.h @@ -5,8 +5,8 @@ struct device; -int asahi_sysfs_register(struct device *dev, int export_enabled); -void asahi_sysfs_unregister(struct device *dev); -void asahi_stats_set_snapshot_ptr(unsigned long long p); +int asahi_sysfs_register(struct device *dev, const void *snapshot, + int export_enabled, void **handle); +void asahi_sysfs_unregister(struct device *dev, void *handle); #endif /* _ASAHI_SYSFS_H */ diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs index 214aa3436b052..1b0eb7c6ea50f 100644 --- a/drivers/gpu/drm/asahi/sysfs_exports.rs +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -1,46 +1,37 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! FFI declarations for the C `sysfs.c` shim. -//! -//! The Rust driver keeps the firmware stats in [`crate::stats::StatsSnapshot`] -//! and uses these shims to register / unregister the sysfs file and to publish -//! the snapshot pointer for the C-side `show` callback. +//! Per-device sysfs registration. The caller retains the snapshot Arc until +//! unregister has drained active readers and freed the C attribute wrapper. -use core::ffi::{c_int, c_ulonglong, c_void}; +use core::ffi::{c_int, c_void}; +use crate::stats::StatsSnapshot; extern "C" { - /// Register the `agx_stats` sysfs file on `dev`; `export_enabled` 0 - /// makes the file print `unsupported`. Returns 0 on success. - fn asahi_sysfs_register(dev: *mut c_void, export_enabled: c_int) -> c_int; - - /// Unregister the `agx_stats` sysfs file and clear the snapshot pointer. - fn asahi_sysfs_unregister(dev: *mut c_void); - - /// Publish the snapshot pointer for the C-side `show` callback (the C - /// static `asahi_stats_snapshot_ptr` is the single storage location). - fn asahi_stats_set_snapshot_ptr(p: c_ulonglong); + fn asahi_sysfs_register( + dev: *mut c_void, + snapshot: *const StatsSnapshot, + export_enabled: c_int, + handle: *mut *mut c_void, + ) -> c_int; + fn asahi_sysfs_unregister(dev: *mut c_void, handle: *mut c_void); } -/// Register the sysfs file. Safe to call from `AsahiDriver::probe` after the -/// DRM device has been registered. +/// Register only after initialization succeeds, keeping the snapshot Arc alive. pub(crate) fn register( dev: *mut kernel::bindings::device, + snapshot: *const StatsSnapshot, export_enabled: bool, -) -> kernel::error::Result { - let ret = unsafe { asahi_sysfs_register(dev.cast(), export_enabled as c_int) }; +) -> kernel::error::Result<*mut c_void> { + let mut handle = core::ptr::null_mut(); + let ret = unsafe { asahi_sysfs_register(dev.cast(), snapshot, export_enabled as c_int, &mut handle) }; if ret < 0 { Err(kernel::error::Error::from_errno(ret)) } else { - Ok(()) + Ok(handle) } } -/// Unregister the sysfs file. Safe to call from the device release path. -pub(crate) fn unregister(dev: *mut kernel::bindings::device) { - unsafe { asahi_sysfs_unregister(dev.cast()) }; -} - -/// Publish the address of the live `StatsSnapshot` to the C shim. -pub(crate) fn set_snapshot_ptr(ptr: *const crate::stats::StatsSnapshot) { - unsafe { asahi_stats_set_snapshot_ptr(ptr as c_ulonglong) }; +/// Drain readers before the caller drops its snapshot Arc. +pub(crate) fn unregister(dev: *mut kernel::bindings::device, handle: *mut c_void) { + unsafe { asahi_sysfs_unregister(dev.cast(), handle) }; } From dec58d5f3a1eba197772fa0bce7930faaeb6b8a5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 03:44:23 -0400 Subject: [PATCH 1229/2680] drm/asahi: pack G17 tiler heap blocks contiguously in their backing Each tiler heap (TVB) block is 128 KiB and is mapped at a 160 KiB VA stride, so that an unmapped 32 KiB guard follows it. The backing object used the same stride: block i was mapped from offset i * 160 KiB, and an object of n blocks was (n - 1) * 160 KiB + 128 KiB long. Every block but the last therefore carried 32 KiB of allocated, zeroed and pinned backing that no GPU mapping covers, 20 % of all tiler heap memory. Map block i from offset i * 128 KiB and size the backing to n * 128 KiB. The VA placement, the guards and the addresses handed to the firmware are unchanged. At the in-world peak of a measured Subnautica session the render pairs held 2,057 blocks in 49 objects, so this returns 62.7 MiB. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5f6490414a683eef149ba6de38cbc27ba62f9a26) --- drivers/gpu/drm/asahi/g17/buffer.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index 10bcf13c68789..dd1aed70a2bf0 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -340,21 +340,17 @@ pub(crate) struct Tvb { extensions: KVec, } impl Tvb { + /// Blocks are packed in the backing; only their VA placement keeps the + /// stride and its trailing guard. fn bytes(blocks: usize) -> Result { - blocks - .checked_sub(1) - .and_then(|v| v.checked_mul(cfg::TVB_BLOCK_STRIDE as usize)) - .and_then(|v| v.checked_add(cfg::TVB_BLOCK_SIZE)) - .ok_or(EOVERFLOW) + blocks.checked_mul(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW) } fn map_block( object: &KernelObject, vm: &mmu::Vm, relative: usize, ) -> Result { - let start = relative - .checked_mul(cfg::TVB_BLOCK_STRIDE as usize) - .ok_or(EOVERFLOW)?; + let start = relative.checked_mul(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW)?; let source = start..start.checked_add(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW)?; let mapping = compact_alias( object, From 7aab010ec260eeb74b527996d6e45ee8084a4724 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 03:51:06 -0400 Subject: [PATCH 1230/2680] drm/asahi: allocate only the G17 boot pool blocks that are used Boot allocates three groups of 1 MiB pool blocks that no GPU mapping ever covers and nothing writes: - render blocks 23..27: the render pool maps blocks 0..21 and keeps the compute page list in block 22, but allocated 28 blocks; - the 23rd block of the device-wide compute backing: compute VMs map 22 of the 23 blocks; - the backing of the two zeroed compute blocks: their VAs fall inside the compute pool's page-list mapping, and the objects were never aliased anywhere. Each block is a zeroed, pinned kernel object for the life of the device. Allocate only the mapped render blocks plus the page-list block, only the mapped compute blocks, and no zero-block backing. Every firmware-visible address and table is unchanged; this returns 8 MiB. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit d2a2c2fa97b994595209843720ad78b99b93f16a) --- drivers/gpu/drm/asahi/g17/initdata.rs | 8 ++------ drivers/gpu/drm/asahi/hw/t8140.rs | 5 +++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 4419056789dea..f2ec60633631f 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -23,7 +23,6 @@ const CONTEXT_SIZE: usize = 0x20000; const REGION_A_SIZE: usize = core::mem::size_of::(); const REGION_C_SIZE: usize = 0x1000; -const COMPUTE_BLOCKS_ALLOCATED: usize = 23; const COMPUTE_BLOCKS_MAPPED: usize = 22; const COMPUTE_BLOCKS_POPULATED: usize = 21; const COMPUTE_CONTROL_BLOCKS: u32 = 25; @@ -65,8 +64,8 @@ impl ComputeShared { KernelObject::backing(alloc.dev, low::RENDER_PAGE_LIST_SIZE, CpuMap::WriteCombined)?; let runs = KernelObject::backing(alloc.dev, low::RENDER_RUN_LIST_SIZE, CpuMap::WriteCombined)?; - let mut blocks = KVec::with_capacity(COMPUTE_BLOCKS_ALLOCATED, GFP_KERNEL)?; - for _ in 0..COMPUTE_BLOCKS_ALLOCATED { + let mut blocks = KVec::with_capacity(COMPUTE_BLOCKS_MAPPED, GFP_KERNEL)?; + for _ in 0..COMPUTE_BLOCKS_MAPPED { blocks.push( KernelObject::backing(alloc.dev, abi::FREE_LIST_BLOCK_SIZE, CpuMap::WriteCombined)?, GFP_KERNEL, @@ -873,9 +872,6 @@ impl InitData { let compute_state = self.add(KernelObject::backing(env.dev, mmu::UAT_PGSZ, WB)?)?; self.object_mut(compute_state)? .write(0, abi::FreeListState::COMPUTE)?; - for _ in &low::COMPUTE_ZERO_BLOCKS { - self.add(KernelObject::backing(env.dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; - } let compute_runs = self.add(KernelObject::new( env.dev, env.lower, diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 2e73aed1a9797..143ba720d06f2 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -196,8 +196,9 @@ pub(crate) mod low { pub(crate) const RENDER_BLOCKS: u64 = 0x70_0022_0000; /// Distance between consecutive blocks. pub(crate) const BLOCK_STRIDE: u64 = 0x10_8000; - /// Render blocks allocated at boot. - pub(crate) const RENDER_BLOCKS_ALLOCATED: usize = 28; + /// Render blocks allocated at boot: the mapped blocks and the block that + /// holds the compute page list. + pub(crate) const RENDER_BLOCKS_ALLOCATED: usize = COMPUTE_PAGE_LIST_BLOCK + 1; /// Render blocks mapped at boot. pub(crate) const RENDER_BLOCKS_MAPPED: usize = 22; /// Render blocks in the free list at boot. From a1446564976c8659cf564838f6cc41c778bd2c4a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 04:49:41 -0400 Subject: [PATCH 1231/2680] drm/asahi: give back idle G17 tiler heap growth beyond recent use A render pair's tiler heap (TVB) grows on demand, by firmware request or from the sampled page high-water, and never shrinks. A pair that grew for one heavy scene keeps that growth for its whole life, also after the client returns to a light scene or goes idle; in a measured Subnautica session the game's pairs held 260 MiB of growth until it exited. The macOS driver shrinks its parameter buffers (AGXParameterManagement::checkForShrink). It folds the page high-water into an average once per check, average = max((sample + 4 * average) / 5, sample). For each check in which the average stays below a quarter of the capacity it removes one 512 KiB block, and a buffer that has been idle for ten seconds returns to its minimum. Apply the same policy from the event worker, with a minimum that keeps what recent work used, so a shrink never makes the next pass at the recent load grow again: - once per second, sample the page high-water (sharing the sample with the growth check) and update the average; a period with a sample or a prepared pass also enters a window of the last ten such periods; - each second the average is below a quarter of the capacity earns 512 KiB of shrink; - a pair with no pass prepared or in flight for ten seconds may shrink to its floor; - the floor is the largest of the primary backing, the blocks of the last pass layout, and the size below which the automatic growth check would grow again for the largest sample in the window. No shrink goes below it. A shrink removes whole trailing growth extensions only, and only while the pair is idle and every block is back in the inventory ring. The kept blocks are laid out as a fresh heap is (ring entries and page list from the read cursor, state counts, both inventory cursors), and the PB descriptor row is reloaded, as an idle-host growth does. The removed blocks' client aliases leave the VM cache under the device mutex; the worker unmaps and frees them after it. A shrink is best effort and leaves the heap unchanged if anything fails. While some heap can still shrink, the worker wakes at least once per second. If the inventory of an idle heap does not hold every block, the heap is left as it is and the first such heaps are reported once each with their inventory cursors. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 272955b009fe844d4d06361365f166d75b6b451b) --- drivers/gpu/drm/asahi/g17/buffer.rs | 259 +++++++++++++++++++- drivers/gpu/drm/asahi/g17/queue/render.rs | 1 + drivers/gpu/drm/asahi/g17/runtime.rs | 24 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 14 ++ drivers/gpu/drm/asahi/g17/runtime/render.rs | 74 ++++++ drivers/gpu/drm/asahi/mmu/lifetime.rs | 34 +++ 6 files changed, 401 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index dd1aed70a2bf0..1f98ed7bf1fb6 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -333,6 +333,13 @@ pub(crate) struct TvbGrowth { /// Sparse 128-KiB data blocks at 160-KiB backing strides. Firmware owns their /// contents; CPU zeroing occurs only on fresh backing, never on pool reuse. +/// Aliases and backing of growth extensions a shrink removed from an idle +/// heap. Field order unmaps the aliases before freeing their backing. +pub(crate) struct TvbRelease { + _mappings: KVec, + _extensions: KVec, +} + pub(crate) struct Tvb { addresses: KVec, // Retain sparse-block backing until the pool and its aliases retire. @@ -394,6 +401,18 @@ impl Tvb { pub(crate) fn addresses(&self) -> &[u64] { &self.addresses } + /// Blocks of each growth extension; extensions are packed like the primary. + fn extension_blocks(extension: &KernelObject) -> usize { + extension.size() / cfg::TVB_BLOCK_SIZE + } + /// Blocks of the primary backing, which a heap never gives back. + fn primary_blocks(&self) -> usize { + self.extensions + .iter() + .fold(self.blocks(), |blocks, extension| { + blocks.saturating_sub(Self::extension_blocks(extension)) + }) + } pub(crate) fn prepare_growth( &mut self, @@ -536,8 +555,50 @@ pub(crate) struct Manager { scratch: u64, buffer_id: u8, generation: u64, + shrink: Shrink, +} + +/// Native tiler-heap shrink policy, after macOS +/// `AGXParameterManagement::checkForShrink` (AGXG17X 0x8bd5e20). Once per +/// period the page high-water folds into an average, +/// `max((sample + 4 * average) / 5, sample)`. Each period whose average +/// stays below a quarter of the capacity earns one 512 KiB block of shrink, +/// and a heap idle for ten seconds returns to its minimum. Only an idle heap +/// is rebuilt here, by whole growth extensions. +/// +/// Unlike macOS, the minimum keeps what the recent work needs: never below +/// the size at which the automatic growth check would grow the heap again +/// for the largest high-water sample of the last ten active periods. A +/// shrink then never makes the next pass at the recent load grow back. +const SHRINK_PERIOD_NS: u64 = 1_000_000_000; +const SHRINK_IDLE_NS: u64 = 10_000_000_000; +/// Blocks in one native 512 KiB parameter-buffer block. +const SHRINK_STEP_BLOCKS: usize = 0x8_0000 / cfg::TVB_BLOCK_SIZE; +/// Active periods in the recent-load window. +const SHRINK_WINDOW: usize = (SHRINK_IDLE_NS / SHRINK_PERIOD_NS) as usize; + +#[derive(Default)] +struct Shrink { + last_busy: u64, + period_start: u64, + /// High-water samples since the last period and the last growth check. + period_max: u32, + growth_max: u32, + average: u32, + credit: usize, + /// Blocks the most recent pass layout required. + required: usize, + /// High-water samples of the last active periods (with a sample or a + /// pass) and their largest value. Idle periods leave both unchanged. + window: [u32; SHRINK_WINDOW], + window_next: usize, + busy_peak: u32, + reported: bool, } +/// Bounds the one-line report of heaps that could not be rebuilt. +static SHRINK_REPORTS: AtomicU32 = AtomicU32::new(0); + pub(crate) struct Submitted<'a> { word: &'a AtomicU32, next: u32, @@ -599,6 +660,7 @@ impl Manager { scratch: args.scratch, buffer_id: args.buffer_id, generation: args.generation, + shrink: Shrink::default(), }) } pub(crate) fn buffer_id(&self) -> u8 { @@ -658,7 +720,9 @@ impl Manager { next: word.load(Ordering::Relaxed).wrapping_add(1), }) } - pub(crate) fn automatic_target(&self) -> Result> { + /// Reads the firmware page high-water and requests its reset. Both the + /// growth check and the shrink period see every sample. + fn sample_max_pages(&mut self) -> Result { let max_pages = self .graph .word(fw::CONTROL + core::mem::offset_of!(fw::Control, max_pages))?; @@ -669,8 +733,83 @@ impl Manager { let pages = max_pages.load(Ordering::Relaxed); fence(Ordering::Release); reset.store(1, Ordering::Relaxed); + self.shrink.period_max = self.shrink.period_max.max(pages); + self.shrink.growth_max = self.shrink.growth_max.max(pages); + Ok(pages) + } + pub(crate) fn automatic_target(&mut self) -> Result> { + self.sample_max_pages()?; + let pages = core::mem::take(&mut self.shrink.growth_max); Ok(automatic_growth_target(pages as usize, self.blocks())) } + /// Records the blocks a pass layout requires, the floor of any shrink. + /// A pass is being prepared at `now`; the heap counts as busy. + pub(crate) fn note_required(&mut self, blocks: usize, now: u64) { + self.shrink.required = blocks; + self.shrink.last_busy = now; + } + /// The smallest heap a shrink may leave: the primary backing, the blocks + /// of the last pass layout, and the size below which the automatic growth + /// check would grow again for the recent peak. + fn shrink_floor(&self) -> usize { + let recent = (self.shrink.busy_peak as usize) + .saturating_mul(2) + .div_ceil(cfg::TVB_PAGES_PER_BLOCK); + self.tvb.primary_blocks().max(self.shrink.required).max(recent) + } + /// Whether the heap holds growth beyond its primary backing. + pub(crate) fn grown(&self) -> bool { + !self.tvb.extensions.is_empty() + } + /// Whether the heap holds growth that a shrink could give back. + pub(crate) fn shrinkable(&self) -> bool { + self.tvb.extensions.last().is_some_and(|extension| { + self.blocks() - Tvb::extension_blocks(extension) >= self.shrink_floor() + }) + } + /// Advances the shrink policy at `now` and returns the block count an idle + /// heap should shrink to. `busy` is true while the pair has work in flight + /// or is borrowed; a busy heap only updates its average and recent peak. + /// Call it for every grown heap, so the recent peak can also fall. + pub(crate) fn shrink_target(&mut self, now: u64, busy: bool) -> Result> { + if busy || self.shrink.last_busy == 0 { + self.shrink.last_busy = now; + } + if self.shrink.period_start == 0 { + self.shrink.period_start = now; + } + if now.saturating_sub(self.shrink.period_start) >= SHRINK_PERIOD_NS { + self.sample_max_pages()?; + let sample = core::mem::take(&mut self.shrink.period_max); + let average = (u64::from(sample) + 4 * u64::from(self.shrink.average)) / 5; + self.shrink.average = (average as u32).max(sample); + if sample != 0 || self.shrink.last_busy >= self.shrink.period_start { + let next = self.shrink.window_next; + self.shrink.window[next] = sample; + self.shrink.window_next = (next + 1) % SHRINK_WINDOW; + self.shrink.busy_peak = self.shrink.window.iter().copied().max().unwrap_or(0); + } + self.shrink.period_start = now; + let capacity = self.blocks() * cfg::TVB_PAGES_PER_BLOCK; + if (self.shrink.average as usize).saturating_mul(4) < capacity { + self.shrink.credit = self.shrink.credit.saturating_add(SHRINK_STEP_BLOCKS); + } else { + self.shrink.credit = 0; + } + } + if !self.shrinkable() { + self.shrink.credit = 0; + return Ok(None); + } + if busy { + return Ok(None); + } + if now.saturating_sub(self.shrink.last_busy) >= SHRINK_IDLE_NS { + return Ok(Some(self.shrink_floor())); + } + let last = self.tvb.extensions.last().map_or(0, Tvb::extension_blocks); + Ok((last != 0 && self.shrink.credit >= last).then(|| self.blocks() - last)) + } fn cursors(&self) -> Result { Ok(Cursors { total: self.graph.word(fw::CONTROL)?.load(Ordering::Relaxed), @@ -872,6 +1011,124 @@ impl Manager { } } +impl Manager { + /// Rebuilds an idle heap with whole trailing growth extensions removed, + /// keeping at least `target` blocks. The kept blocks are laid out as + /// `Graph::init` lays out a fresh heap, starting at the firmware read + /// cursor. This runs only while every block is back in the inventory, as + /// an idle-host growth requires the complete snapshot unchanged. + /// + /// `truncate(blocks, kept, out)` moves the removed blocks' client aliases + /// out of the VM cache. Returns the reloaded PB row and the released + /// aliases and backing, which the caller frees after the device mutex. + /// None leaves the heap, the firmware records and the VM unchanged. + pub(crate) fn shrink( + &mut self, + target: usize, + descriptor: [u32; 4], + truncate: impl FnOnce(usize, usize, &mut KVec) -> Result, + ) -> Result> { + let blocks = self.blocks(); + let mut kept = blocks; + let mut dropped = 0; + for extension in self.tvb.extensions.iter().rev() { + let size = Tvb::extension_blocks(extension); + if kept - size < target.max(self.tvb.primary_blocks()) { + break; + } + kept -= size; + dropped += 1; + } + if dropped == 0 { + return Ok(None); + } + let before = self.cursors()?; + let retained = (before.total as usize + cfg::TVB_BLOCK_SLOTS - before.read as usize) + % cfg::TVB_BLOCK_SLOTS; + if !before.valid() || before.total != before.committed || retained != blocks { + if !self.shrink.reported && SHRINK_REPORTS.fetch_add(1, Ordering::Relaxed) < 8 { + self.shrink.reported = true; + pr_info!( + "G17: tiler heap {} kept {} blocks: inventory {}/{}/{} holds {}\n", + self.buffer_id, + blocks, + before.total, + before.committed, + before.read, + retained + ); + } + return Ok(None); + } + // Allocation failure only postpones the shrink to a later pass. + let (Ok(mappings), Ok(extensions)) = ( + KVec::with_capacity(blocks - kept, GFP_NOWAIT), + KVec::with_capacity(dropped, GFP_NOWAIT), + ) else { + return Ok(None); + }; + let mut release = TvbRelease { + _mappings: mappings, + _extensions: extensions, + }; + for &address in &self.tvb.addresses[..kept] { + fw::compact_page(address)? + .checked_add(cfg::TVB_PAGES_PER_BLOCK as u32 - 1) + .ok_or(EOVERFLOW)?; + } + let base = self.graph.pointer(0, fw::SIZE)?.cast::(); + self.graph.word(0)?; + // The callback moves aliases only after checking the cached owner, + // its block count and the spare capacity. + truncate(blocks, kept, &mut release._mappings)?; + + // From here every index, address and CPU view has been checked. + let write = |offset: usize, value: u32| { + // SAFETY: The full graph and base alignment were checked above. + // Every offset is an aligned field or a checked array index of + // Graph. No mutable reference aliases firmware-owned fields. + unsafe { AtomicU32::from_ptr(base.add(offset / 4)).store(value, Ordering::Relaxed) }; + }; + for (index, &address) in self.tvb.addresses[..kept].iter().enumerate() { + // The same pure conversion succeeded for every kept block above. + let Ok(first) = fw::compact_page(address) else { + continue; + }; + let slot = (before.read as usize + index) % cfg::TVB_BLOCK_SLOTS; + let at = fw::BLOCK_TABLE + slot * size_of::(); + write(at, first); + write(at + 4, 0); + for page in 0..cfg::TVB_PAGES_PER_BLOCK { + let entry = index * cfg::TVB_PAGES_PER_BLOCK + page; + write(fw::PAGE_LIST + entry * 4, first + page as u32); + } + } + let total = ((before.read as usize + kept) % cfg::TVB_BLOCK_SLOTS) as u32; + let pages = (kept * cfg::TVB_PAGES_PER_BLOCK) as u32; + write(core::mem::offset_of!(fw::State, pages), pages); + write(core::mem::offset_of!(fw::State, producer), total); + write(core::mem::offset_of!(fw::State, consumer), before.read); + write(core::mem::offset_of!(fw::State, last_page), pages - 1); + write(fw::CONTROL, total); + fence(Ordering::SeqCst); + write(fw::CONTROL + 4, total); + fence(Ordering::SeqCst); + + self.tvb.addresses.truncate(kept); + for _ in 0..dropped { + if let Some(extension) = self.tvb.extensions.pop() { + append_reserved(&mut release._extensions, extension); + } + } + self.shrink.credit = self.shrink.credit.saturating_sub(blocks - kept); + self.shrink.reported = false; + Ok(Some(( + fw::reload_descriptor(descriptor, self.page_list_client, pages), + release, + ))) + } +} + /// Caller reserves all needed entries before any firmware-visible mutation. fn append_reserved(values: &mut KVec, value: T) { let length = values.len(); diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index d82ee84c1283f..0bf477c0344fb 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -909,6 +909,7 @@ impl Pair { return Err(EINVAL); }; let required = target::Layout::new(pass, self.clusters)?.tvb_blocks; + self.memory.manager.note_required(required, crate::g17::now_ns()); if self.in_flight() { return Ok(None); } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 4c7aa6a442cc1..e4f19397e5f92 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -656,6 +656,8 @@ pub(super) struct DeferredBatch { vacated: KVVec, /// Backing of render pools vacated for evicted idle clients. render: KVVec, + /// Tiler-heap extensions removed from idle heaps, with their aliases. + tvb: KVVec, } impl DeferredBatch { const DEPTH: usize = if crate::hw::t8140::queues::RENDER_DEPTH as usize @@ -676,14 +678,17 @@ impl DeferredBatch { Self::with_capacity(2 * Self::DEPTH, 0) } fn with_capacity(capacity: usize, detached: usize) -> Result { + let render_slots = if detached != 0 { + crate::hw::t8140::queues::RENDER_SLOTS + } else { + 0 + }; Ok(Self { values: KVVec::with_capacity(capacity, GFP_KERNEL)?, bindings: KVVec::with_capacity(detached, GFP_KERNEL)?, vacated: KVVec::with_capacity(detached, GFP_KERNEL)?, - render: KVVec::with_capacity( - if detached != 0 { crate::hw::t8140::queues::RENDER_SLOTS } else { 0 }, - GFP_KERNEL, - )?, + render: KVVec::with_capacity(render_slots, GFP_KERNEL)?, + tvb: KVVec::with_capacity(render_slots, GFP_KERNEL)?, }) } /// Keeps the detached owner's unmapping and VM teardown out of the device @@ -702,6 +707,14 @@ impl DeferredBatch { drop(backing); } } + pub(super) fn tvb_room(&self) -> bool { + self.tvb.len() < self.tvb.capacity() + } + pub(super) fn defer_tvb(&mut self, release: super::buffer::TvbRelease) { + if let Err(release) = self.tvb.push_within_capacity(release) { + drop(release); + } + } pub(super) fn defer_vacated(&mut self, vacated: super::queue::compute::Vacated) { if let Err(vacated) = self.vacated.push_within_capacity(vacated) { drop(vacated); @@ -753,6 +766,9 @@ impl DeferredBatch { for backing in self.render.drain_all() { drop(backing); } + for release in self.tvb.drain_all() { + drop(release); + } } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 99f3caf66bebe..1d1f551377145 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -13,6 +13,9 @@ use crate::g17::{ }; use kernel::{prelude::*, sync::Arc}; +/// Wake interval while a tiler heap may still shrink: the native check period. +const SHRINK_WAKE_NS: u64 = 1_000_000_000; + struct Service<'a> { firmware: &'a mut Firmware, deferred: &'a mut DeferredBatch, @@ -387,10 +390,14 @@ impl crate::g17::Shared { firmware.reclaim_drained_renders(deferred)?; firmware.service_compute_exit(deferred)?; firmware.service_compute_pools(deferred)?; + let mut shrinking = false; if !firmware.recovery.pending() { firmware .queues .vacate_evicted_render_pools(&firmware.init, deferred)?; + shrinking = firmware + .queues + .shrink_render_heaps(&firmware.init, now, deferred); } { let mut reclaim = self.reclaim.lock(); @@ -414,6 +421,13 @@ impl crate::g17::Shared { if polling { self.queue_poll(); } + // A heap with growth to give back is re-evaluated at least once per + // shrink period, also while the device is otherwise idle. + let idle = if shrinking { + Some(idle.map_or(SHRINK_WAKE_NS, |delay| delay.min(SHRINK_WAKE_NS))) + } else { + idle + }; if let Some(delay) = idle { self.queue_idle(delay); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 71b21f98479a6..460889be64e2b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -647,6 +647,80 @@ impl Registry { || !self.render.retired.is_empty() || !self.teardown.is_empty() } + /// Applies the native tiler-heap shrink policy (`buffer::Manager:: + /// shrink_target`) to every grown heap at `now` and rebuilds the heaps of + /// idle pairs that are due, removing whole growth extensions. The removed + /// aliases and backing go to the worker's deferred batch, which frees them + /// after the device mutex. A shrink is best effort: any failure leaves the + /// heap as it was. Returns whether a heap still holds growth it could give + /// back, so the worker keeps a periodic wake. + pub(in crate::g17) fn shrink_render_heaps( + &mut self, + init: &crate::g17::initdata::InitData, + now: u64, + deferred: &mut super::DeferredBatch, + ) -> bool { + let mut pending = false; + for (slot, entry) in self.render.entries.iter_mut().enumerate() { + let Some(entry) = entry.as_mut() else { + continue; + }; + let borrowed = entry.constructing || entry.growing || entry.preparing.is_some(); + let Some(pair) = entry.pair.as_mut() else { + continue; + }; + if !pair.manager().grown() { + continue; + } + let busy = borrowed || !pair.idle_for_context(pair.context()).unwrap_or(false); + if let Ok(Some(target)) = pair.manager().shrink_target(now, busy) { + if deferred.tvb_room() { + if let Ok(slot) = u8::try_from(slot) { + // Ignoring the result keeps the heap unchanged on failure. + let _ = Self::shrink_pair(init, slot, pair, target, deferred); + } + } + } + pending |= pair.manager().shrinkable(); + } + pending + } + + fn shrink_pair( + init: &crate::g17::initdata::InitData, + slot: u8, + pair: &mut Pair, + target: usize, + deferred: &mut super::DeferredBatch, + ) -> Result { + use core::sync::atomic::{fence, Ordering}; + let offset = usize::from(pair.buffer_id()) * 16; + let table = init.pb_descriptor_table()?; + let words = [ + table.word(offset)?, + table.word(offset + 4)?, + table.word(offset + 8)?, + table.word(offset + 12)?, + ]; + fence(Ordering::Acquire); + let descriptor = words.each_ref().map(|word| word.load(Ordering::Relaxed)); + let vm = pair.context().vm().clone(); + let pool = pair.pool_id(); + let shrunk = pair + .manager() + .shrink(target, descriptor, |blocks, kept, out| { + vm.truncate_render_pool_mappings(slot, pool, blocks, kept, out) + })?; + if let Some((reloaded, release)) = shrunk { + for (word, value) in words.into_iter().zip(reloaded) { + word.store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + deferred.defer_tvb(release); + } + Ok(()) + } + /// Gives back the USC backing of render pools whose client VM pressure /// reclaim evicted (an idle VM) and whose final free-list release was /// consumed. The vacated backings go to the worker's deferred batch, which diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index ea3edaca6ef38..98006b9459c98 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -238,6 +238,40 @@ impl Vm { Ok(()) } + /// Moves the aliases of the last `blocks - kept` heap blocks of the exact + /// cached owner into `out`, which the caller drops after its locks. The + /// growth path appends one alias per added block, so these are the + /// blocks a shrink removed. Nothing moves unless all of them fit in `out`. + pub(crate) fn truncate_render_pool_mappings( + &self, + slot: u8, + id: u64, + blocks: usize, + kept: usize, + out: &mut KVec, + ) -> Result { + let count = blocks.checked_sub(kept).ok_or(EINVAL)?; + if id == 0 || count == 0 || out.capacity() - out.len() < count { + return Err(EINVAL); + } + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut driver = lifetime.driver.lock(); + let pool = Option::as_mut(&mut *driver) + .and_then(|driver| driver.render.get_mut(usize::from(slot))) + .and_then(Option::as_mut) + .filter(|pool| pool.id == id) + .ok_or(EINVAL)?; + if pool.blocks != blocks || pool.mappings.len() < count { + return Err(EBUSY); + } + for _ in 0..count { + let mapping = pool.mappings.pop().ok_or(EIO)?; + out.push_within_capacity(mapping).map_err(|_| EIO)?; + } + pool.blocks = kept; + Ok(()) + } + /// A delayed teardown must not remove a successor generation. Remove /// under the cache lock, then unmap after dropping that lock. pub(crate) fn clear_render_pool_mappings_if_owner(&self, slot: u8, id: u64) { From 3570a36aa0ccaeb5843be0d473eb08c982f3d86c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 04:50:48 -0400 Subject: [PATCH 1232/2680] drm/asahi: give back warm G17 compute pools after ten idle seconds A compute queue whose owner closed or exited stays installed for reuse, and up to four such queues (the warm cap) keep their 23 MiB USC private-memory pool populated for as long as no newer queue displaces them. On J700, the kernel's GEM after each game exit sat 54-78 MiB above its level before the game started, mostly in these warm pools. The macOS driver frees a client's USC pools with the client and also shrinks idle pools. Releasing every pool as soon as its owner is gone (a cap of zero) would make each new client re-arm fresh backing in its first submission, which costs about 2.3 ms on a cold start (measured at cap 0 on hardware). Keep the warm cap for bursts of short-lived clients, and additionally release a retained pool once it has been idle for ten seconds, the same period as the idle tiler-heap shrink. The event worker schedules its wake for the next expiry, and retries 100 ms later when a pass could not release every due pool. Queues without a release time stay bounded by the cap only, as before. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c01af7a9e73c789c6a9f98547585a7048927af30) --- drivers/gpu/drm/asahi/g17/runtime.rs | 66 +++++++++++++++------ drivers/gpu/drm/asahi/g17/runtime/events.rs | 7 ++- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index e4f19397e5f92..be22a0cab1321 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -176,30 +176,42 @@ impl Registry { queue.bind_owner(owner, context.clone()) } - /// Publishes releases for the oldest idle retained pools beyond `cap`, at - /// most four per pass. Returns the number published. + /// Delay before retrying pools a turn could not release. + const POOL_RETRY_NS: u64 = 100_000_000; + /// Publishes releases for the oldest idle retained pools beyond `cap` and + /// for every pool released at least `expiry` before `now`, at most four + /// per pass. Returns the number published and the delay until the next + /// retained pool expires or a retry is due. pub(super) fn vacate_idle_pools( &mut self, init: &super::initdata::InitData, cap: usize, - ) -> Result { + now: u64, + expiry: u64, + ) -> Result<(usize, Option)> { // Count first: event turns are frequent and the steady state has no excess. - let forced = self - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .filter(|queue| queue.vacate_candidate() && queue.needs_fresh_backing()) - .count(); - let count = self + let mut forced = 0; + let mut count = 0; + let mut next = None::; + for queue in self .compute .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) .filter(|queue| queue.vacate_candidate()) - .count(); + { + count += 1; + let released = queue.released_at(); + let age = now.saturating_sub(released); + if queue.needs_fresh_backing() || (released != 0 && age >= expiry) { + forced += 1; + } else if released != 0 { + let left = expiry - age; + next = Some(next.map_or(left, |next| next.min(left))); + } + } if count <= cap && forced == 0 { - return Ok(0); + return Ok((0, next)); } let mut candidates: KVec<(u64, u8)> = KVec::with_capacity(count, GFP_KERNEL)?; for queue in self @@ -213,6 +225,7 @@ impl Registry { let key = if queue.needs_fresh_backing() { 0 } else { queue.released_at() }; candidates.push((key, queue.qid()), GFP_KERNEL)?; } + // Recycled and expired pools are the oldest keys and sort first. candidates.sort_unstable(); let excess = candidates.len().saturating_sub(cap).max(forced); let mut published = 0; @@ -230,7 +243,11 @@ impl Registry { } published += 1; } - Ok(published) + if published < excess { + // Beyond this turn's bound, or not yet releasable: retry soon. + next = Some(next.map_or(Self::POOL_RETRY_NS, |next| next.min(Self::POOL_RETRY_NS))); + } + Ok((published, next)) } /// Frees the backing of every pool whose release the firmware consumed. @@ -1161,16 +1178,24 @@ impl super::Shared { /// Idle retained compute queues that keep their USC pool populated; older idle /// pools give their backing back. const COMPUTE_WARM_CAP: usize = 4; +/// A retained pool idle this long gives its backing back regardless of the +/// cap: a burst of short-lived clients still finds warm pools, but the pools +/// of clients that are gone do not stay resident. The macOS driver frees a +/// client's USC pools with the client and has an idle shrink; ten seconds +/// matches its idle parameter-buffer shrink. +const COMPUTE_WARM_EXPIRY_NS: u64 = 10_000_000_000; impl super::Firmware { /// Event-worker service step: witness consumed pool releases, then release the /// oldest idle retained pools beyond the warm cap. /// Detached owner bindings and vacated pool backing go to `deferred`, which /// the worker finishes after releasing the device mutex. + /// Returns the delay until a retained pool expires, for the worker's wake. pub(in crate::g17) fn service_compute_pools( &mut self, deferred: &mut DeferredBatch, - ) -> Result { + now: u64, + ) -> Result> { for queue in self .queues .compute @@ -1185,14 +1210,19 @@ impl super::Firmware { } self.queues.observe_pool_releases(&self.init, deferred)?; if self.render_control_backpressured() { - return Ok(()); + return Ok(None); } - let published = self.queues.vacate_idle_pools(&self.init, COMPUTE_WARM_CAP)?; + let (published, next) = self.queues.vacate_idle_pools( + &self.init, + COMPUTE_WARM_CAP, + now, + COMPUTE_WARM_EXPIRY_NS, + )?; if published != 0 { fence(Ordering::SeqCst); self.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; } - Ok(()) + Ok(next) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 1d1f551377145..a7ff3e4f0ecc3 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -389,7 +389,7 @@ impl crate::g17::Shared { let idle = firmware.idle_check(now, in_flight)?; firmware.reclaim_drained_renders(deferred)?; firmware.service_compute_exit(deferred)?; - firmware.service_compute_pools(deferred)?; + let pools = firmware.service_compute_pools(deferred, now)?; let mut shrinking = false; if !firmware.recovery.pending() { firmware @@ -428,6 +428,11 @@ impl crate::g17::Shared { } else { idle }; + // A retained compute pool gives its backing back when it expires. + let idle = match (idle, pools) { + (Some(delay), Some(expiry)) => Some(delay.min(expiry)), + (delay, expiry) => delay.or(expiry), + }; if let Some(delay) = idle { self.queue_idle(delay); } From 67c78321a2613e864d4e51d361bfe2f5ff238478 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 04:26:27 -0400 Subject: [PATCH 1233/2680] drm/asahi: keep compaction away from GEM pages held for the GPU MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every populated asahi GEM object keeps its shmem pages referenced: by the device SG table, a kernel vmap or a PFN CPU mapping. Those pages sit on the unevictable LRU and cannot migrate, but the compaction migrate scanner still isolates them (compact_unevictable_allowed) and every migration fails. In-world on J700, with about 5 GB of GEM held, background compaction isolated about 3,400 pages a second and 97 % of the migrations failed (pgmigrate_fail 3,203/s against pgmigrate_success 104/s). Mark the shmem mapping of a held object inaccessible. Compaction skips folios of inaccessible mappings before isolating them, and migration refuses them; the flag comes with unevictable, as mapping_set_inaccessible() sets it. Nothing reads or writes a GEM shmem file through its mapping. Kernel objects are marked when created; other objects when they take a device SG table. Releasing idle backing to reclaim, or purging, clears the mark, so released pages can be swapped and migrated again. [ryan: adapted to 7.2.10 — minimal; for DJ: 7.2 has no owned_sg_table(), so held_sg_table() wraps the borrowed sg_table(dev) and the mark is applied where the 7.2 mmu.rs and bind.rs take the device SG table (the populate step of map_in_range(), map_at() and bind_object(), prefetch_sgt_segments() and prepare_bind_object())] Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 2d7de7690cc0f9c8d3a74b08c33999593b720aac) --- drivers/gpu/drm/asahi/gem.rs | 52 +++++++++++++++++++++++++- drivers/gpu/drm/asahi/mmu.rs | 8 ++-- drivers/gpu/drm/asahi/mmu/bind.rs | 2 +- drivers/gpu/drm/asahi/mmu/residency.rs | 2 + 4 files changed, 58 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 39ab0c354fa26..7fd3ab9fc6c1c 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -16,7 +16,8 @@ use kernel::{ VMapOwned, // }, BaseObject, - DriverObject, // + DriverObject, + IntoGEMObject, // }, // }, error::Result, @@ -28,6 +29,7 @@ use kernel::{ use core::ops::Range; use core::sync::atomic::{ AtomicU64, + AtomicUsize, Ordering, // }; @@ -64,6 +66,53 @@ pub(crate) struct AsahiObject { /// Type alias for the shmem GEM object type for this driver. pub(crate) type Object = shmem::Object; +/// Marks whether the pages of `gem` are held for the device. +/// +/// Populated asahi objects keep their pages referenced (device SG table, kernel vmap or a PFN +/// CPU mapping), so the pages cannot migrate. Unmarked, the compaction migrate scanner still +/// isolates them from the unevictable LRU and every migration fails. The shmem mapping of a +/// held object is therefore marked inaccessible, which compaction and migration check and which +/// comes with unevictable, as `mapping_set_inaccessible()` sets it; nothing reads or writes a GEM +/// shmem file through its mapping. Releasing idle backing or purging clears the mark, so the +/// released pages can be swapped and migrated again. +pub(crate) fn set_mapping_held(gem: &Object, held: bool) { + let raw = gem.as_raw(); + // SAFETY: `gem` keeps the GEM object alive. A shmem-backed object's `filp` and its + // `f_mapping` are set at initialization and never change; imported objects have none. + let mapping = unsafe { + let filp = (*raw).filp; + if filp.is_null() { + return; + } + (*filp).f_mapping + }; + if mapping.is_null() { + return; + } + // SAFETY: `flags` is the aligned flag word of the live mapping. The kernel updates it only + // with atomic bit operations on the same word. + let flags = + unsafe { AtomicUsize::from_ptr(core::ptr::addr_of_mut!((*mapping).flags).cast::()) }; + let inaccessible = 1usize << kernel::bindings::mapping_flags_AS_INACCESSIBLE; + let unevictable = 1usize << kernel::bindings::mapping_flags_AS_UNEVICTABLE; + if held { + flags.fetch_or(inaccessible | unevictable, Ordering::Relaxed); + } else { + flags.fetch_and(!inaccessible, Ordering::Relaxed); + } +} + +/// Takes the SG table of `gem`, which holds its pages for the device, and marks the +/// object held (see [`set_mapping_held`]). +pub(crate) fn held_sg_table<'a>( + gem: &'a Object, + dev: &'a kernel::device::Device, +) -> Result<&'a kernel::scatterlist::SGTable> { + let sgt = gem.sg_table(dev)?; + set_mapping_held(gem, true); + Ok(sgt) +} + /// VM-private BOs share their assigned VM's reservation object. Validate this /// before acquiring mapping state; generic GPUVM accepts external shareable BOs. pub(crate) fn validate_vm_binding(gem: &Object, vm: &mmu::Vm) -> Result { @@ -194,6 +243,7 @@ fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> }, )?; + set_mapping_held(&gem, true); mod_pr_debug!("AsahiObject new kernel object id={}\n", gem.id); Ok(ObjectRef::new(gem)) } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 67a95fcf02773..d7ab2767d7171 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1595,7 +1595,7 @@ fn prefetch_sgt_segments( dev: &device::Device, gem: &gem::Object, ) -> Result> { - let sgt = gem.sg_table(dev)?; + let sgt = crate::gem::held_sg_table(gem, dev)?; // self.segments. let mut segments = KVVec::with_capacity(sgt.iter().size_hint().0, GFP_KERNEL)?; @@ -1846,7 +1846,7 @@ impl Vm { // TODO: hack, populate the GEM object sg table first { let dev = unsafe { self.dev.as_bound() }; - let _sgt = gem.sg_table(dev)?; + let _sgt = crate::gem::held_sg_table(gem, dev)?; } let vm_bo = self .gpuvm @@ -1927,7 +1927,7 @@ impl Vm { // TODO: hack, populate the GEM object sg table first { let dev = unsafe { self.dev.as_bound() }; - let _sgt = gem.sg_table(dev)?; + let _sgt = crate::gem::held_sg_table(gem, dev)?; } let vm_bo = self .gpuvm @@ -2008,7 +2008,7 @@ impl Vm { // TODO: hack, populate the GEM object sg table first { let dev = unsafe { self.dev.as_bound() }; - let _sgt = gem.sg_table(dev)?; + let _sgt = crate::gem::held_sg_table(gem, dev)?; } let vm_bo = self .gpuvm diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index 01163edf879de..aea632b336f57 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -80,7 +80,7 @@ impl Vm { // SAFETY: As in `Vm::bind_object()`, the device is bound while its file can issue binds. let dev = unsafe { self.dev.as_bound() }; // Populate the scatter-gather table before any GPUVM lock is taken. - gem.sg_table(dev)?; + crate::gem::held_sg_table(gem, dev)?; let vm_bo = self .gpuvm .obtain(gem, pin_init!(VmBoData { segments <- new_mutex!(None) }))?; diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index f912813b65d2b..5370b87ccf1b5 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -439,6 +439,7 @@ impl Vm { *segments = None; } if object.purge_locked().is_ok() { + crate::gem::set_mapping_held(object, false); purged = purged.saturating_add(object.size() / kernel::page::PAGE_SIZE); } else { gate.note_evicted(); @@ -517,6 +518,7 @@ impl Vm { // tables above. Kernel mappings and other SG owners are excluded; // the C/Rust backing helper rechecks exclusion before release. if object.release_idle_pages_locked().is_ok() { + crate::gem::set_mapping_held(object, false); unpinned = unpinned.saturating_add(object.size() / kernel::page::PAGE_SIZE); } Ok(unpinned < target) From d93b47d389053e4296bb6faf913d976202ea441e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 10 Oct 2026 00:18:43 -0400 Subject: [PATCH 1234/2680] drm/asahi: complete page-table stores before GPU TLB invalidation Use the existing DSB helper before outer-shareable TLB invalidations. A sequentially consistent Rust fence emits DMB and does not provide the completion required before the GPU discards its cached translations. Publish new page-table leaves even when no context currently names the root, so a later binding cannot expose incomplete table stores. Keep invalidation ranges, queue admission and submission batching intact. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 6dd0f73fd3b2bade5415b31346aadeb2f38d446e) --- drivers/gpu/drm/asahi/g17/kick.rs | 2 +- drivers/gpu/drm/asahi/mmu.rs | 36 ++++++++++++++++++------------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index f109c8261476e..3b78b9443a86d 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -308,7 +308,7 @@ impl Queue { if self.id.engine != DataMaster::Compute { return Err(EINVAL); } - fence(Ordering::SeqCst); + mem::sync(); for asid in 0..4 { mem::tlbi_range_or_asid(asid, self.low_va() as usize, KICK_RING_SIZE); mem::tlbi_range_or_asid(asid, self.firmware_va() as usize, KICK_RING_SIZE); diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index d7ab2767d7171..b36e5e4f73b64 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -593,7 +593,7 @@ impl gpuvm::DriverGpuVm for VmInner { } if let Some(asid) = self.slot() { - fence(Ordering::SeqCst); + mem::sync(); self.tlbi_range(asid as u8, unmap_start, unmap_range as usize); mod_dev_dbg!( self.dev, @@ -643,7 +643,7 @@ impl VmInner { } if let Some(asid) = self.slot() { - fence(Ordering::SeqCst); + mem::sync(); self.tlbi_range(asid as u8, va.addr(), va.length() as usize); mod_dev_dbg!( self.dev, @@ -668,7 +668,7 @@ impl VmInner { let result = self.page_table.discard_partial_map(addr..addr + size); if self.uat_inner.firmware == UatFirmware::Handoff { if let Some(asid) = self.slot() { - fence(Ordering::SeqCst); + mem::sync(); self.tlbi_range(asid as u8, addr, size as usize); mem::sync(); } @@ -741,6 +741,7 @@ impl VmInner { /// already invalidate. fn tlbi_contexts_naming_root(&self, iova: u64, size: usize) { if self.uat_inner.firmware == UatFirmware::Handoff { + mem::sync(); return; } @@ -749,11 +750,14 @@ impl VmInner { } fn tlbi_context_mask(&self, iova: u64, size: usize, contexts: u64) { + // Complete new leaves even if no context currently names this root: + // a later context can publish it without repeating the map operation. + // DMB orders accesses but does not complete stores before TLBI; the + // GPU uses outer-shareable invalidation and the barrier covers it. + mem::sync(); if contexts == 0 { return; } - - fence(Ordering::SeqCst); for ctx in 0..UAT_NUM_CTX { if contexts & (1 << ctx) != 0 { mem::tlbi_range_or_asid(ctx as u8, iova as usize, size); @@ -929,7 +933,7 @@ impl ExecutionRoot { owned || (low == 0 && high == 0) }; if release { - fence(Ordering::SeqCst); + mem::sync(); mem::tlbi_asid(self.id); mem::sync(); self.inner.lock().execution_contexts &= !(1u64 << id); @@ -1167,7 +1171,7 @@ impl KernelMapping { self.iova()..self.iova() + prefix as u64, PROT_GPU_FW_PRIV_RW, )?; - fence(Ordering::SeqCst); + mem::sync(); mem::tlbi_all(); mem::sync(); let mut after = [0; MAX_PAGES]; @@ -1217,7 +1221,7 @@ impl KernelMapping { self.size() ); } - fence(Ordering::SeqCst); + mem::sync(); // If we don't have (and have never had) a VM slot, just return let slot = match owner.slot() { @@ -1380,7 +1384,7 @@ impl Drop for KernelMapping { } if let Some(asid) = owner.slot() { - fence(Ordering::SeqCst); + mem::sync(); owner.tlbi_range(asid as u8, self.iova(), self.size()); mod_dev_dbg!( owner.dev, @@ -2318,6 +2322,7 @@ impl Drop for VmInner { self.id, idx ); + mem::sync(); mem::tlbi_asid(idx as u8); mem::sync(); } @@ -2399,7 +2404,7 @@ impl Uat { if contexts == 0 { return Err(ENOENT); } - fence(Ordering::SeqCst); + mem::sync(); for context in 0..UAT_NUM_CTX { if contexts & (1u64 << context) != 0 { mem::tlbi_asid(context as u8); @@ -2459,6 +2464,7 @@ impl Uat { core::mem::drop(uat_inner); // Make sure all TLB entries from the previous owner of this ASID are gone + mem::sync(); mem::tlbi_asid(idx as u8); mem::sync(); } @@ -2526,7 +2532,7 @@ impl Uat { roots.ttb0.store(low, Ordering::Release); roots.ttb1.store(high, Ordering::Release); } - fence(Ordering::SeqCst); + mem::sync(); mem::tlbi_asid(id as u8); mem::sync(); Ok(context) @@ -2573,7 +2579,7 @@ impl Uat { }) }?; - fence(Ordering::SeqCst); + mem::sync(); mem::tlbi_all(); mem::sync(); Ok(snapshot) @@ -2618,7 +2624,7 @@ impl Uat { }) }??; - fence(Ordering::SeqCst); + mem::sync(); mem::tlbi_all(); mem::sync(); Ok(()) @@ -2643,7 +2649,7 @@ impl Uat { .store(tagged_root(vm.ttb(), KERNEL_ALIAS_CTX), Ordering::Release); inner.handoff().unlock(); core::mem::drop(inner); - fence(Ordering::SeqCst); + mem::sync(); mem::tlbi_asid(KERNEL_ALIAS_CTX as u8); mem::sync(); Ok(()) @@ -2798,7 +2804,7 @@ impl Uat { impl Drop for Uat { fn drop(&mut self) { // Make sure we flush the TLBs - fence(Ordering::SeqCst); + mem::sync(); mem::tlbi_all(); mem::sync(); } From 2e3515c1d28a00d725198ff375ca838b95897ecd Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 10 Oct 2026 00:18:43 -0400 Subject: [PATCH 1235/2680] drm/asahi: remove tiler-shrink diagnostic logging Keep an incomplete idle inventory unchanged without a global report counter or per-heap diagnostic state. Correct the backing-layout comment and place it on Tvb rather than the deferred release container. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 019ad857ddfa08ed2473eae34db5b8b8d98b068e) --- drivers/gpu/drm/asahi/g17/buffer.rs | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index 1f98ed7bf1fb6..f648f8f7e0a92 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -331,8 +331,6 @@ pub(crate) struct TvbGrowth { extension: KernelObject, } -/// Sparse 128-KiB data blocks at 160-KiB backing strides. Firmware owns their -/// contents; CPU zeroing occurs only on fresh backing, never on pool reuse. /// Aliases and backing of growth extensions a shrink removed from an idle /// heap. Field order unmaps the aliases before freeing their backing. pub(crate) struct TvbRelease { @@ -340,6 +338,8 @@ pub(crate) struct TvbRelease { _extensions: KVec, } +/// Packed 128-KiB data blocks mapped at guarded 160-KiB virtual strides. +/// Firmware owns their contents; CPU zeroing occurs only on fresh backing. pub(crate) struct Tvb { addresses: KVec, // Retain sparse-block backing until the pool and its aliases retire. @@ -593,12 +593,8 @@ struct Shrink { window: [u32; SHRINK_WINDOW], window_next: usize, busy_peak: u32, - reported: bool, } -/// Bounds the one-line report of heaps that could not be rebuilt. -static SHRINK_REPORTS: AtomicU32 = AtomicU32::new(0); - pub(crate) struct Submitted<'a> { word: &'a AtomicU32, next: u32, @@ -1046,18 +1042,6 @@ impl Manager { let retained = (before.total as usize + cfg::TVB_BLOCK_SLOTS - before.read as usize) % cfg::TVB_BLOCK_SLOTS; if !before.valid() || before.total != before.committed || retained != blocks { - if !self.shrink.reported && SHRINK_REPORTS.fetch_add(1, Ordering::Relaxed) < 8 { - self.shrink.reported = true; - pr_info!( - "G17: tiler heap {} kept {} blocks: inventory {}/{}/{} holds {}\n", - self.buffer_id, - blocks, - before.total, - before.committed, - before.read, - retained - ); - } return Ok(None); } // Allocation failure only postpones the shrink to a later pass. @@ -1121,7 +1105,6 @@ impl Manager { } } self.shrink.credit = self.shrink.credit.saturating_sub(blocks - kept); - self.shrink.reported = false; Ok(Some(( fw::reload_descriptor(descriptor, self.page_list_client, pages), release, From 74e6a4bd2bc58c7e4f882a6c147c6f534697db96 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 10 Oct 2026 04:45:59 -0400 Subject: [PATCH 1236/2680] drm/asahi: reuse the render state mapping for auxiliary framebuffers Each render pair keeps sixteen generations, with a 32 KiB auxiliary framebuffer in each generation. Allocating a second alias from the fixed 4 MiB auxiliary arena exhausts that arena as logical contexts accumulate, returning ENOSPC and stale GL readback despite spare backing memory. Use the existing full state mapping and its auxiliary offset. The state owner already retains both the pages and their client-VM mapping for the required lifetime. This preserves queue depth and concurrent contexts. Tested on A18 Pro with 16 retained contexts, 128 immediate context recreations, targeted native GL/Zink rendering, native/x86-64/i686 compute oracles and MRT, and GL/GLES glmark2 scene validation. Unknown scene validations remain unknown; this is not a CTS conformance claim. Assisted-by: LLM Signed-off-by: Diljit Singh --- drivers/gpu/drm/asahi/g17/queue/render/memory.rs | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index fc5a05e9bf8a9..85319585b3d13 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -259,7 +259,6 @@ impl Compact { pub(super) struct Generation { deflake: mmu::KernelMapping, - auxiliary: mmu::KernelMapping, fragment_client: mmu::KernelMapping, state: KernelObject, fragment: KernelObject, @@ -285,15 +284,6 @@ impl Generation { mmu::PROT_GPU_FW_SHARED_RW, mmu::UAT_PGSZ, )?; - let auxiliary_range = cfg::AUXILIARY; - let auxiliary = state.map_range( - vm, - storage::AUXILIARY..storage::AUXILIARY + storage::AUXILIARY_SIZE, - auxiliary_range, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, - 0, - )?; let fragment = alloc.kernel( mmu::UAT_PGSZ, mmu::UAT_PGSZ as u64, @@ -311,7 +301,6 @@ impl Generation { KBox::new( Self { deflake, - auxiliary, fragment_client, state, fragment, @@ -349,7 +338,10 @@ impl Generation { self.state.gpu_va() + storage::TA_STATUS as u64, self.fragment_client.iova(), self.fragment.gpu_va(), - self.auxiliary.iova(), + // The retained state already maps these pages into the client VM. + // The auxiliary framebuffer accepts a full GPU address; reusing this + // view avoids exhausting the fixed 4 MiB alias arena across pairs. + self.state.gpu_va() + storage::AUXILIARY as u64, ] } } From 234886ff61630999fcd10f0658472a55deabd403 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 22:12:20 +1100 Subject: [PATCH 1237/2680] drm/asahi: describe the G17 tiler-heap shrink policy in hardware terms Rewrite the comment on the G17 tiler-heap shrink policy so it states the hardware and firmware behaviour directly. The edit keeps its line count; comments only, no functional change. [ryan: split from commit 9266541b8d35 ("treewide: describe hardware behaviour in comments"), keeping only its drivers/gpu/drm/asahi hunk; the other files belong to other topics] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g17/buffer.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index f648f8f7e0a92..d9b0c887deea0 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -558,8 +558,8 @@ pub(crate) struct Manager { shrink: Shrink, } -/// Native tiler-heap shrink policy, after macOS -/// `AGXParameterManagement::checkForShrink` (AGXG17X 0x8bd5e20). Once per +/// Native tiler-heap shrink policy, matching the heap behaviour seen +/// under macOS. Once per /// period the page high-water folds into an average, /// `max((sample + 4 * average) / 5, sample)`. Each period whose average /// stays below a quarter of the capacity earns one 512 KiB block of shrink, From 2735c59d28a4a02f7cdaf380a87d2e411447d589 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 19:02:56 +1100 Subject: [PATCH 1238/2680] rust: drm: gem: shmem: let drivers allocate the objects the core creates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The DRM core allocates some GEM objects itself. drm_gem_shmem_prime_import_sg_table() and drm_gem_shmem_dumb_create() both go through __drm_gem_shmem_create(), which allocates a bare struct drm_gem_shmem_object unless the driver provides gem_create_object. The shmem abstraction offers both helpers but no gem_create_object, while lookup_handle() and the GEM object callbacks take every object of the driver to be an Object. An imported dma-buf or a dumb buffer therefore reaches the driver as an Object whose Rust fields lie past the end of its allocation. Add an optional DriverObject::core_object_args() that gives the arguments for such objects. When a driver provides it, a gem_create_object callback allocates the whole Object and leaves the shmem part to the core, as the C shmem drivers do with their own gem_create_object. [ryan: adapted to 7.2.10 — the callback builds the 7.2 shmem Object, which also carries the set-once SG-table Devres and its lock; asahi-wip still leaves gem_create_object unset, so the bug is present there too] Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/kernel/drm/gem/mod.rs | 9 +++++++ rust/kernel/drm/gem/shmem.rs | 50 +++++++++++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index 8c978f5fd7abd..7eabfbfdb23db 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -118,6 +118,15 @@ pub trait DriverObject: Sync + Send + Sized + 'static { ) -> Result::Object>> { unimplemented!() } + + /// Arguments for an object that the DRM core allocates on the driver's behalf, such as a + /// dma-buf imported from another device or a dumb buffer. + /// + /// The shmem helpers build the driver data of such an object from these arguments. If the + /// core then fails to initialize the object, it frees it without dropping the driver data. + fn core_object_args(_dev: &drm::Device, _size: usize) -> Result { + build_error!(crate::error::VTABLE_DEFAULT_ERROR) + } } /// Trait that represents a GEM object subtype diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index fdf53dabc3cff..ee19618da5d40 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -366,6 +366,50 @@ impl Object { Ok(obj) } + /// `gem_create_object` callback, for the objects the DRM core allocates on the driver's + /// behalf (dma-buf imports and dumb buffers) before it initializes their shmem part. + /// + /// Every GEM object of the driver is taken to be an `Object`, these included, so build one + /// with the driver data from [`DriverObject::core_object_args`]. + extern "C" fn create_object_callback( + raw_dev: *mut bindings::drm_device, + size: usize, + ) -> *mut bindings::drm_gem_object { + // SAFETY: The DRM core only calls this for a device of `T::Driver`, which outlives the + // call. + let dev = unsafe { Device::::from_raw(raw_dev) }; + + let new = T::core_object_args(dev, size).and_then(|args| { + KBox::try_pin_init( + try_pin_init!(Self { + obj <- Opaque::init_zeroed(), + parent_resv_obj: None, + sgt_res: ManuallyDrop::new(SetOnce::new()), + sgt_lock <- new_mutex!(()), + sgt_access: AtomicUsize::new(0), + inner <- T::new(dev, size, args), + }), + GFP_KERNEL, + ) + }); + let new = match new { + Ok(new) => new, + Err(e) => return e.to_ptr(), + }; + + // SAFETY: We never move out of `self`. The DRM core owns the object from here on and + // releases it through `free_callback()`. An object the core then fails to initialize is + // freed with `kfree()`, which the `KBox` allocator allows, without dropping `inner`. + let new = KBox::into_raw(unsafe { Pin::into_inner_unchecked(new) }); + + // SAFETY: `new` is a valid object that nothing else references yet. + unsafe { + let obj = (*new).obj.get(); + (*obj).base.funcs = &Self::VTABLE; + &raw mut (*obj).base + } + } + fn acquire_sgt_access(&self, permanent: bool) -> Result { let mut state = self.sgt_access.load(Ordering::Acquire); loop { @@ -511,7 +555,11 @@ impl driver::AllocImpl for Object { type Driver = T::Driver; const ALLOC_OPS: driver::AllocOps = driver::AllocOps { - gem_create_object: None, + gem_create_object: if T::HAS_CORE_OBJECT_ARGS { + Some(Self::create_object_callback) + } else { + None + }, prime_handle_to_fd: None, prime_fd_to_handle: None, gem_prime_import: None, From 9132096da0d9d4e044ec86e741b15cee2795fd4c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 19:02:58 +1100 Subject: [PATCH 1239/2680] drm/asahi: allocate imported and dumb GEM objects as asahi objects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A dma-buf imported from another device comes back from drm_gem_shmem_prime_import_sg_table() as a bare 456-byte struct drm_gem_shmem_object. gnome-shell imports one at startup and binds it on every frame. ASAHI_VM_BIND then reads the object's flags and the SG lease counter behind its end, and updates the counter there. KASAN reports it on every boot: BUG: KASAN: slab-out-of-bounds in ...ASAHI_VM_BIND+0xb7c/0x1840 [asahi] Read of size 4 at addr ffff00002ac179f0 by task gnome-shell/1175 BUG: KASAN: slab-out-of-bounds in ...Object::owned_sg_table+0x48/0x11c [asahi] Write of size 8 at addr ffff00002ac179d0 by task gnome-shell/1175 ... The buggy address is located 40 bytes to the right of allocated 456-byte region [ffff00002ac17800, ffff00002ac179c8) A dumb buffer created on the card node has the same problem. Give the DRM core the arguments for the objects it allocates, so that imports and dumb buffers are allocated as whole asahi objects: shared, neither VM-private nor kernel-owned, like a GEM_CREATE object without flags. They now also get the driver's GEM callbacks, so closing the last handle in a file unbinds them from that file's VMs, as it does for every other object. [ryan: adapted to 7.2.10 — the callback takes an AsahiDrmDevice, the 7.2 name of the DRM device type] Fixes: 80df573af9ef ("rust: drm: gem: shmem: Add DRM shmem helper abstraction") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/gem.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 7fd3ab9fc6c1c..f0a3d919ce718 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -301,6 +301,16 @@ impl DriverObject for AsahiObject { }) } + /// Objects the DRM core allocates for us (dma-bufs imported from other devices and dumb + /// buffers) are shared user objects, like a GEM_CREATE object without flags. + fn core_object_args(_dev: &AsahiDrmDevice, _size: usize) -> Result { + Ok(AsahiObjConfig { + flags: 0, + exportable: true, + kernel: false, + }) + } + /// Callback to drop all mappings for a GEM object owned by a given `File` fn close(obj: &::Object, file: &drm::gem::DriverFile) { // fn close(obj: &Object, file: &DrmFile) { From 9a9da62f8347cb881deac895052fff3f717002bf Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Sun, 11 Oct 2026 04:06:15 -0400 Subject: [PATCH 1240/2680] drm/asahi: preserve TA prefix dependencies across render pairs A fragment-only dependency leaves TA with its physical ring parent. When covered renders migrate between pairs, that parent does not order their TA work. In the Neo Canvas Arcs reproducer, normal two-pair execution stalls, while retaining the cross-pair TA predecessor allows progress. Collect the actual TA QID and kick timestamp alongside the fragment frontier, including predecessors whose graphs are borrowed for preparation. Attach foreign TA predecessors to TA while keeping the fragment frontier on FRG. Same-pair TA dependencies remain implicit; independent prefixes, compute dependencies, pair admission and queue depth are unchanged. Neo validation with production Mesa: the clean four-file patch completed 60 seconds of the failing 18,000-arc workload, and 2,196 targeted CTS outcomes produced 1,026 Pass / 1,170 NotSupported / zero failures or GPU faults. Host checks cover borrowed/resident frontiers, nonadjacent QIDs, unequal and wrapped timestamps, failed producers and barrier capacity. Same-binary opt-in comparison completed pixel-checked resolve repetitions with overlapping timing ranges (median 6.203s off / 6.178s on); capped 2,000-arc browser runs retained their 60 Hz pacing. The tested module uses the deployed651 kernel/PRIME API. Its four changed dependency files and their pre-patch bases match this public tree exactly. The precise firmware resource cycle is not independently established. --- drivers/gpu/drm/asahi/g17/queue/render.rs | 19 ++++++++-- .../asahi/g17/queue/render/dependencies.rs | 36 +++++++++++++++++++ .../asahi/g17/runtime/render/dependencies.rs | 25 +++++++++++++ .../drm/asahi/g17/runtime/render/submit.rs | 13 +++++++ 4 files changed, 91 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 0bf477c0344fb..24a274e1af62a 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -43,6 +43,8 @@ use retirement::{Ticket, Tracker}; pub(crate) struct RenderDependencies { pub(crate) render: [KickDependency; 2], pub(crate) render_count: usize, + pub(crate) tiling: [KickDependency; 2], + pub(crate) tiling_count: usize, pub(crate) compute: Option, } @@ -50,6 +52,8 @@ impl RenderDependencies { pub(crate) const EMPTY: Self = Self { render: [KickDependency::ZERO; 2], render_count: 0, + tiling: [KickDependency::ZERO; 2], + tiling_count: 0, compute: None, }; pub(crate) fn render(&self) -> &[KickDependency] { @@ -403,8 +407,8 @@ impl Pair { /// Kick barriers of one render. The fragment kick always waits for its own tiling kick; /// with the fragment-barriers flag the resolved prefix orders the fragment kick too and the -/// tiling kick keeps only its implicit parent, otherwise the prefix orders the tiling kick. -/// A compute dependency always orders the tiling kick. +/// tiling kick preserves the TA prefix across physical pairs. Otherwise the render prefix +/// orders the tiling kick. A compute dependency always orders the tiling kick. struct RenderBarriers { tiling: [KickDependency; 3], tiling_count: usize, @@ -430,6 +434,17 @@ fn render_barriers( tiling_count += 1; } } + if fragment_stage { + // The implicit parent covers only this physical ring. Preserve the logical + // TA prefix when a predecessor was published on another pair, without + // making this TA kick wait for that predecessor's fragment completion. + for dependency in &dependencies.tiling[..dependencies.tiling_count] { + if dependency.qid() != own_tiling.qid() { + tiling_barriers[tiling_count] = *dependency; + tiling_count += 1; + } + } + } if let Some(compute) = dependencies.compute { tiling_barriers[tiling_count] = compute; tiling_count += 1; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs index be1d9fc9c902c..53a0fa6e449ed 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs @@ -15,6 +15,7 @@ use kernel::{prelude::*, sync::Arc}; struct Predecessor { packet: Arc, word: u64, + ta_word: u64, } /// A loan of the graph cannot hide work already published on that pair. Fence @@ -68,6 +69,21 @@ impl Dependencies { } Ok(()) } + pub(crate) fn collect_tiling( + &self, + owner: u64, + status: &Arc, + prefix: Option, + frontier: &mut Frontier, + ) -> Result { + if self.owner != Some(owner) { + return Ok(()); + } + for entry in self.entries.iter().flatten() { + include(frontier, status, prefix, &entry.packet, entry.ta_word)?; + } + Ok(()) + } } impl Pair { @@ -81,6 +97,8 @@ impl Pair { entries: core::array::from_fn(|_| { active.next().map(|entry| Predecessor { packet: entry.packet.clone(), + ta_word: KickDependency::new(self.qids()[0], entry.kicks[0]) + .map_or(u64::MAX, KickDependency::word), word: KickDependency::new(qid, entry.kicks[1]) .map_or(u64::MAX, KickDependency::word), }) @@ -106,4 +124,22 @@ impl Pair { } Ok(()) } + pub(crate) fn collect_tiling_dependencies( + &self, + owner: u64, + status: &Arc, + prefix: Option, + frontier: &mut Frontier, + ) -> Result { + if self.owner != Some(owner) { + return Ok(()); + } + let qid = self.qids()[0]; + for entry in self.active.iter() { + let word = + KickDependency::new(qid, entry.kicks[0]).map_or(u64::MAX, KickDependency::word); + include(frontier, status, prefix, &entry.packet, word)?; + } + Ok(()) + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs index 09e4322bf42ce..2b14b4548e0db 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs @@ -31,4 +31,29 @@ impl Registry { } Ok(frontier) } + pub(in crate::g17::runtime) fn tiling_prefix( + &self, + owner: u64, + packet: &Arc, + ) -> Result { + let mut frontier = Frontier::new(); + for entry in self.render.entries.iter().flatten() { + if let Some(pair) = entry.pair.as_ref() { + pair.collect_tiling_dependencies( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut frontier, + )?; + } else if let Some(dependencies) = entry.borrowed_dependencies.as_ref() { + dependencies.collect_tiling( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut frontier, + )?; + } + } + Ok(frontier) + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index dbf7175c5cd4b..13381b62da86c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -187,6 +187,19 @@ impl Registry { .ok_or(EOVERFLOW)? = *dependency; dependencies.render_count += 1; } + if dependencies.render_count != 0 + && matches!(&packet.command, crate::g17::command::Validated::Render { pass, .. } + if pass.fragment_barriers) + { + let tiling = self.tiling_prefix(owner, packet)?; + for dependency in tiling.as_slice() { + *dependencies + .tiling + .get_mut(dependencies.tiling_count) + .ok_or(EOVERFLOW)? = *dependency; + dependencies.tiling_count += 1; + } + } if let Some(queue) = self.compute(owner) { dependencies.compute = queue.dependency( owner, From 8e35a81643835cab5e7dcad3177eaa3e981546a2 Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Sun, 11 Oct 2026 19:48:24 -0400 Subject: [PATCH 1241/2680] drm/asahi: provision G17P ray traversal pools from Gen1 SCS demand Program the Gen1 scratch and intersection-table registers from validated compute requests. Publish the optional compute command tail in the UAPI and retain zero-tail compatibility for ordinary compute submissions. Include each enabled USC's SCS small-page, large-page and tail inventories in physical compute pool admission. Allocate the admitted capacity for new or vacant queues, reuse retained pools only when they fit, and grow owned queues after prior work drains and firmware consumes the free-list release. Keep old aliases and backing until the replacement is published. Preserve the pending capacity high-water mark across competing requests. This replaces the private fixed 64MiB RT pool experiment. Ordinary compute keeps its existing 21MiB floor, while the captured 304-byte Gen1 profile requires 41MiB on Neo's five enabled cores. Diagnostic logging and RT-unit control experiments are excluded. Validation: seven actual-source host capacity/lifecycle tests; matched651 module import/modinfo audit; Neo hardware 99 RT CTS passes (two unsupported), repeated 16384-ray dispatches at local 64/256 and four concurrent clients, with fourteen checked runtime outputs and no new captured kernel faults. --- drivers/gpu/drm/asahi/g17/command.rs | 22 ++- drivers/gpu/drm/asahi/g17/freelist.rs | 26 ++- drivers/gpu/drm/asahi/g17/fw/compute.rs | 187 +++++++++++++++++-- drivers/gpu/drm/asahi/g17/fw/compute_pool.rs | 25 +++ drivers/gpu/drm/asahi/g17/fw/initdata.rs | 6 +- drivers/gpu/drm/asahi/g17/fw/mod.rs | 1 + drivers/gpu/drm/asahi/g17/initdata.rs | 9 +- drivers/gpu/drm/asahi/g17/job.rs | 2 +- drivers/gpu/drm/asahi/g17/mod.rs | 3 + drivers/gpu/drm/asahi/g17/queue/compute.rs | 61 +++++- drivers/gpu/drm/asahi/g17/queue/mod.rs | 7 +- drivers/gpu/drm/asahi/g17/runtime.rs | 69 ++++++- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 4 +- include/uapi/drm/asahi_drm.h | 23 ++- 14 files changed, 388 insertions(+), 57 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/fw/compute_pool.rs diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 83191ddbcb1c3..dd3be1a81e1d8 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -3,7 +3,7 @@ //! Decode a submission and validate its user resources before publishing any work. use super::fw::{ - compute::ScratchRequest, + compute::{IntersectionTables, ScratchRequest}, kick::WriteRange, render::{RenderPass, UscProgram, ZlsSurface}, }; @@ -14,6 +14,7 @@ use kernel::{prelude::*, uapi}; const MAX_ATTACHMENTS: usize = crate::microseq::MAX_ATTACHMENTS; const USC_WINDOW_SIZE: u64 = 1 << 32; const MAX_SAMPLERS: u32 = 1024; +const COMPUTE_BASE_SIZE: usize = core::mem::offset_of!(uapi::drm_asahi_cmd_compute, prior_cdm) + size_of::(); const SAMPLER_SIZE: u64 = 8; const PAGE_SIZE: u64 = 0x4000; const ZLS_METADATA_STRIDE: u64 = 0x80; @@ -78,7 +79,7 @@ impl Attachments { #[allow(clippy::large_enum_variant)] pub(crate) enum Payload { Render(uapi::drm_asahi_cmd_render, [Attachments; 2]), - Compute(uapi::drm_asahi_cmd_compute, Attachments), + Compute(uapi::drm_asahi_cmd_compute, Attachments, IntersectionTables), } pub(crate) struct Command { @@ -130,7 +131,11 @@ impl<'a> Parser<'a> { Payload::Render(payload(bytes)?, [self.attachments[0], self.attachments[1]]) } else { self.counts[1] += 1; - Payload::Compute(payload(bytes)?, self.attachments[2]) + Payload::Compute( + payload(&bytes[..bytes.len().min(COMPUTE_BASE_SIZE)])?, + self.attachments[2], + IntersectionTables::parse(bytes.get(COMPUTE_BASE_SIZE..).unwrap_or(&[]))?, + ) }; return Ok(Some(Command { barriers, payload })); } @@ -283,6 +288,7 @@ pub(crate) enum Validated { timestamps: [uapi::drm_asahi_timestamps; 2], }, Compute { + intersection: IntersectionTables, scratch: ScratchRequest, usc_base: u64, cdm_va: u64, @@ -297,7 +303,7 @@ impl Payload { pub(crate) fn prior(&self) -> [u64; 2] { match self { Self::Render(cmd, _) => [cmd.prior_vdm, cmd.prior_cdm], - Self::Compute(cmd, _) => [cmd.prior_vdm, cmd.prior_cdm], + Self::Compute(cmd, _, _) => [cmd.prior_vdm, cmd.prior_cdm], } } @@ -415,7 +421,7 @@ impl Payload { timestamps: [cmd.ts_vtx, cmd.ts_frag], }) } - Self::Compute(cmd, attachments) => { + Self::Compute(cmd, attachments, intersection) => { if cmd.flags != 0 || cmd.cdm_ctrl_stream_base & 3 != 0 || cmd.cdm_ctrl_stream_end & 3 != 0 @@ -432,7 +438,13 @@ impl Payload { )?; sampler(space, cmd.sampler_heap, cmd.sampler_count)?; attachments.validate(space)?; + if intersection.enabled() { + require_range(space, intersection.pool, intersection.pool_bytes, Access::Read)?; + let code = intersection.default_program & 0x0000_ffff_ffff_ffff; + if code != 0 { require_range(space, code, 64, Access::Read)?; } + } Ok(Validated::Compute { + intersection: *intersection, scratch: ScratchRequest::new(cmd.scs_layout)?, usc_base: window.base, cdm_va: cmd.cdm_ctrl_stream_base, diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index c2c9470d10aa3..2612a9302ce28 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -8,7 +8,7 @@ //! reference and are released only after the owner's teardown is acknowledged. use super::fw::freelist::*; -use super::fw::initdata::{FreeListArgs, FreeListClass, FreeListControl, FREE_LIST_BLOCK_SIZE}; +use super::fw::initdata::{FreeListArgs, FreeListClass, FreeListControl, FREE_LIST_BLOCK_PAGES, FREE_LIST_BLOCK_SIZE}; use super::object::{Allocator, CpuMap, KernelObject}; use crate::mmu; use core::sync::atomic::{fence, AtomicBool, AtomicU64, Ordering}; @@ -90,9 +90,15 @@ impl Pages { layout, }) } - /// Fresh zeroed compute backing, built without the device mutex. - pub(crate) fn compute(alloc: &Allocator<'_>) -> Result { - Self::new(alloc, COMPUTE_BLOCKS, RunLayout::Compute) + /// Fresh compute backing at the admitted capacity, built off-lock. + pub(crate) fn compute_sized(alloc: &Allocator<'_>, blocks: usize) -> Result { + if blocks < COMPUTE_BLOCKS || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) { + return Err(EINVAL); + } + Self::new(alloc, blocks, RunLayout::Compute) + } + pub(crate) fn map_client(&self, vm: &mmu::Vm) -> Result { + self.backing.map_alias(vm, self.page_list_va(), mmu::PROT_GPU_SHARED_RW) } fn page_list_va(&self) -> u64 { self.backing.gpu_va() @@ -437,12 +443,16 @@ pub(crate) struct ComputePool { id: u16, } impl ComputePool { - pub(crate) fn new(alloc: &Allocator<'_>, qid: u8) -> Result { + pub(crate) fn fits(&self, blocks: usize) -> bool { + self.pages.as_ref().is_some_and(|pages| pages.blocks >= blocks) + } + + pub(crate) fn new(alloc: &Allocator<'_>, qid: u8, blocks: usize) -> Result { if qid > super::fw::kick::QID_MAX { return Err(EINVAL); } let id = u16::from(qid) + 1; - let pages = Pages::new(alloc, COMPUTE_BLOCKS, RunLayout::Compute)?; + let pages = Pages::compute_sized(alloc, blocks)?; let mut control = alloc.kernel( CONTROL_SIZE, mmu::UAT_PGSZ as u64, @@ -462,7 +472,7 @@ impl ComputePool { class: FreeListClass::One, page_list_va: pages.page_list_va(), run_list_va: pages.run_list_va(), - blocks: COMPUTE_BLOCKS as u32, + blocks: blocks as u32, state_va: state.gpu_va(), }), )?; @@ -517,7 +527,7 @@ impl ComputePool { class: FreeListClass::One, page_list_va: pages.page_list_va(), run_list_va: pages.run_list_va(), - blocks: COMPUTE_BLOCKS as u32, + blocks: pages.blocks as u32, state_va: self.state.gpu_va(), }); header.unk_54[..8].copy_from_slice(&bytes); diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 5b7489cfe93f4..a4775abc6b7eb 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -23,57 +23,114 @@ pub(crate) const COMPUTE_QOS_CLASS: u8 = 0x08; /// Event mask of compute kicks. pub(crate) const COMPUTE_KICK_EVENT_MASK: [u64; 4] = [0, 0, 4, 0]; -const COMPUTE_REGISTERS: usize = 40; +const COMPUTE_REGISTERS: usize = 41; const COMPUTE_REGISTERS_MAX: usize = COMPUTE_REGISTERS + 1; /// Validated per-command shader-context scratch request. +/// +/// The layout word is Apple's packed SCS layout: depth (bits 0:15), levels +/// (16:23), cliques (24:31), user (32:47) and driver (48:63). The region the +/// hardware needs is derived with G17P's Gen1 section model: fixed prefixes +/// totalling [`Self::FIXED_BYTES`], then per-thread stack, per-thread driver +/// and per-thread user bytes. Gen1 has no per-clique level payload. The scratch +/// registers the program writes are the native `generateRegisterList` +/// formulas use `com.apple.AGXG17P`; the +/// disabled program keeps the native disabled constants. #[derive(Clone, Copy)] pub(crate) struct ScratchRequest { pages: u16, + window: u64, + stack: u64, + driver: u64, + tail: u64, } impl ScratchRequest { + /// Native ScsLayoutGen1::getSectionEnd(4). const FIXED_BYTES: u64 = 0x58000; const THREADS_PER_CLIQUE: u64 = 32; const BYTES_PER_STACK_ENTRY: u64 = 8; const PAGE_SHIFT: u32 = 12; const PAGE_SIZE: u64 = 1 << Self::PAGE_SHIFT; const PAGES_PER_MIB: u64 = 1 << (20 - Self::PAGE_SHIFT); + /// Window word when enabled: bit 24 plus the clique count at 16:23. const WINDOW_ENABLED: u64 = 1 << 24; + /// Native disabled window word, [`UNK_10791`]. + const WINDOW_DISABLED: u64 = UNK_10791; + /// Native disabled stack word, [`UNK_1A0E9`]. + const STACK_DISABLED: u64 = UNK_1A0E9; + /// Native tail reservation when enabled. + const TAIL_ENABLED: u64 = 8; /// Payload-free layouts request no scratch even if depth/cliques are set. - /// Otherwise round the fixed and per-thread sections and add a guard page. + /// Otherwise derive the Gen1 end and round it up with the native two-page + /// guard; the per-register words are the native formulas, so they are + /// checked here against Apple's own values before they reach hardware. pub(crate) fn new(layout: u64) -> Result { let depth = layout & u16::MAX as u64; let cliques = (layout >> 24) & u8::MAX as u64; let user = (layout >> 32) & u16::MAX as u64; let driver = (layout >> 48) & u16::MAX as u64; if user == 0 && driver == 0 { - return Ok(Self { pages: 0 }); + return Ok(Self { + pages: 0, + window: Self::WINDOW_DISABLED, + stack: Self::STACK_DISABLED, + driver: 0, + tail: 0, + }); } let threads = cliques.checked_mul(Self::THREADS_PER_CLIQUE).ok_or(EINVAL)?; - let per_thread = depth.checked_mul(Self::BYTES_PER_STACK_ENTRY) - .and_then(|bytes| bytes.checked_add(user)) - .and_then(|bytes| bytes.checked_add(driver)).ok_or(EINVAL)?; - let bytes = threads.checked_mul(per_thread) - .and_then(|bytes| bytes.checked_add(Self::FIXED_BYTES)) + let stack_bytes = threads + .checked_mul(depth.checked_mul(Self::BYTES_PER_STACK_ENTRY).ok_or(EINVAL)?) + .ok_or(EINVAL)?; + let driver_bytes = threads.checked_mul(driver).ok_or(EINVAL)?; + let user_bytes = threads.checked_mul(user).ok_or(EINVAL)?; + let end10 = Self::FIXED_BYTES + .checked_add(stack_bytes) + .and_then(|bytes| bytes.checked_add(driver_bytes)) + .ok_or(EINVAL)?; + let end = end10 + .checked_add(user_bytes) .and_then(|bytes| bytes.checked_add(2 * Self::PAGE_SIZE - 1)) .ok_or(EINVAL)?; - let pages = bytes >> Self::PAGE_SHIFT; - Ok(Self { pages: u16::try_from(pages).map_err(|_| EINVAL)? }) + let pages = end >> Self::PAGE_SHIFT; + Ok(Self { + pages: u16::try_from(pages).map_err(|_| EINVAL)?, + window: Self::WINDOW_ENABLED | (cliques << 16), + stack: depth & !7, + // G17P's native register list does not publish the Gen2 level + // region register. Preserve the historical zero slot. + driver: 0, + tail: Self::TAIL_ENABLED, + }) } pub(crate) fn enabled(self) -> bool { self.pages != 0 } + pub(crate) fn pages(self) -> u16 { self.pages } + fn size_word(self) -> u64 { let pages = u64::from(self.pages); pages | (pages.div_ceil(Self::PAGES_PER_MIB) << 16) } fn window_word(self) -> u64 { - if self.enabled() { Self::WINDOW_ENABLED } else { UNK_10791 } + self.window + } + + fn stack_word(self) -> u64 { + self.stack + } + + fn driver_word(self) -> u64 { + self.driver + } + + fn tail_word(self) -> u64 { + self.tail } fn checkpoints(self) -> [u16; 4] { @@ -106,10 +163,94 @@ const OPERAND_STATE_0A5C9: u64 = 0x9000; /// Checkpoints into the main register array, as entry counts: after the /// first 0x0a5c9 write, after the final 0x1a440 write (twice), and the full /// count. -const CHECKPOINTS: [u16; 4] = [38, 39, 39, COMPUTE_REGISTERS as u16]; +const CHECKPOINTS: [u16; 4] = [39, 40, 40, COMPUTE_REGISTERS as u16]; + +#[derive(Clone, Copy)] +pub(crate) struct IntersectionTables { + pub(crate) pool: u64, + pub(crate) pool_bytes: u64, + auxiliary: u32, + first: u32, + count: u32, + pub(crate) default_program: u64, + default_control: u64, +} + +impl IntersectionTables { + pub(crate) const NONE: Self = Self { + pool: 0, pool_bytes: 0, auxiliary: 0, first: 0, count: 0, + default_program: 0, default_control: 0, + }; + + pub(crate) fn new(pool: u64, pool_bytes: u64, first: u32, count: u32, + default_program: u64, default_control: u64) -> Result { + if pool == 0 { + if pool_bytes != 0 || first != 0 || count != 0 + || default_program != 0 || default_control != 0 { + return Err(EINVAL); + } + return Ok(Self::NONE); + } + // G17P HAL200 Gen1 publishes a byte-sized table length (25 bits), + // with a plain aligned address. Unlike G17X HAL300, its address has + // no bit 51 and its two pool registers use fixed native values. + // Keep the shared UAPI's 128-byte blocks and 16 KiB allocation units. + let end = pool.checked_add(pool_bytes).ok_or(EINVAL)?; + let blocks_end = u64::from(first) + u64::from(count); + let code = default_program & 0x0000_ffff_ffff_ffff; + if pool & 0x3fff != 0 || pool_bytes == 0 || pool_bytes & 0x3fff != 0 + || end > (1u64 << 48) || (pool_bytes >> 14) >= (1u64 << 25) + || count == 0 || count >= (1 << 18) || blocks_end * 128 > pool_bytes + || code & 63 != 0 + || default_control & !0x0000_7fff_0000_3fff != 0 + || (code == 0 && (default_program != 0 || default_control != 0)) + || (code != 0 && default_control & (1u64 << 40) == 0) { + return Err(EINVAL); + } + Ok(Self { pool, pool_bytes, auxiliary: 0, first, count, default_program, default_control }) + } + + pub(crate) fn parse(bytes: &[u8]) -> Result { + if bytes.get(48..).is_some_and(|tail| tail.iter().any(|&b| b != 0)) { + return Err(EINVAL); + } + let mut tail = [0u8; 48]; + let count = bytes.len().min(tail.len()); + tail[..count].copy_from_slice(&bytes[..count]); + let auxiliary = u32::from_le_bytes(tail[..4].try_into().unwrap()); + if tail[4..8].iter().any(|&b| b != 0) + || (auxiliary != 0 && !(8..=32).contains(&auxiliary)) { + return Err(EINVAL); + } + let mut tables = Self::new( + u64::from_le_bytes(tail[8..16].try_into().unwrap()), + u64::from_le_bytes(tail[16..24].try_into().unwrap()), + u32::from_le_bytes(tail[24..28].try_into().unwrap()), + u32::from_le_bytes(tail[28..32].try_into().unwrap()), + u64::from_le_bytes(tail[32..40].try_into().unwrap()), + u64::from_le_bytes(tail[40..48].try_into().unwrap()), + )?; + if auxiliary != 0 && !tables.enabled() { return Err(EINVAL); } + tables.auxiliary = auxiliary; + Ok(tables) + } + + pub(crate) fn enabled(self) -> bool { self.pool != 0 } + + pub(crate) fn words(self) -> [u64; 6] { + if !self.enabled() { return [0; 6]; } + // AGXMetalG17P 26A428 endComputePass, both command encodings; + // pinned AGXG17P processComputeSetup copies these six words as-is. + [self.pool + u64::from(self.first) * 128, + 0xc0000000 | (u64::from(self.count) << 7), 0x6f00000000, + 0x3feff, self.default_program, self.default_control] + } +} + /// Inputs of one compute descriptor. pub(crate) struct ComputeArgs { + pub(crate) intersection: IntersectionTables, /// Scratch request validated with the userspace command. pub(crate) scratch: ScratchRequest, /// Base of the queue's USC window. @@ -166,6 +307,7 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS let operand = args.operand_state_va; let key = work_key(args.context_id, args.work_state); let ids = OBJECT_IDS.word(); + let intersection = args.intersection.words(); Ok([ RegisterWrite::new(0x017d9, 1), @@ -183,15 +325,20 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS RegisterWrite::new(0x1a440, UNK_1A440), RegisterWrite::new(0x1a458, UNK_1A458), RegisterWrite::new(0x101d9, UNK_101D9), - RegisterWrite::new(0x1a089, 0), - RegisterWrite::new(0x1a091, 0), - RegisterWrite::new(0x1a059, 0), - RegisterWrite::new(0x1a061, 0), - RegisterWrite::new(0x1a0b9, 0), - RegisterWrite::new(0x1a0c1, 0), + RegisterWrite::new(0x1a089, intersection[0]), + RegisterWrite::new(0x1a091, intersection[1]), + RegisterWrite::new(0x1a059, intersection[2]), + RegisterWrite::new(0x1a061, intersection[3]), + RegisterWrite::new(0x1a0b9, intersection[4]), + RegisterWrite::new(0x1a0c1, intersection[5]), RegisterWrite::new(0x101d1, args.scratch.size_word()), - RegisterWrite::new(0x0d479, 0), - RegisterWrite::new(0x1a0e9, UNK_1A0E9), + RegisterWrite::new(0x0d479, if args.intersection.auxiliary == 0 { + args.scratch.tail_word() + } else { + u64::from(args.intersection.auxiliary) + }), + RegisterWrite::new(0x1a0e9, args.scratch.stack_word()), + RegisterWrite::new(0x1a141, args.scratch.driver_word()), RegisterWrite::new(0x107a1, args.scratch.window_word()), RegisterWrite::new(0x0a599, UNK_0A599), RegisterWrite::new(0x0d411, UNK_0D411), diff --git a/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs b/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs new file mode 100644 index 0000000000000..4005daa4e9832 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! G17P compute pool capacity for a validated Gen1 SCS request. + +use kernel::prelude::*; +use super::initdata::{FREE_LIST_BLOCK_PAGES, FREE_LIST_BLOCK_SIZE}; +use super::freelist::{COMPUTE_BLOCKS, PAGE_LIST_SIZE}; + +pub(crate) fn required_blocks(active_cores: u32, scs_pages: u16) -> Result { + if active_cores == 0 { return Err(EINVAL); } + if scs_pages == 0 { return Ok(COMPUTE_BLOCKS); } + // Register0x101d1 counts 4-KiB pages and their 1-MiB large-page inventory. + // Register0x0d479 reserves eight further pages for each enabled USC. + let small = u64::from(scs_pages); + let per_core = small.checked_add(small.div_ceil(256)) + .and_then(|pages| pages.checked_add(8)).ok_or(EOVERFLOW)?; + let extra = per_core.checked_mul(u64::from(active_cores)) + .and_then(|pages| pages.checked_mul(4096)).ok_or(EOVERFLOW)?; + let bytes = extra.checked_add((COMPUTE_BLOCKS * FREE_LIST_BLOCK_SIZE) as u64) + .ok_or(EOVERFLOW)?; + let blocks = usize::try_from(bytes.div_ceil(FREE_LIST_BLOCK_SIZE as u64)) + .map_err(|_| EOVERFLOW)?; + let maximum = PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8); + if blocks > maximum { return Err(E2BIG); } + Ok(blocks) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index 94b03ebe284db..1f25bd0fc1c0d 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -2239,7 +2239,11 @@ static_assert!(size_of::() == 0x20); impl PagePoolDescriptor { /// Complete row for a physical compute queue's independent 21-block pool. pub(crate) const fn compute(page_list_va: u64) -> Self { - let pages = 21 * FREE_LIST_BLOCK_PAGES as u64; + Self::compute_blocks(page_list_va, 21) + } + + pub(crate) const fn compute_blocks(page_list_va: u64, blocks: usize) -> Self { + let pages = blocks as u64 * FREE_LIST_BLOCK_PAGES as u64; Self::new(page_list_va, pages << PAGE_POOL_FREE_PAGES_SHIFT, pages) } diff --git a/drivers/gpu/drm/asahi/g17/fw/mod.rs b/drivers/gpu/drm/asahi/g17/fw/mod.rs index 4f0ae8aa253ed..479bc91d21ca5 100644 --- a/drivers/gpu/drm/asahi/g17/fw/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/mod.rs @@ -14,6 +14,7 @@ use kernel::prelude::*; pub(crate) mod buffer; pub(crate) mod channels; pub(crate) mod compute; +pub(crate) mod compute_pool; pub(crate) mod context; pub(crate) mod freelist; pub(crate) mod initdata; diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index f2ec60633631f..d0dcc85c7f09e 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -285,11 +285,12 @@ impl InitData { /// Initialize a fresh physical compute QID's row before exposing its queue. An installed /// or rebound queue keeps the row's firmware-owned counters and must never call this again. - pub(super) fn initialize_compute_pool(&self, slot: u16, page_list_va: u64) -> Result { + /// Called for a fresh row or after its free-list release was consumed. + pub(super) fn initialize_compute_pool_blocks(&self, slot: u16, page_list_va: u64, blocks: usize) -> Result { if slot == 0 || slot > u16::from(super::fw::kick::QID_MAX) + 1 { return Err(EINVAL); } - let entry = abi::PagePoolDescriptor::compute(page_list_va); + let entry = abi::PagePoolDescriptor::compute_blocks(page_list_va, blocks); let size = core::mem::size_of::(); let offset = usize::from(slot) * size; let object = self.object(self.page_pool)?; @@ -299,8 +300,8 @@ impl InitData { object.dword(offset + core::mem::offset_of!(abi::PagePoolDescriptor, counters))?; let count = object.dword(offset + core::mem::offset_of!(abi::PagePoolDescriptor, page_count))?; - // SAFETY: the whole fresh row was bounds checked, the QID has never been published, - // and the device mutex excludes another host publisher. No other row is touched. + // SAFETY: the whole fresh row was bounds checked, this is a fresh QID or its release + // was consumed, and the device mutex excludes another host publisher. No other row is touched. unsafe { pointer.write_bytes(0, size); } diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index bd22c5d5ff46c..05133b656ee59 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -238,7 +238,7 @@ pub(crate) trait Backend: Send + Sync { } /// Lazily create or rebind this logical queue's physical compute graph. - fn ensure_compute(&self) -> Result; + fn ensure_compute(&self, scs_pages: u16) -> Result; /// Settle outstanding work of a terminal VM before an ioctl error is published. fn fail_vm(&self, error: Error); /// Called after both scheduler entities have cancelled their queued jobs. diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 380eba6a76c33..a469d0eb801d3 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -607,6 +607,7 @@ struct Shared { #[pin] drm: Mutex>, clusters: u32, + active_cores: u32, descriptor_flags: [u32; 2], crashed: AtomicBool, feed: Arc, @@ -856,11 +857,13 @@ impl Gpu { let preparations = preparation::Gate::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; + let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); let shared = Arc::pin_init( pin_init!(Shared { drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), dev: dev.as_ref().as_ref().into(), clusters: id.num_clusters, + active_cores, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), feed, diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 10b66c9ae811b..cae576fe183af 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -416,8 +416,8 @@ pub(crate) struct PoolBacking { } impl PoolBacking { - pub(crate) fn new(alloc: &Allocator<'_>, id: kick::Id) -> Result { - let pool = ComputePool::new(alloc, id.qid())?; + pub(crate) fn new(alloc: &Allocator<'_>, id: kick::Id, blocks: usize) -> Result { + let pool = ComputePool::new(alloc, id.qid(), blocks)?; Ok(Self { pool, id }) } @@ -459,6 +459,8 @@ pub(crate) struct Queue { /// the installed queue record still names them until the next owner binds. retained_owner: Option, pool_phase: PoolPhase, + growing: bool, + growth_blocks: usize, /// A killed queue was recycled: its pool/ring must be vacated and re-armed /// fresh before any owner binds it (set by `recycle_killed`, cleared by `rearm`). needs_fresh_backing: bool, @@ -518,6 +520,8 @@ impl Queue { owner: Some(owner), retained_owner: None, pool_phase: PoolPhase::Populated, + growing: false, + growth_blocks: 0, needs_fresh_backing: false, released_at: 0, free_list_generation: 0, @@ -1042,6 +1046,7 @@ impl Queue { Ok(true) } pub(crate) fn reusable(&self) -> bool { + if self.growing { return false; } self.installed // A pool release in flight must be witnessed before any successor. && !matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) @@ -1238,6 +1243,7 @@ impl Queue { consumer: u32, producer: u32, ) -> Result> { + if self.growing { return Ok(None); } let PoolPhase::ReleaseSent(cursor) = self.pool_phase else { return Ok(None); }; @@ -1274,6 +1280,55 @@ impl Queue { self.needs_fresh_backing = false; Ok((self.pool.id(), self.pool.page_list_va()?)) } + pub(crate) fn pool_fits(&self, required: usize) -> bool { + !self.growing && self.pool_phase == PoolPhase::Populated && self.pool.fits(required) + } + + /// Only a capacity increase waits for this queue's previously published work. + /// Other queues and ordinary submissions keep their existing concurrency. + pub(crate) fn begin_growth(&mut self, init: &crate::g17::initdata::InitData, blocks: usize) -> Result { + if self.growing { + if blocks < self.growth_blocks { return Err(EAGAIN); } + self.growth_blocks = blocks; + return Ok(false); + } + if self.quarantined || self.released || self.exit.is_some() + || self.active.len != 0 || self.replays.len != 0 || self.previous.is_some() + || (self.installed && !self.retirement_ready) + || self.pool_phase != PoolPhase::Populated { return Err(EAGAIN); } + if self.installed { + let record = ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); + let cursor = Rings::publish_control(init, &record)?; + self.pool_phase = PoolPhase::ReleaseSent(cursor); + } else { self.pool_phase = PoolPhase::Vacant; } + self.growing = true; + self.growth_blocks = blocks; + Ok(self.installed) + } + + pub(crate) fn finish_growth(&mut self, init: &crate::g17::initdata::InitData, + pages: &mut Option, + alias: &mut Option, blocks: usize) -> Result { + if !self.growing || blocks < self.growth_blocks { return Err(EAGAIN); } + if let PoolPhase::ReleaseSent(cursor) = self.pool_phase { + let consumer = init.control_consumer()?.load(Ordering::Acquire); + let producer = init.control_producer()?.load(Ordering::Acquire); + if !crate::g17::freelist::control_consumed(consumer, producer, cursor)? { + return Err(EAGAIN); + } + } + let binding = self.binding.as_mut().ok_or(EFAULT)?; + let old_alias = binding.pool_alias.take(); + let old_pages = self.pool.take_pages(); + self.pool.rearm(pages.take().ok_or(EINVAL)?)?; + init.initialize_compute_pool_blocks(self.pool.id(), self.pool.page_list_va()?, blocks)?; + binding.pool_alias = alias.take(); + self.pool_phase = PoolPhase::Populated; + self.growing = false; + self.growth_blocks = 0; + Ok(Vacated { _alias: old_alias, _pages: old_pages, _descriptors: None, _ring: None }) + } + /// Exact host identity remains valid even when several commands share an ioctl ID. pub(crate) fn owns_packet(&self, packet: &Arc) -> bool { self.active @@ -1506,6 +1561,7 @@ impl Queue { }; let (epoch, addresses) = self.publication_preflight(&packet, host)?; let crate::g17::command::Validated::Compute { + intersection, scratch, usc_base, cdm_va, @@ -1540,6 +1596,7 @@ impl Queue { self.graph.ensure_record_slot(slot)?; } let args = ComputeArgs { + intersection: *intersection, scratch: *scratch, usc_base: *usc_base, cdm_va: *cdm_va, diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 47b6cb18a8271..4ba6b33cb855a 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -127,9 +127,9 @@ impl Queue { }) } - fn ensure_compute(&mut self) -> Result { + fn ensure_compute(&mut self, scs_pages: u16) -> Result { // Admission is checked even when this queue already owns a scheduler. - self.backend.ensure_compute()?; + self.backend.ensure_compute(scs_pages)?; if self.compute.is_none() { self.compute = Some(Scheduler::new( &self.dev, @@ -174,7 +174,8 @@ impl Queue { self.prune_frontier(index); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { - self.ensure_compute()?; + let Validated::Compute { scratch, .. } = &command else { return Err(EINVAL); }; + self.ensure_compute(scratch.pages())?; } // Only earlier render ordinals covered by the explicit logical prefix // can delay host publication. These links never become GPU dependencies. diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index be22a0cab1321..ecfa57cd9a5a1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -121,10 +121,10 @@ impl Registry { /// replacement context must succeed before its old binding is displaced. /// A vacant queue (USC backing given back) is offered for re-arming only /// when no populated idle queue exists. - pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc) -> Result { + pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: usize) -> Result { if let Some(queue) = self.compute(owner) { return Ok(if queue.matches(owner, queue.qid(), context)? { - Rebind::Done + if queue.pool_fits(required) { Rebind::Done } else { Rebind::Grow } } else { Rebind::Fresh }); @@ -141,6 +141,7 @@ impl Registry { vacant.get_or_insert(queue.qid()); continue; } + if !queue.pool_fits(required) { continue; } queue.bind_owner(owner, context.clone())?; return Ok(Rebind::Done); } @@ -365,6 +366,7 @@ impl Registry { pub(super) enum Rebind { /// The owner now holds an installed, populated queue. Done, + Grow, /// Only a vacant queue is available; re-arm `qid` with fresh backing. Rearm(u8), /// No retained queue suits the owner: reserve a fresh QID. @@ -549,7 +551,52 @@ impl Backend { result } - fn build_compute(&self) -> Result { + fn grow_compute(&self, required: usize) -> Result { + use kernel::time::{delay::fsleep, Delta}; + { + let state = self.shared.state.lock(); + let firmware = state.as_ref().ok_or(ENODEV)?; + if firmware.queues.compute(self.owner).is_some_and(|queue| queue.pool_fits(required)) { + return Ok(()); + } + } + let lease = self.shared.preparations.enter(false)?; + let dev = self.shared.drm()?; + let uat = self.shared.state.lock().as_ref().ok_or(ENODEV)?.uat.clone(); + let alloc = super::object::Allocator { dev: &dev, uat: &uat }; + let mut pages = Some(super::freelist::Pages::compute_sized(&alloc, required)?); + let mut alias = Some(pages.as_ref().ok_or(EIO)?.map_client(self.context.vm())?); + let mut deferred = DeferredBatch::publication()?; + let started = now_ns(); + loop { + let result = (|| { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { return Err(EAGAIN); } + if self.shared.crashed.load(Ordering::Acquire) { return Err(EIO); } + firmware.scan_compute(None, &mut deferred)?; + let queue = firmware.queues.compute_mut(self.owner).ok_or(EIO)?; + if queue.pool_fits(required) { return Ok(None); } + if queue.begin_growth(&firmware.init, required)? { + fence(Ordering::SeqCst); + firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + } + queue.finish_growth(&firmware.init, &mut pages, &mut alias, required).map(Some) + })(); + deferred.finish(); + match result { + Ok(old) => { drop(old); return Ok(()); } + Err(error) if error == EAGAIN && lease.is_current() + && now_ns().saturating_sub(started) < WORK_RING_WAIT_NS => { + self.shared.queue_events(); + fsleep(Delta::from_millis(1)); + } + Err(error) => return Err(error), + } + } + } + + fn build_compute(&self, required: usize) -> Result { loop { match self.shared_compute() { Err(error) if error == EAGAIN => continue, @@ -572,8 +619,9 @@ impl Backend { { return Err(EIO); } - match firmware.queues.rebind_compute(self.owner, &self.context)? { + match firmware.queues.rebind_compute(self.owner, &self.context, required)? { Rebind::Done => return Ok(()), + Rebind::Grow => { drop(state); return self.grow_compute(required); } Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), Rebind::Fresh => { let reservation = firmware.queues.reserve_compute(self.owner)?; @@ -590,7 +638,7 @@ impl Backend { Err(qid) => { // Fresh zeroed backing is built without the device mutex; the // queue may have been taken or re-armed by another owner since. - let pages = crate::g17::freelist::Pages::compute(&alloc)?; + let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; let ring = compute::DescriptorRing::new(&alloc)?; let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -603,13 +651,13 @@ impl Backend { }; // The firmware consumed this slot's release before the backing // was dropped; the row is rewritten before the queue is kicked. - firmware.init.initialize_compute_pool(slot, page_list)?; + firmware.init.initialize_compute_pool_blocks(slot, page_list, required)?; firmware.queues.bind_rearmed(qid, self.owner, &self.context)?; return Ok(()); } }; let built = (|| { - let pool = compute::PoolBacking::new(&alloc, reservation.id)?; + let pool = compute::PoolBacking::new(&alloc, reservation.id, required)?; { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -618,7 +666,7 @@ impl Backend { } firmware.queues.reserved(reservation)?; let (slot, page_list) = pool.pool_descriptor()?; - firmware.init.initialize_compute_pool(slot, page_list)?; + firmware.init.initialize_compute_pool_blocks(slot, page_list, required)?; } let backing = compute::Backing::new(&alloc, pool)?; // The global scheduler is created after pool/kick backing and @@ -1005,7 +1053,8 @@ impl super::Firmware { } impl Backend { - fn ensure_compute(&self) -> Result { + fn ensure_compute(&self, scs_pages: u16) -> Result { + let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages)?; use kernel::time::{delay::fsleep, Delta}; const SPARED_WAIT_MS: usize = 200; let mut deferred = DeferredBatch::publication()?; @@ -1021,7 +1070,7 @@ impl Backend { deferred.finish(); let pending = pending?; if !pending { - return self.build_compute(); + return self.build_compute(required); } // Other owners continue submitting while this owner gives its // queue-local drain witness time to arrive. diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index ef72699e8e7ce..7509cf539c278 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -70,8 +70,8 @@ impl job::Backend for Backend { Some(self.shared.feed.clone()) } - fn ensure_compute(&self) -> Result { - Backend::ensure_compute(self) + fn ensure_compute(&self, scs_pages: u16) -> Result { + Backend::ensure_compute(self, scs_pages) } fn fail_vm(&self, error: Error) { diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 437ad42aeafda..1053fc26d17c0 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -1453,7 +1453,11 @@ struct drm_asahi_cmd_compute { /** @ts: Timestamps for the compute command */ struct drm_asahi_timestamps ts; - /** @scs_layout: Reserved; must be zero. */ + /** + * @scs_layout: Packed G17P Gen1 shader-context scratch layout, or zero. + * Bits 0:15 contain stack depth, 16:23 levels, 24:31 cliques, + * 32:47 user bytes and 48:63 driver bytes. Non-G17P paths require zero. + */ __u64 scs_layout; /** @@ -1473,6 +1477,23 @@ struct drm_asahi_cmd_compute { * queue's compute command count and the header's cdm_barrier instead. */ __u64 prior_cdm; + + /** @cdm_auxiliary_allocation: G17P auxiliary allocation, zero or 8..32. */ + __u32 cdm_auxiliary_allocation; + /** @reserved: MBZ. */ + __u32 reserved; + /** @intersection_pool: 16-KiB aligned G17P intersection-table address. */ + __u64 intersection_pool; + /** @intersection_pool_bytes: Mapped pool size in bytes, a multiple of 16 KiB. */ + __u64 intersection_pool_bytes; + /** @intersection_first: First 128-byte entry in @intersection_pool. */ + __u32 intersection_first; + /** @intersection_count: Number of 128-byte entries, less than 2^18. */ + __u32 intersection_count; + /** @intersection_default_program: Encoded default G17P program address. */ + __u64 intersection_default_program; + /** @intersection_default_control: Default G17P program control word. */ + __u64 intersection_default_control; }; /** From e911c531941cd57098ed86cc40daa681d740ae57 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 09:40:54 +1100 Subject: [PATCH 1242/2680] drm/vc4: convert encoded match data through pointer-width integer Clang22 rejects a direct void-pointer-to-enum cast under the existing Fedora WERROR configuration. Preserve numeric generation match data by converting through uintptr_t before the enum; retain all drivers and warnings. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/vc4/vc4_drv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/vc4/vc4_drv.c b/drivers/gpu/drm/vc4/vc4_drv.c index 7b64e1ed02d4c..284a971e28b31 100644 --- a/drivers/gpu/drm/vc4/vc4_drv.c +++ b/drivers/gpu/drm/vc4/vc4_drv.c @@ -297,7 +297,7 @@ static int vc4_drm_bind(struct device *dev) dev->coherent_dma_mask = DMA_BIT_MASK(32); - gen = (enum vc4_gen)of_device_get_match_data(dev); + gen = (enum vc4_gen)(uintptr_t)of_device_get_match_data(dev); if (gen > VC4_GEN_4) driver = &vc5_drm_driver; From 3a33c1390ad38aef215595d2ae660cd2465e3122 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 09:45:37 +1100 Subject: [PATCH 1243/2680] drm/pl111: convert encoded match data through pointer-width integer Preserve Versatile and IM-PD1 generation values while avoiding Clang22 direct pointer-to-enum errors under the unchanged Fedora WERROR configuration. Keep all driver selections and strict compiler checks. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/pl111/pl111_versatile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/pl111/pl111_versatile.c b/drivers/gpu/drm/pl111/pl111_versatile.c index 0d8331a3909f4..93ef58ffc91d3 100644 --- a/drivers/gpu/drm/pl111/pl111_versatile.c +++ b/drivers/gpu/drm/pl111/pl111_versatile.c @@ -486,7 +486,7 @@ int pl111_versatile_init(struct drm_device *dev, struct pl111_drm_dev_private *p return 0; } - versatile_clcd_type = (enum versatile_clcd)clcd_id->data; + versatile_clcd_type = (enum versatile_clcd)(uintptr_t)clcd_id->data; /* Versatile Express special handling */ if (versatile_clcd_type == VEXPRESS_CLCD_V2M) { @@ -506,7 +506,7 @@ int pl111_versatile_init(struct drm_device *dev, struct pl111_drm_dev_private *p np = of_find_matching_node_and_match(NULL, impd1_clcd_of_match, &clcd_id); if (np) - versatile_clcd_type = (enum versatile_clcd)clcd_id->data; + versatile_clcd_type = (enum versatile_clcd)(uintptr_t)clcd_id->data; } map = syscon_node_to_regmap(np); From 6ad0561af0f01a46bbfd8f2d9b5b1bf226cd13c5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 14 Oct 2026 19:46:45 -0400 Subject: [PATCH 1244/2680] drm/asahi: recycle settled quarantined G17 compute queues A failed client's compute queue is quarantined. Once its owner closes and the retirement witness has settled every retained command, the queue has no active command, so exit_candidate() never issues a kill for it, and reusable() refuses it because it is still quarantined. It therefore keeps its QID and its USC pool backing until the processor stops. Each Gen1-capable compute pool takes about 43 MiB of the T8140 dynamic lower VA window. On Neo, after roughly 42 failed clients the window is exhausted and every new compute submission fails with ENOSPC until reboot. Return such a queue to service the way recycle_killed() already does after an acknowledged kill: reset the item ring at the firmware consumer cursor and require fresh backing, so the existing vacate path publishes the free-list release and frees the VA. A kill acknowledged before the scheduler page showed the killed state was left Unproven forever, retaining its queue as well. Re-read such kills on later passes and settle them only once the page shows the killed state, also when their commands were retired by other means. Validation on Neo with deliberately failing compute clients: unmodified aurora-wip reaches ENOSPC after about 42 failed clients. With this change, a fresh boot handled 84 failed clients, including a replay of 28 failed device creations, with no ENOSPC and no firmware crash, and the same build passed more than 150 failed clients on one boot. Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 29 ++++++++++++++++++- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 10 +++++-- drivers/gpu/drm/asahi/g17/runtime.rs | 1 + 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index cae576fe183af..f3f050d482bef 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -1192,6 +1192,33 @@ impl Queue { { return Ok(false); } + self.recycle_settled()?; + Ok(true) + } + /// A quarantined queue whose owner closed and whose every retained command was + /// already settled, by the exact retirement witness or by an acknowledged kill, + /// holds no pending command: the firmware consumed its item ring exactly as after + /// a kill. Neither `reusable` nor `exit_candidate` matches it, so without this it + /// keeps its QID and USC pool backing until processor stop, and repeated client + /// failures exhaust the compute pool VA range. Return it to service as + /// `recycle_killed` does. + pub(crate) fn recycle_quarantined_idle(&mut self) -> Result { + if !self.installed + || !self.quarantined + || !self.released + || self.growing + || self.exit.is_some() + || self.awaiting_witness + || self.active.len != 0 + || self.replays.len != 0 + || self.previous.is_some() + { + return Ok(false); + } + self.recycle_settled()?; + Ok(true) + } + fn recycle_settled(&mut self) -> Result { self.graph.reset_items()?; self.exit = None; self.quarantined = false; @@ -1213,7 +1240,7 @@ impl Queue { self.retirement_ready = true; self.released_at = now_ns(); self.needs_fresh_backing = true; - Ok(true) + Ok(()) } pub(crate) fn released_at(&self) -> u64 { self.released_at diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index 10d2a37274800..877fc935dc909 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -156,10 +156,16 @@ impl Queue { complete_qos: &mut impl FnMut(crate::g17::qos::Owner) -> Result, defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { - if !self.exit.as_ref().is_some_and(|exit| exit.phase == Phase::Acknowledged) { + let Some(phase) = self.exit.as_ref().map(|exit| exit.phase) else { + return Ok(()); + }; + if !matches!(phase, Phase::Acknowledged | Phase::Unproven) { return Ok(()); } - if self.active.len != 0 { + // An unproven kill (acknowledged before the scheduler page showed the killed + // state) is re-read on later passes and settles only once the page proves it, + // even after its commands were retired by other means. + if self.active.len != 0 || phase == Phase::Unproven { if !recovery_closed { return Ok(()); } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index ecfa57cd9a5a1..ffaff6d80dbf9 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1253,6 +1253,7 @@ impl super::Firmware { .filter_map(|entry| entry.queue.as_deref_mut()) { queue.recycle_killed()?; + queue.recycle_quarantined_idle()?; if let Some(binding) = queue.detach_exited_owner() { deferred.defer_binding(binding); } From 90c00ed77e8be00d19c73cf60cb4cf62f3fa6d7d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 14 Oct 2026 20:05:00 -0400 Subject: [PATCH 1245/2680] drm/asahi: g17: replay render groups firmware retired without executing A render group can be retired by firmware without its fragment pass ever running: the tiler start record is written, the fragment end record is not, both completion stamps already cover the group, its job list is empty and no recovery is reported. On Neo this happens to an innocent owner (the compositor) at the instant another client's compute command fails with a zero-duration completion record. The front group then polls Pending forever, the 2 s scheduler timeout fires, fail_after_observation takes the strict abandonment witness and fails the owner's VM with EIO. The compositor's device is lost and the desktop freezes. Use the same witness to recognise the abandoned group as soon as it retires, and publish the same command again instead of failing it. In Pair::retire_front a lone, outer-published front group that polls Pending is checked with retirement::abandoned (executed groups are never accepted); the proof has to hold on two observations at least 2 ms apart so a single sample straddling a firmware update cannot trigger it. The dead generation's free-list and QoS accounting is completed exactly as the timeout path does, the record is retired and the packet is parked on the pair with its publication state reset; its completion stays unsignalled and no VM status is recorded. Firmware::replay_render, run by the event worker next to the compute replay and never during recovery, prepares and publishes the parked packet again through the ordinary path (fresh ordinal, descriptors, scene and timestamp record). A closed owner, a cancelled packet or a stale context settle the replay with ECANCELED, which does not poison the VM; other publication failures settle it with their error as before. Completion::begin_replay bounds the attempts to three, after which the previous EIO settlement applies. The original submission's scheduler timeout is ignored while its packet is parked, and a processor stop drains a parked replay like active groups. Nothing of an abandoned group reached its attachments, which only the fragment stage writes, so re-running the tiler and fragment passes from the same command is the first execution as far as render targets are concerned. Tiler-side pipeline statistics may count twice; vertex-stage side effects exist only behind an opt-in diagnostic. Tested on Neo (A18 Pro, G17P) with failing ray-query clients next to an animating compositor: the first incident at 81.455 s was replayed at 81.468 s (slot 4, qids [8, 9], pass [tiler start, 0], stamps covering the group) and the compositor kept rendering with no device loss, where the previous kernels lost the desktop in three out of three runs. With this change and VerboseFaults set, two further boots logged 2 and 3 replayed groups over 13 and 24 failing clients (the first boot differed only in the log string), with no device loss and the compositor still rendering. Boots of earlier builds with the same replay logic ran about 1,500 ray-query, ray-tracing pipeline and mesh conformance cases without failing clients and logged no replay. With the VerboseFaults debug flag, each replay is logged with its slot and the pass record that proved the group was not executed. Co-Authored-By: Claude Fable 5.1 Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/asahi/g17/job.rs | 5 + drivers/gpu/drm/asahi/g17/queue/render.rs | 134 +++++++++++++++++- .../gpu/drm/asahi/g17/queue/render/parked.rs | 1 + drivers/gpu/drm/asahi/g17/runtime/events.rs | 2 + drivers/gpu/drm/asahi/g17/runtime/render.rs | 1 + .../drm/asahi/g17/runtime/render/replay.rs | 105 ++++++++++++++ .../drm/asahi/g17/runtime/render/settle.rs | 2 +- 7 files changed, 247 insertions(+), 3 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/replay.rs diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 05133b656ee59..c27c2bac43b10 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -185,6 +185,11 @@ impl Packet { pub(crate) fn is_published(&self) -> bool { self.publication.load(Ordering::Acquire) == 2 } + /// A published group retired by firmware without executing is published + /// again from the same packet. Publication state returns to "begun". + pub(crate) fn reset_publication_for_replay(&self) { + self.publication.store(1, Ordering::Release); + } pub(crate) fn cancel_requested(&self) -> bool { self.cancelled.load(Ordering::Acquire) } diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 24a274e1af62a..bbf85695970a0 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -130,8 +130,14 @@ struct Active { context_update: bool, failed: bool, drained_result: Option>, + /// First observation of this group retired and unlinked without execution. + abandoned_since: Option, } +/// A firmware-retired, unexecuted group must hold on two observations this far +/// apart before it is republished; a single sample may straddle an update. +const ABANDON_GRACE_NS: u64 = 2_000_000; + /// Installed kick backing remains owned even when this pair has no logical /// owner. Commands retain their exact descriptor and allocation generations. pub(crate) struct Pair { @@ -162,6 +168,8 @@ pub(crate) struct Pair { previous: Option, displaced: Option, context_update: bool, + /// A retired, never-executed group awaiting republication on this pair. + replay: Option>, } impl Pair { #[inline(never)] @@ -243,6 +251,7 @@ impl Pair { terminal: None, previous: None, displaced: None, + replay: None, context_update: true, })) } @@ -271,7 +280,21 @@ impl Pair { &mut self.memory.manager } pub(crate) fn in_flight(&self) -> bool { - self.active.len() != 0 + self.active.len() != 0 || self.replay.is_some() + } + /// A replay or an abandonment grace period needs the worker to poll. + pub(crate) fn replay_polling(&self) -> bool { + self.replay.is_some() || self.active.iter().any(|a| a.abandoned_since.is_some()) + } + pub(crate) fn replay_front(&self) -> Option> { + self.replay.clone() + } + pub(crate) fn finish_replay(&mut self, packet: &Arc) -> Result { + if !self.replay.as_ref().is_some_and(|front| Arc::ptr_eq(front, packet)) { + return Err(EIO); + } + self.replay = None; + Ok(()) } pub(crate) fn published_in_flight(&self) -> bool { !self.quarantined && self.active.iter().any(|active| active.outer_published) @@ -682,6 +705,7 @@ impl Pair { context_update: self.context_update, failed: false, drained_result: None, + abandoned_since: None, }) .map_err(|_| EBUSY)?; self.prepared = None; @@ -904,6 +928,9 @@ impl Pair { .as_ref() .is_some_and(|prepared| Arc::ptr_eq(&prepared.packet, packet)) } + pub(crate) fn replay_contains(&self, packet: &Arc) -> bool { + self.replay.as_ref().is_some_and(|replay| Arc::ptr_eq(replay, packet)) + } pub(crate) fn take_deferred_timeout(&self) -> Option<(Arc, Error)> { self.active.iter().find_map(|active| { active @@ -1079,7 +1106,8 @@ impl Pair { }; let timestamps = match result { Ok(retirement::Poll::Complete(timestamps)) => timestamps, - Ok(retirement::Poll::Retry | retirement::Poll::Pending) => return Ok(false), + Ok(retirement::Poll::Pending) => return self.replay_abandoned(host, defer), + Ok(retirement::Poll::Retry) => return Ok(false), Err(error) => { self.quarantine(error, defer)?; return Ok(false); @@ -1109,6 +1137,105 @@ impl Pair { } } + /// The front group was retired and unlinked by firmware without executing + /// (job list empty, stamps cover it, no recovery, fragment end never + /// written). Nothing of this group reached its attachments, so the same + /// command is published again instead of failing the owner's VM. The + /// proof is the strict timeout witness, taken twice ABANDON_GRACE_NS apart, + /// and only for a lone group whose completion still accepts a replay. + fn replay_abandoned( + &mut self, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + use retirement::Memory; + if self.quarantined || self.active.len() != 1 || self.replay.is_some() { + return Ok(false); + } + let front = self.active.front().ok_or(EIO)?; + if !front.outer_published || front.failed { + return Ok(false); + } + let mut observer = Observer { + memory: &self.memory, + qids: self.qids(), + host, + }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(front.ticket.ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + let abandoned = + retirement::abandoned(&front.ticket, snapshot, stamps, state, false)?; + let now = crate::g17::now_ns(); + let front = self.active.front_mut().ok_or(EIO)?; + if !abandoned { + front.abandoned_since = None; + return Ok(false); + } + let Some(since) = front.abandoned_since else { + front.abandoned_since = Some(now); + return Ok(false); + }; + if now.saturating_sub(since) < ABANDON_GRACE_NS { + return Ok(false); + } + if crate::debug::debug_enabled(crate::debug::DebugFlags::VerboseFaults) { + pr_info!( + "render slot {} retired without execution (pass {:?}); replaying\n", + self.slot, + snapshot.pass + ); + } + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let pool = self.free_list.id(); + let front = self.active.front_mut().ok_or(EIO)?; + if !front.flist_completed { + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.flist_completed = true; + front.qos_completed = true; + } + let ordinal = front.ticket.ordinal; + let packet = front.packet.clone(); + if !packet.completion.begin_replay() { + if !packet.completion.spared() { + packet.completion.status().record(EIO); + if self.terminal.is_none() { + self.terminal = Some((packet.completion.status().clone(), EIO)); + } + } + defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + return Ok(true); + } + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + packet.reset_publication_for_replay(); + self.replay = Some(packet); + Ok(true) + } + + /// The pending replay cannot proceed (closed owner, cancellation or a + /// publication refusal). ECANCELED leaves the VM usable; any other error + /// is the ordinary device-loss outcome the replay tried to avoid. + pub(crate) fn fail_replay( + &mut self, + error: Error, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + let Some(packet) = self.replay.take() else { + return Ok(()); + }; + defer(Deferred::Retired(packet.completion.clone(), Err(error))) + } + /// A closed preparation epoch invalidates only this unpublished descriptor. /// Its completion remains available for a retry under the next epoch. pub(crate) fn discard_prepared(&mut self, packet: &Arc) -> Result { @@ -1292,6 +1419,9 @@ impl Pair { self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; } + if let Some(packet) = self.replay.take() { + defer(Deferred::Drained(packet.completion.clone(), Err(EIO)))?; + } if let Some(previous) = self.previous.as_mut() { previous.ready = true; } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index f712bb7b1b87d..39941de815e07 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -213,6 +213,7 @@ impl ParkedPair { terminal: None, previous: self.previous.take(), displaced: None, + replay: None, context_update: true, })) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index a7ff3e4f0ecc3..3ae1d0a609a33 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -368,6 +368,7 @@ impl crate::g17::Shared { // Replay only at this outer boundary, after recovery has settled. // Publication may service events but must never recurse into replay. firmware.replay_compute(deferred)?; + firmware.replay_render(deferred)?; drop(state); self.observe_renders(masks, deferred)?; let mut state = self.state.lock(); @@ -409,6 +410,7 @@ impl crate::g17::Shared { let backpressured = firmware.render_control_backpressured(); let polling = backpressured || firmware.queues.replay_outer_wait.is_some() + || firmware.queues.render.replay_polling() || firmware.queues.teardown.polling(now) || firmware.compute_exit_polling(now) || firmware diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 460889be64e2b..a47abf2045526 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -11,6 +11,7 @@ mod observe; mod pools; mod prepare; mod recovery; +mod replay; mod settle; mod submit; pub(super) mod teardown; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs new file mode 100644 index 0000000000000..ce9771778acd4 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Republication of render groups that firmware retired without executing. +//! The pair proves the abandonment (retired, unlinked, stamps covering it, no +//! recovery, fragment end never written) and parks the packet; this pass +//! prepares and publishes the same command again under the device mutex. + +use super::super::DeferredBatch; +use crate::g17::{object::Allocator, queue::render::RenderAddresses, Firmware}; +use kernel::prelude::*; + +impl super::RenderState { + pub(in crate::g17) fn replay_polling(&self) -> bool { + self.entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .any(|pair| pair.replay_polling()) + } +} + +impl Firmware { + pub(in crate::g17) fn replay_render(&mut self, deferred: &mut DeferredBatch) -> Result { + if self.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&self.init)? != 0 + { + return Ok(()); + } + for slot in 0..self.queues.render.entries.len() { + let Some((packet, owner, qids, closed, busy)) = + self.queues.render.entries[slot].as_ref().and_then(|entry| { + let pair = entry.pair.as_ref()?; + let packet = pair.replay_front()?; + Some(( + packet, + entry.reservation.owner, + pair.qids(), + entry.closed, + entry.preparing.is_some() + || entry.growing + || entry.pending_timeout + || entry.pending_error.is_some(), + )) + }) + else { + continue; + }; + if busy { + continue; + } + if closed || packet.cancel_requested() || !packet.context.is_current() { + self.with_render(slot, |pair, _| { + pair.fail_replay(ECANCELED, &mut |event| deferred.push(event)) + })?; + continue; + } + if packet.completion.status().get() != 0 { + self.with_render(slot, |pair, _| { + pair.fail_replay(EIO, &mut |event| deferred.push(event)) + })?; + continue; + } + let addresses = RenderAddresses { + stamps: [ + self.init.stamp_addresses(qids[0])?, + self.init.stamp_addresses(qids[1])?, + ], + }; + let dev = self.primary.state.shared.drm()?; + let uat = self.uat.clone(); + let prepared = self.with_render(slot, |pair, _| { + if !pair.room() { + return Err(EBUSY); + } + let alloc = Allocator { + dev: &dev, + uat: &uat, + }; + pair.prepare(&alloc, &packet, addresses) + }); + match prepared { + Ok(()) => {} + Err(error) if error == EBUSY || error == EAGAIN => continue, + Err(error) => { + self.with_render(slot, |pair, _| { + pair.fail_replay(error, &mut |event| deferred.push(event)) + })?; + continue; + } + } + match self.publish_render(owner, slot as u8, &packet, deferred) { + Ok(()) => { + self.with_render(slot, |pair, _| pair.finish_replay(&packet))?; + } + Err(error) if error == EBUSY || error == EAGAIN => continue, + Err(error) => { + self.with_render(slot, |pair, _| { + pair.fail_replay(error, &mut |event| deferred.push(event)) + })?; + } + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index 415d1f391f601..2e88848b93216 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -70,7 +70,7 @@ impl crate::g17::Firmware { .and_then(Option::as_ref) .and_then(|entry| entry.pair.as_ref()) .ok_or(EIO)?; - if !pair.contains_packet(packet) { + if pair.replay_contains(packet) || !pair.contains_packet(packet) { return Ok(()); } let generation = self.recovery.generation(); From 5e997695a9aedd3722052c808936dd6b81d36ae8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 14 Oct 2026 21:40:42 -0400 Subject: [PATCH 1246/2680] drm/asahi: g17: never kill compute commands with ray traversal scratch A closed client's quarantined or cancelled compute queue is ended with a firmware context kill once its owner is gone. For commands that use ray traversal scratch this goes wrong on Neo (T8140) when the traversal stopped making progress: - releasing the killed queue's pool right after the kill was acknowledged crashed the GPU firmware about 40 ms later (3 of 3 cases), taking the whole GPU and the desktop down; - keeping the pool instead avoided the crash, but about 12 s later every following render on the device timed out and the compositor lost its context. Leaving such a queue alone, as a diagnostic build that skipped the kill did on five boots, kept the rest of the GPU working each time. Do not issue the kill while any active command of the queue has traversal scratch enabled. The client still sees its device lost through the existing timeout and quarantine path, the queue stays retained until its commands retire, and every other compute queue keeps the existing kill and recycle behaviour. Tested on Neo with this exact change: a ray tracing dispatch that stopped making progress timed out at 69.7 s and was left unkilled; the compositor kept rendering for the following 40 s, and new compute, ray tracing pipeline and amber conformance work ran normally afterwards. A raygen shader looping forever was also left unkilled; later ray tracing dispatches still completed in 20 ms each and the desktop stayed live. Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/asahi/g17/queue/compute/exit.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index 877fc935dc909..187fd27f8e78e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -83,11 +83,25 @@ impl Transaction { } impl Queue { + /// A command with ray traversal scratch is never killed. On Neo, killing a + /// traversal that stopped making progress either crashed the GPU firmware + /// when the queue's pool was then released, or stalled every later render + /// when it was retained; left alone, the rest of the GPU kept working. Such + /// a queue stays quarantined and retained until its commands retire. + fn active_traversal(&self) -> bool { + self.active.iter().any(|active| { + matches!( + &active.packet.command, + crate::g17::command::Validated::Compute { scratch, .. } if scratch.enabled() + ) + }) + } pub(crate) fn exit_candidate(&self, context: &Arc, age: u64) -> bool { self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) && self.released && self.active.len != 0 && self.exit.is_none() + && !self.active_traversal() && (self.quarantined || self.retired_by_teardown) && age >= if self.quarantined { From 1fc016fe714bd2ea3d90a536147fcfdc62c5f947 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 14 Oct 2026 20:46:34 -0400 Subject: [PATCH 1247/2680] drm/asahi: g17: ray queries in render stages Let render commands carry ray query state for their vertex (tiling) and fragment stages, as compute commands already do. UAPI: - DRM_ASAHI_FEATURE_RENDER_RAY_QUERY advertises the feature. - struct drm_asahi_render_ray_query_stage describes one stage: its ray query scratch layout and intersection tables. drm_asahi_cmd_render gains ray_query[2] at its end; shorter commands from older userspace are read with both stages zeroed. - drm_asahi_queue_create gains render_scs_layout[2], the largest layouts the queue will submit, which sizes the queue's render pool. Zero keeps the existing pool size. Firmware programming: for each stage with ray query scratch, the tiling or fragment register program carries the scratch size, tail, stack and window words and the intersection table registers, and enables SCS with register 0x017d9; its kick sets the SCS-enabled header bit. Stages without ray queries keep the existing register programs unchanged. Validation rejects intersection pools and default programs outside the VM and a scratch layout whose enable state disagrees with the intersection tables. The render pool is sized from both stages' scratch demand at queue creation, and a command needing more than was reserved is rejected. Tested on Neo (A18 Pro) with the matching Mesa series: ray_query builtin.flow and advanced.null_as cases 41 of 48 pass across vertex, tessellation, geometry, fragment, compute and ray tracing stages (the failures are tessellation evaluation and three null acceleration structure cases); position fetch 60 of 60 runnable cases; earlier builds of the same render path passed 883 ray query stage cases. Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/asahi/file.rs | 4 +- drivers/gpu/drm/asahi/g17/command.rs | 23 ++++- drivers/gpu/drm/asahi/g17/context.rs | 14 ++- drivers/gpu/drm/asahi/g17/freelist.rs | 19 ++-- drivers/gpu/drm/asahi/g17/fw/compute.rs | 8 +- drivers/gpu/drm/asahi/g17/fw/compute_pool.rs | 9 +- drivers/gpu/drm/asahi/g17/fw/kick.rs | 8 +- drivers/gpu/drm/asahi/g17/fw/mod.rs | 1 + drivers/gpu/drm/asahi/g17/fw/render.rs | 92 +++++++++++++------- drivers/gpu/drm/asahi/g17/fw/render_rt.rs | 48 ++++++++++ drivers/gpu/drm/asahi/g17/job.rs | 4 + drivers/gpu/drm/asahi/g17/mod.rs | 3 +- drivers/gpu/drm/asahi/g17/queue/compute.rs | 2 +- drivers/gpu/drm/asahi/g17/queue/mod.rs | 11 +++ drivers/gpu/drm/asahi/g17/queue/render.rs | 13 ++- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 4 + drivers/gpu/drm/asahi/queue/mod.rs | 5 ++ include/uapi/drm/asahi_drm.h | 55 ++++++++++++ 18 files changed, 262 insertions(+), 61 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/fw/render_rt.rs diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 3cb5f60a4873a..2027c9ba1c20a 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -1188,7 +1188,7 @@ impl File { let _ = file_vm; core::mem::drop(guard); - let queue = reg_data.gpu.new_queue( + let mut queue = reg_data.gpu.new_queue( vm, ualloc, ualloc_priv, @@ -1197,6 +1197,8 @@ impl File { data.usc_exec_base, )?; + queue.reserve_render_scs(data.render_scs_layout)?; + data.queue_id = resv.index().try_into()?; resv.fill(Arc::pin_init(new_mutex!(queue), GFP_KERNEL)?)?; diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index dd3be1a81e1d8..671e513852c96 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -6,6 +6,7 @@ use super::fw::{ compute::{IntersectionTables, ScratchRequest}, kick::WriteRange, render::{RenderPass, UscProgram, ZlsSurface}, + render_rt::RayQueryStage, }; use crate::util::{AnyBitPattern, Reader}; use core::ops::Range; @@ -14,6 +15,7 @@ use kernel::{prelude::*, uapi}; const MAX_ATTACHMENTS: usize = crate::microseq::MAX_ATTACHMENTS; const USC_WINDOW_SIZE: u64 = 1 << 32; const MAX_SAMPLERS: u32 = 1024; +const RENDER_BASE_SIZE: usize = core::mem::offset_of!(uapi::drm_asahi_cmd_render, prior_cdm) + size_of::(); const COMPUTE_BASE_SIZE: usize = core::mem::offset_of!(uapi::drm_asahi_cmd_compute, prior_cdm) + size_of::(); const SAMPLER_SIZE: u64 = 8; const PAGE_SIZE: u64 = 0x4000; @@ -78,7 +80,7 @@ impl Attachments { // Commands stay inline in the submission plan, avoiding an allocation per render pass. #[allow(clippy::large_enum_variant)] pub(crate) enum Payload { - Render(uapi::drm_asahi_cmd_render, [Attachments; 2]), + Render(uapi::drm_asahi_cmd_render, [Attachments; 2], [RayQueryStage; 2]), Compute(uapi::drm_asahi_cmd_compute, Attachments, IntersectionTables), } @@ -128,7 +130,11 @@ impl<'a> Parser<'a> { == uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER { self.counts[0] += 1; - Payload::Render(payload(bytes)?, [self.attachments[0], self.attachments[1]]) + Payload::Render( + payload(&bytes[..bytes.len().min(RENDER_BASE_SIZE)])?, + [self.attachments[0], self.attachments[1]], + RayQueryStage::parse_pair(bytes.get(RENDER_BASE_SIZE..).unwrap_or(&[]))?, + ) } else { self.counts[1] += 1; Payload::Compute( @@ -284,6 +290,7 @@ fn surface( pub(crate) enum Validated { Render { pass: RenderPass, + ray_query: [RayQueryStage; 2], attachments: [Attachments; 2], timestamps: [uapi::drm_asahi_timestamps; 2], }, @@ -302,7 +309,7 @@ pub(crate) enum Validated { impl Payload { pub(crate) fn prior(&self) -> [u64; 2] { match self { - Self::Render(cmd, _) => [cmd.prior_vdm, cmd.prior_cdm], + Self::Render(cmd, _, _) => [cmd.prior_vdm, cmd.prior_cdm], Self::Compute(cmd, _, _) => [cmd.prior_vdm, cmd.prior_cdm], } } @@ -313,7 +320,7 @@ impl Payload { space: &impl AddressSpace, ) -> Result { match self { - Self::Render(cmd, attachments) => { + Self::Render(cmd, attachments, ray_query) => { if cmd.flags & !RENDER_FLAGS != 0 || !(1..=MAX_DIMENSION).contains(&cmd.width_px) || !(1..=MAX_DIMENSION).contains(&cmd.height_px) @@ -415,8 +422,16 @@ impl Payload { | (u64::from(cmd.bg_eot_partial_rsrc_spec_hi) << 32), rsrc_spec_hi: cmd.flags & RSRC_SPEC_HI != 0, }; + for stage in ray_query { + if stage.intersection.enabled() { + require_range(space, stage.intersection.pool, stage.intersection.pool_bytes, Access::Read)?; + let code = stage.intersection.default_program & 0x0000_ffff_ffff_ffff; + if code != 0 { require_range(space, code, 64, Access::Read)?; } + } + } Ok(Validated::Render { pass, + ray_query: *ray_query, attachments: *attachments, timestamps: [cmd.ts_vtx, cmd.ts_frag], }) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index afd0d1d22f5bb..6bc62a7d44bee 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -14,7 +14,7 @@ use super::{ status::VmStatus, }; use crate::mmu::{self, ExecutionContext, Vm, UAT_PGSZ}; -use core::sync::atomic::{fence, AtomicBool, AtomicU32, Ordering}; +use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicUsize, Ordering}; use identity::{RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; use kernel::{ new_mutex, @@ -120,6 +120,7 @@ pub(crate) struct Context { work: Arc>, work_base: u64, render: Arc>>>, + render_blocks: AtomicUsize, /// Fixed for the context's life. Physical queues bound to it carry this /// profile in every record, ring, doorbell and registration. policy: Policy, @@ -192,6 +193,7 @@ impl Context { work, work_base, render, + render_blocks: AtomicUsize::new(super::fw::freelist::RENDER_BLOCKS), policy, }) } @@ -277,6 +279,14 @@ impl Context { } Ok(result) } + pub(crate) fn reserve_render_blocks(&self, blocks: usize) -> Result { + if self.is_published() || self.render.lock().is_some() { return Err(EBUSY); } + self.render_blocks.store(blocks, Ordering::Release); + Ok(()) + } + pub(crate) fn render_blocks(&self) -> usize { + self.render_blocks.load(Ordering::Acquire) + } pub(crate) fn render_pool( &self, alloc: &Allocator<'_>, @@ -293,7 +303,7 @@ impl Context { return Ok(pool.clone()); } let pool = Arc::new( - RenderPool::new(alloc, ids, render_global, self.vm())?, + RenderPool::new(alloc, ids, render_global, self.vm(), self.render_blocks())?, GFP_KERNEL, )?; *render = Some(pool.clone()); diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 2612a9302ce28..c9c9ebda7f6bc 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -139,8 +139,12 @@ impl RenderBacking { alloc: &Allocator<'_>, render_global: &mmu::Vm, client: &mmu::Vm, + blocks: usize, ) -> Result { - let pages = Pages::new(alloc, RENDER_BLOCKS, RunLayout::Render)?; + if blocks < RENDER_BLOCKS || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) { + return Err(EINVAL); + } + let pages = Pages::new(alloc, blocks, RunLayout::Render)?; let address = pages.page_list_va(); let global = pages .backing @@ -161,6 +165,7 @@ impl RenderBacking { /// consumed and the client VM is evicted; a fresh backing is installed before /// the pool is staged again. pub(crate) struct RenderPool { + blocks: usize, backing: Arc>>, /// Page-list VA of the installed backing; zero while vacant. page_list: AtomicU64, @@ -182,9 +187,10 @@ impl RenderPool { ids: &Arc, render_global: &mmu::Vm, client: &mmu::Vm, + blocks: usize, ) -> Result { let id = RenderId::new(ids)?; - let backing = RenderBacking::new(alloc, render_global, client)?; + let backing = RenderBacking::new(alloc, render_global, client, blocks)?; let address = backing.pages.page_list_va(); let run_list_va = backing.pages.run_list_va(); let state = alloc.kernel( @@ -207,7 +213,7 @@ impl RenderPool { class: FreeListClass::Two, page_list_va: address, run_list_va, - blocks: RENDER_BLOCKS as u32, + blocks: blocks as u32, state_va: state.gpu_va(), }, 0, @@ -222,6 +228,7 @@ impl RenderPool { GFP_KERNEL, )?; Ok(Self { + blocks, backing, page_list: AtomicU64::new(address), epoch: AtomicU64::new(0), @@ -279,7 +286,7 @@ impl RenderPool { if !self.vacant() { return Ok(()); } - self.rearm(RenderBacking::new(alloc, render_global, client)?) + self.rearm(RenderBacking::new(alloc, render_global, client, self.blocks)?) } pub(crate) fn serial(&self) -> u64 { @@ -303,7 +310,7 @@ impl RenderPool { } pub(crate) fn end_va(&self) -> u64 { self.run_list_va() - + (RENDER_BLOCKS * size_of::()) as u64 + + (self.blocks * size_of::()) as u64 } /// Fresh pools are already populated. Reuse requires the previous release @@ -319,7 +326,7 @@ impl RenderPool { class: FreeListClass::Two, page_list_va: self.page_list_va(), run_list_va: self.run_list_va(), - blocks: RENDER_BLOCKS as u32, + blocks: self.blocks as u32, state_va: self.state.gpu_va(), }, submitted, diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index a4775abc6b7eb..0633da7f43541 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -112,16 +112,16 @@ impl ScratchRequest { pub(crate) fn pages(self) -> u16 { self.pages } - fn size_word(self) -> u64 { + pub(crate) fn size_word(self) -> u64 { let pages = u64::from(self.pages); pages | (pages.div_ceil(Self::PAGES_PER_MIB) << 16) } - fn window_word(self) -> u64 { + pub(crate) fn window_word(self) -> u64 { self.window } - fn stack_word(self) -> u64 { + pub(crate) fn stack_word(self) -> u64 { self.stack } @@ -129,7 +129,7 @@ impl ScratchRequest { self.driver } - fn tail_word(self) -> u64 { + pub(crate) fn tail_word(self) -> u64 { self.tail } diff --git a/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs b/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs index 4005daa4e9832..0f0614f6155a1 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs @@ -5,9 +5,9 @@ use kernel::prelude::*; use super::initdata::{FREE_LIST_BLOCK_PAGES, FREE_LIST_BLOCK_SIZE}; use super::freelist::{COMPUTE_BLOCKS, PAGE_LIST_SIZE}; -pub(crate) fn required_blocks(active_cores: u32, scs_pages: u16) -> Result { +pub(crate) fn extra_bytes(active_cores: u32, scs_pages: u16) -> Result { if active_cores == 0 { return Err(EINVAL); } - if scs_pages == 0 { return Ok(COMPUTE_BLOCKS); } + if scs_pages == 0 { return Ok(0); } // Register0x101d1 counts 4-KiB pages and their 1-MiB large-page inventory. // Register0x0d479 reserves eight further pages for each enabled USC. let small = u64::from(scs_pages); @@ -15,6 +15,11 @@ pub(crate) fn required_blocks(active_cores: u32, scs_pages: u16) -> Result Result { + let extra = extra_bytes(active_cores, scs_pages)?; let bytes = extra.checked_add((COMPUTE_BLOCKS * FREE_LIST_BLOCK_SIZE) as u64) .ok_or(EOVERFLOW)?; let blocks = usize::try_from(bytes.div_ceil(FREE_LIST_BLOCK_SIZE as u64)) diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index 47cdce3ca09e5..e07fbfb485a10 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -370,7 +370,7 @@ pub(crate) struct KickArgs<'a> { /// Register arrays bound to the kick. pub(crate) register_arrays: [Option; 4], /// The compute register program includes a shader-context scratch request. - pub(crate) compute_scratch: bool, + pub(crate) scs_enabled: bool, /// Priority class of the queue. pub(crate) priority: u8, } @@ -423,7 +423,7 @@ impl KickEntry { const QID_SHIFT: u32 = 42; const MCACHE_PRESENT: u64 = 1 << 58; const PRIORITY_SHIFT: u32 = 59; - const COMPUTE_SCRATCH: u64 = 1 << 61; + const SCS_ENABLED: u64 = 1 << 61; const QOS_SLOT_SHIFT: u32 = 32; const QOS_CLASS_SHIFT: u32 = 40; const QOS_CLASS_MAX: u8 = 0x1f; @@ -446,8 +446,8 @@ impl KickEntry { let mut header = args.timestamp.get() << Self::TIMESTAMP_SHIFT | (args.qid as u64) << Self::QID_SHIFT | (args.priority as u64) << Self::PRIORITY_SHIFT; - if args.compute_scratch { - header |= Self::COMPUTE_SCRATCH; + if args.scs_enabled { + header |= Self::SCS_ENABLED; } let mcache = match args.mcache { Some(table) => { diff --git a/drivers/gpu/drm/asahi/g17/fw/mod.rs b/drivers/gpu/drm/asahi/g17/fw/mod.rs index 479bc91d21ca5..b6bb6aaea860f 100644 --- a/drivers/gpu/drm/asahi/g17/fw/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/mod.rs @@ -21,6 +21,7 @@ pub(crate) mod initdata; pub(crate) mod kick; pub(crate) mod queue; pub(crate) mod render; +pub(crate) mod render_rt; pub(crate) mod qos; /// Clears a firmware structure in place. diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 23eaa7e7dfba9..85a2385b421b8 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -23,7 +23,7 @@ use super::kick::{ use super::queue::CommandTag; use super::{ clear, offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, - UNK_0D411, UNK_101D9, UNK_10791, UNK_1A0E9, UNK_1C8F8, + UNK_0D411, UNK_101D9, UNK_1C8F8, }; /// Size of a tiling descriptor. @@ -54,7 +54,11 @@ const DEFLAKE_20: u64 = 0x20; const DEFLAKE_2A0: u64 = 0x2a0; const TA_REGISTERS: usize = 73; +const TA_REGISTERS_MAX: usize = TA_REGISTERS + 1; +const TA_SCS_ENABLE_INDEX: usize = 1; const FRAGMENT_REGISTERS: usize = 89; +const FRAGMENT_REGISTERS_MAX: usize = FRAGMENT_REGISTERS + 1; +const FRAGMENT_SCS_ENABLE_INDEX: usize = 1; const PARTIAL_STORE_REGISTERS: usize = 16; const PARTIAL_RESUME_REGISTERS: usize = 23; const PARTIAL_LOAD_REGISTERS: usize = 10; @@ -122,8 +126,6 @@ const UNK_15359_PARTIAL: u64 = 0x20; /// Required value of register 0x15049 and of fragment descriptor field /// +0x1f50. const UNK_15049: u64 = 0x10_0000; -/// Value of the fragment register 0x10791. -const UNK_10791_FRAGMENT: u64 = UNK_10791 | 1 << 9; /// USC program offsets carry tag bits that are not part of the address. const USC_OFFSET_TAG_MASK: u32 = 7; @@ -303,6 +305,7 @@ pub(crate) struct RenderStage { pub(crate) struct RenderArgs<'a> { /// Render-pass state. pub(crate) pass: &'a RenderPass, + pub(crate) ray_query: [super::render_rt::RayQueryStage; 2], /// Kernel-owned objects. pub(crate) resources: &'a RenderResources, /// Submission ordinal of the render queue pair, from zero. @@ -481,7 +484,7 @@ fn ta_registers( args: &RenderArgs<'_>, g: &Geometry, w: &PassWords, -) -> Result<[RegisterWrite; TA_REGISTERS]> { +) -> Result<[RegisterWrite; TA_REGISTERS_MAX]> { let p = args.pass; let r = args.resources; let tilemap = user_offset(r.tilemap_va)?; @@ -497,9 +500,12 @@ fn ta_registers( } let key = work_key(r.context_id, args.ta.state_word); let ids = r.ta_ids.word(); + let rt = args.ray_query[0]; + let ift = rt.intersection.words(); Ok([ RegisterWrite::new(0x01748, 1), + RegisterWrite::new(0x017d9, 1), RegisterWrite::new(0x10141, 0x200), RegisterWrite::new(0x1c039, tilemap), RegisterWrite::new(0x1c9c8, tilemap), @@ -545,16 +551,16 @@ fn ta_registers( RegisterWrite::new(0x1c898, 0), RegisterWrite::new(0x101e1, UNK_101D9), RegisterWrite::new(0x1c9e8, w.layer_mode & LAYER_MODE_PIPE_MASK), - RegisterWrite::new(0x1a099, 0), - RegisterWrite::new(0x1a0a1, 0), - RegisterWrite::new(0x1a069, 0), - RegisterWrite::new(0x1a071, 0), - RegisterWrite::new(0x1a0c9, 0), - RegisterWrite::new(0x1a0d1, 0), - RegisterWrite::new(0x101c9, 0), - RegisterWrite::new(0x0d471, 0), - RegisterWrite::new(0x1a0f1, UNK_1A0E9), - RegisterWrite::new(0x10799, UNK_10791), + RegisterWrite::new(0x1a099, ift[0]), + RegisterWrite::new(0x1a0a1, ift[1]), + RegisterWrite::new(0x1a069, ift[2]), + RegisterWrite::new(0x1a071, ift[3]), + RegisterWrite::new(0x1a0c9, ift[4]), + RegisterWrite::new(0x1a0d1, ift[5]), + RegisterWrite::new(0x101c9, rt.scratch.size_word()), + RegisterWrite::new(0x0d471, rt.scratch.tail_word()), + RegisterWrite::new(0x1a0f1, rt.scratch.stack_word()), + RegisterWrite::new(0x10799, rt.scratch.window_word()), RegisterWrite::new(0x1c830, r.pb_slot as u64), RegisterWrite::new(0x1ca30, r.pm_scratch), RegisterWrite::new(0x16c39, r.pm_scratch), @@ -579,7 +585,7 @@ fn fragment_registers( args: &RenderArgs<'_>, g: &Geometry, w: &PassWords, -) -> [RegisterWrite; FRAGMENT_REGISTERS] { +) -> [RegisterWrite; FRAGMENT_REGISTERS_MAX] { let p = args.pass; let r = args.resources; let mut tile_state = TILE_STATE_BASE @@ -591,9 +597,12 @@ fn fragment_registers( } let key = work_key(r.context_id, args.fragment.state_word); let ids = r.fragment_ids.word(); + let rt = args.ray_query[1]; + let ift = rt.intersection.words(); [ RegisterWrite::new(0x01739, 1), + RegisterWrite::new(0x017d9, 1), RegisterWrite::new(0x10009, w.utile_config), // End-of-tile resource specifier. RegisterWrite::new(0x15379, w.eot_rsrc_spec), @@ -611,7 +620,7 @@ fn fragment_registers( RegisterWrite::new(0x16030, g.macro_size), RegisterWrite::new(0x100d9, g.screen), RegisterWrite::new(0x0a301, 0), - RegisterWrite::new(0x10791, UNK_10791_FRAGMENT), + RegisterWrite::new(0x10791, rt.scratch.window_word() | 1 << 9), RegisterWrite::new(0x16098, r.heapmeta_va), RegisterWrite::new(0x15109, p.scissor_va), RegisterWrite::new(0x15101, p.depth_bias_va), @@ -659,15 +668,15 @@ fn fragment_registers( RegisterWrite::new(0x101e9, UNK_101D9), RegisterWrite::new(0x160a8, 0), RegisterWrite::new(0x16068, tile_state), - RegisterWrite::new(0x1a0a9, 0), - RegisterWrite::new(0x1a0b1, 0), - RegisterWrite::new(0x1a079, 0), - RegisterWrite::new(0x1a081, 0), - RegisterWrite::new(0x1a0d9, 0), - RegisterWrite::new(0x1a0e1, 0), - RegisterWrite::new(0x101c1, 0), - RegisterWrite::new(0x0d469, 0), - RegisterWrite::new(0x1a0f9, UNK_1A0E9), + RegisterWrite::new(0x1a0a9, ift[0]), + RegisterWrite::new(0x1a0b1, ift[1]), + RegisterWrite::new(0x1a079, ift[2]), + RegisterWrite::new(0x1a081, ift[3]), + RegisterWrite::new(0x1a0d9, ift[4]), + RegisterWrite::new(0x1a0e1, ift[5]), + RegisterWrite::new(0x101c1, rt.scratch.size_word()), + RegisterWrite::new(0x0d469, rt.scratch.tail_word()), + RegisterWrite::new(0x1a0f9, rt.scratch.stack_word()), RegisterWrite::new(0x0a5a9, UNK_0A599), RegisterWrite::new(0x0d429, UNK_0D411), RegisterWrite::new(0x160e0, ids), @@ -907,9 +916,16 @@ impl TaDescriptor { self.pb_va = resources.pb_va; self.scene_va = resources.scene_va; self.scene_end_va = resources.scene_end_va; - self.checkpoints = TA_CHECKPOINTS; + let scs_register = args.ray_query[0].scratch.enabled(); + self.checkpoints = TA_CHECKPOINTS.map(|count| count + u16::from(scs_register)); self.unk_48 = (args.ordinal as u32).wrapping_add((args.ordinal / 2) as u32); - self.registers.set(array_va, ®isters)?; + if scs_register { + self.registers.set(array_va, ®isters)?; + } else { + let mut trimmed = [registers[0]; TA_REGISTERS]; + trimmed[TA_SCS_ENABLE_INDEX..].copy_from_slice(®isters[TA_SCS_ENABLE_INDEX + 1..]); + self.registers.set(array_va, &trimmed)?; + } self.tpc_va = resources.tpc_va; self.tpc_size = resources.tpc_size; self.fragment_qid = args.fragment.qid.into(); @@ -963,10 +979,11 @@ impl TaDescriptor { /// descriptor's address as seen by the pass's VM. pub(crate) fn register_bindings( descriptor_va: u64, + scs_register: bool, ) -> Result<[Option; 4]> { let array = RegisterArrayBinding::new( offset_va(descriptor_va, Self::ARRAY_OFFSET)?, - TA_CHECKPOINTS[0] as u8, + TA_CHECKPOINTS[0] as u8 + u8::from(scs_register), )?; Ok([Some(array), None, None, None]) } @@ -1153,13 +1170,14 @@ impl FragmentDescriptor { /// the descriptor's address as seen by the pass's VM. pub(crate) fn register_bindings( descriptor_va: u64, + scs_register: bool, ) -> Result<[Option; 4]> { let array = |index, checkpoint: usize| -> Result> { let va = offset_va(descriptor_va, Self::array_offset(index))?; Ok(Some(RegisterArrayBinding::new(va, checkpoint as u8)?)) }; Ok([ - array(0, FRAGMENT_CHECKPOINTS[0] as usize)?, + array(0, FRAGMENT_CHECKPOINTS[0] as usize + usize::from(scs_register))?, array(1, PARTIAL_STORE_REGISTERS)?, array(2, PARTIAL_RESUME_REGISTERS)?, array(3, PARTIAL_LOAD_REGISTERS)?, @@ -1226,9 +1244,19 @@ impl FragmentSlot { desc.merge_upper_x = pass.merge_upper_x; desc.merge_upper_y = pass.merge_upper_y; desc.tiles = geometry.tiles_x * geometry.tiles_y; - desc.checkpoints = FRAGMENT_CHECKPOINTS; - desc.entries = FRAGMENT_REGISTERS as u16; - desc.arrays[0].set(arrays[0], &fragment_registers(args, &geometry, &words))?; + let scs_register = args.ray_query[1].scratch.enabled(); + let extra = u16::from(scs_register); + desc.checkpoints = FRAGMENT_CHECKPOINTS.map(|count| count + extra); + desc.entries = FRAGMENT_REGISTERS as u16 + extra; + let registers = fragment_registers(args, &geometry, &words); + if scs_register { + desc.arrays[0].set(arrays[0], ®isters)?; + } else { + let mut trimmed = [registers[0]; FRAGMENT_REGISTERS]; + trimmed[FRAGMENT_SCS_ENABLE_INDEX..] + .copy_from_slice(®isters[FRAGMENT_SCS_ENABLE_INDEX + 1..]); + desc.arrays[0].set(arrays[0], &trimmed)?; + } desc.arrays[1].set(arrays[1], &partial_store_registers(pass, &words))?; desc.arrays[2].set(arrays[2], &partial_resume_registers(pass, &words))?; desc.arrays[3].set(arrays[3], &partial_load_registers(pass, &words))?; diff --git a/drivers/gpu/drm/asahi/g17/fw/render_rt.rs b/drivers/gpu/drm/asahi/g17/fw/render_rt.rs new file mode 100644 index 0000000000000..5ae23d84400eb --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/render_rt.rs @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +use super::{compute::{IntersectionTables, ScratchRequest}, compute_pool, + freelist::{PAGE_LIST_SIZE, RENDER_BLOCKS}, + initdata::{FREE_LIST_BLOCK_PAGES, FREE_LIST_BLOCK_SIZE}}; +use kernel::prelude::*; + +#[derive(Clone, Copy)] +pub(crate) struct RayQueryStage { + pub(crate) intersection: IntersectionTables, + pub(crate) scratch: ScratchRequest, +} + +impl RayQueryStage { + fn parse(bytes: &[u8]) -> Result { + let mut wire = [0u8; 56]; + wire[..bytes.len()].copy_from_slice(bytes); + let layout = u64::from_le_bytes(wire[..8].try_into().unwrap()); + let intersection = IntersectionTables::parse(&wire[8..])?; + let scratch = ScratchRequest::new(layout)?; + if scratch.enabled() != intersection.enabled() { return Err(EINVAL); } + Ok(Self { intersection, scratch }) + } + + pub(crate) fn parse_pair(bytes: &[u8]) -> Result<[Self; 2]> { + if bytes.get(112..).is_some_and(|tail| tail.iter().any(|&b| b != 0)) { + return Err(EINVAL); + } + let mut wire = [0u8; 112]; + let count = bytes.len().min(wire.len()); + wire[..count].copy_from_slice(&bytes[..count]); + Ok([Self::parse(&wire[..56])?, Self::parse(&wire[56..])?]) + } +} + +pub(crate) fn required_blocks(active_cores: u32, pages: [u16; 2]) -> Result { + let extra = compute_pool::extra_bytes(active_cores, pages[0])? + .checked_add(compute_pool::extra_bytes(active_cores, pages[1])?) + .and_then(|bytes| bytes.checked_mul(crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER as u64)) + .ok_or(EOVERFLOW)?; + let bytes = extra.checked_add((RENDER_BLOCKS * FREE_LIST_BLOCK_SIZE) as u64) + .ok_or(EOVERFLOW)?; + let blocks = usize::try_from(bytes.div_ceil(FREE_LIST_BLOCK_SIZE as u64)) + .map_err(|_| EOVERFLOW)?; + let maximum = PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8); + if blocks > maximum { return Err(E2BIG); } + Ok(blocks) +} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index c27c2bac43b10..b5a5b3759de6d 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -244,6 +244,10 @@ pub(crate) trait Backend: Send + Sync { /// Lazily create or rebind this logical queue's physical compute graph. fn ensure_compute(&self, scs_pages: u16) -> Result; + fn render_scs_blocks(&self, pages: [u16; 2]) -> Result { + if pages != [0; 2] { return Err(EINVAL); } + Ok(super::fw::freelist::RENDER_BLOCKS) + } /// Settle outstanding work of a terminal VM before an ioctl error is published. fn fail_vm(&self, error: Error); /// Called after both scheduler entities have cancelled their queued jobs. diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index a469d0eb801d3..aad094756d4f5 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1082,7 +1082,8 @@ impl gpu::Gpu for Gpu { uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64 | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS as u64; + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY as u64; params.gpu_generation = hw::GpuGen::G17 as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index f3f050d482bef..b78598cfbb760 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -1678,7 +1678,7 @@ impl Queue { mcache: None, event_mask: COMPUTE_KICK_EVENT_MASK, register_arrays: ComputeDescriptor::register_bindings(descriptor_low, *scratch)?, - compute_scratch: scratch.enabled(), + scs_enabled: scratch.enabled(), priority: policy.priority(), }; let published = host.next_compute_publication()?; diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 4ba6b33cb855a..569b5e289e7d8 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -171,6 +171,10 @@ impl Queue { Validated::Compute { .. } => Engine::Compute, }; let index = engine.index(); + if let Validated::Render { ray_query, .. } = &command { + let blocks = self.backend.render_scs_blocks(ray_query.map(|stage| stage.scratch.pages()))?; + if blocks > self.context.render_blocks() { return Err(EINVAL); } + } self.prune_frontier(index); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { @@ -354,6 +358,13 @@ impl Queue { } impl crate::queue::Queue for Queue { + fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { + let pages = [super::fw::compute::ScratchRequest::new(layouts[0])?.pages(), + super::fw::compute::ScratchRequest::new(layouts[1])?.pages()]; + let blocks = self.backend.render_scs_blocks(pages)?; + self.context.reserve_render_blocks(blocks) + } + fn submit( &mut self, _reg_data: &crate::driver::AsahiRegData<'_>, diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index bbf85695970a0..99be455b48e40 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -321,7 +321,7 @@ impl Pair { return Err(EBUSY); } let Validated::Render { - pass, attachments, .. + pass, attachments, ray_query, .. } = &packet.command else { return Err(EINVAL); @@ -385,6 +385,7 @@ impl Pair { let qids = self.qids(); let args = abi::RenderArgs { pass, + ray_query: *ray_query, resources: &resources, ordinal, ta: abi::RenderStage { @@ -585,7 +586,7 @@ impl Pair { self.memory.graph.queues.gpu_va() + u64::from(qid) * size_of::() as u64 }); - let Validated::Render { pass, .. } = &packet.command else { + let Validated::Render { pass, ray_query, .. } = &packet.command else { return Err(EINVAL); }; let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; @@ -609,13 +610,16 @@ impl Pair { mcache, event_mask: abi::RENDER_KICK_EVENT_MASK, register_arrays: arrays, - compute_scratch: false, + scs_enabled: ray_query[stage].scratch.enabled(), priority: policy.priority(), }; let ta_args = args( 0, tiling_barriers, - abi::TaDescriptor::register_bindings(self.memory.graph.descriptor_client(0, ordinal))?, + abi::TaDescriptor::register_bindings( + self.memory.graph.descriptor_client(0, ordinal), + ray_query[0].scratch.enabled(), + )?, None, ); let fragment_args = args( @@ -623,6 +627,7 @@ impl Pair { &barriers.fragment[..barriers.fragment_count], abi::FragmentDescriptor::register_bindings( self.memory.graph.descriptor_client(1, ordinal), + ray_query[1].scratch.enabled(), )?, mcache, ); diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 7509cf539c278..3d5f441d8428f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -70,6 +70,10 @@ impl job::Backend for Backend { Some(self.shared.feed.clone()) } + fn render_scs_blocks(&self, pages: [u16; 2]) -> Result { + crate::g17::fw::render_rt::required_blocks(self.shared.active_cores, pages) + } + fn ensure_compute(&self, scs_pages: u16) -> Result { Backend::ensure_compute(self, scs_pages) } diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 34fc306b66b4f..6341871961c4c 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -66,6 +66,11 @@ mod render; /// Trait implemented by all versioned queues. pub(crate) trait Queue: Send + Sync { + fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { + if layouts != [0; 2] { return Err(EINVAL); } + Ok(()) + } + fn submit( &mut self, reg_data: &AsahiRegData<'_>, diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 1053fc26d17c0..02e27073cf216 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -298,6 +298,12 @@ enum drm_asahi_feature { * DRM_ASAHI_RENDER_FRAGMENT_BARRIERS. */ DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS = (1UL) << 7, + /** + * @DRM_ASAHI_FEATURE_RENDER_RAY_QUERY: Render commands accept per-stage + * ray query state (struct drm_asahi_render_ray_query_stage) and queues + * accept render_scs_layout. + */ + DRM_ASAHI_FEATURE_RENDER_RAY_QUERY = (1UL) << 8, }; /** @@ -817,6 +823,13 @@ struct drm_asahi_queue_create { * fix a 4GiB VA carveout for USC memory and pass its base address here. */ __u64 usc_exec_base; + + /** + * @render_scs_layout: Largest ray query scratch layouts of the vertex and + * fragment stages this queue will submit, used to size its render pool. + * Requires DRM_ASAHI_FEATURE_RENDER_RAY_QUERY; zero otherwise. + */ + __u64 render_scs_layout[2]; }; /** @@ -1218,6 +1231,42 @@ struct drm_asahi_bg_eot { __u32 rsrc_spec; }; +/** + * struct drm_asahi_render_ray_query_stage - Ray query state of one render stage + * + * Index 0 describes the vertex (tiling) stage and index 1 the fragment stage + * of a render command. An all-zero entry disables ray queries in that stage. + * Requires DRM_ASAHI_FEATURE_RENDER_RAY_QUERY. + */ +struct drm_asahi_render_ray_query_stage { + /** @scs_layout: Per-thread ray query scratch layout, as for compute. */ + __u64 scs_layout; + + /** @intersection_aux: Auxiliary intersection table word. */ + __u32 intersection_aux; + + /** @intersection_pad: MBZ */ + __u32 intersection_pad; + + /** @intersection_pool: GPU VA of the intersection program pool. */ + __u64 intersection_pool; + + /** @intersection_pool_bytes: Size of the intersection program pool. */ + __u64 intersection_pool_bytes; + + /** @intersection_first: First intersection table entry. */ + __u32 intersection_first; + + /** @intersection_count: Number of intersection table entries. */ + __u32 intersection_count; + + /** @intersection_default: GPU VA of the default intersection program. */ + __u64 intersection_default; + + /** @intersection_default_control: Control word of the default program. */ + __u64 intersection_default_control; +}; + /** * struct drm_asahi_cmd_render - Command to submit 3D * @@ -1415,6 +1464,12 @@ struct drm_asahi_cmd_render { * queue's compute command count and the header's cdm_barrier instead. */ __u64 prior_cdm; + + /** + * @ray_query: Ray query state of the vertex and fragment stages. Older + * userspace omits it; a shorter command is read with both stages zeroed. + */ + struct drm_asahi_render_ray_query_stage ray_query[2]; }; /** From 864e8ab29da600c4861af69a97dcf68c35da6ba4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:17:40 +1100 Subject: [PATCH 1248/2680] dt-bindings: gpu: apple,agx: Match the T8140 register layout On T8140 the GFX and GFX1 coprocessor windows are owned by their "apple,t8140-ascwrap-v6" mailbox providers, and the GPU node only keeps the GFX coprocessor control registers ("asc") and the GPU block ("sgx"), which is all the driver requests. The binding still required a third "asc1" region, so dtbs_check rejected t8140-j700.dtb. Describe the two regions the device tree and the driver use. (cherry picked from commit f52e01d6925de5022176c5038ff13cc9ad17559b) Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/gpu/apple,agx.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Documentation/devicetree/bindings/gpu/apple,agx.yaml b/Documentation/devicetree/bindings/gpu/apple,agx.yaml index cc9c78a7624f8..e7afc4dd87e8f 100644 --- a/Documentation/devicetree/bindings/gpu/apple,agx.yaml +++ b/Documentation/devicetree/bindings/gpu/apple,agx.yaml @@ -30,12 +30,10 @@ properties: - const: apple,agx-g14s reg: - minItems: 2 - maxItems: 3 + maxItems: 2 reg-names: - minItems: 2 - maxItems: 3 + maxItems: 2 power-domains: maxItems: 1 @@ -93,12 +91,10 @@ allOf: reg: items: - description: GFX coprocessor control registers - - description: GFX1 coprocessor control registers - description: GPU block MMIO registers reg-names: items: - const: asc - - const: asc1 - const: sgx mboxes: items: From 0fa269cd017a90674da83800d544a52381b13804 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 01:32:28 +1100 Subject: [PATCH 1249/2680] soc: apple: sep: Require the DMA shared-buffer helpers The SEP firmware handoff calls dma_mapping_error(). On this base its Rust helper is compiled only when DMA_SHARED_BUFFER is enabled, but Kconfig allows APPLE_SEP without that symbol. Such configurations cannot link. Express the helper dependency so both built-in and modular SEP builds remain valid when DMA shared buffers are disabled elsewhere. Fixes: 24d20c9828b8 ("soc: apple: add Apple SEP driver (Touch ID and key sealing)") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 62a83309b1e58..558626f0be6be 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -118,6 +118,7 @@ config APPLE_SEP depends on ARCH_APPLE depends on PM depends on RUST + depends on DMA_SHARED_BUFFER depends on OF_DYNAMIC depends on SPI depends on RUST_APPLE_MAILBOX From 2cd0d1af4a6d83bb953b51347bebf0b54994ed9e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 22:12:20 +1100 Subject: [PATCH 1250/2680] reset: apple-cio: Describe the regmap selector in hardware terms Rewrite a set of comments so they state the hardware and firmware behaviour directly: - soc/apple SEP (sks.rs, image.rs, profile.rs, sep.rs): key-store request layouts, timeouts and endpoint init described as the enclave protocol. - soc/apple mailbox: the ASC wrapper CPU start/stop sequences. - accel/ane: register, handshake, ring and command-format notes for the T602x/T8112 Neural Engine, plus the scratch3_ack parameter text. - drm/asahi G17: the tiler-heap shrink policy. - mt7932, reset-apple-cio and two documentation pages: keep the descriptions self-contained. Every edit keeps its line count, so line-dependent data (panic locations, WARN/BUG and dynamic-debug records) is unchanged. Apart from the scratch3_ack module parameter description, the change is comments only; there is no functional change. [ryan: split by topic: the drivers/reset/reset-apple-cio.c hunk of 'treewide: describe hardware behaviour in comments'; its other hunks are in 010-soc, 080-radio, 100-sep, 210-gpu, 240-isp and 245-ane] (cherry picked from commit 9266541b8d35c1cd7146eec9a9b31b39fb7f29cc) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/reset/reset-apple-cio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/reset/reset-apple-cio.c b/drivers/reset/reset-apple-cio.c index 214a2aedce2e4..844aa803880e9 100644 --- a/drivers/reset/reset-apple-cio.c +++ b/drivers/reset/reset-apple-cio.c @@ -30,7 +30,7 @@ * "port = slot % 4", which the device tree expresses as one controller node * per die instead. The others index by slot directly. * - * "regmap" is an ApplePMGR::RegMap enum rather than an address. RegMap 0 is + * "regmap" is a PMGR register-map index rather than an address. RegMap 0 is * the PMGR block itself: t8103 resolves to 0x23b784000 and * t6020 to the pmgr node plus 0xa02c, both of which match this driver. The * t8122 and t6030 selectors are unresolved, so they cannot be wired up From a250922d994eb1b40fb38b16fd58ed8cf2ce22a8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:56:30 +1100 Subject: [PATCH 1251/2680] usb: typec: tipd: Tidy up the CD321x additions to tps6598x.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CD321x support placed the mux handle and its remove flag ahead of the device pointer in struct tps6598x, and included cd321x-pm.h in the middle of the header between a register define and the bit fields. Keep the includes together at the top, document why TPS_REG_MODE lives in the header, and move the mux handle next to the other port objects. Describe what remove_safe_prepared actually means instead of the previous shorthand. No functional change. [ryan: adapted to 7.2.10 — on the 7.2 tipd split the header had the cd321x-pm.h include inside the guard; it moves above it as in the original] (cherry picked from commit ec606af097b9ad4e9bbf9d55d7e25cecf5343b3e) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/tps6598x.h | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index b241541715df0..c730c90a40fe0 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -15,11 +15,12 @@ #include #include +#include "cd321x-pm.h" + #ifndef __TPS6598X_H__ #define __TPS6598X_H__ -#include "cd321x-pm.h" - +/* Also read by the SPMI glue to tell when the controller has woken up */ #define TPS_REG_MODE 0x03 #define TPS_FIELD_GET(_mask, _reg) ((typeof(_mask))(((_reg) & (_mask)) >> __bf_shf(_mask))) @@ -323,9 +324,6 @@ struct tipd_data { }; struct tps6598x { - struct typec_mux *mux; - /* CD remove leaves one open SAFE transaction for generic completion. */ - bool remove_safe_prepared; struct device *dev; struct regmap *regmap; struct mutex lock; /* device lock */ @@ -339,6 +337,12 @@ struct tps6598x { struct usb_pd_identity partner_identity; struct usb_role_switch *role_sw; struct typec_capability typec_cap; + struct typec_mux *mux; + /* + * Set by ->remove() once it has parked the mux in SAFE, so that + * tps6598x_disconnect() only ends that mux transaction. + */ + bool remove_safe_prepared; struct power_supply *psy; struct power_supply_desc psy_desc; From aa07fca5b87351c0bb3ffbcea35c65dc0614caca Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:57:02 +1100 Subject: [PATCH 1252/2680] usb: typec: tipd: Fix coding style in the CD321x update path Fix the issues checkpatch --strict reports in code added for CD321x: misaligned continuation lines, two lines over 100 columns, an else branch after a goto, and a double blank line. Reword the comment on the shared display connector, which used double spaces and repeated itself, and the comment on the SAFE transaction left open by cd321x_remove(). Move the DRM include below the linux/ ones, where the DisplayPort Alt Mode driver has it, instead of ahead of them. No functional change. (cherry picked from commit 21cbed38d507b58251d22e34d840acdaeea3a97c) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/core.c | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 12c0b5330611a..85b5e17635c24 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -6,8 +6,6 @@ * Author: Heikki Krogerus */ -#include - #include #include #include @@ -22,6 +20,7 @@ #include #include #include +#include #include "tps6598x.h" #include "trace.h" @@ -213,7 +212,7 @@ static int tps6598x_read_partner_identity(struct tps6598x *tps) } static int tps6598x_set_data_role(struct tps6598x *tps, - enum typec_data_role role, bool connected) + enum typec_data_role role, bool connected) { enum usb_role role_val; int ret; @@ -810,21 +809,19 @@ static void cd321x_update_work(struct work_struct *work) was_disconnected |= partner_changed; /* - * Report sink absence before any fallible hardware transition. A failed - * shutdown retains the route/PHY owner, but cannot keep the sink present. - * Every Type-C port references the same external DCP connector. Only the - * port which successfully acquired the shared display route may report a - * disconnect. A second DP sink can negotiate Alt Mode even though its mux - * request is rejected with -EBUSY; treating that sink as the connector - * owner would tear down the display carried by the first port. + * Report the sink as gone before any hardware transition that can fail: + * a failed teardown keeps the route, but the sink must not stay present. + * All ports reference the same external display connector, so only the + * port that owns the display route may report a disconnect. A second DP + * sink can enter Alt Mode while its mux request fails with -EBUSY, and + * must not tear down the display carried by the other port. */ if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && dp_route_was_active && (!new_connected || was_disconnected || !dp_connected || !dp_hpd || - dp_hpd_changed || dp_mode_changed)) { + dp_hpd_changed || dp_mode_changed)) drm_connector_oob_hotplug_event(cd321x->connector_fwnode, - connector_status_disconnected); - } + connector_status_disconnected); /* * ACIO carries the tunneled PCIe reset handshake over the still-live @@ -871,9 +868,8 @@ static void cd321x_update_work(struct work_struct *work) ret); cd321x->state = old_state; goto retry; - } else { - cd321x->display_route_active = false; } + cd321x->display_route_active = false; } /* Update Type-C properties */ @@ -934,7 +930,8 @@ static void cd321x_update_work(struct work_struct *work) if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && cd321x->display_route_active && dp_hpd) - drm_connector_oob_hotplug_event(cd321x->connector_fwnode, connector_status_connected); + drm_connector_oob_hotplug_event(cd321x->connector_fwnode, + connector_status_connected); power_supply_changed(tps->psy); cd321x_pm_complete(&cd321x->pm); @@ -948,7 +945,8 @@ static void cd321x_update_work(struct work_struct *work) static void cd321x_queue_status(struct cd321x *cd321x) { cd321x->update_status.status_changed |= cd321x->update_status.status ^ cd321x->tps.status; - cd321x->update_status.data_status_changed |= cd321x->update_status.data_status ^ cd321x->tps.data_status; + cd321x->update_status.data_status_changed |= + cd321x->update_status.data_status ^ cd321x->tps.data_status; cd321x->update_status.status = cd321x->tps.status; cd321x->update_status.pwr_status = cd321x->tps.pwr_status; @@ -2052,12 +2050,11 @@ static int cd321x_remove(struct tps6598x *tps) typec_mux_abort(tps->mux); return ret; } - /* Same task/handle completes without another post-shutdown set callback. */ + /* Leave the SAFE transaction open for tps6598x_disconnect() to end. */ tps->remove_safe_prepared = true; return 0; } - int tipd_init(struct tps6598x *tps) { struct fwnode_handle *fwnode; From a8df0adaf9b48573096a6463533f5dfd95d6bf7e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1253/2680] wifi: mt7932: drop the diagnostic module parameters legacy_irq_test, poll_test and reset_test were bring-up switches. They forced INTx, replaced interrupts with polling of the rings, and ran an extra function reset and firmware boot before the interface was registered. All three default to off, nothing sets them, and none of them is needed to run or debug the station. Remove them with the code only they reached: the polled completion wait, mt_transport_polled() and mt_reset_retained(). With the defaults the driver behaves as before. (cherry picked from commit 41097c6b96cd4f7c18445de1c83ba5350b117b1d) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/Kconfig | 3 +- drivers/net/wireless/mediatek/mt7932/mt7932.c | 106 ++---------------- drivers/net/wireless/mediatek/mt7932/mt7932.h | 2 - .../net/wireless/mediatek/mt7932/recovery.c | 2 +- 4 files changed, 11 insertions(+), 102 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index b12b2362efb97..273c389ad09bd 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -15,8 +15,7 @@ config MT7932_FULLMAC unless those files are in the initramfs. Requires a trained retained PCI link and a translated SID1 DMA domain. Firmware, runtime DMA and calibration negotiation start automatically; - cfg80211 and userspace select the country and network. No test boot - parameters are required for ordinary station operation. + cfg80211 and userspace select the country and network. Once DMA pointers are published the module is pinned until hardware reset. Normal unload is refused, including with active Bluetooth. A failed probe reset retains the binding and DMA; a successful checked diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index ebe7087ee030b..89e36ca28a6d4 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -41,22 +41,6 @@ static int mt_initialize(struct mt7932 *m); static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id); static void mt_shutdown(struct pci_dev *pdev); -static bool legacy_irq_test; -module_param(legacy_irq_test, bool, 0444); -MODULE_PARM_DESC(legacy_irq_test, "Explicit diagnostic use of standard PCI INTx instead of MSI"); -static bool poll_test; -module_param(poll_test, bool, 0444); -MODULE_PARM_DESC(poll_test, "Diagnostic ROM/firmware polling with endpoint IRQ generation masked"); -static bool reset_test; -module_param(reset_test, bool, 0444); -MODULE_PARM_DESC(reset_test, "Qualify one FLR and fresh firmware epoch before publishing an interface, retaining old DMA"); - -bool mt_transport_polled(void) -{ - return poll_test; -} - - bool mt_rf_allowed(struct mt7932 *m) { return !READ_ONCE(m->stopping) && READ_ONCE(m->rf_ready) && !READ_ONCE(m->reg_pending) && @@ -313,7 +297,7 @@ static void mt_service(struct mt7932 *m) for (i = 0; i < ARRAY_SIZE(m->rx); i++) if (m->rx[i].count) mt_rx_drain(m, &m->rx[i]); - if (READ_ONCE(m->running) && !poll_test) + if (READ_ONCE(m->running)) mt_write(m, W + 0x204, m->irq_mask); } @@ -395,22 +379,9 @@ int mt_request_ext(struct mt7932 *m, u8 cid, u8 ext, bool runtime, bool set, boo m->reply_length = 0; spin_unlock_irqrestore(&m->response_lock, flags); ret = mt_submit(m, q, ret); - if (!ret && wait) { - if (poll_test) { - unsigned long deadline = jiffies + msecs_to_jiffies(5000); - - do { - mt_service(m); - if (completion_done(&m->response)) - break; - usleep_range(100, 200); - } while (time_before(jiffies, deadline)); - if (!completion_done(&m->response)) - ret = -ETIMEDOUT; - } else if (!wait_for_completion_timeout(&m->response, msecs_to_jiffies(5000))) { - ret = -ETIMEDOUT; - } - } + if (!ret && wait && + !wait_for_completion_timeout(&m->response, msecs_to_jiffies(5000))) + ret = -ETIMEDOUT; if (READ_ONCE(m->stopping)) ret = -ESHUTDOWN; if (ret == -ETIMEDOUT) { @@ -504,7 +475,7 @@ static int mt_dma_setup(struct mt7932 *m) u32 glo; mt_write(m, W + 0x204, 0); - mt_write(m, 0x10188, poll_test ? 0 : 0xff); + mt_write(m, 0x10188, 0xff); mt_rmw(m, W + 0x208, BIT(0) | BIT(2) | BIT(15) | BIT(21) | BIT(27) | BIT(28), 0); ret = mt_poll(m, W + 0x208, BIT(1) | BIT(3), 0, 100000); if (ret) @@ -598,7 +569,7 @@ static int mt_dma_setup(struct mt7932 *m) mt_write(m, W + 0x2f0, 0x8032800a); mt_rmw(m, 0x2120, 0, BIT(1)); mt_write(m, W + 0x1f4, 0xffff); - mt_write(m, W + 0x204, poll_test ? 0 : IRQ_MASK); + mt_write(m, W + 0x204, IRQ_MASK); dev_info(&m->pdev->dev, "DMA_PUBLISHED glo=%08x TX17=%pad RX0=%pad IPC=%pad\n", mt_read(m, W + 0x208), &m->tx[17].dma, &m->rx[0].dma, &m->ipc_dma); return 0; @@ -636,8 +607,8 @@ static int mt_rom_gate(struct mt7932 *m) dev_info(&m->pdev->dev, "ROM_READ_OK %08x=%08x seq=%u IRQ=%llu\n", addresses[i], get_unaligned_le32(m->reply + 36), m->seq, m->interrupts); } - dev_info(&m->pdev->dev, "ROM_TRANSPORT_OK mode=%s IRQ=%llu (no firmware download or RF yet)\n", - poll_test ? "polled-test" : "interrupt", m->interrupts); + dev_info(&m->pdev->dev, "ROM_TRANSPORT_OK IRQ=%llu (no firmware download or RF yet)\n", + m->interrupts); return 0; } @@ -1113,63 +1084,6 @@ static int mt_initialize(struct mt7932 *m) return ret; } -static int mt_reset_retained(struct mt7932 **epoch) -{ - struct mt7932 *old = *epoch, *fresh; - struct pci_dev *pdev = old->pdev; - int irq = pci_irq_vector(pdev, 0), ret; - - /* No interface/work producers have been published at this test gate. */ - WRITE_ONCE(old->running, false); - mt_write(old, W + 0x204, 0); - devm_free_irq(&pdev->dev, irq, old); - old->irq_requested = false; - ret = mt_ownership(old); - if (!ret) - ret = mt_dma_stop(old); - if (!ret) - ret = mt_function_reset(old); - if (ret) - return ret; - old->dma_owned = false; - if (old->module_pinned) { - old->module_pinned = false; - module_put(THIS_MODULE); - } - dev_info(&pdev->dev, "RESET_TEST: function reset complete; retaining old DMA\n"); - fresh = devm_kzalloc(&pdev->dev, sizeof(*fresh), GFP_KERNEL); - if (!fresh) - return -ENOMEM; - fresh->pdev = pdev; - fresh->bar = old->bar; - fresh->irq_mask = IRQ_MASK; - fresh->vectors_allocated = true; - ether_addr_copy(fresh->mac, old->mac); - init_completion(&fresh->response); - init_completion(&fresh->cal_response); - init_completion(&fresh->reset_retry); - mutex_init(&fresh->command_mutex); - spin_lock_init(&fresh->response_lock); - *epoch = fresh; - pci_set_drvdata(pdev, fresh); - ret = mt_ownership(fresh); - if (ret) - return ret; - ret = mt_poll(fresh, 0xf0140, BIT(4), BIT(4), 200000); - if (ret) - return ret; - mt_write(fresh, W + 0x204, 0); - ret = devm_request_threaded_irq(&pdev->dev, irq, mt_irq, mt_irq_thread, - IRQF_ONESHOT | IRQF_SHARED, "mt7932", fresh); - if (ret) - return ret; - fresh->irq_requested = true; - ret = mt_initialize(fresh); - if (!ret) - dev_info(&pdev->dev, "RESET_TEST: fresh ROM and firmware gates passed\n"); - return ret; -} - /* A failed PCI reset cannot be turned into successful removal: devres would * free still-owned DMA. Keep the remove callback (and therefore its device, * mappings and module text) alive. This explicit retry does not run hardware @@ -1307,7 +1221,7 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) goto retain; mt_write(m, W + 0x204, 0); - ret = pci_alloc_irq_vectors(pdev, 1, 1, legacy_irq_test ? PCI_IRQ_INTX : PCI_IRQ_ALL_TYPES); + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES); if (ret < 0) goto retain; m->vectors_allocated = true; @@ -1338,8 +1252,6 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) irq->chip->name, irq->hwirq); } ret = mt_initialize(m); - if (!ret && reset_test) - ret = mt_reset_retained(&m); if (!ret) ret = mt_register_regulatory_gate(m); if (!ret) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 8151e794bfabf..1c2262952e82e 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -253,6 +253,4 @@ void mt_power_work(struct work_struct *work); int mt_power_init(struct mt7932 *m); void mt_station_update_rate(struct mt7932 *m, const u8 *mac); -bool mt_transport_polled(void); - #endif diff --git a/drivers/net/wireless/mediatek/mt7932/recovery.c b/drivers/net/wireless/mediatek/mt7932/recovery.c index f0b7cc9714c24..cdb327bb27e39 100644 --- a/drivers/net/wireless/mediatek/mt7932/recovery.c +++ b/drivers/net/wireless/mediatek/mt7932/recovery.c @@ -192,7 +192,7 @@ int mt_recovery_gate(struct mt7932 *m) int ret; /* mt_runtime_start() completed configuration before entering this stage. */ - if (mt_transport_polled() || m->runtime_epoch) + if (m->runtime_epoch) return -EINVAL; for (i = 0; i < ARRAY_SIZE(queue_config); i++) { ret = mt_request(m, 0x6b, true, true, false, queue_config[i], 32); From af2c4b76b93015734fbd872f77e781cf95cc4547 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:57:22 +1100 Subject: [PATCH 1254/2680] usb: typec: tipd: Simplify the DisplayPort connector lookup fwnode_find_reference() already fails with -ENOENT when the property is missing and never returns NULL, so the separate presence check and the IS_ERR_OR_NULL() test are redundant. Look the reference up directly and drop the error pointer, as the DisplayPort Alt Mode driver does. No functional change. (cherry picked from commit 1cd13c2e1c3e95514d7a00699bb445fc8b36670b) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/core.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 85b5e17635c24..d41ed0b35957e 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -1507,7 +1507,6 @@ static int cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) { struct cd321x *cd321x = container_of(tps, struct cd321x, tps); - struct fwnode_handle *connector_fwnode = NULL; int ret; INIT_DELAYED_WORK(&cd321x->update_work, cd321x_update_work); @@ -1529,10 +1528,10 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) goto err_unregister_altmodes; } - if (fwnode_property_present(fwnode, "displayport")) - connector_fwnode = fwnode_find_reference(fwnode, "displayport", 0); - if (!IS_ERR_OR_NULL(connector_fwnode)) - cd321x->connector_fwnode = connector_fwnode; + cd321x->connector_fwnode = fwnode_find_reference(fwnode, "displayport", 0); + if (IS_ERR(cd321x->connector_fwnode)) + cd321x->connector_fwnode = NULL; + cd321x->tbt_switch = fwnode_typec_thunderbolt_switch_get(fwnode); if (IS_ERR(cd321x->tbt_switch)) { ret = PTR_ERR(cd321x->tbt_switch); From 5bafbaea32a695523804ce12fec9d36a25a43889 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1255/2680] wifi: mt7932: remove the unused station rate query mt_station_update_rate() queried the firmware for the transmit rate of the associated access point. It has never been called: the reply comes from a transmit vector that the firmware shares between stations, so station reports leave the transmit bitrate out. Remove it, the reply parser and the station fields that only it used. No functional change. (cherry picked from commit 08b907740a4ff01fa7c531363747b10ee1a79c75) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/Makefile | 2 +- drivers/net/wireless/mediatek/mt7932/link.c | 3 - drivers/net/wireless/mediatek/mt7932/mt7932.h | 4 -- drivers/net/wireless/mediatek/mt7932/rate.h | 29 --------- .../wireless/mediatek/mt7932/station_query.c | 65 ------------------- 5 files changed, 1 insertion(+), 102 deletions(-) delete mode 100644 drivers/net/wireless/mediatek/mt7932/station_query.c diff --git a/drivers/net/wireless/mediatek/mt7932/Makefile b/drivers/net/wireless/mediatek/mt7932/Makefile index 9e76679c46a8a..f854d21e5db81 100644 --- a/drivers/net/wireless/mediatek/mt7932/Makefile +++ b/drivers/net/wireless/mediatek/mt7932/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_MT7932_FULLMAC) += mt7932-fullmac.o -mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o station_query.o +mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += rf-test.o diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 9f857417a5ce3..999d0bd16c290 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -532,7 +532,6 @@ void mt_connect_work(struct work_struct *work) m->station_connected = jiffies; m->station_signal_valid = false; m->station_rx_rate_valid = false; - m->station_tx_rate_valid = false; spin_unlock(&m->data_lock); mt_cqm_reset(m); } @@ -643,8 +642,6 @@ int mt_connect(struct wiphy *wiphy, struct net_device *netdev, m->connect_center = 0; memset(&m->connect_chandef, 0, sizeof(m->connect_chandef)); m->connection_generation++; - m->station_rate_query_failed = false; - m->station_rate_query_time = jiffies - HZ; m->connect_cancelled = false; m->peer_valid = false; m->connecting = true; diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 1c2262952e82e..688654acac97d 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -161,9 +161,6 @@ struct mt7932 { struct rate_info station_rx_rate; unsigned long station_rx_rate_time; bool station_rx_rate_valid; - struct rate_info station_tx_rate; - unsigned long station_tx_rate_time, station_rate_query_time; - bool station_tx_rate_valid, station_rate_query_failed; /* Connection quality monitor, protected by response_lock. */ struct ewma_mt7932_signal cqm_signal; unsigned long cqm_sample_time; @@ -251,6 +248,5 @@ int mt_set_power_mgmt(struct wiphy *wiphy, struct net_device *netdev, bool enabled, int timeout); void mt_power_work(struct work_struct *work); int mt_power_init(struct mt7932 *m); -void mt_station_update_rate(struct mt7932 *m, const u8 *mac); #endif diff --git a/drivers/net/wireless/mediatek/mt7932/rate.h b/drivers/net/wireless/mediatek/mt7932/rate.h index d570a69f81f44..7e7c5685d40fd 100644 --- a/drivers/net/wireless/mediatek/mt7932/rate.h +++ b/drivers/net/wireless/mediatek/mt7932/rate.h @@ -65,33 +65,4 @@ static inline bool mt7932_rx_rate(u32 rxv, u8 max_nss, struct rate_info *rate) return true; } -/* Stock CID85 reply: exact version1 body, owned BSS/MAC and pending sequence. - * Byte8's station identity and byte0a remain opaque; no invented WTBL test. - */ -static inline bool mt7932_tx_rate_reply(const struct mt7932_event *event, - u8 sequence, const u8 *peer, u8 max_nss, - struct rate_info *rate) -{ - const u8 *body; - u32 a, b, c, rxv; - - if (!mt7932_event_matches(event, sequence) || event->eid != 0x21 || - event->length != 344) - return false; - body = event->packet + 36; - if (body[0] != 1 || - !(get_unaligned_le32(body + 4) & 1) || body[9] || - memcmp(body + 12, peer, 6)) - return false; - a = get_unaligned_le32(body + 0x9c); - b = get_unaligned_le32(body + 0xa0); - c = get_unaligned_le32(body + 0xa4); - if (((a >> 12) & 15) >= 5) - return false; /* TX HE GI/DCM/allocation fields not qualified by this reply. */ - /* Normalize the independently qualified TX words to the shared fields. */ - rxv = (c & 127) | (((c >> 8) & 7) << 7) | - (((a >> 8) & 7) << 12) | (((b >> 26) & 3) << 15) | - (((a >> 6) & 3) << 22) | (((a >> 12) & 15) << 24); - return mt7932_rx_rate(rxv, max_nss, rate); -} #endif diff --git a/drivers/net/wireless/mediatek/mt7932/station_query.c b/drivers/net/wireless/mediatek/mt7932/station_query.c deleted file mode 100644 index 8382768898172..0000000000000 --- a/drivers/net/wireless/mediatek/mt7932/station_query.c +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -#include "mt7932.h" - -/* Research helper, deliberately not called by production station reporting: - * the reply identity is per-peer, but its vector is shared and unattributed. - * Retained while qualifying a real per-peer rate source, not a public metric. - * One serialized query for the owned peer. Keep the command owner until the - * reply has been checked; disconnect may close admission in IRQ context but - * cannot reuse BSS/station resources across this critical section. - */ -void mt_station_update_rate(struct mt7932 *m, const u8 *mac) -{ - struct mt7932_event event; - struct rate_info rate; - u8 body[28] = {}, peer[ETH_ALEN]; - u32 generation; - unsigned long flags; - bool valid = false; - int ret; - - mutex_lock(&m->command_mutex); - spin_lock_irqsave(&m->response_lock, flags); - if (!m->connected || m->disconnecting || !m->peer_valid || !mt_rf_allowed(m) || - m->station_rate_query_failed || (mac && !ether_addr_equal(mac, m->connect_bssid)) || - time_before(jiffies, m->station_rate_query_time + HZ / 2)) { - spin_unlock_irqrestore(&m->response_lock, flags); - goto unlock; - } - generation = m->connection_generation; - ether_addr_copy(peer, m->connect_bssid); - body[0] = m->peer_ids.station[0]; - memcpy(body + 4, peer, ETH_ALEN); - m->station_rate_query_time = jiffies; - spin_unlock_irqrestore(&m->response_lock, flags); - - ret = mt_request(m, 0x85, true, false, true, body, sizeof(body)); - if (!ret) - ret = mt7932_event_parse(m->reply, m->reply_length, &event); - if (!ret) - valid = mt7932_tx_rate_reply(&event, m->seq, peer, m->phy_cap[4], &rate); - if (valid && (rate.bw != RATE_INFO_BW_20 || - (m->connect_channel > 14 && !rate.flags && - (rate.legacy == 10 || rate.legacy == 20 || rate.legacy == 55 || rate.legacy == 110)))) - valid = false; - - spin_lock_irqsave(&m->response_lock, flags); - if (generation == m->connection_generation && m->connected && - !m->disconnecting && m->peer_valid && ether_addr_equal(peer, m->connect_bssid)) { - /* A timed-out command must not turn every station dump into another - * five-second wait. A new connection/module epoch may try again. - */ - if (ret) - m->station_rate_query_failed = true; - spin_lock(&m->data_lock); - m->station_tx_rate_valid = valid; - if (valid) { - m->station_tx_rate = rate; - m->station_tx_rate_time = jiffies; - } - spin_unlock(&m->data_lock); - } - spin_unlock_irqrestore(&m->response_lock, flags); -unlock: - mutex_unlock(&m->command_mutex); -} From 9b5ef1dc972594fd98c45257fdfab0bc3d1853eb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:57:48 +1100 Subject: [PATCH 1256/2680] usb: typec: tipd: Name the sleep hooks after their PM callbacks The tipd_data hooks called from tipd_prepare() and tipd_complete() were named suspend_prepare and resume_reverify. The latter also runs after an aborted suspend, so "resume" is misleading, and neither name matches the dev_pm_ops callback it serves. Call them prepare and complete, and rename the CD321x implementations to match. No functional change. (cherry picked from commit b2ffd97a8ce544bfdaca0b04bac70c211ffcb933) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/core.c | 20 ++++++++++---------- drivers/usb/typec/tipd/tps6598x.h | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index d41ed0b35957e..95a7d75ba3b1d 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -1037,7 +1037,7 @@ static void cd321x_resume_work(struct work_struct *work) cd321x_retry_revalidation(cd321x, true); } -static void cd321x_suspend_prepare(struct tps6598x *tps) +static void cd321x_prepare(struct tps6598x *tps) { struct cd321x *cd321x = container_of(tps, struct cd321x, tps); @@ -1050,7 +1050,7 @@ static void cd321x_suspend_prepare(struct tps6598x *tps) cancel_delayed_work_sync(&cd321x->update_work); } -static void cd321x_resume_reverify(struct tps6598x *tps) +static void cd321x_complete(struct tps6598x *tps) { struct cd321x *cd321x = container_of(tps, struct cd321x, tps); @@ -2300,8 +2300,8 @@ int tipd_prepare(struct device *dev) { struct tps6598x *tps = dev_get_drvdata(dev); - if (tps->data->suspend_prepare) - tps->data->suspend_prepare(tps); + if (tps->data->prepare) + tps->data->prepare(tps); return 0; } @@ -2312,8 +2312,8 @@ void tipd_complete(struct device *dev) { struct tps6598x *tps = dev_get_drvdata(dev); - if (tps->data->resume_reverify) - tps->data->resume_reverify(tps); + if (tps->data->complete) + tps->data->complete(tps); } EXPORT_SYMBOL_GPL(tipd_complete); @@ -2335,8 +2335,8 @@ static const struct tipd_data tipd_cd321x_data = { .reset = cd321x_reset, .switch_power_state = cd321x_switch_power_state, .connect = cd321x_connect, - .suspend_prepare = cd321x_suspend_prepare, - .resume_reverify = cd321x_resume_reverify, + .prepare = cd321x_prepare, + .complete = cd321x_complete, }; static const struct tipd_data tipd_tps6598x_data = { @@ -2393,8 +2393,8 @@ static const struct tipd_data tipd_sn201202x_data = { .reset = cd321x_reset, .switch_power_state = cd321x_switch_power_state, .connect = cd321x_connect, - .suspend_prepare = cd321x_suspend_prepare, - .resume_reverify = cd321x_resume_reverify, + .prepare = cd321x_prepare, + .complete = cd321x_complete, }; const struct tipd_data *tipd_hw_variants[] = { diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index c730c90a40fe0..1e550956e86b9 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -319,8 +319,8 @@ struct tipd_data { bool (*read_data_status)(struct tps6598x *tps); int (*reset)(struct tps6598x *tps); int (*connect)(struct tps6598x *tps, u32 status); - void (*suspend_prepare)(struct tps6598x *tps); - void (*resume_reverify)(struct tps6598x *tps); + void (*prepare)(struct tps6598x *tps); + void (*complete)(struct tps6598x *tps); }; struct tps6598x { From 3353e5ba7db74e1da6c5bb0ac75de72f750b383d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1257/2680] wifi: mt7932: report driver progress at debug level The driver logged each firmware boot step, every scan request and completion, every association event and each calibration request at info level. A desktop scans every few seconds, so this filled the kernel log with SCAN_REQUEST, SCAN_SUBMITTED, LINK_EVENT and DISCONNECT_EVENT lines, plus hex dumps of firmware replies and DMA addresses at probe. Log routine progress with dev_dbg() and print_hex_dump_debug(), so it is still available through dynamic debug. Errors and warnings keep their level and text. Also drop two diagnostics only used during bring-up: the dump of the MSI configuration and interrupt hierarchy at probe, and the register and descriptor dump on a command timeout. A timed-out command still collects a late reply before reporting the error, and the hex dump of a rejected calibration upload reply goes away, since the error line already has the fields that matter. A few of the demoted messages read device status registers. Those reads now happen only while the message is enabled; each follows the driver's own accesses to the same block, so nothing depends on them. (cherry picked from commit a257e5168baa8048f3187203b7e18e53d21db4e2) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../mediatek/mt7932/calibration_transport.c | 13 +- drivers/net/wireless/mediatek/mt7932/data.c | 6 +- .../net/wireless/mediatek/mt7932/frontend.c | 24 +-- drivers/net/wireless/mediatek/mt7932/link.c | 64 ++++--- drivers/net/wireless/mediatek/mt7932/mt7932.c | 156 +++++++----------- drivers/net/wireless/mediatek/mt7932/mt7932.h | 1 - .../net/wireless/mediatek/mt7932/recovery.c | 6 +- .../net/wireless/mediatek/mt7932/startup.c | 22 +-- 8 files changed, 132 insertions(+), 160 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c index 1d6d4d6985a45..fb5a33dd5aa61 100644 --- a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c +++ b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c @@ -124,12 +124,10 @@ int mt_calibration_gate(struct mt7932 *m) get_unaligned_le32(m->reply + 40)) { dev_err(&m->pdev->dev, "WCAL_REPLY_REJECTED: length=%zu eid=%02x ext=%02x\n", m->reply_length, m->reply[28], m->reply[32]); - print_hex_dump(KERN_INFO, "WCAL_RESULT: ", DUMP_PREFIX_OFFSET, 16, 1, - m->reply, min_t(size_t, m->reply_length, 64), false); ret = -EPROTO; goto out; } - dev_info(&m->pdev->dev, "OWN_WCAL_ACCEPTED: %zu bytes\n", wcal->size); + dev_dbg(&m->pdev->dev, "OWN_WCAL_ACCEPTED: %zu bytes\n", wcal->size); phase = "power-on D6 command/completion"; for (at = 0, group = 0; at < planned; at += count, group++) { count = pieces[at].fragment >> 4; @@ -140,10 +138,10 @@ int mt_calibration_gate(struct mt7932 *m) ret = mt_cal_procedure(m, pieces + at, count, 1, m->preload_version); if (ret) goto out; - dev_info(&m->pdev->dev, "POWER_ON_CAL_GROUP_OK: %u type=%u parameter=%08x fragments=%u\n", - group, pieces[at].type, pieces[at].parameter, count); + dev_dbg(&m->pdev->dev, "POWER_ON_CAL_GROUP_OK: %u type=%u parameter=%08x fragments=%u\n", + group, pieces[at].type, pieces[at].parameter, count); } - dev_info(&m->pdev->dev, "POWER_ON_CAL_COMPLETE: %u groups, %d fragments\n", group, planned); + dev_dbg(&m->pdev->dev, "POWER_ON_CAL_COMPLETE: %u groups, %d fragments\n", group, planned); phase = "queued D7 planning/command/completion"; ret = mt_cal_drain_requests(m, oca); out: @@ -187,7 +185,8 @@ int mt_cal_drain_requests(struct mt7932 *m, const struct firmware *oca) mt7932_cal_request_replies(request), m->preload_version); if (ret) return ret; - dev_info(&m->pdev->dev, "CAL_REQUEST_SERVICED: channel=%u\n", get_unaligned_le32(request + 12)); + dev_dbg(&m->pdev->dev, "CAL_REQUEST_SERVICED: channel=%u\n", + get_unaligned_le32(request + 12)); } return 0; } diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index 9e6a51f8e3987..e4f9d8a9f9cc8 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -313,10 +313,10 @@ int mt_data_prepare(struct mt7932 *m) /* The selected response parser establishes sequence completion only, * not a command-specific firmware status. No payload-status invention. */ - dev_info(&m->pdev->dev, "MAC_RTS_RESPONSE: sequence matched, EID=%02x length=%zu\n", - m->reply[28], m->reply_length); + dev_dbg(&m->pdev->dev, "MAC_RTS_RESPONSE: sequence matched, EID=%02x length=%zu\n", + m->reply[28], m->reply_length); /* Publish the initialized data_lock and DMA state to failure handlers. */ smp_store_release(&m->data_ready, true); - dev_info(&m->pdev->dev, "ETHERNET_PREPARED: TX0 retained coherent slots=512, RX translation enabled\n"); + dev_dbg(&m->pdev->dev, "ETHERNET_PREPARED: TX0 retained coherent slots=512, RX translation enabled\n"); return 0; } diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index aa36c68a10691..7b0259bde004a 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -64,8 +64,8 @@ static void mt_scan_finish_work(struct work_struct *work) } mutex_unlock(&m->command_mutex); if (request) { - dev_info(&m->pdev->dev, "SCAN_FINISHED: seq=%u aborted=%u received-BSS=%llu\n", - cancel[0], info.aborted, m->beacons); + dev_dbg(&m->pdev->dev, "SCAN_FINISHED: seq=%u aborted=%u received-BSS=%llu\n", + cancel[0], info.aborted, m->beacons); cfg80211_scan_done(request, &info); } } @@ -226,8 +226,8 @@ static int mt_scan_submit_next(struct mt7932 *m) ret = mt_request(m, 3, true, true, false, body, sizeof(body)); } if (!ret) - dev_info(&m->pdev->dev, "SCAN_SUBMITTED: seq=%u band=%u channels=%u passive=%u\n", - m->scan_seq, m->scan_band, count, passive); + dev_dbg(&m->pdev->dev, "SCAN_SUBMITTED: seq=%u band=%u channels=%u passive=%u\n", + m->scan_seq, m->scan_band, count, passive); return ret; } @@ -350,9 +350,10 @@ int mt_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) if (request->wdev != &m->wdev) return -ENODEV; mt_retry_missing_policy(m); - dev_info_ratelimited(&m->pdev->dev, "SCAN_REQUEST: channels=%u ssids=%d ies=%zu flags=%08x duration=%u/%u\n", - request->n_channels, request->n_ssids, request->ie_len, request->flags, - request->duration, request->duration_mandatory); + dev_dbg(&m->pdev->dev, + "SCAN_REQUEST: channels=%u ssids=%d ies=%zu flags=%08x duration=%u/%u\n", + request->n_channels, request->n_ssids, request->ie_len, request->flags, + request->duration, request->duration_mandatory); /* cfg80211 lists each advertised channel at most once. */ if (request->n_channels > MT7932_CHANNELS) return -EOPNOTSUPP; @@ -547,7 +548,8 @@ static void mt_cal_work(struct work_struct *work) m->preload_version); if (ret) break; - dev_info(&m->pdev->dev, "D7_CALIBRATION_COMPLETE: channel=%u\n", get_unaligned_le32(request + 12)); + dev_dbg(&m->pdev->dev, "D7_CALIBRATION_COMPLETE: channel=%u\n", + get_unaligned_le32(request + 12)); } release_firmware(oca); if (!ret) @@ -601,7 +603,8 @@ int mt_register_interface(struct mt7932 *m) free_netdev(netdev); return ret; } - dev_info(&m->pdev->dev, "CFG80211_INTERFACE_READY: %s, RF awaits country policy\n", netdev->name); + dev_dbg(&m->pdev->dev, "CFG80211_INTERFACE_READY: %s, RF awaits country policy\n", + netdev->name); return 0; } @@ -637,7 +640,8 @@ int mt_enable_scan(struct mt7932 *m) return ret; } WRITE_ONCE(m->rf_ready, true); - dev_info(&m->pdev->dev, "CFG80211_SCAN_READY: %s, carrier off until WPA2 authorization\n", m->netdev->name); + dev_dbg(&m->pdev->dev, "CFG80211_SCAN_READY: %s, carrier off until WPA2 authorization\n", + m->netdev->name); schedule_work(&m->cal_work); spin_unlock_irqrestore(&m->response_lock, flags); return 0; diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 999d0bd16c290..bcdcb71ce3db9 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -113,11 +113,11 @@ void mt_link_event(struct mt7932 *m, const struct mt7932_event *event) selector = get_unaligned_le32(event->packet + 40); body = event->packet + 48; length = event->length - 48; - dev_info(&m->pdev->dev, "LINK_EVENT: selector=%02x length=%zu\n", selector, length); + dev_dbg(&m->pdev->dev, "LINK_EVENT: selector=%02x length=%zu\n", selector, length); if (selector == 0x44) { if (length < 5 || (!m->connecting && !m->connected && !m->disconnecting)) return; - dev_info(&m->pdev->dev, "DISCONNECT_EVENT: metadata=%u deferred=%u reason-index=%u\n", + dev_dbg(&m->pdev->dev, "DISCONNECT_EVENT: metadata=%u deferred=%u reason-index=%u\n", body[0], body[3], body[4]); if (m->disconnect_seen) return; @@ -179,27 +179,28 @@ void mt_link_event(struct mt7932 *m, const struct mt7932_event *event) * rate-control setting. Bounds were checked by the exporter. */ if (he) - dev_info(&m->pdev->dev, - "ASSOC_HE_CAPABILITIES: epoch=%u rx-map=%04x tx-map=%04x\n", - m->connection_generation, - get_unaligned_le16(he->data + 18), - get_unaligned_le16(he->data + 20)); + dev_dbg(&m->pdev->dev, + "ASSOC_HE_CAPABILITIES: epoch=%u rx-map=%04x tx-map=%04x\n", + m->connection_generation, + get_unaligned_le16(he->data + 18), + get_unaligned_le16(he->data + 20)); } break; } case 0x4d: if (length >= 20) - dev_info(&m->pdev->dev, "ASSOC_START: band=%u width=%u channel=%u extension=%u\n", - get_unaligned_le32(body + 12), body[16], body[17], body[19]); + dev_dbg(&m->pdev->dev, "ASSOC_START: band=%u width=%u channel=%u extension=%u\n", + get_unaligned_le32(body + 12), body[16], body[17], body[19]); if (!mt_channel_event(m, body, length)) m->connect_error = -EPROTO; complete(&m->assoc_start); break; case 0x40: if (length >= 52) - dev_info(&m->pdev->dev, "ASSOC_REPORT: status=%u band=%u channel=%u bss=%u bmc=%u station=%u peer=%u cipher=%08x\n", - get_unaligned_le16(body), body[11], body[12], body[16], body[17], body[20], body[21], - get_unaligned_le32(body + 48)); + dev_dbg(&m->pdev->dev, + "ASSOC_REPORT: status=%u band=%u channel=%u bss=%u bmc=%u station=%u peer=%u cipher=%08x\n", + get_unaligned_le16(body), body[11], body[12], body[16], body[17], + body[20], body[21], get_unaligned_le32(body + 48)); /* OPEN reports retain cipher as numeric metadata only; it neither * installs a key nor changes the immutable requested security mode. */ @@ -213,12 +214,13 @@ void mt_link_event(struct mt7932 *m, const struct mt7932_event *event) m->peer_valid = true; /* HOST decoder confirms the RSN suite namespace, not request enum10. */ m->event_cipher = get_unaligned_le32(body + 48); - dev_info(&m->pdev->dev, "ASSOCIATED: peer=%u station=%u cipher-raw=%u\n", - m->peer_wtbl, body[20], m->event_cipher); + dev_dbg(&m->pdev->dev, "ASSOCIATED: peer=%u station=%u cipher-raw=%u\n", + m->peer_wtbl, body[20], m->event_cipher); break; case 0x42: if (length >= 2) - dev_info(&m->pdev->dev, "CONNECT_REPORT: status=%u peer-owned=%u\n", get_unaligned_le16(body), m->peer_valid); + dev_dbg(&m->pdev->dev, "CONNECT_REPORT: status=%u peer-owned=%u\n", + get_unaligned_le16(body), m->peer_valid); if (length < 2 || get_unaligned_le16(body) || !m->peer_valid) m->connect_error = -ECONNREFUSED; complete(&m->assoc_done); @@ -352,8 +354,9 @@ static int mt_retire_connection(struct mt7932 *m) m->link_failed = true; /* uncertain ownership: recovery required */ spin_unlock_irqrestore(&m->response_lock, flags); mutex_unlock(&m->command_mutex); - dev_info(&m->pdev->dev, "CONNECTION_RETIRED: status=%d%s\n", ret, - ret ? "; recovery required, DMA ownership retained" : "; old TX drained, six IDs released"); + dev_dbg(&m->pdev->dev, "CONNECTION_RETIRED: status=%d%s\n", ret, + ret ? "; recovery required, DMA ownership retained" : + "; old TX drained, six IDs released"); return ret; } @@ -398,7 +401,7 @@ void mt_connect_work(struct work_struct *work) goto free; } if (!m->connect_bss) { - dev_info(&m->pdev->dev, "CONNECT_DISCOVERY: refreshing missing scan candidate\n"); + dev_dbg(&m->pdev->dev, "CONNECT_DISCOVERY: refreshing missing scan candidate\n"); ret = mt_connect_discover(m); if (ret) goto free; @@ -444,7 +447,8 @@ void mt_connect_work(struct work_struct *work) memzero_explicit(m->connect_pmk, sizeof(m->connect_pmk)); if (ret) goto report; - dev_info(&m->pdev->dev, "CONNECT_SUBMITTED: BSSID=%pM channel=%u\n", m->connect_bssid, m->connect_channel); + dev_dbg(&m->pdev->dev, "CONNECT_SUBMITTED: BSSID=%pM channel=%u\n", + m->connect_bssid, m->connect_channel); /* Genuine D7 requests must remain serviceable during asynchronous waits. */ mutex_unlock(&m->command_mutex); ret = wait_for_completion_timeout(&m->assoc_start, msecs_to_jiffies(5000)) ? 0 : -ETIMEDOUT; @@ -460,8 +464,8 @@ void mt_connect_work(struct work_struct *work) ret = ret ?: READ_ONCE(m->connect_error); if (ret) goto report; - dev_info(&m->pdev->dev, "ASSOCIATION_CALIBRATED: primary=%u center=%u\n", - m->connect_channel, m->connect_center); + dev_dbg(&m->pdev->dev, "ASSOCIATION_CALIBRATED: primary=%u center=%u\n", + m->connect_channel, m->connect_center); mt7932_peer_queue(queue, m->peer_ids.station[0], true); ret = mt_request(m, 0x6b, true, true, false, queue, sizeof(queue)); if (ret) @@ -551,10 +555,14 @@ void mt_connect_work(struct work_struct *work) netif_wake_queue(m->netdev); } if (m->connect_open) { - dev_info(&m->pdev->dev, "OPEN_CONNECTED: owned peer, firmware completion; Ethernet=%u\n", m->data_ready); + dev_dbg(&m->pdev->dev, + "OPEN_CONNECTED: owned peer, firmware completion; Ethernet=%u\n", + m->data_ready); } else { cfg80211_port_authorized(m->netdev, m->connect_bssid, NULL, 0, GFP_ATOMIC); - dev_info(&m->pdev->dev, "WPA2_AUTHORIZED: firmware handshake complete; Ethernet=%u\n", m->data_ready); + dev_dbg(&m->pdev->dev, + "WPA2_AUTHORIZED: firmware handshake complete; Ethernet=%u\n", + m->data_ready); } } else { dev_err(&m->pdev->dev, "CONNECT_FAILED: %d; recovery-required=%u\n", ret, m->link_failed); @@ -596,9 +604,11 @@ int mt_connect(struct wiphy *wiphy, struct net_device *netdev, mt_retry_missing_policy(m); if (!mt_rf_allowed(m) || !m->bss_active || netdev != m->netdev) return -EAGAIN; - dev_info_ratelimited(&m->pdev->dev, "CONNECT_REQUEST: WPA=%u pairwise-count=%d group=%08x AKM-count=%d PMK-present=%u MFP=%u IE-bytes=%zu\n", - params->crypto.wpa_versions, params->crypto.n_ciphers_pairwise, params->crypto.cipher_group, - params->crypto.n_akm_suites, !!params->crypto.psk, params->mfp, params->ie_len); + dev_dbg(&m->pdev->dev, + "CONNECT_REQUEST: WPA=%u pairwise-count=%d group=%08x AKM-count=%d PMK-present=%u MFP=%u IE-bytes=%zu\n", + params->crypto.wpa_versions, params->crypto.n_ciphers_pairwise, + params->crypto.cipher_group, params->crypto.n_akm_suites, + !!params->crypto.psk, params->mfp, params->ie_len); if (!params->ssid_len || params->ssid_len > 32) return -EINVAL; if (!open && (params->crypto.wpa_versions != NL80211_WPA_VERSION_2 || @@ -675,7 +685,7 @@ void mt_disconnect_work(struct work_struct *work) m->disconnecting = false; cfg80211_disconnected(m->netdev, reason, NULL, 0, local, GFP_ATOMIC); spin_unlock_irqrestore(&m->response_lock, flags); - dev_info(&m->pdev->dev, "DISCONNECTED: teardown=%d local=%u\n", ret, local); + dev_dbg(&m->pdev->dev, "DISCONNECTED: teardown=%d local=%u\n", ret, local); } int mt_disconnect(struct wiphy *wiphy, struct net_device *netdev, u16 reason) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 89e36ca28a6d4..d8a319cfc12a8 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -169,8 +169,8 @@ static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) ret = mt7932_cal_null(&m->cal_state, packet + 36, event.length >= 36 ? event.length - 36 : 0, m->smart_version); - dev_info(&m->pdev->dev, "CAL_EVENT: seq=%u length=%zu result=%d credits=%u/%u\n", - event.seq, event.length, ret, m->cal_state.received, m->cal_state.expected); + dev_dbg(&m->pdev->dev, "CAL_EVENT: seq=%u length=%zu result=%d credits=%u/%u\n", + event.seq, event.length, ret, m->cal_state.received, m->cal_state.expected); if (ret < 0) mt_rf_fail_locked(m, ret); if (ret) @@ -196,9 +196,10 @@ static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) memcpy(m->cal_requests[at], packet + 36, 16); m->cal_request_seq[at] = event.seq; m->cal_request_count++; - dev_info(&m->pdev->dev, "CAL_REQUEST: seq=%u words=%08x/%08x/%08x/%08x queued=%u\n", - event.seq, get_unaligned_le32(packet + 36), get_unaligned_le32(packet + 40), - get_unaligned_le32(packet + 44), get_unaligned_le32(packet + 48), m->cal_request_count); + dev_dbg(&m->pdev->dev, "CAL_REQUEST: seq=%u words=%08x/%08x/%08x/%08x queued=%u\n", + event.seq, get_unaligned_le32(packet + 36), + get_unaligned_le32(packet + 40), get_unaligned_le32(packet + 44), + get_unaligned_le32(packet + 48), m->cal_request_count); if (READ_ONCE(m->rf_ready)) schedule_work(&m->cal_work); } @@ -385,38 +386,18 @@ int mt_request_ext(struct mt7932 *m, u8 cid, u8 ext, bool runtime, bool set, boo if (READ_ONCE(m->stopping)) ret = -ESHUTDOWN; if (ret == -ETIMEDOUT) { - /* Diagnostic only: never turn a polled late reply into IRQ success. */ unsigned int i; - u16 command; - u32 mask, pending; - unsigned int cap = pci_find_capability(m->pdev, PCI_CAP_ID_MSI); - - pci_read_config_word(m->pdev, PCI_COMMAND, &command); - pci_read_config_dword(m->pdev, cap + PCI_MSI_MASK_64, &mask); - pci_read_config_dword(m->pdev, cap + PCI_MSI_PENDING_64, &pending); - dev_info(&m->pdev->dev, "timeout PCI command=%04x MSI mask=%08x pending=%08x\n", - command, mask, pending); - - dev_info(&m->pdev->dev, "timeout gates MAC=%08x WFDMA=%08x SW=%08x\n", - mt_read(m, 0x10188), mt_read(m, W + 0x204), mt_read(m, W + 0x1f4)); + + /* + * Collect a reply that arrived without its interrupt, so that the + * error below reports it. It does not turn the timeout into success. + */ disable_irq(pci_irq_vector(m->pdev, 0)); - for (i = 0; i < ARRAY_SIZE(m->rx); i++) { - struct mt7932_ring *rx = &m->rx[i]; - - if (!rx->count) - continue; - dev_info(&m->pdev->dev, "timeout RX%u ctrl=%08x CPU=%u DMA=%u\n", i, - le32_to_cpu(rx->desc[rx->tail].control), - mt_read(m, rx->reg + 8), mt_read(m, rx->reg + 12)); - if (le32_to_cpu(rx->desc[rx->tail].control) & DONE) { - dma_rmb(); - print_hex_dump(KERN_INFO, "ROM RX prefix: ", DUMP_PREFIX_OFFSET, - 16, 1, rx->buffers + rx->tail * RX_STRIDE, 40, false); - } - mt_rx_drain(m, rx); - } - dev_info(&m->pdev->dev, "late polled reply length=%zu eid=%02x (not IRQ success)\n", - m->reply_length, m->reply_length > 28 ? m->reply[28] : 0); + for (i = 0; i < ARRAY_SIZE(m->rx); i++) + if (m->rx[i].count) + mt_rx_drain(m, &m->rx[i]); + dev_dbg(&m->pdev->dev, "late reply length=%zu eid=%02x\n", + m->reply_length, m->reply_length > 28 ? m->reply[28] : 0); enable_irq(pci_irq_vector(m->pdev, 0)); } spin_lock_irqsave(&m->response_lock, flags); @@ -459,7 +440,7 @@ static int mt_ownership(struct mt7932 *m) udelay(2); if ((mt_read(m, 0xfe250) >> 16) != 0x7000) return -EIO; - dev_info(&m->pdev->dev, "DRIVER_OWN_OK power=%08x\n", mt_read(m, 0xe00f0)); + dev_dbg(&m->pdev->dev, "DRIVER_OWN_OK power=%08x\n", mt_read(m, 0xe00f0)); return 0; } @@ -570,8 +551,8 @@ static int mt_dma_setup(struct mt7932 *m) mt_rmw(m, 0x2120, 0, BIT(1)); mt_write(m, W + 0x1f4, 0xffff); mt_write(m, W + 0x204, IRQ_MASK); - dev_info(&m->pdev->dev, "DMA_PUBLISHED glo=%08x TX17=%pad RX0=%pad IPC=%pad\n", - mt_read(m, W + 0x208), &m->tx[17].dma, &m->rx[0].dma, &m->ipc_dma); + dev_dbg(&m->pdev->dev, "DMA_PUBLISHED glo=%08x TX17=%pad RX0=%pad IPC=%pad\n", + mt_read(m, W + 0x208), &m->tx[17].dma, &m->rx[0].dma, &m->ipc_dma); return 0; } @@ -604,11 +585,11 @@ static int mt_rom_gate(struct mt7932 *m) if (m->reply_length < 40 || m->reply[28] != 2 || get_unaligned_le32(m->reply + 32) != addresses[i]) return -EPROTO; - dev_info(&m->pdev->dev, "ROM_READ_OK %08x=%08x seq=%u IRQ=%llu\n", - addresses[i], get_unaligned_le32(m->reply + 36), m->seq, m->interrupts); + dev_dbg(&m->pdev->dev, "ROM_READ_OK %08x=%08x seq=%u IRQ=%llu\n", + addresses[i], get_unaligned_le32(m->reply + 36), m->seq, m->interrupts); } - dev_info(&m->pdev->dev, "ROM_TRANSPORT_OK IRQ=%llu (no firmware download or RF yet)\n", - m->interrupts); + dev_dbg(&m->pdev->dev, "ROM_TRANSPORT_OK IRQ=%llu (no firmware download or RF yet)\n", + m->interrupts); return 0; } @@ -663,8 +644,8 @@ static int mt_download(struct mt7932 *m, const u8 *data, struct mt7932_region *r if (ret) return ret; ret = mt_scatter(m, data + r->offset, r->length); - dev_info(&m->pdev->dev, "%s region target=%08x length=%u scatter=%d\n", - ram ? "RAM" : "PATCH", r->target, r->length, ret); + dev_dbg(&m->pdev->dev, "%s region target=%08x length=%u scatter=%d\n", + ram ? "RAM" : "PATCH", r->target, r->length, ret); return ret; } @@ -728,8 +709,8 @@ static int mt_capability_gate(struct mt7932 *m) count = mt7932_capabilities(body, m->reply_length - 36); if (count < 0) { dev_err(&m->pdev->dev, "invalid capability reply len=%zu parse=%d\n", m->reply_length, count); - print_hex_dump(KERN_INFO, "NIC_CAP_REJECTED: ", DUMP_PREFIX_OFFSET, - 16, 1, m->reply, m->reply_length, false); + print_hex_dump_debug("NIC_CAP_REJECTED: ", DUMP_PREFIX_OFFSET, + 16, 1, m->reply, m->reply_length, false); return count; } for (i = 0; i < count; i++) { @@ -737,9 +718,9 @@ static int mt_capability_gate(struct mt7932 *m) u32 length = get_unaligned_le32(body + offset + 4); const u8 *value = body + offset + 8; - dev_info(&m->pdev->dev, "capability TLV type=%x bytes=%u\n", type, length); + dev_dbg(&m->pdev->dev, "capability TLV type=%x bytes=%u\n", type, length); if (type == 7) { - dev_info(&m->pdev->dev, "firmware MAC %pM\n", value); + dev_dbg(&m->pdev->dev, "firmware MAC %pM\n", value); mac = true; } else if (type == 8) { if (!value[4] || value[4] > 4) { @@ -748,13 +729,13 @@ static int mt_capability_gate(struct mt7932 *m) } memcpy(m->phy_cap, value, 12); m->antenna_mask = (1U << value[4]) - 1; - dev_info(&m->pdev->dev, "firmware PHY NSS=%u paths=%02x HT=%u VHT=%u HE=%u\n", - value[4], value[10], value[0], value[1], value[11]); + dev_dbg(&m->pdev->dev, "firmware PHY NSS=%u paths=%02x HT=%u VHT=%u HE=%u\n", + value[4], value[10], value[0], value[1], value[11]); phy = true; } else if (type == 0x38 && length) { m->runtime_cal = value[0]; m->runtime_cal_valid = true; - dev_info(&m->pdev->dev, "RUNTIME_CAL_CAPABILITY: %u\n", value[0]); + dev_dbg(&m->pdev->dev, "RUNTIME_CAL_CAPABILITY: %u\n", value[0]); } else if (type == 0x18 && length >= 4) { m->six_ghz = value[0]; m->six_ghz_valid = value[0] <= 1; @@ -767,10 +748,10 @@ static int mt_capability_gate(struct mt7932 *m) dev_err(&m->pdev->dev, "CAPABILITY_MISSING: MAC=%u PHY=%u\n", mac, phy); return -ENODATA; } - print_hex_dump(KERN_INFO, "NIC_CAP: ", DUMP_PREFIX_OFFSET, 16, 1, - m->reply, m->reply_length, false); - dev_info(&m->pdev->dev, "MCU_CAPABILITY_OK eid=%02x seq=%u (no calibration or RF yet)\n", - m->reply[28], m->seq); + print_hex_dump_debug("NIC_CAP: ", DUMP_PREFIX_OFFSET, 16, 1, + m->reply, m->reply_length, false); + dev_dbg(&m->pdev->dev, "MCU_CAPABILITY_OK eid=%02x seq=%u (no calibration or RF yet)\n", + m->reply[28], m->seq); return 0; } @@ -792,7 +773,7 @@ static int mt_config_gate(struct mt7932 *m) do { mt_tx_clean(&m->tx[17]); if (!m->tx[17].queued) { - dev_info(&m->pdev->dev, "BASIC_MLME_SUBMITTED: TX retired, no semantic ACK or RF startup\n"); + dev_dbg(&m->pdev->dev, "BASIC_MLME_SUBMITTED: TX retired, no semantic ACK or RF startup\n"); return 0; } usleep_range(100, 200); @@ -816,7 +797,7 @@ static int mt_board_gate(struct mt7932 *m) return ret; if (m->reply_length < 44 || m->reply[28] != 0xed || m->reply[32] != 0xc1) return -EPROTO; - dev_info(&m->pdev->dev, "CHIP_UID_QUERY_OK: 8 opaque bytes received, not published\n"); + dev_dbg(&m->pdev->dev, "CHIP_UID_QUERY_OK: 8 opaque bytes received, not published\n"); for (i = 0; i < ARRAY_SIZE(addresses); i++) { put_unaligned_le32(addresses[i], request); ret = mt_request_ext(m, 0xed, 1, true, false, true, request, sizeof(request)); @@ -834,8 +815,8 @@ static int mt_board_gate(struct mt7932 *m) candidate = !(module & 15) ? 2 : !board_byte ? 3 : 1; m->board_type = candidate; m->module_byte = module; - dev_info(&m->pdev->dev, "BOARD_CLASSIFICATION_CAPTURED: candidate=%u module=%02x board-byte=%02x status=%08x/%08x; no buffer branch selected\n", - candidate, module, board_byte, status[0], status[1]); + dev_dbg(&m->pdev->dev, "BOARD_CLASSIFICATION_CAPTURED: candidate=%u module=%02x board-byte=%02x status=%08x/%08x; no buffer branch selected\n", + candidate, module, board_byte, status[0], status[1]); return 0; } @@ -874,7 +855,7 @@ static int mt_precal_gate(struct mt7932 *m) return -EPROTO; } status = get_unaligned_le32(m->reply + 40); - dev_info(&m->pdev->dev, "PPR_SOURCE2_RESULT: status=%08x\n", status); + dev_dbg(&m->pdev->dev, "PPR_SOURCE2_RESULT: status=%08x\n", status); if (status) return -EREMOTEIO; memset(request, 0, sizeof(request)); @@ -884,17 +865,17 @@ static int mt_precal_gate(struct mt7932 *m) if (m->reply_length < 40 || m->reply[28] != 0xed || m->reply[32] != 0xaa) return -EPROTO; m->smart_version = m->reply[38] & 15; - dev_info(&m->pdev->dev, "SMART_VERSION_REPLY: eid=%02x ext=%02x version=%u\n", - m->reply[28], m->reply[32], m->smart_version); + dev_dbg(&m->pdev->dev, "SMART_VERSION_REPLY: eid=%02x ext=%02x version=%u\n", + m->reply[28], m->reply[32], m->smart_version); request[3] = 9; ret = mt_request(m, 0xbf, true, false, true, request, 20); if (ret) return ret; if (m->reply_length < 56) return -EPROTO; - dev_info(&m->pdev->dev, "CALTYPE_REPLY: eid=%02x ext=%02x H=%u P=%u O=%u V=%u marker=%08x\n", - m->reply[28], m->reply[32], m->runtime_cal, m->reply[37], - m->reply[44], m->reply[38], get_unaligned_le32(m->reply + 48)); + dev_dbg(&m->pdev->dev, "CALTYPE_REPLY: eid=%02x ext=%02x H=%u P=%u O=%u V=%u marker=%08x\n", + m->reply[28], m->reply[32], m->runtime_cal, m->reply[37], + m->reply[44], m->reply[38], get_unaligned_le32(m->reply + 48)); /* M=0/T=3 exact host predicate. Stage metadata only after acceptance; * do not select the legacy absent-marker D6 compatibility fallback. */ @@ -902,7 +883,7 @@ static int mt_precal_gate(struct mt7932 *m) m->reply[37] != 1 || m->reply[44] != 1) return -EPROTO; m->preload_version = m->reply[38]; - dev_info(&m->pdev->dev, "PRECAL_HANDSHAKE_OK: own calibration and RF policy still required\n"); + dev_dbg(&m->pdev->dev, "PRECAL_HANDSHAKE_OK: own calibration and RF policy still required\n"); return 0; } @@ -958,8 +939,8 @@ static int mt_firmware_gate(struct mt7932 *m) dev_err(&m->pdev->dev, "local input mediatek/mt7932/IZUBA_W7932_2.bin has invalid region %d\n", i); goto out; } - dev_info(&m->pdev->dev, "stock containers patch=%zu/%d regions RAM=%zu/%d regions\n", - patch->size, patches, ram->size, regions); + dev_dbg(&m->pdev->dev, "stock containers patch=%zu/%d regions RAM=%zu/%d regions\n", + patch->size, patches, ram->size, regions); phase = "patch semaphore command/reply"; put_unaligned_le32(2, request); ret = mt_command(m, 0x10, request, 4); @@ -986,7 +967,7 @@ static int mt_firmware_gate(struct mt7932 *m) goto out; mt_secure_release(m); } - dev_info(&m->pdev->dev, "PATCH_READY\n"); + dev_dbg(&m->pdev->dev, "PATCH_READY\n"); phase = "OTP command/profile"; ret = mt_otp_byte(m, 0x7a, &value); if (!ret) @@ -1041,7 +1022,7 @@ static int mt_firmware_gate(struct mt7932 *m) ret = mt_poll(m, 0xe00f0, 3, 3, 5000000); if (ret) goto out; - dev_info(&m->pdev->dev, "FIRMWARE_READY entry=%08x\n", entry); + dev_dbg(&m->pdev->dev, "FIRMWARE_READY entry=%08x\n", entry); phase = "runtime capability/transport gates"; ret = mt_runtime_start(m); out: @@ -1058,16 +1039,16 @@ void mt_transport_snapshot(struct mt7932 *m) unsigned int i; struct mt7932_ring *tx = &m->tx[17]; - dev_info(&m->pdev->dev, "TRANSPORT: interrupts=%llu packets=%llu pending=%08x mask=%08x TX17 head=%u tail=%u queued=%u DMA=%u\n", - m->interrupts, m->packets, mt_read(m, W + 0x200), mt_read(m, W + 0x204), - tx->head, tx->tail, tx->queued, mt_read(m, tx->reg + 12)); + dev_dbg(&m->pdev->dev, "TRANSPORT: interrupts=%llu packets=%llu pending=%08x mask=%08x TX17 head=%u tail=%u queued=%u DMA=%u\n", + m->interrupts, m->packets, mt_read(m, W + 0x200), mt_read(m, W + 0x204), + tx->head, tx->tail, tx->queued, mt_read(m, tx->reg + 12)); for (i = 0; i < ARRAY_SIZE(m->rx); i++) { struct mt7932_ring *rx = &m->rx[i]; if (rx->count) - dev_info(&m->pdev->dev, "TRANSPORT_RX%u: tail=%u producer=%u DMA=%u control=%08x\n", - i, rx->tail, mt_read(m, rx->reg + 8), mt_read(m, rx->reg + 12), - le32_to_cpu(READ_ONCE(rx->desc[rx->tail].control))); + dev_dbg(&m->pdev->dev, "TRANSPORT_RX%u: tail=%u producer=%u DMA=%u control=%08x\n", + i, rx->tail, mt_read(m, rx->reg + 8), mt_read(m, rx->reg + 12), + le32_to_cpu(READ_ONCE(rx->desc[rx->tail].control))); } } @@ -1155,7 +1136,7 @@ static void mt_remove(struct pci_dev *pdev) wiphy_free(m->wiphy); if (m->vectors_allocated) pci_free_irq_vectors(pdev); - dev_info(&pdev->dev, "REMOVE_COMPLETE: host users joined, PCI function reset, DMA retired\n"); + dev_dbg(&pdev->dev, "REMOVE_COMPLETE: host users joined, PCI function reset, DMA retired\n"); /* All coherent allocations/BAR mappings are devres-owned and now safe * to release. No default-domain ownership manipulation is necessary. */ @@ -1230,27 +1211,6 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) goto retain; m->irq_requested = true; - { - struct irq_data *irq = irq_get_irq_data(pci_irq_vector(pdev, 0)); - unsigned int cap = pci_find_capability(pdev, PCI_CAP_ID_MSI); - u16 control, command; - u32 lo, hi, data, mask; - - pci_read_config_word(pdev, PCI_COMMAND, &command); - pci_read_config_word(pdev, cap + PCI_MSI_FLAGS, &control); - pci_read_config_dword(pdev, cap + PCI_MSI_ADDRESS_LO, &lo); - pci_read_config_dword(pdev, cap + PCI_MSI_ADDRESS_HI, &hi); - pci_read_config_dword(pdev, cap + PCI_MSI_DATA_64, &data); - pci_read_config_dword(pdev, cap + PCI_MSI_MASK_64, &mask); - dev_dbg(&pdev->dev, "PCI command=%04x MSI cap=%x control=%04x addr=%08x:%08x data=%08x\n", - command, cap, control, hi, lo, data); - dev_dbg(&pdev->dev, "MSI mask=%08x MSI-X cap=%x chosen=%s\n", mask, - pci_find_capability(pdev, PCI_CAP_ID_MSIX), - pdev->msix_enabled ? "MSI-X" : pdev->msi_enabled ? "MSI" : "INTx"); - for (; irq; irq = irq->parent_data) - dev_dbg(&pdev->dev, "IRQ %u chip=%s hwirq=%lu\n", irq->irq, - irq->chip->name, irq->hwirq); - } ret = mt_initialize(m); if (!ret) ret = mt_register_regulatory_gate(m); diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 688654acac97d..d6c80ecb315ed 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/net/wireless/mediatek/mt7932/recovery.c b/drivers/net/wireless/mediatek/mt7932/recovery.c index cdb327bb27e39..91ae4b0342608 100644 --- a/drivers/net/wireless/mediatek/mt7932/recovery.c +++ b/drivers/net/wireless/mediatek/mt7932/recovery.c @@ -274,7 +274,7 @@ int mt_recovery_gate(struct mt7932 *m) mt_read(m, m->tx[17].reg + 12)); } else { WRITE_ONCE(m->running, true); - dev_info(&m->pdev->dev, "RUNTIME_DMA_READY: full RX epoch, TX17 reset, firmware NORMAL_STATE\n"); + dev_dbg(&m->pdev->dev, "RUNTIME_DMA_READY: full RX epoch, TX17 reset, firmware NORMAL_STATE\n"); } enable_irq(pci_irq_vector(m->pdev, 0)); if (!ret) { @@ -284,8 +284,8 @@ int mt_recovery_gate(struct mt7932 *m) mt7932_capabilities(m->reply + 36, m->reply_length - 36) < 0)) ret = -EPROTO; if (!ret) - dev_info(&m->pdev->dev, "RUNTIME_QUERY_OK: TX17 capability response, IRQ=%llu\n", - m->interrupts); + dev_dbg(&m->pdev->dev, "RUNTIME_QUERY_OK: TX17 capability response, IRQ=%llu\n", + m->interrupts); } return ret; } diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index a4ed7f5b97739..c949c28254355 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -123,7 +123,7 @@ static int mt_stock_config(struct mt7932 *m) if (ret) goto free; } - dev_info(&m->pdev->dev, "STOCK_CONFIG_SUBMITTED: %u original/PHY-derived entries\n", count); + dev_dbg(&m->pdev->dev, "STOCK_CONFIG_SUBMITTED: %u original/PHY-derived entries\n", count); free: kfree(records); out: @@ -165,9 +165,9 @@ static int mt_publish_policy(struct mt7932 *m, const struct mt7932_reg_snapshot /* Native SETs have no response/atomic policy-commit event. Command * submission is not misrepresented as firmware readback or an ACK. */ - dev_info(&m->pdev->dev, "REGULATORY_POLICY_SUBMITTED: %c%c firmware=%c%c channels=%u (native no-response SETs)\n", - reg->alpha2[0], reg->alpha2[1], reg->domain[0], reg->domain[1], - (reg->length - 12) / 8); + dev_dbg(&m->pdev->dev, "REGULATORY_POLICY_SUBMITTED: %c%c firmware=%c%c channels=%u (native no-response SETs)\n", + reg->alpha2[0], reg->alpha2[1], reg->domain[0], reg->domain[1], + (reg->length - 12) / 8); return 0; } @@ -226,7 +226,7 @@ static int mt_startup_once(struct mt7932 *m) ret = mt_calibration_gate(m); if (!ret) ret = mt_request(m, 0x3e, true, true, false, roam_lock, sizeof(roam_lock)); - dev_info(&m->pdev->dev, "OWN_CALIBRATION_STARTUP_RESULT: %d\n", ret); + dev_dbg(&m->pdev->dev, "OWN_CALIBRATION_STARTUP_RESULT: %d\n", ret); if (!ret) ret = mt_enable_scan(m); if (ret) @@ -385,9 +385,9 @@ static void mt_startup_work(struct work_struct *work) reg.alpha2[0], reg.alpha2[1], generation, ret, m->policy_failed || m->link_failed); else - dev_info(&m->pdev->dev, "REGULATORY_READY: %c%c firmware=%c%c generation=%u current=%u\n", - reg.alpha2[0], reg.alpha2[1], reg.domain[0], reg.domain[1], - generation, !m->reg_pending); + dev_dbg(&m->pdev->dev, "REGULATORY_READY: %c%c firmware=%c%c generation=%u current=%u\n", + reg.alpha2[0], reg.alpha2[1], reg.domain[0], reg.domain[1], + generation, !m->reg_pending); spin_unlock_irqrestore(&m->response_lock, flags); next: mutex_unlock(&m->command_mutex); @@ -496,8 +496,8 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request if (READ_ONCE(m->interface_registered)) schedule_work(&m->startup_work); spin_unlock_irqrestore(&m->response_lock, irqflags); - dev_info_ratelimited(&m->pdev->dev, "REGULATORY_REQUEST: %c%c initiator=%u generation=%u\n", - reg.alpha2[0], reg.alpha2[1], request->initiator, m->reg_generation); + dev_dbg(&m->pdev->dev, "REGULATORY_REQUEST: %c%c initiator=%u generation=%u\n", + reg.alpha2[0], reg.alpha2[1], request->initiator, m->reg_generation); } static const struct cfg80211_ops mt_cfg_ops = { @@ -611,6 +611,6 @@ int mt_register_regulatory_gate(struct mt7932 *m) return ret; WRITE_ONCE(m->interface_registered, true); schedule_work(&m->startup_work); - dev_info(&m->pdev->dev, "REGULATORY_GATE_REGISTERED: cfg80211 country policy required\n"); + dev_dbg(&m->pdev->dev, "REGULATORY_GATE_REGISTERED: cfg80211 country policy required\n"); return 0; } From fe247015487394c430e295532e8eee0d28fdb8cb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:58:38 +1100 Subject: [PATCH 1258/2680] usb: typec: tipd: Don't warn about deferred CD321x mode changes The ACIO Thunderbolt switch and the ATC PHY return -EAGAIN while the PCIe host behind a USB4 port, or the USB host on the other side of the PHY, is still being set up or torn down. cd321x_update_work() already treats that as a deferral: it retries without forcing a reconnect. Yet cd321x_typec_update_mode() logged every such attempt as a warning, so a dock that comes up a little slowly left "failed to configure Type-C connection: -11" in the log even though it then worked. Log -EAGAIN at debug level and keep the warning for real failures. The revalidation message after a tunnel state change is in the same category: it is a routine, bounded recovery step, so make it debug too. (cherry picked from commit 4da8304f1b51e6493b8d2006439d50b03defc059) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/core.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 95a7d75ba3b1d..db2429cce777d 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -716,7 +716,11 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, if (ret) { /* The mux may have changed before the switch failed. */ cd321x->state = old_state; - dev_warn(tps->dev, "failed to configure Type-C connection: %d\n", ret); + /* A provider that is not ready yet asks for a retry. */ + if (ret == -EAGAIN) + dev_dbg(tps->dev, "Type-C connection deferred\n"); + else + dev_warn(tps->dev, "failed to configure Type-C connection: %d\n", ret); } else if (cd321x->state.alt != cd321x->port_altmode_dp) { cd321x->dp_status = 0; cd321x->dp_conf = 0; @@ -1006,7 +1010,7 @@ static void cd321x_resume_work(struct work_struct *work) if (atomic_xchg(&cd321x->link_event, 0)) { if (cd321x_pm_link_event(&cd321x->pm)) { cd321x->state_valid = false; - dev_info(tps->dev, "revalidating cable after tunnel state change\n"); + dev_dbg(tps->dev, "revalidating cable after tunnel state change\n"); } else { dev_warn_ratelimited(tps->dev, "tunnel link recovery budget exhausted\n"); From fa2f7eb0acd0e2d2dc11ba6c7ff00249abd8c6a1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1259/2680] wifi: mt7932: disable link power management only once Probe disabled ASPM and clock PM on the link twice. The first call fails the probe, so that a remove and rescan cannot leave the radio on a link policy it was never tested with. The second call could then only succeed, and its warning for a failure was unreachable. Keep the first call and merge the two comments. No functional change. (cherry picked from commit 559e474df2619fee338b5e989280393ae6d1887d) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index d8a319cfc12a8..13b8f97ae5910 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -1160,17 +1160,16 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = apple_piodma_radio_check(pdev); if (ret) return ret; - /* A remove/rescan may have recreated link state under the global policy. */ + /* + * Station operation was tested with ASPM and clock PM off on this link, + * and a remove/rescan may have restored them under the global policy. + */ ret = pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); if (ret) return ret; domain = iommu_get_domain_for_dev(&pdev->dev); if (!domain || (domain->type != IOMMU_DOMAIN_DMA && domain->type != IOMMU_DOMAIN_DMA_FQ)) return dev_err_probe(&pdev->dev, -EINVAL, "translated DMA domain required\n"); - /* Station operation was qualified with ASPM and clock PM off on this link. */ - ret = pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); - if (ret) - dev_warn(&pdev->dev, "link power management left as found: %d\n", ret); ret = pcim_enable_device(pdev); if (ret) return ret; From 5020414bd1574a1627c0863d8f37dbea166a8cc8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:59:58 +1100 Subject: [PATCH 1260/2680] usb: typec: tipd: Use guard() for the locks in the debugfs interface core.c takes tps->lock with guard() and scoped_guard(), but the debugfs interface added for CD321x open-codes mutex_lock()/mutex_unlock() and needs a goto label to unlock on the -EBUSY path. Use the same guards there, which removes the label and the extra return-value juggling. The critical sections are unchanged: user copies still happen outside tps->lock except for the read of the saved DATA1 result, as before. No functional change. (cherry picked from commit 36ae420b3cde6f3e260f17f5abe5742b7efe7b40) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/debugfs.c | 56 +++++++++++++------------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/drivers/usb/typec/tipd/debugfs.c b/drivers/usb/typec/tipd/debugfs.c index 5a8d576eff0dc..8f35394624fe6 100644 --- a/drivers/usb/typec/tipd/debugfs.c +++ b/drivers/usb/typec/tipd/debugfs.c @@ -122,10 +122,10 @@ static int tipd_debugfs_command_release(struct inode *inode, struct file *file) * may be gone. tipd_debugfs_unregister() has then left "DBMa" itself. */ if (f->holds_dbma && !debugfs_file_get(dentry)) { - mutex_lock(&tps->lock); - if (!--tps->debugfs->dbma_holders) - tipd_debugfs_leave_dbma(tps); - mutex_unlock(&tps->lock); + scoped_guard(mutex, &tps->lock) { + if (!--tps->debugfs->dbma_holders) + tipd_debugfs_leave_dbma(tps); + } debugfs_file_put(dentry); } @@ -166,17 +166,15 @@ static ssize_t tipd_debugfs_command_write(struct file *file, if (!memcmp(cmd->cmd, "DBMa", 4) && buf[4] > 1) return -EINVAL; - mutex_lock(&tps->lock); + guard(mutex)(&tps->lock); /* Preserve inherited mode and the leases held by other open files. */ if ((!memcmp(cmd->cmd, "DBMa", 4) && !buf[4] && (tps->dbma_at_probe || tps->debugfs->dbma_holders > (unsigned int)f->holds_dbma)) || (!memcmp(cmd->cmd, "LOCK", 4) && !memchr_inv(buf + 4, 0, 4) && - (tps->dbma_at_probe || tps->debugfs->dbma_holders))) { - ret = -EBUSY; - goto out; - } + (tps->dbma_at_probe || tps->debugfs->dbma_holders))) + return -EBUSY; ret = tps6598x_exec_cmd_tmo(tps, cmd->cmd, len, len ? &buf[4] : NULL, sizeof(out), out, cmd->timeout_ms, 0); @@ -198,27 +196,20 @@ static ssize_t tipd_debugfs_command_write(struct file *file, } } -out: - mutex_unlock(&tps->lock); - - return ret ? ret : count; + return ret ?: count; } static ssize_t tipd_debugfs_command_read(struct file *file, char __user *ubuf, size_t count, loff_t *ppos) { struct tipd_debugfs_file *f = file->private_data; - ssize_t ret; - mutex_lock(&f->tps->lock); + guard(mutex)(&f->tps->lock); if (!f->have_result) - ret = -ENODATA; - else - ret = simple_read_from_buffer(ubuf, count, ppos, f->result, - sizeof(f->result)); - mutex_unlock(&f->tps->lock); + return -ENODATA; - return ret; + return simple_read_from_buffer(ubuf, count, ppos, f->result, + sizeof(f->result)); } static const struct file_operations tipd_debugfs_command_fops = { @@ -244,9 +235,8 @@ static ssize_t tipd_debugfs_regs_read(struct file *file, char __user *ubuf, if (count > sizeof(buf)) return -EINVAL; - mutex_lock(&tps->lock); - ret = tps6598x_block_read(tps, *ppos, buf, count); - mutex_unlock(&tps->lock); + scoped_guard(mutex, &tps->lock) + ret = tps6598x_block_read(tps, *ppos, buf, count); if (ret) return ret; @@ -268,10 +258,10 @@ void tipd_debugfs_register(struct tps6598x *tps) { struct tipd_debugfs *dbg; - mutex_lock(&tipd_debugfs_root_lock); - if (!tipd_debugfs_root) - tipd_debugfs_root = debugfs_create_dir("tipd", usb_debug_root); - mutex_unlock(&tipd_debugfs_root_lock); + scoped_guard(mutex, &tipd_debugfs_root_lock) { + if (!tipd_debugfs_root) + tipd_debugfs_root = debugfs_create_dir("tipd", usb_debug_root); + } dbg = kzalloc_obj(*dbg); if (!dbg) @@ -295,11 +285,11 @@ void tipd_debugfs_unregister(struct tps6598x *tps) /* Waits for running file operations, later ones fail with -EIO */ debugfs_remove_recursive(dbg->dentry); - mutex_lock(&tps->lock); - if (dbg->dbma_holders) - tipd_debugfs_leave_dbma(tps); - tps->debugfs = NULL; - mutex_unlock(&tps->lock); + scoped_guard(mutex, &tps->lock) { + if (dbg->dbma_holders) + tipd_debugfs_leave_dbma(tps); + tps->debugfs = NULL; + } kfree(dbg); } From 6ba4b81e40ca99e182d7aef45dbbc9b6ef4aaab0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1261/2680] wifi: mt7932: drop internal document names from comments Several comments named internal design notes (NEO_*_CONTRACT and similar) or used bring-up wording such as "campaign" and "explicit experiment". These names mean nothing outside the project. State the facts they stood for instead. Comment-only change. (cherry picked from commit df00dc306318ac4628a2f7ec31ed3e92f5e1a1a5) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/association.h | 9 ++++----- drivers/net/wireless/mediatek/mt7932/calibration.h | 2 +- drivers/net/wireless/mediatek/mt7932/frontend.c | 4 ++-- drivers/net/wireless/mediatek/mt7932/link.c | 2 +- drivers/net/wireless/mediatek/mt7932/packet.h | 4 ++-- drivers/net/wireless/mediatek/mt7932/recovery.c | 10 +++++----- drivers/net/wireless/mediatek/mt7932/reset.c | 2 +- drivers/net/wireless/mediatek/mt7932/scan.h | 6 +++--- 8 files changed, 19 insertions(+), 20 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/association.h b/drivers/net/wireless/mediatek/mt7932/association.h index eb207921906e1..ba733a885e261 100644 --- a/drivers/net/wireless/mediatek/mt7932/association.h +++ b/drivers/net/wireless/mediatek/mt7932/association.h @@ -91,8 +91,8 @@ static inline int mt7932_assoc_capabilities(u8 *out, size_t capacity, } /* Original25G83 pure WPA2 binary-key result: structured type6, not a string. - * NEO_TYPE6_BINARY_PMK_NATIVE_CONTRACT: standard nl80211 PMK maps directly; - * no passphrase API, key derivation, duplicate independent PMK or extra flags. + * The standard nl80211 PMK maps directly; no passphrase API, key derivation, + * duplicate independent PMK or extra flags. */ static inline int mt7932_connect_binary_pmk(u8 *body, size_t capacity, const u8 *pmk, size_t length) @@ -111,8 +111,8 @@ static inline int mt7932_connect_binary_pmk(u8 *body, size_t capacity, } /* Explicit OPEN or PMK-only WPA2/CCMP; never an automatic security fallback. - * Caller owns identities/resources and validated BSS state. - * NEO_WPA2_FULLMAC_ASSOCIATION_AND_ETHERNET_CONTRACT. Never log this body. + * Caller owns identities/resources and validated BSS state. Never log this + * body: it carries the PMK. */ static inline int mt7932_connect_body(u8 *body, size_t capacity, const u8 *ssid, size_t ssid_length, const u8 pmk[32], const u8 bssid[6], @@ -141,7 +141,6 @@ static inline int mt7932_connect_body(u8 *body, size_t capacity, const u8 *ssid, /* NORMAL25G83 original strict keyless OPEN branch, without optional * automatic-authentication policy. Keep this coherent version/auth/ * policy tuple together; no dummy keys or stale security state. - * NEO_NORMAL_STRICT_OPEN_PRODUCER_CONTRACT. */ if (open) { body[1] = 1; diff --git a/drivers/net/wireless/mediatek/mt7932/calibration.h b/drivers/net/wireless/mediatek/mt7932/calibration.h index 85075bce7b255..f13ee11866bad 100644 --- a/drivers/net/wireless/mediatek/mt7932/calibration.h +++ b/drivers/net/wireless/mediatek/mt7932/calibration.h @@ -80,7 +80,7 @@ struct mt7932_cal_segment { }; /* Validate the entire original container before selecting any data to send. - * NEO_MT7932_STOCK_ARTIFACTS_AND_CAL_CONVERSION: raw BLOB, not a transcript. + * The input is the raw BLOB container, not a converted transcript. */ static inline int mt7932_cal_validate(const u8 *data, size_t size) { diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 7b0259bde004a..0a5e700d16ccd 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -89,7 +89,7 @@ void mt_scan_event(struct mt7932 *m, const struct mt7932_event *event) if (m->stopping) return; /* Abort completes userspace locally, not by an invented cancel ACK. - * Keep the sequence reserved until an actual old-campaign done arrives. + * Keep the sequence reserved until the old scan's own done event arrives. */ if (event->length >= 37 && m->retired_scan_seq && event->packet[36] == m->retired_scan_seq) { @@ -127,7 +127,7 @@ void mt_abort_scan(struct wiphy *wiphy, struct wireless_dev *wdev) } /* No command/response lock held. The caller closes new scan admission first. - * Natural EID0d completion fences the campaign; a local cancel alone does not. + * Natural EID0d completion fences the scan; a local cancel alone does not. */ int mt_scan_quiesce(struct mt7932 *m) { diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index bcdcb71ce3db9..55a08f5b86728 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -423,7 +423,7 @@ void mt_connect_work(struct work_struct *work) goto free; /* Initial station policy: framework M=0 branch and no optional flags. * This yields flag2 independently of credentials; no scan-result count - * belongs in this field. The recovered framework uses listen interval20. + * belongs in this field. The original driver uses listen interval 20. * Other platform flag policies remain unsupported, not guessed mappings. */ ret = mt7932_connect_body(body, 856, m->connect_ssid, m->connect_ssid_length, diff --git a/drivers/net/wireless/mediatek/mt7932/packet.h b/drivers/net/wireless/mediatek/mt7932/packet.h index 303cc87fef8b0..a93d04d91fd41 100644 --- a/drivers/net/wireless/mediatek/mt7932/packet.h +++ b/drivers/net/wireless/mediatek/mt7932/packet.h @@ -14,8 +14,8 @@ struct mt7932_rx_frame { bool translated, group4, signal_valid, rate_valid, unicast; }; -/* NEO_PACKET_RX_GROUPS_AND_DATA_TXP_CONTRACT. Strict non-monitor subset; - * descriptor groups are ordered4,1,2,3,5, not by numerical group identifier. +/* Strict non-monitor subset of the RX descriptor; its groups are ordered + * 4, 1, 2, 3, 5, not by numerical group identifier. */ static inline int mt7932_rx_frame(const u8 *packet, size_t size, struct mt7932_rx_frame *out) { diff --git a/drivers/net/wireless/mediatek/mt7932/recovery.c b/drivers/net/wireless/mediatek/mt7932/recovery.c index 91ae4b0342608..fc60bc05b13d6 100644 --- a/drivers/net/wireless/mediatek/mt7932/recovery.c +++ b/drivers/net/wireless/mediatek/mt7932/recovery.c @@ -6,9 +6,6 @@ static int mt_runtime_scheduler(struct mt7932 *m, bool publish); static int mt_runtime_rebuild(struct mt7932 *m); static void mt_runtime_publish(struct mt7932 *m); static int mt_runtime_ack(struct mt7932 *m, u32 bit); -/* First runtime epoch only. Contracts: NEO_IRQ_SELECTION_AND_RUNTIME_HANDSHAKE - * and NEO_RUNTIME_RING_REBUILD_AND_OBSERVED_PRECAL. - */ static int mt_tx_retire(struct mt7932 *m) { @@ -179,7 +176,7 @@ static int mt_runtime_ack(struct mt7932 *m, u32 bit) int mt_recovery_gate(struct mt7932 *m) { /* Original observed 25F84 CID6b bodies, not PL country tables. - * Explicit experiment; first two action2 field meanings remain opaque. + * The meaning of the first two action2 fields is unknown. */ static const u8 queue_config[3][32] = { {0x10,0x11,2,0,3,0,7,0,0x2f,0,2}, @@ -191,7 +188,10 @@ int mt_recovery_gate(struct mt7932 *m) unsigned int i, phase = 0; int ret; - /* mt_runtime_start() completed configuration before entering this stage. */ + /* + * mt_runtime_start() completed configuration before entering this stage, + * which switches to the runtime ring layout once per firmware boot. + */ if (m->runtime_epoch) return -EINVAL; for (i = 0; i < ARRAY_SIZE(queue_config); i++) { diff --git a/drivers/net/wireless/mediatek/mt7932/reset.c b/drivers/net/wireless/mediatek/mt7932/reset.c index 114da7afd69eb..d93080513c77f 100644 --- a/drivers/net/wireless/mediatek/mt7932/reset.c +++ b/drivers/net/wireless/mediatek/mt7932/reset.c @@ -125,7 +125,7 @@ int mt_function_reset(struct mt7932 *m) if (!peer) return mt_function_reset_locked(m); /* The radio functions may share state beyond PCI's advertised FLR scope. - * No qualified contract permits resetting Wi-Fi while Bluetooth is live. + * Resetting Wi-Fi while Bluetooth is live has never been tested. * Trylock avoids inversion with a concurrent sibling probe/remove; hold * it across FLR to prevent Bluetooth starting after the admission check. */ diff --git a/drivers/net/wireless/mediatek/mt7932/scan.h b/drivers/net/wireless/mediatek/mt7932/scan.h index ae26645cc3693..217fc5c9d1e96 100644 --- a/drivers/net/wireless/mediatek/mt7932/scan.h +++ b/drivers/net/wireless/mediatek/mt7932/scan.h @@ -10,8 +10,8 @@ /* Selected ordinary TX17/fullmac wildcard scan, not a pre-RF gate. * Caller must own BSS0 and filter cfg80211 DISABLED channels first. NO_IR - * channels require mt7932_scan_passive() before submission. - * NEO_FIRST_CFG80211_SCAN_CONTRACT: 20ms dwell and meaningful MT7932 tail. + * channels require mt7932_scan_passive() before submission. The body uses + * a 20 ms dwell and fills the MT7932-specific tail. */ static inline int mt7932_scan_body(u8 *body, size_t capacity, u8 seq, const u8 *channels, unsigned int count) @@ -87,7 +87,7 @@ static inline void mt7932_scan_disconnected_passive(u8 body[0x4d4]) } /* This firmware requires its native wildcard template. Directed scans use - * no probe IEs. Forwarding the normal supplicant extras stalled the campaign + * no probe IEs. Forwarding the normal supplicant extras stalled the scan * on J700; arbitrary caller probe-IE carriage remains unsupported here. */ static inline int mt7932_scan_options(u8 body[0x4d4], const u8 *ssid, From dc1e941d5032492a392b7d5c0d1550f41ce62de2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 14:00:55 +1100 Subject: [PATCH 1262/2680] usb: typec: tipd: Name the CD321x mux error label after what it does The err_unregister_mux label in cd321x_register_port() drops the references to the mux and the DisplayPort connector; nothing is unregistered there. Call it err_put_mux, like err_put_switch above it. No functional change. (cherry picked from commit 79c568dece2359566c17115e56e4018d578ed332) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index db2429cce777d..c2f811be2382f 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -1539,7 +1539,7 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) cd321x->tbt_switch = fwnode_typec_thunderbolt_switch_get(fwnode); if (IS_ERR(cd321x->tbt_switch)) { ret = PTR_ERR(cd321x->tbt_switch); - goto err_unregister_mux; + goto err_put_mux; } cd321x->state.alt = NULL; @@ -1569,7 +1569,7 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) err_put_switch: typec_thunderbolt_switch_put(cd321x->tbt_switch); cd321x->tbt_switch = NULL; -err_unregister_mux: +err_put_mux: fwnode_handle_put(cd321x->connector_fwnode); cd321x->connector_fwnode = NULL; typec_mux_put(cd321x->mux); From 2ff55bd28845b543e9c5c67ae4afe0b299cc6e33 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1263/2680] wifi: mt7932: tidy the Kconfig help and file headers Describe what the driver supports and what it needs in the Kconfig help instead of listing bring-up details, use C++ comment style for the SPDX line of every C file, and remove repeated blank lines. Rename the documentation title, which still called the driver a bring-up. No functional change. (cherry picked from commit 5913e895a3a4632a4d85c21faae55abc8060ad13) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 4 +-- drivers/net/wireless/mediatek/mt7932/Kconfig | 34 +++++++++---------- .../mediatek/mt7932/calibration_transport.c | 4 +-- drivers/net/wireless/mediatek/mt7932/data.c | 2 +- .../net/wireless/mediatek/mt7932/frontend.c | 5 +-- drivers/net/wireless/mediatek/mt7932/link.c | 2 +- drivers/net/wireless/mediatek/mt7932/mt7932.c | 2 -- drivers/net/wireless/mediatek/mt7932/mt7932.h | 2 -- .../net/wireless/mediatek/mt7932/recovery.c | 2 +- .../net/wireless/mediatek/mt7932/startup.c | 2 +- 10 files changed, 25 insertions(+), 34 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index f4ad65f0815f7..58e97162bdba3 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -1,7 +1,7 @@ .. SPDX-License-Identifier: GPL-2.0-only -MT7932 radio bring-up on the MacBook Neo -====================================== +MediaTek MT7932 radio on the MacBook Neo +======================================== J700 MT7932 Wi-Fi uses cfg80211 and NetworkManager with firmware-managed WPA2-CCMP authentication. Bluetooth uses a transport on the radio's second diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index 273c389ad09bd..c6120372d1f3b 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -1,26 +1,26 @@ # SPDX-License-Identifier: GPL-2.0-only config MT7932_FULLMAC - tristate "Experimental J700 MT7932 fullmac wireless support" + tristate "MediaTek MT7932 Wi-Fi on the Apple MacBook Neo (J700)" depends on PCI && OF && ARM64 && IOMMU_DMA depends on PCIE_APPLE_PIODMA_DIAG && CFG80211 select FW_LOADER help - Native support for the built-in J700 Wi-Fi endpoint: managed - WPA2-CCMP station operation on 2.4 GHz and on the 5 GHz UNII-1, -2, - -2e and -3 channels that the regulatory domain permits. Radar (DFS) - channels are scanned passively but not joined. - Requires original firmware, own calibration and original-derived - country policy packages in addition to the cfg80211 regulatory database. - The device is probed before the root filesystem is mounted, so say M - unless those files are in the initramfs. - Requires a trained retained PCI link and a translated SID1 DMA domain. - Firmware, runtime DMA and calibration negotiation start automatically; - cfg80211 and userspace select the country and network. - Once DMA pointers are published the module is pinned until hardware - reset. Normal unload is refused, including with active Bluetooth. - A failed probe reset retains the binding and DMA; a successful checked - reset retires ownership before releasing probe resources. - Other MediaTek devices and other machines are not matched. + Support for the Wi-Fi function of the MediaTek MT7932 radio in the + Apple MacBook Neo (J700). The firmware runs association and the + WPA2-PSK handshake; the driver is a cfg80211 station on 2.4 GHz and + on the 5 GHz UNII-1, -2, -2e and -3 channels that the regulatory + domain permits. Radar (DFS) channels are scanned passively but not + joined. Other MediaTek devices and other machines are not matched. + + The driver needs the original firmware, the unit's own calibration + and the country policy packages under mediatek/mt7932/, besides the + cfg80211 regulatory database. The device is probed before the root + filesystem is mounted, so say M unless those files are in the + initramfs. + + Once the device has been given DMA addresses, the module cannot be + unloaded. A function that fails to initialize and cannot be reset + stays bound and keeps its DMA memory. config MT7932_RF_KUNIT_TEST bool "KUnit tests for MT7932 terminal RF failure" if !KUNIT_ALL_TESTS diff --git a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c index fb5a33dd5aa61..4efd5bed77479 100644 --- a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c +++ b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c @@ -1,8 +1,6 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only #include "mt7932.h" - - /* Called only by the serialized command owner, never from the RX consumer. */ int mt_cal_procedure(struct mt7932 *m, struct mt7932_cal_piece *pieces, unsigned int count, unsigned int logical, u8 context_version) diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index e4f9d8a9f9cc8..99ae52977b582 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only #include "mt7932.h" static bool mt_bss_tx_allowed(struct mt7932 *m); diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 0a5e700d16ccd..8d873a4379bfa 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only #include "mt7932.h" static void mt_scan_finish_work(struct work_struct *work); @@ -15,9 +15,6 @@ struct mt7932 *mt_from_wiphy(struct wiphy *wiphy) return *(struct mt7932 **)wiphy_priv(wiphy); } - - - static void mt_scan_finish_work(struct work_struct *work) { struct mt7932 *m = container_of(work, struct mt7932, scan_finish_work); diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 55a08f5b86728..a1c345ec1f64e 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only #include "mt7932.h" static int mt_peer_reserve(struct mt7932 *m); diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 13b8f97ae5910..41d34e9c6b8c7 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -781,7 +781,6 @@ static int mt_config_gate(struct mt7932 *m) return -ETIMEDOUT; } - static int mt_board_gate(struct mt7932 *m) { const u32 addresses[] = {0x70, 0x160}; @@ -1052,7 +1051,6 @@ void mt_transport_snapshot(struct mt7932 *m) } } - static int mt_initialize(struct mt7932 *m) { int ret; diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index d6c80ecb315ed..6994f0c208105 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -2,7 +2,6 @@ #ifndef MT7932_H #define MT7932_H - #include #include #include @@ -31,7 +30,6 @@ #include "rate.h" #include "capabilities.h" - #define W 0xd4000 #define H 0xd6000 #define C 0xd7000 diff --git a/drivers/net/wireless/mediatek/mt7932/recovery.c b/drivers/net/wireless/mediatek/mt7932/recovery.c index fc60bc05b13d6..31168bfa2b489 100644 --- a/drivers/net/wireless/mediatek/mt7932/recovery.c +++ b/drivers/net/wireless/mediatek/mt7932/recovery.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only #include "mt7932.h" static int mt_tx_retire(struct mt7932 *m); diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index c949c28254355..051ab3793d965 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only #include #include "mt7932.h" From 557cbc76b53b6424f6b8555993684b43d44366a7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 15:32:20 +1100 Subject: [PATCH 1264/2680] usb: typec: tipd: Retry a display mux that is only busy The CD321x driver treats -EBUSY from typec_mux_set() as "the shared external display is driven by the other port": it sets up the port's USB path without the display and does not retry. -EBUSY has other sources on the same mux chain, though. The Apple Type-C PHY returns it while a mode or pipe transition is pending, and the DCP route returned it while the previous SAFE transition was still running and while the external DCP had not finished stopping. Those conditions clear on their own, but the port was left without its display until the next cable event. Have the DCP route report the lasting condition, every pipeline driving another port's display, as -ENOSPC, and the transient ones as -EAGAIN. The CD321x driver now skips the retry only for -ENOSPC, through a helper with a KUnit case; -EBUSY and -EAGAIN are retried like any other failure, -EAGAIN without counting against the port. [ryan: split by topic: the drivers/usb/typec/tipd part of 'drm/apple, usb: typec: tipd: Retry a display mux that is only busy'; the drm/apple/dcp.c part, which returns -ENOSPC and -EAGAIN, is in 200-dcp] (cherry picked from commit e786a6a78a972597b763c82e3968fea5874452df) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/cd321x-pm-test.c | 12 ++++++++++++ drivers/usb/typec/tipd/cd321x-pm.h | 13 +++++++++++++ drivers/usb/typec/tipd/core.c | 4 ++-- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/drivers/usb/typec/tipd/cd321x-pm-test.c b/drivers/usb/typec/tipd/cd321x-pm-test.c index de9db7437dce2..f94f4ad57cb75 100644 --- a/drivers/usb/typec/tipd/cd321x-pm-test.c +++ b/drivers/usb/typec/tipd/cd321x-pm-test.c @@ -417,6 +417,17 @@ static void cd321x_busy_provider_keeps_real_failure_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, pm.force_reconnect); } +/* Only a pipeline owned by another port skips the retry. */ +static void cd321x_display_taken_test(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, cd321x_display_taken(-ENOSPC)); + /* A PHY mid-transition or a pending SAFE state must be retried. */ + KUNIT_EXPECT_FALSE(test, cd321x_display_taken(-EBUSY)); + KUNIT_EXPECT_FALSE(test, cd321x_display_taken(-EAGAIN)); + KUNIT_EXPECT_FALSE(test, cd321x_display_taken(-ETIMEDOUT)); + KUNIT_EXPECT_FALSE(test, cd321x_display_taken(0)); +} + static struct kunit_case cd321x_pm_cases[] = { KUNIT_CASE(cd321x_busy_provider_preserves_connection_test), KUNIT_CASE(cd321x_busy_provider_completion_reopens_test), @@ -440,6 +451,7 @@ static struct kunit_case cd321x_pm_cases[] = { KUNIT_CASE(cd321x_read_and_apply_share_budget_test), KUNIT_CASE(cd321x_status_event_after_exhaustion_test), KUNIT_CASE(cd321x_runtime_failure_revalidates_test), + KUNIT_CASE(cd321x_display_taken_test), {} }; diff --git a/drivers/usb/typec/tipd/cd321x-pm.h b/drivers/usb/typec/tipd/cd321x-pm.h index a96392f88bee4..08421d54c570b 100644 --- a/drivers/usb/typec/tipd/cd321x-pm.h +++ b/drivers/usb/typec/tipd/cd321x-pm.h @@ -2,6 +2,8 @@ #ifndef __CD321X_PM_H__ #define __CD321X_PM_H__ +#include + #include #define CD321X_RESUME_ATTEMPTS 3 @@ -140,4 +142,15 @@ static inline void cd321x_pm_remove(struct cd321x_pm_state *pm) pm->attempts_left = 0; } +/* + * A display mux reports -ENOSPC when every display pipeline is driving + * another port. That lasts until the other display goes away, so the port's + * USB path is set up without the display and nothing is retried. Any other + * error, such as -EBUSY from a PHY in the middle of a transition, is retried. + */ +static inline bool cd321x_display_taken(int ret) +{ + return ret == -ENOSPC; +} + #endif /* __CD321X_PM_H__ */ diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index c2f811be2382f..a311a5826952b 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -691,7 +691,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, if (ret) goto out; ret = typec_mux_set(cd321x->mux, &cd321x->state); - *display_busy = ret == -EBUSY; + *display_busy = cd321x_display_taken(ret); if (!ret) { cd321x->dp_status = dp_data.status; cd321x->dp_conf = dp_data.conf; @@ -817,7 +817,7 @@ static void cd321x_update_work(struct work_struct *work) * a failed teardown keeps the route, but the sink must not stay present. * All ports reference the same external display connector, so only the * port that owns the display route may report a disconnect. A second DP - * sink can enter Alt Mode while its mux request fails with -EBUSY, and + * sink can enter Alt Mode while its mux request fails with -ENOSPC, and * must not tear down the display carried by the other port. */ if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && From 61583bab0dc0f18bcb008c1e14c50511c6a9f4ad Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 15:15:37 +1100 Subject: [PATCH 1265/2680] soc: apple: mailbox, rtkit: Allow holding received messages The receive interrupt of an ASC mailbox is requested IRQF_NO_SUSPEND so that RTKit co-processors stay serviceable during system sleep. A co-processor that keeps sending while the system sleeps then wakes a CPU for every message. Add apple_mbox_hold_rx() and apple_mbox_release_rx(), which mask and unmask the receive interrupt so that messages stay queued in the hardware FIFO, and the RTKit wrappers apple_rtkit_hold_rx() and apple_rtkit_resume_rx() for co-processor drivers. (cherry picked from commit 03426fb46c9ce29e5b58db52cfefd09fac156fea) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/mailbox.c | 12 ++++++++++++ drivers/soc/apple/rtkit.c | 12 ++++++++++++ include/linux/soc/apple/mailbox.h | 6 ++++++ include/linux/soc/apple/rtkit.h | 7 +++++++ 4 files changed, 37 insertions(+) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 2e22c78f57d69..aa62432cc5d95 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -512,6 +512,18 @@ void apple_mbox_stop(struct apple_mbox *mbox) } EXPORT_SYMBOL(apple_mbox_stop); +void apple_mbox_hold_rx(struct apple_mbox *mbox) +{ + disable_irq(mbox->irq_recv_not_empty); +} +EXPORT_SYMBOL(apple_mbox_hold_rx); + +void apple_mbox_release_rx(struct apple_mbox *mbox) +{ + enable_irq(mbox->irq_recv_not_empty); +} +EXPORT_SYMBOL(apple_mbox_release_rx); + /* * Stop the provider-owned CPU, then quiesce every transport edge before * devres releases IRQs or MMIO mappings. Device links order consumer removal diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 4390ba10767b9..10f1bd0b39c50 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -743,6 +743,18 @@ int apple_rtkit_poll(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_poll); +void apple_rtkit_hold_rx(struct apple_rtkit *rtk) +{ + apple_mbox_hold_rx(rtk->mbox); +} +EXPORT_SYMBOL_GPL(apple_rtkit_hold_rx); + +void apple_rtkit_resume_rx(struct apple_rtkit *rtk) +{ + apple_mbox_release_rx(rtk->mbox); +} +EXPORT_SYMBOL_GPL(apple_rtkit_resume_rx); + bool apple_rtkit_has_endpoint(struct apple_rtkit *rtk, u8 ep) { return test_bit(ep, rtk->endpoints); diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index e5697975a50b5..8813438326ffc 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -93,6 +93,12 @@ struct apple_mbox *apple_mbox_get_byname(struct device *dev, const char *name); int apple_mbox_start(struct apple_mbox *mbox); void apple_mbox_stop(struct apple_mbox *mbox); int apple_mbox_poll(struct apple_mbox *mbox); +/* + * Leave received messages in the hardware FIFO, without an interrupt, until + * apple_mbox_release_rx(). For consumers that must not wake the system. + */ +void apple_mbox_hold_rx(struct apple_mbox *mbox); +void apple_mbox_release_rx(struct apple_mbox *mbox); int apple_mbox_send(struct apple_mbox *mbox, struct apple_mbox_msg msg, bool atomic); diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index e4b38a8f6e1ba..00b9a21e936d7 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -239,6 +239,13 @@ int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message, */ int apple_rtkit_poll(struct apple_rtkit *rtk); +/* + * Stop taking interrupts for messages from the co-processor, which then + * stay queued in the mailbox until apple_rtkit_resume_rx(). + */ +void apple_rtkit_hold_rx(struct apple_rtkit *rtk); +void apple_rtkit_resume_rx(struct apple_rtkit *rtk); + /* * Checks if an endpoint with a given index exists * From 85e39d01d48acf8ce42e4111ca7fda3c8e41d591 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1266/2680] Bluetooth: mt7932: drop the enable module parameter enable=0 made probe return -ENODEV, to keep the Bluetooth function unbound while the transport was being brought up. Nothing sets it any more, and a function can be kept unbound with the driver core's driver_override or by blacklisting the module instead. Remove the parameter and the documentation that mentions it. (cherry picked from commit c333bdf9d6ffa1ce011e368398f3b31df1caa6e4) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../networking/device_drivers/wifi/mt7932-neo.rst | 2 +- drivers/bluetooth/Kconfig | 7 +++---- drivers/bluetooth/mt7932_bt_pcie.c | 7 ------- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 58e97162bdba3..a5f0f16a9d46f 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -59,7 +59,7 @@ makes the driver core unbind Bluetooth before Wi-Fi; unloading DMA pointers, and refuses function reset while its Bluetooth sibling is live. The probe rejects a function that is not cold, which is what now prevents a reprobe after a failed or uncertain -admission; ``enable=0`` leaves the function unbound. The transport also +admission. The transport also disables ASPM on the shared link. Like the Wi-Fi driver, it reads its firmware while probing, so build it as a module. The tested Wi-Fi driver owns function 0 and Bluetooth owns function 1. diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index a3664e8e4cbdf..689e1329d6f66 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -31,10 +31,9 @@ config BT_MTK7932_PCIE help Support the J700 Bluetooth PCIe function with locally installed firmware and calibration. The function is probed once the Wi-Fi function has - initialized under the mt7932-fullmac driver; the enable=0 parameter - leaves it unbound. Both functions appear before the root filesystem - is mounted and read their firmware when probed, so say M, like the - Wi-Fi driver. + initialized under the mt7932-fullmac driver. Both functions appear + before the root filesystem is mounted and read their firmware when + probed, so say M, like the Wi-Fi driver. System suspend uses the firmware sleep handshake and preserves DMA storage. Hibernation and firmware reloading are unsupported. A controller that stops answering commands is reset over HCI and set diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index e72692e09c320..a6eb62ac47ad5 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -67,10 +67,6 @@ #define BT7932_CAL "mediatek/j700-mt7932-btcal.bin" #define BT7932_ADDR "mediatek/j700-mt7932-bdaddr.bin" -static bool enable = true; -module_param(enable, bool, 0600); -MODULE_PARM_DESC(enable, "Probe the transport (default: on); no effect on an already bound device"); - struct bt7932_geometry { u16 stride; u16 payload; @@ -1129,9 +1125,6 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) u16 command; int ret; - /* enable=0 leaves the function unbound for diagnosis. */ - if (!READ_ONCE(enable)) - return -ENODEV; if (!of_machine_is_compatible("apple,j700") || !of_machine_is_compatible("apple,t8140") || pci_domain_nr(pdev->bus) || pdev->bus->number != 1 || pdev->devfn != PCI_DEVFN(0, 1)) From 792fa777c6c1f5e197f10b3aac5af0bcf8330c8b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:11:02 +1100 Subject: [PATCH 1267/2680] phy: apple: atc: Quiet the DisplayPort AUX and tunnel clock logging atcphy_enable_dp_aux() logged "Applied T8140 DisplayPort setup lists" once per boot. It comes from enabling the AUX channel, not from where the setup lists are applied, and it reports nothing a user needs. atc_tunnel_start_t602x() logged four raw register values at info level on every tunnel clock preflight. They only help when a preflight is refused, so log them with dev_dbg(). No functional change. (cherry picked from commit c2b2c2626c9b6c12c58a328c1e00f70642537e13) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index f3ed26db09234..59144c1c768b4 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1748,10 +1748,8 @@ static int j700_aux_disable(struct apple_atcphy *atcphy) static int atcphy_enable_dp_aux(struct apple_atcphy *atcphy) { - if (atcphy->hw->has_t8140_dp) { - dev_info_once(atcphy->dev, "Applied T8140 DisplayPort setup lists\n"); + if (atcphy->hw->has_t8140_dp) return j700_aux_enable(atcphy); - } /* FIXME */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8122) @@ -2685,9 +2683,9 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy) outputs = readl(atcphy->regs.core + AUSPLL_CLKOUT_MASTER); command = readl(atcphy->regs.core + AUSPLL_APB_CMD_OVERRIDE); status = readl(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT); - dev_info(atcphy->dev, - "USB4 tunnel clock preflight: +7000=%08x +2200=%08x +2000=%08x +7044=%08x\n", - gates, outputs, command, status); + dev_dbg(atcphy->dev, + "USB4 tunnel clock preflight: +7000=%08x +2200=%08x +2000=%08x +7044=%08x\n", + gates, outputs, command, status); /* * Before the first rate request, TX_DP_CTRL0 gate bits read 0xe001: * the enable gates are set, the byte-clock reset is asserted, and the From 9cbb575fc3fed08281cf41b531cdf1fbde2ea7bb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 22:12:20 +1100 Subject: [PATCH 1268/2680] soc: apple: mailbox: Describe the ASCWrap v6 CPU start and stop in hardware terms Rewrite a set of comments so they state the hardware and firmware behaviour directly: - soc/apple SEP (sks.rs, image.rs, profile.rs, sep.rs): key-store request layouts, timeouts and endpoint init described as the enclave protocol. - soc/apple mailbox: the ASC wrapper CPU start/stop sequences. - accel/ane: register, handshake, ring and command-format notes for the T602x/T8112 Neural Engine, plus the scratch3_ack parameter text. - drm/asahi G17: the tiler-heap shrink policy. - mt7932, reset-apple-cio and two documentation pages: keep the descriptions self-contained. Every edit keeps its line count, so line-dependent data (panic locations, WARN/BUG and dynamic-debug records) is unchanged. Apart from the scratch3_ack module parameter description, the change is comments only; there is no functional change. [ryan: split by topic: the drivers/soc/apple/mailbox.c hunk of 'treewide: describe hardware behaviour in comments'; its other hunks are in 080-radio, 100-sep, 170-usb, 210-gpu, 240-isp and 245-ane] (cherry picked from commit 9266541b8d35c1cd7146eec9a9b31b39fb7f29cc) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/mailbox.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index aa62432cc5d95..9c448ec1fa9ec 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -179,7 +179,7 @@ static void apple_mbox_ascwrap_v6_stop_cpu_locked(struct apple_mbox *mbox) cpu_control = (u8 __iomem *)mbox->wrapper_regs + APPLE_ASCWRAP_V6_CPU_CONTROL; - /* AppleASCWrapV6::_runCPU(false): clear RUN, reread, clear bit 5. */ + /* Stop: clear RUN, reread, clear bit 5. */ val = readl(cpu_control); writel(val & ~APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); val = readl(cpu_control); @@ -239,7 +239,7 @@ int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox) cpu_control = (u8 __iomem *)mbox->wrapper_regs + APPLE_ASCWRAP_V6_CPU_CONTROL; - /* AppleASCWrapV6::_runCPU(true): read, set RUN, write. */ + /* Start: read, set RUN, write. */ val = readl(cpu_control); writel(val | APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); mbox->ascwrap_v6_cpu_running = true; From 94828541200e2e9a0a6937e751c8adc944cbaca7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1269/2680] Bluetooth: mt7932: report setup progress at debug level The driver logged eight lines at info level while probing, among them DMA addresses, and another whenever the HCI device was set up, which happens on every power-on because the setup is not persistent. On a transport fault it also printed eleven error lines, one per ring. Log routine progress and the ring state at debug level. Errors keep their level, and a successful controller recovery is still reported at info level. (cherry picked from commit a70f8e8a419cbe16359b49e9ab9144a72e6bf266) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/mt7932_bt_pcie.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index a6eb62ac47ad5..988fdd9aef4fa 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -177,7 +177,7 @@ static void bt7932_fault_locked(struct bt7932 *bt, const char *reason) dev_err(&bt->pdev->dev, "terminal transport fault: %s; owned DMA retained, external reset required\n", reason); for (channel = 0; channel < BT7932_CHANNELS; channel++) - dev_err(&bt->pdev->dev, "ring%u host=%u firmware=%u previous=%u\n", channel, + dev_dbg(&bt->pdev->dev, "ring%u host=%u firmware=%u previous=%u\n", channel, bt7932_counter(bt, bt->host_transfer, channel), bt7932_counter(bt, bt->fw_transfer, channel), bt->rings[channel].previous); pci_clear_master(bt->pdev); @@ -409,8 +409,8 @@ static int bt7932_arena(struct bt7932 *bt) context[0x51] = 13; context[0x53] = 2; dma_wmb(); - dev_info(&bt->pdev->dev, "owned SID2 arena=%pad bytes=%zu image=%pad bytes=%zu, both low32\n", - &bt->arena_dma, bt->arena_size, &bt->image_dma, bt->image_size); + dev_dbg(&bt->pdev->dev, "owned SID2 arena=%pad bytes=%zu image=%pad bytes=%zu, both low32\n", + &bt->arena_dma, bt->arena_size, &bt->image_dma, bt->image_size); return 0; } @@ -724,7 +724,7 @@ static int bt7932_download(struct bt7932 *bt) } writel(0x188d1807, bt->bar + BT7932_CONNECTION_SELECTOR); writel(1, bt->bar + BT7932_SEMAPHORE_RELEASE); - dev_info(&bt->pdev->dev, "BT_IMAGE_ACCEPTED response1 boot0->2; image remains retained\n"); + dev_dbg(&bt->pdev->dev, "BT_IMAGE_ACCEPTED response1 boot0->2; image remains retained\n"); return 0; } @@ -825,7 +825,7 @@ static void bt7932_channels(struct bt7932 *bt) bt->last_sleep = 0; WRITE_ONCE(bt->awake, true); WRITE_ONCE(bt->ipc_ready, true); - dev_info(&bt->pdev->dev, "BT_IPC_READY ten retained advertisements;127 RX offers per channel\n"); + dev_dbg(&bt->pdev->dev, "BT_IPC_READY ten retained advertisements;127 RX offers per channel\n"); } static int bt7932_open(struct hci_dev *hdev) @@ -948,7 +948,7 @@ static int bt7932_setup(struct hci_dev *hdev) HCI_INIT_TIMEOUT); if (ret) goto fault; - dev_info(&bt->pdev->dev, "BT_HCI_SETUP_OK Reset/calibration/PTX/address completed in order\n"); + dev_dbg(&bt->pdev->dev, "BT_HCI_SETUP_OK Reset/calibration/PTX/address completed in order\n"); mutex_lock(&bt->lock); if (bt->recovery == BT7932_RECOVERY_REOPENING) dev_info(&bt->pdev->dev, "controller recovered\n"); @@ -1059,7 +1059,7 @@ static int bt7932_iommu(struct bt7932 *bt) if (ret) dev_err(dev, "BT_IOMMU_REJECT 32-bit DMA mask status=%d\n", ret); else - dev_info(dev, "BT_IOMMU_ADMITTED RID101->SID2 matched translated DART\n"); + dev_dbg(dev, "BT_IOMMU_ADMITTED RID101->SID2 matched translated DART\n"); } of_node_put(target); return ret; @@ -1176,7 +1176,7 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = -ENOMEM; goto fail; } - dev_info(&pdev->dev, "BT_BAR0 resource=%pR\n", &pdev->resource[0]); + dev_dbg(&pdev->dev, "BT_BAR0 resource=%pR\n", &pdev->resource[0]); ret = pci_read_config_dword(pdev, BT7932_BOOT_CONFIG, &boot); if (ret || boot == U32_MAX || (boot & 0xf) || ((boot >> 4) & 7) > 1) { dev_err(&pdev->dev, "BT_IDENTITY_REJECT config_status=%d boot=%#x\n", ret, boot); @@ -1193,7 +1193,7 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = -ENODEV; goto fail; } - dev_info(&pdev->dev, "BT_COLD_ADMISSION ROM=%u bootstate0 chip7932 SID2\n", bt->rom); + dev_dbg(&pdev->dev, "BT_COLD_ADMISSION ROM=%u bootstate0 chip7932 SID2\n", bt->rom); ret = bt7932_read_inputs(bt); if (ret) goto fail; @@ -1254,7 +1254,7 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret < 0) goto retained_fault; bt->registered = true; - dev_info(&pdev->dev, "BT_HCI_REGISTERED; setup/discovery/connection remain separate gates\n"); + dev_dbg(&pdev->dev, "BT_HCI_REGISTERED; setup/discovery/connection remain separate gates\n"); return 0; retained_fault: mutex_lock(&bt->lock); From 6fe9dd976143823f9e3b6a863899da5e5def41ae Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:11:02 +1100 Subject: [PATCH 1270/2680] phy: apple: atc: Use the register field names in the tunnel clock code atc_tunnel_restore(), atc_tunnel_start_t602x() and the T602X tunnel rate code write raw masks where the fields already have names: 0x54 and BIT(2), BIT(4), BIT(6) are the AUSPLL_CLKOUT_MASTER output drivers, the ACIOPHY_SLEEP_CTRL values are the TX_SMALL, TX_BIG and TX_CLAMP fields and their overrides, and 0xc00, 0xc0 and 0xc000 in ACIOPHY_CFG0 are RX_BIG, RX_SMALL and RX_CLAMP. The comments calling the last three "additional override bits" of unconfirmed meaning were wrong; the sequence is the same one atc_tunnel_wake_t8103() writes for the TX and RX lanes. Use the names. No functional change: every write is the same value. (cherry picked from commit ce5577ad5457fb80906c8deed63f380ad7113c38) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 59144c1c768b4..ea40d045d62f9 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2618,9 +2618,9 @@ static void atc_tunnel_restore(struct apple_atcphy *atcphy) */ core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE | DPTX_PCLK2_ENABLE); - core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(2)); - core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(4)); - core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(6)); + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_PCLK_DRVR_EN); + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_PCLK2_DRVR_EN); + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_REFBUFCLK_DRVR_EN); ret = atc_tunnel_command(atcphy, 3); if (ret) dev_warn(atcphy->dev, "USB4 tunnel clock: AUSPLL power-down was not acked\n"); @@ -2693,7 +2693,7 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy) * state only with no enabled PLL output, no outstanding command and * no lock. Any other active gate state is refused. */ - if (outputs & 0x54 || command & AUSPLL_APB_CMD_OVERRIDE_REQ || + if (outputs & AUSPLL_CLKOUT_MASTER_DRVR_EN || command & AUSPLL_APB_CMD_OVERRIDE_REQ || status & ACIOPHY_AUSPLL_LOCK) return -EBUSY; if ((gates & (DPTX_PCLK1_ENABLE | DPTX_PCLK2_ENABLE | DPRX_PCLK_ENABLE)) && @@ -2719,25 +2719,24 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy) core_clear32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP); core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP_OV); udelay(2); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0x30); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0xc0); + /* same for the TX lanes */ + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_SMALL); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_SMALL_OV); udelay(2); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0x03); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0x0c); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_BIG); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_BIG_OV); udelay(2); - core_clear32(atcphy, ACIOPHY_SLEEP_CTRL, 0x300); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0xc00); + core_clear32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_CLAMP); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_CLAMP_OV); udelay(2); - /* Additional override bit alongside RX_BIG_OV; exact meaning not independently confirmed. */ - core_set32(atcphy, ACIOPHY_CFG0, 0xc00); + /* and for the RX lanes */ + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG); core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG_OV); udelay(2); - /* Additional override bit alongside RX_SMALL_OV; exact meaning not independently confirmed. */ - core_set32(atcphy, ACIOPHY_CFG0, 0xc0); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL); core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL_OV); udelay(2); - /* Additional override bit alongside RX_CLAMP_OV; exact meaning not independently confirmed. */ - core_clear32(atcphy, ACIOPHY_CFG0, 0xc000); + core_clear32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP); core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP_OV); udelay(2); core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, BIT(2)); @@ -2805,7 +2804,7 @@ static void atc_tunnel_stop_t602x(struct apple_atcphy *atcphy, unsigned int dpin * them. A mode change still powers it down in atc_tunnel_restore(). */ if (!atcphy->tunnel_users && atcphy->tunnel_saved) - core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, 0x54); + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_DRVR_EN); } /* Run one T602X DP IN adapter's pixel clock at @rate; atcphy->lock held. */ @@ -2830,12 +2829,13 @@ static int atc_tunnel_set_t602x(struct apple_atcphy *atcphy, unsigned int dpin, * its descriptor until the USB4 PHY changes mode instead of * reprogramming a still-locked PLL on every DPMS wake. */ - core_set32(atcphy, AUSPLL_CLKOUT_MASTER, 0x54); + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_DRVR_EN); ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT, value, value & ACIOPHY_AUSPLL_LOCK, 10, 10000); if (ret) - core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, 0x54); + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, + AUSPLL_CLKOUT_MASTER_DRVR_EN); } if (ret) return ret; From c1ea191946021cf0472d2a20d9d321836102735c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:17:38 +1100 Subject: [PATCH 1271/2680] soc: apple: mailbox: Tidy T8140 comments and resume error logging Drop the references to private bring-up notes and to vendor function names from the T8140 comments, and describe tx_irq_unmasked by what it guards. A failed runtime resume when starting the mailbox or the ASCWrap CPU printed four runtime PM internals next to the error code. The error code is what tells the cause apart and the PM core already reports the state through sysfs, so print just the error. No functional change. (cherry picked from commit 7ed160b9b5f28df712e98088d4368188b2e28072) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/mailbox.c | 19 +++++-------------- include/linux/soc/apple/mailbox.h | 8 +++----- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 9c448ec1fa9ec..3d76ebc59d51f 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -70,9 +70,8 @@ static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_RECV0 == 0x8830); /* - * The T8140 AOP "setup port" (ADT aop-exclave-mailbox): the second mailbox - * the J700 AOP firmware boots through. Layout measured on hardware by the - * native m1n1 microphone host (artifacts/j700-native-lpmic-20260906). + * The T8140 AOP "setup port": a second, smaller mailbox the AOP firmware + * uses during its boot. */ #define APPLE_T8140_AOP_SETUP_A2I_CONTROL 0x000 #define APPLE_T8140_AOP_SETUP_I2A_CONTROL 0x004 @@ -227,11 +226,7 @@ int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox) ret = pm_runtime_resume_and_get(mbox->dev); if (ret) { - dev_err(mbox->dev, - "ASC CPU start: runtime resume failed: %d (active=%d status=%d disable_depth=%d runtime_error=%d)\n", - ret, mbox->active, mbox->dev->power.runtime_status, - mbox->dev->power.disable_depth, - mbox->dev->power.runtime_error); + dev_err(mbox->dev, "ASC CPU start: runtime resume failed: %d\n", ret); apple_mbox_clear_resume_error(mbox); goto out_unlock; } @@ -455,11 +450,7 @@ int apple_mbox_start(struct apple_mbox *mbox) ret = pm_runtime_resume_and_get(mbox->dev); if (ret) { - dev_err(mbox->dev, - "mailbox start: runtime resume failed: %d (active=%d status=%d disable_depth=%d runtime_error=%d)\n", - ret, mbox->active, mbox->dev->power.runtime_status, - mbox->dev->power.disable_depth, - mbox->dev->power.runtime_error); + dev_err(mbox->dev, "mailbox start: runtime resume failed: %d\n", ret); apple_mbox_clear_resume_error(mbox); goto out_unlock; } @@ -698,7 +689,7 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->ascwrap_v6.wrapper_size = resource_size(res); mbox->ascwrap_v6.iop_vbar_start = iop_vbar_res->start; mbox->ascwrap_v6.iop_vbar_size = resource_size(iop_vbar_res); - /* resetState() is software-only; the matching stop lives above. */ + /* T8140 implements both the CPU start and the CPU stop. */ mbox->ascwrap_v6.missing = 0; mbox->wrapper_regs = diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index 8813438326ffc..fcb17eaf2d5a3 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -42,8 +42,6 @@ enum apple_mbox_ascwrap_v6_role { * * The apple-mailbox provider owns both mappings. Consumers request CPU * transitions through the provider and never map either resource themselves. - * T8140 has a complete matching start and stop transform. AppleA7IOP's - * resetState() only updates software state and performs no reset MMIO. */ struct apple_mbox_ascwrap_v6_lifecycle { enum apple_mbox_ascwrap_v6_role role; @@ -80,9 +78,9 @@ struct apple_mbox { void *cookie; /* - * Protected by tx_lock. IRQ, send-error and provider-removal paths - * mask an outstanding unmask only once; rebuild every consumer when - * changing this internal structure's layout. + * Protected by tx_lock: set while irq_send_empty is unmasked, so that + * the IRQ handler, a failed send and provider removal mask it exactly + * once between them. */ bool tx_irq_unmasked; }; From eb13250a8e250aefc5a2e1ece2baef7a995c7a4c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1272/2680] Bluetooth: mt7932: remove write-only packet counters tx_packets, rx_packets and diagnostics were counted but never read. The HCI core keeps its own statistics, which the driver updates as before. No functional change. (cherry picked from commit 5500f5ef688913d1d1718fa6f41124d52dfd979e) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/mt7932_bt_pcie.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 988fdd9aef4fa..c8ea23a06a48b 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -140,7 +140,6 @@ struct bt7932 { bool stopping, runtime_forbidden; enum bt7932_recovery recovery; enum bt7932_pm_state pm_state; - u64 tx_packets, rx_packets, diagnostics; }; static void bt7932_ring_doorbell(struct bt7932 *bt, unsigned int bit) @@ -583,7 +582,6 @@ static void bt7932_tx_work(struct work_struct *work) ring->host = (ring->host + 1) & BT7932_MASK; bt7932_publish_counter(bt, bt->host_transfer, channel, ring->host); bt7932_ring_doorbell(bt, geometry[channel].doorbell); - bt->tx_packets++; bt->hdev->stat.byte_tx += skb->len; mutex_unlock(&bt->lock); kfree_skb(skb); @@ -628,10 +626,7 @@ static int bt7932_rx_locked(struct bt7932 *bt, unsigned int channel) hci_skb_pkt_type(skb) = geo->packet; skb_put_data(skb, descriptor + 16, length); bt->hdev->stat.byte_rx += length; - bt->rx_packets++; hci_recv_frame(bt->hdev, skb); - } else if (!geo->packet) { - bt->diagnostics++; } slot = (slot + 1) & BT7932_MASK; } From 52ba2abf2f35c140a45bb7f160a3f333847fac35 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:11:02 +1100 Subject: [PATCH 1273/2680] phy: apple: atc: Share the tunnel rate selector table atc_tunnel_start_t8103() mapped the DP link rate code to the pixel clock selector with its own switch, a copy of atc_tunnel_rate_selector() used by the T602X code. Move the helper up and use it for both. No functional change. (cherry picked from commit 5fe0c2d6c2ed688a3a9874546885df537fdfd606) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 46 +++++++++++++++-------------------------- 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index ea40d045d62f9..149605ffc38c1 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2469,28 +2469,32 @@ static void atc_tunnel_wake_t8103(struct apple_atcphy *atcphy) udelay(2); } -static int atc_tunnel_start_t8103(struct apple_atcphy *atcphy, u8 rate) +/* The pixel clock selector for a DP link rate code. */ +static int atc_tunnel_rate_selector(u8 rate) { - u32 selector, value; - int ret; - - lockdep_assert_held(&atcphy->lock); switch (rate) { case 0x06: /* RBR */ - selector = 4; - break; + return 4; case 0x0a: /* HBR */ - selector = 3; - break; + return 3; case 0x14: /* HBR2 */ - selector = 1; - break; + return 1; case 0x1e: /* HBR3 */ - selector = 0; - break; + return 0; default: return -EINVAL; } +} + +static int atc_tunnel_start_t8103(struct apple_atcphy *atcphy, u8 rate) +{ + int selector, ret; + u32 value; + + lockdep_assert_held(&atcphy->lock); + selector = atc_tunnel_rate_selector(rate); + if (selector < 0) + return selector; if (atcphy->tunnel_clock_on) { if (atcphy->tunnel_rate == rate) return 0; @@ -2646,22 +2650,6 @@ static void atc_tunnel_restore(struct apple_atcphy *atcphy) atcphy->tunnel_rate = 0; } -static int atc_tunnel_rate_selector(u8 rate) -{ - switch (rate) { - case 0x06: - return 4; - case 0x0a: - return 3; - case 0x14: - return 1; - case 0x1e: - return 0; - default: - return -EINVAL; - } -} - /* * Bring the T602X tunnel PLL up with its fixed descriptor. The rate is not * part of it: each DP IN adapter selects its own rate on PCLK1 or PCLK2 in From 14bb91ae0e589e4f2839289b49e3b11dc27433cf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:17:41 +1100 Subject: [PATCH 1274/2680] arm64: dts: apple: t8140: Fold the trailing soc override into the soc node The display, PMC, pinctrl, I2C, SPI, AOP, ISP and ANE nodes were added through a second "&soc { }" block at the end of the same file, indented as if they were still inside the soc node. Move them into the soc node itself, as the other Apple SoC dtsi files do. The compiled DTB is unchanged. (cherry picked from commit 337b14463d5010296efb0b17dfaa11b8f7601aae) Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 89f12fbd90632..93a7f22266d68 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -1177,10 +1177,7 @@ #mbox-cells = <0>; status = "disabled"; }; - }; -}; -&soc { disp0_dart: iommu@412300000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; reg = <0x4 0x12300000 0x0 0xc000>; @@ -1791,6 +1788,7 @@ power-domains = <&ps_ane_sys>; status = "disabled"; }; + }; }; #include "t8140-pmgr.dtsi" From 252b9c1463d200fe886573ef1306d73ac884775b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:12:43 +1100 Subject: [PATCH 1275/2680] soc: apple: pmp: log tunables the boot loader does not provide at debug level While it starts, the PMP firmware asks for tunables by name, and the driver answers from the apple,tunable-* properties the boot loader copied from the Apple Device Tree. On T8140 the firmware asks for four names that the Apple Device Tree does not carry (pmc-max-bwr-latency-us, mc-quota-policy, mc-cue-dsid0-quota and lts-config), so every boot logs four "unknown property" lines, although the firmware starts and runs without them. A missing tunable is not something the user can act on. Log it at debug level, where it is still available when bringing up a new SoC. (cherry picked from commit 23b2b6f4a1c93dbc90768e271c92bc914e5d676f) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/pmp.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs index 82bdf6df6482d..34975a6fd7769 100644 --- a/drivers/soc/apple/pmp.rs +++ b/drivers/soc/apple/pmp.rs @@ -325,7 +325,9 @@ impl PmpData { } size = len as u32; } else { - dev_info!(self.dev, "unknown property {:?}", name_str); + // Not every boot loader provides every tunable the firmware + // asks for, and the firmware starts without them. + dev_dbg!(self.dev, "unknown property {:?}", name_str); } } state.ioreg_entries.push(size, GFP_KERNEL)?; From ba22aa6ca5ff7a959b44e55dcc0973aadd5b3b09 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1276/2680] Bluetooth: mt7932: describe the driver without bring-up wording The file header called the driver an experiment reconstructed from "protocol contracts", two comments referred to a review, and the module description said "Experimental standalone". Describe the hardware and the behaviour instead, and keep the credit for the transport research. No functional change. (cherry picked from commit 08640f32d3eaf3f460f041abe09395a4aec0e5c2) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/mt7932_bt_pcie.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index c8ea23a06a48b..ed405f799cf88 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -1,8 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * J700 MT7932 Bluetooth experiment, reconstructed from the attributed - * protocol contracts and public Linux interfaces. Original transport research: - * DJ (DjDeveloperr), Ace (Acelogic), and Ryan Murray. + * Bluetooth transport for the MediaTek MT7932 radio in the Apple MacBook Neo + * (J700), whose Bluetooth controller is the second function of the radio's + * PCIe device. Transport research by DJ (DjDeveloperr), Ace (Acelogic) and + * Ryan Murray. * * Firmware and board inputs are supplied locally. System sleep preserves the * transport arena and uses the firmware quiesce/restore handshake. Unload @@ -733,7 +734,7 @@ static int bt7932_ipc_state(struct bt7932 *bt, u8 *state) value = readl(bt->bar + BT7932_STATE); if (value == U32_MAX) return -EIO; - /* The reviewed early-state rule replaces memory with the BAR byte. */ + /* Until the state in memory passes 1, the BAR register holds it. */ *state = value & 0xff; } return 0; @@ -816,7 +817,7 @@ static void bt7932_channels(struct bt7932 *bt) bt7932_publish_counter(bt, bt->host_transfer, channel, BT7932_DEPTH - 1); bt7932_ring_doorbell(bt, geometry[channel].doorbell); } - /* The reviewed cold IPC transition starts ACTIVE; no initial sleep ACK. */ + /* The IPC starts active after a cold start; no sleep ACK is due. */ bt->last_sleep = 0; WRITE_ONCE(bt->awake, true); WRITE_ONCE(bt->ipc_ready, true); @@ -1472,7 +1473,7 @@ static struct pci_driver bt7932_driver = { }; module_pci_driver(bt7932_driver); -MODULE_DESCRIPTION("Experimental standalone J700 MT7932 Bluetooth PCIe transport"); +MODULE_DESCRIPTION("MediaTek MT7932 Bluetooth PCIe transport for the Apple MacBook Neo"); MODULE_LICENSE("GPL"); MODULE_FIRMWARE(BT7932_FW_B0); MODULE_FIRMWARE(BT7932_FW_B1); From 003fcf24a7f13cafaea52c211e53137a237598e6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:12:38 +1100 Subject: [PATCH 1277/2680] phy: apple: atc: Fix coding style in the AUX and tunnel clock code Align the readl_poll_timeout() continuations in the tunnel clock code with the open parenthesis and drop a doubled blank line after the J700 AUX register definitions, as checkpatch asks. No functional change. (cherry picked from commit 1a8a7d1e7d1cfe8966a288cd9c57763a73360381) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 149605ffc38c1..1257990c00ef4 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1699,7 +1699,6 @@ static int atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode #define J700_AUX_CLAMP_SELECT BIT(8) #define J700_AUX_CLAMP_ENABLE BIT(9) - static int j700_aux_enable(struct apple_atcphy *atcphy) { u32 clamp = J700_AUX_CLAMP_SELECT | J700_AUX_CLAMP_ENABLE; @@ -2599,7 +2598,7 @@ static int atc_tunnel_command(struct apple_atcphy *atcphy, u32 command) FIELD_PREP(AUSPLL_APB_CMD_OVERRIDE_CMD, command) | AUSPLL_APB_CMD_OVERRIDE_REQ | AUSPLL_APB_CMD_OVERRIDE_UNK28); ret = readl_poll_timeout(atcphy->regs.core + AUSPLL_APB_CMD_OVERRIDE, - value, value & AUSPLL_APB_CMD_OVERRIDE_ACK, 10, 10000); + value, value & AUSPLL_APB_CMD_OVERRIDE_ACK, 10, 10000); core_clear32(atcphy, AUSPLL_APB_CMD_OVERRIDE, AUSPLL_APB_CMD_OVERRIDE_REQ); return ret; } @@ -2688,8 +2687,8 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy) gates != 0x0000e001) return -EBUSY; ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_CMN_SHM_STS_REG0, - value, value & ACIOPHY_CMN_SHM_STS_REG0_CMD_READY, - 10, 10000); + value, value & ACIOPHY_CMN_SHM_STS_REG0_CMD_READY, + 10, 10000); if (ret) return ret; atcphy->tunnel_attempted = true; @@ -2748,7 +2747,7 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy) if (ret) goto restore; ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT, - value, value & ACIOPHY_AUSPLL_LOCK, 10, 10000); + value, value & ACIOPHY_AUSPLL_LOCK, 10, 10000); if (ret) goto restore; ret = atc_tunnel_command(atcphy, 0x2000); From dc4cb9976a46a8cf4fa065c37a12730a67c83482 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:18:12 +1100 Subject: [PATCH 1278/2680] arm64: dts: apple: t8140: Tidy the mailbox and pinctrl comments Shorten the AOP mailbox comments to what the nodes need: the register layout of the setup mailbox lives in the driver, and the interrupt and register numbering of the firmware's own device tree does not help a reader of this one. Also drop a stray blank line in the pinctrl and pmgr nodes. The compiled DTB is unchanged. (cherry picked from commit 6f1b4ab6f2f036dedd104b83df774d6284ecab74) Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 93a7f22266d68..22b7d0a887d19 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -674,7 +674,6 @@ /* child nodes are added in t8140-pmgr.dtsi */ }; - aic: interrupt-controller@301000000 { compatible = "apple,t8140-aic3", "apple,t8122-aic3"; #interrupt-cells = <3>; @@ -1311,7 +1310,6 @@ gpio-ranges = <&pinctrl_ap 0 0 216>; apple,npins = <216>; - /* Live J700 ADT /arm-io/gpio0 interrupts (7 groups). */ interrupt-controller; #interrupt-cells = <2>; interrupt-parent = <&aic>; @@ -1322,7 +1320,7 @@ , , ; - /* ADT no-restore-gpio-pins entries: 50 and 134. */ + /* Pins the firmware lists as no-restore */ gpio-reserved-ranges = <50 1>, <134 1>; status = "disabled"; @@ -1335,7 +1333,6 @@ pinmux = , ; }; - }; avd: avd@381080000 { @@ -1400,13 +1397,9 @@ }; /* - * AOP (always-on processor). Live J700 ADT /arm-io/aop - * (arm-io child + 0x210000000): reg[2] is the AOP MMIO window, - * reg[0] the ASCWrap v6 block whose mailbox sits at +0x8000. - * ADT interrupts [391, 390, 393, 392] = send-not-empty, - * send-empty, recv-not-empty, recv-empty. The firmware - * (j700aop, RTKit 12) is iBoot-loaded and left stopped; Linux - * adopts it and never reloads it. + * AOP (always-on processor). The bootloader loads its RTKit + * firmware and leaves it stopped; Linux starts it and never + * reloads it. */ aop_mbox: mailbox@310608000 { compatible = "apple,t8140-asc-mailbox", "apple,asc-mailbox-v4"; @@ -1422,14 +1415,7 @@ status = "disabled"; }; - /* - * The AOP "setup port": the second mailbox the J700 AOP firmware - * boots through (ADT /arm-io/aop-exclave-mailbox, absolute - * address 0x35067c000, IRQs [397, 394] = recv-not-empty, - * send-empty). Register layout measured on hardware: - * A2I control +0x0 (FULL bit 16), I2A control +0x4 (EMPTY bit - * 17), send +0x180/+0x188, recv +0x1b0/+0x1b8. - */ + /* The AOP "setup port", a second mailbox used while it boots */ aop_setup_mbox: mailbox@35067c000 { compatible = "apple,t8140-aop-setup-mailbox"; reg = <0x3 0x5067c000 0x0 0x4000>; From 5d7ab5f5e36fcc43ae2d05099b325bf7de49bc1f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:20:16 +1100 Subject: [PATCH 1279/2680] pmdomain: apple: pmp-report: remove the raw DVFS debugfs dump MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On T8140 the driver creates /sys/kernel/debug/apple-pmp-/dvfs, which prints four undocumented 16-byte entries of the report region as raw hex. It was a reverse-engineering aid: nothing reads it, it sits in a second debugfs root next to apple-pmp-report/, and it contradicts both the comment on the status file and the binding, which only allow documented entries to be read. The status file already shows what the driver knows about (whether the PMP runs and the fast die-temperature efforts). Remove it together with its forward declarations. While at it, drop a duplicate include and a dev_set_drvdata() call whose value nothing reads since report entries look their region up in the RCU list. No functional change outside debugfs. [ryan: adapted to 7.2.10 — the DVFS dump this removes was never ported (commit e3163ca622d6 ("soc: apple: expose the T8140 PMP DVFS report for diagnosis") was dropped from 010-soc), so only the now-unused dev_set_drvdata() goes] (cherry picked from commit 742366203cc8a1f9356d82a43d52e2cb6fa97621) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmp-report.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index 285815327fb11..be6ece499c2e7 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -158,7 +158,7 @@ static int apple_pmp_report_probe(struct platform_device *pdev) if (IS_ERR(rep->base)) return PTR_ERR(rep->base); rep->offsets = of_device_get_match_data(dev); - dev_set_drvdata(dev, rep); + INIT_LIST_HEAD(&rep->list); ret = devm_add_action_or_reset(dev, apple_pmp_report_release, rep); if (ret) From fe99d0f3fa118d51973fe42565618c41eadb8c7f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:30:25 +1000 Subject: [PATCH 1280/2680] dmaengine: apple-admac: Add T8140 support The ADMACs of the A18 Pro (T8140) audio fabric (admac-evolution = 1 in Apple's device tree) move the per-channel register blocks. Up to T8122 every channel, with TX (even) and RX (odd) channels interleaved, has a 0x200-byte block at 0x8000 + ch * 0x200. On T8140 the TX blocks are at 0x8000 + (ch / 2) * 0x200 and the RX blocks at 0xc000 + (ch / 2) * 0x200; on the MacBook Neo the block of TX channel 2 (channel 4) answers at 0x8400, not 0x8800. The offsets within a block, the descriptor and report FIFO ports and the start and stop registers are unchanged. Compute the per-channel register addresses from a per-variant layout flag and add apple,t8140-admac, which also sets the REG_UNK_28 bit like T8122. The T8140 ADMACs sit behind the AOP DART, whose windows for them lie above 4 GiB (apple,dma-range). Descriptors carry full 64-bit addresses, so set a 42-bit DMA mask at probe; with the default 32-bit mask the PCM buffers cannot be allocated. The mask applies to every variant. All ADMAC nodes in the tree sit behind a DART, so on the other SoCs the IOVAs handed out remain bounded by the DART's aperture. (cherry picked from commit e1193514aabb819e99f11561685b8ad5bce1ed79) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/dma/apple-admac.c | 94 ++++++++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 31 deletions(-) diff --git a/drivers/dma/apple-admac.c b/drivers/dma/apple-admac.c index 75b75081bccfe..a1aeaf3e4dc8f 100644 --- a/drivers/dma/apple-admac.c +++ b/drivers/dma/apple-admac.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -51,15 +52,28 @@ #define REG_TX_SRAM_SIZE 0x0094 #define REG_RX_SRAM_SIZE 0x0098 -#define REG_CHAN_CTL(ch) (0x8000 + (ch) * 0x200) +/* + * Per-channel control blocks. Up to t8122 every channel (TX and RX + * interleaved as raw channel numbers) has a block at 0x8000 + ch * 0x200. + * The t8140 "evolved" ADMAC keeps the TX blocks at 0x8000 + tx * 0x200 and + * moves the RX blocks to 0xc000 + rx * 0x200 (measured on J700: the + * headphone channel base-ns TX2 answers at +0x8400, not +0x8800). The + * descriptor/report FIFO ports and the start/stop bits are unchanged. + */ +#define REG_CHAN_BASE_LEGACY(ch) (0x8000 + (ch) * 0x200) +#define REG_CHAN_BASE_EVOLVED(ch) (((ch) & 1 ? 0xc000 : 0x8000) + ((ch) / 2) * 0x200) +#define REG_CHAN_BASE(ad, ch) ((ad)->evolved_layout ? \ + REG_CHAN_BASE_EVOLVED(ch) : REG_CHAN_BASE_LEGACY(ch)) + +#define REG_CHAN_CTL(ad, ch) (REG_CHAN_BASE(ad, ch) + 0x00) #define REG_CHAN_CTL_RST_RINGS BIT(0) -#define REG_DESC_RING(ch) (0x8070 + (ch) * 0x200) -#define REG_REPORT_RING(ch) (0x8074 + (ch) * 0x200) +#define REG_DESC_RING(ad, ch) (REG_CHAN_BASE(ad, ch) + 0x70) +#define REG_REPORT_RING(ad, ch) (REG_CHAN_BASE(ad, ch) + 0x74) -#define REG_RESIDUE(ch) (0x8064 + (ch) * 0x200) +#define REG_RESIDUE(ad, ch) (REG_CHAN_BASE(ad, ch) + 0x64) -#define REG_BUS_WIDTH(ch) (0x8040 + (ch) * 0x200) +#define REG_BUS_WIDTH(ad, ch) (REG_CHAN_BASE(ad, ch) + 0x40) #define BUS_WIDTH_WORD_SIZE GENMASK(3, 0) #define BUS_WIDTH_FRAME_SIZE GENMASK(7, 4) @@ -69,11 +83,11 @@ #define BUS_WIDTH_FRAME_2_WORDS 0x10 #define BUS_WIDTH_FRAME_4_WORDS 0x20 -#define REG_CHAN_SRAM_CARVEOUT(ch) (0x8050 + (ch) * 0x200) +#define REG_CHAN_SRAM_CARVEOUT(ad, ch) (REG_CHAN_BASE(ad, ch) + 0x50) #define CHAN_SRAM_CARVEOUT_SIZE GENMASK(31, 16) #define CHAN_SRAM_CARVEOUT_BASE GENMASK(15, 0) -#define REG_CHAN_FIFOCTL(ch) (0x8054 + (ch) * 0x200) +#define REG_CHAN_FIFOCTL(ad, ch) (REG_CHAN_BASE(ad, ch) + 0x54) #define CHAN_FIFOCTL_LIMIT GENMASK(31, 16) #define CHAN_FIFOCTL_THRESHOLD GENMASK(15, 0) @@ -83,8 +97,8 @@ #define REG_TX_INTSTATE(idx) (0x0030 + (idx) * 4) #define REG_RX_INTSTATE(idx) (0x0040 + (idx) * 4) #define REG_GLOBAL_INTSTATE(idx) (0x0050 + (idx) * 4) -#define REG_CHAN_INTSTATUS(ch, idx) (0x8010 + (ch) * 0x200 + (idx) * 4) -#define REG_CHAN_INTMASK(ch, idx) (0x8020 + (ch) * 0x200 + (idx) * 4) +#define REG_CHAN_INTSTATUS(ad, ch, idx) (REG_CHAN_BASE(ad, ch) + 0x10 + (idx) * 4) +#define REG_CHAN_INTMASK(ad, ch, idx) (REG_CHAN_BASE(ad, ch) + 0x20 + (idx) * 4) struct admac_data; struct admac_tx; @@ -132,6 +146,7 @@ struct admac_data { struct admac_sram txcache, rxcache; bool set_unk28; + bool evolved_layout; int irq; int irq_index; int nchannels; @@ -154,6 +169,7 @@ struct admac_tx { struct admac_hw { bool set_unk28; + bool evolved_layout; }; static int admac_alloc_sram_carveout(struct admac_data *ad, @@ -325,7 +341,7 @@ static void admac_cyclic_write_desc(struct admac_data *ad, int channo, int i; for (i = 0; i < 4; i++) { - if (readl_relaxed(ad->base + REG_DESC_RING(channo)) & RING_FULL) + if (readl_relaxed(ad->base + REG_DESC_RING(ad, channo)) & RING_FULL) break; admac_cyclic_write_one_desc(ad, channo, tx); } @@ -359,10 +375,10 @@ static u32 admac_cyclic_read_residue(struct admac_data *ad, int channo, int nreports; size_t pos; - ring1 = readl_relaxed(ad->base + REG_REPORT_RING(channo)); - residue1 = readl_relaxed(ad->base + REG_RESIDUE(channo)); - ring2 = readl_relaxed(ad->base + REG_REPORT_RING(channo)); - residue2 = readl_relaxed(ad->base + REG_RESIDUE(channo)); + ring1 = readl_relaxed(ad->base + REG_REPORT_RING(ad, channo)); + residue1 = readl_relaxed(ad->base + REG_RESIDUE(ad, channo)); + ring2 = readl_relaxed(ad->base + REG_REPORT_RING(ad, channo)); + residue2 = readl_relaxed(ad->base + REG_RESIDUE(ad, channo)); if (residue2 > residue1) { /* @@ -423,9 +439,9 @@ static void admac_start_chan(struct admac_chan *adchan) u32 startbit = 1 << (adchan->no / 2); writel_relaxed(STATUS_DESC_DONE | STATUS_ERR, - ad->base + REG_CHAN_INTSTATUS(adchan->no, ad->irq_index)); + ad->base + REG_CHAN_INTSTATUS(ad, adchan->no, ad->irq_index)); writel_relaxed(STATUS_DESC_DONE | STATUS_ERR, - ad->base + REG_CHAN_INTMASK(adchan->no, ad->irq_index)); + ad->base + REG_CHAN_INTMASK(ad, adchan->no, ad->irq_index)); switch (admac_chan_direction(adchan->no)) { case DMA_MEM_TO_DEV: @@ -463,8 +479,8 @@ static void admac_reset_rings(struct admac_chan *adchan) struct admac_data *ad = adchan->host; writel_relaxed(REG_CHAN_CTL_RST_RINGS, - ad->base + REG_CHAN_CTL(adchan->no)); - writel_relaxed(0, ad->base + REG_CHAN_CTL(adchan->no)); + ad->base + REG_CHAN_CTL(ad, adchan->no)); + writel_relaxed(0, ad->base + REG_CHAN_CTL(ad, adchan->no)); } static void admac_start_current_tx(struct admac_chan *adchan) @@ -473,7 +489,7 @@ static void admac_start_current_tx(struct admac_chan *adchan) int ch = adchan->no; admac_reset_rings(adchan); - writel_relaxed(0, ad->base + REG_CHAN_CTL(ch)); + writel_relaxed(0, ad->base + REG_CHAN_CTL(ad, ch)); admac_cyclic_write_one_desc(ad, ch, adchan->current_tx); admac_start_chan(adchan); @@ -573,7 +589,7 @@ static int admac_alloc_chan_resources(struct dma_chan *chan) return ret; writel_relaxed(adchan->carveout, - ad->base + REG_CHAN_SRAM_CARVEOUT(adchan->no)); + ad->base + REG_CHAN_SRAM_CARVEOUT(ad, adchan->no)); return 0; } @@ -613,7 +629,7 @@ static int admac_drain_reports(struct admac_data *ad, int channo) for (count = 0; count < 4; count++) { u32 countval_hi, countval_lo, unk1, flags; - if (readl_relaxed(ad->base + REG_REPORT_RING(channo)) & RING_EMPTY) + if (readl_relaxed(ad->base + REG_REPORT_RING(ad, channo)) & RING_EMPTY) break; countval_lo = readl_relaxed(ad->base + REG_REPORT_READ(channo)); @@ -632,21 +648,21 @@ static void admac_handle_status_err(struct admac_data *ad, int channo) { bool handled = false; - if (readl_relaxed(ad->base + REG_DESC_RING(channo)) & RING_ERR) { - writel_relaxed(RING_ERR, ad->base + REG_DESC_RING(channo)); + if (readl_relaxed(ad->base + REG_DESC_RING(ad, channo)) & RING_ERR) { + writel_relaxed(RING_ERR, ad->base + REG_DESC_RING(ad, channo)); dev_err_ratelimited(ad->dev, "ch%d descriptor ring error\n", channo); handled = true; } - if (readl_relaxed(ad->base + REG_REPORT_RING(channo)) & RING_ERR) { - writel_relaxed(RING_ERR, ad->base + REG_REPORT_RING(channo)); + if (readl_relaxed(ad->base + REG_REPORT_RING(ad, channo)) & RING_ERR) { + writel_relaxed(RING_ERR, ad->base + REG_REPORT_RING(ad, channo)); dev_err_ratelimited(ad->dev, "ch%d report ring error\n", channo); handled = true; } if (unlikely(!handled)) { dev_err(ad->dev, "ch%d unknown error, masking errors as cause of IRQs\n", channo); - admac_modify(ad, REG_CHAN_INTMASK(channo, ad->irq_index), + admac_modify(ad, REG_CHAN_INTMASK(ad, channo, ad->irq_index), STATUS_ERR, 0); } } @@ -658,7 +674,7 @@ static void admac_handle_status_desc_done(struct admac_data *ad, int channo) int nreports; writel_relaxed(STATUS_DESC_DONE, - ad->base + REG_CHAN_INTSTATUS(channo, ad->irq_index)); + ad->base + REG_CHAN_INTSTATUS(ad, channo, ad->irq_index)); spin_lock_irqsave(&adchan->lock, flags); nreports = admac_drain_reports(ad, channo); @@ -678,7 +694,7 @@ static void admac_handle_status_desc_done(struct admac_data *ad, int channo) static void admac_handle_chan_int(struct admac_data *ad, int no) { - u32 cause = readl_relaxed(ad->base + REG_CHAN_INTSTATUS(no, ad->irq_index)); + u32 cause = readl_relaxed(ad->base + REG_CHAN_INTSTATUS(ad, no, ad->irq_index)); if (cause & STATUS_ERR) admac_handle_status_err(ad, no); @@ -753,7 +769,7 @@ static int admac_device_config(struct dma_chan *chan, struct admac_data *ad = adchan->host; bool is_tx = admac_chan_direction(adchan->no) == DMA_MEM_TO_DEV; int wordsize = 0; - u32 bus_width = readl_relaxed(ad->base + REG_BUS_WIDTH(adchan->no)) & + u32 bus_width = readl_relaxed(ad->base + REG_BUS_WIDTH(ad, adchan->no)) & ~(BUS_WIDTH_WORD_SIZE | BUS_WIDTH_FRAME_SIZE); if (ad->set_unk28) { @@ -798,7 +814,7 @@ static int admac_device_config(struct dma_chan *chan, return -EINVAL; } - writel_relaxed(bus_width, ad->base + REG_BUS_WIDTH(adchan->no)); + writel_relaxed(bus_width, ad->base + REG_BUS_WIDTH(ad, adchan->no)); /* * By FIFOCTL_LIMIT we seem to set the maximal number of bytes allowed to be @@ -809,7 +825,7 @@ static int admac_device_config(struct dma_chan *chan, */ writel_relaxed(FIELD_PREP(CHAN_FIFOCTL_LIMIT, 0x30 * wordsize) | FIELD_PREP(CHAN_FIFOCTL_THRESHOLD, 0x18 * wordsize), - ad->base + REG_CHAN_FIFOCTL(adchan->no)); + ad->base + REG_CHAN_FIFOCTL(ad, adchan->no)); return 0; } @@ -832,6 +848,15 @@ static int admac_probe(struct platform_device *pdev) if (!hw) return -EINVAL; + /* + * Descriptors carry 64-bit addresses; the t8140 dart-aop streams put + * the audio windows above 4 GiB (apple,dma-range), so the default + * 32-bit coherent mask would leave the PCM buffers unallocatable. + */ + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(42)); + if (err) + return dev_err_probe(&pdev->dev, err, "Failed to set DMA mask\n"); + ad = devm_kzalloc(&pdev->dev, struct_size(ad, channels, nchannels), GFP_KERNEL); if (!ad) return -ENOMEM; @@ -840,6 +865,7 @@ static int admac_probe(struct platform_device *pdev) ad->dev = &pdev->dev; ad->nchannels = nchannels; ad->set_unk28 = hw->set_unk28; + ad->evolved_layout = hw->evolved_layout; mutex_init(&ad->cache_alloc_lock); /* @@ -962,7 +988,13 @@ static const struct admac_hw admac_t8122_hw = { .set_unk28 = true, }; +static const struct admac_hw admac_t8140_hw = { + .set_unk28 = true, + .evolved_layout = true, +}; + static const struct of_device_id admac_of_match[] = { + { .compatible = "apple,t8140-admac", .data = &admac_t8140_hw }, { .compatible = "apple,t8122-admac", .data = &admac_t8122_hw }, { .compatible = "apple,t8103-admac", .data = &admac_t8013_hw }, { .compatible = "apple,admac", .data = &admac_t8013_hw }, From 62d547c882610c70c16da57e73a0733fab495db5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1281/2680] Bluetooth: mt7932: add KUnit tests for packet and board file checks bt7932_hci_length() decides whether a packet the controller placed in a receive ring is handed to the HCI core, and bt7932_validate_ptx() whether a board file is sent to the controller. Cover both with KUnit, including truncated and over-long packets and corrupted board files built from synthetic data. The tests are built into the driver when BT_MTK7932_PCIE_KUNIT_TEST is enabled, so that they reach the static helpers. (cherry picked from commit 7a01b6e775a1e1bc6db5973f5af6f3d935ff385a) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/Kconfig | 13 +++ drivers/bluetooth/mt7932_bt_pcie.c | 4 + drivers/bluetooth/mt7932_bt_pcie_test.c | 133 ++++++++++++++++++++++++ 3 files changed, 150 insertions(+) create mode 100644 drivers/bluetooth/mt7932_bt_pcie_test.c diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index 689e1329d6f66..a090b54b5f303 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -43,6 +43,19 @@ config BT_MTK7932_PCIE To build as a module, choose M. The module is mt7932_bt_pcie. +config BT_MTK7932_PCIE_KUNIT_TEST + bool "KUnit tests for the MT7932 Bluetooth transport" if !KUNIT_ALL_TESTS + depends on KUNIT=y && BT_MTK7932_PCIE + default KUNIT_ALL_TESTS + help + Build KUnit tests into the MT7932 Bluetooth transport. They check, + without radio hardware, that HCI packets whose length does not + match their header are rejected in both directions, and that a + board file with a bad layout or checksum is never sent to the + controller. + + If unsure, say N. + config BT_HCIBTUSB tristate "HCI USB driver" depends on USB diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index ed405f799cf88..ef10b3d9efc6b 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -1481,3 +1481,7 @@ MODULE_FIRMWARE(BT7932_FW_B1_FALLBACK); MODULE_FIRMWARE(BT7932_PTX); MODULE_FIRMWARE(BT7932_CAL); MODULE_FIRMWARE(BT7932_ADDR); + +#if IS_ENABLED(CONFIG_BT_MTK7932_PCIE_KUNIT_TEST) +#include "mt7932_bt_pcie_test.c" +#endif diff --git a/drivers/bluetooth/mt7932_bt_pcie_test.c b/drivers/bluetooth/mt7932_bt_pcie_test.c new file mode 100644 index 0000000000000..748dcabde731c --- /dev/null +++ b/drivers/bluetooth/mt7932_bt_pcie_test.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for the checks mt7932_bt_pcie applies to HCI packets and to the + * board file before either reaches the controller or the HCI core. Built as + * part of mt7932_bt_pcie.c, so the tests reach its static helpers. + */ +#include + +static void bt7932_hci_length_command_test(struct kunit *test) +{ + /* HCI_Reset (0x0c03) and a command carrying two parameter bytes. */ + static const u8 reset[] = { 0x03, 0x0c, 0x00 }; + static const u8 command[] = { 0x1a, 0xfc, 0x02, 0x11, 0x22, 0x33 }; + + KUNIT_EXPECT_TRUE(test, bt7932_hci_length(HCI_COMMAND_PKT, reset, sizeof(reset))); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_COMMAND_PKT, reset, 2)); + KUNIT_EXPECT_TRUE(test, bt7932_hci_length(HCI_COMMAND_PKT, command, 5)); + /* One byte short of, and one byte past, the declared length. */ + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_COMMAND_PKT, command, 4)); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_COMMAND_PKT, command, 6)); +} + +static void bt7932_hci_length_event_test(struct kunit *test) +{ + /* Command Complete for HCI_Reset with status 0, then one spare byte. */ + static const u8 event[] = { 0x0e, 0x04, 0x01, 0x03, 0x0c, 0x00, 0xff }; + + KUNIT_EXPECT_TRUE(test, bt7932_hci_length(HCI_EVENT_PKT, event, 6)); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_EVENT_PKT, event, 5)); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_EVENT_PKT, event, 7)); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_EVENT_PKT, event, 1)); +} + +static void bt7932_hci_length_data_test(struct kunit *test) +{ + /* ACL and ISO lengths are 16-bit; ISO keeps two flag bits on top. */ + static const u8 acl[] = { 0x01, 0x20, 0x02, 0x00, 0xaa, 0xbb }; + static const u8 sco[] = { 0x01, 0x00, 0x01, 0xcc }; + static const u8 iso[] = { 0x01, 0x00, 0x01, 0xc0, 0xdd }; + static const u8 long_acl[] = { 0x01, 0x20, 0x00, 0x01 }; + + KUNIT_EXPECT_TRUE(test, bt7932_hci_length(HCI_ACLDATA_PKT, acl, sizeof(acl))); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_ACLDATA_PKT, acl, 5)); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_ACLDATA_PKT, acl, 3)); + /* A 256-byte ACL payload must not be satisfied by its header alone. */ + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_ACLDATA_PKT, long_acl, sizeof(long_acl))); + KUNIT_EXPECT_TRUE(test, bt7932_hci_length(HCI_SCODATA_PKT, sco, sizeof(sco))); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_SCODATA_PKT, sco, 3)); + KUNIT_EXPECT_TRUE(test, bt7932_hci_length(HCI_ISODATA_PKT, iso, sizeof(iso))); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_ISODATA_PKT, iso, 4)); +} + +static void bt7932_hci_length_type_test(struct kunit *test) +{ + static const u8 packet[] = { 0x00, 0x00, 0x00, 0x00 }; + + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_VENDOR_PKT, packet, sizeof(packet))); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(0, packet, sizeof(packet))); +} + +#define BT7932_TEST_PTX_SIZE 198 + +/* A well-formed container with synthetic record contents. */ +static void bt7932_test_ptx(u8 *data) +{ + static const u32 types[] = { 0x101, 0x201, 0x301, 0x401 }; + static const u32 sizes[] = { 14, 44, 44, 0 }; + u32 offset = 96, sum, i, j; + + memset(data, 0, BT7932_TEST_PTX_SIZE); + memcpy(data, "BLOB", 4); + put_unaligned_le32(96, data + 4); + put_unaligned_le16(1, data + 8); + put_unaligned_le16(ARRAY_SIZE(types), data + 10); + for (i = 0; i < ARRAY_SIZE(types); i++) { + u8 *record = data + 16 + i * 20; + + for (j = 0, sum = 0; j < sizes[i]; j++) { + data[offset + j] = 0x40 + i * 16 + j; + sum += data[offset + j]; + } + put_unaligned_le32(types[i], record); + put_unaligned_le32(offset, record + 4); + put_unaligned_le32(sizes[i], record + 8); + put_unaligned_le32(sum, record + 12); + offset += sizes[i]; + } +} + +static void bt7932_validate_ptx_test(struct kunit *test) +{ + u8 *data = kunit_kzalloc(test, BT7932_TEST_PTX_SIZE + 1, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, data); + bt7932_test_ptx(data); + KUNIT_EXPECT_EQ(test, bt7932_validate_ptx(data, BT7932_TEST_PTX_SIZE), 0); + KUNIT_EXPECT_EQ(test, bt7932_validate_ptx(data, BT7932_TEST_PTX_SIZE - 1), -EINVAL); + KUNIT_EXPECT_EQ(test, bt7932_validate_ptx(data, BT7932_TEST_PTX_SIZE + 1), -EINVAL); + + /* A payload byte that no longer matches its record's checksum. */ + data[120]++; + KUNIT_EXPECT_EQ(test, bt7932_validate_ptx(data, BT7932_TEST_PTX_SIZE), -EINVAL); + + bt7932_test_ptx(data); + data[0] = 'b'; + KUNIT_EXPECT_EQ(test, bt7932_validate_ptx(data, BT7932_TEST_PTX_SIZE), -EINVAL); + + /* A record of the wrong type in the first slot. */ + bt7932_test_ptx(data); + put_unaligned_le32(0x201, data + 16); + KUNIT_EXPECT_EQ(test, bt7932_validate_ptx(data, BT7932_TEST_PTX_SIZE), -EINVAL); + + /* A record length other than the one its type has. */ + bt7932_test_ptx(data); + put_unaligned_le32(45, data + 16 + 2 * 20 + 8); + KUNIT_EXPECT_EQ(test, bt7932_validate_ptx(data, BT7932_TEST_PTX_SIZE), -EINVAL); +} + +static struct kunit_case bt7932_test_cases[] = { + KUNIT_CASE(bt7932_hci_length_command_test), + KUNIT_CASE(bt7932_hci_length_event_test), + KUNIT_CASE(bt7932_hci_length_data_test), + KUNIT_CASE(bt7932_hci_length_type_test), + KUNIT_CASE(bt7932_validate_ptx_test), + {} +}; + +static struct kunit_suite bt7932_test_suite = { + .name = "mt7932-bt-pcie", + .test_cases = bt7932_test_cases, +}; + +kunit_test_suite(bt7932_test_suite); From f792164d156fd570522ce004b5b7f383740996d1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:12:39 +1100 Subject: [PATCH 1282/2680] phy: apple: atc: Restore the note on the PHY reset ordering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit atcphy_configure() takes the USB3 PHY out of reset after the lane configuration, while the Asahi driver this is based on carries a note that the reference sequence releases the reset earlier, and that the order is the first thing to revisit if a mode does not come up. The note was lost when the J700 paths were added; put it back. No functional change. [ryan: adapted to 7.2.10 — 170-usb had replaced the note with a comment on the T8140/T8142 RCAL condition; the note goes back into that comment block, above it] (cherry picked from commit 820a7cb83800294d42d71ba7710020060515d23c) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 1257990c00ef4..b05c5d30b9b78 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2962,6 +2962,12 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) /* Take the USB3 PHY out of reset */ core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_PHY_RESET_N); /* + * NOTE: Not what macos does, it takes the phy out of reset right before + * writing to ACIOPHY_CFG0_COMMON_SMALL, does lane configuration with phy + * out of reset, and the poll is right after AUS_COMMON_SHIM_BLK_BIAS_REG + * write. Does not seem to matter, but if something does not work, + * consider re-ordering things to macos order. + * * T8122 only calibrates for the USB3 PIPE; the T8140/T8142 PHYs, woken in * every mode, wait for the calibration in every mode as on 7.1. */ From 1d6a8a9c1127794414a7e1dba8a80bee7311cf9c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:12:20 +1100 Subject: [PATCH 1283/2680] arm64: dts: apple: t8140: state why PMGR domains off at boot are always-on The AOP audio fabric domains (audio_a, audio_fr, audio_p, audio_leap_a and audio_leap_c) are described as "on at boot", but the boot loader leaves them off and the pwrstate driver turns them on at probe. They are always-on because the AOP firmware stops answering LPAI power requests when audio_p goes off underneath it, not because of their boot state. Correct the comments. nub_ocla and smc_i2cm0 are off at boot as well and carry no reason. The Apple Device Tree flags nub_ocla critical, which is what makes a domain always-on in the generated PMGR files of other SoCs (t8122, t8132), and flags smc_i2cm0 on, as it does smc_i2cm1 and smc_i2cm2. Say so. No functional change. (cherry picked from commit 21c1d3f21ac49f4223993ed1beefca1d6d6e89cd) Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index a51052308a74a..0a479f7e8e8e5 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -476,10 +476,11 @@ * aop_he_nrcs0/1, aop_i2cm3 and aop_spi1 are off at boot and the HE * NRCS pair does not power up, so those stay unmanaged). The audio * fabric parents (audio_a, audio_fr, audio_p, audio_leap_a, - * audio_leap_c) are on at boot and the AOP firmware relies on them: - * letting genpd switch audio_p off during a runtime-PM cycle of - * audio_leap_mca left the firmware unable to answer LPAI power - * requests, so they are always-on; only the leaves (audio_leap_tx0, + * audio_leap_c) are off at boot, and the AOP firmware relies on them + * once they are on: letting genpd switch audio_p off during a + * runtime-PM cycle of audio_leap_mca left the firmware unable to + * answer LPAI power requests, so they are always-on and the driver + * turns them on at probe; only the leaves (audio_leap_tx0, * audio_leap_mca, audio_leap_ns_a, ...) are runtime-managed. */ ps_aop_filter: power-controller@4000 { @@ -517,7 +518,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "audio_a"; - /* AOP-owned audio fabric: on at boot, the firmware expects it to stay on */ + /* AOP audio fabric: the firmware expects it to stay on */ apple,always-on; power-domains = <&ps_aop_filter>; }; @@ -538,7 +539,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "audio_fr"; - /* AOP-owned audio fabric: on at boot, the firmware expects it to stay on */ + /* AOP audio fabric: the firmware expects it to stay on */ apple,always-on; power-domains = <&ps_audio_a>; }; @@ -549,7 +550,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "audio_p"; - /* AOP-owned audio fabric: on at boot, the firmware expects it to stay on */ + /* AOP audio fabric: the firmware expects it to stay on */ apple,always-on; power-domains = <&ps_audio_fr>; }; @@ -560,7 +561,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "audio_leap_a"; - /* AOP-owned audio fabric: on at boot, the firmware expects it to stay on */ + /* AOP audio fabric: the firmware expects it to stay on */ apple,always-on; power-domains = <&ps_audio_fr>; }; @@ -607,7 +608,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "audio_leap_c"; - /* AOP-owned audio fabric: on at boot, the firmware expects it to stay on */ + /* AOP audio fabric: the firmware expects it to stay on */ apple,always-on; power-domains = <&ps_audio_leap_a>; }; @@ -1112,7 +1113,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "nub_ocla"; - apple,always-on; + apple,always-on; /* critical in the Apple Device Tree */ status = "okay"; }; @@ -1492,7 +1493,7 @@ #reset-cells = <0>; label = "smc_i2cm0"; power-domains = <&ps_smc_fabric>; - apple,always-on; + apple,always-on; /* on in the Apple Device Tree, like smc_i2cm1/2 */ status = "okay"; }; From bf9d99e4b13e921a82040f669ef5da8fd4640a8b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:20:35 +1100 Subject: [PATCH 1284/2680] dt-bindings: apple: use the canonical SPDX form in PMP, PMC and report bindings Upstream bindings, and checkpatch, expect the dual licence to be written as "(GPL-2.0-only OR BSD-2-Clause)". The PMP, PMP report and PMC bindings leave out the parentheses; the other T8140 power bindings already use them. Add them. The licence is unchanged. (cherry picked from commit 83039e319faf3e49ceabc424c7c69180ac997dfd) Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/power/apple,pmc.yaml | 2 +- .../devicetree/bindings/power/apple,t8140-pmp-report.yaml | 2 +- Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/power/apple,pmc.yaml b/Documentation/devicetree/bindings/power/apple,pmc.yaml index 91c77695178c4..a57b90806511b 100644 --- a/Documentation/devicetree/bindings/power/apple,pmc.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmc.yaml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) %YAML 1.2 --- $id: http://devicetree.org/schemas/power/apple,pmc.yaml# diff --git a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml index 0cddaf6136379..781afa1ad9d9f 100644 --- a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml +++ b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) %YAML 1.2 --- $id: http://devicetree.org/schemas/power/apple,t8140-pmp-report.yaml# diff --git a/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml b/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml index 9784c62fba89e..997761ed5ad4d 100644 --- a/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml +++ b/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) %YAML 1.2 --- $id: http://devicetree.org/schemas/soc/apple/apple,pmp.yaml# From 84757ddc5cb39dfa72a64cd4a64f2496a5a74c76 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 18:48:20 -0400 Subject: [PATCH 1285/2680] dt-bindings: apple: describe J813 monitoring and input peripherals Describe the T8142 mailbox, SMC, MTP, DART and PWM resources. Add explicit contracts for runtime temperature discovery and firmware-owned trackpad reset without embedding sensor tables or calibration. [ryan: split by topic: only the apple,dockchannel-hid.yaml hunk, which belongs with the DockChannel HID binding in 090-input; the other binding hunks are in 010-soc, 020-dart and 060-power-thermal] Signed-off-by: Acelogic (cherry picked from commit e2c698ca7bf1084897e6a3388702661c88986d84) --- .../bindings/input/apple,dockchannel-hid.yaml | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml b/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml index 3b82308e0b568..924442134b248 100644 --- a/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml +++ b/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml @@ -23,7 +23,9 @@ description: | trackpad itself and asks the host to pulse a reset line when it needs one. With power method 2 the host announces each power transition to the coprocessor, drives the reset line of the trackpad analog front end (AFE) - itself and then confirms the transition. + itself and then confirms the transition. On systems with firmware-managed + reset, the same notifications bracket a firmware-owned transition and the + host waits 50 ms during power-on without operating a GPIO. properties: compatible: @@ -140,7 +142,27 @@ properties: default: 1 description: The power sequencing protocol of the coprocessor firmware, as - described above. Power method 2 requires apple,afe-reset-gpios. + described above. Power method 2 requires apple,afe-reset-gpios + unless apple,firmware-managed-reset is present. + + apple,firmware-managed-reset: + type: boolean + description: + The coprocessor owns the trackpad reset signal. The host sends + power method 2 notifications without controlling an AFE GPIO. + Mutually exclusive with apple,afe-reset-gpios on the parent. + + allOf: + - if: + required: + - apple,firmware-managed-reset + then: + properties: + apple,power-method: + const: 2 + required: + - apple,power-method + - firmware-name stm: type: object @@ -171,6 +193,18 @@ required: additionalProperties: false +allOf: + - if: + properties: + multi-touch: + required: + - apple,firmware-managed-reset + required: + - multi-touch + then: + properties: + apple,afe-reset-gpios: false + examples: - | #include From bd9778e63fc7617168056b80ad34459aadfd041a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:30:48 +1100 Subject: [PATCH 1286/2680] dmaengine: apple-admac: Wait for the channel to stop in synchronize device_synchronize() must not return while the controller can still access the memory of terminated descriptors, but admac_synchronize() only kills the tasklet and frees the descriptors. terminate_all() posts the channel's STOP write and resets the rings without waiting for the channel to wind down, so a client that reuses or frees its buffer right after dmaengine_synchronize() races the channel's last burst, and nothing even ensures that the STOP write has reached the controller by then. The START registers hold the per-channel enable bits that the STOP registers clear (m1n1's register map calls them TX_EN/RX_EN and TX_EN_CLR/RX_EN_CLR). Poll the channel's bit until it reads back clear before synchronize goes on; the read also pushes out the posted STOP write. A channel only has a burst to finish, so bound the wait generously and warn instead of hanging the caller if it never clears. (cherry picked from commit 9aa531dee41919fc10404dfb6b0877b1182de815) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/dma/apple-admac.c | 41 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/drivers/dma/apple-admac.c b/drivers/dma/apple-admac.c index a1aeaf3e4dc8f..dfaeeba9d8e83 100644 --- a/drivers/dma/apple-admac.c +++ b/drivers/dma/apple-admac.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -42,6 +43,10 @@ #define T603X_UNK_28_VAL 0x200000 +/* + * One bit per channel: a write to START sets the channel's enable bit, a write + * to STOP clears it, and START reads back the channels that are enabled. + */ #define REG_TX_START 0x0000 #define REG_TX_STOP 0x0004 #define REG_RX_START 0x0008 @@ -474,6 +479,35 @@ static void admac_stop_chan(struct admac_chan *adchan) dev_dbg(adchan->host->dev, "ch%d stop\n", adchan->no); } +/* + * Generous bound for a channel to wind down after its enable bit was cleared: + * it only has to finish the burst in flight, a few frames at audio rates. + */ +#define ADMAC_STOP_TIMEOUT_US 100000 + +static void admac_wait_stopped(struct admac_chan *adchan) +{ + struct admac_data *ad = adchan->host; + u32 bit = BIT(adchan->no / 2); + void __iomem *reg; + u32 val; + + switch (admac_chan_direction(adchan->no)) { + case DMA_MEM_TO_DEV: + reg = ad->base + REG_TX_START; + break; + case DMA_DEV_TO_MEM: + reg = ad->base + REG_RX_START; + break; + default: + return; + } + + if (readl_poll_timeout(reg, val, !(val & bit), 10, ADMAC_STOP_TIMEOUT_US)) + dev_warn_ratelimited(ad->dev, "ch%d still enabled %d us after stop\n", + adchan->no, ADMAC_STOP_TIMEOUT_US); +} + static void admac_reset_rings(struct admac_chan *adchan) { struct admac_data *ad = adchan->host; @@ -564,6 +598,13 @@ static void admac_synchronize(struct dma_chan *chan) unsigned long flags; LIST_HEAD(head); + /* + * terminate_all() only requests the stop: make sure the channel no + * longer touches the memory of the terminated descriptors before the + * client is allowed to reuse or free it. + */ + admac_wait_stopped(adchan); + spin_lock_irqsave(&adchan->lock, flags); list_splice_tail_init(&adchan->to_free, &head); spin_unlock_irqrestore(&adchan->lock, flags); From e9207faa7f8837d1cc81b94730c22f98916802ca Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1287/2680] selftests: mt7932: fail on routine driver messages at info level The radio drivers report scans, association and setup steps at debug level. Make the physical network test fail when either driver logs at info or notice level while it runs, so that a return of per-scan log spam is caught on the hardware. The check reads "dmesg --level=notice,info" and counts the messages of mt7932-fullmac and mt7932_bt_pcie newer than the last kernel message before the test. Add unit tests for the parsing. (cherry picked from commit dcd65e980ce86644008d63590b14d08fc374b053) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/net/mt7932_e2e.py | 27 ++++++++++++++++++- .../selftests/drivers/net/mt7932_e2e_test.py | 22 +++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/drivers/net/mt7932_e2e.py b/tools/testing/selftests/drivers/net/mt7932_e2e.py index 258baebce12f3..f66ccaac201a6 100644 --- a/tools/testing/selftests/drivers/net/mt7932_e2e.py +++ b/tools/testing/selftests/drivers/net/mt7932_e2e.py @@ -25,6 +25,12 @@ r'SCAN_INTERFACE_SETUP_FAILED|RF_FAILED|recovery-required=1|WARNING:|' r'UBSAN:|hung_task|blocked for more than|WIFI_FLR_REFUSED_OR_FAILED', re.I) +# Both radio drivers report progress at debug level, so a message of theirs at +# info or notice level during scans, association or traffic is log spam. +driver_message = re.compile(r'^\[\s*(\d+\.\d+)\] (?:mt7932-fullmac|mt7932_bt_pcie) ' + r'[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]: ', re.M) +log_stamp = re.compile(r'^\[\s*(\d+\.\d+)\]', re.M) + def run(argv, timeout=30): return subprocess.run(argv, text=True, stdout=subprocess.PIPE, @@ -42,6 +48,17 @@ def kernel_ok(): return p.stdout +def last_log_time(log): + """Timestamp of the newest kernel log message, or 0.0 if there is none.""" + stamps = log_stamp.findall(log) + return float(stamps[-1]) if stamps else 0.0 + + +def driver_info_messages(log, since): + """Radio driver messages in a ``dmesg --level=notice,info`` log after @since.""" + return [m.group(0) for m in driver_message.finditer(log) if float(m.group(1)) > since] + + def dns_name(packet, offset): """Decode a bounded DNS name, rejecting bad offsets and compression cycles.""" labels, seen, end, size = [], set(), None, 1 @@ -178,8 +195,9 @@ def main(): compatible = Path('/sys/firmware/devicetree/base/compatible').read_bytes().split(b'\0') require(b'apple,j700' in compatible and b'apple,t8140' in compatible, 'Wrong device') out.mkdir(mode=0o700) + started = None try: - kernel_ok() + started = last_log_time(kernel_ok()) profile = run(['nmcli', '-g', 'connection.uuid', 'connection', 'show', 'id', args.profile]) profile_uuid = profile.stdout.strip() require(profile.returncode == 0 and re.fullmatch(r'[0-9a-f-]{36}', profile_uuid), @@ -272,6 +290,13 @@ def main(): final_log.returncode != 0 or result['kernel_faults']): result['result'] = 'WIFI_NETWORK_E2E_FAILED' result['error'] = 'Final kernel log unavailable or contains a driver fault' + info_log = run(['dmesg', '--level=notice,info']) + if started is not None and info_log.returncode == 0: + result['driver_info_messages'] = len(driver_info_messages(info_log.stdout, started)) + if result['result'] in ('WIFI_SCAN_OBSERVED', 'WIFI_PHYSICAL_NETWORK_E2E_PASS') and ( + info_log.returncode != 0 or result.get('driver_info_messages')): + result['result'] = 'WIFI_NETWORK_E2E_FAILED' + result['error'] = 'Radio drivers logged routine events at info level' (out / 'result.json').write_text(json.dumps(result, indent=2) + '\n') print(json.dumps(result, indent=2)) print('ARTIFACT=' + str(out / 'result.json')) diff --git a/tools/testing/selftests/drivers/net/mt7932_e2e_test.py b/tools/testing/selftests/drivers/net/mt7932_e2e_test.py index 22b45b154f365..75d341cad257f 100644 --- a/tools/testing/selftests/drivers/net/mt7932_e2e_test.py +++ b/tools/testing/selftests/drivers/net/mt7932_e2e_test.py @@ -182,5 +182,27 @@ def test_new_fault_markers_are_terminal(self): self.assertIsNotNone(e2e.fault.search(message)) +class DriverLogTest(unittest.TestCase): + LOG = ('[ 10.000001] mt7932-fullmac 0000:01:00.0: probe message\n' + '[ 20.500000] usb 1-1: new high-speed USB device\n' + '[ 21.000000] mt7932-fullmac 0000:01:00.0: SCAN_REQUEST: channels=38\n' + '[ 21.250000] mt7932_bt_pcie 0000:01:00.1: controller recovered\n' + '[ 22.000000] Bluetooth: hci0: unrelated core message\n') + + def test_last_log_time(self): + self.assertEqual(e2e.last_log_time(self.LOG), 22.0) + self.assertEqual(e2e.last_log_time(''), 0.0) + + def test_only_radio_messages_after_start_count(self): + self.assertEqual(len(e2e.driver_info_messages(self.LOG, 0.0)), 3) + self.assertEqual(e2e.driver_info_messages(self.LOG, 20.5), + ['[ 21.000000] mt7932-fullmac 0000:01:00.0: ', + '[ 21.250000] mt7932_bt_pcie 0000:01:00.1: ']) + self.assertEqual(e2e.driver_info_messages(self.LOG, 22.0), []) + + def test_quiet_log_has_no_driver_messages(self): + self.assertEqual(e2e.driver_info_messages('[ 1.000000] usb 1-1: reset\n', 0.0), []) + + if __name__ == '__main__': unittest.main() From 09f495f9c3584ee56dda4c9b33ae418b47ce0e31 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 21:25:06 +1100 Subject: [PATCH 1288/2680] usb: dwc3: Export dwc3_event_buffers_cleanup() The Apple glue calls dwc3_event_buffers_cleanup() when it stops the core, but the function is not exported. With USB_DWC3=y and USB_DWC3_APPLE=m (or both modular) modpost fails: ERROR: modpost: "dwc3_event_buffers_cleanup" [drivers/usb/dwc3/dwc3-apple.ko] undefined! Export it like the other core helpers the glue drivers use (dwc3_enable_susphy(), dwc3_core_exit()). Fixes: f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate") (cherry picked from commit e87b71da6428f91b547d7405a82a038aac2e982d) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index 34627ddeaaa1d..86e9cdf935c0d 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -606,6 +606,7 @@ void dwc3_event_buffers_cleanup(struct dwc3 *dwc) reg = dwc3_readl(dwc, DWC3_GEVNTCOUNT(0)); dwc3_writel(dwc, DWC3_GEVNTCOUNT(0), reg); } +EXPORT_SYMBOL_GPL(dwc3_event_buffers_cleanup); static void dwc3_core_num_eps(struct dwc3 *dwc) { From 32f5a5892a02fc28abf3a0f94291821c8d8308ef Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:13:40 +1100 Subject: [PATCH 1289/2680] arm64: dts: apple: t8140-j700: trim measurement notes from comments The PMC node's comment records how the DCS power model and the decision not to limit the DCS rail were measured, down to the benchmark and the game used. Keep what the values are and why no zone uses the limit; the measurements belong in the history, not in the device tree. Two PCIe comments call themselves experimental; describe what the nodes do instead. No functional change: the compiled device tree is identical. (cherry picked from commit 0d1799c7c179d3b6e914eeaee59da7529e824565) Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 033272cdb32d9..942e360f57c4e 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -424,16 +424,12 @@ &pmc { /* - * System power above DCS state 0 at each DCS state, measured with - * glmark2's terrain scene, GPU-bound, on the J700. + * System power above DCS state 0 at each DCS state, GPU-bound. * - * No thermal zone limits the DCS rail. The PMP already drops its - * DCS vote when the GPU idles, and under load the rail must stay - * where the PMP puts it: a power allocator zone on Tp0t at 95 C - * stepped the limit between states 1 and 4 every 1-2 s in - * Minecraft, swinging the game's GPU submission rate about 2.5x. - * With the limit lifted the rate held within 8% and Tp0t settled - * near 100 C, under the 110 C critical trips of the CPU zones. + * No thermal zone limits the DCS rail: the PMP drops its DCS vote + * when the GPU idles, and a power allocator zone on Tp0t made the + * limit oscillate between states 1 and 4 under GPU load. Without + * it Tp0t stays below the 110 C critical trips of the CPU zones. */ apple,dcs-power-microwatt = <0 919000 1453000 2058000 2407000>; status = "okay"; @@ -1826,7 +1822,7 @@ #include "hwmon-laptop.dtsi" &pcie0 { - /* Experiment: guard stays closed until the retained bootstrap succeeds. */ + /* The PIODMA guard stays closed until the retained bootstrap succeeds. */ apple,piodma = <&pcie0_piodma_diag>; iommu-map = <0x100 &pcie0_dart 1 2>; iommu-map-mask = <0xffff>; @@ -1848,7 +1844,7 @@ }; }; -/* Experimental SID17 bootstrap; arena ownership lasts until full reset. */ +/* SID17 bootstrap; the arena stays owned until a full reset. */ &pcie0_dart { status = "okay"; apple,dma-range = /bits/ 64 <0 0x1000000000>; From fd32c7371dc3b45afde4a6aa70b7710b4cd46115 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:17:39 +1100 Subject: [PATCH 1290/2680] iommu: apple-dart: Demote informational probe messages to debug Every DART with an "apple,dma-offset" property, such as the T8140 SEP DART, prints the aliased DMA window at info level on each boot, and the tunneled PCIe-C DART prints its fixed topology and a pair of messages around applying its tunables. These describe an expected configuration taken from the device tree, so log them at debug level like the plain "Limiting DMA range" message already is, and fold the tunables pair into one message. Also merge the two comments explaining why a tunneled DART's parameter registers are not read, and describe the access ordering requirement without reference to how it was found. No functional change. (cherry picked from commit f80758084279064eb1c60e6bfa56f7649bb86a4d) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 53 +++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 30 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index de3a5fbd3ed7e..84cf4e3805e5f 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -322,12 +322,11 @@ struct apple_dart { /* * PCIe-C is a cable-powered aperture and can report a failed transaction as * an asynchronous external abort unless each access is completed before the - * next instruction retires. A live /dev/mem diagnostic using the same - * Device-nGnRnE attributes and the exact 64-SID reset sequence is stable when - * every access is followed by dsb sy + isb. ioremap_np() supplies the memory - * type, but writel() only supplies the normal I/O ordering barrier on arm64. - * Match the proven access semantics for the tunneled DART while leaving all - * always-on DARTs on the normal fast path. + * next instruction retires. The DART reset sequence is only stable with + * Device-nGnRnE attributes and a dsb sy + isb after every access. + * ioremap_np() supplies the memory type, but writel() only supplies the + * normal I/O ordering barrier on arm64, so add the barriers here for the + * tunneled DART and leave all always-on DARTs on the normal fast path. */ static inline void apple_dart_writel(struct apple_dart *dart, u32 value, u32 offset) @@ -2656,9 +2655,9 @@ static int apple_dart_probe(struct platform_device *pdev) * PCIe-C is a cable-powered aperture whose writes must complete before * the following transaction. The normal arm64 ioremap() mapping is * Device-nGnRE and allowed the first posted DART reset write to surface as - * an asynchronous external abort at a later instruction. The identical - * reset sequence is stable through /dev/mem's Device-nGnRnE mapping, and - * PCI configuration forwarding already requires the same non-posted + * an asynchronous external abort at a later instruction. The same + * reset sequence is stable with a Device-nGnRnE mapping, and PCI + * configuration forwarding already requires the same non-posted * ordering. Keep ordinary always-on DARTs on their existing mapping. */ if (tunneled) @@ -2716,26 +2715,20 @@ static int apple_dart_probe(struct platform_device *pdev) if (ret) goto err_clk_disable; if (tunneled && dart->tunables) { - dev_info(dev, "applying %zu firmware PCIe-C DART tunables\n", - dart->tunables->sz); + dev_dbg(dev, "applying %zu PCIe-C DART tunables\n", + dart->tunables->sz); apple_dart_apply_tunables(dart); - dev_info(dev, "firmware PCIe-C DART tunables applied\n"); } - /* - * The cable-powered T8110 DART does not provide a safely probeable - * parameter block at this point in PCIe tunnel activation. The SError - * raised by these reads is asynchronous, which made the following trace - * boundary look guilty on different boots. The device tree supplies the - * immutable topology: 16 KiB pages, 42-bit addresses and 64 SIDs. Use that - * description before making any DART MMIO access. - */ if (tunneled) { /* - * PARAMS is not safe to read on a tunneled DART. T8110 is - * 16 KiB pages, 42-bit addresses, 64 streams and four levels. - * A T8020 PCIe-C DART is 16 KiB pages, a 32-bit IOVA, a 36-bit - * PA and as many streams as its register block describes. + * The parameter registers of a tunneled DART cannot be read + * safely at this point of PCIe tunnel activation: the read + * raises an asynchronous SError. The topology is fixed + * instead. T8110 is 16 KiB pages, 42-bit addresses, 64 + * streams and four levels. A T8020 PCIe-C DART is 16 KiB + * pages, a 32-bit IOVA, a 36-bit PA and as many streams as its + * register block describes. */ if (dart->hw->type == DART_T8110) { dart->pgsize = 1 << DART_PCIEC_PAGE_SHIFT; @@ -2752,9 +2745,9 @@ static int apple_dart_probe(struct platform_device *pdev) dart->num_streams = dart->hw->max_sid_count; dart->four_level = false; } - dev_info(dev, - "PCIe-C DART using fixed topology: pagesize %#x, %u streams, AS %u -> %u\n", - dart->pgsize, dart->num_streams, dart->ias, dart->oas); + dev_dbg(dev, + "PCIe-C DART using fixed topology: pagesize %#x, %u streams, AS %u -> %u\n", + dart->pgsize, dart->num_streams, dart->ias, dart->oas); goto params_done; } @@ -2826,9 +2819,9 @@ static int apple_dart_probe(struct platform_device *pdev) ret = -EINVAL; goto err_clk_disable; } else { - dev_info(dev, - "aliasing DMA range %pad..%pad at DART offset %pad\n", - &dart->dma_min, &dart->dma_max, &dart->dma_offset); + dev_dbg(dev, + "aliasing DMA range %pad..%pad at DART offset %pad\n", + &dart->dma_min, &dart->dma_max, &dart->dma_offset); } /* From 98d3e5c1256a8dab79201e0108b397ae391db980 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:40:28 +1100 Subject: [PATCH 1291/2680] media: apple: avd: h264: Refuse a slice with no data after its header stream_slice() finds the slice data by skipping two bytes and then the slice header, and hands the decoder the rest of the slice buffer as coded data. It never checks that anything is left: the length is payload_len - off in unsigned arithmetic. A one-byte OUTPUT buffer yields a coded data address one byte past the end of the allocation and a length of 0xffffffff, and the decoder reads far outside the buffer until its DART faults. A header that runs to the end of the buffer yields an empty slice placed just past its end. Find the slice data before the slice is copied, refuse a slice that has no data byte after its header, and push the offset that was found. Add KUnit tests for 0-, 1- and 2-byte payloads, a header that ends one byte before, at and beyond the end of the buffer, and an emulation prevention byte inside the header. Fixes: 0765a0974a15 ("media: apple: avd: add a dedicated "submit" step") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/Kconfig | 12 ++++ drivers/media/platform/apple/avd/Makefile | 1 + .../media/platform/apple/avd/avd-h264-test.c | 66 +++++++++++++++++++ drivers/media/platform/apple/avd/avd-h264.c | 61 ++++++++++++----- drivers/media/platform/apple/avd/avd.h | 5 ++ 5 files changed, 130 insertions(+), 15 deletions(-) create mode 100644 drivers/media/platform/apple/avd/avd-h264-test.c diff --git a/drivers/media/platform/apple/avd/Kconfig b/drivers/media/platform/apple/avd/Kconfig index d01634d938857..ed5ad44276c31 100644 --- a/drivers/media/platform/apple/avd/Kconfig +++ b/drivers/media/platform/apple/avd/Kconfig @@ -17,3 +17,15 @@ config VIDEO_APPLE_AVD Build the decoder firmware from https://github.com/aurora-silicon/avd-fw and install it under /lib/firmware/apple/. J700/T8140 uses the avd-fw-v5-t0.bin image. Firmware is supplied separately from this driver. + +config VIDEO_APPLE_AVD_KUNIT_TEST + tristate "KUnit tests for the Apple video decoder" if !KUNIT_ALL_TESTS + depends on VIDEO_APPLE_AVD && KUNIT + default KUNIT_ALL_TESTS + help + Build KUnit tests for the Apple video decoder. They exercise the + checks that the H.264 path makes on the slices that userspace + queues, such as finding the slice data after the slice header, + at the limits and one past them. + + If unsure, say N. diff --git a/drivers/media/platform/apple/avd/Makefile b/drivers/media/platform/apple/avd/Makefile index 39b438f49f936..4d1877c414d41 100644 --- a/drivers/media/platform/apple/avd/Makefile +++ b/drivers/media/platform/apple/avd/Makefile @@ -2,4 +2,5 @@ apple-avd-y := avd-drv.o avd-v4l2.o avd-hw.o avd-h264.o avd-vp9.o avd-hevc.o \ avd-av1.o avd-av1-entropymode.o +apple-avd-$(CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST) += avd-h264-test.o obj-$(CONFIG_VIDEO_APPLE_AVD) += apple-avd.o diff --git a/drivers/media/platform/apple/avd/avd-h264-test.c b/drivers/media/platform/apple/avd/avd-h264-test.c new file mode 100644 index 0000000000000..d814d2644719a --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-h264-test.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit tests for the H.264 slice checks of the Apple video decoder. + */ + +#include + +#include "avd.h" + +struct avd_slice_data_case { + const char *desc; + const u8 *data; + u32 len; + u32 header_len; + bool found; + u32 off; +}; + +static const u8 avd_test_slice[] = { + 0x25, 0xb8, 0x00, 0x00, 0x04, 0x01, 0xaa, 0xbb, +}; + +/* The same slice with an emulation prevention byte in its header. */ +static const u8 avd_test_slice_epb[] = { + 0x25, 0xb8, 0x00, 0x00, 0x03, 0x01, 0xaa, 0xbb, +}; + +static const struct avd_slice_data_case avd_slice_data_cases[] = { + { "empty payload", avd_test_slice, 0, 0 }, + { "one-byte payload", avd_test_slice, 1, 0 }, + { "two-byte payload", avd_test_slice, 2, 0 }, + { "one byte of data", avd_test_slice, 3, 2, true, 2 }, + { "header ends one byte early", avd_test_slice, 8, 7, true, 7 }, + { "header ends at the end", avd_test_slice, 8, 8 }, + { "header beyond the end", avd_test_slice, 8, 12 }, + { "header without emulation prevention", avd_test_slice, 8, 5, true, 5 }, + { "emulation prevention in the header", avd_test_slice_epb, 8, 5, true, 6 }, +}; + +KUNIT_ARRAY_PARAM_DESC(avd_slice_data, avd_slice_data_cases, desc); + +static void avd_slice_data_test(struct kunit *test) +{ + const struct avd_slice_data_case *c = test->param_value; + u32 off = U32_MAX; + + KUNIT_EXPECT_EQ(test, avd_h264_find_slice_data(c->data, c->len, + c->header_len, &off), + c->found); + if (c->found) + KUNIT_EXPECT_EQ(test, off, c->off); + else + KUNIT_EXPECT_EQ(test, off, U32_MAX); +} + +static struct kunit_case avd_h264_test_cases[] = { + KUNIT_CASE_PARAM(avd_slice_data_test, avd_slice_data_gen_params), + {} +}; + +static struct kunit_suite avd_h264_test_suite = { + .name = "apple-avd-h264", + .test_cases = avd_h264_test_cases, +}; + +kunit_test_suite(avd_h264_test_suite); diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index 8876784820d22..942c3f0a2ede6 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -13,6 +13,7 @@ * Tomasz Figa */ +#include #include #include @@ -46,6 +47,7 @@ struct avd_h264_run { s32 cur_poc; u8 num_valid; + u32 slice_data_off; }; /* state */ @@ -386,6 +388,39 @@ static void stream_weights(struct avd_ctx *ctx, struct avd_h264_run *run) } } +/* + * Find the slice data in a slice: skip the two bytes in front of the slice + * header, then the header_len bytes of the header itself, which do not count + * its emulation prevention bytes. At least one byte of slice data must follow. + */ +VISIBLE_IF_KUNIT bool avd_h264_find_slice_data(const u8 *data, u32 len, + u32 header_len, u32 *off) +{ + u32 pos = 2, bytes_read = 2; + + while (bytes_read < header_len && pos < len) { + if (data[pos - 2] != 0x00 || data[pos - 1] != 0x00 || + data[pos] != 0x03) + bytes_read++; + pos++; + } + + if (pos >= len) + return false; + + *off = pos; + return true; +} + +/* Bytes of slice header in front of the slice data. */ +static u32 avd_h264_header_len(const struct avd_h264_run *run) +{ + /* CABAC slice data starts on a byte boundary. */ + bool cabac = run->pps->flags & V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE; + + return (run->slice_params->header_bit_size + (cabac ? 7 : 0)) / 8; +} + static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) { const struct v4l2_ctrl_h264_decode_params *decode = run->decode_params; @@ -396,21 +431,7 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) u32 payload_len = h264_ctx->active_slice->size; bool en_mode = (pps->flags & V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE) == 0; - const u8 *data = h264_ctx->active_slice->cpu; - - u32 min_off = (sl->header_bit_size + (en_mode ? 0 : 7)) / 8; - - /* emulation byte. header_bit_size does not include this. - * I wonder if its worth it to just not support this. */ - u32 off = 2; - u32 bytes_read = 2; - - while (bytes_read < min_off && off < payload_len) { - if (data[off - 2] != 0x00 || data[off - 1] != 0x00 || - data[off] != 0x03) - bytes_read++; - off++; - } + u32 off = run->slice_data_off; dma_addr_t slc_a84 = h264_ctx->active_slice->addr + off; @@ -796,6 +817,16 @@ static int avd_h264_run(struct avd_ctx *ctx) struct vb2_v4l2_buffer *src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); u32 payload_len = vb2_get_plane_payload(&src->vb2_buf, 0); const u8 *data = vb2_plane_vaddr(&src->vb2_buf, 0); + + if (!data || !avd_h264_find_slice_data(data, payload_len, + avd_h264_header_len(&run), + &run.slice_data_off)) { + dev_dbg_ratelimited(ctx->dev->dev, + "slice has no data after its header\n"); + avd_run_postamble(ctx, &run.base); + return -EINVAL; + } + h264_ctx->active_slice = &h264_ctx->slices[h264_ctx->slice_num]; ret = avd_buf_alloc(ctx->dev, h264_ctx->active_slice, payload_len); if (ret) diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index e5b991a00fa47..0bf17ef883e4d 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -292,6 +292,11 @@ void avd_job_finish(struct avd_ctx *ctx, enum vb2_buffer_state result); void avd_run_preamble(struct avd_ctx *ctx, struct avd_run *run); void avd_run_postamble(struct avd_ctx *ctx, struct avd_run *run); +#if IS_ENABLED(CONFIG_KUNIT) +bool avd_h264_find_slice_data(const u8 *data, u32 len, u32 header_len, + u32 *off); +#endif + extern const struct avd_coded_fmt_ops avd_h264_fmt_ops; extern const struct avd_coded_fmt_ops avd_hevc_fmt_ops; extern const struct avd_coded_fmt_ops avd_vp9_fmt_ops; From d1b8796e6ef26fd6e55e2a223355126af3dde2af Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:40:44 +1100 Subject: [PATCH 1292/2680] media: apple: avd: h264: Bound the slices held for one picture Every slice of a picture is copied into its own coherent DMA buffer and held until the picture has been decoded. The client chooses how many slices it queues with V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF and how large they are, and since the slice table grows on demand nothing limits either: a client that never sends the last slice of a picture makes the driver grow the slice and job tables and pin one DMA buffer per slice until allocations fail. Bound a picture by what H.264 allows and by the memory that it can need. A slice codes at least one macroblock, so a picture has no more slices than macroblocks, counting both fields of a frame, and at most MAX_SLICES, the size of the fixed table that the growable one replaced. A picture of PCM macroblocks is about the size of the decoded picture and emulation prevention adds at most one byte for every two, so the coded data of a picture fits in twice the decoded buffer; a budget of one page per slice keeps room for small pictures cut into many slices. Check both limits before the table grows or the next slice is allocated, count each slice at its page-aligned allocation size, grow the table no further than the limit and allocate it with kcalloc(). When a picture goes over, drop its slices together with the unsubmitted job table that points at them, so that the next picture decodes normally. Add KUnit tests for both limits at the limit and one over, and for a slice size that would wrap 32 bits when page-aligned. Fixes: b00e2172cbb5 ("fixup! media: apple: avd: add a dedicated "submit" step") Assisted-by: LLM Signed-off-by: Ryan Murray --- .../media/platform/apple/avd/avd-h264-test.c | 55 ++++++++++ drivers/media/platform/apple/avd/avd-h264.c | 100 ++++++++++++++---- drivers/media/platform/apple/avd/avd.h | 2 + 3 files changed, 138 insertions(+), 19 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-h264-test.c b/drivers/media/platform/apple/avd/avd-h264-test.c index d814d2644719a..89f0e749b1c05 100644 --- a/drivers/media/platform/apple/avd/avd-h264-test.c +++ b/drivers/media/platform/apple/avd/avd-h264-test.c @@ -4,6 +4,7 @@ */ #include +#include #include "avd.h" @@ -53,8 +54,62 @@ static void avd_slice_data_test(struct kunit *test) KUNIT_EXPECT_EQ(test, off, U32_MAX); } +/* The size of the slice table in avd-h264.c */ +#define AVD_TEST_MAX_SLICES 4096 + +struct avd_slice_fits_case { + const char *desc; + u16 width_mbs; + u16 height_map_units; + bool fields; + size_t held; + size_t held_bytes; + u32 len; + size_t decoded_size; + bool fits; +}; + +static const struct avd_slice_fits_case avd_slice_fits_cases[] = { + { "one-macroblock frame, first slice", 1, 1, false, 0, 0, 16, 384, true }, + { "one-macroblock frame, second slice", 1, 1, false, 1, PAGE_SIZE, 16, 384 }, + { "one-macroblock field pair, second slice", 1, 1, true, 1, PAGE_SIZE, 16, 384, true }, + { "one-macroblock field pair, third slice", 1, 1, true, 2, 2 * PAGE_SIZE, 16, 384 }, + { "320x240, last slice", 20, 15, false, 299, 299 * PAGE_SIZE, 16, 115200, true }, + { "320x240, one slice over", 20, 15, false, 300, 300 * PAGE_SIZE, 16, 115200 }, + { "1920x1088, last slice of the table", 120, 68, false, + AVD_TEST_MAX_SLICES - 1, 0, 16, 3133440, true }, + { "1920x1088, one slice over the table", 120, 68, false, + AVD_TEST_MAX_SLICES, 0, 16, 3133440 }, + { "data up to the budget", 120, 68, false, 1, SZ_512M - PAGE_SIZE, + PAGE_SIZE, SZ_256M, true }, + { "data one byte over the budget", 120, 68, false, 1, + SZ_512M - PAGE_SIZE, PAGE_SIZE + 1, SZ_256M }, + { "no decoded size, data up to the page budget", 120, 68, false, 1, + (AVD_TEST_MAX_SLICES - 1) * PAGE_SIZE, 1, 0, true }, + { "no decoded size, data over the page budget", 120, 68, false, 1, + AVD_TEST_MAX_SLICES * PAGE_SIZE, 1, 0 }, + { "slice of almost 4 GiB", 120, 68, false, 0, 0, U32_MAX, SZ_256M }, +}; + +KUNIT_ARRAY_PARAM_DESC(avd_slice_fits, avd_slice_fits_cases, desc); + +static void avd_slice_fits_test(struct kunit *test) +{ + const struct avd_slice_fits_case *c = test->param_value; + struct v4l2_ctrl_h264_sps sps = { + .pic_width_in_mbs_minus1 = c->width_mbs - 1, + .pic_height_in_map_units_minus1 = c->height_map_units - 1, + .flags = c->fields ? 0 : V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY, + }; + + KUNIT_EXPECT_EQ(test, avd_h264_slice_fits(&sps, c->held, c->held_bytes, + c->len, c->decoded_size), + c->fits); +} + static struct kunit_case avd_h264_test_cases[] = { KUNIT_CASE_PARAM(avd_slice_data_test, avd_slice_data_gen_params), + KUNIT_CASE_PARAM(avd_slice_fits_test, avd_slice_fits_gen_params), {} }; diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index 942c3f0a2ede6..e8bc8d0066ead 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -30,6 +31,12 @@ #define H264_TRANSFORM_8X8_MODE(v) FIELD_PREP(BIT(7), !!(v)) +/* + * Most slices held for one picture. This bounds the slice table, the job + * table and the memory that a client can pin. + */ +#define MAX_SLICES 4096 + struct avd_h264_run { struct avd_run base; @@ -63,6 +70,8 @@ struct avd_h264_ctx { struct avd_buf *active_slice; size_t slice_num; size_t alloc_slice_num; + /* allocated size of the held slices */ + size_t slice_bytes; struct avd_h264_bufs { struct avd_buf inst; @@ -611,11 +620,21 @@ static int avd_h264_validate_sps(struct avd_ctx *ctx, return 0; } +/* Free the slices held for the current picture. */ +static void avd_h264_drop_slices(struct avd_ctx *ctx) +{ + struct avd_h264_ctx *h264_ctx = ctx->priv; + + for (size_t i = 0; i < h264_ctx->slice_num; i++) + avd_buf_free(ctx->dev, &h264_ctx->slices[i]); + h264_ctx->slice_num = 0; + h264_ctx->slice_bytes = 0; +} + static void avd_h264_stop(struct avd_ctx *ctx) { struct avd_h264_ctx *h264_ctx = ctx->priv; struct avd_dev *dev = ctx->dev; - int i; if (!h264_ctx) return; @@ -628,8 +647,7 @@ static void avd_h264_stop(struct avd_ctx *ctx) avd_buf_free(dev, &h264_ctx->bufs.ip_above); avd_buf_free(dev, &h264_ctx->bufs.mv_above_info); - for (i = 0; i < h264_ctx->slice_num; i++) - avd_buf_free(dev, &h264_ctx->slices[i]); + avd_h264_drop_slices(ctx); kfree(h264_ctx->slices); kfree(h264_ctx); @@ -659,9 +677,8 @@ static int avd_h264_start(struct avd_ctx *ctx) /* assume one slice per row */ h264_ctx->alloc_slice_num = ctrl->p_new.p_h264_sps->pic_height_in_map_units_minus1 + 1; - h264_ctx->slices = - kzalloc(sizeof(*h264_ctx->slices) * h264_ctx->alloc_slice_num, - GFP_KERNEL); + h264_ctx->slices = kcalloc(h264_ctx->alloc_slice_num, + sizeof(*h264_ctx->slices), GFP_KERNEL); if (!h264_ctx->slices) { ret = -ENOMEM; goto err_free_ctx; @@ -709,7 +726,7 @@ static void avd_h264_run_preamble(struct avd_ctx *ctx, struct avd_h264_run *run) run->addresses.mv_color = run->base.y_out + (dst_len - mv_color_len); } -static int avd_h264_realloc_slices(struct avd_ctx *ctx) +static int avd_h264_realloc_slices(struct avd_ctx *ctx, size_t max_slices) { struct avd_h264_ctx *h264_ctx = ctx->priv; void *tmp; @@ -719,7 +736,8 @@ static int avd_h264_realloc_slices(struct avd_ctx *ctx) /* grow by half, and by at least one for a one-row picture */ alloc_slice_num = h264_ctx->alloc_slice_num + max_t(size_t, h264_ctx->alloc_slice_num / 2, 1); - h264_ctx->slices = kzalloc(sizeof(*h264_ctx->slices) * alloc_slice_num, + alloc_slice_num = min(alloc_slice_num, max_slices); + h264_ctx->slices = kcalloc(alloc_slice_num, sizeof(*h264_ctx->slices), GFP_KERNEL); if (!h264_ctx->slices) { /* make deallocating a little easier */ @@ -799,6 +817,42 @@ static bool avd_h264_refs_valid(const struct avd_h264_run *run) return true; } +/* + * Every slice codes at least one macroblock, so a picture has no more slices + * than macroblocks, counting both fields of a frame. + */ +static size_t avd_h264_max_slices(const struct v4l2_ctrl_h264_sps *sps) +{ + size_t mbs = (size_t)(sps->pic_width_in_mbs_minus1 + 1) * + (sps->pic_height_in_map_units_minus1 + 1); + + if (!(sps->flags & V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY)) + mbs *= 2; + + return min_t(size_t, mbs, MAX_SLICES); +} + +/* + * Coded data held for one picture. A picture of PCM macroblocks is about the + * size of the decoded one and emulation prevention adds at most one byte for + * every two, so twice the decoded buffer is enough; MAX_SLICES pages leave + * room for small pictures cut into many slices. + */ +static size_t avd_h264_slice_budget(size_t decoded_size) +{ + return max_t(size_t, 2 * decoded_size, MAX_SLICES * PAGE_SIZE); +} + +/* Check that one more slice of len bytes fits in the current picture. */ +VISIBLE_IF_KUNIT bool avd_h264_slice_fits(const struct v4l2_ctrl_h264_sps *sps, + size_t held, size_t held_bytes, + u32 len, size_t decoded_size) +{ + return held < avd_h264_max_slices(sps) && + held_bytes + PAGE_ALIGN((size_t)len) <= + avd_h264_slice_budget(decoded_size); +} + static int avd_h264_run(struct avd_ctx *ctx) { struct avd_h264_ctx *h264_ctx = ctx->priv; @@ -808,12 +862,6 @@ static int avd_h264_run(struct avd_ctx *ctx) avd_h264_run_preamble(ctx, &run); - if (h264_ctx->slice_num >= h264_ctx->alloc_slice_num) { - ret = avd_h264_realloc_slices(ctx); - if (ret) - goto err_free_jobs; - } - struct vb2_v4l2_buffer *src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); u32 payload_len = vb2_get_plane_payload(&src->vb2_buf, 0); const u8 *data = vb2_plane_vaddr(&src->vb2_buf, 0); @@ -827,12 +875,30 @@ static int avd_h264_run(struct avd_ctx *ctx) return -EINVAL; } + if (!avd_h264_slice_fits(run.sps, h264_ctx->slice_num, + h264_ctx->slice_bytes, payload_len, + ctx->decoded_fmt.fmt.pix_mp.plane_fmt[0].sizeimage)) { + dev_err_ratelimited(ctx->dev->dev, + "too many slices for one picture, dropping it\n"); + /* The unsubmitted job table points at the dropped slices. */ + avd_h264_drop_slices(ctx); + ret = -EINVAL; + goto err_free_jobs; + } + + if (h264_ctx->slice_num >= h264_ctx->alloc_slice_num) { + ret = avd_h264_realloc_slices(ctx, avd_h264_max_slices(run.sps)); + if (ret) + goto err_free_jobs; + } + h264_ctx->active_slice = &h264_ctx->slices[h264_ctx->slice_num]; ret = avd_buf_alloc(ctx->dev, h264_ctx->active_slice, payload_len); if (ret) goto err_free_jobs; memcpy(h264_ctx->active_slice->cpu, data, payload_len); h264_ctx->slice_num++; + h264_ctx->slice_bytes += PAGE_ALIGN((size_t)payload_len); h264_ctx->monochrome = run.sps->chroma_format_idc == 0; /* Build the P/B{0,1} ref lists. */ @@ -887,14 +953,10 @@ static void avd_h264_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, struct vb2_v4l2_buffer *dst_buf, enum vb2_buffer_state result) { - struct avd_dev *avd = ctx->dev; struct avd_h264_ctx *h264_ctx = ctx->priv; - int i; if (!(src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF)) { - for (i = 0; i < h264_ctx->slice_num; i++) - avd_buf_free(avd, &h264_ctx->slices[i]); - h264_ctx->slice_num = 0; + avd_h264_drop_slices(ctx); /* 4:0:0 has no chroma to decode; the picture is grey */ if (h264_ctx->monochrome && dst_buf && result == VB2_BUF_STATE_DONE) { diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index 0bf17ef883e4d..fd77a0163e8c5 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -295,6 +295,8 @@ void avd_run_postamble(struct avd_ctx *ctx, struct avd_run *run); #if IS_ENABLED(CONFIG_KUNIT) bool avd_h264_find_slice_data(const u8 *data, u32 len, u32 header_len, u32 *off); +bool avd_h264_slice_fits(const struct v4l2_ctrl_h264_sps *sps, size_t held, + size_t held_bytes, u32 len, size_t decoded_size); #endif extern const struct avd_coded_fmt_ops avd_h264_fmt_ops; From 3a67d4dcfebd1a458bc3b1d33a1c9180eabca8a8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:21:28 +1100 Subject: [PATCH 1293/2680] iommu: apple-dart: Invalidate every stream in a domain bitmap Stream remapping is configured separately for each master's stream map. Two masters attached to the same domain can therefore have independent translation roots. The domain's aggregate bitmap does not imply that all streams are remapped to its first stream. Stopping after that stream can leave stale translations reachable by another master after an unmap. Issue a command for every selected stream, as the T8020 engine already does with its stream-select bitmap. Keep the remap optimization when programming translations; redundant invalidates for remapped streams are preferable to skipping an independent root. This also covers bitmaps used for reset and locked-domain maintenance. Add a KUnit case that attaches singleton masters at SIDs 3 and 5 to one paging domain, maps and unmaps an IOVA, and records the real command loop's writes to check that both streams are maintained. Simulate registers and record writes using a static stub enabled only with the driver tests. Use the T8110 16 KiB table granule and skip kernels whose pages are smaller than the device supports. Fixes: 1143d0263e3a ("iommu: apple-dart: Use remap feature") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/Kconfig | 4 +- drivers/iommu/apple-dart-test.c | 100 ++++++++++++++++++++++++++++++++ drivers/iommu/apple-dart.c | 11 +++- 3 files changed, 111 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index a23c2a126c7ac..f0106834f2e15 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -317,7 +317,7 @@ config APPLE_DART Say Y here if you are using an Apple SoC. config APPLE_DART_KUNIT_TEST - bool "KUnit tests for Apple DART locked domains" if !KUNIT_ALL_TESTS + bool "KUnit tests for Apple DART domains and commands" if !KUNIT_ALL_TESTS depends on APPLE_DART=y && KUNIT=y default KUNIT_ALL_TESTS help @@ -325,6 +325,8 @@ config APPLE_DART_KUNIT_TEST handoff without accessing physical DART hardware. Verify that legacy display roots can be replaced while newer firmware-owned roots retain their mappings and reject conflicting host entries. + Verify that unmapping an IOVA maintains independent remap targets. + That case requires kernel pages of at least 16 KiB. config S390_IOMMU def_bool y if S390 && PCI diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 3280fb0435c6c..cd8686502c58b 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -412,7 +412,107 @@ static void apple_dart_test_gated_irq(struct kunit *test) pm_runtime_disable(dart->dev); } +struct apple_dart_test_commands { + DECLARE_BITMAP(sids, DART_MAX_STREAMS); + unsigned int count; +}; + +static void apple_dart_test_record_write(struct apple_dart *dart, u32 value, + u32 offset) +{ + struct kunit *test = kunit_get_current_test(); + struct apple_dart_test_commands *commands = test->priv; + + if (offset == DART_T8110_TLB_CMD) { + __set_bit(FIELD_GET(DART_T8110_TLB_CMD_STREAM, value), commands->sids); + commands->count++; + } + writel(value, dart->regs + offset); +} + +/* Separate masters on one DART need not share a remap target. */ +static void apple_dart_test_independent_remap_roots(struct kunit *test) +{ + struct apple_dart_master_cfg masters[2] = {}; + struct apple_dart_test_commands commands = {}; + struct iommu_iotlb_gather gather; + struct apple_dart_domain *domain; + struct apple_dart *dart; + size_t mapped = 0; + unsigned int i; + + /* T8110 table pointers must be aligned to its 16 KiB granule. */ + if (PAGE_SIZE < SZ_16K) { + kunit_skip(test, "T8110 needs kernel pages of at least 16 KiB"); + return; + } + + dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart); + domain = kunit_kzalloc(test, sizeof(*domain), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, domain); + dart->dev = kunit_device_register(test, "apple-dart-remap-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dart->dev); + dart->hw = &apple_dart_hw_t8110; + dart->num_streams = 16; + dart->pgsize = SZ_16K; + dart->ias = 32; + dart->oas = 42; + dart->regs = (__force void __iomem *)kunit_kzalloc(test, SZ_16K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart->regs); + spin_lock_init(&dart->lock); + mutex_init(&domain->init_lock); + domain->domain.type = IOMMU_DOMAIN_UNMANAGED; + test->priv = &commands; + kunit_activate_static_stub(test, apple_dart_writel, apple_dart_test_record_write); + pm_runtime_set_active(dart->dev); + pm_runtime_enable(dart->dev); + + for (i = 0; i < ARRAY_SIZE(masters); i++) { + struct device *dev; + unsigned int sid = 3 + 2 * i; + int ret; + + dev = kunit_device_register(test, i ? "dart-master-5" : "dart-master-3"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + dev->iommu = kunit_kzalloc(test, sizeof(*dev->iommu), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dev->iommu); + masters[i].dma_max = DMA_BIT_MASK(32); + masters[i].stream_maps[0].dart = dart; + __set_bit(sid, masters[i].stream_maps[0].sidmap); + dev->iommu->priv = &masters[i]; + ret = apple_dart_attach_dev_paging(&domain->domain, dev, NULL); + dev->iommu = NULL; + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, sid)), + dart->hw->tcr_enabled); + } + + KUNIT_ASSERT_EQ(test, apple_dart_map_pages(&domain->domain, 0, SZ_1M, + SZ_16K, 1, IOMMU_READ | IOMMU_WRITE, + GFP_KERNEL, &mapped), 0); + KUNIT_ASSERT_EQ(test, mapped, (size_t)SZ_16K); + KUNIT_ASSERT_EQ(test, apple_dart_iotlb_sync_map(&domain->domain, 0, SZ_16K), 0); + KUNIT_EXPECT_EQ(test, apple_dart_iova_to_phys(&domain->domain, 0), (phys_addr_t)SZ_1M); + + bitmap_zero(commands.sids, DART_MAX_STREAMS); + commands.count = 0; + iommu_iotlb_gather_init(&gather); + KUNIT_EXPECT_EQ(test, apple_dart_unmap_pages(&domain->domain, 0, SZ_16K, 1, + &gather), (size_t)SZ_16K); + apple_dart_iotlb_sync(&domain->domain, &gather); + KUNIT_EXPECT_EQ(test, apple_dart_iova_to_phys(&domain->domain, 0), (phys_addr_t)0); + KUNIT_EXPECT_EQ(test, commands.count, 2U); + KUNIT_EXPECT_EQ(test, commands.sids[0], BIT(3) | BIT(5)); + + pm_runtime_disable(dart->dev); + free_io_pgtable_ops(domain->pgtbl_ops); + kunit_deactivate_static_stub(test, apple_dart_writel); + test->priv = NULL; +} + static struct kunit_case apple_dart_test_cases[] = { + KUNIT_CASE(apple_dart_test_independent_remap_roots), KUNIT_CASE(apple_dart_test_before_attach), KUNIT_CASE(apple_dart_test_locked_handoff), KUNIT_CASE(apple_dart_test_sync_unknown_roots), diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 84cf4e3805e5f..397c00ecb4efe 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -41,6 +41,10 @@ #include "dma-iommu.h" +#if IS_ENABLED(CONFIG_APPLE_DART_KUNIT_TEST) +#include +#endif + #if IS_ENABLED(CONFIG_KUNIT) && !defined(MODULE) #include #endif @@ -331,6 +335,9 @@ struct apple_dart { static inline void apple_dart_writel(struct apple_dart *dart, u32 value, u32 offset) { +#if IS_ENABLED(CONFIG_APPLE_DART_KUNIT_TEST) + KUNIT_STATIC_STUB_REDIRECT(apple_dart_writel, dart, value, offset); +#endif writel(value, dart->regs + offset); if (unlikely(dart->tunneled)) { mb(); @@ -990,6 +997,7 @@ apple_dart_t8110_hw_tlb_command_range(struct apple_dart_stream_map *stream_map, spin_unlock_irqrestore(&dart->lock, flags); return -EHOSTDOWN; } + /* A domain bitmap can contain independent remap targets. */ for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { u32 val = FIELD_PREP(DART_T8110_TLB_CMD_OP, command) | FIELD_PREP(DART_T8110_TLB_CMD_STREAM, sid); @@ -1018,9 +1026,6 @@ apple_dart_t8110_hw_tlb_command_range(struct apple_dart_stream_map *stream_map, DART_T8110_TLB_CMD); if (ret) break; - /* Since remap is used only the first sid needs to be invalidated. */ - if (apple_dart_remaps_streams(dart)) - break; } spin_unlock_irqrestore(&dart->lock, flags); if (ret) From 30c42119f3ad18c8c6886d0202611327dd6a995f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:58:25 +1100 Subject: [PATCH 1294/2680] drm/apple: Rank preferred timing modes ahead of the score parse_mode() marks a mode that the firmware flags IsPreferred by ORing bit 32 into its score, and enumerate_modes() then picks the highest score. The firmware score is a full 64-bit value, so the OR is not a reliable bonus: a preferred mode whose score already has bit 32 set gains nothing, and a mode that is not preferred beats a preferred one whenever its score exceeds the preferred score with bit 32 set. The winner is marked DRM_MODE_TYPE_PREFERRED, so userspace can be offered the wrong timing. Keep the firmware score as it is, record IsPreferred in the parsed mode, and rank the modes by the pair: a preferred mode ranks above every mode that is not preferred, and the score orders the modes of each group. The external SDR path is unchanged: it parses IsPreferred on its own and never sets the flag. Add KUnit tests for scores below, at and above bit 32 and for two preferred modes. Fixes: 6d20a3172ffe ("drm/apple: Use "IsPreferred" property during mode selection") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/parser.c | 24 +++++++++-- drivers/gpu/drm/apple/parser.h | 7 ++++ drivers/gpu/drm/apple/parser_mode_test.c | 53 ++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 drivers/gpu/drm/apple/parser_mode_test.c diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index b282f798dfcdb..2830b35720b6d 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -14,6 +14,7 @@ appledrm-y += iomfb_v14_7.o appledrm-y += iomfb_h17p.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += iomfb_h17p_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple_resume_test.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += parser_mode_test.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 73e143bf0f3a5..7034310e619e2 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright 2021 Alyssa Rosenzweig */ +#include #include #include #include @@ -811,9 +812,6 @@ static int parse_mode(struct dcp_parse_ctx *handle, out->automatic_rgb8_quirk = handle->automatic_1080p180_rgb8; } - if (is_preferred) - *score |= (1L << 32); - if (out->sdr_rgb.score >= 0) best_color_mode = out->sdr_rgb.id; else if (out->sdr_444.score >= 0) @@ -902,6 +900,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, out->timing_mode_id = id; out->color_mode_id = best_color_mode; + out->preferred = is_preferred; trace_iomfb_timing_mode(handle->dcp, id, *score, horiz.active, vert.active, vert.precise_sync_rate, @@ -1008,6 +1007,19 @@ static bool dcp_180hz_ids_current(struct dcp_parse_ctx handle, return matches == 1; } +/* + * A timing mode that the firmware marks as preferred ranks above every mode + * that it does not mark; the firmware score orders the modes of each group. + */ +VISIBLE_IF_KUNIT bool dcp_mode_ranks_above(bool preferred, s64 score, + bool best_preferred, s64 best_score) +{ + if (preferred != best_preferred) + return preferred; + + return score > best_score; +} + struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, int height_mm, unsigned notch_height, @@ -1019,6 +1031,7 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, struct dcp_display_mode *mode, *modes; struct dcp_display_mode *best_mode = NULL; s64 score, best_score = -1; + bool best_preferred = false; ret = iterator_begin(handle, &it, false); @@ -1054,8 +1067,11 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, /* Process a successful mode */ (*count)++; - if (!mode->automatic_rgb8_quirk && score > best_score) { + if (!mode->automatic_rgb8_quirk && + dcp_mode_ranks_above(mode->preferred, score, best_preferred, + best_score)) { best_score = score; + best_preferred = mode->preferred; best_mode = mode; } } diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 3b21263154754..98e3fbe4a1ded 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -100,6 +100,8 @@ struct dcp_display_mode { struct dcp_color_mode sdr; struct dcp_color_mode best; bool vrr; + /* IsPreferred, outside the external SDR path */ + bool preferred; bool metadata_timing_valid; bool automatic_rgb8_quirk; u64 automatic_connection_sequence; @@ -119,6 +121,11 @@ struct dimension { int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx); bool dcp_property_complete(const void *blob, size_t received, size_t advertised); +#if IS_ENABLED(CONFIG_KUNIT) +bool dcp_mode_ranks_above(bool preferred, s64 score, bool best_preferred, + s64 best_score); +#endif + struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, int height_mm, unsigned notch_height, diff --git a/drivers/gpu/drm/apple/parser_mode_test.c b/drivers/gpu/drm/apple/parser_mode_test.c new file mode 100644 index 0000000000000..40d0c8d9cd196 --- /dev/null +++ b/drivers/gpu/drm/apple/parser_mode_test.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * KUnit tests for the ranking of the timing modes that DCP reports. + */ + +#include +#include + +#include "parser.h" + +struct dcp_mode_rank_case { + const char *desc; + bool preferred; + s64 score; + bool best_preferred; + s64 best_score; + bool above; +}; + +static const struct dcp_mode_rank_case dcp_mode_rank_cases[] = { + { "first mode", false, 0, false, -1, true }, + { "preferred over a score above bit 32", true, 0, false, BIT_ULL(32) + 1, true }, + { "score above bit 32 under a preferred mode", false, BIT_ULL(32) + 1, true, 0 }, + { "preferred with bit 32 already set", true, BIT_ULL(32), false, BIT_ULL(32) + 1, true }, + { "higher of two preferred scores", true, 5, true, 4, true }, + { "lower of two preferred scores", true, 4, true, 5 }, + { "higher of two other scores", false, 7, false, 3, true }, + { "equal ranks keep the first mode", true, 3, true, 3 }, +}; + +KUNIT_ARRAY_PARAM_DESC(dcp_mode_rank, dcp_mode_rank_cases, desc); + +static void dcp_mode_rank_test(struct kunit *test) +{ + const struct dcp_mode_rank_case *c = test->param_value; + + KUNIT_EXPECT_EQ(test, dcp_mode_ranks_above(c->preferred, c->score, + c->best_preferred, + c->best_score), + c->above); +} + +static struct kunit_case dcp_mode_rank_test_cases[] = { + KUNIT_CASE_PARAM(dcp_mode_rank_test, dcp_mode_rank_gen_params), + {} +}; + +static struct kunit_suite dcp_mode_rank_test_suite = { + .name = "apple-dcp-mode-rank", + .test_cases = dcp_mode_rank_test_cases, +}; + +kunit_test_suite(dcp_mode_rank_test_suite); From fbff0ab798eb2c35fb0bf4282011916fc362b08a Mon Sep 17 00:00:00 2001 From: Sven Peter Date: Mon, 24 Aug 2026 18:07:56 +0200 Subject: [PATCH 1295/2680] dt-bindings: thunderbolt: Add Apple USB4/Thunderbolt NHI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Native Host Interface (NHI) provides the TX and RX rings used by the software connection manager on Apple Silicon SoCs. The NHI is part of the ACIO host router and is exposed to the main SoC bus while the ACIO co-processor is running. T6000 and T6020 use one DART stream for the TX and RX control rings and one shared by all other rings, which are used for XDomain connections. This likely improves the isolation of those host-to-host connections. The M3 generation uses a slightly insane set of 24 IOMMU entries. These are probably one DART stream for each TX and RX ring. [ryan: split by topic: only the MAINTAINERS part; the rest is in 170-usb] [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] Signed-off-by: Sven Peter (cherry picked from commit 2e2b60f29328f3445128313fb8aff7e215f3d955) --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 359398921933e..1d710cc4f83bc 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2625,6 +2625,7 @@ F: Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml F: Documentation/devicetree/bindings/soc/apple/apple,sep.yaml F: Documentation/devicetree/bindings/spi/apple,spi.yaml F: Documentation/devicetree/bindings/spmi/apple,spmi.yaml +F: Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml F: Documentation/devicetree/bindings/usb/apple,dwc3.yaml F: Documentation/devicetree/bindings/usb/apple,sn201202x.yaml F: Documentation/devicetree/bindings/watchdog/apple,wdt.yaml From 74fed3619e8d27e1d496fa9b9161e2d97a4eee4f Mon Sep 17 00:00:00 2001 From: Sven Peter Date: Mon, 24 Aug 2026 18:07:56 +0200 Subject: [PATCH 1296/2680] dt-bindings: thunderbolt: Add Apple USB4/Thunderbolt host router MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each Type-C port with USB4/Thunderbolt support on Apple Silicon SoCs comes with a host router called ACIO. It contains a Cortex-M3 co-processor and the hardware blocks implementing the USB4 router. The co-processor exposes the NHI and its DART to the main SoC bus while it is running. ACIO can only be powered on after the Type-C PHY has been switched to USB4/Thunderbolt mode. The Type-C controller provides the cable information required to boot it through the thunderbolt-switch connection. [ryan: split by topic: only the MAINTAINERS part; the rest is in 170-usb] [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] Signed-off-by: Sven Peter (cherry picked from commit ecfa1787557c2840eb5e932180b9a21753c31f80) --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 1d710cc4f83bc..405f6d3f63882 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2625,6 +2625,7 @@ F: Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml F: Documentation/devicetree/bindings/soc/apple/apple,sep.yaml F: Documentation/devicetree/bindings/spi/apple,spi.yaml F: Documentation/devicetree/bindings/spmi/apple,spmi.yaml +F: Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml F: Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml F: Documentation/devicetree/bindings/usb/apple,dwc3.yaml F: Documentation/devicetree/bindings/usb/apple,sn201202x.yaml From 0017c8cfe95cb6e0a8b73179c0886bd44f5200c2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:50:04 +1000 Subject: [PATCH 1297/2680] MAINTAINERS: Add an entry for Apple T8140 and J700 support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a MAINTAINERS entry, with myself as maintainer, for the files this series adds for the A18 Pro (T8140) and the MacBook Neo (J700) that no existing entry covers: the T8140 device trees, the T8140 AOP, AOP audio, AOP ALS and ISP bindings, the apple,asc-mem reserved-memory binding, and the DockChannel UART binding and driver. The T8140 device trees also remain covered by ARM/APPLE MACHINE SUPPORT through arch/arm64/boot/dts/apple/, and changes to the shared Apple drivers (DART, ADMAC, mailbox, pinctrl and PMGR) stay under that entry. [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] (cherry picked from commit 2903144bad08cb641634d3fc5fce841af46f704d) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 405f6d3f63882..a190a1013bc9d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2025,6 +2025,19 @@ L: linux-hwmon@vger.kernel.org S: Odd fixes F: drivers/hwmon/applesmc.c +APPLE T8140 SOC AND J700 (MACBOOK NEO) SUPPORT +M: Ryan Murray +W: https://aurora-silicon.github.io +S: Maintained +F: Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml +F: Documentation/devicetree/bindings/media/apple,t8140-isp.yaml +F: Documentation/devicetree/bindings/reserved-memory/apple,asc-mem.yaml +F: Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml +F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml +F: arch/arm64/boot/dts/apple/t8140* +F: drivers/tty/serial/apple_dockchannel* + APPLETALK NETWORK LAYER L: netdev@vger.kernel.org S: Obsolete From 03e88f3662a504ab79aaa529060e5f5be5aef819 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:20:31 +1000 Subject: [PATCH 1298/2680] MAINTAINERS: Add the J700 idle and thermal files to the T8140 entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The T8140 WFI workaround and the J700 CPU thermal policy are Aurora Silicon code rather than Asahi Linux code, so list their files in the Apple T8140 / J700 entry instead of under ARM/APPLE MACHINE SUPPORT. Changes to the shared drivers (apple-soc-cpufreq, macsmc-hwmon) and to the DT bindings stay with their existing entries. [ryan: adapted to 7.2.10 — only the idle lines; the SMC CPU thermal policy files were removed again on aurora-wip (commit 52006c8e0bc5 ("thermal: Remove the Apple SMC CPU thermal policy")) and are not ported] (cherry picked from commit 73adb45383911623ffe356c8c530fc74cd97d278) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index a190a1013bc9d..9a073efa0f642 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2036,6 +2036,8 @@ F: Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: arch/arm64/boot/dts/apple/t8140* +F: arch/arm64/include/asm/apple-idle.h +F: arch/arm64/kernel/idle-apple.S F: drivers/tty/serial/apple_dockchannel* APPLETALK NETWORK LAYER From 0739e270e488775f86a158cb3b63906534101b50 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:47:04 +1000 Subject: [PATCH 1299/2680] MAINTAINERS: Add the AOP driver and sensor files to the T8140 entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Apple AOP driver, the AOP sensor drivers and their Rust support code now carry the T8140 and J700 support this series adds. List them under the APPLE T8140 SOC AND J700 (MACBOOK NEO) SUPPORT entry; they remain covered by ARM/APPLE MACHINE SUPPORT through drivers/soc/apple/ as well. [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] (cherry picked from commit 0d65c53f46d41515478297a7725ffa3ba6b86f34) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 9a073efa0f642..5437686ab772f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2038,7 +2038,11 @@ F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: arch/arm64/boot/dts/apple/t8140* F: arch/arm64/include/asm/apple-idle.h F: arch/arm64/kernel/idle-apple.S +F: drivers/iio/common/aop_sensors/ +F: drivers/soc/apple/aop.rs F: drivers/tty/serial/apple_dockchannel* +F: rust/kernel/iio/common/aop_sensors.rs +F: rust/kernel/soc/apple/aop.rs APPLETALK NETWORK LAYER L: netdev@vger.kernel.org From 4d4080c6a795099fcb4b76b7a732dd3cf902f351 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:41:43 +1000 Subject: [PATCH 1300/2680] MAINTAINERS: Add the T8140 AOP audio driver to the T8140/J700 entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The entry already lists the driver's binding; add the driver itself. [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] (cherry picked from commit 90a8bea518109ab51f4b6d08a2c94904ddb39bc2) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 5437686ab772f..0673ecee52bdf 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2043,6 +2043,7 @@ F: drivers/soc/apple/aop.rs F: drivers/tty/serial/apple_dockchannel* F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs +F: sound/soc/apple/t8140_aop_audio.rs APPLETALK NETWORK LAYER L: netdev@vger.kernel.org From 0724b063544e77bf124907e64803222c6c018ae4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:05:50 +1000 Subject: [PATCH 1301/2680] arm64: configs: Enable the J700 SMC thermal policy and T8140 AOP audio MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The J700 device tree now requests the SMC CPU thermal policy (apple,cpu-thermal-policy on the SMC hwmon node) and describes the T8140 AOP audio paths, but asahi.config still leaves both drivers off. A defconfig plus the Asahi fragment therefore builds a kernel in which apple-soc-cpufreq refuses the T8140 P-states above 2, because the thermal gate is absent, and the T8140 AOP audio module does not exist. Enable the SMC CPU thermal policy built in (it is a bool, only acts on apple,j700 machines, and its entry points are exported so cpufreq and macsmc-hwmon stay modules) and build the T8140 AOP audio driver as a module. With the base defconfig's CPU_FREQ_THERMAL, Kconfig resolves CONFIG_APPLE_SMC_CPU_THERMAL=y and CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=m while ARM_APPLE_SOC_CPUFREQ and SENSORS_MACSMC_HWMON remain modules. Compile-tested only (Kconfig resolution); not run on hardware. [ryan: adapted to 7.2.10 — only CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=m; the SMC CPU thermal policy was removed again on aurora-wip (commit 52006c8e0bc5 ("thermal: Remove the Apple SMC CPU thermal policy")) in favour of the power allocator thermal zones and is not ported] (cherry picked from commit 8b9f3674c9fcc5fe9b264e2a56d5e51927d38d43) Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/configs/asahi.config | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index a638b5a8ed3c3..0877463349d97 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -33,6 +33,7 @@ CONFIG_DRM_ADP=m CONFIG_DRM_APPLE=m CONFIG_DRM_APPLE_AUDIO=y CONFIG_SND_SOC_APPLE_AOP_AUDIO=m +CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=m CONFIG_SND_SOC_APPLE_MCA=m CONFIG_SND_SOC_APPLE_MACAUDIO=m CONFIG_SND_SOC_CS42L83=m From 2ea321780e25d018c2e0097f6672705674a2fa8a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 02:50:46 +1000 Subject: [PATCH 1302/2680] dt-bindings: soc: apple: Add the ASC4 RTKit helper The MTP coprocessor that serves the keyboard and trackpad of Apple laptops runs RTKit firmware that the operating system only has to boot and keep running; the DockChannel HID transport then talks to it over a FIFO pair. The T8112, T8122, T6020, T6030 and T6031 device trees have carried apple,rtk-helper-asc4 nodes for it without a binding, and the T8140 (J700, MacBook Neo) adds a sixth SoC. Describe the node: the ASC control window, the optional SRAM window from which the firmware may ask the host to map buffers, the mailbox, the IOMMU stream and the #helper-cells property that the existing device trees carry although nothing consumes it. The compatible list covers every in-tree user, including the T6031 node, whose second entry is the T8122 helper compatible, so the existing device trees validate unchanged. No bare apple,rtk-helper-asc4 form is allowed: no device tree uses one, and a node needs its SoC-specific compatibles. Compile-tested only: dt_binding_check on the schema, and dtbs_check on t8112-j413, t8112-j493, t6020-j414s, t6030-j514s, t6031-j514c, t8122-j613 and t8140-j700 with no new warnings. [ryan: split by topic: only the MAINTAINERS part; the rest is in 010-soc] (cherry picked from commit abb2dc8d327254d1c11f26dfab1a190bea5a4da5) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 0673ecee52bdf..eebb01d96697b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2033,6 +2033,7 @@ F: Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml F: Documentation/devicetree/bindings/media/apple,t8140-isp.yaml F: Documentation/devicetree/bindings/reserved-memory/apple,asc-mem.yaml F: Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: arch/arm64/boot/dts/apple/t8140* From cf0d0422b651017eebf95b914d63ce7395791098 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 02:50:46 +1000 Subject: [PATCH 1303/2680] dt-bindings: input: Add the Apple DockChannel HID transport Apple laptops from the M2 generation on connect the built-in keyboard and trackpad to the MTP coprocessor, which announces them as HID interfaces over a DockChannel FIFO pair. The apple,dockchannel-hid node has been in the T8112, T8122, T6020, T6030 and T6031 device trees without a binding. Describe it with the properties those trees use: the four FIFO windows, in either of the two orders the trees have; the tx/rx interrupts; the IOMMU stream; the FIFO size; the helper phandle; the AFE and STM reset lines; and the keyboard, multi-touch, stm, actuator and tp_accel children. Add what the T8140 (J700, MacBook Neo) needs on top. apple,no-stm says that the firmware has no STM interface to report the machine identifiers, so the transport must not wait for one. The apple,j700-multitouch compatible on the multi-touch child says that the trackpad sends a different report layout (a 32-byte header instead of 38 bytes, and an 8-byte trailer). apple,power-method selects the power sequencing protocol of the firmware: with method 1, used by every earlier laptop and therefore the default, the coprocessor sequences the trackpad and asks the host to pulse the reset line; with method 2 the host announces each transition, drives the AFE reset line itself and confirms the transition. The reset line is described with the level that asserts the reset as its active level, so that the driver can use logical GPIO values for both methods. Compile-tested only: dt_binding_check on the schema, and dtbs_check on t8112-j413, t8112-j493, t6020-j414s, t6030-j514s, t6031-j514c, t8122-j613 and t8140-j700 with no new warnings. [ryan: split by topic: only the MAINTAINERS part; the rest is in 090-input] (cherry picked from commit 929abaa6bbf9ccdaba0c52fb75bb7b439dbcc9f2) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index eebb01d96697b..fa17bae0adebe 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2030,6 +2030,7 @@ M: Ryan Murray W: https://aurora-silicon.github.io S: Maintained F: Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml +F: Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml F: Documentation/devicetree/bindings/media/apple,t8140-isp.yaml F: Documentation/devicetree/bindings/reserved-memory/apple,asc-mem.yaml F: Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml From 5928a298d032234687dc6115d61d8b76b4ec1ff2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 02:50:46 +1000 Subject: [PATCH 1304/2680] dt-bindings: soc: apple: Add the DockChannel FIFO bus DockChannel is a set of byte FIFOs between the application processor and a coprocessor. Each channel has a host side and a remote side with a configuration window and a data window each, and the channels of one block share an interrupt window whose interrupts signal FIFO threshold events. The apple,dockchannel node maps that interrupt window, translates the channel windows for its children and is their interrupt controller; the HID transport of the MTP coprocessor is its child. The node has been in the T8112, T8122, T6020, T6030 and T6031 device trees without a binding, and the T8140 (J700, MacBook Neo) adds a sixth SoC. Describe it, with every in-tree SoC compatible, so that the existing device trees validate unchanged. Compile-tested only: dt_binding_check on the schema, and dtbs_check on t8112-j413, t8112-j493, t6020-j414s, t6030-j514s, t6031-j514c, t8122-j613 and t8140-j700 with no new warnings. [ryan: split by topic: only the MAINTAINERS part; the rest is in 010-soc] (cherry picked from commit 99c72a91a608939067975ea2f3eeba0db521b32a) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index fa17bae0adebe..37ea23c76f0a8 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2034,6 +2034,7 @@ F: Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml F: Documentation/devicetree/bindings/media/apple,t8140-isp.yaml F: Documentation/devicetree/bindings/reserved-memory/apple,asc-mem.yaml F: Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml F: Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml From 5578852eebd957def448c2906f17fce51ff4e2af Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:09:27 +1000 Subject: [PATCH 1305/2680] HID: dockchannel: Support MTP power method 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MTP firmware of the M2 and M3 laptops powers the trackpad with what Apple calls power method 1: after the host has sent the trackpad firmware it sends a four-byte RESET_INTERFACE command for the OFF state and one for the ON state, and the coprocessor sequences the trackpad itself, asking the host to pulse the AFE reset line when it needs a pulse. The J700 (MacBook Neo) firmware uses power method 2 and accepts only the nine-byte form of the command: the method in byte 1, the interface in byte 2, the target state in byte 3 and a has-changed flag in byte 4. Each transition is announced with the flag clear; the host then drives the AFE reset line itself, asserting it for a power-off and holding it for 50 ms, or releasing it for a power-on; and the transition is confirmed with the flag set. After the firmware has been sent the trackpad is taken through OFF and then ON, as with method 1. This is the sequence recorded on the J700 during its bring-up. Select the method with the apple,power-method property of the multi-touch node, which defaults to 1, so that nothing changes on the existing machines: they keep their two four-byte commands and keep ignoring the commands' results. With method 2 the AFE reset line is requested in probe, as soon as the device tree asks for the method, so that a GPIO provider that has not probed yet defers the probe instead of failing the first open; it is requested as-is, so that its level does not change before the first transition has been announced. The line is driven with logical values, and the J700 device tree describes it with the level that asserts the reset as its active level. Should the firmware also announce the line as a GPIO request, the same descriptor serves the request. Move the command encoders into a header of their own and add KUnit tests for them, so that the wire format is pinned down without Apple hardware. The method 2 encoder and the first two test cases are based on work by Eryk Wieliczko for Aurora's tree (September 2026); Eryk Wieliczko's Signed-off-by is outstanding. Compile-tested only, including the KUnit test object; not run on hardware. [ryan: split by topic: only the MAINTAINERS part; the rest is in 090-input] [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] (cherry picked from commit 1f5c943c432756039c9e95c1e0afa421729eb6c2) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 37ea23c76f0a8..19bf47feb16fe 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2041,6 +2041,7 @@ F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: arch/arm64/boot/dts/apple/t8140* F: arch/arm64/include/asm/apple-idle.h F: arch/arm64/kernel/idle-apple.S +F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* F: drivers/iio/common/aop_sensors/ F: drivers/soc/apple/aop.rs F: drivers/tty/serial/apple_dockchannel* From 617f8a2b7783800f756eee660a8181a9fc0c76ce Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:10:31 +1000 Subject: [PATCH 1306/2680] Documentation: hid: Describe the Apple J700 C1FE multitouch report Write down the layout of the J700 trackpad's report 0x75 as hid-magicmouse decodes it: the 32-byte header with its length, count and button fields, the 30-byte contact records, of which only the positions are decoded, and the 8-byte trailer, together with the checks a report has to pass and what happens to one that fails them. This is the format the C1FE controller's firmware was captured sending on 2026-09-07; the earlier Apple Silicon laptops use a different header, which is described only in the driver. Based on LiamRay11's description of the C1FE report for Aurora's tree; LiamRay11's Signed-off-by is outstanding. [ryan: split by topic: only the MAINTAINERS part; the rest is in 090-input] (cherry picked from commit 98f1a1d3961f5ef859b83eb6fe75bf0192e4cd6c) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 19bf47feb16fe..787ea799ca853 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2038,6 +2038,7 @@ F: Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml F: Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml +F: Documentation/hid/apple-j700-multitouch.rst F: arch/arm64/boot/dts/apple/t8140* F: arch/arm64/include/asm/apple-idle.h F: arch/arm64/kernel/idle-apple.S From 596e26ffd09c90ac8e6a238f61a36aaf972a2cbd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 03:03:33 +1000 Subject: [PATCH 1307/2680] dt-bindings: usb: Add the VIA Labs VL122 USB 2.0 hub Apple's J700 (MacBook Neo) has a VIA Labs VL122 two-port USB 2.0 hub permanently attached to the USB 2.0 root port of its Type-C USB controller. The hub's two downstream ports go to the high-speed pairs of the two Type-C connectors; only the rear connector also has SuperSpeed lanes, which bypass the hub. On Linux the hub enumerates as USB ID 2109:2122. Describe it with the generic USB hub binding, so that a board can wire the downstream ports to its connectors through the graph. The Type-C port controllers then find no USB role switch on their high-speed path, which is correct for a fixed hub: the controller behind it is a host regardless of what happens at the connectors. Add the schema to the T8140/J700 MAINTAINERS entry. [ryan: split by topic: only the MAINTAINERS part; the rest is in 170-usb] (cherry picked from commit 56e55f303f4d91ba9344b522fba9e70599316bff) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 787ea799ca853..a060801d3ae8c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2038,6 +2038,7 @@ F: Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml F: Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml +F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml F: Documentation/hid/apple-j700-multitouch.rst F: arch/arm64/boot/dts/apple/t8140* F: arch/arm64/include/asm/apple-idle.h From 3a2b1f3b2d2b3ee2a4a12b1b656f0a53861a83a2 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 10:05:51 -0400 Subject: [PATCH 1308/2680] arm64: apple: add initial J813 M5 RAM boot support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the measured J813 CPU affinities, AICv3 windows, DockChannel console and loader-populated framebuffer and memory. Keep the Samsung UART disabled and omit DMA peripherals. Supply a minimal kernel configuration and document the required m1n1/U-Boot handoff and repeatable boot qualification. Adapt the minimal device tree and configuration from the local J813 bring-up at c8f85bcfb03925179bfa6c19c4d28c44efae39ac. Use T8142 CPU compatibles instead of firmware-inherited M3 core names. This initial scope is a single-CPU RAM boot, not complete M5 platform support. [ryan: split by topic: only the Documentation/arch/arm64/apple-j813.rst, Documentation/arch/arm64/index.rst, arch/arm64/configs/j813.config part; the rest is in 000-devicetree (DTS), 010-soc (bindings)] [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] Signed-off-by: Acelogic (cherry picked from commit 80661974aa5b2d2de903a023bf4ca38271261466) --- Documentation/arch/arm64/apple-j813.rst | 94 +++++++++++++++++++++++++ Documentation/arch/arm64/index.rst | 1 + arch/arm64/configs/j813.config | 46 ++++++++++++ 3 files changed, 141 insertions(+) create mode 100644 Documentation/arch/arm64/apple-j813.rst create mode 100644 arch/arm64/configs/j813.config diff --git a/Documentation/arch/arm64/apple-j813.rst b/Documentation/arch/arm64/apple-j813.rst new file mode 100644 index 0000000000000..53260365d580f --- /dev/null +++ b/Documentation/arch/arm64/apple-j813.rst @@ -0,0 +1,94 @@ +.. SPDX-License-Identifier: GPL-2.0 + +===================================== +Apple J813 initial native Linux boot +===================================== + +J813 is the 13-inch M5 MacBook Air (T8142, Mac17,3). This configuration + +affinities are 0x10100 through 0x10103, rather than the 0x100 through 0x103 +values in the Apple Device Tree. + +The CPU compatible strings identify T8142 explicitly. The firmware's reused +``sawtooth`` and ``everest`` names do not identify the M5 microarchitecture. + +Bootloader requirements +======================= + +Use the J813-capable Aurora m1n1 and U-Boot bring-up stack. A stock M1/M2 +loader is not sufficient. Before entering Linux, the loader must: + +* Supply usable memory and reserved regions. T8142 DRAM starts at + 0x10000000000; the DTS memory size is only a loader placeholder. +* Fill CPU release addresses and remove CPUs which have not been started. +* Fill the firmware framebuffer's address, size, geometry and format before + enabling it. +* Leave the console, interrupt controller and display powered. This minimal + DT does not describe their power domains. +* Give U-Boot a T8142 memory map covering its MMIO and high DRAM addresses. + +The AIC driver selects the T8142-specific guest timer masking path from +``apple,t8142-aic3``. Writes to the legacy VM_TMR_FIQ_ENA_EL2 register trap +on this SoC. The driver uses the architectural EL02 timer control masks +instead. Older AIC variants retain their existing path. + +Build +===== + +On an arm64 Linux build host with the usual kernel build dependencies:: + + out="$PWD/../j813-build" + make ARCH=arm64 O="$out" \ + KCONFIG_ALLCONFIG=arch/arm64/configs/j813.config allnoconfig + make ARCH=arm64 O="$out" -j"$(nproc)" W=1 Image apple/t8142-j813.dtb + +cross-compiling with GCC from another architecture. + +With dtschema installed, check the affected bindings and DTB:: + + make ARCH=arm64 O="$out" CHECK_DTBS=y \ + apple/t8142-j813.dtb + +Boot and qualification +====================== + +Use a RAM-only initramfs with a serial shell and BusyBox (including +``mount``, ``sleep``, ``cat``, ``uname`` and ``sha256sum``). Its init must +mount proc, sysfs and devtmpfs, keep PID 1 alive and make no disk mounts. +Record the source commit, any uncommitted patch, final kernel config and +SHA-256 hashes of Image, DTB, initramfs, m1n1 and U-Boot with the console +capture. Verify uploaded payload bytes before handing over to U-Boot. + +The initial command line is:: + + earlycon=dockchannel,0x38812c000 console=tty0 console=ttyDC0 keep_bootcon maxcpus=1 rdinit=/init panic=0 + +Use m1n1 to load Image, the initramfs, the DTB and U-Boot into RAM and +prepare the FDT. Pass that prepared FDT through U-Boot's ``booti`` command. +Do not install this experimental payload into the resident boot image. + +At the shell, capture the following into the host's serial log:: + + echo J813_BOOT_PROOF_BEGIN + uname -a + tr '\000' '\n' < /proc/device-tree/compatible + cat /sys/devices/system/cpu/online + cat /proc/uptime + cat /proc/interrupts + sleep 2 + cat /proc/uptime + cat /proc/interrupts + head -5 /proc/meminfo + cat /proc/mounts + cat /proc/fb + dmesg + echo J813_BOOT_PROOF_END + +Verify T8142/J813 identity, one online CPU, advancing uptime and timer +interrupt counts, an interactive shell, a registered framebuffer and only +RAM-backed filesystems. Repeat after an idle interval and after a fresh +RAM boot. Preserve the complete console transcript and a hash manifest; +an Image build or successful loader exit alone is not a boot result. +Review captures for device identifiers and boot entropy before publishing. + +records its measured resources; the qualified console path is DockChannel. diff --git a/Documentation/arch/arm64/index.rst b/Documentation/arch/arm64/index.rst index e5d0d0c46bf3b..23afe611f3039 100644 --- a/Documentation/arch/arm64/index.rst +++ b/Documentation/arch/arm64/index.rst @@ -9,6 +9,7 @@ ARM64 Architecture acpi_object_usage amu + apple-j813 arm-acpi arm-cca asymmetric-32bit diff --git a/arch/arm64/configs/j813.config b/arch/arm64/configs/j813.config new file mode 100644 index 0000000000000..d8143807c2e3e --- /dev/null +++ b/arch/arm64/configs/j813.config @@ -0,0 +1,46 @@ +# Minimal RAM-only J813 bring-up; use with KCONFIG_ALLCONFIG and allnoconfig. +CONFIG_LOCALVERSION="-aurora-j813-wip" +CONFIG_LOCALVERSION_AUTO=n +CONFIG_ARCH_APPLE=y +CONFIG_ARM64_16K_PAGES=y +CONFIG_ARM64_VA_BITS_47=y +CONFIG_SMP=y +CONFIG_NR_CPUS=16 +CONFIG_ARM64_VHE=y +CONFIG_ARM64_SVE=y +CONFIG_ARM64_PAN=y +CONFIG_ARM64_LSE_ATOMICS=y +CONFIG_EXPERT=y +CONFIG_PRINTK=y +CONFIG_BUG=y +CONFIG_MULTIUSER=y +CONFIG_FUTEX=y +CONFIG_POSIX_TIMERS=y +CONFIG_KALLSYMS=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_RD_GZIP=y +CONFIG_BINFMT_ELF=y +CONFIG_BINFMT_SCRIPT=y +CONFIG_TTY=y +CONFIG_VT=y +CONFIG_VT_CONSOLE=y +CONFIG_UNIX98_PTYS=y +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +CONFIG_PROC_FS=y +CONFIG_SYSFS=y +CONFIG_TMPFS=y +CONFIG_SHMEM=y +CONFIG_SERIAL_SAMSUNG=y +CONFIG_SERIAL_SAMSUNG_CONSOLE=y +CONFIG_SERIAL_APPLE_DOCKCHANNEL=y +CONFIG_SERIAL_APPLE_DOCKCHANNEL_EARLYCON=y +CONFIG_SERIAL_EARLYCON=y +CONFIG_FB=y +CONFIG_FB_DEVICE=y +CONFIG_FB_SIMPLE=y +CONFIG_FRAMEBUFFER_CONSOLE=y +# CONFIG_FRAMEBUFFER_CONSOLE_DEFERRED_TAKEOVER is not set +CONFIG_LOG_BUF_SHIFT=20 +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y From 59f0e1c7f2dcfbf3a5a99034860b57dd0938d8d6 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 10:42:22 -0400 Subject: [PATCH 1309/2680] arm64: dts: apple: use the native FDT handoff on J813 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: split by topic: only the Documentation/arch/arm64/apple-j813.rst part; the rest is in 000-devicetree and 010-soc] [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] (cherry picked from commit 6339da35d6f66d5608149ad60f610ce700095a08) --- Documentation/arch/arm64/apple-j813.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/arch/arm64/apple-j813.rst b/Documentation/arch/arm64/apple-j813.rst index 53260365d580f..706450e61d044 100644 --- a/Documentation/arch/arm64/apple-j813.rst +++ b/Documentation/arch/arm64/apple-j813.rst @@ -23,6 +23,7 @@ loader is not sufficient. Before entering Linux, the loader must: * Fill CPU release addresses and remove CPUs which have not been started. * Fill the firmware framebuffer's address, size, geometry and format before enabling it. +* Prepare DMA protection only for devices enabled in the supplied FDT. The * Leave the console, interrupt controller and display powered. This minimal DT does not describe their power domains. * Give U-Boot a T8142 memory map covering its MMIO and high DRAM addresses. From fa25829d298e0c07b951baeb85b52f2f0000fbed Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 11:57:48 -0400 Subject: [PATCH 1310/2680] arm64: apple: preserve registers across WFI on T8142 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: split by topic: only the Documentation/arch/arm64/apple-j813.rst part; the rest is in 220-userpages] [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] (cherry picked from commit f1d29b8afd4e364f730f666d7c247276c8ba50ca) --- Documentation/arch/arm64/apple-j813.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Documentation/arch/arm64/apple-j813.rst b/Documentation/arch/arm64/apple-j813.rst index 706450e61d044..af322809458a6 100644 --- a/Documentation/arch/arm64/apple-j813.rst +++ b/Documentation/arch/arm64/apple-j813.rst @@ -6,12 +6,36 @@ Apple J813 initial native Linux boot J813 is the 13-inch M5 MacBook Air (T8142, Mac17,3). This configuration +The device tree describes all ten CPU affinities. For the initial one-CPU +baseline, keep secondary cores stopped in m1n1 and use ``maxcpus=1``. +The loader removes unavailable CPUs and fills the release addresses of the +remaining nodes. The P-core affinities are 0x10100 through 0x10103, rather than the 0x100 through 0x103 values in the Apple Device Tree. The CPU compatible strings identify T8142 explicitly. The firmware's reused ``sawtooth`` and ``everest`` names do not identify the M5 microarchitecture. +SMP bring-up +============ + +M5 secondary CPUs can return from WFI with cleared general-purpose registers. +Both M5 MIDRs (parts 0x062 and 0x063) select the Apple register-preserving idle +path. Without it, all ten CPUs can start and then panic in their idle tasks. + +SMP also requires the M5-capable m1n1 secondary-start path: dense PMGR CPU +masks, correct reset vectors and per-core stacks. Verify all ten cores there +before preparing the FDT, and use ``maxcpus=10`` for the Linux test. The +existing J813 DT and configuration support this handoff. + +On J813, this path reached the RAM shell with CPUs 0-9 online. A five-second +sleep advanced every CPU's architectural timer count by 1,251 interrupts. +Ten concurrent, individually pinned jobs each hashed 16 MiB of zeros to +``080acf35a507ac9849cfcba47dc2ad83e01b75663a516279c8b9d243b719643e``. +Verify affinity through ``Cpus_allowed_list`` and retain the console output, +kernel configuration hash and payload hashes when repeating the test. +Hotplug, deep sleep and KVM remain unqualified. + Bootloader requirements ======================= From 3adf9fbe9796cb6ace9b90f171462850242a7147 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 12:37:37 -0400 Subject: [PATCH 1311/2680] arm64: dts: apple: enable bounded J813 CPU frequency control MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe both shared cluster policies and enable the userspace governor in the J813 configuration. Expose only the states up to the observed firmware handoff: E state 2 at 972 MHz and P state 10 at 3516 MHz. State 1 is a 300 MHz floor before the firmware SRAM table entries. Dependent decrement-loop timing on J813 confirmed every exposed state's frequency. Document repeatable state sweeps and qualification limits. [ryan: split by topic: only the Documentation/arch/arm64/apple-j813.rst, arch/arm64/configs/j813.config part; the rest is in 000-devicetree and 060-power-thermal] [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] Signed-off-by: Acelogic (cherry picked from commit 0abc47f7ebe2fcedaaf9c2eb06aaf283fdb4362c) --- Documentation/arch/arm64/apple-j813.rst | 48 +++++++++++++++++++++++++ arch/arm64/configs/j813.config | 5 +++ 2 files changed, 53 insertions(+) diff --git a/Documentation/arch/arm64/apple-j813.rst b/Documentation/arch/arm64/apple-j813.rst index af322809458a6..425db75c78c73 100644 --- a/Documentation/arch/arm64/apple-j813.rst +++ b/Documentation/arch/arm64/apple-j813.rst @@ -36,6 +36,54 @@ Verify affinity through ``Cpus_allowed_list`` and retain the console output, kernel configuration hash and payload hashes when repeating the test. Hotplug, deep sleep and KVM remain unqualified. +CPU frequency bring-up +====================== + +The T8142 cluster controllers use five-bit P-state requests with busy and +command-state readback. The driver preserves firmware voltage, PLL and +CLPC/PMP configuration, and stops issuing requests after a timeout or an +unexpected command state. Unlike T8140, this initial J813 path does not +register a Linux SMC thermal policy. The DT exposes only states up to the +measured firmware handoff values; higher states remain unqualified. + +State 1 is 300 MHz on both clusters. The firmware SRAM voltage tables begin +at state 2: the E cluster exposes 300/972 MHz, and the P cluster exposes +300/1308/1608/1932/2220/2508/2820/3072/3300/3516 MHz. A dependent decrement +loop timed against CNTPCT_EL0 in m1n1 confirmed this mapping on J813. +``scaling_cur_freq`` reports the requested state, not an independent clock +measurement or a guarantee against firmware throttling. + +The configuration enables the userspace governor. On a fresh ten-core boot, +capture a sweep of both policies to the host serial log:: + + set -e + for p in /sys/devices/system/cpu/cpufreq/policy*; do + cat "$p/affected_cpus" + echo userspace > "$p/scaling_governor" + for round in 1 2; do + for f in $(cat "$p/scaling_available_frequencies"); do + echo "$f" > "$p/scaling_setspeed" + test "$(cat "$p/scaling_cur_freq")" = "$f" + echo "$p $f" + sleep 0.02 + done + done + cat "$p/cpuinfo_max_freq" > "$p/scaling_setspeed" + cat "$p/stats/total_trans" + done + dmesg + +Retain the transcript and payload hashes. Qualification also compared three +pairs of pinned, checksum-verified integer workloads at each cluster's +minimum and maximum states and checked every CPU's timer and pinned SHA-256 +job afterwards. Short frequency sweeps do not qualify sustained thermal +behavior, automatic governors, suspend or states above the handoff limit. + +T8142 can also lose registers on WFIT in delay loops. Frequency transition +polling exposed this as a cleared pointer in the cpufreq driver. T8142 uses +the existing counter delay loop, avoiding low-power waits with live kernel +register state. + Bootloader requirements ======================= diff --git a/arch/arm64/configs/j813.config b/arch/arm64/configs/j813.config index d8143807c2e3e..f944f153962ab 100644 --- a/arch/arm64/configs/j813.config +++ b/arch/arm64/configs/j813.config @@ -6,6 +6,11 @@ CONFIG_ARM64_16K_PAGES=y CONFIG_ARM64_VA_BITS_47=y CONFIG_SMP=y CONFIG_NR_CPUS=16 +CONFIG_CPU_FREQ=y +CONFIG_CPU_FREQ_STAT=y +CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE=y +CONFIG_CPU_FREQ_GOV_USERSPACE=y +CONFIG_ARM_APPLE_SOC_CPUFREQ=y CONFIG_ARM64_VHE=y CONFIG_ARM64_SVE=y CONFIG_ARM64_PAN=y From ade2ff22bfa1c7c4cb53b9ba271ab21c622be0a9 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 18:48:20 -0400 Subject: [PATCH 1312/2680] arm64: apple: enable J813 temperatures, input and keyboard backlight Describe the MTP, SMC and PWM resources needed by the existing generic drivers. Request trackpad firmware externally, obtain keyboard layout through the loader, and use runtime sensor discovery with a linear LED brightness scale. [ryan: split by topic: only the arch/arm64/configs/j813.config part; the rest is in 000-devicetree, 060-power-thermal and 090-input] Signed-off-by: Acelogic (cherry picked from commit 14334fd7675b9614deb45828162df9523bbad77b) --- arch/arm64/configs/j813.config | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/arch/arm64/configs/j813.config b/arch/arm64/configs/j813.config index f944f153962ab..13b8116af0710 100644 --- a/arch/arm64/configs/j813.config +++ b/arch/arm64/configs/j813.config @@ -49,3 +49,29 @@ CONFIG_FRAMEBUFFER_CONSOLE=y CONFIG_LOG_BUF_SHIFT=20 CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y +CONFIG_PM=y +CONFIG_MAILBOX=y +CONFIG_APPLE_MAILBOX=y +CONFIG_APPLE_RTKIT=y +CONFIG_APPLE_RTKIT_HELPER=y +CONFIG_APPLE_DOCKCHANNEL=y +CONFIG_IOMMU_SUPPORT=y +CONFIG_APPLE_DART=y +CONFIG_MFD_MACSMC=y +CONFIG_HWMON=y +CONFIG_SENSORS_MACSMC_HWMON=y +CONFIG_INPUT=y +CONFIG_INPUT_EVDEV=y +CONFIG_HID_SUPPORT=y +CONFIG_HID=y +CONFIG_HID_GENERIC=y +CONFIG_HID_DOCKCHANNEL=y +CONFIG_HID_APPLE=y +CONFIG_HID_MAGICMOUSE=y +CONFIG_GPIOLIB=y +CONFIG_NEW_LEDS=y +CONFIG_LEDS_CLASS=y +CONFIG_LEDS_PWM=y +CONFIG_PWM=y +CONFIG_PWM_APPLE=y +CONFIG_FW_LOADER=y From 7033d1390682f99faf803a977dc6446cdbd4aa74 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 18:48:20 -0400 Subject: [PATCH 1313/2680] selftests: apple: document and exercise J813 peripherals on hardware MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document the external firmware contract and qualification limits. Add an explicitly manual RAM-boot check that records temperatures, restores backlight brightness and counts physical input events without retaining key codes or pointer coordinates. [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] Signed-off-by: Acelogic (cherry picked from commit 4dec9439047493c1ae3e17e07e5f14d93f5e646e) --- Documentation/arch/arm64/apple-j813.rst | 73 ++++++++++++ tools/testing/selftests/Makefile | 1 + .../testing/selftests/drivers/apple/Makefile | 5 + .../drivers/apple/j813-peripherals.sh | 104 ++++++++++++++++++ 4 files changed, 183 insertions(+) create mode 100644 tools/testing/selftests/drivers/apple/Makefile create mode 100755 tools/testing/selftests/drivers/apple/j813-peripherals.sh diff --git a/Documentation/arch/arm64/apple-j813.rst b/Documentation/arch/arm64/apple-j813.rst index 425db75c78c73..8ba57a51e3320 100644 --- a/Documentation/arch/arm64/apple-j813.rst +++ b/Documentation/arch/arm64/apple-j813.rst @@ -5,6 +5,9 @@ Apple J813 initial native Linux boot ===================================== J813 is the 13-inch M5 MacBook Air (T8142, Mac17,3). This configuration +provides a RAM-only boot path with AICv3, architectural timers, DockChannel +console and the framebuffer left by the boot firmware. The J813 tree also +describes SMC temperature monitoring, MTP input and the keyboard backlight. The device tree describes all ten CPU affinities. For the initial one-CPU baseline, keep secondary cores stopped in m1n1 and use ``maxcpus=1``. @@ -96,6 +99,8 @@ loader is not sufficient. Before entering Linux, the loader must: * Fill the firmware framebuffer's address, size, geometry and format before enabling it. * Prepare DMA protection only for devices enabled in the supplied FDT. The + ``mtp`` alias enables the MTP DAPF handoff. Leave the protection state of + unrelated AOP, PMP and ISP firmware intact. * Leave the console, interrupt controller and display powered. This minimal DT does not describe their power domains. * Give U-Boot a T8142 memory map covering its MMIO and high DRAM addresses. @@ -115,11 +120,14 @@ On an arm64 Linux build host with the usual kernel build dependencies:: KCONFIG_ALLCONFIG=arch/arm64/configs/j813.config allnoconfig make ARCH=arm64 O="$out" -j"$(nproc)" W=1 Image apple/t8142-j813.dtb +The fragment does not enable KVM or storage drivers. Add +``CROSS_COMPILE=aarch64-linux-gnu-`` to both commands when cross-compiling with GCC from another architecture. With dtschema installed, check the affected bindings and DTB:: make ARCH=arm64 O="$out" CHECK_DTBS=y \ + DT_SCHEMA_FILES=arm/apple.yaml:arm/cpus.yaml:cpufreq/apple,cluster-cpufreq.yaml:opp/opp-v2-base.yaml:interrupt-controller/apple,aic2.yaml:serial/apple,dockchannel-uart.yaml:serial/samsung_uart.yaml:display/simple-framebuffer.yaml:apple,pmgr.yaml:apple,smc-hwmon.yaml:apple,dockchannel-hid.yaml:apple,dart.yaml:apple,mailbox.yaml:apple,smc.yaml:apple,pmgr-pwrstate.yaml:apple,s5l-fpwm.yaml:apple,dockchannel.yaml:apple,rtk-helper-asc4.yaml \ apple/t8142-j813.dtb Boot and qualification @@ -164,4 +172,69 @@ RAM boot. Preserve the complete console transcript and a hash manifest; an Image build or successful loader exit alone is not a boot result. Review captures for device identifiers and boot entropy before publishing. +These boot tests do not establish CPU hotplug, KVM guest timers, +storage, networking or audio support. Input and monitoring require their +separate checks below. The disabled Samsung UART node records its measured resources; the qualified console path is DockChannel. + +Temperature monitoring and built-in input +======================================== + +Temperature monitoring enumerates the running SMC's readable, non-function +``T`` keys with four-byte floating-point values. It exposes their identifiers +through ``/sys/class/hwmon/*/temp*_label`` and millidegrees Celsius through +``temp*_input``. It does not embed a sensor inventory, calibration, inferred +sensor locations, trip temperatures or a new CPU cooling policy. Channel +numbers can change with firmware; identify channels by label. These keys can +include virtual or inactive channels; their number is not a count of physical +sensors, and a readable value does not establish a sensor's location. Read failures, +NaNs and infinities remain errors, rather than fabricated temperatures. +The existing J700 thermal provider and bounded J813 frequency states are +unchanged. Temperature reporting alone does not qualify sustained load. + +MTP provides keyboard and trackpad HID descriptors and identifiers at runtime. +The ``keyboard`` alias lets m1n1 fill ``hid-country-code`` and +``apple,keyboard-layout-id`` from this machine's own boot environment. The +source tree does not assume a US keyboard or contain another machine's layout. +The J813 trackpad uses power method 2 with firmware-owned reset; the host +does not operate an AFE GPIO. Other machines retain their existing sequence. + +The trackpad requires an externally supplied +``/lib/firmware/apple/tpmtfw-j813.bin``. The existing DockChannel firmware +container has a 20-byte little-endian header: magic ``0x46444948`` (``HIDF``), +version 1, header length, data length and interface-byte offset. See +``struct fw_header`` and ``dchid_get_firmware()`` for validation. An interface +offset of zero means no patching. The payload must +be prepared from firmware and any required calibration belonging to the +user's own machine. A filename is a lookup contract, not a firmware payload. +No Apple firmware, captured command buffers or calibration bytes are bundled. +Future IPSW/macOS extraction tools can provide this file without changing the +driver. A missing or invalid file fails trackpad startup; it does not prevent +the keyboard or temperature monitor from registering. + +The loader must preserve the iBoot-loaded MTP image and SRAM, prepare its DMA +protection, and leave MTP ready for Linux's RTKit startup. A prior-stage MTP +client must complete a compatible shutdown before Linux starts; a partially + +The backlight uses the existing PWM and LED drivers. Its boot brightness is +retained when the initial PWM period is valid, otherwise it starts off. +Userspace controls ``/sys/class/leds/kbd_backlight/brightness`` +on a linear 0--255 scale. No captured brightness curve or factory calibration +is included. + +Physical verification +--------------------- + +Place the user's touch firmware in a private RAM initramfs, then run:: + + sh tools/testing/selftests/drivers/apple/j813-peripherals.sh 30 + +The check reads temperatures twice, exercises and restores backlight +brightness, and counts keyboard and trackpad events during a bounded physical +interaction window. It records no key codes or pointer coordinates. Press +and release keys and move/click the built-in trackpad during that window. +Confirm visible brightness changes separately; sysfs readback is not optical +proof. Preserve stdout, the source revision and SHA-256 hashes of the exact +Image, DTB, configuration, initramfs and bootloaders as the repeatable result. +Keep firmware and raw machine data private. Without physical events the input +check fails rather than crediting device registration as working input. diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index b622052ec3e92..a16dbc8bab8a4 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -17,6 +17,7 @@ TARGETS += damon TARGETS += devices/error_logs TARGETS += devices/probe TARGETS += dmabuf-heaps +TARGETS += drivers/apple TARGETS += drivers/dma-buf TARGETS += drivers/ntsync TARGETS += drivers/s390x/uvdevice diff --git a/tools/testing/selftests/drivers/apple/Makefile b/tools/testing/selftests/drivers/apple/Makefile new file mode 100644 index 0000000000000..76bca086dde7a --- /dev/null +++ b/tools/testing/selftests/drivers/apple/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 +# Manual physical test: requires user interaction and changes LED brightness. +TEST_FILES := j813-peripherals.sh + +include ../../lib.mk diff --git a/tools/testing/selftests/drivers/apple/j813-peripherals.sh b/tools/testing/selftests/drivers/apple/j813-peripherals.sh new file mode 100755 index 0000000000000..ecad597fe9113 --- /dev/null +++ b/tools/testing/selftests/drivers/apple/j813-peripherals.sh @@ -0,0 +1,104 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Physical J813 check. Save stdout and the payload SHA-256 manifest together. +# Usage: sh j813-peripherals.sh [event-capture-seconds] +set -eu + +duration=${1:-30} +case "$duration" in *[!0-9]*|'') exit 2;; esac +[ "$duration" -ge 1 ] && [ "$duration" -le 600 ] || exit 2 +[ -r /proc/device-tree/compatible ] || exit 4 +tr '\000' '\n' < /proc/device-tree/compatible | grep -qx apple,j813 || exit 4 +[ "$(id -u)" -eq 0 ] || { echo 'SKIP: run as root'; exit 4; } + +work=$(mktemp -d) +led=/sys/class/leds/kbd_backlight +saved_brightness= +cleanup() +{ + if [ -n "$saved_brightness" ]; then + echo "$saved_brightness" > "$led/brightness" + fi + rm -rf "$work" +} +trap cleanup EXIT +trap 'exit 1' INT TERM + +echo J813_PERIPHERALS_BEGIN +uname -r +zcat /proc/config.gz | sha256sum +cat /sys/devices/system/cpu/online + +hwmon= +for h in /sys/class/hwmon/hwmon*; do + [ -r "$h/name" ] || continue + [ "$(cat "$h/name")" = macsmc_hwmon ] || continue + hwmon=$h +done +[ -n "$hwmon" ] || { echo 'FAIL: no SMC hwmon'; exit 1; } +for sample in 1 2; do + count=0 + for input in "$hwmon"/temp*_input; do + [ -r "$input" ] || continue + value=$(cat "$input") + label=$(cat "${input%_input}_label") + [ "$value" -ge -40000 ] && [ "$value" -le 125000 ] || exit 1 + printf 'temperature sample=%s key=%s millidegrees=%s\n' "$sample" "$label" "$value" + count=$((count + 1)) + done + [ "$count" -gt 0 ] || { echo 'FAIL: no temperature channels'; exit 1; } + sleep 1 +done +echo "PASS: $count readable SMC temperature channels" + +[ -r "$led/brightness" ] || { echo 'FAIL: no keyboard backlight'; exit 1; } +saved_brightness=$(cat "$led/brightness") +maximum=$(cat "$led/max_brightness") +for level in 0 $((maximum / 2)) "$maximum"; do + echo "$level" > "$led/brightness" + [ "$(cat "$led/brightness")" = "$level" ] || exit 1 + printf 'backlight requested=%s readback=%s\n' "$level" "$(cat "$led/brightness")" + sleep 5 +done +echo "$saved_brightness" > "$led/brightness" +echo 'PASS: backlight control and restore (visual confirmation is separate)' + +keyboard= +trackpad= +for e in /sys/class/input/event*; do + [ -r "$e/device/name" ] || continue + case "$(cat "$e/device/name")" in + 'Apple MTP keyboard') keyboard=/dev/input/${e##*/} ;; + 'Apple MTP multi-touch') trackpad=/dev/input/${e##*/} ;; + esac +done +[ -n "$keyboard" ] && [ -n "$trackpad" ] || { + echo 'FAIL: keyboard or trackpad missing'; exit 1; +} +echo "Move/click the built-in trackpad and press/release keys for $duration seconds." +# Count bytes only. Do not retain key codes, pointer coordinates, or raw events. +timeout "$duration" cat "$keyboard" | wc -c > "$work/keyboard" & +kp=$! +timeout "$duration" cat "$trackpad" | wc -c > "$work/trackpad" & +tp=$! +wait "$kp" +wait "$tp" +failed=0 +for name in keyboard trackpad; do + bytes=$(cat "$work/$name") + [ "$bytes" -gt 0 ] && [ $((bytes % 24)) -eq 0 ] || { + echo "FAIL: $name produced no complete arm64 input events" + failed=1 + continue + } + printf 'PASS: %s events=%s\n' "$name" "$((bytes / 24))" +done +dmesg > "$work/dmesg" +fault_pattern='Kernel panic|Oops:|(^|[[:space:]])SError([[:space:]:]|$)' +fault_pattern="$fault_pattern|(^|[[:space:]])BUG:|apple-dart .*:.*(fault|error)" +if grep -E "$fault_pattern" "$work/dmesg"; then + echo 'FAIL: kernel fault detected' + exit 1 +fi +[ "$failed" -eq 0 ] || exit 1 +echo J813_PERIPHERALS_PASS From 714e9126cfc22df8c30d21c28a55c3a4c843781f Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 21:03:28 -0400 Subject: [PATCH 1314/2680] usb: add verified M5 J813 USB3 host support [ryan: split by topic: only the arch/arm64/configs/j813.config part; the rest is in 170-usb] (cherry picked from commit 4426aa459a636b7da282eaabb29144e537a7edf1) --- arch/arm64/configs/j813.config | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/arch/arm64/configs/j813.config b/arch/arm64/configs/j813.config index 13b8116af0710..57fb42b0295c5 100644 --- a/arch/arm64/configs/j813.config +++ b/arch/arm64/configs/j813.config @@ -75,3 +75,29 @@ CONFIG_LEDS_PWM=y CONFIG_PWM=y CONFIG_PWM_APPLE=y CONFIG_FW_LOADER=y +CONFIG_PM_GENERIC_DOMAINS=y +CONFIG_APPLE_PMGR_PWRSTATE=y +CONFIG_SPMI=y +CONFIG_SPMI_APPLE=y +CONFIG_USB_SUPPORT=y +CONFIG_USB=y +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_XHCI_PLATFORM=y +CONFIG_USB_DWC3=y +CONFIG_USB_DWC3_HOST=y +CONFIG_USB_DWC3_APPLE=y +CONFIG_USB_HID=y +CONFIG_BLOCK=y +CONFIG_SCSI=y +CONFIG_BLK_DEV_SD=y +CONFIG_USB_STORAGE=y +CONFIG_USB_UAS=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_EFI_PARTITION=y +CONFIG_MSDOS_PARTITION=y +CONFIG_EXFAT_FS=y +CONFIG_NLS_UTF8=y +CONFIG_TYPEC=y +CONFIG_TYPEC_SN201202X=y +CONFIG_PHY_APPLE_ATC=y +CONFIG_DEBUG_FS=y From 3b0efa346a7ca3a50923b2b1446731cb0dc52e0f Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 21:09:11 -0400 Subject: [PATCH 1315/2680] thunderbolt: prepare M5 Type 7 controller support Handle the T8142 reset request layout, firmware ready state, and receive-ring PDF masks without the older external PDF aperture. Require loader tuning and matching ACIO/NHI generations. Add an opt-in build fragment; controller DT enablement and firmware startup remain separate qualification work. [ryan: split by topic: only the arch/arm64/configs/j813-usb4.config part; the rest is in 170-usb] (cherry picked from commit 4532abf64a799404c7ba1cd22bf92f0445b92606) --- arch/arm64/configs/j813-usb4.config | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 arch/arm64/configs/j813-usb4.config diff --git a/arch/arm64/configs/j813-usb4.config b/arch/arm64/configs/j813-usb4.config new file mode 100644 index 0000000000000..d754400c6eccd --- /dev/null +++ b/arch/arm64/configs/j813-usb4.config @@ -0,0 +1,7 @@ +# Additional build coverage for M5 USB4 controller development. +# Merge with j813.config. This does not enable USB4 device-tree nodes. +CONFIG_PCI=y +CONFIG_PCIE_APPLE=y +CONFIG_USB4=y +CONFIG_USB4_APPLE_SOC=y +CONFIG_RESET_APPLE_CIO=y From 12f6a47fa3e0baee67362e91b6d31f2a2981e50d Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 23:34:51 -0400 Subject: [PATCH 1316/2680] arm64: dts: apple: t8142-j813: add the internal display MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe the J813 display coprocessor (DCP), its mailbox and both display DARTs, with resources read from this machine's Apple Device Tree, plus the dcp/disp0/disp0_piodma aliases the loader uses to hand over the live firmware mappings. The DCP and display DART stay disabled until the loader has reserved those mappings and enables them. Enable the Apple DRM driver in the J813 configuration and document the loader contract and the tested result. Tested on J813 from RAM with public m1n1 and U-Boot plus their J813 display and MTP companion changes: DCP completed startup and drove the internal panel at 2560x1600 (notch hidden) through appledrmfb, DPMS off/on cycles completed, and a KDE Plasma Wayland session ran with the built-in keyboard and trackpad. Rendering is software-only. [ryan: split by topic: only the Documentation/arch/arm64/apple-j813.rst, arch/arm64/configs/j813.config part; the rest is in 000-devicetree] [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] (cherry picked from commit 1f30c693c1f133260b83ddc29e5cb2939cc64346) --- Documentation/arch/arm64/apple-j813.rst | 38 +++++++++++++++++++++++++ arch/arm64/configs/j813.config | 4 +++ 2 files changed, 42 insertions(+) diff --git a/Documentation/arch/arm64/apple-j813.rst b/Documentation/arch/arm64/apple-j813.rst index 8ba57a51e3320..a3819bc60a524 100644 --- a/Documentation/arch/arm64/apple-j813.rst +++ b/Documentation/arch/arm64/apple-j813.rst @@ -173,6 +173,7 @@ an Image build or successful loader exit alone is not a boot result. Review captures for device identifiers and boot entropy before publishing. These boot tests do not establish CPU hotplug, KVM guest timers, +sustained thermal behavior, suspend, GPU acceleration, USB, storage, networking or audio support. Input and monitoring require their separate checks below. The disabled Samsung UART node records its measured resources; the qualified console path is DockChannel. @@ -215,6 +216,9 @@ the keyboard or temperature monitor from registering. The loader must preserve the iBoot-loaded MTP image and SRAM, prepare its DMA protection, and leave MTP ready for Linux's RTKit startup. A prior-stage MTP client must complete a compatible shutdown before Linux starts; a partially +initialized or crashed coprocessor is not a supported handoff. On J813, U-Boot +must not start and stop MTP before Linux: Linux's later restart crashes the +firmware and takes the keyboard and trackpad down. The backlight uses the existing PWM and LED drivers. Its boot brightness is retained when the initial PWM period is valid, otherwise it starts off. @@ -238,3 +242,37 @@ proof. Preserve stdout, the source revision and SHA-256 hashes of the exact Image, DTB, configuration, initramfs and bootloaders as the repeatable result. Keep firmware and raw machine data private. Without physical events the input check fails rather than crediting device registration as working input. + +Native display +============== + +The internal panel is driven by the display coprocessor (DCP) through the +Apple DRM driver. iBoot leaves DCP running with H17-generation (macOS 26) +firmware mapped through locked DARTs. Linux attaches to that live session: +it sends the standard RTKit wake handshake and never stops, resets or reloads +the coprocessor. No firmware file is loaded by Linux. + +The loader must: + +* reserve every live DCP firmware segment and the boot framebuffer, and + export their IOVAs to the ``dcp``, ``disp0`` and ``disp0_piodma`` nodes; +* enable the display DART (the first ``iommus`` entry of ``disp0``) and the + ``dcp`` node only after doing so. + +The display DARTs keep the firmware's root page-table slots. Linux publishes +its own translations only into vacant slots. On ``apple,t8142-dart``, a +firmware slot is accepted when it already translates every page Linux mapped +there to the same physical address, and later Linux mappings into it must +match the firmware's translation. Firmware buffers DCP later asks Linux to +share are resolved through those firmware tables. + +The DCP firmware reports the panel dimensions at startup; the driver does not +embed a panel timing or geometry. The J813 panel reports 2560x1664; the driver +hides its 64-pixel notch area by default, so the mode is 2560x1600 at 60 Hz. +The primary plane scans out IOMFB surface 2. Backlight control and a cursor +plane are not provided yet. + +Boot the RAM payload as above with ``CONFIG_DRM_APPLE``. On J813 this reached +``DCP booted``, registered ``appledrmfb`` as the console framebuffer, and ran +a KDE Plasma Wayland session with the built-in keyboard and trackpad. The +desktop renders in software; GPU acceleration is separate work. diff --git a/arch/arm64/configs/j813.config b/arch/arm64/configs/j813.config index 57fb42b0295c5..9091e74f3e918 100644 --- a/arch/arm64/configs/j813.config +++ b/arch/arm64/configs/j813.config @@ -45,6 +45,10 @@ CONFIG_FB=y CONFIG_FB_DEVICE=y CONFIG_FB_SIMPLE=y CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_DRM=y +CONFIG_DRM_APPLE=y +CONFIG_BACKLIGHT_CLASS_DEVICE=y +CONFIG_DRM_FBDEV_EMULATION=y # CONFIG_FRAMEBUFFER_CONSOLE_DEFERRED_TAKEOVER is not set CONFIG_LOG_BUF_SHIFT=20 CONFIG_IKCONFIG=y From 83d57f41eca42e829403c5fb52cb29f59ba7cf7b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 16:12:48 +1100 Subject: [PATCH 1317/2680] MAINTAINERS: assign new DCP bindings and tests to Aurora support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new DCP bindings, backlight documentation and native tests belong to Aurora's T8140 and J700 support. Move their file patterns to that existing entry so reviews for those files reach their maintainer. Keep the existing Apple DRM driver source pattern under its current display-driver entry. [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] (cherry picked from commit 562c6131ed44b9991ebd05614d12bd623d4e34c2) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index a060801d3ae8c..d569e08ca664b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2029,6 +2029,9 @@ APPLE T8140 SOC AND J700 (MACBOOK NEO) SUPPORT M: Ryan Murray W: https://aurora-silicon.github.io S: Maintained +F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml +F: Documentation/devicetree/bindings/display/apple,dcp.yaml +F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml F: Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml F: Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml F: Documentation/devicetree/bindings/media/apple,t8140-isp.yaml @@ -2039,6 +2042,7 @@ F: Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml +F: Documentation/gpu/apple-dcp-backlight.rst F: Documentation/hid/apple-j700-multitouch.rst F: arch/arm64/boot/dts/apple/t8140* F: arch/arm64/include/asm/apple-idle.h @@ -2050,6 +2054,7 @@ F: drivers/tty/serial/apple_dockchannel* F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs +F: tools/testing/apple/ APPLETALK NETWORK LAYER L: netdev@vger.kernel.org From 138aac5c58ae688389bc7db64a4551430604c57f Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:25:13 -0700 Subject: [PATCH 1318/2680] usb: dwc3: apple: test the USB2 PHY mode choice with KUnit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit selects the USB2 PHY's mode before dwc3 leaves reset for device mode only, and leaves host mode to the xHCI root hub. Selecting host mode there again would bring back sessions without USB2. Move that choice into dwc3_apple_usb2_mode_before_reset(), which also rejects states that are not roles, and move the controller state enum into dwc3-apple-state.h with it. dwc3_apple_core_start() takes both the validation and the mode from the helper, so the choice lives in one place. A KUnit suite pins the helper's results for every state. It covers the helper, not the hardware ordering behind it. [ryan: split by topic: only the MAINTAINERS part; the rest is in 170-usb] [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 65eb108802f4a834ae568967357d1a7a501ad8f1) --- MAINTAINERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index d569e08ca664b..c85413682992f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2700,6 +2700,8 @@ F: drivers/spmi/spmi-apple-controller.c F: drivers/thunderbolt/.kunitconfig.apple-dpin F: drivers/thunderbolt/apple-dpin* F: drivers/thunderbolt/apple.c +F: drivers/usb/dwc3/dwc3-apple-state-test.c +F: drivers/usb/dwc3/dwc3-apple-state.h F: drivers/usb/dwc3/dwc3-apple.c F: drivers/usb/typec/tipd/spmi.c F: drivers/video/backlight/apple_dwi_bl.c From b45089c4f950a70c868bea805186830cb8d4bbeb Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 00:57:55 -0400 Subject: [PATCH 1319/2680] arm64: configs: asahi: build the USB4 connection manager and Apple NHI Build the USB4 connection manager and Apple NHI with the Asahi profile. Keep unrelated SEP and host configuration out of this stack. [Chris Kearney: ported onto aurora-silicon/linux#8 from "arm64: enable Apple USB4 modules and J414/J416 firmware aliases". The J414/J416 underscore aliases are already on this branch, so only the configuration is carried.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 8580f66efb64724316fb9576f09db3772d787081) --- arch/arm64/configs/asahi.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 0877463349d97..61eb48f4a4399 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -49,6 +49,8 @@ CONFIG_HID_DOCKCHANNEL=m CONFIG_SPI_HID_APPLE_OF=m CONFIG_SPI_HID_APPLE_CORE=m CONFIG_USB_DWC3_APPLE=m +CONFIG_USB4=m +CONFIG_USB4_APPLE_SOC=m CONFIG_USB_XHCI_PCI_ASMEDIA=y CONFIG_RTC_DRV_MACSMC=m CONFIG_APPLE_ADMAC=m From 4c93c8d4f96c59b9a7599e82e9cfc03bcfaee6b7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 00:31:01 +1000 Subject: [PATCH 1320/2680] arm64: configs: asahi: enable the T8140 console, boot framebuffer and port controllers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MacBook Neo (J700, T8140) comes up from defconfig + asahi.config with no console, no display and no USB-C port controllers: - its serial console is a DockChannel UART (apple,dockchannel-uart, console=ttyDC0), so SERIAL_APPLE_DOCKCHANNEL and its early console have to be built in; - both USB-C ports are driven by SN201202x controllers on SPMI (apple,sn201202x); - the only display path on this tree is the simple-framebuffer the bootloader hands over, which needs simpledrm. All three were enabled in the kernel qualified on hardware on 2026-09-26 (source tree bac923ab, identical to this branch's base) and bound on that boot: ttyDC0 at 0x30812c000, Type-C port0 and port1 with PD contracts, simpledrm fb0. The other differences between that kernel's configuration and defconfig + asahi.config are lab and distribution choices -- drivers built in for the tethered initramfs, debug options, nftables, zram and Bluetooth profiles -- and stay out of this fragment. [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] (cherry picked from commit a7e6e4f7a1604918f45dc18e7abf6ab5913c4440) Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/configs/asahi.config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 61eb48f4a4399..797405da367ee 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -32,6 +32,7 @@ CONFIG_DRM_ASAHI=m CONFIG_DRM_ADP=m CONFIG_DRM_APPLE=m CONFIG_DRM_APPLE_AUDIO=y +CONFIG_DRM_SIMPLEDRM=y CONFIG_SND_SOC_APPLE_AOP_AUDIO=m CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=m CONFIG_SND_SOC_APPLE_MCA=m @@ -52,6 +53,7 @@ CONFIG_USB_DWC3_APPLE=m CONFIG_USB4=m CONFIG_USB4_APPLE_SOC=m CONFIG_USB_XHCI_PCI_ASMEDIA=y +CONFIG_TYPEC_SN201202X=m CONFIG_RTC_DRV_MACSMC=m CONFIG_APPLE_ADMAC=m CONFIG_APPLE_SIO=m @@ -59,6 +61,8 @@ CONFIG_MFD_MACSMC=m CONFIG_COMMON_CLK_APPLE_NCO=m CONFIG_APPLE_DART=m CONFIG_APPLE_DOCKCHANNEL=m +CONFIG_SERIAL_APPLE_DOCKCHANNEL=y +CONFIG_SERIAL_APPLE_DOCKCHANNEL_EARLYCON=y CONFIG_APPLE_MAILBOX=y CONFIG_APPLE_PMGR_MISC=y CONFIG_APPLE_RTKIT=y From d93e197ed1c86419c06cbf44fdf11be70c58ffc8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 02:06:39 +1000 Subject: [PATCH 1321/2680] arm64: configs: asahi: Default to the power allocator thermal governor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Device-tree thermal zones cannot name their governor, and the step_wise default moves a cooling device by one state per temperature change. The T8140 SMC reports die temperatures about once a second while the die can heat by tens of degrees in that time, so step_wise needs many seconds to reach a safe frequency and then oscillates around the trip. The power allocator moves the cooling devices straight to the state that fits the power budget and splits the budget between clusters by their energy models. Zones without a controlled trip are unaffected. [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] (cherry picked from commit d7ae133578fb57775af980c557d66db52aca8432) Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/configs/asahi.config | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 797405da367ee..ef28135099e69 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -7,6 +7,7 @@ CONFIG_ARM64_16K_PAGES=y CONFIG_ARM64_MEMORY_MODEL_CONTROL=y CONFIG_ARM_APPLE_CPUIDLE=y CONFIG_ARM_APPLE_SOC_CPUFREQ=m +CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y CONFIG_BT_BRCMEXT=y CONFIG_BT_HCIBCM4377=m CONFIG_PCIE_APPLE=m From ad43dacc96bb6fa6610ce14468c28913af1f3abc Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Wed, 7 Oct 2026 09:49:28 -0500 Subject: [PATCH 1322/2680] arm64: asahi.config: build the RDMA core and the soft RDMA drivers as modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Apple Silicon kernel has CONFIG_INFINIBAND unset, so no node has an RDMA device, ib_uverbs or the rdma tools' kernel side. Build the RDMA core, its userspace verbs and management interfaces, and the two software providers (rxe, siw) as modules. With these, rdma-core and perftest work on any Mac with a network interface: `modprobe rdma_rxe; rdma link add rxe0 type rxe netdev ` gives a verbs device that needs no RDMA hardware. Nothing loads unless a user asks for it. All five are =m. In `make defconfig asahi.config` they resolve with their dependencies (NET_UDP_TUNNEL=m, IRQ_POLL=y, DIMLIB=y, DMA_SHARED_BUFFER=y, INFINIBAND_VIRT_DMA=y). [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] (cherry picked from commit 0a89f86921266e267f478fab4c6d3642f8bba53e) --- arch/arm64/configs/asahi.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index ef28135099e69..f5afe7166dbb1 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -93,3 +93,8 @@ CONFIG_TYPEC_TPS6598X=m CONFIG_TYPEC_SN201202X=m CONFIG_RESET_APPLE_CIO=m CONFIG_USB4_APPLE_SOC=m +CONFIG_INFINIBAND=m +CONFIG_INFINIBAND_USER_ACCESS=m +CONFIG_INFINIBAND_USER_MAD=m +CONFIG_RDMA_RXE=m +CONFIG_RDMA_SIW=m From 16dc635626a389e34343f74265e427130cbc312f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:40:33 +1100 Subject: [PATCH 1323/2680] dt-bindings: apple: Describe T8140 idle and display crossbar controllers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document the T8140 CPU idle selector and the existing Apple display crossbar interface. T8140 uses only the WFI clock-gated state; CPU and cluster power-down are not supported. The crossbar compatible follows the controller type reported by the ADT and the existing Asahi driver. Keep CS42L42 supply phandles mandatory while permitting the established CS42L83 board model with platform-managed supplies. Existing public Apple board descriptions already use that model. [ryan: split by topic: only the MAINTAINERS part] [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] (cherry picked from commit 691ba4cea2597906a194b04615cb6833aaece7b9) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index c85413682992f..7ca03439208ed 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2029,12 +2029,14 @@ APPLE T8140 SOC AND J700 (MACBOOK NEO) SUPPORT M: Ryan Murray W: https://aurora-silicon.github.io S: Maintained +F: Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml F: Documentation/devicetree/bindings/display/apple,dcp.yaml F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml F: Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml F: Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml F: Documentation/devicetree/bindings/media/apple,t8140-isp.yaml +F: Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml F: Documentation/devicetree/bindings/reserved-memory/apple,asc-mem.yaml F: Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml F: Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml From eb82e328aa48f2ca0ecb5cc663d63a6e2e7100ee Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 16:15:31 +1100 Subject: [PATCH 1324/2680] dt-bindings: mailbox: Describe the T8140 GPU ASCWrap interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe the existing GPU coprocessor mailbox interface, with wrapper and vector-base resources, two FIFO interrupts and firmware role names. The public T8140 GPU device tree already supplies these properties; without a binding, validation cannot decode the firmware-role string. The description follows the public GPU interface and J700 ADT resource layout. This does not change the GPU device tree or driver behavior. [ryan: split by topic: only the MAINTAINERS part] [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] (cherry picked from commit 1afd820cd7fcc775dc154ce4bfc1f955bc814bfc) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 7ca03439208ed..8bb2ed8c5e70a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2035,6 +2035,7 @@ F: Documentation/devicetree/bindings/display/apple,dcp.yaml F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml F: Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml F: Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml +F: Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml F: Documentation/devicetree/bindings/media/apple,t8140-isp.yaml F: Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml F: Documentation/devicetree/bindings/reserved-memory/apple,asc-mem.yaml From eaf41f43b7e349f0c67ec2ecc98ec25aff3617c7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:54:44 +1100 Subject: [PATCH 1325/2680] arm64: configs: select the J700 radio transport and bootstrap parents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The retained PIODMA bootstrap requires a built-in Apple PCI host, whose DART parent must also be built in. Select those providers in the Asahi fragment and request the existing Bluetooth transport as a module. Add a J700 radio fragment that selects the fullmac and modular Bluetooth paths, while preserving system sleep and disabling the kexec selectors that cannot preserve live firmware DMA ownership. It is applied after the general Asahi fragment. Runtime enumeration and Bluetooth admission remain opt-in and firmware/calibration remain external packages. [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] (cherry picked from commit aca3d365aad20020392346780c8b65249f59ceff) Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/configs/asahi.config | 5 +++-- arch/arm64/configs/j700-radio.config | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/configs/j700-radio.config diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index f5afe7166dbb1..a3fa236d1f20e 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -10,7 +10,8 @@ CONFIG_ARM_APPLE_SOC_CPUFREQ=m CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y CONFIG_BT_BRCMEXT=y CONFIG_BT_HCIBCM4377=m -CONFIG_PCIE_APPLE=m +CONFIG_BT_MTK7932_PCIE=m +CONFIG_PCIE_APPLE=y CONFIG_NVME_APPLE=m CONFIG_BRCMFMAC=m CONFIG_BRCMFMAC_PCIE=y @@ -60,7 +61,7 @@ CONFIG_APPLE_ADMAC=m CONFIG_APPLE_SIO=m CONFIG_MFD_MACSMC=m CONFIG_COMMON_CLK_APPLE_NCO=m -CONFIG_APPLE_DART=m +CONFIG_APPLE_DART=y CONFIG_APPLE_DOCKCHANNEL=m CONFIG_SERIAL_APPLE_DOCKCHANNEL=y CONFIG_SERIAL_APPLE_DOCKCHANNEL_EARLYCON=y diff --git a/arch/arm64/configs/j700-radio.config b/arch/arm64/configs/j700-radio.config new file mode 100644 index 0000000000000..87b09a42d3cc6 --- /dev/null +++ b/arch/arm64/configs/j700-radio.config @@ -0,0 +1,19 @@ +# J700 retained radio bootstrap; merge after defconfig and asahi.config. +# System sleep is retained; active radio firmware cannot cross kexec. +# Firmware/calibration and runtime admission follow the MT7932 documentation. +# CONFIG_KEXEC is not set +# CONFIG_KEXEC_FILE is not set +# CONFIG_KEXEC_HANDOVER is not set +CONFIG_PCI=y +CONFIG_PCIE_APPLE=y +CONFIG_APPLE_DART=y +CONFIG_PCIE_APPLE_PIODMA_DIAG=y +CONFIG_WLAN=y +CONFIG_WLAN_VENDOR_MEDIATEK=y +CONFIG_MT7932_FULLMAC=m +CONFIG_BT=m +CONFIG_BT_BREDR=y +CONFIG_BT_LE=y +CONFIG_BT_MTK7932_PCIE=m +CONFIG_CRYPTO_AES=y +CONFIG_CRYPTO_CMAC=y From 171840b3c2b19b881d78d6e3e586709b911b764c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:22:12 +1100 Subject: [PATCH 1326/2680] pmdomain: apple: pmp-report: expose T8140 fast-die effort MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe the T8140 report layout from the Apple device tree and expose its fast-die effort through the existing PMP report mapping. A stopped PMP reads zero, so check the running bit before accepting an effort. Report entry 0x1c1 contains byte 1 for the P cluster, observed under Linux load, and byte 0 for E, assigned from ADT lane order. The E byte has not yet been observed changing. Do not read entry 0x1c2 or scan the unpopulated part of the report aperture. Expose read-only debugfs status and lane effort values for diagnostics. Port the original Linux-derived implementation by Ryan Murray. Retain the Asahi report driver and use it as the sole owner of the mapping. [ryan: split by topic: only the MAINTAINERS part] [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] (cherry picked from commit f69f1b76fcb04e6bd7d7d066654e612be9761af0) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 8bb2ed8c5e70a..89e141ed32cf3 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2030,6 +2030,9 @@ M: Ryan Murray W: https://aurora-silicon.github.io S: Maintained F: Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml +F: drivers/pmdomain/apple/pmp-report.c +F: drivers/soc/apple/pmp.rs +F: include/linux/soc/apple/pmp-report.h F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml F: Documentation/devicetree/bindings/display/apple,dcp.yaml F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml From 954971301719ba1bce1a661015f3fac812781bdf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:00:11 +1100 Subject: [PATCH 1327/2680] dt-bindings: apple: describe PMP fast-die reporting on T8140 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document the existing PMP firmware driver resources and the report aperture consumed by the fast-die limiter. Define a report phandle and lane index on the cluster DVFS controller, using the measured P lane and ADT E lane ordering. Firmware calibration properties are populated by the bootloader from the Apple device tree. [ryan: split by topic: only the MAINTAINERS part] [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] (cherry picked from commit a1f9aabbe6a9771a4dbf3259bc808713f0daac46) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 89e141ed32cf3..cafc2a3a061f9 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2030,6 +2030,9 @@ M: Ryan Murray W: https://aurora-silicon.github.io S: Maintained F: Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml +F: Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml +F: Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml F: drivers/pmdomain/apple/pmp-report.c F: drivers/soc/apple/pmp.rs F: include/linux/soc/apple/pmp-report.h From 1809161a1b214a065ac320af64e114e1c871d382 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 16:12:22 +1100 Subject: [PATCH 1328/2680] MAINTAINERS: place the J700 PMU RTC under Aurora support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Track the new J700 PMU RTC driver and binding under the existing T8140 and J700 support entry. Keep the existing Apple machine maintenance entry focused on its original files. [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] (cherry picked from commit a87814cc20907f021b4e63ce4d0b6f2a63932161) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index cafc2a3a061f9..9841b01ed0a01 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2045,6 +2045,7 @@ F: Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml F: Documentation/devicetree/bindings/media/apple,t8140-isp.yaml F: Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml F: Documentation/devicetree/bindings/reserved-memory/apple,asc-mem.yaml +F: Documentation/devicetree/bindings/rtc/apple,j700-pmu-rtc.yaml F: Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml F: Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml F: Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml @@ -2058,6 +2059,7 @@ F: arch/arm64/include/asm/apple-idle.h F: arch/arm64/kernel/idle-apple.S F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* F: drivers/iio/common/aop_sensors/ +F: drivers/rtc/rtc-apple-pmu.c F: drivers/soc/apple/aop.rs F: drivers/tty/serial/apple_dockchannel* F: rust/kernel/iio/common/aop_sensors.rs @@ -2660,7 +2662,6 @@ F: Documentation/devicetree/bindings/power/apple* F: Documentation/devicetree/bindings/power/reset/apple,smc-reboot.yaml F: Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml F: Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml -F: Documentation/devicetree/bindings/rtc/apple,j700-pmu-rtc.yaml F: Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml F: Documentation/devicetree/bindings/soc/apple/apple,sep.yaml F: Documentation/devicetree/bindings/spi/apple,spi.yaml @@ -2700,7 +2701,6 @@ F: drivers/power/reset/macsmc-reboot.c F: drivers/power/supply/macsmc-power.c F: drivers/pwm/pwm-apple.c F: drivers/reset/reset-apple-cio.c -F: drivers/rtc/rtc-apple-pmu.c F: drivers/rtc/rtc-macsmc.c F: drivers/soc/apple/* F: tools/asahi/pmp-bootargs-tests.rs From cb3c2e8e37d15ad00881e49cbf9271c5d70ec161 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:15:45 +1100 Subject: [PATCH 1329/2680] wifi: mt7932: depend on cfg80211 instead of selecting it MT7932_FULLMAC selects CFG80211. Built in, that forces CFG80211=y even when RFKILL=m, and Kconfig then warns about an unmet dependency, because CFG80211 depends on RFKILL || !RFKILL. Wireless drivers normally depend on CFG80211 rather than select it. Do the same, so that cfg80211's own constraints decide whether the driver can be built in. The J700 radio fragment relied on the select. Enable cfg80211 there as a module, which also lets it load regulatory.db from the root filesystem. Record the modular cfg80211 in the tested configuration. [ryan: split by topic: only the arch/arm64/configs/j700-radio.config part; the rest is in 080-radio] (cherry picked from commit 81cdd0987ba999a21c1eced2a0a7c19045c9c95c) Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/configs/j700-radio.config | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/configs/j700-radio.config b/arch/arm64/configs/j700-radio.config index 87b09a42d3cc6..f5c5d3c3ec0bf 100644 --- a/arch/arm64/configs/j700-radio.config +++ b/arch/arm64/configs/j700-radio.config @@ -8,6 +8,7 @@ CONFIG_PCI=y CONFIG_PCIE_APPLE=y CONFIG_APPLE_DART=y CONFIG_PCIE_APPLE_PIODMA_DIAG=y +CONFIG_CFG80211=m CONFIG_WLAN=y CONFIG_WLAN_VENDOR_MEDIATEK=y CONFIG_MT7932_FULLMAC=m From 056b4954ef74f72c1c2a7e71ab1ebde2540c0d65 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:05:49 +1100 Subject: [PATCH 1330/2680] arm64: configs: asahi: enable the J700 Wi-Fi driver asahi.config asks for the J700 Bluetooth transport, but not for the Wi-Fi driver whose bound function the transport waits for before it starts. Ask for both, as j700-radio.config already does. (cherry picked from commit b2f686e884399dac0a2462ba44370767753a1589) Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/configs/asahi.config | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index a3fa236d1f20e..f4848da9a4025 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -15,6 +15,7 @@ CONFIG_PCIE_APPLE=y CONFIG_NVME_APPLE=m CONFIG_BRCMFMAC=m CONFIG_BRCMFMAC_PCIE=y +CONFIG_MT7932_FULLMAC=m CONFIG_TOUCHSCREEN_APPLE_Z2=m CONFIG_INPUT_MACSMC_INPUT=m CONFIG_I2C_APPLE=m From 4757bd772c897e22fa60d66e92edf4b821a90269 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 15:16:11 +1100 Subject: [PATCH 1331/2680] dt-bindings: soc: apple: Add the T8140 memory cache controller The memory cache controller of the Apple T8140 holds the system level cache in front of DRAM, split into planes. The boot firmware enables and powers the cache ways of every plane but leaves allocation disabled, so the operating system has to allow it. Describe one register block per plane, and the devices that take over DMA the bootloader left running: allocation must wait until they are bound. [ryan: split by topic: only the MAINTAINERS part; the rest is in 010-soc] (cherry picked from commit 7bfcdc23ebd5224a109efa80afd896d0c9c3dc53) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 9841b01ed0a01..491d3f3f42b1a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2050,6 +2050,7 @@ F: Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml F: Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml F: Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,t8140-mcc.yaml F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml F: Documentation/gpu/apple-dcp-backlight.rst From ef95edb9871f76ae342ce521a19903a78de842ad Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 15:16:21 +1100 Subject: [PATCH 1332/2680] soc: apple: Enable T8140 system level cache allocation On T8140 the boot firmware enables and powers the 12 ways of each of the 4 system level cache planes, but leaves the allocation-disable word of every plane set. The cache therefore holds no data, and every GPU and CPU access that misses the core caches goes to DRAM. Clear the allocation-disable word of each plane once every plane reports as many ways powered as enabled and no flush in progress, and read it back. Do it only after the devices named in the device tree as having taken over DMA from the bootloader are bound, so the cache starts allocating after their hand-off rather than during it. On shutdown, stop allocating and flush the cache before the next boot stage runs. With allocation disabled the cache holds no data, so no maintenance is needed before enabling it. Measured on J700 (MacBook Neo), toggling only this word on a running system: glmark2 build+texture 902 -> 1886, vkcube 2141 -> 5584 fps, glmark2 terrain at 1920x1200 41 -> 66 fps, and the submit-to-completion latency of an isolated GPU job 560 -> 210 us. [ryan: split by topic: only the MAINTAINERS part; the rest is in 010-soc] (cherry picked from commit f4ec00ff19dbfdc6d9bdbf03e8d7ec2798502f1a) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 491d3f3f42b1a..087ac82c2b1f7 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2062,6 +2062,7 @@ F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* F: drivers/iio/common/aop_sensors/ F: drivers/rtc/rtc-apple-pmu.c F: drivers/soc/apple/aop.rs +F: drivers/soc/apple/mcc.c F: drivers/tty/serial/apple_dockchannel* F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs From 6231b2ffd8b0d2c6a31300bdcb191b84875766e1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:23:20 +1100 Subject: [PATCH 1333/2680] wifi: mt7932: report whether the Wi-Fi function initialized When initialization fails and the DMA of function 0 cannot be stopped, mt7932-fullmac keeps the binding to retain that DMA. Whether the Wi-Fi firmware is running therefore cannot be told from the binding, which is what the Bluetooth transport on function 1 waits for before it starts its own firmware on the shared radio. Record when the probe has finished initialization and export mt7932_fullmac_ready() to report it. [ryan: split by topic: only the MAINTAINERS part; the rest is in 080-radio] (cherry picked from commit 826c3346b3ca39728f568e5f6ff0ec78e6c6199d) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 087ac82c2b1f7..8b78dfcda41df 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2035,6 +2035,7 @@ F: Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml F: Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml F: drivers/pmdomain/apple/pmp-report.c F: drivers/soc/apple/pmp.rs +F: include/linux/mt7932.h F: include/linux/soc/apple/pmp-report.h F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml F: Documentation/devicetree/bindings/display/apple,dcp.yaml From 4a7ad7982adccfabe1b74978940e6c6f564b74aa Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 20:57:42 +1100 Subject: [PATCH 1334/2680] arm64: configs: add a J700 base fragment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit defconfig, asahi.config and j700-radio.config give a J700 kernel that boots, but not the configuration it is tested with. The boot path (NVMe/ANS, btrfs, the DockChannel keyboard and trackpad, SMC, SPMI, the USB-C PHY and Type-C) ends up modular and so depends on what the initramfs carries, the per-process 4K user page mode stays off, and several J700 driver choices differ. Add j700.config, to be merged after j700-radio.config, with each group commented with its reason, and cover the J700 fragments with one MAINTAINERS pattern. Together with the device-parity fragment added next, the fragments reproduce the tested configuration. [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] (cherry picked from commit 69b264dc00b60aa612f2d7bc9f214a3e877cb8c8) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + arch/arm64/configs/j700.config | 113 +++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 arch/arm64/configs/j700.config diff --git a/MAINTAINERS b/MAINTAINERS index 8b78dfcda41df..7f1ebd3e608d2 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2057,6 +2057,7 @@ F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml F: Documentation/gpu/apple-dcp-backlight.rst F: Documentation/hid/apple-j700-multitouch.rst F: arch/arm64/boot/dts/apple/t8140* +F: arch/arm64/configs/j700*.config F: arch/arm64/include/asm/apple-idle.h F: arch/arm64/kernel/idle-apple.S F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* diff --git a/arch/arm64/configs/j700.config b/arch/arm64/configs/j700.config new file mode 100644 index 0000000000000..976fd65b7161c --- /dev/null +++ b/arch/arm64/configs/j700.config @@ -0,0 +1,113 @@ +# J700 (MacBook Neo, T8140) base fragment. +# +# Merge after defconfig, asahi.config and j700-radio.config, then add the device-parity fragment: +# make defconfig +# scripts/kconfig/merge_config.sh -m .config arch/arm64/configs/asahi.config arch/arm64/configs/j700-radio.config +# make olddefconfig +# scripts/kconfig/merge_config.sh -m .config arch/arm64/configs/j700.config arch/arm64/configs/j700-device-parity.config +# make olddefconfig +# The result is the configuration the J700 kernel is tested with. + +# Boot NOR (nvram) access for boot-volume switching, and the PMU RTC wake alarm +CONFIG_SPI_APPLE_QSPI=m +CONFIG_RTC_DRV_APPLE_PMU=m + +# Classic Bluetooth profiles (headsets, tethering) +CONFIG_BT_RFCOMM=m +CONFIG_BT_RFCOMM_TTY=y +CONFIG_BT_BNEP=m +CONFIG_BT_BNEP_MC_FILTER=y +CONFIG_BT_BNEP_PROTO_FILTER=y + +# Distribution firmware files are compressed +CONFIG_FW_LOADER_COMPRESS=y +CONFIG_FW_LOADER_COMPRESS_ZSTD=y + +# 48-bit virtual addresses; KASLR is untested on this machine +CONFIG_ARM64_VA_BITS_48=y +# CONFIG_RANDOMIZE_BASE is not set + +# Boot path and platform built in, so storage, keyboard, SMC and USB do not depend on the initramfs +CONFIG_BLK_DEV_NVME=y +CONFIG_NVME_APPLE=y +CONFIG_APPLE_SART=y +CONFIG_APPLE_RTKIT_HELPER=y +CONFIG_BTRFS_FS=y +CONFIG_APPLE_DOCKCHANNEL=y +CONFIG_HID_DOCKCHANNEL=y +CONFIG_HID_APPLE=y +CONFIG_HID_MAGICMOUSE=y +CONFIG_SPMI_APPLE=y +CONFIG_NVMEM_APPLE_SPMI=y +CONFIG_MFD_MACSMC=y +CONFIG_GPIO_MACSMC=y +CONFIG_POWER_RESET_MACSMC=y +CONFIG_SENSORS_MACSMC_HWMON=y +CONFIG_I2C_APPLE=y +CONFIG_PINCTRL_APPLE_GPIO=y +CONFIG_APPLE_WATCHDOG=y +CONFIG_ARM_APPLE_SOC_CPUFREQ=y +CONFIG_PHY_APPLE_ATC=y +CONFIG_USB_DWC3_APPLE=y +CONFIG_TYPEC=y +CONFIG_TYPEC_SN201202X=y +CONFIG_MUX_APPLE_DPXBAR=y +CONFIG_BACKLIGHT_CLASS_DEVICE=y +CONFIG_SYSFB_SIMPLEFB=y +CONFIG_ZRAM=y +CONFIG_UHID=y +CONFIG_RFKILL=y +CONFIG_THERMAL_NETLINK=y +CONFIG_NVME_HWMON=y + +# J700 driver choices +CONFIG_APPLE_CLPC=m +CONFIG_APPLE_PMP=y +CONFIG_DRM_ACCEL=y +CONFIG_DRM_ACCEL_ANE=m +# CONFIG_HID_APPLE_MTP_HAPTIC is not set +CONFIG_IIO=m +# CONFIG_IIO_AOP_SENSOR_LAS is not set +# CONFIG_USB4 is not set + +# Per-process 4K user pages on the 16K kernel (x86 emulation) +CONFIG_ARM64_USER4K_EXPERIMENTAL=y + +# USB Ethernet built in, for a wired link before modules load +CONFIG_USB_USBNET=y +CONFIG_USB_NET_AX8817X=y +CONFIG_USB_NET_AX88179_178A=y +CONFIG_USB_NET_CDC_NCM=y +CONFIG_USB_RTL8152=y +CONFIG_USB_UAS=m + +# Crypto that Bluetooth and Wi-Fi need early +CONFIG_CRYPTO_CCM=y +CONFIG_CRYPTO_ECDH=y +CONFIG_CRYPTO_GCM=y +CONFIG_CRYPTO_HMAC=y +CONFIG_CRYPTO_SHA256=y + +# Utilisation clamping +CONFIG_UCLAMP_TASK=y +CONFIG_UCLAMP_TASK_GROUP=y + +# LSMs named in CONFIG_LSM +CONFIG_SECURITY_LANDLOCK=y +CONFIG_SECURITY_YAMA=y + +# dma-buf heaps and udmabuf (camera and GPU buffer sharing) +CONFIG_DMABUF_HEAPS=y +CONFIG_DMABUF_HEAPS_SYSTEM=y +CONFIG_UDMABUF=y + +# Debugging aids kept in the tested configuration +CONFIG_DYNAMIC_DEBUG=y +CONFIG_FTRACE=y +CONFIG_ENABLE_DEFAULT_TRACERS=y +CONFIG_PM_DEBUG=y +CONFIG_PM_ADVANCED_DEBUG=y + +# Match the tested kernel image: no relative relocation packing, no SCM version suffix +# CONFIG_RELR is not set +# CONFIG_LOCALVERSION_AUTO is not set From 5c50a22fd6dd241ecd966510b3cdce9d23f5cc1f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 20:57:43 +1100 Subject: [PATCH 1335/2680] arm64: configs: add a J700 device-parity fragment The J700 configuration lacks much of what a general-purpose distribution kernel provides: game controllers and other USB or Bluetooth HID devices, most USB-serial and USB network adapters, filesystems found on removable media, dm-crypt, VPN protocols, the nftables and xtables extensions used by firewall and container tools, and binfmt_misc. Add j700-device-parity.config, to be merged after j700.config. It takes the Fedora Asahi Remix 44 kernel configuration as the reference and enables what can attach to or run on the J700, mostly as modules. It changes no option that the other fragments set. Options that change scheduler, memory, security or boot behaviour are left out. (cherry picked from commit d96ce2dea7fe5c23188253d7e998aad0b82d38b6) Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/configs/j700-device-parity.config | 1053 ++++++++++++++++++ 1 file changed, 1053 insertions(+) create mode 100644 arch/arm64/configs/j700-device-parity.config diff --git a/arch/arm64/configs/j700-device-parity.config b/arch/arm64/configs/j700-device-parity.config new file mode 100644 index 0000000000000..55bce7a9ac0af --- /dev/null +++ b/arch/arm64/configs/j700-device-parity.config @@ -0,0 +1,1053 @@ +# J700 (MacBook Neo, T8140) device-parity fragment. +# +# Brings the J700 configuration to device and feature parity with a general-purpose +# distribution kernel (reference: Fedora Asahi Remix 44, 7.1.13-402.asahi.fc44.aarch64+16k): +# drivers for devices that can attach over USB or Bluetooth, and common protocols, +# filesystems and features. Most entries are modules. Nothing here changes an option +# that defconfig, asahi.config, j700-radio.config or j700.config set. +# +# Merge after j700.config; see that file for the full sequence. + +# Prerequisites of the entries below: nftables core, hidraw, uinput and the ALSA sequencer +CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_INET=y +CONFIG_NF_TABLES_IPV4=y +CONFIG_NF_TABLES_IPV6=y +CONFIG_NF_TABLES_NETDEV=y +CONFIG_NFT_CT=m +CONFIG_NFT_LOG=m +CONFIG_NFT_LIMIT=m +CONFIG_NFT_MASQ=m +CONFIG_NFT_NAT=m +CONFIG_NFT_REJECT=m +CONFIG_NFT_REJECT_INET=m +CONFIG_NFT_FIB_INET=m +CONFIG_NFT_FIB_IPV4=m +CONFIG_NFT_FIB_IPV6=m +CONFIG_HIDRAW=y +CONFIG_INPUT_UINPUT=m +CONFIG_SND_SEQUENCER=m +CONFIG_SND_HRTIMER=m +CONFIG_SND_SEQ_DUMMY=m +CONFIG_SND_SEQ_HRTIMER_DEFAULT=y + +# HID / input / game controllers (USB + Bluetooth HID devices, xpad, joydev, USB tablets/touchscreens) +CONFIG_DRAGONRISE_FF=y +CONFIG_GREENASIA_FF=y +CONFIG_HID_A4TECH=m +CONFIG_HID_ACCUTOUCH=m +CONFIG_HID_ACRUX=m +CONFIG_HID_ACRUX_FF=y +CONFIG_HID_ALPS=m +CONFIG_HID_APPLEIR=m +CONFIG_HID_AUREAL=m +CONFIG_HID_BATTERY_STRENGTH=y +CONFIG_HID_BELKIN=m +CONFIG_HID_BETOP_FF=m +CONFIG_HID_BIGBEN_FF=m +CONFIG_HID_CHERRY=m +CONFIG_HID_CHICONY=m +CONFIG_HID_CMEDIA=m +CONFIG_HID_CORSAIR=m +CONFIG_HID_COUGAR=m +CONFIG_HID_CP2112=m +CONFIG_HID_CREATIVE_SB0540=m +CONFIG_HID_CYPRESS=m +CONFIG_HID_DRAGONRISE=m +CONFIG_HID_ELAN=m +CONFIG_HID_ELECOM=m +CONFIG_HID_ELO=m +CONFIG_HID_EMS_FF=m +CONFIG_HID_EVISION=m +CONFIG_HID_EZKEY=m +CONFIG_HID_FT260=m +CONFIG_HID_GEMBIRD=m +CONFIG_HID_GFRM=m +CONFIG_HID_GLORIOUS=m +CONFIG_HID_GOOGLE_STADIA_FF=m +CONFIG_HID_GREENASIA=m +CONFIG_HID_GT683R=m +CONFIG_HID_GYRATION=m +CONFIG_HID_HOLTEK=m +CONFIG_HID_HUAWEI=m +CONFIG_HID_ICADE=m +CONFIG_HID_ITE=m +CONFIG_HID_JABRA=m +CONFIG_HID_KENSINGTON=m +CONFIG_HID_KEYTOUCH=m +CONFIG_HID_KYE=m +CONFIG_HID_KYSONA=m +CONFIG_HID_LCPOWER=m +CONFIG_HID_LED=m +CONFIG_HID_LENOVO=m +CONFIG_HID_LENOVO_GO=m +CONFIG_HID_LENOVO_GO_S=m +CONFIG_HID_LETSKETCH=m +CONFIG_HID_LOGITECH=m +CONFIG_HID_LOGITECH_DJ=m +CONFIG_HID_LOGITECH_HIDPP=m +CONFIG_HID_MACALLY=m +CONFIG_HID_MALTRON=m +CONFIG_HID_MAYFLASH=m +CONFIG_HID_MCP2221=m +CONFIG_HID_MEGAWORLD_FF=m +CONFIG_HID_MICROSOFT=m +CONFIG_HID_MONTEREY=m +CONFIG_HID_NINTENDO=m +CONFIG_HID_NTI=m +CONFIG_HID_NTRIG=m +CONFIG_HID_NVIDIA_SHIELD=m +CONFIG_HID_ORTEK=m +CONFIG_HID_PANTHERLORD=m +CONFIG_HID_PENMOUNT=m +CONFIG_HID_PETALYNX=m +CONFIG_HID_PICOLCD=m +CONFIG_HID_PICOLCD_BACKLIGHT=y +CONFIG_HID_PICOLCD_FB=y +CONFIG_HID_PICOLCD_LEDS=y +CONFIG_HID_PID=y +CONFIG_HID_PLANTRONICS=m +CONFIG_HID_PLAYSTATION=m +CONFIG_HID_PRIMAX=m +CONFIG_HID_PRODIKEYS=m +CONFIG_HID_PXRC=m +CONFIG_HID_RAPOO=m +CONFIG_HID_RAZER=m +CONFIG_HID_RETRODE=m +CONFIG_HID_RMI=m +CONFIG_HID_ROCCAT=m +CONFIG_HID_SAITEK=m +CONFIG_HID_SAMSUNG=m +CONFIG_HID_SEMITEK=m +CONFIG_HID_SIGMAMICRO=m +CONFIG_HID_SMARTJOYPLUS=m +CONFIG_HID_SONY=m +CONFIG_HID_SPEEDLINK=m +CONFIG_HID_STEAM=m +CONFIG_HID_STEELSERIES=m +CONFIG_HID_SUNPLUS=m +CONFIG_HID_THINGM=m +CONFIG_HID_THRUSTMASTER=m +CONFIG_HID_TIVO=m +CONFIG_HID_TOPRE=m +CONFIG_HID_TOPSEED=m +CONFIG_HID_TWINHAN=m +CONFIG_HID_U2FZERO=m +CONFIG_HID_UCLOGIC=m +CONFIG_HID_UDRAW_PS3=m +CONFIG_HID_UNIVERSAL_PIDFF=m +CONFIG_HID_VIEWSONIC=m +CONFIG_HID_VIVALDI=m +CONFIG_HID_WACOM=m +CONFIG_HID_WALTOP=m +CONFIG_HID_WIIMOTE=m +CONFIG_HID_WINWING=m +CONFIG_HID_XIAOMI=m +CONFIG_HID_XINMO=m +CONFIG_HID_ZEROPLUS=m +CONFIG_HID_ZYDACRON=m +CONFIG_HOLTEK_FF=y +CONFIG_INPUT_JOYDEV=m +CONFIG_INPUT_JOYSTICK=y +CONFIG_INPUT_TABLET=y +CONFIG_JOYSTICK_IFORCE=m +CONFIG_JOYSTICK_IFORCE_USB=m +CONFIG_JOYSTICK_PXRC=m +CONFIG_JOYSTICK_XPAD=m +CONFIG_JOYSTICK_XPAD_FF=y +CONFIG_JOYSTICK_XPAD_LEDS=y +CONFIG_LOGIG940_FF=y +CONFIG_LOGIRUMBLEPAD2_FF=y +CONFIG_LOGITECH_FF=y +CONFIG_LOGIWHEELS_FF=y +CONFIG_MOUSE_SYNAPTICS_USB=m +CONFIG_NINTENDO_FF=y +CONFIG_NVIDIA_SHIELD_FF=y +CONFIG_PANTHERLORD_FF=y +CONFIG_PLAYSTATION_FF=y +CONFIG_RMI4_CORE=m +CONFIG_RMI4_F03=y +CONFIG_RMI4_F11=y +CONFIG_RMI4_F12=y +CONFIG_RMI4_F1A=y +CONFIG_RMI4_F21=y +CONFIG_RMI4_F30=y +CONFIG_RMI4_F34=y +CONFIG_RMI4_F3A=y +CONFIG_RMI4_F55=y +CONFIG_SMARTJOYPLUS_FF=y +CONFIG_SONY_FF=y +CONFIG_STEAM_FF=y +CONFIG_TABLET_USB_ACECAD=m +CONFIG_TABLET_USB_AIPTEK=m +CONFIG_TABLET_USB_HANWANG=m +CONFIG_TABLET_USB_KBTAB=m +CONFIG_TABLET_USB_PEGASUS=m +CONFIG_THRUSTMASTER_FF=y +CONFIG_TOUCHSCREEN_USB_3M=y +CONFIG_TOUCHSCREEN_USB_COMPOSITE=m +CONFIG_TOUCHSCREEN_USB_DMC_TSC10=y +CONFIG_TOUCHSCREEN_USB_E2I=y +CONFIG_TOUCHSCREEN_USB_EASYTOUCH=y +CONFIG_TOUCHSCREEN_USB_EGALAX=y +CONFIG_TOUCHSCREEN_USB_ELO=y +CONFIG_TOUCHSCREEN_USB_ETT_TC45USB=y +CONFIG_TOUCHSCREEN_USB_ETURBO=y +CONFIG_TOUCHSCREEN_USB_GENERAL_TOUCH=y +CONFIG_TOUCHSCREEN_USB_GOTOP=y +CONFIG_TOUCHSCREEN_USB_GUNZE=y +CONFIG_TOUCHSCREEN_USB_IDEALTEK=y +CONFIG_TOUCHSCREEN_USB_IRTOUCH=y +CONFIG_TOUCHSCREEN_USB_ITM=y +CONFIG_TOUCHSCREEN_USB_JASTEC=y +CONFIG_TOUCHSCREEN_USB_NEXIO=y +CONFIG_TOUCHSCREEN_USB_PANJIT=y +CONFIG_TOUCHSCREEN_USB_ZYTRONIC=y +CONFIG_USB_HIDDEV=y +CONFIG_ZEROPLUS_FF=y + +# USB device classes: serial adapters, storage sub-drivers, printers/TMC/WDM, misc USB gadgets, usbmon, USB/IP, card readers, maker adapters +CONFIG_APPLE_MFI_FASTCHARGE=m +CONFIG_DLN2_ADC=m +CONFIG_GNSS_USB=m +CONFIG_GPIB=m +CONFIG_GPIB_AGILENT_82357A=m +CONFIG_GPIB_COMMON=m +CONFIG_GPIB_LPVO=m +CONFIG_GPIB_NI_USB=m +CONFIG_GPIO_DLN2=m +CONFIG_GPIO_MPSSE=m +CONFIG_I2C_CP2615=m +CONFIG_I2C_DIOLAN_U2C=m +CONFIG_I2C_DLN2=m +CONFIG_I2C_TINY_USB=m +CONFIG_MEMSTICK=m +CONFIG_MEMSTICK_REALTEK_USB=m +CONFIG_MFD_DLN2=m +CONFIG_MISC_RTSX_USB=m +CONFIG_MMC_REALTEK_USB=m +CONFIG_MMC_USHC=m +CONFIG_MMC_VUB300=m +CONFIG_MSPRO_BLOCK=m +CONFIG_REALTEK_AUTOPM=y +CONFIG_SENSORS_AQUACOMPUTER_D5NEXT=m +CONFIG_SENSORS_ASUS_ROG_RYUJIN=m +CONFIG_SENSORS_CORSAIR_CPRO=m +CONFIG_SENSORS_CORSAIR_PSU=m +CONFIG_SENSORS_GIGABYTE_WATERFORCE=m +CONFIG_SENSORS_NZXT_KRAKEN2=m +CONFIG_SENSORS_NZXT_KRAKEN3=m +CONFIG_SENSORS_NZXT_SMART2=m +CONFIG_SENSORS_POWERZ=m +CONFIG_SPI_CH341=m +CONFIG_SPI_DLN2=m +CONFIG_USBIP_CORE=m +CONFIG_USBIP_HOST=m +CONFIG_USBIP_VHCI_HCD=m +CONFIG_USBPCWATCHDOG=m +CONFIG_USB_ADUTUX=m +CONFIG_USB_APPLEDISPLAY=m +CONFIG_USB_CHAOSKEY=m +CONFIG_USB_EMI26=m +CONFIG_USB_EMI62=m +CONFIG_USB_EZUSB_FX2=m +CONFIG_USB_IDMOUSE=m +CONFIG_USB_IOWARRIOR=m +CONFIG_USB_ISIGHTFW=m +CONFIG_USB_LCD=m +CONFIG_USB_LD=m +CONFIG_USB_LEGOTOWER=m +CONFIG_USB_MDC800=m +CONFIG_USB_MICROTEK=m +CONFIG_USB_MON=m +CONFIG_USB_PRINTER=m +CONFIG_USB_SERIAL_AIRCABLE=m +CONFIG_USB_SERIAL_ARK3116=m +CONFIG_USB_SERIAL_BELKIN=m +CONFIG_USB_SERIAL_CH341=m +CONFIG_USB_SERIAL_CYBERJACK=m +CONFIG_USB_SERIAL_CYPRESS_M8=m +CONFIG_USB_SERIAL_DIGI_ACCELEPORT=m +CONFIG_USB_SERIAL_EDGEPORT=m +CONFIG_USB_SERIAL_EDGEPORT_TI=m +CONFIG_USB_SERIAL_EMPEG=m +CONFIG_USB_SERIAL_F81232=m +CONFIG_USB_SERIAL_F8153X=m +CONFIG_USB_SERIAL_GARMIN=m +CONFIG_USB_SERIAL_GENERIC=y +CONFIG_USB_SERIAL_IPAQ=m +CONFIG_USB_SERIAL_IPW=m +CONFIG_USB_SERIAL_IR=m +CONFIG_USB_SERIAL_IUU=m +CONFIG_USB_SERIAL_KEYSPAN=m +CONFIG_USB_SERIAL_KEYSPAN_PDA=m +CONFIG_USB_SERIAL_KLSI=m +CONFIG_USB_SERIAL_KOBIL_SCT=m +CONFIG_USB_SERIAL_MCT_U232=m +CONFIG_USB_SERIAL_MOS7720=m +CONFIG_USB_SERIAL_MOS7840=m +CONFIG_USB_SERIAL_NAVMAN=m +CONFIG_USB_SERIAL_OMNINET=m +CONFIG_USB_SERIAL_OPTICON=m +CONFIG_USB_SERIAL_OTI6858=m +CONFIG_USB_SERIAL_PL2303=m +CONFIG_USB_SERIAL_QCAUX=m +CONFIG_USB_SERIAL_QT2=m +CONFIG_USB_SERIAL_QUALCOMM=m +CONFIG_USB_SERIAL_SAFE=m +CONFIG_USB_SERIAL_SAFE_PADDED=y +CONFIG_USB_SERIAL_SIERRAWIRELESS=m +CONFIG_USB_SERIAL_SIMPLE=m +CONFIG_USB_SERIAL_SPCP8X5=m +CONFIG_USB_SERIAL_SSU100=m +CONFIG_USB_SERIAL_SYMBOL=m +CONFIG_USB_SERIAL_TI=m +CONFIG_USB_SERIAL_UPD78F0730=m +CONFIG_USB_SERIAL_VISOR=m +CONFIG_USB_SERIAL_WHITEHEAT=m +CONFIG_USB_SERIAL_XR=m +CONFIG_USB_SERIAL_XSENS_MT=m +CONFIG_USB_SEVSEG=m +CONFIG_USB_SISUSBVGA=m +CONFIG_USB_STORAGE_ALAUDA=m +CONFIG_USB_STORAGE_CYPRESS_ATACB=m +CONFIG_USB_STORAGE_DATAFAB=m +CONFIG_USB_STORAGE_ENE_UB6250=m +CONFIG_USB_STORAGE_FREECOM=m +CONFIG_USB_STORAGE_ISD200=m +CONFIG_USB_STORAGE_JUMPSHOT=m +CONFIG_USB_STORAGE_KARMA=m +CONFIG_USB_STORAGE_ONETOUCH=m +CONFIG_USB_STORAGE_REALTEK=m +CONFIG_USB_STORAGE_SDDR09=m +CONFIG_USB_STORAGE_SDDR55=m +CONFIG_USB_STORAGE_USBAT=m +CONFIG_USB_TMC=m +CONFIG_USB_TRANCEVIBRATOR=m +CONFIG_USB_WDM=m +CONFIG_USB_YUREX=m +CONFIG_W1=m +CONFIG_W1_MASTER_DS2490=m +CONFIG_W1_SLAVE_DS2405=m +CONFIG_W1_SLAVE_DS2406=m +CONFIG_W1_SLAVE_DS2408=m +CONFIG_W1_SLAVE_DS2413=m +CONFIG_W1_SLAVE_DS2423=m +CONFIG_W1_SLAVE_DS2430=m +CONFIG_W1_SLAVE_DS2431=m +CONFIG_W1_SLAVE_DS2433=m +CONFIG_W1_SLAVE_DS2433_CRC=y +CONFIG_W1_SLAVE_DS2438=m +CONFIG_W1_SLAVE_DS2780=m +CONFIG_W1_SLAVE_DS2781=m +CONFIG_W1_SLAVE_DS2805=m +CONFIG_W1_SLAVE_DS28E04=m +CONFIG_W1_SLAVE_SMEM=m +CONFIG_W1_SLAVE_THERM=m +CONFIG_XILLYUSB=m + +# USB network adapters: USB Ethernet/tethering/mobile broadband, USB Wi-Fi dongles, USB CAN adapters +CONFIG_AR5523=m +CONFIG_ATH10K_USB=m +CONFIG_ATH6KL=m +CONFIG_ATH6KL_USB=m +CONFIG_ATH9K_HTC=m +CONFIG_BRCMFMAC_USB=y +CONFIG_CAN_8DEV_USB=m +CONFIG_CAN_EMS_USB=m +CONFIG_CAN_ESD_USB=m +CONFIG_CAN_F81604=m +CONFIG_CAN_GS_USB=m +CONFIG_CAN_ISOTP=m +CONFIG_CAN_J1939=m +CONFIG_CAN_KVASER_USB=m +CONFIG_CAN_MCBA_USB=m +CONFIG_CAN_PEAK_USB=m +CONFIG_CARL9170=m +CONFIG_CARL9170_LEDS=y +CONFIG_LIBERTAS_THINFIRM=m +CONFIG_LIBERTAS_THINFIRM_USB=m +CONFIG_MT7601U=m +CONFIG_MT7663U=m +CONFIG_MT76x0U=m +CONFIG_MT76x2U=m +CONFIG_MT7921U=m +CONFIG_MT7925U=m +CONFIG_RT2500USB=m +CONFIG_RT2800USB=m +CONFIG_RT2800USB_RT33XX=y +CONFIG_RT2800USB_RT3573=y +CONFIG_RT2800USB_RT35XX=y +CONFIG_RT2800USB_RT53XX=y +CONFIG_RT2800USB_RT55XX=y +CONFIG_RT2800USB_UNKNOWN=y +CONFIG_RT2X00=m +CONFIG_RT73USB=m +CONFIG_RTL8187=m +CONFIG_RTL8192DU=m +CONFIG_RTL8XXXU=m +CONFIG_RTW88=m +CONFIG_RTW88_8723DU=m +CONFIG_RTW88_8812AU=m +CONFIG_RTW88_8814AU=m +CONFIG_RTW88_8821AU=m +CONFIG_RTW88_8821CU=m +CONFIG_RTW88_8822BU=m +CONFIG_RTW88_8822CU=m +CONFIG_RTW89=m +CONFIG_RTW89_8851BU=m +CONFIG_RTW89_8852AU=m +CONFIG_RTW89_8852BU=m +CONFIG_RTW89_8852CU=m +CONFIG_USB_ALI_M5632=y +CONFIG_USB_AN2720=y +CONFIG_USB_CATC=m +CONFIG_USB_EPSON2888=y +CONFIG_USB_HSO=m +CONFIG_USB_IPHETH=m +CONFIG_USB_KAWETH=m +CONFIG_USB_KC2190=y +CONFIG_USB_NET_AQC111=m +CONFIG_USB_NET_CDC_EEM=m +CONFIG_USB_NET_CDC_MBIM=m +CONFIG_USB_NET_CH9200=m +CONFIG_USB_NET_CX82310_ETH=m +CONFIG_USB_NET_GL620A=m +CONFIG_USB_NET_HUAWEI_CDC_NCM=m +CONFIG_USB_NET_INT51X1=m +CONFIG_USB_NET_KALMIA=m +CONFIG_USB_NET_QMI_WWAN=m +CONFIG_USB_NET_RNDIS_HOST=m +CONFIG_USB_NET_SR9700=m +CONFIG_USB_SIERRA_NET=m +CONFIG_USB_VL600=m +CONFIG_ZD1211RW=m + +# Storage: optical/SCSI generic, device-mapper (LUKS, LVM, verity, integrity), MD RAID, partition tables +CONFIG_AIX_PARTITION=y +CONFIG_BLK_DEV_SR=m +CONFIG_BSD_DISKLABEL=y +CONFIG_CHR_DEV_SG=m +CONFIG_DM_CACHE=m +CONFIG_DM_CACHE_SMQ=m +CONFIG_DM_CRYPT=m +CONFIG_DM_INTEGRITY=m +CONFIG_DM_RAID=m +CONFIG_DM_SNAPSHOT=m +CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_UEVENT=y +CONFIG_DM_VERITY=m +CONFIG_DM_VERITY_FEC=y +CONFIG_DM_WRITECACHE=m +CONFIG_LDM_PARTITION=y +CONFIG_MAC_PARTITION=y +CONFIG_MD_LINEAR=m +CONFIG_MD_RAID0=m +CONFIG_MD_RAID1=m +CONFIG_MD_RAID10=m +CONFIG_MD_RAID456=m +CONFIG_MINIX_SUBPARTITION=y +CONFIG_OSF_PARTITION=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_SGI_PARTITION=y +CONFIG_SOLARIS_X86_PARTITION=y +CONFIG_SUN_PARTITION=y +CONFIG_UNIXWARE_DISKLABEL=y + +# Filesystems for removable media and images, binfmt_misc, SMB client +CONFIG_BINFMT_MISC=m +CONFIG_CIFS=m +CONFIG_CIFS_ALLOW_INSECURE_LEGACY=y +CONFIG_CIFS_DFS_UPCALL=y +CONFIG_CIFS_POSIX=y +CONFIG_CIFS_SWN_UPCALL=y +CONFIG_CIFS_UPCALL=y +CONFIG_CIFS_XATTR=y +CONFIG_EROFS_FS=m +CONFIG_EROFS_FS_BACKED_BY_FILE=y +CONFIG_EROFS_FS_POSIX_ACL=y +CONFIG_EROFS_FS_SECURITY=y +CONFIG_EROFS_FS_XATTR=y +CONFIG_EROFS_FS_ZIP=y +CONFIG_EROFS_FS_ZIP_ACCEL=y +CONFIG_EROFS_FS_ZIP_DEFLATE=y +CONFIG_EROFS_FS_ZIP_LZMA=y +CONFIG_EROFS_FS_ZIP_ZSTD=y +CONFIG_EXFAT_FS=m +CONFIG_F2FS_FS=m +CONFIG_F2FS_FS_COMPRESSION=y +CONFIG_F2FS_FS_LZ4=y +CONFIG_F2FS_FS_LZ4HC=y +CONFIG_F2FS_FS_LZO=y +CONFIG_F2FS_FS_LZORLE=y +CONFIG_F2FS_FS_POSIX_ACL=y +CONFIG_F2FS_FS_SECURITY=y +CONFIG_F2FS_FS_XATTR=y +CONFIG_F2FS_FS_ZSTD=y +CONFIG_F2FS_IOSTAT=y +CONFIG_F2FS_STAT_FS=y +CONFIG_F2FS_UNFAIR_RWSEM=y +CONFIG_HFSPLUS_FS=m +CONFIG_HFS_FS=m +CONFIG_ISO9660_FS=m +CONFIG_JOLIET=y +CONFIG_MSDOS_FS=m +CONFIG_NLS_ASCII=m +CONFIG_NLS_CODEPAGE_1250=m +CONFIG_NLS_CODEPAGE_1251=m +CONFIG_NLS_CODEPAGE_737=m +CONFIG_NLS_CODEPAGE_775=m +CONFIG_NLS_CODEPAGE_850=m +CONFIG_NLS_CODEPAGE_852=m +CONFIG_NLS_CODEPAGE_855=m +CONFIG_NLS_CODEPAGE_857=m +CONFIG_NLS_CODEPAGE_860=m +CONFIG_NLS_CODEPAGE_861=m +CONFIG_NLS_CODEPAGE_862=m +CONFIG_NLS_CODEPAGE_863=m +CONFIG_NLS_CODEPAGE_864=m +CONFIG_NLS_CODEPAGE_865=m +CONFIG_NLS_CODEPAGE_866=m +CONFIG_NLS_CODEPAGE_869=m +CONFIG_NLS_CODEPAGE_874=m +CONFIG_NLS_CODEPAGE_932=m +CONFIG_NLS_CODEPAGE_936=m +CONFIG_NLS_CODEPAGE_949=m +CONFIG_NLS_CODEPAGE_950=m +CONFIG_NLS_ISO8859_13=m +CONFIG_NLS_ISO8859_14=m +CONFIG_NLS_ISO8859_15=m +CONFIG_NLS_ISO8859_2=m +CONFIG_NLS_ISO8859_3=m +CONFIG_NLS_ISO8859_4=m +CONFIG_NLS_ISO8859_5=m +CONFIG_NLS_ISO8859_6=m +CONFIG_NLS_ISO8859_7=m +CONFIG_NLS_ISO8859_8=m +CONFIG_NLS_ISO8859_9=m +CONFIG_NLS_KOI8_R=m +CONFIG_NLS_KOI8_U=m +CONFIG_NLS_MAC_CELTIC=m +CONFIG_NLS_MAC_CENTEURO=m +CONFIG_NLS_MAC_CROATIAN=m +CONFIG_NLS_MAC_CYRILLIC=m +CONFIG_NLS_MAC_GAELIC=m +CONFIG_NLS_MAC_GREEK=m +CONFIG_NLS_MAC_ICELAND=m +CONFIG_NLS_MAC_INUIT=m +CONFIG_NLS_MAC_ROMAN=m +CONFIG_NLS_MAC_ROMANIAN=m +CONFIG_NLS_MAC_TURKISH=m +CONFIG_NLS_UTF8=m +CONFIG_NTFS3_FS=m +CONFIG_NTFS3_FS_POSIX_ACL=y +CONFIG_NTFS3_LZX_XPRESS=y +CONFIG_SQUASHFS_LZ4=y +CONFIG_SQUASHFS_LZO=y +CONFIG_SQUASHFS_XATTR=y +CONFIG_SQUASHFS_XZ=y +CONFIG_SQUASHFS_ZSTD=y +CONFIG_UDF_FS=m +CONFIG_XFS_FS=m +CONFIG_XFS_ONLINE_REPAIR=y +CONFIG_XFS_ONLINE_SCRUB=y +CONFIG_XFS_ONLINE_SCRUB_STATS=y +CONFIG_XFS_POSIX_ACL=y +CONFIG_XFS_QUOTA=y +CONFIG_XFS_RT=y +CONFIG_XFS_SUPPORT_ASCII_CI=y +CONFIG_XFS_SUPPORT_V4=y +CONFIG_ZISOFS=y + +# Network protocols: WireGuard/OpenVPN-DCO/L2TP/PPP/IPsec/kTLS, tunnels, policy routing, nftables+xtables+ipset+ebtables, tc, TCP CC, ss diag, virtual links +CONFIG_BONDING=m +CONFIG_BRIDGE_EBT_802_3=m +CONFIG_BRIDGE_EBT_AMONG=m +CONFIG_BRIDGE_EBT_ARP=m +CONFIG_BRIDGE_EBT_ARPREPLY=m +CONFIG_BRIDGE_EBT_BROUTE=m +CONFIG_BRIDGE_EBT_DNAT=m +CONFIG_BRIDGE_EBT_IP=m +CONFIG_BRIDGE_EBT_IP6=m +CONFIG_BRIDGE_EBT_LIMIT=m +CONFIG_BRIDGE_EBT_LOG=m +CONFIG_BRIDGE_EBT_MARK=m +CONFIG_BRIDGE_EBT_MARK_T=m +CONFIG_BRIDGE_EBT_NFLOG=m +CONFIG_BRIDGE_EBT_PKTTYPE=m +CONFIG_BRIDGE_EBT_REDIRECT=m +CONFIG_BRIDGE_EBT_SNAT=m +CONFIG_BRIDGE_EBT_STP=m +CONFIG_BRIDGE_EBT_T_FILTER=m +CONFIG_BRIDGE_EBT_T_NAT=m +CONFIG_BRIDGE_EBT_VLAN=m +CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_BRIDGE_NF_EBTABLES_LEGACY=m +CONFIG_CLS_U32_MARK=y +CONFIG_CLS_U32_PERF=y +CONFIG_DEFAULT_CUBIC=y +CONFIG_DUMMY=m +CONFIG_GACT_PROB=y +CONFIG_GENEVE=m +CONFIG_IFB=m +CONFIG_INET6_AH=m +CONFIG_INET6_ESP=m +CONFIG_INET6_ESPINTCP=y +CONFIG_INET6_ESP_OFFLOAD=m +CONFIG_INET6_IPCOMP=m +CONFIG_INET_AH=m +CONFIG_INET_DIAG_DESTROY=y +CONFIG_INET_ESP=m +CONFIG_INET_ESPINTCP=y +CONFIG_INET_ESP_OFFLOAD=m +CONFIG_INET_IPCOMP=m +CONFIG_INET_RAW_DIAG=m +CONFIG_INET_UDP_DIAG=m +CONFIG_IP6_NF_FILTER=m +CONFIG_IP6_NF_IPTABLES_LEGACY=m +CONFIG_IP6_NF_MANGLE=m +CONFIG_IP6_NF_MATCH_AH=m +CONFIG_IP6_NF_MATCH_EUI64=m +CONFIG_IP6_NF_MATCH_FRAG=m +CONFIG_IP6_NF_MATCH_HL=m +CONFIG_IP6_NF_MATCH_IPV6HEADER=m +CONFIG_IP6_NF_MATCH_MH=m +CONFIG_IP6_NF_MATCH_OPTS=m +CONFIG_IP6_NF_MATCH_RPFILTER=m +CONFIG_IP6_NF_MATCH_RT=m +CONFIG_IP6_NF_MATCH_SRH=m +CONFIG_IP6_NF_NAT=m +CONFIG_IP6_NF_RAW=m +CONFIG_IP6_NF_TARGET_HL=m +CONFIG_IP6_NF_TARGET_MASQUERADE=m +CONFIG_IP6_NF_TARGET_NPT=m +CONFIG_IP6_NF_TARGET_REJECT=m +CONFIG_IPV6_GRE=m +CONFIG_IPV6_MULTIPLE_TABLES=y +CONFIG_IPV6_TUNNEL=m +CONFIG_IPV6_VTI=m +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_NF_ARPFILTER=m +CONFIG_IP_NF_ARPTABLES=m +CONFIG_IP_NF_ARP_MANGLE=m +CONFIG_IP_NF_FILTER=m +CONFIG_IP_NF_IPTABLES_LEGACY=m +CONFIG_IP_NF_MANGLE=m +CONFIG_IP_NF_MATCH_AH=m +CONFIG_IP_NF_MATCH_ECN=m +CONFIG_IP_NF_MATCH_RPFILTER=m +CONFIG_IP_NF_MATCH_TTL=m +CONFIG_IP_NF_NAT=m +CONFIG_IP_NF_RAW=m +CONFIG_IP_NF_TARGET_ECN=m +CONFIG_IP_NF_TARGET_MASQUERADE=m +CONFIG_IP_NF_TARGET_NETMAP=m +CONFIG_IP_NF_TARGET_REDIRECT=m +CONFIG_IP_NF_TARGET_REJECT=m +CONFIG_IP_NF_TARGET_TTL=m +CONFIG_IP_ROUTE_MULTIPATH=y +CONFIG_IP_SET=m +CONFIG_IP_SET_BITMAP_IP=m +CONFIG_IP_SET_BITMAP_IPMAC=m +CONFIG_IP_SET_BITMAP_PORT=m +CONFIG_IP_SET_HASH_IP=m +CONFIG_IP_SET_HASH_IPMAC=m +CONFIG_IP_SET_HASH_IPMARK=m +CONFIG_IP_SET_HASH_IPPORT=m +CONFIG_IP_SET_HASH_IPPORTIP=m +CONFIG_IP_SET_HASH_IPPORTNET=m +CONFIG_IP_SET_HASH_MAC=m +CONFIG_IP_SET_HASH_NET=m +CONFIG_IP_SET_HASH_NETIFACE=m +CONFIG_IP_SET_HASH_NETNET=m +CONFIG_IP_SET_HASH_NETPORT=m +CONFIG_IP_SET_HASH_NETPORTNET=m +CONFIG_IP_SET_LIST_SET=m +CONFIG_L2TP=m +CONFIG_L2TP_ETH=m +CONFIG_L2TP_IP=m +CONFIG_L2TP_V3=y +CONFIG_NETFILTER_NETLINK_ACCT=m +CONFIG_NETFILTER_NETLINK_GLUE_CT=y +CONFIG_NETFILTER_NETLINK_HOOK=m +CONFIG_NETFILTER_NETLINK_LOG=m +CONFIG_NETFILTER_NETLINK_OSF=m +CONFIG_NETFILTER_NETLINK_QUEUE=m +CONFIG_NETFILTER_XTABLES_LEGACY=y +CONFIG_NETFILTER_XT_CONNMARK=m +CONFIG_NETFILTER_XT_MATCH_BPF=m +CONFIG_NETFILTER_XT_MATCH_CGROUP=m +CONFIG_NETFILTER_XT_MATCH_CLUSTER=m +CONFIG_NETFILTER_XT_MATCH_COMMENT=m +CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m +CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m +CONFIG_NETFILTER_XT_MATCH_CONNMARK=m +CONFIG_NETFILTER_XT_MATCH_CPU=m +CONFIG_NETFILTER_XT_MATCH_DCCP=m +CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m +CONFIG_NETFILTER_XT_MATCH_DSCP=m +CONFIG_NETFILTER_XT_MATCH_ECN=m +CONFIG_NETFILTER_XT_MATCH_ESP=m +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m +CONFIG_NETFILTER_XT_MATCH_HELPER=m +CONFIG_NETFILTER_XT_MATCH_HL=m +CONFIG_NETFILTER_XT_MATCH_IPCOMP=m +CONFIG_NETFILTER_XT_MATCH_IPRANGE=m +CONFIG_NETFILTER_XT_MATCH_L2TP=m +CONFIG_NETFILTER_XT_MATCH_LENGTH=m +CONFIG_NETFILTER_XT_MATCH_LIMIT=m +CONFIG_NETFILTER_XT_MATCH_MAC=m +CONFIG_NETFILTER_XT_MATCH_MARK=m +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m +CONFIG_NETFILTER_XT_MATCH_NFACCT=m +CONFIG_NETFILTER_XT_MATCH_OSF=m +CONFIG_NETFILTER_XT_MATCH_OWNER=m +CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m +CONFIG_NETFILTER_XT_MATCH_POLICY=m +CONFIG_NETFILTER_XT_MATCH_QUOTA=m +CONFIG_NETFILTER_XT_MATCH_RATEEST=m +CONFIG_NETFILTER_XT_MATCH_REALM=m +CONFIG_NETFILTER_XT_MATCH_RECENT=m +CONFIG_NETFILTER_XT_MATCH_SCTP=m +CONFIG_NETFILTER_XT_MATCH_SOCKET=m +CONFIG_NETFILTER_XT_MATCH_STATE=m +CONFIG_NETFILTER_XT_MATCH_STATISTIC=m +CONFIG_NETFILTER_XT_MATCH_STRING=m +CONFIG_NETFILTER_XT_MATCH_TCPMSS=m +CONFIG_NETFILTER_XT_MATCH_TIME=m +CONFIG_NETFILTER_XT_MATCH_U32=m +CONFIG_NETFILTER_XT_NAT=m +CONFIG_NETFILTER_XT_SET=m +CONFIG_NETFILTER_XT_TARGET_AUDIT=m +CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m +CONFIG_NETFILTER_XT_TARGET_CONNMARK=m +CONFIG_NETFILTER_XT_TARGET_CT=m +CONFIG_NETFILTER_XT_TARGET_DSCP=m +CONFIG_NETFILTER_XT_TARGET_HL=m +CONFIG_NETFILTER_XT_TARGET_HMARK=m +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m +CONFIG_NETFILTER_XT_TARGET_LED=m +CONFIG_NETFILTER_XT_TARGET_MARK=m +CONFIG_NETFILTER_XT_TARGET_MASQUERADE=m +CONFIG_NETFILTER_XT_TARGET_NETMAP=m +CONFIG_NETFILTER_XT_TARGET_NFLOG=m +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m +CONFIG_NETFILTER_XT_TARGET_NOTRACK=m +CONFIG_NETFILTER_XT_TARGET_RATEEST=m +CONFIG_NETFILTER_XT_TARGET_REDIRECT=m +CONFIG_NETFILTER_XT_TARGET_TCPMSS=m +CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m +CONFIG_NETFILTER_XT_TARGET_TEE=m +CONFIG_NETFILTER_XT_TARGET_TPROXY=m +CONFIG_NETFILTER_XT_TARGET_TRACE=m +CONFIG_NETLINK_DIAG=m +CONFIG_NET_ACT_BPF=m +CONFIG_NET_ACT_CONNMARK=m +CONFIG_NET_ACT_CSUM=m +CONFIG_NET_ACT_CT=m +CONFIG_NET_ACT_CTINFO=m +CONFIG_NET_ACT_MPLS=m +CONFIG_NET_ACT_NAT=m +CONFIG_NET_ACT_PEDIT=m +CONFIG_NET_ACT_POLICE=m +CONFIG_NET_ACT_SAMPLE=m +CONFIG_NET_ACT_SIMP=m +CONFIG_NET_ACT_SKBEDIT=m +CONFIG_NET_ACT_SKBMOD=m +CONFIG_NET_ACT_TUNNEL_KEY=m +CONFIG_NET_ACT_VLAN=m +CONFIG_NET_CLS_BPF=m +CONFIG_NET_CLS_FLOW=m +CONFIG_NET_CLS_FW=m +CONFIG_NET_CLS_MATCHALL=m +CONFIG_NET_CLS_ROUTE4=m +CONFIG_NET_CLS_U32=m +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_CMP=m +CONFIG_NET_EMATCH_IPSET=m +CONFIG_NET_EMATCH_IPT=m +CONFIG_NET_EMATCH_META=m +CONFIG_NET_EMATCH_NBYTE=m +CONFIG_NET_EMATCH_TEXT=m +CONFIG_NET_EMATCH_U32=m +CONFIG_NET_FOU=m +CONFIG_NET_FOU_IP_TUNNELS=y +CONFIG_NET_IPGRE=m +CONFIG_NET_IPGRE_BROADCAST=y +CONFIG_NET_IPGRE_DEMUX=m +CONFIG_NET_IPIP=m +CONFIG_NET_IPVTI=m +CONFIG_NET_KEY=m +CONFIG_NET_KEY_MIGRATE=y +CONFIG_NET_SCH_CAKE=m +CONFIG_NET_SCH_CHOKE=m +CONFIG_NET_SCH_CODEL=m +CONFIG_NET_SCH_DRR=m +CONFIG_NET_SCH_DUALPI2=m +CONFIG_NET_SCH_ETS=m +CONFIG_NET_SCH_FQ=m +CONFIG_NET_SCH_FQ_CODEL=m +CONFIG_NET_SCH_FQ_PIE=m +CONFIG_NET_SCH_GRED=m +CONFIG_NET_SCH_HFSC=m +CONFIG_NET_SCH_HHF=m +CONFIG_NET_SCH_HTB=m +CONFIG_NET_SCH_MULTIQ=m +CONFIG_NET_SCH_NETEM=m +CONFIG_NET_SCH_PIE=m +CONFIG_NET_SCH_PLUG=m +CONFIG_NET_SCH_PRIO=m +CONFIG_NET_SCH_QFQ=m +CONFIG_NET_SCH_RED=m +CONFIG_NET_SCH_SFB=m +CONFIG_NET_SCH_SFQ=m +CONFIG_NET_SCH_TBF=m +CONFIG_NET_SCH_TEQL=m +CONFIG_NET_TEAM=m +CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m +CONFIG_NET_TEAM_MODE_BROADCAST=m +CONFIG_NET_TEAM_MODE_LOADBALANCE=m +CONFIG_NET_TEAM_MODE_RANDOM=m +CONFIG_NET_TEAM_MODE_ROUNDROBIN=m +CONFIG_NFT_BRIDGE_META=m +CONFIG_NFT_BRIDGE_REJECT=m +CONFIG_NFT_COMPAT=m +CONFIG_NFT_CONNLIMIT=m +CONFIG_NFT_DUP_IPV4=m +CONFIG_NFT_DUP_IPV6=m +CONFIG_NFT_DUP_NETDEV=m +CONFIG_NFT_FIB_NETDEV=m +CONFIG_NFT_FLOW_OFFLOAD=m +CONFIG_NFT_FWD_NETDEV=m +CONFIG_NFT_HASH=m +CONFIG_NFT_NUMGEN=m +CONFIG_NFT_OSF=m +CONFIG_NFT_QUEUE=m +CONFIG_NFT_QUOTA=m +CONFIG_NFT_REDIR=m +CONFIG_NFT_REJECT_NETDEV=m +CONFIG_NFT_SOCKET=m +CONFIG_NFT_TPROXY=m +CONFIG_NFT_TUNNEL=m +CONFIG_NFT_XFRM=m +CONFIG_NF_CONNTRACK_AMANDA=m +CONFIG_NF_CONNTRACK_BRIDGE=m +CONFIG_NF_CONNTRACK_FTP=m +CONFIG_NF_CONNTRACK_H323=m +CONFIG_NF_CONNTRACK_IRC=m +CONFIG_NF_CONNTRACK_LABELS=y +CONFIG_NF_CONNTRACK_MARK=y +CONFIG_NF_CONNTRACK_NETBIOS_NS=m +CONFIG_NF_CONNTRACK_PPTP=m +CONFIG_NF_CONNTRACK_PROCFS=y +CONFIG_NF_CONNTRACK_SANE=m +CONFIG_NF_CONNTRACK_SIP=m +CONFIG_NF_CONNTRACK_SNMP=m +CONFIG_NF_CONNTRACK_TFTP=m +CONFIG_NF_CONNTRACK_TIMEOUT=y +CONFIG_NF_CONNTRACK_TIMESTAMP=y +CONFIG_NF_CONNTRACK_ZONES=y +CONFIG_NF_CT_NETLINK=m +CONFIG_NF_CT_NETLINK_HELPER=m +CONFIG_NF_CT_NETLINK_TIMEOUT=m +CONFIG_NF_DUP_IPV4=m +CONFIG_NF_DUP_IPV6=m +CONFIG_NF_DUP_NETDEV=m +CONFIG_NF_FLOW_TABLE=m +CONFIG_NF_FLOW_TABLE_INET=m +CONFIG_NF_FLOW_TABLE_PROCFS=y +CONFIG_NF_LOG_ARP=m +CONFIG_NF_LOG_IPV4=m +CONFIG_NF_NAT_SNMP_BASIC=m +CONFIG_NF_SOCKET_IPV4=m +CONFIG_NF_SOCKET_IPV6=m +CONFIG_NF_TABLES_ARP=y +CONFIG_NF_TABLES_BRIDGE=m +CONFIG_NF_TPROXY_IPV4=m +CONFIG_NF_TPROXY_IPV6=m +CONFIG_OVPN=m +CONFIG_PACKET_DIAG=m +CONFIG_PPP=m +CONFIG_PPPOE=m +CONFIG_PPPOE_HASH_BITS_4=y +CONFIG_PPPOL2TP=m +CONFIG_PPP_ASYNC=m +CONFIG_PPP_BSDCOMP=m +CONFIG_PPP_DEFLATE=m +CONFIG_PPP_FILTER=y +CONFIG_PPP_MPPE=m +CONFIG_PPP_MULTILINK=y +CONFIG_PPP_SYNC_TTY=m +CONFIG_PPTP=m +CONFIG_PSAMPLE=m +CONFIG_TCP_CONG_ADVANCED=y +CONFIG_TCP_CONG_BBR=m +CONFIG_TCP_CONG_BIC=m +CONFIG_TCP_CONG_CDG=m +CONFIG_TCP_CONG_DCTCP=m +CONFIG_TCP_CONG_HSTCP=m +CONFIG_TCP_CONG_HTCP=m +CONFIG_TCP_CONG_HYBLA=m +CONFIG_TCP_CONG_ILLINOIS=m +CONFIG_TCP_CONG_LP=m +CONFIG_TCP_CONG_NV=m +CONFIG_TCP_CONG_SCALABLE=m +CONFIG_TCP_CONG_VEGAS=m +CONFIG_TCP_CONG_VENO=m +CONFIG_TCP_CONG_WESTWOOD=m +CONFIG_TCP_CONG_YEAH=m +CONFIG_TLS=m +CONFIG_UNIX_DIAG=m +CONFIG_VXLAN=m +CONFIG_WIREGUARD=m +CONFIG_XFRM_INTERFACE=m +CONFIG_XFRM_MIGRATE=y +CONFIG_XFRM_STATISTICS=y +CONFIG_XFRM_USER=m + +# Bluetooth: USB dongle drivers, vhci, MSFT/AOSP vendor extensions (USB controllers only) +CONFIG_BT_AOSPEXT=y +CONFIG_BT_ATH3K=m +CONFIG_BT_HCIBCM203X=m +CONFIG_BT_HCIBFUSB=m +CONFIG_BT_HCIBPA10X=m +CONFIG_BT_HCIBTUSB_AUTOSUSPEND=y +CONFIG_BT_HCIVHCI=m +CONFIG_BT_MSFTEXT=y + +# Containers / virtualisation: CRIU/checkpoint, userfaultfd, misc cgroup, KVM host vhost-net/vhost-vsock/vsock +CONFIG_CGROUP_MISC=y +CONFIG_CHECKPOINT_RESTORE=y +CONFIG_USERFAULTFD=y +CONFIG_VHOST_NET=m +CONFIG_VHOST_VSOCK=m +CONFIG_VSOCKETS=m +CONFIG_VSOCKETS_DIAG=m +CONFIG_VSOCKETS_LOOPBACK=m + +# Crypto: AF_ALG user API (cryptsetup/iwd), dm-crypt/fscrypt modes, VeraCrypt/TrueCrypt ciphers+hashes, IPsec AEADs +CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_AEGIS128_SIMD=y +CONFIG_CRYPTO_BLAKE2B=m +CONFIG_CRYPTO_BLOWFISH=m +CONFIG_CRYPTO_CAMELLIA=m +CONFIG_CRYPTO_CAST5=m +CONFIG_CRYPTO_CAST6=m +CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_CRC32=m +CONFIG_CRYPTO_CRC32C=m +CONFIG_CRYPTO_CRYPTD=m +CONFIG_CRYPTO_CTS=m +CONFIG_CRYPTO_ESSIV=m +CONFIG_CRYPTO_HCTR2=m +CONFIG_CRYPTO_LRW=m +CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_MD4=m +CONFIG_CRYPTO_NULL=m +CONFIG_CRYPTO_PCBC=m +CONFIG_CRYPTO_RMD160=m +CONFIG_CRYPTO_SEQIV=m +CONFIG_CRYPTO_SERPENT=m +CONFIG_CRYPTO_STREEBOG=m +CONFIG_CRYPTO_TWOFISH=m +CONFIG_CRYPTO_USER_API_AEAD=m +CONFIG_CRYPTO_USER_API_HASH=m +CONFIG_CRYPTO_USER_API_SKCIPHER=m +CONFIG_CRYPTO_WP512=m +CONFIG_CRYPTO_XCBC=m +CONFIG_CRYPTO_XXHASH=m + +# Sound: USB audio/MIDI device drivers, MIDI 2.0, virtual MIDI, dummy card, userspace timers +CONFIG_SND_BCD2000=m +CONFIG_SND_DUMMY=m +CONFIG_SND_SEQ_UMP=y +CONFIG_SND_UMP_LEGACY_RAWMIDI=y +CONFIG_SND_USB_6FIRE=m +CONFIG_SND_USB_AUDIO_MIDI_V2=y +CONFIG_SND_USB_CAIAQ=m +CONFIG_SND_USB_CAIAQ_INPUT=y +CONFIG_SND_USB_HIFACE=m +CONFIG_SND_USB_POD=m +CONFIG_SND_USB_PODHD=m +CONFIG_SND_USB_TONEPORT=m +CONFIG_SND_USB_UA101=m +CONFIG_SND_USB_VARIAX=m +CONFIG_SND_UTIMER=y +CONFIG_SND_VIRMIDI=m + +# Video: non-UVC USB webcams (gspca family, Philips pwc), USB displays (DisplayLink USB2 udl, GUD, GM12U320) +CONFIG_DRM_GM12U320=m +CONFIG_DRM_GUD=m +CONFIG_DRM_UDL=m +CONFIG_USB_GL860=m +CONFIG_USB_GSPCA=m +CONFIG_USB_GSPCA_BENQ=m +CONFIG_USB_GSPCA_CONEX=m +CONFIG_USB_GSPCA_CPIA1=m +CONFIG_USB_GSPCA_DTCS033=m +CONFIG_USB_GSPCA_ETOMS=m +CONFIG_USB_GSPCA_FINEPIX=m +CONFIG_USB_GSPCA_JEILINJ=m +CONFIG_USB_GSPCA_JL2005BCD=m +CONFIG_USB_GSPCA_KINECT=m +CONFIG_USB_GSPCA_KONICA=m +CONFIG_USB_GSPCA_MARS=m +CONFIG_USB_GSPCA_MR97310A=m +CONFIG_USB_GSPCA_NW80X=m +CONFIG_USB_GSPCA_OV519=m +CONFIG_USB_GSPCA_OV534=m +CONFIG_USB_GSPCA_OV534_9=m +CONFIG_USB_GSPCA_PAC207=m +CONFIG_USB_GSPCA_PAC7302=m +CONFIG_USB_GSPCA_PAC7311=m +CONFIG_USB_GSPCA_SE401=m +CONFIG_USB_GSPCA_SN9C2028=m +CONFIG_USB_GSPCA_SN9C20X=m +CONFIG_USB_GSPCA_SONIXB=m +CONFIG_USB_GSPCA_SONIXJ=m +CONFIG_USB_GSPCA_SPCA1528=m +CONFIG_USB_GSPCA_SPCA500=m +CONFIG_USB_GSPCA_SPCA501=m +CONFIG_USB_GSPCA_SPCA505=m +CONFIG_USB_GSPCA_SPCA506=m +CONFIG_USB_GSPCA_SPCA508=m +CONFIG_USB_GSPCA_SPCA561=m +CONFIG_USB_GSPCA_SQ905=m +CONFIG_USB_GSPCA_SQ905C=m +CONFIG_USB_GSPCA_SQ930X=m +CONFIG_USB_GSPCA_STK014=m +CONFIG_USB_GSPCA_STK1135=m +CONFIG_USB_GSPCA_STV0680=m +CONFIG_USB_GSPCA_SUNPLUS=m +CONFIG_USB_GSPCA_T613=m +CONFIG_USB_GSPCA_TOPRO=m +CONFIG_USB_GSPCA_TOUPTEK=m +CONFIG_USB_GSPCA_TV8532=m +CONFIG_USB_GSPCA_VC032X=m +CONFIG_USB_GSPCA_VICAM=m +CONFIG_USB_GSPCA_XIRLINK_CIT=m +CONFIG_USB_GSPCA_ZC3XX=m +CONFIG_USB_M5602=m +CONFIG_USB_PWC=m +CONFIG_USB_PWC_INPUT_EVDEV=y +CONFIG_USB_STV06XX=m + +# Other user-facing: Kerberos persistent keyrings, Speakup screen reader, Wine/Proton ntsync +CONFIG_ACCESSIBILITY=y +CONFIG_NTSYNC=m +CONFIG_PERSISTENT_KEYRINGS=y +CONFIG_SPEAKUP=m +CONFIG_SPEAKUP_SYNTH_ACNTSA=m +CONFIG_SPEAKUP_SYNTH_APOLLO=m +CONFIG_SPEAKUP_SYNTH_AUDPTR=m +CONFIG_SPEAKUP_SYNTH_BNS=m +CONFIG_SPEAKUP_SYNTH_DECTLK=m +CONFIG_SPEAKUP_SYNTH_LTLK=m +CONFIG_SPEAKUP_SYNTH_SOFT=m +CONFIG_SPEAKUP_SYNTH_SPKOUT=m +CONFIG_SPEAKUP_SYNTH_TXPRT=m From e956fbe28e847852d200fa8b36ac7ffbbc9e63fd Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:20:59 -0700 Subject: [PATCH 1336/2680] sep: boot Touch ID and Thunderbolt display without command-line switches The merged Thunderbolt, AVD, and SEP drivers match the kernel running on the M1 Pro and the M1 Air. xART writes, identity-keybag creation, and t8103 Type-C display were still off, so a boot from this branch needed switches that kernel does not use. Default those three on. The first boot creates the identity keybag only when it is absent. Declare the mailbox, runtime PM, and firmware-loader dependencies the SEP driver uses. asahi.config now keeps the SEP module: without CONFIG_TRUSTED_KEYS the fragment's CONFIG_APPLE_SEP=m is dropped, and a TEE trust source would claim the key type first. The same fragment enables the Apple USB4 driver. [ryan: split by topic: only CONFIG_TYPEC=m in asahi.config; the USB4 lines are already added by 8580f66efb64 in this series, the TRUSTED_KEYS lines are in 100-sep and the thunderbolt hunk is in 170-usb] (cherry picked from commit aa3411873bfc4e0d07a85df9709cbdcb6148672b) --- arch/arm64/configs/asahi.config | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index f4848da9a4025..99d6fb52ef9dc 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -53,6 +53,7 @@ CONFIG_HID_DOCKCHANNEL=m CONFIG_SPI_HID_APPLE_OF=m CONFIG_SPI_HID_APPLE_CORE=m CONFIG_USB_DWC3_APPLE=m +CONFIG_TYPEC=m CONFIG_USB4=m CONFIG_USB4_APPLE_SOC=m CONFIG_USB_XHCI_PCI_ASMEDIA=y From eb8771eeb4377b47aeeeb7e738baf196666a2293 Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 13:45:26 -0700 Subject: [PATCH 1337/2680] arm64: configs: asahi: enable the Apple Neural Engine driver Build the in-tree ANE driver (aurora-silicon/linux#155) as modules: ane for the M1 family and ane_t6021 for the M2 family. It needs the DRM accel core, which the release config already enables; name it here too so the fragment matches the config the packages are built with. Signed-off-by: Chris Kearney Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit 70d9fa12bfa4de189b4a47da46856f782b7cc43d) --- arch/arm64/configs/asahi.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 99d6fb52ef9dc..a752832944e83 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -35,6 +35,8 @@ CONFIG_DRM_ASAHI=m CONFIG_DRM_ADP=m CONFIG_DRM_APPLE=m CONFIG_DRM_APPLE_AUDIO=y +CONFIG_DRM_ACCEL=y +CONFIG_DRM_ACCEL_ANE=m CONFIG_DRM_SIMPLEDRM=y CONFIG_SND_SOC_APPLE_AOP_AUDIO=m CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=m From d81c177cde2bb9b782a925d14b5dc09cf0c57b76 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 00:04:05 -0700 Subject: [PATCH 1338/2680] arm64: configs: asahi: build igb for Thunderbolt dock Ethernet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thunderbolt 3 docks commonly carry an Intel I210 for their Ethernet port, as the CalDigit TS3 Plus does. With PCIe over Thunderbolt now coming up on M1, M1 Pro/Max and M2 Pro/Max, the I210 enumerates but had no driver. Build igb as a module; the release config carries the same change. Tested on J314s and J293 with a TS3 Plus: igb binds and the link comes up at 1000 Mb/s. [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 2db5b50997e0556e1c350e3cd8fbf635b3b458d6) --- arch/arm64/configs/asahi.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index a752832944e83..186a4131ff09d 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -15,6 +15,8 @@ CONFIG_PCIE_APPLE=y CONFIG_NVME_APPLE=m CONFIG_BRCMFMAC=m CONFIG_BRCMFMAC_PCIE=y +CONFIG_NET_VENDOR_INTEL=y +CONFIG_IGB=m CONFIG_MT7932_FULLMAC=m CONFIG_TOUCHSCREEN_APPLE_Z2=m CONFIG_INPUT_MACSMC_INPUT=m From 3217540d1e684b4b8fb2ee5c0b05c35756944091 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:04:22 -0700 Subject: [PATCH 1339/2680] PCI: apple: admit J700 radios in the shared kernel with runtime guards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow the built-in bootstrap alongside a modular host and enabled sleep/kexec. Enable only on J700/T8140, install lifetime guards before retaining the arena, and defer probe on transition contention. Restrict link power states to the validated radio link before binding and preserve retained host ownership. [ryan: split by topic: only the MAINTAINERS part; the rest is in 080-radio] [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] (cherry picked from commit baca95f5e1f681151771a24137803a5a700d5707) --- MAINTAINERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 7f1ebd3e608d2..e03ceeeba0da4 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2040,6 +2040,7 @@ F: include/linux/soc/apple/pmp-report.h F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml F: Documentation/devicetree/bindings/display/apple,dcp.yaml F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml +F: Documentation/PCI/controller/apple-t8140.rst F: Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml F: Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml F: Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml @@ -2056,6 +2057,7 @@ F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml F: Documentation/gpu/apple-dcp-backlight.rst F: Documentation/hid/apple-j700-multitouch.rst +F: Documentation/networking/device_drivers/wifi/mt7932-neo.rst F: arch/arm64/boot/dts/apple/t8140* F: arch/arm64/configs/j700*.config F: arch/arm64/include/asm/apple-idle.h @@ -2066,6 +2068,7 @@ F: drivers/rtc/rtc-apple-pmu.c F: drivers/soc/apple/aop.rs F: drivers/soc/apple/mcc.c F: drivers/tty/serial/apple_dockchannel* +F: drivers/pci/controller/pcie-apple-piodma* F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs From 4c927009c7f23388a59d516dd66511dfec79a427 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:04:22 -0700 Subject: [PATCH 1340/2680] Bluetooth: build the gated Neo transport with modular Bluetooth Make the transport tristate, require the retained supplier interlock before DMA publication, and pin activated ownership. Retire software callbacks on unsupported removal without claiming DMA quiescence. Enable the shared radio configuration and document the runtime contracts. [ryan: split by topic: only the PIODMA and PCIEASPM lines of asahi.config; the BT_MTK7932_PCIE and MT7932_FULLMAC lines come with aca3d365aad2 and b2f686e88439 in this series, and the Bluetooth Kconfig change is in 080-radio] (cherry picked from commit 9d496d30819938521476fa451c6ecbd4881ce888) --- arch/arm64/configs/asahi.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 186a4131ff09d..852a2463ad2fd 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -15,6 +15,8 @@ CONFIG_PCIE_APPLE=y CONFIG_NVME_APPLE=m CONFIG_BRCMFMAC=m CONFIG_BRCMFMAC_PCIE=y +CONFIG_PCIE_APPLE_PIODMA_DIAG=y +CONFIG_PCIEASPM=y CONFIG_NET_VENDOR_INTEL=y CONFIG_IGB=m CONFIG_MT7932_FULLMAC=m From 0489e15d1d1f17fe4be09944a59bb2813f871a2d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:39:31 -0700 Subject: [PATCH 1341/2680] docs: apple: clarify Neo guard and recovery contracts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe pre-child lifecycle admission, self-pinned Wi-Fi ownership, permanent PCI refusal errno and malformed-policy recovery. Treat teardown guards as accident protection: privileged raw resets, root-port unbind and ASPM changes remain outside the contract. Use desktop-session wording and explicit Hyprland test options, remove misleading country fallback wording and track the moved public header. Round-2 S6 and updated R1/R2 contracts. Documentation and MAINTAINERS only: non-Neo runtime behavior is unchanged. Round-2 S6. Non-Neo runtime behavior remains unchanged. [ryan: split by topic: only the MAINTAINERS part; the rest is in 080-radio] [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] (cherry picked from commit 31b80654584819451eadf1e391fed18b5832313c) --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index e03ceeeba0da4..44ed05913c305 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2069,6 +2069,7 @@ F: drivers/soc/apple/aop.rs F: drivers/soc/apple/mcc.c F: drivers/tty/serial/apple_dockchannel* F: drivers/pci/controller/pcie-apple-piodma* +F: include/linux/soc/apple/pci-apple-piodma.h F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs From bfa4d492c972c5b499c8a0f3002cb762ee24477d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:30:34 -0700 Subject: [PATCH 1342/2680] arm64: configs: asahi: build igc for Thunderbolt 4 dock Ethernet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thunderbolt 4 docks commonly carry an Intel I225 or I226 for their Ethernet port, as the CalDigit TS4 does (8086:5502, I225-LMvP). These use igc, not igb. With PCIe over Thunderbolt coming up on M1, M1 Pro/Max and M2 Pro/Max, the I225 enumerates behind the dock but had no driver. Build igc as a module; the release config carries the same change. Seen on J293 and J314s with a TS4: the I225 enumerates at 0001:07:00.0 with no driver bound. [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit ce2a8326e0e65c0ff3a0b7ab2a4e28529c1ea23d) --- arch/arm64/configs/asahi.config | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 852a2463ad2fd..b29fe81046dbe 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -19,6 +19,7 @@ CONFIG_PCIE_APPLE_PIODMA_DIAG=y CONFIG_PCIEASPM=y CONFIG_NET_VENDOR_INTEL=y CONFIG_IGB=m +CONFIG_IGC=m CONFIG_MT7932_FULLMAC=m CONFIG_TOUCHSCREEN_APPLE_Z2=m CONFIG_INPUT_MACSMC_INPUT=m From ebc98890b2ee73e3f2850a4400220cde9961b7be Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:39:11 +1100 Subject: [PATCH 1343/2680] drm/apple: operate the external DPTX service over AFKV2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The external T8140 controller announces its port on endpoint 0x2c using version-two frames. Select that transport per endpoint, open announced interfaces and map the single physical DCP unit to logical port zero. Encode bounded command descriptors and validate replies against pending owned buffers before completing them. Preserve callback tokens and limit peer-buffer access to active allocations of the same service. Keep the legacy transport for other display profiles. Add host selftests for frame encoding, malformed records, response correlation, command identifiers and the production owned-copy helper. These pass 905 codec assertions and owned-copy cases with sanitizers; hardware qualification of the operational transport remains pending. [ryan: split by topic: only the MAINTAINERS part; the rest is in 200-dcp] [ryan: adapted to 7.2.10 — reproduces the lines of this commit that survive in aurora-wip 04a63b117, at their final place in the file] (cherry picked from commit d063a2ab3be82f5aca9267725e1b84060314489e) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 44ed05913c305..25eef5fa5e6b2 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2012,6 +2012,7 @@ T: git git://anongit.freedesktop.org/drm/drm-misc F: Documentation/devicetree/bindings/display/apple,dcp-typec-routes.yaml F: drivers/gpu/drm/apple/ F: drivers/gpu/drm/apple/dcp-fabric* +F: tools/testing/selftests/drivers/gpu/apple-afkv2/ APPLE PCIE CONTROLLER DRIVER M: Marc Zyngier From 8c51353990ec48d051f7f48a4cf53304f9c3972c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:34:02 +1100 Subject: [PATCH 1344/2680] phy: apple: implement bounded J700 DisplayPort main-link training After correcting the crossbar route, J700 reaches SET_LINK_RATE but the legacy PHY path waits for the wrong command-ready register. Add an isolated T8140 main-link sequence, firmware drive presets, and cleanup ownership that survives failed setup or shutdown. Preserve the other PMA used for USB. Handle the version-two group-one drive command with complete bounds and range checks before changing the PHY. Cache training values only after a successful PHY operation and retain the legacy endpoint behavior. Hardware register facts were audited with explicit user authorization. The implementation and tests are independently written. The copy audit matched only three hardware-field expressions and standard C test headers. No private implementation patch was applied. Validation: 29 PHY state scenarios, 112 callback boundary cases, 100 group routing cases, 905 AFK assertions and production owned-buffer/AUX tests under ASan/UBSan; schema validation and strict checkpatch. Hardware qualification remains pending. [ryan: split by topic: only the MAINTAINERS line for atc-j700-dp.h; the code is in 170-usb and 200-dcp, and the tools/diagnostics lines are not carried because aurora-wip deleted those tools] (cherry picked from commit 85d81b76238d5a3a4782d0ec443b7274cb1f72e0) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 25eef5fa5e6b2..1f600a15b97be 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2065,6 +2065,7 @@ F: arch/arm64/include/asm/apple-idle.h F: arch/arm64/kernel/idle-apple.S F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* F: drivers/iio/common/aop_sensors/ +F: drivers/phy/apple/atc-j700-dp.h F: drivers/rtc/rtc-apple-pmu.c F: drivers/soc/apple/aop.rs F: drivers/soc/apple/mcc.c From 6b9daca033c7962f3a7b9da04aa96557a8938698 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:00:00 +1100 Subject: [PATCH 1345/2680] selftests: Build the Apple DCP tests and list them in MAINTAINERS The Apple DCP selftests under tools/testing/selftests/drivers/gpu/apple-dcp came with the drm/apple topic, but nothing built them from the top-level selftests Makefile and no MAINTAINERS entry covered them. Add the directory to TARGETS and to the T8140 and J700 entry. [ryan: content of the aurora-wip merge commit f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate"), which no commit carries; its drm/apple part is in 200-dcp] Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + tools/testing/selftests/Makefile | 1 + 2 files changed, 2 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 1f600a15b97be..1cf95d4e21913 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2076,6 +2076,7 @@ F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs F: tools/testing/apple/ +F: tools/testing/selftests/drivers/gpu/apple-dcp/ APPLETALK NETWORK LAYER L: netdev@vger.kernel.org diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index a16dbc8bab8a4..38228eec68ef4 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -19,6 +19,7 @@ TARGETS += devices/probe TARGETS += dmabuf-heaps TARGETS += drivers/apple TARGETS += drivers/dma-buf +TARGETS += drivers/gpu/apple-dcp TARGETS += drivers/ntsync TARGETS += drivers/s390x/uvdevice TARGETS += drivers/net From 6dd237f0169b1171f0f288c47b9531d054620ab1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:00:00 +1100 Subject: [PATCH 1346/2680] arm64: configs: j700: Give the kernel a local version The J700 fragment turns LOCALVERSION_AUTO off, so a kernel built from the fragments alone reports a bare 7.2.10, indistinguishable from an Asahi or mainline build of the same release. The published J700 configuration carried a build-specific value instead ("-aurora-g"), written by the release script. Set CONFIG_LOCALVERSION="-aurora" in the fragment. Release builds keep setting their own value on top. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/configs/j700.config | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/configs/j700.config b/arch/arm64/configs/j700.config index 976fd65b7161c..7e77d55a8fe29 100644 --- a/arch/arm64/configs/j700.config +++ b/arch/arm64/configs/j700.config @@ -111,3 +111,4 @@ CONFIG_PM_ADVANCED_DEBUG=y # Match the tested kernel image: no relative relocation packing, no SCM version suffix # CONFIG_RELR is not set # CONFIG_LOCALVERSION_AUTO is not set +CONFIG_LOCALVERSION="-aurora" From 6141b91a10911d92a1f8701aac37e87a0a6c10b4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:00:00 +1100 Subject: [PATCH 1347/2680] arm64: configs: j700: Note why kernel BTI is off with clang 21 and later Since 7.2, ARM64_BTI_KERNEL depends on CLANG_VERSION < 210000 because of LLVM issue 215547, so the clang 22 J700 build no longer has kernel BTI, which the 7.1 build had. That is an upstream toolchain decision, and the fragment does not try to override it. Say so next to the other hardening choices, so that the missing option in /proc/config.gz is not mistaken for a configuration error. Revisit when upstream lifts the dependency. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/configs/j700.config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/configs/j700.config b/arch/arm64/configs/j700.config index 7e77d55a8fe29..1bcf325d6d943 100644 --- a/arch/arm64/configs/j700.config +++ b/arch/arm64/configs/j700.config @@ -23,6 +23,10 @@ CONFIG_BT_BNEP_PROTO_FILTER=y CONFIG_FW_LOADER_COMPRESS=y CONFIG_FW_LOADER_COMPRESS_ZSTD=y +# Kernel BTI (ARM64_BTI_KERNEL) depends on clang < 21 since 7.2 +# (https://github.com/llvm/llvm-project/issues/215547); with clang 22 it is +# off by upstream decision and is not forced on here. + # 48-bit virtual addresses; KASLR is untested on this machine CONFIG_ARM64_VA_BITS_48=y # CONFIG_RANDOMIZE_BASE is not set From f24e2b82a87db7d9d794313b959b0366285dfb7b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:41:40 +1100 Subject: [PATCH 1348/2680] kunit: Add an Apple firmware parser test configuration Provide an arm64 configuration for the Apple AFK, IOMFB, DockChannel HID, SMC and T8140 AOP parser suites. The drivers can be built into a QEMU test kernel without attaching any Apple hardware. (cherry picked from commit 41906dffd02bc11cbe638051da26578559c9dc83) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../kunit/configs/apple_firmware.config | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 tools/testing/kunit/configs/apple_firmware.config diff --git a/tools/testing/kunit/configs/apple_firmware.config b/tools/testing/kunit/configs/apple_firmware.config new file mode 100644 index 0000000000000..61eeab466b809 --- /dev/null +++ b/tools/testing/kunit/configs/apple_firmware.config @@ -0,0 +1,32 @@ +CONFIG_KUNIT=y +CONFIG_ARCH_APPLE=y +CONFIG_PM=y +CONFIG_APPLE_MAILBOX=y +CONFIG_APPLE_RTKIT=y +CONFIG_DRM=y +CONFIG_BACKLIGHT_CLASS_DEVICE=y +CONFIG_DRM_APPLE=y +CONFIG_DRM_APPLE_PARSER_KUNIT_TEST=y +CONFIG_HID=y +CONFIG_INPUT=y +CONFIG_APPLE_DOCKCHANNEL=y +CONFIG_HID_DOCKCHANNEL=y +CONFIG_HID_DOCKCHANNEL_PARSER_KUNIT_TEST=y +CONFIG_MFD_MACSMC=y +CONFIG_MFD_MACSMC_KUNIT_TEST=y +CONFIG_POWER_SUPPLY=y +CONFIG_MACSMC_POWER=y +CONFIG_MACSMC_POWER_KUNIT_TEST=y +CONFIG_RUST=y +CONFIG_SOUND=y +CONFIG_SND=y +CONFIG_SND_SOC=y +CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_FW_LOADER=y +CONFIG_DMADEVICES=y +CONFIG_DMA_ENGINE=y +CONFIG_PM_GENERIC_DOMAINS=y +CONFIG_RUST_OVERFLOW_CHECKS=y +# CONFIG_LOCALVERSION_AUTO is not set From abfdd6e407b1e3eeb0b2e8f62f7c576dfba0d37c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 01:00:58 +1100 Subject: [PATCH 1349/2680] MAINTAINERS: Cover Apple firmware parser tests Include the SMC test alongside the existing SMC driver in the Apple machine support entry. Add the DockChannel parser test and shared KUnit configuration to the existing Apple Silicon entry so changes to them also reach their driver maintainers. (cherry picked from commit 80085a9a9f2a460106e022d436cf65272d92e1eb) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 1cf95d4e21913..2e95e6fb6da1c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2063,6 +2063,7 @@ F: arch/arm64/boot/dts/apple/t8140* F: arch/arm64/configs/j700*.config F: arch/arm64/include/asm/apple-idle.h F: arch/arm64/kernel/idle-apple.S +F: drivers/hid/dockchannel-hid/dockchannel-hid-parser-test.c F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* F: drivers/iio/common/aop_sensors/ F: drivers/phy/apple/atc-j700-dp.h @@ -2076,6 +2077,7 @@ F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs F: tools/testing/apple/ +F: tools/testing/kunit/configs/apple_firmware.config F: tools/testing/selftests/drivers/gpu/apple-dcp/ APPLETALK NETWORK LAYER @@ -2700,7 +2702,7 @@ F: drivers/iommu/apple-dart-test.c F: drivers/iommu/apple-dart.c F: drivers/iommu/io-pgtable-dart.c F: drivers/irqchip/irq-apple-aic.c -F: drivers/mfd/macsmc.c +F: drivers/mfd/macsmc*.c F: drivers/nvme/host/apple.c F: drivers/nvmem/apple-efuses.c F: drivers/nvmem/apple-spmi-nvmem.c @@ -2709,7 +2711,7 @@ F: drivers/phy/apple/.kunitconfig.apple-atc F: drivers/pinctrl/pinctrl-apple-gpio.c F: drivers/pmdomain/apple/ F: drivers/power/reset/macsmc-reboot.c -F: drivers/power/supply/macsmc-power.c +F: drivers/power/supply/macsmc-power*.c F: drivers/pwm/pwm-apple.c F: drivers/reset/reset-apple-cio.c F: drivers/rtc/rtc-macsmc.c From f5e48ff9a26ee0adcdb273f4294424013f740f47 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 23:15:03 +1100 Subject: [PATCH 1350/2680] tools: aurora: Notify about unsupported front-port displays A display adapter on the J700's USB 2.0 front connector can enumerate a Billboard device while leaving the display dark. Explain the connector's limitation and direct the user to the rear USB-C port. Mark front-branch Billboards with udev and verify the board's fixed hub and controller topology in an unprivileged user service. Send a normal libnotify notification only for an active local graphical session. Keep per-plug delivery state across service restarts and rate-limit attempts without changing display configuration or notification preferences. Keep this board-specific hint in userspace: missing partner altmodes in the firmware-managed Type-C driver do not establish a lack of DP support. Document the kernel/userspace split, public implementation survey, installation, rollback and attended acceptance checks. Add synthetic host tests and a private-session-bus libnotify integration test. (cherry picked from commit 600b095055972ae25308f5f63d29adb31468a6e9) Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 5 + .../70-aurora-display-notifier.rules | 7 + tools/aurora-display-notifier/Makefile | 22 ++ tools/aurora-display-notifier/README.md | 263 +++++++++++++++ .../aurora-display-notifier.service.in | 19 ++ tools/aurora-display-notifier/notifier.py | 236 ++++++++++++++ .../org.aurorasilicon.DisplayHint.desktop | 9 + .../tests/integration_dbus.py | 111 +++++++ .../tests/test_notifier.py | 306 ++++++++++++++++++ 9 files changed, 978 insertions(+) create mode 100644 tools/aurora-display-notifier/70-aurora-display-notifier.rules create mode 100644 tools/aurora-display-notifier/Makefile create mode 100644 tools/aurora-display-notifier/README.md create mode 100644 tools/aurora-display-notifier/aurora-display-notifier.service.in create mode 100644 tools/aurora-display-notifier/notifier.py create mode 100644 tools/aurora-display-notifier/org.aurorasilicon.DisplayHint.desktop create mode 100644 tools/aurora-display-notifier/tests/integration_dbus.py create mode 100644 tools/aurora-display-notifier/tests/test_notifier.py diff --git a/MAINTAINERS b/MAINTAINERS index 2e95e6fb6da1c..697c11216b950 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4419,6 +4419,11 @@ F: kernel/audit* F: lib/*audit.c K: \baudit_[a-z_0-9]\+\b +AURORA DISPLAY NOTIFIER +M: Ryan Murray +S: Maintained +F: tools/aurora-display-notifier/ + AUTOFDO BUILD M: Rong Xu M: Han Shen diff --git a/tools/aurora-display-notifier/70-aurora-display-notifier.rules b/tools/aurora-display-notifier/70-aurora-display-notifier.rules new file mode 100644 index 0000000000000..abb7f8fdb0a05 --- /dev/null +++ b/tools/aurora-display-notifier/70-aurora-display-notifier.rules @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only +# The notifier additionally checks apple,j700 and the fixed hub/controller. +# Never launch a desktop process or guess a user's session bus from udev. +ACTION=="add|change", SUBSYSTEM=="usb", ENV{DEVTYPE}=="usb_device", \ + ATTR{bDeviceClass}=="11", ATTR{devpath}=="1.1|1.1.*", \ + ATTRS{idVendor}=="2109", ATTRS{idProduct}=="2122", \ + ENV{AURORA_DISPLAY_UNSUPPORTED}="1" diff --git a/tools/aurora-display-notifier/Makefile b/tools/aurora-display-notifier/Makefile new file mode 100644 index 0000000000000..7ce15c76936c6 --- /dev/null +++ b/tools/aurora-display-notifier/Makefile @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: GPL-2.0-only +PREFIX ?= /usr +LIBEXECDIR ?= $(PREFIX)/libexec +UDEVDIR ?= $(PREFIX)/lib/udev +USERUNITDIR ?= $(PREFIX)/lib/systemd/user +DATADIR ?= $(PREFIX)/share +PYTHON ?= python3 + +.PHONY: test install +test: + PYTHONDONTWRITEBYTECODE=1 $(PYTHON) -m unittest discover -s tests -v + +install: + install -Dm755 notifier.py $(DESTDIR)$(LIBEXECDIR)/aurora-display-notifier + install -Dm644 70-aurora-display-notifier.rules \ + $(DESTDIR)$(UDEVDIR)/rules.d/70-aurora-display-notifier.rules + install -d $(DESTDIR)$(USERUNITDIR) + sed 's|@LIBEXECDIR@|$(LIBEXECDIR)|g' aurora-display-notifier.service.in \ + > $(DESTDIR)$(USERUNITDIR)/aurora-display-notifier.service + chmod 644 $(DESTDIR)$(USERUNITDIR)/aurora-display-notifier.service + install -Dm644 org.aurorasilicon.DisplayHint.desktop \ + $(DESTDIR)$(DATADIR)/applications/org.aurorasilicon.DisplayHint.desktop diff --git a/tools/aurora-display-notifier/README.md b/tools/aurora-display-notifier/README.md new file mode 100644 index 0000000000000..624d5362ffadb --- /dev/null +++ b/tools/aurora-display-notifier/README.md @@ -0,0 +1,263 @@ + +# J700 front USB-C display hint + +The MacBook Neo (J700) front USB-C connector carries USB 2.0 and power, +but has no DisplayPort lanes. A display adapter can enumerate a USB +Billboard device when its alternate mode cannot be entered. This tool +shows an informational desktop notification: + +> This port doesn't support displays. +> To connect a display, use the rear USB-C port. + +It changes no display, USB authorization, power, notification, or saved +device settings. The desktop's notification preferences continue to apply. + +## Recommended split and upstream prospects + +Ship the hint in userspace. The udev rule only marks device-class `0x11` +Billboards on the fixed hub's front branch. An unprivileged user service +verifies the J700 identity and controller topology, then calls libnotify's +`notify-send` through its inherited session bus. There is no root-to-user +bus discovery, udev `RUN`, system service, or desktop-specific extension. +This works with GNOME Shell and with a notification server such as mako. + +No kernel change is included. At the `a837bd5f9a21160256580aeb0829f6930440950c` +base, the public [J700 DT](https://github.com/aurora-silicon/linux/blob/a837bd5f9a21160256580aeb0829f6930440950c/arch/arm64/boot/dts/apple/t8140-j700.dts) +already describes the fixed VIA `2109:2122` hub and its connector graph: +USB downstream path `1.1` is front, `1.2` is rear. Only rear has the +SuperSpeed/DP PHY connection. The controller is `40a280000.usb`. USB bus +numbers, device addresses, and Linux `port0`/`port1` registration order +are deliberately not used to identify the connector. DT source labels +such as `typec1` are also independent of Linux Type-C class numbering. + +The present firmware-managed +[tipd driver](https://github.com/aurora-silicon/linux/blob/a837bd5f9a21160256580aeb0829f6930440950c/drivers/usb/typec/tipd/core.c) +does not register discovered partner altmodes. Absence of those objects +therefore cannot establish whether a port or partner supports DP. The +notifier uses the board's explicit wiring, rather than inferring a missing +capability from missing sysfs objects. + +A generic kernel improvement to expose firmware-discovered partner +capabilities could be upstream-plausible, separately: obtain actual +Discover SVIDs **and Discover Modes VDOs**, register/unregister partner +altmodes on connection changes, and report active state while preserving +firmware ownership of mode entry. A SVID alone is insufficient to invent +a mode descriptor. This needs public controller documentation and tests +for the specific controller variant, discovery failure, disconnect races, +and suspend. It is unnecessary for this hint. A `dev_info` on every PD +connection cannot distinguish a charger from a display adapter; a generic +tipd message naming J700's front/rear ports would mix board UI into a +controller driver. Neither is recommended here. + +Upstream placement for a generic notifier would be a desktop/session +component backed by reliable connector capability and USB-to-Type-C +association data. This implementation is a small Aurora userspace feature +staged under `tools/`, not a proposed Linux kernel interface. Its J700 +mapping belongs in a board support package until a generic interface can +replace it. Upstream acceptance has not been requested or established. + +## Existing public implementations checked + +Source review on 2026-10-16 was limited to public implementation files; +upstream agent/AI instruction files were not read. No implementation was +copied from proprietary sources. + +* [Asahi tipd at 94fb23346d52](https://github.com/AsahiLinux/linux/blob/94fb23346d522edf53722357c426a3e58030beea/drivers/usb/typec/tipd/core.c) + registers port DP/TBT altmodes and partner identity, but has no + `typec_partner_register_altmode` call or Billboard desktop hint in that + driver. This is a bounded driver inspection, not a claim about every + Asahi package. +* [Linux Type-C class API](https://docs.kernel.org/driver-api/usb/typec.html) + represents partner and port modes separately. Exposing real discovered + modes is a driver concern; displaying a desktop notification is a + session concern. The [upstream class implementation at 90201351a862](https://github.com/torvalds/linux/blob/90201351a862182bdabfc1ad1c97b1e1b6ed5931/drivers/usb/typec/class.c) + recognizes the PD identity's Billboard capability as USB 2.0 capability; + that handling does not send a desktop hint. +* [GNOME settings daemon at 4a75955f7481](https://github.com/GNOME/gnome-settings-daemon/tree/4a75955f7481e2bb19fa73fa67456e83d67e8563/plugins) + had no Billboard/alternate-mode handler in the plugin C/header files + searched. Its [USB protection notifications](https://github.com/GNOME/gnome-settings-daemon/blob/4a75955f7481e2bb19fa73fa67456e83d67e8563/plugins/usb-protection/gsd-usb-protection-manager.c) + concern USBGuard authorization, not a connector's DP capability. + This feature does not change those policies. +* [fwupd Analogix at 50181eeefa62](https://github.com/fwupd/fwupd/blob/50181eeefa622a68b123c2994dd4385fe1430765/plugins/analogix/README.md) + uses selected Billboard devices for firmware updates. Its plugin is + not a generic wrong-port display notifier. This tool never claims USB + interfaces, reads BOS through libusb, or sends USB control requests. +* [systemd hwdb at f9d375fc3daf](https://github.com/systemd/systemd/blob/f9d375fc3daff274f7113d757e0cba59e7a332d4/hwdb.d/20-usb-classes.hwdb) + names class `11` as Billboard. The standard rules import that descriptive + metadata; they do not attach a front/rear display hint to it. The + [udev contract](https://github.com/systemd/systemd/blob/f9d375fc3daff274f7113d757e0cba59e7a332d4/man/udev.xml) + also excludes long-running desktop processes from rules. + +## Matching and delivery + +The rule matches `usb_device`, `bDeviceClass=11`, downstream device paths +`1.1` or `1.1.*`, and a common ancestor with USB ID `2109:2122`. The two +`ATTRS` matches refer to the same ancestor. The user process additionally +requires the exact `apple,j700` compatible token, a fixed hub at downstream +path `1`, the direct front child at `1.1`, and controller `40a280000.usb`. +Other machines and rear-port Billboards fail these checks. The rule's +property is a candidate marker, not an authorization or a generic DP ABI. + +The service subscribes to processed udev events before enumerating, and +reconciles every two seconds. It also handles an adapter already attached +at login. All Billboard functions on the same front device instance share +one notification. Disappearance of just the Billboard function does not +reset deduplication while its parent dock stays attached. + +State in `$XDG_RUNTIME_DIR/aurora-display-notifier/` records successful +delivery per USB instance (sysfs path, address, inode, udev initialization +timestamp), plus the last attempt time. It survives service restarts during +the login, contains no peripheral names/serials, and expires with the user +runtime directory. A lock prevents multiple instances for the same user. +Delivery attempts are at least 30 seconds apart. A rapid replug or a +temporarily missing notification server is retried while still attached; +unplugged pending events are discarded. There is a small unavoidable +crash window between notification acceptance and saving success. + +Before sending, logind must report an active local graphical session +owned by the service's UID and attached to a seat. SSH, TTY, inactive +sessions, and logind failures cannot trigger delivery. If the service is +started before the desktop, it waits until that condition holds. It never +selects another user's bus. The user manager/session bus is per UID; +simultaneous graphical logins of the *same UID* share that bus and its +notification server, as they do for other user services. + +Notifications have normal urgency and the server's default expiry. The +[standard notification protocol](https://specifications.freedesktop.org/notification/latest/protocol.html) +is used without actions or settings changes. Users can disable or mask +the service, or use GNOME/mako notification preferences normally. + +Billboard means an alternate-mode failure, not proof of a connected DP +display. Thus the body is conditional on connecting a display. An adapter +advertising another failed alternate mode can also receive this truthful +port-capability hint. A display adapter that never enumerates a Billboard, +or only uses an interface-class Billboard with another device class, is +not detected. USB graphics such as DisplayLink are outside this feature. + +## Install and enable + +Runtime requirements: Python 3.9+, pyudev, libudev/udev, systemd with +logind and a user manager, libnotify's `notify-send`, and a session +notification server. For Fedora these are typically `python3-pyudev` and +`libnotify`; for Arch/Omarchy, `python-pyudev` and `libnotify` (plus mako or +another notification server). The staging Makefile expects GNU install. + +Packagers can stage all files without running hooks: + +```sh +make DESTDIR="$pkgdir" install +``` + +`PREFIX`, `LIBEXECDIR`, `UDEVDIR`, `USERUNITDIR`, and `DATADIR` are +overridable. No service is enabled by the Makefile. After an authorized +installation, reload udev rules with `sudo udevadm control --reload-rules` +and replug the adapter. Existing devices need a new processed add/change +event to obtain the marker; a rules reload alone does not retroactively +mark them. Avoid a global USB trigger. + +For the desktop user: + +```sh +systemctl --user daemon-reload +systemctl --user enable --now aurora-display-notifier.service +``` + +GNOME and [UWSM-managed Hyprland](https://wiki.hypr.land/Useful-Utilities/Systemd-start/) +supply `graphical-session.target`. A Hyprland session that does not manage +that target can execute the following commands from its startup and +shutdown hooks, respectively (use the syntax for the installed compositor +version): + +```sh +systemctl --user start aurora-display-notifier.service +systemctl --user stop aurora-display-notifier.service +``` + +The user service still requires its inherited session bus and logind's +local graphical session. It requires neither DISPLAY/WAYLAND_DISPLAY +imports nor any guessed bus address. +Launching a compositor inside a logind session still classified as `tty` +does not satisfy the graphical-session check; use a properly registered +graphical login for delivery. + +## Host verification + +```sh +make test +dbus-run-session -- env AURORA_TEST_PRIVATE_BUS=1 PYTHONDONTWRITEBYTECODE=1 \ + python3 tests/integration_dbus.py +``` + +The unit tests use synthetic sysfs and udev objects. The second test needs +PyGObject/Gio and `notify-send`; it sends through a fresh private bus to a +mock notification server, never the desktop's real notification server. +It verifies the libnotify message, default expiration, and persistent +deduplication. Neither command requires J700 access or USB writes. + +Validate a staged install on Linux with `udevadm verify` on the rules, +`systemd-analyze --user verify` on the generated service (with its staged +ExecStart path substituted for validation), and `desktop-file-validate`. +There is no kernel/DT change, so kernel builds, KUnit and dtbs_check do +not validate this userspace-only addition. + +## Attended on-Neo acceptance protocol (not executed here) + +The integration operator must obtain controller policy approval and a +session/resource lease explicitly identifying the J700 before any target +inspection, install, service start, or USB replug. Use that controller +session for every target command. Agree the plug sequence with the person +at the Neo. Do not infer that this document grants hardware access. + +1. Record the candidate commit, installed paths and package version. Save + the existing enable/mask/active status of this service and any previous + versions of these four installed files for rollback. Confirm a J700 + identity, a local GNOME or Hyprland graphical login, logind `Active=yes`, + `Remote=no`, matching `User`, and `Type=wayland` or `x11`. Confirm a + notification server owns `org.freedesktop.Notifications` on that user's + session bus. Preserve existing notification preferences; if Do Not + Disturb hides banners, record that and check the notification history. +2. Install this userspace candidate and dependencies with the approved + integration method, reload only these udev rules, then enable/start + the user unit as above. Record `systemctl --user status + aurora-display-notifier.service`. Do not change kernel/DT or display + settings. Start `udevadm monitor --udev --property --subsystem-match=usb` + and `journalctl --user -fu aurora-display-notifier.service` in the + approved session, saving their output as private qualification evidence. +3. With the dock/display disconnected, plug it into **front**. Within a + few seconds of Billboard enumeration, expect exactly one hint with the + text above. Check the Billboard device's `bDeviceClass` is `11` and + `udevadm info --query=property --path=` contains + `AURORA_DISPLAY_UNSUPPORTED=1`. Verify ancestry is the fixed hub's + `1.1` branch under `40a280000.usb`. Do not assume bus number `1` or + Billboard address `1-1.1.5` stays constant. Leave it connected for at + least 35 seconds: no second hint. Restart the user service: no duplicate. +4. Unplug/replug the whole dock in **front** to establish a fresh hint + (the previous step waited over 30 seconds). Immediately unplug/replug + again and keep it connected: at most one new hint, delayed until the + 30-second limit expires. Repeat the rapid unplug/replug cycle, but + remove the dock during that delay: no stale hint after removal. Wait + over 30 seconds from the last hint, then replug and keep it attached: + one new hint. +5. Unplug and connect the same dock/display to **rear**. Expect no new + notification from this service, even if a Billboard enumerates because + of another failure. Verify rear USB 2.0 ancestry uses `1.2`, and check + the external display works with the existing driver. A front-port hint + already in desktop history is not a new rear-port event. +6. Test a plain USB data device and a charger in front: no hint. With no + local active graphical session, test front Billboard enumeration: no + delivery to SSH/TTY or an inactive user's session. On returning to the + local graphical session, a still-connected eligible dock may produce + the pending hint. Confirm user preferences and display configuration + were not changed. Repeat steps 3 and 5 in GNOME and Hyprland/mako. +7. Save pass/fail evidence including failed attempts and the final user + unit journal. Raw USB/session metadata remains private. Stop/disable + the service if it was enabled only for qualification, restore its + original enable/mask/active state and the previous files (or remove + just the four newly installed files), reload rules and user units. + Restart the restored service only if it was previously running. No + kernel rollback is needed. Runtime state can be removed after stopping + all instances if the operator wants to reset the hint for another test. + +Acceptance requires both the visible front-port hint and absence of new +rear-port hints in each desktop. Host tests alone do not prove those +attended results or validate the physical topology on the running DT. diff --git a/tools/aurora-display-notifier/aurora-display-notifier.service.in b/tools/aurora-display-notifier/aurora-display-notifier.service.in new file mode 100644 index 0000000000000..d167a18cb5d79 --- /dev/null +++ b/tools/aurora-display-notifier/aurora-display-notifier.service.in @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0-only +[Unit] +Description=Explain USB-C display limitations +PartOf=graphical-session.target +After=graphical-session.target +ConditionPathExists=/sys/firmware/devicetree/base/compatible + +[Service] +ExecStart=@LIBEXECDIR@/aurora-display-notifier +Restart=on-failure +RestartSec=10 +RuntimeDirectory=aurora-display-notifier +RuntimeDirectoryMode=0700 +RuntimeDirectoryPreserve=yes +UMask=0077 +NoNewPrivileges=yes + +[Install] +WantedBy=graphical-session.target diff --git a/tools/aurora-display-notifier/notifier.py b/tools/aurora-display-notifier/notifier.py new file mode 100644 index 0000000000000..f2358073ba001 --- /dev/null +++ b/tools/aurora-display-notifier/notifier.py @@ -0,0 +1,236 @@ +#!/usr/bin/python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Explain the J700 front USB-C port's display limitation in the user's session.""" + +import fcntl +import json +import logging +import os +from pathlib import Path +import subprocess +import time + + +SUMMARY = "This port doesn't support displays." +BODY = "To connect a display, use the rear USB-C port." +MARKER = "AURORA_DISPLAY_UNSUPPORTED" +INTERVAL = 30.0 + + +def read_attr(path, name): + try: + return (path / name).read_text(encoding="ascii").strip() + except (OSError, UnicodeError): + return "" + + +def is_j700(sysfs): + try: + compatible = (sysfs / "firmware/devicetree/base/compatible").read_bytes() + except OSError: + return False + return b"apple,j700" in compatible.split(b"\0") + + +def front_device(path, sysfs): + """Match the fixed hub's front branch, independently of USB bus numbers.""" + path = path.resolve() + if not path.is_relative_to((sysfs / "devices").resolve()): + return None + for front in (path, *path.parents): + if read_attr(front, "devpath") != "1.1": + continue + hub = front.parent + if (read_attr(hub, "devpath") == "1" + and read_attr(hub, "idVendor") == "2109" + and read_attr(hub, "idProduct") == "2122" + and any(p.name == "40a280000.usb" for p in hub.parents)): + return front + return None + + +def candidate(device, sysfs): + if (device.device_type != "usb_device" + or device.properties.get(MARKER) != "1"): + return None + path = Path(device.sys_path) + if read_attr(path, "bDeviceClass") != "11": + return None + return front_device(path, sysfs) + + +def plug_id(front, initialized): + """A new USB device instance on the front branch starts a new plug.""" + number = read_attr(front, "devnum") + try: + inode = front.stat().st_ino + except OSError: + return None + if not number.isdecimal(): + return None + return f"{front}:{number}:{inode}:{initialized}" + + +def command(args): + return subprocess.run(args, check=True, capture_output=True, text=True, timeout=5).stdout + + +def active_desktop(uid): + """The user manager may outlive the local graphical login (lingering/SSH).""" + try: + sessions = command(["loginctl", "show-user", str(uid), "-p", "Sessions", "--value"]) + for session in sessions.split(): + data = command([ + "loginctl", "show-session", session, "-p", "User", "-p", "Active", + "-p", "Remote", "-p", "Type", "-p", "Seat", + ]) + props = dict(line.split("=", 1) for line in data.splitlines() if "=" in line) + if (props.get("User") == str(uid) and props.get("Active") == "yes" + and props.get("Remote") == "no" and props.get("Seat") + and props.get("Type") in ("wayland", "x11")): + return True + except (OSError, subprocess.SubprocessError): + pass + return False + + +def notify(): + try: + command([ + "notify-send", "--app-name=Aurora Display Hint", "--urgency=normal", + "--icon=video-display-symbolic", + "--hint=string:desktop-entry:org.aurorasilicon.DisplayHint", + "--", SUMMARY, BODY, + ]) + except (OSError, subprocess.SubprocessError): + logging.warning("Could not deliver the display hint; retrying in 30 seconds") + return False + logging.info("Sent the front USB-C display hint") + return True + + +class Notifier: + def __init__(self, state_path, send=notify, active=None, clock=time.monotonic): + self.path = state_path + self.send = send + self.active = active if active is not None else lambda: active_desktop(os.getuid()) + self.clock = clock + self.shown = set() + self.last_attempt = None + try: + data = json.loads(self.path.read_text(encoding="utf-8")) + shown = data["shown"] + stamp = data["last_attempt"] + if (isinstance(shown, list) and all(isinstance(x, str) for x in shown) + and (stamp is None or isinstance(stamp, (int, float)))): + self.shown = set(shown) + self.last_attempt = stamp + except (OSError, ValueError, KeyError, TypeError): + pass + + def save(self): + temporary = self.path.with_suffix(".tmp") + temporary.write_text(json.dumps({ + "shown": sorted(self.shown), "last_attempt": self.last_attempt, + }) + "\n", encoding="utf-8") + temporary.replace(self.path) + + def reconcile(self, plugs): + # Keep successful delivery across restarts, but forget disconnected plugs. + current = self.shown & plugs + if current != self.shown: + self.shown = current + self.save() + if not plugs - self.shown: + return + now = self.clock() + if self.last_attempt is not None and now - self.last_attempt < INTERVAL: + return + if not self.active(): + return + self.last_attempt = self.clock() + self.save() + if self.send(): + self.shown.update(plugs) + self.save() + + +def lookup_device(context, path): + import pyudev + + try: + return pyudev.Devices.from_sys_path(context, path) + except pyudev.DeviceNotFoundError: + return None + + +def collect(context, sysfs, device_from_path): + # Live front devices retain deduplication even if the Billboard function + # disappears and reappears while the dock stays connected. + live = set() + eligible = set() + for device in context.list_devices(subsystem="usb", DEVTYPE="usb_device"): + path = Path(device.sys_path) + front = front_device(path, sysfs) + if front is None: + continue + try: + parent = device_from_path(context, str(front)) + if parent is None: + continue + initialized = parent.properties.get("USEC_INITIALIZED", "") + if not initialized: + continue + identity = plug_id(front, initialized) + except (OSError, ValueError): + continue + if identity is None: + continue + live.add(identity) + if candidate(device, sysfs) is not None: + eligible.add(identity) + return live, eligible + + +def run(context, monitor, sysfs, state, device_from_path): + # Subscribe before enumeration so a plug during startup cannot be lost. + monitor.start() + while True: + live, eligible = collect(context, sysfs, device_from_path) + # Previously shown plugs remain tracked after their Billboard disappears. + state.reconcile(eligible | (live & state.shown)) + # Periodic reconciliation also handles login, failed delivery and a + # dropped uevent. Bound work during a burst of USB enumeration events. + deadline = time.monotonic() + 2 + while time.monotonic() < deadline: + monitor.poll(timeout=max(0, deadline - time.monotonic())) + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + sysfs = Path("/sys") + if not is_j700(sysfs): + return + if os.getuid() == 0: + raise SystemExit("Run this notifier in a regular user's graphical session") + if not os.environ.get("DBUS_SESSION_BUS_ADDRESS"): + raise SystemExit("The user session bus is unavailable") + runtime = Path(os.environ["XDG_RUNTIME_DIR"]) / "aurora-display-notifier" + runtime.mkdir(mode=0o700, exist_ok=True) + # One user manager, one session bus, one notifier, including manual starts. + with (runtime / "lock").open("w", encoding="ascii") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return + import pyudev + + context = pyudev.Context() + monitor = pyudev.Monitor.from_netlink(context, source="udev") + monitor.filter_by(subsystem="usb", device_type="usb_device") + state = Notifier(runtime / "state.json") + run(context, monitor, sysfs, state, lookup_device) + + +if __name__ == "__main__": + main() diff --git a/tools/aurora-display-notifier/org.aurorasilicon.DisplayHint.desktop b/tools/aurora-display-notifier/org.aurorasilicon.DisplayHint.desktop new file mode 100644 index 0000000000000..4197a66afdb24 --- /dev/null +++ b/tools/aurora-display-notifier/org.aurorasilicon.DisplayHint.desktop @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +[Desktop Entry] +Type=Application +Name=Aurora Display Hint +Comment=Explain the front USB-C port's display limitation +Exec=systemctl --user start aurora-display-notifier.service +Icon=video-display-symbolic +NoDisplay=true +X-GNOME-UsesNotifications=true diff --git a/tools/aurora-display-notifier/tests/integration_dbus.py b/tools/aurora-display-notifier/tests/integration_dbus.py new file mode 100644 index 0000000000000..8a7d9020a0410 --- /dev/null +++ b/tools/aurora-display-notifier/tests/integration_dbus.py @@ -0,0 +1,111 @@ +#!/usr/bin/python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise notify-send against a mock server in a PRIVATE dbus-run-session.""" +import os +from pathlib import Path +import threading + +from gi.repository import Gio, GLib + +from test_notifier import Topology, notifier + + +XML = """ + + + + + + + + + + + + + + +""" + + +def main(): + if os.environ.get("AURORA_TEST_PRIVATE_BUS") != "1": + raise SystemExit("Use dbus-run-session with AURORA_TEST_PRIVATE_BUS=1") + bus = Gio.bus_get_sync(Gio.BusType.SESSION, None) + result = bus.call_sync( + "org.freedesktop.DBus", "/org/freedesktop/DBus", "org.freedesktop.DBus", + "RequestName", GLib.Variant("(su)", ("org.freedesktop.Notifications", 4)), + GLib.VariantType.new("(u)"), Gio.DBusCallFlags.NONE, 5000, None, + ) + assert result.unpack() == (1,), "The private test bus must have no notification server" + calls = [] + + def method(connection, sender, path, interface, name, params, invocation): + if name == "GetCapabilities": + reply = GLib.Variant("(as)", (["body"],)) + elif name == "GetServerInformation": + reply = GLib.Variant("(ssss)", ("Test server", "Aurora", "1", "1.2")) + else: + assert name == "Notify" + calls.append(params.unpack()) + reply = GLib.Variant("(u)", (42,)) + invocation.return_value(reply) + + info = Gio.DBusNodeInfo.new_for_xml(XML).interfaces[0] + registration = bus.register_object("/org/freedesktop/Notifications", info, method, None, None) + loop = GLib.MainLoop() + errors = [] + + def exercise(): + fixture = Topology() + fixture.setUp() + try: + assert notifier.is_j700(fixture.sysfs) + events = [fixture.event(fixture.front, ""), fixture.event(fixture.billboard)] + context = type("Context", (), {"list_devices": lambda *args, **kwargs: events})() + parent = type("Device", (), {"properties": {"USEC_INITIALIZED": "1000"}})() + live, eligible = notifier.collect(context, fixture.sysfs, lambda *args: parent) + state = notifier.Notifier(Path(fixture.tmp.name) / "state.json", active=lambda: True) + state.reconcile(eligible) + state.reconcile(eligible) + # A service restart must retain deduplication. + restarted = notifier.Notifier(state.path, active=lambda: True) + restarted.reconcile(eligible) + # Remove the Billboard function; the dock's instance remains live. + events.pop() + live, eligible = notifier.collect(context, fixture.sysfs, lambda *args: parent) + restarted.reconcile(eligible | (live & restarted.shown)) + assert restarted.shown == state.shown + assert len(calls) == 1, calls + app, replacement, icon, summary, body, actions, hints, expiry = calls[0] + assert app == "Aurora Display Hint" + assert replacement == 0 + # libnotify versions may send the icon in app_icon or image-path. + assert (icon == "video-display-symbolic" + or hints.get("image-path") == "video-display-symbolic"), calls[0] + assert (summary, body) == (notifier.SUMMARY, notifier.BODY) + assert actions == [] + assert hints["urgency"] == 1 + assert hints["desktop-entry"] == "org.aurorasilicon.DisplayHint" + assert expiry == -1 # Respect the notification server's expiration policy. + except BaseException as error: + errors.append(error) + finally: + fixture.doCleanups() + GLib.idle_add(loop.quit) + + worker = threading.Thread(target=exercise, daemon=True) + worker.start() + timeout = GLib.timeout_add_seconds(15, loop.quit) + loop.run() + GLib.source_remove(timeout) + worker.join(timeout=1) + bus.unregister_object(registration) + assert not worker.is_alive(), "Timed out waiting for the notifier" + if errors: + raise errors[0] + print("PASS: synthetic front plug -> libnotify -> private D-Bus server; one notification") + + +if __name__ == "__main__": + main() diff --git a/tools/aurora-display-notifier/tests/test_notifier.py b/tools/aurora-display-notifier/tests/test_notifier.py new file mode 100644 index 0000000000000..e59d2d2b61b42 --- /dev/null +++ b/tools/aurora-display-notifier/tests/test_notifier.py @@ -0,0 +1,306 @@ +# SPDX-License-Identifier: GPL-2.0-only +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + + +spec = importlib.util.spec_from_file_location("notifier", Path(__file__).parents[1] / "notifier.py") +notifier = importlib.util.module_from_spec(spec) +spec.loader.exec_module(notifier) + + +class Topology(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.sysfs = Path(self.tmp.name).resolve() + compatible = self.sysfs / "firmware/devicetree/base/compatible" + compatible.parent.mkdir(parents=True) + compatible.write_bytes(b"apple,j700\0apple,t8140\0apple,arm-platform\0") + # Bus 7 deliberately differs from the observed bus 1. + root = self.sysfs / "devices/platform/soc/40a280000.usb/xhci-hcd.9.auto/usb7" + self.hub = self.device(root / "7-1", devpath="1", idVendor="2109", idProduct="2122") + self.front = self.device(self.hub / "7-1.1", devpath="1.1", devnum="4") + self.billboard = self.device(self.front / "7-1.1.5", devpath="1.1.5", bDeviceClass="11") + + def device(self, path, **attributes): + path.mkdir(parents=True, exist_ok=True) + for name, value in attributes.items(): + (path / name).write_text(value + "\n", encoding="ascii") + return path + + def event(self, path, marker="1", kind="usb_device"): + return SimpleNamespace(sys_path=str(path), device_type=kind, + properties={notifier.MARKER: marker}) + + def test_machine_requires_exact_compatible_token(self): + self.assertTrue(notifier.is_j700(self.sysfs)) + (self.sysfs / "firmware/devicetree/base/compatible").write_bytes(b"apple,j700-other\0") + self.assertFalse(notifier.is_j700(self.sysfs)) + + def test_machine_missing(self): + self.assertFalse(notifier.is_j700(self.sysfs / "missing")) + + def test_front_nested_billboard(self): + self.assertEqual(notifier.candidate(self.event(self.billboard), self.sysfs), self.front) + + def test_direct_front_billboard(self): + (self.front / "bDeviceClass").write_text("11\n") + self.assertEqual(notifier.candidate(self.event(self.front), self.sysfs), self.front) + + def test_rear_billboard_is_ignored_even_with_marker(self): + rear = self.device(self.hub / "7-1.2", devpath="1.2", bDeviceClass="11") + nested = self.device(rear / "7-1.2.5", devpath="1.2.5", bDeviceClass="11") + for path in (rear, nested): + self.assertIsNone(notifier.candidate(self.event(path), self.sysfs)) + + def test_requires_marker_and_device_class(self): + for marker, kind in (("", "usb_device"), ("1", "usb_interface")): + self.assertIsNone(notifier.candidate(self.event(self.billboard, marker, kind), self.sysfs)) + (self.billboard / "bDeviceClass").write_text("09\n") + self.assertIsNone(notifier.candidate(self.event(self.billboard), self.sysfs)) + + def test_wrong_fixed_hub_is_ignored(self): + (self.hub / "idProduct").write_text("0817\n") + self.assertIsNone(notifier.candidate(self.event(self.billboard), self.sysfs)) + + def test_wrong_controller_is_ignored(self): + controller = self.hub.parents[2] + controller.rename(controller.with_name("other.usb")) + path = controller.with_name("other.usb") / self.billboard.relative_to(controller) + self.assertIsNone(notifier.candidate(self.event(path), self.sysfs)) + + def test_removed_device_and_outside_sysfs_are_ignored(self): + self.assertIsNone(notifier.candidate(self.event(self.billboard / "gone"), self.sysfs)) + self.assertIsNone(notifier.front_device(Path("/tmp"), self.sysfs)) + + def test_plug_identity_changes_with_device_number(self): + first = notifier.plug_id(self.front, "1000") + (self.front / "devnum").write_text("5\n") + self.assertNotEqual(first, notifier.plug_id(self.front, "1001")) + (self.front / "devnum").unlink() + self.assertIsNone(notifier.plug_id(self.front, "1001")) + + def test_collect_coalesces_and_retains_live_front(self): + second = self.device(self.front / "7-1.1.6", devpath="1.1.6", bDeviceClass="11") + events = [self.event(self.front, ""), self.event(self.billboard), self.event(second)] + context = SimpleNamespace(list_devices=lambda **kwargs: events) + parent = SimpleNamespace(properties={"USEC_INITIALIZED": "1000"}) + live, eligible = notifier.collect(context, self.sysfs, lambda *args: parent) + self.assertEqual(live, eligible) + self.assertEqual(len(eligible), 1) + # Billboard removal does not end the dock's plug. + del events[1:] + live2, eligible2 = notifier.collect(context, self.sysfs, lambda *args: parent) + self.assertEqual(live, live2) + self.assertEqual(eligible2, set()) + + def test_collect_waits_for_front_initialization(self): + context = SimpleNamespace(list_devices=lambda **kwargs: [self.event(self.billboard)]) + parent = SimpleNamespace(properties={}) + self.assertEqual(notifier.collect(context, self.sysfs, lambda *args: parent), (set(), set())) + + def test_collect_tolerates_unplug_race(self): + context = SimpleNamespace(list_devices=lambda **kwargs: [self.event(self.billboard)]) + + def disappeared(*args): + raise OSError("device disappeared") + + self.assertEqual(notifier.collect(context, self.sysfs, disappeared), (set(), set())) + self.assertEqual(notifier.collect(context, self.sysfs, lambda *args: None), (set(), set())) + + def test_lookup_handles_library_unplug_error(self): + class DeviceGone(Exception): + pass + + def disappeared(*args): + raise DeviceGone() + + library = SimpleNamespace(DeviceNotFoundError=DeviceGone, + Devices=SimpleNamespace(from_sys_path=disappeared)) + with patch.dict(sys.modules, {"pyudev": library}): + self.assertIsNone(notifier.lookup_device(None, str(self.front))) + + +class Delivery(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.path = Path(self.tmp.name) / "state.json" + self.now = 100.0 + self.sent = 0 + self.success = True + self.active = True + self.state = self.new_state() + + def new_state(self): + return notifier.Notifier(self.path, self.send, lambda: self.active, lambda: self.now) + + def send(self): + self.sent += 1 + return self.success + + def test_once_per_plug_including_restarts(self): + self.state.reconcile({"plug-a"}) + self.now += 60 + self.state.reconcile({"plug-a"}) + self.new_state().reconcile({"plug-a"}) + self.assertEqual(self.sent, 1) + + def test_replug_waits_for_rate_limit(self): + self.state.reconcile({"plug-a"}) + self.state.reconcile(set()) + self.now += 1 + self.state.reconcile({"plug-b"}) + self.assertEqual(self.sent, 1) + self.now += 29 + self.state.reconcile({"plug-b"}) + self.assertEqual(self.sent, 2) + + def test_slow_session_check_does_not_shorten_rate_limit(self): + sends = [] + checks = [] + + def active(): + checks.append(self.now) + if len(checks) == 1: + self.now += 8 + return True + + def send(): + sends.append(self.now) + # The dispatch timestamp must already be persisted for restarts. + self.assertEqual(json.loads(self.path.read_text())["last_attempt"], self.now) + return True + + state = notifier.Notifier(self.path, send, active, lambda: self.now) + state.reconcile({"plug-a"}) + state.reconcile(set()) + state = notifier.Notifier(self.path, send, active, lambda: self.now) + self.now = 130.0 + state.reconcile({"plug-b"}) + self.assertEqual(sends, [108.0]) + self.assertEqual(checks, [100.0]) + self.now = 138.0 + state.reconcile({"plug-b"}) + self.assertEqual(sends, [108.0, 138.0]) + self.assertGreaterEqual(sends[1] - sends[0], notifier.INTERVAL) + + def test_disconnected_pending_plug_is_never_delivered(self): + self.state.reconcile({"plug-a"}) + self.now += 1 + self.state.reconcile({"plug-b"}) + self.state.reconcile(set()) + self.now += 60 + self.state.reconcile(set()) + self.assertEqual(self.sent, 1) + + def test_inactive_session_defers_until_active(self): + self.active = False + self.state.reconcile({"plug-a"}) + self.assertEqual(self.sent, 0) + self.active = True + self.state.reconcile({"plug-a"}) + self.assertEqual(self.sent, 1) + + def test_delivery_failure_retries_with_rate_limit(self): + self.success = False + self.state.reconcile({"plug-a"}) + self.assertEqual(self.state.shown, set()) + self.now += 29 + self.new_state().reconcile({"plug-a"}) + self.assertEqual(self.sent, 1) + self.success = True + self.now += 1 + self.state.reconcile({"plug-a"}) + self.assertEqual(self.sent, 2) + self.assertEqual(self.state.shown, {"plug-a"}) + + def test_corrupt_state_is_recoverable(self): + for data in ("invalid", "[]", '{"shown": false, "last_attempt": []}'): + self.path.write_text(data) + state = self.new_state() + self.assertEqual(state.shown, set()) + self.assertIsNone(state.last_attempt) + + def test_state_write_failure_prevents_untracked_send(self): + with patch.object(self.state, "save", side_effect=OSError): + with self.assertRaises(OSError): + self.state.reconcile({"plug-a"}) + self.assertEqual(self.sent, 0) + + def test_empty_plugs_never_checks_session_or_notifies(self): + with patch.object(self.state, "active", side_effect=AssertionError): + self.state.reconcile(set()) + self.assertFalse(self.path.exists()) + + def test_state_contains_no_peripheral_names(self): + self.state.reconcile({"sysfs-path:4:123:1000"}) + data = json.loads(self.path.read_text()) + self.assertEqual(set(data), {"shown", "last_attempt"}) + + +class SessionAndTransport(unittest.TestCase): + def test_other_machine_never_checks_session(self): + with patch.object(notifier, "is_j700", return_value=False), \ + patch.object(notifier.os, "getuid", side_effect=AssertionError): + notifier.main() + + def test_root_and_missing_bus_never_start_notifier(self): + with patch.object(notifier, "is_j700", return_value=True), \ + patch.object(notifier.os, "getuid", return_value=0): + with self.assertRaises(SystemExit): + notifier.main() + with patch.object(notifier, "is_j700", return_value=True), \ + patch.object(notifier.os, "getuid", return_value=1000), \ + patch.dict(os.environ, {}, clear=True): + with self.assertRaises(SystemExit): + notifier.main() + + def test_session_gate(self): + valid = "User=1000\nActive=yes\nRemote=no\nType=wayland\nSeat=seat0\n" + cases = [(valid, True)] + [ + (valid.replace(old, new), False) for old, new in ( + ("User=1000", "User=1001"), ("Active=yes", "Active=no"), + ("Remote=no", "Remote=yes"), ("Type=wayland", "Type=tty"), + ("Seat=seat0", "Seat="), + ) + ] + cases.append((valid.replace("wayland", "x11"), True)) + for reply, expected in cases: + with self.subTest(reply=reply), patch.object(notifier, "command", side_effect=["3\n", reply]): + self.assertEqual(notifier.active_desktop(1000), expected) + + def test_logind_errors_fail_closed(self): + for error in (FileNotFoundError(), subprocess.TimeoutExpired("loginctl", 5)): + with patch.object(notifier, "command", side_effect=error): + self.assertFalse(notifier.active_desktop(os.getuid())) + + def test_notification_is_plain_libnotify_on_inherited_bus(self): + with patch.object(notifier, "command", return_value="") as call: + self.assertTrue(notifier.notify()) + args = call.call_args.args[0] + self.assertEqual(args[-3:], ["--", notifier.SUMMARY, notifier.BODY]) + self.assertIn("--urgency=normal", args) + self.assertNotIn("--expire-time", " ".join(args)) + + def test_notification_transport_failure(self): + with patch.object(notifier, "command", side_effect=subprocess.TimeoutExpired("notify-send", 5)): + self.assertFalse(notifier.notify()) + + def test_subprocess_is_bounded_without_shell(self): + with patch.object(subprocess, "run", return_value=SimpleNamespace(stdout="ok")) as run: + self.assertEqual(notifier.command(["example", "literal"]), "ok") + self.assertEqual(run.call_args.kwargs["timeout"], 5) + self.assertNotIn("shell", run.call_args.kwargs) + + +if __name__ == "__main__": + unittest.main() From 70691476397adb33b28f6e5040729c46d94e9aa4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:26:17 +1100 Subject: [PATCH 1351/2680] tools: aurora-wake: Add an unattended suspend-to-idle regression test Suspend several times with an RTC alarm (rtc1, the PMU RTC, on the J700) and check after each resume that the sleep lasted until the alarm, that suspend_stats reports no failures, that every USB device is back, that the power/wakeup settings of USB devices, xHCI and DWC3 are unchanged, and that the log has no xHCI resume errors, warnings or oopses. Needs no one at the machine. (cherry picked from commit 8700f11cd53554f065b4b1ffb53ad7f0af66f4c3) Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-wake/s2idle-rtc-regression.sh | 144 +++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100755 tools/aurora-wake/s2idle-rtc-regression.sh diff --git a/tools/aurora-wake/s2idle-rtc-regression.sh b/tools/aurora-wake/s2idle-rtc-regression.sh new file mode 100755 index 0000000000000..d6eee0910010d --- /dev/null +++ b/tools/aurora-wake/s2idle-rtc-regression.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Unattended suspend-to-idle regression test for Apple silicon (J700 and others). +# +# Suspends with an RTC alarm (rtc1, apple-pmu-rtc, by default) several times and +# checks after every resume that: +# - the sleep lasted until the alarm (no spurious wakeup, e.g. from a running xHC), +# - suspend_stats reports no new failures, +# - every USB device present before is back, +# - the power/wakeup settings of USB devices, xHCI and DWC3 are unchanged, +# - the kernel log has no xHCI resume errors, warnings or oopses. +# +# Usage (as root): +# s2idle-rtc-regression.sh [-c cycles] [-s seconds] [-r rtc] [-o outdir] [-i ignore-regex] +# -i takes an extended regex of known, unrelated log lines to leave out of the +# error check, e.g. -i dcp for a display driver failing its own resume. +# Exit status: 0 when every cycle passed, 1 otherwise, 2 for a setup error. + +set -u + +cycles=3 +secs=20 +rtc=rtc1 +out=/var/tmp/s2idle-rtc-regression-$(date +%Y%m%d-%H%M%S) +ignore= + +while getopts "c:s:r:o:i:" opt; do + case $opt in + c) cycles=$OPTARG ;; + s) secs=$OPTARG ;; + r) rtc=$OPTARG ;; + o) out=$OPTARG ;; + i) ignore=$OPTARG ;; + *) echo "usage: $0 [-c cycles] [-s secs] [-r rtc] [-o outdir] [-i regex]" >&2; exit 2 ;; + esac +done + +[ "$(id -u)" = 0 ] || { echo "must run as root" >&2; exit 2; } +command -v rtcwake > /dev/null || { echo "rtcwake not found" >&2; exit 2; } +[ -e "/sys/class/rtc/$rtc/wakealarm" ] || { echo "$rtc has no wakealarm" >&2; exit 2; } +grep -qw s2idle /sys/power/mem_sleep 2> /dev/null || grep -qw freeze /sys/power/state || + { echo "suspend-to-idle not available" >&2; exit 2; } + +mkdir -p "$out" + +# Errors that must never appear in the log of a suspend/resume cycle. +bad='Host halt failed|xHC restore state timeout|xHC error in resume' +bad+='|Controller not ready at resume|HC died|Clearing Run/Stop bit failed' +bad+='|Controller Save State failed|PM: Some devices failed' +bad+='|PM: .*failed to suspend|PM: .*failed to resume|Freezing .* failed' +bad+='|unable to enumerate USB device|WARNING:|BUG:|Oops|Call trace:' + +usb_devices() { + local d + for d in /sys/bus/usb/devices/*; do + [ -f "$d/idVendor" ] || continue + echo "$(basename "$d") $(cat "$d/idVendor"):$(cat "$d/idProduct")" + done | sort +} + +wakeup_state() { + local d + for d in /sys/bus/usb/devices/* /sys/bus/platform/devices/{*.usb,xhci-hcd.*}; do + [ -f "$d/power/wakeup" ] && echo "$(basename "$d") $(cat "$d/power/wakeup")" + done | sort +} + +# The lines that differ between two files, on one line +changes() { + diff "$1" "$2" | grep '^[<>]' | tr '\n' ' ' +} + +# Drop the log lines the caller declared as known and unrelated +known_filter() { + if [ -n "$ignore" ]; then grep -Ev "$ignore"; else cat; fi +} + +stat_fail() { + cat /sys/power/suspend_stats/fail 2> /dev/null || echo 0 +} + +usb_devices > "$out/usb-before.txt" +wakeup_state > "$out/wakeup-before.txt" +uname -a > "$out/uname.txt" +echo "cycles=$cycles secs=$secs rtc=$rtc ignore=$ignore" > "$out/params.txt" + +failed=0 +for i in $(seq 1 "$cycles"); do + c="$out/cycle-$i" + mkdir -p "$c" + reasons=() + fail_before=$(stat_fail) + dmesg -C + sync + + t0=$(date +%s.%N) + rtcwake -d "$rtc" -m freeze -s "$secs" > "$c/rtcwake.txt" 2>&1 + rc=$? + t1=$(date +%s.%N) + slept=$(awk -v a="$t0" -v b="$t1" 'BEGIN { printf "%.1f", b - a }') + + [ $rc = 0 ] || reasons+=("rtcwake exited $rc") + if awk -v s="$slept" -v w="$secs" 'BEGIN { exit !(s + 1.5 < w) }'; then + wirq=$(cat /sys/power/pm_wakeup_irq 2>&1) + reasons+=("woke early after ${slept}s of ${secs}s (pm_wakeup_irq $wirq)") + fi + [ "$(stat_fail)" = "$fail_before" ] || reasons+=("suspend_stats fail count rose") + + # Give USB up to 15 s to come back. + for _ in $(seq 1 30); do + usb_devices > "$c/usb-after.txt" + cmp -s "$out/usb-before.txt" "$c/usb-after.txt" && break + sleep 0.5 + done + cmp -s "$out/usb-before.txt" "$c/usb-after.txt" || + reasons+=("USB devices differ: $(changes "$out/usb-before.txt" \ + "$c/usb-after.txt")") + + wakeup_state > "$c/wakeup-after.txt" + cmp -s "$out/wakeup-before.txt" "$c/wakeup-after.txt" || + reasons+=("power/wakeup changed: $(changes "$out/wakeup-before.txt" \ + "$c/wakeup-after.txt")") + + dmesg > "$c/dmesg.txt" + grep -E "$bad" "$c/dmesg.txt" | known_filter > "$c/bad-lines.txt" + [ -s "$c/bad-lines.txt" ] && + reasons+=("kernel log errors: $(head -3 "$c/bad-lines.txt" | tr '\n' ' ')") + + if [ ${#reasons[@]} = 0 ]; then + echo "cycle $i: PASS (slept ${slept}s)" | tee -a "$out/summary.txt" + else + failed=1 + echo "cycle $i: FAIL (slept ${slept}s)" | tee -a "$out/summary.txt" + printf ' - %s\n' "${reasons[@]}" | tee -a "$out/summary.txt" + fi +done + +if [ $failed = 0 ]; then + echo "RESULT: PASS ($cycles cycles) evidence: $out" | tee -a "$out/summary.txt" +else + echo "RESULT: FAIL evidence: $out" | tee -a "$out/summary.txt" +fi +exit $failed From 96971db03d9bdd20a42826ad38d41cebbc6a4a56 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:23:56 +1100 Subject: [PATCH 1352/2680] tools: aurora-wake: Harden the suspend-to-idle regression script The regression script cleared the kernel log before every cycle. That threw away the log from before the test, and every message logged between one cycle's snapshot and the next clear, so a late USB recovery failure could vanish from both the saved log and the error check. Save the log before the first cycle instead, check each cycle's part of it from a mark without clearing anything, and check what was logged after the last cycle as well. The mark moves past exactly what was saved, from the same read, so a message logged between two reads lands in the next collection. A log that wrapped meanwhile is checked whole. A cycle count of 0, a negative one or a non-number ran no cycle and still reported a pass. Accept only positive counts and a sleep of at least 5 s, refuse other arguments with exit status 2, and pass only when every requested cycle ran. It ran as root and wrote its evidence to a predictable path under /var/tmp, creating it with mkdir -p, so a symlink planted there could redirect what it writes. Create a fresh private directory with mktemp -d by default, and accept an explicit -o only as an absolute path that does not exist yet, created with mode 0700. The early-wake check measured the whole rtcwake command, so suspend and resume work could hide a wake long before the alarm. Measure the time actually spent suspended as the growth of CLOCK_BOOTTIME over CLOCK_MONOTONIC, and report the whole command time beside it. Some failed checks passed quietly. A kernel log that cannot be read, a suspend failure count that cannot be read, and an -i expression grep rejects, which made the filter hide every error, now fail the cycle or the run instead. Add -n to check the arguments without suspending, and a hardware-free test of the argument handling, the output directory rules and the kernel log collection. (cherry picked from commit 8d9758ed9430358118fb0065b2a1f9801309ca5f) Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-wake/s2idle-rtc-regression.sh | 135 +++++++++++++++--- .../aurora-wake/test-s2idle-rtc-regression.sh | 79 ++++++++++ 2 files changed, 197 insertions(+), 17 deletions(-) create mode 100755 tools/aurora-wake/test-s2idle-rtc-regression.sh diff --git a/tools/aurora-wake/s2idle-rtc-regression.sh b/tools/aurora-wake/s2idle-rtc-regression.sh index d6eee0910010d..89a131aca0b59 100755 --- a/tools/aurora-wake/s2idle-rtc-regression.sh +++ b/tools/aurora-wake/s2idle-rtc-regression.sh @@ -5,44 +5,88 @@ # # Suspends with an RTC alarm (rtc1, apple-pmu-rtc, by default) several times and # checks after every resume that: -# - the sleep lasted until the alarm (no spurious wakeup, e.g. from a running xHC), +# - the time actually spent suspended (CLOCK_BOOTTIME minus CLOCK_MONOTONIC) +# lasted until the alarm: no spurious wakeup, e.g. from a running xHC, # - suspend_stats reports no new failures, # - every USB device present before is back, # - the power/wakeup settings of USB devices, xHCI and DWC3 are unchanged, # - the kernel log has no xHCI resume errors, warnings or oopses. # # Usage (as root): -# s2idle-rtc-regression.sh [-c cycles] [-s seconds] [-r rtc] [-o outdir] [-i ignore-regex] +# s2idle-rtc-regression.sh [-n] [-c cycles] [-s seconds] [-r rtc] [-o new-dir] [-i ignore-regex] # -i takes an extended regex of known, unrelated log lines to leave out of the # error check, e.g. -i dcp for a display driver failing its own resume. -# Exit status: 0 when every cycle passed, 1 otherwise, 2 for a setup error. +# -n only checks the arguments and prints what would run. +# The kernel log is never cleared: it is saved before the first cycle, and each +# cycle checks what was logged since the previous one. +# Exit status: 0 when every requested cycle ran and passed, 1 otherwise, 2 for +# a usage or setup error. set -u cycles=3 secs=20 rtc=rtc1 -out=/var/tmp/s2idle-rtc-regression-$(date +%Y%m%d-%H%M%S) +out= ignore= +dry_run=0 -while getopts "c:s:r:o:i:" opt; do +usage() { + echo "usage: $0 [-n] [-c cycles] [-s secs] [-r rtc] [-o outdir] [-i regex]" >&2 + exit 2 +} + +while getopts "nc:s:r:o:i:" opt; do case $opt in + n) dry_run=1 ;; c) cycles=$OPTARG ;; s) secs=$OPTARG ;; r) rtc=$OPTARG ;; o) out=$OPTARG ;; i) ignore=$OPTARG ;; - *) echo "usage: $0 [-c cycles] [-s secs] [-r rtc] [-o outdir] [-i regex]" >&2; exit 2 ;; + *) usage ;; esac done +[ $OPTIND -gt $# ] || usage + +# A count that runs no cycle must not report a pass. +[[ $cycles =~ ^[1-9][0-9]{0,3}$ ]] || { echo "cycles must be 1-9999: $cycles" >&2; usage; } +[[ $secs =~ ^[1-9][0-9]{0,4}$ ]] && [ "$secs" -ge 5 ] || + { echo "seconds must be 5-99999: $secs" >&2; usage; } +# An invalid expression would make the filter fail and hide every error. +if [ -n "$ignore" ]; then + grep -Eq -- "$ignore" < /dev/null + [ $? -le 1 ] || { echo "invalid -i expression: $ignore" >&2; usage; } +fi + +# The evidence goes to a directory this script creates for itself, with private +# permissions: a fresh one under /var/tmp by default, or the absolute path given +# with -o, which must not exist yet. It never writes through an existing path, +# so a planted symlink cannot redirect what root writes. +if [ -n "$out" ]; then + [[ $out == /* ]] || { echo "the output directory must be an absolute path" >&2; usage; } + [ ! -e "$out" ] && [ ! -L "$out" ] || + { echo "the output directory must not exist yet: $out" >&2; usage; } +fi + +if [ $dry_run = 1 ]; then + echo "would run $cycles cycles of $secs s on $rtc into ${out:-a new /var/tmp directory}" + exit 0 +fi [ "$(id -u)" = 0 ] || { echo "must run as root" >&2; exit 2; } command -v rtcwake > /dev/null || { echo "rtcwake not found" >&2; exit 2; } +command -v python3 > /dev/null || { echo "python3 not found" >&2; exit 2; } [ -e "/sys/class/rtc/$rtc/wakealarm" ] || { echo "$rtc has no wakealarm" >&2; exit 2; } grep -qw s2idle /sys/power/mem_sleep 2> /dev/null || grep -qw freeze /sys/power/state || { echo "suspend-to-idle not available" >&2; exit 2; } -mkdir -p "$out" +if [ -n "$out" ]; then + mkdir -m 700 -- "$out" || { echo "cannot create $out" >&2; exit 2; } +else + out=$(mktemp -d /var/tmp/s2idle-rtc-regression.XXXXXXXX) || + { echo "cannot create an output directory" >&2; exit 2; } +fi # Errors that must never appear in the log of a suspend/resume cycle. bad='Host halt failed|xHC restore state timeout|xHC error in resume' @@ -76,36 +120,75 @@ known_filter() { if [ -n "$ignore" ]; then grep -Ev "$ignore"; else cat; fi } +# The suspend failure count, or "unavailable": a missing count is not zero. stat_fail() { - cat /sys/power/suspend_stats/fail 2> /dev/null || echo 0 + local n + + n=$(cat /sys/power/suspend_stats/fail 2> /dev/null) + [[ $n =~ ^[0-9]+$ ]] && echo "$n" || echo unavailable } +# CLOCK_BOOTTIME and CLOCK_MONOTONIC: only the first one runs while suspended. +clocks() { + python3 -c 'import time; print(time.clock_gettime(time.CLOCK_BOOTTIME), + time.clock_gettime(time.CLOCK_MONOTONIC))' +} + +# --- kernel log collection (also run by test-s2idle-rtc-regression.sh) --- +# Save the kernel log since $mark (a line count) into $1 without clearing it, and +# move $mark past exactly what was saved, from the same read, so that nothing +# logged in between is skipped. A log that wrapped meanwhile no longer starts +# with $log_first: then all of it is saved. A failed read fails. +log_take() { + dmesg > "$out/.dmesg" || return 1 + if [ "$(head -n 1 "$out/.dmesg")" = "$log_first" ]; then + tail -n +"$((mark + 1))" "$out/.dmesg" > "$1" || return 1 + else + { echo "(kernel log wrapped; the whole log follows)"; cat "$out/.dmesg"; } > "$1" || + return 1 + fi + mark=$(wc -l < "$out/.dmesg") +} +# --- end of kernel log collection --- + usb_devices > "$out/usb-before.txt" wakeup_state > "$out/wakeup-before.txt" uname -a > "$out/uname.txt" echo "cycles=$cycles secs=$secs rtc=$rtc ignore=$ignore" > "$out/params.txt" +dmesg > "$out/dmesg-before.txt" || { echo "cannot read the kernel log" >&2; exit 2; } +log_first=$(head -n 1 "$out/dmesg-before.txt") +mark=$(wc -l < "$out/dmesg-before.txt") failed=0 +completed=0 for i in $(seq 1 "$cycles"); do c="$out/cycle-$i" mkdir -p "$c" reasons=() fail_before=$(stat_fail) - dmesg -C sync - t0=$(date +%s.%N) + read -r b0 m0 < <(clocks) rtcwake -d "$rtc" -m freeze -s "$secs" > "$c/rtcwake.txt" 2>&1 rc=$? - t1=$(date +%s.%N) - slept=$(awk -v a="$t0" -v b="$t1" 'BEGIN { printf "%.1f", b - a }') + read -r b1 m1 < <(clocks) + # Time spent suspended, and the whole command with entry and resume + slept=$(awk -v b="$b1" -v a="$b0" -v n="$m1" -v m="$m0" \ + 'BEGIN { printf "%.1f", (b - a) - (n - m) }') + took=$(awk -v b="$b1" -v a="$b0" 'BEGIN { printf "%.1f", b - a }') [ $rc = 0 ] || reasons+=("rtcwake exited $rc") - if awk -v s="$slept" -v w="$secs" 'BEGIN { exit !(s + 1.5 < w) }'; then + # The alarm counts from before suspend entry: allow 3 s for entry. + if awk -v s="$slept" -v w="$secs" 'BEGIN { exit !(s + 3 < w) }'; then wirq=$(cat /sys/power/pm_wakeup_irq 2>&1) reasons+=("woke early after ${slept}s of ${secs}s (pm_wakeup_irq $wirq)") fi - [ "$(stat_fail)" = "$fail_before" ] || reasons+=("suspend_stats fail count rose") + fail_after=$(stat_fail) + if [ "$fail_before" = unavailable ] || [ "$fail_after" = unavailable ]; then + reasons+=("suspend_stats fail count unavailable") + elif [ "$fail_after" != "$fail_before" ]; then + reasons+=("suspend_stats fail count rose") + fi # Give USB up to 15 s to come back. for _ in $(seq 1 30); do @@ -122,20 +205,38 @@ for i in $(seq 1 "$cycles"); do reasons+=("power/wakeup changed: $(changes "$out/wakeup-before.txt" \ "$c/wakeup-after.txt")") - dmesg > "$c/dmesg.txt" + log_take "$c/dmesg.txt" || reasons+=("kernel log unavailable") grep -E "$bad" "$c/dmesg.txt" | known_filter > "$c/bad-lines.txt" [ -s "$c/bad-lines.txt" ] && reasons+=("kernel log errors: $(head -3 "$c/bad-lines.txt" | tr '\n' ' ')") + times="suspended ${slept}s, ${took}s in all" if [ ${#reasons[@]} = 0 ]; then - echo "cycle $i: PASS (slept ${slept}s)" | tee -a "$out/summary.txt" + echo "cycle $i: PASS ($times)" | tee -a "$out/summary.txt" else failed=1 - echo "cycle $i: FAIL (slept ${slept}s)" | tee -a "$out/summary.txt" + echo "cycle $i: FAIL ($times)" | tee -a "$out/summary.txt" printf ' - %s\n' "${reasons[@]}" | tee -a "$out/summary.txt" fi + completed=$((completed + 1)) done +# Whatever was logged after the last check +if ! log_take "$out/dmesg-after.txt"; then + failed=1 + echo "kernel log unavailable after the last cycle" | tee -a "$out/summary.txt" +elif grep -E "$bad" "$out/dmesg-after.txt" | known_filter | grep -q .; then + failed=1 + echo "kernel log errors after the last cycle, see dmesg-after.txt" | + tee -a "$out/summary.txt" +fi +rm -f "$out/.dmesg" + +if [ "$completed" != "$cycles" ]; then + failed=1 + echo "only $completed of $cycles cycles ran" | tee -a "$out/summary.txt" +fi + if [ $failed = 0 ]; then echo "RESULT: PASS ($cycles cycles) evidence: $out" | tee -a "$out/summary.txt" else diff --git a/tools/aurora-wake/test-s2idle-rtc-regression.sh b/tools/aurora-wake/test-s2idle-rtc-regression.sh new file mode 100755 index 0000000000000..32a2b196b6b14 --- /dev/null +++ b/tools/aurora-wake/test-s2idle-rtc-regression.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Hardware-free checks of the argument handling of s2idle-rtc-regression.sh: a +# count that would run no cycle must be refused, not reported as a pass, and the +# output must go to a fresh directory of its own. + +script="$(dirname "$0")/s2idle-rtc-regression.sh" +fails=0 + +expect() { + local want=$1 got + shift + bash "$script" "$@" > /dev/null 2>&1 + got=$? + if [ "$got" = "$want" ]; then + echo "ok: $* -> $got" + else + echo "FAIL: $* -> $got, expected $want" + fails=1 + fi +} + +expect 2 -n -c 0 +expect 2 -n -c -1 +expect 2 -n -c abc +expect 2 -n -c '' +expect 2 -n -s 0 +expect 2 -n -s 2 +expect 2 -n -s x +expect 2 -n extra-argument +expect 0 -n -c 3 -s 20 +expect 0 -n +# An invalid filter would hide every kernel log error. +expect 2 -n -i '[' +expect 0 -n -i 'dcp|DCP' + +# Output only goes to a directory the script creates itself, never to an +# existing path that may be a symlink planted by someone else. +existing=$(mktemp -d) +ln -s "$existing" "$existing.link" +expect 2 -n -o "$existing" +expect 2 -n -o "$existing.link" +expect 2 -n -o relative-dir +expect 0 -n -o "$existing/new" +rm -rf "$existing" "$existing.link" + +# Kernel log collection. A message logged right after a read must land in the +# next collection, and a failed read must fail, not pass as an empty log. +tmp=$(mktemp -d) +sed -n '/^# --- kernel log collection/,/^# --- end of kernel log collection/p' \ + "$script" > "$tmp/lib.sh" +fakelog=$tmp/kmsg +late=0 +dmesg() { + cat "$fakelog" || return 1 + late=$((late + 1)) + echo "WARNING: late $late" >> "$fakelog" +} +printf 'first line\nsecond line\n' > "$fakelog" +# shellcheck source=/dev/null +. "$tmp/lib.sh" +out=$tmp log_first='first line' mark=2 +if log_take "$tmp/c1" && log_take "$tmp/c2" && grep -q 'late 1' "$tmp/c2"; then + echo "ok: a message logged right after a read is in the next collection" +else + echo "FAIL: a message logged between collections was skipped" + fails=1 +fi +fakelog=$tmp/missing +if log_take "$tmp/c3" 2> /dev/null; then + echo "FAIL: a failed kernel log read passed" + fails=1 +else + echo "ok: a failed kernel log read fails" +fi +rm -rf "$tmp" + +exit $fails From b4e186718c317c9d79ab3682ac020e3bb43e204b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:13:09 +1100 Subject: [PATCH 1353/2680] tools: aurora-wake: Add Bluetooth wake tests Tests for Bluetooth wake from suspend-to-idle on the J700: - bt-wake-cycle.sh: one RTC-backstopped sleep with wakeup of the Bluetooth PCI function enabled or disabled; checks that the driver armed (or did not arm) its interrupt, that the sleep was ended by the Bluetooth MSI (or by the RTC with wakeup disabled), that the controller is up and a given device is connected again after resume, and the kernel log. - attended-bt-wake.sh: the same with a real keyboard and a person pressing a key, about two minutes. - bt-wake-power.sh: alternating sleeps with wakeup disabled and enabled, recording Bluetooth and total interrupts, s2idle loop re-entries and battery energy. - hog-keyboard.py: an emulated BLE HID-over-GATT keyboard on another BlueZ host that advertises only when told to "press a key", like an idle keyboard, so that the wake test can run unattended. - bt-wake-emulated.sh: driver for the unattended run: pairs the emulator with the system under test, runs wake-enabled and wake-disabled cycles with an emulated key press during each sleep, and removes the pairing. Hosts and Bluetooth addresses come from a local configuration file outside the tree. Not for upstream. (cherry picked from commit d53c1cd749777fd26e052a34298bacf77a182689) Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-wake/attended-bt-wake.sh | 44 ++++ tools/aurora-wake/bt-wake-cycle.sh | 155 ++++++++++++ tools/aurora-wake/bt-wake-emulated.sh | 138 +++++++++++ tools/aurora-wake/bt-wake-power.sh | 57 +++++ tools/aurora-wake/hog-keyboard.py | 332 ++++++++++++++++++++++++++ 5 files changed, 726 insertions(+) create mode 100755 tools/aurora-wake/attended-bt-wake.sh create mode 100755 tools/aurora-wake/bt-wake-cycle.sh create mode 100755 tools/aurora-wake/bt-wake-emulated.sh create mode 100755 tools/aurora-wake/bt-wake-power.sh create mode 100755 tools/aurora-wake/hog-keyboard.py diff --git a/tools/aurora-wake/attended-bt-wake.sh b/tools/aurora-wake/attended-bt-wake.sh new file mode 100755 index 0000000000000..f8afc8ac87269 --- /dev/null +++ b/tools/aurora-wake/attended-bt-wake.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Attended Bluetooth keyboard wake check (J700), about two minutes, as root. +# +# Usage: attended-bt-wake.sh +# is a paired keyboard with WakeAllowed set (bluetoothctl wake on). +# +# 1. Press a key on the keyboard so that it connects; the script waits for it. +# 2. At SUSPENDING, wait for the screen to go dark, count to ten, and press one +# key on the keyboard. The RTC ends the sleep after 60 s otherwise. +set -u + +addr=${1:-} +[[ $addr =~ ^([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$ ]] || + { echo "usage: $0 " >&2; exit 2; } +here=$(dirname "$(readlink -f "$0")") +out=/var/tmp/bt-wake/attended-$(date +%H%M%S) +mkdir -p "$out" + +bluetoothctl info "$addr" > "$out/device.txt" 2>&1 || + { echo "$addr is not known to BlueZ" >&2; exit 2; } +grep -E "Name|Paired|WakeAllowed|Connected" "$out/device.txt" +grep -q "WakeAllowed: yes" "$out/device.txt" || + { echo "wake is not allowed: bluetoothctl wake $addr on" >&2; exit 2; } + +echo ">>> Press a key on the Bluetooth keyboard so that it connects (60 s)." +for _ in $(seq 1 120); do + bluetoothctl info "$addr" | grep -q "Connected: yes" && break + sleep 0.5 +done +bluetoothctl info "$addr" | grep -q "Connected: yes" || + { echo "the keyboard did not connect" >&2; exit 2; } +echo "connected." + +echo ">>> SUSPENDING. When the screen is dark, count to ten and press ONE key." +"$here/bt-wake-cycle.sh" -w on -s 60 -a "$addr" -o "$out/cycle" +rc=$? +if [ $rc = 0 ]; then + echo "VERDICT: PASS (the Bluetooth keyboard woke the system)" +else + echo "VERDICT: FAIL (see $out/cycle/verdict.txt)" +fi +exit $rc diff --git a/tools/aurora-wake/bt-wake-cycle.sh b/tools/aurora-wake/bt-wake-cycle.sh new file mode 100755 index 0000000000000..3c10162256e97 --- /dev/null +++ b/tools/aurora-wake/bt-wake-cycle.sh @@ -0,0 +1,155 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# One suspend-to-idle cycle for the MT7932 Bluetooth wake test (J700), as root. +# +# Sets power/wakeup of the Bluetooth PCI function, suspends with an RTC alarm as +# a backstop and records what ended the sleep. Something else (a person, or an +# emulated keyboard on another machine) is expected to make a wake-enabled +# Bluetooth HID device reconnect while the system sleeps. +# +# Usage: bt-wake-cycle.sh -w on|off [-s backstop-secs] [-r rtc] [-a bdaddr] [-o outdir] +# -w on expect a Bluetooth wake well before the backstop +# -w off expect no wake: the RTC ends the sleep +# -a Bluetooth address that must be connected again after resume +# The file /suspending appears right before the system suspends. +# Exit status: 0 PASS, 1 FAIL, 2 setup error. + +set -u + +mode= +secs=120 +rtc=rtc1 +addr= +out=/var/tmp/bt-wake/cycle-$(date +%Y%m%d-%H%M%S) +bdf=0000:01:00.1 + +while getopts "w:s:r:a:o:" opt; do + case $opt in + w) mode=$OPTARG ;; + s) secs=$OPTARG ;; + r) rtc=$OPTARG ;; + a) addr=$OPTARG ;; + o) out=$OPTARG ;; + *) exit 2 ;; + esac +done +case $mode in +on|off) ;; +*) echo "usage: $0 -w on|off [-s secs] [-r rtc] [-a bdaddr] [-o outdir]" >&2; exit 2 ;; +esac +[ "$(id -u)" = 0 ] || { echo "must run as root" >&2; exit 2; } +dev=/sys/bus/pci/devices/$bdf +[ -f "$dev/power/wakeup" ] || { echo "$bdf has no power/wakeup" >&2; exit 2; } + +mkdir -p "$out" +irq=$(awk -F: '/mt7932-bt/ { gsub(/ /, "", $1); print $1; exit }' /proc/interrupts) +[ -n "$irq" ] || { echo "no mt7932-bt interrupt" >&2; exit 2; } +# Recorded only: a PME would arrive on the root port's PME interrupt. +pme=$(awk -F: '/PCI-MSI-0000:00:00.0.*PME/ { gsub(/ /, "", $1); print $1; exit }' /proc/interrupts) + +bt_irqs() { + awk -v i="$irq" '$1 == i ":" { + for (k = 2; k <= NF; k++) if ($k ~ /^[0-9]+$/) s += $k; else break + } END { print s + 0 }' /proc/interrupts +} + +orig_wakeup=$(cat "$dev/power/wakeup") +restore() { echo "$orig_wakeup" > "$dev/power/wakeup"; } +trap restore EXIT +[ "$mode" = on ] && echo enabled > "$dev/power/wakeup" || echo disabled > "$dev/power/wakeup" + +echo 'module mt7932_bt_pcie +p' > /sys/kernel/debug/dynamic_debug/control 2> /dev/null +echo 1 > /sys/power/pm_debug_messages + +{ + echo "mode=$mode secs=$secs rtc=$rtc addr=$addr irq=$irq" + uname -r + echo "wakeup=$(cat "$dev/power/wakeup")" \ + "wakeup_count=$(cat "$dev/power/wakeup_count" 2>/dev/null)" + echo "irq_wakeup=$(cat /sys/kernel/irq/"$irq"/wakeup 2>/dev/null)" + echo "bt_irqs=$(bt_irqs)" + bluetoothctl show | grep -E "Powered|PowerState" + [ -n "$addr" ] && bluetoothctl info "$addr" | grep -E "Name|Connected|WakeAllowed" +} > "$out/before.txt" 2>&1 +fail_before=$(cat /sys/power/suspend_stats/fail) +dmesg -C +# HCI trace of the whole cycle: what the host sent at suspend, what woke it. +btmon -w "$out/hci.btsnoop" > /dev/null 2>&1 & +btmon_pid=$! +sleep 1 +sync + +date +%s.%N > "$out/suspending" +t0=$(date +%s.%N) +rtcwake -d "$rtc" -m freeze -s "$secs" > "$out/rtcwake.txt" 2>&1 +rc=$? +t1=$(date +%s.%N) +slept=$(awk -v a="$t0" -v b="$t1" 'BEGIN { printf "%.1f", b - a }') +wirq=$(cat /sys/power/pm_wakeup_irq 2>&1) +irqs_after=$(bt_irqs) +wsrc=other +[ "$wirq" = "$irq" ] && wsrc=bt-msi +[ -n "$pme" ] && [ "$wirq" = "$pme" ] && wsrc=root-port-pme + +# Bluetooth must come back: the controller up and, if given, the device connected. +reconnect=n/a +if [ -n "$addr" ]; then + reconnect=timeout + for i in $(seq 1 60); do + if bluetoothctl info "$addr" 2> /dev/null | grep -q "Connected: yes"; then + reconnect="${i}x0.5s" + break + fi + sleep 0.5 + done +fi +powered=$(bluetoothctl show | awk '/Powered:/ { print $2 }') +kill "$btmon_pid" 2> /dev/null + +dmesg > "$out/dmesg.txt" +{ + echo "rc=$rc slept=${slept}s pm_wakeup_irq=$wirq ($wsrc) bt_irq=$irq pme_irq=$pme" + echo "bt_irqs_after=$irqs_after wakeup_count=$(cat "$dev/power/wakeup_count" 2>/dev/null)" + echo "irq_wakeup_after=$(cat /sys/kernel/irq/"$irq"/wakeup 2>/dev/null)" + echo "reconnect=$reconnect powered=$powered" + grep -E "mt7932|Bluetooth|hci0" "$out/dmesg.txt" | grep -vE "Timekeeping|s2idle" | tail -20 +} > "$out/after.txt" + +reasons=() +[ $rc = 0 ] || reasons+=("rtcwake exited $rc") +[ "$(cat /sys/power/suspend_stats/fail)" = "$fail_before" ] || + reasons+=("suspend_stats fail count rose") +grep -q "system sleep with wakeup armed" "$out/dmesg.txt" +armed=$? +if [ "$mode" = on ]; then + [ $armed = 0 ] || reasons+=("driver did not report wakeup armed") + # The driver wakes through the function's MSI only; a root port PME is + # recorded but is not the designed path. + [ "$wsrc" = bt-msi ] || + reasons+=("pm_wakeup_irq $wirq ($wsrc) is not the Bluetooth MSI $irq") + awk -v s="$slept" -v w="$secs" 'BEGIN { exit !(s < w - 10) }' || + reasons+=("slept ${slept}s of ${secs}s") +else + [ $armed != 0 ] || reasons+=("wakeup armed although disabled") + awk -v s="$slept" -v w="$secs" 'BEGIN { exit !(s + 1.5 >= w) }' || + reasons+=("woke early after ${slept}s (pm_wakeup_irq $wirq)") +fi +[ "$powered" = yes ] || reasons+=("controller not powered after resume") +[ "$reconnect" != timeout ] || reasons+=("$addr did not reconnect within 30 s") +bad='WARNING:|BUG:|Oops|Call trace:|terminal transport fault|failed to (suspend|resume)' +grep -E "$bad" "$out/dmesg.txt" > "$out/bad-lines.txt" +[ -s "$out/bad-lines.txt" ] && + reasons+=("kernel log errors: $(head -2 "$out/bad-lines.txt" | tr '\n' ' ')") + +if [ ${#reasons[@]} = 0 ]; then + echo "VERDICT wake=$mode: PASS slept=${slept}s pm_wakeup_irq=$wirq ($wsrc)" \ + "reconnect=$reconnect" | tee "$out/verdict.txt" + status=0 +else + echo "VERDICT wake=$mode: FAIL slept=${slept}s pm_wakeup_irq=$wirq" | tee "$out/verdict.txt" + printf ' - %s\n' "${reasons[@]}" | tee -a "$out/verdict.txt" + status=1 +fi +echo "evidence: $out" | tee -a "$out/verdict.txt" +exit $status diff --git a/tools/aurora-wake/bt-wake-emulated.sh b/tools/aurora-wake/bt-wake-emulated.sh new file mode 100755 index 0000000000000..64689be5bd715 --- /dev/null +++ b/tools/aurora-wake/bt-wake-emulated.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Unattended Bluetooth wake test with an emulated BLE HID keyboard +# (hog-keyboard.py) on a second BlueZ host. Runs on a third machine that can +# reach both over SSH. +# +# bt-wake-emulated.sh pair pair the emulator with the target +# bt-wake-emulated.sh run [N] N wake-enabled cycles (default 5), then two +# wake-disabled cycles +# bt-wake-emulated.sh unpair remove that pairing on both sides +# +# The lab is described by a local configuration file that is not part of the +# tree, ${BT_WAKE_CONF:-~/.config/aurora/bt-wake.conf}, a shell fragment with: +# TARGET_SSH ssh destination (an ssh_config alias) of the system under test +# EMU_SSH ssh destination of the BlueZ host running the emulator +# EMU_ADDR Bluetooth address of the emulator host's adapter +# TARGET_ADDR Bluetooth address of the target's adapter +# EMU_TOOLS directory holding hog-keyboard.py on the emulator host +# TARGET_SSH_ARGS optional extra ssh/scp options for the target, e.g. -i +# Only the emulator is ever paired or removed; the paired lists of both sides +# are recorded before and after. +set -u + +conf=${BT_WAKE_CONF:-$HOME/.config/aurora/bt-wake.conf} +[ -r "$conf" ] || { echo "no lab configuration: $conf" >&2; exit 2; } +# shellcheck source=/dev/null +. "$conf" +mac='^([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$' +for v in TARGET_SSH EMU_SSH EMU_ADDR TARGET_ADDR EMU_TOOLS; do + [ -n "${!v:-}" ] || { echo "$conf does not set $v" >&2; exit 2; } +done +[[ $EMU_ADDR =~ $mac && $TARGET_ADDR =~ $mac ]] || + { echo "EMU_ADDR and TARGET_ADDR must be Bluetooth addresses" >&2; exit 2; } + +ssh_opts=(-o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=2) +read -r -a target_args <<< "${TARGET_SSH_ARGS:-}" +tgt() { ssh "${ssh_opts[@]}" "${target_args[@]}" "$TARGET_SSH" "$@"; } +box() { ssh "${ssh_opts[@]}" "$EMU_SSH" "$@"; } +CTL=/run/hogkb/ctl +TGT_DIR=/var/tmp/bt-wake +BACKSTOP=120 +TRIGGER_AFTER=20 +out=${OUT:-./bt-wake-run-$(date +%Y%m%d-%H%M%S)} +mkdir -p "$out" + +say() { echo "$(date +%H:%M:%S) $*" | tee -a "$out/run.log"; } +emu() { box "echo '$*' | sudo -n tee $CTL > /dev/null"; } + +emu_start() { + box "pgrep -f '[h]og-keyboard.py' > /dev/null || { + sudo -n mkdir -p /run/hogkb + sudo -n setsid nohup $EMU_TOOLS/hog-keyboard.py > /dev/null 2>&1 < /dev/null & + sleep 3 + }; sudo -n tail -2 /run/hogkb/log" +} + +paired_lists() { + tgt "bluetoothctl devices Paired" > "$out/target-paired-$1.txt" + box "bluetoothctl devices Paired" > "$out/emulator-paired-$1.txt" + say "paired ($1): target $(wc -l < "$out/target-paired-$1.txt")," \ + "emulator host $(wc -l < "$out/emulator-paired-$1.txt")" +} + +case "${1:-}" in +pair) + paired_lists before-pair + emu_start + emu pairable on + sleep 2 + # Target side: discover, pair (Just Works), trust, allow wake. + tgt "{ echo 'agent NoInputNoOutput'; echo default-agent; echo 'scan le'; sleep 12; + echo 'scan off'; echo 'pair $EMU_ADDR'; sleep 15; echo 'trust $EMU_ADDR'; + sleep 1; echo 'connect $EMU_ADDR'; sleep 8; echo 'wake $EMU_ADDR on'; sleep 1; + echo 'info $EMU_ADDR'; sleep 1; } | bluetoothctl" > "$out/pair-target.txt" 2>&1 + emu pairable off + grep -E "Paired|Bonded|Trusted|Connected|WakeAllowed|UUID: Human" \ + "$out/pair-target.txt" | tail -8 | tee -a "$out/run.log" + paired_lists after-pair + ;; +run) + n=${2:-5} + emu_start + tgt "mkdir -p $TGT_DIR" && + scp -q "${target_args[@]}" "$(dirname "$0")/bt-wake-cycle.sh" \ + "$TARGET_SSH:$TGT_DIR/" + pass=0 + plan=() + for _ in $(seq 1 "$n"); do plan+=(on); done + plan+=(off off) + k=0 + for mode in "${plan[@]}"; do + k=$((k + 1)) + d=$TGT_DIR/cycle-$k-$mode + # Start each cycle with the keyboard connected, as in real use. + emu key 0x2c 30 + sleep 8 + say "cycle $k wake=$mode: suspending the target (backstop ${BACKSTOP}s)" + tgt "rm -rf $d; setsid nohup $TGT_DIR/bt-wake-cycle.sh -w $mode -s $BACKSTOP \ + -a $EMU_ADDR -o $d > $d.log 2>&1 < /dev/null &" + for _ in $(seq 1 30); do + tgt "test -f $d/suspending" 2> /dev/null && break + sleep 1 + done + sleep "$TRIGGER_AFTER" + say "cycle $k: emulated key press" + # Keep advertising past the backstop, so that the keyboard also + # comes back after an RTC wake. + emu key 0x2c $((BACKSTOP + 60)) + for _ in $(seq 1 $((BACKSTOP + 120))); do + tgt "test -f $d/verdict.txt" 2> /dev/null && break + sleep 1 + done + tgt "cat $d/verdict.txt $d/after.txt" > "$out/cycle-$k-$mode.txt" 2>&1 + head -4 "$out/cycle-$k-$mode.txt" | tee -a "$out/run.log" + grep -q "PASS" "$out/cycle-$k-$mode.txt" && pass=$((pass + 1)) + box "sudo -n tail -8 /run/hogkb/log" >> "$out/emulator.log" + sleep 10 + done + say "RESULT: $pass of ${#plan[@]} cycles passed (evidence $out, target $TGT_DIR)" + tgt "tar -C $TGT_DIR -czf - ." > "$out/target-evidence.tgz" + [ "$pass" = "${#plan[@]}" ] + ;; +unpair) + paired_lists before-unpair + if grep -qi "$EMU_ADDR" "$out/target-paired-before-unpair.txt"; then + tgt "bluetoothctl remove $EMU_ADDR" | tee -a "$out/run.log" + fi + box "bluetoothctl devices Paired | grep -qi $TARGET_ADDR && + sudo -n bluetoothctl remove $TARGET_ADDR" | tee -a "$out/run.log" + emu quit + paired_lists after-unpair + ;; +*) + echo "usage: $0 pair | run [cycles] | unpair" >&2 + exit 2 + ;; +esac diff --git a/tools/aurora-wake/bt-wake-power.sh b/tools/aurora-wake/bt-wake-power.sh new file mode 100755 index 0000000000000..af5d49dadb216 --- /dev/null +++ b/tools/aurora-wake/bt-wake-power.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Cost of Bluetooth wake in suspend-to-idle on the J700, as root: alternates +# RTC-timed sleeps with the Bluetooth PCI function's wakeup disabled and +# enabled (no Bluetooth activity expected) and records, per sleep: +# - Bluetooth MSIs and all interrupts taken across the sleep, +# - how often the s2idle loop was re-entered (pm_debug "Timekeeping suspended"), +# - the battery energy used, when running on battery (energy_now). +# Usage: bt-wake-power.sh [-s secs] [-n pairs] [-o outdir] +set -u +secs=300 +pairs=2 +out=/var/tmp/bt-wake/power-$(date +%Y%m%d-%H%M%S) +while getopts "s:n:o:" opt; do + case $opt in s) secs=$OPTARG ;; n) pairs=$OPTARG ;; o) out=$OPTARG ;; *) exit 2 ;; esac +done +[ "$(id -u)" = 0 ] || { echo "must run as root" >&2; exit 2; } +dev=/sys/bus/pci/devices/0000:01:00.1 +bat=/sys/class/power_supply/macsmc-battery +irq=$(awk -F: '/mt7932-bt/ { gsub(/ /, "", $1); print $1; exit }' /proc/interrupts) +mkdir -p "$out" +orig=$(cat "$dev/power/wakeup") +trap 'echo "$orig" > "$dev/power/wakeup"' EXIT +echo 1 > /sys/power/pm_debug_messages +echo 'module mt7932_bt_pcie +p' > /sys/kernel/debug/dynamic_debug/control 2> /dev/null + +irq_sum() { # $1: irq number or "all" + awk -v i="$1" '$1 ~ /^[0-9]+:$/ && (i == "all" || $1 == i ":") { + for (k = 2; k <= NF; k++) if ($k ~ /^[0-9]+$/) s += $k; else break + } END { print s + 0 }' /proc/interrupts +} + +echo "mode secs slept bt_msis all_irqs s2idle_entries energy_mWh status armed" | + tee "$out/results.txt" +for p in $(seq 1 "$pairs"); do + for mode in disabled enabled; do + echo "$mode" > "$dev/power/wakeup" + dmesg -C + b0=$(irq_sum "$irq") + a0=$(irq_sum all) + e0=$(cat "$bat/energy_now") + st=$(cat "$bat/status") + t0=$(date +%s.%N) + rtcwake -d rtc1 -m freeze -s "$secs" > /dev/null 2>&1 + t1=$(date +%s.%N) + b1=$(irq_sum "$irq"); a1=$(irq_sum all); e1=$(cat "$bat/energy_now") + dmesg > "$out/dmesg-$p-$mode.txt" + entries=$(grep -c "Timekeeping suspended" "$out/dmesg-$p-$mode.txt") + armed=$(grep -c "system sleep with wakeup armed" "$out/dmesg-$p-$mode.txt") + slept=$(awk -v a="$t0" -v b="$t1" 'BEGIN { printf "%.1f", b - a }') + echo "$mode $secs $slept $((b1 - b0)) $((a1 - a0)) $entries" \ + "$(((e0 - e1) / 1000)) $st $armed" | tee -a "$out/results.txt" + sleep 20 + done +done +echo "evidence: $out" diff --git a/tools/aurora-wake/hog-keyboard.py b/tools/aurora-wake/hog-keyboard.py new file mode 100755 index 0000000000000..d21f9e09d69bf --- /dev/null +++ b/tools/aurora-wake/hog-keyboard.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Emulated BLE HID-over-GATT keyboard on a BlueZ host, for unattended +Bluetooth wake tests of another machine. + +It behaves like a real BLE keyboard that went idle: it advertises only when a +key is "pressed" (a command on the control FIFO), and stops advertising as soon +as a central connects, so a central that disconnects it for suspend is not +woken again until the next key. + +Run as root: hog-keyboard.py [--ctl /run/hogkb/ctl] [--log /run/hogkb/log] +Commands (one per line, written to the FIFO): + pairable on|off adapter discoverable + pairable, advertise for pairing + key [usage [secs]] advertise if needed, wait up to secs (60) for the + central to connect and subscribe, then press+release + adv on|off advertise / stop (debug) + status log connection state + quit +""" +import argparse +import os +import subprocess +import sys +import threading +import time + +import dbus +import dbus.exceptions +import dbus.mainloop.glib +import dbus.service +from gi.repository import GLib + +BLUEZ = "org.bluez" +OM = "org.freedesktop.DBus.ObjectManager" +PROPS = "org.freedesktop.DBus.Properties" +GATT_MGR = "org.bluez.GattManager1" +SVC = "org.bluez.GattService1" +CHR = "org.bluez.GattCharacteristic1" +DSC = "org.bluez.GattDescriptor1" +AGENT = "org.bluez.Agent1" +ADV_INSTANCE = "1" + +# Boot-compatible keyboard with report ID 1: modifiers, reserved, 6 keys. +REPORT_MAP = bytes([ + 0x05, 0x01, 0x09, 0x06, 0xA1, 0x01, 0x85, 0x01, + 0x05, 0x07, 0x19, 0xE0, 0x29, 0xE7, 0x15, 0x00, 0x25, 0x01, + 0x75, 0x01, 0x95, 0x08, 0x81, 0x02, + 0x95, 0x01, 0x75, 0x08, 0x81, 0x01, + 0x95, 0x06, 0x75, 0x08, 0x15, 0x00, 0x25, 0x65, + 0x05, 0x07, 0x19, 0x00, 0x29, 0x65, 0x81, 0x00, + 0xC0, +]) + +LOG = None + + +def log(msg): + line = "%s %s" % (time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime()) + + ".%03d" % (int(time.time() * 1000) % 1000), msg) + print(line, flush=True) + if LOG: + with open(LOG, "a") as f: + f.write(line + "\n") + + +def ay(data): + return dbus.Array([dbus.Byte(b) for b in data], signature="y") + + +class Application(dbus.service.Object): + def __init__(self, bus): + self.path = "/org/aurora/hogkb" + self.services = [] + super().__init__(bus, self.path) + + @dbus.service.method(OM, out_signature="a{oa{sa{sv}}}") + def GetManagedObjects(self): + out = {} + for s in self.services: + out[s.path] = s.props() + for c in s.chars: + out[c.path] = c.props() + for d in c.descs: + out[d.path] = d.props() + return out + + +class Service(dbus.service.Object): + def __init__(self, bus, app, index, uuid): + self.path = "%s/service%d" % (app.path, index) + self.uuid = uuid + self.chars = [] + super().__init__(bus, self.path) + app.services.append(self) + + def props(self): + return {SVC: {"UUID": self.uuid, "Primary": True, + "Characteristics": dbus.Array([c.path for c in self.chars], signature="o")}} + + +class Characteristic(dbus.service.Object): + def __init__(self, bus, svc, index, uuid, flags, value=b""): + self.path = "%s/char%d" % (svc.path, index) + self.svc = svc + self.uuid = uuid + self.flags = flags + self.value = bytes(value) + self.descs = [] + self.notifying = False + super().__init__(bus, self.path) + svc.chars.append(self) + + def props(self): + return {CHR: {"Service": dbus.ObjectPath(self.svc.path), "UUID": self.uuid, + "Flags": self.flags, + "Descriptors": dbus.Array([d.path for d in self.descs], signature="o")}} + + @dbus.service.method(CHR, in_signature="a{sv}", out_signature="ay") + def ReadValue(self, options): + return ay(self.value) + + @dbus.service.method(CHR, in_signature="aya{sv}") + def WriteValue(self, value, options): + self.value = bytes(value) + + @dbus.service.method(CHR) + def StartNotify(self): + self.notifying = True + log("notify on %s" % self.uuid) + + @dbus.service.method(CHR) + def StopNotify(self): + self.notifying = False + log("notify off %s" % self.uuid) + + @dbus.service.signal(PROPS, signature="sa{sv}as") + def PropertiesChanged(self, interface, changed, invalidated): + pass + + def notify(self, value): + self.value = bytes(value) + self.PropertiesChanged(CHR, {"Value": ay(value)}, []) + + +class Descriptor(dbus.service.Object): + def __init__(self, bus, chrc, index, uuid, flags, value): + self.path = "%s/desc%d" % (chrc.path, index) + self.chrc = chrc + self.uuid = uuid + self.flags = flags + self.value = bytes(value) + super().__init__(bus, self.path) + chrc.descs.append(self) + + def props(self): + return {DSC: {"Characteristic": dbus.ObjectPath(self.chrc.path), "UUID": self.uuid, + "Flags": self.flags}} + + @dbus.service.method(DSC, in_signature="a{sv}", out_signature="ay") + def ReadValue(self, options): + return ay(self.value) + + +class Agent(dbus.service.Object): + """Just Works: accept everything (test peripheral only).""" + + @dbus.service.method(AGENT) + def Release(self): + pass + + @dbus.service.method(AGENT, in_signature="os") + def AuthorizeService(self, device, uuid): + log("authorize service %s %s" % (device, uuid)) + + @dbus.service.method(AGENT, in_signature="ou") + def RequestConfirmation(self, device, passkey): + log("confirm %s %06d" % (device, passkey)) + + @dbus.service.method(AGENT, in_signature="o") + def RequestAuthorization(self, device): + log("authorize %s" % device) + + @dbus.service.method(AGENT) + def Cancel(self): + log("agent cancel") + + +class Keyboard: + def __init__(self, bus, adapter): + self.bus = bus + self.adapter_path = adapter + self.adapter = dbus.Interface(bus.get_object(BLUEZ, adapter), PROPS) + self.app = Application(bus) + hid = Service(bus, self.app, 0, "1812") + Characteristic(bus, hid, 0, "2a4a", ["read"], [0x11, 0x01, 0x00, 0x02]) + Characteristic(bus, hid, 1, "2a4b", ["encrypt-read"], REPORT_MAP) + Characteristic(bus, hid, 2, "2a4c", ["write-without-response"], [0]) + Characteristic(bus, hid, 3, "2a4e", ["read", "write-without-response"], [1]) + self.report = Characteristic(bus, hid, 4, "2a4d", ["encrypt-read", "notify"], [0] * 8) + Descriptor(bus, self.report, 0, "2908", ["read"], [0x01, 0x01]) + bat = Service(bus, self.app, 1, "180f") + Characteristic(bus, bat, 0, "2a19", ["read", "notify"], [100]) + dis = Service(bus, self.app, 2, "180a") + # PnP ID: USB vendor source, Linux Foundation, test product. + Characteristic(bus, dis, 0, "2a50", ["read"], [0x02, 0x6b, 0x1d, 0x46, 0x02, 0x01, 0x00]) + self.index = adapter.rsplit("hci", 1)[1] + self.adv = False + self.connected = set() + mgr = dbus.Interface(bus.get_object(BLUEZ, adapter), GATT_MGR) + mgr.RegisterApplication(self.app.path, {}, + reply_handler=lambda: log("GATT application registered"), + error_handler=lambda e: log("GATT register failed: %s" % e)) + self.agent = Agent(bus, "/org/aurora/hogkb/agent") + am = dbus.Interface(bus.get_object(BLUEZ, "/org/bluez"), "org.bluez.AgentManager1") + am.RegisterAgent(self.agent._object_path, "NoInputNoOutput") + am.RequestDefaultAgent(self.agent._object_path) + bus.add_signal_receiver(self.props_changed, dbus_interface=PROPS, + signal_name="PropertiesChanged", path_keyword="path") + + def props_changed(self, interface, changed, invalidated, path=None): + if interface != "org.bluez.Device1" or "Connected" not in changed: + return + if changed["Connected"]: + self.connected.add(path) + log("connected %s" % path) + # A real keyboard stops advertising once its host is back. + self.set_adv(False) + else: + self.connected.discard(path) + log("disconnected %s" % path) + + def set_adv(self, on, discoverable=False): + """ + Legacy MGMT advertising through btmgmt: on some kernels bluetoothd's + extended advertising path rejects the data (Add Extended Advertising + Data: Invalid Parameters), while Add Advertising works. Instance 1, + connectable, HID service UUID; appearance and name in scan response. + """ + if on and not self.adv: + scan_rsp = "0319c103" + "0d09" + b"AuroraTestKB".hex() + cmd = ["btmgmt", "--index", self.index, "add-adv", "-c", "-u", "1812", + "-s", scan_rsp] + if discoverable: + cmd.append("-g") + ret = subprocess.run(cmd + [ADV_INSTANCE], capture_output=True, text=True, + timeout=10, stdin=subprocess.PIPE) + self.adv = "Instance added" in ret.stdout + log("advertising" if self.adv else "advertise failed: %s" % ret.stdout.strip()) + elif not on and self.adv: + subprocess.run(["btmgmt", "--index", self.index, "rm-adv", ADV_INSTANCE], + capture_output=True, timeout=10, stdin=subprocess.PIPE) + self.adv = False + log("advertising stopped") + + def pairable(self, on): + self.adapter.Set("org.bluez.Adapter1", "Pairable", dbus.Boolean(on)) + self.adapter.Set("org.bluez.Adapter1", "Discoverable", dbus.Boolean(on)) + self.set_adv(False) + if on: + self.set_adv(True, discoverable=True) + log("pairable %s" % on) + + def key(self, usage, deadline): + """Advertise until a central is connected and subscribed, then type.""" + if not (self.connected and self.report.notifying): + if not self.adv: + self.set_adv(True) + if time.monotonic() < deadline: + GLib.timeout_add(50, self.key, usage, deadline) + else: + log("key %#x: no central subscribed before the deadline" % usage) + self.set_adv(False) + return False + self.report.notify([0, 0, usage, 0, 0, 0, 0, 0]) + GLib.timeout_add(30, lambda: (self.report.notify([0] * 8), False)[1]) + log("key %#x sent" % usage) + return False + + +def fifo_reader(path, kb): + while True: + with open(path) as f: + for line in f: + words = line.split() + if not words: + continue + GLib.idle_add(command, kb, words) + + +def command(kb, words): + cmd = words[0] + log("command: %s" % " ".join(words)) + if cmd == "pairable": + kb.pairable(words[1:] == ["on"]) + elif cmd == "adv": + kb.set_adv(words[1:] == ["on"]) + elif cmd == "key": + usage = int(words[1], 0) if len(words) > 1 else 0x2c # space + wait = float(words[2]) if len(words) > 2 else 60 + kb.key(usage, time.monotonic() + wait) + elif cmd == "status": + log("connected=%s notifying=%s advertising=%s" % + (sorted(kb.connected), kb.report.notifying, kb.adv)) + elif cmd == "quit": + kb.set_adv(False) + loop.quit() + return False + + +def main(): + global LOG, loop + ap = argparse.ArgumentParser() + ap.add_argument("--ctl", default="/run/hogkb/ctl") + ap.add_argument("--log", default="/run/hogkb/log") + ap.add_argument("--adapter", default="/org/bluez/hci0") + args = ap.parse_args() + os.makedirs(os.path.dirname(args.ctl), exist_ok=True) + if not os.path.exists(args.ctl): + os.mkfifo(args.ctl, 0o600) + LOG = args.log + dbus.mainloop.glib.DBusGMainLoop(set_as_default=True) + bus = dbus.SystemBus() + kb = Keyboard(bus, args.adapter) + threading.Thread(target=fifo_reader, args=(args.ctl, kb), daemon=True).start() + loop = GLib.MainLoop() + log("ready, control FIFO %s" % args.ctl) + loop.run() + + +if __name__ == "__main__": + sys.exit(main()) From 61b6880a7f93db2a5083193e73a7ad77d0e8858c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:22:41 +1100 Subject: [PATCH 1354/2680] accel/ane: Check that handles[0] holds every SUBMIT task descriptor SUBMIT has the engine run td_count task descriptors of td_size bytes. Userspace puts them in the first tsk_size bytes of handles[0], ahead of the weights, and passes a copy of the first one in the bootstrap buffer, where the engine starts. libane does exactly this: it copies the whole program into handles[0] and the first descriptor into a bootstrap buffer of one tile. The ioctl checks tsk_size against handles[0] and td_size against the bootstrap buffer, but nothing ties td_count to the buffer that holds the descriptors. A client can submit a one-page handles[0] with td_size = 4 and td_count = 65535, which describes a 256 KiB descriptor array in a 16 KiB buffer. An engine that follows that layout reads past the end of the buffer: into a DART fault, which wedges the engine until recovery, or into whatever the DART maps next. Check the three sizes in one helper: td_count * td_size, computed with an overflow check, must fit in tsk_size, tsk_size must stay smaller than handles[0], and td_size must fit in the bootstrap buffer. Bounding the descriptor array by the bootstrap buffer instead would refuse every libane program with more than one tile of descriptors. Describe the layout in the UAPI header and add KUnit tests for the exact fits, one word over each limit and a product that overflows. Fixes: 3020d255f1b1 ("accel/ane: Add the Apple Neural Engine driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/accel/ane/Kconfig | 12 ++++ drivers/accel/ane/Makefile | 1 + drivers/accel/ane/ane.h | 5 ++ drivers/accel/ane/ane_drv.c | 31 +++++++-- drivers/accel/ane/ane_drv_test.c | 105 +++++++++++++++++++++++++++++++ include/uapi/drm/ane_accel.h | 8 ++- 6 files changed, 155 insertions(+), 7 deletions(-) create mode 100644 drivers/accel/ane/ane_drv_test.c diff --git a/drivers/accel/ane/Kconfig b/drivers/accel/ane/Kconfig index a1e8766340621..a14e621707a54 100644 --- a/drivers/accel/ane/Kconfig +++ b/drivers/accel/ane/Kconfig @@ -21,3 +21,15 @@ config DRM_ACCEL_ANE To compile this driver as modules, choose M here: the module for the M1 family will be called ane, and the module for the M2 family will be called ane_t6021. + +config DRM_ACCEL_ANE_KUNIT_TEST + tristate "KUnit tests for the Apple Neural Engine driver" if !KUNIT_ALL_TESTS + depends on DRM_ACCEL_ANE && KUNIT + default KUNIT_ALL_TESTS + help + Build KUnit tests for the Apple Neural Engine driver. They check + the buffer sizes that DRM_IOCTL_ANE_SUBMIT accepts against the + task descriptor layout that userspace builds, at the limits, one + past them and with a size product that overflows. + + If unsure, say N. diff --git a/drivers/accel/ane/Makefile b/drivers/accel/ane/Makefile index c0ad4a43bc0c3..8103e33a112a4 100644 --- a/drivers/accel/ane/Makefile +++ b/drivers/accel/ane/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_DRM_ACCEL_ANE) += ane.o ane_t6021.o +obj-$(CONFIG_DRM_ACCEL_ANE_KUNIT_TEST) += ane_drv_test.o ane-y := \ ane_drv.o \ diff --git a/drivers/accel/ane/ane.h b/drivers/accel/ane/ane.h index c9d18fb13f521..1c4dc618c5c9f 100644 --- a/drivers/accel/ane/ane.h +++ b/drivers/accel/ane/ane.h @@ -147,6 +147,11 @@ struct ane_request { u32 bar[DRM_ANE_TILE_COUNT]; }; +#if IS_ENABLED(CONFIG_KUNIT) +int ane_submit_check_sizes(const struct drm_ane_submit *args, size_t cmd_size, + size_t btsp_size); +#endif + /* * Reclaim wedge-preserved BO ranges (unmap, release nodes and pages). * Only legal once DMA is provably quiescent: after a successful diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index c98f2348e82f1..05a5c85be2019 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -1,12 +1,14 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright 2022 Eileen Yoon */ +#include #include #include #include #include #include #include +#include #include #include #include @@ -583,6 +585,26 @@ static int ane_bo_free(struct drm_device *drm, void *data, return 0; } +/* + * SUBMIT runs a chain of td_count task descriptors of td_size bytes. They lie + * in the first tsk_size bytes of handles[0], ahead of the weights, and the + * engine starts from a copy of the first descriptor in the bootstrap buffer. + * Check that each buffer holds what the engine is told to read from it. + */ +VISIBLE_IF_KUNIT int ane_submit_check_sizes(const struct drm_ane_submit *args, + size_t cmd_size, size_t btsp_size) +{ + u32 span; + + if (check_mul_overflow(args->td_count, args->td_size, &span) || + span > args->tsk_size || args->tsk_size >= cmd_size || + args->td_size > btsp_size) + return -EINVAL; + + return 0; +} +EXPORT_SYMBOL_IF_KUNIT(ane_submit_check_sizes); + static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) { struct ane_device *ane = drm->dev_private; @@ -624,9 +646,7 @@ static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) goto put; } gem[bdx] = &bo->base; - if (!bo->iova || - (bdx == CMD_BUF_BDX && - args->tsk_size >= (bo->npages << ane->shift))) { + if (!bo->iova) { err = -EINVAL; goto put; } @@ -647,10 +667,13 @@ static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) goto put; } btsp = &bo->base; - if (!bo->iova || args->td_size > bo->base.size) { + if (!bo->iova) { err = -EINVAL; goto put; } + err = ane_submit_check_sizes(args, gem[CMD_BUF_BDX]->size, btsp->size); + if (err) + goto put; req.btsp_iova = lower_32_bits(bo->iova); /* diff --git a/drivers/accel/ane/ane_drv_test.c b/drivers/accel/ane/ane_drv_test.c new file mode 100644 index 0000000000000..1c687b1476955 --- /dev/null +++ b/drivers/accel/ane/ane_drv_test.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * KUnit tests for the buffer size checks of DRM_IOCTL_ANE_SUBMIT. + */ + +#include +#include + +#include "ane.h" + +struct ane_submit_size_case { + const char *desc; + u64 tsk_size; + u32 td_count; + u32 td_size; + size_t cmd_size; + size_t btsp_size; + int ret; +}; + +static const struct ane_submit_size_case ane_submit_size_cases[] = { + { + /* libane: the program in handles[0], one tile of bootstrap */ + .desc = "100 descriptors of 0x274 bytes", + .tsk_size = 100 * 0x274, .td_count = 100, .td_size = 0x274, + .cmd_size = SZ_64K + SZ_16K, .btsp_size = SZ_16K, + }, + { + .desc = "descriptors fill the task region", + .tsk_size = 0x1000, .td_count = 0x400, .td_size = 4, + .cmd_size = SZ_16K, .btsp_size = SZ_16K, + }, + { + .desc = "one descriptor past the task region", + .tsk_size = 0x1000, .td_count = 0x401, .td_size = 4, + .cmd_size = SZ_16K, .btsp_size = SZ_16K, .ret = -EINVAL, + }, + { + .desc = "65535 descriptors in one page", + .tsk_size = 0x100, .td_count = 0xffff, .td_size = 4, + .cmd_size = SZ_16K, .btsp_size = SZ_16K, .ret = -EINVAL, + }, + { + .desc = "task region one byte short of the buffer", + .tsk_size = SZ_16K - 1, .td_count = 1, .td_size = 4, + .cmd_size = SZ_16K, .btsp_size = SZ_16K, + }, + { + .desc = "task region as large as the buffer", + .tsk_size = SZ_16K, .td_count = 1, .td_size = 4, + .cmd_size = SZ_16K, .btsp_size = SZ_16K, .ret = -EINVAL, + }, + { + .desc = "first descriptor fills the bootstrap buffer", + .tsk_size = SZ_16K, .td_count = 1, .td_size = SZ_16K, + .cmd_size = SZ_32K, .btsp_size = SZ_16K, + }, + { + .desc = "first descriptor one word past the bootstrap buffer", + .tsk_size = SZ_16K + 4, .td_count = 1, .td_size = SZ_16K + 4, + .cmd_size = SZ_32K, .btsp_size = SZ_16K, .ret = -EINVAL, + }, + { + .desc = "largest descriptor array below 4 GiB", + .tsk_size = 0x3fffULL * 0x40000, .td_count = 0x3fff, + .td_size = 0x40000, .cmd_size = SZ_4G, .btsp_size = 0x40000, + }, + { + .desc = "descriptor array of 4 GiB", + .tsk_size = SZ_4G, .td_count = 0x4000, .td_size = 0x40000, + .cmd_size = SZ_4G + SZ_16K, .btsp_size = 0x40000, + .ret = -EINVAL, + }, +}; + +KUNIT_ARRAY_PARAM_DESC(ane_submit_size, ane_submit_size_cases, desc); + +static void ane_submit_sizes_test(struct kunit *test) +{ + const struct ane_submit_size_case *c = test->param_value; + struct drm_ane_submit args = { + .tsk_size = c->tsk_size, + .td_count = c->td_count, + .td_size = c->td_size, + }; + + KUNIT_EXPECT_EQ(test, ane_submit_check_sizes(&args, c->cmd_size, + c->btsp_size), c->ret); +} + +static struct kunit_case ane_submit_test_cases[] = { + KUNIT_CASE_PARAM(ane_submit_sizes_test, ane_submit_size_gen_params), + {} +}; + +static struct kunit_suite ane_submit_test_suite = { + .name = "ane-submit", + .test_cases = ane_submit_test_cases, +}; + +kunit_test_suite(ane_submit_test_suite); + +MODULE_DESCRIPTION("KUnit tests for the Apple Neural Engine SUBMIT checks"); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/include/uapi/drm/ane_accel.h b/include/uapi/drm/ane_accel.h index 76577277b70eb..4e16e1771b53c 100644 --- a/include/uapi/drm/ane_accel.h +++ b/include/uapi/drm/ane_accel.h @@ -108,14 +108,16 @@ struct drm_ane_bo_free { /** * struct drm_ane_submit - DRM_IOCTL_ANE_SUBMIT argument (ABI version 1) - * @tsk_size: In. Size of the task microcode at the start of handles[0], not - * zero. + * @tsk_size: In. Size of the task microcode at the start of handles[0]: the + * @td_count task descriptors of @td_size bytes. Not zero, at least + * @td_count times @td_size, and smaller than handles[0]. * @td_count: In. Number of task descriptors, 1 to 0xffff. * @td_size: In. Size of a task descriptor: a multiple of 4, 4 to 0x40000. * @handles: In. Buffer per tile: handles[0] holds the microcode and the * weights, handles[1] must be 0 (the driver sets it from handles[0]), * and 0 marks an unused tile. - * @btsp_handle: In. Buffer with the bootstrap task descriptor. + * @btsp_handle: In. Buffer with a copy of the first task descriptor, at + * least @td_size bytes. The engine starts from this copy. * @pad: Must be zero. * * A successful SUBMIT returns after the engine completed the work, with the From d0420c2d4d54e97b4e8ba1284992bad8463d75b6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:56:17 +1100 Subject: [PATCH 1355/2680] accel/ane: Describe the hardware without test-system history Many comments in the driver record where, when and in which internal step a behaviour was observed, rather than what the hardware does: the names of test machines, dates, paths to notes outside the tree, review and work-item names, and commit hashes and module builds that are not public, along with benchmark results tied to single systems. None of it helps a reader of the code, and none of it can be checked from the tree. Reword these comments to state the hardware behaviour and the reason for each barrier, bound and workaround, and drop what only records history. Where such a passage also described the code wrongly, follow the code: the T6021 boot unit does perform the boot writes, probe checks only the ane_cpu island, the IPC surface threshold is implemented and the heap floor is the DART page size. The module parameter descriptions and one log message lose their "lab" and "proven configuration" wording. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/accel/ane/ane.h | 12 +- drivers/accel/ane/ane_boost.c | 19 +- drivers/accel/ane/ane_dart.c | 18 +- drivers/accel/ane/ane_drv.c | 70 ++-- drivers/accel/ane/ane_fw_validate.h | 40 +-- drivers/accel/ane/ane_stats.h | 9 +- drivers/accel/ane/ane_t6021.h | 202 +++++------ drivers/accel/ane/ane_t6021_boot.c | 362 ++++++++------------ drivers/accel/ane/ane_t6021_boot.h | 186 +++++----- drivers/accel/ane/ane_t6021_fwload.c | 151 ++++---- drivers/accel/ane/ane_t6021_rtclient_main.c | 225 ++++++------ drivers/accel/ane/ane_tm.c | 36 +- 12 files changed, 583 insertions(+), 747 deletions(-) diff --git a/drivers/accel/ane/ane.h b/drivers/accel/ane/ane.h index 1c4dc618c5c9f..f974f8a31194e 100644 --- a/drivers/accel/ane/ane.h +++ b/drivers/accel/ane/ane.h @@ -38,9 +38,9 @@ struct ane_device { /* The ane SET block (m1n1 ANE.ps_map): pmgr power-state words for * this engine, mapped at probe on every SoC. Read-only, always: - * pmgr reads give recovery its ACTUAL evidence, while direct SET - * writes are firmware-locked and external-abort the SoC (T6001 - * netconsole-named 2026-09-16; T8103 same mechanism). + * pmgr reads give recovery the ACTUAL power state, while direct + * SET writes are firmware-locked and external-abort the SoC + * (T6001 and T8103 alike). */ phys_addr_t ps_base; void __iomem *ps; @@ -103,9 +103,9 @@ struct ane_device { /* * SoC descriptor fact: the set0/base islands hold the tm/tq * register file in retention through any genpd power cycle - * (T6001 bisect evidence 2026-09-16: TQ_EN still reads 0x3000 - * after a 2 s gate), so recovery must drain retained task-manager - * state itself instead of expecting a power-on reset to clear it. + * (on T6001 TQ_EN still reads 0x3000 after a 2 s gate), so + * recovery must drain retained task-manager state itself + * instead of expecting a power-on reset to clear it. * False on T8103, where the cycle is a full POR of the file. */ bool tm_retention; diff --git a/drivers/accel/ane/ane_boost.c b/drivers/accel/ane/ane_boost.c index a13d706f4bc3c..6b6d0791518dc 100644 --- a/drivers/accel/ane/ane_boost.c +++ b/drivers/accel/ane/ane_boost.c @@ -3,17 +3,14 @@ * Engine-busy CPU cluster boost. * * Bandwidth-bound programs run at a speed set by the CPU cluster - * p-states. Measured on jwm1 (T8103, 2026-09-25, ane-linux-experiments - * receipt jwm1-ane-dvfs-boost): with a mostly-sleeping submitter at - * normal priority, 75 of 489 Qwen decode steps take 100-175 ms instead - * of 70, every program in a slow step ~2x its normal wall, the - * submitter's CPU time and run delay unchanged; both clusters pinned - * at their top p-state (performance governor, or SCHED_FIFO for the - * submitter, which schedutil boosts the same way) -> 0 of 489. The - * whole-encoder Parakeet program (140 ms) moves too once the QoS lapses - * mid-submit: timed from submit start, the hold dropped 100 ms into - * every encoder run, median 171 ms (140-238); held to completion, 138.0 - * (137.7-138.7), jwm1 2026-09-25. The likely + * p-states. On T8103, with a mostly-sleeping submitter at normal + * priority, some submits run about twice as long as normal, while the + * submitter's CPU time and run delay stay unchanged; with both clusters + * pinned at their top p-state (performance governor, or SCHED_FIFO for + * the submitter, which schedutil boosts the same way) the slow submits + * disappear. A single long program slows down as well if the QoS + * lapses before it completes, so the hold must last from submit start + * to completion, and the idle delay counts from completion. The likely * mechanism is the second p-state field apple-soc-cpufreq writes with * the cluster p-state on T8103 only (DVFS_CMD PS2, has_ps2); the * memory-controller tuning its author named as unimplemented is still diff --git a/drivers/accel/ane/ane_dart.c b/drivers/accel/ane/ane_dart.c index 0dc0e955182da..f99d5c5b5c0c1 100644 --- a/drivers/accel/ane/ane_dart.c +++ b/drivers/accel/ane/ane_dart.c @@ -174,7 +174,7 @@ bool ane_dart_faulted(struct ane_device *ane, u64 *iova, u32 *status) * A fault is a missing PTE, nothing more: the page tables are provider * owned and unchanged, so the stream-restore apple-dart uses at reset * time is the wrong tool mid-job (register surgery while the faulting - * transaction is retried is what killed jwm1 twice). Instead the faulted + * transaction is retried takes the machine down). Instead the faulted * IOVA is handed a throwaway DART page, so the retried transaction * completes and the engine drains on its own; the scratch pages are * unmapped once the request has been failed cleanly. @@ -184,14 +184,14 @@ bool ane_dart_faulted(struct ane_device *ane, u64 *iova, u32 *status) * allocator knows nothing about a raw iommu_map. A scratch PTE that * outlives its release path (any future bug on the wedge side) would * then sit, untracked, in a range the allocator considers free: the - * next BO_INIT maps onto it and dies in dart_init_pte's "we require an - * unmap first" WARN (m1-test-host 2026-09-25, iommu_map failed at 0x4000). So - * the scratch range is RESERVED in the allocator first: a node-tracked - * mapping can never collide with a later BO_INIT, and a scratch whose - * release is skipped degrades into a bounded, benign leak instead of a - * poisoned address space. Reserve failure means a live BO node already - * owns the faulting page (a wild descriptor, not a hole): refuse and - * let the caller wedge, fail closed. + * next BO_INIT maps onto it and fails with dart_init_pte's "we require + * an unmap first" WARN. So the scratch range is RESERVED in the + * allocator first: a node-tracked mapping can never collide with a + * later BO_INIT, and a scratch whose release is skipped degrades into + * a bounded, benign leak instead of a poisoned address space. Reserve + * failure means a live BO node already owns the faulting page (a wild + * descriptor, not a hole): refuse and let the caller wedge, fail + * closed. */ int ane_dart_drain_fault(struct ane_device *ane, u64 fault_iova, struct ane_dart_scratch *scratch, int *pages) diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 05a5c85be2019..1b61d3f981bd9 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -29,35 +29,33 @@ * map_mode) and read at BO-mmap/iommu-map time: * bit 0 — mark DART descriptors IOMMU_CACHE (cacheable, inner-shareable) * bit 1 — map the CPU vma cacheable instead of write-combine - * 0 = today's WC CPU vmas + NC descriptors; 3 = both cacheable (device - * accesses cohere with the CPU caches). Each axis is toggleable alone for - * attribution. + * 0 = WC CPU vmas + NC descriptors; 3 = both cacheable (device + * accesses cohere with the CPU caches). Each axis is toggleable alone. * - * Coherency evidence for mode 3, both SoCs: - * - T6001/t6001-test-host (apple,t6000-dart x3, APPLE_DART2 PTEs): (1) the ANE sits in + * Why mode 3 is coherent, on both SoCs: + * - T6001 (apple,t6000-dart x3, APPLE_DART2 PTEs): (1) the ANE sits in * the dma-coherent DART class every SoC DMA-API device uses on * arm64/Asahi, where dma-iommu itself programs descriptors with - * IOMMU_CACHE; (2) eiln's pre-GEM driver (4566c89^) mapped BOs cacheable + * IOMMU_CACHE; (2) eiln's pre-GEM driver mapped BOs cacheable * on the CPU while DMAing them through this same iommu_map() path for - * months with no maintenance; (3) empirically, an alternating-pattern - * 16-rep buffer-reuse battery over 6 programs in both directions is - * byte-identical WC+NC vs cached on every rep — stale cache lines would - * surface as the previous pattern's bytes. - * - T8103/m1-test-host (apple,t8103-dart x3, APPLE_DART1 PTEs): the DART1 PTE + * months with no maintenance; (3) buffers reused with alternating + * patterns in both directions read back byte-identical under WC+NC + * and cached mappings — stale cache lines would surface as the + * previous pattern's bytes. + * - T8103 (apple,t8103-dart x3, APPLE_DART1 PTEs): the DART1 PTE * format has no cacheability field (io-pgtable-dart.c dart_prot_to_pte), * so bit 0 is a structural no-op there and the lever is the cached CPU - * vma alone. eiln's pre-GEM precedent ran exactly that on M1 for months. - * The same reuse battery is byte-identical (14/14) and the certified - * Parakeet E2E pins hold bit-for-bit with ane_exec 2561.2 -> 2402.0 ms. + * vma alone. eiln's pre-GEM driver ran exactly that on M1 for months, + * and the same buffer-reuse check reads back byte-identical. * - * Default 3 (cached both sides). 0 is the documented rollback: runtime via - * /sys/module/ane/parameters/map_mode, or by loading the retained prior - * module (ane-96d5a88.ko on both hosts). + * Default 3 (cached both sides). 0 restores write-combine CPU vmas and + * non-cacheable descriptors at runtime via + * /sys/module/ane/parameters/map_mode. */ static int map_mode = 3; module_param(map_mode, int, 0644); MODULE_PARM_DESC(map_mode, - "BO mapping: bit0=IOMMU_CACHE DART descriptors, bit1=cacheable CPU vmas (default 3 = cached; 0 = writecombine + non-cacheable rollback)"); + "BO mapping: bit0=IOMMU_CACHE DART descriptors, bit1=cacheable CPU vmas (default 3 = cached; 0 = writecombine + non-cacheable fallback)"); /* * Producer-side stats (ane_stats sysfs, ane_timeline debugfs), read @@ -138,8 +136,8 @@ static unsigned long ane_iommu_clear_range(struct ane_device *ane, u64 start, * inserted. A VALID PTE in a just-reserved range is therefore a stray * from a torn-down session or a lost unmap. Both map paths clear it once * and retry instead of poisoning every later BO_INIT until reload - * (m1-test-host 2026-09-25: dart_init_pte -EEXIST at 0x4000 for the rest - * of the session). + * (dart_init_pte would return -EEXIST at that IOVA for the rest of the + * session). */ static int ane_iommu_map_each(struct ane_device *ane, struct ane_bo *bo, int prot) @@ -440,8 +438,8 @@ static void ane_gem_free_object(struct drm_gem_object *gem) * DMA. The range's node stays reserved and tracked * until recovery quiesces the engine and reclaims * it, so no later BO_INIT can map over these PTEs - * (a leaked-but-forgotten mapping is what turned - * into dart_init_pte -EEXIST WARNs on m1-test-host). + * (a leaked-but-forgotten mapping turns into + * dart_init_pte -EEXIST WARNs). */ dev_err(ane->dev, "wedged: preserving bo mapping\n"); list_del_init(&bo->node); @@ -866,8 +864,7 @@ static long ane_drm_ioctl(struct file *file, unsigned int cmd, * _IOC_TYPEMASK is the low 8 bits, which encode the *nr*, not the * *type*. The type byte lives at offset _IOC_TYPESHIFT (8). Use * _IOC_TYPE(cmd) to extract the right field; misuse here returns - * -ENOTTY for every legitimate ane_ioctl, which is exactly the - * regression committed and reverted in the lifecycle-reset series. + * -ENOTTY for every legitimate ane_ioctl. */ if (_IOC_TYPE(cmd) != DRM_IOCTL_BASE || (_IOC_NR(cmd) < DRM_COMMAND_BASE && @@ -1102,11 +1099,11 @@ static int ane_attach_genpd(struct ane_device *ane) * device tree by design: DART topology from "iommus", power wiring from * "power-domains", the engine window from "reg", the IRQ by name. The SET * block is the one exception: the device tree carries no range for it, so - * it is compiled in per SoC, and only from a source that has proven the + * it is compiled in per SoC, and only from a source that confirms the * address on hardware (m1n1 proxyclient ANE.ps_map, then a bound device). * A guessed SET base is not a bug but a brick: direct writes to the - * block external-abort the SoC (T6001 named by netconsole 2026-09-16, - * PS_SET0 down at 0x28e08c000; T8103 same mechanism at 0x23b70c000). + * block external-abort the SoC (T6001 at 0x28e08c000 with PS_SET0 down; + * T8103 same mechanism at 0x23b70c000). * The M2 family (T602x, T8112) runs the ANE firmware and binds to * ane_t6021 instead. */ @@ -1119,15 +1116,15 @@ struct ane_soc { }; static const struct ane_soc ane_soc_t8103 = { - /* M1 (m1-test-host). SET block mapped read-only for the recovery ACTUAL - * log and the powered-on guard. Execution proven in the fleet. + /* M1. SET block mapped read-only for the recovery ACTUAL log and the + * powered-on guard. */ .ps_base = 0x23b70c000ULL, }; static const struct ane_soc ane_soc_t6000 = { /* M1 Pro, M1 Max and M1 Ultra die 0 share this compatible and SET base - * (proven on T6001/t6001-test-host); M1 Pro and M1 Ultra are untested. + * (verified on T6001); M1 Pro and M1 Ultra are untested. */ .ps_base = 0x28e08c000ULL, .tm_retention = true, @@ -1207,9 +1204,9 @@ static int ane_platform_probe(struct platform_device *pdev) goto detach_genpd; } - /* m1n1 keys the SET block by ADT node name (ANE.ps_map); the live - * overlays carry no range for it, so the SoC descriptor picks the - * base (see the tier table above). Mapped read-only, always: + /* m1n1 keys the SET block by ADT node name (ANE.ps_map); the device + * tree carries no range for it, so the SoC descriptor picks the + * base (see struct ane_soc above). Mapped read-only, always: * recovery logs its ACTUAL nibbles beside every engine write and * refuses engine MMIO unless the islands read powered on. Unmapped * is tolerated: recovery then just skips the check. @@ -1349,10 +1346,9 @@ static int __maybe_unused ane_runtime_resume(struct device *dev) if (first) { /* Bisect order, each stage named before it runs: genpd * raise is already complete when this callback runs; the - * SET window reads come first (pmgr class, always safe - * per T6001/T8103 bisect evidence), then the engine. A - * hard reset after the last off-box line names the - * killing access exactly. + * SET window reads come first (pmgr class, always safe on + * T6001 and T8103), then the engine. A hard reset after + * the last line logged names the killing access exactly. */ struct platform_device *pdev = to_platform_device(dev); struct resource *eng; diff --git a/drivers/accel/ane/ane_fw_validate.h b/drivers/accel/ane/ane_fw_validate.h index 3f8e82978b4de..0d22a94b8ffb9 100644 --- a/drivers/accel/ane/ane_fw_validate.h +++ b/drivers/accel/ane/ane_fw_validate.h @@ -3,28 +3,26 @@ * ane_fw_validate.h — pinned ANE firmware images, their validator, and * iBoot's runtime patches. * - * Single source of truth shared by the kernel loader - * (ane_t6021_fwload.c) and the offline regression - * (tools/h14_boot_regression.c). Strict EXACT-image assertions (no + * Single source of truth for the kernel loader (ane_t6021_fwload.c); + * it also builds in userspace. Strict EXACT-image assertions (no * generic Mach-O parsing). * - * The images are the macOS 13.5 (22G74) payloads that iBoot preloads on - * the stub every Omarchy install boots: selene for T6020/T6021/T6022 - * (byte-verified live at SEG0 on T6021, notebook 20260926T230146Z) and - * bia for T8112 (receipts/2026-10-01-t8112-ane). Both have 5 load - * commands (2 segments + LC_SYMTAB + LC_UUID + LC_UNIXTHREAD), entry 0. + * The images are the macOS 13.5 (22G74) payloads that iBoot preloads + * for a 13.5 boot stub: selene for T6020/T6021/T6022 and bia for + * T8112. Both have 5 load commands (2 segments + LC_SYMTAB + LC_UUID + + * LC_UNIXTHREAD), entry 0. * * Userspace consumers typedef u8/u32/u64/size_t/bool and provide - * get_unaligned_le32/64, memcmp and memcpy (see that tool's source); - * kernel consumers get these from linux/types.h + linux/string.h. - * This header intentionally includes nothing. + * get_unaligned_le32/64, memcmp and memcpy; kernel consumers get these + * from linux/types.h + linux/string.h. This header intentionally + * includes nothing. */ #ifndef __ANE_FW_VALIDATE_H__ #define __ANE_FW_VALIDATE_H__ -/* FWIM surface size = 0x500000 bytes (Main - * 2026-09-20, audit 751caa4, T6021). Covers the selene vmsize 0x4fc000 - * and the bia vmsize 0x4ec000; NOT derivable from the blob length. +/* FWIM surface size = 0x500000 bytes (T6021). Covers the selene vmsize + * 0x4fc000 and the bia vmsize 0x4ec000; NOT derivable from the blob + * length. */ #define ANE_FW_BUF_SIZE 0x500000 #define ANE_FW_ENTRY_PC 0x0 @@ -252,10 +250,9 @@ ane_fw_validate_blob(const u8 *blob, size_t size, /* * iBoot runtime patches. iBoot places the image at its Mach-O vm layout * (TEXT vm 0, DATA vm segs[1].vmaddr) and then writes the fields below. - * The 17 pre-Linux T6021 captures (receipts/2026-10-01-t602x-independent) - * differ from the archive in these bytes only; only the stack guard - * changes between boots. A driver that runs its own copy writes them - * itself. + * Images captured on T6021 before Linux runs differ from the archive in + * these bytes only; only the stack guard changes between boots. A + * driver that runs its own copy writes them itself. * * Each __rtk_patch_* record is {u32 tag, u32 value length, value}. */ @@ -269,10 +266,9 @@ ane_fw_validate_blob(const u8 *blob, size_t size, * whose key <= the chip revision: header {type 1, flags 3 (20-byte * records), capacity 0x24, count, u32 key}, then {u32 offset, u64 mask, * u64 value} per record (iBoot table offsets carry flag bit 30; the - * block does not). Source: the 25G83 iBoot's records - * (receipts/2026-10-01-t8112-optin). On T6021 the block that the 13.5 - * iBoot leaves in the preloaded image (the captures) equals the 25G83 - * records exactly. + * block does not). Source: the 25G83 iBoot's records. On T6021 the + * block that the 13.5 iBoot leaves in the preloaded image (the + * captures) equals the 25G83 records exactly. */ struct ane_asc_tunable { u32 off; diff --git a/drivers/accel/ane/ane_stats.h b/drivers/accel/ane/ane_stats.h index 768fe97846edc..7a294609dc261 100644 --- a/drivers/accel/ane/ane_stats.h +++ b/drivers/accel/ane/ane_stats.h @@ -3,7 +3,7 @@ * Producer-side ANE counters and submission ring, shared by ane.ko and * ane_t6021.ko. * - * Contract: + * Interface: * - sysfs under /sys/class/accel/accel/device/ane_stats, * mode 0444, ASCII "key value" lines, integers only: busy_ns * (cumulative u64), jobs (cumulative u64). busy_ns counts only @@ -358,7 +358,7 @@ static inline int ane_timeline_show(struct seq_file *m, void *v) #define ANE_STATS_RING_ORDER_DEFAULT 8 /* 256 slots */ /* - * Accounting rule (coreglass producer contract): jobs counts completed + * Accounting rule: jobs counts completed * submissions, one begin/complete pair per engine submission — one per * ANE_SUBMIT on ane.ko (so libane ane_exec is 1 job and ane_exec_loop * with N iterations is N jobs) and one per firmware PROCEDURE_CALL @@ -375,9 +375,8 @@ static inline int ane_timeline_show(struct seq_file *m, void *v) * show callbacks fetch the counters from their real drvdata type * (struct ane_device * on ane.ko, struct ane_rtclient * on * ane_t6021.ko) and pass &...->stats_ctrs here. The formatter never - * sees the device pointer, so the drvdata type confusion that shipped - * in the first round (reading the head of ane_device as counters) - * cannot compile again: there is no cast to remove. + * sees the device pointer, so a drvdata type confusion (reading the + * head of ane_device as counters) cannot compile: there is no cast. */ /* * Consistent counter snapshot across the tiny close/open transition: diff --git a/drivers/accel/ane/ane_t6021.h b/drivers/accel/ane/ane_t6021.h index 4de6c0dfe6b53..d50ed9e6873ec 100644 --- a/drivers/accel/ane/ane_t6021.h +++ b/drivers/accel/ane/ane_t6021.h @@ -2,49 +2,45 @@ /* T6021 (H14 / J414c) Apple Neural Engine — skeleton constants and types. * * Where the constants below come from (none of them needs a device write): - * - reg windows, IRQ, pmgr islands: ADT j414c (DeviceTree.j414cap.im4p, - * receipts/2026-09-18-t6021-engine-layout-mined §2), Linux translation - * +0x200000000 (proven class, pmgr low-32 match). + * - reg windows, IRQ, pmgr islands: ADT j414c (DeviceTree.j414cap.im4p), + * Linux translation +0x200000000 (the pmgr addresses match in the + * low 32 bits). * - ASC cpu block (+0x1400000 h14g) + MBI transport (SCRATCH, channel - * table, +0x1844000 doorbell): the W4-fix receipt plus the live reads - * and the handshake described below; m1n1 ASCRegs = t8103 cross-ref - * only. - * - RTKit MGMT protocol: Asahi rtkit.c semantics, u64 message halves as - * staged in omarchy-ane rtkit/h14_rtkit_hello.py (commit 6ad26b7). + * table, +0x1844000 doorbell): register reads and the handshake + * described below; m1n1 ASCRegs = t8103 cross-ref only. + * - RTKit MGMT protocol: Asahi rtkit.c semantics, u64 message halves. * - RTBuddy endpoint table + doorbell word: the per-endpoint ring sizes, - * fourccs and doorbell bits listed with ane_t6021_eps below - * (receipts/2026-09-18-h14-w2-protocol-decode §3). + * fourccs and doorbell bits listed with ane_t6021_eps below. * - * DT binding (driver + packaging/dt/t6021-ane.dts are the two halves): + * DT binding (Documentation/devicetree/bindings/npu/apple,ane.yaml): * compatible = "apple,t6021-ane" (also "apple,t6020-ane", the * T6022 die-0 "apple,t6022-ane" and "apple,t8112-ane": * struct ane_t602x_soc). The T6021 values follow; T8112 - * (packaging/dt/t8112-ane.dts) has engine 0x26a000000, - * pmgr 0x23b700000+0x18000, set 0x23b724000+0x4000, - * IRQ 520, seven power states 0xc008..0xc038, and a - * fourth window "fuse" (0x23d2c8060+8, chip revision). + * has engine 0x26a000000, pmgr 0x23b700000+0x18000, + * set 0x23b724000+0x4000, IRQ 520, seven power states + * 0xc008..0xc038, and a fourth window "fuse" + * (0x23d2c8060+8, chip revision). * reg/reg-names = "engine" (whole 32 MiB ADT range0, 0x284000000; * the H13-style +0x1c04000 engine delta does not exist - * on this SoC — it is never applied here; a first touch - * external-aborted, proven 2026-09-18), + * on this SoC — it is never applied here; an access + * there external-aborts), * "pmgr" (0x28e080000+0x4034, pmgr1,t6021 island words), - * "set" (0x28e08c000+0x4000, SET window — read-only by - * repo rule: direct SET writes external-abort the SoC) + * "set" (0x28e08c000+0x4000, SET window — read-only: + * direct SET writes external-abort the SoC) * interrupts = one AIC2 level-high interrupt named "ane" (raw 884; - * [INFERENCE W2 §3: the ANE MBI doorbell IRQ]. + * [INFERENCE: the ANE MBI doorbell IRQ]. * dart-ane0 carries raw 885, provider-owned, never * fetched here) * iommus = dart-ane0 streams (apple,t6020-dart/apple,t8110-dart * nodes). The ADT dart-ane0 is one hardware block with a * four-window quartet (0x85800000/85810000/85820000/ - * 85804000, each 0x4000); the installed overlay's - * three-node split (t6001-proven pattern) is kept. - * power-domains = eight, phase1 raise order (sys_mpm→td→base→set1..4, + * 85804000, each 0x4000); the device tree keeps the + * three-node split used on T6001. + * power-domains = eight, raised in order (sys_mpm→td→base→set1..4, * ane_cpu last): ane_sys_mpm@4000, ane_td@4008, * ane_base@4010, ane_set1..4@4018-4030, ane_cpu@2e0. - * ane_td/ane_base were the W3 chain gap (six consumed, - * block access reset the machine); see the - * 2026-09-19 init-sequence receipt §3. + * ane_td and ane_base are required: with only the + * other six raised, block access resets the machine. */ #ifndef __ANE_T6021_H__ @@ -63,7 +59,7 @@ #include "ane_t6021_boot.h" -/* reg windows (packaging/dt/t6021-ane.dts reg-names order) */ +/* reg windows (DT reg-names order) */ enum { ANE_T6021_REG_ENGINE, ANE_T6021_REG_PMGR, @@ -81,35 +77,34 @@ enum { /* ANE-block-relative ASC/RTBuddy addresses. CPU block = ANE+0x1400000 * for h14g: CPU_CONTROL is 0x1400044, without the 0x200000 offset of - * the h16g/h17/h18g parts (whose block is the 0x1600044 one that the - * phase-1 §2 note quoted). + * the h16g/h17/h18g parts (whose block is the 0x1600044 one). * RUN is set at 0x1400044 on this part. * CPU_STATUS +0x48 keeps the m1n1 ASCRegs +4 shape (0x1400048, polled * while the ASC CPU starts). * - * W10 (2026-09-19) CORRECTS the claim that stood here — that the m1n1 - * t8103 mailbox (INBOX_CTRL +0x8110 / INBOX0 +0x8800) "has NO h14g - * analog". It has one, and it is live: see ANE_ASC_MBOX_* below. The - * earlier live test read +0x1608114, the h16g base, which is simply - * the wrong address on this part. The mailbox belongs to the RTBuddy - * transport rather than to the ANE engine interface, which is why the - * engine-side constants never refer to 0x1408xxx. + * The m1n1 t8103 mailbox (INBOX_CTRL +0x8110 / INBOX0 +0x8800) has an + * h14g analog, and it is live: see ANE_ASC_MBOX_* below. +0x1608114, + * the h16g base, is simply the wrong address on this part. The + * mailbox belongs to the RTBuddy transport rather than to the ANE + * engine interface, which is why the engine-side constants never refer + * to 0x1408xxx. * - * W10 also read the CPU block for the first time, on a Linux boot with - * all eight pmgr domains at ACTUAL=0xf and the W8 grant applied: + * Before the firmware is started, on a Linux boot with all eight pmgr + * domains at ACTUAL=0xf and the grant tunables applied, the CPU block + * reads: * CPU_CONTROL 0x285400044 = 0x00000000 RUN=0 * CPU_STATUS 0x285400048 = 0x0000002a RUNNING=0 STOPPED=1 IDLE=1 * RVBAR 0x285050000 = 0x00000001 valid bit set, entry addr 0 - * The IOP has never been started and no firmware image is programmed, + * The IOP has not been started and no firmware image is programmed, * so nothing on this transport can answer the host until it is. */ #define ANE_ASC_CPU_CONTROL 0x1400044 /* RUN = BIT(4) */ #define ANE_ASC_CPU_STATUS 0x1400048 /* m1n1 R_CPU_STATUS shape */ #define ANE_ASC_RVBAR 0x1050000 /* fw entry | valid bit0 */ -#define ANE_ASC_EDPRCR 0x1010310 /* phase1 S2 whitelist */ +#define ANE_ASC_EDPRCR 0x1010310 /* read whitelist */ #define ANE_ASC_VERS 0x1840000 #define ANE_ASC_RTB_STATUS 0x1840088 /* polled: value < 2 */ -#define ANE_ASC_RTB_STATUS_UNK7C 0x184007c /* phase1 S2 whitelist */ +#define ANE_ASC_RTB_STATUS_UNK7C 0x184007c /* read whitelist */ /* ASC blocks iBoot names in the firmware's __rtk_patch records * (RTK_cpu_physical_address 0x285000000 and RTK_cpu_wrapper_physical_address @@ -118,8 +113,8 @@ enum { #define ANE_ASC_CPU_BASE 0x1000000 #define ANE_ASC_WRAPPER_BASE 0x1400000 -/* The real ASC mailbox, mapped live by W10 (read-only) at ASC+0x8000. - * Three independent sources agree on the offsets: the live DT sibling +/* The real ASC mailbox, at ASC+0x8000. + * Three independent sources agree on the offsets: the DT sibling * mbox@2a2408000 ("apple,t6020-asc-mailbox" / "apple,asc-mailbox-v4", * size 0x4000, IRQs send-empty/send-not-empty/recv-empty/ * recv-not-empty) fixes mailbox = asc_base + 0x8000; upstream @@ -127,13 +122,13 @@ enum { * offsets; m1n1 proxyclient/m1n1/hw/asc.py ASCRegs gives the same set * ASC-relative plus the R_MBOX_CTRL field layout. * - * Observed on jw14m2 (2026-09-19), both directions identical: + * Before the firmware is started, both directions read the same: * a2i_control 0x285408110 = 0x00020001 * i2a_control 0x285408114 = 0x00020001 * -> ENABLE=1 EMPTY=1 FULL=0 OVERFLOW=0 FIFOCNT=0 WPTR=0 RPTR=0 * Enabled, and never used: both pointer pairs are still at the origin, - * so not one message has crossed either way since reset. 600 polls of - * i2a_control over 30 s produced zero changes. + * so not one message has crossed either way since reset, and + * i2a_control does not change while polled. * * Receive is POP-ON-READ and 64-bit: upstream does * while (!(i2a_control & EMPTY)) { readq(RECV0); readq(RECV1); } @@ -156,8 +151,8 @@ enum { #define ANE_ASC_MBOX_CTRL_WPTR GENMASK(11, 8) #define ANE_ASC_MBOX_CTRL_ENABLE BIT(0) -/* W10 hazard: a plain READ of ANE+0x1854000 is fabric-fatal (watchdog - * reset ~63 s later, netconsole pinned the pre-log and no value line). +/* Hazard: a plain READ of ANE+0x1854000 is fabric-fatal (the read + * never returns; watchdog reset ~63 s later). * Nothing may touch 0x1854000..0x1c04000; the old engine kill window * 0x1c04000..0x1c28000 still stands. */ @@ -192,23 +187,22 @@ enum { * see ane_t6021_eps), which this driver does not walk yet. * * [INFERENCE: the fw->host doorbell/IRQ bit numbering mirrors the - * host->fw SET bit per Asahi rtkit semantics; pinned live by W5.] + * host->fw SET bit per Asahi rtkit semantics.] * */ #define ANE_MBI_SCRATCH0 0x1840048 /* SCRATCH0..7 = +0x48..+0x64 */ #define ANE_MBI_SCRATCH6 0x1840060 #define ANE_MBI_SCRATCH7 0x1840064 /* Wake/ack handshake words live in ane_t6021_boot.h - * (ANE_T6021_BOOT_WAKE_REQ / ANE_T6021_BOOT_ACK) — single source, shared - * with the userspace boot regression. + * (ANE_T6021_BOOT_WAKE_REQ / ANE_T6021_BOOT_ACK) — single source. */ #define ANE_MBI_DOORBELL 0x1844000 /* write32 (1 << endpoint id) */ -/* NOT a message pair. W10 proved this is a mirror of the 24 MHz +/* NOT a message pair. This is a mirror of the 24 MHz * architectural counter: across 32 samples the absolute difference * against CNTVCT_EL0 is a constant 13-18 counts (the MRS-to-MMIO - * instruction gap), aggregate drift 0.054 ppm over 55.2M counts. The - * "type-0 heartbeat, ~29 ticks/3 s" that W6/W9 recorded as firmware - * liveness was this clock plus a poll count: the RTKit type field + * instruction gap), aggregate drift 0.054 ppm over 55.2M counts. Read + * as a message it looks like a "type-0 heartbeat" from the firmware, + * but that is this clock plus a poll count: the RTKit type field * GENMASK_ULL(59,52) is structurally zero until the counter passes * 2^52 (~6 years of uptime). Kept only so the old reads stay * identifiable in the logs; do not treat as transport. @@ -245,8 +239,8 @@ static inline u64 ane_mbi_msg48_encode(u32 cursor, u32 len) * 3. memcpy(ring + cursor, cmd, len) (DMA-coherent ring) * 4. dma_wmb() (ring visible before bell) * 5. write32(ANE_MBI_MSG_A2I_WR, lo) + write32(+4, hi) — 32-bit - * halves (the AKF message register is word-shaped; W6 proved the - * W5-live abort was NOT the 64-bit writeq — the halves abort too) + * halves (the AKF message register is word-shaped; a host write + * there aborts with 32-bit halves as well as with a 64-bit writeq) * 6. write32(ANE_MBI_DOORBELL, 1 << ep) * 7. on send success only: write_cursor = cursor + len, so a * failed send leaves the cursor at the last @@ -254,8 +248,8 @@ static inline u64 ane_mbi_msg48_encode(u32 cursor, u32 len) * CSNE_CMD_PING = header-only 0x11 on EP1 (INIT) -> doorbell bit * 1 << 1 = 0x2. Only the a2i message register + doorbell are * touched, and only behind the mbi_doorbell opt-in (both stay - * host-write-fatal even behind the W8 grant, W9; SCRATCH family is - * host-writable granted, W9 nonzero latch). + * host-write-fatal even after the grant tunables; the SCRATCH family + * is host-writable after them: a nonzero write latches). */ /* MBI channel-table entry as the firmware publishes it (stride 0x100): @@ -299,11 +293,11 @@ static inline u64 ane_mbi_msg48_encode(u32 cursor, u32 len) #define ANE_RTKIT_EP_TRACEKIT 0xa /* RTBuddy app endpoints — the host opens ids 1..6; - * ring sizes + fourccs from the per-EP config table (W2 §3). fourcc is - * byte-reversed in the table (0x54324643 = "T2FC"). + * ring sizes + fourccs from the firmware's per-EP config table. fourcc + * is byte-reversed in the table (0x54324643 = "T2FC"). */ enum ane_t6021_eps { - ANE_T6021_EP_INIT = 1, /* INIT — CSNE_CMD controller channel (W4) */ + ANE_T6021_EP_INIT = 1, /* INIT — CSNE_CMD controller channel */ ANE_T6021_EP_T2FC, /* fw->host commands */ ANE_T6021_EP_T2FH, /* fw->host commands */ ANE_T6021_EP_T2HS, @@ -312,11 +306,8 @@ enum ane_t6021_eps { ANE_T6021_EP_COUNT = ANE_T6021_EP_T2HT + 1 /* arrays index by id */ }; -/* CSNE command ids — the firmware knows 96 of them - * (full set: receipts/2026-09-18-h14-w2-protocol-decode §4 and - * receipts/2026-09-18-h14-w2-protocol-decode/fw_cmd_table.json). The - * fw parses the id as u16 at wire offset +4 (W4 correction of the W2 - * §4 "offset 0" claim); the +/* CSNE command ids — the firmware knows 96 of them. The + * fw parses the id as u16 at wire offset +4 (not at offset 0); the * controller header for the fw->host direction is a different shape * (u32 id @ +0x8, 0x24 bytes) and never rides host->fw submission. */ @@ -395,9 +386,9 @@ struct ane_t6021 { */ struct mutex mbox_lock; - /* Bring-up state machine — honest semantics (W15): + /* Bring-up state machine: * power_gated: first_resume passed (eight-island gate + - * whitelist; block access proven-safe), + * whitelist; block access safe), * cpu_started: RVBAR programmed + CPU RUN released (fw_boot=1), * fw_alive: fw first-alive ack observed on SCRATCH7 — NOT * the init handshake, @@ -409,8 +400,8 @@ struct ane_t6021 { bool fw_alive; bool booted; - /* W10: the mailbox does exist — at +0x1408xxx, not the h16g - * +0x1608xxx that read-aborted in 2026-09-19 — and it reads a + /* The mailbox does exist — at +0x1408xxx, not the h16g + * +0x1608xxx, which read-aborts — and it reads a * live, enabled, permanently-empty ASC v4 control pair. It is * empty because the ASC CPU is STOPPED with RUN clear and RVBAR * entry 0, i.e. no firmware was ever started. Default off = @@ -419,19 +410,19 @@ struct ane_t6021 { */ bool transport; bool doorbell; /* mbi_doorbell=1: EP rings may write the +0x1844000 - * doorbell + a2i message register (decoded 2026-09-19) + * doorbell + a2i message register */ bool irq_requested; struct ane_t6021_ep ep[ANE_T6021_EP_COUNT]; - /* W13 fw surface (fw_load=1): coherent, DART-mapped via the + /* fw surface (fw_load=1): coherent, DART-mapped via the * device's iommu group. NULL unless loaded. */ void *fw_buf; dma_addr_t fw_iova; u32 fw_size; - /* W16 entry alias: IOVA the fw pages are aliased at (the latched + /* Entry alias: IOVA the fw pages are aliased at (the latched * RVBAR entry); 0 = no alias mapped. The DMA allocator has no * runtime reservation API on this kernel, so instead of a prose * collision bound every DMA allocation site must pass its iova @@ -440,22 +431,21 @@ struct ane_t6021 { u64 fw_alias_iova; /* Per-window mapped extents (iova, bytes) — teardown unmaps * exactly these, because the windows are not assumed adjacent - * (live trace: hole between SEG0 and SEGi) and unmapping a page - * that was never mapped trips dart_unmap_pages (io-pgtable-dart - * .c:319 WARN, 2026-09-26 state-report unwind). Set only on - * successful map. + * (there can be a hole between SEG0 and SEGi) and unmapping a page + * that was never mapped trips a WARN in dart_unmap_pages + * (io-pgtable-dart.c). Set only on successful map. */ #define ANE_FW_ALIAS_MAX_WIN 3 u64 fw_alias_ext_iova[ANE_FW_ALIAS_MAX_WIN]; size_t fw_alias_ext_len[ANE_FW_ALIAS_MAX_WIN]; int fw_alias_extn; - /* W16 pass-3: ane_cpu reset controller (ps RESET via the pmgr + /* ane_cpu reset controller (ps RESET via the pmgr * pwrstate reset_controller ops); NULL when the DT carries no * resets property. */ struct reset_control *cpu_rst; - /* W15 boot allocations (gate-gated: never exist until the + /* Boot allocations (gate-gated: never exist until the * preflight opens and the sequence passes poll A). Ownership per * the wedged-pin rule: held, never freed, while cpu_started. */ @@ -485,7 +475,7 @@ struct ane_t6021 { */ }; -/* Enforced alias-window invariant (W16): the DMA allocator has no +/* Enforced alias-window invariant: the DMA allocator has no * runtime IOVA reservation on this kernel, so every DMA allocation * site must refuse a mapping that overlaps the fw entry alias. */ @@ -505,7 +495,7 @@ void ane_t6021_rtkit_shutdown(struct ane_t6021 *ane); void ane_t6021_rtkit_drain(struct ane_t6021 *ane); irqreturn_t ane_t6021_rtkit_irq_thread(int irq, void *data); -/* ane_t6021_boot.c — W15 boot state resolution (fw_boot=1). An +/* ane_t6021_boot.c — boot state resolution (fw_boot=1). An * explicit boot request fails the probe while the prerequisites in * ane_t6021_boot.c hold (-ENODATA); fw_boot=0 binds status-only. * ane_t6021_boot_start() is the sequence dispatcher proper (all gates @@ -519,10 +509,9 @@ int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, /* ---- CSNE_CMD wire structs (host->fw on the INIT channel) ---- * - * W4, extending W2 §4 (the receipt's "fw parses a u16 id - * at header offset 0" was WRONG — the firmware reads - * the id as u16 at offset +4, for generic commands as well as for - * LOAD_PROGRAM/CREATE_PROCESS/PROCEDURE_CALL). + * The firmware reads the id as u16 at offset +4 (not at header + * offset 0), for generic commands as well as for + * LOAD_PROGRAM/CREATE_PROCESS/PROCEDURE_CALL. * Wire is * little-endian; this module is arm64-only so host order is wire order. * @@ -532,12 +521,11 @@ int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, * ring slot doubles as the response area. The firmware handles * PING/BUILDINFO/BOOT/REG_FILE_LOAD/IPC_ENDPOINT_SET on its * default path, without parsing their fields, so - * their payloads beyond the header are opaque until the W1 live - * exchange pins them. + * their payloads beyond the header are opaque. */ struct ane_csne_hdr { u32 rsvd0; /* bytes 0..3: never read by the fw processor; 0 */ - u16 id; /* PROVEN u16 @ +4 (see above) */ + u16 id; /* u16 @ +4 (see above) */ u8 flags; /* fw-written byte @ +6 (status/scratch) [INFERENCE] */ u8 rsvd7; }; @@ -551,13 +539,13 @@ static inline void ane_csne_hdr_init(struct ane_csne_hdr *h, u16 id) } /* Header-only CSNE_CMDs: BOOT (0x10 — boot-arg surfaces ride - * SCRATCH0-7 / the firmware init boot args, not the command, phase1 §2.5), + * SCRATCH0-7 / the firmware init boot args, not the command), * PING (0x11), BUILDINFO (0x06). sizeof(struct ane_csne_hdr) bytes. */ /* REG_FILE_LOAD (0x05) payload: the 1456 B blob is the fw's own * _rtk_tunables section (0x5b0 bytes) — its transport - * (inline vs shared-memory iova) is [INFERENCE], pinned by W1. + * (inline vs shared-memory iova) is [INFERENCE]. */ struct ane_csne_cmd_reg_file_load { struct ane_csne_hdr hdr; @@ -566,7 +554,7 @@ struct ane_csne_cmd_reg_file_load { /* IPC_ENDPOINT_SET (0x15) payload: binds a host RTBuddy endpoint to a * fw channel. No field parsed by the fw processor — layout - * [INFERENCE], pinned by W1. + * [INFERENCE]. */ struct ane_csne_cmd_ipc_endpoint_set { struct ane_csne_hdr hdr; @@ -575,20 +563,18 @@ struct ane_csne_cmd_ipc_endpoint_set { /* PROCEDURE_CALL (0x204) / INFERENCE_CALL (0x404) — the procedure-call * command family. Field offsets follow what the firmware accepts; - * field names (programId, procedureId, numIoBuffers, W2 §4) are given + * field names (programId, procedureId, numIoBuffers) are given * only where the field's role is known: * +0x08/+0x0c u32 pair validated together * +0x10 u64 passed to the fw validator * +0x18 u32 stats/priority, fw-valid [2,7]; 2 is the value - * proven end-to-end (binding.json stats_type; the - * first-inference receipt's bare calls) + * that works end-to-end for plain calls * +0x28 u32 element count * +0x60 count × 0x30-byte io-buffer records * Commands are whole multiples of the procedure-call size, so they * form same-shape arrays in the ring; this driver submits one * command per slot. INFERENCE_CALL shares the shape [INFERENCE: the - * 0x404 id is not among the decoded ids — it is the W4 submission - * endpoint per the phase-1 workstream plan]. + * 0x404 id is not among the decoded ids]. */ struct ane_csne_io_elem { u8 bytes[0x30]; /* internal layout not decoded */ @@ -629,11 +615,10 @@ ane_csne_cmd_procedure_call_size(unsigned int num_io_buffers) */ #define ANE_CSNE_CMD_MAX_SIZE 0x1b88 -/* ---- LOAD_PROGRAM (0x200) wire contract (2026-09-26, entry item - * 23/23b/23c/23e): the payload is NINE 0x30-byte named section - * records at +0x08. Loading publishes ".bin" for each section - * into the fw program-info symbol table, with value = record+0x18 - * and aux = u32(record+0x20). +/* ---- LOAD_PROGRAM (0x200) wire format: the payload is NINE 0x30-byte + * named section records at +0x08. Loading publishes ".bin" for + * each section into the fw program-info symbol table, with value = + * record+0x18 and aux = u32(record+0x20). * * The fw dereferences record+0x18, so it * carries a fw-addressable pointer — in this driver, the IOVA of a @@ -732,24 +717,23 @@ static inline size_t ane_opsec_size(u32 ops) } /* Submit one CSNE command block on the INIT (EP1) ring: RTBuddy - * endpoint send semantics (W2 §3) — slot alloc with wrap, + * endpoint send semantics — slot alloc with wrap, * memcpy into the ring, 54-bit doorbell word, cursor * advanced only on doorbell success. No synchronous response matching: * fw->host responses arrive on the T2F* channels and are not walked - * yet (W2 §3). Sleeps (mutex) — process context only. + * yet. Sleeps (mutex) — process context only. */ int ane_t6021_csne_submit(struct ane_t6021 *ane, const void *cmd, size_t size); -/* Probe-time one-shot CSNE_CMD_PING on EP1 (W5-live), behind +/* Probe-time one-shot CSNE_CMD_PING on EP1, behind * mbi_doorbell=1 only; watches the fw response surfaces for 3 s. */ void ane_t6021_csne_ping_attempt(struct ane_t6021 *ane); -/* W13/W14 firmware loader (ane_t6021_fwload.c): validate + stage + +/* Firmware loader (ane_t6021_fwload.c): validate + stage + * dart-ane0-map the 13.5 PRELOAD payload behind fw_load=1. This is - * the staging half of the boot contract: the fw DVA source - * and the RVBAR fold are closed (audit commits - * 3762aee/12be074); ane_t6021_boot.c consumes the staged surface. + * the staging half of the boot sequence: it provides the fw DVA that + * the RVBAR fold uses; ane_t6021_boot.c consumes the staged surface. */ int ane_t6021_fwload_probe(struct ane_t6021 *ane); void ane_t6021_fwload_remove(struct ane_t6021 *ane); diff --git a/drivers/accel/ane/ane_t6021_boot.c b/drivers/accel/ane/ane_t6021_boot.c index ccb87ec65b2ff..709bb7df52ef2 100644 --- a/drivers/accel/ane/ane_t6021_boot.c +++ b/drivers/accel/ane/ane_t6021_boot.c @@ -1,131 +1,90 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* - * ane_t6021_boot.c — T6021 boot state reporter (W15). + * ane_t6021_boot.c — T6021 boot sequencer: the kernel io backend, the + * boot sources and the preflight gates for ane_t6021_boot_run(). The + * sequence and its pure helpers (entry fold, latch/entry decode, + * SCRATCH u64 split, init-structure fill) live in ane_t6021_boot.h. * - * READ-ONLY. Main review 2026-09-20: the unresolved pre-CPU power and - * config-table prerequisites BLOCK every boot write, so this unit - * performs no MMIO write at all — it resolves the boot-input state, - * reports it with per-read whitelist provenance, and names the exact - * missing lifecycle datum. The write path (RVBAR entry fold + CPU RUN - * release + first-alive ack) lands only after the prerequisites below - * are pinned; its pure contract units (entry fold, latch/entry decode, - * SCRATCH u64 split, init-structure fill) live in ane_t6021_boot.h and - * are regression-tested host-side (tools/h14_boot_regression.c). - * - * Blocked boot writes, with the blocking prerequisite each one waits on: + * Boot writes, with the prerequisite each one depends on: * - RVBAR entry fold (ENTRY_BASE | fw DVA & ADDR_MASK; single u64 - * store; the read and the write are both 64-bit, per width_proof): - * semantics of the observed RVBAR read 0x1 are unresolved - * (see the rvbar lifecycle fork) AND the pre-CPU block - * below applies regardless. - * - CPU_CONTROL write32 0 then 0x10 (rvbar-width local order): waits - * on the same pre-CPU block; releasing RUN with an unresolved - * entry state is the exact blind start the lane forbids. - * - SCRATCH0/1 publication + SCRATCH7 wake: init-structure fields - * [0x08] and [0x10,0x38) have no pinned firmware consumer - * read-set; zero-filling unknown fields and calling the structure - * valid is forbidden (Main 2026-09-20). + * store; the read and the write are both 64-bit): skipped while + * the RVBAR read has bit0 set (see the RVBAR lifecycle below). + * - CPU_CONTROL write32 0 then 0x10: only once the entry state is + * decided; releasing RUN with an unresolved entry state would be a + * blind start. + * - SCRATCH0/1 publication + SCRATCH7 wake: every init-structure + * field the firmware reads needs a Linux source; zero-filling + * unknown fields and calling the structure valid is not allowed. * - * Pre-CPU prerequisites, updated for pass4 (receipts 2026-09-20 - * pass4 + tools/check_power_anchors.py 33/33, commit - * 04630ef): - * RESOLVED (pass4): the CPU-power branch — the h14g configuration - * selects the PLAIN branch (the same configuration that puts - * CPU_CONTROL at 0x01400044), so the CPU island is powered with - * write32(PMGR+0x2e0, 0x0000000f), validate 0xff — - * AUTO_ENABLE bit28 never set. On Linux this END STATE is exactly - * what the first_resume stage-2/3 gate verifies (act=0xf, - * AUTO_ENABLE clear) before any of this code runs — the write is - * therefore not replicated (ps-word writes are forbidden by the - * standing rule and froze the box 2026-09-19); the gate IS the - * equivalence check. - * STILL OPEN (each blocks its write below): - * 1. CLOSED (pass5 c364f24): the pre-CPU engine table writels - * eng+0xb38/0xb98/0xbf8 <- 0x01ff01ff are REQUIRED every time - * the ANE clocks and power are enabled — they are written - * unconditionally, before the CPU start; the - * order-vs-start question is moot. Receiver PROVEN pass4 - * (the engine register window, reg[0]); count = - * 3. Armed below; the first live firing must ride netconsole - * behind the seam logs. - * 2. Provider first-enable of the device clock and power: the - * set of clock and power gates it enables is only known at run - * time (open). Linux - * substitutes the genpd raise + supplier links (islands verified - * ACTUAL=0xf by the gate); the equivalence datum is the open - * half. - * 3. RVBAR lifecycle — RESOLVED (M2ResetLifecycle, - * rvbar-lifecycle-evidence.json, anchors 38/38 - * check_rvbar_lifecycle_anchors.py): live bit0-set + entry 0 is - * the STALE state; the valid transition is power_off - * (de-init, then PMGR ps 0x2e0 ← 0x00000000 through the PMGR - * window) → power_on → re-init, NEVER an - * RVBAR write while bit0 is set. RVBAR (engine window, - * 0x1050000, u64) is written only after a read64 with bit0 == 0; - * CPU_CONTROL write32 0 then 0x10 runs on BOTH paths; success = - * SCRATCH7 read32 == 0x08042006. Open edge: - * whether ps-off clears bit0 — one live read-only read64 after a - * domain-off answers it. IMPLEMENTATION CAVEAT: the ps-word write - * is the class that froze this host from kernel context - * (2026-09-19 receipt) and the W3 gate demands act=0xf + - * AUTO_ENABLE clear afterwards — the reset vehicle (userspace - * stage vs in-kernel) is a Main decision, so the fold+start path - * stays behind the itemized preflight gates. - * 3. Ack model — TWO-PHASE, SINGLE REGISTER, TRANSITION-BASED - * (rvbar-lifecycle receipt 6288b0b, 57/57 anchors; supersedes the - * pass6 SCRATCH0-beacon reading): pre-CPU, SCRATCH cells are - * cleared and SCRATCH7 is PULSED 1 -> 0 (stale READY/wake - * cleared); after CPU_CONTROL 0 -> 0x10, poll A demands a FRESH - * SCRATCH7 READY 0x08042006 (fw alive; fw_alive). Publication - * (init suballoc DVA -> SCRATCH0/1, dsb st) happens ONLY after - * poll A; then wake 0xf7fbdff9 -> SCRATCH7 releases the fw; poll - * B awaits SCRATCH7 DONE 0x08042006 (booted) and reads back - * SCRATCH0/1. Because of the pulse, a post-CPU READY is - * unambiguous — no stale-ack hazard. The SCRATCH0 marker - * is not a gate. + * Pre-CPU prerequisites: + * CPU power: the h14g configuration selects the PLAIN branch (the + * same configuration that puts CPU_CONTROL at 0x01400044), so the + * CPU island is powered with write32(PMGR+0x2e0, 0x0000000f), + * validate 0xff — AUTO_ENABLE bit28 never set. On Linux genpd + * raises the island before any of this code runs, and probe checks + * its ACTUAL state before the first engine read; the write is + * therefore not replicated (a direct ps-word write from kernel + * context has hung the machine). + * 1. The pre-CPU engine table writels eng+0xb38/0xb98/0xbf8 <- + * 0x01ff01ff (three, into the engine register window, reg[0]) are + * REQUIRED every time the ANE clocks and power are enabled, before + * the CPU start. They have also wedged the machine here, so + * fw_start_table_mode decides whether they run. + * 2. Provider first-enable of the device clock and power: the set of + * clock and power gates it enables is only known at run time. + * Linux substitutes the genpd raise + supplier links. + * 3. RVBAR lifecycle: bit0 set + entry 0 is the STALE state; the + * valid transition is power_off (de-init, then PMGR ps 0x2e0 ← + * 0x00000000 through the PMGR window) → power_on → re-init, NEVER + * an RVBAR write while bit0 is set. RVBAR (engine window, + * 0x1050000, u64) is written only after a read64 with bit0 == 0; + * CPU_CONTROL write32 0 then 0x10 runs on BOTH paths; success = + * SCRATCH7 read32 == 0x08042006. Whether ps-off clears bit0 is not + * known. The ps-word write of that reset is the class that hangs + * the machine from kernel context, so this driver never performs + * it: with bit0 set, the ASC starts at the latched entry. + * 4. Ack model — TWO-PHASE, SINGLE REGISTER, TRANSITION-BASED: + * pre-CPU, SCRATCH cells are cleared and SCRATCH7 is PULSED 1 -> 0 + * (stale READY/wake cleared); after CPU_CONTROL 0 -> 0x10, poll A + * demands a FRESH SCRATCH7 READY 0x08042006 (fw alive; fw_alive). + * Publication (init suballoc DVA -> SCRATCH0/1, dsb st) happens + * ONLY after poll A; then wake 0xf7fbdff9 -> SCRATCH7 releases the + * fw; poll B awaits SCRATCH7 DONE 0x08042006 (booted) and reads + * back SCRATCH0/1. Because of the pulse, a post-CPU READY is + * unambiguous — no stale-ack hazard. The SCRATCH0 marker is not a + * gate. * On the firmware side: it writes READY (0x08042006) to * SCRATCH7, then polls SCRATCH7 every 1 ms until it reads the * wake word 0xf7fbdff9, and writes DONE (0x08042006 again) once * it has booted; the host's poll B waits for that second * 0x08042006. - * 4. Linux allocation map (legacy branch sizes): FWIM surface = - * 0x500000 bytes (Main, audit 751caa4); - * Linux fw_buf is ANE_FW_BUF_SIZE = 0x500000 — the semantic is - * honored directly, covering the image vmsize 0x36c000. 'IPC ' - * surface = a configured size, capped - * — the numeric cap needs the h14g config field map - * (open prerequisite). Init suballoc = 0x174 from a Linux-owned - * pool (pool - * total size open). RTBuddy FW_INIT sizes (64K…) are the OTHER + * 5. Linux allocation map (legacy branch sizes): FWIM surface = + * 0x500000 bytes; Linux fw_buf is ANE_FW_BUF_SIZE = 0x500000, + * covering the image vmsize 0x36c000. 'IPC ' surface = a + * configured size, capped. Init suballoc = 0x174 from a + * Linux-owned pool. RTBuddy FW_INIT sizes (64K…) are the OTHER * branch — never used here. - * 5. Init publication: the fw consumes [0x08]..[0x68] (pass5) and - * the Linux sources of those fields are not pinned ([0x08] = - * the 'IPC ' surface DVA — the surface - * itself is CLOSED (pass5c/5d), its Linux allocation size open - * per item 4; [0x10]/[0x18] = config-size terms with - * the image byte-count closed; [0x30] = the - * load-progress word) — publication cannot fire until every - * fw-read field has a pinned Linux source. + * 6. Init publication: the fw consumes [0x08]..[0x68] ([0x08] = the + * 'IPC ' surface DVA; [0x10]/[0x18] = config-size terms; [0x30] = + * the load-progress word) — publication cannot fire until every + * fw-read field has a Linux source. * * Read whitelist — every MMIO read below, individually: - * - RVBAR eng+0x01050000: read64 width-proven (rvbar-width - * width_proof); W8 and W10 live Linux reads, value 0x1 each. - * - CPU_STATUS eng+0x1400048: phase-1 S2 whitelist; polled while - * the CPU starts; W10 live read - * 0x2a. - * - ASC mailbox controls eng+0x1408110/0x1408114: W10 live reads - * (both 0x00020001; 600 polls / 30 s, zero changes, zero aborts). - * The FATAL mailbox-class reads were the +0x1608xxx h16g analog — + * - RVBAR eng+0x01050000: read64 (the register is 64 bits wide); + * Linux reads have returned 0x1. + * - CPU_STATUS eng+0x1400048: polled while the CPU starts; reads + * 0x2a before it does. + * - ASC mailbox controls eng+0x1408110/0x1408114: both read + * 0x00020001; polling them neither changes them nor aborts. + * The FATAL mailbox-class reads are the +0x1608xxx h16g analog — * a different address never touched here; the 0x1854000..0x1c04000 * fabric-fatal window is likewise never touched. - * - SCRATCH eng+0x1840048..0x1840064: phase-1 S2 whitelist - * (all-zero pre-attach), W5 pre-read log clean, first_resume - * re-reads all eight on every load. + * - SCRATCH eng+0x1840048..0x1840064: all zero before the driver + * attaches. * - TICK eng+0x1160008 (PROGRESS dump only): 24 MHz domain tick, - * proven readable post-release (2026-09-25 handshake receipt); - * two reads 20 ms apart for a delta. engine+0x1140008 is - * deliberately NOT read (readability/width/reset semantics - * unproven, no pre-release baseline — Main review 2026-09-26). + * readable once the CPU is released; two reads 20 ms apart for a + * delta. engine+0x1140008 is deliberately NOT read (readability, + * width and reset semantics unknown, no pre-release baseline). */ #include @@ -142,86 +101,67 @@ #include "ane_t6021_boot.h" /* Retain the firmware nap-prevention counter via the init resource bit - * (lab boot_prevent_nap=1, the proven add-path value). + * (default on). */ static bool boot_prevent_nap = true; module_param(boot_prevent_nap, bool, 0444); MODULE_PARM_DESC(boot_prevent_nap, - "Retain firmware nap-prevention counter via init resource bit (default on: proven add-path value)"); + "Retain firmware nap-prevention counter via init resource bit (default on)"); /* Boot-write gates — ITEMIZED, each a HARD gate: the ENTIRE write * sequence (preboot engine table, scratch clear + pulse, RVBAR * resolution, CPU release, publication, wake) runs start-to-finish - * only when EVERY gate below is true, checked BEFORE any write (Main - * 2026-09-20: no partial boot; remaining semantics stay hard gates, - * not comments). Each flips only with cited proof in a Main-reviewed - * commit that also populates the sources and adds the kernel io - * backend — never a runtime knob. + * only when EVERY gate below is true, checked BEFORE any write (no + * partial boot; remaining semantics stay hard gates, not comments). + * Each is a compile-time constant — never a runtime knob. */ -/* Provider equivalence — CLOSED (Main provider review 2026-09-20): - * Linux genpd attachment + supplier links accepted INSTEAD of - * reproducing Apple runtime index arrays, CONDITIONAL on the live - * first_resume verifying all 8 islands ACTUAL=0xf, BUSY=0, CPU - * AUTO_ENABLE clear before the sequence — that verification runs on - * every probe and the driver performs no direct kernel PMGR writes. +/* Provider equivalence: Linux genpd attachment + supplier links stand + * in for Apple's runtime power index arrays, provided all 8 islands + * are on (ACTUAL=0xf, BUSY=0, CPU AUTO_ENABLE clear) before the + * sequence; probe checks the ane_cpu island before the first engine + * read, and the driver performs no direct kernel PMGR writes. */ static const bool pf_provider_genpd_strategy = true; -static const bool pf_pool_word0_proven = true; /* CONFIRMED (Reset - * 183fd50, traced through - * the firmware's pool - * setup): DDM Params - * word0 = requested bytes = - * 0x40000 — sourced, not - * synthesized +static const bool pf_pool_word0_proven = true; /* DDM Params word0 = + * requested bytes = + * 0x40000, traced + * through the firmware's + * pool setup */ -static const bool pf_heap_floor_pinned = true; /* CLOSED: floor = - * the DART page size = - * 0x4000 (the dart-ane0 - * node's page-size - * property is 0x4000, - * pass6h); +static const bool pf_heap_floor_pinned = true; /* floor = the DART + * page size = 0x4000 + * (the dart-ane0 node's + * page-size property); * sourced in boot_sources */ -static const bool pf_dart_page_floor = true; /* CLOSED ( - * pass6h + exact-node - * linkage): dtree-j414c.txt - * sha256 dd2955…d504, ane0 - * iommu-parent 363 (line - * 11499), dart-ane0 - * page-size 0x4000 (line - * 11510), child mapper-ane0 - * phandle 363 (line 11527). - * Runnable linkage check: - * ane-linux-experiments - * tools/ - * check_dart_ane_pagesize_linkage.py +static const bool pf_dart_page_floor = true; /* j414c ADT: ane0's + * iommu-parent is the + * dart-ane0 child + * mapper-ane0, and + * dart-ane0's page-size + * is 0x4000 */ -static const bool pf_rvbar_lifecycle = true; /* 6288b0b, 57/57 anchors */ -static const bool pf_pass6_init_contract = true; /* cd25b46, 87/87 */ - -/* FINAL authorization gate — CLOSED (Main provider review + pass6h - * numeric closure + 183fd50 word0 confirmation + netconsole Wi-Fi - * marker m2-wifi-check-1789937086 @ 15:44:47.121595Z). Source - * contract closure complete; live attempt authorized by user - * override (autonomous boot/recovery loop). - */ -/* Table-block mode selection (2026-09-20 16:23:07 wedge): +static const bool pf_rvbar_lifecycle = true; +static const bool pf_pass6_init_contract = true; + +/* Table-block mode selection (the table writes have wedged the + * machine): * mode 0 = ABORT before any write (accidental-repeat prevention), * mode 1 = write the table (as on macOS; selected per-run via the * rtclient fw_start_table_mode param), - * mode 2 = SKIP the table (default: the shipped diagnostic). - * W8 write-grant tunables are mode-independent (proven no-abort - * class, w8-run.out: APERTURE_UNLOCKED) and run in every armed mode. + * mode 2 = SKIP the table (default: diagnostic). + * The write-grant tunables are mode-independent (a write class that + * does not abort) and run in every armed mode. */ static const bool pf_main_lifetime_review = true; static bool ane_t6021_boot_preflight_complete(void) { - /* pf_preboot_table_safe is INTENTIONALLY NOT in this product: - * mode 2 (table skip) is the authorized diagnostic state — - * the preflight covers the source contract, the mode selector - * covers the live-fault table gating. + /* The pre-CPU table block is INTENTIONALLY NOT in this product: + * mode 2 (table skip) is a valid diagnostic state — the + * preflight covers the boot sources, the mode selector covers + * the table gating. */ return pf_provider_genpd_strategy && pf_pool_word0_proven && pf_heap_floor_pinned && pf_dart_page_floor && @@ -229,47 +169,37 @@ static bool ane_t6021_boot_preflight_complete(void) pf_main_lifetime_review; } -/* STATIC boot sources — populated per the pinned contract (Main - * authorization 2026-09-20). The 'DDM ' pool (size 0x40000) and the - * memory manager built on top of it are separate objects, which - * audit 018abdb had confused; the pool's first word is its - * requested size (see pool_word0 below). +/* STATIC boot sources. The 'DDM ' pool (size 0x40000) and the + * memory manager built on top of it are separate objects; the pool's + * first word is its requested size (see pool_word0 below). * * Runtime fields (fw_dva, ipc_dva, pool_dma) are - * filled by ane_t6021_boot_prepare() at sequence time (after poll A, - * per "dynamic allocations occur after READY"); heap_floor stays 0 - * here and is supplied together with pf_heap_floor_pinned — NEVER - * used unpinned (the gate above keeps the whole sequence unrun until - * then). + * filled by ane_t6021_boot_prepare() at sequence time (after poll A: + * dynamic allocations occur after READY). */ static const struct ane_t6021_init_sources boot_sources = { .cfg_size = 0x500000, /* FWIM surface size */ .prev_fw_len = 0, /* first boot; static per reload */ - .heap_floor = 0x4000, /* DART page size; node - * authority: the dart-ane0 - * page-size property, 0x4000 - * (exact linkage check - * passing) + .heap_floor = 0x4000, /* DART page size: the + * dart-ane0 page-size + * property, 0x4000 */ .pool_word0 = 0x40000, /* DDM Params word0 = requested - * bytes (CONFIRMED, 183fd50) + * bytes */ }; /* Poll A/B bound: up to 1000 polls with a 1 ms sleep, the same bound - * the firmware's own handshake loop uses (rvbar-lifecycle step 6/10). + * the firmware's own handshake loop uses. */ #define ANE_BOOT_ACK_POLL_US 1000 #define ANE_BOOT_POLL_MS 1000 /* The COMPLETE resolved legacy-boot sequence is implemented ONCE in - * ane_t6021_boot.h (ane_t6021_boot_run) against an io backend, so the - * fake-MMIO trace check exercises the exact code the device will run - * (Main: runnable trace check; implement the device wiring only after - * ALL sources close — until then this file performs no boot write). - * The kernel backend (readl/writeq/udelay wrappers + prepare hook - * doing the pool/IPC allocations and the sourced fill) is the small - * reviewed increment that lands with the gate flip. + * ane_t6021_boot.h (ane_t6021_boot_run) against an io backend, so a + * fake-MMIO trace test can exercise the exact code the device runs. + * The kernel backend below is the readl/writeq/udelay wrappers plus + * the prepare hook doing the pool/IPC allocations and the sourced fill. */ /* ---- kernel io backend for ane_t6021_boot_run() ---- */ @@ -331,22 +261,20 @@ static void ane_boot_phase(void *ctx, const char *what) /* bounded phase marker: one line per block boundary (never * per-poll), for crash attribution. It is a debug message: turn * it on with dynamic debug, and raise the console loglevel to - * see it there. The short drain lets the line reach - * tty0/netconsole/ssh BEFORE the risky write it announces - * (fw-start-debug 2026-09-22: fwstart#2 died with zero capture — - * the marker must beat the write). + * see it there. The short drain lets the line reach a local or + * remote console BEFORE the risky write it announces: if the + * write kills the machine, the marker must already be out. */ dev_dbg(mm->ane->dev, "BOOT-PHASE %s\n", what); msleep(30); } /* S5 prepare — runs strictly AFTER poll A (fw alive), BEFORE the - * SCRATCH0/1 publish. Dynamic allocations occur here per "dynamic - * allocations occur after READY" (Main): the 'DDM ' pool, the 'IPC ' - * surface, and the trust-bounded fw-requested HEAP surface. Every - * allocation is wedged-pin owned from here on (held while - * cpu_started; reboot reclaims). Publishes the pool DVA (suballoc at - * offset 0) as the SCRATCH0/1 halves. + * SCRATCH0/1 publish. Dynamic allocations occur here, after READY: + * the 'DDM ' pool, the 'IPC ' surface, and the trust-bounded + * fw-requested HEAP surface. Every allocation is wedged-pin owned from + * here on (held while cpu_started; reboot reclaims). Publishes the + * pool DVA (suballoc at offset 0) as the SCRATCH0/1 halves. */ static void *ane_boot_alloc(void *ctx, u64 size, u64 *iova) { @@ -368,7 +296,7 @@ static void *ane_boot_alloc(void *ctx, u64 size, u64 *iova) /* Kernel prepare: live SCRATCH3/SCRATCH1 reads feed the SHARED * assembly (ane_t6021_boot_prepare_publish) with the dma_alloc hook — - * the same path the fake-MMIO trace test exercises. Allocations are + * the same path a fake-MMIO trace test can exercise. Allocations are * wedged-pin owned from creation (held while cpu_started). */ static int ane_t6021_boot_prepare(void *ctx, u32 *lo, u32 *hi) @@ -402,10 +330,10 @@ static int ane_t6021_boot_prepare(void *ctx, u32 *lo, u32 *hi) lo, hi); if (!err && boot_prevent_nap) { ((u8 *)a.pool)[ANE_T6021_INIT_TEMPLATE_OFF + 0x84] |= 1; - dev_info(ane->dev, "LAB init resource[0x84] bit0=1: prevent nap\n"); + dev_info(ane->dev, "init resource[0x84] bit0=1: prevent nap\n"); } - /* OWNERSHIP TRANSFER on EVERY path (Main lifetime review): an - * error after the pool/ipc allocations must NOT lose them — a + /* OWNERSHIP TRANSFER on EVERY path: an error after the + * pool/ipc allocations must NOT lose them — a * started CPU may already be fetching, so partial allocations * are RETAINED (wedged-pin holds them; reboot reclaims), never * freed on this path. @@ -427,16 +355,15 @@ static int ane_t6021_boot_prepare(void *ctx, u32 *lo, u32 *hi) * (wedged-pin cleanup refuses to free under a started CPU) and the * probe binds fenced. */ -/* Poll-A-timeout progress dump — PROVEN-readable observables ONLY - * (Main review 2026-09-26): all eight SCRATCH cells (pulse-cleared to 0 - * pre-release, so any nonzero word with SCRATCH7 != READY/WAKE is a - * candidate firmware store), RVBAR, CPU_STATUS, and the 24 MHz domain - * tick at engine+0x1160008 (watched advancing post-release in the - * 2026-09-25 handshake receipt; a static tick separates "ASC domain - * unclocked" from "clocked but silent"). +/* Poll-A-timeout progress dump — observables known to be safe to read + * ONLY: all eight SCRATCH cells (pulse-cleared to 0 pre-release, so any + * nonzero word with SCRATCH7 != READY/WAKE is a candidate firmware + * store), RVBAR, CPU_STATUS, and the 24 MHz domain tick at + * engine+0x1160008 (it advances once the CPU is released; a static + * tick separates "ASC domain unclocked" from "clocked but silent"). * * Deliberately NOT read: engine+0x1140008 (a word the 13.5 stub - * writes). Its readability, width and reset value are unproven + * writes). Its readability, width and reset value are unknown * and no pre-release baseline exists, so an after-value would not * locate PC (could be preexisting iBoot state, write-only, or * normalized). @@ -484,8 +411,8 @@ int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, u64 sres = 0; int r; - /* Wedged-pin module lifetime (Main lifetime review): the ref is - * acquired BEFORE the first write — a started CPU can never + /* Wedged-pin module lifetime: the ref is acquired + * BEFORE the first write — a started CPU can never * outlive the pin, and a dying module refuses the boot before * any write happens. Retained on CPU start (never released: * intentional); released only if no CPU start occurred. @@ -509,8 +436,7 @@ int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, /* Released CPU without READY (poll-A timeout, either transport * mode): capture the 13.5 execution-progress observables NOW, - * while the islands are powered — this is the exact-progress - * datum the next pre-Linux capture cross-checks. + * while the islands are powered. */ if (cs && !fa) ane_t6021_boot_progress_dump(ane); diff --git a/drivers/accel/ane/ane_t6021_boot.h b/drivers/accel/ane/ane_t6021_boot.h index 0677dd046172e..ac0497bf6b010 100644 --- a/drivers/accel/ane/ane_t6021_boot.h +++ b/drivers/accel/ane/ane_t6021_boot.h @@ -1,36 +1,32 @@ /* SPDX-License-Identifier: GPL-2.0-only OR MIT */ /* - * ane_t6021_boot.h — T6021 legacy-boot contract units (pure, shared by - * the kernel sequencer and the userspace regression). - * - * Evidence: live reads and traces under Linux, plus the receipts in - * ane-linux-experiments/receipts/ (commits 3762aee + 12be074), for - * the firmware image with sha256 9f7915c4…. + * ane_t6021_boot.h — T6021 legacy-boot sequence and its helpers (pure, + * written against an io backend so that the kernel sequencer and a + * userspace trace test run the same code). * * - RVBAR fold: read64 of eng+0x01050000; bit0 - * set skips the write (live jw14m2 read 0x1). Otherwise write64 - * ENTRY_BASE | (fw DVA & ADDR_MASK) — single u64 store - * (2026-09-20-h14-rvbar-width.json width_proof). + * set skips the write (Linux reads 0x1 there). Otherwise write64 + * ENTRY_BASE | (fw DVA & ADDR_MASK) — single u64 store (the + * register is 64 bits wide). * - fw DVA = the device address of the staged firmware ("FWIM") * surface, mapped through the ANE DART; the RVBAR fold above starts * the ASC at this address. * Linux correspondence: dma_alloc_coherent on the ane - * platform device is dart-ane0-translated (live iommu group 6, + * platform device is dart-ane0-translated (one iommu group, * three apple,t6020-dart streams, DMA domain). * - SCRATCH0/1 u64 split: low32 → 0x01840048 first, high32 → * 0x0184004c, ordered after the writes that fill the published - * data (legacy-init-publication receipt; each half is a write32). + * data (each half is a write32). * - 0x174 init suballocation: header[0x00] u64 = fw DVA (legacy * branch, the one taken on this provider), [0x68] u32 = 64 count, - * template 256 B at [0x6C,0x16C) with template+0xC0 = 4 RESOLVED + * template 256 B at [0x6C,0x16C) with template+0xC0 = 4 * (the template's initial state); the conditional |=0x10 is - * separate, unqualified and not set. The rest of the block is + * separate, not understood and not set. The rest of the block is * zeroed by the caller (see ane_t6021_init_struct_fill). * * This header intentionally includes nothing. Kernel consumers have * already; userspace consumers typedef u8/u32/u64 and - * include first (tools/h14_boot_regression.c in this - * repository is the shipped runnable check). + * include first. */ #ifndef __ANE_T6021_BOOT_H__ #define __ANE_T6021_BOOT_H__ @@ -42,7 +38,7 @@ #define ANE_T6021_RVBAR_ENTRY_BASE 0x0081000000000001ULL #define ANE_T6021_RVBAR_ADDR_MASK 0xff7efffffffff800ULL -/* CPU_CONTROL release (rvbar-width local order: write32 0 then 0x10; +/* CPU_CONTROL release (write32 0 then 0x10, in that order; * h14g config selects eng+0x01400044). RUN = BIT(4). */ #define ANE_T6021_CPU_RUN_RELEASE 0x10 @@ -50,7 +46,7 @@ /* SCRATCH7 handshake words (RTBuddy and legacy init publication; the * firmware writes the ack). */ -/* SCRATCH0 request band (pass9-corrected): post-READY +/* SCRATCH0 request band: post-READY * SCRATCH0 >= 0x21 REFUSES the boot preparation; below 0x21 proceeds * to the MAX/alloc path. */ @@ -59,9 +55,7 @@ #define ANE_T6021_BOOT_WAKE_REQ 0xf7fbdff9U #define ANE_T6021_BOOT_ACK 0x08042006U -/* Init suballocation geometry (legacy init path; - * legacy-init-publication receipt). - */ +/* Init suballocation geometry (legacy init path). */ #define ANE_T6021_INIT_STRUCT_SIZE 0x174 #define ANE_T6021_INIT_FW_DVA_OFF 0x00 /* u64, legacy branch */ #define ANE_T6021_INIT_COUNT_OFF 0x68 /* u32 = 64 */ @@ -69,9 +63,9 @@ #define ANE_T6021_INIT_TEMPLATE_OFF 0x6c /* 256 B at [0x6C,0x16C) */ #define ANE_T6021_INIT_TEMPLATE_SIZE 0x100 #define ANE_T6021_INIT_TBIT_OFF 0xc0 /* template-relative */ -/* RESOLVED (the init path ORs 4 into this template word, whose - * initial value is zero); the conditional |=0x10 is a separate, still - * unqualified mutation and is NOT set. +/* The init path ORs 4 into this template word, whose initial value + * is zero; the conditional |=0x10 is a separate, not understood + * mutation and is NOT set. */ #define ANE_T6021_INIT_TBIT_VAL 0x4 @@ -81,8 +75,8 @@ static inline u64 ane_t6021_rvbar_compose(u64 iova) } /* The skip condition: bit0 of the RVBAR read (a set bit skips the - * fold; live jw14m2 reads have been 0x1). What a set - * bit names in boot-mode terms is the open lifecycle fork — see + * fold; Linux reads have been 0x1). What a set + * bit means in boot-mode terms is the RVBAR lifecycle — see * ane_t6021_boot.c; this predicate only decodes the bit. */ static inline bool ane_t6021_rvbar_latched(u64 rd) @@ -120,50 +114,37 @@ static inline u64 ane_t6021_scratch64_join(u32 lo, u32 hi) return ((u64)hi << 32) | lo; } -/* Linux sources for the init-suballocation fields (pass5b/5c/5d - * b8b7c35 + pass6c 75a017b; validator tools/check_init_contract.py - * 68/68). EVERY fw-read field is explicitly sourced from the struct +/* Linux sources for the init-suballocation fields. EVERY fw-read + * field is explicitly sourced from the struct * below — nothing is silently zero: [0x00] FWIM DVA, [0x08] 'IPC ' * surface DVA (Linux alloc, size = ane_t6021_ipc_size), [0x10] * cfg_size = 0x500000, [0x18] = 0x10000000 - size, [0x20] heap * DVA, [0x28] heap size, [0x30] fw-load progress, [0x50] boot ordinal, * [0x58] pool DMA base, [0x60] pool word0, [0x68] count 64, - * template[0x00] = 0, template+0xC0 = 4. The struct values pin BEFORE - * the preflight gate can open (Main: no unresolved field silently + * template[0x00] = 0, template+0xC0 = 4. The struct values are fixed + * BEFORE the preflight gate can open (no unresolved field silently * zero; no partial boot). The caller MUST zero the full 0x174 block * first (coherent alloc): gaps [0x34..37]/[0x64..67] are never * written and sit inside fw-read u64 units. */ -/* Init-suballocation contract — pass6 producer semantics - * (rvbar-lifecycle-evidence.json pass6_init_header_producers, commit - * cd25b46, anchors G1-G26; 87/87 total): +/* Init-suballocation field producers: * - * STATIC sources (pinned before the preflight can open): + * STATIC sources (fixed before the preflight can open): * fw_dva -> [0x00] staged selene ('FWIM') surface DVA, - * ipc_dva -> [0x08] 'IPC ' surface DVA (its size calc remains - * the audit lane's open item), + * ipc_dva -> [0x08] 'IPC ' surface DVA, * cfg_size -> [0x10] = 0x500000; [0x18] = * 0x10000000 - cfg_size = 0x0fb00000, * prev_fw_len -> [0x30] previous fw image length (0 on first boot; * driver-static, updated on each reload), - * heap_floor -> [0x28] floor = the DART page size (value - * pinned by the audit lane at preflight close), + * heap_floor -> [0x28] floor = the DART page size, * pool_dma -> [0x58] the 'DDM ' pool surface DVA (Linux allocates * its own 0x40000-byte DDM pool — the size macOS uses), * pool_word0 -> [0x60] first qword of the Linux pool-descriptor - * struct. NOT CLOSED: the fw-side consumer of - * pool+0x00 is untraced and the host never stores it - * (free-list head at manager+0 untouched by - * Allocate). This value is a HARD-GATED source — the - * sequence cannot run until a producer/consumer proof - * pins it; no hypothesis value is shipped. - * PRODUCER OPEN (Main, superseding b21e23f): the - * "pool+0x00 = manager free-list head" claim is - * REJECTED — the pool manager and the Params pool - * are different objects (wrong-object alias likely), and - * Reset6cb048d shows the consumer stores word0 - * VERBATIM at image ops 68 (possible extent use). - * Wait for the exact same-Params producer. + * struct: DDM Params word0, the requested pool size. + * It is NOT the pool manager's free-list head: the + * pool manager and the Params pool are different + * objects, and the firmware's consumer stores word0 + * VERBATIM (possibly as an extent). * * DYNAMIC inputs (read AFTER poll A — the fw publishes its extra-heap * request into SCRATCH3 (cell idx3) and its boot ordinal is @@ -186,20 +167,18 @@ struct ane_t6021_init_sources { u64 pool_word0; }; -/* 'IPC ' surface size (CURRENT_CONTRACT f9602e0; operands proven: - * page_size = the DART page size — 0x4000, one 16K - * DART page, typical; ordinal = captured SCRATCH1 + 1, the boot +/* 'IPC ' surface size (page_size = the DART page size — 0x4000, one + * 16K DART page, typical; ordinal = captured SCRATCH1 + 1, the boot * ordinal): size = max(page_size, ordinal + 1). Used by the kernel * prepare hook to size the 'IPC ' allocation. */ -/* SCRATCH0 read-order note (pass9 phase-1: SCRATCH0 then SCRATCH1 - * after READY): later evidence SUPERSEDES the audit's - * "SCRATCH0 >= 0x21 arms IPC" reading — a value below 0x21 leads - * to the MAX+IPC-alloc path and >=0x21 goes to logging/error, so - * scratch0 < 33 is likely a VALIDITY gate, not an optional-IPC - * condition. Header [0x08] is always filled with the IPC surface - * DVA. Allocation stays UNCONDITIONAL until Reset corrects - * the source; the threshold is NOT implemented. +/* SCRATCH0 read-order note (SCRATCH0 then SCRATCH1 after READY): + * SCRATCH0 >= 0x21 does NOT arm an optional IPC surface — a value + * below 0x21 leads to the MAX+IPC-alloc path and >=0x21 goes to + * logging/error, so scratch0 < 33 is likely a VALIDITY gate, not an + * optional-IPC condition. Header [0x08] is always filled with the IPC + * surface DVA: whenever the boot proceeds, the IPC surface is + * allocated. */ static inline u64 ane_t6021_ipc_size(u64 page_size, u32 captured) { @@ -277,8 +256,8 @@ ane_t6021_init_struct_fill(u8 *buf, const struct ane_t6021_init_sources *s, buf[0x28 + i] = (u8)((u64)heap_size >> (8 * i)); /* [0x30] u32 FULL width: previous fw image length (0 first - * boot). Main review: low-byte-only serialization dropped the - * upper length bits. + * boot). Low-byte-only serialization would drop the upper + * length bits. */ buf[0x30] = (u8)s->prev_fw_len; buf[0x31] = (u8)(s->prev_fw_len >> 8); @@ -290,15 +269,15 @@ ane_t6021_init_struct_fill(u8 *buf, const struct ane_t6021_init_sources *s, buf[ANE_T6021_INIT_COUNT_OFF + 2] = 0; buf[ANE_T6021_INIT_COUNT_OFF + 3] = 0; - /* template[0x00] = a config word, pinned value 0 - * (pass5d); [0x04..0x0B] zero via the caller's zeroing. + /* template[0x00] = a config word, value 0; + * [0x04..0x0B] zero via the caller's zeroing. */ buf[ANE_T6021_INIT_TEMPLATE_OFF + 0] = 0; buf[ANE_T6021_INIT_TEMPLATE_OFF + 1] = 0; buf[ANE_T6021_INIT_TEMPLATE_OFF + 2] = 0; buf[ANE_T6021_INIT_TEMPLATE_OFF + 3] = 0; - /* template+0xC0 = 4: RESOLVED initial state (see + /* template+0xC0 = 4: the initial state (see * ANE_T6021_INIT_TBIT_VAL). The conditional |=0x10 remains unset. */ buf[ANE_T6021_INIT_TEMPLATE_OFF + ANE_T6021_INIT_TBIT_OFF + 0] = @@ -385,7 +364,7 @@ ane_t6021_boot_prepare_publish(const struct ane_t6021_init_sources *s, * HEADER SOURCES come from the ALLOCATIONS: s->ipc_dva/pool_dma * may be zero in the caller's static sources — fill from the * allocator outputs (a->ipc_dva/a->pool_dva), never from s - * (Main review: zero-DVA header bug). + * (or the header would carry zero DVAs). */ { struct ane_t6021_init_sources filled = *s; @@ -402,11 +381,11 @@ ane_t6021_boot_prepare_publish(const struct ane_t6021_init_sources *s, } /* Post-DONE SCRATCH1:SCRATCH0 (read SCRATCH1 first, then SCRATCH0) - * is a fw-produced DEVICE ADDRESS (pass9 phase-2), NOT a return code — - * never dereference it raw. WHITELIST (Main 2026-09-20): only the + * is a fw-produced DEVICE ADDRESS, NOT a return code — + * never dereference it raw. WHITELIST: only the * Linux-owned FWIM and 'IPC ' windows may ever be matched against it; * the 'DDM ' pool is NOT whitelisted. No consumption until range AND - * length checks are proven; until then the raw u64 is stored and + * length checks exist; until then the raw u64 is stored and * logged only. */ @@ -415,8 +394,8 @@ ane_t6021_boot_prepare_publish(const struct ane_t6021_init_sources *s, * Register offsets (ENGINE-relative; the Linux driver maps the engine * window, so kernel offsets == trace offsets) and the io backend. * ane_t6021_boot_run() executes the resolved total order against ANY - * backend: the kernel wraps readl/writeq/udelay; the host regression - * wraps a recording fake (trace asserts: full order, no writes when + * backend: the kernel wraps readl/writeq/udelay; a host test can wrap + * a recording fake (trace asserts: full order, no writes when * the preflight gate is closed, publish strictly after poll A). */ @@ -429,14 +408,14 @@ ane_t6021_boot_prepare_publish(const struct ane_t6021_init_sources *s, #define ANE_T6021_BOOT_REG_SCRATCH1 0x0184004c #define ANE_T6021_BOOT_REG_SCRATCH6 0x01840060 #define ANE_T6021_BOOT_REG_SCRATCH7 0x01840064 -/* 13.5 stub execution-progress observables (M2StartupRecovery 2026-09-26 - * decode of a9c4b771294a6b11…). Only PROVEN-readable engine-window - * registers: the 24 MHz domain tick at VM 0x1160008 (watched advancing - * ~24 MHz post-release, 2026-09-25 handshake receipt) plus SCRATCH, - * RVBAR and CPU_STATUS already on the whitelist. The stub's VM 0x30c - * store to engine+0x1140008 is real, but the register's readability, - * width and reset state are UNPROVEN and no pre-release baseline exists - * — it is deliberately NOT read here (Main review 2026-09-26). +/* 13.5 stub execution-progress observables (from the selene image). + * Only engine-window registers known to be safe to read: the 24 MHz + * domain tick at VM 0x1160008 (it advances at ~24 MHz once the CPU is + * released) plus SCRATCH, RVBAR and CPU_STATUS already on the + * whitelist. The stub's VM 0x30c store to engine+0x1140008 is real, + * but the register's readability, width and reset state are UNKNOWN + * and no pre-release baseline exists — it is deliberately NOT read + * here. */ #define ANE_T6021_BOOT_REG_TICK 0x01160008 #define ANE_T6021_BOOT_TABLE_VALUE 0x01ff01ffU @@ -453,7 +432,7 @@ struct ane_t6021_boot_io { */ void (*publish_barrier)(void *ctx); /* Bounded phase log: emitted ONCE before each write/poll block so a - * crash source survives netconsole (never per-poll). + * crash source reaches a remote console first (never per-poll). */ void (*phase)(void *ctx, const char *what); void (*poll_wait)(void *ctx); /* 1 ms poll delay */ @@ -465,16 +444,15 @@ struct ane_t6021_boot_io { }; struct ane_t6021_boot_cfg { - int preflight_ok; /* EVERY prerequisite closed (Main: no + int preflight_ok; /* EVERY prerequisite closed (no * partial boot — the whole sequence or * nothing) */ - int preboot_table_mode; /* LIVE-FAULT gating of the pre-CPU table - * block (2026-09-20 wedge): + int preboot_table_mode; /* gating of the pre-CPU table block, + * whose writes have wedged the machine: * 0 = ABORT: -EAGAIN, zero writes * (accidental-repeat prevention), - * 1 = WRITE the table block (re-armed - * after the table-base analysis), + * 1 = WRITE the table block, * 2 = SKIP the table block, run the rest * (diagnostic: tests fw-alive without * the pre-CPU table config). @@ -484,11 +462,11 @@ struct ane_t6021_boot_cfg { * endpoint select) instead of 1 * (legacy ChMan/MBI). In RTBuddy mode the fw * may skip READY/DONE; listen on mailbox. - * Run contract: P4 READY is recorded, not + * Run semantics: P4 READY is recorded, not * required; no legacy publish/wake runs; * aliveness is gated by the RTKit HELLO. */ - int stop_after; /* fw-start-debug step bisect (2026-09-22): + int stop_after; /* fw-start-debug step bisect: * 0 = full run; N in 1..4 = stop AFTER * step N completes, return -ECANCELED * (1 grant tunables, 2 scratch, 3 rvbar, @@ -537,24 +515,21 @@ ane_t6021_boot_run(const struct ane_t6021_boot_io *io, *booted = 0; *scratch_result = 0; - /* Main: all gates checked BEFORE any boot write — no partial + /* All gates checked BEFORE any boot write — no partial * sequence when the preflight is closed. */ if (!cfg->preflight_ok) return -ENODATA; - /* pre-CPU engine table (pass4 receiver, pass5 gate: REQUIRED - * every clock/power enable; residual alias/indirect writer - * risk documented in the preflight list). - * LIVE-FAULT 2026-09-20: this block wedged the machine (P0 then - * silence — stall could be the FIRST write; P1 never reached; - * hardware reset 16:24:08) — pf_preboot_table_safe gates it - * OFF; nothing below runs until re-armed. + /* pre-CPU engine table (REQUIRED on every clock/power enable). + * This block has also wedged the machine (after the P0 marker, + * possibly at the FIRST write; P1 never reached), so + * preboot_table_mode decides whether it runs. */ switch (cfg->preboot_table_mode) { case 0: return -EAGAIN; /* BEFORE any write: accidental-repeat - * prevention (Main 2026-09-20) + * prevention */ case 1: io->phase(io->ctx, "P0 preboot-table"); @@ -571,18 +546,17 @@ ane_t6021_boot_run(const struct ane_t6021_boot_io *io, case 2: default: /* diagnostic skip: NO table writes — tests fw-alive - * without the pre-CPU table config (user override - * 2026-09-20). + * without the pre-CPU table config. */ io->phase(io->ctx, "P0 table SKIPPED (diagnostic)"); break; } - /* P-1: W8 write-grant tunables (proven no-abort class, - * w8-run.out: APERTURE_UNLOCKED — 12 engine-aperture writes + /* P-1: write-grant tunables (a write class that does not abort; + * it leaves the aperture unlocked — 12 engine-aperture writes * preparing the aperture; replayed verbatim). - * Per-write before/after discrimination (Main: attempt 3 stalled - * LAST phase = P-1, possibly FIRST tunable write). + * Per-write before/after phase markers, so a stall in P-1 + * names the exact tunable write. */ { static const struct { u32 off; u32 val; } tun[] = { @@ -687,7 +661,7 @@ ane_t6021_boot_run(const struct ane_t6021_boot_io *io, io->phase(io->ctx, "P5 prepare+publish"); /* S5-S6: prepare (alloc/fill; kernel backend owns DMA), then the * publish barrier, then publish suballoc DVA low32/high32, then - * wake. Barrier semantics (Main review 2026-09-20): the kernel + * wake. Barrier semantics: the kernel * backend's dma_wmb() is dmb oshst — it ORDERS the coherent fill * before the device publish; it is NOT a dsb st (which * additionally waits for completion). Ordering suffices for the @@ -724,7 +698,7 @@ ane_t6021_boot_run(const struct ane_t6021_boot_io *io, /* HANDSHAKE COMPLETE != firmware result success: the DONE ack is * the fw's handshake word; the SCRATCH0/1 result u64 it leaves - * behind has UNSOURCED success semantics (pass6: the fw writes a + * behind has UNSOURCED success semantics (the fw writes a * "result" whose meaning is not decoded). Expose it * raw; never infer success from booted alone. */ diff --git a/drivers/accel/ane/ane_t6021_fwload.c b/drivers/accel/ane/ane_t6021_fwload.c index 59688070ada62..68d775a38dcfe 100644 --- a/drivers/accel/ane/ane_t6021_fwload.c +++ b/drivers/accel/ane/ane_t6021_fwload.c @@ -1,66 +1,57 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* - * T602x/T8112 ANE firmware loader — installed-module port of the proven lab - * staging unit (receipts 2026-09-19-h14-w13-boot-contract.md, - * -w14-staging-proven.md; add-path proof - * 2026-09-28-h14-fsm-secure-park-decode/first-inference.md). + * T602x/T8112 ANE firmware loader. * * Implements, behind fw_load=1 (default on): * 1. request_firmware() of the SoC's pinned 13.5 image * (ane_t602x_soc.fw: selene on T602x, bia on T8112) - * 2. Validation via ane_fw_validate.h (shared with the offline - * regression tools/h14_boot_regression.c): sha256 pin, strict + * 2. Validation via ane_fw_validate.h: sha256 pin, strict * exact-image assertions (5 load commands, 2 pinned segments, * entry 0, bounded LC walk). * 3. dma_alloc_coherent on the ANE platform device: the buffer is * DART-mapped through the device's iommu group (stream 0 of the - * three bound instances, H14DartAudit). Segment-wise copy + * three bound instances). Segment-wise copy * (__TEXT fileoff 0x4000 -> vm 0, __DATA -> its vmaddr; vmsize * tail zero from the coherent alloc). Coherent memory needs no * explicit cache clean. - * 4. Which copy runs. By default the staged copy (own memory). The lab + * 4. Which copy runs. By default the staged copy (own memory). The * option fw_alias_reserved=1 (T6021 only) maps the copy iBoot * preloaded at SEG0/SEGi; probe refuses it unless no-map - * /reserved-memory nodes cover both windows (the lab m1n1 adds them, - * the packaged m1n1 1.6.1 does not). The staged copy - * runs: ane_fw_apply_boot_patches() first writes iBoot's runtime - * patches into it (DATA base, RTK_soc, revision, ASC addresses, - * stack guard, ASC tunables), and then it equals the preload byte - * for byte except the random guard (17 captures, - * receipts/2026-10-01-t602x-independent). That needs no reserved - * memory and no preload address, so it is the only mode on SoCs - * without a recorded placement (T6020, T6022, T8112). - * 5. W16 entry alias: the staged fw pages are ALIASED at the latched - * RVBAR entry on the device's default DMA domain. Placement - * contract, receipt receipts/2026-09-20-t6021-entry-alias.md: - * - live RVBAR read64 = 0x10000000001 (bit0 latched, entry - * bits 0x10000000000 = dart-ane vm base); lifecycle rule - * (rvbar-lifecycle 6288b0b): bit0 set => the RVBAR write - * branch is skipped, so the ASC fetches AT the latched entry; - * - netconsole 2026-09-20T17:48:19: apple-dart 285800000.iommu - * "translation fault ... code:0x2 (NO PGD FOR IOVA) at - * 0x100000dca10" from the pre-module userspace CPU release — - * the ASC fetched the entry region while nothing was mapped; - * - current-state table walk (kcore, 2026-09-20): entry - * pgd[16] = 0 while the fw leaves are valid -> the region the - * hardware fetches is unmapped and the fw surface is - * physically NON-contiguous (consecutive leaf PAs differ by - * -0x4000). The alias therefore copies each page's PA from - * the live fw translation (iommu_iova_to_phys), never from - * the kernel VA, and verifies every target page unmapped - * before mapping it. + * /reserved-memory nodes cover both windows (m1n1 1.6.1 adds + * none). The staged copy runs: ane_fw_apply_boot_patches() first + * writes iBoot's runtime patches into it (DATA base, RTK_soc, + * revision, ASC addresses, stack guard, ASC tunables), and then it + * equals the preload byte for byte except the random guard. That + * needs no reserved memory and no preload address, so it is the + * only mode on SoCs without a recorded placement (T6020, T6022, + * T8112). + * 5. Entry alias: the staged fw pages are ALIASED at the latched + * RVBAR entry on the device's default DMA domain. Placement: + * - RVBAR read64 = 0x10000000001 (bit0 latched, entry + * bits 0x10000000000 = dart-ane vm base); lifecycle rule: + * bit0 set => the RVBAR write branch is skipped, so the ASC + * fetches AT the latched entry; + * - a CPU release with nothing mapped there faults in + * apple-dart ("translation fault ... code:0x2 (NO PGD FOR + * IOVA) at 0x100000dca10"): the ASC fetches the entry region; + * - a walk of the DART tables shows entry pgd[16] = 0 while the + * fw leaves are valid -> the region the hardware fetches is + * unmapped and the fw surface is physically NON-contiguous + * (consecutive leaf PAs differ by -0x4000). The alias + * therefore copies each page's PA from the live fw + * translation (iommu_iova_to_phys), never from the kernel VA, + * and verifies every target page unmapped before mapping it. * Driver-managed only: iommu_map() on the attached domain — * apple_dart implements .map_pages + .iotlb_sync_map, so the * driver flushes the DART TLB; no raw TTBR/TCR writes. - * Collision ceiling: the iovad allocates top-down from ~4 TiB - * and total mappable RAM is 94.9 GiB (/proc/iomem), so the - * allocator floor 4 TiB - 96 GiB = 0x3e800000000 is far above - * the alias ceiling 0x10005000000; the per-page zero-check - * additionally guards each map. + * Collision ceiling: the iovad allocates top-down from ~4 TiB, + * so even with all RAM mapped (96 GiB: allocator floor + * 4 TiB - 96 GiB = 0x3e800000000) it stays far above the alias + * ceiling 0x10005000000; the per-page zero-check additionally + * guards each map. * - * NOT implemented: the boot step (CPU release + SCRATCH). The userspace - * hybrid stage owns the W8 grant writes; with the alias in place a - * release no longer faults the entry fetch. + * The boot step itself (CPU release + SCRATCH) is in ane_t6021_boot.c; + * with the alias in place a release no longer faults the entry fetch. * * Error ownership: every failure path releases the firmware and, once * allocated, the coherent buffer, and clears ane->fw_buf — remove() @@ -90,25 +81,24 @@ MODULE_FIRMWARE(ANE_FW_BIA_NAME); static bool fw_load = true; module_param(fw_load, bool, 0444); MODULE_PARM_DESC(fw_load, - "Validate + DART-map the 13.5 PRELOAD payload (default on: the proven add-path configuration)."); + "Validate + DART-map the 13.5 PRELOAD payload (default on)."); static unsigned int fw_extra_ram = 0x200000; module_param(fw_extra_ram, uint, 0444); MODULE_PARM_DESC(fw_extra_ram, - "Page-aligned owned RAM after the 5 MiB firmware allocation (default 0x200000, the proven add-path grant; maximum 16 MiB)."); + "Page-aligned owned RAM after the 5 MiB firmware allocation (default 0x200000; maximum 16 MiB)."); -/* Preloaded-placement alias, lab only: map the iBoot-reserved SEG0/SEGi +/* Preloaded-placement alias: map the iBoot-reserved SEG0/SEGi * phys at the entry IOVAs (the preload, with iBoot's patches in place). * Default off: the own-memory copy needs no reserved RAM, so it runs with - * the packaged m1n1 (T6021 own-memory boot: receipts/2026-10-01-t602x- - * independent, "Boot B"). Only SoCs whose placement is recorded honor it + * a stock m1n1. Only SoCs whose placement is recorded honor it * (ane_t602x_soc.preload_placement), and only over reserved RAM * (ane_t6021_fwload_placement_ok). */ static bool fw_alias_reserved; module_param(fw_alias_reserved, bool, 0444); MODULE_PARM_DESC(fw_alias_reserved, - "Lab, T6021: map reserved SEG0 0x10000848000+0xc4000 at entry and SEG1 0x10001400000+0x438000 after it (probe refuses unless no-map /reserved-memory nodes cover both); 0 (default) = run the staged copy with iBoot's patches replayed (own memory, the only mode on T6020/T6022)."); + "T6021: map reserved SEG0 0x10000848000+0xc4000 at entry and SEG1 0x10001400000+0x438000 after it (probe refuses unless no-map /reserved-memory nodes cover both); 0 (default) = run the staged copy with iBoot's patches replayed (own memory, the only mode on T6020/T6022)."); static bool ane_t6021_fw_alias_is_reserved(struct device *dev) { @@ -126,8 +116,8 @@ static const struct { u64 phys, len; } ane_t6021_fw_preload[] = { }; /* Only a no-map /reserved-memory node that the kernel took at boot keeps - * it off those pages. The lab m1n1 adds ane-firmware@ nodes; the packaged - * m1n1 1.6.1 adds none (receipts/2026-10-01-t6021-disk-boot, "Limits"). + * it off those pages. m1n1 1.6.1 adds none, so this mode needs a boot + * loader that reserves both windows (e.g. as ane-firmware@ nodes). */ static bool ane_t6021_fw_preload_reserved(void) { @@ -178,30 +168,28 @@ bool ane_t6021_fwload_options_ok(void) /* BINDING probe-top predicate. MUST be called before * devm_kzalloc / power / CPU release at every probe site. * The alloc-time check below runs as defense in depth. - * The lab envelope rule: 16 KiB-aligned, <= 16 MiB. Both alias - * modes map the whole allocation, so the grant needs no mode. + * fw_extra_ram must be 16 KiB-aligned and <= 16 MiB. Both alias + * modes map the whole allocation, so the bound needs no mode. */ return fw_extra_ram <= SZ_16M && IS_ALIGNED(fw_extra_ram, ANE_T6021_FW_ALIAS_PAGE); } /* Per-SoC data. RTK_soc_revision on T602x is a measured constant: m1n1 - * gives Linux no chip-revision. T6021: the 17 preload captures (0x11); - * T6020: ANEMinorVersion 1 on community rows 4e8224a39f40, dc53badc436a, - * ea8c35bd20bf, e6069fe959ad; T6022: 17 on row 90c6b9bf3e03 - * (receipts/2026-10-01-community-rows). T6020 and T6022 have the T6021 - * die-0 addresses (13.5 ADTs, t602x-ane.dtsi). + * gives Linux no chip-revision. T6021: 0x11, as in the preloaded images; + * T6020: ANEMinorVersion 1; T6022: ANEMinorVersion 17. T6020 and T6022 + * have the T6021 die-0 addresses (13.5 ADTs). * * pmu_pa: the firmware's power service programs the seven ANE ps words * through its DART at IOVA == PA (the PMU base it is given is the ANE_TD - * word: selene 0x28e084008, bia 0x23b70c010, receipts/2026-10-01-t8112-kit). + * word: selene 0x28e084008, bia 0x23b70c010). * macOS maps the page first; without it the first access faults (NO PMD - * FOR IOVA 0x28e084008, 2026-09-29) and the firmware halts. + * FOR IOVA 0x28e084008) and the firmware halts. * - * T8112 (receipts/2026-10-01-t8112-optin): the revision comes from the - * eFuse words iBoot reads, the ANE_SYS_CPU word is pmgr +0xc008, and the - * macOS (type 0x70) opens PWGATE "set" +0x8b8 before the ps words. The TM - * TD word is not known there, so trace_td is off. + * T8112: the revision comes from the eFuse words iBoot reads, the + * ANE_SYS_CPU word is pmgr +0xc008, and the macOS (type 0x70) opens + * PWGATE "set" +0x8b8 before the ps words. The TM TD word is not known + * there, so trace_td is off. */ const struct ane_t602x_soc ane_t6020_soc = { .soc = 0x6020, .soc_revision = 0x01, @@ -294,15 +282,13 @@ static int ane_t6021_fw_alias_map(struct ane_t6021 *ane, bool reserved) * The windows are adjacent, so the mapping is one * contiguous run of `mapped` bytes starting at entry — * tracked exactly, because teardown must never unmap a - * page that was not mapped (dart_unmap_pages WARNs on - * holes and the 2026-09-26 state-report unwind hit it). + * page that was not mapped (dart_unmap_pages WARNs on holes). * SEG1 IOVA = entry + 0xc4000 = 0x100000c4000 per the * pinned ADT segment-ranges record (0x10000000000 -> * 0x10000848000 0xc4000; 0x100000c4000 -> 0x10001400000 - * 0x438000). Commit e6612e9 shipped 0x1000000c4000 — one - * hex digit off, mapping DATA at 16 TiB instead: every - * fw_alias_reserved boot since 2026-09-24 left the fw - * DATA section unmapped past the SEG0 head. + * 0x438000). Note 0x100000c4000, not 0x1000000c4000: + * one hex digit more maps DATA at 16 TiB and leaves the + * fw DATA section unmapped past the SEG0 head. */ struct { u64 iova, phys, len; } win[] = { { 0x10000000000ull, ane_t6021_fw_preload[0].phys, @@ -313,9 +299,8 @@ static int ane_t6021_fw_alias_map(struct ane_t6021 *ane, bool reserved) unsigned int w, windows = fw_extra_ram ? ARRAY_SIZE(win) : 2; /* The remap is contiguous: SEG1 base = SEG0 base + SEG0 - * len (segment-ranges order). Derived, never hand-written - * — the standalone literal 0x1000000c4000 that shipped in - * e6612e9 was one digit off and mapped DATA at 16 TiB. + * len (segment-ranges order). Derived, never hand-written: + * a standalone literal is easy to get one digit off. * The remap base must also equal the latched entry, or * the fetch head is not where we mapped. */ @@ -378,7 +363,7 @@ static int ane_t6021_fw_alias_map(struct ane_t6021 *ane, bool reserved) err_unmap_mapped: /* Cleanup exactly the per-window bytes we mapped; windows - * are not assumed adjacent (live trace: a hole between + * are not assumed adjacent (there can be a hole between * SEG0 and SEGi), and foreign collision mappings are * never touched. */ @@ -445,9 +430,9 @@ static int ane_t6021_fw_alias_map(struct ane_t6021 *ane, bool reserved) entry, ane->fw_size / ANE_T6021_FW_ALIAS_PAGE, &ane->fw_iova, &pa0); /* The firmware's power service needs the pmgr sub-block mapped - * IOVA == PA in every vehicle (NO PMD FOR IOVA 0x28e084008, - * 2026-09-29); the staged-DMA alias branch skipped it and left - * a bisect run booting a halting fw. + * IOVA == PA in every mode, this staged-DMA alias branch + * included; without it the firmware faults (NO PMD FOR IOVA + * 0x28e084008) and halts. */ return ane_t6021_pmu_map(ane, dom); @@ -556,7 +541,7 @@ int ane_t6021_fwload_probe(struct ane_t6021 *ane) if (fw_extra_ram > SZ_16M || !IS_ALIGNED(fw_extra_ram, ANE_T6021_FW_ALIAS_PAGE)) return -EINVAL; /* The 64-bit coherent mask is set once in ane_t6021_probe, - * BEFORE rtkit_init allocates the rings (W15 review). + * BEFORE rtkit_init allocates the rings. */ ret = request_firmware(&fw, img->name, ane->dev); @@ -619,10 +604,10 @@ void ane_t6021_fwload_remove(struct ane_t6021 *ane) int i; /* Unmap exactly the per-window recorded extents: the - * reserved-alias windows are not assumed adjacent (live - * trace hole between SEG0 and SEGi) and unmapping bytes - * that were never mapped trips dart_unmap_pages - * (io-pgtable-dart.c:319 WARN, 2026-09-26). + * reserved-alias windows are not assumed adjacent (there + * can be a hole between SEG0 and SEGi) and unmapping bytes + * that were never mapped trips a WARN in dart_unmap_pages + * (io-pgtable-dart.c). */ if (dom) for (i = 0; i < ane->fw_alias_extn; i++) diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index eb4ae9e383e23..8cf2d3cea39bc 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -4,51 +4,45 @@ * * One module, two surfaces: * - boot/transport: fenced Linux-context firmware start (fw_start=1 - * default — the proven add-path config), shared boot/fwload - * contract units, then the legacy 13.5 ChMan transport for the - * three load-time commands (LOAD_PROGRAM 0x200, CREATE_PROCESS - * 0x202, PROCEDURE_CALL 0x204) driven through the ioctls in this - * file; + * default), shared boot/fwload helpers, then the legacy 13.5 ChMan + * transport for the three load-time commands (LOAD_PROGRAM 0x200, + * CREATE_PROCESS 0x202, PROCEDURE_CALL 0x204) driven through the + * ioctls in this file; * - DRM accel: BO_INIT/BO_FREE plus PROG_LOAD/PROC_CREATE/EXEC per * include/uapi/drm/ane_accel.h (ABI 2). Sections ride BOs the * user supplies; one global mutex serializes every firmware * command; completion = the legacy exchange's reply PLUS the * firmware's finish event on the IO_T2H ring. EXEC also returns the - * fw's target-to-host slots (the sequencer's per-step drain). + * fw's target-to-host slots (drained after every exchange). * - * Compiled defaults are the load-run.sh parameter list — a bare - * `insmod ane_t6021.ko` is the proven add-path configuration on - * boot 3ab812a3 (fw_load=1 fw_start=1 fw_start_dapf=0 legacy_only=1 - * legacy_query=1 scratch3_ack=1 poll_rx=1 ...) — except hello_wait_ms, - * 1000 in that list and 0 here (see its definition). + * The compiled defaults are the configuration that boots the firmware: + * a bare `insmod ane_t6021.ko` runs the whole sequence (fw_load=1 + * fw_start=1 legacy_only=1 legacy_query=1 scratch3_ack=1 poll_rx=1 ...). * Remaining parameters are overridable from sysfs for bisection only. * - * Division of labor (receipts/2026-09-22-t6021-rtkit-port): - * - CPU start: kernel-context start is the proven-safe route on this - * box — m1n1 RTBuddy path is dormant (W5/W6 MGMT session wall) and - * the 13.5 (22G74) selene image brings up the legacy ChMan - * transport after DONE without RTKit. Boot is fenced in - * ane_t6021_boot.c (entry fold, RVBAR skip-or-fold, scratch, - * RUN, poll A/B, publish/wake) and audited in - * tools/h14_boot_regression.c. + * Division of labor: + * - CPU start: from kernel context — the m1n1 RTBuddy path stays + * dormant (no MGMT session comes up) and the 13.5 (22G74) selene + * image brings up the legacy ChMan transport after DONE without + * RTKit. Boot is fenced in ane_t6021_boot.c (entry fold, RVBAR + * skip-or-fold, scratch, RUN, poll A/B, publish/wake). * - genpd/pmgr: Runtime PM + the DT power-domains binding owns the * raise; probe verifies ACTUAL on ane_cpu (pmgr window) before * the first engine read (G1 gate). The pre-CPU engine table block - * is intentionally NOT written (preboot_table_mode = 2 = SKIP, the - * 2026-09-20-era live-fault discriminator). macOS-form mpm_off is - * skipped: with fw_start_mpm_off=0 (the proven config) the - * boot-raised ane_sys_mpm stays on; the boot writes do not depend - * on it. + * is NOT written by default (preboot_table_mode = 2 = SKIP: the + * block has wedged the machine). The macOS-form mpm_off step is + * skipped: the boot-raised ane_sys_mpm stays on; the boot writes do + * not depend on it. * - mailbox: ane_rtclient_validate_chman sees the 'IPC ' surface the * fw published during boot; the three load-time commands travel * the IO ring (channel 1) over ane_rtclient_legacy_exchange. Each * command advances the cursor on the next 64-byte slot — resending - * to slot 0 wedges the ring (sequencer rule). + * to slot 0 wedges the ring. * - non-posted MMIO (DT "nonposted-mmio" -> IORESOURCE_MEM_NONPOSTED) - * is mandatory throughout the ANE aperture; posted writel froze the - * box. - * - Firmware alias: the staged copy (own memory, default) or, with the - * lab option fw_alias_reserved=1, the iBoot-reserved SEG0/SEGi phys + * is mandatory throughout the ANE aperture; posted writel hangs the + * machine. + * - Firmware alias: the staged copy (own memory, default) or, with + * fw_alias_reserved=1, the iBoot-reserved SEG0/SEGi phys * is aliased at the latched RVBAR entry; the subsequent kernel writes * into the entry region translate through dart-ane0 instead of * faulting. @@ -104,7 +98,7 @@ /* Doorbell (IPI) block, engine + 0x1844000: set +0, pending +0x8000, * ack +0xc000 (an interrupt is handled by reading +0x184c000 and writing - * +0x1850000, with no SoC difference, receipts/2026-10-01-t8112-ane). + * +0x1850000, with no SoC difference). */ #define ANE_IPI_OFF 0x1844000 @@ -112,10 +106,9 @@ #define ANE_ASC_CPU_STATUS_RUNNING BIT(0) #define ANE_ASC_CPU_STATUS_STOPPED BIT(1) -/* Boot-time module parameters — defaults are the proven add-path - * parameter list (boot 3ab812a3 / load-run.sh), except fw_alias_reserved, - * which defaults to own memory (receipts/2026-10-01-t602x-independent, - * "Boot B"). fw_load, fw_extra_ram and fw_alias_reserved live in +/* Boot-time module parameters — defaults are the configuration that + * boots the firmware, with fw_alias_reserved defaulting to own memory. + * fw_load, fw_extra_ram and fw_alias_reserved live in * ane_t6021_fwload.c and boot_prevent_nap in ane_t6021_boot.c (single * registration each). Remaining knobs are overridable from sysfs for * bisection only. @@ -123,12 +116,12 @@ static bool fw_start = true; module_param(fw_start, bool, 0444); MODULE_PARM_DESC(fw_start, - "Fenced Linux-context firmware start: stage + alias, then the boot sequence; default on (proven add-path config)."); + "Fenced Linux-context firmware start: stage + alias, then the boot sequence; default on."); static int fw_start_table_mode = 2; module_param(fw_start_table_mode, int, 0444); MODULE_PARM_DESC(fw_start_table_mode, - "Pre-CPU engine table block: 0 abort, 1 write, 2 skip (default, proven config)."); + "Pre-CPU engine table block: 0 abort, 1 write, 2 skip (default)."); static bool fw_start_rtb_mode; module_param(fw_start_rtb_mode, bool, 0444); @@ -136,16 +129,16 @@ MODULE_PARM_DESC(fw_start_rtb_mode, "RTBuddy/RTKit-app-endpoint select; default off (legacy ChMan transport)."); /* Drive RX by apple_rtkit_poll from a workqueue even though a recv - * IRQ exists (lab poll_rx=1, the proven add-path value: the raw recv - * line is unproven, so the worker is what services HELLO/EPMAP). + * IRQ exists (default on: the raw recv line is not relied on, so the + * worker is what services HELLO/EPMAP). */ static bool poll_rx = true; module_param(poll_rx, bool, 0444); MODULE_PARM_DESC(poll_rx, - "Drive RX by apple_rtkit_poll from a workqueue (default on, proven config)."); + "Drive RX by apple_rtkit_poll from a workqueue (default on)."); /* STARTEP every fw-announced app endpoint (>= 0x20) after a successful - * RTKit handshake (lab start_app_eps=1 default). + * RTKit handshake (default on). */ static bool start_app_eps = true; module_param(start_app_eps, bool, 0444); @@ -160,18 +153,17 @@ MODULE_PARM_DESC(scratch3_ack, static bool legacy_only = true; module_param(legacy_only, bool, 0444); MODULE_PARM_DESC(legacy_only, - "Use the pinned 13.5 legacy ChMan transport (default on, proven add-path config)."); + "Use the pinned 13.5 legacy ChMan transport (default on)."); static bool legacy_query = true; module_param(legacy_query, bool, 0444); MODULE_PARM_DESC(legacy_query, - "Service bounded startup allocations and CONFIG_GET; default on (proven config)."); + "Service bounded startup allocations and CONFIG_GET; default on."); /* 0 (default) skips RTKit in legacy mode, so the ANE mailbox never - * starts. Measured 2026-09-30 (receipts/2026-09-30-t6021-stock-mailbox): - * the 13.5 firmware sent no HELLO on any recorded boot (-ETIME after + * starts: the 13.5 firmware sends no HELLO (-ETIME after * 1000 ms), and starting the mailbox enables AIC2 884, a level line - * that then fired ~700,000 times/s (one CPU of hardirq time). A + * that then fires ~700,000 times/s (one CPU of hardirq time). A * firmware that speaks RTKit needs hello_wait_ms=1000. */ static unsigned int hello_wait_ms; @@ -208,7 +200,7 @@ struct ane_rtclient { struct reset_control *cpu_rst; struct delayed_work poll_work; - /* fw_start=1 view over the shared boot/fwload contract units. */ + /* fw_start=1 view over the shared boot/fwload helpers. */ struct ane_t6021 *fw; /* A CPU we released is (or may be) running: state HELD — surfaces, @@ -229,7 +221,7 @@ struct ane_rtclient { /* One reusable 16 KiB command buffer for every host command * (CONFIG_GET + the three ioctls). Protocol-legal to reuse: an * exchange returns 0 only after the IO slot flipped back to - * host-owned with a zero status (the lab's own completion + * host-owned with a zero status (the completion * predicate), so the firmware has fully consumed the previous * command. The legacy_buffers table stays for fw MALLOC replies, * which the firmware may reference forever (held until reboot). @@ -261,14 +253,14 @@ struct ane_t6021_fd { * for reuse (see ane_t6021_bo_release). Every other BO frees at its last * reference. * - * Per-BO cap raised 2026-09-30: the 256 KiB Qwen-class matvec weights seen - * by the H14 compiler are bounded by `reduction * columns * 2`, which - * reaches 20 MiB at (K,N)=(2048,5120). The same cap serves the - * Qwen4-attention (K,N)=(4096,4096) constant at 32 MiB and any H14 - * softmax/reduction with an 8 MiB table. Total BO bytes across all fds are - * capped separately (bo_total_max_mb) by an atomic counter, enforced at - * alloc and released when the memory is really freed. Program sections stay - * held until reboot, and the 38 Qwen programs alone hold about 2.6 GiB. + * Per-BO cap: matvec weights emitted by the H14 compiler are bounded by + * `reduction * columns * 2`, which reaches 20 MiB at (K,N)=(2048,5120). + * The same cap serves an attention constant at (K,N)=(4096,4096), 32 MiB, + * and any H14 softmax/reduction with an 8 MiB table. Total BO bytes + * across all fds are capped separately (bo_total_max_mb) by an atomic + * counter, enforced at alloc and released when the memory is really + * freed. Program sections stay held until reboot, and one large model's + * programs alone can hold gigabytes. * A cap of 0 refuses every BO_INIT with -ENOSPC; there is no unlimited * value. Every BO also needs IOVA below 4 GiB (32-bit DMA mask), so * allocations fail with -ENOMEM near that bound whatever the cap. @@ -333,8 +325,8 @@ static atomic_t ane_t6021_quarantined = ATOMIC_INIT(0); static LIST_HEAD(ane_t6021_bo_pool); static DEFINE_SPINLOCK(ane_t6021_bo_pool_lock); -/* Final put: the last handle or mapping is gone. The firmware never sees - * a freed IOVA (lab rule), so a fw_ref BO is never freed: a program +/* Final put: the last handle or mapping is gone. The firmware must never + * see a freed IOVA, so a fw_ref BO is never freed: a program * section stays held, because a cached firmware program keeps reading * it; an io BO goes to the pool, unless a quarantined firmware may still * write it. Every other BO frees here, so the BO cap bounds only the @@ -410,7 +402,7 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, unsigned int channel, unsigned int timeout_ms); -/* ---- boot-contract helpers (shared with boot.c / fwload.c) ---- */ +/* ---- boot helpers (shared with boot.c / fwload.c) ---- */ static int ane_rtclient_legacy_alloc(struct ane_rtclient *ane, u64 size) { @@ -549,7 +541,7 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, dma_rmb(); size = READ_ONCE(slot[1]); tag = READ_ONCE(slot[2]); - /* Lab rule: a valid MALLOC slot carries + /* A valid MALLOC slot carries * header 0 and a u32 tag; anything else * is a malformed ring and we stop. */ @@ -585,8 +577,8 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, ane_t6021_chman_layout[channel].size; goto out; } - /* legacy_fast_poll=1 (the proven add-path cadence): poll - * the rings every 50 us instead of sleeping 1-2 ms. + /* Fast poll: check the rings every 50 us instead of + * sleeping 1-2 ms. */ udelay(50); } @@ -598,11 +590,11 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, } /* Time to let the output writes land after the completion signal. - * Measured 2026-09-29 on some boots: the fw ack, the TD counter and the - * TQ words all report done ~0.13 ms before the output reaches DRAM (the - * output read as zeros in about 1 of 5 calls). The finish event arrives - * at the same time for short programs. No signal for "output landed" is - * known, so the wait is a fixed margin of about 8x the lag. + * On some boots the fw ack, the TD counter and the TQ words all report + * done ~0.13 ms before the output reaches DRAM (the output then reads + * as zeros). The finish event arrives at the same time for short + * programs. No signal for "output landed" is known, so the wait is a + * fixed margin of about 8x the lag. */ static unsigned int call_settle_us = 1000; module_param(call_settle_us, uint, 0644); @@ -616,20 +608,15 @@ MODULE_PARM_DESC(call_settle_us, * and also reads the last-committed-TD word. It reads that word only * while the seven ANE pmgr PS words read 0x3ff. No register is written. * - * Provenance (omarchy-ane commits): * - The TD word: engine + ane_t602x_soc.trace_td_off; on T602x TM - * 0x285c00000 + TD window 0x20400 + 0x58, the word the CALL wait polled - * from be2cf130d761ed675ad86f12b21ac1d53d906e1d until - * 3a942d6cf6278526fbc02bf0c4743c5c1b276cdb. 3a942d6 measured its - * layout: the call's nid in bits 23:16 (+1 per call), the index of the - * last task taken in bits 15:0 (receipts/2026-09-30-t6021-call-wait). - * Not known on T8112, so trace_td records nothing there. + * 0x285c00000 + TD window 0x20400 + 0x58. Its layout: the call's nid + * in bits 23:16 (+1 per call), the index of the last task taken in + * bits 15:0. Not known on T8112, so trace_td records nothing there. * - The seven ANE power-state words at ane_t602x_soc.pmu_pa + ps_off * (T602x: DT power-domains ane_sys_mpm 0x4000 .. ane_set4 0x4030 of - * the pmgr at 0x28e080000). The guard "PS words 0x3ff before any TM - * read; a TM read while the compute domains are off hangs the SoC" is - * ane_rtclient_pm_pwrstate_ok in 27e996a6de544a803a71d7a5c4ed11d828d4d049, - * and the CALL wait applied it until 3a942d6. + * the pmgr at 0x28e080000). The guard: PS words 0x3ff before any TM + * read, since a TM read while the compute domains are off hangs the + * SoC. * * Each record has a ktime_get_ns() stamp: CALL before the exchange, ACK * when the firmware acked it, TD for each new TD word value (word = the @@ -747,13 +734,11 @@ static void ane_t6021_trace_free(void) */ #define ANE_CALL_COOKIE 0xADD0 -/* IO_T2H event of a PROCEDURE_CALL, 0x28 bytes (measured 2026-09-30, - * receipts/2026-09-30-t6021-call-wait): u32 sequence, u32 0x300, u64 - * cookie, u32 program id, u32 process id, u32 0, u32 state. The firmware - * posts two per call: state 0 about 0.2-0.5 ms after the ack whatever the - * program length (a 3,597-task program takes its last task 252 ms later; - * receipts/2026-10-01-t6021-trace-td), and state 1 when the procedure has - * finished. +/* IO_T2H event of a PROCEDURE_CALL, 0x28 bytes: u32 sequence, u32 0x300, + * u64 cookie, u32 program id, u32 process id, u32 0, u32 state. The + * firmware posts two per call: state 0 about 0.2-0.5 ms after the ack + * whatever the program length (a long program can take its last task + * hundreds of ms later), and state 1 when the procedure has finished. */ #define ANE_T2H_CALL_COOKIE_OFF 0x08 #define ANE_T2H_CALL_STATE_OFF 0x1c @@ -780,12 +765,11 @@ static const void *ane_rtclient_fw_cpu(struct ane_rtclient *ane, u64 iova, return NULL; } -/* Return every firmware-owned slot on a target-to-host ring to the fw - * (legacy_t2h_ack=1, the lab default): the fw publishes by writing the - * DMA address with bit0 clear; the host returns the slot by setting - * bit0 and ringing the channel's doorbell bit, as the allocation ring - * does. The sequencer drained channels 4/6 after every step; the - * ioctls drain them after every completed exchange. Returns true when +/* Return every firmware-owned slot on a target-to-host ring to the fw: + * the fw publishes by writing the DMA address with bit0 clear; the host + * returns the slot by setting bit0 and ringing the channel's doorbell + * bit, as the allocation ring does. The ioctls drain channels 4/6 after + * every completed exchange. Returns true when * an IO_T2H slot held the finish event of a CALL. */ static bool ane_rtclient_drain_t2h(struct ane_rtclient *ane, @@ -882,10 +866,10 @@ static int ane_rtclient_call_wait_traced(struct ane_rtclient *ane, * (channel 6). The ack, the eight TQ status words and the last-committed * TD word (TM +0x20458: the call's nid in bits 23:16 and the index of the * last task taken in bits 15:0) all report done once the last task has - * been dispatched, not when it has run: measured 2026-09-30, Qwen program - * 20 (20 tasks) showed its last task index 0.22 ms after the ack, posted - * its finish event 3.5 ms after the ack, and a caller that returned at - * the first signal read an all-zero output. Returns 0 when the event + * been dispatched, not when it has run: a 20-task program can show its + * last task index 0.22 ms after the ack but post its finish event 3.5 ms + * after it, and a caller that returns at the first signal reads an + * all-zero output. Returns 0 when the event * arrived, -ETIMEDOUT else. */ static int ane_rtclient_call_wait(struct ane_rtclient *ane, @@ -920,7 +904,7 @@ static int ane_rtclient_command(struct ane_rtclient *ane, ane_t6021_trace->calls++; ane_t6021_trace_add(ANE_TR_CALL, 0); } - /* Producer contract hot path (ane_stats.h): record submit at + /* Producer-side stats hot path (ane_stats.h): record submit at * command enqueue, completion after the call returns, one * begin/complete pair per engine submission. Only * CSNE_CMD_PROCEDURE_CALL is engine work; the control-plane @@ -981,8 +965,8 @@ static int ane_rtclient_command(struct ane_rtclient *ane, ane_stats_complete(&ane->stats_ctrs, &ane->stats_ring, stats_ticket, ktime_get_ns(), 0, 0); /* The fw talks back on the target-to-host rings (fwlog, perf); - * hand those slots back so the rings never fill (the sequencer - * did this per step; same ack, channels 4 and 6). + * hand those slots back so the rings never fill (channels 4 + * and 6). */ ane_rtclient_drain_t2h(ane, 4); ane_rtclient_drain_t2h(ane, 6); @@ -995,8 +979,8 @@ static int ane_rtclient_command(struct ane_rtclient *ane, /* ---- LOAD / CREATE / CALL wrappers ---- */ /* The firmware never frees a program or a process, and its program table - * holds 256 entries: measured 2026-09-29, the 257th LOAD_PROGRAM on one - * boot answered with a protocol error and quarantined the device. Two + * holds 256 entries: the 257th LOAD_PROGRAM after a boot is answered + * with a protocol error, which quarantines the device. Two * loads of byte-identical sections therefore share one firmware program * and one process. The key is SHA-256 over every section's id, size and * bytes, so a client can only reach a program whose bytes it also @@ -1088,9 +1072,9 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, } } - /* Lab contract: the record slot IS the section identity (slot = - * id - 1; tdprop id 7 lands at slot 6 and slot 5 stays empty, - * exactly the proven add-path image). Reject out-of-range and + /* The record slot IS the section identity (slot = + * id - 1; tdprop id 7 lands at slot 6 and slot 5 stays empty + * when id 6 is not supplied). Reject out-of-range and * duplicate ids before building the wire image. */ for (i = 0; i < user->section_count; i++) { @@ -1179,7 +1163,7 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, memset(command->cpu, 0, SZ_16K); /* One 0x30-byte section record per supplied section, placed at * its IDENTITY slot: slot = id - 1 (tdprop id 7 lands at slot - * 6, slot 5 stays empty — exactly the proven add-path image: + * 6, slot 5 stays empty — e.g. an image with * six populated records, ids 1,2,3,4,5 then 7). Unsupplied * slots stay zero-filled (flags = 0). flags bit0 = 1, id at * +0x04, iova at +0x18, size at +0x20. @@ -1226,8 +1210,8 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, } /* Generic binds are accepted for ABI compatibility and unused: * the LOAD record already carries iova + size, and the generic - * section image is prepared by userspace (the proven add path - * ships it inside the section bytes, generic.bin). + * section image is prepared by userspace (it ships inside the + * section bytes, generic.bin). */ /* ProgramId placeholder; the firmware writes the assigned id @@ -1239,11 +1223,10 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, *(u32 *)(cmd + 0x1b8) = cpu_to_le32(U32_MAX); } - /* The proven add-path LOAD command is 0x1C0 bytes: the nine + /* The LOAD command is 0x1C0 bytes: the nine * 0x30 records end at 0x1b8, ProgramId at +0x1b8, 8 bytes of * zero tail. sizeof(struct ane_csne_cmd_load_program) is only - * 0x1b8, so the length is pinned here (h14_seq_first_add.py - * load_step). + * 0x1b8, so the length is pinned here. */ BUILD_BUG_ON(sizeof(struct ane_csne_cmd_load_program) != 0x1b8); ret = ane_rtclient_command(ane, command, @@ -1484,8 +1467,8 @@ static int ane_t6021_bo_init_ioctl(struct drm_device *drm, void *data, return -ENOMEM; } /* Every fw-visible DMA surface must be 16 KiB aligned and clear - * of the firmware entry alias (receipt 2026-09-20-t6021-entry-alias: - * the same invariant ane_rtclient_legacy_alloc enforces). + * of the firmware entry alias (the same invariant + * ane_rtclient_legacy_alloc enforces). */ if (!IS_ALIGNED(bo->dma, SZ_16K) || (ane->fw && !ane_t6021_fw_alias_iova_ok(ane->fw, bo->dma, @@ -1672,9 +1655,8 @@ static int ane_t6021_prog_load_ioctl(struct drm_device *drm, void *data, } /* Firmware perf mode: CSNE_CMD_CH_PROPERTY_WRITE (0x1f), channel 0, - * property 0x10aa, value 1. macOS sends it once during power-on (see - * the omarchy-ane perf-mode receipt); the 13.5 firmware treats it as - * its engine perf-mode switch. It is a runtime + * property 0x10aa, value 1. macOS sends it once during power-on; the + * 13.5 firmware treats it as its engine perf-mode switch. It is a runtime * switch (write 1 to /sys/module/ane_t6021/parameters/fw_perf_mode) so * its effect on call time can be measured on one boot. Only 1 is * accepted: no other value is known to be safe. @@ -1872,8 +1854,8 @@ static const struct apple_rtkit_ops ane_rtclient_rtkit_ops = { .shmem_destroy = ane_rtclient_shmem_destroy, }; -/* ---- poll worker: RX fallback while the recv line is unproven ---- - * (lab poll_rx=1). Runs every 10 ms until the handshake completes, +/* ---- poll worker: RX fallback, the recv line is not relied on ---- + * Runs every 10 ms until the handshake completes, * then every second while poll_rx stays on. Armed BEFORE the host ack * so the fw's HELLO is never missed. */ @@ -1896,7 +1878,7 @@ static void ane_rtclient_post_boot(struct work_struct *w) /* STARTEP every fw-announced app endpoint (>= 0x20; the fw mgmt * dispatcher starts an endpoint on flag bit 1). Same call every Asahi - * RTKit client makes; the lab ran it with start_app_eps=1. + * RTKit client makes. */ static void ane_rtclient_start_app_eps(struct ane_rtclient *ane) { @@ -2033,7 +2015,7 @@ static int ane_rtclient_probe(struct platform_device *pdev) /* Early, BEFORE any allocation/power: legacy_only must never be * rejected after the CPU release, where an unwind could drop - * domains under a running ASC (lab probe order). + * domains under a running ASC. */ if (legacy_only && (!fw_start || fw_start_rtb_mode)) { dev_err(dev, @@ -2132,7 +2114,7 @@ static int ane_rtclient_probe(struct platform_device *pdev) return -EPROBE_DEFER; } - /* Lab fw_start order: staging requires fw_load and an + /* fw_start order: staging requires fw_load and an * IOMMU-mapped device BEFORE any allocation/staging. */ if (!ane_t6021_fwload_requested()) { @@ -2225,8 +2207,7 @@ static int ane_rtclient_probe(struct platform_device *pdev) ANE_T6021_BOOT_ACK); /* hello_wait_ms > 0 only: init rtkit and arm the RX * poll worker BEFORE writing the ack, so a HELLO - * after the ack is not missed. The 13.5 fw sent none - * on any recorded boot. + * after the ack is not missed. The 13.5 fw sends none. */ if (hello_wait_ms && !ane->rtk) { ane->rtk = devm_apple_rtkit_init(dev, ane, @@ -2289,7 +2270,7 @@ static int ane_rtclient_probe(struct platform_device *pdev) if (!cfg_err && legacy_query && ane->chman_ok) { /* CONFIG_GET (opcode 0x03) keeps the legacy ChMan * transport armed for the ioctls; the reply's - * word +0x08 must be nonzero (lab rule). The + * word +0x08 must be nonzero. The * boot heap + 'IPC ' allocations stay HELD until * reboot. */ @@ -2342,7 +2323,7 @@ static int ane_rtclient_probe(struct platform_device *pdev) goto err_pm_or_hold; } /* RX fallback worker first: the fw's HELLO must not be - * missed while the recv line is unproven. + * missed, and the recv line is not relied on. */ schedule_delayed_work(&ane->poll_work, msecs_to_jiffies(10)); deadline = jiffies + msecs_to_jiffies(hello_wait_ms); diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index ba234d57cdf74..e4fab5e3225ff 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -61,12 +61,10 @@ static const int TQ_PRTY_TABLE[ANE_TQ_COUNT] = { 0x1, 0x2, 0x3, 0x4, /* The ane SET block (m1n1 ANE.ps_map) maps the pmgr power-state words * for this engine (set0, base, set1..4). pmgr reads are always safe and - * give recovery its ACTUAL power evidence; the SET words themselves are - * firmware-locked: a direct write external-aborts the SoC. Bisect - * evidence, 2026-09-16: the T6001 kill is netconsole-named at - * 0x28e08c000 (PS_SET0 down, all sets gated); T8103 hard-reset the same - * way at 0x23b70c000 with the 95dbcf3-era gate armed. No code here ever - * writes the SET block. + * give recovery its ACTUAL power state; the SET words themselves are + * firmware-locked: a direct write external-aborts the SoC (T6001 at + * 0x28e08c000 with PS_SET0 down and all sets gated; T8103 hard-resets + * the same way at 0x23b70c000). No code here ever writes the SET block. */ #define ANE_PS_ACTUAL_MASK 0xf0 #define ANE_PS_WORDS 6 /* set0, base, set1..4 */ @@ -91,7 +89,7 @@ u32 ane_ps_act(struct ane_device *ane) /* Bisect probe: per-word named reads of the SET window. Each word logs * before its readl, so a window whose address does not decode on this - * SoC is named by the last off-box line (word index + byte offset) + * SoC is named by the last line logged (word index + byte offset) * instead of a silent hard reset between two other prints. The lines * are debug messages: dynamic debug turns them on. */ @@ -121,7 +119,7 @@ u32 ane_ps_act_probe(struct ane_device *ane) /* Recovery-path MMIO logging. Every write prints before and after, and * every engine read prints its value, each line carrying the pmgr * ACTUAL of the owning partitions. A write that external-aborts the - * SoC is then named by the last line the off-box netconsole carried, + * SoC is then named by the last line that reached a remote console, * and the bisect starts from that register instead of a guess. The * lines are debug messages: dynamic debug turns them on. */ @@ -283,12 +281,12 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) for (;;) { /* Latch-first completion poll. Engine register reads hang - * jwm1 once a DART fault has landed (lockups 1 and 3: the - * journal ends inside the 1-microsecond completion poll, - * before any fault handling ran), so the poll op consults - * the DART error latch first and stops the poll before the - * engine is touched. Residual hazard: a fault landing - * between one iteration's latch read and engine read. + * the SoC (T8103) once a DART fault has landed, inside the + * 1-microsecond completion poll and before any fault + * handling can run, so the poll op consults the DART error + * latch first and stops the poll before the engine is + * touched. Residual hazard: a fault landing between one + * iteration's latch read and engine read. */ err = read_poll_timeout(ane_tm_poll_step, status, status != 0, 1, 1000000, false, @@ -327,7 +325,7 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) } status = tq_read32(ane, TQ_NID1(req->qid)); tq_write32(ane, TQ_NID1(req->qid), status & ~1U); - /* Kept from the qualified completion sequence (clear the + /* Kept from the established completion sequence (clear the * NID valid bit, then the queue status). writel() already * orders the two writes to the engine. */ @@ -393,7 +391,7 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) * wedge clears, BO_FREE unmaps them through the normal path again. * Callers hold engine_lock. * - * T6001 note, bisect evidence 2026-09-16: its set0/base islands are + * T6001 note: its set0/base islands are * firmware-locked (direct pmgr writes external-abort; see the SET block * comment above) and hold the tm/tq file in retention through any * genpd cycle - TQ_EN still reads 0x3000 after a 20 ms and a 2 s gate, @@ -506,7 +504,7 @@ static void ane_tm_drain_retained(struct ane_device *ane) int line, qid; /* Stop the TM first: the cycle leaves the enable|halt latch set - * (TQ_EN 0x3000, 2026-09-16 evidence). Clear both bits so the + * (TQ_EN reads 0x3000). Clear both bits so the * re-arm below starts the TM from a defined stopped state. */ ane_rec_writel(ane, "TM_TQ_EN stop tm+0x0c", @@ -564,8 +562,8 @@ static void ane_tm_drain_retained(struct ane_device *ane) tq_write32(ane, TQ_NID1(qid), nid & ~1U); } - /* Retention evidence for the off-box console: what the dead task - * left in the committed counter and the error latches. + /* Log the retained state: what the dead task left in the + * committed counter and the error latches. */ ane_rec_read32(ane, "TM_COMMITTED tm+0x44 (drain)", ane->engine + ANE_TM_BASE + TM_COMMITTED); From 9e1cf874add1c32e963f73671ca0137a0ad30248 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:56:17 +1100 Subject: [PATCH 1356/2680] accel/ane: Remove blank lines at the end of two files The ANE sysfs ABI description and ane_t6021_boot.c end in blank lines, which git diff --check reports. Remove them. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/ABI/testing/sysfs-class-accel-ane | 2 -- drivers/accel/ane/ane_t6021_boot.c | 1 - 2 files changed, 3 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-class-accel-ane b/Documentation/ABI/testing/sysfs-class-accel-ane index 668bed9c7cb38..af17b34317238 100644 --- a/Documentation/ABI/testing/sysfs-class-accel-ane +++ b/Documentation/ABI/testing/sysfs-class-accel-ane @@ -46,5 +46,3 @@ Description: Users: performance samplers that compute engine utilization and job rate from the two counters. - - diff --git a/drivers/accel/ane/ane_t6021_boot.c b/drivers/accel/ane/ane_t6021_boot.c index 709bb7df52ef2..7c59d18fc6223 100644 --- a/drivers/accel/ane/ane_t6021_boot.c +++ b/drivers/accel/ane/ane_t6021_boot.c @@ -475,4 +475,3 @@ int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, sres); return r; } - From cfb2082189d219127b9bc1f465f3de448d477bc0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:10:00 +1100 Subject: [PATCH 1357/2680] soc: apple: mcc: factor out disabling and flushing planes Move the loop that stops allocation and flushes each plane out of apple_mcc_quiesce() into apple_mcc_disable_planes(), which takes the number of planes to act on, so that the next patch can use it on the planes enabled before a failure. No functional change. (cherry picked from commit 4cf2113dc36329544edc54b09dc775c5888c3975) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/mcc.c | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/drivers/soc/apple/mcc.c b/drivers/soc/apple/mcc.c index 10f5e6e01e8cd..d0f6c1362b308 100644 --- a/drivers/soc/apple/mcc.c +++ b/drivers/soc/apple/mcc.c @@ -72,6 +72,24 @@ static bool apple_mcc_plane_ready(struct apple_mcc *mcc, unsigned int plane) return false; } +/* Stop allocating on the first @count planes, then write back and invalidate them. */ +static void apple_mcc_disable_planes(struct apple_mcc *mcc, unsigned int count) +{ + unsigned int i; + u32 val; + + for (i = 0; i < count; i++) { + writel(1, mcc->planes[i] + MCC_ALLOC_DISABLE); + writel(1, mcc->planes[i] + MCC_CACHE_FLUSH); + } + + for (i = 0; i < count; i++) + if (readl_poll_timeout_atomic(mcc->planes[i] + MCC_CACHE_FLUSH, val, + !(val & MCC_CACHE_FLUSH_BUSY), 10, + MCC_FLUSH_TIMEOUT_US)) + dev_err(mcc->dev, "plane %u flush timed out\n", i); +} + static int apple_mcc_enable(struct apple_mcc *mcc) { unsigned int i; @@ -91,26 +109,12 @@ static int apple_mcc_enable(struct apple_mcc *mcc) return 0; } -/* Stop allocating, then write back and invalidate the cache. */ static void apple_mcc_quiesce(struct apple_mcc *mcc) { - unsigned int i; - u32 val; - if (!mcc->enabled) return; - for (i = 0; i < mcc->nr_planes; i++) { - writel(1, mcc->planes[i] + MCC_ALLOC_DISABLE); - writel(1, mcc->planes[i] + MCC_CACHE_FLUSH); - } - - for (i = 0; i < mcc->nr_planes; i++) - if (readl_poll_timeout_atomic(mcc->planes[i] + MCC_CACHE_FLUSH, val, - !(val & MCC_CACHE_FLUSH_BUSY), 10, - MCC_FLUSH_TIMEOUT_US)) - dev_err(mcc->dev, "plane %u flush timed out\n", i); - + apple_mcc_disable_planes(mcc, mcc->nr_planes); mcc->enabled = false; } From ae137c56d7cb0093c7f18fa6865f4f6a4b3155c7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:10:03 +1100 Subject: [PATCH 1358/2680] soc: apple: mcc: disable the planes already enabled when enabling fails apple_mcc_enable() clears the allocation-disable bit plane by plane and reads it back. If a later plane does not take it, probe returns -EIO with the earlier planes allocating. The driver does not bind, so neither remove() nor shutdown() runs, and nothing stops allocation or writes the cache back before the next kernel or the firmware takes over, which is what the driver promises to do on shutdown. Disable and flush the planes enabled so far, and the one that failed, whose state is unknown, before returning the error. The failure needs a plane whose register does not take the write, which cannot be produced on a working machine or with memory standing in for the registers, so there is no automated test; the normal path is unchanged, and the on-device check is that probe still reports allocation enabled on every plane. (cherry picked from commit 8bf158a74ea7e2a5a7e42082fc882d3508119a09) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/mcc.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/soc/apple/mcc.c b/drivers/soc/apple/mcc.c index d0f6c1362b308..7b8c2e2220c8b 100644 --- a/drivers/soc/apple/mcc.c +++ b/drivers/soc/apple/mcc.c @@ -102,6 +102,12 @@ static int apple_mcc_enable(struct apple_mcc *mcc) writel(0, mcc->planes[i] + MCC_ALLOC_DISABLE); if (readl(mcc->planes[i] + MCC_ALLOC_DISABLE)) { dev_err(mcc->dev, "plane %u did not enable allocation\n", i); + /* + * The driver will not bind, so nothing disables these + * planes on shutdown: disable the ones enabled so far, + * and this one, whose state is unknown, now. + */ + apple_mcc_disable_planes(mcc, i + 1); return -EIO; } } From bdd6e4ea9393a914139ea91e3e74042ee4f8439b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:10:45 +1100 Subject: [PATCH 1359/2680] pmdomain: apple: pmc: undo a partial agent enable apple_pmc_agent_enable() changes the agent's bit rail by rail and stops at the first rail that does not take the write. The rails before it keep the change, and pmc->enable[] records the failed one as changed too. For a dynamic voter domain genpd treats a failed power_on() as the domain staying off and never calls power_off(), so the agent stays enabled on those rails for good; a failed disable likewise leaves it half disabled while genpd keeps the domain on. Snapshot the enables before the change and, on failure, put back the rails changed so far and the one that failed, so the agent is in the same state on every rail and that state matches what genpd records. The failure needs a rail enable word that does not take the write, which memory standing in for the registers cannot produce, so the KUnit test added with the next patch covers the successful path only. (cherry picked from commit 4653ef5a873f16c68a1e9e9080a41395628ca9d4) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmc.c | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 8510750d125aa..be086c5eed0a9 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -151,18 +151,46 @@ static int apple_pmc_enable_update(struct apple_pmc *pmc, unsigned int rail, return apple_pmc_enable_write(pmc, rail); } +/* + * Caller holds pmc->lock. Put the rails up to and including @last back to + * the enables in @prev, after a change failed on @last. + */ +static void apple_pmc_enable_restore(struct apple_pmc *pmc, const u32 *prev, + unsigned int last) +{ + unsigned int rail; + + for (rail = 0; rail <= last; rail++) { + pmc->enable[rail] = prev[rail]; + apple_pmc_enable_write(pmc, rail); + } +} + +/* + * Enable or disable @agent on every rail. If a rail does not take the + * change, undo it on the rails changed so far: genpd keeps a domain whose + * power_on() fails off, and never calls its power_off(), so an agent left + * enabled on some rails would stay enabled for good. + */ static int apple_pmc_agent_enable(struct apple_pmc *pmc, unsigned int agent, bool enable) { + u32 prev[APPLE_PMC_MAX_RAILS]; unsigned long flags; unsigned int rail; int ret = 0; raw_spin_lock_irqsave(&pmc->lock, flags); - for (rail = 0; rail < pmc->hw->rails && !ret; rail++) + memcpy(prev, pmc->enable, sizeof(prev)); + for (rail = 0; rail < pmc->hw->rails; rail++) { ret = apple_pmc_enable_update(pmc, rail, enable ? BIT(agent) : 0, enable ? 0 : BIT(agent)); + if (ret) { + apple_pmc_enable_restore(pmc, prev, rail); + break; + } + } raw_spin_unlock_irqrestore(&pmc->lock, flags); return ret; From 105a06e2fe2cb4e77db3ff93863b136a4bd89025 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:13:13 +1100 Subject: [PATCH 1360/2680] pmdomain: apple: pmc: restore the AP's own DCS vote when a limit is lifted The DCS limit votes through the AP's agent on behalf of the agents it stops honouring. When the limit is lifted, and when the driver is torn down, apple_pmc_dcs_vote() writes 0 into the AP agent's DCS field. That discards whatever the AP agent voted before Linux took over, so a single limit cycle changes the boot configuration of the memory controller. Read the AP agent's DCS vote at probe, before anything writes it, and put it back instead of 0 whenever no limit applies. Add a KUnit test with memory standing in for the registers. It checks that agent enables only change the agent's own bit, and that after a limit cycle the AP agent's DCS vote is the one it started with; the latter fails without this change (the vote reads 0) and passes with it. (cherry picked from commit a86e8e5637c07504ca09b7bfa5639492af065d8c) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/Kconfig | 10 ++++ drivers/pmdomain/apple/pmc-test.c | 95 +++++++++++++++++++++++++++++++ drivers/pmdomain/apple/pmc.c | 22 ++++++- 3 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 drivers/pmdomain/apple/pmc-test.c diff --git a/drivers/pmdomain/apple/Kconfig b/drivers/pmdomain/apple/Kconfig index 2494e7aece065..2a70d364f88c0 100644 --- a/drivers/pmdomain/apple/Kconfig +++ b/drivers/pmdomain/apple/Kconfig @@ -34,4 +34,14 @@ config APPLE_PMC GPU, while its device is powered on. It can also limit the DRAM controller rail as a thermal cooling device. +config APPLE_PMC_KUNIT_TEST + bool "KUnit tests for the Apple PMC voter interface" if !KUNIT_ALL_TESTS + depends on APPLE_PMC && KUNIT=y + default KUNIT_ALL_TESTS + help + Unit tests for the Apple PMC driver. They check that enabling and + disabling a voter agent only changes that agent on each rail, and + that lifting a DCS limit gives the AP agent its own vote back. The + registers are simulated in memory. Only useful for kernel developers. + endif diff --git a/drivers/pmdomain/apple/pmc-test.c b/drivers/pmdomain/apple/pmc-test.c new file mode 100644 index 0000000000000..3f0e277b21255 --- /dev/null +++ b/drivers/pmdomain/apple/pmc-test.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * KUnit tests for the Apple PMC voter interface: agent enables and the + * AP's DCS vote, with memory standing in for the registers. + * + * Included by pmc.c. + */ + +#include + +#define APPLE_PMC_TEST_RAILS_SIZE 0x4000 + +static struct apple_pmc *apple_pmc_test_alloc(struct kunit *test) +{ + const struct apple_pmc_hw *hw = &apple_pmc_hw_t8140; + struct apple_pmc *pmc; + + pmc = kunit_kzalloc(test, sizeof(*pmc), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pmc); + pmc->hw = hw; + raw_spin_lock_init(&pmc->lock); + pmc->voters = (void __iomem *)kunit_kzalloc(test, + hw->agents * hw->agent_stride, + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, (void __force *)pmc->voters); + pmc->rails = (void __iomem *)kunit_kzalloc(test, + APPLE_PMC_TEST_RAILS_SIZE, + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, (void __force *)pmc->rails); + + return pmc; +} + +static void apple_pmc_test_set_dcs_vote(struct apple_pmc *pmc, + unsigned int agent, unsigned int state) +{ + void __iomem *reg = pmc->voters + agent * pmc->hw->agent_stride; + + writel(state << APPLE_PMC_RAIL_SHIFT(pmc->hw->dcs_rail), reg); +} + +/* Enabling and disabling an agent changes its bit on every rail only. */ +static void apple_pmc_test_agent_enable(struct kunit *test) +{ + struct apple_pmc *pmc = apple_pmc_test_alloc(test); + unsigned int rail; + + pmc->enable[0] = BIT(0); + writel(BIT(0), pmc->rails + pmc->hw->rail_enable); + + KUNIT_ASSERT_EQ(test, apple_pmc_agent_enable(pmc, 1, true), 0); + for (rail = 0; rail < pmc->hw->rails; rail++) + KUNIT_EXPECT_EQ(test, apple_pmc_enable_read(pmc, rail), + BIT(1) | (rail ? 0 : BIT(0))); + + KUNIT_ASSERT_EQ(test, apple_pmc_agent_enable(pmc, 1, false), 0); + for (rail = 0; rail < pmc->hw->rails; rail++) + KUNIT_EXPECT_EQ(test, apple_pmc_enable_read(pmc, rail), + rail ? 0 : BIT(0)); +} + +/* Lifting a DCS limit gives the AP's agent back its own vote, not 0. */ +static void apple_pmc_test_dcs_limit_restores_ap_vote(struct kunit *test) +{ + struct apple_pmc *pmc = apple_pmc_test_alloc(test); + const struct apple_pmc_hw *hw = pmc->hw; + + apple_pmc_test_set_dcs_vote(pmc, hw->ap_agent, 2); + apple_pmc_test_set_dcs_vote(pmc, 0, 4); + pmc->enable[hw->dcs_rail] = BIT(0) | BIT(hw->ap_agent); + pmc->ap_dcs_vote = apple_pmc_ap_dcs_read(pmc); + KUNIT_ASSERT_EQ(test, pmc->ap_dcs_vote, 2); + + /* A limit of 3 lets the AP vote at most DCS 1 for the PMP's 4. */ + pmc->dcs_state = 3; + apple_pmc_dcs_vote(pmc); + KUNIT_EXPECT_EQ(test, apple_pmc_ap_dcs_read(pmc), 1); + + pmc->dcs_state = 0; + apple_pmc_dcs_vote(pmc); + KUNIT_EXPECT_EQ(test, apple_pmc_ap_dcs_read(pmc), 2); +} + +static struct kunit_case apple_pmc_test_cases[] = { + KUNIT_CASE(apple_pmc_test_agent_enable), + KUNIT_CASE(apple_pmc_test_dcs_limit_restores_ap_vote), + {} +}; + +static struct kunit_suite apple_pmc_test_suite = { + .name = "apple-pmc", + .test_cases = apple_pmc_test_cases, +}; + +kunit_test_suite(apple_pmc_test_suite); diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index be086c5eed0a9..bce3cd7619b77 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -92,6 +92,8 @@ struct apple_pmc { struct dentry *debugfs; /* Cooling state of the DCS limit: 0 is none, dcs_max holds DCS at 0. */ unsigned long dcs_state; + /* DCS state the AP's agent voted for before probe, kept without a limit. */ + unsigned int ap_dcs_vote; /* Power cost of each DCS state in mW, if a model is described. */ u32 dcs_power[APPLE_PMC_MAX_STATES]; struct delayed_work dcs_work; @@ -315,16 +317,27 @@ static unsigned int apple_pmc_dcs_demand(struct apple_pmc *pmc) return min(demand, hw->dcs_max); } +/* The DCS state the AP's agent votes for. */ +static unsigned int apple_pmc_ap_dcs_read(struct apple_pmc *pmc) +{ + const struct apple_pmc_hw *hw = pmc->hw; + u32 floor = readl(pmc->voters + hw->ap_agent * hw->agent_stride); + + return (floor & APPLE_PMC_RAIL_MASK(hw->dcs_rail)) >> + APPLE_PMC_RAIL_SHIFT(hw->dcs_rail); +} + /* * Caller holds pmc->lock. Vote through the AP's agent for what the other - * agents ask for, capped by the limit; vote nothing without a limit. + * agents ask for, capped by the limit; without a limit, put back the vote + * the AP's agent had before probe. */ static void apple_pmc_dcs_vote(struct apple_pmc *pmc) { const struct apple_pmc_hw *hw = pmc->hw; void __iomem *reg = pmc->voters + hw->ap_agent * hw->agent_stride; u32 mask = APPLE_PMC_RAIL_MASK(hw->dcs_rail); - unsigned int state = 0; + unsigned int state = pmc->ap_dcs_vote; u32 old, val; if (pmc->dcs_state) @@ -703,6 +716,7 @@ static int apple_pmc_probe(struct platform_device *pdev) "PMC is not responding\n"); pmc->enable[rail] = pmc->saved_enable[rail]; } + pmc->ap_dcs_vote = apple_pmc_ap_dcs_read(pmc); count = of_property_count_u32_elems(np, "apple,static-voters"); if (count > 0) { @@ -797,3 +811,7 @@ module_init(apple_pmc_init); MODULE_DESCRIPTION("Apple SoC PMC voter interface driver"); MODULE_LICENSE("Dual MIT/GPL"); + +#if IS_ENABLED(CONFIG_APPLE_PMC_KUNIT_TEST) +#include "pmc-test.c" +#endif From eac6d6241388f94092314df4ff12ec603024d9de Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:21:26 +1100 Subject: [PATCH 1361/2680] soc: apple: clpc: remove the debugfs override of the floor The driver has a writable debugfs file, force, that pins the DCS and fabric floors to fixed states regardless of CPU demand. It was used to measure what each state costs and gains while the default policy was chosen. Nothing uses it any more, and a debugfs file that changes memory-controller performance behind the driver's back is not something to keep in a driver meant for upstream. Remove it. The read-only status file stays. No functional change outside debugfs. (cherry picked from commit dba445e6732685313e036e73dc2753c7ed8cbcb6) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/clpc.c | 55 ++-------------------------------------- 1 file changed, 2 insertions(+), 53 deletions(-) diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index 2d66b98c3a9be..911163f13b0c0 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -35,7 +35,6 @@ #include #include #include -#include #include #define APPLE_CLPC_MAX_CLUSTERS 8 @@ -122,7 +121,6 @@ struct apple_clpc { u8 map[APPLE_CLPC_RAILS][APPLE_CLPC_LEVELS]; unsigned int gpu_state[APPLE_CLPC_RAILS]; struct apple_clpc_vote vote[APPLE_CLPC_RAILS]; - int force[APPLE_CLPC_RAILS]; u32 mask; u32 saved_floor; bool gpu_on; @@ -286,7 +284,6 @@ static void apple_clpc_sample(struct apple_clpc *clpc) unsigned int state[APPLE_CLPC_RAILS]; unsigned long now = jiffies; unsigned int i, rail; - int force; clpc->samples++; if (!cpumask_equal(clpc->covered, cpu_possible_mask) && @@ -312,9 +309,6 @@ static void apple_clpc_sample(struct apple_clpc *clpc) demand[rail] = max(demand[rail], clpc->gpu_state[rail]); state[rail] = apple_clpc_vote(&clpc->vote[rail], demand[rail], now); - force = READ_ONCE(clpc->force[rail]); - if (force >= 0) - state[rail] = force; } if (apple_pmc_floor_update(clpc->pmc, clpc->agent, clpc->mask, @@ -345,12 +339,12 @@ static int apple_clpc_status_show(struct seq_file *s, void *unused) apple_pmc_floor_read(clpc->pmc, clpc->agent), clpc->saved_floor); for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) - seq_printf(s, "%s: rail %u state %u enabled %u force %d map %u,%u,%u,%u gpu %u\n", + seq_printf(s, "%s: rail %u state %u enabled %u map %u,%u,%u,%u gpu %u\n", rail == APPLE_CLPC_DCS ? "dcs" : "soc", clpc->hw->rail[rail], clpc->vote[rail].state, apple_pmc_voter_enabled(clpc->pmc, clpc->agent, clpc->hw->rail[rail]), - READ_ONCE(clpc->force[rail]), clpc->map[rail][0], + clpc->map[rail][0], clpc->map[rail][1], clpc->map[rail][2], clpc->map[rail][3], clpc->gpu_state[rail]); seq_printf(s, "gpu: %s\n", !clpc->gpu_regmap ? "no input" : @@ -370,48 +364,6 @@ static int apple_clpc_status_show(struct seq_file *s, void *unused) } DEFINE_SHOW_ATTRIBUTE(apple_clpc_status); -/* - * Diagnostic override: "DCS SOC" holds each rail at a fixed state, -1 - * returns that rail to demand control; "-1" alone clears both. - */ -static ssize_t apple_clpc_force_write(struct file *file, - const char __user *ubuf, size_t len, - loff_t *pos) -{ - struct apple_clpc *clpc = file->private_data; - int v[APPLE_CLPC_RAILS], n; - unsigned int rail; - char buf[32]; - - if (len >= sizeof(buf)) - return -EINVAL; - if (copy_from_user(buf, ubuf, len)) - return -EFAULT; - buf[len] = '\0'; - - n = sscanf(buf, "%d %d", &v[APPLE_CLPC_DCS], &v[APPLE_CLPC_SOC]); - if (n == 1 && v[APPLE_CLPC_DCS] == -1) - v[APPLE_CLPC_SOC] = -1; - else if (n != 2) - return -EINVAL; - - for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) - if (v[rail] < -1 || v[rail] > (int)clpc->hw->max_state[rail]) - return -EINVAL; - for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) - WRITE_ONCE(clpc->force[rail], v[rail]); - - dev_info(clpc->dev, "diagnostic override: dcs %d soc %d\n", - v[APPLE_CLPC_DCS], v[APPLE_CLPC_SOC]); - return len; -} - -static const struct file_operations apple_clpc_force_fops = { - .owner = THIS_MODULE, - .open = simple_open, - .write = apple_clpc_force_write, -}; - static int apple_clpc_gpu_input(struct apple_clpc *clpc) { struct device_node *ps, *pmgr; @@ -504,7 +456,6 @@ static int apple_clpc_probe(struct platform_device *pdev) if (clpc->hw->rail[rail] >= apple_pmc_rail_count(clpc->pmc)) return -EINVAL; clpc->mask |= APPLE_PMC_RAIL_MASK(clpc->hw->rail[rail]); - clpc->force[rail] = -1; } ret = apple_clpc_parse_map(clpc, dcs_map, APPLE_CLPC_DCS) ?: @@ -555,8 +506,6 @@ static int apple_clpc_probe(struct platform_device *pdev) clpc->debugfs = debugfs_create_dir(dev_name(dev), NULL); debugfs_create_file("status", 0400, clpc->debugfs, clpc, &apple_clpc_status_fops); - debugfs_create_file("force", 0200, clpc->debugfs, clpc, - &apple_clpc_force_fops); ret = devm_add_action_or_reset(dev, apple_clpc_stop, clpc); if (ret) From 794e7843d80eff5278085387c3831910a796c4fb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:22:11 +1100 Subject: [PATCH 1362/2680] soc: apple: clpc: make the demand policy per-SoC data, drop the GPU input The sampling period, busy threshold, hold time and the frequency-to-state tables are module parameters, along with an optional GPU floor. They were knobs for choosing the policy; the values the driver ships with are the measured result, and nothing sets the parameters. Module parameters that change memory-controller policy also become ABI once the driver is upstream. Turn the timing values into constants and move the state tables into the per-SoC data, next to the rails they apply to. The map validation against each rail's highest state goes with the parameters, since the tables can no longer be wrong at runtime. The GPU input only acts when gpu_dcs or gpu_soc is set, and both default to 0, so it has never contributed to the floor: the GPU raises the DCS through the PMP's GFX report instead. Remove it, and with it the read of the PMGR GPU power state through the PMGR syscon. The device tree property it read goes in the following patches. No functional change with the default parameters. (cherry picked from commit 9dfd51203aa80ce8867d98ba03c79fbb283039c2) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/clpc.c | 170 +++++++-------------------------------- 1 file changed, 29 insertions(+), 141 deletions(-) diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index 911163f13b0c0..76bd9d3815acb 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -5,34 +5,28 @@ * On macOS the CLPC raises the floor of the memory controller (DCS) and the * SoC fabric while the CPU clusters or the GPU are busy, through its own * voter agent in the PMC. This driver keeps that agent's floor in step with - * Linux CPU load and, optionally, GPU activity: + * Linux CPU load: * * - every sampling period, each cpufreq policy reports how busy its * busiest CPU was and how fast the policy runs relative to its maximum; * a busy policy requests a DCS and fabric state from a table indexed by * that frequency quartile; - * - while the GPU's PMGR power state reports the GPU as actually powered, - * a fixed GPU floor is requested; * - each rail keeps its last request for a hold time after demand stops, * then drops back to 0, so idle releases the floor. * - * The native mapping from CPU and GPU demand to DCS and fabric states has - * not been decoded. The default tables are measured policy, not a copy of - * macOS, and can be changed with module parameters. + * The native mapping from CPU demand to DCS and fabric states is not known. + * The tables below are measured policy, not a copy of macOS. * * Copyright The Asahi Linux Contributors */ -#include #include #include #include #include -#include #include #include #include -#include #include #include #include @@ -40,52 +34,28 @@ #define APPLE_CLPC_MAX_CLUSTERS 8 #define APPLE_CLPC_LEVELS 4 -/* PMGR power state: ACTUAL is bits [7:4]; 0xf means powered on. */ -#define APPLE_PMGR_PS_ACTUAL GENMASK(7, 4) -#define APPLE_PMGR_PS_ACTIVE 0xf - enum apple_clpc_rail { APPLE_CLPC_DCS, APPLE_CLPC_SOC, APPLE_CLPC_RAILS, }; -static unsigned int sample_ms = 4; -module_param(sample_ms, uint, 0444); -MODULE_PARM_DESC(sample_ms, "Sampling period in ms"); - -static unsigned int busy_pct = 50; -module_param(busy_pct, uint, 0444); -MODULE_PARM_DESC(busy_pct, "Busy threshold of a CPU cluster, percent of the sampling period"); - -static unsigned int hold_ms = 32; -module_param(hold_ms, uint, 0444); -MODULE_PARM_DESC(hold_ms, "Time a rail keeps its last request after demand stops, ms"); - -static char *dcs_map = "1,2,3,4"; -module_param(dcs_map, charp, 0444); -MODULE_PARM_DESC(dcs_map, "DCS state for a busy cluster below 25, 50, 75 and 100 percent of its maximum frequency"); - -static char *soc_map = "0,0,0,0"; -module_param(soc_map, charp, 0444); -MODULE_PARM_DESC(soc_map, "Fabric state for a busy cluster below 25, 50, 75 and 100 percent of its maximum frequency"); - -static unsigned int gpu_dcs; -module_param(gpu_dcs, uint, 0444); -MODULE_PARM_DESC(gpu_dcs, "DCS state while the GPU is powered (0: none)"); - -static unsigned int gpu_soc; -module_param(gpu_soc, uint, 0444); -MODULE_PARM_DESC(gpu_soc, "Fabric state while the GPU is powered (0: none)"); +/* Sampling period */ +#define APPLE_CLPC_SAMPLE_MS 4 +/* Busy threshold of a CPU cluster, percent of the sampling period */ +#define APPLE_CLPC_BUSY_PCT 50 +/* Time a rail keeps its last request after demand stops */ +#define APPLE_CLPC_HOLD_MS 32 /** * struct apple_clpc_hw - per-SoC CLPC parameters * @rail: PMC rail of the DCS and of the fabric - * @max_state: highest state of the DCS and of the fabric + * @map: state of each rail for a busy cluster below 25, 50, 75 and 100 + * percent of its maximum frequency */ struct apple_clpc_hw { unsigned int rail[APPLE_CLPC_RAILS]; - unsigned int max_state[APPLE_CLPC_RAILS]; + u8 map[APPLE_CLPC_RAILS][APPLE_CLPC_LEVELS]; }; struct apple_clpc_cluster { @@ -106,8 +76,6 @@ struct apple_clpc { struct device *pmc; const struct apple_clpc_hw *hw; unsigned int agent; - struct regmap *gpu_regmap; - u32 gpu_offset; /* Samples while a vote is held; idle_work defers to busy CPUs. */ struct delayed_work work; struct delayed_work idle_work; @@ -118,34 +86,14 @@ struct apple_clpc { unsigned long next_scan; u64 *idle; u64 *wall; - u8 map[APPLE_CLPC_RAILS][APPLE_CLPC_LEVELS]; - unsigned int gpu_state[APPLE_CLPC_RAILS]; struct apple_clpc_vote vote[APPLE_CLPC_RAILS]; u32 mask; u32 saved_floor; - bool gpu_on; unsigned long samples; unsigned long update_errors; struct dentry *debugfs; }; -static int apple_clpc_parse_map(struct apple_clpc *clpc, const char *s, - enum apple_clpc_rail rail) -{ - unsigned int v[APPLE_CLPC_LEVELS]; - unsigned int i; - - if (sscanf(s, "%u,%u,%u,%u", &v[0], &v[1], &v[2], &v[3]) != 4) - return -EINVAL; - for (i = 0; i < APPLE_CLPC_LEVELS; i++) { - if (v[i] > clpc->hw->max_state[rail]) - return -EINVAL; - clpc->map[rail][i] = v[i]; - } - - return 0; -} - /* cpufreq policies can appear after probe; pick up new ones as they do. */ static void apple_clpc_find_clusters(struct apple_clpc *clpc) { @@ -220,21 +168,8 @@ static void apple_clpc_sample_cluster(struct apple_clpc *clpc, q = min_t(unsigned int, cl->level / (100 / APPLE_CLPC_LEVELS), APPLE_CLPC_LEVELS - 1); for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) - cl->request[rail] = busy >= busy_pct ? clpc->map[rail][q] : 0; -} - -static bool apple_clpc_gpu_powered(struct apple_clpc *clpc) -{ - u32 ps; - - /* - * Hardware auto-gating powers the GPU down while the requested state - * stays on, so only the actual state says whether the GPU runs. - */ - if (regmap_read(clpc->gpu_regmap, clpc->gpu_offset, &ps)) - return false; - - return FIELD_GET(APPLE_PMGR_PS_ACTUAL, ps) == APPLE_PMGR_PS_ACTIVE; + cl->request[rail] = busy >= APPLE_CLPC_BUSY_PCT ? + clpc->hw->map[rail][q] : 0; } static unsigned int apple_clpc_vote(struct apple_clpc_vote *vote, @@ -244,7 +179,7 @@ static unsigned int apple_clpc_vote(struct apple_clpc_vote *vote, vote->last_demand = now; vote->state = demand; } else if (time_after_eq(now, vote->last_demand + - msecs_to_jiffies(hold_ms))) { + msecs_to_jiffies(APPLE_CLPC_HOLD_MS))) { vote->state = 0; } @@ -275,7 +210,7 @@ static void apple_clpc_queue(struct apple_clpc *clpc, bool idle) */ queue_delayed_work(system_freezable_power_efficient_wq, idle ? &clpc->idle_work : &clpc->work, - msecs_to_jiffies(sample_ms)); + msecs_to_jiffies(APPLE_CLPC_SAMPLE_MS)); } static void apple_clpc_sample(struct apple_clpc *clpc) @@ -300,16 +235,9 @@ static void apple_clpc_sample(struct apple_clpc *clpc) demand[rail] = max(demand[rail], cl->request[rail]); } - clpc->gpu_on = clpc->gpu_regmap && - (clpc->gpu_state[APPLE_CLPC_DCS] || - clpc->gpu_state[APPLE_CLPC_SOC]) && - apple_clpc_gpu_powered(clpc); - for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) { - if (clpc->gpu_on) - demand[rail] = max(demand[rail], clpc->gpu_state[rail]); + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) state[rail] = apple_clpc_vote(&clpc->vote[rail], demand[rail], now); - } if (apple_pmc_floor_update(clpc->pmc, clpc->agent, clpc->mask, apple_clpc_floor(clpc, state))) @@ -339,18 +267,13 @@ static int apple_clpc_status_show(struct seq_file *s, void *unused) apple_pmc_floor_read(clpc->pmc, clpc->agent), clpc->saved_floor); for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) - seq_printf(s, "%s: rail %u state %u enabled %u map %u,%u,%u,%u gpu %u\n", + seq_printf(s, "%s: rail %u state %u enabled %u map %u,%u,%u,%u\n", rail == APPLE_CLPC_DCS ? "dcs" : "soc", clpc->hw->rail[rail], clpc->vote[rail].state, apple_pmc_voter_enabled(clpc->pmc, clpc->agent, clpc->hw->rail[rail]), - clpc->map[rail][0], - clpc->map[rail][1], clpc->map[rail][2], - clpc->map[rail][3], clpc->gpu_state[rail]); - seq_printf(s, "gpu: %s\n", !clpc->gpu_regmap ? "no input" : - !clpc->gpu_state[APPLE_CLPC_DCS] && - !clpc->gpu_state[APPLE_CLPC_SOC] ? "no floor" : - clpc->gpu_on ? "powered" : "off"); + clpc->hw->map[rail][0], clpc->hw->map[rail][1], + clpc->hw->map[rail][2], clpc->hw->map[rail][3]); seq_printf(s, "samples: %lu update errors: %lu\n", clpc->samples, clpc->update_errors); for (i = 0; i < clpc->nr_clusters; i++) @@ -364,34 +287,6 @@ static int apple_clpc_status_show(struct seq_file *s, void *unused) } DEFINE_SHOW_ATTRIBUTE(apple_clpc_status); -static int apple_clpc_gpu_input(struct apple_clpc *clpc) -{ - struct device_node *ps, *pmgr; - int ret; - - ps = of_parse_phandle(clpc->dev->of_node, "apple,gpu-power-state", 0); - if (!ps) - return 0; - - ret = of_property_read_u32(ps, "reg", &clpc->gpu_offset); - pmgr = of_get_parent(ps); - of_node_put(ps); - if (ret) { - of_node_put(pmgr); - return ret; - } - - clpc->gpu_regmap = syscon_node_to_regmap(pmgr); - of_node_put(pmgr); - if (IS_ERR(clpc->gpu_regmap)) { - ret = PTR_ERR(clpc->gpu_regmap); - clpc->gpu_regmap = NULL; - return ret; - } - - return 0; -} - static void apple_clpc_free_clusters(struct apple_clpc *clpc) { unsigned int i; @@ -458,20 +353,6 @@ static int apple_clpc_probe(struct platform_device *pdev) clpc->mask |= APPLE_PMC_RAIL_MASK(clpc->hw->rail[rail]); } - ret = apple_clpc_parse_map(clpc, dcs_map, APPLE_CLPC_DCS) ?: - apple_clpc_parse_map(clpc, soc_map, APPLE_CLPC_SOC); - if (ret) - return dev_err_probe(dev, ret, "invalid state map\n"); - if (gpu_dcs > clpc->hw->max_state[APPLE_CLPC_DCS] || - gpu_soc > clpc->hw->max_state[APPLE_CLPC_SOC]) - return dev_err_probe(dev, -EINVAL, "invalid GPU state\n"); - clpc->gpu_state[APPLE_CLPC_DCS] = gpu_dcs; - clpc->gpu_state[APPLE_CLPC_SOC] = gpu_soc; - - ret = apple_clpc_gpu_input(clpc); - if (ret) - return dev_err_probe(dev, ret, "invalid GPU power state\n"); - clpc->idle = devm_kcalloc(dev, nr_cpu_ids, sizeof(*clpc->idle), GFP_KERNEL); clpc->wall = devm_kcalloc(dev, nr_cpu_ids, sizeof(*clpc->wall), @@ -542,10 +423,17 @@ static int apple_clpc_resume(struct device *dev) static DEFINE_SIMPLE_DEV_PM_OPS(apple_clpc_pm_ops, apple_clpc_suspend, apple_clpc_resume); -/* T8140: DCS states 0-4 on rail 0, fabric states 0-3 on rail 1. */ +/* + * T8140: DCS states 0-4 on rail 0, fabric states 0-3 on rail 1. A busy + * cluster raises the DCS one state per frequency quartile and leaves the + * fabric alone. + */ static const struct apple_clpc_hw apple_clpc_hw_t8140 = { .rail = { [APPLE_CLPC_DCS] = 0, [APPLE_CLPC_SOC] = 1 }, - .max_state = { [APPLE_CLPC_DCS] = 4, [APPLE_CLPC_SOC] = 3 }, + .map = { + [APPLE_CLPC_DCS] = { 1, 2, 3, 4 }, + [APPLE_CLPC_SOC] = { 0, 0, 0, 0 }, + }, }; static const struct of_device_id apple_clpc_of_match[] = { From be81515af4c972db9dbd06267192d8addc67188c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:11:48 +1100 Subject: [PATCH 1363/2680] pmdomain: apple: pmgr-pwrstate: log always-on domains left off at info level The boot loader is not required to leave an always-on domain powered. Several T8140 domains are off when Linux starts: the GPU domain, the AOP audio fabric, an SMC I2C master and the NUB OCLA domain. The device tree marks them always-on for good reasons (firmware dependencies, the Apple Device Tree's critical and on flags, and keeping the GPU powered across system sleep), and the driver already turns them on at probe. That is an expected state, not a problem, yet each such domain produces a warning on every boot. Log it at info level and say what the driver does about it. A domain that cannot be turned on is still reported as an error by apple_pmgr_ps_set(). (cherry picked from commit 10e1825791523747c5f0d4ac44570fe4418ff586) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmgr-pwrstate.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index 47f9593ae1221..053c087c37184 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -366,7 +366,13 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) } else if (of_property_read_bool(node, "apple,always-on")) { ps->genpd.flags |= GENPD_FLAG_ALWAYS_ON; if (!active) { - dev_warn(dev, "always-on domain %s is not on at boot\n", name); + /* + * The boot loader need not leave an always-on domain on, + * so this is not an error: apple_pmgr_ps_set() reports a + * domain that cannot be turned on. + */ + dev_info(dev, "always-on domain %s is off at boot, turning it on\n", + name); /* * Turn it on so pm_genpd_init does not fail. genpd never * powers an always-on domain, so keep requesting it on From d996a6f7b4f2824c01f2624c1016c107b80fe887 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 15:48:35 +1100 Subject: [PATCH 1364/2680] soc: apple: rtkit: Do not warn when buffers are retained on purpose apple_rtkit_free_retaining_buffers() prints a warning every time it is called. Its callers use it deliberately for firmware that is left running: the T8140 NVMe controller on every unbind after its firmware reached RUN, the DCP when it has pinned a live firmware session, and the MTP helper for a session adopted from the bootloader. None of these is a problem, yet each unbind left a warning in the log. A caller that gets here after a failed shutdown is better placed to report that failure, so log the retained buffer sizes at debug level and say in the kernel-doc that a running co-processor is an expected case. (cherry picked from commit 1ef9c2113ecdf2851908e4bd9793876d92152d27) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit.c | 15 ++++++++------- include/linux/soc/apple/rtkit.h | 4 ++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 10f1bd0b39c50..f394ea8402ee8 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -1233,14 +1233,15 @@ void apple_rtkit_free_retaining_buffers(struct apple_rtkit *rtk) apple_rtkit_stop_rx(rtk); /* - * A failed shutdown provides no guarantee that firmware stopped DMA. - * Retain shared buffers, including custom allocation contexts, rather - * than returning their storage to an allocator while it may be live. + * The firmware may still be using the shared buffers, so keep them, + * including custom allocation contexts, rather than returning their + * storage to an allocator. Callers use this on purpose for firmware + * that cannot be stopped, and report a failed shutdown themselves. */ - dev_warn(rtk->dev, - "RTKit: retaining the shared buffers of an unstopped co-processor (ioreport %zu, crashlog %zu, oslog %zu, syslog %zu bytes)\n", - rtk->ioreport_buffer.size, rtk->crashlog_buffer.size, - rtk->oslog_buffer.size, rtk->syslog_buffer.size); + dev_dbg(rtk->dev, + "RTKit: retaining the shared buffers of a running co-processor (ioreport %zu, crashlog %zu, oslog %zu, syslog %zu bytes)\n", + rtk->ioreport_buffer.size, rtk->crashlog_buffer.size, + rtk->oslog_buffer.size, rtk->syslog_buffer.size); kfree(rtk->syslog_msg_buffer); apple_rtkit_release_rx(rtk); kfree(rtk); diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 00b9a21e936d7..6f7d59a267877 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -136,8 +136,8 @@ struct apple_rtkit *apple_rtkit_init_adopted(struct device *dev, void *cookie, void apple_rtkit_free(struct apple_rtkit *rtk); /* - * Detach a co-processor that could not be quiesced without releasing its - * shared buffers. + * Detach a co-processor that is still running, or that could not be + * quiesced, without releasing its shared buffers. * * Stops mailbox callbacks and drains the private RTKit workqueue before * freeing the transport state. Firmware-visible mappings and their allocation From 27de362d83d8b4b2d4f8484b30dfb7c5a3768e88 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:17:39 +1100 Subject: [PATCH 1365/2680] spi: apple-qspi: Do not print a banner on probe The controller version, clock rate and FIFO depths are fixed for a given SoC and are useful only when debugging the driver. Print them at debug level. (cherry picked from commit 152cc6b4d737c3523b56d1c4b0b4cd3928e514b5) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/spi-apple-qspi.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c index 57a572c1e779d..e0d8653e51984 100644 --- a/drivers/spi/spi-apple-qspi.c +++ b/drivers/spi/spi-apple-qspi.c @@ -587,9 +587,9 @@ static int apple_qspi_probe(struct platform_device *pdev) "failed to register SPI controller\n"); version = apple_qspi_p_read(qspi, APPLE_QSPI_P_VERSION); - dev_info(dev, "QSPIMC version %#x, %lu Hz clock, txq=%u rxq=%u\n", - version, clk_get_rate(qspi->clk), qspi->tx_depth, - qspi->rx_depth); + dev_dbg(dev, "QSPIMC version %#x, %lu Hz clock, txq=%u rxq=%u\n", + version, clk_get_rate(qspi->clk), qspi->tx_depth, + qspi->rx_depth); return 0; } From 503a189c428ce6afc38dca42e79678ecd061aec2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:21:30 +1100 Subject: [PATCH 1366/2680] pmdomain: apple: pmc: Test partial enables and probe unwind A failed power_on() must restore both the hardware enables and the software shadow, including the rail that rejected the update. Extend the PMC tests to reject a write on an intermediate rail after the first rail has accepted it, and call the genpd power-on callback directly. Also exercise the probe release action with a nonzero inherited AP DCS vote while a limit is active. It must restore the vote and saved enables without relying on a later power-off callback. Use the existing enable-read helper for readback and permit a KUnit static stub only when the driver tests are enabled. This leaves the normal register accesses unchanged and allows a rejected write to be modeled without physical PMC hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmc-test.c | 73 +++++++++++++++++++++++++++++++ drivers/pmdomain/apple/pmc.c | 11 ++++- 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/drivers/pmdomain/apple/pmc-test.c b/drivers/pmdomain/apple/pmc-test.c index 3f0e277b21255..46b1bdabe022a 100644 --- a/drivers/pmdomain/apple/pmc-test.c +++ b/drivers/pmdomain/apple/pmc-test.c @@ -6,6 +6,7 @@ * Included by pmc.c. */ +#include #include #define APPLE_PMC_TEST_RAILS_SIZE 0x4000 @@ -17,6 +18,8 @@ static struct apple_pmc *apple_pmc_test_alloc(struct kunit *test) pmc = kunit_kzalloc(test, sizeof(*pmc), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, pmc); + pmc->dev = kunit_device_register(test, "apple-pmc-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pmc->dev); pmc->hw = hw; raw_spin_lock_init(&pmc->lock); pmc->voters = (void __iomem *)kunit_kzalloc(test, @@ -81,7 +84,77 @@ static void apple_pmc_test_dcs_limit_restores_ap_vote(struct kunit *test) KUNIT_EXPECT_EQ(test, apple_pmc_ap_dcs_read(pmc), 2); } +struct apple_pmc_test_reject { + unsigned int rail; + u32 enable; + bool rejected; +}; + +static u32 apple_pmc_test_reject_read(struct apple_pmc *pmc, unsigned int rail) +{ + struct kunit *test = kunit_get_current_test(); + struct apple_pmc_test_reject *reject = test->priv; + void __iomem *reg = pmc->rails + pmc->hw->rail_enable + 4 * rail; + u32 val = readl(reg); + + if (rail == reject->rail && !reject->rejected && val != reject->enable) { + reject->rejected = true; + writel(reject->enable, reg); + return reject->enable; + } + return val; +} + +/* Genpd leaves a domain off after power_on() fails on an intermediate rail. */ +static void apple_pmc_test_partial_enable(struct kunit *test) +{ + struct apple_pmc *pmc = apple_pmc_test_alloc(test); + struct apple_pmc_test_reject reject = { .rail = 1, .enable = BIT(0) }; + struct apple_pmc_domain domain = { .pmc = pmc, .agent = 1 }; + unsigned int rail; + + for (rail = 0; rail < pmc->hw->rails; rail++) { + pmc->enable[rail] = BIT(0); + writel(BIT(0), pmc->rails + pmc->hw->rail_enable + 4 * rail); + } + test->priv = &reject; + kunit_activate_static_stub(test, apple_pmc_enable_read, apple_pmc_test_reject_read); + KUNIT_EXPECT_EQ(test, apple_pmc_domain_power_on(&domain.genpd), -EIO); + KUNIT_EXPECT_TRUE(test, reject.rejected); + for (rail = 0; rail < pmc->hw->rails; rail++) { + KUNIT_EXPECT_EQ(test, pmc->enable[rail], BIT(0)); + KUNIT_EXPECT_EQ(test, apple_pmc_enable_read(pmc, rail), BIT(0)); + } + kunit_deactivate_static_stub(test, apple_pmc_enable_read); + test->priv = NULL; +} + +/* Probe unwind must also restore a nonzero inherited AP DCS vote. */ +static void apple_pmc_test_release_restores_ap_vote(struct kunit *test) +{ + struct apple_pmc *pmc = apple_pmc_test_alloc(test); + const struct apple_pmc_hw *hw = pmc->hw; + unsigned int rail; + + INIT_LIST_HEAD(&pmc->domains); + INIT_DELAYED_WORK(&pmc->dcs_work, apple_pmc_dcs_work); + apple_pmc_test_set_dcs_vote(pmc, hw->ap_agent, 2); + pmc->ap_dcs_vote = apple_pmc_ap_dcs_read(pmc); + apple_pmc_test_set_dcs_vote(pmc, 0, 4); + pmc->enable[hw->dcs_rail] = BIT(0) | BIT(hw->ap_agent); + pmc->dcs_state = 3; + apple_pmc_dcs_vote(pmc); + KUNIT_EXPECT_EQ(test, apple_pmc_ap_dcs_read(pmc), 1); + apple_pmc_release(pmc); + KUNIT_EXPECT_EQ(test, pmc->dcs_state, 0UL); + KUNIT_EXPECT_EQ(test, apple_pmc_ap_dcs_read(pmc), 2); + for (rail = 0; rail < hw->rails; rail++) + KUNIT_EXPECT_EQ(test, apple_pmc_enable_read(pmc, rail), pmc->saved_enable[rail]); +} + static struct kunit_case apple_pmc_test_cases[] = { + KUNIT_CASE(apple_pmc_test_partial_enable), + KUNIT_CASE(apple_pmc_test_release_restores_ap_vote), KUNIT_CASE(apple_pmc_test_agent_enable), KUNIT_CASE(apple_pmc_test_dcs_limit_restores_ap_vote), {} diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index bce3cd7619b77..057f41e5f8959 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -40,6 +40,10 @@ #include #include +#if IS_ENABLED(CONFIG_APPLE_PMC_KUNIT_TEST) +#include +#endif + #define APPLE_PMC_MAX_RAILS 8 #define APPLE_PMC_MAX_AGENTS 32 #define APPLE_PMC_MAX_STATES 16 @@ -115,6 +119,9 @@ static struct dentry *apple_pmc_debugfs_root; static u32 apple_pmc_enable_read(struct apple_pmc *pmc, unsigned int rail) { +#if IS_ENABLED(CONFIG_APPLE_PMC_KUNIT_TEST) + KUNIT_STATIC_STUB_REDIRECT(apple_pmc_enable_read, pmc, rail); +#endif return readl(pmc->rails + pmc->hw->rail_enable + 4 * rail); } @@ -125,7 +132,7 @@ static u32 apple_pmc_enable_read(struct apple_pmc *pmc, unsigned int rail) static int apple_pmc_enable_write(struct apple_pmc *pmc, unsigned int rail) { void __iomem *reg = pmc->rails + pmc->hw->rail_enable + 4 * rail; - u32 old = readl(reg); + u32 old = apple_pmc_enable_read(pmc, rail); u32 val = pmc->enable[rail]; if (pmc->dcs_state && rail == pmc->hw->dcs_rail) @@ -134,7 +141,7 @@ static int apple_pmc_enable_write(struct apple_pmc *pmc, unsigned int rail) return 0; writel(val, reg); - if (readl(reg) != val) { + if (apple_pmc_enable_read(pmc, rail) != val) { dev_err_ratelimited(pmc->dev, "rail %u enable %#x -> %#x did not stick\n", rail, old, val); From 81f45e55a62bfc2a686734179d5cda8956c7d8d5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:21:31 +1100 Subject: [PATCH 1367/2680] soc: apple: mcc: Test partial allocation enable and cache handoff An allocation-enable failure can happen after an earlier plane started allocating. Verify that both the successful planes and the failing plane are disabled and flushed before returning the error, even though probe has not marked the controller enabled yet. Add KUnit coverage of an intermediate-plane rejection, normal shutdown, and a cache that is not ready. A static stub for register writes models rejected enables and completed flushes; normal builds retain the same MMIO operations without the test hook. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 10 +++ drivers/soc/apple/mcc-test.c | 118 +++++++++++++++++++++++++++++++++++ drivers/soc/apple/mcc.c | 23 ++++++- 3 files changed, 148 insertions(+), 3 deletions(-) create mode 100644 drivers/soc/apple/mcc-test.c diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 62f2f33e6b56c..2640e418be7c6 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -37,6 +37,16 @@ config APPLE_MCC Say 'y' or 'm' here if you have an Apple T8140 SoC. +config APPLE_MCC_KUNIT_TEST + bool "KUnit tests for Apple MCC allocation and unwind" if !KUNIT_ALL_TESTS + depends on APPLE_MCC=y && KUNIT=y + default KUNIT_ALL_TESTS + help + Test allocation failure after a plane has been enabled, and cache + handoff on shutdown, using simulated registers. Verify that failure + also cleans up the plane whose state is uncertain. These tests do not + access physical hardware and are only useful for kernel developers. + config APPLE_PMGR_MISC bool "Apple SoC PMGR miscellaneous support" depends on PM diff --git a/drivers/soc/apple/mcc-test.c b/drivers/soc/apple/mcc-test.c new file mode 100644 index 0000000000000..46879e534a1b6 --- /dev/null +++ b/drivers/soc/apple/mcc-test.c @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Included by mcc.c to exercise allocation failure and cache handoff. */ +#include +#include +#include + +struct apple_mcc_test { + struct apple_mcc mcc; + unsigned int reject_plane; + unsigned long attempted; + unsigned long disabled; + unsigned long flushed; +}; + +static void apple_mcc_test_write(struct apple_mcc *mcc, unsigned int plane, + u32 val, u32 offset) +{ + struct kunit *test = kunit_get_current_test(); + struct apple_mcc_test *ctx = test->priv; + + if (offset == MCC_ALLOC_DISABLE) { + if (val) { + ctx->disabled |= BIT(plane); + } else { + ctx->attempted |= BIT(plane); + if (plane == ctx->reject_plane) + return; + } + } + if (offset == MCC_CACHE_FLUSH) { + ctx->flushed |= BIT(plane); + /* Model a cache whose flush completes immediately. */ + val = 0; + } + writel(val, mcc->planes[plane] + offset); +} + +static struct apple_mcc_test *apple_mcc_test_alloc(struct kunit *test) +{ + struct apple_mcc_test *ctx; + struct apple_mcc *mcc; + unsigned int i; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ctx); + mcc = &ctx->mcc; + mcc->dev = kunit_device_register(test, "apple-mcc-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, mcc->dev); + mcc->nr_planes = 3; + ctx->reject_plane = MCC_MAX_PLANES; + for (i = 0; i < mcc->nr_planes; i++) { + mcc->planes[i] = (__force void __iomem *)kunit_kzalloc(test, SZ_16K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, mcc->planes[i]); + writel(1, mcc->planes[i] + MCC_ALLOC_DISABLE); + writel(1, mcc->planes[i] + MCC_CACHE_WAYS); + writel(FIELD_PREP(MCC_CACHE_WAYS_POWERED, 1), mcc->planes[i] + MCC_CACHE_POWER); + } + test->priv = ctx; + kunit_activate_static_stub(test, apple_mcc_writel, apple_mcc_test_write); + return ctx; +} + +static void apple_mcc_test_partial_enable(struct kunit *test) +{ + struct apple_mcc_test *ctx = apple_mcc_test_alloc(test); + struct apple_mcc *mcc = &ctx->mcc; + unsigned int i; + + ctx->reject_plane = 1; + KUNIT_EXPECT_EQ(test, apple_mcc_enable(mcc), -EIO); + KUNIT_EXPECT_EQ(test, ctx->attempted, BIT(0) | BIT(1)); + /* The failed plane's state is uncertain, so include it in cleanup. */ + KUNIT_EXPECT_EQ(test, ctx->disabled, BIT(0) | BIT(1)); + KUNIT_EXPECT_EQ(test, ctx->flushed, BIT(0) | BIT(1)); + KUNIT_EXPECT_FALSE(test, mcc->enabled); + for (i = 0; i < mcc->nr_planes; i++) + KUNIT_EXPECT_EQ(test, readl(mcc->planes[i] + MCC_ALLOC_DISABLE), 1U); +} + +static void apple_mcc_test_quiesce(struct kunit *test) +{ + struct apple_mcc_test *ctx = apple_mcc_test_alloc(test); + struct apple_mcc *mcc = &ctx->mcc; + + KUNIT_ASSERT_EQ(test, apple_mcc_enable(mcc), 0); + mcc->enabled = true; + apple_mcc_quiesce(mcc); + KUNIT_EXPECT_EQ(test, ctx->attempted, 7UL); + KUNIT_EXPECT_EQ(test, ctx->disabled, 7UL); + KUNIT_EXPECT_EQ(test, ctx->flushed, 7UL); + KUNIT_EXPECT_FALSE(test, mcc->enabled); +} + +static void apple_mcc_test_not_ready(struct kunit *test) +{ + struct apple_mcc_test *ctx = apple_mcc_test_alloc(test); + struct apple_mcc *mcc = &ctx->mcc; + + writel(0, mcc->planes[1] + MCC_CACHE_WAYS); + KUNIT_EXPECT_EQ(test, apple_mcc_enable(mcc), -EBUSY); + KUNIT_EXPECT_EQ(test, ctx->attempted, 0UL); + KUNIT_EXPECT_EQ(test, ctx->disabled, 0UL); + KUNIT_EXPECT_EQ(test, ctx->flushed, 0UL); +} + +static struct kunit_case apple_mcc_test_cases[] = { + KUNIT_CASE(apple_mcc_test_partial_enable), + KUNIT_CASE(apple_mcc_test_quiesce), + KUNIT_CASE(apple_mcc_test_not_ready), + {} +}; + +static struct kunit_suite apple_mcc_test_suite = { + .name = "apple-mcc", + .test_cases = apple_mcc_test_cases, +}; + +kunit_test_suite(apple_mcc_test_suite); diff --git a/drivers/soc/apple/mcc.c b/drivers/soc/apple/mcc.c index 7b8c2e2220c8b..1bc23d1da22d0 100644 --- a/drivers/soc/apple/mcc.c +++ b/drivers/soc/apple/mcc.c @@ -18,6 +18,10 @@ #include #include +#if IS_ENABLED(CONFIG_APPLE_MCC_KUNIT_TEST) +#include +#endif + #define MCC_ALLOC_DISABLE 0x82c #define MCC_CACHE_WAYS 0x2800 #define MCC_CACHE_WAYS_ENABLED GENMASK(4, 0) @@ -36,6 +40,15 @@ struct apple_mcc { bool enabled; }; +static void apple_mcc_writel(struct apple_mcc *mcc, unsigned int plane, + u32 value, u32 offset) +{ +#if IS_ENABLED(CONFIG_APPLE_MCC_KUNIT_TEST) + KUNIT_STATIC_STUB_REDIRECT(apple_mcc_writel, mcc, plane, value, offset); +#endif + writel(value, mcc->planes[plane] + offset); +} + static int apple_mcc_wait_boot_dma(struct device *dev) { struct of_phandle_iterator it; @@ -79,8 +92,8 @@ static void apple_mcc_disable_planes(struct apple_mcc *mcc, unsigned int count) u32 val; for (i = 0; i < count; i++) { - writel(1, mcc->planes[i] + MCC_ALLOC_DISABLE); - writel(1, mcc->planes[i] + MCC_CACHE_FLUSH); + apple_mcc_writel(mcc, i, 1, MCC_ALLOC_DISABLE); + apple_mcc_writel(mcc, i, 1, MCC_CACHE_FLUSH); } for (i = 0; i < count; i++) @@ -99,7 +112,7 @@ static int apple_mcc_enable(struct apple_mcc *mcc) return -EBUSY; for (i = 0; i < mcc->nr_planes; i++) { - writel(0, mcc->planes[i] + MCC_ALLOC_DISABLE); + apple_mcc_writel(mcc, i, 0, MCC_ALLOC_DISABLE); if (readl(mcc->planes[i] + MCC_ALLOC_DISABLE)) { dev_err(mcc->dev, "plane %u did not enable allocation\n", i); /* @@ -197,3 +210,7 @@ module_platform_driver(apple_mcc_driver); MODULE_AUTHOR("Ryan Murray "); MODULE_DESCRIPTION("Apple T8140 system level cache allocation"); MODULE_LICENSE("Dual MIT/GPL"); + +#if IS_ENABLED(CONFIG_APPLE_MCC_KUNIT_TEST) +#include "mcc-test.c" +#endif From 5fcda733a8c1859ff8cfd450c04653d54f177bf9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:42:23 +1100 Subject: [PATCH 1368/2680] drm/apple: dcp: Initialise work items at probe The vblank, swap watchdog and backlight work items are initialised in dcp_comp_bind(), but system sleep and shutdown cancel them on any probed DCP: dcp_platform_suspend() cancels swap_watchdog_wq unconditionally, and dcp_platform_shutdown() cancels bl_register_wq and bl_update_wq for the internal H17P DCP. A DCP whose DRM component never bound, or whose bind failed before reaching those INIT_WORK() calls (for instance on -EPROBE_DEFER from the PIODMA domain), therefore hands an all-zero work item to cancel_work_sync(). __flush_work() then warns about the missing function on every suspend or reboot. Initialise the work items and the backlight registration mutex once at probe, next to the other work items the platform device owns. Binding again no longer re-initialises them either, which was only harmless because unbind had cancelled them first. Fixes: be6e4f675c23 ("drm/apple: recover Type-C outputs whose swaps DCP drops") Fixes: 43039912593a ("drm: apple: retain adopted H17P DMA resources through reboot") (cherry picked from commit 40ee648c89f4b5ce016464842bdbe76c9f501c79) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 047c653433262..680e0e655e542 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -4096,9 +4096,6 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) of_node_put(panel_np); dcp->fixed_connector_type = DRM_MODE_CONNECTOR_eDP; dcp->connector_type = DRM_MODE_CONNECTOR_eDP; - INIT_WORK(&dcp->bl_register_wq, dcp_work_register_backlight); - mutex_init(&dcp->bl_register_mutex); - INIT_WORK(&dcp->bl_update_wq, dcp_work_update_backlight); } ret = dcp_create_piodma_iommu_dev(dcp); @@ -4143,9 +4140,6 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) // TDOD: mem_desc IDs start at 1, for simplicity just skip '0' entry set_bit(0, dcp->memdesc_map); - INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); - INIT_DELAYED_WORK(&dcp->swap_watchdog_wq, dcp_swap_watchdog); - if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { /* Pin before the first write or mailbox wake can expose DMA. */ @@ -4406,6 +4400,12 @@ static int dcp_platform_probe(struct platform_device *pdev) INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, dcp_placeholder_edid_work); INIT_DELAYED_WORK(&dcp->bl_fallback_wq, dcp_work_backlight_fallback); + /* System sleep and shutdown cancel these even if no bind succeeded. */ + INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); + INIT_DELAYED_WORK(&dcp->swap_watchdog_wq, dcp_swap_watchdog); + INIT_WORK(&dcp->bl_register_wq, dcp_work_register_backlight); + INIT_WORK(&dcp->bl_update_wq, dcp_work_update_backlight); + mutex_init(&dcp->bl_register_mutex); /* Balanced by enable at successful component bind. */ disable_delayed_work(&dcp->typec_reconnect_wq); disable_delayed_work(&dcp->placeholder_edid_wq); From 0fb99aafbf30127a9fc88d190f2587116cef3c58 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:42:25 +1100 Subject: [PATCH 1369/2680] drm/apple: dcp: Do not hold the mailbox of an unbound DCP for sleep dcp_hold_for_sleep() holds the RTKit mailbox of an internal H17P DCP whose panel uses soft DPMS. That state is decided from the connector type, which dcp_comp_bind() sets to eDP as soon as it finds the panel node, before RTKit is initialised. If the bind then fails, dcp->rtk is left NULL (or an error pointer when apple_rtkit_init() itself failed), the IOMFB queue was never stopped and so reads as idle, and the next system suspend calls apple_rtkit_hold_rx() on it and dereferences rtk->mbox. Skip the hold when there is no RTKit instance. There is nothing to hold then, and dcp_release_after_sleep() only releases a hold that was taken. Fixes: 1efb7996cdcf ("drm/apple: Hold internal DCP messages during system sleep") (cherry picked from commit 852496cbcd2f462142a0eb9f0793f6b376876481) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 680e0e655e542..48cb10816ef18 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -4654,7 +4654,9 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) */ static void dcp_hold_for_sleep(struct apple_dcp *dcp) { - if (!dcp_uses_soft_dpms(dcp) || dcp->fw_compat != DCP_FIRMWARE_H17P || + /* A failed component bind leaves no coprocessor session to hold. */ + if (IS_ERR_OR_NULL(dcp->rtk) || + !dcp_uses_soft_dpms(dcp) || dcp->fw_compat != DCP_FIRMWARE_H17P || dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G || READ_ONCE(dcp->crashed)) return; From 406bf2b3ef8d488a30decb16c06886285caf1542 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:42:27 +1100 Subject: [PATCH 1370/2680] drm/apple: dcp: Release the fixed display route with the device A DCP with a fixed HDMI/DP output selects its crossbar state and takes a Type-C mux reference in probe, but neither is ever released for the device: - mux_control_select() keeps the mux control's lock until mux_control_deselect(), and devm_mux_control_get() only drops the reference. dcp_register_typec_routes() runs after the selection, and its devm_phy_get() and devm_mux_control_get() can return -EPROBE_DEFER on a pipeline that also has Type-C routes (dcpext0 on the M2 Pro/Max MacBook Pro). The next probe attempt then blocks in mux_control_select() forever, as does a probe after unbinding the driver. - The Type-C mux reference is dropped in dcp_comp_unbind() instead, so it leaks when probe fails, and every later bind/unbind cycle of the DRM device drops it once more. Release both from a devres action registered right after the crossbar control is obtained. It runs after the Type-C routes are unregistered, which may have handed the pipeline back to its fixed output, so the selection is dropped in whatever state the routes left it. (cherry picked from commit 07f7d0da05db1a5e4955d25a868b90ba5cf6c7e6) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 48cb10816ef18..6b86a46dfe883 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3971,6 +3971,23 @@ static int dcp_connector_type_from_dt(struct device_node *np) return DRM_MODE_CONNECTOR_Unknown; } +/* + * The fixed route's crossbar selection and Type-C mux are taken at probe and + * must be released with the device: a crossbar left selected blocks the next + * probe's mux_control_select() for good. + */ +static void dcp_release_fixed_route(void *data) +{ + struct apple_dcp *dcp = data; + + if (dcp->fixed_route_selected) { + mux_control_deselect(dcp->xbar); + dcp->fixed_route_selected = false; + } + typec_mux_put(dcp->typec_mux); + dcp->typec_mux = NULL; +} + static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) { scoped_guard(mutex, &dcp->hpd_mutex) { @@ -4248,7 +4265,6 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) disable_irq(dcp->hdmi_hpd_irq); dcp_disable_typec_work(dcp, true); - typec_mux_put(dcp->typec_mux); if (dcp->avep) { av_service_disconnect(dcp); @@ -4510,6 +4526,9 @@ static int dcp_platform_probe(struct platform_device *pdev) dev_err(dev, "Failed to get dp-xbar: %ld\n", PTR_ERR(dcp->xbar)); return PTR_ERR(dcp->xbar); } + ret = devm_add_action_or_reset(dev, dcp_release_fixed_route, dcp); + if (ret) + return ret; ret = mux_control_select(dcp->xbar, mux_index); if (ret) dev_warn(dev, "mux_control_select failed: %d\n", ret); From d17fbbe15816bf1f4b06c802b76288ad283ede5a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:43:10 +1100 Subject: [PATCH 1371/2680] drm/apple: dcp: Tidy RTKit start-up logging Two lines are logged at info level on every boot of a machine with an H17P DCP, describing normal start-up rather than anything that needs attention: the retained DMA resources and the RTKit negotiation with a running coprocessor. Log them at debug level. The first was added by commit 97c4777bcb11 ("drm: apple: retain live H17P resources across component teardown") to say that rebinding is unsupported. That is still reported where it matters: probe and component bind refuse a retained session with an error saying a reboot is required. Nothing parses either line. dcp_start() also checks for a missing RTKit instance twice, the second time with an error message for a case the first check already returned on. Keep one check, covering the error pointer left by a failed apple_rtkit_init() as well; apple_drm_init_dcp() already reports the failed start. No functional change. (cherry picked from commit a3446fdfb3f4acb6222140387f4d40ec1d2a248e) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 6b86a46dfe883..55bd178496315 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -104,7 +104,7 @@ static int dcp_pin_live_session(struct apple_dcp *dcp) mutex_lock(&dcp_retained_sessions_lock); list_add_tail(&session->link, &dcp_retained_sessions); mutex_unlock(&dcp_retained_sessions_lock); - dev_info(dcp->dev, "live H17P DMA resources retained until reboot; rebind unsupported\n"); + dev_dbg(dcp->dev, "live H17P DMA resources retained until reboot\n"); return 0; } @@ -3112,23 +3112,12 @@ int dcp_start(struct platform_device *pdev) struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; - if (!dcp->rtk) + /* A failed component bind leaves no RTKit instance to start. */ + if (IS_ERR_OR_NULL(dcp->rtk)) return -ENODEV; init_completion(&dcp->start_done); - /* - * The coprocessor start can fail (or be recovered from) without tearing - * the DRM side down, so this can be reached with no RTKit instance at - * all. Starting an endpoint then dereferences NULL inside - * apple_rtkit_start_ep() and takes the machine with it. - */ - if (!dcp->rtk || IS_ERR(dcp->rtk)) { - dev_err(dcp->dev, - "cannot start endpoints: RTKit is not initialised\n"); - return -ENODEV; - } - /* Make the H17P allocator available before external service discovery. */ if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { ret = apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); @@ -4198,7 +4187,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) goto err_piodma; if (dcp->hw.adopt_live_session) - dev_info(dev, "negotiating RTKit with the running DCP\n"); + dev_dbg(dev, "negotiating RTKit with the running DCP\n"); /* Announce host readiness before awaiting external IOP readiness. */ if (dcp->hw.iomfb_method_profile == From 8ec09bf34abbc3d902a612cd8888314d66557fda Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:43:12 +1100 Subject: [PATCH 1372/2680] drm/apple: dptxep: Remove call counters and tidy tunnel comments The validate, connect, request and release display calls keep per-boot counters only to log them with the caller's return address at debug level. The trace events already record these calls, so drop the counters and their log lines. Several comments in the Thunderbolt DP tunnel paths retell how the code was ported and debugged rather than what it does. Reduce them to the facts a reader of this file needs. No functional change. (cherry picked from commit e2b7198e4255de468d30ba4e5f81db727fa1610b) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dptxep.c | 111 +++++++-------------------------- drivers/gpu/drm/apple/dptxep.h | 7 --- 2 files changed, 21 insertions(+), 97 deletions(-) diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index b9d4729407ed6..98066b69ced73 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -84,10 +84,9 @@ struct dptxport_apcall_set_tiled { }; /* - * Ported from aurora-silicon/linux#8: a Thunderbolt DP tunnel uses the same - * plain CORE|ATC|DIE|CONNECTED target as a direct alt-mode PHY. CORE is the - * DFP port (0 = dpphy, 1/2 = dpin0/dpin1, dcp->dptx_dfp_port), ATC is the - * route's own ATC index -- no separate DPIN field. + * A Thunderbolt DP tunnel uses the same CORE|ATC|DIE|CONNECTED target as a + * direct alt-mode PHY: CORE is the DFP port (0 = dpphy, 1/2 = dpin0/dpin1) + * and ATC the route's own ATC index. */ static u32 dptxport_remote_target(struct apple_dcp *dcp, u8 core, u8 atc, u8 die) @@ -107,29 +106,13 @@ int dptxport_validate_connection(struct apple_epic_service *service, u8 core, u32 target = dptxport_remote_target(service->ep->dcp, core, atc, die); /* - * attributes: role (0 = direct PHY, 1 = Thunderbolt/USB4 DP IN) | - * supportsHPD << 8. A real, hardware-validated reference - * implementation of the equivalent Thunderbolt DP tunnel mechanism - * on a different SoC (aurora-silicon/linux#8) adds this exact role - * bit for its tunnel routes; our own analog-DPIN path has never set - * it, always sending a plain direct-PHY attributes value even - * though this is a genuinely USB4-tunneled connection. DCP firmware - * appears to treat a failed AUX probe as fatal for role=0 but keeps - * training regardless for role=1 (a Thunderbolt tunnel's AUX - * proxying may not respond as fast as a direct PHY's): repeated - * test runs with substantially different Activate/crossbar ordering - * all reached the identical INACTIVE_SINK_DETECTED/DPRX-timeout - * outcome, which points at a signal we are not sending at all - * rather than a timing issue fixable by reordering. + * Attributes: role (0 = direct PHY, 1 = Thunderbolt/USB4 DP IN) | + * supports HPD << 8. Without the DP IN role DCP gives up on a tunnel + * whose AUX channel is slower to answer than a direct PHY's. */ u32 attrs = 0x100 | (dcp_is_usb4_output(service->ep->dcp) ? 1 : 0); trace_dptxport_validate_connection(dptx, core, atc, die); - dptx->validate_calls++; - dev_dbg(service->ep->dcp->dev, - "DPTX validate: call #%u this boot target=0x%x core=%u atc=%u die=%u attrs=0x%x caller=%pS\n", - dptx->validate_calls, target, core, atc, die, - attrs, __builtin_return_address(0)); cmd.target = cpu_to_le32(target); cmd.unk = cpu_to_le32(attrs); @@ -172,11 +155,6 @@ int dptxport_connect(struct apple_epic_service *service, u8 core, u8 atc, u32 target = dptxport_remote_target(service->ep->dcp, core, atc, die); trace_dptxport_connect(dptx, core, atc, die); - dptx->connect_calls++; - dev_dbg(service->ep->dcp->dev, - "DPTX connect: call #%u this boot target=0x%x unk=0x%x caller=%pS\n", - dptx->connect_calls, target, unk_field, - __builtin_return_address(0)); cmd.target = cpu_to_le32(target); cmd.unk = cpu_to_le32(unk_field); @@ -200,35 +178,13 @@ int dptxport_connect(struct apple_epic_service *service, u8 core, u8 atc, int dptxport_request_display(struct apple_epic_service *service) { - struct dptx_port *dptx = service->cookie; - int ret; - - dptx->request_calls++; - dev_dbg(service->ep->dcp->dev, - "DPTX request_display: call #%u this boot caller=%pS\n", - dptx->request_calls, __builtin_return_address(0)); - ret = afk_service_call(service, 0, 6, NULL, 0, 16, NULL, 0, 16); - dev_dbg(service->ep->dcp->dev, - "DPTX request_display: call #%u result=%d\n", - dptx->request_calls, ret); - return ret; + return afk_service_call(service, 0, 6, NULL, 0, 16, NULL, 0, 16); } int dptxport_release_display(struct apple_epic_service *service) { - struct dptx_port *dptx = service->cookie; - int ret; - dcp_180hz_invalidate(service->ep->dcp); - dptx->release_calls++; - dev_dbg(service->ep->dcp->dev, - "DPTX release_display: call #%u this boot caller=%pS\n", - dptx->release_calls, __builtin_return_address(0)); - ret = afk_service_call(service, 0, 7, NULL, 0, 16, NULL, 0, 16); - dev_dbg(service->ep->dcp->dev, - "DPTX release_display: call #%u result=%d\n", - dptx->release_calls, ret); - return ret; + return afk_service_call(service, 0, 7, NULL, 0, 16, NULL, 0, 16); } int dptxport_set_hpd_timeout(struct apple_epic_service *service, bool hpd, @@ -458,23 +414,8 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic dev_dbg(dcp->dev, "USB4/DPTX: SET_ACTIVE_LANE_COUNT %llu\n", lane_count); /* - * dcp_usb4_drm_allowed() (usb4_force_dptx) used to gate this - * for the old manual-training sysfs knob (module_param_cb - * usb4_dptx_train), removed wholesale in 0dc9f50 when - * apple_dcp_tb_dp_tunnel() replaced it with automatic tunnel - * detection -- but usb4_force_dptx itself, and this gate, - * were left behind with no way left to ever set it true. - * dcp_dptx_connect()'s single wait_for_completion_timeout() - * on linkcfg_completion is now shared by both the alt-mode - * and USB4-tunnel paths (also part of 0dc9f50's unification), - * so gating it here meant every tunneled connect timed out - * after a fully successful DPRX/AUX handshake and lane - * negotiation -- firmware does not send FORCE_HOTPLUG_DETECT - * (the only other completion site) for a tunnel. The - * reference (aurora-silicon/linux#8) completes - * linkcfg_completion here unconditionally; usb4_lane_completion - * has no consumer anywhere in this tree, so this drops both the - * dead gate and the dead completion and matches the reference. + * dcp_dptx_connect() waits for this on alt-mode and tunnel + * routes alike: DCP sends no FORCE_HOTPLUG_DETECT for a tunnel. */ complete(&dptx->linkcfg_completion); } @@ -514,11 +455,8 @@ static int dptxport_call_did_change_link_config(struct apple_epic_service *service) { /* - * Ported from aurora-silicon/linux#8: bringing the tunnel crossbar - * connection up (dcp_tunnel_crossbar_up()) now happens in - * dptxport_call(), after this succeeds, gated on - * dptx_tunnel && link_rate -- not inside this handler, and its - * result is never propagated back to DCP as a call failure. + * dptxport_call() brings a tunnel's crossbar connection up after + * this; its result is never reported to DCP as a call failure. */ /* assume the link config did change and wait a little bit */ @@ -580,11 +518,10 @@ static int dptxport_call_set_link_rate(struct apple_epic_service *service, if (phy_set_rate) { /* - * Ported from aurora-silicon/linux#8: a Thunderbolt DP tunnel - * starts/stops its own pixel clock instead of configuring the - * PHY directly, and never fails the apcall over it -- a - * failed clock just keeps the crossbar connection down - * (dcp->tb_clock_ok), which dcp_tunnel_crossbar_up() checks. + * A Thunderbolt DP tunnel starts and stops its own pixel clock + * instead of configuring the PHY, and never fails the call over + * it: a failed clock keeps the crossbar connection down, see + * dcp_tunnel_crossbar_up(). */ if (dptx->atcphy && service->ep->dcp->dptx_tunnel) { dcp_tunnel_set_rate(service->ep->dcp, dptx->atcphy, @@ -664,12 +601,10 @@ dptxport_call_activate(struct apple_epic_service *service, struct apple_dcp *dcp = service->ep->dcp; /* - * Ported from aurora-silicon/linux#8: no crossbar, and no PHY mode - * change for a tunnel (dptx->atcphy must stay in USB4/TBT mode the - * whole connection, see dcp_tunnel_set_rate()). The DP IN adapter is - * only woken (DPTX_INACTIVE=0) here, via dcp_tunnel_dpin_activate(); - * waking it earlier hangs the machine. Activate always replies - * success to DCP. + * No PHY mode change for a tunnel: the ATC PHY stays in USB4/TBT + * mode for the whole connection. The DP IN adapter may only be woken + * (DPTX_INACTIVE=0) from here; waking it earlier hangs the machine. + * Activate always replies success to DCP. */ if (dptx->atcphy && !dcp->phy_managed_by_typec) phy_set_mode_ext(dptx->atcphy, PHY_MODE_DP, dcp->index); @@ -755,11 +690,7 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, switch (idx) { case DPTX_APCALL_WILL_CHANGE_LINKG_CONFIG: - /* - * Ported from aurora-silicon/linux#8: a re-link on an - * established tunnel takes the crossbar connection down - * first. - */ + /* A re-link takes an established tunnel's crossbar down first. */ if (service->ep->dcp->dptx_tunnel && dptx->link_rate) dcp_tunnel_crossbar_down(service->ep->dcp); return dptxport_call_will_change_link_config(service); diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index e731604fb6768..f5096c59e8df8 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -63,13 +63,6 @@ struct dptx_port { u32 link_rate, pending_link_rate; u32 drive_settings[2]; u32 trained_voltage[4], trained_pre[4]; - /* - * Per-boot invocation counters for the validate/connect/request/ - * release AFK calls, used together with the caller return address - * logged alongside them to correlate call ordering when diagnosing - * link-training sequencing issues. - */ - u32 validate_calls, connect_calls, request_calls, release_calls; }; int dptxport_validate_connection(struct apple_epic_service *service, u8 core, From 33ac8110c40bfb889902f49f3c76956fd3cd5cdd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:27:44 +1100 Subject: [PATCH 1373/2680] drm: apple: afk: factor the receive bounds check into a helper afk_recv() open-codes the same "does this queue entry fit before the end of the ring" test twice, once to detect the wrap-around dummy entry and once to reject an entry that still does not fit. Move it into afk_ring_payload_fits() so the predicate can be tested without a live co-processor. No functional change. (cherry picked from commit dbdebdb0ee3ac458cf3b6aa1e6f49099bbc4a743) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 4 ++-- drivers/gpu/drm/apple/afk.h | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 0d598980c96e4..86f9efd82b48f 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -1217,7 +1217,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) * the current dummy queue entry and we have to advance to the next one * which will contain the real data. */ - if (rptr + size + sizeof(*hdr) > ep->rxbfr.bufsz) { + if (!afk_ring_payload_fits(rptr, size, ep->rxbfr.bufsz)) { rptr = 0; hdr = ep->rxbfr.buf + rptr; magic = le32_to_cpu(hdr->magic); @@ -1232,7 +1232,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) } } - if (rptr + size + sizeof(*hdr) > ep->rxbfr.bufsz) { + if (!afk_ring_payload_fits(rptr, size, ep->rxbfr.bufsz)) { dev_warn(ep->dcp->dev, "AFK[ep:%02x]: queue entry out of bounds: 0x%lx > 0x%lx\n", ep->endpoint, rptr + size + sizeof(*hdr), ep->rxbfr.bufsz); diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 0304410bd6e82..a483b8cefa5fb 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -266,6 +266,11 @@ struct afk_qe { u8 data[]; }; +static inline bool afk_ring_payload_fits(u32 ptr, u32 bytes, size_t bufsz) +{ + return ptr + bytes + sizeof(struct afk_qe) <= bufsz; +} + struct epic_hdr { u8 version; __le16 seq; From d52dcdaf08eb7839e1c841d317c01e159c38ba2b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:47:09 +1100 Subject: [PATCH 1374/2680] soc: apple: rtkit-helper: Terminate log messages Six messages lack a trailing newline, and the RTKit initialisation error misspells "initialize". No functional change. (cherry picked from commit efda4e7a8a30dfcc7e2cee451f002128693ec8fe) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit-helper.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c index 3dd941241893b..5f03bed772f26 100644 --- a/drivers/soc/apple/rtkit-helper.c +++ b/drivers/soc/apple/rtkit-helper.c @@ -51,7 +51,7 @@ static int apple_rtkit_helper_shmem_setup(void *cookie, struct apple_rtkit_shmem if (!helper->sram) { dev_err(helper->dev, - "RTKit buffer request with no SRAM region: %pR", &res); + "RTKit buffer request with no SRAM region: %pR\n", &res); return -EFAULT; } @@ -59,7 +59,7 @@ static int apple_rtkit_helper_shmem_setup(void *cookie, struct apple_rtkit_shmem if (res.end < res.start || !resource_contains(helper->sram, &res)) { dev_err(helper->dev, - "RTKit buffer request outside SRAM region: %pR", &res); + "RTKit buffer request outside SRAM region: %pR\n", &res); return -EFAULT; } @@ -139,7 +139,7 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) helper->sram_base = devm_ioremap_resource(dev, helper->sram); if (IS_ERR(helper->sram_base)) return dev_err_probe(dev, PTR_ERR(helper->sram_base), - "Failed to map SRAM region"); + "Failed to map SRAM region\n"); } if (apple_rtkit_helper_live(helper)) { @@ -147,7 +147,7 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) &apple_rtkit_helper_ops); if (IS_ERR(helper->rtk)) return dev_err_probe(dev, PTR_ERR(helper->rtk), - "Failed to adopt RTKit"); + "Failed to adopt RTKit\n"); ret = devm_add_action_or_reset(dev, apple_rtkit_helper_release, helper->rtk); @@ -163,7 +163,7 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) devm_apple_rtkit_init(dev, helper, NULL, 0, &apple_rtkit_helper_ops); if (IS_ERR(helper->rtk)) return dev_err_probe(dev, PTR_ERR(helper->rtk), - "Failed to intialize RTKit"); + "Failed to initialize RTKit\n"); writel_relaxed(APPLE_ASC_CPU_CONTROL_RUN, helper->asc_base + APPLE_ASC_CPU_CONTROL); @@ -174,7 +174,7 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) else ret = apple_rtkit_wake(helper->rtk); if (ret != 0) - return dev_err_probe(dev, ret, "Failed to wake up coprocessor"); + return dev_err_probe(dev, ret, "Failed to wake up coprocessor\n"); return 0; } From cc121e67c807b332e52b075a8721eb89864b9f3f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:28:00 +1100 Subject: [PATCH 1375/2680] drm: apple: afk: bound received queue entries without 32-bit wrap afk_recv() checks that a receive queue entry fits in the ring with "rptr + size + sizeof(*hdr) > bufsz". rptr and size are both u32 read from memory the co-processor writes, so their sum is computed in 32 bits and wraps before it is widened. An entry at offset 0x40 that claims 0xfffffff8 bytes passes both checks, and the entry is then copied into the receive scratch buffer, which is only bufsz bytes long. Since the scratch buffer was introduced this is a heap overflow driven by a single bad length; before it, the handlers read far past the ring. Do the check in afk_ring_payload_fits() with subtractions that cannot wrap, report the entry size and offset instead of a derived end, and add KUnit cases for the exact end, offsets past the ring and wrapping sums. Fixes: 63861816dc0c ("drm/apple: preallocate the AFK receive dispatch buffer") (cherry picked from commit 72afd5aefb9f47a5ac17ef2a9787bfab6dbeb6bb) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 4 ++-- drivers/gpu/drm/apple/afk.h | 3 ++- drivers/gpu/drm/apple/afk_test.c | 21 +++++++++++++++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 86f9efd82b48f..e42f9d997cfde 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -1234,8 +1234,8 @@ static bool afk_recv(struct apple_dcp_afkep *ep) if (!afk_ring_payload_fits(rptr, size, ep->rxbfr.bufsz)) { dev_warn(ep->dcp->dev, - "AFK[ep:%02x]: queue entry out of bounds: 0x%lx > 0x%lx\n", - ep->endpoint, rptr + size + sizeof(*hdr), ep->rxbfr.bufsz); + "AFK[ep:%02x]: queue entry of 0x%x bytes at 0x%x out of bounds\n", + ep->endpoint, size, rptr); return false; } diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index a483b8cefa5fb..d06f1ad9d804a 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -268,7 +268,8 @@ struct afk_qe { static inline bool afk_ring_payload_fits(u32 ptr, u32 bytes, size_t bufsz) { - return ptr + bytes + sizeof(struct afk_qe) <= bufsz; + return ptr <= bufsz && sizeof(struct afk_qe) <= bufsz - ptr && + bytes <= bufsz - ptr - sizeof(struct afk_qe); } struct epic_hdr { diff --git a/drivers/gpu/drm/apple/afk_test.c b/drivers/gpu/drm/apple/afk_test.c index e58a952642045..c3130086c78e1 100644 --- a/drivers/gpu/drm/apple/afk_test.c +++ b/drivers/gpu/drm/apple/afk_test.c @@ -29,8 +29,29 @@ static void afk_ring_stride_test(struct kunit *test) KUNIT_EXPECT_EQ(test, afk_ring_advance(0xfc0, 0x40, 0x1000, stride), 0U); } +static void afk_ring_payload_fits_test(struct kunit *test) +{ + const size_t hdr = sizeof(struct afk_qe); + + /* An entry ending exactly at the end of the ring fits. */ + KUNIT_EXPECT_TRUE(test, afk_ring_payload_fits(0, 0x1000 - hdr, 0x1000)); + KUNIT_EXPECT_TRUE(test, afk_ring_payload_fits(0xfc0, 0x30, 0x1000)); + KUNIT_EXPECT_TRUE(test, afk_ring_payload_fits(0x1000 - hdr, 0, 0x1000)); + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0xfc0, 0x31, 0x1000)); + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0xff8, 0, 0x1000)); + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0x1000, 0, 0x1000)); + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0x1040, 0, 0x1000)); + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0, 0, hdr - 1)); + /* Sizes that wrap a 32-bit offset sum must not pass for a fit. */ + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0x40, 0xfffffff8, 0x1000)); + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0x40, U32_MAX - 0x3f, 0x1000)); + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0x100, U32_MAX - 0xff, 0x1000)); + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0, U32_MAX, 0x1000)); +} + static struct kunit_case afk_ring_cases[] = { KUNIT_CASE(afk_ring_stride_test), + KUNIT_CASE(afk_ring_payload_fits_test), {} }; From 9f295a82ee2a82985c3b11d405373f7de7a2a9fa Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:47:44 +1100 Subject: [PATCH 1376/2680] pmdomain: apple: pmc: keep the DCS limit state when the rail enable fails apple_pmc_dcs_set_cur_state() records the new cooling state before it rewrites the DCS rail enable word, and keeps it when the write does not stick. Two things then go wrong: - Lifting a limit: the rail still honours only the AP agent, but the AP vote has already been put back to its boot value, the polling work is cancelled and dcs_state reads 0. The PMP and GPU votes are ignored with nothing left voting on their behalf, and since a request for state 0 now matches dcs_state, no later call retries the write. - Setting a limit: dcs_state reports a limit the PMC does not apply. Put the limit change into a helper and, when the enable write fails, return to the previous state with the same helper, so the rail, the AP vote, the polling work and dcs_state agree again and the next request retries. On success the sequence of writes is unchanged. The failure needs a rail enable word that does not take the write, which memory standing in for the registers cannot produce, so there is no KUnit case for it; the existing apple-pmc suite covers the successful limit cycle. Fixes: cf8edfc522ce ("pmdomain: apple: pmc: limit the DCS rail as a cooling device") (cherry picked from commit aa566ebf1748663f08a3d4f6f6db18d9aa687e91) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmc.c | 50 +++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 057f41e5f8959..541821fb00777 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -393,36 +393,56 @@ static int apple_pmc_dcs_get_cur_state(struct thermal_cooling_device *cdev, return 0; } +/* + * Caller holds pmc->lock. Vote the capped demand before the other agents + * stop being honoured, and drop that vote only once they are honoured + * again, so that the rail never dips below either. + */ +static int apple_pmc_dcs_limit(struct apple_pmc *pmc, unsigned long state) +{ + unsigned int rail = pmc->hw->dcs_rail; + int ret; + + WRITE_ONCE(pmc->dcs_state, state); + if (state) { + apple_pmc_dcs_vote(pmc); + return apple_pmc_enable_write(pmc, rail); + } + + ret = apple_pmc_enable_write(pmc, rail); + if (!ret) + apple_pmc_dcs_vote(pmc); + + return ret; +} + static int apple_pmc_dcs_set_cur_state(struct thermal_cooling_device *cdev, unsigned long state) { struct apple_pmc *pmc = cdev->devdata; - unsigned int rail = pmc->hw->dcs_rail; - unsigned long flags; + unsigned long flags, old; + bool limited; int ret = 0; if (state > pmc->hw->dcs_max) return -EINVAL; raw_spin_lock_irqsave(&pmc->lock, flags); - if (state != pmc->dcs_state) { + old = pmc->dcs_state; + if (state != old) { + ret = apple_pmc_dcs_limit(pmc, state); /* - * Vote the capped demand before the other agents stop being - * honoured, and drop that vote only once they are honoured - * again, so that the rail never dips below either. + * The rail enable did not change, so neither did the limit: + * go back to the old state, which the next call then retries + * to leave. */ - WRITE_ONCE(pmc->dcs_state, state); - if (state) { - apple_pmc_dcs_vote(pmc); - ret = apple_pmc_enable_write(pmc, rail); - } else { - ret = apple_pmc_enable_write(pmc, rail); - apple_pmc_dcs_vote(pmc); - } + if (ret) + apple_pmc_dcs_limit(pmc, old); } + limited = pmc->dcs_state; raw_spin_unlock_irqrestore(&pmc->lock, flags); - if (state) + if (limited) queue_delayed_work(system_freezable_power_efficient_wq, &pmc->dcs_work, msecs_to_jiffies(APPLE_PMC_DCS_POLL_MS)); From f598f24c0417a302a8492f6bdb3cae9bc6b6764e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:47:47 +1100 Subject: [PATCH 1377/2680] hwmon: macsmc: move the module help text back to its option The KUnit option was inserted between the help paragraphs of SENSORS_MACSMC_HWMON, so the sentence naming the macsmc-hwmon module now ends the help of SENSORS_MACSMC_HWMON_KUNIT_TEST, a bool. Move it back. (cherry picked from commit 953bda88a5a8165e3094960a760c9a005b9bf55a) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hwmon/Kconfig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index 70c71a9148d32..f418775288388 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -1208,6 +1208,9 @@ config SENSORS_MACSMC_HWMON on Apple Silicon devices. Say Y here if you have an Apple Silicon device. + This driver can also be built as a module. If so, the module will + be called macsmc-hwmon. + config SENSORS_MACSMC_HWMON_KUNIT_TEST bool "KUnit tests for the Apple SMC hwmon driver" if !KUNIT_ALL_TESTS depends on SENSORS_MACSMC_HWMON && KUNIT=y @@ -1218,9 +1221,6 @@ config SENSORS_MACSMC_HWMON_KUNIT_TEST that cannot be set up must keep its channel; the tests check that. No SMC is needed. Only useful for kernel developers. - This driver can also be built as a module. If so, the module will - be called macsmc-hwmon. - config SENSORS_MAX1111 tristate "Maxim MAX1111 Serial 8-bit ADC chip and compatibles" depends on SPI_MASTER From 4fcc4e7591a4853afc935b1bcbe94e7cf3f3f88c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:45:28 +1100 Subject: [PATCH 1378/2680] drm: apple: free property dictionaries the connector does not keep dcp_connector_update_dict() keeps the ColorElements, TimingElements, DisplayAttributes and Transport dictionaries for debugfs and clears the caller's pointer for every key, so the buffer of any other key is lost. That used to be harmless because the chunk buffer was device-managed. Since it became a plain kzalloc() allocation, every property transfer with another key leaks up to the 1 MiB the firmware may announce, for as long as the system runs. The switch away from devm is deliberate and stays: a Type-C port can move to another DCP before its dictionaries are replaced, and freeing against the connector's current DCP then fails devres_destroy(). Free the buffer with kfree() when no dictionary takes ownership of it. Fixes: 38c14f6107df ("drm: apple: support ProMotion variable refresh rate") (cherry picked from commit 3e425a8210c10a48e5d9cff7816c2346aee41400) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/connector.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/apple/connector.c b/drivers/gpu/drm/apple/connector.c index c7a2d97087499..27082dc21038e 100644 --- a/drivers/gpu/drm/apple/connector.c +++ b/drivers/gpu/drm/apple/connector.c @@ -148,6 +148,8 @@ void dcp_connector_update_dict(struct apple_connector *connector, const char *ke dcp_connector_set_dict(connector, &connector->display_attributes, chunks); else if (!strcmp(key, "Transport")) dcp_connector_set_dict(connector, &connector->transport, chunks); + else + kfree(chunks->data); chunks->data = NULL; chunks->length = 0; From 65a71639affd3fdf8d99f55a7dc4861c9ba81a83 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:47:49 +1100 Subject: [PATCH 1379/2680] soc: apple: clpc: drop the syscon select and the GPU from Kconfig The driver no longer reads the GPU power state through the PMGR syscon, so it does not need MFD_SYSCON, and the help text should not promise a GPU input. No functional change for configurations that enable it: the PMGR power-state driver selects MFD_SYSCON itself. (cherry picked from commit e6adc6a8417e41dd1c2da4189663df26d4a78581) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 2640e418be7c6..599f3cd761cc1 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -148,12 +148,11 @@ config APPLE_CLPC tristate "Apple host performance controller floors" depends on ARCH_APPLE || COMPILE_TEST depends on APPLE_PMC && CPU_FREQ - select MFD_SYSCON default ARCH_APPLE help Raises the DRAM controller and SoC fabric performance floors of the - host's PMC voter agent while CPU clusters, and optionally the GPU, - are busy, and releases them at idle. + host's PMC voter agent while CPU clusters are busy, and releases + them at idle. Say 'y' or 'm' here if you have an Apple SoC with a PMC. From bc031e3d6a70e72569f34c760adf1086c66647af Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:47:55 +1100 Subject: [PATCH 1380/2680] cpufreq: apple-soc: say why a cluster with verified transitions fails init When the OPP levels are outside the SoC P-state range, the DVFS command register stays busy, or the inherited P-state has no OPP, ->init() returns an error without a message, and the cpufreq core only reports the failure at debug level. The cluster is then left without cpufreq and nothing in the log says why. Report each case once. (cherry picked from commit 17e86f73c68ef70f581a000216e15e40c65d53f9) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/cpufreq/apple-soc-cpufreq.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 5f16b80b746b9..d5172ddd346f3 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -843,6 +843,8 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) dev_pm_opp_put(opp); if (freq_table[i].driver_data < info->min_pstate || freq_table[i].driver_data > info->max_pstate) { + dev_err(cpu_dev, "OPP level %u is not a valid P-state\n", + freq_table[i].driver_data); ret = -EINVAL; goto out_free_cpufreq_table; } @@ -856,13 +858,16 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) ret = readq_poll_timeout_atomic(reg_base + APPLE_DVFS_CMD, cmd, !(cmd & APPLE_DVFS_CMD_BUSY), 2, apple_soc_cpufreq_timeout(info)); - if (ret) + if (ret) { + dev_err(cpu_dev, "DVFS busy at init, command=%#llx\n", cmd); goto out_free_cpufreq_table; + } state = FIELD_GET(APPLE_DVFS_CMD_PS1, cmd); for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) if (freq_table[i].driver_data == state) break; if (freq_table[i].frequency == CPUFREQ_TABLE_END) { + dev_err(cpu_dev, "current P-state %u has no OPP\n", state); ret = -ERANGE; goto out_free_cpufreq_table; } From 2c07632230c505bc934957493cd49e404f9f4c41 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:45:30 +1100 Subject: [PATCH 1381/2680] drm: apple: parser: add KUnit tests for malformed dictionaries The serialized dictionaries come from the DCP firmware, which the driver does not trust. Add a suite that feeds the parser hand-built blobs: a valid EPIC service announcement, the H17P spelling of its keys with and without the H17P flag, strings and dictionaries running past the end of the blob, nesting deeper than the skip limit, a bad header, and the extent check used for delivered property payloads. (cherry picked from commit d2feca3911c45793568bd1408a8bed4b889f97c4) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Makefile | 2 +- drivers/gpu/drm/apple/parser_test.c | 220 ++++++++++++++++++++++++++++ 2 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/apple/parser_test.c diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 2830b35720b6d..b0e3d4cb127c1 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -12,7 +12,7 @@ appledrm-y += iomfb_v12_3.o appledrm-y += iomfb_v13_3.o appledrm-y += iomfb_v14_7.o appledrm-y += iomfb_h17p.o -appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += iomfb_h17p_test.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += iomfb_h17p_test.o parser_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple_resume_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += parser_mode_test.o appledrm-y += epic/dpavservep.o diff --git a/drivers/gpu/drm/apple/parser_test.c b/drivers/gpu/drm/apple/parser_test.c new file mode 100644 index 0000000000000..7dd8ca6eac189 --- /dev/null +++ b/drivers/gpu/drm/apple/parser_test.c @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include +#include +#include +#include + +#include "parser.h" + +/* Serialized dictionary tags: size in bits 0-23, type in 24-28, last in 31. */ +#define T_DICT 1 +#define T_ARRAY 2 +#define T_INT64 4 +#define T_STRING 9 +#define T_BOOL 11 + +struct blob { + u8 data[512]; + size_t len; +}; + +static void blob_align(struct blob *b) +{ + b->len = round_up(b->len, 4); +} + +static void blob_tag(struct blob *b, u32 type, u32 size) +{ + blob_align(b); + put_unaligned_le32(size | type << 24, b->data + b->len); + b->len += 4; +} + +static void blob_init(struct blob *b) +{ + memset(b, 0, sizeof(*b)); + put_unaligned_le32(0xd3, b->data); + b->len = 4; +} + +static void blob_string(struct blob *b, const char *s) +{ + size_t n = strlen(s); + + blob_tag(b, T_STRING, n); + memcpy(b->data + b->len, s, n); + b->len += n; +} + +static void blob_int(struct blob *b, s64 value) +{ + blob_tag(b, T_INT64, 64); + put_unaligned_le64(value, b->data + b->len); + b->len += 8; +} + +static int parse_service(struct kunit *test, const struct blob *b, bool h17p, + const char **name, const char **class, s64 *unit) +{ + struct dcp_parse_ctx ctx; + + KUNIT_ASSERT_EQ(test, parse(b->data, b->len, &ctx), 0); + return parse_epic_service_init(&ctx, name, class, unit, h17p); +} + +static void parser_service_init_test(struct kunit *test) +{ + const char *name, *class; + struct blob b; + s64 unit = -1; + + blob_init(&b); + blob_tag(&b, T_DICT, 4); + blob_string(&b, "EPICName"); + blob_string(&b, "dcpav-service-epic"); + blob_string(&b, "Other"); + blob_tag(&b, T_BOOL, 1); + blob_string(&b, "EPICProviderClass"); + blob_string(&b, "AppleDCPAVService"); + blob_string(&b, "EPICUnit"); + blob_int(&b, 2); + + KUNIT_ASSERT_EQ(test, parse_service(test, &b, false, &name, &class, &unit), 0); + KUNIT_EXPECT_STREQ(test, name, "dcpav-service-epic"); + KUNIT_EXPECT_STREQ(test, class, "AppleDCPAVService"); + KUNIT_EXPECT_EQ(test, unit, 2); + kfree(name); + kfree(class); +} + +static void parser_service_init_h17p_keys_test(struct kunit *test) +{ + const char *name, *class; + struct blob b; + s64 unit = -1; + + blob_init(&b); + blob_tag(&b, T_DICT, 3); + blob_string(&b, "compartment"); + blob_string(&b, "system"); + blob_string(&b, "name"); + blob_string(&b, "system"); + blob_string(&b, "interface-id"); + blob_int(&b, 1); + + KUNIT_EXPECT_EQ(test, parse_service(test, &b, false, &name, &class, &unit), + -ENOENT); + KUNIT_EXPECT_TRUE(test, IS_ERR(name)); + KUNIT_EXPECT_TRUE(test, IS_ERR(class)); + + KUNIT_ASSERT_EQ(test, parse_service(test, &b, true, &name, &class, &unit), 0); + KUNIT_EXPECT_STREQ(test, name, "system"); + KUNIT_EXPECT_STREQ(test, class, "system"); + KUNIT_EXPECT_EQ(test, unit, 1); + kfree(name); + kfree(class); +} + +static void parser_service_init_truncated_test(struct kunit *test) +{ + const char *name, *class; + struct blob b; + s64 unit; + + /* A string length running past the end of the blob. */ + blob_init(&b); + blob_tag(&b, T_DICT, 3); + blob_string(&b, "EPICName"); + blob_tag(&b, T_STRING, 0x100); + memcpy(b.data + b.len, "dcp", 3); + b.len += 3; + + KUNIT_EXPECT_LT(test, parse_service(test, &b, false, &name, &class, &unit), 0); + KUNIT_EXPECT_TRUE(test, IS_ERR(name)); + KUNIT_EXPECT_TRUE(test, IS_ERR(class)); + + /* A dictionary announcing more pairs than the blob holds. */ + blob_init(&b); + blob_tag(&b, T_DICT, 0xffffff); + blob_string(&b, "EPICName"); + blob_string(&b, "dcp"); + + KUNIT_EXPECT_LT(test, parse_service(test, &b, false, &name, &class, &unit), 0); + KUNIT_EXPECT_TRUE(test, IS_ERR(name)); + KUNIT_EXPECT_TRUE(test, IS_ERR(class)); +} + +static void parser_nesting_test(struct kunit *test) +{ + const char *name, *class; + struct blob b; + unsigned int i; + s64 unit; + + /* An ignored value nested far deeper than any real dictionary. */ + blob_init(&b); + blob_tag(&b, T_DICT, 4); + blob_string(&b, "EPICName"); + blob_string(&b, "dcp"); + blob_string(&b, "EPICProviderClass"); + blob_string(&b, "dcp"); + blob_string(&b, "EPICUnit"); + blob_int(&b, 0); + blob_string(&b, "Ignored"); + for (i = 0; i < 64; i++) + blob_tag(&b, T_ARRAY, 1); + blob_tag(&b, T_BOOL, 0); + + KUNIT_EXPECT_LT(test, parse_service(test, &b, false, &name, &class, &unit), 0); + KUNIT_EXPECT_TRUE(test, IS_ERR(name)); + KUNIT_EXPECT_TRUE(test, IS_ERR(class)); +} + +static void parser_bad_header_test(struct kunit *test) +{ + struct dcp_parse_ctx ctx; + struct blob b; + + blob_init(&b); + KUNIT_EXPECT_EQ(test, parse(b.data, 3, &ctx), -EINVAL); + put_unaligned_le32(0xd4, b.data); + KUNIT_EXPECT_EQ(test, parse(b.data, b.len, &ctx), -EINVAL); +} + +static void parser_property_complete_test(struct kunit *test) +{ + struct blob b; + + blob_init(&b); + blob_tag(&b, T_DICT, 1); + blob_string(&b, "Key"); + blob_int(&b, 7); + + KUNIT_EXPECT_TRUE(test, dcp_property_complete(b.data, b.len, b.len)); + /* One unsent byte after the aligned object is tolerated. */ + KUNIT_EXPECT_TRUE(test, dcp_property_complete(b.data, b.len, b.len + 1)); + KUNIT_EXPECT_FALSE(test, dcp_property_complete(b.data, b.len, b.len + 4)); + KUNIT_EXPECT_FALSE(test, dcp_property_complete(b.data, b.len - 4, b.len)); + KUNIT_EXPECT_FALSE(test, dcp_property_complete(b.data, b.len + 4, b.len + 4)); + KUNIT_EXPECT_FALSE(test, dcp_property_complete(NULL, b.len, b.len)); +} + +static struct kunit_case parser_cases[] = { + KUNIT_CASE(parser_service_init_test), + KUNIT_CASE(parser_service_init_h17p_keys_test), + KUNIT_CASE(parser_service_init_truncated_test), + KUNIT_CASE(parser_nesting_test), + KUNIT_CASE(parser_bad_header_test), + KUNIT_CASE(parser_property_complete_test), + {} +}; + +static struct kunit_suite parser_suite = { + .name = "apple-dcp-parser", + .test_cases = parser_cases, +}; + +kunit_test_suite(parser_suite); From 857fcea70c2855aae71d6fc8d85d66de76a9a98f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:47:51 +1100 Subject: [PATCH 1382/2680] soc: apple: mcc: print the boot DMA device before dropping its node apple_mcc_wait_boot_dma() drops the reference the phandle iterator holds on the node and then prints the node with %pOF. Print it first. The node is part of the base device tree, so this is a correctness cleanup rather than a reachable use after free. (cherry picked from commit d5b7c447a7507b36bdb445d254ac2b313373e69c) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/mcc.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/mcc.c b/drivers/soc/apple/mcc.c index 1bc23d1da22d0..b30023ad6fced 100644 --- a/drivers/soc/apple/mcc.c +++ b/drivers/soc/apple/mcc.c @@ -61,9 +61,10 @@ static int apple_mcc_wait_boot_dma(struct device *dev) if (pdev) put_device(&pdev->dev); if (!bound) { + ret = dev_err_probe(dev, -EPROBE_DEFER, "waiting for %pOF to bind\n", + it.node); of_node_put(it.node); - return dev_err_probe(dev, -EPROBE_DEFER, "waiting for %pOF to bind\n", - it.node); + return ret; } } From 295b7dcb90fd0e6b093e30992db632e5ae123f51 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:26:25 +1100 Subject: [PATCH 1383/2680] HID: dockchannel: Build the protocol KUnit test only with the driver The protocol test lives in drivers/hid/dockchannel-hid/, which the HID Makefile enters only for CONFIG_HID_DOCKCHANNEL, but the test depends only on APPLE_DOCKCHANNEL. With the transport driver disabled the test option can be set, or turned on by KUNIT_ALL_TESTS, and the suite is silently not built: a KUnit run reports zero tests for it. Make the test depend on HID_DOCKCHANNEL. Fixes: 1f5c943c4327 ("HID: dockchannel: Support MTP power method 2") (cherry picked from commit d488b7e0f240b905c9a5ec6ceb402121fefdeb7d) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hid/dockchannel-hid/Kconfig b/drivers/hid/dockchannel-hid/Kconfig index a19b543c066cb..5f1da27d445a8 100644 --- a/drivers/hid/dockchannel-hid/Kconfig +++ b/drivers/hid/dockchannel-hid/Kconfig @@ -16,7 +16,7 @@ config HID_DOCKCHANNEL config HID_DOCKCHANNEL_KUNIT_TEST tristate "KUnit tests for the DockChannel HID protocol helpers" if !KUNIT_ALL_TESTS - depends on KUNIT && APPLE_DOCKCHANNEL + depends on KUNIT && HID_DOCKCHANNEL default KUNIT_ALL_TESTS help Build the KUnit tests for the DockChannel HID protocol helpers, From 57dd8fd5a8f99952efba093be7ebe9356e9e7dba Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:06:03 +1100 Subject: [PATCH 1384/2680] mfd: macsmc: Reject short fixed-width replies The raw SMC interface reports how many bytes firmware returned. Copying the requested width instead can expose stale SRAM data, while fixed-width readers may consume bytes that firmware never supplied. Copy only returned bytes and reject oversized replies. Make every fixed-width reader reject short replies with -EIO and clear its value before returning. Keep capacity-based identity string reads unchanged. Fixes: e038d985c982 ("mfd: Add Apple Silicon System Management Controller") Cc: stable@vger.kernel.org (cherry picked from commit 07166c0bf377973be976dec85312426ef998ed68) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/mfd/macsmc.c | 15 ++++++++++----- drivers/power/supply/macsmc-power.c | 4 ++++ drivers/rtc/rtc-macsmc.c | 8 ++++++-- include/linux/mfd/macsmc.h | 16 ++++++++++++++-- 4 files changed, 34 insertions(+), 9 deletions(-) diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 8cdc59a7dd88c..e7ad580f58fca 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -157,6 +157,9 @@ static int apple_smc_rw_locked(struct apple_smc *smc, smc_key key, if (ret < 0) return ret; + if (rsize && ret > rsize) + return -EIO; + if (rsize) { /* * Small data <= 4 bytes is returned as part of the reply @@ -165,9 +168,9 @@ static int apple_smc_rw_locked(struct apple_smc *smc, smc_key key, * Device memory. */ if (rsize <= 4) - memcpy(rbuf, &rdata, rsize); + memcpy(rbuf, &rdata, ret); else - memcpy_fromio(rbuf, smc->shmem.iomem, rsize); + memcpy_fromio(rbuf, smc->shmem.iomem, ret); } return ret; @@ -415,7 +418,7 @@ static int apple_smc_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct device_node *np; struct apple_smc *smc; - __be32 count; + __be32 count = 0; int ret; smc = devm_kzalloc(dev, sizeof(*smc), GFP_KERNEL); @@ -467,8 +470,10 @@ static int apple_smc_probe(struct platform_device *pdev) BLOCKING_INIT_NOTIFIER_HEAD(&smc->event_handlers); ret = apple_smc_read(smc, SMC_KEY(#KEY), &count, sizeof(count)); - if (ret >= 0 && ret != sizeof(count)) - ret = -EINVAL; + if (ret >= 0 && ret != sizeof(count)) { + count = 0; + ret = -EIO; + } if (ret < 0) return dev_err_probe(smc->dev, ret, "Failed to get key count"); smc->key_count = be32_to_cpu(count); diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c index 900d76a305dce..3753c3451e4b7 100644 --- a/drivers/power/supply/macsmc-power.c +++ b/drivers/power/supply/macsmc-power.c @@ -334,6 +334,10 @@ static int macsmc_battery_get_charge_behaviour(struct macsmc_power *power) ret = apple_smc_read(power->smc, SMC_KEY(CHTE), chte_buf, 4); if (ret < 0) return ret; + if (ret != sizeof(chte_buf)) { + memset(chte_buf, 0, sizeof(chte_buf)); + return -EIO; + } if (chte_buf[0] == 0x01) return POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE; diff --git a/drivers/rtc/rtc-macsmc.c b/drivers/rtc/rtc-macsmc.c index 1c1c529df8296..a28dd7c577bde 100644 --- a/drivers/rtc/rtc-macsmc.c +++ b/drivers/rtc/rtc-macsmc.c @@ -74,8 +74,10 @@ static int macsmc_rtc_get_time(struct device *dev, struct rtc_time *tm) ret = apple_smc_read(rtc->smc, SMC_KEY(CLKM), &ctr, RTC_BYTES); if (ret < 0) return ret; - if (ret != RTC_BYTES) + if (ret != RTC_BYTES) { + ctr = 0; return -EIO; + } if (rtc->offset_in_seconds) return macsmc_rtc_get_time_secs(rtc, ctr, tm); @@ -107,8 +109,10 @@ static int macsmc_rtc_set_time(struct device *dev, struct rtc_time *tm) ret = apple_smc_read(rtc->smc, SMC_KEY(CLKM), &ctr, RTC_BYTES); if (ret < 0) return ret; - if (ret != RTC_BYTES) + if (ret != RTC_BYTES) { + ctr = 0; return -EIO; + } if (rtc->offset_in_seconds) { __le64 off_secs; diff --git a/include/linux/mfd/macsmc.h b/include/linux/mfd/macsmc.h index cc09ecce0df74..fbc047b00afd8 100644 --- a/include/linux/mfd/macsmc.h +++ b/include/linux/mfd/macsmc.h @@ -229,7 +229,13 @@ static inline bool apple_smc_key_exists(struct apple_smc *smc, smc_key key) static inline int apple_smc_read_##type(struct apple_smc *smc, smc_key key, type *p) \ { \ int ret = apple_smc_read(smc, key, p, sizeof(*p)); \ - return (ret < 0) ? ret : ((ret != sizeof(*p)) ? -EINVAL : 0); \ + if (ret < 0) \ + return ret; \ + if (ret != sizeof(*p)) { \ + *p = 0; \ + return -EIO; \ + } \ + return 0; \ } \ static inline int apple_smc_write_##type(struct apple_smc *smc, smc_key key, type p) \ { \ @@ -243,7 +249,13 @@ static inline bool apple_smc_key_exists(struct apple_smc *smc, smc_key key) type w, type *r) \ { \ int ret = apple_smc_rw(smc, key, &w, sizeof(w), r, sizeof(*r)); \ - return (ret < 0) ? ret : ((ret != sizeof(*r)) ? -EINVAL : 0); \ + if (ret < 0) \ + return ret; \ + if (ret != sizeof(*r)) { \ + *r = 0; \ + return -EIO; \ + } \ + return 0; \ } APPLE_SMC_TYPE_OPS(u64) From c2585da8d1006d44f603a95978838d87bffdf17c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:45:33 +1100 Subject: [PATCH 1385/2680] drm: apple: parser: bound the timing catalog allocation enumerate_modes() allocates one display mode for every element the firmware announces in the TimingElements array before looking at any of them. The count is a 24-bit field, so a malformed catalog asks kcalloc() for gigabytes, which fails with an allocation warning instead of being rejected as malformed input. Every element takes at least a four-byte tag, so reject counts larger than the rest of the blob can hold, and pass __GFP_NOWARN so a catalog that is merely too large to allocate fails quietly like any other malformed one. Add a KUnit case for both an impossible count and a catalog whose entries are present but invalid. Fixes: 38c14f6107df ("drm: apple: support ProMotion variable refresh rate") (cherry picked from commit 8e741eb93c20f1ba1246f8231e22a8a0e301fee2) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/parser.c | 10 +++++++-- drivers/gpu/drm/apple/parser_test.c | 32 +++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 7034310e619e2..b67c677598d4a 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -1038,8 +1038,14 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, if (ret) return ERR_PTR(ret); - /* Start with a worst case allocation. */ - modes = kcalloc(it.len, sizeof(*modes), GFP_KERNEL); + /* + * Start with a worst case allocation. The count comes from firmware: + * every entry needs at least a tag in what is left of the blob, and a + * catalog too large to allocate fails like any other malformed one. + */ + if (it.len > (handle->len - handle->pos) / sizeof(struct dcp_parse_tag)) + return ERR_PTR(-EINVAL); + modes = kcalloc(it.len, sizeof(*modes), GFP_KERNEL | __GFP_NOWARN); *count = 0; if (!modes) diff --git a/drivers/gpu/drm/apple/parser_test.c b/drivers/gpu/drm/apple/parser_test.c index 7dd8ca6eac189..6795fcdf18864 100644 --- a/drivers/gpu/drm/apple/parser_test.c +++ b/drivers/gpu/drm/apple/parser_test.c @@ -184,6 +184,37 @@ static void parser_bad_header_test(struct kunit *test) KUNIT_EXPECT_EQ(test, parse(b.data, b.len, &ctx), -EINVAL); } +static void parser_mode_count_test(struct kunit *test) +{ + struct dcp_display_mode *modes; + struct dcp_parse_ctx ctx; + unsigned int count = 1; + struct blob b; + + /* An element count that cannot be backed by the bytes present. */ + blob_init(&b); + blob_tag(&b, T_ARRAY, 0xffffff); + blob_tag(&b, T_DICT, 0); + + KUNIT_ASSERT_EQ(test, parse(b.data, b.len, &ctx), 0); + modes = enumerate_modes(&ctx, &count, 0, 0, 0, false, false); + KUNIT_EXPECT_PTR_EQ(test, modes, ERR_PTR(-EINVAL)); + + /* Entries that are present but invalid are skipped, not fatal. */ + blob_init(&b); + blob_tag(&b, T_ARRAY, 2); + blob_tag(&b, T_DICT, 0); + blob_tag(&b, T_DICT, 1); + blob_string(&b, "ID"); + blob_int(&b, 1); + + KUNIT_ASSERT_EQ(test, parse(b.data, b.len, &ctx), 0); + modes = enumerate_modes(&ctx, &count, 0, 0, 0, false, false); + KUNIT_ASSERT_FALSE(test, IS_ERR(modes)); + KUNIT_EXPECT_EQ(test, count, 0); + kfree(modes); +} + static void parser_property_complete_test(struct kunit *test) { struct blob b; @@ -208,6 +239,7 @@ static struct kunit_case parser_cases[] = { KUNIT_CASE(parser_service_init_truncated_test), KUNIT_CASE(parser_nesting_test), KUNIT_CASE(parser_bad_header_test), + KUNIT_CASE(parser_mode_count_test), KUNIT_CASE(parser_property_complete_test), {} }; From c1ee56a9334d4813887505e68c35e256c88e7874 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:47:53 +1100 Subject: [PATCH 1386/2680] pmdomain: apple: pmgr-pwrstate: tidy the rate-limited error messages Align the continuation lines of the messages that became rate-limited, and make the reset/disable timeout report the value it polled rather than the register value read before the write, which is what "now" claims to show. (cherry picked from commit bbfd7c70dc3d1ca553f13f24f30644bd41a40741) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmgr-pwrstate.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index 053c087c37184..d0b75e0b4b671 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -68,7 +68,7 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a /* Resets are synchronous, and only work if the device is powered and clocked. */ if (reg & APPLE_PMGR_RESET && pstate != APPLE_PMGR_PS_ACTIVE) dev_err_ratelimited(ps->dev, "PS %s: powering off with RESET active\n", - genpd->name); + genpd->name); if (pstate != APPLE_PMGR_PS_ACTIVE && (ps->force_disable || ps->force_reset)) { u32 reg_pre = reg & ~(APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | @@ -88,8 +88,9 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a APPLE_PMGR_PS_SET_TIMEOUT); if (ret < 0) - dev_err_ratelimited(ps->dev, "PS %s: Failed to set reset/disable bits (now: 0x%x)\n", - genpd->name, reg); + dev_err_ratelimited(ps->dev, + "PS %s: Failed to set reset/disable bits (now: 0x%x)\n", + genpd->name, cur); } /* Do not replay the hardware-owned BUSY bit in this whole-word write. */ @@ -119,8 +120,9 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a } if (ret < 0) - dev_err_ratelimited(ps->dev, "PS %s: Failed to reach power state 0x%x (requested 0x%x, now: 0x%x)\n", - genpd->name, pstate, reg, cur); + dev_err_ratelimited(ps->dev, + "PS %s: Failed to reach power state 0x%x (requested 0x%x, now: 0x%x)\n", + genpd->name, pstate, reg, cur); if (auto_enable) { /* Not all devices implement this; this is a no-op where not implemented. */ @@ -222,7 +224,8 @@ static int apple_pmgr_reset_assert(struct reset_controller_dev *rcdev, unsigned if (ps->genpd.status == GENPD_STATE_OFF) dev_err_ratelimited(ps->dev, - "PS 0x%x: asserting RESET while powered down\n", ps->offset); + "PS 0x%x: asserting RESET while powered down\n", + ps->offset); dev_dbg(ps->dev, "PS 0x%x: assert reset\n", ps->offset); /* Quiesce device before asserting reset */ @@ -252,7 +255,8 @@ static int apple_pmgr_reset_deassert(struct reset_controller_dev *rcdev, unsigne if (ps->genpd.status == GENPD_STATE_OFF) dev_err_ratelimited(ps->dev, - "PS 0x%x: RESET was deasserted while powered down\n", ps->offset); + "PS 0x%x: RESET was deasserted while powered down\n", + ps->offset); spin_unlock_irqrestore(&ps->genpd.slock, flags); From ad51f7b7758f3d15ef81676f0d12c8da85cea6ad Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:47:08 +1100 Subject: [PATCH 1387/2680] HID: dockchannel: Log routine interface set-up at debug level Every boot logs a run of info lines while the MTP coprocessor announces its interfaces: the announcement and product name of each interface, and the start, firmware upload, reset and GPIO request of the multi-touch interface. They describe the normal sequence and say nothing actionable, so log them at debug level. Failures keep their warnings and errors. The one line per interface that reports it ready stays at info level: it is the only kernel-log evidence that the multi-touch firmware came up, and boot checks rely on it. A DT that sets apple,firmware-managed-reset on an interface that cannot use it made the driver ignore that interface without saying why; report it. No functional change. (cherry picked from commit be53e38a7f80d1dacb6cbe892834610b0b3628dc) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index b079f6a9ded21..4e8e1d611ad27 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -245,8 +245,11 @@ dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) firmware_managed_reset = of_property_read_bool(of_node, "apple,firmware-managed-reset"); if (firmware_managed_reset && - (strcmp(name, "multi-touch") || power_method != DCHID_POWER_METHOD_2)) + (strcmp(name, "multi-touch") || power_method != DCHID_POWER_METHOD_2)) { + dev_err(dchid->dev, "Firmware-managed reset is not supported for %s\n", + name); goto err_put_node; + } if (power_method != DCHID_POWER_METHOD_1 && power_method != DCHID_POWER_METHOD_2) { dev_err(dchid->dev, "Unsupported power method %u for %s\n", @@ -548,8 +551,8 @@ static int dchid_request_gpio(struct dchid_iface *iface) if (iface->gpio) return 0; - dev_info(iface->dchid->dev, "Requesting GPIO %s#%d: %s\n", - iface->name, iface->gpio_id, iface->gpio_name); + dev_dbg(iface->dchid->dev, "Requesting GPIO %s#%d: %s\n", + iface->name, iface->gpio_id, iface->gpio_name); /* A power method 2 interface holds its reset line from probe on. */ if (iface->power_method == DCHID_POWER_METHOD_2 && @@ -584,7 +587,7 @@ static int dchid_start_interface(struct dchid_iface *iface) return -EINPROGRESS; } - dev_info(iface->dchid->dev, "Starting interface %s\n", iface->name); + dev_dbg(iface->dchid->dev, "Starting interface %s\n", iface->name); iface->starting = true; @@ -604,7 +607,7 @@ static int dchid_start_interface(struct dchid_iface *iface) if (fw && size) { /* Send firmware to the device */ - dev_info(iface->dchid->dev, "Sending firmware for %s\n", iface->name); + dev_dbg(iface->dchid->dev, "Sending firmware for %s\n", iface->name); ret = dchid_send_firmware(iface, fw, size); if (ret < 0) { dev_err(iface->dchid->dev, "Failed to send %s firmware\n", iface->name); @@ -612,7 +615,7 @@ static int dchid_start_interface(struct dchid_iface *iface) } /* After loading firmware, multi-touch needs a reset */ - dev_info(iface->dchid->dev, "Resetting %s\n", iface->name); + dev_dbg(iface->dchid->dev, "Resetting %s\n", iface->name); if (iface->power_method == DCHID_POWER_METHOD_2) { ret = dchid_pm2_set_power(iface, DCHID_POWER_STATE_OFF); if (ret < 0) @@ -749,7 +752,7 @@ static void dchid_create_interface_work(struct work_struct *ws) return; } - dev_info(dchid->dev, "New interface %s\n", iface->name); + dev_dbg(dchid->dev, "New interface %s\n", iface->name); /* Start the interface. This is not the entire init process, as firmware is loaded later on device open. */ ret = dchid_enable_interface(iface); @@ -951,8 +954,8 @@ static void dchid_handle_init(struct dockchannel_hid *dchid, void *data, size_t dev_warn(dchid->dev, "Unterminated product name for %s\n", iface->name); } else { - dev_info(dchid->dev, "Product name for %s: %s\n", - iface->name, product); + dev_dbg(dchid->dev, "Product name for %s: %s\n", + iface->name, product); } break; } @@ -1011,7 +1014,7 @@ static void dchid_handle_gpio(struct dockchannel_hid *dchid, void *data, size_t goto err; } - dev_info(dchid->dev, "GPIO command: %s#%d: %d\n", iface->name, cmd->gpio, cmd->cmd); + dev_dbg(dchid->dev, "GPIO command: %s#%d: %d\n", iface->name, cmd->gpio, cmd->cmd); switch (cmd->cmd) { case 3: @@ -1397,7 +1400,7 @@ static int dockchannel_hid_probe(struct platform_device *pdev) return -EIO; } - dev_info(dchid->dev, "Initialized, awaiting packets\n"); + dev_dbg(dchid->dev, "Initialized, awaiting packets\n"); platform_set_drvdata(pdev, dchid); dockchannel_await(dchid->dc, dchid_handle_packet, dchid, sizeof(struct dchid_hdr)); From f7fc803df89921e827b46f79c0c9b2bf3e7f0fdd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:06:33 +1100 Subject: [PATCH 1388/2680] mfd: macsmc: Preserve the key output on lookup failure A failed key lookup may leave its output unwritten, but the unconditional byte swap reads and changes that output anyway. This can read an uninitialized caller value and changes an existing key on error. Receive the wire key in a temporary and publish it only after a successful command. Fixes: e038d985c982 ("mfd: Add Apple Silicon System Management Controller") Cc: stable@vger.kernel.org (cherry picked from commit f52bcacfb392746ca3b3a6aa10a733d3abf91506) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/mfd/macsmc.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index e7ad580f58fca..92f1b987bf888 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -203,11 +203,14 @@ EXPORT_SYMBOL(apple_smc_rw); int apple_smc_get_key_by_index(struct apple_smc *smc, int index, smc_key *key) { + u32 data; int ret; - ret = apple_smc_cmd(smc, SMC_MSG_GET_KEY_BY_INDEX, index, 0, 0, key); + ret = apple_smc_cmd(smc, SMC_MSG_GET_KEY_BY_INDEX, index, 0, 0, &data); + if (ret < 0) + return ret; - *key = swab32(*key); + *key = swab32(data); return ret; } EXPORT_SYMBOL(apple_smc_get_key_by_index); From e6db8d019fd861c8cd19f5197787707e7249ba12 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:29:16 +1100 Subject: [PATCH 1389/2680] drm: apple: afk: validate the receive write pointer and bound a drain afk_recv() checks the read pointer against the ring, but uses the write pointer, which the co-processor owns, only to decide when the ring is empty. The receive worker calls afk_recv() until it reports an empty ring. Since the consumer advances rptr modulo the ring in whole blocks, a write pointer at or past the end of the ring, or one that is not block aligned, is never reached: with every slot holding a valid entry the worker dispatches the same ring contents forever and holds the ordered endpoint queue. A producer that keeps moving a valid wptr ahead of rptr has the same effect. Check both pointers on every read, before any entry is touched: rptr must leave room for a queue entry header, wptr must lie inside the ring, and both must be block aligned, which is how the consumer advances and therefore what the producer has to write for the ring to drain. Treat a violation like the other invalid-entry cases and stop draining. Also bound one RECV notification to as many entries as the ring has blocks. A well-behaved producer cannot have more than that queued when the notification arrives; entries it writes during the pass come with their own RECV notification, which drains them. The pointer and budget checks are pure helpers with KUnit cases for an end-of-ring and an all-ones write pointer, misaligned pointers and a write pointer that keeps moving. (cherry picked from commit 76ab8771541b2aa4feb143f1039b8e2b25e03b63) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 32 +++++++++++++++----- drivers/gpu/drm/apple/afk.h | 35 ++++++++++++++++++++++ drivers/gpu/drm/apple/afk_test.c | 51 ++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index e42f9d997cfde..3615e6b5460f4 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -1189,15 +1189,15 @@ static bool afk_recv(struct apple_dcp_afkep *ep) wptr = le32_to_cpu(*ep->rxbfr.wptr); trace_afk_recv_rwptr_pre(ep, rptr, wptr); - if (rptr == wptr) + if (!afk_ring_ptrs_valid(rptr, wptr, ep->rxbfr.bufsz, ep->rxbfr.stride)) { + dev_warn_ratelimited(ep->dcp->dev, + "AFK[ep:%02x]: invalid receive ring pointers: rptr 0x%x wptr 0x%x\n", + ep->endpoint, rptr, wptr); return false; + } - if (rptr > (ep->rxbfr.bufsz - sizeof(*hdr))) { - dev_warn(ep->dcp->dev, - "AFK[ep:%02x]: rptr out of bounds: 0x%x > 0x%lx\n", - ep->endpoint, rptr, ep->rxbfr.bufsz - sizeof(*hdr)); + if (rptr == wptr) return false; - } dma_rmb(); @@ -1261,6 +1261,23 @@ static bool afk_recv(struct apple_dcp_afkep *ep) return true; } +static bool afk_recv_entry(void *ep) +{ + return afk_recv(ep); +} + +static void afk_recv_drain(struct apple_dcp_afkep *ep) +{ + unsigned int budget = afk_ring_drain_budget(ep->rxbfr.bufsz, + ep->rxbfr.stride); + + if (afk_ring_drain(afk_recv_entry, ep, budget) == budget && + ep->rxbfr.ready && le32_to_cpu(*ep->rxbfr.rptr) != le32_to_cpu(*ep->rxbfr.wptr)) + dev_warn_ratelimited(ep->dcp->dev, + "AFK[ep:%02x]: receive ring did not drain in one pass\n", + ep->endpoint); +} + static void afk_receive_message_worker(struct work_struct *work_) { struct afk_receive_message_work *work; @@ -1295,8 +1312,7 @@ static void afk_receive_message_worker(struct work_struct *work_) break; case RBEP_RECV: - while (afk_recv(work->ep)) - ; + afk_recv_drain(work->ep); break; default: diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index d06f1ad9d804a..dcfa4342e4bc8 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -272,6 +272,41 @@ static inline bool afk_ring_payload_fits(u32 ptr, u32 bytes, size_t bufsz) bytes <= bufsz - ptr - sizeof(struct afk_qe); } +/* + * Check the receive ring pointers before reading an entry. The consumer + * only advances rptr in whole blocks of @stride bytes and stops when it + * meets wptr, so the producer has to keep wptr block aligned and inside + * the ring as well; wptr == bufsz is never written, it wraps to 0. + */ +static inline bool afk_ring_ptrs_valid(u32 rptr, u32 wptr, size_t bufsz, + u32 stride) +{ + return stride && bufsz >= sizeof(struct afk_qe) && + rptr <= bufsz - sizeof(struct afk_qe) && wptr < bufsz && + !(rptr % stride) && !(wptr % stride); +} + +/* + * One RECV notification drains at most as many entries as the ring has + * blocks, so a producer that keeps moving wptr cannot hold the ordered + * receive worker. Entries written after this pass come with their own + * RECV notification. + */ +static inline unsigned int afk_ring_drain_budget(size_t bufsz, u32 stride) +{ + return stride ? bufsz / stride : 1; +} + +static inline unsigned int afk_ring_drain(bool (*recv)(void *ctx), void *ctx, + unsigned int budget) +{ + unsigned int n = 0; + + while (n < budget && recv(ctx)) + n++; + return n; +} + struct epic_hdr { u8 version; __le16 seq; diff --git a/drivers/gpu/drm/apple/afk_test.c b/drivers/gpu/drm/apple/afk_test.c index c3130086c78e1..0621eebe914df 100644 --- a/drivers/gpu/drm/apple/afk_test.c +++ b/drivers/gpu/drm/apple/afk_test.c @@ -49,7 +49,58 @@ static void afk_ring_payload_fits_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0, U32_MAX, 0x1000)); } +static void afk_ring_ptrs_test(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, afk_ring_ptrs_valid(0, 0, 0x1000, 0x40)); + KUNIT_EXPECT_TRUE(test, afk_ring_ptrs_valid(0xfc0, 0x40, 0x1000, 0x40)); + KUNIT_EXPECT_TRUE(test, afk_ring_ptrs_valid(0x80, 0xf80, 0x1000, 0x80)); + /* The write pointer must lie inside the ring. */ + KUNIT_EXPECT_FALSE(test, afk_ring_ptrs_valid(0, 0x1000, 0x1000, 0x40)); + KUNIT_EXPECT_FALSE(test, afk_ring_ptrs_valid(0, U32_MAX, 0x1000, 0x40)); + KUNIT_EXPECT_FALSE(test, afk_ring_ptrs_valid(0x1000, 0, 0x1000, 0x40)); + KUNIT_EXPECT_FALSE(test, afk_ring_ptrs_valid(U32_MAX, 0, 0x1000, 0x40)); + /* Both pointers must be block aligned. */ + KUNIT_EXPECT_FALSE(test, afk_ring_ptrs_valid(0, 0x20, 0x1000, 0x40)); + KUNIT_EXPECT_FALSE(test, afk_ring_ptrs_valid(0x10, 0, 0x1000, 0x40)); + KUNIT_EXPECT_FALSE(test, afk_ring_ptrs_valid(0x40, 0x80, 0x1000, 0x80)); + /* An uninitialised ring has no valid pointers. */ + KUNIT_EXPECT_FALSE(test, afk_ring_ptrs_valid(0, 0, 0x1000, 0)); + KUNIT_EXPECT_FALSE(test, afk_ring_ptrs_valid(0, 0, 0, 0x40)); +} + +struct afk_drain_ctx { + unsigned int calls, entries; +}; + +/* A producer that publishes @entries entries, or one that never stops. */ +static bool afk_drain_recv(void *data) +{ + struct afk_drain_ctx *ctx = data; + + return ++ctx->calls <= ctx->entries; +} + +static void afk_ring_drain_test(struct kunit *test) +{ + struct afk_drain_ctx ctx = { .entries = 3 }; + unsigned int budget = afk_ring_drain_budget(0x1000, 0x40); + + KUNIT_EXPECT_EQ(test, budget, 64U); + KUNIT_EXPECT_EQ(test, afk_ring_drain(afk_drain_recv, &ctx, budget), 3U); + KUNIT_EXPECT_EQ(test, ctx.calls, 4U); + + /* A write pointer that keeps moving ahead exhausts the budget. */ + ctx = (struct afk_drain_ctx){ .entries = UINT_MAX }; + KUNIT_EXPECT_EQ(test, afk_ring_drain(afk_drain_recv, &ctx, budget), budget); + KUNIT_EXPECT_EQ(test, ctx.calls, budget); + + /* Without a ring the single pass reports the missing setup. */ + KUNIT_EXPECT_EQ(test, afk_ring_drain_budget(0, 0), 1U); +} + static struct kunit_case afk_ring_cases[] = { + KUNIT_CASE(afk_ring_ptrs_test), + KUNIT_CASE(afk_ring_drain_test), KUNIT_CASE(afk_ring_stride_test), KUNIT_CASE(afk_ring_payload_fits_test), {} From fb3667db1b14290fa5386f1e12782b2a8a72d134 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:47:54 +1100 Subject: [PATCH 1390/2680] gpio: macsmc: tidy the J700 pcIO helpers Align the parameter list and the condition of macsmc_gpio_is_j700_pcio() with the opening parenthesis, and build the pcIO level bit without OR-ing in a logical negation, which sparse reports as dubious. No functional change. (cherry picked from commit d8c37da9a526312cd02937e2a34a6e80dbe095c8) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpio/gpio-macsmc.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpio/gpio-macsmc.c b/drivers/gpio/gpio-macsmc.c index 471bce2e54b4f..424f19b1c4a5d 100644 --- a/drivers/gpio/gpio-macsmc.c +++ b/drivers/gpio/gpio-macsmc.c @@ -99,11 +99,11 @@ struct macsmc_gpio { }; static bool macsmc_gpio_is_j700_pcio(struct macsmc_gpio *smcgp, - unsigned int offset) + unsigned int offset) { return smcgp->j700_pcio && - (offset == MACSMC_J700_AFE_RESET_GPIO || - offset == MACSMC_J700_RADIO_REG_ON_GPIO); + (offset == MACSMC_J700_AFE_RESET_GPIO || + offset == MACSMC_J700_RADIO_REG_ON_GPIO); } static int macsmc_gpio_nr(smc_key key) @@ -236,7 +236,7 @@ static int macsmc_gpio_set(struct gpio_chip *gc, unsigned int offset, int value) lo = MACSMC_J700_AFE_RESET_LO; hi = MACSMC_J700_AFE_RESET_HI; } - payload = ((u64)hi << 32) | lo | !!value; + payload = ((u64)hi << 32) | lo | (value ? 1 : 0); ret = apple_smc_write_u64(smcgp->smc, SMC_KEY(pcIO), payload); if (ret < 0) { From 9aea5f74be18b534f2d3153e134a385d81fb1ac1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:41:34 +1100 Subject: [PATCH 1391/2680] HID: dockchannel-hid: Test MTP message bounds with KUnit Exercise the receive handlers with short subheaders, oversized bodies, invalid interface numbers and mismatched acknowledgment identifiers. Check packet checksums and response truncation with canaries, and accept a valid descriptor. Include the tests in the driver to use the actual receive handlers without exporting private implementation details. The fixtures have no transport or HID device and cannot communicate with a coprocessor. (cherry picked from commit 98abcc5af1e8d634ae73c67dbb20d36141dea8ee) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/Kconfig | 10 + .../dockchannel-hid-parser-test.c | 204 ++++++++++++++++++ drivers/hid/dockchannel-hid/dockchannel-hid.c | 16 +- 3 files changed, 226 insertions(+), 4 deletions(-) create mode 100644 drivers/hid/dockchannel-hid/dockchannel-hid-parser-test.c diff --git a/drivers/hid/dockchannel-hid/Kconfig b/drivers/hid/dockchannel-hid/Kconfig index 5f1da27d445a8..600fc235b4790 100644 --- a/drivers/hid/dockchannel-hid/Kconfig +++ b/drivers/hid/dockchannel-hid/Kconfig @@ -27,3 +27,13 @@ config HID_DOCKCHANNEL_KUNIT_TEST If unsure, say N. endmenu + +config HID_DOCKCHANNEL_PARSER_KUNIT_TEST + bool "KUnit tests for DockChannel HID message parsing" if !KUNIT_ALL_TESTS + depends on HID_DOCKCHANNEL && KUNIT=y + default KUNIT_ALL_TESTS + help + Build tests that pass malformed MTP frames to the initialization, + input-report and acknowledgment handlers. Cover short headers, + oversized lengths, interface indices and acknowledgment identity. + The tests run without a DockChannel or HID device. diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid-parser-test.c b/drivers/hid/dockchannel-hid/dockchannel-hid-parser-test.c new file mode 100644 index 0000000000000..ef0e8869bd99f --- /dev/null +++ b/drivers/hid/dockchannel-hid/dockchannel-hid-parser-test.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Included by dockchannel-hid.c to exercise the MTP receive handlers. */ +#include +#include + +static int dchid_parser_test_init(struct kunit *test) +{ + struct dockchannel_hid *dchid; + struct dchid_iface *iface; + + dchid = kunit_kzalloc(test, sizeof(*dchid), GFP_KERNEL); + iface = kunit_kzalloc(test, sizeof(*iface), GFP_KERNEL); + if (!dchid || !iface) + return -ENOMEM; + dchid->dev = kunit_device_register(test, "dchid-parser"); + if (IS_ERR(dchid->dev)) + return PTR_ERR(dchid->dev); + iface->dchid = dchid; + iface->name = "test"; + iface->out_report = 0x10; + iface->out_flags = 0x80; + iface->tx_seq = 7; + init_completion(&iface->out_complete); + dchid->ifaces[1] = iface; + test->priv = iface; + return 0; +} + +static void dchid_short_ack_test(struct kunit *test) +{ + struct dchid_iface *iface = test->priv; + struct dchid_hdr hdr = {}; + + for (hdr.length = 0; hdr.length < sizeof(struct dchid_subhdr); hdr.length++) { + dchid_handle_ack(iface, &hdr, NULL); + KUNIT_EXPECT_FALSE(test, completion_done(&iface->out_complete)); + KUNIT_EXPECT_EQ(test, iface->out_report, 0x10); + } +} + +static void dchid_ack_length_test(struct kunit *test) +{ + struct dchid_iface *iface = test->priv; + struct dchid_hdr hdr = { .length = sizeof(struct dchid_subhdr), .seq = 7 }; + struct dchid_subhdr sub = { .flags = 0x80 }; + + /* A report ID is required even when no response body was requested. */ + dchid_handle_ack(iface, &hdr, &sub); + KUNIT_EXPECT_FALSE(test, completion_done(&iface->out_complete)); + sub.length = U16_MAX; + dchid_handle_ack(iface, &hdr, &sub); + KUNIT_EXPECT_FALSE(test, completion_done(&iface->out_complete)); + KUNIT_EXPECT_EQ(test, iface->out_report, 0x10); +} + +static void dchid_ack_identity_test(struct kunit *test) +{ + struct dchid_iface *iface = test->priv; + struct dchid_hdr hdr = { .length = sizeof(struct dchid_subhdr) + 1, .seq = 7 }; + struct { + struct dchid_subhdr sub; + u8 id; + } __packed ack = { .sub = { .flags = 0x80, .length = 1 }, .id = 0x11 }; + + dchid_handle_ack(iface, &hdr, &ack); + KUNIT_EXPECT_FALSE(test, completion_done(&iface->out_complete)); + ack.id = 0x10; + hdr.seq++; + dchid_handle_ack(iface, &hdr, &ack); + KUNIT_EXPECT_FALSE(test, completion_done(&iface->out_complete)); + hdr.seq--; + ack.sub.flags++; + dchid_handle_ack(iface, &hdr, &ack); + KUNIT_EXPECT_FALSE(test, completion_done(&iface->out_complete)); + ack.sub.flags--; + dchid_handle_ack(iface, &hdr, &ack); + KUNIT_EXPECT_TRUE(test, completion_done(&iface->out_complete)); + KUNIT_EXPECT_EQ(test, iface->out_report, -1); + KUNIT_EXPECT_EQ(test, iface->resp_size, 1U); +} + +static void dchid_ack_copy_test(struct kunit *test) +{ + struct dchid_iface *iface = test->priv; + struct dchid_hdr hdr = { .length = sizeof(struct dchid_subhdr) + 5, .seq = 7 }; + struct { + struct dchid_subhdr sub; + u8 data[5]; + } __packed ack = { .sub = { .flags = 0x80, .length = 5 }, + .data = { 0x10, 1, 2, 3, 4 } }; + u8 response[4] = { 0xa5, 0xa5, 0xa5, 0xa5 }; + + iface->resp_buf = response + 1; + iface->resp_size = 2; + dchid_handle_ack(iface, &hdr, &ack); + KUNIT_EXPECT_TRUE(test, completion_done(&iface->out_complete)); + KUNIT_EXPECT_EQ(test, iface->resp_size, 3U); + KUNIT_EXPECT_MEMEQ(test, response, ((u8[]){ 0xa5, 1, 2, 0xa5 }), sizeof(response)); +} + +static void dchid_init_bounds_test(struct kunit *test) +{ + struct dchid_iface *iface = test->priv; + struct dockchannel_hid *dchid = iface->dchid; + struct { + struct dchid_init_hdr hdr; + struct dchid_init_block_hdr block; + } __packed msg = { .hdr = { .iface = 1, .more_packets = 1 } }; + unsigned int len; + + for (len = 0; len < sizeof(msg.hdr); len++) + dchid_handle_init(dchid, NULL, len); + msg.hdr.iface = MAX_INTERFACES; + dchid_handle_init(dchid, &msg, sizeof(msg)); + msg.hdr.iface = U8_MAX; + dchid_handle_init(dchid, &msg, sizeof(msg)); + msg.hdr.iface = 1; + msg.block.type = INIT_HID_DESCRIPTOR; + msg.block.length = U16_MAX; + dchid_handle_init(dchid, &msg, sizeof(msg)); + KUNIT_EXPECT_PTR_EQ(test, iface->hid_desc, NULL); + msg.block.type = INIT_GPIO_REQUEST; + msg.block.length = 0; + dchid_handle_init(dchid, &msg, sizeof(msg)); + KUNIT_EXPECT_FALSE(test, iface->gpio_announced); + KUNIT_EXPECT_PTR_EQ(test, dchid->ifaces[1], iface); +} + +static void dchid_short_input_test(struct kunit *test) +{ + struct dchid_iface *iface = test->priv; + struct dchid_work *work; + unsigned int len; + + /* Any attempted HID delivery would dereference the absent HID device. */ + iface->open = true; + for (len = 0; len < sizeof(struct dchid_subhdr); len++) { + work = kzalloc(struct_size(work, data, len), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, work); + work->iface = iface; + work->hdr.iface = 1; + work->hdr.length = len; + dchid_packet_work(&work->work); + } + work = kzalloc(struct_size(work, data, sizeof(struct dchid_subhdr)), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, work); + work->iface = iface; + work->hdr.iface = 1; + work->hdr.length = sizeof(struct dchid_subhdr); + ((struct dchid_subhdr *)work->data)->length = U16_MAX; + dchid_packet_work(&work->work); +} + +static void dchid_init_descriptor_test(struct kunit *test) +{ + struct dchid_iface *iface = test->priv; + struct { + struct dchid_init_hdr hdr; + struct dchid_init_block_hdr block; + u8 descriptor[4]; + } __packed msg = { + .hdr = { .iface = 1, .more_packets = 1 }, + .block = { .type = INIT_HID_DESCRIPTOR, .length = 4 }, + .descriptor = { 0x05, 0x01, 0x09, 0x06 }, + }; + + dchid_handle_init(iface->dchid, &msg, sizeof(msg)); + KUNIT_ASSERT_NOT_NULL(test, iface->hid_desc); + KUNIT_EXPECT_EQ(test, iface->hid_desc_len, sizeof(msg.descriptor)); + KUNIT_EXPECT_MEMEQ(test, iface->hid_desc, msg.descriptor, sizeof(msg.descriptor)); +} + +static void dchid_packet_checksum_test(struct kunit *test) +{ + struct dchid_hdr hdr = { .hdr_len = sizeof(hdr), .length = 8 }; + u8 body[12] = {}; + u32 checksum; + + put_unaligned_le32(U32_MAX - dchid_checksum(&hdr, sizeof(hdr)), body + 8); + KUNIT_EXPECT_TRUE(test, dchid_verify_checksum(&hdr, body, &checksum)); + KUNIT_EXPECT_EQ(test, checksum, U32_MAX); + body[0] ^= 1; + KUNIT_EXPECT_FALSE(test, dchid_verify_checksum(&hdr, body, &checksum)); +} + +static struct kunit_case dchid_parser_cases[] = { + KUNIT_CASE(dchid_short_ack_test), + KUNIT_CASE(dchid_ack_length_test), + KUNIT_CASE(dchid_ack_identity_test), + KUNIT_CASE(dchid_ack_copy_test), + KUNIT_CASE(dchid_init_bounds_test), + KUNIT_CASE(dchid_short_input_test), + KUNIT_CASE(dchid_init_descriptor_test), + KUNIT_CASE(dchid_packet_checksum_test), + {} +}; + +static struct kunit_suite dchid_parser_suite = { + .name = "dockchannel-hid-parser", + .init = dchid_parser_test_init, + .test_cases = dchid_parser_cases, +}; + +kunit_test_suite(dchid_parser_suite); diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 4e8e1d611ad27..fbc8e4c849607 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -306,6 +306,13 @@ static u32 dchid_checksum(void *p, size_t length) return sum; } +static bool dchid_verify_checksum(struct dchid_hdr *hdr, void *data, u32 *checksum) +{ + *checksum = dchid_checksum(hdr, sizeof(*hdr)); + *checksum += dchid_checksum(data, hdr->length + sizeof(u32)); + return *checksum == U32_MAX; +} + static int dchid_send(struct dchid_iface *iface, u32 flags, void *msg, size_t size) { u32 checksum = 0xffffffff; @@ -1202,10 +1209,7 @@ static void dchid_handle_packet(void *cookie, size_t avail) goto done; } - checksum = dchid_checksum(&hdr, sizeof(hdr)); - checksum += dchid_checksum(dchid->pkt_buf, hdr.length + 4); - - if (checksum != 0xffffffff) { + if (!dchid_verify_checksum(&hdr, dchid->pkt_buf, &checksum)) { dev_err(dchid->dev, "Checksum mismatch (iface %d): 0x%08x != 0xffffffff\n", hdr.iface, checksum); goto done; @@ -1459,3 +1463,7 @@ module_platform_driver(dockchannel_hid_driver); MODULE_DESCRIPTION("Apple DockChannel HID transport driver"); MODULE_AUTHOR("Hector Martin "); MODULE_LICENSE("Dual MIT/GPL"); + +#if IS_ENABLED(CONFIG_HID_DOCKCHANNEL_PARSER_KUNIT_TEST) +#include "dockchannel-hid-parser-test.c" +#endif From 1c7e9a253ca62b5851e9a5c84da955bbbf23ec1a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:06:49 +1100 Subject: [PATCH 1392/2680] mfd: macsmc: Reject out-of-range key indices The GPIO key-range lookup asks for key_count - 1. A malformed zero key count therefore reaches this interface as -1 and is converted to an unsigned mailbox argument. Indices at or above the advertised key count can likewise be sent to the coprocessor. Validate the index against the key count before issuing the command. Fixes: e038d985c982 ("mfd: Add Apple Silicon System Management Controller") Cc: stable@vger.kernel.org (cherry picked from commit 9356b9795c74be3599933b78f417ff2454ecd12d) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/mfd/macsmc.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 92f1b987bf888..5dd8f8bc21956 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -206,6 +206,9 @@ int apple_smc_get_key_by_index(struct apple_smc *smc, int index, smc_key *key) u32 data; int ret; + if (index < 0 || index >= smc->key_count) + return -EINVAL; + ret = apple_smc_cmd(smc, SMC_MSG_GET_KEY_BY_INDEX, index, 0, 0, &data); if (ret < 0) return ret; From d5cae8f2a4bfc2b735b8498ba1582aa73faeb4a1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:30:04 +1100 Subject: [PATCH 1393/2680] drm: apple: parser: cap the number of timing modes enumerate_modes() allocates one struct dcp_display_mode (320 bytes) for every element of the firmware's TimingElements array. The previous fix only rejects counts the blob cannot hold, so a 1 MiB property transfer of empty dictionaries can still request about 80 MiB. The published catalog is deliberately never freed while mode readers may hold pointers into it (see dcp_modes_snapshot()), so repeated deliveries retain each oversized array. Reject catalogs with more than DCP_MAX_TIMING_MODES (1024) elements before allocating. DCP reports one element per timing it derived from the sink's EDID and DisplayID; every catalog in the J700 boot and hotplug logs collected so far has at most 14 entries (1, 6 or 14 are the only non-zero counts seen). 1024 is far above anything an EDID with extension blocks and DisplayID can describe in practice, and bounds the array at 320 KiB. The retention itself is left as it is. Add KUnit cases for a catalog at the maximum, which is accepted, and one element above it, which is rejected. Fixes: 38c14f6107df ("drm: apple: support ProMotion variable refresh rate") (cherry picked from commit 0661976c0102c35fb2ee281c9215550e73ecd383) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/parser.c | 8 +++++-- drivers/gpu/drm/apple/parser.h | 8 +++++++ drivers/gpu/drm/apple/parser_test.c | 36 +++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index b67c677598d4a..a7a7061647aea 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -1040,11 +1040,15 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, /* * Start with a worst case allocation. The count comes from firmware: - * every entry needs at least a tag in what is left of the blob, and a - * catalog too large to allocate fails like any other malformed one. + * every entry needs at least a tag in what is left of the blob, and no + * display has more timings than DCP_MAX_TIMING_MODES. The catalog is + * retained once published, so reject anything larger before + * allocating for it. */ if (it.len > (handle->len - handle->pos) / sizeof(struct dcp_parse_tag)) return ERR_PTR(-EINVAL); + if (it.len > DCP_MAX_TIMING_MODES) + return ERR_PTR(-E2BIG); modes = kcalloc(it.len, sizeof(*modes), GFP_KERNEL | __GFP_NOWARN); *count = 0; diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 98e3fbe4a1ded..5a2a84614c536 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -119,6 +119,14 @@ struct dimension { s64 pixel_repetition; }; +/* + * Upper bound on the TimingElements catalog. DCP reports one element per + * timing it derived from the sink's EDID and DisplayID, and every catalog + * logged on J700 so far holds at most 14. The cap leaves ample headroom + * while keeping the retained array (320 bytes per entry) below 1 MiB. + */ +#define DCP_MAX_TIMING_MODES 1024 + int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx); bool dcp_property_complete(const void *blob, size_t received, size_t advertised); #if IS_ENABLED(CONFIG_KUNIT) diff --git a/drivers/gpu/drm/apple/parser_test.c b/drivers/gpu/drm/apple/parser_test.c index 6795fcdf18864..7fc03175a2324 100644 --- a/drivers/gpu/drm/apple/parser_test.c +++ b/drivers/gpu/drm/apple/parser_test.c @@ -215,6 +215,41 @@ static void parser_mode_count_test(struct kunit *test) kfree(modes); } +/* Build an array of @count empty dictionaries in a test allocation. */ +static struct dcp_display_mode *parse_empty_modes(struct kunit *test, + unsigned int count, + unsigned int *found) +{ + size_t len = 8 + 4 * count; + struct dcp_parse_ctx ctx; + unsigned int i; + u8 *data; + + data = kunit_kzalloc(test, len, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, data); + put_unaligned_le32(0xd3, data); + put_unaligned_le32(count | T_ARRAY << 24, data + 4); + for (i = 0; i < count; i++) + put_unaligned_le32(T_DICT << 24, data + 8 + 4 * i); + + KUNIT_ASSERT_EQ(test, parse(data, len, &ctx), 0); + return enumerate_modes(&ctx, found, 0, 0, 0, false, false); +} + +static void parser_mode_cap_test(struct kunit *test) +{ + struct dcp_display_mode *modes; + unsigned int count = 1; + + modes = parse_empty_modes(test, DCP_MAX_TIMING_MODES, &count); + KUNIT_ASSERT_FALSE(test, IS_ERR(modes)); + KUNIT_EXPECT_EQ(test, count, 0); + kfree(modes); + + modes = parse_empty_modes(test, DCP_MAX_TIMING_MODES + 1, &count); + KUNIT_EXPECT_PTR_EQ(test, modes, ERR_PTR(-E2BIG)); +} + static void parser_property_complete_test(struct kunit *test) { struct blob b; @@ -240,6 +275,7 @@ static struct kunit_case parser_cases[] = { KUNIT_CASE(parser_nesting_test), KUNIT_CASE(parser_bad_header_test), KUNIT_CASE(parser_mode_count_test), + KUNIT_CASE(parser_mode_cap_test), KUNIT_CASE(parser_property_complete_test), {} }; From 08db05202b3be00fd79f1da126d16b0124a0a8aa Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:36:03 +1100 Subject: [PATCH 1394/2680] iommu: apple-dart: Invalidate every DART even if a TLB sync step fails apple_dart_domain_flush_tlb_range() publishes the host page table into locked roots and then invalidates the TLB of each DART of the domain. If publishing fails for a locked stream, it skips the invalidation of that DART, and if powering a DART or publishing fails it returns before looking at the remaining DARTs of the domain. This function also backs iotlb_sync(). After an unmap the DMA layer frees the pages as soon as it returns, so a skipped invalidation leaves translations to freed memory in the TLB, and the device can then read or write pages that have been reused. Publishing fails, for example, when a firmware-owned root slot changed under a locked stream, which is independent of the unmap that has to be flushed. Always invalidate a DART that could be powered, whatever the outcome of publishing, go on with the remaining DARTs, and return the first error. The map path still sees the error and fails the mapping. Leaving firmware-owned root slots out of range invalidates is deliberate, see commit dd5ac6fdffb0 ("iommu: apple-dart: leave live firmware translations out of invalidates"), and still holds: a failed publication does not change which slots apple_dart_fw_invalidate() skips, it only no longer suppresses the flush of the others. Neither the skip nor the early return was explained when it was added. Add a KUnit case with a two-DART domain whose locked stream fails to publish, and check that both DARTs receive a TLB command. Fixes: 79379798f12f ("iommu/apple-dart: Support T8140 DARTs and firmware-owned stream roots") (cherry picked from commit 0faa62fa8908a8d0e5491160466034030037698b) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart-test.c | 71 +++++++++++++++++++++++++++++++++ drivers/iommu/apple-dart.c | 42 ++++++++++--------- 2 files changed, 95 insertions(+), 18 deletions(-) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index cd8686502c58b..e0fbd3218146a 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -168,6 +168,76 @@ static void apple_dart_test_firmware_roots(struct kunit *test) KUNIT_EXPECT_EQ(test, live[1], 0x4001ULL); } +static struct apple_dart *apple_dart_test_t8110(struct kunit *test, const char *name) +{ + struct apple_dart *dart; + + dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart); + dart->dev = kunit_device_register(test, name); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dart->dev); + dart->regs = (__force void __iomem *)kunit_kzalloc(test, SZ_16K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart->regs); + dart->hw = &apple_dart_hw_t8110; + dart->version = 0x0202; + dart->num_streams = 1; + spin_lock_init(&dart->lock); + pm_runtime_set_active(dart->dev); + pm_runtime_enable(dart->dev); + return dart; +} + +static void apple_dart_test_unmap_flush_after_publish_failure(struct kunit *test) +{ + struct iommu_iotlb_gather gather = { .start = 0, .end = SZ_16K - 1 }; + struct apple_dart_stream_map stream = {}; + struct apple_dart_domain *domain; + struct apple_dart *locked, *other; + struct io_pgtable *iop; + u64 live[] = { 0x1001, 0, 0, 0 }; + u64 owned[ARRAY_SIZE(live)] = {}; + u64 ours[] = { 0x3001, 0, 0, 0 }; + u32 sentinel = 0x40; + + /* A locked stream whose root slot firmware holds: publishing fails. */ + locked = apple_dart_test_t8110(test, "apple-dart-flush-locked"); + locked->locked = true; + locked->pgsize = sizeof(live); + locked->locked_ttbr[0][0] = live; + locked->locked_owned[0][0] = owned; + other = apple_dart_test_t8110(test, "apple-dart-flush-other"); + other->pgsize = SZ_16K; + + iop = kunit_kzalloc(test, sizeof(*iop), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, iop); + iop->cfg.apple_dart_cfg.n_ttbrs = 1; + iop->cfg.apple_dart_cfg.ttbr[0] = ours; + domain = kunit_kzalloc(test, sizeof(*domain), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, domain); + domain->pgtbl_ops = &iop->ops; + domain->stream_maps[0].dart = locked; + atomic_long_set(&domain->stream_maps[0].sidmap[0], 1); + domain->stream_maps[1].dart = other; + atomic_long_set(&domain->stream_maps[1].sidmap[0], 1); + + writel(sentinel, locked->regs + DART_T8110_TLB_CMD); + writel(sentinel, other->regs + DART_T8110_TLB_CMD); + stream.dart = locked; + __set_bit(0, stream.sidmap); + KUNIT_ASSERT_EQ(test, apple_dart_hw_sync_locked(&iop->cfg, &stream, true), + -EBUSY); + + /* The pages are freed after this: both DARTs must drop them. */ + apple_dart_iotlb_sync(&domain->domain, &gather); + KUNIT_EXPECT_NE(test, readl(locked->regs + DART_T8110_TLB_CMD), sentinel); + KUNIT_EXPECT_NE(test, readl(other->regs + DART_T8110_TLB_CMD), sentinel); + KUNIT_EXPECT_EQ(test, live[0], 0x1001ULL); + KUNIT_EXPECT_EQ(test, atomic_read(&locked->dev->power.usage_count), 0); + KUNIT_EXPECT_EQ(test, atomic_read(&other->dev->power.usage_count), 0); + pm_runtime_disable(locked->dev); + pm_runtime_disable(other->dev); +} + static void apple_dart_test_gated_commands(struct kunit *test) { struct apple_dart_stream_map stream = {}; @@ -517,6 +587,7 @@ static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_locked_handoff), KUNIT_CASE(apple_dart_test_sync_unknown_roots), KUNIT_CASE(apple_dart_test_firmware_roots), + KUNIT_CASE(apple_dart_test_unmap_flush_after_publish_failure), KUNIT_CASE(apple_dart_test_gated_commands), KUNIT_CASE(apple_dart_test_resume_commands_failure), KUNIT_CASE(apple_dart_test_tunnel_restore), diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 397c00ecb4efe..7543d35c4105a 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -1343,8 +1343,15 @@ static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, struct apple_dart_atomic_stream_map *map; struct apple_dart_stream_map stream; struct io_pgtable_cfg *cfg = &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; - int i, j, ret; + const u32 op = DART_T8110_TLB_CMD_OP_FLUSH_SID; + int i, j, ret, err = 0; + u64 start, end; + /* + * After an unmap, the caller frees the pages once this returns. Every + * DART that can be reached must therefore be invalidated, even if + * publishing the page table or powering another DART failed. + */ for_each_stream_map(i, domain, map) { stream.dart = map->dart; for (j = 0; j < BITS_TO_LONGS(stream.dart->num_streams); j++) @@ -1355,31 +1362,30 @@ static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, if (ret < 0) { dev_err_ratelimited(stream.dart->dev, "failed to power DART for TLB synchronization: %d\n", ret); - return ret; + err = err ?: ret; + continue; } ret = stream.dart->locked ? apple_dart_hw_sync_locked(cfg, &stream, true) : 0; - if (ret) + if (ret) { dev_err_ratelimited(stream.dart->dev, "failed to publish host page-table entries: %d\n", ret); - else { - /* The range registers take the DVA the device issues. */ - if (range && stream.dart->fw_mirror && stream.dart->version >= 0x0202) - ret = apple_dart_fw_invalidate(&stream, first + domain->dma_offset, - last + domain->dma_offset); - else if (range && stream.dart->locked && stream.dart->version >= 0x0202) - ret = apple_dart_t8110_hw_tlb_command_range(&stream, - DART_T8110_TLB_CMD_OP_FLUSH_SID, true, - first + domain->dma_offset, - last + domain->dma_offset); - else - ret = stream.dart->hw->invalidate_tlb(&stream); + err = err ?: ret; } + /* The range registers take the DVA the device issues. */ + start = first + domain->dma_offset; + end = last + domain->dma_offset; + if (range && stream.dart->fw_mirror && stream.dart->version >= 0x0202) + ret = apple_dart_fw_invalidate(&stream, start, end); + else if (range && stream.dart->locked && stream.dart->version >= 0x0202) + ret = apple_dart_t8110_hw_tlb_command_range(&stream, op, true, + start, end); + else + ret = stream.dart->hw->invalidate_tlb(&stream); pm_runtime_put(stream.dart->dev); - if (ret) - return ret; + err = err ?: ret; } - return 0; + return err; } static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain) From 7812c6b0a802642ed4f17c92d2083fe60728d3c4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:30:07 +1100 Subject: [PATCH 1395/2680] soc: apple: rtkit: Only parse an initial crash log in inherited sessions The first message on the crashlog endpoint is normally the buffer request, and type 1 is used both for that request and for a crash report. Since commit c07d044a0311 ("soc: apple: rtkit: recognise an initial crash log in adopted sessions") the handler reads the header of a freshly set up buffer and reports a crash when it finds a valid crash log there. That check is only needed for a session inherited from the bootloader, which has already negotiated its buffer and can only be reporting a crash. For a session that went through the handshake it is wrong: a co-processor that places its own buffers (DCP, the T8140 GPU) hands over memory that is not cleared by Linux. If that memory still holds the log of an earlier crash, for example after apple_rtkit_reinit() recovered from one, the buffer request is reported as a new crash, the instance is marked crashed and the client starts recovery again. Return after setting up the buffer unless the session is inherited, as the handler did before the inherited case was added. That commit relied on a fresh session's buffer being zeroed, which holds for Linux's own coherent allocations but not for firmware-placed ones. The inherited-session handling it added is kept unchanged. Fixes: c07d044a0311 ("soc: apple: rtkit: recognise an initial crash log in adopted sessions") (cherry picked from commit 39aced01ad973b68b6c593a5e4ad353ef45c2a62) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index f394ea8402ee8..43e1cbdd117f5 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -422,6 +422,14 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) if (apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer, APPLE_RTKIT_EP_CRASHLOG, msg)) goto unmapped; + /* + * In a session that went through the handshake, the first + * message on this endpoint is the buffer request. Firmware may + * place that buffer in memory which still holds the log of an + * earlier crash, so only look for one in an inherited session. + */ + if (!rtk->crashlog_inherited) + return; if (rtk->crashlog_buffer.size < sizeof(header)) goto unmapped; From 32cffd82c9a02af4bf1754da2027a8f8f88c1534 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:44:44 +1100 Subject: [PATCH 1396/2680] HID: dockchannel-hid: Reject unaligned packet lengths before checksumming The receive checksum sums whole words and warns about any trailing bytes. A peer can provide an unaligned packet length and a checksum that matches the truncated sum. Such a packet both triggers a warning and passes the checksum test despite bytes being left unchecked. Reject lengths that are not word aligned before computing the checksum. The packet body has already been consumed, so this keeps the next packet at its original boundary. Test all three unaligned lengths using packets whose truncated checksums would otherwise match. (cherry picked from commit cce3ac8ff460e55eb2805f353ea2c223bae308ca) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../dockchannel-hid/dockchannel-hid-parser-test.c | 14 ++++++++++++++ drivers/hid/dockchannel-hid/dockchannel-hid.c | 5 +++++ 2 files changed, 19 insertions(+) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid-parser-test.c b/drivers/hid/dockchannel-hid/dockchannel-hid-parser-test.c index ef0e8869bd99f..56efa764aaec6 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid-parser-test.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid-parser-test.c @@ -183,7 +183,21 @@ static void dchid_packet_checksum_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, dchid_verify_checksum(&hdr, body, &checksum)); } +static void dchid_unaligned_checksum_test(struct kunit *test) +{ + struct dchid_hdr hdr = { .hdr_len = sizeof(hdr) }; + u8 body[16] = {}; + u32 checksum; + + for (hdr.length = 9; hdr.length < 12; hdr.length++) { + put_unaligned_le32(U32_MAX - dchid_checksum(&hdr, sizeof(hdr)), body + 8); + memset(body + 12, 0xa5, 4); + KUNIT_EXPECT_FALSE(test, dchid_verify_checksum(&hdr, body, &checksum)); + } +} + static struct kunit_case dchid_parser_cases[] = { + KUNIT_CASE(dchid_unaligned_checksum_test), KUNIT_CASE(dchid_short_ack_test), KUNIT_CASE(dchid_ack_length_test), KUNIT_CASE(dchid_ack_identity_test), diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index fbc8e4c849607..e47de457f7f04 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -308,6 +308,11 @@ static u32 dchid_checksum(void *p, size_t length) static bool dchid_verify_checksum(struct dchid_hdr *hdr, void *data, u32 *checksum) { + if (!IS_ALIGNED(hdr->length, sizeof(u32))) { + *checksum = 0; + return false; + } + *checksum = dchid_checksum(hdr, sizeof(*hdr)); *checksum += dchid_checksum(data, hdr->length + sizeof(u32)); return *checksum == U32_MAX; From 95d0b4ba8ed623d92e351272695c225f3cae96e7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:06:59 +1100 Subject: [PATCH 1397/2680] mfd: macsmc: Hold the mutex while decoding key metadata apple_smc_cmd() releases the command mutex before key metadata is copied from shared SRAM. Another key transaction can overwrite that SRAM between the reply and the copy, returning metadata for a different key. Hold the mutex through the command and metadata copy. Fixes: e038d985c982 ("mfd: Add Apple Silicon System Management Controller") Cc: stable@vger.kernel.org (cherry picked from commit 04eb4b4433a6d3e2518939c6f89656fd75007e9a) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/mfd/macsmc.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 5dd8f8bc21956..cf25327a7bd8b 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -223,7 +223,9 @@ int apple_smc_get_key_info(struct apple_smc *smc, smc_key key, struct apple_smc_ u8 key_info[6]; int ret; - ret = apple_smc_cmd(smc, SMC_MSG_GET_KEY_INFO, key, 0, 0, NULL); + guard(mutex)(&smc->mutex); + + ret = apple_smc_cmd_locked(smc, SMC_MSG_GET_KEY_INFO, key, 0, 0, NULL); if (ret >= 0 && info) { memcpy_fromio(key_info, smc->shmem.iomem, sizeof(key_info)); info->size = key_info[0]; From adde6eaeeba4832423becfec4c5b5e74b20ddfe7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:41:32 +1100 Subject: [PATCH 1398/2680] drm/apple: Test firmware message bounds with KUnit Exercise AFKv2 frame and command lengths, owned DMA spans, callback capacities and reply correlation, plus short EPIC announcements and replies and invalid command slots. Keep the existing ring payload predicate in a helper so it can be tested without a live coprocessor. Test the H17P IOMFB receive dispatcher with short records, invalid contexts, excessive nesting, malformed tags and overflowing lengths. Include a valid callback to check that accepted messages still reach their handler. [ryan: carried on top of the j700/harden-all AFK commits, which already add a wrap-safe afk_ring_payload_fits(); this commit's own version of that helper is dropped, as is commit 8d73dcf37979 ("drm/apple: Avoid overflow when checking AFK receive payloads") (superseded by those commits)] (cherry picked from commit 458ec5f7f4b7ed0ffccbb9bda4888e54386e16f7) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Kconfig | 10 ++ drivers/gpu/drm/apple/Makefile | 2 + drivers/gpu/drm/apple/afk-parser-test.c | 106 ++++++++++++ drivers/gpu/drm/apple/afk.c | 4 + drivers/gpu/drm/apple/afkv2-test.c | 195 ++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb-parser-test.c | 154 +++++++++++++++++ drivers/gpu/drm/apple/iomfb.c | 4 + 7 files changed, 475 insertions(+) create mode 100644 drivers/gpu/drm/apple/afk-parser-test.c create mode 100644 drivers/gpu/drm/apple/afkv2-test.c create mode 100644 drivers/gpu/drm/apple/iomfb-parser-test.c diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 9c1ce96bc0dbb..2c2d64c364420 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -43,3 +43,13 @@ config DRM_APPLE_KUNIT_TEST This option is intended for developer test kernels. Also test the Type-C modeset, hotplug and service ownership state, and the system resume of a CRTC whose mode is no longer admissible. + +config DRM_APPLE_PARSER_KUNIT_TEST + bool "KUnit tests for Apple firmware message parsers" if !KUNIT_ALL_TESTS + depends on DRM_APPLE && KUNIT=y + default KUNIT_ALL_TESTS + help + Build tests for AFKv2 lengths, DMA ownership and reply correlation, + legacy EPIC replies, and H17P IOMFB callback envelopes and sizes. + The tests use allocated buffers and fake devices and do not require + an Apple display controller or any firmware. diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index b0e3d4cb127c1..66d11e865ec6c 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -26,3 +26,5 @@ obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk_test.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_backlight_test.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk-service-test.o + +obj-$(CONFIG_DRM_APPLE_PARSER_KUNIT_TEST) += afkv2-test.o diff --git a/drivers/gpu/drm/apple/afk-parser-test.c b/drivers/gpu/drm/apple/afk-parser-test.c new file mode 100644 index 0000000000000..fd8c5cb0c26e5 --- /dev/null +++ b/drivers/gpu/drm/apple/afk-parser-test.c @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Included by afk.c to exercise the EPIC receive handlers. */ +#include +#include + +static int afk_parser_test_init(struct kunit *test) +{ + struct apple_dcp_afkep *ep; + struct apple_dcp *dcp; + + ep = kunit_kzalloc(test, sizeof(*ep), GFP_KERNEL); + dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + if (!ep || !dcp) + return -ENOMEM; + dcp->dev = kunit_device_register(test, "afk-parser"); + if (IS_ERR(dcp->dev)) + return PTR_ERR(dcp->dev); + dcp->fw_compat = DCP_FIRMWARE_H17P; + ep->dcp = dcp; + ep->num_channels = 1; + ep->services[0].channel = 1; + ep->services[0].enabled = true; + spin_lock_init(&ep->services[0].lock); + test->priv = ep; + return 0; +} + +static void afk_compact_header_test(struct kunit *test) +{ + u8 record[sizeof(struct epic_hdr) + sizeof(struct epic_sub_hdr_h17p)] = {}; + unsigned int len; + + for (len = 0; len < sizeof(record); len++) + KUNIT_EXPECT_FALSE(test, afk_validate_h17p_header(NULL, len)); + put_unaligned_le32(sizeof(record) - 8, record + 4); + KUNIT_EXPECT_TRUE(test, afk_validate_h17p_header(record, sizeof(record))); + KUNIT_EXPECT_FALSE(test, afk_validate_h17p_header(record, sizeof(record) + 1)); + put_unaligned_le32(U32_MAX, record + 4); + KUNIT_EXPECT_FALSE(test, afk_validate_h17p_header(record, sizeof(record))); +} + +static void afk_short_epic_test(struct kunit *test) +{ + struct apple_dcp_afkep *ep = test->priv; + unsigned int len; + + for (len = 0; len < sizeof(struct epic_hdr) + sizeof(struct epic_sub_hdr_h17p); len++) + afk_recv_handle(ep, 0, EPIC_TYPE_NOTIFY, NULL, len); + for (len = 0; len < 32; len++) + afk_recv_handle_init(ep, 2, NULL, len); + KUNIT_EXPECT_EQ(test, ep->num_channels, 1U); +} + +static void afk_short_reply_test(struct kunit *test) +{ + struct apple_dcp_afkep *ep = test->priv; + struct epic_cmd_info *pending = &ep->services[0].cmds[0]; + unsigned int len; + + set_bit(0, ep->services[0].cmd_map); + pending->tag = 0x200; + for (len = 0; len < sizeof(struct epic_cmd); len++) { + afk_recv_handle_reply(ep, 1, 0x200, NULL, len); + KUNIT_EXPECT_FALSE(test, pending->done); + } +} + +static void afk_reply_index_test(struct kunit *test) +{ + struct apple_dcp_afkep *ep = test->priv; + struct epic_cmd_info *pending = &ep->services[0].cmds[0]; + struct epic_cmd reply = { .retcode = cpu_to_le32(7) }; + + set_bit(0, ep->services[0].cmd_map); + pending->tag = 0x200; + afk_recv_handle_reply(ep, 1, MAX_PENDING_CMDS, &reply, sizeof(reply)); + KUNIT_EXPECT_FALSE(test, pending->done); + afk_recv_handle_reply(ep, 1, 0x2ff, &reply, sizeof(reply)); + KUNIT_EXPECT_FALSE(test, pending->done); + afk_recv_handle_reply(ep, 1, 0x300, &reply, sizeof(reply)); + KUNIT_EXPECT_FALSE(test, pending->done); + afk_recv_handle_reply(ep, 2, 0x200, NULL, sizeof(reply)); + KUNIT_EXPECT_FALSE(test, pending->done); + afk_recv_handle_reply(ep, 1, 0x200, &reply, sizeof(reply)); + KUNIT_EXPECT_TRUE(test, pending->done); + KUNIT_EXPECT_EQ(test, pending->retcode, 7U); + reply.retcode = cpu_to_le32(9); + afk_recv_handle_reply(ep, 1, 0x200, &reply, sizeof(reply)); + KUNIT_EXPECT_EQ(test, pending->retcode, 7U); +} + +static struct kunit_case afk_parser_cases[] = { + KUNIT_CASE(afk_compact_header_test), + KUNIT_CASE(afk_short_epic_test), + KUNIT_CASE(afk_short_reply_test), + KUNIT_CASE(afk_reply_index_test), + {} +}; + +static struct kunit_suite afk_parser_suite = { + .name = "apple-epic-parser", + .init = afk_parser_test_init, + .test_cases = afk_parser_cases, +}; + +kunit_test_suite(afk_parser_suite); diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 3615e6b5460f4..5f69e8ad80f29 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -2086,3 +2086,7 @@ static void afk_remove_service_debugfs(struct apple_epic_service *srv) } #endif + +#if IS_ENABLED(CONFIG_DRM_APPLE_PARSER_KUNIT_TEST) +#include "afk-parser-test.c" +#endif diff --git a/drivers/gpu/drm/apple/afkv2-test.c b/drivers/gpu/drm/apple/afkv2-test.c new file mode 100644 index 0000000000000..646e0fd3c5471 --- /dev/null +++ b/drivers/gpu/drm/apple/afkv2-test.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "afk.h" +#include "afkv2-codec.h" + +static void afkv2_frame_bounds_test(struct kunit *test) +{ + u8 record[AFKV2_HEADER_SIZE + 8] = {}; + struct afkv2_frame frame; + size_t size; + + for (size = 0; size < AFKV2_HEADER_SIZE; size++) + KUNIT_EXPECT_EQ(test, afkv2_decode_frame(record, size, false, &frame), + -EINVAL); + KUNIT_EXPECT_EQ(test, afkv2_decode_frame(NULL, sizeof(record), false, &frame), + -EINVAL); + KUNIT_EXPECT_EQ(test, afkv2_decode_frame(record, SIZE_MAX, false, &frame), + -E2BIG); + afkv2_put32(record + 4, U32_MAX); + KUNIT_EXPECT_EQ(test, afkv2_decode_frame(record, sizeof(record), false, &frame), + -EINVAL); + afkv2_put32(record + 4, 15); + KUNIT_EXPECT_EQ(test, afkv2_decode_frame(record, sizeof(record), true, &frame), + -EINVAL); + afkv2_put32(record + 4, 16); + KUNIT_EXPECT_EQ(test, afkv2_decode_frame(record, sizeof(record), false, &frame), + -EINVAL); + KUNIT_ASSERT_EQ(test, afkv2_decode_frame(record, sizeof(record), true, &frame), 0); + KUNIT_EXPECT_EQ(test, frame.payload_size, 0U); + KUNIT_EXPECT_EQ(test, frame.padding_size, 8U); + afkv2_put32(record + 4, sizeof(record) - 8); + KUNIT_ASSERT_EQ(test, afkv2_decode_frame(record, sizeof(record), false, &frame), 0); + KUNIT_EXPECT_EQ(test, frame.payload_size, 8U); + KUNIT_EXPECT_PTR_EQ(test, frame.payload, record + AFKV2_HEADER_SIZE); + record[17] = AFKV2_RESPONSE + 1; + KUNIT_EXPECT_EQ(test, afkv2_decode_frame(record, sizeof(record), false, &frame), + -EPROTO); +} + +static void afkv2_command_bounds_test(struct kunit *test) +{ + u8 payload[AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE] = { 1 }; + struct afkv2_command command; + size_t size; + + for (size = 0; size < sizeof(payload); size++) + KUNIT_EXPECT_EQ(test, afkv2_decode_command(payload, size, &command), -EINVAL); + KUNIT_ASSERT_EQ(test, afkv2_decode_command(payload, sizeof(payload), &command), 0); + afkv2_put64(payload + 8, U64_MAX); + afkv2_put32(payload + 24, 2); + KUNIT_EXPECT_EQ(test, afkv2_decode_command(payload, sizeof(payload), &command), + -EINVAL); + afkv2_put32(payload + 24, 1); + KUNIT_EXPECT_EQ(test, afkv2_decode_command(payload, sizeof(payload), &command), 0); + afkv2_put64(payload + 8, 0); + KUNIT_EXPECT_EQ(test, afkv2_decode_command(payload, sizeof(payload), &command), + -EINVAL); + afkv2_put64(payload + 8, 0x1000); + afkv2_put32(payload + 24, AFKV2_MAX_PAYLOAD + 1); + KUNIT_EXPECT_EQ(test, afkv2_decode_command(payload, sizeof(payload), &command), + -EINVAL); + payload[0] = 2; + KUNIT_EXPECT_EQ(test, afkv2_decode_command(payload, sizeof(payload), &command), + -EINVAL); + payload[0] = 0; + payload[2] = 1; + KUNIT_EXPECT_EQ(test, afkv2_decode_command(payload, sizeof(payload), &command), + -EINVAL); +} + +static void afkv2_owned_span_test(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, afkv2_owned_span(0x1000, 0x100, 0x1000, 0x100)); + KUNIT_EXPECT_TRUE(test, afkv2_owned_span(0x1000, 0x100, 0x10ff, 1)); + KUNIT_EXPECT_FALSE(test, afkv2_owned_span(0x1000, 0x100, 0xfff, 1)); + KUNIT_EXPECT_FALSE(test, afkv2_owned_span(0x1000, 0x100, 0x10ff, 2)); + KUNIT_EXPECT_FALSE(test, afkv2_owned_span(0x1000, 0x100, 0x1000, SIZE_MAX)); + KUNIT_EXPECT_FALSE(test, afkv2_owned_span(U64_MAX, 2, U64_MAX, 1)); + KUNIT_EXPECT_TRUE(test, afkv2_owned_span(U64_MAX, 1, U64_MAX, 1)); +} + +static void afkv2_service_reply_bounds_test(struct kunit *test) +{ + u8 reply[68] = {}; + size_t size = SIZE_MAX; + unsigned int len; + + afkv2_put16(reply + 2, 3); + afkv2_put32(reply + 4, 8); + afkv2_put32(reply + 8, 4); + afkv2_put32(reply + 12, 0x69706378); + for (len = 0; len < 64; len++) + KUNIT_EXPECT_EQ(test, afkv2_service_reply_size(reply, len, 3, 8, 4, &size), + -EPROTO); + KUNIT_ASSERT_EQ(test, afkv2_service_reply_size(reply, sizeof(reply), 3, 8, 4, + &size), 0); + KUNIT_EXPECT_EQ(test, size, 4U); + KUNIT_EXPECT_EQ(test, afkv2_service_reply_size(reply, sizeof(reply), 3, 8, 3, + &size), -EMSGSIZE); + afkv2_put32(reply + 8, U32_MAX); + KUNIT_EXPECT_EQ(test, afkv2_service_reply_size(reply, sizeof(reply), 3, 8, + SIZE_MAX, &size), -EMSGSIZE); +} + +static void afkv2_response_bounds_test(struct kunit *test) +{ + struct afkv2_frame frame = { .category = AFKV2_RESPONSE, .interface = 4, .type = 9 }; + struct afkv2_command command = { + .id = 0xf1, .oob = true, .rx_address = 0x1000, .rx_size = 64, + .tx_address = 0x2000, .tx_size = 32, + }; + + KUNIT_EXPECT_EQ(test, afkv2_check_response(&frame, &command, 4, 9, 0xf1, + 0x1000, 64, 0x2000, 32), 0); + command.rx_size++; + KUNIT_EXPECT_EQ(test, afkv2_check_response(&frame, &command, 4, 9, 0xf1, + 0x1000, 64, 0x2000, 32), -EPROTO); + command.rx_size--; + command.id = 0x01; + KUNIT_EXPECT_EQ(test, afkv2_check_response(&frame, &command, 4, 9, 0xf1, + 0x1000, 64, 0x2000, 32), -EPROTO); + command.id = 0xf1; + command.rx_address++; + KUNIT_EXPECT_EQ(test, afkv2_check_response(&frame, &command, 4, 9, 0xf1, + 0x1000, 64, 0x2000, 32), -EPROTO); +} + +static void afkv2_callback_bounds_test(struct kunit *test) +{ + struct afkv2_command command = { + .oob = true, .rx_size = 64, .tx_size = 32, .value = 64, + }; + size_t request = SIZE_MAX, response = SIZE_MAX; + + KUNIT_ASSERT_EQ(test, afkv2_callback_lengths(&command, 32, 256, &request, &response), 0); + KUNIT_EXPECT_EQ(test, request, 32U); + KUNIT_EXPECT_EQ(test, response, 64U); + command.tx_size = 31; + KUNIT_EXPECT_EQ(test, afkv2_callback_lengths(&command, 32, 256, &request, &response), + -EINVAL); + command.tx_size = 32; + command.rx_size = 65; + KUNIT_EXPECT_EQ(test, afkv2_callback_lengths(&command, 32, 256, &request, &response), + -EINVAL); + command.value = U32_MAX; + command.rx_size = 257; + KUNIT_EXPECT_EQ(test, afkv2_callback_lengths(&command, 32, 256, &request, &response), + -EINVAL); +} + +static void afkv2_inline_response_test(struct kunit *test) +{ + struct afkv2_frame frame = { .category = AFKV2_RESPONSE, .interface = 4, .type = 9 }; + struct afkv2_command command = { .id = 3, .inline_size = 64 }; + + KUNIT_EXPECT_EQ(test, afkv2_check_inline_response(&frame, &command, 4, 9, 3, 64), 0); + command.inline_size++; + KUNIT_EXPECT_EQ(test, afkv2_check_inline_response(&frame, &command, 4, 9, 3, 64), + -EPROTO); + command.inline_size--; + frame.token = 1; + KUNIT_EXPECT_EQ(test, afkv2_check_inline_response(&frame, &command, 4, 9, 3, 64), + -EPROTO); +} + +static void afk_ring_payload_bounds_test(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, afk_ring_payload_fits(0x40, 0x100, 0x1000)); + KUNIT_EXPECT_TRUE(test, afk_ring_payload_fits(0xfc0, 0x30, 0x1000)); + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0xfc0, 0x31, 0x1000)); + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0x1000, 0, 0x1000)); +} + +static struct kunit_case afkv2_parser_cases[] = { + KUNIT_CASE(afk_ring_payload_bounds_test), + KUNIT_CASE(afkv2_frame_bounds_test), + KUNIT_CASE(afkv2_command_bounds_test), + KUNIT_CASE(afkv2_owned_span_test), + KUNIT_CASE(afkv2_service_reply_bounds_test), + KUNIT_CASE(afkv2_response_bounds_test), + KUNIT_CASE(afkv2_callback_bounds_test), + KUNIT_CASE(afkv2_inline_response_test), + {} +}; + +static struct kunit_suite afkv2_parser_suite = { + .name = "apple-afkv2-parser", + .test_cases = afkv2_parser_cases, +}; + +kunit_test_suite(afkv2_parser_suite); + +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/iomfb-parser-test.c b/drivers/gpu/drm/apple/iomfb-parser-test.c new file mode 100644 index 0000000000000..e829ff43bda4a --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-parser-test.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Included by iomfb.c to exercise the receive dispatcher itself. */ +#include +#include + +static int iomfb_parser_test_init(struct kunit *test) +{ + struct apple_dcp *dcp; + + dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + if (!dcp) + return -ENOMEM; + dcp->dev = kunit_device_register(test, "iomfb-parser"); + if (IS_ERR(dcp->dev)) + return PTR_ERR(dcp->dev); + dcp->fw_compat = DCP_FIRMWARE_H17P; + test->priv = dcp; + return 0; +} + +static void iomfb_tag_test(struct kunit *test) +{ + char tag[] = "411D"; + unsigned int i; + + KUNIT_EXPECT_EQ(test, dcp_parse_tag(tag), 114); + for (i = 0; i < sizeof(tag) - 1; i++) { + char saved = tag[i]; + + tag[i] = 0xff; + KUNIT_EXPECT_EQ(test, dcp_parse_tag(tag), -EINVAL); + tag[i] = saved; + } + tag[0] = '/'; + KUNIT_EXPECT_EQ(test, dcp_parse_tag(tag), -EINVAL); + tag[0] = ':'; + KUNIT_EXPECT_EQ(test, dcp_parse_tag(tag), -EINVAL); +} + +static void iomfb_short_callback_test(struct kunit *test) +{ + struct apple_dcp *dcp = test->priv; + unsigned int len; + + /* A NULL record must never be read before the envelope is checked. */ + for (len = 0; len < sizeof(struct dcp_packet_header); len++) { + dcp->crashed = false; + dcpep_handle_cb(dcp, DCP_CONTEXT_CB, NULL, len, 0); + KUNIT_EXPECT_TRUE(test, dcp->crashed); + KUNIT_EXPECT_EQ(test, dcp->ch_cb.depth, 0); + } +} + +static void iomfb_callback_envelope_test(struct kunit *test) +{ + struct apple_dcp *dcp = test->priv; + static const struct { + u32 length; + u16 offset; + u8 depth; + } cases[] = { + { sizeof(struct dcp_packet_header), 0x8000, 0 }, + { sizeof(struct dcp_packet_header), 0x7ff8, 0 }, + { U32_MAX, 0, 0 }, + { sizeof(struct dcp_packet_header), 0, DCP_MAX_CALL_DEPTH }, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(cases); i++) { + dcp->crashed = false; + dcp->ch_cb.depth = cases[i].depth; + dcpep_handle_cb(dcp, DCP_CONTEXT_CB, NULL, cases[i].length, cases[i].offset); + KUNIT_EXPECT_TRUE(test, dcp->crashed); + KUNIT_EXPECT_EQ(test, dcp->ch_cb.depth, cases[i].depth); + } + dcp->crashed = false; + dcpep_handle_cb(dcp, 1, NULL, sizeof(struct dcp_packet_header), 0); + KUNIT_EXPECT_TRUE(test, dcp->crashed); +} + +static void iomfb_callback_lengths_test(struct kunit *test) +{ + struct apple_dcp *dcp = test->priv; + struct dcp_packet_header record = { .tag = { '4', '1', '1', 'D' } }; + static const u32 lengths[][2] = { + { U32_MAX, 1 }, { 1, U32_MAX }, { 1, 0 }, { 0, 1 }, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(lengths); i++) { + record.in_len = lengths[i][0]; + record.out_len = lengths[i][1]; + dcp->crashed = false; + dcpep_handle_cb(dcp, DCP_CONTEXT_CB, &record, sizeof(record), 0); + KUNIT_EXPECT_TRUE(test, dcp->crashed); + KUNIT_EXPECT_EQ(test, dcp->ch_cb.depth, 0); + } + memcpy(record.tag, "/11D", 4); + record.in_len = 0; + record.out_len = 0; + dcp->crashed = false; + dcpep_handle_cb(dcp, DCP_CONTEXT_CB, &record, sizeof(record), 0); + KUNIT_EXPECT_TRUE(test, dcp->crashed); +} + +static bool iomfb_parser_callback(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + put_unaligned_le32(0x12345678, out); + return false; +} + +static void iomfb_callback_size_test(struct kunit *test) +{ + struct apple_dcp *dcp = test->priv; + iomfb_cb_handler *handlers; + struct dcp_packet_header *record; + size_t size = sizeof(*record) + 0x1044 + 0x1004; + + handlers = kunit_kcalloc(test, IOMFB_MAX_CB, sizeof(*handlers), GFP_KERNEL); + record = kunit_kzalloc(test, size + 4, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, handlers); + KUNIT_ASSERT_NOT_NULL(test, record); + handlers[114] = iomfb_parser_callback; + dcp->cb_handlers = handlers; + memcpy(record->tag, "411D", 4); + record->in_len = 0x1044; + record->out_len = 0x1008; + dcpep_handle_cb(dcp, DCP_CONTEXT_CB, record, size + 4, 0); + KUNIT_EXPECT_TRUE(test, dcp->crashed); + KUNIT_EXPECT_EQ(test, dcp->ch_cb.depth, 0); + dcp->crashed = false; + record->out_len = 0x1004; + dcpep_handle_cb(dcp, DCP_CONTEXT_CB, record, size, 0); + KUNIT_EXPECT_FALSE(test, dcp->crashed); + KUNIT_EXPECT_EQ(test, dcp->ch_cb.depth, 1); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)(record + 1) + 0x1044), 0x12345678U); +} + +static struct kunit_case iomfb_parser_cases[] = { + KUNIT_CASE(iomfb_tag_test), + KUNIT_CASE(iomfb_short_callback_test), + KUNIT_CASE(iomfb_callback_envelope_test), + KUNIT_CASE(iomfb_callback_lengths_test), + KUNIT_CASE(iomfb_callback_size_test), + {} +}; + +static struct kunit_suite iomfb_parser_suite = { + .name = "apple-iomfb-parser", + .init = iomfb_parser_test_init, + .test_cases = iomfb_parser_cases, +}; + +kunit_test_suite(iomfb_parser_suite); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 00856206b8c70..108c6a42edb99 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1906,3 +1906,7 @@ void iomfb_shutdown(struct apple_dcp *dcp) break; } } + +#if IS_ENABLED(CONFIG_DRM_APPLE_PARSER_KUNIT_TEST) +#include "iomfb-parser-test.c" +#endif From d28218d865cc5a536e9c66a06d2819f55e09c8aa Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:31:34 +1100 Subject: [PATCH 1399/2680] spi: apple: Program the hardware CS delays for every message apple_spi_set_cs_timing() writes the CS delay registers and the CS override bit once, when the SPI core calls it from spi_setup(). The controller uses auto_runtime_pm without an autosuspend delay, so it is runtime suspended right after spi_setup() and after every message, and its power domain can then be gated. The block comes back with its registers at their reset values, which disables the hardware delays. Because the controller provides set_cs_timing, the SPI core no longer inserts the software CS setup and hold delays either. The fingerprint sensor on this bus therefore runs without any CS setup or hold time once the domain has been gated. The registers are also shared by all devices on the bus, so the last device to call spi_setup() would set the timing for all of them. Keep the delay calculation and validation in set_cs_timing, but write the registers from prepare_message, where the controller is powered and the device of the message is known. The hardware delays exist on purpose for the Mesa sensor and are only installed on its bus, see commit 1d0fbf391a2a ("spi: apple: install hardware CS timing only on the Mesa sensor's bus"). That scope and the delay values are unchanged; only the point where the registers are written moves. Fixes: 4a356a39160b ("soc: apple: add production SEP FileVault integration") (cherry picked from commit 9c86fc1e7b5454852b131baecd35df1184c3cedd) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/spi-apple.c | 88 ++++++++++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 31 deletions(-) diff --git a/drivers/spi/spi-apple.c b/drivers/spi/spi-apple.c index f18cc6e1b8d0e..74f3c1f09d739 100644 --- a/drivers/spi/spi-apple.c +++ b/drivers/spi/spi-apple.c @@ -190,22 +190,6 @@ static void apple_spi_init(struct apple_spi *spi) reg_write(spi, APPLE_SPI_DELAY_POST_CYCLES, 0); } -static int apple_spi_prepare_message(struct spi_controller *ctlr, struct spi_message *msg) -{ - struct apple_spi *spi = spi_controller_get_devdata(ctlr); - struct spi_device *device = msg->spi; - - u32 cfg = ((device->mode & SPI_CPHA ? APPLE_SPI_CFG_CPHA : 0) | - (device->mode & SPI_CPOL ? APPLE_SPI_CFG_CPOL : 0) | - (device->mode & SPI_LSB_FIRST ? APPLE_SPI_CFG_LSB_FIRST : 0)); - - /* Update core config */ - reg_mask(spi, APPLE_SPI_CFG, - APPLE_SPI_CFG_CPHA | APPLE_SPI_CFG_CPOL | APPLE_SPI_CFG_LSB_FIRST, cfg); - - return 0; -} - static void apple_spi_set_cs(struct spi_device *device, bool is_high) { struct apple_spi *spi = spi_controller_get_devdata(device->controller); @@ -232,30 +216,32 @@ static int apple_spi_delay_cycles(struct apple_spi *spi, struct spi_delay *delay return 0; } -static int apple_spi_set_cs_timing(struct spi_device *device) +static int apple_spi_cs_timing_cycles(struct apple_spi *spi, + struct spi_device *device, + u32 *setup_cycles, u32 *hold_cycles) { - struct apple_spi *spi = spi_controller_get_devdata(device->controller); - u32 setup_cycles, hold_cycles; int ret; /* This controller path has no hardware representation for inactive time. */ if (device->cs_inactive.value) return -EOPNOTSUPP; - ret = apple_spi_delay_cycles(spi, &device->cs_setup, &setup_cycles); - if (ret) - return ret; - ret = apple_spi_delay_cycles(spi, &device->cs_hold, &hold_cycles); + ret = apple_spi_delay_cycles(spi, &device->cs_setup, setup_cycles); if (ret) return ret; + return apple_spi_delay_cycles(spi, &device->cs_hold, hold_cycles); +} +static void apple_spi_write_cs_timing(struct apple_spi *spi, u32 setup_cycles, + u32 hold_cycles) +{ if (!setup_cycles && !hold_cycles) { reg_mask(spi, APPLE_SPI_SHIFTCFG, APPLE_SPI_SHIFTCFG_OVERRIDE_CS, 0); reg_write(spi, APPLE_SPI_DELAY_PRE, 0); reg_write(spi, APPLE_SPI_DELAY_POST, 0); reg_write(spi, APPLE_SPI_DELAY_POST_CYCLES, 0); - return 0; + return; } reg_write(spi, APPLE_SPI_DELAY_PRE, @@ -267,14 +253,54 @@ static int apple_spi_set_cs_timing(struct spi_device *device) APPLE_SPI_DELAY_POST_FLAGS); reg_mask(spi, APPLE_SPI_SHIFTCFG, 0, APPLE_SPI_SHIFTCFG_OVERRIDE_CS); +} + +/* + * Only validate the delays here. The registers are programmed for each + * message in apple_spi_prepare_message(): their contents are lost when + * runtime PM gates the controller's power domain between messages, and + * they are shared by all devices on the bus. + */ +static int apple_spi_set_cs_timing(struct spi_device *device) +{ + struct apple_spi *spi = spi_controller_get_devdata(device->controller); + u32 setup_cycles, hold_cycles; + int ret; + + ret = apple_spi_cs_timing_cycles(spi, device, &setup_cycles, + &hold_cycles); + if (ret) + return ret; + + dev_dbg(&device->dev, "hardware CS setup=%u hold=%u cycles\n", + setup_cycles, hold_cycles); + return 0; +} + +static int apple_spi_prepare_message(struct spi_controller *ctlr, struct spi_message *msg) +{ + struct apple_spi *spi = spi_controller_get_devdata(ctlr); + struct spi_device *device = msg->spi; + + u32 cfg = ((device->mode & SPI_CPHA ? APPLE_SPI_CFG_CPHA : 0) | + (device->mode & SPI_CPOL ? APPLE_SPI_CFG_CPOL : 0) | + (device->mode & SPI_LSB_FIRST ? APPLE_SPI_CFG_LSB_FIRST : 0)); + + /* Update core config */ + reg_mask(spi, APPLE_SPI_CFG, + APPLE_SPI_CFG_CPHA | APPLE_SPI_CFG_CPOL | APPLE_SPI_CFG_LSB_FIRST, cfg); + + if (ctlr->set_cs_timing) { + u32 setup_cycles, hold_cycles; + int ret; + + ret = apple_spi_cs_timing_cycles(spi, device, &setup_cycles, + &hold_cycles); + if (ret) + return ret; + apple_spi_write_cs_timing(spi, setup_cycles, hold_cycles); + } - dev_dbg(&device->dev, - "hardware CS setup=%u hold=%u cycles: pre=%#08x post=%#08x post_cycles=%#08x shiftcfg=%#08x\n", - setup_cycles, hold_cycles, - reg_read(spi, APPLE_SPI_DELAY_PRE), - reg_read(spi, APPLE_SPI_DELAY_POST), - reg_read(spi, APPLE_SPI_DELAY_POST_CYCLES), - reg_read(spi, APPLE_SPI_SHIFTCFG)); return 0; } From 9a08dca9e7183068cfe702535a321e3673d4dde7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 01:03:04 +1100 Subject: [PATCH 1400/2680] iommu: apple-dart: Return IOMEM_ERR_PTR() from the tunneled MMIO helper apple_dart_platform_ioremap_np_resource() returns a void __iomem pointer but builds its error values with ERR_PTR(), which sparse reports as an address space mismatch. Use IOMEM_ERR_PTR(). No functional change. (cherry picked from commit dbc0391a3a19b2148bb6c37c5f9765ca7fa9acc4) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 7543d35c4105a..5cfa98a956916 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -2610,20 +2610,20 @@ apple_dart_platform_ioremap_np_resource(struct platform_device *pdev, res = platform_get_resource(pdev, IORESOURCE_MEM, index); if (!res) - return ERR_PTR(-EINVAL); + return IOMEM_ERR_PTR(-EINVAL); if (!devm_request_mem_region(dev, res->start, resource_size(res), dev_name(dev))) - return ERR_PTR(-EBUSY); + return IOMEM_ERR_PTR(-EBUSY); regs = ioremap_np(res->start, resource_size(res)); if (!regs) - return ERR_PTR(-ENOMEM); + return IOMEM_ERR_PTR(-ENOMEM); ret = devm_add_action_or_reset(dev, apple_dart_iounmap_np, (__force void *)regs); if (ret) - return ERR_PTR(ret); + return IOMEM_ERR_PTR(ret); *res_out = res; return regs; From 0af2be2f9628b204896d12d449a620fce12d0887 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:33:08 +1100 Subject: [PATCH 1401/2680] spi: apple-qspi: Restore the controller configuration after system sleep The controller is reset and configured once, in probe: interrupt enables, FIFO thresholds and the fixed timing registers written by apple_qspi_init_config(). The driver has no PM callbacks, so when the QSPI power domain is gated during system sleep these registers come back with their reset values and are never written again. Only the clock divider and the mode are programmed for each operation. The first flash access after resume, typically an NVRAM read or write, then runs with an unconfigured controller. If it fails to complete it costs a full transfer timeout before the timeout path resets the block. Add system sleep callbacks that stop the controller across suspend and run the same reset and configuration as probe on resume. Fixes: 9057400a21c4 ("spi: apple: add T8140 QSPIMC controller") (cherry picked from commit 085e077be1d7956704b7234ea26183ea28c74ce6) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/spi-apple-qspi.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c index e0d8653e51984..2142b14e167bf 100644 --- a/drivers/spi/spi-apple-qspi.c +++ b/drivers/spi/spi-apple-qspi.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -593,6 +594,35 @@ static int apple_qspi_probe(struct platform_device *pdev) return 0; } +static int apple_qspi_suspend(struct device *dev) +{ + struct spi_controller *ctlr = dev_get_drvdata(dev); + + return spi_controller_suspend(ctlr); +} + +/* + * The controller loses its configuration when its power domain is gated + * during system sleep. Restore it before the first operation after resume. + */ +static int apple_qspi_resume(struct device *dev) +{ + struct spi_controller *ctlr = dev_get_drvdata(dev); + struct apple_qspi *qspi = spi_controller_get_devdata(ctlr); + int ret; + + ret = apple_qspi_hw_reset(qspi); + if (ret) { + dev_err(dev, "controller reset failed on resume: %d\n", ret); + return ret; + } + + return spi_controller_resume(ctlr); +} + +static DEFINE_SIMPLE_DEV_PM_OPS(apple_qspi_pm_ops, apple_qspi_suspend, + apple_qspi_resume); + static const struct of_device_id apple_qspi_of_match[] = { { .compatible = "apple,t8140-qspi" }, {} @@ -604,6 +634,7 @@ static struct platform_driver apple_qspi_driver = { .driver = { .name = "apple-qspi", .of_match_table = apple_qspi_of_match, + .pm = pm_sleep_ptr(&apple_qspi_pm_ops), }, }; module_platform_driver(apple_qspi_driver); From e6e947f8cccbc4b1b3705e4bb57c3672b2dcffb7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:09:12 +1100 Subject: [PATCH 1402/2680] HID: dockchannel-hid: Keep parser tests in the subsystem menu The parser KUnit option appears after the DockChannel HID menu closes, so menuconfig presents it outside the subsystem it tests. Move the option above endmenu without changing its dependencies. (cherry picked from commit 060616f0e5b3d8fb6068c67ab4039bb1b2b94183) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/Kconfig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/hid/dockchannel-hid/Kconfig b/drivers/hid/dockchannel-hid/Kconfig index 600fc235b4790..8889385d929bf 100644 --- a/drivers/hid/dockchannel-hid/Kconfig +++ b/drivers/hid/dockchannel-hid/Kconfig @@ -26,8 +26,6 @@ config HID_DOCKCHANNEL_KUNIT_TEST If unsure, say N. -endmenu - config HID_DOCKCHANNEL_PARSER_KUNIT_TEST bool "KUnit tests for DockChannel HID message parsing" if !KUNIT_ALL_TESTS depends on HID_DOCKCHANNEL && KUNIT=y @@ -37,3 +35,5 @@ config HID_DOCKCHANNEL_PARSER_KUNIT_TEST input-report and acknowledgment handlers. Cover short headers, oversized lengths, interface indices and acknowledgment identity. The tests run without a DockChannel or HID device. + +endmenu From bedd3d5b1994f7716b355cfba924d312bdcc78e1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:16:02 +1100 Subject: [PATCH 1403/2680] media: apple: isp: Do not trust the batch count when a batch fails isp_submit_buffers() builds the buffer batch in the command area, which lies in the IPC surface the firmware reads and writes. When the firmware does not acknowledge the batch, the error path walks the batch again to put its buffers back on the pending list, and takes the number of entries from the batch header in that shared memory. Firmware that wrote to the area in the meantime, or that misbehaves, can make the driver walk far past the 40 entries the area holds, reading beyond the surface. Count the entries in a local variable while building the batch, write the count to the header once, and use the local count in the error path. The batch the firmware sees is unchanged. (cherry picked from commit 187f13551b80e04718d59e733edda8ac4878b9e0) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-v4l2.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 23eab4bf498a9..072b8523279c2 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -153,7 +153,7 @@ static int isp_submit_buffers(struct apple_isp *isp, bool pre_start) struct isp_channel *chan = isp->chan_bh; struct isp_message *req = &chan->req; struct isp_buffer *buf, *tmp; - unsigned int num_rendered = 0; + unsigned int num_rendered = 0, num_buffers = 0; unsigned long flags; size_t offset; int err; @@ -180,7 +180,7 @@ static int isp_submit_buffers(struct apple_isp *isp, bool pre_start) bufd->iovas[0] = meta->iova; bufd->flags[0] = 0x40000000; bufd++; - bl->num_buffers++; + num_buffers++; meta->submitted = true; } @@ -199,7 +199,7 @@ static int isp_submit_buffers(struct apple_isp *isp, bool pre_start) bufd->iovas[0] = meta->iova; bufd->flags[0] = 0x40000000; bufd++; - bl->num_buffers++; + num_buffers++; meta->submitted = true; } @@ -222,7 +222,7 @@ static int isp_submit_buffers(struct apple_isp *isp, bool pre_start) buf->surfs[0].iova, buf->surfs[0].iova + buf->surfs[0].size); */ bufd++; - bl->num_buffers++; + num_buffers++; num_rendered++; /* @@ -236,6 +236,12 @@ static int isp_submit_buffers(struct apple_isp *isp, bool pre_start) send: spin_unlock_irqrestore(&isp->buf_lock, flags); + /* + * The firmware can write to the command area, so the error path below + * goes by the count kept here rather than the one in the batch. + */ + bl->num_buffers = num_buffers; + req->arg0 = isp->cmd_iova; req->arg1 = max_t(u64, ISP_IPC_BUFEXC_STAT_SIZE, ((uintptr_t)bufd - (uintptr_t)bl)); @@ -255,7 +261,7 @@ static int isp_submit_buffers(struct apple_isp *isp, bool pre_start) spin_lock_irqsave(&isp->buf_lock, flags); bufd = &bl->buffers[0]; - for (int i = 0; i < bl->num_buffers; i++, bufd++) { + for (unsigned int i = 0; i < num_buffers; i++, bufd++) { list_for_each_entry_safe_reverse( buf, tmp, &isp->bufs_submitted, link) { if (bufd->iovas[0] == buf->surfs[0].iova) { From 3563595ef9c52c6599bfb04a3fef540f75047f86 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:41:37 +1100 Subject: [PATCH 1404/2680] mfd: macsmc: Test key and fixed-width readers with KUnit Use KUnit static stubs for the command, SRAM, and battery read paths to exercise the real readers without a mailbox. Check request bounds, inline and SRAM values, short and oversized replies, key lookup errors, index validation, metadata locking, and the CHTE charge-behaviour path. The short CHTE cases poison the unread buffer while returning zero or two bytes. They fail if the caller inspects data from a short reply. (cherry picked from commit bdf1eea58e4b537505cf4cfda00f2719dcca5e0a) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/mfd/Kconfig | 10 + drivers/mfd/macsmc-test.c | 221 +++++++++++++++++++++++ drivers/mfd/macsmc.c | 20 +- drivers/power/supply/Kconfig | 10 + drivers/power/supply/macsmc-power-test.c | 58 ++++++ drivers/power/supply/macsmc-power.c | 16 +- 6 files changed, 332 insertions(+), 3 deletions(-) create mode 100644 drivers/mfd/macsmc-test.c create mode 100644 drivers/power/supply/macsmc-power-test.c diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index 763ce6a34782b..4d7a7f0fe1ea5 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -343,6 +343,16 @@ config MFD_MACSMC various features. The MFD core device handles this protocol and exposes it to the sub-devices. +config MFD_MACSMC_KUNIT_TEST + bool "KUnit tests for Apple SMC key readers" if !KUNIT_ALL_TESTS + depends on MFD_MACSMC && KUNIT=y + default KUNIT_ALL_TESTS + help + Build tests for SMC key/value readers, request limits and metadata + decoding. Static stubs replace mailbox commands and SRAM access, + allowing malformed replies and output buffer limits to be checked + without a running SMC. + config MFD_MADERA tristate "Cirrus Logic Madera codecs" select MFD_CORE diff --git a/drivers/mfd/macsmc-test.c b/drivers/mfd/macsmc-test.c new file mode 100644 index 0000000000000..c4f9329dbc01c --- /dev/null +++ b/drivers/mfd/macsmc-test.c @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Included by macsmc.c; the command and SRAM transports are stubbed. */ +#include + +struct macsmc_test_context { + struct apple_smc smc; + u8 shmem[SMC_MAX_SIZE]; + u32 data; + int result; + unsigned int calls; + bool require_sram_lock; +}; + +static int macsmc_test_command(struct apple_smc *smc, u64 cmd, u64 arg, + u64 size, u64 wsize, u32 *ret_data) +{ + struct macsmc_test_context *ctx = container_of(smc, struct macsmc_test_context, smc); + + KUNIT_EXPECT_TRUE(kunit_get_current_test(), mutex_is_locked(&smc->mutex)); + ctx->calls++; + if (ret_data && ctx->result >= 0) + *ret_data = ctx->data; + return ctx->result; +} + +static void macsmc_test_read_sram(struct apple_smc *smc, void *buf, size_t size) +{ + struct macsmc_test_context *ctx = container_of(smc, struct macsmc_test_context, smc); + + if (ctx->require_sram_lock) + KUNIT_EXPECT_TRUE(kunit_get_current_test(), mutex_is_locked(&smc->mutex)); + memcpy(buf, ctx->shmem, size); +} + +static int macsmc_test_init(struct kunit *test) +{ + struct macsmc_test_context *ctx; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + mutex_init(&ctx->smc.mutex); + ctx->smc.shmem.iomem = (void __iomem *)ctx->shmem; + ctx->smc.key_count = 3; + test->priv = ctx; + kunit_activate_static_stub(test, apple_smc_cmd_locked, macsmc_test_command); + kunit_activate_static_stub(test, apple_smc_read_sram, macsmc_test_read_sram); + return 0; +} + +static void macsmc_value_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + u8 byte = 0; + u32 word = 0; + + ctx->data = 0x12345678; + ctx->result = sizeof(word); + KUNIT_EXPECT_EQ(test, apple_smc_read_u32(&ctx->smc, SMC_KEY(TEST), &word), 0); + KUNIT_EXPECT_EQ(test, word, ctx->data); + ctx->result = sizeof(byte); + KUNIT_EXPECT_EQ(test, apple_smc_read_u8(&ctx->smc, SMC_KEY(TEST), &byte), 0); + KUNIT_EXPECT_EQ(test, byte, 0x78); + ctx->result = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, apple_smc_read_u8(&ctx->smc, SMC_KEY(TEST), &byte), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, byte, 0x78); +} + +static void macsmc_fixed_value_short_reply_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + u32 word = 0xa5a5a5a5; + u8 byte = 0xa5; + + ctx->data = 0x12345678; + ctx->result = 2; + KUNIT_EXPECT_EQ(test, apple_smc_read_u32(&ctx->smc, SMC_KEY(TEST), &word), -EIO); + KUNIT_EXPECT_EQ(test, word, 0U); + word = 0xa5a5a5a5; + KUNIT_EXPECT_EQ(test, + apple_smc_rw_u32(&ctx->smc, SMC_KEY(TEST), 0, &word), -EIO); + KUNIT_EXPECT_EQ(test, word, 0U); + ctx->result = 0; + KUNIT_EXPECT_EQ(test, apple_smc_read_u8(&ctx->smc, SMC_KEY(TEST), &byte), -EIO); + KUNIT_EXPECT_EQ(test, byte, 0); +} + +static void macsmc_request_bounds_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + u8 byte; + + KUNIT_EXPECT_EQ(test, apple_smc_read(&ctx->smc, SMC_KEY(TEST), &byte, + SMC_MAX_SIZE + 1), -EINVAL); + KUNIT_EXPECT_EQ(test, apple_smc_read(&ctx->smc, SMC_KEY(TEST), &byte, SIZE_MAX), + -EINVAL); + KUNIT_EXPECT_EQ(test, apple_smc_write(&ctx->smc, SMC_KEY(TEST), &byte, SIZE_MAX), + -EINVAL); + KUNIT_EXPECT_EQ(test, apple_smc_read(&ctx->smc, SMC_KEY(TEST), &byte, 0), -EINVAL); + KUNIT_EXPECT_EQ(test, ctx->calls, 0U); +} + +static void macsmc_sram_value_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + u8 value[SMC_MAX_SIZE + 2]; + unsigned int i; + + for (i = 0; i < sizeof(ctx->shmem); i++) + ctx->shmem[i] = i; + memset(value, 0xa5, sizeof(value)); + ctx->result = SMC_MAX_SIZE; + KUNIT_EXPECT_EQ(test, apple_smc_read(&ctx->smc, SMC_KEY(TEST), value + 1, + SMC_MAX_SIZE), SMC_MAX_SIZE); + KUNIT_EXPECT_MEMEQ(test, value + 1, ctx->shmem, SMC_MAX_SIZE); + KUNIT_EXPECT_EQ(test, value[0], 0xa5); + KUNIT_EXPECT_EQ(test, value[SMC_MAX_SIZE + 1], 0xa5); +} + +static void macsmc_key_info_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + struct apple_smc_key_info info = {}; + const u8 wire[] = { 255, 'u', 'i', '3', '2', APPLE_SMC_READABLE }; + + memcpy(ctx->shmem, wire, sizeof(wire)); + KUNIT_EXPECT_EQ(test, apple_smc_get_key_info(&ctx->smc, SMC_KEY(TEST), &info), 0); + KUNIT_EXPECT_EQ(test, info.size, 255); + KUNIT_EXPECT_EQ(test, info.type_code, SMC_KEY(ui32)); + KUNIT_EXPECT_EQ(test, info.flags, APPLE_SMC_READABLE); + ctx->result = -EIO; + memset(ctx->shmem, 0, sizeof(wire)); + KUNIT_EXPECT_EQ(test, apple_smc_get_key_info(&ctx->smc, SMC_KEY(TEST), &info), -EIO); + KUNIT_EXPECT_EQ(test, info.size, 255); +} + +static void macsmc_short_value_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + u8 value[4]; + + memset(value, 0xa5, sizeof(value)); + ctx->data = 0x12345678; + ctx->result = 0; + KUNIT_EXPECT_EQ(test, apple_smc_read(&ctx->smc, SMC_KEY(TEST), value, 4), 0); + KUNIT_EXPECT_MEMEQ(test, value, ((u8[]){ 0xa5, 0xa5, 0xa5, 0xa5 }), 4); + ctx->result = 2; + KUNIT_EXPECT_EQ(test, apple_smc_read(&ctx->smc, SMC_KEY(TEST), value, 4), 2); + KUNIT_EXPECT_MEMEQ(test, value, ((u8[]){ 0x78, 0x56, 0xa5, 0xa5 }), 4); + memset(value, 0xa5, sizeof(value)); + ctx->result = 5; + KUNIT_EXPECT_EQ(test, apple_smc_read(&ctx->smc, SMC_KEY(TEST), value, 4), -EIO); + KUNIT_EXPECT_MEMEQ(test, value, ((u8[]){ 0xa5, 0xa5, 0xa5, 0xa5 }), 4); + ctx->result = 7; + memset(ctx->shmem, 0x3c, sizeof(ctx->shmem)); + memset(ctx->shmem + 16, 0xa5, 8); + KUNIT_EXPECT_EQ(test, apple_smc_read(&ctx->smc, SMC_KEY(TEST), ctx->shmem + 16, 8), 7); + KUNIT_EXPECT_EQ(test, ctx->shmem[23], 0xa5); +} + +static void macsmc_key_error_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + smc_key key = SMC_KEY(TEST); + + ctx->result = -EIO; + KUNIT_EXPECT_EQ(test, apple_smc_get_key_by_index(&ctx->smc, 0, &key), -EIO); + KUNIT_EXPECT_EQ(test, key, SMC_KEY(TEST)); + ctx->result = 0; + ctx->data = swab32(SMC_KEY(TEST)); + KUNIT_EXPECT_EQ(test, apple_smc_get_key_by_index(&ctx->smc, 2, &key), 0); + KUNIT_EXPECT_EQ(test, key, SMC_KEY(TEST)); +} + +static void macsmc_key_index_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + smc_key key = SMC_KEY(TEST); + int index = -1; + + KUNIT_EXPECT_EQ(test, + apple_smc_get_key_by_index(&ctx->smc, READ_ONCE(index), &key), + -EINVAL); + KUNIT_EXPECT_EQ(test, apple_smc_get_key_by_index(&ctx->smc, 3, &key), -EINVAL); + KUNIT_EXPECT_EQ(test, apple_smc_get_key_by_index(&ctx->smc, INT_MAX, &key), -EINVAL); + KUNIT_EXPECT_EQ(test, ctx->calls, 0U); + KUNIT_EXPECT_EQ(test, key, SMC_KEY(TEST)); +} + +static void macsmc_key_info_lock_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + struct apple_smc_key_info info; + + ctx->require_sram_lock = true; + KUNIT_EXPECT_EQ(test, apple_smc_get_key_info(&ctx->smc, SMC_KEY(TEST), &info), 0); + KUNIT_EXPECT_FALSE(test, mutex_is_locked(&ctx->smc.mutex)); + KUNIT_EXPECT_EQ(test, apple_smc_get_key_info(&ctx->smc, SMC_KEY(TEST), NULL), 0); + KUNIT_EXPECT_FALSE(test, mutex_is_locked(&ctx->smc.mutex)); +} + +static struct kunit_case macsmc_parser_cases[] = { + KUNIT_CASE(macsmc_key_info_lock_test), + KUNIT_CASE(macsmc_key_index_test), + KUNIT_CASE(macsmc_key_error_test), + KUNIT_CASE(macsmc_short_value_test), + KUNIT_CASE(macsmc_fixed_value_short_reply_test), + KUNIT_CASE(macsmc_value_test), + KUNIT_CASE(macsmc_request_bounds_test), + KUNIT_CASE(macsmc_sram_value_test), + KUNIT_CASE(macsmc_key_info_test), + {} +}; + +static struct kunit_suite macsmc_parser_suite = { + .name = "macsmc-parser", + .init = macsmc_test_init, + .test_cases = macsmc_parser_cases, +}; + +kunit_test_suite(macsmc_parser_suite); diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index cf25327a7bd8b..55d52c684d38b 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -5,6 +5,8 @@ * Copyright The Asahi Linux Contributors */ +#include + #include #include #include @@ -57,6 +59,13 @@ static const struct mfd_cell apple_smc_devs[] = { static const struct mfd_cell apple_smc_low_gpio_dev = MFD_CELL_OF("macsmc-low-gpio", NULL, NULL, 0, 0, "apple,smc-low-gpio"); +static void apple_smc_read_sram(struct apple_smc *smc, void *buf, size_t size) +{ + if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(apple_smc_read_sram, smc, buf, size); + memcpy_fromio(buf, smc->shmem.iomem, size); +} + static int apple_smc_cmd_locked(struct apple_smc *smc, u64 cmd, u64 arg, u64 size, u64 wsize, u32 *ret_data) { @@ -65,6 +74,9 @@ static int apple_smc_cmd_locked(struct apple_smc *smc, u64 cmd, u64 arg, u64 msg; lockdep_assert_held(&smc->mutex); + if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(apple_smc_cmd_locked, smc, cmd, arg, size, wsize, + ret_data); if (smc->boot_stage != APPLE_SMC_INITIALIZED) return -EIO; @@ -170,7 +182,7 @@ static int apple_smc_rw_locked(struct apple_smc *smc, smc_key key, if (rsize <= 4) memcpy(rbuf, &rdata, ret); else - memcpy_fromio(rbuf, smc->shmem.iomem, ret); + apple_smc_read_sram(smc, rbuf, ret); } return ret; @@ -227,7 +239,7 @@ int apple_smc_get_key_info(struct apple_smc *smc, smc_key key, struct apple_smc_ ret = apple_smc_cmd_locked(smc, SMC_MSG_GET_KEY_INFO, key, 0, 0, NULL); if (ret >= 0 && info) { - memcpy_fromio(key_info, smc->shmem.iomem, sizeof(key_info)); + apple_smc_read_sram(smc, key_info, sizeof(key_info)); info->size = key_info[0]; info->type_code = get_unaligned_be32(&key_info[1]); info->flags = key_info[5]; @@ -536,3 +548,7 @@ MODULE_AUTHOR("Hector Martin "); MODULE_AUTHOR("Sven Peter "); MODULE_LICENSE("Dual MIT/GPL"); MODULE_DESCRIPTION("Apple SMC driver"); + +#if IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST) +#include "macsmc-test.c" +#endif diff --git a/drivers/power/supply/Kconfig b/drivers/power/supply/Kconfig index f0ede1cecb6a8..c535296dd5e80 100644 --- a/drivers/power/supply/Kconfig +++ b/drivers/power/supply/Kconfig @@ -1186,4 +1186,14 @@ config MACSMC_POWER Say Y or M here if you have an Apple Silicon based Mac. +config MACSMC_POWER_KUNIT_TEST + bool "KUnit tests for Apple SMC battery replies" if !KUNIT_ALL_TESTS + depends on MACSMC_POWER && KUNIT=y + default KUNIT_ALL_TESTS + help + Build tests for malformed fixed-width battery replies. The tests + exercise zero-length and short CHTE replies and check that the + charge-behaviour reader rejects them. Static stubs replace SMC + reads, so the tests need no Apple hardware. + endif # POWER_SUPPLY diff --git a/drivers/power/supply/macsmc-power-test.c b/drivers/power/supply/macsmc-power-test.c new file mode 100644 index 0000000000000..78f7f1b877beb --- /dev/null +++ b/drivers/power/supply/macsmc-power-test.c @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Included by macsmc-power.c; the charge-behaviour reader is stubbed. */ +#include + +struct macsmc_power_test_context { + struct macsmc_power power; + int result; +}; + +static int macsmc_power_test_read_chte(struct macsmc_power *power, u8 *buf) +{ + struct macsmc_power_test_context *ctx = + container_of(power, struct macsmc_power_test_context, power); + + memset(buf, 0xa5, 4); + buf[0] = 1; + return ctx->result; +} + +static int macsmc_power_test_init(struct kunit *test) +{ + struct macsmc_power_test_context *ctx; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + ctx->power.has_chte = true; + test->priv = ctx; + kunit_activate_static_stub(test, macsmc_power_read_chte, + macsmc_power_test_read_chte); + return 0; +} + +static void macsmc_power_short_chte_test(struct kunit *test) +{ + struct macsmc_power_test_context *ctx = test->priv; + + ctx->result = 0; + KUNIT_EXPECT_EQ(test, macsmc_battery_get_charge_behaviour(&ctx->power), -EIO); + ctx->result = 2; + KUNIT_EXPECT_EQ(test, macsmc_battery_get_charge_behaviour(&ctx->power), -EIO); + ctx->result = 4; + KUNIT_EXPECT_EQ(test, macsmc_battery_get_charge_behaviour(&ctx->power), + POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE); +} + +static struct kunit_case macsmc_power_test_cases[] = { + KUNIT_CASE(macsmc_power_short_chte_test), + {} +}; + +static struct kunit_suite macsmc_power_test_suite = { + .name = "macsmc-power", + .init = macsmc_power_test_init, + .test_cases = macsmc_power_test_cases, +}; + +kunit_test_suite(macsmc_power_test_suite); diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c index 3753c3451e4b7..be10128414758 100644 --- a/drivers/power/supply/macsmc-power.c +++ b/drivers/power/supply/macsmc-power.c @@ -22,6 +22,8 @@ #include #include +#include + #define MAX_STRING_LENGTH 256 /* @@ -316,6 +318,14 @@ static int macsmc_battery_get_status(struct macsmc_power *power) return macsmc_battery_charging_status(power); } +static int macsmc_power_read_chte(struct macsmc_power *power, u8 *buf) +{ + if (IS_ENABLED(CONFIG_MACSMC_POWER_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(macsmc_power_read_chte, power, buf); + + return apple_smc_read(power->smc, SMC_KEY(CHTE), buf, 4); +} + static int macsmc_battery_get_charge_behaviour(struct macsmc_power *power) { int ret; @@ -331,7 +341,7 @@ static int macsmc_battery_get_charge_behaviour(struct macsmc_power *power) } if (power->has_chte) { - ret = apple_smc_read(power->smc, SMC_KEY(CHTE), chte_buf, 4); + ret = macsmc_power_read_chte(power, chte_buf); if (ret < 0) return ret; if (ret != sizeof(chte_buf)) { @@ -1123,3 +1133,7 @@ MODULE_LICENSE("Dual MIT/GPL"); MODULE_DESCRIPTION("Apple SMC battery and power management driver"); MODULE_AUTHOR("Hector Martin "); MODULE_AUTHOR("Michael Reeves "); + +#if IS_ENABLED(CONFIG_MACSMC_POWER_KUNIT_TEST) +#include "macsmc-power-test.c" +#endif From 4f86c3aaf1b4603a972f18873668ecf19e55ff87 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:34:16 +1100 Subject: [PATCH 1405/2680] nvme-apple: Reword two post-M4 handoff comments Describe what the inherited-buffer check and the restart of a stopped post-M4 coprocessor rely on, without the references to particular boot stages and their test history. No functional change. (cherry picked from commit 9e2861cbe6f72900640e53f2c28aa87e87460766) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 6ef14f7bf96c2..14a7194773a8d 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -410,9 +410,8 @@ static bool apple_nvme_in_inherited_rtkit_range(struct apple_nvme *anv, /* * A range property or a partial System RAM intersection does not prove - * ownership. Every page must be reserved and present in the linear map. - * Native m1n1 reserves the predecessor's inherited allocation prefix. - * RAM excluded by a resident EL2 stage needs a separate cache contract. + * ownership. Every page must be reserved and present in the linear map, + * as the bootloader leaves the buffers it hands over. */ if (region_intersects(bfr->iova, bfr->size, IORESOURCE_SYSTEM_RAM, IORES_DESC_NONE) != REGION_INTERSECTS) @@ -1640,8 +1639,8 @@ static void apple_nvme_reset_work(struct work_struct *work) * Post-M4 ANS cannot be reset through PMGR: the reset * assert raises an SError. A previous boot stage that * stopped the coprocessor cleanly leaves its firmware - * resident, so mirror the qualified U-Boot handoff: - * set RUN and run a fresh RTKit boot handshake. + * resident, so set RUN and run a fresh RTKit boot + * handshake. */ dev_dbg(anv->dev, "post-M4 firmware handoff: restarting stopped ANS without reset\n"); From e34c99fd7ef0d55598004a197c429f4bf2aa95c4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 01:21:43 +1100 Subject: [PATCH 1406/2680] drm/apple: remove duplicate AFK service test registration When DRM_APPLE_KUNIT_TEST=m, afk-service-test.o is added to modules.order both by drivers/gpu/drm/Makefile and by the Apple driver Makefile. The duplicate entries cause modules_check to fail with a module name conflict. Keep the registration in drivers/gpu/drm/apple/Makefile next to the driver and remove the redundant top-level entry. Fixes: 18d69dc80648 ("drm/apple: close external admission and recovery races") (cherry picked from commit 5680427fe7f9aacc8f32146855845ae10650356b) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index 200ff122fee5c..8bc9a05484f21 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -229,7 +229,6 @@ obj-$(CONFIG_DRM_PANTHOR) += panthor/ obj-$(CONFIG_DRM_TYR) += tyr/ obj-$(CONFIG_DRM_APPLE) += apple/ obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple/iomfb-state-test.o -obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple/afk-service-test.o obj-$(CONFIG_DRM_ASPEED_GFX) += aspeed/ obj-$(CONFIG_DRM_MCDE) += mcde/ obj-$(CONFIG_DRM_TIDSS) += tidss/ From 600c9e6a9fa40d509971e8f305c3136eccafeda2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 06:50:57 +1100 Subject: [PATCH 1407/2680] drm/apple: afk: Export afk_service_retry() to the AFK service test afk_service_put() is an inline helper in afk.h that ends by calling afk_service_retry(), which lives in afk.c inside appledrm. The AFK service lifetime test is built as its own module and uses afk_service_put(), so with DRM_APPLE=m and DRM_APPLE_KUNIT_TEST=m (as KUNIT_ALL_TESTS=m selects) modpost fails: ERROR: modpost: "afk_service_retry" [drivers/gpu/drm/apple/afk-service-test.ko] undefined! Export the function to the KUnit test namespace and import that namespace in the test, as other drivers do for tests that live in their own module. The export is compiled out without CONFIG_KUNIT. Fixes: 7357e51053e4 ("drm/apple: scope deferred DPAV recovery to current interfaces") (cherry picked from commit 31f77ecbff1c2f8f8c680c3fa9775757e0ce9791) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk-service-test.c | 1 + drivers/gpu/drm/apple/afk.c | 3 +++ 2 files changed, 4 insertions(+) diff --git a/drivers/gpu/drm/apple/afk-service-test.c b/drivers/gpu/drm/apple/afk-service-test.c index a86df389deb17..1cf589ae7a417 100644 --- a/drivers/gpu/drm/apple/afk-service-test.c +++ b/drivers/gpu/drm/apple/afk-service-test.c @@ -238,4 +238,5 @@ static struct kunit_suite afk_service_suite = { }; kunit_test_suite(afk_service_suite); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 5f69e8ad80f29..729a14fe9cdf2 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -13,6 +13,8 @@ #include #include +#include + #include "afk.h" #include "afkv2-codec.h" #include "trace.h" @@ -66,6 +68,7 @@ void afk_service_retry(struct apple_dcp_afkep *ep) queue_work(ep->wq, &ep->service_work); spin_unlock_irqrestore(&ep->lock, flags); } +EXPORT_SYMBOL_IF_KUNIT(afk_service_retry); static void afk_send(struct apple_dcp_afkep *ep, u64 message) { From 26a9dc8733fd4322eac4b007cbe6ccc683e7da7b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:46:27 +1100 Subject: [PATCH 1408/2680] soc: apple: aop: Decode ring entry headers without pointer casts The ring entry header is read by casting a byte array to a QEHeader reference and written by viewing the struct as a byte slice, both in unsafe blocks with no safety comment, and clippy flags the casts. Encode and decode the header field by field as little-endian words instead, as the driver already does for the EPIC headers, and hoist the duplicated QEH_SIZE constant. While at it, address the remaining clippy findings in the driver: add the missing safety comments, use as_char_ptr() and ptr::from_ref() instead of pointer casts when filling the platform_device_info, and drop a no-op u64 cast in patch_bootargs(). No functional change. (cherry picked from commit ca667646525a896ab42d67098e2976a053293375) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 52 +++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 69123e033e9db..57c3648f957ea 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -213,6 +213,7 @@ fn le_u64(b: &[u8], off: usize) -> u64 { #[inline(always)] fn mem_sync() { + // SAFETY: a full-system barrier touches no memory or registers of ours. unsafe { asm!("dsb sy"); } @@ -227,6 +228,29 @@ struct QEHeader { ty: u32, } +/// The size of a ring entry header. +const QEH_SIZE: usize = mem::size_of::(); + +impl QEHeader { + fn from_bytes(b: &[u8; QEH_SIZE]) -> QEHeader { + QEHeader { + magic: le_u32(b, 0), + size: le_u32(b, 4), + channel: le_u32(b, 8), + ty: le_u32(b, 12), + } + } + + fn to_bytes(self) -> [u8; QEH_SIZE] { + let mut b = [0; QEH_SIZE]; + b[0..4].copy_from_slice(&{ self.magic }.to_le_bytes()); + b[4..8].copy_from_slice(&{ self.size }.to_le_bytes()); + b[8..12].copy_from_slice(&{ self.channel }.to_le_bytes()); + b[12..16].copy_from_slice(&{ self.ty }.to_le_bytes()); + b + } +} + #[repr(C, packed)] #[derive(Clone, Copy, Default)] struct EPICHeader { @@ -601,11 +625,10 @@ impl AFKEndpoint { mem_sync(); let base = buf_offset + block_size * 3; let mut msg_buf = KVec::new(); - const QEH_SIZE: usize = mem::size_of::(); while wptr as usize != rptr { let mut qeh_bytes = [0; QEH_SIZE]; self.memcpy_from_iomem(base + rptr, &mut qeh_bytes)?; - let mut qeh = unsafe { &*(qeh_bytes.as_ptr() as *const QEHeader) }; + let mut qeh = QEHeader::from_bytes(&qeh_bytes); if qeh.magic != QE_MAGIC1 && qeh.magic != QE_MAGIC2 { let magic = qeh.magic; dev_err!( @@ -620,7 +643,7 @@ impl AFKEndpoint { if qeh.size as usize > room { rptr = 0; self.memcpy_from_iomem(base + rptr, &mut qeh_bytes)?; - qeh = unsafe { &*(qeh_bytes.as_ptr() as *const QEHeader) }; + qeh = QEHeader::from_bytes(&qeh_bytes); if qeh.magic != QE_MAGIC1 && qeh.magic != QE_MAGIC2 { let magic = qeh.magic; @@ -643,7 +666,7 @@ impl AFKEndpoint { self.drop_message(&client.dev, fmt!("short message ({} bytes)", msg_buf.len())) } Some((header, msg)) => { - if let Err(e) = self.handle_ipc(client, qeh, &header, msg) { + if let Err(e) = self.handle_ipc(client, &qeh, &header, msg) { let channel = qeh.channel; self.drop_message( &client.dev, @@ -789,7 +812,6 @@ impl AFKEndpoint { mem_sync(); let rptr = self.iomem_read32(buf_offset + block_size)? as usize; let mut wptr = self.iomem_read32(buf_offset + block_size * 2)? as usize; - const QEH_SIZE: usize = mem::size_of::(); if wptr < rptr && wptr + QEH_SIZE >= rptr { dev_err!(client.dev, "Tx buffer full at endpoint {}", self.index); return Err(EIO); @@ -801,16 +823,11 @@ impl AFKEndpoint { channel, ty, }; - let qeh_bytes = unsafe { - slice::from_raw_parts( - &qeh as *const QEHeader as *const u8, - mem::size_of::(), - ) - }; - self.memcpy_to_iomem(base + wptr, qeh_bytes)?; + let qeh_bytes = qeh.to_bytes(); + self.memcpy_to_iomem(base + wptr, &qeh_bytes)?; if payload_len > buf_size.checked_sub(wptr + QEH_SIZE).ok_or(EIO)? { wptr = 0; - self.memcpy_to_iomem(base + wptr, qeh_bytes)?; + self.memcpy_to_iomem(base + wptr, &qeh_bytes)?; } self.memcpy_to_iomem(base + wptr + QEH_SIZE, header)?; self.memcpy_to_iomem(base + wptr + QEH_SIZE + header.len(), data)?; @@ -1235,11 +1252,11 @@ impl WorkItem for AopServiceRegisterWork { .and_then(|x| x.get_child_by_name(this.name)); let info = bindings::platform_device_info { parent: this.data.dev.as_raw(), - name: this.name.as_ptr() as *const _, + name: this.name.as_char_ptr(), id: bindings::PLATFORM_DEVID_AUTO, res: ptr::null_mut(), num_res: 0, - data: &this.service as *const EPICService as *const _, + data: ptr::from_ref(&this.service).cast(), size_data: mem::size_of::(), dma_mask: 0, fwnode: fwnode @@ -1484,7 +1501,7 @@ impl AopData { if *k != key { continue; } - arg_bytes[idx..idx + size].copy_from_slice(&(*v as u64).to_le_bytes()[..size]); + arg_bytes[idx..idx + size].copy_from_slice(&v.to_le_bytes()[..size]); break; } idx += size; @@ -2296,6 +2313,7 @@ impl platform::Driver for AopDriver { ); return Err(ENODEV); } + // SAFETY: nothing of this device has DMA in flight before probe. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::new::<42>())? }; let aop_req = pdev.io_request_by_index(0).ok_or(EINVAL)?; let aop_mmio = aop_req.iomap_sized::()?; @@ -2358,6 +2376,8 @@ impl Drop for AopDriver { } } +// SAFETY: the wrapped `Arc` is `Send + Sync` (the trait requires +// both), so the handle may move to the thread that unbinds the device. unsafe impl Send for AopDriver {} module_platform_driver! { From 2c1d690b5f5b3f4452d24e89e0f800fd1242f0d7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 06:59:14 +1100 Subject: [PATCH 1409/2680] soc: apple: aop: Validate the receive ring before taking entries recv_rb() takes the read and write pointers of a receive ring from the shared buffer, which the firmware can write, and checks neither against the ring: - The read pointer is used for a header copy before anything establishes that it lies in the ring; only the check against the whole allocation stops the copy, so it can read from the other ring or the ring header blocks. - The write pointer is only the loop terminator, re-read after every entry. A value the read pointer cannot reach, at or past the end of the ring or off the block grid, keeps the loop dispatching entries forever and holds the RTKit receive worker. - When an entry does not fit before the end of the ring, its header is read again from the top, and only the magic of that second header is checked. Its size sizes the message buffer and the copy, so a bad value asks for an allocation of up to 4 GiB or copies past the ring. - An entry that ends where it started, or a wrapped entry that ends on the read pointer it came from, leaves the read pointer where it was, and the same entry is dispatched again. Move the ring arithmetic into an aop_ring module of pure functions and check, before the first entry and after every re-read of the write pointer, that both pointers are block boundaries inside the ring with room for an entry header. An entry, wrapped or not, must fit in the ring and must move the read pointer. Refuse the ring with EIO otherwise, as for a bad magic, and log the pointers or the header that failed. Also refuse a ring whose size is not a whole number of blocks, which the entry alignment relies on. Bound one pass to the number of blocks in the ring, the most entries it can hold, so that a producer that keeps moving the write pointer cannot hold the worker either; entries beyond that are taken on the doorbell that announces them. The checks have a KUnit suite, apple_aop_ring. Fixes: 5ba018e083e7 ("soc: apple: aop: Bounds-check ring accesses and access pointers volatile") (cherry picked from commit 71e6b6173b9d00a4ffb01924da1c7d77be766676) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 77 +++++++----- drivers/soc/apple/aop_ring.rs | 221 ++++++++++++++++++++++++++++++++++ 2 files changed, 270 insertions(+), 28 deletions(-) create mode 100644 drivers/soc/apple/aop_ring.rs diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 57c3648f957ea..2e3a55899aa6b 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -188,6 +188,8 @@ fn align_up(v: usize, a: usize) -> usize { (v + a - 1) & !(a - 1) } +mod aop_ring; + /// Index into the AFK endpoint tables for RTKit endpoint `ep`, if it is one /// of the endpoints this driver drives. fn afk_endpoint_index(ep: u8) -> Option { @@ -621,43 +623,62 @@ impl AFKEndpoint { } }; let mut rptr = self.iomem_read32(buf_offset + block_size)? as usize; - let mut wptr = self.iomem_read32(buf_offset + block_size * 2)?; + let mut wptr = self.iomem_read32(buf_offset + block_size * 2)? as usize; mem_sync(); let base = buf_offset + block_size * 3; let mut msg_buf = KVec::new(); - while wptr as usize != rptr { - let mut qeh_bytes = [0; QEH_SIZE]; - self.memcpy_from_iomem(base + rptr, &mut qeh_bytes)?; - let mut qeh = QEHeader::from_bytes(&qeh_bytes); - if qeh.magic != QE_MAGIC1 && qeh.magic != QE_MAGIC2 { - let magic = qeh.magic; + let ring = match aop_ring::Ring::new(buf_size, block_size) { + Ok(ring) => ring, + Err(e) => { + dev_err!( + client.dev, + "Invalid ring on ep {}: size {:#x} block {:#x}", + self.index, + buf_size, + block_size + ); + return Err(e); + } + }; + // Entries the producer adds while this pass runs beyond one ring's + // worth are left for the doorbell that announces them. + let mut budget = ring.capacity(); + loop { + if ring.check_ptr(rptr).is_err() || ring.check_ptr(wptr).is_err() { dev_err!( client.dev, - "Invalid magic on ep {}, got {:x}", + "Ring pointers out of range on ep {}: rptr {:#x} wptr {:#x}", self.index, - magic + rptr, + wptr ); return Err(EIO); } - let room = buf_size.checked_sub(rptr + QEH_SIZE).ok_or(EIO)?; - if qeh.size as usize > room { - rptr = 0; - self.memcpy_from_iomem(base + rptr, &mut qeh_bytes)?; - qeh = QEHeader::from_bytes(&qeh_bytes); - - if qeh.magic != QE_MAGIC1 && qeh.magic != QE_MAGIC2 { - let magic = qeh.magic; - dev_err!( - client.dev, - "Invalid magic on ep {}, got {:x}", - self.index, - magic - ); - return Err(EIO); - } + if wptr == rptr || budget == 0 { + break; } + budget -= 1; + let mut last = QEHeader::default(); + let read = ring.read_entry(rptr, |ptr| { + let mut qeh_bytes = [0; QEH_SIZE]; + self.memcpy_from_iomem(base + ptr, &mut qeh_bytes)?; + last = QEHeader::from_bytes(&qeh_bytes); + Ok(last) + }); + let Ok((qeh, entry, next)) = read else { + let (magic, size) = (last.magic, last.size); + dev_err!( + client.dev, + "Invalid entry on ep {} at {:#x}: magic {:x} size {:#x}", + self.index, + rptr, + magic, + size + ); + return Err(EIO); + }; msg_buf.resize(qeh.size as usize, 0, GFP_KERNEL)?; - self.memcpy_from_iomem(base + rptr + QEH_SIZE, &mut msg_buf)?; + self.memcpy_from_iomem(base + entry + QEH_SIZE, &mut msg_buf)?; // A message the endpoint cannot handle is skipped. The ring // position is still good, and not advancing past the entry would // read it again on every doorbell and stall the endpoint for good. @@ -682,10 +703,10 @@ impl AFKEndpoint { } } } - rptr = align_up(rptr + QEH_SIZE + qeh.size as usize, block_size) % buf_size; + rptr = next; mem_sync(); self.iomem_write32(buf_offset + block_size, rptr as u32)?; - wptr = self.iomem_read32(buf_offset + block_size * 2)?; + wptr = self.iomem_read32(buf_offset + block_size * 2)? as usize; mem_sync(); } Ok(()) diff --git a/drivers/soc/apple/aop_ring.rs b/drivers/soc/apple/aop_ring.rs new file mode 100644 index 0000000000000..d854cc221091b --- /dev/null +++ b/drivers/soc/apple/aop_ring.rs @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounds for the entries of an AFK ring. +//! +//! The ring pointers and the entry headers live in memory the firmware can +//! write. These checks keep every entry inside its ring, independent of the +//! shared allocation that also holds the other ring and the header blocks. + +use super::{QEHeader, QEH_SIZE, QE_MAGIC1, QE_MAGIC2}; +use kernel::prelude::*; + +/// A ring of `size` bytes of entries made of `block`-byte blocks. Entries +/// start on a block boundary. +pub(super) struct Ring { + size: usize, + block: usize, +} + +impl Ring { + pub(super) fn new(size: usize, block: usize) -> Result { + if size < QEH_SIZE || !block.is_power_of_two() || size % block != 0 { + return Err(EIO); + } + Ok(Self { size, block }) + } + + /// The most entries the ring can hold, one per block. One receive pass + /// takes at most this many, so that a producer that keeps moving the + /// write pointer cannot hold the receive worker. + pub(super) fn capacity(&self) -> usize { + self.size / self.block + } + + /// Checks a read or write pointer: it must be a block boundary inside + /// the ring with room for an entry header. + pub(super) fn check_ptr(&self, ptr: usize) -> Result { + if ptr >= self.size || ptr % self.block != 0 || self.size - ptr < QEH_SIZE { + return Err(EIO); + } + Ok(()) + } + + /// Returns the pointer after an entry of `len` bytes at `ptr`, if the + /// entry fits in the ring. + fn end(&self, ptr: usize, len: usize) -> Result { + self.check_ptr(ptr)?; + let end = ptr + .checked_add(QEH_SIZE) + .and_then(|v| v.checked_add(len)) + .ok_or(EIO)?; + let aligned = end.checked_add(self.block - 1).ok_or(EIO)? & !(self.block - 1); + if aligned > self.size { + return Err(EIO); + } + Ok(aligned % self.size) + } + + /// Finds the entry at read pointer `ptr`, reading entry headers with + /// `read`. An entry that does not fit before the end of the ring starts + /// again at the top, where its header is read a second time. Returns the + /// header, where the entry is and the read pointer after it. + pub(super) fn read_entry( + &self, + mut ptr: usize, + mut read: impl FnMut(usize) -> Result, + ) -> Result<(QEHeader, usize, usize)> { + self.check_ptr(ptr)?; + let previous = ptr; + let mut header = read(ptr)?; + Self::check_magic(&header)?; + if header.size as usize > self.size - ptr - QEH_SIZE { + ptr = 0; + header = read(ptr)?; + Self::check_magic(&header)?; + } + // The header at the top is firmware-written too: it must fit in the + // ring before it sizes an allocation or a copy. + let next = self.end(ptr, header.size as usize)?; + // An entry that leaves the read pointer where it was would be taken + // again and again. + if next == ptr || next == previous { + return Err(EIO); + } + Ok((header, ptr, next)) + } + + fn check_magic(header: &QEHeader) -> Result { + if header.magic != QE_MAGIC1 && header.magic != QE_MAGIC2 { + return Err(EIO); + } + Ok(()) + } +} + +#[kunit_tests(apple_aop_ring)] +mod tests { + use super::*; + + const BUF: usize = 0x1000; + const BLOCK: usize = 0x40; + + fn header(size: u32) -> QEHeader { + QEHeader { + magic: QE_MAGIC1, + size, + ..QEHeader::default() + } + } + + #[test] + fn invalid_layout_and_alignment_overflow_are_rejected() -> Result { + assert!(Ring::new(0, 64).is_err()); + assert!(Ring::new(256, 0).is_err()); + assert!(Ring::new(256, 3).is_err()); + assert!(Ring::new(255, 64).is_err()); + // no room for an entry header + assert!(Ring::new(QEH_SIZE - 1, 1).is_err()); + let ring = Ring::new(usize::MAX - 63, 64)?; + assert!(ring.end(0, usize::MAX - QEH_SIZE).is_err()); + Ok(()) + } + + #[test] + fn pointers_inside_the_ring_are_accepted() -> Result { + let ring = Ring::new(BUF, BLOCK)?; + for ptr in [0, 0x40, 0xfc0] { + assert!(ring.check_ptr(ptr).is_ok()); + } + Ok(()) + } + + #[test] + fn pointers_at_or_past_the_end_are_refused() -> Result { + let ring = Ring::new(BUF, BLOCK)?; + for ptr in [BUF, u32::MAX as usize, 0x10_0000, usize::MAX] { + assert!(ring.check_ptr(ptr).is_err()); + } + Ok(()) + } + + #[test] + fn misaligned_pointers_are_refused() -> Result { + let ring = Ring::new(BUF, BLOCK)?; + for ptr in [0x10, 0x44, 0x7f, 0xbf] { + assert!(ring.check_ptr(ptr).is_err()); + } + Ok(()) + } + + #[test] + fn invalid_pointers_are_rejected_before_reading() -> Result { + let ring = Ring::new(256, 64)?; + for ptr in [256, 320, usize::MAX, 1, 255] { + let mut reads = 0; + assert!(ring + .read_entry(ptr, |_| { + reads += 1; + Ok(header(0)) + }) + .is_err()); + assert_eq!(reads, 0); + } + Ok(()) + } + + #[test] + fn oversized_wrapped_header_is_rejected() -> Result { + let ring = Ring::new(256, 64)?; + let mut reads = 0; + let result = ring.read_entry(192, |ptr| { + reads += 1; + Ok(header(if ptr == 0 { 256 } else { 64 })) + }); + assert!(result.is_err()); + assert_eq!(reads, 2); + assert!(ring + .read_entry(192, |ptr| Ok(header(if ptr == 0 { u32::MAX } else { 64 }))) + .is_err()); + Ok(()) + } + + #[test] + fn wrapped_entry_stays_in_ring() -> Result { + let ring = Ring::new(256, 64)?; + let (_, ptr, next) = + ring.read_entry(192, |ptr| Ok(header(if ptr == 0 { 48 } else { 64 })))?; + assert_eq!((ptr, next), (0, 64)); + assert_eq!(ring.read_entry(192, |_| Ok(header(48)))?.2, 0); + Ok(()) + } + + #[test] + fn receive_entry_must_advance_the_read_pointer() -> Result { + let ring = Ring::new(256, 64)?; + assert!(ring.read_entry(0, |_| Ok(header(240))).is_err()); + assert!(ring + .read_entry(192, |ptr| Ok(header(if ptr == 0 { 176 } else { 64 }))) + .is_err()); + Ok(()) + } + + #[test] + fn one_pass_takes_at_most_a_ring_of_entries() -> Result { + let ring = Ring::new(BUF, BLOCK)?; + assert_eq!(ring.capacity(), 64); + // A producer that moves the write pointer one block ahead of every + // entry taken never lets the pointers meet; the pass still ends. The + // test itself gives up after two rings' worth. + let (mut rptr, mut wptr, mut taken) = (0, BLOCK, 0); + let mut budget = ring.capacity(); + while wptr != rptr && budget != 0 && taken < 2 * BUF / BLOCK { + budget -= 1; + let (_, _, next) = ring.read_entry(rptr, |_| Ok(header(16)))?; + taken += 1; + rptr = next; + wptr = (wptr + BLOCK) % BUF; + } + assert_eq!(taken, 64); + Ok(()) + } +} From 1b2fdae17b80e40f2cfef74f4ba604138dc40000 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 06:59:56 +1100 Subject: [PATCH 1410/2680] soc: apple: aop: Do not overrun unread entries in the transmit ring write_entry() refuses an entry only when the write pointer sits less than one header below the read pointer. An entry that does not fit before the end of the ring is moved to the top without looking at the read pointer, and one placed below the read pointer is not checked against it either. Either way the host overwrites entries the firmware has not read yet, and an entry that ends exactly on the read pointer makes a full ring read as empty. The pointers read back from the shared buffer are not validated, and an entry larger than the ring is written past it. Work out where the entry goes, and whether it fits, from both pointers: at the write pointer if it fits before the end of the ring, at the top of the ring otherwise, and in either case only if it ends before the read pointer. Validate the pointers first, as the receive path does. A ring without room fails the call with EIO and logs "Tx buffer full" as before. Order the entry before the write pointer that publishes it. Add KUnit cases for the placement to apple_aop_ring. Fixes: 5ba018e083e7 ("soc: apple: aop: Bounds-check ring accesses and access pointers volatile") (cherry picked from commit 13d14a5444c77496e3b0914b3ef833b1f04a9c35) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 40 ++++++++------ drivers/soc/apple/aop_ring.rs | 98 +++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 17 deletions(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 2e3a55899aa6b..6641e1d501362 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -184,10 +184,6 @@ const ALS_CALIBRATION_LEN: usize = 80; const ALS_CALIBRATION_OPERATION: u64 = 0x0746_b8d6_6515_2e31; const ALS_CALIBRATION_BODY_LEN: u64 = 56; -fn align_up(v: usize, a: usize) -> usize { - (v + a - 1) & !(a - 1) -} - mod aop_ring; /// Index into the AFK endpoint tables for RTKit endpoint `ep`, if it is one @@ -832,29 +828,39 @@ impl AFKEndpoint { let base = buf_offset + block_size * 3; mem_sync(); let rptr = self.iomem_read32(buf_offset + block_size)? as usize; - let mut wptr = self.iomem_read32(buf_offset + block_size * 2)? as usize; - if wptr < rptr && wptr + QEH_SIZE >= rptr { - dev_err!(client.dev, "Tx buffer full at endpoint {}", self.index); + let wptr = self.iomem_read32(buf_offset + block_size * 2)? as usize; + let ring = aop_ring::Ring::new(buf_size, block_size)?; + if ring.check_ptr(rptr).is_err() || ring.check_ptr(wptr).is_err() { + dev_err!( + client.dev, + "Tx ring pointers out of range at endpoint {}: rptr {:#x} wptr {:#x}", + self.index, + rptr, + wptr + ); return Err(EIO); } - let payload_len = header.len() + data.len(); + let payload_len = header.len().checked_add(data.len()).ok_or(EIO)?; + let Ok((entry, next)) = ring.write_entry(rptr, wptr, payload_len) else { + dev_err!(client.dev, "Tx buffer full at endpoint {}", self.index); + return Err(EIO); + }; let qeh = QEHeader { magic: QE_MAGIC1, - size: payload_len as u32, + size: u32::try_from(payload_len).map_err(|_| EIO)?, channel, ty, }; let qeh_bytes = qeh.to_bytes(); - self.memcpy_to_iomem(base + wptr, &qeh_bytes)?; - if payload_len > buf_size.checked_sub(wptr + QEH_SIZE).ok_or(EIO)? { - wptr = 0; + if entry != wptr { self.memcpy_to_iomem(base + wptr, &qeh_bytes)?; } - self.memcpy_to_iomem(base + wptr + QEH_SIZE, header)?; - self.memcpy_to_iomem(base + wptr + QEH_SIZE + header.len(), data)?; - wptr = align_up(wptr + QEH_SIZE + payload_len, block_size) % buf_size; - self.iomem_write32(buf_offset + block_size * 2, wptr as u32)?; - Ok(wptr as u64 | (AFK_OPC_SEND << 48)) + self.memcpy_to_iomem(base + entry, &qeh_bytes)?; + self.memcpy_to_iomem(base + entry + QEH_SIZE, header)?; + self.memcpy_to_iomem(base + entry + QEH_SIZE + header.len(), data)?; + mem_sync(); + self.iomem_write32(buf_offset + block_size * 2, next as u32)?; + Ok(next as u64 | (AFK_OPC_SEND << 48)) } fn epic_notify( &mut self, diff --git a/drivers/soc/apple/aop_ring.rs b/drivers/soc/apple/aop_ring.rs index d854cc221091b..ccbb661331e2a 100644 --- a/drivers/soc/apple/aop_ring.rs +++ b/drivers/soc/apple/aop_ring.rs @@ -90,6 +90,43 @@ impl Ring { } Ok(()) } + + /// Places an entry of `len` bytes after its header, given the read and + /// write pointers. It goes at `wptr`, or at the top of the ring when it + /// does not fit before the end (its header at `wptr` then tells the + /// consumer to wrap). It never runs into the unread entries from `rptr` + /// on, and never ends on `rptr`, where a full ring would read as empty. + /// Returns where the entry goes and the write pointer after it. + pub(super) fn write_entry( + &self, + rptr: usize, + wptr: usize, + len: usize, + ) -> Result<(usize, usize)> { + self.check_ptr(rptr)?; + self.check_ptr(wptr)?; + // Reject an entry the whole ring cannot take before even writing + // the wrap header. + let head_next = self.end(0, len)?; + let ptr = if len > self.size - wptr - QEH_SIZE { + 0 + } else { + wptr + }; + let next = if ptr == 0 { + head_next + } else { + self.end(ptr, len)? + }; + if head_next == 0 + || next == rptr + || (wptr < rptr && (ptr == 0 || next == 0 || next > rptr)) + || (ptr < wptr && (rptr == 0 || next > rptr)) + { + return Err(ENOSPC); + } + Ok((ptr, next)) + } } #[kunit_tests(apple_aop_ring)] @@ -159,6 +196,8 @@ mod tests { }) .is_err()); assert_eq!(reads, 0); + assert!(ring.write_entry(ptr, 0, 1).is_err()); + assert!(ring.write_entry(0, ptr, 1).is_err()); } Ok(()) } @@ -218,4 +257,63 @@ mod tests { assert_eq!(taken, 64); Ok(()) } + + #[test] + fn transmit_wrap_checks_whole_entry_and_free_space() -> Result { + let ring = Ring::new(256, 64)?; + assert!(ring.write_entry(128, 192, 256).is_err()); + assert!(ring.write_entry(128, 192, usize::MAX).is_err()); + assert_eq!(ring.write_entry(192, 192, 64)?, (0, 128)); + Ok(()) + } + + #[test] + fn transmit_cannot_catch_read_pointer() -> Result { + let ring = Ring::new(256, 64)?; + assert!(ring.write_entry(64, 192, 64).is_err()); + assert!(ring.write_entry(128, 64, 64).is_err()); + assert!(ring.write_entry(0, 192, 48).is_err()); + assert!(ring.write_entry(192, 64, 176).is_err()); + assert!(ring.write_entry(192, 192, 240).is_err()); + assert_eq!(ring.write_entry(192, 64, 48)?, (64, 128)); + Ok(()) + } + + #[test] + fn an_entry_that_fits_goes_at_the_write_pointer() -> Result { + let ring = Ring::new(BUF, BLOCK)?; + assert_eq!(ring.write_entry(0, 0, 0x100 - QEH_SIZE)?.0, 0); + assert_eq!(ring.write_entry(0x200, 0x100, 0x80 - QEH_SIZE)?.0, 0x100); + assert_eq!(ring.write_entry(0x40, 0xf00, 0x100 - QEH_SIZE)?, (0xf00, 0)); + Ok(()) + } + + #[test] + fn a_wrapping_entry_goes_to_the_top_when_there_is_room() -> Result { + let ring = Ring::new(BUF, BLOCK)?; + assert_eq!( + ring.write_entry(0x200, 0xfc0, 0x100 - QEH_SIZE)?, + (0, 0x100) + ); + Ok(()) + } + + #[test] + fn a_wrapping_entry_never_overruns_unread_entries() -> Result { + let ring = Ring::new(BUF, BLOCK)?; + // the unread entries start at 0x80; the wrapped entry would cover them + assert!(ring.write_entry(0x80, 0xfc0, 0x100 - QEH_SIZE).is_err()); + assert!(ring.write_entry(0, 0xfc0, 0x100 - QEH_SIZE).is_err()); + Ok(()) + } + + #[test] + fn an_entry_never_reaches_the_read_pointer() -> Result { + let ring = Ring::new(BUF, BLOCK)?; + // ending on rptr would make a full ring read as empty + assert!(ring.write_entry(0x200, 0x100, 0x100 - QEH_SIZE).is_err()); + assert!(ring.write_entry(0x200, 0x100, 0x110 - QEH_SIZE).is_err()); + assert!(ring.write_entry(0, 0xf00, 0x100 - QEH_SIZE).is_err()); + Ok(()) + } } From d13e32ebafa3ea6e4ff2352072a6f7d39d851bc0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:46:24 +1100 Subject: [PATCH 1411/2680] ASoC: apple: t8140-aop-audio: Refuse card names that do not fit The card's id, short and long names are built from the apple,chassis-name and apple,machine-kind properties and copied into ALSA's fixed-size fields with a slice copy. A source longer than its field (the id has room for 15 characters, so any chassis name over seven characters) indexes out of range, which is a Rust panic and an oops in probe. Make copy_str() return EINVAL when the string does not fit and fail the probe with an error message instead. The names on the J700 fit, so nothing changes there. Add a KUnit suite for the copy: a name that fits is copied with its NUL, and one longer than the field is refused and leaves the field untouched. Fixes: eeb112648f2a ("ASoC: apple: Add T8140 AOP audio driver") (cherry picked from commit 10640d1dd059e9e54aa368b2925f31f1b1eb880d) Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 58 +++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index 596f717d02b36..d024293997114 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -1550,8 +1550,14 @@ const PCM_PERIOD_BYTES_MAX: usize = 0x10000; /// 16384 frames (341 ms), while still allowing the largest PipeWire quantum. const SPKR_BUFFER_MAX: usize = 128 * 1024; -fn copy_str(target: &mut [u8], source: &[u8]) { - target[..source.len()].copy_from_slice(source) +/// Copies the NUL-terminated `source` into the fixed-size C string field +/// `target`; EINVAL if it does not fit. +fn copy_str(target: &mut [u8], source: &[u8]) -> Result { + target + .get_mut(..source.len()) + .ok_or(EINVAL)? + .copy_from_slice(source); + Ok(()) } // ---- ASoC component: DPCM front-end PCMs on the ADMAC channels, back-end @@ -2604,16 +2610,19 @@ impl SndSocT8140AopDriver { let id_str = CString::try_from_fmt(fmt!("Apple{}AOP", *chassis))?; let shortname = CString::try_from_fmt(fmt!("{} {} AOP", *machine_kind, *chassis))?; let longname = CString::try_from_fmt(fmt!("{} {} AOP Audio", *machine_kind, *chassis))?; - // SAFETY: `card` was created by `snd_card_new` above; the fixed-size name fields are filled - // with NUL-terminated strings that fit. - unsafe { - copy_str( - &mut (*this.card).driver, - c"t8140-aop-audio".to_bytes_with_nul(), + // SAFETY: `card` was created by `snd_card_new` above and is not + // registered yet, so nothing else accesses it. + let card = unsafe { &mut *this.card }; + let named = copy_str(&mut card.driver, c"t8140-aop-audio".to_bytes_with_nul()) + .and_then(|()| copy_str(&mut card.id, id_str.to_bytes_with_nul())) + .and_then(|()| copy_str(&mut card.shortname, shortname.to_bytes_with_nul())) + .and_then(|()| copy_str(&mut card.longname, longname.to_bytes_with_nul())); + if let Err(e) = named { + dev_err!( + data.dev, + "apple,chassis-name or apple,machine-kind too long for the card names\n" ); - copy_str(&mut (*this.card).id, id_str.to_bytes_with_nul()); - copy_str(&mut (*this.card).shortname, shortname.to_bytes_with_nul()); - copy_str(&mut (*this.card).longname, longname.to_bytes_with_nul()); + return Err(e); } this.new_pcm(0, c"lpai", c"Low-Power Microphone", &Self::LPAI_OPS)?; @@ -2664,14 +2673,15 @@ impl SndSocT8140AopDriver { return Err(Error::from_errno(ret)); } // SAFETY: `pcm` was created by `snd_pcm_new` above; the ops are static and the private - // data is an `Arc` released by `pcm_free_private`; `name` fits the fixed-size field. + // data is an `Arc` released by `pcm_free_private`. unsafe { bindings::snd_pcm_set_ops(pcm, bindings::SNDRV_PCM_STREAM_CAPTURE as i32, ops); (*pcm).private_data = self.data.clone().into_foreign().cast(); (*pcm).private_free = Some(pcm_free_private); (*pcm).info_flags = 0; - copy_str(&mut (*pcm).name, name.to_bytes_with_nul()); } + // SAFETY: as above. + copy_str(unsafe { &mut (*pcm).name }, name.to_bytes_with_nul())?; if device == 0 { // The low-power microphone's buffer is filled by the driver from // the source ring; the array's is set up on its DMA device at @@ -2961,3 +2971,25 @@ mod tests { assert_eq!(f32_bits_of_q31(i32::MIN), 0xbf80_0000); // -1.0 } } + +/// The card and PCM names are copied into ALSA's fixed-size fields from +/// device-tree strings. +#[kunit_tests(snd_soc_t8140_aop_audio_names)] +mod name_tests { + use super::*; + + #[test] + fn a_name_that_fits_is_copied_with_its_nul() { + let mut field = [0xffu8; 16]; + assert!(copy_str(&mut field, c"AppleJ700AOP".to_bytes_with_nul()).is_ok()); + assert_eq!(&field[..13], b"AppleJ700AOP\0"); + } + + #[test] + fn a_name_longer_than_its_field_is_refused() { + let mut field = [0u8; 16]; + let long = c"AppleJ700LongerChassisAOP".to_bytes_with_nul(); + assert!(copy_str(&mut field, long).is_err()); + assert_eq!(field, [0u8; 16]); + } +} From 684164dcd3125b733a895556155b06297578e757 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:46:25 +1100 Subject: [PATCH 1412/2680] ASoC: apple: t8140-aop-audio: Let power request sequences wrap The power request sequence numbers are taken with fetch_add(1) + 1. The atomic add wraps, but the "+ 1" on its result does not: with CONFIG_RUST_OVERFLOW_CHECKS=y the request after u32::MAX of them panics instead of sending sequence 0. Use wrapping_add() so the counter wraps like the firmware's u32 field. This takes billions of requests, so it is not a practical problem, but the panic serves no purpose. Fixes: eeb112648f2a ("ASoC: apple: Add T8140 AOP audio driver") (cherry picked from commit 53d54b3c1b67bc10145b8291de74634da0043e47) Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index d024293997114..b0362e9b1e580 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -664,7 +664,7 @@ impl SndSocT8140AopData { } fn set_power(&self, state: u32) -> Result<()> { - let sequence = self.sequence.fetch_add(1, Relaxed) + 1; + let sequence = self.sequence.fetch_add(1, Relaxed).wrapping_add(1); let req = PowerRequest::new(0, sequence, state, 1)?; let ret = self.epic_wrapped_call(&AudioSetDeviceProp::new( AUDIO_DEV_LPAI, @@ -732,7 +732,7 @@ impl SndSocT8140AopData { /// Property 202 with the service's sequence and a 24 MHz timestamp; /// `pw0 ` is confirmed through property 200 like the native host does. fn service_set_power(&self, svc: &Service, state: u32) -> Result<()> { - let sequence = svc.sequence.fetch_add(1, Relaxed) + 1; + let sequence = svc.sequence.fetch_add(1, Relaxed).wrapping_add(1); let req = PowerRequest::new(svc.dev_id, sequence, state, 0)?; let ret = self.epic_wrapped_call(&AudioSetDeviceProp::new( svc.dev_id, From 688bf977a6a731fd438bdf93ae188a2de6cd23c1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:41:39 +1100 Subject: [PATCH 1413/2680] ASoC: apple: Test T8140 AOP producer reports with KUnit Separate producer position decoding and cursor validation from stream handling so that malformed reports can be checked without an audio device. Keep the existing counter checks and publish report readiness before cursor validation as before. Cover short fields and reports, unaligned little-endian values, addition and multiplication overflow, and cursors outside the owned ring. (cherry picked from commit ac29a1cd288091504adddb7cee3bd451a5068ede) Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 142 +++++++++++++++++++++++++---- 1 file changed, 123 insertions(+), 19 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index b0362e9b1e580..d181bf206e06c 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -1072,33 +1072,54 @@ fn copy_span( Ok(dst) } +#[derive(Debug, PartialEq, Eq)] +enum ProducerReportError { + Short, + Overflow, +} + +fn producer_position(report: &[u8]) -> core::result::Result<(u64, u64), ProducerReportError> { + if report.len() < REPORT_MIN_LEN { + return Err(ProducerReportError::Short); + } + let count = le_u64(report, 0x40).map_err(|_| ProducerReportError::Short)?; + let absolute = count + .checked_add(1) + .and_then(|frames| frames.checked_mul(LPAI_FRAME_BYTES as u64)) + .ok_or(ProducerReportError::Overflow)?; + let cursor = le_u64(report, 0x60).map_err(|_| ProducerReportError::Short)?; + Ok((absolute, cursor)) +} + +fn producer_cursor_matches(absolute: u64, cursor: u64, ring_bytes: usize) -> bool { + ring_bytes != 0 && cursor == absolute % ring_bytes as u64 +} + impl ReportListener for SndSocT8140AopData { fn process_report(&self, _subtype: u16, report: &[u8]) -> Result<()> { - if report.len() < REPORT_MIN_LEN { - if !self.report_warnings[0].xchg(true, Relaxed) { - dev_warn!( - self.dev, - "short lpai producer report ({} bytes)\n", - report.len() - ); + let (absolute, cursor) = match producer_position(report) { + Ok(position) => position, + Err(ProducerReportError::Short) => { + if !self.report_warnings[0].xchg(true, Relaxed) { + dev_warn!( + self.dev, + "short lpai producer report ({} bytes)\n", + report.len() + ); + } + return Ok(()); } - return Ok(()); - } - let count = le_u64(report, 0x40)?; - let Some(absolute) = count - .checked_add(1) - .and_then(|frames| frames.checked_mul(LPAI_FRAME_BYTES as u64)) - else { - if !self.report_warnings[1].xchg(true, Relaxed) { - dev_warn!(self.dev, "overflowing lpai producer counter\n"); + Err(ProducerReportError::Overflow) => { + if !self.report_warnings[1].xchg(true, Relaxed) { + dev_warn!(self.dev, "overflowing lpai producer counter\n"); + } + return Ok(()); } - return Ok(()); }; let frames = absolute / LPAI_FRAME_BYTES as u64; self.reported.store(true, Relaxed); - let cursor = le_u64(report, 0x60)?; let ring_bytes = self.ring_bytes.load(Relaxed) as usize; - if ring_bytes == 0 || cursor != absolute % ring_bytes as u64 { + if !producer_cursor_matches(absolute, cursor, ring_bytes) { if !self.report_warnings[2].xchg(true, Relaxed) { dev_warn!( self.dev, @@ -2993,3 +3014,86 @@ mod name_tests { assert_eq!(field, [0u8; 16]); } } + +#[kunit_tests(snd_soc_t8140_aop_audio_parser)] +mod parser_tests { + use super::*; + + #[test] + fn short_reply_fields_are_rejected() { + let data = [0u8; REPORT_MIN_LEN]; + for len in 0..4 { + assert_eq!(le_u32(&data[..len], 0), Err(EINVAL)); + } + for len in 0..8 { + assert_eq!(le_u64(&data[..len], 0), Err(EINVAL)); + } + for len in 0..REPORT_MIN_LEN { + assert_eq!(le_u64(&data[..len], 0x60), Err(EINVAL)); + } + } + + #[test] + fn unaligned_fields_at_the_end_of_a_reply_are_decoded() { + let data = [0xff, 1, 2, 3, 4, 5, 6, 7, 8]; + assert_eq!(le_u32(&data, 5), Ok(0x0807_0605)); + assert_eq!(le_u64(&data, 1), Ok(0x0807_0605_0403_0201)); + assert_eq!(le_u32(&data, 6), Err(EINVAL)); + assert_eq!(le_u64(&data, 2), Err(EINVAL)); + } + + fn producer_report(count: u64, cursor: u64) -> [u8; REPORT_MIN_LEN] { + let mut report = [0u8; REPORT_MIN_LEN]; + report[0x40..0x48].copy_from_slice(&count.to_le_bytes()); + report[0x60..0x68].copy_from_slice(&cursor.to_le_bytes()); + report + } + + #[test] + fn short_producer_reports_are_rejected() { + let report = producer_report(3199, LPAI_PERIOD_BYTES as u64); + for len in 0..REPORT_MIN_LEN { + assert_eq!( + producer_position(&report[..len]), + Err(ProducerReportError::Short) + ); + } + assert_eq!( + producer_position(&report), + Ok((LPAI_PERIOD_BYTES as u64, LPAI_PERIOD_BYTES as u64)) + ); + } + + #[test] + fn overflowing_producer_positions_are_rejected() { + for count in [u64::MAX, u64::MAX / LPAI_FRAME_BYTES as u64] { + assert_eq!( + producer_position(&producer_report(count, 0)), + Err(ProducerReportError::Overflow) + ); + } + let count = u64::MAX / LPAI_FRAME_BYTES as u64 - 1; + assert_eq!( + producer_position(&producer_report(count, 0)), + Ok(((count + 1) * LPAI_FRAME_BYTES as u64, 0)) + ); + } + + #[test] + fn producer_cursors_must_match_the_owned_ring() { + assert!(producer_cursor_matches(RING_BYTES as u64, 0, RING_BYTES)); + assert!(producer_cursor_matches( + RING_BYTES as u64 + 8, + 8, + RING_BYTES + )); + assert!(!producer_cursor_matches(8, 8, 0)); + assert!(!producer_cursor_matches( + 8, + RING_BYTES as u64 + 8, + RING_BYTES + )); + assert!(!producer_cursor_matches(8, u64::MAX, RING_BYTES)); + assert!(!producer_cursor_matches(8, 7, RING_BYTES)); + } +} From 3f3133fab3b6839924e8c552c576c9ed121bc271 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 07:27:24 +1100 Subject: [PATCH 1414/2680] soc: apple: sep: Refuse a damaged host-state store instead of erasing it Store::open() initialises the host-state store whenever the file is shorter than the store or its header does not match: it zeroes all 72 blocks and writes an empty superblock. That was by design while the store held records that a seed import restored after a reset. Since the SEP's anti-replay records moved to the shared xART gigalocker, though, the file holds Linux state that nothing regenerates: the Touch ID identity index and the lockout state. A truncated file, a damaged header or a store in a format this driver does not know is now erased on the next probe, together with the evidence needed to recover it. Initialise the store only when this open created the file and it is empty, and refuse an existing file that is short or has an unknown header without writing to it. That includes what an interrupted creation leaves: in general the driver cannot tell it from a damaged store, so the decision is the administrator's. The log line says when the file is empty or all zero: that is all an interrupted creation leaves, and such a file holds nothing to keep. Creating a new store is logged too: on a machine that has used Touch ID it means the old store was removed. Opening also has two ways to lose or leak a record of a store with a valid header. A pending write intent that names a used block makes it free that block, discarding the committed record, and a record length beyond its 32 KiB block makes read() allocate up to 4 GiB and return the following blocks. No writer of the format produces either; refuse them as well instead of acting on them. The on-disk format does not change. Fixes: 29fd07c333c0 ("soc: apple: use shared xART gigalocker") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/shim.h | 1 + drivers/soc/apple/shim.rs | 16 ++++++- drivers/soc/apple/store.rs | 84 ++++++++++++++++++++++++++++++++-- drivers/soc/apple/store_shim.c | 17 +++++++ 4 files changed, 113 insertions(+), 5 deletions(-) diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index f43e038fe4205..83839b9353691 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -42,6 +42,7 @@ void sep_hwrng_unregister(void *mem); /* -- store_shim.c ------------------------------------------------------- */ void *sep_store_open(const char *path); +void *sep_store_open_host(const char *path, bool *created); void *sep_store_open_trunc(const char *path); void *sep_store_open_ro(const char *path); void *sep_store_open_block(const char *path, int writable); diff --git a/drivers/soc/apple/shim.rs b/drivers/soc/apple/shim.rs index 4e25060d15d25..7e7f489e59481 100644 --- a/drivers/soc/apple/shim.rs +++ b/drivers/soc/apple/shim.rs @@ -7,6 +7,7 @@ use kernel::prelude::*; extern "C" { fn sep_store_open(path: *const c_char) -> *mut c_void; + fn sep_store_open_host(path: *const c_char, created: *mut bool) -> *mut c_void; fn sep_store_open_trunc(path: *const c_char) -> *mut c_void; fn sep_store_open_ro(path: *const c_char) -> *mut c_void; fn sep_store_open_block(path: *const c_char, writable: c_int) -> *mut c_void; @@ -29,7 +30,7 @@ pub(crate) fn random_bytes(buf: &mut [u8]) -> Result<()> { /// Backing-store file handle. /// /// # Invariants -/// `handle` is a live `struct file *` from `sep_store_open`. +/// `handle` is a live `struct file *` from one of the store-open shims. pub(crate) struct StoreFile { handle: *mut c_void, } @@ -56,6 +57,19 @@ impl StoreFile { Ok(StoreFile { handle }) } + /// Opens the file like [`StoreFile::open`], and also reports whether this + /// call created it. + pub(crate) fn open_host(path: &CStr) -> Result<(StoreFile, bool)> { + let mut created = false; + // SAFETY: `path` is NUL-terminated and `created` is valid for writes + // for the duration of the call. The shim returns a live file or an + // error pointer. + let handle = unsafe { + kernel::error::from_err_ptr(sep_store_open_host(path.as_char_ptr(), &mut created))? + }; + Ok((StoreFile { handle }, created)) + } + pub(crate) fn open_trunc(path: &CStr) -> Result { // SAFETY: `path` is NUL-terminated; the shim returns NULL on failure. let handle = unsafe { sep_store_open_trunc(path.as_char_ptr()) }; diff --git a/drivers/soc/apple/store.rs b/drivers/soc/apple/store.rs index dc37025ce3b8a..03bd6f1bf0a01 100644 --- a/drivers/soc/apple/store.rs +++ b/drivers/soc/apple/store.rs @@ -73,6 +73,20 @@ const SLOT_ENTRY_SIZE: usize = 24; const INTENT_NONE: u8 = 0; const INTENT_WRITE: u8 = 1; +/// Why an existing file is refused. It is logged, so it never carries record +/// contents. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Damage { + /// The file is shorter than the store. + Size, + /// The magic, version or geometry is not this format's. + Header, + /// The slot's record is longer than its block. + Slot(usize), + /// The pending write intent names a block that holds a committed record. + Intent, +} + // Not internally locked; the caller holds a mutex around all access. pub(crate) struct Store { file: shim::StoreFile, @@ -92,9 +106,34 @@ fn le64(buf: &[u8], off: usize) -> u64 { u64::from_le_bytes(v) } +/// Describes an existing file that holds no store as "empty" or "all zero". +/// That is all an interrupted creation leaves, and such a file holds nothing +/// to keep; anything else may be what is left of a store. Reads at most a +/// store's worth of the file. +fn describe(file: &shim::StoreFile, size: u64) -> Option<&'static str> { + if size == 0 { + return Some("empty"); + } + if size > STORE_SIZE as u64 { + return None; + } + let mut buf = KVec::new(); + buf.resize(BLOCK_SIZE, 0, GFP_KERNEL).ok()?; + let mut off = 0; + while off < size { + let len = (size - off).min(BLOCK_SIZE as u64) as usize; + file.read_exact(off, &mut buf[..len]).ok()?; + if buf[..len].iter().any(|&b| b != 0) { + return None; + } + off += len as u64; + } + Some("all zero") +} + impl Store { pub(crate) fn open() -> Result { - let file = shim::StoreFile::open(STORE_PATH)?; + let (file, created) = shim::StoreFile::open_host(STORE_PATH)?; let mut store = Store { file, @@ -104,11 +143,18 @@ impl Store { fresh: false, }; + // Only a new, empty file is initialised. An existing file may hold the + // only copy of the host state, so one that this driver cannot use is + // refused and left as it is. let size = store.file.size()?; - if size < STORE_SIZE as u64 { + if created && size == 0 { store.create()?; + pr_info!("created a new host-state store\n"); return Ok(store); } + if size < STORE_SIZE as u64 { + return Err(store.refuse(Damage::Size, size)); + } let mut sb = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; sb.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; @@ -119,8 +165,7 @@ impl Store { || le32(&sb, SB_SLOT_COUNT) as usize != SLOT_COUNT || le32(&sb, SB_BLOCK_SIZE) as usize != BLOCK_SIZE { - store.create()?; - return Ok(store); + return Err(store.refuse(Damage::Header, size)); } store.generation = le64(&sb, SB_GENERATION); @@ -134,6 +179,10 @@ impl Store { uuid, len: le32(&sb, off + 18), }; + // A longer record would be read from the blocks of the next slots. + if store.slots[i].used && store.slots[i].len as usize > MAX_VALUE { + return Err(store.refuse(Damage::Slot(i), size)); + } } // Crash recovery: writes are copy-on-write, so discard the intent, never complete it. @@ -142,6 +191,12 @@ impl Store { store.recovered = true; let slot = u16::from_le_bytes([sb[SB_INTENT_SLOT], sb[SB_INTENT_SLOT + 1]]) as usize; if intent_kind == INTENT_WRITE && slot < SLOT_COUNT { + // The intent names the free block that was being filled. One + // that names a committed record is damage, and freeing that + // block would discard the record. + if store.slots[slot].used { + return Err(store.refuse(Damage::Intent, size)); + } store.slots[slot] = Slot::FREE; } store.commit()?; @@ -150,6 +205,27 @@ impl Store { Ok(store) } + /// Logs why the existing file is refused, and returns the error. Nothing + /// is written to the file. + fn refuse(&self, damage: Damage, size: u64) -> Error { + let what = match damage { + Damage::Size | Damage::Header => describe(&self.file, size), + _ => None, + }; + match what { + Some(what) => pr_err!( + "host-state store failed validation ({:?}, {}), refusing it\n", + damage, + what + ), + None => pr_err!( + "host-state store failed validation ({:?}), refusing it\n", + damage + ), + } + EIO + } + fn create(&mut self) -> Result<()> { self.slots = [Slot::FREE; SLOT_COUNT]; self.generation = 1; diff --git a/drivers/soc/apple/store_shim.c b/drivers/soc/apple/store_shim.c index bdfd5dfbb6391..4c89f92eb5f13 100644 --- a/drivers/soc/apple/store_shim.c +++ b/drivers/soc/apple/store_shim.c @@ -51,6 +51,23 @@ void *sep_store_open(const char *path) return IS_ERR(f) ? NULL : f; } +/* + * As sep_store_open(), but reports whether this call created the file, so that + * only a new file is ever initialised. Returns an error pointer on failure. + */ +void *sep_store_open_host(const char *path, bool *created) +{ + struct file *f; + + *created = false; + f = open_as_kernel(path, O_RDWR | O_CREAT | O_EXCL | O_LARGEFILE, 0600); + if (PTR_ERR_OR_ZERO(f) == -EEXIST) + return open_as_kernel(path, O_RDWR | O_LARGEFILE, 0); + if (!IS_ERR(f)) + *created = true; + return f; +} + /* * Opens truncated to empty, for callers that rewrite the whole file each time, * so a shorter record cannot leave stale trailing bytes from a larger one. From d5dc6eeb3761b8971fd21fd1a63ec54bf4d5abeb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 07:27:25 +1100 Subject: [PATCH 1415/2680] soc: apple: sep: Make the host store's backing I/O replaceable Reach the backing file through a small trait and make the store generic over it, with the kernel file as the default. The driver does not change; tests can then exercise the on-disk format, interrupted writes and damaged files in memory, without a filesystem or a SEP. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/store.rs | 40 ++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/drivers/soc/apple/store.rs b/drivers/soc/apple/store.rs index 03bd6f1bf0a01..12a7429f51e38 100644 --- a/drivers/soc/apple/store.rs +++ b/drivers/soc/apple/store.rs @@ -87,9 +87,36 @@ enum Damage { Intent, } +/// Backing I/O of a [`Store`], separate so that the format handling can be +/// tested without a filesystem. +pub(crate) trait StoreIo { + fn size(&self) -> Result; + fn read_exact(&self, off: u64, buf: &mut [u8]) -> Result; + fn write_all(&self, off: u64, buf: &[u8]) -> Result; + fn sync(&self) -> Result; +} + +impl StoreIo for shim::StoreFile { + fn size(&self) -> Result { + shim::StoreFile::size(self) + } + + fn read_exact(&self, off: u64, buf: &mut [u8]) -> Result { + shim::StoreFile::read_exact(self, off, buf) + } + + fn write_all(&self, off: u64, buf: &[u8]) -> Result { + shim::StoreFile::write_all(self, off, buf) + } + + fn sync(&self) -> Result { + shim::StoreFile::sync(self) + } +} + // Not internally locked; the caller holds a mutex around all access. -pub(crate) struct Store { - file: shim::StoreFile, +pub(crate) struct Store { + file: F, slots: [Slot; SLOT_COUNT], generation: u64, pub(crate) recovered: bool, @@ -110,7 +137,7 @@ fn le64(buf: &[u8], off: usize) -> u64 { /// That is all an interrupted creation leaves, and such a file holds nothing /// to keep; anything else may be what is left of a store. Reads at most a /// store's worth of the file. -fn describe(file: &shim::StoreFile, size: u64) -> Option<&'static str> { +fn describe(file: &F, size: u64) -> Option<&'static str> { if size == 0 { return Some("empty"); } @@ -132,9 +159,14 @@ fn describe(file: &shim::StoreFile, size: u64) -> Option<&'static str> { } impl Store { - pub(crate) fn open() -> Result { + pub(crate) fn open() -> Result { let (file, created) = shim::StoreFile::open_host(STORE_PATH)?; + Self::from_file(file, created) + } +} +impl Store { + fn from_file(file: F, created: bool) -> Result { let mut store = Store { file, slots: [Slot::FREE; SLOT_COUNT], From 23b782092a3d7eb256156c7fad75fd6c746f6840 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 07:27:25 +1100 Subject: [PATCH 1416/2680] soc: apple: sep: Validate the whole host-store superblock before use Store::open() checks the header, the record lengths and the block a write intent names, and takes the rest of the superblock as it is. A used slot can carry an unknown kind, or a key that another slot also has, so that one record shadows the other. Bytes the format leaves zero can hold anything, and an intent of an unknown type is cleared with a write. Decode the slot table and the intent first, and use them only if they are what the writer of this format produces: zero in every byte it does not use, free slots that are all zero, used slots with a known kind, a length that fits the slot's block and a key no other slot has, and either no intent or a write intent that names a free block. The file must also be exactly the size of the store. Anything else is refused without writing, and every open of such a file logs the reason. The format does not change, so a store written by an earlier kernel opens in place, and an earlier kernel opens what this one writes. Opening no longer writes. An interrupted copy-on-write leaves an intent naming a block that is still free in the committed table, and the next write replaces it, so clearing it at open gained nothing. Drop the recovered and fresh flags, which nothing reads. The slot table is rewritten in place, so the format relies on the device writing the first 4 KiB of the superblock atomically, as the NVMe storage of Apple silicon Macs does with its 4 KiB logical blocks. Say so where the layout is defined, and check at build time that every field a reader uses lies in that range. Fixes: 24d20c9828b8 ("soc: apple: add Apple SEP driver (Touch ID and key sealing)") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 5 +- drivers/soc/apple/store.rs | 549 +++++++++++++++++++++++++++++++++---- 2 files changed, 496 insertions(+), 58 deletions(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 558626f0be6be..13ca68d633474 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -160,8 +160,9 @@ config APPLE_SEP_KUNIT_TEST help Build KUnit tests for the pure logic of the Apple SEP driver: the mailbox message encoding and decoding, the shared-memory registration - limits and the rules that keep a keybag's lookup identity apart from - the identity the enclave reports. The tests touch no hardware. + limits, the rules that keep a keybag's lookup identity apart from + the identity the enclave reports, and the format and validation of + the Linux host-state store. The tests touch no hardware. If unsure, say N. diff --git a/drivers/soc/apple/store.rs b/drivers/soc/apple/store.rs index 12a7429f51e38..94fc976275405 100644 --- a/drivers/soc/apple/store.rs +++ b/drivers/soc/apple/store.rs @@ -67,23 +67,40 @@ const SB_GENERATION: usize = 28; const SB_INTENT_KIND: usize = 36; const SB_INTENT_SLOT: usize = 37; const SB_INTENT_TYPE: usize = 39; +const SB_RESERVED: usize = 40; const SB_SLOT_TABLE: usize = 64; const SLOT_ENTRY_SIZE: usize = 24; +const SB_SLOT_TABLE_END: usize = SB_SLOT_TABLE + SLOT_COUNT * SLOT_ENTRY_SIZE; + +/// The superblock is rewritten in place, so the format relies on the device +/// writing at least this much of it atomically: a torn write of the slot table +/// would mix entries of two tables. Every field a reader uses lies within it. +/// NVMe storage on Apple silicon has 4 KiB logical blocks. +const ATOMIC_WRITE: usize = 4096; +static_assert!(SB_SLOT_TABLE_END <= ATOMIC_WRITE); const INTENT_NONE: u8 = 0; const INTENT_WRITE: u8 = 1; -/// Why an existing file is refused. It is logged, so it never carries record +fn valid_kind(kind: u8) -> bool { + (0xf0..=0xf5).contains(&kind) +} + +/// Why an existing store is refused. It is logged, so it never carries record /// contents. #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum Damage { - /// The file is shorter than the store. + /// The file is not exactly [`STORE_SIZE`] bytes long. Size, /// The magic, version or geometry is not this format's. Header, - /// The slot's record is longer than its block. + /// A byte outside the header fields and the slot table is not zero. + Reserved, + /// The slot entry is neither all zero (free) nor a valid record. Slot(usize), - /// The pending write intent names a block that holds a committed record. + /// The slot repeats the key of an earlier slot. + Duplicate(usize), + /// The pending intent is not a write into a free slot. Intent, } @@ -119,8 +136,6 @@ pub(crate) struct Store { file: F, slots: [Slot; SLOT_COUNT], generation: u64, - pub(crate) recovered: bool, - pub(crate) fresh: bool, } fn le32(buf: &[u8], off: usize) -> u32 { @@ -158,6 +173,72 @@ fn describe(file: &F, size: u64) -> Option<&'static str> { Some("all zero") } +/// Decodes the slot table of the superblock `sb` into `slots`. +/// +/// Accepts exactly what this format's writer produces: the version 1 header, +/// zero in every byte it does not use, free slots that are all zero, used slots +/// with a known kind, a length that fits the slot's block and a key no other +/// slot has, and either no intent or a write intent naming a free slot. +fn decode(sb: &[u8], slots: &mut [Slot; SLOT_COUNT]) -> Result<(), Damage> { + if sb[SB_MAGIC..SB_MAGIC + 16] != MAGIC + || le32(sb, SB_VERSION) != VERSION + || le32(sb, SB_SLOT_COUNT) as usize != SLOT_COUNT + || le32(sb, SB_BLOCK_SIZE) as usize != BLOCK_SIZE + { + return Err(Damage::Header); + } + if sb[SB_RESERVED..SB_SLOT_TABLE].iter().any(|&b| b != 0) + || sb[SB_SLOT_TABLE_END..BLOCK_SIZE].iter().any(|&b| b != 0) + { + return Err(Damage::Reserved); + } + + for i in 0..SLOT_COUNT { + let off = SB_SLOT_TABLE + i * SLOT_ENTRY_SIZE; + let entry = &sb[off..off + SLOT_ENTRY_SIZE]; + if entry.iter().all(|&b| b == 0) { + slots[i] = Slot::FREE; + continue; + } + let len = le32(entry, 18); + if entry[0] != 1 + || !valid_kind(entry[1]) + || len as usize > MAX_VALUE + || entry[22..].iter().any(|&b| b != 0) + { + return Err(Damage::Slot(i)); + } + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&entry[2..18]); + let key = Key { + kind: entry[1], + uuid, + }; + if slots[..i].iter().any(|slot| slot.matches(&key)) { + return Err(Damage::Duplicate(i)); + } + slots[i] = Slot { + used: true, + kind: key.kind, + uuid, + len, + }; + } + + let slot = u16::from_le_bytes([sb[SB_INTENT_SLOT], sb[SB_INTENT_SLOT + 1]]) as usize; + let intent_valid = match sb[SB_INTENT_KIND] { + INTENT_NONE => slot == 0 && sb[SB_INTENT_TYPE] == 0, + // The intent is written along with the committed table, in which the + // block it names is still free. + INTENT_WRITE => slot < SLOT_COUNT && !slots[slot].used && valid_kind(sb[SB_INTENT_TYPE]), + _ => false, + }; + if !intent_valid { + return Err(Damage::Intent); + } + Ok(()) +} + impl Store { pub(crate) fn open() -> Result { let (file, created) = shim::StoreFile::open_host(STORE_PATH)?; @@ -171,12 +252,10 @@ impl Store { file, slots: [Slot::FREE; SLOT_COUNT], generation: 0, - recovered: false, - fresh: false, }; // Only a new, empty file is initialised. An existing file may hold the - // only copy of the host state, so one that this driver cannot use is + // only copy of the host state, so one that does not validate is // refused and left as it is. let size = store.file.size()?; if created && size == 0 { @@ -184,56 +263,23 @@ impl Store { pr_info!("created a new host-state store\n"); return Ok(store); } - if size < STORE_SIZE as u64 { - return Err(store.refuse(Damage::Size, size)); - } let mut sb = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; sb.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; - store.file.read_exact(0, &mut sb)?; - - if sb[SB_MAGIC..SB_MAGIC + 16] != MAGIC - || le32(&sb, SB_VERSION) != VERSION - || le32(&sb, SB_SLOT_COUNT) as usize != SLOT_COUNT - || le32(&sb, SB_BLOCK_SIZE) as usize != BLOCK_SIZE - { - return Err(store.refuse(Damage::Header, size)); + let decoded = if size == STORE_SIZE as u64 { + store.file.read_exact(0, &mut sb)?; + decode(&sb, &mut store.slots) + } else { + Err(Damage::Size) + }; + if let Err(damage) = decoded { + return Err(store.refuse(damage, size)); } + // Writes are copy-on-write, so an interrupted one is discarded, never + // completed. Its intent names a free block and the next write replaces + // it, so opening writes nothing. store.generation = le64(&sb, SB_GENERATION); - for i in 0..SLOT_COUNT { - let off = SB_SLOT_TABLE + i * SLOT_ENTRY_SIZE; - let mut uuid = [0u8; 16]; - uuid.copy_from_slice(&sb[off + 2..off + 18]); - store.slots[i] = Slot { - used: sb[off] != 0, - kind: sb[off + 1], - uuid, - len: le32(&sb, off + 18), - }; - // A longer record would be read from the blocks of the next slots. - if store.slots[i].used && store.slots[i].len as usize > MAX_VALUE { - return Err(store.refuse(Damage::Slot(i), size)); - } - } - - // Crash recovery: writes are copy-on-write, so discard the intent, never complete it. - let intent_kind = sb[SB_INTENT_KIND]; - if intent_kind != INTENT_NONE { - store.recovered = true; - let slot = u16::from_le_bytes([sb[SB_INTENT_SLOT], sb[SB_INTENT_SLOT + 1]]) as usize; - if intent_kind == INTENT_WRITE && slot < SLOT_COUNT { - // The intent names the free block that was being filled. One - // that names a committed record is damage, and freeing that - // block would discard the record. - if store.slots[slot].used { - return Err(store.refuse(Damage::Intent, size)); - } - store.slots[slot] = Slot::FREE; - } - store.commit()?; - } - Ok(store) } @@ -261,7 +307,6 @@ impl Store { fn create(&mut self) -> Result<()> { self.slots = [Slot::FREE; SLOT_COUNT]; self.generation = 1; - self.fresh = true; let mut zero = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; zero.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; @@ -320,7 +365,7 @@ impl Store { } pub(crate) fn read(&mut self, key: &Key) -> Result>> { - if !(0xf0..=0xf5).contains(&key.kind) { + if !valid_kind(key.kind) { return Err(EINVAL); } let Some(idx) = self.find(key) else { @@ -336,7 +381,7 @@ impl Store { } pub(crate) fn write(&mut self, key: &Key, value: &[u8]) -> Result<()> { - if !(0xf0..=0xf5).contains(&key.kind) { + if !valid_kind(key.kind) { return Err(EINVAL); } if value.len() > MAX_VALUE { @@ -395,3 +440,395 @@ pub(crate) const fn crc16_ccitt_false(data: &[u8]) -> u16 { // Canonical CCITT-FALSE check value; guards against a reflected/XOR variant. static_assert!(crc16_ccitt_false(b"123456789") == 0x29B1); + +#[cfg(CONFIG_APPLE_SEP_KUNIT_TEST)] +#[::kernel::macros::kunit_tests(apple_sep_host_store)] +mod tests { + use super::*; + use core::cell::{Cell, RefCell}; + + /// A backing file in memory. A power cut can be armed on a write: that + /// write keeps only a prefix of its data, and every later call fails as if + /// the machine had stopped. + struct MemoryFile { + bytes: RefCell>, + writes: Cell, + cut: Cell>, + dead: Cell, + } + + impl MemoryFile { + fn new() -> Self { + Self { + bytes: RefCell::new(VVec::new()), + writes: Cell::new(0), + cut: Cell::new(None), + dead: Cell::new(false), + } + } + + fn copy(&self) -> Result { + let file = Self::new(); + file.bytes + .borrow_mut() + .extend_from_slice(&self.bytes.borrow(), GFP_KERNEL)?; + Ok(file) + } + + /// Cuts the power during the `nth` write from now, after `keep` bytes. + fn cut(&self, nth: usize, keep: usize) { + self.cut.set(Some((self.writes.get() + nth, keep))); + } + + fn restore_power(&self) { + self.cut.set(None); + self.dead.set(false); + } + + fn poke(&self, off: usize, data: &[u8]) { + self.bytes.borrow_mut()[off..off + data.len()].copy_from_slice(data); + } + + fn same(&self, other: &Self) -> bool { + self.bytes.borrow()[..] == other.bytes.borrow()[..] + } + } + + impl StoreIo for &MemoryFile { + fn size(&self) -> Result { + if self.dead.get() { + return Err(EIO); + } + Ok(self.bytes.borrow().len() as u64) + } + + fn read_exact(&self, off: u64, buf: &mut [u8]) -> Result { + if self.dead.get() { + return Err(EIO); + } + let bytes = self.bytes.borrow(); + let start = off as usize; + buf.copy_from_slice(bytes.get(start..start + buf.len()).ok_or(EIO)?); + Ok(()) + } + + fn write_all(&self, off: u64, buf: &[u8]) -> Result { + if self.dead.get() { + return Err(EIO); + } + let nth = self.writes.get(); + self.writes.set(nth + 1); + let keep = match self.cut.get() { + Some((at, keep)) if at == nth => { + self.dead.set(true); + keep.min(buf.len()) + } + _ => buf.len(), + }; + let start = off as usize; + let mut bytes = self.bytes.borrow_mut(); + if bytes.len() < start + keep { + bytes.resize(start + keep, 0, GFP_KERNEL)?; + } + bytes[start..start + keep].copy_from_slice(&buf[..keep]); + if self.dead.get() { + Err(EIO) + } else { + Ok(()) + } + } + + fn sync(&self) -> Result { + if self.dead.get() { + Err(EIO) + } else { + Ok(()) + } + } + } + + /// Offset of `byte` within the table entry of `slot`. + const fn entry(slot: usize, byte: usize) -> usize { + SB_SLOT_TABLE + slot * SLOT_ENTRY_SIZE + byte + } + + /// A store this driver created and then wrote `records` to. + fn written(records: &[(Key, &[u8])]) -> Result { + let file = MemoryFile::new(); + let mut store = Store::from_file(&file, true)?; + for (key, value) in records { + store.write(key, value)?; + } + Ok(file) + } + + /// Opens an existing store, checking that opening wrote nothing. + fn reopen(file: &MemoryFile) -> Result> { + let writes = file.writes.get(); + let store = Store::from_file(file, false)?; + if file.writes.get() != writes { + return Err(EINVAL); + } + Ok(store) + } + + /// Whether opening `file` fails and leaves it exactly as it was. + fn refused(file: &MemoryFile) -> Result { + let before = file.copy()?; + let writes = file.writes.get(); + Ok(Store::from_file(file, false).is_err() + && file.writes.get() == writes + && file.same(&before)) + } + + fn damage(file: &MemoryFile) -> Option { + let mut slots = [Slot::FREE; SLOT_COUNT]; + decode(&file.bytes.borrow()[..BLOCK_SIZE], &mut slots).err() + } + + /// Whether a store holding two records, patched with `data` at `off`, is + /// refused because of `slot`. + fn refuses_slot(off: usize, data: &[u8], slot: usize) -> Result { + let file = written(&[(Key::root(0xf1), b"index"), (Key::root(0xf5), b"")])?; + file.poke(off, data); + Ok(damage(&file) == Some(Damage::Slot(slot)) && refused(&file)?) + } + + /// Whether a store holding one record is refused with the given intent. + fn refuses_intent(kind: u8, slot: u16, record_kind: u8) -> Result { + let file = written(&[(Key::root(0xf1), b"index")])?; + file.poke(SB_INTENT_KIND, &[kind]); + file.poke(SB_INTENT_SLOT, &slot.to_le_bytes()); + file.poke(SB_INTENT_TYPE, &[record_kind]); + Ok(damage(&file) == Some(Damage::Intent) && refused(&file)?) + } + + #[test] + fn new_file_is_initialised() -> Result { + let file = written(&[])?; + assert_eq!(file.bytes.borrow().len(), STORE_SIZE); + assert!(damage(&file).is_none()); + assert!(reopen(&file)?.read(&Key::root(0xf1))?.is_none()); + Ok(()) + } + + #[test] + fn existing_files_are_never_initialised() -> Result { + // Empty, short, long and all-zero files are what an interrupted + // creation or damage leaves; none of them may be erased. + for len in [0, 1, BLOCK_SIZE, STORE_SIZE - 1, STORE_SIZE, STORE_SIZE + 1] { + let file = MemoryFile::new(); + file.bytes.borrow_mut().resize(len, 0, GFP_KERNEL)?; + assert!(refused(&file)?); + } + Ok(()) + } + + #[test] + fn only_an_empty_new_file_is_initialised() -> Result { + // Should a file that was there be reported as created, a store in it + // is still opened in place, and anything else refused. + let key = Key::root(0xf1); + let file = written(&[(key, b"index")])?; + let writes = file.writes.get(); + let mut store = Store::from_file(&file, true)?; + assert!(store.read(&key)?.as_deref() == Some(&b"index"[..])); + assert_eq!(file.writes.get(), writes); + + let file = MemoryFile::new(); + file.bytes.borrow_mut().resize(BLOCK_SIZE, 0, GFP_KERNEL)?; + assert!(Store::from_file(&file, true).is_err()); + assert_eq!(file.writes.get(), 0); + assert_eq!(file.bytes.borrow().len(), BLOCK_SIZE); + Ok(()) + } + + /// How a refusal of `file` describes it. + fn described(file: &MemoryFile) -> Result> { + Ok(describe(&file, file.size()?)) + } + + #[test] + fn interrupted_creation_is_refused() -> Result { + // Creation writes every block and then the superblock. Until the + // superblock is on disk the file is not a store, and holds nothing. + for nth in [0, 1, BLOCK_COUNT - 1, BLOCK_COUNT] { + for keep in [0, BLOCK_SIZE / 2] { + let file = MemoryFile::new(); + file.cut(nth, keep); + assert!(Store::from_file(&file, true).is_err()); + file.restore_power(); + if nth == BLOCK_COUNT && keep > 0 { + assert!(reopen(&file).is_ok()); + } else { + assert!(refused(&file)?); + assert!(described(&file)?.is_some()); + } + } + } + Ok(()) + } + + #[test] + fn refusals_tell_an_interrupted_creation_from_a_store() -> Result { + for (len, what) in [ + (0, Some("empty")), + (1, Some("all zero")), + (BLOCK_SIZE + 1, Some("all zero")), + (STORE_SIZE, Some("all zero")), + // Creation never writes past the store. + (STORE_SIZE + 1, None), + ] { + let file = MemoryFile::new(); + file.bytes.borrow_mut().resize(len, 0, GFP_KERNEL)?; + assert!(refused(&file)?); + assert!(described(&file)? == what); + } + + // A truncated store, and a single byte of data, may be what is left of + // the host state. + let file = written(&[(Key::root(0xf1), b"index")])?; + file.bytes.borrow_mut().truncate(2 * BLOCK_SIZE); + assert!(refused(&file)?); + assert!(described(&file)?.is_none()); + let file = MemoryFile::new(); + file.bytes.borrow_mut().resize(STORE_SIZE, 0, GFP_KERNEL)?; + file.poke(STORE_SIZE - 1, &[1]); + assert!(refused(&file)?); + assert!(described(&file)?.is_none()); + Ok(()) + } + + #[test] + fn foreign_headers_are_refused() -> Result { + let file = written(&[])?; + file.poke(SB_MAGIC, b"X"); + assert_eq!(damage(&file), Some(Damage::Header)); + assert!(refused(&file)?); + // Other versions, including 2, and other geometries. + for (off, value) in [ + (SB_VERSION, 0), + (SB_VERSION, 2), + (SB_SLOT_COUNT, SLOT_COUNT as u32 - 1), + (SB_BLOCK_SIZE, BLOCK_SIZE as u32 / 2), + ] { + let file = written(&[])?; + file.poke(off, &value.to_le_bytes()); + assert_eq!(damage(&file), Some(Damage::Header)); + assert!(refused(&file)?); + } + Ok(()) + } + + #[test] + fn unused_bytes_must_be_zero() -> Result { + for off in [ + SB_RESERVED, + SB_SLOT_TABLE - 1, + SB_SLOT_TABLE_END, + BLOCK_SIZE - 1, + ] { + let file = written(&[])?; + file.poke(off, &[1]); + assert_eq!(damage(&file), Some(Damage::Reserved)); + assert!(refused(&file)?); + } + Ok(()) + } + + #[test] + fn slots_must_be_records_or_zero() -> Result { + // Slot 0 holds 0xf1, slot 1 holds an empty 0xf5 and slot 2 is free. + assert!(refuses_slot(entry(2, 1), &[0xf0], 2)?); + assert!(refuses_slot(entry(2, 18), &[1], 2)?); + assert!(refuses_slot(entry(0, 0), &[2], 0)?); + assert!(refuses_slot(entry(0, 1), &[0xef], 0)?); + assert!(refuses_slot(entry(0, 1), &[0xf6], 0)?); + assert!(refuses_slot(entry(1, 22), &[1], 1)?); + // A length beyond the slot's block would be read from the next ones. + let over = MAX_VALUE as u32 + 1; + assert!(refuses_slot(entry(1, 18), &over.to_le_bytes(), 1)?); + assert!(refuses_slot(entry(1, 18), &u32::MAX.to_le_bytes(), 1)?); + Ok(()) + } + + #[test] + fn keys_must_be_unique() -> Result { + let other = Key { + kind: 0xf1, + uuid: [7; 16], + }; + let file = written(&[(Key::root(0xf1), b"a"), (other, b"b")])?; + // The same kind under another UUID is another key. + assert!(damage(&file).is_none()); + file.poke(entry(1, 2), &[0; 16]); + assert_eq!(damage(&file), Some(Damage::Duplicate(1))); + assert!(refused(&file)?); + Ok(()) + } + + #[test] + fn intents_must_name_a_free_block() -> Result { + // Slot 0 holds the record and slot 1 is free. + assert!(refuses_intent(INTENT_WRITE, 0, 0xf2)?); + assert!(refuses_intent(INTENT_WRITE, SLOT_COUNT as u16, 0xf2)?); + assert!(refuses_intent(INTENT_WRITE, 1, 0xef)?); + assert!(refuses_intent(2, 1, 0xf2)?); + assert!(refuses_intent(INTENT_NONE, 1, 0)?); + assert!(refuses_intent(INTENT_NONE, 0, 0xf2)?); + assert!(!refuses_intent(INTENT_WRITE, 1, 0xf2)?); + Ok(()) + } + + #[test] + fn records_are_never_discarded_or_overread() -> Result { + // An intent naming a committed record's block would discard the + // record, and a length beyond its block would be read from the next. + let file = written(&[(Key::root(0xf1), b"index")])?; + file.poke(SB_INTENT_KIND, &[INTENT_WRITE]); + file.poke(SB_INTENT_TYPE, &[0xf1]); + assert!(refused(&file)?); + let file = written(&[(Key::root(0xf1), b"index")])?; + file.poke(entry(0, 18), &(MAX_VALUE as u32 + 1).to_le_bytes()); + assert!(refused(&file)?); + Ok(()) + } + + #[test] + fn interrupted_write_is_discarded_without_writing() -> Result { + let key = Key::root(0xf1); + let file = written(&[(key, b"old")])?; + // The intent and the new block reach the disk, the new table does not. + file.cut(2, 0); + assert!(reopen(&file)?.write(&key, b"new").is_err()); + file.restore_power(); + + // The intent is still pending, and opening leaves it there. + assert_eq!(file.bytes.borrow()[SB_INTENT_KIND], INTENT_WRITE); + let mut store = reopen(&file)?; + assert!(store.read(&key)?.as_deref() == Some(&b"old"[..])); + store.write(&key, b"newer")?; + assert_eq!(file.bytes.borrow()[SB_INTENT_KIND], INTENT_NONE); + assert!(reopen(&file)?.read(&key)?.as_deref() == Some(&b"newer"[..])); + Ok(()) + } + + #[test] + fn a_full_block_is_the_largest_record() -> Result { + let mut full = KVec::new(); + full.resize(MAX_VALUE, 0x5a, GFP_KERNEL)?; + let key = Key::root(0xf2); + let file = written(&[(key, &full)])?; + let mut store = reopen(&file)?; + assert!(store.read(&key)?.as_deref() == Some(&full[..])); + full.push(0, GFP_KERNEL)?; + assert!(store.write(&key, &full).is_err()); + + // The last slot's block ends where the file does. + file.poke(entry(SLOT_COUNT - 1, 0), &[1, 0xf3]); + file.poke(entry(SLOT_COUNT - 1, 18), &(MAX_VALUE as u32).to_le_bytes()); + let value = reopen(&file)?.read(&Key::root(0xf3))?; + assert_eq!(value.map(|v| v.len()), Some(MAX_VALUE)); + Ok(()) + } +} From 68f2669cd7626e47e781cc22fbfe2060bd5ee805 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 07:27:25 +1100 Subject: [PATCH 1417/2680] soc: apple: sep: Test host-store compatibility with the earlier driver The host store has to stay readable in both directions: a machine can boot this kernel and then fall back to an earlier one with the same file, or the other way round. Build the earlier driver's store.rs into the tests as store_shipped.rs, changed only to reach its file through StoreIo, and check that - both drivers write byte-identical stores for the same operations, and reject the same invalid writes without writing, - each opens the other's store in place and reads the same records, - a write cut off at any of its write calls, keeping any multiple of 4 KiB of that call, leaves a store that both drivers open, in either order, holding the old value until the new table is on disk and the new value after that. The earlier driver also documents the behaviour this series removes: it erases a truncated store and one with an unknown version, both of which this driver refuses. A tools test undoes the changes to store_shipped.rs and compares the result with the shipped store.rs by its git blob id, so the copy cannot drift from the driver it stands for. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/store.rs | 198 +++++++++++++ drivers/soc/apple/store_shipped.rs | 287 +++++++++++++++++++ tools/aurora-sep/tests/test_store_shipped.py | 47 +++ 3 files changed, 532 insertions(+) create mode 100644 drivers/soc/apple/store_shipped.rs create mode 100644 tools/aurora-sep/tests/test_store_shipped.py diff --git a/drivers/soc/apple/store.rs b/drivers/soc/apple/store.rs index 94fc976275405..0cbedb2de632d 100644 --- a/drivers/soc/apple/store.rs +++ b/drivers/soc/apple/store.rs @@ -441,6 +441,14 @@ pub(crate) const fn crc16_ccitt_false(data: &[u8]) -> u16 { // Canonical CCITT-FALSE check value; guards against a reflected/XOR variant. static_assert!(crc16_ccitt_false(b"123456789") == 0x29B1); +// The driver as it shipped before this validation, changed only to reach its +// file through `StoreIo`. The tests use it as the reference for compatibility +// in both directions. +#[cfg(CONFIG_APPLE_SEP_KUNIT_TEST)] +#[allow(dead_code)] +#[path = "store_shipped.rs"] +mod shipped; + #[cfg(CONFIG_APPLE_SEP_KUNIT_TEST)] #[::kernel::macros::kunit_tests(apple_sep_host_store)] mod tests { @@ -831,4 +839,194 @@ mod tests { assert_eq!(value.map(|v| v.len()), Some(MAX_VALUE)); Ok(()) } + + /// Opens `file` with the shipped driver. + fn open_shipped(file: &MemoryFile) -> Result> { + shipped::Store::open(file) + } + + /// `key` as the shipped driver spells it. + fn as_shipped(key: &Key) -> shipped::Key { + shipped::Key { + kind: key.kind, + uuid: key.uuid, + } + } + + /// A record value whose every byte differs from those of other seeds. + fn value(len: usize, seed: u8) -> Result> { + let mut v = KVec::with_capacity(len, GFP_KERNEL)?; + for i in 0..len { + v.push(seed.wrapping_add(i as u8).wrapping_mul(31), GFP_KERNEL)?; + } + Ok(v) + } + + /// Every kind, a non-root key, an empty value, full blocks and overwrites. + fn history() -> Result)>> { + let mut ops = KVec::new(); + let other = Key { + kind: 0xf0, + uuid: [0x5c; 16], + }; + for (key, len, seed) in [ + (Key::root(0xf1), 300, 1), + (Key::root(0xf5), 0, 2), + (Key::root(0xf2), MAX_VALUE, 3), + (other, 16, 4), + (Key::root(0xf1), 600, 5), + (Key::root(0xf3), 1, 6), + (Key::root(0xf4), MAX_VALUE - 1, 7), + (Key::root(0xf0), 16, 8), + (Key::root(0xf5), 12, 9), + ] { + ops.push((key, value(len, seed)?), GFP_KERNEL)?; + } + Ok(ops) + } + + /// Whether this driver and the shipped one read the same value for every + /// key in `ops` from `file`. + fn both_read(file: &MemoryFile, ops: &[(Key, KVec)]) -> Result { + let mut new = reopen(file)?; + let mut shipped = open_shipped(file)?; + if shipped.fresh { + return Ok(false); + } + for (key, _) in ops { + if new.read(key)?.as_deref() != shipped.read(&as_shipped(key))?.as_deref() { + return Ok(false); + } + } + Ok(true) + } + + #[test] + fn writes_match_the_shipped_driver_byte_for_byte() -> Result { + let ours = MemoryFile::new(); + let theirs = MemoryFile::new(); + let mut new = Store::from_file(&ours, true)?; + let mut shipped = open_shipped(&theirs)?; + assert!(ours.same(&theirs)); + for (key, value) in history()?.iter() { + new.write(key, value)?; + shipped.write(&as_shipped(key), value)?; + assert!(ours.same(&theirs)); + } + Ok(()) + } + + #[test] + fn both_drivers_reject_the_same_writes() -> Result { + let mut long = KVec::new(); + long.resize(MAX_VALUE + 1, 0x5a, GFP_KERNEL)?; + let ours = written(&[(Key::root(0xf1), b"index")])?; + let theirs = ours.copy()?; + let mut new = reopen(&ours)?; + let mut shipped = open_shipped(&theirs)?; + for (key, value) in [ + (Key::root(0xef), &b"x"[..]), + (Key::root(0xf6), &b"x"[..]), + (Key::root(0xf2), &long[..]), + ] { + let ours_failed = new.write(&key, value).err(); + assert!(ours_failed.is_some()); + assert!(ours_failed == shipped.write(&as_shipped(&key), value).err()); + } + assert!(new.read(&Key::root(0xef)).is_err()); + assert!(shipped.read(&as_shipped(&Key::root(0xef))).is_err()); + assert!(ours.same(&theirs)); + Ok(()) + } + + #[test] + fn each_driver_opens_the_others_store_in_place() -> Result { + let ops = history()?; + let (first, second) = ops.split_at(4); + + // Shipped writer, new reader, then the other way round. + let file = MemoryFile::new(); + let mut shipped = open_shipped(&file)?; + for (key, value) in first { + shipped.write(&as_shipped(key), value)?; + } + assert!(both_read(&file, &ops)?); + let mut new = reopen(&file)?; + for (key, value) in second { + new.write(key, value)?; + } + assert!(both_read(&file, &ops)?); + for (key, _) in ops.iter() { + let stored = open_shipped(&file)?.read(&as_shipped(key))?; + let last = ops + .iter() + .rev() + .find(|(k, _)| k == key) + .map(|(_, v)| &v[..]); + assert!(stored.as_deref() == last); + } + Ok(()) + } + + #[test] + fn torn_writes_open_in_either_driver() -> Result { + let key = Key::root(0xf2); + let old = value(MAX_VALUE, 0x11)?; + let new = value(MAX_VALUE, 0x22)?; + // A write is an intent superblock, the new block, then the new table. + // Each is cut off after every multiple of the atomic write size. + for writer_is_new in [false, true] { + for nth in 0..3 { + for keep in (0..=BLOCK_SIZE).step_by(ATOMIC_WRITE) { + let file = MemoryFile::new(); + open_shipped(&file)?.write(&as_shipped(&key), &old)?; + open_shipped(&file)?.write(&as_shipped(&Key::root(0xf5)), b"lockout")?; + file.cut(nth, keep); + let failed = if writer_is_new { + reopen(&file)?.write(&key, &new).is_err() + } else { + open_shipped(&file)?.write(&as_shipped(&key), &new).is_err() + }; + assert!(failed); + file.restore_power(); + + // Only a table that reached the disk publishes the value. + let expect = if nth == 2 && keep > 0 { &new } else { &old }; + let crashed = file.copy()?; + for new_first in [false, true] { + let file = crashed.copy()?; + if new_first { + assert!(reopen(&file)?.read(&key)?.as_deref() == Some(&expect[..])); + } + let mut shipped = open_shipped(&file)?; + assert!(!shipped.fresh); + assert!(shipped.read(&as_shipped(&key))?.as_deref() == Some(&expect[..])); + let lockout = reopen(&file)?.read(&Key::root(0xf5))?; + assert!(lockout.as_deref() == Some(&b"lockout"[..])); + assert!(reopen(&file)?.read(&key)?.as_deref() == Some(&expect[..])); + } + } + } + } + Ok(()) + } + + #[test] + fn the_shipped_driver_erased_what_this_one_refuses() -> Result { + let key = Key::root(0xf1); + let truncated = written(&[(key, b"index")])?; + truncated + .bytes + .borrow_mut() + .truncate(STORE_SIZE - BLOCK_SIZE); + let unknown = written(&[(key, b"index")])?; + unknown.poke(SB_VERSION, &2u32.to_le_bytes()); + for file in [truncated, unknown] { + assert!(refused(&file)?); + let mut shipped = open_shipped(&file)?; + assert!(shipped.fresh); + assert!(shipped.read(&as_shipped(&key))?.is_none()); + } + Ok(()) + } } diff --git a/drivers/soc/apple/store_shipped.rs b/drivers/soc/apple/store_shipped.rs new file mode 100644 index 0000000000000..74785d8923076 --- /dev/null +++ b/drivers/soc/apple/store_shipped.rs @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Linux-only state. SEP anti-replay records live in `xart_store`. + +use super::StoreIo; +use kernel::prelude::*; + +pub(crate) const STORE_PATH: &CStr = c"/var/lib/aurora-sep-host-state.bin"; + +const BLOCK_SIZE: usize = 0x8000; +const BLOCK_COUNT: usize = 72; +pub(crate) const STORE_SIZE: usize = BLOCK_SIZE * BLOCK_COUNT; +const SLOT_COUNT: usize = BLOCK_COUNT - 1; +pub(crate) const MAX_VALUE: usize = BLOCK_SIZE; + +const MAGIC: [u8; 16] = *b"AURORA-SEP-STOR\x01"; +const VERSION: u32 = 1; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct Key { + pub(crate) kind: u8, + pub(crate) uuid: [u8; 16], +} + +impl Key { + pub(crate) const fn root(kind: u8) -> Key { + Key { + kind, + uuid: [0u8; 16], + } + } +} + +#[derive(Clone, Copy)] +struct Slot { + used: bool, + kind: u8, + uuid: [u8; 16], + len: u32, +} + +impl Slot { + const FREE: Slot = Slot { + used: false, + kind: 0, + uuid: [0; 16], + len: 0, + }; + + fn matches(&self, key: &Key) -> bool { + self.used && self.kind == key.kind && self.uuid == key.uuid + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Intent { + None, + Write { slot: u16, kind: u8 }, +} + +const SB_MAGIC: usize = 0; +const SB_VERSION: usize = 16; +const SB_SLOT_COUNT: usize = 20; +const SB_BLOCK_SIZE: usize = 24; +const SB_GENERATION: usize = 28; +const SB_INTENT_KIND: usize = 36; +const SB_INTENT_SLOT: usize = 37; +const SB_INTENT_TYPE: usize = 39; +const SB_SLOT_TABLE: usize = 64; +const SLOT_ENTRY_SIZE: usize = 24; + +const INTENT_NONE: u8 = 0; +const INTENT_WRITE: u8 = 1; + +// Not internally locked; the caller holds a mutex around all access. +pub(crate) struct Store { + file: F, + slots: [Slot; SLOT_COUNT], + generation: u64, + pub(crate) recovered: bool, + pub(crate) fresh: bool, +} + +fn le32(buf: &[u8], off: usize) -> u32 { + u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]]) +} + +fn le64(buf: &[u8], off: usize) -> u64 { + let mut v = [0u8; 8]; + v.copy_from_slice(&buf[off..off + 8]); + u64::from_le_bytes(v) +} + +impl Store { + pub(crate) fn open(file: F) -> Result { + let mut store = Store { + file, + slots: [Slot::FREE; SLOT_COUNT], + generation: 0, + recovered: false, + fresh: false, + }; + + let size = store.file.size()?; + if size < STORE_SIZE as u64 { + store.create()?; + return Ok(store); + } + + let mut sb = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; + sb.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; + store.file.read_exact(0, &mut sb)?; + + if sb[SB_MAGIC..SB_MAGIC + 16] != MAGIC + || le32(&sb, SB_VERSION) != VERSION + || le32(&sb, SB_SLOT_COUNT) as usize != SLOT_COUNT + || le32(&sb, SB_BLOCK_SIZE) as usize != BLOCK_SIZE + { + store.create()?; + return Ok(store); + } + + store.generation = le64(&sb, SB_GENERATION); + for i in 0..SLOT_COUNT { + let off = SB_SLOT_TABLE + i * SLOT_ENTRY_SIZE; + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&sb[off + 2..off + 18]); + store.slots[i] = Slot { + used: sb[off] != 0, + kind: sb[off + 1], + uuid, + len: le32(&sb, off + 18), + }; + } + + // Crash recovery: writes are copy-on-write, so discard the intent, never complete it. + let intent_kind = sb[SB_INTENT_KIND]; + if intent_kind != INTENT_NONE { + store.recovered = true; + let slot = u16::from_le_bytes([sb[SB_INTENT_SLOT], sb[SB_INTENT_SLOT + 1]]) as usize; + if intent_kind == INTENT_WRITE && slot < SLOT_COUNT { + store.slots[slot] = Slot::FREE; + } + store.commit()?; + } + + Ok(store) + } + + fn create(&mut self) -> Result<()> { + self.slots = [Slot::FREE; SLOT_COUNT]; + self.generation = 1; + self.fresh = true; + + let mut zero = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; + zero.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; + for b in 0..BLOCK_COUNT { + self.file.write_all((b * BLOCK_SIZE) as u64, &zero)?; + } + + self.commit()?; + Ok(()) + } + + fn commit(&mut self) -> Result<()> { + self.write_superblock(Intent::None)?; + self.file.sync() + } + + fn write_superblock(&mut self, intent: Intent) -> Result<()> { + let mut sb = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; + sb.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; + + sb[SB_MAGIC..SB_MAGIC + 16].copy_from_slice(&MAGIC); + sb[SB_VERSION..SB_VERSION + 4].copy_from_slice(&VERSION.to_le_bytes()); + sb[SB_SLOT_COUNT..SB_SLOT_COUNT + 4].copy_from_slice(&(SLOT_COUNT as u32).to_le_bytes()); + sb[SB_BLOCK_SIZE..SB_BLOCK_SIZE + 4].copy_from_slice(&(BLOCK_SIZE as u32).to_le_bytes()); + sb[SB_GENERATION..SB_GENERATION + 8].copy_from_slice(&self.generation.to_le_bytes()); + + let (ikind, islot, itype) = match intent { + Intent::None => (INTENT_NONE, 0u16, 0u8), + Intent::Write { slot, kind } => (INTENT_WRITE, slot, kind), + }; + sb[SB_INTENT_KIND] = ikind; + sb[SB_INTENT_SLOT..SB_INTENT_SLOT + 2].copy_from_slice(&islot.to_le_bytes()); + sb[SB_INTENT_TYPE] = itype; + + for (i, slot) in self.slots.iter().enumerate() { + let off = SB_SLOT_TABLE + i * SLOT_ENTRY_SIZE; + sb[off] = u8::from(slot.used); + sb[off + 1] = slot.kind; + sb[off + 2..off + 18].copy_from_slice(&slot.uuid); + sb[off + 18..off + 22].copy_from_slice(&slot.len.to_le_bytes()); + } + + self.file.write_all(0, &sb) + } + + fn find(&self, key: &Key) -> Option { + self.slots.iter().position(|s| s.matches(key)) + } + + fn find_free(&self) -> Option { + self.slots.iter().position(|s| !s.used) + } + + fn block_offset(slot: usize) -> u64 { + ((slot + 1) * BLOCK_SIZE) as u64 + } + + pub(crate) fn read(&mut self, key: &Key) -> Result>> { + if !(0xf0..=0xf5).contains(&key.kind) { + return Err(EINVAL); + } + let Some(idx) = self.find(key) else { + return Ok(None); + }; + let len = self.slots[idx].len as usize; + let mut value = KVec::with_capacity(len, GFP_KERNEL)?; + value.resize(len, 0, GFP_KERNEL)?; + if len > 0 { + self.file.read_exact(Self::block_offset(idx), &mut value)?; + } + Ok(Some(value)) + } + + pub(crate) fn write(&mut self, key: &Key, value: &[u8]) -> Result<()> { + if !(0xf0..=0xf5).contains(&key.kind) { + return Err(EINVAL); + } + if value.len() > MAX_VALUE { + return Err(ENOSPC); + } + + let old = self.find(key); + let fresh = self.find_free().ok_or(ENOSPC)?; + + // 1. Record the intent, durably, before touching any data block. + self.write_superblock(Intent::Write { + slot: fresh as u16, + kind: key.kind, + })?; + self.file.sync()?; + + // 2. Fill the new block and make it durable. + if !value.is_empty() { + self.file.write_all(Self::block_offset(fresh), value)?; + } + self.file.sync()?; + + // 3. Switch the slot table, release the old block, clear the intent. + self.slots[fresh] = Slot { + used: true, + kind: key.kind, + uuid: key.uuid, + len: value.len() as u32, + }; + if let Some(old) = old { + self.slots[old] = Slot::FREE; + } + self.generation = self.generation.wrapping_add(1); + self.commit() + } +} + +pub(crate) const fn crc16_ccitt_false(data: &[u8]) -> u16 { + let mut crc: u16 = 0xffff; + let mut i = 0; + while i < data.len() { + crc ^= (data[i] as u16) << 8; + let mut bit = 0; + while bit < 8 { + crc = if crc & 0x8000 != 0 { + (crc << 1) ^ 0x1021 + } else { + crc << 1 + }; + bit += 1; + } + i += 1; + } + crc +} + +// Canonical CCITT-FALSE check value; guards against a reflected/XOR variant. +static_assert!(crc16_ccitt_false(b"123456789") == 0x29B1); diff --git a/tools/aurora-sep/tests/test_store_shipped.py b/tools/aurora-sep/tests/test_store_shipped.py new file mode 100644 index 0000000000000..290f1b1a2b97a --- /dev/null +++ b/tools/aurora-sep/tests/test_store_shipped.py @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: MIT +"""The host store's KUnit suite tests against the driver as it shipped. + +drivers/soc/apple/store_shipped.rs is drivers/soc/apple/store.rs as it was +before the store validated what it reads, changed only so that the tests can +give it an in-memory file. Undoing exactly those changes must give back the +shipped file, identified by its git blob id. +""" + +import hashlib +from pathlib import Path +import unittest + +SHIPPED = Path(__file__).resolve().parents[3] / "drivers/soc/apple/store_shipped.rs" +BLOB = "dc37025ce3b8adbde9df751b6169342c532f0c08" + +# (as shipped, as in store_shipped.rs) +SEAM = ( + ("use crate::shim;\n", + "use super::StoreIo;\n"), + ("pub(crate) struct Store {\n" + " file: shim::StoreFile,\n", + "pub(crate) struct Store {\n" + " file: F,\n"), + ("impl Store {\n" + " pub(crate) fn open() -> Result {\n" + " let file = shim::StoreFile::open(STORE_PATH)?;\n" + "\n", + "impl Store {\n" + " pub(crate) fn open(file: F) -> Result {\n"), +) + + +@unittest.skipUnless(SHIPPED.is_file(), "not in a kernel tree") +class Seam(unittest.TestCase): + def test_only_the_io_seam_differs(self): + text = SHIPPED.read_text() + for shipped, changed in SEAM: + self.assertEqual(text.count(changed), 1, changed) + text = text.replace(changed, shipped) + data = text.encode() + blob = hashlib.sha1(b"blob %d\0" % len(data) + data).hexdigest() + self.assertEqual(blob, BLOB) + + +if __name__ == "__main__": + unittest.main() From 1183a832a53be8a20a4b679be49f8e6343d97b66 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 07:35:29 +1100 Subject: [PATCH 1418/2680] tools: aurora-sep: Document the host-state store and add a read-only check Describe the host-state store: its format, why it has no checksum and why it relies on 4 KiB atomic writes, what the driver refuses and how that is logged, how to copy the store consistently, and how to recover without erasing state. Only a file in which every byte is zero, which is all an interrupted creation leaves, may be removed; anything else, including a truncated store, needs a recovery decision. Add host-state-check.py, which applies the driver's validation rules to a store or a private copy of it without writing, and prints only the verdict. A store can then be checked before booting a kernel that would refuse it, or an earlier kernel that would erase it. When it refuses a file for its size or header, it also says whether the file is empty or all zero, or else whether it still has a header and how many records, so that an interrupted creation can be told from a truncated store without printing anything the store holds. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/HOST-STATE.md | 105 +++++++++ tools/aurora-sep/host-state-check.py | 150 ++++++++++++ .../aurora-sep/tests/test_host_state_check.py | 223 ++++++++++++++++++ 3 files changed, 478 insertions(+) create mode 100644 tools/aurora-sep/HOST-STATE.md create mode 100755 tools/aurora-sep/host-state-check.py create mode 100644 tools/aurora-sep/tests/test_host_state_check.py diff --git a/tools/aurora-sep/HOST-STATE.md b/tools/aurora-sep/HOST-STATE.md new file mode 100644 index 0000000000000..2226c36840e48 --- /dev/null +++ b/tools/aurora-sep/HOST-STATE.md @@ -0,0 +1,105 @@ +# Linux host-state store + +`/var/lib/aurora-sep-host-state.bin` holds SEP state that only Linux keeps: +the Touch ID identity index and the lockout state. Nothing regenerates it. +The catacombs have files of their own, and the SEP's anti-replay records live +in the shared xART gigalocker. + +## Format + +The file is exactly 72 blocks of 32 KiB (2359296 bytes). Block 0 is the +superblock: the magic `AURORA-SEP-STOR\x01`, version 1, the slot count 71, the +block size, a generation, a write intent and a table of 71 slot entries. The +record of slot *n* is in block *n* + 1. Writes are copy-on-write: the driver +records an intent naming a free block, writes the record there, then writes the +new table. + +The table is rewritten in place, so the format relies on the device writing the +first 4 KiB of the superblock atomically. Every field the driver reads lies in +that range, so after a crash it finds either the old table or the new one. +NVMe storage on Apple silicon has 4 KiB logical blocks +(`/sys/block//queue/logical_block_size`). A copy of the file taken while +the driver writes has no such guarantee; see "Copying the store". + +Every kernel writes this format byte for byte the same way, so any mix of +kernels can use one valid store in any boot order. The driver's KUnit suite +builds the earlier driver from its own source and checks this in both +directions, including writes cut off at every step. + +There is deliberately no checksum. Earlier kernels rewrite the whole superblock +on each write, zeroing every byte they do not use, and never look past the +store's 72 blocks. A checksum inside the superblock would be erased, and one +after the store would be stale, after any write by an earlier kernel, and a +reader could not tell either from damage. + +## What the driver refuses + +The driver creates the store only when the file does not exist. It never +initialises, repairs or rewrites an existing file that fails validation. Each +time it opens the file, at probe and again when Touch ID is activated, it logs +`host-state store failed validation (REASON), refusing it`, and Touch ID stays +unavailable for that boot. Nothing is written. + +| Reason | Meaning | +| --- | --- | +| `Size` | not exactly 2359296 bytes: truncated, extended or an interrupted creation | +| `Header` | the magic, version, slot count or block size is wrong | +| `Reserved` | a byte that the format leaves zero is not zero | +| `Slot(n)` | slot *n* is neither all zero nor a valid record | +| `Duplicate(n)` | slot *n* has the same key as an earlier slot | +| `Intent` | the pending write intent does not name a free block | + +A `Size` or `Header` reason is followed by `, empty` or `, all zero` when the +file, no longer than a store, holds nothing but zeros. Without that qualifier +the file holds data, which may be what is left of a store. + +Kernels from before this validation erase a file that is short or has another +header, and use the other cases unchecked. A kernel that carries only the fix +that stopped the erasing refuses a short file, an unknown header, a record +longer than its block and an intent that names a used block, with the same log +line, and uses the other cases unchecked. + +## Copying the store + +Copy the store only while nothing uses Touch ID, for example with fprintd +stopped, and take two copies in a row. Use them only if they are identical: + +```sh +umask 077 +cp /var/lib/aurora-sep-host-state.bin copy-1 +cp /var/lib/aurora-sep-host-state.bin copy-2 +cmp copy-1 copy-2 && echo consistent +``` + +## Checking a store + +`host-state-check.py FILE` applies the same rules without writing and prints +`ok: N records` or `refused: REASON`, never record contents. For `Size` and +`Header` it describes the file as the driver does, and for a `Size` that is not +all zero it adds `header present, N records` (for example a truncated store) +or `no header`. Check a consistent copy rather than the live file: a read that +overlaps a write by the driver can see a refusal that is not there. + +## Recovery + +Keep a private copy of the store (mode 0600, never published) before any +recovery step, and before booting a kernel that would erase it. + +- A file in which every byte is zero holds nothing. It is what an interrupted + first creation leaves, by any kernel, until the superblock is written, and + the driver reports it as `Size, empty`, `Size, all zero` or + `Header, all zero`. Confirm it on the file itself: + + ```sh + f=/var/lib/aurora-sep-host-state.bin + cmp -n "$(stat -c %s "$f")" "$f" /dev/zero && echo all-zero + ``` + + Only if that prints `all-zero`, remove the file; the driver creates a new + store on the next boot. +- Anything else needs a recovery decision, including a file that is only + short: a truncated store usually keeps its header and its first records + (`Size, header present, N records`). Do not delete the store or restore an + older copy to make Touch ID work again: the SEP's anti-replay state has moved + on since that copy, so old records may be refused or replay old state. Do not + change the Linux OS identity to work around a refused store. diff --git a/tools/aurora-sep/host-state-check.py b/tools/aurora-sep/host-state-check.py new file mode 100755 index 0000000000000..d553456d60b83 --- /dev/null +++ b/tools/aurora-sep/host-state-check.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Check a SEP host-state store the way the kernel driver does, read-only. + +Prints "ok" with the number of records, or the reason the driver would refuse +the file in the words of its kernel log line, and never any record contents. +A file refused for its size or header is also described as empty, all zero, +holding a header and N records, or holding no header, so that what an +interrupted creation leaves can be told from a damaged store. Exit status 0 +means the driver accepts the file, 1 that it refuses it, 2 that it could not be +read. +""" + +import argparse +import os +import struct +import sys + +BLOCK_SIZE = 0x8000 +BLOCK_COUNT = 72 +STORE_SIZE = BLOCK_SIZE * BLOCK_COUNT +SLOT_COUNT = BLOCK_COUNT - 1 +MAX_VALUE = BLOCK_SIZE +MAGIC = b"AURORA-SEP-STOR\x01" +VERSION = 1 +SB_INTENT_KIND = 36 +SB_INTENT_SLOT = 37 +SB_INTENT_TYPE = 39 +SB_RESERVED = 40 +SB_SLOT_TABLE = 64 +SLOT_ENTRY_SIZE = 24 +SB_SLOT_TABLE_END = SB_SLOT_TABLE + SLOT_COUNT * SLOT_ENTRY_SIZE +INTENT_NONE = 0 +INTENT_WRITE = 1 +KINDS = range(0xF0, 0xF6) + + +def has_header(sb): + """Whether sb starts with this format's header.""" + if len(sb) < 28: + return False + version, slots, block = struct.unpack_from(" STORE_SIZE: + return False + f.seek(0) + left = size + while left: + chunk = f.read(min(left, BLOCK_SIZE)) + if not chunk or any(chunk): + return False + left -= len(chunk) + return True + + +def records(n): + return f"{n} record" + ("" if n == 1 else "s") + + +def describe(f, size, sb, reason): + """Describe a file refused for its size or header, without its contents.""" + if size == 0: + return "empty" + if is_zero(f, size): + return "all zero" + if reason != "Size": + return None + if not has_header(sb): + return "no header" + used = sum(1 for i in range(SLOT_COUNT) + if SB_SLOT_TABLE + i * SLOT_ENTRY_SIZE < len(sb) + and sb[SB_SLOT_TABLE + i * SLOT_ENTRY_SIZE]) + return f"header present, {records(used)}" + + +def damage(size, sb): + """Return None if the driver accepts the store, else why it refuses it.""" + if size != STORE_SIZE or len(sb) != BLOCK_SIZE: + return "Size" + if not has_header(sb): + return "Header" + if any(sb[SB_RESERVED:SB_SLOT_TABLE]) or any(sb[SB_SLOT_TABLE_END:]): + return "Reserved" + + keys = set() + free = [] + for i in range(SLOT_COUNT): + off = SB_SLOT_TABLE + i * SLOT_ENTRY_SIZE + entry = sb[off:off + SLOT_ENTRY_SIZE] + if not any(entry): + free.append(True) + continue + length = struct.unpack_from(" MAX_VALUE \ + or any(entry[22:]): + return f"Slot({i})" + key = bytes(entry[1:18]) + if key in keys: + return f"Duplicate({i})" + keys.add(key) + free.append(False) + + kind = sb[SB_INTENT_KIND] + slot = struct.unpack_from(" Date: Fri, 16 Oct 2026 01:24:02 +1100 Subject: [PATCH 1419/2680] wifi: mt7932: drop received data the hardware did not decrypt On a WPA2 connection the driver hands every header-translated data frame from the associated peer to the network stack. Nothing checks that the hardware decrypted it: a frame that arrived in plaintext, or that was decrypted with a cipher or key length other than the one the key expects, is delivered like any other. The receive descriptor reports both: word1 carries the cipher the frame was decrypted with (zero when it was not) and the cipher and cipher-length mismatch flags, in the same layout as the connac2 devices that mt76 handles. Only a cipher mismatch on a translated frame was rejected; the cipher-length mismatch flag was ignored. Keep the cipher and both mismatch flags in the parsed frame and, on a protected connection, deliver only frames that were decrypted without a mismatch. The firmware runs the WPA2 handshake itself, so the host never needs to accept an unprotected EAPOL frame. An open connection is unchanged. Add KUnit tests with synthetic descriptors: decrypted, plaintext, cipher-length and cipher mismatch, integrity errors, every truncation of the descriptor and its Ethernet header, and a length beyond the buffer. Fixes: 8709ef5c7d40 ("wifi: add existing J700 MT7932 fullmac implementation") (cherry picked from commit 43363f1eb7ac812adbbfa56ee4c5ee0198a90306) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/Kconfig | 10 ++ drivers/net/wireless/mediatek/mt7932/Makefile | 1 + drivers/net/wireless/mediatek/mt7932/data.c | 7 + .../net/wireless/mediatek/mt7932/input-test.c | 133 ++++++++++++++++++ drivers/net/wireless/mediatek/mt7932/packet.h | 29 +++- 5 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 drivers/net/wireless/mediatek/mt7932/input-test.c diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index c6120372d1f3b..f09fb1f32012b 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -31,3 +31,13 @@ config MT7932_RF_KUNIT_TEST admission paths with synthetic devices, without radio hardware. Check association wakeups, retained peer ownership and refusal to reopen RF admission after a terminal calibration error. + +config MT7932_KUNIT_TEST + bool "KUnit tests for MT7932 firmware input validation" if !KUNIT_ALL_TESTS + depends on KUNIT=y && MT7932_FULLMAC + default KUNIT_ALL_TESTS + help + Exercise the validation of what the firmware and the device hand + to the driver, with synthetic, truncated and corrupted input and + without radio hardware: receive descriptors, including frames the + hardware did not decrypt, and command replies. diff --git a/drivers/net/wireless/mediatek/mt7932/Makefile b/drivers/net/wireless/mediatek/mt7932/Makefile index f854d21e5db81..97b1b760372c1 100644 --- a/drivers/net/wireless/mediatek/mt7932/Makefile +++ b/drivers/net/wireless/mediatek/mt7932/Makefile @@ -2,3 +2,4 @@ obj-$(CONFIG_MT7932_FULLMAC) += mt7932-fullmac.o mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += rf-test.o +mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += input-test.o diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index 99ae52977b582..ba06db134414e 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -234,6 +234,13 @@ void mt_data_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame) m->netdev->stats.rx_dropped++; goto unlock; } + if (!mt7932_rx_protection_ok(frame, m->connect_open)) { + m->netdev->stats.rx_dropped++; + dev_dbg_ratelimited(&m->pdev->dev, + "RX_UNPROTECTED_DROPPED: cipher=%u mismatch=%u\n", + frame->sec_mode, frame->cipher_mismatch); + goto unlock; + } skb = netdev_alloc_skb_ip_align(m->netdev, frame->length); if (!skb) { m->netdev->stats.rx_dropped++; diff --git a/drivers/net/wireless/mediatek/mt7932/input-test.c b/drivers/net/wireless/mediatek/mt7932/input-test.c new file mode 100644 index 0000000000000..409103ed18740 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/input-test.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for the validation of firmware input: RX descriptors and + * command replies. The firmware and the device are treated as untrusted. + */ +#include + +#include "mt7932.h" + +#define RX_HEADER 24 +#define RX_ETHERNET 14 +#define RX_PAYLOAD 40 +#define RX_SIZE (RX_HEADER + RX_ETHERNET + RX_PAYLOAD) + +/* A translated (Ethernet) unicast data frame on channel 36 for WTBL 5. */ +static void rx_build(u8 *packet, u32 extra_w1) +{ + memset(packet, 0, RX_SIZE); + put_unaligned_le32(RX_SIZE | (2U << 27), packet); + put_unaligned_le32(5 | extra_w1, packet + 4); + put_unaligned_le32(BIT(13), packet + 8); + put_unaligned_le32((36U << 8) | (1U << 16), packet + 12); + memset(packet + RX_HEADER, 0xa5, RX_ETHERNET + RX_PAYLOAD); + put_unaligned_be16(ETH_P_IP, packet + RX_HEADER + 12); +} + +static u32 rx_cipher(unsigned int mode) +{ + return mode << MT7932_RXD1_SEC_MODE_SHIFT; +} + +static void mt7932_rx_decrypted_test(struct kunit *test) +{ + struct mt7932_rx_frame frame; + u8 packet[RX_SIZE]; + + rx_build(packet, rx_cipher(4)); + KUNIT_ASSERT_EQ(test, mt7932_rx_frame(packet, sizeof(packet), &frame), 0); + KUNIT_EXPECT_TRUE(test, frame.translated); + KUNIT_EXPECT_EQ(test, frame.sec_mode, 4); + KUNIT_EXPECT_EQ(test, frame.wtbl, 5); + KUNIT_EXPECT_EQ(test, frame.length, (size_t)(RX_ETHERNET + RX_PAYLOAD)); + KUNIT_EXPECT_TRUE(test, mt7932_rx_protection_ok(&frame, false)); + KUNIT_EXPECT_TRUE(test, mt7932_rx_protection_ok(&frame, true)); +} + +/* A plaintext frame on a protected connection is never delivered. */ +static void mt7932_rx_plaintext_test(struct kunit *test) +{ + struct mt7932_rx_frame frame; + u8 packet[RX_SIZE]; + + rx_build(packet, 0); + KUNIT_ASSERT_EQ(test, mt7932_rx_frame(packet, sizeof(packet), &frame), 0); + KUNIT_EXPECT_EQ(test, frame.sec_mode, 0); + KUNIT_EXPECT_FALSE(test, mt7932_rx_protection_ok(&frame, false)); + /* An open connection has nothing to decrypt. */ + KUNIT_EXPECT_TRUE(test, mt7932_rx_protection_ok(&frame, true)); +} + +/* Decrypted with the wrong cipher or key length: refused on WPA2. */ +static void mt7932_rx_mismatch_test(struct kunit *test) +{ + struct mt7932_rx_frame frame; + u8 packet[RX_SIZE]; + + rx_build(packet, rx_cipher(4) | MT7932_RXD1_CIPHER_LEN_MISMATCH); + KUNIT_ASSERT_EQ(test, mt7932_rx_frame(packet, sizeof(packet), &frame), 0); + KUNIT_EXPECT_TRUE(test, frame.cipher_mismatch); + KUNIT_EXPECT_FALSE(test, mt7932_rx_protection_ok(&frame, false)); + /* A translated frame with a cipher mismatch is rejected by the parser. */ + rx_build(packet, rx_cipher(4) | MT7932_RXD1_CIPHER_MISMATCH); + KUNIT_EXPECT_EQ(test, mt7932_rx_frame(packet, sizeof(packet), &frame), -EBADMSG); +} + +/* ICV, TKIP MIC and FCS errors are rejected before any field is used. */ +static void mt7932_rx_integrity_test(struct kunit *test) +{ + struct mt7932_rx_frame frame; + u8 packet[RX_SIZE]; + unsigned int bit; + + for (bit = 25; bit <= 27; bit++) { + rx_build(packet, rx_cipher(4) | BIT(bit)); + KUNIT_EXPECT_EQ(test, mt7932_rx_frame(packet, sizeof(packet), &frame), + -EBADMSG); + } +} + +/* Short buffers and length fields beyond the buffer never parse. */ +static void mt7932_rx_short_test(struct kunit *test) +{ + struct mt7932_rx_frame frame; + u8 packet[RX_SIZE], *copy; + size_t cut; + + rx_build(packet, rx_cipher(4)); + for (cut = 0; cut < RX_HEADER + RX_ETHERNET; cut++) { + /* An exact-size copy lets KASAN catch any read past the end. */ + copy = kmemdup(packet, cut, GFP_KERNEL); + KUNIT_ASSERT_TRUE(test, copy || !cut); + if (cut >= 4) + put_unaligned_le32(cut | (2U << 27), copy); + KUNIT_EXPECT_LT_MSG(test, mt7932_rx_frame(copy, cut, &frame), 0, + "length %zu", cut); + kfree(copy); + } + /* The descriptor claims more than the DMA buffer holds. */ + copy = kmemdup(packet, RX_SIZE - 1, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, copy); + KUNIT_EXPECT_EQ(test, mt7932_rx_frame(copy, RX_SIZE - 1, &frame), -EMSGSIZE); + kfree(copy); + /* Optional groups (3 and 5) that do not fit. */ + rx_build(packet, rx_cipher(4) | BIT(13) | BIT(15)); + put_unaligned_le32((RX_HEADER + 20) | (2U << 27), packet); + KUNIT_EXPECT_EQ(test, mt7932_rx_frame(packet, RX_HEADER + 20, &frame), -EMSGSIZE); +} + +static struct kunit_case mt7932_input_test_cases[] = { + KUNIT_CASE(mt7932_rx_decrypted_test), + KUNIT_CASE(mt7932_rx_plaintext_test), + KUNIT_CASE(mt7932_rx_mismatch_test), + KUNIT_CASE(mt7932_rx_integrity_test), + KUNIT_CASE(mt7932_rx_short_test), + {} +}; + +static struct kunit_suite mt7932_input_test_suite = { + .name = "mt7932-input", + .test_cases = mt7932_input_test_cases, +}; + +kunit_test_suite(mt7932_input_test_suite); diff --git a/drivers/net/wireless/mediatek/mt7932/packet.h b/drivers/net/wireless/mediatek/mt7932/packet.h index a93d04d91fd41..d7480c81872b2 100644 --- a/drivers/net/wireless/mediatek/mt7932/packet.h +++ b/drivers/net/wireless/mediatek/mt7932/packet.h @@ -2,6 +2,8 @@ #ifndef MT7932_PACKET_H #define MT7932_PACKET_H +#include + #include "protocol.h" #include "channels.h" @@ -10,10 +12,20 @@ struct mt7932_rx_frame { size_t length; u32 timestamp, rcpi, rxv; u16 frame_control, sequence_control, qos_control, wtbl; - u8 channel, amsdu; + u8 channel, amsdu, sec_mode; bool translated, group4, signal_valid, rate_valid, unicast; + bool cipher_mismatch; }; +/* RX descriptor word1: the cipher the hardware decrypted the frame with (0 + * when it did not decrypt it), and the cipher and cipher-length mismatch + * flags. The layout is the connac2 one. + */ +#define MT7932_RXD1_SEC_MODE_SHIFT 16 +#define MT7932_RXD1_SEC_MODE_MASK 0x1f +#define MT7932_RXD1_CIPHER_MISMATCH BIT(23) +#define MT7932_RXD1_CIPHER_LEN_MISMATCH BIT(24) + /* Strict non-monitor subset of the RX descriptor; its groups are ordered * 4, 1, 2, 3, 5, not by numerical group identifier. */ @@ -54,6 +66,9 @@ static inline int mt7932_rx_frame(const u8 *packet, size_t size, struct mt7932_r out->wtbl = w1 & 0x3ff; out->unicast = ((w3 >> 16) & 3) == 1; out->amsdu = w4 & 3; + out->sec_mode = (w1 >> MT7932_RXD1_SEC_MODE_SHIFT) & MT7932_RXD1_SEC_MODE_MASK; + out->cipher_mismatch = w1 & (MT7932_RXD1_CIPHER_MISMATCH | + MT7932_RXD1_CIPHER_LEN_MISMATCH); if (!mt7932_channel_band(out->channel)) return -EOPNOTSUPP; if ((out->translated && (w2 & (1U << 27))) || (!out->translated && out->amsdu)) @@ -103,6 +118,18 @@ static inline int mt7932_rx_frame(const u8 *packet, size_t size, struct mt7932_r return 0; } +/* + * On a protected connection, deliver only frames the hardware decrypted with + * the cipher and key length it expected. The firmware runs the WPA2 handshake + * itself, so the host has no unprotected EAPOL to accept. Nothing is + * decrypted on an open connection. + */ +static inline bool mt7932_rx_protection_ok(const struct mt7932_rx_frame *frame, + bool open) +{ + return open || (frame->sec_mode && !frame->cipher_mismatch); +} + static inline int mt7932_rx_signal(const struct mt7932_rx_frame *frame, u8 antenna_mask) { int signal = -128; From 951957af90c3f292d447a1bc5aa6697ed52a6e10 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 01:24:57 +1100 Subject: [PATCH 1420/2680] wifi: mt7932: complete a command only with the event that answers it A command that waits for a reply is completed by the first event, not otherwise dispatched, that carries its sequence number. Nothing checks that the event answers that command. A delayed event after the 8-bit sequence has wrapped, or a malformed one, can complete a different command; most callers then check the reply, but the RTS setup accepts any matching sequence and goes on to publish data readiness. Record the event that answers the command being sent and require it as well as the sequence. Extended commands answer with the extended event (0xed), the runtime capability query (0x8a) with 0xec, other runtime queries with an event of their own ID, the boot ROM's register read with 0x02, its patch semaphore with 0x04 and its status commands with 0x01, the event numbers other connac devices use. These are also the events the callers already check for, and the RTS reply is the extended event. The ROM's OTP read keeps sequence-only matching for now: no public source documents its reply event. Log the event it receives at debug level so that it can be measured and pinned. Add KUnit tests for the matcher and the reply table, including same-sequence events with the wrong event ID. Fixes: 8709ef5c7d40 ("wifi: add existing J700 MT7932 fullmac implementation") (cherry picked from commit 47dbbd82a926e90b6884c029ca9e343a8ce51ca4) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../net/wireless/mediatek/mt7932/input-test.c | 57 +++++++++++++++++++ drivers/net/wireless/mediatek/mt7932/mt7932.c | 5 +- drivers/net/wireless/mediatek/mt7932/mt7932.h | 2 + .../net/wireless/mediatek/mt7932/protocol.h | 32 ++++++++++- 4 files changed, 93 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/input-test.c b/drivers/net/wireless/mediatek/mt7932/input-test.c index 409103ed18740..9ee9253f34a30 100644 --- a/drivers/net/wireless/mediatek/mt7932/input-test.c +++ b/drivers/net/wireless/mediatek/mt7932/input-test.c @@ -116,12 +116,69 @@ static void mt7932_rx_short_test(struct kunit *test) KUNIT_EXPECT_EQ(test, mt7932_rx_frame(packet, RX_HEADER + 20, &frame), -EMSGSIZE); } +static void event_build(u8 *packet, u8 eid, u8 seq, u8 option) +{ + memset(packet, 0, 64); + put_unaligned_le32(64 | (7U << 27), packet); + put_unaligned_le16(40, packet + 24); + packet[28] = eid; + packet[29] = seq; + packet[30] = option; +} + +/* Only the event that answers a command completes it. */ +static void mt7932_event_reply_test(struct kunit *test) +{ + struct mt7932_event event; + u8 packet[64]; + int eid = mt7932_reply_eid(0xed, true); + + event_build(packet, 0xed, 9, 0); + KUNIT_ASSERT_EQ(test, mt7932_event_parse(packet, sizeof(packet), &event), 0); + KUNIT_EXPECT_TRUE(test, mt7932_event_matches(&event, 9, eid)); + /* Another sequence, the no-reply option bit, or nothing waiting. */ + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 10, eid)); + event.option = 4; + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 9, eid)); + event.option = 0; + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 0, eid)); + /* An unrelated event that happens to carry the waiting sequence. */ + event_build(packet, 0x99, 9, 0); + KUNIT_ASSERT_EQ(test, mt7932_event_parse(packet, sizeof(packet), &event), 0); + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 9, eid)); + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 9, mt7932_reply_eid(0x8a, true))); + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 9, mt7932_reply_eid(0x03, false))); + /* The ROM's patch semaphore answers with its own event only. */ + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 9, mt7932_reply_eid(0x10, false))); + event_build(packet, 0x04, 9, 0); + KUNIT_ASSERT_EQ(test, mt7932_event_parse(packet, sizeof(packet), &event), 0); + KUNIT_EXPECT_TRUE(test, mt7932_event_matches(&event, 9, mt7932_reply_eid(0x10, false))); + event_build(packet, 0x01, 9, 0); + KUNIT_ASSERT_EQ(test, mt7932_event_parse(packet, sizeof(packet), &event), 0); + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 9, mt7932_reply_eid(0x10, false))); +} + +static void mt7932_event_reply_eid_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0xed, false), 0xed); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0xed, true), 0xed); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0x8a, true), 0xec); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0xbf, true), 0xbf); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0x03, false), 0x02); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0x01, false), 0x01); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0x05, false), 0x01); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0x10, false), 0x04); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0x50, false), -1); +} + static struct kunit_case mt7932_input_test_cases[] = { KUNIT_CASE(mt7932_rx_decrypted_test), KUNIT_CASE(mt7932_rx_plaintext_test), KUNIT_CASE(mt7932_rx_mismatch_test), KUNIT_CASE(mt7932_rx_integrity_test), KUNIT_CASE(mt7932_rx_short_test), + KUNIT_CASE(mt7932_event_reply_test), + KUNIT_CASE(mt7932_event_reply_eid_test), {} }; diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 41d34e9c6b8c7..e879b46356994 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -205,7 +205,7 @@ static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) } goto unlock; } - if (mt7932_event_matches(&event, m->waiting) && !m->reply_length) { + if (mt7932_event_matches(&event, m->waiting, m->waiting_eid) && !m->reply_length) { memcpy(m->reply, packet, event.length); m->reply_length = event.length; complete(&m->response); @@ -377,6 +377,7 @@ int mt_request_ext(struct mt7932 *m, u8 cid, u8 ext, bool runtime, bool set, boo reinit_completion(&m->response); spin_lock_irqsave(&m->response_lock, flags); m->waiting = wait ? m->seq : 0; + m->waiting_eid = mt7932_reply_eid(cid, runtime); m->reply_length = 0; spin_unlock_irqrestore(&m->response_lock, flags); ret = mt_submit(m, q, ret); @@ -679,6 +680,8 @@ static int mt_otp_byte(struct mt7932 *m, u32 address, u8 *value) if (m->reply_length < 40) return -EPROTO; *value = m->reply[36]; + dev_dbg(&m->pdev->dev, "OTP_REPLY: eid=%02x word=%08x address=%08x length=%zu\n", + m->reply[28], get_unaligned_le32(m->reply + 32), address, m->reply_length); return 0; } diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 6994f0c208105..37c34872fe66f 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -75,6 +75,8 @@ struct mt7932 { unsigned int cal_request_head, cal_request_count; spinlock_t response_lock; u8 seq, waiting; + /* Event that answers the waiting command; see mt7932_reply_eid(). */ + int waiting_eid; u8 mac[ETH_ALEN]; u8 reply[MT7932_RX_CAPACITY]; size_t reply_length; diff --git a/drivers/net/wireless/mediatek/mt7932/protocol.h b/drivers/net/wireless/mediatek/mt7932/protocol.h index 00751e130ef44..e7f8820a9ff8c 100644 --- a/drivers/net/wireless/mediatek/mt7932/protocol.h +++ b/drivers/net/wireless/mediatek/mt7932/protocol.h @@ -65,9 +65,37 @@ static inline int mt7932_event_parse(const u8 *packet, size_t length, return 0; } -static inline bool mt7932_event_matches(const struct mt7932_event *e, u8 seq) +/* + * The event that answers a command waiting for a reply, or -1 for any event. + * Extended commands answer with the extended event, the capability query + * with its own event, and the boot ROM's status commands with the generic + * one. The ROM's register read answers with the access-register event and + * its patch semaphore with the patch-semaphore event, as on other connac + * devices. Other runtime commands answer with an event of their own ID. + * The ROM's OTP read alone still matches on the sequence: its reply event + * is not documented, and mt_otp_byte() records the one it receives. + */ +static inline int mt7932_reply_eid(u8 cid, bool runtime) { - return seq && e->seq == seq && !(e->option & 4); + if (cid == 0xed) + return 0xed; + if (runtime) + return cid == 0x8a ? 0xec : cid; + switch (cid) { + case 0x03: + return 0x02; + case 0x10: + return 0x04; + case 0x50: + return -1; + default: + return 0x01; + } +} + +static inline bool mt7932_event_matches(const struct mt7932_event *e, u8 seq, int eid) +{ + return seq && e->seq == seq && !(e->option & 4) && (eid < 0 || e->eid == eid); } #endif From 701335cc7f6784079c43f705578e0fe5496276bf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 01:25:22 +1100 Subject: [PATCH 1421/2680] wifi: mt7932: refuse hibernation The driver supports only suspend-to-idle and refuses other suspend states in its .suspend callback. Hibernation does not go through .suspend: its freeze phase calls .freeze, which the driver does not provide, so the PCI core lets it pass with the firmware still running. The firmware keeps writing to the receive rings while the memory image is saved, and after a restore the kernel resumes with ring state that the firmware started by the booting kernel does not have. Refuse the freeze and poweroff phases. The PM core then aborts hibernation before the device changes state, so there is nothing to undo. Supporting hibernation needs DMA retirement on freeze and a full reinitialisation, including the regulatory pass, on restore. Fixes: b4d89a96d6b0 ("wifi: mt7932: keep the function running across suspend-to-idle") (cherry picked from commit b5c3a48a07eb169431f557f2a00df3b9f8e72320) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index e879b46356994..fb07f9fcb790f 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -1259,8 +1259,24 @@ static int mt_suspend(struct device *dev) return 0; } +/* + * Hibernation is refused. Its freeze phase does not call .suspend, so without + * this the firmware would keep writing to memory while the image is saved, + * and the restored kernel would resume with ring state the firmware of the + * booting kernel does not have. Supporting it needs DMA retirement and a full + * reinitialisation on restore. Returning an error here has no side effect to + * undo: the PM core aborts hibernation before this device changes state. + */ +static int mt_freeze(struct device *dev) +{ + dev_err(dev, "hibernation is not supported\n"); + return -EOPNOTSUPP; +} + static const struct dev_pm_ops mt_pm_ops = { .suspend = mt_suspend, + .freeze = mt_freeze, + .poweroff = mt_freeze, }; static const struct pci_device_id mt_ids[] = { From 8ccc132c3dd1fd01dc648a94c924a49dc7ababb5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 22:11:50 +1100 Subject: [PATCH 1422/2680] soc: apple: sep: Name the failed step of a Touch ID sensor registration When the enclave did not register the sensor, the log said so and nothing else. Eight steps can fail on the way: an enclave refusal of REGISTER_SENSOR, REGISTER_SENSOR_SERIAL, the session share or its commit, and a failed serial read, nonce read or session exchange on the sensor side. Most of them returned false without a word, so a failure could not be told apart from any other. Name the step that failed with the enclave status or the error, and how long after the sensor was powered on. Then report what the sensor shows afterwards: its state, the patch acknowledgment, whether it presents an identifier and how long a frame it holds. The sensor data itself is never logged. The sensor_trace parameter also logs each step as it completes, with its time since power-on. The host test that runs wake_sensor() gets the new fields in its mock. (cherry picked from commit 43731176c824f8f00ad227cf7769ae9e7264a09a) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 137 ++++++++++++++++----- drivers/soc/apple/sensor.rs | 8 ++ drivers/soc/apple/sep.rs | 9 ++ tools/aurora-sep/tests/test_sensor_wake.py | 15 ++- 4 files changed, 134 insertions(+), 35 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index ba21814a32a50..d60cec976fa26 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1701,6 +1701,64 @@ impl SepData { } } + fn sensor_ms(&self) -> u64 { + shim::boottime_ns().saturating_sub(self.sensor_power_ns.load(Relaxed)) / 1_000_000 + } + + fn sensor_trace(&self, step: &CStr) { + if *module_parameters::sensor_trace.value() != 0 { + dev_info!(self.dev, "sensor: {} done at {} ms\n", step, self.sensor_ms()); + } + } + + /// Reports a failed registration step with what the sensor shows + /// afterwards: its state, patch acknowledgment, whether it presents an + /// identifier and the length of any frame it holds. Never the + /// identifier, the serial or any key material. + fn sensor_step_failed(&self, step: &CStr) { + let ms = self.sensor_ms(); + match sensor::status() { + Ok(st) => dev_warn!( + self.dev, + "sensor: {} failed {} ms after power-on; sensor now state {} ack {:#04x} identifier {} holding {} bytes{}\n", + step, + ms, + st.state, + st.patch_ack(), + st.identifier_to_register().is_some(), + st.available().unwrap_or(0), + if st.is_silent() { c" (silent)" } else { c"" } + ), + Err(e) => dev_warn!( + self.dev, + "sensor: {} failed {} ms after power-on; its status read failed too ({:?})\n", + step, + ms, + e + ), + } + } + + /// One enclave request of the registration. A refusal is named. + fn sensor_enclave_step(&self, op: &crate::sbio::SbioOp) -> Option> { + match self.sbio_transfer(op) { + Ok(done) if done.status.is_ok() => { + self.sensor_trace(op.name()); + Some(done.payload) + } + Ok(done) => { + dev_warn!(self.dev, "sensor: {} refused with status {}\n", op.name(), done.status); + self.sensor_step_failed(op.name()); + None + } + Err(e) => { + dev_warn!(self.dev, "sensor: {} transfer failed: {:?}\n", op.name(), e); + self.sensor_step_failed(op.name()); + None + } + } + } + fn register_sensor(&self, id: &sensor::Identifier) -> bool { let stage = self.bringup.load(Relaxed); dev_info!(self.dev, "sbio: registering sensor at bring-up stage {}\n", stage); @@ -1714,9 +1772,12 @@ impl SepData { return false; } }; - if !self.sensor_session_request(0x48, c"PLATFORM_IDENTITY", &platform) - || self.sbio_expect_ok(&crate::sbio::sbio_register_sensor(id)).is_none() - { + if !self.sensor_session_request(0x48, c"PLATFORM_IDENTITY", &platform) { + self.sensor_step_failed(c"PLATFORM_IDENTITY"); + return false; + } + self.sensor_trace(c"PLATFORM_IDENTITY"); + if self.sensor_enclave_step(&crate::sbio::sbio_register_sensor(id)).is_none() { return false; } } @@ -1724,7 +1785,7 @@ impl SepData { if stage >= BRINGUP_ESTABLISHED { let _ = self.log_identity_count(c"before CLEAR_STATE for sensor re-registration"); if self - .sbio_expect_ok(&crate::sbio::sbio_clear_state()) + .sensor_enclave_step(&crate::sbio::sbio_clear_state()) .is_none() { return false; @@ -1752,23 +1813,24 @@ impl SepData { let serial = match sensor::read_sensor_serial() { Ok(serial) => serial, - Err(_) => { + Err(e) => { + dev_warn!(self.dev, "sensor: serial read failed: {:?}\n", e); + self.sensor_step_failed(c"serial read"); return false; } }; + self.sensor_trace(c"serial read"); let op = crate::sbio::sbio_register_sensor_serial(&serial); - match self.sbio_call(&op) { - SbioOutcome::Ok(_payload) => { - if self.profile.nonce_session && !self.authenticate_sensor_session() { - return false; - } - self.bringup.store(BRINGUP_ESTABLISHED, Relaxed); - let _ = self.log_identity_count(c"after sensor serial registration"); - true - } - _ => false, + if self.sensor_enclave_step(&op).is_none() { + return false; + } + if self.profile.nonce_session && !self.authenticate_sensor_session() { + return false; } + self.bringup.store(BRINGUP_ESTABLISHED, Relaxed); + let _ = self.log_identity_count(c"after sensor serial registration"); + true } fn sensor_platform_identity() -> Result<[u8; 12]> { @@ -1802,12 +1864,16 @@ impl SepData { Ok(nonce) => nonce, Err(error) => { dev_err!(self.dev, "sensor: nonce read failed: {:?}\n", error); + self.sensor_step_failed(c"nonce read"); return false; } }; + self.sensor_trace(c"nonce read"); if !self.sensor_session_request(0x81, c"SENSOR_NONCE", &nonce) { + self.sensor_step_failed(c"SENSOR_NONCE"); return false; } + self.sensor_trace(c"SENSOR_NONCE"); if !self.sensor_key_loaded.load(Relaxed) { let firmware = match kernel::firmware::Firmware::request( @@ -1826,9 +1892,11 @@ impl SepData { if firmware.data().is_empty() || !self.sensor_session_request(0x86, c"SENSOR_ROOT_KEY", firmware.data()) { + self.sensor_step_failed(c"SENSOR_ROOT_KEY"); return false; } self.sensor_key_loaded.store(true, Relaxed); + self.sensor_trace(c"SENSOR_ROOT_KEY"); } // Each identification starts a new sensor-side exchange, even though // the signed key remains loaded for the lifetime of the enclave. @@ -1837,6 +1905,7 @@ impl SepData { fn wake_sensor(&self) -> Option { let _source = sensor::power_source(); + self.sensor_power_ns.store(shim::boottime_ns(), Relaxed); let cycled = sensor::power_cycle(); if !sensor::cs_timing().is_hardware() { dev_warn!( @@ -1877,6 +1946,13 @@ impl SepData { answered_without_identifier = true; continue; }; + if *module_parameters::sensor_trace.value() != 0 { + dev_info!( + self.dev, + "sensor: identifier present {} ms after power-on\n", + self.sensor_ms() + ); + } if !self.register_sensor(&id) { dev_warn!(self.dev, "sensor: the enclave did not register the sensor\n"); return None; @@ -2090,43 +2166,37 @@ impl SepData { } fn establish_session(&self) -> bool { - let share = match self.sbio_call(&crate::sbio::sbio_request_session_share()) { - SbioOutcome::Ok(sh) => sh, - SbioOutcome::Status16 => { - return false; - } - SbioOutcome::PrerequisiteMissing => { - return false; - } - SbioOutcome::Other => { - return false; - } + let Some(share) = self.sensor_enclave_step(&crate::sbio::sbio_request_session_share()) + else { + return false; }; // 0x30 reserved for this reply; length is a floor, take the first 40 if share.len() < sensor::SESSION_SHARE_LEN { + dev_warn!(self.dev, "sensor: short session share ({} bytes)\n", share.len()); return false; } let mut out = [0u8; sensor::SESSION_SHARE_LEN]; out.copy_from_slice(&share[..sensor::SESSION_SHARE_LEN]); // relay it: class 0x72, no CRC, no padding - if sensor::send_session_share(&out).is_err() { + if let Err(e) = sensor::send_session_share(&out) { + dev_warn!(self.dev, "sensor: sending the session share failed: {:?}\n", e); + self.sensor_step_failed(c"session share send"); return false; } let reply = match sensor::read_session_reply() { Ok(r) => r, - Err(_) => { + Err(e) => { + dev_warn!(self.dev, "sensor: session reply read failed: {:?}\n", e); + self.sensor_step_failed(c"session reply read"); return false; } }; - match self.sbio_transfer(&crate::sbio::sbio_commit_session_share(&reply)) { - Ok(done) if done.status.is_ok() => true, - Ok(_) => false, - Err(_) => false, - } + self.sensor_enclave_step(&crate::sbio::sbio_commit_session_share(&reply)) + .is_some() } // Each failure names its step: a report that only says "sequence counter @@ -2180,6 +2250,7 @@ impl SepData { Ok(done) if done.status.is_ok() => true, Ok(done) => { dev_warn!(self.dev, "sequence counter: the enclave refused the sensor's reply, status {}\n", done.status); + self.sensor_step_failed(c"challenge commit"); false } Err(e) => { diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index 3fcba35e39428..dc3df5c6b1d34 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -635,6 +635,14 @@ impl Identifier { } impl Status { + /// The length of the frame the sensor offers, if it offers one. + pub(crate) fn available(&self) -> Option { + match self.offset12() { + Offset12::Available(n) => Some(n), + _ => None, + } + } + pub(crate) fn offset12(&self) -> Offset12 { match self.state { STATE_IDLE => Offset12::Identifier(u16::from_le_bytes([ diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 6d1182b17b8f4..066f2a4c655b5 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -627,6 +627,10 @@ struct SepData { last_capture_end_ns: Atomic, + // Boot-time clock at the start of the last sensor power cycle, for + // reporting how far into a bring-up a step failed. + sensor_power_ns: Atomic, + device_view_synced: Atomic, keybag_designated: Atomic, @@ -851,6 +855,7 @@ impl SepData { enrol_material <- new_mutex!(None), enrol_identity_candidates <- new_mutex!(KVec::new()), last_capture_end_ns: Atomic::new(0), + sensor_power_ns: Atomic::new(0), device_view_synced: Atomic::new(false), phase: Atomic::new(PHASE_ATTACH), ool_xarm <- new_mutex!(Some(ool_xarm)), @@ -2472,6 +2477,10 @@ module! { default: 0, description: "Low 64 bits of an explicit xART OS UUID", }, + sensor_trace: u8 { + default: 0, + description: "Trace Touch ID sensor registration steps and timing, never sensor data", + }, xarm_trace: u8 { default: 0, description: "Opt-in xART opcode/status trace without payloads or key material", diff --git a/tools/aurora-sep/tests/test_sensor_wake.py b/tools/aurora-sep/tests/test_sensor_wake.py index c9bb8c6af7579..cd1c9c74a793e 100644 --- a/tools/aurora-sep/tests/test_sensor_wake.py +++ b/tools/aurora-sep/tests/test_sensor_wake.py @@ -22,6 +22,15 @@ def test_post_registration_status_drives_patch_readiness(self): #![allow(dead_code, unused_variables)] use std::cell::RefCell; macro_rules! dev_warn { ($($args:tt)*) => {} } +macro_rules! dev_info { ($($args:tt)*) => {} } +use std::sync::atomic::{AtomicU64, Ordering::Relaxed}; +mod shim { pub fn boottime_ns()->u64 {0} } +mod module_parameters { + pub struct Param(u8); + impl Param { pub fn value(&self)->&u8 {&self.0} } + #[allow(non_upper_case_globals)] pub static sensor_trace: Param = Param(0); +} +struct Profile { nonce_session: bool } macro_rules! dev_err { ($($args:tt)*) => {} } mod kernel { pub mod time { pub struct Delta; @@ -60,8 +69,10 @@ def test_post_registration_status_drives_patch_readiness(self): pub fn patch_ack(&self)->u8 {self.ack} } } -struct Driver; +struct Driver { profile: Profile, sensor_power_ns: AtomicU64 } impl Driver { + fn new()->Self { Driver { profile: Profile { nonce_session: false }, sensor_power_ns: AtomicU64::new(0) } } + fn sensor_ms(&self)->u64 {0} fn register_sensor(&self,_:&sensor::Identifier)->bool { MOCK.with(|m| {let mut m=m.borrow_mut();assert_eq!(m.reads,1);m.registered=true;}); true } @@ -80,7 +91,7 @@ def test_post_registration_status_drives_patch_readiness(self): (Some(sensor::Status{state:0,ack:0,silent:false}),true,Some(0)), ] { MOCK.with(|m| *m.borrow_mut()=Mock{next,..Mock::default()}); - assert_eq!(Driver.wake_sensor().is_some(),ready); + assert_eq!(Driver::new().wake_sensor().is_some(),ready); MOCK.with(|m| {let m=m.borrow();assert_eq!(m.reads,2);assert!(m.registered);assert_eq!(m.patch,patch);}); } } From ab6cc0067d5b13a6c1d81751a418089d16650176 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:34:11 +1100 Subject: [PATCH 1423/2680] wifi: mt7932: let a cancelled scan batch retire before a disconnect gives up The disconnect worker quiesces scanning before it retires the connection. mt_scan_quiesce() waits for the current batch to finish, aborts the scan and returns -EBUSY if a cancelled batch has not yet been reported finished. The disconnect worker treats that as fatal and sets link_failed, which nothing clears: the radio stays unusable until the next boot. A multi-batch scan makes this reachable without any firmware fault. When the awaited batch finishes, the scan completion work may submit the next batch before the abort is seen. The abort then cancels that batch, and its end is reported by the firmware a little later, after the quiesce has already returned -EBUSY. cfg80211 disconnects on every suspend, so a background scan at suspend time is enough. Wait for the cancelled batch to retire, bounded to two seconds, as mt_scan_retire_wait() already does for an address change. Only a scan that the firmware never ends still fails the link. Add a KUnit test that drives mt_scan_quiesce() with a synthetic scan: the completion work has already submitted the next batch when the abort arrives, and the firmware reports its end 300 ms later (success) or never (-EBUSY after the two-second bound). Fixes: 8709ef5c7d40 ("wifi: add existing J700 MT7932 fullmac implementation") (cherry picked from commit cd6b06ffbc497cd783cf07147e61a897ab6757dc) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../net/wireless/mediatek/mt7932/frontend.c | 26 +++--- .../net/wireless/mediatek/mt7932/rf-test.c | 91 +++++++++++++++++++ 2 files changed, 105 insertions(+), 12 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 8d873a4379bfa..84170c8f61326 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -123,18 +123,6 @@ void mt_abort_scan(struct wiphy *wiphy, struct wireless_dev *wdev) spin_unlock_irqrestore(&m->response_lock, flags); } -/* No command/response lock held. The caller closes new scan admission first. - * Natural EID0d completion fences the scan; a local cancel alone does not. - */ -int mt_scan_quiesce(struct mt7932 *m) -{ - if (READ_ONCE(m->scan_request) || READ_ONCE(m->retired_scan_seq)) - wait_for_completion_timeout(&m->scan_done, msecs_to_jiffies(5000)); - mt_abort_scan(m->wiphy, &m->wdev); - flush_work(&m->scan_finish_work); - return READ_ONCE(m->retired_scan_seq) ? -EBUSY : 0; -} - /* Wait at most @timeout in total for a cancelled scan to retire. Callers * may hold RTNL, so this never waits longer than asked. A stopping driver * completes scan_done for good, which would turn the wait into a spin. @@ -153,6 +141,20 @@ static void mt_scan_retire_wait(struct mt7932 *m, unsigned long timeout) } } +/* No command/response lock held. The caller closes new scan admission first. + * Natural EID0d completion fences the scan; a local cancel alone does not. + * The batch awaited here may already have been followed by the next one, + * which the abort then cancels: give the firmware time to end that one too, + * rather than declaring the link unrecoverable while its cancel is in flight. + */ +int mt_scan_quiesce(struct mt7932 *m) +{ + if (READ_ONCE(m->scan_request) || READ_ONCE(m->retired_scan_seq)) + wait_for_completion_timeout(&m->scan_done, msecs_to_jiffies(5000)); + mt_scan_retire_wait(m, msecs_to_jiffies(2000)); + return READ_ONCE(m->retired_scan_seq) ? -EBUSY : 0; +} + /* command_mutex held: one RF-band/type batch at a time, with matching EID0d * completion before the next batch. A request without SSIDs is passive; * active requests must also listen passively on NO_IR channels. diff --git a/drivers/net/wireless/mediatek/mt7932/rf-test.c b/drivers/net/wireless/mediatek/mt7932/rf-test.c index da078caa2444a..c95f4057a2305 100644 --- a/drivers/net/wireless/mediatek/mt7932/rf-test.c +++ b/drivers/net/wireless/mediatek/mt7932/rf-test.c @@ -298,6 +298,96 @@ static void mt_startup_band1_queue_drain_test(struct kunit *test) m->cal_test_context = NULL; } +struct mt_scan_retire_test_context { + struct delayed_work retire; + struct mt7932 *m; + bool late; +}; + +static struct mt_scan_retire_test_context *mt_scan_retire_ctx; + +/* The firmware reports the end of the cancelled batch a little later. */ +static void mt_scan_retire_test_late(struct work_struct *work) +{ + struct mt_scan_retire_test_context *ctx = + container_of(to_delayed_work(work), struct mt_scan_retire_test_context, retire); + struct mt7932 *m = ctx->m; + unsigned long flags; + + spin_lock_irqsave(&m->response_lock, flags); + m->retired_scan_seq = 0; + complete(&m->scan_done); + spin_unlock_irqrestore(&m->response_lock, flags); +} + +/* + * The scan completion work had already submitted the next batch when the + * abort arrived: the abort cancels it, and its end is still to come. + */ +static void mt_scan_retire_test_finish(struct work_struct *work) +{ + struct mt_scan_retire_test_context *ctx = mt_scan_retire_ctx; + struct mt7932 *m = ctx->m; + unsigned long flags; + + spin_lock_irqsave(&m->response_lock, flags); + if (m->scan_request && m->scan_aborted) { + m->scan_request = NULL; + m->retired_scan_seq = 8; + } + spin_unlock_irqrestore(&m->response_lock, flags); + if (ctx->late) + schedule_delayed_work(&ctx->retire, msecs_to_jiffies(300)); +} + +static void mt_scan_quiesce_retirement_test(struct kunit *test) +{ + static const struct cfg80211_ops ops = {}; + struct mt_scan_retire_test_context ctx = {}; + struct cfg80211_scan_request *scan; + struct mt7932 *m = test->priv; + unsigned long start; + unsigned int round; + + scan = kunit_kzalloc(test, sizeof(*scan), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, scan); + m->wiphy = wiphy_new(&ops, sizeof(m)); + KUNIT_ASSERT_NOT_NULL(test, m->wiphy); + *(struct mt7932 **)wiphy_priv(m->wiphy) = m; + init_completion(&m->scan_done); + INIT_WORK(&m->scan_finish_work, mt_scan_retire_test_finish); + INIT_DELAYED_WORK(&ctx.retire, mt_scan_retire_test_late); + ctx.m = m; + mt_scan_retire_ctx = &ctx; + for (round = 0; round < 2; round++) { + ctx.late = !round; + reinit_completion(&m->scan_done); + m->scan_request = scan; + scan->wdev = &m->wdev; + m->scan_aborted = false; + m->retired_scan_seq = 0; + /* The batch the quiesce waits for has just finished. */ + complete(&m->scan_done); + start = jiffies; + if (ctx.late) { + /* A late but genuine end retires the scan: no failure. */ + KUNIT_EXPECT_EQ(test, mt_scan_quiesce(m), 0); + KUNIT_EXPECT_EQ(test, m->retired_scan_seq, 0); + } else { + /* A batch the firmware never ends still fails, after the bound. */ + KUNIT_EXPECT_EQ(test, mt_scan_quiesce(m), -EBUSY); + KUNIT_EXPECT_EQ(test, m->retired_scan_seq, 8); + KUNIT_EXPECT_GE(test, jiffies - start, msecs_to_jiffies(1900)); + KUNIT_EXPECT_LE(test, jiffies - start, msecs_to_jiffies(3000)); + } + cancel_delayed_work_sync(&ctx.retire); + flush_work(&m->scan_finish_work); + } + mt_scan_retire_ctx = NULL; + wiphy_free(m->wiphy); + m->wiphy = NULL; +} + static struct kunit_case mt_rf_test_cases[] = { KUNIT_CASE(mt_startup_band1_queue_drain_test), KUNIT_CASE(mt_missing_policy_admission_retry_test), @@ -305,6 +395,7 @@ static struct kunit_case mt_rf_test_cases[] = { KUNIT_CASE(mt_rf_failure_wakes_association_test), KUNIT_CASE(mt_rf_failure_blocks_startup_test), KUNIT_CASE(mt_rf_failure_preserves_first_error_test), + KUNIT_CASE(mt_scan_quiesce_retirement_test), {} }; From 513e05f95951aa16d284fd441979294f0c7c379a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 22:29:43 +1100 Subject: [PATCH 1424/2680] soc: apple: sep: Clear a partial sensor registration before retrying it On a profile with a nonce session, every registration identifies the sensor first: PLATFORM_IDENTITY, then REGISTER_SENSOR. The enclave state is only cleared once the bring-up stage reads ESTABLISHED. If a registration fails after REGISTER_SENSOR was accepted, the stage stays FRESH while the enclave holds the partial registration. Each retry then identified the sensor again without clearing it. On boot d1962b3a, all 42 retries over 800 s failed that way, and Touch ID stayed unusable until a reboot. Count the enclave as holding the sensor once REGISTER_SENSOR has been submitted, answered or not, since a lost or late reply does not mean the enclave did not act on it. Clear its sensor state before registering again whenever it may hold one. Restarting the bring-up from FRESH does not change that, because only CLEAR_STATE empties the enclave. Profiles without a nonce session keep their flow. The decision is a plain function, with KUnit tests for the first registration, the retry after a partial one, re-registration and the other profiles. (cherry picked from commit ef320e3fffa34b00bddb40d3122513eab3d10b71) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/registration.rs | 116 ++++++++++++++++++++++++++++++ drivers/soc/apple/sbio.rs | 26 +++++-- drivers/soc/apple/sep.rs | 11 ++- 3 files changed, 146 insertions(+), 7 deletions(-) create mode 100644 drivers/soc/apple/registration.rs diff --git a/drivers/soc/apple/registration.rs b/drivers/soc/apple/registration.rs new file mode 100644 index 0000000000000..95edd6a0b65b5 --- /dev/null +++ b/drivers/soc/apple/registration.rs @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Ryan Murray + +//! What a Touch ID sensor registration has to send to the enclave. +//! +//! A registration identifies the sensor (REGISTER_SENSOR), registers its +//! serial and, on profiles with a nonce session, authenticates it. Any of +//! those steps can fail after the enclave has already accepted an earlier +//! one, and then the enclave holds a partial registration. On boot +//! d1962b3a every retry that registered the sensor again without clearing +//! it failed, 42 of them over 800 s. Re-registration that identifies the +//! sensor and then clears the enclave's sensor state, the order every +//! verify uses, has not failed on an idle machine. So the next attempt +//! clears whenever the enclave may hold any registration, complete or not. + +/// Nothing is registered with the enclave yet. +pub(crate) const FRESH: u32 = 0; +/// The sensor is identified; profiles without a nonce session register the +/// serial next. +pub(crate) const IDENTIFIED: u32 = 1; +/// The registration completed. +pub(crate) const ESTABLISHED: u32 = 2; + +/// The enclave requests one registration attempt makes, in order. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Plan { + /// PLATFORM_IDENTITY and REGISTER_SENSOR, before anything else. A + /// nonce-session profile identifies the sensor on every reset. + pub(crate) identify_first: bool, + /// CLEAR_STATE, after any identification. + pub(crate) clear_state: bool, + /// REGISTER_SENSOR after the clear, on a profile without a nonce session. + pub(crate) register_sensor: bool, +} + +/// Plans one registration attempt. +/// +/// `stage` is the bring-up stage and `holds_sensor` whether an +/// identification of the sensor was submitted to the enclave since it +/// booted, answered or not: a lost reply does not mean the enclave did not +/// act on it. `sepos13` says whether the profile starts from the serial, as +/// the 13.5 key store does. +pub(crate) fn plan(stage: u32, holds_sensor: bool, nonce_session: bool, sepos13: bool) -> Plan { + Plan { + identify_first: nonce_session, + clear_state: stage >= ESTABLISHED || (nonce_session && holds_sensor), + register_sensor: stage == FRESH && !nonce_session && !sepos13, + } +} + +#[kernel::macros::kunit_tests(apple_sep_registration)] +mod tests { + use super::*; + + #[test] + fn first_registration_after_boot_does_not_clear() { + assert_eq!( + plan(FRESH, false, true, false), + Plan { + identify_first: true, + clear_state: false, + register_sensor: false + } + ); + } + + #[test] + fn retry_after_a_partial_registration_clears() { + // REGISTER_SENSOR was accepted, then the serial, nonce or session + // step failed: the bring-up stage is still FRESH. + let retry = plan(FRESH, true, true, false); + assert!(retry.identify_first); + assert!(retry.clear_state); + } + + #[test] + fn re_registration_after_a_complete_one_clears() { + assert!(plan(ESTABLISHED, true, true, false).clear_state); + assert!(plan(ESTABLISHED, false, false, false).clear_state); + } + + #[test] + fn a_fresh_restart_still_clears_what_the_enclave_holds() { + // Restarting the bring-up from FRESH does not erase the enclave's + // registration; only CLEAR_STATE does. + assert!(plan(FRESH, true, true, false).clear_state); + } + + #[test] + fn profiles_without_a_nonce_session_are_unchanged() { + assert_eq!( + plan(FRESH, true, false, false), + Plan { + identify_first: false, + clear_state: false, + register_sensor: true + } + ); + assert_eq!( + plan(IDENTIFIED, true, false, false), + Plan { + identify_first: false, + clear_state: false, + register_sensor: false + } + ); + assert_eq!( + plan(FRESH, false, false, true), + Plan { + identify_first: false, + clear_state: false, + register_sensor: false + } + ); + } +} diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index d60cec976fa26..e15af135453ac 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1761,9 +1761,24 @@ impl SepData { fn register_sensor(&self, id: &sensor::Identifier) -> bool { let stage = self.bringup.load(Relaxed); - dev_info!(self.dev, "sbio: registering sensor at bring-up stage {}\n", stage); + let plan = registration::plan( + stage, + self.sensor_held.load(Relaxed), + self.profile.nonce_session, + matches!(self.profile.key_store, profile::KeyStore::Sepos13 { .. }), + ); + dev_info!( + self.dev, + "sbio: registering sensor at bring-up stage {}{}\n", + stage, + if plan.clear_state && stage < BRINGUP_ESTABLISHED { + c", clearing the enclave's partial registration first" + } else { + c"" + } + ); - if self.profile.nonce_session { + if plan.identify_first { // J700 needs identification before CLEAR_STATE on every reset. let platform = match Self::sensor_platform_identity() { Ok(platform) => platform, @@ -1777,12 +1792,15 @@ impl SepData { return false; } self.sensor_trace(c"PLATFORM_IDENTITY"); + // Counted as held from submission on: a reply that is lost or + // late does not mean the enclave did not act on the request. + self.sensor_held.store(true, Relaxed); if self.sensor_enclave_step(&crate::sbio::sbio_register_sensor(id)).is_none() { return false; } } - if stage >= BRINGUP_ESTABLISHED { + if plan.clear_state { let _ = self.log_identity_count(c"before CLEAR_STATE for sensor re-registration"); if self .sensor_enclave_step(&crate::sbio::sbio_clear_state()) @@ -1799,7 +1817,7 @@ impl SepData { // The 13.5 J313 flow begins with REGISTER_SENSOR_SERIAL (0x17). // The newer REGISTER_SENSOR (0x80) returned 0x16 on this firmware. self.bringup.store(BRINGUP_IDENTIFIED, Relaxed); - } else if stage == BRINGUP_FRESH && !self.profile.nonce_session { + } else if plan.register_sensor { let op = crate::sbio::sbio_register_sensor(id); match self.sbio_call(&op) { SbioOutcome::Ok(_payload) => { diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 066f2a4c655b5..9608d92ef776a 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -25,6 +25,7 @@ mod profile; mod proto; mod refkey; mod refkey_seal; +mod registration; mod rxring; mod sbio; mod scrd; @@ -251,9 +252,9 @@ enum SbioOutcome { Other, } -const BRINGUP_FRESH: u32 = 0; -const BRINGUP_IDENTIFIED: u32 = 1; -const BRINGUP_ESTABLISHED: u32 = 2; +const BRINGUP_FRESH: u32 = registration::FRESH; +const BRINGUP_IDENTIFIED: u32 = registration::IDENTIFIED; +const BRINGUP_ESTABLISHED: u32 = registration::ESTABLISHED; pub(crate) struct PatchLoaded(()); @@ -640,6 +641,9 @@ struct SepData { touchid_failed: Atomic, bringup: Atomic, + // The enclave accepted an identification of the sensor since it booted, + // even if the registration it began then failed. + sensor_held: Atomic, #[pin] ool_sks: Mutex>, @@ -841,6 +845,7 @@ impl SepData { sks_initialized: Atomic::new(false), sks_wedged: Atomic::new(0), bringup: Atomic::new(BRINGUP_FRESH), + sensor_held: Atomic::new(false), keybag_designated: Atomic::new(false), bringup_started: Atomic::new(false), touchid_started: Atomic::new(false), From ac0033fd641b15614043cfcdab3d5692e1c5d23d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:34:59 +1100 Subject: [PATCH 1425/2680] wifi: mt7932: report the connection outcome outside the response lock The connect worker publishes its outcome with response_lock held and interrupts disabled, so that a late firmware success cannot raise the carrier after a cancellation. Some of those cfg80211 calls take bottom-half spinlocks: - a connect cancelled while it was in progress, which happens whenever the system suspends during a connect, drops the BSS reference with cfg80211_put_bss(), which takes rdev->bss_lock with spin_lock_bh(); - cfg80211_connect_done() looks the BSS up again, under the same lock, when it has dropped off the scan list, and drops the reference when it cannot allocate its event. Re-enabling bottom halves with interrupts disabled is not allowed: lockdep reports it, and pending softirqs would then run with response_lock held. Keep deciding the outcome, the connection flags and the carrier under the lock, then make the cfg80211 calls after dropping it, with GFP_KERNEL. The ordering against cancellation still holds: - mt_disconnect() and a firmware disconnect event both leave the report to the disconnect worker, which retires the connection under command_mutex first, and this worker holds command_mutex until its own report is queued; - an RF failure reports a lost connection directly. While the worker is publishing, the failure now marks the connection lost instead, and the worker reports it once its own result has been queued, so cfg80211 never sees the disconnect before the connect. A connect cancelled by userspace or by suspend is not a driver failure: log it at debug level instead of as an error. Add a KUnit test for an RF failure during publication: the failure marks the connection lost and leaves the report to the worker. Fixes: 8709ef5c7d40 ("wifi: add existing J700 MT7932 fullmac implementation") (cherry picked from commit f69dfc9ee6a4bf3bd9fe83fc06cd09b68dfa5474) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/link.c | 72 +++++++++++++------ drivers/net/wireless/mediatek/mt7932/mt7932.h | 4 ++ .../net/wireless/mediatek/mt7932/rf-test.c | 25 +++++++ 3 files changed, 78 insertions(+), 23 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index a1c345ec1f64e..4e9ff20623da0 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -71,11 +71,17 @@ void mt_rf_fail_locked(struct mt7932 *m, int error) complete_all(&m->assoc_done); complete_all(&m->discovery_done); } else if (m->connected) { - /* Hardware ownership is uncertain: report failure, retain the peer. */ + /* Hardware ownership is uncertain: report failure, retain the peer. + * A connect worker still publishing the connection reports this + * after it, so that cfg80211 sees the two in order. + */ m->connected = false; m->disconnecting = true; - cfg80211_disconnected(m->netdev, WLAN_REASON_UNSPECIFIED, - NULL, 0, true, GFP_ATOMIC); + if (m->connect_publishing) + m->connect_lost = true; + else + cfg80211_disconnected(m->netdev, WLAN_REASON_UNSPECIFIED, + NULL, 0, true, GFP_ATOMIC); } if (m->scan_request) { m->scan_aborted = true; @@ -388,7 +394,8 @@ void mt_connect_work(struct work_struct *work) struct cfg80211_connect_resp_params response = {}; u8 *body, mac[8] = {}, queue[32]; unsigned long flags; - bool submitted = false, peer_owned = false; + bool submitted = false, peer_owned = false, cancelled; + u16 reason; int ret; mutex_lock(&m->command_mutex); @@ -504,7 +511,10 @@ void mt_connect_work(struct work_struct *work) } /* Serialize the final outcome with cancellation. In particular, a late * firmware success must not publish carrier/authorization after .deauth. - * cfg80211 queues these notifications; GFP_ATOMIC keeps this IRQ-safe. + * The cfg80211 calls may take bottom-half locks, so they are made after + * response_lock is dropped. command_mutex keeps the disconnect worker, + * which retires the connection before reporting its end, behind them; + * an RF failure in the meantime leaves its report to this worker. */ spin_lock_irqsave(&m->response_lock, flags); if (!ret && (m->connect_error || !mt_rf_allowed(m))) { @@ -541,33 +551,49 @@ void mt_connect_work(struct work_struct *work) } m->connected = !ret; m->disconnecting = false; - if (m->connect_cancelled) { - if (response.links[0].bss) - cfg80211_put_bss(m->wiphy, response.links[0].bss); - cfg80211_disconnected(m->netdev, m->disconnect_reason, NULL, 0, true, GFP_ATOMIC); - } else { - cfg80211_connect_done(m->netdev, &response, GFP_ATOMIC); - } + cancelled = m->connect_cancelled; + reason = m->disconnect_reason; + m->connect_publishing = true; if (!ret) { schedule_work(&m->power_work); if (READ_ONCE(m->data_ready)) { netif_carrier_on(m->netdev); netif_wake_queue(m->netdev); } - if (m->connect_open) { - dev_dbg(&m->pdev->dev, - "OPEN_CONNECTED: owned peer, firmware completion; Ethernet=%u\n", - m->data_ready); - } else { - cfg80211_port_authorized(m->netdev, m->connect_bssid, NULL, 0, GFP_ATOMIC); - dev_dbg(&m->pdev->dev, - "WPA2_AUTHORIZED: firmware handshake complete; Ethernet=%u\n", - m->data_ready); - } + } + spin_unlock_irqrestore(&m->response_lock, flags); + + if (cancelled) { + if (response.links[0].bss) + cfg80211_put_bss(m->wiphy, response.links[0].bss); + cfg80211_disconnected(m->netdev, reason, NULL, 0, true, GFP_KERNEL); } else { - dev_err(&m->pdev->dev, "CONNECT_FAILED: %d; recovery-required=%u\n", ret, m->link_failed); + cfg80211_connect_done(m->netdev, &response, GFP_KERNEL); + } + if (!ret && !m->connect_open) + cfg80211_port_authorized(m->netdev, m->connect_bssid, NULL, 0, GFP_KERNEL); + + spin_lock_irqsave(&m->response_lock, flags); + m->connect_publishing = false; + if (m->connect_lost) { + m->connect_lost = false; + cfg80211_disconnected(m->netdev, WLAN_REASON_UNSPECIFIED, + NULL, 0, true, GFP_ATOMIC); } spin_unlock_irqrestore(&m->response_lock, flags); + if (!ret && m->connect_open) + dev_dbg(&m->pdev->dev, + "OPEN_CONNECTED: owned peer, firmware completion; Ethernet=%u\n", + READ_ONCE(m->data_ready)); + else if (!ret) + dev_dbg(&m->pdev->dev, + "WPA2_AUTHORIZED: firmware handshake complete; Ethernet=%u\n", + READ_ONCE(m->data_ready)); + else if (cancelled) + dev_dbg(&m->pdev->dev, "CONNECT_CANCELLED: %d\n", ret); + else + dev_err(&m->pdev->dev, "CONNECT_FAILED: %d; recovery-required=%u\n", + ret, READ_ONCE(m->link_failed)); mutex_unlock(&m->command_mutex); } diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 37c34872fe66f..0460b23e81645 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -147,6 +147,10 @@ struct mt7932 { u8 connect_ssid_length, connect_channel, peer_wtbl; bool connecting, connected, peer_valid, link_failed, connect_open; bool disconnecting, disconnect_seen, disconnect_event_received, disconnect_local; + /* The connect worker is reporting its outcome; an RF failure meanwhile + * sets connect_lost for the worker to report. Under response_lock. + */ + bool connect_publishing, connect_lost; u16 disconnect_reason; int connect_error; u32 event_cipher; diff --git a/drivers/net/wireless/mediatek/mt7932/rf-test.c b/drivers/net/wireless/mediatek/mt7932/rf-test.c index c95f4057a2305..8d12c9a9ee697 100644 --- a/drivers/net/wireless/mediatek/mt7932/rf-test.c +++ b/drivers/net/wireless/mediatek/mt7932/rf-test.c @@ -388,6 +388,30 @@ static void mt_scan_quiesce_retirement_test(struct kunit *test) m->wiphy = NULL; } +/* + * An RF failure while the connect worker is publishing a new connection + * must not report the disconnect ahead of the connect: it leaves the report + * to the worker. (cfg80211 is not called here: the synthetic netdev has no + * wireless device, so a direct report would crash the test.) + */ +static void mt_rf_failure_during_publication_test(struct kunit *test) +{ + struct mt7932 *m = test->priv; + + m->connecting = false; + m->connected = true; + m->connect_publishing = true; + mt_rf_test_fail(m, -EIO); + KUNIT_EXPECT_TRUE(test, m->connect_lost); + KUNIT_EXPECT_FALSE(test, m->connected); + KUNIT_EXPECT_TRUE(test, m->disconnecting); + KUNIT_EXPECT_FALSE(test, netif_carrier_ok(m->netdev)); + /* Ownership is uncertain after an RF failure: keep the peer. */ + KUNIT_EXPECT_TRUE(test, m->peer_valid); + m->connect_lost = false; + m->connect_publishing = false; +} + static struct kunit_case mt_rf_test_cases[] = { KUNIT_CASE(mt_startup_band1_queue_drain_test), KUNIT_CASE(mt_missing_policy_admission_retry_test), @@ -396,6 +420,7 @@ static struct kunit_case mt_rf_test_cases[] = { KUNIT_CASE(mt_rf_failure_blocks_startup_test), KUNIT_CASE(mt_rf_failure_preserves_first_error_test), KUNIT_CASE(mt_scan_quiesce_retirement_test), + KUNIT_CASE(mt_rf_failure_during_publication_test), {} }; From 7d1d76dd6dc17e23676e2c76a3fc068097359d83 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 22:30:12 +1100 Subject: [PATCH 1426/2680] soc: apple: sep: Re-read a sensor frame that is still on offer A read of a sensor frame can fail on the bus in two ways. The bytes come back garbled, or the sensor does not take the frame. Either way it keeps offering the frame at the same length. Boot 0874c215 showed both: the nonce read failed its CRC, and afterwards the sensor still offered the complete 43-byte nonce frame with a valid CRC. On boot 7f0c01be the serial read succeeded but the serial frame was not taken. The nonce command that followed then waited behind it until it timed out. Check the CRC of the serial, nonce, session and challenge replies. The session and challenge replies were relayed unchecked, but they carry the same CRC; it held on every boot measured. The calibration module reply keeps its own handling. When a reply fails its CRC and the sensor still offers a frame of that length, read it again, at most twice. Before a command that expects a reply, take any stale frame the sensor still offers, so the reply is not queued behind it. Two such frames at most: if one is still on offer after that, fail the step rather than send the command, because a read does not prove the sensor took the frame. Report both, since either means an exchange went wrong on the bus. (cherry picked from commit c61197b7d5e1c9e845f5c207f4ed9259ff975950) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 15 +++++ drivers/soc/apple/sensor.rs | 130 +++++++++++++++++++++++++++--------- 2 files changed, 112 insertions(+), 33 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index e15af135453ac..3a6c054f49a6e 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1711,11 +1711,25 @@ impl SepData { } } + /// Reports frames the sensor exchange had to read again or discard. + fn note_sensor_recoveries(&self) { + let (reread, drained) = sensor::take_recoveries(); + if reread != 0 || drained != 0 { + dev_warn!( + self.dev, + "sensor: re-read {} frame(s) the sensor still offered after a garbled read, discarded {} stale frame(s)\n", + reread, + drained + ); + } + } + /// Reports a failed registration step with what the sensor shows /// afterwards: its state, patch acknowledgment, whether it presents an /// identifier and the length of any frame it holds. Never the /// identifier, the serial or any key material. fn sensor_step_failed(&self, step: &CStr) { + self.note_sensor_recoveries(); let ms = self.sensor_ms(); match sensor::status() { Ok(st) => dev_warn!( @@ -1847,6 +1861,7 @@ impl SepData { return false; } self.bringup.store(BRINGUP_ESTABLISHED, Relaxed); + self.note_sensor_recoveries(); let _ = self.log_identity_count(c"after sensor serial registration"); true } diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index dc3df5c6b1d34..c0f3a6159f74c 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -5,6 +5,7 @@ //! The fingerprint sensor over SPI. use kernel::prelude::*; +use kernel::sync::atomic::{Atomic, Relaxed}; use kernel::time::delay::fsleep; use kernel::time::Delta; @@ -118,6 +119,7 @@ const CMD_START_CAPTURE: [u8; CMD_LEN] = [0x80, 0x40, 0x00, 0x07, 0x00, 0x00, 0x // Precedes the patch blob. const CMD_SETUP_PATCH_ENABLE: [u8; CMD_LEN] = [0x80, 0x60, 0x00, 0x07, 0x00, 0x00, 0x00]; const CMD_GET_SENSOR_SERIAL: [u8; CMD_LEN] = [0x80, 0x70, 0x00, 0x07, 0x00, 0x00, 0x00]; +const CMD_GET_NONCE: [u8; CMD_LEN] = [0x80, 0x35, 0x00, 0x07, 0x00, 0x00, 0x00]; const READ_CMD_LEN: usize = 11; const CMD_READ_PREFIX: [u8; CMD_LEN] = [0x80, 0x13, 0x00, 0x0b, 0x00, 0x00, 0x00]; @@ -475,26 +477,18 @@ fn poll_until_ready_any(accepted: &[usize]) -> Result { } pub(crate) fn send_session_share(share: &[u8; SESSION_SHARE_LEN]) -> Result<()> { + drain_stale_frames()?; send_framed(TYPE_SESSION, share) } pub(crate) fn send_challenge(challenge: &[u8; CHALLENGE_LEN]) -> Result<()> { + drain_stale_frames()?; send_framed(TYPE_SEQUENCE, challenge) } static_assert!(FRAME_HEADER + SESSION_SHARE_LEN == 0x2f); static_assert!(FRAME_HEADER + CHALLENGE_LEN == 0x47); -fn read_framed(total: usize, at: usize, out: &mut [u8]) -> Result<()> { - let mut buf = [0xffu8; MAX_REPLY]; - read_reply(total, &mut buf)?; - if at + out.len() > total { - return Err(EINVAL); - } - out.copy_from_slice(&buf[at..at + out.len()]); - Ok(()) -} - fn poll_until_ready(expected: usize) -> Result<()> { for _ in 0..READ_POLL_ATTEMPTS { let st = status()?; @@ -511,11 +505,6 @@ fn poll_until_ready(expected: usize) -> Result<()> { const READ_POLL_MS: i64 = 5; const READ_POLL_ATTEMPTS: usize = 200; -fn read_reply(total: usize, buf: &mut [u8; MAX_REPLY]) -> Result<()> { - poll_until_ready(total)?; - read_reply_ready(total, buf) -} - fn read_reply_ready(total: usize, buf: &mut [u8; MAX_REPLY]) -> Result<()> { if total > MAX_REPLY { return Err(EINVAL); @@ -538,22 +527,99 @@ fn read_reply_ready(total: usize, buf: &mut [u8; MAX_REPLY]) -> Result<()> { }) } -fn read_framed_verified(total: usize, at: usize, out: &mut [u8]) -> Result<()> { - if total < CRC_LEN || at + out.len() > total - CRC_LEN { - return Err(EINVAL); +/// Frames the sensor still offered after the read that should have taken +/// them, and stale frames found on offer before a command. Both mean an +/// exchange went wrong on the bus; the caller reports them. +static REREADS: Atomic = Atomic::new(0); +static DRAINED: Atomic = Atomic::new(0); + +/// Takes the counts of re-read and drained frames since the last call. +pub(crate) fn take_recoveries() -> (u32, u32) { + (REREADS.xchg(0, Relaxed), DRAINED.xchg(0, Relaxed)) +} + +// A garbled read the sensor did not take is read once more, twice at most. +const FRAME_READS: usize = 3; +// Stale frames taken before a command; the sensor queues at most a couple. +const DRAIN_FRAMES: usize = 2; + +/// A reply buffer, wiped however the read that fills it ends. +struct FrameBuf([u8; MAX_REPLY]); + +impl FrameBuf { + fn new() -> Self { + FrameBuf([0xff; MAX_REPLY]) } - let mut buf = [0xffu8; MAX_REPLY]; - read_reply(total, &mut buf)?; +} - let split = total - CRC_LEN; - let expected = u16::from_le_bytes([buf[split], buf[split + 1]]); - let computed = crc16_ansi(&buf[..split]); - if expected != computed { - return Err(EIO); +impl Drop for FrameBuf { + fn drop(&mut self) { + for b in self.0.iter_mut() { + // SAFETY: a valid, uniquely borrowed byte. + unsafe { core::ptr::write_volatile(b, 0) }; + } + } +} + +fn frame_crc_valid(frame: &[u8]) -> bool { + let split = frame.len() - CRC_LEN; + u16::from_le_bytes([frame[split], frame[split + 1]]) == crc16_ansi(&frame[..split]) +} + +/// Takes any frame the sensor still offers before a new command, so the +/// reply to that command is not mistaken for, or queued behind, a stale one. +/// Fails if the sensor still offers a frame once the budget is spent: a +/// read is no proof the sensor took the frame. +fn drain_stale_frames() -> Result<()> { + for _ in 0..DRAIN_FRAMES { + let Some(count) = status()?.available().filter(|n| *n != 0) else { + return Ok(()); + }; + let count = count as usize; + if count > MAX_REPLY { + return Err(EIO); + } + let mut buf = FrameBuf::new(); + read_reply_ready(count, &mut buf.0)?; + DRAINED.fetch_add(1, Relaxed); + } + match status()?.available() { + None | Some(0) => Ok(()), + Some(_) => Err(EBUSY), + } +} + +/// Sends `cmd` and reads its `total`-byte reply, checking the reply's CRC. +fn command_frame(cmd: &[u8; CMD_LEN], total: usize, at: usize, out: &mut [u8]) -> Result<()> { + drain_stale_frames()?; + command(cmd)?; + read_frame(total, at, out) +} + +/// Reads a `total`-byte reply once the sensor offers it and checks its CRC. +/// A read the sensor did not take leaves the frame on offer at the same +/// length: only then is it read again. +fn read_frame(total: usize, at: usize, out: &mut [u8]) -> Result<()> { + if total < CRC_LEN || total > MAX_REPLY || at + out.len() > total - CRC_LEN { + return Err(EINVAL); } + poll_until_ready(total)?; - out.copy_from_slice(&buf[at..at + out.len()]); - Ok(()) + let mut buf = FrameBuf::new(); + for attempt in 0..FRAME_READS { + if attempt > 0 { + if status()?.available() != Some(total as u32) { + break; + } + REREADS.fetch_add(1, Relaxed); + } + read_reply_ready(total, &mut buf.0)?; + if frame_crc_valid(&buf.0[..total]) { + out.copy_from_slice(&buf.0[at..at + out.len()]); + return Ok(()); + } + } + Err(EIO) } const MAX_REPLY: usize = 0x4f; @@ -576,17 +642,15 @@ impl SensorSerial { } pub(crate) fn read_sensor_serial() -> Result { - command(&CMD_GET_SENSOR_SERIAL)?; let mut serial = [0u8; SENSOR_SERIAL_LEN]; - read_framed_verified(SENSOR_SERIAL_REPLY_LEN, SENSOR_SERIAL_AT, &mut serial)?; + command_frame(&CMD_GET_SENSOR_SERIAL, SENSOR_SERIAL_REPLY_LEN, SENSOR_SERIAL_AT, &mut serial)?; Ok(SensorSerial(serial)) } /// Read the nonce as one frame; a separate acknowledgment would consume it. pub(crate) fn session_nonce() -> Result<[u8; 32]> { - command(&[0x80, 0x35, 0, 7, 0, 0, 0])?; let mut value = [0; 32]; - read_framed_verified(9 + value.len() + CRC_LEN, 9, &mut value)?; + command_frame(&CMD_GET_NONCE, 9 + value.len() + CRC_LEN, 9, &mut value)?; if value.iter().all(|byte| *byte == 0) { return Err(EIO); } @@ -595,13 +659,13 @@ pub(crate) fn session_nonce() -> Result<[u8; 32]> { pub(crate) fn read_session_reply() -> Result<[u8; SESSION_SHARE_LEN]> { let mut share = [0u8; SESSION_SHARE_LEN]; - read_framed(SESSION_REPLY_LEN, SESSION_REPLY_AT, &mut share)?; + read_frame(SESSION_REPLY_LEN, SESSION_REPLY_AT, &mut share)?; Ok(share) } pub(crate) fn read_challenge_reply() -> Result<[u8; CHALLENGE_LEN]> { let mut reply = [0u8; CHALLENGE_LEN]; - read_framed(CHALLENGE_REPLY_LEN, CHALLENGE_REPLY_AT, &mut reply)?; + read_frame(CHALLENGE_REPLY_LEN, CHALLENGE_REPLY_AT, &mut reply)?; Ok(reply) } From 0336531db8f85fe30be88a8c0ab33934d64fdc32 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 01:27:46 +1100 Subject: [PATCH 1427/2680] wifi: mt7932: keep the link when a beacon hint only lifts NO_IR In the world regulatory domain, cfg80211 lifts NO_IR on a channel once it has heard a beacon there, and calls the notifier because the driver asks for channel-change callbacks. The changed channel flags make a new snapshot, which starts a full regulatory pass, and the pass ends any connection before it programs the firmware. So the first beacon heard on each new channel during a background scan drops a working connection, only to permit more than the firmware currently allows. While associated, keep the stricter domain the firmware already has when the new snapshot differs from the applied one only by NO_IR lifted on some channels: same country, firmware domain, channel list and limits. Record it as deferred and run the pass when the connection has ended, before the disconnect worker lets a new connect start. Any other change, including a restriction, is applied at once as before and replaces a deferred relaxation. Until the deferred pass runs, a scan may probe on such a channel, which cfg80211 now permits, while the firmware domain still marks it passive. Add KUnit tests for the relaxation check. (cherry picked from commit 4e1a415fc82544593b57dffd7cc26e621b505ecd) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/Kconfig | 3 +- drivers/net/wireless/mediatek/mt7932/Makefile | 1 + drivers/net/wireless/mediatek/mt7932/link.c | 1 + drivers/net/wireless/mediatek/mt7932/mt7932.h | 6 + .../mediatek/mt7932/regulatory-test.c | 103 ++++++++++++++++++ .../net/wireless/mediatek/mt7932/regulatory.h | 32 ++++++ .../net/wireless/mediatek/mt7932/startup.c | 32 ++++++ 7 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 drivers/net/wireless/mediatek/mt7932/regulatory-test.c diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index f09fb1f32012b..49a6fbad0e67e 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -40,4 +40,5 @@ config MT7932_KUNIT_TEST Exercise the validation of what the firmware and the device hand to the driver, with synthetic, truncated and corrupted input and without radio hardware: receive descriptors, including frames the - hardware did not decrypt, and command replies. + hardware did not decrypt, command replies, regulatory snapshots + and country power packages. diff --git a/drivers/net/wireless/mediatek/mt7932/Makefile b/drivers/net/wireless/mediatek/mt7932/Makefile index 97b1b760372c1..781d4530fbee1 100644 --- a/drivers/net/wireless/mediatek/mt7932/Makefile +++ b/drivers/net/wireless/mediatek/mt7932/Makefile @@ -3,3 +3,4 @@ obj-$(CONFIG_MT7932_FULLMAC) += mt7932-fullmac.o mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += rf-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += input-test.o +mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += regulatory-test.o diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 4e9ff20623da0..ccfd4d4124fce 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -710,6 +710,7 @@ void mt_disconnect_work(struct work_struct *work) spin_lock_irqsave(&m->response_lock, flags); m->disconnecting = false; cfg80211_disconnected(m->netdev, reason, NULL, 0, local, GFP_ATOMIC); + mt_reg_apply_deferred_locked(m); spin_unlock_irqrestore(&m->response_lock, flags); dev_dbg(&m->pdev->dev, "DISCONNECTED: teardown=%d local=%u\n", ret, local); } diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 0460b23e81645..aae173b97cc13 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -114,6 +114,11 @@ struct mt7932 { bool interface_registered, interface_up; u32 reg_generation, reg_attempted; struct mt7932_reg_snapshot reg_desired; + /* A relaxation kept from the firmware while associated; applied by the + * next pass once the connection has ended. Under response_lock. + */ + struct mt7932_reg_snapshot reg_deferred; + bool reg_deferral; /* Channels the applied country package leaves out; under RTNL. */ DECLARE_BITMAP(policy_disabled, MT7932_CHANNELS); struct net_device *netdev; @@ -187,6 +192,7 @@ struct mt7932 { bool mt_rf_allowed(struct mt7932 *m); void mt_retry_missing_policy(struct mt7932 *m); +void mt_reg_apply_deferred_locked(struct mt7932 *m); int mt_net_open(struct net_device *netdev); int mt_request_input(struct mt7932 *m, const struct firmware **fw, const char *name); /* response_lock held; closes RF admission and reports terminal host state. */ diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory-test.c b/drivers/net/wireless/mediatek/mt7932/regulatory-test.c new file mode 100644 index 0000000000000..1292c450ff54b --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/regulatory-test.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for the regulatory snapshot logic and, as local files the + * driver must not trust, the country power packages. + */ +#include + +#include "mt7932.h" + +/* A world-domain snapshot: channels 1, 6, 36 and 149, the 5 GHz ones NO_IR. */ +static void reg_build(struct mt7932_reg_snapshot *reg) +{ + static const u16 channels[] = {1, 6, 36, 149}; + unsigned int i; + + memset(reg, 0, sizeof(*reg)); + memcpy(reg->alpha2, "00", 2); + memcpy(reg->domain, "XZ", 2); + reg->domain[4] = 1; + reg->domain[5] = 2; + reg->domain[8] = 2; + reg->domain[9] = 2; + for (i = 0; i < ARRAY_SIZE(channels); i++) { + put_unaligned_le16(channels[i], reg->domain + 12 + i * 8); + put_unaligned_le32(channels[i] > 14 ? IEEE80211_CHAN_NO_IR : 0, + reg->domain + 16 + i * 8); + reg->power[i] = 20; + } + reg->length = 12 + ARRAY_SIZE(channels) * 8; +} + +static void reg_set_flags(struct mt7932_reg_snapshot *reg, unsigned int entry, u32 flags) +{ + put_unaligned_le32(flags, reg->domain + 16 + entry * 8); +} + +/* A beacon hint lifting NO_IR only relaxes; nothing else does. */ +static void mt7932_reg_relaxation_test(struct kunit *test) +{ + struct mt7932_reg_snapshot *prev, *next; + + prev = kunit_kzalloc(test, sizeof(*prev), GFP_KERNEL); + next = kunit_kzalloc(test, sizeof(*next), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, prev); + KUNIT_ASSERT_NOT_NULL(test, next); + reg_build(prev); + + reg_build(next); + reg_set_flags(next, 2, 0); + KUNIT_EXPECT_TRUE(test, mt7932_reg_only_relaxes(prev, next)); + /* An identical snapshot is not treated as a change by the caller, + * but it is trivially within the applied domain. + */ + KUNIT_EXPECT_TRUE(test, mt7932_reg_only_relaxes(prev, prev)); + + /* NO_IR added: a restriction. */ + reg_build(next); + reg_set_flags(next, 0, IEEE80211_CHAN_NO_IR); + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + /* Radar or a bandwidth limit appears, or another flag goes away. */ + reg_build(next); + reg_set_flags(next, 2, IEEE80211_CHAN_NO_IR | IEEE80211_CHAN_RADAR); + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + reg_set_flags(prev, 3, IEEE80211_CHAN_NO_IR | IEEE80211_CHAN_NO_HT40); + reg_build(next); + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + reg_build(prev); + /* A new country, power limit, channel or channel order. */ + reg_build(next); + memcpy(next->alpha2, "AU", 2); + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + reg_build(next); + next->power[1] = 19; + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + reg_build(next); + put_unaligned_le16(40, next->domain + 12 + 3 * 8); + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + reg_build(next); + put_unaligned_le16(44, next->domain + 12 + 4 * 8); + next->domain[9] = 3; + next->length += 8; + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + /* A failed snapshot never relaxes, and a corrupt length is refused. */ + reg_build(next); + next->error = -EOPNOTSUPP; + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + reg_build(next); + prev->length = sizeof(prev->domain) + 8; + next->length = prev->length; + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); +} + +static struct kunit_case mt7932_policy_test_cases[] = { + KUNIT_CASE(mt7932_reg_relaxation_test), + {} +}; + +static struct kunit_suite mt7932_policy_test_suite = { + .name = "mt7932-regulatory", + .test_cases = mt7932_policy_test_cases, +}; + +kunit_test_suite(mt7932_policy_test_suite); diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory.h b/drivers/net/wireless/mediatek/mt7932/regulatory.h index fd65e07d9506a..3a8c1ffc74015 100644 --- a/drivers/net/wireless/mediatek/mt7932/regulatory.h +++ b/drivers/net/wireless/mediatek/mt7932/regulatory.h @@ -5,6 +5,8 @@ /* Private, versioned packaging of original-derived country policy. This is * not firmware code and does not replace cfg80211's regulatory database. */ +#include + #include "channels.h" #define MT7932_POLICY_TABLES 13 @@ -102,6 +104,36 @@ static inline int mt7932_policy_parse(struct mt7932_policy *out, return at == size ? 0 : -EINVAL; } +/* + * Whether @next differs from the applied @prev only by NO_IR lifted on some + * channels, as a beacon hint does in the world domain: same country, firmware + * domain header, channel list and limits, and every channel's flags equal to + * the applied ones less, at most, NO_IR. Such a change only permits more; the + * firmware may keep the stricter domain until it is idle. + */ +static inline bool mt7932_reg_only_relaxes(const struct mt7932_reg_snapshot *prev, + const struct mt7932_reg_snapshot *next) +{ + unsigned int i, count; + + if (prev->error || next->error || prev->length != next->length || + prev->length < 12 || prev->length > sizeof(prev->domain) || + memcmp(prev->alpha2, next->alpha2, 2) || + memcmp(prev->domain, next->domain, 12) || + memcmp(prev->power, next->power, sizeof(prev->power))) + return false; + count = (prev->length - 12) / 8; + for (i = 0; i < count; i++) { + const u8 *a = prev->domain + 12 + i * 8, *b = next->domain + 12 + i * 8; + u32 was = get_unaligned_le32(a + 4), now = get_unaligned_le32(b + 4); + + if (memcmp(a, b, 4) || (now & ~was) || + ((was & ~now) & ~IEEE80211_CHAN_NO_IR)) + return false; + } + return true; +} + /* Every entry of a validated package starts with its channel number and is * followed by per-rate power limits. A country that does not permit a * channel sets all of that channel's limits to this marker. diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 051ab3793d965..679c9193f85bc 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -279,6 +279,20 @@ static void mt_policy_disable(struct mt7932 *m, const struct mt7932_policy *poli * Consume the flag under the same lock as the worker publication, so repeated * userspace requests cannot continually invalidate an in-flight attempt. */ +/* response_lock held: start the pass a deferred relaxation is waiting for. */ +void mt_reg_apply_deferred_locked(struct mt7932 *m) +{ + lockdep_assert_held(&m->response_lock); + if (!m->reg_deferral || m->stopping) + return; + m->reg_deferral = false; + m->reg_desired = m->reg_deferred; + m->reg_generation++; + WRITE_ONCE(m->reg_pending, true); + if (READ_ONCE(m->interface_registered)) + schedule_work(&m->startup_work); +} + void mt_retry_missing_policy(struct mt7932 *m) { unsigned long flags; @@ -462,7 +476,24 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request spin_unlock_irqrestore(&m->response_lock, irqflags); return; } + if (!m->reg_pending && m->connected && !m->disconnecting && + mt7932_reg_only_relaxes(&m->reg_desired, ®) && + memcmp(®, &m->reg_desired, sizeof(reg))) { + /* A pass would end the connection only to permit more than the + * firmware now allows. Keep the stricter applied domain and run + * the pass once the connection has ended. + */ + m->reg_deferred = reg; + m->reg_deferral = true; + spin_unlock_irqrestore(&m->response_lock, irqflags); + for_each_set_bit(i, m->policy_disabled, MT7932_CHANNELS) + mt_channel(m, i)->flags |= IEEE80211_CHAN_DISABLED; + dev_dbg(&m->pdev->dev, "REGULATORY_RELAXATION_DEFERRED: %c%c until disconnected\n", + reg.alpha2[0], reg.alpha2[1]); + return; + } if (!memcmp(®, &m->reg_desired, sizeof(reg)) && !m->reg_pending) { + m->reg_deferral = false; spin_unlock_irqrestore(&m->response_lock, irqflags); /* The applied package still holds, but cfg80211 has just * rebuilt the channel flags without its disables. @@ -472,6 +503,7 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request return; } m->reg_desired = reg; + m->reg_deferral = false; m->reg_generation++; /* The next package pass records its own set. */ bitmap_zero(m->policy_disabled, MT7932_CHANNELS); From 10f592aa1d29a71b0c0952a5b70985fcd61a1312 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 11:53:45 +1100 Subject: [PATCH 1428/2680] usb: typec: tipd: Stop waiting for a USB4 router that never binds On the M1 and M2 machines every CD321x Type-C connector is linked through its port@2 endpoint to the USB4 router serving that port, and cd321x_register_port() looks up the router's Thunderbolt switch with fwnode_typec_thunderbolt_switch_get(). The lookup returns -EPROBE_DEFER until the router driver (USB4_APPLE_SOC) has registered the switch. So when that driver is not built, or the router fails to probe, every linked PD controller stays deferred for good: no Type-C port, no partners, no USB role switching and no DisplayPort alternate mode, although the router only takes part in USB4 and Thunderbolt links. Kernels configured for the M1 Pro and M2 from configurations that predate the router driver show exactly that: only the connector-less MagSafe controller binds, and the others stay in devices_deferred with no recorded reason. Handle the switch like other providers that may never get a driver: keep deferring while the driver core still waits for missing dependencies, and once driver_deferred_probe_check_state() gives up, register the port without a switch, as for a connector with no router link. Ports whose router registers its switch in time behave as before. Fixes: fc76d0f20c29 ("usb: typec: tipd: Hook up tbt switch for Apple platforms") (cherry picked from commit 7fd578b198e007701218f87f34a562545e54be29) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/core.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index a311a5826952b..7c4fa7adcbd5c 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -1507,6 +1507,27 @@ static void cd321x_stop_work(struct cd321x *cd321x) cancel_delayed_work_sync(&cd321x->update_work); } +/* + * The USB4/Thunderbolt router is only needed once a USB4 or Thunderbolt link + * comes up. If its driver is not built or never binds, do not leave power + * delivery, USB and DisplayPort on this port without a driver as well: stop + * waiting for the router when the driver core stops waiting for missing + * dependencies. + */ +static struct typec_thunderbolt_switch * +cd321x_tbt_switch_get(struct device *dev, struct fwnode_handle *fwnode) +{ + struct typec_thunderbolt_switch *sw; + + sw = fwnode_typec_thunderbolt_switch_get(fwnode); + if (sw != ERR_PTR(-EPROBE_DEFER) || + driver_deferred_probe_check_state(dev) == -EPROBE_DEFER) + return sw; + + dev_info(dev, "USB4/Thunderbolt router unavailable, continuing without it\n"); + return NULL; +} + static int cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) { @@ -1536,7 +1557,7 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) if (IS_ERR(cd321x->connector_fwnode)) cd321x->connector_fwnode = NULL; - cd321x->tbt_switch = fwnode_typec_thunderbolt_switch_get(fwnode); + cd321x->tbt_switch = cd321x_tbt_switch_get(tps->dev, fwnode); if (IS_ERR(cd321x->tbt_switch)) { ret = PTR_ERR(cd321x->tbt_switch); goto err_put_mux; From 78111b9ab293a5468bd3ac1564de17db031cf688 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 22:30:52 +1100 Subject: [PATCH 1429/2680] soc: apple: sep: Retry a failed attach-time sensor bring-up When the bring-up at attach failed, nothing retried it until the first verify, minutes later. That verify was then refused before it touched the sensor, and gdm offered no fingerprint at login. A bring-up that fails on the bus leaves nothing a new attempt cannot undo. Each attempt power-cycles the sensor and, after a failed one, clears the enclave partial registration first. That is the same path every verify takes, and it has never failed on an idle machine. Try up to three times at attach, logging each failed attempt. A power cycle that fails ends the attempt before registration: a sensor that was not reset may still hold the session of the attempt before. (cherry picked from commit 6c394131cf7e66c132be30a2f0ebd0562ca478dd) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 3a6c054f49a6e..2ee4674a6457b 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1080,12 +1080,28 @@ impl SepData { fn attach_bringup(&self) { if self.profile.nonce_session { - if !self.bring_sensor_online() { - dev_warn!( - self.dev, - "sensor: authenticated attach incomplete; capture unavailable\n" - ); + // A bring-up that fails on the bus leaves nothing a new attempt + // cannot undo: each attempt power-cycles the sensor and, after a + // failed one, clears the enclave's partial registration first, + // as every verify does. + const ATTEMPTS: u32 = 3; + for attempt in 1..=ATTEMPTS { + if self.bring_sensor_online() { + return; + } + if attempt < ATTEMPTS { + dev_warn!( + self.dev, + "sensor: attach-time bring-up attempt {} of {} failed; starting again\n", + attempt, + ATTEMPTS + ); + } } + dev_warn!( + self.dev, + "sensor: authenticated attach incomplete; capture unavailable\n" + ); return; } if !sensor::is_bound() { @@ -1940,6 +1956,12 @@ impl SepData { let _source = sensor::power_source(); self.sensor_power_ns.store(shim::boottime_ns(), Relaxed); let cycled = sensor::power_cycle(); + if !cycled && self.profile.nonce_session { + // A sensor that was not reset may still hold the session of an + // earlier attempt; registering it again cannot be trusted. + dev_err!(self.dev, "sensor: the power cycle failed; not registering the sensor\n"); + return None; + } if !sensor::cs_timing().is_hardware() { dev_warn!( self.dev, From a94299d9d3c47be5478b41c962a2334fabcaece1 Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Tue, 13 Oct 2026 14:20:26 +0200 Subject: [PATCH 1430/2680] arm64: dts: apple: t8140-j700: give the boot framebuffer its panel simpledrm reads the panel's physical size through the framebuffer's panel phandle. Without it the boot display reports 96 dpi, and compositors pick a scale of 1 for the 219 ppi panel. Link it as the other Apple laptops do. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 27da3a5d53ca11f2aeff4916d25f40308c378a07) --- arch/arm64/boot/dts/apple/t8140-j700.dts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 942e360f57c4e..affdd0149be64 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -52,6 +52,8 @@ power-domains = <&ps_disp_cpu>; /* Format properties will be added by loader */ status = "disabled"; + panel = <&panel>; + post-init-providers = <&panel>; }; }; From 9ba865c8a1e2aff41e5671fce3864e0c2e266425 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:50:13 +1100 Subject: [PATCH 1431/2680] wifi: mt7932: test the power package parser The power packages are files from the local system, so the parser and the channel filter treat them as untrusted input. Add KUnit tests with synthetic country and world packages: valid packages, a package that names another country, every truncation, trailing data, header, mode, table length, entry count and table type bounds, the world package's 2.4 GHz channel order, the not-permitted marker, the channel filter and 2,000 random corruptions. No driver behaviour changes. (cherry picked from commit 5cbf717b9d47f3717b4d30403e325ff59d47eecb) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../mediatek/mt7932/regulatory-test.c | 330 +++++++++++++++++- 1 file changed, 328 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory-test.c b/drivers/net/wireless/mediatek/mt7932/regulatory-test.c index 1292c450ff54b..225f3b8f35a0c 100644 --- a/drivers/net/wireless/mediatek/mt7932/regulatory-test.c +++ b/drivers/net/wireless/mediatek/mt7932/regulatory-test.c @@ -1,9 +1,11 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * KUnit tests for the regulatory snapshot logic and, as local files the - * driver must not trust, the country power packages. + * KUnit tests for the regulatory snapshot logic and the power package + * parser and channel filter. The packages are local files, so + * the parser treats them as untrusted. */ #include +#include #include "mt7932.h" @@ -90,8 +92,332 @@ static void mt7932_reg_relaxation_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); } +#define PKG_MODE_SIZE 328 +#define PKG_ENTRY_SIZE 122 + +/* Channels of the synthetic 2.4 and 5 GHz tables 1..9, in table order. */ +static const u8 pkg_channels[] = { + 1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, + 36, 40, 44, 48, 52, 56, 60, 64, + 100, 104, 108, 112, 116, 120, 124, 128, + 132, 136, 140, 144, 149, 153, 157, 161, + 38, 46, 54, 62, 102, 110, 118, 126, + 134, 142, 151, 159, 42, 58, 106, 122, + 138, 155, 50, 114, 130, 146, 163, 167, + 165, 169, 171, 173, 175, +}; + +static unsigned int pkg_count(unsigned int table, bool world) +{ + if (table == 2) + return world ? 6 : 5; + return table == 9 ? 5 : 8; +} + +static size_t pkg_size(bool world) +{ + static const u16 others[] = {532, 144, 58, 265}; + size_t size = 16 + (world ? 2 : 3) * PKG_MODE_SIZE; + unsigned int i; + + for (i = 1; i <= 9; i++) + size += 44 + pkg_count(i, world) * PKG_ENTRY_SIZE; + for (i = 0; i < ARRAY_SIZE(others); i++) + size += others[i]; + return size; +} + +/* Build a structurally valid package. Channel 14 is only in the world one. */ +static u8 *pkg_build(struct kunit *test, const char *alpha2, size_t *size) +{ + static const u16 lengths[] = {532, 144, 58, 265}; + static const u8 types[] = {5, 1, 4, 2}; + bool world = !memcmp(alpha2, "XZ", 2); + unsigned int i, j, modes = world ? 2 : 3, channel = 0; + u8 *data, *p; + + *size = pkg_size(world); + data = kunit_kzalloc(test, *size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, data); + memcpy(data, "J7RP", 4); + put_unaligned_le16(1, data + 4); + memcpy(data + 6, alpha2, 2); + data[8] = modes; + data[9] = MT7932_POLICY_TABLES; + put_unaligned_le32(*size, data + 12); + p = data + 16; + for (i = 0; i < modes; i++, p += PKG_MODE_SIZE) + put_unaligned_le32(17 + i, p + 4); + for (i = 0; i < MT7932_POLICY_TABLES; i++) { + if (i >= 1 && i <= 9) { + unsigned int count = pkg_count(i, world); + + p[0] = 3; + p[1] = 0x0a; + put_unaligned_le16(44 + count * PKG_ENTRY_SIZE, p + 2); + p[4] = count; + p[5] = i <= 2 ? 1 : 2; + p[6] = i == 2 || i == 9; + memcpy(p + 8, alpha2, 2); + for (j = 0; j < count; j++) { + u8 *entry = p + 44 + j * PKG_ENTRY_SIZE; + + /* A country package has no channel 14 entry. */ + if (!world && pkg_channels[channel] == 14) + channel++; + entry[0] = pkg_channels[channel++]; + memset(entry + 1, 30, PKG_ENTRY_SIZE - 1); + } + p += 44 + count * PKG_ENTRY_SIZE; + } else { + unsigned int k = i ? i - 9 : 0; + + p[1] = 2; + put_unaligned_le16(lengths[k], p + 2); + p[7] = types[k]; + if (i == 10) + memcpy(p + 8, alpha2, 2); + p += lengths[k]; + } + } + KUNIT_ASSERT_PTR_EQ(test, p, data + *size); + return data; +} + +/* The entry of @channel in a parsed package, or NULL. */ +static u8 *pkg_entry(const struct mt7932_policy *policy, unsigned int channel) +{ + unsigned int i, j; + + for (i = 1; i <= 9; i++) + for (j = 0; j < policy->table[i][4]; j++) + if (policy->table[i][44 + j * PKG_ENTRY_SIZE] == channel) + return (u8 *)policy->table[i] + 44 + j * PKG_ENTRY_SIZE; + return NULL; +} + +static void mt7932_policy_valid_test(struct kunit *test) +{ + struct mt7932_policy policy; + size_t size; + u8 *data; + + data = pkg_build(test, "AU", &size); + KUNIT_EXPECT_EQ(test, size, 10447); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&policy, data, size, "AU"), 0); + KUNIT_EXPECT_EQ(test, policy.modes, 3); + KUNIT_EXPECT_PTR_EQ(test, policy.table[0], data + 16 + 3 * PKG_MODE_SIZE); + KUNIT_EXPECT_EQ(test, policy.length[12], 265); + KUNIT_EXPECT_TRUE(test, mt7932_policy_permits(&policy, 36)); + KUNIT_EXPECT_FALSE(test, mt7932_policy_permits(&policy, 14)); + + data = pkg_build(test, "XZ", &size); + KUNIT_EXPECT_EQ(test, size, 10241); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&policy, data, size, "XZ"), 0); + KUNIT_EXPECT_EQ(test, policy.modes, 2); + KUNIT_EXPECT_TRUE(test, mt7932_policy_permits(&policy, 14)); +} + +/* A package serves only the country it names, in the header and table 10. */ +static void mt7932_policy_country_mismatch_test(struct kunit *test) +{ + struct mt7932_policy policy; + size_t size; + u8 *data; + + data = pkg_build(test, "AU", &size); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "XZ"), -EINVAL); + /* A country package renamed to the world one is not the world one. */ + memcpy(data + 6, "XZ", 2); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "XZ"), -EINVAL); +} + +/* Every prefix of a valid package, with a consistent header, is rejected. */ +static void mt7932_policy_truncation_test(struct kunit *test) +{ + struct mt7932_policy policy; + size_t size, cut; + u8 *data, *copy; + + data = pkg_build(test, "PL", &size); + for (cut = 0; cut < size; cut++) { + /* An exact-size copy lets KASAN catch any read past the end. */ + copy = kmemdup(data, cut, GFP_KERNEL); + KUNIT_ASSERT_TRUE(test, copy || !cut); + if (cut >= 16) + put_unaligned_le32(cut, copy + 12); + KUNIT_EXPECT_EQ_MSG(test, mt7932_policy_parse(&policy, copy, cut, "PL"), + -EINVAL, "length %zu", cut); + kfree(copy); + } +} + +static void mt7932_policy_header_test(struct kunit *test) +{ + struct mt7932_policy policy; + size_t size; + u8 *data, *longer; + + data = pkg_build(test, "PL", &size); + /* Trailing bytes after the last table. */ + longer = kunit_kzalloc(test, size + 4, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, longer); + memcpy(longer, data, size); + put_unaligned_le32(size + 4, longer + 12); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, longer, size + 4, "PL"), -EINVAL); + + data[4] = 2; + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + data[4] = 1; + data[8] = 4; + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + data[8] = 3; + data[9] = 12; + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + data[9] = MT7932_POLICY_TABLES; + put_unaligned_le32(size - 1, data + 12); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + put_unaligned_le32(size, data + 12); + /* Mode command lengths are bounded by the 320-byte body. */ + put_unaligned_le32(321, data + 16 + 4); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + put_unaligned_le32(0, data + 16 + 4); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + put_unaligned_le32(17, data + 16 + 4); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), 0); +} + +/* Table lengths and entry counts must agree with the table type. */ +static void mt7932_policy_table_test(struct kunit *test) +{ + struct mt7932_policy policy; + size_t size; + u8 *data, *table; + + data = pkg_build(test, "PL", &size); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), 0); + table = (u8 *)policy.table[3]; + /* An entry count that would run past the table. */ + table[4] = 9; + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + table[4] = 8; + /* A table length that claims the rest of the file. */ + put_unaligned_le16(0xffff, table + 2); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + put_unaligned_le16(44 + 8 * PKG_ENTRY_SIZE, table + 2); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), 0); + table = (u8 *)policy.table[10]; + table[7] = 2; + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); +} + +/* The world package's 2.4 GHz tables must list channels 1..14 in order. */ +static void mt7932_policy_world_order_test(struct kunit *test) +{ + struct mt7932_policy policy; + size_t size; + u8 *data; + + data = pkg_build(test, "XZ", &size); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&policy, data, size, "XZ"), 0); + pkg_entry(&policy, 9)[0] = 10; + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "XZ"), -EINVAL); +} + +static void mt7932_policy_permits_test(struct kunit *test) +{ + struct mt7932_policy policy; + size_t size; + u8 *data, *entry; + + data = pkg_build(test, "PL", &size); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), 0); + entry = pkg_entry(&policy, 144); + KUNIT_ASSERT_NOT_NULL(test, entry); + memset(entry + 1, MT7932_POLICY_NOT_PERMITTED, PKG_ENTRY_SIZE - 1); + KUNIT_EXPECT_FALSE(test, mt7932_policy_permits(&policy, 144)); + /* One rate with a limit is enough to permit the channel. */ + entry[PKG_ENTRY_SIZE - 1] = 8; + KUNIT_EXPECT_TRUE(test, mt7932_policy_permits(&policy, 144)); + /* A channel the package does not list is not permitted. */ + KUNIT_EXPECT_FALSE(test, mt7932_policy_permits(&policy, 177)); +} + +static void mt7932_policy_filter_test(struct kunit *test) +{ + static const u16 channels[] = {1, 6, 13, 36, 144, 149, 165}; + struct mt7932_reg_snapshot *reg; + struct mt7932_policy policy; + unsigned int i; + size_t size; + u8 *data; + + reg = kunit_kzalloc(test, sizeof(*reg), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, reg); + data = pkg_build(test, "PL", &size); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), 0); + memset(pkg_entry(&policy, 144) + 1, MT7932_POLICY_NOT_PERMITTED, + PKG_ENTRY_SIZE - 1); + memset(pkg_entry(&policy, 6) + 1, MT7932_POLICY_NOT_PERMITTED, + PKG_ENTRY_SIZE - 1); + memcpy(reg->domain, "PL", 2); + for (i = 0; i < ARRAY_SIZE(channels); i++) { + put_unaligned_le16(channels[i], reg->domain + 12 + i * 8); + put_unaligned_le32(i, reg->domain + 16 + i * 8); + } + reg->domain[8] = 3; + reg->domain[9] = 4; + reg->length = 12 + ARRAY_SIZE(channels) * 8; + + mt7932_policy_filter(reg, &policy); + KUNIT_EXPECT_EQ(test, reg->length, 12 + 5 * 8); + KUNIT_EXPECT_EQ(test, reg->domain[8], 2); + KUNIT_EXPECT_EQ(test, reg->domain[9], 3); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(reg->domain + 12 + 8), 13); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(reg->domain + 12 + 3 * 8), 149); + /* Each entry keeps its own channel flags. */ + KUNIT_EXPECT_EQ(test, get_unaligned_le32(reg->domain + 16 + 3 * 8), 5); + /* The vacated tail is cleared. */ + KUNIT_EXPECT_TRUE(test, !memchr_inv(reg->domain + 12 + 5 * 8, 0, 2 * 8)); +} + +/* Random corruption must never be accepted in a way that reads out of bounds. */ +static void mt7932_policy_corruption_test(struct kunit *test) +{ + struct rnd_state rnd; + struct mt7932_policy policy; + unsigned int round, flips, channel; + size_t size; + u8 *data, *copy; + + prandom_seed_state(&rnd, 7932); + data = pkg_build(test, "AU", &size); + copy = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, copy); + for (round = 0; round < 2000; round++) { + memcpy(copy, data, size); + for (flips = 1 + prandom_u32_state(&rnd) % 4; flips; flips--) + copy[prandom_u32_state(&rnd) % size] ^= 1 << (prandom_u32_state(&rnd) % 8); + if (mt7932_policy_parse(&policy, copy, size, "AU")) + continue; + for (channel = 0; channel < 256; channel++) + mt7932_policy_permits(&policy, channel); + } +} + static struct kunit_case mt7932_policy_test_cases[] = { KUNIT_CASE(mt7932_reg_relaxation_test), + KUNIT_CASE(mt7932_policy_valid_test), + KUNIT_CASE(mt7932_policy_country_mismatch_test), + KUNIT_CASE(mt7932_policy_truncation_test), + KUNIT_CASE(mt7932_policy_header_test), + KUNIT_CASE(mt7932_policy_table_test), + KUNIT_CASE(mt7932_policy_world_order_test), + KUNIT_CASE(mt7932_policy_permits_test), + KUNIT_CASE(mt7932_policy_filter_test), + KUNIT_CASE(mt7932_policy_corruption_test), {} }; From 750aac9d54858966b075213f7713f7b9c3716cda Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 11:54:02 +1100 Subject: [PATCH 1432/2680] usb: typec: tipd: Add KUnit tests for the USB4 router lookup Test cd321x_tbt_switch_get() on a software node graph shaped like the M1 and M2 device trees: a connector whose port@2 is linked to a node with the thunderbolt-switch property. A connector without that link, and a router that has registered its switch, must not consult the driver core. A router that has not registered its switch must defer the port while the driver core keeps waiting, and must no longer hold the port back once it gives up. Move the driver core query into cd321x_tbt_switch_keep_waiting() so that the tests can redirect it with a static stub. Without KUnit both helpers stay static and the stub compiles away. (cherry picked from commit 6b59e8a697dcb31b4d7ff91354d127f48ed4f0b4) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/Kconfig | 4 +- drivers/usb/typec/tipd/Makefile | 2 +- drivers/usb/typec/tipd/cd321x-tbt-test.c | 236 +++++++++++++++++++++++ drivers/usb/typec/tipd/core.c | 17 +- drivers/usb/typec/tipd/tps6598x.h | 6 + 5 files changed, 260 insertions(+), 5 deletions(-) create mode 100644 drivers/usb/typec/tipd/cd321x-tbt-test.c diff --git a/drivers/usb/typec/tipd/Kconfig b/drivers/usb/typec/tipd/Kconfig index a447a4622ea76..aae3e81cca848 100644 --- a/drivers/usb/typec/tipd/Kconfig +++ b/drivers/usb/typec/tipd/Kconfig @@ -29,11 +29,13 @@ config TYPEC_SN201202X module will be called sn201202x.ko. config TYPEC_TIPD_KUNIT_TEST - tristate "KUnit tests for CD321x resume state" if !KUNIT_ALL_TESTS + tristate "KUnit tests for CD321x" if !KUNIT_ALL_TESTS depends on KUNIT && TYPEC_TPS6598X_CORE default KUNIT_ALL_TESTS help Test suspend gating, fresh cable-state revalidation, bounded retries and removal during resume. The tests exercise the state transitions shared with the driver, including aborted and repeated sleep cycles. + Also test that a USB4/Thunderbolt router which never registers its + switch does not keep the port from probing. These tests run without accessing a physical Type-C controller. diff --git a/drivers/usb/typec/tipd/Makefile b/drivers/usb/typec/tipd/Makefile index 1a7605e7e57ec..198d57654b577 100644 --- a/drivers/usb/typec/tipd/Makefile +++ b/drivers/usb/typec/tipd/Makefile @@ -12,4 +12,4 @@ tps6598x-y := i2c.o obj-$(CONFIG_TYPEC_SN201202X) += sn201202x.o sn201202x-y := spmi.o -obj-$(CONFIG_TYPEC_TIPD_KUNIT_TEST) += cd321x-pm-test.o +obj-$(CONFIG_TYPEC_TIPD_KUNIT_TEST) += cd321x-pm-test.o cd321x-tbt-test.o diff --git a/drivers/usb/typec/tipd/cd321x-tbt-test.c b/drivers/usb/typec/tipd/cd321x-tbt-test.c new file mode 100644 index 0000000000000..63858db80d509 --- /dev/null +++ b/drivers/usb/typec/tipd/cd321x-tbt-test.c @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for the CD321x USB4/Thunderbolt router lookup + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "tps6598x.h" + +/* + * A Type-C connector whose port@2 is linked to a USB4 router, as on the M1 + * and M2 machines, and a connector without such a link. + */ +static const struct software_node cd321x_test_router_ep; +static const struct software_node cd321x_test_connector_ep; + +static const struct property_entry cd321x_test_router_props[] = { + PROPERTY_ENTRY_BOOL("thunderbolt-switch"), + { } +}; + +static const struct software_node cd321x_test_router = { + .name = "cd321x-test-router", + .properties = cd321x_test_router_props, +}; + +static const struct software_node cd321x_test_router_port = { + .name = "port@1", + .parent = &cd321x_test_router, +}; + +static const struct software_node_ref_args cd321x_test_router_refs[] = { + SOFTWARE_NODE_REFERENCE(&cd321x_test_connector_ep), +}; + +static const struct property_entry cd321x_test_router_ep_props[] = { + PROPERTY_ENTRY_REF_ARRAY("remote-endpoint", cd321x_test_router_refs), + { } +}; + +static const struct software_node cd321x_test_router_ep = { + .name = "endpoint", + .parent = &cd321x_test_router_port, + .properties = cd321x_test_router_ep_props, +}; + +static const struct software_node cd321x_test_connector = { + .name = "cd321x-test-connector", +}; + +static const struct software_node cd321x_test_connector_port = { + .name = "port@2", + .parent = &cd321x_test_connector, +}; + +static const struct software_node_ref_args cd321x_test_connector_refs[] = { + SOFTWARE_NODE_REFERENCE(&cd321x_test_router_ep), +}; + +static const struct property_entry cd321x_test_connector_ep_props[] = { + PROPERTY_ENTRY_REF_ARRAY("remote-endpoint", cd321x_test_connector_refs), + { } +}; + +static const struct software_node cd321x_test_connector_ep = { + .name = "endpoint", + .parent = &cd321x_test_connector_port, + .properties = cd321x_test_connector_ep_props, +}; + +static const struct software_node cd321x_test_plain_connector = { + .name = "cd321x-test-plain-connector", +}; + +static const struct software_node * const cd321x_test_nodes[] = { + &cd321x_test_router, + &cd321x_test_router_port, + &cd321x_test_router_ep, + &cd321x_test_connector, + &cd321x_test_connector_port, + &cd321x_test_connector_ep, + &cd321x_test_plain_connector, + NULL +}; + +struct cd321x_tbt_test { + struct device *dev; + bool keep_waiting; + unsigned int keep_waiting_calls; +}; + +/* Stands in for the driver core's answer on whether to keep waiting. */ +static bool cd321x_tbt_test_keep_waiting(struct device *dev) +{ + struct cd321x_tbt_test *ctx = kunit_get_current_test()->priv; + + ctx->keep_waiting_calls++; + return ctx->keep_waiting; +} + +static void cd321x_tbt_test_unregister_nodes(void *unused) +{ + software_node_unregister_node_group(cd321x_test_nodes); +} + +static int cd321x_tbt_test_init(struct kunit *test) +{ + struct cd321x_tbt_test *ctx; + int ret; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ctx); + test->priv = ctx; + + ret = software_node_register_node_group(cd321x_test_nodes); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = kunit_add_action_or_reset(test, cd321x_tbt_test_unregister_nodes, + NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + + ctx->dev = kunit_device_register(test, "cd321x"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx->dev); + + kunit_activate_static_stub(test, cd321x_tbt_switch_keep_waiting, + cd321x_tbt_test_keep_waiting); + return 0; +} + +static int cd321x_tbt_test_set(struct typec_thunderbolt_switch_dev *sw, + const struct typec_thunderbolt_switch_data *data) +{ + return 0; +} + +static void cd321x_tbt_test_unregister_switch(void *sw) +{ + typec_thunderbolt_switch_unregister(sw); +} + +/* A connector without a router link never waits. */ +static void cd321x_tbt_no_router_test(struct kunit *test) +{ + struct cd321x_tbt_test *ctx = test->priv; + struct typec_thunderbolt_switch *sw; + + sw = cd321x_tbt_switch_get(ctx->dev, + software_node_fwnode(&cd321x_test_plain_connector)); + KUNIT_EXPECT_NULL(test, sw); + KUNIT_EXPECT_EQ(test, ctx->keep_waiting_calls, 0); +} + +/* A router whose driver has registered its switch is used. */ +static void cd321x_tbt_router_bound_test(struct kunit *test) +{ + struct cd321x_tbt_test *ctx = test->priv; + struct typec_thunderbolt_switch_desc desc = { + .fwnode = software_node_fwnode(&cd321x_test_router), + .set = cd321x_tbt_test_set, + }; + struct typec_thunderbolt_switch_dev *sw_dev; + struct typec_thunderbolt_switch *sw; + struct device *router; + int ret; + + router = kunit_device_register(test, "router"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, router); + /* The lookup takes a reference on the router driver's module. */ + KUNIT_ASSERT_NOT_NULL(test, router->driver); + + sw_dev = typec_thunderbolt_switch_register(router, &desc); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sw_dev); + ret = kunit_add_action_or_reset(test, cd321x_tbt_test_unregister_switch, + sw_dev); + KUNIT_ASSERT_EQ(test, ret, 0); + + sw = cd321x_tbt_switch_get(ctx->dev, + software_node_fwnode(&cd321x_test_connector)); + KUNIT_EXPECT_NOT_ERR_OR_NULL(test, sw); + KUNIT_EXPECT_EQ(test, ctx->keep_waiting_calls, 0); + typec_thunderbolt_switch_put(sw); +} + +/* A router that has not registered its switch yet defers the port. */ +static void cd321x_tbt_router_pending_test(struct kunit *test) +{ + struct cd321x_tbt_test *ctx = test->priv; + struct typec_thunderbolt_switch *sw; + + ctx->keep_waiting = true; + sw = cd321x_tbt_switch_get(ctx->dev, + software_node_fwnode(&cd321x_test_connector)); + KUNIT_EXPECT_PTR_EQ(test, sw, ERR_PTR(-EPROBE_DEFER)); + KUNIT_EXPECT_EQ(test, ctx->keep_waiting_calls, 1); +} + +/* + * Once the driver core stops waiting for missing dependencies, a router that + * never registered its switch, for example because its driver is not built, + * no longer keeps the port from probing. + */ +static void cd321x_tbt_router_never_binds_test(struct kunit *test) +{ + struct cd321x_tbt_test *ctx = test->priv; + struct typec_thunderbolt_switch *sw; + + ctx->keep_waiting = false; + sw = cd321x_tbt_switch_get(ctx->dev, + software_node_fwnode(&cd321x_test_connector)); + KUNIT_EXPECT_NULL(test, sw); + KUNIT_EXPECT_EQ(test, ctx->keep_waiting_calls, 1); +} + +static struct kunit_case cd321x_tbt_cases[] = { + KUNIT_CASE(cd321x_tbt_no_router_test), + KUNIT_CASE(cd321x_tbt_router_bound_test), + KUNIT_CASE(cd321x_tbt_router_pending_test), + KUNIT_CASE(cd321x_tbt_router_never_binds_test), + {} +}; + +static struct kunit_suite cd321x_tbt_suite = { + .name = "cd321x-tbt", + .init = cd321x_tbt_test_init, + .test_cases = cd321x_tbt_cases, +}; + +kunit_test_suite(cd321x_tbt_suite); +MODULE_DESCRIPTION("KUnit tests for the CD321x USB4/Thunderbolt router lookup"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 7c4fa7adcbd5c..6534ddb21f76b 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -6,6 +6,8 @@ * Author: Heikki Krogerus */ +#include +#include #include #include #include @@ -1507,6 +1509,15 @@ static void cd321x_stop_work(struct cd321x *cd321x) cancel_delayed_work_sync(&cd321x->update_work); } +/* Separate so that the KUnit tests can choose the driver core's answer. */ +VISIBLE_IF_KUNIT bool cd321x_tbt_switch_keep_waiting(struct device *dev) +{ + KUNIT_STATIC_STUB_REDIRECT(cd321x_tbt_switch_keep_waiting, dev); + + return driver_deferred_probe_check_state(dev) == -EPROBE_DEFER; +} +EXPORT_SYMBOL_IF_KUNIT(cd321x_tbt_switch_keep_waiting); + /* * The USB4/Thunderbolt router is only needed once a USB4 or Thunderbolt link * comes up. If its driver is not built or never binds, do not leave power @@ -1514,19 +1525,19 @@ static void cd321x_stop_work(struct cd321x *cd321x) * waiting for the router when the driver core stops waiting for missing * dependencies. */ -static struct typec_thunderbolt_switch * +VISIBLE_IF_KUNIT struct typec_thunderbolt_switch * cd321x_tbt_switch_get(struct device *dev, struct fwnode_handle *fwnode) { struct typec_thunderbolt_switch *sw; sw = fwnode_typec_thunderbolt_switch_get(fwnode); - if (sw != ERR_PTR(-EPROBE_DEFER) || - driver_deferred_probe_check_state(dev) == -EPROBE_DEFER) + if (sw != ERR_PTR(-EPROBE_DEFER) || cd321x_tbt_switch_keep_waiting(dev)) return sw; dev_info(dev, "USB4/Thunderbolt router unavailable, continuing without it\n"); return NULL; } +EXPORT_SYMBOL_IF_KUNIT(cd321x_tbt_switch_get); static int cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 1e550956e86b9..bca532b9019f0 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -462,4 +462,10 @@ static inline void tipd_debugfs_unregister(struct tps6598x *tps) { } static inline void tipd_debugfs_exit(void) { } #endif +#if IS_ENABLED(CONFIG_KUNIT) +bool cd321x_tbt_switch_keep_waiting(struct device *dev); +struct typec_thunderbolt_switch * +cd321x_tbt_switch_get(struct device *dev, struct fwnode_handle *fwnode); +#endif + #endif /* __TPS6598X_H__ */ From 998328ddfa3b165cc57dfbabae0c7090f79cfa29 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:17:40 +1100 Subject: [PATCH 1433/2680] soc: apple: sep: Drop an SBIO reply that belongs to an earlier exchange An SBIO reply is matched to the waiting exchange by opcode alone, and an error marker is matched before the opcode is checked. The receive work reads a reply before it takes the reassembly lock. If the requester times out in between and a retry begins the same opcode, the late reply completes the retry with the earlier attempt result. Bring-up now retries the registration, so the window is real. Number the exchanges. The receive work samples the number before it reads a reply and passes it with the chunk; a chunk sampled before the current exchange began is dropped. Replies to the current exchange can only arrive after it began, since its request is sent after that. (cherry picked from commit e4321646f86a1db4eea5e91e11424ac4df2304dd) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 5 ++- drivers/soc/apple/transfer.rs | 83 ++++++++++++++++++++++++++++++++++- 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 2ee4674a6457b..45c290f5523e5 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2617,6 +2617,9 @@ impl SepData { if !self.sbio_ready.load(Relaxed) { return; } + // Before reading the reply: a requester that times out meanwhile and + // begins a new exchange must not receive it. + let generation = self.sbio_rx.lock().generation(); // notification and the enclave's DMA are not ordered; buffer may still be poison if !self.ool_await_written(&self.ool_sbio, 0, transfer::HEADER_LEN) { @@ -2655,7 +2658,7 @@ impl SepData { } }; - let progress = self.sbio_rx.lock().on_chunk(marker, &packet, &payload); + let progress = self.sbio_rx.lock().on_chunk(generation, marker, &packet, &payload); match progress { transfer::Progress::NeedMore(cont) => { diff --git a/drivers/soc/apple/transfer.rs b/drivers/soc/apple/transfer.rs index 1f77f45162576..a4f5f0df3e308 100644 --- a/drivers/soc/apple/transfer.rs +++ b/drivers/soc/apple/transfer.rs @@ -177,6 +177,10 @@ pub(crate) struct Reassembly { done: Option, sending: Option, grants: u32, + // Counts exchanges begun. A reply is matched only by its opcode, so a + // late reply to an exchange that timed out would otherwise complete the + // next exchange with the same opcode. + generation: u64, } impl Reassembly { @@ -186,14 +190,22 @@ impl Reassembly { done: None, sending: None, grants: 0, + generation: 0, } } + /// The exchange a message arriving now can belong to. Sampled before the + /// message is read and passed back to [`Self::on_chunk`]. + pub(crate) fn generation(&self) -> u64 { + self.generation + } + pub(crate) fn begin(&mut self, opcode: u32) -> Result<()> { if self.active.is_some() { return Err(EBUSY); } self.done = None; + self.generation = self.generation.wrapping_add(1); self.active = Some(Active { opcode, total: 0, @@ -242,7 +254,16 @@ impl Reassembly { self.abort(); } - pub(crate) fn on_chunk(&mut self, marker: u8, packet: &Packet, payload: &[u8]) -> Progress { + /// Takes one chunk of a reply. `generation` is [`Self::generation`] as + /// sampled when the message arrived: a chunk that arrived before the + /// current exchange began belongs to an earlier one and is dropped. + pub(crate) fn on_chunk( + &mut self, + generation: u64, + marker: u8, + packet: &Packet, + payload: &[u8], + ) -> Progress { if marker < MARKER_FIRST { return Progress::Notification; } @@ -262,6 +283,9 @@ impl Reassembly { let Some(active_opcode) = self.active.as_ref().map(|a| a.opcode) else { return Progress::Ignored; }; + if generation != self.generation { + return Progress::Ignored; + } // Checked before the opcode test below, deliberately. if marker == MARKER_ERROR { @@ -371,3 +395,60 @@ impl Reassembly { self.done = Some(Completed { status, payload }); } } + +#[kernel::macros::kunit_tests(apple_sep_transfer)] +mod tests { + use super::*; + + const OPCODE: u32 = 0x48; + + fn reply(err: u32) -> Packet { + Packet { + version: VERSION, + total: 0, + offset: 0, + flags: 0, + err, + opcode: OPCODE, + chunk: 0, + } + } + + #[test] + fn a_reply_to_the_current_exchange_completes_it() { + let mut rx = Reassembly::new(); + assert!(rx.begin(OPCODE).is_ok()); + let generation = rx.generation(); + assert!(matches!( + rx.on_chunk(generation, MARKER_FIRST, &reply(0), &[]), + Progress::Complete + )); + assert!(rx.take_done().is_some_and(|done| done.status.is_ok())); + } + + #[test] + fn a_late_reply_does_not_complete_the_next_exchange() { + let mut rx = Reassembly::new(); + assert!(rx.begin(OPCODE).is_ok()); + // The reply to the first exchange is being read when it times out. + let late = rx.generation(); + rx.abort(); + assert!(rx.take_done().is_some()); + // A retry begins the same opcode before the late reply is delivered. + assert!(rx.begin(OPCODE).is_ok()); + assert!(matches!( + rx.on_chunk(late, MARKER_FIRST, &reply(0), &[]), + Progress::Ignored + )); + assert!(matches!( + rx.on_chunk(late, MARKER_ERROR, &reply(0x16), &[]), + Progress::Ignored + )); + assert!(rx.take_done().is_none()); + let current = rx.generation(); + assert!(matches!( + rx.on_chunk(current, MARKER_FIRST, &reply(0), &[]), + Progress::Complete + )); + } +} From 63bb160b15e9c528bc64753b179ffa0589b7e7b4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 05:00:00 +1100 Subject: [PATCH 1434/2680] arm64: dts: apple: Fix T600x display IOMMU arguments The non-Ultra USB4 routing override replaced the display IOMMU list with one-cell specifiers, although Asahi converted these DARTs to five cells. The parser consumes the next DART phandle as the DMA-window length, producing an invalid window and omitting display DARTs. Keep all three display engines from aurora-wip, using the five-cell 0..0xfbffffff aperture from the Asahi 7.2 display nodes. [ryan: adapted to 7.2.10 -- complete the argument conversion missed in commit c1e9d83af92c ("arm64: dts: apple: add non-Ultra T600x USB4 tunnel routing"), including the later non-Ultra display override.] Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index b7efe3826fe9c..230e6d8c91630 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -162,7 +162,9 @@ #ifdef APPLE_T600X_NON_ULTRA &display { - iommus = <&disp0_dart 0>, <&dispext0_dart 0>, <&dispext1_dart 0>; + iommus = <&disp0_dart 0 0x0 0x0 0x0 0xfc000000>, + <&dispext0_dart 0 0x0 0x0 0x0 0xfc000000>, + <&dispext1_dart 0 0x0 0x0 0x0 0xfc000000>; }; &dispext1_dart { From da0ad2596581af8cd649328f7db5280ce3f85dd9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 05:00:00 +1100 Subject: [PATCH 1435/2680] arm64: dts: apple: Fix T602x display IOMMU overrides The display argument audit found the same missed five-cell conversion in the M2 Max laptop and M2 Pro/Max desktop routing overrides. A short list either truncates a specifier or treats another DART phandle as part of its aperture, losing display engines. Use the Asahi 7.2 aperture for the existing display DARTs. Keep the two additional M2 Max DARTs with their own one-cell binding and legacy aperture, and preserve every Aurora routing candidate. [ryan: adapted to 7.2.10 -- match each display consumer to its provider argument count after the Asahi display-DART conversion.] Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6021-display2.dtsi | 4 +++- arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6021-display2.dtsi b/arch/arm64/boot/dts/apple/t6021-display2.dtsi index c518139f1fb4b..5bd77f4c78854 100644 --- a/arch/arm64/boot/dts/apple/t6021-display2.dtsi +++ b/arch/arm64/boot/dts/apple/t6021-display2.dtsi @@ -258,7 +258,9 @@ }; &display { - iommus = <&disp0_dart 0>, <&dispext0_dart 0>, <&dispext1_dart 0>, + iommus = <&disp0_dart 0 0x100 0x0 0x10 0x0>, + <&dispext0_dart 0 0x100 0x0 0x10 0x0>, + <&dispext1_dart 0 0x100 0x0 0x10 0x0>, <&dispext2_dart 0>, <&dispext3_dart 0>; }; diff --git a/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi b/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi index 64adaec8f3f11..822760c80ae59 100644 --- a/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi @@ -64,7 +64,8 @@ }; &display { - iommus = <&dispext0_dart 0>, <&dispext1_dart 0>; + iommus = <&dispext0_dart 0 0x100 0x0 0x10 0x0>, + <&dispext1_dart 0 0x100 0x0 0x10 0x0>; }; &dispext1_dart { From 2a758a244dbb1fd7ae5d4d7eeedf7384547f8c41 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 05:00:00 +1100 Subject: [PATCH 1436/2680] wifi: brcmfmac: Remove the premature authorization notification Asahi 7.2 carries two offload authorization notifications. The first queues port authorization before cfg80211_connect_done(), so __cfg80211_port_authorized() warns that the wdev is not connected. Remove the earlier duplicate and keep the notification after connect_done(), including its successful-connection guard. Leave the roaming notification alone. This retains the intended 4-way handshake offload support without authorizing an unconnected interface. [ryan: adapted to 7.2.10 -- remove the stale first variant inherited from Asahi, retaining the later variant and Aurora roaming support.] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index f9b390ff8dc09..197d31fcb9173 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -6723,12 +6723,6 @@ brcmf_bss_connect_done(struct brcmf_cfg80211_info *cfg, if (completed) { brcmf_get_assoc_ies(cfg, ifp); brcmf_update_bss_info(cfg, ifp); - if ((profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) || - (profile->use_fwsup == BRCMF_PROFILE_FWSUP_PSK)) { - cfg80211_port_authorized(ndev, profile->bssid, - NULL, 0, GFP_KERNEL); - brcmf_dbg(CONN, "Report port authorized\n"); - } set_bit(BRCMF_VIF_STATUS_CONNECTED, &ifp->vif->sme_state); conn_params.status = WLAN_STATUS_SUCCESS; From 0117e6ecea1733de2509b50b2c8c02184e4067b1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 05:00:00 +1100 Subject: [PATCH 1437/2680] arm64: configs: j700: Enable the CIO reset controller explicitly Without the Apple CIO reset controller, Apple USB4 host routers cannot start, even when USB4_APPLE_SOC is enabled. Enable it built in for the J700, which matches RESET_APPLE_CIO=y in the published 7.1 J700 configuration, so it is available before modular USB4 routers probe. The shared Asahi fragment already enables the driver as a module. [ryan: adapted to 7.2.11 -- asahi.config now sets CONFIG_RESET_APPLE_CIO=m itself; only the J700 fragment is changed.] Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/configs/j700.config | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/configs/j700.config b/arch/arm64/configs/j700.config index 1bcf325d6d943..fa88b33e7fae6 100644 --- a/arch/arm64/configs/j700.config +++ b/arch/arm64/configs/j700.config @@ -52,6 +52,7 @@ CONFIG_PINCTRL_APPLE_GPIO=y CONFIG_APPLE_WATCHDOG=y CONFIG_ARM_APPLE_SOC_CPUFREQ=y CONFIG_PHY_APPLE_ATC=y +CONFIG_RESET_APPLE_CIO=y CONFIG_USB_DWC3_APPLE=y CONFIG_TYPEC=y CONFIG_TYPEC_SN201202X=y From a6b50a114e0a0550eb9de4f9e7a9968ebe50fd1e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 09:17:28 +1100 Subject: [PATCH 1438/2680] arm64: dts: apple: t600x: Keep the DCS sleep performance state at 7 The pmgr-misc nodes now come from the upstream series, whose driver carries the T600x memory controller (DCS) floor of performance state 7 in its match data, so the node drops "apple,dcs-min-ps". This tree still has the original driver, which reads that property and otherwise moves DCS to performance state 1 in suspend_noirq. The older T600x node set the floor to 7, and every M1 Pro suspend we have logged ran at 7. The first suspend to reach suspend_noirq with the 7.2 device tree hung the machine. Restore the property until the upstream driver replaces the one here. [ryan: adapted to 7.2.10 -- restore the property dropped from the pmgr_misc node of commit 01a79e36ec5a ("arm64: dts: apple: Add pmgr-misc nodes to t60xx"), as set by 3ebb2e200c11.] Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t600x-die0.dtsi | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index 2cbe84a3d5625..008276bc52d67 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -73,6 +73,7 @@ reg = <0x2 0x8e20c000 0 0x400>, <0x2 0x8e20c800 0 0x400>; reg-names = "fabric-ps", "dcs-ps"; + apple,dcs-min-ps = <7>; }; pmp_dart: iommu@28e300000 { From 96666c8e5cecad201015139b6c98d4b875dac70f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 04:18:30 +1100 Subject: [PATCH 1439/2680] drm/apple: Treat a missing legacy EDID service as optional Legacy DCP firmware can start the DPAVSERV endpoint without announcing an EDID service. Stock accepts expiry of the advisory discovery wait, while Aurora reports an endpoint failure even though the endpoint remains live. Keep the discovery window so an early announcement can precede get_modes, and return success when it expires. Only the announcement callback may publish a service. Log expiry and legacy publication at debug level so a late announcement can be distinguished from continued absence without a warning for an optional service. Leave AFK startup errors and H17P's asynchronous admission unchanged. Extend the production-function host tests to check expiry and publication diagnostics as well as optional discovery and genuine startup failures. Fixes: 014aed99ea14 ("drm/apple: retrieve external H17P EDID through bounded AFKV2 inline calls") (cherry picked from commit 672846413c5933fc4254ad3daf2279d6bd588f35) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/epic/dpavservep.c | 18 ++-- .../selftests/drivers/gpu/apple-dcp/edid.py | 99 ++++++++++++++++++- 2 files changed, 109 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 2476209719c08..39b808306f624 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -138,6 +138,9 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, dev_dbg(dcp->dev, "DCPAVSERV: AFKV2 EDID service ready on channel %u\n", service->channel); dcpavserv_notify_edid(service); + } else { + /* Also report announcements arriving after the discovery window. */ + dev_dbg(dcp->dev, "DCPAVSERV: legacy EDID service published\n"); } } @@ -488,12 +491,15 @@ int dpavservep_init(struct apple_dcp *dcp) if (ep->wire_v2) return 0; - ret = wait_for_completion_timeout(&dcp->dcpavserv.enable_completion, - msecs_to_jiffies(1000)); - if (!ret) { - dev_warn(dcp->dev, "DCPAVSERV: timed out waiting for EDID service announcement\n"); - return -ETIMEDOUT; - } + /* + * Legacy firmware need not announce an EDID service. Give it the same + * discovery window so an early announcement can precede get_modes, + * but absence is not an endpoint failure. + * Only dcpavserv_init() may publish a service that actually arrived. + */ + if (!wait_for_completion_timeout(&dcp->dcpavserv.enable_completion, + msecs_to_jiffies(1000))) + dev_dbg(dcp->dev, "DCPAVSERV: no legacy EDID service within 1 s\n"); return 0; } diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py index 69ebe810f1c2d..e247ce3a5489e 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py @@ -45,9 +45,10 @@ def function(file,signature): #define cpu_to_le32(x) (x) #define dev_warn(...) ((void)0) #define dev_err(...) do { diagnostic_errors++; } while (0) +#define dev_dbg(dev,fmt,...) do { if(!strcmp(fmt,"DCPAVSERV: legacy EDID service published\n")) legacy_publishes++; } while (0) #define dev_info(...) do { diagnostic_information++; } while (0) -static unsigned diagnostic_errors,diagnostic_information; +static unsigned diagnostic_errors,diagnostic_information,legacy_publishes; #define DCP_FIRMWARE_H17P 17 #define EPIC_TYPE_NOTIFY 0 #define EPIC_CAT_REPORT 0 @@ -339,7 +340,8 @@ def function(file,signature): dpavservep_detach(&d);afk_service_publish(legacy);dcpavserv_init(legacy,name,native_class,0); assert(!d.dcpavserv.service && legacy->torndown); legacy=prepare(13,0,name,native_class);assert(legacy); - d.typec_generation++;afk_service_publish(legacy);dcpavserv_init(legacy,name,native_class,0); + legacy_publishes=0;d.typec_generation++;afk_service_publish(legacy);dcpavserv_init(legacy,name,native_class,0); + assert(legacy_publishes==1); assert(d.dcpavserv.service==legacy && legacy->users==1);dpavservep_detach(&d); assert(dcpdpserv_prepare(legacy,"dcpdp-service-epic",native_class,0)==0); ep.wire_v2=true;ep.endpoint=0x2a;d.external_physically_absent=false; @@ -573,6 +575,7 @@ def function(file, signature): #define scoped_guard(k,p) for(bool *_lock __attribute__((cleanup(release)))=acquire(p); _lock; _lock=once(_lock)) static void mutex_lock(bool *p) { (void)acquire(p); } static void mutex_unlock(bool *p) { (void)once(p); } +#define dev_dbg(...) ((void)0) #define dev_info(...) ((void)0) #define dev_err(...) ((void)0) #define trace_dcpavserv_init(...) ((void)0) @@ -932,3 +935,95 @@ def function(text, signature): } ''' return prefix + '\n'.join(parts) + suffix + + +def startup(root): + source = (root / 'drivers/gpu/drm/apple/epic/dpavservep.c').read_text() + return r''' +#include +#include +#include +#include +#include +#define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 +#define DPAVSERV_ENDPOINT 0x28 +#define DPAVSERV_ENDPOINT_V2 0x2a +#define IS_ERR(p) ((uintptr_t)(p) > (uintptr_t)-4096) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +#define ERR_PTR(e) ((void *)(intptr_t)(e)) +#define dev_warn(...) do { warnings++; } while (0) +#define dev_dbg(...) do { debug++; } while (0) +struct apple_dcp; +struct apple_dcp_afkep { + struct apple_dcp *dcp; + unsigned endpoint; + bool wire_v2,inline_v2,match_epic_name; +}; +struct apple_dcp { + void *dev; + struct { int iomfb_method_profile; } hw; + struct { unsigned enable_completion; bool enabled; } dcpavserv; + struct apple_dcp_afkep *dcpavservep; +}; +static const unsigned dpavservep_ops[] = {0}; +static struct apple_dcp_afkep endpoint; +static int init_error,start_error; +static unsigned warnings,waits,starts,shutdowns,detaches,debug; +static bool announce; +static void init_completion(unsigned *c) { *c=0; } +static unsigned msecs_to_jiffies(unsigned ms) { return ms; } +static unsigned wait_for_completion_timeout(unsigned *c,unsigned timeout) +{ assert(timeout==1000);waits++;return *c; } +static struct apple_dcp_afkep *afk_init(struct apple_dcp *d,unsigned ep,const void *ops) +{ + assert(ops==dpavservep_ops); + endpoint=(struct apple_dcp_afkep){.dcp=d,.endpoint=ep}; + return init_error ? ERR_PTR(init_error) : &endpoint; +} +static int afk_start(struct apple_dcp_afkep *ep) +{ + starts++; + assert(ep==&endpoint&&ep->match_epic_name); + if(announce&&!start_error) { + ep->dcp->dcpavserv.enabled=true; + ep->dcp->dcpavserv.enable_completion=1; + } + return start_error; +} +static void afk_shutdown(struct apple_dcp_afkep *ep) +{ assert(ep==&endpoint);shutdowns++; } +static void dpavservep_detach(struct apple_dcp *d) +{ detaches++;d->dcpavserv.enabled=false; } +''' + block(source, 'int dpavservep_init(') + r''' +int main(void) +{ + for(unsigned v2=0;v2<2;v2++) { + for(unsigned early=0;early<2;early++) { + struct apple_dcp d={.hw={.iomfb_method_profile=v2?2:0}}; + announce=early;waits=warnings=starts=shutdowns=detaches=debug=0; + assert(dpavservep_init(&d)==0); + assert(starts==1&&waits==!v2&&!warnings&&!shutdowns&&!detaches); + assert(d.dcpavservep==&endpoint&&endpoint.wire_v2==v2&&!endpoint.inline_v2); + assert(endpoint.endpoint==(v2?DPAVSERV_ENDPOINT_V2:DPAVSERV_ENDPOINT)); + /* A missing optional service must not be fabricated as enabled. */ + assert(d.dcpavserv.enabled==early); + assert(debug==(!v2&&!early)); + const int errors[]={-ENOMEM,-ETIMEDOUT}; + for(unsigned i=0;i Date: Fri, 16 Oct 2026 04:31:43 +1100 Subject: [PATCH 1440/2680] usb: typec: tipd: Apply SAFE after ordinary USB role shutdown Ordinary Apple USB ports must stop DWC3 before the mux changes the shared PHY lanes. Early SAFE during removal or probe rollback instead reaches ATC with a live PIPE. Prepare display shutdown first and hold the SAFE transaction across role NONE, applying it after successful role shutdown. Let mux providers declare this ordering requirement and use one predicate for disconnect and removal. ATC declares it only for ordinary ports, so a fixed host still applies SAFE while its backend clock is available, even if its firmware graph exposes a no-op role switch. Other providers retain the existing early order. This keeps the fixed-host divergence from Asahi because its shared PIPE must be parked before the host clock disappears. An unregistered role switch returns -EOPNOTSUPP after controller teardown; accept that shutdown result only when it also reports role NONE. Keep unsupported requests on a live role as errors. On other errors abort without applying SAFE and retain recovery ownership. Reject an ordinary-port SAFE against a live PIPE with -EBUSY so the next probe cannot warn or change live lanes. The removal transaction remains open until generic disconnect finishes; the prepared flag prevents an early second application. Cover the ordering, fixed-host no-op switch, absent switch, errors and re-probe in host tests, and the live-PIPE guard in KUnit without MMIO. Fixes: 6f004361c61b ("usb: typec: serialize display teardown and preserve recovery ownership") [ryan: split by topic: only the apple-dcp selftests part; the rest is in 170-usb] (cherry picked from commit 4b36583930a5bef1e5bcf0942ea833ac55c4d748) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/gpu/apple-dcp/Makefile | 2 +- .../drivers/gpu/apple-dcp/quiesce.py | 78 +++++++++++++++++-- .../selftests/drivers/gpu/apple-dcp/run.py | 3 +- .../drivers/gpu/apple-dcp/transaction.py | 6 +- 4 files changed, 79 insertions(+), 10 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile index bbb39cada55bf..619da7189c114 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT TEST_PROGS := run.sh -TEST_FILES := run.py common.py edid.py recovery.py rate.py phy.py lifecycle.py pm.py idle.py boot.py dwc3 README +TEST_FILES := run.py common.py edid.py recovery.py rate.py phy.py lifecycle.py pm.py idle.py boot.py quiesce.py transaction.py accounting.py dwc3 README include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py index 6e7b9f69b2a25..e422e9acfa0ae 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py @@ -12,6 +12,7 @@ def make_unit(source, parent=False, reference=None): #include #include #include +#include #include #include typedef unsigned u32; @@ -25,6 +26,7 @@ def make_unit(source, parent=False, reference=None): #define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) #define to_platform_device(p) (p) #define typec_mux_get_drvdata(m) ((m)->data) +#define dev_dbg(...) ((void)0) #define dev_warn(...) ((void)0) #define dev_info(...) ((void)0) #define dev_name(p) "device" @@ -38,7 +40,7 @@ def make_unit(source, parent=False, reference=None): #define TPS_STATUS_TO_TYPEC_VCONN(s) (s) #define TPS_STATUS_TO_TYPEC_DATAROLE(s) (s) struct typec_mux_state { void *alt; unsigned long mode; void *data; }; -struct typec_mux_dev { bool removed; void (*commit)(struct typec_mux_dev *,struct typec_mux_state *); void (*abort)(struct typec_mux_dev *,struct typec_mux_state *); int (*finish)(struct typec_mux_dev *,struct typec_mux_state *); void *data; int (*set)(struct typec_mux_dev *,struct typec_mux_state *); int (*prepare)(struct typec_mux_dev *,struct typec_mux_state *); }; +struct typec_mux_dev { bool removed, safe_after_role; void (*commit)(struct typec_mux_dev *,struct typec_mux_state *); void (*abort)(struct typec_mux_dev *,struct typec_mux_state *); int (*finish)(struct typec_mux_dev *,struct typec_mux_state *); void *data; int (*set)(struct typec_mux_dev *,struct typec_mux_state *); int (*prepare)(struct typec_mux_dev *,struct typec_mux_state *); }; struct task_struct { int unused; }; static _Thread_local struct task_struct this_task; #define current (&this_task) @@ -65,6 +67,8 @@ def make_unit(source, parent=False, reference=None): static struct apple_dcp_typec_port *sole_port(void) { return &port; } static char events[100];static unsigned count; static int power_error,phy_error,tbt_error,role_error,last_error,phy_calls,reset_calls,gpio_calls,role_calls,stops,unregisters; +static bool ordinary_port,controller_active,phy_off,safe_noop; + static void event(char c) { assert(count+1external; } static int dcp_dptx_disconnect_oob(void *dev,u32 index) { (void)dev;assert(index==0);event('Q');stops++;if(power_error){display.external_error=power_error;return power_error;}display.external_cable=false;display.external_stopped=true;display.dptxport[0].connected=false;display.typec_cable_connected=false;connector.connected=false;return 0; } @@ -85,7 +89,8 @@ def make_unit(source, parent=False, reference=None): struct typec_thunderbolt_switch_data { int state; }; static void cd321x_stop_work(struct cd321x *c) { (void)c;event('W'); } static int typec_thunderbolt_switch_set(void *p,void *s) { (void)p;(void)s;event('B');return tbt_error; } -static int usb_role_switch_set_role(void *p,int r) { (void)p;assert(r==USB_ROLE_NONE);event('R');role_calls++;return role_error; } +static int usb_role_switch_set_role(void *p,int r) { (void)p;assert(r==USB_ROLE_NONE);if(ordinary_port&&!role_error){controller_active=false;phy_off=true;}event('R');role_calls++;return role_error; } +static enum usb_role __attribute__((unused)) usb_role_switch_get_role(void *p) { (void)p;return controller_active?USB_ROLE_HOST:USB_ROLE_NONE; } static void typec_unregister_partner(void *p) { (void)p;event('P'); } #define typec_set_pwr_opmode(...) ((void)0) #define typec_set_pwr_role(...) ((void)0) @@ -102,9 +107,12 @@ def make_unit(source, parent=False, reference=None): static void gpiod_set_value_cansleep(void *p,int v) { (void)p;assert(v==1);gpio_calls++;event('G'); } '''.replace('REMOVE_RETURN','void' if parent else 'int') - functions='\n'.join(block(mux, marker) for marker in ('static void typec_mux_unlock(', 'void typec_mux_abort(', 'int typec_mux_begin(', 'int typec_mux_apply(', 'int typec_mux_end(', 'int typec_mux_prepare(')) if not parent else block((reference/'drivers/usb/typec/mux.c').read_text(),'int typec_mux_prepare(')+'\n' + policy=block(mux,'bool typec_mux_safe_after_role(')+'\n' if 'bool typec_mux_safe_after_role(' in mux else '' + functions=policy+'\n'.join(block(mux, marker) for marker in ('static void typec_mux_unlock(', 'void typec_mux_abort(', 'int typec_mux_begin(', 'int typec_mux_apply(', 'int typec_mux_end(', 'int typec_mux_prepare(')) if not parent else block((reference/'drivers/usb/typec/mux.c').read_text(),'int typec_mux_prepare(')+'\n' functions+=block(mux,'int typec_mux_set(')+'\n'+block(dcp,'static int dcp_typec_route_prepare(')+'\n'+block(dcp,'static int dcp_typec_route_finish(')+'\n'+block(dcp,'static void dcp_typec_route_commit(')+'\n' functions+=block(dcp,'static struct apple_dcp_typec_route *\ndcp_typec_rebalance_locked(')+'\n' + if 'static bool tps6598x_safe_after_role(' in core: + functions+=block(core,'static bool tps6598x_safe_after_role(')+'\n' if not parent:functions+=block(core,'static void tps6598x_drop_partner(')+'\n'+block(core,'static int tps6598x_set_data_role(')+'\n' functions+=block(core,'static '+('void' if parent else 'int')+' tps6598x_disconnect(')+'\n'+block(core,'static '+('void' if parent else 'int')+' cd321x_remove(')+'\n'+block(core,'void tipd_remove(')+'\n' controls=r''' @@ -113,12 +121,13 @@ def make_unit(source, parent=False, reference=None): static struct typec_port class_port; static struct tipd_data data; -static int phy_set(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;assert(s->mode==TYPEC_STATE_SAFE);event('L');phy_calls++;return phy_error; } +static int phy_set(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;assert(s->mode==TYPEC_STATE_SAFE);event('L');phy_calls++;if(ordinary_port&&controller_active)return -EBUSY;if(ordinary_port){assert(phy_off);safe_noop=true;}return phy_error; } static int display_set(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;(void)s;event('S');if(port.owner&&display.external&&connector.connected) {int r=dcp_dptx_disconnect_oob(display.dev,0);if(r)return r;}if(port.safe_pending)return 0;port.owner=NULL;return 0; } static void unregister_port(struct tps6598x *t) { (void)t;unregisters++;event('U'); } static int reset(struct tps6598x *t) { (void)t;reset_calls++;event('X');return 0; } static void initial(struct cd321x *c) { count=0;events[0]=0;power_error=phy_error=tbt_error=role_error=last_error=phy_calls=reset_calls=gpio_calls=role_calls=stops=unregisters=deactivate_calls=0; + ordinary_port=false;controller_active=true;phy_off=safe_noop=false; connector=(struct connector){.connected=true};display=(struct apple_dcp){.external=true,.external_cable=true,.dptxport={{.connected=true}},.typec_cable_connected=true,.typec_connector=&connector}; port=(struct apple_dcp_typec_port){.owner=&route,.applied_valid=true,.dp_wanted=true,.dp_hpd=true,.hpd=true};route=(struct apple_dcp_typec_route){.port=&port,.dcp=&display,.selected=true}; phy_provider=(struct typec_mux_dev){.set=phy_set};display_provider=(struct typec_mux_dev){.data=&route,.set=display_set,.prepare=dcp_typec_route_prepare,.finish=dcp_typec_route_finish,.commit=dcp_typec_route_commit}; @@ -130,7 +139,62 @@ def make_unit(source, parent=False, reference=None): int main(int argc,char **argv) { (void)argc;struct cd321x c;struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE}; initial(&c); - if(argv[1][0]=='o') {tipd_remove(&c.tps);fprintf(stderr,"ordered remove: %s\n",events);assert(position('Q')>=0&&position('Q')=0&&position('Q')=0&&position('Q')=0&&position('Q')tps.irq=1;data.register_port=register_port;data.read_data_status=read_data_status;data.connect=connect; + initial(c);ordinary_port=true;phy_provider.safe_after_role=true;c->tps.irq=1;data.register_port=register_port;data.read_data_status=read_data_status;data.connect=connect; connect_error=irq_error=register_error=node_puts=mux_get_error=0; } int main(void) { @@ -337,7 +337,7 @@ def probe(root): probe_initial(&c);irq_error=-ENOSPC;phy_error=-ETIMEDOUT; assert(tipd_init(&c.tps)==-ENOSPC&&reset_calls==0&&gpio_calls==0&&port.owner==&route&&port.safe_pending&&unregisters==1&&node_puts==1); probe_initial(&c);connect_error=-EAGAIN;role_error=-EBUSY; - assert(tipd_init(&c.tps)==-EAGAIN&&reset_calls==0&&gpio_calls==0&&phy_calls==1&&port.owner==&route&&port.safe_pending); + assert(tipd_init(&c.tps)==-EAGAIN&&reset_calls==0&&gpio_calls==0&&phy_calls==0&&port.owner==&route&&port.safe_pending); probe_initial(&c);irq_error=-ENOSPC; assert(tipd_init(&c.tps)==-ENOSPC&&reset_calls==1&&!port.owner&&!port.safe_pending&&unregisters==1); probe_initial(&c);mux_get_error=-ENODEV; @@ -392,7 +392,7 @@ def clock_order(root): unit = unit.replace(block(unit, 'int main('), block(unit, 'int main(').replace('int main(', 'int __attribute__((unused)) prior_main(', 1), 1) unit = unit.replace('static int power_error,', 'static bool backend_clock=true,parked;\nstatic int power_error,') unit = unit.replace("event('R');role_calls++;return role_error;", "event('R');role_calls++;if(!role_error)backend_clock=false;return role_error;", 1) - unit = unit.replace("event('L');phy_calls++;return phy_error;", "event('L');phy_calls++;if(parked)return 0;if(!backend_clock)return -ENODEV;if(!phy_error)parked=true;return phy_error;", 1) + unit = unit.replace("event('L');phy_calls++;", "event('L');phy_calls++;if(parked)return 0;if(!backend_clock)return -ENODEV;if(!phy_error)parked=true;", 1) unit += r""" int main(void) { (void)current;struct cd321x c;initial(&c); From 6d4144ae92febdde0cf32d697fd05f44fafc8ddd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 04:31:43 +1100 Subject: [PATCH 1441/2680] phy: apple: atc: Remember mux requests that keep the lane mode USB role shutdown can power the PHY off while its saved Type-C request is still USB3. A subsequent SAFE request needs no lane work, but both mux fast paths return without saving it. A later USB2 PHY initialization can then restore the old USB3 request instead of respecting SAFE. Record every accepted request whose lanes already match, both before acquiring USB runtime ownership and in the locked recheck. Preserve the capability checks, PIPE retries and fixed-hub no-op bypass. Test SAFE after independent shutdown on both paths and call the real USB2 initializer with a KUnit configuration stub that counts lane requests. Extend the host role mock through the production PHY power-off, mux recheck and USB2 init bodies, leaving only MMIO operations mocked. The host cases now verify that role NONE powers the PHY OFF, SAFE requires no lane work, and USB2 init cannot restore stale USB3 after SAFE. Fixes: 48836ec5a5b6 ("phy: apple: atc: Keep USB2 up when the port sits behind a fixed hub") [ryan: split by topic: only the apple-dcp selftests part; the rest is in 170-usb] (cherry picked from commit 2757f43e143ed2851d01cde331df9631e15d4329) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../drivers/gpu/apple-dcp/quiesce.py | 71 ++++++++++++++++--- 1 file changed, 63 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py index e422e9acfa0ae..fdab2d0b2cdac 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py @@ -6,6 +6,7 @@ def make_unit(source, parent=False, reference=None): reference=source if reference is None else reference core=(source/'drivers/usb/typec/tipd/core.c').read_text();mux=(source/'drivers/usb/typec/mux.c').read_text() + atc=(source/'drivers/phy/apple/atc.c').read_text() dcp=(reference/'drivers/gpu/drm/apple/dcp.c').read_text() prefix=r''' #include @@ -68,7 +69,10 @@ def make_unit(source, parent=False, reference=None): static char events[100];static unsigned count; static int power_error,phy_error,tbt_error,role_error,last_error,phy_calls,reset_calls,gpio_calls,role_calls,stops,unregisters; static bool ordinary_port,controller_active,phy_off,safe_noop; - +static int ordinary_power_off(void); +static int ordinary_safe(void); +static void ordinary_initial(void); +static void ordinary_usb2_check(void); static void event(char c) { assert(count+1external; } static int dcp_dptx_disconnect_oob(void *dev,u32 index) { (void)dev;assert(index==0);event('Q');stops++;if(power_error){display.external_error=power_error;return power_error;}display.external_cable=false;display.external_stopped=true;display.dptxport[0].connected=false;display.typec_cable_connected=false;connector.connected=false;return 0; } @@ -89,7 +93,7 @@ def make_unit(source, parent=False, reference=None): struct typec_thunderbolt_switch_data { int state; }; static void cd321x_stop_work(struct cd321x *c) { (void)c;event('W'); } static int typec_thunderbolt_switch_set(void *p,void *s) { (void)p;(void)s;event('B');return tbt_error; } -static int usb_role_switch_set_role(void *p,int r) { (void)p;assert(r==USB_ROLE_NONE);if(ordinary_port&&!role_error){controller_active=false;phy_off=true;}event('R');role_calls++;return role_error; } +static int usb_role_switch_set_role(void *p,int r) { (void)p;assert(r==USB_ROLE_NONE);if(ordinary_port&&!role_error){int ret=ordinary_power_off();if(ret)return ret;controller_active=false;phy_off=true;}event('R');role_calls++;return role_error; } static enum usb_role __attribute__((unused)) usb_role_switch_get_role(void *p) { (void)p;return controller_active?USB_ROLE_HOST:USB_ROLE_NONE; } static void typec_unregister_partner(void *p) { (void)p;event('P'); } #define typec_set_pwr_opmode(...) ((void)0) @@ -115,19 +119,70 @@ def make_unit(source, parent=False, reference=None): functions+=block(core,'static bool tps6598x_safe_after_role(')+'\n' if not parent:functions+=block(core,'static void tps6598x_drop_partner(')+'\n'+block(core,'static int tps6598x_set_data_role(')+'\n' functions+=block(core,'static '+('void' if parent else 'int')+' tps6598x_disconnect(')+'\n'+block(core,'static '+('void' if parent else 'int')+' cd321x_remove(')+'\n'+block(core,'void tipd_remove(')+'\n' + ordinary_phy=r''' +/* Actual ordinary-port ATC power-off, mux recheck and USB2 init; MMIO leaves. */ +#undef WARN_ON_ONCE +#define WARN_ON_ONCE(x) ({ bool hit=(x);phy_warns+=hit;hit; }) +#define fallthrough __attribute__((fallthrough)) +enum atcphy_mode { APPLE_ATCPHY_MODE_OFF, APPLE_ATCPHY_MODE_USB2, + APPLE_ATCPHY_MODE_USB3, APPLE_ATCPHY_MODE_USB3_DP, APPLE_ATCPHY_MODE_DP, + APPLE_ATCPHY_MODE_TBT, APPLE_ATCPHY_MODE_USB4 }; +enum atcphy_pipehandler_state { ATCPHY_PIPEHANDLER_STATE_DUMMY, ATCPHY_PIPEHANDLER_STATE_USB3 }; +struct apple_atcphy { + void *dev; int lock; + const struct { bool has_usb4,has_t8140_dp; } *hw; + struct { void *usb2phy; } regs; + bool fixed_usb2,host_active,mode_dirty,pipe_pending,usb_off_pending,ss_tunables,dp_tunables; + enum atcphy_mode mode,typec_mode; + enum atcphy_pipehandler_state pipe_state; +}; +struct phy { struct apple_atcphy *data; }; +#define phy_get_drvdata(p) ((p)->data) +#define USB2PHY_SIG 0 +#define USB2PHY_SIG_HOST 1 +static struct { enum atcphy_pipehandler_state pipehandler_state; bool enable_dp_aux; } atcphy_modes[7]; +static struct apple_atcphy ordinary_phy; +static struct phy usb_phy={.data=&ordinary_phy}; +static unsigned phy_warns,configure_calls,lane_requests; +static int atcphy_park_pipehandler(struct apple_atcphy *p) { p->pipe_state=ATCPHY_PIPEHANDLER_STATE_DUMMY;return 0; } +static int atcphy_configure_pipehandler(struct apple_atcphy *p,bool on) { (void)p;(void)on;assert(false);return -EINVAL; } +static int atcphy_configure(struct apple_atcphy *p,enum atcphy_mode mode) { configure_calls++;lane_requests+=mode!=APPLE_ATCPHY_MODE_OFF;p->mode=mode;p->mode_dirty=false;return 0; } +static void set32(void *p,unsigned v) { (void)p;(void)v;assert(false); } +static void atcphy_usb2_power_on(struct apple_atcphy *p) { (void)p;assert(false); } +''' + '\n'.join(block(atc, marker) for marker in ( + 'static int atcphy_usb3_power_off(', 'static int atcphy_mux_set_locked(', + 'static int atcphy_usb2_init(')) + r''' +static int ordinary_power_off(void) { return atcphy_usb3_power_off(&usb_phy); } +static void ordinary_initial(void) { + ordinary_phy=(struct apple_atcphy){.mode=APPLE_ATCPHY_MODE_USB3,.typec_mode=APPLE_ATCPHY_MODE_USB3,.pipe_state=ATCPHY_PIPEHANDLER_STATE_USB3}; + phy_warns=configure_calls=lane_requests=0; +} +static int ordinary_safe(void) { + unsigned before=configure_calls; + int ret=atcphy_mux_set_locked(&ordinary_phy,APPLE_ATCPHY_MODE_OFF); + assert(!phy_warns); + if(!ret){assert(ordinary_phy.mode==APPLE_ATCPHY_MODE_OFF);assert(configure_calls==before);} + return ret; +} +static void ordinary_usb2_check(void) { + unsigned before=lane_requests; + assert(atcphy_usb2_init(&usb_phy)==0); + assert(ordinary_phy.typec_mode==APPLE_ATCPHY_MODE_OFF&&lane_requests==before); +} +''' controls=r''' static struct typec_mux_dev phy_provider,display_provider; static struct typec_mux mux; static struct typec_port class_port; static struct tipd_data data; -static int phy_set(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;assert(s->mode==TYPEC_STATE_SAFE);event('L');phy_calls++;if(ordinary_port&&controller_active)return -EBUSY;if(ordinary_port){assert(phy_off);safe_noop=true;}return phy_error; } +static int phy_set(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;assert(s->mode==TYPEC_STATE_SAFE);event('L');phy_calls++;if(ordinary_port){int ret=ordinary_safe();if(ret)return ret;assert(phy_off);safe_noop=true;}return phy_error; } static int display_set(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;(void)s;event('S');if(port.owner&&display.external&&connector.connected) {int r=dcp_dptx_disconnect_oob(display.dev,0);if(r)return r;}if(port.safe_pending)return 0;port.owner=NULL;return 0; } static void unregister_port(struct tps6598x *t) { (void)t;unregisters++;event('U'); } static int reset(struct tps6598x *t) { (void)t;reset_calls++;event('X');return 0; } static void initial(struct cd321x *c) { count=0;events[0]=0;power_error=phy_error=tbt_error=role_error=last_error=phy_calls=reset_calls=gpio_calls=role_calls=stops=unregisters=deactivate_calls=0; - ordinary_port=false;controller_active=true;phy_off=safe_noop=false; + ordinary_port=false;controller_active=true;phy_off=safe_noop=false;ordinary_initial(); connector=(struct connector){.connected=true};display=(struct apple_dcp){.external=true,.external_cable=true,.dptxport={{.connected=true}},.typec_cable_connected=true,.typec_connector=&connector}; port=(struct apple_dcp_typec_port){.owner=&route,.applied_valid=true,.dp_wanted=true,.dp_hpd=true,.hpd=true};route=(struct apple_dcp_typec_route){.port=&port,.dcp=&display,.selected=true}; phy_provider=(struct typec_mux_dev){.set=phy_set};display_provider=(struct typec_mux_dev){.data=&route,.set=display_set,.prepare=dcp_typec_route_prepare,.finish=dcp_typec_route_finish,.commit=dcp_typec_route_commit}; @@ -153,7 +208,7 @@ def make_unit(source, parent=False, reference=None): if(!strcmp(argv[1],"ordinary-role-reprobe"))role_error=-EBUSY; if(!strncmp(argv[1],"ordinary-unsupported-",21))role_error=-EOPNOTSUPP; if(!strncmp(argv[1],"ordinary-unregistered-",22)) { - role_error=-EOPNOTSUPP;controller_active=false;phy_off=true; + role_error=-EOPNOTSUPP;assert(ordinary_power_off()==0);controller_active=false;phy_off=true; } if(!strcmp(argv[1],"ordinary-disconnect")||!strcmp(argv[1],"ordinary-unregistered-disconnect")||!strcmp(argv[1],"ordinary-unsupported-disconnect")) { int ret=tps6598x_disconnect(&c.tps,0); @@ -163,7 +218,7 @@ def make_unit(source, parent=False, reference=None): } else { assert(ret==0); assert(position('Q')>=0&&position('Q')=0&&position('Q') Date: Fri, 16 Oct 2026 18:15:49 +1100 Subject: [PATCH 1442/2680] drm/apple: Require a complete reply to a DPTX HPD request dptxport_set_hpd_timeout() takes an HPD request as acknowledged when the last word of the firmware's 20-byte reply echoes the HPD state it was sent. It reads the reply with afk_service_call_timeout(), which copies only as much of the reply as its header declares and zero-fills the rest of the output. A reply declaring a shorter body therefore reads as an echo of 0 and acknowledges HPD low, and a reply declaring more than its buffer holds is clipped and accepted for either state. Teardown depends on that acknowledgment: the external H17P DCP releases the display, and with it the Type-C route, only once the firmware has acknowledged HPD low. Make afk_service_call_timeout_len() available. It checks the length a reply declares against what arrived and against the response space the caller reserved, which now includes the caller's output padding, and it reports how much of the output the reply filled. Use it for HPD requests, and refuse a reply that does not fill the whole response before comparing the echo. This asks no more of the firmware for HPD low than every DCP already answers for HPD high, whose echo of 1 cannot come from zero fill. The only other user of the length check, the H17P EDID read, reserves no output padding and is unchanged. The apple-afkv2 service-call test extracts the now global function. Fixes: 5c29016e370b ("drm: apple: recover Type-C displays across link interruptions") (cherry picked from commit 88837bda2a82ece2f9b930ff926983fe4711af00) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 21 +++++++------------ drivers/gpu/drm/apple/afk.h | 10 +++++++++ drivers/gpu/drm/apple/dptxep.c | 13 ++++++++++-- .../gpu/apple-afkv2/extract-service.py | 3 ++- 4 files changed, 31 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 729a14fe9cdf2..4f19df565cfaf 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -1827,12 +1827,6 @@ int afk_service_call(struct apple_epic_service *service, u16 group, u32 command, output_pad, MSEC_PER_SEC); } -static int afk_service_call_timeout_len(struct apple_epic_service *service, u16 group, - u32 command, const void *data, size_t data_len, - size_t data_pad, void *output, size_t output_len, - size_t output_pad, unsigned int timeout_ms, - size_t *received); - int afk_service_call_len(struct apple_epic_service *service, u16 group, u32 command, const void *data, size_t data_len, size_t data_pad, void *output, size_t output_len, @@ -1853,11 +1847,11 @@ int afk_service_call_timeout(struct apple_epic_service *service, u16 group, timeout_ms, NULL); } -static int afk_service_call_timeout_len(struct apple_epic_service *service, u16 group, - u32 command, const void *data, size_t data_len, - size_t data_pad, void *output, size_t output_len, - size_t output_pad, unsigned int timeout_ms, - size_t *received) +int afk_service_call_timeout_len(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, unsigned int timeout_ms, + size_t *received) { struct epic_service_call *call; size_t wire_received = 0, result_size; @@ -1910,11 +1904,12 @@ static int afk_service_call_timeout_len(struct apple_epic_service *service, u16 goto out; } if (received) { + /* The reply may use the padding reserved after the output. */ ret = afkv2_service_reply_size(bfr, wire_received, group, command, - output_len, &result_size); + output_len + output_pad, &result_size); if (ret) goto out; - *received = result_size; + *received = min(result_size, output_len); } if (le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || le16_to_cpu(call->group) != group || diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index dcfa4342e4bc8..a85ed77597755 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -460,5 +460,15 @@ int afk_service_call_len(struct apple_epic_service *service, u16 group, u32 command, const void *data, size_t data_len, size_t data_pad, void *output, size_t output_len, size_t output_pad, size_t *received); +/* + * As afk_service_call_timeout(), but the length the reply declares is checked + * against what arrived and against @output_len + @output_pad, and @received + * is set to the number of bytes of @output that the reply filled. + */ +int afk_service_call_timeout_len(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, unsigned int timeout_ms, + size_t *received); #endif diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 98066b69ced73..06a1a3a093b08 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -191,6 +191,7 @@ int dptxport_set_hpd_timeout(struct apple_epic_service *service, bool hpd, unsigned int timeout_ms) { struct dcpdptx_hotplug_cmd cmd, resp; + size_t received; int ret; memset(&cmd, 0, sizeof(cmd)); @@ -198,10 +199,18 @@ int dptxport_set_hpd_timeout(struct apple_epic_service *service, bool hpd, if (hpd) cmd.unk = cpu_to_le32(1); - ret = afk_service_call_timeout(service, 8, 8, &cmd, sizeof(cmd), 12, - &resp, sizeof(resp), 12, timeout_ms); + ret = afk_service_call_timeout_len(service, 8, 8, &cmd, sizeof(cmd), 12, + &resp, sizeof(resp), 12, timeout_ms, + &received); if (ret) return ret; + /* Past the end of a short reply there is only zero fill, not an echo. */ + if (received < sizeof(resp)) { + dev_warn(service->ep->dcp->dev, + "set_hpd: short reply, %zu of %zu bytes (sent hpd=%d)\n", + received, sizeof(resp), hpd); + return -EPROTO; + } if (le32_to_cpu(resp.unk) != hpd) { dev_warn(service->ep->dcp->dev, "set_hpd: unk reply 0x%x (sent hpd=%d)\n", diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py index ab95f5c422710..19381f919d7e8 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py @@ -6,7 +6,7 @@ import sys source = pathlib.Path(sys.argv[1]).read_text() -match = re.search(r"static int afk_service_call_timeout_len\([^;]*?\)\n\{", source) +match = re.search(r"^int afk_service_call_timeout_len\([^;]*?\)\n\{", source, re.M) if not match: raise RuntimeError("production service-call definition not found") start = match.start() @@ -25,6 +25,7 @@ #define kzalloc(n,f) calloc(1,n) #define kfree(p) free(p) #define max(a,b) ((a)>(b)?(a):(b)) +#define min(a,b) ((a)<(b)?(a):(b)) #define cpu_to_le16(x) (x) #define cpu_to_le32(x) (x) #define le16_to_cpu(x) (x) From e75fd4a23699166aebbd2d75a3b4a42490a0dd0d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 18:15:49 +1100 Subject: [PATCH 1443/2680] selftests: apple-afkv2: Check the reply length of a DPTX HPD request Extract dptxport_set_hpd_timeout() with the actual service-call body and its timeout wrapper, and answer the HPD request with matched, successful replies in legacy and AFK-v2 framing. For both HPD states, a reply is refused when its declared body stops before the HPD echo (0 to 16 bytes), when it declares more than its buffer holds (36 and 65535 bytes), when it was cut short in transit, or when the echo is wrong. Complete (20 bytes) and padded (32 bytes) replies are accepted. Without "drm/apple: Require a complete reply to a DPTX HPD request", 18 of these checks fail: every short reply acknowledges HPD low and every oversized one is accepted. (cherry picked from commit d1cadb0ca747ae36337d1250379430f920adb0a6) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../drivers/gpu/apple-afkv2/Makefile | 2 +- .../selftests/drivers/gpu/apple-afkv2/README | 5 + .../drivers/gpu/apple-afkv2/extract-hpd.py | 150 ++++++++++++++++++ .../gpu/apple-afkv2/run-afkv2-tests.sh | 10 ++ 4 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/extract-hpd.py diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile b/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile index df1b033d9cb32..a792549ee9f14 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile @@ -4,6 +4,6 @@ export HOSTCC export AFKV2_SOURCE_ROOT := $(abspath ../../../../../..) TEST_PROGS := run-afkv2-tests.sh -TEST_FILES := codec.c owned-prefix.c owned-suffix.c extract-owned.py inline-edid.c extract-service.py extract-send.py extract-edid.py extract-ring.py extract-notify.py host-include +TEST_FILES := codec.c owned-prefix.c owned-suffix.c extract-owned.py inline-edid.c extract-service.py extract-hpd.py extract-send.py extract-edid.py extract-ring.py extract-notify.py host-include include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/README b/tools/testing/selftests/drivers/gpu/apple-afkv2/README index 064cdcf018595..bffcd55fba642 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/README +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/README @@ -28,6 +28,11 @@ EDID header/checksum/block limits, and continued rejection of inline responses by the OOB matcher. The actual service-call body is extracted for short reply, error status and timeout checks; its wire sender is mocked. +The DPTX HPD request is extracted with that service-call body and its timeout +wrapper. With legacy and AFK-v2 framing, a reply whose declared body stops +before the HPD echo, or runs past the bytes that arrived, is refused for both +HPD states; complete and padded replies are accepted and a wrong echo refused. + The real sender is also extracted to test a 128-byte mocked command payload with a 32 KiB response capacity, OOB behavior, send failures and timeout retention. The two production EDID readers are tested independently to diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-hpd.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-hpd.py new file mode 100644 index 0000000000000..6957d92f390e2 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-hpd.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the production DPTX HPD request with mocked service-call replies.""" +import pathlib +import re +import sys + + +def function(source, name): + match = re.search(r"^(?:static )?int " + name + r"\([^;]*?\)\n\{", source, re.M) + if not match: + raise RuntimeError("production definition of " + name + " not found") + end, depth = match.end(), 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + # Older trees keep the call body static; the unit declares it below. + return re.sub(r"^static ", "", source[match.start():end]) + + +afk = pathlib.Path(sys.argv[1]).read_text() +dptx = pathlib.Path(sys.argv[2]).read_text() +command = re.search(r"^struct dcpdptx_hotplug_cmd \{.*?\} __attribute__\(\(packed\)\);", + dptx, re.M | re.S) +if not command: + raise RuntimeError("production HPD request layout not found") +prefix = r''' +#undef NDEBUG +#include +#include +#include +#include "afkv2-codec.h" +typedef u32 __le32; +#define GFP_KERNEL 0 +#define kzalloc(n,f) calloc(1,n) +#define kfree(p) free(p) +#define max(a,b) ((a)>(b)?(a):(b)) +#define min(a,b) ((a)<(b)?(a):(b)) +#define cpu_to_le16(x) (x) +#define cpu_to_le32(x) (x) +#define le16_to_cpu(x) (x) +#define le32_to_cpu(x) (x) +#define EPIC_SERVICE_CALL_MAGIC 0x69706378 +#define EPIC_SUBTYPE_STD_SERVICE 0xc0 +#define dev_warn(...) do {} while (0) +#define dev_dbg(...) do {} while (0) +struct epic_service_call { + u8 pad0[2]; u16 group; u32 command, data_len, magic; u8 pad1[48]; +}; +struct dcp { void *dev; }; +struct ops { const char *name; }; +struct ep { struct dcp *dcp; int endpoint; bool inline_v2; }; +struct apple_epic_service { struct ep *ep; struct ops *ops; u32 channel; }; +int afk_service_call_timeout_len(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, size_t data_pad, void *output, + size_t output_len, size_t output_pad, unsigned int timeout_ms, size_t *received); +int afk_service_call_timeout(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, size_t data_pad, void *output, + size_t output_len, size_t output_pad, unsigned int timeout_ms); +int dptxport_set_hpd_timeout(struct apple_epic_service *service, bool hpd, + unsigned int timeout_ms); +/* + * A matched, successful reply in the request's buffer: a 64-byte header, then + * the body the firmware declares, of which the echo is the word at 16. An + * AFK-v2 transport reports how much of the buffer arrived; a legacy one fills + * it. Without a length query AFK-v2 refuses anything short of the buffer. + */ +static bool wire_v2; +static size_t declared, arrived; +static u32 echo_flip; +static int afk_send_command_timeout_len(struct apple_epic_service *s, u8 type, + const void *payload, size_t payload_len, void *output, size_t output_len, + u32 *retcode, unsigned int timeout_ms, size_t *received) +{ + struct epic_service_call *call = output; + u8 *body = (u8 *)output + sizeof(*call); + u32 echo; + + (void)s; (void)payload; (void)timeout_ms; + assert(type == EPIC_SUBTYPE_STD_SERVICE && payload_len == output_len); + assert(output_len == sizeof(*call) + 32 && call->data_len == 32); + assert(call->group == 8 && call->command == 8); + if (wire_v2 && !received && arrived < output_len - sizeof(*call)) + return -EMSGSIZE; + memcpy(&echo, body + 16, sizeof(echo)); + echo ^= echo_flip; + memcpy(body + 16, &echo, sizeof(echo)); + call->data_len = declared; + *retcode = 0; + if (received) + *received = sizeof(*call) + (wire_v2 ? arrived : output_len - sizeof(*call)); + return 0; +} +''' +suffix = r''' +static int failures; +static void expect(bool ok, const char *what, bool hpd, int ret) +{ + printf("%s %s: %s hpd=%d declared=%zu arrived=%zu ret=%d\n", ok ? "ok" : "FAIL", + wire_v2 ? "afk-v2" : "legacy", what, hpd, declared, arrived, ret); + failures += !ok; +} +int main(void) +{ + static const size_t lengths[] = { 0, 4, 8, 12, 16, 20, 32, 36, 65535 }; + struct dcp dcp = { 0 }; + struct ops ops = { "AppleDCPDPTXRemotePort" }; + struct ep ep = { .dcp = &dcp }; + struct apple_epic_service service = { .ep = &ep, .ops = &ops, .channel = 1 }; + + for (int v2 = 0; v2 < 2; v2++) { + wire_v2 = v2; + for (size_t i = 0; i < sizeof(lengths) / sizeof(lengths[0]); i++) { + for (int hpd = 0; hpd < 2; hpd++) { + int ret; + + declared = lengths[i]; arrived = 32; echo_flip = 0; + ret = dptxport_set_hpd_timeout(&service, hpd, 1000); + if (declared < sizeof(struct dcpdptx_hotplug_cmd)) + expect(ret < 0, "short reply refused", hpd, ret); + else if (declared > 32) + expect(ret < 0, "reply longer than its buffer refused", hpd, ret); + else + expect(ret == 0, "complete reply accepted", hpd, ret); + } + } + for (int hpd = 0; hpd < 2; hpd++) { + int ret; + + declared = 20; arrived = 32; echo_flip = 1; + ret = dptxport_set_hpd_timeout(&service, hpd, 1000); + expect(ret == -EINVAL, "wrong echo refused", hpd, ret); + if (!wire_v2) + continue; + declared = 20; arrived = 16; echo_flip = 0; + ret = dptxport_set_hpd_timeout(&service, hpd, 1000); + expect(ret < 0, "reply cut short in transit refused", hpd, ret); + } + } + printf("%d HPD reply checks failed\n", failures); + if (failures) + return 1; + puts("PASS: extracted production DPTX HPD request: short, oversized, wrong-echo and complete replies, legacy and AFK-v2"); + return 0; +} +''' +unit = (prefix + command.group(0) + "\n" + function(afk, "afk_service_call_timeout_len") + "\n" + + function(afk, "afk_service_call_timeout") + "\n" + + function(dptx, "dptxport_set_hpd_timeout") + suffix) +pathlib.Path(sys.argv[3]).write_text(unit) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh b/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh index 4348eebc78246..5351b80edbcc5 100755 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh @@ -53,6 +53,16 @@ $afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ "$afkv2_tmp/service.c" -o "$afkv2_tmp/service" "$afkv2_tmp/service" +# The DPTX HPD request: only a complete reply carries the HPD echo. +"$afkv2_python" "$afkv2_dir/extract-hpd.py" \ + "$afkv2_root/drivers/gpu/drm/apple/afk.c" \ + "$afkv2_root/drivers/gpu/drm/apple/dptxep.c" "$afkv2_tmp/hpd.c" +# shellcheck disable=SC2086 +$afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_tmp/hpd.c" -o "$afkv2_tmp/hpd" +"$afkv2_tmp/hpd" + "$afkv2_python" "$afkv2_dir/extract-send.py" \ "$afkv2_root/drivers/gpu/drm/apple/afk.c" "$afkv2_tmp/send.c" # shellcheck disable=SC2086 From 920373b8f60dd4ca18eaab1cd5d9fd2e734ac3d6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 15:09:06 +1100 Subject: [PATCH 1444/2680] drm/apple: Withdraw a Type-C DPTX link that never comes up When the external H17P DCP connects a Type-C DisplayPort sink and no link configuration arrives within two seconds, dcp_dptx_connect_raw() releases the display and returns -ETIMEDOUT. dcp_external_link_locked() then latches that error in external_error, which keeps the external session blocked: every later connect, stop, route deactivation and SAFE transition of the Type-C route returns it until the driver is rebound. The latch exists so that a firmware request whose reply may still arrive is never raced by a retry. A missing link configuration is not such a request: every request of the attempt has been answered, and the sink went away while the firmware trained it, as when a dock is pulled or bounces while it is being plugged in: DPTXPort: firmware reports link fault 22 on target 0:0 (x5) dcp_dptx_connect: timed out waiting for port 0 link configuration sn201202x 1-0c: failed to place Type-C mux in safe mode: -110 (every later cable event, until reboot) Withdraw such an attempt the way the external teardown does: drop HPD, then release the display. When the firmware acknowledges both with complete replies, nothing of the attempt is outstanding, so mark the port released, report the attempt as retryable and do not latch it; the next cable or HPD event connects again. If either request fails, the firmware state is unknown: the port stays marked as holding the display and the error is latched as before. Other DCPs keep their behaviour: they only release the display, and their reconnect worker retries as it did. Log the outcome either way, so that a capture shows whether the withdrawal ran and what the firmware answered. Fixes: 35f4ccf26053 ("drm/apple: add bounded external stop qualification path") (cherry picked from commit 566ff3adbc7c873817bdd6d9968939e5e099bbe1) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 58 ++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 55bd178496315..5963621ec7cf4 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2488,13 +2488,19 @@ bool dcp_has_typec_routes(struct platform_device *pdev) #define DPTX_RECONNECT_DELAY msecs_to_jiffies(1000) #define DPTX_RECONNECT_RETRIES 5 -static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *waiting_service) +/* + * Connect DPTX port @port. On failure, @retryable (if given) tells whether + * the attempt left nothing outstanding in the firmware, so that a later + * connection may simply try again: the service was not published yet, or a + * link that never came up was withdrawn and the firmware acknowledged that. + */ +static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable) { unsigned long timeout; - int ret = 0; + int ret = 0, err; - if (waiting_service) - *waiting_service = false; + if (retryable) + *retryable = false; if (!dcp->phy) { dev_warn(dcp->dev, "dcp_dptx_connect: missing phy\n"); return -ENODEV; @@ -2511,8 +2517,8 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *waiting_s dev_warn(dcp->dev, "dcp_dptx_connect: dptx service for port %d not enabled\n", port); ret = -ENODEV; /* No firmware request has been made; the first publication can retry. */ - if (waiting_service && !READ_ONCE(dcp->dptxport[port].closed)) { - *waiting_service = true; + if (retryable && !READ_ONCE(dcp->dptxport[port].closed)) { + *retryable = true; ret = -EAGAIN; } goto out_unlock; @@ -2577,7 +2583,7 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *waiting_s "dcp_dptx_connect: timed out waiting for port %u link configuration\n", port); ret = -ETIMEDOUT; - goto out_disconnect; + goto out_withdraw; } dev_dbg(dcp->dev, "dcp_dptx_connect: waited %d ms for link\n", @@ -2593,9 +2599,39 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *waiting_s return 0; -out_disconnect: +out_withdraw: mutex_lock(&dcp->hpd_mutex); + if (!dcp_is_external_h17p(dcp)) { + dcp->dptxport[port].connected = false; + goto out_release; + } + /* + * No link configuration arrived, typically because the sink went away + * while the firmware trained it. Withdraw the request the way a + * disconnect does, HPD first. Only once the firmware has acknowledged + * both is nothing of this attempt outstanding; until then the port + * stays held. + */ + err = -ENODEV; + if (dptx_service_available(&dcp->dptxport[port])) { + err = dcp_is_typec_output(dcp) ? + dptxport_set_hpd(dcp->dptxport[port].service, false) : 0; + if (!err) + err = dptxport_release_display(dcp->dptxport[port].service); + } + if (err) { + dev_err(dcp->dev, + "dcp_dptx_connect: port %u not withdrawn, still held: %d\n", + port, err); + goto out_unlock; + } + dev_info(dcp->dev, "dcp_dptx_connect: port %u withdrawn, the next connection retries\n", + port); dcp->dptxport[port].connected = false; + if (retryable) + *retryable = true; + goto out_unlock; + out_release: dptxport_release_display(dcp->dptxport[port].service); @@ -2631,7 +2667,7 @@ static void dcp_external_publish_generation(struct apple_dcp *dcp) /* external_lock is never taken by the serialized IOMFB receive callbacks. */ static int dcp_external_link_locked(struct apple_dcp *dcp) { - bool waiting_service; + bool retryable; int ret; lockdep_assert_held(&dcp->external_lock); @@ -2641,8 +2677,8 @@ static int dcp_external_link_locked(struct apple_dcp *dcp) return dcp->external_error; if (!dcp->external_cable) return -ENODEV; - ret = dcp_dptx_connect_raw(dcp, 0, &waiting_service); - if (ret && !waiting_service) + ret = dcp_dptx_connect_raw(dcp, 0, &retryable); + if (ret && !retryable) dcp->external_error = ret; return ret; } From 33c4e5a8aad1dac344f5abce99ce638a01fdaea4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 18:18:56 +1100 Subject: [PATCH 1445/2680] selftests: apple-dcp: Replay a fast dock replug that loses the DPTX link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the sequence of a dock pulled while its display link trains, with the actual external-session functions of drm/apple and a firmware model whose link configuration never arrives while the sink is gone, and whose requests can fail, be refused or go unanswered: - withdrawn-link: the route comes back after the plug-time bounce, the link times out and is withdrawn, the next connect tries again, the unplug tears the session down and the next plug connects. - unacknowledged-withdrawal: HPD low or the release times out, or HPD low gets an incomplete reply. The session stays held and latched. - unanswered-request: validation, connection, the display request or HPD assertion times out, with or without a failed cleanup, on a session whose pipe never powered or was stopped before. The port no longer reads connected, yet the failure stays latched across the unplug and the next plug is refused: the firmware may still carry the request out. - refused-request: a display request or HPD assertion the firmware refuses stays latched, as before. - live-failure: a powered pipe whose power-off failed holds everything. quiesce-before-safe gains withdrawn-still-latched: with a failure latched and the port reading released, a SAFE transition is still refused and never reaches the Type-C PHY. [ryan: adapted to 7.2.10 — context only: the quiesce-before-safe case list also holds the #332 cases, as on aurora-wip] (cherry picked from commit 97f64ff057dcd611611eab761979090670610079) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/gpu/apple-dcp/README | 4 +- .../drivers/gpu/apple-dcp/lifecycle.py | 169 ++++++++++++++++++ .../drivers/gpu/apple-dcp/quiesce.py | 2 + .../selftests/drivers/gpu/apple-dcp/run.py | 4 +- 4 files changed, 177 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/README b/tools/testing/selftests/drivers/gpu/apple-dcp/README index b04b6b9934193..1f5e3cc8cd68d 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/README +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/README @@ -29,7 +29,9 @@ The modules contain public synthetic host boundaries and retained assertions: faulted ACK/BIST cleanup, dirty-route retry, OFF/new-calibration epochs, warm reset and legacy T8103/T8122/T8140/T8142 route controls. * lifecycle: actual physical-disconnect authority and ownership despite failed - teardown; DWC3/core/PHY PM and reset-reference recovery; exact firmware + teardown; a fast dock replug whose DPTX link never configures (withdrawn, + unacknowledged, unanswered, refused and live failures); DWC3/core/PHY PM + and reset-reference recovery; exact firmware tuple/resource admission, unsupported VRR rejection and power reply bounds; 15,552 D134 dispatcher scope/length/context and legacy reply cases. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index d5b56fe489490..e21ca96e971df 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -366,6 +366,175 @@ def function(file, name): source = prefix + pm + shim + funcs + tests return source +def bounce(root): + """Fast dock replug: the sink vanishes while the DPTX link trains. + + The physical-disconnect fixture with the actual dcp_dptx_connect_raw() + and a firmware model whose link configuration may never arrive and + whose requests may fail, be refused or go unanswered. + """ + unit = disconnect(root) + dcp = (root / 'drivers/gpu/drm/apple/dcp.c').read_text() + m = re.search(r'^static int dcp_dptx_connect_raw\(', dcp, re.M) + assert m, 'dcp_dptx_connect_raw' + raw = block(dcp, m.start()) + + def swap(old, new): + nonlocal unit + assert unit.count(old) == 1, old + unit = unit.replace(old, new) + + swap(' struct { bool connected; void *service; } dptxport[1];', + ' struct { bool connected, closed; void *service, *atcphy; int linkcfg_completion; } dptxport[1];\n' + ' void *phy, *dev, *active_typec_route;\n' + ' int dptx_die, dptx_phy, dptx_dfp_port, connector_type;') + old_raw = unit[unit.index('static int dcp_dptx_connect_raw('):] + old_raw = old_raw[:old_raw.index('return raw_error; }') + len('return raw_error; }')] + swap(old_raw, 'static int dcp_dptx_connect_raw(struct apple_dcp *, u32, bool *);') + swap('static int dptxport_set_hpd(void *p, bool on)\n{ (void)p; assert(!on); return hpd_error; }', + 'static bool fw_hpd, fw_display, link_trains;\n' + 'static int hpd_drops, connects, release_error_once, assert_error;\n' + '/* A request that times out may still be carried out; a refused one is not. */\n' + 'static int dptxport_set_hpd(void *p, bool on)\n' + '{ (void)p; if (on) { fw_hpd |= assert_error != -EINVAL; return assert_error; }\n' + ' hpd_drops++; if (hpd_error) return hpd_error; fw_hpd = false; return 0; }') + swap('static int dptxport_release_display(void *p)\n{ (void)p; releases++; return release_error; }', + 'static int dptxport_release_display(void *p)\n' + '{ (void)p; releases++; if (release_error) return release_error;\n' + ' if (release_error_once) { int r = release_error_once; release_error_once = 0; return r; }\n' + ' fw_display = false; return 0; }') + swap('int main(void)\n{', 'static int __attribute__((unused)) prior_main(void)\n{') + firmware = r''' +typedef uint8_t u8; +#define dev_dbg(...) ((void)0) +#define dev_info(...) ((void)0) +#define DRM_MODE_CONNECTOR_DisplayPort 10 +#define DPTX_CONNECT_TIMEOUT 2000 +#define DPTX_TUNNEL_CONNECT_TIMEOUT 8000 +static void mutex_lock(int *m) { (void)m; } +static void mutex_unlock(int *m) { (void)m; } +static bool dcp_is_usb4_output(struct apple_dcp *d) { (void)d; return false; } +static bool apple_dp_tunnel_t602x(void) { return false; } +static void reinit_completion(int *c) { *c = 0; } +/* The firmware configures the link only while a sink answers it. */ +static unsigned long wait_for_completion_timeout(int *c, unsigned long t) +{ if (link_trains) *c = 1; return *c ? t : 0; } +static unsigned jiffies_to_msecs(unsigned long t) { return t; } +static void usleep_range(int a, int b) { (void)a; (void)b; } +static void av_service_connect(struct apple_dcp *d) { (void)d; } +static int validate_error, connect_error, request_error; +static int dptxport_validate_connection(void *s, int core, int atc, int die) +{ (void)s; (void)core; (void)atc; (void)die; return validate_error; } +static int dptxport_connect(void *s, int core, int atc, int die, bool typec) +{ (void)s; (void)core; (void)atc; (void)die; (void)typec; connects++; return connect_error; } +static int dptxport_request_display(void *s) +{ (void)s; fw_display |= request_error != -EINVAL; return request_error; } +static int dptxport_set_hpd_timeout(void *s, bool on, int t) { (void)t; return dptxport_set_hpd(s, on); } +''' + tests = r''' +static struct platform_device device = { .data = &display }; +/* The session the dock leaves behind after its plug-time route bounce. */ +static void bounced(void) +{ + init(); display.phy = (void *)1; fw_hpd = fw_display = true; + release_error_once = 0; link_trains = true; + CHECK(dcp_external_disconnect(&display, true) == 0); + CHECK(display.external_stopped && !display.dptxport[0].connected && !display.external_error); + connects = hpd_drops = 0; + validate_error = connect_error = request_error = assert_error = 0; +} +/* The failure stays latched across the unplug, and the next plug is refused. */ +static void stays_latched(int error) +{ + int before; + + validate_error = connect_error = request_error = assert_error = 0; + hpd_error = release_error = 0; + CHECK(display.external_error == error); + CHECK(dcp_external_disconnect(&display, true) == error); + CHECK(display.external_error == error); + before = connects; link_trains = true; + CHECK(dcp_dptx_connect_oob(&device, 0) == error && connects == before); +} +int main(void) +{ + /* The route comes back while the sink no longer answers: withdrawn, not latched. */ + bounced(); link_trains = false; + CHECK(dcp_dptx_connect_oob(&device, 0) == -ETIMEDOUT); + CHECK(!display.external_error && !display.dptxport[0].connected); + CHECK(!fw_hpd && !fw_display && hpd_drops == 1); + /* tipd's own OOB connect after the mux tries again rather than refusing. */ + CHECK(dcp_dptx_connect_oob(&device, 0) == -ETIMEDOUT && connects == 2); + /* The unplug's teardown (OOB disconnect, then SAFE) succeeds. */ + CHECK(dcp_external_disconnect(&display, true) == 0 && !display.external_error); + /* The next plug connects: the display recovers without a reboot. */ + link_trains = true; + CHECK(dcp_dptx_connect_oob(&device, 0) == 0); + CHECK(display.dptxport[0].connected && fw_hpd && fw_display && connects == 3); + printf("case withdrawn-link complete\n"); + + /* + * The firmware does not acknowledge the withdrawal: HPD low or the release + * times out, or HPD low gets an incomplete reply. The session stays held. + */ + for (int which = 0; which < 3; which++) { + bounced(); link_trains = false; + if (which == 0) hpd_error = -ETIMEDOUT; + if (which == 1) release_error_once = -ETIMEDOUT; + if (which == 2) hpd_error = -EPROTO; + CHECK(dcp_dptx_connect_oob(&device, 0) == -ETIMEDOUT); + CHECK(display.external_error == -ETIMEDOUT && display.dptxport[0].connected); + stays_latched(-ETIMEDOUT); + } + printf("case unacknowledged-withdrawal complete\n"); + + /* + * A request goes unanswered, so the firmware may still carry it out, with + * its cleanup failing as well or not, on a session that never powered its + * pipe or that stopped it before. A port no longer marked connected is no + * proof that the firmware let go of it: the failure stays latched. + */ + for (int fault = 0; fault < 4; fault++) + for (int cleanup = 0; cleanup < 2; cleanup++) + for (int stopped = 0; stopped < 2; stopped++) { + bounced(); + if (!stopped) + display.external_power_attempted = display.external_stopped = false; + if (fault == 0) validate_error = -ETIMEDOUT; + if (fault == 1) connect_error = -ETIMEDOUT; + if (fault == 2) request_error = -ETIMEDOUT; + if (fault == 3) assert_error = -ETIMEDOUT; + if (cleanup) release_error = -ETIMEDOUT; + CHECK(dcp_dptx_connect_oob(&device, 0) == -ETIMEDOUT); + stays_latched(-ETIMEDOUT); + } + printf("case unanswered-request complete\n"); + + /* A request the firmware refuses is latched as before, released display or not. */ + for (int fault = 0; fault < 2; fault++) { + bounced(); + if (fault == 0) request_error = -EINVAL; + if (fault == 1) assert_error = -EINVAL; + CHECK(dcp_dptx_connect_oob(&device, 0) == -EINVAL); + CHECK(!display.dptxport[0].connected); + stays_latched(-EINVAL); + } + printf("case refused-request complete\n"); + + /* A live failure (powered pipe never acknowledged off) still holds everything. */ + init(); display.phy = (void *)1; power_error = -ETIMEDOUT; connects = 0; + CHECK(dcp_external_disconnect(&display, true) == -ETIMEDOUT); + power_error = 0; + CHECK(dcp_external_disconnect(&display, true) == -ETIMEDOUT); + CHECK(display.external_error == -ETIMEDOUT && display.pipe_enabled_h17p); + CHECK(dcp_dptx_connect_oob(&device, 0) == -ETIMEDOUT && connects == 0); + printf("case live-failure complete\n"); + + printf("%d assertions, %d failures\n", checks, failures); return !!failures; +} +''' + return unit + firmware + raw + '\n' + tests + def teardown(root): files = {n: (root / f).read_text() for n, f in { diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py index fdab2d0b2cdac..af56c57419131 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py @@ -257,6 +257,8 @@ def make_unit(source, parent=False, reference=None): if(argv[1][0]=='m') {power_error=-EIO;assert(typec_mux_set(&mux,&safe)==-EIO&&phy_calls==0&&port.owner==&route);} if(argv[1][0]=='d') {assert(typec_mux_prepare(&mux,&safe)==0&&stops==1&&phy_calls==0&&port.owner==&route);assert(typec_mux_prepare(&mux,&safe)==0&&stops==1&&port.owner==&route);} if(argv[1][0]=='e') {display.external_error=-EIO;port.hpd=false;display.typec_cable_connected=false;connector.connected=false;assert(typec_mux_set(&mux,&safe)==-EIO&&phy_calls==0&&port.owner==&route);} + /* A latched failure is no less live once the port reads released: SAFE stays refused. */ + if(argv[1][0]=='w') {display.external_error=-ETIMEDOUT;display.dptxport[0].connected=false;display.external_stopped=true;port.hpd=false;assert(typec_mux_set(&mux,&safe)==-ETIMEDOUT&&phy_calls==0&&port.owner==&route&&deactivate_calls==0);} if(argv[1][0]=='l') {display.external=false;assert(typec_mux_prepare(&mux,&safe)==0&&stops==1&&port.safe_pending&&port.owner==&route);} if(argv[1][0]=='f') {assert(typec_mux_prepare(&mux,&safe)==0&&port.safe_pending);dcp_typec_rebalance_locked(NULL,0);assert(port.owner==&route&&route.selected&&deactivate_calls==0);port.safe_pending=false;dcp_typec_rebalance_locked(NULL,0);assert(port.owner==NULL&&deactivate_calls==1);} if(argv[1][0]=='t') {route.tunnel=true;assert(typec_mux_prepare(&mux,&safe)==0&&stops==0&&port.dp_wanted);} diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py index 9b697de8d2a12..bf24520697696 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -66,7 +66,7 @@ def main(): ('phy', 'usb-unreachable', lambda r: pm.ownership(r, True), ((),), ()), ('lifecycle', 'cpuidle-s2idle', idle.s2idle, ((),), ('-Wno-unused-parameter', '-Wno-unused-variable')), ('lifecycle', 'typec-probe', pm.probe, ((),), ()), - ('lifecycle', 'quiesce-before-safe', quiesce.make_unit, tuple((c,) for c in ('ordered','power','ack','role','bolt','mux','duplicate','error-latched','legacy','fabric','tunnel','generic-disconnect','ordinary-remove','ordinary-disconnect','ordinary-role-error','ordinary-phy-error','ordinary-role-reprobe','ordinary-unsupported-remove','ordinary-unsupported-disconnect','ordinary-unregistered-remove','ordinary-unregistered-disconnect','noop-fixed-remove','noop-fixed-disconnect')), ()), + ('lifecycle', 'quiesce-before-safe', quiesce.make_unit, tuple((c,) for c in ('ordered','power','ack','role','bolt','mux','duplicate','error-latched','withdrawn-still-latched','legacy','fabric','tunnel','generic-disconnect','ordinary-remove','ordinary-disconnect','ordinary-role-error','ordinary-phy-error','ordinary-role-reprobe','ordinary-unsupported-remove','ordinary-unsupported-disconnect','ordinary-unregistered-remove','ordinary-unregistered-disconnect','noop-fixed-remove','noop-fixed-disconnect')), ()), ('lifecycle', 'probe-mux-pm-dispatch', pm.probe_dispatch, ((),), ()), ('phy', 'parking', lambda r: parking_unit(r), parking_cases, ()), ('phy', 'adverse-retry', phy.adverse, tuple((c,) for c in ('bist', 'init', 'mux', 'gate', 'dp-cache', 'off', 'success')), ()), @@ -78,6 +78,8 @@ def main(): ('phy', 'aux', phy.aux, ((),), ()), ('lifecycle', 'physical-disconnect', lifecycle.disconnect, ((),), ('-Wno-unused-variable', '-Wno-unused-function', '-Wno-unused-but-set-variable')), + ('lifecycle', 'fast-replug-bounce', lifecycle.bounce, ((),), + ('-Wno-unused-variable', '-Wno-unused-function', '-Wno-unused-but-set-variable')), ('lifecycle', 'dwc3-teardown', lifecycle.teardown, (('all',),), ()), ('lifecycle', 'resource-profile', lifecycle.profile, ((),), ('-Wno-sign-compare',)), ('lifecycle', 'firmware-provider', lifecycle.firmware, ((),), ()), From 903e8ff6b5c64b92a8fa2132ace6abecb34d83d1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 18:36:04 +1100 Subject: [PATCH 1446/2680] drm/apple: Say which step of an external display stop failed dcp_external_stop_locked() keeps a session whose teardown failed blocked and owned, and every later SAFE transition of its Type-C route then returns the stored error. The failure itself is silent: the first line in the log is tipd's "failed to place Type-C mux in safe mode", which does not tell a queue that never settled from a pipe power-off or a DPTX withdrawal the firmware did not acknowledge. Name the step that failed, once, when the session is latched. (cherry picked from commit 8f2ae6f50b5cf1af90e13d8f7dfe584edecd7361) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 5963621ec7cf4..561a4d9686a98 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2718,6 +2718,7 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) /* The generation token belongs to the CRTC that was actually enabled. */ static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation) { + const char *step; int ret; lockdep_assert_held(&dcp->external_lock); @@ -2732,11 +2733,16 @@ static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation) /* Route setup can call this before IOMFB or its shared memory exists. */ if (dcp->external_stopped || !dcp->external_power_attempted) goto release_link; + step = "settling the queue"; ret = iomfb_queue_settle(dcp, msecs_to_jiffies(500)); - if (!ret) + if (!ret) { + step = "powering the pipe off"; ret = iomfb_external_power_h17p(dcp, false); - if (!ret) + } + if (!ret) { + step = "settling the queue after power-off"; ret = iomfb_queue_settle(dcp, msecs_to_jiffies(500)); + } if (ret) goto failed; /* This is a transport acknowledgment, not permission to free scanout. */ @@ -2744,6 +2750,7 @@ static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation) release_link: scoped_guard(mutex, &dcp->hpd_mutex) { if (dcp->dptxport[0].connected) { + step = "withdrawing the DPTX link"; if (!dptx_service_available(&dcp->dptxport[0])) { ret = -ENODEV; goto failed; @@ -2763,6 +2770,7 @@ static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation) return 0; failed: /* A retry could race a late reply. Keep this session blocked and owned. */ + dev_err(dcp->dev, "external display stop failed %s, session held: %d\n", step, ret); dcp->external_error = ret; return ret; } From 07858f2100f0e8c308f1d6f3f47a5057384b1992 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:09:43 +1100 Subject: [PATCH 1447/2680] drm/apple: Record an external display announced while its pipe is off MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While its pipe is stopped, an external H17P display passes no hotplug to DRM: dcp_external_hotplug() parks the callback until the pipe starts again, so that the unplug a DPMS-off stop causes does not tear down the connector. It parks the announcement the firmware sends once it has read a returning display as well, before dcp_firmware_metadata_hotplug() can record that announcement. That is the case on resume from system sleep. Suspend stops the pipe and advances the cable generation, and the firmware announces the display again after resume while the CRTC is still off. The modes that need an announcement for the current generation, such as 1080p at 180 Hz on a two-lane HBR3 link, therefore stay unavailable until userspace enables the CRTC at some other mode and the firmware announces the display a second time. Record an announcement while the pipe is stopped, without notifying the connector, as for every hotplug parked there. Accept it only once the link of the current cable generation has been requested again: a callback that arrives before that belongs to a link already released, whatever the generation counters say by then. An unplug is still not recorded, so a DPMS-off stop keeps the link proof of the mode that DPMS on restores. The apple-dcp lifecycle selftest copies dcp_external_hotplug() into its model of the driver, so give that model the new call as well. [ryan: adapted to 7.2.10 — context only: next to 200-dcp's parser_mode_test.o in the Makefile] (cherry picked from commit d6935850b08e560d9f782d6ee6146ef9d56042e3) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 46 +++++- .../gpu/drm/apple/dcp_external_resume_test.c | 141 ++++++++++++++++++ .../drivers/gpu/apple-dcp/lifecycle.py | 1 + 5 files changed, 182 insertions(+), 8 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp_external_resume_test.c diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 66d11e865ec6c..79c6d920c7c0e 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -15,6 +15,7 @@ appledrm-y += iomfb_h17p.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += iomfb_h17p_test.o parser_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple_resume_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += parser_mode_test.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_resume_test.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 314e9861f756c..2989258121fc1 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -498,6 +498,7 @@ struct apple_dcp { void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, bool connected); void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected); +void dcp_firmware_metadata_announced(struct apple_dcp *dcp); void dcp_firmware_metadata_begin(struct apple_dcp *dcp); void dcp_firmware_metadata_attributes(struct apple_dcp *dcp, const struct dcp_display_metadata *attributes); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 561a4d9686a98..f8d43d13e8e9d 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1963,6 +1963,41 @@ void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, boo spin_unlock_irqrestore(&dcp->metadata.lock, flags); } +static void dcp_firmware_metadata_present_locked(struct apple_dcp *dcp) +{ + lockdep_assert_held(&dcp->metadata.lock); + dcp->metadata.hpd_seen = true; + /* + * Fresh proof may precede HPD assertion after a same-cable loss. + * Keep invalidated firmware metadata and old receipt tickets revoked. + */ + if (READ_ONCE(dcp->typec_cable_connected) && + dcp->metadata.epoch.generation == READ_ONCE(dcp->typec_generation)) + dcp->metadata.epoch.connected = true; +} + +/* + * While its pipe is stopped an external display reports no hotplug to DRM + * (see dcp_external_hotplug()). When the firmware announces the display + * there, as when the link returns from system sleep with the CRTC off, the + * announcement still holds for the modes that need it, so record it. The + * connector is not notified, just as for an announcement on DPMS wake. + * + * Only a link requested for the current cable generation can be announced: + * a callback that arrives before that belongs to a link already released. + */ +void dcp_firmware_metadata_announced(struct apple_dcp *dcp) +{ + unsigned long flags; + + if (!dcp_firmware_metadata_enabled(dcp) || + !READ_ONCE(dcp->dptxport[0].connected)) + return; + spin_lock_irqsave(&dcp->metadata.lock, flags); + dcp_firmware_metadata_present_locked(dcp); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); +} + void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected) { struct apple_connector *con = READ_ONCE(dcp->connector); @@ -1972,14 +2007,7 @@ void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected) return; spin_lock_irqsave(&dcp->metadata.lock, flags); if (connected) { - dcp->metadata.hpd_seen = true; - /* - * Fresh proof may precede HPD assertion after a same-cable loss. - * Keep invalidated firmware metadata and old receipt tickets revoked. - */ - if (READ_ONCE(dcp->typec_cable_connected) && - dcp->metadata.epoch.generation == READ_ONCE(dcp->typec_generation)) - dcp->metadata.epoch.connected = true; + dcp_firmware_metadata_present_locked(dcp); } else if (dcp->metadata.hpd_seen) { dcp->metadata.hpd_seen = false; dcp->rate_proof.connection_sequence++; @@ -2834,6 +2862,8 @@ bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected) if (dcp->iomfb.presentation_paused) { dcp->external_hotplug_pending = true; dcp->external_hotplug_connected = connected; + if (connected) + dcp_firmware_metadata_announced(dcp); /* Publish initial/new-cable discovery, never same-cable DPMS wake. */ if (connected && READ_ONCE(dcp->external_cable) && !READ_ONCE(dcp->external_error) && diff --git a/drivers/gpu/drm/apple/dcp_external_resume_test.c b/drivers/gpu/drm/apple/dcp_external_resume_test.c new file mode 100644 index 0000000000000..409a8cd8ffd8b --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_external_resume_test.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include +#include + +#include "dcp-internal.h" + +/* + * An external H17P DCP back from system sleep on the same cable: suspend + * stopped its pipe and advanced the cable generation, and the CRTC is off. + */ +struct external_resume_test { + struct apple_dcp *dcp; +}; + +/* What the firmware hotplug callback does, under the same lock. */ +static bool external_resume_hotplug(struct apple_dcp *dcp, bool connected) +{ + guard(mutex)(&dcp->iomfb.lock); + return dcp_external_hotplug(dcp, connected); +} + +static struct external_resume_test *external_resume_init(struct kunit *test) +{ + struct external_resume_test *priv; + struct apple_dcp *dcp; + struct device *dev; + + priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, priv); + dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dcp); + dev = kunit_device_register(test, "dcp-external-resume"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + dcp->dev = dev; + dcp->hw.iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL; + spin_lock_init(&dcp->metadata.lock); + mutex_init(&dcp->iomfb.lock); + + /* Connected and announced before the sleep. */ + dcp->typec_cable_connected = true; + dcp->typec_generation = 1; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, true); + dcp_firmware_metadata_hotplug(dcp, true); + dcp->external_cable = true; + dcp->external_power_attempted = true; + dcp->external_generation = 1; + dcp->external_rx_generation = 1; + dcp->external_crtc_generation = 1; + + /* Suspend: the pipe stopped, then the cable generation advanced. */ + dcp->external_stopped = true; + dcp->iomfb.presentation_paused = true; + dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); + /* Resume: the reconnect work requested the link again. */ + dcp->dptxport[0].connected = true; + + priv->dcp = dcp; + + return priv; +} + +/* The announcement after the link returns must count with the pipe stopped. */ +static void external_paused_announce_recorded_test(struct kunit *test) +{ + struct external_resume_test *priv = external_resume_init(test); + struct apple_dcp *dcp = priv->dcp; + + KUNIT_ASSERT_FALSE(test, dcp->metadata.epoch.connected); + /* A stopped pipe consumes the callback. */ + KUNIT_EXPECT_TRUE(test, external_resume_hotplug(dcp, true)); + + KUNIT_EXPECT_TRUE(test, dcp->metadata.epoch.connected); + KUNIT_EXPECT_EQ(test, dcp->metadata.epoch.generation, dcp->typec_generation); + /* It is replayed to DRM only when the pipe starts again. */ + KUNIT_EXPECT_TRUE(test, dcp->external_hotplug_pending); + KUNIT_EXPECT_TRUE(test, dcp->external_hotplug_connected); +} + +/* The unplug a DPMS-off stop causes must not revoke the link proof. */ +static void external_paused_unplug_ignored_test(struct kunit *test) +{ + struct external_resume_test *priv = external_resume_init(test); + struct apple_dcp *dcp = priv->dcp; + u64 connection_sequence; + + KUNIT_EXPECT_TRUE(test, external_resume_hotplug(dcp, true)); + connection_sequence = dcp->rate_proof.connection_sequence; + KUNIT_EXPECT_TRUE(test, external_resume_hotplug(dcp, false)); + + KUNIT_EXPECT_TRUE(test, dcp->metadata.epoch.connected); + KUNIT_EXPECT_EQ(test, dcp->rate_proof.connection_sequence, connection_sequence); +} + +/* An announcement on an older cable generation is not current. */ +static void external_paused_stale_announce_test(struct kunit *test) +{ + struct external_resume_test *priv = external_resume_init(test); + struct apple_dcp *dcp = priv->dcp; + + dcp->typec_generation++; + KUNIT_EXPECT_TRUE(test, external_resume_hotplug(dcp, true)); + + KUNIT_EXPECT_FALSE(test, dcp->metadata.epoch.connected); +} + +/* + * A callback delayed past a release of the link is not an announcement of + * the next one, even once software has moved both generations on. + */ +static void external_paused_released_link_test(struct kunit *test) +{ + struct external_resume_test *priv = external_resume_init(test); + struct apple_dcp *dcp = priv->dcp; + + dcp->dptxport[0].connected = false; + dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); + KUNIT_EXPECT_TRUE(test, external_resume_hotplug(dcp, true)); + + KUNIT_EXPECT_FALSE(test, dcp->metadata.epoch.connected); +} + +static struct kunit_case external_resume_cases[] = { + KUNIT_CASE(external_paused_announce_recorded_test), + KUNIT_CASE(external_paused_unplug_ignored_test), + KUNIT_CASE(external_paused_stale_announce_test), + KUNIT_CASE(external_paused_released_link_test), + {} +}; + +static struct kunit_suite external_resume_suite = { + .name = "apple-dcp-external-resume", + .test_cases = external_resume_cases, +}; + +kunit_test_suite(external_resume_suite); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index e21ca96e971df..030c0c34ff575 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -139,6 +139,7 @@ def function(file, name): static void dcp_firmware_metadata_hotplug(struct apple_dcp *d, bool c) { (void)d; (void)c; } static void dcp_firmware_metadata_connection(struct apple_dcp *d, u64 g, bool c) { (void)d; (void)g; (void)c; } +static void dcp_firmware_metadata_announced(struct apple_dcp *d) { (void)d; } static int dcp_external_disconnect(struct apple_dcp *, bool); static struct apple_dcp display; static void drm_connector_oob_hotplug_event(void *node, int status) From 17db00a7289000b9e600e46204480c99d126bcea Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:31:23 +1100 Subject: [PATCH 1448/2680] drm/apple: Tell when an external display's modes are in for its cable The modes of an external H17P display reach the driver in pieces after its link comes up: the TimingElements catalogue, the AV service its EDID is read from, the firmware's announcement of the display and, for the 1080p 180 Hz exception mode, the link rate and lane count DCP configured. They arrive in no fixed order. A probe made before the last of them lists an incomplete set of modes, and userspace picks a fallback mode from it. Nothing can tell yet whether they are in for the current cable. In particular a disconnect revokes the link and announcement facts but keeps the catalogue, which only the next TimingElements replaces, so a non-empty catalogue can still be the previous connection's. Record the cable generation and connection sequence of the receipt that filled the catalogue, as the exception mode's own ticket already does, and add dcp_external_modes_ready(): the catalogue is the current connection's, its EDID service is up, and an exception mode in it passes every check of dcp_180hz_mode_current() except the EDID, which only the probe reads. That needs the existing check split into its link and its EDID parts; the 180 Hz admission itself is unchanged. Callers follow in later patches. The KUnit suite feeds the facts in each order, replugs with the previous catalogue retained, advances the generation as system sleep does and drops the downstream link, and checks that the 180 Hz mode is admitted exactly when the predicate says the connection is ready. (cherry picked from commit 722fe8472504934b140d98a20a077130bd94cfef) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/dcp-180hz.c | 103 ++++- drivers/gpu/drm/apple/dcp-180hz.h | 1 + drivers/gpu/drm/apple/dcp-internal.h | 3 + .../gpu/drm/apple/dcp_external_ready_test.c | 431 ++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 6 + 6 files changed, 526 insertions(+), 19 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp_external_ready_test.c diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 79c6d920c7c0e..06927b4ad4e6a 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -16,6 +16,7 @@ appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += iomfb_h17p_test.o parser_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple_resume_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += parser_mode_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_resume_test.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_ready_test.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/dcp-180hz.c b/drivers/gpu/drm/apple/dcp-180hz.c index 8eb4166aa6dbb..09bbc5bd20ad7 100644 --- a/drivers/gpu/drm/apple/dcp-180hz.c +++ b/drivers/gpu/drm/apple/dcp-180hz.c @@ -146,34 +146,49 @@ void dcp_180hz_notify(struct apple_dcp *dcp) schedule_work(&con->hotplug_wq); } -bool dcp_180hz_mode_current(struct apple_dcp *dcp, const struct dcp_display_mode *mode) +/* + * Everything the exception mode needs except the EDID, which only the next + * probe reads: a catalogue received for the current cable and connection, + * the AV service of that receipt, an announced display and the configured + * link rate and lane count. + */ +static bool dcp_180hz_link_current(struct apple_dcp *dcp, const struct dcp_display_mode *mode) { struct apple_connector *con = dcp->connector; - const struct edid *raw; lockdep_assert_held(&dcp->hpd_mutex); lockdep_assert_held(&dcp->dcpavserv.lock); lockdep_assert_held(&dcp->metadata.lock); - if (!con || dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL || - !dcp_180hz_direct_port(dcp->dptxport[0].service) || - !mode->unsafe_rgb8_ticket.associated || - mode->unsafe_rgb8_ticket.generation != dcp->typec_generation || - mode->automatic_connection_sequence != dcp->rate_proof.connection_sequence || - mode->automatic_catalog_sequence != dcp->rate_proof.catalog_sequence || - !dcp->metadata.epoch.connected || !dcp->typec_cable_connected || dcp->quiescing || - READ_ONCE(con->dcp) != to_platform_device(dcp->dev) || + return con && dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && + dcp_180hz_direct_port(dcp->dptxport[0].service) && + mode->unsafe_rgb8_ticket.associated && + mode->unsafe_rgb8_ticket.generation == dcp->typec_generation && + mode->automatic_connection_sequence == dcp->rate_proof.connection_sequence && + mode->automatic_catalog_sequence == dcp->rate_proof.catalog_sequence && + dcp->metadata.epoch.connected && dcp->typec_cable_connected && !dcp->quiescing && + READ_ONCE(con->dcp) == to_platform_device(dcp->dev) && + dcp_180hz_timing_epoch(mode, dcp->dcpavserv.epoch) && + dcp->dptxport[0].connected && dcp->dptxport[0].enabled && !dcp->dptxport[0].closed && + dcp_180hz_fact_current(dcp, &dcp->rate_proof.rate) && + dcp_180hz_fact_current(dcp, &dcp->rate_proof.lanes) && + dcp_180hz_payload(dcp->rate_proof.rate.value, dcp->rate_proof.lanes.value, + mode->mode.clock) && + mode->sdr_rgb.depth == 8 && mode->sdr_rgb.format == DCP_COLOR_FORMAT_RGB && + mode->sdr_rgb.range == DCP_COLOR_YCBCR_RANGE_FULL && + mode->sdr_rgb.eotf == DCP_EOTF_SDR_GAMMA && + mode->sdr_rgb.colorimetry == DCP_COLORIMETRY_SRGB; +} + +bool dcp_180hz_mode_current(struct apple_dcp *dcp, const struct dcp_display_mode *mode) +{ + struct apple_connector *con = dcp->connector; + const struct edid *raw; + + if (!dcp_180hz_link_current(dcp, mode) || !dcpavserv_edid_service_current(dcp, con->drm_edid_service_epoch) || - !dcp_180hz_timing_epoch(mode, dcp->dcpavserv.epoch) || !con->drm_edid || con->drm_edid_firmware_derived || READ_ONCE(con->edid_invalidation_pending) || - con->drm_edid_generation != dcp->typec_generation || - !dcp->dptxport[0].connected || !dcp->dptxport[0].enabled || dcp->dptxport[0].closed || - !dcp_180hz_fact_current(dcp, &dcp->rate_proof.rate) || - !dcp_180hz_fact_current(dcp, &dcp->rate_proof.lanes) || - !dcp_180hz_payload(dcp->rate_proof.rate.value, dcp->rate_proof.lanes.value, mode->mode.clock) || - mode->sdr_rgb.depth != 8 || mode->sdr_rgb.format != DCP_COLOR_FORMAT_RGB || - mode->sdr_rgb.range != DCP_COLOR_YCBCR_RANGE_FULL || - mode->sdr_rgb.eotf != DCP_EOTF_SDR_GAMMA || mode->sdr_rgb.colorimetry != DCP_COLORIMETRY_SRGB) + con->drm_edid_generation != dcp->typec_generation) return false; raw = drm_edid_raw(con->drm_edid); /* The transport validates extent/checksums; this quirk accepts only its 256-byte form. */ @@ -181,6 +196,56 @@ bool dcp_180hz_mode_current(struct apple_dcp *dcp, const struct dcp_display_mode dcp_180hz_edid((const u8 *)raw, 256, &mode->mode); } +/* Caller holds metadata.lock. */ +static bool dcp_catalog_current(struct apple_dcp *dcp) +{ + const struct dcp_metadata_ticket *ticket = &dcp->metadata.catalog_ticket; + + return ticket->associated && dcp->nr_modes && + ticket->generation == dcp->typec_generation && + ticket->generation == dcp->metadata.epoch.generation && + dcp->metadata.catalog_connection_sequence == dcp->rate_proof.connection_sequence; +} + +/** + * dcp_external_modes_ready - whether a probe would list the current modes + * @dcp: an external H17P DCP + * + * The modes of an external display arrive in pieces, in no fixed order: the + * TimingElements catalogue, the AV service the EDID is read from, the + * firmware's announcement and the link configuration. A connection is + * ready once the catalogue is the current cable's, its EDID service is up + * and, if the catalogue holds an exception mode, that mode passes every + * check of dcp_180hz_mode_current() but the EDID the probe will read. A + * catalogue retained from an earlier connection never counts. + * + * Caller holds hpd_mutex. + */ +bool dcp_external_modes_ready(struct apple_dcp *dcp) +{ + bool ready, exception = false, proven = false; + unsigned long flags; + unsigned int i; + + lockdep_assert_held(&dcp->hpd_mutex); + guard(mutex)(&dcp->dcpavserv.lock); + if (!dcpavserv_edid_current(dcp, dcp->typec_generation) || + !dcpavserv_edid_service_current(dcp, dcp->dcpavserv.epoch)) + return false; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + ready = dcp_catalog_current(dcp); + for (i = 0; ready && i < dcp->nr_modes; i++) { + if (!dcp->modes[i].automatic_rgb8_quirk) + continue; + exception = true; + proven |= dcp_180hz_link_current(dcp, &dcp->modes[i]); + } + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + + return ready && (!exception || proven); +} + bool dcp_180hz_pending(struct apple_dcp *dcp, const struct dcp_display_mode *mode) { struct apple_connector *con = dcp->connector; diff --git a/drivers/gpu/drm/apple/dcp-180hz.h b/drivers/gpu/drm/apple/dcp-180hz.h index 4995dae4cf648..87e9258a6782b 100644 --- a/drivers/gpu/drm/apple/dcp-180hz.h +++ b/drivers/gpu/drm/apple/dcp-180hz.h @@ -28,6 +28,7 @@ void dcp_180hz_invalidate(struct apple_dcp *dcp); void dcp_180hz_notify(struct apple_dcp *dcp); void dcp_180hz_catalog_changed(struct apple_dcp *dcp); bool dcp_180hz_mode_current(struct apple_dcp *dcp, const struct dcp_display_mode *mode); +bool dcp_external_modes_ready(struct apple_dcp *dcp); bool dcp_180hz_pending(struct apple_dcp *dcp, const struct dcp_display_mode *mode); void dcp_180hz_retry(struct apple_dcp *dcp); int dcp_180hz_init(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 2989258121fc1..18ce4e77a49fc 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -304,6 +304,9 @@ struct apple_dcp { struct dcp_display_metadata attributes; struct drm_display_mode preferred; struct dcp_metadata_ticket attributes_ticket, timing_ticket; + /* The connection whose TimingElements filled the mode catalogue. */ + struct dcp_metadata_ticket catalog_ticket; + u64 catalog_connection_sequence; bool attributes_valid, timing_valid, hpd_seen; } metadata; diff --git a/drivers/gpu/drm/apple/dcp_external_ready_test.c b/drivers/gpu/drm/apple/dcp_external_ready_test.c new file mode 100644 index 0000000000000..80685ef9e0ad8 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_external_ready_test.c @@ -0,0 +1,431 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include +#include +#include + +#include + +#include "afk.h" +#include "connector.h" +#include "dcp-internal.h" +#include "parser.h" + +/* + * An external H17P DCP driving the QG270 S3 at 1920x1080 and 180 Hz over a + * direct two-lane HBR3 link: the facts its first probe needs arrive one by + * one, in any order. + */ +struct ready_test { + struct apple_dcp *dcp; + struct apple_connector *con; + struct apple_dcp_afkep dptx_ep; + struct apple_epic_service dptx; + struct apple_epic_service av; + struct dcp_display_mode modes[2]; +}; + +static const struct drm_display_mode ready_mode_180 = { + .clock = 414090, + .hdisplay = 1920, .hsync_start = 1968, .hsync_end = 2000, .htotal = 2080, + .vdisplay = 1080, .vsync_start = 1083, .vsync_end = 1088, .vtotal = 1106, + .flags = DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC, +}; + +static const struct drm_display_mode ready_mode_60 = { + .clock = 148500, + .hdisplay = 1920, .hsync_start = 2008, .hsync_end = 2052, .htotal = 2200, + .vdisplay = 1080, .vsync_start = 1084, .vsync_end = 1089, .vtotal = 1125, + .flags = DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC, +}; + +static void ready_checksum(u8 *block) +{ + u8 sum = 0; + int i; + + for (i = 0; i < 127; i++) + sum += block[i]; + block[127] = -sum; +} + +/* A 256-byte EDID that admits ready_mode_180. */ +static const struct drm_edid *ready_edid(struct kunit *test) +{ + static const u8 header[] = { 0, 255, 255, 255, 255, 255, 255, 0 }; + static const u8 dtd[] = { + 0xc1, 0xa1, 0x80, 0xa0, 0x70, 0x38, 0x1a, 0x40, 0x30, + 0x20, 0x35, 0x00, 0, 0, 0, 0, 0, 0x1a, + }; + static const u8 name[] = { + 0, 0, 0, 0xfc, 0, 'Q', 'G', '2', '7', '0', ' ', 'S', '3', '\n', + ' ', ' ', ' ', ' ', + }; + static const u8 hf_vsdb[] = { 0x67, 0xd8, 0x5d, 0xc4, 1, 0x54, 0x80, 0 }; + const struct drm_edid *edid; + u8 *p; + + p = kunit_kzalloc(test, 256, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, p); + memcpy(p, header, sizeof(header)); + p[8] = 0x04; + p[9] = 0x72; + p[10] = 0x4f; + p[11] = 0x0b; + p[18] = 1; + p[19] = 4; + p[20] = 0x80; + memcpy(p + 54, dtd, sizeof(dtd)); + memcpy(p + 72, name, sizeof(name)); + p[126] = 1; + ready_checksum(p); + p[128] = 2; + p[129] = 3; + p[130] = 4 + sizeof(hf_vsdb); + memcpy(p + 132, hf_vsdb, sizeof(hf_vsdb)); + ready_checksum(p + 128); + + edid = drm_edid_alloc(p, 256); + KUNIT_ASSERT_NOT_NULL(test, edid); + return edid; +} + +static void ready_free_edid(void *data) +{ + drm_edid_free(data); +} + +static struct ready_test *ready_init(struct kunit *test) +{ + struct platform_device *pdev; + struct ready_test *t; + struct apple_dcp *dcp; + + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t); + dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dcp); + t->con = kunit_kzalloc(test, sizeof(*t->con), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t->con); + pdev = kunit_platform_device_alloc(test, "dcp-external-ready", PLATFORM_DEVID_AUTO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev); + KUNIT_ASSERT_EQ(test, kunit_platform_device_add(test, pdev), 0); + + t->dcp = dcp; + dcp->dev = &pdev->dev; + dcp->hw.iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL; + mutex_init(&dcp->hpd_mutex); + mutex_init(&dcp->dcpavserv.lock); + spin_lock_init(&dcp->metadata.lock); + dcp->connector = t->con; + t->con->dcp = pdev; + t->con->connected = true; + + /* A direct route: the PHY and route are only compared, never used. */ + dcp->phy = kunit_kzalloc(test, 1, GFP_KERNEL); + dcp->active_typec_route = kunit_kzalloc(test, sizeof(*dcp->active_typec_route), + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dcp->phy); + KUNIT_ASSERT_NOT_NULL(test, dcp->active_typec_route); + t->dptx_ep.dcp = dcp; + t->dptx_ep.endpoint = DPTX_ENDPOINT_V2; + t->dptx_ep.wire_v2 = true; + t->dptx.ep = &t->dptx_ep; + t->dptx.cookie = &dcp->dptxport[0]; + t->dptx.enabled = true; + dcp->dptxport[0].service = &t->dptx; + dcp->dptxport[0].enabled = true; + dcp->dptxport[0].connected = true; + dcp->dptxport[0].atcphy = dcp->phy; + + t->modes[0].mode = ready_mode_60; + t->modes[1].mode = ready_mode_180; + t->modes[1].automatic_rgb8_quirk = true; + t->modes[1].sdr_rgb = (struct dcp_color_mode) { + .depth = 8, + .format = DCP_COLOR_FORMAT_RGB, + .range = DCP_COLOR_YCBCR_RANGE_FULL, + .eotf = DCP_EOTF_SDR_GAMMA, + .colorimetry = DCP_COLORIMETRY_SRGB, + }; + + /* A new cable. */ + dcp->typec_cable_connected = true; + dcp->typec_generation = 1; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, true); + + return t; +} + +/* What dcpep_process_chunks() publishes for a complete TimingElements. */ +static void ready_catalogue(struct ready_test *t, unsigned int nr_modes) +{ + struct apple_dcp *dcp = t->dcp; + struct dcp_metadata_ticket ticket; + unsigned long flags; + unsigned int i; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + ticket = dcp_metadata_begin(&dcp->metadata.epoch); + ticket.associated = dcp->metadata.epoch.generation == dcp->typec_generation && + dcp->typec_cable_connected; + dcp->rate_proof.catalog_sequence++; + for (i = 0; i < nr_modes; i++) { + if (!t->modes[i].automatic_rgb8_quirk) + continue; + t->modes[i].unsafe_rgb8_ticket = ticket; + t->modes[i].unsafe_rgb8_service_epoch = dcp->dcpavserv.epoch; + t->modes[i].automatic_service_pending = !dcp->dcpavserv.enabled && + !dcp->dcpavserv.service; + t->modes[i].automatic_connection_sequence = dcp->rate_proof.connection_sequence; + t->modes[i].automatic_catalog_sequence = dcp->rate_proof.catalog_sequence; + } + dcp->modes = t->modes; + dcp->nr_modes = nr_modes; + dcp->metadata.catalog_ticket = ticket; + dcp->metadata.catalog_ticket.associated &= nr_modes > 0; + dcp->metadata.catalog_connection_sequence = dcp->rate_proof.connection_sequence; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); +} + +/* The AV service the EDID is read from comes up (dcpavserv_init()). */ +static void ready_service(struct ready_test *t) +{ + struct apple_dcp *dcp = t->dcp; + + guard(mutex)(&dcp->dcpavserv.lock); + dcp->dcpavserv.service = &t->av; + dcp->dcpavserv.enabled = true; + dcp->dcpavserv.epoch++; +} + +/* The firmware announces the display (dcpep_cb_hotplug()). */ +static void ready_announce(struct ready_test *t) +{ + dcp_firmware_metadata_hotplug(t->dcp, true); +} + +/* The DPTX endpoint reports the link rate and lane count it configured. */ +static void ready_link(struct ready_test *t, u32 rate, u32 lanes) +{ + struct dcp_180hz_token token; + + token = dcp_180hz_config_begin(&t->dptx, false); + dcp_180hz_config_complete(&t->dptx, false, token, rate, true); + token = dcp_180hz_config_begin(&t->dptx, true); + dcp_180hz_config_complete(&t->dptx, true, token, lanes, true); +} + +static bool ready(struct ready_test *t) +{ + guard(mutex)(&t->dcp->hpd_mutex); + return dcp_external_modes_ready(t->dcp); +} + +/* What dcp_get_modes() decides for the 180 Hz mode once it has read the EDID. */ +static bool ready_probe_lists_180(struct kunit *test, struct ready_test *t) +{ + struct apple_dcp *dcp = t->dcp; + const struct drm_edid *edid = ready_edid(test); + unsigned long flags; + bool listed; + + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, ready_free_edid, (void *)edid), 0); + guard(mutex)(&dcp->hpd_mutex); + guard(mutex)(&dcp->dcpavserv.lock); + t->con->drm_edid = edid; + t->con->drm_edid_generation = dcp->typec_generation; + t->con->drm_edid_service_epoch = dcp->dcpavserv.epoch; + spin_lock_irqsave(&dcp->metadata.lock, flags); + listed = dcp_180hz_mode_current(dcp, &t->modes[1]); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + t->con->drm_edid = NULL; + return listed; +} + +static void ready_complete_test(struct kunit *test) +{ + struct ready_test *t = ready_init(test); + + ready_catalogue(t, 2); + ready_service(t); + ready_announce(t); + ready_link(t, 0x1e, 2); + + KUNIT_EXPECT_TRUE(test, ready(t)); + KUNIT_EXPECT_TRUE(test, ready_probe_lists_180(test, t)); +} + +enum ready_fact { READY_CATALOGUE, READY_SERVICE, READY_ANNOUNCE, READY_LINK, READY_FACTS }; + +static void ready_fact(struct ready_test *t, enum ready_fact fact) +{ + switch (fact) { + case READY_CATALOGUE: + ready_catalogue(t, 2); + break; + case READY_SERVICE: + ready_service(t); + break; + case READY_ANNOUNCE: + ready_announce(t); + break; + default: + ready_link(t, 0x1e, 2); + break; + } +} + +/* + * Back from system sleep on the same cable: suspend advanced the generation + * without a new connection, so the firmware has to announce the display + * again. (A new cable counts as announced from dcp_dptx_connect_oob().) + * Each fact can come last; until it has, a probe would miss the 180 Hz mode. + */ +static void ready_any_order_test(struct kunit *test) +{ + enum ready_fact last, fact; + + for (last = 0; last < READY_FACTS; last++) { + struct ready_test *t = ready_init(test); + + t->dcp->typec_generation++; + dcp_firmware_metadata_connection(t->dcp, t->dcp->typec_generation, false); + + for (fact = 0; fact < READY_FACTS; fact++) + if (fact != last) + ready_fact(t, fact); + KUNIT_EXPECT_FALSE_MSG(test, ready(t), "fact %d missing", last); + KUNIT_EXPECT_FALSE_MSG(test, ready_probe_lists_180(test, t), + "fact %d missing", last); + + ready_fact(t, last); + KUNIT_EXPECT_TRUE_MSG(test, ready(t), "fact %d last", last); + KUNIT_EXPECT_TRUE_MSG(test, ready_probe_lists_180(test, t), + "fact %d last", last); + } +} + +/* + * A replug keeps the previous connection's catalogue until TimingElements + * replaces it: that catalogue and a new EDID service must not count. + */ +static void ready_retained_catalogue_test(struct kunit *test) +{ + struct ready_test *t = ready_init(test); + struct apple_dcp *dcp = t->dcp; + + ready_catalogue(t, 2); + ready_service(t); + ready_announce(t); + ready_link(t, 0x1e, 2); + KUNIT_ASSERT_TRUE(test, ready(t)); + + /* Unplug (dcp_external_disconnect()), then a new cable. */ + dcp->typec_cable_connected = false; + dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); + dcp->dcpavserv.enabled = false; + dcp->dcpavserv.service = NULL; + dcp->dcpavserv.epoch++; + dcp->typec_cable_connected = true; + dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, true); + + ready_service(t); + ready_announce(t); + ready_link(t, 0x1e, 2); + KUNIT_EXPECT_EQ(test, dcp->nr_modes, 2); + KUNIT_EXPECT_FALSE(test, ready(t)); + + ready_catalogue(t, 2); + KUNIT_EXPECT_TRUE(test, ready(t)); +} + +/* + * System sleep advances the generation of a cable that stays attached. + * Facts from before it, and a callback delivered after it, prove nothing. + */ +static void ready_generation_advance_test(struct kunit *test) +{ + struct ready_test *t = ready_init(test); + struct apple_dcp *dcp = t->dcp; + + ready_catalogue(t, 2); + ready_service(t); + ready_announce(t); + ready_link(t, 0x1e, 2); + KUNIT_ASSERT_TRUE(test, ready(t)); + + dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); + KUNIT_EXPECT_FALSE(test, ready(t)); + + /* A late announcement of the old link, with the new generation set. */ + ready_announce(t); + KUNIT_EXPECT_FALSE(test, ready(t)); + KUNIT_EXPECT_FALSE(test, ready_probe_lists_180(test, t)); + + ready_catalogue(t, 2); + ready_link(t, 0x1e, 2); + KUNIT_EXPECT_TRUE(test, ready(t)); +} + +/* The display behind a dock that stays attached drops its link. */ +static void ready_downstream_loss_test(struct kunit *test) +{ + struct ready_test *t = ready_init(test); + + ready_catalogue(t, 2); + ready_service(t); + ready_announce(t); + ready_link(t, 0x1e, 2); + KUNIT_ASSERT_TRUE(test, ready(t)); + + dcp_firmware_metadata_hotplug(t->dcp, false); + KUNIT_EXPECT_FALSE(test, ready(t)); +} + +/* Without an exception mode, the catalogue and the EDID service suffice. */ +static void ready_ordinary_catalogue_test(struct kunit *test) +{ + struct ready_test *t = ready_init(test); + + ready_catalogue(t, 1); + KUNIT_EXPECT_FALSE(test, ready(t)); + ready_service(t); + KUNIT_EXPECT_TRUE(test, ready(t)); +} + +/* A link configured at another rate never proves the 180 Hz mode. */ +static void ready_other_link_test(struct kunit *test) +{ + struct ready_test *t = ready_init(test); + + ready_catalogue(t, 2); + ready_service(t); + ready_announce(t); + ready_link(t, 0x14, 2); + KUNIT_EXPECT_FALSE(test, ready(t)); + KUNIT_EXPECT_FALSE(test, ready_probe_lists_180(test, t)); +} + +static struct kunit_case ready_cases[] = { + KUNIT_CASE(ready_complete_test), + KUNIT_CASE(ready_any_order_test), + KUNIT_CASE(ready_retained_catalogue_test), + KUNIT_CASE(ready_generation_advance_test), + KUNIT_CASE(ready_downstream_loss_test), + KUNIT_CASE(ready_ordinary_catalogue_test), + KUNIT_CASE(ready_other_link_test), + {} +}; + +static struct kunit_suite ready_suite = { + .name = "apple-dcp-external-ready", + .test_cases = ready_cases, +}; + +kunit_test_suite(ready_suite); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 7bf13ff8f12d0..8f3d658381f53 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1022,6 +1022,7 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, spin_lock_irqsave(&dcp->metadata.lock, flags); dcp->rate_proof.catalog_sequence++; + dcp->metadata.catalog_ticket.associated = false; spin_unlock_irqrestore(&dcp->metadata.lock, flags); dcp_180hz_catalog_changed(dcp); } @@ -1070,6 +1071,11 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, modes[i].automatic_catalog_sequence = dcp->rate_proof.catalog_sequence; dcp->modes = modes; dcp->nr_modes = nr_modes; + /* Only a complete receipt for a current cable is that cable's catalogue. */ + dcp->metadata.catalog_ticket = rate_ticket; + dcp->metadata.catalog_ticket.associated &= nr_modes > 0; + dcp->metadata.catalog_connection_sequence = + dcp->chunks.automatic_connection_sequence; spin_unlock_irqrestore(&dcp->metadata.lock, flags); /* Notify removals as well as newly eligible modes, outside the lock. */ dcp_180hz_catalog_changed(dcp); From 936d68712791fa4fabdb2d4b9ff579b30d82a731 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:31:23 +1100 Subject: [PATCH 1449/2680] drm/apple: Add a state machine for announcing an external display MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Userspace that probes an external display before its modes are in picks a mode from an incomplete list and switches again once the rest arrives. That costs a second mode set, which the monitor has to resynchronize to, both when a display is plugged in and when its CRTC comes back from system sleep disabled and userspace sets its mode again. Add the policy for when DRM reports such a display and when it sends the hotplug event for it, as a pure helper that a later patch wires up: - A new connection is reported once it is ready, or after 1 s without that. - A reported connection stays reported until the display or the cable goes away, so a link retrained on the same cable does not withdraw it. A cable known to be gone wins at once. - A reported connection can hold its next hotplug event, never the connection itself, until it is ready again, or for 2 s without that. - The timeout of a hold runs from when the caller starts it, which it does only once the link can come back, and only for the cable generation it was started in. A hold that outlives its generation, as across another system sleep, waits for the next one from the start, even if its old deadline is still to come. A hold whose link failed ends at once. - A hold that ends with the connection reported owes userspace one hotplug event, as does a reported connection that ends, until the hotplug worker hands it out. Whoever ends a hold can tell from that whether it has to queue the worker, and evaluating the connection again does not owe a second event. The state also carries the flag with which a later patch stops the announcement for teardown and system sleep. The KUnit suite covers both kinds of hold, their timeouts, a jiffies wrap, the generation rule, failure, unplug during a hold, the event a release owes and a replug the hotplug worker only sees once the new link is up. [ryan: adapted to 7.2.10 — context only: 200-dcp's DRM_APPLE_PARSER_KUNIT_TEST entry follows the extended help text] (cherry picked from commit 28758de011dde69d28d19168eb7adcf08d7e7aa9) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Kconfig | 3 +- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/dcp-announce-test.c | 404 ++++++++++++++++++++++ drivers/gpu/drm/apple/dcp-announce.h | 151 ++++++++ 4 files changed, 558 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/apple/dcp-announce-test.c create mode 100644 drivers/gpu/drm/apple/dcp-announce.h diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 2c2d64c364420..8445d82991a93 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -42,7 +42,8 @@ config DRM_APPLE_KUNIT_TEST larger rings, as well as malformed ring size combinations. This option is intended for developer test kernels. Also test the Type-C modeset, hotplug and service ownership state, - and the system resume of a CRTC whose mode is no longer admissible. + the system resume of a CRTC whose mode is no longer admissible, and + when an external display is ready to be reported to userspace. config DRM_APPLE_PARSER_KUNIT_TEST bool "KUnit tests for Apple firmware message parsers" if !KUNIT_ALL_TESTS diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 06927b4ad4e6a..a992d5366bdad 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -26,6 +26,7 @@ appledrm-$(CONFIG_TRACING) += trace.o obj-$(CONFIG_DRM_APPLE) += appledrm.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk_test.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_backlight_test.o +obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp-announce-test.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk-service-test.o diff --git a/drivers/gpu/drm/apple/dcp-announce-test.c b/drivers/gpu/drm/apple/dcp-announce-test.c new file mode 100644 index 0000000000000..bdfd9ef4db774 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-announce-test.c @@ -0,0 +1,404 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include + +#include "dcp-announce.h" + +#define TIMEOUT msecs_to_jiffies(DCP_ANNOUNCE_TIMEOUT_MS) +#define RESUME_TIMEOUT msecs_to_jiffies(DCP_ANNOUNCE_RESUME_TIMEOUT_MS) + +static const struct dcp_announce_facts link_only = { + .generation = 1, .connected = true, .present = true, +}; + +static const struct dcp_announce_facts link_ready = { + .generation = 1, .connected = true, .present = true, .ready = true, +}; + +static const struct dcp_announce_facts unplugged = { + .generation = 2, +}; + +static void announce_ready_test(struct kunit *test) +{ + struct dcp_announce a = { }; + + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_ready, 100), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, a.announced); + KUNIT_EXPECT_FALSE(test, a.holding); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_CONNECTED)); +} + +static void announce_waits_for_ready_test(struct kunit *test) +{ + struct dcp_announce a = { }; + + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 100), + DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_TRUE(test, a.holding); + KUNIT_EXPECT_FALSE(test, a.announced); + KUNIT_EXPECT_FALSE(test, a.timed); + KUNIT_EXPECT_FALSE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_HOLD)); + dcp_announce_start(&a, 100); + KUNIT_EXPECT_EQ(test, a.deadline, 100 + TIMEOUT); + + /* Neither re-evaluation nor starting again extends the deadline. */ + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 100 + TIMEOUT - 1), + DCP_ANNOUNCE_HOLD); + dcp_announce_start(&a, 100 + TIMEOUT - 1); + KUNIT_EXPECT_EQ(test, a.deadline, 100 + TIMEOUT); + + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_ready, 100 + TIMEOUT - 1), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_FALSE(test, a.holding); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_CONNECTED)); +} + +static void announce_timeout_test(struct kunit *test) +{ + struct dcp_announce a = { }; + + dcp_announce_update(&a, &link_only, 100); + dcp_announce_start(&a, 100); + KUNIT_EXPECT_FALSE(test, dcp_announce_expired(&a, 1, 100 + TIMEOUT - 1)); + KUNIT_EXPECT_TRUE(test, dcp_announce_expired(&a, 1, 100 + TIMEOUT)); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 100 + TIMEOUT), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, a.announced); + KUNIT_EXPECT_FALSE(test, a.holding); + /* A timer that fires after the hold ended finds nothing to release. */ + KUNIT_EXPECT_FALSE(test, dcp_announce_expired(&a, 1, 100 + 2 * TIMEOUT)); +} + +static void announce_timeout_wraps_test(struct kunit *test) +{ + unsigned long start = ULONG_MAX - TIMEOUT / 2; + struct dcp_announce a = { }; + + dcp_announce_update(&a, &link_only, start); + dcp_announce_start(&a, start); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, start + TIMEOUT - 1), + DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, start + TIMEOUT), + DCP_ANNOUNCE_CONNECTED); +} + +/* A failed link will not get ready; report it without waiting. */ +static void announce_failed_test(struct kunit *test) +{ + struct dcp_announce_facts f = link_only; + struct dcp_announce a = { }; + + f.failed = true; + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &f, 0), DCP_ANNOUNCE_CONNECTED); +} + +/* A link retrained on the same cable must not withdraw the connection. */ +static void announce_keeps_reported_test(struct kunit *test) +{ + struct dcp_announce_facts f = link_only; + struct dcp_announce a = { }; + + dcp_announce_update(&a, &link_ready, 0); + f.generation = 2; + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &f, 10 * TIMEOUT), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_FALSE(test, a.holding); +} + +/* A missing cable wins over a link DCP has not dropped yet. */ +static void announce_absent_wins_test(struct kunit *test) +{ + struct dcp_announce_facts f = link_ready; + struct dcp_announce a = { }; + + dcp_announce_update(&a, &link_ready, 0); + f.present = false; + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &f, 1), DCP_ANNOUNCE_DISCONNECTED); + KUNIT_EXPECT_FALSE(test, a.announced); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_DISCONNECTED)); +} + +static void announce_disconnect_during_hold_test(struct kunit *test) +{ + struct dcp_announce a = { }; + + dcp_announce_update(&a, &link_only, 0); + dcp_announce_start(&a, 0); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &unplugged, 1), + DCP_ANNOUNCE_DISCONNECTED); + KUNIT_EXPECT_FALSE(test, a.holding); + KUNIT_EXPECT_FALSE(test, dcp_announce_owed(&a)); + KUNIT_EXPECT_FALSE(test, dcp_announce_expired(&a, 1, TIMEOUT)); + + /* The next connection gets a full timeout of its own. */ + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 5 * TIMEOUT), + DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_FALSE(test, a.timed); + dcp_announce_start(&a, 5 * TIMEOUT); + KUNIT_EXPECT_EQ(test, a.deadline, 5 * TIMEOUT + TIMEOUT); + KUNIT_EXPECT_FALSE(test, dcp_announce_expired(&a, 1, TIMEOUT)); +} + +/* + * A replug that the hotplug worker only sees once the new link is up: the + * old connection still needs its event while the new one is held. + */ +static void announce_replug_test(struct kunit *test) +{ + struct dcp_announce_facts f = link_only; + struct dcp_announce a = { }; + + dcp_announce_update(&a, &link_ready, 0); + dcp_announce_notify(&a, DCP_ANNOUNCE_CONNECTED); + dcp_announce_end(&a); + f.generation = 2; + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &f, 1), DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_HOLD)); + KUNIT_EXPECT_FALSE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_HOLD)); + f.ready = true; + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &f, 2), DCP_ANNOUNCE_CONNECTED); +} + +/* Ending a connection that was never reported owes userspace nothing. */ +static void announce_end_unreported_test(struct kunit *test) +{ + struct dcp_announce a = { }; + + dcp_announce_update(&a, &link_only, 0); + dcp_announce_end(&a); + KUNIT_EXPECT_FALSE(test, a.withdrawn); + KUNIT_EXPECT_FALSE(test, a.holding); +} + +/* Only a reported connection has an event to hold. */ +static void hold_unreported_test(struct kunit *test) +{ + struct dcp_announce a = { }; + + KUNIT_EXPECT_FALSE(test, dcp_announce_hold(&a, 1)); + KUNIT_EXPECT_FALSE(test, a.holding); +} + +static struct dcp_announce held(u64 generation) +{ + struct dcp_announce a = { .announced = true }; + + dcp_announce_hold(&a, generation); + return a; +} + +/* + * A held event keeps the connection reported, and its timeout runs only once + * started: the link cannot come back before then. + */ +static void hold_reported_test(struct kunit *test) +{ + struct dcp_announce a = held(1); + + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 0), DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_TRUE(test, a.announced); + KUNIT_EXPECT_FALSE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_HOLD)); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 100 * RESUME_TIMEOUT), + DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_FALSE(test, dcp_announce_expired(&a, 1, 100 * RESUME_TIMEOUT)); + + dcp_announce_start(&a, 1000); + KUNIT_EXPECT_EQ(test, a.deadline, 1000 + RESUME_TIMEOUT); + /* Starting again does not move the deadline. */ + dcp_announce_start(&a, 2000); + KUNIT_EXPECT_EQ(test, a.deadline, 1000 + RESUME_TIMEOUT); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 1000 + RESUME_TIMEOUT - 1), + DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_TRUE(test, dcp_announce_expired(&a, 1, 1000 + RESUME_TIMEOUT)); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 1000 + RESUME_TIMEOUT), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_CONNECTED)); + KUNIT_EXPECT_FALSE(test, a.holding); +} + +static void hold_ready_test(struct kunit *test) +{ + struct dcp_announce a = held(1); + + dcp_announce_start(&a, 0); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 1), DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_ready, 2), DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_CONNECTED)); + KUNIT_EXPECT_FALSE(test, dcp_announce_expired(&a, 1, RESUME_TIMEOUT)); +} + +static void hold_failed_test(struct kunit *test) +{ + struct dcp_announce_facts f = link_only; + struct dcp_announce a = held(1); + + f.failed = true; + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &f, 0), DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, a.announced); +} + +/* An unplug while the event is held is reported at once. */ +static void hold_unplug_test(struct kunit *test) +{ + struct dcp_announce a = held(1); + + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &unplugged, 0), + DCP_ANNOUNCE_DISCONNECTED); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_DISCONNECTED)); + KUNIT_EXPECT_FALSE(test, a.holding); +} + +/* + * A hold that outlives its cable generation, as across another sleep, waits + * for the next one from the start: the old deadline is not carried over. + */ +static void hold_next_generation_test(struct kunit *test) +{ + struct dcp_announce_facts f = link_only; + struct dcp_announce a = held(1); + + dcp_announce_start(&a, 0); + f.generation = 2; + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &f, RESUME_TIMEOUT), + DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_EQ(test, a.generation, 2); + KUNIT_EXPECT_FALSE(test, dcp_announce_expired(&a, 2, 10 * RESUME_TIMEOUT)); + dcp_announce_start(&a, 10 * RESUME_TIMEOUT); + KUNIT_EXPECT_EQ(test, a.deadline, 11 * RESUME_TIMEOUT); +} + +/* + * The deadline of one generation never ends the hold of the next, even + * before an update has moved the hold on. + */ +static void hold_deadline_generation_test(struct kunit *test) +{ + struct dcp_announce a = held(1); + + dcp_announce_start(&a, 0); + KUNIT_EXPECT_TRUE(test, dcp_announce_expired(&a, 1, RESUME_TIMEOUT)); + KUNIT_EXPECT_FALSE(test, dcp_announce_expired(&a, 2, RESUME_TIMEOUT)); +} + +/* A new connection seen first in a later generation restarts its timeout. */ +static void announce_next_generation_test(struct kunit *test) +{ + struct dcp_announce_facts f = link_only; + struct dcp_announce a = { }; + + dcp_announce_update(&a, &f, 0); + dcp_announce_start(&a, 0); + f.generation = 2; + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &f, TIMEOUT), DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_FALSE(test, a.timed); + dcp_announce_start(&a, 3 * TIMEOUT); + KUNIT_EXPECT_EQ(test, a.deadline, 4 * TIMEOUT); +} + +/* Until its timeout is started, a hold waits however long it takes. */ +static void announce_untimed_test(struct kunit *test) +{ + struct dcp_announce a = { }; + + dcp_announce_update(&a, &link_only, 0); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 100 * TIMEOUT), + DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_FALSE(test, dcp_announce_expired(&a, 1, 100 * TIMEOUT)); +} + +/* + * However a new connection is released, and whoever evaluates it, its event + * is owed until it is handed out, and it is handed out once. + */ +static void announce_release_owed_test(struct kunit *test) +{ + struct dcp_announce a = { }; + + dcp_announce_update(&a, &link_only, 0); + dcp_announce_start(&a, 0); + KUNIT_EXPECT_FALSE(test, dcp_announce_owed(&a)); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, TIMEOUT), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, dcp_announce_owed(&a)); + /* Evaluating it again does not hand the event out. */ + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, TIMEOUT + 1), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, dcp_announce_owed(&a)); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_CONNECTED)); + KUNIT_EXPECT_FALSE(test, dcp_announce_owed(&a)); +} + +/* An evaluation after the hold ended and its event went out owes nothing. */ +static void hold_release_once_test(struct kunit *test) +{ + struct dcp_announce a = held(1); + + dcp_announce_start(&a, 0); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, RESUME_TIMEOUT), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, dcp_announce_owed(&a)); + dcp_announce_notify(&a, DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 2 * RESUME_TIMEOUT), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_FALSE(test, dcp_announce_owed(&a)); +} + +/* + * A new hold before the event of the last one went out holds that event + * too: it goes out once, when the new hold ends. + */ +static void hold_again_test(struct kunit *test) +{ + struct dcp_announce a = held(1); + + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_ready, 0), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, dcp_announce_hold(&a, 2)); + KUNIT_EXPECT_FALSE(test, dcp_announce_owed(&a)); + KUNIT_EXPECT_FALSE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_HOLD)); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_ready, 1), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, dcp_announce_owed(&a)); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_CONNECTED)); + KUNIT_EXPECT_FALSE(test, dcp_announce_owed(&a)); +} + +static struct kunit_case announce_cases[] = { + KUNIT_CASE(announce_ready_test), + KUNIT_CASE(announce_waits_for_ready_test), + KUNIT_CASE(announce_timeout_test), + KUNIT_CASE(announce_timeout_wraps_test), + KUNIT_CASE(announce_failed_test), + KUNIT_CASE(announce_keeps_reported_test), + KUNIT_CASE(announce_absent_wins_test), + KUNIT_CASE(announce_disconnect_during_hold_test), + KUNIT_CASE(announce_replug_test), + KUNIT_CASE(announce_end_unreported_test), + KUNIT_CASE(announce_next_generation_test), + KUNIT_CASE(announce_untimed_test), + KUNIT_CASE(announce_release_owed_test), + KUNIT_CASE(hold_unreported_test), + KUNIT_CASE(hold_reported_test), + KUNIT_CASE(hold_ready_test), + KUNIT_CASE(hold_failed_test), + KUNIT_CASE(hold_unplug_test), + KUNIT_CASE(hold_next_generation_test), + KUNIT_CASE(hold_deadline_generation_test), + KUNIT_CASE(hold_release_once_test), + KUNIT_CASE(hold_again_test), + {} +}; + +static struct kunit_suite announce_suite = { + .name = "apple-dcp-announce", + .test_cases = announce_cases, +}; + +kunit_test_suite(announce_suite); + +MODULE_DESCRIPTION("KUnit tests for the Apple DCP external connection announcement"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/dcp-announce.h b/drivers/gpu/drm/apple/dcp-announce.h new file mode 100644 index 0000000000000..cb360e48eb316 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-announce.h @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __APPLE_DCP_ANNOUNCE_H__ +#define __APPLE_DCP_ANNOUNCE_H__ + +#include +#include + +/* + * The modes of an external display arrive in pieces after its link comes up + * (see dcp_external_modes_ready()). Userspace that probes before they are + * all in picks a mode from an incomplete list, then switches again once the + * rest arrives: a second mode set the monitor has to resynchronize to. + * + * - A new connection is reported to DRM once it is ready, or once + * DCP_ANNOUNCE_TIMEOUT_MS have passed without that. + * - Once reported, a connection stays reported until the display or the + * cable goes away: a link retrained on the same cable, for instance after + * system sleep, does not withdraw it. A cable that is gone is reported as + * such at once, even while DCP still holds the old link. + * - A reported connection can hold its next hotplug event, never the + * connection itself, until it is ready again, or until + * DCP_ANNOUNCE_RESUME_TIMEOUT_MS have passed without that. That is the + * event that has userspace set the mode of a CRTC resumed disabled. + * + * The timeout of a hold runs from dcp_announce_start(), and only for the + * cable generation it was started in. A hold that outlives its generation, + * as across another system sleep, waits for the next one from the start. + * + * A hold that ends with the connection reported owes userspace a hotplug + * event, as does a reported connection that ends, until + * dcp_announce_notify() hands that event out. + */ +#define DCP_ANNOUNCE_TIMEOUT_MS 1000 +#define DCP_ANNOUNCE_RESUME_TIMEOUT_MS 2000 + +enum dcp_announce_status { + DCP_ANNOUNCE_DISCONNECTED, + DCP_ANNOUNCE_HOLD, /* the hotplug event is held */ + DCP_ANNOUNCE_CONNECTED, +}; + +struct dcp_announce { + bool announced; /* DRM reports the connection */ + bool withdrawn; /* a reported connection ended, no event sent yet */ + bool released; /* a hold ended, no event sent yet */ + bool holding; /* waiting for the connection to be ready */ + bool timed; /* the hold of @generation ends at @deadline */ + bool stopped; /* nothing may be queued for the announcement */ + u64 generation; /* the cable generation of the hold */ + unsigned long deadline; +}; + +struct dcp_announce_facts { + u64 generation; /* the current cable generation */ + bool connected; /* DCP reports the display */ + bool present; /* the cable is not known to be gone */ + bool ready; /* a probe would list the modes of this connection */ + bool failed; /* the link failed; it will not get ready */ +}; + +/* The connection ended, whether DRM was told about it or not. */ +static inline void dcp_announce_end(struct dcp_announce *a) +{ + a->withdrawn |= a->announced; + a->announced = false; + a->holding = false; +} + +/* + * Hold the next hotplug event of a reported connection until it is ready on + * cable generation @generation. + */ +static inline bool dcp_announce_hold(struct dcp_announce *a, u64 generation) +{ + if (!a->announced) + return false; + a->holding = true; + a->timed = false; + a->generation = generation; + return true; +} + +/* Start the timeout of a hold that has none for its generation yet. */ +static inline void dcp_announce_start(struct dcp_announce *a, unsigned long now) +{ + if (!a->holding || a->timed) + return; + a->timed = true; + a->deadline = now + msecs_to_jiffies(a->announced ? DCP_ANNOUNCE_RESUME_TIMEOUT_MS : + DCP_ANNOUNCE_TIMEOUT_MS); +} + +static inline bool dcp_announce_expired(const struct dcp_announce *a, u64 generation, + unsigned long now) +{ + return a->holding && a->timed && a->generation == generation && + !time_before(now, a->deadline); +} + +/* Whether a hotplug event is owed that dcp_announce_notify() has not handed out. */ +static inline bool dcp_announce_owed(const struct dcp_announce *a) +{ + return a->withdrawn || (a->released && !a->holding); +} + +static inline enum dcp_announce_status +dcp_announce_update(struct dcp_announce *a, const struct dcp_announce_facts *f, + unsigned long now) +{ + if (!f->connected || !f->present) { + dcp_announce_end(a); + return DCP_ANNOUNCE_DISCONNECTED; + } + if (!a->announced && !a->holding) { + a->holding = true; + a->timed = false; + a->generation = f->generation; + } + if (a->holding && a->generation != f->generation) { + a->generation = f->generation; + a->timed = false; + } + if (a->holding && !f->ready && !f->failed && + !dcp_announce_expired(a, f->generation, now)) + return DCP_ANNOUNCE_HOLD; + if (a->holding) { + a->holding = false; + a->released = true; + } + a->announced = true; + return DCP_ANNOUNCE_CONNECTED; +} + +/* + * Whether a hotplug event is due, which hands out any event owed. A held + * connection changes nothing userspace can see, unless a reported one ended + * before it. + */ +static inline bool dcp_announce_notify(struct dcp_announce *a, + enum dcp_announce_status status) +{ + bool due = status != DCP_ANNOUNCE_HOLD || a->withdrawn; + + if (due) { + a->withdrawn = false; + a->released = false; + } + return due; +} + +#endif /* __APPLE_DCP_ANNOUNCE_H__ */ From 3dc4ddef54246a71409feb982327ee7c5a86b004 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:31:24 +1100 Subject: [PATCH 1450/2680] drm/apple: Report an external display connected once its modes are in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A physical disconnect of an external H17P display first marks the cable absent, advances its generation and retires the AV service the EDID is read from, and only then stops the link and clears the connector's connected bit. A hotplug event in that window has userspace probe a connector that is still connected but has no EDID. GNOME then rejects the stored configuration of an "unknown unknown" monitor. The reverse happens on plug: DCP can report the link before the EDID service, the catalogue of the new connection or the link configuration the 180 Hz mode needs, and userspace picks a fallback mode from what it finds. Wire up the announcement state with dcp_external_modes_ready() as its readiness, for the external H17P pipeline only. detect() evaluates it and reports what it says; it never waits. The hotplug worker sends no event while a connection is held: the last missing fact queues it again, through the paths that already do that, or the timeout work does. A physical disconnect and a new cable end the connection. The timeout is a delayed work item on the freezable workqueue, which follows the Type-C work across bind, unbind and system sleep. It starts the timeout of a hold, so a hold cannot run out while the link cannot come back, and evaluates the hold itself under hpd_mutex, reconciling it with the current cable generation before any deadline counts. Queueing that work counts a deadline only for the generation it was set in, so a hold that crossed another sleep is reconciled as soon as the work is enabled again, not at the deadline of the last generation. Whoever ends a hold, the timeout work or a probe, queues the hotplug worker for the event it owes under that lock. The event of a new connection is therefore not lost to a probe that reports it first, and a timeout work that loses that race adds no second event. Teardown and system sleep stop the announcement under hpd_mutex before they drain the timeout work, and nothing queues the timeout work or the hotplug worker for a stopped announcement. The teardown that keeps the DMA resources of a live DCP then drains the hotplug worker as before, so neither is left queued for the quiesced pipeline, even when the worker was evaluating a hold while it ran. An event that comes due while the announcement is stopped is sent once its work is enabled again. On every other pipeline detect() still returns the connector's connected bit as before, and teardown is unchanged. The KUnit suite runs detect(), the timeout work, the hotplug worker, the sleep and teardown paths: a probe that releases a new connection before its timeout work runs, a probe that races a running timeout work, the real hotplug worker running during unbind and during shutdown, a probe after the DRM device outlived the teardown, and holds that cross another sleep, whose timeout work has to run as soon as it is enabled again. A hook, built only with the KUnit option, pauses the timeout work and the teardown at the points where they race. The apple-dcp lifecycle selftest copies the external disconnect and connect paths into its model of the driver, so give that model the announcement they now end. [ryan: adapted to 7.2.10 — context only: 200-dcp's dcp_release_fixed_route() stays before dcp_disable_typec_work()] (cherry picked from commit 12c1046b9e313d0620570dfd427fbae75085b072) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/apple_drv.c | 4 +- drivers/gpu/drm/apple/connector.h | 1 + drivers/gpu/drm/apple/dcp-internal.h | 34 ++ drivers/gpu/drm/apple/dcp.c | 38 +- .../drm/apple/dcp_external_announce_test.c | 384 ++++++++++++++++++ drivers/gpu/drm/apple/iomfb.c | 138 ++++++- .../drivers/gpu/apple-dcp/lifecycle.py | 3 + 8 files changed, 596 insertions(+), 7 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp_external_announce_test.c diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index a992d5366bdad..0f0b80bb43ef3 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -17,6 +17,7 @@ appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple_resume_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += parser_mode_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_resume_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_ready_test.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_announce_test.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index aa80399b3be40..a338e0a2fd008 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -122,8 +122,8 @@ apple_connector_detect(struct drm_connector *connector, bool force) { struct apple_connector *apple_connector = to_apple_connector(connector); - return apple_connector->connected ? connector_status_connected : - connector_status_disconnected; + return dcp_connector_connected(apple_connector) ? + connector_status_connected : connector_status_disconnected; } static void apple_connector_oob_hotplug(struct drm_connector *connector, diff --git a/drivers/gpu/drm/apple/connector.h b/drivers/gpu/drm/apple/connector.h index e04bc2e119a5e..38772d6731c2c 100644 --- a/drivers/gpu/drm/apple/connector.h +++ b/drivers/gpu/drm/apple/connector.h @@ -15,6 +15,7 @@ struct apple_connector; #include "dcp-internal.h" void dcp_hotplug(struct work_struct *work); +bool dcp_connector_connected(struct apple_connector *connector); void dcp_retrain_oob(struct apple_connector *connector); /* diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 18ce4e77a49fc..584bf3b71f182 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -18,6 +18,7 @@ #include "dptxep.h" #include "dcp_backlight.h" +#include "dcp-announce.h" #include "dcp-display-metadata.h" #include "dcp-metadata-epoch.h" #include "dcp-180hz.h" @@ -206,6 +207,11 @@ struct apple_dcp_hw_data { u32 firmware_request; }; +enum dcp_test_point { + DCP_TEST_ANNOUNCE_TIMER_DONE, /* past hpd_mutex in the timeout work */ + DCP_TEST_UNBIND_ANNOUNCE_STOPPED, /* unbind drained the timeout work */ +}; + /* TODO: move IOMFB members to its own struct */ struct apple_dcp { /* An adopted session has no confirmed firmware DMA stop boundary. */ @@ -443,6 +449,13 @@ struct apple_dcp { struct completion typec_iomfb_hpd_ready; struct delayed_work typec_reconnect_wq; u32 typec_reconnect_tries; + /* hpd_mutex: what DRM reports for the external display, and when */ + struct dcp_announce announce; + struct delayed_work announce_wq; +#if IS_ENABLED(CONFIG_DRM_APPLE_KUNIT_TEST) + /* Lets a test stop work where it races, see dcp_test_point(). */ + void (*test_point)(struct apple_dcp *dcp, enum dcp_test_point point); +#endif struct gpio_desc *hdmi_hpd; struct gpio_desc *hdmi_pwren; @@ -535,6 +548,27 @@ void iomfb_queue_crc_h17p(struct apple_dcp *dcp, u32 swap_id); void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); void dcp_handle_hotplug_actions(struct apple_dcp *dcp, unsigned int action); bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected); +void dcp_announce_arm(struct apple_dcp *dcp); +void dcp_announce_work(struct work_struct *work); + +/* A KUnit test can wait here for the work it races with. */ +static inline void dcp_test_point(struct apple_dcp *dcp, enum dcp_test_point point) +{ +#if IS_ENABLED(CONFIG_DRM_APPLE_KUNIT_TEST) + void (*test_point)(struct apple_dcp *dcp, enum dcp_test_point point); + + test_point = READ_ONCE(dcp->test_point); + if (test_point) + test_point(dcp, point); +#endif +} + +#if IS_ENABLED(CONFIG_KUNIT) +void dcp_comp_unbind(struct device *dev, struct device *main, void *data); +void dcp_platform_shutdown(struct platform_device *pdev); +void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable); +void dcp_enable_typec_work(struct apple_dcp *dcp); +#endif int dcp_backlight_init(struct apple_dcp *dcp, struct device *dev); void dcp_backlight_stop(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index f8d43d13e8e9d..3904642a7cb5a 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright 2021 Alyssa Rosenzweig */ +#include #include #include #include @@ -2817,6 +2818,7 @@ static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical) WRITE_ONCE(dcp->typec_cable_connected, false); dcp->typec_generation++; dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); + dcp_announce_end(&dcp->announce); } /* HPD is fenced; retire the AV owner after releasing hpd_mutex. */ dpavservep_detach(dcp); @@ -3046,6 +3048,7 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) if (new_cable) { dcp->typec_generation++; dcp_firmware_metadata_connection(dcp, dcp->typec_generation, true); + dcp_announce_end(&dcp->announce); } dcp->placeholder_retried = false; } @@ -4051,29 +4054,45 @@ static void dcp_release_fixed_route(void *data) dcp->typec_mux = NULL; } -static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) +VISIBLE_IF_KUNIT void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) { scoped_guard(mutex, &dcp->hpd_mutex) { if (release_cable) WRITE_ONCE(dcp->typec_cable_connected, false); dcp->typec_generation++; dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); + /* Nothing arms the timeout work past its drain below. */ + dcp->announce.stopped = true; } if (release_cable && dcp_is_external_h17p(dcp)) dpavservep_detach(dcp); /* Block new enqueues as well as draining users of the AFK endpoints. */ disable_delayed_work_sync(&dcp->typec_reconnect_wq); disable_delayed_work_sync(&dcp->placeholder_edid_wq); + disable_delayed_work_sync(&dcp->announce_wq); cancel_work_sync(&dcp->external_stop_work); } -static void dcp_enable_typec_work(struct apple_dcp *dcp) +VISIBLE_IF_KUNIT void dcp_enable_typec_work(struct apple_dcp *dcp) { enable_delayed_work(&dcp->typec_reconnect_wq); enable_delayed_work(&dcp->placeholder_edid_wq); + enable_delayed_work(&dcp->announce_wq); /* A cable can be routed before the DRM component binds. */ if (READ_ONCE(dcp->typec_cable_connected)) mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); + if (dcp_is_external_h17p(dcp)) { + struct apple_connector *con = READ_ONCE(dcp->connector); + + guard(mutex)(&dcp->hpd_mutex); + dcp->announce.stopped = false; + /* A hold that crossed system sleep waits for the new link from now. */ + dcp_announce_arm(dcp); + /* Send an event that came due while stopped. */ + if (dcp_announce_owed(&dcp->announce) && con && + READ_ONCE(con->dcp) == to_platform_device(dcp->dev)) + schedule_work(&con->hotplug_wq); + } } static bool dcp_external_is_idle(struct apple_dcp *dcp) @@ -4285,7 +4304,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) * We need to shutdown DCP before tearing down the display subsystem. Otherwise * the DCP will crash and briefly flash a green screen of death. */ -static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) +VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, void *data) { struct apple_dcp *dcp = dev_get_drvdata(dev); @@ -4315,6 +4334,14 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) cancel_work_sync(&dcp->bl_update_wq); } cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + /* + * Stop the announcement before draining the hotplug work: that + * work arms the timeout work, which in turn queues it. + */ + scoped_guard(mutex, &dcp->hpd_mutex) + dcp->announce.stopped = true; + disable_delayed_work_sync(&dcp->announce_wq); + dcp_test_point(dcp, DCP_TEST_UNBIND_ANNOUNCE_STOPPED); cancel_work_sync(&dcp->vblank_wq); if (dcp->connector) cancel_work_sync(&dcp->connector->hotplug_wq); @@ -4478,6 +4505,7 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp_typec_reconnect_work); INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, dcp_placeholder_edid_work); + INIT_DELAYED_WORK(&dcp->announce_wq, dcp_announce_work); INIT_DELAYED_WORK(&dcp->bl_fallback_wq, dcp_work_backlight_fallback); /* System sleep and shutdown cancel these even if no bind succeeded. */ INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); @@ -4488,6 +4516,8 @@ static int dcp_platform_probe(struct platform_device *pdev) /* Balanced by enable at successful component bind. */ disable_delayed_work(&dcp->typec_reconnect_wq); disable_delayed_work(&dcp->placeholder_edid_wq); + disable_delayed_work(&dcp->announce_wq); + dcp->announce.stopped = true; platform_set_drvdata(pdev, dcp); @@ -4650,7 +4680,7 @@ static void dcp_platform_remove(struct platform_device *pdev) component_del(&pdev->dev, &dcp_comp_ops); } -static void dcp_platform_shutdown(struct platform_device *pdev) +VISIBLE_IF_KUNIT void dcp_platform_shutdown(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); diff --git a/drivers/gpu/drm/apple/dcp_external_announce_test.c b/drivers/gpu/drm/apple/dcp_external_announce_test.c new file mode 100644 index 0000000000000..58a0c1247acd8 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_external_announce_test.c @@ -0,0 +1,384 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include +#include +#include +#include +#include + +#include + +#include "connector.h" +#include "dcp.h" +#include "dcp-internal.h" + +/* + * The announcement of an external H17P display, through the detect(), the + * hotplug worker, the timeout work and the teardown the driver runs. A + * stub stands in for the hotplug worker and counts the events it would + * send, except where the real one has to race with teardown. + */ +struct announce_test { + struct apple_dcp dcp; + struct platform_device *pdev; + struct apple_connector con; + struct drm_device drm; + atomic_t events; + + /* dcp_test_point() to pause at once, or -1 */ + int pause_at; + struct completion paused, resume; + + struct completion hotplug_entered, hotplug_release, hotplug_done; + struct task_struct *teardown; + struct completion torn_down; +}; + +static void announce_test_noop(struct work_struct *work) +{ +} + +static void announce_test_event(struct work_struct *work) +{ + struct announce_test *t = container_of(work, struct announce_test, con.hotplug_wq); + + atomic_inc(&t->events); +} + +/* The real hotplug worker, held back until the test lets it run. */ +static void announce_test_running_hotplug(struct work_struct *work) +{ + struct announce_test *t = container_of(work, struct announce_test, con.hotplug_wq); + + complete(&t->hotplug_entered); + wait_for_completion(&t->hotplug_release); + dcp_hotplug(work); + complete(&t->hotplug_done); +} + +static void announce_test_point(struct apple_dcp *dcp, enum dcp_test_point point) +{ + struct announce_test *t = container_of(dcp, struct announce_test, dcp); + + if (cmpxchg(&t->pause_at, point, -1) != point) + return; + complete(&t->paused); + wait_for_completion(&t->resume); +} + +static void announce_test_exit(void *data) +{ + struct announce_test *t = data; + + WRITE_ONCE(t->pause_at, -1); + complete_all(&t->resume); + complete_all(&t->hotplug_release); + if (t->teardown) + wait_for_completion_timeout(&t->torn_down, 10 * HZ); + disable_delayed_work_sync(&t->dcp.announce_wq); + cancel_work_sync(&t->con.hotplug_wq); +} + +/* + * A display on cable generation 2, connected, whose modes are not in: no + * catalogue has arrived for this connection. + */ +static struct announce_test *announce_test_init(struct kunit *test) +{ + struct announce_test *t; + struct apple_dcp *dcp; + + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t); + dcp = &t->dcp; + t->pdev = kunit_platform_device_alloc(test, "dcp-announce", PLATFORM_DEVID_AUTO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->pdev); + KUNIT_ASSERT_EQ(test, kunit_platform_device_add(test, t->pdev), 0); + platform_set_drvdata(t->pdev, dcp); + + dcp->dev = &t->pdev->dev; + dcp->hw.iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL; + iomfb_queue_init(dcp); + mutex_init(&dcp->hpd_mutex); + mutex_init(&dcp->dcpavserv.lock); + spin_lock_init(&dcp->metadata.lock); + INIT_DELAYED_WORK(&dcp->announce_wq, dcp_announce_work); + INIT_DELAYED_WORK(&dcp->typec_reconnect_wq, announce_test_noop); + INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, announce_test_noop); + INIT_DELAYED_WORK(&dcp->bl_fallback_wq, announce_test_noop); + INIT_DELAYED_WORK(&dcp->backlight.fade_work, announce_test_noop); + INIT_WORK(&dcp->external_stop_work, announce_test_noop); + INIT_WORK(&dcp->vblank_wq, announce_test_noop); + INIT_WORK(&t->con.hotplug_wq, announce_test_event); + mutex_init(&t->drm.mode_config.mutex); + + t->pause_at = -1; + init_completion(&t->paused); + init_completion(&t->resume); + init_completion(&t->hotplug_entered); + init_completion(&t->hotplug_release); + init_completion(&t->hotplug_done); + init_completion(&t->torn_down); + dcp->test_point = announce_test_point; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, announce_test_exit, t), 0); + + t->con.base.dev = &t->drm; + t->con.dcp = t->pdev; + t->con.connected = true; + dcp->connector = &t->con; + dcp->typec_cable_connected = true; + dcp->typec_generation = 2; + dcp_firmware_metadata_connection(dcp, 2, false); + + return t; +} + +/* A hold of @generation whose timeout ends at @deadline. */ +static void announce_test_hold(struct announce_test *t, bool announced, u64 generation, + unsigned long deadline) +{ + struct dcp_announce *a = &t->dcp.announce; + + a->announced = announced; + a->holding = true; + a->timed = true; + a->generation = generation; + a->deadline = deadline; +} + +static bool announce_test_detect(struct announce_test *t) +{ + bool connected = dcp_connector_connected(&t->con); + + flush_work(&t->con.hotplug_wq); + return connected; +} + +/* + * A new connection whose timeout expired, probed before the timeout work + * runs: the probe reports it and stops the timer, so it has to queue the + * hotplug event the timer would have. Nothing else will. + */ +static void announce_cold_timeout_probe_test(struct kunit *test) +{ + struct announce_test *t = announce_test_init(test); + + announce_test_hold(t, false, 2, jiffies - 1); + mod_delayed_work(system_freezable_wq, &t->dcp.announce_wq, 60 * HZ); + + KUNIT_ASSERT_TRUE(test, announce_test_detect(t)); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->dcp.announce_wq)); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 1); +} + +/* + * A held event whose timeout expired, released by a probe while the + * timeout work is still running: the event goes out once. + */ +static void announce_timer_probe_once_test(struct kunit *test) +{ + struct announce_test *t = announce_test_init(test); + + announce_test_hold(t, true, 2, jiffies - 1); + t->pause_at = DCP_TEST_ANNOUNCE_TIMER_DONE; + mod_delayed_work(system_freezable_wq, &t->dcp.announce_wq, 0); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&t->paused, HZ), 0); + + KUNIT_EXPECT_TRUE(test, announce_test_detect(t)); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 1); + KUNIT_EXPECT_FALSE(test, t->dcp.announce.holding); + + complete_all(&t->resume); + flush_delayed_work(&t->dcp.announce_wq); + flush_work(&t->con.hotplug_wq); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 1); +} + +/* + * The timeout work of a hold that outlived its cable generation, as when + * work is enabled after another sleep: it starts a full timeout for the + * current generation rather than wait out the deadline of the last one. + * Nothing is ready, so no event goes out yet. + */ +static void announce_timer_next_generation_test(struct kunit *test) +{ + struct announce_test *t = announce_test_init(test); + struct dcp_announce *a = &t->dcp.announce; + unsigned long old_deadline = jiffies + HZ, before = jiffies; + + announce_test_hold(t, true, 1, old_deadline); + dcp_announce_work(&t->dcp.announce_wq.work); + flush_work(&t->con.hotplug_wq); + + KUNIT_EXPECT_TRUE(test, a->holding); + KUNIT_EXPECT_EQ(test, a->generation, t->dcp.typec_generation); + KUNIT_EXPECT_TRUE(test, a->timed); + KUNIT_EXPECT_NE(test, a->deadline, old_deadline); + KUNIT_EXPECT_TRUE(test, time_after_eq(a->deadline, before + + msecs_to_jiffies(DCP_ANNOUNCE_RESUME_TIMEOUT_MS))); + KUNIT_EXPECT_TRUE(test, delayed_work_pending(&t->dcp.announce_wq)); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 0); +} + +static int announce_test_unbind(void *data) +{ + struct announce_test *t = data; + + dcp_comp_unbind(t->dcp.dev, NULL, NULL); + complete(&t->torn_down); + return 0; +} + +static int announce_test_shutdown(void *data) +{ + struct announce_test *t = data; + + dcp_platform_shutdown(t->pdev); + complete(&t->torn_down); + return 0; +} + +/* + * The real hotplug worker evaluates a held connection after a teardown that + * retains the DMA resources stopped the timeout work: neither may be left + * queued once the teardown returns. + */ +static void announce_teardown_running_hotplug(struct kunit *test, + int (*teardown)(void *data)) +{ + struct announce_test *t = announce_test_init(test); + struct apple_dcp *dcp = &t->dcp; + + dcp->retain_dma = true; + announce_test_hold(t, true, 2, jiffies + 60 * HZ); + mod_delayed_work(system_freezable_wq, &dcp->announce_wq, 60 * HZ); + INIT_WORK(&t->con.hotplug_wq, announce_test_running_hotplug); + schedule_work(&t->con.hotplug_wq); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&t->hotplug_entered, HZ), 0); + + t->pause_at = DCP_TEST_UNBIND_ANNOUNCE_STOPPED; + t->teardown = kthread_run(teardown, t, "dcp-announce-teardown"); + KUNIT_ASSERT_FALSE(test, IS_ERR(t->teardown)); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&t->paused, HZ), 0); + complete(&t->hotplug_release); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&t->hotplug_done, HZ), 0); + complete_all(&t->resume); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&t->torn_down, HZ), 0); + + KUNIT_EXPECT_TRUE(test, dcp->quiescing); + KUNIT_EXPECT_TRUE(test, dcp->announce.holding); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&dcp->announce_wq)); + KUNIT_EXPECT_FALSE(test, work_pending(&t->con.hotplug_wq)); +} + +static void announce_unbind_running_hotplug_test(struct kunit *test) +{ + announce_teardown_running_hotplug(test, announce_test_unbind); +} + +static void announce_shutdown_running_hotplug_test(struct kunit *test) +{ + announce_teardown_running_hotplug(test, announce_test_shutdown); +} + +/* + * The DRM device outlives a teardown that retains the DMA resources, so + * userspace can still probe the connector. A probe that ends an expired + * hold queues no work for the stopped pipeline. + */ +static void announce_probe_after_unbind_test(struct kunit *test) +{ + struct announce_test *t = announce_test_init(test); + + t->dcp.retain_dma = true; + announce_test_hold(t, true, 2, jiffies - 1); + dcp_comp_unbind(t->dcp.dev, NULL, NULL); + + KUNIT_EXPECT_TRUE(test, announce_test_detect(t)); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 0); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->dcp.announce_wq)); +} + +/* + * A new connection released by a probe while its work is disabled for + * sleep: its event goes out once the work is enabled again. + */ +static void announce_release_while_stopped_test(struct kunit *test) +{ + struct announce_test *t = announce_test_init(test); + + announce_test_hold(t, false, 2, jiffies + 60 * HZ); + dcp_disable_typec_work(&t->dcp, false); + t->dcp.external_error = -ETIMEDOUT; + + KUNIT_EXPECT_TRUE(test, announce_test_detect(t)); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 0); + + dcp_enable_typec_work(&t->dcp); + flush_work(&t->con.hotplug_wq); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 1); +} + +/* + * A hold whose timeout had just started when the system slept again. The + * timeout work that enabling queues runs at once, not at the deadline of + * the last cable generation, and waits a full timeout for the new one, as + * for a hold that crossed a single sleep. Nothing flushes that work, as + * a flush runs delayed work at once. + */ +static void announce_test_sleep_again(struct kunit *test, bool announced, + unsigned int timeout_ms) +{ + struct announce_test *t = announce_test_init(test); + struct dcp_announce *a = &t->dcp.announce; + unsigned long timeout = msecs_to_jiffies(timeout_ms), before; + + announce_test_hold(t, announced, 2, jiffies + timeout); + mod_delayed_work(system_freezable_wq, &t->dcp.announce_wq, timeout); + dcp_disable_typec_work(&t->dcp, false); + t->pause_at = DCP_TEST_ANNOUNCE_TIMER_DONE; + before = jiffies; + dcp_enable_typec_work(&t->dcp); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&t->paused, timeout / 2), 0); + + KUNIT_EXPECT_TRUE(test, a->holding); + KUNIT_EXPECT_EQ(test, a->generation, 3); + KUNIT_EXPECT_TRUE(test, a->timed); + KUNIT_EXPECT_TRUE(test, time_after_eq(a->deadline, before + timeout)); + KUNIT_EXPECT_TRUE(test, time_before_eq(a->deadline, jiffies + timeout)); + KUNIT_EXPECT_TRUE(test, delayed_work_pending(&t->dcp.announce_wq)); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 0); + complete_all(&t->resume); +} + +static void announce_resume_hold_sleep_again_test(struct kunit *test) +{ + announce_test_sleep_again(test, true, DCP_ANNOUNCE_RESUME_TIMEOUT_MS); +} + +static void announce_new_connection_sleep_again_test(struct kunit *test) +{ + announce_test_sleep_again(test, false, DCP_ANNOUNCE_TIMEOUT_MS); +} + +static struct kunit_case external_announce_cases[] = { + KUNIT_CASE(announce_cold_timeout_probe_test), + KUNIT_CASE(announce_timer_probe_once_test), + KUNIT_CASE(announce_timer_next_generation_test), + KUNIT_CASE(announce_unbind_running_hotplug_test), + KUNIT_CASE(announce_shutdown_running_hotplug_test), + KUNIT_CASE(announce_probe_after_unbind_test), + KUNIT_CASE(announce_release_while_stopped_test), + KUNIT_CASE(announce_resume_hold_sleep_again_test), + KUNIT_CASE(announce_new_connection_sleep_again_test), + {} +}; + +static struct kunit_suite external_announce_suite = { + .name = "apple-dcp-external-announce", + .test_cases = external_announce_cases, +}; + +kunit_test_suite(external_announce_suite); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 108c6a42edb99..5f7b2cbf886ed 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -794,11 +794,131 @@ static void dcp_consume_edid_invalidation(struct apple_connector *connector) dcp_clear_connector_edid(connector); } +/* + * The external H17P pipeline learns the modes of each cable connection from + * firmware messages that can lag the link; see dcp-announce.h. + */ +static bool dcp_announce_gated(struct apple_dcp *dcp, + struct apple_connector *connector) +{ + return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && + READ_ONCE(dcp->connector) == connector; +} + +/* + * Queue the timeout work of a held connection: at once for a hold whose + * timeout has not started for the current cable generation, as only that + * work starts it. + */ +void dcp_announce_arm(struct apple_dcp *dcp) +{ + struct dcp_announce *a = &dcp->announce; + unsigned long now = jiffies, delay = 0; + + lockdep_assert_held(&dcp->hpd_mutex); + if (a->stopped || !a->holding) + return; + if (a->timed && a->generation == dcp->typec_generation && + time_before(now, a->deadline)) + delay = a->deadline - now; + mod_delayed_work(system_freezable_wq, &dcp->announce_wq, delay); +} + +/* + * Evaluate what DRM is to report for the connection. @start starts the + * timeout of a held connection; only the timeout work does that. + */ +static enum dcp_announce_status +dcp_announce_eval(struct apple_dcp *dcp, struct apple_connector *connector, + bool start) +{ + struct dcp_announce *a = &dcp->announce; + struct dcp_announce_facts facts = { + .generation = dcp->typec_generation, + .connected = READ_ONCE(connector->connected), + .present = !READ_ONCE(dcp->external_physically_absent), + .failed = READ_ONCE(dcp->external_error) || READ_ONCE(dcp->crashed), + }; + enum dcp_announce_status status; + unsigned long now = jiffies; + bool held = a->holding, announced = a->announced; + + lockdep_assert_held(&dcp->hpd_mutex); + if (facts.connected && facts.present && !facts.failed) + facts.ready = dcp_external_modes_ready(dcp); + status = dcp_announce_update(a, &facts, now); + if (status == DCP_ANNOUNCE_HOLD) { + if (start) + dcp_announce_start(a, now); + dcp_announce_arm(dcp); + } else if (held) { + /* The hold ended: its timeout work has nothing left to do. */ + cancel_delayed_work(&dcp->announce_wq); + if (status == DCP_ANNOUNCE_CONNECTED && !facts.ready && !facts.failed) + dev_info(dcp->dev, "display modes not ready after %u ms\n", + announced ? DCP_ANNOUNCE_RESUME_TIMEOUT_MS : + DCP_ANNOUNCE_TIMEOUT_MS); + } + return status; +} + +/* + * Evaluate the connection outside the hotplug worker. An event that this + * makes owed is the worker's to send, so queue it, unless the announcement + * is stopped: dcp_enable_typec_work() queues it then. + */ +static void dcp_announce_poll(struct apple_dcp *dcp, struct apple_connector *connector, + bool start) +{ + bool owed = dcp_announce_owed(&dcp->announce); + + dcp_announce_eval(dcp, connector, start); + if (!owed && dcp_announce_owed(&dcp->announce) && !dcp->announce.stopped) + schedule_work(&connector->hotplug_wq); +} + +/* + * The timeout of a held connection. It runs on the freezable workqueue, as + * the link work does, so the timeout of a hold starts here, once the link + * can come back for the hold's generation. Everything it does is under + * hpd_mutex: a hold that ended meanwhile, or an announcement stopped for + * teardown or sleep, gets nothing more from here. + */ +void dcp_announce_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, + announce_wq); + struct apple_connector *con = READ_ONCE(dcp->connector); + + scoped_guard(mutex, &dcp->hpd_mutex) { + if (dcp->announce.holding && !dcp->announce.stopped && con && + dcp_announce_gated(dcp, con) && + READ_ONCE(con->dcp) == to_platform_device(dcp->dev)) + dcp_announce_poll(dcp, con, true); + } + dcp_test_point(dcp, DCP_TEST_ANNOUNCE_TIMER_DONE); +} + +/* The connection state DRM reports for @connector. */ +bool dcp_connector_connected(struct apple_connector *connector) +{ + struct platform_device *pdev = READ_ONCE(connector->dcp); + struct apple_dcp *dcp = pdev ? platform_get_drvdata(pdev) : NULL; + + if (!dcp || !dcp_announce_gated(dcp, connector)) + return READ_ONCE(connector->connected); + + guard(mutex)(&dcp->hpd_mutex); + dcp_announce_poll(dcp, connector, false); + return dcp->announce.announced; +} + void dcp_hotplug(struct work_struct *work) { struct apple_connector *connector; struct platform_device *pdev; struct apple_dcp *dcp; + bool connected; int ret; connector = container_of(work, struct apple_connector, hotplug_wq); @@ -827,12 +947,28 @@ void dcp_hotplug(struct work_struct *work) dcp_clear_connector_edid(connector); } + if (dcp_announce_gated(dcp, connector)) { + enum dcp_announce_status status; + bool notify; + + scoped_guard(mutex, &dcp->hpd_mutex) { + status = dcp_announce_eval(dcp, connector, false); + notify = dcp_announce_notify(&dcp->announce, status); + connected = dcp->announce.announced; + } + /* The last missing fact or the timeout queues this work again. */ + if (!notify) + return; + } else { + connected = READ_ONCE(connector->connected); + } + /* * DCP defers link training until we set a display mode. But we set * display modes from atomic_flush, so userspace needs to trigger a * flush, or the CRTC gets no signal. */ - if (connector->base.state && !READ_ONCE(dcp->mode_state.valid) && connector->connected && + if (connector->base.state && !READ_ONCE(dcp->mode_state.valid) && connected && !(dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x())) { drm_connector_set_link_status_property(&connector->base, DRM_MODE_LINK_STATUS_BAD); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 030c0c34ff575..b580b1a978e3a 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -92,6 +92,7 @@ def function(file, name): static void mod_delayed_work(int q, struct delayed_work *w, int t) { (void)q; (void)t; w->work.pending = true; } struct apple_connector { bool connected, edid_invalidation_pending; }; +struct dcp_announce { int ends; }; struct apple_dcp { int external_lock, hpd_mutex; struct { int lock; bool presentation_paused; } iomfb; @@ -107,6 +108,7 @@ def function(file, name): u64 external_generation, external_rx_generation, external_crtc_generation; u64 external_stop_generation, typec_generation; int external_error, mode_state; void *avep; + struct dcp_announce announce; }; struct platform_device { struct apple_dcp *data; }; static struct apple_dcp *platform_get_drvdata(struct platform_device *p) { return p->data; } @@ -140,6 +142,7 @@ def function(file, name): static void dcp_firmware_metadata_connection(struct apple_dcp *d, u64 g, bool c) { (void)d; (void)g; (void)c; } static void dcp_firmware_metadata_announced(struct apple_dcp *d) { (void)d; } +static void dcp_announce_end(struct dcp_announce *a) { a->ends++; } static int dcp_external_disconnect(struct apple_dcp *, bool); static struct apple_dcp display; static void drm_connector_oob_hotplug_event(void *node, int status) From 9f31f4f38f1c939e2f114009f30a692323ba0521 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:31:25 +1100 Subject: [PATCH 1451/2680] drm/apple: Hold the resume hotplug of an external display until it is ready MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit System sleep releases the link of an external H17P display, and with it the proof of a mode such as 1080p at 180 Hz on a two-lane HBR3 link. A CRTC at that mode is resumed disabled with its link marked bad, and a hotplug event has userspace set its mode again. That event goes out from the resume handler, before the link work has even run. Userspace probes at once, finds the 180 Hz mode missing and sets a 60 Hz mode, then sets 180 Hz again once the firmware has announced the display and the mode is admitted, about 0.7 s after thaw. The monitor has to lock onto two links in a row. Have the DCP of such a CRTC send that event itself, once the display's modes are in for the new cable generation, including every check of the 180 Hz admission except the EDID the probe reads. If they are not in 2 s after the freezable link work can run, as with a monitor switched off during sleep, it sends the event anyway. A probe in the meantime finds the display connected with the modes it has, and nothing waits under a lock. Unplugging the display, a new cable or a failed link releases the event at once. Other CRTCs still get their event from the resume handler. The hold arms the timeout work as any other does, so a DCP whose work is still stopped for sleep arms it when that work is enabled. The firmware announces the returning display while the pipe is still stopped, and a failed reconnect sets no fact the worker otherwise hears about, so both now queue a re-evaluation of a held display, unless the announcement is stopped. apple_drm_drop_stale_crtcs() returns the mask of the CRTCs it disabled so that the resume handler knows whose events to hand over. [ryan: adapted to 7.2.10 — struct drm_atomic_state is struct drm_atomic_commit] (cherry picked from commit 921e1d336c99e7a928f59aa0b5c7b6f4eca419d5) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 23 ++- drivers/gpu/drm/apple/apple_resume.c | 10 +- drivers/gpu/drm/apple/apple_resume.h | 2 +- drivers/gpu/drm/apple/apple_resume_test.c | 10 +- drivers/gpu/drm/apple/dcp.c | 56 ++++++ drivers/gpu/drm/apple/dcp.h | 1 + .../gpu/drm/apple/dcp_external_resume_test.c | 159 ++++++++++++++++++ 7 files changed, 248 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index a338e0a2fd008..421dbf8d2a68f 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -942,16 +942,35 @@ static int apple_platform_suspend(struct device *dev) return 0; } +/* + * The DCP of an external display sends the hotplug event for its CRTC + * itself, once the modes of the new link are in; see dcp_hold_resume_event(). + * Returns false if any of @crtcs needs the event now. + */ +static bool apple_drm_hold_resume_events(struct apple_drm_private *apple, u32 crtcs) +{ + struct drm_crtc *crtc; + bool held = true; + + drm_for_each_crtc(crtc, &apple->drm) { + if (crtcs & drm_crtc_mask(crtc)) + held &= dcp_hold_resume_event(to_apple_crtc(crtc)->dcp); + } + return held; +} + static int apple_platform_resume(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); - bool dropped = false; + u32 dropped = 0; if (apple) { if (apple->drm.mode_config.suspend_state) dropped = apple_drm_drop_stale_crtcs(apple->drm.mode_config.suspend_state); - drm_mode_config_helper_resume(&apple->drm); /* Have userspace set the mode of a CRTC resumed disabled again. */ + if (dropped && apple_drm_hold_resume_events(apple, dropped)) + dropped = 0; + drm_mode_config_helper_resume(&apple->drm); if (dropped) drm_kms_helper_hotplug_event(&apple->drm); } diff --git a/drivers/gpu/drm/apple/apple_resume.c b/drivers/gpu/drm/apple/apple_resume.c index caf1c814c5b98..dc8e07b1a6e1f 100644 --- a/drivers/gpu/drm/apple/apple_resume.c +++ b/drivers/gpu/drm/apple/apple_resume.c @@ -27,9 +27,9 @@ * The suspend state holds every CRTC, plane and connector already, so it is * edited directly rather than through helpers that need the modeset locks. * - * Returns true if a CRTC is to be resumed disabled. + * Returns the mask of the CRTCs to be resumed disabled. */ -bool apple_drm_drop_stale_crtcs(struct drm_atomic_commit *state) +u32 apple_drm_drop_stale_crtcs(struct drm_atomic_commit *state) { const struct drm_crtc_helper_funcs *funcs; struct drm_connector_state *conn_state; @@ -38,7 +38,7 @@ bool apple_drm_drop_stale_crtcs(struct drm_atomic_commit *state) struct drm_connector *connector; struct drm_plane *plane; struct drm_crtc *crtc; - bool dropped = false; + u32 dropped = 0; int i, j; for_each_new_crtc_in_state(state, crtc, crtc_state, i) { @@ -64,7 +64,7 @@ bool apple_drm_drop_stale_crtcs(struct drm_atomic_commit *state) continue; /* Detaching from a CRTC already in the state cannot fail. */ if (WARN_ON(drm_atomic_set_crtc_for_connector(conn_state, NULL))) - return true; + return dropped | drm_crtc_mask(crtc); /* * Nor does a connector without a CRTC keep an encoder: * the resume check warns about one left routed. @@ -76,7 +76,7 @@ bool apple_drm_drop_stale_crtcs(struct drm_atomic_commit *state) conn_state->link_status = DRM_LINK_STATUS_BAD; } crtc_state->active = false; - dropped = true; + dropped |= drm_crtc_mask(crtc); if (WARN_ON(drm_atomic_set_mode_for_crtc(crtc_state, NULL))) break; } diff --git a/drivers/gpu/drm/apple/apple_resume.h b/drivers/gpu/drm/apple/apple_resume.h index 20b4c2951463f..2bb1002cbe4bb 100644 --- a/drivers/gpu/drm/apple/apple_resume.h +++ b/drivers/gpu/drm/apple/apple_resume.h @@ -8,6 +8,6 @@ struct drm_atomic_commit; -bool apple_drm_drop_stale_crtcs(struct drm_atomic_commit *state); +u32 apple_drm_drop_stale_crtcs(struct drm_atomic_commit *state); #endif /* __APPLE_RESUME_H__ */ diff --git a/drivers/gpu/drm/apple/apple_resume_test.c b/drivers/gpu/drm/apple/apple_resume_test.c index f9144fa02babb..878e0ab8b59d6 100644 --- a/drivers/gpu/drm/apple/apple_resume_test.c +++ b/drivers/gpu/drm/apple/apple_resume_test.c @@ -187,7 +187,7 @@ static void resume_stale_crtc_disabled_test(struct kunit *test) priv->admissible = false; state = priv->drm.mode_config.suspend_state; /* The caller tells userspace about the CRTC it had and lost. */ - KUNIT_EXPECT_TRUE(test, apple_drm_drop_stale_crtcs(state)); + KUNIT_EXPECT_EQ(test, apple_drm_drop_stale_crtcs(state), drm_crtc_mask(priv->crtc)); /* The resume check warns about an encoder routed without a CRTC. */ conn_state = drm_atomic_get_new_connector_state(state, &priv->connector); @@ -218,8 +218,8 @@ static void resume_admissible_crtc_restored_test(struct kunit *test) struct resume_test *priv = resume_test_init(test); KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); - KUNIT_EXPECT_FALSE(test, - apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state)); + KUNIT_EXPECT_EQ(test, apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state), + 0U); KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); KUNIT_EXPECT_TRUE(test, priv->crtc->state->enable); @@ -256,8 +256,8 @@ static void resume_stale_dpms_off_crtc_disabled_test(struct kunit *test) KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); priv->admissible = false; - KUNIT_EXPECT_TRUE(test, - apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state)); + KUNIT_EXPECT_EQ(test, apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state), + drm_crtc_mask(priv->crtc)); KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); KUNIT_EXPECT_FALSE(test, priv->crtc->state->enable); KUNIT_EXPECT_NULL(test, priv->connector.state->crtc); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 3904642a7cb5a..a536adca44645 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1977,6 +1977,21 @@ static void dcp_firmware_metadata_present_locked(struct apple_dcp *dcp) dcp->metadata.epoch.connected = true; } +/* + * A fact a held connection waits for has changed: evaluate it again. The + * unlocked checks may miss a hold that has just started; such a hold ends at + * its timeout instead. A stopped announcement is evaluated again when its + * work is enabled; teardown quiesces the callers before draining the worker. + */ +static void dcp_announce_kick(struct apple_dcp *dcp) +{ + struct apple_connector *con = READ_ONCE(dcp->connector); + + if (READ_ONCE(dcp->announce.holding) && !READ_ONCE(dcp->announce.stopped) && con && + READ_ONCE(con->dcp) == to_platform_device(dcp->dev)) + schedule_work(&con->hotplug_wq); +} + /* * While its pipe is stopped an external display reports no hotplug to DRM * (see dcp_external_hotplug()). When the firmware announces the display @@ -1997,6 +2012,7 @@ void dcp_firmware_metadata_announced(struct apple_dcp *dcp) spin_lock_irqsave(&dcp->metadata.lock, flags); dcp_firmware_metadata_present_locked(dcp); spin_unlock_irqrestore(&dcp->metadata.lock, flags); + dcp_announce_kick(dcp); } void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected) @@ -2984,6 +3000,9 @@ static void dcp_typec_reconnect_work(struct work_struct *work) dcp->typec_reconnect_tries = 0; return; } + /* A display held for this link will not get ready. */ + if (READ_ONCE(dcp->external_error)) + dcp_announce_kick(dcp); if (++dcp->typec_reconnect_tries < (dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x() ? @@ -2997,6 +3016,43 @@ static void dcp_typec_reconnect_work(struct work_struct *work) dcp->typec_reconnect_tries, ret); } +/** + * dcp_hold_resume_event - hold the resume hotplug event of an external display + * @pdev: the DCP of a CRTC resumed disabled + * + * System sleep releases the link of an external H17P display, and with it + * the proof of modes such as 1080p at 180 Hz, so apple_drm_drop_stale_crtcs() + * resumes the CRTC of such a mode disabled with its link marked bad. A + * hotplug event then has userspace set the mode again. Sent at once, it has + * userspace probe before the link is back, pick a fallback mode, and set its + * own mode again once that is offered: two mode sets the monitor has to + * resynchronize to. + * + * Hold that event, not the connection, until the display's modes are in for + * the new cable generation, or for DCP_ANNOUNCE_RESUME_TIMEOUT_MS after the + * link work can run again, then send it once from the hotplug worker. No + * lock is held while waiting; a probe in the meantime reports the display + * connected with the modes it has. + * + * Returns false if the caller must send the event itself. + */ +bool dcp_hold_resume_event(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + struct apple_connector *con = READ_ONCE(dcp->connector); + + if (!dcp_is_external_h17p(dcp) || !con || READ_ONCE(con->dcp) != pdev) + return false; + + guard(mutex)(&dcp->hpd_mutex); + if (!READ_ONCE(con->connected) || !dcp->typec_cable_connected || + !dcp_announce_hold(&dcp->announce, dcp->typec_generation)) + return false; + /* Freezable, like the link work: the timeout starts after thaw. */ + dcp_announce_arm(dcp); + return true; +} + static void disconnected_hpd_event(struct apple_connector *con) { if (!con) diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index d57c533b63822..266083c14a546 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -84,6 +84,7 @@ void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message); int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port); int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); +bool dcp_hold_resume_event(struct platform_device *pdev); struct apple_dcp; struct phy; diff --git a/drivers/gpu/drm/apple/dcp_external_resume_test.c b/drivers/gpu/drm/apple/dcp_external_resume_test.c index 409a8cd8ffd8b..cf3db9324d3ec 100644 --- a/drivers/gpu/drm/apple/dcp_external_resume_test.c +++ b/drivers/gpu/drm/apple/dcp_external_resume_test.c @@ -2,9 +2,13 @@ /* Copyright 2026 Ryan Murray */ #include +#include #include #include +#include +#include "connector.h" +#include "dcp.h" #include "dcp-internal.h" /* @@ -125,11 +129,166 @@ static void external_paused_released_link_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, dcp->metadata.epoch.connected); } +/* + * The same display as apple_platform_resume() finds it: reported before the + * sleep, its CRTC just resumed disabled, the link not back yet. + */ +struct resume_hold_test { + struct platform_device *pdev; + struct apple_dcp *dcp; + struct apple_connector *con; + struct apple_connector connector; + atomic_t events; + atomic_t timer_runs; +}; + +static void resume_hold_event(struct work_struct *work) +{ + struct resume_hold_test *t = container_of(work, struct resume_hold_test, + connector.hotplug_wq); + + atomic_inc(&t->events); +} + +static void resume_hold_timer(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, + announce_wq); + struct resume_hold_test *t = container_of(dcp->connector, struct resume_hold_test, + connector); + + atomic_inc(&t->timer_runs); +} + +static void resume_hold_exit(void *data) +{ + struct resume_hold_test *t = data; + + cancel_delayed_work_sync(&t->dcp->announce_wq); + cancel_work_sync(&t->con->hotplug_wq); +} + +static struct resume_hold_test *resume_hold_init(struct kunit *test) +{ + struct resume_hold_test *t; + struct apple_dcp *dcp; + + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t); + dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dcp); + t->dcp = dcp; + t->con = &t->connector; + t->pdev = kunit_platform_device_alloc(test, "dcp-resume-hold", PLATFORM_DEVID_AUTO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->pdev); + KUNIT_ASSERT_EQ(test, kunit_platform_device_add(test, t->pdev), 0); + platform_set_drvdata(t->pdev, dcp); + + dcp->dev = &t->pdev->dev; + dcp->hw.iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL; + mutex_init(&dcp->hpd_mutex); + mutex_init(&dcp->dcpavserv.lock); + spin_lock_init(&dcp->metadata.lock); + INIT_DELAYED_WORK(&dcp->announce_wq, resume_hold_timer); + INIT_WORK(&t->con->hotplug_wq, resume_hold_event); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, resume_hold_exit, t), 0); + + t->con->dcp = t->pdev; + t->con->connected = true; + dcp->connector = t->con; + dcp->announce.announced = true; + dcp->typec_cable_connected = true; + dcp->typec_generation = 2; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); + + return t; +} + +static bool resume_hold_detect(struct resume_hold_test *t) +{ + bool connected = dcp_connector_connected(t->con); + + flush_work(&t->con->hotplug_wq); + return connected; +} + +/* The event is held, never the connection: a probe still finds it. */ +static void resume_hold_armed_test(struct kunit *test) +{ + struct resume_hold_test *t = resume_hold_init(test); + + KUNIT_EXPECT_TRUE(test, dcp_hold_resume_event(t->pdev)); + KUNIT_EXPECT_TRUE(test, t->dcp->announce.holding); + KUNIT_EXPECT_EQ(test, t->dcp->announce.generation, 2); + /* + * The timer work starts the timeout once it can run, so it is queued + * at once. It may have run already: count its runs after a flush. + */ + KUNIT_EXPECT_FALSE(test, t->dcp->announce.timed); + flush_delayed_work(&t->dcp->announce_wq); + KUNIT_EXPECT_EQ(test, atomic_read(&t->timer_runs), 1); + + KUNIT_EXPECT_TRUE(test, resume_hold_detect(t)); + KUNIT_EXPECT_TRUE(test, t->dcp->announce.holding); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 0); +} + +/* A connection never reported has no resume event to hold. */ +static void resume_hold_unreported_test(struct kunit *test) +{ + struct resume_hold_test *t = resume_hold_init(test); + + t->dcp->announce.announced = false; + KUNIT_EXPECT_FALSE(test, dcp_hold_resume_event(t->pdev)); + KUNIT_EXPECT_FALSE(test, t->dcp->announce.holding); +} + +/* Other pipelines keep sending the event at once. */ +static void resume_hold_internal_test(struct kunit *test) +{ + struct resume_hold_test *t = resume_hold_init(test); + + t->dcp->hw.iomfb_method_profile = DCP_IOMFB_METHODS_DEFAULT; + KUNIT_EXPECT_FALSE(test, dcp_hold_resume_event(t->pdev)); + KUNIT_EXPECT_FALSE(test, t->dcp->announce.holding); +} + +/* A cable unplugged during the sleep is reported at once, with one event. */ +static void resume_hold_unplug_test(struct kunit *test) +{ + struct resume_hold_test *t = resume_hold_init(test); + + KUNIT_ASSERT_TRUE(test, dcp_hold_resume_event(t->pdev)); + t->dcp->external_physically_absent = true; + + KUNIT_EXPECT_FALSE(test, resume_hold_detect(t)); + KUNIT_EXPECT_FALSE(test, t->dcp->announce.holding); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 1); +} + +/* A link that failed will not get ready: the event is not held for it. */ +static void resume_hold_failed_link_test(struct kunit *test) +{ + struct resume_hold_test *t = resume_hold_init(test); + + KUNIT_ASSERT_TRUE(test, dcp_hold_resume_event(t->pdev)); + t->dcp->external_error = -ETIMEDOUT; + + KUNIT_EXPECT_TRUE(test, resume_hold_detect(t)); + KUNIT_EXPECT_FALSE(test, t->dcp->announce.holding); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 1); +} + static struct kunit_case external_resume_cases[] = { KUNIT_CASE(external_paused_announce_recorded_test), KUNIT_CASE(external_paused_unplug_ignored_test), KUNIT_CASE(external_paused_stale_announce_test), KUNIT_CASE(external_paused_released_link_test), + KUNIT_CASE(resume_hold_armed_test), + KUNIT_CASE(resume_hold_unreported_test), + KUNIT_CASE(resume_hold_internal_test), + KUNIT_CASE(resume_hold_unplug_test), + KUNIT_CASE(resume_hold_failed_link_test), {} }; From dc9a33aac67a3db89cf77c428780de084b26f985 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 15:13:55 +1100 Subject: [PATCH 1452/2680] drm/apple: Let an external display announcement lose its connection An external display can come back on a new link while userspace still scans out to it through an active CRTC. A Type-C route bounce, for instance, tears the DisplayPort link down and brings it back within a second or two, quicker than a compositor reconfigures. If the driver then cannot set the mode of that CRTC again, the display stays dark while DRM still reports it connected, and a compositor that finds the display it had on its next probe has no reason to set a mode. Teach the announcement state machine a lost connection: one whose link could not be set up again on its cable generation. It is reported gone, which owes a hotplug event, until userspace releases the CRTC (dcp_announce_retry()) or the cable generation changes. It is then a new connection, held until its modes are in like any other. The loss is reported with one event. A connection that stays gone sends no more, whatever facts arrive meanwhile: each repeated event would only restart a compositor's wait for the output to stay disconnected (Mutter debounces a disconnect for three seconds and restarts that wait on every event). Nothing loses a connection yet. (cherry picked from commit 1e54dce092ecae6d341040d8e79f376ff0bde6b8) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-announce-test.c | 89 +++++++++++++++++++++++ drivers/gpu/drm/apple/dcp-announce.h | 57 ++++++++++++++- 2 files changed, 142 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-announce-test.c b/drivers/gpu/drm/apple/dcp-announce-test.c index bdfd9ef4db774..ec5d7dd250cc3 100644 --- a/drivers/gpu/drm/apple/dcp-announce-test.c +++ b/drivers/gpu/drm/apple/dcp-announce-test.c @@ -367,6 +367,91 @@ static void hold_again_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, dcp_announce_owed(&a)); } +/* A reported connection whose link was not restored is withdrawn once. */ +static void lost_withdraws_test(struct kunit *test) +{ + struct dcp_announce_facts lost = link_ready; + struct dcp_announce a = { }; + + KUNIT_ASSERT_EQ(test, dcp_announce_update(&a, &link_ready, 1), + DCP_ANNOUNCE_CONNECTED); + KUNIT_ASSERT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_CONNECTED)); + + dcp_announce_lose(&a, 1); + KUNIT_EXPECT_TRUE(test, dcp_announce_lost(&a, 1)); + lost.lost = true; + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &lost, 2), + DCP_ANNOUNCE_DISCONNECTED); + KUNIT_EXPECT_FALSE(test, a.announced); + KUNIT_EXPECT_TRUE(test, dcp_announce_owed(&a)); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_DISCONNECTED)); + KUNIT_EXPECT_FALSE(test, dcp_announce_owed(&a)); + + /* Modes still in for the lost generation do not bring it back. */ + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &lost, 3), + DCP_ANNOUNCE_DISCONNECTED); + KUNIT_EXPECT_FALSE(test, dcp_announce_owed(&a)); + /* Nor does an unchanged loss send its event again. */ + KUNIT_EXPECT_FALSE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_DISCONNECTED)); +} + +/* A loss belongs to its cable generation: the next cable is not lost. */ +static void lost_generation_test(struct kunit *test) +{ + struct dcp_announce a = { }; + + dcp_announce_lose(&a, 1); + KUNIT_EXPECT_TRUE(test, dcp_announce_lost(&a, 1)); + KUNIT_EXPECT_FALSE(test, dcp_announce_lost(&a, 2)); + + /* Releasing a CRTC on the next cable retries nothing, and forgets the loss. */ + KUNIT_EXPECT_FALSE(test, dcp_announce_retry(&a, 2)); + KUNIT_EXPECT_FALSE(test, dcp_announce_lost(&a, 1)); +} + +/* A lost connection whose CRTC was released is a new connection. */ +static void lost_retry_test(struct kunit *test) +{ + struct dcp_announce_facts lost = link_only; + struct dcp_announce a = { }; + + KUNIT_ASSERT_EQ(test, dcp_announce_update(&a, &link_ready, 1), + DCP_ANNOUNCE_CONNECTED); + dcp_announce_lose(&a, 1); + lost.lost = true; + KUNIT_ASSERT_EQ(test, dcp_announce_update(&a, &lost, 2), + DCP_ANNOUNCE_DISCONNECTED); + KUNIT_ASSERT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_DISCONNECTED)); + + KUNIT_EXPECT_TRUE(test, dcp_announce_retry(&a, 1)); + KUNIT_EXPECT_FALSE(test, dcp_announce_lost(&a, 1)); + KUNIT_EXPECT_FALSE(test, dcp_announce_retry(&a, 1)); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 3), + DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_FALSE(test, a.announced); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_ready, 4), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, dcp_announce_owed(&a)); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_CONNECTED)); +} + +/* A connection lost again after its retry waits for the next cable. */ +static void lost_retry_once_test(struct kunit *test) +{ + struct dcp_announce a = { }; + + dcp_announce_lose(&a, 1); + KUNIT_EXPECT_TRUE(test, dcp_announce_retry(&a, 1)); + dcp_announce_lose(&a, 1); + KUNIT_EXPECT_FALSE(test, dcp_announce_retry(&a, 1)); + KUNIT_EXPECT_TRUE(test, dcp_announce_lost(&a, 1)); + + /* The next cable starts afresh. */ + dcp_announce_lose(&a, 2); + KUNIT_EXPECT_TRUE(test, dcp_announce_retry(&a, 2)); + KUNIT_EXPECT_FALSE(test, dcp_announce_lost(&a, 2)); +} + static struct kunit_case announce_cases[] = { KUNIT_CASE(announce_ready_test), KUNIT_CASE(announce_waits_for_ready_test), @@ -390,6 +475,10 @@ static struct kunit_case announce_cases[] = { KUNIT_CASE(hold_deadline_generation_test), KUNIT_CASE(hold_release_once_test), KUNIT_CASE(hold_again_test), + KUNIT_CASE(lost_withdraws_test), + KUNIT_CASE(lost_generation_test), + KUNIT_CASE(lost_retry_test), + KUNIT_CASE(lost_retry_once_test), {} }; diff --git a/drivers/gpu/drm/apple/dcp-announce.h b/drivers/gpu/drm/apple/dcp-announce.h index cb360e48eb316..8a954b016af52 100644 --- a/drivers/gpu/drm/apple/dcp-announce.h +++ b/drivers/gpu/drm/apple/dcp-announce.h @@ -21,6 +21,12 @@ * connection itself, until it is ready again, or until * DCP_ANNOUNCE_RESUME_TIMEOUT_MS have passed without that. That is the * event that has userspace set the mode of a CRTC resumed disabled. + * - A reported connection whose link came back under an active CRTC, but + * could not be set up again, is lost: it is reported gone until userspace + * releases that CRTC or the cable generation changes, and is then a new + * connection. Releasing the CRTC brings it back once per cable + * generation. Userspace may keep driving an output it already uses + * whatever its link status says, but not a display that went away. * * The timeout of a hold runs from dcp_announce_start(), and only for the * cable generation it was started in. A hold that outlives its generation, @@ -46,7 +52,11 @@ struct dcp_announce { bool holding; /* waiting for the connection to be ready */ bool timed; /* the hold of @generation ends at @deadline */ bool stopped; /* nothing may be queued for the announcement */ + bool lost; /* the connection of @lost_generation is lost */ + bool retried; /* a loss of @retried_generation was retried */ u64 generation; /* the cable generation of the hold */ + u64 lost_generation; + u64 retried_generation; unsigned long deadline; }; @@ -56,6 +66,7 @@ struct dcp_announce_facts { bool present; /* the cable is not known to be gone */ bool ready; /* a probe would list the modes of this connection */ bool failed; /* the link failed; it will not get ready */ + bool lost; /* see dcp_announce_lost() */ }; /* The connection ended, whether DRM was told about it or not. */ @@ -97,6 +108,43 @@ static inline bool dcp_announce_expired(const struct dcp_announce *a, u64 genera !time_before(now, a->deadline); } +/* + * The link of the active CRTC of the connection on cable generation + * @generation could not be set up again. + */ +static inline void dcp_announce_lose(struct dcp_announce *a, u64 generation) +{ + a->lost = true; + a->lost_generation = generation; +} + +static inline bool dcp_announce_lost(const struct dcp_announce *a, u64 generation) +{ + return a->lost && a->lost_generation == generation; +} + +/* + * Userspace released the CRTC of the connection on cable generation + * @generation. A lost connection is a new connection from now on, once per + * cable generation: one lost again after that stays lost until the cable + * generation changes, so that a link that cannot come up does not have + * userspace switch between the display and none for as long as it is + * attached. Returns whether the connection is to be announced again. + */ +static inline bool dcp_announce_retry(struct dcp_announce *a, u64 generation) +{ + if (!dcp_announce_lost(a, generation)) { + a->lost = false; + return false; + } + if (a->retried && a->retried_generation == generation) + return false; + a->lost = false; + a->retried = true; + a->retried_generation = generation; + return true; +} + /* Whether a hotplug event is owed that dcp_announce_notify() has not handed out. */ static inline bool dcp_announce_owed(const struct dcp_announce *a) { @@ -107,7 +155,7 @@ static inline enum dcp_announce_status dcp_announce_update(struct dcp_announce *a, const struct dcp_announce_facts *f, unsigned long now) { - if (!f->connected || !f->present) { + if (!f->connected || !f->present || f->lost) { dcp_announce_end(a); return DCP_ANNOUNCE_DISCONNECTED; } @@ -133,13 +181,14 @@ dcp_announce_update(struct dcp_announce *a, const struct dcp_announce_facts *f, /* * Whether a hotplug event is due, which hands out any event owed. A held - * connection changes nothing userspace can see, unless a reported one ended - * before it. + * connection, or one that stays gone, changes nothing userspace can see, + * unless a reported one ended before it. Repeating the event of a lost + * connection would only restart a compositor's wait for it to stay gone. */ static inline bool dcp_announce_notify(struct dcp_announce *a, enum dcp_announce_status status) { - bool due = status != DCP_ANNOUNCE_HOLD || a->withdrawn; + bool due = status == DCP_ANNOUNCE_CONNECTED || a->withdrawn; if (due) { a->withdrawn = false; From ad87212941dba5aa1ae341e1115f0f164850d60c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 15:13:55 +1100 Subject: [PATCH 1453/2680] drm/apple: Set the mode of an external display again when its link returns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the DisplayPort route of an external H17P display bounces under an active CRTC, the hotplug worker finds the display back without a valid DCP mode and retrains the CRTC with drm_atomic_helper_reset_crtc(). For the 180 Hz exception mode that commit fails its mode fixup: the mode is admitted only against the EDID a probe read for the current cable generation, and nothing has probed the new link yet. The retrain fails with -EINVAL, the link status is left bad and nothing tries again. A compositor that keeps driving an output whatever its link status says, and that finds the same display and modes on its next probe, then keeps the CRTC on a dead link and the monitor stays dark. On the external H17P pipeline, make that recovery end either in a picture or in a disconnect: - If userspace keeps a CRTC active on the connector while its link is down, probe the connector first, as userspace would, so the mode admission sees the EDID of the current link. Then set the mode of the CRTC again, with its link status good. - If that fails, or leaves the link down, mark the link bad and lose the connection. The display is reported disconnected, with a hotplug event, until userspace switches the CRTC off (dcp_poweroff() calls dcp_external_crtc_released()) or the cable generation changes. It is then announced again as a new connection. Both are decided under the modeset locks of the mode set, so a CRTC that userspace switches off meanwhile is either off before the attempt, and left alone, or found lost when it goes off. A route that bounces again during the attempt leaves the link to the newer connection. A mode set of the external display that fails, including one refused because its pipe did not start, now also queues the hotplug worker: atomic_enable cannot report the failure, and the CRTC would otherwise stay active without a link. A pipe that did not start also takes its old mode with it, so that the worker does not take the previous mode for a link; a pipe whose presentation is only paused keeps its mode. A lost connection comes back once per cable generation when its CRTC is switched off, so a link that cannot come up does not have userspace switch the display on and off while it stays attached. The recovery never writes external_error and does not wait for it to clear: a latched failure only makes the mode set fail like any other, and the physical unplug that releases the latch also starts a new cable generation. The worker gets here only when it sends its event, so after the hold of a new connection has ended, and the 180 Hz admission rules are unchanged. Other DCP pipelines, the M1 and M2 ones included, keep the existing retraining. A KUnit suite runs the real hotplug worker, detect() and timeout work against a DRM KUnit device whose CRTC admits its mode the way the exception mode is admitted. It covers a bounce under an active CRTC, the mode gone on the new link, a mode set that leaves the link down, a bounce after a loss, a bounce during the probe, an inactive CRTC, a connector without state yet, userspace switching the CRTC off during the mode set, a link that never comes up, and a DCP of another kind. A new test point counts the hotplug events. Four cases run the real dcp_crtc_atomic_modeset() and H17P mode set path, with a test hook answering IOMFB method calls in place of the firmware: the bounce, a pipe that was stopped while its loss was buffered and then fails to activate, a paused presentation, and unchanged facts after a reported loss, which must not send more events. [ryan: adapted to 7.2.10 — struct drm_atomic_state is struct drm_atomic_commit, allocated and released with drm_atomic_commit_alloc() and drm_atomic_commit_put()] (cherry picked from commit 10bab06f30a5272c7ab64653869ef1b3af3bc5f2) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/dcp-internal.h | 6 + drivers/gpu/drm/apple/dcp.c | 1 + .../gpu/drm/apple/dcp_external_relink_test.c | 849 ++++++++++++++++++ drivers/gpu/drm/apple/iomfb.c | 201 ++++- 5 files changed, 1053 insertions(+), 5 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp_external_relink_test.c diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 0f0b80bb43ef3..645279b811131 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -18,6 +18,7 @@ appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += parser_mode_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_resume_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_ready_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_announce_test.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_relink_test.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 584bf3b71f182..072bd2f2146b6 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -36,6 +36,7 @@ struct apple_dcp; struct apple_dcp_afkep; +struct dcp_method_entry; /* Snapshot of a completed present, independent of the latest DRM state. */ struct iomfb_scanout_h17p { @@ -210,6 +211,7 @@ struct apple_dcp_hw_data { enum dcp_test_point { DCP_TEST_ANNOUNCE_TIMER_DONE, /* past hpd_mutex in the timeout work */ DCP_TEST_UNBIND_ANNOUNCE_STOPPED, /* unbind drained the timeout work */ + DCP_TEST_HOTPLUG_EVENT, /* the hotplug worker sends its event */ }; /* TODO: move IOMFB members to its own struct */ @@ -455,6 +457,9 @@ struct apple_dcp { #if IS_ENABLED(CONFIG_DRM_APPLE_KUNIT_TEST) /* Lets a test stop work where it races, see dcp_test_point(). */ void (*test_point)(struct apple_dcp *dcp, enum dcp_test_point point); + /* Lets a test answer IOMFB method calls in place of the firmware. */ + void (*test_push)(struct apple_dcp *dcp, const struct dcp_method_entry *call, + dcp_callback_t cb, void *cookie); #endif struct gpio_desc *hdmi_hpd; @@ -550,6 +555,7 @@ void dcp_handle_hotplug_actions(struct apple_dcp *dcp, unsigned int action); bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected); void dcp_announce_arm(struct apple_dcp *dcp); void dcp_announce_work(struct work_struct *work); +void dcp_external_crtc_released(struct apple_dcp *dcp); /* A KUnit test can wait here for the work it races with. */ static inline void dcp_test_point(struct apple_dcp *dcp, enum dcp_test_point point) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a536adca44645..9d41a32eeeac7 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3548,6 +3548,7 @@ void dcp_poweroff(struct platform_device *pdev) ret = dcp_external_stop_locked(dcp, dcp->external_crtc_generation); if (ret) dev_err(dcp->dev, "external display stop failed: %d\n", ret); + dcp_external_crtc_released(dcp); return; } diff --git a/drivers/gpu/drm/apple/dcp_external_relink_test.c b/drivers/gpu/drm/apple/dcp_external_relink_test.c new file mode 100644 index 0000000000000..5b48c42610a35 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_external_relink_test.c @@ -0,0 +1,849 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "connector.h" +#include "dcp.h" +#include "dcp-internal.h" +#include "iomfb.h" +#include "parser.h" + +/* + * An external H17P display whose DisplayPort route goes away and comes back + * while userspace keeps its CRTC active, through the real hotplug worker, + * detect(), timeout work and atomic commits. The CRTC stands in for the DCP + * pipe: as for the 180 Hz exception mode, its mode is admitted only once a + * probe has read the EDID of the current cable generation, and setting it + * brings the link up unless the test says otherwise. + */ +struct relink_test { + struct drm_device drm; + struct apple_dcp dcp; + struct apple_connector con; + struct platform_device *pdev; + struct drm_plane *plane; + struct drm_crtc *crtc; + struct drm_encoder encoder; + struct drm_framebuffer fb; + struct drm_display_mode *mode; + + u64 probed_generation; /* the cable generation of the last probe */ + bool offered; /* the display offers the mode */ + bool trains; /* setting the mode brings the link up */ + bool bounce_on_probe; /* the route bounces again during a probe */ + bool real_modeset; /* atomic_enable is the real dcp_crtc_atomic_modeset() */ + bool force_modeset; /* a userspace commit changes the connectors */ + u32 reply; /* what the firmware answers a call with */ + int modeset_ret; /* the result of the real mode set */ + struct dcp_display_mode catalog; + bool hold_modeset; /* a mode set waits for @modeset_release */ + struct completion modeset_entered, modeset_release, userspace_done; + int userspace_ret; + atomic_t probes, checks, modesets, events; +}; + +static struct relink_test *to_relink_test(struct drm_device *drm) +{ + return container_of(drm, struct relink_test, drm); +} + +static void relink_test_noop(struct work_struct *work) +{ +} + +static void relink_test_point(struct apple_dcp *dcp, enum dcp_test_point point) +{ + struct relink_test *t = container_of(dcp, struct relink_test, dcp); + + if (point == DCP_TEST_HOTPLUG_EVENT) + atomic_inc(&t->events); +} + +static enum drm_connector_status relink_test_detect(struct drm_connector *connector, bool force) +{ + return dcp_connector_connected(to_apple_connector(connector)) ? + connector_status_connected : connector_status_disconnected; +} + +/* As dcp_get_modes(): what a probe reads belongs to the current cable. */ +static int relink_test_get_modes(struct drm_connector *connector) +{ + struct relink_test *t = to_relink_test(connector->dev); + struct drm_display_mode *mode; + + atomic_inc(&t->probes); + scoped_guard(mutex, &t->dcp.hpd_mutex) { + t->probed_generation = t->dcp.typec_generation; + if (t->bounce_on_probe) + t->dcp.typec_generation++; + } + if (!t->offered) + return 0; + mode = drm_mode_duplicate(connector->dev, t->mode); + if (!mode) + return 0; + drm_mode_probed_add(connector, mode); + return 1; +} + +/* As lookup_mode() for the exception mode, under the same lock. */ +static bool relink_test_mode_fixup(struct drm_crtc *crtc, + const struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode) +{ + struct relink_test *t = to_relink_test(crtc->dev); + + atomic_inc(&t->checks); + guard(mutex)(&t->dcp.hpd_mutex); + return t->offered && t->probed_generation == t->dcp.typec_generation; +} + +/* + * As dcp_crtc_atomic_modeset(): the mode set trains the link, or fails and + * has the hotplug worker recover the CRTC. + */ +/* The firmware for the real mode set path: it answers every method call. */ +static void relink_test_push(struct apple_dcp *dcp, const struct dcp_method_entry *call, + dcp_callback_t cb, void *cookie) +{ + struct relink_test *t = container_of(dcp, struct relink_test, dcp); + + cb(dcp, &t->reply, cookie); +} + +static void relink_test_crtc_enable(struct drm_crtc *crtc, struct drm_atomic_commit *state) +{ + struct relink_test *t = to_relink_test(crtc->dev); + unsigned int action; + + atomic_inc(&t->modesets); + if (t->hold_modeset) { + complete(&t->modeset_entered); + wait_for_completion(&t->modeset_release); + } + if (t->real_modeset) { + t->modeset_ret = dcp_crtc_atomic_modeset(crtc, state); + return; + } + dcp_mode_begin(&t->dcp.mode_state); + action = dcp_mode_finish(&t->dcp.mode_state, t->trains, &t->con.connected); + if (!t->trains) + action |= DCP_HOTPLUG_NOTIFY; + dcp_handle_hotplug_actions(&t->dcp, action); +} + +/* As dcp_poweroff() for the external display. */ +static void relink_test_crtc_disable(struct drm_crtc *crtc, struct drm_atomic_commit *state) +{ + struct relink_test *t = to_relink_test(crtc->dev); + + if (drm_atomic_get_new_crtc_state(state, crtc)->active) + return; + guard(mutex)(&t->dcp.external_lock); + dcp_mode_invalidate(&t->dcp.mode_state); + dcp_external_crtc_released(&t->dcp); +} + +static const struct drm_crtc_funcs relink_test_crtc_funcs = { + .reset = drm_atomic_helper_crtc_reset, + .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state, +}; + +static const struct drm_crtc_helper_funcs relink_test_crtc_helpers = { + .mode_fixup = relink_test_mode_fixup, + .atomic_enable = relink_test_crtc_enable, + .atomic_disable = relink_test_crtc_disable, +}; + +/* Scanout itself is not under test; the plane helpers require an update. */ +static void relink_test_plane_update(struct drm_plane *plane, struct drm_atomic_commit *state) +{ +} + +static const struct drm_plane_helper_funcs relink_test_plane_helpers = { + .atomic_update = relink_test_plane_update, +}; + +static const struct drm_connector_helper_funcs relink_test_connector_helpers = { + .get_modes = relink_test_get_modes, +}; + +static const struct drm_connector_funcs relink_test_connector_funcs = { + .fill_modes = drm_helper_probe_single_connector_modes, + .detect = relink_test_detect, + .reset = drm_atomic_helper_connector_reset, + .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, +}; + +static void relink_test_fb_destroy(struct drm_framebuffer *fb) +{ + drm_framebuffer_cleanup(fb); +} + +static const struct drm_framebuffer_funcs relink_test_fb_funcs = { + .destroy = relink_test_fb_destroy, +}; + +static void relink_test_exit(void *data) +{ + struct relink_test *t = data; + + complete_all(&t->modeset_release); + wait_for_completion_timeout(&t->userspace_done, 10 * HZ); + disable_delayed_work_sync(&t->dcp.announce_wq); + disable_work_sync(&t->con.hotplug_wq); + drm_atomic_helper_shutdown(&t->drm); + drm_framebuffer_put(&t->fb); +} + +/* Commit @active for the CRTC, as userspace would. */ +static int relink_test_commit(struct relink_test *t, bool active, bool init) +{ + struct drm_connector_state *conn_state; + struct drm_plane_state *plane_state; + struct drm_modeset_acquire_ctx ctx; + struct drm_crtc_state *crtc_state; + struct drm_atomic_commit *state; + int ret; + + drm_modeset_acquire_init(&ctx, 0); +retry: + state = drm_atomic_commit_alloc(&t->drm); + if (!state) { + ret = -ENOMEM; + goto out; + } + state->acquire_ctx = &ctx; + crtc_state = drm_atomic_get_crtc_state(state, t->crtc); + if (IS_ERR(crtc_state)) { + ret = PTR_ERR(crtc_state); + goto put; + } + crtc_state->active = active; + if (t->force_modeset) + crtc_state->connectors_changed = true; + if (init) { + conn_state = drm_atomic_get_connector_state(state, &t->con.base); + if (IS_ERR(conn_state)) { + ret = PTR_ERR(conn_state); + goto put; + } + plane_state = drm_atomic_get_plane_state(state, t->plane); + if (IS_ERR(plane_state)) { + ret = PTR_ERR(plane_state); + goto put; + } + ret = drm_atomic_set_crtc_for_connector(conn_state, t->crtc) ?: + drm_atomic_set_mode_for_crtc(crtc_state, t->mode) ?: + drm_atomic_set_crtc_for_plane(plane_state, t->crtc); + if (ret) + goto put; + drm_atomic_set_fb_for_plane(plane_state, &t->fb); + plane_state->crtc_w = t->mode->hdisplay; + plane_state->crtc_h = t->mode->vdisplay; + plane_state->src_w = t->mode->hdisplay << 16; + plane_state->src_h = t->mode->vdisplay << 16; + } + ret = drm_atomic_commit(state); +put: + drm_atomic_commit_put(state); + if (ret == -EDEADLK) { + drm_modeset_backoff(&ctx); + goto retry; + } +out: + drm_modeset_drop_locks(&ctx); + drm_modeset_acquire_fini(&ctx); + return ret; +} + +/* + * Let the hotplug worker run until it is idle. A held connection keeps its + * timeout work queued, which is flushed too but may stay queued. + */ +static void relink_test_settle(struct kunit *test, struct relink_test *t) +{ + int i; + + for (i = 0; i < 8; i++) { + flush_delayed_work(&t->dcp.announce_wq); + flush_work(&t->con.hotplug_wq); + if (!work_pending(&t->con.hotplug_wq) && + (t->dcp.announce.holding || !delayed_work_pending(&t->dcp.announce_wq))) + return; + } + KUNIT_FAIL(test, "the hotplug worker did not settle"); +} + +static void relink_test_hotplug(struct relink_test *t) +{ + schedule_work(&t->con.hotplug_wq); + flush_work(&t->con.hotplug_wq); +} + +/* + * End the hold of a new connection by its timeout: the modes of this + * fixture never get ready. + */ +static void relink_test_expire(struct kunit *test, struct relink_test *t) +{ + scoped_guard(mutex, &t->dcp.hpd_mutex) { + t->dcp.announce.timed = true; + t->dcp.announce.deadline = jiffies - 1; + } + mod_delayed_work(system_freezable_wq, &t->dcp.announce_wq, 0); + relink_test_settle(test, t); +} + +/* As dcp_external_disconnect() for a route that went away. */ +static void relink_test_tear(struct relink_test *t) +{ + struct apple_dcp *dcp = &t->dcp; + + scoped_guard(mutex, &dcp->hpd_mutex) { + WRITE_ONCE(dcp->typec_cable_connected, false); + dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); + dcp_announce_end(&dcp->announce); + } + dcp_mode_invalidate(&dcp->mode_state); + WRITE_ONCE(t->con.connected, false); + relink_test_hotplug(t); +} + +/* + * As dcp_dptx_connect_oob() for the route coming back, then the firmware's + * connected callback, then the end of the new connection's hold. + */ +static void relink_test_reconnect(struct kunit *test, struct relink_test *t) +{ + struct apple_dcp *dcp = &t->dcp; + + scoped_guard(mutex, &dcp->hpd_mutex) { + WRITE_ONCE(dcp->typec_cable_connected, true); + dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, true); + dcp_announce_end(&dcp->announce); + } + WRITE_ONCE(t->con.connected, true); + relink_test_hotplug(t); + KUNIT_ASSERT_TRUE(test, dcp->announce.holding); + relink_test_expire(test, t); +} + +static bool relink_test_connected(struct relink_test *t) +{ + return relink_test_detect(&t->con.base, false) == connector_status_connected; +} + +static bool relink_test_link_up(struct relink_test *t) +{ + return READ_ONCE(t->dcp.mode_state.valid); +} + +/* + * A display on cable generation 2, reported connected, with userspace + * scanning out to it at a mode admitted for that generation. + */ +static struct relink_test *relink_test_init(struct kunit *test) +{ + struct apple_crtc *apple_crtc; + struct relink_test *t; + struct apple_dcp *dcp; + struct drm_device *drm; + struct device *dev; + int ret; + + dev = drm_kunit_helper_alloc_device(test); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + t = drm_kunit_helper_alloc_drm_device(test, dev, struct relink_test, drm, + DRIVER_MODESET | DRIVER_ATOMIC); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t); + drm = &t->drm; + dcp = &t->dcp; + + t->pdev = kunit_platform_device_alloc(test, "dcp-relink", PLATFORM_DEVID_AUTO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->pdev); + KUNIT_ASSERT_EQ(test, kunit_platform_device_add(test, t->pdev), 0); + platform_set_drvdata(t->pdev, dcp); + dcp->dev = &t->pdev->dev; + dcp->hw.iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL; + iomfb_queue_init(dcp); + mutex_init(&dcp->external_lock); + mutex_init(&dcp->hpd_mutex); + mutex_init(&dcp->dcpavserv.lock); + spin_lock_init(&dcp->metadata.lock); + spin_lock_init(&dcp->mode_state.lock); + INIT_DELAYED_WORK(&dcp->announce_wq, dcp_announce_work); + INIT_WORK(&dcp->vblank_wq, relink_test_noop); + INIT_WORK(&t->con.hotplug_wq, dcp_hotplug); + dcp->test_point = relink_test_point; + dcp->typec_cable_connected = true; + dcp->typec_generation = 2; + dcp_firmware_metadata_connection(dcp, 2, true); + dcp->connector = &t->con; + t->con.dcp = t->pdev; + t->con.connected = true; + t->offered = true; + t->trains = true; + t->probed_generation = 2; + init_completion(&t->modeset_entered); + init_completion(&t->modeset_release); + init_completion(&t->userspace_done); + complete_all(&t->userspace_done); + + t->plane = drm_kunit_helper_create_primary_plane(test, drm, NULL, + &relink_test_plane_helpers, + NULL, 0, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->plane); + /* An apple_crtc, so that the real mode set path finds its DCP. */ + apple_crtc = drmm_kzalloc(drm, sizeof(*apple_crtc), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, apple_crtc); + apple_crtc->dcp = t->pdev; + t->crtc = &apple_crtc->base; + ret = drmm_crtc_init_with_planes(drm, t->crtc, t->plane, NULL, + &relink_test_crtc_funcs, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + drm_crtc_helper_add(t->crtc, &relink_test_crtc_helpers); + ret = drmm_encoder_init(drm, &t->encoder, NULL, DRM_MODE_ENCODER_TMDS, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + t->encoder.possible_crtcs = drm_crtc_mask(t->crtc); + ret = drmm_connector_init(drm, &t->con.base, &relink_test_connector_funcs, + DRM_MODE_CONNECTOR_USB, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + drm_connector_helper_add(&t->con.base, &relink_test_connector_helpers); + KUNIT_ASSERT_EQ(test, drm_connector_attach_encoder(&t->con.base, &t->encoder), 0); + drm_mode_config_reset(drm); + /* The device is never registered, so it has nobody to send uevents to. */ + dev_set_uevent_suppress(drm->primary->kdev, true); + + t->mode = drm_kunit_display_mode_from_cea_vic(test, drm, 16); + KUNIT_ASSERT_NOT_NULL(test, t->mode); + t->fb.dev = drm; + t->fb.format = drm_format_info(DRM_FORMAT_XRGB8888); + t->fb.width = t->mode->hdisplay; + t->fb.height = t->mode->vdisplay; + t->fb.pitches[0] = t->mode->hdisplay * 4; + t->fb.modifier = DRM_FORMAT_MOD_LINEAR; + KUNIT_ASSERT_EQ(test, drm_framebuffer_init(drm, &t->fb, &relink_test_fb_funcs), 0); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, relink_test_exit, t), 0); + + KUNIT_ASSERT_EQ(test, relink_test_commit(t, true, true), 0); + KUNIT_ASSERT_TRUE(test, t->crtc->state->active); + KUNIT_ASSERT_TRUE(test, relink_test_link_up(t)); + KUNIT_ASSERT_EQ(test, atomic_read(&t->modesets), 1); + dcp->announce.announced = true; + + return t; +} + +/* + * The Neo sequence: the route of an active CRTC goes away and comes back, + * with no valid mode, and before userspace has probed the new cable. The + * mode is set again, and the link is good. + */ +static void relink_bounce_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + int events; + + relink_test_tear(t); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); + /* Userspace has not reacted: its CRTC is still on. */ + KUNIT_ASSERT_TRUE(test, t->crtc->state->active); + + events = atomic_read(&t->events); + relink_test_reconnect(test, t); + + KUNIT_EXPECT_EQ(test, atomic_read(&t->modesets), 2); + KUNIT_EXPECT_TRUE(test, relink_test_link_up(t)); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_GOOD); + KUNIT_EXPECT_TRUE(test, relink_test_connected(t)); + KUNIT_EXPECT_GT(test, atomic_read(&t->events), events); +} + +/* + * The display on the new link no longer offers the mode: the link is bad + * and the display gone until userspace switches the CRTC off, and then it + * is a new connection. + */ +static void relink_mode_gone_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + int events, checks; + + relink_test_tear(t); + t->offered = false; + events = atomic_read(&t->events); + relink_test_reconnect(test, t); + + KUNIT_EXPECT_EQ(test, atomic_read(&t->modesets), 1); + KUNIT_EXPECT_FALSE(test, relink_test_link_up(t)); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_BAD); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); + KUNIT_EXPECT_GT(test, atomic_read(&t->events), events); + + /* Later firmware facts neither bring it back nor try again. */ + checks = atomic_read(&t->checks); + relink_test_hotplug(t); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); + KUNIT_EXPECT_EQ(test, atomic_read(&t->checks), checks); + + /* Userspace takes the display away, so it is new when it is back. */ + KUNIT_ASSERT_EQ(test, relink_test_commit(t, false, false), 0); + relink_test_settle(test, t); + KUNIT_EXPECT_TRUE(test, t->dcp.announce.holding); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); + events = atomic_read(&t->events); + relink_test_expire(test, t); + KUNIT_EXPECT_TRUE(test, relink_test_connected(t)); + KUNIT_EXPECT_GT(test, atomic_read(&t->events), events); + KUNIT_EXPECT_EQ(test, atomic_read(&t->modesets), 1); +} + +/* The mode is set again but the link does not come up: as above. */ +static void relink_link_fails_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + int events; + + relink_test_tear(t); + t->trains = false; + events = atomic_read(&t->events); + relink_test_reconnect(test, t); + + KUNIT_EXPECT_EQ(test, atomic_read(&t->modesets), 2); + KUNIT_EXPECT_FALSE(test, relink_test_link_up(t)); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_BAD); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); + KUNIT_EXPECT_GT(test, atomic_read(&t->events), events); +} + +/* A loss belongs to its cable: the next one is set up again. */ +static void relink_next_cable_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + relink_test_tear(t); + t->offered = false; + relink_test_reconnect(test, t); + KUNIT_ASSERT_FALSE(test, relink_test_connected(t)); + + relink_test_tear(t); + t->offered = true; + relink_test_reconnect(test, t); + KUNIT_EXPECT_EQ(test, atomic_read(&t->modesets), 2); + KUNIT_EXPECT_TRUE(test, relink_test_link_up(t)); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_GOOD); + KUNIT_EXPECT_TRUE(test, relink_test_connected(t)); +} + +/* A CRTC userspace switched off is left to userspace. */ +static void relink_inactive_crtc_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + KUNIT_ASSERT_EQ(test, relink_test_commit(t, false, false), 0); + relink_test_settle(test, t); + relink_test_tear(t); + relink_test_reconnect(test, t); + + KUNIT_EXPECT_EQ(test, atomic_read(&t->probes), 0); + KUNIT_EXPECT_EQ(test, atomic_read(&t->modesets), 1); + KUNIT_EXPECT_TRUE(test, relink_test_connected(t)); +} + +/* + * Other DCPs keep their retraining: the CRTC is reset with its link marked + * bad, without a probe. + */ +static void relink_other_profile_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + t->dcp.hw.iomfb_method_profile = DCP_IOMFB_METHODS_DEFAULT; + dcp_mode_invalidate(&t->dcp.mode_state); + relink_test_hotplug(t); + + KUNIT_EXPECT_EQ(test, atomic_read(&t->probes), 0); + KUNIT_EXPECT_EQ(test, atomic_read(&t->modesets), 2); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_BAD); + KUNIT_EXPECT_TRUE(test, relink_test_connected(t)); +} + +/* A route that bounces again during the probe leaves it to the new cable. */ +static void relink_newer_cable_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + relink_test_tear(t); + t->bounce_on_probe = true; + relink_test_reconnect(test, t); + + KUNIT_EXPECT_EQ(test, atomic_read(&t->probes), 1); + KUNIT_EXPECT_EQ(test, atomic_read(&t->modesets), 1); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_GOOD); +} + +/* Before drm_mode_config_reset() the connector has no state to recover. */ +static void relink_no_state_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + struct drm_connector_state *state = t->con.base.state; + + t->con.base.state = NULL; + dcp_mode_invalidate(&t->dcp.mode_state); + relink_test_hotplug(t); + t->con.base.state = state; + + KUNIT_EXPECT_EQ(test, atomic_read(&t->probes), 0); + KUNIT_EXPECT_EQ(test, atomic_read(&t->modesets), 1); +} + +static int relink_test_userspace_off(void *data) +{ + struct relink_test *t = data; + + t->userspace_ret = relink_test_commit(t, false, false); + complete(&t->userspace_done); + return 0; +} + +/* + * Userspace switches the CRTC off while the mode set that fails to bring + * the link up is in flight. Whichever goes first, the display is not left + * lost with its CRTC off: it comes back as a new connection. + */ +static void relink_userspace_off_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + struct task_struct *task; + + relink_test_tear(t); + t->trains = false; + t->hold_modeset = true; + reinit_completion(&t->userspace_done); + + /* The reconnect, up to the mode set of the end of its hold. */ + scoped_guard(mutex, &t->dcp.hpd_mutex) { + WRITE_ONCE(t->dcp.typec_cable_connected, true); + t->dcp.typec_generation++; + dcp_firmware_metadata_connection(&t->dcp, t->dcp.typec_generation, true); + dcp_announce_end(&t->dcp.announce); + } + WRITE_ONCE(t->con.connected, true); + relink_test_hotplug(t); + scoped_guard(mutex, &t->dcp.hpd_mutex) { + t->dcp.announce.timed = true; + t->dcp.announce.deadline = jiffies - 1; + } + mod_delayed_work(system_freezable_wq, &t->dcp.announce_wq, 0); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&t->modeset_entered, 5 * HZ), 0); + + task = kthread_run(relink_test_userspace_off, t, "relink-userspace"); + if (IS_ERR(task)) + complete(&t->userspace_done); + KUNIT_ASSERT_FALSE(test, IS_ERR(task)); + msleep(50); + t->hold_modeset = false; + complete_all(&t->modeset_release); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&t->userspace_done, 5 * HZ), 0); + KUNIT_ASSERT_EQ(test, t->userspace_ret, 0); + relink_test_settle(test, t); + + KUNIT_EXPECT_FALSE(test, t->crtc->state->active); + relink_test_expire(test, t); + KUNIT_EXPECT_TRUE(test, relink_test_connected(t)); +} + +/* + * A link that never comes up: lost, back once after userspace switches the + * CRTC off, then lost for good when the mode set userspace makes on it + * fails too, until the next cable. + */ +static void relink_retry_once_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + int modesets; + + relink_test_tear(t); + t->trains = false; + relink_test_reconnect(test, t); + KUNIT_ASSERT_FALSE(test, relink_test_connected(t)); + + /* Userspace switches the CRTC off: the display comes back once. */ + KUNIT_ASSERT_EQ(test, relink_test_commit(t, false, false), 0); + relink_test_settle(test, t); + relink_test_expire(test, t); + KUNIT_ASSERT_TRUE(test, relink_test_connected(t)); + + /* Its mode set fails: lost again, without a mode set of the kernel's own. */ + modesets = atomic_read(&t->modesets); + KUNIT_ASSERT_EQ(test, relink_test_commit(t, true, false), 0); + relink_test_settle(test, t); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_BAD); + KUNIT_EXPECT_EQ(test, atomic_read(&t->modesets), modesets + 2); + + /* Switched off again, it stays gone on this cable. */ + KUNIT_ASSERT_EQ(test, relink_test_commit(t, false, false), 0); + relink_test_settle(test, t); + KUNIT_EXPECT_FALSE(test, t->dcp.announce.holding); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); + + /* The next cable is announced again. */ + relink_test_tear(t); + t->trains = true; + relink_test_reconnect(test, t); + KUNIT_EXPECT_TRUE(test, relink_test_connected(t)); +} + +/* + * Mode sets go through the real dcp_crtc_atomic_modeset() and the H17P mode + * set path, with the test answering for the firmware. + */ +static void relink_test_real(struct relink_test *t) +{ + t->real_modeset = true; + t->dcp.external_running = true; + t->dcp.fw_compat = DCP_FIRMWARE_H17P; + t->catalog.mode = *t->mode; + t->dcp.modes = &t->catalog; + t->dcp.nr_modes = 1; + t->dcp.test_push = relink_test_push; +} + +/* The bounce, through the real mode set: the link comes back. */ +static void relink_real_modeset_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + relink_test_real(t); + relink_test_tear(t); + relink_test_reconnect(test, t); + KUNIT_EXPECT_EQ(test, t->modeset_ret, 0); + KUNIT_EXPECT_TRUE(test, relink_test_link_up(t)); + KUNIT_EXPECT_TRUE(test, relink_test_connected(t)); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_GOOD); +} + +/* + * The pipe was stopped while its loss was still buffered by the paused + * hotplug callback, and the next activation fails before its mode set + * begins: the old mode must not pass for a link. + */ +static void relink_early_epipe_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + t->dcp.external_running = true; + t->dcp.external_cable = true; + t->dcp.external_power_attempted = true; + t->dcp.fw_compat = DCP_FIRMWARE_H17P; + scoped_guard(mutex, &t->dcp.iomfb.lock) { + t->dcp.iomfb.presentation_paused = true; + KUNIT_EXPECT_TRUE(test, dcp_external_hotplug(&t->dcp, false)); + } + KUNIT_ASSERT_TRUE(test, relink_test_link_up(t)); + KUNIT_ASSERT_TRUE(test, t->dcp.external_hotplug_pending); + t->dcp.external_running = false; + t->dcp.external_stopped = true; + t->real_modeset = true; + t->force_modeset = true; + KUNIT_ASSERT_EQ(test, relink_test_commit(t, true, false), 0); + KUNIT_ASSERT_EQ(test, t->modeset_ret, -EPIPE); + relink_test_settle(test, t); + KUNIT_EXPECT_FALSE(test, relink_test_link_up(t)); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_BAD); +} + +/* A presentation paused on purpose is not a failed link. */ +static void relink_paused_kept_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + t->dcp.external_running = true; + t->dcp.fw_compat = DCP_FIRMWARE_H17P; + scoped_guard(mutex, &t->dcp.iomfb.lock) + t->dcp.iomfb.presentation_paused = true; + t->real_modeset = true; + t->force_modeset = true; + KUNIT_ASSERT_EQ(test, relink_test_commit(t, true, false), 0); + KUNIT_EXPECT_EQ(test, t->modeset_ret, -EPIPE); + relink_test_settle(test, t); + KUNIT_EXPECT_TRUE(test, relink_test_link_up(t)); + KUNIT_EXPECT_TRUE(test, relink_test_connected(t)); + scoped_guard(mutex, &t->dcp.iomfb.lock) + t->dcp.iomfb.presentation_paused = false; +} + +/* + * Once a lost connection was reported, unchanged facts send no more events: + * each would restart a compositor's wait for the output to stay gone. + */ +static void relink_lost_quiet_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + int events, i; + + relink_test_tear(t); + t->trains = false; + relink_test_reconnect(test, t); + KUNIT_ASSERT_FALSE(test, relink_test_connected(t)); + relink_test_settle(test, t); + events = atomic_read(&t->events); + for (i = 0; i < 5; i++) + relink_test_hotplug(t); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), events); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); +} + +static struct kunit_case relink_cases[] = { + KUNIT_CASE(relink_bounce_test), + KUNIT_CASE(relink_mode_gone_test), + KUNIT_CASE(relink_link_fails_test), + KUNIT_CASE(relink_next_cable_test), + KUNIT_CASE(relink_inactive_crtc_test), + KUNIT_CASE(relink_other_profile_test), + KUNIT_CASE(relink_newer_cable_test), + KUNIT_CASE(relink_no_state_test), + KUNIT_CASE(relink_userspace_off_test), + KUNIT_CASE(relink_retry_once_test), + KUNIT_CASE(relink_real_modeset_test), + KUNIT_CASE(relink_early_epipe_test), + KUNIT_CASE(relink_paused_kept_test), + KUNIT_CASE(relink_lost_quiet_test), + {} +}; + +static struct kunit_suite relink_suite = { + .name = "apple-dcp-external-relink", + .test_cases = relink_cases, +}; + +kunit_test_suite(relink_suite); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 5f7b2cbf886ed..aaca48f7b38f1 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -623,6 +624,14 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca { struct dcp_method_entry resolved = *call; +#if IS_ENABLED(CONFIG_DRM_APPLE_KUNIT_TEST) + /* A KUnit test answers for the firmware. */ + if (dcp->test_push) { + dcp->test_push(dcp, call, cb, cookie); + return; + } +#endif + /* Reply chains stay on the serialized receiver; callers enqueue copies. */ if (iomfb_uses_queue(dcp) && READ_ONCE(dcp->iomfb.owner) != current) { if (iomfb_queue_command(dcp, oob, call, in_len, out_len, @@ -838,6 +847,7 @@ dcp_announce_eval(struct apple_dcp *dcp, struct apple_connector *connector, .connected = READ_ONCE(connector->connected), .present = !READ_ONCE(dcp->external_physically_absent), .failed = READ_ONCE(dcp->external_error) || READ_ONCE(dcp->crashed), + .lost = dcp_announce_lost(a, dcp->typec_generation), }; enum dcp_announce_status status; unsigned long now = jiffies; @@ -913,12 +923,174 @@ bool dcp_connector_connected(struct apple_connector *connector) return dcp->announce.announced; } +/* The CRTC userspace keeps active on @connector while its link is down. */ +static struct drm_crtc *dcp_external_unlinked_crtc(struct apple_dcp *dcp, + struct apple_connector *connector) +{ + struct drm_connector_state *conn_state = connector->base.state; + struct drm_crtc *crtc = conn_state ? conn_state->crtc : NULL; + + drm_modeset_lock_assert_held(&connector->base.dev->mode_config.connection_mutex); + if (!crtc || !crtc->state || !crtc->state->active || + READ_ONCE(dcp->mode_state.valid)) + return NULL; + return crtc; +} + +static bool dcp_external_needs_relink(struct apple_dcp *dcp, + struct apple_connector *connector) +{ + struct drm_device *dev = connector->base.dev; + struct drm_modeset_acquire_ctx ctx; + bool unlinked = false; + int ret; + + DRM_MODESET_LOCK_ALL_BEGIN(dev, ctx, 0, ret); + unlinked = !!dcp_external_unlinked_crtc(dcp, connector); + DRM_MODESET_LOCK_ALL_END(dev, ctx, ret); + + return unlinked; +} + +/* Commit @crtc with its mode set again and the link of @connector good. */ +static int dcp_external_commit_relink(struct apple_connector *connector, + struct drm_crtc *crtc, + struct drm_modeset_acquire_ctx *ctx) +{ + struct drm_connector_state *conn_state; + struct drm_crtc_state *crtc_state; + struct drm_atomic_commit *state; + int ret; + + state = drm_atomic_commit_alloc(crtc->dev); + if (!state) + return -ENOMEM; + state->acquire_ctx = ctx; + crtc_state = drm_atomic_get_crtc_state(state, crtc); + if (IS_ERR(crtc_state)) { + ret = PTR_ERR(crtc_state); + goto out; + } + conn_state = drm_atomic_get_connector_state(state, &connector->base); + if (IS_ERR(conn_state)) { + ret = PTR_ERR(conn_state); + goto out; + } + crtc_state->connectors_changed = true; + conn_state->link_status = DRM_LINK_STATUS_GOOD; + ret = drm_atomic_commit(state); +out: + drm_atomic_commit_put(state); + + return ret; +} + +/* + * Set the mode of the CRTC userspace keeps active on @connector again. If + * that fails, or leaves the link down, mark the link bad and lose the + * connection of cable generation @generation. Both are decided under the + * modeset locks, so a CRTC that userspace switches off is either switched + * off first and left alone, or found lost by dcp_external_crtc_released(). + * + * Returns the error that lost the connection, or 0. + */ +static int dcp_external_reset_crtc(struct apple_dcp *dcp, + struct apple_connector *connector, u64 generation) +{ + struct drm_device *dev = connector->base.dev; + struct drm_modeset_acquire_ctx ctx; + struct drm_crtc *crtc; + int ret, err = 0; + + DRM_MODESET_LOCK_ALL_BEGIN(dev, ctx, 0, ret); + + err = 0; + crtc = dcp_external_unlinked_crtc(dcp, connector); + if (!crtc) + goto unlock; + ret = dcp_external_commit_relink(connector, crtc, &ctx); + if (ret == -EDEADLK) + goto unlock; + if (!ret && READ_ONCE(dcp->mode_state.valid)) + goto unlock; + + err = ret ?: -EIO; + ret = 0; + scoped_guard(mutex, &dcp->hpd_mutex) { + if (generation == dcp->typec_generation) { + connector->base.state->link_status = DRM_LINK_STATUS_BAD; + dcp_announce_lose(&dcp->announce, generation); + /* The caller's event is the one this owes. */ + dcp_announce_notify(&dcp->announce, + dcp_announce_eval(dcp, connector, false)); + } else { + /* A newer connection gets a link of its own. */ + err = 0; + } + } +unlock: + DRM_MODESET_LOCK_ALL_END(dev, ctx, ret); + + return ret ?: err; +} + +/* + * The link of an external display came back, or was disturbed, under a CRTC + * that userspace keeps active. Userspace sees the display it had and need + * not set its mode again, so set it here. A mode that needs its link + * proven, such as the 180 Hz exception mode, is admitted only against the + * EDID a probe read for the current cable generation (see + * dcp_180hz_mode_current()), so probe first, as userspace would. + * + * A link that cannot be set up again is marked bad, and the connection is + * lost: reported gone until userspace releases the CRTC, see + * dcp-announce.h. The caller sends the hotplug event. + */ +static void dcp_external_relink(struct apple_dcp *dcp, + struct apple_connector *connector) +{ + struct drm_device *dev = connector->base.dev; + u64 generation; + int ret; + + if (!dcp_external_needs_relink(dcp, connector)) + return; + + scoped_guard(mutex, &dcp->hpd_mutex) + generation = dcp->typec_generation; + scoped_guard(mutex, &dev->mode_config.mutex) + connector->base.funcs->fill_modes(&connector->base, + dev->mode_config.max_width, + dev->mode_config.max_height); + + ret = dcp_external_reset_crtc(dcp, connector, generation); + if (ret) + dev_warn(dcp->dev, "display link not restored (%d), reporting it disconnected\n", + ret); +} + +/* + * The CRTC of the external display was switched off. If its connection was + * lost, report the display again, as a new connection, once per cable + * generation (see dcp_announce_retry()). + */ +void dcp_external_crtc_released(struct apple_dcp *dcp) +{ + struct apple_connector *con = READ_ONCE(dcp->connector); + + guard(mutex)(&dcp->hpd_mutex); + if (dcp_announce_retry(&dcp->announce, dcp->typec_generation) && + !dcp->announce.stopped && con && + READ_ONCE(con->dcp) == to_platform_device(dcp->dev)) + schedule_work(&con->hotplug_wq); +} + void dcp_hotplug(struct work_struct *work) { struct apple_connector *connector; struct platform_device *pdev; struct apple_dcp *dcp; - bool connected; + bool connected, gated; int ret; connector = container_of(work, struct apple_connector, hotplug_wq); @@ -947,7 +1119,8 @@ void dcp_hotplug(struct work_struct *work) dcp_clear_connector_edid(connector); } - if (dcp_announce_gated(dcp, connector)) { + gated = dcp_announce_gated(dcp, connector); + if (gated) { enum dcp_announce_status status; bool notify; @@ -968,8 +1141,11 @@ void dcp_hotplug(struct work_struct *work) * display modes from atomic_flush, so userspace needs to trigger a * flush, or the CRTC gets no signal. */ - if (connector->base.state && !READ_ONCE(dcp->mode_state.valid) && connected && - !(dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x())) { + if (gated) { + if (connector->base.state && !READ_ONCE(dcp->mode_state.valid) && connected) + dcp_external_relink(dcp, connector); + } else if (connector->base.state && !READ_ONCE(dcp->mode_state.valid) && connected && + !(dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x())) { drm_connector_set_link_status_property(&connector->base, DRM_MODE_LINK_STATUS_BAD); @@ -987,6 +1163,7 @@ void dcp_hotplug(struct work_struct *work) ret); } + dcp_test_point(dcp, DCP_TEST_HOTPLUG_EVENT); drm_kms_helper_connector_hotplug_event(&connector->base); } @@ -1499,8 +1676,19 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, guard(mutex)(&dcp->external_lock); if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && (!dcp->external_running || dcp->external_error || READ_ONCE(dcp->quiescing) || - READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.presentation_paused))) + READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.presentation_paused))) { + /* + * The pipe of this CRTC did not start, so it has no link and its + * old mode is gone with it: the hotplug worker sets it up again + * or reports the display lost. A pipe that runs with its + * presentation paused on purpose keeps its mode. + */ + if (!dcp->external_running || dcp->external_error) { + dcp_mode_invalidate(&dcp->mode_state); + dcp_handle_hotplug_actions(dcp, DCP_HOTPLUG_NOTIFY); + } return -EPIPE; + } crtc_state = drm_atomic_get_new_crtc_state(commit, crtc); if (!crtc_state) @@ -1538,6 +1726,9 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, action = dcp_mode_finish(&dcp->mode_state, !ret, dcp->connector ? &dcp->connector->connected : NULL); + /* An external mode set that failed leaves the CRTC without a link too. */ + if (ret && dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) + action |= DCP_HOTPLUG_NOTIFY; dcp_handle_hotplug_actions(dcp, action); return ret; From 2fb61850cb3e0ccdc4523284117cb19c7987e6e0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 22:25:25 +1100 Subject: [PATCH 1454/2680] phy: apple: atc: Look up the Type-C target mode before the mux update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit atcphy_mux_set() takes the USB3 root hub of a port behind a fixed hub before atcphy_mux_set_locked() works out which PHY mode the new Type-C state needs. Move that lookup into atcphy_typec_target_mode() and do it first, under the PHY lock, so that the mux update can look at the target before it decides what else it needs. atcphy_mux_set_locked() now takes the PHY and the target mode. The lookup keeps its error messages and still maps the off state of a fixed-hub port to USB2. No functional change. [ryan: adapted to 7.2.10 — without the mux dynamic-debug lines of commit 294891122c09 ("usb: typec: expose mux callback results through dynamic debug"), which 170-usb does not carry; they are not moved or reworded] (cherry picked from commit 7156d4c0a7b65e2389a29931d15c082b41c66845) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index b05c5d30b9b78..272bfafc1082e 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -3634,20 +3634,18 @@ static void atcphy_usb3_pm_put(struct atcphy_usb3_pm *pm) #endif } -static int atcphy_mux_set_locked(struct typec_mux_dev *mux, struct typec_mux_state *state) +/* The PHY mode a Type-C mux state asks for; unsupported states turn the PHY off. */ +static enum atcphy_mode atcphy_typec_target_mode(struct apple_atcphy *atcphy, + const struct typec_mux_state *state) { - struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); enum atcphy_mode target_mode; - int ret; - - guard(mutex)(&atcphy->lock); if (state->mode == TYPEC_STATE_SAFE) { target_mode = APPLE_ATCPHY_MODE_OFF; } else if (state->mode == TYPEC_STATE_USB) { target_mode = APPLE_ATCPHY_MODE_USB3; } else if (!state->alt && state->mode == TYPEC_MODE_USB4) { - struct enter_usb_data *data = state->data; + const struct enter_usb_data *data = state->data; u32 eudo_usb_mode = FIELD_GET(EUDO_USB_MODE_MASK, data->eudo); switch (eudo_usb_mode) { @@ -3701,6 +3699,14 @@ static int atcphy_mux_set_locked(struct typec_mux_dev *mux, struct typec_mux_sta if (atcphy->fixed_usb2 && target_mode == APPLE_ATCPHY_MODE_OFF) target_mode = APPLE_ATCPHY_MODE_USB2; + return target_mode; +} + +static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode target_mode) +{ + int ret; + + guard(mutex)(&atcphy->lock); if (atcphy->mode == target_mode && !atcphy->mode_dirty) { /* Retry a failed PIPE setup without disturbing the configured lanes. */ if (atcphy->fixed_usb2 && atcphy->host_active && @@ -3784,12 +3790,16 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta { struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); struct atcphy_usb3_pm pm = {}; + enum atcphy_mode target_mode; int ret; + scoped_guard(mutex, &atcphy->lock) + target_mode = atcphy_typec_target_mode(atcphy, state); + ret = atcphy_usb3_pm_get(atcphy, &pm); if (ret) return ret; - ret = atcphy_mux_set_locked(mux, state); + ret = atcphy_mux_set_locked(atcphy, target_mode); atcphy_usb3_pm_put(&pm); return ret; } From b8b43c56a4735468149ad05e4a2a8c8b2e2824ac Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 19:46:08 +1100 Subject: [PATCH 1455/2680] drm/apple: Retire an H17P present that the firmware dropped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An external H17P display can lose its sink during its first mode set, for instance when a dock is pulled while the display comes up. The firmware fails the mode set after about two seconds, then accepts the next present and drops it, because its pipe is not enabled. It never completes that present, and it reports no abort for it either: DPTXPort: firmware reports link fault 22 on target 0:0 set_digital_out_mode finished:1820 swap not completed, retraining the display IOMFB transaction timed out: cmd=0 oob=0 present=1 accepted=1 swap=162 The swap watchdog already completes the flip for userspace and retrains the display one second after such a present was accepted. Since the H17P commands and presents were serialized, however, a present that was accepted and never completed also holds every command queued behind it: the queue waits until the ten-second transaction timeout declares the coprocessor crashed, and a teardown that waits for the queue to settle in the meantime fails. When the watchdog gives up on such a present, also retire it from the present state, so that the queue moves on. Nothing from the firmware ended the present, so unlike an aborted one it might still complete, and it might still be read. Until the firmware completes a later present, which shows that it is past every earlier one, a completion of any retired present is therefore ignored rather than taken for a protocol error, and the framebuffers of the retired presents, and those they would have displaced, stay referenced. A mode set has to succeed before another present is sent, so this holds at most one present per failed plug; past eight in a row without a completed present, the queue is left to its timeout as before. A watchdog run that finds its present already finished, or a newer present that re-armed it, now does nothing. Other firmware and the H17G method profile have no present state, and their watchdog is unchanged. Add the KUnit suite apple-dcp-dropped-swap, which replays the sequence through the real queue, watchdog and completion callback. [ryan: adapted to 7.2.10 — drm_atomic_state_put() is drm_atomic_commit_put()] Fixes: 5f9574c081d5 ("drm: apple: serialize H17P command and present transactions") (cherry picked from commit 8f7e2263a66adfc129eaec9f74b05b783f1d5d60) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/dcp-internal.h | 12 + drivers/gpu/drm/apple/dcp.c | 13 +- drivers/gpu/drm/apple/dcp_dropped_swap_test.c | 481 ++++++++++++++++++ drivers/gpu/drm/apple/iomfb.c | 74 ++- drivers/gpu/drm/apple/iomfb_h17p.c | 18 + drivers/gpu/drm/apple/iomfb_h17p.h | 43 ++ drivers/gpu/drm/apple/iomfb_h17p_test.c | 61 +++ drivers/gpu/drm/apple/iomfb_template.c | 12 + 9 files changed, 707 insertions(+), 8 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp_dropped_swap_test.c diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 645279b811131..85cf69e5a1e04 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -19,6 +19,7 @@ appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_resume_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_ready_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_announce_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_relink_test.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_dropped_swap_test.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 072bd2f2146b6..5cc4bf77e2aca 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -42,6 +42,8 @@ struct dcp_method_entry; struct iomfb_scanout_h17p { struct dcp_swap_submit_req_h17p request; struct drm_framebuffer *fb[SWAP_SURFACES]; + /* On iomfb.dropped while the firmware may still read a dropped present. */ + struct list_head link; }; /* One operation, including all of its nested replies and present completion. */ @@ -382,6 +384,8 @@ struct apple_dcp { /* Completes a Type-C swap that DCP dropped, and recovers the pipe. */ struct delayed_work swap_watchdog_wq; unsigned int swap_watchdog_retrains; + /* When the latest arm expires, in jiffies. */ + unsigned long swap_watchdog_deadline; /* List of referenced framebuffers pending a completed replacement swap. * The commit and RTKit work queues share it under this lock. @@ -514,6 +518,11 @@ struct apple_dcp { atomic_t opaque_x_state; struct iomfb_scanout_h17p *scanout; struct iomfb_scanout_h17p *next_scanout; + /* + * Presents the firmware dropped, until it completes a later one: + * at most one per present retired in present_state_h17p. + */ + struct list_head dropped; } iomfb; }; @@ -532,6 +541,7 @@ void iomfb_present_backlight_h17p(struct apple_dcp *dcp); bool iomfb_present_brightness_only_h17p(struct apple_dcp *dcp); bool iomfb_present_complete_h17p(struct apple_dcp *dcp); void iomfb_present_failed_h17p(struct apple_dcp *dcp); +bool iomfb_retire_dropped_present_h17p(struct apple_dcp *dcp); bool iomfb_apply_backlight_h17p(struct apple_dcp *dcp, const struct dcp_swap_submit_req_h17p *request, struct dcp_present_h17p *wire); @@ -574,6 +584,8 @@ void dcp_comp_unbind(struct device *dev, struct device *main, void *data); void dcp_platform_shutdown(struct platform_device *pdev); void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable); void dcp_enable_typec_work(struct apple_dcp *dcp); +void dcp_swap_watchdog(struct work_struct *work); +void iomfb_scanout_put_h17p(struct apple_dcp *dcp, struct iomfb_scanout_h17p *scanout); #endif int dcp_backlight_init(struct apple_dcp *dcp, struct device *dev); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 9d41a32eeeac7..5324dc5bf5655 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2277,14 +2277,18 @@ static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) * not asserted HPD yet. Userspace would then wait for the flip until the * commit times out, stalling every output it drives. Complete the flip, * mark the mode invalid so later commits do not wait for DCP, and let the - * hotplug worker re-apply the active CRTC. + * hotplug worker re-apply the active CRTC. On H17P the dropped present is + * also retired, so that the commands queued behind it can run. */ -static void dcp_swap_watchdog(struct work_struct *work) +VISIBLE_IF_KUNIT void dcp_swap_watchdog(struct work_struct *work) { struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, swap_watchdog_wq); + if (!iomfb_retire_dropped_present_h17p(dcp)) + return; + dev_warn(dcp->dev, "swap not completed, retraining the display\n"); dcp_mode_invalidate(&dcp->mode_state); dcp_drm_crtc_vblank(dcp->crtc); @@ -2295,9 +2299,12 @@ static void dcp_swap_watchdog(struct work_struct *work) void dcp_swap_watchdog_arm(struct apple_dcp *dcp) { - if (dcp_is_typec_output(dcp)) + if (dcp_is_typec_output(dcp)) { + dcp->swap_watchdog_deadline = jiffies + + msecs_to_jiffies(DCP_SWAP_WATCHDOG_MS); mod_delayed_work(system_wq, &dcp->swap_watchdog_wq, msecs_to_jiffies(DCP_SWAP_WATCHDOG_MS)); + } } void dcp_swap_watchdog_complete(struct apple_dcp *dcp) diff --git a/drivers/gpu/drm/apple/dcp_dropped_swap_test.c b/drivers/gpu/drm/apple/dcp_dropped_swap_test.c new file mode 100644 index 0000000000000..8a16ef2efd9b8 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_dropped_swap_test.c @@ -0,0 +1,481 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "dcp.h" +#include "dcp-internal.h" + +/* + * A dock leaves during the first mode set of an external H17P display. The + * firmware fails the mode set, accepts the next present and drops it without + * a completion or an abort. The teardown that follows waits for the IOMFB + * queue to settle. This replays that sequence through the real queue and + * swap watchdog; the firmware is the test, which answers the present as the + * receive callbacks would and never completes it. + */ +#define DROPPED_SWAP_ID 162 + +struct dropped_test; + +struct dropped_present { + struct iomfb_transaction transaction; + struct dropped_test *t; + struct iomfb_scanout_h17p *scanout; + u32 swap_id; + bool started, released; +}; + +struct dropped_test { + struct apple_dcp dcp; + /* Only its event lock: the watchdog completes the flip. */ + struct drm_device drm; + struct apple_crtc crtc; + struct platform_device *pdev; + /* fb[0] is on screen, and present i shows fb[i + 1]. */ + struct drm_framebuffer fb[4]; + struct dropped_present present[3]; + unsigned int freed; +}; + +static void dropped_test_noop(struct work_struct *work) +{ +} + +static void dropped_test_noop_delayed(struct work_struct *work) +{ +} + +static void dropped_test_fb_free(struct kref *kref) +{ + struct drm_framebuffer *fb = container_of(kref, struct drm_framebuffer, + base.refcount); + struct dropped_test *t = fb->dev ? container_of(fb->dev, struct dropped_test, drm) : NULL; + + if (t) + t->freed++; +} + +static unsigned int dropped_test_refs(struct drm_framebuffer *fb) +{ + return kref_read(&fb->base.refcount); +} + +/* As iomfb_atomic_start() and the accepted submit reply of dcp_swapped(). */ +static void dropped_test_start(struct apple_dcp *dcp, struct iomfb_transaction *transaction) +{ + struct dropped_present *p = container_of(transaction, struct dropped_present, transaction); + + dcp->iomfb.next_scanout = p->scanout; + p->started = dcp_present_begin_h17p(&dcp->present_state_h17p, p->swap_id) && + dcp_present_submit_h17p(&dcp->present_state_h17p, p->swap_id, true); + dcp_swap_watchdog_arm(dcp); +} + +/* As iomfb_atomic_release(). */ +static void dropped_test_release(struct iomfb_transaction *transaction) +{ + struct dropped_present *p = container_of(transaction, struct dropped_present, transaction); + + iomfb_scanout_put_h17p(&p->t->dcp, p->scanout); + p->released = true; +} + +static struct iomfb_scanout_h17p * +dropped_test_scanout(struct kunit *test, struct drm_framebuffer *fb) +{ + struct iomfb_scanout_h17p *scanout = kzalloc_obj(*scanout); + + KUNIT_ASSERT_NOT_NULL(test, scanout); + INIT_LIST_HEAD(&scanout->link); + scanout->fb[0] = fb; + drm_framebuffer_get(fb); + return scanout; +} + +static void dropped_test_scanout_free(struct iomfb_scanout_h17p *scanout) +{ + unsigned int i; + + for (i = 0; i < SWAP_SURFACES; i++) + if (scanout->fb[i]) + drm_framebuffer_put(scanout->fb[i]); + kfree(scanout); +} + +static void dropped_test_exit(void *data) +{ + struct dropped_test *t = data; + struct apple_dcp *dcp = &t->dcp; + struct iomfb_scanout_h17p *scanout, *next; + + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); + iomfb_queue_stop(dcp); + guard(mutex)(&dcp->iomfb.lock); + list_for_each_entry_safe(scanout, next, &dcp->iomfb.dropped, link) { + list_del(&scanout->link); + dropped_test_scanout_free(scanout); + } + if (dcp->iomfb.next_scanout && dcp->iomfb.next_scanout != dcp->iomfb.scanout) + dropped_test_scanout_free(dcp->iomfb.next_scanout); + if (dcp->iomfb.scanout) + dropped_test_scanout_free(dcp->iomfb.scanout); +} + +static struct dropped_test *dropped_test_init(struct kunit *test, + enum dcp_firmware_version fw, int profile) +{ + struct dropped_test *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + struct apple_dcp *dcp; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, t); + dcp = &t->dcp; + t->pdev = kunit_platform_device_alloc(test, "dcp-dropped", PLATFORM_DEVID_AUTO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->pdev); + KUNIT_ASSERT_EQ(test, kunit_platform_device_add(test, t->pdev), 0); + platform_set_drvdata(t->pdev, dcp); + dcp->dev = &t->pdev->dev; + dcp->fw_compat = fw; + dcp->hw.iomfb_method_profile = profile; + dcp->fixed_connector_type = DRM_MODE_CONNECTOR_USB; + + iomfb_queue_init(dcp); + mutex_init(&dcp->external_lock); + mutex_init(&dcp->hpd_mutex); + mutex_init(&dcp->swapped_out_fbs_lock); + INIT_LIST_HEAD(&dcp->swapped_out_fbs); + spin_lock_init(&dcp->mode_state.lock); + spin_lock_init(&dcp->backlight.lock); + INIT_WORK(&dcp->vblank_wq, dropped_test_noop); + INIT_DELAYED_WORK(&dcp->swap_watchdog_wq, dcp_swap_watchdog); + INIT_DELAYED_WORK(&dcp->typec_reconnect_wq, dropped_test_noop_delayed); + INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, dropped_test_noop_delayed); + + spin_lock_init(&t->drm.event_lock); + t->crtc.base.dev = &t->drm; + dcp->crtc = &t->crtc; + dcp->mode_state.valid = true; + + for (i = 0; i < ARRAY_SIZE(t->fb); i++) { + t->fb[i].dev = &t->drm; + kref_init(&t->fb[i].base.refcount); + t->fb[i].base.free_cb = dropped_test_fb_free; + } + /* The display shows fb[0] from a present completed before the plug. */ + dcp->iomfb.scanout = dropped_test_scanout(test, &t->fb[0]); + for (i = 0; i < ARRAY_SIZE(t->present); i++) { + t->present[i].t = t; + t->present[i].swap_id = DROPPED_SWAP_ID + i; + t->present[i].transaction.start = dropped_test_start; + t->present[i].transaction.release = dropped_test_release; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dropped_test_exit, t), 0); + return t; +} + +static struct dropped_test *dropped_test_external(struct kunit *test) +{ + return dropped_test_init(test, DCP_FIRMWARE_H17P, DCP_IOMFB_METHODS_H17P_EXTERNAL); +} + +/* Queue present @i and let the queue send it; the firmware accepts it. */ +static void dropped_test_present(struct kunit *test, struct dropped_test *t, unsigned int i, + struct drm_framebuffer *fb) +{ + struct dropped_present *p = &t->present[i]; + + p->scanout = dropped_test_scanout(test, fb); + KUNIT_ASSERT_EQ(test, iomfb_queue(&t->dcp, &p->transaction), 0); + flush_work(&t->dcp.iomfb.work); + KUNIT_ASSERT_TRUE(test, p->started); +} + +/* As dcpep_cb_swap_complete() for @swap_id on the receive path. */ +static bool dropped_test_complete(struct dropped_test *t, u32 swap_id) +{ + struct apple_dcp *dcp = &t->dcp; + + guard(mutex)(&dcp->iomfb.lock); + if (!dcp_present_complete_h17p(&dcp->present_state_h17p, swap_id)) + return false; + iomfb_present_complete_h17p(dcp); + dcp_swap_watchdog_complete(dcp); + return true; +} + +/* The firmware completes @swap_id: the real receive callback handles it. */ +static int dropped_test_deliver(struct dropped_test *t, u32 swap_id) +{ + guard(mutex)(&t->dcp.iomfb.lock); + return iomfb_h17p_deliver_swap_complete(&t->dcp, swap_id); +} + +/* Present @i is dropped: the watchdog retires it and retrains the display. */ +static void dropped_test_retire(struct kunit *test, struct dropped_test *t, unsigned int i) +{ + dropped_test_present(test, t, i, &t->fb[i + 1]); + KUNIT_ASSERT_EQ(test, iomfb_queue_settle(&t->dcp, msecs_to_jiffies(2000)), 0); + /* The queue settles before the watchdog retrains, outside the lock. */ + flush_delayed_work(&t->dcp.swap_watchdog_wq); + KUNIT_ASSERT_TRUE(test, dcp_present_was_retired_h17p(&t->dcp.present_state_h17p, + t->present[i].swap_id)); +} + +/* + * The captured sequence: the teardown's queue settle times out on the dropped + * present. The swap watchdog retires it a second after it was accepted, so a + * retry settles, the queue moves on and the coprocessor is not declared + * crashed. + */ +static void dropped_present_retired_test(struct kunit *test) +{ + struct dropped_test *t = dropped_test_external(test); + struct apple_dcp *dcp = &t->dcp; + + dropped_test_present(test, t, 0, &t->fb[1]); + KUNIT_EXPECT_EQ(test, iomfb_queue_settle(dcp, msecs_to_jiffies(500)), -ETIMEDOUT); + + KUNIT_EXPECT_EQ(test, iomfb_queue_settle(dcp, msecs_to_jiffies(2000)), 0); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dcp->crashed)); + KUNIT_EXPECT_TRUE(test, t->present[0].released); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&dcp->iomfb.timeout)); + KUNIT_EXPECT_FALSE(test, dcp->present_state_h17p.pending); + KUNIT_EXPECT_TRUE(test, dcp_present_was_retired_h17p(&dcp->present_state_h17p, + DROPPED_SWAP_ID)); + /* The firmware did not abort it. */ + KUNIT_EXPECT_FALSE(test, dcp_present_was_aborted_h17p(&dcp->present_state_h17p, + DROPPED_SWAP_ID)); + /* + * The display was retrained as before. The queue settles as soon as + * the present is retired; the watchdog retrains after that, outside + * the IOMFB lock. + */ + flush_delayed_work(&dcp->swap_watchdog_wq); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dcp->mode_state.valid)); +} + +/* + * The firmware might still read a dropped present: its framebuffers, and the + * one it would have replaced, stay referenced until a later present + * completes. A late completion of the dropped present releases nothing. + */ +static void dropped_present_buffers_test(struct kunit *test) +{ + struct dropped_test *t = dropped_test_external(test); + struct apple_dcp *dcp = &t->dcp; + + dropped_test_present(test, t, 0, &t->fb[1]); + KUNIT_ASSERT_EQ(test, iomfb_queue_settle(dcp, msecs_to_jiffies(2000)), 0); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[1]), 2); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[0]), 2); + KUNIT_EXPECT_FALSE(test, list_empty(&dcp->iomfb.dropped)); + + KUNIT_EXPECT_FALSE(test, dropped_test_complete(t, DROPPED_SWAP_ID)); + KUNIT_EXPECT_TRUE(test, dcp_present_was_retired_h17p(&dcp->present_state_h17p, + DROPPED_SWAP_ID)); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[1]), 2); + + /* The next mode set's present is sent and completes. */ + dropped_test_present(test, t, 1, &t->fb[2]); + KUNIT_EXPECT_TRUE(test, dropped_test_complete(t, DROPPED_SWAP_ID + 1)); + KUNIT_EXPECT_TRUE(test, list_empty(&dcp->iomfb.dropped)); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[1]), 1); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[0]), 1); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[2]), 2); + KUNIT_EXPECT_PTR_EQ(test, dcp->iomfb.scanout, t->present[1].scanout); + KUNIT_EXPECT_EQ(test, t->freed, 0); +} + +/* + * A watchdog that was already running when its present completed must not + * retire the next present, which was accepted just now. + */ +static void completed_present_kept_test(struct kunit *test) +{ + struct dropped_test *t = dropped_test_external(test); + struct apple_dcp *dcp = &t->dcp; + + dropped_test_present(test, t, 0, &t->fb[1]); + KUNIT_ASSERT_TRUE(test, dropped_test_complete(t, DROPPED_SWAP_ID)); + /* As iomfb_queue_advance() after the completion. */ + scoped_guard(mutex, &dcp->iomfb.lock) { + dcp->iomfb.active = NULL; + cancel_delayed_work(&dcp->iomfb.timeout); + t->present[0].transaction.release(&t->present[0].transaction); + } + dropped_test_present(test, t, 1, &t->fb[2]); + + dcp_swap_watchdog(&dcp->swap_watchdog_wq.work); + KUNIT_EXPECT_TRUE(test, dcp->present_state_h17p.pending); + KUNIT_EXPECT_FALSE(test, dcp_present_was_aborted_h17p(&dcp->present_state_h17p, + DROPPED_SWAP_ID + 1)); + KUNIT_EXPECT_FALSE(test, t->present[1].released); + KUNIT_EXPECT_TRUE(test, READ_ONCE(dcp->mode_state.valid)); +} + +/* + * Without a completed present for IOMFB_DROPPED_MAX dropped ones, the next one + * is not retired: the display is retrained as before and the queue is left to + * its timeout. + */ +static void dropped_present_bounded_test(struct kunit *test) +{ + struct dropped_test *t = dropped_test_external(test); + struct apple_dcp *dcp = &t->dcp; + struct dcp_present_state_h17p *state = &dcp->present_state_h17p; + unsigned int i; + + /* As IOMFB_DROPPED_MAX retirements leave the present state and list. */ + for (i = 0; i < IOMFB_DROPPED_MAX; i++) { + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(state, 100 + i)); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(state, 100 + i, true)); + KUNIT_ASSERT_TRUE(test, dcp_present_retire_h17p(state, 100 + i)); + list_add_tail(&dropped_test_scanout(test, &t->fb[0])->link, &dcp->iomfb.dropped); + } + dropped_test_present(test, t, 0, &t->fb[1]); + KUNIT_EXPECT_EQ(test, iomfb_queue_settle(dcp, msecs_to_jiffies(1500)), -ETIMEDOUT); + /* The watchdog has given up on the present; let it finish retraining. */ + flush_delayed_work(&dcp->swap_watchdog_wq); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dcp->mode_state.valid)); + KUNIT_EXPECT_TRUE(test, state->pending); + KUNIT_EXPECT_FALSE(test, dcp_present_was_retired_h17p(state, DROPPED_SWAP_ID)); + KUNIT_EXPECT_FALSE(test, t->present[0].released); + KUNIT_EXPECT_EQ(test, list_count_nodes(&dcp->iomfb.dropped), IOMFB_DROPPED_MAX); +} + +/* Other firmware keeps the watchdog it had: no present state is touched. */ +static void dcp_dropped_legacy_watchdog(struct kunit *test, enum dcp_firmware_version fw, + int profile) +{ + struct dropped_test *t = dropped_test_init(test, fw, profile); + struct apple_dcp *dcp = &t->dcp; + + dcp->present_state_h17p.pending = true; + dcp->present_state_h17p.accepted = true; + dcp->swap_watchdog_deadline = jiffies + HZ; + dcp_swap_watchdog(&dcp->swap_watchdog_wq.work); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dcp->mode_state.valid)); + KUNIT_EXPECT_TRUE(test, dcp->present_state_h17p.pending); + KUNIT_EXPECT_FALSE(test, dcp->present_state_h17p.has_last_aborted); + KUNIT_EXPECT_EQ(test, dcp->present_state_h17p.nr_retired, 0U); + dcp->present_state_h17p.pending = false; +} + +static void legacy_watchdog_test(struct kunit *test) +{ + dcp_dropped_legacy_watchdog(test, DCP_FIRMWARE_V_13_5, 0); +} + +/* H17P firmware with the H17G method profile keeps its transport too. */ +static void h17g_watchdog_test(struct kunit *test) +{ + dcp_dropped_legacy_watchdog(test, DCP_FIRMWARE_H17P, DCP_IOMFB_METHODS_H17G); +} + +/* + * Two plugs in a row drop their present: the watchdog retires 162, a mode set + * succeeds, and it retires 163. Both scanouts are kept. The firmware then + * completes 162 after all: a late completion, not an unexpected one. The + * coprocessor is not declared crashed, nothing is released, and the next stop + * finds the queue settled. + */ +static void older_retired_completion_test(struct kunit *test) +{ + struct dropped_test *t = dropped_test_external(test); + struct apple_dcp *dcp = &t->dcp; + + dropped_test_retire(test, t, 0); + /* The recovery mode set succeeds. */ + WRITE_ONCE(dcp->mode_state.valid, true); + dropped_test_retire(test, t, 1); + KUNIT_ASSERT_EQ(test, list_count_nodes(&dcp->iomfb.dropped), 2); + KUNIT_ASSERT_FALSE(test, READ_ONCE(dcp->crashed)); + + KUNIT_ASSERT_EQ(test, dropped_test_deliver(t, DROPPED_SWAP_ID), 0); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dcp->crashed)); + KUNIT_EXPECT_EQ(test, list_count_nodes(&dcp->iomfb.dropped), 2); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[0]), 2); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[1]), 2); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[2]), 2); + KUNIT_ASSERT_EQ(test, dropped_test_deliver(t, DROPPED_SWAP_ID + 1), 0); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dcp->crashed)); + KUNIT_EXPECT_EQ(test, iomfb_queue_settle(dcp, msecs_to_jiffies(100)), 0); + + /* A completion of a present that was never sent is still unexpected. */ + KUNIT_ASSERT_EQ(test, dropped_test_deliver(t, DROPPED_SWAP_ID - 1), 0); + KUNIT_EXPECT_TRUE(test, READ_ONCE(dcp->crashed)); +} + +/* + * As above, with present 164 of the next mode set in flight when 162 + * completes: that completion neither completes 164 nor stops its watchdog, + * and releases nothing. The completion of 164 shows that the firmware is past + * 162 and 163: their scanouts are released and they are forgotten, so a + * completion of 162 after that is unexpected again. + */ +static void retired_completion_in_flight_test(struct kunit *test) +{ + struct dropped_test *t = dropped_test_external(test); + struct apple_dcp *dcp = &t->dcp; + struct dcp_present_state_h17p *state = &dcp->present_state_h17p; + + dropped_test_retire(test, t, 0); + dropped_test_retire(test, t, 1); + dropped_test_present(test, t, 2, &t->fb[3]); + + KUNIT_ASSERT_EQ(test, dropped_test_deliver(t, DROPPED_SWAP_ID), 0); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dcp->crashed)); + KUNIT_EXPECT_TRUE(test, state->pending); + KUNIT_EXPECT_EQ(test, state->swap_id, DROPPED_SWAP_ID + 2); + KUNIT_EXPECT_PTR_EQ(test, dcp->iomfb.active, &t->present[2].transaction); + KUNIT_EXPECT_FALSE(test, t->present[2].transaction.completed); + KUNIT_EXPECT_TRUE(test, delayed_work_pending(&dcp->swap_watchdog_wq)); + KUNIT_EXPECT_EQ(test, list_count_nodes(&dcp->iomfb.dropped), 2); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[1]), 2); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[2]), 2); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[0]), 2); + + KUNIT_ASSERT_EQ(test, dropped_test_deliver(t, DROPPED_SWAP_ID + 2), 0); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dcp->crashed)); + KUNIT_EXPECT_FALSE(test, state->pending); + KUNIT_EXPECT_TRUE(test, t->present[2].transaction.completed); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&dcp->swap_watchdog_wq)); + KUNIT_EXPECT_TRUE(test, list_empty(&dcp->iomfb.dropped)); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[1]), 1); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[2]), 1); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[0]), 1); + KUNIT_EXPECT_PTR_EQ(test, dcp->iomfb.scanout, t->present[2].scanout); + KUNIT_EXPECT_FALSE(test, dcp_present_was_retired_h17p(state, DROPPED_SWAP_ID)); + KUNIT_EXPECT_FALSE(test, dcp_present_was_retired_h17p(state, DROPPED_SWAP_ID + 1)); + + KUNIT_ASSERT_EQ(test, dropped_test_deliver(t, DROPPED_SWAP_ID), 0); + KUNIT_EXPECT_TRUE(test, READ_ONCE(dcp->crashed)); +} + +static struct kunit_case dropped_swap_cases[] = { + KUNIT_CASE(dropped_present_retired_test), + KUNIT_CASE(dropped_present_buffers_test), + KUNIT_CASE(completed_present_kept_test), + KUNIT_CASE(dropped_present_bounded_test), + KUNIT_CASE(legacy_watchdog_test), + KUNIT_CASE(h17g_watchdog_test), + KUNIT_CASE(older_retired_completion_test), + KUNIT_CASE(retired_completion_in_flight_test), + {} +}; + +static struct kunit_suite dropped_swap_suite = { + .name = "apple-dcp-dropped-swap", + .test_cases = dropped_swap_cases, +}; + +kunit_test_suite(dropped_swap_suite); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index aaca48f7b38f1..76da1567ff870 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright 2021 Alyssa Rosenzweig */ +#include #include #include #include @@ -385,6 +386,7 @@ void iomfb_queue_init(struct apple_dcp *dcp) { mutex_init(&dcp->iomfb.lock); INIT_LIST_HEAD(&dcp->iomfb.pending); + INIT_LIST_HEAD(&dcp->iomfb.dropped); INIT_WORK(&dcp->iomfb.work, iomfb_queue_work); INIT_DELAYED_WORK(&dcp->iomfb.timeout, iomfb_queue_timeout); INIT_DELAYED_WORK(&dcp->iomfb.backlight_retry, iomfb_backlight_retry); @@ -1773,6 +1775,7 @@ iomfb_scanout_prepare_h17p(struct apple_dcp *dcp, struct drm_crtc *crtc, scanout = kzalloc_obj(*scanout); if (!scanout) return NULL; + INIT_LIST_HEAD(&scanout->link); if (dcp->iomfb.scanout) { scanout->request = dcp->iomfb.scanout->request; for (i = 0; i < SWAP_SURFACES; i++) { @@ -1900,8 +1903,14 @@ void iomfb_present_failed_h17p(struct apple_dcp *dcp) bool iomfb_present_complete_h17p(struct apple_dcp *dcp) { struct iomfb_transaction *transaction = dcp->iomfb.active; + struct iomfb_scanout_h17p *dropped, *next; lockdep_assert_held(&dcp->iomfb.lock); + /* The firmware is past every present before this one. */ + list_for_each_entry_safe(dropped, next, &dcp->iomfb.dropped, link) { + list_del(&dropped->link); + iomfb_scanout_release_h17p(dropped); + } if (transaction) { transaction->completed = true; if (transaction->backlight_reserved) { @@ -1917,6 +1926,65 @@ bool iomfb_present_complete_h17p(struct apple_dcp *dcp) return true; } +/* + * The firmware accepts presents while an external pipe is not enabled, for + * instance after a mode set that failed because the sink went away, and then + * drops them: neither a completion nor an abort follows. Such a present + * would hold the queue until its timeout declares the coprocessor crashed, + * and a teardown waiting for the queue to settle would fail. + * + * Called by the swap watchdog: retire the present. The firmware might still + * complete it or read it: until the firmware completes a later present, a + * completion of it is ignored, and the framebuffers of the present and those + * it displaced stay referenced. Without a completed present for + * IOMFB_DROPPED_MAX retired ones, the queue is left to its timeout. Returns + * false if the present ended or a newer one re-armed the watchdog in the + * meantime. Other firmware has no present state. + */ +bool iomfb_retire_dropped_present_h17p(struct apple_dcp *dcp) +{ + struct dcp_present_state_h17p *state = &dcp->present_state_h17p; + u32 swap_id; + + if (!iomfb_uses_queue(dcp)) + return true; + + guard(mutex)(&dcp->iomfb.lock); + swap_id = state->swap_id; + if (time_before(jiffies, dcp->swap_watchdog_deadline) || + !state->pending || !state->accepted) + return false; + if (!dcp_present_retire_h17p(state, swap_id)) + return true; + + iomfb_present_failed_h17p(dcp); + if (!iomfb_present_brightness_only_h17p(dcp)) { + dcp_unarm_retained_framebuffers(dcp, swap_id); + if (dcp->iomfb.next_scanout) { + list_add_tail(&dcp->iomfb.next_scanout->link, + &dcp->iomfb.dropped); + dcp->iomfb.next_scanout = NULL; + } + } + iomfb_queue_advance(dcp); + return true; +} + +/* Release a present's scanout unless it is displayed or was dropped. */ +VISIBLE_IF_KUNIT void iomfb_scanout_put_h17p(struct apple_dcp *dcp, + struct iomfb_scanout_h17p *scanout) +{ + /* A transaction that never started is released without the lock. */ + if (!scanout) + return; + lockdep_assert_held(&dcp->iomfb.lock); + if (scanout == dcp->iomfb.scanout || !list_empty(&scanout->link)) + return; + if (dcp->iomfb.next_scanout == scanout) + dcp->iomfb.next_scanout = NULL; + iomfb_scanout_release_h17p(scanout); +} + static void iomfb_backlight_start(struct apple_dcp *dcp, struct iomfb_transaction *transaction) { @@ -1999,11 +2067,7 @@ static void iomfb_atomic_release(struct iomfb_transaction *transaction) struct iomfb_atomic_transaction *atomic = container_of(transaction, struct iomfb_atomic_transaction, transaction); - if (atomic->scanout && atomic->dcp->iomfb.scanout != atomic->scanout) { - if (atomic->dcp->iomfb.next_scanout == atomic->scanout) - atomic->dcp->iomfb.next_scanout = NULL; - iomfb_scanout_release_h17p(atomic->scanout); - } + iomfb_scanout_put_h17p(atomic->dcp, atomic->scanout); drm_atomic_commit_put(atomic->state); kfree(atomic); } diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index a77de05c538c1..1460d883372ea 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -817,6 +817,24 @@ int iomfb_h17p_first_unbounded_callback(void) return tag; return -1; } + +/* + * Hands a completion of @swap_id to the swap_complete_ap_gated callback, as + * the firmware would. The caller holds the IOMFB lock, as the receiver does. + */ +int iomfb_h17p_deliver_swap_complete(struct apple_dcp *dcp, u32 swap_id) +{ + struct dc_swap_complete_resp_h17p *resp; + + lockdep_assert_held(&dcp->iomfb.lock); + resp = kzalloc_obj(*resp); + if (!resp) + return -ENOMEM; + resp->swap_id = swap_id; + cb_handlers[590](dcp, 590, NULL, resp); + kfree(resp); + return 0; +} #endif #undef DCP_FW_VER diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 75856d1921562..9880e82078178 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -16,6 +16,9 @@ static_assert(sizeof(struct dcp_surface_h17p) == 0x22c); static_assert(sizeof(struct dcp_swap_submit_req_h17p) == 0xe9c); static_assert(sizeof(struct dcp_swap_submit_resp_h17p) == 0x0c); +/* Retired presents kept, with their scanouts, until a later one completes. */ +#define IOMFB_DROPPED_MAX 8 + /* Owned by the RTKit receive thread, independently of DRM state lifetime. */ struct dcp_present_state_h17p { u32 swap_id; @@ -26,6 +29,12 @@ struct dcp_present_state_h17p { /* The most recent aborted present, whose late completion is ignored. */ bool has_last_aborted; u32 last_aborted; + /* + * Presents retired without a completion or an abort, until the firmware + * completes a later one: a late completion of any of them is ignored. + */ + unsigned int nr_retired; + u32 retired[IOMFB_DROPPED_MAX]; }; static inline bool @@ -80,6 +89,37 @@ dcp_present_was_aborted_h17p(const struct dcp_present_state_h17p *state, return state->has_last_aborted && state->last_aborted == swap_id; } +/* + * The swap watchdog retires an accepted present that neither completed nor + * was aborted in time. Nothing from the firmware ended it, so unlike an + * aborted one it may still complete: it is kept until a later present + * completes. Without room for it, the present stays in flight. + */ +static inline bool +dcp_present_retire_h17p(struct dcp_present_state_h17p *state, u32 swap_id) +{ + if (!state->pending || !state->accepted || state->swap_id != swap_id || + state->nr_retired >= IOMFB_DROPPED_MAX) + return false; + state->pending = false; + state->accepted = false; + state->retired[state->nr_retired++] = swap_id; + return true; +} + +/* Whether @swap_id was retired; a completion for it is late, not unexpected. */ +static inline bool +dcp_present_was_retired_h17p(const struct dcp_present_state_h17p *state, + u32 swap_id) +{ + unsigned int i; + + for (i = 0; i < state->nr_retired; i++) + if (state->retired[i] == swap_id) + return true; + return false; +} + static inline bool dcp_present_complete_h17p(struct dcp_present_state_h17p *state, u32 swap_id) { @@ -87,6 +127,8 @@ dcp_present_complete_h17p(struct dcp_present_state_h17p *state, u32 swap_id) return false; state->pending = false; state->accepted = false; + /* The firmware is past every present before this one. */ + state->nr_retired = 0; return true; } @@ -124,6 +166,7 @@ void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, #if IS_ENABLED(CONFIG_DRM_APPLE_KUNIT_TEST) int iomfb_h17p_first_unbounded_callback(void); +int iomfb_h17p_deliver_swap_complete(struct apple_dcp *dcp, u32 swap_id); #endif #undef DCP_FW_VER diff --git a/drivers/gpu/drm/apple/iomfb_h17p_test.c b/drivers/gpu/drm/apple/iomfb_h17p_test.c index c5955836700ca..8ad8815e98b0c 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p_test.c +++ b/drivers/gpu/drm/apple/iomfb_h17p_test.c @@ -144,11 +144,72 @@ static void h17p_present_abort_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, dcp_present_was_aborted_h17p(&state, 9)); } +static void h17p_present_retire_test(struct kunit *test) +{ + struct dcp_present_state_h17p state = {}; + u32 id; + + /* Only an accepted present in flight is retired. */ + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(&state, 7)); + KUNIT_EXPECT_FALSE(test, dcp_present_retire_h17p(&state, 7)); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 7, true)); + KUNIT_EXPECT_FALSE(test, dcp_present_retire_h17p(&state, 6)); + KUNIT_EXPECT_TRUE(test, dcp_present_retire_h17p(&state, 7)); + KUNIT_EXPECT_FALSE(test, state.pending); + KUNIT_EXPECT_FALSE(test, dcp_present_complete_h17p(&state, 7)); + /* A late completion for it is recognisable; the firmware did not abort it. */ + KUNIT_EXPECT_TRUE(test, dcp_present_was_retired_h17p(&state, 7)); + KUNIT_EXPECT_FALSE(test, dcp_present_was_aborted_h17p(&state, 7)); + + /* A second retirement keeps the first. */ + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(&state, 8)); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 8, true)); + KUNIT_EXPECT_TRUE(test, dcp_present_retire_h17p(&state, 8)); + KUNIT_EXPECT_TRUE(test, dcp_present_was_retired_h17p(&state, 7)); + KUNIT_EXPECT_TRUE(test, dcp_present_was_retired_h17p(&state, 8)); + + /* An abort is recorded apart from them. */ + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(&state, 9)); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 9, true)); + KUNIT_EXPECT_TRUE(test, dcp_present_abort_h17p(&state, 9)); + KUNIT_EXPECT_TRUE(test, dcp_present_was_aborted_h17p(&state, 9)); + KUNIT_EXPECT_FALSE(test, dcp_present_was_retired_h17p(&state, 9)); + KUNIT_EXPECT_TRUE(test, dcp_present_was_retired_h17p(&state, 7)); + + /* A late completion of 7 does not complete present 10 in flight. */ + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(&state, 10)); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 10, true)); + KUNIT_EXPECT_FALSE(test, dcp_present_complete_h17p(&state, 7)); + KUNIT_EXPECT_TRUE(test, state.pending); + KUNIT_EXPECT_TRUE(test, dcp_present_was_retired_h17p(&state, 7)); + + /* Completing 10 shows the firmware is past 7 and 8; the abort is kept. */ + KUNIT_EXPECT_TRUE(test, dcp_present_complete_h17p(&state, 10)); + KUNIT_EXPECT_FALSE(test, dcp_present_was_retired_h17p(&state, 7)); + KUNIT_EXPECT_FALSE(test, dcp_present_was_retired_h17p(&state, 8)); + KUNIT_EXPECT_TRUE(test, dcp_present_was_aborted_h17p(&state, 9)); + + /* At most IOMFB_DROPPED_MAX are kept: the next present stays in flight. */ + for (id = 100; id < 100 + IOMFB_DROPPED_MAX; id++) { + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(&state, id)); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, id, true)); + KUNIT_ASSERT_TRUE(test, dcp_present_retire_h17p(&state, id)); + } + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(&state, id)); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, id, true)); + KUNIT_EXPECT_FALSE(test, dcp_present_retire_h17p(&state, id)); + KUNIT_EXPECT_TRUE(test, state.pending); + KUNIT_EXPECT_TRUE(test, dcp_present_was_retired_h17p(&state, 100)); + KUNIT_EXPECT_FALSE(test, dcp_present_was_retired_h17p(&state, id)); + KUNIT_EXPECT_TRUE(test, dcp_present_complete_h17p(&state, id)); +} + static struct kunit_case h17p_present_cases[] = { KUNIT_CASE(h17p_surface_wire_test), KUNIT_CASE(h17p_opaque_x_parameter_test), KUNIT_CASE(h17p_callback_bounds_test), KUNIT_CASE(h17p_present_abort_test), + KUNIT_CASE(h17p_present_retire_test), {} }; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 8f3d658381f53..76c13c4f77ae2 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -265,6 +265,18 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, resp->swap_id); return; } + /* + * The swap watchdog retired it: its framebuffers stay + * referenced until a later present completes, and a present + * in flight now is a different one. + */ + if (dcp_present_was_retired_h17p(&dcp->present_state_h17p, + resp->swap_id)) { + dev_warn_ratelimited(dcp->dev, + "completion for retired present %u ignored\n", + resp->swap_id); + return; + } dev_err(dcp->dev, "unexpected present completion %u\n", resp->swap_id); dcp->crashed = true; return; From 6ab0e936c225cc14af942cf993d7c925754cc7b5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 22:26:25 +1100 Subject: [PATCH 1456/2680] phy: apple: atc: Don't wait for USB enumeration when the mux keeps its mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a port behind a fixed hub, atcphy_mux_set() takes the device locks of the USB3 root hub and of the device on its port before it looks at the new Type-C state. hub_wq holds the root hub's lock for a whole hub event, and a SuperSpeed enumeration that keeps failing holds it for its full set of retries, each with 5 s command timeouts. The CD321x driver also calls the mux for DisplayPort HPD and status updates that keep the pin assignment. Such an update touches neither the lanes nor the PIPE, but it still waits for hub_wq, and the HPD event it carries reaches the display only afterwards. On a J700 with a USB-C dock (DisplayPort plus USB3), the display came back 27 s after a cable flip, right after hub_wq gave up on the dock's SuperSpeed hub: 22:06:46 usb 2-1: new SuperSpeed Plus Gen 2x1 USB device number 20 using xhci-hcd 22:06:46 usb 2-1: device descriptor read/8, error -71 22:06:51 xhci-hcd xhci-hcd.0.auto: Timeout while waiting for setup device command ... 22:07:08 usb usb2-port1: unable to enumerate USB device 22:07:08 phy-apple-atc 40b000000.phy: Pipehandler lock not acked. 22:07:11 apple-dcp 416e00000.dcp: dcp_hotplug() connected:1 ... Return early, before any USB reference is taken, when the target mode is the current one, no failed transition is pending and the PIPE already sits on the backend that mode needs. atcphy_mux_set_locked() keeps its own check for the case where the state changes in between. A mode change still takes the root hub, as the PIPE has to be handed over while the controller is quiet. [ryan: adapted to 7.2.10 — context only: the mux dynamic-debug line of commit 294891122c09 ("usb: typec: expose mux callback results through dynamic debug") is not carried] (cherry picked from commit b0dee3d08c1afc2d3e76631935619a8192b86335) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 272bfafc1082e..90979abce1b13 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -3702,6 +3702,22 @@ static enum atcphy_mode atcphy_typec_target_mode(struct apple_atcphy *atcphy, return target_mode; } +/* + * Whether a mux update leaves both the lanes and the PIPE as they are. That is + * the case when the mode does not change and no earlier PIPE handoff to a + * running controller behind a fixed hub is still outstanding. A DisplayPort + * HPD or status update with an unchanged pin assignment is one of these. + */ +static bool atcphy_mux_is_noop(const struct apple_atcphy *atcphy, enum atcphy_mode target_mode) +{ + if (atcphy->mode != target_mode || atcphy->mode_dirty) + return false; + + return !(atcphy->fixed_usb2 && atcphy->host_active && + (atcphy->pipe_state != atcphy_modes[target_mode].pipehandler_state || + atcphy->pipe_pending)); +} + static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode target_mode) { int ret; @@ -3793,8 +3809,17 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta enum atcphy_mode target_mode; int ret; - scoped_guard(mutex, &atcphy->lock) + scoped_guard(mutex, &atcphy->lock) { target_mode = atcphy_typec_target_mode(atcphy, state); + /* + * Only a lane or PIPE change needs the USB3 root hub below. + * Taking it means waiting for the hub's ongoing work, which + * includes a whole enumeration with its timeouts, and would + * hold up a DisplayPort update that changes neither. + */ + if (atcphy_mux_is_noop(atcphy, target_mode)) + return 0; + } ret = atcphy_usb3_pm_get(atcphy, &pm); if (ret) From 9ef50a62f1085ef5f1654db7e8a37046096f01b7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 19:48:59 +1100 Subject: [PATCH 1457/2680] drm/apple: Retry an external display stop whose queue did not settle dcp_external_stop_locked() waits up to 500 ms for the IOMFB queue to settle before it asks the firmware to power the external pipe off. If the queue does not settle in time, the error is latched in external_error, which keeps the session blocked: every later SAFE transition of the Type-C route, route deactivation and connection returns it until the driver is rebound. The latch exists so that a retry never races a late reply to a request that timed out. When the queue does not settle, the stop has not sent any request yet, so there is no such reply. This happens when a dock leaves during the first mode set: a present that the firmware dropped keeps the queue busy until the swap watchdog retires it, and the SAFE transition that tipd makes meanwhile latches the stop: set_digital_out_mode finished:1820 sn201202x 1-0c: failed to place Type-C mux in safe mode: -110 external display activation failed: -110 sn201202x 1-0c: failed to place Type-C mux in safe mode: -110 (every later cable event, until reboot) Return such a timeout without latching it. The session stays owned: presentation stays paused, the pipe stays marked enabled and the DPTX port stays held, so the route cannot be deactivated and the Type-C PHY is not touched until a stop succeeds. Then stop again: - a SAFE transition stops the session again, which tipd's next revalidation of the cable does, or the next cable event once those retries are spent; - nothing retries a CRTC that is switched off, so the stop retries itself a second later, after the swap watchdog, at most three times; - an enable of the CRTC on the same cable resumes the deferred session instead of powering a pipe on that was never powered off. A crashed or stopped queue, a failed power-off and anything after the power-off request are still latched. The apple-dcp host selftests that extract dcp_external_stop_locked() get the new fields in their fixture. Fixes: 35f4ccf26053 ("drm/apple: add bounded external stop qualification path") (cherry picked from commit 9a56c642b83f09006ddada51da317f38692bbc8c) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 4 ++ drivers/gpu/drm/apple/dcp.c | 43 ++++++++++++++++++- drivers/gpu/drm/apple/dcp_dropped_swap_test.c | 43 +++++++++++++++++-- .../drm/apple/dcp_external_announce_test.c | 1 + .../drivers/gpu/apple-dcp/lifecycle.py | 6 ++- 5 files changed, 92 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 5cc4bf77e2aca..c65b96541a9f0 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -250,8 +250,12 @@ struct apple_dcp { bool external_hotplug_pending; bool external_hotplug_connected; bool external_stopped; + /* The last stop sent nothing and stays owned; see external_stop_retry_wq. */ + bool external_stop_deferred; + unsigned int external_stop_retries; int external_error; struct work_struct external_stop_work; + struct delayed_work external_stop_retry_wq; /* firmware version and compatible firmware version */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 5324dc5bf5655..915510dd3ed04 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2743,6 +2743,14 @@ static int dcp_external_start_locked(struct apple_dcp *dcp) ret = dcp_external_link_locked(dcp); if (ret || dcp->external_running) return ret; + /* A deferred stop sent nothing: the pipe is still powered. */ + if (dcp->external_stop_deferred) { + dcp->external_stop_deferred = false; + dcp->external_stop_retries = 0; + dcp->external_running = true; + iomfb_pause_presentation(dcp, false); + return 0; + } iomfb_pause_presentation(dcp, true); /* Even a timed-out first activation can have changed firmware state. */ dcp->external_power_attempted = true; @@ -2767,6 +2775,8 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) return dcp_external_link_locked(dcp); } +#define DCP_EXTERNAL_STOP_RETRIES 3 + /* The generation token belongs to the CRTC that was actually enabled. */ static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation) { @@ -2787,6 +2797,21 @@ static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation) goto release_link; step = "settling the queue"; ret = iomfb_queue_settle(dcp, msecs_to_jiffies(500)); + if (ret == -ETIMEDOUT) { + /* + * Nothing was sent, so no late reply can race a retry. The + * session stays owned, but not latched: a SAFE transition, the + * retry work or the next enable of the CRTC takes it from here. + */ + dev_warn(dcp->dev, "external display stop deferred: IOMFB queue busy\n"); + dcp->external_stop_deferred = true; + if (dcp->external_stop_retries++ < DCP_EXTERNAL_STOP_RETRIES) + mod_delayed_work(system_wq, &dcp->external_stop_retry_wq, + msecs_to_jiffies(DCP_SWAP_WATCHDOG_MS)); + return ret; + } + dcp->external_stop_deferred = false; + dcp->external_stop_retries = 0; if (!ret) { step = "powering the pipe off"; ret = iomfb_external_power_h17p(dcp, false); @@ -2857,6 +2882,17 @@ static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical) return ret; } +/* Stop a deferred session again once the swap watchdog had its turn. */ +static void dcp_external_stop_retry(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, + external_stop_retry_wq); + + guard(mutex)(&dcp->external_lock); + if (dcp->external_stop_deferred) + dcp_external_stop_locked(dcp, dcp->external_generation); +} + static void dcp_external_stop_work(struct work_struct *work) { struct apple_dcp *dcp = container_of(work, struct apple_dcp, external_stop_work); @@ -3099,6 +3135,7 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) cancel_delayed_work_sync(&dcp->typec_reconnect_wq); cancel_delayed_work_sync(&dcp->placeholder_edid_wq); cancel_work_sync(&dcp->external_stop_work); + cancel_delayed_work_sync(&dcp->external_stop_retry_wq); guard(mutex)(&dcp->external_lock); new_cable = !dcp->external_cable; if (new_cable) @@ -3448,6 +3485,7 @@ void dcp_poweron(struct platform_device *pdev) cancel_delayed_work_sync(&dcp->typec_reconnect_wq); cancel_delayed_work_sync(&dcp->placeholder_edid_wq); cancel_work_sync(&dcp->external_stop_work); + cancel_delayed_work_sync(&dcp->external_stop_retry_wq); guard(mutex)(&dcp->external_lock); ret = dcp_external_start_locked(dcp); if (!ret) @@ -3553,7 +3591,7 @@ void dcp_poweroff(struct platform_device *pdev) cancel_delayed_work_sync(&dcp->placeholder_edid_wq); guard(mutex)(&dcp->external_lock); ret = dcp_external_stop_locked(dcp, dcp->external_crtc_generation); - if (ret) + if (ret && !dcp->external_stop_deferred) dev_err(dcp->dev, "external display stop failed: %d\n", ret); dcp_external_crtc_released(dcp); return; @@ -4135,6 +4173,7 @@ VISIBLE_IF_KUNIT void dcp_disable_typec_work(struct apple_dcp *dcp, bool release disable_delayed_work_sync(&dcp->placeholder_edid_wq); disable_delayed_work_sync(&dcp->announce_wq); cancel_work_sync(&dcp->external_stop_work); + cancel_delayed_work_sync(&dcp->external_stop_retry_wq); } VISIBLE_IF_KUNIT void dcp_enable_typec_work(struct apple_dcp *dcp) @@ -4380,6 +4419,7 @@ VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, v scoped_guard(mutex, &dcp->iomfb.lock) WRITE_ONCE(dcp->quiescing, true); cancel_work_sync(&dcp->external_stop_work); + cancel_delayed_work_sync(&dcp->external_stop_retry_wq); if (dcp->crtc && !dcp->drm_retained) { drm_dev_get(dcp->crtc->base.dev); dcp->drm_retained = true; @@ -4534,6 +4574,7 @@ static int dcp_platform_probe(struct platform_device *pdev) mutex_init(&dcp->hpd_mutex); mutex_init(&dcp->external_lock); INIT_WORK(&dcp->external_stop_work, dcp_external_stop_work); + INIT_DELAYED_WORK(&dcp->external_stop_retry_wq, dcp_external_stop_retry); mutex_init(&dcp->tb_lock); spin_lock_init(&dcp->mode_state.lock); spin_lock_init(&dcp->metadata.lock); diff --git a/drivers/gpu/drm/apple/dcp_dropped_swap_test.c b/drivers/gpu/drm/apple/dcp_dropped_swap_test.c index 8a16ef2efd9b8..b45010d38fac1 100644 --- a/drivers/gpu/drm/apple/dcp_dropped_swap_test.c +++ b/drivers/gpu/drm/apple/dcp_dropped_swap_test.c @@ -18,9 +18,9 @@ * A dock leaves during the first mode set of an external H17P display. The * firmware fails the mode set, accepts the next present and drops it without * a completion or an abort. The teardown that follows waits for the IOMFB - * queue to settle. This replays that sequence through the real queue and - * swap watchdog; the firmware is the test, which answers the present as the - * receive callbacks would and never completes it. + * queue to settle. This replays that sequence through the real queue, swap + * watchdog and external stop; the firmware is the test, which answers the + * present as the receive callbacks would and never completes it. */ #define DROPPED_SWAP_ID 162 @@ -118,6 +118,7 @@ static void dropped_test_exit(void *data) struct iomfb_scanout_h17p *scanout, *next; cancel_delayed_work_sync(&dcp->swap_watchdog_wq); + cancel_delayed_work_sync(&dcp->external_stop_retry_wq); iomfb_queue_stop(dcp); guard(mutex)(&dcp->iomfb.lock); list_for_each_entry_safe(scanout, next, &dcp->iomfb.dropped, link) { @@ -159,6 +160,8 @@ static struct dropped_test *dropped_test_init(struct kunit *test, INIT_DELAYED_WORK(&dcp->swap_watchdog_wq, dcp_swap_watchdog); INIT_DELAYED_WORK(&dcp->typec_reconnect_wq, dropped_test_noop_delayed); INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, dropped_test_noop_delayed); + /* A retried stop would send the power-off, which this test cannot answer. */ + INIT_DELAYED_WORK(&dcp->external_stop_retry_wq, dropped_test_noop_delayed); spin_lock_init(&t->drm.event_lock); t->crtc.base.dev = &t->drm; @@ -322,6 +325,39 @@ static void completed_present_kept_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, READ_ONCE(dcp->mode_state.valid)); } +/* + * The teardown of the captured sequence: a queue that does not settle before + * the power-off is sent leaves no firmware request outstanding. The stop is + * not latched and keeps everything owned; once the watchdog has retired the + * dropped present, a retry finds the queue settled. + */ +static void stop_deferred_test(struct kunit *test) +{ + struct dropped_test *t = dropped_test_external(test); + struct apple_dcp *dcp = &t->dcp; + + dcp->external_prepared = true; + dcp->external_cable = true; + dcp->external_power_attempted = true; + dcp->external_running = true; + dcp->dptxport[0].connected = true; + dcp->pipe_enabled_h17p = true; + dropped_test_present(test, t, 0, &t->fb[1]); + + dcp_poweroff(t->pdev); + KUNIT_EXPECT_EQ(test, dcp->external_error, 0); + KUNIT_EXPECT_FALSE(test, dcp->external_stopped); + KUNIT_EXPECT_TRUE(test, READ_ONCE(dcp->pipe_enabled_h17p)); + KUNIT_EXPECT_TRUE(test, READ_ONCE(dcp->dptxport[0].connected)); + KUNIT_EXPECT_TRUE(test, dcp->iomfb.presentation_paused); + /* Nothing else retries a CRTC that is off: the stop retries itself. */ + KUNIT_EXPECT_TRUE(test, dcp->external_stop_deferred); + KUNIT_EXPECT_TRUE(test, delayed_work_pending(&dcp->external_stop_retry_wq)); + + KUNIT_EXPECT_EQ(test, iomfb_queue_settle(dcp, msecs_to_jiffies(2000)), 0); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dcp->crashed)); +} + /* * Without a completed present for IOMFB_DROPPED_MAX dropped ones, the next one * is not retired: the display is retrained as before and the queue is left to @@ -465,6 +501,7 @@ static struct kunit_case dropped_swap_cases[] = { KUNIT_CASE(dropped_present_retired_test), KUNIT_CASE(dropped_present_buffers_test), KUNIT_CASE(completed_present_kept_test), + KUNIT_CASE(stop_deferred_test), KUNIT_CASE(dropped_present_bounded_test), KUNIT_CASE(legacy_watchdog_test), KUNIT_CASE(h17g_watchdog_test), diff --git a/drivers/gpu/drm/apple/dcp_external_announce_test.c b/drivers/gpu/drm/apple/dcp_external_announce_test.c index 58a0c1247acd8..6dd9792541af2 100644 --- a/drivers/gpu/drm/apple/dcp_external_announce_test.c +++ b/drivers/gpu/drm/apple/dcp_external_announce_test.c @@ -110,6 +110,7 @@ static struct announce_test *announce_test_init(struct kunit *test) INIT_DELAYED_WORK(&dcp->bl_fallback_wq, announce_test_noop); INIT_DELAYED_WORK(&dcp->backlight.fade_work, announce_test_noop); INIT_WORK(&dcp->external_stop_work, announce_test_noop); + INIT_DELAYED_WORK(&dcp->external_stop_retry_wq, announce_test_noop); INIT_WORK(&dcp->vblank_wq, announce_test_noop); INIT_WORK(&t->con.hotplug_wq, announce_test_event); mutex_init(&t->drm.mode_config.mutex); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index b580b1a978e3a..608e7c1872202 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -84,7 +84,9 @@ def function(file, name): #define DCP_HOTPLUG_NOTIFY 1 struct work_struct { bool pending; }; struct delayed_work { struct work_struct work; }; -static int system_freezable_wq; +static int system_freezable_wq, system_wq; +#define DCP_SWAP_WATCHDOG_MS 1000 +#define DCP_EXTERNAL_STOP_RETRIES 3 static int schedule_work(struct work_struct *w) { w->pending = true; return 1; } static void cancel_work_sync(struct work_struct *w) { w->pending = false; } static void cancel_delayed_work_sync(struct delayed_work *w) { w->work.pending = false; } @@ -101,6 +103,8 @@ def function(file, name): struct delayed_work typec_reconnect_wq, placeholder_edid_wq; struct work_struct external_stop_work; bool external_running, external_stopped, external_power_attempted; + bool external_stop_deferred; unsigned int external_stop_retries; + struct delayed_work external_stop_retry_wq; bool external_cable, typec_cable_connected, typec_crtc_off, pipe_enabled_h17p; bool external_physically_absent, external_prepared, placeholder_retried; int typec_reconnect_tries; From 3a432f43ceea65ea6d9b16e08e8b1c9e89b8cebc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 22:26:25 +1100 Subject: [PATCH 1458/2680] phy: apple: atc: Add KUnit tests for the Type-C mux decisions Test how atcphy_typec_target_mode() maps Type-C states to PHY modes, on Type-C ports and on ports behind a fixed hub, and when atcphy_mux_is_noop() lets a mux update skip the lane and PIPE work: a DisplayPort update with the same pin assignment does, while a new mode, a failed transition and a PIPE handoff still owed to a running controller do not. The tests are built into the driver, so that they can reach its static functions. (cherry picked from commit 39bb6d308b70024e3a49097462160b877390ab9c) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/Kconfig | 14 ++++ drivers/phy/apple/atc-test.c | 132 +++++++++++++++++++++++++++++++++++ drivers/phy/apple/atc.c | 4 ++ 3 files changed, 150 insertions(+) create mode 100644 drivers/phy/apple/atc-test.c diff --git a/drivers/phy/apple/Kconfig b/drivers/phy/apple/Kconfig index 26cd09c05497c..e67f51241c29b 100644 --- a/drivers/phy/apple/Kconfig +++ b/drivers/phy/apple/Kconfig @@ -13,6 +13,20 @@ config PHY_APPLE_ATC If M is selected the module will be called 'phy-apple-atc'. +config PHY_APPLE_ATC_KUNIT_TEST + bool "KUnit tests for the Apple Type-C PHY" if !KUNIT_ALL_TESTS + depends on PHY_APPLE_ATC && KUNIT + depends on PHY_APPLE_ATC=m || KUNIT=y + default KUNIT_ALL_TESTS + help + Build the KUnit tests for the Type-C mux decisions of the Apple + Type-C PHY driver into the driver. They check how Type-C states + map to PHY modes and which mux updates may skip the lane and + PIPE work. The tests need no hardware and run when the driver + is loaded. + + If unsure, say N. + config PHY_APPLE_DPTX tristate "Apple DPTX PHY" depends on ARCH_APPLE || COMPILE_TEST diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c new file mode 100644 index 0000000000000..e4452d1ab5218 --- /dev/null +++ b/drivers/phy/apple/atc-test.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * KUnit tests for the Type-C mux decisions of the Apple Type-C PHY driver. + * This file is included from atc.c so that it can test static functions. + */ + +#include +#include + +static struct apple_atcphy *atcphy_test_alloc(struct kunit *test, bool fixed_usb2) +{ + struct apple_atcphy *atcphy; + + atcphy = kunit_kzalloc(test, sizeof(*atcphy), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, atcphy); + atcphy->fixed_usb2 = fixed_usb2; + atcphy->mode = APPLE_ATCPHY_MODE_USB3_DP; + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_USB3; + atcphy->host_active = fixed_usb2; + + return atcphy; +} + +/* The mode lookup reports unsupported states against the PHY device */ +static struct apple_atcphy *atcphy_test_alloc_dev(struct kunit *test, bool fixed_usb2) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc(test, fixed_usb2); + + atcphy->dev = kunit_device_register(test, "atcphy-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, atcphy->dev); + + return atcphy; +} + +static void atcphy_test_target_mode(struct kunit *test) +{ + struct typec_altmode dp = { .svid = USB_TYPEC_DP_SID }; + struct typec_mux_state state = {}; + struct apple_atcphy *atcphy = atcphy_test_alloc_dev(test, false); + + state.mode = TYPEC_STATE_SAFE; + KUNIT_EXPECT_EQ(test, atcphy_typec_target_mode(atcphy, &state), APPLE_ATCPHY_MODE_OFF); + + state.mode = TYPEC_STATE_USB; + KUNIT_EXPECT_EQ(test, atcphy_typec_target_mode(atcphy, &state), APPLE_ATCPHY_MODE_USB3); + + state.alt = &dp; + state.mode = TYPEC_DP_STATE_C; + KUNIT_EXPECT_EQ(test, atcphy_typec_target_mode(atcphy, &state), APPLE_ATCPHY_MODE_DP); + state.mode = TYPEC_DP_STATE_D; + KUNIT_EXPECT_EQ(test, atcphy_typec_target_mode(atcphy, &state), + APPLE_ATCPHY_MODE_USB3_DP); + state.mode = TYPEC_DP_STATE_E; + KUNIT_EXPECT_EQ(test, atcphy_typec_target_mode(atcphy, &state), APPLE_ATCPHY_MODE_DP); + state.mode = TYPEC_DP_STATE_A; + KUNIT_EXPECT_EQ(test, atcphy_typec_target_mode(atcphy, &state), APPLE_ATCPHY_MODE_OFF); +} + +/* A port behind a fixed hub keeps USB2 for its hub whenever a mode turns the PHY off */ +static void atcphy_test_target_mode_fixed_hub(struct kunit *test) +{ + struct typec_altmode dp = { .svid = USB_TYPEC_DP_SID }; + struct typec_mux_state state = {}; + struct apple_atcphy *atcphy = atcphy_test_alloc_dev(test, true); + + state.mode = TYPEC_STATE_SAFE; + KUNIT_EXPECT_EQ(test, atcphy_typec_target_mode(atcphy, &state), APPLE_ATCPHY_MODE_USB2); + + state.alt = &dp; + state.mode = TYPEC_DP_STATE_A; + KUNIT_EXPECT_EQ(test, atcphy_typec_target_mode(atcphy, &state), APPLE_ATCPHY_MODE_USB2); + state.mode = TYPEC_DP_STATE_D; + KUNIT_EXPECT_EQ(test, atcphy_typec_target_mode(atcphy, &state), + APPLE_ATCPHY_MODE_USB3_DP); +} + +static void atcphy_test_noop_same_mode(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc(test, true); + + /* A DisplayPort HPD or status update keeps the pin assignment */ + KUNIT_EXPECT_TRUE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); + + /* The same holds while the controller does not use the USB3 PHY */ + atcphy->host_active = false; + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + KUNIT_EXPECT_TRUE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); + + /* Ports without a fixed hub never route the PIPE from the mux */ + atcphy = atcphy_test_alloc(test, false); + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + atcphy->pipe_pending = true; + KUNIT_EXPECT_TRUE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); +} + +static void atcphy_test_noop_needs_work(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc(test, true); + + /* A new pin assignment or the safe state rebuild the lanes */ + KUNIT_EXPECT_FALSE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_DP)); + KUNIT_EXPECT_FALSE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB2)); + + /* So does a failed transition or an orientation change */ + atcphy->mode_dirty = true; + KUNIT_EXPECT_FALSE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); + atcphy->mode_dirty = false; + + /* A running controller is still owed its USB3 backend */ + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + KUNIT_EXPECT_FALSE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); + + /* or a handoff that has not completed */ + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_USB3; + atcphy->pipe_pending = true; + KUNIT_EXPECT_FALSE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); +} + +static struct kunit_case atcphy_mux_test_cases[] = { + KUNIT_CASE(atcphy_test_target_mode), + KUNIT_CASE(atcphy_test_target_mode_fixed_hub), + KUNIT_CASE(atcphy_test_noop_same_mode), + KUNIT_CASE(atcphy_test_noop_needs_work), + {} +}; + +static struct kunit_suite atcphy_mux_test_suite = { + .name = "phy-apple-atc-mux", + .test_cases = atcphy_mux_test_cases, +}; + +kunit_test_suite(atcphy_mux_test_suite); diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 90979abce1b13..37347a64d94ad 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -4194,3 +4194,7 @@ module_platform_driver(atcphy_driver); MODULE_AUTHOR("Sven Peter "); MODULE_DESCRIPTION("Apple Type-C PHY driver"); MODULE_LICENSE("GPL"); + +#if IS_ENABLED(CONFIG_PHY_APPLE_ATC_KUNIT_TEST) +#include "atc-test.c" +#endif From fc4a99163451e6f8bcabdcdffb956b8b9429ab86 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 19:48:59 +1100 Subject: [PATCH 1459/2680] selftests: apple-dcp: Replay a dock that leaves during the first mode set Add dropped-swap-teardown, with tipd's actual update and retry work and the actual external-session functions of drm/apple, against an IOMFB queue that a present the firmware dropped keeps busy until the swap watchdog retires it, or until the transaction timeout declares the coprocessor crashed: - captured: tipd's out-of-band disconnect and its SAFE find the queue busy. The stop is deferred, not latched, and the session stays owned with nothing sent. The next revalidation stops the session, SAFE reaches the PHY and the next plug connects. - retired-during-retry: the watchdog retires the present while the retry waits for the queue. - never-retired: nothing retires the present. The session stays owned through all of tipd's retries. Once the coprocessor is declared crashed, the stop latches and the next plug is refused. - after-power-off: a settle that times out after the power-off request still latches and holds the session. - crashed-queue: a crashed queue still latches. - crtc-off: the same deferral for a CRTC switched off on the same cable. - stop-retry: the deferred stop retries itself, and stops retrying after three deferrals; the session stays owned. - enable-after-deferral: enabling the CRTC on the same cable resumes the session without a second power-on; with the cable gone it is refused. The SAFE prepare of the DCP's Type-C route is modelled by its H17P branch; quiesce-before-safe runs the real one. (cherry picked from commit 96adf952fcc9dfb4014c899e94ea26685fb629d4) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/gpu/apple-dcp/README | 5 +- .../drivers/gpu/apple-dcp/lifecycle.py | 190 ++++++++++++++++++ .../selftests/drivers/gpu/apple-dcp/run.py | 2 + 3 files changed, 196 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/README b/tools/testing/selftests/drivers/gpu/apple-dcp/README index 1f5e3cc8cd68d..33373ea702c30 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/README +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/README @@ -30,7 +30,10 @@ The modules contain public synthetic host boundaries and retained assertions: warm reset and legacy T8103/T8122/T8140/T8142 route controls. * lifecycle: actual physical-disconnect authority and ownership despite failed teardown; a fast dock replug whose DPTX link never configures (withdrawn, - unacknowledged, unanswered, refused and live failures); DWC3/core/PHY PM + unacknowledged, unanswered, refused and live failures); a dock that leaves + during the first mode set, whose dropped present keeps the IOMFB queue busy + (deferred and retried stop, never-retired, post-power-off and crashed-queue + latches); DWC3/core/PHY PM and reset-reference recovery; exact firmware tuple/resource admission, unsupported VRR rejection and power reply bounds; 15,552 D134 dispatcher scope/length/context and legacy reply cases. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 608e7c1872202..db72bc324ee11 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -543,6 +543,196 @@ def swap(old, new): ''' return unit + firmware + raw + '\n' + tests +def dropped(root): + """A dock leaves during the first mode set, and the next swap is dropped. + + The physical-disconnect fixture with tipd's actual update and retry work + and the actual external stop, against an IOMFB queue that a dropped + present keeps busy until the swap watchdog retires it, or until the + transaction timeout declares the coprocessor crashed. Time is the + queue's own clock, advanced by the waits of the stop and of tipd. + """ + unit = disconnect(root) + dcp = (root / 'drivers/gpu/drm/apple/dcp.c').read_text() + extra = block(dcp, dcp.index('static int dcp_external_start_locked(')) + # Without a retry of its own, a deferred stop of a CRTC that is off waits. + retry = 'static void dcp_external_stop_retry(' + extra += '\n' + (block(dcp, dcp.index(retry)) if retry in dcp else + retry + 'struct work_struct *work) { (void)work; }') + + def swap(old, new): + nonlocal unit + assert unit.count(old) == 1, old + unit = unit.replace(old, new) + + swap('static int iomfb_queue_settle(struct apple_dcp *d, int t)\n' + '{ (void)d; (void)t; settles++; return settle_error; }', + '/* Milliseconds since the firmware accepted the dropped present. */\n' + 'static int now, retire_at, crash_at, after_power_error;\n' + 'static bool pending, crashed, powered;\n' + 'static int iomfb_queue_settle(struct apple_dcp *d, int t)\n' + '{ (void)d; settles++; if (settle_error) return settle_error;\n' + ' if (powered) { powered = false; if (after_power_error) return after_power_error; }\n' + ' if (pending && retire_at >= 0 && now + t >= retire_at) {\n' + ' if (now < retire_at) now = retire_at; pending = false; }\n' + ' if (pending && now + t >= crash_at) { if (now < crash_at) now = crash_at; crashed = true; }\n' + ' if (crashed) return -EIO;\n' + ' if (pending) { now += t; return -ETIMEDOUT; }\n' + ' return 0; }') + swap('static int iomfb_external_power_h17p(struct apple_dcp *d, bool on)\n' + '{ (void)d; assert(!on); powers++; return power_error; }', + 'static int iomfb_external_power_h17p(struct apple_dcp *d, bool on)\n' + '{ (void)d; if (on) { power_ons++; return 0; }\n' + ' assert(!pending); powers++; powered = !power_error; return power_error; }') + swap('static int settle_error,', 'static int power_ons, settle_error,') + # The H17P branch of dcp_typec_route_prepare(), which quiesce-before-safe + # runs for real inside typec_mux_set(). + swap('static int typec_set_mode(void *p, int s)\n' + '{ (void)p; assert(s == TYPEC_STATE_SAFE); safes++; return safe_error; }', + 'static int phy_safes;\n' + 'static int typec_set_mode(void *p, int s)\n' + '{ int r; (void)p; assert(s == TYPEC_STATE_SAFE); safes++;\n' + ' if (display.external_error) return display.external_error;\n' + ' if (display.external_cable || !display.external_stopped || display.dptxport[0].connected) {\n' + ' r = dcp_external_disconnect(&display, true); if (r) return r; }\n' + ' phy_safes++; return 0; }') + swap('int main(void)\n{', 'static int __attribute__((unused)) prior_main(void)\n{') + tests = r''' +static struct platform_device device = { .data = &display }; +/* The dock has left; the firmware accepted a present it will not complete. */ +static void left(int retire) +{ + init(); now = 0; retire_at = retire; crash_at = 10000; after_power_error = 0; + pending = true; crashed = powered = false; phy_safes = power_ons = 0; +} +/* tipd's revalidation of the cable after its debounce, as resume_work. */ +static bool revalidate(void) +{ + now += CD321X_DEBOUNCE_DELAY_MS; + if (!cd321x_pm_begin_read(&port.pm)) return false; + cd321x_pm_snapshot_ready(&port.pm); cd321x_update_work(&port.update_work.work); + return true; +} +/* Nothing of the session has been released, and nothing was sent. */ +static void owned(void) +{ + CHECK(!display.external_stopped && display.dptxport[0].connected); + CHECK(display.pipe_enabled_h17p && !display.typec_crtc_off); + CHECK(display.iomfb.presentation_paused && !display.external_running); + CHECK(powers == 0 && releases == 0 && phy_safes == 0); +} +int main(void) +{ + /* + * The capture: tipd's out-of-band disconnect and then its SAFE stop the + * session while the dropped present is in flight. Each settle times out + * after 500 ms, before the power-off is sent: not latched. The watchdog + * retires the present, and tipd's next revalidation stops the session and + * reaches the PHY. + */ + left(1500); + cd321x_update_work(&port.update_work.work); + CHECK(safes == 1 && settles == 2 && now == 1000); + CHECK(!display.external_error && !display.external_cable && !sink.connected); + owned(); + CHECK(port.pm.phase == CD321X_PM_REVALIDATE && !port.state_valid); + CHECK(revalidate()); + CHECK(safes == 2 && phy_safes == 1 && powers == 1 && releases == 1); + CHECK(!display.external_error && display.external_stopped); + CHECK(!display.dptxport[0].connected && !display.pipe_enabled_h17p && display.typec_crtc_off); + CHECK(port.state_valid && port.pm.phase == CD321X_PM_RUNNING); + /* The next plug connects. */ + CHECK(dcp_dptx_connect_oob(&device, 0) == 0); + CHECK(display.dptxport[0].connected && display.external_cable && !display.external_error); + printf("case captured complete\n"); + + /* The watchdog retires the present during the retry's settle. */ + left(1900); + cd321x_update_work(&port.update_work.work); + CHECK(!display.external_error); owned(); + CHECK(revalidate() && phy_safes == 1 && display.external_stopped && now == 1900); + printf("case retired-during-retry complete\n"); + + /* + * Nothing retires the present. The stop is retried within tipd's budget + * and the session stays owned throughout; once the transaction timeout + * declares the coprocessor crashed, the stop latches and the next plug is + * refused: the protected case. + */ + left(-1); + cd321x_update_work(&port.update_work.work); + while (revalidate()) + ; + CHECK(safes == 1 + CD321X_RESUME_ATTEMPTS && port.pm.phase == CD321X_PM_STALE); + CHECK(!display.external_error); owned(); + now = 12000; + CHECK(cd321x_pm_event(&port.pm) && revalidate()); + CHECK(display.external_error == -EIO); owned(); + CHECK(dcp_dptx_connect_oob(&device, 0) == -EIO && raw_links == 0); + printf("case never-retired complete\n"); + + /* A settle after the power-off was sent still latches and holds. */ + left(0); pending = false; after_power_error = -ETIMEDOUT; + cd321x_update_work(&port.update_work.work); + CHECK(powers == 1 && display.external_error == -ETIMEDOUT); + CHECK(display.dptxport[0].connected && display.pipe_enabled_h17p && !display.external_stopped); + CHECK(revalidate() && phy_safes == 0 && powers == 1 && releases == 0); + printf("case after-power-off complete\n"); + + /* A crashed or stopped queue is no proof that DMA stopped: latched. */ + left(1000); settle_error = -EIO; + cd321x_update_work(&port.update_work.work); + CHECK(display.external_error == -EIO); settle_error = 0; + CHECK(revalidate() && phy_safes == 0 && powers == 0); + printf("case crashed-queue complete\n"); + + /* The same deferral for a CRTC switched off on the same cable. */ + left(1000); + CHECK(dcp_external_disconnect(&display, false) == -ETIMEDOUT); + CHECK(!display.external_error && display.external_cable); owned(); + now = 1000; + CHECK(dcp_external_disconnect(&display, false) == 0 && display.external_stopped && powers == 1); + printf("case crtc-off complete\n"); + + /* Nothing else retries a CRTC that is off: the stop retries itself, bounded. */ + left(1000); + CHECK(dcp_external_disconnect(&display, false) == -ETIMEDOUT); + CHECK(display.external_stop_deferred && display.external_stop_retry_wq.work.pending); + now = 1000; display.external_stop_retry_wq.work.pending = false; + dcp_external_stop_retry(&display.external_stop_retry_wq.work); + CHECK(display.external_stopped && !display.external_stop_deferred && powers == 1); + CHECK(!display.external_stop_retry_wq.work.pending && !display.external_stop_retries); + left(-1); crash_at = 1 << 30; + for (int i = 0; i <= DCP_EXTERNAL_STOP_RETRIES; i++) { + display.external_stop_retry_wq.work.pending = false; + dcp_external_stop_retry(&display.external_stop_retry_wq.work); + if (!i) CHECK(dcp_external_disconnect(&display, false) == -ETIMEDOUT); + } + CHECK(!display.external_stop_retry_wq.work.pending && !display.external_error); owned(); + printf("case stop-retry complete\n"); + + /* + * The CRTC is enabled again on the same cable before any stop went + * through: the pipe was never powered off, so it is resumed, not powered + * on a second time. + */ + left(1000); + CHECK(dcp_external_disconnect(&display, false) == -ETIMEDOUT); owned(); + CHECK(dcp_external_start_locked(&display) == 0); + CHECK(power_ons == 0 && powers == 0 && display.external_running); + CHECK(!display.iomfb.presentation_paused && !display.external_stop_deferred); + CHECK(display.pipe_enabled_h17p && display.dptxport[0].connected); + /* Once the cable is gone, the deferred session is not resumed. */ + left(1000); + CHECK(dcp_external_disconnect(&display, true) == -ETIMEDOUT); + CHECK(dcp_external_start_locked(&display) == -ENODEV && power_ons == 0); owned(); + printf("case enable-after-deferral complete\n"); + + printf("%d assertions, %d failures\n", checks, failures); return !!failures; +} +''' + return unit + extra + '\n' + tests + def teardown(root): files = {n: (root / f).read_text() for n, f in { diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py index bf24520697696..5c3b6968d1def 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -80,6 +80,8 @@ def main(): ('-Wno-unused-variable', '-Wno-unused-function', '-Wno-unused-but-set-variable')), ('lifecycle', 'fast-replug-bounce', lifecycle.bounce, ((),), ('-Wno-unused-variable', '-Wno-unused-function', '-Wno-unused-but-set-variable')), + ('lifecycle', 'dropped-swap-teardown', lifecycle.dropped, ((),), + ('-Wno-unused-variable', '-Wno-unused-function', '-Wno-unused-but-set-variable')), ('lifecycle', 'dwc3-teardown', lifecycle.teardown, (('all',),), ()), ('lifecycle', 'resource-profile', lifecycle.profile, ((),), ('-Wno-sign-compare',)), ('lifecycle', 'firmware-provider', lifecycle.firmware, ((),), ()), From 06aaa2dfe6a6148df6e8024798a91350967c3969 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:37:41 +1100 Subject: [PATCH 1460/2680] phy: apple: atc: Split the mux and switch callbacks from their PHY work Move the bodies of atcphy_mux_set() and atcphy_sw_set() into atcphy_mux_update() and atcphy_set_orientation(), which take the PHY rather than the Type-C mux or switch device. The callbacks only look up the PHY and call them. This lets the driver's KUnit tests run a whole mux update and an orientation change on a PHY without registering Type-C devices. No functional change. (cherry picked from commit 64ef3b3a84f0a3faf5d4c5f333cb52284f5075fa) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 37347a64d94ad..59ff76e7aa9ed 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -3381,9 +3381,8 @@ static int atcphy_probe_rcdev(struct apple_atcphy *atcphy) return devm_reset_controller_register(atcphy->dev, &atcphy->rcdev); } -static int atcphy_sw_set(struct typec_switch_dev *sw, enum typec_orientation orientation) +static int atcphy_set_orientation(struct apple_atcphy *atcphy, enum typec_orientation orientation) { - struct apple_atcphy *atcphy = typec_switch_get_drvdata(sw); bool swap_lanes; guard(mutex)(&atcphy->lock); @@ -3412,6 +3411,11 @@ static int atcphy_sw_set(struct typec_switch_dev *sw, enum typec_orientation ori return 0; } +static int atcphy_sw_set(struct typec_switch_dev *sw, enum typec_orientation orientation) +{ + return atcphy_set_orientation(typec_switch_get_drvdata(sw), orientation); +} + static void atcphy_typec_switch_unregister(void *data) { typec_switch_unregister(data); @@ -3802,9 +3806,8 @@ static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode t return atcphy_configure(atcphy, target_mode); } -static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state) +static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux_state *state) { - struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); struct atcphy_usb3_pm pm = {}; enum atcphy_mode target_mode; int ret; @@ -3829,6 +3832,11 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta return ret; } +static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state) +{ + return atcphy_mux_update(typec_mux_get_drvdata(mux), state); +} + static void atcphy_typec_mux_unregister(void *data) { typec_mux_unregister(data); From 1bbaab74a4764e6fd93c2af7efd33eaa42626ae9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 02:05:56 +1100 Subject: [PATCH 1461/2680] iommu: apple-dart: Drain runtime PM before injecting test errors The gated-IRQ test injects a runtime-PM error after a successful IRQ, whose pm_runtime_put() queues asynchronous idle work. A pending PM callback can overwrite runtime_error after the injection, allowing the next IRQ to run its handler and making the test fail intermittently. Holding power.lock during injection does not wait for that callback. Drain runtime-PM work before injecting the error so that the failed-resume case reliably exercises the error path. Keep the IRQ, register and usage count assertions. (cherry picked from commit c6466de02fc51a0de273dddba59af7df9a9c46f6) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart-test.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index e0fbd3218146a..850e42297439c 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -471,6 +471,9 @@ static void apple_dart_test_gated_irq(struct kunit *test) KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 0)), 0x1234U); KUNIT_EXPECT_EQ(test, atomic_read(&dart->dev->power.usage_count), 0); + /* Drain the work queued by the successful IRQ's pm_runtime_put(). */ + pm_runtime_barrier(dart->dev); + /* Failed runtime resume must not run the handler or leak its reference. */ writel(0, dart->regs + DART_TCR(dart, 0)); scoped_guard(spinlock_irqsave, &dart->dev->power.lock) From 853e443b4e6344ab28f050025ad2e8fe6d85ad6f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 20:09:20 +1100 Subject: [PATCH 1462/2680] drm/apple: Drop the mode of an external display whose link faults The DPTX endpoint of the external H17P display reports a sink that stops answering, a busy timeout or a link that did not start through its DEVICE_NOT_RESPONDING, DEVICE_BUSY_TIMEOUT and DEVICE_NOT_STARTED callbacks. The driver only revoked the 180 Hz rate proof there. The mode stayed valid, so a link that faulted after its mode set, without an HPD change, stayed reported connected with link-status GOOD and nothing set its mode again; and a fault during the mode set that recovers the link was undone when the mode set completed. On the external H17P display, treat such a fault as the end of the mode: invalidate it, which a mode set in flight then reports as failed when it finishes, and queue the hotplug worker when a set mode was lost, so that it sets the mode of the active CRTC again or reports the display lost. Before any mode was set there is nothing to recover and nothing is queued. dcp_mode_invalidate() now says whether the mode was valid. The 180 Hz admission conditions are unchanged. Other profiles keep their behaviour. (cherry picked from commit 90324ce24af9d4974e68b62ed0ad2e0fc465ee08) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 16 +++++ .../gpu/drm/apple/dcp_external_relink_test.c | 66 +++++++++++++++++++ drivers/gpu/drm/apple/dptxep.c | 2 +- drivers/gpu/drm/apple/iomfb-state.h | 9 ++- 5 files changed, 92 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index c65b96541a9f0..589da7130aec0 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -570,6 +570,7 @@ bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected); void dcp_announce_arm(struct apple_dcp *dcp); void dcp_announce_work(struct work_struct *work); void dcp_external_crtc_released(struct apple_dcp *dcp); +void dcp_external_link_fault(struct apple_dcp *dcp); /* A KUnit test can wait here for the work it races with. */ static inline void dcp_test_point(struct apple_dcp *dcp, enum dcp_test_point point) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 915510dd3ed04..a9cd020ca734a 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2313,6 +2313,22 @@ void dcp_swap_watchdog_complete(struct apple_dcp *dcp) dcp->swap_watchdog_retrains = 0; } +/* + * The firmware reports that the DisplayPort sink stopped answering, or that + * the link did not start. On the external H17P display, the mode set + * before, or being set, has no link any more: drop it, also across a mode + * set in flight, and have the hotplug worker set the mode of the CRTC again + * or report the display lost (dcp_external_relink()). A mode set still in + * flight reports this itself when it finishes, and before any mode was set + * there is nothing to recover. + */ +void dcp_external_link_fault(struct apple_dcp *dcp) +{ + dcp_180hz_invalidate(dcp); + if (dcp_is_external_h17p(dcp) && dcp_mode_invalidate(&dcp->mode_state)) + dcp_handle_hotplug_actions(dcp, DCP_HOTPLUG_NOTIFY); +} + static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) { struct apple_dcp *dcp = cookie; diff --git a/drivers/gpu/drm/apple/dcp_external_relink_test.c b/drivers/gpu/drm/apple/dcp_external_relink_test.c index 5b48c42610a35..04a405a8e813d 100644 --- a/drivers/gpu/drm/apple/dcp_external_relink_test.c +++ b/drivers/gpu/drm/apple/dcp_external_relink_test.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -55,6 +56,7 @@ struct relink_test { bool bounce_on_probe; /* the route bounces again during a probe */ bool real_modeset; /* atomic_enable is the real dcp_crtc_atomic_modeset() */ bool force_modeset; /* a userspace commit changes the connectors */ + bool fault_in_call; /* the link faults before the firmware answers */ u32 reply; /* what the firmware answers a call with */ int modeset_ret; /* the result of the real mode set */ struct dcp_display_mode catalog; @@ -129,7 +131,10 @@ static void relink_test_push(struct apple_dcp *dcp, const struct dcp_method_entr dcp_callback_t cb, void *cookie) { struct relink_test *t = container_of(dcp, struct relink_test, dcp); + bool modeset = !strcmp(call->name, "dcpep_set_digital_out_mode"); + if (modeset && t->fault_in_call) + dcp_external_link_fault(dcp); cb(dcp, &t->reply, cookie); } @@ -823,6 +828,64 @@ static void relink_lost_quiet_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); } +/* + * The link faults while the recovery mode set waits for the firmware, which + * then answers: the mode set did not bring the link up. + */ +static void relink_fault_during_modeset_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + relink_test_real(t); + t->fault_in_call = true; + relink_test_tear(t); + relink_test_reconnect(test, t); + KUNIT_EXPECT_FALSE(test, relink_test_link_up(t)); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_BAD); +} + +/* + * The link faults after a completed mode set, with no hotplug: the mode is + * gone at once, and the worker sets it again. + */ +static void relink_fault_after_modeset_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + int modesets; + + relink_test_real(t); + relink_test_tear(t); + relink_test_reconnect(test, t); + KUNIT_ASSERT_EQ(test, t->modeset_ret, 0); + KUNIT_ASSERT_TRUE(test, relink_test_link_up(t)); + /* See the fault's effect before the worker it queues can run. */ + disable_work_sync(&t->con.hotplug_wq); + modesets = atomic_read(&t->modesets); + dcp_external_link_fault(&t->dcp); + KUNIT_EXPECT_FALSE(test, relink_test_link_up(t)); + KUNIT_EXPECT_TRUE(test, t->dcp.mode_state.invalidated); + enable_work(&t->con.hotplug_wq); + relink_test_hotplug(t); + relink_test_settle(test, t); + KUNIT_EXPECT_GT(test, atomic_read(&t->modesets), modesets); + KUNIT_EXPECT_TRUE(test, relink_test_link_up(t)); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_GOOD); +} + +/* A fault with no mode set queues nothing: there is nothing to recover. */ +static void relink_fault_inactive_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + KUNIT_ASSERT_EQ(test, relink_test_commit(t, false, false), 0); + relink_test_settle(test, t); + disable_work_sync(&t->con.hotplug_wq); + dcp_external_link_fault(&t->dcp); + KUNIT_EXPECT_FALSE(test, work_pending(&t->con.hotplug_wq)); + enable_work(&t->con.hotplug_wq); +} + static struct kunit_case relink_cases[] = { KUNIT_CASE(relink_bounce_test), KUNIT_CASE(relink_mode_gone_test), @@ -838,6 +901,9 @@ static struct kunit_case relink_cases[] = { KUNIT_CASE(relink_early_epipe_test), KUNIT_CASE(relink_paused_kept_test), KUNIT_CASE(relink_lost_quiet_test), + KUNIT_CASE(relink_fault_during_modeset_test), + KUNIT_CASE(relink_fault_after_modeset_test), + KUNIT_CASE(relink_fault_inactive_test), {} }; diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 06a1a3a093b08..275d64d98d037 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -744,7 +744,7 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, case DPTX_APCALL_DEVICE_NOT_RESPONDING: case DPTX_APCALL_DEVICE_BUSY_TIMEOUT: case DPTX_APCALL_DEVICE_NOT_STARTED: - dcp_180hz_invalidate(service->ep->dcp); + dcp_external_link_fault(service->ep->dcp); dev_warn(service->ep->dcp->dev, "DPTXPort: firmware reports link fault %u on target %u:%u\n", idx, service->ep->dcp->dptx_die, diff --git a/drivers/gpu/drm/apple/iomfb-state.h b/drivers/gpu/drm/apple/iomfb-state.h index a9ada00a8528b..4cf48977ffd74 100644 --- a/drivers/gpu/drm/apple/iomfb-state.h +++ b/drivers/gpu/drm/apple/iomfb-state.h @@ -32,14 +32,21 @@ static inline void dcp_mode_begin(struct dcp_mode_state *state) spin_unlock_irqrestore(&state->lock, flags); } -static inline void dcp_mode_invalidate(struct dcp_mode_state *state) +/* + * The mode is gone, also for a mode set in flight, which then does not + * complete it. Returns whether a set mode was valid until now. + */ +static inline bool dcp_mode_invalidate(struct dcp_mode_state *state) { unsigned long flags; + bool valid; spin_lock_irqsave(&state->lock, flags); + valid = state->valid; state->valid = false; state->invalidated = true; spin_unlock_irqrestore(&state->lock, flags); + return valid; } /* The caller holds state->lock. */ From 9ffeebc10fab7dbdff8b6726ba319a8b7e33aa90 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:38:03 +1100 Subject: [PATCH 1463/2680] phy: apple: atc: Test that a mux update keeping its mode skips the USB3 root hub The existing tests check atcphy_mux_is_noop() on its own, so they still pass if atcphy_mux_update() stops returning early and takes the USB3 root hub again for every DisplayPort HPD or status update. Run whole mux updates on a PHY behind a fixed hub that has no device tree node. Taking the root hub then fails with -ENODEV at the controller lookup, before any register access. An update with the same pin assignment and orientation must return 0. A new pin assignment, a pending PIPE handoff, a PIPE away from the USB3 backend of a running host, and a cable flip reported through atcphy_set_orientation() must all reach the root hub. (cherry picked from commit 0aeb0acde3e1962a1d0e617d8d06da7741bfe78e) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-test.c | 68 ++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index e4452d1ab5218..a8c0c57d09859 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -116,11 +116,79 @@ static void atcphy_test_noop_needs_work(struct kunit *test) KUNIT_EXPECT_FALSE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); } +/* + * A J700-style PHY behind a fixed hub, in DisplayPort + USB3 with the PIPE on + * USB3 for a running host, but without a device tree node. A mux update that + * goes for the USB3 root hub then fails with -ENODEV at the controller lookup, + * before any register access, which tells the two paths apart. + */ +static struct apple_atcphy *atcphy_test_alloc_fixed_hub(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_dev(test, true); + + atcphy->hw = &atcphy_hw_t8140; + mutex_init(&atcphy->lock); + + return atcphy; +} + +static const struct typec_altmode atcphy_test_dp = { .svid = USB_TYPEC_DP_SID }; + +static int atcphy_test_update_dp(struct apple_atcphy *atcphy, unsigned long pin) +{ + struct typec_mux_state state = { + .alt = (struct typec_altmode *)&atcphy_test_dp, + .mode = pin, + }; + + return atcphy_mux_update(atcphy, &state); +} + +static void atcphy_test_update_skips_usb(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + + /* An HPD or status update with the same pin assignment */ + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + KUNIT_EXPECT_EQ(test, atcphy->mode, APPLE_ATCPHY_MODE_USB3_DP); + KUNIT_EXPECT_FALSE(test, atcphy->mode_dirty); + + /* The orientation the lanes already use */ + KUNIT_EXPECT_EQ(test, atcphy_set_orientation(atcphy, TYPEC_ORIENTATION_NORMAL), 0); + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); +} + +static void atcphy_test_update_takes_usb(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + + /* A new pin assignment */ + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), -ENODEV); + KUNIT_EXPECT_EQ(test, atcphy->mode, APPLE_ATCPHY_MODE_USB3_DP); + + /* A PIPE handoff that has not completed */ + atcphy->pipe_pending = true; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), -ENODEV); + atcphy->pipe_pending = false; + + /* A running host whose PIPE is not on the USB3 backend */ + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), -ENODEV); + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_USB3; + + /* A cable flip moves the USB3 and DisplayPort lanes */ + KUNIT_EXPECT_EQ(test, atcphy_set_orientation(atcphy, TYPEC_ORIENTATION_REVERSE), 0); + KUNIT_EXPECT_TRUE(test, atcphy->mode_dirty); + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), -ENODEV); +} + static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_target_mode), KUNIT_CASE(atcphy_test_target_mode_fixed_hub), KUNIT_CASE(atcphy_test_noop_same_mode), KUNIT_CASE(atcphy_test_noop_needs_work), + KUNIT_CASE(atcphy_test_update_skips_usb), + KUNIT_CASE(atcphy_test_update_takes_usb), {} }; From 85131eb5c88547c392493f709d294f05c9b5408d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:10:49 +1100 Subject: [PATCH 1464/2680] iommu/apple-dart: Do not reset a DART that kept its state across sleep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit apple_dart_resume() resets every DART on resume, clearing its page table pointers and disabling translation before restoring the saved values. That is only correct when the DART lost power. A DART whose power domain stayed on still has its state, and may still be translating for a consumer that kept running through the sleep: dwc3-apple keeps its core up across system sleep as a syscore device, which holds the shared power domain on, and with xHCI now left running in suspend-to-idle the controller can write events or touch its scratchpad while the DART is resumed. A DMA that hits the reset window faults, and xHCI halts on a Host System Error. Compare the translation registers with the values saved at suspend. When they still match and at least one page table is valid, the DART did not lose power: leave it alone and only invalidate its TLBs. Otherwise reset and restore it as before. A powered-off DART reads back with no valid page table, so it always takes the reset path. The TLBs are flushed with one command per stream rather than one command for a map of all streams. Where the streams of a map are remapped onto its first stream, a map flush only reaches that first stream, which would leave stale entries of every other stream behind. Two consequences beyond the USB case: - The suspend and resume callbacks are shared by runtime PM and system sleep. Every runtime resume of an unlocked DART whose power domain stayed on now also skips the reset and only invalidates the TLBs. This applies to all DARTs the driver handles, on other Apple SoCs as well, including the t8110 ones. - "Retained" is judged only by TCR and TTBR matching the values saved at suspend. On that path the error status and the stream enables are no longer reset; they are left as the DART kept them. [ryan: adapted to 7.2.10 — context only: the resume tests follow the 7.2.10 DART tests in apple-dart-test.c; no code change] (cherry picked from commit 82494bf5c6b07739782eeac837b9f6a96c70ca36) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart-test.c | 91 +++++++++++++++++++++++++++++++++ drivers/iommu/apple-dart.c | 56 ++++++++++++++++++++ 2 files changed, 147 insertions(+) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 850e42297439c..fcde0a6a6ff6f 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -584,6 +584,93 @@ static void apple_dart_test_independent_remap_roots(struct kunit *test) test->priv = NULL; } +static struct apple_dart *apple_dart_test_sleeping(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_tunnel(test); + unsigned int sid; + + dart->tunneled = false; + dart->hw = &apple_dart_hw_t8103; + dart->num_streams = 16; + for (sid = 0; sid < dart->num_streams; sid++) + writel(0x80 + sid, dart->regs + DART_TCR(dart, sid)); + writel(dart->hw->ttbr_valid | 0x1000, dart->regs + DART_TTBR(dart, 2, 0)); + KUNIT_ASSERT_EQ(test, apple_dart_suspend(dart->dev), 0); + return dart; +} + +static void apple_dart_test_resume_retained(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_sleeping(test); + u32 enable = dart->hw->enable_streams; + + /* The domain stayed on and a consumer may still be doing DMA: no reset. */ + writel(0, dart->regs + enable); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); + KUNIT_EXPECT_EQ(test, readl(dart->regs + enable), 0U); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 2)), 0x82U); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TTBR(dart, 2, 0)), + dart->hw->ttbr_valid | 0x1000); +} + +static void apple_dart_test_resume_lost(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_sleeping(test); + + /* Power was cut: reset and restore the saved translations. */ + memset_io(dart->regs, 0, SZ_16K); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); + KUNIT_EXPECT_EQ(test, readl(dart->regs + dart->hw->enable_streams), U32_MAX); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 2)), 0x82U); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TTBR(dart, 2, 0)), + dart->hw->ttbr_valid | 0x1000); +} + +static void apple_dart_test_resume_unused(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_tunnel(test); + + /* Matching registers without any page table prove nothing: reset. */ + dart->tunneled = false; + dart->hw = &apple_dart_hw_t8103; + dart->num_streams = 16; + KUNIT_ASSERT_EQ(test, apple_dart_suspend(dart->dev), 0); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); + KUNIT_EXPECT_EQ(test, readl(dart->regs + dart->hw->enable_streams), U32_MAX); +} + +/* + * A DART that kept its state is flushed stream by stream. Where the streams of + * a map are remapped onto its first stream, as on DARTs that program only that + * stream, a single flush of all streams would reach stream 0 and nothing else. + */ +static void apple_dart_test_resume_retained_each_stream(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_tunnel(test); + unsigned int sid; + u32 cmd; + + dart->tunneled = false; + dart->hw = &apple_dart_hw_t8110; + dart->num_streams = 16; + for (sid = 0; sid < dart->num_streams; sid++) + writel(dart->hw->tcr_enabled, dart->regs + DART_TCR(dart, sid)); + writel(dart->hw->ttbr_valid | 0x1000, dart->regs + DART_TTBR(dart, 0, 0)); + writel(dart->hw->ttbr_valid | 0x2000, dart->regs + DART_TTBR(dart, 5, 0)); + KUNIT_ASSERT_EQ(test, apple_dart_suspend(dart->dev), 0); + + writel(0, dart->regs + DART_T8110_TLB_CMD); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); + /* Commands go out in stream order: the last one names the last stream. */ + cmd = readl(dart->regs + DART_T8110_TLB_CMD); + KUNIT_EXPECT_EQ(test, FIELD_GET(DART_T8110_TLB_CMD_STREAM, cmd), dart->num_streams - 1); + KUNIT_EXPECT_EQ(test, FIELD_GET(DART_T8110_TLB_CMD_OP, cmd), + (u32)DART_T8110_TLB_CMD_OP_FLUSH_SID); + /* Nothing was reset. */ + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TTBR(dart, 5, 0)), + dart->hw->ttbr_valid | 0x2000); +} + static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_independent_remap_roots), KUNIT_CASE(apple_dart_test_before_attach), @@ -598,6 +685,10 @@ static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_tunnel_reset_failure), KUNIT_CASE(apple_dart_test_failed_host_pm), KUNIT_CASE(apple_dart_test_gated_irq), + KUNIT_CASE(apple_dart_test_resume_retained), + KUNIT_CASE(apple_dart_test_resume_lost), + KUNIT_CASE(apple_dart_test_resume_unused), + KUNIT_CASE(apple_dart_test_resume_retained_each_stream), {} }; diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 5cfa98a956916..a89394c20ca06 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -3177,6 +3177,58 @@ static __maybe_unused int apple_dart_suspend(struct device *dev) return 0; } +/* + * Whether the DART still holds the translation state that apple_dart_suspend() + * saved, because its power domain stayed on across system sleep. That is the + * case while a consumer that keeps running in suspend-to-idle, such as a DWC3 + * controller in host mode, holds the domain on. Such a consumer may still be + * doing DMA, so the DART must not be reset under it. + */ +static bool apple_dart_state_retained(struct apple_dart *dart) +{ + bool translating = false; + unsigned int sid, idx; + + for (sid = 0; sid < dart->num_streams; sid++) { + if (apple_dart_readl(dart, DART_TCR(dart, sid)) != dart->save_tcr[sid]) + return false; + + for (idx = 0; idx < dart->hw->ttbr_count; idx++) { + u32 ttbr = apple_dart_readl(dart, DART_TTBR(dart, sid, idx)); + + if (ttbr != dart->save_ttbr[sid][idx]) + return false; + if (ttbr & dart->hw->ttbr_valid) + translating = true; + } + } + + /* A DART that lost power reads back as reset, without valid page tables */ + return translating; +} + +/* + * Flush the TLBs of every stream with a command of its own. A stream map flush + * is not enough for the whole DART: where the streams of a map are remapped + * onto its first stream, it only reaches that first stream. + */ +static int apple_dart_invalidate_each_stream(struct apple_dart *dart) +{ + unsigned int sid; + int ret; + + for (sid = 0; sid < dart->num_streams; sid++) { + struct apple_dart_stream_map stream_map = { .dart = dart }; + + __set_bit(sid, stream_map.sidmap); + ret = dart->hw->invalidate_tlb(&stream_map); + if (ret) + return ret; + } + + return 0; +} + static __maybe_unused int apple_dart_resume(struct device *dev) { struct apple_dart *dart = dev_get_drvdata(dev); @@ -3201,6 +3253,10 @@ static __maybe_unused int apple_dart_resume(struct device *dev) if (dart->locked) return 0; + /* Nothing was lost, only drop what the TLBs may still hold. */ + if (apple_dart_state_retained(dart)) + return apple_dart_invalidate_each_stream(dart); + ret = apple_dart_hw_reset(dart); if (ret) { dev_err(dev, "Failed to reset DART on resume\n"); From f70b808e64ab70bf1252b2d7547eef55864d270a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 20:09:20 +1100 Subject: [PATCH 1465/2680] drm/apple: Fail an external mode set that gets no reply The completion of set_digital_out_mode ignored its reply, so a mode set that the transport completed without one counted as a success. On the external H17P display, fail such a mode set with -EIO. The 32-bit result itself is kept and logged when it is nonzero, but not acted on: its meaning is not established for H17P. The firmware's own log shows a nonzero result for a mode set that failed after the sink went away, and on other firmware within 0.1 ms for internal-panel requests that repeat the previous one or come first after boot. A link that does not come up is reported through the DPTX fault callbacks instead. (cherry picked from commit f2e5e5330dee2f64e13d3ebbdfbf44fbf48e2d6e) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../gpu/drm/apple/dcp_external_relink_test.c | 37 ++++++++++++++++++- drivers/gpu/drm/apple/iomfb_template.c | 22 +++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp_external_relink_test.c b/drivers/gpu/drm/apple/dcp_external_relink_test.c index 04a405a8e813d..dc80ca636c753 100644 --- a/drivers/gpu/drm/apple/dcp_external_relink_test.c +++ b/drivers/gpu/drm/apple/dcp_external_relink_test.c @@ -57,6 +57,7 @@ struct relink_test { bool real_modeset; /* atomic_enable is the real dcp_crtc_atomic_modeset() */ bool force_modeset; /* a userspace commit changes the connectors */ bool fault_in_call; /* the link faults before the firmware answers */ + bool no_reply; /* the firmware answers a call with no reply */ u32 reply; /* what the firmware answers a call with */ int modeset_ret; /* the result of the real mode set */ struct dcp_display_mode catalog; @@ -135,7 +136,7 @@ static void relink_test_push(struct apple_dcp *dcp, const struct dcp_method_entr if (modeset && t->fault_in_call) dcp_external_link_fault(dcp); - cb(dcp, &t->reply, cookie); + cb(dcp, modeset && t->no_reply ? NULL : &t->reply, cookie); } static void relink_test_crtc_enable(struct drm_crtc *crtc, struct drm_atomic_commit *state) @@ -886,6 +887,38 @@ static void relink_fault_inactive_test(struct kunit *test) enable_work(&t->con.hotplug_wq); } +/* A mode set the firmware gives no reply to has failed. */ +static void relink_no_reply_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + relink_test_real(t); + t->no_reply = true; + relink_test_tear(t); + relink_test_reconnect(test, t); + KUNIT_EXPECT_LT(test, t->modeset_ret, 0); + KUNIT_EXPECT_FALSE(test, relink_test_link_up(t)); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_BAD); +} + +/* + * A nonzero result alone is not a failure: its meaning is not established, + * and it is also seen for a mode set that repeats the current mode. + */ +static void relink_nonzero_reply_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + relink_test_real(t); + t->reply = 1; + relink_test_tear(t); + relink_test_reconnect(test, t); + KUNIT_EXPECT_EQ(test, t->modeset_ret, 0); + KUNIT_EXPECT_TRUE(test, relink_test_link_up(t)); + KUNIT_EXPECT_TRUE(test, relink_test_connected(t)); +} + static struct kunit_case relink_cases[] = { KUNIT_CASE(relink_bounce_test), KUNIT_CASE(relink_mode_gone_test), @@ -904,6 +937,8 @@ static struct kunit_case relink_cases[] = { KUNIT_CASE(relink_fault_during_modeset_test), KUNIT_CASE(relink_fault_after_modeset_test), KUNIT_CASE(relink_fault_inactive_test), + KUNIT_CASE(relink_no_reply_test), + KUNIT_CASE(relink_nonzero_reply_test), {} }; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 76c13c4f77ae2..b403a6c61e2e9 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -44,6 +44,7 @@ struct dcp_wait_cookie { struct kref refcount; struct completion done; u32 status; + bool replied; }; static void release_wait_cookie(struct kref *ref) @@ -2235,6 +2236,10 @@ static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, struct dcp_wait_cookie *wait = cookie; if (wait) { + if (data) { + WRITE_ONCE(wait->status, *(u32 *)data); + WRITE_ONCE(wait->replied, true); + } complete(&wait->done); kref_put(&wait->refcount, release_wait_cookie); } @@ -2363,6 +2368,23 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, "set_digital_out_mode finished with %d to spare\n", jiffies_to_msecs(ret)); } + /* + * The external H17P display reports a link that does not come up + * through its DPTX fault callbacks. What the 32-bit result of the + * mode set means is not established: a nonzero one is seen both for + * a mode set that failed and for one repeating the current mode. Log + * it, and fail only a mode set that got no reply at all. + */ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { + if (!READ_ONCE(cookie->replied)) { + dev_err(dcp->dev, "set_digital_out_mode got no reply\n"); + kref_put(&cookie->refcount, release_wait_cookie); + return -EIO; + } + if (READ_ONCE(cookie->status)) + dev_info(dcp->dev, "set_digital_out_mode returned %#x\n", + READ_ONCE(cookie->status)); + } kref_put(&cookie->refcount, release_wait_cookie); dcp->vrr_enabled = mode->vrr && crtc_state->vrr_enabled; From ed58152ad76cf3e7852e1a80c1c437a882a75f3e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 04:31:43 +1100 Subject: [PATCH 1466/2680] usb: typec: tipd: Apply SAFE after ordinary USB role shutdown Ordinary Apple USB ports must stop DWC3 before the mux changes the shared PHY lanes. Early SAFE during removal or probe rollback instead reaches ATC with a live PIPE. Prepare display shutdown first and hold the SAFE transaction across role NONE, applying it after successful role shutdown. Let mux providers declare this ordering requirement and use one predicate for disconnect and removal. ATC declares it only for ordinary ports, so a fixed host still applies SAFE while its backend clock is available, even if its firmware graph exposes a no-op role switch. Other providers retain the existing early order. This keeps the fixed-host divergence from Asahi because its shared PIPE must be parked before the host clock disappears. An unregistered role switch returns -EOPNOTSUPP after controller teardown; accept that shutdown result only when it also reports role NONE. Keep unsupported requests on a live role as errors. On other errors abort without applying SAFE and retain recovery ownership. Reject an ordinary-port SAFE against a live PIPE with -EBUSY so the next probe cannot warn or change live lanes. The removal transaction remains open until generic disconnect finishes; the prepared flag prevents an early second application. Cover the ordering, fixed-host no-op switch, absent switch, errors and re-probe in host tests, and the live-PIPE guard in KUnit without MMIO. Fixes: 6f004361c61b ("usb: typec: serialize display teardown and preserve recovery ownership") [ryan: split by topic: only the ATC PHY, Type-C mux and tipd part; the apple-dcp selftests part is in 200-dcp] (cherry picked from commit 4b36583930a5bef1e5bcf0942ea833ac55c4d748) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-test.c | 39 ++++++++++++++++++++++++++++++ drivers/phy/apple/atc.c | 11 +++++++++ drivers/usb/typec/mux.c | 18 ++++++++++++++ drivers/usb/typec/mux.h | 1 + drivers/usb/typec/tipd/core.c | 40 +++++++++++++++++++++++-------- drivers/usb/typec/tipd/tps6598x.h | 4 ++-- include/linux/usb/typec_mux.h | 8 +++++++ 7 files changed, 109 insertions(+), 12 deletions(-) diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index a8c0c57d09859..d04d8031b9c09 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -182,6 +182,44 @@ static void atcphy_test_update_takes_usb(struct kunit *test) KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), -ENODEV); } +struct atcphy_test_configure { + unsigned int calls; + unsigned int lane_requests; +}; + +/* Count lane requests instead of touching MMIO, including stale USB3 requests. */ +static int atcphy_test_configure_stub(struct apple_atcphy *atcphy, enum atcphy_mode mode) +{ + struct kunit *test = kunit_get_current_test(); + struct atcphy_test_configure *config = test->priv; + + lockdep_assert_held(&atcphy->lock); + config->calls++; + if (mode != APPLE_ATCPHY_MODE_OFF) { + config->lane_requests++; + return -EINVAL; + } + return 0; +} + +/* Rebinding after a failed role shutdown must not touch a live PIPE. */ +static void atcphy_test_safe_live_pipe(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_dev(test, false); + struct atcphy_test_configure config = {}; + + test->priv = &config; + kunit_activate_static_stub(test, atcphy_configure, atcphy_test_configure_stub); + atcphy->hw = &atcphy_hw_t8103; + atcphy->typec_mode = APPLE_ATCPHY_MODE_USB3; + mutex_init(&atcphy->lock); + KUNIT_EXPECT_EQ(test, atcphy_mux_set_locked(atcphy, APPLE_ATCPHY_MODE_OFF), -EBUSY); + KUNIT_EXPECT_EQ(test, config.calls, 0U); + KUNIT_EXPECT_EQ(test, atcphy->mode, APPLE_ATCPHY_MODE_USB3_DP); + KUNIT_EXPECT_EQ(test, atcphy->typec_mode, APPLE_ATCPHY_MODE_USB3); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_USB3); +} + static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_target_mode), KUNIT_CASE(atcphy_test_target_mode_fixed_hub), @@ -189,6 +227,7 @@ static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_noop_needs_work), KUNIT_CASE(atcphy_test_update_skips_usb), KUNIT_CASE(atcphy_test_update_takes_usb), + KUNIT_CASE(atcphy_test_safe_live_pipe), {} }; diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 59ff76e7aa9ed..5f70825049194 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -22,6 +22,8 @@ * Author: Sven Peter */ +#include + #include #include #include @@ -2859,6 +2861,9 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) int ret = 0; u32 reg; + if (IS_ENABLED(CONFIG_PHY_APPLE_ATC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(atcphy_configure, atcphy, mode); + lockdep_assert_held(&atcphy->lock); if (atcphy->pipe_pending) return -EBUSY; @@ -3742,6 +3747,11 @@ static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode t return 0; } + /* A re-probe after failed role shutdown must leave the live PIPE alone. */ + if (!atcphy->fixed_usb2 && target_mode == APPLE_ATCPHY_MODE_OFF && + atcphy->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY) + return -EBUSY; + switch (target_mode) { case APPLE_ATCPHY_MODE_TBT: case APPLE_ATCPHY_MODE_USB4: @@ -3849,6 +3859,7 @@ static int atcphy_probe_mux(struct apple_atcphy *atcphy) .drvdata = atcphy, .fwnode = atcphy->dev->fwnode, .set = atcphy_mux_set, + .safe_after_role = !atcphy->fixed_usb2, }; mux = typec_mux_register(atcphy->dev, &mux_desc); diff --git a/drivers/usb/typec/mux.c b/drivers/usb/typec/mux.c index 4170d238c01f7..549f68585cf42 100644 --- a/drivers/usb/typec/mux.c +++ b/drivers/usb/typec/mux.c @@ -317,6 +317,23 @@ static void typec_mux_context_put(struct typec_mux_context *context) kfree(context); } +/* Immutable provider policy; the handle pins each provider until put. */ +bool typec_mux_safe_after_role(struct typec_mux *mux) +{ + unsigned int i; + + if (IS_ERR_OR_NULL(mux)) + return false; + + for (i = 0; i < mux->num_mux_devs; i++) { + if (mux->mux_devs[i]->safe_after_role) + return true; + } + + return false; +} +EXPORT_SYMBOL_GPL(typec_mux_safe_after_role); + static int mux_fwnode_match(struct device *dev, const void *fwnode) { if (!is_typec_mux_dev(dev)) @@ -651,6 +668,7 @@ typec_mux_register(struct device *parent, const struct typec_mux_desc *desc) mux_dev->finish = desc->finish; mux_dev->commit = desc->commit; mux_dev->abort = desc->abort; + mux_dev->safe_after_role = desc->safe_after_role; device_initialize(&mux_dev->dev); mux_dev->dev.parent = parent; diff --git a/drivers/usb/typec/mux.h b/drivers/usb/typec/mux.h index 42e44211fd553..823c5529ca675 100644 --- a/drivers/usb/typec/mux.h +++ b/drivers/usb/typec/mux.h @@ -20,6 +20,7 @@ struct typec_mux_dev { typec_mux_notify_fn_t commit; typec_mux_notify_fn_t abort; bool removed; + bool safe_after_role; }; #define to_typec_switch_dev(_dev_) container_of(_dev_, struct typec_switch_dev, dev) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 6534ddb21f76b..98d3237f11458 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -228,7 +228,9 @@ static int tps6598x_set_data_role(struct tps6598x *tps, role_val = USB_ROLE_NONE; ret = usb_role_switch_set_role(tps->role_sw, role_val); - if (ret) + /* An unregistered role switch has already stopped its controller. */ + if (ret && (connected || ret != -EOPNOTSUPP || + usb_role_switch_get_role(tps->role_sw) != USB_ROLE_NONE)) return ret; typec_set_data_role(tps->port, role); return 0; @@ -289,6 +291,12 @@ static void tps6598x_drop_partner(struct tps6598x *tps, u32 status) typec_set_vconn_role(tps->port, TPS_STATUS_TO_TYPEC_VCONN(status)); } +/* The mux provider knows whether this is a fixed host or a switched PHY. */ +static bool tps6598x_safe_after_role(struct tps6598x *tps) +{ + return tps->role_sw && typec_mux_safe_after_role(tps->mux); +} + static int tps6598x_disconnect(struct tps6598x *tps, u32 status) { struct typec_mux_state safe = { .mode = TYPEC_STATE_SAFE }; @@ -302,13 +310,16 @@ static int tps6598x_disconnect(struct tps6598x *tps, u32 status) ret = typec_mux_begin(tps->mux, &safe); if (ret) return ret; - /* Park while the host still supplies the PIPE backend clock. */ - ret = typec_mux_apply(tps->mux); - if (ret) - return ret; + /* Leave shared PHY shutdown to its USB role consumer. */ + if (!tps6598x_safe_after_role(tps)) { + ret = typec_mux_apply(tps->mux); + if (ret) + return ret; + } } ret = tps6598x_set_data_role(tps, TPS_STATUS_TO_TYPEC_DATAROLE(status), false); if (ret) { + /* Keep recovery ownership; a live PHY must not receive SAFE. */ typec_mux_abort(tps->mux); return ret; } @@ -2076,12 +2087,21 @@ static int cd321x_remove(struct tps6598x *tps) ret = typec_mux_begin(tps->mux, &safe); if (ret) return ret; - /* Park while the host still supplies the PIPE backend clock. */ - ret = typec_mux_apply(tps->mux); - if (ret) - return ret; + /* + * Ordinary PHYs need role shutdown before SAFE. A fixed host must park + * now while its controller supplies the PIPE backend clock, even when + * the firmware graph exposes a no-op role switch for that host. + */ + if (!tps6598x_safe_after_role(tps)) { + ret = typec_mux_apply(tps->mux); + if (ret) + return ret; + } ret = usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); - if (ret) { + /* An unregistered switch means the controller is already stopped. */ + if (ret && (ret != -EOPNOTSUPP || + usb_role_switch_get_role(tps->role_sw) != USB_ROLE_NONE)) { + /* Keep recovery ownership; a live PHY must not receive SAFE. */ typec_mux_abort(tps->mux); return ret; } diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index bca532b9019f0..ac1713d151c3c 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -339,8 +339,8 @@ struct tps6598x { struct typec_capability typec_cap; struct typec_mux *mux; /* - * Set by ->remove() once it has parked the mux in SAFE, so that - * tps6598x_disconnect() only ends that mux transaction. + * Set by ->remove() while its SAFE transaction is still open, so that + * tps6598x_disconnect() ends it after shutting down the USB role. */ bool remove_safe_prepared; diff --git a/include/linux/usb/typec_mux.h b/include/linux/usb/typec_mux.h index a18e4e0141adf..185da86c43d25 100644 --- a/include/linux/usb/typec_mux.h +++ b/include/linux/usb/typec_mux.h @@ -103,6 +103,8 @@ struct typec_mux_desc { typec_mux_set_fn_t finish; typec_mux_notify_fn_t commit; typec_mux_notify_fn_t abort; + /* SAFE needs the USB role consumer to stop the shared PHY first. */ + bool safe_after_role; }; #if IS_ENABLED(CONFIG_TYPEC) @@ -114,6 +116,7 @@ void typec_mux_put(struct typec_mux *mux); int typec_mux_begin(struct typec_mux *mux, struct typec_mux_state *state); /* Apply the providers while retaining the transaction and ownership. */ int typec_mux_apply(struct typec_mux *mux); +bool typec_mux_safe_after_role(struct typec_mux *mux); int typec_mux_end(struct typec_mux *mux); void typec_mux_abort(struct typec_mux *mux); int typec_mux_prepare(struct typec_mux *mux, struct typec_mux_state *state); @@ -142,6 +145,11 @@ static inline int typec_mux_begin(struct typec_mux *mux, struct typec_mux_state static inline int typec_mux_apply(struct typec_mux *mux) { return 0; } +static inline bool typec_mux_safe_after_role(struct typec_mux *mux) +{ + return false; +} + static inline int typec_mux_end(struct typec_mux *mux) { return 0; } static inline void typec_mux_abort(struct typec_mux *mux) {} From 84e3024f82cdc45f69cd1fb4bbb19c1f71a3cee0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:11:32 +1100 Subject: [PATCH 1467/2680] iommu/apple-dart: Keep a retained DART in bypass from being reset The resume path only treats a DART as having kept its state when a stream has a valid page table. A stream of an identity domain bypasses the DART without one, for example with iommu.passthrough=1 or where the default domain type is identity. Such a DART reads back exactly as it was saved, its consumer may still be doing DMA, and it was reset all the same, which disables DMA on every stream before the saved state is restored. Count a stream whose TCR still holds the bypass setting of its generation as live as well, on DARTs that support bypass. A DART that lost power reads back with neither, so it still takes the reset path. This completes "iommu/apple-dart: Do not reset a DART that kept its state across sleep". Test it for a T8103 and a T8110 DART; both tests fail without this change. (cherry picked from commit 39bc76211b30ca23b29eb740ffdf0ae8cf2ed5d3) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart-test.c | 36 +++++++++++++++++++++++++++++++++ drivers/iommu/apple-dart.c | 18 ++++++++++++----- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index fcde0a6a6ff6f..1ea1a7fb0fb10 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -671,6 +671,40 @@ static void apple_dart_test_resume_retained_each_stream(struct kunit *test) dart->hw->ttbr_valid | 0x2000); } +/* + * A stream of an identity domain bypasses the DART without a page table. Its + * consumer may still be doing DMA, so a DART that kept that state is not reset. + */ +static void apple_dart_test_resume_retained_bypass_hw(struct kunit *test, + const struct apple_dart_hw *hw) +{ + struct apple_dart *dart = apple_dart_test_tunnel(test); + u32 enable; + + dart->tunneled = false; + dart->hw = hw; + dart->num_streams = 16; + dart->supports_bypass = true; + enable = dart->hw->enable_streams; + writel(dart->hw->tcr_bypass, dart->regs + DART_TCR(dart, 3)); + KUNIT_ASSERT_EQ(test, apple_dart_suspend(dart->dev), 0); + + writel(0, dart->regs + enable); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); + KUNIT_EXPECT_EQ(test, readl(dart->regs + enable), 0U); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 3)), dart->hw->tcr_bypass); +} + +static void apple_dart_test_resume_retained_bypass_t8103(struct kunit *test) +{ + apple_dart_test_resume_retained_bypass_hw(test, &apple_dart_hw_t8103); +} + +static void apple_dart_test_resume_retained_bypass_t8110(struct kunit *test) +{ + apple_dart_test_resume_retained_bypass_hw(test, &apple_dart_hw_t8110); +} + static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_independent_remap_roots), KUNIT_CASE(apple_dart_test_before_attach), @@ -689,6 +723,8 @@ static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_resume_lost), KUNIT_CASE(apple_dart_test_resume_unused), KUNIT_CASE(apple_dart_test_resume_retained_each_stream), + KUNIT_CASE(apple_dart_test_resume_retained_bypass_t8103), + KUNIT_CASE(apple_dart_test_resume_retained_bypass_t8110), {} }; diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index a89394c20ca06..2ce9dfc6a0663 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -3186,12 +3186,17 @@ static __maybe_unused int apple_dart_suspend(struct device *dev) */ static bool apple_dart_state_retained(struct apple_dart *dart) { - bool translating = false; + bool live = false; unsigned int sid, idx; for (sid = 0; sid < dart->num_streams; sid++) { - if (apple_dart_readl(dart, DART_TCR(dart, sid)) != dart->save_tcr[sid]) + u32 tcr = apple_dart_readl(dart, DART_TCR(dart, sid)); + + if (tcr != dart->save_tcr[sid]) return false; + /* An identity domain bypasses the DART without any page table. */ + if (dart->supports_bypass && dart->hw->tcr_bypass && tcr == dart->hw->tcr_bypass) + live = true; for (idx = 0; idx < dart->hw->ttbr_count; idx++) { u32 ttbr = apple_dart_readl(dart, DART_TTBR(dart, sid, idx)); @@ -3199,12 +3204,15 @@ static bool apple_dart_state_retained(struct apple_dart *dart) if (ttbr != dart->save_ttbr[sid][idx]) return false; if (ttbr & dart->hw->ttbr_valid) - translating = true; + live = true; } } - /* A DART that lost power reads back as reset, without valid page tables */ - return translating; + /* + * A DART that lost power reads back as reset: no stream bypasses it and + * none has a valid page table. + */ + return live; } /* From de6d8ff6bc58c9a3cea50d6e605a82213ab627c8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 12:29:57 +1100 Subject: [PATCH 1468/2680] selftests: apple-dcp: Refresh extraction for current host regressions The host runner fails 16 of 55 cases on the current drivers because its extraction fixtures still expect the old ATC, CD321x, DPTX and RTKit boundaries. Share a lane-only adapter around the actual Type-C mode mapper and locked mux body. Extract orientation and the complete PM update path separately, model scoped locking, and disable KUnit redirects in host fixtures. Check that clean same-mode updates bypass PM while dirty or pending work still needs ownership. Extract the optional-router lookup for CD321x probe, model its nullable boundary, and check both deferred probing and router fallback. Refresh the DPTX callback name and catalogue/announcement state, and follow RTKit's debug notification handling for opaque OSLOG types 2 through 5. Retain the strict compiler and sanitizer settings and all 55 runner cases. The full host run improves from 39 pass / 16 fail to 55 pass / 0 fail. No production driver changes are needed. (cherry picked from commit 1dbd5f2b44d3fe9bf0430e5a3a66429506bd5fbe) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/gpu/apple-dcp/README | 12 ++++- .../selftests/drivers/gpu/apple-dcp/boot.py | 21 ++++---- .../selftests/drivers/gpu/apple-dcp/common.py | 16 ++++++ .../drivers/gpu/apple-dcp/dwc3/prefix.c | 3 ++ .../drivers/gpu/apple-dcp/lifecycle.py | 6 +-- .../selftests/drivers/gpu/apple-dcp/phy.py | 22 +++++--- .../selftests/drivers/gpu/apple-dcp/pm.py | 51 +++++++++++++------ .../selftests/drivers/gpu/apple-dcp/rate.py | 37 +++++++++----- 8 files changed, 117 insertions(+), 51 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/README b/tools/testing/selftests/drivers/gpu/apple-dcp/README index 33373ea702c30..ad39902e5e259 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/README +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/README @@ -75,8 +75,18 @@ or admit a hardware trial. Token/sequence semantics remain a separate protocol fact to establish; the tests do not manufacture a firmware echo guarantee. The boot group executes the actual OSLOG dispatcher with synthetic fixed-width -words. Known opaque event types remain visible, unknown types retain warnings, +words. Opaque event types 2-5 use debug logging, unknown types retain warnings, and only the existing buffer-request path can allocate/ACK buffers. The EDID service-rebind group also distinguishes intentional optional EOPNOTSUPP from required admission failures and genuine optional-service errors, and proves refused services never OPEN or publish. No captured firmware values are used. + +The ATC lane/PIPE and DWC3 teardown fixtures use a shared host adapter around +both the extracted Type-C target-mode mapper and the extracted locked mux body. +They omit USB PM at that boundary. The PM fixtures replace the adapter with the +actual mux update and callback, including the scoped no-op check: a clean +same-mode request needs neither root-hub lookup nor PM, while dirty lanes, +pending PIPE work and a mismatched PIPE still require ownership. KUnit static +redirects are disabled in host fixtures so the production bodies execute. +CD321x probe uses the actual optional-router lookup with a modeled deferred +probe decision; both supplier waiting and optional-router fallback are checked. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py b/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py index bf95df845443d..1f6fc49e90a68 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py @@ -19,37 +19,36 @@ def oslog(root): #define APPLE_RTKIT_EP_OSLOG 8 struct buffer { unsigned sentinel; }; struct apple_rtkit { void *dev; struct buffer oslog_buffer; }; -static unsigned warnings, information, requests; +static unsigned warnings, notifications, requests; static u64 observed; -static u8 observed_type; #define dev_warn(dev,fmt,msg) do { (void)(dev); (void)(fmt); warnings++; observed=(msg); } while (0) -#define dev_info(dev,fmt,type,msg) do { (void)(dev); (void)(fmt); information++; observed_type=(type); observed=(msg); } while (0) +#define dev_dbg(dev,fmt,msg) do { (void)(dev); (void)(fmt); notifications++; observed=(msg); } while (0) static void apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, struct buffer *b, u8 ep, u64 msg) { assert(b==&rtk->oslog_buffer && ep==8); requests++; observed=msg; } ''' + mask + '\n' + block(source, 'static void apple_rtkit_oslog_rx(') + r''' -static void reset(void) { warnings=information=requests=0; observed=0; observed_type=0; } +static void reset(void) { warnings=notifications=requests=0; observed=0; } int main(void) { struct apple_rtkit rtk={.oslog_buffer={.sentinel=0x12345678}}; - for (u8 type=3; type<=5; type++) { + for (u8 type=2; type<=5; type++) { for (unsigned edge=0; edge<2; edge++) { u64 msg=((u64)type<<56) | (edge ? ((1ULL<<56)-1) : 0); reset(); apple_rtkit_oslog_rx(&rtk,msg); - assert(!warnings && information==1 && !requests); - assert(observed==msg && observed_type==type); + assert(!warnings && notifications==1 && !requests); + assert(observed==msg); assert(rtk.oslog_buffer.sentinel==0x12345678); } } puts("PASS recognized opaque types preserve both payload bounds without buffer/ACK operations"); - const u8 unknown[]={0,2,6,0xff}; + const u8 unknown[]={0,6,0xff}; for (unsigned i=0;im); } +#define CONFIG_PHY_APPLE_ATC_KUNIT_TEST 0 +#define IS_ENABLED(option) (option) +#define KUNIT_STATIC_STUB_REDIRECT(...) ((void)0) #define guard(kind) \ struct held held_guard __attribute__((cleanup(release))) = hold enum phy_mode { PHY_MODE_USB_HOST, PHY_MODE_USB_DEVICE }; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index db72bc324ee11..7756046d7c07a 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -3,7 +3,7 @@ """Actual disconnect authority, DWC3 teardown and firmware resource admission.""" from pathlib import Path import re -from common import block, source_defines +from common import block, atc_mux_without_pm, source_defines def disconnect(root): # Copyright 2026 Ryan Murray @@ -749,11 +749,11 @@ def fn(n, name): s = files[n] normalized = n == 'atc' and name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in s if normalized: - name = 'atcphy_mux_set_locked' + return atc_mux_without_pm(s) m = re.search(r'^(?:static [^\n]*\b|(?:int|void) )' + name + r'\(', s, re.M) assert m, (n, name) result = block(s, m.start()) - return result.replace('atcphy_mux_set_locked(', 'atcphy_mux_set(', 1) if normalized else result + return result decls = [] for name in ('atcphy_generation', 'atcphy_lane_mode', 'atcphy_pipehandler_state', 'atcphy_mode', 'atcphy_dp_link_rate'): diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py index fbb1951c0f0b1..e29f767e80a29 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py @@ -3,7 +3,7 @@ """Actual PHY parking, failure/retry, masked context and OFF controls.""" from pathlib import Path import re -from common import block, source_defines +from common import block, source_defines, atc_mux_without_pm def parking(root): @@ -13,8 +13,7 @@ def parking(root): def function(source, name): if name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in source: - marker = re.search(r'^static [^\n]+\batcphy_mux_set_locked\(', source, re.M).group() - return block(source, marker).replace('atcphy_mux_set_locked(', 'atcphy_mux_set(', 1) + return atc_mux_without_pm(source) return block(source, re.search(r'^static [^\n]+\b' + name + r'\(', source, re.M).group()) @@ -133,7 +132,7 @@ def function(source, name): parts.append(dp) for name in ('atcphy_configure_lanes', 'j700_aux_enable', 'j700_aux_disable', 'atcphy_enable_dp_aux', 'atcphy_disable_dp_aux', 'atcphy_power_off', - 'atcphy_power_on', 'atcphy_configure', 'atcphy_sw_set', 'atcphy_mux_set', + 'atcphy_power_on', 'atcphy_configure', 'atcphy_set_orientation', 'atcphy_sw_set', 'atcphy_mux_set', 'atcphy_usb3_set_mode', 'atcphy_usb3_power_off'): parts.append(function(atc, name)) production = '\n'.join(parts) @@ -163,6 +162,9 @@ def function(source, name): #define dev_err report #define lockdep_assert_held(...) ((void)0) #define WARN_ON_ONCE(v) assert(!(v)) +#define CONFIG_PHY_APPLE_ATC_KUNIT_TEST 0 +#define IS_ENABLED(option) (option) +#define KUNIT_STATIC_STUB_REDIRECT(...) ((void)0) #define guard(kind) (void) #define fallthrough __attribute__((fallthrough)) #define ndelay(n) ((void)(n)) @@ -1672,8 +1674,7 @@ def block(marker): def function(name): if name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in source: - marker = re.search(r'^static [^\n]+\batcphy_mux_set_locked\(', source, re.M).group() - return block(marker).replace('atcphy_mux_set_locked(', 'atcphy_mux_set(', 1) + return atc_mux_without_pm(source) return block(re.search(r'^static [^\n]+\b' + name + r'\(', source, re.M).group()) @@ -1773,6 +1774,9 @@ def function(name): #define dev_err(...) ((void)0) #define lockdep_assert_held(...) ((void)0) #define WARN_ON_ONCE(v) assert(!(v)) +#define CONFIG_PHY_APPLE_ATC_KUNIT_TEST 0 +#define IS_ENABLED(option) (option) +#define KUNIT_STATIC_STUB_REDIRECT(...) ((void)0) #define guard(kind) (void) #define fallthrough __attribute__((fallthrough)) #define usleep_range(...) ((void)0) @@ -2023,8 +2027,7 @@ def unwind(root): def function(source, name): if name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in source: - marker = re.search(r'^static [^\n]+\batcphy_mux_set_locked\(', source, re.M).group() - return block(source, marker).replace('atcphy_mux_set_locked(', 'atcphy_mux_set(', 1) + return atc_mux_without_pm(source) return block(source, re.search(r'^static [^\n]+\b' + name + r'\(', source, re.M).group()) @@ -2131,6 +2134,9 @@ def function(source, name): #define dev_err report #define lockdep_assert_held(...) ((void)0) #define WARN_ON_ONCE(v) assert(!(v)) +#define CONFIG_PHY_APPLE_ATC_KUNIT_TEST 0 +#define IS_ENABLED(option) (option) +#define KUNIT_STATIC_STUB_REDIRECT(...) ((void)0) #define guard(kind) (void) #define fallthrough __attribute__((fallthrough)) #define ndelay(n) ((void)(n)) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index 6125dbf312b8f..27b207fd7b8f8 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -10,7 +10,8 @@ def ownership(root, usb_unreachable=False): ns=phy.parking(root) s=ns['atc'];has_pm='static int atcphy_usb3_pm_get(' in s prod=ns['production'].replace('struct apple_atcphy {','struct apple_atcphy {\n struct device_node *np;',1) - if has_pm: prod=prod.replace('static int atcphy_mux_set(', 'static int atcphy_mux_set_locked(',1) + if has_pm: + prod = prod.replace(block(prod, 'static int atcphy_mux_set('), '') # Non-mux fixture entry points continue to exercise actual source. Only the new # wrapper owns PM; the fixture's original helpers call it through this declaration. prefix=ns['prefix']+'\nstruct device_node;\nstatic int atcphy_mux_set(struct typec_mux_dev *, struct typec_mux_state *);\n' @@ -35,8 +36,9 @@ def ownership(root, usb_unreachable=False): prefix=prefix.replace('#define guard(kind) (void)',r''' struct lock_guard { int *lock; }; static struct lock_guard lock_take(int *lock) { assert(!*lock); *lock=1; return (struct lock_guard){lock}; } -static void lock_release(struct lock_guard *g) { assert(*g->lock==1); *g->lock=0; } +static void lock_release(struct lock_guard *g) { if(g->lock) { assert(*g->lock==1); *g->lock=0; } } #define guard(kind) struct lock_guard __attribute__((cleanup(lock_release))) _guard = lock_take +#define scoped_guard(kind, ptr) for (struct lock_guard __attribute__((cleanup(lock_release))) _scope = lock_take(ptr); _scope.lock; lock_release(&_scope), _scope.lock=NULL) ''') # Basic USB/device APIs are modeled, but get/put/wrapper and all parking/ # restoration bodies are extracted verbatim from the chosen public source. @@ -240,13 +242,21 @@ def ownership(root, usb_unreachable=False): assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==-ETIMEDOUT&&p.mode_dirty&&p.pipe_pending);balanced(&p); bist_status_stuck=false;recover();assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==0);balanced(&p); } - /* Scope and clean same-mode preserve MMIO guards; fixed no-op briefly owns PM. */ + /* Clean same-mode updates bypass PM; dirty lanes or PIPE retry still need it. */ struct typec_altmode alt={.svid=USB_TYPEC_DP_SID}; struct typec_mux_dev mux={&p};struct typec_mux_state same={.mode=TYPEC_DP_STATE_D,.alt=&alt}; prepare(&p,false,true,false);before=writes;assert(atcphy_mux_set(&mux,&same)==0&&resumes==0&&writes==before);balanced(&p); prepare(&p,true,false,false);before=writes;assert(atcphy_mux_set(&mux,&same)==0&&resumes==0&&writes==before);balanced(&p); prepare(&p,true,true,false);before=writes; - assert(atcphy_mux_set(&mux,&same)==0&&writes==before&&resumes==1);balanced(&p); + root_missing=true;resume_error=-EIO; + assert(atcphy_mux_set(&mux,&same)==0&&writes==before&&resumes==0);balanced(&p); + for(unsigned int retry=0;retry<3;retry++){ + prepare(&p,true,true,false);before=writes;root_missing=true; + if(retry==0)p.mode_dirty=true; + if(retry==1)p.pipe_pending=true; + if(retry==2)p.pipe_state=ATCPHY_PIPEHANDLER_STATE_DUMMY; + assert(atcphy_mux_set(&mux,&same)==-EAGAIN&&writes==before&&resumes==0);balanced(&p); + } /* Binding/identity failure must not reach USB PM or PHY programming. */ for(unsigned int missing=0;missing<3;missing++){ prepare(&p,true,true,false);before=writes; @@ -302,7 +312,8 @@ def ownership(root, usb_unreachable=False): helpers+=ns['function'](s,'atcphy_usb3_link_get')+'\n'+ns['function'](s,'atcphy_usb3_link_put')+'\n' helpers+=ns['function'](s,'atcphy_usb3_pm_find')+'\n#endif\n' helpers+=ns['function'](s,'atcphy_usb3_pm_get')+'\n'+ns['function'](s,'atcphy_usb3_pm_put')+'\n' - # The fixture extractor adapts the locked body; extract the wrapper directly. + helpers+=ns['function'](s,'atcphy_mux_is_noop')+'\n'+ns['function'](s,'atcphy_mux_update')+'\n' + # Lane fixtures provide the target mapping and locked body; use the real callback here. marker=re.search(r'^static int atcphy_mux_set\(',s,re.M).group() helpers+=ns['block'](s,marker)+'\n' else: @@ -346,6 +357,8 @@ def function(text, name): #include #define EPROBE_DEFER 517 #define IS_ERR(p) ((intptr_t)(p)<0) +#define ERR_PTR(e) ((void *)(intptr_t)(e)) +#define dev_info(...) ((void)0) #define PTR_ERR(p) ((int)(intptr_t)(p)) #define IS_ERR_OR_NULL(p) (!(p)||IS_ERR(p)) #define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) @@ -356,7 +369,8 @@ def function(text, name): struct work { bool pending; }; struct fwnode_handle { int refs; }; struct typec_mux_state { void *alt,*data; unsigned long mode; }; -struct tps6598x { int lock; void *port; }; +struct device; +struct tps6598x { int lock; void *port; struct device *dev; }; struct notifier_block { int (*notifier_call)(struct notifier_block *,unsigned long,void *); }; ''' pm = source('drivers/usb/typec/tipd/cd321x-pm.h').replace('#include ', '') @@ -375,7 +389,9 @@ def function(text, name): static struct fwnode_handle connector; static int mux_error, notifier_error, switch_error; static int ports,altmodes,muxes,switches,notifiers,stop_calls; -static bool displayport=true; +struct typec_thunderbolt_switch; +static bool keep_waiting=true; +static bool cd321x_tbt_switch_keep_waiting(struct device *d) { (void)d;return keep_waiting; } static int tps6598x_register_port(struct tps6598x *t,struct fwnode_handle *f) { (void)f;t->port=t;ports++;return 0; } @@ -383,7 +399,6 @@ def function(text, name): c->port_altmode_dp=c;c->port_altmode_tbt=c;altmodes+=2;return 0; } static void *fwnode_typec_mux_get(struct fwnode_handle *f) { (void)f;muxes++;return &muxes; } -static bool fwnode_property_present(struct fwnode_handle *f,const char *s) { (void)f;(void)s;return displayport; } static struct fwnode_handle *fwnode_find_reference(struct fwnode_handle *f,const char *s,int n) { (void)f;(void)s;(void)n;connector.refs++;return &connector; } @@ -396,23 +411,25 @@ def function(text, name): static int typec_set_mode(void *p,unsigned long m) { (void)p;assert(m==TYPEC_STATE_SAFE);return mux_error; } static int cd321x_tbt_notify(struct notifier_block *n,unsigned long e,void *d) { (void)n;(void)e;(void)d;return 0; } static int typec_thunderbolt_switch_register_notifier(void *s,struct notifier_block *n) { - (void)s;(void)n;if(notifier_error)return notifier_error;notifiers++;return 0; + (void)n;if(!s)return 0;if(notifier_error)return notifier_error;notifiers++;return 0; } -static void typec_thunderbolt_switch_put(void *s) { assert(s&&switches>0);switches--; } +static void typec_thunderbolt_switch_put(void *s) { if(s){assert(switches>0);switches--;} } static void typec_mux_put(void *m) { assert(m&&muxes>0);muxes--; } static void typec_unregister_altmode(void *a) { if(a){assert(altmodes>0);altmodes--;} } static void typec_unregister_port(void *p) { assert(p&&ports>0);ports--; } static void cd321x_stop_work(struct cd321x *c) { - stop_calls++;if(c->tbt_notifier_registered){notifiers--;c->tbt_notifier_registered=false;} + stop_calls++;if(c->tbt_notifier_registered){if(c->tbt_switch)notifiers--;c->tbt_notifier_registered=false;} cd321x_pm_remove(&c->pm);c->resume_work.pending=c->update_work.pending=false; } ''' core = source('drivers/usb/typec/tipd/core.c') - production = function(core, 'cd321x_register_port') + '\n' + function(core, 'cd321x_unregister_port') + switch_get = block(core, 'VISIBLE_IF_KUNIT struct typec_thunderbolt_switch *\ncd321x_tbt_switch_get(') + production = switch_get.replace('VISIBLE_IF_KUNIT', 'static', 1) + '\n' + production += function(core, 'cd321x_register_port') + '\n' + function(core, 'cd321x_unregister_port') controls = r''' static void reset(struct cd321x *c) { memset(c,0,sizeof(*c));connector.refs=0; - mux_error=notifier_error=switch_error=0;stop_calls=0; + mux_error=notifier_error=switch_error=0;stop_calls=0;keep_waiting=true; assert(!ports&&!altmodes&&!muxes&&!switches&&!notifiers); } static void balanced(void) { @@ -422,6 +439,7 @@ def function(text, name): struct cd321x c; struct cd321x_pm_state budget; cd321x_pm_init(&budget);assert(cd321x_pm_ready(&budget)); + assert(cd321x_display_taken(-ENOSPC)&&!cd321x_display_taken(-EBUSY)&&!cd321x_display_taken(0)); for(unsigned int i=0;ilock);*g->lock=false; } +#define guard(kind) struct rate_guard __attribute__((cleanup(rate_unlock))) _guard = rate_lock static bool dcp_is_usb4_output(struct apple_dcp *d) { return d->usb4; } static void schedule_work(struct work_struct *w) { w->scheduled++; } static const struct edid *drm_edid_raw(const struct drm_edid *e) { return (const void *)e->data; } +static bool dcpavserv_edid_current(struct apple_dcp *d, u64 generation) { + assert(d->hpd_mutex);return d->typec_cable_connected&&generation==d->typec_generation&&!d->quiescing; +} static bool dcpavserv_edid_service_current(struct apple_dcp *d, u64 e) { assert(d->dcpavserv.lock); return e==d->dcpavserv.epoch && d->dcpavserv.enabled && d->dcpavserv.service; } @@ -423,8 +436,8 @@ def case(value, automatic=True, internal=False, external=True, admitted=True): static void qualify(struct apple_dcp *d, struct apple_epic_service *s) { struct dptxport_apcall_link_rate rate={.link_rate=0x1e}, rate_reply={}; struct dptxport_apcall_set_active_lane_count lanes={.lane_count=2}, lanes_reply={}; - assert(!dptxport_call_qualified_config(s,false,&rate,sizeof(rate),&rate_reply,sizeof(rate_reply))); - assert(!dptxport_call_qualified_config(s,true,&lanes,sizeof(lanes),&lanes_reply,sizeof(lanes_reply))); + assert(!dptxport_call_link_config(s,false,&rate,sizeof(rate),&rate_reply,sizeof(rate_reply))); + assert(!dptxport_call_link_config(s,true,&lanes,sizeof(lanes),&lanes_reply,sizeof(lanes_reply))); (void)d; } static bool current(struct apple_dcp *d,const struct dcp_display_mode *m) { @@ -496,9 +509,9 @@ def case(value, automatic=True, internal=False, external=True, admitted=True): DENY(m.sdr_rgb.depth,10);DENY(m.sdr_rgb.colorimetry,1);DENY(m.sdr_rgb.range,1);DENY(m.mode.clock,414091); struct dptxport_apcall_link_rate rate={.link_rate=0x1e}, response={}; configure_error=-EIO; - assert(dptxport_call_qualified_config(&s,false,&rate,sizeof(rate),&response,sizeof(response))==-EIO); + assert(dptxport_call_link_config(&s,false,&rate,sizeof(rate),&response,sizeof(response))==-EIO); assert(!current(&d,&m));configure_error=0;qualify(&d,&s); - assert(dptxport_call_qualified_config(&s,false,&rate,sizeof(rate)-1,&response,sizeof(response))==-EINVAL); + assert(dptxport_call_link_config(&s,false,&rate,sizeof(rate)-1,&response,sizeof(response))==-EINVAL); assert(!current(&d,&m));qualify(&d,&s); dcp_firmware_metadata_connection(&d,7,false);dcp_firmware_metadata_connection(&d,7,true); assert(!current(&d,&m));qualify(&d,&s);assert(!current(&d,&m)); @@ -543,7 +556,6 @@ def publication(root): };''') state = state.replace('struct { bool lock, enabled; void *service; u64 epoch; } dcpavserv;', 'struct { bool lock, enabled; void *service; u64 epoch; int enable_completion; } dcpavserv;') state = state.replace('u64 typec_generation; unsigned int nr_modes;', '''u64 typec_generation; unsigned int nr_modes; - struct dcp_display_mode *modes; struct apple_epic *dcpavservep; int fw_compat,width_mm,height_mm,fixed_connector_type; unsigned notch_height; struct { size_t length,received; void *data; bool contiguous,payload_complete; @@ -551,7 +563,7 @@ def publication(root): struct dcp_metadata_ticket metadata_ticket,mode_ticket; } chunks;''') - start = state.index('static bool dcpavserv_edid_service_current(') + start = state.index('static bool dcpavserv_edid_current(') state = state[:start] stubs = r''' @@ -604,7 +616,6 @@ def publication(root): } static const struct drm_edid *dcp_firmware_edid(struct apple_dcp *d,u64 *generation) { (void)d; (void)generation; return ERR_PTR(-ENODATA); } static bool dcp_firmware_metadata_current(struct apple_dcp *d,u64 generation) { (void)d; (void)generation; return false; } -static bool dcp_has_panel(struct apple_dcp *d) { return d->fixed_connector_type==DRM_CONNECTOR_eDP; } static struct drm_display_mode *drm_mode_duplicate(struct drm_device *dev,const struct drm_display_mode *m) { assert(dev->mode_config.mutex);struct drm_display_mode *copy=malloc(sizeof(*copy));assert(copy);*copy=*m; if(invalidate_during_duplicate) { invalidate_during_duplicate=false;dcp_180hz_invalidate(probing); } @@ -691,9 +702,9 @@ def publication(root): static void timings(void) { start_receipt();finish_receipt(); } static void callbacks(bool lanes) { if(lanes) { struct dptxport_apcall_set_active_lane_count a={.lane_count=2},b={}; - assert(!dptxport_call_qualified_config(&s,true,&a,sizeof(a),&b,sizeof(b))); } + assert(!dptxport_call_link_config(&s,true,&a,sizeof(a),&b,sizeof(b))); } else { struct dptxport_apcall_link_rate a={.link_rate=0x1e},b={}; - assert(!dptxport_call_qualified_config(&s,false,&a,sizeof(a),&b,sizeof(b))); } + assert(!dptxport_call_link_config(&s,false,&a,sizeof(a),&b,sizeof(b))); } } static void probe(bool expected) { drm.mode_config.mutex=true;unsigned before=published; @@ -753,7 +764,7 @@ def publication(root): reset();callbacks(false);callbacks(true);timings();admit(0);fail_update=true;probe(false);assert(!con.drm_edid);cleanup(); reset();callbacks(false);callbacks(true);timings();admit(0);transport_edid.data[10]^=1;checksum(transport_edid.data);probe(false);cleanup();transport_edid=raw; reset();callbacks(false);callbacks(true);timings();admit(0);configure_error=-EIO; - struct dptxport_apcall_link_rate req={.link_rate=0x1e},resp={};assert(dptxport_call_qualified_config(&s,false,&req,sizeof(req),&resp,sizeof(resp))==-EIO); + struct dptxport_apcall_link_rate req={.link_rate=0x1e},resp={};assert(dptxport_call_link_config(&s,false,&req,sizeof(req),&resp,sizeof(resp))==-EIO); configure_error=0;probe(false);cleanup(); puts("PASS: 24 callback/timing/AV arrival permutations and real get_modes/lookup publication; early configuration needs no second callback; CLOSE, physical reconnect, receipt epochs and publication races deny stale proof"); } From dbe232b2a70147fcb94a13c4acb8d511802712aa Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 04:31:43 +1100 Subject: [PATCH 1469/2680] phy: apple: atc: Remember mux requests that keep the lane mode USB role shutdown can power the PHY off while its saved Type-C request is still USB3. A subsequent SAFE request needs no lane work, but both mux fast paths return without saving it. A later USB2 PHY initialization can then restore the old USB3 request instead of respecting SAFE. Record every accepted request whose lanes already match, both before acquiring USB runtime ownership and in the locked recheck. Preserve the capability checks, PIPE retries and fixed-hub no-op bypass. Test SAFE after independent shutdown on both paths and call the real USB2 initializer with a KUnit configuration stub that counts lane requests. Extend the host role mock through the production PHY power-off, mux recheck and USB2 init bodies, leaving only MMIO operations mocked. The host cases now verify that role NONE powers the PHY OFF, SAFE requires no lane work, and USB2 init cannot restore stale USB3 after SAFE. Fixes: 48836ec5a5b6 ("phy: apple: atc: Keep USB2 up when the port sits behind a fixed hub") [ryan: split by topic: only the drivers/phy/apple part; the apple-dcp selftests part is in 200-dcp] (cherry picked from commit 2757f43e143ed2851d01cde331df9631e15d4329) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-test.c | 56 ++++++++++++++++++++++++++++++++++++ drivers/phy/apple/atc.c | 6 +++- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index d04d8031b9c09..d3a760a7d62fd 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -182,6 +182,39 @@ static void atcphy_test_update_takes_usb(struct kunit *test) KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), -ENODEV); } +/* USB shutdown can turn the lanes off before the SAFE mux request arrives. */ +static struct apple_atcphy *atcphy_test_alloc_powered_off(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_dev(test, false); + + atcphy->hw = &atcphy_hw_t8103; + atcphy->mode = APPLE_ATCPHY_MODE_OFF; + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + atcphy->typec_mode = APPLE_ATCPHY_MODE_USB3; + mutex_init(&atcphy->lock); + + return atcphy; +} + +static void atcphy_test_safe_after_poweroff(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_powered_off(test); + struct typec_mux_state safe = { .mode = TYPEC_STATE_SAFE }; + + KUNIT_EXPECT_EQ(test, atcphy_mux_update(atcphy, &safe), 0); + KUNIT_EXPECT_EQ(test, atcphy->typec_mode, APPLE_ATCPHY_MODE_OFF); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); +} + +/* The locked recheck must also record SAFE if shutdown raced the mode lookup. */ +static void atcphy_test_locked_safe_after_poweroff(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_powered_off(test); + + KUNIT_EXPECT_EQ(test, atcphy_mux_set_locked(atcphy, APPLE_ATCPHY_MODE_OFF), 0); + KUNIT_EXPECT_EQ(test, atcphy->typec_mode, APPLE_ATCPHY_MODE_OFF); +} + struct atcphy_test_configure { unsigned int calls; unsigned int lane_requests; @@ -220,6 +253,26 @@ static void atcphy_test_safe_live_pipe(struct kunit *test) KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_USB3); } +static void atcphy_test_usb2_init_after_safe(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_powered_off(test); + struct typec_mux_state safe = { .mode = TYPEC_STATE_SAFE }; + struct atcphy_test_configure config = {}; + struct phy *phy; + + phy = kunit_kzalloc(test, sizeof(*phy), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, phy); + phy_set_drvdata(phy, atcphy); + test->priv = &config; + kunit_activate_static_stub(test, atcphy_configure, atcphy_test_configure_stub); + + KUNIT_ASSERT_EQ(test, atcphy_mux_update(atcphy, &safe), 0); + KUNIT_EXPECT_EQ(test, atcphy_usb2_init(phy), 0); + KUNIT_EXPECT_EQ(test, config.calls, 1U); + KUNIT_EXPECT_EQ(test, config.lane_requests, 0U); + KUNIT_EXPECT_EQ(test, atcphy->mode, APPLE_ATCPHY_MODE_OFF); +} + static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_target_mode), KUNIT_CASE(atcphy_test_target_mode_fixed_hub), @@ -227,7 +280,10 @@ static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_noop_needs_work), KUNIT_CASE(atcphy_test_update_skips_usb), KUNIT_CASE(atcphy_test_update_takes_usb), + KUNIT_CASE(atcphy_test_safe_after_poweroff), + KUNIT_CASE(atcphy_test_locked_safe_after_poweroff), KUNIT_CASE(atcphy_test_safe_live_pipe), + KUNIT_CASE(atcphy_test_usb2_init_after_safe), {} }; diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 5f70825049194..e721533fa8eae 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -3733,6 +3733,7 @@ static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode t guard(mutex)(&atcphy->lock); if (atcphy->mode == target_mode && !atcphy->mode_dirty) { + atcphy->typec_mode = target_mode; /* Retry a failed PIPE setup without disturbing the configured lanes. */ if (atcphy->fixed_usb2 && atcphy->host_active && (atcphy->pipe_state != atcphy_modes[target_mode].pipehandler_state || @@ -3830,8 +3831,11 @@ static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux * includes a whole enumeration with its timeouts, and would * hold up a DisplayPort update that changes neither. */ - if (atcphy_mux_is_noop(atcphy, target_mode)) + if (atcphy_mux_is_noop(atcphy, target_mode)) { + /* Record matching requests for the next PHY init to restore. */ + atcphy->typec_mode = target_mode; return 0; + } } ret = atcphy_usb3_pm_get(atcphy, &pm); From bd5b7a87c248abe8eac32a8a38e771eae2c0441f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 08:55:11 +1100 Subject: [PATCH 1470/2680] iommu/apple-dart: Restore the translations when the reset on resume fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit apple_dart_resume() returns as soon as apple_dart_hw_reset() fails. By then the reset has disabled translation on every stream and cleared the page table pointers, so the DART is left without any translation and every DMA of its consumers faults until something resets it again. On the J700 this happened to the DART of the PCIe radio: "TLB command 0x1 for stream 0 timed out" on resume, after which the Bluetooth function could not restore its transport and Wi-Fi lost its DMA until a reboot. The reset can only fail in its last steps, after the translation registers are cleared: the unlock of a T8110 v2.2 or later DART that locks on errors, and the TLB flush. Move those steps, with the error acknowledgment and unmasking around the unlock, into apple_dart_hw_finish_reset(). It leaves the translation registers alone and reports the step that failed. On resume, write the saved page table pointers and translation controls back whatever the reset returned. They still describe the page tables in use, and this path is only taken when the DART did not keep its state, so its TLBs hold nothing from before the sleep. Then, if the reset failed, repeat all of its last steps once, not only the flush. When the unlock fails, the reset returns before it unmasks the error interrupt and flushes the TLBs, and a flush that completes later says nothing about either. If they fail again, resume returns the error. After a failed system resume the PM core leaves the DART runtime suspended, with the restored registers matching the saved ones, so its next runtime resume would take the path for a DART that kept its state and only flush the TLBs. Have that resume repeat the last steps of the reset instead. The tests cover an unlock that times out while the command engine is busy, followed by stream flushes that complete, through a KUnit stub of the unlock, and an unlock that never gets through. They size the fake register space for the T8110 layout, whose exception records start at 16 KiB, and put a guard page after it. [ryan: adapted to 7.2.10 — context only: apple_dart_hw_finish_reset() follows Asahi's T8020/T8110 remap helpers. The 7.2.10 reset sequence is otherwise unchanged, and resume keeps its tunnel, command-gating and power_retained checks ahead of the retained-state check] (cherry picked from commit d9fc214474191909db140a5ba565a014819bd6de) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart-test.c | 181 +++++++++++++++++++++++++++++++- drivers/iommu/apple-dart.c | 94 ++++++++++++----- 2 files changed, 246 insertions(+), 29 deletions(-) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 1ea1a7fb0fb10..49819ea998042 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-only /* Included by apple-dart.c to exercise domain and command lifecycles. */ #include +#include #include +#include static void apple_dart_test_before_attach(struct kunit *test) { @@ -320,16 +322,28 @@ static void apple_dart_test_resume_commands_failure(struct kunit *test) pm_runtime_disable(dart->dev); } +/* + * Large enough for the T8110 layout, whose exception records reach up to SID + * 255 below 0xc000. The page after it is vmalloc's guard, so an access beyond + * any layout faults instead of reaching other memory. + */ +#define APPLE_DART_TEST_REGS_SIZE SZ_64K + +KUNIT_DEFINE_ACTION_WRAPPER(apple_dart_test_vfree, vfree, const void *); + static struct apple_dart *apple_dart_test_tunnel(struct kunit *test) { struct apple_dart *dart; + void *regs; dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, dart); dart->dev = kunit_device_register(test, "apple-dart-tunnel-test"); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dart->dev); - dart->regs = (__force void __iomem *)kunit_kzalloc(test, SZ_16K, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, dart->regs); + regs = vzalloc(APPLE_DART_TEST_REGS_SIZE); + KUNIT_ASSERT_NOT_NULL(test, regs); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, apple_dart_test_vfree, regs), 0); + dart->regs = (__force void __iomem *)regs; dart->hw = &apple_dart_hw_t8103_usb4; dart->num_streams = dart->hw->max_sid_count; dart->tunneled = true; @@ -353,7 +367,7 @@ static void apple_dart_test_tunnel_restore(struct kunit *test) dart->regs + DART_TTBR(dart, sid, idx)); } KUNIT_ASSERT_EQ(test, apple_dart_save_tunnel_state(dart->dev), 0); - memset_io(dart->regs, 0, SZ_16K); + memset_io(dart->regs, 0, APPLE_DART_TEST_REGS_SIZE); KUNIT_ASSERT_EQ(test, apple_dart_restore_tunnel_state(dart->dev), 0); for (sid = 0; sid < dart->num_streams; sid++) { KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, sid)), @@ -618,7 +632,7 @@ static void apple_dart_test_resume_lost(struct kunit *test) struct apple_dart *dart = apple_dart_test_sleeping(test); /* Power was cut: reset and restore the saved translations. */ - memset_io(dart->regs, 0, SZ_16K); + memset_io(dart->regs, 0, APPLE_DART_TEST_REGS_SIZE); KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); KUNIT_EXPECT_EQ(test, readl(dart->regs + dart->hw->enable_streams), U32_MAX); KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 2)), 0x82U); @@ -705,6 +719,161 @@ static void apple_dart_test_resume_retained_bypass_t8110(struct kunit *test) apple_dart_test_resume_retained_bypass_hw(test, &apple_dart_hw_t8110); } +/* + * The PCIe DART of the J700 radio: two streams translating for a function + * that keeps running in suspend-to-idle. + */ +static struct apple_dart *apple_dart_test_radio(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_tunnel(test); + unsigned int sid; + + dart->tunneled = false; + dart->hw = &apple_dart_hw_t8110; + dart->num_streams = 16; + dart->version = 0x0202; + for (sid = 1; sid <= 2; sid++) { + writel(dart->hw->tcr_enabled, dart->regs + DART_TCR(dart, sid)); + writel(dart->hw->ttbr_valid | (0x1000 * sid), + dart->regs + DART_TTBR(dart, sid, 0)); + } + KUNIT_ASSERT_EQ(test, apple_dart_suspend(dart->dev), 0); + return dart; +} + +static void apple_dart_test_radio_flush_timeout(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_radio(test); + u32 busy = DART_T8110_TLB_CMD_BUSY | + FIELD_PREP(DART_T8110_TLB_CMD_OP, DART_T8110_TLB_CMD_OP_FLUSH_SID); + + /* A flush that never completes must not cost the streams their tables. */ + writel(busy, dart->regs + DART_T8110_TLB_CMD); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 1)), dart->hw->tcr_enabled); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TTBR(dart, 2, 0)), + dart->hw->ttbr_valid | 0x2000); + + /* The same after a power loss, where the DART is reset first. */ + memset_io(dart->regs, 0, APPLE_DART_TEST_REGS_SIZE); + writel(busy, dart->regs + DART_T8110_TLB_CMD); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 2)), dart->hw->tcr_enabled); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TTBR(dart, 1, 0)), + dart->hw->ttbr_valid | 0x1000); +} + +/* The radio DART after a power loss, set to lock on errors. */ +static struct apple_dart *apple_dart_test_radio_lost(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_radio(test); + + memset_io(dart->regs, 0, APPLE_DART_TEST_REGS_SIZE); + writel(DART_T8110_DIAG_LOCK_ON_ERR, dart->regs + DART_T8110_DIAG_LOCK); + /* The reset unmasks the error interrupt only after the unlock. */ + writel(0x55, dart->regs + DART_T8110_ERROR_MASK); + return dart; +} + +/* The last command flushed the last stream, and nothing is pending. */ +static void apple_dart_test_expect_flushed(struct kunit *test, struct apple_dart *dart) +{ + u32 cmd = readl(dart->regs + DART_T8110_TLB_CMD); + + KUNIT_EXPECT_FALSE(test, cmd & DART_T8110_TLB_CMD_BUSY); + KUNIT_EXPECT_EQ(test, FIELD_GET(DART_T8110_TLB_CMD_OP, cmd), + (u32)DART_T8110_TLB_CMD_OP_FLUSH_SID); + KUNIT_EXPECT_EQ(test, FIELD_GET(DART_T8110_TLB_CMD_STREAM, cmd), dart->num_streams - 1); +} + +static int apple_dart_test_unlocks; + +/* + * The command engine is still busy when the reset unlocks the DART, and goes + * idle just after the unlock gave up waiting for it. + */ +static int apple_dart_test_unlock_late(struct apple_dart *dart) +{ + struct kunit *test = kunit_get_current_test(); + bool first = !apple_dart_test_unlocks++; + int ret; + + if (first) + writel(DART_T8110_TLB_CMD_BUSY, dart->regs + DART_T8110_TLB_CMD); + kunit_deactivate_static_stub(test, apple_dart_t8110_v2_unlock); + ret = apple_dart_t8110_v2_unlock(dart); + kunit_activate_static_stub(test, apple_dart_t8110_v2_unlock, + apple_dart_test_unlock_late); + if (first) + writel(0, dart->regs + DART_T8110_TLB_CMD); + return ret; +} + +static void apple_dart_test_radio_unlock_timeout(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_radio_lost(test); + + apple_dart_test_unlocks = 0; + kunit_activate_static_stub(test, apple_dart_t8110_v2_unlock, + apple_dart_test_unlock_late); + + /* The stream flushes after the failed unlock complete... */ + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); + apple_dart_test_expect_flushed(test, dart); + /* ...but the reset only ends once the unlock and the error mask are done. */ + KUNIT_EXPECT_EQ(test, apple_dart_test_unlocks, 2); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_ERROR_MASK), 0U); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 1)), dart->hw->tcr_enabled); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TTBR(dart, 2, 0)), + dart->hw->ttbr_valid | 0x2000); +} + +static void apple_dart_test_radio_unlock_stuck(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_radio_lost(test); + + /* An unlock that never gets through fails resume, with the tables back. */ + writel(DART_T8110_TLB_CMD_BUSY, dart->regs + DART_T8110_TLB_CMD); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_ERROR_MASK), 0x55U); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 2)), dart->hw->tcr_enabled); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TTBR(dart, 1, 0)), + dart->hw->ttbr_valid | 0x1000); + + /* + * The next resume finds the tables as they were saved, and must still + * unlock the DART and unmask its errors, not only flush the TLBs. + */ + writel(0, dart->regs + DART_T8110_TLB_CMD); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); + apple_dart_test_expect_flushed(test, dart); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_ERROR_MASK), 0U); +} + +static int apple_dart_test_flush_calls; + +static int apple_dart_test_flush_once_late(struct apple_dart_stream_map *stream_map) +{ + return apple_dart_test_flush_calls++ ? 0 : -ETIMEDOUT; +} + +static void apple_dart_test_resume_reset_retry(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_sleeping(test); + struct apple_dart_hw hw = apple_dart_hw_t8103; + + /* The reset's flush times out once; the second flush completes. */ + hw.invalidate_tlb = apple_dart_test_flush_once_late; + dart->hw = &hw; + apple_dart_test_flush_calls = 0; + memset_io(dart->regs, 0, APPLE_DART_TEST_REGS_SIZE); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); + KUNIT_EXPECT_GT(test, apple_dart_test_flush_calls, 1); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 2)), 0x82U); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TTBR(dart, 2, 0)), + dart->hw->ttbr_valid | 0x1000); +} + static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_independent_remap_roots), KUNIT_CASE(apple_dart_test_before_attach), @@ -723,6 +892,10 @@ static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_resume_lost), KUNIT_CASE(apple_dart_test_resume_unused), KUNIT_CASE(apple_dart_test_resume_retained_each_stream), + KUNIT_CASE(apple_dart_test_radio_flush_timeout), + KUNIT_CASE(apple_dart_test_radio_unlock_timeout), + KUNIT_CASE(apple_dart_test_radio_unlock_stuck), + KUNIT_CASE(apple_dart_test_resume_reset_retry), KUNIT_CASE(apple_dart_test_resume_retained_bypass_t8103), KUNIT_CASE(apple_dart_test_resume_retained_bypass_t8110), {} diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 2ce9dfc6a0663..0e270f92754e3 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -11,6 +11,7 @@ * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd. */ +#include #include #include #include @@ -294,6 +295,8 @@ struct apple_dart { u32 tunnel_state_saved : 1; u32 tunnel_state_restored : 1; bool commands_gated; + /* Resume restored the translations, but the last steps of its reset failed. */ + bool reset_incomplete; struct apple_tunable *tunables; u16 version; @@ -1226,6 +1229,8 @@ static int apple_dart_t8110_v2_unlock(struct apple_dart *dart) { struct apple_dart_stream_map stream_map = { .dart = dart }; + KUNIT_STATIC_STUB_REDIRECT(apple_dart_t8110_v2_unlock, dart); + if (!(readl(dart->regs + DART_T8110_DIAG_LOCK) & DART_T8110_DIAG_LOCK_ON_ERR)) return 0; @@ -1299,6 +1304,41 @@ apple_dart_t8110_hw_enable_remap(struct apple_dart *dart, int sid, int remap_to) writel(tcr, dart->regs + DART_TCR(dart, sid)); } +/* + * The last steps of a reset, which leave the translation registers alone: + * acknowledge the errors latched so far, unlock a DART that locks on errors, + * unmask the error interrupt and flush the TLBs. + */ +static int apple_dart_hw_finish_reset(struct apple_dart *dart) +{ + struct apple_dart_stream_map stream_map = { .dart = dart }; + int ret; + + /* Newer revisions expose per-stream exception records. */ + if (dart->hw->type == DART_T8110 && dart->version >= 0x0202) { + apple_dart_drain_stale_exceptions(dart); + ret = apple_dart_t8110_v2_unlock(dart); + if (ret) { + dev_err(dart->dev, "failed to unlock after reset: %d\n", ret); + return ret; + } + } else { + /* Clear pending errors before the interrupt is unmasked. */ + apple_dart_writel(dart, apple_dart_readl(dart, dart->hw->error), + dart->hw->error); + } + + if (dart->hw->type == DART_T8110) + apple_dart_writel(dart, 0, DART_T8110_ERROR_MASK); + + bitmap_set(stream_map.sidmap, 0, dart->num_streams); + ret = dart->hw->invalidate_tlb(&stream_map); + if (ret) + dev_err(dart->dev, "failed to flush the TLBs after reset: %d\n", ret); + + return ret; +} + static int apple_dart_hw_reset(struct apple_dart *dart) { struct apple_dart_stream_map stream_map; @@ -1317,24 +1357,7 @@ static int apple_dart_hw_reset(struct apple_dart *dart) apple_dart_writel(dart, U32_MAX, dart->hw->enable_streams + 4 * i); - /* Newer revisions expose per-stream exception records. */ - if (dart->hw->type == DART_T8110 && dart->version >= 0x0202) { - int ret; - - apple_dart_drain_stale_exceptions(dart); - ret = apple_dart_t8110_v2_unlock(dart); - if (ret) - return ret; - } else { - /* Clear pending errors before the interrupt is unmasked. */ - apple_dart_writel(dart, apple_dart_readl(dart, dart->hw->error), - dart->hw->error); - } - - if (dart->hw->type == DART_T8110) - apple_dart_writel(dart, 0, DART_T8110_ERROR_MASK); - - return dart->hw->invalidate_tlb(&stream_map); + return apple_dart_hw_finish_reset(dart); } static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, @@ -3261,16 +3284,25 @@ static __maybe_unused int apple_dart_resume(struct device *dev) if (dart->locked) return 0; - /* Nothing was lost, only drop what the TLBs may still hold. */ - if (apple_dart_state_retained(dart)) - return apple_dart_invalidate_each_stream(dart); + if (apple_dart_state_retained(dart)) { + /* Nothing was lost, only drop what the TLBs may still hold. */ + if (!dart->reset_incomplete) + return apple_dart_invalidate_each_stream(dart); - ret = apple_dart_hw_reset(dart); - if (ret) { - dev_err(dev, "Failed to reset DART on resume\n"); + /* A failed resume restored the translations: finish its reset. */ + ret = apple_dart_hw_finish_reset(dart); + dart->reset_incomplete = ret; return ret; } + ret = apple_dart_hw_reset(dart); + + /* + * Restore the translations even if the reset failed. It can only fail + * in its last steps, after clearing them, and the saved registers still + * point to the page tables in use. Leaving them cleared would fault + * every DMA of the consumers until the next reset. + */ for (sid = 0; sid < dart->num_streams; sid++) { for (idx = 0; idx < dart->hw->ttbr_count; idx++) apple_dart_writel(dart, dart->save_ttbr[sid][idx], @@ -3279,7 +3311,19 @@ static __maybe_unused int apple_dart_resume(struct device *dev) DART_TCR(dart, sid)); } - return 0; + /* + * Then repeat all of those last steps once. A TLB flush that completes + * now does not show that the DART was unlocked and its error interrupt + * unmasked. + */ + if (ret) { + dev_warn(dev, "reset on resume failed (%d), retrying\n", ret); + ret = apple_dart_hw_finish_reset(dart); + } + + /* Should they fail again, the next resume repeats them. */ + dart->reset_incomplete = ret; + return ret; } static DEFINE_RUNTIME_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume, NULL); From c703d40182064ab208738fca48babba78b0dcc89 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 13:30:00 +1100 Subject: [PATCH 1471/2680] arm64: dts: apple: t602x: Let the old PMGR misc driver bind again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pmgr-misc nodes now come from the upstream series, whose T602x node lists only "apple,t6020-pmgr-misc". This tree still has the original driver, which matches only "apple,t6000-pmgr-misc". It no longer binds on M2 Pro/Max, so the fabric and the memory controller (DCS) stay at their active performance states through suspend-to-idle. The older T602x node listed "apple,t6000-pmgr-misc" as a fallback. Restore the fallback. Like the older node, this sets no "apple,*-min-ps" property, so the driver lowers both to its default sleep performance state, as it did before. [ryan: adapted to 7.2.10 — restore the fallback compatible dropped from the pmgr_misc node of commit 01a79e36ec5a ("arm64: dts: apple: Add pmgr-misc nodes to t60xx"), as set by 3ebb2e200c11. Superseded when the upstream v3 pmgr-misc driver, which matches "apple,t6020-pmgr-misc" itself, lands in 010-soc; drop this commit then.] Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t602x-die0.dtsi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t602x-die0.dtsi b/arch/arm64/boot/dts/apple/t602x-die0.dtsi index 6001004903632..140a175a47687 100644 --- a/arch/arm64/boot/dts/apple/t602x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-die0.dtsi @@ -84,7 +84,7 @@ }; pmgr_misc: power-management@28e20c000 { - compatible = "apple,t6020-pmgr-misc"; + compatible = "apple,t6020-pmgr-misc", "apple,t6000-pmgr-misc"; reg = <0x2 0x8e20c000 0 0x400>, <0x2 0x8e20c400 0 0x400>; reg-names = "fabric-ps", "dcs-ps"; From c562a804d11356740c86698d725e5f4d05466e0e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 14:00:00 +1100 Subject: [PATCH 1472/2680] selftests: apple-dcp: Follow the 7.2.10 driver boundaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On 7.2.10 the host runner fails 17 of its 55 cases, all of which pass on aurora-wip. The fixtures still extract the 7.1 driver boundaries; the drivers need no change. ATC (11 cases): commit 9073d508ef83 ("phy: apple: atc: Add t8122 thunderbolt support.") moved the setup after the tunables into per-SoC atcphy_hw callbacks, and atcphy_configure() now compares against the all-modes one for its RCAL wait. Extract the callbacks together with atcphy_configure(), and give every fixture profile the callback of the SoC it models. A new adverse-retry case checks all four real profiles: T8103 sets its overrides without RCAL, T8122 wakes and waits for the USB3 PIPE only, T8140/T8142 do both in every mode. Parser (3 cases): commit 14922c6beaf9 ("drm/apple: Rank preferred timing modes ahead of the score") marks dcp_mode_ranks_above() VISIBLE_IF_KUNIT. Define it as does without CONFIG_KUNIT, like the fixture's IS_ENABLED(). VRR: commit 5164f7e7ff8e ("drm: Rename struct drm_atomic_state to drm_atomic_commit"). Probe rollback: commit a2b95607926d ("usb: typec: tipd: don't send GAID when probe fails in APP mode") resets the controller after a failed probe only if probe loaded a patch. Probe a patched controller by default, so that the checks that SAFE, role and registration errors block the reset keep their meaning, and check that an unpatched one is not reset. Service rebind: commit 4db65629ef98 ("drm: apple: afk: validate the receive write pointer and bound a drain") drains RECV through afk_recv_drain(), bounded by the receive ring. Extract it with the afk.h ring helpers, give the endpoint a ring, and check that the drain never reaches its bound. The strict compiler and sanitizer settings are unchanged. The runner goes from 38 pass / 17 fail to 55 pass / 0 fail. [ryan: adapted to 7.2.10 — selftest-only counterpart of the 7.2 ATC callbacks, KUnit visibility, drm_atomic_commit, the tipd stable fix and the bounded AFK drain] Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/gpu/apple-dcp/common.py | 8 ++++ .../drivers/gpu/apple-dcp/dwc3/cases.c | 7 ++++ .../drivers/gpu/apple-dcp/lifecycle.py | 12 +++--- .../selftests/drivers/gpu/apple-dcp/phy.py | 42 +++++++++++++++++-- .../selftests/drivers/gpu/apple-dcp/rate.py | 3 ++ .../drivers/gpu/apple-dcp/recovery.py | 16 +++++-- .../selftests/drivers/gpu/apple-dcp/run.py | 3 +- .../drivers/gpu/apple-dcp/transaction.py | 11 +++-- 8 files changed, 86 insertions(+), 16 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py index db10625064000..e895a7ac203e4 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py @@ -45,6 +45,14 @@ def run_unit(name, unit, output, cases=((),), includes=(), extra=()): return hashlib.sha256(unit.encode()).hexdigest() +# Asahi 7.2 moved the generation-specific setup after the tunables into +# per-SoC atcphy_hw callbacks; atcphy_configure() calls them and compares +# against the all-modes one for the RCAL wait. +ATC_POST_TUNABLE = ('atcphy_configure_post_tunable_t8103', 'atcphy_wake_common_t8122', + 'atcphy_configure_post_tunable_t8122', + 'atcphy_configure_post_tunable_all_modes') + + def atc_mux_without_pm(source): """Lane/teardown fixtures omit USB PM, keeping mode mapping and lane work real. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/cases.c b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/cases.c index 3e92ba747494f..dafb4d6822bee 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/cases.c +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/cases.c @@ -54,6 +54,12 @@ struct fixture { struct phy p2, p3, extra; struct dwc3_apple apple; }; +/* The post-tunable callback of each modelled SoC: T8103, T8122, T8140 */ +static void (*const post_tunable[])(struct apple_atcphy *, enum atcphy_mode) = { + atcphy_configure_post_tunable_t8103, + atcphy_configure_post_tunable_t8122, + atcphy_configure_post_tunable_all_modes, +}; static void initial(struct fixture *f, unsigned int gen, bool fixed, bool pm) { memset(f, 0, sizeof(*f)); @@ -68,6 +74,7 @@ static void initial(struct fixture *f, unsigned int gen, bool fixed, bool pm) .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, }; + hardware.configure_post_tunable = post_tunable[gen]; f->atc = (struct apple_atcphy){ .hw = &hardware, .regs.core = (unsigned char *)core_regs, diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 7756046d7c07a..a76d3a5fc56d8 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -3,7 +3,7 @@ """Actual disconnect authority, DWC3 teardown and firmware resource admission.""" from pathlib import Path import re -from common import block, atc_mux_without_pm, source_defines +from common import block, atc_mux_without_pm, source_defines, ATC_POST_TUNABLE def disconnect(root): # Copyright 2026 Ryan Murray @@ -779,7 +779,8 @@ def fn(n, name): 'atcphy_configure_pipehandler', 'atcphy_park_pipehandler', 'atcphy_get_mode_config', 'atcphy_dp_configure', 'j700_field_update', 'j700_aux_disable', 'atcphy_disable_dp_aux', - 'atcphy_power_off', 'atcphy_power_on', 'atcphy_configure', 'atcphy_usb2_init', 'atcphy_usb3_power_off', + 'atcphy_power_off', 'atcphy_power_on', *ATC_POST_TUNABLE, 'atcphy_configure', + 'atcphy_usb2_init', 'atcphy_usb3_power_off', 'atcphy_usb3_set_mode', 'atcphy_usb2_power_off', '_atcphy_dwc3_reset_assert', 'atcphy_dwc3_reset_assert', 'atcphy_mux_set'), 'phy': ('phy_init', 'phy_exit', 'phy_power_on', 'phy_power_off'), @@ -1346,11 +1347,12 @@ def vrr(root): struct platform_device { struct apple_dcp *data; }; struct drm_crtc { struct platform_device *dcp; }; struct drm_crtc_state { bool vrr_enabled, modeset; }; -struct drm_atomic_state { struct drm_crtc_state crtc; }; +/* mainline 7.2 renamed struct drm_atomic_state to struct drm_atomic_commit */ +struct drm_atomic_commit { struct drm_crtc_state crtc; }; #define to_apple_crtc(x) (x) #define platform_get_drvdata(x) ((x)->data) static struct drm_crtc_state *drm_atomic_get_new_crtc_state( - struct drm_atomic_state *s, struct drm_crtc *c) + struct drm_atomic_commit *s, struct drm_crtc *c) { (void)c; return &s->crtc; } static bool drm_atomic_crtc_needs_modeset(struct drm_crtc_state *s) { return s->modeset; } @@ -1362,7 +1364,7 @@ def vrr(root): struct apple_dcp dcp = { .connector = &connector }; struct platform_device pdev = { .data = &dcp }; struct drm_crtc crtc = { .dcp = &pdev }; - struct drm_atomic_state state = {0}; + struct drm_atomic_commit state = {0}; unsigned int cases = 0; /* Check initial modesets, plane-only commits and link recovery states. */ for (unsigned int profile = 0; profile < 3; profile++) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py index e29f767e80a29..fc212a968c8d2 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py @@ -3,7 +3,7 @@ """Actual PHY parking, failure/retry, masked context and OFF controls.""" from pathlib import Path import re -from common import block, source_defines, atc_mux_without_pm +from common import block, source_defines, atc_mux_without_pm, ATC_POST_TUNABLE def parking(root): @@ -132,7 +132,8 @@ def function(source, name): parts.append(dp) for name in ('atcphy_configure_lanes', 'j700_aux_enable', 'j700_aux_disable', 'atcphy_enable_dp_aux', 'atcphy_disable_dp_aux', 'atcphy_power_off', - 'atcphy_power_on', 'atcphy_configure', 'atcphy_set_orientation', 'atcphy_sw_set', 'atcphy_mux_set', + 'atcphy_power_on', *ATC_POST_TUNABLE, 'atcphy_configure', 'atcphy_set_orientation', + 'atcphy_sw_set', 'atcphy_mux_set', 'atcphy_usb3_set_mode', 'atcphy_usb3_power_off'): parts.append(function(atc, name)) production = '\n'.join(parts) @@ -193,6 +194,7 @@ def function(source, name): suffix = r''' static struct atcphy_hw hw = { .has_t8140_dp = true, .gen = ATCPHY_GENERATION_T8122, + .configure_post_tunable = atcphy_configure_post_tunable_all_modes, .aciophy_lane_mode = ACIOPHY_LANE_MODE_T8122, .aciophy_crossbar = ACIOPHY_CROSSBAR_T8122, .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, @@ -536,6 +538,9 @@ def function(source, name): for (unsigned int generation = 0; generation < 4; generation++) { hw.gen = generation == 0 ? ATCPHY_GENERATION_T8103 : ATCPHY_GENERATION_T8122; + hw.configure_post_tunable = generation == 0 ? atcphy_configure_post_tunable_t8103 : + generation == 1 ? atcphy_configure_post_tunable_t8122 : + atcphy_configure_post_tunable_all_modes; hw.has_t8140_dp = generation == 2; hw.has_usb4 = !hw.has_t8140_dp; hw.aciophy_lane_mode = generation == 0 ? ACIOPHY_LANE_MODE_T8103 : @@ -1036,6 +1041,33 @@ def adverse(root): puts("PASS: all matched hardware profiles fixed/ordinary host/device success and duplicate cache controls"); } +static void post_tunable_profiles(void) +{ + /* + * Asahi 7.2 per-SoC setup after the tunables: T8103 sets its overrides in + * every mode and never waits for RCAL; T8122 wakes its common block and + * waits for RCAL for the USB3 PIPE only; T8140/T8142 do both in every + * mode, DisplayPort included, as on 7.1. + */ + const enum atcphy_mode modes[]={APPLE_ATCPHY_MODE_USB3,APPLE_ATCPHY_MODE_USB3_DP, + APPLE_ATCPHY_MODE_DP}; + for (unsigned int gen=0; gen<4; gen++) + for (unsigned int m=0; m=2 || (gen==1 && usb3); + assert(rcal==wake); + assert(!!(registers[AUS_COMMON_SHIM_BLK_BIAS_REG/4]&AUS_COMMON_SHIM_BLK_BIAS_REG_BGBIAS_OV)==wake); + assert(!!(registers[ACIOPHY_SLEEP_CTRL/4]&ACIOPHY_SLEEP_CTRL_TX_CLAMP_OV)==(gen==0)); + } + puts("PASS: actual per-SoC post-tunable callbacks and RCAL wait for every hardware profile"); +} + static void pending_gate(void) { for (unsigned int gen=0;gen<2;gen++) { @@ -1129,6 +1161,7 @@ def adverse(root): else if (!strcmp(argv[1],"init")) retry_callbacks(false); else if (!strcmp(argv[1],"mux")) retry_callbacks(true); else if (!strcmp(argv[1],"success")) successful_scope(); + else if (!strcmp(argv[1],"post-tunable")) post_tunable_profiles(); else assert(false); return 0; } @@ -2107,9 +2140,11 @@ def function(source, name): for name in ('core_read32', 'atcphy_get_mode_config', 'j700_field_update', 'j700_pma_idle'): parts.append(function(atc, name)) parts.append(dp) + # The fixture hw below is the T8140 profile: only its post-tunable callback. for name in ('atcphy_configure_lanes', 'j700_aux_enable', 'j700_aux_disable', 'atcphy_enable_dp_aux', 'atcphy_disable_dp_aux', 'atcphy_power_off', - 'atcphy_power_on', 'atcphy_configure', 'atcphy_mux_set'): + 'atcphy_power_on', 'atcphy_wake_common_t8122', + 'atcphy_configure_post_tunable_all_modes', 'atcphy_configure', 'atcphy_mux_set'): parts.append(function(atc, name)) production = '\n'.join(parts) prefix = r''' @@ -2156,6 +2191,7 @@ def function(source, name): suffix = r''' static struct atcphy_hw hw = { .has_t8140_dp = true, .gen = ATCPHY_GENERATION_T8122, + .configure_post_tunable = atcphy_configure_post_tunable_all_modes, .aciophy_lane_mode = 0x60, .aciophy_crossbar = 0x64, }; static void initial(struct apple_atcphy *p, bool fixed, bool swapped) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py index 339b225a7bf7b..06d82d45aae10 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py @@ -120,6 +120,9 @@ def wire(value, out): #undef static_assert #define static_assert(c) _Static_assert(c, #c) #define IS_ENABLED(x) 0 +/* without CONFIG_KUNIT, as IS_ENABLED() above */ +#define VISIBLE_IF_KUNIT static +#define EXPORT_SYMBOL_IF_KUNIT(symbol) #define U32_MAX UINT32_MAX #define U16_MAX UINT16_MAX #define S64_MAX INT64_MAX diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py b/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py index 310af408b0f96..6c1ef39f86bfc 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py @@ -293,8 +293,10 @@ def run(name,text,cases): #define RBEP_RECV 0x85 #define complete_all(p) complete(p) ''') + afh=(root/'drivers/gpu/drm/apple/afk.h').read_text() queued=queued.replace('struct apple_dcp_afkep { u8 *command_tags;bool lock,stopping;', - 'struct apple_dcp_afkep { u8 *command_tags;int started,stopped,txbfr,rxbfr;void *debugfs_entry;bool lock,stopping;') + 'typedef u32 __le32;\n'+extract(afh,'struct afk_ringbuffer {')+';\n' + 'struct apple_dcp_afkep { u8 *command_tags;int started,stopped;struct afk_ringbuffer txbfr,rxbfr;void *debugfs_entry;bool lock,stopping;') queued=queued.replace('bool external_physically_absent,hpd_mutex,quiescing,typec_cable_connected;', 'bool retain_dma,external_physically_absent,hpd_mutex,quiescing,typec_cable_connected;') queued=queued.replace('static void run_retries(void) { assert(!ep.lock);if(ep.service_work.pending) { ep.service_work.pending=false;afk_service_worker(&ep.service_work); } }', @@ -305,8 +307,9 @@ def run(name,text,cases): queued+=r''' struct afk_receive_message_work { struct apple_dcp_afkep *ep;u64 message;struct work_struct work; }; static void afk_getbuf(struct apple_dcp_afkep *e,u64 m) { (void)e;(void)m;assert(!"unexpected GETBUF"); } - static void afk_init_rxtx(struct apple_dcp_afkep *e,u64 m,int *r) { (void)e;(void)m;(void)r;assert(!"unexpected INIT"); } - static unsigned int rx_step;static bool close_pending; + static void afk_init_rxtx(struct apple_dcp_afkep *e,u64 m,struct afk_ringbuffer *r) { (void)e;(void)m;(void)r;assert(!"unexpected INIT"); } + static unsigned int rx_step,drain_warnings;static bool close_pending;static __le32 rx_rptr,rx_wptr; + #define dev_warn_ratelimited(...) (drain_warnings++) static bool afk_recv(struct apple_dcp_afkep *e) { assert(e==&ep);rx_running=true; if(rx_step==0) closed(13,0x6a01,0); @@ -318,11 +321,15 @@ def run(name,text,cases): } ''' af=(root/'drivers/gpu/drm/apple/afk.c').read_text() - for signature in ['static void afk_receive_message_worker(', 'int afk_receive_message(', 'void afk_quiesce(', 'static void afk_release_context(']: + for signature in ['static inline unsigned int afk_ring_drain_budget(', 'static inline unsigned int afk_ring_drain(']: + queued+=extract(afh,signature)+'\n' + for signature in ['static bool afk_recv_entry(', 'static void afk_recv_drain(', + 'static void afk_receive_message_worker(', 'int afk_receive_message(', 'void afk_quiesce(', 'static void afk_release_context(']: queued+=extract(af,signature)+'\n' queued+=r''' int main(int argc,char **argv) { assert(argc==2);setup();reuse_dma=true;INIT_WORK(&ep.service_work,afk_service_worker); + ep.rxbfr=(struct afk_ringbuffer){.ready=true,.rptr=&rx_rptr,.wptr=&rx_wptr,.bufsz=0x1000,.stride=0x40}; announce(13,name,native_class,13);struct apple_epic_service *old=d.dcpavserv.service;assert(old); assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack();run_retries(); unsigned int before=opens; @@ -350,6 +357,7 @@ def run(name,text,cases): afk_release_context(empty); cleanup(); } + assert(!drain_warnings); printf("PASS actual RX worker/quiesce: %s\n",argv[1]); } ''' diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py index 5c3b6968d1def..44fbdbab441b9 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -69,7 +69,8 @@ def main(): ('lifecycle', 'quiesce-before-safe', quiesce.make_unit, tuple((c,) for c in ('ordered','power','ack','role','bolt','mux','duplicate','error-latched','withdrawn-still-latched','legacy','fabric','tunnel','generic-disconnect','ordinary-remove','ordinary-disconnect','ordinary-role-error','ordinary-phy-error','ordinary-role-reprobe','ordinary-unsupported-remove','ordinary-unsupported-disconnect','ordinary-unregistered-remove','ordinary-unregistered-disconnect','noop-fixed-remove','noop-fixed-disconnect')), ()), ('lifecycle', 'probe-mux-pm-dispatch', pm.probe_dispatch, ((),), ()), ('phy', 'parking', lambda r: parking_unit(r), parking_cases, ()), - ('phy', 'adverse-retry', phy.adverse, tuple((c,) for c in ('bist', 'init', 'mux', 'gate', 'dp-cache', 'off', 'success')), ()), + ('phy', 'adverse-retry', phy.adverse, tuple((c,) for c in ('bist', 'init', 'mux', 'gate', 'dp-cache', 'off', 'success', + 'post-tunable')), ()), ('phy', 'masked-context', phy.usb_context, ((),), ()), ('phy', 'off-hardening', phy.hardening, ((),), ()), ('phy', 'warm-reset', phy.warm_reset, ((),), ()), diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py index 44666a4997c8a..77e3eb7782341 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py @@ -301,12 +301,13 @@ def probe(root): #define enable_irq_wake(...) ((void)0) #define dev_err_probe(...) (-EINVAL) struct fwnode_handle { int unused; }; -static int connect_error,irq_error,register_error,node_puts,mux_get_error; +static int connect_error,irq_error,register_error,node_puts,mux_get_error,controller_mode; static void *devm_gpiod_get_optional(void *d,const char *s,int v) { (void)d;(void)s;(void)v;return NULL; } static void msleep(unsigned ms) { (void)ms; } static bool device_is_compatible(void *d,const char *s) { (void)d;(void)s;return false; } static int tps6598x_read32(struct tps6598x *t,int reg,u32 *v) { (void)t;(void)reg;*v=1;return 0; } -static int tps6598x_check_mode(struct tps6598x *t) { (void)t;return 0; } +static int tps6598x_check_mode(struct tps6598x *t) { (void)t;return controller_mode; } +static int apply_patch(struct tps6598x *t) { (void)t;return 0; } static int tps6598x_write64(struct tps6598x *t,int reg,unsigned v) { (void)t;(void)reg;(void)v;return 0; } static bool tps6598x_read_status(struct tps6598x *t,u32 *s) { (void)t;*s=TPS_STATUS_PLUG_PRESENT;return true; } static bool tps6598x_read_power_status(struct tps6598x *t) { (void)t;return true; } @@ -331,6 +332,8 @@ def probe(root): static void probe_initial(struct cd321x *c) { initial(c);ordinary_port=true;phy_provider.safe_after_role=true;c->tps.irq=1;data.register_port=register_port;data.read_data_status=read_data_status;data.connect=connect; connect_error=irq_error=register_error=node_puts=mux_get_error=0; + /* tipd_init() resets only a controller it patched (stable a2b95607926d, mainline c5df31f10880) */ + controller_mode=TPS_MODE_PTCH;data.init=apply_patch; } int main(void) { struct cd321x c; @@ -340,11 +343,13 @@ def probe(root): assert(tipd_init(&c.tps)==-EAGAIN&&reset_calls==0&&gpio_calls==0&&phy_calls==0&&port.owner==&route&&port.safe_pending); probe_initial(&c);irq_error=-ENOSPC; assert(tipd_init(&c.tps)==-ENOSPC&&reset_calls==1&&!port.owner&&!port.safe_pending&&unregisters==1); + probe_initial(&c);irq_error=-ENOSPC;controller_mode=0; + assert(tipd_init(&c.tps)==-ENOSPC&&reset_calls==0&&!port.owner&&!port.safe_pending&&unregisters==1); probe_initial(&c);mux_get_error=-ENODEV; assert(tipd_init(&c.tps)==-ENODEV&&reset_calls==0&&stops==0&&phy_calls==0&&unregisters==0&&node_puts==1); probe_initial(&c);register_error=-EIO; assert(tipd_init(&c.tps)==-EIO&&reset_calls==0&&stops==0&&phy_calls==0&&unregisters==0); - puts("PASS actual probe rollback preserves primary errors and blocks reset after SAFE/role/registration errors"); + puts("PASS actual probe rollback preserves primary errors, blocks reset after SAFE/role/registration errors and resets only a patched controller"); } """ return unit From ea70989f95ea316265482048c397519ba9dda499 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 15:30:00 +1100 Subject: [PATCH 1473/2680] KEYS: trusted: Keep the state of a runtime source out of read-only data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit apple_sep registers its trusted-key source from module init, after mark_readonly(). Commit cc99abbe2aa7 ("keys/trusted_keys: mark 'migratable' as __ro_after_init"), new in v7.2, turns the store that register_trusted_key_source() then makes to migratable into a write to a read-only page. On 7.2.10 the SEP probe Oopses with a permission fault, ESR 0x9600004f, at register_trusted_key_source+0x1b4/0x1d4, called from sep_tk_register+0x40 [apple_sep], and Touch ID and the SEP trusted keys are gone. unregister_trusted_key_source() makes the same store. migratable and trusted_key_exit belong to the built-in source that init_trusted() picks, and mainline sets them nowhere else. Leave them to init_trusted() and keep the state of a source registered later apart: its migration policy goes to trusted_key_source_migratable, which a new payload reads once, and its exit() is reached through the registered source. The built-in policy stays read-only after init. Source selection, the RNG choice, static-call dispatch and the non-migratable SEP keys are unchanged. [ryan: adapted to 7.2.10 — on 7.1 migratable is writable, so the store only faults from v7.2 on. Mainline still has no runtime registration: it chooses the source in init_trusted() alone.] Fixes: 24d20c9828b8 ("soc: apple: add Apple SEP driver (Touch ID and key sealing)") Assisted-by: LLM Signed-off-by: Ryan Murray --- security/keys/trusted-keys/trusted_core.c | 26 +++++++++++++++++------ 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/security/keys/trusted-keys/trusted_core.c b/security/keys/trusted-keys/trusted_core.c index de218b1b1f5a6..77db540b420ed 100644 --- a/security/keys/trusted-keys/trusted_core.c +++ b/security/keys/trusted-keys/trusted_core.c @@ -70,6 +70,12 @@ static unsigned char migratable __ro_after_init; static DEFINE_MUTEX(trusted_key_source_lock); static bool trusted_key_source_active; static struct trusted_key_source *trusted_key_source_registered; +/* + * trusted_key_exit and migratable belong to the built-in source and only + * init_trusted() sets them; migratable is read-only after init. A source + * registered later keeps its migration policy here, -1 while there is none. + */ +static int trusted_key_source_migratable = -1; /* Defined below; registration needs it before its definition. */ static int kernel_get_random(unsigned char *key, size_t key_len); @@ -142,8 +148,7 @@ int register_trusted_key_source(struct trusted_key_source *src) static_call_update(trusted_key_seal, src->ops->seal); static_call_update(trusted_key_unseal, src->ops->unseal); static_call_update(trusted_key_get_random, get_random); - trusted_key_exit = src->ops->exit; - migratable = src->ops->migratable; + WRITE_ONCE(trusted_key_source_migratable, src->ops->migratable); trusted_key_source_active = true; trusted_key_source_registered = src; @@ -174,10 +179,9 @@ void unregister_trusted_key_source(struct trusted_key_source *src) static_call_update(trusted_key_unseal, trusted_key_absent_unseal); static_call_update(trusted_key_get_random, trusted_key_absent_get_random); - if (trusted_key_exit) - trusted_key_exit(); - trusted_key_exit = NULL; - migratable = 0; + if (src->ops->exit) + src->ops->exit(); + WRITE_ONCE(trusted_key_source_migratable, -1); trusted_key_source_active = false; trusted_key_source_registered = NULL; @@ -266,6 +270,14 @@ static int datablob_parse(char **datablob, struct trusted_key_payload *p) return ret; } +/* The migration policy of the source that seals new keys. */ +static unsigned char trusted_key_migratable(void) +{ + int m = READ_ONCE(trusted_key_source_migratable); + + return m < 0 ? migratable : m; +} + static struct trusted_key_payload *trusted_payload_alloc(struct key *key) { struct trusted_key_payload *p = NULL; @@ -278,7 +290,7 @@ static struct trusted_key_payload *trusted_payload_alloc(struct key *key) if (!p) goto err; - p->migratable = migratable; + p->migratable = trusted_key_migratable(); err: return p; } From e99548b5a58fe1126c124ac0ac157664376c9840 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 15:30:00 +1100 Subject: [PATCH 1474/2680] KEYS: trusted: Test trusted-key sources registered at runtime Add KUnit tests for register_trusted_key_source() and unregister_trusted_key_source(), with a synthetic source that touches no hardware and no real key material. They cover dispatch to the source and back to the absent stubs when it withdraws, the migration policy that each registration brings (a key update included), a second source being refused and unable to withdraw the first, a failing init(), and incomplete sources. Built in, KUnit runs before mark_readonly(), while read-only-after-init data can still be written: there, the store that the previous commit removes goes unnoticed, and the suite skips one case to say so. Loaded as a module after boot, as apple_sep is, the suite catches it: without the previous commit, it Oopses in register_trusted_key_source(). Assisted-by: LLM Signed-off-by: Ryan Murray --- security/keys/trusted-keys/Kconfig | 20 + security/keys/trusted-keys/Makefile | 2 + .../trusted-keys/tests/trusted_source_kunit.c | 342 ++++++++++++++++++ 3 files changed, 364 insertions(+) create mode 100644 security/keys/trusted-keys/tests/trusted_source_kunit.c diff --git a/security/keys/trusted-keys/Kconfig b/security/keys/trusted-keys/Kconfig index e5a4a53aeab2f..3a3799e4134eb 100644 --- a/security/keys/trusted-keys/Kconfig +++ b/security/keys/trusted-keys/Kconfig @@ -80,3 +80,23 @@ config TRUSTED_KEYS_PKWM if !HAVE_TRUSTED_KEYS comment "No trust source selected!" endif + +config TRUSTED_SOURCE_KUNIT_TEST + tristate "KUnit tests for trusted-key sources registered at runtime" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + This builds unit tests for register_trusted_key_source() and + unregister_trusted_key_source(), using a synthetic source that + touches no hardware and no real key material. + + Built in, they run before mark_readonly(), while read-only-after-init + data can still be written. Build them as a module and load it after + boot to test registration the way a source module such as apple_sep + does it. They skip when another source holds the slot; a source + module that registers while they run gets -EBUSY. + + For more information on KUnit and unit tests in general, + please refer to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. diff --git a/security/keys/trusted-keys/Makefile b/security/keys/trusted-keys/Makefile index 5fc053a21dadf..a940ac3f0d666 100644 --- a/security/keys/trusted-keys/Makefile +++ b/security/keys/trusted-keys/Makefile @@ -18,3 +18,5 @@ trusted-$(CONFIG_TRUSTED_KEYS_CAAM) += trusted_caam.o trusted-$(CONFIG_TRUSTED_KEYS_DCP) += trusted_dcp.o trusted-$(CONFIG_TRUSTED_KEYS_PKWM) += trusted_pkwm.o + +obj-$(CONFIG_TRUSTED_SOURCE_KUNIT_TEST) += tests/trusted_source_kunit.o diff --git a/security/keys/trusted-keys/tests/trusted_source_kunit.c b/security/keys/trusted-keys/tests/trusted_source_kunit.c new file mode 100644 index 0000000000000..f0491743f8d5a --- /dev/null +++ b/security/keys/trusted-keys/tests/trusted_source_kunit.c @@ -0,0 +1,342 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for trusted-key sources registered at runtime, through + * register_trusted_key_source() and unregister_trusted_key_source(). + * + * The source under test is synthetic: it touches no hardware and seals + * nothing for real. A test is skipped when another source holds the slot. + * + * Copyright 2026 Ryan Murray + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define FAKE_NAME "trusted_source_kunit" + +struct trusted_source_fake { + struct trusted_key_ops ops; + struct trusted_key_source src; + int init_ret; + int inits, exits, seals, unseals; +}; + +/* The ops have no context argument; the tests run one at a time. */ +static struct trusted_source_fake *fake; + +static int fake_init(void) +{ + fake->inits++; + return fake->init_ret; +} + +static void fake_exit(void) +{ + fake->exits++; +} + +/* Not a cipher: an invertible stand-in, so that unseal can be checked. */ +static int fake_seal(struct trusted_key_payload *p, char *datablob) +{ + unsigned int i; + + for (i = 0; i < p->key_len; i++) + p->blob[i] = p->key[i] ^ 0x5a; + p->blob_len = p->key_len; + fake->seals++; + return 0; +} + +static int fake_unseal(struct trusted_key_payload *p, char *datablob) +{ + unsigned int i; + + if (p->blob_len < MIN_KEY_SIZE || p->blob_len > MAX_KEY_SIZE) + return -EINVAL; + for (i = 0; i < p->blob_len; i++) + p->key[i] = p->blob[i] ^ 0x5a; + p->key_len = p->blob_len; + fake->unseals++; + return 0; +} + +static int fake_get_random(unsigned char *key, size_t key_len) +{ + size_t i; + + for (i = 0; i < key_len; i++) + key[i] = i; + return key_len; +} + +KUNIT_DEFINE_ACTION_WRAPPER(trusted_source_withdraw, + unregister_trusted_key_source, + struct trusted_key_source *); +KUNIT_DEFINE_ACTION_WRAPPER(trusted_source_key_put, key_put, struct key *); + +/* Register the fake, or skip if another source owns or claims the slot. */ +static void fake_register(struct kunit *test) +{ + int ret = register_trusted_key_source(&fake->src); + + if (ret == -EBUSY || ret == -ENODEV) + kunit_skip(test, "another trusted-key source holds the slot (%d)", + ret); + KUNIT_ASSERT_EQ(test, ret, 0); +} + +/* Instantiate a trusted key from @cmd; it is put when the test ends. */ +static struct key *fake_key(struct kunit *test, const char *cmd) +{ + struct key *key; + int ret; + + key = key_alloc(&key_type_trusted, FAKE_NAME, GLOBAL_ROOT_UID, + GLOBAL_ROOT_GID, current_cred(), KEY_POS_ALL, + KEY_ALLOC_NOT_IN_QUOTA, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, key); + + ret = key_instantiate_and_link(key, cmd, strlen(cmd), NULL, NULL); + if (ret) { + key_put(key); + return ERR_PTR(ret); + } + ret = kunit_add_action_or_reset(test, trusted_source_key_put, key); + KUNIT_ASSERT_EQ(test, ret, 0); + return key; +} + +static unsigned char fake_key_migratable(struct key *key) +{ + struct trusted_key_payload *p; + unsigned char migratable; + + down_read(&key->sem); + p = dereference_key_locked(key); + migratable = p->migratable; + up_read(&key->sem); + return migratable; +} + +/* + * Built in, KUnit runs before mark_readonly(), while read-only-after-init + * data can still be written. Loaded as a module after boot, as apple_sep + * is, every registration in this suite runs with that protection on. + */ +static void trusted_source_after_mark_readonly(struct kunit *test) +{ + int i; + + if (system_state < SYSTEM_RUNNING) + kunit_skip(test, "built in: runs before mark_readonly()"); + + for (i = 0; i < 2; i++) { + fake_register(test); + KUNIT_EXPECT_FALSE(test, IS_ERR(fake_key(test, "new 32"))); + unregister_trusted_key_source(&fake->src); + } + KUNIT_EXPECT_EQ(test, fake->inits, 2); + KUNIT_EXPECT_EQ(test, fake->exits, 2); + KUNIT_EXPECT_EQ(test, fake->seals, 2); +} + +/* Keys are sealed and unsealed by the source until it withdraws. */ +static void trusted_source_dispatch(struct kunit *test) +{ + struct trusted_key_payload *p; + unsigned char *plain; + unsigned int len; + struct key *key; + char *load; + + plain = kunit_kzalloc(test, MAX_KEY_SIZE, GFP_KERNEL); + load = kunit_kzalloc(test, sizeof("load ") + 2 * MAX_BLOB_SIZE, + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, plain); + KUNIT_ASSERT_NOT_NULL(test, load); + + fake_register(test); + KUNIT_EXPECT_EQ(test, fake->inits, 1); + + key = fake_key(test, "new 32"); + KUNIT_ASSERT_FALSE(test, IS_ERR(key)); + KUNIT_EXPECT_EQ(test, fake->seals, 1); + + down_read(&key->sem); + p = dereference_key_locked(key); + len = min_t(unsigned int, p->key_len, MAX_KEY_SIZE); + memcpy(plain, p->key, len); + memcpy(load, "load ", 5); + bin2hex(load + 5, p->blob, + min_t(unsigned int, p->blob_len, MAX_BLOB_SIZE)); + up_read(&key->sem); + KUNIT_EXPECT_EQ(test, len, 32U); + + key = fake_key(test, load); + KUNIT_ASSERT_FALSE(test, IS_ERR(key)); + KUNIT_EXPECT_EQ(test, fake->unseals, 1); + + down_read(&key->sem); + p = dereference_key_locked(key); + KUNIT_EXPECT_EQ(test, p->key_len, len); + KUNIT_EXPECT_MEMEQ(test, p->key, plain, len); + up_read(&key->sem); + + unregister_trusted_key_source(&fake->src); + KUNIT_EXPECT_EQ(test, fake->exits, 1); + + /* Dispatch is back on the absent stubs, not on the departed source. */ + KUNIT_EXPECT_EQ(test, PTR_ERR(fake_key(test, "new 32")), -ENODEV); + KUNIT_EXPECT_EQ(test, PTR_ERR(fake_key(test, load)), -ENODEV); + KUNIT_EXPECT_EQ(test, fake->seals, 1); + KUNIT_EXPECT_EQ(test, fake->unseals, 1); +} + +/* Each registration brings its own policy; a withdrawn one leaves none. */ +static void trusted_source_migratable(struct kunit *test) +{ + static const unsigned char policy[] = { 1, 0, 1 }; + struct key *key; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(policy); i++) { + fake->ops.migratable = policy[i]; + fake_register(test); + + key = fake_key(test, "new 32"); + KUNIT_ASSERT_FALSE(test, IS_ERR(key)); + KUNIT_EXPECT_EQ(test, fake_key_migratable(key), policy[i]); + + /* Only a migratable key may be resealed. */ + KUNIT_EXPECT_EQ(test, key_update(make_key_ref(key, true), + "update", 6), + policy[i] ? 0 : -EPERM); + KUNIT_EXPECT_EQ(test, fake_key_migratable(key), policy[i]); + + unregister_trusted_key_source(&fake->src); + } + KUNIT_EXPECT_EQ(test, fake->seals, 3 + 2); +} + +/* A second source is refused, and cannot withdraw the first. */ +static void trusted_source_exclusive(struct kunit *test) +{ + struct trusted_key_source other = { + .name = FAKE_NAME, + .ops = &fake->ops, + }; + + fake_register(test); + KUNIT_EXPECT_EQ(test, register_trusted_key_source(&other), -EBUSY); + KUNIT_EXPECT_EQ(test, register_trusted_key_source(&fake->src), -EBUSY); + KUNIT_EXPECT_EQ(test, fake->inits, 1); + + unregister_trusted_key_source(&other); + KUNIT_EXPECT_EQ(test, fake->exits, 0); + KUNIT_EXPECT_FALSE(test, IS_ERR(fake_key(test, "new 32"))); + KUNIT_EXPECT_EQ(test, fake->seals, 1); + + unregister_trusted_key_source(&fake->src); + KUNIT_EXPECT_EQ(test, fake->exits, 1); +} + +/* A failed init() leaves no source behind, and registration can retry. */ +static void trusted_source_init_failure(struct kunit *test) +{ + int ret; + + fake->init_ret = -EIO; + ret = register_trusted_key_source(&fake->src); + if (ret == -EBUSY || ret == -ENODEV) + kunit_skip(test, "another trusted-key source holds the slot (%d)", + ret); + KUNIT_EXPECT_EQ(test, ret, -EIO); + KUNIT_EXPECT_EQ(test, fake->inits, 1); + KUNIT_EXPECT_EQ(test, fake->exits, 0); + KUNIT_EXPECT_EQ(test, PTR_ERR(fake_key(test, "new 32")), -ENODEV); + + fake->init_ret = 0; + fake_register(test); + KUNIT_EXPECT_FALSE(test, IS_ERR(fake_key(test, "new 32"))); + unregister_trusted_key_source(&fake->src); + KUNIT_EXPECT_EQ(test, fake->inits, 2); + KUNIT_EXPECT_EQ(test, fake->exits, 1); +} + +static void trusted_source_expect_invalid(struct kunit *test, + struct trusted_key_source *src) +{ + int ret = register_trusted_key_source(src); + + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + if (!ret) + unregister_trusted_key_source(src); +} + +/* Incomplete sources are refused before anything is called. */ +static void trusted_source_invalid(struct kunit *test) +{ + struct trusted_key_ops no_seal = fake->ops; + struct trusted_key_source no_name = { .ops = &fake->ops }; + struct trusted_key_source no_ops = { .name = FAKE_NAME }; + struct trusted_key_source bad_ops = { + .name = FAKE_NAME, + .ops = &no_seal, + }; + + no_seal.seal = NULL; + trusted_source_expect_invalid(test, NULL); + trusted_source_expect_invalid(test, &no_name); + trusted_source_expect_invalid(test, &no_ops); + trusted_source_expect_invalid(test, &bad_ops); + KUNIT_EXPECT_EQ(test, fake->inits, 0); +} + +static int trusted_source_init(struct kunit *test) +{ + fake = kunit_kzalloc(test, sizeof(*fake), GFP_KERNEL); + if (!fake) + return -ENOMEM; + + fake->ops.init = fake_init; + fake->ops.seal = fake_seal; + fake->ops.unseal = fake_unseal; + fake->ops.get_random = fake_get_random; + fake->ops.exit = fake_exit; + fake->src.name = FAKE_NAME; + fake->src.ops = &fake->ops; + + /* However the test ends; a no-op unless the fake is registered. */ + return kunit_add_action_or_reset(test, trusted_source_withdraw, + &fake->src); +} + +static struct kunit_case trusted_source_cases[] = { + KUNIT_CASE(trusted_source_after_mark_readonly), + KUNIT_CASE(trusted_source_dispatch), + KUNIT_CASE(trusted_source_migratable), + KUNIT_CASE(trusted_source_exclusive), + KUNIT_CASE(trusted_source_init_failure), + KUNIT_CASE(trusted_source_invalid), + {} +}; + +static struct kunit_suite trusted_source_suite = { + .name = "trusted_source", + .init = trusted_source_init, + .test_cases = trusted_source_cases, +}; + +kunit_test_suite(trusted_source_suite); + +MODULE_DESCRIPTION("KUnit tests for trusted-key sources registered at runtime"); +MODULE_LICENSE("GPL"); From 770be78eb3b1fa5f6b5e72d09387c93a839e3bb9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 14:56:33 +1100 Subject: [PATCH 1475/2680] soc: apple: aop: Accept a transmit entry at a write pointer of zero write_entry() takes an entry placed at 0 to be one that wrapped to the top of the ring. An entry that fits at the write pointer is placed at 0 as well when the write pointer itself is 0, which it is whenever the previous entry ended exactly at the end of the ring. Until the firmware has read that entry, the read pointer is above the write pointer, and the check for a wrapped entry below the read pointer refuses every entry although [0, rptr) is free. The call fails with "Tx buffer full" and EIO until the firmware catches up. Decide whether the entry wraps from its length and the write pointer, as the placement already does, and check that instead of the placement. Add KUnit cases for an entry at a write pointer of 0 with an unread entry above it. [ryan: Fixes: names commit d7ef97006a66 ("soc: apple: aop: Do not overrun unread entries in the transmit ring"), the 7.2.10 port of 13d14a5444c7.] Fixes: d7ef97006a66 ("soc: apple: aop: Do not overrun unread entries in the transmit ring") (cherry picked from commit 30339518b20cf6bdce03af002c174097fa186c28) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop_ring.rs | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/drivers/soc/apple/aop_ring.rs b/drivers/soc/apple/aop_ring.rs index ccbb661331e2a..41608ba7699b5 100644 --- a/drivers/soc/apple/aop_ring.rs +++ b/drivers/soc/apple/aop_ring.rs @@ -108,20 +108,20 @@ impl Ring { // Reject an entry the whole ring cannot take before even writing // the wrap header. let head_next = self.end(0, len)?; - let ptr = if len > self.size - wptr - QEH_SIZE { - 0 - } else { - wptr - }; - let next = if ptr == 0 { + // The entry wraps when it does not fit before the end of the ring. + // It then goes at 0, but so does an entry that fits at a write + // pointer of 0: where it goes does not tell whether it wrapped. + let wrapped = len > self.size - wptr - QEH_SIZE; + let ptr = if wrapped { 0 } else { wptr }; + let next = if wrapped { head_next } else { self.end(ptr, len)? }; if head_next == 0 || next == rptr - || (wptr < rptr && (ptr == 0 || next == 0 || next > rptr)) - || (ptr < wptr && (rptr == 0 || next > rptr)) + || (wptr < rptr && (wrapped || next == 0 || next > rptr)) + || (wrapped && (rptr == 0 || next > rptr)) { return Err(ENOSPC); } @@ -307,6 +307,19 @@ mod tests { Ok(()) } + #[test] + fn an_entry_fits_at_a_write_pointer_of_zero() -> Result { + let ring = Ring::new(BUF, BLOCK)?; + // The last entry ended at the end of the ring and is still unread, + // so the write pointer is 0 below the read pointer: [0, rptr) is free. + assert_eq!(ring.write_entry(0xf80, 0, 0x40 - QEH_SIZE)?, (0, 0x40)); + assert_eq!(ring.write_entry(0x80, 0, 0x40 - QEH_SIZE)?, (0, 0x40)); + // It still never reaches the read pointer. + assert!(ring.write_entry(0x40, 0, 0x40 - QEH_SIZE).is_err()); + assert!(ring.write_entry(0x80, 0, 0x80 - QEH_SIZE).is_err()); + Ok(()) + } + #[test] fn an_entry_never_reaches_the_read_pointer() -> Result { let ring = Ring::new(BUF, BLOCK)?; From 1c52d90829929d9b71b3857aeab92f14f14b31a7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 21:09:11 +1100 Subject: [PATCH 1476/2680] usb: typec: tipd: cd321x: Never leave interrupt events unacknowledged MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CD321x and SN201202x controllers latch their events in INT_EVENT1 until the host writes them to INT_CLEAR1, and raise their interrupt only when an event is latched while none were pending. When cd321x_interrupt() fails to read the latched events it returns without acknowledging them, so the controller never interrupts again, and the port ignores every later plug, power and Alternate Mode change until the driver is rebound. A failed write to INT_CLEAR1 does the same, and a failed status read after the acknowledgment drops the change behind the event. On the J700 the rear-port SN201202x hits this after s2idle: its event handler runs while the controller is still in SPMI sleep, the register select times out, "cd321x_interrupt: failed to read events" is logged, and the port stays dead for the rest of the boot. The resume revalidation reads a fresh snapshot but leaves the latched events alone, so it does not recover the port either. Acknowledge latched events at the start of every snapshot read, which covers them, and have an interrupt that could not read, acknowledge or apply its events schedule such a read with a full retry budget. A port whose earlier recovery gave up is included, since no further interrupt can arrive to reopen it. Failures during suspend or probe need nothing extra, because both end with a snapshot read. A failed read says nothing about the cable, so it does not force a reconnect. Keep the acknowledgment apart from the bounded budget for reading and applying the cable state. While the events stay latched, no interrupt can restart a recovery that gave up, so a snapshot read that fails to acknowledge them is not charged to that budget and never gives up: it is retried after a delay that doubles from 500 ms to at most 30 s. Sleep cancels the retry and the resume read takes it over, and removal ends it. Only the first failure of a series is logged. Add KUnit tests that model the controller's event latch behind a regmap and drive the interrupt handler and the snapshot worker. They check that the next event still raises an interrupt after a failed event read, clear or status read, after a failed read on a port that had given up, and after events latched across a sleep cycle; also when the event reads or clears keep failing long after the cable-state budget has run out, including across a sleep cycle, before the bus recovers. They also check that removal stops the retries. Without this change, all of them fail except the one where the interrupt succeeds. State machine tests check the retry delay and that the acknowledgment leaves the cable-state budget alone. [ryan: adapted to 7.2.10 — the IRQ test takes the CD321x data from tipd_hw_variants[TIPD_VARIANT_CD321X]: 7.2.10 keeps the variant data static behind that exported table instead of exporting tipd_cd321x_data] Fixes: 45188f27b3d0 ("usb: typec: tipd: Add support for Apple CD321X") (cherry picked from commit 7e97f2f5ee4553790b00af41e36a9aa4c480ffe4) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/Kconfig | 4 + drivers/usb/typec/tipd/Makefile | 3 +- drivers/usb/typec/tipd/cd321x-irq-test.c | 349 +++++++++++++++++++++++ drivers/usb/typec/tipd/cd321x-pm-test.c | 108 +++++++ drivers/usb/typec/tipd/cd321x-pm.h | 47 +++ drivers/usb/typec/tipd/core.c | 73 ++++- drivers/usb/typec/tipd/tps6598x.h | 1 + 7 files changed, 575 insertions(+), 10 deletions(-) create mode 100644 drivers/usb/typec/tipd/cd321x-irq-test.c diff --git a/drivers/usb/typec/tipd/Kconfig b/drivers/usb/typec/tipd/Kconfig index aae3e81cca848..7c2527913111d 100644 --- a/drivers/usb/typec/tipd/Kconfig +++ b/drivers/usb/typec/tipd/Kconfig @@ -31,11 +31,15 @@ config TYPEC_SN201202X config TYPEC_TIPD_KUNIT_TEST tristate "KUnit tests for CD321x" if !KUNIT_ALL_TESTS depends on KUNIT && TYPEC_TPS6598X_CORE + select REGMAP default KUNIT_ALL_TESTS help Test suspend gating, fresh cable-state revalidation, bounded retries and removal during resume. The tests exercise the state transitions shared with the driver, including aborted and repeated sleep cycles. + Also test that interrupt events which could not be read or cleared + are still acknowledged, however long that keeps failing, so that the + controller keeps raising its interrupt. Also test that a USB4/Thunderbolt router which never registers its switch does not keep the port from probing. These tests run without accessing a physical Type-C controller. diff --git a/drivers/usb/typec/tipd/Makefile b/drivers/usb/typec/tipd/Makefile index 198d57654b577..e2e3d9271dd26 100644 --- a/drivers/usb/typec/tipd/Makefile +++ b/drivers/usb/typec/tipd/Makefile @@ -12,4 +12,5 @@ tps6598x-y := i2c.o obj-$(CONFIG_TYPEC_SN201202X) += sn201202x.o sn201202x-y := spmi.o -obj-$(CONFIG_TYPEC_TIPD_KUNIT_TEST) += cd321x-pm-test.o cd321x-tbt-test.o +obj-$(CONFIG_TYPEC_TIPD_KUNIT_TEST) += cd321x-pm-test.o cd321x-tbt-test.o \ + cd321x-irq-test.o diff --git a/drivers/usb/typec/tipd/cd321x-irq-test.c b/drivers/usb/typec/tipd/cd321x-irq-test.c new file mode 100644 index 0000000000000..ff7aca3f9f4a5 --- /dev/null +++ b/drivers/usb/typec/tipd/cd321x-irq-test.c @@ -0,0 +1,349 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for CD321x interrupt event acknowledgment + */ + +#include +#include +#include +#include + +#include "tps6598x.h" + +#define TEST_REG_INT_EVENT1 0x14 +#define TEST_REG_INT_CLEAR1 0x18 +#define TEST_REG_STATUS 0x1a + +/* + * The controller latches events in INT_EVENT1 until they are written to + * INT_CLEAR1, and raises its interrupt only when an event is latched while + * none were pending. + */ +struct cd321x_test_ctrl { + u64 latched; + unsigned int interrupts; + unsigned int failing_event_reads; + unsigned int failing_clear_writes; + unsigned int failing_status_reads; +}; + +struct cd321x_test { + struct cd321x cd; + struct cd321x_test_ctrl ctrl; +}; + +static void cd321x_test_raise(struct cd321x_test_ctrl *ctrl, u64 event) +{ + if (!ctrl->latched) + ctrl->interrupts++; + ctrl->latched |= event; +} + +static int cd321x_test_read(void *context, const void *reg, size_t reg_size, + void *val, size_t val_size) +{ + struct cd321x_test_ctrl *ctrl = context; + + memset(val, 0, val_size); + switch (*(const u8 *)reg) { + case TEST_REG_INT_EVENT1: + if (ctrl->failing_event_reads) { + ctrl->failing_event_reads--; + return -ETIMEDOUT; + } + memcpy(val, &ctrl->latched, min(val_size, sizeof(ctrl->latched))); + break; + case TEST_REG_STATUS: + if (ctrl->failing_status_reads) { + ctrl->failing_status_reads--; + return -ETIMEDOUT; + } + break; + } + + return 0; +} + +static int cd321x_test_write(void *context, const void *data, size_t count) +{ + struct cd321x_test_ctrl *ctrl = context; + const u8 *buf = data; + u64 clear = 0; + + if (buf[0] == TEST_REG_INT_CLEAR1) { + if (ctrl->failing_clear_writes) { + ctrl->failing_clear_writes--; + return -ETIMEDOUT; + } + memcpy(&clear, buf + 1, min(count - 1, sizeof(clear))); + ctrl->latched &= ~clear; + } + + return 0; +} + +static const struct regmap_bus cd321x_test_bus = { + .read = cd321x_test_read, + .write = cd321x_test_write, + .reg_format_endian_default = REGMAP_ENDIAN_NATIVE, + .val_format_endian_default = REGMAP_ENDIAN_NATIVE, +}; + +static void cd321x_test_stop(void *data) +{ + struct cd321x *cd = data; + + scoped_guard(mutex, &cd->tps.lock) + cd321x_pm_remove(&cd->pm); + cancel_delayed_work_sync(&cd->resume_work); + cancel_delayed_work_sync(&cd->update_work); +} + +static struct cd321x_test *cd321x_test_port(struct kunit *test) +{ + struct cd321x_test *t; + struct device *dev; + + dev = kunit_device_register(test, "cd321x-irq-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t); + + mutex_init(&t->cd.tps.lock); + t->cd.tps.dev = dev; + t->cd.tps.data = tipd_hw_variants[TIPD_VARIANT_CD321X]; + t->cd.tps.regmap = devm_regmap_init(dev, &cd321x_test_bus, &t->ctrl, + &tps6598x_regmap_config); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->cd.tps.regmap); + dev_set_drvdata(dev, &t->cd.tps); + + cd321x_init_work(&t->cd); + t->cd.pm.phase = CD321X_PM_RUNNING; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, cd321x_test_stop, &t->cd), 0); + + return t; +} + +static irqreturn_t cd321x_test_interrupt(struct cd321x_test *t) +{ + return t->cd.tps.data->irq_handler(0, &t->cd.tps); +} + +/* The read of a snapshot that was scheduled to recover the events. */ +static void cd321x_test_recover(struct kunit *test, struct cd321x_test *t) +{ + KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); + flush_delayed_work(&t->cd.resume_work); + scoped_guard(mutex, &t->cd.tps.lock) + KUNIT_EXPECT_TRUE(test, cd321x_pm_can_update(&t->cd.pm)); +} + +/* The time until the next recovery read, which must be scheduled. */ +static unsigned long cd321x_test_retry_delay(struct kunit *test, + struct cd321x_test *t) +{ + KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); + return t->cd.resume_work.timer.expires - jiffies; +} + +/* + * Keep the events from being acknowledged, by failing every access counted by + * @failures, for much longer than the cable-state budget lasts. Once the bus + * works again, the events must still be acknowledged, so that the next cable + * event raises an interrupt. + */ +static void cd321x_test_ack_outage(struct kunit *test, struct cd321x_test *t, + unsigned int *failures) +{ + unsigned long first, delay = 0; + unsigned int i; + + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + *failures = UINT_MAX; + cd321x_test_interrupt(t); + first = cd321x_test_retry_delay(test, t); + + for (i = 0; i < 4 * CD321X_RESUME_ATTEMPTS; i++) { + flush_delayed_work(&t->cd.resume_work); + KUNIT_EXPECT_NE(test, t->cd.pm.phase, CD321X_PM_STALE); + delay = cd321x_test_retry_delay(test, t); + } + /* Retried more and more slowly, but still at least once a minute. */ + KUNIT_EXPECT_GE(test, delay, 8 * first); + KUNIT_EXPECT_LE(test, delay, msecs_to_jiffies(60 * MSEC_PER_SEC)); + KUNIT_EXPECT_EQ(test, t->ctrl.interrupts, 1U); + + *failures = 0; + cd321x_test_recover(test, t); + KUNIT_EXPECT_EQ(test, t->ctrl.latched, 0); + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_DATA_STATUS_UPDATE); + KUNIT_EXPECT_EQ(test, t->ctrl.interrupts, 2U); + KUNIT_EXPECT_EQ(test, cd321x_test_interrupt(t), IRQ_HANDLED); + KUNIT_EXPECT_EQ(test, t->ctrl.latched, 0); +} + +static void cd321x_irq_acknowledges_events_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + KUNIT_EXPECT_EQ(test, cd321x_test_interrupt(t), IRQ_HANDLED); + KUNIT_EXPECT_EQ(test, t->ctrl.latched, 0); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); +} + +static void cd321x_failed_event_read_recovers_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + t->ctrl.failing_event_reads = 1; + cd321x_test_interrupt(t); + KUNIT_ASSERT_NE(test, t->ctrl.latched, 0); + + cd321x_test_recover(test, t); + KUNIT_EXPECT_EQ(test, t->ctrl.latched, 0); + /* The next cable event must reach the host again. */ + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + KUNIT_EXPECT_EQ(test, t->ctrl.interrupts, 2U); +} + +static void cd321x_failed_status_read_recovers_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + t->ctrl.failing_status_reads = 1; + cd321x_test_interrupt(t); + /* Acknowledged, but the cable state behind the event was not read. */ + KUNIT_ASSERT_EQ(test, t->ctrl.latched, 0); + + cd321x_test_recover(test, t); +} + +static void cd321x_failed_read_after_giving_up_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + + t->cd.pm.phase = CD321X_PM_STALE; + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_DATA_STATUS_UPDATE); + t->ctrl.failing_event_reads = 1; + cd321x_test_interrupt(t); + + cd321x_test_recover(test, t); + KUNIT_EXPECT_EQ(test, t->ctrl.latched, 0); +} + +static void cd321x_failed_clear_recovers_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + t->ctrl.failing_clear_writes = 1; + cd321x_test_interrupt(t); + KUNIT_ASSERT_NE(test, t->ctrl.latched, 0); + + cd321x_test_recover(test, t); + KUNIT_EXPECT_EQ(test, t->ctrl.latched, 0); + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + KUNIT_EXPECT_EQ(test, t->ctrl.interrupts, 2U); +} + +static void cd321x_event_reads_failing_for_long_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + + cd321x_test_ack_outage(test, t, &t->ctrl.failing_event_reads); +} + +static void cd321x_clear_writes_failing_for_long_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + + cd321x_test_ack_outage(test, t, &t->ctrl.failing_clear_writes); +} + +static void cd321x_ack_retry_across_sleep_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + struct device *dev = t->cd.tps.dev; + unsigned int i; + + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + t->ctrl.failing_event_reads = UINT_MAX; + cd321x_test_interrupt(t); + flush_delayed_work(&t->cd.resume_work); + + /* Sleep cancels the retry, and the resume read takes it over. */ + KUNIT_ASSERT_EQ(test, tipd_prepare(dev), 0); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); + tipd_complete(dev); + for (i = 0; i < 2 * CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); + flush_delayed_work(&t->cd.resume_work); + } + + t->ctrl.failing_event_reads = 0; + cd321x_test_recover(test, t); + KUNIT_EXPECT_EQ(test, t->ctrl.latched, 0); + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + KUNIT_EXPECT_EQ(test, t->ctrl.interrupts, 2U); +} + +static void cd321x_ack_retry_ends_on_removal_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + t->ctrl.failing_event_reads = UINT_MAX; + cd321x_test_interrupt(t); + flush_delayed_work(&t->cd.resume_work); + KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); + + scoped_guard(mutex, &t->cd.tps.lock) + cd321x_pm_remove(&t->cd.pm); + /* A retry that was already queued neither reads nor queues another. */ + flush_delayed_work(&t->cd.resume_work); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); + KUNIT_EXPECT_EQ(test, t->ctrl.failing_event_reads, UINT_MAX - 2); +} + +static void cd321x_resume_acknowledges_events_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + struct device *dev = t->cd.tps.dev; + + KUNIT_ASSERT_EQ(test, tipd_prepare(dev), 0); + /* Latched while the interrupt was off, and never delivered. */ + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_POWER_STATUS_UPDATE); + tipd_complete(dev); + + cd321x_test_recover(test, t); + KUNIT_EXPECT_EQ(test, t->ctrl.latched, 0); + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + KUNIT_EXPECT_EQ(test, t->ctrl.interrupts, 2U); +} + +static struct kunit_case cd321x_irq_cases[] = { + KUNIT_CASE(cd321x_irq_acknowledges_events_test), + KUNIT_CASE(cd321x_failed_event_read_recovers_test), + KUNIT_CASE(cd321x_failed_status_read_recovers_test), + KUNIT_CASE(cd321x_failed_read_after_giving_up_test), + KUNIT_CASE(cd321x_failed_clear_recovers_test), + KUNIT_CASE(cd321x_event_reads_failing_for_long_test), + KUNIT_CASE(cd321x_clear_writes_failing_for_long_test), + KUNIT_CASE(cd321x_ack_retry_across_sleep_test), + KUNIT_CASE(cd321x_ack_retry_ends_on_removal_test), + KUNIT_CASE(cd321x_resume_acknowledges_events_test), + {} +}; + +static struct kunit_suite cd321x_irq_suite = { + .name = "cd321x-irq", + .test_cases = cd321x_irq_cases, +}; + +kunit_test_suite(cd321x_irq_suite); +MODULE_DESCRIPTION("KUnit tests for CD321x interrupt event acknowledgment"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/drivers/usb/typec/tipd/cd321x-pm-test.c b/drivers/usb/typec/tipd/cd321x-pm-test.c index f94f4ad57cb75..977df770dcff5 100644 --- a/drivers/usb/typec/tipd/cd321x-pm-test.c +++ b/drivers/usb/typec/tipd/cd321x-pm-test.c @@ -417,6 +417,109 @@ static void cd321x_busy_provider_keeps_real_failure_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, pm.force_reconnect); } +static void cd321x_lost_event_rereads_test(struct kunit *test) +{ + static const enum cd321x_pm_phase phases[] = { + CD321X_PM_RUNNING, CD321X_PM_REVALIDATE, + CD321X_PM_APPLY, CD321X_PM_STALE, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(phases); i++) { + struct cd321x_pm_state pm = { .phase = phases[i] }; + + /* Even an exhausted recovery must read the latched events. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_event_lost(&pm)); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_REVALIDATE); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_EXPECT_FALSE(test, pm.force_reconnect); + KUNIT_EXPECT_TRUE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_EQ(test, pm.attempts_left, CD321X_RESUME_ATTEMPTS - 1U); + } +} + +static void cd321x_lost_event_left_to_resume_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + cd321x_pm_init(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_event_lost(&pm)); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_INITIALIZING); + KUNIT_ASSERT_TRUE(test, cd321x_pm_ready(&pm)); + + /* A read failing during suspend is redone by the resume read. */ + cd321x_pm_prepare(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_event_lost(&pm)); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_PREPARED); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_EXPECT_TRUE(test, cd321x_pm_begin_read(&pm)); + + cd321x_pm_remove(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_event_lost(&pm)); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_REMOVED); +} + +static void cd321x_lost_event_keeps_reconnect_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm, true)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_event_lost(&pm)); + /* A failed read is no evidence either way about the cable. */ + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); +} + +static void cd321x_unacked_events_outlast_budget_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int i, delay = 0; + + KUNIT_ASSERT_TRUE(test, cd321x_pm_event_lost(&pm)); + for (i = 0; i < 4 * CD321X_RESUME_ATTEMPTS; i++) { + unsigned int next; + + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + next = cd321x_pm_ack_failed(&pm); + /* Never given up, and not charged to the cable-state budget... */ + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_REVALIDATE); + KUNIT_EXPECT_EQ(test, pm.attempts_left, (unsigned int)CD321X_RESUME_ATTEMPTS); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + /* ...only retried less and less often, down to a limit. */ + KUNIT_EXPECT_EQ(test, next, i ? min(2 * delay, CD321X_ACK_RETRY_MAX_MS) : + CD321X_ACK_RETRY_MS); + delay = next; + } + KUNIT_EXPECT_EQ(test, delay, CD321X_ACK_RETRY_MAX_MS); + + /* Once acknowledged, the cable state has its bounded budget again. */ + KUNIT_EXPECT_TRUE(test, cd321x_pm_acked(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_acked(&pm)); + for (i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_EQ(test, cd321x_pm_retry(&pm, true), + i + 1 < CD321X_RESUME_ATTEMPTS); + } + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); +} + +static void cd321x_unacked_events_across_sleep_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int delay; + + KUNIT_ASSERT_TRUE(test, cd321x_pm_event_lost(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + delay = cd321x_pm_ack_failed(&pm); + cd321x_pm_prepare(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + /* The resume read takes over the retries, without starting afresh. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_GT(test, cd321x_pm_ack_failed(&pm), delay); + cd321x_pm_remove(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); +} + /* Only a pipeline owned by another port skips the retry. */ static void cd321x_display_taken_test(struct kunit *test) { @@ -451,6 +554,11 @@ static struct kunit_case cd321x_pm_cases[] = { KUNIT_CASE(cd321x_read_and_apply_share_budget_test), KUNIT_CASE(cd321x_status_event_after_exhaustion_test), KUNIT_CASE(cd321x_runtime_failure_revalidates_test), + KUNIT_CASE(cd321x_lost_event_rereads_test), + KUNIT_CASE(cd321x_lost_event_left_to_resume_test), + KUNIT_CASE(cd321x_lost_event_keeps_reconnect_test), + KUNIT_CASE(cd321x_unacked_events_outlast_budget_test), + KUNIT_CASE(cd321x_unacked_events_across_sleep_test), KUNIT_CASE(cd321x_display_taken_test), {} }; diff --git a/drivers/usb/typec/tipd/cd321x-pm.h b/drivers/usb/typec/tipd/cd321x-pm.h index 08421d54c570b..3879c8a660dcc 100644 --- a/drivers/usb/typec/tipd/cd321x-pm.h +++ b/drivers/usb/typec/tipd/cd321x-pm.h @@ -4,10 +4,13 @@ #include +#include #include #define CD321X_RESUME_ATTEMPTS 3 #define CD321X_LINK_EVENTS 3 +#define CD321X_ACK_RETRY_MS 500 +#define CD321X_ACK_RETRY_MAX_MS 30000 enum cd321x_pm_phase { CD321X_PM_RUNNING, @@ -24,6 +27,8 @@ struct cd321x_pm_state { enum cd321x_pm_phase phase; unsigned int attempts_left; unsigned int link_events_left; + /* Non-zero while events may be left unacknowledged: the retry delay. */ + unsigned int ack_retry_ms; bool force_reconnect; }; @@ -127,6 +132,48 @@ static inline bool cd321x_pm_event(struct cd321x_pm_state *pm) return true; } +/* + * Events that could not be read and acknowledged stay latched in the + * controller, which then sends no further interrupt. Only a fresh read can + * acknowledge them, so request one with a full budget even if an earlier + * recovery gave up. Sleep and probe already end with such a read. + */ +static inline bool cd321x_pm_event_lost(struct cd321x_pm_state *pm) +{ + if (pm->phase == CD321X_PM_INITIALIZING || + pm->phase == CD321X_PM_PREPARED || pm->phase == CD321X_PM_REMOVED) + return false; + pm->phase = CD321X_PM_REVALIDATE; + pm->attempts_left = CD321X_RESUME_ATTEMPTS; + return true; +} + +/* + * Until the latched events are acknowledged, the controller raises no + * interrupt that could restart a recovery that gave up. So a snapshot read + * that fails to acknowledge them does not count against the cable-state + * budget, and is never given up: it gives back the attempt taken by + * cd321x_pm_begin_read() and is retried after a delay that doubles up to + * CD321X_ACK_RETRY_MAX_MS. Sleep cancels the retry and the resume read takes + * it over; removal ends it. Returns the delay in milliseconds. + */ +static inline unsigned int cd321x_pm_ack_failed(struct cd321x_pm_state *pm) +{ + pm->attempts_left++; + pm->ack_retry_ms = clamp(2 * pm->ack_retry_ms, CD321X_ACK_RETRY_MS, + CD321X_ACK_RETRY_MAX_MS); + return pm->ack_retry_ms; +} + +/* Returns true if acknowledging the events had failed before. */ +static inline bool cd321x_pm_acked(struct cd321x_pm_state *pm) +{ + bool retried = pm->ack_retry_ms; + + pm->ack_retry_ms = 0; + return retried; +} + static inline void cd321x_pm_complete(struct cd321x_pm_state *pm) { if (pm->phase == CD321X_PM_APPLY) { diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 98d3237f11458..74dcec2a2030f 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -1001,6 +1001,33 @@ static int cd321x_connect(struct tps6598x *tps, u32 status) return 0; } +/* + * The controller raises its interrupt only when events are latched while none + * were pending, so latched events that are never acknowledged silence the port + * for good. Acknowledge them before a snapshot read: the snapshot covers them. + */ +static int cd321x_ack_events(struct tps6598x *tps) +{ + u64 event; + int ret; + + ret = tps6598x_read64(tps, TPS_REG_INT_EVENT1, &event); + if (ret || !event) + return ret; + trace_cd321x_irq(event); + + return tps6598x_write64(tps, TPS_REG_INT_CLEAR1, event); +} + +/* Recover events that an interrupt could not read, acknowledge or apply. */ +static void cd321x_event_lost(struct cd321x *cd321x) +{ + lockdep_assert_held(&cd321x->tps.lock); + if (cd321x_pm_event_lost(&cd321x->pm)) + mod_delayed_work(system_freezable_wq, &cd321x->resume_work, + msecs_to_jiffies(CD321X_DEBOUNCE_DELAY_MS)); +} + /* * An attached cable need not generate an IRQ after sleep. Read a fresh * snapshot and reapply it with the mode cache invalidated, allowing the @@ -1013,6 +1040,7 @@ static void cd321x_resume_work(struct work_struct *work) struct cd321x, resume_work); struct tps6598x *tps = &cd321x->tps; u32 status; + int ret; guard(mutex)(&tps->lock); if (cd321x->pm.phase == CD321X_PM_INITIALIZING || @@ -1032,6 +1060,18 @@ static void cd321x_resume_work(struct work_struct *work) if (!cd321x_pm_begin_read(&cd321x->pm)) return; + ret = cd321x_ack_events(tps); + if (ret) { + /* Retried until it succeeds, so report only the first failure. */ + if (!cd321x->pm.ack_retry_ms) + dev_err(tps->dev, "failed to acknowledge events: %d\n", ret); + mod_delayed_work(system_freezable_wq, &cd321x->resume_work, + msecs_to_jiffies(cd321x_pm_ack_failed(&cd321x->pm))); + return; + } + if (cd321x_pm_acked(&cd321x->pm)) + dev_info(tps->dev, "acknowledged events after retrying\n"); + if (!tps6598x_read_status(tps, &status)) goto retry; @@ -1079,6 +1119,7 @@ static void cd321x_complete(struct tps6598x *tps) static irqreturn_t cd321x_interrupt(int irq, void *data) { struct tps6598x *tps = data; + struct cd321x *cd321x = container_of(tps, struct cd321x, tps); u64 event = 0; u32 status; int ret; @@ -1088,21 +1129,25 @@ static irqreturn_t cd321x_interrupt(int irq, void *data) ret = tps6598x_read64(tps, TPS_REG_INT_EVENT1, &event); if (ret) { dev_err(tps->dev, "%s: failed to read events\n", __func__); - goto err_unlock; + goto err_lost; } trace_cd321x_irq(event); if (!event) goto err_unlock; - tps6598x_write64(tps, TPS_REG_INT_CLEAR1, event); + ret = tps6598x_write64(tps, TPS_REG_INT_CLEAR1, event); + if (ret) { + dev_err(tps->dev, "%s: failed to clear events\n", __func__); + goto err_lost; + } if (!tps6598x_read_status(tps, &status)) - goto err_unlock; + goto err_lost; if (event & APPLE_CD_REG_INT_POWER_STATUS_UPDATE) { if (!tps6598x_read_power_status(tps)) - goto err_unlock; + goto err_lost; if (TPS_POWER_STATUS_PWROPMODE(tps->pwr_status) == TYPEC_PWR_MODE_PD) { if (tps6598x_read_partner_identity(tps)) { dev_err(tps->dev, "failed to read partner identity\n"); @@ -1113,11 +1158,14 @@ static irqreturn_t cd321x_interrupt(int irq, void *data) if (event & APPLE_CD_REG_INT_DATA_STATUS_UPDATE) if (!tps->data->read_data_status(tps)) - goto err_unlock; + goto err_lost; /* Can be called uncondtionally since it will check for any changes itself */ cd321x_connect(tps, status); + goto err_unlock; +err_lost: + cd321x_event_lost(cd321x); err_unlock: mutex_unlock(&tps->lock); @@ -1550,16 +1598,23 @@ cd321x_tbt_switch_get(struct device *dev, struct fwnode_handle *fwnode) } EXPORT_SYMBOL_IF_KUNIT(cd321x_tbt_switch_get); +/* Separate so that the KUnit tests can drive the workers without a port. */ +VISIBLE_IF_KUNIT void cd321x_init_work(struct cd321x *cd321x) +{ + INIT_DELAYED_WORK(&cd321x->update_work, cd321x_update_work); + INIT_DELAYED_WORK(&cd321x->resume_work, cd321x_resume_work); + atomic_set(&cd321x->link_event, 0); + cd321x_pm_init(&cd321x->pm); +} +EXPORT_SYMBOL_IF_KUNIT(cd321x_init_work); + static int cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) { struct cd321x *cd321x = container_of(tps, struct cd321x, tps); int ret; - INIT_DELAYED_WORK(&cd321x->update_work, cd321x_update_work); - INIT_DELAYED_WORK(&cd321x->resume_work, cd321x_resume_work); - atomic_set(&cd321x->link_event, 0); - cd321x_pm_init(&cd321x->pm); + cd321x_init_work(cd321x); ret = tps6598x_register_port(tps, fwnode); if (ret) diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index ac1713d151c3c..fb7c4decd3018 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -463,6 +463,7 @@ static inline void tipd_debugfs_exit(void) { } #endif #if IS_ENABLED(CONFIG_KUNIT) +void cd321x_init_work(struct cd321x *cd321x); bool cd321x_tbt_switch_keep_waiting(struct device *dev); struct typec_thunderbolt_switch * cd321x_tbt_switch_get(struct device *dev, struct fwnode_handle *fwnode); From 1a81d6dbd21f7544e41040f7545468fe77c34f2c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 23:06:16 +1100 Subject: [PATCH 1477/2680] usb: typec: tipd: cd321x: Recover from a failed partner identity read When cd321x_interrupt() fails to read the identity of a PD partner after a power status change, it logs the error, clears the identity and goes on to queue the cable update. The event has been acknowledged by then, and nothing reads the identity again. The update then registers a new partner with an all-zero identity, or, for a partner that is already registered, takes the changed identity for a different partner and tears down a working connection to register it again. Handle the failure like the other failed reads in the interrupt: leave the queued state alone and schedule a fresh snapshot read, which reads the identity together with the rest of the cable state, retrying within the usual budget. Add a KUnit test whose identity read fails once: the identity in use must be kept until the fresh read has succeeded, and then be replaced. Without this change it fails. Fixes: 36c791c1996d ("usb: typec: tipd: Update partner identity when power status was updated") (cherry picked from commit 6d51a7a3cf1b4be7fdbdd18699b633db078881eb) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/cd321x-irq-test.c | 45 ++++++++++++++++++++++++ drivers/usb/typec/tipd/core.c | 2 +- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/drivers/usb/typec/tipd/cd321x-irq-test.c b/drivers/usb/typec/tipd/cd321x-irq-test.c index ff7aca3f9f4a5..d8744fc5a02f2 100644 --- a/drivers/usb/typec/tipd/cd321x-irq-test.c +++ b/drivers/usb/typec/tipd/cd321x-irq-test.c @@ -13,6 +13,8 @@ #define TEST_REG_INT_EVENT1 0x14 #define TEST_REG_INT_CLEAR1 0x18 #define TEST_REG_STATUS 0x1a +#define TEST_REG_POWER_STATUS 0x3f +#define TEST_REG_RX_IDENTITY_SOP 0x48 /* * The controller latches events in INT_EVENT1 until they are written to @@ -25,6 +27,10 @@ struct cd321x_test_ctrl { unsigned int failing_event_reads; unsigned int failing_clear_writes; unsigned int failing_status_reads; + unsigned int failing_identity_reads; + u32 status; + u16 pwr_status; + struct usb_pd_identity identity; }; struct cd321x_test { @@ -58,6 +64,19 @@ static int cd321x_test_read(void *context, const void *reg, size_t reg_size, ctrl->failing_status_reads--; return -ETIMEDOUT; } + memcpy(val, &ctrl->status, min(val_size, sizeof(ctrl->status))); + break; + case TEST_REG_POWER_STATUS: + memcpy(val, &ctrl->pwr_status, min(val_size, sizeof(ctrl->pwr_status))); + break; + case TEST_REG_RX_IDENTITY_SOP: + if (ctrl->failing_identity_reads) { + ctrl->failing_identity_reads--; + return -ETIMEDOUT; + } + /* The identity follows a status byte. */ + memcpy(val + 1, &ctrl->identity, + min(val_size - 1, sizeof(ctrl->identity))); break; } @@ -234,6 +253,31 @@ static void cd321x_failed_read_after_giving_up_test(struct kunit *test) KUNIT_EXPECT_EQ(test, t->ctrl.latched, 0); } +static void cd321x_failed_identity_read_recovers_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + const struct usb_pd_identity old = { .id_header = 0x11 }; + + t->cd.tps.partner_identity = old; + t->cd.update_status.partner_identity = old; + t->ctrl.status = TPS_STATUS_PLUG_PRESENT; + t->ctrl.pwr_status = FIELD_PREP(TPS_POWER_STATUS_TYPEC_CURRENT_MASK, + TYPEC_PWR_MODE_PD); + t->ctrl.identity.id_header = 0x22; + t->ctrl.failing_identity_reads = 1; + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_POWER_STATUS_UPDATE); + cd321x_test_interrupt(t); + KUNIT_ASSERT_EQ(test, t->ctrl.latched, 0); + + /* An incomplete read must not replace the identity in use... */ + KUNIT_EXPECT_EQ(test, t->cd.update_status.partner_identity.id_header, + old.id_header); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.update_work)); + /* ...but be completed by a fresh one. */ + cd321x_test_recover(test, t); + KUNIT_EXPECT_EQ(test, t->cd.update_status.partner_identity.id_header, 0x22U); +} + static void cd321x_failed_clear_recovers_test(struct kunit *test) { struct cd321x_test *t = cd321x_test_port(test); @@ -329,6 +373,7 @@ static struct kunit_case cd321x_irq_cases[] = { KUNIT_CASE(cd321x_failed_event_read_recovers_test), KUNIT_CASE(cd321x_failed_status_read_recovers_test), KUNIT_CASE(cd321x_failed_read_after_giving_up_test), + KUNIT_CASE(cd321x_failed_identity_read_recovers_test), KUNIT_CASE(cd321x_failed_clear_recovers_test), KUNIT_CASE(cd321x_event_reads_failing_for_long_test), KUNIT_CASE(cd321x_clear_writes_failing_for_long_test), diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 74dcec2a2030f..9799500cbabfd 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -1151,7 +1151,7 @@ static irqreturn_t cd321x_interrupt(int irq, void *data) if (TPS_POWER_STATUS_PWROPMODE(tps->pwr_status) == TYPEC_PWR_MODE_PD) { if (tps6598x_read_partner_identity(tps)) { dev_err(tps->dev, "failed to read partner identity\n"); - tps->partner_identity = (struct usb_pd_identity) {0}; + goto err_lost; } } } From eaa46403a4e4d3e7e7381b1c0558e8f468e84a9a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 21:09:11 +1100 Subject: [PATCH 1478/2680] usb: typec: tipd: sn201202x: Keep events unhandled while the controller sleeps sn201202x_suspend() puts the controller into SPMI sleep, and sn201202x_resume() wakes it again, but its event interrupt stays enabled for the whole time. An event latched around suspend is handled while the controller sleeps: during suspend until the interrupt is suspended, and during resume as soon as the interrupt is replayed after the noirq phase, before the wakeup command has been sent. The register select then times out and the event read fails. On the J700 this shows up after s2idle as "sn201202x 1-0c: cd321x_interrupt: failed to read events", logged during device resume, with no further rear-port event for the rest of the boot. Disable the event interrupt before the controller is put to sleep, and enable it again once the controller is awake, or when suspend fails. An interrupt arriving meanwhile is replayed when it is enabled, and the resume revalidation acknowledges anything latched during sleep. A resume whose wakeup fails still enables the interrupt, so that a later event can be handled once the controller answers. Polled controllers have no event interrupt and are unaffected. Fixes: 419cafb82d4c ("usb: typec: tipd: Add sn201202x support") (cherry picked from commit c2dd5d18206eb916b45ba66a6a17e856546abb2e) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/spmi.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/drivers/usb/typec/tipd/spmi.c b/drivers/usb/typec/tipd/spmi.c index 59d563aebfff5..9aa7c8e28e050 100644 --- a/drivers/usb/typec/tipd/spmi.c +++ b/drivers/usb/typec/tipd/spmi.c @@ -293,9 +293,12 @@ static int __maybe_unused sn201202x_resume(struct device *dev) int err; err = sn201202x_wake(sn); - if (err) - return err; - return tipd_resume(tps); + if (!err) + err = tipd_resume(tps); + /* The resume revalidation acknowledges events latched while asleep. */ + if (tps->irq) + enable_irq(tps->irq); + return err; } static int __maybe_unused sn201202x_suspend(struct device *dev) @@ -304,9 +307,16 @@ static int __maybe_unused sn201202x_suspend(struct device *dev) struct sn201202x *sn = tps_to_sn(tps); int err; + /* + * A sleeping controller cannot be read, so its events must not be + * handled until it is awake again. An interrupt arriving meanwhile is + * replayed when it is enabled. + */ + if (tps->irq) + disable_irq(tps->irq); err = tipd_suspend(tps); if (err) - return err; + goto out_enable; reinit_completion(&sn->sleep_completion); err = spmi_command_sleep(sn->sdev); if (err) @@ -320,6 +330,9 @@ static int __maybe_unused sn201202x_suspend(struct device *dev) out_resume: tipd_resume(tps); +out_enable: + if (tps->irq) + enable_irq(tps->irq); return err; } From a125ff7acafb580e0ee79522524a5bc34904c28d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 23:11:20 +1100 Subject: [PATCH 1479/2680] usb: typec: tipd: sn201202x: Wake the controller when suspend fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sn201202x_suspend() fails the suspend when the controller does not confirm the SLEEP command within 100 ms, and rolls back with tipd_resume(). The controller accepted the command, though, so it may well be asleep with only the confirmation missing. The PM core does not resume a device whose suspend failed, so nothing wakes the controller: the rollback reads its registers, its event interrupt is enabled again, and every event read then fails until the next system sleep. Send WAKEUP before rolling back once SLEEP was accepted. A failed wakeup is logged, and the suspend still fails with its own error. A SLEEP command that was rejected leaves the controller awake and needs no wakeup, as before. Failing the suspend, where the Asahi driver only logs the missing confirmation, is kept, so that the system does not sleep with the controller in an unknown state. Make the system sleep callbacks visible to KUnit and test them on a simulated SPMI bus: a sleep cycle keeps the event interrupt off while the controller sleeps, a failed wakeup at resume still enables it, a rejected SLEEP is not followed by a wakeup, and a SLEEP that is never confirmed is followed by a wakeup before the event interrupt is enabled again, also when that wakeup fails. Without this change the last two fail. [ryan: adapted to 7.2.10 — the PM test takes the SN201202x data from tipd_hw_variants[TIPD_VARIANT_SN201202X]: 7.2.10 keeps the variant data static behind that exported table instead of exporting tipd_sn201202x_data] Fixes: 38c14f6107df ("drm: apple: support ProMotion variable refresh rate") (cherry picked from commit 0a77048229afe1488764ebe47272cf5e36174e25) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/Kconfig | 11 + drivers/usb/typec/tipd/Makefile | 1 + drivers/usb/typec/tipd/sn201202x-pm-test.c | 240 +++++++++++++++++++++ drivers/usb/typec/tipd/spmi.c | 16 +- drivers/usb/typec/tipd/tps6598x.h | 2 + 5 files changed, 266 insertions(+), 4 deletions(-) create mode 100644 drivers/usb/typec/tipd/sn201202x-pm-test.c diff --git a/drivers/usb/typec/tipd/Kconfig b/drivers/usb/typec/tipd/Kconfig index 7c2527913111d..b8a904446109a 100644 --- a/drivers/usb/typec/tipd/Kconfig +++ b/drivers/usb/typec/tipd/Kconfig @@ -43,3 +43,14 @@ config TYPEC_TIPD_KUNIT_TEST Also test that a USB4/Thunderbolt router which never registers its switch does not keep the port from probing. These tests run without accessing a physical Type-C controller. + +config TYPEC_SN201202X_KUNIT_TEST + tristate "KUnit tests for SN201202x" if !KUNIT_ALL_TESTS + depends on KUNIT && TYPEC_SN201202X + select REGMAP + default KUNIT_ALL_TESTS + help + Test that the SN201202x events are not handled while the controller + sleeps, and that a system suspend that fails leaves the controller + awake. These tests run on a simulated SPMI bus, without accessing a + physical Type-C controller. diff --git a/drivers/usb/typec/tipd/Makefile b/drivers/usb/typec/tipd/Makefile index e2e3d9271dd26..fcb10573a05b1 100644 --- a/drivers/usb/typec/tipd/Makefile +++ b/drivers/usb/typec/tipd/Makefile @@ -14,3 +14,4 @@ sn201202x-y := spmi.o obj-$(CONFIG_TYPEC_TIPD_KUNIT_TEST) += cd321x-pm-test.o cd321x-tbt-test.o \ cd321x-irq-test.o +obj-$(CONFIG_TYPEC_SN201202X_KUNIT_TEST) += sn201202x-pm-test.o diff --git a/drivers/usb/typec/tipd/sn201202x-pm-test.c b/drivers/usb/typec/tipd/sn201202x-pm-test.c new file mode 100644 index 0000000000000..1082774dba56f --- /dev/null +++ b/drivers/usb/typec/tipd/sn201202x-pm-test.c @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for SN201202x system sleep + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "tps6598x.h" + +/* + * A controller on a simulated SPMI bus. It confirms SLEEP and WAKEUP with the + * interrupts the driver waits for, unless the SLEEP confirmation gets lost, + * and it does not answer register accesses while it is asleep. + */ +struct sn201202x_test { + struct sn201202x sn; + struct device *dev; + int irq; + bool asleep; + bool sleep_unconfirmed; + int sleep_error; + int wakeup_error; + unsigned int wakeups; + bool irq_enabled_at_wakeup; +}; + +static bool sn201202x_test_irq_enabled(struct sn201202x_test *t) +{ + return !irqd_irq_disabled(irq_get_irq_data(t->irq)); +} + +static int sn201202x_test_cmd(struct spmi_controller *ctrl, u8 opcode, u8 sid) +{ + struct sn201202x_test *t = spmi_controller_get_drvdata(ctrl); + + switch (opcode) { + case SPMI_CMD_SLEEP: + if (t->sleep_error) + return t->sleep_error; + t->asleep = true; + if (!t->sleep_unconfirmed) + complete(&t->sn.sleep_completion); + return 0; + case SPMI_CMD_WAKEUP: + t->wakeups++; + t->irq_enabled_at_wakeup = sn201202x_test_irq_enabled(t); + if (t->wakeup_error) + return t->wakeup_error; + t->asleep = false; + complete(&t->sn.wake_completion); + return 0; + } + + return -EOPNOTSUPP; +} + +static int sn201202x_test_read(void *context, const void *reg, size_t reg_size, + void *val, size_t val_size) +{ + struct sn201202x_test *t = context; + + /* A sleeping controller never confirms the register selection. */ + if (t->asleep) + return -ETIMEDOUT; + memset(val, 0, val_size); + if (*(const u8 *)reg == TPS_REG_MODE) + memcpy(val, "APP ", min(val_size, 4)); + + return 0; +} + +static int sn201202x_test_write(void *context, const void *data, size_t count) +{ + struct sn201202x_test *t = context; + + return t->asleep ? -ETIMEDOUT : 0; +} + +static const struct regmap_bus sn201202x_test_bus = { + .read = sn201202x_test_read, + .write = sn201202x_test_write, + .reg_format_endian_default = REGMAP_ENDIAN_NATIVE, + .val_format_endian_default = REGMAP_ENDIAN_NATIVE, +}; + +static irqreturn_t sn201202x_test_irq(int irq, void *data) +{ + return IRQ_HANDLED; +} + +static void sn201202x_test_put_device(void *data) +{ + spmi_device_put(data); +} + +static void sn201202x_test_free_irq_desc(void *data) +{ + irq_free_desc((unsigned long)data); +} + +static void sn201202x_test_free_irq(void *data) +{ + struct sn201202x_test *t = data; + + free_irq(t->irq, t); +} + +static struct sn201202x_test *sn201202x_test_port(struct kunit *test) +{ + struct spmi_controller *ctrl; + struct sn201202x_test *t; + struct tps6598x *tps; + + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t); + t->dev = kunit_device_register(test, "sn201202x-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->dev); + + ctrl = devm_spmi_controller_alloc(t->dev, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctrl); + spmi_controller_set_drvdata(ctrl, t); + ctrl->cmd = sn201202x_test_cmd; + t->sn.sdev = spmi_device_alloc(ctrl); + KUNIT_ASSERT_NOT_NULL(test, t->sn.sdev); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, sn201202x_test_put_device, + t->sn.sdev), 0); + init_completion(&t->sn.select_completion); + init_completion(&t->sn.sleep_completion); + init_completion(&t->sn.wake_completion); + + tps = &t->sn.cd.tps; + tps->dev = t->dev; + tps->data = tipd_hw_variants[TIPD_VARIANT_SN201202X]; + tps->regmap = devm_regmap_init(t->dev, &sn201202x_test_bus, t, + &tps6598x_regmap_config); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, tps->regmap); + dev_set_drvdata(t->dev, tps); + + /* The event interrupt, which is never raised. */ + t->irq = irq_alloc_desc(NUMA_NO_NODE); + KUNIT_ASSERT_GE(test, t->irq, 0); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, sn201202x_test_free_irq_desc, + (void *)(unsigned long)t->irq), 0); + irq_set_chip_and_handler(t->irq, &dummy_irq_chip, handle_simple_irq); + irq_clear_status_flags(t->irq, IRQ_NOREQUEST); + KUNIT_ASSERT_EQ(test, request_irq(t->irq, sn201202x_test_irq, 0, + "sn201202x-test", t), 0); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, sn201202x_test_free_irq, t), 0); + tps->irq = t->irq; + + return t; +} + +static void sn201202x_sleep_cycle_test(struct kunit *test) +{ + struct sn201202x_test *t = sn201202x_test_port(test); + + KUNIT_ASSERT_EQ(test, sn201202x_suspend(t->dev), 0); + KUNIT_EXPECT_TRUE(test, t->asleep); + /* No event may be handled while the controller cannot be read. */ + KUNIT_EXPECT_FALSE(test, sn201202x_test_irq_enabled(t)); + + KUNIT_EXPECT_EQ(test, sn201202x_resume(t->dev), 0); + KUNIT_EXPECT_EQ(test, t->wakeups, 1U); + KUNIT_EXPECT_FALSE(test, t->irq_enabled_at_wakeup); + KUNIT_EXPECT_TRUE(test, sn201202x_test_irq_enabled(t)); +} + +static void sn201202x_failed_wakeup_test(struct kunit *test) +{ + struct sn201202x_test *t = sn201202x_test_port(test); + + KUNIT_ASSERT_EQ(test, sn201202x_suspend(t->dev), 0); + t->wakeup_error = -EIO; + KUNIT_EXPECT_EQ(test, sn201202x_resume(t->dev), -EIO); + /* Events are handled again once the controller answers. */ + KUNIT_EXPECT_TRUE(test, sn201202x_test_irq_enabled(t)); +} + +static void sn201202x_rejected_sleep_test(struct kunit *test) +{ + struct sn201202x_test *t = sn201202x_test_port(test); + + t->sleep_error = -EIO; + KUNIT_EXPECT_EQ(test, sn201202x_suspend(t->dev), -EIO); + KUNIT_EXPECT_FALSE(test, t->asleep); + KUNIT_EXPECT_EQ(test, t->wakeups, 0U); + KUNIT_EXPECT_TRUE(test, sn201202x_test_irq_enabled(t)); +} + +static void sn201202x_unconfirmed_sleep_test(struct kunit *test) +{ + struct sn201202x_test *t = sn201202x_test_port(test); + + t->sleep_unconfirmed = true; + KUNIT_EXPECT_EQ(test, sn201202x_suspend(t->dev), -ETIMEDOUT); + /* No resume follows a failed suspend, so the rollback must wake it... */ + KUNIT_EXPECT_EQ(test, t->wakeups, 1U); + KUNIT_EXPECT_FALSE(test, t->asleep); + /* ...before its events are handled again. */ + KUNIT_EXPECT_FALSE(test, t->irq_enabled_at_wakeup); + KUNIT_EXPECT_TRUE(test, sn201202x_test_irq_enabled(t)); +} + +static void sn201202x_unconfirmed_sleep_failed_wakeup_test(struct kunit *test) +{ + struct sn201202x_test *t = sn201202x_test_port(test); + + t->sleep_unconfirmed = true; + t->wakeup_error = -EIO; + /* The suspend error is reported, not the one from the rollback. */ + KUNIT_EXPECT_EQ(test, sn201202x_suspend(t->dev), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, t->wakeups, 1U); + KUNIT_EXPECT_TRUE(test, sn201202x_test_irq_enabled(t)); +} + +static struct kunit_case sn201202x_pm_cases[] = { + KUNIT_CASE(sn201202x_sleep_cycle_test), + KUNIT_CASE(sn201202x_failed_wakeup_test), + KUNIT_CASE(sn201202x_rejected_sleep_test), + KUNIT_CASE(sn201202x_unconfirmed_sleep_test), + KUNIT_CASE(sn201202x_unconfirmed_sleep_failed_wakeup_test), + {} +}; + +static struct kunit_suite sn201202x_pm_suite = { + .name = "sn201202x-pm", + .test_cases = sn201202x_pm_cases, +}; + +kunit_test_suite(sn201202x_pm_suite); +MODULE_DESCRIPTION("KUnit tests for SN201202x system sleep"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/drivers/usb/typec/tipd/spmi.c b/drivers/usb/typec/tipd/spmi.c index 9aa7c8e28e050..d70a617d4084a 100644 --- a/drivers/usb/typec/tipd/spmi.c +++ b/drivers/usb/typec/tipd/spmi.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include #include @@ -286,7 +287,7 @@ static void sn201202x_remove(struct spmi_device *device) spmi_command_sleep(sn->sdev); } -static int __maybe_unused sn201202x_resume(struct device *dev) +VISIBLE_IF_KUNIT int __maybe_unused sn201202x_resume(struct device *dev) { struct tps6598x *tps = dev_get_drvdata(dev); struct sn201202x *sn = tps_to_sn(tps); @@ -300,12 +301,13 @@ static int __maybe_unused sn201202x_resume(struct device *dev) enable_irq(tps->irq); return err; } +EXPORT_SYMBOL_IF_KUNIT(sn201202x_resume); -static int __maybe_unused sn201202x_suspend(struct device *dev) +VISIBLE_IF_KUNIT int __maybe_unused sn201202x_suspend(struct device *dev) { struct tps6598x *tps = dev_get_drvdata(dev); struct sn201202x *sn = tps_to_sn(tps); - int err; + int err, ret; /* * A sleeping controller cannot be read, so its events must not be @@ -324,10 +326,15 @@ static int __maybe_unused sn201202x_suspend(struct device *dev) if (!sn->polled && !wait_for_completion_timeout(&sn->sleep_completion, msecs_to_jiffies(100))) { err = -ETIMEDOUT; - goto out_resume; + goto out_wake; } return 0; +out_wake: + /* It accepted SLEEP, so it may be asleep without having confirmed it. */ + ret = sn201202x_wake(sn); + if (ret) + dev_err(dev, "failed to wake up after failed suspend: %d\n", ret); out_resume: tipd_resume(tps); out_enable: @@ -335,6 +342,7 @@ static int __maybe_unused sn201202x_suspend(struct device *dev) enable_irq(tps->irq); return err; } +EXPORT_SYMBOL_IF_KUNIT(sn201202x_suspend); MODULE_DEVICE_TABLE(of, sn201202x_of_match); diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index fb7c4decd3018..e401989785cb4 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -467,6 +467,8 @@ void cd321x_init_work(struct cd321x *cd321x); bool cd321x_tbt_switch_keep_waiting(struct device *dev); struct typec_thunderbolt_switch * cd321x_tbt_switch_get(struct device *dev, struct fwnode_handle *fwnode); +int sn201202x_suspend(struct device *dev); +int sn201202x_resume(struct device *dev); #endif #endif /* __TPS6598X_H__ */ From 35bc4bbcd583f728096303cde5886fe4d92e1c19 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 13:33:57 +1100 Subject: [PATCH 1480/2680] usb: typec: tipd: cd321x: Don't race the recovery read in the IRQ test cd321x_test_recover() asserted that the recovery read was still pending before flushing it. The driver queues that read with no delay, so on an SMP test machine a worker can run it before the assertion, and the test fails although the driver behaved correctly. With four CPUs, cd321x_resume_acknowledges_events_test failed 1 time in 5. Check the read's effect instead. After the flush the port must accept updates again, which it only does once the recovery read has run. Removing the resume path's mod_delayed_work() still makes the test fail. Fixes: 7e97f2f5ee45 ("usb: typec: tipd: cd321x: Never leave interrupt events unacknowledged") (cherry picked from commit 2e36dd688f13ab22023fc9fcbf0ebf32dba61c8a) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/cd321x-irq-test.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/usb/typec/tipd/cd321x-irq-test.c b/drivers/usb/typec/tipd/cd321x-irq-test.c index d8744fc5a02f2..c14a0cdd16726 100644 --- a/drivers/usb/typec/tipd/cd321x-irq-test.c +++ b/drivers/usb/typec/tipd/cd321x-irq-test.c @@ -148,10 +148,13 @@ static irqreturn_t cd321x_test_interrupt(struct cd321x_test *t) return t->cd.tps.data->irq_handler(0, &t->cd.tps); } -/* The read of a snapshot that was scheduled to recover the events. */ +/* + * Run the snapshot read that was scheduled to recover the events. It is queued + * without delay, so on SMP a worker may already have run it: check its effect, + * not whether it is still pending. + */ static void cd321x_test_recover(struct kunit *test, struct cd321x_test *t) { - KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); flush_delayed_work(&t->cd.resume_work); scoped_guard(mutex, &t->cd.tps.lock) KUNIT_EXPECT_TRUE(test, cd321x_pm_can_update(&t->cd.pm)); From 9890d01e30117dfe197fb1dfbbaf211400e97af5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 10:34:12 +1100 Subject: [PATCH 1481/2680] usb: dwc3: apple: Arm the core interrupt for system wakeup in host mode USB devices cannot wake an Apple silicon machine from suspend-to-idle. The glue device is never made wakeup capable, so dwc3_core_probe() records sys_wakeup = false, dwc3_host_init() leaves the xHCI platform device with wakeup disabled, and xhci_suspend() clears the root port wake bits. Nothing arms the controller interrupt for wakeup either, so suspend_device_irqs() disables it like any other device interrupt. The core itself is already kept running across system sleep whenever it is up (it is a syscore device, see dwc3_apple_init()), so the controller is in a state to report port events. What is missing is the wakeup policy and the wake interrupt: - Initialise the glue device as a wakeup source, enabled by default as usb_hcd_pci_probe() does for PCI host controllers. This happens before the first dwc3_core_probe(), which then passes the setting on to the xHCI device as on other dwc3 platforms, so xhci_suspend() keeps the root port wake-on-connect/disconnect/overcurrent bits and the root hubs stay armed for remote wakeup. Userspace can still turn it off through the power/wakeup attribute of the glue device. - While the controller is in host mode, register the core interrupt, which xHCI shares, as the wake interrupt of the glue device, and drop it again when host mode is torn down. The PM core arms it in the noirq phase when wakeup is enabled. Device mode gets no wake interrupt: a gadget is resumed by its host, and arming bus activity there would only cause spurious wakeups. This needs the AIC to accept enable_irq_wake(), which "irqchip/apple-aic: Allow interrupts to wake the system" provides; before it, the interrupt could not be armed at all. Not yet verified on hardware: whether this DWC3 integration raises its interrupt for a port status change after xhci_suspend() has halted the controller (the xHCI specification leaves wake signalling from the halted state to the platform). If it does not, the wake interrupt is armed but never fires, behaviour is otherwise unchanged, and a follow-up has to find the platform wake path instead. (cherry picked from commit 13e221102941d2404d6c55254869e1328f4a8ddf) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/dwc3-apple.c | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index e1d1104db98a3..1e2ee195a82f8 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -13,6 +13,8 @@ #include #include #include +#include +#include #include #include @@ -78,7 +80,7 @@ * Type-C mux for the SuperSpeed lanes, and the PHY follows those mode changes on its own. * * Whenever the core is up, for either kind of port, it is kept running across system sleep, see - * dwc3_apple_init(). + * dwc3_apple_init(). In host mode its interrupt is then also armed as the wake interrupt. * * enum dwc3_apple_state is defined in dwc3-apple-state.h. */ @@ -87,6 +89,7 @@ * struct dwc3_apple - Apple-specific DWC3 USB controller * @dwc: Core DWC3 structure * @dev: Pointer to the device structure + * @irq: Core interrupt, shared with xHCI in host mode and used as its wake interrupt * @mmio_resource: Resource to be passed to dwc3_core_probe * @apple_regs: Apple-specific DWC3 registers * @reset: Reset control @@ -107,6 +110,7 @@ struct dwc3_apple { struct dwc3 dwc; struct device *dev; + int irq; struct resource *mmio_resource; void __iomem *apple_regs; @@ -438,6 +442,18 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st goto core_exit; } + /* + * xHCI shares the core interrupt and is suspended with its root port wake bits set when + * this device may wake the system (dwc3_host_init() passes that on). Arm the interrupt + * for wakeup while in host mode, so that a port event during suspend-to-idle ends it. + * Device mode has no wake source: a gadget is woken by its host. + */ + if (state == DWC3_APPLE_HOST) { + ret = dev_pm_set_wake_irq(appledwc->dev, appledwc->irq); + if (ret) + dev_warn(appledwc->dev, "Failed to set up the wake IRQ, err=%d\n", ret); + } + appledwc->state = state; /* * A connected cable leaves the Apple DWC3 core and its PHY as one live, @@ -474,6 +490,7 @@ static int dwc3_apple_exit(struct dwc3_apple *appledwc) appledwc->state = DWC3_APPLE_NO_CABLE; break; case DWC3_APPLE_HOST: + dev_pm_clear_wake_irq(appledwc->dev); dwc3_host_exit(&appledwc->dwc); appledwc->state = DWC3_APPLE_NO_CABLE; break; @@ -622,6 +639,18 @@ static int dwc3_apple_probe(struct platform_device *pdev) return dev_err_probe(dev, PTR_ERR(appledwc->apple_regs), "Failed to map Apple-specific MMIO\n"); + appledwc->irq = platform_get_irq(pdev, 0); + if (appledwc->irq < 0) + return appledwc->irq; + + /* + * Like a PCI xHCI controller, allow USB devices to wake the system by default. This has + * to be set before dwc3_core_probe(), which hands it on to the xHCI device. + */ + ret = devm_device_init_wakeup(dev); + if (ret) + return dev_err_probe(dev, ret, "Failed to set up wakeup\n"); + /* * On this platform, DWC3 can only be brought up after parts of the PHY have been * initialized with knowledge of the target mode and cable orientation from typec_set_mux. From 4d1d94cec1f5d9756fdd90f1f46cb266b95c3a65 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 12:06:26 +1100 Subject: [PATCH 1482/2680] usb: hub: keep the VL122 awake instead of ignoring its remote wakeup The VIA Labs VL122 USB 2.0 hub (2109:2122) on the J700 never signals a downstream connect while it is runtime suspended, so it was given USB_QUIRK_IGNORE_REMOTE_WAKEUP. That kept it out of runtime suspend, but it also leaves the hub without a wakeup setting at all, so it is never armed for remote wakeup in system sleep. Every USB 2.0 keyboard and mouse on the machine sits behind this hub, so none of them could wake it. The problem is runtime PM, not remote wakeup: mark the hub with HUB_QUIRK_DISABLE_AUTOSUSPEND, as is done for other hubs with broken runtime suspend, and drop the device quirk so its wakeup setting is available again for system sleep. (cherry picked from commit 8d33f6497da5a1cbe0d212bbe378ebade7e365f2) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/core/hub.c | 7 +++++++ drivers/usb/core/quirks.c | 4 ---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index b119f5b6065c4..b9372e13c978f 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -48,6 +48,8 @@ #define USB_PRODUCT_USB5534B 0x5534 #define USB_VENDOR_CYPRESS 0x04b4 #define USB_PRODUCT_CY7C65632 0x6570 +#define USB_VENDOR_VIA_LABS 0x2109 +#define USB_PRODUCT_VL122 0x2122 #define USB_VENDOR_TEXAS_INSTRUMENTS 0x0451 #define USB_PRODUCT_TUSB8041_USB3 0x8140 #define USB_PRODUCT_TUSB8041_USB2 0x8142 @@ -6043,6 +6045,11 @@ static const struct usb_device_id hub_id_table[] = { | USB_DEVICE_ID_MATCH_PRODUCT, .idVendor = USB_VENDOR_TEXAS_INSTRUMENTS, .idProduct = USB_PRODUCT_TUSB8041_USB3, + .driver_info = HUB_QUIRK_DISABLE_AUTOSUSPEND}, + { .match_flags = USB_DEVICE_ID_MATCH_VENDOR + | USB_DEVICE_ID_MATCH_PRODUCT, + .idVendor = USB_VENDOR_VIA_LABS, + .idProduct = USB_PRODUCT_VL122, .driver_info = HUB_QUIRK_DISABLE_AUTOSUSPEND}, { .match_flags = USB_DEVICE_ID_MATCH_VENDOR | USB_DEVICE_ID_MATCH_PRODUCT, diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c index ae941ec487e44..b5b577f0b931c 100644 --- a/drivers/usb/core/quirks.c +++ b/drivers/usb/core/quirks.c @@ -583,10 +583,6 @@ static const struct usb_device_id usb_quirk_list[] = { /* VLI disk */ { USB_DEVICE(0x2109, 0x0711), .driver_info = USB_QUIRK_NO_LPM }, - /* VIA Labs, Inc. VL122 USB2.0 Hub: claims remote wakeup, never sends it */ - { USB_DEVICE(0x2109, 0x2122), .driver_info = - USB_QUIRK_IGNORE_REMOTE_WAKEUP }, - /* VIA Labs, Inc. USB2.0 Hub */ { USB_DEVICE(0x2109, 0x2817), .driver_info = USB_QUIRK_NO_LPM }, From 30bb9b1b2ce3c310cfc604d6eb44770d72ee05eb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 12:44:20 +1100 Subject: [PATCH 1483/2680] usb: xhci: plat: Allow the controller to keep running in suspend-to-idle xhci_suspend() halts the controller (clears USBCMD.R/S) and saves its state. A halted xHC writes no events to its event ring, so it raises no interrupt either: wake events from a halted controller are a separate signal, which on PCI is PME# and on other platforms a PHY or glue wake interrupt (dwc3-qcom, dwc3-imx8mp, xhci-mtk, xhci-tegra all have one). A platform without such a signal cannot be woken by USB devices from system sleep at all, even when it keeps the controller powered and its interrupt armed for wakeup. For suspend-to-idle there is a simple way out: do not halt the controller. The root hubs are still bus suspended with their wake bits set, so the ports are in U3 and downstream devices are armed for remote wakeup, but a remote wakeup or port change now raises a Port Status Change Event and with it the interrupt, which wakes the system once the platform arms it with enable_irq_wake(). Add a property, "xhci-run-in-s2idle", for glue drivers to request this. xhci-plat then skips xhci_suspend() and xhci_resume() across a system suspend when the target state is suspend-to-idle and the controller may wake the system with a wakeup source below it, the same way it already leaves the controller alone while a sideband instance is active. In all other cases, including hibernation and when wakeup is disabled through power/wakeup, the controller is halted as before. Event handling needs nothing new: until the interrupt is suspended an event goes through xhci_irq(), whose usb_hcd_resume_root_hub() reports a wakeup event and aborts the suspend; once it is armed for wakeup, the first interrupt ends suspend-to-idle and is handled when device interrupts are resumed. (cherry picked from commit 4d9fbe0bed99fff546a1ccd4221a9be20fa15b3d) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/host/xhci-plat.c | 37 ++++++++++++++++++++++++++++++++++++ drivers/usb/host/xhci-plat.h | 1 + drivers/usb/host/xhci.h | 1 + 3 files changed, 39 insertions(+) diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index 074d9c731639f..7f874e848f954 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "xhci.h" @@ -277,6 +278,9 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s if (device_property_read_bool(tmpdev, "xhci-skip-phy-init-quirk")) xhci->quirks |= XHCI_SKIP_PHY_INIT; + if (device_property_read_bool(tmpdev, "xhci-run-in-s2idle")) + xhci->quirks |= XHCI_RUN_IN_S2IDLE; + device_property_read_u32(tmpdev, "imod-interval-ns", &xhci->imod_interval); device_property_read_u16(tmpdev, "num-hc-interrupters", @@ -494,6 +498,24 @@ static int xhci_plat_suspend_common(struct device *dev) return 0; } +/* + * Some platforms have no wake signal for a halted xHC: a halted controller writes no events, so + * its interrupt cannot fire, and on PCI the remote wakeup or port change would be signalled as a + * PME instead. Where the platform keeps the controller powered and clocked in suspend-to-idle, + * leave it running instead, with the root hubs suspended and armed for wakeup as usual. A remote + * wakeup or port change then raises a Port Status Change Event, and the interrupt, armed for + * wakeup by the platform, ends suspend-to-idle. + */ +static bool xhci_plat_run_in_s2idle(struct device *dev) +{ + struct usb_hcd *hcd = dev_get_drvdata(dev); + struct xhci_hcd *xhci = hcd_to_xhci(hcd); + + return (xhci->quirks & XHCI_RUN_IN_S2IDLE) && + pm_suspend_target_state == PM_SUSPEND_TO_IDLE && + device_may_wakeup(dev) && device_wakeup_path(dev); +} + static int xhci_plat_suspend(struct device *dev) { struct usb_hcd *hcd = dev_get_drvdata(dev); @@ -505,6 +527,15 @@ static int xhci_plat_suspend(struct device *dev) return 0; } + if (xhci_plat_run_in_s2idle(dev)) { + /* A runtime suspended controller is halted, start it again. */ + if (pm_runtime_suspended(dev)) + pm_runtime_resume(dev); + priv->running_in_s2idle = 1; + dev_dbg(dev, "wakeup enabled, keep running in suspend-to-idle.\n"); + return 0; + } + return xhci_plat_suspend_common(dev); } @@ -570,6 +601,12 @@ static int xhci_plat_resume(struct device *dev) return 0; } + if (priv->running_in_s2idle) { + priv->running_in_s2idle = 0; + dev_dbg(dev, "kept running in suspend-to-idle, skip resume.\n"); + return 0; + } + return xhci_plat_resume_common(dev, false); } diff --git a/drivers/usb/host/xhci-plat.h b/drivers/usb/host/xhci-plat.h index 00751d8518310..eaaa9bc3e843f 100644 --- a/drivers/usb/host/xhci-plat.h +++ b/drivers/usb/host/xhci-plat.h @@ -17,6 +17,7 @@ struct xhci_plat_priv { unsigned long long quirks; bool power_lost; unsigned sideband_at_suspend:1; + unsigned running_in_s2idle:1; void (*plat_start)(struct usb_hcd *); int (*init_quirk)(struct usb_hcd *); int (*suspend_quirk)(struct usb_hcd *); diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index b38327abce172..dfae388d0f660 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -1649,6 +1649,7 @@ struct xhci_hcd { #define XHCI_ETRON_HOST BIT_ULL(49) #define XHCI_LIMIT_ENDPOINT_INTERVAL_9 BIT_ULL(50) #define XHCI_ASMEDIA_FW_QUIRK BIT_ULL(51) +#define XHCI_RUN_IN_S2IDLE BIT_ULL(52) unsigned int num_active_eps; unsigned int limit_active_eps; From 25fa4fbae182bf21ca987ca622fcc9cbe1879560 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 12:44:20 +1100 Subject: [PATCH 1484/2680] usb: dwc3: Let glue drivers keep xHCI running in suspend-to-idle Add a dwc3 software property for glue drivers whose controller stays powered in suspend-to-idle but has no wake signal besides the core interrupt, and pass it on to the xHCI platform device as "xhci-run-in-s2idle". The property array now holds up to seven entries, so make room for them and for the terminating empty entry. (cherry picked from commit f52428867a055b05c8b20e172e471b5d19221b54) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/core.c | 3 +++ drivers/usb/dwc3/core.h | 3 +++ drivers/usb/dwc3/glue.h | 4 ++++ drivers/usb/dwc3/host.c | 5 ++++- 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index 86e9cdf935c0d..2380cdfc9a2c3 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -1701,6 +1701,9 @@ static void dwc3_get_software_properties(struct dwc3 *dwc, if (properties->needs_full_reinit) dwc->needs_full_reinit = true; + if (properties->xhci_run_in_s2idle) + dwc->xhci_run_in_s2idle = true; + dwc->gsbuscfg0_reqinfo = DWC3_GSBUSCFG0_REQINFO_UNSPECIFIED; if (properties->gsbuscfg0_reqinfo != diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h index bb58bf290d5a8..c611938aab86e 100644 --- a/drivers/usb/dwc3/core.h +++ b/drivers/usb/dwc3/core.h @@ -1134,6 +1134,8 @@ struct dwc3_glue_ops { * @usb2_gadget_lpm_disable: set to disable usb2 lpm for gadget * @needs_full_reinit: set to indicate the core may lose power and need full * initialization during system pm + * @xhci_run_in_s2idle: set to keep xHCI running in suspend-to-idle when it may + * wake the system * @disable_scramble_quirk: set if we enable the disable scramble quirk * @u2exit_lfps_quirk: set if we enable u2exit lfps quirk * @u2ss_inp3_quirk: set if we enable P3 OK for U2/SS Inactive quirk @@ -1398,6 +1400,7 @@ struct dwc3 { unsigned usb2_lpm_disable:1; unsigned usb2_gadget_lpm_disable:1; unsigned needs_full_reinit:1; + unsigned xhci_run_in_s2idle:1; unsigned disable_scramble_quirk:1; unsigned u2exit_lfps_quirk:1; diff --git a/drivers/usb/dwc3/glue.h b/drivers/usb/dwc3/glue.h index db5f32df712bf..0737f533a8cd0 100644 --- a/drivers/usb/dwc3/glue.h +++ b/drivers/usb/dwc3/glue.h @@ -14,10 +14,14 @@ * @gsbuscfg0_reqinfo: Value to be programmed in the GSBUSCFG0.REQINFO field * @needs_full_reinit: indicate the controller may not remain power during system * pm and need full initialization + * @xhci_run_in_s2idle: the controller stays powered in suspend-to-idle and has no + * wake signal of its own, so xHCI has to keep running for + * USB devices to wake the system */ struct dwc3_properties { u32 gsbuscfg0_reqinfo; unsigned needs_full_reinit:1; + unsigned xhci_run_in_s2idle:1; }; #define DWC3_DEFAULT_PROPERTIES ((struct dwc3_properties){ \ diff --git a/drivers/usb/dwc3/host.c b/drivers/usb/dwc3/host.c index 96b588bd08cdc..8cbefa118b8a2 100644 --- a/drivers/usb/dwc3/host.c +++ b/drivers/usb/dwc3/host.c @@ -129,7 +129,7 @@ static int dwc3_host_get_irq(struct dwc3 *dwc) int dwc3_host_init(struct dwc3 *dwc) { - struct property_entry props[6]; + struct property_entry props[8]; struct platform_device *xhci; int ret, irq; int prop_idx = 0; @@ -188,6 +188,9 @@ int dwc3_host_init(struct dwc3 *dwc) props[prop_idx++] = PROPERTY_ENTRY_U16("num-hc-interrupters", dwc->num_hc_interrupters); + if (dwc->xhci_run_in_s2idle) + props[prop_idx++] = PROPERTY_ENTRY_BOOL("xhci-run-in-s2idle"); + if (prop_idx) { ret = device_create_managed_software_node(&xhci->dev, props, NULL); if (ret) { From 1a638f680b7b315fe33dc2717e3b412753f7717a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 12:44:21 +1100 Subject: [PATCH 1485/2680] usb: dwc3: apple: Keep xHCI running in suspend-to-idle With the core interrupt armed for wakeup, USB devices still cannot wake the J700 (T8140) from suspend-to-idle. Tested with a USB keyboard behind the on-board VL122 hub and an external hub, with the whole chain from the root hub to the keyboard set to power/wakeup=enabled: a keypress lights the keyboard up again, so it signals remote wakeup and its hubs pass it on, but the xHCI interrupt never fires during the sleep. The reason is that xhci_suspend() halts the controller. A halted xHC writes no events and raises no interrupt; a remote wakeup is then only visible as a platform wake signal, and this controller has none: the DWC3 node has a single interrupt, and the Type-C PHY has none. The core and its power domain are kept up across system sleep anyway (it is a syscore device while it is up), so ask xHCI to keep running in suspend-to-idle when it may wake the system. The remote wakeup then raises a Port Status Change Event on the running controller, and the core interrupt, already armed as the wake interrupt in host mode, ends suspend-to-idle. Not yet verified on hardware with this change. (cherry picked from commit 825856a1ff4c8e8432ab3a26b3d616f11306f3f3) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/dwc3-apple.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index 1e2ee195a82f8..b24789b574561 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -80,7 +80,8 @@ * Type-C mux for the SuperSpeed lanes, and the PHY follows those mode changes on its own. * * Whenever the core is up, for either kind of port, it is kept running across system sleep, see - * dwc3_apple_init(). In host mode its interrupt is then also armed as the wake interrupt. + * dwc3_apple_init(). In host mode its interrupt is then also armed as the wake interrupt, and + * xHCI keeps running in suspend-to-idle so that it can raise it. * * enum dwc3_apple_state is defined in dwc3-apple-state.h. */ @@ -212,6 +213,11 @@ static int dwc3_apple_core_probe(struct dwc3_apple *appledwc) probe_data.ignore_clocks_and_resets = true; probe_data.skip_core_init_mode = true; probe_data.properties = DWC3_DEFAULT_PROPERTIES; + /* + * There is no wake interrupt besides the core interrupt, and a halted xHC raises none, + * see dwc3_apple_init(). + */ + probe_data.properties.xhci_run_in_s2idle = true; /* * This runs from the role switch callback, so a failure leaves the glue @@ -443,9 +449,12 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st } /* - * xHCI shares the core interrupt and is suspended with its root port wake bits set when - * this device may wake the system (dwc3_host_init() passes that on). Arm the interrupt - * for wakeup while in host mode, so that a port event during suspend-to-idle ends it. + * xHCI shares the core interrupt and suspends its root hubs with their wake bits set when + * this device may wake the system (dwc3_host_init() passes that on). The controller has no + * separate wake signal, and a halted xHC writes no events and so raises no interrupt, so + * xHCI is told to keep running in suspend-to-idle instead of halting (the core stays + * powered, see below). Arm the interrupt for wakeup while in host mode, so that the Port + * Status Change Event of a remote wakeup or port change ends suspend-to-idle. * Device mode has no wake source: a gadget is woken by its host. */ if (state == DWC3_APPLE_HOST) { From 94b2abbaae62b3c7079a5b0608bf0743b5317445 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:26:15 +1100 Subject: [PATCH 1486/2680] usb: xhci: plat: Add KUnit tests for property quirks and suspend-to-idle Move the mapping of boolean device properties to quirks, and the decision whether to keep the controller running in suspend-to-idle, into helpers that do not need a probed controller, and test them: every property maps to its quirk, "xhci-run-in-s2idle" included, and the controller keeps running only for suspend-to-idle with the quirk set, wakeup enabled and a wakeup source below it. No functional change. (cherry picked from commit dbc72dfdaf1a65b80c8555ee48c01c3374e56f93) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/host/Kconfig | 12 +++ drivers/usb/host/Makefile | 1 + drivers/usb/host/xhci-plat-test.c | 117 ++++++++++++++++++++++++++++++ drivers/usb/host/xhci-plat.c | 72 +++++++++++------- drivers/usb/host/xhci-plat.h | 8 ++ 5 files changed, 183 insertions(+), 27 deletions(-) create mode 100644 drivers/usb/host/xhci-plat-test.c diff --git a/drivers/usb/host/Kconfig b/drivers/usb/host/Kconfig index b0b8b33c9969a..812fd8ce5f9e9 100644 --- a/drivers/usb/host/Kconfig +++ b/drivers/usb/host/Kconfig @@ -76,6 +76,18 @@ config USB_XHCI_PLATFORM If unsure, say N. +config USB_XHCI_PLATFORM_KUNIT_TEST + tristate "KUnit tests for the generic xHCI platform driver" if !KUNIT_ALL_TESTS + depends on USB_XHCI_PLATFORM && KUNIT + default KUNIT_ALL_TESTS + help + Test how the generic xHCI platform driver turns device properties + into quirks, and when it keeps the controller running across + suspend-to-idle instead of halting it. The tests need no xHCI + hardware. + + If unsure, say N. + config USB_XHCI_HISTB tristate "xHCI support for HiSilicon STB SoCs" depends on USB_XHCI_PLATFORM && (ARCH_HISI || COMPILE_TEST) diff --git a/drivers/usb/host/Makefile b/drivers/usb/host/Makefile index 3d0043e37d96b..a7872f781b7be 100644 --- a/drivers/usb/host/Makefile +++ b/drivers/usb/host/Makefile @@ -76,6 +76,7 @@ xhci-pci-$(CONFIG_USB_XHCI_PCI_ASMEDIA) += xhci-pci-asmedia.o obj-$(CONFIG_USB_XHCI_PCI_PROM21) += xhci-pci-prom21.o obj-$(CONFIG_USB_XHCI_PCI_RENESAS) += xhci-pci-renesas.o obj-$(CONFIG_USB_XHCI_PLATFORM) += xhci-plat-hcd.o +obj-$(CONFIG_USB_XHCI_PLATFORM_KUNIT_TEST) += xhci-plat-test.o obj-$(CONFIG_USB_XHCI_HISTB) += xhci-histb.o obj-$(CONFIG_USB_XHCI_RCAR) += xhci-rcar-hcd.o obj-$(CONFIG_USB_XHCI_MTK) += xhci-mtk-hcd.o diff --git a/drivers/usb/host/xhci-plat-test.c b/drivers/usb/host/xhci-plat-test.c new file mode 100644 index 0000000000000..410cf99da08d6 --- /dev/null +++ b/drivers/usb/host/xhci-plat-test.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit tests for the generic xHCI platform driver + */ + +#include +#include +#include +#include +#include +#include + +#include "xhci.h" +#include "xhci-plat.h" + +/* Only suspend-to-idle with the quirk, wakeup enabled and a wakeup path keeps it running. */ +static void xhci_plat_keep_running_test(struct kunit *test) +{ + static const suspend_state_t targets[] = { + PM_SUSPEND_ON, PM_SUSPEND_TO_IDLE, PM_SUSPEND_STANDBY, PM_SUSPEND_MEM, + }; + unsigned int i, combo; + + for (i = 0; i < ARRAY_SIZE(targets); i++) { + for (combo = 0; combo < 8; combo++) { + u64 quirks = (combo & 1) ? XHCI_RUN_IN_S2IDLE : 0; + bool may_wakeup = combo & 2, wakeup_path = combo & 4; + bool expected = combo == 7 && targets[i] == PM_SUSPEND_TO_IDLE; + + bool kept = xhci_plat_keep_running(quirks, targets[i], may_wakeup, + wakeup_path); + + KUNIT_EXPECT_EQ_MSG(test, kept, expected, + "target %d quirk %d may_wakeup %d wakeup_path %d", + targets[i], combo & 1, may_wakeup, wakeup_path); + } + } +} + +/* Other quirks do not stand in for the suspend-to-idle one. */ +static void xhci_plat_keep_running_other_quirks_test(struct kunit *test) +{ + KUNIT_EXPECT_FALSE(test, xhci_plat_keep_running(~XHCI_RUN_IN_S2IDLE, PM_SUSPEND_TO_IDLE, + true, true)); + KUNIT_EXPECT_TRUE(test, xhci_plat_keep_running(~0ULL, PM_SUSPEND_TO_IDLE, true, true)); +} + +static u64 xhci_plat_test_quirks(struct kunit *test, const struct property_entry *props) +{ + struct device *dev; + + dev = kunit_device_register(test, "xhci-plat-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + KUNIT_ASSERT_EQ(test, device_create_managed_software_node(dev, props, NULL), 0); + + return xhci_plat_property_quirks(dev); +} + +/* A device without properties gets no quirks. */ +static void xhci_plat_no_property_quirks_test(struct kunit *test) +{ + static const struct property_entry props[] = { {} }; + + KUNIT_EXPECT_EQ(test, xhci_plat_test_quirks(test, props), 0ULL); +} + +/* The property a glue driver sets for suspend-to-idle maps to its quirk, and only to it. */ +static void xhci_plat_run_in_s2idle_property_test(struct kunit *test) +{ + static const struct property_entry props[] = { + PROPERTY_ENTRY_BOOL("xhci-run-in-s2idle"), + {} + }; + + KUNIT_EXPECT_EQ(test, xhci_plat_test_quirks(test, props), XHCI_RUN_IN_S2IDLE); +} + +/* Every boolean property maps to its quirk. */ +static void xhci_plat_all_property_quirks_test(struct kunit *test) +{ + static const struct property_entry props[] = { + PROPERTY_ENTRY_BOOL("usb2-lpm-disable"), + PROPERTY_ENTRY_BOOL("usb3-lpm-capable"), + PROPERTY_ENTRY_BOOL("quirk-broken-port-ped"), + PROPERTY_ENTRY_BOOL("xhci-sg-trb-cache-size-quirk"), + PROPERTY_ENTRY_BOOL("write-64-hi-lo-quirk"), + PROPERTY_ENTRY_BOOL("xhci-missing-cas-quirk"), + PROPERTY_ENTRY_BOOL("xhci-skip-phy-init-quirk"), + PROPERTY_ENTRY_BOOL("xhci-run-in-s2idle"), + {} + }; + + KUNIT_EXPECT_EQ(test, xhci_plat_test_quirks(test, props), + XHCI_HW_LPM_DISABLE | XHCI_LPM_SUPPORT | XHCI_BROKEN_PORT_PED | + XHCI_SG_TRB_CACHE_SIZE_QUIRK | XHCI_WRITE_64_HI_LO | XHCI_MISSING_CAS | + XHCI_SKIP_PHY_INIT | XHCI_RUN_IN_S2IDLE); +} + +static struct kunit_case xhci_plat_test_cases[] = { + KUNIT_CASE(xhci_plat_keep_running_test), + KUNIT_CASE(xhci_plat_keep_running_other_quirks_test), + KUNIT_CASE(xhci_plat_no_property_quirks_test), + KUNIT_CASE(xhci_plat_run_in_s2idle_property_test), + KUNIT_CASE(xhci_plat_all_property_quirks_test), + {} +}; + +static struct kunit_suite xhci_plat_test_suite = { + .name = "xhci_plat", + .test_cases = xhci_plat_test_cases, +}; + +kunit_test_suite(xhci_plat_test_suite); + +MODULE_DESCRIPTION("KUnit tests for the generic xHCI platform driver"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index 7f874e848f954..8a4413f817e72 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -8,6 +8,7 @@ * A lot of code borrowed from the Linux xHCI driver. */ +#include #include #include #include @@ -154,6 +155,39 @@ static const struct of_device_id usb_xhci_of_match[] = { MODULE_DEVICE_TABLE(of, usb_xhci_of_match); #endif +/* The quirks that the boolean properties of a device, or of one of its parents, ask for */ +VISIBLE_IF_KUNIT u64 xhci_plat_property_quirks(struct device *dev) +{ + u64 quirks = 0; + + if (device_property_read_bool(dev, "usb2-lpm-disable")) + quirks |= XHCI_HW_LPM_DISABLE; + + if (device_property_read_bool(dev, "usb3-lpm-capable")) + quirks |= XHCI_LPM_SUPPORT; + + if (device_property_read_bool(dev, "quirk-broken-port-ped")) + quirks |= XHCI_BROKEN_PORT_PED; + + if (device_property_read_bool(dev, "xhci-sg-trb-cache-size-quirk")) + quirks |= XHCI_SG_TRB_CACHE_SIZE_QUIRK; + + if (device_property_read_bool(dev, "write-64-hi-lo-quirk")) + quirks |= XHCI_WRITE_64_HI_LO; + + if (device_property_read_bool(dev, "xhci-missing-cas-quirk")) + quirks |= XHCI_MISSING_CAS; + + if (device_property_read_bool(dev, "xhci-skip-phy-init-quirk")) + quirks |= XHCI_SKIP_PHY_INIT; + + if (device_property_read_bool(dev, "xhci-run-in-s2idle")) + quirks |= XHCI_RUN_IN_S2IDLE; + + return quirks; +} +EXPORT_SYMBOL_IF_KUNIT(xhci_plat_property_quirks); + int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const struct xhci_plat_priv *priv_match) { const struct hc_driver *driver; @@ -256,30 +290,7 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s /* Iterate over all parent nodes for finding quirks */ for (tmpdev = &pdev->dev; tmpdev; tmpdev = tmpdev->parent) { - - if (device_property_read_bool(tmpdev, "usb2-lpm-disable")) - xhci->quirks |= XHCI_HW_LPM_DISABLE; - - if (device_property_read_bool(tmpdev, "usb3-lpm-capable")) - xhci->quirks |= XHCI_LPM_SUPPORT; - - if (device_property_read_bool(tmpdev, "quirk-broken-port-ped")) - xhci->quirks |= XHCI_BROKEN_PORT_PED; - - if (device_property_read_bool(tmpdev, "xhci-sg-trb-cache-size-quirk")) - xhci->quirks |= XHCI_SG_TRB_CACHE_SIZE_QUIRK; - - if (device_property_read_bool(tmpdev, "write-64-hi-lo-quirk")) - xhci->quirks |= XHCI_WRITE_64_HI_LO; - - if (device_property_read_bool(tmpdev, "xhci-missing-cas-quirk")) - xhci->quirks |= XHCI_MISSING_CAS; - - if (device_property_read_bool(tmpdev, "xhci-skip-phy-init-quirk")) - xhci->quirks |= XHCI_SKIP_PHY_INIT; - - if (device_property_read_bool(tmpdev, "xhci-run-in-s2idle")) - xhci->quirks |= XHCI_RUN_IN_S2IDLE; + xhci->quirks |= xhci_plat_property_quirks(tmpdev); device_property_read_u32(tmpdev, "imod-interval-ns", &xhci->imod_interval); @@ -506,14 +517,21 @@ static int xhci_plat_suspend_common(struct device *dev) * wakeup or port change then raises a Port Status Change Event, and the interrupt, armed for * wakeup by the platform, ends suspend-to-idle. */ +VISIBLE_IF_KUNIT bool xhci_plat_keep_running(u64 quirks, suspend_state_t target, + bool may_wakeup, bool wakeup_path) +{ + return (quirks & XHCI_RUN_IN_S2IDLE) && target == PM_SUSPEND_TO_IDLE && + may_wakeup && wakeup_path; +} +EXPORT_SYMBOL_IF_KUNIT(xhci_plat_keep_running); + static bool xhci_plat_run_in_s2idle(struct device *dev) { struct usb_hcd *hcd = dev_get_drvdata(dev); struct xhci_hcd *xhci = hcd_to_xhci(hcd); - return (xhci->quirks & XHCI_RUN_IN_S2IDLE) && - pm_suspend_target_state == PM_SUSPEND_TO_IDLE && - device_may_wakeup(dev) && device_wakeup_path(dev); + return xhci_plat_keep_running(xhci->quirks, pm_suspend_target_state, + device_may_wakeup(dev), device_wakeup_path(dev)); } static int xhci_plat_suspend(struct device *dev) diff --git a/drivers/usb/host/xhci-plat.h b/drivers/usb/host/xhci-plat.h index eaaa9bc3e843f..5a43502ca27a9 100644 --- a/drivers/usb/host/xhci-plat.h +++ b/drivers/usb/host/xhci-plat.h @@ -34,4 +34,12 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, void xhci_plat_remove(struct platform_device *dev); extern const struct dev_pm_ops xhci_plat_pm_ops; +#if IS_ENABLED(CONFIG_KUNIT) +#include + +u64 xhci_plat_property_quirks(struct device *dev); +bool xhci_plat_keep_running(u64 quirks, suspend_state_t target, bool may_wakeup, + bool wakeup_path); +#endif + #endif /* _XHCI_PLAT_H */ From c87cbecccf89509982a425116f69c20a6936cf1b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:26:16 +1100 Subject: [PATCH 1487/2680] usb: dwc3: Add KUnit tests for the xHCI device properties Move the filling of the property array for the xHCI platform device into dwc3_host_xhci_properties() and test it: "xhci-run-in-s2idle" is passed on only when the glue asks for it, and with every optional property set the array still ends in an empty entry. No functional change. (cherry picked from commit 0612d22d1796d962799c67af3b5da1fca9a32544) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/Kconfig | 10 +++ drivers/usb/dwc3/Makefile | 1 + drivers/usb/dwc3/core.h | 6 ++ drivers/usb/dwc3/dwc3-host-test.c | 115 ++++++++++++++++++++++++++++++ drivers/usb/dwc3/host.c | 79 ++++++++++++-------- 5 files changed, 181 insertions(+), 30 deletions(-) create mode 100644 drivers/usb/dwc3/dwc3-host-test.c diff --git a/drivers/usb/dwc3/Kconfig b/drivers/usb/dwc3/Kconfig index 3e033ddc86d3d..c862e55ae3a37 100644 --- a/drivers/usb/dwc3/Kconfig +++ b/drivers/usb/dwc3/Kconfig @@ -51,6 +51,16 @@ config USB_DWC3_DUAL_ROLE endchoice +config USB_DWC3_HOST_KUNIT_TEST + tristate "KUnit tests for the DWC3 host glue" if !KUNIT_ALL_TESTS + depends on KUNIT && (USB_DWC3_HOST || USB_DWC3_DUAL_ROLE) + default KUNIT_ALL_TESTS + help + Test the properties with which the DWC3 driver configures its + xHCI platform device. The tests need no DWC3 hardware. + + If unsure, say N. + comment "Platform Glue Driver Support" config USB_DWC3_OMAP diff --git a/drivers/usb/dwc3/Makefile b/drivers/usb/dwc3/Makefile index 76578fc4a5da4..57e0475016b6c 100644 --- a/drivers/usb/dwc3/Makefile +++ b/drivers/usb/dwc3/Makefile @@ -44,6 +44,7 @@ endif obj-$(CONFIG_USB_DWC3_AM62) += dwc3-am62.o obj-$(CONFIG_USB_DWC3_APPLE) += dwc3-apple.o +obj-$(CONFIG_USB_DWC3_HOST_KUNIT_TEST) += dwc3-host-test.o obj-$(CONFIG_USB_DWC3_APPLE_KUNIT_TEST) += dwc3-apple-state-test.o obj-$(CONFIG_USB_DWC3_OMAP) += dwc3-omap.o obj-$(CONFIG_USB_DWC3_EXYNOS) += dwc3-exynos.o diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h index c611938aab86e..150af8cad6e0b 100644 --- a/drivers/usb/dwc3/core.h +++ b/drivers/usb/dwc3/core.h @@ -1675,6 +1675,12 @@ static inline void dwc3_pre_run_stop(struct dwc3 *dwc, bool is_on) #if IS_ENABLED(CONFIG_USB_DWC3_HOST) || IS_ENABLED(CONFIG_USB_DWC3_DUAL_ROLE) int dwc3_host_init(struct dwc3 *dwc); void dwc3_host_exit(struct dwc3 *dwc); +/* Most properties dwc3_host_xhci_properties() fills in, not counting the empty entry */ +#define DWC3_XHCI_MAX_PROPERTIES 7 +#if IS_ENABLED(CONFIG_KUNIT) +struct property_entry; +unsigned int dwc3_host_xhci_properties(const struct dwc3 *dwc, struct property_entry *props); +#endif #else static inline int dwc3_host_init(struct dwc3 *dwc) { return 0; } diff --git a/drivers/usb/dwc3/dwc3-host-test.c b/drivers/usb/dwc3/dwc3-host-test.c new file mode 100644 index 0000000000000..6c50ea82bbff9 --- /dev/null +++ b/drivers/usb/dwc3/dwc3-host-test.c @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit tests for the properties dwc3 gives its xHCI device + */ + +#include +#include +#include +#include +#include +#include + +#include "core.h" + +struct dwc3_host_test { + struct dwc3 *dwc; + struct property_entry props[DWC3_XHCI_MAX_PROPERTIES + 1]; +}; + +static int dwc3_host_test_init(struct kunit *test) +{ + struct dwc3_host_test *t; + + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t); + t->dwc = kunit_kzalloc(test, sizeof(*t->dwc), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t->dwc); + /* Fill with garbage: the helper has to clear what it does not set. */ + memset(t->props, 0xa5, sizeof(t->props)); + test->priv = t; + + return 0; +} + +static const struct property_entry *dwc3_host_test_find(const struct property_entry *props, + const char *name) +{ + for (; props->name; props++) + if (!strcmp(props->name, name)) + return props; + + return NULL; +} + +/* Without the glue asking for it, xHCI is not told to keep running in suspend-to-idle. */ +static void dwc3_host_no_run_in_s2idle_test(struct kunit *test) +{ + struct dwc3_host_test *t = test->priv; + unsigned int n; + + n = dwc3_host_xhci_properties(t->dwc, t->props); + + KUNIT_EXPECT_LE(test, n, DWC3_XHCI_MAX_PROPERTIES); + KUNIT_EXPECT_NULL(test, t->props[n].name); + KUNIT_EXPECT_NULL(test, dwc3_host_test_find(t->props, "xhci-run-in-s2idle")); + KUNIT_EXPECT_NOT_NULL(test, dwc3_host_test_find(t->props, "xhci-sg-trb-cache-size-quirk")); +} + +/* The glue property reaches xHCI as a boolean property. */ +static void dwc3_host_run_in_s2idle_test(struct kunit *test) +{ + struct dwc3_host_test *t = test->priv; + const struct property_entry *prop; + unsigned int n; + + t->dwc->xhci_run_in_s2idle = true; + n = dwc3_host_xhci_properties(t->dwc, t->props); + + KUNIT_EXPECT_NULL(test, t->props[n].name); + prop = dwc3_host_test_find(t->props, "xhci-run-in-s2idle"); + KUNIT_ASSERT_NOT_NULL(test, prop); + /* A boolean property has no value. */ + KUNIT_EXPECT_EQ(test, prop->length, 0); +} + +/* With every optional property set, the array still ends in an empty entry. */ +static void dwc3_host_all_properties_test(struct kunit *test) +{ + struct dwc3_host_test *t = test->priv; + unsigned int n; + + t->dwc->ip = DWC3_IP; + t->dwc->revision = DWC3_REVISION_290A; + t->dwc->usb3_lpm_capable = true; + t->dwc->usb2_lpm_disable = true; + t->dwc->xhci_run_in_s2idle = true; + n = dwc3_host_xhci_properties(t->dwc, t->props); + + KUNIT_EXPECT_EQ(test, n, DWC3_XHCI_MAX_PROPERTIES); + KUNIT_EXPECT_NULL(test, t->props[n].name); + KUNIT_EXPECT_NOT_NULL(test, dwc3_host_test_find(t->props, "usb3-lpm-capable")); + KUNIT_EXPECT_NOT_NULL(test, dwc3_host_test_find(t->props, "usb2-lpm-disable")); + KUNIT_EXPECT_NOT_NULL(test, dwc3_host_test_find(t->props, "quirk-broken-port-ped")); + KUNIT_EXPECT_NOT_NULL(test, dwc3_host_test_find(t->props, "num-hc-interrupters")); + KUNIT_EXPECT_NOT_NULL(test, dwc3_host_test_find(t->props, "xhci-run-in-s2idle")); +} + +static struct kunit_case dwc3_host_test_cases[] = { + KUNIT_CASE(dwc3_host_no_run_in_s2idle_test), + KUNIT_CASE(dwc3_host_run_in_s2idle_test), + KUNIT_CASE(dwc3_host_all_properties_test), + {} +}; + +static struct kunit_suite dwc3_host_test_suite = { + .name = "dwc3_host", + .init = dwc3_host_test_init, + .test_cases = dwc3_host_test_cases, +}; + +kunit_test_suite(dwc3_host_test_suite); + +MODULE_DESCRIPTION("KUnit tests for the dwc3 xHCI device properties"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/drivers/usb/dwc3/host.c b/drivers/usb/dwc3/host.c index 8cbefa118b8a2..a138a5f34ebd7 100644 --- a/drivers/usb/dwc3/host.c +++ b/drivers/usb/dwc3/host.c @@ -7,6 +7,7 @@ * Authors: Felipe Balbi , */ +#include #include #include #include @@ -127,9 +128,55 @@ static int dwc3_host_get_irq(struct dwc3 *dwc) return irq; } +/** + * dwc3_host_xhci_properties - Fill in the properties of the xHCI device + * @dwc: Pointer to our controller context structure + * @props: Array of DWC3_XHCI_MAX_PROPERTIES + 1 entries + * + * Return: the number of properties, which are followed by an empty entry. + */ +VISIBLE_IF_KUNIT unsigned int dwc3_host_xhci_properties(const struct dwc3 *dwc, + struct property_entry *props) +{ + unsigned int prop_idx = 0; + + memset(props, 0, sizeof(*props) * (DWC3_XHCI_MAX_PROPERTIES + 1)); + + props[prop_idx++] = PROPERTY_ENTRY_BOOL("xhci-sg-trb-cache-size-quirk"); + + props[prop_idx++] = PROPERTY_ENTRY_BOOL("write-64-hi-lo-quirk"); + + if (dwc->usb3_lpm_capable) + props[prop_idx++] = PROPERTY_ENTRY_BOOL("usb3-lpm-capable"); + + if (dwc->usb2_lpm_disable) + props[prop_idx++] = PROPERTY_ENTRY_BOOL("usb2-lpm-disable"); + + /** + * WORKAROUND: dwc3 revisions <=3.00a have a limitation + * where Port Disable command doesn't work. + * + * The suggested workaround is that we avoid Port Disable + * completely. + * + * This following flag tells XHCI to do just that. + */ + if (DWC3_VER_IS_WITHIN(DWC3, ANY, 300A)) + props[prop_idx++] = PROPERTY_ENTRY_BOOL("quirk-broken-port-ped"); + + props[prop_idx++] = PROPERTY_ENTRY_U16("num-hc-interrupters", + dwc->num_hc_interrupters); + + if (dwc->xhci_run_in_s2idle) + props[prop_idx++] = PROPERTY_ENTRY_BOOL("xhci-run-in-s2idle"); + + return prop_idx; +} +EXPORT_SYMBOL_IF_KUNIT(dwc3_host_xhci_properties); + int dwc3_host_init(struct dwc3 *dwc) { - struct property_entry props[8]; + struct property_entry props[DWC3_XHCI_MAX_PROPERTIES + 1]; struct platform_device *xhci; int ret, irq; int prop_idx = 0; @@ -161,35 +208,7 @@ int dwc3_host_init(struct dwc3 *dwc) goto err; } - memset(props, 0, sizeof(struct property_entry) * ARRAY_SIZE(props)); - - props[prop_idx++] = PROPERTY_ENTRY_BOOL("xhci-sg-trb-cache-size-quirk"); - - props[prop_idx++] = PROPERTY_ENTRY_BOOL("write-64-hi-lo-quirk"); - - if (dwc->usb3_lpm_capable) - props[prop_idx++] = PROPERTY_ENTRY_BOOL("usb3-lpm-capable"); - - if (dwc->usb2_lpm_disable) - props[prop_idx++] = PROPERTY_ENTRY_BOOL("usb2-lpm-disable"); - - /** - * WORKAROUND: dwc3 revisions <=3.00a have a limitation - * where Port Disable command doesn't work. - * - * The suggested workaround is that we avoid Port Disable - * completely. - * - * This following flag tells XHCI to do just that. - */ - if (DWC3_VER_IS_WITHIN(DWC3, ANY, 300A)) - props[prop_idx++] = PROPERTY_ENTRY_BOOL("quirk-broken-port-ped"); - - props[prop_idx++] = PROPERTY_ENTRY_U16("num-hc-interrupters", - dwc->num_hc_interrupters); - - if (dwc->xhci_run_in_s2idle) - props[prop_idx++] = PROPERTY_ENTRY_BOOL("xhci-run-in-s2idle"); + prop_idx = dwc3_host_xhci_properties(dwc, props); if (prop_idx) { ret = device_create_managed_software_node(&xhci->dev, props, NULL); From b4bfd66d1fdf5e73f01ab2aa1785f9e4857bbe52 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:26:16 +1100 Subject: [PATCH 1488/2680] usb: core: Add KUnit tests for the VIA Labs VL122 hub quirks Test that the VL122 hub (2109:2122) no longer has USB_QUIRK_IGNORE_REMOTE_WAKEUP and gets HUB_QUIRK_DISABLE_AUTOSUSPEND instead, and that neighbouring entries still match as before. Move the HUB_QUIRK_* flags to hub.h and add small accessors for the device quirk list and the hub id table that the test can call. No functional change. (cherry picked from commit 0ca3772f141f4b95ca9cb77b2068c5b16c7bfd22) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/core/Kconfig | 11 ++++ drivers/usb/core/Makefile | 1 + drivers/usb/core/hub.c | 15 ++++-- drivers/usb/core/hub.h | 9 ++++ drivers/usb/core/quirks.c | 10 +++- drivers/usb/core/usb-quirks-test.c | 87 ++++++++++++++++++++++++++++++ drivers/usb/core/usb.h | 3 ++ 7 files changed, 132 insertions(+), 4 deletions(-) create mode 100644 drivers/usb/core/usb-quirks-test.c diff --git a/drivers/usb/core/Kconfig b/drivers/usb/core/Kconfig index 58e3ca7e47939..fbeab9ee46202 100644 --- a/drivers/usb/core/Kconfig +++ b/drivers/usb/core/Kconfig @@ -97,6 +97,17 @@ config USB_OTG_FSM Implements OTG Finite State Machine as specified in On-The-Go and Embedded Host Supplement to the USB Revision 2.0 Specification. +config USB_QUIRKS_KUNIT_TEST + tristate "KUnit tests for USB device and hub quirk matching" if !KUNIT_ALL_TESTS + depends on USB && KUNIT + default KUNIT_ALL_TESTS + help + Test that the USB core matches devices and hubs against its quirk + tables as intended for the entries these tests cover. The tests + need no USB hardware. + + If unsure, say N. + config USB_LEDS_TRIGGER_USBPORT tristate "USB port LED trigger" depends on USB && LEDS_TRIGGERS diff --git a/drivers/usb/core/Makefile b/drivers/usb/core/Makefile index 60ea76160122b..bf228db5e15f9 100644 --- a/drivers/usb/core/Makefile +++ b/drivers/usb/core/Makefile @@ -21,5 +21,6 @@ usbcore-y += ../misc/onboard_usb_dev_pdevs.o endif obj-$(CONFIG_USB) += usbcore.o +obj-$(CONFIG_USB_QUIRKS_KUNIT_TEST) += usb-quirks-test.o obj-$(CONFIG_USB_LEDS_TRIGGER_USBPORT) += ledtrig-usbport.o diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index b9372e13c978f..173a7fe809611 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -10,6 +10,7 @@ * Released under the GPLv2 only. */ +#include #include #include #include @@ -57,9 +58,6 @@ #define USB_PRODUCT_USB4913 0x4913 #define USB_PRODUCT_USB4914 0x4914 #define USB_PRODUCT_USB4915 0x4915 -#define HUB_QUIRK_CHECK_PORT_AUTOSUSPEND BIT(0) -#define HUB_QUIRK_DISABLE_AUTOSUSPEND BIT(1) -#define HUB_QUIRK_REDUCE_FRAME_INTR_BINTERVAL BIT(2) #define USB_TP_TRANSMISSION_DELAY 40 /* ns */ #define USB_TP_TRANSMISSION_DELAY_MAX 65535 /* ns */ @@ -6073,6 +6071,17 @@ static const struct usb_device_id hub_id_table[] = { { } /* Terminating entry */ }; +#if IS_ENABLED(CONFIG_KUNIT) +/* The HUB_QUIRK_* flags that hub_id_table gives a hub interface */ +unsigned long usb_hub_id_quirks(struct usb_interface *intf) +{ + const struct usb_device_id *id = usb_match_id(intf, hub_id_table); + + return id ? id->driver_info : 0; +} +EXPORT_SYMBOL_IF_KUNIT(usb_hub_id_quirks); +#endif + MODULE_DEVICE_TABLE(usb, hub_id_table); static struct usb_driver hub_driver = { diff --git a/drivers/usb/core/hub.h b/drivers/usb/core/hub.h index 9ebc5ef54a325..da3235930b0c8 100644 --- a/drivers/usb/core/hub.h +++ b/drivers/usb/core/hub.h @@ -17,6 +17,11 @@ #include #include "usb.h" +/* Quirks that hub_id_table assigns through driver_info */ +#define HUB_QUIRK_CHECK_PORT_AUTOSUSPEND BIT(0) +#define HUB_QUIRK_DISABLE_AUTOSUSPEND BIT(1) +#define HUB_QUIRK_REDUCE_FRAME_INTR_BINTERVAL BIT(2) + struct usb_hub { struct device *intfdev; /* the "interface" device */ struct usb_device *hdev; @@ -183,3 +188,7 @@ static inline int hub_port_debounce_be_stable(struct usb_hub *hub, { return hub_port_debounce(hub, port1, false); } + +#if IS_ENABLED(CONFIG_KUNIT) +unsigned long usb_hub_id_quirks(struct usb_interface *intf); +#endif diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c index b5b577f0b931c..94a8f711da9fc 100644 --- a/drivers/usb/core/quirks.c +++ b/drivers/usb/core/quirks.c @@ -6,6 +6,7 @@ * Copyright (c) 2007 Greg Kroah-Hartman */ +#include #include #include #include @@ -767,9 +768,16 @@ static u32 usb_detect_dynamic_quirks(struct usb_device *udev) /* * Detect any quirks the device has, and do any housekeeping for it if needed. */ +/* The quirks that usb_quirk_list gives a device */ +VISIBLE_IF_KUNIT u32 usb_detect_device_list_quirks(struct usb_device *udev) +{ + return usb_detect_static_quirks(udev, usb_quirk_list); +} +EXPORT_SYMBOL_IF_KUNIT(usb_detect_device_list_quirks); + void usb_detect_quirks(struct usb_device *udev) { - udev->quirks = usb_detect_static_quirks(udev, usb_quirk_list); + udev->quirks = usb_detect_device_list_quirks(udev); /* * Pixart-based mice would trigger remote wakeup issue on AMD diff --git a/drivers/usb/core/usb-quirks-test.c b/drivers/usb/core/usb-quirks-test.c new file mode 100644 index 0000000000000..6426f7951ff0e --- /dev/null +++ b/drivers/usb/core/usb-quirks-test.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit tests for USB device and hub quirk matching + */ + +#include +#include +#include +#include +#include + +/* hub.h includes usb.h, which has no include guard */ +#include "hub.h" + +struct usb_quirks_test { + struct usb_device udev; + struct usb_interface intf; + struct usb_host_interface alt; +}; + +static struct usb_quirks_test *usb_quirks_test_hub(struct kunit *test, u16 vid, u16 pid) +{ + struct usb_quirks_test *t; + + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t); + + t->udev.descriptor.idVendor = cpu_to_le16(vid); + t->udev.descriptor.idProduct = cpu_to_le16(pid); + t->udev.descriptor.bDeviceClass = USB_CLASS_HUB; + t->alt.desc.bInterfaceClass = USB_CLASS_HUB; + t->intf.altsetting = &t->alt; + t->intf.cur_altsetting = &t->alt; + t->intf.num_altsetting = 1; + t->intf.dev.parent = &t->udev.dev; + + return t; +} + +/* + * The VIA Labs VL122 hub on the Apple J700 misbehaves in runtime suspend. It stays + * out of autosuspend, but keeps its wakeup setting for system sleep. + */ +static void usb_quirks_vl122_test(struct kunit *test) +{ + struct usb_quirks_test *t = usb_quirks_test_hub(test, 0x2109, 0x2122); + + KUNIT_EXPECT_FALSE(test, usb_detect_device_list_quirks(&t->udev) & + USB_QUIRK_IGNORE_REMOTE_WAKEUP); + KUNIT_EXPECT_TRUE(test, usb_hub_id_quirks(&t->intf) & HUB_QUIRK_DISABLE_AUTOSUSPEND); +} + +/* Another VIA Labs hub keeps its own device quirk and no hub quirk. */ +static void usb_quirks_other_via_hub_test(struct kunit *test) +{ + struct usb_quirks_test *t = usb_quirks_test_hub(test, 0x2109, 0x2817); + + KUNIT_EXPECT_EQ(test, usb_detect_device_list_quirks(&t->udev), USB_QUIRK_NO_LPM); + KUNIT_EXPECT_EQ(test, usb_hub_id_quirks(&t->intf), 0UL); +} + +/* A hub that no table names gets no quirks. */ +static void usb_quirks_generic_hub_test(struct kunit *test) +{ + struct usb_quirks_test *t = usb_quirks_test_hub(test, 0x1d6b, 0x0002); + + KUNIT_EXPECT_EQ(test, usb_detect_device_list_quirks(&t->udev), 0U); + KUNIT_EXPECT_EQ(test, usb_hub_id_quirks(&t->intf), 0UL); +} + +static struct kunit_case usb_quirks_test_cases[] = { + KUNIT_CASE(usb_quirks_vl122_test), + KUNIT_CASE(usb_quirks_other_via_hub_test), + KUNIT_CASE(usb_quirks_generic_hub_test), + {} +}; + +static struct kunit_suite usb_quirks_test_suite = { + .name = "usb_quirks", + .test_cases = usb_quirks_test_cases, +}; + +kunit_test_suite(usb_quirks_test_suite); + +MODULE_DESCRIPTION("KUnit tests for USB device and hub quirk matching"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/drivers/usb/core/usb.h b/drivers/usb/core/usb.h index a9b37aeb515be..c8880c20f1563 100644 --- a/drivers/usb/core/usb.h +++ b/drivers/usb/core/usb.h @@ -36,6 +36,9 @@ extern int usb_authorize_device(struct usb_device *); extern void usb_deauthorize_interface(struct usb_interface *); extern void usb_authorize_interface(struct usb_interface *); extern void usb_detect_quirks(struct usb_device *udev); +#if IS_ENABLED(CONFIG_KUNIT) +u32 usb_detect_device_list_quirks(struct usb_device *udev); +#endif extern void usb_detect_interface_quirks(struct usb_device *udev); extern void usb_release_quirk_list(void); extern bool usb_endpoint_is_ignored(struct usb_device *udev, From b841ad4b8b3532c63db63b2572abb86db936643f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 18:20:43 +1100 Subject: [PATCH 1489/2680] usb: core: Keep a hub awake in suspend-to-idle when it cannot pass on wakeup The VIA Labs VL122 USB 2.0 hub on the Apple J700 does not pass a remote wakeup from a downstream device upstream while it is suspended itself, the same limitation that makes it miss downstream connects in runtime suspend. With the whole chain armed and the xHC running in suspend-to-idle, a keypress on a keyboard behind it never reaches the root port: PORTSC stays in U3 without a link state change, with the keyboard behind a dock and behind a plain USB 2.0 hub alike. The hub does see the resume while it is awake: an active hub completes a downstream resume on its port and reports it through its status change endpoint. Keep such a hub awake in suspend-to-idle when a device below it may wake the system, as the USB offload support already does for devices that stay in use. Every device between it and the root hub, the root hub included, stays awake too, so that the root port stays in U0 and the running controller keeps polling the hub's interrupt endpoint. The devices below the hub are suspended as before. A remote wakeup then completes a transfer on the hub's status endpoint, and the controller interrupt, armed for wakeup, ends suspend-to-idle. This needs a controller that keeps running in suspend-to-idle; xhci-plat says so through a new runs_in_s2idle flag of struct usb_hcd. Mark the VL122 with a new USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP; it keeps HUB_QUIRK_DISABLE_AUTOSUSPEND for runtime PM. The cost is an active hub and an active root port while a wake source sits below the hub. Tested on the J700 with a USB keyboard behind the VL122 and a dock, lid open and closed: a keypress ends suspend-to-idle through the xHCI interrupt, 3.7 s from entering suspend when pressed right away. (cherry picked from commit a28673c796d5475d6aeb0f05052a28f233a09031) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/core/driver.c | 56 +++++++++++++++++++++++++----- drivers/usb/core/quirks.c | 4 +++ drivers/usb/core/usb-quirks-test.c | 10 +++--- drivers/usb/host/xhci-plat.c | 9 +++++ include/linux/usb.h | 1 + include/linux/usb/hcd.h | 2 ++ include/linux/usb/quirks.h | 7 ++++ 7 files changed, 76 insertions(+), 13 deletions(-) diff --git a/drivers/usb/core/driver.c b/drivers/usb/core/driver.c index 7f33fe5ba03bd..2dc8796823de3 100644 --- a/drivers/usb/core/driver.c +++ b/drivers/usb/core/driver.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -1387,6 +1388,37 @@ static int usb_resume_interface(struct usb_device *udev, return status; } +/* + * Whether to keep a device awake in suspend-to-idle so that a remote wakeup + * from below can reach the host controller: a hub that does not pass one on + * while it is suspended itself, and every device between such a hub and the + * root hub, root hub included. This only works while the host controller keeps + * running, so its interrupt reports the hub's status change. + */ +static bool usb_stay_awake_for_wakeup(struct usb_device *udev, pm_message_t msg) +{ + struct usb_hcd *hcd = bus_to_hcd(udev->bus); + struct usb_device *child; + int port1; + + if (msg.event != PM_EVENT_SUSPEND || !hcd->runs_in_s2idle || + pm_suspend_target_state != PM_SUSPEND_TO_IDLE || + !device_may_wakeup(hcd->self.controller)) + return false; + + if ((udev->quirks & USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP) && + udev->do_remote_wakeup && device_wakeup_path(&udev->dev)) + return true; + + /* Children are done with their suspend before their parent starts */ + usb_hub_for_each_child(udev, port1, child) { + if (child->awake_in_sleep) + return true; + } + + return false; +} + /** * usb_suspend_both - suspend a USB device and its interfaces * @udev: the usb_device to suspend @@ -1430,6 +1462,9 @@ static int usb_suspend_both(struct usb_device *udev, pm_message_t msg) dev_dbg(&udev->dev, "device offloaded, skip suspend.\n"); offload_active = true; } + udev->awake_in_sleep = usb_stay_awake_for_wakeup(udev, msg); + if (udev->awake_in_sleep) + dev_dbg(&udev->dev, "kept awake to pass on remote wakeup\n"); /* Suspend all the interfaces and then udev itself */ if (udev->actconfig) { @@ -1442,9 +1477,9 @@ static int usb_suspend_both(struct usb_device *udev, pm_message_t msg) * interrupt urbs, allowing interrupt events to be * handled during system suspend. */ - if (offload_active && intf->needs_remote_wakeup) { - dev_dbg(&intf->dev, - "device offloaded, skip suspend.\n"); + if ((offload_active || udev->awake_in_sleep) && + intf->needs_remote_wakeup) { + dev_dbg(&intf->dev, "interface kept active, skip suspend.\n"); continue; } status = usb_suspend_interface(udev, intf, msg); @@ -1457,7 +1492,7 @@ static int usb_suspend_both(struct usb_device *udev, pm_message_t msg) } } if (status == 0) { - if (!offload_active) + if (!offload_active && !udev->awake_in_sleep) status = usb_suspend_device(udev, msg); /* @@ -1501,7 +1536,7 @@ static int usb_suspend_both(struct usb_device *udev, pm_message_t msg) /* If the suspend succeeded then prevent any more URB submissions * and flush any outstanding URBs. */ - } else { + } else if (!udev->awake_in_sleep) { udev->can_submit = 0; if (!offload_active) { for (i = 0; i < 16; ++i) { @@ -1512,8 +1547,10 @@ static int usb_suspend_both(struct usb_device *udev, pm_message_t msg) } done: - if (status != 0) + if (status != 0) { + udev->awake_in_sleep = 0; usb_offload_set_pm_locked(udev, false); + } dev_vdbg(&udev->dev, "%s: status %d\n", __func__, status); return status; } @@ -1544,7 +1581,9 @@ static int usb_resume_both(struct usb_device *udev, pm_message_t msg) int i; struct usb_interface *intf; bool offload_active = false; + bool awake = udev->awake_in_sleep; + udev->awake_in_sleep = 0; if (udev->state == USB_STATE_NOTATTACHED) { status = -ENODEV; goto done; @@ -1572,9 +1611,8 @@ static int usb_resume_both(struct usb_device *udev, pm_message_t msg) * pending interrupt urbs, allowing interrupt events * to be handled during system suspend. */ - if (offload_active && intf->needs_remote_wakeup) { - dev_dbg(&intf->dev, - "device offloaded, skip resume.\n"); + if ((offload_active || awake) && intf->needs_remote_wakeup) { + dev_dbg(&intf->dev, "interface kept active, skip resume.\n"); continue; } usb_resume_interface(udev, intf, msg, diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c index 94a8f711da9fc..bf38174963e5c 100644 --- a/drivers/usb/core/quirks.c +++ b/drivers/usb/core/quirks.c @@ -584,6 +584,10 @@ static const struct usb_device_id usb_quirk_list[] = { /* VLI disk */ { USB_DEVICE(0x2109, 0x0711), .driver_info = USB_QUIRK_NO_LPM }, + /* VIA Labs, Inc. VL122 USB2.0 Hub */ + { USB_DEVICE(0x2109, 0x2122), .driver_info = + USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP }, + /* VIA Labs, Inc. USB2.0 Hub */ { USB_DEVICE(0x2109, 0x2817), .driver_info = USB_QUIRK_NO_LPM }, diff --git a/drivers/usb/core/usb-quirks-test.c b/drivers/usb/core/usb-quirks-test.c index 6426f7951ff0e..81ebc58238b0d 100644 --- a/drivers/usb/core/usb-quirks-test.c +++ b/drivers/usb/core/usb-quirks-test.c @@ -38,15 +38,17 @@ static struct usb_quirks_test *usb_quirks_test_hub(struct kunit *test, u16 vid, } /* - * The VIA Labs VL122 hub on the Apple J700 misbehaves in runtime suspend. It stays - * out of autosuspend, but keeps its wakeup setting for system sleep. + * The VIA Labs VL122 hub on the Apple J700 does not pass on remote wakeup while it + * is suspended. It stays out of autosuspend and keeps its wakeup setting, and is + * kept awake in suspend-to-idle instead. */ static void usb_quirks_vl122_test(struct kunit *test) { struct usb_quirks_test *t = usb_quirks_test_hub(test, 0x2109, 0x2122); + u32 quirks = usb_detect_device_list_quirks(&t->udev); - KUNIT_EXPECT_FALSE(test, usb_detect_device_list_quirks(&t->udev) & - USB_QUIRK_IGNORE_REMOTE_WAKEUP); + KUNIT_EXPECT_FALSE(test, quirks & USB_QUIRK_IGNORE_REMOTE_WAKEUP); + KUNIT_EXPECT_TRUE(test, quirks & USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP); KUNIT_EXPECT_TRUE(test, usb_hub_id_quirks(&t->intf) & HUB_QUIRK_DISABLE_AUTOSUSPEND); } diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index 8a4413f817e72..6d2aa0d8f7492 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -367,6 +367,15 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s goto put_usb3_hcd; } + /* + * Let the USB core keep hubs awake that need it for wakeup, see + * XHCI_RUN_IN_S2IDLE. Only now are all quirk sources merged, the xhci_hcd + * quirks module parameter included. + */ + hcd->runs_in_s2idle = !!(xhci->quirks & XHCI_RUN_IN_S2IDLE); + if (xhci->shared_hcd) + xhci->shared_hcd->runs_in_s2idle = hcd->runs_in_s2idle; + device_enable_async_suspend(&pdev->dev); pm_runtime_put_noidle(&pdev->dev); diff --git a/include/linux/usb.h b/include/linux/usb.h index 49ab8dbb885f6..482d37aff4a72 100644 --- a/include/linux/usb.h +++ b/include/linux/usb.h @@ -708,6 +708,7 @@ struct usb_device { unsigned usb2_hw_lpm_allowed:1; unsigned usb3_lpm_u1_enabled:1; unsigned usb3_lpm_u2_enabled:1; + unsigned awake_in_sleep:1; int string_langid; /* static strings from the device */ diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h index 181db044c4d2e..f4520d9922a2b 100644 --- a/include/linux/usb/hcd.h +++ b/include/linux/usb/hcd.h @@ -160,6 +160,8 @@ struct usb_hcd { unsigned tpl_support:1; /* OTG & EH TPL support */ unsigned cant_recv_wakeups:1; /* wakeup requests from downstream aren't received */ + unsigned runs_in_s2idle:1; + /* HC keeps running in suspend-to-idle with wakeup on */ unsigned int irq; /* irq allocated */ void __iomem *regs; /* device memory/io */ diff --git a/include/linux/usb/quirks.h b/include/linux/usb/quirks.h index a4043b33c2c2a..8bcceb0d02bd8 100644 --- a/include/linux/usb/quirks.h +++ b/include/linux/usb/quirks.h @@ -84,4 +84,11 @@ /* Use a 255 bytes config descriptor request mirroring windows behavior */ #define USB_QUIRK_WINDOWS_CONFIG_REQ_SIZE BIT(19) +/* + * Hub does not pass a downstream remote wakeup upstream while it is suspended + * itself. Keep it awake in suspend-to-idle when a device below may wake the + * system and the host controller keeps running. + */ +#define USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP BIT(20) + #endif /* __LINUX_USB_QUIRKS_H */ From 0c43d9cfa548f3ce8076e79dbafa7db0ace5e8e4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:31:23 +1100 Subject: [PATCH 1490/2680] usb: xhci: plat: Report a controller that stopped in suspend-to-idle as dead A controller left running in suspend-to-idle is expected to come back still running. If it stopped all the same, because power was cut or it hit a Host System Error or Host Controller Error, resume skipped xhci_resume() and handed the root hubs a dead controller. It is not brought back here. Reinitialising it, as xhci_resume() does when told power was lost, is only safe on a quiesced controller: one whose devices were suspended and whose URBs were flushed. In suspend-to-idle devices may have been kept awake, with URBs queued and their interrupt and hub retry paths live, and a reset under them frees state those paths still use. Report the controller dead instead, the path for a controller that stops responding: first mask its interrupts and halt it with xhci_halt(), as the other error paths do, unless its registers no longer answer, since the USB core ignores the interrupts of a dead HCD and a pending one would never be acknowledged; then xhci_hc_died() gives back every pending URB and lets the USB core disconnect the devices. USB then needs the controller to be bound again. The new xhci_quiesce_dying() and xhci_hc_died() are exported for this. The check for a stopped controller and the quiescing are tested with KUnit; the quiescing test fails when the interrupts are left enabled. (cherry picked from commit 6ad2760be257077ee50e6279a94b02aea153a772) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/host/xhci-plat-test.c | 48 +++++++++++++++++++++++++++++++ drivers/usb/host/xhci-plat.c | 31 +++++++++++++++++++- drivers/usb/host/xhci-plat.h | 1 + drivers/usb/host/xhci-ring.c | 1 + drivers/usb/host/xhci.c | 12 ++++++++ drivers/usb/host/xhci.h | 1 + 6 files changed, 93 insertions(+), 1 deletion(-) diff --git a/drivers/usb/host/xhci-plat-test.c b/drivers/usb/host/xhci-plat-test.c index 410cf99da08d6..844e71d7ecdb8 100644 --- a/drivers/usb/host/xhci-plat-test.c +++ b/drivers/usb/host/xhci-plat-test.c @@ -45,6 +45,52 @@ static void xhci_plat_keep_running_other_quirks_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, xhci_plat_keep_running(~0ULL, PM_SUSPEND_TO_IDLE, true, true)); } +/* A running controller is left alone; a halted, failed or unpowered one is not. */ +static void xhci_plat_stopped_test(struct kunit *test) +{ + /* Running, with an event pending and a port change */ + KUNIT_EXPECT_FALSE(test, xhci_plat_stopped(0)); + KUNIT_EXPECT_FALSE(test, xhci_plat_stopped(STS_EINT | STS_PORT)); + /* Halted, Host System Error, Host Controller Error */ + KUNIT_EXPECT_TRUE(test, xhci_plat_stopped(STS_HALT)); + KUNIT_EXPECT_TRUE(test, xhci_plat_stopped(STS_FATAL | STS_HALT)); + KUNIT_EXPECT_TRUE(test, xhci_plat_stopped(STS_HCE)); + /* Register reads of a powered-off block */ + KUNIT_EXPECT_TRUE(test, xhci_plat_stopped(~0U)); +} + +/* + * A controller reported dead is halted with its interrupts masked first: + * the USB core ignores the interrupts of a dead HCD, so one left enabled + * and pending would never be acknowledged. + */ +static void xhci_plat_quiesce_dying_test(struct kunit *test) +{ + struct xhci_op_regs *op; + struct xhci_hcd *xhci; + + xhci = kunit_kzalloc(test, sizeof(*xhci), GFP_KERNEL); + op = kunit_kzalloc(test, sizeof(*op), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, xhci); + KUNIT_ASSERT_NOT_NULL(test, op); + xhci->op_regs = (__force struct xhci_op_regs __iomem *)op; + + /* A Host System Error with an event pending, interrupts enabled */ + writel(CMD_RUN | XHCI_IRQS, &xhci->op_regs->command); + writel(STS_HALT | STS_FATAL | STS_EINT, &xhci->op_regs->status); + xhci_quiesce_dying(xhci); + KUNIT_EXPECT_EQ(test, readl(&xhci->op_regs->command) & XHCI_IRQS, 0U); + KUNIT_EXPECT_TRUE(test, xhci->xhc_state & XHCI_STATE_HALTED); + + /* Registers that no longer answer are left alone. */ + xhci->xhc_state = 0; + writel(~0U, &xhci->op_regs->status); + writel(XHCI_IRQS, &xhci->op_regs->command); + xhci_quiesce_dying(xhci); + KUNIT_EXPECT_EQ(test, readl(&xhci->op_regs->command), (u32)XHCI_IRQS); + KUNIT_EXPECT_FALSE(test, xhci->xhc_state & XHCI_STATE_HALTED); +} + static u64 xhci_plat_test_quirks(struct kunit *test, const struct property_entry *props) { struct device *dev; @@ -99,6 +145,8 @@ static void xhci_plat_all_property_quirks_test(struct kunit *test) static struct kunit_case xhci_plat_test_cases[] = { KUNIT_CASE(xhci_plat_keep_running_test), KUNIT_CASE(xhci_plat_keep_running_other_quirks_test), + KUNIT_CASE(xhci_plat_stopped_test), + KUNIT_CASE(xhci_plat_quiesce_dying_test), KUNIT_CASE(xhci_plat_no_property_quirks_test), KUNIT_CASE(xhci_plat_run_in_s2idle_property_test), KUNIT_CASE(xhci_plat_all_property_quirks_test), diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index 6d2aa0d8f7492..3f11e5609f613 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -534,6 +534,13 @@ VISIBLE_IF_KUNIT bool xhci_plat_keep_running(u64 quirks, suspend_state_t target, } EXPORT_SYMBOL_IF_KUNIT(xhci_plat_keep_running); +/* Whether a controller that was left running reads back as stopped or failed */ +VISIBLE_IF_KUNIT bool xhci_plat_stopped(u32 usbsts) +{ + return usbsts == ~(u32)0 || (usbsts & (STS_HALT | STS_FATAL | STS_HCE)); +} +EXPORT_SYMBOL_IF_KUNIT(xhci_plat_stopped); + static bool xhci_plat_run_in_s2idle(struct device *dev) { struct usb_hcd *hcd = dev_get_drvdata(dev); @@ -629,8 +636,30 @@ static int xhci_plat_resume(struct device *dev) } if (priv->running_in_s2idle) { + struct xhci_hcd *xhci = hcd_to_xhci(hcd); + u32 status = readl(&xhci->op_regs->status); + priv->running_in_s2idle = 0; - dev_dbg(dev, "kept running in suspend-to-idle, skip resume.\n"); + if (!xhci_plat_stopped(status)) { + dev_dbg(dev, "kept running in suspend-to-idle, skip resume.\n"); + return 0; + } + + /* + * The controller was left running on the platform's promise that it + * stays powered, and stopped all the same: power was cut, or it hit an + * error. It is not brought back here. Devices may have been kept awake + * through the sleep, with URBs queued and their interrupt and retry + * paths live, and a reset under them is not safe. Mask its interrupts and + * halt it, and report it dead, which gives back every pending URB and + * lets the USB core disconnect the devices. USB then needs the controller + * bound again. + */ + dev_err(dev, "stopped in suspend-to-idle (USBSTS %#010x)\n", status); + spin_lock_irq(&xhci->lock); + xhci_quiesce_dying(xhci); + xhci_hc_died(xhci); + spin_unlock_irq(&xhci->lock); return 0; } diff --git a/drivers/usb/host/xhci-plat.h b/drivers/usb/host/xhci-plat.h index 5a43502ca27a9..1956db312b8aa 100644 --- a/drivers/usb/host/xhci-plat.h +++ b/drivers/usb/host/xhci-plat.h @@ -40,6 +40,7 @@ extern const struct dev_pm_ops xhci_plat_pm_ops; u64 xhci_plat_property_quirks(struct device *dev); bool xhci_plat_keep_running(u64 quirks, suspend_state_t target, bool may_wakeup, bool wakeup_path); +bool xhci_plat_stopped(u32 usbsts); #endif #endif /* _XHCI_PLAT_H */ diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c index 8908f5c27e526..429c34b38788b 100644 --- a/drivers/usb/host/xhci-ring.c +++ b/drivers/usb/host/xhci-ring.c @@ -1421,6 +1421,7 @@ void xhci_hc_died(struct xhci_hcd *xhci) if (notify) usb_hc_died(xhci_to_hcd(xhci)); } +EXPORT_SYMBOL_GPL(xhci_hc_died); /* * When we get a completion for a Set Transfer Ring Dequeue Pointer command, diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index 5c759ad28199f..66872b7a7f355 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -138,6 +138,18 @@ int xhci_halt(struct xhci_hcd *xhci) return ret; } +/* + * Stop a controller that is about to be reported dead: mask its interrupts and + * halt it, as the other error paths do before xhci_hc_died(). A controller whose + * registers no longer answer is left alone. Called with xhci->lock held. + */ +void xhci_quiesce_dying(struct xhci_hcd *xhci) +{ + if (readl(&xhci->op_regs->status) != ~(u32)0) + xhci_halt(xhci); +} +EXPORT_SYMBOL_GPL(xhci_quiesce_dying); + /* * Set the run bit and wait for the host to be running. */ diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index dfae388d0f660..616c308be72e5 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -1986,6 +1986,7 @@ struct xhci_hub *xhci_get_rhub(struct usb_hcd *hcd); enum usb_link_tunnel_mode xhci_port_is_tunneled(struct xhci_hcd *xhci, struct xhci_port *port); void xhci_hc_died(struct xhci_hcd *xhci); +void xhci_quiesce_dying(struct xhci_hcd *xhci); #ifdef CONFIG_PM int xhci_bus_suspend(struct usb_hcd *hcd); From e82cc6b800edba79dee5d5fa9eb5ca8bc222d96b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:32:07 +1100 Subject: [PATCH 1491/2680] usb: xhci: plat: Fail system suspend when the controller does not resume Before a system suspend, a runtime suspended controller is resumed: it is halted, and both the halting path and the path that keeps it running in suspend-to-idle need it running. The result of pm_runtime_resume() was ignored. When it failed, the keep-running path marked the controller as running and reported success, so the system went to sleep without the wake source it was promised, and the halting path went on to suspend a controller that was not running. Fail the system suspend with the error instead, before anything is marked. The runtime resume is a helper with KUnit tests, built with CONFIG_PM only; the error test fails when the result is ignored. Fixes: 4d9fbe0bed99 ("usb: xhci: plat: Allow the controller to keep running in suspend-to-idle") (cherry picked from commit 7dfc99ae6302ec23f0e9fcefb349fe6d94ff0a02) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/host/xhci-plat-test.c | 53 +++++++++++++++++++++++++++++++ drivers/usb/host/xhci-plat.c | 25 ++++++++++++--- drivers/usb/host/xhci-plat.h | 1 + 3 files changed, 74 insertions(+), 5 deletions(-) diff --git a/drivers/usb/host/xhci-plat-test.c b/drivers/usb/host/xhci-plat-test.c index 844e71d7ecdb8..00646f7ba2f5a 100644 --- a/drivers/usb/host/xhci-plat-test.c +++ b/drivers/usb/host/xhci-plat-test.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include "xhci.h" @@ -91,6 +92,54 @@ static void xhci_plat_quiesce_dying_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, xhci->xhc_state & XHCI_STATE_HALTED); } +#ifdef CONFIG_PM +static int xhci_plat_test_rpm_ret; + +static int xhci_plat_test_runtime_resume(struct device *dev) +{ + return xhci_plat_test_rpm_ret; +} + +static struct dev_pm_domain xhci_plat_test_pm_domain = { + .ops = { .runtime_resume = xhci_plat_test_runtime_resume }, +}; + +static struct device *xhci_plat_test_rpm_device(struct kunit *test, int rpm_ret) +{ + struct device *dev = kunit_device_register(test, "xhci-plat-rpm-test"); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + xhci_plat_test_rpm_ret = rpm_ret; + dev->pm_domain = &xhci_plat_test_pm_domain; + pm_runtime_set_suspended(dev); + pm_runtime_enable(dev); + return dev; +} + +/* A runtime resume that fails before system sleep fails the system suspend. */ +static void xhci_plat_resume_from_runtime_error_test(struct kunit *test) +{ + struct device *dev = xhci_plat_test_rpm_device(test, -ETIMEDOUT); + + KUNIT_EXPECT_EQ(test, xhci_plat_resume_from_runtime(dev), -ETIMEDOUT); + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); + pm_runtime_disable(dev); + dev->pm_domain = NULL; +} + +/* A successful one, and an active controller, let it go ahead. */ +static void xhci_plat_resume_from_runtime_ok_test(struct kunit *test) +{ + struct device *dev = xhci_plat_test_rpm_device(test, 0); + + KUNIT_EXPECT_EQ(test, xhci_plat_resume_from_runtime(dev), 0); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); + KUNIT_EXPECT_EQ(test, xhci_plat_resume_from_runtime(dev), 0); + pm_runtime_disable(dev); + dev->pm_domain = NULL; +} +#endif /* CONFIG_PM */ + static u64 xhci_plat_test_quirks(struct kunit *test, const struct property_entry *props) { struct device *dev; @@ -147,6 +196,10 @@ static struct kunit_case xhci_plat_test_cases[] = { KUNIT_CASE(xhci_plat_keep_running_other_quirks_test), KUNIT_CASE(xhci_plat_stopped_test), KUNIT_CASE(xhci_plat_quiesce_dying_test), +#ifdef CONFIG_PM + KUNIT_CASE(xhci_plat_resume_from_runtime_error_test), + KUNIT_CASE(xhci_plat_resume_from_runtime_ok_test), +#endif KUNIT_CASE(xhci_plat_no_property_quirks_test), KUNIT_CASE(xhci_plat_run_in_s2idle_property_test), KUNIT_CASE(xhci_plat_all_property_quirks_test), diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index 3f11e5609f613..aec29654680e3 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -490,14 +490,28 @@ void xhci_plat_remove(struct platform_device *dev) } EXPORT_SYMBOL_GPL(xhci_plat_remove); +/* A runtime suspended controller is halted: bring it back before system sleep. */ +VISIBLE_IF_KUNIT int xhci_plat_resume_from_runtime(struct device *dev) +{ + int ret; + + if (!pm_runtime_suspended(dev)) + return 0; + + ret = pm_runtime_resume(dev); + return ret < 0 ? ret : 0; +} +EXPORT_SYMBOL_IF_KUNIT(xhci_plat_resume_from_runtime); + static int xhci_plat_suspend_common(struct device *dev) { struct usb_hcd *hcd = dev_get_drvdata(dev); struct xhci_hcd *xhci = hcd_to_xhci(hcd); int ret; - if (pm_runtime_suspended(dev)) - pm_runtime_resume(dev); + ret = xhci_plat_resume_from_runtime(dev); + if (ret) + return ret; ret = xhci_priv_suspend_quirk(hcd); if (ret) @@ -562,9 +576,10 @@ static int xhci_plat_suspend(struct device *dev) } if (xhci_plat_run_in_s2idle(dev)) { - /* A runtime suspended controller is halted, start it again. */ - if (pm_runtime_suspended(dev)) - pm_runtime_resume(dev); + int ret = xhci_plat_resume_from_runtime(dev); + + if (ret) + return ret; priv->running_in_s2idle = 1; dev_dbg(dev, "wakeup enabled, keep running in suspend-to-idle.\n"); return 0; diff --git a/drivers/usb/host/xhci-plat.h b/drivers/usb/host/xhci-plat.h index 1956db312b8aa..da55117cbf3f3 100644 --- a/drivers/usb/host/xhci-plat.h +++ b/drivers/usb/host/xhci-plat.h @@ -41,6 +41,7 @@ u64 xhci_plat_property_quirks(struct device *dev); bool xhci_plat_keep_running(u64 quirks, suspend_state_t target, bool may_wakeup, bool wakeup_path); bool xhci_plat_stopped(u32 usbsts); +int xhci_plat_resume_from_runtime(struct device *dev); #endif #endif /* _XHCI_PLAT_H */ From 88b242d520e3f81ce814bfd26765da158d2bca64 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 16:55:52 +1100 Subject: [PATCH 1492/2680] usb: xhci: Add KUnit static stubs for PORTSC accesses The change bits in PORTSC are cleared by writing 1, and a port posts a Port Status Change Event only for a change made while none of them is set (xHCI 1.2, section 4.19.2). KUnit tests of the event handling fake the registers in memory, where writing 1 to a change bit leaves it set, so they can show neither. Nor can they let the port change between two accesses of the driver, as the controller may. Let a test stand in for xhci_portsc_writel() and xhci_portsc_readl(), so that it can apply each write the way the controller does, and change the port just before a chosen read. Without CONFIG_KUNIT the redirections compile away, and with it, they only look for a stub while a KUnit test is running. The next change uses them. (cherry picked from commit 5261b17aa6855626f86c9c35844921beaa279c8f) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/host/xhci.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index 66872b7a7f355..7019daa1d0376 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -8,6 +8,7 @@ * Some code borrowed from the Linux EHCI driver. */ +#include #include #include #include @@ -43,6 +44,8 @@ MODULE_PARM_DESC(quirks, "Bit flags for quirks to be enabled as default"); void xhci_portsc_writel(struct xhci_port *port, u32 val) { + KUNIT_STATIC_STUB_REDIRECT(xhci_portsc_writel, port, val); + trace_xhci_portsc_writel(port, val); writel(val, &port->port_reg->portsc); } @@ -50,6 +53,8 @@ EXPORT_SYMBOL_GPL(xhci_portsc_writel); u32 xhci_portsc_readl(struct xhci_port *port) { + KUNIT_STATIC_STUB_REDIRECT(xhci_portsc_readl, port); + return readl(&port->port_reg->portsc); } EXPORT_SYMBOL_GPL(xhci_portsc_readl); From e7e00b14e20c4f9df9bea1406764a9156ef797fb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 14:30:31 +1100 Subject: [PATCH 1493/2680] usb: xhci: Wake from suspend-to-idle only for wakeup events A controller that is kept running in suspend-to-idle (XHCI_RUN_IN_S2IDLE) posts a Port Status Change Event for every change on every root port, and with its interrupt armed for wakeup any of them ends suspend-to-idle. On a J700 the SuperSpeed link of the rear port, behind a debug cable, drops to SS.Inactive every few seconds. Once a wake-armed keyboard keeps the controller running, that link change wakes the system within two seconds, although its root hub is suspended with wakeup disabled: usb usb2: bus suspend, wakeup 0 xhci-hcd xhci-hcd.0.auto: wakeup enabled, keep running in suspend-to-idle. PM: Triggering wakeup from IRQ 93 xhci-hcd xhci-hcd.0.auto: Get port status 2-1 read: 0x4002c0, return 0x4002c0 A halted controller signals only some events as a wakeup. The wake bits in PORTSC, which xhci_bus_suspend() sets only when the root hub may wake the system, select connect, disconnect and over-current changes, and a device signals resume only when the USB core enabled its remote wakeup. While the controller runs those bits select nothing, so make the same choice in software instead: when the controller is running in suspend-to-idle, report a hard wakeup event for exactly those port changes, for a transfer completed by a hub that the USB core kept awake to pass on a remote wakeup (the only devices with transfers in flight), for a function wake notification, and for a controller that stopped or reports a Host Controller or Host System Error, so that resume finds it. Such an error wakes the system even if xhci_halt() then times out, as a failing controller may never halt. This is how macsmc-input decides on wakeups from its suspend-to-idle interrupt. Do not resume or poll the root hub for any other port change, so no wakeup event is reported and no hub status URB completes while the system sleeps. Do not leave the change set in PORTSC either: a port posts a Port Status Change Event only for a change made while none of its change bits is set (xHCI 1.2, section 4.19.2), so a later change of that port that should wake the system, such as a connect, would post none. Clear the change bits and hold them in struct xhci_port instead, where GetPortStatus and the root hub status data report them to the hub driver as before, until it clears them. Clearing a change bit is part of the decision, and the event handling clears none before it, not even the link change of a USB 2 port, which it clears at once when the system is awake. The port may change between a read of PORTSC and the write that clears its change bits: - A change latched in between, while other change bits were still set, posts no event, so PORTSC is read again after each write, and judged. - A device that signals resume while PLC is still set, for example from its U3 entry, sets no new PLC, and the write clears the only change bit of that remote wakeup. Only the link state, now Resume, shows it. The event handling then carries on as if PLC were still set, so that it starts the resume timing of a USB 2 port, or moves a SuperSpeed link to U0. When running in suspend-to-idle begins, the root hubs stop being polled until resume, as for a halted controller, and ports whose change bits are still set, for example for a hub driver that froze before it cleared them, are handled as if they had just posted an event. On resume, the root hubs of ports with held changes are resumed, and all root hubs are polled again, as xhci_resume() does. The interface to the platform glue is xhci_run_in_s2idle_begin() and xhci_run_in_s2idle_end(), and the running state moves from the glue into struct xhci_hcd so the event handling can see it. Add KUnit cases for the decision, and for the event path through xhci_irq() on a controller faked in memory, whose PORTSC takes writes and posts events as the controller does. Before this change, the link change resumes and polls the root hub there, neither a remote wakeup nor a controller error reports a wakeup event, and a port whose link change is left set posts no event for a later connect, disconnect, over-current or remote wakeup. Further cases let the device below a USB 2 or SuperSpeed port signal resume while a change is decided on, just before each PORTSC read in turn and just before the write that clears PLC, both on an event and when running in suspend-to-idle begins: the system wakes, and the resume is taken on. This only decides between events: the controller interrupt is still armed for wakeup, so the IRQ core still ends suspend-to-idle before the handler runs. The following changes keep it enabled instead. (cherry picked from commit e9573a16e76c5f0f19efaae643fa6c395f258d7b) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/host/Kconfig | 5 +- drivers/usb/host/xhci-hub.c | 8 +- drivers/usb/host/xhci-plat-test.c | 948 ++++++++++++++++++++++++++++++ drivers/usb/host/xhci-plat.c | 9 +- drivers/usb/host/xhci-plat.h | 1 - drivers/usb/host/xhci-ring.c | 173 +++++- drivers/usb/host/xhci.c | 83 ++- drivers/usb/host/xhci.h | 12 + 8 files changed, 1224 insertions(+), 15 deletions(-) diff --git a/drivers/usb/host/Kconfig b/drivers/usb/host/Kconfig index 812fd8ce5f9e9..6b6034e505fcc 100644 --- a/drivers/usb/host/Kconfig +++ b/drivers/usb/host/Kconfig @@ -82,8 +82,9 @@ config USB_XHCI_PLATFORM_KUNIT_TEST default KUNIT_ALL_TESTS help Test how the generic xHCI platform driver turns device properties - into quirks, and when it keeps the controller running across - suspend-to-idle instead of halting it. The tests need no xHCI + into quirks, when it keeps the controller running across + suspend-to-idle instead of halting it, and which events of a + controller kept running wake the system. The tests need no xHCI hardware. If unsure, say N. diff --git a/drivers/usb/host/xhci-hub.c b/drivers/usb/host/xhci-hub.c index b0264bd8577ae..e5661c4673dbd 100644 --- a/drivers/usb/host/xhci-hub.c +++ b/drivers/usb/host/xhci-hub.c @@ -619,7 +619,8 @@ static void xhci_clear_port_change_bit(struct xhci_hcd *xhci, u16 wValue, struct /* Should never happen */ return; } - /* Change bits are all write 1 to clear */ + /* Change bits are all write 1 to clear, also when held in software */ + port->held_changes &= ~status; xhci_portsc_writel(port, portsc | status); portsc = xhci_portsc_readl(port); @@ -1144,6 +1145,9 @@ static u32 xhci_get_port_status(struct usb_hcd *hcd, struct xhci_bus_state *bus_ rhub = xhci_get_rhub(hcd); port = rhub->ports[portnum]; + /* Changes held while running in suspend-to-idle count as set */ + portsc |= port->held_changes; + /* common wPortChange bits */ if (portsc & PORT_CSC) status |= USB_PORT_STAT_C_CONNECTION << 16; @@ -1660,6 +1664,8 @@ int xhci_hub_status_data(struct usb_hcd *hcd, char *buf) break; } trace_xhci_hub_status_data(ports[i], temp); + /* Changes held while running in suspend-to-idle count as set */ + temp |= ports[i]->held_changes & mask; if ((temp & mask) != 0 || (bus_state->port_c_suspend & 1 << i) || diff --git a/drivers/usb/host/xhci-plat-test.c b/drivers/usb/host/xhci-plat-test.c index 00646f7ba2f5a..e781f5662564b 100644 --- a/drivers/usb/host/xhci-plat-test.c +++ b/drivers/usb/host/xhci-plat-test.c @@ -4,12 +4,15 @@ */ #include +#include #include #include #include #include #include #include +#include +#include #include "xhci.h" #include "xhci-plat.h" @@ -140,6 +143,88 @@ static void xhci_plat_resume_from_runtime_ok_test(struct kunit *test) } #endif /* CONFIG_PM */ +/* + * Port changes seen while kept running in suspend-to-idle that are not + * wakeups. The first is from a J700 whose SuperSpeed link on the rear port + * kept dropping to SS.Inactive while its root hub was suspended with wakeup + * disabled: PLC, SS.Inactive, powered, no wake bits. + */ +static void xhci_s2idle_port_no_wakeup_test(struct kunit *test) +{ + static const u32 armed_bits = PORT_WKCONN_E | PORT_WKDISC_E | PORT_WKOC_E; + static const u32 changes[] = { + PORT_PLC | XDEV_INACTIVE | PORT_POWER, + PORT_PLC | XDEV_POLLING | PORT_POWER, + PORT_PLC | XDEV_RXDETECT | PORT_POWER, + PORT_PLC | XDEV_COMP_MODE | PORT_POWER, + /* A device that was resumed and is back in U0 or in U3 */ + PORT_PLC | XDEV_U0 | PORT_POWER | PORT_PE | PORT_CONNECT, + PORT_PLC | XDEV_U3 | PORT_POWER | PORT_PE | PORT_CONNECT, + /* Warm and port reset, enable and config error changes */ + PORT_WRC | PORT_RC | XDEV_U0 | PORT_POWER | PORT_PE | PORT_CONNECT, + PORT_PEC | XDEV_DISABLED | PORT_POWER | PORT_CONNECT, + PORT_CEC | XDEV_U0 | PORT_POWER | PORT_PE | PORT_CONNECT, + /* Resume state, but the link change was not what was reported */ + PORT_RC | XDEV_RESUME | PORT_POWER | PORT_PE | PORT_CONNECT, + }; + unsigned int i; + + KUNIT_EXPECT_FALSE(test, xhci_s2idle_port_wakeup(0x4002c0)); + + for (i = 0; i < ARRAY_SIZE(changes); i++) { + /* Not even with the root hub armed for wakeup */ + KUNIT_EXPECT_FALSE_MSG(test, xhci_s2idle_port_wakeup(changes[i]), + "portsc %#x", changes[i]); + KUNIT_EXPECT_FALSE_MSG(test, xhci_s2idle_port_wakeup(changes[i] | armed_bits), + "portsc %#x", changes[i] | armed_bits); + } +} + +/* Remote wakeup signalled by a device below the port wakes the system. */ +static void xhci_s2idle_port_remote_wakeup_test(struct kunit *test) +{ + u32 resume = PORT_PLC | XDEV_RESUME | PORT_POWER | PORT_PE | PORT_CONNECT; + + KUNIT_EXPECT_TRUE(test, xhci_s2idle_port_wakeup(resume)); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_port_wakeup(resume | PORT_WKDISC_E | PORT_WKOC_E)); +} + +/* Connect, disconnect and over-current wake it only if the port was armed for them. */ +static void xhci_s2idle_port_armed_change_test(struct kunit *test) +{ + u32 connect = PORT_CSC | PORT_CONNECT | XDEV_POLLING | PORT_POWER; + u32 disconnect = PORT_CSC | XDEV_RXDETECT | PORT_POWER; + u32 overcurrent = PORT_OCC | PORT_OC | XDEV_DISABLED | PORT_POWER; + + /* As xhci_bus_suspend() arms an empty and a connected port */ + KUNIT_EXPECT_TRUE(test, xhci_s2idle_port_wakeup(connect | PORT_WKCONN_E | PORT_WKOC_E)); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_port_wakeup(disconnect | PORT_WKDISC_E | PORT_WKOC_E)); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_port_wakeup(overcurrent | PORT_WKOC_E)); + + /* Root hub wakeup disabled: nothing armed */ + KUNIT_EXPECT_FALSE(test, xhci_s2idle_port_wakeup(connect)); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_port_wakeup(disconnect)); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_port_wakeup(overcurrent)); + + /* A wake bit only stands for its own change */ + KUNIT_EXPECT_FALSE(test, xhci_s2idle_port_wakeup(overcurrent | PORT_WKCONN_E | + PORT_WKDISC_E)); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_port_wakeup(PORT_PLC | XDEV_INACTIVE | PORT_POWER | + PORT_WKOC_E)); +} + +/* Only a hub kept awake for wakeup completing a transfer wakes the system. */ +static void xhci_s2idle_transfer_wakeup_test(struct kunit *test) +{ + struct usb_device *udev = kunit_kzalloc(test, sizeof(*udev), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, udev); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_transfer_wakeup(NULL)); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_transfer_wakeup(udev)); + udev->awake_in_sleep = 1; + KUNIT_EXPECT_TRUE(test, xhci_s2idle_transfer_wakeup(udev)); +} + static u64 xhci_plat_test_quirks(struct kunit *test, const struct property_entry *props) { struct device *dev; @@ -203,6 +288,10 @@ static struct kunit_case xhci_plat_test_cases[] = { KUNIT_CASE(xhci_plat_no_property_quirks_test), KUNIT_CASE(xhci_plat_run_in_s2idle_property_test), KUNIT_CASE(xhci_plat_all_property_quirks_test), + KUNIT_CASE(xhci_s2idle_port_no_wakeup_test), + KUNIT_CASE(xhci_s2idle_port_remote_wakeup_test), + KUNIT_CASE(xhci_s2idle_port_armed_change_test), + KUNIT_CASE(xhci_s2idle_transfer_wakeup_test), {} }; @@ -211,7 +300,866 @@ static struct kunit_suite xhci_plat_test_suite = { .test_cases = xhci_plat_test_cases, }; +#ifdef CONFIG_PM_SLEEP +/* + * The event handling of a controller with one SuperSpeed root port, faked in + * memory, whose root hub is suspended. PORTSC writes and port changes go + * through xhci_s2idle_test_portsc_writel() and xhci_s2idle_test_port_change(), + * which act as the controller does. + */ +struct xhci_s2idle_test { + struct usb_hcd *hcd; + struct xhci_hcd *xhci; + struct xhci_op_regs *op; + struct xhci_port_regs *port_regs; + __le32 __iomem *portsc; + union xhci_trb *trbs; + unsigned int events; + /* A change that the controller latches just before the next PORTSC write */ + u32 set_before_write; + /* The PORTSC read just before which the device below signals resume, from 1 */ + unsigned int resume_at_read; + unsigned int reads; + /* The generic xHCI root hub methods, as the USB core calls them */ + struct hc_driver driver; + struct device *dev; + char name[32]; +}; + +static void xhci_s2idle_test_rh_resume(struct work_struct *work) +{ +} + +static void xhci_s2idle_test_rh_timer(struct timer_list *timer) +{ +} + +KUNIT_DEFINE_ACTION_WRAPPER(xhci_s2idle_test_rh_timer_delete, timer_delete_sync, + struct timer_list *); + +/* + * A PORTSC write as the controller takes it: the change bits and PED are + * cleared by writing 1, the link state is only written with LWS, port power + * and the wake bits take the value written, and the read-only bits keep + * theirs. + */ +static void xhci_s2idle_test_portsc_writel(struct xhci_port *port, u32 val) +{ + struct xhci_s2idle_test *t = kunit_get_current_test()->priv; + u32 w1c = PORT_PE | PORT_CHANGE_MASK; + u32 rw = PORT_POWER | PORT_WKCONN_E | PORT_WKDISC_E | PORT_WKOC_E; + u32 old = readl(&port->port_reg->portsc) | t->set_before_write; + u32 now = (old & ~(w1c | rw)) | (old & w1c & ~val) | (val & rw); + + if (val & PORT_LINK_STROBE) + now = (now & ~PORT_PLS_MASK) | (val & PORT_PLS_MASK); + t->set_before_write = 0; + writel(now, &port->port_reg->portsc); +} + +/* The controller posts a Port Status Change Event for root port 1 */ +static void xhci_s2idle_test_post_event(struct xhci_s2idle_test *t) +{ + union xhci_trb *trb = &t->trbs[t->events++]; + + trb->generic.field[0] = cpu_to_le32(1 << 24); + trb->generic.field[2] = cpu_to_le32(COMP_SUCCESS << 24); + trb->generic.field[3] = cpu_to_le32(TRB_TYPE(TRB_PORT_STATUS) | TRB_CYCLE); + writel(readl(&t->xhci->op_regs->status) | STS_EINT, &t->xhci->op_regs->status); +} + +/* + * The controller changes the port. It posts an event only for a change made + * while no change bit is set, see xHCI 1.2 section 4.19.2, and returns whether + * it did. + */ +static bool xhci_s2idle_test_port_change(struct xhci_s2idle_test *t, u32 clear, u32 set) +{ + u32 old = readl(t->portsc); + u32 now = (old & ~clear) | set; + + writel(now, t->portsc); + if ((old & PORT_CHANGE_MASK) || !(now & PORT_CHANGE_MASK)) + return false; + xhci_s2idle_test_post_event(t); + return true; +} + +/* The J700 rear port: its SuperSpeed link drops to SS.Inactive, as in 0x4002c0 */ +static bool xhci_s2idle_test_link_inactive(struct xhci_s2idle_test *t) +{ + return xhci_s2idle_test_port_change(t, PORT_PLS_MASK, XDEV_INACTIVE | PORT_PLC); +} + +/* A device is attached, and its link trains to U0 */ +static bool xhci_s2idle_test_connect(struct xhci_s2idle_test *t) +{ + return xhci_s2idle_test_port_change(t, PORT_PLS_MASK, + XDEV_U0 | PORT_CSC | PORT_CONNECT | PORT_PE); +} + +/* The device below the port is detached */ +static bool xhci_s2idle_test_disconnect(struct xhci_s2idle_test *t) +{ + u32 gone = PORT_PLS_MASK | DEV_SPEED_MASK | PORT_PE | PORT_CONNECT; + + return xhci_s2idle_test_port_change(t, gone, XDEV_RXDETECT | PORT_CSC); +} + +/* The port detects an over-current, and is disabled */ +static bool xhci_s2idle_test_overcurrent(struct xhci_s2idle_test *t) +{ + return xhci_s2idle_test_port_change(t, PORT_PLS_MASK, XDEV_DISABLED | PORT_OCC | PORT_OC); +} + +/* The device below the port signals resume */ +static bool xhci_s2idle_test_remote_wakeup(struct xhci_s2idle_test *t) +{ + return xhci_s2idle_test_port_change(t, PORT_PLS_MASK, XDEV_RESUME | PORT_PLC); +} + +/* A PORTSC read, with the device below signalling resume just before the one chosen */ +static u32 xhci_s2idle_test_portsc_readl(struct xhci_port *port) +{ + struct xhci_s2idle_test *t = kunit_get_current_test()->priv; + + if (++t->reads == t->resume_at_read) + xhci_s2idle_test_remote_wakeup(t); + return readl(&port->port_reg->portsc); +} + +static unsigned int xhci_s2idle_test_count; + +/* The port at @portsc, with an event posted for its change bits if any are set */ +static struct xhci_s2idle_test *xhci_s2idle_test_init(struct kunit *test, u32 usbsts, u32 portsc) +{ + static const struct xhci_driver_overrides no_overrides; + struct xhci_s2idle_test *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + struct xhci_interrupter *ir, **irs; + struct xhci_intr_reg *ir_set; + struct xhci_segment *seg; + struct xhci_ring *ring; + struct xhci_port *port, **ports; + union xhci_trb *trbs; + struct xhci_hcd *xhci; + + KUNIT_ASSERT_NOT_NULL(test, t); + t->hcd = kunit_kzalloc(test, sizeof(*t->hcd) + sizeof(*xhci), GFP_KERNEL); + t->op = kunit_kzalloc(test, sizeof(*t->op), GFP_KERNEL); + t->port_regs = kunit_kzalloc(test, sizeof(*t->port_regs), GFP_KERNEL); + ir = kunit_kzalloc(test, sizeof(*ir), GFP_KERNEL); + irs = kunit_kzalloc(test, sizeof(*irs), GFP_KERNEL); + ring = kunit_kzalloc(test, sizeof(*ring), GFP_KERNEL); + seg = kunit_kzalloc(test, sizeof(*seg), GFP_KERNEL); + trbs = kunit_kcalloc(test, TRBS_PER_SEGMENT, sizeof(*trbs), GFP_KERNEL); + port = kunit_kzalloc(test, sizeof(*port), GFP_KERNEL); + ports = kunit_kzalloc(test, sizeof(*ports), GFP_KERNEL); + KUNIT_ASSERT_TRUE(test, t->hcd && t->op && t->port_regs && ir && irs && ring && seg && + trbs && port && ports); + ir_set = kunit_kzalloc(test, sizeof(*ir_set), GFP_KERNEL); + t->hcd->self.root_hub = kunit_kzalloc(test, sizeof(*t->hcd->self.root_hub), GFP_KERNEL); + KUNIT_ASSERT_TRUE(test, ir_set && t->hcd->self.root_hub); + ir->ir_set = (__force struct xhci_intr_reg __iomem *)ir_set; + + test->priv = t; + kunit_activate_static_stub(test, xhci_portsc_writel, xhci_s2idle_test_portsc_writel); + + /* + * The controller device may wake the system, as for running in s2idle. + * A test may set up more than one controller, so number them. + */ + snprintf(t->name, sizeof(t->name), "xhci-s2idle-test.%u", xhci_s2idle_test_count++); + t->dev = kunit_device_register(test, t->name); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->dev); + KUNIT_ASSERT_EQ(test, devm_device_init_wakeup(t->dev), 0); + + xhci = hcd_to_xhci(t->hcd); + t->xhci = xhci; + spin_lock_init(&xhci->lock); + xhci->main_hcd = t->hcd; + xhci->shared_hcd = t->hcd; + xhci->op_regs = (__force struct xhci_op_regs __iomem *)t->op; + xhci_init_driver(&t->driver, &no_overrides); + t->hcd->driver = &t->driver; + t->hcd->self.controller = t->dev; + t->hcd->speed = HCD_USB3; + t->hcd->state = HC_STATE_SUSPENDED; + /* A registered and pollable root hub, as for usb_hcd_resume_root_hub() */ + t->hcd->rh_registered = 1; + t->hcd->rh_pollable = 1; + t->hcd->uses_new_polling = 1; + spin_lock_init(&t->hcd->self.root_hub->dev.power.lock); + INIT_WORK(&t->hcd->wakeup_work, xhci_s2idle_test_rh_resume); + timer_setup(&t->hcd->rh_timer, xhci_s2idle_test_rh_timer, 0); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, xhci_s2idle_test_rh_timer_delete, + &t->hcd->rh_timer), 0); + + seg->trbs = trbs; + seg->dma = 0x10000; + seg->next = seg; + ring->type = TYPE_EVENT; + ring->first_seg = seg; + ring->deq_seg = seg; + ring->dequeue = trbs; + ring->cycle_state = 1; + ir->event_ring = ring; + irs[0] = ir; + xhci->interrupters = irs; + t->trbs = trbs; + + port->port_reg = (__force struct xhci_port_regs __iomem *)t->port_regs; + port->rhub = &xhci->usb3_rhub; + init_completion(&port->rexit_done); + init_completion(&port->u3exit_done); + ports[0] = port; + xhci->hw_ports = port; + xhci->max_ports = 1; + xhci->usb3_rhub.ports = ports; + xhci->usb3_rhub.num_ports = 1; + xhci->usb3_rhub.hcd = t->hcd; + t->portsc = &port->port_reg->portsc; + + writel(CMD_RUN | CMD_EIE, &xhci->op_regs->command); + writel(usbsts, &xhci->op_regs->status); + writel(portsc, t->portsc); + if (portsc & PORT_CHANGE_MASK) + xhci_s2idle_test_post_event(t); + + return t; +} + +/* Make the port a USB 2 one, on the root hub of a USB 2 bus */ +static void xhci_s2idle_test_usb2(struct xhci_s2idle_test *t) +{ + struct xhci_hcd *xhci = t->xhci; + + xhci->usb2_rhub = xhci->usb3_rhub; + memset(&xhci->usb3_rhub, 0, sizeof(xhci->usb3_rhub)); + xhci->hw_ports->rhub = &xhci->usb2_rhub; + t->hcd->speed = HCD_USB2; +} + +static unsigned long xhci_s2idle_test_wakeups(struct xhci_s2idle_test *t) +{ + return t->dev->power.wakeup->event_count; +} + +/* + * Whether the event handling took on a remote wakeup: the resume timing of a + * USB 2 port runs, a SuperSpeed link was moved to U0. + */ +static bool xhci_s2idle_test_resuming(struct xhci_s2idle_test *t) +{ + if (t->hcd->speed < HCD_USB3) + return t->xhci->hw_ports->resume_timestamp && + test_bit(0, &t->xhci->usb2_rhub.bus_state.resuming_ports); + return (readl(t->portsc) & PORT_PLS_MASK) == XDEV_U0; +} + +/* Whether the root hub was resumed or polled for the event */ +static bool xhci_s2idle_test_rh_resumed(struct xhci_s2idle_test *t) +{ + flush_work(&t->hcd->wakeup_work); + return test_bit(HCD_FLAG_WAKEUP_PENDING, &t->hcd->flags); +} + +static bool xhci_s2idle_test_rh_polled(struct xhci_s2idle_test *t) +{ + return test_bit(HCD_FLAG_POLL_RH, &t->hcd->flags); +} + +/* Whether the root hub reports a change of the port to the hub driver */ +static bool xhci_s2idle_test_rh_change(struct xhci_s2idle_test *t) +{ + char buf[2] = {}; + + return t->driver.hub_status_data(t->hcd, buf) > 0 && (buf[0] & BIT(1)); +} + +/* The port status and changes that the hub driver reads */ +static u32 xhci_s2idle_test_port_status(struct kunit *test, struct xhci_s2idle_test *t) +{ + __le32 status; + + KUNIT_ASSERT_EQ(test, t->driver.hub_control(t->hcd, GetPortStatus, 0, 1, (char *)&status, + sizeof(status)), 0); + return le32_to_cpu(status); +} + +static void xhci_s2idle_test_clear_change(struct kunit *test, struct xhci_s2idle_test *t, + u16 feature) +{ + KUNIT_ASSERT_EQ(test, t->driver.hub_control(t->hcd, ClearPortFeature, feature, 1, NULL, 0), + 0); +} + +static irqreturn_t xhci_s2idle_test_irq(struct xhci_s2idle_test *t) +{ + irqreturn_t ret; + + local_irq_disable(); + ret = xhci_irq(t->hcd); + local_irq_enable(); + return ret; +} + +/* A SuperSpeed port in link training, with nothing connected */ +#define XHCI_S2IDLE_TEST_EMPTY (XDEV_POLLING | PORT_POWER) +/* A SuperSpeed port with a device suspended below it */ +#define XHCI_S2IDLE_TEST_SUSPENDED (XDEV_U3 | XDEV_SS | PORT_POWER | PORT_PE | PORT_CONNECT) +/* A USB 2 port with a high-speed device suspended below it */ +#define XHCI_S2IDLE_TEST_USB2_SUSPENDED (XDEV_U3 | XDEV_HS | PORT_POWER | PORT_PE | PORT_CONNECT) + +/* + * The J700 failure: a SuperSpeed link drops to SS.Inactive while running in + * suspend-to-idle. No wakeup, and the root hub is neither resumed nor polled. + * The change is cleared in PORTSC, so that the port posts an event for its + * next one, and held for the hub driver. + */ +static void xhci_s2idle_link_change_held_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_EMPTY); + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_link_inactive(t)); + KUNIT_ASSERT_EQ(test, readl(t->portsc), 0x4002c0U); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_irq(t), IRQ_HANDLED); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_test_rh_resumed(t)); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_test_rh_polled(t)); + KUNIT_EXPECT_EQ(test, readl(t->portsc) & PORT_CHANGE_MASK, 0U); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_port_status(test, t) >> 16, + (u32)USB_PORT_STAT_C_LINK_STATE); + xhci_run_in_s2idle_end(t->xhci); +} + +/* + * On resume the root hub is resumed and polled again, and the hub driver reads + * the held change as if it had stayed in PORTSC, until it clears it. + */ +static void xhci_s2idle_link_change_resume_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_EMPTY); + u32 status; + + xhci_run_in_s2idle_begin(t->xhci); + xhci_s2idle_test_link_inactive(t); + xhci_s2idle_test_irq(t); + xhci_run_in_s2idle_end(t->xhci); + KUNIT_EXPECT_FALSE(test, t->xhci->running_in_s2idle); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_rh_resumed(t)); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_rh_polled(t)); + /* The root hub is suspended: the poll keeps the change for its status URB */ + KUNIT_EXPECT_TRUE(test, test_bit(HCD_FLAG_POLL_PENDING, &t->hcd->flags)); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_rh_change(t)); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + + status = xhci_s2idle_test_port_status(test, t); + KUNIT_EXPECT_EQ(test, status >> 16, (u32)USB_PORT_STAT_C_LINK_STATE); + KUNIT_EXPECT_EQ(test, status & USB_PORT_STAT_LINK_STATE, (u32)USB_SS_PORT_LS_SS_INACTIVE); + + xhci_s2idle_test_clear_change(test, t, USB_PORT_FEAT_C_PORT_LINK_STATE); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_port_status(test, t) >> 16, 0U); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_test_rh_change(t)); +} + +/* Outside suspend-to-idle the same change goes to the USB core at once, as before. */ +static void xhci_s2idle_link_change_awake_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, STS_EINT, 0x4002c0); + + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_irq(t), IRQ_HANDLED); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_rh_resumed(t)); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_rh_polled(t)); + KUNIT_EXPECT_TRUE(test, readl(t->portsc) & PORT_PLC); +} + +/* Remote wakeup from a SuperSpeed device below the port wakes the system. */ +static void xhci_s2idle_remote_wakeup_event_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_SUSPENDED); + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_remote_wakeup(t)); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_irq(t), IRQ_HANDLED); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_rh_resumed(t)); + /* The event handling took the remote wakeup on and moved the link to U0 */ + KUNIT_EXPECT_EQ(test, readl(t->portsc) & PORT_PLS_MASK, (u32)XDEV_U0); + xhci_run_in_s2idle_end(t->xhci); +} + +/* + * A held change does not keep the port from posting an event for the next + * one: a device attached to a port armed for connect wakes the system. + */ +static void xhci_s2idle_connect_after_held_change_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_EMPTY | + PORT_WKCONN_E | PORT_WKOC_E); + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_link_inactive(t)); + xhci_s2idle_test_irq(t); + KUNIT_ASSERT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_connect(t)); + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + xhci_run_in_s2idle_end(t->xhci); +} + +/* Nor an over-current on a port armed for it */ +static void xhci_s2idle_overcurrent_after_held_change_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_EMPTY | + PORT_WKCONN_E | PORT_WKOC_E); + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_link_inactive(t)); + xhci_s2idle_test_irq(t); + KUNIT_ASSERT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_overcurrent(t)); + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + xhci_run_in_s2idle_end(t->xhci); +} + +/* Nor the disconnect of a device whose link failed, on a port armed for it */ +static void xhci_s2idle_disconnect_after_held_change_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_SUSPENDED | + PORT_WKDISC_E | PORT_WKOC_E); + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_link_inactive(t)); + xhci_s2idle_test_irq(t); + KUNIT_ASSERT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_disconnect(t)); + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + xhci_run_in_s2idle_end(t->xhci); +} + +/* + * Nor a remote wakeup: the over-current change of a port whose root hub may + * not wake the system is held, and the suspended device below it then signals + * resume. + */ +static void xhci_s2idle_remote_wakeup_after_held_change_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_SUSPENDED); + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_port_change(t, 0, PORT_OCC)); + xhci_s2idle_test_irq(t); + KUNIT_ASSERT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_remote_wakeup(t)); + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + KUNIT_EXPECT_EQ(test, readl(t->portsc) & PORT_PLS_MASK, (u32)XDEV_U0); + xhci_run_in_s2idle_end(t->xhci); + KUNIT_EXPECT_TRUE(test, (xhci_s2idle_test_port_status(test, t) >> 16) & + USB_PORT_STAT_C_OVERCURRENT); +} + +/* + * A connect that the controller latches while the link change is cleared + * posts no event, as the port still had a change bit set. The event handling + * reads PORTSC again after the write, and wakes the system for it. + */ +static void xhci_s2idle_connect_during_hold_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_EMPTY | + PORT_WKCONN_E | PORT_WKOC_E); + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_link_inactive(t)); + t->set_before_write = PORT_CSC | PORT_CONNECT; + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_EQ(test, t->set_before_write, 0U); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + /* Left in PORTSC, for the hub driver after the wakeup */ + KUNIT_EXPECT_TRUE(test, readl(t->portsc) & PORT_CSC); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_rh_resumed(t)); + xhci_run_in_s2idle_end(t->xhci); +} + +/* + * One that does not wake the system is held as well, so the port is left with + * no change bit set, and the hub driver reads both changes on resume. + */ +static void xhci_s2idle_change_during_hold_test(struct kunit *test) +{ + /* The root hub may not wake the system, so the port is not armed */ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_EMPTY); + u32 status; + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_link_inactive(t)); + t->set_before_write = PORT_CSC | PORT_CONNECT; + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_EQ(test, t->set_before_write, 0U); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + KUNIT_EXPECT_EQ(test, readl(t->portsc) & PORT_CHANGE_MASK, 0U); + xhci_run_in_s2idle_end(t->xhci); + + status = xhci_s2idle_test_port_status(test, t); + KUNIT_EXPECT_EQ(test, status >> 16, + (u32)(USB_PORT_STAT_C_CONNECTION | USB_PORT_STAT_C_LINK_STATE)); + KUNIT_EXPECT_TRUE(test, status & USB_PORT_STAT_CONNECTION); +} + +/* + * A change that the port latched before, and that the hub driver had no + * chance to clear, is held once running in suspend-to-idle begins: the port + * then posts an event for the remote wakeup of the device below it. + */ +static void xhci_s2idle_change_before_begin_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_SUSPENDED); + + /* Its event comes in before, and the change is left for the hub driver */ + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_port_change(t, 0, PORT_OCC)); + xhci_s2idle_test_irq(t); + KUNIT_ASSERT_TRUE(test, readl(t->portsc) & PORT_OCC); + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + KUNIT_EXPECT_EQ(test, readl(t->portsc) & PORT_CHANGE_MASK, 0U); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_remote_wakeup(t)); + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + xhci_run_in_s2idle_end(t->xhci); + KUNIT_EXPECT_TRUE(test, (xhci_s2idle_test_port_status(test, t) >> 16) & + USB_PORT_STAT_C_OVERCURRENT); +} + +/* + * One that wakes the system wakes it when running in suspend-to-idle begins, + * and is left in PORTSC. + */ +static void xhci_s2idle_wakeup_before_begin_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_EMPTY | + PORT_WKCONN_E | PORT_WKOC_E); + + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_connect(t)); + xhci_s2idle_test_irq(t); + KUNIT_ASSERT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + KUNIT_EXPECT_TRUE(test, readl(t->portsc) & PORT_CSC); + xhci_run_in_s2idle_end(t->xhci); +} + +/* A USB 2 remote wakeup wakes the system and starts the resume timing. */ +static void xhci_s2idle_usb2_remote_wakeup_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, + XHCI_S2IDLE_TEST_USB2_SUSPENDED); + + xhci_s2idle_test_usb2(t); + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_remote_wakeup(t)); + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_resuming(t)); + xhci_run_in_s2idle_end(t->xhci); +} + +/* + * A remote wakeup that starts while PLC is set, as after U3 entry, sets no new + * PLC and so posts no event, and clearing the old PLC removes its only change + * bit. Its link state still shows it: the system wakes, and the event handling + * takes the resume on. + */ +static void xhci_s2idle_resume_during_plc_clear(struct kunit *test, u32 portsc, bool usb2) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, portsc); + + if (usb2) + xhci_s2idle_test_usb2(t); + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_port_change(t, 0, PORT_PLC)); + t->set_before_write = PORT_PLC | XDEV_RESUME; + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_EQ(test, t->set_before_write, 0U); + KUNIT_EXPECT_EQ(test, t->events, 1U); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_resuming(t)); + xhci_run_in_s2idle_end(t->xhci); +} + +static void xhci_s2idle_resume_during_plc_clear_test(struct kunit *test) +{ + xhci_s2idle_resume_during_plc_clear(test, XHCI_S2IDLE_TEST_SUSPENDED, false); +} + +static void xhci_s2idle_usb2_resume_during_plc_clear_test(struct kunit *test) +{ + xhci_s2idle_resume_during_plc_clear(test, XHCI_S2IDLE_TEST_USB2_SUSPENDED, true); +} + +/* So it does for a PLC still set when running in suspend-to-idle begins. */ +static void xhci_s2idle_resume_during_begin_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_SUSPENDED); + + writel(readl(t->portsc) | PORT_PLC, t->portsc); + t->set_before_write = PORT_PLC | XDEV_RESUME; + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_EXPECT_EQ(test, t->set_before_write, 0U); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_resuming(t)); + xhci_run_in_s2idle_end(t->xhci); +} + +/* + * A remote wakeup at any point of the decision on an over-current change that + * does not wake the system: the device signals resume just before each PORTSC + * read in turn. While the change is set, its PLC posts no event of its own. The + * system wakes all the same, and the resume is taken on. When the decision is + * over before the chosen read, the change is held and nothing wakes. + */ +static void xhci_s2idle_resume_during_hold(struct kunit *test, u32 portsc, bool usb2, + bool latched) +{ + unsigned int read; + + for (read = 1; ; read++) { + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, portsc); + + if (usb2) + xhci_s2idle_test_usb2(t); + kunit_activate_static_stub(test, xhci_portsc_readl, + xhci_s2idle_test_portsc_readl); + if (latched) { + /* Left for the hub driver before running in suspend-to-idle */ + writel(readl(t->portsc) | PORT_OCC, t->portsc); + t->resume_at_read = read; + xhci_run_in_s2idle_begin(t->xhci); + } else { + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_port_change(t, 0, PORT_OCC)); + t->reads = 0; + t->resume_at_read = read; + } + xhci_s2idle_test_irq(t); + + if (t->reads < read) { + KUNIT_EXPECT_GE(test, read, 3U); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + KUNIT_EXPECT_EQ(test, t->xhci->hw_ports->held_changes, (u32)PORT_OCC); + xhci_run_in_s2idle_end(t->xhci); + break; + } + KUNIT_EXPECT_GE_MSG(test, xhci_s2idle_test_wakeups(t), 1UL, + "resume before read %u", read); + KUNIT_EXPECT_TRUE_MSG(test, xhci_s2idle_test_resuming(t), + "resume before read %u", read); + xhci_run_in_s2idle_end(t->xhci); + } +} + +static void xhci_s2idle_resume_during_hold_test(struct kunit *test) +{ + xhci_s2idle_resume_during_hold(test, XHCI_S2IDLE_TEST_SUSPENDED, false, false); +} + +static void xhci_s2idle_usb2_resume_during_hold_test(struct kunit *test) +{ + xhci_s2idle_resume_during_hold(test, XHCI_S2IDLE_TEST_USB2_SUSPENDED, true, false); +} + +static void xhci_s2idle_resume_during_begin_hold_test(struct kunit *test) +{ + xhci_s2idle_resume_during_hold(test, XHCI_S2IDLE_TEST_SUSPENDED, false, true); +} + +static void xhci_s2idle_usb2_resume_during_begin_hold_test(struct kunit *test) +{ + xhci_s2idle_resume_during_hold(test, XHCI_S2IDLE_TEST_USB2_SUSPENDED, true, true); +} + +/* + * Every change the hub driver reads is held across the sleep, and each is + * cleared on its own by ClearPortFeature after resume. None comes back in the + * next sleep. + */ +static void xhci_s2idle_held_clear_test(struct kunit *test) +{ + static const struct { + u32 hw, usb; + u16 feature; + } changes[] = { + { PORT_CSC, USB_PORT_STAT_C_CONNECTION, USB_PORT_FEAT_C_CONNECTION }, + { PORT_OCC, USB_PORT_STAT_C_OVERCURRENT, USB_PORT_FEAT_C_OVER_CURRENT }, + { PORT_RC, USB_PORT_STAT_C_RESET, USB_PORT_FEAT_C_RESET }, + { PORT_WRC, USB_PORT_STAT_C_BH_RESET, USB_PORT_FEAT_C_BH_PORT_RESET }, + { PORT_PLC, USB_PORT_STAT_C_LINK_STATE, USB_PORT_FEAT_C_PORT_LINK_STATE }, + { PORT_CEC, USB_PORT_STAT_C_CONFIG_ERROR, USB_PORT_FEAT_C_PORT_CONFIG_ERROR }, + }; + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_SUSPENDED); + u32 hw = 0, usb = 0; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(changes); i++) { + hw |= changes[i].hw; + usb |= changes[i].usb; + } + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_port_change(t, 0, hw)); + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + KUNIT_EXPECT_EQ(test, readl(t->portsc) & PORT_CHANGE_MASK, 0U); + xhci_run_in_s2idle_end(t->xhci); + + for (i = 0; i < ARRAY_SIZE(changes); i++) { + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_port_status(test, t) >> 16, usb); + xhci_s2idle_test_clear_change(test, t, changes[i].feature); + usb &= ~changes[i].usb; + } + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_port_status(test, t) >> 16, 0U); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_test_rh_change(t)); + + xhci_run_in_s2idle_begin(t->xhci); + xhci_run_in_s2idle_end(t->xhci); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_port_status(test, t) >> 16, 0U); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_test_rh_change(t)); +} + +/* + * A USB 2 link change is cleared but not held: the hub driver never reads one, + * it reads the end of a resume from port_c_suspend instead. + */ +static void xhci_s2idle_usb2_plc_not_held_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, + XHCI_S2IDLE_TEST_USB2_SUSPENDED); + + xhci_s2idle_test_usb2(t); + writel(readl(t->portsc) | PORT_PLC | PORT_OCC, t->portsc); + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + KUNIT_EXPECT_EQ(test, t->xhci->hw_ports->held_changes, (u32)PORT_OCC); + KUNIT_EXPECT_EQ(test, readl(t->portsc) & PORT_CHANGE_MASK, 0U); + xhci_run_in_s2idle_end(t->xhci); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_port_status(test, t) >> 16, + (u32)USB_PORT_STAT_C_OVERCURRENT); + xhci_s2idle_test_clear_change(test, t, USB_PORT_FEAT_C_OVER_CURRENT); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_test_rh_change(t)); +} + +/* + * Root hub polling stops until resume, as for a halted controller: the event + * handling decides on every change, and a poll while the system sleeps would + * only pass held changes to the hub driver of a root hub kept awake. + */ +static void xhci_s2idle_rh_polling_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_EMPTY); + + /* Polling, as for a change shortly before the system went to sleep */ + set_bit(HCD_FLAG_POLL_RH, &t->hcd->flags); + mod_timer(&t->hcd->rh_timer, jiffies + HZ); + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_test_rh_polled(t)); + KUNIT_EXPECT_FALSE(test, timer_pending(&t->hcd->rh_timer)); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_link_inactive(t)); + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_test_rh_polled(t)); + KUNIT_EXPECT_FALSE(test, timer_pending(&t->hcd->rh_timer)); + + xhci_run_in_s2idle_end(t->xhci); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_rh_polled(t)); + KUNIT_EXPECT_TRUE(test, timer_pending(&t->hcd->rh_timer)); +} + +/* A controller error halts it and wakes the system, so that resume deals with it. */ +static void xhci_s2idle_controller_error_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, STS_EINT | STS_HCE | STS_HALT, + 0); + + xhci_run_in_s2idle_begin(t->xhci); + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_TRUE(test, t->xhci->xhc_state & XHCI_STATE_HALTED); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + xhci_run_in_s2idle_end(t->xhci); +} + +/* + * So it does if the failing controller never halts, with its interrupts + * masked all the same, for a Host Controller and a Host System Error. + */ +static void xhci_s2idle_error_no_halt(struct kunit *test, u32 error) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, STS_EINT | error, 0); + + xhci_run_in_s2idle_begin(t->xhci); + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_FALSE(test, t->xhci->xhc_state & XHCI_STATE_HALTED); + KUNIT_EXPECT_EQ(test, readl(&t->xhci->op_regs->command) & (CMD_RUN | XHCI_IRQS), 0U); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + xhci_run_in_s2idle_end(t->xhci); +} + +static void xhci_s2idle_controller_error_no_halt_test(struct kunit *test) +{ + xhci_s2idle_error_no_halt(test, STS_HCE); +} + +static void xhci_s2idle_system_error_no_halt_test(struct kunit *test) +{ + xhci_s2idle_error_no_halt(test, STS_FATAL); +} + +static struct kunit_case xhci_s2idle_test_cases[] = { + KUNIT_CASE(xhci_s2idle_link_change_held_test), + KUNIT_CASE(xhci_s2idle_link_change_resume_test), + KUNIT_CASE(xhci_s2idle_link_change_awake_test), + KUNIT_CASE(xhci_s2idle_remote_wakeup_event_test), + KUNIT_CASE(xhci_s2idle_connect_after_held_change_test), + KUNIT_CASE(xhci_s2idle_overcurrent_after_held_change_test), + KUNIT_CASE(xhci_s2idle_disconnect_after_held_change_test), + KUNIT_CASE(xhci_s2idle_remote_wakeup_after_held_change_test), + KUNIT_CASE(xhci_s2idle_connect_during_hold_test), + KUNIT_CASE(xhci_s2idle_change_during_hold_test), + KUNIT_CASE(xhci_s2idle_change_before_begin_test), + KUNIT_CASE(xhci_s2idle_wakeup_before_begin_test), + KUNIT_CASE(xhci_s2idle_usb2_remote_wakeup_test), + KUNIT_CASE(xhci_s2idle_resume_during_plc_clear_test), + KUNIT_CASE(xhci_s2idle_usb2_resume_during_plc_clear_test), + KUNIT_CASE(xhci_s2idle_resume_during_begin_test), + KUNIT_CASE(xhci_s2idle_resume_during_hold_test), + KUNIT_CASE(xhci_s2idle_usb2_resume_during_hold_test), + KUNIT_CASE(xhci_s2idle_resume_during_begin_hold_test), + KUNIT_CASE(xhci_s2idle_usb2_resume_during_begin_hold_test), + KUNIT_CASE(xhci_s2idle_held_clear_test), + KUNIT_CASE(xhci_s2idle_usb2_plc_not_held_test), + KUNIT_CASE(xhci_s2idle_rh_polling_test), + KUNIT_CASE(xhci_s2idle_controller_error_test), + KUNIT_CASE(xhci_s2idle_controller_error_no_halt_test), + KUNIT_CASE(xhci_s2idle_system_error_no_halt_test), + {} +}; + +static struct kunit_suite xhci_s2idle_test_suite = { + .name = "xhci_s2idle", + .test_cases = xhci_s2idle_test_cases, +}; + +kunit_test_suites(&xhci_plat_test_suite, &xhci_s2idle_test_suite); +#else kunit_test_suite(xhci_plat_test_suite); +#endif /* CONFIG_PM_SLEEP */ MODULE_DESCRIPTION("KUnit tests for the generic xHCI platform driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index aec29654680e3..d38d96ec37a56 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -580,7 +580,7 @@ static int xhci_plat_suspend(struct device *dev) if (ret) return ret; - priv->running_in_s2idle = 1; + xhci_run_in_s2idle_begin(hcd_to_xhci(hcd)); dev_dbg(dev, "wakeup enabled, keep running in suspend-to-idle.\n"); return 0; } @@ -643,6 +643,7 @@ static int xhci_plat_resume(struct device *dev) { struct usb_hcd *hcd = dev_get_drvdata(dev); struct xhci_plat_priv *priv = hcd_to_xhci_priv(hcd); + struct xhci_hcd *xhci = hcd_to_xhci(hcd); if (priv->sideband_at_suspend) { priv->sideband_at_suspend = 0; @@ -650,12 +651,11 @@ static int xhci_plat_resume(struct device *dev) return 0; } - if (priv->running_in_s2idle) { - struct xhci_hcd *xhci = hcd_to_xhci(hcd); + if (xhci->running_in_s2idle) { u32 status = readl(&xhci->op_regs->status); - priv->running_in_s2idle = 0; if (!xhci_plat_stopped(status)) { + xhci_run_in_s2idle_end(xhci); dev_dbg(dev, "kept running in suspend-to-idle, skip resume.\n"); return 0; } @@ -675,6 +675,7 @@ static int xhci_plat_resume(struct device *dev) xhci_quiesce_dying(xhci); xhci_hc_died(xhci); spin_unlock_irq(&xhci->lock); + xhci_run_in_s2idle_end(xhci); return 0; } diff --git a/drivers/usb/host/xhci-plat.h b/drivers/usb/host/xhci-plat.h index da55117cbf3f3..d002aac38ec20 100644 --- a/drivers/usb/host/xhci-plat.h +++ b/drivers/usb/host/xhci-plat.h @@ -17,7 +17,6 @@ struct xhci_plat_priv { unsigned long long quirks; bool power_lost; unsigned sideband_at_suspend:1; - unsigned running_in_s2idle:1; void (*plat_start)(struct usb_hcd *); int (*init_quirk)(struct usb_hcd *); int (*suspend_quirk)(struct usb_hcd *); diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c index 429c34b38788b..c6f4df660cdb7 100644 --- a/drivers/usb/host/xhci-ring.c +++ b/drivers/usb/host/xhci-ring.c @@ -52,6 +52,7 @@ * endpoint rings; it generates events on the event ring for these. */ +#include #include #include #include @@ -1957,6 +1958,113 @@ static void handle_vendor_event(struct xhci_hcd *xhci, handle_cmd_completion(xhci, &event->event_cmd); } +/* + * A controller kept running in suspend-to-idle interrupts for every event, see + * xhci_run_in_s2idle_begin(). The event handling decides which of them wake + * the system, as the platform has no other wake logic that would. + */ +void xhci_s2idle_wakeup(struct xhci_hcd *xhci) +{ + pm_wakeup_hard_event(xhci_to_hcd(xhci)->self.controller); +} + +/* + * Whether a port change is one that a halted controller would wake the system + * on: resume signalling, which a device only sends once the USB core enabled + * its remote wakeup, or a connect, disconnect or over-current change that + * xhci_bus_suspend() armed the port for, which it does only when the root hub + * may wake the system. Any other change, such as a SuperSpeed link going to + * SS.Inactive, does not wake it. + */ +VISIBLE_IF_KUNIT bool xhci_s2idle_port_wakeup(u32 portsc) +{ + if ((portsc & PORT_PLC) && (portsc & PORT_PLS_MASK) == XDEV_RESUME) + return true; + + if ((portsc & PORT_CSC) && (portsc & (PORT_WKCONN_E | PORT_WKDISC_E))) + return true; + + return (portsc & PORT_OCC) && (portsc & PORT_WKOC_E); +} +EXPORT_SYMBOL_IF_KUNIT(xhci_s2idle_port_wakeup); + +/** + * xhci_s2idle_port_change - Decide on a port change in suspend-to-idle + * @xhci: The controller, running in suspend-to-idle, with its lock held + * @port: The port + * @portsc: PORTSC as the event handling read it, updated as it is read again + * + * A port posts a Port Status Change Event only for a change made while none of + * its change bits is set, see xHCI 1.2 section 4.19.2. Left set in PORTSC until + * the hub driver clears it after resume, a change that does not wake the system + * would keep the port from posting one for a later change that does, such as a + * connect. Clear the change bits and hold them in software instead, where + * xhci_get_port_status() and xhci_hub_status_data() report them as before. + * + * Clearing them is part of the decision. A change that the port latched in the + * meantime, while other change bits were still set, posted no event, so PORTSC + * is read and judged again after each write. And a remote wakeup that started + * while PLC was set sets no new PLC, so clearing PLC removes its only change + * bit: if the link is then in Resume, PLC is put back into @portsc for the + * event handling, as if it had stayed set. + * + * Return: true if a change of the port wakes the system, or if the port keeps + * changing. The event handling then carries on with @portsc. + */ +static bool xhci_s2idle_port_change(struct xhci_hcd *xhci, struct xhci_port *port, + u32 *portsc) +{ + u32 reported = PORT_CHANGE_MASK; + int retries = 4; + + /* The hub driver sees no USB 2 link state change, the event handling deals with it */ + if (port->rhub == &xhci->usb2_rhub) + reported &= ~PORT_PLC; + + while (!xhci_s2idle_port_wakeup(*portsc)) { + u32 changes = *portsc & PORT_CHANGE_MASK; + + if (!changes) + return false; + if (retries-- == 0) + return true; + + xhci_dbg(xhci, "port %d-%d: no s2idle wakeup, change left for resume\n", + port->rhub->hcd->self.busnum, port->hcd_portnum + 1); + port->held_changes |= changes & reported; + xhci_portsc_writel(port, xhci_port_state_to_neutral(*portsc) | changes); + *portsc = xhci_portsc_readl(port); + + if ((changes & PORT_PLC) && (*portsc & PORT_PLS_MASK) == XDEV_RESUME) + *portsc |= PORT_PLC; + } + + return true; +} + +/* + * Only hubs that the USB core kept awake to pass on a remote wakeup from below + * have transfers in flight, see usb_suspend_both(): a completed one reports + * a change on one of their ports. + */ +VISIBLE_IF_KUNIT bool xhci_s2idle_transfer_wakeup(const struct usb_device *udev) +{ + return udev && udev->awake_in_sleep; +} +EXPORT_SYMBOL_IF_KUNIT(xhci_s2idle_transfer_wakeup); + +static void xhci_s2idle_transfer_event(struct xhci_hcd *xhci, + struct xhci_transfer_event *event) +{ + unsigned int slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags)); + struct xhci_virt_device *vdev = xhci->devs[slot_id]; + + if (vdev && xhci_s2idle_transfer_wakeup(vdev->udev)) { + xhci_dbg(xhci, "slot %u transfer: s2idle wakeup\n", slot_id); + xhci_s2idle_wakeup(xhci); + } +} + static void handle_device_notification(struct xhci_hcd *xhci, union xhci_trb *event) { @@ -1972,6 +2080,8 @@ static void handle_device_notification(struct xhci_hcd *xhci, xhci_dbg(xhci, "Device Wake Notification event for slot ID %u\n", slot_id); + if (xhci->running_in_s2idle) + xhci_s2idle_wakeup(xhci); udev = xhci->devs[slot_id]->udev; if (udev && udev->parent) usb_wakeup_notification(udev->parent, udev->portnum); @@ -2058,11 +2168,6 @@ static void handle_port_status(struct xhci_hcd *xhci, union xhci_trb *event) trace_xhci_handle_port_status(port, portsc); - if (hcd->state == HC_STATE_SUSPENDED) { - xhci_dbg(xhci, "resume root hub\n"); - usb_hcd_resume_root_hub(hcd); - } - if (vdev && (portsc & PORT_PLS_MASK) == XDEV_INACTIVE) { if (!(portsc & PORT_RESET)) vdev->flags |= VDEV_PORT_ERROR; @@ -2070,6 +2175,26 @@ static void handle_port_status(struct xhci_hcd *xhci, union xhci_trb *event) vdev->flags &= ~VDEV_PORT_ERROR; } + /* + * Running in suspend-to-idle, wake the system only as a halted + * controller would. Hold any other change for the USB core until + * resume, and do not resume or poll the root hub for it now. Nothing + * below may clear a change bit before this decision, see + * xhci_s2idle_port_change(). + */ + if (xhci->running_in_s2idle) { + if (!xhci_s2idle_port_change(xhci, port, &portsc)) + return; + xhci_dbg(xhci, "port %d-%d: s2idle wakeup\n", + hcd->self.busnum, hcd_portnum + 1); + xhci_s2idle_wakeup(xhci); + } + + if (hcd->state == HC_STATE_SUSPENDED) { + xhci_dbg(xhci, "resume root hub\n"); + usb_hcd_resume_root_hub(hcd); + } + if ((portsc & PORT_PLC) && (portsc & PORT_PLS_MASK) == XDEV_RESUME) { xhci_dbg(xhci, "port resume event for port %d\n", port_id); @@ -2180,6 +2305,33 @@ static void handle_port_status(struct xhci_hcd *xhci, union xhci_trb *event) spin_lock(&xhci->lock); } +/** + * xhci_s2idle_latched_changes - Decide on the changes latched before suspend-to-idle + * @xhci: The controller, just set running in suspend-to-idle, with its lock held + * + * A port whose change bits are still set, such as one whose changes the hub + * driver had no chance to clear before it froze, posts no event for its next + * change. Handle each such port as if it had just posted an event, so that its + * changes are held, or wake the system, as for any other event. + */ +void xhci_s2idle_latched_changes(struct xhci_hcd *xhci) +{ + union xhci_trb event = {}; + unsigned int i; + + event.generic.field[2] = cpu_to_le32(COMP_SUCCESS << 24); + for (i = 0; i < xhci->max_ports; i++) { + struct xhci_port *port = &xhci->hw_ports[i]; + + if (!port->rhub || !port->rhub->hcd || + port->hcd_portnum == DUPLICATE_ENTRY || + !(xhci_portsc_readl(port) & PORT_CHANGE_MASK)) + continue; + event.generic.field[0] = cpu_to_le32((i + 1) << 24); + handle_port_status(xhci, &event); + } +} + static void xhci_clear_hub_tt_buffer(struct xhci_hcd *xhci, struct xhci_td *td, struct xhci_virt_ep *ep) { @@ -3018,6 +3170,8 @@ static int xhci_handle_event_trb(struct xhci_hcd *xhci, struct xhci_interrupter handle_port_status(xhci, event); break; case TRB_TRANSFER: + if (xhci->running_in_s2idle) + xhci_s2idle_transfer_event(xhci, &event->trans_event); handle_tx_event(xhci, ir, &event->trans_event); break; case TRB_DEV_NOTE: @@ -3225,6 +3379,15 @@ irqreturn_t xhci_irq(struct usb_hcd *hcd) /* This is the handler of the primary interrupter */ xhci_handle_events(xhci, xhci->interrupters[0], false); out: + /* + * A controller that stopped or failed in suspend-to-idle is dealt with + * on resume. A Host Controller or Host System Error wakes the system + * even if xhci_halt() timed out: a failing controller may never halt. + */ + if (xhci->running_in_s2idle && + ((status & (STS_HCE | STS_FATAL)) || + (xhci->xhc_state & (XHCI_STATE_DYING | XHCI_STATE_HALTED)))) + xhci_s2idle_wakeup(xhci); spin_unlock(&xhci->lock); return ret; diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index 7019daa1d0376..652676aa927d2 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -944,7 +944,7 @@ static bool xhci_pending_portevent(struct xhci_hcd *xhci) ports = xhci->usb2_rhub.ports; while (port_index--) { portsc = xhci_portsc_readl(ports[port_index]); - if (portsc & PORT_CHANGE_MASK || + if (portsc & PORT_CHANGE_MASK || ports[port_index]->held_changes || (portsc & PORT_PLS_MASK) == XDEV_RESUME) return true; } @@ -952,7 +952,7 @@ static bool xhci_pending_portevent(struct xhci_hcd *xhci) ports = xhci->usb3_rhub.ports; while (port_index--) { portsc = xhci_portsc_readl(ports[port_index]); - if (portsc & (PORT_CHANGE_MASK | PORT_CAS) || + if (portsc & (PORT_CHANGE_MASK | PORT_CAS) || ports[port_index]->held_changes || (portsc & PORT_PLS_MASK) == XDEV_RESUME) return true; } @@ -1305,6 +1305,85 @@ int xhci_resume(struct xhci_hcd *xhci, bool power_lost, bool is_auto_resume) return retval; } EXPORT_SYMBOL_GPL(xhci_resume); + +/** + * xhci_run_in_s2idle_begin - Leave the controller running in suspend-to-idle + * @xhci: The controller, with its root hubs suspended or kept awake for wakeup + * + * For platforms without a wake signal for a halted controller. A running + * controller interrupts for every event, not just for the ones that a halted + * controller would signal as a wakeup, so with the interrupt kept enabled the + * event handling makes that decision instead: it wakes the system for those + * events and holds any other port change for the USB core until resume, see + * xhci_s2idle_port_wakeup() and xhci_s2idle_port_change(). + * + * The root hubs are not polled until resume, as for a halted controller in + * xhci_suspend(). Changes that the ports latched before, and that the hub + * driver has not cleared yet, are then decided on as if the ports had just + * posted an event for them, so that each port posts an event for its next + * change. + */ +void xhci_run_in_s2idle_begin(struct xhci_hcd *xhci) +{ + struct usb_hcd *hcds[] = { xhci->usb2_rhub.hcd, xhci->usb3_rhub.hcd }; + unsigned int i; + + spin_lock_irq(&xhci->lock); + xhci->running_in_s2idle = 1; + spin_unlock_irq(&xhci->lock); + + for (i = 0; i < ARRAY_SIZE(hcds); i++) { + if (!hcds[i]) + continue; + clear_bit(HCD_FLAG_POLL_RH, &hcds[i]->flags); + timer_delete_sync(&hcds[i]->rh_timer); + } + + spin_lock_irq(&xhci->lock); + xhci_s2idle_latched_changes(xhci); + spin_unlock_irq(&xhci->lock); +} +EXPORT_SYMBOL_GPL(xhci_run_in_s2idle_begin); + +/** + * xhci_run_in_s2idle_end - Hand the controller back after suspend-to-idle + * @xhci: The controller, still running + * + * Passes the port changes that were held during suspend-to-idle up to the USB + * core and polls the root hubs again, as xhci_resume() does for a halted + * controller, unless the controller has been reported dead in the meantime. + */ +void xhci_run_in_s2idle_end(struct xhci_hcd *xhci) +{ + struct xhci_hub *rhubs[] = { &xhci->usb2_rhub, &xhci->usb3_rhub }; + bool held[ARRAY_SIZE(rhubs)] = {}; + unsigned int i, j; + bool dying; + + spin_lock_irq(&xhci->lock); + xhci->running_in_s2idle = 0; + dying = xhci->xhc_state & XHCI_STATE_DYING; + for (i = 0; i < ARRAY_SIZE(rhubs); i++) { + for (j = 0; j < rhubs[i]->num_ports; j++) + held[i] |= !!rhubs[i]->ports[j]->held_changes; + } + spin_unlock_irq(&xhci->lock); + + if (dying) + return; + + for (i = 0; i < ARRAY_SIZE(rhubs); i++) { + struct usb_hcd *hcd = rhubs[i]->hcd; + + if (!hcd) + continue; + if (held[i] && hcd->state == HC_STATE_SUSPENDED) + usb_hcd_resume_root_hub(hcd); + set_bit(HCD_FLAG_POLL_RH, &hcd->flags); + usb_hcd_poll_rh_status(hcd); + } +} +EXPORT_SYMBOL_GPL(xhci_run_in_s2idle_end); #endif /* CONFIG_PM */ /*-------------------------------------------------------------------------*/ diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index 616c308be72e5..e75ce7f7b23bc 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -1483,6 +1483,8 @@ struct xhci_port { unsigned int lpm_incapable:1; unsigned long resume_timestamp; bool rexit_active; + /* Change bits held for the hub driver, see xhci_s2idle_port_change() */ + u32 held_changes; /* Slot ID is the index of the device directly connected to the port */ int slot_id; struct completion rexit_done; @@ -1662,6 +1664,8 @@ struct xhci_hcd { unsigned broken_suspend:1; /* Indicates that omitting hcd is supported if root hub has no ports */ unsigned allow_single_roothub:1; + /* Kept running in suspend-to-idle, see xhci_run_in_s2idle_begin() */ + unsigned running_in_s2idle:1; /* cached extended protocol port capabilities */ struct xhci_port_cap *port_caps; unsigned int num_port_caps; @@ -1902,6 +1906,8 @@ int xhci_ext_cap_init(struct xhci_hcd *xhci); int xhci_suspend(struct xhci_hcd *xhci, bool do_wakeup); int xhci_resume(struct xhci_hcd *xhci, bool power_lost, bool is_auto_resume); +void xhci_run_in_s2idle_begin(struct xhci_hcd *xhci); +void xhci_run_in_s2idle_end(struct xhci_hcd *xhci); irqreturn_t xhci_irq(struct usb_hcd *hcd); irqreturn_t xhci_msi_irq(int irq, void *hcd); @@ -1987,6 +1993,12 @@ enum usb_link_tunnel_mode xhci_port_is_tunneled(struct xhci_hcd *xhci, struct xhci_port *port); void xhci_hc_died(struct xhci_hcd *xhci); void xhci_quiesce_dying(struct xhci_hcd *xhci); +void xhci_s2idle_wakeup(struct xhci_hcd *xhci); +void xhci_s2idle_latched_changes(struct xhci_hcd *xhci); +#if IS_ENABLED(CONFIG_KUNIT) +bool xhci_s2idle_port_wakeup(u32 portsc); +bool xhci_s2idle_transfer_wakeup(const struct usb_device *udev); +#endif #ifdef CONFIG_PM int xhci_bus_suspend(struct usb_hcd *hcd); From b1415b654b433ceb843ff9870304088e04f55091 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 14:31:42 +1100 Subject: [PATCH 1494/2680] usb: xhci: plat: Keep the interrupt enabled when running in suspend-to-idle The event handling of a controller kept running in suspend-to-idle now decides which events wake the system, but its handler never runs then. The glue arms the interrupt for wakeup, and the IRQ core disables an armed interrupt on its first trigger and ends suspend-to-idle without calling the handler. On the J700 that is still IRQ 93 waking the system for a link change on a root port with wakeup disabled. Request the interrupt with IRQF_NO_SUSPEND when the platform asks for XHCI_RUN_IN_S2IDLE, so that suspend_device_irqs() leaves it enabled and xhci_irq() runs while the system sleeps. As the suspend and interrupts documentation describes, the handler of an IRQF_NO_SUSPEND interrupt runs in suspend-to-idle and the system only wakes if it reports a wakeup, which the event handling now does with pm_wakeup_hard_event(). The Apple mailbox interrupts that carry the SMC lid and power button events to macsmc-input work the same way. Interrupts that are no wakeup now cost a pass through the handler, after which suspend-to-idle carries on. A halted controller is unaffected: it writes no events, so it raises no interrupt, and with its hardware marked inaccessible usb_hcd_irq() would not touch it. Only the primary HCD requests the interrupt, and the DWC3 core requests nothing on it in host mode, so it has no other user that would also be kept enabled. IRQF_SHARED is kept as before. Kept enabled, the interrupt is no longer armed by the wake interrupt that the glue sets up for it; the next change removes that. The flags are fixed when the interrupt is requested, so drop XHCI_RUN_IN_S2IDLE again if it only arrives later, from the xhci_hcd quirks module parameter: running without the interrupt could never wake the system. Add a KUnit case for the flags. (cherry picked from commit acf1861397d812b05b671a2c75fb92f6e615ac28) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/host/xhci-plat-test.c | 11 ++++++++++ drivers/usb/host/xhci-plat.c | 35 ++++++++++++++++++++++++------- drivers/usb/host/xhci-plat.h | 1 + 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/drivers/usb/host/xhci-plat-test.c b/drivers/usb/host/xhci-plat-test.c index e781f5662564b..b07b3f985ea26 100644 --- a/drivers/usb/host/xhci-plat-test.c +++ b/drivers/usb/host/xhci-plat-test.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -49,6 +50,15 @@ static void xhci_plat_keep_running_other_quirks_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, xhci_plat_keep_running(~0ULL, PM_SUSPEND_TO_IDLE, true, true)); } +/* Only a controller kept running in suspend-to-idle keeps its interrupt enabled across it. */ +static void xhci_plat_irq_flags_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, xhci_plat_irq_flags(0), (unsigned long)IRQF_SHARED); + KUNIT_EXPECT_EQ(test, xhci_plat_irq_flags(~XHCI_RUN_IN_S2IDLE), (unsigned long)IRQF_SHARED); + KUNIT_EXPECT_EQ(test, xhci_plat_irq_flags(XHCI_RUN_IN_S2IDLE), + (unsigned long)(IRQF_SHARED | IRQF_NO_SUSPEND)); +} + /* A running controller is left alone; a halted, failed or unpowered one is not. */ static void xhci_plat_stopped_test(struct kunit *test) { @@ -279,6 +289,7 @@ static void xhci_plat_all_property_quirks_test(struct kunit *test) static struct kunit_case xhci_plat_test_cases[] = { KUNIT_CASE(xhci_plat_keep_running_test), KUNIT_CASE(xhci_plat_keep_running_other_quirks_test), + KUNIT_CASE(xhci_plat_irq_flags_test), KUNIT_CASE(xhci_plat_stopped_test), KUNIT_CASE(xhci_plat_quiesce_dying_test), #ifdef CONFIG_PM diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index d38d96ec37a56..ad0e2807a862b 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -188,6 +188,17 @@ VISIBLE_IF_KUNIT u64 xhci_plat_property_quirks(struct device *dev) } EXPORT_SYMBOL_IF_KUNIT(xhci_plat_property_quirks); +/* + * A controller kept running in suspend-to-idle decides in its event handling which events wake the + * system, see xhci_run_in_s2idle_begin(). Its interrupt handler has to run while the system + * sleeps, so keep the interrupt enabled across suspend instead of arming it for wakeup. + */ +VISIBLE_IF_KUNIT unsigned long xhci_plat_irq_flags(u64 quirks) +{ + return IRQF_SHARED | ((quirks & XHCI_RUN_IN_S2IDLE) ? IRQF_NO_SUSPEND : 0); +} +EXPORT_SYMBOL_IF_KUNIT(xhci_plat_irq_flags); + int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const struct xhci_plat_priv *priv_match) { const struct hc_driver *driver; @@ -195,6 +206,7 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s struct xhci_hcd *xhci; struct resource *res; struct usb_hcd *hcd, *usb3_hcd; + unsigned long irqflags; int ret; int irq; struct xhci_plat_priv *priv = NULL; @@ -326,7 +338,8 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s if (priv && (priv->quirks & XHCI_SG_TRB_CACHE_SIZE_QUIRK)) xhci->quirks |= XHCI_SG_TRB_CACHE_SIZE_QUIRK; - ret = usb_add_hcd(hcd, irq, IRQF_SHARED); + irqflags = xhci_plat_irq_flags(xhci->quirks); + ret = usb_add_hcd(hcd, irq, irqflags); if (ret) goto disable_usb_phy; @@ -362,15 +375,22 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s if (xhci->shared_hcd) { xhci->shared_hcd->rsrc_start = hcd->rsrc_start; xhci->shared_hcd->rsrc_len = hcd->rsrc_len; - ret = usb_add_hcd(xhci->shared_hcd, irq, IRQF_SHARED); + ret = usb_add_hcd(xhci->shared_hcd, irq, irqflags); if (ret) goto put_usb3_hcd; } + /* + * Running in suspend-to-idle needs the interrupt requested above for it, + * so a later quirk source, such as the xhci_hcd quirks module parameter, + * cannot turn it on. + */ + if (!(irqflags & IRQF_NO_SUSPEND)) + xhci->quirks &= ~XHCI_RUN_IN_S2IDLE; + /* * Let the USB core keep hubs awake that need it for wakeup, see - * XHCI_RUN_IN_S2IDLE. Only now are all quirk sources merged, the xhci_hcd - * quirks module parameter included. + * XHCI_RUN_IN_S2IDLE. */ hcd->runs_in_s2idle = !!(xhci->quirks & XHCI_RUN_IN_S2IDLE); if (xhci->shared_hcd) @@ -536,9 +556,10 @@ static int xhci_plat_suspend_common(struct device *dev) * Some platforms have no wake signal for a halted xHC: a halted controller writes no events, so * its interrupt cannot fire, and on PCI the remote wakeup or port change would be signalled as a * PME instead. Where the platform keeps the controller powered and clocked in suspend-to-idle, - * leave it running instead, with the root hubs suspended and armed for wakeup as usual. A remote - * wakeup or port change then raises a Port Status Change Event, and the interrupt, armed for - * wakeup by the platform, ends suspend-to-idle. + * leave it running instead, with the root hubs suspended and armed for wakeup as usual. The + * controller then interrupts for every event, its interrupt stays enabled (see + * xhci_plat_irq_flags()), and the event handling wakes the system for the events that a halted + * controller would signal. */ VISIBLE_IF_KUNIT bool xhci_plat_keep_running(u64 quirks, suspend_state_t target, bool may_wakeup, bool wakeup_path) diff --git a/drivers/usb/host/xhci-plat.h b/drivers/usb/host/xhci-plat.h index d002aac38ec20..97d7b8475b9be 100644 --- a/drivers/usb/host/xhci-plat.h +++ b/drivers/usb/host/xhci-plat.h @@ -37,6 +37,7 @@ extern const struct dev_pm_ops xhci_plat_pm_ops; #include u64 xhci_plat_property_quirks(struct device *dev); +unsigned long xhci_plat_irq_flags(u64 quirks); bool xhci_plat_keep_running(u64 quirks, suspend_state_t target, bool may_wakeup, bool wakeup_path); bool xhci_plat_stopped(u32 usbsts); From cfb1f8c99fe42810f466b63ece5b93eac83aa704 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:29:34 +1100 Subject: [PATCH 1495/2680] selftests: apple-dcp: Model the DWC3 wake interrupt in the lifecycle tests The controller-lifecycle and dwc3-teardown tests build the actual DWC3 glue teardown into host units. Since the glue arms the core interrupt for wakeup in host mode, it calls dev_pm_set_wake_irq() and dev_pm_clear_wake_irq() and keeps the interrupt number in struct dwc3_apple, none of which the units provided, so neither compiled and none of their cases ran. Model the wake interrupt in both units, and check in the controller-lifecycle test that host teardown drops it again. Fixes: 13e221102941 ("usb: dwc3: apple: Arm the core interrupt for system wakeup in host mode") (cherry picked from commit 63ad4f5ad9c252d1bd8c86cf27118fc56f34736a) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/gpu/apple-dcp/dwc3/mocks.c | 16 ++++++++++++++++ .../drivers/gpu/apple-dcp/teardown_pm.py | 6 +++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c index 777e12dc36135..08bd53fac72e1 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c @@ -65,6 +65,7 @@ struct dwc3 { struct dwc3_apple { struct dwc3 dwc; enum dwc3_apple_state state; + int irq; struct reset_controller_dev *reset; int lock; void *dev; @@ -219,6 +220,21 @@ static void dev_pm_syscore_device(void *d, bool s) (void)d; syscore = s; } +static bool wake_irq_set; + +static int dev_pm_set_wake_irq(void *d, int irq) +{ + (void)d; + (void)irq; + wake_irq_set = true; + return 0; +} + +static void dev_pm_clear_wake_irq(void *d) +{ + (void)d; + wake_irq_set = 0; +} static void dwc3_enable_susphy(struct dwc3 *d, bool s) { (void)d; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py index 1f17261136a9c..6321c337529cb 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py @@ -80,6 +80,8 @@ def unit_for(source): static void *platform_get_drvdata(struct platform_device *d) { return d==&controller?(void *)&apple_context.dwc:(void *)&hcd; } static void dwc3_enable_susphy(struct dwc3 *d,bool v) { (void)d;assert(v);susphy_calls++;observed_controller_write(); } static void device_init_wakeup(struct device *d,bool b) { (void)d;(void)b; } +static bool wake_irq_set; +static void dev_pm_clear_wake_irq(struct device *d) { assert(d==&controller.dev);wake_irq_set=false; } static void platform_device_unregister(struct platform_device *d); static void dwc3_gadget_exit(struct dwc3 *d) { (void)d;assert(0); } static int dwc3_apple_core_stop(struct dwc3_apple *d) { (void)d;return 0; } @@ -140,7 +142,7 @@ def unit_for(source): xhci_context=(struct xhci_hcd){0}; inject_remove=true;removal_started=blocked_attempt=release_remove=false; atomic_store(&early_write,false);atomic_store(&handoff_active,true); - susphy_calls=host_remove_calls=0; + susphy_calls=host_remove_calls=0;wake_irq_set=true; int ret=safe(&p); atomic_store(&handoff_active,false); pthread_mutex_lock(&schedule_mutex);release_remove=true;pthread_cond_broadcast(&schedule_cond);pthread_mutex_unlock(&schedule_mutex); @@ -148,6 +150,8 @@ def unit_for(source): fprintf(stderr,"connected teardown target=%d ret=%d early_write=%d susphy=%d host_remove=%d\n",remove_target,ret,atomic_load(&early_write),susphy_calls,host_remove_calls); assert(ret==0&&!atomic_load(&early_write)&&host_remove_calls==1); assert(susphy_calls==(remove_target?0:1));balanced(&p); + /* Host teardown drops the wake interrupt it armed. */ + assert(remove_target||!wake_irq_set); } puts("PASS actual PM wrapper connected to driver-core/DWC3/xHCI removal; synchronous teardown waits until owned transaction ends"); return 0; From edb14a1f6e367830815725db3bde4d512e162237 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 14:33:20 +1100 Subject: [PATCH 1496/2680] usb: dwc3: apple: Do not arm the core interrupt for wakeup In host mode the core interrupt was set up as the wake interrupt of the glue, so that the Port Status Change Event of a running xHCI would end suspend-to-idle. Armed for wakeup, it ended suspend-to-idle for every event of the controller, which is what made a link change on a root port with wakeup disabled wake the J700. xHCI now keeps its interrupt enabled across suspend-to-idle when it keeps running there and wakes the system itself, for the events a halted controller would signal. A wake interrupt on the same line is never armed then, as the interrupt is no longer suspended, and the PM documentation advises against combining the two on one interrupt. For a halted controller it never served a purpose: a halted xHC writes no events, so the interrupt cannot fire. Drop the wake interrupt and the interrupt number that was kept only for it. The glue stays wakeup capable and enabled by default, which dwc3_host_init() passes on to the xHCI device and which decides whether xHCI keeps running. Drop the wake interrupt from the models of the glue in the DCP lifecycle selftests again as well. [ryan: split by topic: only the DWC3 Apple glue part; the apple-dcp selftests part is in 200-dcp] Fixes: 13e221102941 ("usb: dwc3: apple: Arm the core interrupt for system wakeup in host mode") (cherry picked from commit dc024263b82f53638559d5c40ca17f0c924eb625) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/dwc3-apple.c | 32 +++++--------------------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index b24789b574561..49953a518eec9 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -80,8 +79,8 @@ * Type-C mux for the SuperSpeed lanes, and the PHY follows those mode changes on its own. * * Whenever the core is up, for either kind of port, it is kept running across system sleep, see - * dwc3_apple_init(). In host mode its interrupt is then also armed as the wake interrupt, and - * xHCI keeps running in suspend-to-idle so that it can raise it. + * dwc3_apple_init(). In host mode xHCI then keeps running in suspend-to-idle when a USB device + * may wake the system, as a halted xHC could not signal the wakeup. * * enum dwc3_apple_state is defined in dwc3-apple-state.h. */ @@ -90,7 +89,6 @@ * struct dwc3_apple - Apple-specific DWC3 USB controller * @dwc: Core DWC3 structure * @dev: Pointer to the device structure - * @irq: Core interrupt, shared with xHCI in host mode and used as its wake interrupt * @mmio_resource: Resource to be passed to dwc3_core_probe * @apple_regs: Apple-specific DWC3 registers * @reset: Reset control @@ -111,7 +109,6 @@ struct dwc3_apple { struct dwc3 dwc; struct device *dev; - int irq; struct resource *mmio_resource; void __iomem *apple_regs; @@ -214,8 +211,9 @@ static int dwc3_apple_core_probe(struct dwc3_apple *appledwc) probe_data.skip_core_init_mode = true; probe_data.properties = DWC3_DEFAULT_PROPERTIES; /* - * There is no wake interrupt besides the core interrupt, and a halted xHC raises none, - * see dwc3_apple_init(). + * The controller has no wake signal of its own, and a halted xHC writes no events, so its + * interrupt cannot fire either. xHCI keeps running instead and decides which of its events + * wake the system; the core stays powered across system sleep, see dwc3_apple_init(). */ probe_data.properties.xhci_run_in_s2idle = true; @@ -448,21 +446,6 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st goto core_exit; } - /* - * xHCI shares the core interrupt and suspends its root hubs with their wake bits set when - * this device may wake the system (dwc3_host_init() passes that on). The controller has no - * separate wake signal, and a halted xHC writes no events and so raises no interrupt, so - * xHCI is told to keep running in suspend-to-idle instead of halting (the core stays - * powered, see below). Arm the interrupt for wakeup while in host mode, so that the Port - * Status Change Event of a remote wakeup or port change ends suspend-to-idle. - * Device mode has no wake source: a gadget is woken by its host. - */ - if (state == DWC3_APPLE_HOST) { - ret = dev_pm_set_wake_irq(appledwc->dev, appledwc->irq); - if (ret) - dev_warn(appledwc->dev, "Failed to set up the wake IRQ, err=%d\n", ret); - } - appledwc->state = state; /* * A connected cable leaves the Apple DWC3 core and its PHY as one live, @@ -499,7 +482,6 @@ static int dwc3_apple_exit(struct dwc3_apple *appledwc) appledwc->state = DWC3_APPLE_NO_CABLE; break; case DWC3_APPLE_HOST: - dev_pm_clear_wake_irq(appledwc->dev); dwc3_host_exit(&appledwc->dwc); appledwc->state = DWC3_APPLE_NO_CABLE; break; @@ -648,10 +630,6 @@ static int dwc3_apple_probe(struct platform_device *pdev) return dev_err_probe(dev, PTR_ERR(appledwc->apple_regs), "Failed to map Apple-specific MMIO\n"); - appledwc->irq = platform_get_irq(pdev, 0); - if (appledwc->irq < 0) - return appledwc->irq; - /* * Like a PCI xHCI controller, allow USB devices to wake the system by default. This has * to be set before dwc3_core_probe(), which hands it on to the xHCI device. From 143d4df3037fbefce27b6e5713ccc514f4953105 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 14:33:20 +1100 Subject: [PATCH 1497/2680] usb: dwc3: apple: Do not arm the core interrupt for wakeup In host mode the core interrupt was set up as the wake interrupt of the glue, so that the Port Status Change Event of a running xHCI would end suspend-to-idle. Armed for wakeup, it ended suspend-to-idle for every event of the controller, which is what made a link change on a root port with wakeup disabled wake the J700. xHCI now keeps its interrupt enabled across suspend-to-idle when it keeps running there and wakes the system itself, for the events a halted controller would signal. A wake interrupt on the same line is never armed then, as the interrupt is no longer suspended, and the PM documentation advises against combining the two on one interrupt. For a halted controller it never served a purpose: a halted xHC writes no events, so the interrupt cannot fire. Drop the wake interrupt and the interrupt number that was kept only for it. The glue stays wakeup capable and enabled by default, which dwc3_host_init() passes on to the xHCI device and which decides whether xHCI keeps running. Drop the wake interrupt from the models of the glue in the DCP lifecycle selftests again as well. [ryan: split by topic: only the apple-dcp selftests part; the DWC3 Apple glue part is in 170-usb] Fixes: 13e221102941 ("usb: dwc3: apple: Arm the core interrupt for system wakeup in host mode") (cherry picked from commit dc024263b82f53638559d5c40ca17f0c924eb625) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/gpu/apple-dcp/dwc3/mocks.c | 16 ---------------- .../drivers/gpu/apple-dcp/teardown_pm.py | 6 +----- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c index 08bd53fac72e1..777e12dc36135 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c @@ -65,7 +65,6 @@ struct dwc3 { struct dwc3_apple { struct dwc3 dwc; enum dwc3_apple_state state; - int irq; struct reset_controller_dev *reset; int lock; void *dev; @@ -220,21 +219,6 @@ static void dev_pm_syscore_device(void *d, bool s) (void)d; syscore = s; } -static bool wake_irq_set; - -static int dev_pm_set_wake_irq(void *d, int irq) -{ - (void)d; - (void)irq; - wake_irq_set = true; - return 0; -} - -static void dev_pm_clear_wake_irq(void *d) -{ - (void)d; - wake_irq_set = 0; -} static void dwc3_enable_susphy(struct dwc3 *d, bool s) { (void)d; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py index 6321c337529cb..1f17261136a9c 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py @@ -80,8 +80,6 @@ def unit_for(source): static void *platform_get_drvdata(struct platform_device *d) { return d==&controller?(void *)&apple_context.dwc:(void *)&hcd; } static void dwc3_enable_susphy(struct dwc3 *d,bool v) { (void)d;assert(v);susphy_calls++;observed_controller_write(); } static void device_init_wakeup(struct device *d,bool b) { (void)d;(void)b; } -static bool wake_irq_set; -static void dev_pm_clear_wake_irq(struct device *d) { assert(d==&controller.dev);wake_irq_set=false; } static void platform_device_unregister(struct platform_device *d); static void dwc3_gadget_exit(struct dwc3 *d) { (void)d;assert(0); } static int dwc3_apple_core_stop(struct dwc3_apple *d) { (void)d;return 0; } @@ -142,7 +140,7 @@ def unit_for(source): xhci_context=(struct xhci_hcd){0}; inject_remove=true;removal_started=blocked_attempt=release_remove=false; atomic_store(&early_write,false);atomic_store(&handoff_active,true); - susphy_calls=host_remove_calls=0;wake_irq_set=true; + susphy_calls=host_remove_calls=0; int ret=safe(&p); atomic_store(&handoff_active,false); pthread_mutex_lock(&schedule_mutex);release_remove=true;pthread_cond_broadcast(&schedule_cond);pthread_mutex_unlock(&schedule_mutex); @@ -150,8 +148,6 @@ def unit_for(source): fprintf(stderr,"connected teardown target=%d ret=%d early_write=%d susphy=%d host_remove=%d\n",remove_target,ret,atomic_load(&early_write),susphy_calls,host_remove_calls); assert(ret==0&&!atomic_load(&early_write)&&host_remove_calls==1); assert(susphy_calls==(remove_target?0:1));balanced(&p); - /* Host teardown drops the wake interrupt it armed. */ - assert(remove_target||!wake_irq_set); } puts("PASS actual PM wrapper connected to driver-core/DWC3/xHCI removal; synchronous teardown waits until owned transaction ends"); return 0; From 84f08bf7031f639024469f805a7f5d5371b44eae Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 21:50:08 +1100 Subject: [PATCH 1498/2680] usb: core: Tell whether a remote wakeup from below reaches a hub A host controller that has to keep running in suspend-to-idle for USB devices to wake the system should do so only when one of them can. The power/wakeup setting of a device does not tell: whether its remote wakeup gets through also depends on the hubs on its way, and the USB core decides on that as it suspends each device. Add usb_wakeup_reaches(), which tells, once the devices below a hub are suspended, whether one of them can wake it. A device signals a remote wakeup only if the USB core enabled that when it suspended it. A hub passes one on from below whatever its own wakeup setting, which only arms it for connect, disconnect and over-current changes (USB 2.0, section 7.1.7.7, and commit 7aba8d014341 ("USB: don't enable remote wakeup by default")). The exception is a hub with USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP: while it is suspended, it does not pass one on from below, so the devices below it count only while it is kept awake. The quirk is about passing a wakeup on, so such a hub armed for its own changes still counts. Add KUnit tests for it on device trees built in memory, among them the one behind the front port of a J700. (cherry picked from commit 69b23ab68ffa1fde47aa1fe140beb757e144bea7) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/core/Kconfig | 11 ++ drivers/usb/core/Makefile | 1 + drivers/usb/core/hub.c | 36 +++++++ drivers/usb/core/usb-wakeup-test.c | 161 +++++++++++++++++++++++++++++ include/linux/usb/hcd.h | 6 ++ 5 files changed, 215 insertions(+) create mode 100644 drivers/usb/core/usb-wakeup-test.c diff --git a/drivers/usb/core/Kconfig b/drivers/usb/core/Kconfig index fbeab9ee46202..1251c543f6cb0 100644 --- a/drivers/usb/core/Kconfig +++ b/drivers/usb/core/Kconfig @@ -108,6 +108,17 @@ config USB_QUIRKS_KUNIT_TEST If unsure, say N. +config USB_WAKEUP_KUNIT_TEST + tristate "KUnit tests for USB wakeup in system sleep" if !KUNIT_ALL_TESTS + depends on USB && PM && KUNIT + default KUNIT_ALL_TESTS + help + Test which USB devices the USB core finds able to wake the system + once it has suspended them, on device trees built in memory. The + tests need no USB hardware. + + If unsure, say N. + config USB_LEDS_TRIGGER_USBPORT tristate "USB port LED trigger" depends on USB && LEDS_TRIGGERS diff --git a/drivers/usb/core/Makefile b/drivers/usb/core/Makefile index bf228db5e15f9..64468ad4a6d51 100644 --- a/drivers/usb/core/Makefile +++ b/drivers/usb/core/Makefile @@ -22,5 +22,6 @@ endif obj-$(CONFIG_USB) += usbcore.o obj-$(CONFIG_USB_QUIRKS_KUNIT_TEST) += usb-quirks-test.o +obj-$(CONFIG_USB_WAKEUP_KUNIT_TEST) += usb-wakeup-test.o obj-$(CONFIG_USB_LEDS_TRIGGER_USBPORT) += ledtrig-usbport.o diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index 173a7fe809611..97a043aaa62c5 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -3468,6 +3468,42 @@ unsigned usb_wakeup_enabled_descendants(struct usb_device *udev) } EXPORT_SYMBOL_GPL(usb_wakeup_enabled_descendants); +/** + * usb_wakeup_reaches - Whether a remote wakeup from below reaches a hub + * @hdev: The hub, with the devices below it suspended for system sleep + * + * A device signals a remote wakeup only if the USB core enabled that when it + * suspended it: resume signalling, or for a hub, a connect, disconnect or + * over-current change on one of its ports. A hub passes on the remote wakeup + * of a device below it whatever its own wakeup setting (USB 2.0, section + * 7.1.7.7). A hub with USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP does not while it is + * suspended, but still signals its own; a hub kept awake does both. + * + * Context: System sleep, once the devices below @hdev are suspended, so that + * neither they nor their wakeup settings change. Takes no locks. + * + * Return: true if a device below @hdev can wake it up. + */ +bool usb_wakeup_reaches(struct usb_device *hdev) +{ + struct usb_device *child; + int port1; + + usb_hub_for_each_child(hdev, port1, child) { + /* A hub kept awake reports any change on its status endpoint */ + u32 quirks = child->awake_in_sleep ? 0 : child->quirks; + + if (child->do_remote_wakeup) + return true; + if (!(quirks & USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP) && + usb_wakeup_reaches(child)) + return true; + } + + return false; +} +EXPORT_SYMBOL_GPL(usb_wakeup_reaches); + /* * usb_port_suspend - suspend a usb device's upstream port * @udev: device that's no longer in active use, not a root hub diff --git a/drivers/usb/core/usb-wakeup-test.c b/drivers/usb/core/usb-wakeup-test.c new file mode 100644 index 0000000000000..4e8658d316dab --- /dev/null +++ b/drivers/usb/core/usb-wakeup-test.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit tests for the USB core's wakeup decisions in system sleep + */ + +#include +#include +#include +#include +#include + +/* hub.h includes usb.h, which has no include guard */ +#include "hub.h" + +#define USB_WAKEUP_TEST_PORTS 4U + +/* A suspended USB device, with the hub driver state of a hub */ +struct usb_wakeup_test_dev { + struct usb_device udev; + struct usb_host_config config; + struct usb_interface intf; + struct usb_hub hub; + struct usb_port ports[USB_WAKEUP_TEST_PORTS]; + struct usb_port *port_list[USB_WAKEUP_TEST_PORTS]; +}; + +/* A hub or another device, attached to the first free port of @parent if any */ +static struct usb_wakeup_test_dev *usb_wakeup_test_add(struct kunit *test, + struct usb_wakeup_test_dev *parent, + bool hub) +{ + struct usb_wakeup_test_dev *d = kunit_kzalloc(test, sizeof(*d), GFP_KERNEL); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, d); + if (hub) { + for (i = 0; i < USB_WAKEUP_TEST_PORTS; i++) + d->port_list[i] = &d->ports[i]; + d->hub.hdev = &d->udev; + d->hub.ports = d->port_list; + d->config.interface[0] = &d->intf; + usb_set_intfdata(&d->intf, &d->hub); + d->udev.actconfig = &d->config; + d->udev.maxchild = USB_WAKEUP_TEST_PORTS; + } + if (parent) { + for (i = 0; i < USB_WAKEUP_TEST_PORTS && parent->ports[i].child; i++) + ; + KUNIT_ASSERT_LT(test, i, USB_WAKEUP_TEST_PORTS); + parent->ports[i].child = &d->udev; + d->udev.parent = &parent->udev; + } + + return d; +} + +/* + * The defaults behind the front port of an Apple J700: a keyboard with remote + * wakeup enabled, below a hub, below the VIA Labs VL122 hub on the root port, + * every hub with wakeup disabled. The VL122 does not pass a remote wakeup on + * while it is suspended, so the keyboard's does not reach the root hub, unless + * the VL122 is kept awake. + */ +static void usb_wakeup_reaches_vl122_test(struct kunit *test) +{ + struct usb_wakeup_test_dev *root = usb_wakeup_test_add(test, NULL, true); + struct usb_wakeup_test_dev *vl122 = usb_wakeup_test_add(test, root, true); + struct usb_wakeup_test_dev *hub = usb_wakeup_test_add(test, vl122, true); + struct usb_wakeup_test_dev *keyboard = usb_wakeup_test_add(test, hub, false); + + vl122->udev.quirks = USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP; + keyboard->udev.do_remote_wakeup = 1; + KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&hub->udev)); + KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&vl122->udev)); + KUNIT_EXPECT_FALSE(test, usb_wakeup_reaches(&root->udev)); + + vl122->udev.awake_in_sleep = 1; + KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&root->udev)); +} + +/* Any other hub passes it on, whatever its own wakeup setting. */ +static void usb_wakeup_reaches_hub_test(struct kunit *test) +{ + struct usb_wakeup_test_dev *root = usb_wakeup_test_add(test, NULL, true); + struct usb_wakeup_test_dev *hub = usb_wakeup_test_add(test, root, true); + struct usb_wakeup_test_dev *keyboard = usb_wakeup_test_add(test, hub, false); + + usb_wakeup_test_add(test, hub, false); + usb_wakeup_test_add(test, root, false); + KUNIT_EXPECT_FALSE(test, usb_wakeup_reaches(&root->udev)); + + keyboard->udev.do_remote_wakeup = 1; + KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&root->udev)); +} + +/* + * A hub armed for wakeup signals connect, disconnect and over-current changes + * on its ports, also if it does not pass wakeups on from below while suspended. + */ +static void usb_wakeup_reaches_armed_hub_test(struct kunit *test) +{ + struct usb_wakeup_test_dev *root = usb_wakeup_test_add(test, NULL, true); + struct usb_wakeup_test_dev *hub = usb_wakeup_test_add(test, root, true); + + hub->udev.do_remote_wakeup = 1; + KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&root->udev)); + + hub->udev.quirks = USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP; + KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&root->udev)); +} + +/* + * A hub that does not pass wakeups on from below while it is suspended, armed + * for wakeup, with wakeup disabled for the root hub and the device below it: + * its own changes reach the root hub. A remote wakeup from below it does not. + */ +static void usb_wakeup_reaches_armed_quirky_hub_test(struct kunit *test) +{ + struct usb_wakeup_test_dev *root = usb_wakeup_test_add(test, NULL, true); + struct usb_wakeup_test_dev *hub = usb_wakeup_test_add(test, root, true); + struct usb_wakeup_test_dev *keyboard = usb_wakeup_test_add(test, hub, false); + + hub->udev.quirks = USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP; + hub->udev.do_remote_wakeup = 1; + KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&root->udev)); + + hub->udev.do_remote_wakeup = 0; + keyboard->udev.do_remote_wakeup = 1; + KUNIT_EXPECT_FALSE(test, usb_wakeup_reaches(&root->udev)); +} + +/* A root hub with nothing below, and a device that is no hub, reach nothing. */ +static void usb_wakeup_reaches_nothing_test(struct kunit *test) +{ + struct usb_wakeup_test_dev *root = usb_wakeup_test_add(test, NULL, true); + struct usb_wakeup_test_dev *keyboard; + + KUNIT_EXPECT_FALSE(test, usb_wakeup_reaches(&root->udev)); + keyboard = usb_wakeup_test_add(test, root, false); + keyboard->udev.do_remote_wakeup = 1; + KUNIT_EXPECT_FALSE(test, usb_wakeup_reaches(&keyboard->udev)); +} + +static struct kunit_case usb_wakeup_test_cases[] = { + KUNIT_CASE(usb_wakeup_reaches_vl122_test), + KUNIT_CASE(usb_wakeup_reaches_hub_test), + KUNIT_CASE(usb_wakeup_reaches_armed_hub_test), + KUNIT_CASE(usb_wakeup_reaches_armed_quirky_hub_test), + KUNIT_CASE(usb_wakeup_reaches_nothing_test), + {} +}; + +static struct kunit_suite usb_wakeup_test_suite = { + .name = "usb_wakeup", + .test_cases = usb_wakeup_test_cases, +}; + +kunit_test_suite(usb_wakeup_test_suite); + +MODULE_DESCRIPTION("KUnit tests for the USB core's wakeup decisions in system sleep"); +MODULE_LICENSE("GPL"); diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h index f4520d9922a2b..604b9dda94317 100644 --- a/include/linux/usb/hcd.h +++ b/include/linux/usb/hcd.h @@ -692,6 +692,7 @@ extern wait_queue_head_t usb_kill_urb_queue; #ifdef CONFIG_PM extern unsigned usb_wakeup_enabled_descendants(struct usb_device *udev); +bool usb_wakeup_reaches(struct usb_device *hdev); extern void usb_root_hub_lost_power(struct usb_device *rhdev); extern int hcd_bus_suspend(struct usb_device *rhdev, pm_message_t msg); extern int hcd_bus_resume(struct usb_device *rhdev, pm_message_t msg); @@ -701,6 +702,11 @@ static inline unsigned usb_wakeup_enabled_descendants(struct usb_device *udev) { return 0; } + +static inline bool usb_wakeup_reaches(struct usb_device *hdev) +{ + return false; +} static inline void usb_hcd_resume_root_hub(struct usb_hcd *hcd) { return; From a8ed01efd19042ca0d1d5333f3386bc786418819 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 14:46:43 +1100 Subject: [PATCH 1499/2680] selftests: apple-dcp: Follow the tipd lost-IRQ recovery helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The host runner fails 5 of 55 cases since the tipd lost-IRQ recovery because the typec-probe, probe-mux-pm-dispatch, physical-disconnect, fast-replug-bounce and dropped-swap-teardown units no longer compile. They inline cd321x-pm.h, which now includes for the acknowledgment retry backoff, and the probe units do not extract the new cd321x_init_work() called by cd321x_register_port(). Inline the header through one shared helper that provides a host clamp() in place of minmax.h, and extract the actual cd321x_init_work() for both probe units. Check in typec-probe that a lost event reopens even a stale port with a full budget, that a failed acknowledgment keeps the budget with a backoff capped at CD321X_ACK_RETRY_MAX_MS, and that sleep and removal leave lost events to the resume read. The dispatch unit drops these helpers like the other ones it does not exercise. Retain the strict compiler and sanitizer settings and all 55 runner cases. The full host run improves from 50 pass / 5 fail to 55 pass / 0 fail. No production driver changes are needed. [ryan: adapted to 7.2.10 — context only: lifecycle.py also imports ATC_POST_TUNABLE, added by commit b69728b754b1 ("selftests: apple-dcp: Follow the 7.2.10 driver boundaries")] (cherry picked from commit befa72f4f47d58327130236376ceeab26b45ee5c) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/gpu/apple-dcp/common.py | 9 +++++++ .../drivers/gpu/apple-dcp/lifecycle.py | 5 ++-- .../selftests/drivers/gpu/apple-dcp/pm.py | 24 ++++++++++++++++--- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py index e895a7ac203e4..61d34b1593cc0 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py @@ -25,6 +25,15 @@ def source_defines(source): definitions.append('\n'.join(definition)) return definitions +def tipd_pm_header(root): + """CD321x PM state machine for host units, with a host clamp() for minmax.h.""" + header = (root / 'drivers/usb/typec/tipd/cd321x-pm.h').read_text() + header = header.replace('#include ', '') + # Single line: probe-mux-pm-dispatch filters duplicate definitions per line. + return header.replace('#include ', + '#define clamp(val, lo, hi) ({ __auto_type _v = (val); ' + '__typeof__(_v) _lo = (lo), _hi = (hi); _v < _lo ? _lo : _v > _hi ? _hi : _v; })') + def run_unit(name, unit, output, cases=((),), includes=(), extra=()): work = output / name work.mkdir(parents=True, exist_ok=True) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index a76d3a5fc56d8..0b3fdcaa81e8c 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -3,7 +3,7 @@ """Actual disconnect authority, DWC3 teardown and firmware resource admission.""" from pathlib import Path import re -from common import block, atc_mux_without_pm, source_defines, ATC_POST_TUNABLE +from common import block, atc_mux_without_pm, source_defines, ATC_POST_TUNABLE, tipd_pm_header def disconnect(root): # Copyright 2026 Ryan Murray @@ -21,8 +21,7 @@ def function(file, name): pos += 1 return s[m.start():pos] - pm = (r / 'drivers/usb/typec/tipd/cd321x-pm.h').read_text() - pm = pm.replace('#include ', '') + pm = tipd_pm_header(r) core = 'drivers/usb/typec/tipd/core.c' dcp = 'drivers/gpu/drm/apple/dcp.c' funcs = '\n'.join(function(f, n) for f, n in ( diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index 27b207fd7b8f8..ac1249af085ca 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -4,7 +4,7 @@ from pathlib import Path import os, re, subprocess import phy -from common import block +from common import block, tipd_pm_header def ownership(root, usb_unreachable=False): ns=phy.parking(root) @@ -373,7 +373,7 @@ def function(text, name): struct tps6598x { int lock; void *port; struct device *dev; }; struct notifier_block { int (*notifier_call)(struct notifier_block *,unsigned long,void *); }; ''' - pm = source('drivers/usb/typec/tipd/cd321x-pm.h').replace('#include ', '') + pm = tipd_pm_header(root) mocks = r''' struct cd321x { struct tps6598x tps; @@ -425,6 +425,8 @@ def function(text, name): core = source('drivers/usb/typec/tipd/core.c') switch_get = block(core, 'VISIBLE_IF_KUNIT struct typec_thunderbolt_switch *\ncd321x_tbt_switch_get(') production = switch_get.replace('VISIBLE_IF_KUNIT', 'static', 1) + '\n' + init_work = block(core, 'VISIBLE_IF_KUNIT void cd321x_init_work(') + production += init_work.replace('VISIBLE_IF_KUNIT', 'static', 1) + '\n' production += function(core, 'cd321x_register_port') + '\n' + function(core, 'cd321x_unregister_port') controls = r''' static void reset(struct cd321x *c) { @@ -453,6 +455,22 @@ def function(text, name): assert(!cd321x_pm_can_update(&budget)&&cd321x_pm_resume(&budget)); assert(cd321x_pm_link_event(&budget)); puts("PASS actual retry budget becomes stale without a fresh event"); + struct cd321x_pm_state lost={0}; + cd321x_pm_init(&lost);assert(!cd321x_pm_event_lost(&lost)&&cd321x_pm_ready(&lost)); + for(unsigned int i=0;i Date: Thu, 15 Oct 2026 10:36:40 +1100 Subject: [PATCH 1500/2680] Bluetooth: mt7932: Report the wakeup setting of the PCI function The driver sets no hdev->wakeup callback, so hci_register_dev() leaves HCI_CONN_FLAG_REMOTE_WAKEUP out of the supported connection flags. bluetoothd marks paired HID devices as wake-allowed and is refused ("Bad flag given (0x1) vs supported (0xe)") for each of them on every boot, and hci_suspend_sync() always takes the disconnect-only path. Return device_may_wakeup() of the PCI function, as hci_bcm4377 does for the Bluetooth function of the other Apple PCIe combo chips. The flag is then supported and the setting is honest: wakeup on the PCI function is off by default, so by default suspend behaves exactly as before. When userspace enables it, hci_suspend_sync() additionally leaves page scan and LE passive scan set up for the devices flagged for remote wakeup before bt7932_suspend() quiesces the transport. This does not make Bluetooth wake the system yet. What is still missing, none of which can be verified today: - bt7932_suspend() quiesces the firmware with the quiesce/restore handshake. It is not known whether the firmware keeps scanning in that state and signals the host when a wake-enabled device connects. - Nothing arms the MSI vector for wakeup, and it could not be armed: the pcie-apple MSI chips implement no irq_set_wake, so enable_irq_wake() fails on the MSI as it did on every AIC interrupt. - pcie-apple has no WAKE# handling, and whether a PME from the function reaches a host interrupt in suspend-to-idle has not been tested. (cherry picked from commit 54f3e0228a22e1a00ebeff349a47d3316c8ce23b) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/mt7932_bt_pcie.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index ef10b3d9efc6b..00aa10affe6d1 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -1012,6 +1012,19 @@ static void bt7932_hw_error(struct hci_dev *hdev, u8 code) bt_dev_err(hdev, "hardware error 0x%2.2x", code); } +/* + * Follows the wakeup setting of the PCI function, which is off by default. + * Only while it is on does hci_suspend_sync() leave page scan and LE passive + * scan set up for the devices flagged for remote wakeup. The transport is + * quiesced for system sleep either way, and no interrupt is armed for wakeup. + */ +static bool bt7932_wakeup(struct hci_dev *hdev) +{ + struct bt7932 *bt = hci_get_drvdata(hdev); + + return device_may_wakeup(&bt->pdev->dev); +} + static int bt7932_iommu(struct bt7932 *bt) { struct device *dev = &bt->pdev->dev; @@ -1216,6 +1229,7 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) bt->hdev->setup = bt7932_setup; bt->hdev->reset = bt7932_reset; bt->hdev->hw_error = bt7932_hw_error; + bt->hdev->wakeup = bt7932_wakeup; hci_set_quirk(bt->hdev, HCI_QUIRK_RESET_ON_CLOSE); hci_set_quirk(bt->hdev, HCI_QUIRK_NON_PERSISTENT_SETUP); /* System sleep suspends HCI from bt7932_suspend(), after the freeze. */ From 40a69946cadad6bd04d9b78a14aaab95fd5e5010 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 00:40:27 +1100 Subject: [PATCH 1501/2680] usb: core: Keep a hub awake in suspend-to-idle for its own wakeup While it is suspended, the VIA Labs VL122 hub on the Apple J700 passes no remote wakeup on from below (USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP), and it does not signal one for a connect, disconnect or over-current change on its own ports either. That is what made it miss downstream connects in runtime suspend: with the hub suspended and its root port armed for wakeup, a dock attached to either connector left the root port in U3, without a link state change, until something resumed the hub from the host side, see commit 601d47365fcf ("usb: core: ignore remote wakeup on the VIA VL122 hub"). So in suspend-to-idle, a VL122 with its wakeup enabled and nothing below it that can wake is suspended like any other hub armed for those changes, and none of them wakes the system. Add USB_QUIRK_HUB_NO_SUSPENDED_PORT_WAKEUP for that, and set it for the VL122. Keep a hub with it awake in suspend-to-idle when its wakeup is enabled and the host controller keeps running, as is done for a device below a hub that does not pass wakeups on, so that its status endpoint reports those changes. usb_wakeup_reaches() counts its own wakeup only while it is kept awake. The cost is an active hub and root port while wakeup is enabled for the hub, which it is not by default. Add KUnit tests that the VL122 armed for wakeup, with nothing below it that can wake, is kept awake, and so is the root hub above it, and that a hub with only the other quirk is not, but still counts as able to wake. Fixes: 8d33f6497da5 ("usb: hub: keep the VL122 awake instead of ignoring its remote wakeup") (cherry picked from commit cdb68b78f648455b8be5a841a419c8e0ff856c6b) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/core/driver.c | 18 +++-- drivers/usb/core/hub.c | 8 ++- drivers/usb/core/quirks.c | 3 +- drivers/usb/core/usb-quirks-test.c | 7 +- drivers/usb/core/usb-wakeup-test.c | 105 ++++++++++++++++++++++++++++- drivers/usb/core/usb.h | 1 + drivers/usb/host/xhci-ring.c | 6 +- include/linux/usb/quirks.h | 8 +++ 8 files changed, 140 insertions(+), 16 deletions(-) diff --git a/drivers/usb/core/driver.c b/drivers/usb/core/driver.c index 2dc8796823de3..b16aed3b6bff8 100644 --- a/drivers/usb/core/driver.c +++ b/drivers/usb/core/driver.c @@ -25,6 +25,7 @@ * */ +#include #include #include #include @@ -1390,12 +1391,14 @@ static int usb_resume_interface(struct usb_device *udev, /* * Whether to keep a device awake in suspend-to-idle so that a remote wakeup - * from below can reach the host controller: a hub that does not pass one on - * while it is suspended itself, and every device between such a hub and the - * root hub, root hub included. This only works while the host controller keeps - * running, so its interrupt reports the hub's status change. + * can reach the host controller: a hub that does not pass one on from below + * while it is suspended itself, a hub with wakeup enabled that does not signal + * the changes of its own ports while it is suspended, and every device between + * such a hub and the root hub, root hub included. This only works while the + * host controller keeps running, so its interrupt reports the hub's status + * change. */ -static bool usb_stay_awake_for_wakeup(struct usb_device *udev, pm_message_t msg) +VISIBLE_IF_KUNIT bool usb_stay_awake_for_wakeup(struct usb_device *udev, pm_message_t msg) { struct usb_hcd *hcd = bus_to_hcd(udev->bus); struct usb_device *child; @@ -1406,6 +1409,10 @@ static bool usb_stay_awake_for_wakeup(struct usb_device *udev, pm_message_t msg) !device_may_wakeup(hcd->self.controller)) return false; + if ((udev->quirks & USB_QUIRK_HUB_NO_SUSPENDED_PORT_WAKEUP) && + udev->do_remote_wakeup) + return true; + if ((udev->quirks & USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP) && udev->do_remote_wakeup && device_wakeup_path(&udev->dev)) return true; @@ -1418,6 +1425,7 @@ static bool usb_stay_awake_for_wakeup(struct usb_device *udev, pm_message_t msg) return false; } +EXPORT_SYMBOL_IF_KUNIT(usb_stay_awake_for_wakeup); /** * usb_suspend_both - suspend a USB device and its interfaces diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index 97a043aaa62c5..df313698b0918 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -3476,8 +3476,9 @@ EXPORT_SYMBOL_GPL(usb_wakeup_enabled_descendants); * suspended it: resume signalling, or for a hub, a connect, disconnect or * over-current change on one of its ports. A hub passes on the remote wakeup * of a device below it whatever its own wakeup setting (USB 2.0, section - * 7.1.7.7). A hub with USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP does not while it is - * suspended, but still signals its own; a hub kept awake does both. + * 7.1.7.7). While it is suspended, a hub with USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP + * does not, and one with USB_QUIRK_HUB_NO_SUSPENDED_PORT_WAKEUP does not signal + * its own; a hub kept awake does both. * * Context: System sleep, once the devices below @hdev are suspended, so that * neither they nor their wakeup settings change. Takes no locks. @@ -3493,7 +3494,8 @@ bool usb_wakeup_reaches(struct usb_device *hdev) /* A hub kept awake reports any change on its status endpoint */ u32 quirks = child->awake_in_sleep ? 0 : child->quirks; - if (child->do_remote_wakeup) + if (child->do_remote_wakeup && + !(quirks & USB_QUIRK_HUB_NO_SUSPENDED_PORT_WAKEUP)) return true; if (!(quirks & USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP) && usb_wakeup_reaches(child)) diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c index bf38174963e5c..50c564c46b48a 100644 --- a/drivers/usb/core/quirks.c +++ b/drivers/usb/core/quirks.c @@ -586,7 +586,8 @@ static const struct usb_device_id usb_quirk_list[] = { /* VIA Labs, Inc. VL122 USB2.0 Hub */ { USB_DEVICE(0x2109, 0x2122), .driver_info = - USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP }, + USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP | + USB_QUIRK_HUB_NO_SUSPENDED_PORT_WAKEUP }, /* VIA Labs, Inc. USB2.0 Hub */ { USB_DEVICE(0x2109, 0x2817), .driver_info = USB_QUIRK_NO_LPM }, diff --git a/drivers/usb/core/usb-quirks-test.c b/drivers/usb/core/usb-quirks-test.c index 81ebc58238b0d..5587e1d4cfa0c 100644 --- a/drivers/usb/core/usb-quirks-test.c +++ b/drivers/usb/core/usb-quirks-test.c @@ -38,9 +38,9 @@ static struct usb_quirks_test *usb_quirks_test_hub(struct kunit *test, u16 vid, } /* - * The VIA Labs VL122 hub on the Apple J700 does not pass on remote wakeup while it - * is suspended. It stays out of autosuspend and keeps its wakeup setting, and is - * kept awake in suspend-to-idle instead. + * The VIA Labs VL122 hub on the Apple J700 signals no remote wakeup while it is + * suspended, neither for its own ports nor from below. It stays out of autosuspend + * and keeps its wakeup setting, and is kept awake in suspend-to-idle instead. */ static void usb_quirks_vl122_test(struct kunit *test) { @@ -49,6 +49,7 @@ static void usb_quirks_vl122_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, quirks & USB_QUIRK_IGNORE_REMOTE_WAKEUP); KUNIT_EXPECT_TRUE(test, quirks & USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP); + KUNIT_EXPECT_TRUE(test, quirks & USB_QUIRK_HUB_NO_SUSPENDED_PORT_WAKEUP); KUNIT_EXPECT_TRUE(test, usb_hub_id_quirks(&t->intf) & HUB_QUIRK_DISABLE_AUTOSUSPEND); } diff --git a/drivers/usb/core/usb-wakeup-test.c b/drivers/usb/core/usb-wakeup-test.c index 4e8658d316dab..d39c71c9c459e 100644 --- a/drivers/usb/core/usb-wakeup-test.c +++ b/drivers/usb/core/usb-wakeup-test.c @@ -3,8 +3,11 @@ * KUnit tests for the USB core's wakeup decisions in system sleep */ +#include #include +#include #include +#include #include #include #include @@ -14,6 +17,10 @@ #define USB_WAKEUP_TEST_PORTS 4U +/* The device quirks of the VIA Labs VL122 hub */ +#define USB_WAKEUP_TEST_VL122 (USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP | \ + USB_QUIRK_HUB_NO_SUSPENDED_PORT_WAKEUP) + /* A suspended USB device, with the hub driver state of a hub */ struct usb_wakeup_test_dev { struct usb_device udev; @@ -49,6 +56,7 @@ static struct usb_wakeup_test_dev *usb_wakeup_test_add(struct kunit *test, KUNIT_ASSERT_LT(test, i, USB_WAKEUP_TEST_PORTS); parent->ports[i].child = &d->udev; d->udev.parent = &parent->udev; + d->udev.bus = parent->udev.bus; } return d; @@ -68,7 +76,7 @@ static void usb_wakeup_reaches_vl122_test(struct kunit *test) struct usb_wakeup_test_dev *hub = usb_wakeup_test_add(test, vl122, true); struct usb_wakeup_test_dev *keyboard = usb_wakeup_test_add(test, hub, false); - vl122->udev.quirks = USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP; + vl122->udev.quirks = USB_WAKEUP_TEST_VL122; keyboard->udev.do_remote_wakeup = 1; KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&hub->udev)); KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&vl122->udev)); @@ -141,12 +149,106 @@ static void usb_wakeup_reaches_nothing_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, usb_wakeup_reaches(&keyboard->udev)); } +/* + * The VL122 does not signal the changes of its own ports while it is suspended + * either: armed for wakeup, it reaches the root hub only while it is kept awake. + */ +static void usb_wakeup_reaches_armed_vl122_test(struct kunit *test) +{ + struct usb_wakeup_test_dev *root = usb_wakeup_test_add(test, NULL, true); + struct usb_wakeup_test_dev *vl122 = usb_wakeup_test_add(test, root, true); + + vl122->udev.quirks = USB_WAKEUP_TEST_VL122; + vl122->udev.do_remote_wakeup = 1; + KUNIT_EXPECT_FALSE(test, usb_wakeup_reaches(&root->udev)); + + vl122->udev.awake_in_sleep = 1; + KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&root->udev)); +} + +static void usb_wakeup_test_restore_target(void *state) +{ + pm_suspend_target_state = *(suspend_state_t *)state; +} + +/* + * A root hub on a host controller that keeps running in suspend-to-idle, with + * wakeup enabled, as the system goes to suspend-to-idle. + */ +static struct usb_wakeup_test_dev *usb_wakeup_test_s2idle_root(struct kunit *test) +{ + struct usb_wakeup_test_dev *root = usb_wakeup_test_add(test, NULL, true); + struct usb_hcd *hcd = kunit_kzalloc(test, sizeof(*hcd), GFP_KERNEL); + struct device *dev = kunit_device_register(test, "usb-wakeup-test"); + suspend_state_t *target = kunit_kzalloc(test, sizeof(*target), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, hcd); + KUNIT_ASSERT_NOT_NULL(test, target); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + KUNIT_ASSERT_EQ(test, devm_device_init_wakeup(dev), 0); + hcd->self.controller = dev; + hcd->runs_in_s2idle = 1; + root->udev.bus = &hcd->self; + + *target = pm_suspend_target_state; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, usb_wakeup_test_restore_target, + target), 0); + pm_suspend_target_state = PM_SUSPEND_TO_IDLE; + + return root; +} + +/* + * The VL122 armed for wakeup, with nothing below it that can wake: it is kept + * awake in suspend-to-idle for the changes of its own ports, and so is the root + * hub above it, but not for other system sleep transitions, or on a controller + * that halts or may not wake the system. A hub that signals those changes while + * it is suspended is not kept awake for them. + */ +static void usb_wakeup_stay_awake_armed_vl122_test(struct kunit *test) +{ + struct usb_wakeup_test_dev *root = usb_wakeup_test_s2idle_root(test); + struct usb_wakeup_test_dev *vl122 = usb_wakeup_test_add(test, root, true); + struct usb_hcd *hcd = bus_to_hcd(root->udev.bus); + + usb_wakeup_test_add(test, vl122, false); + vl122->udev.quirks = USB_WAKEUP_TEST_VL122; + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + + vl122->udev.do_remote_wakeup = 1; + KUNIT_EXPECT_TRUE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_FREEZE)); + + pm_suspend_target_state = PM_SUSPEND_MEM; + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + pm_suspend_target_state = PM_SUSPEND_TO_IDLE; + + hcd->runs_in_s2idle = 0; + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + hcd->runs_in_s2idle = 1; + + device_set_wakeup_enable(hcd->self.controller, false); + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + device_set_wakeup_enable(hcd->self.controller, true); + + vl122->udev.awake_in_sleep = 1; + KUNIT_EXPECT_TRUE(test, usb_stay_awake_for_wakeup(&root->udev, PMSG_SUSPEND)); + KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&root->udev)); + + vl122->udev.awake_in_sleep = 0; + vl122->udev.quirks = USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP; + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&root->udev)); +} + static struct kunit_case usb_wakeup_test_cases[] = { KUNIT_CASE(usb_wakeup_reaches_vl122_test), KUNIT_CASE(usb_wakeup_reaches_hub_test), KUNIT_CASE(usb_wakeup_reaches_armed_hub_test), KUNIT_CASE(usb_wakeup_reaches_armed_quirky_hub_test), KUNIT_CASE(usb_wakeup_reaches_nothing_test), + KUNIT_CASE(usb_wakeup_reaches_armed_vl122_test), + KUNIT_CASE(usb_wakeup_stay_awake_armed_vl122_test), {} }; @@ -159,3 +261,4 @@ kunit_test_suite(usb_wakeup_test_suite); MODULE_DESCRIPTION("KUnit tests for the USB core's wakeup decisions in system sleep"); MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/drivers/usb/core/usb.h b/drivers/usb/core/usb.h index c8880c20f1563..771902dcb3d1b 100644 --- a/drivers/usb/core/usb.h +++ b/drivers/usb/core/usb.h @@ -38,6 +38,7 @@ extern void usb_authorize_interface(struct usb_interface *); extern void usb_detect_quirks(struct usb_device *udev); #if IS_ENABLED(CONFIG_KUNIT) u32 usb_detect_device_list_quirks(struct usb_device *udev); +bool usb_stay_awake_for_wakeup(struct usb_device *udev, pm_message_t msg); #endif extern void usb_detect_interface_quirks(struct usb_device *udev); extern void usb_release_quirk_list(void); diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c index c6f4df660cdb7..5ffe833e5e45a 100644 --- a/drivers/usb/host/xhci-ring.c +++ b/drivers/usb/host/xhci-ring.c @@ -2043,9 +2043,9 @@ static bool xhci_s2idle_port_change(struct xhci_hcd *xhci, struct xhci_port *por } /* - * Only hubs that the USB core kept awake to pass on a remote wakeup from below - * have transfers in flight, see usb_suspend_both(): a completed one reports - * a change on one of their ports. + * Only hubs that the USB core kept awake for a remote wakeup have transfers in + * flight, see usb_suspend_both(): a completed one reports a change on one of + * their ports. */ VISIBLE_IF_KUNIT bool xhci_s2idle_transfer_wakeup(const struct usb_device *udev) { diff --git a/include/linux/usb/quirks.h b/include/linux/usb/quirks.h index 8bcceb0d02bd8..a77d3eaf8dc4c 100644 --- a/include/linux/usb/quirks.h +++ b/include/linux/usb/quirks.h @@ -91,4 +91,12 @@ */ #define USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP BIT(20) +/* + * Hub does not signal a remote wakeup for a connect, disconnect or over-current + * change on its own ports while it is suspended. Keep it awake in + * suspend-to-idle when its wakeup is enabled and the host controller keeps + * running. + */ +#define USB_QUIRK_HUB_NO_SUSPENDED_PORT_WAKEUP BIT(21) + #endif /* __LINUX_USB_QUIRKS_H */ From 1d1c18ad5308cee4242561f30d7f8fa85f5cc8ed Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:33:44 +1100 Subject: [PATCH 1502/2680] drm/apple: afk: Do not warn when a retired channel is closed afk_recv_handle_teardown() warns when the co-processor closes a channel that has no live service. On M1 and M2 that should not happen, because the driver never gives up a service on its own. Since the H17P reconnect support, it gives up services in two places: - dpavservep_detach() retires the DCPAVSERV EDID service when the external display goes away, before the firmware closes its channel; - afk_service_discard() drops a service whose preparation declined it, such as the optional AppleDCPDP2HDMI service on its second channel. The firmware then closes both channels on every unplug and the driver prints two "teardown for disabled channel" warnings, although it has nothing left to do. Log a teardown at debug level when the slot that last served the channel is torn down, and keep the warning for a channel no slot knows. A slot that is reused for another channel forgets the old one, so an unexpected close is still reported. Rename that warning to say "unknown", which is the only case left. Add KUnit cases for a retired, a declined, a reused and a live slot. (cherry picked from commit c7c9a17ef9980368d3cf928bdc400ce7235c544a) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk-service-test.c | 33 ++++++++++++++++++++++++ drivers/gpu/drm/apple/afk.c | 19 ++++++++++++-- drivers/gpu/drm/apple/afk.h | 16 ++++++++++++ 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/afk-service-test.c b/drivers/gpu/drm/apple/afk-service-test.c index 1cf589ae7a417..a64064758a906 100644 --- a/drivers/gpu/drm/apple/afk-service-test.c +++ b/drivers/gpu/drm/apple/afk-service-test.c @@ -218,6 +218,37 @@ static void dptx_service_duplicate_close_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, port.enabled); } +static void afk_service_retired_channel_test(struct kunit *test) +{ + struct apple_epic_service *service = test->priv; + + /* A slot that never served the channel does not know it. */ + KUNIT_EXPECT_FALSE(test, afk_service_retired(service, 1)); + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 1)); + /* A live service is found by afk_service_matches(), not here. */ + KUNIT_EXPECT_FALSE(test, afk_service_retired(service, 1)); + /* Retired locally (display unplugged) before the co-processor closes it. */ + afk_test_retire(service); + KUNIT_EXPECT_FALSE(test, afk_service_matches(service, 1)); + KUNIT_EXPECT_TRUE(test, afk_service_retired(service, 1)); + KUNIT_EXPECT_FALSE(test, afk_service_retired(service, 2)); + /* Reusing the slot forgets the old channel. */ + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); + KUNIT_EXPECT_FALSE(test, afk_service_retired(service, 1)); + KUNIT_EXPECT_FALSE(test, afk_service_retired(service, 2)); +} + +static void afk_service_declined_channel_test(struct kunit *test) +{ + struct apple_epic_service *service = test->priv; + + /* An announcement whose preparation we declined is closed later too. */ + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &prepared_ops, 3)); + afk_service_discard(service); + KUNIT_EXPECT_FALSE(test, afk_service_matches(service, 3)); + KUNIT_EXPECT_TRUE(test, afk_service_retired(service, 3)); +} + static struct kunit_case afk_service_cases[] = { KUNIT_CASE(afk_service_admission_test), KUNIT_CASE(dptx_service_identity_test), @@ -228,6 +259,8 @@ static struct kunit_case afk_service_cases[] = { KUNIT_CASE(afk_service_completed_reader_test), KUNIT_CASE(afk_service_tag_sequence_test), KUNIT_CASE(afk_service_unconverted_owner_test), + KUNIT_CASE(afk_service_retired_channel_test), + KUNIT_CASE(afk_service_declined_channel_test), { } }; diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 4f19df565cfaf..b59d9edf22f0a 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -344,6 +344,17 @@ static struct apple_epic_service *afk_epic_find_service(struct apple_dcp_afkep * return NULL; } +static bool afk_epic_channel_retired(struct apple_dcp_afkep *ep, u32 channel) +{ + u32 i; + + for (i = 0; i < ep->num_channels; i++) + if (afk_service_retired(&ep->services[i], channel)) + return true; + + return false; +} + static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, enum epic_type etype, enum epic_category ecat, u8 stype, const void *payload, size_t payload_len, u64 token); @@ -579,8 +590,12 @@ static void afk_recv_handle_teardown(struct apple_dcp_afkep *ep, u32 channel) service = afk_epic_find_service(ep, channel); if (!service) { - dev_warn(ep->dcp->dev, "AFK[ep:%02x]: teardown for disabled channel %u\n", - ep->endpoint, channel); + if (afk_epic_channel_retired(ep, channel)) + dev_dbg(ep->dcp->dev, "AFK[ep:%02x]: teardown for retired channel %u\n", + ep->endpoint, channel); + else + dev_warn(ep->dcp->dev, "AFK[ep:%02x]: teardown for unknown channel %u\n", + ep->endpoint, channel); return; } diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index a85ed77597755..b548c0bbfbcbc 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -191,6 +191,22 @@ static inline bool afk_service_matches(struct apple_epic_service *service, return found; } +/* + * True if this slot last served @channel and was retired or discarded on our + * side: the co-processor may still close that channel afterwards. + */ +static inline bool afk_service_retired(struct apple_epic_service *service, + u32 channel) +{ + unsigned long flags; + bool retired; + + spin_lock_irqsave(&service->lock, flags); + retired = service->channel == channel && service->torndown; + spin_unlock_irqrestore(&service->lock, flags); + return retired; +} + /* The caller holds service->lock. */ static inline bool afk_service_reclaimable(struct apple_epic_service *service) { From c3eb72d03f4ff97761c0a60ac639006dd7a1a098 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:24:02 +1100 Subject: [PATCH 1503/2680] PCI: apple: Let MSIs wake the system enable_irq_wake() fails with -ENXIO for every MSI of a function behind this host bridge. irq_set_irq_wake() only looks at the top-level chip, which for a PCI function is the per-device PCI/MSI chip built from the generic template; msi_lib_init_dev_msi_info() gives it neither an .irq_set_wake callback nor IRQCHIP_SKIP_SET_WAKE, and the MSI bottom chip of this driver does not forward the request either. A function driver that arms its interrupt for wakeup, and the PCIe port PME service, are therefore left without a wake interrupt, suspend_device_irqs() disables the vector, and the function cannot end suspend-to-idle. Each vector is an AIC interrupt of its own (see apple_msi_domain_alloc()), so it can wake the system exactly when that AIC line can. Forward the wake setting down the hierarchy: give the bottom chip irq_chip_set_wake_parent(), and set the same callback on the per-device chip from an init_dev_msi_info() wrapper, as the GICv3 ITS does to adjust its device domains. With the AIC accepting wake requests ("irqchip/ apple-aic: Allow interrupts to wake the system"), enable_irq_wake() on a PCI MSI then succeeds, the vector stays enabled across suspend-to-idle and its next message wakes the system. Nothing is armed by this change alone: it only lets drivers that ask for a wake interrupt get one. (cherry picked from commit 9ccab018d67f2aaacd7ca9a70e35c7eceb6b0da4) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pci/controller/pcie-apple.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 3db824f642279..f1dd255d90d2b 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -474,6 +474,7 @@ static struct irq_chip apple_msi_bottom_chip = { .irq_eoi = irq_chip_eoi_parent, .irq_set_affinity = irq_chip_set_affinity_parent, .irq_set_type = irq_chip_set_type_parent, + .irq_set_wake = irq_chip_set_wake_parent, .irq_compose_msi_msg = apple_msi_compose_msg, }; @@ -1774,6 +1775,26 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, return ret; } +/* + * Each MSI vector is an AIC interrupt of its own, so a vector can wake the + * system exactly when the AIC line behind it can. Forward the wake setting + * from the per-device PCI/MSI chip down to the AIC, which otherwise fails + * enable_irq_wake() for every MSI and leaves PCIe functions without a way + * to end suspend-to-idle. + */ +static bool apple_msi_init_dev_msi_info(struct device *dev, + struct irq_domain *domain, + struct irq_domain *real_parent, + struct msi_domain_info *info) +{ + if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info)) + return false; + + if (!info->chip->irq_set_wake) + info->chip->irq_set_wake = irq_chip_set_wake_parent; + return true; +} + static const struct msi_parent_ops apple_msi_parent_ops = { .supported_flags = (MSI_GENERIC_FLAGS_MASK | MSI_FLAG_PCI_MSIX | @@ -1783,7 +1804,7 @@ static const struct msi_parent_ops apple_msi_parent_ops = { MSI_FLAG_PCI_MSI_MASK_PARENT), .chip_flags = MSI_CHIP_FLAG_SET_EOI, .bus_select_token = DOMAIN_BUS_PCI_MSI, - .init_dev_msi_info = msi_lib_init_dev_msi_info, + .init_dev_msi_info = apple_msi_init_dev_msi_info, }; static int apple_msi_init(struct apple_pcie *pcie) From eb2d9e5caca2f9a77769bba87105b6ca15b480d0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 21:50:09 +1100 Subject: [PATCH 1504/2680] usb: xhci: plat: Keep running in suspend-to-idle only when USB can wake XHCI_RUN_IN_S2IDLE keeps the controller running in suspend-to-idle when wakeup is enabled for it and for a device below it, so that the device can wake the system. That costs power, and a device with wakeup enabled does not mean that its remote wakeup gets through. On a J700 at its defaults, wakeup is enabled for the controller and, by usbhid, for the USB keyboards, and disabled for the hubs and root hubs. The VIA Labs VL122 hub between the keyboards and the root hub passes no remote wakeup on while it is suspended, and the USB core keeps it awake only when its own wakeup is enabled. So the controller was kept running, and a key press did not wake the system: usb usb1: bus suspend, wakeup 0 usb usb2: bus suspend, wakeup 0 xhci-hcd xhci-hcd.0.auto: wakeup enabled, keep running in suspend-to-idle. Keep it running only when a USB device can wake the system through one of its root hubs: the root hub is armed for connect, disconnect and over-current changes, or kept awake for a hub below it, or the remote wakeup of a device below reaches it, see usb_wakeup_reaches(). Halt it as before otherwise. Add a KUnit test of the decision, with the J700 defaults among its cases. (cherry picked from commit ca00d3fc3aad0f4978458be7f9fc8ec5c899a487) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/host/xhci-plat-test.c | 38 +++++++++++++++++++++++++++---- drivers/usb/host/xhci-plat.c | 27 ++++++++++++++++++---- drivers/usb/host/xhci-plat.h | 3 ++- 3 files changed, 57 insertions(+), 11 deletions(-) diff --git a/drivers/usb/host/xhci-plat-test.c b/drivers/usb/host/xhci-plat-test.c index b07b3f985ea26..49cbd15e4ed34 100644 --- a/drivers/usb/host/xhci-plat-test.c +++ b/drivers/usb/host/xhci-plat-test.c @@ -18,7 +18,7 @@ #include "xhci.h" #include "xhci-plat.h" -/* Only suspend-to-idle with the quirk, wakeup enabled and a wakeup path keeps it running. */ +/* Only suspend-to-idle with the quirk, wakeup enabled and a USB wakeup keeps it running. */ static void xhci_plat_keep_running_test(struct kunit *test) { static const suspend_state_t targets[] = { @@ -29,19 +29,46 @@ static void xhci_plat_keep_running_test(struct kunit *test) for (i = 0; i < ARRAY_SIZE(targets); i++) { for (combo = 0; combo < 8; combo++) { u64 quirks = (combo & 1) ? XHCI_RUN_IN_S2IDLE : 0; - bool may_wakeup = combo & 2, wakeup_path = combo & 4; + bool may_wakeup = combo & 2, usb_wakeup = combo & 4; bool expected = combo == 7 && targets[i] == PM_SUSPEND_TO_IDLE; bool kept = xhci_plat_keep_running(quirks, targets[i], may_wakeup, - wakeup_path); + usb_wakeup); KUNIT_EXPECT_EQ_MSG(test, kept, expected, - "target %d quirk %d may_wakeup %d wakeup_path %d", - targets[i], combo & 1, may_wakeup, wakeup_path); + "target %d quirk %d may_wakeup %d usb_wakeup %d", + targets[i], combo & 1, may_wakeup, usb_wakeup); } } } +/* + * Something below a root hub has to be able to wake the system, which the USB core decided as it + * suspended the devices. The J700 defaults are the first case: wakeup is enabled for a keyboard, + * but disabled for the VL122 hub between it and the root hub, which does not pass the remote + * wakeup on while it is suspended, and for the root hub. Nothing can wake the system through it. + */ +static void xhci_plat_usb_wakeup_test(struct kunit *test) +{ + struct usb_hcd *hcd = kunit_kzalloc(test, sizeof(*hcd), GFP_KERNEL); + struct usb_device *rhdev = kunit_kzalloc(test, sizeof(*rhdev), GFP_KERNEL); + + KUNIT_ASSERT_TRUE(test, hcd && rhdev); + KUNIT_EXPECT_FALSE(test, xhci_plat_usb_wakeup(NULL)); + KUNIT_EXPECT_FALSE(test, xhci_plat_usb_wakeup(hcd)); + hcd->self.root_hub = rhdev; + KUNIT_EXPECT_FALSE(test, xhci_plat_usb_wakeup(hcd)); + + /* Armed for connect, disconnect and over-current changes */ + rhdev->do_remote_wakeup = 1; + KUNIT_EXPECT_TRUE(test, xhci_plat_usb_wakeup(hcd)); + rhdev->do_remote_wakeup = 0; + + /* Kept awake for a hub below, see usb_suspend_both() */ + rhdev->awake_in_sleep = 1; + KUNIT_EXPECT_TRUE(test, xhci_plat_usb_wakeup(hcd)); +} + /* Other quirks do not stand in for the suspend-to-idle one. */ static void xhci_plat_keep_running_other_quirks_test(struct kunit *test) { @@ -289,6 +316,7 @@ static void xhci_plat_all_property_quirks_test(struct kunit *test) static struct kunit_case xhci_plat_test_cases[] = { KUNIT_CASE(xhci_plat_keep_running_test), KUNIT_CASE(xhci_plat_keep_running_other_quirks_test), + KUNIT_CASE(xhci_plat_usb_wakeup_test), KUNIT_CASE(xhci_plat_irq_flags_test), KUNIT_CASE(xhci_plat_stopped_test), KUNIT_CASE(xhci_plat_quiesce_dying_test), diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index ad0e2807a862b..b82ed7dfdc864 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -559,16 +559,32 @@ static int xhci_plat_suspend_common(struct device *dev) * leave it running instead, with the root hubs suspended and armed for wakeup as usual. The * controller then interrupts for every event, its interrupt stays enabled (see * xhci_plat_irq_flags()), and the event handling wakes the system for the events that a halted - * controller would signal. + * controller would signal. That costs power, so do it only when a USB device can wake the system + * through one of its root hubs, see xhci_plat_usb_wakeup(). */ VISIBLE_IF_KUNIT bool xhci_plat_keep_running(u64 quirks, suspend_state_t target, - bool may_wakeup, bool wakeup_path) + bool may_wakeup, bool usb_wakeup) { return (quirks & XHCI_RUN_IN_S2IDLE) && target == PM_SUSPEND_TO_IDLE && - may_wakeup && wakeup_path; + may_wakeup && usb_wakeup; } EXPORT_SYMBOL_IF_KUNIT(xhci_plat_keep_running); +/* + * Whether a USB device can wake the system through the root hub of @hcd, once the USB core has + * suspended them all: the root hub is armed for connect, disconnect and over-current changes, or + * kept awake for a hub below it, or the remote wakeup of a device below reaches it. A device with + * wakeup enabled does not count if a hub on its way does not pass the wakeup on. + */ +VISIBLE_IF_KUNIT bool xhci_plat_usb_wakeup(struct usb_hcd *hcd) +{ + struct usb_device *rhdev = hcd ? hcd->self.root_hub : NULL; + + return rhdev && (rhdev->do_remote_wakeup || rhdev->awake_in_sleep || + usb_wakeup_reaches(rhdev)); +} +EXPORT_SYMBOL_IF_KUNIT(xhci_plat_usb_wakeup); + /* Whether a controller that was left running reads back as stopped or failed */ VISIBLE_IF_KUNIT bool xhci_plat_stopped(u32 usbsts) { @@ -581,8 +597,9 @@ static bool xhci_plat_run_in_s2idle(struct device *dev) struct usb_hcd *hcd = dev_get_drvdata(dev); struct xhci_hcd *xhci = hcd_to_xhci(hcd); - return xhci_plat_keep_running(xhci->quirks, pm_suspend_target_state, - device_may_wakeup(dev), device_wakeup_path(dev)); + return xhci_plat_keep_running(xhci->quirks, pm_suspend_target_state, device_may_wakeup(dev), + xhci_plat_usb_wakeup(xhci->main_hcd) || + xhci_plat_usb_wakeup(xhci->shared_hcd)); } static int xhci_plat_suspend(struct device *dev) diff --git a/drivers/usb/host/xhci-plat.h b/drivers/usb/host/xhci-plat.h index 97d7b8475b9be..0c4affdb5236d 100644 --- a/drivers/usb/host/xhci-plat.h +++ b/drivers/usb/host/xhci-plat.h @@ -39,7 +39,8 @@ extern const struct dev_pm_ops xhci_plat_pm_ops; u64 xhci_plat_property_quirks(struct device *dev); unsigned long xhci_plat_irq_flags(u64 quirks); bool xhci_plat_keep_running(u64 quirks, suspend_state_t target, bool may_wakeup, - bool wakeup_path); + bool usb_wakeup); +bool xhci_plat_usb_wakeup(struct usb_hcd *hcd); bool xhci_plat_stopped(u32 usbsts); int xhci_plat_resume_from_runtime(struct device *dev); #endif From 6007d5749e5db28bb2e7e298e86aec39f84c1650 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:33:44 +1100 Subject: [PATCH 1505/2680] drm/apple: iomfb: Log the J700 power-LUT fallback at info level The H17P DCP asks for five power-LUT tables through the provider property callback. Commit 22778725c286 ("drm: apple: retain J700 provider replies with older loaders") answers with the J700 tables when the device tree lacks them, because the loader in use predates those properties, and warns once that it does so. No released loader adds these properties yet, so the warning appears on every J700 boot. The reply carries the same bytes as the disp0 entries of the Apple Device Tree, and both displays work with it, so there is no fault to report. Log the fallback at info level instead. The message still names the missing property, and properties that are present but malformed are still errors. (cherry picked from commit bcd2e166aadb9457956435a57ad9a61b92d8467f) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 1460d883372ea..e6c5d85db122b 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -453,7 +453,7 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, memset(reply, 0, sizeof(*reply)); memcpy(reply->data, property->legacy_data, property->length); reply->length = cpu_to_le32(property->length); - dev_warn_once(dcp->dev, + dev_info_once(dcp->dev, "missing provider property %s (%u bytes); using the J700 defaults\n", property->dt_name, property->length); return true; From 9d7b90e70d54ef479ba7a373bc89332fcfaecb7b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:35:15 +1100 Subject: [PATCH 1506/2680] rust: types: keep the OwnableMut safety section in its doc comment A blank source line separates the OwnableMut description from its safety section. Mark it as a documentation paragraph so Clippy does not reject the kernel crate with empty_line_after_doc_comments before checking Rust drivers. (cherry picked from commit e6dc18912d8788972ee18ce8f038745d2571ac32) Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/kernel/types.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 76b9d8589945e..0a83266e492ac 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -456,7 +456,7 @@ pub unsafe trait Ownable { /// /// When this type is implemented for an Ownable type, it allows Owned to be dereferenced /// into a &mut T. - +/// /// # Safety /// /// Implementers must ensure that the kernel never mutates the underlying type while From 30a28f610695a24abd62180fd6589e2e267b73b1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:12:53 +1100 Subject: [PATCH 1507/2680] Bluetooth: mt7932: Keep the controller running for wakeup in suspend-to-idle A paired keyboard or mouse cannot wake the machine. When wakeup is enabled on the PCI function, hci_suspend_dev() sets up page scan and LE passive scan for the devices flagged for remote wakeup, but bt7932_suspend() then quiesces the firmware with the quiesce handshake and lets the PCI core put the function into D3hot, and nothing arms the interrupt. No connection request or advertising report from a wake device can reach the host. In suspend-to-idle with wakeup enabled and the controller up, leave the transport running and wake through its MSI instead: - Track wake reports from the start of the suspend callback, before waiting for queued HCI work and before hci_suspend_dev(). An event that the HCI core records as a remote wake reason (a BR/EDR connection request or completion, an LE advertising report while no discovery runs) is reported with pm_wakeup_dev_event(), so a report that arrives before the interrupt is suspended aborts the suspend rather than being consumed unnoticed. A Hardware Error does the same, so that HCI recovery runs on an awake system. Command replies and vendor events never count, so the suspend setup cannot abort itself. - Arm the MSI with enable_irq_wake(). If it cannot be armed, quiesce as before. - Block transmission until resume. HCI recovery, whose work is not frozen for system sleep and which would need to transmit, waits in the open callback until resume instead of failing its setup. - Save the PCI state in suspend_noirq, which keeps the PCI core from moving the function out of D0, and fail the suspend there if the transport faulted after it was armed. - Hold the interrupt off from resume_noirq until resume, after the PCI core has restored that state. - Whatever raises a terminal fault while that saved state is outstanding saves it again after clearing bus mastering, so that the PCI core never restores bus mastering for a failed transport. A suspend aborted before suspend_noirq has no saved state to restore. With wakeup disabled (the default), with the controller down, or for a user channel, suspend is unchanged. The firmware does not raise interrupts while idle (none in 60 s on an idle J700), so an armed MSI only fires for traffic the wake setup asked for. The wake signal is the function's MSI only; endpoint PME is not enabled on this path. Why the quiesce exists, and why skipping it here holds: "Bluetooth: mt7932: support modules and system suspend" quiesces the firmware so that the PCI power transitions of system sleep never meet a running firmware. This path makes no PCI power transition: the saved state leaves the function in D0 with bus mastering and the DMA arena untouched, the arrangement that "wifi: mt7932: keep the function running across suspend-to-idle" qualified for the Wi-Fi function after finding that clearing bus mastering under running firmware hangs the SoC. The PCIe link and host bridge stay up for that function anyway. The quiesce still runs whenever wakeup is off. ASPM and clock PM stay disabled on the link as the transport was admitted, which also keeps the link in L0 for the MSI. HCI is still suspended only from bt7932_suspend(), after the queued HCI work ("Bluetooth: mt7932: don't cancel HCI commands at suspend entry"). KUnit covers which received events end the sleep. This needs "PCI: apple: Let MSIs wake the system" for enable_irq_wake() to succeed. (cherry picked from commit 7d9f156ec2dd1ce4489a98d805ae37a3e37e18c4) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/Kconfig | 5 +- drivers/bluetooth/mt7932_bt_pcie.c | 257 ++++++++++++++++++- drivers/bluetooth/mt7932_bt_pcie_wake_test.c | 81 ++++++ 3 files changed, 336 insertions(+), 7 deletions(-) create mode 100644 drivers/bluetooth/mt7932_bt_pcie_wake_test.c diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index a090b54b5f303..b35637d575a34 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -50,9 +50,10 @@ config BT_MTK7932_PCIE_KUNIT_TEST help Build KUnit tests into the MT7932 Bluetooth transport. They check, without radio hardware, that HCI packets whose length does not - match their header are rejected in both directions, and that a + match their header are rejected in both directions, that a board file with a bad layout or checksum is never sent to the - controller. + controller, and which received events end a system sleep with + Bluetooth wakeup enabled. If unsure, say N. diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 00aa10affe6d1..6c0c4b3d33c09 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -25,8 +25,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -139,6 +141,15 @@ struct bt7932 { bool enabled, regions, vectors, irq_requested, retained; bool ipc_ready, opened, awake, waking, fault, registered; bool stopping, runtime_forbidden; + /* + * Suspend-to-idle with the transport left running for wakeup, see + * bt7932_suspend(). Written under lock except irq_held. + */ + bool wake_tracking; /* wake reports become PM wakeup events */ + bool wake_armed; /* the MSI is armed for wakeup */ + bool tx_blocked; /* nothing is sent until resume */ + bool irq_held; /* IRQ off until the PCI state is restored */ + bool pci_saved; /* saved PCI state not yet restored */ enum bt7932_recovery recovery; enum bt7932_pm_state pm_state; }; @@ -181,6 +192,12 @@ static void bt7932_fault_locked(struct bt7932 *bt, const char *reason) bt7932_counter(bt, bt->host_transfer, channel), bt7932_counter(bt, bt->fw_transfer, channel), bt->rings[channel].previous); pci_clear_master(bt->pdev); + /* + * Whatever raised the fault, a saved state that the PCI core has yet + * to restore must not bring bus mastering back: save it again. + */ + if (bt->pci_saved) + pci_save_state(bt->pdev); if (bt->irq_requested) disable_irq_nosync(bt->irq); wake_up_all(&bt->wake_wait); @@ -222,6 +239,64 @@ static bool bt7932_hci_length(u8 packet, const u8 *data, size_t len) return len == expected; } +enum bt7932_wake { + BT7932_WAKE_NONE, + /* A remote device asks for the host. */ + BT7932_WAKE_REMOTE, + /* An LE advertising report: remote, unless discovery is running. */ + BT7932_WAKE_ADVERTISING, + /* The controller lost its state; HCI recovery is coming. */ + BT7932_WAKE_ERROR, +}; + +/* + * Classifies a received event for a system sleep with wakeup armed. The + * remote events are those that the HCI core records as a remote wake reason + * while suspended (see hci_store_wake_reason()): a BR/EDR connection request + * or completion, or an LE advertising report. A Hardware Error must end the + * sleep too, so that recovery runs on an awake system. Command Complete and + * Status, vendor events and the replies to the suspend setup never count. + * The caller has checked the length against the event header. + */ +static enum bt7932_wake bt7932_wake_event(const u8 *data, size_t len) +{ + switch (data[0]) { + case HCI_EV_CONN_COMPLETE: + case HCI_EV_CONN_REQUEST: + case HCI_EV_SYNC_CONN_COMPLETE: + return BT7932_WAKE_REMOTE; + case HCI_EV_HARDWARE_ERROR: + return BT7932_WAKE_ERROR; + case HCI_EV_LE_META: + if (len < HCI_EVENT_HDR_SIZE + 1) + return BT7932_WAKE_NONE; + switch (data[HCI_EVENT_HDR_SIZE]) { + case HCI_EV_LE_ADVERTISING_REPORT: + case HCI_EV_LE_DIRECT_ADV_REPORT: + case HCI_EV_LE_EXT_ADV_REPORT: + return BT7932_WAKE_ADVERTISING; + } + return BT7932_WAKE_NONE; + default: + return BT7932_WAKE_NONE; + } +} + +/* Caller owns lock and has found wake tracking on. */ +static bool bt7932_wake_report(struct bt7932 *bt, const u8 *data, size_t len) +{ + switch (bt7932_wake_event(data, len)) { + case BT7932_WAKE_REMOTE: + case BT7932_WAKE_ERROR: + return true; + case BT7932_WAKE_ADVERTISING: + /* Discovery is paused once HCI suspends; until then it is not a wake. */ + return READ_ONCE(bt->hdev->discovery.state) == DISCOVERY_STOPPED; + default: + return false; + } +} + static int bt7932_validate_ptx(const u8 *data, size_t size) { static const u32 types[] = { 0x101, 0x201, 0x301, 0x401 }; @@ -485,7 +560,7 @@ static int bt7932_wake_and_lock(struct bt7932 *bt, bool require_open) for (;;) { mutex_lock(&bt->lock); if (bt->fault || bt->stopping || bt->pm_state != BT7932_RUNNING || - (require_open && !bt->opened)) { + bt->tx_blocked || (require_open && !bt->opened)) { mutex_unlock(&bt->lock); return -EHOSTDOWN; } @@ -627,6 +702,15 @@ static int bt7932_rx_locked(struct bt7932 *bt, unsigned int channel) hci_skb_pkt_type(skb) = geo->packet; skb_put_data(skb, descriptor + 16, length); bt->hdev->stat.byte_rx += length; + /* + * From the start of bt7932_suspend() until resume, a + * wake report ends the system sleep that is under way: + * before the interrupt is suspended, nothing else would + * remember that it arrived. + */ + if (geo->packet == HCI_EVENT_PKT && bt->wake_tracking && + bt7932_wake_report(bt, descriptor + 16, length)) + pm_wakeup_dev_event(&bt->pdev->dev, 0, true); hci_recv_frame(bt->hdev, skb); } slot = (slot + 1) & BT7932_MASK; @@ -829,6 +913,14 @@ static int bt7932_open(struct hci_dev *hdev) struct bt7932 *bt = hci_get_drvdata(hdev); int ret = 0; + /* + * The HCI core's recovery work is not frozen for system sleep. While + * the transport sleeps with wakeup armed nothing may be sent, so a + * recovery that reopens the device waits here for resume rather than + * failing its setup and faulting the transport. bt7932_resume() + * unblocks it before it resumes HCI, which waits for this open. + */ + wait_event(bt->wake_wait, !READ_ONCE(bt->tx_blocked)); mutex_lock(&bt->lock); if (bt->fault || !bt->ipc_ready || bt->stopping || bt->pm_state != BT7932_RUNNING) @@ -868,7 +960,8 @@ static int bt7932_send(struct hci_dev *hdev, struct sk_buff *skb) int ret; if (READ_ONCE(bt->fault) || !READ_ONCE(bt->opened) || - READ_ONCE(bt->stopping) || READ_ONCE(bt->pm_state) != BT7932_RUNNING) + READ_ONCE(bt->stopping) || READ_ONCE(bt->tx_blocked) || + READ_ONCE(bt->pm_state) != BT7932_RUNNING) return -EHOSTDOWN; if (channel < 0 || skb->len > geometry[channel].payload) return -EMSGSIZE; @@ -1015,8 +1108,8 @@ static void bt7932_hw_error(struct hci_dev *hdev, u8 code) /* * Follows the wakeup setting of the PCI function, which is off by default. * Only while it is on does hci_suspend_sync() leave page scan and LE passive - * scan set up for the devices flagged for remote wakeup. The transport is - * quiesced for system sleep either way, and no interrupt is armed for wakeup. + * scan set up for the devices flagged for remote wakeup, and only then does + * bt7932_suspend() keep the transport running with its interrupt armed. */ static bool bt7932_wakeup(struct hci_dev *hdev) { @@ -1371,20 +1464,133 @@ static bool bt7932_hci_pm(struct bt7932 *bt) return !hci_dev_test_flag(bt->hdev, HCI_USER_CHANNEL); } +/* + * Suspend-to-idle can keep the transport running so that the controller can + * wake the system: with wakeup enabled on the PCI function, the controller + * up and HCI suspended by this driver. + */ +static bool bt7932_may_wake(struct bt7932 *bt) +{ + return device_may_wakeup(&bt->pdev->dev) && pm_suspend_no_platform() && + bt7932_hci_pm(bt) && test_bit(HCI_UP, &bt->hdev->flags); +} + +static void bt7932_track_wake(struct bt7932 *bt, bool on) +{ + mutex_lock(&bt->lock); + bt->wake_tracking = on; + mutex_unlock(&bt->lock); +} + +/* + * Leave the firmware running with its MSI armed. It keeps the scans that + * hci_suspend_dev() set up and reports a match like any other event. Returns + * 0 when armed, -EAGAIN when the transport should be quiesced instead, or + * -EIO when it failed. + */ +static int bt7932_arm_wakeup(struct bt7932 *bt) +{ + struct device *dev = &bt->pdev->dev; + int ret; + + ret = enable_irq_wake(bt->irq); + if (ret) { + dev_warn(dev, "cannot arm the interrupt for wakeup: %d\n", ret); + return -EAGAIN; + } + + mutex_lock(&bt->lock); + ret = bt->fault ? -EIO : 0; + if (!ret) { + bt->wake_armed = true; + /* A failed wake handshake would fault the transport. */ + bt->tx_blocked = true; + } + mutex_unlock(&bt->lock); + if (ret) { + disable_irq_wake(bt->irq); + return ret; + } + /* Drop what HCI queued after its suspend, as the quiesce does. */ + cancel_work_sync(&bt->tx_work); + skb_queue_purge(&bt->tx_queue); + return 0; +} + +/* + * Ends the armed state on resume, or after a suspend that was aborted. The + * interrupt is still disabled if bt7932_resume_noirq() held it. + */ +static void bt7932_disarm_wakeup(struct bt7932 *bt) +{ + disable_irq_wake(bt->irq); + mutex_lock(&bt->lock); + bt->wake_armed = false; + bt->tx_blocked = false; + bt->wake_tracking = false; + bt->pci_saved = false; + mutex_unlock(&bt->lock); + wake_up_all(&bt->wake_wait); + if (bt->irq_held) { + bt->irq_held = false; + enable_irq(bt->irq); + } +} + +/* + * With wakeup enabled, the transport stays up through suspend-to-idle + * instead of being quiesced: + * + * - Wake tracking starts before anything else, so that a wake report or a + * Hardware Error is never consumed unnoticed (bt7932_rx_locked()). + * - bt7932_arm_wakeup() arms the MSI and blocks TX. HCI recovery, which + * would need TX, waits in bt7932_open() until resume. + * - bt7932_suspend_noirq() saves the PCI state, which keeps the PCI core + * from moving the function out of D0, once the handler can no longer + * run, and fails the suspend if the transport faulted in the meantime. + * - bt7932_resume_noirq() holds the interrupt off until the PCI core has + * restored that state and called bt7932_resume(). + * - A terminal fault, from whatever source, saves the state again with + * bus mastering clear while it is outstanding, so restoring it never + * brings bus mastering back for a failed transport. + * + * The PCIe link and host bridge stay up in suspend-to-idle anyway, as the + * Wi-Fi function on the same link is never stopped, and ASPM stays off on + * this link as the transport was admitted. + */ static int bt7932_suspend(struct device *dev) { struct bt7932 *bt = pci_get_drvdata(to_pci_dev(dev)); + bool wake; int ret; if (READ_ONCE(bt->fault)) return -EIO; if (!bt->registered) return 0; + wake = bt7932_may_wake(bt); + if (wake) + bt7932_track_wake(bt, true); if (bt7932_hci_pm(bt)) { bt7932_wait_hci_idle(bt); ret = hci_suspend_dev(bt->hdev); - if (ret) + if (ret) { + bt7932_track_wake(bt, false); + return ret; + } + } + if (wake) { + ret = bt7932_arm_wakeup(bt); + if (!ret) { + dev_dbg(dev, "system sleep with wakeup armed\n"); + return 0; + } + bt7932_track_wake(bt, false); + if (ret != -EAGAIN) { + dev_err(dev, "transport failed while arming wakeup\n"); + hci_resume_dev(bt->hdev); return ret; + } } ret = bt7932_quiesce(bt); if (!ret) { @@ -1400,11 +1606,49 @@ static int bt7932_suspend(struct device *dev) return ret; } +static int bt7932_suspend_noirq(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct bt7932 *bt = pci_get_drvdata(pdev); + int ret = 0; + + if (!bt->wake_armed) + return 0; + mutex_lock(&bt->lock); + if (bt->fault) { + ret = -EIO; + } else { + pci_save_state(pdev); + bt->pci_saved = true; + } + mutex_unlock(&bt->lock); + if (ret) + dev_err(dev, "transport failed while suspending with wakeup armed\n"); + return ret; +} + +static int bt7932_resume_noirq(struct device *dev) +{ + struct bt7932 *bt = pci_get_drvdata(to_pci_dev(dev)); + + if (bt->wake_armed) { + disable_irq(bt->irq); + bt->irq_held = true; + } + return 0; +} + static int bt7932_resume(struct device *dev) { struct bt7932 *bt = pci_get_drvdata(to_pci_dev(dev)); int ret; + if (bt->wake_armed) { + bt7932_disarm_wakeup(bt); + if (READ_ONCE(bt->fault)) + return -EIO; + return bt7932_hci_pm(bt) ? hci_resume_dev(bt->hdev) : 0; + } if (READ_ONCE(bt->fault)) return -EIO; if (!bt->registered) @@ -1428,6 +1672,8 @@ static int bt7932_freeze(struct device *dev) static const struct dev_pm_ops bt7932_pm_ops = { .suspend = bt7932_suspend, + .suspend_noirq = bt7932_suspend_noirq, + .resume_noirq = bt7932_resume_noirq, .resume = bt7932_resume, .freeze = bt7932_freeze, .thaw = bt7932_resume, @@ -1498,4 +1744,5 @@ MODULE_FIRMWARE(BT7932_ADDR); #if IS_ENABLED(CONFIG_BT_MTK7932_PCIE_KUNIT_TEST) #include "mt7932_bt_pcie_test.c" +#include "mt7932_bt_pcie_wake_test.c" #endif diff --git a/drivers/bluetooth/mt7932_bt_pcie_wake_test.c b/drivers/bluetooth/mt7932_bt_pcie_wake_test.c new file mode 100644 index 0000000000000..61e93ce1f31db --- /dev/null +++ b/drivers/bluetooth/mt7932_bt_pcie_wake_test.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for how mt7932_bt_pcie classifies received events while the + * system suspends with wakeup armed. Built as part of mt7932_bt_pcie.c, + * so the tests reach its static helpers. + */ +#include + +static void bt7932_wake_event_bredr_test(struct kunit *test) +{ + /* Connection Request: bdaddr, class of device, link type. */ + static const u8 request[] = { HCI_EV_CONN_REQUEST, 10, + 1, 2, 3, 4, 5, 6, 0x40, 0x25, 0x00, 0x01 }; + static const u8 complete[] = { HCI_EV_CONN_COMPLETE, 11, 0x00, 0x01, 0x00, + 1, 2, 3, 4, 5, 6, 0x01, 0x00 }; + static const u8 sync[] = { HCI_EV_SYNC_CONN_COMPLETE, 1, 0x00 }; + + KUNIT_EXPECT_EQ(test, bt7932_wake_event(request, sizeof(request)), BT7932_WAKE_REMOTE); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(complete, sizeof(complete)), BT7932_WAKE_REMOTE); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(sync, sizeof(sync)), BT7932_WAKE_REMOTE); +} + +static void bt7932_wake_event_le_test(struct kunit *test) +{ + static const u8 adv[] = { HCI_EV_LE_META, 2, HCI_EV_LE_ADVERTISING_REPORT, 1 }; + static const u8 direct[] = { HCI_EV_LE_META, 1, HCI_EV_LE_DIRECT_ADV_REPORT }; + static const u8 ext[] = { HCI_EV_LE_META, 1, HCI_EV_LE_EXT_ADV_REPORT }; + /* LE Connection Complete and Read Remote Features Complete. */ + static const u8 conn[] = { HCI_EV_LE_META, 1, HCI_EV_LE_CONN_COMPLETE }; + static const u8 features[] = { HCI_EV_LE_META, 1, 0x04 }; + static const u8 no_subevent[] = { HCI_EV_LE_META, 0 }; + + KUNIT_EXPECT_EQ(test, bt7932_wake_event(adv, sizeof(adv)), BT7932_WAKE_ADVERTISING); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(direct, sizeof(direct)), BT7932_WAKE_ADVERTISING); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(ext, sizeof(ext)), BT7932_WAKE_ADVERTISING); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(conn, sizeof(conn)), BT7932_WAKE_NONE); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(features, sizeof(features)), BT7932_WAKE_NONE); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(no_subevent, sizeof(no_subevent)), + BT7932_WAKE_NONE); +} + +/* The replies to the commands that set up suspend must never end it. */ +static void bt7932_wake_event_replies_test(struct kunit *test) +{ + /* Command Complete for LE Set Scan Enable, Command Status. */ + static const u8 cc[] = { HCI_EV_CMD_COMPLETE, 4, 0x01, 0x0c, 0x20, 0x00 }; + static const u8 cs[] = { HCI_EV_CMD_STATUS, 4, 0x00, 0x01, 0x0d, 0x20 }; + static const u8 disconn[] = { HCI_EV_DISCONN_COMPLETE, 4, 0x00, 0x01, 0x00, 0x15 }; + /* Vendor events, including the MediaTek vendor channel reply. */ + static const u8 vendor[] = { HCI_EV_VENDOR, 1, 0x00 }; + static const u8 mtk[] = { 0xe4, 5, 0x02, 0x01, 0x01, 0x00, 0x00 }; + + KUNIT_EXPECT_EQ(test, bt7932_wake_event(cc, sizeof(cc)), BT7932_WAKE_NONE); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(cs, sizeof(cs)), BT7932_WAKE_NONE); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(disconn, sizeof(disconn)), BT7932_WAKE_NONE); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(vendor, sizeof(vendor)), BT7932_WAKE_NONE); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(mtk, sizeof(mtk)), BT7932_WAKE_NONE); +} + +/* A Hardware Error ends the sleep so that recovery runs awake. */ +static void bt7932_wake_event_error_test(struct kunit *test) +{ + static const u8 error[] = { HCI_EV_HARDWARE_ERROR, 1, 0x00 }; + + KUNIT_EXPECT_EQ(test, bt7932_wake_event(error, sizeof(error)), BT7932_WAKE_ERROR); +} + +static struct kunit_case bt7932_wake_test_cases[] = { + KUNIT_CASE(bt7932_wake_event_bredr_test), + KUNIT_CASE(bt7932_wake_event_le_test), + KUNIT_CASE(bt7932_wake_event_replies_test), + KUNIT_CASE(bt7932_wake_event_error_test), + {} +}; + +static struct kunit_suite bt7932_wake_test_suite = { + .name = "mt7932-bt-wake", + .test_cases = bt7932_wake_test_cases, +}; + +kunit_test_suite(bt7932_wake_test_suite); From f1ac80ffc07784f9b37880e3ea7beb7d2873408f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 21:50:09 +1100 Subject: [PATCH 1508/2680] usb: core: Keep a hub awake for any device below that can wake the system The VIA Labs VL122 hub (USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP) passes no remote wakeup from below on while it is suspended, so in suspend-to-idle the USB core keeps it awake, with its status endpoint running, for a device below that may wake the system. It does so only when wakeup is enabled for the hub itself, though. Hubs have it disabled by default, while usbhid enables it for keyboards, so at its defaults a J700 never keeps the VL122 awake, and a USB keyboard behind it cannot wake the system. The hub's own setting is the wrong condition for a wakeup from below. It already keeps the VL122 awake for the changes of its own ports (USB_QUIRK_HUB_NO_SUSPENDED_PORT_WAKEUP). Any other hub passes on a remote wakeup from below whatever its own wakeup setting, which only arms it for connect, disconnect and over-current changes (USB 2.0, section 7.1.7.7, and commit 7aba8d014341 ("USB: don't enable remote wakeup by default")). Keep the VL122 awake whenever the remote wakeup of a device below can reach it, see usb_wakeup_reaches(), so that it behaves as such a hub does, and the wakeup setting of the keyboard decides, as it does behind any other hub. This costs power at the defaults: with a USB keyboard below the VL122, the hub and the host controller stay up in every suspend-to-idle. To avoid that, disable wakeup for the keyboard, or for the host controller. As the hub reports any change of its ports on its status endpoint, a connect or disconnect on one of them also wakes the system while it is kept awake. Add KUnit tests that the VL122 is kept awake at the J700 defaults, and not outside suspend-to-idle, on a controller that halts or may not wake the system, or without a device below that can wake it. Fixes: a28673c796d5 ("usb: core: Keep a hub awake in suspend-to-idle when it cannot pass on wakeup") (cherry picked from commit 859fce0b1b565a85ef48668c688e70c0faec9cc4) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/core/driver.c | 7 +++- drivers/usb/core/usb-wakeup-test.c | 57 ++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/drivers/usb/core/driver.c b/drivers/usb/core/driver.c index b16aed3b6bff8..4a9bdcf8bb11a 100644 --- a/drivers/usb/core/driver.c +++ b/drivers/usb/core/driver.c @@ -1397,6 +1397,11 @@ static int usb_resume_interface(struct usb_device *udev, * such a hub and the root hub, root hub included. This only works while the * host controller keeps running, so its interrupt reports the hub's status * change. + * + * Other hubs pass on the remote wakeup of a device below them whatever their + * own wakeup setting, which only arms them for connect, disconnect and + * over-current changes (USB 2.0, sections 7.1.7.7 and 11.4.4). So does a hub + * kept awake here: it is kept awake for any device below that can wake it. */ VISIBLE_IF_KUNIT bool usb_stay_awake_for_wakeup(struct usb_device *udev, pm_message_t msg) { @@ -1414,7 +1419,7 @@ VISIBLE_IF_KUNIT bool usb_stay_awake_for_wakeup(struct usb_device *udev, pm_mess return true; if ((udev->quirks & USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP) && - udev->do_remote_wakeup && device_wakeup_path(&udev->dev)) + usb_wakeup_reaches(udev)) return true; /* Children are done with their suspend before their parent starts */ diff --git a/drivers/usb/core/usb-wakeup-test.c b/drivers/usb/core/usb-wakeup-test.c index d39c71c9c459e..e563dd7822886 100644 --- a/drivers/usb/core/usb-wakeup-test.c +++ b/drivers/usb/core/usb-wakeup-test.c @@ -241,6 +241,61 @@ static void usb_wakeup_stay_awake_armed_vl122_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&root->udev)); } +/* + * The defaults behind the front port of a J700: the VL122 hub, which does not + * pass a remote wakeup on while it is suspended, has wakeup disabled, as hubs + * have by default, and a keyboard below it has it enabled. The VL122 is kept + * awake for the keyboard, and so is the root hub above it. + */ +static void usb_wakeup_stay_awake_vl122_test(struct kunit *test) +{ + struct usb_wakeup_test_dev *root = usb_wakeup_test_s2idle_root(test); + struct usb_wakeup_test_dev *vl122 = usb_wakeup_test_add(test, root, true); + struct usb_wakeup_test_dev *hub = usb_wakeup_test_add(test, vl122, true); + struct usb_wakeup_test_dev *keyboard = usb_wakeup_test_add(test, hub, false); + + vl122->udev.quirks = USB_WAKEUP_TEST_VL122; + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + + keyboard->udev.do_remote_wakeup = 1; + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&keyboard->udev, PMSG_SUSPEND)); + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&hub->udev, PMSG_SUSPEND)); + KUNIT_EXPECT_TRUE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + + vl122->udev.awake_in_sleep = 1; + KUNIT_EXPECT_TRUE(test, usb_stay_awake_for_wakeup(&root->udev, PMSG_SUSPEND)); +} + +/* + * Nothing is kept awake for other system sleep transitions, for a controller + * that halts or may not wake the system, or with no device below that can wake. + */ +static void usb_wakeup_stay_awake_not_test(struct kunit *test) +{ + struct usb_wakeup_test_dev *root = usb_wakeup_test_s2idle_root(test); + struct usb_wakeup_test_dev *vl122 = usb_wakeup_test_add(test, root, true); + struct usb_wakeup_test_dev *keyboard = usb_wakeup_test_add(test, vl122, false); + struct usb_hcd *hcd = bus_to_hcd(root->udev.bus); + + vl122->udev.quirks = USB_WAKEUP_TEST_VL122; + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + keyboard->udev.do_remote_wakeup = 1; + KUNIT_ASSERT_TRUE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_FREEZE)); + + pm_suspend_target_state = PM_SUSPEND_MEM; + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + pm_suspend_target_state = PM_SUSPEND_TO_IDLE; + + hcd->runs_in_s2idle = 0; + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + hcd->runs_in_s2idle = 1; + + device_set_wakeup_enable(hcd->self.controller, false); + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); +} + static struct kunit_case usb_wakeup_test_cases[] = { KUNIT_CASE(usb_wakeup_reaches_vl122_test), KUNIT_CASE(usb_wakeup_reaches_hub_test), @@ -249,6 +304,8 @@ static struct kunit_case usb_wakeup_test_cases[] = { KUNIT_CASE(usb_wakeup_reaches_nothing_test), KUNIT_CASE(usb_wakeup_reaches_armed_vl122_test), KUNIT_CASE(usb_wakeup_stay_awake_armed_vl122_test), + KUNIT_CASE(usb_wakeup_stay_awake_vl122_test), + KUNIT_CASE(usb_wakeup_stay_awake_not_test), {} }; From 304f5e0c20d71c8a93a371689347bc87ed4918bd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 22:59:46 +1100 Subject: [PATCH 1509/2680] soc: apple: sep: Say why the sensor status after registration was unusable After a sensor registration the status is read again before the patch acknowledgment is trusted. When that read is unusable the bring-up stops with "status unavailable after registration", which covers two different failures: the SPI transfer failed, or the sensor answered with an all-zero (silent) status. They point at different causes, the bus or the sensor, and the line does not say which, nor how long after power-on it happened. Name which one it was and the time since the sensor was powered on, as the other registration steps already do. The start of the message is unchanged, so existing log searches still match. No functional change. (cherry picked from commit 3f7970de6b41e28e17c38e356cdc6f6e5fd36ad2) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 45c290f5523e5..95d4adbbc96ce 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2017,8 +2017,21 @@ impl SepData { // acknowledgement sampled before CLEAR_STATE and authentication. let st = match sensor::status() { Ok(st) if !st.is_silent() => st, - _ => { - dev_warn!(self.dev, "sensor: status unavailable after registration\n"); + Ok(_) => { + dev_warn!( + self.dev, + "sensor: status unavailable after registration: silent {} ms after power-on\n", + self.sensor_ms() + ); + return None; + } + Err(e) => { + dev_warn!( + self.dev, + "sensor: status unavailable after registration: read failed ({:?}) {} ms after power-on\n", + e, + self.sensor_ms() + ); return None; } }; From a1fcb9944dcc182f9021ec486b5e2cf7b361a43f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 13:25:59 +1100 Subject: [PATCH 1510/2680] selftests: apple-dcp: Follow the retired-channel teardown and LUT log level The service-rebind group extracts afk_recv_handle_teardown(), which now asks afk_epic_channel_retired() whether a teardown names a channel that was retired before it logs. Extract that helper and the afk_service_retired() check it uses, so the group keeps compiling the production teardown path. The firmware-provider group compiles trampoline_provider_property_h17p(), which now reports the J700 power-LUT fallback with dev_info_once(). Give its host prelude a no-op dev_info_once() next to dev_warn_once(). No expectation changes; all 55 host tests pass. (cherry picked from commit 7543b0f32df84cff9478d643ab5fb5366ae7f217) Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/testing/selftests/drivers/gpu/apple-dcp/edid.py | 1 + tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py index e247ce3a5489e..fc86c7f8be7df 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py @@ -158,6 +158,7 @@ def function(file,signature): function('epic/dpavservep.c','static void dcpavserv_teardown('),function('epic/dpavservep.c','void dpavservep_detach('), function('afk.c','static const struct apple_epic_service_ops *\nafk_match_service('), function('afk.c','static struct apple_epic_service *afk_epic_find_service('), + function('afk.h','static inline bool afk_service_retired('),function('afk.c','static bool afk_epic_channel_retired('), function('afk.c','static void afk_discard_pending_open('),function('afk.c','static void afk_retry_service_init('),function('afk.c','static void afk_service_worker('),function('afk.c','static void afk_recv_handle_init('), function('epic/dpavservep.c','static int dcpdpserv_prepare('), function('epic/dpavservep.c','static void dcpdpserv_teardown('),function('afk.c','static void afk_recv_handle_teardown('), diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 0b3fdcaa81e8c..2219d39962273 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -974,6 +974,7 @@ def firmware(root): #define trace_iomfb_callback(d, t, n) ((void)(d), (void)(t)) #define dev_err(...) ((void)0) #define dev_warn_once(...) ((void)0) +#define dev_info_once(...) ((void)0) struct device_node { bool internal, external, present; int length; }; struct device { struct device_node *of_node; }; struct apple_dcp { struct { int iomfb_method_profile; } hw; struct device *dev; bool crashed; }; From 5c0952e5f0529cd2975e2b2c9fd05ce1aec763e8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:38:15 +1100 Subject: [PATCH 1511/2680] drm/asahi: clean up the statistics Rust style MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use infallible conversions for the sysfs enable flag and firmware utilization, and let field access dereference the DRM reference. Document why the registration helpers may pass the driver's device and optional snapshot to C. This clears the Clippy findings in the statistics code without changing its accounting, ownership or teardown order. Apply rustfmt to the statistics registration code in the driver and its sysfs wrapper. [ryan: adapted to 7.2.10 — without the driver.rs part: the 7.2.10 probe already takes the statistics snapshot from the GPU manager, builds AsahiPlatformDriverData one field per line and has the G17 sysfs registration on one line, so nothing is left to restyle there] (cherry picked from commit 748b2a25f2ca960017eef868e2323a27e07f0fe2) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/channel.rs | 12 +++++++----- drivers/gpu/drm/asahi/sysfs_exports.rs | 15 +++++++++++++-- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/channel.rs b/drivers/gpu/drm/asahi/channel.rs index 01fd1d9f2d0bc..e756d397591ec 100644 --- a/drivers/gpu/drm/asahi/channel.rs +++ b/drivers/gpu/drm/asahi/channel.rs @@ -666,11 +666,13 @@ impl StatsChannel::ver { // rem * util * 1e9 / (100 * ts_hz), which // cannot overflow u64 (rem < ts_hz <= 24e6 // on supported SoCs). - let util = [*util1, *util2, *util3, *util4] - .into_iter() - .max() - .unwrap_or(0) - .min(100) as u64; + let util = u64::from( + [*util1, *util2, *util3, *util4] + .into_iter() + .max() + .unwrap_or(0) + .min(100), + ); let ts = timestamp.0; if self.last_util_ts != 0 && ts > self.last_util_ts { let delta = ts - self.last_util_ts; diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs index 1b0eb7c6ea50f..079623d1ef210 100644 --- a/drivers/gpu/drm/asahi/sysfs_exports.rs +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -3,8 +3,8 @@ //! Per-device sysfs registration. The caller retains the snapshot Arc until //! unregister has drained active readers and freed the C attribute wrapper. -use core::ffi::{c_int, c_void}; use crate::stats::StatsSnapshot; +use core::ffi::{c_int, c_void}; extern "C" { fn asahi_sysfs_register( @@ -23,7 +23,16 @@ pub(crate) fn register( export_enabled: bool, ) -> kernel::error::Result<*mut c_void> { let mut handle = core::ptr::null_mut(); - let ret = unsafe { asahi_sysfs_register(dev.cast(), snapshot, export_enabled as c_int, &mut handle) }; + // SAFETY: The bound driver keeps the device and optional snapshot alive until + // unregister drains readers. `handle` is writable for the duration of the call. + let ret = unsafe { + asahi_sysfs_register( + dev.cast(), + snapshot, + c_int::from(export_enabled), + &mut handle, + ) + }; if ret < 0 { Err(kernel::error::Error::from_errno(ret)) } else { @@ -33,5 +42,7 @@ pub(crate) fn register( /// Drain readers before the caller drops its snapshot Arc. pub(crate) fn unregister(dev: *mut kernel::bindings::device, handle: *mut c_void) { + // SAFETY: `handle` was returned by register for this device. The driver still + // owns the device and optional snapshot while the C helper drains readers. unsafe { asahi_sysfs_unregister(dev.cast(), handle) }; } From 6d1b70da6d39570e437542b45a286f1e177d829a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:12:55 +1100 Subject: [PATCH 1512/2680] Bluetooth: mt7932: Tell the controller that the host sleeps When it suspends the system, macOS sends the controller one vendor command, opcode 0xfc6f with seven 0xff parameter bytes, before the PCI function changes power state, and sends nothing when the system wakes (observed in our research of the J700 macOS Bluetooth transport; no code was taken from it). 0xfc6f is the MediaTek vendor channel that btmtk, btmtksdio and btmtkuart also use; those drivers wait for a vendor event in reply, so it is not answered by Command Complete. Send the same command when the transport is left running for wakeup, after hci_suspend_dev() has set up the wake scans and before the interrupt is armed. Place it on the command ring directly, outside the HCI command flow, and wait up to 500 ms until the controller has taken it from the ring. The ring publication is factored out of the TX work for this, so that the command has a result: - Nothing was published (the controller could not be reached without a fault, or the ring was full): wakeup is not armed and the transport is quiesced as without wakeup. - Published but not taken in time: the command may still take effect, so the quiesce cannot be relied on. The suspend fails, HCI is resumed and the controller is reset through HCI recovery. - The transport faulted: the suspend fails. The response contract is not established: any reply is a vendor event, which is never treated as a wake report, but one that arrived after the interrupt is suspended would wake the system. The quiesced path is left alone. (cherry picked from commit 7bfd6f96aa12422914c204b342b569d42fa4eedf) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/mt7932_bt_pcie.c | 141 +++++++++++++++++++++++------ 1 file changed, 113 insertions(+), 28 deletions(-) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 6c0c4b3d33c09..0ac2f2471c4ce 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -62,6 +62,9 @@ #define BT7932_CONTROL_POOL_SIZE (BT7932_DEPTH * BT7932_CONTROL_SIZE) #define BT7932_COMPLETION_SIZE (BT7932_DEPTH * 0x18) #define BT7932_TX_QUEUE_MAX 128 +/* Vendor command telling the controller that the host goes to sleep. */ +#define BT7932_OP_HOST_SLEEP 0xfc6f +#define BT7932_HOST_SLEEP_PLEN 7 #define BT7932_FW_B0 "mediatek/MT7932B0_OS_TypeB_0.1.44.0_241001003711.bin" #define BT7932_FW_B1 "mediatek/MT7932B1_OS_TypeB_2.0.177.0_260706180356.bin" @@ -616,51 +619,64 @@ static int bt7932_tx_progress(struct bt7932 *bt, unsigned int channel) return 0; } +/* + * Caller owns lock with the firmware awake. Returns -ENOSPC while the ring is + * full, or -EPROTO after faulting the transport. + */ +static int bt7932_publish_locked(struct bt7932 *bt, unsigned int channel, + const u8 *data, size_t len) +{ + struct bt7932_ring *ring = &bt->rings[channel]; + u8 *descriptor; + + if (bt7932_tx_progress(bt, channel)) { + bt7932_fault_locked(bt, "TX consumer exceeded outstanding descriptors"); + return -EPROTO; + } + if (((ring->host + 1) & BT7932_MASK) == ring->previous) + return -ENOSPC; + descriptor = bt->arena + ring->offset + ring->host * geometry[channel].stride; + memset(descriptor, 0, geometry[channel].stride); + descriptor[0] = 2; + descriptor[1] = len; + descriptor[2] = len >> 8; + descriptor[3] = len >> 16; + put_unaligned_le16(ring->host, descriptor + 12); + descriptor[15] = 2; + memcpy(descriptor + 16, data, len); + ring->host = (ring->host + 1) & BT7932_MASK; + bt7932_publish_counter(bt, bt->host_transfer, channel, ring->host); + bt7932_ring_doorbell(bt, geometry[channel].doorbell); + bt->hdev->stat.byte_tx += len; + return 0; +} + static void bt7932_tx_work(struct work_struct *work) { struct bt7932 *bt = container_of(work, struct bt7932, tx_work); struct sk_buff *skb; + int ret; while ((skb = skb_dequeue(&bt->tx_queue))) { int channel = bt7932_tx_channel(hci_skb_pkt_type(skb)); - struct bt7932_ring *ring; - u16 consumer; - u8 *descriptor; if (bt7932_wake_and_lock(bt, true)) { kfree_skb(skb); skb_queue_purge(&bt->tx_queue); return; } - ring = &bt->rings[channel]; - if (bt7932_tx_progress(bt, channel)) { - bt7932_fault_locked(bt, "TX consumer exceeded outstanding descriptors"); - mutex_unlock(&bt->lock); - kfree_skb(skb); - skb_queue_purge(&bt->tx_queue); - return; - } - consumer = ring->previous; - if (((ring->host + 1) & BT7932_MASK) == consumer) { + ret = bt7932_publish_locked(bt, channel, skb->data, skb->len); + if (ret == -ENOSPC) { skb_queue_head(&bt->tx_queue, skb); mutex_unlock(&bt->lock); return; } - descriptor = bt->arena + ring->offset + ring->host * geometry[channel].stride; - memset(descriptor, 0, geometry[channel].stride); - descriptor[0] = 2; - descriptor[1] = skb->len; - descriptor[2] = skb->len >> 8; - descriptor[3] = skb->len >> 16; - put_unaligned_le16(ring->host, descriptor + 12); - descriptor[15] = 2; - memcpy(descriptor + 16, skb->data, skb->len); - ring->host = (ring->host + 1) & BT7932_MASK; - bt7932_publish_counter(bt, bt->host_transfer, channel, ring->host); - bt7932_ring_doorbell(bt, geometry[channel].doorbell); - bt->hdev->stat.byte_tx += skb->len; mutex_unlock(&bt->lock); kfree_skb(skb); + if (ret) { + skb_queue_purge(&bt->tx_queue); + return; + } } } @@ -1482,11 +1498,62 @@ static void bt7932_track_wake(struct bt7932 *bt, bool on) mutex_unlock(&bt->lock); } +/* + * Before system sleep, the host tells the controller that it is going to + * sleep with a vendor command whose parameters are all ones; nothing is sent + * when the host is back. It is answered by a vendor event, if at all, so + * place it on the command ring directly rather than through the HCI command + * flow, and wait until the controller has taken it. + * + * Returns -EAGAIN if nothing was published (the controller could not be + * reached without a fault, or the ring was full), -EBUSY if the command was + * published but not taken in time, so that it may still take effect, or + * -EIO if the transport faulted. + */ +static int bt7932_host_sleep(struct bt7932 *bt) +{ + u8 cmd[HCI_COMMAND_HDR_SIZE + BT7932_HOST_SLEEP_PLEN]; + unsigned long deadline; + u16 target; + int ret; + + put_unaligned_le16(BT7932_OP_HOST_SLEEP, cmd); + cmd[2] = BT7932_HOST_SLEEP_PLEN; + memset(cmd + HCI_COMMAND_HDR_SIZE, 0xff, BT7932_HOST_SLEEP_PLEN); + + ret = bt7932_wake_and_lock(bt, true); + if (ret) + return READ_ONCE(bt->fault) ? -EIO : -EAGAIN; + ret = bt7932_publish_locked(bt, 1, cmd, sizeof(cmd)); + target = bt->rings[1].host; + mutex_unlock(&bt->lock); + if (ret == -ENOSPC) + return -EAGAIN; + if (ret) + return -EIO; + + deadline = jiffies + msecs_to_jiffies(500); + for (;;) { + mutex_lock(&bt->lock); + ret = bt->fault ? -EIO : 0; + if (!ret && bt7932_counter(bt, bt->fw_transfer, 1) == target) { + mutex_unlock(&bt->lock); + return 0; + } + mutex_unlock(&bt->lock); + if (ret) + return ret; + if (time_after(jiffies, deadline)) + return -EBUSY; + usleep_range(1000, 2000); + } +} + /* * Leave the firmware running with its MSI armed. It keeps the scans that * hci_suspend_dev() set up and reports a match like any other event. Returns - * 0 when armed, -EAGAIN when the transport should be quiesced instead, or - * -EIO when it failed. + * 0 when armed, -EAGAIN when the transport should be quiesced instead, + * -EBUSY when the controller state is uncertain, or -EIO when it failed. */ static int bt7932_arm_wakeup(struct bt7932 *bt) { @@ -1499,6 +1566,13 @@ static int bt7932_arm_wakeup(struct bt7932 *bt) return -EAGAIN; } + ret = bt7932_host_sleep(bt); + if (ret) { + disable_irq_wake(bt->irq); + dev_warn(dev, "host sleep command failed: %d\n", ret); + return ret; + } + mutex_lock(&bt->lock); ret = bt->fault ? -EIO : 0; if (!ret) { @@ -1586,6 +1660,17 @@ static int bt7932_suspend(struct device *dev) return 0; } bt7932_track_wake(bt, false); + if (ret == -EBUSY) { + /* + * The host sleep command may still take effect, so the + * quiesce cannot be relied on. Fail this suspend and + * reset the controller through HCI recovery. + */ + dev_err(dev, "host sleep command not taken; resetting the controller\n"); + hci_resume_dev(bt->hdev); + bt7932_reset(bt->hdev); + return ret; + } if (ret != -EAGAIN) { dev_err(dev, "transport failed while arming wakeup\n"); hci_resume_dev(bt->hdev); From d1319d0d0808ad89f9513eb9b9f564c3e2e07baf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 10:33:13 +1100 Subject: [PATCH 1513/2680] irqchip/apple-aic: Allow interrupts to wake the system Neither the AIC nor the AIC2/AIC3 irq_chip implements .irq_set_wake or sets IRQCHIP_SKIP_SET_WAKE, so irq_set_irq_wake() fails with -ENXIO for every hardware interrupt on Apple silicon. Drivers that arm their interrupt for wakeup, directly or through dev_pm_set_wake_irq(), are silently left without a wake interrupt: dev_pm_arm_wake_irq() ignores the error, suspend_device_irqs() then disables the line like any other, and an event on it cannot end suspend-to-idle. Only interrupts requested with IRQF_NO_SUSPEND, such as the Apple mailbox interrupts that carry the SMC lid and power button events, are still delivered, which is why those are the only things that wake these machines today. Suspend-to-idle is the only system sleep state on these SoCs. The AIC stays powered in it and delivers unmasked interrupts to the CPUs idling in WFI as usual, so there is no wake logic to program. Set IRQCHIP_SKIP_SET_WAKE on both chips, like the GICv3 driver does, so that enable_irq_wake() succeeds and the IRQ core keeps wake-armed interrupts enabled and turns their next event into a system wakeup. IRQCHIP_MASK_ON_SUSPEND is deliberately not set. The IRQ core already disables the non-wake interrupts lazily, and masks and records any that fire during suspend so they are replayed on resume. Masking them in hardware up front gains nothing on a controller that has no separate wake path, and would risk losing events from the edge-like MSIs that the PCIe controller delivers through the AIC. The FIQ chip is left alone: it only carries per-CPU timer, PMU and fast IPI interrupts, none of which are system wake sources. (cherry picked from commit 43c7fc8f82a65f2e3807f41f03421df2b27ed3de) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/irqchip/irq-apple-aic.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index eb3a648f2f533..3214501a10635 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -484,6 +484,13 @@ static int aic_irq_set_type(struct irq_data *d, unsigned int type) return (type == IRQ_TYPE_LEVEL_HIGH || type == IRQ_TYPE_EDGE_RISING) ? 0 : -EINVAL; } +/* + * The only system sleep state on these SoCs is suspend-to-idle, in which the + * AIC stays powered and keeps delivering every unmasked interrupt to the CPUs + * idling in WFI. There is no separate wake logic to configure, so any IRQ may + * be a wakeup source: let enable_irq_wake() succeed and leave it to the IRQ + * core to keep wake-armed interrupts enabled across suspend_device_irqs(). + */ static struct irq_chip aic_chip = { .name = "AIC", .irq_mask = aic_irq_mask, @@ -491,6 +498,7 @@ static struct irq_chip aic_chip = { .irq_eoi = aic_irq_eoi, .irq_set_affinity = aic_irq_set_affinity, .irq_set_type = aic_irq_set_type, + .flags = IRQCHIP_SKIP_SET_WAKE, }; static struct irq_chip aic2_chip = { @@ -499,6 +507,7 @@ static struct irq_chip aic2_chip = { .irq_unmask = aic_irq_unmask, .irq_eoi = aic_irq_eoi, .irq_set_type = aic_irq_set_type, + .flags = IRQCHIP_SKIP_SET_WAKE, }; /* From 5bcb026d5bf7b61198388968a4b7cce426b7e647 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:31:14 +1100 Subject: [PATCH 1514/2680] ASoC: apple: t8140-aop-audio: use infallible conversions in tests The volume table contains u32 gains. Widen them with From when checking the speaker ceiling and volume steps so the KUnit build passes Clippy's cast_lossless lint. Keep the test values and assertions unchanged. (cherry picked from commit caa00345cc1c397e2e09c47569b2335eb9543618) Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index d181bf206e06c..5fb78c22ac381 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -2924,7 +2924,7 @@ mod tests { #[test] fn the_top_of_the_volume_range_is_the_amplifier_full_scale() { assert_eq!( - SPKR_VOLUME_Q31[SPKR_VOLUME_MAX as usize] as i64, + i64::from(SPKR_VOLUME_Q31[SPKR_VOLUME_MAX as usize]), SPKR_CEILING_Q31 ); assert_eq!(SPKR_CEILING_Q31, 1 << 30); @@ -2935,7 +2935,10 @@ mod tests { fn the_volume_table_falls_in_half_db_steps_to_mute() { assert_eq!(SPKR_VOLUME_Q31[0], 0); for i in 2..=SPKR_VOLUME_MAX as usize { - let (lo, hi) = (SPKR_VOLUME_Q31[i - 1] as u128, SPKR_VOLUME_Q31[i] as u128); + let (lo, hi) = ( + u128::from(SPKR_VOLUME_Q31[i - 1]), + u128::from(SPKR_VOLUME_Q31[i]), + ); assert!(hi > lo); // hi / lo == 10^(0.5/20) to within the table's rounding let expect = lo * HALF_DB_PPB / 1_000_000_000; @@ -2945,7 +2948,7 @@ mod tests { #[test] fn the_locked_level_is_20_db_below_full_scale() { - let locked = SPKR_VOLUME_Q31[LOCKED_VOLUME] as i64; + let locked = i64::from(SPKR_VOLUME_Q31[LOCKED_VOLUME]); // 10 * locked == ceiling to within the table's rounding (0.1 ppm) assert!((10 * locked - SPKR_CEILING_Q31).abs() <= 10); } From b7ae994991e891a96e1a4ab0e0b0c1c864ec67a1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 14:36:08 +1100 Subject: [PATCH 1515/2680] usb: xhci: plat: Hold a runtime PM reference in the resume-from-runtime tests xhci_plat_resume_from_runtime_ok_test resumes a mock device with no runtime PM reference held. The PM core then queues an idle check, which tries a runtime suspend. The mock has no suspend callback, so the attempt fails and the device returns to active, but for a moment its status is RPM_SUSPENDING. A test that reads pm_runtime_active() then sees false. Under lockdep with two CPUs this failed 1 run in 10. The real caller cannot see this: system suspend calls the suspend callbacks with the reference that device_prepare() takes. Hold one in the tests too, and drop it when they release the device. Fixes: 7dfc99ae6302 ("usb: xhci: plat: Fail system suspend when the controller does not resume") (cherry picked from commit 2911b586c885463279d04d560137c147c05ca099) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/host/xhci-plat-test.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/drivers/usb/host/xhci-plat-test.c b/drivers/usb/host/xhci-plat-test.c index 49cbd15e4ed34..ca4814898714e 100644 --- a/drivers/usb/host/xhci-plat-test.c +++ b/drivers/usb/host/xhci-plat-test.c @@ -152,10 +152,24 @@ static struct device *xhci_plat_test_rpm_device(struct kunit *test, int rpm_ret) xhci_plat_test_rpm_ret = rpm_ret; dev->pm_domain = &xhci_plat_test_pm_domain; pm_runtime_set_suspended(dev); + /* + * System suspend runs the suspend callbacks with a runtime PM reference + * held (device_prepare()). Hold one here too, or the idle check queued + * after a resume starts a runtime suspend while the test looks at the + * state. + */ + pm_runtime_get_noresume(dev); pm_runtime_enable(dev); return dev; } +static void xhci_plat_test_rpm_release(struct device *dev) +{ + pm_runtime_disable(dev); + pm_runtime_put_noidle(dev); + dev->pm_domain = NULL; +} + /* A runtime resume that fails before system sleep fails the system suspend. */ static void xhci_plat_resume_from_runtime_error_test(struct kunit *test) { @@ -163,8 +177,7 @@ static void xhci_plat_resume_from_runtime_error_test(struct kunit *test) KUNIT_EXPECT_EQ(test, xhci_plat_resume_from_runtime(dev), -ETIMEDOUT); KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); - pm_runtime_disable(dev); - dev->pm_domain = NULL; + xhci_plat_test_rpm_release(dev); } /* A successful one, and an active controller, let it go ahead. */ @@ -175,8 +188,7 @@ static void xhci_plat_resume_from_runtime_ok_test(struct kunit *test) KUNIT_EXPECT_EQ(test, xhci_plat_resume_from_runtime(dev), 0); KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); KUNIT_EXPECT_EQ(test, xhci_plat_resume_from_runtime(dev), 0); - pm_runtime_disable(dev); - dev->pm_domain = NULL; + xhci_plat_test_rpm_release(dev); } #endif /* CONFIG_PM */ From e369318e7291fe81c86ff5bff5a4880261addfda Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 23:01:33 +1100 Subject: [PATCH 1516/2680] soc: apple: sep: Retry a failed sensor bring-up before a verify Every verify powers the sensor up and registers it with the enclave again before it captures. The first verify after a system suspend on a MacBook Neo failed that way: the re-registration completed, the status read straight after it was unusable, and the verify ended with "the sensor did not come back online". libfprint reports that as an error, so fprintd ends the verify and the lock screen drops its fingerprint prompt although the sensor works. The first registration after boot fails in the same ways now and then, and commit 6c394131cf7e ("soc: apple: sep: Retry a failed attach-time sensor bring-up") recovers it by starting again from a power-cycled sensor, which clears the enclave's registration before the new one. The verify path never got that retry, and the first verify after a suspend is where a failed re-registration has been seen: in the kernel logs on record, re-registrations that did not follow a suspend have not failed, and one of the four that first followed one failed like this. Before commit 4db9a3896199 ("soc: apple: recheck sensor status after SEP registration") the same verify fell to state 9 instead. Move attach's bounded retry into a helper and use it for a verify too, on the same nonce-session profiles. A verify asks before each new attempt whether it is still wanted, so a cancel, a close or a suspend does not wait for further attempts. Other profiles keep their single attempt. Attempts cost tens of milliseconds each and are logged. Authentication is unchanged: each attempt still needs the enclave to accept the session commit, and the match itself is the same. tools/aurora-sep/tests/test_verify_bringup.py runs the verify and attach code with mocks; it fails without this change. (cherry picked from commit af1c264b6816b07cce2943a99bc6f91d0f10b37f) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 71 +++++-- tools/aurora-sep/tests/test_verify_bringup.py | 196 ++++++++++++++++++ 2 files changed, 246 insertions(+), 21 deletions(-) create mode 100644 tools/aurora-sep/tests/test_verify_bringup.py diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 95d4adbbc96ce..92d7fb73f65b8 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1078,30 +1078,40 @@ impl SepData { Ok(()) } - fn attach_bringup(&self) { - if self.profile.nonce_session { - // A bring-up that fails on the bus leaves nothing a new attempt - // cannot undo: each attempt power-cycles the sensor and, after a - // failed one, clears the enclave's partial registration first, - // as every verify does. - const ATTEMPTS: u32 = 3; - for attempt in 1..=ATTEMPTS { - if self.bring_sensor_online() { - return; - } - if attempt < ATTEMPTS { - dev_warn!( - self.dev, - "sensor: attach-time bring-up attempt {} of {} failed; starting again\n", - attempt, - ATTEMPTS - ); - } + /// Brings the sensor up, starting again after a failed attempt for as + /// long as `wanted` says the result is still needed. + /// + /// A bring-up that fails on the bus leaves nothing a new attempt cannot + /// undo: each attempt power-cycles the sensor and, after a failed one, + /// clears the enclave's partial registration first, as every verify does. + fn bring_sensor_online_retrying(&self, purpose: &CStr, wanted: impl Fn() -> bool) -> bool { + const ATTEMPTS: u32 = 3; + for attempt in 1..=ATTEMPTS { + if self.bring_sensor_online() { + return true; + } + if attempt == ATTEMPTS || !wanted() { + break; } dev_warn!( self.dev, - "sensor: authenticated attach incomplete; capture unavailable\n" + "sensor: {} bring-up attempt {} of {} failed; starting again\n", + purpose, + attempt, + ATTEMPTS ); + } + false + } + + fn attach_bringup(&self) { + if self.profile.nonce_session { + if !self.bring_sensor_online_retrying(c"attach-time", || true) { + dev_warn!( + self.dev, + "sensor: authenticated attach incomplete; capture unavailable\n" + ); + } return; } if !sensor::is_bound() { @@ -1375,6 +1385,25 @@ impl SepData { self.refresh_match_credential(); } + /// Whether the verify this work item runs is still wanted: not cancelled, + /// not ended by a suspend and not closed. + fn verify_wanted(&self) -> bool { + !self.shutting_down.load(Relaxed) + && bio::verify_is_live(&self.bio_session.lock(), self.capture_epoch.load(Relaxed)) + } + + // Every verify registers the sensor again. On a profile with a nonce + // session that registration can fail as the attach-time one does, most + // often the first one after a system suspend, and a new attempt recovers + // it in the same way. Without one, the failure ends the user's + // authentication even though the sensor works. + fn bring_sensor_online_for_verify(&self) -> bool { + if !self.profile.nonce_session { + return self.bring_sensor_online(); + } + self.bring_sensor_online_retrying(c"verify", || self.verify_wanted()) + } + pub(crate) fn run_verify(&self) { self.retry_restore_proof(); if !self.templates_restored.load(Relaxed) { @@ -1401,7 +1430,7 @@ impl SepData { self.settle_before_capture(); // The enclave refuses a capture from an uncalibrated sensor (0x65 answers 1). - if !self.bring_sensor_online() { + if !self.bring_sensor_online_for_verify() { dev_err!(self.dev, "verify: the sensor did not come back online\n"); self.finish_verify(bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), token_bytes); let _ = sensor::idle(); diff --git a/tools/aurora-sep/tests/test_verify_bringup.py b/tools/aurora-sep/tests/test_verify_bringup.py new file mode 100644 index 0000000000000..a4e3f4572d0f1 --- /dev/null +++ b/tools/aurora-sep/tests/test_verify_bringup.py @@ -0,0 +1,196 @@ +# SPDX-License-Identifier: MIT +"""Execute the actual verify and attach sensor bring-up code with mocks. + +The first verify after a system suspend re-registers the sensor, and on a +nonce-session profile that registration can fail in a way a new attempt from +a power-cycled sensor recovers. Attach makes up to three attempts; a verify +must too, but stop as soon as the verify is no longer wanted. +""" +from pathlib import Path +import re +import subprocess +import tempfile +import unittest + +APPLE = Path(__file__).resolve().parents[3] / 'drivers/soc/apple' + + +def method(source, name): + """The text of method `name` in `source`, or '' if there is none.""" + match = re.search(r'\n (pub\(crate\) )?fn ' + name + r'\b', source) + if not match: + return '' + opening = source.index('{', match.end()) + depth, end = 1, opening + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[match.start():end] + + +PRELUDE = r''' +#![allow(dead_code, unused_variables, unused_imports)] +use std::cell::RefCell; +use std::ffi::CStr; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering::Relaxed}; +macro_rules! dev_err { ($($args:tt)*) => {} } +macro_rules! dev_warn { ($($args:tt)*) => {} } +macro_rules! dev_info { ($($args:tt)*) => {} } + +const ENROL_STATUS_SENSOR: u32 = 1; +const ENROL_STATUS_ENCLAVE: u32 = 2; +const SBIO_PROBE_USER_ID: u32 = 1; + +#[derive(Default)] +struct Mock { + /// Outcomes of successive bring-up attempts; missing ones fail. + script: Vec, + attempts: usize, + /// The verify stops being wanted once this many attempts have run. + cancel_after: Option, + captured: bool, + finished: Option, +} +thread_local! { static MOCK: RefCell = RefCell::new(Mock::default()); } + +mod bio { + pub const TOKEN_LEN: usize = 4; + #[derive(Clone, Copy, Debug, PartialEq)] + pub enum VerifyOutcome { Matched(u8), NoMatch, Failed(u32) } + pub struct Session; + pub fn verify_is_live(_: &Session, _: u64) -> bool { + super::MOCK.with(|m| { + let m = m.borrow(); + m.cancel_after.map_or(true, |n| m.attempts < n) + }) + } +} +mod sbio { + pub struct UserId; + impl UserId { pub fn new(_: u32) -> Option { Some(UserId) } } +} +mod sensor { + pub fn idle() -> Result<(), ()> { Ok(()) } + pub fn power(_: bool) -> bool { true } + pub fn is_bound() -> bool { true } +} +mod diag { + pub fn sensor_bind() -> u32 { 0 } + pub fn sensor_brought_up(_: u32, _: bool) {} +} +struct PatchLoaded(()); +struct Profile { nonce_session: bool } +struct SessionLock(bio::Session); +impl SessionLock { fn lock(&self) -> &bio::Session { &self.0 } } + +struct Driver { + dev: (), + profile: Profile, + templates_restored: AtomicBool, + shutting_down: AtomicBool, + capture_epoch: AtomicU64, + bio_session: SessionLock, +} + +impl Driver { + fn new(nonce_session: bool) -> Self { + Driver { + dev: (), + profile: Profile { nonce_session }, + templates_restored: AtomicBool::new(true), + shutting_down: AtomicBool::new(false), + capture_epoch: AtomicU64::new(1), + bio_session: SessionLock(bio::Session), + } + } + fn bring_sensor_online(&self) -> bool { + MOCK.with(|m| { + let mut m = m.borrow_mut(); + let ok = m.script.get(m.attempts).copied().unwrap_or(false); + m.attempts += 1; + ok + }) + } + fn retry_restore_proof(&self) {} + fn mint_token_bytes(&self) -> Option<[u8; bio::TOKEN_LEN]> { Some([1; bio::TOKEN_LEN]) } + fn settle_before_capture(&self) {} + fn verify_one_image(&self) -> bio::VerifyOutcome { + MOCK.with(|m| m.borrow_mut().captured = true); + bio::VerifyOutcome::Matched(1) + } + fn note_capture_end(&self) {} + fn save_after_match(&self, _: sbio::UserId) -> bool { true } + fn finish_verify(&self, outcome: bio::VerifyOutcome, _: [u8; bio::TOKEN_LEN]) { + MOCK.with(|m| { + let mut m = m.borrow_mut(); + assert!(m.finished.is_none(), "the verify reported twice"); + m.finished = Some(outcome); + }); + } + fn wake_sensor(&self) -> Option { Some(PatchLoaded(())) } + fn complete_bringup(&self, _: PatchLoaded) -> bool { true } +''' + +MAIN = r''' +} + +fn run(script: &[bool], cancel_after: Option, nonce: bool, verify: bool) -> Mock { + MOCK.with(|m| *m.borrow_mut() = Mock { script: script.to_vec(), cancel_after, ..Mock::default() }); + let driver = Driver::new(nonce); + if verify { driver.run_verify() } else { driver.attach_bringup() } + MOCK.with(|m| std::mem::take(&mut *m.borrow_mut())) +} + +fn main() { + use bio::VerifyOutcome::*; + + // The case seen after a suspend: the first attempt fails, the next works. + let m = run(&[false, true], None, true, true); + assert!(m.captured && m.finished == Some(Matched(1)) && m.attempts == 2, + "a verify gave up after one failed sensor bring-up (attempts {}, finished {:?})", + m.attempts, m.finished); + + // A sensor that never comes up: three attempts, then a sensor failure. + let m = run(&[], None, true, true); + assert_eq!((m.attempts, m.captured, m.finished), (3, false, Some(Failed(ENROL_STATUS_SENSOR)))); + + // Cancelled or suspending during the first attempt: no second one. + let m = run(&[false, true], Some(1), true, true); + assert_eq!((m.attempts, m.captured, m.finished), (1, false, Some(Failed(ENROL_STATUS_SENSOR)))); + + // A first attempt that works is the only one. + let m = run(&[true], None, true, true); + assert_eq!((m.attempts, m.captured, m.finished), (1, true, Some(Matched(1)))); + + // Profiles without a nonce session keep their single attempt. + let m = run(&[false, true], None, false, true); + assert_eq!((m.attempts, m.captured, m.finished), (1, false, Some(Failed(ENROL_STATUS_SENSOR)))); + + // Attach keeps its three attempts. + assert_eq!(run(&[false, false, true], None, true, false).attempts, 3); + assert_eq!(run(&[], None, true, false).attempts, 3); + assert_eq!(run(&[true], None, true, false).attempts, 1); + assert_eq!(run(&[], None, false, false).attempts, 0); +} +''' + + +class VerifyBringupTest(unittest.TestCase): + def test_verify_retries_a_failed_bring_up_like_attach(self): + source = (APPLE / 'sbio.rs').read_text() + methods = [method(source, name) for name in ( + 'run_verify', 'attach_bringup', + 'bring_sensor_online_retrying', 'bring_sensor_online_for_verify', + 'verify_wanted')] + self.assertTrue(methods[0] and methods[1]) + program = PRELUDE + '\n'.join(m for m in methods if m) + MAIN + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'test.rs').write_text(program) + subprocess.run(['rustc', '--edition=2021', str(root / 'test.rs'), + '-o', str(root / 'test')], check=True) + subprocess.run([str(root / 'test')], check=True) + + +if __name__ == '__main__': + unittest.main() From 95cf1ce2be3bd6cd4983fd79384d0bd6f7ebd1a2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 14:55:54 +1100 Subject: [PATCH 1517/2680] drm/apple: dcp: Initialise the start completion before checking RTKit dcp_start() returns for a missing or failed RTKit instance before it initialises start_done, which dcp_wait_ready() then waits on. No caller reaches dcp_start() after a failed component bind today, but nothing keeps it that way. Initialise the completion first, so a wait after a failed start times out like any other. Fixes: c0e5730f8058 ("drm/apple: dcp: Tidy RTKit start-up logging") (cherry picked from commit b1f7d09dcf33e3bb06e81d846783b698d604be5b) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a9cd020ca734a..7051471cc9fb9 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3305,12 +3305,12 @@ int dcp_start(struct platform_device *pdev) struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; - /* A failed component bind leaves no RTKit instance to start. */ + /* Even if nothing starts, dcp_wait_ready() must find it initialised. */ + init_completion(&dcp->start_done); + if (IS_ERR_OR_NULL(dcp->rtk)) return -ENODEV; - init_completion(&dcp->start_done); - /* Make the H17P allocator available before external service discovery. */ if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { ret = apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); From bc3e0f43c30a24e7053d88124662751b5fbdedf7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:22:46 +1100 Subject: [PATCH 1518/2680] rust: refresh examples for the Asahi kernel APIs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kernel doctests still use standard-library vectors and strings, the old resource-based IoMem constructor, and a fallible memset call. Adapt the examples to KVec, IoRequest, pinned devres initialization and the current whole-map memset API so they compile and run in the kernel test harness. Mark the declaration example for the crate-private flags macro as an internal illustration. External doctest crates cannot import that macro; keep its visibility and the runtime APIs unchanged. [ryan: adapted to 7.2.10 — only the KVec and Drain examples: 7.2.10 has no rust/kernel/iosys_map.rs, its io.rs no longer carries the old IoMem example, and types.rs no longer defines declare_flags_type!()] (cherry picked from commit d4f2ff81de0c83043c151a71a58840db7a916753) Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/kernel/alloc/kvec.rs | 24 ++++++++++++++---------- rust/kernel/alloc/kvec/drain.rs | 11 +++++++---- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index 93ab7a5f0048e..969acadca2cc3 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -798,14 +798,14 @@ where /// # Examples /// /// ``` - /// let mut v = vec![1, 2, 3]; - /// let u: Vec<_> = v.drain(1..).collect(); + /// let mut v = kernel::kvec![1, 2, 3]?; + /// assert!(v.drain(1..).eq([2, 3])); /// assert_eq!(v, &[1]); - /// assert_eq!(u, &[2, 3]); /// /// // A full range clears the vector, like `clear()` does /// v.drain(..); /// assert_eq!(v, &[]); + /// # Ok::<(), Error>(()) /// ``` pub fn drain(&mut self, range: R) -> Drain<'_, T, A> where @@ -1757,25 +1757,29 @@ impl TryFrom> for [T; N] { /// # Examples /// /// ``` - /// assert_eq!(vec![1, 2, 3].try_into(), Ok([1, 2, 3])); - /// assert_eq!(>::new().try_into(), Ok([])); + /// assert_eq!(kernel::kvec![1, 2, 3]?.try_into(), Ok([1, 2, 3])); + /// assert_eq!(KVec::::new().try_into(), Ok([])); + /// # Ok::<(), Error>(()) /// ``` /// /// If the length doesn't match, the input comes back in `Err`: /// ``` - /// let r: Result<[i32; 4], _> = (0..10).collect::>().try_into(); - /// assert_eq!(r, Err(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9])); + /// let r: Result<[i32; 4], _> = kernel::kvec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]?.try_into(); + /// assert_eq!(r, Err(kernel::kvec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]?)); + /// # Ok::<(), Error>(()) /// ``` /// /// If you're fine with just getting a prefix of the `Vec`, /// you can call [`.truncate(N)`](Vec::truncate) first. /// ``` - /// let mut v = String::from("hello world").into_bytes(); - /// v.sort(); + /// let mut v = KVec::new(); + /// v.extend_from_slice(b"hello world", GFP_KERNEL)?; + /// v.sort_unstable(); /// v.truncate(2); - /// let [a, b]: [_; 2] = v.try_into().unwrap(); + /// let [a, b]: [_; 2] = v.try_into().map_err(|_| EINVAL)?; /// assert_eq!(a, b' '); /// assert_eq!(b, b'd'); + /// # Ok::<(), Error>(()) /// ``` fn try_from(mut vec: Vec) -> Result<[T; N], Vec> { if vec.len() != N { diff --git a/rust/kernel/alloc/kvec/drain.rs b/rust/kernel/alloc/kvec/drain.rs index 035878fd11284..381c07882c935 100644 --- a/rust/kernel/alloc/kvec/drain.rs +++ b/rust/kernel/alloc/kvec/drain.rs @@ -22,8 +22,10 @@ use super::{Allocator, Vec}; /// # Example /// /// ``` -/// let mut v = vec![0, 1, 2]; -/// let iter: std::vec::Drain<'_, _> = v.drain(..); +/// let mut v = kernel::kvec![0, 1, 2]?; +/// let iter = v.drain(..); +/// assert_eq!(iter.as_slice(), &[0, 1, 2]); +/// # Ok::<(), Error>(()) /// ``` // #[stable(feature = "drain", since = "1.6.0")] pub struct Drain< @@ -54,11 +56,12 @@ impl<'a, T, A: Allocator> Drain<'a, T, A> { /// # Examples /// /// ``` - /// let mut vec = vec!['a', 'b', 'c']; + /// let mut vec = kernel::kvec!['a', 'b', 'c']?; /// let mut drain = vec.drain(..); /// assert_eq!(drain.as_slice(), &['a', 'b', 'c']); - /// let _ = drain.next().unwrap(); + /// assert_eq!(drain.next(), Some('a')); /// assert_eq!(drain.as_slice(), &['b', 'c']); + /// # Ok::<(), Error>(()) /// ``` #[must_use] // #[stable(feature = "vec_drain_as_slice", since = "1.46.0")] From c89bc220f6d6d57afbbd43c540a9d139d24a6830 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 21:44:22 +1100 Subject: [PATCH 1519/2680] Bluetooth: mt7932: Enable wakeup by default A paired Bluetooth keyboard or mouse can wake the system only while wakeup is enabled on the PCI function of the controller, see bt7932_wakeup() and bt7932_may_wake(). The PCI core leaves it disabled, so at its defaults a J700 is not woken by a Bluetooth keyboard: the tests that qualified Bluetooth wakeup enabled it from userspace. Enable it once the controller is registered, as btmtksdio does for MediaTek SDIO controllers and hci_bcm for Broadcom UART ones. Which paired device may wake the system stays the user's choice, through the WakeAllowed property that BlueZ keeps per device, and power/wakeup still turns wakeup off for the whole controller. This costs power in suspend-to-idle while Bluetooth is up: the controller keeps running with its interrupt armed, scanning for the devices allowed to wake the system. (cherry picked from commit f22b055cef58cc509dacbba2d95e265ce69097c2) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/mt7932_bt_pcie.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 0ac2f2471c4ce..72910ef976f8e 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -1122,7 +1122,7 @@ static void bt7932_hw_error(struct hci_dev *hdev, u8 code) } /* - * Follows the wakeup setting of the PCI function, which is off by default. + * Follows the wakeup setting of the PCI function, see bt7932_probe(). * Only while it is on does hci_suspend_sync() leave page scan and LE passive * scan set up for the devices flagged for remote wakeup, and only then does * bt7932_suspend() keep the transport running with its interrupt armed. @@ -1373,6 +1373,12 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret < 0) goto retained_fault; bt->registered = true; + /* + * Let paired devices wake the system by default, as btmtksdio does + * for MediaTek SDIO controllers: BlueZ still decides per device + * (WakeAllowed), and power/wakeup turns it off for the controller. + */ + device_set_wakeup_enable(&pdev->dev, true); dev_dbg(&pdev->dev, "BT_HCI_REGISTERED; setup/discovery/connection remain separate gates\n"); return 0; retained_fault: From b3dfaa56ac3424bfdbf0ae5cd5e339eda0408cc3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 14:55:54 +1100 Subject: [PATCH 1520/2680] spi: apple-qspi: Resume the controller even if its reset fails apple_qspi_resume() returns before spi_controller_resume() when the controller reset times out. The controller then stays marked suspended and every later spi_sync() on it fails with -ESHUTDOWN until reboot, while spi-mem operations, which do not look at that flag, go on using it anyway. Log the failure and resume the controller regardless. A transfer that then times out resets the controller again, as it does outside of system sleep. Fixes: ed0925cdf091 ("spi: apple-qspi: Restore the controller configuration after system sleep") (cherry picked from commit 02694d65655baae0536d47beef4f0356145069f4) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/spi-apple-qspi.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c index 2142b14e167bf..a7edd0ff4e8d5 100644 --- a/drivers/spi/spi-apple-qspi.c +++ b/drivers/spi/spi-apple-qspi.c @@ -611,11 +611,14 @@ static int apple_qspi_resume(struct device *dev) struct apple_qspi *qspi = spi_controller_get_devdata(ctlr); int ret; + /* + * Reopen the controller even if the reset failed. Left suspended, it + * would fail every transfer until reboot; a transfer that times out + * resets it again instead. + */ ret = apple_qspi_hw_reset(qspi); - if (ret) { + if (ret) dev_err(dev, "controller reset failed on resume: %d\n", ret); - return ret; - } return spi_controller_resume(ctlr); } From 30181c88b4daa145396d166a3dcb3dc52eb2bb54 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 15:30:05 +1100 Subject: [PATCH 1521/2680] soc: apple: aop: Test that a wrapped entry is refused below the read pointer When the write pointer is below the read pointer, the unread entries include [0, wptr), so a transmit entry must never wrap to the top of the ring there, even when it would end before the read pointer. write_entry() refuses that through its `wptr < rptr && wrapped` term, which commit 033fa5e3c095 ("soc: apple: aop: Accept a transmit entry at a write pointer of zero") rewrote, but no test pinned it: removing the term left every case passing. Add a case with two such entries. Neither ends on or past the read pointer, so only that term refuses them. (cherry picked from commit 35d6b26e5c9d8bf22ce6284bf0594084ffd2dcdf) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop_ring.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/soc/apple/aop_ring.rs b/drivers/soc/apple/aop_ring.rs index 41608ba7699b5..b07dede89fe23 100644 --- a/drivers/soc/apple/aop_ring.rs +++ b/drivers/soc/apple/aop_ring.rs @@ -320,6 +320,18 @@ mod tests { Ok(()) } + #[test] + fn a_wrapped_entry_is_refused_below_the_read_pointer() -> Result { + let ring = Ring::new(BUF, BLOCK)?; + // wptr < rptr: [0, wptr) holds unread entries, so an entry must + // not wrap even when it would end before the read pointer. + assert!(ring.write_entry(0xfc0, 0xf80, 0x80).is_err()); + assert!(ring + .write_entry(0xf80, 0xf00, 0x100 - QEH_SIZE + 1) + .is_err()); + Ok(()) + } + #[test] fn an_entry_never_reaches_the_read_pointer() -> Result { let ring = Ring::new(BUF, BLOCK)?; From e703b5b4ec6a6c5b7781edd0d896c3b47cbb1843 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 18:30:00 +1100 Subject: [PATCH 1522/2680] usb: typec: tipd: cd321x: Don't race the resume read in the sleep retry test cd321x_ack_retry_across_sleep_test() asserted that the resume read was pending right after tipd_complete(). cd321x_complete() queues that read with no delay, so on an SMP test machine a worker can already have taken it off the queue when the assertion runs, and the test fails although the driver behaved correctly. It failed once in the arm64 KUnit run with lockdep and the USB, DART and ATC PHY tests built in. Flush first, then check that the failed read queued its retry again. A retry is queued with a delay of at least CD321X_ACK_RETRY_MS, so it is still pending when it is checked. Removing the mod_delayed_work() of cd321x_complete(), or the retry of a failed acknowledgment in cd321x_resume_work(), still makes the test fail. Fixes: a8c7348c5091 ("usb: typec: tipd: cd321x: Never leave interrupt events unacknowledged") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/cd321x-irq-test.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/usb/typec/tipd/cd321x-irq-test.c b/drivers/usb/typec/tipd/cd321x-irq-test.c index c14a0cdd16726..5062d45616773 100644 --- a/drivers/usb/typec/tipd/cd321x-irq-test.c +++ b/drivers/usb/typec/tipd/cd321x-irq-test.c @@ -326,8 +326,9 @@ static void cd321x_ack_retry_across_sleep_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); tipd_complete(dev); for (i = 0; i < 2 * CD321X_RESUME_ATTEMPTS; i++) { - KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); + /* Queued first without delay: a worker may already be running it. */ flush_delayed_work(&t->cd.resume_work); + KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); } t->ctrl.failing_event_reads = 0; From a8c6adadada970d581bbe96aa240e2c1a5776b4f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 16:57:00 +1100 Subject: [PATCH 1523/2680] KEYS: trusted: Keep a source that registers before init_trusted() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A trusted-key source module that is built in registers from its probe: apple_sep with CONFIG_APPLE_SEP=y probes at device_initcall, before init_trusted() runs at late_initcall. init_trusted() then looks for a built-in source, finds none, and points the dispatch at the absent stubs, without taking the source mutex or checking for a registered source. That source still holds the slot, so nothing can take it again. Every new trusted key now fails with -ENODEV after registration was logged as a success, and the key type stays registered. A modular apple_sep loads after init_trusted() and is not affected. Mainline chooses the source once, in init_trusted(). Keep it that way, and have init_trusted() defer: under the source mutex, it leaves the slot and the dispatch to a source that is already active, and otherwise chooses as before. Whichever source comes up first keeps the slot, and a later one still gets -EBUSY; trusted.source= and trusted.rng= are checked on both paths, as before. The choice moves into trusted_key_core_init(), which the KUnit tests call with synthetic sources, in both orders: a source that registers before the core's init keeps the slot and its migration policy, and one that registers after it replaces the absent stubs. [ryan: adapted to 7.2.10 — context only: trusted-type.h declares trusted_key_core_init() above the CONFIG_TRUSTED_KEYS_DEBUG block that 7.2.10 has in place of the TRUSTED_DEBUG define, as in the 7.2.10 port] Fixes: bac0846c9087 ("soc: apple: add Apple SEP driver (Touch ID and key sealing)") (cherry picked from commit 4f15946bdd603c77e344d922e39fd1949466fdb2 ("KEYS: trusted: Keep a source that registers before init_trusted()")) Assisted-by: LLM Signed-off-by: Ryan Murray --- include/keys/trusted-type.h | 5 ++ security/keys/trusted-keys/Kconfig | 5 +- .../trusted-keys/tests/trusted_source_kunit.c | 73 ++++++++++++++++++- security/keys/trusted-keys/trusted_core.c | 70 ++++++++++++++---- 4 files changed, 134 insertions(+), 19 deletions(-) diff --git a/include/keys/trusted-type.h b/include/keys/trusted-type.h index e95387b11f925..fb2ec53ebce06 100644 --- a/include/keys/trusted-type.h +++ b/include/keys/trusted-type.h @@ -86,6 +86,11 @@ extern struct key_type key_type_trusted; int register_trusted_key_source(struct trusted_key_source *src); void unregister_trusted_key_source(struct trusted_key_source *src); +#if IS_ENABLED(CONFIG_KUNIT) +/* For the KUnit tests of the trusted-key core only. */ +int trusted_key_core_init(const struct trusted_key_source *sources, size_t n); +#endif + #ifdef CONFIG_TRUSTED_KEYS_DEBUG extern bool trusted_debug; diff --git a/security/keys/trusted-keys/Kconfig b/security/keys/trusted-keys/Kconfig index 3a3799e4134eb..899f80d95a663 100644 --- a/security/keys/trusted-keys/Kconfig +++ b/security/keys/trusted-keys/Kconfig @@ -87,8 +87,9 @@ config TRUSTED_SOURCE_KUNIT_TEST default KUNIT_ALL_TESTS help This builds unit tests for register_trusted_key_source() and - unregister_trusted_key_source(), using a synthetic source that - touches no hardware and no real key material. + unregister_trusted_key_source(), called before and after the + core's own init, using synthetic sources that touch no hardware + and no real key material. Built in, they run before mark_readonly(), while read-only-after-init data can still be written. Build them as a module and load it after diff --git a/security/keys/trusted-keys/tests/trusted_source_kunit.c b/security/keys/trusted-keys/tests/trusted_source_kunit.c index f0491743f8d5a..6a5eb3e79d756 100644 --- a/security/keys/trusted-keys/tests/trusted_source_kunit.c +++ b/security/keys/trusted-keys/tests/trusted_source_kunit.c @@ -1,7 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-only /* * KUnit tests for trusted-key sources registered at runtime, through - * register_trusted_key_source() and unregister_trusted_key_source(). + * register_trusted_key_source() and unregister_trusted_key_source(), before + * and after the core's own init. * * The source under test is synthetic: it touches no hardware and seals * nothing for real. A test is skipped when another source holds the slot. @@ -22,12 +23,17 @@ #include #define FAKE_NAME "trusted_source_kunit" +#define FAKE_BUILTIN_NAME "trusted_source_kunit_builtin" struct trusted_source_fake { struct trusted_key_ops ops; struct trusted_key_source src; int init_ret; int inits, exits, seals, unseals; + /* A built-in source for trusted_key_core_init(). */ + struct trusted_key_ops builtin_ops; + struct trusted_key_source builtin; + int builtin_inits; }; /* The ops have no context argument; the tests run one at a time. */ @@ -44,6 +50,13 @@ static void fake_exit(void) fake->exits++; } +/* A built-in source that finds no device, as each of them does on Apple. */ +static int fake_builtin_init(void) +{ + fake->builtin_inits++; + return -ENODEV; +} + /* Not a cipher: an invertible stand-in, so that unseal can be checked. */ static int fake_seal(struct trusted_key_payload *p, char *datablob) { @@ -149,6 +162,56 @@ static void trusted_source_after_mark_readonly(struct kunit *test) KUNIT_EXPECT_EQ(test, fake->seals, 2); } +/* + * A source module that is built in, as apple_sep can be, registers from its + * probe at device_initcall, before init_trusted() runs at late_initcall. + * The core then leaves it the slot and the dispatch. + */ +static void trusted_source_before_core_init(struct kunit *test) +{ + struct key *key; + + fake->ops.migratable = 1; + fake_register(test); + + KUNIT_EXPECT_EQ(test, trusted_key_core_init(&fake->builtin, 1), 0); + KUNIT_EXPECT_EQ(test, fake->builtin_inits, 0); + + key = fake_key(test, "new 32"); + KUNIT_ASSERT_FALSE(test, IS_ERR(key)); + KUNIT_EXPECT_EQ(test, fake->seals, 1); + KUNIT_EXPECT_EQ(test, fake_key_migratable(key), 1); + + /* The source still holds the slot, so it can withdraw. */ + unregister_trusted_key_source(&fake->src); + KUNIT_EXPECT_EQ(test, fake->exits, 1); + KUNIT_EXPECT_EQ(test, PTR_ERR(fake_key(test, "new 32")), -ENODEV); + KUNIT_EXPECT_EQ(test, fake->seals, 1); +} + +/* + * A source module registers after the core's init found no built-in + * source, as a modular apple_sep does. + */ +static void trusted_source_after_core_init(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, trusted_key_core_init(&fake->builtin, 1), 0); + if (!fake->builtin_inits) + kunit_skip(test, "another trusted-key source holds the slot"); + + /* With no source, the absent stubs answer. */ + KUNIT_EXPECT_EQ(test, PTR_ERR(fake_key(test, "new 32")), -ENODEV); + + fake_register(test); + KUNIT_EXPECT_FALSE(test, IS_ERR(fake_key(test, "new 32"))); + KUNIT_EXPECT_EQ(test, fake->seals, 1); + + unregister_trusted_key_source(&fake->src); + KUNIT_EXPECT_EQ(test, fake->inits, 1); + KUNIT_EXPECT_EQ(test, fake->exits, 1); + KUNIT_EXPECT_EQ(test, fake->builtin_inits, 1); +} + /* Keys are sealed and unsealed by the source until it withdraws. */ static void trusted_source_dispatch(struct kunit *test) { @@ -315,6 +378,11 @@ static int trusted_source_init(struct kunit *test) fake->src.name = FAKE_NAME; fake->src.ops = &fake->ops; + fake->builtin_ops = fake->ops; + fake->builtin_ops.init = fake_builtin_init; + fake->builtin.name = FAKE_BUILTIN_NAME; + fake->builtin.ops = &fake->builtin_ops; + /* However the test ends; a no-op unless the fake is registered. */ return kunit_add_action_or_reset(test, trusted_source_withdraw, &fake->src); @@ -322,6 +390,8 @@ static int trusted_source_init(struct kunit *test) static struct kunit_case trusted_source_cases[] = { KUNIT_CASE(trusted_source_after_mark_readonly), + KUNIT_CASE(trusted_source_before_core_init), + KUNIT_CASE(trusted_source_after_core_init), KUNIT_CASE(trusted_source_dispatch), KUNIT_CASE(trusted_source_migratable), KUNIT_CASE(trusted_source_exclusive), @@ -338,5 +408,6 @@ static struct kunit_suite trusted_source_suite = { kunit_test_suite(trusted_source_suite); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); MODULE_DESCRIPTION("KUnit tests for trusted-key sources registered at runtime"); MODULE_LICENSE("GPL"); diff --git a/security/keys/trusted-keys/trusted_core.c b/security/keys/trusted-keys/trusted_core.c index 77db540b420ed..5387f66316ff1 100644 --- a/security/keys/trusted-keys/trusted_core.c +++ b/security/keys/trusted-keys/trusted_core.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -491,18 +492,45 @@ static int kernel_get_random(unsigned char *key, size_t key_len) return get_random_bytes_wait(key, key_len) ?: key_len; } -static int __init init_trusted(void) +/** + * trusted_key_core_init - give the source slot to a built-in source + * @sources: the built-in sources, in order of preference + * @n: the number of @sources + * + * A source module that is built in, such as apple_sep, can register from + * its probe at device_initcall, before init_trusted() runs at + * late_initcall. Such a source keeps the slot: the built-in sources are + * not tried, and the dispatch stays with it. Otherwise the first built-in + * source to initialise takes the slot, and with none the dispatch goes to + * the absent stubs, so that a source module can register later. + * + * init_trusted() calls this once, with trusted_key_sources. The KUnit tests + * call it with synthetic sources of their own. + * + * Return: 0, or the error of trusted.rng or of a built-in source's init(). + */ +VISIBLE_IF_KUNIT int +trusted_key_core_init(const struct trusted_key_source *sources, size_t n) { int (*get_random)(unsigned char *key, size_t key_len); /* -ENODEV when the built-in array is empty or nothing matches, so a * later module source can still register instead of hitting -EBUSY. */ - int i, ret = -ENODEV; + int ret = -ENODEV; + size_t i; + + mutex_lock(&trusted_key_source_lock); + + /* A source module that is built in may have registered first. */ + if (trusted_key_source_active) { + ret = 0; + goto out; + } - for (i = 0; i < ARRAY_SIZE(trusted_key_sources); i++) { + for (i = 0; i < n; i++) { if (trusted_key_source && - strncmp(trusted_key_source, trusted_key_sources[i].name, - strlen(trusted_key_sources[i].name))) + strncmp(trusted_key_source, sources[i].name, + strlen(sources[i].name))) continue; /* @@ -510,31 +538,32 @@ static int __init init_trusted(void) * well as trusted.rng=$trusted.source if the trust source * defines its own get_random callback. */ - get_random = trusted_key_sources[i].ops->get_random; + get_random = sources[i].ops->get_random; if (trusted_rng && strcmp(trusted_rng, "default")) { if (!strcmp(trusted_rng, "kernel")) { get_random = kernel_get_random; - } else if (strcmp(trusted_rng, trusted_key_sources[i].name) || + } else if (strcmp(trusted_rng, sources[i].name) || !get_random) { pr_warn("Unsupported RNG. Supported: kernel"); if (get_random) - pr_cont(", %s", trusted_key_sources[i].name); + pr_cont(", %s", sources[i].name); pr_cont(", default\n"); - return -EINVAL; + ret = -EINVAL; + goto out; } } if (!get_random) get_random = kernel_get_random; - ret = trusted_key_sources[i].ops->init(); + ret = sources[i].ops->init(); if (!ret) { - static_call_update(trusted_key_seal, trusted_key_sources[i].ops->seal); - static_call_update(trusted_key_unseal, trusted_key_sources[i].ops->unseal); + static_call_update(trusted_key_seal, sources[i].ops->seal); + static_call_update(trusted_key_unseal, sources[i].ops->unseal); static_call_update(trusted_key_get_random, get_random); - trusted_key_exit = trusted_key_sources[i].ops->exit; - migratable = trusted_key_sources[i].ops->migratable; + trusted_key_exit = sources[i].ops->exit; + migratable = sources[i].ops->migratable; } if (!ret || ret != -ENODEV) @@ -555,14 +584,23 @@ static int __init init_trusted(void) static_call_update(trusted_key_unseal, trusted_key_absent_unseal); static_call_update(trusted_key_get_random, trusted_key_absent_get_random); - return 0; + ret = 0; + goto out; } if (!ret) trusted_key_source_active = true; - +out: + mutex_unlock(&trusted_key_source_lock); return ret; } +EXPORT_SYMBOL_IF_KUNIT(trusted_key_core_init); + +static int __init init_trusted(void) +{ + return trusted_key_core_init(trusted_key_sources, + ARRAY_SIZE(trusted_key_sources)); +} static void __exit cleanup_trusted(void) { From bbe82f98ee43a199893fc6b1411f63b0df28d683 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 17:43:29 +1100 Subject: [PATCH 1524/2680] soc: apple: sep: Recover the entropy exchange after a request times out Every GET_ENTROPY request carries the same reserved tag, and the exchange ends as soon as its waiter gives up. A reply that comes after its request timed out is then taken as the answer to the next request, and from there on every answer is one request late. Nothing stops the hwrng fill thread from asking again either. Each retry is one more message the enclave is not reading, in the inbox it shares with the key store, Touch ID and the xART service, and once that is full every send to the enclave waits 500 ms for room and fails. Count the requests that the mailbox took and the enclave has not answered. The enclave answers in order, so only the reply that settles the newest one is its answer, and earlier replies are dropped. After a request goes unanswered, or the mailbox has no room for it, hold further requests back with EAGAIN for 1 s, doubling with each failure up to 60 s, and resume as soon as a late reply shows that the enclave is answering again. A request made once the wait is over counts what is still owed as lost, so a reply that never comes cannot stop the channel for good. A held-back hwrng read returns -EAGAIN and is not logged as a failure; the core tries again 10 s later. A held-back survey attempt is not counted against the five the survey gets. KUnit tests (apple_sep_control) cover a reply in time, a late reply, a reply that comes after the retry, a lost reply, a full mailbox, a failed send, the back-off and a request abandoned for removal. Fixes: bac0846c9087 ("soc: apple: add Apple SEP driver (Touch ID and key sealing)") (cherry picked from commit f771659aba5e9ab34342df9a90499e0bb71a76c8 ("soc: apple: sep: Recover the entropy exchange after a request times out")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/control.rs | 247 ++++++++++++++++++++++++++++++++++- drivers/soc/apple/sep.rs | 37 +++++- 2 files changed, 272 insertions(+), 12 deletions(-) diff --git a/drivers/soc/apple/control.rs b/drivers/soc/apple/control.rs index beb6b608415de..aeda23f5ef42e 100644 --- a/drivers/soc/apple/control.rs +++ b/drivers/soc/apple/control.rs @@ -23,13 +23,44 @@ impl Slot { }; } +/// After an entropy request goes unanswered, the next one waits this long, +/// doubling with each further failure up to [`ENTROPY_RETRY_MAX_MS`]. +pub(crate) const ENTROPY_RETRY_MIN_MS: u64 = 1000; +pub(crate) const ENTROPY_RETRY_MAX_MS: u64 = 60_000; + struct EntropySink { value: Option, busy: bool, + // Requests made and not answered yet, oldest first. A request that timed + // out may still be answered, and every reply carries the reserved tag, + // so only their order tells them apart. + owed: u32, + // A request went unanswered: none is sent before this time, in ms. + retry_at: Option, + retry_ms: u64, +} + +/// How an entropy exchange ended. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum EntropyEnd { + /// The reply was taken. + Answered, + /// No reply in time. It may still come. + TimedOut, + /// The mailbox did not take the request; `full` if it ran out of time + /// waiting for room, so the enclave is not reading it either. + NotSent { full: bool }, + /// Given up for a removal. The reply may still come. + Abandoned, } pub(crate) enum Delivery { Entropy, + /// The answer to an entropy request that had timed out. `recovered` if + /// requests were being held back until the enclave answered again. + EntropyLate { + recovered: bool, + }, Matched, Unmatched, } @@ -52,6 +83,9 @@ impl ControlState { entropy: EntropySink { value: None, busy: false, + owed: 0, + retry_at: None, + retry_ms: ENTROPY_RETRY_MIN_MS, }, } } @@ -125,8 +159,7 @@ impl ControlState { pub(crate) fn deliver(&mut self, reply: proto::ControlReply) -> Delivery { // The reserved entropy tag is claimed first, outstanding request or not. if reply.tag == proto::TAG_ENTROPY { - self.entropy.value = Some(reply.data_lo); - return Delivery::Entropy; + return self.deliver_entropy(reply.data_lo); } if let Some(slot) = self @@ -146,12 +179,51 @@ impl ControlState { Delivery::Unmatched } - pub(crate) fn entropy_begin(&mut self) -> Result<()> { + // The enclave answers in order, so a reply belongs to the oldest request + // still owed. Only the one that settles the newest request is its answer; + // any before it answer requests that timed out, and are dropped. + fn deliver_entropy(&mut self, value: u32) -> Delivery { + let entropy = &mut self.entropy; + if entropy.owed == 0 { + // Answers a request already written off as lost. + return Delivery::Unmatched; + } + entropy.owed -= 1; + if entropy.owed == 0 && entropy.busy { + entropy.value = Some(value); + return Delivery::Entropy; + } + // The enclave is answering again, so stop holding requests back. + let recovered = !entropy.busy && entropy.retry_at.take().is_some(); + if recovered { + entropy.retry_ms = ENTROPY_RETRY_MIN_MS; + } + Delivery::EntropyLate { recovered } + } + + /// Claims the entropy exchange for a request to be sent at `now_ms`. + /// + /// Refused with EAGAIN after a request the enclave did not answer, until + /// it answers or the retry time comes. A request that fails then doubles + /// the wait. Each claim counts as owed until [`Self::entropy_end`], + /// because the reply can arrive before the sender returns from the + /// mailbox. + pub(crate) fn entropy_begin(&mut self, now_ms: u64) -> Result<()> { if self.entropy.busy { return Err(EBUSY); } + if let Some(at) = self.entropy.retry_at { + if now_ms < at { + return Err(EAGAIN); + } + // Nothing came back in all that time. Count those replies as + // lost, or the answer to this request would be taken for one of + // theirs and no request would be answered again. + self.entropy.owed = 0; + } self.entropy.busy = true; self.entropy.value = None; + self.entropy.owed = self.entropy.owed.saturating_add(1); Ok(()) } @@ -159,7 +231,172 @@ impl ControlState { self.entropy.value.take() } - pub(crate) fn entropy_end(&mut self) { - self.entropy.busy = false; + /// Ends the exchange claimed by [`Self::entropy_begin`], at `now_ms`. + pub(crate) fn entropy_end(&mut self, how: EntropyEnd, now_ms: u64) { + let entropy = &mut self.entropy; + entropy.busy = false; + let failed = match how { + EntropyEnd::Answered => { + entropy.retry_at = None; + entropy.retry_ms = ENTROPY_RETRY_MIN_MS; + false + } + EntropyEnd::TimedOut => true, + EntropyEnd::NotSent { full } => { + entropy.owed = entropy.owed.saturating_sub(1); + full + } + EntropyEnd::Abandoned => false, + }; + if failed { + entropy.retry_at = Some(now_ms.saturating_add(entropy.retry_ms)); + entropy.retry_ms = + core::cmp::min(entropy.retry_ms.saturating_mul(2), ENTROPY_RETRY_MAX_MS); + } + } +} + +#[kernel::macros::kunit_tests(apple_sep_control)] +mod tests { + use super::*; + + fn entropy(value: u32) -> proto::ControlReply { + proto::ControlReply { + tag: proto::TAG_ENTROPY, + data_lo: value, + } + } + + // One request answered in time, as the hwrng read makes it. + fn draw(state: &mut ControlState, now_ms: u64, value: u32) -> Option { + state.entropy_begin(now_ms).ok()?; + let delivered = matches!(state.deliver(entropy(value)), Delivery::Entropy); + let taken = state.entropy_take(); + state.entropy_end( + if taken.is_some() { + EntropyEnd::Answered + } else { + EntropyEnd::TimedOut + }, + now_ms, + ); + if delivered { + taken + } else { + None + } + } + + #[test] + fn a_request_is_answered() { + let mut state = ControlState::new(); + assert_eq!(draw(&mut state, 0, 0x1234_5678), Some(0x1234_5678)); + assert_eq!(draw(&mut state, 1, 0x9abc_def0), Some(0x9abc_def0)); + } + + #[test] + fn a_late_reply_is_not_the_next_requests_answer() { + let mut state = ControlState::new(); + assert!(state.entropy_begin(0).is_ok()); + state.entropy_end(EntropyEnd::TimedOut, 0); + // The retry is held back until the enclave answers. + assert_eq!(state.entropy_begin(1), Err(EAGAIN)); + // It answers the request that timed out: dropped, and reads resume. + assert!(matches!( + state.deliver(entropy(1)), + Delivery::EntropyLate { recovered: true } + )); + assert_eq!(draw(&mut state, 2, 2), Some(2)); + } + + #[test] + fn a_reply_later_than_the_retry_shifts_nothing_after_it() { + let mut state = ControlState::new(); + assert!(state.entropy_begin(0).is_ok()); + state.entropy_end(EntropyEnd::TimedOut, 0); + // The retry goes out once its time comes, the first reply written off. + let at = ENTROPY_RETRY_MIN_MS; + assert!(state.entropy_begin(at).is_ok()); + // That reply comes after all. The retry takes it: it is enclave + // output nobody has used. + assert!(matches!(state.deliver(entropy(1)), Delivery::Entropy)); + assert_eq!(state.entropy_take(), Some(1)); + state.entropy_end(EntropyEnd::Answered, at); + // The retry's own reply is then owed to nobody. + assert!(matches!(state.deliver(entropy(2)), Delivery::Unmatched)); + assert_eq!(draw(&mut state, at + 1, 3), Some(3)); + } + + #[test] + fn a_request_abandoned_for_removal_does_not_shift_the_answers() { + let mut state = ControlState::new(); + assert!(state.entropy_begin(0).is_ok()); + state.entropy_end(EntropyEnd::Abandoned, 0); + assert!(state.entropy_begin(0).is_ok()); + assert!(matches!( + state.deliver(entropy(1)), + Delivery::EntropyLate { recovered: false } + )); + assert_eq!(state.entropy_take(), None); + assert!(matches!(state.deliver(entropy(2)), Delivery::Entropy)); + assert_eq!(state.entropy_take(), Some(2)); + state.entropy_end(EntropyEnd::Answered, 0); + } + + #[test] + fn a_lost_reply_does_not_stop_the_channel_for_good() { + let mut state = ControlState::new(); + assert!(state.entropy_begin(0).is_ok()); + state.entropy_end(EntropyEnd::TimedOut, 0); + // That reply never comes. The retry must still get its own answer. + let at = ENTROPY_RETRY_MIN_MS; + assert_eq!(draw(&mut state, at, 9), Some(9)); + assert_eq!(draw(&mut state, at + 1, 10), Some(10)); + } + + #[test] + fn a_full_mailbox_holds_requests_back_too() { + let mut state = ControlState::new(); + assert!(state.entropy_begin(0).is_ok()); + state.entropy_end(EntropyEnd::NotSent { full: true }, 0); + assert_eq!(state.entropy_begin(ENTROPY_RETRY_MIN_MS - 1), Err(EAGAIN)); + // Nothing was owed: the next request's reply is its own. + assert_eq!(draw(&mut state, ENTROPY_RETRY_MIN_MS, 4), Some(4)); + } + + #[test] + fn a_failed_send_owes_nothing() { + let mut state = ControlState::new(); + assert!(state.entropy_begin(0).is_ok()); + state.entropy_end(EntropyEnd::NotSent { full: false }, 0); + assert_eq!(draw(&mut state, 0, 8), Some(8)); + } + + #[test] + fn retries_back_off_up_to_a_limit() { + let mut state = ControlState::new(); + let mut now = 0; + let mut wait = ENTROPY_RETRY_MIN_MS; + for _ in 0..12 { + assert!(state.entropy_begin(now).is_ok()); + state.entropy_end(EntropyEnd::TimedOut, now); + assert_eq!(state.entropy_begin(now + wait - 1), Err(EAGAIN)); + now += wait; + wait = core::cmp::min(wait * 2, ENTROPY_RETRY_MAX_MS); + } + assert_eq!(wait, ENTROPY_RETRY_MAX_MS); + // An answer resets the wait. + assert_eq!(draw(&mut state, now, 1), Some(1)); + assert!(state.entropy_begin(now).is_ok()); + state.entropy_end(EntropyEnd::TimedOut, now); + assert!(state.entropy_begin(now + ENTROPY_RETRY_MIN_MS).is_ok()); + state.entropy_end(EntropyEnd::Abandoned, now); + } + + #[test] + fn a_reply_nobody_is_owed_is_dropped() { + let mut state = ControlState::new(); + assert!(matches!(state.deliver(entropy(1)), Delivery::Unmatched)); + assert_eq!(draw(&mut state, 0, 2), Some(2)); } } diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 9608d92ef776a..a495f12814efb 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -207,6 +207,10 @@ fn sks_declared_sizes() -> (usize, usize) { (0x8000, 0x4000) } +fn boottime_ms() -> u64 { + shim::boottime_ns() / 1_000_000 +} + struct Hex<'a>(&'a [u8]); impl kernel::fmt::Display for Hex<'_> { @@ -1160,25 +1164,28 @@ impl SepData { return Err(EINVAL); } - self.control.lock().entropy_begin()?; + self.control.lock().entropy_begin(boottime_ms())?; let msg = proto::encode_control(&op, proto::TAG_ENTROPY); if let Err(e) = self.send(msg) { - self.control.lock().entropy_end(); + let full = e == ETIMEDOUT; + self.control + .lock() + .entropy_end(control::EntropyEnd::NotSent { full }, boottime_ms()); return Err(e); } let mut remaining = time::msecs_to_jiffies(op.timeout_ms()); let mut guard = self.control.lock(); - let result = loop { + let (result, end) = loop { if let Some(value) = guard.entropy_take() { - break Ok(value); + break (Ok(value), control::EntropyEnd::Answered); } if self.rng_shutdown.load(Relaxed) { - break Err(ECANCELED); + break (Err(ECANCELED), control::EntropyEnd::Abandoned); } if remaining == 0 { - break Err(ETIMEDOUT); + break (Err(ETIMEDOUT), control::EntropyEnd::TimedOut); } match self .control_wq @@ -1189,7 +1196,7 @@ impl SepData { CondVarTimeoutResult::Timeout => remaining = 0, } }; - guard.entropy_end(); + guard.entropy_end(end, boottime_ms()); result } @@ -1228,6 +1235,12 @@ impl SepData { this.rng_survey_attempts.store(attempt, Relaxed); match this.survey_hwrng() { Ok(()) => dev_info!(this.dev, "hwrng: registered after ready-phase survey (attempt {})\n", attempt), + // Holding back after a request that went unanswered: this + // attempt did not ask the enclave anything. + Err(e) if e == EAGAIN => { + this.rng_survey_attempts.store(attempt - 1, Relaxed); + Self::arm_settle(this); + } Err(e) if attempt < RNG_SURVEY_ATTEMPTS => { dev_warn!(this.dev, "hwrng: survey attempt {} failed ({:?}); retrying\n", attempt, e); Self::arm_settle(this); @@ -1834,6 +1847,11 @@ impl SepData { // must return a negative errno, not 0: a blocking reader would spin on 0 let e = failure.unwrap_or(EIO); + // Holding back after a request that went unanswered. Nothing failed + // in this read; the core tries again later. + if e == EAGAIN { + return e.to_errno(); + } let n = self.rng_failures.load(Relaxed).wrapping_add(1); self.rng_failures.store(n, Relaxed); if n == 1 { @@ -2081,6 +2099,11 @@ impl SepData { control::Delivery::Entropy => { self.control_wq.notify_all(); } + control::Delivery::EntropyLate { recovered: true } => dev_info!( + self.dev, + "hwrng: the enclave answered a request that had timed out; reads resume\n" + ), + control::Delivery::EntropyLate { recovered: false } => {} control::Delivery::Matched => self.control_wq.notify_all(), control::Delivery::Unmatched => {} } From 77866550dddf3391e499a93f6f41aa97319944a0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 17:43:29 +1100 Subject: [PATCH 1525/2680] soc: apple: sep: Hold entropy requests while the system sleeps The hwrng fill thread is not frozen for system sleep, and nothing in the driver held it back, so it could ask the enclave for entropy while devices were suspended. On an M1 Pro (T6000) one such request, sent during the 5 s pm_test=platform hold, was never answered. After the resume the enclave sent nothing more and did not read its mailbox again: every send timed out waiting for room, so /dev/hwrng stayed dead until reboot, and no other enclave service could be reached either. With the hwrng source deselected over three suspend cycles the enclave kept working, so it survives the sleep itself; it does not survive a request in that window. Hold entropy requests from the start of a suspend or hibernation until its end is reported, on the PM notifier that already holds Touch ID captures. Its window contains the whole of device suspend. A request already sent is first given time to be answered, which the enclave can still do then. A read in the window returns -EAGAIN, which the core treats as a source with nothing to give, and it tries again after the resume. KUnit tests cover a read while the system sleeps and a request in flight when the sleep begins. Fixes: bac0846c9087 ("soc: apple: add Apple SEP driver (Touch ID and key sealing)") (cherry picked from commit 1b0451249175e77ba9775e276d75a1d6e2310bf1 ("soc: apple: sep: Hold entropy requests while the system sleeps")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/control.rs | 57 ++++++++++++++++++++++++++++++++---- drivers/soc/apple/pm.rs | 9 ++++-- drivers/soc/apple/sep.rs | 41 +++++++++++++++++++++++--- 3 files changed, 96 insertions(+), 11 deletions(-) diff --git a/drivers/soc/apple/control.rs b/drivers/soc/apple/control.rs index aeda23f5ef42e..5eb9d51c9f421 100644 --- a/drivers/soc/apple/control.rs +++ b/drivers/soc/apple/control.rs @@ -35,6 +35,8 @@ struct EntropySink { // out may still be answered, and every reply carries the reserved tag, // so only their order tells them apart. owed: u32, + // The system is going to sleep: no request may be sent. + asleep: bool, // A request went unanswered: none is sent before this time, in ms. retry_at: Option, retry_ms: u64, @@ -84,6 +86,7 @@ impl ControlState { value: None, busy: false, owed: 0, + asleep: false, retry_at: None, retry_ms: ENTROPY_RETRY_MIN_MS, }, @@ -203,15 +206,18 @@ impl ControlState { /// Claims the entropy exchange for a request to be sent at `now_ms`. /// - /// Refused with EAGAIN after a request the enclave did not answer, until - /// it answers or the retry time comes. A request that fails then doubles - /// the wait. Each claim counts as owed until [`Self::entropy_end`], - /// because the reply can arrive before the sender returns from the - /// mailbox. + /// Refused with EAGAIN while the system sleeps, and after a request the + /// enclave did not answer, until it answers or the retry time comes. A + /// request that fails then doubles the wait. Each claim counts as owed + /// until [`Self::entropy_end`], because the reply can arrive before the + /// sender returns from the mailbox. pub(crate) fn entropy_begin(&mut self, now_ms: u64) -> Result<()> { if self.entropy.busy { return Err(EBUSY); } + if self.entropy.asleep { + return Err(EAGAIN); + } if let Some(at) = self.entropy.retry_at { if now_ms < at { return Err(EAGAIN); @@ -254,6 +260,21 @@ impl ControlState { core::cmp::min(entropy.retry_ms.saturating_mul(2), ENTROPY_RETRY_MAX_MS); } } + + /// The system is going to sleep: refuse requests until + /// [`Self::entropy_wake`]. Returns whether one is still in flight. + pub(crate) fn entropy_sleep(&mut self) -> bool { + self.entropy.asleep = true; + self.entropy.busy + } + + pub(crate) fn entropy_busy(&self) -> bool { + self.entropy.busy + } + + pub(crate) fn entropy_wake(&mut self) { + self.entropy.asleep = false; + } } #[kernel::macros::kunit_tests(apple_sep_control)] @@ -294,6 +315,32 @@ mod tests { assert_eq!(draw(&mut state, 1, 0x9abc_def0), Some(0x9abc_def0)); } + #[test] + fn no_request_is_sent_while_the_system_sleeps() { + let mut state = ControlState::new(); + assert!(!state.entropy_sleep()); + assert_eq!(state.entropy_begin(0), Err(EAGAIN)); + assert_eq!(state.entropy_begin(ENTROPY_RETRY_MAX_MS * 10), Err(EAGAIN)); + state.entropy_wake(); + assert_eq!(draw(&mut state, 1, 7), Some(7)); + } + + #[test] + fn a_request_in_flight_finishes_before_sleep() { + let mut state = ControlState::new(); + assert!(state.entropy_begin(0).is_ok()); + // The sleep has to wait for this one. + assert!(state.entropy_sleep()); + assert!(matches!(state.deliver(entropy(5)), Delivery::Entropy)); + assert_eq!(state.entropy_take(), Some(5)); + state.entropy_end(EntropyEnd::Answered, 0); + assert!(!state.entropy_busy()); + // And nothing new starts until the system is awake. + assert_eq!(state.entropy_begin(0), Err(EAGAIN)); + state.entropy_wake(); + assert_eq!(draw(&mut state, 0, 6), Some(6)); + } + #[test] fn a_late_reply_is_not_the_next_requests_answer() { let mut state = ControlState::new(); diff --git a/drivers/soc/apple/pm.rs b/drivers/soc/apple/pm.rs index a517e6e448732..b276890c27c78 100644 --- a/drivers/soc/apple/pm.rs +++ b/drivers/soc/apple/pm.rs @@ -4,8 +4,10 @@ //! talking to the sensor over SPI, so one left running into a suspend is still //! active while devices go down. A J313 that suspended with the lock screen's //! capture in flight did not wake. Before the system suspends, end any capture -//! in progress and hold new ones until it has resumed. The notifier itself -//! is in `pm_shim.c`; the capture handling is on [`SepData`]. +//! in progress and hold new ones until it has resumed. The hwrng fill thread +//! is not frozen either, so entropy requests are held over the same window. +//! The notifier itself is in `pm_shim.c`; the capture and entropy handling is +//! on [`SepData`]. use core::sync::atomic::{AtomicPtr, Ordering}; @@ -51,13 +53,16 @@ unsafe extern "C" fn pm_event(entering: bool) -> c_int { // notifier call, so it is live for this borrow. let sep = unsafe { as ForeignOwnable>::borrow(ptr) }; if entering { + sep.rng_sleep_prepare(); if let Err(error) = sep.sleep_prepare() { // A failing notifier is not included in the PM chain's rollback. sep.sleep_finished(); + sep.rng_sleep_finished(); return error.to_errno(); } } else { sep.sleep_finished(); + sep.rng_sleep_finished(); } 0 } diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index a495f12814efb..662078489c205 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -173,6 +173,10 @@ const PROTECTED_DATA_AVAILABLE: bool = true; const RNG_MAX_WORDS_PER_READ: usize = 16; +// How long going to sleep waits for an entropy request in flight: the +// mailbox's 500 ms wait for room, then the wait for the reply. +const RNG_SLEEP_DRAIN_MS: u32 = proto::CONTROL_TIMEOUT_MS + 1000; + extern "C" { fn sep_cancel_work_sync(work: *mut c_void); fn sep_cancel_delayed_work_sync(work: *mut c_void); @@ -1200,6 +1204,35 @@ impl SepData { result } + /// The system is about to suspend: send no entropy request until it has + /// resumed, and let one already sent be answered while the enclave can + /// still answer it. On a T6000 a request sent while devices were + /// suspended was never answered, and the enclave did not read its mailbox + /// again after the resume. + pub(crate) fn rng_sleep_prepare(&self) { + if !self.control.lock().entropy_sleep() { + return; + } + let mut waited: u32 = 0; + while self.control.lock().entropy_busy() { + if waited >= RNG_SLEEP_DRAIN_MS { + dev_warn!( + self.dev, + "hwrng: a request was still in flight after {} ms; suspending anyway\n", + RNG_SLEEP_DRAIN_MS + ); + return; + } + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(10)); + waited += 10; + } + } + + /// The system has resumed, or the suspend was aborted. + pub(crate) fn rng_sleep_finished(&self) { + self.control.lock().entropy_wake(); + } + fn control_survey(&self) { for param in [0x00u8, 0x01, 0xff] { let _ = self.control_request(&proto::op_nop(param)); @@ -1235,8 +1268,8 @@ impl SepData { this.rng_survey_attempts.store(attempt, Relaxed); match this.survey_hwrng() { Ok(()) => dev_info!(this.dev, "hwrng: registered after ready-phase survey (attempt {})\n", attempt), - // Holding back after a request that went unanswered: this - // attempt did not ask the enclave anything. + // Asleep, or holding back after a request that went unanswered: + // this attempt did not ask the enclave anything. Err(e) if e == EAGAIN => { this.rng_survey_attempts.store(attempt - 1, Relaxed); Self::arm_settle(this); @@ -1847,8 +1880,8 @@ impl SepData { // must return a negative errno, not 0: a blocking reader would spin on 0 let e = failure.unwrap_or(EIO); - // Holding back after a request that went unanswered. Nothing failed - // in this read; the core tries again later. + // Asleep, or holding back after a request that went unanswered. + // Nothing failed in this read; the core tries again later. if e == EAGAIN { return e.to_errno(); } From 4e5cd66956d82db3aebc14d0a7a55f7178cbea36 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 17:19:43 +1100 Subject: [PATCH 1526/2680] usb: xhci: plat: Call the interrupt handler through the HC driver in tests The suspend-to-idle tests call xhci_irq() directly, but that function is not exported. With the xHCI core built in and the tests built as a module, as with KUNIT_ALL_TESTS=m on a distribution config, modpost fails: ERROR: modpost: "xhci_irq" [drivers/usb/host/xhci-plat-test.ko] undefined! The tests already set up the generic xHCI hc_driver with xhci_init_driver() and call its hub_control and hub_status_data methods. Call the interrupt handler through its irq method as well, which is xhci_irq() and is how the USB core invokes it from usb_hcd_irq(). Fixes: e9573a16e76c ("usb: xhci: Wake from suspend-to-idle only for wakeup events") (cherry picked from commit 0c44f4c9bd8571cf59f13d06bd0cfd0b873c193a ("usb: xhci: plat: Call the interrupt handler through the HC driver in tests")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/host/xhci-plat-test.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/usb/host/xhci-plat-test.c b/drivers/usb/host/xhci-plat-test.c index ca4814898714e..8583b63e5fe76 100644 --- a/drivers/usb/host/xhci-plat-test.c +++ b/drivers/usb/host/xhci-plat-test.c @@ -371,7 +371,7 @@ struct xhci_s2idle_test { /* The PORTSC read just before which the device below signals resume, from 1 */ unsigned int resume_at_read; unsigned int reads; - /* The generic xHCI root hub methods, as the USB core calls them */ + /* The generic xHCI root hub and interrupt methods, as the USB core calls them */ struct hc_driver driver; struct device *dev; char name[32]; @@ -649,7 +649,7 @@ static irqreturn_t xhci_s2idle_test_irq(struct xhci_s2idle_test *t) irqreturn_t ret; local_irq_disable(); - ret = xhci_irq(t->hcd); + ret = t->driver.irq(t->hcd); local_irq_enable(); return ret; } From 538d9a1d3f3b34401740eb146e0855d413dd2a7f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 18:29:41 +1100 Subject: [PATCH 1527/2680] soc: apple: sep: Take an SBIO reply header only once it has landed The receive work waits for an SBIO reply header to leave the poison fill, but takes it as soon as any one byte has changed. The mailbox notification is not ordered against the enclave's DMA, so the header can be only partly visible. With chunk still poison, the payload read fails its bounds check; with version still poison, the reassembly rejects the reply. Both end the exchange with the NotAnswered sentinel, so the log says that no message arrived at all when one did. An M1 Pro logged exactly that for INIT_SBIO_COMMUNICATION about 3 ms after powering the sensor. That is when its reply normally arrives, and far short of the 5 s SBIO timeout, which returns ETIMEDOUT instead. Wait until every header word that taking the message reads has left the fill, within the existing 200 x 1 ms poll budget: all but flags for a data chunk, and err for an error report. No valid reply holds the fill in those words. A grant or a notification reads no header, so it is no longer held up by the buffer. Before the first reply of a boot, a grant used to wait 200 ms and then end the exchange that it was granting. An error report has no payload, so none is read. Report a reply whose header never fully lands, or fails a check, as Malformed. A header that stays entirely poison is still reported as BufferNeverWritten, and NotAnswered is kept for host-side failures. This is host-side only. No message to the enclave is added, removed or reordered, and no xART path is touched. KUnit tests (apple_sep_transfer) cover a torn header, a header that lands late, one that never lands, one left partly written, poison in flags, an error report, a grant and a reply failing its checks. Fixes: bac0846c9087 ("soc: apple: add Apple SEP driver (Touch ID and key sealing)") (cherry picked from commit d70b99b2ed147b35eca50b85cdea4c09aade0441 ("soc: apple: sep: Take an SBIO reply header only once it has landed")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 54 +++--- drivers/soc/apple/sep.rs | 20 +++ drivers/soc/apple/transfer.rs | 305 +++++++++++++++++++++++++++++++++- 3 files changed, 355 insertions(+), 24 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 92d7fb73f65b8..d0a731e5aa11b 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2664,31 +2664,31 @@ impl SepData { let generation = self.sbio_rx.lock().generation(); // notification and the enclave's DMA are not ordered; buffer may still be poison - if !self.ool_await_written(&self.ool_sbio, 0, transfer::HEADER_LEN) { - self.sbio_fail_unwritten(); - return; - } - - let header = match self.ool_read(&self.ool_sbio, 0, transfer::HEADER_LEN) { - Ok(h) => h, - Err(_) => { - self.sbio_fail(); - return; - } - }; - let packet = match transfer::Packet::decode(&header) { + let packet = match transfer::take_header( + &SbioHeader(self), + marker, + OOL_POISON_OUTBOUND, + OOL_WRITE_POLL_ATTEMPTS, + ) { Ok(p) => p, - Err(_) => { - self.sbio_fail(); + Err(status) => { + self.sbio_fail_with(status); return; } }; - let chunk = packet.chunk as usize; + let chunk = if transfer::carries_payload(marker) { + packet.chunk as usize + } else { + 0 + }; let payload = if chunk == 0 { KVec::new() + } else if chunk > OOL_SIZE_SBIO - transfer::HEADER_LEN { + self.sbio_fail_with(transfer::DeviceStatus::Malformed); + return; } else if !self.ool_await_written(&self.ool_sbio, transfer::HEADER_LEN, chunk) { - self.sbio_fail_unwritten(); + self.sbio_fail_with(transfer::DeviceStatus::BufferNeverWritten); return; } else { match self.ool_read(&self.ool_sbio, transfer::HEADER_LEN, chunk) { @@ -2741,10 +2741,8 @@ impl SepData { self.sbio_wq.notify_all(); } - fn sbio_fail_unwritten(&self) { - self.sbio_rx - .lock() - .abort_with(transfer::DeviceStatus::BufferNeverWritten); + fn sbio_fail_with(&self, status: transfer::DeviceStatus) { + self.sbio_rx.lock().abort_with(status); self.sbio_wq.notify_all(); } @@ -3460,6 +3458,20 @@ impl SepData { } } +/// The SBIO reply header, as [`transfer::take_header`] samples it. +struct SbioHeader<'a>(&'a SepData); + +impl transfer::HeaderSlot for SbioHeader<'_> { + fn sample(&self, header: &mut [u8; transfer::HEADER_LEN]) -> bool { + self.0.ool_peek(&self.0.ool_sbio, 0, header) + } + + fn pause(&self) { + let poll = kernel::time::Delta::from_millis(i64::from(OOL_WRITE_POLL_MS)); + kernel::time::delay::fsleep(poll); + } +} + pub(crate) struct SbioOp { opcode: u16, payload: [u8; SBIO_MAX_PAYLOAD], diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 662078489c205..b93feb4dd5d9c 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -169,6 +169,10 @@ const OOL_WRITE_POLL_ATTEMPTS: u32 = 200; const OOL_POISON_INBOUND: u8 = 0xA5; const OOL_POISON_OUTBOUND: u8 = 0x5A; +// transfer::take_header() tells a landed SBIO header from the fill by its +// words, which a valid reply cannot hold as four equal nonzero bytes. +static_assert!(OOL_POISON_OUTBOUND != 0); + const PROTECTED_DATA_AVAILABLE: bool = true; const RNG_MAX_WORDS_PER_READ: usize = 16; @@ -1527,6 +1531,22 @@ impl SepData { Ok(payload) } + /// Copies `out.len()` bytes at `off` of the outbound buffer into `out`; + /// `false` once the buffers are gone or the range is outside them. + fn ool_peek(&self, slot: &Mutex>, off: usize, out: &mut [u8]) -> bool { + let guard = slot.lock(); + let Some(buffers) = guard.as_ref() else { + return false; + }; + if off + out.len() > buffers.allocated { + return false; + } + // SAFETY: the region is within the allocation as checked above, and + // the driver is the only host-side accessor. + out.copy_from_slice(unsafe { &buffers.outbound.as_ref()[off..off + out.len()] }); + true + } + fn ool_await_written(&self, slot: &Mutex>, off: usize, len: usize) -> bool { for _ in 0..OOL_WRITE_POLL_ATTEMPTS { { diff --git a/drivers/soc/apple/transfer.rs b/drivers/soc/apple/transfer.rs index a4f5f0df3e308..d82ca1a394eb1 100644 --- a/drivers/soc/apple/transfer.rs +++ b/drivers/soc/apple/transfer.rs @@ -20,6 +20,8 @@ pub(crate) enum DeviceStatus { ReportedWithoutStatus, NotAnswered, BufferNeverWritten, + /// A reply arrived, but its header never fully landed or failed a check. + Malformed, } impl DeviceStatus { @@ -28,7 +30,8 @@ impl DeviceStatus { DeviceStatus::Answered(err) => Some(*err), DeviceStatus::ReportedWithoutStatus | DeviceStatus::NotAnswered - | DeviceStatus::BufferNeverWritten => None, + | DeviceStatus::BufferNeverWritten + | DeviceStatus::Malformed => None, } } @@ -53,6 +56,10 @@ impl kernel::fmt::Display for DeviceStatus { f, "NONE — a payload notification arrived but the outbound buffer was never written (a race with the enclave's DMA, not a silent enclave)" ), + DeviceStatus::Malformed => write!( + f, + "NONE — a reply arrived but its header was left partly written or failed a check (host sentinel, not an enclave value)" + ), } } } @@ -80,7 +87,102 @@ fn le32(buf: &[u8], word: usize) -> u32 { u32::from_le_bytes([buf[o], buf[o + 1], buf[o + 2], buf[o + 3]]) } +// The header words, numbered as in `Packet::decode`, that taking a message +// reads. A data chunk reads all but flags; an error report reads only err. +// None of them can hold a fill of four equal nonzero bytes in a valid reply: +// version is VERSION, opcode is an echoed 16-bit opcode, and total, offset +// and chunk are at most MAX_TRANSACTION. err is the enclave's status. A +// poison err taken as landed would be reported as a refusal the enclave did +// not make. If the enclave ever answered with that value, waiting costs the +// poll budget and the exchange fails either way. +const DATA_WORDS: [usize; 6] = [0, 1, 2, 4, 5, 6]; +const ERROR_WORDS: [usize; 1] = [4]; + +static_assert!(MAX_TRANSACTION < 0x0101_0101); + +fn header_words_read(marker: u8) -> &'static [usize] { + match marker { + MARKER_FIRST | MARKER_NEXT => &DATA_WORDS, + MARKER_ERROR => &ERROR_WORDS, + // A grant or a notification: `Reassembly::on_chunk` reads no header. + _ => &[], + } +} + +/// Whether a message with `marker` has a payload after its header. +pub(crate) fn carries_payload(marker: u8) -> bool { + matches!(marker, MARKER_FIRST | MARKER_NEXT) +} + +fn header_landed(header: &[u8; HEADER_LEN], words: &[usize], poison: u8) -> bool { + let poison = u32::from_le_bytes([poison; 4]); + words.iter().all(|&word| le32(header, word) != poison) +} + +/// The host's view of the header at the start of an endpoint's outbound +/// buffer, which the enclave fills by DMA. +pub(crate) trait HeaderSlot { + /// Copies the header out; `false` once the buffers are gone. + fn sample(&self, header: &mut [u8; HEADER_LEN]) -> bool; + /// Waits before the next sample. + fn pause(&self); +} + +/// Takes the header of a message with `marker` from `slot`, sampling it up to +/// `attempts` times until every word that taking the message reads has left +/// the `poison` fill, a nonzero byte. The mailbox notification is not ordered +/// against the enclave's DMA, so the header can be unwritten, or only partly +/// written, when the message arrives: one changed byte is not enough. A +/// message whose handling reads no header is taken without looking at the +/// buffer. +pub(crate) fn take_header( + slot: &impl HeaderSlot, + marker: u8, + poison: u8, + attempts: u32, +) -> Result { + let words = header_words_read(marker); + if words.is_empty() { + return Ok(Packet::NONE); + } + + let mut header = [poison; HEADER_LEN]; + let mut landed = false; + for attempt in 0..attempts { + if attempt > 0 { + slot.pause(); + } + if !slot.sample(&mut header) { + return Err(DeviceStatus::NotAnswered); + } + if header_landed(&header, words, poison) { + landed = true; + break; + } + } + + if !landed { + return Err(if header.iter().all(|&b| b == poison) { + DeviceStatus::BufferNeverWritten + } else { + DeviceStatus::Malformed + }); + } + Packet::decode(&header).map_err(|_| DeviceStatus::Malformed) +} + impl Packet { + /// Stands in for the header of a message whose handling reads none of it. + pub(crate) const NONE: Packet = Packet { + version: 0, + total: 0, + offset: 0, + flags: 0, + err: 0, + opcode: 0, + chunk: 0, + }; + pub(crate) fn decode(buf: &[u8]) -> Result { if buf.len() < HEADER_LEN { return Err(EINVAL); @@ -251,7 +353,7 @@ impl Reassembly { } fn fail(&mut self) { - self.abort(); + self.abort_with(DeviceStatus::Malformed); } /// Takes one chunk of a reply. `generation` is [`Self::generation`] as @@ -368,7 +470,7 @@ impl Reassembly { .extend_from_slice(payload, GFP_KERNEL) .is_err() { - self.fail(); + self.abort(); return Progress::Failed; } @@ -399,8 +501,11 @@ impl Reassembly { #[kernel::macros::kunit_tests(apple_sep_transfer)] mod tests { use super::*; + use core::cell::Cell; const OPCODE: u32 = 0x48; + const POISON: u8 = 0x5A; + const ATTEMPTS: u32 = 200; fn reply(err: u32) -> Packet { Packet { @@ -414,6 +519,79 @@ mod tests { } } + fn data(total: u32, offset: u32, chunk: u32) -> Packet { + Packet { + total, + offset, + chunk, + ..reply(0) + } + } + + /// An outbound buffer header that the enclave's DMA fills word by word, + /// from word 0 up, while the host pauses between samples. + struct Dma { + header: Cell<[u8; HEADER_LEN]>, + incoming: Cell<[u8; HEADER_LEN]>, + landed: Cell, + per_pause: Cell, + samples: Cell, + } + + impl Dma { + fn new() -> Dma { + Dma { + header: Cell::new([POISON; HEADER_LEN]), + incoming: Cell::new([POISON; HEADER_LEN]), + landed: Cell::new(0), + per_pause: Cell::new(0), + samples: Cell::new(0), + } + } + + /// The enclave starts writing `packet`: `now` words are visible at + /// once, and `per_pause` more after each pause. + fn deliver(&self, packet: &Packet, now: usize, per_pause: usize) { + let mut bytes = [0u8; HEADER_LEN]; + let _ = packet.encode(&mut bytes); + self.incoming.set(bytes); + self.landed.set(0); + self.per_pause.set(per_pause); + self.land(now); + } + + fn land(&self, words: usize) { + let landed = core::cmp::min(self.landed.get() + words, HEADER_WORDS); + let mut header = self.header.get(); + header[..landed * 4].copy_from_slice(&self.incoming.get()[..landed * 4]); + self.header.set(header); + self.landed.set(landed); + } + + /// One word becomes visible on its own. + fn show(&self, word: usize, value: u32) { + let mut header = self.header.get(); + header[word * 4..word * 4 + 4].copy_from_slice(&value.to_le_bytes()); + self.header.set(header); + } + } + + impl HeaderSlot for Dma { + fn sample(&self, header: &mut [u8; HEADER_LEN]) -> bool { + self.samples.set(self.samples.get() + 1); + *header = self.header.get(); + true + } + + fn pause(&self) { + self.land(self.per_pause.get()); + } + } + + fn take(dma: &Dma, marker: u8) -> Result { + take_header(dma, marker, POISON, ATTEMPTS) + } + #[test] fn a_reply_to_the_current_exchange_completes_it() { let mut rx = Reassembly::new(); @@ -451,4 +629,125 @@ mod tests { Progress::Complete )); } + + #[test] + fn a_torn_header_is_taken_once_it_has_landed() { + let dma = Dma::new(); + // version..flags are visible; err, opcode and chunk are still poison. + dma.deliver(&data(8, 0, 8), 4, 1); + let taken = take(&dma, MARKER_FIRST); + assert!(taken.is_ok()); + let Ok(packet) = taken else { + return; + }; + assert_eq!(packet.chunk, 8); + assert_eq!(packet.err, 0); + assert_eq!(packet.opcode, OPCODE); + + let mut rx = Reassembly::new(); + assert!(rx.begin(OPCODE).is_ok()); + let generation = rx.generation(); + assert!(matches!( + rx.on_chunk(generation, MARKER_FIRST, &packet, &[0u8; 8]), + Progress::Complete + )); + assert!(rx.take_done().is_some_and(|done| done.status.is_ok())); + } + + #[test] + fn a_late_header_is_taken_whole() { + let dma = Dma::new(); + // Nothing is visible when the message arrives; then a word per pause. + dma.deliver(&data(8, 0, 8), 0, 1); + let taken = take(&dma, MARKER_FIRST); + assert!(taken.is_ok()); + let Ok(packet) = taken else { + return; + }; + assert_eq!(packet.total, 8); + assert_eq!(packet.chunk, 8); + assert_eq!(packet.opcode, OPCODE); + assert_eq!(dma.samples.get(), HEADER_WORDS as u32 + 1); + } + + #[test] + fn a_header_that_never_lands_is_reported_as_never_written() { + let dma = Dma::new(); + assert!(matches!( + take(&dma, MARKER_FIRST), + Err(DeviceStatus::BufferNeverWritten) + )); + assert_eq!(dma.samples.get(), ATTEMPTS); + } + + #[test] + fn a_header_left_partly_written_is_reported_as_malformed() { + let dma = Dma::new(); + dma.deliver(&data(8, 0, 8), 4, 0); + assert!(matches!( + take(&dma, MARKER_FIRST), + Err(DeviceStatus::Malformed) + )); + } + + #[test] + fn flags_may_hold_the_poison() { + let dma = Dma::new(); + let packet = Packet { + flags: u32::from_le_bytes([POISON; 4]), + ..data(8, 0, 8) + }; + dma.deliver(&packet, HEADER_WORDS, 0); + assert!(take(&dma, MARKER_FIRST).is_ok()); + assert_eq!(dma.samples.get(), 1); + } + + #[test] + fn an_error_report_reads_only_its_status() { + let dma = Dma::new(); + dma.show(4, 0x16); + let taken = take(&dma, MARKER_ERROR); + assert!(taken.is_ok()); + let Ok(packet) = taken else { + return; + }; + assert_eq!(packet.err, 0x16); + + let mut rx = Reassembly::new(); + assert!(rx.begin(OPCODE).is_ok()); + let generation = rx.generation(); + assert!(matches!( + rx.on_chunk(generation, MARKER_ERROR, &packet, &[]), + Progress::Complete + )); + assert!(rx + .take_done() + .is_some_and(|done| done.status.answered() == Some(0x16))); + } + + #[test] + fn a_grant_or_a_notification_reads_no_header() { + let dma = Dma::new(); + assert!(take(&dma, MARKER_REQUEST).is_ok()); + assert!(take(&dma, MARKER_FIRST - 1).is_ok()); + assert_eq!(dma.samples.get(), 0); + } + + #[test] + fn a_reply_failing_a_check_is_reported_as_malformed() { + let mut rx = Reassembly::new(); + assert!(rx.begin(OPCODE).is_ok()); + let generation = rx.generation(); + let bad = Packet { + version: VERSION + 1, + ..reply(0) + }; + assert!(matches!( + rx.on_chunk(generation, MARKER_FIRST, &bad, &[]), + Progress::Failed + )); + assert!(rx + .take_done() + .is_some_and(|done| matches!(done.status, DeviceStatus::Malformed))); + } } From 2510adbce55f7ddbcb425a784b4a994c4fd08d34 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 17:19:44 +1100 Subject: [PATCH 1528/2680] usb: typec: tipd: cd321x: Add a module description to the PM tests modpost warns when the CD321x PM tests are built as a module: WARNING: modpost: missing MODULE_DESCRIPTION() in drivers/usb/typec/tipd/cd321x-pm-test.o Add one, as the other tipd test modules have. Fixes: c2514e492d2e ("thunderbolt: restore M1 tunneled PCIe state and recover late link loss") (cherry picked from commit 67a37af9b8c7fabb66b2ab3e54caa23db6d59aed ("usb: typec: tipd: cd321x: Add a module description to the PM tests")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/cd321x-pm-test.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/usb/typec/tipd/cd321x-pm-test.c b/drivers/usb/typec/tipd/cd321x-pm-test.c index 977df770dcff5..08adc096cbb7c 100644 --- a/drivers/usb/typec/tipd/cd321x-pm-test.c +++ b/drivers/usb/typec/tipd/cd321x-pm-test.c @@ -569,4 +569,5 @@ static struct kunit_suite cd321x_pm_suite = { }; kunit_test_suite(cd321x_pm_suite); +MODULE_DESCRIPTION("KUnit tests for CD321x system sleep and cable-state revalidation"); MODULE_LICENSE("GPL"); From 1f724d4392028e951e5ad04f395f33e4f0795997 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 18:29:41 +1100 Subject: [PATCH 1529/2680] soc: apple: sep: Poison an SBIO reply header again once it is read Waiting for the reply header to leave the poison fill only works while the header region holds the fill, and it does so only until the first reply of a boot. Every later header lands on top of the one before it. If its notification overtakes the DMA, the receive work finds the previous header complete and takes it for the new one. A reply to the same opcode then completes the exchange with the previous answer, and a continuation chunk fails its offset check. Fill the header with the poison again as soon as it has been looked at, before the message is processed. That is before any continuation is sent, and before the requester wakes to send its next request. The receive work is the only reader of the header, so this cannot race a read. A grant or a notification does not read the header and leaves it alone. This is host-side only. Nothing sent to the enclave changes. KUnit tests (apple_sep_transfer) cover the next exchange's reply, a continuation chunk and the refill itself. Fixes: bac0846c9087 ("soc: apple: add Apple SEP driver (Touch ID and key sealing)") (cherry picked from commit f66113f26bcbafd5ebb00bf5f619406c6d27788c ("soc: apple: sep: Poison an SBIO reply header again once it is read")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 5 +++ drivers/soc/apple/sep.rs | 18 ++++++++ drivers/soc/apple/transfer.rs | 82 +++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index d0a731e5aa11b..7a9afe8fbebf2 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -3466,6 +3466,11 @@ impl transfer::HeaderSlot for SbioHeader<'_> { self.0.ool_peek(&self.0.ool_sbio, 0, header) } + fn repoison(&self) { + self.0 + .ool_repoison(&self.0.ool_sbio, 0, transfer::HEADER_LEN); + } + fn pause(&self) { let poll = kernel::time::Delta::from_millis(i64::from(OOL_WRITE_POLL_MS)); kernel::time::delay::fsleep(poll); diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index b93feb4dd5d9c..69d3d52ffa5b4 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -1547,6 +1547,24 @@ impl SepData { true } + /// Fills `len` bytes at `off` of the outbound buffer with the poison + /// again, once the message they held has been read. + fn ool_repoison(&self, slot: &Mutex>, off: usize, len: usize) { + let guard = slot.lock(); + let Some(buffers) = guard.as_ref() else { + return; + }; + if off + len > buffers.allocated { + return; + } + // SAFETY: the region is within the allocation as checked above, and + // the driver is the only host-side accessor. The enclave writes the + // region only to deliver a message, and the one it held has been + // looked at. If part of it was still landing, that part may land over + // the fill afterwards, as a header that has not landed yet does. + unsafe { buffers.outbound.as_mut()[off..off + len].fill(OOL_POISON_OUTBOUND) }; + } + fn ool_await_written(&self, slot: &Mutex>, off: usize, len: usize) -> bool { for _ in 0..OOL_WRITE_POLL_ATTEMPTS { { diff --git a/drivers/soc/apple/transfer.rs b/drivers/soc/apple/transfer.rs index d82ca1a394eb1..3323e7e2cfc09 100644 --- a/drivers/soc/apple/transfer.rs +++ b/drivers/soc/apple/transfer.rs @@ -124,6 +124,8 @@ fn header_landed(header: &[u8; HEADER_LEN], words: &[usize], poison: u8) -> bool pub(crate) trait HeaderSlot { /// Copies the header out; `false` once the buffers are gone. fn sample(&self, header: &mut [u8; HEADER_LEN]) -> bool; + /// Fills the header with the poison again. + fn repoison(&self); /// Waits before the next sample. fn pause(&self); } @@ -135,6 +137,9 @@ pub(crate) trait HeaderSlot { /// written, when the message arrives: one changed byte is not enough. A /// message whose handling reads no header is taken without looking at the /// buffer. +/// +/// Once looked at, the header is poisoned again whatever the outcome, so the +/// next message cannot take it for its own. pub(crate) fn take_header( slot: &impl HeaderSlot, marker: u8, @@ -160,6 +165,7 @@ pub(crate) fn take_header( break; } } + slot.repoison(); if !landed { return Err(if header.iter().all(|&b| b == poison) { @@ -574,6 +580,10 @@ mod tests { header[word * 4..word * 4 + 4].copy_from_slice(&value.to_le_bytes()); self.header.set(header); } + + fn poisoned(&self) -> bool { + self.header.get().iter().all(|&b| b == POISON) + } } impl HeaderSlot for Dma { @@ -583,6 +593,10 @@ mod tests { true } + fn repoison(&self) { + self.header.set([POISON; HEADER_LEN]); + } + fn pause(&self) { self.land(self.per_pause.get()); } @@ -750,4 +764,72 @@ mod tests { .take_done() .is_some_and(|done| matches!(done.status, DeviceStatus::Malformed))); } + + #[test] + fn the_previous_replys_header_is_not_taken_for_the_next() { + let dma = Dma::new(); + dma.deliver(&data(8, 0, 8), HEADER_WORDS, 0); + assert!(take(&dma, MARKER_FIRST).is_ok_and(|p| p.chunk == 8)); + + // The next exchange's message arrives before any of its header. + dma.deliver(&data(4, 0, 4), 0, HEADER_WORDS); + let taken = take(&dma, MARKER_FIRST); + assert!(taken.is_ok()); + let Ok(packet) = taken else { + return; + }; + assert_eq!(packet.chunk, 4); + assert_eq!(packet.total, 4); + } + + #[test] + fn a_continuation_does_not_take_the_first_chunks_header() { + let dma = Dma::new(); + let mut rx = Reassembly::new(); + assert!(rx.begin(OPCODE).is_ok()); + let generation = rx.generation(); + + dma.deliver(&data(16, 0, 8), HEADER_WORDS, 0); + let taken = take(&dma, MARKER_FIRST); + assert!(taken.is_ok()); + let Ok(first) = taken else { + return; + }; + assert!(matches!( + rx.on_chunk(generation, MARKER_FIRST, &first, &[0u8; 8]), + Progress::NeedMore(_) + )); + + // The second chunk's message arrives before its header lands. + dma.deliver(&data(16, 8, 8), 0, 2); + let taken = take(&dma, MARKER_NEXT); + assert!(taken.is_ok()); + let Ok(next) = taken else { + return; + }; + assert!(matches!( + rx.on_chunk(generation, MARKER_NEXT, &next, &[0u8; 8]), + Progress::Complete + )); + assert!(rx + .take_done() + .is_some_and(|done| done.status.is_ok() && done.payload.len() == 16)); + } + + #[test] + fn a_header_is_poisoned_again_once_looked_at() { + let dma = Dma::new(); + dma.deliver(&data(8, 0, 8), HEADER_WORDS, 0); + assert!(take(&dma, MARKER_FIRST).is_ok()); + assert!(dma.poisoned()); + + dma.deliver(&data(8, 0, 8), 4, 0); + assert!(take(&dma, MARKER_FIRST).is_err()); + assert!(dma.poisoned()); + + // A grant leaves the buffer alone. + dma.show(4, 0x16); + assert!(take(&dma, MARKER_REQUEST).is_ok()); + assert!(!dma.poisoned()); + } } From 22fba3c88c62ca69dddba35273214221a9b1eb19 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 17:21:46 +1100 Subject: [PATCH 1530/2680] drm/apple: Add module descriptions to the KUnit test modules modpost warns about four of the Apple display KUnit test modules when DRM_APPLE_KUNIT_TEST is built as a module, as KUNIT_ALL_TESTS=m does on a distribution config: WARNING: modpost: missing MODULE_DESCRIPTION() in drivers/gpu/drm/apple/afk_test.o WARNING: modpost: missing MODULE_DESCRIPTION() in drivers/gpu/drm/apple/dcp_backlight_test.o WARNING: modpost: missing MODULE_DESCRIPTION() in drivers/gpu/drm/apple/afk-service-test.o WARNING: modpost: missing MODULE_DESCRIPTION() in drivers/gpu/drm/apple/iomfb-state-test.o Add one to each, as dcp-announce-test has. Fixes: 1ff8bb4e3bd8 ("drm: apple: afk: derive the ring header stride from INITRB") Fixes: 0da3b48fc8f7 ("drm: apple: test H17P backlight takeover and DPMS policy") Fixes: 5837706b10e3 ("drm/apple: pin DP service owners before recycling retired AFK slots") Fixes: fb40feb283b6 ("drm/apple: preserve mode invalidation across hotplug and modeset") (cherry picked from commit ee77f893e79f0ee43e11e685dd81e38832bbb0b4 ("drm/apple: Add module descriptions to the KUnit test modules")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk-service-test.c | 1 + drivers/gpu/drm/apple/afk_test.c | 1 + drivers/gpu/drm/apple/dcp_backlight_test.c | 1 + drivers/gpu/drm/apple/iomfb-state-test.c | 1 + 4 files changed, 4 insertions(+) diff --git a/drivers/gpu/drm/apple/afk-service-test.c b/drivers/gpu/drm/apple/afk-service-test.c index a64064758a906..358d0edf34519 100644 --- a/drivers/gpu/drm/apple/afk-service-test.c +++ b/drivers/gpu/drm/apple/afk-service-test.c @@ -271,5 +271,6 @@ static struct kunit_suite afk_service_suite = { }; kunit_test_suite(afk_service_suite); +MODULE_DESCRIPTION("KUnit tests for the Apple AFK EPIC service lifetime"); MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/afk_test.c b/drivers/gpu/drm/apple/afk_test.c index 0621eebe914df..3bbed1096144d 100644 --- a/drivers/gpu/drm/apple/afk_test.c +++ b/drivers/gpu/drm/apple/afk_test.c @@ -113,4 +113,5 @@ static struct kunit_suite afk_ring_suite = { kunit_test_suite(afk_ring_suite); +MODULE_DESCRIPTION("KUnit tests for the Apple AFK ring buffers"); MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index a6fb45958b168..70379571a5dd3 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -528,4 +528,5 @@ static struct kunit_suite backlight_suite = { kunit_test_suite(backlight_suite); +MODULE_DESCRIPTION("KUnit tests for the Apple DCP backlight"); MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/iomfb-state-test.c b/drivers/gpu/drm/apple/iomfb-state-test.c index d751625a94058..97ba9924d454f 100644 --- a/drivers/gpu/drm/apple/iomfb-state-test.c +++ b/drivers/gpu/drm/apple/iomfb-state-test.c @@ -124,4 +124,5 @@ static struct kunit_suite dcp_mode_suite = { }; kunit_test_suite(dcp_mode_suite); +MODULE_DESCRIPTION("KUnit tests for the Apple DCP mode and connection state"); MODULE_LICENSE("Dual MIT/GPL"); From 80320a91b369da21dcbb9baaf822fe7374730f6b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 18:10:53 +1100 Subject: [PATCH 1531/2680] usb: typec: tipd: cd321x: Don't race the resume read in the IRQ test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cd321x_ack_retry_across_sleep_test() asserted that the resume read was pending right after tipd_complete(). The driver queues that read with no delay, so on an SMP test machine a worker can already be running it, with the pending bit clear, and the test fails although the driver behaved correctly. With two CPUs it failed in 1 of 20 runs. Wait for the resume read with flush_work() instead. Unlike flush_delayed_work(), it does not run early the retry that the read schedules, so the loop that follows still sees exactly one retry pending after each read. Check that the resume read tried to read the events. The other pending and timer checks in the test look either at work that nothing can queue at that point, or at a retry scheduled at least 500 ms ahead. Not queueing the resume read, or giving up the retry after the cable-state budget, still makes the test fail, and now so does queueing the resume read late. [ryan: adapted to 7.2.10 — the test takes the CD321x data from tipd_hw_variants[TIPD_VARIANT_CD321X], and the retry loop is this version's again: it replaces the 7.2.10-only fix of the same race in commit 11e0ce667c92 ("usb: typec: tipd: cd321x: Don't race the resume read in the sleep retry test"), which flushed the work before checking that it was pending] Fixes: 7e97f2f5ee45 ("usb: typec: tipd: cd321x: Never leave interrupt events unacknowledged") (cherry picked from commit 7ec5a7211cda47308f4eb0c6eda4ee240f17a801 ("usb: typec: tipd: cd321x: Don't race the resume read in the IRQ test")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/cd321x-irq-test.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/usb/typec/tipd/cd321x-irq-test.c b/drivers/usb/typec/tipd/cd321x-irq-test.c index 5062d45616773..5eb17581527ef 100644 --- a/drivers/usb/typec/tipd/cd321x-irq-test.c +++ b/drivers/usb/typec/tipd/cd321x-irq-test.c @@ -314,7 +314,7 @@ static void cd321x_ack_retry_across_sleep_test(struct kunit *test) { struct cd321x_test *t = cd321x_test_port(test); struct device *dev = t->cd.tps.dev; - unsigned int i; + unsigned int i, left; cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); t->ctrl.failing_event_reads = UINT_MAX; @@ -324,11 +324,18 @@ static void cd321x_ack_retry_across_sleep_test(struct kunit *test) /* Sleep cancels the retry, and the resume read takes it over. */ KUNIT_ASSERT_EQ(test, tipd_prepare(dev), 0); KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); + left = t->ctrl.failing_event_reads; tipd_complete(dev); + /* + * The resume read is queued without delay, so on SMP a worker may be + * running it already. Wait for it, without running the retry it + * schedules early, and check that it tried to read the events. + */ + flush_work(&t->cd.resume_work.work); + KUNIT_EXPECT_EQ(test, t->ctrl.failing_event_reads, left - 1); for (i = 0; i < 2 * CD321X_RESUME_ATTEMPTS; i++) { - /* Queued first without delay: a worker may already be running it. */ - flush_delayed_work(&t->cd.resume_work); KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); + flush_delayed_work(&t->cd.resume_work); } t->ctrl.failing_event_reads = 0; From 99b5c643b40ad5bb8e3a6c5ea56af4778075a241 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 18:59:35 +1100 Subject: [PATCH 1532/2680] soc: apple: sep: Poison the SBIO reply header before each host send A grant leaves the outbound header alone, because the reply may already be landing behind it. The enclave's grant may itself have written a header there, as the host's own grant writes a request-next header in the inbound buffer, and nothing reads or refills it. After a multi-chunk send, such as a Touch ID capture relay, the reply could take that header once its own has not landed yet: a data reply would fail its offset check as malformed, and an error report would read err 0 and hide the enclave's status. Poison the header again just before each chunk of a request is sent, the first and every later one, through a new send_for_reply(). The enclave is waiting for the host there, so no header can be landing. A continuation is sent straight after its chunk's header was taken and refilled, so it needs no change. A grant still leaves the buffer alone. Add KUnit cases for a grant's header followed by a data reply and by an error report, and for the refill coming before the send. The first two fail with the refill taken out of send_for_reply(). Fixes: bac0846c9087 ("soc: apple: add Apple SEP driver (Touch ID and key sealing)") (cherry picked from commit d6b7ba26ef7fb1cde5a17ea4702e1ae690d6a48e ("soc: apple: sep: Poison the SBIO reply header before each host send")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 4 +- drivers/soc/apple/transfer.rs | 83 ++++++++++++++++++++++++++++++++++- 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 7a9afe8fbebf2..e3f6b613cc605 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2859,7 +2859,9 @@ impl SepData { crate::sbio::encode_sbio_next(opcode, seq) } .ok_or(EPERM)?; - self.send(msg)?; + // The last grant's header may sit in the outbound buffer; the + // reply to this chunk must not take it. + transfer::send_for_reply(&SbioHeader(self), || self.send(msg))?; offset += chunk; first = false; diff --git a/drivers/soc/apple/transfer.rs b/drivers/soc/apple/transfer.rs index 3323e7e2cfc09..082baf3ff1bf5 100644 --- a/drivers/soc/apple/transfer.rs +++ b/drivers/soc/apple/transfer.rs @@ -139,7 +139,9 @@ pub(crate) trait HeaderSlot { /// buffer. /// /// Once looked at, the header is poisoned again whatever the outcome, so the -/// next message cannot take it for its own. +/// next message cannot take it for its own. A message whose header is not +/// looked at, such as a grant, leaves the buffer alone: the reply may already +/// be landing behind it. [`send_for_reply`] poisons that header instead. pub(crate) fn take_header( slot: &impl HeaderSlot, marker: u8, @@ -177,6 +179,19 @@ pub(crate) fn take_header( Packet::decode(&header).map_err(|_| DeviceStatus::Malformed) } +/// Poisons the header in `slot` again, then calls `send` for a host message +/// that asks the enclave for its next one: a request, or the next chunk of +/// one. The enclave may have written a header there that nothing read, such +/// as a grant's, and the next reply must not take it for its own. Until +/// `send`, the enclave is waiting for the host, so no header can be landing. +pub(crate) fn send_for_reply( + slot: &impl HeaderSlot, + send: impl FnOnce() -> Result, +) -> Result { + slot.repoison(); + send() +} + impl Packet { /// Stands in for the header of a message whose handling reads none of it. pub(crate) const NONE: Packet = Packet { @@ -827,9 +842,73 @@ mod tests { assert!(take(&dma, MARKER_FIRST).is_err()); assert!(dma.poisoned()); - // A grant leaves the buffer alone. + // A grant leaves the buffer alone; the host's next send poisons it. dma.show(4, 0x16); assert!(take(&dma, MARKER_REQUEST).is_ok()); assert!(!dma.poisoned()); + assert!(send_for_reply(&dma, || Ok(())).is_ok()); + assert!(dma.poisoned()); + } + + /// The enclave's grant writes a request-next header, as the host's own + /// grant does, and the host sends its next chunk. + fn grant_then_send(dma: &Dma) { + dma.deliver( + &Packet::request_next(OPCODE, 0x3fe4, 0x8000), + HEADER_WORDS, + 0, + ); + assert!(take(dma, MARKER_REQUEST).is_ok()); + assert!(send_for_reply(dma, || Ok(())).is_ok()); + } + + #[test] + fn a_grants_header_is_not_taken_for_the_reply() { + let dma = Dma::new(); + let mut rx = Reassembly::new(); + assert!(rx.begin(OPCODE).is_ok()); + rx.begin_send(OPCODE); + let generation = rx.generation(); + grant_then_send(&dma); + assert!(matches!( + rx.on_chunk(generation, MARKER_REQUEST, &Packet::NONE, &[]), + Progress::Grant + )); + rx.end_send(); + + // The reply arrives before any of its header. + dma.deliver(&data(8, 0, 8), 0, HEADER_WORDS); + let taken = take(&dma, MARKER_FIRST); + assert!(taken.is_ok()); + let Ok(packet) = taken else { + return; + }; + assert_eq!(packet.offset, 0); + assert_eq!(packet.chunk, 8); + assert!(matches!( + rx.on_chunk(generation, MARKER_FIRST, &packet, &[0u8; 8]), + Progress::Complete + )); + assert!(rx.take_done().is_some_and(|done| done.status.is_ok())); + } + + #[test] + fn a_grants_header_does_not_hide_an_error_status() { + let dma = Dma::new(); + grant_then_send(&dma); + dma.deliver(&reply(0x16), 0, HEADER_WORDS); + assert!(take(&dma, MARKER_ERROR).is_ok_and(|p| p.err == 0x16)); + } + + #[test] + fn the_header_is_poisoned_before_the_host_sends() { + let dma = Dma::new(); + dma.deliver(&data(8, 0, 8), HEADER_WORDS, 0); + let sent = send_for_reply(&dma, || { + assert!(dma.poisoned()); + Ok(7u32) + }); + assert!(sent.is_ok_and(|v| v == 7)); + assert!(send_for_reply(&dma, || -> Result<()> { Err(EIO) }).is_err()); } } From 8fa72d1a2ff92d778c6bb10a0b8f6117a6dd3b24 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 19:43:03 +1100 Subject: [PATCH 1533/2680] selftests: apple-dcp: Follow the fixed-hub SuperSpeed handover MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The host runner fails 31 of 55 cases on the fixed-hub SuperSpeed handover and the PIPE park retry, because their units no longer compile. The ATC mux body moved to atcphy_mux_switch(), with atcphy_mux_check(), atcphy_mux_park() and atcphy_usb3_port_ready() split out, and atcphy_mux_set_locked() takes a third argument. The dummy handoff and the USB3 set_mode now call atcphy_pipe_live() and atcphy_usb3_route(). The PHY state has usb3_port_off and park_owed, and the mux update powers the SuperSpeed root port off and on and queues retries. Extract the split helpers wherever the fixtures extract their callers, and add the new state to the fixtures' copies of the PHY. The lane adapter now runs the actual atcphy_mux_switch() under atcphy_mux_set_locked(), as the ordinary-port quiesce unit does. Both check that no SuperSpeed port comes up, since they take none down. The PM fixtures extract the actual port power steps, retries and atcphy_mux_change(). They model the root port's power: powering it off disconnects the device on it, and the PIPE mux may not be written while the held root's port is up. A power-on asserts that the PIPE is on the USB3 backend of a clean running host. Three behaviours change on purpose, and the assertions that check them follow the source: - The link on the port is disconnected rather than resumed. The selective-suspend link checks still run for sources that resume the link. With the handover, the same scenarios check instead that SAFE takes the port down and keeps it down, and that a USB3 mode brings it back. They also check that a port that fails to go down stops the change before any PHY write, that a port someone else switched off stays theirs, and that a failed power-on fails the change, queues a retry and is completed by the same mode. The root state changes that were injected at the child lookup are injected at the root resume. That resume also starts the controller-lifecycle removal race. - dwc3 has left host mode once a USB3 power-off or controller reset fails, so the parking checks expect host_active cleared, and no lane rebuild from a stopped fixed-hub controller. - A fixed-hub host whose PIPE cannot be routed still starts. The orientation and warm-reset checks expect 0 there. The retry check expects a parked PIPE that the next mux update routes, or, if the lock stays held, the existing pending/acknowledged-retry sequence. Keep the existing assertions: for sources without the handover the expectations are the same as before. The new runner passes 55 of 55 on the drivers of commit 3abf4a56b59e ("Merge pull request #345 from aurora-silicon/fix/harden-all-20261017"), on the handover alone and with the park retry. Reverting each of five handover behaviours in atc.c fails it. No production driver changes are needed. [ryan: adapted to 7.2.10 — context only: common.py, lifecycle.py and phy.py keep ATC_POST_TUNABLE, the per-SoC post-tunable callbacks of commit b69728b754b1 ("selftests: apple-dcp: Follow the 7.2.10 driver boundaries"), next to the new ATC_SPLIT helpers] (cherry picked from commit db757e1f92b02bbf265b936b34865c12eb54be60 ("selftests: apple-dcp: Follow the fixed-hub SuperSpeed handover")) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/gpu/apple-dcp/README | 12 ++ .../selftests/drivers/gpu/apple-dcp/common.py | 33 ++++- .../drivers/gpu/apple-dcp/dwc3/mocks.c | 1 + .../drivers/gpu/apple-dcp/lifecycle.py | 4 +- .../selftests/drivers/gpu/apple-dcp/phy.py | 72 ++++++++--- .../selftests/drivers/gpu/apple-dcp/pm.py | 119 +++++++++++++++++- .../drivers/gpu/apple-dcp/quiesce.py | 13 +- .../drivers/gpu/apple-dcp/teardown_pm.py | 3 +- 8 files changed, 229 insertions(+), 28 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/README b/tools/testing/selftests/drivers/gpu/apple-dcp/README index ad39902e5e259..906e56b7bf613 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/README +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/README @@ -90,3 +90,15 @@ pending PIPE work and a mismatched PIPE still require ownership. KUnit static redirects are disabled in host fixtures so the production bodies execute. CD321x probe uses the actual optional-router lookup with a modeled deferred probe decision; both supplier waiting and optional-router fallback are checked. + +With the fixed-hub SuperSpeed handover the locked mux body is +atcphy_mux_switch(), and the PM fixtures run the actual USB side around it: +the SuperSpeed root port goes down before the PIPE moves, the device on it +with it, and comes up only once the PIPE is back on the USB3 backend of the +running host. A modeled port refuses PIPE moves while it is up, and its +power-off and power-on failures, a port someone else switched off and the +retries the update queues are checked. The adapter of the lane fixtures takes +no port down and checks that none comes up. A failed USB3 power-off or reset +leaves dwc3 out of host mode, and a fixed-hub host whose PIPE cannot be routed +still starts. Each fixture detects these from the source, so the expectations +for sources without the handover remain. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py index 61d34b1593cc0..90c6188f56646 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py @@ -62,6 +62,20 @@ def run_unit(name, unit, output, cases=((),), includes=(), extra=()): 'atcphy_configure_post_tunable_all_modes') +# Helpers that the fixed-hub SuperSpeed handover split out of extracted functions +ATC_SPLIT = ('atcphy_pipe_live', 'atcphy_usb3_route') + + +def atc_function(source, name): + """The actual ATC function @name, or nothing for a source that does not have it. + + The fixed-hub SuperSpeed handover splits helpers out of the functions that the + fixtures extract; older sources keep that code inline. + """ + marker = re.search(r'^static [^\n]+\b' + name + r'\(', source, re.M) + return block(source, marker.start()) + '\n' if marker else '' + + def atc_mux_without_pm(source): """Lane/teardown fixtures omit USB PM, keeping mode mapping and lane work real. @@ -69,10 +83,27 @@ def atc_mux_without_pm(source): """ mapping = block(source, 'static enum atcphy_mode atcphy_typec_target_mode(') locked = block(source, 'static int atcphy_mux_set_locked(') - return mapping + '\n' + locked + r''' + if 'static int atcphy_mux_switch(' not in source: + return mapping + '\n' + locked + r''' static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state) { struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); return atcphy_mux_set_locked(atcphy, atcphy_typec_target_mode(atcphy, state)); } +''' + # The locked body is atcphy_mux_switch(); atcphy_mux_set_locked() takes the lock + # and tells whether a SuperSpeed port that the USB side took down may come up. + helpers = ''.join(atc_function(source, name) for name in + ('atcphy_usb3_port_ready', 'atcphy_mux_check', 'atcphy_mux_park')) + return mapping + '\n' + helpers + block(source, 'static int atcphy_mux_switch(') + '\n' + locked + r''' +static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state) +{ + struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); + bool port_on; + int ret = atcphy_mux_set_locked(atcphy, atcphy_typec_target_mode(atcphy, state), &port_on); + + /* Without the USB side no SuperSpeed port went down, so none may come up. */ + assert(!port_on); + return ret; +} ''' diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c index 777e12dc36135..7d4ca0d1ca3cc 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c @@ -14,6 +14,7 @@ struct apple_atcphy { unsigned int dp_usb_context_valid, dp_usb_context_dirty; u32 dp_usb_context[2][27]; bool pipe_pending, usb_off_pending, usb_off_drained; + bool usb3_port_off, park_owed; enum atcphy_mode mode, typec_mode; enum atcphy_pipehandler_state pipe_state; }; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 2219d39962273..eaa193d0aff9d 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -774,13 +774,13 @@ def fn(n, name): 'atcphy_configure_pipehandler_usb3_host_t8103', 'atcphy_configure_pipehandler_usb3_host_t8122', 'atcphy_configure_pipehandler_usb3', 'atcphy_configure_pipehandler_usb4', - 'atcphy_enable_dummy_phy', 'atcphy_configure_pipehandler_dummy', + 'atcphy_enable_dummy_phy', 'atcphy_pipe_live', 'atcphy_configure_pipehandler_dummy', 'atcphy_configure_pipehandler', 'atcphy_park_pipehandler', 'atcphy_get_mode_config', 'atcphy_dp_configure', 'j700_field_update', 'j700_aux_disable', 'atcphy_disable_dp_aux', 'atcphy_power_off', 'atcphy_power_on', *ATC_POST_TUNABLE, 'atcphy_configure', 'atcphy_usb2_init', 'atcphy_usb3_power_off', - 'atcphy_usb3_set_mode', 'atcphy_usb2_power_off', + 'atcphy_usb3_route', 'atcphy_usb3_set_mode', 'atcphy_usb2_power_off', '_atcphy_dwc3_reset_assert', 'atcphy_dwc3_reset_assert', 'atcphy_mux_set'), 'phy': ('phy_init', 'phy_exit', 'phy_power_on', 'phy_power_off'), 'core': ('dwc3_phy_power_on', 'dwc3_phy_power_off', 'dwc3_phy_exit', 'dwc3_core_exit'), diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py index fc212a968c8d2..c7f6a7b109aa8 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py @@ -3,7 +3,8 @@ """Actual PHY parking, failure/retry, masked context and OFF controls.""" from pathlib import Path import re -from common import block, source_defines, atc_mux_without_pm, ATC_POST_TUNABLE +from common import (block, source_defines, atc_mux_without_pm, ATC_POST_TUNABLE, + atc_function, ATC_SPLIT) def parking(root): @@ -14,6 +15,8 @@ def parking(root): def function(source, name): if name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in source: return atc_mux_without_pm(source) + if name in ATC_SPLIT: + return atc_function(source, name) return block(source, re.search(r'^static [^\n]+\b' + name + r'\(', source, re.M).group()) @@ -39,6 +42,7 @@ def function(source, name): unsigned int dp_usb_context_valid, dp_usb_context_dirty; u32 dp_usb_context[2][27]; bool pipe_pending, usb_off_pending, usb_off_drained; + bool usb3_port_off, park_owed; enum atcphy_mode mode, typec_mode; enum atcphy_pipehandler_state pipe_state; }; @@ -123,7 +127,7 @@ def function(source, name): 'atcphy_configure_pipehandler_usb3_host_t8122', 'atcphy_configure_pipehandler_usb3_host_t8103', 'atcphy_configure_pipehandler_usb3', 'atcphy_configure_pipehandler_usb4', - 'atcphy_enable_dummy_phy', 'atcphy_configure_pipehandler_dummy', + 'atcphy_enable_dummy_phy', 'atcphy_pipe_live', 'atcphy_configure_pipehandler_dummy', 'atcphy_configure_pipehandler', 'atcphy_park_pipehandler', 'atcphy_usb2_power_off', '_atcphy_dwc3_reset_assert', 'atcphy_dwc3_reset_assert'): parts.append(function(atc, name)) @@ -134,11 +138,17 @@ def function(source, name): 'atcphy_enable_dp_aux', 'atcphy_disable_dp_aux', 'atcphy_power_off', 'atcphy_power_on', *ATC_POST_TUNABLE, 'atcphy_configure', 'atcphy_set_orientation', 'atcphy_sw_set', 'atcphy_mux_set', - 'atcphy_usb3_set_mode', 'atcphy_usb3_power_off'): + 'atcphy_usb3_route', 'atcphy_usb3_set_mode', 'atcphy_usb3_power_off'): parts.append(function(atc, name)) production = '\n'.join(parts) if 'static int j700_usb_capture(' in dp: production = '#define HAS_USB_CONTEXT\n' + production + # With the fixed-hub SuperSpeed handover, dwc3 has left host mode once it + # powers the USB3 PHY off or asserts its reset, and a fixed-hub host starts + # even if its PIPE cannot be routed. + production = ('#define STOPPED_HOST_PARKS ' + str(int('static bool atcphy_pipe_live(' in atc)) + + '\n#define FIXED_HOST_STARTS ' + str(int('static int atcphy_usb3_route(' in atc)) + + '\n' + production) prefix = r''' #include #include @@ -274,6 +284,13 @@ def function(source, name): }; return atcphy_mux_set(&mux, &state); } +/* + * Whether a USB3 power-off or controller reset that failed leaves dwc3 a host. + * With the fixed-hub SuperSpeed handover it has left host mode either way. + */ +#define FAILED_STOP_HOST (!STOPPED_HOST_PARKS) +/* A host start whose PIPE cannot be routed fails, except on a fixed-hub port with the handover. */ +#define HOST_START_ERROR(p, err) (FIXED_HOST_STARTS && (p)->fixed_usb2 ? 0 : (err)) static void unchanged(struct apple_atcphy *p, bool dirty, bool host) { bool expected_dirty = dirty || (p->hw->has_t8140_dp && p->fixed_usb2 && host); @@ -346,10 +363,10 @@ def function(source, name): int ret = atcphy_usb3_power_off(&phy); observed("power-off parking failure", ret); assert(ret == -ETIMEDOUT); - unchanged(&p, dirty, true); + unchanged(&p, dirty, FAILED_STOP_HOST); if (!fault) { assert(atcphy_usb3_power_off(&phy) == -ETIMEDOUT); - unchanged(&p, dirty, true); + unchanged(&p, dirty, FAILED_STOP_HOST); } recover(); assert(atcphy_usb3_power_off(&phy) == 0); @@ -375,15 +392,15 @@ def function(source, name): int ret = atcphy_dwc3_reset_assert(&p.rcdev, 0); observed("reset parking failure", ret); assert(ret == -ETIMEDOUT); - unchanged(&p, dirty, true); + unchanged(&p, dirty, FAILED_STOP_HOST); if (!fault) { assert(atcphy_dwc3_reset_assert(&p.rcdev, 0) == -ETIMEDOUT); - unchanged(&p, dirty, true); + unchanged(&p, dirty, FAILED_STOP_HOST); } recover(); assert(atcphy_dwc3_reset_assert(&p.rcdev, 0) == 0); assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY && !p.host_active); - assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && p.mode_dirty == (dirty || fixed)); + assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && p.mode_dirty == (dirty || (fixed && FAILED_STOP_HOST))); assert(p.dp_owned_pmas == 3 && p.dp_idle_pmas == (dirty ? 1U : 0U)); assert(p.aux_power_requested && writes == 0); assert(reset_writes == 2 && usb2_writes > 0); @@ -405,7 +422,7 @@ def function(source, name): fail_poll = 1; /* The first DP stop ACK cannot complete. */ int ret = atcphy_usb3_power_off(&phy); observed("power-off DP cleanup failure", ret); - assert(ret == -ETIMEDOUT && p.mode_dirty && p.host_active); + assert(ret == -ETIMEDOUT && p.mode_dirty && p.host_active == FAILED_STOP_HOST); assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && p.dp_owned_pmas == 3); assert(p.aux_power_requested && usb2_writes == 0 && reset_writes == 0); struct phy_configure_opts_dp training = { @@ -422,7 +439,7 @@ def function(source, name): p.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; fail_poll = 1; assert(atcphy_usb3_power_off(&phy) == -ETIMEDOUT); - assert(p.mode_dirty && p.host_active && p.dp_owned_pmas == 3); + assert(p.mode_dirty && p.host_active == FAILED_STOP_HOST && p.dp_owned_pmas == 3); recover(); assert(atcphy_usb3_power_off(&phy) == 0); assert(p.mode == APPLE_ATCPHY_MODE_OFF && !p.mode_dirty && !p.host_active); @@ -435,7 +452,7 @@ def function(source, name): p.aux_power_requested = false; fail_poll = fault; /* BIG/SMALL sleep ACK. */ assert(atcphy_usb3_power_off(&phy) == -ETIMEDOUT); - assert(p.mode == APPLE_ATCPHY_MODE_USB2 && p.mode_dirty && p.host_active); + assert(p.mode == APPLE_ATCPHY_MODE_USB2 && p.mode_dirty && p.host_active == FAILED_STOP_HOST); recover(); assert(atcphy_usb3_power_off(&phy) == 0 && !p.mode_dirty && !p.host_active); } @@ -449,7 +466,7 @@ def function(source, name): p.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; fail_poll = fault; assert(atcphy_usb3_power_off(&phy) == -ETIMEDOUT); - assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && p.mode_dirty && p.host_active); + assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && p.mode_dirty && p.host_active == FAILED_STOP_HOST); assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); assert(usb2_writes == 0 && reset_writes == 0); unsigned int owned = p.dp_owned_pmas, idle = p.dp_idle_pmas; @@ -562,7 +579,7 @@ def function(source, name): assert(p.mode_dirty); struct phy phy = { &p }; unsigned int before = writes; - assert(atcphy_usb3_set_mode(&phy, PHY_MODE_USB_HOST, 0) == -EBUSY); + assert(atcphy_usb3_set_mode(&phy, PHY_MODE_USB_HOST, 0) == HOST_START_ERROR(&p, -EBUSY)); assert(writes == before); } require_parked = true; @@ -1003,6 +1020,23 @@ def adverse(root): struct phy phy={&p}; if (unlock) hold_unlock_ack=true; else poll_errors[2]=-EIO; +#if FIXED_HOST_STARTS + if (fixed) { + /* A fixed-hub host starts, with its PIPE parked if the lock lets go. */ + assert(atcphy_usb3_set_mode(&phy,PHY_MODE_USB_HOST,0)==0); + assert(p.mode_dirty && p.pipe_pending==(bool)unlock); + if (!unlock) { + /* Parked: the next mux update restores the lanes and routes the PIPE. */ + assert(p.pipe_state==ATCPHY_PIPEHANDLER_STATE_DUMMY); + memset(poll_errors,0,sizeof(poll_errors)); + assert((mux ? select_mode(&p,APPLE_ATCPHY_MODE_USB3) : atcphy_usb2_init(&phy))==0); + assert(!p.mode_dirty && !p.pipe_pending); + assert(atcphy_usb3_set_mode(&phy,PHY_MODE_USB_HOST,0)==0); + assert(p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3 && !p.pipe_pending); + continue; + } + } else +#endif assert(atcphy_usb3_set_mode(&phy,PHY_MODE_USB_HOST,0)== (unlock ? -ETIMEDOUT : -EIO)); assert(p.mode_dirty && p.pipe_pending); @@ -1379,7 +1413,7 @@ def warm_reset(root): assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); struct phy phy = { &p }; unsigned int before = writes; - assert(atcphy_usb3_set_mode(&phy, PHY_MODE_USB_HOST, 0) == -EBUSY); + assert(atcphy_usb3_set_mode(&phy, PHY_MODE_USB_HOST, 0) == HOST_START_ERROR(&p, -EBUSY)); assert(j700_dp_configure(&p, &rate) == -EBUSY && writes == before); fail_caller = NULL; assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); @@ -1708,6 +1742,8 @@ def block(marker): def function(name): if name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in source: return atc_mux_without_pm(source) + if name in ATC_SPLIT: + return atc_function(source, name) return block(re.search(r'^static [^\n]+\b' + name + r'\(', source, re.M).group()) @@ -1727,6 +1763,7 @@ def function(name): unsigned int dp_usb_context_valid, dp_usb_context_dirty; u32 dp_usb_context[2][27]; bool pipe_pending, usb_off_pending, usb_off_drained; + bool usb3_port_off, park_owed; bool swap_lanes; enum atcphy_mode mode, typec_mode; enum atcphy_pipehandler_state pipe_state; @@ -1782,9 +1819,9 @@ def function(name): 'atcphy_configure_pipehandler_usb3_host_t8122', 'atcphy_configure_pipehandler_usb3_host_t8103', 'atcphy_configure_pipehandler_usb3', 'atcphy_configure_pipehandler_usb4', - 'atcphy_enable_dummy_phy', 'atcphy_configure_pipehandler_dummy', + 'atcphy_enable_dummy_phy', 'atcphy_pipe_live', 'atcphy_configure_pipehandler_dummy', 'atcphy_configure_pipehandler', 'atcphy_park_pipehandler', - 'atcphy_usb3_set_mode', 'atcphy_mux_set'): + 'atcphy_usb3_route', 'atcphy_usb3_set_mode', 'atcphy_mux_set'): parts.append(function(name)) production = '\n'.join(parts) prefix = r''' @@ -2061,6 +2098,8 @@ def unwind(root): def function(source, name): if name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in source: return atc_mux_without_pm(source) + if name in ATC_SPLIT: + return atc_function(source, name) return block(source, re.search(r'^static [^\n]+\b' + name + r'\(', source, re.M).group()) @@ -2085,6 +2124,7 @@ def function(source, name): unsigned int dp_usb_context_valid, dp_usb_context_dirty; u32 dp_usb_context[2][27]; bool pipe_pending, usb_off_pending, usb_off_drained; + bool usb3_port_off, park_owed; enum atcphy_mode mode, typec_mode; enum atcphy_pipehandler_state pipe_state; }; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index ac1249af085ca..be73d45fadcb2 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -4,12 +4,30 @@ from pathlib import Path import os, re, subprocess import phy -from common import block, tipd_pm_header +from common import block, tipd_pm_header, atc_function def ownership(root, usb_unreachable=False): ns=phy.parking(root) s=ns['atc'];has_pm='static int atcphy_usb3_pm_get(' in s - prod=ns['production'].replace('struct apple_atcphy {','struct apple_atcphy {\n struct device_node *np;',1) + # The fixed-hub SuperSpeed handover powers the SuperSpeed root port off around + # a change, where the PM wrapper used to resume the link on that port. + port_power='static int atcphy_usb3_port_off(' in s + link_resume='static int atcphy_usb3_link_get(' in s + fields='\n struct device_node *np;' + if port_power: + fields+='\n struct delayed_work usb3_port_work, park_work;\n unsigned int usb3_port_retries, park_retries;' + prod=ns['production'].replace('struct apple_atcphy {','struct apple_atcphy {'+fields,1) + if port_power: + # The PIPE may only move while the SuperSpeed port of the held root is down. + mux_write='if (p == (void *)&pipe_regs[PIPEHANDLER_MUX_CTRL / 4]) mux_writes++;' + assert mux_write in prod + prod=prod.replace(mux_write,'if (p == (void *)&pipe_regs[PIPEHANDLER_MUX_CTRL / 4]) { mux_writes++; assert(!pipe_change_window() || usb3_port_down); }',1) + prod=r''' +struct work_struct { int unused; }; +struct delayed_work { struct work_struct work; unsigned int queued; unsigned long delay; }; +static bool usb3_port_down; +static bool pipe_change_window(void); +'''+prod if has_pm: prod = prod.replace(block(prod, 'static int atcphy_mux_set('), '') # Non-mux fixture entry points continue to exercise actual source. Only the new @@ -106,10 +124,16 @@ def ownership(root, usb_unreachable=False): static struct usb_device *usb_get_dev(struct usb_device *u) { u->refs++;return u; } static void usb_put_dev(struct usb_device *u) { assert(u->refs>0);u->refs--; } static int usb_for_each_dev(void *data,int (*fn)(struct usb_device *,void *)) { if(root_missing)return 0;for(unsigned int i=0;i<4;i++){int r=fn(&roots[i],data);if(r)return r;}return 0; } -static struct usb_device *usb_hub_find_child(struct usb_device *u, int port) { assert(u==&roots[3]&&root_locked&&port==1); +/* Root state changes while the root is held: at the lookup of its child or, without one, at its resume. */ +static void root_held_events(struct usb_device *u) { if(detach_at_lookup)usb_set_device_state(u,USB_STATE_NOTATTACHED); if(quiesce_at_lookup)hcd.state=HC_STATE_QUIESCING; +} +#if LINK_RESUME +static struct usb_device *usb_hub_find_child(struct usb_device *u, int port) { assert(u==&roots[3]&&root_locked&&port==1); + root_held_events(u); return link_present?&link_device:NULL; } +#endif static void usb_lock_device(struct usb_device *u) { if(u==&roots[3]){assert(!root_locked);if(REQUIRES_PLATFORM_LOCKS)assert(controller_locked&&host_locked);root_locked=true;if(detach_on_lock)u->state=0;} else {assert(u==&link_device&&root_locked&&!link_locked);link_locked=true;} @@ -134,6 +158,7 @@ def ownership(root, usb_unreachable=False): struct lock_guard g=lock_take(&pm_phy->lock);lock_release(&g); if(resume_error)return resume_error; pm_refs++;stuck_ack=link_present&&link_device.state!=USB_STATE_NOTATTACHED&&link_suspended;if(detach_on_resume)roots[3].state=0; + if(!LINK_RESUME)root_held_events(&roots[3]); if(inaccessible_on_resume)clear_bit(HCD_FLAG_HW_ACCESSIBLE,&hcd.flags); if(dead_on_resume)set_bit(HCD_FLAG_DEAD,&hcd.flags); return 0; @@ -151,6 +176,43 @@ def ownership(root, usb_unreachable=False): if(d==&link_device.dev){assert(link_locked&&link_pm_refs==1);link_pm_refs--;} else {assert(d==&roots[3].dev&&!link_locked&&pm_refs==1);pm_refs--;} } +#define dev_info(...) ((void)0) +#if PORT_POWER +#define to_delayed_work(w) container_of(w, struct delayed_work, work) +#define msecs_to_jiffies(ms) ((unsigned long)(ms)) +struct workqueue_struct { int unused; }; +static struct workqueue_struct freezable_wq; +#define system_freezable_wq (&freezable_wq) +static unsigned int port_offs, port_ons; +static int port_off_error, port_on_error; +static bool port_disabled; +static bool mod_delayed_work(struct workqueue_struct *wq, struct delayed_work *w, unsigned long delay) +{ assert(wq==system_freezable_wq&&!pm_phy->lock);w->queued++;w->delay=delay;return true; } +static bool pipe_change_window(void) { return root_locked; } +/* Port power of the held root; the device on it goes, and with it its hold on the PIPE. */ +static int usb_hub_port_power_off(struct usb_device *u, int port) +{ + assert(u==&roots[3]&&port==1&&root_locked&&pm_refs==1&&!pm_phy->lock); + port_offs++; + if(port_off_error)return port_off_error; + if(port_disabled)return 1; + if(link_present&&link_device.state!=USB_STATE_NOTATTACHED){link_device.state=USB_STATE_NOTATTACHED;stuck_ack=false;} + usb3_port_down=true; + return 0; +} +/* Only once the PIPE is back on the USB3 backend of the running host. */ +static int usb_hub_port_power_on(struct usb_device *u, int port) +{ + assert(u==&roots[3]&&port==1&&root_locked&&pm_refs==1&&!pm_phy->lock); + assert(usb3_port_down&&pm_phy->host_active&&!pm_phy->mode_dirty&&!pm_phy->pipe_pending); + assert(pm_phy->pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3); + port_ons++; + if(port_on_error)return port_on_error; + if(port_disabled)return 1; + usb3_port_down=false; + return 0; +} +#endif ''' suffix=ns['suffix'].replace('int main(', 'int __attribute__((unused)) fixture_main(',1) controls=r''' @@ -175,6 +237,9 @@ def ownership(root, usb_unreachable=False): link_present=link_suspended=link_locked=false; link_pm_refs=link_resumes=link_error=link_detach_on_resume=link_dead_on_resume=0; detach_at_lookup=quiesce_at_lookup=0; +#if PORT_POWER + usb3_port_down=port_disabled=false;port_offs=port_ons=0;port_off_error=port_on_error=0; +#endif } static int safe(struct apple_atcphy *p) { @@ -272,6 +337,7 @@ def ownership(root, usb_unreachable=False): int ret=safe(&p);fprintf(stderr,"post-child admission event %u returned %d writes %u\n",event,ret,writes-before); assert(ret==-ENODEV&&writes==before);balanced(&p); } +#if LINK_RESUME /* Selectively suspended upstream link is not resumed by root ownership alone. */ for(unsigned int swap=0;swap<2;swap++){ prepare(&p,true,true,swap);link_present=link_suspended=stuck_ack=true; @@ -291,6 +357,38 @@ def ownership(root, usb_unreachable=False): prepare(&p,true,true,false);link_present=link_suspended=true;fail_caller="atcphy_pipehandler_lock";before=writes; assert(safe(&p)==-ETIMEDOUT&&writes==before&&p.mode_dirty&&p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3);balanced(&p); fail_caller=NULL;assert(safe(&p)==0);balanced(&p); +#else + /* The device on the SuperSpeed port goes with the port, before the PIPE moves. */ + for(unsigned int swap=0;swap<2;swap++){ + prepare(&p,true,true,swap);link_present=link_suspended=stuck_ack=true; + int link_ret=safe(&p);fprintf(stderr,"port-off SAFE returned %d, root resumes %d, port offs %u\n",link_ret,resumes,port_offs); + assert(link_ret==0&&resumes==1&&link_resumes==0&&link_device.state==USB_STATE_NOTATTACHED); + /* SAFE has no USB3: the port stays off until a USB3 mode has the PIPE back. */ + assert(port_offs==1&&!port_ons&&usb3_port_down&&p.usb3_port_off);balanced(&p); + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==0&&p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3); + assert(port_offs==2&&port_ons==1&&!usb3_port_down&&!p.usb3_port_off);balanced(&p); + } + /* A port that does not go down stops the change before any PHY programming. */ + prepare(&p,true,true,false);link_present=link_suspended=true;port_off_error=-EIO;before=writes; + unsigned int retained=p.dp_usb_context_valid,owners=p.dp_owned_pmas; + assert(safe(&p)==-EIO&&writes==before&&p.dp_usb_context_valid==retained&&p.dp_owned_pmas==owners); + assert(!port_ons&&!usb3_port_down&&!p.usb3_port_off);balanced(&p); + port_off_error=0;assert(safe(&p)==0&&usb3_port_down);balanced(&p); + /* A port that someone else switched off stays theirs. */ + prepare(&p,true,true,false);port_disabled=usb3_port_down=true; + assert(safe(&p)==0&&port_offs==1&&!p.usb3_port_off);balanced(&p); + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==0&&!port_ons&&usb3_port_down);balanced(&p); + /* A port that does not come back fails the change; the same mode then only powers it on. */ + prepare(&p,true,true,false);assert(safe(&p)==0);port_on_error=-EIO; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==-EIO&&port_ons==1&&p.usb3_port_off&&p.usb3_port_work.queued==1); + balanced(&p);port_on_error=0;before=writes; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==0&&port_ons==2&&!p.usb3_port_off&&writes==before);balanced(&p); + /* PM resume is NOT proof of ACK; the port is down and stays down. */ + prepare(&p,true,true,false);link_present=link_suspended=true;fail_caller="atcphy_pipehandler_lock";before=writes; + assert(safe(&p)==-ETIMEDOUT&&writes==before&&p.mode_dirty&&p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3); + assert(usb3_port_down&&p.usb3_port_off&&!port_ons);balanced(&p); + fail_caller=NULL;assert(safe(&p)==0);balanced(&p); +#endif for(unsigned int count=0;count<2;count++){ prepare(&p,true,true,false);roots[3].maxchild=count?2:0;before=writes; assert(safe(&p)==(count?-ENODEV:-EAGAIN)&&writes==before);balanced(&p); @@ -308,11 +406,20 @@ def ownership(root, usb_unreachable=False): if has_pm: helpers=ns['block'](s,'struct atcphy_usb3_pm {')+';\n' helpers+='#if IS_REACHABLE(CONFIG_USB)\n' - if 'static int atcphy_usb3_link_get(' in s: + if link_resume: helpers+=ns['function'](s,'atcphy_usb3_link_get')+'\n'+ns['function'](s,'atcphy_usb3_link_put')+'\n' + helpers+=atc_function(s,'atcphy_usb3_is_root') helpers+=ns['function'](s,'atcphy_usb3_pm_find')+'\n#endif\n' helpers+=ns['function'](s,'atcphy_usb3_pm_get')+'\n'+ns['function'](s,'atcphy_usb3_pm_put')+'\n' - helpers+=ns['function'](s,'atcphy_mux_is_noop')+'\n'+ns['function'](s,'atcphy_mux_update')+'\n' + # The port power steps and the retries that the mux update queues; the + # delayed work functions themselves are left to the KUnit tests. + helpers+=''.join(atc_function(s,name) for name in + ('atcphy_usb3_port_off','atcphy_usb3_port_on','atcphy_usb3_port_retry_delay', + 'atcphy_usb3_port_retry','atcphy_mux_in_mode')) + helpers+=ns['function'](s,'atcphy_mux_is_noop')+'\n' + helpers+=''.join(atc_function(s,name) for name in + ('atcphy_park_retry_delay','atcphy_park_retry','atcphy_park_due','atcphy_mux_change')) + helpers+=ns['function'](s,'atcphy_mux_update')+'\n' # Lane fixtures provide the target mapping and locked body; use the real callback here. marker=re.search(r'^static int atcphy_mux_set\(',s,re.M).group() helpers+=ns['block'](s,marker)+'\n' @@ -326,7 +433,7 @@ def ownership(root, usb_unreachable=False): ('static void typec_mux_unlock(', 'void typec_mux_abort(', 'int typec_mux_begin(', 'int typec_mux_apply(', 'int typec_mux_end(', 'int typec_mux_set('))+'\n' hcd_source=(root/'drivers/usb/core/hcd.c').read_text() hcd_death=ns['block'](hcd_source,'void usb_hc_died (')+'\n' - text=prefix+prod+'\n#define REQUIRES_PLATFORM_LOCKS '+str(int('device_lock(&pm->controller->dev)' in s))+'\n#define USB_REACHABLE '+str(int(not usb_unreachable))+'\n#define HCD_DEAD_ONLY '+str(int(False))+'\n'+pmfixture+helpers+typec_caller+hcd_death+suffix+controls + text=prefix+prod+'\n#define REQUIRES_PLATFORM_LOCKS '+str(int('device_lock(&pm->controller->dev)' in s))+'\n#define PORT_POWER '+str(int(port_power))+'\n#define LINK_RESUME '+str(int(link_resume))+'\n#define USB_REACHABLE '+str(int(not usb_unreachable))+'\n#define HCD_DEAD_ONLY '+str(int(False))+'\n'+pmfixture+helpers+typec_caller+hcd_death+suffix+controls return text def probe(root): diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py index af56c57419131..09557684964bf 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only # Copyright 2026 Ryan Murray """Actual mux prepare/set and TPS removal ordering; firmware/PHY leaves are mocks.""" -from common import block +from common import block, atc_function def make_unit(source, parent=False, reference=None): reference=source if reference is None else reference @@ -118,12 +118,16 @@ def make_unit(source, parent=False, reference=None): if 'static bool tps6598x_safe_after_role(' in core: functions+=block(core,'static bool tps6598x_safe_after_role(')+'\n' if not parent:functions+=block(core,'static void tps6598x_drop_partner(')+'\n'+block(core,'static int tps6598x_set_data_role(')+'\n' + # The fixed-hub SuperSpeed handover moved the locked mux body to atcphy_mux_switch(); + # atcphy_mux_set_locked() also reports whether a SuperSpeed port may come back up. + split=[atc_function(atc,name) for name in ('atcphy_usb3_port_ready','atcphy_mux_check','atcphy_mux_park','atcphy_mux_switch')] functions+=block(core,'static '+('void' if parent else 'int')+' tps6598x_disconnect(')+'\n'+block(core,'static '+('void' if parent else 'int')+' cd321x_remove(')+'\n'+block(core,'void tipd_remove(')+'\n' ordinary_phy=r''' /* Actual ordinary-port ATC power-off, mux recheck and USB2 init; MMIO leaves. */ #undef WARN_ON_ONCE #define WARN_ON_ONCE(x) ({ bool hit=(x);phy_warns+=hit;hit; }) #define fallthrough __attribute__((fallthrough)) +#define KUNIT_STATIC_STUB_REDIRECT(...) ((void)0) enum atcphy_mode { APPLE_ATCPHY_MODE_OFF, APPLE_ATCPHY_MODE_USB2, APPLE_ATCPHY_MODE_USB3, APPLE_ATCPHY_MODE_USB3_DP, APPLE_ATCPHY_MODE_DP, APPLE_ATCPHY_MODE_TBT, APPLE_ATCPHY_MODE_USB4 }; @@ -133,6 +137,7 @@ def make_unit(source, parent=False, reference=None): const struct { bool has_usb4,has_t8140_dp; } *hw; struct { void *usb2phy; } regs; bool fixed_usb2,host_active,mode_dirty,pipe_pending,usb_off_pending,ss_tunables,dp_tunables; + bool usb3_port_off,park_owed; enum atcphy_mode mode,typec_mode; enum atcphy_pipehandler_state pipe_state; }; @@ -149,7 +154,7 @@ def make_unit(source, parent=False, reference=None): static int atcphy_configure(struct apple_atcphy *p,enum atcphy_mode mode) { configure_calls++;lane_requests+=mode!=APPLE_ATCPHY_MODE_OFF;p->mode=mode;p->mode_dirty=false;return 0; } static void set32(void *p,unsigned v) { (void)p;(void)v;assert(false); } static void atcphy_usb2_power_on(struct apple_atcphy *p) { (void)p;assert(false); } -''' + '\n'.join(block(atc, marker) for marker in ( +''' + ''.join(split) + '\n'.join(block(atc, marker) for marker in ( 'static int atcphy_usb3_power_off(', 'static int atcphy_mux_set_locked(', 'static int atcphy_usb2_init(')) + r''' static int ordinary_power_off(void) { return atcphy_usb3_power_off(&usb_phy); } @@ -170,6 +175,10 @@ def make_unit(source, parent=False, reference=None): assert(ordinary_phy.typec_mode==APPLE_ATCPHY_MODE_OFF&&lane_requests==before); } ''' + if split[-1]: + # An ordinary port takes no SuperSpeed port down, so none may come up. + ordinary_phy=ordinary_phy.replace(' int ret=atcphy_mux_set_locked(&ordinary_phy,APPLE_ATCPHY_MODE_OFF);', + ' bool port_on;\n int ret=atcphy_mux_set_locked(&ordinary_phy,APPLE_ATCPHY_MODE_OFF,&port_on);\n assert(!port_on);',1) controls=r''' static struct typec_mux_dev phy_provider,display_provider; static struct typec_mux mux; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py index 1f17261136a9c..822ba20aa87b0 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py @@ -58,7 +58,8 @@ def unit_for(source): else {assert(d==&controller.dev&&controller_locked);if(!remover)atomic_store(&handoff_active,false);controller_locked=false;pthread_mutex_unlock(&controller_mutex);} }''' unit=unit.replace(old,new) - # The scheduler injection is a child-lookup interleaving, as in the review. + # The scheduler injection is a child-lookup interleaving, as in the review, or + # the root resume for a source that takes the SuperSpeed port down instead. unit=unit.replace('if(detach_at_lookup)usb_set_device_state(u,USB_STATE_NOTATTACHED);','if(inject_remove){inject_remove=false;start_remove();}\n if(detach_at_lookup)usb_set_device_state(u,USB_STATE_NOTATTACHED);',1) unit=unit.replace('static void pm_runtime_put_noidle(struct device *d) {','static void pm_runtime_put_noidle(struct device *d) {\n if(remover&&d==&host_device)return;',1) # Old source lacks platform-lock helpers. Keep fixture-only API definitions From 113a235982f485520aa7257874930d475895e054 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 19:34:47 +1100 Subject: [PATCH 1534/2680] usb: typec: mux: Let a provider declare its timeouts transient A mux provider can time out on a change that its hardware holds up for a while, and that goes through later unchanged. On a J700 the ATC PHY cannot park the PIPE of the controller behind the fixed hub while that controller still drives it, which a partner renegotiating power can keep up for several seconds. Such a timeout says nothing wrong about the state asked for, and a Type-C driver that gives the state up after a few quick retries leaves the port in the wrong mode until the next cable event. Let a provider declare that -ETIMEDOUT from its set callback is such a hold-up. That is a property of the provider, not of a mux handle: one handle can hold several providers, and on the rear connector of a J700 it holds the PHY and a DCP display route, whose own timeouts can be permanent. So note, for each request, whether the provider that failed it timed out having declared its timeouts transient, and let the consumer ask with typec_mux_timeout_transient() after a request that timed out. Nothing declares it yet, so no consumer changes. The apple-dcp host selftests extract the transaction code; their mux shims gain the two new fields. [ryan: split by topic: only the apple-dcp selftests part; the Type-C mux part is in 170-usb] (cherry picked from commit 373c3271edb7da0489b5ab4ec63e1e199ade74f8 ("usb: typec: mux: Let a provider declare its timeouts transient")) Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/testing/selftests/drivers/gpu/apple-dcp/pm.py | 4 ++-- tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index be73d45fadcb2..c3dbcdac5a3a7 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -39,7 +39,7 @@ def ownership(root, usb_unreachable=False): int (*prepare)(struct typec_mux_dev *, struct typec_mux_state *); int (*finish)(struct typec_mux_dev *, struct typec_mux_state *); void (*commit)(struct typec_mux_dev *, struct typec_mux_state *); - void (*abort)(struct typec_mux_dev *, struct typec_mux_state *);bool removed; }; + void (*abort)(struct typec_mux_dev *, struct typec_mux_state *);bool removed, transient_timeouts; }; struct typec_mux_context { struct { bool held; } lock; }; struct task_struct { int unused; };static struct task_struct test_task; #define current (&test_task) @@ -48,7 +48,7 @@ def ownership(root, usb_unreachable=False): static void srcu_read_unlock(int *s,int i) { (void)s;(void)i; } #define mutex_trylock(m) ((m)->held ? 0 : ((m)->held=true)) #define mutex_unlock(m) ((m)->held=false) - struct typec_mux { unsigned int num_mux_devs; struct typec_mux_dev **mux_devs; struct typec_mux_context *context; struct typec_mux_state transaction; struct task_struct *owner; int srcu_index; unsigned prepared; bool applied; }; + struct typec_mux { unsigned int num_mux_devs; struct typec_mux_dev **mux_devs; struct typec_mux_context *context; struct typec_mux_state transaction; struct task_struct *owner; int srcu_index; unsigned prepared; bool applied, timeout_transient; }; #define IS_ERR_OR_NULL(p) (!(p))''') prefix=prefix.replace('#define WARN_ON_ONCE(v) assert(!(v))', 'static bool checked_warn(bool v) { assert(!v);return v; }\n#define WARN_ON_ONCE(v) checked_warn(v)') prefix=prefix.replace('#define guard(kind) (void)',r''' diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py index 09557684964bf..88e07fb5bbfa5 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py @@ -41,7 +41,7 @@ def make_unit(source, parent=False, reference=None): #define TPS_STATUS_TO_TYPEC_VCONN(s) (s) #define TPS_STATUS_TO_TYPEC_DATAROLE(s) (s) struct typec_mux_state { void *alt; unsigned long mode; void *data; }; -struct typec_mux_dev { bool removed, safe_after_role; void (*commit)(struct typec_mux_dev *,struct typec_mux_state *); void (*abort)(struct typec_mux_dev *,struct typec_mux_state *); int (*finish)(struct typec_mux_dev *,struct typec_mux_state *); void *data; int (*set)(struct typec_mux_dev *,struct typec_mux_state *); int (*prepare)(struct typec_mux_dev *,struct typec_mux_state *); }; +struct typec_mux_dev { bool removed, safe_after_role; void (*commit)(struct typec_mux_dev *,struct typec_mux_state *); void (*abort)(struct typec_mux_dev *,struct typec_mux_state *); int (*finish)(struct typec_mux_dev *,struct typec_mux_state *); void *data; int (*set)(struct typec_mux_dev *,struct typec_mux_state *); int (*prepare)(struct typec_mux_dev *,struct typec_mux_state *); bool transient_timeouts; }; struct task_struct { int unused; }; static _Thread_local struct task_struct this_task; #define current (&this_task) @@ -53,7 +53,7 @@ def make_unit(source, parent=False, reference=None): static void srcu_read_unlock(int *s,int i) { (void)s;(void)i; } static int mutex_trylock(struct mutex *m) { return pthread_mutex_trylock(&m->value)==0; } static void mutex_unlock(struct mutex *m) { assert(!pthread_mutex_unlock(&m->value)); } -struct typec_mux { struct typec_mux_context *context; struct typec_mux_state transaction; struct task_struct *owner; int srcu_index; unsigned prepared; bool applied; unsigned num_mux_devs; struct typec_mux_dev *mux_devs[3]; }; +struct typec_mux { struct typec_mux_context *context; struct typec_mux_state transaction; struct task_struct *owner; int srcu_index; unsigned prepared; bool applied; unsigned num_mux_devs; struct typec_mux_dev *mux_devs[3]; bool timeout_transient; }; struct connector { bool connected; }; struct apple_dcp { int external_lock,external_error; bool external_cable,external_stopped; struct { bool connected; } dptxport[1]; void *hdmi_hpd; bool active; bool external,typec_cable_connected; void *dev; struct connector *typec_connector; }; struct apple_dcp_typec_route; From a3e91f3fb052cfd4ef6c7133b0540f578fa23760 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 19:45:51 +1100 Subject: [PATCH 1535/2680] usb: typec: tipd: Keep applying a Type-C state that a busy mux holds up When a mux update fails, the CD321x update work hands the port to the cable-state recovery: a fresh read and a new attempt at the whole state, three times, then nothing until the next controller interrupt. Each attempt takes about a second, the resume read and the update being debounced by half a second each. On the rear port of a J700 that is not enough. The PHY cannot park its PIPE while the controller behind the fixed hub still drives it, and a Mac renegotiating power, in bursts of up to about 5.5 s, kept that up past the last attempt. Its safe state failed with -ETIMEDOUT each time, so the USB state of the new data role was never asked for, and with no further interrupt the Mac stayed on USB2 for the rest of the boot. Such a failure says nothing about the cable state, which was read fine. When the mux provider that failed a change timed out having declared its timeouts transient, do as for events that could not be acknowledged: do not charge the attempt to the cable-state budget. Apply the whole state again from a fresh read, starting with the safe state, after a delay that doubles from 500 ms up to 10 s. A controller event during the backoff brings the retry forward to the usual debounce, so that its state is applied promptly and the retry that was due later finds nothing left to do. Sleep cancels the retry and the resume read takes it over; removal, or any failure that is not such a hold-up, ends it, the latter going back to the bounded retries with no backoff left over. Only the first failure of such a series is reported, and the change that finally goes through is logged. The provider cannot tell a held-up lock from its other timeouts, so a PHY that stays stuck is not retried for good: after 16 retries, some two minutes and over twenty times the longest burst seen, the port gives up with one warning, as the budget does, and the next controller event starts over. The rear connector's mux handle holds the DCP display route as well as the PHY, and the route declares nothing, so whether a change was held up is asked of the provider that failed it, right after each mux request. The safe state of the update work now goes through the driver's own handle too, the one it asks, rather than the port's: both are the connector's, with the same providers, and the state is the same. Only the PHY of a J700 fixed-hub port declares its timeouts transient. For every other provider, any M1 or M2 port's among them and the DCP display routes, and for any other error, the driver behaves as before. The new tests drive the interrupt, resume read and update work of a port whose mux is refused for longer than the budget lasts, and expect the safe state and then the USB state once it lets go, with no further event, also on a J700 rear connector, where the display route follows the PHY on the handle. They also cover the backoff and its limit, an event that takes over from the backoff, sleep and removal during the retries, giving up on a PHY that stays stuck until the next event, a series that another error ends, and that other muxes, the display route on the J700 connector, and other errors keep the bounded retries. The apple-dcp host selftests follow: the units that extract the update work extract the new helpers with it, against a mux that does not declare its timeouts transient, and the probe unit runs the new state transitions. [ryan: split by topic: only the apple-dcp selftests part; the tipd part is in 170-usb] Fixes: 70f7c21e7d32 ("phy: apple: atc: Route the PIPE from the Type-C mux on a fixed-hub port") (cherry picked from commit ed289aabb019c8dfb1ee8575ba55430d8f43089c ("usb: typec: tipd: Keep applying a Type-C state that a busy mux holds up")) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../drivers/gpu/apple-dcp/lifecycle.py | 22 +++++++++- .../selftests/drivers/gpu/apple-dcp/pm.py | 42 ++++++++++++++++++- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index eaa193d0aff9d..d0eeb17b69240 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -24,6 +24,10 @@ def function(file, name): pm = tipd_pm_header(r) core = 'drivers/usb/typec/tipd/core.c' dcp = 'drivers/gpu/drm/apple/dcp.c' + # The retry of a change that a mux with transient timeouts held up + held = [n for n in ('cd321x_mux_held', 'cd321x_mux_set', 'cd321x_mux_still_held', + 'cd321x_retry_held_mux', 'cd321x_update_done') + if re.search(r'^static \w+ ' + n + r'\(', (r / core).read_text(), re.M)] funcs = '\n'.join(function(f, n) for f, n in ( ('drivers/gpu/drm/apple/iomfb.c', 'iomfb_pause_presentation'), (dcp, 'dcp_external_report_disconnected'), @@ -35,6 +39,7 @@ def function(file, name): (dcp, 'dcp_external_link_locked'), (dcp, 'dcp_dptx_connect_oob'), (core, 'cd321x_retry_revalidation'), + *((core, n) for n in held), (core, 'cd321x_update_work'))) prefix = r''' #include @@ -167,7 +172,11 @@ def function(file, name): struct cd321x { struct tps6598x tps; struct cd321x_status update_status; struct cd321x_pm_state pm; struct delayed_work update_work, resume_work; struct typec_mux_state state; bool state_valid, display_route_active, cur_partner_is_pd; - struct usb_pd_identity cur_partner_identity; void *connector_fwnode, *tbt_switch; }; + struct usb_pd_identity cur_partner_identity; void *connector_fwnode, *tbt_switch, *mux; }; +/* No mux here declares its timeouts transient: every failure is budgeted. */ +static bool typec_mux_timeout_transient(void *m) { (void)m; return false; } +#define dev_dbg(...) ((void)0) +#define dev_info(...) ((void)0) static int tbt_error, role_error, safe_error, mode_error; static int tbts, roles, safes; static enum usb_role usb_role_switch_get_role(enum usb_role *r) { return *r; } @@ -179,6 +188,9 @@ def function(file, name): { (void)p; assert(s == TYPEC_STATE_SAFE); safes++; return safe_error; } static int cd321x_typec_update_mode(struct tps6598x *t, struct cd321x_status *s, bool *busy) { (void)t; (void)s; *busy = mode_error == -EBUSY; return mode_error; } +/* SAFE through the driver's own mux handle is the same SAFE request. */ +static int __attribute__((unused)) typec_mux_set(void *m, struct typec_mux_state *s) +{ (void)m; return typec_set_mode(NULL, s->mode); } static void typec_unregister_partner(void *p) { (void)p; } static void *typec_register_partner(void *p, struct typec_partner_desc *d) { (void)p; (void)d; return (void *)1; } @@ -370,6 +382,14 @@ def function(file, name): printf("%d assertions, %d failures\n", checks, failures); return !!failures; } ''' + # The mode update also tells whether a mux held it up. + if re.search(r'bool \*display_busy,\s*bool \*held\)', (r / core).read_text()): + old = ('static int cd321x_typec_update_mode(struct tps6598x *t, struct cd321x_status *s, bool *busy)\n' + '{ (void)t; (void)s; *busy = mode_error == -EBUSY; return mode_error; }') + assert shim.count(old) == 1 + shim = shim.replace(old, 'static int cd321x_typec_update_mode(struct tps6598x *t, struct cd321x_status *s, bool *busy,\n' + ' bool *held)\n' + '{ (void)t; (void)s; *busy = mode_error == -EBUSY; *held = false; return mode_error; }') source = prefix + pm + shim + funcs + tests return source diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index c3dbcdac5a3a7..557162b627907 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -611,6 +611,44 @@ def function(text, name): } ''' + # A mux that holds a change up is retried outside the cable-state budget. + if 'cd321x_pm_mux_held(' in pm: + lost = ' puts("PASS actual lost events reopen a stale port; unacknowledged events keep their budget with capped backoff");\n' + assert controls.count(lost) == 1 + controls = controls.replace(lost, lost + r''' struct cd321x_pm_state held={0}; + held.phase=CD321X_PM_RUNNING; + for(unsigned int i=0,delay=0;i<4*CD321X_RESUME_ATTEMPTS;i++){ + unsigned int expected=i?(2*delay Date: Sat, 17 Oct 2026 13:31:41 +1100 Subject: [PATCH 1536/2680] selftests: apple-dcp: Compile the CD321x update with its DisplayPort helpers The physical-disconnect fixture, which fast-replug-bounce and dropped-swap-teardown build on, extracts cd321x_update_work(). The hotplug decisions it takes are now the inline helpers in cd321x-dp.h, reported through cd321x_dp_notify(), and the port keeps a pending IRQ_HPD and whether a sink was reported. When cd321x-dp.h is present, inline it (without its kernel includes) and extract cd321x_dp_notify() with the update work. Give the fixture's port the two new flags, turn its connector status values into the enum the helpers name (same values: the OOB mock still tears the display down on disconnected only, and an unknown status is a retrain, not a teardown), and supply DP_STATUS_IRQ_HPD and a no-op trace point. The fixtures keep their assertions; all 55 host tests pass. [ryan: rebased onto aurora-wip 0daed2db + ATC PIPE park v3, whose commit "usb: typec: tipd: Keep applying a Type-C state that a busy mux holds up" changed this fixture too: it extracts v3's held-mux helpers, adds the mux handle and its shims, and gives the stubbed cd321x_typec_update_mode() its held out-parameter. Kept all of that; cd321x-dp.h is inlined and cd321x_dp_notify() extracted as reviewed, the two port flags join v3's struct, and the source takes the header after v3's shim.] Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit bd98d3083bfab0b040d59beceeb6439dcf0807b2 ("selftests: apple-dcp: Compile the CD321x update with its DisplayPort helpers")) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../drivers/gpu/apple-dcp/lifecycle.py | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index d0eeb17b69240..9069b8f73d243 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -28,7 +28,22 @@ def function(file, name): held = [n for n in ('cd321x_mux_held', 'cd321x_mux_set', 'cd321x_mux_still_held', 'cd321x_retry_held_mux', 'cd321x_update_done') if re.search(r'^static \w+ ' + n + r'\(', (r / core).read_text(), re.M)] - funcs = '\n'.join(function(f, n) for f, n in ( + # The DisplayPort hotplug decisions live in cd321x-dp.h when present. + dp_header = r / 'drivers/usb/typec/tipd/cd321x-dp.h' + dp = '' + if dp_header.exists(): + dp = '\n'.join(l for l in dp_header.read_text().splitlines() + if not l.startswith('#include')) + dp = r''' +#ifndef BIT +#define BIT(n) (1U << (n)) +#endif +#define CD321X_DATA_STATUS_HPD_IRQ (1U << 9) +#define DP_STATUS_IRQ_HPD BIT(8) +#define trace_cd321x_dp_hotplug(...) ((void)0) +''' + dp + '\n' + notify = (('drivers/usb/typec/tipd/core.c', 'cd321x_dp_notify'),) if dp else () + funcs = '\n'.join(function(f, n) for f, n in notify + ( ('drivers/gpu/drm/apple/iomfb.c', 'iomfb_pause_presentation'), (dcp, 'dcp_external_report_disconnected'), (dcp, 'dcp_external_publish_generation'), @@ -155,8 +170,8 @@ def function(file, name): static struct apple_dcp display; static void drm_connector_oob_hotplug_event(void *node, int status) { assert(node); if (!status) dcp_external_disconnect(&display, true); } -#define connector_status_disconnected 0 -#define connector_status_connected 1 +enum drm_connector_status { connector_status_disconnected, connector_status_connected, + connector_status_unknown }; ''' shim = r''' enum usb_role { USB_ROLE_NONE, USB_ROLE_HOST, USB_ROLE_DEVICE }; @@ -172,6 +187,7 @@ def function(file, name): struct cd321x { struct tps6598x tps; struct cd321x_status update_status; struct cd321x_pm_state pm; struct delayed_work update_work, resume_work; struct typec_mux_state state; bool state_valid, display_route_active, cur_partner_is_pd; + bool dp_irq_hpd, dp_sink_reported; struct usb_pd_identity cur_partner_identity; void *connector_fwnode, *tbt_switch, *mux; }; /* No mux here declares its timeouts transient: every failure is budgeted. */ static bool typec_mux_timeout_transient(void *m) { (void)m; return false; } @@ -390,7 +406,7 @@ def function(file, name): shim = shim.replace(old, 'static int cd321x_typec_update_mode(struct tps6598x *t, struct cd321x_status *s, bool *busy,\n' ' bool *held)\n' '{ (void)t; (void)s; *busy = mode_error == -EBUSY; *held = false; return mode_error; }') - source = prefix + pm + shim + funcs + tests + source = prefix + pm + shim + dp + funcs + tests return source def bounce(root): From 6644cf8c6176c2e68a7522fc1d20a352ca01793b Mon Sep 17 00:00:00 2001 From: Wes Grimes Date: Sat, 3 Oct 2026 17:42:37 -0400 Subject: [PATCH 1537/2680] drm/apple: read SupportsBacklightControl from the display attributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DCP describes a connected display in its DisplayAttributes dictionary. For external displays whose brightness DCP can set, such as the Apple Studio Display, that includes SupportsBacklightControl alongside BacklightLuminanceMin/Max (4 and 600 nits there). Parse the flag and record it per pipeline, cleared for every new display and never set for an integrated panel, which has its own backlight path. Nothing uses it yet. [ryan: adapted to 7.2.10 — parse inside the bounded iterator_begin() dictionary walk and keep the H17P external metadata context] Signed-off-by: Wes Grimes (cherry picked from commit be289d9ca14dc4cc8c52b2b06906d9c19cf636f5) --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/iomfb_template.c | 6 +++++- drivers/gpu/drm/apple/parser.c | 6 +++++- drivers/gpu/drm/apple/parser.h | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 589da7130aec0..6bc4dcaf6bfe5 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -379,6 +379,8 @@ struct apple_dcp { /* Attributes of the connected display */ int width_mm, height_mm; + /* an external display whose backlight DCP can drive */ + bool ext_backlight; unsigned notch_height; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index b403a6c61e2e9..1be48abd5de5e 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1103,14 +1103,18 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, } else if (!strcmp(req->key, "DisplayAttributes")) { struct dcp_parse_ctx metadata_ctx = ctx; + bool backlight_control; + ret = parse_display_attributes(&ctx, &dcp->width_mm, - &dcp->height_mm); + &dcp->height_mm, &backlight_control); if (ret) { dev_warn(dcp->dev, "failed to parse display attribs\n"); return false; } + WRITE_ONCE(dcp->ext_backlight, + backlight_control && !dcp_has_panel(dcp)); dcp_set_dimensions(dcp); if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { struct dcp_display_metadata metadata; diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index a7a7061647aea..823a575c0b564 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -1104,12 +1104,14 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, } int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, - int *height_mm) + int *height_mm, bool *backlight_control) { int ret = 0; struct iterator it; s64 width_cm = 0, height_cm = 0; + *backlight_control = false; + ret = iterator_begin(handle, &it, true); if (ret) return ret; @@ -1123,6 +1125,8 @@ int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, ret = parse_int(it.handle, &width_cm); else if (!strcmp(key, "MaxVerticalImageSize")) ret = parse_int(it.handle, &height_cm); + else if (!strcmp(key, "SupportsBacklightControl")) + ret = parse_bool(it.handle, backlight_control); else ret = skip(it.handle); diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 5a2a84614c536..1f5612e672887 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -140,7 +140,7 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, bool internal, bool external_sdr); int parse_display_metadata(struct dcp_parse_ctx *handle, struct dcp_display_metadata *out); int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, - int *height_mm); + int *height_mm, bool *backlight_control); int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, const char **class, s64 *unit, bool h17p_keys); From efb78fd5ad3ca461d056d0889cfb34b6f08982ce Mon Sep 17 00:00:00 2001 From: Wes Grimes Date: Sat, 3 Oct 2026 17:43:02 -0400 Subject: [PATCH 1538/2680] drm/apple: drive the backlight of external displays MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Apple Studio Display has a USB HID brightness control (VESA usage page 0x82, feature report 1), but on firmware 17.0 the panel ignores it, from Linux and from macOS alike. macOS sets the brightness through DCP instead: with every HID interface of the display seized, the DisplayServices brightness API still dims the panel. DCP accepts the same swap backlight fields for such a display as for an integrated panel. bl_value is then a signed 32-bit level, 0 for the dimmest and S32_MAX for the brightest; negative values clamp to the dimmest. Whether the steps in between are linear in luminance is not known, so the level is exposed on a unitless 0-1000 scale. Register a BACKLIGHT_RAW device, "apple--bl", for the connector of a display that reports SupportsBacklightControl. It is a child of the connector's sysfs device, so userspace can tell which output it belongs to. The connector's hotplug work registers it when such a display connects and drops it otherwise. Disconnects reported outside that work queue it, so registration always runs without the caller's locks. The late_register and early_unregister hooks bound it to the connector's own lifetime. The backlight core stops calling update_status once unregistration returns, so a brightness write that races an unplug is safe. The level is kept per connector across replug and sent with the first swap after every power-on. A new level is stored before brightness.update is set, and the swap clears update with xchg() before reading the level. A level written while a swap is being built thus either goes out with that swap or leaves update set, so the commit that the write triggers sends it. Integrated panels keep their existing path. Tested on j416s (MacBook Pro 16" M2 Pro) with a Studio Display on the right and "left 1" ports: brightnessctl without root, unplug and replug (level restored), s2idle, and unbinding the DRM device while brightness writes race the teardown. On "left 2" the device registers, but the level was not exercised. The integrated panel's backlight is unaffected. [ryan: adapted to 7.2.10 — our disconnected_hpd_event already defers cleanup to hotplug_wq, which now syncs the backlight; never report a level for H17P external pipes] Signed-off-by: Wes Grimes (cherry picked from commit b0f9e8d2eac247ef1164cb3e261211f22158be07) --- drivers/gpu/drm/apple/apple_drv.c | 14 +++ drivers/gpu/drm/apple/connector.h | 15 +++ drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp_backlight.c | 133 +++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb.c | 2 + drivers/gpu/drm/apple/iomfb_template.c | 9 ++ 6 files changed, 174 insertions(+) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 421dbf8d2a68f..a0f3dbb155c84 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -312,6 +312,16 @@ static void appledrm_connector_cleanup(struct drm_connector *connector) kfree(apple_connector); } +static int apple_connector_late_register(struct drm_connector *connector) +{ + return apple_connector_backlight_late_register(to_apple_connector(connector)); +} + +static void apple_connector_early_unregister(struct drm_connector *connector) +{ + apple_connector_backlight_early_unregister(to_apple_connector(connector)); +} + static const struct drm_connector_funcs apple_connector_funcs = { .fill_modes = drm_helper_probe_single_connector_modes, .destroy = appledrm_connector_cleanup, @@ -321,6 +331,8 @@ static const struct drm_connector_funcs apple_connector_funcs = { .detect = apple_connector_detect, .debugfs_init = apple_connector_debugfs_init, .oob_hotplug_event = apple_connector_oob_hotplug, + .late_register = apple_connector_late_register, + .early_unregister = apple_connector_early_unregister, }; /* @@ -383,6 +395,7 @@ static int apple_connector_create(struct drm_device *drm, return -ENOMEM; mutex_init(&connector->chunk_lock); + apple_connector_backlight_init(connector); drm_connector_helper_add(&connector->base, &apple_connector_helper_funcs); if (attach_fwnode) @@ -549,6 +562,7 @@ static int apple_probe_typec_ports(struct drm_device *drm, return -ENOMEM; mutex_init(&connector->chunk_lock); + apple_connector_backlight_init(connector); drm_connector_helper_add(&connector->base, &apple_connector_helper_funcs); diff --git a/drivers/gpu/drm/apple/connector.h b/drivers/gpu/drm/apple/connector.h index 38772d6731c2c..654825124ee71 100644 --- a/drivers/gpu/drm/apple/connector.h +++ b/drivers/gpu/drm/apple/connector.h @@ -18,6 +18,11 @@ void dcp_hotplug(struct work_struct *work); bool dcp_connector_connected(struct apple_connector *connector); void dcp_retrain_oob(struct apple_connector *connector); +void apple_connector_backlight_init(struct apple_connector *connector); +void apple_connector_backlight_sync(struct apple_connector *connector); +int apple_connector_backlight_late_register(struct apple_connector *connector); +void apple_connector_backlight_early_unregister(struct apple_connector *connector); + /* * How many DCP pipelines may offer a route to one Type-C port. The Type-C mux * class caps the mode-switch providers per connector (TYPEC_MUX_MAX_DEVS) and @@ -58,6 +63,16 @@ struct apple_connector { struct dcp_chunks timing_elements; struct dcp_chunks display_attributes; struct dcp_chunks transport; + + /* + * Backlight of an external display that DCP can drive, registered + * while such a display is connected. bl_lock serialises + * registration against the connector's sysfs lifetime. + */ + struct mutex bl_lock; + struct backlight_device *bl_dev; + bool bl_allowed; /* between late_register and early_unregister */ + int bl_level; /* kept across replug, applied on every power-on */ }; #define to_apple_connector(x) container_of(x, struct apple_connector, base) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 6bc4dcaf6bfe5..d177e6aba274d 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -600,6 +600,7 @@ void dcp_backlight_stop(struct apple_dcp *dcp); void dcp_backlight_finish_fade(struct apple_dcp *dcp); int dcp_backlight_register(struct apple_dcp *dcp); int dcp_backlight_update(struct apple_dcp *dcp); +s32 dcp_ext_backlight_value(struct apple_dcp *dcp); bool dcp_has_panel(struct apple_dcp *dcp); #define DCP_AUDIO_MAX_CHANS 15 diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 1552bb7ca9384..8f3643a0bda3f 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -5,12 +5,15 @@ #include #include #include +#include #include +#include #include #include #include "linux/jiffies.h" +#include "connector.h" #include "dcp.h" #include "dcp-internal.h" @@ -562,3 +565,133 @@ int dcp_backlight_register(struct apple_dcp *dcp) return 0; } + +/* + * External displays that report SupportsBacklightControl, such as the Apple + * Studio Display, take the same swap backlight fields as an integrated + * panel. bl_value is then a signed 32-bit level from 0 (dimmest) to S32_MAX + * (brightest); DCP converts it for the display. The backlight device belongs + * to the connector, so userspace can tell which output it dims. + */ +#define DCP_EXT_BL_MAX 1000 + +/* The bl_value for the display on this DCP, or -1 if it has no backlight. */ +s32 dcp_ext_backlight_value(struct apple_dcp *dcp) +{ + struct apple_connector *connector = READ_ONCE(dcp->connector); + + if (!connector || !READ_ONCE(dcp->ext_backlight)) + return -1; + /* H17P external swap brightness fields are unverified; leave them. */ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) + return -1; + + return div_u64((u64)READ_ONCE(connector->bl_level) * S32_MAX, + DCP_EXT_BL_MAX); +} + +static int dcp_ext_backlight_update_status(struct backlight_device *bd) +{ + struct apple_connector *connector = bl_get_data(bd); + struct platform_device *pdev; + struct apple_dcp *dcp; + + WRITE_ONCE(connector->bl_level, backlight_get_brightness(bd)); + + /* An unrouted port picks the level up when it powers on again. */ + pdev = READ_ONCE(connector->dcp); + if (!pdev) + return 0; + + dcp = platform_get_drvdata(pdev); + /* pairs with the xchg() in iomfb_flush: it sees the level or leaves update set */ + smp_store_release(&dcp->brightness.update, true); + + return dcp_backlight_update(dcp); +} + +static const struct backlight_ops dcp_ext_backlight_ops = { + .update_status = dcp_ext_backlight_update_status, +}; + +static void apple_connector_backlight_register(struct apple_connector *connector) +{ + struct backlight_properties props = { + .type = BACKLIGHT_RAW, + .max_brightness = DCP_EXT_BL_MAX, + .brightness = connector->bl_level, + }; + struct backlight_device *bd; + char name[32]; + + snprintf(name, sizeof(name), "apple-%s-bl", connector->base.name); + bd = backlight_device_register(name, connector->base.kdev, connector, + &dcp_ext_backlight_ops, &props); + if (IS_ERR(bd)) { + drm_warn(connector->base.dev, + "%s: failed to register backlight: %pe\n", + connector->base.name, bd); + return; + } + + connector->bl_dev = bd; +} + +static void apple_connector_backlight_unregister(struct apple_connector *connector) +{ + backlight_device_unregister(connector->bl_dev); + connector->bl_dev = NULL; +} + +void apple_connector_backlight_init(struct apple_connector *connector) +{ + mutex_init(&connector->bl_lock); + connector->bl_level = DCP_EXT_BL_MAX; +} + +/* + * Register the backlight while a display that supports it is connected, and + * drop it otherwise. Called from the connector's hotplug work. + */ +void apple_connector_backlight_sync(struct apple_connector *connector) +{ + struct platform_device *pdev; + bool want = false; + + mutex_lock(&connector->bl_lock); + + pdev = READ_ONCE(connector->dcp); + if (connector->bl_allowed && READ_ONCE(connector->connected) && pdev) { + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + want = READ_ONCE(dcp->ext_backlight); + } + + if (want && !connector->bl_dev) + apple_connector_backlight_register(connector); + else if (!want && connector->bl_dev) + apple_connector_backlight_unregister(connector); + + mutex_unlock(&connector->bl_lock); +} + +int apple_connector_backlight_late_register(struct apple_connector *connector) +{ + mutex_lock(&connector->bl_lock); + connector->bl_allowed = true; + mutex_unlock(&connector->bl_lock); + + /* a display may have connected before the connector was registered */ + apple_connector_backlight_sync(connector); + + return 0; +} + +void apple_connector_backlight_early_unregister(struct apple_connector *connector) +{ + mutex_lock(&connector->bl_lock); + connector->bl_allowed = false; + if (connector->bl_dev) + apple_connector_backlight_unregister(connector); + mutex_unlock(&connector->bl_lock); +} diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 76da1567ff870..2782205592aa7 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1099,6 +1099,8 @@ void dcp_hotplug(struct work_struct *work) scoped_guard(mutex, &connector->base.dev->mode_config.mutex) dcp_consume_edid_invalidation(connector); + apple_connector_backlight_sync(connector); + pdev = READ_ONCE(connector->dcp); if (!pdev) { /* a Type-C port unrouted after this was queued */ scoped_guard(mutex, &connector->base.dev->mode_config.mutex) { diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 1be48abd5de5e..784879e5a9b79 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -2538,6 +2538,15 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru req->swap.bl_value = dcp->brightness.dac; req->swap.bl_power = 0x40; dcp->brightness.update = false; + } else if (!dcp_has_panel(dcp) && xchg(&dcp->brightness.update, false)) { + /* read after clearing: a newer level sets update again and commits */ + s32 bl = dcp_ext_backlight_value(dcp); + + if (bl >= 0) { + req->swap.bl_unk = 1; + req->swap.bl_value = bl; + req->swap.bl_power = 0x40; + } } if (crtc_state->color_mgmt_changed) { From da245f4ed4b9fc483905af850d3196f91024b8f6 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:03:15 -0700 Subject: [PATCH 1539/2680] drm/apple: read SupportsBacklightControl leniently parse_bool() rejects a value of any other type, and the error fails the whole DisplayAttributes dictionary: no physical size, and a failed reply to DCP. Before the key was read it was skipped, so a display that sent it as, say, an integer would regress. Read a mismatched value as false and skip it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 07be8ecccdc17f551fc4adf8b19b1f3d233249d9) --- drivers/gpu/drm/apple/parser.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 823a575c0b564..30f470c326163 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -231,6 +231,22 @@ static int parse_bool(struct dcp_parse_ctx *handle, bool *b) return 0; } +/* + * For an optional flag: a value of another type reads as false instead of + * failing the whole dictionary, which the key did not do before it was read. + */ +static int parse_bool_or_skip(struct dcp_parse_ctx *handle, bool *b) +{ + struct dcp_parse_ctx at = *handle; + + if (!parse_bool(handle, b)) + return 0; + + *handle = at; + *b = false; + return skip(handle); +} + #if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) static int parse_blob(struct dcp_parse_ctx *handle, size_t size, u8 const **blob) { @@ -1126,7 +1142,7 @@ int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, else if (!strcmp(key, "MaxVerticalImageSize")) ret = parse_int(it.handle, &height_cm); else if (!strcmp(key, "SupportsBacklightControl")) - ret = parse_bool(it.handle, backlight_control); + ret = parse_bool_or_skip(it.handle, backlight_control); else ret = skip(it.handle); From 9e520bf5909647e316535bb360ce1e54531c007c Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:03:15 -0700 Subject: [PATCH 1540/2680] drm/apple: external backlight: change nothing until a level is chosen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes from review of the external display backlight, so that adding it changes nothing for anyone who does not use it: - Full brightness on connect. bl_level started at the maximum and every power-on sets brightness.update, so each boot, plug, DPMS wake and Type-C reroute turned a Studio Display to full brightness, which Omarchy's brightness keys cannot yet undo. bl_level now starts unset and nothing is sent until userspace writes a level. The device reports the maximum meanwhile; systemd-backlight writes that back when it has nothing saved, and that write is ignored. - fbdev blanking reached the device (no controls_device), and a blanked device reports brightness 0, which was stored and sent: the display would come back at its dimmest. Ignore blanking; display power belongs to DRM. - The name. apple-DP-1-bl sorts before the panel's apple-panel-bl, and Omarchy takes the first backlight as the built-in screen's, so with a Studio Display attached the laptop's brightness keys moved the Studio Display. Name it dcp--bl, which sorts after apple-panel-bl in any locale. Userspace that follows the connector (omarchy-mac#687) is unaffected. - A second hotplug event on every disconnect: disconnected_hpd_event() queued the whole hotplug work, which sends its own event. Use a work item that only syncs the backlight, and sync at the end of the hotplug work, after its event. Sync when a Type-C port loses its pipeline too. - Lifetime: the hotplug and backlight work items could run after the connector was freed at teardown; cancel them first. - Writes committed synchronously with the backlight core's ops_lock held, and returned -EDEADLK to userspace. Commit from a work item, retrying on -EDEADLK, so a slider's writes collapse into a few commits; the stored level goes out with the next swap if a commit fails. - The update flag was consumed by swaps that do_swap() then drops, so a level could be lost until the next power-on. - ext_backlight was never cleared: clear it before parsing new display attributes, on disconnect, and when the pipeline returns to its fixed output. [ryan: adapted to 7.2.10 — keep our deferred disconnect (hotplug_wq owns EDID/property cleanup and the event); clear ext_backlight and queue bl_sync_wq there] Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit c37ee1d7845cde1d772f2174f44cb9f387a9b01a) --- drivers/gpu/drm/apple/apple_drv.c | 5 ++ drivers/gpu/drm/apple/connector.h | 8 ++- drivers/gpu/drm/apple/dcp.c | 14 +++++ drivers/gpu/drm/apple/dcp_backlight.c | 86 +++++++++++++++++++++++--- drivers/gpu/drm/apple/iomfb.c | 5 +- drivers/gpu/drm/apple/iomfb_template.c | 12 +++- 6 files changed, 117 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index a0f3dbb155c84..f12bfe36e8eb8 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -304,6 +304,11 @@ static void appledrm_connector_cleanup(struct drm_connector *connector) { struct apple_connector *apple_connector = to_apple_connector(connector); + /* Queued by the last disconnect at shutdown; they use the connector. */ + cancel_work_sync(&apple_connector->hotplug_wq); + cancel_work_sync(&apple_connector->bl_sync_wq); + cancel_work_sync(&apple_connector->bl_commit_wq); + mutex_destroy(&apple_connector->bl_lock); drm_connector_cleanup(connector); kfree(apple_connector->color_elements.data); kfree(apple_connector->timing_elements.data); diff --git a/drivers/gpu/drm/apple/connector.h b/drivers/gpu/drm/apple/connector.h index 654825124ee71..5b908fb85520d 100644 --- a/drivers/gpu/drm/apple/connector.h +++ b/drivers/gpu/drm/apple/connector.h @@ -70,9 +70,15 @@ struct apple_connector { * registration against the connector's sysfs lifetime. */ struct mutex bl_lock; + struct work_struct bl_sync_wq; /* apple_connector_backlight_sync() */ + struct work_struct bl_commit_wq; /* sends a new level */ struct backlight_device *bl_dev; bool bl_allowed; /* between late_register and early_unregister */ - int bl_level; /* kept across replug, applied on every power-on */ + /* + * Kept across replug and applied on every power-on, once userspace has + * chosen one; -1 until then, and nothing is sent. + */ + int bl_level; }; #define to_apple_connector(x) container_of(x, struct apple_connector, base) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 7051471cc9fb9..915e9418a68a4 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -535,6 +535,8 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) WRITE_ONCE(connector->connected, false); /* hotplug work queued before this checks for it */ WRITE_ONCE(connector->dcp, NULL); + /* no pipeline, so no backlight */ + schedule_work(&connector->bl_sync_wq); /* Unrouted: the port could go to any of its pipelines again. */ if (connector->port_encoder) @@ -543,6 +545,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) } dcp->typec_connector = NULL; dcp->connector = dcp->fixed_connector; + WRITE_ONCE(dcp->ext_backlight, false); if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_USB) { dcp->connector_type = dcp->fixed_connector_type; @@ -3114,13 +3117,24 @@ bool dcp_hold_resume_event(struct platform_device *pdev) static void disconnected_hpd_event(struct apple_connector *con) { + struct platform_device *pdev; + if (!con) return; + pdev = READ_ONCE(con->dcp); + if (pdev) { + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + WRITE_ONCE(dcp->ext_backlight, false); + } + /* May run with DRM modeset locks held; defer property/cache cleanup. */ WRITE_ONCE(con->connected, false); WRITE_ONCE(con->edid_invalidation_pending, true); schedule_work(&con->hotplug_wq); + /* Drop the display's backlight, outside the caller's locks. */ + schedule_work(&con->bl_sync_wq); } static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 8f3643a0bda3f..3728d587628ba 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -575,10 +575,15 @@ int dcp_backlight_register(struct apple_dcp *dcp) */ #define DCP_EXT_BL_MAX 1000 -/* The bl_value for the display on this DCP, or -1 if it has no backlight. */ +/* + * The bl_value for the display on this DCP, or -1 to send none: it has no + * backlight, or userspace has not chosen a level yet, so the display keeps + * its own. + */ s32 dcp_ext_backlight_value(struct apple_dcp *dcp) { struct apple_connector *connector = READ_ONCE(dcp->connector); + int level; if (!connector || !READ_ONCE(dcp->ext_backlight)) return -1; @@ -586,17 +591,33 @@ s32 dcp_ext_backlight_value(struct apple_dcp *dcp) if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) return -1; - return div_u64((u64)READ_ONCE(connector->bl_level) * S32_MAX, - DCP_EXT_BL_MAX); + level = READ_ONCE(connector->bl_level); + if (level < 0) + return -1; + + return div_u64((u64)level * S32_MAX, DCP_EXT_BL_MAX); } static int dcp_ext_backlight_update_status(struct backlight_device *bd) { struct apple_connector *connector = bl_get_data(bd); + int level = bd->props.brightness; struct platform_device *pdev; struct apple_dcp *dcp; - WRITE_ONCE(connector->bl_level, backlight_get_brightness(bd)); + /* The display's power belongs to DRM, not to fbdev blanking. */ + if (backlight_is_blank(bd)) + return 0; + + /* + * Until a level is chosen the device reports DCP_EXT_BL_MAX, and + * systemd-backlight writes what it reads back when it has nothing + * saved. Sending that would turn the display to full brightness. + */ + if (READ_ONCE(connector->bl_level) < 0 && level == DCP_EXT_BL_MAX) + return 0; + + WRITE_ONCE(connector->bl_level, level); /* An unrouted port picks the level up when it powers on again. */ pdev = READ_ONCE(connector->dcp); @@ -607,11 +628,44 @@ static int dcp_ext_backlight_update_status(struct backlight_device *bd) /* pairs with the xchg() in iomfb_flush: it sees the level or leaves update set */ smp_store_release(&dcp->brightness.update, true); - return dcp_backlight_update(dcp); + /* + * Commit from a work item: a slider's stream of writes collapses into + * a few commits, and none of them blocks with ops_lock held. + */ + schedule_work(&connector->bl_commit_wq); + return 0; +} + +static void apple_connector_backlight_commit_work(struct work_struct *work) +{ + struct apple_connector *connector = + container_of(work, struct apple_connector, bl_commit_wq); + struct platform_device *pdev = READ_ONCE(connector->dcp); + int ret; + + if (!pdev) + return; + + do { + ret = dcp_backlight_update(platform_get_drvdata(pdev)); + } while (ret == -EDEADLK); + + /* update stays set, so the next swap carries the level anyway */ + if (ret) + drm_dbg(connector->base.dev, "%s: backlight commit: %d\n", + connector->base.name, ret); +} + +/* Never follow fbdev blanking, which would store a level of 0. */ +static bool dcp_ext_backlight_controls_device(struct backlight_device *bd, + struct device *display_dev) +{ + return false; } static const struct backlight_ops dcp_ext_backlight_ops = { .update_status = dcp_ext_backlight_update_status, + .controls_device = dcp_ext_backlight_controls_device, }; static void apple_connector_backlight_register(struct apple_connector *connector) @@ -619,12 +673,18 @@ static void apple_connector_backlight_register(struct apple_connector *connector struct backlight_properties props = { .type = BACKLIGHT_RAW, .max_brightness = DCP_EXT_BL_MAX, - .brightness = connector->bl_level, + .brightness = connector->bl_level < 0 ? DCP_EXT_BL_MAX : + connector->bl_level, }; struct backlight_device *bd; char name[32]; - snprintf(name, sizeof(name), "apple-%s-bl", connector->base.name); + /* + * Sorts after the integrated panel's apple-panel-bl in any locale: + * userspace that takes the first backlight as the built-in screen's + * (Omarchy's omarchy-hw-display) keeps the panel. + */ + snprintf(name, sizeof(name), "dcp-%s-bl", connector->base.name); bd = backlight_device_register(name, connector->base.kdev, connector, &dcp_ext_backlight_ops, &props); if (IS_ERR(bd)) { @@ -643,10 +703,18 @@ static void apple_connector_backlight_unregister(struct apple_connector *connect connector->bl_dev = NULL; } +static void apple_connector_backlight_sync_work(struct work_struct *work) +{ + apple_connector_backlight_sync(container_of(work, struct apple_connector, + bl_sync_wq)); +} + void apple_connector_backlight_init(struct apple_connector *connector) { mutex_init(&connector->bl_lock); - connector->bl_level = DCP_EXT_BL_MAX; + INIT_WORK(&connector->bl_sync_wq, apple_connector_backlight_sync_work); + INIT_WORK(&connector->bl_commit_wq, apple_connector_backlight_commit_work); + connector->bl_level = -1; } /* @@ -694,4 +762,6 @@ void apple_connector_backlight_early_unregister(struct apple_connector *connecto if (connector->bl_dev) apple_connector_backlight_unregister(connector); mutex_unlock(&connector->bl_lock); + /* nothing can queue it again once the device is gone */ + cancel_work_sync(&connector->bl_commit_wq); } diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 2782205592aa7..ab591667899a1 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1099,8 +1099,6 @@ void dcp_hotplug(struct work_struct *work) scoped_guard(mutex, &connector->base.dev->mode_config.mutex) dcp_consume_edid_invalidation(connector); - apple_connector_backlight_sync(connector); - pdev = READ_ONCE(connector->dcp); if (!pdev) { /* a Type-C port unrouted after this was queued */ scoped_guard(mutex, &connector->base.dev->mode_config.mutex) { @@ -1110,6 +1108,7 @@ void dcp_hotplug(struct work_struct *work) } } drm_kms_helper_connector_hotplug_event(&connector->base); + apple_connector_backlight_sync(connector); return; } dcp = platform_get_drvdata(pdev); @@ -1169,6 +1168,8 @@ void dcp_hotplug(struct work_struct *work) dcp_test_point(dcp, DCP_TEST_HOTPLUG_EVENT); drm_kms_helper_connector_hotplug_event(&connector->base); + /* after the event, so registering a backlight cannot delay it */ + apple_connector_backlight_sync(connector); } /* diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 784879e5a9b79..f3f20ed14e330 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1105,6 +1105,8 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, bool backlight_control; + /* a display whose attributes do not parse gets no backlight */ + WRITE_ONCE(dcp->ext_backlight, false); ret = parse_display_attributes(&ctx, &dcp->width_mm, &dcp->height_mm, &backlight_control); @@ -2538,8 +2540,14 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru req->swap.bl_value = dcp->brightness.dac; req->swap.bl_power = 0x40; dcp->brightness.update = false; - } else if (!dcp_has_panel(dcp) && xchg(&dcp->brightness.update, false)) { - /* read after clearing: a newer level sets update again and commits */ + } else if (!dcp_has_panel(dcp) && READ_ONCE(dcp->mode_state.valid) && + dcp->connector && READ_ONCE(dcp->connector->connected) && + xchg(&dcp->brightness.update, false)) { + /* + * Consumed only when do_swap() will send the swap, so a level is + * not lost to a dropped one. Read after clearing: a newer level + * sets update again and commits. + */ s32 bl = dcp_ext_backlight_value(dcp); if (bl >= 0) { From 534cb566bc569303e9a5479235590970e806d63d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:19:27 -0700 Subject: [PATCH 1541/2680] drm/apple: external backlight: fixes from a second review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - DisplayAttributes cleared ext_backlight before parsing and set it after, so the hotplug work could run in between and unregister the backlight of a connected display, with nothing to bring it back. And since a disconnect clears the flag, attributes arriving after the hotplug work left a reconnected display without one. Parse into a local, write the flag once, and sync the backlight when it changes. - parse_bool_or_skip() returned skip()'s error, failing the whole dictionary on a value the parser cannot skip. Before the key was read that error was ignored; ignore it again. - The hotplug work was initialised only after a step that can fail, and connector cleanup now cancels it: on that error path cancel_work_sync() would warn on a zeroed work. Initialise it first, in both connector paths. - After a bl_power=4 write, brightness writes were held back until bl_power went back to 0, although bl_power does not turn this display off. Store and send the level whatever the power state; fbdev blanking still never reaches the device. [ryan: adapted to 7.2.10 — drop the parse_bool_or_skip() change that ignored an unskippable value: our bounded dictionary walk already fails on unskippable values for every key, so keep that error] Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 58bcb316a9217f8f57e447dba0aa957741e88e1d) --- drivers/gpu/drm/apple/apple_drv.c | 6 ++++-- drivers/gpu/drm/apple/dcp_backlight.c | 10 +++++----- drivers/gpu/drm/apple/iomfb_template.c | 20 ++++++++++++++------ 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index f12bfe36e8eb8..b7595a1aeec3b 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -401,6 +401,8 @@ static int apple_connector_create(struct drm_device *drm, mutex_init(&connector->chunk_lock); apple_connector_backlight_init(connector); + /* before anything can fail: cleanup cancels it */ + INIT_WORK(&connector->hotplug_wq, dcp_hotplug); drm_connector_helper_add(&connector->base, &apple_connector_helper_funcs); if (attach_fwnode) @@ -417,7 +419,6 @@ static int apple_connector_create(struct drm_device *drm, connector->base.polled = DRM_CONNECTOR_POLL_HPD; connector->connected = false; connector->dcp = dcp; - INIT_WORK(&connector->hotplug_wq, dcp_hotplug); ret = drm_connector_attach_encoder(&connector->base, &encoder->base); if (ret) @@ -568,6 +569,8 @@ static int apple_probe_typec_ports(struct drm_device *drm, mutex_init(&connector->chunk_lock); apple_connector_backlight_init(connector); + /* before anything can fail: cleanup cancels it */ + INIT_WORK(&connector->hotplug_wq, dcp_hotplug); drm_connector_helper_add(&connector->base, &apple_connector_helper_funcs); @@ -586,7 +589,6 @@ static int apple_probe_typec_ports(struct drm_device *drm, connector->base.polled = DRM_CONNECTOR_POLL_HPD; connector->connected = false; connector->dcp = NULL; - INIT_WORK(&connector->hotplug_wq, dcp_hotplug); for (i = 0; i < num_dcp; i++) { if (!dcp_typec_port_has_candidate(port_idx, dcp[i])) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 3728d587628ba..6d2f85197004e 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -605,11 +605,11 @@ static int dcp_ext_backlight_update_status(struct backlight_device *bd) struct platform_device *pdev; struct apple_dcp *dcp; - /* The display's power belongs to DRM, not to fbdev blanking. */ - if (backlight_is_blank(bd)) - return 0; - /* + * The display's power belongs to DRM: fbdev blanking never reaches + * this device (controls_device), and bl_power does not turn the + * display off, so the level is stored and sent whatever their state. + * * Until a level is chosen the device reports DCP_EXT_BL_MAX, and * systemd-backlight writes what it reads back when it has nothing * saved. Sending that would turn the display to full brightness. @@ -656,7 +656,7 @@ static void apple_connector_backlight_commit_work(struct work_struct *work) connector->base.name, ret); } -/* Never follow fbdev blanking, which would store a level of 0. */ +/* Never follow fbdev blanking: DRM owns the external display's power. */ static bool dcp_ext_backlight_controls_device(struct backlight_device *bd, struct device *display_dev) { diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index f3f20ed14e330..4e815759c2f6b 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1102,21 +1102,29 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, dcp_firmware_metadata_timing(dcp); } else if (!strcmp(req->key, "DisplayAttributes")) { struct dcp_parse_ctx metadata_ctx = ctx; + bool backlight_control = false, ext; - bool backlight_control; - - /* a display whose attributes do not parse gets no backlight */ - WRITE_ONCE(dcp->ext_backlight, false); ret = parse_display_attributes(&ctx, &dcp->width_mm, &dcp->height_mm, &backlight_control); + /* + * Written once, so the hotplug work never sees a working display + * without its backlight; a display whose attributes do not parse + * gets none. The attributes can arrive after the hotplug work has + * run, so sync the backlight when the answer changes. + */ + ext = !ret && backlight_control && !dcp_has_panel(dcp); + if (ext != READ_ONCE(dcp->ext_backlight)) { + WRITE_ONCE(dcp->ext_backlight, ext); + if (dcp->connector) + schedule_work(&dcp->connector->bl_sync_wq); + } + if (ret) { dev_warn(dcp->dev, "failed to parse display attribs\n"); return false; } - WRITE_ONCE(dcp->ext_backlight, - backlight_control && !dcp_has_panel(dcp)); dcp_set_dimensions(dcp); if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { struct dcp_display_metadata metadata; From af60d14d692123ad673993b564c9a6f3c653e5d1 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:34:18 -0700 Subject: [PATCH 1542/2680] drm/apple: retain external brightness until swap submission MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A brightness commit sends the CTM asynchronously before do_swap(). If HPD drops or a Type-C retrain invalidates the mode before the CTM reply, do_swap() drops the swap although iomfb_flush() already consumed brightness.update. The level can remain unapplied until a later power cycle. Prepare external backlight fields in do_swap() after its link check instead. Retain the update when display attributes or a chosen level are not yet available. Keep the integrated panel path unchanged. [ryan: adapted to 7.2.10 — keep the firmware-versioned swap_start request; never consume the brightness flag on H17P external pipes] Assisted-by: Codex Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit b1db13d4281aeeee27390a7daa2341ff963dad9d) --- drivers/gpu/drm/apple/dcp_backlight.c | 2 +- drivers/gpu/drm/apple/iomfb_template.c | 38 ++++++++++++++------------ 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 6d2f85197004e..777932478f4b4 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -625,7 +625,7 @@ static int dcp_ext_backlight_update_status(struct backlight_device *bd) return 0; dcp = platform_get_drvdata(pdev); - /* pairs with the xchg() in iomfb_flush: it sees the level or leaves update set */ + /* Pairs with do_swap(): it sees the level or leaves update set. */ smp_store_release(&dcp->brightness.update, true); /* diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 4e815759c2f6b..de0072e8d87ca 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -2211,12 +2211,31 @@ static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_start_req) start_req = { 0 }; + struct DCP_FW_NAME(dcp_swap_submit_req) *req = &DCP_FW_UNION(dcp->swap); if (READ_ONCE(dcp->mode_state.valid) && dcp->connector && - READ_ONCE(dcp->connector->connected)) + READ_ONCE(dcp->connector->connected)) { + /* The CTM reply may arrive after a link loss. Consume the + * external level only here, once the swap can be submitted. + */ + if (!dcp_has_panel(dcp) && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL && + xchg(&dcp->brightness.update, false)) { + s32 bl = dcp_ext_backlight_value(dcp); + + if (bl >= 0) { + req->swap.bl_unk = 1; + req->swap.bl_value = bl; + req->swap.bl_power = 0x40; + } else { + /* DisplayAttributes may still be on their way. */ + smp_store_release(&dcp->brightness.update, true); + } + } dcp_swap_start(dcp, false, &start_req, dcp_swap_started, NULL); - else + } else { dcp_drm_crtc_vblank(dcp->crtc); + } } #if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) @@ -2548,21 +2567,6 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru req->swap.bl_value = dcp->brightness.dac; req->swap.bl_power = 0x40; dcp->brightness.update = false; - } else if (!dcp_has_panel(dcp) && READ_ONCE(dcp->mode_state.valid) && - dcp->connector && READ_ONCE(dcp->connector->connected) && - xchg(&dcp->brightness.update, false)) { - /* - * Consumed only when do_swap() will send the swap, so a level is - * not lost to a dropped one. Read after clearing: a newer level - * sets update again and commits. - */ - s32 bl = dcp_ext_backlight_value(dcp); - - if (bl >= 0) { - req->swap.bl_unk = 1; - req->swap.bl_value = bl; - req->swap.bl_power = 0x40; - } } if (crtc_state->color_mgmt_changed) { From 4e585a5185683016472f3705a682f116d6ea9c3a Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:34:39 -0700 Subject: [PATCH 1543/2680] drm/apple: replay external brightness after late attributes On replug or reroute, the first power-on swap can precede SupportsBacklightControl. A connector retains its chosen level, but learning the capability later only registers its backlight. With a static desktop there may be no further swap to restore the level. When syncing a connected backlight-capable output, mark its current pipeline for brightness update and queue the existing commit worker if a level has been chosen. Unset levels still leave the display alone. Assisted-by: Codex Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 68e899eb0692c0f78c8345f025e0a63dabb2a6a4) --- drivers/gpu/drm/apple/dcp_backlight.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 777932478f4b4..123de87b52b93 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -740,6 +740,18 @@ void apple_connector_backlight_sync(struct apple_connector *connector) else if (!want && connector->bl_dev) apple_connector_backlight_unregister(connector); + /* Attributes and HPD can arrive after the first swap on reconnect. + * Replay a chosen level on the current pipeline, even on a static + * desktop where userspace has no further commits to submit. + */ + if (want && READ_ONCE(connector->bl_level) >= 0) { + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + /* Publish the retained level before do_swap() consumes update. */ + smp_store_release(&dcp->brightness.update, true); + schedule_work(&connector->bl_commit_wq); + } + mutex_unlock(&connector->bl_lock); } From 53b11d0ac6aa1bd76af42eebce4d7fe2265e1915 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:35:03 -0700 Subject: [PATCH 1544/2680] drm/apple: quiesce connector workers before DCP unbind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unbinding apple-drm unregisters connector backlights but only cancels hotplug and backlight-sync work much later, at connector destruction. A pending hotplug can retrain a CRTC after component_unbind_all() has freed DCP endpoints, its clock or its RTKit instance; backlight-sync also dereferences that component state. Disable and drain all connector work before unregister and component unbind, including bind-error paths. Disabling also rejects work queued by callbacks during DCP shutdown. Do this outside connector and modeset locks, before the resources used by those workers are released. [ryan: adapted to 7.2.10 — keep drm_dev_unplug() for unbind; quiesce connector work before it] Assisted-by: Codex Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit d2b7591cfc74db3b0434a771ed3d120d801797be) --- drivers/gpu/drm/apple/apple_drv.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index b7595a1aeec3b..13aab4aede1fc 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -687,6 +687,26 @@ static bool apple_dcp_usable(struct device_node *np) dcp_h17p_external_supported(np)); } +static void apple_drm_quiesce_connectors(struct drm_device *drm) +{ + struct drm_connector_list_iter iter; + struct drm_connector *connector; + + drm_connector_list_iter_begin(drm, &iter); + drm_for_each_connector_iter(connector, &iter) { + struct apple_connector *apple_connector = to_apple_connector(connector); + + /* DCP callbacks can still enqueue these until component unbind + * has drained RTKit. Stop their users before releasing endpoints, + * clocks and IOMMUs, and reject enqueues during that release. + */ + disable_work_sync(&apple_connector->hotplug_wq); + disable_work_sync(&apple_connector->bl_sync_wq); + disable_work_sync(&apple_connector->bl_commit_wq); + } + drm_connector_list_iter_end(&iter); +} + static int apple_drm_init_dcp(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); @@ -803,7 +823,7 @@ static int apple_drm_init(struct device *dev) ret = drmm_mode_config_init(&apple->drm); if (ret) - goto err_unbind; + goto err_components; /* * IOMFB::UPPipeDCP_H13P::verify_surfaces produces the error "plane @@ -851,6 +871,8 @@ static int apple_drm_init(struct device *dev) return 0; err_unbind: + apple_drm_quiesce_connectors(&apple->drm); +err_components: component_unbind_all(dev, NULL); return ret; } @@ -859,6 +881,7 @@ static void apple_drm_uninit(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); + apple_drm_quiesce_connectors(&apple->drm); drm_dev_unplug(&apple->drm); drm_atomic_helper_shutdown(&apple->drm); From 38969ce61f95224033abb40d8aa2b1e5a053de8c Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:37:41 -0700 Subject: [PATCH 1545/2680] drm/apple: clear backlight capability when a sink is replaced MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The firmware hotplug callback updates connector->connected directly and never calls disconnected_hpd_event(). It therefore retains ext_backlight from the previous sink, unlike a Type-C cable teardown. On a subsequent HPD assertion before fresh attributes, an ordinary monitor can acquire a backlight device and receive a retained Studio Display brightness level. Activating a Type-C route can also inherit capability from the fixed output it replaces. Clear the capability on firmware-reported disconnects and when activating a different output, after filtering synthetic Type-C DPMS disconnects. The next sink must advertise its own support before receiving backlight fields. [ryan: adapted to 7.2.10 — clear the capability before the external H17P hotplug hand-off can return early] Assisted-by: Codex Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit d096336061ce4ec64df66aee059e912319a0e0d1) --- drivers/gpu/drm/apple/dcp.c | 1 + drivers/gpu/drm/apple/iomfb_template.c | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 915e9418a68a4..54350ff436884 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -416,6 +416,7 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, dcp->phy = route->phy; dcp->dptx_phy = route->dptx_phy; dcp->connector_type = DRM_MODE_CONNECTOR_USB; + WRITE_ONCE(dcp->ext_backlight, false); if (connector) { WRITE_ONCE(connector->dcp, to_platform_device(dcp->dev)); dcp->typec_connector = connector; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index de0072e8d87ca..f1561b0fe7884 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1996,6 +1996,13 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) if (dcp_is_typec_output(dcp) && *connected && dcp->nr_modes) complete_all(&dcp->typec_iomfb_hpd_ready); + /* A firmware HPD loss can bypass disconnected_hpd_event(), for + * example on the fixed output. Do not lend its capability to the next + * sink while that sink's DisplayAttributes are still arriving. + */ + if (!*connected) + WRITE_ONCE(dcp->ext_backlight, false); + if (dcp_external_hotplug(dcp, !!(*connected))) return; From fbd0ffd445e8699757d331dd8377e2a237b44c71 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:40:43 -0700 Subject: [PATCH 1546/2680] drm/apple: detach the Type-C fabric from an unbound DRM device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The DCP platform devices and Type-C mux providers survive apple-drm unbind. Their port connectors and DCP CRTC/connector pointers still reference objects freed by DRM cleanup. A subsequent mux notification or tunnel teardown can dereference those stale objects or queue hotplug/backlight work into freed connector memory. DPTX ports also retain enabled=true and a service pointer into the released AFK endpoint, and reject replacement services on rebind. After component unbind drains firmware callbacks, clear the fabric and DCP references to this DRM instance under the fabric lock, including bind-error cleanup. Serialize DPTX endpoint shutdown with mux/tunnel callbacks and clear its port service state before they can use it again. Retain physical route ownership so a later bind can adopt it. [ryan: adapted to 7.2.10 — leave the CRTC/connector references of a retain_dma DCP, which pinned this DRM device for its late firmware callbacks] Assisted-by: Codex Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit d72c51e384426a7f7c67494d97d1d73810e43fcc) --- drivers/gpu/drm/apple/apple_drv.c | 4 +++ drivers/gpu/drm/apple/dcp.c | 53 +++++++++++++++++++++++++++++++ drivers/gpu/drm/apple/dcp.h | 1 + 3 files changed, 58 insertions(+) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 13aab4aede1fc..946f3a0af206c 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -872,6 +872,9 @@ static int apple_drm_init(struct device *dev) err_unbind: apple_drm_quiesce_connectors(&apple->drm); + component_unbind_all(dev, NULL); + dcp_unlink(&apple->drm); + return ret; err_components: component_unbind_all(dev, NULL); return ret; @@ -887,6 +890,7 @@ static void apple_drm_uninit(struct device *dev) component_unbind_all(dev, NULL); + dcp_unlink(&apple->drm); dev_set_drvdata(dev, NULL); } diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 54350ff436884..7568458f90e27 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3300,6 +3300,47 @@ void dcp_link(struct platform_device *pdev, struct apple_crtc *crtc, } } +/* Called after component unbind has drained all firmware callbacks. The + * platform devices and their Type-C muxes can outlive this DRM instance. + */ +void dcp_unlink(struct drm_device *drm) +{ + struct apple_dcp_typec_port *port; + struct drm_crtc *crtc; + + guard(mutex)(&dcp_typec_fabric_lock); + + list_for_each_entry(port, &dcp_typec_ports, link) { + if (port->connector && port->connector->base.dev == drm) + port->connector = NULL; + if (port->secondary_connector && + port->secondary_connector->base.dev == drm) + port->secondary_connector = NULL; + } + + drm_for_each_crtc(crtc, drm) { + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + struct apple_dcp *dcp; + + if (!apple_crtc->dcp) + continue; + dcp = platform_get_drvdata(apple_crtc->dcp); + if (dcp->crtc != apple_crtc) + continue; + /* + * A DCP that kept its DMA pinned this DRM device for its late + * firmware callbacks; its references stay valid, keep them. + */ + if (READ_ONCE(dcp->retain_dma)) + continue; + dcp->crtc = NULL; + dcp->connector = NULL; + dcp->fixed_connector = NULL; + dcp->typec_connector = NULL; + WRITE_ONCE(dcp->ext_backlight, false); + } +} + bool dcp_fw_compat_is_12_x(struct platform_device *pdev) { @@ -4498,8 +4539,20 @@ VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, v } if (dcp->dptxep) { + /* Mux/tunnel callbacks must stop using the service before its + * endpoint is released. Firmware callbacks are drained by AFK. + */ + guard(mutex)(&dcp_typec_fabric_lock); + afk_shutdown(dcp->dptxep); dcp->dptxep = NULL; + scoped_guard(mutex, &dcp->hpd_mutex) { + for (int i = 0; i < ARRAY_SIZE(dcp->dptxport); i++) { + dcp->dptxport[i].enabled = false; + dcp->dptxport[i].connected = false; + dcp->dptxport[i].service = NULL; + } + } } if (!IS_ERR_OR_NULL(dcp->dptx_controller_ep)) { diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 266083c14a546..bb56b6607b065 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -57,6 +57,7 @@ bool dcp_fw_compat_is_12_x(struct platform_device *pdev); unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev); void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, struct apple_connector *connector); +void dcp_unlink(struct drm_device *drm); int dcp_start(struct platform_device *pdev); int dcp_wait_ready(struct platform_device *pdev, u64 timeout); void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit); From 10c076bb08e8d4184682b1f4de34fa207d74509f Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:42:58 -0700 Subject: [PATCH 1547/2680] drm/apple: stop panel backlight users before DCP teardown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The panel backlight is registered by asynchronous work after the component bind devres group has closed. Its devres can therefore live until platform-device removal, beyond apple-drm unbind and CRTC destruction. A subsequent brightness write dereferences the freed CRTC. Panel backlight work is also only canceled after DCP endpoints and its IOMMU have been released. Disable and drain both panel backlight workers at the start of component unbind, then explicitly unregister the managed backlight while its resources still exist. This also prevents late firmware callbacks from re-registering it during shutdown and lets a later bind create a fresh backlight. [ryan: adapted to 7.2.10 — the panel backlight works are initialised at probe here, so cancel them (twice: before the endpoints go and again after) instead of disabling them, and stop the fade actor first] Assisted-by: Codex Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 8d12238c30f115e13f614ed98b536d61982e3189) --- drivers/gpu/drm/apple/dcp.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 7568458f90e27..dde910f3684ac 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -4532,6 +4532,25 @@ VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, v dcp_disable_typec_work(dcp, true); + if (dcp->fixed_connector_type == DRM_MODE_CONNECTOR_eDP) { + /* Registration runs asynchronously and its devres can belong + * to the platform device rather than the component bind group. + * Stop callbacks and userspace writes while the CRTC, RTKit and + * IOMMU they use are still available. The works are set up at + * probe, so cancel rather than disable them: a rebind uses them. + * Firmware callbacks may still queue them until the endpoints + * are shut down, so they are cancelled again below. + */ + cancel_delayed_work_sync(&dcp->bl_fallback_wq); + dcp_backlight_stop(dcp); + cancel_work_sync(&dcp->bl_register_wq); + cancel_work_sync(&dcp->bl_update_wq); + if (dcp->brightness.bl_dev) { + devm_backlight_device_unregister(dev, dcp->brightness.bl_dev); + dcp->brightness.bl_dev = NULL; + } + } + if (dcp->avep) { av_service_disconnect(dcp); afk_shutdown(dcp->avep); From a6253cd5f4ffc66f2e105f7ffb69f96e66e58356 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:44:20 -0700 Subject: [PATCH 1548/2680] drm/apple: reject late vblank work during component unbind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit component_unbind() releases the managed RTKit instance only after dcp_comp_unbind() returns. A callback delivered while RTKit drains can therefore queue vblank or swap-watchdog work after cancel_work_sync() has returned. That work can outlive CRTC teardown and dereference a stale CRTC. Disable and drain both work items before endpoint and IOMMU teardown, so late firmware callbacks cannot enqueue them again. Component bind reinitializes them for the next DRM instance. [ryan: adapted to 7.2.10 — this tree sets the works up at probe, not bind: record that unbind disabled them and enable them again at the next bind] Assisted-by: Codex Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit b300d661702e0d9bce8e70d3290a75ddcbdf8668) --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/dcp.c | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index d177e6aba274d..86eca0c186ef2 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -222,6 +222,8 @@ struct apple_dcp { bool retain_dma; bool quiescing; bool drm_retained; + /* vblank_wq and swap_watchdog_wq disabled by the last unbind */ + bool vblank_works_disabled; struct device *dev; struct platform_device *piodma; bool piodma_created; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index dde910f3684ac..f26a6d4770670 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -4292,6 +4292,12 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) return dev_err_probe(dev, -EBUSY, "Live DCP session cannot be rebound; reboot required\n"); + if (dcp->vblank_works_disabled) { + enable_work(&dcp->vblank_wq); + enable_delayed_work(&dcp->swap_watchdog_wq); + dcp->vblank_works_disabled = false; + } + if (dcp->external_prepared) return 0; @@ -4531,6 +4537,13 @@ VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, v disable_irq(dcp->hdmi_hpd_irq); dcp_disable_typec_work(dcp, true); + /* RTKit is released after this unbind callback, and can still deliver + * a final swap completion or hotplug while its receive queue drains. + * The works are set up at probe; the next bind enables them again. + */ + disable_delayed_work_sync(&dcp->swap_watchdog_wq); + disable_work_sync(&dcp->vblank_wq); + dcp->vblank_works_disabled = true; if (dcp->fixed_connector_type == DRM_MODE_CONNECTOR_eDP) { /* Registration runs asynchronously and its devres can belong @@ -4607,9 +4620,6 @@ VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, v cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); } - cancel_delayed_work_sync(&dcp->swap_watchdog_wq); - cancel_work_sync(&dcp->vblank_wq); - devm_clk_put(dev, dcp->clk); dcp->clk = NULL; /* optional and possibly NULL, released with the bind's devres group */ From 35e96cb83a677588cedba10b6cbed3e5719b7ebd Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 09:47:08 -0700 Subject: [PATCH 1549/2680] drm/apple, media: apple: avd: T6030 internal panel, external display and AVD fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bring up the T6030 internal panel and notch, the DCP 14.7 IOMFB interface and the external display (DCPEXT) path over Thunderbolt, and take the AVD playback reliability series. [ryan: adapted to 7.2.10 — drivers/media/platform/apple/avd/* (AVD series) excluded, it belongs to the 240-isp bucket. Asahi's 14.7 IOMFB template moved to iomfb_v14_7_tpl.c/.h (push/pop_macro guarded header) under the new DCP_FIRMWARE_V_14_7_TEMPLATE; generic 14.7.0 M1/M2 DCPs still map to it with the template poweron/poweroff/sleep/start/modeset/flush/shutdown and v13_5 audio paths; only apple,t6030-dcp/-dcpext use the native DCP_FIRMWARE_V_14_7 backend kept as iomfb_v14_7.c/.h. dcp-link.h merged: our H17P payload helpers keep their signatures (apple_dcp_h17p_rt_bw_encode_reply(request, reply, ...)), the DCPLink RPC-view variant becomes apple_dcp_h17p_rt_bw_encode_reply_view(). The enum stays in version_utils.h; DPAV_CTRL_ENDPOINT aliases our DPTX_CONTROLLER_ENDPOINT (0x24). Shared-path changes gated to the native 14.7 pipeline: AFK NOTIFY/REPLY results, short IOReturn replies, unmatched DPAV binding and the START_ACK drain (via our bounded afk_recv_drain()); DPTX set_rate/activate/DidChange failure replies; the T602X DP IN routing; in_atomic() mailbox sends; the ibootep re-init -EBUSY. dptxport services by name (dcp-lpdptx-port-epic, empty class) are admitted through our dptxport_prepare() only for 14.7; ibootep keeps our H17P dispext-service ops; the route deactivate keeps our deactivate_error handling. Kept apple,dcpext in the DRM DCP match table (theirs dropped it) and exclude apple,t6030-dcpext in apple_dcp_usable(). The T6030 internal bind/unbind enable/disable our probe-disabled Type-C work and the external probe enables its reconnect work; dcp_dptx_connect's out_disconnect maps to our out_withdraw. Adapted to 7.2 (drm_atomic_commit, dcp->mode_state instead of valid_mode, enumerate_modes(..., external_sdr=false)). Needs apple_pmp_report_wait_ready() from 4336e52 (170-usb).] Co-Authored-By: Claude Opus 5.5 (1M context) (cherry picked from commit 107036c9c1676a692ceced65cd300e5b8e4ca6a3) --- drivers/gpu/drm/apple/Kconfig | 1 + drivers/gpu/drm/apple/Makefile | 5 +- drivers/gpu/drm/apple/afk.c | 76 +- drivers/gpu/drm/apple/apple_drv.c | 24 +- drivers/gpu/drm/apple/av.c | 2 +- drivers/gpu/drm/apple/dcp-internal.h | 22 + drivers/gpu/drm/apple/dcp-lifecycle.h | 81 ++ drivers/gpu/drm/apple/dcp-link.h | 738 +++++++++- drivers/gpu/drm/apple/dcp.c | 320 ++++- drivers/gpu/drm/apple/dcp.h | 1 + drivers/gpu/drm/apple/dcpext_drm.c | 299 ++++ drivers/gpu/drm/apple/dcpext_drm.h | 11 + drivers/gpu/drm/apple/dcpext_mode.h | 11 + drivers/gpu/drm/apple/dcpext_scanout.c | 591 ++++++++ drivers/gpu/drm/apple/dcpext_scanout.h | 17 + drivers/gpu/drm/apple/dptxep.c | 179 ++- drivers/gpu/drm/apple/dptxep.h | 3 + drivers/gpu/drm/apple/ibootep.c | 558 +++++++- drivers/gpu/drm/apple/ibootep.h | 22 + drivers/gpu/drm/apple/iomfb.c | 17 +- drivers/gpu/drm/apple/iomfb_h17p.c | 2 +- drivers/gpu/drm/apple/iomfb_v12_3.c | 2 +- drivers/gpu/drm/apple/iomfb_v13_3.c | 2 +- drivers/gpu/drm/apple/iomfb_v14_7.c | 1633 ++++++++++++++++++++-- drivers/gpu/drm/apple/iomfb_v14_7.h | 31 +- drivers/gpu/drm/apple/iomfb_v14_7_link.c | 464 ++++++ drivers/gpu/drm/apple/iomfb_v14_7_link.h | 84 ++ drivers/gpu/drm/apple/iomfb_v14_7_swap.h | 118 ++ drivers/gpu/drm/apple/iomfb_v14_7_tpl.c | 125 ++ drivers/gpu/drm/apple/iomfb_v14_7_tpl.h | 37 + drivers/gpu/drm/apple/plane.c | 29 + drivers/gpu/drm/apple/plane.h | 4 + drivers/gpu/drm/apple/version_utils.h | 10 + 33 files changed, 5348 insertions(+), 171 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp-lifecycle.h create mode 100644 drivers/gpu/drm/apple/dcpext_drm.c create mode 100644 drivers/gpu/drm/apple/dcpext_drm.h create mode 100644 drivers/gpu/drm/apple/dcpext_mode.h create mode 100644 drivers/gpu/drm/apple/dcpext_scanout.c create mode 100644 drivers/gpu/drm/apple/dcpext_scanout.h create mode 100644 drivers/gpu/drm/apple/ibootep.h create mode 100644 drivers/gpu/drm/apple/iomfb_v14_7_link.c create mode 100644 drivers/gpu/drm/apple/iomfb_v14_7_link.h create mode 100644 drivers/gpu/drm/apple/iomfb_v14_7_swap.h create mode 100644 drivers/gpu/drm/apple/iomfb_v14_7_tpl.c create mode 100644 drivers/gpu/drm/apple/iomfb_v14_7_tpl.h diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 8445d82991a93..621beb052be14 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -9,6 +9,7 @@ config DRM_APPLE select DRM_CLIENT_SELECTION select DRM_KMS_HELPER select DRM_KMS_DMA_HELPER + select DRM_GEM_SHMEM_HELPER select DRM_GEM_DMA_HELPER select VIDEOMODE_HELPERS depends on BACKLIGHT_CLASS_DEVICE diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 85cf69e5a1e04..11500d7e54a7a 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -8,9 +8,12 @@ appledrm-$(CONFIG_DRM_APPLE_AUDIO) += audio.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += av.o appledrm-y += connector.o dcp-display-metadata.o dcp-180hz.o appledrm-y += ibootep.o +appledrm-y += dcpext_scanout.o +appledrm-y += dcpext_drm.o appledrm-y += iomfb_v12_3.o appledrm-y += iomfb_v13_3.o -appledrm-y += iomfb_v14_7.o +appledrm-y += iomfb_v14_7_tpl.o +appledrm-y += iomfb_v14_7.o iomfb_v14_7_link.o appledrm-y += iomfb_h17p.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += iomfb_h17p_test.o parser_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple_resume_test.o diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index b59d9edf22f0a..c57520bb68421 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -495,6 +495,15 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, else ops = afk_match_service(ep, service_name); + /* T6030 14.7: the DPAV controller endpoint binds whatever it announces. */ + if (!ops && ep->dcp->fw_compat == DCP_FIRMWARE_V_14_7 && ep->ops && + !strcmp(ep->ops->name, "dcpav-controller-epic")) { + dev_info(ep->dcp->dev, + "AFK[ep:%02x]: binding unmatched DPAV service %s on channel %d\n", + ep->endpoint, service_name, channel); + ops = ep->ops; + } + if (!ops) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: unable to match service %s on channel %d\n", @@ -791,6 +800,38 @@ static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, return; } + /* + * 14.7 answers a command to an unpublished interface with a NOTIFY + * whose category is REPLY and whose payload is one IOReturn + * (kIOReturnNoDevice is 0xe00002c0). That is the command result. + */ + if (ep->dcp->fw_compat == DCP_FIRMWARE_V_14_7 && + eshdr->category == EPIC_CAT_REPLY && payload_size >= 4 && + payload_size < sizeof(struct epic_cmd)) { + u32 rc; + u16 tag = le16_to_cpu(eshdr->tag); + u8 idx = tag & 0xff; + unsigned long flags; + + memcpy(&rc, payload, sizeof(rc)); + rc = le32_to_cpu(rc); + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: channel %d short reply type %#x tag %#x ret %#x\n", + ep->endpoint, channel, type, tag, rc); + if (idx < MAX_PENDING_CMDS) { + spin_lock_irqsave(&service->lock, flags); + if (!service->cmds[idx].done && + service->cmds[idx].tag == tag) { + service->cmds[idx].done = true; + service->cmds[idx].retcode = rc; + if (service->cmds[idx].completion) + complete(service->cmds[idx].completion); + } + spin_unlock_irqrestore(&service->lock, flags); + } + return; + } + dev_err(ep->dcp->dev, "AFK[ep:%02x]: channel %d received unhandled standard service message: %x / %x\n", ep->endpoint, channel, type, eshdr->category); @@ -1169,7 +1210,21 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, subtype == EPIC_SUBTYPE_TEARDOWN) return afk_recv_handle_teardown(ep, channel); - if (type == EPIC_TYPE_REPLY && eshdr->category == EPIC_CAT_REPLY) + /* + * T6030 14.7 returns a command result as a NOTIFY whose category is + * REPLY and whose body is a normal epic_cmd. The tag check inside the + * reply handler ignores anything that is not one of our commands. + */ + if (ep->dcp->fw_compat == DCP_FIRMWARE_V_14_7 && + (type == EPIC_TYPE_REPLY || type == EPIC_TYPE_NOTIFY) && + eshdr->category == EPIC_CAT_REPLY && + payload_size >= sizeof(struct epic_cmd)) + return afk_recv_handle_reply(ep, channel, + le16_to_cpu(eshdr->tag), payload, + payload_size); + + if (ep->dcp->fw_compat != DCP_FIRMWARE_V_14_7 && + type == EPIC_TYPE_REPLY && eshdr->category == EPIC_CAT_REPLY) return afk_recv_handle_reply(ep, channel, le16_to_cpu(eshdr->tag), payload, payload_size); @@ -1310,6 +1365,10 @@ static void afk_receive_message_worker(struct work_struct *work_) break; case RBEP_START_ACK: + /* T6030 14.7 may queue messages before it acks the start. */ + if (work->ep->dcp->fw_compat == DCP_FIRMWARE_V_14_7 && + work->ep->rxbfr.ready) + afk_recv_drain(work->ep); complete_all(&work->ep->started); break; @@ -1795,6 +1854,21 @@ static int afk_send_command_timeout_len(struct apple_epic_service *service, u8 t "AFK[ep:%02x]: %s (chan:%u) command type 0x%x tag 0x%04x timed out after %u ms\n", ep->endpoint, service->ops->name, service->channel, type, tag, timeout_ms); + if (ep->dcp->external && ep->txbfr.ready && ep->rxbfr.ready) { + /* Diagnostic snapshot of shared RAM, not coprocessor MMIO. + * Unequal TX pointers mean firmware has not drained the queue. + */ + dma_rmb(); + dev_warn(ep->dcp->dev, + "AFK timeout channel=%u tag=%#x TX r/w=%#x/%#x RX r/w=%#x/%#x running=%d crashed=%d\n", + service->channel, tag, + le32_to_cpu(READ_ONCE(*ep->txbfr.rptr)), + le32_to_cpu(READ_ONCE(*ep->txbfr.wptr)), + le32_to_cpu(READ_ONCE(*ep->rxbfr.rptr)), + le32_to_cpu(READ_ONCE(*ep->rxbfr.wptr)), + apple_rtkit_is_running(ep->dcp->rtk), + apple_rtkit_is_crashed(ep->dcp->rtk)); + } afk_service_put(service); return -ETIMEDOUT; } diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 946f3a0af206c..04234d75f4334 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -464,6 +464,17 @@ static int apple_probe_per_dcp(struct device *dev, if (bitmap_empty(iomfb_surfaces, DCP_MAX_PLANES)) return dev_err_probe(dev, -EINVAL, "No usable display surfaces\n"); + if (dcp_fw_compat_is_14_7(dcp)) { + planes[0] = apple_plane_init_v14_7(drm, 1U << num, 0, + DRM_PLANE_TYPE_PRIMARY); + if (IS_ERR(planes[0])) + return PTR_ERR(planes[0]); + ret = drm_plane_create_zpos_immutable_property(planes[0], 0); + if (ret) + return ret; + zpos = 1; + } + /* * BUG: * Some DCPs support more than two surfaces. For some reason, these @@ -474,6 +485,8 @@ static int apple_probe_per_dcp(struct device *dev, * from the surface. */ for_each_set_bit(surf, iomfb_surfaces, DCP_MAX_PLANES) { + if (dcp_fw_compat_is_14_7(dcp)) + break; plane_type = (zpos == 0) ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY; interchange = (zpos <= 1 || surf != top_surf); @@ -676,13 +689,20 @@ static int apple_get_fb_resource(struct device *dev, const char *name, static const struct of_device_id apple_dcp_id_tbl[] = { { .compatible = "apple,dcp" }, { .compatible = "apple,dcpext" }, + /* T6030: apple_dcp_usable() keeps its dcpext outside the DRM device. */ + { .compatible = "apple,t6030-dcp" }, {}, }; -/* A supported firmware-described external provider participates in component matching. */ +/* + * A supported firmware-described external provider participates in component + * matching. The T6030 external processor runs a separate session and never + * joins this DRM device. + */ static bool apple_dcp_usable(struct device_node *np) { return of_device_is_available(np) && + !of_device_is_compatible(np, "apple,t6030-dcpext") && (!of_device_is_compatible(np, "apple,t8140-dcpext") || dcp_h17p_external_supported(np)); } @@ -975,6 +995,7 @@ static void apple_platform_remove(struct platform_device *pdev) static const struct of_device_id of_match[] = { { .compatible = "apple,display-subsystem" }, + { .compatible = "apple,t6030-display-subsystem" }, {} }; MODULE_DEVICE_TABLE(of, of_match); @@ -1045,7 +1066,6 @@ static struct platform_driver apple_platform_driver = { }; - static int __init appledrm_register(void) { if (drm_firmware_drivers_only()) diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c index 049fcc5dae086..052950d7a4ff7 100644 --- a/drivers/gpu/drm/apple/av.c +++ b/drivers/gpu/drm/apple/av.c @@ -394,7 +394,7 @@ int avep_init(struct apple_dcp *dcp) audiosrv_data->cmds = dcp_av_audio_cmds_v12_3; break; case DCP_FIRMWARE_V_13_5: - case DCP_FIRMWARE_V_14_7: + case DCP_FIRMWARE_V_14_7_TEMPLATE: audiosrv_data->cmds = dcp_av_audio_cmds_v13_5; break; default: diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 86eca0c186ef2..3c957afe53113 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -28,6 +28,7 @@ #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" #include "iomfb_v14_7.h" +#include "iomfb_v14_7_tpl.h" #include "epic/dpavservep.h" #include "version_utils.h" @@ -96,6 +97,11 @@ enum { DCP_EXPERT_ENDPOINT = 0x22, DISP0_ENDPOINT = 0x23, DPTX_CONTROLLER_ENDPOINT = 0x24, + /* + * The same endpoint on T6030 14.7 firmware: the DPAV controller that + * publishes the port service used by DPTX 0x2a. + */ + DPAV_CTRL_ENDPOINT = 0x24, DPAVSERV_ENDPOINT = 0x28, DPAVSERV_ENDPOINT_V2 = 0x2a, AV_ENDPOINT = 0x29, @@ -217,6 +223,8 @@ enum dcp_test_point { }; /* TODO: move IOMFB members to its own struct */ +struct dcpext_scanout; + struct apple_dcp { /* An adopted session has no confirmed firmware DMA stop boundary. */ bool retain_dma; @@ -262,6 +270,15 @@ struct apple_dcp { /* firmware version and compatible firmware version */ enum dcp_firmware_version fw_compat; + bool external; + struct dcpext_scanout *dcpext_scanout; + bool external_link_ready; /* Full connect/HPD handshake completed. */ + bool external_suspended; /* hpd_mutex serializes PM against explicit start */ + struct work_struct external_work; + atomic_t external_requested; + + /* DCP_FIRMWARE_V_14_7 state; outlives this device once RTKit runs. */ + struct apple_dcp_v14 *v14; /* Coprocessor control register */ void __iomem *coproc_reg; @@ -428,6 +445,11 @@ struct apple_dcp { struct apple_dcp_afkep *dptx_controller_ep; struct completion dptx_controller_ready; + struct apple_dcp_afkep *dpavctrlep; + struct apple_epic_service *dpav_ctrl; + struct completion dpav_ctrl_ready; + bool dpav_ctrl_open; + struct dptx_port dptxport[2]; /* debugfs entries */ diff --git a/drivers/gpu/drm/apple/dcp-lifecycle.h b/drivers/gpu/drm/apple/dcp-lifecycle.h new file mode 100644 index 0000000000000..3b370a1245e1b --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-lifecycle.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright The Asahi Linux Contributors */ + +#ifndef __APPLE_DCP_LIFECYCLE_H__ +#define __APPLE_DCP_LIFECYCLE_H__ + +#include +#include +#include + +#define APPLE_DCP_COPROC_CPU_CONTROL_RUN BIT(4) +#define APPLE_DCP_COPROC_CPU_CONTROL_STOP_SECOND_CLEAR BIT(5) +#define APPLE_DCP_COPROC_CPU_STATUS_RUNNING BIT(0) +#define APPLE_DCP_COPROC_CPU_STATUS_STOPPED BIT(1) + +enum apple_dcp_coprocessor_start { + APPLE_DCP_COPROCESSOR_COLD_START, + APPLE_DCP_COPROCESSOR_ADOPT_AND_RESTART, + APPLE_DCP_COPROCESSOR_QUIESCED_RESTART, + APPLE_DCP_COPROCESSOR_IDLE_RESTART, + APPLE_DCP_COPROCESSOR_ASC_STOPPED_RESTART, +}; + +enum apple_dcp_loader_handoff { + APPLE_DCP_LOADER_HANDOFF_NONE, + APPLE_DCP_LOADER_HANDOFF_RTKIT_QUIESCED, + APPLE_DCP_LOADER_HANDOFF_RTKIT_IDLE, + APPLE_DCP_LOADER_HANDOFF_ASC_STOPPED, +}; + +static inline bool apple_dcp_coprocessor_is_active(u32 cpu_control, u32 cpu_status) +{ + if (cpu_control & APPLE_DCP_COPROC_CPU_CONTROL_RUN || + cpu_status & APPLE_DCP_COPROC_CPU_STATUS_RUNNING) + return true; + + return !(cpu_status & APPLE_DCP_COPROC_CPU_STATUS_STOPPED); +} + +static inline enum apple_dcp_coprocessor_start +apple_dcp_coprocessor_start_mode(enum apple_dcp_loader_handoff handoff, + bool can_adopt, + u32 cpu_control, u32 cpu_status) +{ + if (handoff == APPLE_DCP_LOADER_HANDOFF_ASC_STOPPED) + return APPLE_DCP_COPROCESSOR_ASC_STOPPED_RESTART; + if (handoff == APPLE_DCP_LOADER_HANDOFF_RTKIT_IDLE) + return APPLE_DCP_COPROCESSOR_IDLE_RESTART; + if (handoff == APPLE_DCP_LOADER_HANDOFF_RTKIT_QUIESCED) + return APPLE_DCP_COPROCESSOR_QUIESCED_RESTART; + + if (can_adopt && + apple_dcp_coprocessor_is_active(cpu_control, cpu_status)) + return APPLE_DCP_COPROCESSOR_ADOPT_AND_RESTART; + + return APPLE_DCP_COPROCESSOR_COLD_START; +} + +static inline bool +apple_dcp_needs_adoption(enum apple_dcp_coprocessor_start start_mode) +{ + return start_mode == APPLE_DCP_COPROCESSOR_ADOPT_AND_RESTART; +} + +static inline bool +apple_dcp_needs_fresh_boot(enum apple_dcp_coprocessor_start start_mode) +{ + return start_mode == APPLE_DCP_COPROCESSOR_ASC_STOPPED_RESTART; +} + +static inline u32 apple_dcp_coprocessor_stop_first(u32 control) +{ + return control & ~APPLE_DCP_COPROC_CPU_CONTROL_RUN; +} + +static inline u32 apple_dcp_coprocessor_stop_second(u32 control) +{ + return control & ~APPLE_DCP_COPROC_CPU_CONTROL_STOP_SECOND_CLEAR; +} + +#endif /* __APPLE_DCP_LIFECYCLE_H__ */ diff --git a/drivers/gpu/drm/apple/dcp-link.h b/drivers/gpu/drm/apple/dcp-link.h index ae1fcabdfc0b2..33d74777df5cf 100644 --- a/drivers/gpu/drm/apple/dcp-link.h +++ b/drivers/gpu/drm/apple/dcp-link.h @@ -4,29 +4,232 @@ #ifndef __APPLE_DCP_LINK_H__ #define __APPLE_DCP_LINK_H__ +#include #include +#include +#include #include +#include +#include #include #include /* - * H17-generation DCP firmware uses the ordinary IOMFB transport once the - * INIT/SetShmem handshake is done: the same six fixed channels (CMD 0x00000, - * OOBCMD 0x08000, ASYNC 0x40000, OOBASYNC 0x48000, CB 0x60000, - * OOBCB 0x68000), selected by the context field of the message. Records are - * allocated on each channel's stack at - * off_next = off_cur + align_up(12 + in_len + out_len, 0x40) - * which is the rule dcp_packet_start() and DCP_PACKET_ALIGNMENT implement in - * iomfb.c. + * Two users share this header: * - * This header only describes the callback payloads whose layout differs from - * older firmware. + * - The DCPLink transport (APPLE_DCP_LINK_*) of the 14.7 firmware session + * that the boot loader hands over on T6030 (iomfb_v14_7_link.c). + * + * - H17-generation DCP firmware, which uses the ordinary IOMFB transport once + * the INIT/SetShmem handshake is done: the same six fixed channels (CMD + * 0x00000, OOBCMD 0x08000, ASYNC 0x40000, OOBASYNC 0x48000, CB 0x60000, + * OOBCB 0x68000), selected by the context field of the message. Records + * are allocated on each channel's stack at + * off_next = off_cur + align_up(12 + in_len + out_len, 0x40) + * which is the rule dcp_packet_start() and DCP_PACKET_ALIGNMENT implement + * in iomfb.c. For H17P this header only describes the callback payloads + * whose layout differs from older firmware (apple_dcp_h17p_*). */ +#define APPLE_DCP_LINK_PROTOCOL_VERSION 4 +#define APPLE_DCP_LINK_ENDPOINT 0x37 +#define APPLE_DCP_LINK_STREAM_COUNT 8 +#define APPLE_DCP_LINK_STREAMS_PER_SIDE 4 +#define APPLE_DCP_LINK_STREAM_STATE_SIZE 0x68 +#define APPLE_DCP_LINK_STATE_SIZE 0x3d0 +#define APPLE_DCP_LINK_LOCAL_STREAMS_OFFSET 0x078 +#define APPLE_DCP_LINK_REMOTE_STREAMS_OFFSET 0x218 +#define APPLE_DCP_LINK_RPC_MEMORY_SIZE 0x0c0000 +#define APPLE_DCP_LINK_RPC_PAGE_SIZE 0x004000 +#define APPLE_DCP_LINK_SIDE_MEMORY_SIZE 0x040000 +#define APPLE_DCP_LINK_STREAM_HALF_SIZE 0x020000 +#define APPLE_DCP_LINK_STREAM_BUFFER_SIZE 0x008000 +#define APPLE_DCP_LINK_HEAP_OFFSET 0x080000 +#define APPLE_DCP_LINK_HEAP_SIZE 0x040000 +#define APPLE_DCP_LINK_RPC_ALIGNMENT 4 +#define APPLE_DCP_LINK_SHARED_VAR_MEMORY_SIZE 0x4000000 + +#define APPLE_DCP_LINK_INIT_FLAG_64M 0x04000000 + +#define APPLE_DCP_LINK_MSG_FIRMWARE_INIT 0 +#define APPLE_DCP_LINK_MSG_READY 1 +#define APPLE_DCP_LINK_MSG_RPC 2 +#define APPLE_DCP_LINK_MSG_RPC_REPLY 0x42 +#define APPLE_DCP_LINK_MSG_INIT 0x40 +#define APPLE_DCP_LINK_MSG_REMOTE BIT_ULL(8) +#define APPLE_DCP_LINK_MSG_NESTED BIT_ULL(9) +#define APPLE_DCP_LINK_MSG_STREAM GENMASK_ULL(15, 10) +#define APPLE_DCP_LINK_MSG_OFFSET GENMASK_ULL(31, 16) +#define APPLE_DCP_LINK_MSG_SIZE GENMASK_ULL(47, 32) +#define APPLE_DCP_LINK_MSG_HASH GENMASK_ULL(47, 16) +#define APPLE_DCP_LINK_MSG_VERSION GENMASK_ULL(63, 48) +#define APPLE_DCP_LINK_REPLY_CONTROL_MASK 0xff3e + +#define APPLE_DCP_LINK_CALL_D003 0x44303033 #define APPLE_DCP_H17P_CLOCK_ID_154 0x154 #define APPLE_DCP_H17P_CLOCK_ID_194 0x194 +enum apple_dcp_link_state { + APPLE_DCP_LINK_WAIT_READY, + APPLE_DCP_LINK_READY, +}; + +enum apple_dcp_link_handoff_phase { + APPLE_DCP_LINK_HANDOFF_NONE, + APPLE_DCP_LINK_HANDOFF_FIRMWARE_INIT, + APPLE_DCP_LINK_HANDOFF_READY, +}; + +enum apple_dcp_link_adopt_action { + APPLE_DCP_LINK_ADOPT_INVALID, + APPLE_DCP_LINK_ADOPT_START_ENDPOINT, + APPLE_DCP_LINK_ADOPT_SEND_INIT, + APPLE_DCP_LINK_ADOPT_START_IOMFB, +}; + +static inline bool +apple_dcp_link_handoff_message_valid(enum apple_dcp_link_handoff_phase phase, u64 message) +{ + u8 type = message & GENMASK_ULL(1, 0); + + if (phase == APPLE_DCP_LINK_HANDOFF_FIRMWARE_INIT) + return type == APPLE_DCP_LINK_MSG_FIRMWARE_INIT && + !(message & GENMASK_ULL(7, 6)); + if (phase == APPLE_DCP_LINK_HANDOFF_READY) + return type == APPLE_DCP_LINK_MSG_READY; + + return false; +} + +static inline enum apple_dcp_link_adopt_action +apple_dcp_link_adopt_decide(enum apple_dcp_link_handoff_phase phase, + u64 message, bool descriptor_zero, + bool descriptor_valid, bool streams_idle) +{ + if (!streams_idle) + return APPLE_DCP_LINK_ADOPT_INVALID; + + switch (phase) { + case APPLE_DCP_LINK_HANDOFF_NONE: + return message == 0 && descriptor_zero ? + APPLE_DCP_LINK_ADOPT_START_ENDPOINT : + APPLE_DCP_LINK_ADOPT_INVALID; + case APPLE_DCP_LINK_HANDOFF_FIRMWARE_INIT: + return (descriptor_zero || descriptor_valid) && + apple_dcp_link_handoff_message_valid(phase, message) ? + APPLE_DCP_LINK_ADOPT_SEND_INIT : + APPLE_DCP_LINK_ADOPT_INVALID; + case APPLE_DCP_LINK_HANDOFF_READY: + return descriptor_valid && + apple_dcp_link_handoff_message_valid(phase, message) ? + APPLE_DCP_LINK_ADOPT_START_IOMFB : + APPLE_DCP_LINK_ADOPT_INVALID; + default: + return APPLE_DCP_LINK_ADOPT_INVALID; + } +} + +static inline u16 apple_dcp_link_negotiate_version(u16 remote_version) +{ + return remote_version < APPLE_DCP_LINK_PROTOCOL_VERSION ? + remote_version : APPLE_DCP_LINK_PROTOCOL_VERSION; +} + +enum apple_dcp_link_handshake_action { + APPLE_DCP_LINK_HANDSHAKE_NONE, + APPLE_DCP_LINK_HANDSHAKE_START_IOMFB, +}; + +enum apple_dcp_h17p_callback { + APPLE_DCP_H17P_CB_DID_BOOT = 0x44303030, + APPLE_DCP_H17P_CB_DID_POWER_ON = 0x44303031, + APPLE_DCP_H17P_CB_WILL_POWER_OFF = 0x44303032, + APPLE_DCP_H17P_CB_RT_BANDWIDTH = APPLE_DCP_LINK_CALL_D003, + APPLE_DCP_H17P_CB_FRAME_SYNC = 0x44303036, + APPLE_DCP_H17P_CB_MATCH_PMU = 0x44313030, + APPLE_DCP_H17P_CB_START_HARDWARE_BOOT = 0x44313230, + APPLE_DCP_H17P_CB_ALLOCATE_BANDWIDTH = 0x44313239, + APPLE_DCP_H17P_CB_MAP_BUF = 0x44323031, + APPLE_DCP_H17P_CB_UNMAP_BUF = 0x44323032, + APPLE_DCP_H17P_CB_MATCH_PMU_2 = 0x44323036, + APPLE_DCP_H17P_CB_MATCH_BACKLIGHT = 0x44323037, + APPLE_DCP_H17P_CB_GET_TIME = 0x44323039, + APPLE_DCP_H17P_CB_PROP_PUBLISH = 0x44333030, + APPLE_DCP_H17P_CB_GET_UINT_PROP = 0x44343031, + APPLE_DCP_H17P_CB_GET_FREQUENCY = 0x44343038, + APPLE_DCP_H17P_CB_MAP_REG = 0x44343131, + APPLE_DCP_H17P_CB_SET_PROPERTY_INT = 0x44343134, + APPLE_DCP_H17P_CB_SET_PROPERTY_BOOL = 0x44343135, + APPLE_DCP_H17P_CB_ALLOCATE_BUFFER = 0x44343531, + APPLE_DCP_H17P_CB_MAP_PHYSICAL = 0x44343532, + APPLE_DCP_H17P_CB_RELEASE_MEM_DESC = 0x44343534, + APPLE_DCP_H17P_CB_POWER_UP_DART = 0x44353732, + APPLE_DCP_H17P_CB_HOTPLUG = 0x44353735, + APPLE_DCP_H17P_CB_POWERSTATE_NOTIFY = 0x44353736, + APPLE_DCP_H17P_CB_CREATE_DEFAULT_FB = 0x44353836, + APPLE_DCP_H17P_CB_CLEAR_DEFAULT_FB = 0x44353837, + APPLE_DCP_H17P_CB_SWAP_NOTIFY = 0x44353838, + APPLE_DCP_H17P_CB_SWAP_INFO = 0x44353839, + APPLE_DCP_H17P_CB_SWAP_COMPLETE = 0x44353934, + APPLE_DCP_H17P_CB_BATCHED_SWAP_COMPLETE = 0x44353935, + APPLE_DCP_H17P_CB_SWAP_COMPLETE_INTENT = 0x44353936, + APPLE_DCP_H17P_CB_ABORT_SWAP = 0x44353937, + APPLE_DCP_H17P_CB_ENABLE_BACKLIGHT = 0x44353938, +}; + +enum apple_dcp_h17p_call { + APPLE_DCP_H17P_CALL_LATE_INIT = 0x41303030, + APPLE_DCP_H17P_CALL_SETUP_VIDEO_LIMITS = 0x41303234, + APPLE_DCP_H17P_CALL_UP_SET_CREATE_DFB = 0x41333839, + APPLE_DCP_H17P_CALL_VI_SET_TEMPERATURE_HINT = 0x41333930, + APPLE_DCP_H17P_CALL_START_SIGNAL = 0x41343031, + APPLE_DCP_H17P_CALL_SWAP_START = 0x41343036, + APPLE_DCP_H17P_CALL_SWAP_SUBMIT = 0x41343037, + APPLE_DCP_H17P_CALL_SET_DISPLAY_DEVICE = 0x41343131, + APPLE_DCP_H17P_CALL_IS_MAIN_DISPLAY = 0x41343132, + APPLE_DCP_H17P_CALL_SET_DIGITAL_OUT_MODE = 0x41343133, + APPLE_DCP_H17P_CALL_SET_MATRIX = 0x41343233, + APPLE_DCP_H17P_CALL_SET_PARAMETER = 0x41343439, + APPLE_DCP_H17P_CALL_CREATE_DEFAULT_FB = 0x41343533, + APPLE_DCP_H17P_CALL_VIDEO_POWER_SAVINGS = 0x41343537, + APPLE_DCP_H17P_CALL_FIRST_CLIENT_OPEN = 0x41343635, + APPLE_DCP_H17P_CALL_LAST_CLIENT_CLOSE = 0x41343636, + APPLE_DCP_H17P_CALL_DISPLAY_REFRESH = 0x41343734, + APPLE_DCP_H17P_CALL_FLUSH_SUPPORTS_POWER = 0x41343737, + APPLE_DCP_H17P_CALL_ABORT_SWAPS = 0x41343738, + APPLE_DCP_H17P_CALL_SET_POWER_STATE = 0x41343833, +}; + +struct apple_dcp_link_method { + u32 call; + u32 input_size; + u32 output_size; +}; + +enum apple_dcp_link_side { + APPLE_DCP_LINK_SIDE_AP = 0, + APPLE_DCP_LINK_SIDE_DCP = 1, +}; + +struct apple_dcp_link_stream_layout { + u32 local_offset; + u32 remote_offset; + u32 capacity; +}; + +struct apple_dcp_link_init_desc { + __le64 shared_dva; + __le32 flags; + __le32 version; +} __packed; + +struct apple_dcp_link_rpc_header { + __le32 call; + __le32 input_size; + __le32 output_size; +} __packed; + struct apple_dcp_h17p_clock_request { __le32 service; __le32 clock_id; @@ -46,10 +249,27 @@ struct apple_dcp_h17p_rt_bw_reply { __le32 status; } __packed; +struct apple_dcp_link_rpc_view { + u32 call; + u32 input_size; + u32 output_size; + u32 offset; + u32 total_size; + const void *input; + void *output; +}; + +static_assert(sizeof(struct apple_dcp_link_init_desc) == 0x10); +static_assert(sizeof(struct apple_dcp_link_rpc_header) == 0x0c); static_assert(sizeof(struct apple_dcp_h17p_clock_request) == 0x08); static_assert(sizeof(struct apple_dcp_h17p_rt_bw_config) == 0x10); static_assert(sizeof(struct apple_dcp_h17p_rt_bw_request) == 0x04); static_assert(sizeof(struct apple_dcp_h17p_rt_bw_reply) == 0x14); +static_assert(APPLE_DCP_LINK_STREAM_BUFFER_SIZE * + APPLE_DCP_LINK_STREAMS_PER_SIDE == + APPLE_DCP_LINK_STREAM_HALF_SIZE); +static_assert(APPLE_DCP_LINK_HEAP_OFFSET + APPLE_DCP_LINK_HEAP_SIZE == + APPLE_DCP_LINK_RPC_MEMORY_SIZE); static inline int apple_dcp_h17p_clock_rate(const struct apple_dcp_h17p_clock_request *request, @@ -70,6 +290,489 @@ apple_dcp_h17p_clock_rate(const struct apple_dcp_h17p_clock_request *request, } } +static inline int +apple_dcp_h17p_callback_method(u32 call, struct apple_dcp_link_method *method) +{ + struct apple_dcp_link_method found = { .call = call }; + + if (!method) + return -EINVAL; + + switch (call) { + case APPLE_DCP_H17P_CB_WILL_POWER_OFF: + case APPLE_DCP_H17P_CB_MATCH_PMU: + case APPLE_DCP_H17P_CB_CLEAR_DEFAULT_FB: + break; + case APPLE_DCP_H17P_CB_DID_BOOT: + case APPLE_DCP_H17P_CB_DID_POWER_ON: + case APPLE_DCP_H17P_CB_START_HARDWARE_BOOT: + case APPLE_DCP_H17P_CB_MATCH_PMU_2: + case APPLE_DCP_H17P_CB_MATCH_BACKLIGHT: + found.output_size = 4; + break; + case APPLE_DCP_H17P_CB_RT_BANDWIDTH: + found.input_size = 4; + found.output_size = 20; + break; + case APPLE_DCP_H17P_CB_FRAME_SYNC: + found.input_size = 84; + found.output_size = 80; + break; + case APPLE_DCP_H17P_CB_ALLOCATE_BANDWIDTH: + found.input_size = 28; + found.output_size = 20; + break; + case APPLE_DCP_H17P_CB_MAP_BUF: + found.input_size = 12; + found.output_size = 16; + break; + case APPLE_DCP_H17P_CB_UNMAP_BUF: + case APPLE_DCP_H17P_CB_SWAP_NOTIFY: + found.input_size = 24; + break; + case APPLE_DCP_H17P_CB_GET_TIME: + found.output_size = 8; + break; + case APPLE_DCP_H17P_CB_POWERSTATE_NOTIFY: + case APPLE_DCP_H17P_CB_ABORT_SWAP: + case APPLE_DCP_H17P_CB_ENABLE_BACKLIGHT: + found.input_size = 4; + break; + case APPLE_DCP_H17P_CB_PROP_PUBLISH: + found.input_size = 16; + break; + case APPLE_DCP_H17P_CB_SWAP_COMPLETE: + found.input_size = 1840; + break; + case APPLE_DCP_H17P_CB_SWAP_COMPLETE_INTENT: + found.input_size = 20; + break; + case APPLE_DCP_H17P_CB_GET_UINT_PROP: + found.input_size = 80; + found.output_size = 12; + break; + case APPLE_DCP_H17P_CB_GET_FREQUENCY: + found.input_size = 8; + found.output_size = 8; + break; + case APPLE_DCP_H17P_CB_MAP_REG: + found.input_size = 16; + found.output_size = 28; + break; + case APPLE_DCP_H17P_CB_SET_PROPERTY_INT: + found.input_size = 80; + found.output_size = 4; + break; + case APPLE_DCP_H17P_CB_SET_PROPERTY_BOOL: + found.input_size = 76; + found.output_size = 4; + break; + case APPLE_DCP_H17P_CB_ALLOCATE_BUFFER: + found.input_size = 20; + found.output_size = 20; + break; + case APPLE_DCP_H17P_CB_MAP_PHYSICAL: + found.input_size = 8; + found.output_size = 4; + break; + case APPLE_DCP_H17P_CB_RELEASE_MEM_DESC: + case APPLE_DCP_H17P_CB_POWER_UP_DART: + found.input_size = 4; + found.output_size = 4; + break; + case APPLE_DCP_H17P_CB_HOTPLUG: + found.input_size = 88; + found.output_size = 76; + break; + case APPLE_DCP_H17P_CB_CREATE_DEFAULT_FB: + found.input_size = 8; + found.output_size = 4; + break; + case APPLE_DCP_H17P_CB_SWAP_INFO: + found.input_size = 228; + found.output_size = 224; + break; + case APPLE_DCP_H17P_CB_BATCHED_SWAP_COMPLETE: + found.output_size = 1024; + break; + default: + return -ENOENT; + } + + *method = found; + return 0; +} + +static inline int +apple_dcp_link_stream_layout(enum apple_dcp_link_side side, u8 stream, + bool remote, + struct apple_dcp_link_stream_layout *layout) +{ + u32 local_side; + u32 remote_side; + u32 stream_offset; + + if (!layout || side > APPLE_DCP_LINK_SIDE_DCP || + stream >= APPLE_DCP_LINK_STREAMS_PER_SIDE) + return -EINVAL; + + local_side = side * APPLE_DCP_LINK_SIDE_MEMORY_SIZE; + remote_side = (side ^ 1) * APPLE_DCP_LINK_SIDE_MEMORY_SIZE; + stream_offset = stream * APPLE_DCP_LINK_STREAM_BUFFER_SIZE; + + if (remote) { + layout->local_offset = local_side + + APPLE_DCP_LINK_STREAM_HALF_SIZE + + stream_offset; + layout->remote_offset = remote_side + stream_offset; + } else { + layout->local_offset = local_side + stream_offset; + layout->remote_offset = remote_side + + APPLE_DCP_LINK_STREAM_HALF_SIZE + + stream_offset; + } + layout->capacity = APPLE_DCP_LINK_STREAM_BUFFER_SIZE; + + return 0; +} + +static inline int apple_dcp_h17p_method(u32 call, + struct apple_dcp_link_method *method) +{ + struct apple_dcp_link_method found = { .call = call }; + + if (!method) + return -EINVAL; + + switch (call) { + case APPLE_DCP_H17P_CALL_UP_SET_CREATE_DFB: + case APPLE_DCP_H17P_CALL_SETUP_VIDEO_LIMITS: + case APPLE_DCP_H17P_CALL_FIRST_CLIENT_OPEN: + break; + case APPLE_DCP_H17P_CALL_LATE_INIT: + found.input_size = 4; + found.output_size = 4; + break; + case APPLE_DCP_H17P_CALL_START_SIGNAL: + case APPLE_DCP_H17P_CALL_VI_SET_TEMPERATURE_HINT: + case APPLE_DCP_H17P_CALL_IS_MAIN_DISPLAY: + case APPLE_DCP_H17P_CALL_CREATE_DEFAULT_FB: + case APPLE_DCP_H17P_CALL_DISPLAY_REFRESH: + found.output_size = 4; + break; + case APPLE_DCP_H17P_CALL_SWAP_START: + case APPLE_DCP_H17P_CALL_SET_POWER_STATE: + found.input_size = 16; + found.output_size = 8; + break; + case APPLE_DCP_H17P_CALL_SWAP_SUBMIT: + found.input_size = 0x1d30; + found.output_size = 12; + break; + case APPLE_DCP_H17P_CALL_SET_DISPLAY_DEVICE: + case APPLE_DCP_H17P_CALL_VIDEO_POWER_SAVINGS: + case APPLE_DCP_H17P_CALL_LAST_CLIENT_CLOSE: + found.input_size = 4; + found.output_size = 4; + break; + case APPLE_DCP_H17P_CALL_SET_DIGITAL_OUT_MODE: + case APPLE_DCP_H17P_CALL_ABORT_SWAPS: + found.input_size = 8; + found.output_size = 4; + break; + case APPLE_DCP_H17P_CALL_SET_MATRIX: + found.input_size = 80; + found.output_size = 4; + break; + case APPLE_DCP_H17P_CALL_SET_PARAMETER: + found.input_size = 40; + found.output_size = 4; + break; + case APPLE_DCP_H17P_CALL_FLUSH_SUPPORTS_POWER: + found.input_size = 4; + break; + default: + return -ENOENT; + } + + *method = found; + return 0; +} + +static inline void +apple_dcp_link_init_desc_encode(struct apple_dcp_link_init_desc *desc, + u64 shared_var_dva, bool use_64m_window) +{ + desc->shared_dva = cpu_to_le64(shared_var_dva); + desc->flags = cpu_to_le32(use_64m_window ? APPLE_DCP_LINK_INIT_FLAG_64M : 0); + desc->version = cpu_to_le32(APPLE_DCP_LINK_PROTOCOL_VERSION); +} + +static inline bool +apple_dcp_link_init_desc_valid(const struct apple_dcp_link_init_desc *desc) +{ + u64 shared_dva; + u32 flags; + + if (!desc) + return false; + + shared_dva = le64_to_cpu(desc->shared_dva); + flags = le32_to_cpu(desc->flags); + + if (le32_to_cpu(desc->version) != APPLE_DCP_LINK_PROTOCOL_VERSION) + return false; + if (!shared_dva) + return flags == 0; + + return flags == APPLE_DCP_LINK_INIT_FLAG_64M && + !(shared_dva & (APPLE_DCP_LINK_RPC_PAGE_SIZE - 1)) && + !(shared_dva & ~GENMASK_ULL(47, 0)); +} + +static inline bool +apple_dcp_link_init_desc_zero(const struct apple_dcp_link_init_desc *desc) +{ + return desc && !le64_to_cpu(desc->shared_dva) && + !le32_to_cpu(desc->flags) && !le32_to_cpu(desc->version); +} + +static inline bool +apple_dcp_link_stream_memory_idle(const void *memory, size_t size) +{ + const u8 *bytes = memory; + size_t offset; + + if (!memory || size < APPLE_DCP_LINK_RPC_MEMORY_SIZE) + return false; + + for (offset = sizeof(struct apple_dcp_link_init_desc); + offset < APPLE_DCP_LINK_RPC_MEMORY_SIZE; offset++) { + if (bytes[offset]) + return false; + } + + return true; +} + +static inline int apple_dcp_link_init_message(u64 rpc_dva, u64 *message) +{ + if (!message) + return -EINVAL; + if (rpc_dva & ~GENMASK_ULL(47, 0)) + return -ERANGE; + + *message = (rpc_dva << 16) | APPLE_DCP_LINK_MSG_INIT; + return 0; +} + +static inline enum apple_dcp_link_handshake_action +apple_dcp_link_handshake_step(enum apple_dcp_link_state *state, u64 message) +{ + u8 type; + + if (!state) + return APPLE_DCP_LINK_HANDSHAKE_NONE; + + type = message & GENMASK_ULL(1, 0); + switch (type) { + case APPLE_DCP_LINK_MSG_FIRMWARE_INIT: + return APPLE_DCP_LINK_HANDSHAKE_NONE; + case APPLE_DCP_LINK_MSG_READY: + if (*state != APPLE_DCP_LINK_WAIT_READY) + return APPLE_DCP_LINK_HANDSHAKE_NONE; + *state = APPLE_DCP_LINK_READY; + return APPLE_DCP_LINK_HANDSHAKE_START_IOMFB; + default: + return APPLE_DCP_LINK_HANDSHAKE_NONE; + } +} + +static inline int apple_dcp_link_ready_reply(u64 message, u16 *version, + u32 *firmware_hash) +{ + if (!version || !firmware_hash) + return -EINVAL; + if ((message & GENMASK_ULL(1, 0)) != APPLE_DCP_LINK_MSG_READY) + return -EPROTO; + + *version = FIELD_GET(APPLE_DCP_LINK_MSG_VERSION, message); + *firmware_hash = FIELD_GET(APPLE_DCP_LINK_MSG_HASH, message); + return 0; +} + +static inline int apple_dcp_link_rpc_payload_size(u32 input_size, + u32 output_size, u32 *size) +{ + u32 total; + + if (!size) + return -EINVAL; + if (check_add_overflow(input_size, output_size, &total) || + check_add_overflow(total, + (u32)sizeof(struct apple_dcp_link_rpc_header), + &total)) + return -EOVERFLOW; + + *size = total; + return 0; +} + +static inline int apple_dcp_link_align_payload(u32 size, u32 alignment, + u32 *aligned_size) +{ + u32 padded; + + if (!aligned_size || !is_power_of_2(alignment)) + return -EINVAL; + if (check_add_overflow(size, alignment - 1, &padded)) + return -EOVERFLOW; + + *aligned_size = padded & -alignment; + return 0; +} + +static inline int apple_dcp_link_rpc_message(u32 offset, u32 size, bool nested, + u64 *message) +{ + if (!message) + return -EINVAL; + if (offset > U16_MAX || size > U16_MAX) + return -ERANGE; + + *message = APPLE_DCP_LINK_MSG_RPC | + FIELD_PREP(APPLE_DCP_LINK_MSG_OFFSET, offset) | + FIELD_PREP(APPLE_DCP_LINK_MSG_SIZE, size); + if (nested) + *message |= APPLE_DCP_LINK_MSG_NESTED; + + return 0; +} + +static inline int apple_dcp_link_stream_message(u64 message, u8 stream, + bool remote, u64 *encoded) +{ + if (!encoded) + return -EINVAL; + if (stream >= APPLE_DCP_LINK_STREAMS_PER_SIDE) + return -ERANGE; + + message &= ~(APPLE_DCP_LINK_MSG_REMOTE | APPLE_DCP_LINK_MSG_STREAM); + message |= FIELD_PREP(APPLE_DCP_LINK_MSG_STREAM, stream); + if (remote) + message |= APPLE_DCP_LINK_MSG_REMOTE; + + *encoded = message; + return 0; +} + +static inline u8 apple_dcp_link_message_stream(u64 message) +{ + return FIELD_GET(APPLE_DCP_LINK_MSG_STREAM, message); +} + +static inline bool apple_dcp_link_message_is_remote(u64 message) +{ + return message & APPLE_DCP_LINK_MSG_REMOTE; +} + +static inline u64 apple_dcp_link_rpc_reply(u64 request) +{ + return (request & (APPLE_DCP_LINK_MSG_VERSION | + APPLE_DCP_LINK_REPLY_CONTROL_MASK)) | + APPLE_DCP_LINK_MSG_RPC_REPLY; +} + +static inline int apple_dcp_link_rpc_decode(u64 message, void *buffer, + u32 capacity, + struct apple_dcp_link_rpc_view *view) +{ + struct apple_dcp_link_rpc_header *header; + u32 expected_size; + u32 offset; + u32 total_size; + int ret; + + if (!buffer || !view) + return -EINVAL; + if ((message & GENMASK_ULL(1, 0)) != APPLE_DCP_LINK_MSG_RPC || + (message & GENMASK_ULL(7, 6)) == BIT_ULL(6)) + return -EPROTO; + + offset = FIELD_GET(APPLE_DCP_LINK_MSG_OFFSET, message); + total_size = FIELD_GET(APPLE_DCP_LINK_MSG_SIZE, message); + if (offset > capacity || total_size > capacity - offset || + total_size < sizeof(*header)) + return -EMSGSIZE; + + header = buffer + offset; + ret = apple_dcp_link_rpc_payload_size(le32_to_cpu(header->input_size), + le32_to_cpu(header->output_size), + &expected_size); + if (ret) + return ret; + if (expected_size != total_size) + return -EMSGSIZE; + + view->call = le32_to_cpu(header->call); + view->input_size = le32_to_cpu(header->input_size); + view->output_size = le32_to_cpu(header->output_size); + view->offset = offset; + view->total_size = total_size; + view->input = header + 1; + view->output = (u8 *)(header + 1) + view->input_size; + return 0; +} + +static inline int apple_dcp_link_stack_push(u32 top, u32 capacity, + u32 payload_size, u32 alignment, + u32 *offset, u32 *next_top) +{ + u32 aligned_size; + int ret; + + if (!offset || !next_top || top > capacity) + return -EINVAL; + + ret = apple_dcp_link_align_payload(payload_size, alignment, &aligned_size); + if (ret) + return ret; + if (aligned_size > capacity - top) + return -ENOSPC; + + *offset = top; + *next_top = top + aligned_size; + return 0; +} + +static inline int apple_dcp_link_stack_pop(u32 top, u32 payload_size, + u32 alignment, u32 *next_top) +{ + u32 aligned_size; + int ret; + + if (!next_top) + return -EINVAL; + + ret = apple_dcp_link_align_payload(payload_size, alignment, &aligned_size); + if (ret) + return ret; + if (aligned_size > top) + return -ERANGE; + + *next_top = top - aligned_size; + return 0; +} + +static inline void +apple_dcp_link_rpc_header_encode(struct apple_dcp_link_rpc_header *header, + u32 call, u32 input_size, u32 output_size) +{ + header->call = cpu_to_le32(call); + header->input_size = cpu_to_le32(input_size); + header->output_size = cpu_to_le32(output_size); +} + static inline void apple_dcp_h17p_rt_bw_encode(struct apple_dcp_h17p_rt_bw_config *config, u64 scratch, u64 clock_request) @@ -92,4 +795,19 @@ apple_dcp_h17p_rt_bw_encode_reply(const struct apple_dcp_h17p_rt_bw_request *req return 0; } +/* The same, for a call record seen through a DCPLink RPC view. */ +static inline int +apple_dcp_h17p_rt_bw_encode_reply_view(struct apple_dcp_link_rpc_view *view, + u64 scratch, u64 clock_request) +{ + if (!view || view->call != APPLE_DCP_LINK_CALL_D003) + return -EINVAL; + if (view->input_size < sizeof(struct apple_dcp_h17p_rt_bw_request) || + view->output_size < sizeof(struct apple_dcp_h17p_rt_bw_reply)) + return -EMSGSIZE; + + return apple_dcp_h17p_rt_bw_encode_reply(view->input, view->output, + scratch, clock_request); +} + #endif /* __APPLE_DCP_LINK_H__ */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index f26a6d4770670..f322df62ead64 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -45,10 +45,12 @@ #include "afk.h" #include "av.h" #include "dcp.h" +#include "dcpext_scanout.h" #include "dcp-internal.h" #include "dcp-external-bw.h" #include "dptx-service.h" #include "iomfb.h" +#include "ibootep.h" #include "parser.h" #include "trace.h" @@ -185,6 +187,10 @@ static bool show_notch; module_param(show_notch, bool, 0644); MODULE_PARM_DESC(show_notch, "Use the full display height and shows the notch"); +static bool dcpext_probe; +module_param(dcpext_probe, bool, 0444); +MODULE_PARM_DESC(dcpext_probe, "Opt into experimental T6030 external DCP probe (may hang)"); + bool hdmi_audio; module_param(hdmi_audio, bool, 0644); MODULE_PARM_DESC(hdmi_audio, "Enable unstable HDMI audio support"); @@ -256,6 +262,13 @@ static bool dcp_typec_route_fixed_output_busy(struct apple_dcp_typec_route *rout if (dcp->fixed_connector_type == DRM_MODE_CONNECTOR_USB) return false; + /* + * The 14.7 firmware drives the internal panel on the IOMFB path. + * The DPTX service is a separate output, so the live panel does not + * occupy the Type-C route. + */ + if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) + return false; if (dcp->fixed_connector && dcp->fixed_connector->connected) return true; if (dcp->hdmi_hpd && gpiod_get_value_cansleep(dcp->hdmi_hpd)) @@ -1281,6 +1294,15 @@ static int dcp_tunnel_dpin_locked(struct apple_dcp *dcp, bool active) return dcp->tb_dpin_set_active(dcp->tb_dpin_ctx, active); } +/* Limited to the T6030 14.7 pipeline; other T602X tunnels keep their order. */ +static bool dcp_t602x_tunnel_route(struct apple_dcp_typec_route *route) +{ + return route && route->tunnel && route->active_xbar && + route->dcp->fw_compat == DCP_FIRMWARE_V_14_7 && + of_device_is_compatible(route->active_xbar->chip->dev.parent->of_node, + "apple,t6020-display-crossbar"); +} + /* * Thunderbolt DP IN, from DCP's DidChangeLinkConfiguration once a link rate * is set: bring the crossbar connection up (FIFO/PCLK/ATC enables) now that @@ -1309,6 +1331,9 @@ int dcp_tunnel_crossbar_up(struct apple_dcp *dcp) ret = mux_control_try_select(route->active_xbar, route->mux_index); if (!ret) route->xbar_up = true; + /* T602X DP IN selection reserves and routes the mux only. */ + if (!ret && dcp_t602x_tunnel_route(route)) + ret = dcp_dpxbar_link(route->active_xbar, true); } else { ret = dcp_dpxbar_link(route->active_xbar, true); } @@ -1349,6 +1374,13 @@ int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate) guard(mutex)(&dcp->tb_lock); if (!dcp->dptx_tunnel) return -ENODEV; + route = dcp->active_typec_route; + /* The initial T6030 clock port supports core0 -> DP IN0 (PCLK1). + * Other routes require explicit PCLK slot selection and accounting. + */ + if (link_rate && of_machine_is_compatible("apple,t6030") && + (!route || route->mux_index != 0 || dcp->dptx_dfp_port != 1)) + return -EOPNOTSUPP; fn = symbol_get(apple_atc_dp_tunnel_rate); if (!fn) { dev_err(dcp->dev, "phy-apple-atc not loaded, no DP tunnel clock\n"); @@ -1373,9 +1405,24 @@ int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate) /* Thunderbolt DP IN: DCP Activate/Deactivate */ int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active) { + struct apple_dcp_typec_route *route; + int ret; + guard(mutex)(&dcp->tb_lock); if (!dcp->dptx_tunnel) return 0; + /* Route the upstream engine before firmware begins AUX negotiation. + * T602X DP IN .set does not enable clocks; DidChange does that later. + */ + route = dcp->active_typec_route; + if (active && dcp_t602x_tunnel_route(route)) { + if (!route->xbar_up) { + ret = mux_control_try_select(route->active_xbar, route->mux_index); + if (ret) + return ret; + route->xbar_up = true; + } + } return dcp_tunnel_dpin_locked(dcp, active); } @@ -1386,6 +1433,13 @@ int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active) * display pipeline there and tell DCP a display is attached, so it trains * the link (and completes DPRX) through the tunnel. */ +static bool dcp_tb_services_ready(struct apple_dcp *dcp) +{ + if (!dcp->external || dcp->fw_compat != DCP_FIRMWARE_V_14_7) + return true; + return smp_load_acquire(&dcp->dptxport[0].enabled) && ibootep_is_ready(dcp); +} + int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, bool active, int (*set_active)(void *ctx, bool active), void *ctx) @@ -1394,6 +1448,7 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, struct apple_dcp_typec_route *candidate, *best = NULL, *planned = NULL; struct apple_dcp_typec_route **slot; unsigned int best_score = UINT_MAX; + bool waiting_for_external = false; struct mux_control *ctl; struct apple_dcp *dcp; bool ordered; @@ -1472,6 +1527,21 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, port->secondary_connector : port->connector)) continue; + /* A boot-present tunnel may precede the explicit firmware start. + * Do not claim its route or spend firmware reconnect attempts + * until both command services have been published. Acquire + * pairs with RemotePort publication; iBoot checks its cookie + * with acquire ordering and refuses a stopping service. + */ + /* Retained scanout after link loss must never be reactivated. */ + if (dcpext_scanout_terminal(candidate->dcp)) { + ret = -ESHUTDOWN; + goto err_reorder; + } + if (!dcp_tb_services_ready(candidate->dcp)) { + waiting_for_external = true; + continue; + } score = dcp_typec_route_score(candidate); /* * DPIN0 prefers the hybrid dcpext0, which completes tunneled @@ -1488,7 +1558,7 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, } } if (!best) { - ret = -EBUSY; + ret = waiting_for_external ? -EAGAIN : -EBUSY; goto err_reorder; } @@ -1521,17 +1591,27 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, dev_info(dcp->dev, "display routed to Thunderbolt DP tunnel dpin%u\n", dpin); + if (dcp->fw_compat == DCP_FIRMWARE_V_14_7 && dcp->dptxep && + !dcp->dptxport[0].enabled) + dev_warn(dcp->dev, "DPTX port not announced, not opening the controller\n"); + /* * The DP IN adapter may only be woken (DPTX_INACTIVE=0) while DCP * drives the DPTX, i.e. from DCP's Activate call; waking it earlier * hangs the machine. dptxep calls set_active back from Activate and * Deactivate (set above, before the route became a tunnel). */ - if (!dcp->typec_connector) + if (!dcp->typec_connector && !dcp->external) dev_warn(dcp->dev, "no Type-C connector for the DP tunnel\n"); WRITE_ONCE(dcp->typec_cable_connected, true); port->hpd = true; - if (dcp->typec_connector) + /* External firmware link bring-up precedes its DRM connector. Run it + * after returning to the tunnel manager, outside the fabric lock. + */ + if (dcp->external) { + dcp->typec_reconnect_tries = 0; + mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); + } else if (dcp->typec_connector) dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); return 0; @@ -2499,11 +2579,22 @@ static const struct apple_rtkit_ops external_h17p_rtkit_ops = { void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message) { + bool native_14_7 = dcp->fw_compat == DCP_FIRMWARE_V_14_7; + int ret; + if (READ_ONCE(dcp->quiescing)) return; trace_dcp_send_msg(dcp, endpoint, message); - apple_rtkit_send_message(dcp->rtk, endpoint, message, NULL, - true); + /* + * The adopted T6030 14.7 session shares this mailbox with the panel + * link. A non-sleeping send fails while that FIFO is full, and the + * DisplayPort handshake then waits for a reply that was never sent. + */ + ret = apple_rtkit_send_message(dcp->rtk, endpoint, message, NULL, + native_14_7 ? in_atomic() : true); + if (ret && native_14_7) + dev_warn_ratelimited(dcp->dev, "DCP send ep %02x failed: %d\n", + endpoint, ret); } int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commit) @@ -2513,6 +2604,9 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commi struct drm_crtc_state *crtc_state; bool needs_modeset; + if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) + return iomfb_v14_7_atomic_check(dcp, crtc, commit); + if (dcp->crashed) return -EINVAL; @@ -2577,14 +2671,21 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable dev_warn(dcp->dev, "dcp_dptx_connect: missing phy\n"); return -ENODEV; } + /* @port selects the upstream RemotePort service/core. dptx_dfp_port + * is the downstream address: dpphy=0, dpin0=1, dpin1=2. + */ dev_dbg(dcp->dev, - "%s(port=%d) target=%u:%u typec=%d route=%s conn_type=%d connected=%d\n", - __func__, port, dcp->dptx_die, dcp->dptx_phy, - dcp_is_typec_output(dcp), + "%s(port=%d) die=%u atc=%u dfp_port=%u tunnel=%d typec=%d route=%s conn_type=%d connected=%d\n", + __func__, port, dcp->dptx_die, dcp->dptx_phy, dcp->dptx_dfp_port, + dcp->dptx_tunnel, dcp_is_typec_output(dcp), dcp->active_typec_route ? "borrowed" : "fixed", dcp->connector_type, dcp->dptxport[port].connected); mutex_lock(&dcp->hpd_mutex); + if (dcp->external && dcpext_scanout_terminal(dcp)) { + mutex_unlock(&dcp->hpd_mutex); + return -ESHUTDOWN; + } if (!dptx_service_available(&dcp->dptxport[port])) { dev_warn(dcp->dev, "dcp_dptx_connect: dptx service for port %d not enabled\n", port); ret = -ENODEV; @@ -2598,6 +2699,8 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable if (dcp->dptxport[port].connected) goto out_unlock; + if (dcp->external) + smp_store_release(&dcp->external_link_ready, false); reinit_completion(&dcp->dptxport[port].linkcfg_completion); dcp->dptxport[port].atcphy = dcp->phy; @@ -2663,8 +2766,23 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable usleep_range(5, 10); - if (dcp->connector_type == DRM_MODE_CONNECTOR_DisplayPort) - dptxport_set_hpd(dcp->dptxport[port].service, true); + if (dcp->connector_type == DRM_MODE_CONNECTOR_DisplayPort) { + ret = dptxport_set_hpd(dcp->dptxport[port].service, true); + /* T6030 external: out_withdraw drops the request as out_disconnect did. */ + if (ret && dcp->external) + goto out_withdraw; + } + if (dcp->external) { + mutex_lock(&dcp->hpd_mutex); + if (!dcp->dptxport[port].connected || + !READ_ONCE(dcp->typec_cable_connected) || READ_ONCE(dcp->crashed) || + dcpext_scanout_terminal(dcp)) { + mutex_unlock(&dcp->hpd_mutex); + return -ENOLINK; + } + smp_store_release(&dcp->external_link_ready, true); + mutex_unlock(&dcp->hpd_mutex); + } if (dcp->avep) av_service_connect(dcp); @@ -3055,7 +3173,8 @@ static void dcp_typec_reconnect_work(struct work_struct *work) typec_reconnect_wq); int ret; - if (!READ_ONCE(dcp->typec_cable_connected)) + if ((dcp->external && dcpext_scanout_terminal(dcp)) || + !READ_ONCE(dcp->typec_cable_connected)) return; ret = dcp_dptx_connect(dcp, 0); @@ -3143,9 +3262,14 @@ static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) if (dcp_is_external_h17p(dcp)) return dcp_external_disconnect(dcp, false); + /* Release the caller's RemotePort service, not the downstream DFP port. */ dev_info(dcp->dev, "%s(port=%d)\n", __func__, port); mutex_lock(&dcp->hpd_mutex); + if (dcp->external) { + smp_store_release(&dcp->external_link_ready, false); + dcpext_scanout_invalidate(dcp); + } if (dptx_service_available(&dcp->dptxport[port]) && dcp->dptxport[port].connected) { dptxport_release_display(dcp->dptxport[port].service); dcp->dptxport[port].connected = false; @@ -3198,7 +3322,7 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) } ret = dcp_dptx_connect(dcp, port); - if (ret && dcp_is_typec_output(dcp)) + if (ret && ret != -ESHUTDOWN && dcp_is_typec_output(dcp)) mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, DPTX_RECONNECT_DELAY); @@ -3349,6 +3473,13 @@ bool dcp_fw_compat_is_12_x(struct platform_device *pdev) return dcp->fw_compat == DCP_FIRMWARE_V_12_3; } +bool dcp_fw_compat_is_14_7(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return dcp->fw_compat == DCP_FIRMWARE_V_14_7; +} + unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); @@ -3364,6 +3495,21 @@ int dcp_start(struct platform_device *pdev) /* Even if nothing starts, dcp_wait_ready() must find it initialised. */ init_completion(&dcp->start_done); + /* + * The T6030 firmware session is adopted for the internal panel. + * Open DPTX on that same RTKit only when the firmware advertised it. + */ + if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { + if (dcp->external) { + complete(&dcp->start_done); + return 0; + } + ret = iomfb_v14_7_start(dcp); + if (ret) + return ret; + return 0; + } + if (IS_ERR_OR_NULL(dcp->rtk)) return -ENODEV; @@ -3456,9 +3602,12 @@ static void _dcp_poweroff(struct apple_dcp *dcp) case DCP_FIRMWARE_V_13_5: iomfb_poweroff_v13_3(dcp); break; - case DCP_FIRMWARE_V_14_7: + case DCP_FIRMWARE_V_14_7_TEMPLATE: iomfb_poweroff_v14_7_0(dcp); break; + case DCP_FIRMWARE_V_14_7: + iomfb_v14_7_poweroff(dcp); + break; case DCP_FIRMWARE_H17P: iomfb_poweroff_h17p(dcp); break; @@ -3524,9 +3673,12 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) case DCP_FIRMWARE_V_13_5: iomfb_sleep_v13_3(dcp); break; - case DCP_FIRMWARE_V_14_7: + case DCP_FIRMWARE_V_14_7_TEMPLATE: iomfb_sleep_v14_7_0(dcp); break; + case DCP_FIRMWARE_V_14_7: + iomfb_v14_7_poweroff(dcp); + break; case DCP_FIRMWARE_H17P: iomfb_sleep_h17p(dcp); break; @@ -3630,9 +3782,12 @@ void dcp_poweron(struct platform_device *pdev) case DCP_FIRMWARE_V_13_5: iomfb_poweron_v13_3(dcp); break; - case DCP_FIRMWARE_V_14_7: + case DCP_FIRMWARE_V_14_7_TEMPLATE: iomfb_poweron_v14_7_0(dcp); break; + case DCP_FIRMWARE_V_14_7: + iomfb_v14_7_poweron(dcp); + break; case DCP_FIRMWARE_H17P: iomfb_poweron_h17p(dcp); break; @@ -4175,6 +4330,19 @@ static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) return DCP_FIRMWARE_UNKNOWN; } + /* + * T6030 runs the 14.7 firmware session handed over by the boot loader + * natively (iomfb_v14_7.c); other 14.7.0 DCPs keep the template below. + */ + if (of_device_is_compatible(dev->of_node, "apple,t6030-dcp") || + of_device_is_compatible(dev->of_node, "apple,t6030-dcpext")) { + if (ret >= 0 && !strcmp(compat_str, "14.7.0")) + return DCP_FIRMWARE_V_14_7; + dev_err(dev, "T6030 display not started: DCP firmware-compat %s is not 14.7.0\n", + compat_str); + return DCP_FIRMWARE_UNKNOWN; + } + if (strncmp(compat_str, "12.3.0", sizeof(compat_str)) == 0) return DCP_FIRMWARE_V_12_3; /* @@ -4191,7 +4359,7 @@ static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) return DCP_FIRMWARE_V_13_5; else if (strncmp(compat_str, "14.7.0", sizeof(compat_str)) == 0 && (strncmp(fw_str, "14.7.0", sizeof(compat_str)) == 0)) - return DCP_FIRMWARE_V_14_7; + return DCP_FIRMWARE_V_14_7_TEMPLATE; dev_err(dev, "DCP firmware-compat %s (FW: %s) is not supported\n", compat_str, fw_str); @@ -4378,6 +4546,17 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) dcp->connector_type = DRM_MODE_CONNECTOR_eDP; } + /* The running T6030 firmware is adopted as is. */ + if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { + if (dcp->external) + return 0; + ret = iomfb_v14_7_bind(dcp); + /* Balances the disable at probe, as below. */ + if (!ret) + dcp_enable_typec_work(dcp); + return ret; + } + ret = dcp_create_piodma_iommu_dev(dcp); if (ret) return dev_err_probe(dev, ret, @@ -4492,6 +4671,13 @@ VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, v if (!dcp) return; + if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { + if (!dcp->external) + dcp_disable_typec_work(dcp, false); + iomfb_v14_7_unbind(dcp); + return; + } + if (dcp->retain_dma) { /* No firmware stop is acknowledged: detach host users, never DMA. */ scoped_guard(mutex, &dcp->iomfb.lock) @@ -4659,7 +4845,8 @@ static int dcp_platform_probe(struct platform_device *pdev) * device trees. This prevents replacing simpledrm and ending up without * display. */ - if (!of_property_present(dev->of_node, "apple,bw-scratch") && + if (fw_compat != DCP_FIRMWARE_V_14_7 && + !of_property_present(dev->of_node, "apple,bw-scratch") && !hw->firmware_clock) return dev_err_probe(dev, -ENODEV, "Incompatible devicetree! " "Use devicetree matching this kernel.\n"); @@ -4679,6 +4866,7 @@ static int dcp_platform_probe(struct platform_device *pdev) iomfb_queue_init(dcp); dcp->fw_compat = fw_compat; + dcp->external = of_device_is_compatible(dev->of_node, "apple,t6030-dcpext"); dcp->dev = dev; /* * Type-C and Thunderbolt routes can be activated as soon as they are @@ -4739,6 +4927,14 @@ static int dcp_platform_probe(struct platform_device *pdev) platform_set_drvdata(pdev, dcp); + if (fw_compat == DCP_FIRMWARE_V_14_7 && !dcp->external) { + ret = iomfb_v14_7_probe(dcp); + if (ret) + return ret; + } + if (dcp->external) + dcp->hw.num_dptx_ports = 2; + dcp->phy = devm_phy_optional_get(dev, "dp-phy"); if (IS_ERR(dcp->phy)) { dev_err(dev, "Failed to get dp-phy: %ld\n", PTR_ERR(dcp->phy)); @@ -4754,7 +4950,9 @@ static int dcp_platform_probe(struct platform_device *pdev) "apple,iomfb-surfaces", sizeof(u32)); - if (num_surfs == 0 || num_surfs == -ENODATA) { + if (fw_compat == DCP_FIRMWARE_V_14_7) { + set_bit(0, dcp->iomfb_surfaces); + } else if (num_surfs == 0 || num_surfs == -ENODATA) { set_bit(0, dcp->iomfb_surfaces); set_bit(1, dcp->iomfb_surfaces); } else if (num_surfs < 0) { @@ -4875,6 +5073,17 @@ static int dcp_platform_probe(struct platform_device *pdev) if (ret) return ret; + /* + * The T6030 external processor is not part of the panel's DRM device. + * Joining that component set would hold the internal screen until + * dcpext binds. Its tunnel bring-up runs the Type-C reconnect work, + * which component bind would otherwise enable. + */ + if (dcp->external) { + enable_delayed_work(&dcp->typec_reconnect_wq); + return iomfb_v14_7_external_start(dcp); + } + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { /* Complete optional startup before the DRM master can depend on it. */ @@ -4895,6 +5104,10 @@ static int dcp_platform_probe(struct platform_device *pdev) static void dcp_platform_remove(struct platform_device *pdev) { + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + if (dcp && dcp->external) + return; component_del(&pdev->dev, &dcp_comp_ops); } @@ -4902,6 +5115,9 @@ VISIBLE_IF_KUNIT void dcp_platform_shutdown(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); + if (dcp && dcp->external) + return; + if (dcp && dcp->retain_dma && dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { dcp_comp_unbind(&pdev->dev, NULL, NULL); @@ -5009,10 +5225,61 @@ static void dcp_release_after_sleep(struct apple_dcp *dcp) iomfb_pause_presentation(dcp, false); } +/* dpm_prepare completes for every device before any dpm_suspend callback. + * Veto early so Thunderbolt cannot tear down a working tunnel first. The + * same HPD lock serializes this gate against explicit firmware startup. + */ +static int dcp_platform_prepare(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + + if (!dcp->external) + return 0; + mutex_lock(&dcp->hpd_mutex); + if (atomic_read(&dcp->external_requested) || + (dcp->rtk && apple_rtkit_is_running(dcp->rtk)) || + dcpext_scanout_requested(dcp)) { + mutex_unlock(&dcp->hpd_mutex); + dev_warn(dev, "external firmware/scanout attempted: refusing PM prepare; reboot required for retained DMA\n"); + return -EBUSY; + } + WRITE_ONCE(dcp->external_suspended, true); + mutex_unlock(&dcp->hpd_mutex); + return 0; +} + +static void dcp_platform_complete(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + + if (!dcp->external) + return; + mutex_lock(&dcp->hpd_mutex); + WRITE_ONCE(dcp->external_suspended, false); + mutex_unlock(&dcp->hpd_mutex); +} + static int dcp_platform_suspend(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); + /* Serialize PM against explicit bring-up, including queued startup work. + * Before a startup attempt the device remains suspendable. After any + * attempt we cannot prove DMA/firmware quiescence, even on failure. + */ + if (dcp->external) { + mutex_lock(&dcp->hpd_mutex); + if (atomic_read(&dcp->external_requested) || + (dcp->rtk && apple_rtkit_is_running(dcp->rtk)) || + dcpext_scanout_requested(dcp)) { + mutex_unlock(&dcp->hpd_mutex); + dev_warn(dev, "external firmware/scanout attempted: suspend refused; retained DMA requires reboot\n"); + return -EBUSY; + } + WRITE_ONCE(dcp->external_suspended, true); + mutex_unlock(&dcp->hpd_mutex); + } + if (dcp_is_external_h17p(dcp)) { int ret; @@ -5073,8 +5340,11 @@ static int dcp_platform_resume(struct device *dev) return 0; } -static DEFINE_SIMPLE_DEV_PM_OPS(dcp_platform_pm_ops, - dcp_platform_suspend, dcp_platform_resume); +static const struct dev_pm_ops dcp_platform_pm_ops = { + .prepare = pm_sleep_ptr(dcp_platform_prepare), + .complete = pm_sleep_ptr(dcp_platform_complete), + SYSTEM_SLEEP_PM_OPS(dcp_platform_suspend, dcp_platform_resume) +}; static const struct apple_dcp_hw_data apple_dcp_hw_t6020 = { @@ -5127,6 +5397,14 @@ static const struct apple_dcp_hw_data apple_dcp_hw_dcp = { .num_dptx_ports = 0, }; +static const struct apple_dcp_hw_data apple_dcp_hw_t6030 = { + .num_dptx_ports = 1, +}; + +static const struct apple_dcp_hw_data apple_dcp_hw_t6030_dcpext = { + .num_dptx_ports = 1, +}; + static const struct apple_dcp_hw_data apple_dcp_hw_dcpext = { .num_dptx_ports = 2, }; @@ -5137,6 +5415,8 @@ static const struct of_device_id of_match[] = { { .compatible = "apple,t8140-dcp", .data = &apple_dcp_hw_t8140, }, { .compatible = "apple,t8140-dcpext", .data = &apple_dcp_hw_t8140_external, }, { .compatible = "apple,t8142-dcp", .data = &apple_dcp_hw_t8142, }, + { .compatible = "apple,t6030-dcp", .data = &apple_dcp_hw_t6030, }, + { .compatible = "apple,t6030-dcpext", .data = &apple_dcp_hw_t6030_dcpext, }, { .compatible = "apple,dcp", .data = &apple_dcp_hw_dcp, }, { .compatible = "apple,dcpext", .data = &apple_dcp_hw_dcpext, }, {} diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index bb56b6607b065..a1ace94a0404f 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -54,6 +54,7 @@ bool dcp_typec_dual_stream(void); void dcp_typec_reorder(void); bool dcp_is_typec_only(struct platform_device *pdev); bool dcp_fw_compat_is_12_x(struct platform_device *pdev); +bool dcp_fw_compat_is_14_7(struct platform_device *pdev); unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev); void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, struct apple_connector *connector); diff --git a/drivers/gpu/drm/apple/dcpext_drm.c b/drivers/gpu/drm/apple/dcpext_drm.c new file mode 100644 index 0000000000000..08748f79a6b01 --- /dev/null +++ b/drivers/gpu/drm/apple/dcpext_drm.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Fixed iBoot scanout, with CPU shadow updates. No firmware calls per commit. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "dcp.h" +#include "dcpext_mode.h" +#include "dcpext_drm.h" + +#define EXT_WIDTH DCPEXT_WIDTH +#define EXT_HEIGHT DCPEXT_HEIGHT +#define EXT_STRIDE DCPEXT_STRIDE + +struct dcpext_drm { + struct drm_device drm; + struct drm_plane plane; + struct drm_crtc crtc; + struct drm_encoder encoder; + struct drm_connector connector; + void *pixels; + size_t size; + u32 stride; + /* Owned by the retained scanout, never by the detachable parent device. */ + atomic_t *terminal_error; +}; + +/* The firmware selected this geometry/rate; timing changes are not supported. */ +static const struct drm_display_mode ext_mode = { + DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED, + 594000, 3840, 4016, 4104, 4400, 0, + 2160, 2168, 2178, 2250, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) +}; + +static bool ext_rect_valid(int x1, int y1, int x2, int y2, u32 pitch, size_t size) +{ + u64 end; + + if (x1 < 0 || y1 < 0 || x2 <= x1 || y2 <= y1 || + x2 > EXT_WIDTH || y2 > EXT_HEIGHT || pitch < EXT_STRIDE) + return false; + end = (u64)(y2 - 1) * pitch + (u64)x2 * 4; + return end <= size; +} + +static int ext_plane_check(struct drm_plane *plane, struct drm_atomic_commit *state) +{ + struct drm_plane_state *ps = drm_atomic_get_new_plane_state(state, plane); + struct drm_crtc_state *cs = NULL; + struct drm_gem_object *obj; + int ret; + + if (ps->crtc) + cs = drm_atomic_get_new_crtc_state(state, ps->crtc); + ret = drm_atomic_helper_check_plane_state(ps, cs, DRM_PLANE_NO_SCALING, + DRM_PLANE_NO_SCALING, false, false); + if (ret || !ps->visible) + return ret; + if (atomic_read(container_of(plane->dev, struct dcpext_drm, drm)->terminal_error)) + return -ENOLINK; + /* Keep source/destination coordinates identical for bounded damage copies. */ + if (ps->src_x || ps->src_y || ps->src_w != EXT_WIDTH << 16 || + ps->src_h != EXT_HEIGHT << 16 || ps->crtc_x || ps->crtc_y || + ps->crtc_w != EXT_WIDTH || ps->crtc_h != EXT_HEIGHT || + ps->fb->format->format != DRM_FORMAT_XRGB8888) + return -EINVAL; + obj = drm_gem_fb_get_obj(ps->fb, 0); + if (!obj || ps->fb->offsets[0] > obj->size || + !ext_rect_valid(0, 0, EXT_WIDTH, EXT_HEIGHT, ps->fb->pitches[0], + obj->size - ps->fb->offsets[0])) + return -EINVAL; + return 0; +} + +static void ext_plane_update(struct drm_plane *plane, struct drm_atomic_commit *state) +{ + struct dcpext_drm *ext = container_of(plane->dev, struct dcpext_drm, drm); + struct drm_plane_state *ps = drm_atomic_get_new_plane_state(state, plane); + struct drm_plane_state *old = drm_atomic_get_old_plane_state(state, plane); + struct drm_shadow_plane_state *shadow = to_drm_shadow_plane_state(ps); + struct drm_framebuffer *fb = ps->fb; + struct drm_gem_object *obj; + struct drm_atomic_helper_damage_iter iter; + struct drm_rect damage, bounds = DRM_RECT_INIT(0, 0, EXT_WIDTH, EXT_HEIGHT); + int idx, x, y; + + if (!ps->visible || !fb || atomic_read(ext->terminal_error)) + return; + obj = drm_gem_fb_get_obj(fb, 0); + if (drm_gem_fb_begin_cpu_access(fb, DMA_FROM_DEVICE)) + return; + if (!drm_dev_enter(plane->dev, &idx)) + goto end_access; + drm_atomic_helper_damage_iter_init(&iter, old, ps); + drm_atomic_for_each_plane_damage(&iter, &damage) { + if (!drm_rect_intersect(&damage, &bounds)) + continue; + if (WARN_ON_ONCE(!obj || fb->offsets[0] > obj->size || + !ext_rect_valid(damage.x1, damage.y1, damage.x2, damage.y2, + fb->pitches[0], obj->size - fb->offsets[0]) || + !ext_rect_valid(damage.x1, damage.y1, damage.x2, damage.y2, + ext->stride, ext->size))) + continue; + for (y = damage.y1; y < damage.y2; ++y) { + u8 *dst = ext->pixels + (size_t)y * ext->stride + damage.x1 * 4; + + iosys_map_memcpy_from(dst, &shadow->data[0], + (size_t)y * fb->pitches[0] + damage.x1 * 4, + (damage.x2 - damage.x1) * 4); + /* Firmware BGRA requires opaque alpha; XRGB's X is undefined. */ + for (x = 0; x < damage.x2 - damage.x1; ++x) + dst[x * 4 + 3] = 0xff; + } + } + dma_wmb(); + drm_dev_exit(idx); +end_access: + drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE); +} + +static void ext_plane_disable(struct drm_plane *plane, struct drm_atomic_commit *state) +{ + struct dcpext_drm *ext = container_of(plane->dev, struct dcpext_drm, drm); + u32 *dst = ext->pixels; + int idx, i; + + if (atomic_read(ext->terminal_error) || !drm_dev_enter(plane->dev, &idx)) + return; + for (i = 0; i < EXT_WIDTH * EXT_HEIGHT; ++i) + dst[i] = cpu_to_le32(0xff000000); + dma_wmb(); + drm_dev_exit(idx); +} + +static const struct drm_plane_funcs ext_plane_funcs = { + DRM_GEM_SHADOW_PLANE_FUNCS, + .update_plane = drm_atomic_helper_update_plane, + .disable_plane = drm_atomic_helper_disable_plane, + .destroy = drm_plane_cleanup, +}; +static const struct drm_plane_helper_funcs ext_plane_helpers = { + DRM_GEM_SHADOW_PLANE_HELPER_FUNCS, + .atomic_check = ext_plane_check, + .atomic_update = ext_plane_update, + .atomic_disable = ext_plane_disable, +}; +static enum drm_mode_status ext_mode_valid(struct drm_crtc *crtc, + const struct drm_display_mode *mode) +{ + return drm_mode_equal(mode, &ext_mode) ? MODE_OK : MODE_BAD; +} +static int ext_crtc_check(struct drm_crtc *crtc, struct drm_atomic_commit *state) +{ + struct drm_crtc_state *cs = drm_atomic_get_new_crtc_state(state, crtc); + + return cs->enable ? drm_atomic_helper_check_crtc_primary_plane(cs) : 0; +} +static const struct drm_crtc_funcs ext_crtc_funcs = { + .reset = drm_atomic_helper_crtc_reset, + .set_config = drm_atomic_helper_set_config, + .page_flip = drm_atomic_helper_page_flip, + .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state, + .destroy = drm_crtc_cleanup, +}; +static const struct drm_crtc_helper_funcs ext_crtc_helpers = { + .mode_valid = ext_mode_valid, + .atomic_check = ext_crtc_check, +}; +static int ext_get_modes(struct drm_connector *connector) +{ + return drm_connector_helper_get_modes_fixed(connector, &ext_mode); +} +static enum drm_connector_status ext_detect(struct drm_connector *connector, bool force) +{ + struct dcpext_drm *ext = container_of(connector->dev, struct dcpext_drm, drm); + + return atomic_read(ext->terminal_error) ? connector_status_disconnected : + connector_status_connected; +} +static const struct drm_connector_funcs ext_connector_funcs = { + .reset = drm_atomic_helper_connector_reset, + .fill_modes = drm_helper_probe_single_connector_modes, + .detect = ext_detect, + .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, + .destroy = drm_connector_cleanup, +}; +static const struct drm_connector_helper_funcs ext_connector_helpers = { + .get_modes = ext_get_modes, +}; +static const struct drm_encoder_funcs ext_encoder_funcs = { + .destroy = drm_encoder_cleanup, +}; +static const struct drm_mode_config_funcs ext_mode_funcs = { + .fb_create = drm_gem_fb_create_with_dirty, + .atomic_check = drm_atomic_helper_check, + .atomic_commit = drm_atomic_helper_commit, +}; +DEFINE_DRM_GEM_FOPS(ext_fops); +static const struct drm_driver ext_driver = { + DRM_GEM_SHMEM_DRIVER_OPS, + .driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC, + .fops = &ext_fops, + .name = "apple-dcpext-shadow", + .desc = "Apple external fixed iBoot shadow scanout", + .major = 1, + .minor = 0, +}; + +static void ext_unregister(void *data) +{ + struct drm_device *drm = data; + + drm_dev_unplug(drm); + drm_atomic_helper_shutdown(drm); +} + +int dcpext_drm_register(struct apple_dcp *dcp, void *pixels, size_t size, u32 stride, + atomic_t *terminal_error, struct drm_device **retained_drm) +{ + static const u32 formats[] = { DRM_FORMAT_XRGB8888 }; + static const u64 modifiers[] = { DRM_FORMAT_MOD_LINEAR, DRM_FORMAT_MOD_INVALID }; + struct dcpext_drm *ext; + struct drm_device *drm; + int ret; + + if (!terminal_error || !retained_drm || atomic_read(terminal_error) || !pixels || !IS_ALIGNED((unsigned long)pixels, 4) || stride != EXT_STRIDE || + !ext_rect_valid(0, 0, EXT_WIDTH, EXT_HEIGHT, stride, size)) + return -EINVAL; + ext = devm_drm_dev_alloc(dcp->dev, &ext_driver, struct dcpext_drm, drm); + if (IS_ERR(ext)) + return PTR_ERR(ext); + drm = &ext->drm; + ext->pixels = pixels; + ext->size = size; + ext->stride = stride; + ext->terminal_error = terminal_error; + ret = drmm_mode_config_init(drm); + if (ret) + return ret; + drm->mode_config.min_width = EXT_WIDTH; + drm->mode_config.max_width = EXT_WIDTH; + drm->mode_config.min_height = EXT_HEIGHT; + drm->mode_config.max_height = EXT_HEIGHT; + drm->mode_config.preferred_depth = 24; + drm->mode_config.funcs = &ext_mode_funcs; + ret = drm_universal_plane_init(drm, &ext->plane, 0, &ext_plane_funcs, + formats, ARRAY_SIZE(formats), modifiers, DRM_PLANE_TYPE_PRIMARY, NULL); + if (ret) + return ret; + drm_plane_helper_add(&ext->plane, &ext_plane_helpers); + drm_plane_enable_fb_damage_clips(&ext->plane); + ret = drm_crtc_init_with_planes(drm, &ext->crtc, &ext->plane, NULL, + &ext_crtc_funcs, NULL); + if (ret) + return ret; + drm_crtc_helper_add(&ext->crtc, &ext_crtc_helpers); + ret = drm_encoder_init(drm, &ext->encoder, &ext_encoder_funcs, + DRM_MODE_ENCODER_TMDS, NULL); + if (ret) + return ret; + ext->encoder.possible_crtcs = drm_crtc_mask(&ext->crtc); + ret = drm_connector_init(drm, &ext->connector, &ext_connector_funcs, + DRM_MODE_CONNECTOR_DisplayPort); + if (ret) + return ret; + drm_connector_helper_add(&ext->connector, &ext_connector_helpers); + ret = drm_connector_attach_encoder(&ext->connector, &ext->encoder); + if (ret) + return ret; + drm_mode_config_reset(drm); + /* No vblank IRQ: the atomic helper emits completion events after copies. + * These events do not mean a physical vblank; tearing remains possible. + * Deliberately no fbdev/client setup, console takeover, or render node. + */ + ret = drm_dev_register(drm, 0); + if (ret) + return ret; + ret = devm_add_action_or_reset(dcp->dev, ext_unregister, drm); + if (ret) + return ret; + /* Keep notification work safe even after parent devres teardown. */ + drm_dev_get(drm); + WRITE_ONCE(*retained_drm, drm); + dev_info(dcp->dev, "external shadow DRM registered: fixed 3840x2160, CPU copies, no hardware vblank\n"); + return 0; +} diff --git a/drivers/gpu/drm/apple/dcpext_drm.h b/drivers/gpu/drm/apple/dcpext_drm.h new file mode 100644 index 0000000000000..e9ef1db3d96af --- /dev/null +++ b/drivers/gpu/drm/apple/dcpext_drm.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __APPLE_DCPEXT_DRM_H__ +#define __APPLE_DCPEXT_DRM_H__ +#include +#include +struct apple_dcp; +struct drm_device; +/* Caller retains the already-scanning buffer and serializes one registration. */ +int dcpext_drm_register(struct apple_dcp *dcp, void *pixels, size_t size, u32 stride, + atomic_t *terminal_error, struct drm_device **retained_drm); +#endif diff --git a/drivers/gpu/drm/apple/dcpext_mode.h b/drivers/gpu/drm/apple/dcpext_mode.h new file mode 100644 index 0000000000000..f87ae41784f6e --- /dev/null +++ b/drivers/gpu/drm/apple/dcpext_mode.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __APPLE_DCPEXT_MODE_H__ +#define __APPLE_DCPEXT_MODE_H__ +/* Focused 4K candidate: keep geometry consistent across firmware, DMA and KMS. + * The separately retained 1080p kernel remains the working fallback. + */ +#define DCPEXT_WIDTH 3840 +#define DCPEXT_HEIGHT 2160 +#define DCPEXT_STRIDE (DCPEXT_WIDTH * 4) +#define DCPEXT_MAX_BUFFER_SIZE (64U * 1024 * 1024) +#endif diff --git a/drivers/gpu/drm/apple/dcpext_scanout.c b/drivers/gpu/drm/apple/dcpext_scanout.c new file mode 100644 index 0000000000000..d0593453cf9a7 --- /dev/null +++ b/drivers/gpu/drm/apple/dcpext_scanout.c @@ -0,0 +1,591 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Local T6030 external first-pixel diagnostic. No automatic scanout startup. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dcp.h" +#include "dcpext_mode.h" +#include "dcpext_scanout.h" +#include "dcpext_drm.h" +#include "ibootep.h" + +#define SCANOUT_WIDTH DCPEXT_WIDTH +#define SCANOUT_HEIGHT DCPEXT_HEIGHT +#define SCANOUT_STRIDE DCPEXT_STRIDE +#define SCANOUT_IOVA_BASE (1ULL << 40) +#define SCANOUT_IOVA_SIZE (1ULL << 36) + +struct dcpext_scanout { + struct apple_dcp *dcp; + struct device *firmware_dev; + struct device_node *node; + struct platform_device *pdev; + struct iommu_group *group; + struct iommu_domain *firmware_domain; + struct iommu_domain *scanout_domain; + void *pixels; + dma_addr_t iova; + size_t size; + size_t mapped; + struct sg_table sgt; + struct work_struct work; + struct work_struct desktop_work; + atomic_t desktop_requested; + atomic_t terminal_error; + bool desktop_registered; + struct drm_device *retained_drm; + struct work_struct invalidate_work; + bool pattern_ready; + atomic_t requested; + bool stopping; + bool published; /* Firmware callbacks may retain this object until reboot. */ + struct device_attribute attr; + struct device_attribute desktop_attr; + struct device_attribute status_attr; + struct attribute *attrs[4]; + struct attribute_group attr_group; +}; + +static void dcpext_invalidate_work(struct work_struct *work) +{ + struct dcpext_scanout *scanout = container_of(work, struct dcpext_scanout, invalidate_work); + struct drm_device *drm = READ_ONCE(scanout->retained_drm); + + /* Never notify while holding the Type-C fabric or HPD locks. */ + if (drm && !READ_ONCE(scanout->stopping)) + drm_kms_helper_hotplug_event(drm); +} + +static void scanout_fail(struct dcpext_scanout *scanout, int error) +{ + if (READ_ONCE(scanout->stopping)) + return; + if (error >= 0) + error = -EIO; + if (!atomic_cmpxchg(&scanout->terminal_error, 0, error)) + schedule_work(&scanout->invalidate_work); +} + +bool dcpext_scanout_requested(struct apple_dcp *dcp) +{ + struct dcpext_scanout *scanout = smp_load_acquire(&dcp->dcpext_scanout); + + return scanout && atomic_read(&scanout->requested); +} + +bool dcpext_scanout_terminal(struct apple_dcp *dcp) +{ + struct dcpext_scanout *scanout = smp_load_acquire(&dcp->dcpext_scanout); + + return scanout && atomic_read(&scanout->terminal_error); +} + +void dcpext_scanout_fault(struct apple_dcp *dcp, int error) +{ + struct dcpext_scanout *scanout = smp_load_acquire(&dcp->dcpext_scanout); + + if (scanout) + scanout_fail(scanout, error); +} + +void dcpext_scanout_invalidate(struct apple_dcp *dcp) +{ + struct dcpext_scanout *scanout = smp_load_acquire(&dcp->dcpext_scanout); + + /* Initial unplug remains allowed; only a started scanout owns DMA. */ + if (scanout && atomic_read(&scanout->requested)) + scanout_fail(scanout, -ENOLINK); +} + +static bool scanout_link_ready(struct dcpext_scanout *scanout) +{ + struct apple_dcp *dcp = scanout->dcp; + + return !READ_ONCE(dcp->crashed) && !atomic_read(&scanout->terminal_error) && + smp_load_acquire(&dcp->external_link_ready) && + (!READ_ONCE(dcp->dptx_tunnel) || READ_ONCE(dcp->tb_clock_ok)) && + READ_ONCE(dcp->typec_cable_connected) && + READ_ONCE(dcp->dptxport[0].connected) && + READ_ONCE(dcp->dptxport[0].enabled); +} + +static ssize_t dcpext_status_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct dcpext_scanout *scanout = container_of(attr, struct dcpext_scanout, status_attr); + bool requested = atomic_read(&scanout->requested); + bool ready = smp_load_acquire(&scanout->pattern_ready); + bool desktop_requested = atomic_read(&scanout->desktop_requested); + bool registered = smp_load_acquire(&scanout->desktop_registered); + int error = atomic_read(&scanout->terminal_error); + bool fw_ready = !READ_ONCE(scanout->dcp->crashed) && scanout->dcp->rtk && apple_rtkit_is_running(scanout->dcp->rtk) && + ibootep_is_ready(scanout->dcp) && + smp_load_acquire(&scanout->dcp->dptxport[0].enabled); + const char *phase = error ? "terminal" : registered ? "desktop_registered" : + desktop_requested ? "desktop_pending" : ready ? "pattern_ready" : + requested ? "pattern_pending" : "ready"; + + return sysfs_emit(buf, "schema=1 phase=%s fw_ready=%u pattern_requested=%u pattern_ready=%u desktop_requested=%u desktop_registered=%u terminal_error=%d link_ready=%u width=%u height=%u stride=%u\n", + phase, fw_ready, requested, ready, desktop_requested, registered, error, + scanout_link_ready(scanout), SCANOUT_WIDTH, SCANOUT_HEIGHT, SCANOUT_STRIDE); +} + +static bool scanout_iova_valid(u64 iova, size_t size) +{ + return size && IS_ALIGNED(iova, SZ_16K) && IS_ALIGNED(size, SZ_16K) && + iova >= SCANOUT_IOVA_BASE && size <= SCANOUT_IOVA_SIZE && + iova - SCANOUT_IOVA_BASE <= SCANOUT_IOVA_SIZE - size; +} + +static void scanout_fill_pattern(void *pixels, size_t size) +{ + static const u32 bars[] = { + 0xffffffff, 0xffffff00, 0xff00ffff, 0xff00ff00, + 0xffff00ff, 0xffff0000, 0xff0000ff, 0xff000000, + }; + __le32 *pixel = pixels; + u32 x, y; + + memset(pixels, 0, size); + for (y = 0; y < SCANOUT_HEIGHT; y++) + for (x = 0; x < SCANOUT_WIDTH; x++) + pixel[y * SCANOUT_WIDTH + x] = + cpu_to_le32(bars[x / (SCANOUT_WIDTH / ARRAY_SIZE(bars))]); +} + +/* Before mapping, require a hole in scanout and the expected firmware mapping. + * Afterwards, require both DART domains to resolve every 16K page identically. + * SG addresses describe physical backing, never DMA addresses from another map. + */ +static int scanout_verify_buffer(struct dcpext_scanout *scanout, bool mapped) +{ + struct scatterlist *sg; + size_t total = 0, off; + unsigned int i; + + if (!scanout_iova_valid(scanout->iova, scanout->size)) + return -ERANGE; + for_each_sg(scanout->sgt.sgl, sg, scanout->sgt.orig_nents, i) { + phys_addr_t phys = sg_phys(sg); + size_t length = sg->length; + + if (!phys || !length || !IS_ALIGNED(phys, SZ_16K) || + !IS_ALIGNED(length, SZ_16K) || length > scanout->size - total) + return -EINVAL; + for (off = 0; off < length; off += SZ_16K) { + u64 iova = scanout->iova + total + off; + phys_addr_t firmware = iommu_iova_to_phys(scanout->firmware_domain, iova); + phys_addr_t display = iommu_iova_to_phys(scanout->scanout_domain, iova); + + if (firmware != phys + off) + return -EFAULT; + if (mapped ? display != firmware : display != 0) + return mapped ? -EFAULT : -EBUSY; + } + total += length; + } + return total == scanout->size ? 0 : -EINVAL; +} + +static int scanout_map_buffer(struct dcpext_scanout *scanout) +{ + struct scatterlist *sg; + unsigned int i; + int ret; + + ret = scanout_verify_buffer(scanout, false); + if (ret) + return ret; + for_each_sg(scanout->sgt.sgl, sg, scanout->sgt.orig_nents, i) { + ret = iommu_map(scanout->scanout_domain, scanout->iova + scanout->mapped, + sg_phys(sg), sg->length, IOMMU_READ | IOMMU_WRITE, GFP_KERNEL); + if (ret) + return ret; + scanout->mapped += sg->length; + } + return scanout_verify_buffer(scanout, true); +} + +static bool scanout_node_verified(struct device_node *node) +{ + u32 marker; + + return node && of_device_is_available(node) && + of_device_is_compatible(node, "apple,t6030-dispext-scanout") && + !of_property_read_u32(node, "apple,t6030-dispext-handoff", &marker) && marker == 1 && + !of_property_read_u32(node, "apple,t6030-scanout-verified", &marker) && marker == 1; +} + +struct scanout_group_check { + struct device *expected; + unsigned int count; +}; + +static int scanout_group_member(struct device *dev, void *data) +{ + struct scanout_group_check *check = data; + + check->count++; + return dev == check->expected ? 0 : -EBUSY; +} + +static int scanout_check_domains(struct dcpext_scanout *scanout) +{ + struct device *dev = &scanout->pdev->dev; + struct scanout_group_check check = { .expected = dev }; + struct iommu_group *firmware_group; + int ret; + + /* These raw display mappings have no IOMMU_CACHE attribute. The tested + * firmware allocation is uncached coherent memory on a noncoherent device. + */ + if (dev_is_dma_coherent(scanout->firmware_dev) || dev_is_dma_coherent(dev)) + return -EOPNOTSUPP; + /* No driver or second DMA client may allocate addresses in this domain. */ + if (device_is_bound(dev)) + return -EBUSY; + scanout->firmware_domain = iommu_get_domain_for_dev(scanout->firmware_dev); + scanout->scanout_domain = iommu_get_domain_for_dev(dev); + if (IS_ERR_OR_NULL(scanout->firmware_domain) || IS_ERR_OR_NULL(scanout->scanout_domain) || + scanout->firmware_domain == scanout->scanout_domain || + !iommu_is_dma_domain(scanout->firmware_domain) || + !iommu_is_dma_domain(scanout->scanout_domain) || + !(scanout->scanout_domain->pgsize_bitmap & SZ_16K)) + return -EINVAL; + scanout->group = iommu_group_get(dev); + if (!scanout->group) + return -ENODEV; + firmware_group = iommu_group_get(scanout->firmware_dev); + ret = firmware_group && firmware_group != scanout->group ? 0 : -EBUSY; + if (firmware_group) + iommu_group_put(firmware_group); + if (ret) + return ret; + ret = iommu_group_for_each_dev(scanout->group, &check, scanout_group_member); + if (ret) + return ret; + return check.count == 1 ? 0 : -EBUSY; +} + +/* + * The early gate predates external firmware startup. Recheck immediately + * before attaching: neither firmware nor another client may have populated + * the inherited roots in the meantime. This function performs no MMIO writes. + */ +static int scanout_recheck_dart(struct device_node *dart) +{ + struct resource regs, table, first = {}; + struct device_node *region = NULL; + struct reserved_mem *rmem; + void __iomem *mmio; + u32 state[6], marker; + u64 phys, size; + void *root; + int i, ret = -EINVAL; + + if (of_property_read_u32(dart, "apple,t6030-dispext-handoff", &marker) || marker != 1 || + of_property_count_u32_elems(dart, "apple,inherited-dart-state") != 6 || + of_property_read_u32_array(dart, "apple,inherited-dart-state", state, 6) || + state[0] != 0 || state[3] != 4 || + of_count_phandle_with_args(dart, "memory-region", NULL) != 2 || + of_address_to_resource(dart, 0, ®s) || + regs.start != 0x2d1304000ULL || resource_size(®s) != SZ_16K) + return -EINVAL; + mmio = ioremap(regs.start, resource_size(®s)); + if (!mmio) + return -ENOMEM; + if (!(readl(mmio + 0x200) & BIT(0))) + goto out; + for (i = 0; i < 2; i++) { + u32 sid = state[3 * i], tcr = state[3 * i + 1], ttbr = state[3 * i + 2]; + + ret = -EINVAL; + if ((tcr & (BIT(0) | BIT(1) | BIT(3))) != BIT(0) || !(ttbr & BIT(0)) || + (ttbr & ~(GENMASK(29, 2) | BIT(0))) || + readl(mmio + 0x1000 + 4 * sid) != tcr || + readl(mmio + 0x1400 + 4 * sid) != ttbr) + goto out; + phys = (u64)(ttbr & GENMASK(29, 2)) << 12; + region = of_parse_phandle(dart, "memory-region", i); + if (!region || !of_property_read_bool(region, "no-map") || + of_property_read_bool(region, "reusable") || + of_address_to_resource(region, 0, &table) || table.end < table.start) + goto out; + size = resource_size(&table); + rmem = of_reserved_mem_lookup(region); + if (!rmem || size < SZ_16K || size > SZ_1M || + !IS_ALIGNED(table.start, SZ_16K) || !IS_ALIGNED(size, SZ_16K) || + rmem->base != table.start || rmem->size != size || + phys < table.start || phys > table.end - SZ_16K + 1 || + (i && table.start <= first.end && first.start <= table.end)) + goto out; + if (!i) + first = table; + root = memremap(phys, SZ_16K, MEMREMAP_WB); + if (!root) { + ret = -ENOMEM; + goto out; + } + ret = memchr_inv(root, 0, SZ_16K) ? -EBUSY : 0; + memunmap(root); + if (ret) + goto out; + of_node_put(region); + region = NULL; + } +out: + of_node_put(region); + iounmap(mmio); + return ret; +} + +static int scanout_create_device(struct dcpext_scanout *scanout) +{ + struct of_phandle_args spec; + struct platform_device *existing; + int ret; + + if (PAGE_SIZE != SZ_16K || !scanout_node_verified(scanout->node)) + return -EINVAL; + if (of_count_phandle_with_args(scanout->node, "iommus", "#iommu-cells") != 1) + return -EINVAL; + ret = of_parse_phandle_with_args(scanout->node, "iommus", "#iommu-cells", 0, &spec); + if (ret) + return ret; + ret = spec.args_count == 1 && spec.args[0] == 0 && + of_device_is_compatible(spec.np, "apple,t8110-dart") && + of_device_is_available(spec.np) ? 0 : -EINVAL; + if (!ret) + ret = scanout_recheck_dart(spec.np); + of_node_put(spec.np); + if (ret) + return ret; + existing = of_find_device_by_node(scanout->node); + if (existing) { + put_device(&existing->dev); + return -EBUSY; + } + scanout->pdev = of_platform_device_create(scanout->node, NULL, scanout->firmware_dev); + if (!scanout->pdev) + return -ENODEV; + /* Once attached, never destroy this consumer of the locked inherited DART. */ + ret = dma_set_mask_and_coherent(&scanout->pdev->dev, DMA_BIT_MASK(42)); + if (!ret) + ret = of_dma_configure(&scanout->pdev->dev, scanout->node, true); + if (ret) + return ret; + return scanout_check_domains(scanout); +} + +static void dcpext_pattern_work(struct work_struct *work) +{ + struct dcpext_scanout *scanout = container_of(work, struct dcpext_scanout, work); + struct device *dev = scanout->firmware_dev; + const char *step = "scanout DMA consumer"; + int ret; + + if (READ_ONCE(scanout->stopping) || atomic_read(&scanout->terminal_error)) + return; + ret = scanout_create_device(scanout); + if (ret) + goto fail; + scanout->size = ALIGN((size_t)SCANOUT_STRIDE * SCANOUT_HEIGHT, SZ_16K); + step = "coherent framebuffer allocation"; + scanout->pixels = dma_alloc_coherent(dev, scanout->size, &scanout->iova, GFP_KERNEL); + if (!scanout->pixels) { + ret = -ENOMEM; + goto fail; + } + if (!scanout_iova_valid(scanout->iova, scanout->size)) { + ret = -ERANGE; + goto fail; + } + step = "coherent backing scatterlist"; + ret = dma_get_sgtable(dev, &scanout->sgt, scanout->pixels, scanout->iova, scanout->size); + if (ret) + goto fail; + step = "scanout IOVA mapping and page verification"; + ret = scanout_map_buffer(scanout); + if (ret) + goto fail; + scanout_fill_pattern(scanout->pixels, scanout->size); + dma_wmb(); + dev_info(dev, "external pattern: verified %zu bytes at %pad in separate firmware/scanout domains; BGRA 3840x2160 stride %u\n", + scanout->size, &scanout->iova, SCANOUT_STRIDE); + step = "iBoot first-pattern commands"; + if (atomic_read(&scanout->terminal_error)) { + ret = -ENOLINK; + goto fail; + } + ret = ibootep_present_pattern(scanout->dcp, scanout->iova, scanout->size, SCANOUT_STRIDE); + if (ret) + goto fail; + if (atomic_read(&scanout->terminal_error)) { + ret = -ENOLINK; + goto fail; + } + smp_store_release(&scanout->pattern_ready, true); + dev_info(dev, "external pattern commands accepted; all scanout resources retained until reboot\n"); + return; +fail: + scanout_fail(scanout, ret); + dev_err(dev, "external pattern stopped at %s: %d (%zu bytes mapped); no retry, resources retained until reboot\n", + step, ret, scanout->mapped); +} + +static ssize_t dcpext_pattern_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct dcpext_scanout *scanout = container_of(attr, struct dcpext_scanout, attr); + struct apple_dcp *dcp = scanout->dcp; + + if (!sysfs_streq(buf, "1")) + return -EINVAL; + if (READ_ONCE(scanout->stopping) || !dcp->external || !dcp->rtk || + !apple_rtkit_is_running(dcp->rtk) || READ_ONCE(dcp->crashed) || !scanout_node_verified(scanout->node)) + return -ENODEV; + /* Serialize the first request against the disconnect invalidation hook. */ + mutex_lock(&dcp->hpd_mutex); + if (atomic_read(&scanout->terminal_error) || !scanout_link_ready(scanout)) { + mutex_unlock(&dcp->hpd_mutex); + return -ENOLINK; + } + if (atomic_cmpxchg(&scanout->requested, 0, 1)) { + mutex_unlock(&dcp->hpd_mutex); + return -EALREADY; + } + mutex_unlock(&dcp->hpd_mutex); + /* Firmware DMA and callbacks may outlive any error returned by this worker. */ + __module_get(THIS_MODULE); + queue_work(system_unbound_wq, &scanout->work); + return count; +} + +/* Separate explicit step: keep a successful pattern visible until requested. */ +static void dcpext_desktop_work(struct work_struct *work) +{ + struct dcpext_scanout *scanout = container_of(work, struct dcpext_scanout, desktop_work); + int ret; + + if (READ_ONCE(scanout->stopping) || atomic_read(&scanout->terminal_error) || + !smp_load_acquire(&scanout->pattern_ready)) + return; + ret = dcpext_drm_register(scanout->dcp, scanout->pixels, scanout->size, + SCANOUT_STRIDE, &scanout->terminal_error, &scanout->retained_drm); + if (ret) { + scanout_fail(scanout, ret); + dev_err(scanout->firmware_dev, "external desktop registration failed: %d; scanout retained\n", ret); + } else { + smp_store_release(&scanout->desktop_registered, true); + if (atomic_read(&scanout->terminal_error)) + schedule_work(&scanout->invalidate_work); + } +} + +static ssize_t dcpext_desktop_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct dcpext_scanout *scanout = container_of(attr, struct dcpext_scanout, desktop_attr); + + if (!sysfs_streq(buf, "1")) + return -EINVAL; + if (READ_ONCE(scanout->stopping)) + return -ENODEV; + if (atomic_read(&scanout->terminal_error)) + return -ENOLINK; + if (!smp_load_acquire(&scanout->pattern_ready)) + return -EAGAIN; + if (atomic_cmpxchg(&scanout->desktop_requested, 0, 1)) + return -EALREADY; + queue_work(system_unbound_wq, &scanout->desktop_work); + return count; +} + +static void dcpext_scanout_cleanup(void *data) +{ + struct dcpext_scanout *scanout = data; + + WRITE_ONCE(scanout->stopping, true); + cancel_work_sync(&scanout->desktop_work); + cancel_work_sync(&scanout->work); + cancel_work_sync(&scanout->invalidate_work); + if (atomic_read(&scanout->requested) || READ_ONCE(scanout->published)) + return; /* Firmware callbacks and possible DMA survive parent cleanup. */ + WRITE_ONCE(scanout->dcp->dcpext_scanout, NULL); + of_node_put(scanout->node); + put_device(scanout->firmware_dev); + kfree(scanout); +} + +int dcpext_scanout_register(struct apple_dcp *dcp) +{ + struct device_node *node; + struct dcpext_scanout *scanout; + int ret; + + if (!dcp->external || dcp->fw_compat != DCP_FIRMWARE_V_14_7) + return -ENODEV; + node = of_get_child_by_name(dcp->dev->of_node, "scanout"); + if (!scanout_node_verified(node)) { + of_node_put(node); + return -ENODEV; + } + scanout = kzalloc(sizeof(*scanout), GFP_KERNEL); + if (!scanout) { + of_node_put(node); + return -ENOMEM; + } + scanout->dcp = dcp; + scanout->firmware_dev = get_device(dcp->dev); + scanout->node = node; + atomic_set(&scanout->requested, 0); + atomic_set(&scanout->terminal_error, 0); + INIT_WORK(&scanout->invalidate_work, dcpext_invalidate_work); + INIT_WORK(&scanout->work, dcpext_pattern_work); + atomic_set(&scanout->desktop_requested, 0); + INIT_WORK(&scanout->desktop_work, dcpext_desktop_work); + sysfs_attr_init(&scanout->attr.attr); + scanout->attr.attr.name = "dcpext_pattern"; + scanout->attr.attr.mode = 0200; + scanout->attr.store = dcpext_pattern_store; + scanout->attrs[0] = &scanout->attr.attr; + sysfs_attr_init(&scanout->desktop_attr.attr); + scanout->desktop_attr.attr.name = "dcpext_desktop"; + scanout->desktop_attr.attr.mode = 0200; + scanout->desktop_attr.store = dcpext_desktop_store; + scanout->attrs[1] = &scanout->desktop_attr.attr; + sysfs_attr_init(&scanout->status_attr.attr); + scanout->status_attr.attr.name = "dcpext_status"; + scanout->status_attr.attr.mode = 0444; + scanout->status_attr.show = dcpext_status_show; + scanout->attrs[2] = &scanout->status_attr.attr; + scanout->attr_group.attrs = scanout->attrs; + ret = devm_add_action_or_reset(dcp->dev, dcpext_scanout_cleanup, scanout); + if (ret) + return ret; + WRITE_ONCE(scanout->published, true); + smp_store_release(&dcp->dcpext_scanout, scanout); + ret = devm_device_add_group(dcp->dev, &scanout->attr_group); + if (!ret) + dev_info(dcp->dev, "external scanout handoff verified; explicit dcpext_pattern=1 is available\n"); + return ret; +} diff --git a/drivers/gpu/drm/apple/dcpext_scanout.h b/drivers/gpu/drm/apple/dcpext_scanout.h new file mode 100644 index 0000000000000..37392ccfa5499 --- /dev/null +++ b/drivers/gpu/drm/apple/dcpext_scanout.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef _APPLE_DCPEXT_SCANOUT_H +#define _APPLE_DCPEXT_SCANOUT_H + +#include +struct apple_dcp; + +/* Register the explicit one-shot diagnostic only after the iBoot service exists. */ +int dcpext_scanout_register(struct apple_dcp *dcp); + +/* Caller holds hpd_mutex when invalidating a physical link. */ +void dcpext_scanout_fault(struct apple_dcp *dcp, int error); +void dcpext_scanout_invalidate(struct apple_dcp *dcp); +bool dcpext_scanout_terminal(struct apple_dcp *dcp); +bool dcpext_scanout_requested(struct apple_dcp *dcp); + +#endif diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 275d64d98d037..e9a6db2b60b3f 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -8,11 +8,13 @@ #include #include #include +#include #include #include #include "afk.h" +#include #include "dcp.h" #include "dptxep.h" #include "dcp-180hz.h" @@ -533,8 +535,14 @@ static int dptxport_call_set_link_rate(struct apple_epic_service *service, * dcp_tunnel_crossbar_up(). */ if (dptx->atcphy && service->ep->dcp->dptx_tunnel) { - dcp_tunnel_set_rate(service->ep->dcp, dptx->atcphy, - link_rate); + ret = dcp_tunnel_set_rate(service->ep->dcp, dptx->atcphy, + link_rate); + /* T6030 14.7: refuse a rate whose tunnel clock failed. */ + if (ret && service->ep->dcp->fw_compat == DCP_FIRMWARE_V_14_7) { + reply->retcode = cpu_to_le32(1); + reply->link_rate = cpu_to_le32(dptx->link_rate); + return 0; + } } else if (dptx->atcphy) { dptx->phy_ops.dp.link_rate = phy_link_rate; dptx->phy_ops.dp.set_rate = 1; @@ -608,6 +616,7 @@ dptxport_call_activate(struct apple_epic_service *service, { struct dptx_port *dptx = service->cookie; struct apple_dcp *dcp = service->ep->dcp; + int ret; /* * No PHY mode change for a tunnel: the ATC PHY stays in USB4/TBT @@ -617,8 +626,18 @@ dptxport_call_activate(struct apple_epic_service *service, */ if (dptx->atcphy && !dcp->phy_managed_by_typec) phy_set_mode_ext(dptx->atcphy, PHY_MODE_DP, dcp->index); - if (dcp->dptx_tunnel) - dcp_tunnel_dpin_activate(dcp, true); + if (dcp->dptx_tunnel) { + ret = dcp_tunnel_dpin_activate(dcp, true); + /* T6030 14.7: report a failed tunnel activation to DCP. */ + if (ret && dcp->fw_compat == DCP_FIRMWARE_V_14_7) { + dev_err(dcp->dev, "DPTX tunnel activation failed: %d\n", ret); + if (reply_size < sizeof(__le32)) + return -EINVAL; + memcpy(reply, data, min(reply_size, data_size)); + put_unaligned_le32(1, reply); + return 0; + } + } memcpy(reply, data, min(reply_size, data_size)); if (reply_size >= 4) @@ -691,6 +710,9 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, trace_dptxport_apcall(dptx, 0, idx, data_size); dev_dbg(service->ep->dcp->dev, "DPTXPort: APCALL %u (%zu bytes)\n", idx, data_size); + if (service->ep->dcp->external) + dev_info(service->ep->dcp->dev, "DPTX callback %u size %zu reply %zu\n", + idx, data_size, reply_size); if (data_size && service->ep->dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL) print_hex_dump_debug("DPTXPort: apcall data: ", @@ -706,8 +728,18 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, case DPTX_APCALL_DID_CHANGE_LINK_CONFIG: { int ret = dptxport_call_did_change_link_config(service); - if (!ret && service->ep->dcp->dptx_tunnel && dptx->link_rate) - dcp_tunnel_crossbar_up(service->ep->dcp); + if (!ret && service->ep->dcp->dptx_tunnel && dptx->link_rate) { + int xret = dcp_tunnel_crossbar_up(service->ep->dcp); + + /* T6030 14.7: fail the link config without a crossbar. */ + if (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_14_7) + ret = xret; + } + if (ret && service->ep->dcp->fw_compat == DCP_FIRMWARE_V_14_7 && + service->ep->dcp->dptx_tunnel && reply_size >= sizeof(__le32)) { + put_unaligned_le32(1, reply); + return 0; + } return ret; } case DPTX_APCALL_GET_MAX_LINK_RATE: @@ -806,8 +838,20 @@ static int dptxport_prepare(struct apple_epic_service *service, const char *name const char *class, s64 unit) { struct apple_dcp *dcp = service->ep->dcp; - int port = dptx_service_port(name, class, unit, service->ep->wire_v2, - dcp->index, dcp->hw.num_dptx_ports); + int port; + + /* + * T6030 14.7 also announces the low-power port service, and may leave + * its class empty. + */ + if (dcp->fw_compat == DCP_FIRMWARE_V_14_7 && name && + (!strcmp(name, "dcpdptx-port-epic") || + !strcmp(name, "dcp-lpdptx-port-epic")) && + (!class || !class[0] || !strcmp(class, "AppleDCPDPTXRemotePort"))) + port = unit >= 0 && unit < 2 ? unit : -EINVAL; + else + port = dptx_service_port(name, class, unit, service->ep->wire_v2, + dcp->index, dcp->hw.num_dptx_ports); if (port < 0) return port; @@ -931,6 +975,21 @@ static const struct apple_epic_service_ops dptxep_ops[] = { .call = dptxport_call, .call_v2 = dptxport_group_call, }, + /* T6030 14.7 announces the port by name, possibly without a class. */ + { + .name = "dcpdptx-port-epic", + .prepare = dptxport_prepare, + .teardown = dptxport_teardown, + .init = dptxport_init, + .call = dptxport_call, + }, + { + .name = "dcp-lpdptx-port-epic", + .prepare = dptxport_prepare, + .teardown = dptxport_teardown, + .init = dptxport_init, + .call = dptxport_call, + }, {} }; @@ -1000,6 +1059,17 @@ int dptxep_init(struct apple_dcp *dcp) ret = wait_for_completion_timeout(&dcp->dptxport[port].enable_completion, timeout); if (!ret) { + /* + * 14.7 does not announce the port during boot. A fake + * channel 0 is interface_id 0, and the firmware answers + * kIOReturnNoDevice (0xe00002c0). Leave the endpoint + * up so a later announce can still bind. + */ + if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { + dev_info(dcp->dev, + "DPTX port %u not announced yet\n", port); + return 0; + } dev_warn(dcp->dev, "DPTX transport ready but port %u service not announced\n", port); return -ETIMEDOUT; @@ -1011,3 +1081,96 @@ int dptxep_init(struct apple_dcp *dcp) return dcp_180hz_init(dcp); } + +static void dpav_ctrl_bind(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + struct apple_dcp *dcp = service->ep->dcp; + + dev_info(dcp->dev, + "DPAV service %s class %s unit %lld channel %u\n", + name ? name : "", class ? class : "", unit, service->channel); + if (name && !strcmp(name, "dcpav-controller-epic")) { + dcp->dpav_ctrl = service; + complete_all(&dcp->dpav_ctrl_ready); + } +} + +static const struct apple_epic_service_ops dpav_ctrl_ops[] = { + { + .name = "dcpav-controller-epic", + .init = dpav_ctrl_bind, + }, + { + .name = "DCPAVController", + .init = dpav_ctrl_bind, + }, + { + .name = "dcpdp-controller-epic", + .init = dpav_ctrl_bind, + }, + {} +}; + +static void dptx_av_bind(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + dev_info(service->ep->dcp->dev, + "AV service %s class %s unit %lld channel %u\n", + name ? name : "", class ? class : "", unit, service->channel); +} + +static const struct apple_epic_service_ops dptx_av_ops[] = { + { .name = "DCPAVSimpleVideoInterface", .init = dptx_av_bind }, + { .name = "dcpav-video-interface-epic", .init = dptx_av_bind }, + { .name = "IOAVVideoInterface", .init = dptx_av_bind }, + { .name = "IOAVController", .init = dptx_av_bind }, + {} +}; + +int dptx_prepare_interfaces(struct apple_dcp *dcp) +{ + int ret; + + if (apple_rtkit_has_endpoint(dcp->rtk, AV_ENDPOINT) && !dcp->avep) { + dcp->avep = afk_init(dcp, AV_ENDPOINT, dptx_av_ops); + if (IS_ERR(dcp->avep)) { + dev_warn(dcp->dev, "AV endpoint init failed: %ld\n", + PTR_ERR(dcp->avep)); + dcp->avep = NULL; + } else { + ret = afk_start(dcp->avep); + dev_info(dcp->dev, "AV endpoint start: %d\n", ret); + } + } + + if (apple_rtkit_has_endpoint(dcp->rtk, DPAVSERV_ENDPOINT) && + !dcp->dcpavservep) { + ret = dpavservep_init(dcp); + dev_info(dcp->dev, "DPAV service endpoint: %d\n", ret); + } + + return 0; +} + +int dpav_ctrl_init(struct apple_dcp *dcp) +{ + int ret; + + init_completion(&dcp->dpav_ctrl_ready); + dcp->dpavctrlep = afk_init(dcp, DPAV_CTRL_ENDPOINT, dpav_ctrl_ops); + if (IS_ERR(dcp->dpavctrlep)) + return PTR_ERR(dcp->dpavctrlep); + + ret = afk_start(dcp->dpavctrlep); + if (ret) { + dev_warn(dcp->dev, "DPAV controller endpoint failed to start: %d\n", + ret); + return 0; + } + + if (!wait_for_completion_timeout(&dcp->dpav_ctrl_ready, + msecs_to_jiffies(1000))) + dev_info(dcp->dev, "DPAV controller did not announce a service\n"); + return 0; +} diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index f5096c59e8df8..0ff6152732645 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -75,4 +75,7 @@ int dptxport_set_hpd(struct apple_epic_service *service, bool hpd); int dptxport_set_hpd_timeout(struct apple_epic_service *service, bool hpd, unsigned int timeout_ms); +struct apple_dcp; +int dpav_ctrl_init(struct apple_dcp *dcp); +int dptx_prepare_interfaces(struct apple_dcp *dcp); #endif diff --git a/drivers/gpu/drm/apple/ibootep.c b/drivers/gpu/drm/apple/ibootep.c index 3cada415c249a..b7e746d28cbc5 100644 --- a/drivers/gpu/drm/apple/ibootep.c +++ b/drivers/gpu/drm/apple/ibootep.c @@ -1,14 +1,559 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright 2023 */ -#include +#include +#include +#include +#include +#include +#include #include "afk.h" #include "dcp.h" +#include "dcpext_mode.h" +#include "dcpext_scanout.h" +#include "ibootep.h" + +/* m1n1 src/dcp_iboot.c: iBoot commands have their own wrapper inside EPIC. */ +#define IBOOT_QUERY_RX_SIZE 0x4000 +#define IBOOT_MODE_SIZE 24 +#define IBOOT_PATTERN_WIDTH DCPEXT_WIDTH +#define IBOOT_PATTERN_HEIGHT DCPEXT_HEIGHT +#define IBOOT_PATTERN_STRIDE DCPEXT_STRIDE + +enum iboot_query_op { + IBOOT_GET_HPD = 3, + IBOOT_GET_TIMING_MODES = 4, + IBOOT_GET_COLOR_MODES = 5, +}; + +/* Packed layouts from m1n1 src/dcp_iboot.h and swap_set_layer_cmd_v13_3. */ +struct iboot_plane { + __le32 unk1; + __le64 addr; + __le32 tile_size; + __le32 stride; + __le32 unk2[4]; + __le32 addr_format; + __le32 unk3; +} __packed; + +struct iboot_layer { + struct iboot_plane planes[3]; + __le32 unk; + __le32 plane_count; + __le32 width; + __le32 height; + __le32 surface_format; + __le32 colorspace; + __le32 eotf; + u8 transform; + u8 padding[3]; +} __packed; + +struct iboot_swap_layer_v13_3 { + __le32 unk; + __le32 layer_id; + struct iboot_layer layer; + __le32 unk3; + __le32 unk4; + __le32 src[4]; + __le32 dst[4]; + __le32 unk2; +} __packed; + +static_assert(sizeof(struct iboot_plane) == 44); +static_assert(sizeof(struct iboot_layer) == 164); +static_assert(sizeof(struct iboot_swap_layer_v13_3) == 216); + +struct iboot_query { + struct apple_epic_service *service; + struct work_struct work; + atomic_t busy; + bool stopping; + bool pattern_requested; + bool timing_valid; + bool color_valid; + /* Preserve the exact queried records, including firmware-owned padding. */ + u8 timing[IBOOT_MODE_SIZE]; + u8 color[IBOOT_MODE_SIZE]; +}; + +/* Small wire helpers kept separate so malformed replies can be host-tested. */ +static int iboot_build_query(u32 op, __le32 request[4]) +{ + if (op < IBOOT_GET_HPD || op > IBOOT_GET_COLOR_MODES) + return -EINVAL; + request[0] = cpu_to_le32(op); + request[1] = cpu_to_le32(16); + request[2] = 0; + request[3] = 0; + return 0; +} + +static int iboot_reply_payload(const u8 *reply, size_t capacity, u32 op, + const u8 **payload, size_t *size) +{ + u32 length; + + if (capacity < 8 || get_unaligned_le32(reply) != op) + return -EPROTO; + length = get_unaligned_le32(reply + 4); + if (length < 8 || length > capacity) + return -EPROTO; + *payload = reply + 8; + *size = length - 8; + return 0; +} + +static int iboot_parse_hpd(const u8 *payload, size_t size, bool *hpd, + u32 *timings, u32 *colors) +{ + if (size < 12 || payload[0] > 1) + return -EPROTO; + *hpd = payload[0]; + *timings = get_unaligned_le32(payload + 4); + *colors = get_unaligned_le32(payload + 8); + /* A complete mode list must fit the fixed, bounded reply buffer. */ + if (*timings > (IBOOT_QUERY_RX_SIZE - 12) / IBOOT_MODE_SIZE || + *colors > (IBOOT_QUERY_RX_SIZE - 12) / IBOOT_MODE_SIZE) + return -EOVERFLOW; + return 0; +} + +static int iboot_parse_modes(const u8 *payload, size_t size, u32 *count) +{ + u32 i; + + if (size < 4) + return -EPROTO; + *count = get_unaligned_le32(payload); + if (*count > (size - 4) / IBOOT_MODE_SIZE) + return -EPROTO; + for (i = 0; i < *count; i++) + if (get_unaligned_le32(payload + 4 + i * IBOOT_MODE_SIZE) > 1) + return -EPROTO; + return 0; +} + +static bool iboot_pattern_mode(u32 op, const u8 *mode) +{ + if (get_unaligned_le32(mode) != 1) + return false; + if (op == IBOOT_GET_TIMING_MODES) + return get_unaligned_le32(mode + 4) == IBOOT_PATTERN_WIDTH && + get_unaligned_le32(mode + 8) == IBOOT_PATTERN_HEIGHT && + /* Observed alternate 4K timing; never fall back to generic CVT. */ + get_unaligned_le32(mode + 12) == 0x003bff23U; + if (op == IBOOT_GET_COLOR_MODES) + return get_unaligned_le32(mode + 4) == 1 && /* BT.601/709 */ + get_unaligned_le32(mode + 8) == 1 && /* SDR */ + get_unaligned_le32(mode + 12) == 1 && /* RGB */ + get_unaligned_le32(mode + 16) == 32; + return false; +} + +static int iboot_pattern_params(u64 iova, size_t size, u32 stride) +{ + /* The caller owns address allocation and both DART mappings. */ + if (!iova || (iova & 0x3fff) || stride != IBOOT_PATTERN_STRIDE || + size < (size_t)IBOOT_PATTERN_STRIDE * IBOOT_PATTERN_HEIGHT || + size > DCPEXT_MAX_BUFFER_SIZE || iova >= (1ULL << 42) || + size > (1ULL << 42) - iova) + return -EINVAL; + return 0; +} + +static void iboot_build_pattern_layer(struct iboot_swap_layer_v13_3 *cmd, + u64 iova, u32 stride) +{ + memset(cmd, 0, sizeof(*cmd)); + /* Layer zero, one linear BGRA plane with opaque alpha in the supplied pixels. */ + cmd->layer.planes[0].addr = cpu_to_le64(iova); + cmd->layer.planes[0].stride = cpu_to_le32(stride); + cmd->layer.planes[0].addr_format = cpu_to_le32(1); + cmd->layer.plane_count = cpu_to_le32(1); + cmd->layer.width = cpu_to_le32(IBOOT_PATTERN_WIDTH); + cmd->layer.height = cpu_to_le32(IBOOT_PATTERN_HEIGHT); + cmd->layer.surface_format = cpu_to_le32(1); /* BGRA */ + cmd->layer.colorspace = cpu_to_le32(1); /* sRGB */ + cmd->layer.eotf = cpu_to_le32(1); /* SDR */ + cmd->src[0] = cmd->dst[0] = cpu_to_le32(IBOOT_PATTERN_WIDTH); + cmd->src[1] = cmd->dst[1] = cpu_to_le32(IBOOT_PATTERN_HEIGHT); +} + +static int iboot_build_pattern_request(u32 op, const void *data, size_t data_size, + u8 *request, size_t capacity, size_t *size) +{ + size_t expected; + + switch (op) { + case 2: expected = 1; break; /* SetPower */ + case 6: expected = 2 * IBOOT_MODE_SIZE; break; /* SetMode */ + case 15: expected = 0; break; /* SwapBegin */ + case 16: expected = sizeof(struct iboot_swap_layer_v13_3); break; + case 18: expected = 12; break; /* SwapEnd */ + default: return -EINVAL; + } + if (data_size != expected || capacity < 16 + expected || + (expected && !data)) + return -EINVAL; + memset(request, 0, 16 + expected); + put_unaligned_le32(op, request); + put_unaligned_le32(16 + expected, request + 4); + if (expected) + memcpy(request + 16, data, expected); + *size = 16 + expected; + return 0; +} + +static int iboot_read_query(struct iboot_query *query, u32 op, u8 *reply, + const u8 **payload, size_t *size) +{ + struct apple_epic_service *service = query->service; + __le32 request[4]; + u32 retcode = 0; + int ret; + + if (READ_ONCE(query->stopping) || READ_ONCE(service->torndown)) + return -ENODEV; + ret = iboot_build_query(op, request); + if (ret) + return ret; + memset(reply, 0, IBOOT_QUERY_RX_SIZE); + ret = afk_send_command(service, EPIC_SUBTYPE_STD_SERVICE, request, + sizeof(request), reply, IBOOT_QUERY_RX_SIZE, &retcode); + if (ret) + return ret; + if (retcode) { + dev_warn(service->ep->dcp->dev, "iBoot query %u returned %#x\n", op, retcode); + return -EIO; + } + ret = iboot_reply_payload(reply, IBOOT_QUERY_RX_SIZE, op, payload, size); + if (ret) + dev_warn(service->ep->dcp->dev, + "iBoot query %u malformed header: opcode=%#x length=%#x\n", + op, get_unaligned_le32(reply), get_unaligned_le32(reply + 4)); + return ret; +} + +static int iboot_log_modes(struct iboot_query *query, u32 op, u8 *reply) +{ + struct device *dev = query->service->ep->dcp->dev; + const u8 *payload; + size_t size; + u32 count, i, selected = U32_MAX; + int ret; + + ret = iboot_read_query(query, op, reply, &payload, &size); + if (ret) + return ret; + ret = iboot_parse_modes(payload, size, &count); + if (ret) + return ret; + for (i = 0; i < count; i++) { + const u8 *mode = payload + 4 + i * IBOOT_MODE_SIZE; + + if (!iboot_pattern_mode(op, mode)) + continue; + if (op == IBOOT_GET_TIMING_MODES) { + memcpy(query->timing, mode, IBOOT_MODE_SIZE); + query->timing_valid = true; + } else { + memcpy(query->color, mode, IBOOT_MODE_SIZE); + query->color_valid = true; + } + selected = i; + break; + } + dev_info(dev, "iBoot %s modes: %u\n", + op == IBOOT_GET_TIMING_MODES ? "timing" : "color", count); + /* Count is bounded by the validated reply length, at most 682 records. */ + for (i = 0; i < count; i++) { + const u8 *mode = payload + 4 + i * IBOOT_MODE_SIZE; + u32 valid = get_unaligned_le32(mode); + + dev_info(dev, "iBoot mode op=%u index=%u selected=%u raw24=%*phN\n", + op, i, i == selected, IBOOT_MODE_SIZE, mode); + + if (op == IBOOT_GET_TIMING_MODES) { + u32 fps = get_unaligned_le32(mode + 12); + + dev_info(dev, "iBoot timing %u: valid=%u %ux%u fps=%u.%04u rawfps=%#x\n", + i, valid, get_unaligned_le32(mode + 4), + get_unaligned_le32(mode + 8), fps >> 16, + ((fps & 0xffff) * 10000) >> 16, fps); + } else { + dev_info(dev, "iBoot color %u: valid=%u colorimetry=%u eotf=%u encoding=%u bpp=%u\n", + i, valid, get_unaligned_le32(mode + 4), + get_unaligned_le32(mode + 8), get_unaligned_le32(mode + 12), + get_unaligned_le32(mode + 16)); + } + } + return 0; +} + +static int iboot_refresh_modes(struct iboot_query *query, u8 *reply) +{ + struct device *dev = query->service->ep->dcp->dev; + const u8 *payload; + size_t size; + u32 timings, colors; + bool hpd; + int ret; + + query->timing_valid = false; + query->color_valid = false; + ret = iboot_read_query(query, IBOOT_GET_HPD, reply, &payload, &size); + if (ret) + return ret; + ret = iboot_parse_hpd(payload, size, &hpd, &timings, &colors); + if (ret) + return ret; + dev_info(dev, "iBoot HPD=%u timing-count=%u color-count=%u\n", hpd, timings, colors); + if (!hpd) + return -ENOLINK; + ret = iboot_log_modes(query, IBOOT_GET_TIMING_MODES, reply); + if (!ret) + ret = iboot_log_modes(query, IBOOT_GET_COLOR_MODES, reply); + return ret; +} + +static void iboot_query_work(struct work_struct *work) +{ + struct iboot_query *query = container_of(work, struct iboot_query, work); + struct device *dev = query->service->ep->dcp->dev; + u8 *reply; + int ret; + + reply = kzalloc(IBOOT_QUERY_RX_SIZE, GFP_KERNEL); + if (!reply) { + ret = -ENOMEM; + goto done; + } + ret = iboot_refresh_modes(query, reply); + kfree(reply); +done: + if (ret && ret != -ENOLINK) + dev_warn(dev, "iBoot mode query failed: %d\n", ret); + atomic_set_release(&query->busy, 0); +} + +static struct iboot_query *iboot_find_query(struct apple_dcp *dcp) +{ + struct apple_dcp_afkep *ep = READ_ONCE(dcp->ibootep); + u32 i; + + if (!dcp->external || !ep) + return ERR_PTR(-ENODEV); + for (i = 0; i < AFK_MAX_CHANNEL; i++) { + struct iboot_query *query = smp_load_acquire(&ep->services[i].cookie); + + if (!query) + continue; + if (READ_ONCE(query->stopping) || READ_ONCE(query->service->torndown)) + return ERR_PTR(-ENODEV); + return query; + } + return ERR_PTR(-EAGAIN); +} + +bool ibootep_is_ready(struct apple_dcp *dcp) +{ + return !IS_ERR(iboot_find_query(dcp)); +} + +int ibootep_query_modes(struct apple_dcp *dcp) +{ + struct iboot_query *query = iboot_find_query(dcp); + + if (IS_ERR(query)) + return PTR_ERR(query); + if (atomic_cmpxchg(&query->busy, 0, 1)) + return -EBUSY; + /* AFK replies run on ep->wq; waiting on that same queue deadlocks. */ + queue_work(system_unbound_wq, &query->work); + return 0; +} + +/* Observational only: inner status fields for mutation ops are not decoded. + * AFK currently exposes capacity rather than the actual DMA reply length. + * Never trust the firmware length to enlarge this fixed diagnostic snapshot. + */ +static void iboot_log_pattern_reply(struct device *dev, u32 op, const u8 *reply) +{ + const u8 *payload; + size_t size; + int header = iboot_reply_payload(reply, IBOOT_QUERY_RX_SIZE, op, &payload, &size); + + dev_info(dev, "iBoot command %u EPIC success: inner-op=%u declared-len=%u header-check=%d raw64=%*phN (capacity snapshot, inner status unknown)\n", + op, get_unaligned_le32(reply), get_unaligned_le32(reply + 4), + header, 64, reply); +} + +static int iboot_pattern_command(struct iboot_query *query, u32 op, + const void *data, size_t data_size, u8 *reply) +{ + u8 request[16 + sizeof(struct iboot_swap_layer_v13_3)]; + size_t size; + u32 retcode = 0; + int ret; + + if (READ_ONCE(query->stopping) || READ_ONCE(query->service->torndown)) + return -ENODEV; + ret = iboot_build_pattern_request(op, data, data_size, request, sizeof(request), &size); + if (ret) + return ret; + memset(reply, 0, IBOOT_QUERY_RX_SIZE); + ret = afk_send_command(query->service, EPIC_SUBTYPE_STD_SERVICE, + request, size, reply, IBOOT_QUERY_RX_SIZE, &retcode); + if (retcode) { + dev_warn(query->service->ep->dcp->dev, + "iBoot pattern command %u returned %#x\n", op, retcode); + return -EIO; + } + if (!ret) + iboot_log_pattern_reply(query->service->ep->dcp->dev, op, reply); + return ret; +} + +int ibootep_present_pattern(struct apple_dcp *dcp, u64 iova, size_t size, u32 stride) +{ + struct iboot_query *query; + struct iboot_swap_layer_v13_3 layer; + u8 modes[2 * IBOOT_MODE_SIZE], power = 1, end[12] = {}; + const u8 *payload; + u8 *reply; + size_t payload_size; + u32 step = IBOOT_GET_HPD, swap_id = 0; + int ret; + + ret = iboot_pattern_params(iova, size, stride); + if (ret) + return ret; + query = iboot_find_query(dcp); + if (IS_ERR(query)) + return PTR_ERR(query); + if (dcp->fw_compat != DCP_FIRMWARE_V_14_7) + return -EOPNOTSUPP; + /* The caller uses its own sleepable worker, not the ordered AFK RX queue. */ + might_sleep(); + if (atomic_cmpxchg(&query->busy, 0, 1)) + return -EBUSY; + if (query->pattern_requested) { + ret = -EALREADY; + goto done; + } + /* Even failure is one-shot; a timeout leaves the firmware state uncertain. */ + query->pattern_requested = true; + reply = kzalloc(IBOOT_QUERY_RX_SIZE, GFP_KERNEL); + if (!reply) { + ret = -ENOMEM; + goto done; + } + ret = iboot_refresh_modes(query, reply); + if (ret) + goto free; + if (!query->timing_valid || !query->color_valid) { + ret = -ENODATA; + goto free; + } + memcpy(modes, query->timing, IBOOT_MODE_SIZE); + memcpy(modes + IBOOT_MODE_SIZE, query->color, IBOOT_MODE_SIZE); + iboot_build_pattern_layer(&layer, iova, stride); + step = 2; + ret = iboot_pattern_command(query, step, &power, sizeof(power), reply); + if (ret) + goto free; + /* Match m1n1's Sonoma power-on settling interval before changing timing. */ + msleep(100); + step = 6; + ret = iboot_pattern_command(query, step, modes, sizeof(modes), reply); + if (ret) + goto free; + step = 15; + ret = iboot_pattern_command(query, step, NULL, 0, reply); + if (ret) + goto free; + ret = iboot_reply_payload(reply, IBOOT_QUERY_RX_SIZE, step, &payload, &payload_size); + if (ret || payload_size < 20) { + ret = -EPROTO; + goto free; + } + swap_id = get_unaligned_le32(payload + 12); + step = 16; + ret = iboot_pattern_command(query, step, &layer, sizeof(layer), reply); + if (ret) + goto free; + step = 18; + ret = iboot_pattern_command(query, step, end, sizeof(end), reply); + if (!ret) + dev_info(dcp->dev, "iBoot EPIC sequence completed for 3840x2160 BGRA pattern; physical scanout unverified, stride %u IOVA %#llx swap %u; buffer retained\n", + stride, iova, swap_id); +free: + if (ret) + dev_warn(dcp->dev, "iBoot first-pattern step %u failed: %d; no retry, buffer must remain mapped\n", + step, ret); + kfree(reply); +done: + atomic_set_release(&query->busy, 0); + return ret; +} + +static ssize_t dcpext_modes_query_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + bool query; + int ret = kstrtobool(buf, &query); + + if (ret || !query) + return -EINVAL; + ret = ibootep_query_modes(dev_get_drvdata(dev)); + return ret ?: count; +} +static DEVICE_ATTR_WO(dcpext_modes_query); +static struct attribute *iboot_query_attrs[] = { &dev_attr_dcpext_modes_query.attr, NULL }; +static const struct attribute_group iboot_query_group = { .attrs = iboot_query_attrs }; + +static void iboot_query_cancel(void *data) +{ + struct iboot_query *query = data; + + WRITE_ONCE(query->stopping, true); + cancel_work_sync(&query->work); +} static void disp_service_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { + struct apple_dcp *dcp = service->ep->dcp; + struct iboot_query *query; + int ret; + + if (!dcp->external) + return; + query = devm_kzalloc(dcp->dev, sizeof(*query), GFP_KERNEL); + if (!query) + return; + query->service = service; + atomic_set(&query->busy, 0); + INIT_WORK(&query->work, iboot_query_work); + ret = devm_add_action_or_reset(dcp->dev, iboot_query_cancel, query); + if (ret) + return; + /* Remove the sysfs entry before cancelling work during device cleanup. */ + ret = devm_device_add_group(dcp->dev, &iboot_query_group); + if (ret) { + dev_warn(dcp->dev, "iBoot query interface unavailable: %d\n", ret); + return; + } + smp_store_release(&service->cookie, query); + ret = dcpext_scanout_register(dcp); + if (ret && ret != -ENODEV) + dev_warn(dcp->dev, "external pattern interface unavailable: %d\n", ret); + dev_info(dcp->dev, "external iBoot disp0 service ready on channel %u; mode queries are manual\n", + service->channel); } @@ -31,12 +576,19 @@ static const struct apple_epic_service_ops external_ibootep_ops[] = { int ibootep_init(struct apple_dcp *dcp) { const struct apple_epic_service_ops *ops = ibootep_ops; + int ret; + /* The T6030 external start path may race a second start. */ + if (dcp->ibootep && dcp->fw_compat == DCP_FIRMWARE_V_14_7) + return -EBUSY; if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) ops = external_ibootep_ops; dcp->ibootep = afk_init(dcp, DISP0_ENDPOINT, ops); - if (IS_ERR(dcp->ibootep)) - return PTR_ERR(dcp->ibootep); + if (IS_ERR(dcp->ibootep)) { + ret = PTR_ERR(dcp->ibootep); + dcp->ibootep = NULL; + return ret; + } return afk_start(dcp->ibootep); } diff --git a/drivers/gpu/drm/apple/ibootep.h b/drivers/gpu/drm/apple/ibootep.h new file mode 100644 index 0000000000000..f742333e15625 --- /dev/null +++ b/drivers/gpu/drm/apple/ibootep.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef _APPLE_DCP_IBOOTEP_H +#define _APPLE_DCP_IBOOTEP_H + +#include + +struct apple_dcp; + +/* Read-only publication check; does not send firmware commands. */ +bool ibootep_is_ready(struct apple_dcp *dcp); + +/* Queue external HPD/timing/color queries; never call synchronously from AFK RX. */ +int ibootep_query_modes(struct apple_dcp *dcp); + +/* Explicit, one-shot external presentation, called from sleepable work outside + * the AFK receive queue. Buffer: 3840x2160, stride 15360, bytes B,G,R,0xff. + * Caller verifies both DART mappings and retains the buffer until reboot, + * including after an error: commands may already have reached firmware. + */ +int ibootep_present_pattern(struct apple_dcp *dcp, u64 iova, size_t size, u32 stride); + +#endif diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index ab591667899a1..1f9cbcb8ed72f 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1717,9 +1717,12 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, case DCP_FIRMWARE_V_13_5: ret = iomfb_modeset_v13_3(dcp, crtc_state); break; - case DCP_FIRMWARE_V_14_7: + case DCP_FIRMWARE_V_14_7_TEMPLATE: ret = iomfb_modeset_v14_7_0(dcp, crtc_state); break; + case DCP_FIRMWARE_V_14_7: + ret = iomfb_v14_7_modeset(dcp, crtc_state); + break; case DCP_FIRMWARE_H17P: ret = iomfb_modeset_h17p(dcp, crtc_state); break; @@ -2112,11 +2115,15 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit) struct platform_device *pdev = to_apple_crtc(crtc)->dcp; struct apple_dcp *dcp = platform_get_drvdata(pdev); + if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { + iomfb_v14_7_flush(dcp, crtc, commit); + return; + } + if (iomfb_uses_queue(dcp)) { iomfb_queue_atomic(dcp, crtc, commit); return; } - /* * DCP does not complete swaps after a link loss. A plane-only commit * arriving between the reconnect callback and the required modeset @@ -2154,7 +2161,7 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit) case DCP_FIRMWARE_V_13_5: iomfb_flush_v13_3(dcp, crtc, commit); break; - case DCP_FIRMWARE_V_14_7: + case DCP_FIRMWARE_V_14_7_TEMPLATE: iomfb_flush_v14_7_0(dcp, crtc, commit); break; case DCP_FIRMWARE_H17P: @@ -2175,7 +2182,7 @@ static void iomfb_start(struct apple_dcp *dcp) case DCP_FIRMWARE_V_13_5: iomfb_start_v13_3(dcp); break; - case DCP_FIRMWARE_V_14_7: + case DCP_FIRMWARE_V_14_7_TEMPLATE: iomfb_start_v14_7_0(dcp); break; case DCP_FIRMWARE_H17P: @@ -2289,7 +2296,7 @@ void iomfb_shutdown(struct apple_dcp *dcp) case DCP_FIRMWARE_V_13_5: iomfb_shutdown_v13_3(dcp); break; - case DCP_FIRMWARE_V_14_7: + case DCP_FIRMWARE_V_14_7_TEMPLATE: iomfb_shutdown_v14_7_0(dcp); break; case DCP_FIRMWARE_H17P: diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index e6c5d85db122b..4f13cee71c872 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -8,7 +8,7 @@ #include "iomfb_h17p.h" #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" -#include "iomfb_v14_7.h" +#include "iomfb_v14_7_tpl.h" #include "version_utils.h" struct dcp_callback_size { diff --git a/drivers/gpu/drm/apple/iomfb_v12_3.c b/drivers/gpu/drm/apple/iomfb_v12_3.c index 9b87a35148231..bf4fa32d32c59 100644 --- a/drivers/gpu/drm/apple/iomfb_v12_3.c +++ b/drivers/gpu/drm/apple/iomfb_v12_3.c @@ -4,7 +4,7 @@ /* Include all per-version headers before defining DCP_FW ourself. */ #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" -#include "iomfb_v14_7.h" +#include "iomfb_v14_7_tpl.h" #include "iomfb_h17p.h" #include "version_utils.h" diff --git a/drivers/gpu/drm/apple/iomfb_v13_3.c b/drivers/gpu/drm/apple/iomfb_v13_3.c index 574c957f1643b..c25e7c4a0378c 100644 --- a/drivers/gpu/drm/apple/iomfb_v13_3.c +++ b/drivers/gpu/drm/apple/iomfb_v13_3.c @@ -4,7 +4,7 @@ /* Include all per-version headers before defining DCP_FW ourself. */ #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" -#include "iomfb_v14_7.h" +#include "iomfb_v14_7_tpl.h" #include "iomfb_h17p.h" #include "version_utils.h" diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index ccf3310267604..e0a9a82d3452e 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -1,119 +1,1534 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright The Asahi Linux Contributors */ -/* Include all per-version headers before defining DCP_FW ourself. */ -#include "iomfb_v12_3.h" -#include "iomfb_v13_3.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "afk.h" +#include "dcp.h" +#include "dcp-internal.h" +#include "dcpext_scanout.h" +#include "dcp-lifecycle.h" +#include "iomfb_internal.h" #include "iomfb_v14_7.h" -#include "iomfb_h17p.h" -#include "version_utils.h" - -static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { - IOMFB_METHOD("A000", dcpep_late_init_signal), - IOMFB_METHOD("A029", dcpep_setup_video_limits), - IOMFB_METHOD("A131", iomfbep_a131_pmu_service_matched), - IOMFB_METHOD("A132", iomfbep_a132_backlight_service_matched), - IOMFB_METHOD("A377", dcpep_set_create_dfb), - IOMFB_METHOD("A378", iomfbep_a358_vi_set_temperature_hint), - IOMFB_METHOD("A401", dcpep_start_signal), - IOMFB_METHOD("A406", dcpep_swap_start), - IOMFB_METHOD("A407", dcpep_swap_submit), - IOMFB_METHOD("A409", dcpep_set_display_device), - IOMFB_METHOD("A410", dcpep_is_main_display), - IOMFB_METHOD("A411", dcpep_set_digital_out_mode), - IOMFB_METHOD("A421", iomfbep_set_matrix), - IOMFB_METHOD("A425", iomfbep_get_color_remap_mode), - IOMFB_METHOD("A440", dcpep_set_parameter_dcp), - IOMFB_METHOD("A444", dcpep_create_default_fb), - IOMFB_METHOD("A448", dcpep_enable_disable_video_power_savings), - IOMFB_METHOD("A455", dcpep_first_client_open), - IOMFB_METHOD("A456", iomfbep_last_client_close), - IOMFB_METHOD("A463", dcpep_set_display_refresh_properties), - IOMFB_METHOD("A466", dcpep_flush_supports_power), - IOMFB_METHOD("A467", iomfbep_abort_swaps_dcp), - //IOMFB_METHOD("A471", dcpep_update_dfb_surface), - IOMFB_METHOD("A472", dcpep_set_power_state), +#include "iomfb_v14_7_link.h" +#include "iomfb_v14_7_swap.h" +#include "parser.h" + +#define A(n) DCP_V14_TAG('A', n) +#define D(n) DCP_V14_TAG('D', n) + +/* The image whose method and callback layouts this file implements. */ +#define DCP_V14_FIRMWARE_UUID "DDF38191-93B3-324A-BC8F-643006F5AC82" + +#define DCP_V14_CPU_CONTROL 0x44 +#define DCP_V14_CPU_STATUS 0x48 +#define DCP_V14_PMP_TIMEOUT msecs_to_jiffies(30000) + +/* + * The PMP report's wait-ready entry point is optional at link time: until a + * provider implements it, behave as its !CONFIG_APPLE_PMP_REPORT stub does. + */ +static int dcp_v14_pmp_wait_ready(struct device_node *entry, + unsigned long timeout) +{ + typeof(&apple_pmp_report_wait_ready) wait; + int ret; + + wait = symbol_get(apple_pmp_report_wait_ready); + if (!wait) + return -ENODEV; + ret = wait(entry, timeout); + symbol_put(apple_pmp_report_wait_ready); + return ret; +} +#define DCP_V14_RTKIT_RETRIES 4 +#define DCP_V14_MAX_PROPERTIES 256 +#define DCP_V14_MAX_RAW 32 +#define DCP_V14_MAX_RAW_BYTES SZ_8M +#define DCP_V14_MAX_BUFFERS 64 +#define DCP_V14_MAX_BUFFER SZ_16M +#define DCP_V14_MAX_BUFFER_BYTES SZ_64M +/* IOMFB layer of the primary plane. */ +#define DCP_V14_LAYER 0 +#define DCP_V14_BLACK 0xff000000 + +struct dcp_v14_property { + u32 service; + char key[64]; + u64 value; +}; + +struct dcp_v14_raw { + u32 service; + char key[64]; + void *data; + u32 size; }; -#define DCP_FW v14_7_0 -#define DCP_FW_VER DCP_FW_VERSION(14, 7, 0) - -#include "iomfb_template.c" - -static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { - [0] = trampoline_true, /* did_boot_signal */ - [1] = trampoline_true, /* did_power_on_signal */ - [2] = trampoline_nop, /* will_power_off_signal */ - [3] = trampoline_rt_bandwidth, - // [6] = trampoline_set_frame_sync_props, - [6] = trampoline_nop, - [100] = iomfbep_cb_match_pmu_service, - [101] = trampoline_zero, /* get_display_default_stride */ - [102] = trampoline_nop, /* set_number_property */ - [103] = trampoline_nop, /* trigger_user_cal_loader */ - [104] = trampoline_nop, /* set_boolean_property */ - [107] = trampoline_nop, /* remove_property */ - [108] = trampoline_true, /* create_provider_service */ - [109] = trampoline_true, /* create_product_service */ - [110] = trampoline_true, /* create_pmu_service */ - [111] = trampoline_true, /* create_iomfb_service */ - [112] = trampoline_create_backlight_service, - [113] = trampoline_true, /* create_nvram_service? */ - //[114] = trampoline_get_tiling_state, - //[115] = trampoline_false, /* set_tiling_state */ - [121] = dcpep_cb_boot_1, /* is_dark_boot */ - [122] = trampoline_false, /* is_dark_boot / is_waking_from_hibernate*/ - [123] = trampoline_false, - [125] = trampoline_read_edt_data, - [127] = trampoline_prop_start, - [128] = trampoline_prop_chunk, - [129] = trampoline_prop_end, - // [129] = trampoline_allocate_bandwidth, - [201] = trampoline_map_piodma, - [202] = trampoline_unmap_piodma, - [206] = iomfbep_cb_match_pmu_service_2, - [207] = iomfbep_cb_match_backlight_service, - [208] = trampoline_nop, /* update_backlight_factor_prop */ - [209] = trampoline_get_time, - [300] = trampoline_pr_publish, - [401] = trampoline_get_uint_prop, - [404] = trampoline_nop, /* sr_set_uint_prop */ - [406] = trampoline_set_fx_prop, - [408] = trampoline_get_frequency, - [411] = trampoline_map_reg, - [413] = trampoline_true, /* sr_set_property_dict */ - [414] = trampoline_sr_set_property_int, - [415] = trampoline_true, /* sr_set_property_bool */ - [451] = trampoline_allocate_buffer, - [452] = trampoline_map_physical, - [454] = trampoline_release_mem_desc, - [552] = trampoline_true, /* set_property_dict_0 */ - [561] = trampoline_true, /* set_property_dict */ - [563] = trampoline_true, /* set_property_int */ - [565] = trampoline_true, /* set_property_bool */ - [567] = trampoline_true, /* set_property_str */ - [574] = trampoline_zero, /* power_up_dart */ - [576] = trampoline_hotplug, - [577] = trampoline_nop, /* powerstate_notify */ - //[582] = trampoline_create_dfb_surface, /* create_default_fb_surface */ - [582] = trampoline_true, - [584] = trampoline_nop, /* IOMobileFramebufferAP::clear_default_surface */ - [588] = trampoline_nop, /* resize_default_fb_surface_gated */ - [589] = trampoline_swap_complete, - [591] = trampoline_swap_complete_intent_gated, - [592] = trampoline_abort_swap_ap_gated, - [593] = trampoline_enable_backlight_message_ap_gated, - [594] = trampoline_nop, /* IOMobileFramebufferAP::setSystemConsoleMode */ - [596] = trampoline_false, /* IOMobileFramebufferAP::isDFBAllocated */ - [597] = trampoline_false, /* IOMobileFramebufferAP::preserveContents */ - [598] = trampoline_nop, /* find_swap_function_gated */ +struct dcp_v14_buffer { + void *cpu; + dma_addr_t iova; + phys_addr_t phys; + size_t size; + bool retired; + bool piodma_mapped; }; -void DCP_FW_NAME(iomfb_start)(struct apple_dcp *dcp) + +struct apple_dcp_v14 { + struct device *dev; + /* Cleared when KMS unbinds; the firmware session outlives it. */ + struct apple_dcp *dcp; + struct apple_rtkit *rtk; + struct dcp_v14_link link; + + /* Owns the RPC stream: start, swaps and idle callbacks. */ + struct mutex lock; + struct work_struct idle_work; + bool failed; + bool started; + + /* Boot framebuffer and native panel timing (notch rows included). */ + u32 stride; + u32 fb_width, fb_height; + u32 panel_width, panel_height; + u64 clock_rate; + + struct dcp_v14_property properties[DCP_V14_MAX_PROPERTIES]; + u32 property_count; + struct dcp_v14_raw raw[DCP_V14_MAX_RAW]; + u32 raw_count, raw_bytes; + void *chunk; + u32 chunk_size, chunk_offset; + u64 analytics; + + struct dcp_v14_buffer buffers[DCP_V14_MAX_BUFFERS]; + u32 buffer_count; + u64 buffer_bytes; + struct platform_device *piodma; + struct iommu_domain *piodma_domain; + + /* Scanned out until the next swap completes. */ + struct drm_framebuffer *active_fb; + u64 swaps; + u64 swap_ns_max; +}; + +/* One DCP session per boot: RTKit is never started twice. */ +static bool dcp_v14_session; + +static int dcp_v14_callback(void *cookie, u32 tag, const void *input, u32 in_size, + void *output, u32 out_size); + +/* Called with the lock held, or from a callback. */ +static int dcp_v14_call(struct apple_dcp_v14 *v14, u32 tag, const void *in, u32 in_size, + void *out, u32 out_size, u32 completion) +{ + int ret; + + if (v14->failed) + return -EIO; + dev_dbg(v14->dev, "call %#x in %u out %u\n", tag, in_size, out_size); + ret = dcp_v14_link_call(&v14->link, tag, in, in_size, out, out_size, completion, + dcp_v14_callback, v14); + if (ret) { + v14->failed = true; + dev_err(v14->dev, "DCP call %#x failed: %d; recovery requires a reboot\n", + tag, ret); + } + return ret; +} + +/* A call with an optional u32 input of 1 and an optional u32 result to check. */ +static int dcp_v14_simple_call(struct apple_dcp_v14 *v14, u32 tag, bool input, bool reply, + bool expected) +{ + __le32 one = cpu_to_le32(1), result = 0; + int ret; + + ret = dcp_v14_call(v14, tag, input ? &one : NULL, input ? 4 : 0, + reply ? &result : NULL, reply ? 4 : 0, 0); + if (!ret && reply && le32_to_cpu(result) != expected) + return -EPROTO; + return ret; +} + +static struct dcp_v14_property *dcp_v14_property(struct apple_dcp_v14 *v14, u32 service, + const u8 *key, bool create) +{ + struct dcp_v14_property *p; + u32 i; + + if (!memchr(key, 0, 64)) + return ERR_PTR(-EINVAL); + for (i = 0; i < v14->property_count; i++) { + p = &v14->properties[i]; + if (p->service == service && !strcmp(p->key, key)) + return p; + } + if (!create) + return NULL; + if (v14->property_count == DCP_V14_MAX_PROPERTIES) + return ERR_PTR(-ENOSPC); + p = &v14->properties[v14->property_count++]; + p->service = service; + strscpy(p->key, key, sizeof(p->key)); + return p; +} + +static int dcp_v14_raw_property(struct apple_dcp_v14 *v14, u32 service, const u8 *key, + const void *data, u32 size) +{ + u32 i, old_size = 0; + void *copy; + + if (!memchr(key, 0, 64) || size > SZ_1M) + return -EINVAL; + for (i = 0; i < v14->raw_count; i++) + if (v14->raw[i].service == service && !strcmp(v14->raw[i].key, key)) + break; + if (i == DCP_V14_MAX_RAW) + return -ENOSPC; + if (i < v14->raw_count) + old_size = v14->raw[i].size; + if (v14->raw_bytes - old_size + size > DCP_V14_MAX_RAW_BYTES) + return -ENOSPC; + copy = kvmemdup(data, size ?: 1, GFP_KERNEL); + if (!copy) + return -ENOMEM; + if (i == v14->raw_count) + v14->raw_count++; + kvfree(v14->raw[i].data); + v14->raw[i].service = service; + strscpy(v14->raw[i].key, key, sizeof(v14->raw[i].key)); + v14->raw[i].data = copy; + v14->raw[i].size = size; + v14->raw_bytes += size - old_size; + return 0; +} + +/* The firmware's allocate-buffer callback wants a physically contiguous buffer. */ +static int dcp_v14_alloc(struct apple_dcp_v14 *v14, u64 request, u32 *id) +{ + struct dcp_v14_buffer *buf; + struct sg_table table; + dma_addr_t iova; + size_t size; + void *cpu; + int ret; + + if (!request || request > DCP_V14_MAX_BUFFER) + return -EINVAL; + size = ALIGN(request, SZ_16K); + if (v14->buffer_count == DCP_V14_MAX_BUFFERS || + v14->buffer_bytes + size > DCP_V14_MAX_BUFFER_BYTES) + return -ENOSPC; + cpu = dma_alloc_attrs(v14->dev, size, &iova, GFP_KERNEL, DMA_ATTR_FORCE_CONTIGUOUS); + if (!cpu) + return -ENOMEM; + ret = dma_get_sgtable_attrs(v14->dev, &table, cpu, iova, size, + DMA_ATTR_FORCE_CONTIGUOUS); + if (ret) + goto free; + if (table.orig_nents != 1 || table.sgl->length < size) { + sg_free_table(&table); + ret = -ERANGE; + goto free; + } + buf = &v14->buffers[v14->buffer_count]; + buf->phys = sg_phys(table.sgl); + sg_free_table(&table); + memset(cpu, 0, size); + dma_wmb(); + buf->cpu = cpu; + buf->iova = iova; + buf->size = size; + v14->buffer_bytes += size; + /* The firmware takes id 0 as a failure. */ + *id = ++v14->buffer_count; + dev_info(v14->dev, "DCP buffer %u: %zu bytes at %pad\n", *id, size, &iova); + return 0; +free: + dma_free_attrs(v14->dev, size, cpu, iova, DMA_ATTR_FORCE_CONTIGUOUS); + return ret; +} + +/* Maps a firmware buffer at the same address for the PIODMA stream. */ +static int dcp_v14_map_piodma(struct apple_dcp_v14 *v14, u32 id) +{ + struct dcp_v14_buffer *buf = &v14->buffers[id - 1]; + struct device_node *node; + size_t offset; + u32 marker; + int ret; + + if (buf->retired) + return -EINVAL; + if (buf->piodma_mapped) + return 0; + if (!v14->piodma) { + node = of_get_child_by_name(v14->dev->of_node, "piodma"); + if (!node || !of_device_is_available(node) || + of_property_read_u32(node, "apple,t6030-handoff", &marker) || marker != 1) { + of_node_put(node); + return -ENODEV; + } + v14->piodma = of_platform_device_create(node, NULL, v14->dev); + if (!v14->piodma) { + of_node_put(node); + return -ENOMEM; + } + ret = dma_set_mask_and_coherent(&v14->piodma->dev, DMA_BIT_MASK(42)); + if (!ret) + ret = of_dma_configure(&v14->piodma->dev, node, true); + of_node_put(node); + /* Never destroy a device attached to a locked DART stream. */ + if (ret) + return ret; + v14->piodma_domain = iommu_get_domain_for_dev(&v14->piodma->dev); + } + if (IS_ERR_OR_NULL(v14->piodma_domain)) + return -EIO; + ret = iommu_map(v14->piodma_domain, buf->iova, buf->phys, buf->size, + IOMMU_READ | IOMMU_WRITE, GFP_KERNEL); + if (ret) + return ret; + for (offset = 0; offset < buf->size; offset += SZ_16K) + if (iommu_iova_to_phys(v14->piodma_domain, buf->iova + offset) != + buf->phys + offset) + return -EIO; + buf->piodma_mapped = true; + dev_info(v14->dev, "PIODMA mapped DCP buffer %u: %zu bytes at %pad\n", + id, buf->size, &buf->iova); + return 0; +} + +/* Runs on the thread that owns the RPC stream; the lock is held. */ +static int dcp_v14_callback(void *cookie, u32 tag, const void *input, u32 in_size, + void *output, u32 out_size) +{ + struct apple_dcp_v14 *v14 = cookie; + const u8 *in = input; + u8 *out = output; + struct dcp_v14_property *p; + u32 count, offset, service, key_offset, id; + int ret; + +#define SHAPE(i, o) (in_size == (i) && out_size == (o)) + /* get_time */ + if (tag == D(209) && SHAPE(0, 8)) { + put_unaligned_le64(ktime_to_ms(ktime_get_real()), out); + return 0; + } + /* No default framebuffer is allocated by this host. */ + if ((tag == D(596) && SHAPE(0, 4)) || (tag == D(582) && SHAPE(8, 4))) + return 0; + /* + * Tiling state get and set: event, parameter, u32 value and a nullable + * byte. Like the M1/M2 hosts: no tiled display, no setter. + */ + if ((tag == D(115) && SHAPE(16, 8)) || (tag == D(116) && SHAPE(16, 4))) { + if (in[12] > 1) + return -EINVAL; + if (tag == D(115)) + out[4] = 1; + return 0; + } + /* + * Analytics event: name[64], serialized dictionary[4096], nullable + * flag. The dictionary is in/out and the firmware parses it even on + * failure, so it is echoed back unchanged; status 0 accepts it. + */ + if (tag == D(114) && SHAPE(4164, 4100)) { + if (!memchr(in, 0, 64) || in[4160] > 1 || (!in[4160] && in[64] != 'd')) + return -EINVAL; + if (!in[4160]) + memcpy(out, in + 64, 4096); + put_unaligned_le32(0, out + 4096); + v14->analytics++; + dev_dbg(v14->dev, "analytics event %llu: %.64s\n", v14->analytics, in); + return 0; + } + /* Swap information for a completed swap; the link checks its id. */ + if (tag == D(589) && SHAPE(0x6f0, 0)) + return 0; + if ((tag == D(588) && SHAPE(8, 0)) || (tag == D(598) && SHAPE(0, 0))) + return 0; + /* Main-display query, answered by asking the firmware. */ + if (tag == D(599) && SHAPE(0, 0)) + return dcp_v14_simple_call(v14, A(410), false, true, true); + /* Service creation and boot signals. */ + if ((tag == D(108) || tag == D(109) || tag == D(110) || tag == D(111) || + tag == D(112) || tag == D(113) || tag == D(0) || tag == D(1)) && SHAPE(0, 4)) { + out[0] = 1; + return 0; + } + /* Hotplug: echo the tiled-display record when present. */ + if (tag == D(576) && SHAPE(88, 76)) { + if (!(in[84] & 1)) + memcpy(out, in + 8, 76); + return 0; + } + if ((tag == D(577) && SHAPE(4, 0)) || (tag == D(300) && SHAPE(16, 0))) + return 0; + /* Chunked property transfer: start, chunk, end. */ + if (tag == D(127) && SHAPE(4, 4)) { + count = get_unaligned_le32(in); + if (v14->chunk || !count || count > 0x100001) + return -EINVAL; + v14->chunk_size = count - 1; + v14->chunk_offset = 0; + v14->chunk = kvzalloc(max_t(u32, 1, v14->chunk_size), GFP_KERNEL); + if (!v14->chunk) + return -ENOMEM; + out[0] = 1; + return 0; + } + if (tag == D(128) && SHAPE(0x1008, 4)) { + offset = get_unaligned_le32(in + 0x1000); + count = get_unaligned_le32(in + 0x1004); + if (!v14->chunk || offset != v14->chunk_offset || count > 4096 || + offset > v14->chunk_size || count > v14->chunk_size - offset) + return -EINVAL; + memcpy(v14->chunk + offset, in, count); + v14->chunk_offset += count; + out[0] = 1; + return 0; + } + if (tag == D(129) && SHAPE(64, 4)) { + if (!v14->chunk || v14->chunk_offset != v14->chunk_size || !memchr(in, 0, 64)) + return -EINVAL; + ret = dcp_v14_raw_property(v14, 0, in, v14->chunk, v14->chunk_size); + if (ret) + return ret; + kvfree(v14->chunk); + v14->chunk = NULL; + dev_dbg(v14->dev, "property %.64s: %u bytes\n", in, v14->chunk_size); + out[0] = 1; + return 0; + } + /* + * Display clock frequencies: index 0 is the display clock iBoot set + * up, index 1 is not used. Nothing here programs a clock. + */ + if (tag == D(408) && SHAPE(8, 8)) { + count = get_unaligned_le32(in + 4); + if (memcmp(in, "VORP", 4) || count > 1) + return -EINVAL; + put_unaligned_le64(count ? 0 : v14->clock_rate, out); + dev_dbg(v14->dev, "display clock %u: %llu Hz\n", count, get_unaligned_le64(out)); + return 0; + } + if (tag == D(125) && SHAPE(100, 36)) { + count = get_unaligned_le32(in + 64); + if (count > 8) + return -EINVAL; + memcpy(out, in + 68, count * 4); + return 0; + } + /* Frame sync properties: echo them when present. */ + if (tag == D(6) && SHAPE(60, 56)) { + if (!(in[56] & 1)) + memcpy(out, in, 56); + return 0; + } + /* Dark boot and hibernation wake: no. */ + if ((tag == D(122) || tag == D(123)) && SHAPE(0, 4)) + return 0; + /* Late boot: the calls the firmware expects from its host, in order. */ + if (tag == D(121) && SHAPE(0, 4)) { + static const struct { + u32 tag; + bool input, reply, expected; + } sequence[] = { + { A(444), false, true, true }, + { A(29), false, false, false }, + { A(466), true, false, false }, + { A(0), true, true, true }, + { A(463), false, true, true }, + }; + + for (count = 0; count < ARRAY_SIZE(sequence); count++) { + ret = dcp_v14_simple_call(v14, sequence[count].tag, sequence[count].input, + sequence[count].reply, sequence[count].expected); + if (ret) + return ret; + } + out[0] = 1; + return 0; + } + /* Default stride: that of the boot framebuffer. */ + if (tag == D(101) && SHAPE(0, 4)) { + put_unaligned_le32(v14->stride, out); + return 0; + } + /* Real-time bandwidth: explicitly declined. */ + if (tag == D(3) && SHAPE(4, 60)) { + put_unaligned_le32(1, out + 56); + return 0; + } + if (tag == D(451) && SHAPE(20, 28)) { + u32 alignment = get_unaligned_le32(in + 12); + u64 size = get_unaligned_le64(in + 4); + struct dcp_v14_buffer *buf; + + dev_dbg(v14->dev, "allocation flags %#x size %llu alignment %u\n", + get_unaligned_le32(in), size, alignment); + if (get_unaligned_le32(in) != 0x703 || !is_power_of_2(alignment) || + alignment > SZ_16K || in[16] || in[17] || in[18]) + return -EINVAL; + ret = dcp_v14_alloc(v14, size, &id); + if (ret) + return ret; + buf = &v14->buffers[id - 1]; + if ((buf->phys | buf->iova) & (alignment - 1)) + return -EINVAL; + put_unaligned_le64(buf->phys, out); + put_unaligned_le64(buf->iova, out + 8); + put_unaligned_le64(buf->size, out + 16); + put_unaligned_le32(id, out + 24); + return 0; + } + /* Map a buffer for PIODMA; the AP virtual address is not used. */ + if (tag == D(201) && SHAPE(12, 20)) { + u64 buffer = get_unaligned_le64(in); + + if (!buffer || buffer > v14->buffer_count || get_unaligned_le32(in + 8) != 1) + return -EINVAL; + ret = dcp_v14_map_piodma(v14, buffer); + if (ret) + return ret; + put_unaligned_le64(0, out); + put_unaligned_le64(v14->buffers[buffer - 1].iova, out + 8); + put_unaligned_le32(0, out + 16); + return 0; + } + /* Release: the buffer stays allocated and mapped until reboot. */ + if (tag == D(454) && SHAPE(4, 4)) { + id = get_unaligned_le32(in); + if (!id || id > v14->buffer_count || v14->buffers[id - 1].retired) + return -EINVAL; + v14->buffers[id - 1].retired = true; + out[0] = 1; + return 0; + } + /* Raw property read. */ + if (tag == D(400) && SHAPE(76, 0xc04)) { + count = get_unaligned_le32(in + 68); + service = get_unaligned_le32(in); + if (count > 0xc00 || (in[72] & 1) || !memchr(in + 4, 0, 64)) + return -EINVAL; + for (offset = 0; offset < v14->raw_count; offset++) { + struct dcp_v14_raw *raw = &v14->raw[offset]; + + if (raw->service != service || strcmp(raw->key, in + 4)) + continue; + if (raw->size > count) + return -ENOSPC; + memcpy(out, raw->data, raw->size); + put_unaligned_le32(raw->size, out + 0xc00); + break; + } + return 0; + } + /* Unsigned property read. */ + if (tag == D(401) && SHAPE(80, 12)) { + /* + * The PMU temperature the M1/M2 mini-LED hosts report: a fixed + * placeholder in centidegrees, not a measurement. + */ + if (!memcmp(in, "SUMP", 4) && !strncmp(in + 4, "Temperature", 64)) { + put_unaligned_le64(3029, out); + out[8] = 1; + return 0; + } + p = dcp_v14_property(v14, get_unaligned_le32(in), in + 4, false); + if (IS_ERR(p)) + return PTR_ERR(p); + if (p) { + put_unaligned_le64(p->value, out); + out[8] = 1; + } else { + dev_dbg(v14->dev, "no host property %#x %.64s\n", + get_unaligned_le32(in), in + 4); + } + return 0; + } + /* Dictionary properties, kept raw. */ + if ((tag == D(413) && SHAPE(4168, 4)) || + ((tag == D(552) || tag == D(561)) && SHAPE(4164, 4)) || + (tag == D(567) && SHAPE(128, 4))) { + key_offset = tag == D(413) ? 4 : 0; + service = key_offset ? get_unaligned_le32(in) : 0; + if (tag != D(567) && (in[key_offset + 4160] & 1)) { + out[0] = 1; + return 0; + } + count = tag == D(567) ? strnlen(in + 64, 64) : 4096; + ret = dcp_v14_raw_property(v14, service, in + key_offset, in + key_offset + 64, + count); + if (!ret) + out[0] = 1; + return ret; + } + /* Number and boolean properties. */ + if ((tag == D(563) && SHAPE(76, 4)) || (tag == D(565) && SHAPE(72, 4)) || + (tag == D(414) && SHAPE(80, 4)) || (tag == D(415) && SHAPE(76, 4)) || + ((tag == D(102) || tag == D(104)) && SHAPE(68, 0))) { + key_offset = (tag == D(414) || tag == D(415)) ? 4 : 0; + service = key_offset ? get_unaligned_le32(in) : 0; + count = (tag == D(563) || tag == D(414)) ? 8 : 4; + if (out_size && (in[key_offset + 64 + count] & 1)) { + out[0] = 1; + return 0; + } + p = dcp_v14_property(v14, service, in + key_offset, true); + if (IS_ERR(p)) + return PTR_ERR(p); + p->value = count == 8 ? get_unaligned_le64(in + key_offset + 64) : + get_unaligned_le32(in + key_offset + 64); + if (tag == D(104)) + p->value = in[64]; + if (out_size) + out[0] = 1; + return 0; + } + /* + * Property operations a host-service proxy replays when it starts + * with queued operations. They return nothing. + */ + if ((tag == D(402) && SHAPE(76, 0)) || (tag == D(404) && SHAPE(72, 0)) || + (tag == D(406) && SHAPE(72, 0)) || (tag == D(407) && SHAPE(72, 0))) { + dev_dbg(v14->dev, "proxy property callback %#x\n", tag); + return 0; + } + /* Property removal. */ + if (tag == D(107) && SHAPE(64, 0)) { + p = dcp_v14_property(v14, 0, in, false); + if (IS_ERR(p)) + return PTR_ERR(p); + if (p) { + *p = v14->properties[--v14->property_count]; + memset(&v14->properties[v14->property_count], 0, sizeof(*p)); + } + return 0; + } + /* PMU and backlight service matching, answered by the firmware. */ + if ((tag == D(206) || tag == D(207)) && SHAPE(0, 4)) { + ret = dcp_v14_simple_call(v14, tag == D(206) ? A(131) : A(132), false, true, + false); + if (!ret) + out[0] = 1; + return ret; + } + if (tag == D(100) && SHAPE(0, 0)) { + __le32 result; + + return dcp_v14_call(v14, A(374), NULL, 0, &result, sizeof(result), 0); + } +#undef SHAPE + dev_err(v14->dev, "unhandled DCP callback %#x %u/%u\n", tag, in_size, out_size); + return -EOPNOTSUPP; +} + +/* Firmware notifications that arrive while no call is in progress. */ +static void dcp_v14_idle(struct work_struct *work) +{ + struct apple_dcp_v14 *v14 = container_of(work, struct apple_dcp_v14, idle_work); + int ret = 0; + + mutex_lock(&v14->lock); + while (!v14->failed && !ret) + ret = dcp_v14_link_pump(&v14->link, 0, dcp_v14_callback, v14); + if (ret && ret != -ETIMEDOUT && ret != -EAGAIN) { + v14->failed = true; + dev_err(v14->dev, "DCP notification failed: %d; recovery requires a reboot\n", ret); + } + mutex_unlock(&v14->lock); +} + +static void dcp_v14_crashed(void *cookie, const void *crashlog, size_t crashlog_size) +{ + struct apple_dcp_v14 *v14 = cookie; + struct apple_dcp *dcp = READ_ONCE(v14->dcp); + + WRITE_ONCE(v14->failed, true); + if (dcp) { + WRITE_ONCE(dcp->crashed, true); + if (dcp->external) + dcpext_scanout_fault(dcp, -EIO); + } + dev_err(v14->dev, "DCP firmware crashed; its buffers are kept until reboot\n"); + dcp_v14_link_fail(&v14->link); +} + +static void dcp_v14_recv(void *cookie, u8 endpoint, u64 message) +{ + struct apple_dcp_v14 *v14 = cookie; + + if (endpoint == DISP0_ENDPOINT && v14->dcp && v14->dcp->external) { + if (v14->dcp->ibootep) + afk_receive_message(v14->dcp->ibootep, message); + return; + } + if (endpoint == DPTX_ENDPOINT) { + dev_info(v14->dev, "DPTX message %#llx\n", message); + if (v14->dcp && v14->dcp->dptxep) + afk_receive_message(v14->dcp->dptxep, message); + return; + } + if (endpoint == DPAV_CTRL_ENDPOINT) { + dev_info(v14->dev, "DPAV message %#llx\n", message); + if (v14->dcp && v14->dcp->dpavctrlep) + afk_receive_message(v14->dcp->dpavctrlep, message); + return; + } + if (endpoint == AV_ENDPOINT) { + dev_info(v14->dev, "AV message %#llx\n", message); + if (v14->dcp && v14->dcp->avep) + afk_receive_message(v14->dcp->avep, message); + return; + } + if (endpoint == DPAVSERV_ENDPOINT) { + dev_info(v14->dev, "DPAVSERV message %#llx\n", message); + if (v14->dcp && v14->dcp->dcpavservep) + afk_receive_message(v14->dcp->dcpavservep, message); + return; + } + if (endpoint != APPLE_DCP_LINK_ENDPOINT) { + dev_dbg(v14->dev, "ignored endpoint %#x message %#llx\n", endpoint, message); + return; + } + dcp_v14_link_receive(&v14->link, message); +} + +/* + * Buffers the firmware already owns must lie in a reserved region the boot + * loader described, and be mapped linearly there. The OS log is a physical + * address, not a DART address. + */ +static int dcp_v14_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct apple_dcp_v14 *v14 = cookie; + struct device *dev = v14->dev; + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + phys_addr_t phys; + int i; + + if (!bfr->size || bfr->size > SZ_16M || !domain) + return -EINVAL; + if (!bfr->iova) { + bfr->buffer = dma_alloc_coherent(dev, bfr->size, &bfr->iova, GFP_KERNEL); + if (!bfr->buffer) + return -ENOMEM; + memset(bfr->buffer, 0, bfr->size); + dma_wmb(); + return 0; + } + phys = iommu_iova_to_phys(domain, bfr->iova); + for (i = 0; ; i++) { + struct device_node *node = of_parse_phandle(dev->of_node, "memory-region", i); + struct resource res; + size_t off; + bool oslog, valid; + + if (!node) + return -ERANGE; + oslog = of_property_read_bool(node, "apple,dcp-os-log"); + valid = !of_address_to_resource(node, 0, &res); + of_node_put(node); + if (!valid) + continue; + if (oslog && bfr->iova == res.start) + phys = res.start; + if (phys < res.start || phys > res.end || bfr->size - 1 > res.end - phys) + continue; + if (!oslog) { + for (off = 0; off < bfr->size; off += SZ_16K) + if (iommu_iova_to_phys(domain, bfr->iova + off) != phys + off) + return -ERANGE; + if (iommu_iova_to_phys(domain, bfr->iova + bfr->size - 1) != + phys + bfr->size - 1) + return -ERANGE; + } + bfr->buffer = memremap(phys, bfr->size, MEMREMAP_WB); + if (!bfr->buffer) + return -ENOMEM; + bfr->is_mapped = true; + dev_dbg(dev, "RTKit buffer at %pad: %pa, %zu bytes\n", &bfr->iova, &phys, + bfr->size); + return 0; + } +} + +static void dcp_v14_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct apple_dcp_v14 *v14 = cookie; + + if (bfr->is_mapped) + memunmap(bfr->buffer); + else + dma_free_coherent(v14->dev, bfr->size, bfr->buffer, bfr->iova); +} + +static const struct apple_rtkit_ops dcp_v14_rtkit_ops = { + .crashed = dcp_v14_crashed, + .recv_message = dcp_v14_recv, + .shmem_setup = dcp_v14_shmem_setup, + .shmem_destroy = dcp_v14_shmem_destroy, +}; + +/* Freed only if RTKit never started: the firmware may call into it until reboot. */ +static void dcp_v14_release(void *data) +{ + struct apple_dcp_v14 *v14 = data; + + if (!v14->rtk) + kfree(v14); +} + +/* Read only: bind maps and claims these registers. */ +static int dcp_v14_cpu_running(struct device *dev) +{ + struct resource *res; + void __iomem *coproc; + u32 control; + + res = platform_get_resource_byname(to_platform_device(dev), IORESOURCE_MEM, "coproc"); + if (!res) + return -ENODEV; + coproc = ioremap_np(res->start, resource_size(res)); + if (!coproc) + return -ENOMEM; + control = readl(coproc + DCP_V14_CPU_CONTROL); + iounmap(coproc); + if (!(control & APPLE_DCP_COPROC_CPU_CONTROL_RUN)) + return dev_err_probe(dev, -EBUSY, + "T6030 display not started: the DCP CPU is stopped (%#x); the PMP is not started for it\n", + control); + return 0; +} + +int iomfb_v14_7_probe(struct apple_dcp *dcp) +{ + struct device *dev = dcp->dev; + struct device_node *np = dev->of_node, *entry; + struct apple_dcp_v14 *v14; + const char *uuid = NULL; + u32 marker = 0; + int ret; + + if (of_property_read_string(np, "apple,firmware-uuid", &uuid) || + strcmp(uuid, DCP_V14_FIRMWARE_UUID)) + return dev_err_probe(dev, -ENODEV, + "T6030 display not started: DCP firmware %s is not supported\n", + uuid ?: "(unknown)"); + if (of_property_read_u32(np, "apple,t6030-handoff", &marker) || marker != 1) + return dev_err_probe(dev, -ENODEV, + "T6030 display not started: no boot loader display handoff\n"); + if (!iommu_get_domain_for_dev(dev)) + return dev_err_probe(dev, -ENODEV, + "T6030 display not started: the DCP has no DART domain\n"); + ret = dcp_v14_cpu_running(dev); + if (ret) + return ret; + + entry = of_parse_phandle(np, "apple,pmp-report", 0); + if (!entry) + return dev_err_probe(dev, -ENODEV, + "T6030 display not started: no apple,pmp-report, the PMP is not described\n"); + ret = dcp_v14_pmp_wait_ready(entry, DCP_V14_PMP_TIMEOUT); + of_node_put(entry); + if (ret == -EPROBE_DEFER) + return dev_err_probe(dev, ret, "waiting for the PMP report\n"); + if (ret) + return dev_err_probe(dev, ret, + "T6030 display not started: the PMP has not acknowledged the display request; the boot framebuffer stays\n"); + + v14 = kzalloc_obj(*v14); + if (!v14) + return -ENOMEM; + v14->dev = dev; + v14->dcp = dcp; + mutex_init(&v14->lock); + INIT_WORK(&v14->idle_work, dcp_v14_idle); + ret = devm_add_action_or_reset(dev, dcp_v14_release, v14); + if (ret) + return ret; + dcp->v14 = v14; + dev_info(dev, "PMP running with the display request acknowledged\n"); + return 0; +} + +/* The boot framebuffer gives the stride and, with the notch rows, the panel size. */ +static int dcp_v14_geometry(struct apple_dcp_v14 *v14) +{ + struct device_node *fb = of_find_compatible_node(NULL, NULL, "simple-framebuffer"); + u32 notch = 0; + int ret; + + if (!fb) + return -ENODEV; + ret = of_property_read_u32(fb, "width", &v14->fb_width); + if (!ret) + ret = of_property_read_u32(fb, "height", &v14->fb_height); + if (!ret) + ret = of_property_read_u32(fb, "stride", &v14->stride); + of_node_put(fb); + if (ret) + return ret; + /* The boot loader hides the notch rows from the boot framebuffer. */ + of_property_read_u32(v14->dev->of_node, "apple,notch-height", ¬ch); + /* The firmware takes this stride as its default: 4 bytes per pixel. */ + if (!v14->fb_width || !v14->fb_height || v14->stride != v14->fb_width * 4 || + notch > MAX_NOTCH_HEIGHT) + return -EINVAL; + v14->panel_width = v14->fb_width; + v14->panel_height = v14->fb_height + notch; + return 0; +} + +static void dcpext_bringup(struct work_struct *work); + +/* Verify the mappings installed by the IOMMU core before touching the ASC. */ +static int dcpext_verify_memory(struct apple_dcp *dcp) +{ + static const char *const names[] = {"asc-firmware", "dcp_data", "heap"}; + static const u64 iovas[] = {0x10040000000ULL, 0x10080000000ULL, 0x100c0000000ULL}; + struct device *dev = dcp->dev; + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + u32 ready; + int i; + + if (!domain || of_property_read_u32(dev->of_node, + "apple,t6030-dcpext-memory-ready", &ready) || ready != 1) + return -EINVAL; + for (i = 0; i < ARRAY_SIZE(names); i++) { + struct device_node *mem; + struct resource res; + u64 off, size; + int idx = of_property_match_string(dev->of_node, "memory-region-names", names[i]); + if (idx < 0) + return idx; + mem = of_parse_phandle(dev->of_node, "memory-region", idx); + if (!mem) + return -EINVAL; + if (!of_property_present(mem, "no-map") || of_address_to_resource(mem, 0, &res)) { + of_node_put(mem); + return -EINVAL; + } + of_node_put(mem); + size = resource_size(&res); + if (!size || size > SZ_256M || !IS_ALIGNED(res.start, SZ_16K) || + !IS_ALIGNED(size, SZ_16K)) + return -EINVAL; + for (off = 0; off < size; off += SZ_16K) { + if (iommu_iova_to_phys(domain, iovas[i] + off) != res.start + off) { + dev_err(dev, "dcpext %s mapping mismatch at %#llx; CPU untouched\n", + names[i], iovas[i] + off); + return -EINVAL; + } + } + dev_info(dev, "dcpext verified %s: %pa + %#llx at %#llx\n", + names[i], &res.start, size, iovas[i]); + } + return 0; +} + +static ssize_t dcpext_start_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + bool start; + int ret = kstrtobool(buf, &start); + + if (ret || !start) + return -EINVAL; + /* Serialize the one-shot handoff with idle system suspend. */ + guard(mutex)(&dcp->hpd_mutex); + if (READ_ONCE(dcp->external_suspended)) + return -EBUSY; + if (!dcp_v14_session) + return -EAGAIN; + if (atomic_cmpxchg(&dcp->external_requested, 0, 1)) + return -EBUSY; + /* Keep code and firmware callbacks alive after this one-shot request. */ + __module_get(THIS_MODULE); + schedule_work(&dcp->external_work); + return count; +} +static DEVICE_ATTR_WO(dcpext_start); + +static struct attribute *dcpext_attrs[] = { &dev_attr_dcpext_start.attr, NULL }; +static const struct attribute_group dcpext_group = { .attrs = dcpext_attrs }; + +static void dcpext_cancel(void *data) +{ + struct apple_dcp *dcp = data; + + cancel_work_sync(&dcp->external_work); +} + +int iomfb_v14_7_external_start(struct apple_dcp *dcp) +{ + int ret; + + /* This path does not run component bind, which sets the panel's mask. + * The external DART window also starts above 1 TiB. + */ + ret = dma_set_mask_and_coherent(dcp->dev, DMA_BIT_MASK(42)); + if (ret) + return dev_err_probe(dcp->dev, ret, "dcpext requires 42-bit DMA\n"); + ret = dcpext_verify_memory(dcp); + if (ret) + return ret; + atomic_set(&dcp->external_requested, 0); + INIT_WORK(&dcp->external_work, dcpext_bringup); + ret = devm_add_action_or_reset(dcp->dev, dcpext_cancel, dcp); + if (ret) + return ret; + ret = devm_device_add_group(dcp->dev, &dcpext_group); + if (!ret) + dev_info(dcp->dev, "dcpext memory verified; waiting for explicit dcpext_start\n"); + return ret; +} + +/* Refuse startup unless both the hardware floor and PMP vote are in place. */ +static int dcpext_check_power(struct apple_dcp *dcp) { - dcp->cb_handlers = cb_handlers; + struct device *dev = dcp->dev; + struct device_node *ps, *entry; + struct regmap *map; + const char *label; + u32 offset, value; + int ret; - dcp_start_signal(dcp, false, dcp_started, NULL); + ps = of_parse_phandle(dev->of_node, "power-domains", 0); + if (!ps) + return -ENODEV; + ret = of_property_read_string(ps, "label", &label); + if (ret || strcmp(label, "dispext0_cpu")) { + of_node_put(ps); + return -EINVAL; + } + ret = of_property_read_u32(ps, "reg", &offset); + map = syscon_node_to_regmap(ps->parent); + of_node_put(ps); + if (ret) + return ret; + if (IS_ERR(map)) + return PTR_ERR(map); + ret = regmap_read(map, offset, &value); + if (ret) + return ret; + if (((value >> 16) & 0xf) != 0xf || ((value >> 4) & 0xf) != 0xf) + return dev_err_probe(dev, -EIO, + "dcpext power floor is not active: %#x\n", value); + + entry = of_parse_phandle(dev->of_node, "apple,pmp-report", 0); + if (!entry) + return -ENODEV; + ret = dcp_v14_pmp_wait_ready(entry, DCP_V14_PMP_TIMEOUT); + of_node_put(entry); + return ret; } -#undef DCP_FW_VER -#undef DCP_FW +static void dcpext_bringup(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(work, struct apple_dcp, external_work); + struct apple_dcp_v14 *v14; + struct device *dev = dcp->dev; + struct resource *res; + struct apple_rtkit *rtk; + u32 control; + int ret, n; + + ret = dcpext_check_power(dcp); + if (ret) { + dev_err(dev, "dcpext startup refused: power prerequisites failed: %d\n", ret); + return; + } + dev_info(dev, "dcpext CPU power floor active and PMP request acknowledged\n"); + + res = platform_get_resource_byname(to_platform_device(dev), + IORESOURCE_MEM, "coproc"); + if (!res) { + dev_err(dev, "dcpext has no coproc register\n"); + return; + } + dcp->coproc_reg = devm_ioremap_resource(dev, res); + if (IS_ERR(dcp->coproc_reg)) { + dev_err(dev, "dcpext coproc map failed: %ld\n", + PTR_ERR(dcp->coproc_reg)); + dcp->coproc_reg = NULL; + return; + } + + control = readl(dcp->coproc_reg + DCP_V14_CPU_CONTROL); + dev_info(dev, "dcpext CPU control %#x\n", control); + ret = dcpext_verify_memory(dcp); + if (ret) + return; + + v14 = kzalloc_obj(*v14); + if (!v14) + return; + v14->dev = dev; + v14->dcp = dcp; + mutex_init(&v14->lock); + INIT_WORK(&v14->idle_work, dcp_v14_idle); + dcp->v14 = v14; + dcp_v14_link_init(&v14->link, dev, NULL); + + rtk = apple_rtkit_init(dev, v14, "mbox", 0, &dcp_v14_rtkit_ops); + if (IS_ERR(rtk)) { + dev_err(dev, "dcpext RTKit init failed: %ld\n", PTR_ERR(rtk)); + return; + } + v14->rtk = rtk; + v14->link.rtk = rtk; + dcp->rtk = rtk; + + if (!(control & APPLE_DCP_COPROC_CPU_CONTROL_RUN)) { + writel(control | APPLE_DCP_COPROC_CPU_CONTROL_RUN, + dcp->coproc_reg + DCP_V14_CPU_CONTROL); + dev_info(dev, "dcpext CPU started by explicit request\n"); + } + ret = apple_rtkit_wake(rtk); + for (n = 0; ret == -ETIME && n < DCP_V14_RTKIT_RETRIES; n++) + ret = apple_rtkit_boot(rtk); + if (ret) { + dev_err(dev, "dcpext RTKit did not wake: %d\n", ret); + return; + } + + dev_info(dev, "dcpext RTKit session running\n"); + if (apple_rtkit_has_endpoint(rtk, DPAV_CTRL_ENDPOINT)) + dpav_ctrl_init(dcp); + if (apple_rtkit_has_endpoint(rtk, DPTX_ENDPOINT)) + dptxep_init(dcp); + /* External mode discovery only; this does not power or modeset a display. */ + if (apple_rtkit_has_endpoint(rtk, DISP0_ENDPOINT)) { + ret = ibootep_init(dcp); + if (ret) + dev_err(dev, "dcpext mode-query endpoint failed: %d\n", ret); + } +} + +int iomfb_v14_7_bind(struct apple_dcp *dcp) +{ + struct apple_dcp_v14 *v14 = dcp->v14; + struct device *dev = dcp->dev; + struct apple_rtkit *rtk; + u32 control, status; + struct clk *clk; + int ret, n; + + if (!v14) + return -ENODEV; + if (v14->rtk || dcp_v14_session) + return dev_err_probe(dev, -EBUSY, + "T6030 display not started: an earlier DCP session is kept; reboot to restart the display\n"); + + control = readl(dcp->coproc_reg + DCP_V14_CPU_CONTROL); + status = readl(dcp->coproc_reg + DCP_V14_CPU_STATUS); + dev_info(dev, "DCP CPU control %#x status %#x\n", control, status); + if (!(control & APPLE_DCP_COPROC_CPU_CONTROL_RUN)) + return dev_err_probe(dev, -EBUSY, + "T6030 display not started: the DCP CPU is stopped, and interrupted firmware is never resumed\n"); + + ret = dcp_v14_geometry(v14); + if (ret) + return dev_err_probe(dev, ret, + "T6030 display not started: no usable boot framebuffer\n"); + clk = clk_get(dev, NULL); + if (IS_ERR(clk)) + return dev_err_probe(dev, PTR_ERR(clk), + "T6030 display not started: no display clock\n"); + v14->clock_rate = clk_get_rate(clk); + clk_put(clk); + if (!v14->clock_rate) + return dev_err_probe(dev, -EINVAL, + "T6030 display not started: the display clock has no rate\n"); + + dcp_v14_link_init(&v14->link, dev, NULL); + rtk = apple_rtkit_init(dev, v14, "mbox", 0, &dcp_v14_rtkit_ops); + if (IS_ERR(rtk)) + return dev_err_probe(dev, PTR_ERR(rtk), + "T6030 display not started: RTKit init failed\n"); + /* From here nothing is freed and the module stays loaded. */ + v14->rtk = rtk; + v14->link.rtk = rtk; + dcp->rtk = rtk; + dcp_v14_session = true; + __module_get(THIS_MODULE); + + ret = apple_rtkit_wake(rtk); + for (n = 0; ret == -ETIME && n < DCP_V14_RTKIT_RETRIES; n++) + ret = apple_rtkit_boot(rtk); + if (ret) { + v14->failed = true; + return dev_err_probe(dev, ret, + "T6030 display not started: the DCP RTKit session did not wake; the boot framebuffer stays\n"); + } + dev_info(dev, "DCP RTKit session running\n"); + /* The internal panel uses IOMFB only; dock DPTX belongs to dcpext. */ + return 0; +} + +void iomfb_v14_7_unbind(struct apple_dcp *dcp) +{ + struct apple_dcp_v14 *v14 = dcp->v14; + + dcp->active = false; + dcp_mode_invalidate(&dcp->mode_state); + if (!v14) + return; + WRITE_ONCE(v14->dcp, NULL); + if (v14->rtk) + dev_info(dcp->dev, "display unbound; the DCP session and its buffers are kept until reboot\n"); +} + +static int dcp_v14_status_show(struct seq_file *m, void *unused) +{ + struct apple_dcp_v14 *v14 = m->private; + struct drm_framebuffer *fb; + int ret; + + ret = mutex_lock_interruptible(&v14->lock); + if (ret) + return ret; + fb = v14->active_fb; + seq_printf(m, "started %d\nfailed %d\nswaps %llu\nswap_max_us %llu\n", + v14->started, v14->failed, v14->swaps, v14->swap_ns_max / NSEC_PER_USEC); + seq_printf(m, "active_fb %u\nimported %d\n", fb ? fb->base.id : 0, + fb && fb->obj[0]->import_attach); + seq_printf(m, "panel %ux%u\nboot_fb %ux%u stride %u\nclock %llu\n", + v14->panel_width, v14->panel_height, v14->fb_width, v14->fb_height, + v14->stride, v14->clock_rate); + seq_printf(m, "buffers %u bytes %llu\nproperties %u raw %u\nanalytics %llu\n", + v14->buffer_count, v14->buffer_bytes, v14->property_count, v14->raw_count, + v14->analytics); + mutex_unlock(&v14->lock); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(dcp_v14_status); + +/* One mode: the native timing of the boot panel, less the hidden notch rows. */ +static int dcp_v14_mode(struct apple_dcp *dcp, struct apple_dcp_v14 *v14) +{ + struct dcp_display_mode *modes, *best = NULL; + struct dcp_parse_ctx ctx; + unsigned int count, i; + void *blob = NULL; + u32 size = 0; + int ret; + + for (i = 0; i < v14->raw_count; i++) { + if (!v14->raw[i].service && !strcmp(v14->raw[i].key, "PreferredTimingElements")) { + size = v14->raw[i].size; + blob = v14->raw[i].data; + break; + } + } + if (!blob) + return -ENODATA; + ret = parse(blob, size, &ctx); + if (ret) + return ret; + ctx.dcp = dcp; + modes = enumerate_modes(&ctx, &count, dcp->panel.width_mm, dcp->panel.height_mm, + dcp->notch_height, true, false); + if (IS_ERR(modes)) + return PTR_ERR(modes); + for (i = 0; i < count; i++) { + struct drm_display_mode *m = &modes[i].mode; + int hz = drm_mode_vrefresh(m); + + dev_dbg(dcp->dev, "timing %u/%u: " DRM_MODE_FMT "%s\n", i + 1, count, + DRM_MODE_ARG(m), m->type & DRM_MODE_TYPE_PREFERRED ? " best" : ""); + if (m->hdisplay != v14->panel_width || + m->vdisplay != v14->panel_height - dcp->notch_height || + (hz != 60 && hz != 120)) + continue; + if (!best || (m->type & DRM_MODE_TYPE_PREFERRED)) + best = &modes[i]; + } + if (!best) { + dev_err(dcp->dev, "no %ux%u timing at 60 or 120 Hz among %u\n", v14->panel_width, + v14->panel_height - dcp->notch_height, count); + kfree(modes); + return -EINVAL; + } + best->mode.type |= DRM_MODE_TYPE_PREFERRED; + dcp->modes = kmemdup(best, sizeof(*best), GFP_KERNEL); + kfree(modes); + if (!dcp->modes) + return -ENOMEM; + dcp->nr_modes = 1; + return 0; +} + +int iomfb_v14_7_start(struct apple_dcp *dcp) +{ + struct apple_dcp_v14 *v14 = dcp->v14; + const struct drm_display_mode *mode; + const char *step; + int ret; + + if (!v14 || !v14->rtk || v14->failed || v14->started) + return -ENODEV; + + step = "DCPLink"; + ret = dcp_v14_link_start(&v14->link); + if (ret) + goto fail; + + mutex_lock(&v14->lock); + step = "start signal"; + ret = dcp_v14_simple_call(v14, A(401), false, true, true); + if (!ret) { + step = "first client open"; + ret = dcp_v14_simple_call(v14, A(455), false, false, false); + } + mutex_unlock(&v14->lock); + if (ret) + goto fail; + dcp_v14_link_set_idle_work(&v14->link, &v14->idle_work); + + step = "panel mode"; + mutex_lock(&v14->lock); + ret = dcp_v14_mode(dcp, v14); + mutex_unlock(&v14->lock); + if (ret) + goto fail; + + v14->started = true; + /* Never removed, like the session it describes. */ + debugfs_create_file("dcp-t6030", 0400, NULL, v14, &dcp_v14_status_fops); + dcp->connector->connected = true; + dcp_set_dimensions(dcp); + scoped_guard(spinlock_irqsave, &dcp->mode_state.lock) + dcp->mode_state.valid = true; + dcp->active = true; + complete(&dcp->start_done); + + mode = &dcp->modes[0].mode; + dev_info(dcp->dev, "T6030 display started: %ux%u@%d, %u notch rows %s, %ux%u mm\n", + mode->hdisplay, mode->vdisplay, drm_mode_vrefresh(mode), + dcp->notch_height ?: v14->panel_height - v14->fb_height, + dcp->notch_height ? "hidden" : "shown", mode->width_mm, mode->height_mm); + return 0; +fail: + v14->failed = true; + dev_err(dcp->dev, + "T6030 display not started: %s failed: %d; the boot framebuffer stays (reboot to retry)\n", + step, ret); + return ret; +} + +/* Swap start, then the swap; returns once the firmware completed that swap. */ +static int dcp_v14_swap(struct apple_dcp_v14 *v14, const u8 *surface, u64 iova, + u32 width, u32 height, u32 dst_y) +{ + __le32 start[4] = {}, started[2], result[3]; + u8 *swap; + u32 id; + int ret; + + if (surface && (!iova || !width || width > v14->panel_width || !height || + dst_y > v14->panel_height || height > v14->panel_height - dst_y)) + return -EINVAL; + swap = kmalloc(DCP_V14_SWAP_SIZE, GFP_KERNEL); + if (!swap) + return -ENOMEM; + + mutex_lock(&v14->lock); + ret = dcp_v14_call(v14, A(406), start, sizeof(start), started, sizeof(started), 0); + if (!ret && le32_to_cpu(started[1])) + ret = -EIO; + id = le32_to_cpu(started[0]); + if (!ret && !id) + ret = -EPROTO; + if (!ret) { + dcp_v14_encode_swap(swap, id, DCP_V14_BLACK, surface, iova, width, height, + dst_y, DCP_V14_LAYER); + ret = dcp_v14_call(v14, A(407), swap, DCP_V14_SWAP_SIZE, result, sizeof(result), + id); + /* The swap status sits at byte 5 of the reply. */ + if (!ret && get_unaligned_le32((u8 *)result + 5)) + ret = -EIO; + } + if (ret) + v14->failed = true; + mutex_unlock(&v14->lock); + + kfree(swap); + return ret; +} + +/* Shows @fb (or only the black background) and keeps it until the next swap. */ +static bool dcp_v14_present(struct apple_dcp_v14 *v14, struct drm_framebuffer *fb, u32 dst_y) +{ + u8 surface[DCP_V14_SURFACE_SIZE]; + struct drm_framebuffer *old; + u64 iova = 0, start, elapsed, swaps; + int ret; + + if (READ_ONCE(v14->failed)) + return false; + if (fb) { + drm_framebuffer_get(fb); + dcp_v14_encode_surface(surface, fb->pitches[0], fb->width, fb->height, + fb->format->format == DRM_FORMAT_XRGB8888); + iova = drm_fb_dma_get_gem_obj(fb, 0)->dma_addr; + } + start = ktime_get_ns(); + ret = dcp_v14_swap(v14, fb ? surface : NULL, iova, fb ? fb->width : 0, + fb ? fb->height : 0, dst_y); + elapsed = ktime_get_ns() - start; + if (ret) { + /* Either framebuffer may still be scanned out: keep both. */ + dev_err(v14->dev, "native DCP flip failed %d; buffers pinned until reboot\n", ret); + return false; + } + mutex_lock(&v14->lock); + old = v14->active_fb; + v14->active_fb = fb; + swaps = ++v14->swaps; + v14->swap_ns_max = max(v14->swap_ns_max, elapsed); + mutex_unlock(&v14->lock); + if (old) + drm_framebuffer_put(old); + if (swaps <= 3) + dev_info(v14->dev, "swap %llu complete: surface %d, imported %d, %llu us\n", + swaps, !!fb, fb && fb->obj[0]->import_attach, elapsed / NSEC_PER_USEC); + return true; +} + +/* Releases helper waiters without reporting a completed flip. */ +static void dcp_v14_cancel_event(struct apple_dcp *dcp) +{ + struct apple_crtc *crtc = dcp->crtc; + struct drm_pending_vblank_event *event; + unsigned long flags; + + spin_lock_irqsave(&crtc->base.dev->event_lock, flags); + event = crtc->event; + crtc->event = NULL; + spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); + if (!event) + return; + if (event->base.fence) { + dma_fence_set_error(event->base.fence, -EIO); + dma_fence_signal(event->base.fence); + } + if (event->base.completion) { + complete_all(event->base.completion); + if (event->base.completion_release) + event->base.completion_release(event->base.completion); + event->base.completion = NULL; + } + drm_event_cancel_free(crtc->base.dev, &event->base); +} + +/* Only the layout the firmware was qualified with: one full-mode linear plane. */ +int iomfb_v14_7_atomic_check(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *state) +{ + struct apple_dcp_v14 *v14 = dcp->v14; + struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + struct drm_plane_state *p = drm_atomic_get_new_plane_state(state, crtc->primary); + struct drm_gem_dma_object *obj; + struct drm_framebuffer *fb; + u32 width, height; + + if (dcp->crashed || !v14 || READ_ONCE(v14->failed)) + return -EIO; + if (!crtc_state || !crtc_state->active || !p || p->crtc != crtc || !p->visible) + return 0; + + fb = p->fb; + width = crtc_state->mode.hdisplay; + height = crtc_state->mode.vdisplay; + if (fb->format->format != DRM_FORMAT_XRGB8888 && + fb->format->format != DRM_FORMAT_ARGB8888) + return -EINVAL; + if (fb->modifier != DRM_FORMAT_MOD_LINEAR || fb->offsets[0] || + fb->pitches[0] < width * 4 || (fb->pitches[0] & 63)) + return -EINVAL; + if (p->src_x || p->src_y || p->crtc_x || p->crtc_y || + p->src_w != width << 16 || p->src_h != height << 16 || + p->crtc_w != width || p->crtc_h != height || + fb->width != width || fb->height != height) + return -EINVAL; + obj = drm_fb_dma_get_gem_obj(fb, 0); + if (!obj || !obj->dma_addr || (u64)fb->pitches[0] * height > obj->base.size) + return -EINVAL; + return 0; +} + +int iomfb_v14_7_modeset(struct apple_dcp *dcp, struct drm_crtc_state *crtc_state) +{ + /* The firmware keeps the timing it booted with, which is the only mode. */ + if (!lookup_mode(dcp, &crtc_state->mode)) + return -EINVAL; + scoped_guard(spinlock_irqsave, &dcp->mode_state.lock) + dcp->mode_state.valid = true; + return 0; +} + +void iomfb_v14_7_flush(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *state) +{ + struct drm_plane_state *p = drm_atomic_get_new_plane_state(state, crtc->primary); + struct drm_framebuffer *fb; + + if (!p) + p = crtc->primary->state; + fb = p && p->visible ? p->fb : NULL; + dcp->swap_start = ktime_get(); + if (dcp->v14 && dcp_v14_present(dcp->v14, fb, dcp->notch_height)) + dcp_drm_crtc_page_flip(dcp, ktime_get()); + else + dcp_v14_cancel_event(dcp); +} + +void iomfb_v14_7_poweron(struct apple_dcp *dcp) +{ + /* The panel stays on; the next swap shows the plane again. */ +} + +void iomfb_v14_7_poweroff(struct apple_dcp *dcp) +{ + /* Blank to black; the panel and the DCP stay powered. */ + if (dcp->v14 && dcp->v14->started) + dcp_v14_present(dcp->v14, NULL, 0); +} diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.h b/drivers/gpu/drm/apple/iomfb_v14_7.h index 7fd9b47fd372b..b3f1efd380fdf 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7.h @@ -1,17 +1,32 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright The Asahi Linux Contributors */ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ #ifndef __APPLE_IOMFB_V14_7_H__ #define __APPLE_IOMFB_V14_7_H__ -#include "version_utils.h" +#include -#define DCP_FW v14_7_0 -#define DCP_FW_VER DCP_FW_VERSION(14, 7, 0) +struct apple_dcp; +struct apple_dcp_v14; +struct drm_atomic_commit; +struct drm_crtc; +struct drm_crtc_state; -#include "iomfb_template.h" +/* Platform probe: firmware identity, boot loader handoff, PMP acknowledgment. */ +int iomfb_v14_7_probe(struct apple_dcp *dcp); +/* Component bind: the running DCP's RTKit session. */ +int iomfb_v14_7_bind(struct apple_dcp *dcp); +int iomfb_v14_7_external_start(struct apple_dcp *dcp); +/* Component unbind: KMS goes away; the firmware session is kept until reboot. */ +void iomfb_v14_7_unbind(struct apple_dcp *dcp); +/* DCPLink, start signal, first client open and the panel mode. */ +int iomfb_v14_7_start(struct apple_dcp *dcp); -#undef DCP_FW_VER -#undef DCP_FW +int iomfb_v14_7_atomic_check(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *state); +int iomfb_v14_7_modeset(struct apple_dcp *dcp, struct drm_crtc_state *crtc_state); +void iomfb_v14_7_flush(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *state); +void iomfb_v14_7_poweron(struct apple_dcp *dcp); +void iomfb_v14_7_poweroff(struct apple_dcp *dcp); #endif /* __APPLE_IOMFB_V14_7_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_link.c b/drivers/gpu/drm/apple/iomfb_v14_7_link.c new file mode 100644 index 0000000000000..5eeb86e96f282 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v14_7_link.c @@ -0,0 +1,464 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iomfb_v14_7_link.h" + +#define DCP_V14_LINK_ALIGNMENT 64 +#define DCP_V14_LINK_READY_MSG ((64ULL << 16) | 0x101) +#define DCP_V14_READY_TIMEOUT msecs_to_jiffies(3000) +#define DCP_V14_CALL_TIMEOUT (5 * HZ) +#define DCP_V14_TAG_D589 DCP_V14_TAG('D', 589) +#define DCP_V14_D589_IN_SIZE 0x6f0 + +enum dcp_v14_event_kind { + DCP_V14_EVENT_REPLY = 1, + DCP_V14_EVENT_CALLBACK = 2, +}; + +struct dcp_v14_event { + struct list_head list; + u32 kind; + u32 size; + u64 message; + u8 data[]; +}; + +/* Called with the lock held. */ +static void link_fail(struct dcp_v14_link *link) +{ + if (!link->failed) + dev_err(link->dev, "DCP link stopped; firmware buffers are kept until reboot\n"); + link->failed = true; + wake_up(&link->wait); +} + +/* Called with the lock held. */ +static void link_event(struct dcp_v14_link *link, u32 kind, u64 message, + const void *data, u32 size) +{ + struct dcp_v14_event *e; + + if (link->event_count >= DCP_V14_LINK_MAX_EVENTS || + size > APPLE_DCP_LINK_STREAM_BUFFER_SIZE) { + link_fail(link); + return; + } + e = kmalloc(struct_size(e, data, size), GFP_KERNEL); + if (!e) { + link_fail(link); + return; + } + e->kind = kind; + e->size = size; + e->message = message; + if (size) + memcpy(e->data, data, size); + list_add_tail(&e->list, &link->events); + link->event_count++; + wake_up(&link->wait); + if (link->idle_work) + queue_work(system_unbound_wq, link->idle_work); +} + +void dcp_v14_link_init(struct dcp_v14_link *link, struct device *dev, + struct apple_rtkit *rtk) +{ + link->dev = dev; + link->rtk = rtk; + mutex_init(&link->lock); + init_waitqueue_head(&link->wait); + INIT_LIST_HEAD(&link->events); + init_completion(&link->ready); +} + +void dcp_v14_link_fail(struct dcp_v14_link *link) +{ + mutex_lock(&link->lock); + link_fail(link); + mutex_unlock(&link->lock); + complete(&link->ready); +} + +bool dcp_v14_link_failed(struct dcp_v14_link *link) +{ + return READ_ONCE(link->failed); +} + +void dcp_v14_link_set_idle_work(struct dcp_v14_link *link, struct work_struct *work) +{ + mutex_lock(&link->lock); + link->idle_work = work; + if (link->event_count) + queue_work(system_unbound_wq, work); + mutex_unlock(&link->lock); +} + +/* Host-driven start: endpoint, INIT with the RPC memory, then READY. */ +int dcp_v14_link_start(struct dcp_v14_link *link) +{ + struct apple_dcp_link_init_desc *desc; + u64 message; + int ret; + + link->rpc = dma_alloc_coherent(link->dev, APPLE_DCP_LINK_RPC_MEMORY_SIZE, + &link->rpc_dva, GFP_KERNEL); + if (!link->rpc) + return -ENOMEM; + memset(link->rpc, 0, APPLE_DCP_LINK_RPC_MEMORY_SIZE); + desc = link->rpc; + apple_dcp_link_init_desc_encode(desc, 0, false); + /* This firmware reads the version field as the RPC alignment. */ + desc->version = cpu_to_le32(DCP_V14_LINK_ALIGNMENT); + ret = apple_dcp_link_init_message(link->rpc_dva, &message); + if (ret) + return ret; + + link->ready_result = -ETIMEDOUT; + ret = apple_rtkit_start_ep(link->rtk, APPLE_DCP_LINK_ENDPOINT); + if (ret) + return ret; + dma_wmb(); + ret = apple_rtkit_send_message(link->rtk, APPLE_DCP_LINK_ENDPOINT, message, + NULL, false); + if (ret) + return ret; + if (!wait_for_completion_timeout(&link->ready, DCP_V14_READY_TIMEOUT)) + return -ETIMEDOUT; + + mutex_lock(&link->lock); + ret = link->failed ? -EIO : link->ready_result; + if (!ret) + link->enabled = true; + mutex_unlock(&link->lock); + if (!ret) + dev_info(link->dev, "DCPLink ready, RPC alignment %d\n", DCP_V14_LINK_ALIGNMENT); + return ret; +} + +/* RTKit receive worker: check, record and queue; never blocks on a call. */ +void dcp_v14_link_receive(struct dcp_v14_link *link, u64 message) +{ + struct apple_dcp_link_stream_layout layout; + struct apple_dcp_link_rpc_view view; + u8 stream = apple_dcp_link_message_stream(message); + bool remote = apple_dcp_link_message_is_remote(message); + bool nested; + + mutex_lock(&link->lock); + if (link->failed) + goto out; + + if (!link->enabled) { + /* Firmware-side INIT needs no answer; READY ends the handshake. */ + if ((message & 3) == APPLE_DCP_LINK_MSG_FIRMWARE_INIT) + goto out; + link->ready_result = message == DCP_V14_LINK_READY_MSG ? 0 : -EPROTO; + if (link->ready_result) + dev_err(link->dev, "unexpected DCPLink ready message %#llx\n", message); + mutex_unlock(&link->lock); + complete(&link->ready); + return; + } + + if ((message & 3) != APPLE_DCP_LINK_MSG_RPC) { + dev_err(link->dev, "unexpected DCPLink message %#llx\n", message); + goto error; + } + dma_rmb(); + + if (message & BIT_ULL(6)) { + /* The reply to the innermost call, echoed in place. */ + nested = link->calls[1].active; + if (stream || !link->calls[nested].active || + link->calls[nested].remote != remote || + apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, 0, remote, &layout)) + goto error; + if (memcmp(link->rpc + layout.local_offset + link->calls[nested].offset, + &link->calls[nested].header, sizeof(link->calls[nested].header))) + goto error; + link_event(link, DCP_V14_EVENT_REPLY, message, + link->rpc + layout.local_offset + link->calls[nested].offset, + link->calls[nested].size); + link->calls[nested].active = false; + goto out; + } + + if (link->callback_count >= DCP_V14_LINK_MAX_CALLBACKS || + apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, stream, remote, &layout) || + apple_dcp_link_rpc_decode(message, link->rpc + layout.remote_offset, + layout.capacity, &view)) + goto error; + if ((view.call >> 24) != 'D') { + dev_err(link->dev, "DCPLink callback with tag %#x\n", view.call); + goto error; + } + link->callbacks[link->callback_count].message = message; + link->callbacks[link->callback_count].output = view.output; + link->callbacks[link->callback_count].size = view.output_size; + link->callbacks[link->callback_count].call = view.call; + link->callback_count++; + dev_dbg(link->dev, "callback %#x in %u out %u\n", + view.call, view.input_size, view.output_size); + link_event(link, DCP_V14_EVENT_CALLBACK, message, + link->rpc + layout.remote_offset + view.offset, view.total_size); + goto out; +error: + link_fail(link); +out: + mutex_unlock(&link->lock); +} + +/* Places a call packet (header, input, zeroed output) and sends it. */ +static int link_submit(struct dcp_v14_link *link, const struct apple_dcp_link_rpc_header *rpc, + u32 size, bool nested) +{ + struct apple_dcp_link_stream_layout layout; + u32 total, offset = 0, input_size, output_size; + bool remote = false; + u64 message; + int ret; + + mutex_lock(&link->lock); + if (!link->enabled || link->failed) { + ret = -EIO; + goto out; + } + if (link->calls[nested].active || (nested && !link->callback_count) || + (!nested && link->callback_count)) { + ret = -EBUSY; + goto out; + } + if (nested) { + u64 callback = link->callbacks[link->callback_count - 1].message; + + if (apple_dcp_link_message_stream(callback)) { + ret = -EOPNOTSUPP; + goto out; + } + /* A nested call goes to the callback's side of stream 0, after the outer call. */ + remote = apple_dcp_link_message_is_remote(callback); + if (link->calls[0].active && link->calls[0].remote == remote) + offset = ALIGN(link->calls[0].size, DCP_V14_LINK_ALIGNMENT); + } + input_size = le32_to_cpu(rpc->input_size); + output_size = le32_to_cpu(rpc->output_size); + ret = apple_dcp_link_rpc_payload_size(input_size, output_size, &total); + if (ret || total != size) { + ret = -EMSGSIZE; + goto out; + } + ret = apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, 0, remote, &layout); + if (ret) + goto out; + if (offset + size > layout.capacity) { + ret = -EMSGSIZE; + goto out; + } + memcpy(link->rpc + layout.local_offset + offset, rpc, size); + memset(link->rpc + layout.local_offset + offset + sizeof(*rpc) + input_size, 0, + output_size); + link->calls[nested].header = *rpc; + link->calls[nested].size = size; + link->calls[nested].offset = offset; + link->calls[nested].remote = remote; + link->calls[nested].active = true; + apple_dcp_link_rpc_message(offset, size, false, &message); + apple_dcp_link_stream_message(message, 0, remote, &message); + dma_wmb(); + ret = apple_rtkit_send_message(link->rtk, APPLE_DCP_LINK_ENDPOINT, message, NULL, false); + if (ret) + link_fail(link); +out: + mutex_unlock(&link->lock); + return ret; +} + +/* Writes the innermost callback's output in place and acknowledges it. */ +static int link_reply(struct dcp_v14_link *link, u64 message, const void *output, u32 size) +{ + unsigned int top; + int ret = -EINVAL; + + mutex_lock(&link->lock); + if (!link->enabled || link->failed) { + ret = -EIO; + goto out; + } + if (!link->callback_count) + goto out; + top = link->callback_count - 1; + if (message != link->callbacks[top].message || size != link->callbacks[top].size) + goto out; + memcpy(link->callbacks[top].output, output, size); + dma_wmb(); + ret = apple_rtkit_send_message(link->rtk, APPLE_DCP_LINK_ENDPOINT, + apple_dcp_link_rpc_reply(message), NULL, false); + if (ret) + link_fail(link); + else + link->callback_count--; +out: + mutex_unlock(&link->lock); + return ret; +} + +/* Uninterruptible: an abandoned call would leave the firmware mid-call. */ +static struct dcp_v14_event *link_next_event(struct dcp_v14_link *link, unsigned long timeout) +{ + struct dcp_v14_event *e; + + if (!wait_event_timeout(link->wait, + READ_ONCE(link->event_count) || READ_ONCE(link->failed), + timeout)) + return ERR_PTR(-ETIMEDOUT); + mutex_lock(&link->lock); + if (link->failed || list_empty(&link->events)) { + mutex_unlock(&link->lock); + return ERR_PTR(-EIO); + } + e = list_first_entry(&link->events, struct dcp_v14_event, list); + list_del(&e->list); + link->event_count--; + mutex_unlock(&link->lock); + return e; +} + +static int link_callback(struct dcp_v14_link *link, struct dcp_v14_event *e, + dcp_v14_callback_fn callback, void *cookie, u32 *completed) +{ + struct apple_dcp_link_rpc_header *h = (void *)e->data; + u32 in, out, tag; + int ret; + + if (e->kind != DCP_V14_EVENT_CALLBACK || e->size < sizeof(*h) || !callback) + return -EPROTO; + in = le32_to_cpu(h->input_size); + out = le32_to_cpu(h->output_size); + tag = le32_to_cpu(h->call); + if (in > e->size - sizeof(*h) || out != e->size - sizeof(*h) - in) + return -EPROTO; + memset(e->data + sizeof(*h) + in, 0, out); + link->callback_depth++; + ret = callback(cookie, tag, h + 1, in, e->data + sizeof(*h) + in, out); + link->callback_depth--; + if (ret) + return ret; + if (tag == DCP_V14_TAG_D589 && in == DCP_V14_D589_IN_SIZE && completed) + *completed = get_unaligned_le32(h + 1); + return link_reply(link, e->message, e->data + sizeof(*h) + in, out); +} + +int dcp_v14_link_pump(struct dcp_v14_link *link, unsigned long timeout, + dcp_v14_callback_fn callback, void *cookie) +{ + struct dcp_v14_event *e = link_next_event(link, timeout); + int ret; + + if (IS_ERR(e)) + return PTR_ERR(e); + ret = link_callback(link, e, callback, cookie, NULL); + kfree(e); + return ret; +} + +/* + * Calls @tag and handles the callbacks that arrive until its reply, and, for + * a non-zero @completion_id, until the swap-complete callback for that swap. + * Must not be called concurrently with itself or dcp_v14_link_pump(), except + * from inside a callback handler. + */ +int dcp_v14_link_call(struct dcp_v14_link *link, u32 tag, const void *input, + u32 input_size, void *output, u32 output_size, + u32 completion_id, dcp_v14_callback_fn callback, void *cookie) +{ + struct apple_dcp_link_rpc_header *packet; + unsigned long deadline = jiffies + DCP_V14_CALL_TIMEOUT; + u64 expected = APPLE_DCP_LINK_MSG_RPC_REPLY; + u32 size, completed = 0; + bool nested, replied = false; + int ret; + + ret = apple_dcp_link_rpc_payload_size(input_size, output_size, &size); + if (ret || size > APPLE_DCP_LINK_STREAM_BUFFER_SIZE) + return -EMSGSIZE; + packet = kzalloc(size, GFP_KERNEL); + if (!packet) + return -ENOMEM; + apple_dcp_link_rpc_header_encode(packet, tag, input_size, output_size); + if (input_size) + memcpy(packet + 1, input, input_size); + +retry: + /* + * Handle queued notifications before an outer call. A queued + * callback is not an enclosing call until its handler runs. + */ + if (!link->callback_depth) { + while (READ_ONCE(link->event_count)) { + ret = dcp_v14_link_pump(link, 0, callback, cookie); + if (ret) { + kfree(packet); + return ret; + } + } + } + mutex_lock(&link->lock); + nested = link->callback_depth != 0; + if (nested && !link->callback_count) { + mutex_unlock(&link->lock); + kfree(packet); + return -EPROTO; + } + /* The reply to a nested call carries the callback's side. */ + if (nested) + expected |= link->callbacks[link->callback_count - 1].message & + APPLE_DCP_LINK_MSG_REMOTE; + mutex_unlock(&link->lock); + ret = link_submit(link, packet, size, nested); + /* A callback arrived between the drain and the submit. */ + if (ret == -EBUSY && !nested && time_before(jiffies, deadline)) + goto retry; + kfree(packet); + if (ret) + return ret; + + while (time_before(jiffies, deadline)) { + struct dcp_v14_event *e = link_next_event(link, max_t(long, deadline - jiffies, 0)); + struct apple_dcp_link_rpc_header *h; + + if (IS_ERR(e)) + return PTR_ERR(e); + h = (void *)e->data; + if (e->kind == DCP_V14_EVENT_REPLY) { + if (replied || e->size != size || e->message != expected || + le32_to_cpu(h->call) != tag || + le32_to_cpu(h->input_size) != input_size || + le32_to_cpu(h->output_size) != output_size) { + ret = -EPROTO; + } else { + if (output_size) + memcpy(output, e->data + sizeof(*h) + input_size, + output_size); + replied = true; + } + } else { + ret = link_callback(link, e, callback, cookie, &completed); + if (!ret && completion_id && completed && completed != completion_id) + ret = -EPROTO; + } + kfree(e); + if (ret) + return ret; + if (replied && (!completion_id || completed == completion_id)) + return 0; + } + return -ETIMEDOUT; +} diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_link.h b/drivers/gpu/drm/apple/iomfb_v14_7_link.h new file mode 100644 index 0000000000000..95e6ccdbb63d2 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v14_7_link.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ + +#ifndef __APPLE_IOMFB_V14_7_LINK_H__ +#define __APPLE_IOMFB_V14_7_LINK_H__ + +#include +#include +#include +#include +#include +#include + +#include "dcp-link.h" + +#define DCP_V14_LINK_MAX_EVENTS 32 +#define DCP_V14_LINK_MAX_CALLBACKS 8 + +/* Method and callback tags: 'A' or 'D' followed by three decimal digits. */ +#define DCP_V14_TAG(c, n) (((u32)(c) << 24) | ((u32)('0' + (n) / 100) << 16) | \ + ((u32)('0' + (n) / 10 % 10) << 8) | (u32)('0' + (n) % 10)) + +struct apple_rtkit; +struct work_struct; + +/* + * Handles one callback from the firmware. @out is zeroed and is sent back + * as the reply when the handler returns 0; any error stops the link. + */ +typedef int (*dcp_v14_callback_fn)(void *cookie, u32 tag, const void *in, + u32 in_size, void *out, u32 out_size); + +struct dcp_v14_link { + struct device *dev; + struct apple_rtkit *rtk; + void *rpc; + dma_addr_t rpc_dva; + + /* Protects everything below; never held across a callback handler. */ + struct mutex lock; + wait_queue_head_t wait; + struct list_head events; + unsigned int event_count; + unsigned int callback_count; + /* Callback handlers running on the caller's stack. */ + unsigned int callback_depth; + bool failed; + bool enabled; + struct work_struct *idle_work; + + struct completion ready; + int ready_result; + + /* Our calls: [0] outer, [1] made from inside a callback. */ + struct { + bool active; + bool remote; + u32 offset; + u32 size; + struct apple_dcp_link_rpc_header header; + } calls[2]; + + /* Firmware callbacks awaiting a reply, innermost last. */ + struct { + u64 message; + void *output; + u32 size; + u32 call; + } callbacks[DCP_V14_LINK_MAX_CALLBACKS]; +}; + +void dcp_v14_link_init(struct dcp_v14_link *link, struct device *dev, + struct apple_rtkit *rtk); +int dcp_v14_link_start(struct dcp_v14_link *link); +void dcp_v14_link_receive(struct dcp_v14_link *link, u64 message); +void dcp_v14_link_fail(struct dcp_v14_link *link); +bool dcp_v14_link_failed(struct dcp_v14_link *link); +void dcp_v14_link_set_idle_work(struct dcp_v14_link *link, struct work_struct *work); +int dcp_v14_link_call(struct dcp_v14_link *link, u32 tag, const void *input, + u32 input_size, void *output, u32 output_size, + u32 completion_id, dcp_v14_callback_fn callback, void *cookie); +int dcp_v14_link_pump(struct dcp_v14_link *link, unsigned long timeout, + dcp_v14_callback_fn callback, void *cookie); + +#endif /* __APPLE_IOMFB_V14_7_LINK_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_swap.h b/drivers/gpu/drm/apple/iomfb_v14_7_swap.h new file mode 100644 index 0000000000000..aab3351dd9eea --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v14_7_swap.h @@ -0,0 +1,118 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ + +#ifndef __APPLE_IOMFB_V14_7_SWAP_H__ +#define __APPLE_IOMFB_V14_7_SWAP_H__ + +#include +#include +#include + +#define DCP_V14_SWAP_SIZE 0x1b58 +#define DCP_V14_SURFACE_SIZE 0x22c + +/* Offsets in the swap record. Rectangles are x, y, w, h (u32 each). */ +#define DCP_V14_SWAP_ID 0x98 +#define DCP_V14_SWAP_SURF_ID(p) (0x9c + (p) * 4) +#define DCP_V14_SWAP_SRC(p) (0xac + (p) * 16) /* in the surface */ +#define DCP_V14_SWAP_DST(p) (0x10c + (p) * 16) /* on the panel */ +#define DCP_V14_SWAP_ENABLED 0x14c +#define DCP_V14_SWAP_COMPLETED 0x150 +#define DCP_V14_SWAP_BACKGROUND 0x154 +#define DCP_V14_SWAP_SURFACE(p) (0x508 + (p) * DCP_V14_SURFACE_SIZE) +#define DCP_V14_SWAP_IOVA(p) (0xdb8 + (p) * 8) +#define DCP_V14_SWAP_SURF_NULL(p) (0x1b4b + (p)) +#define DCP_V14_SWAP_SURF_NULL_COUNT 10 + +#define DCP_V14_SWAP_LAYERS 0x7 +#define DCP_V14_SWAP_SET_BACKGROUND BIT(31) + +/* Offsets in a surface record. */ +#define DCP_V14_SURF_OPAQUE 0x02 +#define DCP_V14_SURF_FORMAT 0x0b +#define DCP_V14_SURF_STRIDE 0x15 +#define DCP_V14_SURF_WIDTH 0x21 +#define DCP_V14_SURF_HEIGHT 0x25 +#define DCP_V14_SURF_SIZE 0x29 +#define DCP_V14_SURF_PLANE_WIDTH 0x59 +#define DCP_V14_SURF_PLANE_HEIGHT 0x5d +#define DCP_V14_SURF_PLANE_STRIDE 0x69 +#define DCP_V14_SURF_PLANE_SIZE 0x6d + +/* 'BGRA': 8-bit linear, ARGB8888 in DRM terms. */ +#define DCP_V14_FORMAT_BGRA 0x42475241 + +/* + * A single-plane linear BGRA surface covering the whole framebuffer. The + * firmware has no XRGB format: an opaque surface blends as premultiplied + * alpha over the black background, which shows XRGB pixels unchanged. + */ +static inline void dcp_v14_encode_surface(u8 *s, u32 stride, u32 width, u32 height, + bool opaque) +{ + u32 bytes = stride * height; + + memset(s, 0, DCP_V14_SURFACE_SIZE); + s[DCP_V14_SURF_OPAQUE] = opaque; + put_unaligned_le32(1, s + 0x03); + put_unaligned_le32(1, s + 0x07); + put_unaligned_le32(DCP_V14_FORMAT_BGRA, s + DCP_V14_SURF_FORMAT); + s[0x13] = 13; + s[0x14] = 12; + put_unaligned_le32(stride, s + DCP_V14_SURF_STRIDE); + put_unaligned_le16(1, s + 0x19); + s[0x1b] = 1; + s[0x1c] = 1; + put_unaligned_le32(width, s + DCP_V14_SURF_WIDTH); + put_unaligned_le32(height, s + DCP_V14_SURF_HEIGHT); + put_unaligned_le32(bytes, s + DCP_V14_SURF_SIZE); + put_unaligned_le32(1, s + 0x35); + put_unaligned_le64(1, s + 0x51); + put_unaligned_le32(width, s + DCP_V14_SURF_PLANE_WIDTH); + put_unaligned_le32(height, s + DCP_V14_SURF_PLANE_HEIGHT); + put_unaligned_le32(stride, s + DCP_V14_SURF_PLANE_STRIDE); + put_unaligned_le32(bytes, s + DCP_V14_SURF_PLANE_SIZE); + put_unaligned_le16(4, s + 0x71); + s[0x73] = 1; + s[0x74] = 1; + put_unaligned_le64(1, s + 0x149); +} + +static inline void dcp_v14_put_rect(u8 *r, u32 x, u32 y, u32 w, u32 h) +{ + put_unaligned_le32(x, r); + put_unaligned_le32(y, r + 4); + put_unaligned_le32(w, r + 8); + put_unaligned_le32(h, r + 12); +} + +/* + * The swap record for swap @id. @background fills every panel pixel that no + * surface covers: the whole panel when @surface is NULL. With a surface on + * @layer, the whole @width x @height surface is read (source 0,0) and shown + * unscaled at 0,@dst_y on the panel: 0 for the full panel, the notch height + * to keep the rows above it black. + */ +static inline void dcp_v14_encode_swap(u8 *swap, u32 id, u32 background, const u8 *surface, + u64 iova, u32 width, u32 height, u32 dst_y, u32 layer) +{ + memset(swap, 0, DCP_V14_SWAP_SIZE); + put_unaligned_le32(id, swap + DCP_V14_SWAP_ID); + put_unaligned_le32(DCP_V14_SWAP_SET_BACKGROUND | DCP_V14_SWAP_LAYERS, + swap + DCP_V14_SWAP_ENABLED); + put_unaligned_le32(DCP_V14_SWAP_SET_BACKGROUND | DCP_V14_SWAP_LAYERS, + swap + DCP_V14_SWAP_COMPLETED); + put_unaligned_le32(background, swap + DCP_V14_SWAP_BACKGROUND); + memset(swap + DCP_V14_SWAP_SURF_NULL(0), 1, DCP_V14_SWAP_SURF_NULL_COUNT); + swap[0x1b56] = 1; + swap[0x1b57] = 1; + if (surface) { + memcpy(swap + DCP_V14_SWAP_SURFACE(layer), surface, DCP_V14_SURFACE_SIZE); + put_unaligned_le64(iova, swap + DCP_V14_SWAP_IOVA(layer)); + put_unaligned_le32(1, swap + DCP_V14_SWAP_SURF_ID(layer)); + dcp_v14_put_rect(swap + DCP_V14_SWAP_SRC(layer), 0, 0, width, height); + dcp_v14_put_rect(swap + DCP_V14_SWAP_DST(layer), 0, dst_y, width, height); + swap[DCP_V14_SWAP_SURF_NULL(layer)] = 0; + } +} + +#endif /* __APPLE_IOMFB_V14_7_SWAP_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_tpl.c b/drivers/gpu/drm/apple/iomfb_v14_7_tpl.c new file mode 100644 index 0000000000000..6ad66e7c84690 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v14_7_tpl.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +/* + * 14.7 firmware through the classic IOMFB template, for the generic M1/M2 + * 14.7.0 firmware (DCP_FIRMWARE_V_14_7_TEMPLATE). Not to be confused with + * iomfb_v14_7.c, the native DCPLink backend for T6030 panels. + */ + +/* Include all per-version headers before defining DCP_FW ourself. */ +#include "iomfb_v12_3.h" +#include "iomfb_v13_3.h" +#include "iomfb_v14_7_tpl.h" +#include "iomfb_h17p.h" +#include "version_utils.h" + +static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { + IOMFB_METHOD("A000", dcpep_late_init_signal), + IOMFB_METHOD("A029", dcpep_setup_video_limits), + IOMFB_METHOD("A131", iomfbep_a131_pmu_service_matched), + IOMFB_METHOD("A132", iomfbep_a132_backlight_service_matched), + IOMFB_METHOD("A377", dcpep_set_create_dfb), + IOMFB_METHOD("A378", iomfbep_a358_vi_set_temperature_hint), + IOMFB_METHOD("A401", dcpep_start_signal), + IOMFB_METHOD("A406", dcpep_swap_start), + IOMFB_METHOD("A407", dcpep_swap_submit), + IOMFB_METHOD("A409", dcpep_set_display_device), + IOMFB_METHOD("A410", dcpep_is_main_display), + IOMFB_METHOD("A411", dcpep_set_digital_out_mode), + IOMFB_METHOD("A421", iomfbep_set_matrix), + IOMFB_METHOD("A425", iomfbep_get_color_remap_mode), + IOMFB_METHOD("A440", dcpep_set_parameter_dcp), + IOMFB_METHOD("A444", dcpep_create_default_fb), + IOMFB_METHOD("A448", dcpep_enable_disable_video_power_savings), + IOMFB_METHOD("A455", dcpep_first_client_open), + IOMFB_METHOD("A456", iomfbep_last_client_close), + IOMFB_METHOD("A463", dcpep_set_display_refresh_properties), + IOMFB_METHOD("A466", dcpep_flush_supports_power), + IOMFB_METHOD("A467", iomfbep_abort_swaps_dcp), + //IOMFB_METHOD("A471", dcpep_update_dfb_surface), + IOMFB_METHOD("A472", dcpep_set_power_state), +}; + +#define DCP_FW v14_7_0 +#define DCP_FW_VER DCP_FW_VERSION(14, 7, 0) + +#include "iomfb_template.c" + +static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { + [0] = trampoline_true, /* did_boot_signal */ + [1] = trampoline_true, /* did_power_on_signal */ + [2] = trampoline_nop, /* will_power_off_signal */ + [3] = trampoline_rt_bandwidth, + // [6] = trampoline_set_frame_sync_props, + [6] = trampoline_nop, + [100] = iomfbep_cb_match_pmu_service, + [101] = trampoline_zero, /* get_display_default_stride */ + [102] = trampoline_nop, /* set_number_property */ + [103] = trampoline_nop, /* trigger_user_cal_loader */ + [104] = trampoline_nop, /* set_boolean_property */ + [107] = trampoline_nop, /* remove_property */ + [108] = trampoline_true, /* create_provider_service */ + [109] = trampoline_true, /* create_product_service */ + [110] = trampoline_true, /* create_pmu_service */ + [111] = trampoline_true, /* create_iomfb_service */ + [112] = trampoline_create_backlight_service, + [113] = trampoline_true, /* create_nvram_service? */ + //[114] = trampoline_get_tiling_state, + //[115] = trampoline_false, /* set_tiling_state */ + [121] = dcpep_cb_boot_1, /* is_dark_boot */ + [122] = trampoline_false, /* is_dark_boot / is_waking_from_hibernate*/ + [123] = trampoline_false, + [125] = trampoline_read_edt_data, + [127] = trampoline_prop_start, + [128] = trampoline_prop_chunk, + [129] = trampoline_prop_end, + // [129] = trampoline_allocate_bandwidth, + [201] = trampoline_map_piodma, + [202] = trampoline_unmap_piodma, + [206] = iomfbep_cb_match_pmu_service_2, + [207] = iomfbep_cb_match_backlight_service, + [208] = trampoline_nop, /* update_backlight_factor_prop */ + [209] = trampoline_get_time, + [300] = trampoline_pr_publish, + [401] = trampoline_get_uint_prop, + [404] = trampoline_nop, /* sr_set_uint_prop */ + [406] = trampoline_set_fx_prop, + [408] = trampoline_get_frequency, + [411] = trampoline_map_reg, + [413] = trampoline_true, /* sr_set_property_dict */ + [414] = trampoline_sr_set_property_int, + [415] = trampoline_true, /* sr_set_property_bool */ + [451] = trampoline_allocate_buffer, + [452] = trampoline_map_physical, + [454] = trampoline_release_mem_desc, + [552] = trampoline_true, /* set_property_dict_0 */ + [561] = trampoline_true, /* set_property_dict */ + [563] = trampoline_true, /* set_property_int */ + [565] = trampoline_true, /* set_property_bool */ + [567] = trampoline_true, /* set_property_str */ + [574] = trampoline_zero, /* power_up_dart */ + [576] = trampoline_hotplug, + [577] = trampoline_nop, /* powerstate_notify */ + //[582] = trampoline_create_dfb_surface, /* create_default_fb_surface */ + [582] = trampoline_true, + [584] = trampoline_nop, /* IOMobileFramebufferAP::clear_default_surface */ + [588] = trampoline_nop, /* resize_default_fb_surface_gated */ + [589] = trampoline_swap_complete, + [591] = trampoline_swap_complete_intent_gated, + [592] = trampoline_abort_swap_ap_gated, + [593] = trampoline_enable_backlight_message_ap_gated, + [594] = trampoline_nop, /* IOMobileFramebufferAP::setSystemConsoleMode */ + [596] = trampoline_false, /* IOMobileFramebufferAP::isDFBAllocated */ + [597] = trampoline_false, /* IOMobileFramebufferAP::preserveContents */ + [598] = trampoline_nop, /* find_swap_function_gated */ +}; +void DCP_FW_NAME(iomfb_start)(struct apple_dcp *dcp) +{ + dcp->cb_handlers = cb_handlers; + + dcp_start_signal(dcp, false, dcp_started, NULL); +} + +#undef DCP_FW_VER +#undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_tpl.h b/drivers/gpu/drm/apple/iomfb_v14_7_tpl.h new file mode 100644 index 0000000000000..cfa0a24dac31b --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v14_7_tpl.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright The Asahi Linux Contributors */ + +/* + * The classic IOMFB (dcpep) protocol of 14.7 firmware, from the shared + * IOMFB template, for the generic M1/M2 14.7.0 firmware + * (DCP_FIRMWARE_V_14_7_TEMPLATE). Not to be confused with iomfb_v14_7.h, + * the native DCPLink session for T6030 processors handed over by the boot + * loader. + */ + +#ifndef __APPLE_IOMFB_V14_7_TPL_H__ +#define __APPLE_IOMFB_V14_7_TPL_H__ + +#include "version_utils.h" + +/* + * dcp-internal.h includes this header, also from inside another template + * instantiation (iomfb_h17p.c): keep that one's DCP_FW. + */ +#pragma push_macro("DCP_FW") +#pragma push_macro("DCP_FW_VER") +#undef DCP_FW +#undef DCP_FW_VER + +#define DCP_FW v14_7_0 +#define DCP_FW_VER DCP_FW_VERSION(14, 7, 0) + +#include "iomfb_template.h" + +#undef DCP_FW_VER +#undef DCP_FW + +#pragma pop_macro("DCP_FW_VER") +#pragma pop_macro("DCP_FW") + +#endif /* __APPLE_IOMFB_V14_7_TPL_H__ */ diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index 22729e810391f..ad2f9d4edf100 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -704,3 +704,32 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, return &plane->base; } + +/* The formats the T6030 (14.x firmware) scanout was qualified with. */ +static const u32 dcp_primary_formats_14_7[] = { + DRM_FORMAT_XRGB8888, + DRM_FORMAT_ARGB8888, +}; + +struct drm_plane *apple_plane_init_v14_7(struct drm_device *dev, + unsigned long possible_crtcs, + u32 iomfb_surf, + enum drm_plane_type type) +{ + struct apple_plane *plane; + + if (type != DRM_PLANE_TYPE_PRIMARY) + return ERR_PTR(-EINVAL); + + plane = drmm_universal_plane_alloc(dev, struct apple_plane, base, possible_crtcs, + &apple_plane_funcs, dcp_primary_formats_14_7, + ARRAY_SIZE(dcp_primary_formats_14_7), + format_modifiers_no_ix, type, NULL); + if (IS_ERR(plane)) + return ERR_CAST(plane); + + drm_plane_helper_add(&plane->base, &apple_primary_plane_helper_funcs); + plane->iomfb_surf = iomfb_surf; + + return &plane->base; +} diff --git a/drivers/gpu/drm/apple/plane.h b/drivers/gpu/drm/apple/plane.h index d07cb52e12c9c..d7905573276f4 100644 --- a/drivers/gpu/drm/apple/plane.h +++ b/drivers/gpu/drm/apple/plane.h @@ -39,5 +39,9 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, bool supports_xrgb2101010, enum drm_plane_type type, bool interchange); +struct drm_plane *apple_plane_init_v14_7(struct drm_device *dev, + unsigned long possible_crtcs, + u32 iomfb_surf, + enum drm_plane_type type); #endif /* __APPLE_PLANE_H__ */ diff --git a/drivers/gpu/drm/apple/version_utils.h b/drivers/gpu/drm/apple/version_utils.h index 5fd801dadc259..c2684bdb121d4 100644 --- a/drivers/gpu/drm/apple/version_utils.h +++ b/drivers/gpu/drm/apple/version_utils.h @@ -24,7 +24,17 @@ enum dcp_firmware_version { DCP_FIRMWARE_UNKNOWN, DCP_FIRMWARE_V_12_3, DCP_FIRMWARE_V_13_5, + /* + * 14.7 firmware on T6030 (apple,t6030-dcp/-dcpext), driven natively + * through the DCPLink session that the boot loader hands over + * (iomfb_v14_7.c). + */ DCP_FIRMWARE_V_14_7, + /* + * Generic 14.7.0 firmware on M1/M2 DCPs, driven through the classic + * IOMFB template (iomfb_v14_7_tpl.c), as on Asahi. + */ + DCP_FIRMWARE_V_14_7_TEMPLATE, DCP_FIRMWARE_H17P, }; From 90e8ec815f6890f49773ace14a6d8160899756b1 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 11:05:42 -0700 Subject: [PATCH 1550/2680] drm/apple: fall back to standard 4K timings for the first external pattern The first external pattern only accepted the 3840x2160 59.9966 Hz reduced-blanking timing reported through TB3 docks. Through a USB4 dock (CalDigit TS4) the same monitor reports standard 59.94 Hz and 60 Hz timings instead, so no timing matched and the pattern failed with -ENODATA. Rank the acceptable 3840x2160 timings: 59.9966 Hz first (unchanged for TB3 docks), then 59.94 Hz, then 60 Hz. 59.94 Hz is preferred over 60 Hz because firmware can resolve 60 Hz on a 4K120 monitor to a DSC timing, which does not produce a picture on this path yet. Among equal ranks the last entry wins, since the monitor's own timings follow the firmware's fixed list. Tested on J516S with a CalDigit TS4: 59.94 Hz is selected and the firmware sets 3840x2160@59 without DSC. The monitor still does not show a picture over the USB4 tunnel; that is tracked separately. Refs iconidentify/m3-roadmap#49 Co-Authored-By: Claude Opus 5.5 (1M context) (cherry picked from commit 575ce54748941d7afddca11648bed25a64450c28) --- drivers/gpu/drm/apple/ibootep.c | 42 +++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/ibootep.c b/drivers/gpu/drm/apple/ibootep.c index b7e746d28cbc5..c1403c52f4f4a 100644 --- a/drivers/gpu/drm/apple/ibootep.c +++ b/drivers/gpu/drm/apple/ibootep.c @@ -136,15 +136,34 @@ static int iboot_parse_modes(const u8 *payload, size_t size, u32 *count) return 0; } +/* + * 3840x2160 timings the first pattern may use, best first: the reduced-blanking + * 59.9966 Hz timing seen through TB3 docks, then the standard 59.94 Hz and 60 Hz + * timings that monitors report over faster USB4 links. 59.94 Hz comes before + * 60 Hz because firmware can resolve "60 Hz" to a DSC timing on 4K120 monitors, + * and DSC output does not yet produce a picture on this path. Returns 0 for + * anything else. + */ +static int iboot_pattern_timing_rank(const u8 *mode) +{ + static const u32 fps[] = { 0x003bff23U, 0x003bf0a8U, 0x003c0000U }; + u32 i; + + if (get_unaligned_le32(mode + 4) != IBOOT_PATTERN_WIDTH || + get_unaligned_le32(mode + 8) != IBOOT_PATTERN_HEIGHT) + return 0; + for (i = 0; i < ARRAY_SIZE(fps); i++) + if (get_unaligned_le32(mode + 12) == fps[i]) + return ARRAY_SIZE(fps) - i; + return 0; +} + static bool iboot_pattern_mode(u32 op, const u8 *mode) { if (get_unaligned_le32(mode) != 1) return false; if (op == IBOOT_GET_TIMING_MODES) - return get_unaligned_le32(mode + 4) == IBOOT_PATTERN_WIDTH && - get_unaligned_le32(mode + 8) == IBOOT_PATTERN_HEIGHT && - /* Observed alternate 4K timing; never fall back to generic CVT. */ - get_unaligned_le32(mode + 12) == 0x003bff23U; + return iboot_pattern_timing_rank(mode) > 0; if (op == IBOOT_GET_COLOR_MODES) return get_unaligned_le32(mode + 4) == 1 && /* BT.601/709 */ get_unaligned_le32(mode + 8) == 1 && /* SDR */ @@ -254,15 +273,24 @@ static int iboot_log_modes(struct iboot_query *query, u32 op, u8 *reply) for (i = 0; i < count; i++) { const u8 *mode = payload + 4 + i * IBOOT_MODE_SIZE; + int rank; + if (!iboot_pattern_mode(op, mode)) continue; if (op == IBOOT_GET_TIMING_MODES) { + /* Best rank wins; among equals the last, since the monitor's own + * timings follow the firmware's fixed list. */ + rank = iboot_pattern_timing_rank(mode); + if (selected != U32_MAX && + rank < iboot_pattern_timing_rank(query->timing)) + continue; memcpy(query->timing, mode, IBOOT_MODE_SIZE); query->timing_valid = true; - } else { - memcpy(query->color, mode, IBOOT_MODE_SIZE); - query->color_valid = true; + selected = i; + continue; } + memcpy(query->color, mode, IBOOT_MODE_SIZE); + query->color_valid = true; selected = i; break; } From b786e48a350cab2dc26906522bd24cc9cbea7af8 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 13:50:25 -0700 Subject: [PATCH 1551/2680] drm/apple: dcpext: ride through one hotplug bounce after the first pattern Some DP-to-HDMI converters (seen with a CalDigit TS4 feeding a 4K120 monitor through a converter flashed for 4K120 on Macs) drop HPD once, about 7 s after the first mode set, and bring it back ~1.5 s later. The source is expected to reconnect and set the mode again; the M1 display path does exactly that. The external path latched -ENOLINK on the first link loss instead, so the monitor went back to standby. On the first link loss after the pattern (before or after desktop registration, which reuses the same retained buffer) wait up to 10 s instead of latching. The disconnect that re-routing the returning tunnel performs is part of the same bounce and keeps waiting. When the link is ready again, re-arm the one-shot iBoot pattern request and send the mode and swap commands again on the retained buffer. A second loss, or no return within the window, latches exactly as before. Tested on J516S with a CalDigit TS4 and the converter: the link drops at +6.6 s, returns at +1.5 s, the mode is set again (3840x2160@59, no DSC) and the external desktop stays up. Refs iconidentify/m3-roadmap#49 Refs iconidentify/m3-roadmap#28 Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01PffktdvF6JDnCYo5jZ6Rtv (cherry picked from commit fd4aecfd4decd2cd435b63fa042c1945e8c5d777) --- drivers/gpu/drm/apple/dcp.c | 1 + drivers/gpu/drm/apple/dcpext_scanout.c | 85 +++++++++++++++++++++++++- drivers/gpu/drm/apple/dcpext_scanout.h | 1 + drivers/gpu/drm/apple/ibootep.c | 13 +++- drivers/gpu/drm/apple/ibootep.h | 2 + 5 files changed, 99 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index f322df62ead64..8afb3522f8b49 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2781,6 +2781,7 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable return -ENOLINK; } smp_store_release(&dcp->external_link_ready, true); + dcpext_scanout_link_restored(dcp); mutex_unlock(&dcp->hpd_mutex); } diff --git a/drivers/gpu/drm/apple/dcpext_scanout.c b/drivers/gpu/drm/apple/dcpext_scanout.c index d0593453cf9a7..5bd89d2b63429 100644 --- a/drivers/gpu/drm/apple/dcpext_scanout.c +++ b/drivers/gpu/drm/apple/dcpext_scanout.c @@ -35,6 +35,16 @@ #define SCANOUT_IOVA_BASE (1ULL << 40) #define SCANOUT_IOVA_SIZE (1ULL << 36) +/* + * Some docks and monitors drop HPD once shortly after the first mode is set and + * bring it back within a second or two. Tolerate exactly one such bounce after + * the first pattern: hold the retained buffer, wait for the link to return, and + * send the pattern commands again. Anything else still latches a terminal error. + */ +#define SCANOUT_BOUNCE_GRACE_MS 10000 +#define SCANOUT_REPATTERN_DELAY_MS 1500 +enum { SCANOUT_BOUNCE_NONE, SCANOUT_BOUNCE_PENDING, SCANOUT_BOUNCE_USED }; + struct dcpext_scanout { struct apple_dcp *dcp; struct device *firmware_dev; @@ -57,6 +67,9 @@ struct dcpext_scanout { struct work_struct invalidate_work; bool pattern_ready; atomic_t requested; + atomic_t bounce; /* SCANOUT_BOUNCE_* */ + struct delayed_work bounce_work; + struct delayed_work repattern_work; bool stopping; bool published; /* Firmware callbacks may retain this object until reboot. */ struct device_attribute attr; @@ -113,8 +126,71 @@ void dcpext_scanout_invalidate(struct apple_dcp *dcp) struct dcpext_scanout *scanout = smp_load_acquire(&dcp->dcpext_scanout); /* Initial unplug remains allowed; only a started scanout owns DMA. */ - if (scanout && atomic_read(&scanout->requested)) - scanout_fail(scanout, -ENOLINK); + if (!scanout || !atomic_read(&scanout->requested)) + return; + /* The desktop reuses the pattern's retained buffer, so the same restore + * covers a bounce before or after desktop registration. */ + if (smp_load_acquire(&scanout->pattern_ready)) { + int prev = atomic_cmpxchg(&scanout->bounce, SCANOUT_BOUNCE_NONE, SCANOUT_BOUNCE_PENDING); + + if (prev == SCANOUT_BOUNCE_NONE) { + dev_info(scanout->firmware_dev, "external link lost after the first pattern; waiting up to %u ms for it to return\n", + SCANOUT_BOUNCE_GRACE_MS); + schedule_delayed_work(&scanout->bounce_work, msecs_to_jiffies(SCANOUT_BOUNCE_GRACE_MS)); + return; + } + /* Re-routing the returning tunnel disconnects the port once more + * before reconnecting it; that is still the same bounce. */ + if (prev == SCANOUT_BOUNCE_PENDING) + return; + } + scanout_fail(scanout, -ENOLINK); +} + +static void dcpext_bounce_timeout(struct work_struct *work) +{ + struct dcpext_scanout *scanout = + container_of(to_delayed_work(work), struct dcpext_scanout, bounce_work); + + if (atomic_cmpxchg(&scanout->bounce, SCANOUT_BOUNCE_PENDING, SCANOUT_BOUNCE_USED) != + SCANOUT_BOUNCE_PENDING) + return; + dev_err(scanout->firmware_dev, "external link did not return within %u ms\n", SCANOUT_BOUNCE_GRACE_MS); + scanout_fail(scanout, -ENOLINK); +} + +static void dcpext_repattern_work(struct work_struct *work) +{ + struct dcpext_scanout *scanout = + container_of(to_delayed_work(work), struct dcpext_scanout, repattern_work); + int ret; + + if (READ_ONCE(scanout->stopping) || atomic_read(&scanout->terminal_error)) + return; + ret = ibootep_rearm_pattern(scanout->dcp); + if (!ret) + ret = ibootep_present_pattern(scanout->dcp, scanout->iova, scanout->size, SCANOUT_STRIDE); + if (ret) { + scanout_fail(scanout, ret); + dev_err(scanout->firmware_dev, "external pattern not restored after the link returned: %d; resources retained until reboot\n", + ret); + return; + } + dev_info(scanout->firmware_dev, "external pattern restored after one hotplug bounce\n"); +} + +/* Caller holds hpd_mutex; the external link has just become ready again. */ +void dcpext_scanout_link_restored(struct apple_dcp *dcp) +{ + struct dcpext_scanout *scanout = smp_load_acquire(&dcp->dcpext_scanout); + + if (!scanout || atomic_cmpxchg(&scanout->bounce, SCANOUT_BOUNCE_PENDING, SCANOUT_BOUNCE_USED) != + SCANOUT_BOUNCE_PENDING) + return; + cancel_delayed_work(&scanout->bounce_work); + dev_info(scanout->firmware_dev, "external link returned; sending the pattern commands again\n"); + queue_delayed_work(system_unbound_wq, &scanout->repattern_work, + msecs_to_jiffies(SCANOUT_REPATTERN_DELAY_MS)); } static bool scanout_link_ready(struct dcpext_scanout *scanout) @@ -527,6 +603,8 @@ static void dcpext_scanout_cleanup(void *data) WRITE_ONCE(scanout->stopping, true); cancel_work_sync(&scanout->desktop_work); cancel_work_sync(&scanout->work); + cancel_delayed_work_sync(&scanout->bounce_work); + cancel_delayed_work_sync(&scanout->repattern_work); cancel_work_sync(&scanout->invalidate_work); if (atomic_read(&scanout->requested) || READ_ONCE(scanout->published)) return; /* Firmware callbacks and possible DMA survive parent cleanup. */ @@ -561,6 +639,9 @@ int dcpext_scanout_register(struct apple_dcp *dcp) atomic_set(&scanout->terminal_error, 0); INIT_WORK(&scanout->invalidate_work, dcpext_invalidate_work); INIT_WORK(&scanout->work, dcpext_pattern_work); + atomic_set(&scanout->bounce, SCANOUT_BOUNCE_NONE); + INIT_DELAYED_WORK(&scanout->bounce_work, dcpext_bounce_timeout); + INIT_DELAYED_WORK(&scanout->repattern_work, dcpext_repattern_work); atomic_set(&scanout->desktop_requested, 0); INIT_WORK(&scanout->desktop_work, dcpext_desktop_work); sysfs_attr_init(&scanout->attr.attr); diff --git a/drivers/gpu/drm/apple/dcpext_scanout.h b/drivers/gpu/drm/apple/dcpext_scanout.h index 37392ccfa5499..626399cf321e4 100644 --- a/drivers/gpu/drm/apple/dcpext_scanout.h +++ b/drivers/gpu/drm/apple/dcpext_scanout.h @@ -11,6 +11,7 @@ int dcpext_scanout_register(struct apple_dcp *dcp); /* Caller holds hpd_mutex when invalidating a physical link. */ void dcpext_scanout_fault(struct apple_dcp *dcp, int error); void dcpext_scanout_invalidate(struct apple_dcp *dcp); +void dcpext_scanout_link_restored(struct apple_dcp *dcp); bool dcpext_scanout_terminal(struct apple_dcp *dcp); bool dcpext_scanout_requested(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/ibootep.c b/drivers/gpu/drm/apple/ibootep.c index c1403c52f4f4a..272d3de78a987 100644 --- a/drivers/gpu/drm/apple/ibootep.c +++ b/drivers/gpu/drm/apple/ibootep.c @@ -272,7 +272,6 @@ static int iboot_log_modes(struct iboot_query *query, u32 op, u8 *reply) return ret; for (i = 0; i < count; i++) { const u8 *mode = payload + 4 + i * IBOOT_MODE_SIZE; - int rank; if (!iboot_pattern_mode(op, mode)) @@ -445,6 +444,18 @@ static int iboot_pattern_command(struct iboot_query *query, u32 op, return ret; } +int ibootep_rearm_pattern(struct apple_dcp *dcp) +{ + struct iboot_query *query = iboot_find_query(dcp); + + if (IS_ERR(query)) + return PTR_ERR(query); + if (atomic_read(&query->busy)) + return -EBUSY; + WRITE_ONCE(query->pattern_requested, false); + return 0; +} + int ibootep_present_pattern(struct apple_dcp *dcp, u64 iova, size_t size, u32 stride) { struct iboot_query *query; diff --git a/drivers/gpu/drm/apple/ibootep.h b/drivers/gpu/drm/apple/ibootep.h index f742333e15625..3d9a76e5a45bc 100644 --- a/drivers/gpu/drm/apple/ibootep.h +++ b/drivers/gpu/drm/apple/ibootep.h @@ -18,5 +18,7 @@ int ibootep_query_modes(struct apple_dcp *dcp); * including after an error: commands may already have reached firmware. */ int ibootep_present_pattern(struct apple_dcp *dcp, u64 iova, size_t size, u32 stride); +/* Allow one more pattern request, for the retained buffer after a hotplug bounce. */ +int ibootep_rearm_pattern(struct apple_dcp *dcp); #endif From 65abe9701c865edf52384f5f2e05fb04bcd7ec8a Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 12:32:40 -0700 Subject: [PATCH 1552/2680] drm/apple: add opt-in firmware-completed external buffer swaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copy complete opaque frames into a second retained buffer and wait for the exact submitted iBoot swap before reusing the former front buffer. Validate actual AFK reply lengths before reading swap IDs. Bound failures, cancel failed completion events, and retain both DMA buffers after uncertain completion or link loss. Keep the path opt-in and firmware-gated pending the J516S hardware check. Add host sanitizer coverage for transport bounds and delayed ACK cleanup, packet layouts, transaction failures, buffer ownership and full 4K conversion. CPU copies and software completion timestamps remain. [ryan: adapted to 7.2.10 — Ported only this commit's own delta (nothing from the held 0aa391057881). AFK: our afk_send_command_timeout_len() body becomes afk_send_command_common() with an extra legacy-EPIC reply_len output (AFKv2 endpoints return -EOPNOTSUPP there; our received/reply_length handling is unchanged); afk_send_command_with_reply_len() and afk_service_call_with_reply_len() are implemented on it, and our afk_service_call*/afk_send_command* paths keep their behaviour. dcpext_scanout_link_restored() keeps the one-bounce cmpxchg and also returns for page-flip mode. dcpext_drm.c adapted to drm_atomic_commit. Dropped tools/testing/selftests/drm/apple/test-afk-replies.py (and its README line): it compiles the upstream afk.c function shapes against host stubs, which our AFKv2 afk.c does not have; the other three host tests pass against this tree.] Refs: iconidentify/m3-roadmap#61 (cherry picked from commit 29095d9d22dda109ae9bbc717c9ab60065791948) --- drivers/gpu/drm/apple/afk.c | 138 +++++++++- drivers/gpu/drm/apple/afk.h | 11 + drivers/gpu/drm/apple/afk_reply.h | 28 +++ drivers/gpu/drm/apple/dcpext_drm.c | 130 +++++++++- drivers/gpu/drm/apple/dcpext_scanout.c | 123 ++++++++- drivers/gpu/drm/apple/dcpext_scanout.h | 12 + drivers/gpu/drm/apple/ibootep.c | 108 ++++++++ drivers/gpu/drm/apple/ibootep.h | 6 + tools/testing/selftests/drm/apple/README | 28 +++ .../selftests/drm/apple/test-dcpext-copy.py | 95 +++++++ .../drm/apple/test-dcpext-present.py | 238 ++++++++++++++++++ .../selftests/drm/apple/test-iboot-wire.py | 171 +++++++++++++ 12 files changed, 1066 insertions(+), 22 deletions(-) create mode 100644 drivers/gpu/drm/apple/afk_reply.h create mode 100644 tools/testing/selftests/drm/apple/README create mode 100644 tools/testing/selftests/drm/apple/test-dcpext-copy.py create mode 100644 tools/testing/selftests/drm/apple/test-dcpext-present.py create mode 100644 tools/testing/selftests/drm/apple/test-iboot-wire.py diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index c57520bb68421..af19176f97c79 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -17,6 +18,7 @@ #include "afk.h" #include "afkv2-codec.h" +#include "afk_reply.h" #include "trace.h" struct afk_receive_message_work { @@ -676,6 +678,8 @@ static void afk_recv_handle_reply(struct apple_dcp_afkep *ep, u32 channel, service->cmds[idx].done = true; service->cmds[idx].retcode = le32_to_cpu(cmd->retcode); + service->cmds[idx].reply_len = le32_to_cpu(cmd->rxlen); + service->cmds[idx].reply_len_valid = true; if (service->cmds[idx].free_on_ack) { /* defer freeing until we're no longer in atomic context */ rxbuf = service->cmds[idx].rxbuf; @@ -1705,10 +1709,21 @@ static u16 afk_command_tag(struct apple_epic_service *service, u8 slot) return tag; } +static int afk_send_command_common(struct apple_epic_service *service, u8 type, + const void *payload, size_t payload_len, + void *output, size_t output_len, u32 *retcode, + unsigned int timeout_ms, size_t *received, + size_t *reply_len); + static int afk_send_command_timeout_len(struct apple_epic_service *service, u8 type, const void *payload, size_t payload_len, void *output, size_t output_len, u32 *retcode, - unsigned int timeout_ms, size_t *received); + unsigned int timeout_ms, size_t *received) +{ + return afk_send_command_common(service, type, payload, payload_len, + output, output_len, retcode, timeout_ms, + received, NULL); +} int afk_send_command_timeout(struct apple_epic_service *service, u8 type, const void *payload, size_t payload_len, @@ -1719,10 +1734,15 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, output, output_len, retcode, timeout_ms, NULL); } -static int afk_send_command_timeout_len(struct apple_epic_service *service, u8 type, - const void *payload, size_t payload_len, - void *output, size_t output_len, u32 *retcode, - unsigned int timeout_ms, size_t *received) +/* + * @reply_len (legacy EPIC only): return just the bytes the firmware reported + * in its reply instead of the whole allocated output. + */ +static int afk_send_command_common(struct apple_epic_service *service, u8 type, + const void *payload, size_t payload_len, + void *output, size_t output_len, u32 *retcode, + unsigned int timeout_ms, size_t *received, + size_t *reply_len) { struct epic_cmd cmd; void *inline_command = NULL; @@ -1737,10 +1757,20 @@ static int afk_send_command_timeout_len(struct apple_epic_service *service, u8 t if (received) *received = 0; + if (reply_len) { + *reply_len = 0; + if (payload_len > U32_MAX || output_len > U32_MAX) + return -EOVERFLOW; + } service = afk_service_get(service); if (!service) return -ENODEV; ep = service->ep; + /* AFKv2 replies report their length through reply_length instead. */ + if (reply_len && ep->wire_v2) { + ret = -EOPNOTSUPP; + goto err_put_service; + } if (ep->wire_v2 && (!service->channel || service->channel >= AFKV2_INTERFACES || !ep->command_tags)) { ret = -ENODEV; @@ -1806,6 +1836,8 @@ static int afk_send_command_timeout_len(struct apple_epic_service *service, u8 t service->cmds[idx].tag = tag; service->cmds[idx].type = type; service->cmds[idx].reply_length = 0; + service->cmds[idx].reply_len = 0; + service->cmds[idx].reply_len_valid = false; service->cmds[idx].rxbuf = rxbuf; service->cmds[idx].txbuf = txbuf; service->cmds[idx].rxbuf_dma = rxbuf_dma; @@ -1878,6 +1910,19 @@ static int afk_send_command_timeout_len(struct apple_epic_service *service, u8 t ret = 0; if (retcode) *retcode = service->cmds[idx].retcode; + if (reply_len) { + ret = afk_reply_size_valid(service->cmds[idx].reply_len_valid, + service->cmds[idx].reply_len, output_len, 0); + if (!ret) { + *reply_len = service->cmds[idx].reply_len; + dma_rmb(); + if (output && output_len) { + memset(output, 0, output_len); + memcpy(output, rxbuf, *reply_len); + } + } + goto err_free_cmd; + } if (ep->wire_v2 && received) copy_len = service->cmds[idx].reply_length; if (ep->wire_v2 && !service->cmds[idx].retcode && @@ -2030,6 +2075,89 @@ int afk_service_call_timeout_len(struct apple_epic_service *service, u16 group, return ret; } +int afk_send_command_with_reply_len(struct apple_epic_service *service, u8 type, + const void *payload, size_t payload_len, void *output, + size_t output_len, u32 *retcode, size_t *reply_len) +{ + if (!reply_len) + return -EINVAL; + return afk_send_command_common(service, type, payload, payload_len, + output, output_len, retcode, MSEC_PER_SEC, + NULL, reply_len); +} + +int afk_service_call_with_reply_len(struct apple_epic_service *service, + u16 group, u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, size_t *reply_len) +{ + struct epic_service_call *call; + void *bfr; + size_t bfr_len, input_size, output_size, received = 0; + int ret; + u32 retcode; + u32 retlen; + + if (!reply_len) + return -EINVAL; + *reply_len = 0; + if (!service) + return -ENODEV; + if (check_add_overflow(data_len, data_pad, &input_size) || + check_add_overflow(output_len, output_pad, &output_size) || + check_add_overflow(max(input_size, output_size), sizeof(*call), + &bfr_len) || bfr_len > U32_MAX) + return -EOVERFLOW; + + bfr = kzalloc(bfr_len, GFP_KERNEL); + if (!bfr) + return -ENOMEM; + + call = bfr; + + memset(call, 0, sizeof(*call)); + call->group = cpu_to_le16(group); + call->command = cpu_to_le32(command); + call->data_len = cpu_to_le32(data_len + data_pad); + call->magic = cpu_to_le32(EPIC_SERVICE_CALL_MAGIC); + + memcpy(bfr + sizeof(*call), data, data_len); + + ret = afk_send_command_common(service, EPIC_SUBTYPE_STD_SERVICE, bfr, + bfr_len, bfr, bfr_len, &retcode, + MSEC_PER_SEC, NULL, &received); + if (ret) + goto out; + if (retcode) { + ret = -EINVAL; + goto out; + } + ret = afk_reply_size_valid(true, received, bfr_len, sizeof(*call)); + if (ret) + goto out; + if (le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || + le16_to_cpu(call->group) != group || + le32_to_cpu(call->command) != command) { + ret = -EINVAL; + goto out; + } + + retlen = le32_to_cpu(call->data_len); + ret = afk_service_body_size_valid(received, sizeof(*call), retlen, + output_len); + if (ret) + goto out; + *reply_len = retlen; + if (output && output_len) { + memset(output, 0, output_len); + memcpy(output, bfr + sizeof(*call), retlen); + } + +out: + kfree(bfr); + return ret; +} + #if IS_ENABLED(CONFIG_DRM_APPLE_DEBUG) #define AFK_DEBUGFS_MAX_REPLY 8192 diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index b548c0bbfbcbc..716b9fe595c64 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -37,6 +37,9 @@ struct epic_cmd_info { dma_addr_t txbuf_dma; size_t rxlen; size_t txlen; + /* DMA allocation lengths above stay immutable until buffers are freed. */ + size_t reply_len; + bool reply_len_valid; u32 retcode; bool done; @@ -487,4 +490,12 @@ int afk_service_call_timeout_len(struct apple_epic_service *service, u16 group, size_t output_pad, unsigned int timeout_ms, size_t *received); +/* Strict variants return only firmware-reported bytes; legacy callers unchanged. */ +int afk_send_command_with_reply_len(struct apple_epic_service *service, u8 type, + const void *payload, size_t payload_len, void *output, + size_t output_len, u32 *retcode, size_t *reply_len); +int afk_service_call_with_reply_len(struct apple_epic_service *service, + u16 group, u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, size_t *reply_len); #endif diff --git a/drivers/gpu/drm/apple/afk_reply.h b/drivers/gpu/drm/apple/afk_reply.h new file mode 100644 index 0000000000000..4f6d1c30e1a96 --- /dev/null +++ b/drivers/gpu/drm/apple/afk_reply.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef _DRM_APPLE_AFK_REPLY_H +#define _DRM_APPLE_AFK_REPLY_H + +#include +#include + +/* Allocation capacity must never be replaced with firmware's returned length. */ +static inline int afk_reply_size_valid(bool valid, size_t received, + size_t capacity, size_t minimum) +{ + if (!valid || received < minimum) + return -EPROTO; + if (received > capacity) + return -EMSGSIZE; + return 0; +} + +static inline int afk_service_body_size_valid(size_t received, size_t header, + size_t claimed, size_t capacity) +{ + if (received < header || claimed > received - header) + return -EPROTO; + if (claimed > capacity) + return -EMSGSIZE; + return 0; +} +#endif diff --git a/drivers/gpu/drm/apple/dcpext_drm.c b/drivers/gpu/drm/apple/dcpext_drm.c index 08748f79a6b01..27a794881a6da 100644 --- a/drivers/gpu/drm/apple/dcpext_drm.c +++ b/drivers/gpu/drm/apple/dcpext_drm.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Fixed iBoot scanout, with CPU shadow updates. No firmware calls per commit. */ +/* Fixed iBoot mode, CPU shadow updates, optionally synchronized buffer swaps. */ #include +#include #include #include #include @@ -14,9 +15,11 @@ #include #include #include +#include #include "dcp.h" #include "dcpext_mode.h" #include "dcpext_drm.h" +#include "dcpext_scanout.h" #define EXT_WIDTH DCPEXT_WIDTH #define EXT_HEIGHT DCPEXT_HEIGHT @@ -31,6 +34,9 @@ struct dcpext_drm { void *pixels; size_t size; u32 stride; + struct apple_dcp *dcp; + bool pageflips; + u8 *row; /* Owned by the retained scanout, never by the detachable parent device. */ atomic_t *terminal_error; }; @@ -83,6 +89,66 @@ static int ext_plane_check(struct drm_plane *plane, struct drm_atomic_commit *st return 0; } +/* A failed copy/firmware transaction must not become a successful fake flip. */ +static void ext_cancel_flip(struct dcpext_drm *ext, struct drm_atomic_commit *state, int error) +{ + struct drm_crtc_state *cs = drm_atomic_get_new_crtc_state(state, &ext->crtc); + struct drm_pending_vblank_event *event; + unsigned long flags; + + if (!cs) + return; + spin_lock_irqsave(&ext->drm.event_lock, flags); + event = cs->event; + cs->event = NULL; + spin_unlock_irqrestore(&ext->drm.event_lock, flags); + if (!event) + return; + if (event->base.fence) { + dma_fence_set_error(event->base.fence, error); + dma_fence_signal(event->base.fence); + } + if (event->base.completion) { + complete_all(event->base.completion); + if (event->base.completion_release) + event->base.completion_release(event->base.completion); + event->base.completion = NULL; + } + drm_event_cancel_free(&ext->drm, &event->base); +} + +static int ext_copy_frame(struct dcpext_drm *ext, struct drm_plane_state *ps) +{ + struct drm_shadow_plane_state *shadow = to_drm_shadow_plane_state(ps); + struct drm_framebuffer *fb = ps->fb; + struct dcpext_frame frame; + int x, y, ret; + + ret = dcpext_scanout_begin_frame(ext->dcp, &frame); + if (ret) + return ret; + if (!ext_rect_valid(0, 0, EXT_WIDTH, EXT_HEIGHT, ext->stride, frame.size)) { + dcpext_scanout_end_frame(ext->dcp, false); + return -EINVAL; + } + /* The idle buffer contains an older frame, so damage alone is insufficient. + * Copy the complete current framebuffer before publishing its address. + */ + for (y = 0; y < EXT_HEIGHT; y++) { + u8 *dst = frame.pixels + (size_t)y * ext->stride; + + iosys_map_memcpy_from(ext->row, &shadow->data[0], + (size_t)y * fb->pitches[0], EXT_STRIDE); + for (x = 0; x < EXT_WIDTH; x++) + ext->row[x * 4 + 3] = 0xff; + /* Convert in cached memory, then stream to coherent scanout memory. + * Byte stores directly to that mapping make full 4K copies costly. + */ + memcpy(dst, ext->row, EXT_STRIDE); + } + return dcpext_scanout_end_frame(ext->dcp, true); +} + static void ext_plane_update(struct drm_plane *plane, struct drm_atomic_commit *state) { struct dcpext_drm *ext = container_of(plane->dev, struct dcpext_drm, drm); @@ -93,15 +159,25 @@ static void ext_plane_update(struct drm_plane *plane, struct drm_atomic_commit * struct drm_gem_object *obj; struct drm_atomic_helper_damage_iter iter; struct drm_rect damage, bounds = DRM_RECT_INIT(0, 0, EXT_WIDTH, EXT_HEIGHT); - int idx, x, y; + int idx, x, y, ret; - if (!ps->visible || !fb || atomic_read(ext->terminal_error)) + if (!ps->visible || !fb) return; + ret = atomic_read(ext->terminal_error); + if (ret) + goto cancel; obj = drm_gem_fb_get_obj(fb, 0); - if (drm_gem_fb_begin_cpu_access(fb, DMA_FROM_DEVICE)) - return; - if (!drm_dev_enter(plane->dev, &idx)) + ret = drm_gem_fb_begin_cpu_access(fb, DMA_FROM_DEVICE); + if (ret) + goto cancel; + if (!drm_dev_enter(plane->dev, &idx)) { + ret = -ENODEV; goto end_access; + } + if (ext->pageflips) { + ret = ext_copy_frame(ext, ps); + goto exit_device; + } drm_atomic_helper_damage_iter_init(&iter, old, ps); drm_atomic_for_each_plane_damage(&iter, &damage) { if (!drm_rect_intersect(&damage, &bounds)) @@ -124,22 +200,45 @@ static void ext_plane_update(struct drm_plane *plane, struct drm_atomic_commit * } } dma_wmb(); +exit_device: drm_dev_exit(idx); end_access: drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE); +cancel: + if (ext->pageflips && ret) + ext_cancel_flip(ext, state, ret); } static void ext_plane_disable(struct drm_plane *plane, struct drm_atomic_commit *state) { struct dcpext_drm *ext = container_of(plane->dev, struct dcpext_drm, drm); u32 *dst = ext->pixels; - int idx, i; + struct dcpext_frame frame; + int idx, i, ret; - if (atomic_read(ext->terminal_error) || !drm_dev_enter(plane->dev, &idx)) + ret = atomic_read(ext->terminal_error); + if (ret || !drm_dev_enter(plane->dev, &idx)) { + if (ext->pageflips) + ext_cancel_flip(ext, state, ret ? ret : -ENODEV); return; + } + if (ext->pageflips) { + ret = dcpext_scanout_begin_frame(ext->dcp, &frame); + if (ret) { + ext_cancel_flip(ext, state, ret); + goto exit_device; + } + dst = frame.pixels; + } for (i = 0; i < EXT_WIDTH * EXT_HEIGHT; ++i) dst[i] = cpu_to_le32(0xff000000); dma_wmb(); + if (ext->pageflips) { + ret = dcpext_scanout_end_frame(ext->dcp, true); + if (ret) + ext_cancel_flip(ext, state, ret); + } +exit_device: drm_dev_exit(idx); } @@ -247,6 +346,13 @@ int dcpext_drm_register(struct apple_dcp *dcp, void *pixels, size_t size, u32 st ext->size = size; ext->stride = stride; ext->terminal_error = terminal_error; + ext->dcp = dcp; + ext->pageflips = dcpext_scanout_pageflips(dcp); + if (ext->pageflips) { + ext->row = drmm_kmalloc(drm, EXT_STRIDE, GFP_KERNEL); + if (!ext->row) + return -ENOMEM; + } ret = drmm_mode_config_init(drm); if (ret) return ret; @@ -281,8 +387,9 @@ int dcpext_drm_register(struct apple_dcp *dcp, void *pixels, size_t size, u32 st if (ret) return ret; drm_mode_config_reset(drm); - /* No vblank IRQ: the atomic helper emits completion events after copies. - * These events do not mean a physical vblank; tearing remains possible. + /* No vblank IRQ: the helper timestamps completion in software. In pageflip + * mode the copy/commit also waits for firmware completion before this event; + * the legacy single-buffer mode remains unsynchronized and may tear. * Deliberately no fbdev/client setup, console takeover, or render node. */ ret = drm_dev_register(drm, 0); @@ -294,6 +401,7 @@ int dcpext_drm_register(struct apple_dcp *dcp, void *pixels, size_t size, u32 st /* Keep notification work safe even after parent devres teardown. */ drm_dev_get(drm); WRITE_ONCE(*retained_drm, drm); - dev_info(dcp->dev, "external shadow DRM registered: fixed 3840x2160, CPU copies, no hardware vblank\n"); + dev_info(dcp->dev, "external shadow DRM registered: fixed 3840x2160, CPU copies, %s\n", + ext->pageflips ? "firmware-completed double-buffer swaps" : "no hardware vblank"); return 0; } diff --git a/drivers/gpu/drm/apple/dcpext_scanout.c b/drivers/gpu/drm/apple/dcpext_scanout.c index 5bd89d2b63429..17e74998d321e 100644 --- a/drivers/gpu/drm/apple/dcpext_scanout.c +++ b/drivers/gpu/drm/apple/dcpext_scanout.c @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include #include #include @@ -35,6 +37,10 @@ #define SCANOUT_IOVA_BASE (1ULL << 40) #define SCANOUT_IOVA_SIZE (1ULL << 36) +static bool dcpext_pageflip; +module_param(dcpext_pageflip, bool, 0444); +MODULE_PARM_DESC(dcpext_pageflip, "Use two retained external buffers and wait for firmware swap completion"); + /* * Some docks and monitors drop HPD once shortly after the first mode is set and * bring it back within a second or two. Tolerate exactly one such bounce after @@ -56,6 +62,12 @@ struct dcpext_scanout { void *pixels; dma_addr_t iova; size_t size; + size_t frame_size; + bool pageflips; + unsigned int front; + struct mutex present_lock; + u64 frames_completed; + u64 present_ns_max; size_t mapped; struct sg_table sgt; struct work_struct work; @@ -75,7 +87,8 @@ struct dcpext_scanout { struct device_attribute attr; struct device_attribute desktop_attr; struct device_attribute status_attr; - struct attribute *attrs[4]; + struct device_attribute present_attr; + struct attribute *attrs[5]; struct attribute_group attr_group; }; @@ -93,7 +106,7 @@ static void scanout_fail(struct dcpext_scanout *scanout, int error) { if (READ_ONCE(scanout->stopping)) return; - if (error >= 0) + if (error >= 0 || (scanout->pageflips && error == -ENOLINK)) error = -EIO; if (!atomic_cmpxchg(&scanout->terminal_error, 0, error)) schedule_work(&scanout->invalidate_work); @@ -128,6 +141,13 @@ void dcpext_scanout_invalidate(struct apple_dcp *dcp) /* Initial unplug remains allowed; only a started scanout owns DMA. */ if (!scanout || !atomic_read(&scanout->requested)) return; + /* Link loss can race a submitted swap. Until recovery also accounts for + * its completion, keep both buffers and require a fresh boot in this mode. + */ + if (scanout->pageflips) { + scanout_fail(scanout, -EIO); + return; + } /* The desktop reuses the pattern's retained buffer, so the same restore * covers a bounce before or after desktop registration. */ if (smp_load_acquire(&scanout->pattern_ready)) { @@ -184,7 +204,8 @@ void dcpext_scanout_link_restored(struct apple_dcp *dcp) { struct dcpext_scanout *scanout = smp_load_acquire(&dcp->dcpext_scanout); - if (!scanout || atomic_cmpxchg(&scanout->bounce, SCANOUT_BOUNCE_PENDING, SCANOUT_BOUNCE_USED) != + if (!scanout || scanout->pageflips || + atomic_cmpxchg(&scanout->bounce, SCANOUT_BOUNCE_PENDING, SCANOUT_BOUNCE_USED) != SCANOUT_BOUNCE_PENDING) return; cancel_delayed_work(&scanout->bounce_work); @@ -205,6 +226,77 @@ static bool scanout_link_ready(struct dcpext_scanout *scanout) READ_ONCE(dcp->dptxport[0].enabled); } +bool dcpext_scanout_pageflips(struct apple_dcp *dcp) +{ + /* Pair with publication of the initialized scanout in register(). */ + struct dcpext_scanout *scanout = smp_load_acquire(&dcp->dcpext_scanout); + + return scanout && scanout->pageflips; +} + +int dcpext_scanout_begin_frame(struct apple_dcp *dcp, struct dcpext_frame *frame) +{ + /* Pair with register(); drm_dev_enter() protects the parent lifetime. */ + struct dcpext_scanout *scanout = smp_load_acquire(&dcp->dcpext_scanout); + + if (!scanout || !scanout->pageflips) + return -EOPNOTSUPP; + mutex_lock(&scanout->present_lock); + /* Observe the buffers/mappings published before pattern_ready. */ + if (READ_ONCE(scanout->stopping) || !scanout_link_ready(scanout) || + !smp_load_acquire(&scanout->pattern_ready)) { + mutex_unlock(&scanout->present_lock); + return -ENOLINK; + } + frame->pixels = scanout->pixels + (scanout->front ^ 1) * scanout->frame_size; + frame->size = scanout->frame_size; + return 0; +} + +int dcpext_scanout_end_frame(struct apple_dcp *dcp, bool present) +{ + /* Same publication/lifetime protection as begin_frame(). */ + struct dcpext_scanout *scanout = smp_load_acquire(&dcp->dcpext_scanout); + unsigned int next = scanout->front ^ 1; + u64 start; + int ret = 0; + + lockdep_assert_held(&scanout->present_lock); + if (!present) + goto out; + if (READ_ONCE(scanout->stopping) || !scanout_link_ready(scanout)) { + ret = -ENOLINK; + goto out; + } + dma_wmb(); + start = ktime_get_ns(); + ret = ibootep_present_frame(dcp, scanout->iova + next * scanout->frame_size, + scanout->frame_size, SCANOUT_STRIDE); + if (ret) { + /* Even -ENOLINK from a presentation command is uncertain DMA state; + * use a terminal error that the hotplug restore path cannot clear. + */ + scanout_fail(scanout, -EIO); + goto out; + } + scanout->front = next; + scanout->frames_completed++; + scanout->present_ns_max = max(scanout->present_ns_max, ktime_get_ns() - start); +out: + mutex_unlock(&scanout->present_lock); + return ret; +} + +static ssize_t dcpext_present_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct dcpext_scanout *scanout = container_of(attr, struct dcpext_scanout, present_attr); + + return sysfs_emit(buf, "pageflips=%u buffers=%u completed=%llu front=%u wait_max_us=%llu\n", + scanout->pageflips, scanout->pageflips ? 2 : 1, + READ_ONCE(scanout->frames_completed), READ_ONCE(scanout->front), + READ_ONCE(scanout->present_ns_max) / NSEC_PER_USEC); +} + static ssize_t dcpext_status_show(struct device *dev, struct device_attribute *attr, char *buf) { struct dcpext_scanout *scanout = container_of(attr, struct dcpext_scanout, status_attr); @@ -484,7 +576,8 @@ static void dcpext_pattern_work(struct work_struct *work) ret = scanout_create_device(scanout); if (ret) goto fail; - scanout->size = ALIGN((size_t)SCANOUT_STRIDE * SCANOUT_HEIGHT, SZ_16K); + scanout->frame_size = ALIGN((size_t)SCANOUT_STRIDE * SCANOUT_HEIGHT, SZ_16K); + scanout->size = scanout->frame_size * (scanout->pageflips ? 2 : 1); step = "coherent framebuffer allocation"; scanout->pixels = dma_alloc_coherent(dev, scanout->size, &scanout->iova, GFP_KERNEL); if (!scanout->pixels) { @@ -512,7 +605,13 @@ static void dcpext_pattern_work(struct work_struct *work) ret = -ENOLINK; goto fail; } - ret = ibootep_present_pattern(scanout->dcp, scanout->iova, scanout->size, SCANOUT_STRIDE); + ret = ibootep_present_pattern(scanout->dcp, scanout->iova, + scanout->frame_size, SCANOUT_STRIDE); + if (!ret && scanout->pageflips) { + step = "initial firmware swap completion"; + ret = ibootep_present_frame(scanout->dcp, scanout->iova, + scanout->frame_size, SCANOUT_STRIDE); + } if (ret) goto fail; if (atomic_read(&scanout->terminal_error)) { @@ -565,7 +664,7 @@ static void dcpext_desktop_work(struct work_struct *work) if (READ_ONCE(scanout->stopping) || atomic_read(&scanout->terminal_error) || !smp_load_acquire(&scanout->pattern_ready)) return; - ret = dcpext_drm_register(scanout->dcp, scanout->pixels, scanout->size, + ret = dcpext_drm_register(scanout->dcp, scanout->pixels, scanout->frame_size, SCANOUT_STRIDE, &scanout->terminal_error, &scanout->retained_drm); if (ret) { scanout_fail(scanout, ret); @@ -618,10 +717,15 @@ int dcpext_scanout_register(struct apple_dcp *dcp) { struct device_node *node; struct dcpext_scanout *scanout; + const char *uuid; int ret; if (!dcp->external || dcp->fw_compat != DCP_FIRMWARE_V_14_7) return -ENODEV; + if (dcpext_pageflip && + (of_property_read_string(dcp->dev->of_node, "apple,firmware-uuid", &uuid) || + strcmp(uuid, "DDF38191-93B3-324A-BC8F-643006F5AC82"))) + return -EOPNOTSUPP; node = of_get_child_by_name(dcp->dev->of_node, "scanout"); if (!scanout_node_verified(node)) { of_node_put(node); @@ -635,6 +739,8 @@ int dcpext_scanout_register(struct apple_dcp *dcp) scanout->dcp = dcp; scanout->firmware_dev = get_device(dcp->dev); scanout->node = node; + scanout->pageflips = dcpext_pageflip; + mutex_init(&scanout->present_lock); atomic_set(&scanout->requested, 0); atomic_set(&scanout->terminal_error, 0); INIT_WORK(&scanout->invalidate_work, dcpext_invalidate_work); @@ -659,6 +765,11 @@ int dcpext_scanout_register(struct apple_dcp *dcp) scanout->status_attr.attr.mode = 0444; scanout->status_attr.show = dcpext_status_show; scanout->attrs[2] = &scanout->status_attr.attr; + sysfs_attr_init(&scanout->present_attr.attr); + scanout->present_attr.attr.name = "dcpext_present"; + scanout->present_attr.attr.mode = 0444; + scanout->present_attr.show = dcpext_present_show; + scanout->attrs[3] = &scanout->present_attr.attr; scanout->attr_group.attrs = scanout->attrs; ret = devm_add_action_or_reset(dcp->dev, dcpext_scanout_cleanup, scanout); if (ret) diff --git a/drivers/gpu/drm/apple/dcpext_scanout.h b/drivers/gpu/drm/apple/dcpext_scanout.h index 626399cf321e4..aa195170f8e28 100644 --- a/drivers/gpu/drm/apple/dcpext_scanout.h +++ b/drivers/gpu/drm/apple/dcpext_scanout.h @@ -5,6 +5,18 @@ #include struct apple_dcp; +struct dcpext_frame { + void *pixels; + size_t size; +}; + +bool dcpext_scanout_pageflips(struct apple_dcp *dcp); +/* begin holds the presentation mutex until end, including on discard. + * Callers must hold drm_dev_enter() across the pair to exclude removal. + */ +int dcpext_scanout_begin_frame(struct apple_dcp *dcp, struct dcpext_frame *frame); +int dcpext_scanout_end_frame(struct apple_dcp *dcp, bool present); + /* Register the explicit one-shot diagnostic only after the iBoot service exists. */ int dcpext_scanout_register(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/ibootep.c b/drivers/gpu/drm/apple/ibootep.c index 272d3de78a987..f473258e778e1 100644 --- a/drivers/gpu/drm/apple/ibootep.c +++ b/drivers/gpu/drm/apple/ibootep.c @@ -72,6 +72,8 @@ struct iboot_query { atomic_t busy; bool stopping; bool pattern_requested; + bool presentation_failed; + u32 last_frame_swap; bool timing_valid; bool color_valid; /* Preserve the exact queried records, including firmware-owned padding. */ @@ -212,6 +214,9 @@ static int iboot_build_pattern_request(u32 op, const void *data, size_t data_siz case 15: expected = 0; break; /* SwapBegin */ case 16: expected = sizeof(struct iboot_swap_layer_v13_3); break; case 18: expected = 12; break; /* SwapEnd */ + case 19: /* SwapWait */ + expected = 16; + break; default: return -EINVAL; } if (data_size != expected || capacity < 16 + expected || @@ -444,6 +449,109 @@ static int iboot_pattern_command(struct iboot_query *query, u32 op, return ret; } +/* Unlike startup diagnostics, presentation needs the actual received length + * before trusting the swap id. Mutation operations may have no DMA body; + * their outer EPIC return code carries the operation's success or failure. + */ +static int iboot_frame_command(struct iboot_query *query, u32 op, + const void *data, size_t data_size, u8 *reply, + size_t *received) +{ + u8 request[16 + sizeof(struct iboot_swap_layer_v13_3)]; + size_t size; + u32 retcode = 0; + int ret; + + if (READ_ONCE(query->stopping) || READ_ONCE(query->service->torndown)) + return -ENODEV; + ret = iboot_build_pattern_request(op, data, data_size, request, + sizeof(request), &size); + if (ret) + return ret; + ret = afk_send_command_with_reply_len(query->service, EPIC_SUBTYPE_STD_SERVICE, + request, size, reply, IBOOT_QUERY_RX_SIZE, &retcode, received); + if (ret) + return ret; + return retcode ? -EIO : 0; +} + +int ibootep_present_frame(struct apple_dcp *dcp, u64 iova, size_t size, u32 stride) +{ + struct iboot_swap_layer_v13_3 layer; + struct iboot_query *query; + __le32 end[3] = {}, wait[4] = {}; + const u8 *payload; + size_t received, payload_size; + u32 id = 0, step = 15; + u8 *reply; + int ret; + + ret = iboot_pattern_params(iova, size, stride); + if (ret) + return ret; + if (dcp->fw_compat != DCP_FIRMWARE_V_14_7) + return -EOPNOTSUPP; + query = iboot_find_query(dcp); + if (IS_ERR(query)) + return PTR_ERR(query); + might_sleep(); + if (atomic_cmpxchg(&query->busy, 0, 1)) + return -EBUSY; + if (!query->pattern_requested || query->presentation_failed) { + ret = -EIO; + goto done; + } + reply = kzalloc(IBOOT_QUERY_RX_SIZE, GFP_KERNEL); + if (!reply) { + ret = -ENOMEM; + goto done; + } + ret = iboot_frame_command(query, step, NULL, 0, reply, &received); + if (ret) + goto failed; + ret = iboot_reply_payload(reply, received, step, &payload, &payload_size); + if (ret || payload_size < 20) { + ret = -EPROTO; + goto failed; + } + id = get_unaligned_le32(payload + 12); + if (!id || id == query->last_frame_swap) { + ret = -EPROTO; + goto failed; + } + iboot_build_pattern_layer(&layer, iova, stride); + step = 16; + ret = iboot_frame_command(query, step, &layer, sizeof(layer), reply, &received); + if (ret) + goto failed; + step = 18; + ret = iboot_frame_command(query, step, end, sizeof(end), reply, &received); + if (ret) + goto failed; + /* Display zero, exact swap id, wait for completion, no extra delay. + * Mode 1 waits while this id remains in the pending/current transaction + * queue. It does not wait for the newly displayed frame to be replaced. + * AFK bounds the host wait; on timeout neither buffer may be reused. + */ + wait[1] = cpu_to_le32(id); + wait[2] = cpu_to_le32(1); + step = 19; + ret = iboot_frame_command(query, step, wait, sizeof(wait), reply, &received); + if (ret) + goto failed; + query->last_frame_swap = id; + kfree(reply); + goto done; +failed: + query->presentation_failed = true; + dev_err(dcp->dev, "external frame swap %u failed at step %u: %d; buffers retained, no retry\n", + id, step, ret); + kfree(reply); +done: + atomic_set_release(&query->busy, 0); + return ret; +} + int ibootep_rearm_pattern(struct apple_dcp *dcp) { struct iboot_query *query = iboot_find_query(dcp); diff --git a/drivers/gpu/drm/apple/ibootep.h b/drivers/gpu/drm/apple/ibootep.h index 3d9a76e5a45bc..aa44c2f8754e4 100644 --- a/drivers/gpu/drm/apple/ibootep.h +++ b/drivers/gpu/drm/apple/ibootep.h @@ -21,4 +21,10 @@ int ibootep_present_pattern(struct apple_dcp *dcp, u64 iova, size_t size, u32 st /* Allow one more pattern request, for the retained buffer after a hotplug bounce. */ int ibootep_rearm_pattern(struct apple_dcp *dcp); +/* Submit a retained buffer without changing power/timing. Success follows + * SwapWait for this swap, not merely the SetSwapEnd command acknowledgment. + * Failure leaves DMA ownership uncertain: retain every submitted buffer. + */ +int ibootep_present_frame(struct apple_dcp *dcp, u64 iova, size_t size, u32 stride); + #endif diff --git a/tools/testing/selftests/drm/apple/README b/tools/testing/selftests/drm/apple/README new file mode 100644 index 0000000000000..bc0d80a86b500 --- /dev/null +++ b/tools/testing/selftests/drm/apple/README @@ -0,0 +1,28 @@ +Apple external-display host checks +================================= + +Run with Python 3 and a C compiler with address/undefined sanitizers: + + python3 tools/testing/selftests/drm/apple/test-iboot-wire.py + python3 tools/testing/selftests/drm/apple/test-dcpext-present.py + python3 tools/testing/selftests/drm/apple/test-dcpext-copy.py + +These compile the driver's actual functions with host transport/DMA stubs. +They exercise received-length validation and late-ACK cleanup, packet bounds, +presentation ordering, failure latching and buffer ownership. They do not +establish firmware behavior or visual correctness; those need a hardware boot. + +The candidate path is opt-in with appledrm.dcpext_pageflip=1 and limited to the +14.7 firmware UUID checked in dcpext_scanout_register(). It retains two mapped +buffers, copies a complete opaque frame into the idle one, then sends +SwapBegin/SetLayer/SwapEnd/SwapWait. The exact Begin ID is passed to Wait; +successful Wait is required before the old front buffer is reused. The host +wait is bounded by AFK. Failure retains both buffers and stops presentation. +Link loss is terminal in this candidate mode, including hotplug bounces after +startup: replug recovery must first account for an in-flight transaction. + +The read-only dcpext_present sysfs attribute reports mode, retained-buffer +count, completed desktop swaps, front-buffer index and maximum command/wait +duration in microseconds. Counters are diagnostics, not hardware timestamps. +DRM still timestamps completion in software after Wait; this is neither a +vblank interrupt implementation nor zero-copy scanout. diff --git a/tools/testing/selftests/drm/apple/test-dcpext-copy.py b/tools/testing/selftests/drm/apple/test-dcpext-copy.py new file mode 100644 index 0000000000000..e22a47ddf261e --- /dev/null +++ b/tools/testing/selftests/drm/apple/test-dcpext-copy.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Check full-frame conversion, padded source pitch and retained front pixels.""" +from pathlib import Path +import os +import subprocess +import tempfile + +base = Path(__file__).resolve().parents[5] / "drivers/gpu/drm/apple" +source = (base / "dcpext_drm.c").read_text() + + +def function(start, end): + return source[source.index(start):source.index(end)] + + +preamble = r''' +#include +#include +#include +#include +#include +#include +#include +#include "dcpext_mode.h" +typedef uint8_t u8; +typedef uint32_t u32; +typedef uint64_t u64; +#define EXT_WIDTH DCPEXT_WIDTH +#define EXT_HEIGHT DCPEXT_HEIGHT +#define EXT_STRIDE DCPEXT_STRIDE +struct iosys_map { void *vaddr; }; +struct drm_framebuffer { u32 pitches[1]; }; +struct drm_plane_state { struct drm_framebuffer *fb; }; +struct drm_shadow_plane_state { struct drm_plane_state base; struct iosys_map data[1]; }; +#define to_drm_shadow_plane_state(p) ((struct drm_shadow_plane_state *)(p)) +struct dcpext_drm { void *dcp; u8 *row; u32 stride; }; +struct dcpext_frame { void *pixels; size_t size; }; +static struct dcpext_frame back; +static bool begin_fails, held; +static unsigned presented, discarded; +static int dcpext_scanout_begin_frame(void *dcp, struct dcpext_frame *frame) { + (void)dcp; + if(begin_fails) return -ENOLINK; + assert(!held); held=true; *frame=back; return 0; +} +static int dcpext_scanout_end_frame(void *dcp, bool present) { + (void)dcp; assert(held); held=false; + if(present) presented++; else discarded++; + return 0; +} +static void iosys_map_memcpy_from(void *dst, const struct iosys_map *src, + size_t offset, size_t size) { + memcpy(dst,(const u8 *)src->vaddr+offset,size); +} +''' +tests = r''' +int main(void) { + const size_t size=(size_t)EXT_STRIDE*EXT_HEIGHT; + const u32 pitch=EXT_STRIDE+64; + u8 *source=malloc((size_t)pitch*EXT_HEIGHT); + u8 *buffers=malloc(size*2); + assert(source && buffers); + memset(buffers,0xab,size*2); + for(unsigned y=0;y +#include +#include +#include +#include +#include +#include +#include +#include "dcpext_mode.h" +typedef uint8_t u8; +typedef uint32_t u32; +typedef uint64_t u64; +typedef u32 __le32; +typedef u64 __le64; +#define __packed __attribute__((packed)) +#define static_assert _Static_assert +#define cpu_to_le32(x) (x) +#define cpu_to_le64(x) (x) +#define READ_ONCE(x) (x) +#define smp_load_acquire(p) (*(p)) +#define ARRAY_SIZE(x) (sizeof(x)/sizeof((x)[0])) +#define IBOOT_QUERY_RX_SIZE 0x4000 +#define IBOOT_MODE_SIZE 24 +#define IBOOT_PATTERN_WIDTH DCPEXT_WIDTH +#define IBOOT_PATTERN_HEIGHT DCPEXT_HEIGHT +#define IBOOT_PATTERN_STRIDE DCPEXT_STRIDE +#define SCANOUT_STRIDE DCPEXT_STRIDE +#define IBOOT_GET_HPD 3 +#define IBOOT_GET_TIMING_MODES 4 +#define IBOOT_GET_COLOR_MODES 5 +#define EPIC_SUBTYPE_STD_SERVICE 0x20 +#define DCP_FIRMWARE_V_14_7 147 +#define GFP_KERNEL 0 +#define IS_ERR(p) ((intptr_t)(p) < 0) +#define PTR_ERR(p) ((intptr_t)(p)) +#define might_sleep() do {} while(0) +#define dev_err(...) do {} while(0) +#define dma_wmb() do {} while(0) +#define max(a,b) ((a)>(b)?(a):(b)) +static bool alloc_fail; +static unsigned live; +static void *kzalloc(size_t n, int flags) { + (void)flags; + if (alloc_fail) return NULL; + void *p = calloc(1,n); assert(p); live++; return p; +} +static void kfree(void *p) { if(p) live--; free(p); } +static int atomic_cmpxchg(int *p, int old, int new) { + int value=*p; if(value==old) *p=new; return value; +} +#define atomic_set_release(p,v) (*(p)=(v)) +static u32 get_unaligned_le32(const void *p) { u32 v; memcpy(&v,p,4); return v; } +static void put_unaligned_le32(u32 v, void *p) { memcpy(p,&v,4); } +struct dcpext_scanout; +struct apple_dcp { int fw_compat; struct dcpext_scanout *dcpext_scanout; }; +struct apple_epic_service { bool torndown; }; +struct iboot_query { + struct apple_epic_service *service; + int busy; + bool stopping, pattern_requested, presentation_failed; + u32 last_frame_swap; +}; +static struct apple_epic_service service; +static struct iboot_query query; +static struct iboot_query *iboot_find_query(struct apple_dcp *dcp) { + (void)dcp; return &query; +} +static unsigned calls, fail_step, duplicate_id, short_reply, remote_error; +static u32 returned_id; +static u64 submitted_iova; +static int afk_send_command_with_reply_len(struct apple_epic_service *s, u8 type, + const void *request, size_t size, void *output, size_t capacity, + u32 *retcode, size_t *received) { + const unsigned sequence[]={15,16,18,19}; + const u8 *wire=request; + unsigned op=get_unaligned_le32(wire); + assert(s==&service && type==EPIC_SUBTYPE_STD_SERVICE); + assert(calls<4 && op==sequence[calls++]); + assert(get_unaligned_le32(wire+4)==size && capacity==IBOOT_QUERY_RX_SIZE); + if(op==fail_step) return -ETIMEDOUT; + *retcode=remote_error && op==19 ? 1 : 0; + *received=0; + memset(output,0,capacity); + if(op==15) { + returned_id=duplicate_id ? query.last_frame_swap : query.last_frame_swap+1; + put_unaligned_le32(15,output); + put_unaligned_le32(28,(u8 *)output+4); + put_unaligned_le32(returned_id,(u8 *)output+20); + *received=short_reply ? 27 : 28; + } else if(op==16) { + assert(size==232); + memcpy(&submitted_iova,wire+28,8); + } else if(op==18) { + assert(size==28); + for(unsigned i=16;i<28;i++) assert(wire[i]==0); + } else { + assert(size==32 && get_unaligned_le32(wire+16)==0); + assert(get_unaligned_le32(wire+20)==returned_id); + assert(get_unaligned_le32(wire+24)==1 && get_unaligned_le32(wire+28)==0); + } + return 0; +} +struct dcpext_frame { void *pixels; size_t size; }; +struct dcpext_scanout { + bool pageflips, stopping, pattern_ready, link; + int present_lock, terminal_error; + void *pixels; + size_t frame_size; + unsigned front; + u64 iova, frames_completed, present_ns_max; +}; +static void mutex_lock(int *p) { assert(!*p); *p=1; } +static void mutex_unlock(int *p) { assert(*p); *p=0; } +#define lockdep_assert_held(p) assert(*(p)) +static bool scanout_link_ready(struct dcpext_scanout *s) { return s->link && !s->terminal_error; } +static void scanout_fail(struct dcpext_scanout *s, int error) { s->terminal_error=error; } +static u64 ktime_get_ns(void) { static u64 clock; return ++clock; } +''' + +tests = r''' +static void reset(void) { + assert(!live); + query=(struct iboot_query){.service=&service,.pattern_requested=true}; + calls=fail_step=duplicate_id=short_reply=remote_error=0; + alloc_fail=false; + service.torndown=false; +} +int main(void) { + struct apple_dcp dcp={.fw_compat=DCP_FIRMWARE_V_14_7}; + const u64 address=0x10000000000ULL; + const size_t bytes=(size_t)DCPEXT_STRIDE*DCPEXT_HEIGHT; + reset(); + assert(!ibootep_present_frame(&dcp,address,bytes,DCPEXT_STRIDE)); + assert(calls==4 && query.last_frame_swap==1 && !query.busy && !live); + assert(submitted_iova==address); + calls=0; + assert(!ibootep_present_frame(&dcp,address+bytes,bytes,DCPEXT_STRIDE)); + assert(calls==4 && query.last_frame_swap==2 && submitted_iova==address+bytes); + unsigned steps[]={15,16,18,19}; + for(unsigned i=0;i<4;i++) { + reset(); fail_step=steps[i]; + assert(ibootep_present_frame(&dcp,address,bytes,DCPEXT_STRIDE)==-ETIMEDOUT); + assert(calls==i+1 && query.presentation_failed && !query.last_frame_swap); + assert(!query.busy && !live); + assert(ibootep_present_frame(&dcp,address,bytes,DCPEXT_STRIDE)==-EIO); + assert(calls==i+1); + } + reset(); short_reply=1; + assert(ibootep_present_frame(&dcp,address,bytes,DCPEXT_STRIDE)==-EPROTO); + assert(calls==1 && query.presentation_failed); + reset(); duplicate_id=1; query.last_frame_swap=10; + assert(ibootep_present_frame(&dcp,address,bytes,DCPEXT_STRIDE)==-EPROTO); + assert(calls==1 && query.last_frame_swap==10); + reset(); remote_error=1; + assert(ibootep_present_frame(&dcp,address,bytes,DCPEXT_STRIDE)==-EIO); + assert(calls==4 && query.presentation_failed && !query.last_frame_swap); + reset(); query.busy=1; + assert(ibootep_present_frame(&dcp,address,bytes,DCPEXT_STRIDE)==-EBUSY); + assert(!calls && query.busy); + reset(); alloc_fail=true; + assert(ibootep_present_frame(&dcp,address,bytes,DCPEXT_STRIDE)==-ENOMEM); + assert(!calls && !query.busy && !query.presentation_failed); + reset(); query.stopping=true; + assert(ibootep_present_frame(&dcp,address,bytes,DCPEXT_STRIDE)==-ENODEV); + assert(!calls && query.presentation_failed); + reset(); + struct dcpext_scanout scan={.pageflips=true,.pattern_ready=true,.link=true, + .pixels=calloc(2,bytes),.frame_size=bytes,.iova=address}; + assert(scan.pixels); + dcp.dcpext_scanout=&scan; + struct dcpext_frame frame; + assert(!dcpext_scanout_begin_frame(&dcp,&frame)); + assert(frame.pixels==(u8 *)scan.pixels+bytes && frame.size==bytes && scan.present_lock); + assert(!dcpext_scanout_end_frame(&dcp,false)); + assert(!calls && !scan.front && !scan.present_lock); + for(unsigned i=0;i<6;i++) { + calls=0; + assert(!dcpext_scanout_begin_frame(&dcp,&frame)); + assert(frame.pixels==(u8 *)scan.pixels+(i%2 ? 0 : bytes)); + assert(!dcpext_scanout_end_frame(&dcp,true)); + assert(calls==4 && scan.front==(i%2 ? 0 : 1)); + assert(submitted_iova==address+scan.front*bytes && scan.frames_completed==i+1); + } + calls=0; fail_step=19; + assert(!dcpext_scanout_begin_frame(&dcp,&frame)); + assert(dcpext_scanout_end_frame(&dcp,true)==-ETIMEDOUT); + assert(scan.front==0 && scan.frames_completed==6 && scan.terminal_error==-EIO); + assert(!scan.present_lock && !live); + assert(dcpext_scanout_begin_frame(&dcp,&frame)==-ENOLINK); + assert(calls==4 && !scan.present_lock); + scan.terminal_error=0; scan.link=false; + assert(dcpext_scanout_begin_frame(&dcp,&frame)==-ENOLINK); + scan.link=true; scan.stopping=true; + assert(dcpext_scanout_begin_frame(&dcp,&frame)==-ENOLINK); + free(scan.pixels); + puts("PASS: actual frame sequence, exact swap wait, malformed/failed replies, no retry, buffer reuse only after completion"); +} +''' + +layouts = iboot[iboot.index("struct iboot_plane {"):iboot.index("struct iboot_query {")] +helpers = iboot[iboot.index("static int iboot_build_query("):iboot.index("static int iboot_read_query(")] +body = preamble + layouts + helpers +body += function(iboot, "iboot_frame_command") + function(iboot, "ibootep_present_frame") +body += function(scanout, "dcpext_scanout_begin_frame") + function(scanout, "dcpext_scanout_end_frame") +with tempfile.TemporaryDirectory(prefix="dcpext-present-test-") as tmp: + path = Path(tmp) + (path / "test.c").write_text(body + tests) + subprocess.run([os.environ.get("CC", "cc"), "-Wall", "-Wextra", "-Werror", + "-Wno-unused-function", "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", "-I", str(base), + "-o", str(path / "test"), str(path / "test.c")], check=True) + subprocess.run([str(path / "test")], check=True) diff --git a/tools/testing/selftests/drm/apple/test-iboot-wire.py b/tools/testing/selftests/drm/apple/test-iboot-wire.py new file mode 100644 index 0000000000000..950609e3c238d --- /dev/null +++ b/tools/testing/selftests/drm/apple/test-iboot-wire.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile and exercise the actual ibootep.c wire helpers, no hardware access.""" +from pathlib import Path +import os +import subprocess +import tempfile + +base = Path(__file__).resolve().parents[5] / 'drivers/gpu/drm/apple' +src = (base / 'ibootep.c').read_text() +helpers = src[src.index('static int iboot_build_query('):src.index('static int iboot_read_query(')] +layouts = src[src.index('struct iboot_plane {'):src.index('struct iboot_query {')] +prefix = r''' +#include +#include "dcpext_mode.h" +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint32_t u32; +typedef uint64_t u64; +typedef uint32_t __le32; +typedef uint64_t __le64; +#define cpu_to_le32(x) (x) +#define cpu_to_le64(x) (x) +#define __packed __attribute__((packed)) +#define static_assert _Static_assert +#define IBOOT_QUERY_RX_SIZE 0x4000 +#define IBOOT_MODE_SIZE 24 +#define IBOOT_PATTERN_WIDTH 3840 +#define IBOOT_PATTERN_HEIGHT 2160 +#define IBOOT_PATTERN_STRIDE 15360 +enum { IBOOT_GET_HPD=3, IBOOT_GET_TIMING_MODES=4, IBOOT_GET_COLOR_MODES=5 }; +static u32 get_unaligned_le32(const void *p_) { + const u8 *p = p_; + return p[0] | ((u32)p[1]<<8) | ((u32)p[2]<<16) | ((u32)p[3]<<24); +} +static void put32(u8 *p, u32 x) { + for (unsigned i=0; i<4; i++) p[i] = x >> (i*8); +} +#define put_unaligned_le32(v,p) put32(p,v) +''' +tests = r''' +int main(void) { + __le32 req[4]; + for (u32 op=0; op<20; op++) { + memset(req, 0xaa, sizeof(req)); + int ret=iboot_build_query(op, req); + if (op<3 || op>5) { assert(ret==-EINVAL); continue; } + assert(ret==0 && req[0]==op && req[1]==16 && req[2]==0 && req[3]==0); + } + u8 reply[IBOOT_QUERY_RX_SIZE] = {0}; + const u8 *p; + size_t n; + put32(reply, 3); put32(reply+4, 20); + assert(!iboot_reply_payload(reply, sizeof(reply), 3, &p, &n)); + assert(p==reply+8 && n==12); + assert(iboot_reply_payload(reply, 7, 3, &p, &n)==-EPROTO); + assert(iboot_reply_payload(reply, sizeof(reply), 4, &p, &n)==-EPROTO); + put32(reply+4, 7); + assert(iboot_reply_payload(reply, sizeof(reply), 3, &p, &n)==-EPROTO); + put32(reply+4, sizeof(reply)+1); + assert(iboot_reply_payload(reply, sizeof(reply), 3, &p, &n)==-EPROTO); + put32(reply+4, 20); + assert(!iboot_reply_payload(reply, sizeof(reply), 3, &p, &n)); + bool hpd; u32 nt, nc; + assert(!iboot_parse_hpd(p,n,&hpd,&nt,&nc) && !hpd && !nt && !nc); + reply[8]=1; put32(reply+12, 4); put32(reply+16, 7); + assert(!iboot_parse_hpd(p,n,&hpd,&nt,&nc) && hpd && nt==4 && nc==7); + assert(iboot_parse_hpd(p,11,&hpd,&nt,&nc)==-EPROTO); + reply[8]=2; + assert(iboot_parse_hpd(p,n,&hpd,&nt,&nc)==-EPROTO); + reply[8]=1; put32(reply+12, UINT32_MAX); + assert(iboot_parse_hpd(p,n,&hpd,&nt,&nc)==-EOVERFLOW); + memset(reply, 0, sizeof(reply)); + put32(reply, 4); put32(reply+4, 8+4+2*24); put32(reply+8, 2); + put32(reply+12, 1); put32(reply+16, 3840); put32(reply+20, 2160); + put32(reply+24, 60U<<16); + assert(!iboot_reply_payload(reply, sizeof(reply), 4, &p, &n)); + u32 count; + assert(!iboot_parse_modes(p,n,&count) && count==2); + assert(iboot_parse_modes(p,3,&count)==-EPROTO); + assert(iboot_parse_modes(p,n-1,&count)==-EPROTO); + put32(reply+12, 2); + assert(iboot_parse_modes(p,n,&count)==-EPROTO); + put32(reply+12, 1); put32(reply+8, UINT32_MAX); + assert(iboot_parse_modes(p,n,&count)==-EPROTO); + put32(reply+8, 0); + assert(!iboot_parse_modes(p,4,&count) && count==0); + /* Every bounded length/count pair: no helper may read past a declared list. */ + for (size_t size=0; size<200; size++) { + for (u32 c=0; c<16; c++) { + memset(reply,0,sizeof(reply)); put32(reply,c); + int r=iboot_parse_modes(reply,size,&count); + assert((r==0)==(size>=4 && c<=(size-4)/24)); + } + } + u8 mode[24]={0}; + put32(mode,1); put32(mode+4,3840); put32(mode+8,2160); put32(mode+12,60U<<16); + assert(iboot_pattern_mode(4,mode)); + put32(mode+4,1920);put32(mode+8,1080);assert(!iboot_pattern_mode(4,mode)); + put32(mode+4,3840);put32(mode+8,2160); + put32(mode+12,(60U<<16)-1); + assert(!iboot_pattern_mode(4,mode)); + put32(mode+4,1); put32(mode+8,1); put32(mode+12,1); put32(mode+16,32); + assert(iboot_pattern_mode(5,mode)); + put32(mode+16,30); assert(!iboot_pattern_mode(5,mode)); + put32(mode+16,32); put32(mode,0); assert(!iboot_pattern_mode(5,mode)); + assert(!iboot_pattern_mode(6,mode)); + u64 iova=0x10100100000ULL; + size_t fb_size=15360*2160; + assert(!iboot_pattern_params(iova,fb_size,15360)); + assert(iboot_pattern_params(0,fb_size,15360)==-EINVAL); + assert(iboot_pattern_params(iova+1,fb_size,15360)==-EINVAL); + assert(iboot_pattern_params(iova,fb_size-1,15360)==-EINVAL); + assert(iboot_pattern_params(iova,fb_size,15364)==-EINVAL); + assert(iboot_pattern_params(iova,65U*1024*1024,15360)==-EINVAL); + assert(iboot_pattern_params((1ULL<<42)-0x4000,fb_size,15360)==-EINVAL); + assert(iboot_pattern_params(UINT64_MAX,fb_size,15360)==-EINVAL); + struct iboot_swap_layer_v13_3 layer; + iboot_build_pattern_layer(&layer,iova,15360); + assert(sizeof(layer)==216); + assert(offsetof(struct iboot_swap_layer_v13_3,src)==180); + assert(offsetof(struct iboot_swap_layer_v13_3,dst)==196); + u8 expected[216]={0}; + /* Independent byte fixture: upstream m1n1 >=13.3, opaque linear BGRA. */ + put32(expected+12,(u32)iova); put32(expected+16,(u32)(iova>>32)); + put32(expected+24,15360); put32(expected+44,1); + put32(expected+144,1); put32(expected+148,3840); put32(expected+152,2160); + put32(expected+156,1); put32(expected+160,1); put32(expected+164,1); + put32(expected+180,3840); put32(expected+184,2160); + put32(expected+196,3840); put32(expected+200,2160); + assert(!memcmp(&layer,expected,sizeof(expected))); + u8 request[232], power=1, modes[48]={0}, end[12]={0}; + size_t request_size; + assert(!iboot_build_pattern_request(2,&power,1,request,sizeof(request),&request_size)); + assert(request_size==17 && request[0]==2 && request[4]==17 && request[16]==1); + assert(!iboot_build_pattern_request(6,modes,48,request,sizeof(request),&request_size)); + assert(request_size==64); + assert(!iboot_build_pattern_request(15,NULL,0,request,sizeof(request),&request_size)); + assert(request_size==16); + assert(!iboot_build_pattern_request(16,&layer,sizeof(layer),request,sizeof(request),&request_size)); + assert(request_size==232 && request[0]==16 && request[4]==232); + assert(!memcmp(request+16,expected,216)); + for (size_t i=8;i<16;i++) assert(request[i]==0); + assert(!iboot_build_pattern_request(18,end,12,request,sizeof(request),&request_size)); + assert(request_size==28); + __le32 wait[4] = {0, 123, 1, 0}; + assert(!iboot_build_pattern_request(19,wait,sizeof(wait),request,sizeof(request),&request_size)); + assert(request_size==32 && request[0]==19 && request[4]==32); + assert(get_unaligned_le32(request+16)==0 && get_unaligned_le32(request+20)==123); + assert(get_unaligned_le32(request+24)==1 && get_unaligned_le32(request+28)==0); + assert(iboot_build_pattern_request(19,wait,12,request,sizeof(request),&request_size)==-EINVAL); + assert(iboot_build_pattern_request(1,&layer,sizeof(layer),request,sizeof(request),&request_size)==-EINVAL); + assert(iboot_build_pattern_request(16,&layer,sizeof(layer)-8,request,sizeof(request),&request_size)==-EINVAL); + assert(iboot_build_pattern_request(16,&layer,sizeof(layer),request,sizeof(request)-1,&request_size)==-EINVAL); + assert(iboot_build_pattern_request(2,NULL,1,request,sizeof(request),&request_size)==-EINVAL); + puts("PASS: actual iBoot query/pattern helpers, >=13.3 layer bytes, bounds and malformed replies"); +} +''' +with tempfile.TemporaryDirectory(prefix='iboot-query-test-') as path: + path=Path(path) + (path/'test.c').write_text(prefix+layouts+helpers+tests) + subprocess.run([os.environ.get('CC','cc'), '-Wall', '-Wextra', '-Werror', '-I', str(base), + '-fsanitize=address,undefined', '-fno-omit-frame-pointer', + '-o', str(path/'test'), str(path/'test.c')],check=True) + subprocess.run([str(path/'test')],check=True) From ce18f50a50d94feb0844e3e3aee65ec2f00dadea Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 12:51:51 -0700 Subject: [PATCH 1553/2680] drm/apple: handle the 14.7 SwapBegin reply-length mismatch The first candidate boot stopped at SwapBegin with EPROTO and never registered the external desktop. The gated firmware emits a 284-byte inner length but returns the 28-byte Begin record through EPIC. Accept only that exact opcode/declared/received-length combination; continue rejecting truncation and other invalid lengths. The real presentation host test reproduces the failed boot before this change and passes afterward. Add parser cases for every truncated length, wrong op, nearby incorrect length and the ordinary 28-byte envelope. No live firmware retry was attempted after failure. (cherry picked from commit b8e901f453371606e2ac403f53b196be335e33e5) --- drivers/gpu/drm/apple/ibootep.c | 35 +++++++++++++++---- .../drm/apple/test-dcpext-present.py | 3 +- .../selftests/drm/apple/test-iboot-wire.py | 14 ++++++++ 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/ibootep.c b/drivers/gpu/drm/apple/ibootep.c index f473258e778e1..e9613d3f0a9de 100644 --- a/drivers/gpu/drm/apple/ibootep.c +++ b/drivers/gpu/drm/apple/ibootep.c @@ -123,6 +123,31 @@ static int iboot_parse_hpd(const u8 *payload, size_t size, bool *hpd, return 0; } +static int iboot_parse_swap_begin(const u8 *reply, size_t received, u32 *id) +{ + const u8 *payload; + size_t size; + int ret; + + /* The gated 14.7 firmware writes 284 in this wrapper but returns only + * the 28-byte Begin record through EPIC. Accept that exact mismatch, + * bounded by the actual transport length; never read the absent tail. + */ + if (received == 28 && get_unaligned_le32(reply) == 15 && + get_unaligned_le32(reply + 4) == 284) { + payload = reply + 8; + size = 20; + } else { + ret = iboot_reply_payload(reply, received, 15, &payload, &size); + if (ret) + return ret; + } + if (size < 20) + return -EPROTO; + *id = get_unaligned_le32(payload + 12); + return 0; +} + static int iboot_parse_modes(const u8 *payload, size_t size, u32 *count) { u32 i; @@ -480,8 +505,7 @@ int ibootep_present_frame(struct apple_dcp *dcp, u64 iova, size_t size, u32 stri struct iboot_swap_layer_v13_3 layer; struct iboot_query *query; __le32 end[3] = {}, wait[4] = {}; - const u8 *payload; - size_t received, payload_size; + size_t received; u32 id = 0, step = 15; u8 *reply; int ret; @@ -509,12 +533,9 @@ int ibootep_present_frame(struct apple_dcp *dcp, u64 iova, size_t size, u32 stri ret = iboot_frame_command(query, step, NULL, 0, reply, &received); if (ret) goto failed; - ret = iboot_reply_payload(reply, received, step, &payload, &payload_size); - if (ret || payload_size < 20) { - ret = -EPROTO; + ret = iboot_parse_swap_begin(reply, received, &id); + if (ret) goto failed; - } - id = get_unaligned_le32(payload + 12); if (!id || id == query->last_frame_swap) { ret = -EPROTO; goto failed; diff --git a/tools/testing/selftests/drm/apple/test-dcpext-present.py b/tools/testing/selftests/drm/apple/test-dcpext-present.py index 67a0c72f9cbe0..98adf005de243 100644 --- a/tools/testing/selftests/drm/apple/test-dcpext-present.py +++ b/tools/testing/selftests/drm/apple/test-dcpext-present.py @@ -110,7 +110,8 @@ def function(source, name): if(op==15) { returned_id=duplicate_id ? query.last_frame_swap : query.last_frame_swap+1; put_unaligned_le32(15,output); - put_unaligned_le32(28,(u8 *)output+4); + /* The audited 14.7 Begin wrapper says 284; EPIC returns 28 bytes. */ + put_unaligned_le32(284,(u8 *)output+4); put_unaligned_le32(returned_id,(u8 *)output+20); *received=short_reply ? 27 : 28; } else if(op==16) { diff --git a/tools/testing/selftests/drm/apple/test-iboot-wire.py b/tools/testing/selftests/drm/apple/test-iboot-wire.py index 950609e3c238d..bfaef16a0503a 100644 --- a/tools/testing/selftests/drm/apple/test-iboot-wire.py +++ b/tools/testing/selftests/drm/apple/test-iboot-wire.py @@ -159,6 +159,20 @@ assert(iboot_build_pattern_request(16,&layer,sizeof(layer)-8,request,sizeof(request),&request_size)==-EINVAL); assert(iboot_build_pattern_request(16,&layer,sizeof(layer),request,sizeof(request)-1,&request_size)==-EINVAL); assert(iboot_build_pattern_request(2,NULL,1,request,sizeof(request),&request_size)==-EINVAL); + /* 14.7 Begin has a 284-byte wrapper claim but returns a 28-byte record. */ + u32 swap_id; + memset(reply,0,sizeof(reply)); + put32(reply,15); put32(reply+4,284); put32(reply+20,123); + assert(!iboot_parse_swap_begin(reply,28,&swap_id) && swap_id==123); + for (size_t received=0;received<28;received++) + assert(iboot_parse_swap_begin(reply,received,&swap_id)==-EPROTO); + assert(iboot_parse_swap_begin(reply,29,&swap_id)==-EPROTO); + put32(reply+4,283); + assert(iboot_parse_swap_begin(reply,28,&swap_id)==-EPROTO); + put32(reply+4,284); put32(reply,14); + assert(iboot_parse_swap_begin(reply,28,&swap_id)==-EPROTO); + put32(reply,15); put32(reply+4,28); + assert(!iboot_parse_swap_begin(reply,28,&swap_id) && swap_id==123); puts("PASS: actual iBoot query/pattern helpers, >=13.3 layer bytes, bounds and malformed replies"); } ''' From a79b26dae9798a27abf6d12dd66007c4efc2469d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 13:37:04 -0700 Subject: [PATCH 1554/2680] drm/apple: repair idle scanout using two frames of damage (cherry picked from commit 3e8c83096391d8dbd6231987e6032712697f3d6a) --- drivers/gpu/drm/apple/dcpext_drm.c | 53 +++++++-- tools/testing/selftests/drm/apple/README | 10 +- .../selftests/drm/apple/test-dcpext-copy.py | 103 +++++++++++++----- 3 files changed, 127 insertions(+), 39 deletions(-) diff --git a/drivers/gpu/drm/apple/dcpext_drm.c b/drivers/gpu/drm/apple/dcpext_drm.c index 27a794881a6da..ee4b2bd185b76 100644 --- a/drivers/gpu/drm/apple/dcpext_drm.c +++ b/drivers/gpu/drm/apple/dcpext_drm.c @@ -37,6 +37,9 @@ struct dcpext_drm { struct apple_dcp *dcp; bool pageflips; u8 *row; + /* Damage from the last completed frame; the idle buffer is one older. */ + struct drm_rect previous_damage; + bool damage_valid; /* Owned by the retained scanout, never by the detachable parent device. */ atomic_t *terminal_error; }; @@ -117,10 +120,13 @@ static void ext_cancel_flip(struct dcpext_drm *ext, struct drm_atomic_commit *st drm_event_cancel_free(&ext->drm, &event->base); } -static int ext_copy_frame(struct dcpext_drm *ext, struct drm_plane_state *ps) +static int ext_copy_frame(struct dcpext_drm *ext, struct drm_plane_state *old, + struct drm_plane_state *ps) { struct drm_shadow_plane_state *shadow = to_drm_shadow_plane_state(ps); struct drm_framebuffer *fb = ps->fb; + struct drm_rect bounds = DRM_RECT_INIT(0, 0, EXT_WIDTH, EXT_HEIGHT); + struct drm_rect damage, copy; struct dcpext_frame frame; int x, y, ret; @@ -131,22 +137,46 @@ static int ext_copy_frame(struct dcpext_drm *ext, struct drm_plane_state *ps) dcpext_scanout_end_frame(ext->dcp, false); return -EINVAL; } - /* The idle buffer contains an older frame, so damage alone is insufficient. - * Copy the complete current framebuffer before publishing its address. + /* The idle buffer contains frame N-2. Repair both N-1's damage and the + * current damage from the current framebuffer, without touching scanout. + * Invalid history forces two complete copies, initializing both buffers. */ - for (y = 0; y < EXT_HEIGHT; y++) { - u8 *dst = frame.pixels + (size_t)y * ext->stride; + if (!ext->damage_valid) + damage = bounds; + else if (!drm_atomic_helper_damage_merged(old, ps, &damage) || + !drm_rect_intersect(&damage, &bounds)) + damage = (struct drm_rect) { 0 }; + copy = damage; + if (ext->damage_valid && drm_rect_visible(&ext->previous_damage)) { + if (!drm_rect_visible(©)) { + copy = ext->previous_damage; + } else { + copy.x1 = min(copy.x1, ext->previous_damage.x1); + copy.y1 = min(copy.y1, ext->previous_damage.y1); + copy.x2 = max(copy.x2, ext->previous_damage.x2); + copy.y2 = max(copy.y2, ext->previous_damage.y2); + } + } + for (y = copy.y1; drm_rect_visible(©) && y < copy.y2; y++) { + size_t offset = (size_t)y * fb->pitches[0] + copy.x1 * 4; + size_t bytes = drm_rect_width(©) * 4; + u8 *dst = frame.pixels + (size_t)y * ext->stride + copy.x1 * 4; - iosys_map_memcpy_from(ext->row, &shadow->data[0], - (size_t)y * fb->pitches[0], EXT_STRIDE); - for (x = 0; x < EXT_WIDTH; x++) + iosys_map_memcpy_from(ext->row, &shadow->data[0], offset, bytes); + for (x = 0; x < drm_rect_width(©); x++) ext->row[x * 4 + 3] = 0xff; /* Convert in cached memory, then stream to coherent scanout memory. * Byte stores directly to that mapping make full 4K copies costly. */ - memcpy(dst, ext->row, EXT_STRIDE); + memcpy(dst, ext->row, bytes); + } + ret = dcpext_scanout_end_frame(ext->dcp, true); + /* Ordered atomic commits advance history only after a completed swap. */ + if (!ret) { + ext->previous_damage = damage; + ext->damage_valid = true; } - return dcpext_scanout_end_frame(ext->dcp, true); + return ret; } static void ext_plane_update(struct drm_plane *plane, struct drm_atomic_commit *state) @@ -175,7 +205,7 @@ static void ext_plane_update(struct drm_plane *plane, struct drm_atomic_commit * goto end_access; } if (ext->pageflips) { - ret = ext_copy_frame(ext, ps); + ret = ext_copy_frame(ext, old, ps); goto exit_device; } drm_atomic_helper_damage_iter_init(&iter, old, ps); @@ -223,6 +253,7 @@ static void ext_plane_disable(struct drm_plane *plane, struct drm_atomic_commit return; } if (ext->pageflips) { + ext->damage_valid = false; ret = dcpext_scanout_begin_frame(ext->dcp, &frame); if (ret) { ext_cancel_flip(ext, state, ret); diff --git a/tools/testing/selftests/drm/apple/README b/tools/testing/selftests/drm/apple/README index bc0d80a86b500..210a00b4c80d8 100644 --- a/tools/testing/selftests/drm/apple/README +++ b/tools/testing/selftests/drm/apple/README @@ -14,13 +14,21 @@ establish firmware behavior or visual correctness; those need a hardware boot. The candidate path is opt-in with appledrm.dcpext_pageflip=1 and limited to the 14.7 firmware UUID checked in dcpext_scanout_register(). It retains two mapped -buffers, copies a complete opaque frame into the idle one, then sends +buffers, prepares a complete opaque frame in the idle one, then sends SwapBegin/SetLayer/SwapEnd/SwapWait. The exact Begin ID is passed to Wait; successful Wait is required before the old front buffer is reused. The host wait is bounded by AFK. Failure retains both buffers and stops presentation. Link loss is terminal in this candidate mode, including hotplug bounces after startup: replug recovery must first account for an in-flight transaction. +After initializing both buffers, the copy includes the current and previous +frame's damage: the idle buffer contains frame N-2. Missing damage or invalid +history requires full updates; disabling the plane invalidates history. History +advances only after a completed swap. The copy test compares 48 frames with a +complete reference, covering disjoint/overlapping/empty/clipped damage, reset, +opaque alpha, padded pitch, failed swaps and untouched front-buffer pixels. +It also checks that small updates read less than one percent of a full frame. + The read-only dcpext_present sysfs attribute reports mode, retained-buffer count, completed desktop swaps, front-buffer index and maximum command/wait duration in microseconds. Counters are diagnostics, not hardware timestamps. diff --git a/tools/testing/selftests/drm/apple/test-dcpext-copy.py b/tools/testing/selftests/drm/apple/test-dcpext-copy.py index e22a47ddf261e..9922207bb53d5 100644 --- a/tools/testing/selftests/drm/apple/test-dcpext-copy.py +++ b/tools/testing/selftests/drm/apple/test-dcpext-copy.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only OR MIT -"""Check full-frame conversion, padded source pitch and retained front pixels.""" +"""Exercise actual two-buffer copy/history code against complete reference frames.""" from pathlib import Path import os import subprocess @@ -29,16 +29,33 @@ def function(start, end): #define EXT_WIDTH DCPEXT_WIDTH #define EXT_HEIGHT DCPEXT_HEIGHT #define EXT_STRIDE DCPEXT_STRIDE +#define min(a,b) ((a)<(b)?(a):(b)) +#define max(a,b) ((a)>(b)?(a):(b)) +struct drm_rect { int x1,y1,x2,y2; }; +#define DRM_RECT_INIT(x,y,w,h) {x,y,(x)+(w),(y)+(h)} +static bool drm_rect_visible(const struct drm_rect *r) { return r->x1x2 && r->y1y2; } +static int drm_rect_width(const struct drm_rect *r) { return r->x2-r->x1; } +static bool drm_rect_intersect(struct drm_rect *a,const struct drm_rect *b) { + a->x1=max(a->x1,b->x1);a->y1=max(a->y1,b->y1); + a->x2=min(a->x2,b->x2);a->y2=min(a->y2,b->y2);return drm_rect_visible(a); +} struct iosys_map { void *vaddr; }; struct drm_framebuffer { u32 pitches[1]; }; -struct drm_plane_state { struct drm_framebuffer *fb; }; +struct drm_plane_state { struct drm_framebuffer *fb; struct drm_rect damage; bool full; }; struct drm_shadow_plane_state { struct drm_plane_state base; struct iosys_map data[1]; }; #define to_drm_shadow_plane_state(p) ((struct drm_shadow_plane_state *)(p)) -struct dcpext_drm { void *dcp; u8 *row; u32 stride; }; +/* The core helper supplies clipped/merged damage, or full damage when absent. */ +static bool drm_atomic_helper_damage_merged(const struct drm_plane_state *old, + const struct drm_plane_state *ps,struct drm_rect *out) { + (void)old;*out=ps->full?(struct drm_rect)DRM_RECT_INIT(0,0,EXT_WIDTH,EXT_HEIGHT):ps->damage; + return drm_rect_visible(out); +} +struct dcpext_drm { void *dcp; u8 *row; u32 stride; struct drm_rect previous_damage; bool damage_valid; }; struct dcpext_frame { void *pixels; size_t size; }; static struct dcpext_frame back; -static bool begin_fails, held; +static bool begin_fails, end_fails, held; static unsigned presented, discarded; +static size_t copied; static int dcpext_scanout_begin_frame(void *dcp, struct dcpext_frame *frame) { (void)dcp; if(begin_fails) return -ENOLINK; @@ -46,41 +63,73 @@ def function(start, end): } static int dcpext_scanout_end_frame(void *dcp, bool present) { (void)dcp; assert(held); held=false; + if(end_fails)return -EIO; if(present) presented++; else discarded++; return 0; } static void iosys_map_memcpy_from(void *dst, const struct iosys_map *src, size_t offset, size_t size) { - memcpy(dst,(const u8 *)src->vaddr+offset,size); + memcpy(dst,(const u8 *)src->vaddr+offset,size);copied+=size; } ''' tests = r''' +static void paint(u8 *source,u8 *reference,u32 pitch,struct drm_rect r,unsigned seed) { + struct drm_rect bounds=DRM_RECT_INIT(0,0,EXT_WIDTH,EXT_HEIGHT); + if(!drm_rect_intersect(&r,&bounds))return; + for(int y=r.y1;y=2 && step<12)assert(copied Date: Sun, 27 Sep 2026 18:26:32 -0700 Subject: [PATCH 1555/2680] drm/apple: retain sparse tiled damage for external buffer repair (cherry picked from commit 4a02c1cf7a12313c891ec490c1330d072c90b4ac) --- drivers/gpu/drm/apple/dcpext_drm.c | 116 +++++++++++++----- .../selftests/drm/apple/test-dcpext-copy.py | 78 +++++++++--- 2 files changed, 146 insertions(+), 48 deletions(-) diff --git a/drivers/gpu/drm/apple/dcpext_drm.c b/drivers/gpu/drm/apple/dcpext_drm.c index ee4b2bd185b76..1d9981b5def3f 100644 --- a/drivers/gpu/drm/apple/dcpext_drm.c +++ b/drivers/gpu/drm/apple/dcpext_drm.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Fixed iBoot mode, CPU shadow updates, optionally synchronized buffer swaps. */ #include +#include #include #include #include @@ -24,6 +25,10 @@ #define EXT_WIDTH DCPEXT_WIDTH #define EXT_HEIGHT DCPEXT_HEIGHT #define EXT_STRIDE DCPEXT_STRIDE +#define EXT_TILE_SIZE 32 +#define EXT_TILE_COLS DIV_ROUND_UP(EXT_WIDTH, EXT_TILE_SIZE) +#define EXT_TILE_ROWS DIV_ROUND_UP(EXT_HEIGHT, EXT_TILE_SIZE) +#define EXT_TILE_COUNT (EXT_TILE_COLS * EXT_TILE_ROWS) struct dcpext_drm { struct drm_device drm; @@ -38,7 +43,9 @@ struct dcpext_drm { bool pageflips; u8 *row; /* Damage from the last completed frame; the idle buffer is one older. */ - struct drm_rect previous_damage; + DECLARE_BITMAP(previous_damage, EXT_TILE_COUNT); + DECLARE_BITMAP(current_damage, EXT_TILE_COUNT); + DECLARE_BITMAP(copy_damage, EXT_TILE_COUNT); bool damage_valid; /* Owned by the retained scanout, never by the detachable parent device. */ atomic_t *terminal_error; @@ -120,15 +127,63 @@ static void ext_cancel_flip(struct dcpext_drm *ext, struct drm_atomic_commit *st drm_event_cancel_free(&ext->drm, &event->base); } -static int ext_copy_frame(struct dcpext_drm *ext, struct drm_plane_state *old, - struct drm_plane_state *ps) +/* Bound history storage without merging distant clips into one large box. + * A tile conservatively includes unchanged edge pixels, never omits damage. + * Scratch maps are protected by the retained scanout's presentation lock. + */ +static void ext_damage_tiles(struct dcpext_drm *ext, struct drm_plane_state *old, + struct drm_plane_state *ps) +{ + struct drm_atomic_helper_damage_iter iter; + struct drm_rect bounds = DRM_RECT_INIT(0, 0, EXT_WIDTH, EXT_HEIGHT); + struct drm_rect damage; + unsigned int x1, x2, y, y2; + + if (!ext->damage_valid) { + bitmap_fill(ext->current_damage, EXT_TILE_COUNT); + return; + } + bitmap_zero(ext->current_damage, EXT_TILE_COUNT); + drm_atomic_helper_damage_iter_init(&iter, old, ps); + drm_atomic_for_each_plane_damage(&iter, &damage) { + if (!drm_rect_intersect(&damage, &bounds)) + continue; + x1 = damage.x1 / EXT_TILE_SIZE; + x2 = DIV_ROUND_UP(damage.x2, EXT_TILE_SIZE); + y2 = DIV_ROUND_UP(damage.y2, EXT_TILE_SIZE); + for (y = damage.y1 / EXT_TILE_SIZE; y < y2; y++) + bitmap_set(ext->current_damage, y * EXT_TILE_COLS + x1, x2 - x1); + } +} + +static void ext_copy_rect(struct dcpext_drm *ext, struct drm_plane_state *ps, + const struct dcpext_frame *frame, const struct drm_rect *copy) { struct drm_shadow_plane_state *shadow = to_drm_shadow_plane_state(ps); struct drm_framebuffer *fb = ps->fb; - struct drm_rect bounds = DRM_RECT_INIT(0, 0, EXT_WIDTH, EXT_HEIGHT); - struct drm_rect damage, copy; + int x, y; + + for (y = copy->y1; y < copy->y2; y++) { + size_t offset = (size_t)y * fb->pitches[0] + copy->x1 * 4; + size_t bytes = drm_rect_width(copy) * 4; + u8 *dst = frame->pixels + (size_t)y * ext->stride + copy->x1 * 4; + + iosys_map_memcpy_from(ext->row, &shadow->data[0], offset, bytes); + for (x = 0; x < drm_rect_width(copy); x++) + ext->row[x * 4 + 3] = 0xff; + /* Convert in cached memory, then stream to coherent scanout memory. + * Byte stores directly to that mapping make full 4K copies costly. + */ + memcpy(dst, ext->row, bytes); + } +} + +static int ext_copy_frame(struct dcpext_drm *ext, struct drm_plane_state *old, + struct drm_plane_state *ps) +{ struct dcpext_frame frame; - int x, y, ret; + unsigned int row, first, end, bit, next; + int ret; ret = dcpext_scanout_begin_frame(ext->dcp, &frame); if (ret) @@ -141,39 +196,32 @@ static int ext_copy_frame(struct dcpext_drm *ext, struct drm_plane_state *old, * current damage from the current framebuffer, without touching scanout. * Invalid history forces two complete copies, initializing both buffers. */ - if (!ext->damage_valid) - damage = bounds; - else if (!drm_atomic_helper_damage_merged(old, ps, &damage) || - !drm_rect_intersect(&damage, &bounds)) - damage = (struct drm_rect) { 0 }; - copy = damage; - if (ext->damage_valid && drm_rect_visible(&ext->previous_damage)) { - if (!drm_rect_visible(©)) { - copy = ext->previous_damage; - } else { - copy.x1 = min(copy.x1, ext->previous_damage.x1); - copy.y1 = min(copy.y1, ext->previous_damage.y1); - copy.x2 = max(copy.x2, ext->previous_damage.x2); - copy.y2 = max(copy.y2, ext->previous_damage.y2); - } - } - for (y = copy.y1; drm_rect_visible(©) && y < copy.y2; y++) { - size_t offset = (size_t)y * fb->pitches[0] + copy.x1 * 4; - size_t bytes = drm_rect_width(©) * 4; - u8 *dst = frame.pixels + (size_t)y * ext->stride + copy.x1 * 4; + ext_damage_tiles(ext, old, ps); + if (ext->damage_valid) + bitmap_or(ext->copy_damage, ext->current_damage, + ext->previous_damage, EXT_TILE_COUNT); + else + bitmap_copy(ext->copy_damage, ext->current_damage, EXT_TILE_COUNT); + for (row = 0; row < EXT_TILE_ROWS; row++) { + first = row * EXT_TILE_COLS; + end = first + EXT_TILE_COLS; + bit = find_next_bit(ext->copy_damage, end, first); + while (bit < end) { + struct drm_rect copy; - iosys_map_memcpy_from(ext->row, &shadow->data[0], offset, bytes); - for (x = 0; x < drm_rect_width(©); x++) - ext->row[x * 4 + 3] = 0xff; - /* Convert in cached memory, then stream to coherent scanout memory. - * Byte stores directly to that mapping make full 4K copies costly. - */ - memcpy(dst, ext->row, bytes); + next = find_next_zero_bit(ext->copy_damage, end, bit); + copy.x1 = (bit - first) * EXT_TILE_SIZE; + copy.x2 = min((next - first) * EXT_TILE_SIZE, EXT_WIDTH); + copy.y1 = row * EXT_TILE_SIZE; + copy.y2 = min((row + 1) * EXT_TILE_SIZE, EXT_HEIGHT); + ext_copy_rect(ext, ps, &frame, ©); + bit = find_next_bit(ext->copy_damage, end, next); + } } ret = dcpext_scanout_end_frame(ext->dcp, true); /* Ordered atomic commits advance history only after a completed swap. */ if (!ret) { - ext->previous_damage = damage; + bitmap_copy(ext->previous_damage, ext->current_damage, EXT_TILE_COUNT); ext->damage_valid = true; } return ret; diff --git a/tools/testing/selftests/drm/apple/test-dcpext-copy.py b/tools/testing/selftests/drm/apple/test-dcpext-copy.py index 9922207bb53d5..b917413c10724 100644 --- a/tools/testing/selftests/drm/apple/test-dcpext-copy.py +++ b/tools/testing/selftests/drm/apple/test-dcpext-copy.py @@ -29,6 +29,28 @@ def function(start, end): #define EXT_WIDTH DCPEXT_WIDTH #define EXT_HEIGHT DCPEXT_HEIGHT #define EXT_STRIDE DCPEXT_STRIDE +#define DIV_ROUND_UP(n,d) (((n)+(d)-1)/(d)) +#define BITS_PER_LONG (8*sizeof(unsigned long)) +#define DECLARE_BITMAP(name,n) unsigned long name[DIV_ROUND_UP(n,BITS_PER_LONG)] +/* Small host equivalents of the kernel's generic bitmap primitives. */ +static void bitmap_zero(unsigned long *b,unsigned n) { memset(b,0,DIV_ROUND_UP(n,BITS_PER_LONG)*sizeof(*b)); } +static void bitmap_fill(unsigned long *b,unsigned n) { memset(b,255,DIV_ROUND_UP(n,BITS_PER_LONG)*sizeof(*b)); } +static void bitmap_copy(unsigned long *d,const unsigned long *s,unsigned n) { memcpy(d,s,DIV_ROUND_UP(n,BITS_PER_LONG)*sizeof(*d)); } +static void bitmap_or(unsigned long *d,const unsigned long *a,const unsigned long *b,unsigned n) { + for(unsigned i=0;i(b)?(a):(b)) struct drm_rect { int x1,y1,x2,y2; }; @@ -41,16 +63,31 @@ def function(start, end): } struct iosys_map { void *vaddr; }; struct drm_framebuffer { u32 pitches[1]; }; -struct drm_plane_state { struct drm_framebuffer *fb; struct drm_rect damage; bool full; }; +struct drm_plane_state { struct drm_framebuffer *fb; struct drm_rect clips[300]; unsigned count; bool full; }; struct drm_shadow_plane_state { struct drm_plane_state base; struct iosys_map data[1]; }; #define to_drm_shadow_plane_state(p) ((struct drm_shadow_plane_state *)(p)) -/* The core helper supplies clipped/merged damage, or full damage when absent. */ -static bool drm_atomic_helper_damage_merged(const struct drm_plane_state *old, - const struct drm_plane_state *ps,struct drm_rect *out) { - (void)old;*out=ps->full?(struct drm_rect)DRM_RECT_INIT(0,0,EXT_WIDTH,EXT_HEIGHT):ps->damage; - return drm_rect_visible(out); +/* The driver clips each core-supplied rectangle to the fixed plane bounds. */ +struct drm_atomic_helper_damage_iter { const struct drm_plane_state *ps; unsigned next; }; +static void drm_atomic_helper_damage_iter_init(struct drm_atomic_helper_damage_iter *iter, + const struct drm_plane_state *old,const struct drm_plane_state *ps) { + (void)old;iter->ps=ps;iter->next=0; +} +static bool iter_next(struct drm_atomic_helper_damage_iter *iter,struct drm_rect *out) { + if(iter->ps->full) { + if(iter->next++)return false; + *out=(struct drm_rect)DRM_RECT_INIT(0,0,EXT_WIDTH,EXT_HEIGHT);return true; + } + if(iter->next>=iter->ps->count)return false; + *out=iter->ps->clips[iter->next++];return true; } -struct dcpext_drm { void *dcp; u8 *row; u32 stride; struct drm_rect previous_damage; bool damage_valid; }; +#define drm_atomic_for_each_plane_damage(iter,rect) while(iter_next(iter,rect)) +/* TILE_DEFINES */ +struct dcpext_drm { + void *dcp; u8 *row; u32 stride; bool damage_valid; + DECLARE_BITMAP(previous_damage,EXT_TILE_COUNT); + DECLARE_BITMAP(current_damage,EXT_TILE_COUNT); + DECLARE_BITMAP(copy_damage,EXT_TILE_COUNT); +}; struct dcpext_frame { void *pixels; size_t size; }; static struct dcpext_frame back; static bool begin_fails, end_fails, held; @@ -102,8 +139,20 @@ def function(start, end): if(step==13)d=(struct drm_rect)DRM_RECT_INIT(-20,-10,40,30); if(step==14 || step==15)d=(struct drm_rect){0}; if(step==16)d=(struct drm_rect)DRM_RECT_INIT(4000,2300,10,10); - shadow.base.full=step==20;shadow.base.damage=d; - paint(source,reference,pitch,shadow.base.full?full:d,step); + shadow.base.full=step==20;shadow.base.clips[0]=d;shadow.base.count=1; + /* Far-apart regions, tile boundaries, overlap and many clips. The + * old bounding-box copy moves almost a full frame for these corners. */ + if(step>=28) { + shadow.base.count=step==47?300:8; + for(unsigned i=0;i=2 && step<12)assert(copied=2 && step<12) || step>=28)assert(copied Date: Tue, 29 Sep 2026 09:17:09 -0700 Subject: [PATCH 1556/2680] drm/apple: import M3 DCP-ext bring-up modules Eryk's M3 display bring-up lives as a standalone out-of-tree module package under drivers/gpu/drm/apple/tools/m3-dcp/: DCP and DCP-ext RTKit endpoints, the HDMI PHY, bridge and xbar, Type-C probe and PTD voting, and the power, DMA and mode plumbing around them. It has never been integrated here, so the incremental M3 HDMI work on Eryk's eryk_m3_gpu branch had nothing to apply against. This imports the subtree as its baseline. Nothing in the kernel's own build changes. The package carries its own Makefile of `obj-m` targets and is referenced by neither drivers/gpu/drm/apple/Makefile nor its Kconfig, so an ordinary kernel build does not compile it and the in-tree drm/apple DCP driver is untouched. The modules are built deliberately, out of tree, against a configured kernel: make -C /lib/modules/$(uname -r)/build \ M=$PWD/drivers/gpu/drm/apple/tools/m3-dcp modules Imported from Eryk's eryk_m3_gpu branch at the parent of "drm/apple: bound M3 timing parser by publication size". (cherry picked from commit e12006c83d1da9ce20a2a3694a9a2ebab351798d) --- drivers/gpu/drm/apple/tools/m3-dcp/Makefile | 25 + drivers/gpu/drm/apple/tools/m3-dcp/README.md | 180 ++++ .../apple/tools/m3-dcp/dcpext_afk_channel.h | 212 ++++ .../drm/apple/tools/m3-dcp/dcpext_session.c | 343 +++++++ .../drm/apple/tools/m3-dcp/dcpext_session.h | 96 ++ .../drm/apple/tools/m3-dcp/dcpext_syslog.h | 49 + .../drm/apple/tools/m3-dcp/m3_dcp_bridge.c | 842 ++++++++++++++++ .../drm/apple/tools/m3-dcp/m3_dcp_bridge.h | 34 + .../apple/tools/m3-dcp/m3_dcp_brightness.h | 33 + .../apple/tools/m3-dcp/m3_dcp_clock_restore.c | 39 + .../gpu/drm/apple/tools/m3-dcp/m3_dcp_hold.c | 57 ++ .../gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.c | 730 ++++++++++++++ .../gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.h | 9 + .../drm/apple/tools/m3-dcp/m3_dcp_native.c | 739 ++++++++++++++ .../drm/apple/tools/m3-dcp/m3_dcp_native.h | 23 + .../drm/apple/tools/m3-dcp/m3_dcp_parser.c | 39 + .../drm/apple/tools/m3-dcp/m3_dcp_ptd_vote.c | 114 +++ .../gpu/drm/apple/tools/m3-dcp/m3_dcp_query.c | 65 ++ .../gpu/drm/apple/tools/m3-dcp/m3_dcp_regs.c | 62 ++ .../apple/tools/m3-dcp/m3_dcp_rtkit_main.c | 434 +++++++++ .../gpu/drm/apple/tools/m3-dcp/m3_dcp_state.h | 15 + .../gpu/drm/apple/tools/m3-dcp/m3_dcp_vote.c | 37 + .../drm/apple/tools/m3-dcp/m3_dcpext_audio.h | 193 ++++ .../apple/tools/m3-dcp/m3_dcpext_audio_api.h | 15 + .../tools/m3-dcp/m3_dcpext_audio_format.h | 190 ++++ .../apple/tools/m3-dcp/m3_dcpext_audio_pcm.c | 343 +++++++ .../gpu/drm/apple/tools/m3-dcp/m3_dcpext_av.h | 69 ++ .../apple/tools/m3-dcp/m3_dcpext_connector.c | 422 ++++++++ .../apple/tools/m3-dcp/m3_dcpext_connector.h | 68 ++ .../drm/apple/tools/m3-dcp/m3_dcpext_dma.c | 330 +++++++ .../drm/apple/tools/m3-dcp/m3_dcpext_frame.h | 141 +++ .../drm/apple/tools/m3-dcp/m3_dcpext_iboot.h | 114 +++ .../drm/apple/tools/m3-dcp/m3_dcpext_kms.c | 341 +++++++ .../drm/apple/tools/m3-dcp/m3_dcpext_kms.h | 25 + .../tools/m3-dcp/m3_dcpext_kms_adapter.h | 114 +++ .../drm/apple/tools/m3-dcp/m3_dcpext_modes.c | 477 +++++++++ .../drm/apple/tools/m3-dcp/m3_dcpext_modes.h | 54 ++ .../drm/apple/tools/m3-dcp/m3_dcpext_native.c | 780 +++++++++++++++ .../drm/apple/tools/m3-dcp/m3_dcpext_native.h | 27 + .../tools/m3-dcp/m3_dcpext_native_adapter.h | 148 +++ .../tools/m3-dcp/m3_dcpext_native_query.h | 37 + .../drm/apple/tools/m3-dcp/m3_dcpext_power.c | 66 ++ .../drm/apple/tools/m3-dcp/m3_dcpext_regs.c | 123 +++ .../drm/apple/tools/m3-dcp/m3_dcpext_rpc.c | 604 ++++++++++++ .../drm/apple/tools/m3-dcp/m3_dcpext_rpc.h | 54 ++ .../drm/apple/tools/m3-dcp/m3_dcpext_rtkit.c | 3 + .../apple/tools/m3-dcp/m3_dcpext_rtkit_main.c | 2 + .../tools/m3-dcp/m3_dcpext_secondary_main.c | 3 + .../tools/m3-dcp/m3_dcpext_secondary_power.c | 3 + .../apple/tools/m3-dcp/m3_dcpext_session.h | 913 ++++++++++++++++++ .../gpu/drm/apple/tools/m3-dcp/m3_epic_tx.h | 54 ++ .../drm/apple/tools/m3-dcp/m3_hdmi_bridge.c | 85 ++ .../drm/apple/tools/m3-dcp/m3_hdmi_control.h | 25 + .../gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy.c | 192 ++++ .../apple/tools/m3-dcp/m3_hdmi_phy_common.h | 65 ++ .../apple/tools/m3-dcp/m3_hdmi_phy_dp-drive.h | 32 + .../apple/tools/m3-dcp/m3_hdmi_phy_dp-main.h | 159 +++ .../apple/tools/m3-dcp/m3_hdmi_phy_dp-pll.h | 95 ++ .../tools/m3-dcp/m3_hdmi_phy_lane-power.h | 75 ++ .../apple/tools/m3-dcp/m3_hdmi_phy_power.h | 81 ++ .../gpu/drm/apple/tools/m3-dcp/m3_hdmi_xbar.c | 68 ++ .../gpu/drm/apple/tools/m3-dcp/m3_hdmi_xbar.h | 145 +++ .../drm/apple/tools/m3-dcp/m3_typec_probe.c | 92 ++ .../gpu/drm/apple/tools/m3-dcp/m3_usbc_phy.c | 283 ++++++ .../drm/apple/tools/m3-dcp/m3_usbc_ptd_vote.c | 3 + .../drm/apple/tools/m3-dcp/syncobj-probe.py | 84 ++ .../apple/tools/m3-dcp/validation/.gitignore | 11 + .../apple/tools/m3-dcp/validation/README.md | 18 + .../m3-dcp/validation/dcpext_session_test.c | 478 +++++++++ .../validation/panel-dpi-20260921/README.md | 27 + .../validation/panel-dpi-20260921/inspect.py | 19 + .../panel-dpi-20260921/screenshot.sha256 | 1 + 72 files changed, 11928 insertions(+) create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/Makefile create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/README.md create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/dcpext_afk_channel.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/dcpext_session.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/dcpext_session.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/dcpext_syslog.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_bridge.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_bridge.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_brightness.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_clock_restore.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_hold.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_native.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_native.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_parser.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_ptd_vote.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_query.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_regs.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_rtkit_main.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_state.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_vote.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_api.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_format.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_pcm.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_av.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_dma.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_frame.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_iboot.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms_adapter.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_adapter.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_query.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_power.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_regs.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rpc.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rpc.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rtkit.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rtkit_main.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_secondary_main.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_secondary_power.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_epic_tx.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_bridge.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_control.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_common.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-drive.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-main.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-pll.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_lane-power.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_power.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_xbar.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_xbar.h create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_typec_probe.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_usbc_phy.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_usbc_ptd_vote.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/syncobj-probe.py create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/validation/.gitignore create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/validation/README.md create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/validation/dcpext_session_test.c create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/README.md create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/inspect.py create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/screenshot.sha256 diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/Makefile b/drivers/gpu/drm/apple/tools/m3-dcp/Makefile new file mode 100644 index 0000000000000..817e86bbdc95a --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/Makefile @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +obj-m += m3_dcp_rtkit.o +m3_dcp_rtkit-y := m3_dcp_rtkit_main.o m3_dcp_bridge.o m3_dcp_native.o m3_dcp_kms.o m3_dcp_parser.o +obj-m += m3_dcp_regs.o +obj-m += m3_dcp_hold.o +obj-m += m3_dcp_vote.o +obj-m += m3_dcp_ptd_vote.o +obj-m += m3_dcp_clock_restore.o +obj-m += m3_dcp_query.o +obj-m += m3_dcpext_regs.o +obj-m += m3_dcpext_power.o +obj-m += m3_dcpext_dma.o +obj-m += m3_dcpext_rtkit.o +m3_dcpext_rtkit-y := m3_dcpext_rtkit_main.o m3_dcpext_rpc.o m3_dcpext_native.o m3_dcpext_modes.o m3_dcpext_kms.o m3_dcpext_connector.o +obj-m += m3_hdmi_phy.o +obj-m += m3_hdmi_bridge.o +obj-m += m3_hdmi_xbar.o +obj-m += m3_typec_probe.o +obj-m += m3_usbc_ptd_vote.o +obj-m += m3_dcpext_secondary_power.o +obj-m += m3_dcpext_secondary.o +m3_dcpext_secondary-y := m3_dcpext_secondary_main.o m3_dcpext_rpc.o m3_dcpext_native.o m3_dcpext_modes.o m3_dcpext_kms.o m3_dcpext_connector.o +obj-m += m3_usbc_phy.o + +obj-m += m3_dcpext_audio_pcm.o diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/README.md b/drivers/gpu/drm/apple/tools/m3-dcp/README.md new file mode 100644 index 0000000000000..831daef79f940 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/README.md @@ -0,0 +1,180 @@ +# J514S native DCP checkpoint + +Explicit-load native display prototype for the M3 Pro J514S, using the inherited +DCP firmware UUID DDF38191-93B3-324A-BC8F-643006F5AC82. The corresponding captured +TEXT SHA256 is a0af36d10a8abf2435918ba360e697529ddfaed56b89f9466e76d566b296e876. +No firmware image is included here. Do not generalize to another M3 or firmware. + +The isolated m1n1 handoff reserves every DCP firmware segment and publishes the +locked display SID0 and PIODMA SID4 mappings. The power-hold helper uses PMGR's +existing syscon to retain DISP_SYS/FE/CPU before PMP startup; forcing CPU_RUN +on a suspended DCP is not a supported recovery method. Reset by reboot. + +Build against the configured UNSTABLE kernel with ARCH=arm64 LLVM=1 and this +directory as M=. Requires DRM_GEM_DMA_HELPER, BACKLIGHT_CLASS_DEVICE and the normal Apple RTKit, DART, +mailbox and PMP providers. A partial kernel build may leave Module.symvers +without the already-built drm_dma_helper exports; restore that dependency's +symbol metadata before modpost. Do not bypass missing-symbol checks. + +Load m3_dcp_hold before the M3 PMP overlay/thermal/bridge modules. Under the M3 +watchdog/thermal guard, load drm_dma_helper and backlight, then m3_dcp_rtkit kernel_client=1. +After successful startup, root may write background or kms to the control +parameter. KMS takeover removes simpledrm and exposes one native3024x1964 eDP +mode. Stop the display manager before a manual takeover. No automatic probe or +module unload: firmware-visible allocations remain pinned until reboot. + +Hardware evidence on2026-09-18 includes successful service initialization, +solid-background completion, three real KMS flips over three prefilled buffers, +all5,939,136 expected snapshot pixels, and the normal Plasma greeter on native +DCP after allowing llvmpipe's3072x1984 padded allocation. The visible framebuffer +and plane must remain3024x1964. Preferred timing metadata says120Hz; the bounded +flip test measured60FPS. A host snapshot is not optical panel readback. + +This is a fixed-mode display checkpoint, not a full accelerated-desktop claim. +GPU rendering qualification is separate. Linear XRGB/ARGB only; display domains +are held active, runtime display power management remains unqualified, and the +initial firmware panel charge-parity diagnostic is retained in evidence. A +plane disable blanks to black; panel power-off has not been qualified. + +Workspace tools/m3-dcp contains the guarded bootstrap, persistent recovery latch, +offline firmware decoders, lifecycle capture and userspace flip/login checks. +Evidence: _provenance/m3-dcp-20260918 in the source-collection workspace. + +## Visible panel fix and reboot validation (2026-09-18) + +The earlier checkpoint established software completions and host-buffer pixels, +but the user reported a dark physical panel. PMP startup alone caused the +blackout. The legacy device-power mailbox only returns a constant ACK on this +PMP v2 firmware; it does not establish the requested DISP power state. + +Captured macOS PTD state and driver analysis identified request index280 and +ACK index284. The loader now optionally seeds DISP bit7 and ANS bit16 through +an ioremap_np mapping before starting PMP. DISP requires ACK; ANS does not. +The native client verifies status1 and the DISP request/ACK before takeover. +Use the recorded lifecycle --hold --ptd-seed sequence, not the old mailbox. + +The user confirmed visible native3024x1964 login output both after takeover +and after a normal UNSTABLE reboot. Guarded login checks observed new frames +and zero reported failures; all five PMP channels remained below47C. This is +visible internal-display support, not accelerated-desktop qualification. + +The exact bootstrap/diagnostic source snapshot is in +../../asahi/m3_lab/tools/m3-dcp; the PMP loader is in its sibling +m3-boot-export. Snapshot hashes and validation summaries are under m3_lab. +Runtime deployment remains /root/m3-gpu-tools: lifecycle.py is installed as +m3-dcp-lifecycle.py and start.py as m3-dcp-start.py; boot.py, boot-client.py, +control.py and m3-dcp.service retain their names. The existing guard and +watchdog helpers are required. Preserve the failure latch, normal display +manager, original Asahi fallback and five-second GRUB menu. + +## Panel dimensions (2026-09-21) + +The J514S firmware's serialized DisplayAttributes contains product identity but +no image dimensions. The board's DCP panel child therefore supplies width-mm=302 +and height-mm=196, following the existing M1/M2 panel binding. These are the +full 3024x1964 panel dimensions at Apple's specified 254 ppi, rounded to whole +millimetres (https://support.apple.com/en-ie/117736). Do not use the notch-hidden +189 mm height with full-height native scanout. + +KMS copies these dimensions into its preferred mode and DRM connector display +information on each mode probe. An old DT without panel data retains display +operation and emits a warning rather than fabricating dimensions or an EDID. +There is no desktop scale in the driver. Deploy both the module and updated +J514S DT in the m1n1 bundle. tools/m3-dcp/prepare-boot.py preserves the other +40 boards and U-Boot; verify the installed bundle before replacement. + +KWin preserves previously saved output configurations. Removing just the scale +key from an existing setup did not trigger automatic scale selection: KWin +retained 1x. To validate automatic selection on this single-panel installation, +stop the display manager, terminate the user session, wait for KWin to exit, +then archive kwinoutputconfig.json before restarting. Retain the archive for +restoring other output preferences; do not discard multi-monitor layouts. + +After a reboot into the corrected driver, DRM and KWin both reported 302x196 mm. +With the old setup archived, unmodified KWin 6.7.4 selected 1.7x automatically +(logical desktop 1779x1155) and saved that choice. No replacement scale was +supplied. KWin's current heuristic uses a 150-DPI target for internal outputs +without a detected laptop lid switch; its laptop-with-lid target is 125 DPI. +This distinction is userspace/input policy, not a reason to alter panel size. +The guarded real Plasma check, GPU rendering and screenshot passed. Evidence: +validation/panel-dpi-20260921 (raw screenshot retained in workspace build). + +## DRM timeline synchronization (2026-09-22) + +Internal and external prototype DRM devices now expose the core SYNCOBJ and +SYNCOBJ_TIMELINE features, as the main apple_drv.c already does. A display-only +node can import fences from the separate render node; it needs no local shader +engine for DRM core to implement timelines. Without these flags, KWin on J514S +did not advertise wp_linux_drm_syncobj_manager_v1. After the guarded normal +boot, it does, and the Vulkan game confirms explicit=1 instead of implicit sync. + +`syncobj-probe.py DISPLAY_NODE RENDER_NODE` is CPU-only: it checks capability +queries, exports/imports an opaque timeline in both directions, verifies timeout +before signaling point 7, then successful wait and eventfd notification. It +does not submit GPU work or count as GPU synchronization qualification. The +J514S card1/renderD128 probe passes both directions. Native Vulkan 0 A.D. runs +with real explicit-sync presentation under the normal M3 guard, at 98.102 FPS +in the short fixed-scene sample. The previous implicit diagnostic was 94.901; +this is not a matched sustained performance claim and remains below 120 FPS. + +Boot: cd0c4c47-1206-435b-a4b5-b33f8f3a1e73. Private evidence: +`build/m3-0ad-120/dcp-syncobj/` and `0ad-120-vk-explicit-sync/`. External module +variants build, but no external monitor is attached in this qualification. + +## Zero-nit internal-panel recovery (2026-09-24) + +The recurrent internal-dark/external-working state was retained on boot +`ab566889-62ce-4df2-b5d1-63ab03423a80`. KMS was active and completing frames, +while the qualified firmware snapshot reported backlight enabled, correction +1.0, and `last_backlight_nits = 0`. Direct writes to cached brightness fields +were ineffective; they are not a supported interface and were restored. + +Firmware analysis identified the M3 A407 brightness-update byte at 0x354 and +unaligned binary64 nits at 0x35c. M4 uses 0x35e for the latter. The M3 firmware +copies these to transaction+0x76a/+0x778 and converts nits into runtime property +19 through its normal backlight pipeline. A serialized A406/A407 transaction +requesting 140 nits completed successfully (swap 33762) and a subsequent RAM +snapshot reported 140 nits. There was no reboot, DCP reload or desktop restart. +The user subsequently confirmed both screens visible: the internal panel +recovered and the external monitor remained working. This is optical +confirmation of the live recovery, not validation of the unloaded replacement +driver or of repeated wake/hotplug cycles. + +The normal driver now exposes `apple-panel-bl` through the backlight class, +with an SDR range of 0..500 nits and an initial request of 140 nits. It sends +zero on CRTC disable and the retained request on enable. Explicit writes to +the backlight device re-send the request even if its value is unchanged, and +work with an idle compositor by retaining and re-submitting the active FB. +Power-on/CRTC-enable invalidate the last acknowledged brightness; only a +completed, successful swap updates that cache. Background-only diagnostic +commands retain the existing brightness. No raw RAM writes are part of this +implementation. + +Validation so far: the live 0-to-140 command above, an arm64 module build for +a 7.1.9 Asahi development kernel, and the actual production encoder tested against the +host IEEE754 implementation for all 501 levels, including unchanged bytes +outside the two wire fields, under ASan/UBSan. Run the offline test with: + +``` +python3 drivers/gpu/drm/asahi/m3_lab/tools/m3-dcp/test-brightness.py \ + drivers/gpu/drm/apple/tools/m3-dcp +``` + +The complete replacement driver has not been loaded in the preserved boot. +Backlight-class integration, repeated DPMS cycles and hotplug still need a +subsequent hardware validation; the initiating firmware power event has not +yet been identified. Do not describe this checkpoint as a fully qualified +wake/hotplug regression fix. + +One temporary diagnostic had an unrelated host-kernel BTI oops before its +first RPC: a private static driver function was called indirectly without a +BTI landing pad. Its abandoned mutex blocked internal submissions for about +422 seconds. The dead owner's address was verified from the register dump; +a later idle worker had reused that task allocation. A same-boot, pointer- +and birth-time-qualified recovery released that abandoned lock; ordinary KMS +submissions resumed. The successful diagnostic used a range-checked direct +branch. Neither diagnostic nor manual mutex recovery belongs in the driver. +The failed module remains pinned and the kernel is tainted until a future +reboot; no forced unload or reboot was attempted. All raw captures, temporary +helper sources/binaries and logs remain local under `build/m3-panel-wake` and +`tools/m3-dcp/live-brightness` in the source-collection workspace. diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_afk_channel.h b/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_afk_channel.h new file mode 100644 index 0000000000000..309fe5320d3d5 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_afk_channel.h @@ -0,0 +1,212 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Bounded additional AFK channel, independent state for concurrent services. + * Adapted from this probe's Asahi-derived afk_probe ring/handshake code. + * Authors/references: Sven Peter, Asahi Linux Contributors; dynamic ring + * block layout from Eileen Yoon / m1n1 rbep.py. No hardware ownership here: + * caller retains every allocation until ALL endpoints and RTKit quiesce. + * Single serialized caller; callbacks must not reenter this engine. + */ +#ifndef DCPEXT_AFK_CHANNEL_H +#define DCPEXT_AFK_CHANNEL_H +/* Announcement-only M3 copy: the unused M4 ring writer is omitted. */ +struct afkc; +struct afkc_ops { + int (*send)(void *cookie, u8 endpoint, u64 message); + int (*allocate)(void *cookie, u32 size, u8 **buffer, u64 *dva); + int (*record)(void *cookie, const u8 *payload, u32 size); +}; +struct afkc_ring { + u8 *header, *data; + u32 size, block, offset, extent; +}; +struct afkc { + const struct afkc_ops *ops; + void *cookie; + u8 endpoint; + bool started, running, stopping, stopped; + bool streaming; /* Desktop owner accounts traffic across endpoints. */ + int error; + u32 messages, buffer_size, tag, capture_size; + u32 max_payload; /* Zero preserves the 256-byte default; explicit maximum4096. */ + u8 *buffer; + struct afkc_ring rings[2]; + u8 capture[32768]; +}; + +static int afkc_fail(struct afkc *c, int error) +{ + if (!c->error) + c->error = error; + return c->error; +} + +static int afkc_send(struct afkc *c, u32 type, u64 payload) +{ + int ret = c->ops->send(c->cookie, c->endpoint, (u64)type << 48 | payload); + return ret ? afkc_fail(c, ret) : 0; +} + +static int afkc_start(struct afkc *c, const struct afkc_ops *ops, void *cookie, u8 ep) +{ + int ret; + if (c->started || !ops || !ops->send || !ops->allocate || !ops->record || + ep < 0x20 || ep > 0x3f) + return -EINVAL; + c->ops = ops; + c->cookie = cookie; + c->endpoint = ep; + c->started = true; /* Published start may take effect even on send error. */ + ret = ops->send(cookie, 0, (5ULL << 52) | ((u64)ep << 32) | 2); + return ret ? afkc_fail(c, ret) : afkc_send(c, 0x80, 0); +} + +static bool afkc_tx_empty(struct afkc *c) +{ + struct afkc_ring *r = &c->rings[0]; + u32 rd, wr; + if (!r->header) + return false; + rd = READ_ONCE(*(u32 *)(r->header + r->block)); + wr = READ_ONCE(*(u32 *)(r->header + 2 * r->block)); + dma_rmb(); + return rd < r->size && !(rd & (r->block - 1)) && rd == wr; +} + +static int afkc_drain(struct afkc *c) +{ + struct afkc_ring *r = &c->rings[1]; + u32 rd, wr, bytes, total; + u8 *qe; + int ret; + if (!c->running || !r->header) + return -EPROTO; + for (u32 n = 0; n < 64; n++) { + rd = READ_ONCE(*(u32 *)(r->header + r->block)); + wr = READ_ONCE(*(u32 *)(r->header + 2 * r->block)); + dma_rmb(); + if (rd >= r->size || wr >= r->size || (rd | wr) & (r->block - 1)) + return -EPROTO; + if (rd == wr) + return 0; + qe = r->data + rd; + if (get_unaligned_le32(qe) != 0x20504f49) + return -EPROTO; + bytes = get_unaligned_le32(qe + 4); + if (bytes > r->size - 16) + return -EPROTO; + if (bytes + 16 > r->size - rd) { + if (wr >= rd || !wr) + return -EPROTO; + rd = 0; + qe = r->data; + if (get_unaligned_le32(qe) != 0x20504f49) + return -EPROTO; + bytes = get_unaligned_le32(qe + 4); + } + if (bytes < 8 || bytes > r->size - rd - 16) + return -EPROTO; + total = ALIGN(bytes + 16, r->block); + if (c->streaming && bytes + 12 > sizeof(c->capture) - c->capture_size) + c->capture_size = 0; + if (total > r->size - rd || (rd < wr && total > wr - rd) || + bytes + 12 > sizeof(c->capture) - c->capture_size) + return -EOVERFLOW; + /* Preserve the existing offline capture framing. */ + memcpy(c->capture + c->capture_size, qe + 8, 8); + put_unaligned_le32(bytes, c->capture + c->capture_size + 8); + memcpy(c->capture + c->capture_size + 12, qe + 16, bytes); + c->capture_size += bytes + 12; + ret = c->ops->record(c->cookie, qe + 16, bytes); + rd += total; + if (rd == r->size) + rd = 0; + dma_mb(); + WRITE_ONCE(*(u32 *)(r->header + r->block), rd); + if (ret) + return ret; + } + return -E2BIG; +} + +static int afkc_receive(struct afkc *c, u64 message) +{ + u32 type = message >> 48, size, offset, index, data_size, block; + u64 dva; + struct afkc_ring *r, *other; + int ret = 0; + if (c->error) + return c->error; + if (!c->started || c->stopped || (!c->streaming && ++c->messages > 128)) + return afkc_fail(c, -EPROTO); + switch (type) { + case 0xa0: + case 0x8c: + break; + case 0x89: + size = ((message >> 16) & 0xffff) * 64; + if (c->buffer || c->stopping || !size || size > 65536) + return afkc_fail(c, -EINVAL); + ret = c->ops->allocate(c->cookie, size, &c->buffer, &dva); + if (ret) + break; + c->buffer_size = size; + c->tag = message & 0xffff; + if (!c->buffer || dva < (1ULL << 40) || + dva - (1ULL << 40) >= (1ULL << 36) || dva & 0x3fff) + return afkc_fail(c, -EINVAL); + ret = afkc_send(c, 0xa1, dva); + break; + case 0x8a: + case 0x8b: + index = type == 0x8a ? 0 : 1; + r = &c->rings[index]; + other = &c->rings[!index]; + offset = ((message >> 32) & 0xffff) * 64; + size = ((message >> 16) & 0xffff) * 64; + if (!c->buffer || r->header || c->stopping || size < 256 || + (message & 0xffff) != c->tag || offset >= c->buffer_size || + size > c->buffer_size - offset || (other->header && + offset < other->offset + other->extent && other->offset < offset + size)) + return afkc_fail(c, -EINVAL); + data_size = READ_ONCE(*(u32 *)(c->buffer + offset)); + if (data_size >= size || (size - data_size) % 3) + return afkc_fail(c, -EPROTO); + block = (size - data_size) / 3; + if (block < 64 || block > 256 || !is_power_of_2(block) || + data_size < block || data_size % block) + return afkc_fail(c, -EPROTO); + *r = (struct afkc_ring){.header = c->buffer + offset, + .data = c->buffer + offset + 3 * block, .size = data_size, + .block = block, .offset = offset, .extent = size}; + if (other->header) + ret = afkc_send(c, 0xa3, 0); + break; + case 0x86: + if (!c->rings[0].header || !c->rings[1].header || c->running || c->stopping) + return afkc_fail(c, -EPROTO); + c->running = true; + break; + case 0x85: + ret = afkc_drain(c); + break; + case 0xc1: + if (!c->stopping || !afkc_tx_empty(c)) + return afkc_fail(c, -EPROTO); + c->stopped = true; + break; + default: + ret = -EOPNOTSUPP; + } + return ret ? afkc_fail(c, ret) : 0; +} + +static int afkc_stop(struct afkc *c) +{ + if (c->error) + return c->error; + if (!c->running || c->stopping || !afkc_tx_empty(c)) + return -EAGAIN; + c->stopping = true; + return afkc_send(c, 0xc0, 0); +} +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_session.c b/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_session.c new file mode 100644 index 0000000000000..a3c5a468bc3ab --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_session.c @@ -0,0 +1,343 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* See dcpext_session.h for attribution and the hardware adapter contract. */ +#include "dcpext_session.h" +#ifdef __KERNEL__ +#include +#else +#include +#endif + +#define MSG(type) ((u64)(type) << 52) +#define SYSTEM_PROFILE 0x11fU /* J514S 14.6 session-1 capture, 2026-09-21 */ +#define SESSION_MS 8000 +#define MAX_MESSAGES 256 +#define DVA_SIZE (1ULL << 36) /* Two 11-bit levels plus 14-bit page offset */ +#define DVA_BASE (1ULL << 40) /* M4 firmware wire address, not a table index */ + +static bool valid_buffer_address(unsigned int ep, bool inherited, u64 addr, u32 size) +{ + u64 base = DVA_BASE, limit = DVA_SIZE; + + if (ep == 8 && inherited) { + /* OSLog can report a host physical address; the callback must + * qualify the exact OS-log reservation before accepting it. + */ + base = 0; + limit = 1ULL << 42; + } + return addr && !(addr & 0xfff) && addr >= base && + addr - base < limit && size <= limit - (addr - base); +} + +static int fail(struct dcpext_session *s, int error) +{ + if (!s->error) + s->error = error < 0 ? error : -EIO; + s->phase = DCPEXT_FAILED; + return s->error; +} + +int dcpext_session_poll(struct dcpext_session *s) +{ + u64 now; + + if (s->phase == DCPEXT_FAILED) + return s->error; + if (!s->ops || s->phase == DCPEXT_NEW) + return -EINVAL; + if (s->phase == DCPEXT_QUIESCED) + return 0; + now = s->ops->now_ms(s->cookie); + if (now < s->last_time) + return fail(s, -ERANGE); + s->last_time = now; + if (s->runtime) { + if (s->phase != DCPEXT_RUNNING) + return fail(s, -EPROTO); + if (now - s->runtime_window >= 1000) { + s->runtime_window = now; + s->runtime_messages = 0; + } + } else if (now >= s->deadline) + return fail(s, -ETIMEDOUT); + return 0; +} + +int dcpext_session_enter_runtime(struct dcpext_session *s) +{ + int ret = dcpext_session_poll(s); + + if (ret) + return ret; + if (s->phase != DCPEXT_RUNNING || s->runtime) + return -EINVAL; + s->runtime = true; + s->runtime_window = s->last_time; + s->runtime_messages = 0; + return 0; +} + +int dcpext_session_runtime_message(struct dcpext_session *s) +{ + int ret = dcpext_session_poll(s); + + if (ret) + return ret; + if (s->runtime && ++s->runtime_messages > 8192) + return fail(s, -EOVERFLOW); + return 0; +} + +int dcpext_session_leave_runtime(struct dcpext_session *s) +{ + int ret = dcpext_session_poll(s); + + if (ret) + return ret; + if (!s->runtime || s->last_time > ~(u64)0 - SESSION_MS) + return -EINVAL; + s->runtime = false; + s->deadline = s->last_time + SESSION_MS; + s->received = 0; + return 0; +} + +static int send(struct dcpext_session *s, u8 ep, u64 msg) +{ + int ret = dcpext_session_poll(s); + + if (ret) + return ret; + /* A failed transport return does not establish that delivery failed. */ + s->touched = true; + ret = s->ops->send(s->cookie, ep, msg); + if (ret) + return fail(s, ret); + return dcpext_session_poll(s); +} + +int dcpext_session_begin(struct dcpext_session *s, + const struct dcpext_session_ops *ops, void *cookie) +{ + u64 now, duration = s->duration_ms ? s->duration_ms : SESSION_MS; + + if (duration > 60000 || duration < SESSION_MS) + return -EINVAL; + if (s->message_limit && + !((s->message_limit == 512 && (duration == 16000 || duration == 60000)) || + (s->message_limit == 8192 && duration == 16000))) + return -EINVAL; + if (s->phase != DCPEXT_NEW || s->touched || s->ops) + return -EBUSY; + if (!ops || !ops->send || !ops->buffer || !ops->now_ms) + return -EINVAL; + now = ops->now_ms(cookie); + if (now > ~(u64)0 - duration) + return -ERANGE; + s->ops = ops; + s->cookie = cookie; + s->last_time = now; + s->deadline = now + duration; + s->phase = DCPEXT_HELLO; + /* Linux apple_rtkit_wake and m1n1 rtkit_boot request INIT (0x220), + * including the reinitialization bit, and wait for ON (0x20). + * ON alone wakes an inherited session but does not request this reset + * of protocol state. Never issue CPU reset/run register writes here. + */ + return send(s, 0, MSG(6) | 0x220); +} + +static int ap_on_if_ready(struct dcpext_session *s) +{ + if (!s->epmap_done || !s->iop_on) + return 0; + s->phase = DCPEXT_AP_ON; + return send(s, 0, MSG(0xb) | 0x20); +} + +static int management(struct dcpext_session *s, u64 msg) +{ + unsigned int type = (msg >> 52) & 0xff; + unsigned int base, ep, state = msg & 0xffff; + u32 bitmap; + bool last; + int ret; + + switch (type) { + case 1: + if (s->phase != DCPEXT_HELLO || s->hello || + (msg & 0xffff) > 12 || ((msg >> 16) & 0xffff) < 12) + return fail(s, -EPROTO); + s->hello = true; + s->phase = DCPEXT_EPMAP; + return send(s, 0, MSG(2) | (12 << 16) | 12); + case 8: + base = (msg >> 32) & 7; + bitmap = (u32)msg; + last = (msg >> 51) & 1; + if (s->phase != DCPEXT_EPMAP || !s->hello || + (s->bases & (1U << base))) + return fail(s, -EPROTO); + s->bases |= 1U << base; + s->endpoints[base] = bitmap; + if ((!base && bitmap != SYSTEM_PROFILE) || + (last && !(s->bases & 1))) + return fail(s, -EOPNOTSUPP); + ret = send(s, 0, MSG(8) | ((u64)base << 32) | + (last ? 1ULL << 51 : 1)); + if (ret || !last) + return ret; + s->epmap_done = true; + s->phase = DCPEXT_IOP_ON; + for (ep = 1; ep < 32; ep++) { + if (!(SYSTEM_PROFILE & (1U << ep))) + continue; + ret = send(s, 0, MSG(5) | ((u64)ep << 32) | 2); + if (ret) + return ret; + s->started |= 1U << ep; + } + return ap_on_if_ready(s); + case 7: + if (state == 0x20 && s->hello && !s->iop_on && + (s->phase == DCPEXT_EPMAP || s->phase == DCPEXT_IOP_ON)) { + s->iop_on = true; + return ap_on_if_ready(s); + } + if (state == (s->sleep_iop ? 1 : 0x10) && s->phase == DCPEXT_IOP_QUIESCE) { + s->phase = DCPEXT_QUIESCED; + return 0; + } + return fail(s, -EPROTO); + case 0xb: + if (state == 0x20 && s->phase == DCPEXT_AP_ON) { + s->phase = DCPEXT_RUNNING; + return 0; + } + if (state == 0x10 && s->phase == DCPEXT_AP_QUIESCE && !s->ap_quiesced) { + s->ap_quiesced = true; + return s->defer_iop_quiesce ? 0 : dcpext_session_finish_quiesce(s); + } + return fail(s, -EPROTO); + default: + return fail(s, -EOPNOTSUPP); + } +} + +static int get_buffer(struct dcpext_session *s, unsigned int ep, u64 msg) +{ + struct dcpext_buffer *b = &s->buffers[ep]; + u64 requested, dva = 0, reply; + u32 size; + int ret; + + if (b->ready) + return fail(s, ep == 1 ? -EIO : -EPROTO); /* EP1 repeat = crash */ + if (ep == 8) { + size = (msg >> 36) & 0xfffff; + requested = (msg & ((1ULL << 36) - 1)) << 12; + } else { + size = ((msg >> 44) & 0xff) << 12; + requested = msg & ((1ULL << 44) - 1); + } + if (!size || size > 0x100000 || + (requested && !valid_buffer_address(ep, true, requested, size))) + return fail(s, -EINVAL); + ret = s->ops->buffer(s->cookie, ep, requested, size, &dva); + if (ret) + return fail(s, ret); + /* Record ownership before any further operation can fail. */ + b->dva = dva; + b->size = size; + b->inherited = requested != 0; + b->ready = true; + ret = dcpext_session_poll(s); + if (ret) + return ret; + if (!valid_buffer_address(ep, requested != 0, dva, size) || + (requested && dva != requested)) + return fail(s, -EINVAL); + /* M4 firmware-owned buffers are validated without a host reply. */ + if (requested) + return 0; + if (ep == 8) + reply = (1ULL << 56) | ((u64)size << 36) | (dva >> 12); + else + reply = MSG(1) | ((u64)(size >> 12) << 44) | dva; + return send(s, ep, reply); +} + +int dcpext_session_receive(struct dcpext_session *s, unsigned int ep, u64 msg) +{ + unsigned int type; + int ret = dcpext_session_poll(s); + + if (ret) + return ret; + if (s->phase == DCPEXT_QUIESCED || (!s->runtime && + ++s->received > (s->message_limit ? s->message_limit : MAX_MESSAGES))) + return fail(s, -EPROTO); + if (!ep) + return management(s, msg); + /* This diagnostic never starts any application endpoint. */ + if (ep >= 32 || !(s->started & (1U << ep))) + return fail(s, -EPROTO); + type = ep == 8 ? msg >> 56 : (msg >> 52) & 0xff; + if ((ep == 1 || ep == 2 || ep == 4 || ep == 8) && type == 1) + return get_buffer(s, ep, msg); + /* AppleDCP 1041.120.7 transport at image 0x34c94 consumes a type-2 + * reply as the log ring's read counter and clears its outstanding bit. + * Without this reply the AP power task waits forever in 0x34708. + * Discard the log contents, acknowledging precisely the notified counter; + * it is not an address, byte extent, or proof of display/DMA retirement. + * Only channel zero has a qualified buffer in this session engine. + */ + if (ep == 8) { + s->oslog_notices++; + if (type == 2) { + if (!s->buffers[8].ready || (msg & 0x00ffffff00000000ULL)) + return fail(s, -EPROTO); + return send(s, ep, msg); + } + /* Types 3/4/5 describe log metadata and require no reply. */ + return 0; + } + /* System messages are serviced during both boot and shutdown. Syslog + * payload is deliberately not dereferenced by this diagnostic. + */ + if (ep == 2 && type == 8) + return 0; + if ((ep == 2 && type == 5) || (ep == 4 && (type == 8 || type == 0xc))) + return send(s, ep, msg); + return fail(s, -EOPNOTSUPP); +} + +int dcpext_session_quiesce(struct dcpext_session *s) +{ + int ret = dcpext_session_poll(s); + + if (ret) + return ret; + /* Reject the old endpoint-only attempt without sending IOP quiesce. */ + if (s->phase != DCPEXT_RUNNING) + return -EAGAIN; + if (s->runtime) { + ret = dcpext_session_leave_runtime(s); + if (ret) + return ret; + } + s->phase = DCPEXT_AP_QUIESCE; + return send(s, 0, MSG(0xb) | 0x10); +} + +int dcpext_session_finish_quiesce(struct dcpext_session *s) +{ + int ret = dcpext_session_poll(s); + + if (ret) + return ret; + if (s->phase != DCPEXT_AP_QUIESCE || !s->ap_quiesced) + return -EAGAIN; + s->phase = DCPEXT_IOP_QUIESCE; + return send(s, 0, MSG(6) | (s->sleep_iop ? 1 : 0x10)); +} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_session.h b/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_session.h new file mode 100644 index 0000000000000..c263bbd6a16c5 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_session.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Transport-independent RTKit system-session diagnostic. No MMIO or DMA API. + * Protocol reference: Asahi Linux Contributors, drivers/soc/apple/rtkit.c + * in linux/linux-displayport at base fba91e5d9577 (see provenance manifest). + * This component starts no application endpoints and cannot read EDID itself. + */ +#ifndef DCPEXT_SESSION_H +#define DCPEXT_SESSION_H + +#ifdef __KERNEL__ +#include +#else +#include +#include +typedef uint8_t u8; +typedef uint32_t u32; +typedef uint64_t u64; +#endif + +enum dcpext_phase { + DCPEXT_NEW, DCPEXT_HELLO, DCPEXT_EPMAP, DCPEXT_IOP_ON, + DCPEXT_AP_ON, DCPEXT_RUNNING, DCPEXT_AP_QUIESCE, + DCPEXT_IOP_QUIESCE, DCPEXT_QUIESCED, DCPEXT_FAILED, +}; + +struct dcpext_buffer { + u64 dva; + u32 size; + bool ready, inherited; +}; + +struct dcpext_session_ops { + /* Single serialized caller. All callbacks must be bounded, non-reentrant. + * send must supply the DMA write barrier before mailbox publication. + * recv's caller supplies the DMA read barrier after mailbox receipt. + */ + u64 (*now_ms)(void *cookie); + int (*send)(void *cookie, u8 ep, u64 msg); + /* Nonzero requested address: validate the ENTIRE inherited mapping + * against live reserved RAM; do not clear/rewrite firmware-owned memory. + * M4 DART wire addresses retain their 1-TiB bias. EP8 may instead name + * the physical OS-log carveout: accept only an exact live reservation, + * never a generic physical-address fallback or an invented DART map. + * Zero DVA: obtain a zeroed, mapped host buffer of at least size bytes. + * Retain ownership even on errors: the engine never frees anything. + * This callback neither sends messages nor starts firmware execution. + */ + int (*buffer)(void *cookie, u8 ep, u64 requested, u32 size, u64 *dva); +}; + +struct dcpext_session { + const struct dcpext_session_ops *ops; + void *cookie; + enum dcpext_phase phase; + int error; + u64 deadline, last_time; + u32 duration_ms; /* 0=8s; explicit bounded image observation, at most60s. */ + u32 message_limit; /* 0=256; native+AV 512; bounded 16s KMS 8192. */ + u32 endpoints[8]; + u32 started; + u8 bases; + unsigned int received; + bool runtime; /* Explicit post-startup desktop state; RPCs keep their deadlines. */ + u64 runtime_window; + u32 runtime_messages; + bool hello, epmap_done, iop_on, touched; + bool sleep_iop; /* Linux shutdown: AP quiesced, then restartable IOP sleep. */ + bool defer_iop_quiesce, ap_quiesced; + unsigned int oslog_notices; + struct dcpext_buffer buffers[9]; +}; + +/* Zero-initialize a new instance before begin; never reinitialize a live one. + * begin admits only the system endpoint profile seen on J514S 14.6 DCPEXT. + * The adapter must separately qualify the firmware UUID and DART handoff. + */ +int dcpext_session_begin(struct dcpext_session *s, + const struct dcpext_session_ops *ops, void *cookie); +int dcpext_session_poll(struct dcpext_session *s); +int dcpext_session_enter_runtime(struct dcpext_session *s); +int dcpext_session_leave_runtime(struct dcpext_session *s); +/* Adapter accounts every mailbox endpoint, including application traffic. */ +int dcpext_session_runtime_message(struct dcpext_session *s); +int dcpext_session_receive(struct dcpext_session *s, unsigned int ep, u64 msg); +int dcpext_session_quiesce(struct dcpext_session *s); +/* Optional ordering experiment: only after AP ACK, let the owner stop its + * application channels while IOP remains awake, then request IOP quiescence. + */ +int dcpext_session_finish_quiesce(struct dcpext_session *s); + +/* QUIESCED means both protocol ACKs arrived. It is NOT a cleanup function or + * proof of real hardware quiescence. Before freeing, the hardware adapter must + * also stop/drain its transport and complete DART removal/invalidation. + * FAILED is sticky; late ACKs cannot permit freeing possibly reachable RAM. + */ +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_syslog.h b/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_syslog.h new file mode 100644 index 0000000000000..bae0e9ad1d0a5 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_syslog.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Capture RTKit syslog slots before acknowledgement allows firmware reuse. + * Format/offsets: Asahi Linux Contributors, drivers/soc/apple/rtkit.c, + * apple_rtkit_syslog_rx_init/log at kernel fba91e5d9577. + * This helper only reads a caller-qualified host buffer. No address mapping, + * firmware commands or string interpretation. Output records are private: + * little-endian u64 mailbox message, u32 slot size, then the complete slot. + */ +#ifndef DCPEXT_SYSLOG_H +#define DCPEXT_SYSLOG_H +struct dcpext_syslog { + u32 last_index, message_size, used; + bool initialized; + u8 capture[1048576]; +}; + +static int dcpext_syslog_observe(struct dcpext_syslog *s, u64 message, + const u8 *buffer, u32 size) +{ + u32 type = (message >> 52) & 0xff; + u32 index = message & 0xff, stride, offset; + if (type == 8) { + if (s->initialized) + return -EPROTO; + s->last_index = index; /* Native/Linux admission uses inclusive max. */ + s->message_size = (message >> 24) & 0xff; + if (!s->message_size) + return -EINVAL; + s->initialized = true; + return 0; + } + if (type != 5) + return 0; + if (!s->initialized || !buffer || index > s->last_index) + return -EINVAL; + stride = 0x20 + s->message_size; + offset = index * stride; + if (offset > size || stride > size - offset) + return -ERANGE; + if (s->used > sizeof(s->capture) || + stride + 12 > sizeof(s->capture) - s->used) + return -ENOSPC; + put_unaligned_le64(message, s->capture + s->used); + put_unaligned_le32(stride, s->capture + s->used + 8); + memcpy(s->capture + s->used + 12, buffer + offset, stride); + s->used += stride + 12; + return 0; +} +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_bridge.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_bridge.c new file mode 100644 index 0000000000000..d2de75da97583 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_bridge.c @@ -0,0 +1,842 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Root-only RPC transport for bounded firmware diagnostics, not a display ABI. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../dcp-link.h" +#include "m3_dcp_bridge.h" + +#define MAX_EVENTS 32 +#define MAX_CALLBACKS 8 +#define MAX_BUFFERS 64 + +/* Diagnostic ABI: allocations are retained until reboot, including releases. */ +#define BRIDGE_ALLOC_BUFFER _IOWR('M', 0x40, struct m3_dcp_buffer) +#define BRIDGE_RETIRE_BUFFER _IOW('M', 0x41, __u32) +#define BRIDGE_MAP_PIODMA _IOWR('M', 0x42, struct m3_dcp_buffer) + +struct scanout_request { + __u32 width, height, stride, format; + __u64 size, dva; +}; +#define BRIDGE_ALLOC_SCANOUT _IOWR('M', 0x43, struct scanout_request) + +/* Write op 1: submit, message=0 (normal) or 1 (nested), data=RPC packet. + * Write op 2: callback reply, message=original message, data=output bytes. + * Read kind 1: command reply, kind 2: callback, kind 3: unexpected message. + * All headers are little endian. One complete record per read/write. + */ +struct record_header { + __le32 kind; + __le32 size; + __le64 message; +}; + +struct bridge_event { + struct list_head list; + struct record_header header; + u8 data[]; +}; + +struct m3_dcp_bridge { + struct miscdevice misc; + struct device *dev; + struct apple_rtkit *rtkit; + void *rpc; + struct mutex lock; + wait_queue_head_t wait; + struct list_head events; + unsigned int event_count, callback_count; + bool failed; + bool enabled; + bool kernel_client; + u32 kernel_callback_depth; + struct work_struct *kernel_work; + u32 buffer_count; + u64 buffer_bytes; + struct platform_device *piodma; + struct iommu_domain *piodma_domain; + struct platform_device *scanout; + void *scanout_cpu; + struct scanout_request scanout_info; + struct { + void *cpu; + struct m3_dcp_buffer info; + bool retired; + bool piodma_mapped; + } buffers[MAX_BUFFERS]; + struct { + bool active; + bool remote; + u32 offset; + u32 size; + struct apple_dcp_link_rpc_header header; + } calls[2]; + struct { + u64 message; + void *output; + u32 size; + u32 call; + u64 allocation_size; + u64 mapping_id; + } callbacks[MAX_CALLBACKS]; +}; + +static void fail(struct m3_dcp_bridge *b) +{ + b->failed = true; + dev_err(b->dev, "DCP bridge stopped; retain firmware buffers until reboot\n"); + wake_up_interruptible(&b->wait); +} + +static void event(struct m3_dcp_bridge *b, u32 kind, u64 message, + const void *data, u32 size) +{ + struct bridge_event *e; + + if (b->event_count >= MAX_EVENTS || size > APPLE_DCP_LINK_STREAM_BUFFER_SIZE) { + fail(b); + return; + } + e = kmalloc(sizeof(*e) + size, GFP_KERNEL); + if (!e) { + fail(b); + return; + } + e->header = (struct record_header) { + .kind = cpu_to_le32(kind), .size = cpu_to_le32(size), + .message = cpu_to_le64(message), + }; + if (size) + memcpy(e->data, data, size); + list_add_tail(&e->list, &b->events); + b->event_count++; + wake_up_interruptible(&b->wait); + if (b->kernel_work) + queue_work(system_unbound_wq, b->kernel_work); +} + +void m3_dcp_bridge_crashed(struct m3_dcp_bridge *b) +{ + mutex_lock(&b->lock); + fail(b); + mutex_unlock(&b->lock); +} + +void m3_dcp_bridge_receive(struct m3_dcp_bridge *b, u64 message) +{ + struct apple_dcp_link_stream_layout layout; + struct apple_dcp_link_rpc_view view; + u8 stream = apple_dcp_link_message_stream(message); + bool remote = apple_dcp_link_message_is_remote(message); + bool nested; + int ret; + + mutex_lock(&b->lock); + if (b->failed) + goto out; + if ((message & 3) != APPLE_DCP_LINK_MSG_RPC) { + event(b, 3, message, NULL, 0); + goto out; + } + dma_rmb(); + if (message & BIT_ULL(6)) { + nested = b->calls[1].active; + if (stream || !b->calls[nested].active || b->calls[nested].remote != remote || + apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, 0, remote, &layout)) + goto error; + if (memcmp(b->rpc + layout.local_offset + b->calls[nested].offset, + &b->calls[nested].header, sizeof(b->calls[nested].header))) + goto error; + event(b, 1, message, b->rpc + layout.local_offset + b->calls[nested].offset, + b->calls[nested].size); + b->calls[nested].active = false; + goto out; + } + if (b->callback_count >= MAX_CALLBACKS || + apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, stream, remote, &layout)) + goto error; + ret = apple_dcp_link_rpc_decode(message, b->rpc + layout.remote_offset, + layout.capacity, &view); + if (ret) + goto error; + if ((view.call >> 24) != 'D') { + u32 off; + + /* Preserve stream-placement evidence without acknowledging bad data. */ + for (off = 0; off < APPLE_DCP_LINK_HEAP_OFFSET; + off += APPLE_DCP_LINK_STREAM_BUFFER_SIZE) { + const __le32 *h = b->rpc + off; + + dev_err(b->dev, "M3 RPC header offset=%#x words=%#x/%#x/%#x\n", + off, le32_to_cpu(h[0]), le32_to_cpu(h[1]), le32_to_cpu(h[2])); + } + goto error; + } + b->callbacks[b->callback_count].message = message; + b->callbacks[b->callback_count].output = view.output; + b->callbacks[b->callback_count].size = view.output_size; + b->callbacks[b->callback_count].call = view.call; + b->callbacks[b->callback_count].allocation_size = 0; + b->callbacks[b->callback_count].mapping_id = 0; + if (view.call == 0x44343531 && view.input_size == 20 && view.output_size == 28) { + __le64 allocation_size; + + memcpy(&allocation_size, view.input + 4, sizeof(allocation_size)); + b->callbacks[b->callback_count].allocation_size = le64_to_cpu(allocation_size); + } + if (view.call == 0x44323031 && view.input_size == 12 && view.output_size == 20) { + __le64 mapping_id; + + memcpy(&mapping_id, view.input, sizeof(mapping_id)); + b->callbacks[b->callback_count].mapping_id = le64_to_cpu(mapping_id); + } + b->callback_count++; + dev_dbg(b->dev, "M3 DCP callback tag=%#x input=%u output=%u\n", + view.call, view.input_size, view.output_size); + event(b, 2, message, b->rpc + layout.remote_offset + view.offset, view.total_size); + goto out; +error: + fail(b); +out: + mutex_unlock(&b->lock); +} + +static ssize_t bridge_read(struct file *file, char __user *user, size_t count, loff_t *pos) +{ + struct m3_dcp_bridge *b = container_of(file->private_data, struct m3_dcp_bridge, misc); + struct bridge_event *e; + size_t size; + int ret; + +again: + mutex_lock(&b->lock); + if (list_empty(&b->events)) { + ret = b->failed ? -EIO : -EAGAIN; + mutex_unlock(&b->lock); + if (ret == -EIO || file->f_flags & O_NONBLOCK) + return ret; + ret = wait_event_interruptible(b->wait, READ_ONCE(b->event_count) || READ_ONCE(b->failed)); + if (ret) + return ret; + goto again; + } + e = list_first_entry(&b->events, struct bridge_event, list); + size = sizeof(e->header) + le32_to_cpu(e->header.size); + ret = -EMSGSIZE; + if (count < size) + goto out; + ret = -EFAULT; + if (copy_to_user(user, &e->header, size)) + goto out; + list_del(&e->list); + b->event_count--; + kfree(e); + ret = size; +out: + mutex_unlock(&b->lock); + return ret; +} + +static ssize_t bridge_request(struct m3_dcp_bridge *b, struct record_header *record, size_t count) +{ + struct apple_dcp_link_stream_layout layout; + struct apple_dcp_link_rpc_header *rpc; + u64 message, control; + u32 kind, size, total, offset = 0; + bool nested, remote = false; + int ret; + + kind = le32_to_cpu(record->kind); + size = le32_to_cpu(record->size); + control = le64_to_cpu(record->message); + ret = -EINVAL; + if (size != count - sizeof(*record)) + goto free_record; + mutex_lock(&b->lock); + if (!b->enabled) { + ret = -EAGAIN; + goto out; + } + if (b->failed) { + ret = -EIO; + goto out; + } + if (kind == 2) { + unsigned int top; + + if (!b->callback_count) + goto out; + top = b->callback_count - 1; + if (control != b->callbacks[top].message || size != b->callbacks[top].size) + goto out; + memcpy(b->callbacks[top].output, record + 1, size); + dma_wmb(); + ret = apple_rtkit_send_message(b->rtkit, APPLE_DCP_LINK_ENDPOINT, + apple_dcp_link_rpc_reply(control), NULL, false); + if (ret) + fail(b); + else + b->callback_count--; + goto out; + } + if (kind != 1 || control > 1 || size < sizeof(*rpc)) + goto out; + nested = control; + if (b->calls[nested].active || (nested && !b->callback_count) || + (!nested && b->callback_count)) { + ret = -EBUSY; + goto out; + } + if (nested) { + u64 callback = b->callbacks[b->callback_count - 1].message; + + if (apple_dcp_link_message_stream(callback)) { + ret = -EOPNOTSUPP; + goto out; + } + /* Re-entry uses the callback's stream without the new-stream IRQ flag. */ + remote = apple_dcp_link_message_is_remote(callback); + if (b->calls[0].active && b->calls[0].remote == remote) + offset = ALIGN(b->calls[0].size, 64); + } + rpc = (void *)(record + 1); + ret = apple_dcp_link_rpc_payload_size(le32_to_cpu(rpc->input_size), + le32_to_cpu(rpc->output_size), &total); + if (ret || total != size) { + ret = -EMSGSIZE; + goto out; + } + ret = apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, 0, remote, &layout); + if (ret) + goto out; + if (offset + size > layout.capacity) { + ret = -EMSGSIZE; + goto out; + } + memcpy(b->rpc + layout.local_offset + offset, rpc, size); + memset(b->rpc + layout.local_offset + offset + sizeof(*rpc) + le32_to_cpu(rpc->input_size), + 0, le32_to_cpu(rpc->output_size)); + b->calls[nested].header = *rpc; + b->calls[nested].size = size; + b->calls[nested].offset = offset; + b->calls[nested].remote = remote; + b->calls[nested].active = true; + /* M3 bit 9 requires a free stream; it is not a recursive-call marker. */ + apple_dcp_link_rpc_message(offset, size, false, &message); + apple_dcp_link_stream_message(message, 0, remote, &message); + dma_wmb(); + ret = apple_rtkit_send_message(b->rtkit, APPLE_DCP_LINK_ENDPOINT, message, NULL, false); + if (ret) + fail(b); +out: + mutex_unlock(&b->lock); +free_record: + kfree(record); + return ret ? ret : count; +} + +static ssize_t bridge_write(struct file *file, const char __user *user, size_t count, loff_t *pos) +{ + struct m3_dcp_bridge *b = container_of(file->private_data, struct m3_dcp_bridge, misc); + struct record_header *record; + + if (count < sizeof(*record) || count > sizeof(*record) + APPLE_DCP_LINK_STREAM_BUFFER_SIZE) + return -EMSGSIZE; + record = memdup_user(user, count); + if (IS_ERR(record)) + return PTR_ERR(record); + return bridge_request(b, record, count); +} + +static __poll_t bridge_poll(struct file *file, poll_table *wait) +{ + struct m3_dcp_bridge *b = container_of(file->private_data, struct m3_dcp_bridge, misc); + __poll_t mask = 0; + + poll_wait(file, &b->wait, wait); + if (READ_ONCE(b->event_count)) + mask |= EPOLLIN | EPOLLRDNORM; + if (READ_ONCE(b->failed)) + mask |= EPOLLERR; + return mask; +} + +static int map_piodma_buffer(struct m3_dcp_bridge *b, u32 id) +{ + struct m3_dcp_buffer *info = &b->buffers[id - 1].info; + struct device_node *node; + u64 offset; + int ret; + + if (b->buffers[id - 1].piodma_mapped) + return 0; + if (!b->piodma) { + node = of_get_child_by_name(b->dev->of_node, "piodma"); + if (!node || !of_device_is_available(node) || + !of_property_read_bool(node, "apple,j514s-inherited-mappings")) { + of_node_put(node); + return -ENODEV; + } + b->piodma = of_platform_device_create(node, NULL, b->dev); + if (!b->piodma) { + of_node_put(node); + return -ENOMEM; + } + ret = dma_set_mask_and_coherent(&b->piodma->dev, DMA_BIT_MASK(42)); + if (!ret) + ret = of_dma_configure(&b->piodma->dev, node, true); + of_node_put(node); + /* Never destroy an attached locked DART stream on an error path. */ + if (ret) + return ret; + b->piodma_domain = iommu_get_domain_for_dev(&b->piodma->dev); + } + if (IS_ERR_OR_NULL(b->piodma_domain)) + return -EIO; + ret = iommu_map(b->piodma_domain, info->dva, info->physical, info->size, + IOMMU_READ | IOMMU_WRITE, GFP_KERNEL); + if (ret) + return ret; + for (offset = 0; offset < info->size; offset += SZ_16K) { + if (iommu_iova_to_phys(b->piodma_domain, info->dva + offset) != info->physical + offset) { + fail(b); + return -EIO; + } + } + b->buffers[id - 1].piodma_mapped = true; + dev_info(b->dev, "M3 PIODMA mapped buffer %u size=%llu dva=%#llx\n", + id, info->size, info->dva); + return 0; +} + +static int allocate_scanout(struct m3_dcp_bridge *b, struct scanout_request *request) +{ + static const u32 colors[] = { + 0xffffffff, 0xffffff00, 0xff00ffff, 0xff00ff00, + 0xffff00ff, 0xffff0000, 0xff0000ff, 0xff202020, + }; + struct scanout_request info = *request; + struct device_node *node; + dma_addr_t dva; + u32 *pixels; + u32 x, y; + int ret; + + /* One diagnostic allocation per boot; native J514S panel dimensions. */ + if (b->scanout_cpu || b->calls[0].active || b->calls[1].active || b->callback_count || + info.width != 3024 || info.height != 1964 || info.stride || info.format || info.size || info.dva) + return -EINVAL; + if (!b->scanout) { + node = of_find_node_by_path("disp0"); + if (!node || !of_device_is_available(node) || + !of_device_is_compatible(node, "apple,t6030-display-diagnostics") || + !of_property_read_bool(node, "apple,j514s-inherited-mappings")) { + of_node_put(node); + return -ENODEV; + } + b->scanout = of_find_device_by_node(node); + if (!b->scanout) { + of_node_put(node); + return -ENODEV; + } + ret = dma_set_mask_and_coherent(&b->scanout->dev, DMA_BIT_MASK(42)); + if (!ret) + ret = of_dma_configure(&b->scanout->dev, node, true); + of_node_put(node); + /* Retain the device reference and inherited mappings until reboot. */ + if (ret) + return ret; + } + info.stride = info.width * 4; + info.size = ALIGN((u64)info.stride * info.height, SZ_16K); + info.format = 0x42475241; /* DCP BGRA8: wire FourCC ARGB */ + pixels = dma_alloc_coherent(&b->scanout->dev, info.size, &dva, GFP_KERNEL); + if (!pixels) + return -ENOMEM; + info.dva = dva; + /* Fill once; there is no per-frame CPU framebuffer copy. */ + for (y = 0; y < info.height; y++) { + for (x = 0; x < info.width; x++) { + u32 color = colors[x * ARRAY_SIZE(colors) / info.width]; + + if (x < 4 || y < 4 || x >= info.width - 4 || y >= info.height - 4) + color = 0xffffffff; + pixels[y * info.width + x] = cpu_to_le32(color); + } + } + dma_wmb(); + b->scanout_cpu = pixels; + b->scanout_info = info; + dev_info(b->dev, "M3 native scanout allocated %ux%u stride=%u size=%llu dva=%#llx\n", + info.width, info.height, info.stride, info.size, info.dva); + *request = info; + return 0; +} + +static int bridge_buffer_request(struct m3_dcp_bridge *b, unsigned int command, + struct m3_dcp_buffer *info) +{ + struct m3_dcp_buffer request = *info; + struct sg_table table; + dma_addr_t dva; + void *cpu; + u32 id; + size_t size; + int ret = -EINVAL; + + if (command != BRIDGE_ALLOC_BUFFER && command != BRIDGE_RETIRE_BUFFER && + command != BRIDGE_MAP_PIODMA && command != BRIDGE_ALLOC_SCANOUT) + return -ENOTTY; + mutex_lock(&b->lock); + if (b->failed || !b->enabled) { + ret = -EIO; + goto out; + } + if (command == BRIDGE_ALLOC_SCANOUT) { + ret = allocate_scanout(b, (struct scanout_request *)info); + goto out; + } + if (command == BRIDGE_RETIRE_BUFFER) { + id = request.id; + ret = -EINVAL; + if (!id || id > b->buffer_count || b->buffers[id - 1].retired) + goto out; + /* Keep mappings and memory pinned: this diagnostic has no DMA fence. */ + b->buffers[id - 1].retired = true; + ret = 0; + goto out; + } + ret = -EINVAL; + if (command == BRIDGE_MAP_PIODMA) { + id = request.id; + if (!b->callback_count || !id || id > b->buffer_count || + b->buffers[id - 1].retired || request.size || request.physical || request.dva || request.flags || + b->callbacks[b->callback_count - 1].call != 0x44323031 || + b->callbacks[b->callback_count - 1].mapping_id != id) + goto out; + ret = map_piodma_buffer(b, id); + if (!ret) + *info = b->buffers[id - 1].info; + goto out; + } + if (!b->callback_count || request.flags || request.id || request.dva || request.physical || + !request.size || request.size > SZ_16M || + b->callbacks[b->callback_count - 1].call != 0x44343531 || + b->callbacks[b->callback_count - 1].allocation_size != request.size) + goto out; + size = ALIGN(request.size, SZ_16K); + ret = -ENOSPC; + if (b->buffer_count == MAX_BUFFERS || b->buffer_bytes + size > SZ_64M) + goto out; + /* D451 requests both a physical address and a DCP address. Require a + * physically contiguous backing allocation before advertising either. + */ + cpu = dma_alloc_attrs(b->dev, size, &dva, GFP_KERNEL, DMA_ATTR_FORCE_CONTIGUOUS); + ret = -ENOMEM; + if (!cpu) + goto out; + ret = dma_get_sgtable_attrs(b->dev, &table, cpu, dva, size, DMA_ATTR_FORCE_CONTIGUOUS); + if (ret) + goto free_dma; + ret = -ERANGE; + if (table.orig_nents != 1 || table.sgl->length < size) { + sg_free_table(&table); + goto free_dma; + } + request.physical = sg_phys(table.sgl); + sg_free_table(&table); + request.size = size; + request.dva = dva; + request.id = b->buffer_count + 1; /* firmware treats zero as failure */ + memset(cpu, 0, size); + dma_wmb(); + b->buffers[b->buffer_count].cpu = cpu; + b->buffers[b->buffer_count].info = request; + b->buffer_count++; + b->buffer_bytes += size; + /* One allocation per callback. Retain even on copy_to_user failure. */ + b->callbacks[b->callback_count - 1].allocation_size = 0; + dev_info(b->dev, "M3 DCP buffer id=%u size=%zu dma=%pad\n", request.id, size, &dva); + *info = request; + ret = 0; + goto out; +free_dma: + dma_free_attrs(b->dev, size, cpu, dva, DMA_ATTR_FORCE_CONTIGUOUS); +out: + mutex_unlock(&b->lock); + return ret; +} + +static long bridge_ioctl(struct file *file, unsigned int command, unsigned long argument) +{ + struct m3_dcp_bridge *b = container_of(file->private_data, struct m3_dcp_bridge, misc); + void __user *user = (void __user *)argument; + struct m3_dcp_buffer request = {}; + int ret; + + if (command == BRIDGE_RETIRE_BUFFER) { + if (copy_from_user(&request.id, user, sizeof(request.id))) + return -EFAULT; + } else if (command == BRIDGE_ALLOC_BUFFER || command == BRIDGE_MAP_PIODMA || + command == BRIDGE_ALLOC_SCANOUT) { + if (copy_from_user(&request, user, sizeof(request))) + return -EFAULT; + } else { + return -ENOTTY; + } + ret = bridge_buffer_request(b, command, &request); + if (!ret && command != BRIDGE_RETIRE_BUFFER && copy_to_user(user, &request, sizeof(request))) + ret = -EFAULT; + return ret; +} + +static const struct file_operations bridge_fops = { + .owner = THIS_MODULE, + .read = bridge_read, + .write = bridge_write, + .poll = bridge_poll, + .unlocked_ioctl = bridge_ioctl, +}; + +struct m3_dcp_bridge *m3_dcp_bridge_create(struct device *dev, + struct apple_rtkit *rtkit, void *rpc, bool kernel_client) +{ + struct m3_dcp_bridge *b = devm_kzalloc(dev, sizeof(*b), GFP_KERNEL); + int ret; + + if (!b) + return ERR_PTR(-ENOMEM); + b->dev = dev; + b->kernel_client = kernel_client; + b->rtkit = rtkit; + b->rpc = rpc; + mutex_init(&b->lock); + init_waitqueue_head(&b->wait); + INIT_LIST_HEAD(&b->events); + b->misc = (struct miscdevice) { + .minor = MISC_DYNAMIC_MINOR, .name = "m3-dcp-rpc", .mode = 0600, + .fops = &bridge_fops, .parent = dev, + }; + ret = kernel_client ? 0 : misc_register(&b->misc); + if (ret) + return ERR_PTR(ret); + return b; +} + +void m3_dcp_bridge_activate(struct m3_dcp_bridge *b) +{ + mutex_lock(&b->lock); + b->enabled = true; + mutex_unlock(&b->lock); + dev_info(b->dev, "M3 DCP %s ready\n", b->kernel_client ? "kernel RPC client" : "RPC bridge /dev/m3-dcp-rpc"); +} + +/* One kernel worker owns this client, including recursive callbacks. */ +static struct bridge_event *kernel_event(struct m3_dcp_bridge *b, unsigned long timeout) +{ + struct bridge_event *e; + long ret; + + ret = wait_event_interruptible_timeout(b->wait, + READ_ONCE(b->event_count) || READ_ONCE(b->failed), timeout); + if (ret <= 0) + return ERR_PTR(ret ?: -ETIMEDOUT); + mutex_lock(&b->lock); + if (b->failed || list_empty(&b->events)) { + mutex_unlock(&b->lock); + return ERR_PTR(-EIO); + } + e = list_first_entry(&b->events, struct bridge_event, list); + list_del(&e->list); + b->event_count--; + mutex_unlock(&b->lock); + return e; +} + +static int kernel_send(struct m3_dcp_bridge *b, u32 kind, u64 message, + const void *data, u32 size) +{ + struct record_header *record; + ssize_t ret; + + if (size > APPLE_DCP_LINK_STREAM_BUFFER_SIZE) + return -EMSGSIZE; + record = kmalloc(sizeof(*record) + size, GFP_KERNEL); + if (!record) + return -ENOMEM; + *record = (struct record_header) { + .kind = cpu_to_le32(kind), .size = cpu_to_le32(size), + .message = cpu_to_le64(message), + }; + if (size) + memcpy(record + 1, data, size); + ret = bridge_request(b, record, sizeof(*record) + size); + return ret < 0 ? ret : 0; +} + +static int kernel_callback(struct m3_dcp_bridge *b, struct bridge_event *e, + m3_dcp_callback_fn callback, void *cookie, u32 *completed) +{ + struct apple_dcp_link_rpc_header *h = (void *)e->data; + u32 size = le32_to_cpu(e->header.size), in, out, tag; + int ret; + + if (le32_to_cpu(e->header.kind) != 2 || size < sizeof(*h)) + return -EPROTO; + in = le32_to_cpu(h->input_size); + out = le32_to_cpu(h->output_size); + tag = le32_to_cpu(h->call); + if (in > size - sizeof(*h) || out != size - sizeof(*h) - in || !callback) + return -EPROTO; + memset(e->data + sizeof(*h) + in, 0, out); + b->kernel_callback_depth++; + ret = callback(b, cookie, tag, h + 1, in, e->data + sizeof(*h) + in, out); + b->kernel_callback_depth--; + if (ret) + return ret; + if (tag == 0x44353839 && in == 0x6f0 && completed) + *completed = le32_to_cpup((__le32 *)(h + 1)); + return kernel_send(b, 2, le64_to_cpu(e->header.message), e->data + sizeof(*h) + in, out); +} + +int m3_dcp_bridge_pump(struct m3_dcp_bridge *b, unsigned long timeout, + m3_dcp_callback_fn callback, void *cookie) +{ + struct bridge_event *e; + int ret; + + if (!b->kernel_client) + return -EBUSY; + e = kernel_event(b, timeout); + if (IS_ERR(e)) + return PTR_ERR(e); + ret = kernel_callback(b, e, callback, cookie, NULL); + kfree(e); + return ret; +} + +int m3_dcp_bridge_call(struct m3_dcp_bridge *b, u32 tag, + const void *input, u32 input_size, void *output, u32 output_size, + u32 completion_id, m3_dcp_callback_fn callback, void *cookie) +{ + struct apple_dcp_link_rpc_header *packet; + unsigned long deadline = jiffies + 5 * HZ; + u32 size, completed = 0; + u64 expected = 0x42; + bool nested, replied = false; + int ret; + + if (!b->kernel_client) + return -EBUSY; + ret = apple_dcp_link_rpc_payload_size(input_size, output_size, &size); + if (ret || size > APPLE_DCP_LINK_STREAM_BUFFER_SIZE) + return -EMSGSIZE; + packet = kzalloc(size, GFP_KERNEL); + if (!packet) + return -ENOMEM; + apple_dcp_link_rpc_header_encode(packet, tag, input_size, output_size); + if (input_size) + memcpy(packet + 1, input, input_size); +retry: + /* Drain asynchronous notifications before starting an outer request. + * A queued callback is not recursive entry until its handler is running. + */ + if (!b->kernel_callback_depth) { + while (READ_ONCE(b->event_count)) { + ret = m3_dcp_bridge_pump(b, 0, callback, cookie); + if (ret) { + kfree(packet); + return ret; + } + } + } + mutex_lock(&b->lock); + nested = b->kernel_callback_depth != 0; + if (nested && !b->callback_count) { + mutex_unlock(&b->lock); + kfree(packet); + return -EPROTO; + } + if (nested) + expected |= b->callbacks[b->callback_count - 1].message & BIT_ULL(8); + mutex_unlock(&b->lock); + ret = kernel_send(b, 1, nested, packet, size); + if (ret == -EBUSY && !nested && time_before(jiffies, deadline)) + goto retry; /* A callback arrived between draining and submitting. */ + kfree(packet); + if (ret) + return ret; + while (time_before(jiffies, deadline)) { + long remaining = deadline - jiffies; + struct bridge_event *e = kernel_event(b, max(remaining, 0L)); + struct apple_dcp_link_rpc_header *h; + + if (IS_ERR(e)) + return PTR_ERR(e); + h = (void *)e->data; + if (le32_to_cpu(e->header.kind) == 1) { + if (replied || le32_to_cpu(e->header.size) != size || + le64_to_cpu(e->header.message) != expected || + le32_to_cpu(h->call) != tag || le32_to_cpu(h->input_size) != input_size || + le32_to_cpu(h->output_size) != output_size) { + ret = -EPROTO; + } else { + if (output_size) + memcpy(output, e->data + sizeof(*h) + input_size, output_size); + replied = true; + } + } else { + ret = kernel_callback(b, e, callback, cookie, &completed); + if (completion_id && completed && completed != completion_id) + ret = -EPROTO; + } + kfree(e); + if (ret) + return ret; + if (replied && (!completion_id || completed == completion_id)) + return 0; + } + return -ETIMEDOUT; +} + +int m3_dcp_bridge_alloc(struct m3_dcp_bridge *b, struct m3_dcp_buffer *info) +{ + return bridge_buffer_request(b, BRIDGE_ALLOC_BUFFER, info); +} + +int m3_dcp_bridge_map(struct m3_dcp_bridge *b, struct m3_dcp_buffer *info) +{ + return bridge_buffer_request(b, BRIDGE_MAP_PIODMA, info); +} + +int m3_dcp_bridge_retire(struct m3_dcp_bridge *b, u32 id) +{ + struct m3_dcp_buffer info = { .id = id }; + + return bridge_buffer_request(b, BRIDGE_RETIRE_BUFFER, &info); +} + +void m3_dcp_bridge_set_work(struct m3_dcp_bridge *b, struct work_struct *work) +{ + mutex_lock(&b->lock); + b->kernel_work = work; + if (b->event_count) + queue_work(system_unbound_wq, work); + mutex_unlock(&b->lock); +} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_bridge.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_bridge.h new file mode 100644 index 0000000000000..3e47f3f727dce --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_bridge.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef M3_DCP_BRIDGE_H +#define M3_DCP_BRIDGE_H + +struct device; +struct work_struct; +struct apple_rtkit; +struct m3_dcp_bridge; + +struct m3_dcp_buffer { + u64 size, physical, dva; + u32 id, flags; +}; + +typedef int (*m3_dcp_callback_fn)(struct m3_dcp_bridge *bridge, void *cookie, + u32 tag, const void *input, u32 input_size, + void *output, u32 output_size); + +struct m3_dcp_bridge *m3_dcp_bridge_create(struct device *dev, + struct apple_rtkit *rtkit, void *rpc, bool kernel_client); +void m3_dcp_bridge_receive(struct m3_dcp_bridge *bridge, u64 message); +void m3_dcp_bridge_activate(struct m3_dcp_bridge *bridge); +void m3_dcp_bridge_crashed(struct m3_dcp_bridge *bridge); +int m3_dcp_bridge_call(struct m3_dcp_bridge *bridge, u32 tag, + const void *input, u32 input_size, void *output, u32 output_size, + u32 completion_id, m3_dcp_callback_fn callback, void *cookie); +int m3_dcp_bridge_pump(struct m3_dcp_bridge *bridge, unsigned long timeout, + m3_dcp_callback_fn callback, void *cookie); +int m3_dcp_bridge_alloc(struct m3_dcp_bridge *bridge, struct m3_dcp_buffer *info); +int m3_dcp_bridge_map(struct m3_dcp_bridge *bridge, struct m3_dcp_buffer *info); +int m3_dcp_bridge_retire(struct m3_dcp_bridge *bridge, u32 id); + +void m3_dcp_bridge_set_work(struct m3_dcp_bridge *bridge, struct work_struct *work); +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_brightness.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_brightness.h new file mode 100644 index 0000000000000..b7b391ed30deb --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_brightness.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef M3_DCP_BRIGHTNESS_H +#define M3_DCP_BRIGHTNESS_H + +#include +#include + +#define M3_PANEL_MAX_NITS 500 +#define M3_PANEL_DEFAULT_NITS 140 + +/* J514S TEXT SHA a0af36d10a8abf2435918ba360e697529ddfaed56b89f9466e76d566b296e876: + * 133880..13389c copies wire[354] and wire[35c] to transaction[76a]/[778]. + * 17cd68..17cdc0 converts binary64 nits to runtime property 19 (16.16). + * The M4 double is at 35e: that layout must not be reused on M3. + * Use firmware panel calibration; this is an SDR brightness range. + */ +static inline void m3_dcp_encode_brightness(u8 *swap, unsigned int nits) +{ + u64 bits = 0; + + /* Encode an exactly representable integer as IEEE754 binary64 without + * using floating point in the kernel. Caller bounds nits to 0..500. + */ + if (nits) { + unsigned int exponent = fls(nits) - 1; + + bits = ((u64)(1023 + exponent) << 52) | + (((u64)nits << (52 - exponent)) & ((1ULL << 52) - 1)); + } + swap[0x354] = 1; + put_unaligned_le64(bits, swap + 0x35c); +} +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_clock_restore.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_clock_restore.c new file mode 100644 index 0000000000000..8c4318c59af54 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_clock_restore.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* J514S diagnostic, not a clock driver. Restore only the single observed + * before/after PMP difference at the ADT-qualified display clock register. + * Do not infer this bit's semantics or use on another boot state/platform. + * Run under the initialization watchdog, with native DCP unattached. + */ +#include +#include +#include +#include + +static int __init restore(void) +{ + void __iomem *reg; + u32 before, after; + + if (!of_machine_is_compatible("apple,j514s") || + !of_machine_is_compatible("apple,t6030")) + return -ENODEV; + reg = ioremap_np(0x350040000ULL, 0x4000); + if (!reg) + return -ENOMEM; + before = readl(reg + 0x64); + if (before != 0x87100000) { + pr_err("M3 DCP clock restore refuses unexpected %#x\n", before); + iounmap(reg); + return -EINVAL; + } + writel(0x85100000, reg + 0x64); + after = readl(reg + 0x64); + pr_info("M3 DCP clock restore before=%#x immediate=%#x\n", before, after); + msleep(100); + pr_info("M3 DCP clock restore after100ms=%#x\n", readl(reg + 0x64)); + iounmap(reg); + return 0; +} +module_init(restore); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("J514S restore observed pre-PMP display clock bit; reboot to remove"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_hold.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_hold.c new file mode 100644 index 0000000000000..42fd6729b68eb --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_hold.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Explicit J514S experiment: retain inherited display domains across PMP boot. + * No unload: keep the power floor until reboot, including on probe failure. + */ +#include +#include +#include +#include +#include + +static int __init hold(void) +{ + const u32 offsets[] = {0x1c0, 0x258, 0x10000}; + struct regmap *pmgr; + struct device_node *node; + int ret; + u32 before[ARRAY_SIZE(offsets)]; + + if (!of_machine_is_compatible("apple,j514s")) + return -ENODEV; + node = of_find_node_by_path("/soc/power-management@350700000"); + if (!node) + return -ENODEV; + pmgr = syscon_node_to_regmap(node); + of_node_put(node); + if (IS_ERR(pmgr)) + return PTR_ERR(pmgr); + for (unsigned int i = 0; i < ARRAY_SIZE(offsets); i++) { + pr_info("M3 DCP hold reading PMGR +%#x through syscon\n", offsets[i]); + msleep(100); + ret = regmap_read(pmgr, offsets[i], &before[i]); + if (ret) + return ret; + pr_info("M3 DCP hold before PMGR +%#x = %#x\n", offsets[i], before[i]); + if ((before[i] & 0xff) != 0xff || (before[i] & BIT(31))) { + return -EBUSY; + } + } + for (unsigned int i = 0; i < ARRAY_SIZE(offsets); i++) { + /* Same PS_MIN field used by apple,min-state in pmgr-pwrstate.c. + * Preserve automatic control and clear write-one-to-clear history. + */ + pr_info("M3 DCP hold raising PMGR +%#x minimum to active\n", offsets[i]); + msleep(100); + ret = regmap_update_bits(pmgr, offsets[i], GENMASK(19, 16) | GENMASK(9, 8), + GENMASK(19, 16)); + if (ret) + return ret; + ret = regmap_read(pmgr, offsets[i], &before[i]); + if (ret) + return ret; + pr_info("M3 DCP hold after PMGR +%#x = %#x\n", offsets[i], before[i]); + } + return 0; +} +module_init(hold); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.c new file mode 100644 index 0000000000000..c4df618902473 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.c @@ -0,0 +1,730 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* J514S native SID0 scanout. Explicit-load, fixed inherited panel mode only. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "m3_dcp_native.h" +#include "m3_dcp_kms.h" +#include "m3_dcp_brightness.h" + +/* Diagnostic history capacity, unrelated to submission or buffering limits. */ +#define M3_PRESENT_HISTORY 256 +struct m3_present_record { + u64 sequence, submitted_ns, completed_ns; + u32 framebuffer; + s32 status; +}; + +struct m3_kms { + struct drm_device drm; + struct drm_plane plane; + struct drm_crtc crtc; + struct drm_encoder encoder; + struct drm_connector connector; + struct drm_display_mode mode; + struct m3_dcp_native *native; + u32 brightness_nits; + /* Keep scanout pinned independently of atomic state until real completion. */ + struct drm_framebuffer *active_fb, *failed_fb; + struct mutex present_lock; + bool failed; + u64 completed; + u64 swap_total_ns, swap_max_ns; + spinlock_t history_lock; + u64 history_count; + struct m3_present_record history[M3_PRESENT_HISTORY]; +}; + +static struct m3_kms *to_m3(struct drm_device *drm) +{ + return container_of(drm, struct m3_kms, drm); +} + +static enum drm_mode_status m3_mode_valid(struct drm_crtc *crtc, + const struct drm_display_mode *mode) +{ + return drm_mode_equal(mode, &to_m3(crtc->dev)->mode) ? MODE_OK : MODE_BAD; +} + +static int m3_plane_check(struct drm_plane *plane, struct drm_atomic_state *state) +{ + struct drm_plane_state *p = drm_atomic_get_new_plane_state(state, plane); + struct drm_crtc_state *c; + struct drm_gem_dma_object *obj; + u64 bytes; + int ret; + + if (READ_ONCE(to_m3(plane->dev)->failed)) + return -EIO; + if (!p->crtc) + return 0; + c = drm_atomic_get_crtc_state(state, p->crtc); + if (IS_ERR(c)) + return PTR_ERR(c); + ret = drm_atomic_helper_check_plane_state(p, c, DRM_PLANE_NO_SCALING, + DRM_PLANE_NO_SCALING, false, true); + if (ret || !p->visible) + return ret; + /* Only the qualified full-panel, uncompressed, single-plane layout. */ + if (p->src_x || p->src_y || p->crtc_x || p->crtc_y || + p->src_w != 3024U << 16 || p->src_h != 1964U << 16 || + p->crtc_w != 3024 || p->crtc_h != 1964 || + p->fb->width != 3024 || p->fb->height != 1964 || + p->fb->modifier != DRM_FORMAT_MOD_LINEAR || p->fb->offsets[0] || + p->fb->pitches[0] < 12096 || (p->fb->pitches[0] & 63)) + return -EINVAL; + obj = drm_fb_dma_get_gem_obj(p->fb, 0); + bytes = (u64)p->fb->pitches[0] * p->fb->height; + if (!obj || bytes > obj->base.size || bytes > U32_MAX || !obj->dma_addr) + return -EINVAL; + return 0; +} + +static void m3_plane_update(struct drm_plane *plane, struct drm_atomic_state *state) +{ + /* One RPC contains the complete atomic state, submitted by CRTC flush. */ +} + +static const struct drm_plane_helper_funcs m3_plane_helpers = { + .prepare_fb = drm_gem_plane_helper_prepare_fb, + .atomic_check = m3_plane_check, + .atomic_update = m3_plane_update, + .atomic_disable = m3_plane_update, +}; + +static const struct drm_plane_funcs m3_plane_funcs = { + .update_plane = drm_atomic_helper_update_plane, + .disable_plane = drm_atomic_helper_disable_plane, + .destroy = drm_plane_cleanup, + .reset = drm_atomic_helper_plane_reset, + .atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_plane_destroy_state, +}; + +static void m3_surface(u8 *s, struct drm_framebuffer *fb) +{ + u32 stride = fb->pitches[0], bytes = stride * fb->height; + + memset(s, 0, 0x22c); + /* DCP has no XRGB8. Premultiplied RGB over black preserves XRGB pixels. */ + s[2] = fb->format->format == DRM_FORMAT_XRGB8888; + put_unaligned_le32(1, s + 3); + put_unaligned_le32(1, s + 7); + put_unaligned_le32(0x42475241, s + 0xb); + s[0x13] = 13; + s[0x14] = 12; + put_unaligned_le32(stride, s + 0x15); + put_unaligned_le16(1, s + 0x19); + s[0x1b] = s[0x1c] = 1; + put_unaligned_le32(fb->width, s + 0x21); + put_unaligned_le32(fb->height, s + 0x25); + put_unaligned_le32(bytes, s + 0x29); + put_unaligned_le32(1, s + 0x35); + put_unaligned_le64(1, s + 0x51); + put_unaligned_le32(fb->width, s + 0x59); + put_unaligned_le32(fb->height, s + 0x5d); + put_unaligned_le32(stride, s + 0x69); + put_unaligned_le32(bytes, s + 0x6d); + put_unaligned_le16(4, s + 0x71); + s[0x73] = s[0x74] = 1; + put_unaligned_le64(1, s + 0x149); +} + +static void m3_finish_event(struct drm_crtc *crtc, struct drm_atomic_state *state, bool success) +{ + struct drm_crtc_state *c = drm_atomic_get_new_crtc_state(state, crtc); + struct drm_pending_vblank_event *event = c->event; + unsigned long flags; + + c->event = NULL; + if (!event) + return; + if (!success) { + /* Release helper waiters without reporting a successful page flip. */ + if (event->base.fence) { + dma_fence_set_error(event->base.fence, -EIO); + dma_fence_signal(event->base.fence); + } + if (event->base.completion) { + complete_all(event->base.completion); + event->base.completion_release(event->base.completion); + } + drm_event_cancel_free(crtc->dev, &event->base); + return; + } + spin_lock_irqsave(&crtc->dev->event_lock, flags); + drm_crtc_send_vblank_event(crtc, event); + spin_unlock_irqrestore(&crtc->dev->event_lock, flags); +} + +static bool m3_present_locked(struct m3_kms *m3, struct drm_framebuffer *fb) +{ + u8 surface[0x22c]; + dma_addr_t dva = 0; + u64 start, elapsed; + int ret; + + lockdep_assert_held(&m3->present_lock); + if (m3->failed) + return false; + if (fb) { + drm_framebuffer_get(fb); + m3_surface(surface, fb); + dva = drm_fb_dma_get_gem_obj(fb, 0)->dma_addr; + } + start = ktime_get_ns(); + ret = m3_dcp_native_swap(m3->native, fb ? surface : NULL, dva, + fb ? fb->width : 0, fb ? fb->height : 0, + fb ? m3->brightness_nits : 0); + elapsed = ktime_get_ns() - start; + /* Never make a timing reader wait on present_lock or copy scanout pixels. */ + spin_lock(&m3->history_lock); + m3->history[m3->history_count % M3_PRESENT_HISTORY] = + (struct m3_present_record) { + .sequence = m3->history_count + 1, + .submitted_ns = start, + .completed_ns = start + elapsed, + .framebuffer = fb ? fb->base.id : 0, + .status = ret, + }; + m3->history_count++; + spin_unlock(&m3->history_lock); + if (ret) { + /* Either buffer may still be scanned out. Never unmap them on error. */ + m3->failed_fb = fb; + WRITE_ONCE(m3->failed, true); + drm_err(&m3->drm, "native DCP flip failed %d; buffers pinned until reboot\n", ret); + return false; + } + if (m3->active_fb) + drm_framebuffer_put(m3->active_fb); + m3->active_fb = fb; + m3->completed++; + m3->swap_total_ns += elapsed; + m3->swap_max_ns = max(m3->swap_max_ns, elapsed); + if (m3->completed <= 3 || !(m3->completed % 600)) + drm_info(&m3->drm, "native KMS completed=%llu surface=%d imported=%d\n", + m3->completed, !!fb, fb && !!fb->obj[0]->import_attach); + return true; +} + +static bool m3_present(struct m3_kms *m3, struct drm_framebuffer *fb) +{ + bool success; + + mutex_lock(&m3->present_lock); + success = m3_present_locked(m3, fb); + mutex_unlock(&m3->present_lock); + return success; +} + +static int m3_backlight_update(struct backlight_device *backlight) +{ + struct m3_kms *m3 = bl_get_data(backlight); + int ret = 0; + + /* Serialize with KMS and retain the active framebuffer through D589. + * Brightness must also work when an idle compositor submits no frames. + * An inactive CRTC retains the request until its next enabling swap. + */ + mutex_lock(&m3->present_lock); + if (m3->failed) { + ret = -EIO; + goto out; + } + m3->brightness_nits = backlight_get_brightness(backlight); + /* A repeated userspace request must repair firmware state as well. */ + m3_dcp_native_brightness_invalidate(m3->native); + if (m3->active_fb && !m3_present_locked(m3, m3->active_fb)) + ret = -EIO; +out: + mutex_unlock(&m3->present_lock); + return ret; +} + +static const struct backlight_ops m3_backlight_ops = { + .update_status = m3_backlight_update, +}; + +static void m3_crtc_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) +{ + struct m3_kms *m3 = to_m3(crtc->dev); + bool success; + + if (!drm_atomic_get_new_crtc_state(state, crtc)->active) + return; + success = m3_present(m3, drm_atomic_get_new_plane_state(state, &m3->plane)->fb); + if (success) + drm_crtc_handle_vblank(crtc); + m3_finish_event(crtc, state, success); +} + +static void m3_crtc_enable(struct drm_crtc *crtc, struct drm_atomic_state *state) +{ + const struct drm_vblank_crtc_config config = { .offdelay_ms = 0 }; + + /* Firmware may reset brightness while the panel is powered down. */ + m3_dcp_native_brightness_invalidate(to_m3(crtc->dev)->native); + drm_crtc_vblank_on_config(crtc, &config); +} + +static void m3_crtc_disable(struct drm_crtc *crtc, struct drm_atomic_state *state) +{ + bool success = m3_present(to_m3(crtc->dev), NULL); + + if (success) + drm_crtc_handle_vblank(crtc); + m3_finish_event(crtc, state, success); + drm_crtc_vblank_off(crtc); +} + +static int m3_enable_vblank(struct drm_crtc *crtc) { return 0; } +static void m3_disable_vblank(struct drm_crtc *crtc) { } + +static const struct drm_crtc_helper_funcs m3_crtc_helpers = { + .mode_valid = m3_mode_valid, + .atomic_enable = m3_crtc_enable, + .atomic_disable = m3_crtc_disable, + .atomic_flush = m3_crtc_flush, +}; + +static const struct drm_crtc_funcs m3_crtc_funcs = { + .set_config = drm_atomic_helper_set_config, + .page_flip = drm_atomic_helper_page_flip, + .destroy = drm_crtc_cleanup, + .reset = drm_atomic_helper_crtc_reset, + .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state, + .enable_vblank = m3_enable_vblank, + .disable_vblank = m3_disable_vblank, +}; + +static int m3_get_modes(struct drm_connector *connector) +{ + struct drm_display_mode *mode = drm_mode_duplicate(connector->dev, + &to_m3(connector->dev)->mode); + if (!mode) + return 0; + connector->display_info.width_mm = mode->width_mm; + connector->display_info.height_mm = mode->height_mm; + mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED; + drm_mode_probed_add(connector, mode); + return 1; +} + +static enum drm_connector_status m3_detect(struct drm_connector *connector, bool force) +{ + return connector_status_connected; +} + +static const struct drm_connector_helper_funcs m3_connector_helpers = { + .get_modes = m3_get_modes, +}; + +static const struct drm_connector_funcs m3_connector_funcs = { + .detect = m3_detect, + .fill_modes = drm_helper_probe_single_connector_modes, + .destroy = drm_connector_cleanup, + .reset = drm_atomic_helper_connector_reset, + .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, +}; + +static const struct drm_encoder_funcs m3_encoder_funcs = { + .destroy = drm_encoder_cleanup, +}; + +static int m3_atomic_check(struct drm_device *drm, struct drm_atomic_state *state) +{ + struct m3_kms *m3 = to_m3(drm); + struct drm_crtc_state *c; + int ret; + + if (READ_ONCE(m3->failed)) + return -EIO; + c = drm_atomic_get_crtc_state(state, &m3->crtc); + if (IS_ERR(c)) + return PTR_ERR(c); + ret = drm_atomic_add_affected_planes(state, &m3->crtc); + return ret ?: drm_atomic_helper_check(drm, state); +} + +static void m3_commit_tail(struct drm_atomic_state *state) +{ + /* Each flush waits for its matching D589 before releasing the previous FB. + * Waiting for an additional vblank here would stall an idle display. + */ + drm_atomic_helper_commit_modeset_disables(state->dev, state); + drm_atomic_helper_commit_modeset_enables(state->dev, state); + drm_atomic_helper_commit_planes(state->dev, state, DRM_PLANE_COMMIT_ACTIVE_ONLY); + drm_atomic_helper_commit_hw_done(state); + drm_atomic_helper_cleanup_planes(state->dev, state); +} + +static const struct drm_mode_config_funcs m3_mode_funcs = { + .fb_create = drm_gem_fb_create, + .atomic_check = m3_atomic_check, + .atomic_commit = drm_atomic_helper_commit, +}; +static const struct drm_mode_config_helper_funcs m3_mode_helpers = { + .atomic_commit_tail = m3_commit_tail, +}; + +static int m3_dumb_create(struct drm_file *file, struct drm_device *drm, + struct drm_mode_create_dumb *args) +{ + /* llvmpipe pads its storage to 64-pixel tiles. The scanout framebuffer + * and plane still have to match the exact native panel extent. */ + if (args->bpp != 32 || !args->width || args->width > ALIGN(3024, 64) || + !args->height || args->height > ALIGN(1964, 64)) + return -EINVAL; + args->pitch = ALIGN(args->width * 4, 64); + args->size = PAGE_ALIGN((u64)args->pitch * args->height); + return drm_gem_dma_dumb_create_internal(file, drm, args); +} + +DEFINE_DRM_GEM_DMA_FOPS(m3_fops); +static const struct drm_driver m3_driver = { + /* DRM core owns syncobj timelines, including fences imported from the + * separate render device. Compositors query this display node too. */ + .driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC | + DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE, + DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(m3_dumb_create), + .fops = &m3_fops, + .name = "m3-dcp", + .desc = "J514S native DCP scanout", + .major = 0, + .minor = 1, +}; + + +static int m3_status(struct seq_file *seq, void *unused) +{ + struct drm_debugfs_entry *entry = seq->private; + struct m3_kms *m3 = to_m3(entry->dev); + struct drm_framebuffer *fb; + + mutex_lock(&m3->present_lock); + fb = m3->active_fb; + seq_printf(seq, "completed %llu\nfailed %u\nactive_fb %u\nimported %u\n", + m3->completed, m3->failed, fb ? fb->base.id : 0, + fb && !!fb->obj[0]->import_attach); + seq_printf(seq, "swap_total_us %llu\nswap_max_us %llu\n", + m3->swap_total_ns / 1000, m3->swap_max_ns / 1000); + mutex_unlock(&m3->present_lock); + return 0; +} + +static int m3_analytics(struct seq_file *seq, void *unused) +{ + struct drm_debugfs_entry *entry = seq->private; + struct m3_kms *m3 = to_m3(entry->dev); + + m3_dcp_native_analytics_show(m3->native, seq); + return 0; +} + +static int m3_completion(struct seq_file *seq, void *unused) +{ + struct drm_debugfs_entry *entry = seq->private; + struct m3_kms *m3 = to_m3(entry->dev); + + m3_dcp_native_completion_show(m3->native, seq); + return 0; +} + +static int m3_present_history(struct seq_file *seq, void *unused) +{ + struct drm_debugfs_entry *entry = seq->private; + struct m3_kms *m3 = to_m3(entry->dev); + struct m3_present_record *records; + u64 count, first, i; + + records = kmalloc(sizeof(m3->history), GFP_KERNEL); + if (!records) + return -ENOMEM; + spin_lock(&m3->history_lock); + count = m3->history_count; + memcpy(records, m3->history, sizeof(m3->history)); + spin_unlock(&m3->history_lock); + seq_puts(seq, "sequence submitted_ns completed_ns framebuffer status\n"); + first = count > M3_PRESENT_HISTORY ? count - M3_PRESENT_HISTORY : 0; + for (i = first; i < count; i++) { + struct m3_present_record *r = &records[i % M3_PRESENT_HISTORY]; + + seq_printf(seq, "%llu %llu %llu %u %d\n", r->sequence, + r->submitted_ns, r->completed_ns, r->framebuffer, r->status); + } + kfree(records); + return 0; +} + +struct m3_snapshot { + size_t size; + u8 bytes[]; +}; + +static int m3_snapshot_open(struct inode *inode, struct file *file) +{ + struct m3_kms *m3 = inode->i_private; + struct m3_snapshot *snapshot; + struct drm_framebuffer *fb; + struct dma_buf *dmabuf = NULL; + struct iosys_map map = {}; + u32 bytes; + int ret; + + ret = mutex_lock_interruptible(&m3->present_lock); + if (ret) + return ret; + fb = m3->active_fb; + if (m3->failed || !fb) { + ret = -ENODATA; + goto unlock; + } + bytes = fb->pitches[0] * fb->height; + if (bytes > SZ_32M) { + ret = -E2BIG; + goto unlock; + } + snapshot = kvmalloc(sizeof(*snapshot) + 32 + bytes, GFP_KERNEL); + if (!snapshot) { + ret = -ENOMEM; + goto unlock; + } + /* Only this root-only diagnostic maps or reads imported GPU pixels. */ + if (fb->obj[0]->import_attach) { + dmabuf = fb->obj[0]->import_attach->dmabuf; + ret = dma_buf_begin_cpu_access(dmabuf, DMA_FROM_DEVICE); + if (ret) + goto free; + ret = dma_buf_vmap_unlocked(dmabuf, &map); + if (ret) { + dma_buf_end_cpu_access(dmabuf, DMA_FROM_DEVICE); + goto free; + } + } else { + ret = drm_gem_dma_vmap(drm_fb_dma_get_gem_obj(fb, 0), &map); + if (ret) + goto free; + } + snapshot->size = 32 + bytes; + put_unaligned_le32(0x4246334d, snapshot->bytes); /* M3FB */ + put_unaligned_le32(fb->width, snapshot->bytes + 4); + put_unaligned_le32(fb->height, snapshot->bytes + 8); + put_unaligned_le32(fb->pitches[0], snapshot->bytes + 12); + put_unaligned_le32(fb->format->format, snapshot->bytes + 16); + put_unaligned_le32(fb->base.id, snapshot->bytes + 20); + put_unaligned_le32(!!dmabuf, snapshot->bytes + 24); + put_unaligned_le32(bytes, snapshot->bytes + 28); + iosys_map_memcpy_from(snapshot->bytes + 32, &map, 0, bytes); + if (dmabuf) { + dma_buf_vunmap_unlocked(dmabuf, &map); + ret = dma_buf_end_cpu_access(dmabuf, DMA_FROM_DEVICE); + if (ret) + goto free; + } + file->private_data = snapshot; + mutex_unlock(&m3->present_lock); + return 0; +free: + kvfree(snapshot); +unlock: + mutex_unlock(&m3->present_lock); + return ret; +} + +static ssize_t m3_snapshot_read(struct file *file, char __user *data, size_t size, loff_t *pos) +{ + struct m3_snapshot *snapshot = file->private_data; + + return simple_read_from_buffer(data, size, pos, snapshot->bytes, snapshot->size); +} + +static int m3_snapshot_release(struct inode *inode, struct file *file) +{ + kvfree(file->private_data); + return 0; +} + +static const struct file_operations m3_snapshot_fops = { + .owner = THIS_MODULE, + .open = m3_snapshot_open, + .read = m3_snapshot_read, + .release = m3_snapshot_release, + .llseek = default_llseek, +}; + +static void m3_panel_size(struct m3_kms *m3) +{ + struct device_node *dcp, *panel; + u32 width = 0, height = 0; + + /* This firmware's DisplayAttributes omits image dimensions. As with the + * M1/M2 internal panels, use board data rather than inventing an EDID or + * encoding a desktop scale. Native scanout includes the notch region. + */ + dcp = of_find_node_by_path("dcp"); + panel = of_get_compatible_child(dcp, "apple,panel"); + of_node_put(dcp); + if (panel) { + of_property_read_u32(panel, "width-mm", &width); + of_property_read_u32(panel, "height-mm", &height); + of_node_put(panel); + } + if (!width || !height || width > 2000 || height > 2000) { + drm_warn(&m3->drm, "panel physical size missing or invalid; update J514S device tree\n"); + return; + } + m3->mode.width_mm = width; + m3->mode.height_mm = height; + drm_info(&m3->drm, "panel physical size %ux%u mm from device tree\n", width, height); +} + +int m3_dcp_kms_register(struct m3_dcp_native *native, bool takeover) +{ + static const u32 formats[] = { DRM_FORMAT_XRGB8888, DRM_FORMAT_ARGB8888 }; + static const u64 modifiers[] = { DRM_FORMAT_MOD_LINEAR, DRM_FORMAT_MOD_INVALID }; + struct backlight_properties backlight_props = { + .type = BACKLIGHT_RAW, + .max_brightness = M3_PANEL_MAX_NITS, + .brightness = M3_PANEL_DEFAULT_NITS, + }; + struct backlight_device *backlight; + struct device_node *node; + struct platform_device *pdev; + struct m3_kms *m3; + struct drm_device *drm; + const void *timing; + u32 size; + int ret; + + node = of_find_node_by_path("disp0"); + if (!node || !of_device_is_available(node) || + !of_device_is_compatible(node, "apple,t6030-display-diagnostics") || + !of_property_read_bool(node, "apple,j514s-inherited-mappings")) { + of_node_put(node); + return -ENODEV; + } + pdev = of_find_device_by_node(node); + if (!pdev) { + of_node_put(node); + return -ENODEV; + } + /* Native client and its device mappings remain pinned until reboot. */ + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(42)); + if (!ret) + ret = of_dma_configure(&pdev->dev, node, true); + of_node_put(node); + if (ret) + return ret; + m3 = devm_drm_dev_alloc(&pdev->dev, &m3_driver, struct m3_kms, drm); + if (IS_ERR(m3)) + return PTR_ERR(m3); + drm = &m3->drm; + m3->native = native; + m3->brightness_nits = M3_PANEL_DEFAULT_NITS; + mutex_init(&m3->present_lock); + spin_lock_init(&m3->history_lock); + timing = m3_dcp_native_property(native, "PreferredTimingElements", &size); + if (!timing) + return -ENODATA; + ret = m3_dcp_preferred_mode(timing, size, &m3->mode); + kfree(timing); + if (ret) + return ret; + m3_panel_size(m3); + ret = drmm_mode_config_init(drm); + if (ret) + return ret; + drm->mode_config.min_width = drm->mode_config.max_width = 3024; + drm->mode_config.min_height = drm->mode_config.max_height = 1964; + drm->mode_config.funcs = &m3_mode_funcs; + drm->mode_config.helper_private = &m3_mode_helpers; + ret = drm_universal_plane_init(drm, &m3->plane, 1, &m3_plane_funcs, formats, + ARRAY_SIZE(formats), modifiers, DRM_PLANE_TYPE_PRIMARY, NULL); + if (ret) + return ret; + drm_plane_helper_add(&m3->plane, &m3_plane_helpers); + ret = drm_crtc_init_with_planes(drm, &m3->crtc, &m3->plane, NULL, &m3_crtc_funcs, NULL); + if (ret) + return ret; + drm_crtc_helper_add(&m3->crtc, &m3_crtc_helpers); + ret = drm_encoder_init(drm, &m3->encoder, &m3_encoder_funcs, DRM_MODE_ENCODER_TMDS, NULL); + if (ret) + return ret; + m3->encoder.possible_crtcs = 1; + ret = drm_connector_init(drm, &m3->connector, &m3_connector_funcs, DRM_MODE_CONNECTOR_eDP); + if (ret) + return ret; + drm_connector_helper_add(&m3->connector, &m3_connector_helpers); + ret = drm_connector_attach_encoder(&m3->connector, &m3->encoder); + if (ret) + return ret; + ret = drm_vblank_init(drm, 1); + if (ret) + return ret; + drm_mode_config_reset(drm); + if (takeover) { + struct resource fb; + + node = of_find_compatible_node(NULL, NULL, "simple-framebuffer"); + ret = node ? of_address_to_resource(node, 0, &fb) : -ENODEV; + of_node_put(node); + if (ret) + return ret; + ret = aperture_remove_conflicting_devices(fb.start, resource_size(&fb), m3_driver.name); + if (ret) + return ret; + } + drm_debugfs_add_file(drm, "m3_status", m3_status, NULL); + drm_debugfs_add_file(drm, "m3_analytics", m3_analytics, NULL); + drm_debugfs_add_file(drm, "m3_completion", m3_completion, NULL); + drm_debugfs_add_file(drm, "m3_present_history", m3_present_history, NULL); + backlight = devm_backlight_device_register(&pdev->dev, "apple-panel-bl", + &pdev->dev, m3, &m3_backlight_ops, + &backlight_props); + if (IS_ERR(backlight)) + return PTR_ERR(backlight); + ret = drm_dev_register(drm, 0); + if (!ret) + debugfs_create_file("m3_scanout", 0400, drm->debugfs_root, m3, &m3_snapshot_fops); + if (!ret) + drm_info(drm, "M3 native KMS ready: %ux%u at %d Hz, SID0 DMA scanout\n", + m3->mode.hdisplay, m3->mode.vdisplay, drm_mode_vrefresh(&m3->mode)); + return ret; +} + +MODULE_IMPORT_NS("DMA_BUF"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.h new file mode 100644 index 0000000000000..ae1b38a12a553 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef M3_DCP_KMS_H +#define M3_DCP_KMS_H +#include +struct m3_dcp_native; +struct drm_display_mode; +int m3_dcp_preferred_mode(const void *blob, u32 size, struct drm_display_mode *mode); +int m3_dcp_kms_register(struct m3_dcp_native *native, bool takeover); +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_native.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_native.c new file mode 100644 index 0000000000000..a48d4272f42e1 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_native.c @@ -0,0 +1,739 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Hash-qualified J514S DCP protocol; a single kernel client owns all RPCs. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "m3_dcp_bridge.h" +#include "m3_dcp_native.h" +#include "m3_dcp_brightness.h" + +#define A(n) M3_DCP_TAG('A', n) +#define D(n) M3_DCP_TAG('D', n) +#define MAX_PROPERTIES 256 +#define MAX_RAW_PROPERTIES 32 + +struct m3_dcp_property { + u32 service; + char key[64]; + u64 value; +}; + +struct m3_dcp_native { + struct device *dev; + struct m3_dcp_bridge *bridge; + struct mutex lock; + struct work_struct idle_work; + bool failed; + bool opened; + bool brightness_valid; + u32 brightness_nits; + u32 stride; + u32 diagnostic_plane; + bool diagnostic_flat_surface; + struct m3_dcp_property properties[MAX_PROPERTIES]; + u32 property_count; + struct { u32 service; char key[64]; void *data; u32 size; } raw[MAX_RAW_PROPERTIES]; + u32 raw_count, raw_bytes; + void *chunk; + u32 chunk_size, chunk_offset; + u64 completion_count; + u8 completion_last[0x6f0]; + u64 analytics_count; + u8 analytics_last[4164]; +}; + +static void idle_callbacks(struct work_struct *work); + +static int callback(struct m3_dcp_bridge *b, void *cookie, u32 tag, + const void *input, u32 in_size, void *output, u32 out_size); + +static int call(struct m3_dcp_native *dcp, u32 tag, const void *in, u32 ins, + void *out, u32 outs, u32 completion) +{ + int ret; + + if (dcp->failed) + return -EIO; + dev_dbg(dcp->dev, "M3 kernel RPC %#x input=%u output=%u\n", tag, ins, outs); + ret = m3_dcp_bridge_call(dcp->bridge, tag, in, ins, out, outs, + completion, callback, dcp); + if (ret) { + dcp->failed = true; + dev_err(dcp->dev, "M3 kernel RPC %#x failed: %d; recovery requires reboot\n", tag, ret); + } + return ret; +} + +static int simple_call(struct m3_dcp_native *dcp, u32 tag, bool input, bool reply, bool expected) +{ + __le32 one = cpu_to_le32(1), result = 0; + int ret = call(dcp, tag, input ? &one : NULL, input ? 4 : 0, + reply ? &result : NULL, reply ? 4 : 0, 0); + + if (!ret && reply && le32_to_cpu(result) != expected) + return -EPROTO; + return ret; +} + +static struct m3_dcp_property *property(struct m3_dcp_native *dcp, + u32 service, const u8 *key, bool create) +{ + struct m3_dcp_property *p; + u32 i; + + if (!memchr(key, 0, 64)) + return ERR_PTR(-EINVAL); + for (i = 0; i < dcp->property_count; i++) { + p = &dcp->properties[i]; + if (p->service == service && !strcmp(p->key, key)) + return p; + } + if (!create) + return NULL; + if (dcp->property_count == MAX_PROPERTIES) + return ERR_PTR(-ENOSPC); + p = &dcp->properties[dcp->property_count++]; + p->service = service; + strscpy(p->key, key, sizeof(p->key)); + return p; +} + +static int raw_property(struct m3_dcp_native *dcp, u32 service, const u8 *key, + const void *data, u32 size) +{ + void *copy; + u32 i, old_size = 0; + + if (!memchr(key, 0, 64) || size > SZ_1M) + return -EINVAL; + for (i = 0; i < dcp->raw_count; i++) + if (dcp->raw[i].service == service && !strcmp(dcp->raw[i].key, key)) + break; + if (i == MAX_RAW_PROPERTIES) + return -ENOSPC; + if (i < dcp->raw_count) + old_size = dcp->raw[i].size; + if (dcp->raw_bytes - old_size + size > SZ_8M) + return -ENOSPC; + copy = kvmemdup(data, size ?: 1, GFP_KERNEL); + if (!copy) + return -ENOMEM; + if (i == dcp->raw_count) + dcp->raw_count++; + kvfree(dcp->raw[i].data); + dcp->raw[i].service = service; + strscpy(dcp->raw[i].key, key, 64); + dcp->raw[i].data = copy; + dcp->raw[i].size = size; + dcp->raw_bytes += size - old_size; + return 0; +} + +static int callback(struct m3_dcp_bridge *b, void *cookie, u32 tag, + const void *input, u32 in_size, void *output, u32 out_size) +{ + struct m3_dcp_native *dcp = cookie; + const u8 *in = input; + u8 *out = output; + struct m3_dcp_property *p; + struct m3_dcp_buffer buffer = {}; + u32 count, offset, service, key_offset; + int ret; + + dev_dbg(dcp->dev, "M3 kernel callback %#x input=%u output=%u\n", tag, in_size, out_size); +#define SHAPE(i, o) (in_size == (i) && out_size == (o)) + if (tag == D(209) && SHAPE(0, 8)) { + put_unaligned_le64(ktime_to_ms(ktime_get_real()), out); + return 0; + } + if (tag == D(596) && SHAPE(0, 4)) + return 0; /* No default framebuffer was allocated by this host. */ + /* J514S 1e5e14/1e5e9c: event, parameter, u32 value, nullable byte. + * The getter copies output[0] back and returns output[4] as a boolean. + * Match the M2 host: no display tiling, and no setter implementation. + */ + if ((tag == D(115) && SHAPE(16, 8)) || + (tag == D(116) && SHAPE(16, 4))) { + if (in[12] > 1) + return -EINVAL; + dev_info(dcp->dev, "M3 tiling callback %#x event=%u parameter=%u value=%u null=%u\n", + tag, get_unaligned_le32(in), get_unaligned_le32(in + 4), + get_unaligned_le32(in + 8), in[12]); + if (tag == D(115)) + out[4] = 1; + return 0; + } + /* The transport zeroes output, so absent optional metadata stays absent. */ + if (tag == D(114) && SHAPE(4164, 4100)) { + /* J514S CoreAnalyticsSendEvent: name[64], serialized dictionary[4096], + * nullable flag. Retain the latest event locally. The dictionary is + * in/out: firmware parses it even on failure, so a zero-filled reply + * is invalid. Return its own serialized dictionary without changes. + * The final u32 is the method status (zero = accepted). + */ + if (!memchr(in, 0, 64) || in[4160] > 1 || + (!in[4160] && in[64] != 'd')) + return -EINVAL; + memcpy(dcp->analytics_last, in, sizeof(dcp->analytics_last)); + dcp->analytics_count++; + if (!in[4160]) + memcpy(out, in + 64, 4096); + put_unaligned_le32(0, out + 4096); + dev_info(dcp->dev, "M3 analytics event %llu: %.64s (retained locally)\n", + dcp->analytics_count, in); + return 0; + } + if (tag == D(589) && SHAPE(0x6f0, 0)) { + memcpy(dcp->completion_last, in, sizeof(dcp->completion_last)); + dcp->completion_count++; + return 0; + } + if ((tag == D(588) && SHAPE(8, 0)) || + (tag == D(598) && SHAPE(0, 0))) + return 0; + if (tag == D(599) && SHAPE(0, 0)) + return simple_call(dcp, A(410), false, true, true); + if ((tag == D(108) || tag == D(109) || tag == D(110) || tag == D(111) || + tag == D(112) || tag == D(113) || tag == D(0) || tag == D(1)) && SHAPE(0, 4)) { + out[0] = 1; + return 0; + } + if (tag == D(576) && SHAPE(88, 76)) { + if (!(in[84] & 1)) + memcpy(out, in + 8, 76); + return 0; + } + if ((tag == D(577) && SHAPE(4, 0)) || (tag == D(300) && SHAPE(16, 0))) + return 0; + if (tag == D(127) && SHAPE(4, 4)) { + count = get_unaligned_le32(in); + if (dcp->chunk || !count || count > 0x100001) + return -EINVAL; + dcp->chunk_size = count - 1; + dcp->chunk_offset = 0; + dcp->chunk = kvzalloc(max_t(u32, 1, dcp->chunk_size), GFP_KERNEL); + if (!dcp->chunk) + return -ENOMEM; + out[0] = 1; + return 0; + } + if (tag == D(128) && SHAPE(0x1008, 4)) { + offset = get_unaligned_le32(in + 0x1000); + count = get_unaligned_le32(in + 0x1004); + if (!dcp->chunk || offset != dcp->chunk_offset || count > 4096 || + offset > dcp->chunk_size || count > dcp->chunk_size - offset) + return -EINVAL; + memcpy(dcp->chunk + offset, in, count); + dcp->chunk_offset += count; + out[0] = 1; + return 0; + } + if (tag == D(129) && SHAPE(64, 4)) { + if (!dcp->chunk || dcp->chunk_offset != dcp->chunk_size || !memchr(in, 0, 64) || + dcp->raw_count == MAX_RAW_PROPERTIES || dcp->raw_bytes + dcp->chunk_size > SZ_8M) + return -EINVAL; + ret = raw_property(dcp, 0, in, dcp->chunk, dcp->chunk_size); + if (ret) + return ret; + kvfree(dcp->chunk); + dcp->chunk = NULL; + dev_info(dcp->dev, "M3 property %.64s: %u bytes\n", in, dcp->chunk_size); + out[0] = 1; + return 0; + } + if (tag == D(408) && SHAPE(8, 8)) { + count = get_unaligned_le32(in + 4); + if (memcmp(in, "VORP", 4) || count > 1) + return -EINVAL; + /* J514S boot ADT: disp0 clock-ids 348/412 index arm-io's + * clock-frequencies at 92/156 (IDs minus 256): 712 MHz / 0. + * The 48 MHz reference-clock fallback makes updateFrequencies + * reject its input because it is below the 402.28 MHz video clock. + * Report the inherited metadata; do not program clock registers. + */ + put_unaligned_le64(count ? 0 : 712000000, out); + dev_info(dcp->dev, "M3 display clock %u: %llu Hz (boot ADT)\n", + count, get_unaligned_le64(out)); + return 0; + } + if (tag == D(125) && SHAPE(100, 36)) { + count = get_unaligned_le32(in + 64); + if (count > 8) + return -EINVAL; + memcpy(out, in + 68, count * 4); + return 0; + } + if (tag == D(6) && SHAPE(60, 56)) { + if (!(in[56] & 1)) + memcpy(out, in, 56); + return 0; + } + if ((tag == D(122) || tag == D(123)) && SHAPE(0, 4)) + return 0; + if (tag == D(121) && SHAPE(0, 4)) { + static const struct { u32 tag; bool in, reply, expected; } sequence[] = { + {A(444), false, true, true}, {A(29)}, + {A(466), true}, {A(0), true, true, true}, {A(463), false, true, true}, + }; + for (count = 0; count < ARRAY_SIZE(sequence); count++) { + ret = simple_call(dcp, sequence[count].tag, sequence[count].in, + sequence[count].reply, sequence[count].expected); + if (ret) + return ret; + } + out[0] = 1; + return 0; + } + if (tag == D(101) && SHAPE(0, 4)) { + put_unaligned_le32(dcp->stride, out); + return 0; + } + if (tag == D(3) && SHAPE(4, 60)) { + put_unaligned_le32(1, out + 56); /* Explicitly decline bandwidth service. */ + return 0; + } + if (tag == D(451) && SHAPE(20, 28)) { + u32 alignment = get_unaligned_le32(in + 12); + + dev_info(dcp->dev, "M3 allocation flags=%#x size=%llu alignment=%u null=%u/%u/%u\n", + get_unaligned_le32(in), get_unaligned_le64(in + 4), alignment, + in[16], in[17], in[18]); + if (get_unaligned_le32(in) != 0x703 || !is_power_of_2(alignment) || alignment > SZ_16K || + in[16] || in[17] || in[18]) + return -EINVAL; + buffer.size = get_unaligned_le64(in + 4); + ret = m3_dcp_bridge_alloc(b, &buffer); + if (ret) + return ret; + if ((buffer.physical | buffer.dva) & (alignment - 1)) + return -EINVAL; + put_unaligned_le64(buffer.physical, out); + put_unaligned_le64(buffer.dva, out + 8); + put_unaligned_le64(buffer.size, out + 16); + put_unaligned_le32(buffer.id, out + 24); + return 0; + } + if (tag == D(201) && SHAPE(12, 20)) { + u64 id = get_unaligned_le64(in); + + if (id > U32_MAX || get_unaligned_le32(in + 8) != 1) + return -EINVAL; + buffer.id = id; + ret = m3_dcp_bridge_map(b, &buffer); + if (ret) + return ret; + /* Same vaddr/dva/status layout as M2, verified at TEXT 0x18b330. + * The AP virtual-address output is unused by this host. + */ + put_unaligned_le64(0, out); + put_unaligned_le64(buffer.dva, out + 8); + put_unaligned_le32(0, out + 16); + return 0; + } + if (tag == D(454) && SHAPE(4, 4)) { + ret = m3_dcp_bridge_retire(b, get_unaligned_le32(in)); + if (!ret) + out[0] = 1; + return ret; + } + if (tag == D(582) && SHAPE(8, 4)) + return 0; /* Optional default surface is not allocated by this host. */ + if (tag == D(400) && SHAPE(76, 0xc04)) { + count = get_unaligned_le32(in + 68); + service = get_unaligned_le32(in); + if (count > 0xc00 || (in[72] & 1) || !memchr(in + 4, 0, 64)) + return -EINVAL; + for (offset = 0; offset < dcp->raw_count; offset++) { + if (dcp->raw[offset].service != service || strcmp(dcp->raw[offset].key, in + 4)) + continue; + if (dcp->raw[offset].size > count) + return -ENOSPC; + memcpy(out, dcp->raw[offset].data, dcp->raw[offset].size); + put_unaligned_le32(dcp->raw[offset].size, out + 0xc00); + break; + } + return 0; + } + if (tag == D(401) && SHAPE(80, 12)) { + /* Match the existing M2 mini-LED host's PMUS compatibility + * response. This is a fixed calibration placeholder, NOT a + * measured M3 temperature and never a thermal-guard input. + * Firmware 19a4c4..19a4e8 converts centidegrees to 16.16. + */ + if (!memcmp(in, "SUMP", 4) && + !strncmp(in + 4, "Temperature", 64)) { + put_unaligned_le64(3029, out); + out[8] = 1; + dev_info_ratelimited(dcp->dev, + "M3 PMUS Temperature: M2 compatibility placeholder 3029\n"); + return 0; + } + p = property(dcp, get_unaligned_le32(in), in + 4, false); + if (IS_ERR(p)) + return PTR_ERR(p); + if (p) { + put_unaligned_le64(p->value, out); + out[8] = 1; + } else + dev_info_ratelimited(dcp->dev, + "M3 missing host property service=%#x key=%.64s\n", + get_unaligned_le32(in), in + 4); + return 0; + } + if ((tag == D(413) && SHAPE(4168, 4)) || + ((tag == D(552) || tag == D(561)) && SHAPE(4164, 4)) || + (tag == D(567) && SHAPE(128, 4))) { + key_offset = tag == D(413) ? 4 : 0; + service = key_offset ? get_unaligned_le32(in) : 0; + if (tag != D(567) && (in[key_offset + 4160] & 1)) { + out[0] = 1; + return 0; + } + count = tag == D(567) ? strnlen(in + 64, 64) : 4096; + ret = raw_property(dcp, service, in + key_offset, in + key_offset + 64, count); + if (!ret) + out[0] = 1; + return ret; + } + if ((tag == D(563) && SHAPE(76, 4)) || (tag == D(565) && SHAPE(72, 4)) || + (tag == D(414) && SHAPE(80, 4)) || (tag == D(415) && SHAPE(76, 4)) || + ((tag == D(102) || tag == D(104)) && SHAPE(68, 0))) { + key_offset = (tag == D(414) || tag == D(415)) ? 4 : 0; + service = key_offset ? get_unaligned_le32(in) : 0; + count = (tag == D(563) || tag == D(414)) ? 8 : 4; + if (out_size && (in[key_offset + 64 + count] & 1)) { + out[0] = 1; + return 0; + } + p = property(dcp, service, in + key_offset, true); + if (IS_ERR(p)) + return PTR_ERR(p); + p->value = count == 8 ? get_unaligned_le64(in + key_offset + 64) : + get_unaligned_le32(in + key_offset + 64); + if (tag == D(104)) + p->value = in[64]; + if (out_size) + out[0] = 1; + return 0; + } + if (tag == D(406) && SHAPE(72, 0)) + return 0; + if (tag == D(107) && SHAPE(64, 0)) { + p = property(dcp, 0, in, false); + if (IS_ERR(p)) + return PTR_ERR(p); + if (p) { + *p = dcp->properties[--dcp->property_count]; + memset(&dcp->properties[dcp->property_count], 0, sizeof(*p)); + } + return 0; + } + if ((tag == D(206) || tag == D(207)) && SHAPE(0, 4)) { + ret = simple_call(dcp, tag == D(206) ? A(131) : A(132), false, true, false); + if (!ret) + out[0] = 1; + return ret; + } + if (tag == D(100) && SHAPE(0, 0)) + { + __le32 result; + return call(dcp, A(374), NULL, 0, &result, 4, 0); + } + dev_err(dcp->dev, "M3 unhandled kernel callback %#x %u/%u\n", tag, in_size, out_size); + return -EOPNOTSUPP; +#undef SHAPE +} + +int m3_dcp_native_open(struct m3_dcp_native *dcp) +{ + int ret; + + mutex_lock(&dcp->lock); + if (dcp->opened) { + ret = -EALREADY; + } else { + ret = simple_call(dcp, A(455), false, false, false); + if (!ret) { + dcp->opened = true; + dev_info(dcp->dev, "M3 first-client-open complete\n"); + } + } + mutex_unlock(&dcp->lock); + return ret; +} + +struct m3_dcp_native *m3_dcp_native_start(struct device *dev, struct m3_dcp_bridge *bridge, + bool defer_open) +{ + struct m3_dcp_native *dcp; + struct device_node *fb; + int ret; + + dcp = devm_kzalloc(dev, sizeof(*dcp), GFP_KERNEL); + if (!dcp) + return ERR_PTR(-ENOMEM); + dcp->dev = dev; + dcp->bridge = bridge; + mutex_init(&dcp->lock); + INIT_WORK(&dcp->idle_work, idle_callbacks); + fb = of_find_compatible_node(NULL, NULL, "simple-framebuffer"); + ret = fb ? of_property_read_u32(fb, "stride", &dcp->stride) : -ENODEV; + of_node_put(fb); + if (ret || dcp->stride != 12096) + return ERR_PTR(-EINVAL); + ret = simple_call(dcp, A(401), false, true, true); + if (!ret && !defer_open) + ret = m3_dcp_native_open(dcp); + if (ret) + return ERR_PTR(ret); + m3_dcp_bridge_set_work(bridge, &dcp->idle_work); + if (defer_open) + dev_info(dev, "M3 DCP kernel startup complete; first-client-open deferred\n"); + else + dev_info(dev, "M3 DCP kernel startup and first-client-open complete\n"); + return dcp; +} + +/* J514S wrappers: A409@13f7d4 (4/4), A472@1421f4 (12/8), + * A411@13f8a8 (8/4). A472's output status is at byte 4; bytes 9/10 + * are additional boolean inputs and byte 11 is the output-pointer null flag. + * Follow M2's display-handle 0, then power-state 1 sequence explicitly. + */ +int m3_dcp_native_panel(struct m3_dcp_native *dcp, unsigned int action) +{ + u8 input[80] = {}, output[8] = {}; + int ret; + + mutex_lock(&dcp->lock); + switch (action) { + case 1: + dcp->brightness_valid = false; + ret = call(dcp, A(409), input, 4, output, 4, 0); + /* Existing internal-display handle 0 returns 2. */ + if (!ret && get_unaligned_le32(output) != 2) + ret = -EIO; + if (!ret) { + put_unaligned_le64(1, input); + ret = call(dcp, A(472), input, 12, output, 8, 0); + if (!ret && get_unaligned_le32(output + 4)) + ret = -EIO; + } + break; + case 2: + /* PreferredTimingElements: color 1, timing 2, 3024x1964. */ + put_unaligned_le32(1, input); + put_unaligned_le32(2, input + 4); + ret = call(dcp, A(411), input, 8, output, 4, 0); + if (!ret && get_unaligned_le32(output)) + ret = -EIO; + break; + case 3: + ret = call(dcp, A(479), input, 4, output, 8, 0); + break; + case 4: + /* Native stub kernel ab5a53c; firmware wrapper 140fe4. + * Same initialization used by the M2 host before client open. + */ + ret = call(dcp, A(448), input, 4, output, 4, 0); + if (!ret && get_unaligned_le32(output)) + ret = -EIO; + break; + case 5: + /* Native stub kernel ab57948; firmware wrapper 13fe2c. + * Location 9 is the identity CTM used by the M2 KMS path. + */ + put_unaligned_le32(9, input); + put_unaligned_le64(1ULL << 32, input + 4); + put_unaligned_le64(1ULL << 32, input + 4 + 4 * 8); + put_unaligned_le64(1ULL << 32, input + 4 + 8 * 8); + ret = call(dcp, A(421), input, 80, output, 4, 0); + if (!ret && get_unaligned_le32(output)) + ret = -EIO; + break; + case 6: + case 7: + /* Reversible diagnostic: M2's linear primary uses IOMFB plane 2. + * The next swap clears the other two primary slots atomically. + */ + dcp->diagnostic_plane = action == 6 ? 2 : 0; + put_unaligned_le32(dcp->diagnostic_plane, output); + ret = 0; + break; + case 8: + case 9: + /* Captured 26A428 linear BGRA uses the non-planar IOSurface + * representation. SurfaceContents fields match the stub driver. + */ + dcp->diagnostic_flat_surface = action == 8; + ret = 0; + break; + default: + ret = -EINVAL; + } + dev_info(dcp->dev, "M3 panel action %u ret=%d output=%#x/%#x\n", + action, ret, get_unaligned_le32(output), get_unaligned_le32(output + 4)); + if (ret) + dcp->failed = true; + mutex_unlock(&dcp->lock); + return ret; +} + +static int native_swap(struct m3_dcp_native *dcp, const void *surface, u64 dva, + u32 width, u32 height, u32 background, int brightness_nits) +{ + __le32 start[4] = {}, started[2], result[3]; + u8 *swap; + u32 id; + int ret; + + if (brightness_nits < -1 || brightness_nits > M3_PANEL_MAX_NITS) + return -EINVAL; + if (surface && (!dva || !width || width > 3024 || !height || height > 1964)) + return -EINVAL; + swap = kzalloc(0x1b58, GFP_KERNEL); + if (!swap) + return -ENOMEM; + mutex_lock(&dcp->lock); + ret = call(dcp, A(406), start, sizeof(start), started, sizeof(started), 0); + if (ret || le32_to_cpu(started[1])) { + ret = ret ?: -EIO; + goto out; + } + id = le32_to_cpu(started[0]); + if (!id) { + ret = -EPROTO; + goto out; + } + put_unaligned_le32(id, swap + 0x98); + put_unaligned_le32(0x80000007, swap + 0x14c); + put_unaligned_le32(0x80000007, swap + 0x150); + /* A disabled plane must blank to black, including compositor DPMS. */ + put_unaligned_le32(background, swap + 0x154); + memset(swap + 0x1b4b, 1, 10); + swap[0x1b56] = swap[0x1b57] = 1; + if (brightness_nits >= 0 && (!dcp->brightness_valid || + dcp->brightness_nits != brightness_nits)) + m3_dcp_encode_brightness(swap, brightness_nits); + if (surface) { + u32 plane = dcp->diagnostic_plane; + u8 *s = swap + 0x508 + plane * 0x22c; + + memcpy(s, surface, 0x22c); + if (dcp->diagnostic_flat_surface) { + put_unaligned_le32(0, s + 3); + put_unaligned_le32(0, s + 7); + s[0x14] = 1; /* Captured sRGB BGRA surface. */ + put_unaligned_le16(4, s + 0x19); + memset(s + 0x39, 0, 0x1ed - 0x39); + put_unaligned_le64(1, s + 0x51); + put_unaligned_le64(1, s + 0x149); + } + put_unaligned_le64(dva, swap + 0xdb8 + plane * 8); + put_unaligned_le32(1, swap + 0x9c + plane * 4); + put_unaligned_le32(width, swap + 0xb4 + plane * 16); + put_unaligned_le32(height, swap + 0xb8 + plane * 16); + put_unaligned_le32(width, swap + 0x114 + plane * 16); + put_unaligned_le32(height, swap + 0x118 + plane * 16); + swap[0x1b4b + plane] = 0; + } + ret = call(dcp, A(407), swap, 0x1b58, result, sizeof(result), id); + if (!ret && get_unaligned_le32((u8 *)result + 5)) + ret = -EIO; + /* Cache only after the matching frame completion and successful reply. */ + if (!ret && brightness_nits >= 0) { + dcp->brightness_nits = brightness_nits; + dcp->brightness_valid = true; + } + if (!ret && !surface) + dev_info(dcp->dev, "M3 kernel swap %u completed, surface=%d\n", id, !!surface); +out: + if (ret) + dcp->failed = true; + mutex_unlock(&dcp->lock); + kfree(swap); + return ret; +} + +int m3_dcp_native_swap(struct m3_dcp_native *dcp, const void *surface, u64 dva, + u32 width, u32 height, int brightness_nits) +{ + return native_swap(dcp, surface, dva, width, height, 0xff000000, brightness_nits); +} + +int m3_dcp_native_background(struct m3_dcp_native *dcp, u32 color) +{ + return native_swap(dcp, NULL, 0, 0, 0, color, -1); +} + +void m3_dcp_native_brightness_invalidate(struct m3_dcp_native *dcp) +{ + mutex_lock(&dcp->lock); + dcp->brightness_valid = false; + mutex_unlock(&dcp->lock); +} + +void *m3_dcp_native_property(struct m3_dcp_native *dcp, const char *key, u32 *size) +{ + void *copy = NULL; + u32 i; + + mutex_lock(&dcp->lock); + for (i = 0; i < dcp->raw_count; i++) { + if (!dcp->raw[i].service && !strcmp(dcp->raw[i].key, key)) { + *size = dcp->raw[i].size; + copy = kmemdup(dcp->raw[i].data, *size, GFP_KERNEL); + break; + } + } + mutex_unlock(&dcp->lock); + return copy; +} + +void m3_dcp_native_analytics_show(struct m3_dcp_native *dcp, struct seq_file *seq) +{ + mutex_lock(&dcp->lock); + seq_printf(seq, "events %llu\n", dcp->analytics_count); + if (dcp->analytics_count) { + seq_printf(seq, "last_event %.64s\ndictionary_present %u\n", + dcp->analytics_last, !dcp->analytics_last[4160]); + if (!dcp->analytics_last[4160]) + seq_hex_dump(seq, "", DUMP_PREFIX_OFFSET, 16, 1, + dcp->analytics_last + 64, 4096, false); + } + mutex_unlock(&dcp->lock); +} + +void m3_dcp_native_completion_show(struct m3_dcp_native *dcp, struct seq_file *seq) +{ + mutex_lock(&dcp->lock); + seq_printf(seq, "callbacks %llu\n", dcp->completion_count); + if (dcp->completion_count) + seq_hex_dump(seq, "", DUMP_PREFIX_OFFSET, 16, 1, + dcp->completion_last, sizeof(dcp->completion_last), false); + mutex_unlock(&dcp->lock); +} + +static void idle_callbacks(struct work_struct *work) +{ + struct m3_dcp_native *dcp = container_of(work, struct m3_dcp_native, idle_work); + int ret = 0; + + /* Share RPC ownership with atomic flips; this worker never steals an + * in-flight request's reply. Firmware notifications also need service + * when the desktop is static or the CRTC is disabled. */ + mutex_lock(&dcp->lock); + while (!dcp->failed && !ret) + ret = m3_dcp_bridge_pump(dcp->bridge, 0, callback, dcp); + if (ret && ret != -ETIMEDOUT && ret != -EAGAIN) { + dcp->failed = true; + dev_err(dcp->dev, "M3 idle callback failed %d; recovery requires reboot\n", ret); + } + mutex_unlock(&dcp->lock); +} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_native.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_native.h new file mode 100644 index 0000000000000..7516ed8a1df5f --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_native.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef M3_DCP_NATIVE_H +#define M3_DCP_NATIVE_H + +#define M3_DCP_TAG(c, n) (((u32)(c) << 24) | ((u32)('0' + (n) / 100) << 16) | \ + ((u32)('0' + (n) / 10 % 10) << 8) | (u32)('0' + (n) % 10)) +struct m3_dcp_native; +struct m3_dcp_bridge; +struct device; +struct seq_file; +void m3_dcp_native_completion_show(struct m3_dcp_native *dcp, struct seq_file *seq); +void m3_dcp_native_analytics_show(struct m3_dcp_native *dcp, struct seq_file *seq); +struct m3_dcp_native *m3_dcp_native_start(struct device *dev, struct m3_dcp_bridge *bridge, + bool defer_open); +int m3_dcp_native_open(struct m3_dcp_native *dcp); +int m3_dcp_native_swap(struct m3_dcp_native *dcp, const void *surface, u64 dva, + u32 width, u32 height, int brightness_nits); +void m3_dcp_native_brightness_invalidate(struct m3_dcp_native *dcp); +int m3_dcp_native_background(struct m3_dcp_native *dcp, u32 color); +int m3_dcp_native_panel(struct m3_dcp_native *dcp, unsigned int action); +/* Caller owns the returned snapshot and must kfree it. */ +void *m3_dcp_native_property(struct m3_dcp_native *dcp, const char *key, u32 *size); +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_parser.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_parser.c new file mode 100644 index 0000000000000..676c129209d9a --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_parser.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Reuse the existing serialized-property decoder without an apple_dcp object. */ +#include "m3_dcp_kms.h" +#define parse m3_property_parse +#define enumerate_modes m3_property_enumerate_modes +#define parse_display_attributes m3_property_display_attributes +#define parse_epic_service_init m3_property_epic_service_init +#define parse_sound_constraints m3_property_sound_constraints +#define parse_sound_mode m3_property_sound_mode +#define parse_system_log_mnits m3_property_system_log_mnits +#include "../../trace.h" +#define trace_iomfb_color_mode(...) do { } while (0) +#define trace_iomfb_timing_mode(...) do { } while (0) +#define trace_avep_sound_mode(...) do { } while (0) +#include "../../parser.c" + +int m3_dcp_preferred_mode(const void *blob, u32 size, struct drm_display_mode *mode) +{ + struct dcp_parse_ctx ctx = {}; + struct dcp_display_mode *modes; + unsigned int count; + int ret; + + ret = m3_property_parse(blob, size, &ctx); + if (ret) + return ret; + modes = m3_property_enumerate_modes(&ctx, &count, 0, 0, 0); + if (IS_ERR(modes)) + return PTR_ERR(modes); + ret = -EINVAL; + /* The qualified J514S session has one preferred physical panel timing. */ + if (count == 1 && modes[0].mode.hdisplay == 3024 && modes[0].mode.vdisplay == 1964 && + (drm_mode_vrefresh(&modes[0].mode) == 60 || drm_mode_vrefresh(&modes[0].mode) == 120)) { + *mode = modes[0].mode; + ret = 0; + } + kfree(modes); + return ret; +} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_ptd_vote.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_ptd_vote.c new file mode 100644 index 0000000000000..3edb157331f8c --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_ptd_vote.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* J514S PMP v2 display request, qualified by macOS pmgr-76 and ADT. + * Initialization diagnostic only, under the lifecycle watchdog. Reboot to + * discard the request. Never submit display or GPU work from this module. + */ +#include +#include +#include +#include +#include + +#ifndef M3_PTD_INSTANCE +#define M3_PTD_INSTANCE 0 +#endif +static bool external = M3_PTD_INSTANCE != 0; +module_param(external, bool, 0400); +MODULE_PARM_DESC(external, "Also request qualified J514S DISPEXT0; retain until reboot"); + +static int __init vote(void) +{ + struct device_node *node; + const u8 *table; + void __iomem *ptd; + u64 request, ack, mask, ack_mask = BIT_ULL(7); + int length, i, found = 0; + + pr_info("M3 DCP PTD diagnostic entered; no register access yet\n"); + if (!of_machine_is_compatible("apple,j514s") || + !of_machine_is_compatible("apple,t6030")) + return -ENODEV; + node = of_find_node_by_path("/soc/pmp@350500000"); + if (!node) + return -ENODEV; + table = of_get_property(node, "apple,tunable-soc-device", &length); + if (!table || length != 21 * 0x7c || + get_unaligned_le32(table + 7 * 0x7c) != 8 || + memcmp(table + 7 * 0x7c + 0x74, "DISP\0\0\0\0", 8)) + goto invalid; + /* macOS also votes for the in-use ANS controller (bit16). Without + * that dependency, firmware DVFS may stop storage before logs flush. + */ + if (get_unaligned_le32(table + 16 * 0x7c) != 17 || + memcmp(table + 16 * 0x7c + 0x74, "ANS\0\0\0\0\0", 8)) + goto invalid; + if (external) { + /* J514S retained boot ADT, distinct from DISP at index7. The + * power protocol is shared; do not copy macOS26's device indices. */ + unsigned int index = 8 + M3_PTD_INSTANCE; + if (get_unaligned_le32(table + index * 0x7c) != index + 1 || + memcmp(table + index * 0x7c + 0x74, M3_PTD_INSTANCE ? "DISPEXT1" : "DISPEXT0", 8)) + goto invalid; + ack_mask |= BIT_ULL(index); + } + table = of_get_property(node, "apple,tunable-ptd-range", &length); + if (!table || length % 32) + goto invalid; + for (i = 0; i < length; i += 32) { + u32 id = get_unaligned_le32(table + i); + if (id != 10 && id != 11) + continue; + if (get_unaligned_le32(table + i + 4) != (id == 10 ? 280 : 284) || + get_unaligned_le32(table + i + 8) != 4) + goto invalid; + found |= id == 10 ? 1 : 2; + } + if (found != 3) + goto invalid; + of_node_put(node); + pr_info("M3 DCP PTD display/ANS and range tables validated\n"); + ptd = ioremap_np(0x3503c0000ULL, 0x14000); + if (!ptd) + return -ENOMEM; + pr_info("M3 DCP PTD mapping ready; reading status\n"); + msleep(100); + for (i = 0; i < 100 && readq(ptd + 16) != 1; i++) + usleep_range(1000, 2000); + if (readq(ptd + 16) != 1) { + iounmap(ptd); + return -EAGAIN; + } + pr_info("M3 DCP PTD status ready; reading request/ack\n"); + msleep(100); + request = readq(ptd + 280 * 16); + ack = readq(ptd + 284 * 16); + pr_info("M3 DCP PTD display before request=%#llx ack=%#llx\n", request, ack); + /* Preserve every other client bit. No other diagnostic writer may run. */ + /* DISP flags0xb require acknowledgement; ANS flags0 do not. Match + * ApplePMGR's flags-bit1 check, rather than requiring ANS in the ACK. + */ + if ((request ^ ack) & ack_mask) { + iounmap(ptd); + return -EBUSY; + } + mask = ack_mask | BIT_ULL(16); + pr_info("M3 DCP PTD writing display/ANS request through non-posted mapping\n"); + msleep(100); + writeq(request | mask, ptd + 0x10000 + 280 * 8); + for (i = 0; i < 100; i++) { + ack = readq(ptd + 284 * 16); + if ((ack & ack_mask) == ack_mask) + break; + usleep_range(1000, 2000); + } + pr_info("M3 DCP PTD display after request=%#llx ack=%#llx polls=%d\n", + readq(ptd + 280 * 16), ack, i); + iounmap(ptd); + return (ack & ack_mask) == ack_mask ? 0 : -ETIMEDOUT; +invalid: + of_node_put(node); + return -EINVAL; +} +module_init(vote); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("J514S isolated PMP v2 PTD display request; reboot to remove"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_query.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_query.c new file mode 100644 index 0000000000000..18476a9659958 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_query.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Repeat the already-qualified A410 query under an external watchdog. */ +#include +#include +#include +#include +#include "../../dcp-link.h" +#include "m3_dcp_state.h" +static bool dump; +module_param(dump, bool, 0400); + +static int __init query(void) +{ + struct device_node *node; + struct platform_device *pdev; + struct m3_dcp_rtkit *dcp; + int ret; + + if (!of_machine_is_compatible("apple,j514s")) + return -ENODEV; + node = of_find_node_by_path("/soc/dcp@28ec00000"); + pdev = of_find_device_by_node(node); + of_node_put(node); + if (!pdev) + return -ENODEV; + if (!pdev->dev.driver || strcmp(pdev->dev.driver->name, "m3-dcp-rtkit")) { + ret = -ENODEV; + goto out; + } + dcp = platform_get_drvdata(pdev); + if (dump && dcp && dcp->rpc) { + for (unsigned int offset = 0; offset < 0x80000; offset += 0x20000) { + pr_info("M3 DCP RPC snapshot offset=%#x\n", offset); + print_hex_dump(KERN_INFO, "M3 RPC ", DUMP_PREFIX_OFFSET, 16, 1, + dcp->rpc + offset, 256, false); + } + ret = 0; + goto out; + } + if (!dcp || dcp->result || dcp->awaiting_rpc || !dcp->rpc) { + ret = -EBUSY; + goto out; + } + reinit_completion(&dcp->ready); + apple_dcp_link_rpc_header_encode(dcp->rpc, 0x41343130, 0, 4); + *(__le32 *)(dcp->rpc + 12) = cpu_to_le32(0xa5); + dcp->result = -ETIMEDOUT; + dcp->awaiting_rpc = true; + dma_wmb(); + ret = apple_rtkit_send_message(dcp->rtkit, APPLE_DCP_LINK_ENDPOINT, + (16ULL << 32) | APPLE_DCP_LINK_MSG_RPC, NULL, false); + if (!ret) + ret = wait_for_completion_timeout(&dcp->ready, msecs_to_jiffies(3000)) ? + dcp->result : -ETIMEDOUT; + pr_info("M3 repeated A410 query ret=%d value=%u\n", ret, + le32_to_cpup((__le32 *)(dcp->rpc + 12))); +out: + put_device(&pdev->dev); + return ret; +} +static void __exit done(void) {} +module_init(query); +module_exit(done); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("J514S one-shot query of an existing DCP diagnostic session"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_regs.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_regs.c new file mode 100644 index 0000000000000..37a37cfaa5658 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_regs.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* J514S readl-only diagnostic: never use vector loads on DART registers. */ +#include +#include +#include +#include +static int __init probe(void) +{ + const phys_addr_t bases[] = {0x28d30c000ULL, 0x28d304000ULL}; + const unsigned int sid[] = {5, 0, 4}; + if (!of_machine_is_compatible("apple,j514s")) return -ENODEV; + for (unsigned int i=0; i<3; i++) { + phys_addr_t base=bases[i != 0]; + const unsigned int offsets[] = {0x200,0x1000+4*sid[i],0x1400+4*sid[i]}; + void __iomem *regs=ioremap(base,0x4000); + if (!regs) return -ENOMEM; + for (unsigned int n=0; n<3; n++) { + pr_info("M3 DCP reading %pap + %#x using readl\n", &base, offsets[n]); + msleep(100); + pr_info("M3 DCP register %pap + %#x = %#x\n", &base, offsets[n], readl(regs+offsets[n])); + } + iounmap(regs); + } + { + /* Display clock registers from J514S ADT clock-frequencies-regs entries + * 92/156, plus the three real display power-state registers. Parent + * LW00/LS0/IOA0/AFI and display leaf entries have no_ps: do not map them. + */ + const phys_addr_t addresses[] = {0x28ec00044ULL,0x28ec00048ULL, + 0x3507001c0ULL,0x350700258ULL,0x350710000ULL, + 0x350040064ULL,0x350040164ULL,0x28e850000ULL,0x28e850004ULL}; + for (unsigned int i=0; i +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../dcp-link.h" + +#include "m3_dcp_state.h" +#include "m3_dcp_bridge.h" +#include "m3_dcp_native.h" +#include "m3_dcp_kms.h" +#include +#include + +static bool kernel_client; +module_param(kernel_client, bool, 0400); +static bool defer_client_open; +module_param(defer_client_open, bool, 0400); +MODULE_PARM_DESC(defer_client_open, "Pause native startup before A455 for staged takeover diagnosis"); + +static struct m3_dcp_rtkit *active; +static DEFINE_MUTEX(control_lock); +static bool kms_registered; +static bool client_opened; +static void save_properties(struct m3_dcp_rtkit *dcp); +static int control_set(const char *value, const struct kernel_param *param) +{ + int ret; + + guard(mutex)(&control_lock); + if (!active || IS_ERR_OR_NULL(active->native) || active->result) + return -EBUSY; + if (sysfs_streq(value, "client-open")) { + if (client_opened) + return -EALREADY; + ret = m3_dcp_native_open(active->native); + if (!ret) { + client_opened = true; + save_properties(active); + } + } else if (!client_opened) { + return -EAGAIN; + } else if (sysfs_streq(value, "background")) { + ret = m3_dcp_native_swap(active->native, NULL, 0, 0, 0, -1); + } else if (sysfs_streq(value, "background-green")) { + ret = m3_dcp_native_background(active->native, 0xff00ff00); + } else if (sysfs_streq(value, "panel-on")) { + ret = m3_dcp_native_panel(active->native, 1); + } else if (sysfs_streq(value, "panel-mode")) { + ret = m3_dcp_native_panel(active->native, 2); + } else if (sysfs_streq(value, "panel-state")) { + ret = m3_dcp_native_panel(active->native, 3); + } else if (sysfs_streq(value, "panel-no-idle")) { + ret = m3_dcp_native_panel(active->native, 4); + } else if (sysfs_streq(value, "panel-identity")) { + ret = m3_dcp_native_panel(active->native, 5); + } else if (sysfs_streq(value, "panel-plane2")) { + ret = m3_dcp_native_panel(active->native, 6); + } else if (sysfs_streq(value, "panel-plane0")) { + ret = m3_dcp_native_panel(active->native, 7); + } else if (sysfs_streq(value, "panel-flat")) { + ret = m3_dcp_native_panel(active->native, 8); + } else if (sysfs_streq(value, "panel-planar")) { + ret = m3_dcp_native_panel(active->native, 9); + } else if (sysfs_streq(value, "kms")) { + if (kms_registered) + return -EALREADY; + ret = m3_dcp_kms_register(active->native, true); + if (!ret) + kms_registered = true; + } else { + return -EINVAL; + } + active->result = ret; + return ret; +} +static const struct kernel_param_ops control_ops = { .set = control_set }; +module_param_cb(control, &control_ops, NULL, 0200); +MODULE_PARM_DESC(control, "Explicit guarded diagnostics after native startup: background or kms"); + +static void save_properties(struct m3_dcp_rtkit *dcp) +{ + const char *names[] = { "PreferredTimingElements", "DisplayAttributes" }; + struct dentry *dir = debugfs_create_dir("m3-dcp-properties", NULL); + + for (unsigned int i = 0; i < ARRAY_SIZE(names); i++) { + struct debugfs_blob_wrapper *blob = devm_kzalloc(dcp->dev, sizeof(*blob), GFP_KERNEL); + u32 size; + void *data; + + if (!blob) + continue; + data = m3_dcp_native_property(dcp->native, names[i], &size); + if (!data) + continue; + /* Immutable startup snapshots; this diagnostic remains pinned. */ + blob->data = data; + blob->size = size; + debugfs_create_blob(names[i], 0400, dir, blob); + } +} + +static bool start_link; +module_param(start_link, bool, 0400); +MODULE_PARM_DESC(start_link, "Also negotiate DCPLink, without issuing display RPCs"); +static bool query_display; +module_param(query_display, bool, 0400); +MODULE_PARM_DESC(query_display, "Issue the M3 A410 read-only main-display query after READY"); +static bool power_vote = true; +module_param(power_vote, bool, 0400); +MODULE_PARM_DESC(power_vote, "Require PMP DISP vote; false permits only the watchdog-bounded startup/query diagnostic before PMP"); +static void crashed(void *cookie, const void *log, size_t size) +{ + struct m3_dcp_rtkit *dcp = cookie; + + dcp->result = -EIO; + dev_err(dcp->dev, "DCP firmware crashed; retain all DMA allocations until reboot\n"); + if (dcp->bridge) + m3_dcp_bridge_crashed(dcp->bridge); + complete(&dcp->ready); +} + +static void receive(void *cookie, u8 ep, u64 message) +{ + struct m3_dcp_rtkit *dcp = cookie; + u32 alignment; + + dev_dbg(dcp->dev, "DCP app rx ep=%#x message=%#llx\n", ep, message); + if (ep != APPLE_DCP_LINK_ENDPOINT) + return; + if (dcp->bridge) { + m3_dcp_bridge_receive(dcp->bridge, message); + return; + } + if (dcp->awaiting_rpc) { + u32 output; + + dma_rmb(); + output = le32_to_cpup((__le32 *)(dcp->rpc + 12)); + if (message != APPLE_DCP_LINK_MSG_RPC_REPLY || output > 1) { + dcp->result = -EPROTO; + } else { + dcp->result = 0; + dev_info(dcp->dev, "M3 A410 reply: is_main_display=%u\n", output); + } + dcp->awaiting_rpc = false; + complete(&dcp->ready); + return; + } + if ((message & 3) == APPLE_DCP_LINK_MSG_FIRMWARE_INIT) + return; + /* J514S TEXT 0x1178b8: READY encodes alignment at bit 16, + * with no firmware hash. 0x1161c0 supplies the firmware side bit. + */ + alignment = message >> 16; + if (message != ((64ULL << 16) | 0x101)) { + dcp->result = -EPROTO; + } else { + dcp->result = 0; + dev_info(dcp->dev, "M3 DCPLink READY alignment=%u\n", alignment); + } + complete(&dcp->ready); +} + +static int shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct m3_dcp_rtkit *dcp = cookie; + struct iommu_domain *domain = iommu_get_domain_for_dev(dcp->dev); + phys_addr_t phys; + + if (!bfr->size || bfr->size > SZ_16M || !domain) + return -EINVAL; + if (!bfr->iova) { + bfr->buffer = dma_alloc_coherent(dcp->dev, bfr->size, &bfr->iova, GFP_KERNEL); + if (!bfr->buffer) + return -ENOMEM; + memset(bfr->buffer, 0, bfr->size); + dma_wmb(); + dev_info(dcp->dev, "DCP shmem allocated size=%#zx dva=%pad\n", + bfr->size, &bfr->iova); + return 0; + } + phys = iommu_iova_to_phys(domain, bfr->iova); + /* Restrict firmware-supplied addresses to a declared reserved region. */ + for (int i = 0; ; i++) { + struct device_node *node = of_parse_phandle(dcp->dev->of_node, "memory-region", i); + struct resource res; + bool oslog, valid; + + if (!node) + return -ERANGE; + oslog = of_property_read_bool(node, "apple,dcp-os-log"); + valid = !of_address_to_resource(node, 0, &res); + of_node_put(node); + if (!valid) + continue; + /* The OS-log carveout is a host physical buffer, not a DART alias. */ + if (oslog && bfr->iova == res.start) + phys = res.start; + if (phys < res.start || phys > res.end || bfr->size - 1 > res.end - phys) + continue; + if (!oslog) { + for (size_t off = 0; off < bfr->size; off += SZ_16K) + if (iommu_iova_to_phys(domain, bfr->iova + off) != phys + off) + return -ERANGE; + if (iommu_iova_to_phys(domain, bfr->iova + bfr->size - 1) != phys + bfr->size - 1) + return -ERANGE; + } + bfr->buffer = memremap(phys, bfr->size, MEMREMAP_WB); + if (!bfr->buffer) + return -ENOMEM; + bfr->is_mapped = true; + dev_info(dcp->dev, "DCP shmem inherited size=%#zx dva=%pad phys=%pap\n", + bfr->size, &bfr->iova, &phys); + return 0; + } +} + +static void shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct m3_dcp_rtkit *dcp = cookie; + + if (bfr->is_mapped) + memunmap(bfr->buffer); + else + dma_free_coherent(dcp->dev, bfr->size, bfr->buffer, bfr->iova); +} + +static const struct apple_rtkit_ops ops = { + .crashed = crashed, + .recv_message = receive, + .shmem_setup = shmem_setup, + .shmem_destroy = shmem_destroy, +}; + +/* PMP v2's legacy device-power mailbox only acknowledges the message on + * this firmware. The lifecycle must seed the PTD request before PMP starts. + * Verify the request and firmware acknowledgement without changing votes. + */ +static int keep_display_power(struct device *dev) +{ + struct device_node *node = of_find_node_by_path("/soc/pmp@350500000"); + const u8 *table; + void __iomem *ptd; + u64 status, request, ack; + int length, i, found = 0; + + if (!node) + return -ENODEV; + table = of_get_property(node, "apple,tunable-soc-device", &length); + if (!table || length != 21 * 0x7c || + get_unaligned_le32(table + 7 * 0x7c) != 8 || + !(get_unaligned_le32(table + 7 * 0x7c + 8) & BIT(1)) || + memcmp(table + 7 * 0x7c + 0x74, "DISP\0\0\0\0", 8)) + goto invalid; + table = of_get_property(node, "apple,tunable-ptd-range", &length); + if (!table || length % 32) + goto invalid; + for (i = 0; i < length; i += 32) { + u32 id = get_unaligned_le32(table + i); + + if (id != 10 && id != 11) + continue; + if (get_unaligned_le32(table + i + 4) != (id == 10 ? 280 : 284) || + get_unaligned_le32(table + i + 8) != 4) + goto invalid; + found |= id == 10 ? 1 : 2; + } + if (found != 3) + goto invalid; + of_node_put(node); + ptd = ioremap_np(0x3503c0000ULL, 0x4000); + if (!ptd) + return -ENOMEM; + status = readq(ptd + 16); + request = readq(ptd + 280 * 16); + ack = readq(ptd + 284 * 16); + iounmap(ptd); + dev_info(dev, "PMP PTD display status=%#llx request=%#llx ack=%#llx\n", + status, request, ack); + return status == 1 && (request & ack & BIT_ULL(7)) ? 0 : -EAGAIN; +invalid: + of_node_put(node); + return -EINVAL; +} + +static int probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct m3_dcp_rtkit *dcp; + void __iomem *cpu; + const char *uuid; + u64 message; + int ret; + + if (kernel_client && (!power_vote || query_display)) + return -EINVAL; + if (defer_client_open && !kernel_client) + return -EINVAL; + if (!of_machine_is_compatible("apple,j514s") || + of_property_read_string(dev->of_node, "apple,firmware-uuid", &uuid) || + strcmp(uuid, "DDF38191-93B3-324A-BC8F-643006F5AC82")) + return -ENODEV; + if (!iommu_get_domain_for_dev(dev)) + return -ENODEV; + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); + if (ret) + return ret; + cpu = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(cpu)) + return PTR_ERR(cpu); + dev_info(dev, "M3 DCP inherited CPU control=%#x status=%#x\n", readl(cpu + 0x44), readl(cpu + 0x48)); + if (!(readl(cpu + 0x44) & BIT(4))) + return dev_err_probe(dev, -EBUSY, "DCP CPU stopped during handoff; do not resume interrupted firmware\n"); + if (power_vote) { + ret = keep_display_power(dev); + if (ret) + return dev_err_probe(dev, ret, "DISP power vote failed\n"); + } + if (!(readl(cpu + 0x44) & BIT(4))) + return dev_err_probe(dev, -EIO, "DCP CPU stopped after DISP power vote\n"); + dcp = devm_kzalloc(dev, sizeof(*dcp), GFP_KERNEL); + if (!dcp) + return -ENOMEM; + dcp->dev = dev; + init_completion(&dcp->ready); + platform_set_drvdata(pdev, dcp); + dcp->rtkit = apple_rtkit_init(dev, dcp, "mbox", 0, &ops); + if (IS_ERR(dcp->rtkit)) + return dev_err_probe(dev, PTR_ERR(dcp->rtkit), "RTKit init failed\n"); + /* This bring-up client has no qualified hot teardown. Retain its DMA and + * RTKit objects even on startup failure; recovery is a guarded reboot. + */ + __module_get(THIS_MODULE); + ret = apple_rtkit_wake(dcp->rtkit); + for (int n = 0; ret == -ETIME && n < 4; n++) + ret = apple_rtkit_boot(dcp->rtkit); + if (ret) + goto retained; + dev_info(dev, "M3 native RTKit boot complete; running=%d\n", apple_rtkit_is_running(dcp->rtkit)); + if (!start_link && !query_display && !kernel_client) + goto retained; + dcp->rpc = dma_alloc_coherent(dev, APPLE_DCP_LINK_RPC_MEMORY_SIZE, + &dcp->rpc_dva, GFP_KERNEL); + if (!dcp->rpc) { + ret = -ENOMEM; + goto retained; + } + memset(dcp->rpc, 0, APPLE_DCP_LINK_RPC_MEMORY_SIZE); + apple_dcp_link_init_desc_encode(dcp->rpc, 0, false); + /* The imported descriptor names this field version; M3 uses alignment. */ + ((struct apple_dcp_link_init_desc *)dcp->rpc)->version = cpu_to_le32(64); + ret = apple_dcp_link_init_message(dcp->rpc_dva, &message); + if (ret) + goto retained; + dcp->result = -ETIMEDOUT; + ret = apple_rtkit_start_ep(dcp->rtkit, APPLE_DCP_LINK_ENDPOINT); + if (ret) + goto retained; + dma_wmb(); + ret = apple_rtkit_send_message(dcp->rtkit, APPLE_DCP_LINK_ENDPOINT, message, NULL, false); + if (ret) + goto retained; + if (!wait_for_completion_timeout(&dcp->ready, msecs_to_jiffies(3000))) + ret = -ETIMEDOUT; + else + ret = dcp->result; + if (!ret && kernel_client) { + dcp->bridge = m3_dcp_bridge_create(dev, dcp->rtkit, dcp->rpc, true); + if (IS_ERR(dcp->bridge)) { + ret = PTR_ERR(dcp->bridge); + dcp->bridge = NULL; + goto retained; + } + m3_dcp_bridge_activate(dcp->bridge); + dcp->native = m3_dcp_native_start(dev, dcp->bridge, defer_client_open); + ret = PTR_ERR_OR_ZERO(dcp->native); + if (!ret) { + client_opened = !defer_client_open; + if (client_opened) + save_properties(dcp); + active = dcp; + } + } + if (!ret && query_display) { + /* __TEXT 0x13f84c: no input; bool result serialized in four bytes. + * One request on AP stream 0 at offset 0, already 64-byte aligned. + */ + reinit_completion(&dcp->ready); + apple_dcp_link_rpc_header_encode(dcp->rpc, 0x41343130, 0, 4); + *(__le32 *)(dcp->rpc + 12) = cpu_to_le32(0xa5); + dcp->result = -ETIMEDOUT; + dcp->awaiting_rpc = true; + dma_wmb(); + ret = apple_rtkit_send_message(dcp->rtkit, APPLE_DCP_LINK_ENDPOINT, + (16ULL << 32) | APPLE_DCP_LINK_MSG_RPC, NULL, false); + if (!ret) { + if (!wait_for_completion_timeout(&dcp->ready, msecs_to_jiffies(3000))) + ret = -ETIMEDOUT; + else + ret = dcp->result; + } + } +retained: + dcp->result = ret; + dev_info(dev, "M3 native DCP diagnostic finished ret=%d; resources retained until reboot\n", ret); + return 0; +} + +static const struct of_device_id matches[] = { + { .compatible = "apple,t6030-dcp" }, + { } +}; +/* Deliberately no module alias: this diagnostic is loaded explicitly. */ +static struct platform_driver m3_dcp_driver = { + .probe = probe, + .driver = { + .name = "m3-dcp-rtkit", + .of_match_table = matches, + .suppress_bind_attrs = true, + }, +}; +module_platform_driver(m3_dcp_driver); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("J514S native RTKit/DMA DCPLink startup diagnostic"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_state.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_state.h new file mode 100644 index 0000000000000..c24c8ca074f57 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_state.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include +#include +/* Private layout shared by the explicit startup and one-shot query probes. */ +struct m3_dcp_rtkit { + struct device *dev; + struct apple_rtkit *rtkit; + struct completion ready; + void *rpc; + dma_addr_t rpc_dva; + int result; + bool awaiting_rpc; + struct m3_dcp_bridge *bridge; + struct m3_dcp_native *native; +}; diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_vote.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_vote.c new file mode 100644 index 0000000000000..4fe311d3f2b0c --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_vote.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Isolate the existing J514S DISP request from native DCP attachment. + * Initialization-only diagnostic: use the bounded lifecycle watchdog. + * Keep the vote until reboot; no unload or display submissions. + */ +#include +#include +#include +#include +#include + +static int __init vote(void) +{ + struct device_node *node; + const u8 *table; + int length, ret = -EINVAL; + + if (!of_machine_is_compatible("apple,j514s") || + !of_machine_is_compatible("apple,t6030")) + return -ENODEV; + node = of_find_node_by_path("/soc/pmp@350500000"); + if (!node) + return -ENODEV; + table = of_get_property(node, "apple,tunable-soc-device", &length); + if (table && length == 21 * 0x7c && + get_unaligned_le32(table + 7 * 0x7c) == 8 && + !memcmp(table + 7 * 0x7c + 0x74, "DISP\0\0\0\0", 8)) + ret = apple_pmp_set_device_power(0x0f, 8, 1); + of_node_put(node); + pr_info("M3 DCP isolated PMP DISP request returned %d\n", ret); + if (!ret) + msleep(20); + return ret; +} +module_init(vote); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("J514S isolated PMP display power request; reboot to remove"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio.h new file mode 100644 index 0000000000000..2387fb93ed03f --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio.h @@ -0,0 +1,193 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* J514S old-EPIC audio transport. Service framing follows Asahi av.c and + * m1n1 EPICStandardService; serialized lifetime follows the M4 audio port. + * No PCM/DMA playback is initiated by this transport. */ +#define M3_AUDIO_MAX (256 * 1024) +static struct afkc audio_afk; +static u32 audio_channel, audio_command, audio_group, audio_bytes, audio_received; +static u16 audio_tag; +static u32 audio_generation; +static bool audio_owned; +static bool audio_announced, audio_pending; +static int audio_error, audio_result = -ENODATA; +static u8 *audio_tx, *audio_rx; +static u64 audio_tx_dva, audio_rx_dva; +module_param(audio_result, int, 0400); + +static int audio_record(void *cookie, const u8 *p, u32 size) +{ + const u8 *h = audio_afk.capture + audio_afk.capture_size - size - 12; + u32 channel = get_unaligned_le32(h), kind = get_unaligned_le32(h + 4); + if (size < 40 || p[0] != 2 || p[20] != 4) return -EPROTO; + if (!p[21] && get_unaligned_le16(p + 22) == 0x30) { + pr_info("m3_audio: announcement channel=%u name=%.*s bytes=%u\n", channel, min(size-40,32U), p+40, size); + if (size < 76 || memcmp(p+40,"dispext0:dcpav-audio-interface",sizeof("dispext0:dcpav-audio-interface")-1) || get_unaligned_le32(p+72)!=0xd3) + return 0; + if (audio_announced) return -EALREADY; + audio_channel=channel; audio_generation++; audio_announced=true; return 0; + } + if (channel != audio_channel) return 0; + if (!p[21]) { + if (get_unaligned_le16(p+22)==0x32) { + audio_generation++;audio_announced=false; audio_pending=false; audio_error=-ENODEV; + } + return 0; + } + if (!audio_pending || (kind!=0 && kind!=4) || size<70 || p[21]!=0x20 || + get_unaligned_le16(p+22)!=0xc0 || get_unaligned_le16(p+32)!=audio_tag) + return -EPROTO; + if (get_unaligned_le32(p+40)) return -EREMOTEIO; + if (get_unaligned_le64(p+44)!=audio_rx_dva || get_unaligned_le32(p+60)>audio_bytes) + return -EPROTO; + dma_rmb(); + audio_received=get_unaligned_le32(p+60); + audio_pending=false; + return 0; +} +static const struct afkc_ops audio_ops={.send=session_send,.allocate=afk_allocate,.record=audio_record}; +static int audio_receive(u64 message) +{ + int ret=afkc_receive(&audio_afk,message); + if(ret) {audio_error=ret;audio_pending=false;} + return 0; /* Audio faults must not tear down video or free in-flight DMA. */ +} +static int audio_wait(bool announcement) +{ + u64 until=session_now(NULL)+2000; + int ret; + while(announcement ? !audio_announced : audio_pending) { + if(audio_error)return audio_error; + ret=native_poll(NULL,msecs_to_jiffies(5)); + if(ret && ret!=-ETIMEDOUT)return ret; + if(session_now(NULL)>=until)return -ETIMEDOUT; + } + return audio_error; +} +static int audio_call_locked(u16 group,u32 command,const void *body,u32 bytes) +{ + u8 wire[70]={};int ret; + lockdep_assert_held(&native_route_lock); + if(port || usb_c || !native_kms_ready || native_kms_finished || !native_kms_connected || + desktop_disconnected || READ_ONCE(desktop_stop) || !session.runtime)return -ENODEV; + if(audio_error)return audio_error; + if(bytes>M3_AUDIO_MAX+48)return -E2BIG; + if(!audio_afk.started) { + if(!(session.endpoints[1]&BIT(9)))return -ENODEV; + ret=afk_allocate(NULL,M3_AUDIO_MAX+112,&audio_tx,&audio_tx_dva);if(ret)goto fail; + ret=afk_allocate(NULL,M3_AUDIO_MAX+112,&audio_rx,&audio_rx_dva);if(ret)goto fail; + audio_afk.streaming=true; + ret=afkc_start(&audio_afk,&audio_ops,NULL,0x29);if(ret)goto fail; + } + ret=audio_wait(true);if(ret)goto fail; + if(audio_pending){ret=-EBUSY;goto fail;} + audio_bytes=bytes+64;audio_group=group;audio_command=command;audio_tag++; + memset(audio_tx,0,audio_bytes);memset(audio_rx,0,audio_bytes); + put_unaligned_le16(group,audio_tx+2);put_unaligned_le32(command,audio_tx+4); + put_unaligned_le32(bytes,audio_tx+8);put_unaligned_le32(0x69706378,audio_tx+12); + memcpy(audio_tx+64,body,bytes); + wire[0]=2;put_unaligned_le16(epic_seq++,wire+1);put_unaligned_le32(30,wire+16); + wire[20]=4;wire[21]=0x30;put_unaligned_le16(0xc0,wire+22);put_unaligned_le16(audio_tag,wire+32); + put_unaligned_le64(audio_rx_dva,wire+44);put_unaligned_le64(audio_tx_dva,wire+52); + put_unaligned_le32(audio_bytes,wire+60);put_unaligned_le32(audio_bytes,wire+64); + audio_pending=true;audio_received=0;dma_wmb(); + ret=old_epic_write_channel(&audio_afk,audio_channel,3,wire,sizeof(wire));if(ret)goto fail; + ret=audio_wait(false);if(ret)goto fail; + if(audio_received<64 || get_unaligned_le16(audio_rx+2)!=group || + get_unaligned_le32(audio_rx+4)!=command || get_unaligned_le32(audio_rx+12)!=0x69706378 || + get_unaligned_le32(audio_rx+8)>audio_received-64) {ret=-EPROTO;goto fail;} + audio_result=0; + pr_info("m3_audio: group=%u command=%u response=%u body=%u\n",group,command,audio_received,get_unaligned_le32(audio_rx+8)); + return 0; +fail: + audio_result=audio_error=ret;audio_pending=false; + pr_err("m3_audio: error=%d; all firmware DMA retained until reboot\n",ret); + return ret; +} +static ssize_t audio_request_write(struct file *file,const char __user *buf,size_t count,loff_t *offset) +{ + u8 *request;u32 group,command,bytes;int ret; + if(count<8 || count>M3_AUDIO_MAX+56)return -EINVAL; + request=memdup_user(buf,count);if(IS_ERR(request))return PTR_ERR(request); + group=get_unaligned_le32(request);command=get_unaligned_le32(request+4);bytes=count-8; + /* Initial metadata qualification only: no stream/prepare requests. */ + if(!((group==0 && ((command==4 && bytes==32)||(command==5 && bytes==16))) || + (group==1 && (command==16 || command==18) && bytes>=48))) {kfree(request);return -EINVAL;} + mutex_lock(&native_route_lock); + ret=audio_owned ? -EBUSY : audio_call_locked(group,command,request+8,bytes); + mutex_unlock(&native_route_lock);kfree(request); + return ret?ret:count; +} +static ssize_t audio_response_read(struct file *file,char __user *buf,size_t count,loff_t *offset) +{ + ssize_t ret; + mutex_lock(&native_route_lock); + ret=simple_read_from_buffer(buf,count,offset,audio_rx,audio_received); + mutex_unlock(&native_route_lock);return ret; +} +static ssize_t audio_capture_read(struct file *file,char __user *buf,size_t count,loff_t *offset) +{ + ssize_t ret; + mutex_lock(&native_route_lock); + ret=simple_read_from_buffer(buf,count,offset,audio_afk.capture,audio_afk.capture_size); + mutex_unlock(&native_route_lock);return ret; +} +static const struct file_operations audio_request_fops={.owner=THIS_MODULE,.write=audio_request_write}; +static const struct file_operations audio_response_fops={.owner=THIS_MODULE,.read=audio_response_read,.llseek=default_llseek}; +static const struct file_operations audio_capture_fops={.owner=THIS_MODULE,.read=audio_capture_read,.llseek=default_llseek}; +static void audio_debugfs(struct dentry *dir) +{ + if(port)return; + debugfs_create_file("audio-request",0200,dir,NULL,&audio_request_fops); + debugfs_create_file("audio-response",0400,dir,NULL,&audio_response_fops); + debugfs_create_file("audio-capture",0400,dir,NULL,&audio_capture_fops); +} + +#if DCPEXT_INSTANCE == 0 +#include "m3_dcpext_audio_api.h" +static u64 audio_current_generation(void) +{return ((u64)audio_generation<<32)|(u32)native_kms_generation;} +static bool audio_live(void) +{return !port && !usb_c && native_kms_ready && !native_kms_finished && native_kms_connected && + !desktop_disconnected && !READ_ONCE(desktop_stop) && session.runtime;} +static int audio_identify(char *name,size_t size,u64 *generation) +{ + int ret=0; + mutex_lock(&native_route_lock); + if(!audio_live()){ret=-ENODEV;goto out;} + strscpy(name,"M3 HDMI",size); + if(av_edid_bytes>=128)for(u32 off=54;off<=108;off+=18) { + const u8 *d=av_edid+off; + if(d[0] || d[1] || d[2] || d[3]!=0xfc)continue; + char display[14];memcpy(display,d+5,13);display[13]=0; + for(u32 i=0;i<13;i++)if(display[i]=='\n'){display[i]=0;break;} + strscpy(name,display,size);break; + } + *generation=audio_current_generation(); +out:mutex_unlock(&native_route_lock);return ret; +} +static int audio_pcm_call(u16 group,u32 command,void *data,u32 bytes,u64 *generation) +{ + int ret; + if(!data || !generation)return -EINVAL; + mutex_lock(&native_route_lock); + if(!audio_live() || (*generation && *generation!=audio_current_generation())){ret=-ENODEV;goto out;} + /* Lazy endpoint publication may establish the first audio generation. */ + bool first=!audio_afk.started; + ret=audio_call_locked(group,command,data,bytes); + if(ret)goto out; + if(!first && *generation && *generation!=audio_current_generation()){ret=-ENODEV;goto out;} + *generation=audio_current_generation(); + u32 received=get_unaligned_le32(audio_rx+8); + if(received>bytes){ret=-EPROTO;goto out;} + memset(data,0,bytes);memcpy(data,audio_rx+64,received); +out:mutex_unlock(&native_route_lock);return ret; +} +static const struct m3_dcpext_audio_ops audio_pcm_ops={.call=audio_pcm_call,.identify=audio_identify}; +const struct m3_dcpext_audio_ops *m3_dcpext_audio_get(unsigned int controller) +{ + if(controller)return NULL; + mutex_lock(&native_route_lock);audio_owned=true;mutex_unlock(&native_route_lock); + return &audio_pcm_ops; +} +EXPORT_SYMBOL_GPL(m3_dcpext_audio_get); +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_api.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_api.h new file mode 100644 index 0000000000000..10082d1845ac6 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_api.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef M3_DCPEXT_AUDIO_H +#define M3_DCPEXT_AUDIO_H +#include +#define M3_AUDIO_ELEMENTS_MAX (256 * 1024) + +/* Calls serialize with scanout/hotplug. Generation binds a PCM stream to one + * connected sink; an unplug/replug must never reuse its opaque audio cookie. + */ +struct m3_dcpext_audio_ops { + int (*call)(u16 group, u32 command, void *data, u32 bytes, u64 *generation); + int (*identify)(char *name, size_t size, u64 *generation); +}; +const struct m3_dcpext_audio_ops *m3_dcpext_audio_get(unsigned int port); +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_format.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_format.h new file mode 100644 index 0000000000000..f54440fc15a93 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_format.h @@ -0,0 +1,190 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Bounded OSSerialize format selection. Encoding: Hector Martin / Asahi; + * format fields: Martin Povišer / Asahi parser.c. M3 14.6 cookies are 32 bytes, verified in live metadata-2. + * Initial PCM qualification is stereo S16_LE at 48 kHz only. + */ +#ifndef M3_DCPEXT_AUDIO_FORMAT_H +#define M3_DCPEXT_AUDIO_FORMAT_H +struct audio_object { + const u8 *raw; + u32 size, pos, budget; +}; +static int audio_object_tag(struct audio_object *o, u32 *type, u32 *count) +{ + u32 v; + o->pos = (o->pos + 3) & ~3U; + if (!o->budget-- || o->pos > o->size || o->size - o->pos < 4) + return -EPROTO; + v = get_unaligned_le32(o->raw + o->pos); + o->pos += 4; + if (v & 0x60000000) + return -EPROTO; + *type = (v >> 24) & 31; + *count = v & 0xffffff; + return 0; +} +static int audio_object_bytes(struct audio_object *o, u32 n, const u8 **p) +{ + if (o->pos > o->size || n > o->size - o->pos) + return -EPROTO; + *p = o->raw + o->pos; + o->pos += n; + return 0; +} +static int audio_object_skip(struct audio_object *o, unsigned int depth) +{ + u32 t, n; + const u8 *p; + int ret; + if (depth > 12 || audio_object_tag(o, &t, &n)) + return -EPROTO; + switch (t) { + case 1: + case 2: + if (n > 256) + return -E2BIG; + for (u32 i = 0; i < n * (t == 1 ? 2 : 1); i++) { + ret = audio_object_skip(o, depth + 1); + if (ret) + return ret; + } + return 0; + case 4: return n == 64 ? audio_object_bytes(o, 8, &p) : -EPROTO; + case 9: + case 10: return audio_object_bytes(o, n, &p); + case 11: return n <= 1 ? 0 : -EPROTO; + default: return -EPROTO; + } +} +static int audio_object_string(struct audio_object *o, const u8 **p, u32 *n) +{ + u32 t; + if (audio_object_tag(o, &t, n) || t != 9) + return -EPROTO; + return audio_object_bytes(o, *n, p); +} +static bool audio_key(const u8 *p, u32 n, const char *key) +{ + return n == strlen(key) && !memcmp(p, key, n); +} +static int audio_object_number(struct audio_object *o, u64 *value) +{ + u32 t, n; + const u8 *p; + if (audio_object_tag(o, &t, &n) || t != 4 || n != 64 || + audio_object_bytes(o, 8, &p)) + return -EPROTO; + *value = get_unaligned_le64(p); + return 0; +} +static int audio_stereo_layout(struct audio_object *o) +{ + u32 t, count; + bool found = false; + if (audio_object_tag(o, &t, &count) || t != 2 || count > 256) + return -EPROTO; + for (u32 i = 0; i < count; i++) { + u32 pairs; + u64 channels = 0, active = 0; + bool stereo = false; + if (audio_object_tag(o, &t, &pairs) || t != 1 || pairs > 64) + return -EPROTO; + for (u32 j = 0; j < pairs; j++) { + const u8 *key; + u32 len; + int ret; + if (audio_object_string(o, &key, &len)) + return -EPROTO; + if (audio_key(key, len, "ChannelCount")) + ret = audio_object_number(o, &channels); + else if (audio_key(key, len, "ActiveChannelCount")) + ret = audio_object_number(o, &active); + else if (audio_key(key, len, "ChannelLayout")) { + u32 n; + const u8 *left, *right; + u32 ln, rn; + struct audio_object saved = *o; + ret = audio_object_tag(o, &t, &n); + if (!ret && t == 2 && n == 2 && + !audio_object_string(o, &left, &ln) && + !audio_object_string(o, &right, &rn)) + stereo = audio_key(left, ln, "Front Left") && audio_key(right, rn, "Front Right"); + *o = saved; + ret = audio_object_skip(o, 0); + } else + ret = audio_object_skip(o, 0); + if (ret) + return ret; + } + found |= channels == 2 && active == 2 && stereo; + } + return found ? 1 : 0; +} +static int m3_audio_select_stereo(const u8 *raw, u32 bytes, u8 cookie[32]) +{ + struct audio_object o = { .raw = raw, .size = bytes, .pos = 4, .budget = 32768 }; + u32 t, count; + unsigned int matches = 0; + if (bytes < 8 || bytes > 256 * 1024 || get_unaligned_le32(raw) != 0xd3 || + audio_object_tag(&o, &t, &count) || t != 2 || count > 256) + return -EPROTO; + for (u32 i = 0; i < count; i++) { + const u8 *blob = NULL; + u32 pairs, blob_size = 0, seen = 0; + u64 fields[6] = {0}; + bool is_virtual = true, stereo = false; + static const char * const keys[] = { "ElementType", "Format", "ChannelCount", "SampleSize", "LinkSampleRate", "StreamSampleRate" }; + if (audio_object_tag(&o, &t, &pairs) || t != 1 || pairs > 64) + return -EPROTO; + for (u32 j = 0; j < pairs; j++) { + const u8 *key; + u32 len, k; + int ret; + if (audio_object_string(&o, &key, &len)) + return -EPROTO; + for (k = 0; k < 6; k++) + if (audio_key(key, len, keys[k])) + break; + if (k < 6) { + if (seen & (1U << k)) + return -EPROTO; + seen |= 1U << k; + ret = audio_object_number(&o, &fields[k]); + } else if (audio_key(key, len, "IsVirtual")) { + u32 n; + if (seen & 64) + return -EPROTO; + seen |= 64; + ret = audio_object_tag(&o, &t, &n); + if (ret || t != 11 || n > 1) + return -EPROTO; + is_virtual = n; + } else if (audio_key(key, len, "ElementData")) { + if (blob || audio_object_tag(&o, &t, &blob_size) || t != 10) + return -EPROTO; + ret = audio_object_bytes(&o, blob_size, &blob); + } else if (audio_key(key, len, "AudioChannelLayoutElements")) { + ret = audio_stereo_layout(&o); + stereo = ret == 1; + if (ret > 0) + ret = 0; + } else + ret = audio_object_skip(&o, 0); + if (ret) + return ret; + } + if (seen != 127 || is_virtual || !stereo || fields[0] != 2 || + fields[1] != 1 || fields[2] != 2 || fields[3] != 16 || + fields[4] != 48000 || fields[5] != 48000) + continue; + if (!blob || blob_size != 32 || get_unaligned_le32(blob) != 1 || + get_unaligned_le32(blob + 4) != 2 || get_unaligned_le32(blob + 8) != 16 || + get_unaligned_le32(blob + 12) != 48000 || + get_unaligned_le32(blob + 16) || get_unaligned_le32(blob + 20)) + return -EPROTO; + memcpy(cookie, blob, 32); + matches++; + } + return matches == 1 ? 0 : -ENODATA; +} +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_pcm.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_pcm.c new file mode 100644 index 0000000000000..2ffe80576e45c --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_pcm.c @@ -0,0 +1,343 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* M3 port of M4 display PCM integration. Audio command/DMA ordering follows Martin + * Povišer's Asahi audio.c; the transport/cookie and connected-sink lifetime + * are supplied by the M3 old-EPIC transport. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "m3_dcpext_audio_api.h" +#include "m3_dcpext_audio_format.h" + +static u8 metadata_capture[M3_AUDIO_ELEMENTS_MAX + 48]; +static struct debugfs_blob_wrapper metadata_blob = { + .data = metadata_capture, .size = sizeof(metadata_capture), +}; +static struct dentry *metadata_debug; +static bool metadata_only = true; +module_param(metadata_only, bool, 0400); +MODULE_PARM_DESC(metadata_only, "Discover fresh formats without registering PCM for initial qualification"); +static unsigned int controller; +module_param(controller, uint, 0400); +MODULE_PARM_DESC(controller, "DCPEXT instance whose DPA DMA route is described by this device (initially 0)"); + +struct m3_audio_pcm { + struct device *dev; + const struct m3_dcpext_audio_ops *ops; + struct dma_chan *chan; + struct snd_card *card; + u64 generation; + u8 cookie[32]; + char name[32]; + bool opened, prepared, started, pinned; +}; + +static int m3_audio_simple(struct m3_audio_pcm *a, u32 cmd, bool cookie) +{ + u8 data[64] = {}; + u32 size = cmd == 4 ? 32 : cmd == 5 ? 16 : 64; + int ret; + if (cookie || (cmd >= 6 && cmd <= 11)) + memcpy(data, a->cookie, sizeof(a->cookie)); + ret = a->ops->call(0, cmd, data, size, &a->generation); + /* M4 link methods return IOReturn at byte 48 of the 64-byte body. */ + if (!ret && size == 64 && get_unaligned_le32(data + 48)) { + dev_err(a->dev, "audio command %u service status %#x\n", cmd, + get_unaligned_le32(data + 48)); + ret = -EREMOTEIO; + } + return ret; +} + +static int m3_audio_elements(struct m3_audio_pcm *a) +{ + u8 *data; + u64 used; + int ret; + data = kzalloc(M3_AUDIO_ELEMENTS_MAX + 48, GFP_KERNEL); + if (!data) + return -ENOMEM; + put_unaligned_le64(M3_AUDIO_ELEMENTS_MAX, data); + ret = a->ops->call(1, 16, data, M3_AUDIO_ELEMENTS_MAX + 48, &a->generation); + if (ret) + goto out; + memcpy(metadata_capture, data, sizeof(metadata_capture)); + used = get_unaligned_le64(data + 32); + dev_info(a->dev, "audio metadata capacity=%llu status=%#x used=%llu\n", + get_unaligned_le64(data), get_unaligned_le32(data + 16), used); + if (get_unaligned_le64(data) != M3_AUDIO_ELEMENTS_MAX || get_unaligned_le32(data + 16) || + used < 8 || used > M3_AUDIO_ELEMENTS_MAX) { + ret = -EPROTO; + goto out; + } + ret = m3_audio_select_stereo(data + 48, used, a->cookie); + if (!ret) + dev_info(a->dev, "%s: fresh stereo 48kHz S16 cookie, metadata %llu bytes\n", a->name, used); +out: + kfree(data); + return ret; +} + +static int m3_audio_open_service(struct m3_audio_pcm *a) +{ + int ret; + a->generation = 0; + ret = a->ops->identify(a->name, sizeof(a->name), &a->generation); + if (ret) + return ret; + ret = m3_audio_simple(a, 4, false); + if (ret) + return ret; + a->opened = true; + ret = m3_audio_elements(a); + if (ret) { + m3_audio_simple(a, 5, false); + a->opened = false; + } + return ret; +} + +static const struct snd_pcm_hardware m3_audio_hw = { + .info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | + SNDRV_PCM_INFO_INTERLEAVED | SNDRV_PCM_INFO_BATCH, + .formats = SNDRV_PCM_FMTBIT_S16_LE, + .rates = SNDRV_PCM_RATE_48000, + .rate_min = 48000, .rate_max = 48000, + .channels_min = 2, .channels_max = 2, + .buffer_bytes_max = 256 * 1024, + .period_bytes_min = 256, .period_bytes_max = 64 * 1024, + .periods_min = 4, .periods_max = 32, + .fifo_size = 16, +}; + +static int m3_pcm_open(struct snd_pcm_substream *s) +{ + struct m3_audio_pcm *a = s->pcm->private_data; + int ret; + struct dma_tx_state state; + if (dmaengine_tx_status(a->chan, 0, &state) == DMA_ERROR) + return -EIO; + if (!s->dma_buffer.area || s->dma_buffer.bytes != 256 * 1024) + return -ENOMEM; + ret = m3_audio_open_service(a); + if (ret) + return ret; + s->runtime->hw = m3_audio_hw; + /* The DMA helper applies the integer-period constraint itself. The + * constraint API may return positive success, not just zero. + */ + ret = snd_dmaengine_pcm_open(s, a->chan); + if (ret) { + m3_audio_simple(a, 5, false); + a->opened = false; + } + return ret; +} +static int m3_pcm_hw_params(struct snd_pcm_substream *s, struct snd_pcm_hw_params *p) +{ + struct m3_audio_pcm *a = s->pcm->private_data; + struct dma_slave_config cfg = {}; + int ret = snd_hwparams_to_dma_slave_config(s, p, &cfg); + if (ret) + return ret; + cfg.direction = DMA_MEM_TO_DEV; + /* DPA data entry is always 32 bits, including stereo 16-bit PCM. */ + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + return dmaengine_slave_config(a->chan, &cfg); +} +static int m3_pcm_prepare(struct snd_pcm_substream *s) +{ + struct m3_audio_pcm *a = s->pcm->private_data; + int ret; + + /* ALSA permits prepare again after drop/XRUN or before a first start. + * DCP retains the prepared link until unprepare and rejects duplicate + * prepare commands. The PCM format is fixed for this open lifetime. + */ + if (a->prepared) + return 0; + ret = m3_audio_simple(a, 6, true); + if (!ret) + a->prepared = true; + return ret; +} +static int m3_pcm_trigger(struct snd_pcm_substream *s, int cmd) +{ + struct m3_audio_pcm *a = s->pcm->private_data; + int ret; + if (cmd == SNDRV_PCM_TRIGGER_START) { + ret = m3_audio_simple(a, 7, true); + if (ret) + return ret; + a->started = true; + /* Bring-up uses one fixed coherent buffer for the card lifetime. + * Keep it and callback storage alive even if SIO cannot retire DMA. + * TODO: removable PCM after SIO error recovery is qualified. + */ + if (!a->pinned) { + __module_get(THIS_MODULE); + a->pinned = true; + } + ret = snd_dmaengine_pcm_trigger(s, cmd); + if (ret) { + m3_audio_simple(a, 10, false); + a->started = false; + } + return ret; + } + if (cmd == SNDRV_PCM_TRIGGER_STOP) { + ret = snd_dmaengine_pcm_trigger(s, cmd); + if (ret) + return ret; + /* Drain DMA before the link is stopped; sync_stop is sleepable. */ + return 0; + } + return -EINVAL; +} +static int m3_pcm_sync_stop(struct snd_pcm_substream *s) +{ + struct m3_audio_pcm *a = s->pcm->private_data; + int ret = snd_dmaengine_pcm_sync_stop(s); + if (!ret && a->started) { + ret = m3_audio_simple(a, 10, false); + a->started = false; + } + return ret; +} +static int m3_pcm_hw_free(struct snd_pcm_substream *s) +{ + struct m3_audio_pcm *a = s->pcm->private_data; + int ret = 0; + if (a->prepared) { + ret = m3_audio_simple(a, 11, false); + a->prepared = false; + } + return ret; +} +static int m3_pcm_close(struct snd_pcm_substream *s) +{ + struct m3_audio_pcm *a = s->pcm->private_data; + int ret = snd_dmaengine_pcm_close(s); + if (a->opened) { + int err = m3_audio_simple(a, 5, false); + a->opened = false; + if (!ret) + ret = err; + } + return ret; +} +static const struct snd_pcm_ops m3_pcm_ops = { + .open = m3_pcm_open, .close = m3_pcm_close, + .hw_params = m3_pcm_hw_params, .hw_free = m3_pcm_hw_free, + .prepare = m3_pcm_prepare, .trigger = m3_pcm_trigger, + .sync_stop = m3_pcm_sync_stop, .pointer = snd_dmaengine_pcm_pointer, +}; + +static int m3_audio_probe(struct platform_device *pdev) +{ + struct m3_audio_pcm *a; + struct snd_pcm *pcm; + int ret; + /* This first DT node describes DCPEXT0 -> DPA1 -> SIO 0x66. + * Never redirect it to a controller whose DMA mapping is unqualified. + */ + if (controller != 0) + return -EINVAL; + a = devm_kzalloc(&pdev->dev, sizeof(*a), GFP_KERNEL); + if (!a) + return -ENOMEM; + a->dev = &pdev->dev; + a->ops = m3_dcpext_audio_get(controller); + if (!a->ops) + return -EPROBE_DEFER; + a->chan = of_dma_request_slave_channel(pdev->dev.of_node, "tx"); + if (IS_ERR(a->chan)) + return dev_err_probe(a->dev, PTR_ERR(a->chan), "SIO channel unavailable\n"); + platform_set_drvdata(pdev, a); + pm_runtime_enable(a->dev); + ret = pm_runtime_resume_and_get(a->dev); + if (ret < 0) + goto release; + ret = m3_audio_open_service(a); + if (ret) + goto power; + ret = m3_audio_simple(a, 5, false); + a->opened = false; + if (ret) + goto power; + if (metadata_only) { + dev_info(a->dev, "metadata-only qualification completed; no PCM or DMA issued\n"); + return 0; + } + ret = snd_card_new(a->dev, -1, "M3HDMI0", THIS_MODULE, 0, &a->card); + if (ret) + goto power; + strscpy(a->card->driver, "M3HDMI", sizeof(a->card->driver)); + snprintf(a->card->shortname, sizeof(a->card->shortname), "HDMI %s", a->name); + snprintf(a->card->longname, sizeof(a->card->longname), "Apple M3 DCPEXT0 HDMI - %s", a->name); + ret = snd_pcm_new(a->card, a->card->shortname, 0, 1, 0, &pcm); + if (ret) + goto card; + pcm->private_data = a; + pcm->nonatomic = true; + strscpy(pcm->name, a->card->shortname, sizeof(pcm->name)); + snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_PLAYBACK, &m3_pcm_ops); + ret = snd_pcm_set_managed_buffer_all(pcm, SNDRV_DMA_TYPE_DEV, + a->chan->device->dev, 256 * 1024, 0); + if (ret) + goto card; + ret = snd_card_register(a->card); + if (!ret) + return 0; +card: + snd_card_free(a->card); +power: + pm_runtime_put_sync(a->dev); +release: + pm_runtime_disable(a->dev); + dma_release_channel(a->chan); + return ret; +} +static void m3_audio_remove(struct platform_device *pdev) +{ + struct m3_audio_pcm *a = platform_get_drvdata(pdev); + if (a->card) + snd_card_free(a->card); + dma_release_channel(a->chan); + pm_runtime_put_sync(a->dev); + pm_runtime_disable(a->dev); +} +static const struct of_device_id m3_audio_match[] = { + { .compatible = "apple,j514s-hdmi-audio" }, {} +}; +MODULE_DEVICE_TABLE(of, m3_audio_match); +static struct platform_driver m3_audio_driver = { + .probe = m3_audio_probe, .remove = m3_audio_remove, + .driver = { .name = "m3-dcpext-audio", .of_match_table = m3_audio_match, + .suppress_bind_attrs = true }, +}; +static int __init m3_audio_init(void) +{ + int ret; + metadata_debug = debugfs_create_blob("m3-hdmi-elements", 0400, NULL, &metadata_blob); + ret = platform_driver_register(&m3_audio_driver); + if (ret) + debugfs_remove(metadata_debug); + return ret; +} +static void __exit m3_audio_exit(void) +{ + platform_driver_unregister(&m3_audio_driver); + debugfs_remove(metadata_debug); +} +module_init(m3_audio_init); +module_exit(m3_audio_exit); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple M3 external display PCM audio"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_av.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_av.h new file mode 100644 index 0000000000000..f99243bfd3dbd --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_av.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Old EPIC AV service lifecycle: m1n1 EPICStandardService group4 open/close; + * Asahi dpavservep copyEDID group1/command7. Buffers retained until reboot. */ +static bool av_read_ready; +static bool av_announced,av_pending,av_opened,av_closed,av_edid_done; +static u32 av_channel,av_command,av_group,av_size,av_tag; +static u8 *av_rx,*av_tx;static u64 av_rx_dva,av_tx_dva;static u8 av_edid[1024];static u32 av_edid_bytes; +static int av_send(u16 group,u32 command,u32 bytes) +{ + u8 *tx,wire[70]={};u64 txd;int ret; + if(av_pending || !av_announced)return -EINVAL; + if(bytes>1144)return -E2BIG; + if(!av_rx){ret=afk_allocate(NULL,1144,&av_rx,&av_rx_dva);if(ret)return ret;} + if(!av_tx){ret=afk_allocate(NULL,1144,&av_tx,&av_tx_dva);if(ret)return ret;} + tx=av_tx;txd=av_tx_dva;memset(tx,0,1144);memset(av_rx,0,1144); + put_unaligned_le16(group,tx+2);put_unaligned_le32(command,tx+4); + put_unaligned_le32(bytes-64,tx+8);put_unaligned_le32(0x69706378,tx+12); + if(group==1)put_unaligned_le64(1032,tx+64); + wire[0]=2;put_unaligned_le16(epic_seq++,wire+1);put_unaligned_le32(30,wire+16); + wire[20]=4;wire[21]=0x30;put_unaligned_le16(0xc0,wire+22);put_unaligned_le16(av_tag,wire+32); + put_unaligned_le64(av_rx_dva,wire+44);put_unaligned_le64(txd,wire+52); + put_unaligned_le32(bytes,wire+60);put_unaligned_le32(bytes,wire+64); + av_pending=true;av_group=group;av_command=command;av_size=bytes; + pr_info("m3_dcpext_av: sending group=%u command=%u bytes=%u\n",group,command,bytes); + return old_epic_write_channel(&aux_afk,av_channel,3,wire,sizeof(wire)); +} +static int av_record(u32 channel,u32 kind,const u8 *p,u32 size) +{ + if(size>=76 && p[0]==2 && p[20]==4 && !p[21] && get_unaligned_le16(p+22)==0x30 && + !memcmp(p+40,port?"dispext1:dcpav-service-epic:0":"dispext0:dcpav-service-epic:0",sizeof("dispext0:dcpav-service-epic:0")) && get_unaligned_le32(p+72)==0xd3){ + if(av_announced)return -EALREADY; + av_announced=true;av_closed=false;av_channel=channel;pr_info("m3_dcpext_av: service announced channel=%u\n",channel);return 0; + } + if(size>=40 && av_announced && channel==av_channel && p[0]==2 && p[20]==4 && !p[21] && get_unaligned_le16(p+22)==0x32){ + pr_info("m3_dcpext_av: service teardown report; no DMA released\n"); + /* Teardown is not proof that a pending DMA has stopped. */ + if(av_pending)return -EBUSY; + av_announced=av_opened=false;av_closed=true;return 0; + } + if(!av_pending || channel!=av_channel || (kind!=0 && kind!=4))return 0; + if(size<70 || p[21]!=0x20 || get_unaligned_le16(p+22)!=0xc0 || get_unaligned_le16(p+32)!=av_tag || + get_unaligned_le32(p+40) || get_unaligned_le64(p+44)!=av_rx_dva || + get_unaligned_le32(p+60)>av_size)return -EPROTO; + dma_rmb(); + if(get_unaligned_le16(av_rx+2)!=av_group || get_unaligned_le32(av_rx+4)!=av_command || + get_unaligned_le32(av_rx+12)!=0x69706378)return -EPROTO; + av_pending=false;av_tag++; + if(av_group==4){av_opened=av_command==6;av_closed=av_command==7;} + else { + u64 used=get_unaligned_le64(av_rx+96);const u8 *edid=av_rx+120;u32 bytes; + if(get_unaligned_le64(av_rx+64)!=1032 || used<136 || used>1032 || + get_unaligned_le32(av_rx+112)!=0xd3)return -EPROTO; + bytes=used-8; + if(bytes%128 || bytes!=128*((u32)edid[126]+1) || memcmp(edid,"\0\xff\xff\xff\xff\xff\xff\0",8) || + get_unaligned_le32(av_rx+116)!=(0x8a000000|bytes))return -EBADMSG; + for(u32 at=0;at +#include +#include +#include +#include +#include +#include +#include "m3_dcpext_connector.h" +#define M3_DCPEXT_EDID_RAW_MAX 1024 + +struct m3_connector_state { + struct drm_connector_state base; + u64 generation; +}; + +static struct m3_dcpext_connector *to_m3(struct drm_connector *connector) +{ + return container_of(connector, struct m3_dcpext_connector, base); +} + +static void mode_geometry(const struct drm_display_mode *mode, + struct m3_dcpext_mode *geometry) +{ + *geometry = (struct m3_dcpext_mode) { + .width = mode->hdisplay, .height = mode->vdisplay, + .clock_khz = mode->clock, .htotal = mode->htotal, + .vtotal = mode->vtotal, + .interlaced = mode->flags & DRM_MODE_FLAG_INTERLACE, + .doublescan = mode->flags & DRM_MODE_FLAG_DBLSCAN, + }; +} + +static int select_mode(struct m3_dcpext_connector *c, + const struct drm_display_mode *mode, u8 timing[24]) +{ + struct m3_dcpext_mode geometry; + + /* Firmware does not accept porch timings here. Only modes produced by + * DRM's EDID parser are eligible; never accept arbitrary custom timings + * just because their size and rounded refresh match a firmware record. + */ + if (mode->vscan > 1 || mode->flags & ~(DRM_MODE_FLAG_PHSYNC | + DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)) + return -EINVAL; + mode_geometry(mode, &geometry); + return m3_dcpext_mode_select(&geometry, c->timings, + c->timings_count, timing); +} + +static bool mode_known(struct m3_dcpext_connector *c, + const struct drm_display_mode *mode) +{ + for (u32 i = 0; i < c->modes_count; i++) + if (drm_mode_equal(mode, &c->modes[i])) + return true; + return false; +} + +static int select_native_mode(struct m3_dcpext_connector *c, + const struct drm_display_mode *mode, u32 *selected) +{ + struct m3_dcpext_native_mode requested = {}; + int ret; + + if (mode->vscan > 1 || mode->flags & ~(DRM_MODE_FLAG_PHSYNC | + DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC) || + mode->hsync_start < mode->hdisplay || mode->hsync_end < mode->hsync_start || + mode->vsync_start < mode->vdisplay || mode->vsync_end < mode->vsync_start) + return -EINVAL; + mode_geometry(mode, &requested.geometry); + requested.hfront = mode->hsync_start - mode->hdisplay; + requested.hsync = mode->hsync_end - mode->hsync_start; + requested.vfront = mode->vsync_start - mode->vdisplay; + requested.vsync = mode->vsync_end - mode->vsync_start; + requested.hpositive = mode->flags & DRM_MODE_FLAG_PHSYNC; + requested.vpositive = mode->flags & DRM_MODE_FLAG_PVSYNC; + ret = m3_dcpext_native_mode_select(&requested, c->native_modes, + c->native_modes_count, selected); + /* This first HDMI path qualifies exact 1080p60 only. Do not advertise an + * EDID 59.94 mode while silently selecting the firmware's 60-Hz ID. */ + if (!ret && requested.geometry.clock_khz != c->native_modes[*selected].geometry.clock_khz) + ret = -EOPNOTSUPP; + return ret; +} + +static int get_modes(struct drm_connector *connector) +{ + struct m3_dcpext_connector *c = to_m3(connector); + struct drm_display_mode *mode, *next, *modes; + u32 count = 0; + u32 selected; + u8 timing[24]; + int ret; + + mutex_lock(&c->lock); + kfree(c->modes); + c->modes = NULL; + c->modes_count = 0; + ret = drm_edid_connector_update(connector, c->connected ? c->edid : NULL); + if (ret || !c->connected || !c->edid) + goto out; + drm_edid_connector_add_modes(connector); + list_for_each_entry_safe(mode, next, &connector->probed_modes, head) { + if (c->native_modes ? select_native_mode(c, mode, &selected) : + select_mode(c, mode, timing)) { + list_del(&mode->head); + drm_mode_destroy(connector->dev, mode); + } else { + count++; + } + } + modes = kcalloc(count, sizeof(*modes), GFP_KERNEL); + if (!modes) { + /* mode_valid will reject all modes, including core fallbacks. */ + count = 0; + goto out; + } + c->modes = modes; + list_for_each_entry(mode, &connector->probed_modes, head) + drm_mode_copy(&modes[c->modes_count++], mode); + /* EDID reports sink capability; v1 transports only RGB at eight bpc. */ + connector->display_info.bpc = 8; + connector->display_info.color_formats = BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444); + out: + mutex_unlock(&c->lock); + return count; +} + +static enum drm_connector_status detect(struct drm_connector *connector, bool force) +{ + struct m3_dcpext_connector *c = to_m3(connector); + enum drm_connector_status status; + + mutex_lock(&c->lock); + status = c->connected ? connector_status_connected : connector_status_disconnected; + mutex_unlock(&c->lock); + return status; +} + +static enum drm_mode_status mode_valid(struct drm_connector *connector, + const struct drm_display_mode *mode) +{ + struct m3_dcpext_connector *c = to_m3(connector); + bool valid; + + mutex_lock(&c->lock); + valid = c->connected && c->edid && mode_known(c, mode); + mutex_unlock(&c->lock); + return valid ? MODE_OK : MODE_BAD; +} + +static void destroy_state(struct drm_connector *connector, + struct drm_connector_state *state) +{ + struct m3_connector_state *s = container_of(state, struct m3_connector_state, base); + + __drm_atomic_helper_connector_destroy_state(state); + kfree(s); +} + +static void reset(struct drm_connector *connector) +{ + struct m3_connector_state *s; + + if (connector->state) + destroy_state(connector, connector->state); + connector->state = NULL; + s = kzalloc(sizeof(*s), GFP_KERNEL); + if (s) + __drm_atomic_helper_connector_reset(connector, &s->base); +} + +static struct drm_connector_state *duplicate_state(struct drm_connector *connector) +{ + struct m3_connector_state *s; + + if (!connector->state) + return NULL; + s = kmemdup(container_of(connector->state, struct m3_connector_state, base), + sizeof(*s), GFP_KERNEL); + if (!s) + return NULL; + __drm_atomic_helper_connector_duplicate_state(connector, &s->base); + return &s->base; +} + +static int atomic_check(struct drm_connector *connector, struct drm_atomic_state *state) +{ + struct m3_dcpext_connector *c = to_m3(connector); + struct drm_connector_state *cs = drm_atomic_get_new_connector_state(state, connector); + struct drm_crtc_state *crtc; + struct m3_connector_state *s = container_of(cs, struct m3_connector_state, base); + int ret = 0; + + if (!cs->crtc) { + s->generation = 0; + return 0; + } + crtc = drm_atomic_get_new_crtc_state(state, cs->crtc); + if (!crtc || !crtc->enable) + return -EINVAL; + mutex_lock(&c->lock); + if (!c->connected || !c->edid || !mode_known(c, &crtc->mode)) + ret = -ENODEV; + else + s->generation = c->generation; + mutex_unlock(&c->lock); + return ret; +} + +static void destroy(struct drm_connector *connector) +{ + struct m3_dcpext_connector *c = to_m3(connector); + + /* Owner has stopped connection workers and drained atomic commits. */ + drm_connector_cleanup(connector); + drm_edid_free(c->edid); + kfree(c->modes); + kfree(c->native_modes); +} + +static const struct drm_connector_funcs connector_funcs = { + .detect = detect, + .fill_modes = drm_helper_probe_single_connector_modes, + .destroy = destroy, + .reset = reset, + .atomic_duplicate_state = duplicate_state, + .atomic_destroy_state = destroy_state, +}; + +static const struct drm_connector_helper_funcs connector_helpers = { + .get_modes = get_modes, + .mode_valid = mode_valid, + .atomic_check = atomic_check, +}; + +int m3_dcpext_connector_init(struct drm_device *drm, + struct m3_dcpext_connector *c, struct drm_encoder *encoder, int connector_type) +{ + int ret; + + mutex_init(&c->lock); + c->generation = 1; + ret = drm_connector_init(drm, &c->base, &connector_funcs, connector_type); + if (ret) + return ret; + c->base.polled = DRM_CONNECTOR_POLL_HPD; + c->base.interlace_allowed = false; + c->base.doublescan_allowed = false; + drm_connector_helper_add(&c->base, &connector_helpers); + ret = drm_connector_attach_encoder(&c->base, encoder); + if (ret) + drm_connector_cleanup(&c->base); + return ret; +} + +u64 m3_dcpext_connector_invalidate(struct m3_dcpext_connector *c, bool connected) +{ + u64 generation; + + mutex_lock(&c->base.dev->mode_config.mutex); + mutex_lock(&c->lock); + generation = ++c->generation; + c->connected = connected; + drm_edid_free(c->edid); + c->edid = NULL; + kfree(c->modes); + c->modes = NULL; + c->modes_count = 0; + c->timings_count = 0; + kfree(c->native_modes); + c->native_modes = NULL; + c->native_modes_count = 0; + c->native_generation = 0; + drm_edid_connector_update(&c->base, NULL); + mutex_unlock(&c->lock); + mutex_unlock(&c->base.dev->mode_config.mutex); + return generation; +} + +static int publish_snapshot(struct m3_dcpext_connector *c, u64 generation, + const u8 *raw, u32 bytes, const u8 *timings, u32 count, + const struct m3_dcpext_native_mode *native, u64 native_generation) +{ + const struct drm_edid *edid; + struct m3_dcpext_native_mode *native_copy = NULL; + int ret = 0; + + if (!raw || bytes < 128 || bytes > M3_DCPEXT_EDID_RAW_MAX || + bytes % 128 || bytes != ((u32)raw[126] + 1) * 128 || + !count || (!native && (!timings || count > 64)) || + (native && (timings || !native_generation || count > M3_DCPEXT_NATIVE_MAX_MODES))) + return -EINVAL; + if (native) { + for (u32 i = 0; i < count; i++) { + u32 rate; + if (m3_dcpext_mode_validate(&native[i].geometry, &rate) || + !native[i].hsync || !native[i].vsync || + (u64)native[i].geometry.width + native[i].hfront + native[i].hsync > native[i].geometry.htotal || + (u64)native[i].geometry.height + native[i].vfront + native[i].vsync > native[i].geometry.vtotal) + return -EINVAL; + } + } + edid = drm_edid_alloc(raw, bytes); + if (!edid) + return -ENOMEM; + if (!drm_edid_valid(edid)) { + drm_edid_free(edid); + return -EINVAL; + } + if (native) { + native_copy = kmemdup(native, count * sizeof(*native), GFP_KERNEL); + if (!native_copy) { + drm_edid_free(edid); + return -ENOMEM; + } + } + mutex_lock(&c->base.dev->mode_config.mutex); + mutex_lock(&c->lock); + if (!c->connected || c->generation != generation) { + ret = -ESTALE; + } else if (c->edid) { + /* A new snapshot requires invalidation, even on the same cable. */ + ret = -EALREADY; + } else { + c->edid = edid; + edid = NULL; + if (native) { + c->native_modes = native_copy; + native_copy = NULL; + c->native_modes_count = count; + c->native_generation = native_generation; + } else { + memcpy(c->timings, timings, count * 24); + c->timings_count = count; + } + } + mutex_unlock(&c->lock); + mutex_unlock(&c->base.dev->mode_config.mutex); + drm_edid_free(edid); + kfree(native_copy); + return ret; +} + +int m3_dcpext_connector_publish(struct m3_dcpext_connector *c, u64 generation, + const u8 *raw, u32 bytes, const u8 *timings, u32 count) +{ + return publish_snapshot(c, generation, raw, bytes, timings, count, NULL, 0); +} + +int m3_dcpext_connector_publish_native(struct m3_dcpext_connector *c, + u64 generation, const u8 *raw, u32 bytes, + const struct m3_dcpext_native_mode *modes, + u32 count, u64 native_generation) +{ + if (!modes) + return -EINVAL; + return publish_snapshot(c, generation, raw, bytes, NULL, count, modes, native_generation); +} + +void m3_dcpext_connector_hotplug(struct m3_dcpext_connector *c) +{ + drm_kms_helper_connector_hotplug_event(&c->base); +} + +int m3_dcpext_connector_commit_mode(struct m3_dcpext_connector *c, + const struct drm_connector_state *state, + const struct drm_display_mode *mode, + struct m3_dcpext_mode *geometry, u8 timing[24]) +{ + const struct m3_connector_state *s; + int ret; + + if (!state || state->connector != &c->base || !state->crtc || !mode || !geometry || !timing) + return -EINVAL; + s = container_of(state, struct m3_connector_state, base); + mutex_lock(&c->lock); + if (!c->connected || !c->edid || s->generation != c->generation) + ret = -ESTALE; + else if (c->native_modes) + ret = -EOPNOTSUPP; + else if (!mode_known(c, mode)) + ret = -EINVAL; + else { + ret = select_mode(c, mode, timing); + if (!ret) + mode_geometry(mode, geometry); + } + mutex_unlock(&c->lock); + return ret; +} + +int m3_dcpext_connector_commit_native(struct m3_dcpext_connector *c, + const struct drm_connector_state *state, + const struct drm_display_mode *mode, + struct m3_dcpext_native_mode *selected, + u64 *native_generation) +{ + const struct m3_connector_state *s; + u32 index; + int ret; + + if (!state || state->connector != &c->base || !state->crtc || + !mode || !selected || !native_generation) + return -EINVAL; + s = container_of(state, struct m3_connector_state, base); + mutex_lock(&c->lock); + if (!c->connected || !c->edid || s->generation != c->generation) + ret = -ESTALE; + else if (!c->native_modes || !mode_known(c, mode)) + ret = -EINVAL; + else { + ret = select_native_mode(c, mode, &index); + if (!ret) { + *selected = c->native_modes[index]; + *native_generation = c->native_generation; + } + } + mutex_unlock(&c->lock); + return ret; +} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.h new file mode 100644 index 0000000000000..573a1874ace6a --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef M3_DCPEXT_CONNECTOR_H +#define M3_DCPEXT_CONNECTOR_H + +#include +#include +#include "m3_dcpext_modes.h" + +struct drm_edid; +struct drm_encoder; + +/* One native HDMI port, independent of the assigned DCPEXT controller. + * The route owner serializes invalidate/publish against hardware commits. + * Lock order: route owner -> mode_config.mutex -> lock. DRM callbacks take + * only lock (mode_config.mutex is already held during probing). + * No firmware operations or userspace notifications run with lock held. + */ +struct m3_dcpext_connector { + struct drm_connector base; + struct mutex lock; + u64 generation; + bool connected; + const struct drm_edid *edid; + u8 timings[64 * 24]; + u32 timings_count; + struct m3_dcpext_native_mode *native_modes; + u32 native_modes_count; + u64 native_generation; + struct drm_display_mode *modes; + u32 modes_count; +}; + +int m3_dcpext_connector_init(struct drm_device *drm, + struct m3_dcpext_connector *connector, + struct drm_encoder *encoder, int connector_type); +/* Call for every HPD/route change, before dispatching new firmware queries. + * Returns the generation to attach to all asynchronous query results. + * Callers must not hold DRM locks. Hotplug notification is explicit below. + */ +u64 m3_dcpext_connector_invalidate(struct m3_dcpext_connector *connector, + bool connected); +/* Publish one coherent EDID + firmware timing snapshot. Stale replies cannot + * repopulate a disconnected/reassigned port. Original EDID bytes are retained. + */ +int m3_dcpext_connector_publish(struct m3_dcpext_connector *connector, + u64 generation, const u8 *edid, u32 bytes, + const u8 *timings, u32 count); +int m3_dcpext_connector_publish_native(struct m3_dcpext_connector *connector, + u64 generation, const u8 *edid, u32 bytes, + const struct m3_dcpext_native_mode *modes, + u32 count, u64 native_generation); +/* Owner calls after releasing its route lock, with the DRM device registered. */ +void m3_dcpext_connector_hotplug(struct m3_dcpext_connector *connector); +/* Recheck a checked atomic state's generation immediately before hardware + * submission, under the route owner's lock. Returns a complete firmware record. + * This does not implement scanout or imply presentation/retirement completion. + */ +int m3_dcpext_connector_commit_mode(struct m3_dcpext_connector *connector, + const struct drm_connector_state *state, + const struct drm_display_mode *mode, + struct m3_dcpext_mode *geometry, + u8 timing[24]); +int m3_dcpext_connector_commit_native(struct m3_dcpext_connector *connector, + const struct drm_connector_state *state, + const struct drm_display_mode *mode, + struct m3_dcpext_native_mode *selected, + u64 *native_generation); +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_dma.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_dma.c new file mode 100644 index 0000000000000..536389856308a --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_dma.c @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* J514S external DART additive mapping diagnostic. No firmware messages. + * Adapted from the M4 m4_dcpext_dma_probe mapper; no M4 PHY/session code. + * T6030 SID5 TCR=0xc01 and roots qualified in the guarded 2026-09-21 audit. + * DVA 0x40000000 belongs to M3 firmware TEXT: use empty slot 0x20000000. + * PTE encoding and ordering: Asahi io-pgtable-dart.c, fba91e5d9577. + * T8110 SID flush: Asahi apple-dart.c, same revision. + * Reservations: M3 m1n1 9b9fa13 handoff and same-boot J514S table audit. + * This is not a DMA execution test: the private DVA is never sent to DCP. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef DCPEXT_INSTANCE +#define DCPEXT_INSTANCE 0 +#endif + +#define LEASE_PAGE 0x4000 +#ifdef DCPEXT_WITH_SESSION +#define LEASE_SIZE 0x400000 +#else +#define LEASE_SIZE 0x10000 +#endif +#define LEASE_DVA 0x20000000ULL +#define LEASE_SLOT (LEASE_DVA >> 25) +#define PTE_ADDRESS GENMASK_ULL(39, 10) +/* T6030 DCP is not dma-coherent. Match dma-iommu + dart_prot_to_pte: + * shared host buffers need DART2 NO_CACHE, unlike firmware-owned TEXT/DATA. + */ +#define PTE_RW (GENMASK_ULL(51, 40) | BIT_ULL(1) | BIT_ULL(0)) + +static unsigned int port = DCPEXT_INSTANCE; +module_param(port, uint, 0400); +static int result = -EINPROGRESS; +module_param(result, int, 0400); +MODULE_PARM_DESC(result, "0 only after private mapping removal and inherited root comparison"); +static struct device *dma_dev; +static void *pool, *leaf; +static dma_addr_t pool_dma; +static u64 *root, *saved; +static void __iomem *regs; +static phys_addr_t dart_base; +static bool region_owned, pinned; +#ifdef DCPEXT_WITH_SESSION +static bool native_desktop; +module_param(native_desktop,bool,0400); +static bool native_kms_trial; +module_param(native_kms_trial,bool,0400); +static bool native_frame; +module_param(native_frame,bool,0400); +static bool native_startup; +module_param(native_startup,bool,0400); +#include "m3_dcpext_frame.h" +#include "m3_dcpext_session.h" +#endif + +static u64 encode_address(phys_addr_t address) +{ + return (address >> 4) & PTE_ADDRESS; +} + +static int fabric_powered(void); +static int remove_private_mapping(void); +static void cleanup(void); + + +static int flush_sid(void) +{ + u32 status; + + /* DART table walks are coherent, as in apple_dart_finalize_domain. */ + pr_info("m3_dcpext_dma: flushing SID5\n"); + msleep(20); + dma_wmb(); + writel(0x100 | 5, regs + 0x80); + return readl_poll_timeout(regs + 0x80, status, !(status & BIT(31)), 1, 10000); +} + +static int fabric_powered(void) +{ + struct device_node *node; + struct regmap *map; + u32 state, i, offset = port ? 0x3c0 : 0x3b8; + int ret; + + node = of_find_node_by_path("/soc/power-management@350700000"); + if (!node || !of_device_is_compatible(node, "apple,t6030-pmgr")) { + of_node_put(node); + return -ENODEV; + } + map = syscon_node_to_regmap(node); + of_node_put(node); + if (IS_ERR(map)) + return PTR_ERR(map); + for (i = 0; i < 2; i++) { + ret = regmap_read(map, offset + (i ? (port ? 0x30 : 0x20) : 0), &state); + if (ret) + return ret; + if ((state & 0xff) != 0xff || + state & (BIT(31) | BIT(28) | BIT(12) | BIT(11) | BIT(10))) + return -EHOSTDOWN; + } + return 0; +} + +static void cleanup(void) +{ +#ifdef DCPEXT_WITH_SESSION + session_cleanup(); +#endif + if (root) + memunmap(root); + if (leaf) + free_page((unsigned long)leaf); + if (pool) + dma_free_coherent(dma_dev, LEASE_SIZE, pool, pool_dma); + if (dma_dev) + root_device_unregister(dma_dev); + if (regs) + iounmap(regs); + if (region_owned) + release_mem_region(dart_base, 0x4000); + kfree(saved); +} + +/* Called only after protocol completion, or by the no-firmware mapper. On + * any mismatch all allocations and the module reference remain owned. */ +static int remove_private_mapping(void) +{ + u64 entry = encode_address(virt_to_phys(leaf)) | 1; + unsigned int i; + int ret; + + /* Verify only our private entries; inherited mappings must be unchanged. */ + for (i = 0; i < LEASE_SIZE / LEASE_PAGE; i++) + if ((((u64 *)leaf)[i] & PTE_ADDRESS) << 4 != pool_dma + i * LEASE_PAGE) { + ret = -EIO; + return ret; + } + ret = -EAGAIN; + if (cmpxchg64_relaxed(&root[LEASE_SLOT], entry, 0) != entry) + return ret; + ret = flush_sid(); + if (ret) + return ret; + if (memcmp(root, saved, LEASE_PAGE) || (readl(regs + 0x100) & BIT(31))) { + ret = -EIO; + return ret; + } + return 0; +} + +static int __init probe_init(void) +{ + struct device_node *dcp = NULL, *dart = NULL, *ram = NULL; + struct resource resource, reservation; + phys_addr_t root_pa, leaf_pa; + const char *uuid; + char alias[16]; + u32 marker, ttbr, tcr, protect, params[2]; + u64 entry; + int count, segment_bytes, i, ret = -ENODEV; + + if (port > 1 || PAGE_SIZE != LEASE_PAGE || + !of_machine_is_compatible("apple,j514s") || + !of_machine_is_compatible("apple,t6030")) + return -ENODEV; + snprintf(alias, sizeof(alias), "dcpext%u", port); + dcp = of_find_node_by_path(alias); + if (!dcp || of_device_is_available(dcp) || + !of_device_is_compatible(dcp, "apple,t6030-dcpext") || + of_property_read_u32(dcp, "apple,j514s-dcpext-reservations", &marker) || marker != 1 || + of_property_read_string(dcp, "apple,firmware-uuid", &uuid) || + strcmp(uuid, "DDF38191-93B3-324A-BC8F-643006F5AC82")) + goto done; + if (!of_get_property(dcp, "apple,j514s-dcpext-segments", &segment_bytes) || + segment_bytes <= 0 || segment_bytes % 32) + goto done; + count = of_property_count_u32_elems(dcp, "memory-region"); + if (count != segment_bytes / 32 + 3 || + of_property_read_u32_array(dcp, "iommus", params, 2) || params[1] != 5) + goto done; + dart = of_find_node_by_phandle(params[0]); + ram = of_parse_phandle(dcp, "memory-region", count - 3); + dart_base = port ? 0x2d530c000ULL : 0x2d130c000ULL; + if (!dart || of_device_is_available(dart) || + !of_device_is_compatible(dart, "apple,t8110-dart") || + of_address_to_resource(dart, 0, &resource) || resource.start != dart_base || + resource_size(&resource) != 0x4000 || !ram || + !of_device_is_compatible(ram, "apple,dart-mem") || + !of_property_read_bool(ram, "no-map") || + of_address_to_resource(ram, 0, &reservation)) + goto done; + ret = fabric_powered(); + if (ret) + goto done; + ret = -EBUSY; + if (!request_mem_region(dart_base, 0x4000, "m3-dcpext-dma-probe")) + goto done; + region_owned = true; + /* Match of_iomap/devm_ioremap_resource: T6030 soc is nonposted-mmio. */ + if (!(resource.flags & IORESOURCE_MEM_NONPOSTED)) { + ret = -EINVAL; + goto done; + } + regs = ioremap_np(dart_base, 0x4000); + ret = -ENOMEM; + if (!regs) + goto done; + protect = readl(regs + 0x200); + tcr = readl(regs + 0x1000 + 5 * 4); + ttbr = readl(regs + 0x1400 + 5 * 4); + root_pa = (u64)((ttbr & GENMASK(29, 2)) >> 2) << 14; + pr_info("m3_dcpext_dma: protect=%#x tcr=%#x ttbr=%#x root=%pap reservation=%pr error=%#x\n", + protect, tcr, ttbr, &root_pa, &reservation, readl(regs + 0x100)); + ret = -EINVAL; + /* T8110 bits 11:8 are ignored while REMAP_EN (bit 7) is clear. + * Firmware also leaves 0x801 here; require all active controls exactly + * as before and recheck the untouched full TCR before publication. */ + if (!(protect & 1) || (tcr & ~GENMASK(11, 8)) != 1 || !(ttbr & 1) || + root_pa < reservation.start || root_pa > reservation.end || + reservation.end - root_pa + 1 < LEASE_PAGE || (readl(regs + 0x100) & BIT(31))) + goto done; + root = memremap(root_pa, LEASE_PAGE, MEMREMAP_WB); + ret = -ENOMEM; + if (!root) + goto done; + saved = kmemdup(root, LEASE_PAGE, GFP_KERNEL); + leaf = (void *)get_zeroed_page(GFP_KERNEL); + if (!saved || !leaf) + goto done; + ret = -EBUSY; + if (READ_ONCE(root[LEASE_SLOT])) + goto done; +#ifdef DCPEXT_WITH_SESSION + ret = session_prepare(dcp); + if (!ret) ret = session_tables(&reservation); + if (!ret) ret = session_audit(); + if (ret) goto done; +#endif + dma_dev = root_device_register(port ? "m3-dcpext1" : "m3-dcpext-dma-probe"); + if (IS_ERR(dma_dev)) { + ret = PTR_ERR(dma_dev); + dma_dev = NULL; + goto done; + } + /* No OF node / IOMMU attachment: obtain a host-owned direct allocation. + * Require identity DMA-to-physical translation for the DART leaf. + */ + ret = dma_coerce_mask_and_coherent(dma_dev, DMA_BIT_MASK(42)); + if (ret) + goto done; + pool = dma_alloc_coherent(dma_dev, LEASE_SIZE, &pool_dma, GFP_KERNEL); + ret = -ENOMEM; + if (!pool) + goto done; + leaf_pa = virt_to_phys(leaf); + pr_info("m3_dcpext_dma: pool=%pad physical=%#llx leaf=%pap\n", + &pool_dma, (u64)dma_to_phys(dma_dev, pool_dma), &leaf_pa); + ret = -EINVAL; + if ((pool_dma | leaf_pa) & (LEASE_PAGE - 1) || + dma_to_phys(dma_dev, pool_dma) != pool_dma || + pool_dma + LEASE_SIZE > BIT_ULL(42) || leaf_pa >= BIT_ULL(42)) + goto done; + for (i = 0; i < LEASE_SIZE / LEASE_PAGE; i++) + ((u64 *)leaf)[i] = encode_address(pool_dma + i * LEASE_PAGE) | PTE_RW; + memset(pool, 0xa5, LEASE_SIZE); + dma_wmb(); + entry = encode_address(leaf_pa) | 1; + ret = -EAGAIN; + if (readl(regs + 0x1400 + 5 * 4) != ttbr || + readl(regs + 0x1000 + 5 * 4) != tcr || + readl(regs + 0x200) != protect || memcmp(root, saved, LEASE_PAGE)) + goto done; + /* A published table must survive failed invalidation/teardown. */ + __module_get(THIS_MODULE); + pinned = true; + if (cmpxchg64_relaxed(&root[LEASE_SLOT], 0, entry)) { + module_put(THIS_MODULE); + pinned = false; + goto done; + } + ret = flush_sid(); + if (ret) + goto done; + pr_info("m3_dcpext_dma_probe: port=%u SID5 private DVA=%#llx bytes=%#x mapped; no firmware notification\n", + port, LEASE_DVA, LEASE_SIZE); +#ifdef DCPEXT_WITH_SESSION + ret = frame_prepare_publish(dcp,count); + if(!ret)ret = pio_prepare_publish(dcp,count); + if(!ret)ret = session_start(); + goto done; /* Keep every DMA allocation and module reference until reboot. */ +#endif + ret = remove_private_mapping(); + if (ret) + goto done; + module_put(THIS_MODULE); + pinned = false; + pr_info("m3_dcpext_dma_probe: private mapping removed; inherited root unchanged\n"); +done: + pr_info("m3_dcpext_dma: result=%d published=%u\n", ret, pinned); + of_node_put(ram); + of_node_put(dart); + of_node_put(dcp); + result = ret; + if (pinned) { + pr_info("m3_dcpext_dma_probe: result=%d; retaining all allocations/resources until reboot\n", ret); + return 0; + } + cleanup(); + return ret; +} + +static void __exit probe_exit(void) +{ +} +module_init(probe_init); +module_exit(probe_exit); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("J514S external DART additive mapping/remove diagnostic, no firmware execution"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_frame.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_frame.h new file mode 100644 index 0000000000000..03c97a40518c1 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_frame.h @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Paged static image adapted from M4 dcpext_frame_map.h. M3 SID5/FE0, + * qualified reservations and nonposted accesses. All allocations are retained + * until reboot, including on partial publication: the caller is already pinned. + */ +#include +#define FRAME_DVA 0x30000000ULL +#define FRAME_SLOT (FRAME_DVA >> 25) +#define FRAME_BYTES ALIGN(1920 * 1080 * 4 + 24 * LEASE_PAGE, LEASE_PAGE) +static bool iboot_image; +module_param(iboot_image,bool,0400); +static void *frame_pixels,*frame_ram; +static u64 *frame_leaf,*frame_root; +static void __iomem *frame_regs; +static dma_addr_t frame_dma[2*FRAME_BYTES/LEASE_PAGE]; +static u64 encode_address(phys_addr_t address); +static int flush_sid(void); +static int frame_prepare_publish(struct device_node *dcp,int count) +{ + struct device_node *ram,*disp,*dart; + struct resource reservation,res; + u32 params[2],ttbr,tcr,protect,status; + phys_addr_t pa;int ret=-EINVAL; + if(!iboot_image && !native_startup)return 0; + if(!pinned)return -EINVAL; + ram=of_parse_phandle(dcp,"memory-region",count-2); + if(!ram)return -EINVAL; + if(!of_device_is_compatible(ram,"apple,dart-mem") || !of_property_read_bool(ram,"no-map") || + of_address_to_resource(ram,0,&reservation))goto ram_out; + if(resource_size(&reservation)>0x100000 || resource_size(&reservation)>2)<<14; + ret=-EINVAL; + if((tcr & ~GENMASK(11,8))!=5 || !(ttbr&1) || !(protect&1) || pareservation.end || + reservation.end-pa+1=BIT_ULL(42)){ret=-ERANGE;goto ram_out;} + for(unsigned int y=0;y<1080;y++)for(unsigned int x=0;x<1920;x++){ + static const u32 bars[]={0xffff0000,0xff00ff00,0xff0000ff,0xffffff00,0xffff00ff,0xff00ffff,0xffffffff,0xff202020}; + ((u32 *)frame_pixels)[y*1920+x]=x<8 || y<8 || x>=1912 || y>=1072?0xffffffff:bars[x*8/1920]; + } + for(unsigned int i=0;i=BIT_ULL(42) || frame_dma[i]&(LEASE_PAGE-1) || dma_to_phys(dma_dev,frame_dma[i])!=frame_dma[i]){ + ret=-ERANGE;goto ram_out; + } + frame_leaf[i]=encode_address(frame_dma[i])|PTE_RW; + } + ret=-EAGAIN; + if(readl(frame_regs+0x1400)!=ttbr || readl(frame_regs+0x1000)!=tcr || readl(frame_regs+0x200)!=protect)goto ram_out; + dma_wmb(); + if(cmpxchg64_relaxed(&root[FRAME_SLOT],0,encode_address(virt_to_phys(frame_leaf))|1) || + cmpxchg64_relaxed(&frame_root[FRAME_SLOT],0,encode_address(virt_to_phys(frame_leaf))|1))goto ram_out; + ret=flush_sid();if(ret)goto ram_out; + dma_wmb();writel(0x100,frame_regs+0x80); + ret=readl_poll_timeout(frame_regs+0x80,status,!(status&BIT(31)),1,10000); + if(!ret && (readl(frame_regs+0x100)&BIT(31)))ret=-EIO; + pr_info("m3_dcpext_frame: SID5/FE0 DVA=%#llx bytes=%#lx mapping result=%d; retained until reboot\n",FRAME_DVA,(unsigned long)FRAME_BYTES,ret); +ram_out: + of_node_put(ram);return ret; +} + +/* Native D451/D201 allocations share the retained host arena with PIODMA. + * SID4 root has its own reservation; never alias or replace inherited leaves. */ +static bool pio_published; +static int pio_prepare_publish(struct device_node *dcp,int count) +{ + struct device_node *ram; + struct resource res; + u32 ttbr,tcr,status; + phys_addr_t pa; + u64 *pio_root; + void *mapping; + int ret=-EINVAL; + if(!native_startup)return 0; + if(!frame_regs || !pinned)return -EINVAL; + ram=of_parse_phandle(dcp,"memory-region",count-1); + if(!ram)return -EINVAL; + if(!of_device_is_compatible(ram,"apple,dart-mem") || !of_property_read_bool(ram,"no-map") || + of_address_to_resource(ram,0,&res))goto out; + ttbr=readl(frame_regs+0x1410);tcr=readl(frame_regs+0x1010); + pa=(u64)((ttbr&GENMASK(29,2))>>2)<<14; + pr_info("m3_dcpext_native: PIODMA TCR=%#x TTBR=%#x root=%pap reservation=%pR\n",tcr,ttbr,&pa,&res); + /* T8110 TCR bits11:8 are the remap selector; bit7 enables remapping. + * Boot values a05/b05/e05 all have remapping disabled. Qualify every active + * control bit, preserve the ignored selector, and recheck the exact register + * before publication. Definitions: drivers/iommu/apple-dart.c. */ + if((tcr!=0xa05 && (tcr & ~GENMASK(11,8))!=5) || !(ttbr&1) || pares.end || res.end-pa+10x100000 || (res.start|resource_size(&res))&(LEASE_PAGE-1))goto out; + mapping=memremap(res.start,resource_size(&res),MEMREMAP_WB); + if(!mapping){ret=-ENOMEM;goto out;} + pio_root=mapping+pa-res.start; + ret=-EBUSY; + if(READ_ONCE(pio_root[LEASE_SLOT]) || readl(frame_regs+0x1410)!=ttbr || + readl(frame_regs+0x1010)!=tcr)goto out; + dma_wmb(); + if(cmpxchg64_relaxed(&pio_root[LEASE_SLOT],0,encode_address(virt_to_phys(leaf))|1))goto out; + dma_wmb();writel(0x104,frame_regs+0x80); + ret=readl_poll_timeout(frame_regs+0x80,status,!(status&BIT(31)),1,10000); + if(!ret && (readl(frame_regs+0x100)&BIT(31)))ret=-EIO; + if(!ret)pio_published=true; + pr_info("m3_dcpext_native: PIODMA SID4 arena mapped ret=%d; retained until reboot\n",ret); +out: + of_node_put(ram);return ret; +} + +static void frame_sync(unsigned int slot,bool for_cpu) +{ + unsigned int first=slot*(FRAME_BYTES/LEASE_PAGE); + for(unsigned int i=first;i=108 && p[0]==2 && p[20]==4 && p[21]==0 && + get_unaligned_le16(p+22)==0x30 && !memcmp(p+40,"disp0-service",14)){ + if(ib_announced)return -EPROTO; + ib_announced=true;ib_channel=channel;return 0; + } + if(size<70 || p[0]!=2 || p[20]!=4 || p[21]!=0x20 || + get_unaligned_le16(p+22)!=0xc0)return 0; + if(!ib_pending || channel!=ib_channel || get_unaligned_le16(p+32)!=ib_tag || + get_unaligned_le64(p+44)!=ib_rx_dva)return -EPROTO; + bytes=get_unaligned_le32(p+60); + pr_info("m3_dcpext_iboot: reply op=%u status=%#x bytes=%u\n",ib_operation,get_unaligned_le32(p+40),bytes); + if(get_unaligned_le32(p+40))return -EREMOTEIO; + if(bytes>4096)return -EOVERFLOW; + dma_rmb(); + if(ib_operation!=2 && ib_operation!=6 && ib_operation!=1){ + if(bytes<8 || get_unaligned_le32(ib_rx)!=ib_operation || + get_unaligned_le32(ib_rx+4)>bytes || get_unaligned_le32(ib_rx+4)<8)return -EPROTO; + if(ib_operation==3){ + if(bytes<20 || ib_rx[8]>1)return -EPROTO; + if(!ib_rx[8] || !get_unaligned_le32(ib_rx+12) || !get_unaligned_le32(ib_rx+16)){ + if(++ib_attempts>=16)return -ENODEV; + ib_next_query=session_now(NULL)+250;ib_pending=false;ib_tag++;return 0; + } + ib_timings=get_unaligned_le32(ib_rx+12);ib_colors=get_unaligned_le32(ib_rx+16); + if(!ib_timings || !ib_colors || ib_timings>128 || ib_colors>128)return -ERANGE; + pr_info("m3_dcpext_iboot: HPD=1 timing_count=%u color_count=%u\n",ib_timings,ib_colors); + }else{ + if(bytes<12)return -EPROTO; + count=get_unaligned_le32(ib_rx+8); + if(count!=(ib_operation==4?ib_timings:ib_colors) || bytes<12+count*24)return -EPROTO; + for(unsigned int i=0;i=ARRAY_SIZE(ops))return -EPROTO; + if(iboot_image && ib_stage==6 && session_now(NULL) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "m3_dcpext_kms.h" + +struct m3_dcpext_kms { + struct drm_device drm; + struct drm_plane plane; + struct drm_crtc crtc; + struct drm_encoder encoder; + struct m3_dcpext_connector connector; + const struct m3_dcpext_kms_ops *ops; + void *cookie; + struct mutex *route_lock; + /* Outlives the hardware owner while userspace retains DRM references. */ + struct mutex owner_lock; + bool detached; + bool failed; + u64 completed; +}; + +static struct m3_dcpext_kms *to_kms(struct drm_device *drm) +{ + return container_of(drm, struct m3_dcpext_kms, drm); +} + +static int plane_check(struct drm_plane *plane, struct drm_atomic_state *state) +{ + struct drm_plane_state *p = drm_atomic_get_new_plane_state(state, plane); + struct drm_crtc_state *c; + struct drm_gem_object *obj; + u64 extent; + int ret; + + if (!p->crtc) + return 0; + c = drm_atomic_get_crtc_state(state, p->crtc); + if (IS_ERR(c)) + return PTR_ERR(c); + ret = drm_atomic_helper_check_plane_state(p, c, DRM_PLANE_NO_SCALING, + DRM_PLANE_NO_SCALING, false, true); + if (ret || !p->visible) + return ret; + /* Native v1 supports one full-screen linear RGB plane; no cropping. */ + if (p->src_x || p->src_y || p->crtc_x || p->crtc_y || + p->src_w != (u32)c->mode.hdisplay << 16 || + p->src_h != (u32)c->mode.vdisplay << 16 || + p->crtc_w != c->mode.hdisplay || p->crtc_h != c->mode.vdisplay || + p->fb->width != c->mode.hdisplay || p->fb->height != c->mode.vdisplay || + p->fb->modifier != DRM_FORMAT_MOD_LINEAR || + p->fb->pitches[0] < p->fb->width * 4) + return -EINVAL; + obj = drm_gem_fb_get_obj(p->fb, 0); + extent = (u64)p->fb->pitches[0] * (p->fb->height - 1) + + (u64)p->fb->width * 4 + p->fb->offsets[0]; + return obj && extent <= obj->size ? 0 : -EINVAL; +} + +static void plane_update(struct drm_plane *plane, struct drm_atomic_state *state) {} +static const struct drm_plane_helper_funcs plane_helpers = { + .prepare_fb = drm_gem_plane_helper_prepare_fb, + .atomic_check = plane_check, + .atomic_update = plane_update, + .atomic_disable = plane_update, +}; +static const struct drm_plane_funcs plane_funcs = { + .update_plane = drm_atomic_helper_update_plane, + .disable_plane = drm_atomic_helper_disable_plane, + .destroy = drm_plane_cleanup, + .reset = drm_atomic_helper_plane_reset, + .atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_plane_destroy_state, +}; + +static int present(struct m3_dcpext_kms *kms, struct drm_atomic_state *state, + bool enable) +{ + struct drm_crtc_state *c = drm_atomic_get_new_crtc_state(state, &kms->crtc); + struct drm_connector_state *connector; + struct drm_framebuffer *fb = NULL; + struct m3_dcpext_native_mode mode; + struct iosys_map map[DRM_FORMAT_MAX_PLANES], data[DRM_FORMAT_MAX_PLANES]; + u64 generation = 0; + int ret; + + if (enable) { + struct drm_plane_state *p = drm_atomic_get_new_plane_state(state, &kms->plane); + if (!p) + return -EINVAL; + fb = p->fb; + } + if (fb) { + ret = drm_gem_fb_begin_cpu_access(fb, DMA_FROM_DEVICE); + if (ret) + return ret; + ret = drm_gem_fb_vmap(fb, map, data); + if (ret) + goto end_access; + } + mutex_lock(&kms->owner_lock); + if (kms->detached) { + ret = -ENODEV; + goto unlock_owner; + } + mutex_lock(kms->route_lock); + ret = kms->failed ? -EIO : 0; + if (!ret && enable) { + connector = drm_atomic_get_new_connector_state(state, &kms->connector.base); + ret = connector ? m3_dcpext_connector_commit_native(&kms->connector, + connector, &c->mode, &mode, &generation) : -EINVAL; + } + if (!ret) + ret = kms->ops->present(kms->cookie, enable ? &mode : NULL, generation, + fb ? &data[0] : NULL, fb ? fb->pitches[0] : 0, + fb && fb->format->format == DRM_FORMAT_XRGB8888); + if (!ret && enable) + kms->completed++; + if (ret && ret != -ESTALE && ret != -ENODEV) { + kms->failed = true; + drm_err(&kms->drm, "external commit failed %d; scanout retained by owner\n", ret); + } + mutex_unlock(kms->route_lock); +unlock_owner: + mutex_unlock(&kms->owner_lock); + if (fb) + drm_gem_fb_vunmap(fb, map); +end_access: + if (fb) + drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE); + return ret; +} + +static void finish_event(struct drm_crtc *crtc, struct drm_atomic_state *state, int ret) +{ + struct drm_crtc_state *c = drm_atomic_get_new_crtc_state(state, crtc); + struct drm_pending_vblank_event *event = c->event; + unsigned long flags; + + if (!ret && c->active) + drm_crtc_handle_vblank(crtc); + if (!event) + return; + c->event = NULL; + if (ret) { + /* Cancel the userspace event without claiming a presentation, but + * release the helper dependency and signal an error on its fence. + * Otherwise a rejected commit blocks every later modeset/unplug. */ + if (event->base.fence) { + dma_fence_set_error(event->base.fence, ret); + dma_fence_signal(event->base.fence); + } + if (event->base.completion) { + complete_all(event->base.completion); + event->base.completion_release(event->base.completion); + } + drm_event_cancel_free(crtc->dev, &event->base); + return; + } + spin_lock_irqsave(&crtc->dev->event_lock, flags); + drm_crtc_send_vblank_event(crtc, event); + spin_unlock_irqrestore(&crtc->dev->event_lock, flags); +} + +static void crtc_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) +{ + if (drm_atomic_get_new_crtc_state(state, crtc)->active) + finish_event(crtc, state, present(to_kms(crtc->dev), state, true)); +} +static void crtc_enable(struct drm_crtc *crtc, struct drm_atomic_state *state) +{ + const struct drm_vblank_crtc_config config = { .offdelay_ms = 0 }; + drm_crtc_vblank_on_config(crtc, &config); +} +static void crtc_disable(struct drm_crtc *crtc, struct drm_atomic_state *state) +{ + int ret = present(to_kms(crtc->dev), state, false); + /* An active-to-active modeset completes after its new surface, not + * after temporarily powering off the old timing. */ + if (!drm_atomic_get_new_crtc_state(state, crtc)->active) + finish_event(crtc, state, ret); + drm_crtc_vblank_off(crtc); +} +static int enable_vblank(struct drm_crtc *crtc) { return 0; } +static void disable_vblank(struct drm_crtc *crtc) {} +static const struct drm_crtc_helper_funcs crtc_helpers = { + .atomic_enable = crtc_enable, + .atomic_disable = crtc_disable, + .atomic_flush = crtc_flush, +}; +static const struct drm_crtc_funcs crtc_funcs = { + .set_config = drm_atomic_helper_set_config, + .page_flip = drm_atomic_helper_page_flip, + .destroy = drm_crtc_cleanup, + .reset = drm_atomic_helper_crtc_reset, + .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state, + .enable_vblank = enable_vblank, + .disable_vblank = disable_vblank, +}; +static const struct drm_encoder_funcs encoder_funcs = { .destroy = drm_encoder_cleanup }; + +static int atomic_check(struct drm_device *drm, struct drm_atomic_state *state) +{ + struct m3_dcpext_kms *kms = to_kms(drm); + struct drm_crtc_state *crtc; + struct drm_connector_state *connector; + int ret; + + crtc = drm_atomic_get_crtc_state(state, &kms->crtc); + if (IS_ERR(crtc)) + return PTR_ERR(crtc); + if (READ_ONCE(kms->failed) && crtc->active) + return -EIO; + connector = drm_atomic_get_connector_state(state, &kms->connector.base); + if (IS_ERR(connector)) + return PTR_ERR(connector); + ret = drm_atomic_add_affected_planes(state, &kms->crtc); + return ret ?: drm_atomic_helper_check(drm, state); +} +static void commit_tail(struct drm_atomic_state *state) +{ + drm_atomic_helper_commit_modeset_disables(state->dev, state); + drm_atomic_helper_commit_modeset_enables(state->dev, state); + drm_atomic_helper_commit_planes(state->dev, state, DRM_PLANE_COMMIT_ACTIVE_ONLY); + /* present already waited for firmware completion; no timer vblank. */ + drm_atomic_helper_commit_hw_done(state); + drm_atomic_helper_cleanup_planes(state->dev, state); +} +static const struct drm_mode_config_funcs mode_funcs = { + .fb_create = drm_gem_fb_create, + .atomic_check = atomic_check, + .atomic_commit = drm_atomic_helper_commit, +}; +static const struct drm_mode_config_helper_funcs mode_helpers = { .atomic_commit_tail = commit_tail }; +DEFINE_DRM_GEM_FOPS(kms_fops); +static const struct drm_driver kms_driver = { + /* The common DRM syncobj implementation supports imported render fences + * without requiring a render engine on the display device. */ + .driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC | + DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE, + DRM_GEM_SHMEM_DRIVER_OPS, + .fops = &kms_fops, + .name = "m3-dcpext", + .desc = "J514S native HDMI", + .major = 0, .minor = 1, +}; + +struct m3_dcpext_kms *m3_dcpext_kms_create(struct device *dev, + const struct m3_dcpext_kms_ops *ops, void *cookie, struct mutex *route_lock, int connector_type) +{ + static const u32 formats[] = { DRM_FORMAT_XRGB8888, DRM_FORMAT_ARGB8888 }; + static const u64 modifiers[] = { DRM_FORMAT_MOD_LINEAR, DRM_FORMAT_MOD_INVALID }; + struct m3_dcpext_kms *kms; + struct drm_device *drm; + int ret; + + if (!ops || !ops->present || !route_lock) + return ERR_PTR(-EINVAL); + kms = devm_drm_dev_alloc(dev, &kms_driver, struct m3_dcpext_kms, drm); + if (IS_ERR(kms)) + return kms; + drm = &kms->drm; + kms->ops = ops; + kms->cookie = cookie; + kms->route_lock = route_lock; + mutex_init(&kms->owner_lock); + ret = drmm_mode_config_init(drm); + if (ret) + return ERR_PTR(ret); + drm->mode_config.min_width = drm->mode_config.min_height = 1; + drm->mode_config.max_width = M3_DCPEXT_MAX_WIDTH; + drm->mode_config.max_height = M3_DCPEXT_MAX_HEIGHT; + drm->mode_config.funcs = &mode_funcs; + drm->mode_config.helper_private = &mode_helpers; + ret = drm_universal_plane_init(drm, &kms->plane, 1, &plane_funcs, formats, + ARRAY_SIZE(formats), modifiers, DRM_PLANE_TYPE_PRIMARY, NULL); + if (ret) + return ERR_PTR(ret); + drm_plane_helper_add(&kms->plane, &plane_helpers); + ret = drm_crtc_init_with_planes(drm, &kms->crtc, &kms->plane, NULL, &crtc_funcs, NULL); + if (ret) + return ERR_PTR(ret); + drm_crtc_helper_add(&kms->crtc, &crtc_helpers); + ret = drm_encoder_init(drm, &kms->encoder, &encoder_funcs, DRM_MODE_ENCODER_TMDS, NULL); + if (ret) + return ERR_PTR(ret); + kms->encoder.possible_crtcs = 1; + ret = m3_dcpext_connector_init(drm, &kms->connector, &kms->encoder, connector_type); + if (ret) + return ERR_PTR(ret); + ret = drm_vblank_init(drm, 1); + if (ret) + return ERR_PTR(ret); + drm_mode_config_reset(drm); + return kms; +} +struct m3_dcpext_connector *m3_dcpext_kms_connector(struct m3_dcpext_kms *kms) +{ + return &kms->connector; +} +int m3_dcpext_kms_register(struct m3_dcpext_kms *kms) +{ + return drm_dev_register(&kms->drm, 0); +} +void m3_dcpext_kms_unplug(struct m3_dcpext_kms *kms) +{ + drm_dev_unplug(&kms->drm); + drm_atomic_helper_shutdown(&kms->drm); + /* Shutdown performs the final disable while the owner is still alive. + * Serialize with any late commit before severing callback references. + * Existing DRM/GEM file references may survive physical device removal. + */ + mutex_lock(&kms->owner_lock); + kms->detached = true; + kms->ops = NULL; + kms->cookie = NULL; + kms->route_lock = NULL; + mutex_unlock(&kms->owner_lock); +} +MODULE_IMPORT_NS("DMA_BUF"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.h new file mode 100644 index 0000000000000..628c8fd5f021d --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef M3_DCPEXT_KMS_H +#define M3_DCPEXT_KMS_H +#include "m3_dcpext_connector.h" +struct device; +struct iosys_map; +struct m3_dcpext_kms; + +/* The route mutex serializes callbacks/hotplug and hardware commits. Pixels + * are CPU-readable until present returns; the owner copies them into its + * retained scanout slots. Success requires matching firmware completion. + * Disable has mode/pixels NULL; an active mode with pixels NULL presents only + * the background. Neither operation may free uncertain DMA resources. */ +struct m3_dcpext_kms_ops { + int (*present)(void *cookie, const struct m3_dcpext_native_mode *mode, + u64 generation, const struct iosys_map *pixels, + u32 source_pitch, bool opaque); +}; +struct m3_dcpext_kms *m3_dcpext_kms_create(struct device *dev, + const struct m3_dcpext_kms_ops *ops, void *cookie, + struct mutex *route_lock, int connector_type); +struct m3_dcpext_connector *m3_dcpext_kms_connector(struct m3_dcpext_kms *kms); +int m3_dcpext_kms_register(struct m3_dcpext_kms *kms); +void m3_dcpext_kms_unplug(struct m3_dcpext_kms *kms); +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms_adapter.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms_adapter.h new file mode 100644 index 0000000000000..897e1dcefe68c --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms_adapter.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* M4 native KMS ownership pattern adapted to J514S old RPC protocol. */ +#include +#include "m3_dcpext_kms.h" +static DEFINE_MUTEX(native_route_lock); +static struct m3_dcpext_kms *native_kms; +static bool native_kms_ready,native_kms_finished,native_kms_powered=true; +static bool native_kms_connected; +static unsigned int native_kms_frames,native_kms_slot; +static u64 native_kms_until,native_kms_generation; +module_param(native_kms_ready,bool,0400); +module_param(native_kms_frames,uint,0400); +static int native_kms_present(void *cookie,const struct m3_dcpext_native_mode *mode, + u64 generation,const struct iosys_map *pixels,u32 source_pitch,bool opaque) +{ + u8 surface[0x22c];void *destination;u64 dva;int ret; + lockdep_assert_held(&native_route_lock); + /* Hotplug is an ordinary stale atomic commit, not a poisoned device. */ + if(desktop_disconnected || !native_kms_connected)return mode?-ESTALE:0; + if(!mode){ + if(!native_kms_powered)return 0; + ret=m3_dcpext_native_power(native_client,false); + if(!ret)native_kms_powered=false;return ret; + } + if(native_kms_finished || !native_kms_ready || generation!=native_kms_generation || + generation!=m3_dcpext_native_generation(native_client))return -ESTALE; + if(mode->geometry.width!=1920 || mode->geometry.height!=1080 || (pixels && source_pitch<7680))return -EINVAL; + if(!native_kms_powered){ret=m3_dcpext_native_power(native_client,true);if(ret)return ret;native_kms_powered=true;} + if(!pixels)return m3_dcpext_native_background(native_client,0xff000000); + /* Initial color bars occupy A. Copy to B first; alternate only on D589. */ + unsigned int slot=native_kms_slot?0:1; + destination=frame_pixels+slot*FRAME_BYTES; + dva=(FRAME_DVA+slot*FRAME_BYTES)|BIT_ULL(40); + frame_sync(slot,true); + for(u32 y=0;y<1080;y++)iosys_map_memcpy_from(destination+y*7680,pixels,(size_t)y*source_pitch,7680); + frame_sync(slot,false);dma_wmb();native_surface(surface,opaque); + ret=m3_dcpext_native_swap(native_client,surface,dva,1920,1080); + if(!ret){native_kms_slot^=1;native_kms_frames++;} + return ret; +} +static const struct m3_dcpext_kms_ops native_kms_ops={.present=native_kms_present}; +static int native_kms_publish(void) +{ + struct m3_dcpext_native_mode *modes; + const struct m3_dcpext_native_mode wanted={ + .geometry={1920,1080,148500,2200,1125,false,false}, + .hfront=88,.hsync=44,.vfront=4,.vsync=5,.hpositive=true,.vpositive=true}; + struct m3_dcpext_connector *connector; + void *blob;u32 bytes=0,count,index,w,h;u64 generation;int ret; + if(native_kms_connected || !av_edid_done)return 0; + blob=m3_dcpext_native_property(native_client,"DisplayAttributes",&bytes); + if(!blob)return 0; + ret=m3_dcpext_native_dimensions(blob,bytes,&w,&h);kfree(blob);if(ret)return ret; + blob=m3_dcpext_native_property(native_client,"TimingElements",&bytes); + if(!blob)return 0; + modes=kcalloc(M3_DCPEXT_NATIVE_MAX_MODES,sizeof(*modes),GFP_KERNEL); + if(!modes){kfree(blob);return -ENOMEM;} + ret=m3_dcpext_native_modes_parse(blob,bytes,modes,M3_DCPEXT_NATIVE_MAX_MODES,&count);kfree(blob); + if(!ret)ret=m3_dcpext_native_mode_select(&wanted,modes,count,&index); + if(ret)goto out; + native_kms_generation=m3_dcpext_native_generation(native_client); + if(!native_kms){ + native_kms=m3_dcpext_kms_create(dma_dev,&native_kms_ops,NULL,&native_route_lock, + usb_c?DRM_MODE_CONNECTOR_DisplayPort:DRM_MODE_CONNECTOR_HDMIA); + if(IS_ERR(native_kms)){ret=PTR_ERR(native_kms);native_kms=NULL;goto out;} + } + connector=m3_dcpext_kms_connector(native_kms); + mutex_unlock(&native_route_lock); + generation=m3_dcpext_connector_invalidate(connector,true); + ret=m3_dcpext_connector_publish_native(connector,generation,av_edid,av_edid_bytes,&modes[index],1,native_kms_generation); + mutex_lock(&native_route_lock); + if(ret)goto out; + native_kms_connected=true;native_kms_powered=true;native_kms_slot=0; + if(native_kms_ready){ + mutex_unlock(&native_route_lock); + m3_dcpext_connector_hotplug(connector); + mutex_lock(&native_route_lock); + pr_info("m3_dcpext_native: HDMI reconnected: fresh EDID and timing generation %llu\n",native_kms_generation); + goto out; + } + if(native_desktop){ret=dcpext_session_enter_runtime(&session);if(ret)goto out;afk.streaming=true;aux_afk.streaming=true;} + native_kms_ready=true;native_kms_until=session_now(NULL)+8000; + ret=m3_dcpext_kms_register(native_kms); + if(ret)native_kms_ready=false; + else { + pr_info("m3_dcpext_native: HDMI DRM registered: 1080p60 %ux%u mm, desktop=%u\n",w,h,native_desktop); + if(native_desktop)desktop_notify(0); + } +out: + kfree(modes);return ret; +} +/* Keep DRM and the firmware session alive while the cable is absent. */ +static void native_kms_disconnect(void) +{ + lockdep_assert_held(&native_route_lock); + native_kms_connected=false; + native_frame_submitted=false;av_read_ready=false;av_edid_done=false; + mutex_unlock(&native_route_lock); + m3_dcpext_connector_invalidate(m3_dcpext_kms_connector(native_kms),false); + m3_dcpext_connector_hotplug(m3_dcpext_kms_connector(native_kms)); + mutex_lock(&native_route_lock); +} +/* Never call with route lock held: final disable runs through present. */ +static void native_kms_unplug(void) +{ + if(!native_kms_ready || native_kms_finished)return; + m3_dcpext_connector_invalidate(m3_dcpext_kms_connector(native_kms),false); + m3_dcpext_connector_hotplug(m3_dcpext_kms_connector(native_kms)); + m3_dcpext_kms_unplug(native_kms); + mutex_lock(&native_route_lock); + native_kms_finished=true;native_frame_done=true; + pr_info("m3_dcpext_native: HDMI KMS trial finished; %u completed frames\n",native_kms_frames); + mutex_unlock(&native_route_lock); +} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c new file mode 100644 index 0000000000000..75fee95341180 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c @@ -0,0 +1,477 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include "m3_dcpext_modes.h" +#ifdef __KERNEL__ +#include +#include +#include +#else +#include +#include +static u32 get_unaligned_le32(const void *p) { u32 v; memcpy(&v, p, 4); return v; } +static u64 get_unaligned_le64(const void *p) { u64 v; memcpy(&v, p, 8); return v; } +#endif + +int m3_dcpext_surface_validate(u32 width, u32 height, u32 pitch, + u64 dva, u64 mapped_bytes) +{ + u64 bytes; + + /* Scope is the qualified external controller's linear SDR path. */ + if (!width || width > M3_DCPEXT_MAX_WIDTH || !height || height > M3_DCPEXT_MAX_HEIGHT || + pitch < width * 4 || (pitch & 63) || !dva || (dva & 0x3fff)) + return -EINVAL; + bytes = (u64)pitch * height; + if (bytes > mapped_bytes || dva >= (1ULL << 36) || + bytes > (1ULL << 36) - dva) + return -ERANGE; + return 0; +} + +int m3_dcpext_mode_validate(const struct m3_dcpext_mode *m, u32 *fps_16_16) +{ + u64 fps, total; + + if (!m || !fps_16_16 || !m->width || + m->width > M3_DCPEXT_MAX_WIDTH || !m->height || m->height > M3_DCPEXT_MAX_HEIGHT || !m->clock_khz || + m->htotal <= m->width || m->vtotal <= m->height || + m->htotal > 65535 || m->vtotal > 65535) + return -EINVAL; + if (m->interlaced || m->doublescan) + return -EOPNOTSUPP; + /* 4 * 2.7 Gbit/s * 8/10, divided by 24 wire bits per RGB pixel. */ + if ((u64)m->clock_khz * 24 > M3_DCPEXT_MAX_PAYLOAD_KBPS) + return -ERANGE; + total = (u64)m->htotal * m->vtotal; + fps = ((u64)m->clock_khz * 1000 * 65536 + total / 2) / total; + if (fps < 24 * 65536 || fps > 120 * 65536) + return -ERANGE; + *fps_16_16 = fps; + return 0; +} + +int m3_dcpext_mode_select(const struct m3_dcpext_mode *m, + const u8 *timings, u32 count, u8 selected[24]) +{ + const u8 *best = NULL; + u64 fps, best_delta = ~(u64)0; + + int ret; + u32 rate; + bool ambiguous = false; + + if (!timings || !selected || count > 64) + return -EINVAL; + ret = m3_dcpext_mode_validate(m, &rate); + if (ret) + return ret; + fps = rate; + for (u32 i = 0; i < count; i++) { + const u8 *r = timings + i * 24; + u32 reported = get_unaligned_le32(r + 12); + u64 delta = fps > reported ? fps - reported : reported - fps; + + if (get_unaligned_le32(r) != 1 || + get_unaligned_le32(r + 4) != m->width || + get_unaligned_le32(r + 8) != m->height || + get_unaligned_le32(r + 16) || get_unaligned_le32(r + 20)) + continue; + /* Captured DMT records use nominal integer rates: e.g. 800x600 + * 60.316 Hz -> 60, 640x480 72.809 Hz -> 72. Prefer the closest + * reported rate; fractional 59.94 and 60 remain distinguishable. + */ + if (delta >= 65536 || delta > best_delta) + continue; + if (delta < best_delta) + ambiguous = false; + else if (best && memcmp(best, r, 24)) + ambiguous = true; + best = r; + best_delta = delta; + } + if (!best) + return -ENODATA; + if (ambiguous) + return -ENOTUNIQ; + memcpy(selected, best, 24); + return 0; +} + +/* OSSerialize tags follow the existing Asahi parser and m1n1 fw/common.py. + * Bound every container and skip, including unknown fields. Keep this parser + * independent of DRM so captured and malformed publications use the same code + * in host sanitizer tests and in the driver. */ +struct native_value { + u32 type, count, begin, end; +}; +struct native_blob { + const u8 *data; + u32 bytes; +}; + +static int native_value_read(const struct native_blob *b, u32 *pos, + u32 depth, u32 *budget, struct native_value *v) +{ + struct native_value child; + u32 tag, children = 0, payload = 0; + int ret; + + if (depth > 16 || !*budget || *pos > b->bytes) + return -EINVAL; + --*budget; + *pos = (*pos + 3) & ~3U; + if (*pos > b->bytes || b->bytes - *pos < 4) + return -EINVAL; + tag = get_unaligned_le32(b->data + *pos); + *pos += 4; + *v = (struct native_value) { + .type = (tag >> 24) & 31, .count = tag & 0xffffff, + .begin = *pos, + }; + if (tag & 0x60000000) + return -EINVAL; + switch (v->type) { + case 1: + case 2: + if (v->count > 4096) + return -E2BIG; + children = v->count * (v->type == 1 ? 2 : 1); + break; + case 4: + if (v->count != 64) + return -EINVAL; + payload = 8; + break; + case 9: + case 10: + payload = v->count; + break; + case 11: + if (v->count > 1) + return -EINVAL; + break; + default: + return -EINVAL; + } + if (payload > b->bytes - *pos) + return -EINVAL; + *pos += payload; + for (u32 i = 0; i < children; i++) { + ret = native_value_read(b, pos, depth + 1, budget, &child); + if (ret) + return ret; + if (v->type == 1 && !(i & 1) && child.type != 9) + return -EINVAL; + } + v->end = *pos; + return 0; +} + +static int native_member(const struct native_blob *b, + const struct native_value *dict, const char *name, + struct native_value *out) +{ + struct native_value key, value; + u32 pos = dict->begin, budget = 65536; + bool found = false; + int ret; + + if (dict->type != 1) + return -EINVAL; + for (u32 i = 0; i < dict->count; i++) { + ret = native_value_read(b, &pos, 0, &budget, &key); + if (!ret) + ret = native_value_read(b, &pos, 0, &budget, &value); + if (ret || key.type != 9) + return -EINVAL; + if (key.count != strlen(name) || + memcmp(b->data + key.begin, name, key.count)) + continue; + if (found) + return -EINVAL; + *out = value; + found = true; + } + return found ? 0 : -ENOENT; +} + +static int native_uint(const struct native_blob *b, const struct native_value *d, + const char *name, u32 *out) +{ + struct native_value v; + u64 number; + int ret = native_member(b, d, name, &v); + + if (ret) + return ret; + if (v.type != 4) + return -EINVAL; + number = get_unaligned_le64(b->data + v.begin); + if (number > 0xffffffffULL) + return -ERANGE; + *out = number; + return 0; +} + +static int native_bool(const struct native_blob *b, const struct native_value *d, + const char *name, bool *out) +{ + struct native_value v; + int ret = native_member(b, d, name, &v); + + if (ret) + return ret; + if (v.type != 11) + return -EINVAL; + *out = v.count; + return 0; +} + +static int native_color_excluded(const struct native_blob *b, + const struct native_value *mode, + const char *name, u32 id) +{ + struct native_value array, value; + u32 pos, budget = 65536; + int ret = native_member(b, mode, name, &array); + + if (ret || array.type != 2) + return ret ?: -EINVAL; + pos = array.begin; + for (u32 i = 0; i < array.count; i++) { + ret = native_value_read(b, &pos, 0, &budget, &value); + if (ret || value.type != 4) + return -EINVAL; + if (get_unaligned_le64(b->data + value.begin) == id) + return 1; + } + return 0; +} + +static int native_color(const struct native_blob *b, + const struct native_value *mode, u32 *id) +{ + struct native_value array, color; + u32 pos, budget = 65536, best_score = 0; + bool found = false; + int ret = native_member(b, mode, "ColorModes", &array); + + if (ret || array.type != 2 || array.count > 64) + return -EINVAL; + pos = array.begin; + for (u32 i = 0; i < array.count; i++) { + u32 candidate, score, depth, encoding, eotf, range, colorimetry; + bool virtual; + + ret = native_value_read(b, &pos, 0, &budget, &color); + if (ret || native_bool(b, &color, "IsVirtual", &virtual)) + return -EINVAL; + if (virtual) + continue; + if (native_uint(b, &color, "ID", &candidate) || + native_uint(b, &color, "Score", &score) || + native_uint(b, &color, "Depth", &depth) || + native_uint(b, &color, "PixelEncoding", &encoding) || + native_uint(b, &color, "EOTF", &eotf) || + native_uint(b, &color, "Colorimetry", &colorimetry) || + native_uint(b, &color, "DynamicRange", &range)) + return -EINVAL; + if (depth != 8 || encoding || eotf || range || + (colorimetry != 10 && colorimetry != 16)) + continue; + ret = native_color_excluded(b, mode, "UnsafeColorElementIDs", candidate); + if (!ret) + ret = native_color_excluded(b, mode, "DSCRequiredColorElementIDs", candidate); + if (ret < 0) + return ret; + if (ret) + continue; + if (!found || score > best_score || (score == best_score && candidate < *id)) { + *id = candidate; + best_score = score; + found = true; + } + } + return found ? 0 : -EOPNOTSUPP; +} + +static int native_dimension(const struct native_blob *b, + const struct native_value *mode, const char *name, + u32 *active, u32 *total, u32 *front, u32 *sync, + bool *positive, u32 *rate) +{ + struct native_value dimension; + u32 back, polarity, repetition; + int ret = native_member(b, mode, name, &dimension); + + if (ret || native_uint(b, &dimension, "Active", active) || + native_uint(b, &dimension, "Total", total) || + native_uint(b, &dimension, "FrontPorch", front) || + native_uint(b, &dimension, "SyncWidth", sync) || + native_uint(b, &dimension, "BackPorch", &back) || + native_uint(b, &dimension, "SyncPolarity", &polarity) || + native_uint(b, &dimension, "PixelRepetition", &repetition) || + native_uint(b, &dimension, "PreciseSyncRate", rate)) + return -EINVAL; + if (!*active || !*sync || !*total || *total > 65535 || + (u64)*active + *front + *sync + back != *total || polarity > 1) + return -EINVAL; + if (repetition) + return -EOPNOTSUPP; + *positive = polarity; + return 0; +} + +/* 14.6 publishes IsSplit but omits pipe counts. Missing counts mean the + * single pipe only for a non-split timing; malformed/present counts still fail. */ +static int native_pipe_count(const struct native_blob *b, + const struct native_value *v, const char *key, + u32 *count) +{ + int ret = native_uint(b, v, key, count); + if (ret == -ENOENT) { *count = 1; return 0; } + return ret; +} + +static int native_mode(const struct native_blob *b, const struct native_value *v, + struct m3_dcpext_native_mode *out) +{ + struct m3_dcpext_mode *m = &out->geometry; + u32 horizontal_rate, rate, pipes_h, pipes_v, ignored; + bool virtual, interlaced, split; + u64 clock; + int ret; + + memset(out, 0, sizeof(*out)); + if (native_bool(b, v, "IsVirtual", &virtual)) + return -EINVAL; + if (virtual) + return -EOPNOTSUPP; + if (native_bool(b, v, "IsInterlaced", &interlaced) || + native_bool(b, v, "IsSplit", &split) || + native_bool(b, v, "IsPreferred", &out->preferred) || + native_pipe_count(b, v, "HorizontalPipeCount", &pipes_h) || + native_pipe_count(b, v, "VerticalPipeCount", &pipes_v) || + native_uint(b, v, "ID", &out->timing_id)) + return -EINVAL; + if (interlaced || split || pipes_h != 1 || pipes_v != 1) + return -EOPNOTSUPP; + ret = native_dimension(b, v, "HorizontalAttributes", &m->width, + &m->htotal, &out->hfront, &out->hsync, + &out->hpositive, &horizontal_rate); + if (!ret) + ret = native_dimension(b, v, "VerticalAttributes", &m->height, + &m->vtotal, &out->vfront, &out->vsync, + &out->vpositive, &rate); + if (ret) + return ret; + if (rate < 24 * 65536 || rate > 120 * 65536 || + m->width > M3_DCPEXT_MAX_WIDTH || m->height > M3_DCPEXT_MAX_HEIGHT) + return -EOPNOTSUPP; + clock = (u64)m->htotal * m->vtotal * rate; + m->clock_khz = (clock + 32768000) / 65536000; + ret = m3_dcpext_mode_validate(m, &ignored); + if (ret) + return -EOPNOTSUPP; + return native_color(b, v, &out->color_id); +} + +int m3_dcpext_native_modes_parse(const void *data, u32 bytes, + struct m3_dcpext_native_mode *modes, + u32 capacity, u32 *count) +{ + struct native_blob blob = { .data = data, .bytes = bytes }; + struct native_value root, value; + struct m3_dcpext_native_mode mode; + u32 pos = 4, budget = 65536, used = 0; + int ret; + + if (!count) + return -EINVAL; + *count = 0; + if (!data || !modes || !capacity || capacity > M3_DCPEXT_NATIVE_MAX_MODES || + bytes < 8 || bytes > 0x100000 || get_unaligned_le32(data) != 0xd3) + return -EINVAL; + ret = native_value_read(&blob, &pos, 0, &budget, &root); + if (ret || root.type != 2 || root.count > M3_DCPEXT_NATIVE_MAX_MODES || + ((pos + 3) & ~3U) != bytes) + return -EINVAL; + pos = root.begin; + budget = 65536; + for (u32 i = 0; i < root.count; i++) { + ret = native_value_read(&blob, &pos, 0, &budget, &value); + if (!ret) + ret = native_mode(&blob, &value, &mode); + if (ret == -EOPNOTSUPP) + continue; + if (ret) + return ret; + for (u32 j = 0; j < used; j++) + if (modes[j].timing_id == mode.timing_id) + return -EINVAL; + if (used == capacity) + return -ENOSPC; + modes[used++] = mode; + } + *count = used; + return 0; +} + +int m3_dcpext_native_mode_select(const struct m3_dcpext_native_mode *requested, + const struct m3_dcpext_native_mode *modes, + u32 count, u32 *selected) +{ + const struct m3_dcpext_mode *g; + u32 rate, best = 0, best_delta = ~0U; + bool found = false, ambiguous = false; + int ret; + + if (!requested || !modes || !selected || count > M3_DCPEXT_NATIVE_MAX_MODES) + return -EINVAL; + g = &requested->geometry; + ret = m3_dcpext_mode_validate(g, &rate); + if (ret) + return ret; + for (u32 i = 0; i < count; i++) { + const struct m3_dcpext_native_mode *m = &modes[i]; + u32 delta, candidate_rate; + + if (m3_dcpext_mode_validate(&m->geometry, &candidate_rate) || + g->width != m->geometry.width || g->height != m->geometry.height || + g->htotal != m->geometry.htotal || g->vtotal != m->geometry.vtotal || + requested->hfront != m->hfront || requested->hsync != m->hsync || + requested->vfront != m->vfront || requested->vsync != m->vsync || + requested->hpositive != m->hpositive || requested->vpositive != m->vpositive) + continue; + delta = rate > candidate_rate ? rate - candidate_rate : candidate_rate - rate; + /* Native DMT publications round rates (800x600 says 60 rather than + * 60.317 Hz). Keep EDID's exact clock, match all porches/polarities, + * and choose the uniquely closest native rate within one Hz. */ + if (delta >= 65536 || delta > best_delta) + continue; + if (found && delta == best_delta) + ambiguous = true; + else + ambiguous = false; + best = i; + best_delta = delta; + found = true; + } + if (!found) + return -ENODATA; + if (ambiguous) + return -ENOTUNIQ; + *selected = best; + return 0; +} + +int m3_dcpext_native_dimensions(const void *data,u32 bytes,u32 *width_mm,u32 *height_mm) +{ + struct native_blob b={.data=data,.bytes=bytes};struct native_value root; + u32 pos=4,budget=65536,w,h;int ret; + if(!data || bytes<8 || bytes>0x100000 || !width_mm || !height_mm || get_unaligned_le32(data)!=0xd3)return -EINVAL; + ret=native_value_read(&b,&pos,0,&budget,&root); + if(ret || root.type!=1 || ((pos+3)&~3U)!=bytes)return -EINVAL; + if(native_uint(&b,&root,"MaxHorizontalImageSize",&w) || + native_uint(&b,&root,"MaxVerticalImageSize",&h) || !w || !h || w>255 || h>255)return -EINVAL; + *width_mm=w*10;*height_mm=h*10;return 0; +} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.h new file mode 100644 index 0000000000000..fc725270df3fb --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef M3_DCPEXT_MODES_H +#define M3_DCPEXT_MODES_H +#ifdef __KERNEL__ +#include +#else +#include +#include +typedef uint8_t u8;typedef uint32_t u32;typedef uint64_t u64; +#endif + +/* Linear SDR scanout allocation and four-lane HBR transport ceiling. */ +#define M3_DCPEXT_MAX_WIDTH 3840 +#define M3_DCPEXT_MAX_HEIGHT 2160 +#define M3_DCPEXT_MAX_PITCH (M3_DCPEXT_MAX_WIDTH * 4) +#define M3_DCPEXT_MAX_PAYLOAD_KBPS 8640000 + +/* Mode data is supplied by DRM's EDID parser, never invented here. */ +struct m3_dcpext_mode { + u32 width, height, clock_khz, htotal, vtotal; + bool interlaced, doublescan; +}; + +#define M3_DCPEXT_NATIVE_MAX_MODES 128 +struct m3_dcpext_native_mode { + struct m3_dcpext_mode geometry; + u32 hfront, hsync, vfront, vsync; + u32 color_id, timing_id; + bool hpositive, vpositive, preferred; +}; + +/* Decode the current native TimingElements publication. Only real, uncompressed + * 8bpc full-range RGB/SDR modes within the transport policy are admitted. + * No allocation; count is zero on error. IDs belong to this publication only. */ +int m3_dcpext_native_modes_parse(const void *blob, u32 bytes, + struct m3_dcpext_native_mode *modes, + u32 capacity, u32 *count); +/* Match full EDID timing geometry to a uniquely closest native timing. */ +int m3_dcpext_native_mode_select(const struct m3_dcpext_native_mode *requested, + const struct m3_dcpext_native_mode *modes, + u32 count, u32 *selected); + +/* The current hardware policy is four HBR lanes, 8bpc RGB. Mode records are + * the complete firmware-provided 24-byte records, including reserved fields. + * The selected record is returned unchanged. Unknown flags are not stripped. + */ +int m3_dcpext_mode_validate(const struct m3_dcpext_mode *mode, u32 *fps_16_16); +int m3_dcpext_mode_select(const struct m3_dcpext_mode *mode, + const u8 *timings, u32 count, u8 selected[24]); +/* One linear BGRA plane, caller-owned mapping in both external DARTs. */ +int m3_dcpext_surface_validate(u32 width, u32 height, u32 pitch, + u64 dva, u64 mapped_bytes); +int m3_dcpext_native_dimensions(const void *,u32 bytes,u32 *width_mm,u32 *height_mm); +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.c new file mode 100644 index 0000000000000..d82233c76b461 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.c @@ -0,0 +1,780 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Hash-qualified J514S DCP protocol; a single kernel client owns all RPCs. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "m3_dcpext_rpc.h" +#include "m3_dcpext_native.h" + +#define A(n) M3_DCP_TAG('A', n) +#define D(n) M3_DCP_TAG('D', n) +#define MAX_PROPERTIES 256 +#define MAX_RAW_PROPERTIES 32 + +struct m3_dcp_property { + u32 service; + char key[64]; + u64 value; +}; + +struct m3_dcpext_native { + struct device *dev; + struct m3_dcpext_rpc *bridge; + struct mutex lock; + bool failed; + bool opened; + u32 stride; + u64 mode_generation; + u32 diagnostic_plane; + bool diagnostic_flat_surface; + struct m3_dcp_property properties[MAX_PROPERTIES]; + u32 property_count; + struct { u32 service; char key[64]; void *data; u32 size; } raw[MAX_RAW_PROPERTIES]; + u32 raw_count, raw_bytes; + void *chunk; + u32 chunk_size, chunk_offset; + u64 completion_count; + u8 completion_last[0x6f0]; + u64 analytics_count; + u8 analytics_last[4164]; +}; + + +static int callback(struct m3_dcpext_rpc *b, void *cookie, u32 tag, + const void *input, u32 in_size, void *output, u32 out_size); + +static int call(struct m3_dcpext_native *dcp, u32 tag, const void *in, u32 ins, + void *out, u32 outs, u32 completion) +{ + int ret; + + if (dcp->failed) + return -EIO; + dev_dbg(dcp->dev, "M3 kernel RPC %#x input=%u output=%u\n", tag, ins, outs); + ret = m3_dcpext_rpc_call(dcp->bridge, tag, in, ins, out, outs, + completion, callback, dcp); + if (ret) { + dcp->failed = true; + dev_err(dcp->dev, "M3 kernel RPC %#x failed: %d; recovery requires reboot\n", tag, ret); + } + return ret; +} + +static int simple_call(struct m3_dcpext_native *dcp, u32 tag, bool input, bool reply, bool expected) +{ + __le32 one = cpu_to_le32(1), result = 0; + int ret = call(dcp, tag, input ? &one : NULL, input ? 4 : 0, + reply ? &result : NULL, reply ? 4 : 0, 0); + + if (!ret && reply && le32_to_cpu(result) != expected) + return -EPROTO; + return ret; +} + +static struct m3_dcp_property *property(struct m3_dcpext_native *dcp, + u32 service, const u8 *key, bool create) +{ + struct m3_dcp_property *p; + u32 i; + + if (!memchr(key, 0, 64)) + return ERR_PTR(-EINVAL); + for (i = 0; i < dcp->property_count; i++) { + p = &dcp->properties[i]; + if (p->service == service && !strcmp(p->key, key)) + return p; + } + if (!create) + return NULL; + if (dcp->property_count == MAX_PROPERTIES) + return ERR_PTR(-ENOSPC); + p = &dcp->properties[dcp->property_count++]; + p->service = service; + strscpy(p->key, key, sizeof(p->key)); + return p; +} + +static int raw_property(struct m3_dcpext_native *dcp, u32 service, const u8 *key, + const void *data, u32 size) +{ + void *copy; + u32 i, old_size = 0; + + if (!memchr(key, 0, 64) || size > SZ_1M) + return -EINVAL; + for (i = 0; i < dcp->raw_count; i++) + if (dcp->raw[i].service == service && !strcmp(dcp->raw[i].key, key)) + break; + if (i == MAX_RAW_PROPERTIES) + return -ENOSPC; + if (i < dcp->raw_count) + old_size = dcp->raw[i].size; + if (dcp->raw_bytes - old_size + size > SZ_8M) + return -ENOSPC; + copy = kvmemdup(data, size ?: 1, GFP_KERNEL); + if (!copy) + return -ENOMEM; + if (i == dcp->raw_count) + dcp->raw_count++; + kvfree(dcp->raw[i].data); + dcp->raw[i].service = service; + strscpy(dcp->raw[i].key, key, 64); + dcp->raw[i].data = copy; + dcp->raw[i].size = size; + dcp->raw_bytes += size - old_size; + if(!service && !strcmp(key,"TimingElements"))dcp->mode_generation++; + return 0; +} + +static int callback(struct m3_dcpext_rpc *b, void *cookie, u32 tag, + const void *input, u32 in_size, void *output, u32 out_size) +{ + struct m3_dcpext_native *dcp = cookie; + const u8 *in = input; + u8 *out = output; + struct m3_dcp_property *p; + struct m3_dcpext_buffer buffer = {}; + u32 count, offset, service, key_offset; + int ret; + + dev_dbg(dcp->dev, "M3 kernel callback %#x input=%u output=%u\n", tag, in_size, out_size); +#define SHAPE(i, o) (in_size == (i) && out_size == (o)) + if (tag == D(591) && SHAPE(20, 0)) { + /* Old DCP protocol: packed id/bool/intent/width/height, then padding. + * Firmware-generated intent swaps do not retire an A407 framebuffer. + * The first captured M3 notification has intent 15 at 1920x1080. */ + u32 intent = get_unaligned_le32(in + 5); + if (in[4] > 1 || (intent != 6 && intent != 7 && intent != 15)) + return -EOPNOTSUPP; + dev_info(dcp->dev, "M3 intent swap id=%u flag=%u intent=%u size=%ux%u; no submitted buffer retired\n", + get_unaligned_le32(in), in[4], intent, + get_unaligned_le32(in + 9), get_unaligned_le32(in + 13)); + return 0; + } + /* D002 is the external will-power-off notification; no DMA ownership release. */ + if (tag == D(2) && SHAPE(0, 0)) + return 0; + if (tag == D(574) && SHAPE(4, 4)) { + /* M3 TEXT13e810 stores one bool byte; the other three are padding. + * m1n1's matching old protocol names this powerUpDART. */ + if (in[0] > 1) + return -EINVAL; + return m3_dcpext_rpc_dart_power(b, in[0]); + } + if (tag == D(209) && SHAPE(0, 8)) { + put_unaligned_le64(ktime_to_ms(ktime_get_real()), out); + return 0; + } + if (tag == D(596) && SHAPE(0, 4)) + return 0; /* No default framebuffer was allocated by this host. */ + /* J514S 1e5e14/1e5e9c: event, parameter, u32 value, nullable byte. + * The getter copies output[0] back and returns output[4] as a boolean. + * Match the M2 host: no display tiling, and no setter implementation. + */ + if ((tag == D(115) && SHAPE(16, 8)) || + (tag == D(116) && SHAPE(16, 4))) { + if (in[12] > 1) + return -EINVAL; + dev_info(dcp->dev, "M3 tiling callback %#x event=%u parameter=%u value=%u null=%u\n", + tag, get_unaligned_le32(in), get_unaligned_le32(in + 4), + get_unaligned_le32(in + 8), in[12]); + if (tag == D(115)) + out[4] = 1; + return 0; + } + /* The transport zeroes output, so absent optional metadata stays absent. */ + if (tag == D(114) && SHAPE(4164, 4100)) { + /* J514S CoreAnalyticsSendEvent: name[64], serialized dictionary[4096], + * nullable flag. Retain the latest event locally. The dictionary is + * in/out: firmware parses it even on failure, so a zero-filled reply + * is invalid. Return its own serialized dictionary without changes. + * The final u32 is the method status (zero = accepted). + */ + if (!memchr(in, 0, 64) || in[4160] > 1 || + (!in[4160] && in[64] != 'd')) + return -EINVAL; + memcpy(dcp->analytics_last, in, sizeof(dcp->analytics_last)); + dcp->analytics_count++; + if (!in[4160]) + memcpy(out, in + 64, 4096); + put_unaligned_le32(0, out + 4096); + dev_info(dcp->dev, "M3 analytics event %llu: %.64s (retained locally)\n", + dcp->analytics_count, in); + return 0; + } + if (tag == D(589) && SHAPE(0x6f0, 0)) { + memcpy(dcp->completion_last, in, sizeof(dcp->completion_last)); + dcp->completion_count++; + return 0; + } + if ((tag == D(588) && SHAPE(8, 0)) || + (tag == D(598) && SHAPE(0, 0))) + return 0; + if (tag == D(599) && SHAPE(0, 0)) + return simple_call(dcp, A(410), false, true, false); + if ((tag == D(108) || tag == D(109) || tag == D(110) || tag == D(111) || + tag == D(112) || tag == D(113) || tag == D(0) || tag == D(1)) && SHAPE(0, 4)) { + out[0] = 1; + return 0; + } + if (tag == D(576) && SHAPE(88, 76)) { + if (!(in[84] & 1)) + memcpy(out, in + 8, 76); + return 0; + } + if ((tag == D(577) && SHAPE(4, 0)) || (tag == D(300) && SHAPE(16, 0))) + return 0; + if (tag == D(127) && SHAPE(4, 4)) { + count = get_unaligned_le32(in); + if (dcp->chunk || !count || count > 0x100001) + return -EINVAL; + dcp->chunk_size = count - 1; + dcp->chunk_offset = 0; + dcp->chunk = kvzalloc(max_t(u32, 1, dcp->chunk_size), GFP_KERNEL); + if (!dcp->chunk) + return -ENOMEM; + out[0] = 1; + return 0; + } + if (tag == D(128) && SHAPE(0x1008, 4)) { + offset = get_unaligned_le32(in + 0x1000); + count = get_unaligned_le32(in + 0x1004); + if (!dcp->chunk || offset != dcp->chunk_offset || count > 4096 || + offset > dcp->chunk_size || count > dcp->chunk_size - offset) + return -EINVAL; + memcpy(dcp->chunk + offset, in, count); + dcp->chunk_offset += count; + out[0] = 1; + return 0; + } + if (tag == D(129) && SHAPE(64, 4)) { + if (!dcp->chunk || dcp->chunk_offset != dcp->chunk_size || !memchr(in, 0, 64) || + dcp->raw_count == MAX_RAW_PROPERTIES || dcp->raw_bytes + dcp->chunk_size > SZ_8M) + return -EINVAL; + ret = raw_property(dcp, 0, in, dcp->chunk, dcp->chunk_size); + if (ret) + return ret; + kvfree(dcp->chunk); + dcp->chunk = NULL; + dev_info(dcp->dev, "M3 property %.64s: %u bytes\n", in, dcp->chunk_size); + out[0] = 1; + return 0; + } + if (tag == D(408) && SHAPE(8, 8)) { + count = get_unaligned_le32(in + 4); + if (memcmp(in, "VORP", 4) || count > 1) + return -EINVAL; + /* J514S boot ADT: dispext0 IDs353/414 (indices97/158), + * dispext1 IDs356/416 (indices100/160), both 935MHz/0. */ + put_unaligned_le64(count ? 0 : 935000000, out); + dev_info(dcp->dev, "M3 display clock %u: %llu Hz (external boot ADT)\n", + count, get_unaligned_le64(out)); + return 0; + } + if (tag == D(125) && SHAPE(100, 36)) { + count = get_unaligned_le32(in + 64); + if (count > 8) + return -EINVAL; + memcpy(out, in + 68, count * 4); + return 0; + } + if (tag == D(6) && SHAPE(60, 56)) { + if (!(in[56] & 1)) + memcpy(out, in, 56); + return 0; + } + if ((tag == D(122) || tag == D(123)) && SHAPE(0, 4)) + return 0; + if (tag == D(121) && SHAPE(0, 4)) { + static const struct { u32 tag; bool in, reply, expected; } sequence[] = { + {A(444), false, true, false}, {A(29)}, + {A(466), true}, {A(0), true, true, true}, {A(463), false, true, true}, + }; + for (count = 0; count < ARRAY_SIZE(sequence); count++) { + ret = simple_call(dcp, sequence[count].tag, sequence[count].in, + sequence[count].reply, sequence[count].expected); + if (ret) + return ret; + } + out[0] = 1; + return 0; + } + if (tag == D(101) && SHAPE(0, 4)) { + put_unaligned_le32(dcp->stride, out); + return 0; + } + if (tag == D(3) && SHAPE(4, 60)) { + put_unaligned_le32(1, out + 56); /* Explicitly decline bandwidth service. */ + return 0; + } + if (tag == D(451) && SHAPE(20, 28)) { + u32 alignment = get_unaligned_le32(in + 12); + + dev_info(dcp->dev, "M3 allocation flags=%#x size=%llu alignment=%u null=%u/%u/%u\n", + get_unaligned_le32(in), get_unaligned_le64(in + 4), alignment, + in[16], in[17], in[18]); + if (get_unaligned_le32(in) != 0x703 || !is_power_of_2(alignment) || alignment > SZ_16K || + in[16] || in[17] || in[18]) + return -EINVAL; + buffer.size = get_unaligned_le64(in + 4); + ret = m3_dcpext_rpc_alloc(b, &buffer); + if (ret) + return ret; + if ((buffer.physical | buffer.dva) & (alignment - 1)) + return -EINVAL; + put_unaligned_le64(buffer.physical, out); + put_unaligned_le64(buffer.dva, out + 8); + put_unaligned_le64(buffer.size, out + 16); + put_unaligned_le32(buffer.id, out + 24); + return 0; + } + if (tag == D(201) && SHAPE(12, 20)) { + u64 id = get_unaligned_le64(in); + + if (id > U32_MAX || get_unaligned_le32(in + 8) != 1) + return -EINVAL; + buffer.id = id; + ret = m3_dcpext_rpc_map(b, &buffer); + if (ret) + return ret; + /* Same vaddr/dva/status layout as M2, verified at TEXT 0x18b330. + * The AP virtual-address output is unused by this host. + */ + put_unaligned_le64(0, out); + put_unaligned_le64(buffer.dva, out + 8); + put_unaligned_le32(0, out + 16); + return 0; + } + if (tag == D(454) && SHAPE(4, 4)) { + ret = m3_dcpext_rpc_retire(b, get_unaligned_le32(in)); + if (!ret) + out[0] = 1; + return ret; + } + if (tag == D(582) && SHAPE(8, 4)) + return 0; /* Optional default surface is not allocated by this host. */ + if (tag == D(400) && SHAPE(76, 0xc04)) { + count = get_unaligned_le32(in + 68); + service = get_unaligned_le32(in); + if (count > 0xc00 || (in[72] & 1) || !memchr(in + 4, 0, 64)) + return -EINVAL; + for (offset = 0; offset < dcp->raw_count; offset++) { + if (dcp->raw[offset].service != service || strcmp(dcp->raw[offset].key, in + 4)) + continue; + if (dcp->raw[offset].size > count) + return -ENOSPC; + memcpy(out, dcp->raw[offset].data, dcp->raw[offset].size); + put_unaligned_le32(dcp->raw[offset].size, out + 0xc00); + break; + } + return 0; + } + if (tag == D(401) && SHAPE(80, 12)) { + /* Match the existing M2 mini-LED host's PMUS compatibility + * response. This is a fixed calibration placeholder, NOT a + * measured M3 temperature and never a thermal-guard input. + * Firmware 19a4c4..19a4e8 converts centidegrees to 16.16. + */ + if (!memcmp(in, "SUMP", 4) && + !strncmp(in + 4, "Temperature", 64)) { + put_unaligned_le64(3029, out); + out[8] = 1; + dev_info_ratelimited(dcp->dev, + "M3 PMUS Temperature: M2 compatibility placeholder 3029\n"); + return 0; + } + p = property(dcp, get_unaligned_le32(in), in + 4, false); + if (IS_ERR(p)) + return PTR_ERR(p); + if (p) { + put_unaligned_le64(p->value, out); + out[8] = 1; + } else + dev_info_ratelimited(dcp->dev, + "M3 missing host property service=%#x key=%.64s\n", + get_unaligned_le32(in), in + 4); + return 0; + } + if ((tag == D(413) && SHAPE(4168, 4)) || + ((tag == D(552) || tag == D(561)) && SHAPE(4164, 4)) || + (tag == D(567) && SHAPE(128, 4))) { + key_offset = tag == D(413) ? 4 : 0; + service = key_offset ? get_unaligned_le32(in) : 0; + if (tag != D(567) && (in[key_offset + 4160] & 1)) { + out[0] = 1; + return 0; + } + count = tag == D(567) ? strnlen(in + 64, 64) : 4096; + ret = raw_property(dcp, service, in + key_offset, in + key_offset + 64, count); + if (!ret) + out[0] = 1; + return ret; + } + if ((tag == D(563) && SHAPE(76, 4)) || (tag == D(565) && SHAPE(72, 4)) || + (tag == D(414) && SHAPE(80, 4)) || (tag == D(415) && SHAPE(76, 4)) || + ((tag == D(102) || tag == D(104)) && SHAPE(68, 0))) { + key_offset = (tag == D(414) || tag == D(415)) ? 4 : 0; + service = key_offset ? get_unaligned_le32(in) : 0; + count = (tag == D(563) || tag == D(414)) ? 8 : 4; + if (out_size && (in[key_offset + 64 + count] & 1)) { + out[0] = 1; + return 0; + } + p = property(dcp, service, in + key_offset, true); + if (IS_ERR(p)) + return PTR_ERR(p); + p->value = count == 8 ? get_unaligned_le64(in + key_offset + 64) : + get_unaligned_le32(in + key_offset + 64); + if (tag == D(104)) + p->value = in[64]; + if (out_size) + out[0] = 1; + return 0; + } + if (tag == D(406) && SHAPE(72, 0)) + return 0; + if (tag == D(107) && SHAPE(64, 0)) { + if(!memchr(in,0,64))return -EINVAL; + if(!strcmp(in,"TimingElements"))dcp->mode_generation++; + for(u32 i=0;iraw_count;i++) { + if(dcp->raw[i].service || strcmp(dcp->raw[i].key,in))continue; + dcp->raw_bytes-=dcp->raw[i].size;kvfree(dcp->raw[i].data); + dcp->raw[i]=dcp->raw[--dcp->raw_count]; + memset(&dcp->raw[dcp->raw_count],0,sizeof(dcp->raw[0]));break; + } + p = property(dcp, 0, in, false); + if (IS_ERR(p)) + return PTR_ERR(p); + if (p) { + *p = dcp->properties[--dcp->property_count]; + memset(&dcp->properties[dcp->property_count], 0, sizeof(*p)); + } + return 0; + } + if (tag == D(207) && SHAPE(0, 4)) { + /* HDMI has no panel backlight. Match iomfb_template.c's external + * display path: acknowledge the request without A132, which + * registers a panel backlight service with firmware. + */ + out[0] = 1; + return 0; + } + if (tag == D(206) && SHAPE(0, 4)) { + ret = simple_call(dcp, A(131), false, true, true); + if (!ret) + out[0] = 1; + return ret; + } + if (tag == D(100) && SHAPE(0, 0)) + { + __le32 result; + return call(dcp, A(374), NULL, 0, &result, 4, 0); + } + dev_err(dcp->dev, "M3 unhandled kernel callback %#x %u/%u\n", tag, in_size, out_size); + return -EOPNOTSUPP; +#undef SHAPE +} + +int m3_dcpext_native_open(struct m3_dcpext_native *dcp) +{ + int ret; + + mutex_lock(&dcp->lock); + if (dcp->opened) { + ret = -EALREADY; + } else { + ret = simple_call(dcp, A(455), false, false, false); + if (!ret) { + dcp->opened = true; + dev_info(dcp->dev, "M3 first-client-open complete\n"); + } + } + mutex_unlock(&dcp->lock); + return ret; +} + +struct m3_dcpext_native *m3_dcpext_native_start(struct device *dev, struct m3_dcpext_rpc *bridge, + bool defer_open) +{ + struct m3_dcpext_native *dcp; + int ret; + + dcp = devm_kzalloc(dev, sizeof(*dcp), GFP_KERNEL); + if (!dcp) + return ERR_PTR(-ENOMEM); + dcp->dev = dev; + dcp->bridge = bridge; + mutex_init(&dcp->lock); + dcp->stride = 7680; + ret = simple_call(dcp, A(401), false, true, true); + if (!ret && !defer_open) + ret = m3_dcpext_native_open(dcp); + if (ret) + return ERR_PTR(ret); + if (defer_open) + dev_info(dev, "M3 DCP kernel startup complete; first-client-open deferred\n"); + else + dev_info(dev, "M3 DCP kernel startup and first-client-open complete\n"); + return dcp; +} + +/* J514S wrappers: A409@13f7d4 (4/4), A472@1421f4 (12/8), + * A411@13f8a8 (8/4). A472's output status is at byte 4; bytes 9/10 + * are additional boolean inputs and byte 11 is the output-pointer null flag. + * Follow M2's display-handle 0, then power-state 1 sequence explicitly. + */ +int m3_dcpext_native_panel(struct m3_dcpext_native *dcp, unsigned int action) +{ + u8 input[80] = {}, output[8] = {}; + int ret; + + mutex_lock(&dcp->lock); + switch (action) { + case 1: + ret = call(dcp, A(409), input, 4, output, 4, 0); + /* Existing internal-display handle 0 returns 2. */ + if (!ret && get_unaligned_le32(output) != 2) + ret = -EIO; + if (!ret) { + put_unaligned_le64(1, input); + ret = call(dcp, A(472), input, 12, output, 8, 0); + if (!ret && get_unaligned_le32(output + 4)) + ret = -EIO; + } + break; + case 2: + /* PreferredTimingElements: color 1, timing 2, 3024x1964. */ + put_unaligned_le32(1, input); + put_unaligned_le32(2, input + 4); + ret = call(dcp, A(411), input, 8, output, 4, 0); + if (!ret && get_unaligned_le32(output)) + ret = -EIO; + break; + case 3: + ret = call(dcp, A(479), input, 4, output, 8, 0); + break; + case 4: + /* Native stub kernel ab5a53c; firmware wrapper 140fe4. + * Same initialization used by the M2 host before client open. + */ + ret = call(dcp, A(448), input, 4, output, 4, 0); + if (!ret && get_unaligned_le32(output)) + ret = -EIO; + break; + case 5: + /* Native stub kernel ab57948; firmware wrapper 13fe2c. + * Location 9 is the identity CTM used by the M2 KMS path. + */ + put_unaligned_le32(9, input); + put_unaligned_le64(1ULL << 32, input + 4); + put_unaligned_le64(1ULL << 32, input + 4 + 4 * 8); + put_unaligned_le64(1ULL << 32, input + 4 + 8 * 8); + ret = call(dcp, A(421), input, 80, output, 4, 0); + if (!ret && get_unaligned_le32(output)) + ret = -EIO; + break; + case 6: + case 7: + /* Reversible diagnostic: M2's linear primary uses IOMFB plane 2. + * The next swap clears the other two primary slots atomically. + */ + dcp->diagnostic_plane = action == 6 ? 2 : 0; + put_unaligned_le32(dcp->diagnostic_plane, output); + ret = 0; + break; + case 8: + case 9: + /* Captured 26A428 linear BGRA uses the non-planar IOSurface + * representation. SurfaceContents fields match the stub driver. + */ + dcp->diagnostic_flat_surface = action == 8; + ret = 0; + break; + default: + ret = -EINVAL; + } + dev_info(dcp->dev, "M3 panel action %u ret=%d output=%#x/%#x\n", + action, ret, get_unaligned_le32(output), get_unaligned_le32(output + 4)); + if (ret) + dcp->failed = true; + mutex_unlock(&dcp->lock); + return ret; +} + +/* External mode IDs come from this session's TimingElements publication. */ +int m3_dcpext_native_mode(struct m3_dcpext_native *dcp, u32 color, u32 timing) +{ + u8 input[8], output[4]; int ret; + put_unaligned_le32(color, input); put_unaligned_le32(timing, input + 4); + mutex_lock(&dcp->lock); + ret = call(dcp, A(411), input, sizeof(input), output, sizeof(output), 0); + if (!ret && get_unaligned_le32(output)) ret = -EIO; + if (ret) dcp->failed = true; + mutex_unlock(&dcp->lock); + return ret; +} +int m3_dcpext_native_power(struct m3_dcpext_native *dcp, bool on) +{ + u8 input[12] = {}, output[8] = {}; int ret; + put_unaligned_le64(on, input); + mutex_lock(&dcp->lock); + ret = call(dcp, A(472), input, sizeof(input), output, sizeof(output), 0); + if (!ret && get_unaligned_le32(output + 4)) ret = -EIO; + if (ret) dcp->failed = true; + mutex_unlock(&dcp->lock); + return ret; +} + +static int native_swap(struct m3_dcpext_native *dcp, const void *surface, u64 dva, + u32 width, u32 height, u32 background) +{ + __le32 start[4] = {}, started[2], result[3]; + u8 *swap; + u32 id; + int ret; + + if (surface && (!dva || !width || width > 3024 || !height || height > 1964)) + return -EINVAL; + swap = kzalloc(0x1b58, GFP_KERNEL); + if (!swap) + return -ENOMEM; + mutex_lock(&dcp->lock); + ret = call(dcp, A(406), start, sizeof(start), started, sizeof(started), 0); + if (ret || le32_to_cpu(started[1])) { + ret = ret ?: -EIO; + goto out; + } + id = le32_to_cpu(started[0]); + if (!id) { + ret = -EPROTO; + goto out; + } + put_unaligned_le32(id, swap + 0x98); + put_unaligned_le32(0x80000007, swap + 0x14c); + put_unaligned_le32(0x80000007, swap + 0x150); + /* A disabled plane must blank to black, including compositor DPMS. */ + put_unaligned_le32(background, swap + 0x154); + memset(swap + 0x1b4b, 1, 10); + swap[0x1b56] = swap[0x1b57] = 1; + if (surface) { + u32 plane = dcp->diagnostic_plane; + u8 *s = swap + 0x508 + plane * 0x22c; + + memcpy(s, surface, 0x22c); + if (dcp->diagnostic_flat_surface) { + put_unaligned_le32(0, s + 3); + put_unaligned_le32(0, s + 7); + s[0x14] = 1; /* Captured sRGB BGRA surface. */ + put_unaligned_le16(4, s + 0x19); + memset(s + 0x39, 0, 0x1ed - 0x39); + put_unaligned_le64(1, s + 0x51); + put_unaligned_le64(1, s + 0x149); + } + put_unaligned_le64(dva, swap + 0xdb8 + plane * 8); + put_unaligned_le32(1, swap + 0x9c + plane * 4); + put_unaligned_le32(width, swap + 0xb4 + plane * 16); + put_unaligned_le32(height, swap + 0xb8 + plane * 16); + put_unaligned_le32(width, swap + 0x114 + plane * 16); + put_unaligned_le32(height, swap + 0x118 + plane * 16); + swap[0x1b4b + plane] = 0; + } + ret = call(dcp, A(407), swap, 0x1b58, result, sizeof(result), id); + if (!ret && get_unaligned_le32((u8 *)result + 5)) + ret = -EIO; + if (!ret && !surface) + dev_info(dcp->dev, "M3 kernel swap %u completed, surface=%d\n", id, !!surface); +out: + if (ret) + dcp->failed = true; + mutex_unlock(&dcp->lock); + kfree(swap); + return ret; +} + +int m3_dcpext_native_swap(struct m3_dcpext_native *dcp, const void *surface, u64 dva, + u32 width, u32 height) +{ + return native_swap(dcp, surface, dva, width, height, 0xff000000); +} + +int m3_dcpext_native_background(struct m3_dcpext_native *dcp, u32 color) +{ + return native_swap(dcp, NULL, 0, 0, 0, color); +} + +void *m3_dcpext_native_property(struct m3_dcpext_native *dcp, const char *key, u32 *size) +{ + void *copy = NULL; + u32 i; + + mutex_lock(&dcp->lock); + for (i = 0; i < dcp->raw_count; i++) { + if (!dcp->raw[i].service && !strcmp(dcp->raw[i].key, key)) { + *size = dcp->raw[i].size; + copy = kmemdup(dcp->raw[i].data, *size, GFP_KERNEL); + break; + } + } + mutex_unlock(&dcp->lock); + return copy; +} + +void m3_dcpext_native_analytics_show(struct m3_dcpext_native *dcp, struct seq_file *seq) +{ + mutex_lock(&dcp->lock); + seq_printf(seq, "events %llu\n", dcp->analytics_count); + if (dcp->analytics_count) { + seq_printf(seq, "last_event %.64s\ndictionary_present %u\n", + dcp->analytics_last, !dcp->analytics_last[4160]); + if (!dcp->analytics_last[4160]) + seq_hex_dump(seq, "", DUMP_PREFIX_OFFSET, 16, 1, + dcp->analytics_last + 64, 4096, false); + } + mutex_unlock(&dcp->lock); +} + +void m3_dcpext_native_completion_show(struct m3_dcpext_native *dcp, struct seq_file *seq) +{ + mutex_lock(&dcp->lock); + seq_printf(seq, "callbacks %llu\n", dcp->completion_count); + if (dcp->completion_count) + seq_hex_dump(seq, "", DUMP_PREFIX_OFFSET, 16, 1, + dcp->completion_last, sizeof(dcp->completion_last), false); + mutex_unlock(&dcp->lock); +} + +int m3_dcpext_native_pump(struct m3_dcpext_native *dcp, unsigned long timeout) +{ + int ret; + mutex_lock(&dcp->lock); + ret=m3_dcpext_rpc_pump(dcp->bridge, timeout, callback, dcp); + mutex_unlock(&dcp->lock); + return ret; +} + +void m3_dcpext_native_invalidate_sink(struct m3_dcpext_native *dcp) +{ + mutex_lock(&dcp->lock); + dcp->mode_generation++; + for(u32 i=0;iraw_count;) { + if(dcp->raw[i].service || + (strcmp(dcp->raw[i].key,"TimingElements") && + strcmp(dcp->raw[i].key,"DisplayAttributes"))){i++;continue;} + dcp->raw_bytes-=dcp->raw[i].size;kvfree(dcp->raw[i].data); + dcp->raw[i]=dcp->raw[--dcp->raw_count]; + memset(&dcp->raw[dcp->raw_count],0,sizeof(dcp->raw[0])); + } + mutex_unlock(&dcp->lock); +} + +u64 m3_dcpext_native_generation(struct m3_dcpext_native *dcp) +{ + u64 generation;mutex_lock(&dcp->lock);generation=dcp->mode_generation;mutex_unlock(&dcp->lock);return generation; +} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.h new file mode 100644 index 0000000000000..6b26dd32371c0 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef M3_DCPEXT_NATIVE_H +#define M3_DCPEXT_NATIVE_H + +#define M3_DCP_TAG(c, n) (((u32)(c) << 24) | ((u32)('0' + (n) / 100) << 16) | \ + ((u32)('0' + (n) / 10 % 10) << 8) | (u32)('0' + (n) % 10)) +struct m3_dcpext_native; +struct m3_dcpext_rpc; +struct device; +struct seq_file; +void m3_dcpext_native_completion_show(struct m3_dcpext_native *dcp, struct seq_file *seq); +void m3_dcpext_native_analytics_show(struct m3_dcpext_native *dcp, struct seq_file *seq); +struct m3_dcpext_native *m3_dcpext_native_start(struct device *dev, struct m3_dcpext_rpc *bridge, + bool defer_open); +int m3_dcpext_native_open(struct m3_dcpext_native *dcp); +int m3_dcpext_native_swap(struct m3_dcpext_native *dcp, const void *surface, u64 dva, + u32 width, u32 height); +int m3_dcpext_native_background(struct m3_dcpext_native *dcp, u32 color); +int m3_dcpext_native_panel(struct m3_dcpext_native *dcp, unsigned int action); +/* Caller owns the returned snapshot and must kfree it. */ +void *m3_dcpext_native_property(struct m3_dcpext_native *dcp, const char *key, u32 *size); +int m3_dcpext_native_pump(struct m3_dcpext_native *, unsigned long timeout); +int m3_dcpext_native_mode(struct m3_dcpext_native *, u32 color, u32 timing); +int m3_dcpext_native_power(struct m3_dcpext_native *, bool on); +u64 m3_dcpext_native_generation(struct m3_dcpext_native *); +void m3_dcpext_native_invalidate_sink(struct m3_dcpext_native *); +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_adapter.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_adapter.h new file mode 100644 index 0000000000000..8ee3cf4826a20 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_adapter.h @@ -0,0 +1,148 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include "m3_dcpext_rpc.h" +#include "m3_dcpext_native.h" +#include "m3_dcpext_modes.h" +static bool native_frame_done, native_frame_submitted; +static u64 native_frame_until; +static struct m3_dcpext_rpc *native_rpc; +static struct m3_dcpext_native *native_client; +static bool native_started; +static unsigned int received; +static int session_dispatch(u64 endpoint,u64 message); +static int native_receive(u64 message) +{ + if(!native_rpc)return native_query_receive(message); + m3_dcpext_rpc_receive(native_rpc,message);return 0; +} +static int native_poll(void *cookie,unsigned long timeout) +{ + u64 deadline=session_now(NULL)+jiffies_to_msecs(timeout); + int ret; + do { + u64 msg,ep; + ret=dcpext_session_poll(&session);if(ret)return ret; + if(!(readl(session_fifo+0x114)&BIT(17))){ + msg=readq(session_fifo+0x830);ep=readq(session_fifo+0x838);dma_rmb(); + return session_dispatch(ep,msg); + } + usleep_range(100,200); + }while(session_now(NULL)size || info->size>0x100000 || native_buffers_used==ARRAY_SIZE(native_buffers))return -E2BIG; + ret=session_buffer(NULL,0x37,0,info->size,&dva);if(ret)return ret; + info->size=ALIGN(info->size,LEASE_PAGE);info->dva=dva;info->physical=pool_dma+offset; + info->id=++native_buffers_used;native_buffers[info->id-1].info=*info; + return 0; +} +static int native_map(void *cookie,struct m3_dcpext_buffer *info) +{ + if(!pio_published || !info->id || info->id>native_buffers_used || native_buffers[info->id-1].retired)return -EINVAL; + *info=native_buffers[info->id-1].info;return 0; +} +static int native_retire(void *cookie,u32 id) +{ + if(!id || id>native_buffers_used || native_buffers[id-1].retired)return -EINVAL; + native_buffers[id-1].retired=true;return 0; +} +static int native_power(void *cookie,bool on) +{ + if(!pio_published)return -EUCLEAN; + pr_info("m3_dcpext_native: firmware DART power=%u; session vote retained\n",on);return 0; +} +static u8 native_capture[0x800000]; +static u32 native_capture_used; +static int native_record(void *cookie,u32 kind,u64 msg,const void *data,u32 size) +{ + u8 *p=native_capture+native_capture_used; + if(session.runtime)return 0; + if(size>sizeof(native_capture)-native_capture_used-16)return -ENOSPC; + put_unaligned_le32(kind,p);put_unaligned_le32(size,p+4);put_unaligned_le64(msg,p+8); + memcpy(p+16,data,size);native_capture_used+=size+16;return 0; +} +static const struct m3_dcpext_rpc_ops native_ops={.send=session_send,.poll=native_poll, + .alloc=native_alloc,.map=native_map,.retire=native_retire,.dart_power=native_power,.record=native_record}; +static void native_surface(u8 *s,bool opaque) +{ + memset(s,0,0x22c);s[2]=opaque; + /* Qualified M3 linear BGRA s, plane 0. */ + put_unaligned_le32(1,s+3);put_unaligned_le32(1,s+7); + put_unaligned_le32(0x42475241,s+0xb);s[0x13]=13;s[0x14]=12; + put_unaligned_le32(7680,s+0x15);put_unaligned_le16(1,s+0x19); + s[0x1b]=s[0x1c]=1; + put_unaligned_le32(1920,s+0x21);put_unaligned_le32(1080,s+0x25); + put_unaligned_le32(1920*1080*4,s+0x29);put_unaligned_le32(1,s+0x35); + put_unaligned_le64(1,s+0x51);put_unaligned_le32(1920,s+0x59); + put_unaligned_le32(1080,s+0x5d);put_unaligned_le32(7680,s+0x69); + put_unaligned_le32(1920*1080*4,s+0x6d);put_unaligned_le16(4,s+0x71); + s[0x73]=s[0x74]=1;put_unaligned_le64(1,s+0x149); +} +static int native_kms_publish(void); +static int native_frame_progress(void) +{ + struct m3_dcpext_native_mode *modes; + const struct m3_dcpext_native_mode wanted = { + .geometry = { .width=1920, .height=1080, .clock_khz=148500, .htotal=2200, .vtotal=1125 }, + .hfront=88, .hsync=44, .vfront=4, .vsync=5, .hpositive=true, .vpositive=true, + }; + u32 bytes=0, count=0, selected; void *blob; int ret; + u8 surface[0x22c] = {}; + if (!native_frame || native_frame_done) return 0; + if(desktop_disconnected || desktop_hpd_dirty || desktop_hpd_rpc || desktop_link_stage)return 0; + if (native_frame_submitted) { + if(native_kms_trial)return native_kms_publish(); + if (session_now(NULL) +#include +#include +#include +#include + +#ifndef DCPEXT_INSTANCE +#define DCPEXT_INSTANCE 0 +#endif + +static struct platform_device *owner; +static struct device *domain; +static int result = -EINPROGRESS; +module_param(result, int, 0400); + +static int __init acquire(void) +{ + struct device_node *node; + u32 version; + int ret; + + if (!of_machine_is_compatible("apple,j514s") || + !of_machine_is_compatible("apple,t6030")) + return -ENODEV; + node = of_find_node_by_path(DCPEXT_INSTANCE ? "dcpext1" : "dcpext0"); + if (!node) + return -ENODEV; + if (of_device_is_available(node) || + of_property_read_u32(node, "apple,j514s-dcpext-reservations", &version) || + version != 1) { + of_node_put(node); + return -EINVAL; + } + owner = platform_device_alloc(DCPEXT_INSTANCE ? "m3-dcpext1-power" : "m3-dcpext-power", PLATFORM_DEVID_NONE); + if (!owner) { + of_node_put(node); + return -ENOMEM; + } + device_set_node(&owner->dev, of_fwnode_handle(node)); + ret = platform_device_add(owner); + if (ret) { + platform_device_put(owner); + of_node_put(node); + return ret; + } + domain = dev_pm_domain_attach_by_id(&owner->dev, 0); + if (IS_ERR_OR_NULL(domain)) { + ret = domain ? PTR_ERR(domain) : -ENODEV; + platform_device_unregister(owner); + of_node_put(node); + return ret; + } + pr_info("M3 DCPEXT%u acquiring CPU/FE/SYS domains via genpd\n", DCPEXT_INSTANCE); + result = pm_runtime_resume_and_get(domain); + pr_info("M3 DCPEXT%u power acquisition result=%d; retained until reboot\n", DCPEXT_INSTANCE, result); + return 0; +} + +module_init(acquire); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("J514S guarded external display power-domain acquisition"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_regs.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_regs.c new file mode 100644 index 0000000000000..9b1bf39519da0 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_regs.c @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* J514S reservation/power audit. Explicit load under the M3 watchdog guard. + * No power, DART, mailbox or PHY writes. Read DART only if the relevant + * real PMGR domains are on. no_ps ADT entries have no register to inspect. + */ +#include +#include +#include +#include +#include +#include +#include + +static unsigned int port; +module_param(port, uint, 0400); +static bool read_dart; +module_param(read_dart, bool, 0400); +MODULE_PARM_DESC(read_dart, "Inspect powered DART after validating reservations"); +static bool read_cpu; +module_param(read_cpu, bool, 0400); + +static int __init audit(void) +{ + const u32 power[2][3] = {{0x3b8, 0x3d8, 0x3e0}, {0x3c0, 0x3f0, 0x3f8}}; + const phys_addr_t coproc[] = {0x2d2c00000ULL, 0x2d6c00000ULL}; + const phys_addr_t dart[] = {0x2d130c000ULL, 0x2d530c000ULL}; + const phys_addr_t scanout[] = {0x2d1304000ULL, 0x2d5304000ULL}; + struct device_node *node; + struct regmap *pmgr; + struct resource resource; + char alias[16]; + u32 marker, value; + bool powered = true; + int ret; + + if (port > 1 || !of_machine_is_compatible("apple,j514s") || + !of_machine_is_compatible("apple,t6030")) + return -ENODEV; + snprintf(alias, sizeof(alias), "dcpext%u", port); + node = of_find_node_by_path(alias); + if (!node) + return -ENODEV; + ret = of_property_read_u32(node, "apple,j514s-dcpext-reservations", &marker); + if (ret || marker != 1 || of_device_is_available(node) || + !of_device_is_compatible(node, "apple,t6030-dcpext") || + of_address_to_resource(node, 0, &resource) || resource.start != coproc[port] || + of_count_phandle_with_args(node, "memory-region", NULL) != 7) { + of_node_put(node); + return -EINVAL; + } + for (unsigned int i = 0; i < 7; i++) { + struct device_node *memory = of_parse_phandle(node, "memory-region", i); + + if (!memory || !of_property_read_bool(memory, "no-map") || + of_address_to_resource(memory, 0, &resource)) { + of_node_put(memory); + of_node_put(node); + return -EINVAL; + } + pr_info("M3 DCPEXT%u protected region %u: %pr\n", port, i, &resource); + of_node_put(memory); + } + of_node_put(node); + node = of_find_node_by_path("/soc/power-management@350700000"); + if (!node) + return -ENODEV; + pmgr = syscon_node_to_regmap(node); + of_node_put(node); + if (IS_ERR(pmgr)) + return PTR_ERR(pmgr); + for (unsigned int i = 0; i < 3; i++) { + ret = regmap_read(pmgr, power[port][i], &value); + if (ret) + return ret; + pr_info("M3 DCPEXT%u PMGR +%#x = %#x\n", port, power[port][i], value); + /* CPU auto-gating is normal on T6030; require the parent + * SYS/FE fabric to remain active before optional ASC reads. + */ + if ((value & 0xff) != 0xff || + value & (BIT(31) | BIT(12) | BIT(11) | BIT(10)) || + (i < 2 && (value & BIT(28)))) + powered = false; + } + if ((!read_dart && !read_cpu) || !powered) { + pr_info("M3 DCPEXT%u no DART reads: requested=%u powered=%u\n", + port, read_dart, powered); + return 0; + } + if (read_cpu) { + void __iomem *regs = ioremap(coproc[port], 0x4000); + + if (!regs) + return -ENOMEM; + pr_info("M3 DCPEXT%u reading ASC control/status\n", port); + msleep(20); + pr_info("M3 DCPEXT%u ASC control=%#x status=%#x\n", + port, readl(regs + 0x44), readl(regs + 0x48)); + iounmap(regs); + } + for (unsigned int i = 0; read_dart && i < 3; i++) { + unsigned int sid = i == 0 ? 5 : i == 1 ? 0 : 4; + phys_addr_t base = i == 0 ? dart[port] : scanout[port]; + const unsigned int offsets[] = {0x200, 0x1000 + sid * 4, 0x1400 + sid * 4}; + void __iomem *regs = ioremap(base, 0x4000); + + if (!regs) + return -ENOMEM; + for (unsigned int n = 0; n < ARRAY_SIZE(offsets); n++) { + pr_info("M3 DCPEXT%u reading %pap + %#x\n", port, &base, offsets[n]); + msleep(20); + pr_info("M3 DCPEXT%u DART %pap + %#x = %#x\n", + port, &base, offsets[n], readl(regs + offsets[n])); + } + iounmap(regs); + } + return 0; +} + +static void __exit done(void) {} +module_init(audit); +module_exit(done); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("J514S external display reservation and scalar register audit"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rpc.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rpc.c new file mode 100644 index 0000000000000..87eb859dac115 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rpc.c @@ -0,0 +1,604 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* M4 transport snapshot bce0fd0d685485 adapted for M3 D589 completion + * and D201 20-byte output. M3 external startup remains a bounded probe. */ +#include +#include +#include +#include +#include +#include +#include "../../dcp-link.h" +#include "m3_dcpext_rpc.h" +#define MAX_EVENTS 32 +#define MAX_CALLBACKS 8 +struct record_header { + __le32 kind; + __le32 size; + __le64 message; +}; + +struct bridge_event { + struct list_head list; + struct record_header header; + u8 data[]; +}; + +struct m3_dcpext_rpc { + struct device *dev; + const struct m3_dcpext_rpc_ops *ops; + void *cookie; + void *rpc; + struct mutex lock; + struct list_head events; + unsigned int event_count, callback_count; + bool failed; + bool enabled; + u32 kernel_callback_depth; + struct { + bool active; + bool remote; + u32 offset; + u32 size; + struct apple_dcp_link_rpc_header header; + } calls[2]; + struct { + u64 message; + void *output; + u32 size; + u32 call; + u64 allocation_size; + u64 mapping_id; + } callbacks[MAX_CALLBACKS]; +}; + +static void fail(struct m3_dcpext_rpc *b) +{ + b->failed = true; + dev_err(b->dev, + "DCP bridge stopped; retain firmware buffers until reboot\n"); +} + +static void event(struct m3_dcpext_rpc *b, u32 kind, u64 message, + const void *data, u32 size) +{ + struct bridge_event *e; + + if (b->ops->record && + b->ops->record(b->cookie, kind, message, data, size)) { + fail(b); + return; + } + if (b->event_count >= MAX_EVENTS || + size > APPLE_DCP_LINK_STREAM_BUFFER_SIZE) { + fail(b); + return; + } + e = kmalloc(sizeof(*e) + size, GFP_KERNEL); + if (!e) { + fail(b); + return; + } + e->header = (struct record_header){ + .kind = cpu_to_le32(kind), + .size = cpu_to_le32(size), + .message = cpu_to_le64(message), + }; + if (size) + memcpy(e->data, data, size); + list_add_tail(&e->list, &b->events); + b->event_count++; +} + +void m3_dcpext_rpc_crashed(struct m3_dcpext_rpc *b) +{ + mutex_lock(&b->lock); + fail(b); + mutex_unlock(&b->lock); +} + +void m3_dcpext_rpc_receive(struct m3_dcpext_rpc *b, u64 message) +{ + struct apple_dcp_link_stream_layout layout; + struct apple_dcp_link_rpc_view view; + u8 stream = apple_dcp_link_message_stream(message); + bool remote = apple_dcp_link_message_is_remote(message); + bool nested; + int ret; + + mutex_lock(&b->lock); + if (b->failed) + goto out; + if ((message & 3) != APPLE_DCP_LINK_MSG_RPC) { + event(b, 3, message, NULL, 0); + goto out; + } + dma_rmb(); + if (message & BIT_ULL(6)) { + nested = b->calls[1].active; + if (stream || !b->calls[nested].active || + b->calls[nested].remote != remote || + apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, 0, + remote, &layout)) + goto error; + if (memcmp(b->rpc + layout.local_offset + + b->calls[nested].offset, + &b->calls[nested].header, + sizeof(b->calls[nested].header))) + goto error; + event(b, 1, message, + b->rpc + layout.local_offset + b->calls[nested].offset, + b->calls[nested].size); + b->calls[nested].active = false; + goto out; + } + if (b->callback_count >= MAX_CALLBACKS || + apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, stream, remote, + &layout)) + goto error; + ret = apple_dcp_link_rpc_decode(message, b->rpc + layout.remote_offset, + layout.capacity, &view); + if (ret) + goto error; + if ((view.call >> 24) != 'D') { + u32 off; + + /* Preserve stream-placement evidence without acknowledging bad data. */ + for (off = 0; off < APPLE_DCP_LINK_HEAP_OFFSET; + off += APPLE_DCP_LINK_STREAM_BUFFER_SIZE) { + const __le32 *h = b->rpc + off; + + dev_err(b->dev, + "M3 RPC header offset=%#x words=%#x/%#x/%#x\n", + off, le32_to_cpu(h[0]), le32_to_cpu(h[1]), + le32_to_cpu(h[2])); + } + goto error; + } + b->callbacks[b->callback_count].message = message; + b->callbacks[b->callback_count].output = view.output; + b->callbacks[b->callback_count].size = view.output_size; + b->callbacks[b->callback_count].call = view.call; + b->callbacks[b->callback_count].allocation_size = 0; + b->callbacks[b->callback_count].mapping_id = 0; + if (view.call == 0x44343531 && view.input_size == 20 && + view.output_size == 28) { + __le64 allocation_size; + + memcpy(&allocation_size, view.input + 4, + sizeof(allocation_size)); + b->callbacks[b->callback_count].allocation_size = + le64_to_cpu(allocation_size); + } + if (view.call == 0x44323031 && view.input_size == 12 && + view.output_size == 20) { + __le64 mapping_id; + + memcpy(&mapping_id, view.input, sizeof(mapping_id)); + b->callbacks[b->callback_count].mapping_id = + le64_to_cpu(mapping_id); + } + b->callback_count++; + dev_dbg(b->dev, "M3 DCP callback tag=%#x input=%u output=%u\n", + view.call, view.input_size, view.output_size); + event(b, 2, message, b->rpc + layout.remote_offset + view.offset, + view.total_size); + goto out; +error: + fail(b); +out: + mutex_unlock(&b->lock); +} + +static ssize_t bridge_request(struct m3_dcpext_rpc *b, + struct record_header *record, size_t count) +{ + struct apple_dcp_link_stream_layout layout; + struct apple_dcp_link_rpc_header *rpc; + u64 message, control; + u32 kind, size, total, offset = 0; + bool nested, remote = false; + int ret; + + kind = le32_to_cpu(record->kind); + size = le32_to_cpu(record->size); + control = le64_to_cpu(record->message); + ret = -EINVAL; + if (size != count - sizeof(*record)) + goto free_record; + mutex_lock(&b->lock); + if (!b->enabled) { + ret = -EAGAIN; + goto out; + } + if (b->failed) { + ret = -EIO; + goto out; + } + if (kind == 2) { + unsigned int top; + + if (!b->callback_count) + goto out; + top = b->callback_count - 1; + if (control != b->callbacks[top].message || + size != b->callbacks[top].size) + goto out; + if (b->ops->record && + b->ops->record(b->cookie, 4, control, record + 1, size)) { + ret = -ENOSPC; + fail(b); + goto out; + } + memcpy(b->callbacks[top].output, record + 1, size); + dma_wmb(); + ret = b->ops->send(b->cookie, APPLE_DCP_LINK_ENDPOINT, + apple_dcp_link_rpc_reply(control)); + if (ret) + fail(b); + else + b->callback_count--; + goto out; + } + if (kind != 1 || control > 1 || size < sizeof(*rpc)) + goto out; + nested = control; + if (b->calls[nested].active || (nested && !b->callback_count) || + (!nested && b->callback_count)) { + ret = -EBUSY; + goto out; + } + if (nested) { + u64 callback = b->callbacks[b->callback_count - 1].message; + + if (apple_dcp_link_message_stream(callback)) { + ret = -EOPNOTSUPP; + goto out; + } + /* Re-entry uses the callback's stream without the new-stream IRQ flag. */ + remote = apple_dcp_link_message_is_remote(callback); + if (b->calls[0].active && b->calls[0].remote == remote) + offset = ALIGN(b->calls[0].size, 64); + } + rpc = (void *)(record + 1); + ret = apple_dcp_link_rpc_payload_size(le32_to_cpu(rpc->input_size), + le32_to_cpu(rpc->output_size), + &total); + if (ret || total != size) { + ret = -EMSGSIZE; + goto out; + } + ret = apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, 0, remote, + &layout); + if (ret) + goto out; + if (offset + size > layout.capacity) { + ret = -EMSGSIZE; + goto out; + } + if (b->ops->record && + b->ops->record(b->cookie, 3, control, rpc, size)) { + ret = -ENOSPC; + fail(b); + goto out; + } + memcpy(b->rpc + layout.local_offset + offset, rpc, size); + memset(b->rpc + layout.local_offset + offset + sizeof(*rpc) + + le32_to_cpu(rpc->input_size), + 0, le32_to_cpu(rpc->output_size)); + b->calls[nested].header = *rpc; + b->calls[nested].size = size; + b->calls[nested].offset = offset; + b->calls[nested].remote = remote; + b->calls[nested].active = true; + /* M3 bit 9 requires a free stream; it is not a recursive-call marker. */ + apple_dcp_link_rpc_message(offset, size, false, &message); + apple_dcp_link_stream_message(message, 0, remote, &message); + dma_wmb(); + ret = b->ops->send(b->cookie, APPLE_DCP_LINK_ENDPOINT, message); + if (ret) + fail(b); +out: + mutex_unlock(&b->lock); +free_record: + kfree(record); + return ret ? ret : count; +} + +struct m3_dcpext_rpc *m3_dcpext_rpc_create(struct device *dev, void *rpc, + const struct m3_dcpext_rpc_ops *ops, + void *cookie) +{ + struct m3_dcpext_rpc *b; + if (!dev || !rpc || !ops || !ops->send || !ops->poll) + return ERR_PTR(-EINVAL); + b = kzalloc(sizeof(*b), GFP_KERNEL); + if (!b) + return ERR_PTR(-ENOMEM); + b->dev = dev; + b->rpc = rpc; + b->ops = ops; + b->cookie = cookie; + b->enabled = true; + mutex_init(&b->lock); + INIT_LIST_HEAD(&b->events); + return b; +} +void m3_dcpext_rpc_destroy(struct m3_dcpext_rpc *b) +{ + struct bridge_event *e, *next; + list_for_each_entry_safe(e, next, &b->events, list) { + list_del(&e->list); + kfree(e); + } + kfree(b); +} +static struct bridge_event *kernel_event(struct m3_dcpext_rpc *b, + unsigned long timeout) +{ + struct bridge_event *e; + long ret; + + unsigned long deadline = jiffies + timeout; + + while (!READ_ONCE(b->event_count) && !READ_ONCE(b->failed)) { + long remaining = deadline - jiffies; + + if (!timeout || remaining <= 0) + return ERR_PTR(-ETIMEDOUT); + ret = b->ops->poll(b->cookie, remaining); + if (ret) + return ERR_PTR(ret); + } + mutex_lock(&b->lock); + if (b->failed || list_empty(&b->events)) { + mutex_unlock(&b->lock); + return ERR_PTR(-EIO); + } + e = list_first_entry(&b->events, struct bridge_event, list); + list_del(&e->list); + b->event_count--; + mutex_unlock(&b->lock); + return e; +} + +static int kernel_send(struct m3_dcpext_rpc *b, u32 kind, u64 message, + const void *data, u32 size) +{ + struct record_header *record; + ssize_t ret; + + if (size > APPLE_DCP_LINK_STREAM_BUFFER_SIZE) + return -EMSGSIZE; + record = kmalloc(sizeof(*record) + size, GFP_KERNEL); + if (!record) + return -ENOMEM; + *record = (struct record_header){ + .kind = cpu_to_le32(kind), + .size = cpu_to_le32(size), + .message = cpu_to_le64(message), + }; + if (size) + memcpy(record + 1, data, size); + ret = bridge_request(b, record, sizeof(*record) + size); + return ret < 0 ? ret : 0; +} + +static int kernel_callback(struct m3_dcpext_rpc *b, struct bridge_event *e, + m3_dcpext_callback_fn callback, void *cookie, + u32 *completed) +{ + struct apple_dcp_link_rpc_header *h = (void *)e->data; + u32 size = le32_to_cpu(e->header.size), in, out, tag; + int ret; + + if (le32_to_cpu(e->header.kind) != 2 || size < sizeof(*h)) + return -EPROTO; + in = le32_to_cpu(h->input_size); + out = le32_to_cpu(h->output_size); + tag = le32_to_cpu(h->call); + if (in > size - sizeof(*h) || out != size - sizeof(*h) - in || + !callback) + return -EPROTO; + memset(e->data + sizeof(*h) + in, 0, out); + b->kernel_callback_depth++; + ret = callback(b, cookie, tag, h + 1, in, e->data + sizeof(*h) + in, + out); + b->kernel_callback_depth--; + if (ret) + return ret; + if (tag == 0x44353839 && in == 0x6f0 && completed) + *completed = le32_to_cpup((__le32 *)(h + 1)); + return kernel_send(b, 2, le64_to_cpu(e->header.message), + e->data + sizeof(*h) + in, out); +} + +int m3_dcpext_rpc_pump(struct m3_dcpext_rpc *b, unsigned long timeout, + m3_dcpext_callback_fn callback, void *cookie) +{ + struct bridge_event *e; + int ret; + + e = kernel_event(b, timeout); + if (IS_ERR(e)) + return PTR_ERR(e); + ret = kernel_callback(b, e, callback, cookie, NULL); + kfree(e); + return ret; +} + +static int rpc_call(struct m3_dcpext_rpc *b, u32 tag, const void *input, + u32 input_size, void *output, u32 output_size, + u32 completion_id, m3_dcpext_callback_fn callback, + void *cookie, bool (*valid)(void *), bool *rejected) +{ + struct apple_dcp_link_rpc_header *packet; + unsigned long deadline = jiffies + 5 * HZ; + u32 size, completed = 0; + u64 expected = 0x42; + bool nested, replied = false; + int ret; + + ret = apple_dcp_link_rpc_payload_size(input_size, output_size, &size); + if (ret || size > APPLE_DCP_LINK_STREAM_BUFFER_SIZE) + return -EMSGSIZE; + packet = kzalloc(size, GFP_KERNEL); + if (!packet) + return -ENOMEM; + apple_dcp_link_rpc_header_encode(packet, tag, input_size, output_size); + if (input_size) + memcpy(packet + 1, input, input_size); +retry: + /* Drain asynchronous notifications before starting an outer request. + * A queued callback is not recursive entry until its handler is running. + */ + if (!b->kernel_callback_depth) { + while (READ_ONCE(b->event_count)) { + if (time_after_eq(jiffies, deadline)) { + kfree(packet); + return -ETIMEDOUT; + } + ret = m3_dcpext_rpc_pump(b, 0, callback, cookie); + if (ret) { + kfree(packet); + return ret; + } + } + } + mutex_lock(&b->lock); + nested = b->kernel_callback_depth != 0; + if (nested && !b->callback_count) { + mutex_unlock(&b->lock); + kfree(packet); + return -EPROTO; + } + if (nested) + expected |= b->callbacks[b->callback_count - 1].message & + BIT_ULL(8); + mutex_unlock(&b->lock); + if (valid && !valid(cookie)) { + *rejected = true; + kfree(packet); + return -ESTALE; + } + ret = kernel_send(b, 1, nested, packet, size); + if (ret == -EBUSY && !nested && time_before(jiffies, deadline)) + goto retry; /* A callback arrived between draining and submitting. */ + kfree(packet); + if (ret) + return ret; + while (time_before(jiffies, deadline)) { + long remaining = deadline - jiffies; + struct bridge_event *e = kernel_event(b, max(remaining, 0L)); + struct apple_dcp_link_rpc_header *h; + + if (IS_ERR(e)) + return PTR_ERR(e); + h = (void *)e->data; + if (le32_to_cpu(e->header.kind) == 1) { + if (replied || le32_to_cpu(e->header.size) != size || + le64_to_cpu(e->header.message) != expected || + le32_to_cpu(h->call) != tag || + le32_to_cpu(h->input_size) != input_size || + le32_to_cpu(h->output_size) != output_size) { + ret = -EPROTO; + } else { + if (output_size) + memcpy(output, + e->data + sizeof(*h) + + input_size, + output_size); + replied = true; + } + } else { + ret = kernel_callback(b, e, callback, cookie, + &completed); + if (completion_id && completed && + completed != completion_id) + ret = -EPROTO; + } + kfree(e); + if (ret) + return ret; + if (replied && (!completion_id || completed == completion_id)) + return 0; + } + return -ETIMEDOUT; +} + +int m3_dcpext_rpc_alloc(struct m3_dcpext_rpc *b, struct m3_dcpext_buffer *info) +{ + u32 top; + int ret = -EINVAL; + mutex_lock(&b->lock); + if (!b->failed && b->callback_count && b->ops->alloc) { + top = b->callback_count - 1; + if (b->callbacks[top].call == 0x44343531 && info->size && + b->callbacks[top].allocation_size == info->size && + !info->id && !info->flags && !info->dva && + !info->physical) { + ret = b->ops->alloc(b->cookie, info); + if (!ret) + b->callbacks[top].allocation_size = 0; + } + } + mutex_unlock(&b->lock); + return ret; +} +int m3_dcpext_rpc_map(struct m3_dcpext_rpc *b, struct m3_dcpext_buffer *info) +{ + u32 top; + int ret = -EINVAL; + mutex_lock(&b->lock); + if (!b->failed && b->callback_count && b->ops->map) { + top = b->callback_count - 1; + if (b->callbacks[top].call == 0x44323031 && info->id && + b->callbacks[top].mapping_id == info->id && !info->flags && + !info->size && !info->dva && !info->physical) + ret = b->ops->map(b->cookie, info); + } + mutex_unlock(&b->lock); + return ret; +} +int m3_dcpext_rpc_dart_power(struct m3_dcpext_rpc *b, bool on) +{ + int ret = -EOPNOTSUPP; + mutex_lock(&b->lock); + if (!b->failed && b->callback_count && b->ops->dart_power && + b->callbacks[b->callback_count - 1].call == 0x44353734) + ret = b->ops->dart_power(b->cookie, on); + mutex_unlock(&b->lock); + return ret; +} + +int m3_dcpext_rpc_retire(struct m3_dcpext_rpc *b, u32 id) +{ + if (!id || !b->ops->retire) + return -EINVAL; + return b->ops->retire(b->cookie, id); +} + +int m3_dcpext_rpc_call(struct m3_dcpext_rpc *b, u32 tag, const void *input, + u32 input_size, void *output, u32 output_size, + u32 completion_id, m3_dcpext_callback_fn callback, + void *cookie) +{ + return m3_dcpext_rpc_call_checked(b, tag, input, input_size, output, + output_size, completion_id, callback, cookie, NULL); +} + +int m3_dcpext_rpc_call_checked(struct m3_dcpext_rpc *b, u32 tag, const void *input, + u32 input_size, void *output, u32 output_size, + u32 completion_id, m3_dcpext_callback_fn callback, + void *cookie, bool (*valid)(void *)) +{ + bool rejected = false; + int ret; + + if ((input_size && !input) || (output_size && !output)) + return -EINVAL; + ret = rpc_call(b, tag, input, input_size, output, output_size, + completion_id, callback, cookie, valid, &rejected); + if (ret && !rejected) + m3_dcpext_rpc_crashed(b); + return ret; +} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rpc.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rpc.h new file mode 100644 index 0000000000000..6fd2064fd2b01 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rpc.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef M3_DCPEXT_RPC_H +#define M3_DCPEXT_RPC_H +#include +struct device; +struct m3_dcpext_rpc; +struct m3_dcpext_buffer { + u64 size, physical, dva; + u32 id, flags; +}; +/* Owner retains RPC storage and every DMA allocation until qualified firmware + * quiescence, including failures. poll dispatches mailbox messages back into + * receive; it must honor the owner's absolute deadline/watchdog/thermal gates. + * Calls/pump are serialized by the owner; receive may run on another worker. + * send must not synchronously reenter receive (the RPC mutex is held). + */ +struct m3_dcpext_rpc_ops { + int (*send)(void *cookie, u8 endpoint, u64 message); + int (*poll)(void *cookie, unsigned long timeout); + int (*alloc)(void *cookie, struct m3_dcpext_buffer *buffer); + int (*map)(void *cookie, struct m3_dcpext_buffer *buffer); + int (*retire)(void *cookie, u32 id); + /* Firmware power vote; owner may retain an independent session vote. */ + int (*dart_power)(void *cookie, bool on); + /* Optional bounded diagnostic sink: kind 1 reply, 2 callback, + * 3 host request, 4 callback reply. Failure stops transport. */ + int (*record)(void *cookie, u32 kind, u64 message, const void *data, + u32 size); +}; +typedef int (*m3_dcpext_callback_fn)(struct m3_dcpext_rpc *, void *, u32, + const void *, u32, void *, u32); +struct m3_dcpext_rpc *m3_dcpext_rpc_create(struct device *, void *memory, + const struct m3_dcpext_rpc_ops *, + void *cookie); +void m3_dcpext_rpc_receive(struct m3_dcpext_rpc *, u64 message); +void m3_dcpext_rpc_crashed(struct m3_dcpext_rpc *); +/* Only after all callbacks/workers stop and firmware quiescence is proven. */ +void m3_dcpext_rpc_destroy(struct m3_dcpext_rpc *); +int m3_dcpext_rpc_call(struct m3_dcpext_rpc *, u32 tag, const void *, u32, + void *, u32, u32 completion_id, m3_dcpext_callback_fn, + void *); +/* Check owner state after draining callbacks and before each send attempt. + * The owner serializes state changes with calls/pump. A false check returns + * -ESTALE without sending this request or poisoning the transport. */ +int m3_dcpext_rpc_call_checked(struct m3_dcpext_rpc *, u32 tag, const void *, u32, + void *, u32, u32 completion_id, + m3_dcpext_callback_fn, void *, bool (*valid)(void *)); +int m3_dcpext_rpc_pump(struct m3_dcpext_rpc *, unsigned long timeout, + m3_dcpext_callback_fn, void *); +int m3_dcpext_rpc_alloc(struct m3_dcpext_rpc *, struct m3_dcpext_buffer *); +int m3_dcpext_rpc_map(struct m3_dcpext_rpc *, struct m3_dcpext_buffer *); +int m3_dcpext_rpc_retire(struct m3_dcpext_rpc *, u32 id); +int m3_dcpext_rpc_dart_power(struct m3_dcpext_rpc *, bool on); +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rtkit.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rtkit.c new file mode 100644 index 0000000000000..f678591eb96ef --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rtkit.c @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#define DCPEXT_WITH_SESSION +#include "m3_dcpext_dma.c" diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rtkit_main.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rtkit_main.c new file mode 100644 index 0000000000000..65c408cb40bed --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rtkit_main.c @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include "m3_dcpext_rtkit.c" diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_secondary_main.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_secondary_main.c new file mode 100644 index 0000000000000..0d0f0e9f08b51 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_secondary_main.c @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#define DCPEXT_INSTANCE 1 +#include "m3_dcpext_rtkit.c" diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_secondary_power.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_secondary_power.c new file mode 100644 index 0000000000000..6689776e93f9d --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_secondary_power.c @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#define DCPEXT_INSTANCE 1 +#include "m3_dcpext_power.c" diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h new file mode 100644 index 0000000000000..338808ba96bc8 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h @@ -0,0 +1,913 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* J514S 14.6 RTKit system-only handshake. Based on the M4 session adapter + * and Asahi mailbox/RTKit transport. No application endpoint or PHY access. + * All allocations and power references are retained until reboot, even after + * successful protocol quiescence. That ACK alone is not proof of DMA stop. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "m3_hdmi_control.h" +/* Controller identity selects firmware/memory; connector routing is separate. */ +static bool usb_c=DCPEXT_INSTANCE!=0; +module_param(usb_c,bool,0400); +/* Opt-in compatibility experiment for eventual shared-stream routing. + * Keep the normal HDMI four-lane/HBR policy until separately qualified. + */ +static bool hdmi_test_two_lane_hbr2; +module_param(hdmi_test_two_lane_hbr2,bool,0400); +static unsigned int link_max_lanes(void) +{return usb_c || hdmi_test_two_lane_hbr2 ? 2 : 4;} +static const struct m3_usbc_route_ops *usbc_ops; +#include "dcpext_session.c" +#include "dcpext_syslog.h" +static struct dcpext_syslog syslog_capture; +static struct debugfs_blob_wrapper syslog_blob; +static struct dcpext_session session; +static DECLARE_COMPLETION(desktop_started); +static DECLARE_COMPLETION(desktop_finished); +static bool desktop_stop,desktop_done; +static int desktop_result=-EINPROGRESS; +extern int m3_hdmi_bridge_hpd(void); +static int (*desktop_read_hpd)(void); +static u64 desktop_hpd_next; +static unsigned int desktop_hpd_low_samples; +static unsigned int desktop_hpd_high_samples; +static bool desktop_disconnected; +static bool desktop_hpd_dirty, desktop_hpd_rpc, desktop_hpd_value; +static u64 desktop_hpd_deadline; +static unsigned int desktop_link_stage; +/* Bounded diagnostic HPD cycle; actual GPIO is sampled again after 3 s. */ +static bool desktop_retrain; +static u64 desktop_retrain_until; +module_param(desktop_retrain,bool,0600); +module_param(desktop_disconnected,bool,0400); +module_param(desktop_stop,bool,0600); +module_param(desktop_done,bool,0400); +module_param(desktop_result,int,0400); +static void desktop_notify(int ret) +{WRITE_ONCE(desktop_result,ret);complete_all(&desktop_started);} + +static struct regmap *session_pmgr; +static bool cpu_no_auto; +module_param(cpu_no_auto, bool, 0400); +static bool probe_nmi,nmi_sent; +module_param(probe_nmi,bool,0400); +static bool probe_ping; +module_param(probe_ping, bool, 0400); +static bool ping_sent, ping_replied; +static void __iomem *session_cpu, *session_fifo; +static bool session_cpu_owned, session_fifo_owned; +static phys_addr_t session_base; +static struct resource session_table_ram; +static void *session_table_map, *session_table_saved; +static size_t session_used; +static struct { + struct resource ram; + u64 remap; + u32 size, flags; +} session_segments[32]; +static unsigned int session_nsegments; +static void session_cleanup(void) +{ + if (session_table_map) memunmap(session_table_map); + kfree(session_table_saved); + if (session_fifo) iounmap(session_fifo); + if (session_cpu) iounmap(session_cpu); + if (session_fifo_owned) release_mem_region(session_base + 0x8000, 0x4000); + if (session_cpu_owned) release_mem_region(session_base, 0x4000); +} +static int session_prepare(struct device_node *dcp) +{ + struct device_node *node; + struct resource res; + const u8 *metadata; + struct regmap *map; + u32 power; + int bytes, ret; + + session_base = port ? 0x2d6c00000ULL : 0x2d2c00000ULL; + if (of_address_to_resource(dcp, 0, &res) || res.start != session_base || + resource_size(&res) != 0x4000 || !(res.flags & IORESOURCE_MEM_NONPOSTED)) + return -ENODEV; + metadata = of_get_property(dcp, "apple,j514s-dcpext-segments", &bytes); + if (!metadata || bytes <= 0 || bytes % 32 || bytes / 32 > 32) + return -EINVAL; + session_nsegments = bytes / 32; + for (unsigned int i = 0; i < session_nsegments; i++) { + node = of_parse_phandle(dcp, "memory-region", i); + ret = -EINVAL; + if (node && of_device_is_available(node) && + of_property_read_bool(node, "no-map") && + of_device_is_compatible(node, "apple,asc-mem")) + ret = of_address_to_resource(node, 0, &session_segments[i].ram); + of_node_put(node); + if (ret) + return ret; + session_segments[i].remap = get_unaligned_be64(metadata + i * 32 + 16); + session_segments[i].size = get_unaligned_be32(metadata + i * 32 + 24); + session_segments[i].flags = get_unaligned_be32(metadata + i * 32 + 28); + if (!session_segments[i].size || + session_segments[i].ram.start != get_unaligned_be64(metadata + i * 32) || + resource_size(&session_segments[i].ram) != ALIGN((u64)session_segments[i].size, LEASE_PAGE)) + return -EINVAL; + } + /* No CPU control writes. SYS/FE checks are in the shared mapper. */ + node = of_find_node_by_path("/soc/power-management@350700000"); + if (!node) + return -ENODEV; + map = syscon_node_to_regmap(node); + of_node_put(node); + if (IS_ERR(map)) + return PTR_ERR(map); + session_pmgr = map; + ret = regmap_read(map, port ? 0x3f8 : 0x3e0, &power); + if (ret) + return ret; + if ((power & 15) != 15 || power & (BIT(31) | BIT(12) | BIT(11) | BIT(10))) + return -EHOSTDOWN; + if (!request_mem_region(session_base, 0x4000, "m3-dcpext-session")) + return -EBUSY; + session_cpu_owned = true; + if (!request_mem_region(session_base + 0x8000, 0x4000, "m3-dcpext-session")) + return -EBUSY; + session_fifo_owned = true; + /* Mapping here does not touch the registers; fabric is checked before + * session_run accesses these windows. + */ + session_cpu = ioremap_np(session_base, 0x4000); + session_fifo = ioremap_np(session_base + 0x8000, 0x4000); + return session_cpu && session_fifo ? 0 : -ENOMEM; +} + +static int session_tables(const struct resource *reservation) +{ + session_table_ram = *reservation; + if (!resource_size(reservation) || resource_size(reservation) > 0x100000) + return -EINVAL; + session_table_map = memremap(reservation->start, resource_size(reservation), MEMREMAP_WB); + if (!session_table_map) + return -ENOMEM; + session_table_saved = kmemdup(session_table_map, resource_size(reservation), GFP_KERNEL); + return session_table_saved ? 0 : -ENOMEM; +} + +static int session_translate(u64 wire, phys_addr_t *physical) +{ + u64 dva, entry, leaf_pa, pte; + u64 *table; + + if ((wire >> 36) != 0x10) + return -ERANGE; + dva = wire & GENMASK_ULL(35, 0); + entry = READ_ONCE(root[dva >> 25]); + if (!(entry & 1)) + return -ENOENT; + leaf_pa = (entry & PTE_ADDRESS) << 4; + if (leaf_pa < session_table_ram.start || leaf_pa > session_table_ram.end || + session_table_ram.end - leaf_pa + 1 < LEASE_PAGE) + return -ERANGE; + table = session_table_map + leaf_pa - session_table_ram.start; + pte = READ_ONCE(table[(dva >> 14) & 2047]); + if (!(pte & 1)) + return -ENOENT; + *physical = ((pte & PTE_ADDRESS) << 4) | (dva & (LEASE_PAGE - 1)); + return 0; +} + +static int session_buffer(void *cookie, u8 ep, u64 requested, u32 size, u64 *dva) +{ + phys_addr_t pa, checked; + size_t rounded = ALIGN(size, LEASE_PAGE); + int ret; + + if (!requested) { + if (rounded > LEASE_SIZE - session_used) + return -ENOSPC; + memset(pool + session_used, 0, rounded); + *dva = BIT_ULL(40) | (LEASE_DVA + session_used); + session_used += rounded; + pr_info("m3_dcpext_session: ep=%u host buffer size=%#x dva=%#llx\n", ep, size, *dva); + return 0; + } + for (unsigned int i = 0; i < session_nsegments; i++) { + /* ADT flags 0xa OS-log region is a host physical carveout, not + * a DART mapping. Exact base and declared size are mandatory. + */ + if (ep == 8 && session_segments[i].flags == 0xa && + requested == session_segments[i].ram.start && + requested == session_segments[i].remap && size <= session_segments[i].size) { + *dva = requested; + pr_info("m3_dcpext_session: ep=8 reserved physical OSLog size=%#x address=%#llx\n", size, requested); + return 0; + } + } + ret = session_translate(requested, &pa); + if (ret) + return ret; + for (unsigned int i = 0; i < session_nsegments; i++) { + struct resource *r = &session_segments[i].ram; + if (session_segments[i].flags == 0xa || pa < r->start || pa > r->end || + size - 1 > r->end - pa) + continue; + /* Verify every page boundary, including an unaligned first page. */ + for (u64 off = 0; off < size; ) { + ret = session_translate(requested + off, &checked); + if (ret || checked != pa + off) + return -ERANGE; + off += LEASE_PAGE - ((requested + off) & (LEASE_PAGE - 1)); + } + ret = session_translate(requested + size - 1, &checked); + if (ret || checked != pa + size - 1) + return -ERANGE; + *dva = requested; + pr_info("m3_dcpext_session: ep=%u inherited size=%#x dva=%#llx physical=%#llx\n", ep, size, requested, (u64)pa); + return 0; + } + return -ERANGE; +} + +static u64 session_now(void *cookie) +{ + return ktime_get_boottime_ns() / NSEC_PER_MSEC; +} + + +/* Validate ALL inherited mappings against the bootloader-owned segments, + * not just addresses requested by the firmware later. Never follow a table + * pointer outside the reserved page-table storage. */ +static int session_audit(void) +{ + for (unsigned int i = 0; i < LEASE_PAGE / 8; i++) { + u64 entry = READ_ONCE(root[i]), pa; + u64 *table; + if (!(entry & 1)) continue; + pa = (entry & PTE_ADDRESS) << 4; + if (pa < session_table_ram.start || pa > session_table_ram.end || + session_table_ram.end - pa + 1 < LEASE_PAGE) return -ERANGE; + table = session_table_map + pa - session_table_ram.start; + for (unsigned int j = 0; j < LEASE_PAGE / 8; j++) { + u64 pte = READ_ONCE(table[j]), va = ((u64)i << 25) | ((u64)j << 14); + bool found = false; + if (!(pte & 1)) continue; + pa = (pte & PTE_ADDRESS) << 4; + for (unsigned int n = 0; n < session_nsegments; n++) { + u64 start = session_segments[n].remap & GENMASK_ULL(35, 0); + if (session_segments[n].flags == 0xa) continue; + if (va >= start && va - start < ALIGN((u64)session_segments[n].size, LEASE_PAGE) && + pa == session_segments[n].ram.start + va - start) found = true; + } + if (!found) return -ERANGE; + } + } + return 0; +} +static int session_send(void *cookie, u8 ep, u64 message) +{ + u32 status; + int ret = readl_poll_timeout(session_fifo + 0x110, status, !(status & BIT(16)), 100, 100000); + if (ret) return ret; + dma_wmb(); + writeq(message, session_fifo + 0x800); + writeq(ep, session_fifo + 0x808); + if(!session.runtime)pr_info("m3_dcpext_session: TX ep=%#x msg=%#llx\n", ep, message); + return 0; +} +#include "dcpext_afk_channel.h" +static bool quiesce_first; +module_param(quiesce_first, bool, 0400); +static unsigned int afk_endpoint; +module_param(afk_endpoint, uint, 0400); +MODULE_PARM_DESC(afk_endpoint, "Optional announcement-only AFK endpoint, 0x20..0x2b; zero for system-only"); +static struct afkc afk, aux_afk; +static unsigned int aux_endpoint; +module_param(aux_endpoint, uint, 0400); +static bool validate_connection; +module_param(validate_connection, bool, 0400); +static bool connect_port; +module_param(connect_port, bool, 0400); +static bool validation_sent, validation_reply, port0_announced; +static u32 port0_channel; + +static u64 afk_deadline; +static int afk_allocate(void *cookie, u32 size, u8 **buffer, u64 *dva) +{ + size_t offset = session_used; + int ret = session_buffer(NULL, afk_endpoint, 0, size, dva); + if (!ret) *buffer = pool + offset; + return ret; +} +#include "m3_dcpext_native_query.h" +static bool request_display; +module_param(request_display, bool, 0400); +static bool hpd_low_only; +module_param(hpd_low_only,bool,0400); +static bool hotplug; +module_param(hotplug, bool, 0400); +static u64 hpd_until; +static unsigned int rpc_stage, rpc_command, rpc_bytes, rpc_tag, rpc_target; +static u8 *rpc_rx, *rpc_tx; +static u64 rpc_rx_dva, rpc_tx_dva; +static bool rpc_pending; +static u16 epic_seq; +static unsigned int link_lanes; +module_param(link_lanes,uint,0400); +#include "m3_epic_tx.h" +static int old_epic_write_channel(struct afkc *c,u32 channel,u32 kind,u8 *payload,u32 bytes) +{ + u32 next;int ret; + if(!c->running || c->stopping || c->error)return -EINVAL; + ret=m3_epic_ring_write(&c->rings[0],channel,kind,payload,bytes,&next); + return ret ?: afkc_send(c,0xa2,next); +} +static int old_epic_write(u32 channel,u32 kind,u8 *payload,u32 bytes) +{return old_epic_write_channel(&afk,channel,kind,payload,bytes);} +#include "m3_dcpext_iboot.h" +static int link_activate(void) +{ + int ret;int (*fn)(void); + if(usb_c)return usbc_ops->activate(port); + fn=symbol_get(m3_hdmi_phy_activate);if(!fn)return -ENODEV; + ret=fn();symbol_put(m3_hdmi_phy_activate);return ret; +} +static int link_deactivate(void) +{ + int ret;int (*fn)(void); + if(usb_c)return usbc_ops->deactivate(port); + fn=symbol_get(m3_hdmi_phy_deactivate);if(!fn)return -ENODEV; + ret=fn();symbol_put(m3_hdmi_phy_deactivate);return ret; +} +static int link_xbar_down(void) +{ + int ret;int (*fn)(void); + if(usb_c)return usbc_ops->xbar_down(port); + fn=symbol_get(m3_hdmi_xbar_deactivate);if(!fn)return -ENODEV; + ret=fn();symbol_put(m3_hdmi_xbar_deactivate);return ret; +} +static int link_xbar_up(void) +{ + int ret;int (*fn)(void); + if(usb_c)return usbc_ops->xbar_up(port); + fn=symbol_get(m3_hdmi_xbar_activate);if(!fn)return -ENODEV; + ret=fn();symbol_put(m3_hdmi_xbar_activate);return ret; +} +static int link_get_rate(void) +{ + int ret;int (*fn)(void); + if(usb_c)return usbc_ops->get_rate(port); + fn=symbol_get(m3_hdmi_phy_get_rate);if(!fn)return -ENODEV; + ret=fn();symbol_put(m3_hdmi_phy_get_rate);return ret; +} +static int link_set_rate(unsigned int rate) +{ + int ret;int (*fn)(unsigned int); + if(usb_c)return usbc_ops->set_rate(port,rate); + fn=symbol_get(m3_hdmi_phy_set_rate);if(!fn)return -ENODEV; + ret=fn(rate);symbol_put(m3_hdmi_phy_set_rate);return ret; +} +static int link_set_drive(const unsigned int *drive) +{ + u32 hdmi_drive[12]; + int ret;int (*fn)(const unsigned int *); + if(usb_c)return usbc_ops->set_drive(port,drive); + /* The HDMI PHY retains its qualified four-lane physical orientation. + * DPTX encodes only the two negotiated lanes in the diagnostic mode. + * Program equal pair settings, as required by the existing PHY helper; + * this does not claim a new physical lane order or power down either pair. + */ + if(hdmi_test_two_lane_hbr2){ + memcpy(hdmi_drive,drive,6*sizeof(*drive)); + memcpy(hdmi_drive+6,drive,6*sizeof(*drive)); + drive=hdmi_drive; + } + fn=symbol_get(m3_hdmi_phy_set_drive);if(!fn)return -ENODEV; + ret=fn(drive);symbol_put(m3_hdmi_phy_set_drive);return ret; +} +static int link_hpd(void) +{return usbc_ops->hpd(port);} +static int dptx_rpc_send(u32 command) +{ + u8 *tx,wire[70]={};u64 tx_dva;u32 bytes; + int ret; + if (rpc_pending) return -EBUSY; + bytes=command==12?112:(command==11||command==8)?96:80; + /* One outstanding RPC: reuse its pinned buffers only after its reply. */ + if(!rpc_rx){ret=afk_allocate(NULL,112,&rpc_rx,&rpc_rx_dva);if(ret)return ret;} + if(!rpc_tx){ret=afk_allocate(NULL,112,&rpc_tx,&rpc_tx_dva);if(ret)return ret;} + tx=rpc_tx;tx_dva=rpc_tx_dva;memset(tx,0,112);memset(rpc_rx,0,112); + put_unaligned_le32(command,tx+4);put_unaligned_le32(bytes-64,tx+8); + put_unaligned_le32(0x69706378,tx+12); + if(command==8){put_unaligned_le16(8,tx+2);put_unaligned_le32(desktop_hpd_rpc?desktop_hpd_value:rpc_stage==2 && !hpd_low_only,tx+80);} + if(command==11||command==12){ + put_unaligned_le32(command==12?0x100:0,tx+64); + rpc_target=0x8030; + if(usb_c){ + ret=usbc_ops->target(port);if(ret<0)return ret; + rpc_target=ret; + } + put_unaligned_le32(rpc_target,tx+68); + } + wire[0]=2;put_unaligned_le16(epic_seq++,wire+1); + put_unaligned_le32(30,wire+16);wire[20]=4;wire[21]=0x30; + put_unaligned_le16(0xc0,wire+22);put_unaligned_le16(rpc_tag,wire+32); + put_unaligned_le64(rpc_rx_dva,wire+44);put_unaligned_le64(tx_dva,wire+52); + put_unaligned_le32(bytes,wire+60);put_unaligned_le32(bytes,wire+64); + rpc_command=command;rpc_bytes=bytes;rpc_pending=true; + pr_info("m3_dcpext_session: DPTX command=%u tag=%u bytes=%u\n",command,rpc_tag,bytes); + return old_epic_write(port0_channel,3,wire,sizeof(wire)); +} +#include "m3_dcpext_av.h" +static u32 link_drive[8]; +static int dptx_apcall(u32 channel,const u8 *payload,u32 size) +{ + u8 reply[512];u32 idx,len;u8 *data; + int ret; + if(size<108 || size>sizeof(reply) || get_unaligned_le32(payload+56)!=0x69706378) return -EPROTO; + idx=get_unaligned_le32(payload+48);len=get_unaligned_le32(payload+52); + if(len<4||len>size-108)return -EPROTO; + pr_info("m3_dcpext_session: DPTX callback=%u size=%u\n",idx,len); + memcpy(reply,payload,size);data=reply+108; + put_unaligned_le32(0,data); + switch(idx){ + case 0: + ret=link_activate();if(ret)return ret; + break; + case 1: + ret=link_xbar_down();if(ret)return ret; + ret=link_deactivate();if(ret)return ret; + break; + case 2: /* 14.6 ABI: two words, voltage and preemphasis (no kind). */ + if(len!=32)return -EINVAL; + put_unaligned_le32(3,data+16);put_unaligned_le32(3,data+20);break; + case 3: { + u32 drive[12]={}; + if(len!=32+8*link_max_lanes() || get_unaligned_le32(data+16)!=link_max_lanes() || link_lanes!=link_max_lanes())return -EOPNOTSUPP; + for(unsigned int i=0;i3 || drive[3*i+2]>3 || drive[3*i+1]+drive[3*i+2]>3)return -EINVAL; + } + ret=link_set_drive(drive);if(ret)return ret; + memset(link_drive,0,sizeof(link_drive)); + for(unsigned int i=0;i<2*link_max_lanes();i++)link_drive[i]=get_unaligned_le32(data+32+i*4); + pr_info("m3_dcpext_session: drive %u/%u %u/%u %u/%u %u/%u applied\n", + link_drive[0],link_drive[1],link_drive[2],link_drive[3],link_drive[4],link_drive[5],link_drive[6],link_drive[7]); + break; + } + case 4: + if(len!=32+8*link_max_lanes() || get_unaligned_le32(payload+108)!=link_max_lanes() || link_lanes!=link_max_lanes())return -EOPNOTSUPP; + put_unaligned_le32(link_max_lanes(),data);put_unaligned_le32(0,data+16); + for(unsigned int i=0;i<2*link_max_lanes();i++)put_unaligned_le32(link_drive[i],data+32+i*4); + break; + case 5: + if(len!=16)return -EINVAL; + ret=link_xbar_down();if(ret)return ret;break; + case 6: + if(len!=16)return -EINVAL; + ret=link_get_rate();if(ret<0)return ret; + if(ret){ret=link_xbar_up();if(ret)return ret;}break; + case 7: + if(len!=32)return -EINVAL;put_unaligned_le32(usb_c || hdmi_test_two_lane_hbr2 ? 20 : 10,data+16);break; + case 8: + if(len!=32)return -EINVAL; + ret=link_get_rate();if(ret<0)return ret; + put_unaligned_le32(ret,data+16);break; + case 9: + if(len!=32 || get_unaligned_le32(data+20))return -EINVAL; + ret=link_xbar_down();if(ret)return ret; + ret=link_set_rate(get_unaligned_le32(data+16));if(ret)return ret;break; + case 10:case 11: + if(len!=32)return -EINVAL;put_unaligned_le64(idx==10?link_max_lanes():link_lanes,data+16);break; + case 12: + if(len!=32 || get_unaligned_le64(data+16)>link_max_lanes() || + (get_unaligned_le64(data+16)!=link_max_lanes() && get_unaligned_le64(data+16)!=0))return -EOPNOTSUPP; + link_lanes=get_unaligned_le32(data+16);break; + case 15: + if(len!=32 || get_unaligned_le64(data+16))return -EOPNOTSUPP;break; + case 20: /* DPTX_APCALL_INACTIVE_SINK_DETECTED: notification only. */ + if(len!=16)return -EINVAL;break; + case 21: /* DPTX_APCALL_SET_TILED_DISPLAY_HINTS: retain/report hints. */ + if(len!=144)return -EINVAL; + pr_info("m3_dcpext_session: display hints width=%u height=%u pixel_clock=%u\n", + get_unaligned_le32(data+56),get_unaligned_le32(data+60),get_unaligned_le32(data+64)); + break; + case 18: /* Host owns the HDMI HPD line and sends the hotplug event. */ + if(len<32)return -EINVAL;put_unaligned_le32(hotplug,data+16);break; + case 13:case 14:case 16: + if(len<32)return -EINVAL;put_unaligned_le32(0,data+16);break; + default: + /* Never ACK an unimplemented hardware transition. */ + print_hex_dump(KERN_INFO,"m3_dcpext_unhandled: ",DUMP_PREFIX_OFFSET,16,1,data,len,true); + return -EOPNOTSUPP; + } + reply[0]=2;put_unaligned_le16(epic_seq++,reply+1);reply[21]=0x20; + reply[35]=8;put_unaligned_le32(size-44,reply+36); + return old_epic_write(channel,8,reply,size); +} +static int afk_record(void *cookie, const u8 *payload, u32 size) +{ + u32 channel = get_unaligned_le32(afk.capture + afk.capture_size - size - 12); + u32 kind = get_unaligned_le32(afk.capture + afk.capture_size - size - 8); + if (size >= 76 && payload[0] == 2 && payload[20] == 4 && !payload[21] && + get_unaligned_le16(payload + 22) == 0x30 && + !memcmp(payload + 40, port?"dispext1:dcpdptx-port-epic:0":"dispext0:dcpdptx-port-epic:0", sizeof("dispext0:dcpdptx-port-epic:0")) && + get_unaligned_le32(payload + 72) == 0xd3) { + port0_announced = true; + port0_channel = channel; + } + if (rpc_pending && (kind==0 || kind==4) && channel==port0_channel && + size>=70 && payload[0]==2 && payload[20]==4 && payload[21]==0x20 && + get_unaligned_le16(payload+22)==0xc0 && get_unaligned_le16(payload+32)==rpc_tag) { + pr_info("m3_dcpext_session: RPC response status=%#x bytes=%u\n",get_unaligned_le32(payload+40),get_unaligned_le32(payload+60)); + if(get_unaligned_le32(payload+40) || get_unaligned_le64(payload+44)!=rpc_rx_dva || + get_unaligned_le32(payload+60)!=rpc_bytes)return -EPROTO; + dma_rmb(); + if(get_unaligned_le32(rpc_rx+12)!=0x69706378 || + get_unaligned_le32(rpc_rx+4)!=rpc_command)return -EPROTO; + if((rpc_command==11||rpc_command==12) && + (get_unaligned_le32(rpc_rx+64)!=(rpc_command==12?0x100:0) || + get_unaligned_le32(rpc_rx+68)!=rpc_target))return -EPROTO; + pr_info("m3_dcpext_session: DPTX command=%u reply accepted\n",rpc_command); + rpc_pending=false;rpc_tag++; + if(desktop_hpd_rpc){ + desktop_hpd_rpc=false; + pr_info("m3_dcpext_session: runtime link stage=%u HPD=%u acknowledged\n",desktop_link_stage,desktop_hpd_value); + desktop_link_stage=(desktop_link_stage==3 || desktop_link_stage==5)?0:desktop_link_stage+1; + return 0; + } + rpc_stage++; + if(hotplug && rpc_stage==3)hpd_until=session_now(NULL)+2000; + validation_reply = !request_display || rpc_stage==(hotplug?5:3); + } + if(kind==0 && channel==port0_channel && size>=40 && payload[0]==2 && + payload[20]==4 && payload[21]==0x10 && get_unaligned_le16(payload+22)==0xc0) + return dptx_apcall(channel,payload,size); + pr_info("m3_dcpext_session: AFK record ep=%#x bytes=%u\n", afk_endpoint, size); + print_hex_dump(KERN_INFO, "m3_dcpext_afk: ", DUMP_PREFIX_OFFSET, 16, 1, + payload, min(size, 256U), true); + return 0; +} +static const struct afkc_ops afk_ops = { + .send = session_send, .allocate = afk_allocate, .record = afk_record, +}; +static bool log_sent, log_replied; +static int aux_record(void *cookie, const u8 *payload, u32 size) +{ + pr_info("m3_dcpext_session: auxiliary AFK ep=%#x bytes=%u\n",aux_endpoint,size); + if(aux_endpoint==0x28){ + const u8 *h=aux_afk.capture+aux_afk.capture_size-size-12; + int ret=av_record(get_unaligned_le32(h),get_unaligned_le32(h+4),payload,size);if(ret)return ret; + } + if(iboot_modes && aux_endpoint==0x23){ + u32 channel=get_unaligned_le32(aux_afk.capture+aux_afk.capture_size-size-12); + int ret=ib_record(channel,payload,size);if(ret)return ret; + } + if(aux_endpoint==0x20 && size>=72 && payload[21]==0 && !memcmp(payload+40,"system",7)){ + u8 *tx,*rx,wire[70]={};u64 txd,rxd;int ret; + static const u8 property[]={0x14,0,0,0,'g','A','F','K','C','o','n','f','i','g','L','o','g','M','a','s','k',0,0,0,0xd3,0,0,0,0x40,0,0,0x84,0xff,0xff,0,0,0,0,0,0}; + u32 channel=get_unaligned_le32(aux_afk.capture+aux_afk.capture_size-size-12); + if(log_sent)return -EPROTO; + ret=afk_allocate(NULL,sizeof(property),&tx,&txd);if(ret)return ret; + ret=afk_allocate(NULL,sizeof(property),&rx,&rxd);if(ret)return ret; + memcpy(tx,property,sizeof(property));wire[0]=2;put_unaligned_le32(30,wire+16); + wire[20]=4;wire[21]=0x30;put_unaligned_le16(0x43,wire+22); + put_unaligned_le64(rxd,wire+44);put_unaligned_le64(txd,wire+52); + put_unaligned_le32(sizeof(property),wire+60);put_unaligned_le32(sizeof(property),wire+64); + log_sent=true;return old_epic_write_channel(&aux_afk,channel,3,wire,sizeof(wire)); + } + if(aux_endpoint==0x20 && log_sent && size>=70 && payload[21]==0x20 && get_unaligned_le16(payload+22)==0x43){ + if(get_unaligned_le32(payload+40))return -EREMOTEIO; + log_replied=true;pr_info("m3_dcpext_session: firmware verbose logging enabled\n"); + } + print_hex_dump(KERN_INFO,"m3_dcpext_aux: ",DUMP_PREFIX_OFFSET,16,1,payload,min(size,256U),true); + return 0; +} +static const struct afkc_ops aux_ops={.send=session_send,.allocate=afk_allocate,.record=aux_record}; +#include "m3_dcpext_native_adapter.h" +#include "m3_dcpext_audio.h" +static int session_dispatch(u64 endpoint,u64 message) +{ + if(!session.runtime)pr_info("m3_dcpext_session: RX ep=%#llx msg=%#llx\n",endpoint,message); + if(session.runtime){int ret=dcpext_session_runtime_message(&session);if(ret)return ret;} + else if(++received>(native_kms_trial?8192:native_startup?2048:512))return -E2BIG; + if(!(endpoint&0xff) && (message>>52)==4 && ping_sent && !ping_replied){ping_replied=true;return 0;} + if(native_query && (endpoint&0xff)==0x37)return native_receive(message); + if(audio_afk.started && (endpoint&0xff)==0x29)return audio_receive(message); + if(aux_endpoint && (endpoint&0xff)==aux_endpoint)return afkc_receive(&aux_afk,message); + if(afk_endpoint && (endpoint&0xff)==afk_endpoint)return afkc_receive(&afk,message); + if ((endpoint&0xff)==2) { + struct dcpext_buffer *b=&session.buffers[2]; + u64 offset=b->dva & ~BIT_ULL(40); int ret; + if (b->ready && (offsetsession_used || b->size>session_used-(offset-LEASE_DVA))) return -ERANGE; + if(session.runtime && syslog_capture.used>sizeof(syslog_capture.capture)-512)syslog_capture.used=0; + ret=dcpext_syslog_observe(&syslog_capture,message,b->ready?pool+offset-LEASE_DVA:NULL,b->size); + if(ret)return ret; + } + return dcpext_session_receive(&session,endpoint&0xff,message); +} + +static struct debugfs_blob_wrapper ram_snapshot[2][32]; +static void snapshot_ram(struct dentry *dir, unsigned int sample) +{ + u32 pmgr_state; + regmap_read(session_pmgr, port?0x3f8:0x3e0, &pmgr_state); + pr_info("m3_dcpext_session: CPU PMGR=%#x\n", pmgr_state); + pr_info("m3_dcpext_session: ASC status=%#x timer_mask_set=%#x timer_mask_clear=%#x\n", readl(session_cpu+0x48), readl(session_cpu+0x1010), readl(session_cpu+0x1018)); + pr_info("m3_dcpext_session: RAM sample=%u host cntpct=%#llx boottime_ms=%llu\n", sample, read_sysreg(cntpct_el0), session_now(NULL)); + for(unsigned int i=0;i0x1000000)continue; + map=memremap(session_segments[i].ram.start,size,MEMREMAP_WB); + if(!map)continue; + ram_snapshot[sample][i].data=vmalloc(size);ram_snapshot[sample][i].size=size; + if(ram_snapshot[sample][i].data){ + /* Reserved no-map RAM is written only by DCP. This host alias is + * read-only: discard clean stale lines before every asynchronous sample. + * All segment boundaries are page aligned; never discard adjacent data. */ + for (unsigned long at=(unsigned long)map; at<(unsigned long)map+size; at+=64) + asm volatile("dc ivac, %0" :: "r"(at) : "memory"); + dsb(sy); + memcpy(ram_snapshot[sample][i].data,map,size); + snprintf(name,sizeof(name),"firmware-ram-%u%s",i,sample?"":"-early"); + debugfs_create_blob(name,0400,dir,&ram_snapshot[sample][i]); + pr_info("m3_dcpext_session: snapshot %s flags=%#x pa=%#llx remap=%#llx size=%#x\n",name,session_segments[i].flags,(u64)session_segments[i].ram.start,session_segments[i].remap,size); + } + memunmap(map); + } +} +static ssize_t pool_read(struct file *file,char __user *buf,size_t count,loff_t *offset) +{ + dma_rmb();return simple_read_from_buffer(buf,count,offset,pool,session_used); +} +static const struct file_operations pool_fops={.owner=THIS_MODULE,.read=pool_read,.llseek=default_llseek}; +static ssize_t aux_read(struct file *file,char __user *buf,size_t count,loff_t *offset) +{ + return simple_read_from_buffer(buf,count,offset,aux_afk.capture,aux_afk.capture_size); +} +static const struct file_operations aux_fops={.owner=THIS_MODULE,.read=aux_read,.llseek=default_llseek}; +static ssize_t afk_read(struct file *file, char __user *buf, size_t count, loff_t *offset) +{ + return simple_read_from_buffer(buf, count, offset, afk.capture, afk.capture_size); +} +static const struct file_operations afk_fops = { + .owner = THIS_MODULE, .read = afk_read, .llseek = default_llseek, +}; +static int session_run(void) +{ + static const struct dcpext_session_ops ops = { + .now_ms = session_now, .send = session_send, .buffer = session_buffer, + }; + u32 state; + struct dentry *dir = NULL; + u64 early_at; + bool early_saved = false; + int ret; + if(native_desktop && !native_kms_trial)return -EINVAL; + if(native_kms_trial && aux_endpoint!=0x28)return -EINVAL; + if(native_kms_trial && !native_frame)return -EINVAL; + if(native_frame && !native_startup)return -EINVAL; + if(probe_nmi && !iboot_image)return -EINVAL; + if(native_startup && (!native_query || !hotplug || iboot_modes))return -EINVAL; + if(iboot_image && !iboot_modes)return -EINVAL; + if(iboot_modes && (aux_endpoint!=0x23 || !hotplug || !cpu_no_auto))return -EINVAL; + if(aux_endpoint && ((aux_endpoint!=0x23 && aux_endpoint!=0x24 && aux_endpoint!=0x20 && aux_endpoint!=0x28) || !hotplug))return -EINVAL; + if (hotplug && !request_display) return -EINVAL; + if (request_display && !connect_port) return -EINVAL; + if (connect_port && !validate_connection) return -EINVAL; + if (validate_connection && afk_endpoint != 0x2a) return -EINVAL; + if (afk_endpoint && (afk_endpoint < 0x20 || afk_endpoint > 0x2b)) return -EINVAL; + if(port && !usb_c)return -EINVAL; + if(usb_c && hdmi_test_two_lane_hbr2)return -EINVAL; + link_lanes=link_max_lanes(); + if(usb_c){ + const struct m3_usbc_route_ops *(*get_ops)(void)=symbol_get(m3_usbc_get_ops); + if(!get_ops)return -ENODEV; + /* Retain the module reference with the session, including on failure. */ + usbc_ops=get_ops(); + ret=usbc_ops->prepare(port);if(ret)return ret; + } + if(native_desktop){ + /* Retain the GPIO owner's module reference with this session's resources. + * Diagnostics without a desktop do not require the bridge module. */ + desktop_read_hpd=usb_c?link_hpd:symbol_get(m3_hdmi_bridge_hpd); + if(!desktop_read_hpd)return -ENODEV; + ret=desktop_read_hpd(); + if(ret<0 || (!usb_c && ret!=1))return ret<0?ret:-ENOLINK; + } + state = readl(session_cpu + 0x44); + /* Fresh boot only. CPU is stopped and no protocol session is inherited. + * Standard Asahi dcp.c start: RUN only, preserve vector/reset state. */ + if (state != 0 || !(readl(session_fifo + 0x114) & BIT(17))) return -EBUSY; + if (cpu_no_auto) { + ret=regmap_update_bits(session_pmgr,port?0x3f8:0x3e0,BIT(28),0); + if(ret)return ret; + ret=regmap_read_poll_timeout(session_pmgr,port?0x3f8:0x3e0,state, + !(state&BIT(28)) && (state&0xff)==0xff,1,1000); + if(ret)return ret; + pr_info("m3_dcpext_session: diagnostic CPU auto-gating disabled PMGR=%#x\n",state); + } + writel(BIT(4), session_cpu + 0x44); + if (readl(session_cpu + 0x44) != BIT(4)) return -EIO; + pr_info("m3_dcpext_session: CPU RUN set; firmware vector unchanged\n"); + session.defer_iop_quiesce = quiesce_first; + if (afk_endpoint) dir = debugfs_create_dir(port ? "m3_dcpext_session1" : "m3_dcpext_session", NULL); + if(dir)audio_debugfs(dir); + early_at = session_now(NULL) + 2000; + if(iboot_modes || native_startup){session.duration_ms=16000;session.message_limit=512;} + ret = dcpext_session_begin(&session, &ops, NULL); + mutex_lock(&native_route_lock); + while (!ret && session.phase != DCPEXT_QUIESCED) { + if(native_desktop && native_kms_ready && !native_kms_finished && + !READ_ONCE(desktop_stop) && session_now(NULL)>=desktop_hpd_next){ + int level=desktop_read_hpd(); + desktop_hpd_next=session_now(NULL)+200; + if(level<0){ret=level;break;} + if(READ_ONCE(desktop_retrain) && native_kms_connected && !desktop_link_stage){ + WRITE_ONCE(desktop_retrain,false); + desktop_retrain_until=session_now(NULL)+3000; + pr_info("m3_dcpext_session: diagnostic three-second HPD-low cycle\n"); + } + if(session_now(NULL)=11){ + desktop_disconnected=true;desktop_hpd_dirty=true; + native_kms_disconnect(); + pr_info("m3_dcpext_session: HDMI disconnected; DRM retained for reconnect\n"); + } + }else{ + desktop_hpd_low_samples=0; + if(desktop_disconnected && ++desktop_hpd_high_samples>=3){ + desktop_disconnected=false;desktop_hpd_dirty=true; + pr_info("m3_dcpext_session: HDMI cable reconnected; retraining\n"); + } + } + } + if(desktop_hpd_rpc && session_now(NULL)>=desktop_hpd_deadline){ret=-ETIMEDOUT;break;} + if((desktop_hpd_dirty || desktop_link_stage) && !rpc_pending && !av_pending && + (!READ_ONCE(desktop_stop) || desktop_link_stage)){ + if(!desktop_link_stage){ + desktop_hpd_dirty=false;desktop_hpd_value=!desktop_disconnected; + /* Match the native DPTX lifecycle: low/release, then connect/request/high. + * HPD-high alone reopens AV but leaves the sink inactive after removal. */ + desktop_link_stage=desktop_hpd_value?1:4; + m3_dcpext_native_invalidate_sink(native_client); + av_edid_done=false;av_read_ready=false;native_frame_submitted=false; + } + /* USB-C discovery needs its AUX route before request/HPD, including + * reconnect at main-link rate zero (same ordering as M4). */ + if(usb_c && desktop_link_stage==1){ + ret=link_xbar_up(); + if(ret==-EAGAIN){ + /* No crossbar write occurred: mux setup has not caught up with HPD. + * Return to offline polling, without losing the live firmware session. */ + desktop_disconnected=true;desktop_hpd_dirty=false;desktop_link_stage=0; + desktop_hpd_high_samples=0;ret=0; + pr_info("m3_dcpext_session: waiting for Type-C PHY before reconnect\n"); + goto runtime_link_deferred; + } + if(ret)break; + } + desktop_hpd_rpc=true;desktop_hpd_deadline=session_now(NULL)+5000; + ret=dptx_rpc_send(desktop_link_stage==1?11:desktop_link_stage==2?6:desktop_link_stage==5?7:8); + if(ret)break; + } +runtime_link_deferred: + if(native_kms_ready && !native_kms_finished && !desktop_link_stage && (READ_ONCE(desktop_stop) || (!native_desktop && session_now(NULL)>=native_kms_until))){ + mutex_unlock(&native_route_lock);native_kms_unplug();mutex_lock(&native_route_lock); + if(session.runtime){ret=dcpext_session_leave_runtime(&session);received=0;if(ret)break;} + } + u64 message, endpoint; + if (hotplug && !early_saved && session_now(NULL) >= early_at) { + snapshot_ram(dir, 0); + early_saved = true; + if (probe_ping) { + ping_sent = true; + ret = session_send(NULL, 0, 3ULL << 52); + if (ret) break; + } + } + if (probe_nmi && !nmi_sent && ib_pending && ib_operation==6 && session_now(NULL)>=early_at+4000) { + /* AppleASCWrapV6::_generateNMI legacy fallback, 26A428 +0x1004/1014. + * Intentionally request a diagnostic FIQ after the mode command stalls. + * This trial always requires reboot; no firmware code is patched. */ + nmi_sent=true; + pr_warn("m3_dcpext_session: diagnostic ASC NMI fallback; expect firmware crash/reboot recovery\n"); + writel(0x10,session_cpu+0x1004);writel(1,session_cpu+0x1014); + } + ret = dcpext_session_poll(&session); + if (ret) break; + if (session.phase == DCPEXT_RUNNING) { + if(native_query && !native_phase){ret=native_query_start();if(ret)break;} + if(aux_endpoint && !aux_afk.started){ + ret=afkc_start(&aux_afk,&aux_ops,NULL,aux_endpoint);if(ret)break; + } + if (afk_endpoint && !afk.started && (!aux_endpoint || (aux_afk.running && (aux_endpoint!=0x20 || log_replied)))) { + if (!(session.endpoints[1] & BIT(afk_endpoint - 32))) { ret = -ENODEV; break; } + ret = afkc_start(&afk, &afk_ops, NULL, afk_endpoint); + afk_deadline = session_now(NULL) + 1500; + if (ret) break; + } + if (validate_connection && port0_announced && !validation_sent) { + validation_sent=true; + ret = dptx_rpc_send(connect_port?11:12); + if (ret) break; + } + ret=av_progress();if(ret)break; + ret=native_progress();if(ret)break; + if(native_kms_connected && !native_kms_finished && native_kms_generation!=m3_dcpext_native_generation(native_client)){ + pr_info("m3_dcpext_native: timing publication changed; refreshing HDMI connector\n"); + native_kms_disconnect(); + } + ret=ib_poll();if(ret)break; + if (request_display && validation_sent && !rpc_pending && rpc_stage>0 && rpc_stage<(hotplug?5:3) && + (!hotplug || rpc_stage!=3 || (session_now(NULL)>=hpd_until && (!iboot_modes || ib_done) && (!native_frame || native_frame_done)))) { + /* Type-C negotiation can be stable while a waking HDMI adapter pulses + * HPD low. Prepare AUX first, then announce the actual high level. */ + int level=port && desktop_read_hpd && rpc_stage==2?desktop_read_hpd():1; + if(level<0){ret=level;break;} + if(level)ret=dptx_rpc_send(rpc_stage==1?6:hotplug && rpc_stage<4?8:7); + if(ret)break; + } + if (afk_endpoint && afk.running && !afk.stopping && session_now(NULL) >= afk_deadline && (!validate_connection || validation_reply) && (!native_query || native_phase==3)) { + ret = quiesce_first ? dcpext_session_quiesce(&session) : afkc_stop(&afk); + if (ret) break; + } + if (afk.stopped && aux_endpoint && aux_afk.running && !aux_afk.stopping && (aux_endpoint!=0x28 || !av_announced || av_closed)) { + ret=afkc_stop(&aux_afk);if(ret)break; + } + if ((!afk_endpoint || afk.stopped) && (!aux_endpoint || aux_afk.stopped)) { + pr_info("m3_dcpext_session: application endpoints stopped; requesting AP/IOP quiescence\n"); + ret = dcpext_session_quiesce(&session); + if (ret) break; + } + } + if (quiesce_first && session.phase == DCPEXT_AP_QUIESCE && session.ap_quiesced) { + if (!afk.stopping) ret = afkc_stop(&afk); + else if (afk.stopped) ret = dcpext_session_finish_quiesce(&session); + if (ret) break; + } + if (readl(session_fifo + 0x114) & BIT(17)) { + mutex_unlock(&native_route_lock); + if(session.runtime)usleep_range(1000,1500);else usleep_range(100,200); + mutex_lock(&native_route_lock); + continue; + } + message = readq(session_fifo + 0x830); + endpoint = readq(session_fifo + 0x838); + dma_rmb(); + ret=session_dispatch(endpoint,message); + mutex_unlock(&native_route_lock);cond_resched();mutex_lock(&native_route_lock); + } + mutex_unlock(&native_route_lock); + if(native_kms_ready && !native_kms_finished)native_kms_unplug(); + if (afk_endpoint) { + syslog_blob.data=syslog_capture.capture;syslog_blob.size=syslog_capture.used; + debugfs_create_blob("syslog-capture",0400,dir,&syslog_blob); + if(native_startup)debugfs_create_file("native-capture",0400,dir,NULL,&native_capture_fops); + if(aux_endpoint)debugfs_create_file("aux-capture",0400,dir,NULL,&aux_fops); + debugfs_create_file("session-pool",0400,dir,NULL,&pool_fops); + if(hotplug)snapshot_ram(dir, 1); + debugfs_create_file("afk-capture", 0400, dir, NULL, &afk_fops); + pr_info("m3_dcpext_session: AFK running=%u stopped=%u error=%d captured=%u\n", + afk.running, afk.stopped, afk.error, afk.capture_size); + } + pr_info("m3_dcpext_session: result=%d phase=%u messages=%u; retaining all resources until reboot\n", + ret, session.phase, session.received); + return ret; +} + +static int session_thread(void *unused) +{ + int ret=session_run(); + desktop_notify(ret);WRITE_ONCE(desktop_done,true);complete_all(&desktop_finished); + return ret; +} +static int desktop_reboot(struct notifier_block *nb,unsigned long event,void *data) +{ + WRITE_ONCE(desktop_stop,true); + if(!wait_for_completion_timeout(&desktop_finished,msecs_to_jiffies(20000))) + pr_err("m3_dcpext: shutdown timed out; all DMA retained\n"); + return NOTIFY_DONE; +} +static struct notifier_block desktop_reboot_nb={.notifier_call=desktop_reboot}; +static int session_start(void) +{ + struct task_struct *task;int ret; + if(!native_desktop)return session_run(); + ret=register_reboot_notifier(&desktop_reboot_nb);if(ret)return ret; + task=kthread_run(session_thread,NULL,"m3-hdmi"); + if(IS_ERR(task)){unregister_reboot_notifier(&desktop_reboot_nb);return PTR_ERR(task);} + if(!wait_for_completion_timeout(&desktop_started,msecs_to_jiffies(20000))){ + WRITE_ONCE(desktop_stop,true);return -ETIMEDOUT; + } + return READ_ONCE(desktop_result); +} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_epic_tx.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_epic_tx.h new file mode 100644 index 0000000000000..4879e0db722ff --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_epic_tx.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* M4 AFK ring admission with old EPIC channel/kind queue fields. */ +/* Wrap using duplicate queue headers, as in Sven Peter's Asahi afk_send_epic: + * one at the tail and one before the complete payload at offset zero. Never + * split a payload or overwrite unread blocks. Keep one block empty so equal + * pointers mean empty. All admission checks precede shared-memory writes; + * published packets stay mapped even if the mailbox notification fails. + */ +static int m3_epic_ring_write(struct afkc_ring *r, u32 channel, u32 kind, + const u8 *data, u32 bytes, u32 *published) +{ + u32 rd, wr, total, next; + u8 *qe; + bool wrap; + + if (!r || !r->header || !r->data || !data || !published || + r->block < 64 || r->block > 256 || (r->block & (r->block - 1)) || + r->size < r->block || (r->size & (r->block - 1)) || + bytes < 24 || bytes > 1024) + return -EINVAL; + total = ALIGN(bytes + 16, r->block); + rd = READ_ONCE(*(u32 *)(r->header + r->block)); + wr = READ_ONCE(*(u32 *)(r->header + 2 * r->block)); + dma_rmb(); + if (rd >= r->size || wr >= r->size || (rd | wr) & (r->block - 1)) + return -EPROTO; + wrap = total > r->size - wr; + if (total >= r->size || (wrap && (wr < rd || total >= rd)) || + (wr < rd && total >= rd - wr)) + return -ENOSPC; + next = wrap ? total : wr + total; + if (next == r->size) + next = 0; + if (next == rd) + return -ENOSPC; + qe = r->data + wr; + if (wrap) { + memset(qe, 0, 16); + put_unaligned_le32(0x20504f49, qe); + put_unaligned_le32(bytes, qe + 4); + put_unaligned_le32(channel,qe+8);put_unaligned_le32(kind,qe+12); + qe = r->data; + } + memset(qe, 0, total); + put_unaligned_le32(0x20504f49, qe); + put_unaligned_le32(bytes, qe + 4); + put_unaligned_le32(channel,qe+8);put_unaligned_le32(kind,qe+12); + memcpy(qe + 16, data, bytes); + dma_wmb(); + WRITE_ONCE(*(u32 *)(r->header + 2 * r->block), next); + dma_wmb(); + *published = next; + return 0; +} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_bridge.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_bridge.c new file mode 100644 index 0000000000000..0912b258189ee --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_bridge.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* J514S native HDMI bridge power/HPD diagnostic. GPIO API only. + * ADT dp2hdmi-gpio0: gP0f HDMI power, gP1a converter power; AOP GPIO47 HPD. + * Native AppleHDMIPortController prefers converter power over reset when both + * exist. Do not toggle the reset, CEC or force-DFU lines. Retain ownership. + */ +#include +#include +#include +#include +#include +#include +static bool power; +module_param(power, bool, 0400); +static int result = -EINPROGRESS; +module_param(result, int, 0400); +static int hpd = -1; +module_param(hpd, int, 0400); +static struct device *owner; +static struct gpio_desc *port, *bridge, *detect; +/* The session owns firmware notifications; this module owns the GPIO. */ +int m3_hdmi_bridge_hpd(void); +int m3_hdmi_bridge_hpd(void) +{ + if (result || IS_ERR_OR_NULL(detect)) return -ENODEV; + return gpiod_get_value_cansleep(detect); +} +EXPORT_SYMBOL_GPL(m3_hdmi_bridge_hpd); +static struct gpiod_lookup_table lines = { + .dev_id = "m3-hdmi-bridge", + .table = { + GPIO_LOOKUP("macsmc-pmu-gpio", 15, "port-power", GPIO_ACTIVE_HIGH), + GPIO_LOOKUP("macsmc-pmu-gpio", 26, "bridge-power", GPIO_ACTIVE_HIGH), + GPIO_LOOKUP("374824000.pinctrl", 47, "hpd", GPIO_ACTIVE_HIGH), + { } + } +}; +static int __init probe(void) +{ + int ret, p, b; + if (!of_machine_is_compatible("apple,j514s") || + !of_machine_is_compatible("apple,t6030")) return -ENODEV; + owner = root_device_register("m3-hdmi-bridge"); + if (IS_ERR(owner)) return PTR_ERR(owner); + gpiod_add_lookup_table(&lines); + port = gpiod_get(owner, "port-power", GPIOD_ASIS); + if (IS_ERR(port)) {ret=PTR_ERR(port);goto done;} + bridge = gpiod_get(owner, "bridge-power", GPIOD_ASIS); + if (IS_ERR(bridge)) {ret=PTR_ERR(bridge);goto done;} + detect = gpiod_get(owner, "hpd", GPIOD_IN); + if (IS_ERR(detect)) {ret=PTR_ERR(detect);goto done;} + if (gpiod_get_direction(port) != 0 || gpiod_get_direction(bridge) != 0) { + ret=-EINVAL;goto done; + } + p=gpiod_get_value_cansleep(port);b=gpiod_get_value_cansleep(bridge); + hpd=gpiod_get_value_cansleep(detect); + pr_info("m3_hdmi_bridge: before port=%d bridge=%d hpd=%d\n",p,b,hpd); + if (p<0 || b<0 || hpd<0) {ret=-EIO;goto done;} + if (power) { + ret=gpiod_set_value_cansleep(port, 1);if(ret)goto done; + ret=gpiod_set_value_cansleep(bridge, 1);if(ret)goto done; + msleep(500); + /* The converter can pulse HPD low while starting. Wait for the real + * connector signal, rather than injecting HPD high into an absent sink. */ + for(unsigned int sample=0,stable=0;sample<100;sample++){ + hpd=gpiod_get_value_cansleep(detect); + if(hpd<0){ret=hpd;goto done;} + stable=hpd?stable+1:0; + if(stable==5)break; + msleep(50); + if(sample==99){ret=-ENODEV;goto done;} + } + p=gpiod_get_value_cansleep(port);b=gpiod_get_value_cansleep(bridge); + hpd=gpiod_get_value_cansleep(detect); + pr_info("m3_hdmi_bridge: powered port=%d bridge=%d hpd=%d\n",p,b,hpd); + } + ret=(p<0 || b<0 || hpd<0 || (power && (p!=1 || b!=1))) ? -EIO : 0; +done: + result=ret; + pr_info("m3_hdmi_bridge: result=%d power=%u; GPIO ownership retained until reboot\n",ret,power); + return 0; +} +module_init(probe); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("J514S native HDMI converter power and HPD diagnostic"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_control.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_control.h new file mode 100644 index 0000000000000..93033bed047ae --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_control.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef M3_HDMI_CONTROL_H +#define M3_HDMI_CONTROL_H +bool m3_hdmi_phy_ready(void); +int m3_hdmi_phy_deactivate(void); +int m3_hdmi_phy_activate(void); +int m3_hdmi_phy_set_rate(unsigned int rate); +int m3_hdmi_phy_get_rate(void); +int m3_hdmi_phy_set_drive(const unsigned int drive[12]); +int m3_hdmi_xbar_activate(void); +int m3_hdmi_xbar_deactivate(void); +struct m3_usbc_route_ops { + int (*prepare)(unsigned int slot); + int (*hpd)(unsigned int slot); + int (*target)(unsigned int slot); + int (*activate)(unsigned int slot); + int (*deactivate)(unsigned int slot); + int (*xbar_up)(unsigned int slot); + int (*xbar_down)(unsigned int slot); + int (*set_rate)(unsigned int slot, unsigned int rate); + int (*get_rate)(unsigned int slot); + int (*set_drive)(unsigned int slot, const unsigned int drive[12]); +}; +const struct m3_usbc_route_ops *m3_usbc_get_ops(void); +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy.c new file mode 100644 index 0000000000000..53d13e56a92a0 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy.c @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* J514S HDMI ATC3 power, common/AUX and standalone main-link diagnostic. + * Addresses: J514S ADT atc-phy3, native 26A428 descriptor capture and T6030 DT. + * Retain genpd/device/MMIO ownership until reboot, including on failure. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "m3_hdmi_control.h" +#include "m3_hdmi_phy_power.h" +#include "m3_hdmi_phy_common.h" +#include "m3_hdmi_phy_dp-main.h" +#include "m3_hdmi_phy_dp-drive.h" +static int result = -EINPROGRESS; +module_param(result, int, 0400); +static bool roundtrip; +module_param(roundtrip, bool, 0400); +static unsigned int main_rate; +module_param(main_rate, uint, 0400); +static bool common; +module_param(common, bool, 0400); +static unsigned int calibration[12]; +static unsigned int calibration_count; +module_param_array(calibration, uint, &calibration_count, 0400); +static const u32 offsets[] = {0x800, 0x2224, 0x2230, 0x2240, 0x22b0, 0x22e4, + 0x2334, 0x2248, 0x16008, 0x1600c, 0xa04, 0x260c}; +static const u32 masks[] = {0x3f0000, 0xc00, 0x1f, 0x18007f, 0x8700000, 0x60001, + 0xf800000, 0x1ff00, 0x38, 0x1f, 0x7c, 0x3ff}; +static struct device *owner; +static void __iomem *core; +static bool phy_active; +bool m3_hdmi_phy_ready(void) +{ + return result==0 && core && phy_active && (readl(core+0x804)&1) && + (!!(readl(core+0x7034)&8)==!!main_rate) && readl(core+0x60)==0x492; +} +EXPORT_SYMBOL_GPL(m3_hdmi_phy_ready); +int m3_hdmi_phy_deactivate(void) +{ + int ret; + if(!m3_hdmi_phy_ready())return -EHOSTDOWN; + ret=main_rate?m3_hdmi_atc_dp_main_stop(core,3):0; + if(ret)return ret; + m3_hdmi_atc_aux_stop(core); + m3_hdmi_atc_common_stop(core); + ret=m3_hdmi_atc_power_down(core); + pr_info("m3_hdmi_phy: deactivation result=%d\n",ret); + if(!ret){main_rate=0;phy_active=false;} + return ret; +} +EXPORT_SYMBOL_GPL(m3_hdmi_phy_deactivate); +int m3_hdmi_phy_get_rate(void) +{ + return m3_hdmi_phy_ready()?main_rate:-EHOSTDOWN; +} +EXPORT_SYMBOL_GPL(m3_hdmi_phy_get_rate); +int m3_hdmi_phy_set_rate(unsigned int rate) +{ + int ret; + if(rate!=0 && rate!=6 && rate!=10 && rate!=20)return -EINVAL; + if(!m3_hdmi_phy_ready())return -EHOSTDOWN; + if(rate==main_rate)return 0; + ret=main_rate?m3_hdmi_atc_dp_main_stop(core,3):0; + if(!ret && rate)ret=m3_hdmi_atc_dp_main_prepare(core,3,rate,false); + if(ret)result=ret;else main_rate=rate; + pr_info("m3_hdmi_phy: set rate=%u result=%d PLL=%#x\n",rate,ret,readl(core+0x7034)); + return ret; +} +EXPORT_SYMBOL_GPL(m3_hdmi_phy_set_rate); +int m3_hdmi_phy_set_drive(const unsigned int drive[12]) +{ + /* 26A428 AppleT8122TypeCPhy fallback at VA0xfffffe000783cac0. + * Captured J514S ATC3 ADT has no training-table overrides. */ + static const u32 table[16]={0x3f024,0x237013,0x37200b,0x46e004, + 0x3f01c,0x23700b,0x372004,0x3f000,0x3f013,0x237004,0x3f000,0x3f000, + 0x3f004,0x3f000,0x3f000,0x3f000}; + if(!m3_hdmi_phy_ready() || !main_rate)return -EHOSTDOWN; + for(unsigned int i=0;i<12;i+=3) + if(drive[i] || drive[i+1]>3 || drive[i+2]>3 || drive[i+1]+drive[i+2]>3)return -EINVAL; + /* Until native four-lane orientation order is qualified, require equal + * settings on each pair. This makes both possible pair orders identical. */ + if(memcmp(drive,drive+6,6*sizeof(*drive)))return -EOPNOTSUPP; + for(unsigned int i=0;i<4;i++){ + u32 preset=table[drive[3*i+1]*4+drive[3*i+2]]; + int ret=m3_hdmi_atc_dp_drive_preset(core,i/2,i&1,preset); + if(ret){result=ret;return ret;} + pr_info("m3_hdmi_phy: drive lane=%u voltage=%u emphasis=%u preset=%#x\n",i,drive[3*i+1],drive[3*i+2],preset); + } + return 0; +} +EXPORT_SYMBOL_GPL(m3_hdmi_phy_set_drive); +static int phy_start(bool lanes) +{ + int ret=m3_hdmi_atc_power_up_reset(core); + if(ret)return ret; + for (unsigned int i = 0; i < ARRAY_SIZE(offsets); i++) { + m3_hdmi_atc_component_update(core, offsets[i], masks[i], calibration[i]); + if ((readl(core + offsets[i]) & masks[i]) != calibration[i]) { ret = -EIO; return ret; } + } + if (lanes) { + /* Native mode3 is four-lane DP, fixed HDMI orientation. */ + m3_hdmi_atc_component_update(core, 0x60, 0xfff, 0x492); + m3_hdmi_atc_component_update(core, 0x64, 0x1f, 0x14); + m3_hdmi_atc_component_update(core, 0x64, 0x1ffe0, 0x22000); + } + /* Native phy_init_partC releases bit4, waits100us, then cmn_init. */ + udelay(10); + m3_hdmi_atc_component_update(core, 0x20000, 0, BIT(4)); + udelay(100); + ret = m3_hdmi_atc_common_start(core); + if (ret) return ret; + m3_hdmi_atc_aux_start(core); + if ((readl(core + 0x16400) & 0x30f) != 0x10f || (readl(core + 0x16000) & 1)) ret = -EIO; + pr_info("m3_hdmi_phy: common ready=%#x AUX ctrl=%#x cfg=%#x result=%d\n", + readl(core + 0x804), readl(core + 0x16400), readl(core + 0x16000), ret); + if (!ret && main_rate) { + ret = m3_hdmi_atc_dp_main_prepare(core, 3, main_rate, false); + pr_info("m3_hdmi_phy: main rate=%u result=%d request=%#x PLL=%#x lanes=%#x xbar=%#x\n", + main_rate, ret, readl(core + 0x2000), readl(core + 0x7034), + readl(core + 0x60), readl(core + 0x64)); + } + return ret; +} +int m3_hdmi_phy_activate(void) +{ + int ret; + if(result || !core || !common)return -EHOSTDOWN; + if(phy_active)return m3_hdmi_phy_ready()?0:-EIO; + /* DPTX will choose the link rate; restore common/AUX and four-lane mode. */ + main_rate=0; + ret=phy_start(true); + if(!ret)phy_active=true;else result=ret; + pr_info("m3_hdmi_phy: reactivation result=%d\n",ret); + return ret; +} +EXPORT_SYMBOL_GPL(m3_hdmi_phy_activate); +static int __init probe(void) +{ + struct of_phandle_args args = {}; + const char *label; + int ret; + if (main_rate && (!common || main_rate != 10)) return -EINVAL; + if (common) { + if (roundtrip || calibration_count != ARRAY_SIZE(offsets)) return -EINVAL; + for (unsigned int i = 0; i < ARRAY_SIZE(offsets); i++) + if (calibration[i] & ~masks[i]) return -EINVAL; + } + if (!of_machine_is_compatible("apple,j514s") || + !of_machine_is_compatible("apple,t6030")) return -ENODEV; + args.np = of_find_node_by_path("/soc/power-management@350700000/power-controller@4f8"); + if (!args.np) return -ENODEV; + if (of_property_read_string(args.np, "label", &label) || strcmp(label, "atc3_common")) { + of_node_put(args.np); return -EINVAL; + } + owner = root_device_register("m3-hdmi-phy"); + if (IS_ERR(owner)) { of_node_put(args.np); return PTR_ERR(owner); } + ret = of_genpd_add_device(&args, owner); + of_node_put(args.np); + if (ret) { root_device_unregister(owner); return ret; } + pm_runtime_set_suspended(owner); + pm_runtime_enable(owner); + ret = pm_runtime_resume_and_get(owner); + if (ret < 0) goto done; + if (!request_mem_region(0x1303000000ULL, 0x30000, "m3-hdmi-phy")) { + ret = -EBUSY; goto done; + } + core = ioremap_np(0x1303000000ULL, 0x30000); + if (!core) { ret = -ENOMEM; goto done; } + pr_info("m3_hdmi_phy: ATC3 ctrl=%#x status=%#x misc=%#x\n", + readl(core + 0x20000), readl(core + 0x20004), readl(core + 0x20008)); + ret = m3_hdmi_atc_power_idle(core); + if (ret || (!roundtrip && !common)) goto done; + if(roundtrip)ret = m3_hdmi_atc_power_up_reset(core); + if (!ret && roundtrip) ret = m3_hdmi_atc_power_down(core); + if (!ret && common) { + ret=phy_start(!!main_rate); + } +done: + phy_active=!ret && common; + result = ret; + pr_info("m3_hdmi_phy: result=%d roundtrip=%u; resources retained until reboot\n", ret, roundtrip); + return 0; +} +module_init(probe); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("J514S HDMI ATC3 power and common/AUX diagnostic"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_common.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_common.h new file mode 100644 index 0000000000000..6b2eb453e9c50 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_common.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Basic common/AUX sequence agrees with exact M3 26A428 Unicorn traces. + * Shared M4 implementation copied after comparison, not its calibrations. + * Main-link and lane routing are deliberately absent at this stage. + */ +static void m3_hdmi_atc_component_update(void __iomem *regs, u32 offset, u32 clear, u32 set) +{ + writel((readl(regs + offset) & ~clear) | set, regs + offset); +} +static int m3_hdmi_atc_common_start(void __iomem *regs) +{ + u32 status; + m3_hdmi_atc_component_update(regs, 8, 0, 4); + m3_hdmi_atc_component_update(regs, 8, 0, 8); + udelay(1); + m3_hdmi_atc_component_update(regs, 8, 0, 1); + m3_hdmi_atc_component_update(regs, 8, 0, 2); + udelay(1); + m3_hdmi_atc_component_update(regs, 8, 0x10, 0); + m3_hdmi_atc_component_update(regs, 8, 0, 0x20); + udelay(1); + m3_hdmi_atc_component_update(regs, 0xa00, 0, 2); + udelay(15); + return readl_poll_timeout(regs + 0x804, status, status & 1, 1, 100); +} + +#define ATC_AUX_CTRL 0x16400 +#define ATC_AUX_CFG 0x16000 + +static void m3_hdmi_atc_aux_start(void __iomem *regs) +{ + m3_hdmi_atc_component_update(regs, ATC_AUX_CTRL, 0, 3); + udelay(1); + m3_hdmi_atc_component_update(regs, ATC_AUX_CTRL, 0, 0xc); + udelay(1); + m3_hdmi_atc_component_update(regs, ATC_AUX_CTRL, 0x300, 0x100); + udelay(1); + m3_hdmi_atc_component_update(regs, ATC_AUX_CFG, 1, 0); + udelay(1); +} + + +static void m3_hdmi_atc_common_stop(void __iomem *regs) +{ + m3_hdmi_atc_component_update(regs, 0xa00, 2, 0); + udelay(1); + m3_hdmi_atc_component_update(regs, 8, 0, 0x10); + udelay(1); + m3_hdmi_atc_component_update(regs, 8, 1, 0); + udelay(1); + m3_hdmi_atc_component_update(regs, 8, 4, 0); + udelay(1); +} + +static void m3_hdmi_atc_aux_stop(void __iomem *regs) +{ + m3_hdmi_atc_component_update(regs, ATC_AUX_CFG, 0, 1); + udelay(1); + m3_hdmi_atc_component_update(regs, ATC_AUX_CTRL, 0, 0x200); + udelay(1); + m3_hdmi_atc_component_update(regs, ATC_AUX_CTRL, 8, 0); + udelay(1); + m3_hdmi_atc_component_update(regs, ATC_AUX_CTRL, 2, 0); + udelay(1); +} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-drive.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-drive.h new file mode 100644 index 0000000000000..9eb48d09c1330 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-drive.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Shared per-device sequencer: caller owns MMIO, power and device lock. + * Independent packed DP drive-preset register operations. Native behavioral + * reference: AppleM3TypeCPhy::aciophy_dptx_set_{txeq,rxtxeq}_preset, + * 26A428 image; boolean argument false enables the packed preset. Offline qualified only; + * the standalone HBR test used the fixed initial preset, not sink training. + * Caller supplies a qualified 24-bit preset and owns the selected PHY pair. + */ +#ifndef APPLE_ATCPHY_M3_DP_DRIVE_H +#define APPLE_ATCPHY_M3_DP_DRIVE_H + +static int m3_hdmi_atc_dp_drive_preset(void __iomem *regs, unsigned int lane, bool tx, u32 packed) +{ + u32 select, amplitude, base; + if (lane > 1 || packed & 0xff000000) + return -EINVAL; + base = lane * 0x7000; + select = ((packed & 3) << 14) | ((packed >> 2) & 0xf); + amplitude = (((packed >> 6) & 0x3ffff) << 1) | 1; + if (tx) { + m3_hdmi_dp_pll_update(regs, base + 0xc0d0, 0x100000, 0x100000); + m3_hdmi_dp_pll_update(regs, base + 0xc050, 0xc00f, select); + m3_hdmi_dp_pll_update(regs, base + 0xc0d4, 0x10000000, 0x10000000); + m3_hdmi_dp_pll_update(regs, base + 0xc0d0, 0x7ffff, amplitude); + } else { + m3_hdmi_dp_pll_update(regs, base + 0x9210, 0xc00f, select); + m3_hdmi_dp_pll_update(regs, base + 0x9248, 0x10000000, 0x10000000); + m3_hdmi_dp_pll_update(regs, base + 0x9244, 0x7ffff, amplitude); + } + return 0; +} +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-main.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-main.h new file mode 100644 index 0000000000000..0b9c66d47c621 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-main.h @@ -0,0 +1,159 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Adapted from the M4 sequencer, checked against J514S 26A428 native + * main_init and its callees. Experimental; no trained link claim. + */ +#ifndef APPLE_ATCPHY_M3_DP_MAIN_H +#define APPLE_ATCPHY_M3_DP_MAIN_H +#include "m3_hdmi_phy_dp-pll.h" +#include "m3_hdmi_phy_lane-power.h" + +static void m3_hdmi_dp_lane_update(void __iomem *regs, unsigned int lane, u32 offset, u32 mask, u32 value) +{ + m3_hdmi_dp_pll_update(regs, offset + lane * 0x7000, mask, value); +} + +static void m3_hdmi_dp_lane_tx_initial_eq(void __iomem *regs, unsigned int lane) +{ + m3_hdmi_dp_lane_update(regs, lane, 0xc050, 0xc00f, 0); + m3_hdmi_dp_lane_update(regs, lane, 0xc0d4, 0x10000000, 0x10000000); + m3_hdmi_dp_lane_update(regs, lane, 0xc0d0, 0x7ffff, 1); +} + +static void m3_hdmi_dp_lane_rx_initial_eq(void __iomem *regs, unsigned int lane) +{ + m3_hdmi_dp_lane_update(regs, lane, 0x9210, 0xc00f, 0); + m3_hdmi_dp_lane_update(regs, lane, 0x9248, 0x10000000, 0x10000000); + m3_hdmi_dp_lane_update(regs, lane, 0x9244, 0x7ffff, 0x1f81); +} + +static void m3_hdmi_dp_lane_tx_prepare(void __iomem *regs, unsigned int lane, unsigned int rate) +{ + m3_hdmi_atc_lane_tx_power(regs, lane, true); + writel(0x0aafb800, regs + 0xd044 + lane * 0x7000); + ndelay(500); + m3_hdmi_dp_lane_update(regs, lane, 0xd044, 0x1000000, 0x1000000); + ndelay(250); + m3_hdmi_dp_lane_update(regs, lane, 0xd044, 0x4000000, 0x4000000); + ndelay(250); + m3_hdmi_dp_lane_update(regs, lane, 0xd044, 0x7c0, 0x300); + ndelay(250); + m3_hdmi_dp_lane_update(regs, lane, 0xd044, 0x6c000, 0); + ndelay(250); + m3_hdmi_dp_lane_update(regs, lane, 0xc0d0, 0x100000, 0x100000); + m3_hdmi_dp_lane_tx_initial_eq(regs, lane); + m3_hdmi_dp_lane_update(regs, lane, 0xd044, 0x100000, 0x100000); + ndelay(250); + m3_hdmi_dp_lane_update(regs, lane, 0xd044, 0x400000, 0x400000); + udelay(1); + m3_hdmi_dp_lane_update(regs, lane, 0xd004, 0x3000000, 0x2000000); + m3_hdmi_dp_lane_update(regs, lane, 0xd004, 0x30000000, 0x30000000); + m3_hdmi_dp_lane_update(regs, lane, 0xd00c, 0x7000000, rate == 6 ? 0x5000000 : 0x4000000); + m3_hdmi_dp_lane_update(regs, lane, 0xd010, 0x600, 0x600); + m3_hdmi_dp_lane_update(regs, lane, 0xd01c, 0xc00000, 0x800000); +} + +static void m3_hdmi_dp_lane_rx_prepare(void __iomem *regs, unsigned int lane, unsigned int rate) +{ + m3_hdmi_dp_lane_update(regs, lane, 0x9160, 1, 1); + m3_hdmi_dp_lane_update(regs, lane, 0x91dc, 0x18000, 0x18000); + m3_hdmi_dp_lane_update(regs, lane, 0xb0a4, 3, 2); + m3_hdmi_dp_lane_update(regs, lane, 0xb0a8, 0x18000, 0x10000); + m3_hdmi_dp_lane_update(regs, lane, 0xb06c, 0x18000, 0x10000); + m3_hdmi_dp_lane_update(regs, lane, 0xb0d4, 0x1f1f80, 0x121180); + m3_hdmi_dp_lane_update(regs, lane, 0xb064, 0xfc00, 0xb800); + m3_hdmi_dp_lane_update(regs, lane, 0xb0d8, 7, 4); + m3_hdmi_dp_lane_update(regs, lane, 0xb0dc, 0x18, 0); + m3_hdmi_dp_lane_update(regs, lane, 0xb0d4, 0xe000, 0xc000); + ndelay(100); + m3_hdmi_atc_lane_rx_power(regs, lane, true); + m3_hdmi_dp_lane_update(regs, lane, 0xb06c, 0x800, 0x800); + m3_hdmi_dp_lane_update(regs, lane, 0x91d4, 3, 1); + ndelay(10); + m3_hdmi_dp_lane_update(regs, lane, 0x91d4, 2, 2); + m3_hdmi_dp_lane_update(regs, lane, 0xb06c, 0x600, 0xc00); + ndelay(100); + m3_hdmi_dp_lane_update(regs, lane, 0xb0d4, 0x50000, 0x50000); + m3_hdmi_dp_lane_update(regs, lane, 0xb0d8, 1, 1); + ndelay(500); + m3_hdmi_dp_lane_update(regs, lane, 0xb0d4, 0x40000, 0); + ndelay(200); + m3_hdmi_dp_lane_update(regs, lane, 0xb068, 0x180, 0x180); + ndelay(200); + m3_hdmi_dp_lane_update(regs, lane, 0xb068, 0x60, 0x60); + ndelay(200); + m3_hdmi_dp_lane_update(regs, lane, 0xb0d8, 3, 3); + udelay(1); + m3_hdmi_dp_lane_update(regs, lane, 0xb0d8, 3, 2); + udelay(1); + m3_hdmi_dp_lane_update(regs, lane, 0xb0d8, 3, 0); + ndelay(750); + m3_hdmi_dp_lane_update(regs, lane, 0xb0a8, 0x60000, 0x60000); + udelay(1); + m3_hdmi_dp_lane_update(regs, lane, 0xb06c, 0x6000, 0x4000); + m3_hdmi_dp_lane_rx_initial_eq(regs, lane); + m3_hdmi_dp_lane_update(regs, lane, 0xb0a8, 0x380000, rate == 6 ? 0x280000 : 0x200000); + m3_hdmi_dp_lane_update(regs, lane, 0xb0a8, 0xc00000, 0x800000); + ndelay(50); + m3_hdmi_dp_lane_update(regs, lane, 0xb030, 0xc00, 0xc00); + ndelay(100); +} + +static int m3_hdmi_atc_dp_main_prepare(void __iomem *regs, unsigned int pair_mask, unsigned int rate, bool ssc) +{ + int ret; + if ((!pair_mask || pair_mask > 3) || + (rate != 6 && rate != 10 && rate != 20) || ssc) + return -EOPNOTSUPP; + ret = m3_hdmi_dp_pll_prepare(regs, pair_mask, rate, ssc); + if (ret) + return ret; + for (unsigned int lane = 0; lane < 2; lane++) { + if (!(pair_mask & (1u << lane))) continue; + m3_hdmi_dp_lane_tx_prepare(regs, lane, rate); + m3_hdmi_dp_lane_rx_prepare(regs, lane, rate); + } + m3_hdmi_dp_pll_update(regs, 0x7000, 1, 0); + return 0; +} + +static int m3_hdmi_atc_dp_main_stop(void __iomem *regs, unsigned int pair_mask) +{ + unsigned int lane; + if (!pair_mask || pair_mask > 3) + return -EOPNOTSUPP; + m3_hdmi_dp_pll_update(regs, 0x7000, 1, 1); + for(lane=0;lane<2;lane++){ + if(!(pair_mask & (1u<> 1) & 1) == expected, 1, timeout_us); +} + +/* Capture the original ACK for both phases, not the current request bit. */ +static u32 m3_hdmi_dp_pll_request_begin(void __iomem *regs) +{ + u32 old = readl(regs + DP_PLL_REQUEST); + u32 ack = (old >> 1) & 1; + writel((old & 0xe0000006) | 0x10000000 | (ack ^ 1), + regs + DP_PLL_REQUEST); + return ack; +} + +static int m3_hdmi_dp_pll_prepare(void __iomem *regs, unsigned int pair_mask, unsigned int rate, bool ssc) +{ + u32 freq_a, freq_b, freq_c, ack, status; + int ret; + /* Spread spectrum is outside this diagnostic. */ + if ((!pair_mask || pair_mask > 3) || ssc) + return -EOPNOTSUPP; + switch (rate) { + case 0x06: freq_a = 0x1e0e021c; freq_b = 0; freq_c = 0x156600; break; + case 0x0a: freq_a = 0x1e0e01c2; freq_b = 0x07fffffe; freq_c = 0x155200; break; + case 0x14: freq_a = 0x1e0e01c2; freq_b = 0x07fffffe; freq_c = 0x554800; break; + case 0x1e: freq_a = 0x1e0e02a3; freq_b = 0x0bff7ffc; freq_c = 0x564800; break; + default: return -EOPNOTSUPP; + } + m3_hdmi_dp_pll_update(regs, 0x2234, 3, 0); + m3_hdmi_dp_pll_update(regs, 0x7000, 0x1ff0, 0x490); + writel(freq_a, regs + 0x2080); + writel(freq_b, regs + 0x2084); + writel(freq_c, regs + 0x2088); + m3_hdmi_dp_pll_update(regs, 0x2208, 0x1f0000, 0x70000); + m3_hdmi_dp_pll_update(regs, 0x2218, 1, 1); + m3_hdmi_dp_pll_update(regs, 0x2200, 8, 8); + ack = m3_hdmi_dp_pll_request_begin(regs); + ret = m3_hdmi_dp_pll_ack(regs, ack ^ 1, 30); + if (ret) + return ret; + ret = readl_poll_timeout(regs + DP_PLL_STATUS, status, + status & 8, 1, 20); + if (ret) + return ret; + m3_hdmi_dp_pll_update(regs, DP_PLL_REQUEST, 0x1ffffff9, 0x10014000 | ack); + ret = m3_hdmi_dp_pll_ack(regs, ack, 5); + if (ret) + return ret; + for (unsigned int lane = 0; lane < 2; lane++) + if (pair_mask & (1u << lane)) + m3_hdmi_dp_pll_update(regs, 0xb064 + lane * 0x7000, 0xc, 8); + m3_hdmi_dp_pll_update(regs, 0x7000, 0xc, 0xc); + udelay(1); + return 0; +} + +static int m3_hdmi_dp_pll_stop_after_lanes(void __iomem *regs) +{ + u32 ack, status; + int ret; + m3_hdmi_dp_pll_update(regs, 0x7000, 0xc, 8); + ack = m3_hdmi_dp_pll_request_begin(regs); + ret = m3_hdmi_dp_pll_ack(regs, ack ^ 1, 500); + if (ret) + return ret; + m3_hdmi_dp_pll_update(regs, DP_PLL_REQUEST, 0x1ffffff9, 0x10000018 | ack); + ret = m3_hdmi_dp_pll_ack(regs, ack, 500); + if (ret) + return ret; + ret = readl_poll_timeout(regs + DP_PLL_STATUS, status, + !(status & 8), 1, 5); + if (ret) + return ret; + m3_hdmi_dp_pll_update(regs, 0x2200, 8, 0); + return 0; +} +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_lane-power.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_lane-power.h new file mode 100644 index 0000000000000..c58be84d3a9a6 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_lane-power.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Adapted from the M4 sequencer, checked against J514S 26A428 native + * main_init and its callees. Experimental; no trained link claim. + */ +#ifndef APPLE_ATCPHY_M3_LANE_POWER_H +#define APPLE_ATCPHY_M3_LANE_POWER_H + +/* M3 uses a single read per selected-lane bit update. */ +static void m3_hdmi_atc_lane_power_field(void __iomem *regs, u32 offset, + unsigned int shift, unsigned int lane, bool enable) +{ + u32 value = readl(regs + offset), bit = 1u << (shift + lane); + writel(enable ? value | bit : value & ~bit, regs + offset); +} + +static int m3_hdmi_atc_lane_tx_power(void __iomem *regs, unsigned int lane, bool enable) +{ + if (lane > 1) + return -EINVAL; + if (enable) { + m3_hdmi_atc_lane_power_field(regs, 0x10c, 4, lane, true); + m3_hdmi_atc_lane_power_field(regs, 0x10c, 6, lane, true); + udelay(1); + m3_hdmi_atc_lane_power_field(regs, 0x10c, 0, lane, true); + m3_hdmi_atc_lane_power_field(regs, 0x10c, 2, lane, true); + udelay(1); + m3_hdmi_atc_lane_power_field(regs, 0x10c, 8, lane, false); + m3_hdmi_atc_lane_power_field(regs, 0x10c, 10, lane, true); + udelay(1); + } else { + m3_hdmi_atc_lane_power_field(regs, 0x10c, 8, lane, true); + udelay(1); + m3_hdmi_atc_lane_power_field(regs, 0x10c, 0, lane, false); + udelay(1); + m3_hdmi_atc_lane_power_field(regs, 0x10c, 4, lane, false); + udelay(1); + } + return 0; +} + +static int m3_hdmi_atc_lane_rx_power(void __iomem *regs, unsigned int lane, bool enable) +{ + if (lane > 1) + return -EINVAL; + if (enable) { + m3_hdmi_atc_lane_power_field(regs, 0xc, 4, lane, true); + m3_hdmi_atc_lane_power_field(regs, 0xc, 6, lane, true); + udelay(1); + m3_hdmi_atc_lane_power_field(regs, 0xc, 0, lane, true); + m3_hdmi_atc_lane_power_field(regs, 0xc, 2, lane, true); + udelay(1); + m3_hdmi_atc_lane_power_field(regs, 8, 10, lane, true); + m3_hdmi_atc_lane_power_field(regs, 8, 12, lane, true); + udelay(1); + m3_hdmi_atc_lane_power_field(regs, 8, 6, lane, true); + m3_hdmi_atc_lane_power_field(regs, 8, 8, lane, true); + udelay(1); + m3_hdmi_atc_lane_power_field(regs, 8, 14, lane, false); + m3_hdmi_atc_lane_power_field(regs, 8, 16, lane, true); + udelay(1); + } else { + m3_hdmi_atc_lane_power_field(regs, 8, 14, lane, true); + udelay(1); + m3_hdmi_atc_lane_power_field(regs, 8, 6, lane, false); + udelay(1); + m3_hdmi_atc_lane_power_field(regs, 8, 10, lane, false); + udelay(1); + m3_hdmi_atc_lane_power_field(regs, 0xc, 0, lane, false); + udelay(1); + m3_hdmi_atc_lane_power_field(regs, 0xc, 4, lane, false); + udelay(1); + } + return 0; +} +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_power.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_power.h new file mode 100644 index 0000000000000..6bef5442d7247 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_power.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef M3_HDMI_PHY_POWER_H +#define M3_HDMI_PHY_POWER_H +/* Copied from M4 power component after matching the M3 26A428 native + * powerup trace (typec-power, 2026-09-21). Register masks/order agree. + * Power/reset only; the PHY reset remains asserted throughout. + * Startup follows native AppleT8122TypeCPhy::aciophy_phy_powerup_rst. + * Reverse transition follows Sven Peter / Asahi atcphy_power_off, using + * the power/misc bits independently confirmed in the J514S ATC3 native routines. + * No USB2, tunables, lane routing, common initialization or AUX changes. + */ +static int m3_hdmi_atc_power_idle(void __iomem *regs) +{ + return readl(regs + 0x20000) == 4 && + !readl(regs + 0x20004) && !readl(regs + 0x20008) ? 0 : -EBUSY; +} + +static void m3_hdmi_atc_power_update(void __iomem *regs, u32 offset, u32 clear, u32 set) +{ + u32 value = readl(regs + offset); + writel((value & ~clear) | set, regs + offset); + udelay(1); +} + +static int m3_hdmi_atc_power_wait(void __iomem *regs, u32 mask, u32 value) +{ + u32 status; + return readl_poll_timeout(regs + 0x20004, status, + (status & mask) == value, 1, 1000); +} + +static int m3_hdmi_atc_power_up_reset(void __iomem *regs) +{ + int ret; + /* Caller has checked idle and pinned its powered device before entry. */ + ndelay(500); + m3_hdmi_atc_power_update(regs, 0x20008, 0, 1); + m3_hdmi_atc_power_update(regs, 0x20000, 0, 1); + ret = m3_hdmi_atc_power_wait(regs, 1, 1); + if (ret) + return ret; + m3_hdmi_atc_power_update(regs, 0x20000, 0, 2); + ret = m3_hdmi_atc_power_wait(regs, 2, 2); + if (ret) + return ret; + m3_hdmi_atc_power_update(regs, 0x20000, 4, 0); + m3_hdmi_atc_power_update(regs, 0x20000, 0, 8); + if (readl(regs + 0x20000) != 0xb || readl(regs + 0x20008) != 1) + return -EIO; + pr_info("m3_hdmi_phy: reset-held PHY powerup reached control=0xb status=%#x misc=1\n", + readl(regs + 0x20004)); + return 0; +} + +static int m3_hdmi_atc_power_down(void __iomem *regs) +{ + int ret; + /* Keep PHY reset asserted, restore clamp, then remove APB reset release + * and the big/small power requests in that order. No USB2 reset call. + */ + m3_hdmi_atc_power_update(regs, 0x20000, 0x10, 0); + m3_hdmi_atc_power_update(regs, 0x20000, 0, 4); + m3_hdmi_atc_power_update(regs, 0x20008, 5, 0); + m3_hdmi_atc_power_update(regs, 0x20000, 8, 0); + m3_hdmi_atc_power_update(regs, 0x20000, 2, 0); + ret = m3_hdmi_atc_power_wait(regs, 2, 0); + if (ret) + return ret; + m3_hdmi_atc_power_update(regs, 0x20000, 1, 0); + ret = m3_hdmi_atc_power_wait(regs, 1, 0); + if (ret) + return ret; + if (readl(regs + 0x20000) != 4 || readl(regs + 0x20004) || + readl(regs + 0x20008)) + return -EIO; + pr_info("m3_hdmi_phy: power roundtrip restored control=4 status=0 misc=0\n"); + return 0; /* Caller verifies PMGR before dropping the pin. */ +} + + +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_xbar.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_xbar.c new file mode 100644 index 0000000000000..c84ac3aa7ae20 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_xbar.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* J514S DCPEXT0->ATC3 crossbar. Acquire ATC3_COMMON; the caller also holds + * DCPEXT0 power. Retain all ownership until reboot, including failed writes. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "m3_hdmi_control.h" +#include "m3_hdmi_xbar.h" +static int result = -EINPROGRESS; +module_param(result, int, 0400); +static bool roundtrip; +module_param(roundtrip, bool, 0400); +static struct device *owner; +static struct m3_hdmi_dpxbar xbar; +int m3_hdmi_xbar_deactivate(void) +{ + if(result || !xbar.regs)return -EHOSTDOWN; + return m3_hdmi_dpxbar_down(&xbar); +} +EXPORT_SYMBOL_GPL(m3_hdmi_xbar_deactivate); +int m3_hdmi_xbar_activate(void) +{ + if(result || !xbar.regs)return -EHOSTDOWN; + return xbar.selected?0:m3_hdmi_dpxbar_up(&xbar); +} +EXPORT_SYMBOL_GPL(m3_hdmi_xbar_activate); +static int __init probe(void) +{ + struct of_phandle_args args = {}; + const char *label; + int ret; + if (!of_machine_is_compatible("apple,j514s") || + !of_machine_is_compatible("apple,t6030")) return -ENODEV; + args.np = of_find_node_by_path("/soc/power-management@350700000/power-controller@4f8"); + if (!args.np) return -ENODEV; + if (of_property_read_string(args.np, "label", &label) || strcmp(label, "atc3_common")) { + of_node_put(args.np); return -EINVAL; + } + owner = root_device_register("m3-hdmi-xbar"); + if (IS_ERR(owner)) { of_node_put(args.np); return PTR_ERR(owner); } + ret = of_genpd_add_device(&args, owner); + of_node_put(args.np); + if (ret) { root_device_unregister(owner); return ret; } + pm_runtime_set_suspended(owner); + pm_runtime_enable(owner); + ret = pm_runtime_resume_and_get(owner); + if (ret < 0) goto done; + if (!request_mem_region(0x130304c000ULL, 0x4000, "m3-hdmi-xbar")) { + ret = -EBUSY; goto done; + } + xbar.regs = ioremap_np(0x130304c000ULL, 0x4000); + if (!xbar.regs) { ret = -ENOMEM; goto done; } + ret = m3_hdmi_dpxbar_up(&xbar); + if (!ret && roundtrip) ret = m3_hdmi_dpxbar_down(&xbar); +done: + result = ret; + pr_info("m3_hdmi_xbar: result=%d; resources retained until reboot\n", ret); + return 0; +} +module_init(probe); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("J514S HDMI ATC3 display crossbar diagnostic"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_xbar.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_xbar.h new file mode 100644 index 0000000000000..f6d26824d5b50 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_xbar.h @@ -0,0 +1,145 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* M4 T602x sequencer adapted to captured J514S ATC3 routing. Native + * 26A428 connect/up/down model confirms source8000 -> destination8030. + * Status mirrors in that offline model do not prove real clock readiness. + */ +#ifndef M3_HDMI_DPXBAR_H +#define M3_HDMI_DPXBAR_H +/* Caller holds its device lock and a runtime PM reference throughout each + * operation. selected records an attempted route until a verified idle state; + * it is deliberately retained after uncertain writes or failed shutdown. */ +struct m3_hdmi_dpxbar { + void __iomem *regs; + bool selected; + /* J514S DCPEXT0 UFP endpoint0/port0 maps to selector0. */ + u32 selector; +}; +static const u32 dpxbar_offsets[] = { 0, 4, 8, 0xc, 0x10, 0x14, 0x18, + 0x1c, 0x20, 0x24, 0x28, 0x2c, 0x30, 0x34, + 0x800, 0x804, 0x808, 0x80c, 0x810, 0x814, 0x818, 0x81c }; +static const u32 dpxbar_idle[] = { 0, 0x1ff, 0, 0, 0, 0x1ff, 0, + 0, 0, 0x111, 0, 0, 0, 0, 0, 0x1ff, 0, 0, 0x1ff, 0, 0, 0x111 }; + +static int m3_hdmi_dpxbar_idle(struct m3_hdmi_dpxbar *xbar) +{ + for (unsigned int i = 0; i < ARRAY_SIZE(dpxbar_offsets); i++) { + u32 value = readl(xbar->regs + dpxbar_offsets[i]); + if (dpxbar_offsets[i] == 0x34 && value == 0x100) + continue; + if (value != dpxbar_idle[i]) { + pr_err("m3_hdmi_dpxbar: idle mismatch offset=%#x value=%#x expected=%#x\n", + dpxbar_offsets[i], value, dpxbar_idle[i]); + return -EBUSY; + } + } + return 0; +} + +static void m3_hdmi_dpxbar_update(struct m3_hdmi_dpxbar *xbar, u32 offset, u32 clear, u32 set) +{ + writel((readl(xbar->regs + offset) & ~clear) | set, + xbar->regs + offset); +} + +static int m3_hdmi_dpxbar_up(struct m3_hdmi_dpxbar *xbar) +{ + u32 selector = xbar->selector; + u32 bit, shift; + int ret; + + if (selector != 0 && selector != 2) + return -EINVAL; + bit = 1U << selector; + shift = 2 * selector; + ret = m3_hdmi_dpxbar_idle(xbar); + if (ret) + return ret; + /* Caller already pins the powered PHY and published DART tables. */ + xbar->selected = true; + m3_hdmi_dpxbar_update(xbar, 0x30, 0xf00000, selector << 20); + m3_hdmi_dpxbar_update(xbar, 0x30, 0xf00, selector << 8); + if (readl(xbar->regs + 0x30) != (selector << 20 | selector << 8)) + return -EIO; + /* Native connection-up uses source bit 1 and physical-output bit 0x100. + * Clock class 1 follows native pclkForConnection for attributes 0x100. + */ + m3_hdmi_dpxbar_update(xbar, 4, bit, 0); + m3_hdmi_dpxbar_update(xbar, 0x14, bit, 0); + m3_hdmi_dpxbar_update(xbar, 0x24, 0x100, 0); + udelay(1); + if (readl(xbar->regs + 0x804) & bit || + readl(xbar->regs + 0x810) & bit || + readl(xbar->regs + 0x81c) & 0x100) + return -ETIMEDOUT; + m3_hdmi_dpxbar_update(xbar, 8, 0, bit); + m3_hdmi_dpxbar_update(xbar, 0x18, 3U << shift, 1U << shift); + m3_hdmi_dpxbar_update(xbar, 0x28, 0x300, 0x100); + m3_hdmi_dpxbar_update(xbar, 0, 0, bit); + m3_hdmi_dpxbar_update(xbar, 0xc, 0, bit); + m3_hdmi_dpxbar_update(xbar, 0x1c, 0, 0x100); + m3_hdmi_dpxbar_update(xbar, 0x34, 0, 0x100); + m3_hdmi_dpxbar_update(xbar, 0x2c, 0, bit); + pr_info("m3_hdmi_dpxbar: selector%u up mux=%#x source_clock=%#x output_clock=%#x\n", + selector, readl(xbar->regs + 0x30), readl(xbar->regs), + readl(xbar->regs + 0x1c)); + if (readl(xbar->regs) != bit || readl(xbar->regs + 4) != (0x1ff & ~bit) || + readl(xbar->regs + 8) != bit || readl(xbar->regs + 0xc) != bit || + readl(xbar->regs + 0x14) != (0x1ff & ~bit) || + readl(xbar->regs + 0x18) != (1U << shift) || + readl(xbar->regs + 0x1c) != 0x100 || + readl(xbar->regs + 0x24) != 0x11 || + readl(xbar->regs + 0x28) != 0x100 || + readl(xbar->regs + 0x2c) != bit || + readl(xbar->regs + 0x34) != 0x100) + return -EIO; + return 0; +} + +static int m3_hdmi_dpxbar_down(struct m3_hdmi_dpxbar *xbar) +{ + u32 source, read_clock, output; + u32 selector = xbar->selector; + u32 bit, shift; + if (!xbar->selected) + return 0; + if (selector != 0 && selector != 2) + return -EINVAL; + bit = 1U << selector; + shift = 2 * selector; + m3_hdmi_dpxbar_update(xbar, 0x2c, bit, 0); + m3_hdmi_dpxbar_update(xbar, 0, bit, 0); + m3_hdmi_dpxbar_update(xbar, 0xc, bit, 0); + m3_hdmi_dpxbar_update(xbar, 0x1c, 0x100, 0); + udelay(1); + source = readl(xbar->regs + 0x800); + read_clock = readl(xbar->regs + 0x808); + output = readl(xbar->regs + 0x814); + /* Native takeConnectionDown logs these intermediate samples and + * continues with inverse-clock gates below (9135918..9135a08). + * They are not its completion condition. Require the full idle + * snapshot after those writes; never release on a stale final status. + */ + pr_info("m3_hdmi_dpxbar: intermediate stop status source=%#x read=%#x output=%#x\n", + source, read_clock, output); + m3_hdmi_dpxbar_update(xbar, 8, bit, 0); + m3_hdmi_dpxbar_update(xbar, 0x18, 3U << shift, 0); + m3_hdmi_dpxbar_update(xbar, 0x28, 0x300, 0); + m3_hdmi_dpxbar_update(xbar, 4, 0, bit); + m3_hdmi_dpxbar_update(xbar, 0x14, 0, bit); + m3_hdmi_dpxbar_update(xbar, 0x24, 0, 0x100); + /* Native takeConnectionDown leaves bit8 at0x34 set. Hardware in the + * retained trial ignores a zero write; do not require an unnecessary + * recovery for this native parked value. Clocks/status must be stopped, + * and subsequent RTKit AP+IOP quiescence still gates DART teardown. + * Restore the reversible selector fields after clocks stop. + */ + m3_hdmi_dpxbar_update(xbar, 0x30, 0xf00f00, 0); + if (m3_hdmi_dpxbar_idle(xbar)) + return -EIO; + xbar->selected = false; + pr_info("m3_hdmi_dpxbar: clocks stopped and selector restored; native parked register0x34=%#x\n", + readl(xbar->regs + 0x34)); + return 0; +} + +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_typec_probe.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_typec_probe.c new file mode 100644 index 0000000000000..bebd56e2d6be1 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_typec_probe.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Read-only J514S Type-C owner snapshot through the exported cache API. + * No private TIPD layout dependency, second bus owner, or PHY writes. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../usb/typec/tipd/tps6598x.h" + +static bool right_dp_ready, right_dp_partner, any_dp_ready, any_dp_partner; +static const char *controller_names[]={"1-0c","1-0a","2-08"}; +static struct spmi_device *controllers[3]; +static unsigned int dp_partner_mask, dp_ready_mask; +static int state_get(char *buffer, const struct kernel_param *parameter) +{ + struct cd321x_dp_snapshot state; + bool partner,ready; + int ret; + int (*snapshot)(struct device *, struct cd321x_dp_snapshot *); + snapshot = symbol_get(cd321x_get_dp_state); + if (!snapshot) return -EOPNOTSUPP; + right_dp_ready=right_dp_partner=any_dp_ready=any_dp_partner=false; + dp_partner_mask=dp_ready_mask=0; + for(unsigned int i=0;idev,&state); + if(ret){symbol_put(cd321x_get_dp_state);return ret;} + partner=(state.status & TPS_STATUS_PLUG_PRESENT) && + (state.data_status & TPS_DATA_STATUS_DP_CONNECTION) && + TPS_DATA_STATUS_DP_SPEC_PIN_ASSIGNMENT(state.data_status)==TPS_DATA_STATUS_DP_SPEC_PIN_ASSIGNMENT_D; + ready=partner && (state.dp_status & DP_STATUS_HPD_STATE); + any_dp_partner|=partner;any_dp_ready|=ready; + if(partner)dp_partner_mask|=BIT(i); + if(ready)dp_ready_mask|=BIT(i); + if(i==2){right_dp_partner=partner;right_dp_ready=ready;} + } + symbol_put(cd321x_get_dp_state); + if(parameter->arg==&dp_partner_mask || parameter->arg==&dp_ready_mask) + return param_get_uint(buffer, parameter); + return param_get_bool(buffer, parameter); +} +static const struct kernel_param_ops state_ops = { .get = state_get }; +module_param_cb(right_dp_ready, &state_ops, &right_dp_ready, 0400); +module_param_cb(right_dp_partner, &state_ops, &right_dp_partner, 0400); +module_param_cb(any_dp_ready, &state_ops, &any_dp_ready, 0400); +module_param_cb(any_dp_partner, &state_ops, &any_dp_partner, 0400); +module_param_cb(dp_partner_mask, &state_ops, &dp_partner_mask, 0400); +module_param_cb(dp_ready_mask, &state_ops, &dp_ready_mask, 0400); + +static int __init m3_typec_probe(void) +{ + struct device_node *node; + int ret, count = 0; + int (*snapshot)(struct device *, struct cd321x_dp_snapshot *); + + if (!of_machine_is_compatible("apple,j514s")) return -ENODEV; + snapshot = symbol_get(cd321x_get_dp_state); + if (!snapshot) return -EOPNOTSUPP; + for_each_compatible_node(node, NULL, "apple,sn201202x") { + struct spmi_device *sdev = spmi_find_device_by_of_node(node); + struct cd321x_dp_snapshot state; + if (!sdev) continue; + ret = snapshot(&sdev->dev, &state); + if (!ret) { + pr_info("m3_typec_probe: %s status=%#x data=%#x dp_status=%#x IRQs=%llu\n", + dev_name(&sdev->dev), state.status, state.data_status, + state.dp_status, (unsigned long long)state.hpd_irqs); + for(unsigned int i=0;idev),controller_names[i])){ + get_device(&sdev->dev);controllers[i]=sdev;break; + } + count++; + } + spmi_device_put(sdev); + } + symbol_put(cd321x_get_dp_state); + return count ? 0 : -ENODEV; +} +static void __exit m3_typec_exit(void) +{ + for(unsigned int i=0;i +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../usb/typec/tipd/tps6598x.h" +#include "m3_hdmi_control.h" +#include "m3_hdmi_xbar.h" + +struct m3_usbc_port { + const char *controller_name, *phy_name; + resource_size_t address; + struct spmi_device *controller; + struct phy *phy; + struct m3_hdmi_dpxbar xbar; + int owner; +}; +static struct m3_usbc_port ports[] = { + { .controller_name="1-0c", .phy_name="dp0", .address=0x70304c000ULL, .owner=-1 }, + { .controller_name="1-0a", .phy_name="dp1", .address=0xb0304c000ULL, .owner=-1 }, + { .controller_name="2-08", .phy_name="dp", .address=0xf0304c000ULL, .owner=-1 }, +}; +struct m3_usbc_route { + unsigned int active_port, rate; + bool flipped, powered, prepared; + int result; +}; +static struct m3_usbc_route routes[2] = { + { .active_port=0 }, { .active_port=2 }, +}; +static DEFINE_MUTEX(route_lock); +static int result=-EINPROGRESS; +module_param(result,int,0400); +module_param_named(active_port,routes[1].active_port,uint,0400); +module_param_named(rate,routes[1].rate,uint,0400); +module_param_named(powered,routes[1].powered,bool,0400); +module_param_named(primary_active_port,routes[0].active_port,uint,0400); +module_param_named(primary_rate,routes[0].rate,uint,0400); +module_param_named(primary_powered,routes[0].powered,bool,0400); +module_param_named(primary_result,routes[0].result,int,0400); +module_param_named(secondary_result,routes[1].result,int,0400); + +static int partner(unsigned int index, bool *orientation, bool require_hpd) +{ + struct cd321x_dp_snapshot snapshot;u32 status,data,hpd;int ret; + int (*get_snapshot)(struct device *,struct cd321x_dp_snapshot *); + if(!ports[index].controller)return -ENODEV; + get_snapshot=symbol_get(cd321x_get_dp_state); + if(!get_snapshot)return -EOPNOTSUPP; + ret=get_snapshot(&ports[index].controller->dev,&snapshot); + symbol_put(cd321x_get_dp_state);if(ret)return ret; + status=snapshot.status;data=snapshot.data_status;hpd=snapshot.dp_status; + *orientation=!!(status&TPS_STATUS_PLUG_UPSIDE_DOWN); + return !!((status&TPS_STATUS_PLUG_PRESENT) && (data&TPS_DATA_STATUS_DP_CONNECTION) && + TPS_DATA_STATUS_DP_SPEC_PIN_ASSIGNMENT(data)==TPS_DATA_STATUS_DP_SPEC_PIN_ASSIGNMENT_D && + (!require_hpd || (hpd&DP_STATUS_HPD_STATE))); +} +/* Read only cached negotiation and software mode while an inactive port's + * clocks may be off. Never steal a PHY held by the other DCP session. */ +static int candidate(unsigned int slot, bool require_hpd) +{ + unsigned int index;bool orientation;int ret; + for(unsigned int n=0;n=0 && ports[index].owner!=slot)continue; + ret=partner(index,&orientation,require_hpd); + if(ret<0)return ret; + if(!ret)continue; + ret=phy_validate(ports[index].phy,PHY_MODE_DP,0,&opts); + if(ret==-ENOLINK || ret==-EBUSY)continue; + if(ret)return ret; + if(opts.dp.lanes==2)return index; + } + return -ENOLINK; +} +static int aux(struct m3_usbc_port *p, bool enable) +{ + int ret;int (*fn)(struct phy *,bool)=symbol_get(apple_atc_m3_dp_aux); + if(!fn)return -EOPNOTSUPP; + ret=fn(p->phy,enable);symbol_put(apple_atc_m3_dp_aux);return ret; +} +static int acquire(unsigned int slot, bool require_hpd) +{ + struct m3_usbc_route *r=&routes[slot];struct m3_usbc_port *p; + bool orientation;int ret,index; + if(r->powered)return 0; + if(r->rate)return -EBUSY; + index=candidate(slot,require_hpd);if(index<0)return index; + p=&ports[index]; + ret=partner(index,&orientation,false);if(ret!=1)return ret<0?ret:-ENOLINK; + ret=phy_power_on(p->phy);if(ret)return ret; + /* All later failures retain the exclusive claim and power for recovery. */ + p->owner=slot;r->active_port=index;r->flipped=orientation;r->powered=true; + p->xbar.selector=2*slot; + if(!p->xbar.regs){ + if(!request_mem_region(p->address,0x4000,"m3-usbc-dpxbar"))return -EBUSY; + p->xbar.regs=ioremap_np(p->address,0x4000); + if(!p->xbar.regs)return -ENOMEM; + } + ret=m3_hdmi_dpxbar_idle(&p->xbar); + pr_info("m3_usbc_phy: DCPEXT%u selected ATC%u target=%#x selector=%u flipped=%u result=%d\n", + slot,r->active_port,0x8000|(r->active_port<<4),p->xbar.selector,r->flipped,ret); + return ret; +} +static int activate(unsigned int slot) +{ + struct m3_usbc_route *r=&routes[slot];union phy_configure_opts opts={}; + struct m3_usbc_port *p;int ret;bool orientation; + if(!r->powered)return -EHOSTDOWN; + p=&ports[r->active_port]; + ret=partner(r->active_port,&orientation,false); + if(ret!=1 || orientation!=r->flipped)return ret<0?ret:-ENOLINK; + ret=phy_validate(p->phy,PHY_MODE_DP,0,&opts); + if(ret || opts.dp.lanes!=2)return ret ?: -ENOLINK; + return aux(p,true); +} +static int set_rate(unsigned int slot, unsigned int next) +{ + struct m3_usbc_route *r=&routes[slot];struct m3_usbc_port *p=&ports[r->active_port]; + union phy_configure_opts opts={};int ret; + if(next!=0 && next!=6 && next!=10 && next!=20)return -EOPNOTSUPP; + if(next==r->rate)return 0; + if(!r->powered)return -EHOSTDOWN; + if(p->xbar.selected)return -EBUSY; + if(next){ret=activate(slot);if(ret)return ret;} + opts.dp.set_rate=1;opts.dp.link_rate=next*270;opts.dp.lanes=2; + ret=phy_configure(p->phy,&opts); + if(ret)r->result=ret;else r->rate=next; + pr_info("m3_usbc_phy: DCPEXT%u DP rate=%u result=%d\n",slot,next,ret); + return ret; +} +static int check(unsigned int slot) +{ + if(slot>=ARRAY_SIZE(routes))return -EINVAL; + return result ?: routes[slot].result; +} +static int route_prepare(unsigned int slot) +{ + int ret;guard(mutex)(&route_lock); + ret=check(slot);if(ret)return ret; + if(routes[slot].prepared)return -EBUSY; + ret=acquire(slot,false); + if(!ret)ret=m3_hdmi_dpxbar_up(&ports[routes[slot].active_port].xbar); + routes[slot].result=ret;routes[slot].prepared=!ret; + return ret; +} +static int route_hpd(unsigned int slot) +{ + bool orientation;int ret;guard(mutex)(&route_lock); + ret=check(slot);if(ret)return ret; + if(routes[slot].powered){ + ret=partner(routes[slot].active_port,&orientation,true); + return ret<=0?ret:orientation==routes[slot].flipped; + } + ret=candidate(slot,true); + return ret==-ENOLINK?0:ret<0?ret:1; +} +static int route_target(unsigned int slot) +{ + int ret;guard(mutex)(&route_lock); + ret=check(slot);if(ret)return ret; + /* Keep the released destination until the next explicit acquisition. */ + return 0x8000 | (routes[slot].active_port<<4); +} +static int route_activate(unsigned int slot) +{ + int ret;guard(mutex)(&route_lock); + ret=check(slot);return ret ?: activate(slot); +} +static int route_xbar_down(unsigned int slot) +{ + int ret;guard(mutex)(&route_lock); + ret=check(slot);if(ret)return ret; + /* Once released, the old physical port may belong to the other session. */ + if(!routes[slot].powered)return 0; + return m3_hdmi_dpxbar_down(&ports[routes[slot].active_port].xbar); +} +static int route_xbar_up(unsigned int slot) +{ + struct m3_hdmi_dpxbar *xbar;int ret;guard(mutex)(&route_lock); + ret=check(slot);if(ret)return ret; + ret=acquire(slot,true); + if(!routes[slot].powered && (ret==-ENOLINK || ret==-EBUSY))return -EAGAIN; + if(ret)return ret; + xbar=&ports[routes[slot].active_port].xbar; + return xbar->selected?0:m3_hdmi_dpxbar_up(xbar); +} +static int route_set_rate(unsigned int slot, unsigned int next) +{ + int ret;guard(mutex)(&route_lock); + ret=check(slot);return ret ?: set_rate(slot,next); +} +static int route_deactivate(unsigned int slot) +{ + struct m3_usbc_route *r;struct m3_usbc_port *p;int ret; + guard(mutex)(&route_lock); + ret=check(slot);if(ret)return ret; + r=&routes[slot];if(!r->powered)return 0; + p=&ports[r->active_port]; + ret=m3_hdmi_dpxbar_down(&p->xbar); + if(!ret)ret=set_rate(slot,0); + if(!ret)ret=aux(p,false); + if(!ret)ret=phy_power_off(p->phy); + if(!ret){r->powered=false;p->owner=-1;} + return ret; +} +static int route_get_rate(unsigned int slot) +{ + int ret;guard(mutex)(&route_lock); + ret=check(slot);return ret ?: routes[slot].rate; +} +static int route_set_drive(unsigned int slot, const unsigned int drive[12]) +{ + union phy_configure_opts opts={};int ret;guard(mutex)(&route_lock); + ret=check(slot);if(ret)return ret; + ret=activate(slot);if(ret)return ret; + if(!routes[slot].rate)return -EHOSTDOWN; + for(unsigned int i=0;i<2;i++){ + if(drive[i*3])return -EINVAL; + opts.dp.voltage[i]=drive[i*3+1];opts.dp.pre[i]=drive[i*3+2]; + } + opts.dp.set_voltages=1;opts.dp.lanes=2; + return phy_configure(ports[routes[slot].active_port].phy,&opts); +} +static const struct m3_usbc_route_ops route_ops={ + .prepare=route_prepare,.hpd=route_hpd,.target=route_target, + .activate=route_activate,.deactivate=route_deactivate, + .xbar_up=route_xbar_up,.xbar_down=route_xbar_down, + .set_rate=route_set_rate,.get_rate=route_get_rate,.set_drive=route_set_drive, +}; +const struct m3_usbc_route_ops *m3_usbc_get_ops(void) +{return &route_ops;} +EXPORT_SYMBOL_GPL(m3_usbc_get_ops); +static int __init probe(void) +{ + struct device_node *node;int ret=-ENODEV;int (*supported)(struct phy *); + if(!of_machine_is_compatible("apple,j514s"))return -ENODEV; + for_each_compatible_node(node,NULL,"apple,sn201202x"){ + struct spmi_device *sdev=spmi_find_device_by_of_node(node); + if(!sdev)continue; + for(unsigned int i=0;idev),ports[i].controller_name)){ + get_device(&sdev->dev);ports[i].controller=sdev;break; + } + spmi_device_put(sdev); + } + node=of_find_node_by_path("dcpext1");if(!node)goto done; + for(unsigned int i=0;iops->power_on || !phy->ops->power_off){ret=-EOPNOTSUPP;break;} + ret=phy_init(phy);if(ret)break; + ret=phy_set_mode(phy,PHY_MODE_DP);if(ret)break; + } + of_node_put(node); + done: + result=ret; + pr_info("m3_usbc_phy: two-session three-port consumer ready result=%d\n",ret); + return 0; +} +module_init(probe); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("J514S exclusive USB-C DP routing for two external DCP engines"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_usbc_ptd_vote.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_usbc_ptd_vote.c new file mode 100644 index 0000000000000..6f45f41d8c116 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_usbc_ptd_vote.c @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#define M3_PTD_INSTANCE 1 +#include "m3_dcp_ptd_vote.c" diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/syncobj-probe.py b/drivers/gpu/drm/apple/tools/m3-dcp/syncobj-probe.py new file mode 100644 index 0000000000000..99667062edb81 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/syncobj-probe.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""CPU-only DRM timeline/import/eventfd check; does not submit GPU work. + +Usage: syncobj-probe.py DISPLAY_NODE RENDER_NODE +The JSON result identifies both nodes. Exercise imported timelines in both +directions, requiring an unsignaled point to time out before CPU signaling it. +""" +import ctypes as C +import errno +import json +import os +import select +import sys + +drm = C.CDLL('libdrm.so.2', use_errno=True) +U32, U64, INT = C.c_uint32, C.c_uint64, C.c_int +P32, P64 = C.POINTER(U32), C.POINTER(U64) +signatures = { + 'drmGetCap': [INT, U64, P64], + 'drmSyncobjCreate': [INT, U32, P32], + 'drmSyncobjDestroy': [INT, U32], + 'drmSyncobjHandleToFD': [INT, U32, C.POINTER(INT)], + 'drmSyncobjFDToHandle': [INT, INT, P32], + 'drmSyncobjTimelineSignal': [INT, P32, P64, U32], + 'drmSyncobjTimelineWait': [INT, P32, P64, U32, C.c_int64, U32, P32], + 'drmSyncobjEventfd': [INT, U32, U64, INT, U32], +} +for name, args in signatures.items(): + fn = getattr(drm, name) + fn.argtypes, fn.restype = args, INT + + +def check(result): + if result: + raise OSError(C.get_errno(), f'libdrm returned {result}') + + +assert len(sys.argv) == 3, __doc__ +fds, owned, handles = [], [], [] +report = {'scope': 'CPU-signaled synchronization; no GPU execution', 'nodes': [], + 'directions': []} +try: + for path in sys.argv[1:]: + fd = os.open(path, os.O_RDWR | os.O_CLOEXEC) + fds.append(fd) + caps = {} + for name, cap in [('syncobj', 0x13), ('timeline', 0x14)]: + value = U64() + check(drm.drmGetCap(fd, cap, C.byref(value))) + caps[name] = value.value + assert value.value == 1, (path, name, value.value) + report['nodes'].append({'path': os.path.realpath(path), **caps}) + for src, dst in [(fds[0], fds[1]), (fds[1], fds[0])]: + original, imported, object_fd = U32(), U32(), INT(-1) + check(drm.drmSyncobjCreate(src, 0, C.byref(original))) + handles.append((src, original.value)) + check(drm.drmSyncobjHandleToFD(src, original, C.byref(object_fd))) + owned.append(object_fd.value) + check(drm.drmSyncobjFDToHandle(dst, object_fd, C.byref(imported))) + handles.append((dst, imported.value)) + point = U64(7) + C.set_errno(0) + result = drm.drmSyncobjTimelineWait(dst, C.byref(imported), + C.byref(point), 1, 0, 2, None) # WAIT_FOR_SUBMIT, zero deadline + assert result == -errno.ETIME, (result, C.get_errno()) + event = os.eventfd(0, os.EFD_CLOEXEC | os.EFD_NONBLOCK) + owned.append(event) + check(drm.drmSyncobjEventfd(dst, imported, point, event, 0)) + assert not select.select([event], [], [], 0)[0] + check(drm.drmSyncobjTimelineSignal(src, C.byref(original), C.byref(point), 1)) + check(drm.drmSyncobjTimelineWait(dst, C.byref(imported), + C.byref(point), 1, 0, 2, None)) + assert select.select([event], [], [], 1)[0] == [event] + assert os.eventfd_read(event) == 1 + report['directions'].append({'source': fds.index(src), + 'destination': fds.index(dst), 'point': point.value, + 'pre_signal_timeout': True, 'post_signal_wait': True, 'eventfd': True}) + report['status'] = 'Pass' + print(json.dumps(report, indent=2)) +finally: + for fd, handle in reversed(handles): + drm.drmSyncobjDestroy(fd, handle) + for fd in owned + fds: + os.close(fd) diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/validation/.gitignore b/drivers/gpu/drm/apple/tools/m3-dcp/validation/.gitignore new file mode 100644 index 0000000000000..7422560f59a23 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/validation/.gitignore @@ -0,0 +1,11 @@ +# Raw evidence stays local; commit summaries, hashes and helper source only. +* +!*/ +!.gitignore +!*.md +!*.sha256 +!*.py +!*.c +!*.h +!*.rs +!*.sh diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/validation/README.md b/drivers/gpu/drm/apple/tools/m3-dcp/validation/README.md new file mode 100644 index 0000000000000..e6c91c9632e47 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/validation/README.md @@ -0,0 +1,18 @@ +# Validation evidence policy + +Keep concise validation summaries, checksums and helper source in Git. Raw +screenshots, CTS result files, generated case catalogs, kernel logs and telemetry +remain local and are ignored. Existing published evidence is retained unchanged; +this policy removes generated data only from the previously unpublished work. + +On 2026-09-21, 52 unpublished M3 commits were rebuilt to omit generated evidence. +Source code was verified unchanged. Older commit IDs in summaries can be mapped +through `history-map.md` in the M3 lab validation directory. Raw artifact paths in +those summaries refer to local retained evidence, not files shipped in this tree. + +The original history is preserved locally in +`refs/backup/m3-before-artifact-cleanup-20260921` and in the verified incremental +bundle `build/m3-push-audit/unpublished-with-raw-evidence.bundle` in the source +collection workspace. Its prerequisite is the published commit +`4ddd13c6bf5941dc23a298a054b390850cb2294d`. No raw evidence was discarded, and no +published history was rewritten. diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/validation/dcpext_session_test.c b/drivers/gpu/drm/apple/tools/m3-dcp/validation/dcpext_session_test.c new file mode 100644 index 0000000000000..bb46e59a56ba2 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/validation/dcpext_session_test.c @@ -0,0 +1,478 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Host-only protocol tests. Synthetic requests after the captured EPMAP; + * no claim that external firmware has completed this conversation. + */ +#include "dcpext_session.h" +#include +#include +#include +#include + +#define MSG(t) ((u64)(t) << 52) +struct message { u8 ep; u64 word; }; +struct harness { + struct dcpext_session s; + u64 now; + struct message tx[1024]; + unsigned int n, buffers, fail_at, buffer_fault; +}; + +static u64 now_ms(void *cookie) +{ + return ((struct harness *)cookie)->now; +} + +static int send_message(void *cookie, u8 ep, u64 msg) +{ + struct harness *h = cookie; + assert(h->n < 1024); + h->tx[h->n++] = (struct message){ep, msg}; + return h->fail_at && h->n == h->fail_at ? -EIO : 0; +} + +static int prepare_buffer(void *cookie, u8 ep, u64 requested, u32 size, u64 *dva) +{ + struct harness *h = cookie; + assert(ep == 1 || ep == 2 || ep == 4 || ep == 8); + assert(size && size <= 0x100000); + h->buffers++; + if (h->buffer_fault == 1) + return -ENOMEM; + *dva = requested ? requested : 0x10040000000ULL + (u64)ep * 0x100000; + if (h->buffer_fault == 2) + *dva += 1; + if (h->buffer_fault == 3) + *dva = 0x11000000000ULL - 0x1000; + if (h->buffer_fault == 4) + h->now = h->s.deadline; + if (h->buffer_fault == 5) + *dva = 0x10041000000ULL; /* changed inherited address */ + if (h->buffer_fault == 6) + *dva = 0x40000000; /* table address without the DCP wire bias */ + return 0; +} + +static const struct dcpext_session_ops ops = { + .now_ms = now_ms, .send = send_message, .buffer = prepare_buffer, +}; + +static void begin(struct harness *h) +{ + memset(h, 0, sizeof(*h)); + h->now = 100; + assert(dcpext_session_begin(&h->s, &ops, h) == 0); + assert(h->s.touched && h->s.phase == DCPEXT_HELLO); + assert(h->n == 1 && h->tx[0].word == 0x60000000000220ULL); +} + +static void rx(struct harness *h, u8 ep, u64 msg) +{ + assert(dcpext_session_receive(&h->s, ep, msg) == 0); +} + +static void epmap(struct harness *h, bool early_iop) +{ + /* Actual J514S external hello and system map from session-1. */ + rx(h, 0, 0x100000000c000cULL); + assert(h->tx[1].word == 0x200000000c000cULL); + rx(h, 0, 0x8000000000011fULL); + assert(h->tx[2].word == 0x80000000000001ULL); + if (early_iop) + rx(h, 0, MSG(7) | 0x20); + assert(dcpext_session_quiesce(&h->s) == -EAGAIN); + assert(h->n == 3); /* specifically no premature IOP-quiesce message */ + rx(h, 0, 0x88000100800fffULL); + assert(h->tx[3].word == 0x88000100000000ULL); + const unsigned int endpoints[] = {1, 2, 3, 4, 8}; + for (unsigned int i = 0; i < sizeof(endpoints) / sizeof(*endpoints); i++) + assert(h->tx[4 + i].word == (MSG(5) | (u64)endpoints[i] << 32 | 2)); + assert(h->s.started == 0x11e); + assert(h->s.endpoints[1] == 0x00800fff); + assert(h->s.phase == (early_iop ? DCPEXT_AP_ON : DCPEXT_IOP_ON)); +} + +static void system_buffers(struct harness *h) +{ + unsigned int n = h->n; + /* Synthetic conversation based on Linux's system endpoint definitions. + * The firmware-owned DVA values below are test inputs, not hardware reads. + */ + rx(h, 1, MSG(1) | (16ULL << 44) | 0x10006631000ULL); + assert(h->n == n && h->s.buffers[1].inherited); + rx(h, 2, MSG(1) | (4ULL << 44)); + assert(h->tx[n].ep == 2); + assert(h->tx[n++].word == (MSG(1) | (4ULL << 44) | 0x10040200000ULL)); + rx(h, 4, MSG(1) | (3ULL << 44)); + assert(h->tx[n].ep == 4); + assert(h->tx[n++].word == (MSG(1) | (3ULL << 44) | 0x10040400000ULL)); + rx(h, 8, (1ULL << 56) | (0x24000ULL << 36) | (0x103e1b32000ULL >> 12)); + assert(h->n == n && h->s.buffers[8].size == 0x24000); + assert(h->s.buffers[8].dva == 0x103e1b32000ULL && h->s.buffers[8].inherited); + rx(h, 2, MSG(8) | (128ULL << 24) | 16); + rx(h, 2, MSG(5) | 3); + assert(h->tx[n].ep == 2 && h->tx[n++].word == (MSG(5) | 3)); + rx(h, 4, MSG(8) | 123); + assert(h->tx[n].ep == 4 && h->tx[n++].word == (MSG(8) | 123)); + rx(h, 4, MSG(0xc) | 321); + assert(h->tx[n].ep == 4 && h->tx[n++].word == (MSG(0xc) | 321)); + assert(h->n == n && h->buffers == 4); +} + +static void running(struct harness *h) +{ + begin(h); + epmap(h, false); + system_buffers(h); + rx(h, 0, MSG(7) | 0x20); + assert(h->s.phase == DCPEXT_AP_ON); + assert(h->tx[h->n - 1].word == (MSG(0xb) | 0x20)); + rx(h, 0, MSG(0xb) | 0x20); + assert(h->s.phase == DCPEXT_RUNNING); +} + +static void quiesce(struct harness *h) +{ + unsigned int n = h->n; + assert(dcpext_session_quiesce(&h->s) == 0); + assert(h->s.phase == DCPEXT_AP_QUIESCE); + assert(h->tx[n++].word == (MSG(0xb) | 0x10)); + /* Logging can still arrive while AP quiescence is pending. */ + rx(h, 4, MSG(0xc) | 0x55); + assert(h->tx[n++].word == (MSG(0xc) | 0x55)); + rx(h, 0, MSG(0xb) | 0x10); + assert(h->s.phase == DCPEXT_IOP_QUIESCE); + assert(h->tx[n++].word == (MSG(6) | 0x10)); + rx(h, 0, MSG(7) | 0x10); + assert(h->s.phase == DCPEXT_QUIESCED && h->n == n); + assert(h->buffers == 4); /* engine never discards buffer ownership */ +} + +static void sticky_failure(struct harness *h, int error) +{ + unsigned int n = h->n; + assert(h->s.phase == DCPEXT_FAILED && h->s.error == error); + assert(dcpext_session_receive(&h->s, 0, MSG(7) | 0x10) == error); + assert(dcpext_session_quiesce(&h->s) == error); + assert(dcpext_session_poll(&h->s) == error); + assert(h->n == n && h->s.phase == DCPEXT_FAILED); +} + +int main(void) +{ + struct harness h; + unsigned int n; + + /* AP acknowledgement permits channel shutdown, never DMA release. */ + running(&h); + h.s.defer_iop_quiesce = true; + assert(dcpext_session_finish_quiesce(&h.s) == -EAGAIN); + assert(!dcpext_session_quiesce(&h.s)); + n = h.n; + assert(dcpext_session_finish_quiesce(&h.s) == -EAGAIN); + rx(&h, 0, MSG(0xb) | 0x10); + assert(h.s.ap_quiesced && h.s.phase == DCPEXT_AP_QUIESCE && h.n == n); + rx(&h, 4, MSG(0xc) | 0x55); + assert(!dcpext_session_finish_quiesce(&h.s)); + assert(h.s.phase == DCPEXT_IOP_QUIESCE); + rx(&h, 0, MSG(7) | 0x10); + assert(h.s.phase == DCPEXT_QUIESCED); + running(&h); + h.s.defer_iop_quiesce = true; + assert(!dcpext_session_quiesce(&h.s)); + rx(&h, 0, MSG(0xb) | 0x10); + assert(dcpext_session_receive(&h.s, 0, MSG(0xb) | 0x10) == -EPROTO); + sticky_failure(&h, -EPROTO); + assert(dcpext_session_finish_quiesce(&h.s) == -EPROTO); + puts("PASS: deferred IOP waits for AP ACK; duplicate ACK fails with retained ownership"); + + running(&h); + h.s.sleep_iop = true; + assert(!dcpext_session_quiesce(&h.s)); + rx(&h, 0, MSG(0xb) | 0x10); + assert(h.tx[h.n - 1].word == (MSG(6) | 1)); + rx(&h, 0, MSG(7) | 1); + assert(h.s.phase == DCPEXT_QUIESCED); + running(&h); + h.s.sleep_iop = true; + assert(!dcpext_session_quiesce(&h.s)); + rx(&h, 0, MSG(0xb) | 0x10); + assert(dcpext_session_receive(&h.s, 0, MSG(7) | 0x10) == -EPROTO); + sticky_failure(&h, -EPROTO); + puts("PASS: restartable sleep requires AP quiescence then exact IOP sleep ACK; wrong ACK retains buffers"); + + running(&h); + for (unsigned int type = 0; type < 256; type++) + if (type != 1) { + running(&h); + n = h.n; + rx(&h, 8, (u64)type << 56 | 0x3060); + assert(h.n == n + (type == 2) && h.buffers == 4 && h.s.oslog_notices == 1); + if (type == 2) + assert(h.tx[n].ep == 8 && h.tx[n].word == 0x0200000000003060ULL); + } + /* The absolute message bound must still terminate an OSLog flood. */ + running(&h); + int flood_result = 0; + for (unsigned int i = 0; i < 300 && !flood_result; i++) + flood_result = dcpext_session_receive(&h.s, 8, 0x0200000000003060ULL); + assert(flood_result == -EPROTO); + sticky_failure(&h, -EPROTO); + /* Exact captured notification must advance the firmware's read counter. */ + running(&h); + n = h.n; + rx(&h, 8, 0x0200000000003060ULL); + assert(h.n == n + 1 && h.buffers == 4 && h.s.oslog_notices == 1); + assert(h.tx[n].ep == 8 && h.tx[n].word == 0x0200000000003060ULL); + puts("PASS: captured OSLog type 2 acknowledged; metadata opaque, no dereference, flood bound preserved"); + quiesce(&h); + puts("PASS: captured hello/map, system starts, buffer layouts/ACKs, AP then IOP quiescence"); + assert(dcpext_session_receive(&h.s, 2, MSG(5)) == -EPROTO); + sticky_failure(&h, -EPROTO); /* unexpected traffic invalidates completion */ + + /* Read counters wrap and are cumulative, not offsets into the buffer. */ + running(&h); + assert(!dcpext_session_quiesce(&h.s)); + const u64 counters[] = {0x31e1, 0x379b, 0xffffffff, 0}; + for (unsigned int i = 0; i < sizeof(counters) / sizeof(*counters); i++) { + u64 notice = (2ULL << 56) | counters[i]; + n = h.n; + rx(&h, 8, notice); + assert(h.n == n + 1 && h.tx[n].ep == 8 && h.tx[n].word == notice); + assert(h.s.phase == DCPEXT_AP_QUIESCE && !h.s.ap_quiesced); + } + running(&h); + h.fail_at = h.n + 1; + assert(dcpext_session_receive(&h.s, 8, 0x02000000000031e1ULL) == -EIO); + assert(h.s.buffers[8].ready); + sticky_failure(&h, -EIO); + begin(&h); + epmap(&h, false); + n = h.n; + assert(dcpext_session_receive(&h.s, 8, 0x02000000000031e1ULL) == -EPROTO); + assert(h.n == n); + for (unsigned int bit = 32; bit < 56; bit++) { + running(&h); + n = h.n; + assert(dcpext_session_receive(&h.s, 8, (2ULL << 56) | (1ULL << bit)) == -EPROTO); + assert(h.n == n); + sticky_failure(&h, -EPROTO); + } + puts("PASS: OSLog drain during AP shutdown, counter wrap, missing buffer, malformed fields and uncertain ACK failure"); + + begin(&h); + epmap(&h, true); + system_buffers(&h); + rx(&h, 0, MSG(0xb) | 0x20); + quiesce(&h); + puts("PASS: IOP acknowledgement before last map, with AP start deferred until system starts"); + + begin(&h); + epmap(&h, false); + rx(&h, 8, (1ULL << 56) | (0x23456ULL << 36)); + assert(h.s.buffers[8].size == 0x23456 && !h.s.buffers[8].inherited); + assert(h.tx[h.n - 1].word == ((1ULL << 56) | (0x23456ULL << 36) | (0x10040800000ULL >> 12))); + puts("PASS: OSLog byte size and page-shifted address, including non-page-multiple size"); + begin(&h); + epmap(&h, false); + /* Buffer size/address and notification words from the retained INTERNAL + * J713 kernel-client log. External firmware has not supplied these yet. + */ + n = h.n; + rx(&h, 8, (1ULL << 56) | (0x5ff8ULL << 36) | (0x103e1b32000ULL >> 12)); + rx(&h, 8, 0x3010042100000a1ULL); + rx(&h, 8, 0x4000020d51e4021ULL); + rx(&h, 8, 0x4000040d28078a1ULL); + rx(&h, 8, 0x4000060d51e4001ULL); + assert(h.n == n && h.s.oslog_notices == 4); + assert(h.s.buffers[8].dva == 0x103e1b32000ULL && h.s.buffers[8].size == 0x5ff8); + puts("PASS: saved internal-DCP OSLog buffer fields/notices; no dereference, reply or false protocol failure"); + + /* Fault every TX in the complete conversation. Delivery is uncertain; + * the engine must never report quiescence after any failed send. + */ + running(&h); + quiesce(&h); + n = h.n; + for (unsigned int failed = 1; failed <= n; failed++) { + struct harness recorded = h, f = {0}; + f.fail_at = failed; + int ret = dcpext_session_begin(&f.s, &ops, &f); + const struct message input[] = { + {0, 0x100000000c000cULL}, {0, 0x8000000000011fULL}, + {0, 0x88000100800fffULL}, + {1, MSG(1) | (16ULL << 44) | 0x10006631000ULL}, + {2, MSG(1) | (4ULL << 44)}, {4, MSG(1) | (3ULL << 44)}, + {8, (1ULL << 56) | (0x24000ULL << 36) | (0x103e1b32000ULL >> 12)}, + {2, MSG(8) | (128ULL << 24) | 16}, {2, MSG(5) | 3}, + {4, MSG(8) | 123}, {4, MSG(0xc) | 321}, + {0, MSG(7) | 0x20}, {0, MSG(0xb) | 0x20}, + {4, MSG(0xc) | 0x55}, {0, MSG(0xb) | 0x10}, {0, MSG(7) | 0x10}, + }; + for (unsigned int i = 0; !ret && i < sizeof(input) / sizeof(*input); i++) { + if (i == 13) + ret = dcpext_session_quiesce(&f.s); + if (!ret) + ret = dcpext_session_receive(&f.s, input[i].ep, input[i].word); + } + assert(ret == -EIO && f.n == failed && f.s.touched); + for (unsigned int i = 0; i < f.n; i++) + assert(f.tx[i].ep == recorded.tx[i].ep && f.tx[i].word == recorded.tx[i].word); + sticky_failure(&f, -EIO); + } + printf("PASS: delivery failure injected at all %u sends; failures remain sticky\n", n); + + for (unsigned int fault = 1; fault <= 6; fault++) { + begin(&h); + epmap(&h, false); + h.buffer_fault = fault; + n = h.n; + int ret = dcpext_session_receive(&h.s, 1, MSG(1) | (16ULL << 44) | 0x10006631000ULL); + assert(ret == (fault == 1 ? -ENOMEM : fault == 4 ? -ETIMEDOUT : -EINVAL)); + assert(h.n == n && h.buffers == 1); + if (fault != 1) + assert(h.s.buffers[1].ready); + sticky_failure(&h, ret); + } + puts("PASS: buffer allocation/validation/timeouts retain failure state and suppress replies"); + + const u64 bad_requests[] = {MSG(1), MSG(1) | (1ULL << 44) | (1ULL << 36), + MSG(1) | (1ULL << 44) | 0x40000001, MSG(1) | (2ULL << 44) | 0x10ffffff000ULL}; + for (unsigned int i = 0; i < sizeof(bad_requests) / sizeof(*bad_requests); i++) { + begin(&h); + epmap(&h, false); + assert(dcpext_session_receive(&h.s, 2, bad_requests[i]) == -EINVAL); + assert(!h.buffers); + sticky_failure(&h, -EINVAL); + } + for (unsigned int ep = 1; ep <= 8; ep++) { + if (ep != 1 && ep != 2 && ep != 4 && ep != 8) + continue; + running(&h); + int ret = dcpext_session_receive(&h.s, ep, ep == 8 ? 1ULL << 56 : MSG(1)); + assert(ret == (ep == 1 ? -EIO : -EPROTO) && h.buffers == 4); + sticky_failure(&h, ret); + } + puts("PASS: zero/out-of-aperture/unaligned/cross-boundary requests; crash and duplicate buffers"); + + /* Expire every phase where firmware may still own shared memory. */ + for (unsigned int phase = DCPEXT_HELLO; phase <= DCPEXT_IOP_QUIESCE; phase++) { + begin(&h); + if (phase >= DCPEXT_EPMAP) + rx(&h, 0, 0x100000000c000cULL); + if (phase >= DCPEXT_IOP_ON) + rx(&h, 0, 0x8800000000011fULL); + if (phase >= DCPEXT_AP_ON) + rx(&h, 0, MSG(7) | 0x20); + if (phase >= DCPEXT_RUNNING) + rx(&h, 0, MSG(0xb) | 0x20); + if (phase >= DCPEXT_AP_QUIESCE) + assert(dcpext_session_quiesce(&h.s) == 0); + if (phase >= DCPEXT_IOP_QUIESCE) + rx(&h, 0, MSG(0xb) | 0x10); + assert(h.s.phase == phase); + h.now = h.s.deadline; + assert(dcpext_session_poll(&h.s) == -ETIMEDOUT); + sticky_failure(&h, -ETIMEDOUT); + } + puts("PASS: fixed deadline covers every live phase; late quiesce ACK never permits cleanup"); + /* Explicit observation budget stays finite and cannot be renewed. */ + begin(&h); + h.s = (struct dcpext_session){.duration_ms = 60001}; + assert(dcpext_session_begin(&h.s, &ops, &h) == -EINVAL && !h.s.touched); + h.s = (struct dcpext_session){.duration_ms = 60000}; + assert(!dcpext_session_begin(&h.s, &ops, &h)); + assert(h.s.deadline == h.now + 60000); + h.now = h.s.deadline; + assert(dcpext_session_poll(&h.s) == -ETIMEDOUT); + sticky_failure(&h, -ETIMEDOUT); + + begin(&h); + h.now--; + assert(dcpext_session_poll(&h.s) == -ERANGE); + sticky_failure(&h, -ERANGE); + begin(&h); + assert(dcpext_session_receive(&h.s, 0, 0x100000000d000dULL) == -EPROTO); + begin(&h); + rx(&h, 0, 0x100000000c000cULL); + assert(dcpext_session_receive(&h.s, 0, 0x8800000000059fULL) == -EOPNOTSUPP); + begin(&h); + rx(&h, 0, 0x100000000c000cULL); + rx(&h, 0, 0x8000000000011fULL); + assert(dcpext_session_receive(&h.s, 0, 0x8800000000011fULL) == -EPROTO); + running(&h); + assert(dcpext_session_quiesce(&h.s) == 0); + assert(dcpext_session_receive(&h.s, 0, MSG(7) | 0x10) == -EPROTO); + sticky_failure(&h, -EPROTO); /* IOP quiescence before AP is not sufficient */ + running(&h); + assert(dcpext_session_receive(&h.s, 0x28, 0) == -EPROTO); + assert(dcpext_session_begin(&h.s, &ops, &h) == -EBUSY); + running(&h); + while (h.s.received < 256) + rx(&h, 2, MSG(5)); + assert(dcpext_session_receive(&h.s, 2, MSG(5)) == -EPROTO); + sticky_failure(&h, -EPROTO); + puts("PASS: backwards clock, version/profile/map mismatch, wrong ACK order, unsolicited app EP, reuse and message flood"); + /* The retained 40-surface run reached message 257 during power-off. + * Its explicit larger budget still terminates a flood and cannot extend + * the absolute deadline. Native display plus AV startup can also use + * this budget, without extending its 16-second deadline. */ + begin(&h); + h.s = (struct dcpext_session){.message_limit=512}; + assert(dcpext_session_begin(&h.s,&ops,&h)==-EINVAL && !h.s.touched); + h.s = (struct dcpext_session){.duration_ms=16000,.message_limit=512}; + assert(!dcpext_session_begin(&h.s,&ops,&h)); + assert(h.s.deadline == h.now + 16000); + h.now += 16000; + assert(dcpext_session_poll(&h.s) == -ETIMEDOUT); + h.s = (struct dcpext_session){.duration_ms=16000,.message_limit=8192}; + assert(!dcpext_session_begin(&h.s,&ops,&h)); + assert(h.s.deadline == h.now + 16000); + h.s = (struct dcpext_session){.duration_ms=60000,.message_limit=8192}; + assert(dcpext_session_begin(&h.s,&ops,&h)==-EINVAL && !h.s.touched); + h.s = (struct dcpext_session){.duration_ms=15000,.message_limit=512}; + assert(dcpext_session_begin(&h.s,&ops,&h)==-EINVAL && !h.s.touched); + h.s = (struct dcpext_session){.duration_ms=16000}; + assert(!dcpext_session_begin(&h.s,&ops,&h)); + h.s = (struct dcpext_session){.duration_ms=60000,.message_limit=513}; + assert(dcpext_session_begin(&h.s,&ops,&h)==-EINVAL && !h.s.touched); + h.n = 0; + h.s = (struct dcpext_session){.duration_ms=60000,.message_limit=512}; + assert(!dcpext_session_begin(&h.s,&ops,&h)); + epmap(&h,false); + system_buffers(&h); + rx(&h,0,MSG(7)|0x20);rx(&h,0,MSG(0xb)|0x20); + while(h.s.received<512) rx(&h,8,0x0200000000003060ULL); + assert(dcpext_session_receive(&h.s,8,0x0200000000003060ULL)==-EPROTO); + sticky_failure(&h,-EPROTO); + puts("PASS: explicit 60s surface-trial message budget accepts message 257, rejects 513 and preserves sticky failure"); + /* A desktop becomes persistent only after qualified startup; leaving it + * reestablishes one bounded teardown, and floods still fail permanently. */ + running(&h); + assert(!dcpext_session_enter_runtime(&h.s)); + assert(dcpext_session_enter_runtime(&h.s) == -EINVAL); + for (unsigned int second = 0; second < 120; second++) { + h.now += 1000; + assert(!dcpext_session_poll(&h.s)); + for (unsigned int m = 0; m < 512; m++) + assert(!dcpext_session_runtime_message(&h.s)); + } + assert(!dcpext_session_leave_runtime(&h.s)); + assert(!h.s.runtime && h.s.deadline == h.now + 8000); + assert(dcpext_session_leave_runtime(&h.s) == -EINVAL); + h.now += 8000; + assert(dcpext_session_poll(&h.s) == -ETIMEDOUT); + assert(dcpext_session_enter_runtime(&h.s) == -ETIMEDOUT); + running(&h);assert(!dcpext_session_enter_runtime(&h.s)); + for (unsigned int m = 0; m < 8192; m++) + assert(!dcpext_session_runtime_message(&h.s)); + assert(dcpext_session_runtime_message(&h.s) == -EOVERFLOW); + h.now += 2000;assert(dcpext_session_poll(&h.s) == -EOVERFLOW); + running(&h);assert(!dcpext_session_enter_runtime(&h.s)); + assert(!dcpext_session_quiesce(&h.s)); + assert(!h.s.runtime && h.s.phase == DCPEXT_AP_QUIESCE); + h.now += 8000;assert(dcpext_session_poll(&h.s) == -ETIMEDOUT); + begin(&h);assert(dcpext_session_enter_runtime(&h.s) == -EINVAL); + puts("PASS: explicit desktop admission, sustained runtime, all-endpoint rate bound and bounded teardown"); + puts("OFFLINE ONLY: no mailbox, DMA, external firmware completion, EDID or image was tested."); + return 0; +} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/README.md b/drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/README.md new file mode 100644 index 0000000000000..703e0e618283a --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/README.md @@ -0,0 +1,27 @@ +# J514S physical panel dimensions validation + +Source fix: cc1765d1fe7e. Installed module and m1n1 bundle hashes are in deployment.json. +The bootloader and every byte outside the J514S DT remained unchanged; device-tree.diff +contains only the panel node with 302x196 mm dimensions. + +Reboot 366814ff-4281-477d-babf-c3506c8d08d7 completed guarded native DCP startup. DRM GETCONNECTOR +and KWin independently reported 302x196 mm at 3024x1964, approximately 254 DPI. +Initially removing only the saved scale key retained 1x through KWin's existing-setup +path. After stopping the display manager, terminating eryk's session and waiting for +KWin to exit, the single-panel configuration was archived. No scale was supplied. +KWin 6.7.4 then selected 1.7x automatically (1779x1155 logical desktop). Only keyboard +and touchpad input devices were present, consistent with its 150-DPI internal-output +heuristic without a lid switch. Upstream source consulted: +https://invent.kde.org/plasma/kwin/-/blob/Plasma/6.7/src/outputconfigurationstore.cpp +Panel specification: https://support.apple.com/en-ie/117736 + +Guarded real Plasma/Konsole/screenshot validation completed, peak five-channel PMP +sample 69.50 C, with the existing 60-second watchdog and M3 thermal protections. +The screenshot was visually checked: normally rendered desktop, readable controls. +The temporary validation Konsole closed; normal persistent autologin remains active. +A subsequent native KMS status showed completed=257, failed=0. + +Target backups: /root/m3-dcp-dpi-backup-20260921 (old module, boot bundle, config), +and /home/eryk/.config/kwinoutputconfig.json.before-panel-dpi-autodetect. +Raw boot evidence and screenshot are retained under build/m3-dcp-dpi in the workspace. +This validates dimensions and automatic scaling, not new refresh-rate or recording support. diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/inspect.py b/drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/inspect.py new file mode 100644 index 0000000000000..4177b49dd7587 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/inspect.py @@ -0,0 +1,19 @@ +import ctypes as C,json,os,subprocess +from pathlib import Path +class Connector(C.Structure): + _fields_=[(n,C.c_uint32) for n in ('connector_id','encoder_id','connector_type','connector_type_id','connection','mmWidth','mmHeight')] +lib=C.CDLL('libdrm.so.2');lib.drmModeGetConnector.argtypes=[C.c_int,C.c_uint32];lib.drmModeGetConnector.restype=C.POINTER(Connector);lib.drmModeFreeConnector.argtypes=[C.POINTER(Connector)] +connectors=[] +for path in Path('/sys/class/drm').glob('card*-eDP-*'): + fd=os.open('/dev/dri/'+path.name.split('-')[0],os.O_RDONLY) + c=lib.drmModeGetConnector(fd,int((path/'connector_id').read_text())) + assert c + connectors.append({'name':path.name,'physical_mm':[c.contents.mmWidth,c.contents.mmHeight],'modes':(path/'modes').read_text().splitlines()}) + lib.drmModeFreeConnector(c);os.close(fd) +node=Path('/sys/firmware/devicetree/base') +dcp=node/(node/'aliases/dcp').read_bytes().rstrip(b'\0').decode().lstrip('/') +physical=[int.from_bytes((dcp/'panel'/k).read_bytes(),'big') for k in ('width-mm','height-mm')] +env=dict(os.environ,XDG_RUNTIME_DIR='/run/user/1000',DBUS_SESSION_BUS_ADDRESS='unix:path=/run/user/1000/bus') +q=subprocess.check_output(['runuser','-u','eryk','--','busctl','--user','--json=short','call','org.kde.KWin','/KWin','org.kde.KWin','supportInformation'],env=env,text=True,timeout=5) +info=json.loads(q)['data'][0] +print(json.dumps({'boot_id':Path('/proc/sys/kernel/random/boot_id').read_text().strip(),'device_tree_mm':physical,'drm_connectors':connectors,'kwin_support':info,'kwin_config':json.loads(Path('/home/eryk/.config/kwinoutputconfig.json').read_text())},indent=2)) diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/screenshot.sha256 b/drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/screenshot.sha256 new file mode 100644 index 0000000000000..762488d9d8389 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/screenshot.sha256 @@ -0,0 +1 @@ +3eca5616349eb5983c328c59c1d404fcc29cc6d6cff6866f20daeee80b32532c build/m3-dcp-dpi/dcp-dpi-autoscale/desktop.png From 44801ef2a633d4e21d1683d2eb321366e3291d7e Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 29 Sep 2026 09:17:18 -0700 Subject: [PATCH 1557/2680] drm/apple: bound M3 timing parser by publication size (cherry picked from commit 14187cbb08f1c5eb5f90514fe16f212895311985 on Eryk Wieliczko's eryk_m3_gpu branch; m3-dcp modules only, m3_lab scripts omitted) (cherry picked from commit c3e261cf6152524218456ea71707d2cfcc79defd) --- drivers/gpu/drm/apple/tools/m3-dcp/README.md | 12 ++++++++++++ .../gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c | 16 ++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/README.md b/drivers/gpu/drm/apple/tools/m3-dcp/README.md index 831daef79f940..9f88d810ecb2a 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/README.md +++ b/drivers/gpu/drm/apple/tools/m3-dcp/README.md @@ -178,3 +178,15 @@ The failed module remains pinned and the kernel is tainted until a future reboot; no forced unload or reboot was attempted. All raw captures, temporary helper sources/binaries and logs remain local under `build/m3-panel-wake` and `tools/m3-dcp/live-brightness` in the source-collection workspace. +## Large HDMI timing publications (2026-09-29) + +A connected HDMI monitor published 57 timing records in 1,013,116 bytes. The +OSSerialize traversal exhausted the old fixed 65,536-value budget despite all +byte/container/depth limits being satisfied. That returned EINVAL before the +first frame; the automatic late-display guard retained the watchdog and the +machine reset. Budgets now derive from the publication size (one tag consumes +at least four bytes). The 1 MiB publication, nesting and container bounds are +unchanged. Offline parsing now admits 29 modes under the existing HBR/SDR +policy. The host sanitizer regression includes an oversized value-count fixture +and the real capture, alongside malformed/truncated inputs. No mode bandwidth +or color restrictions are relaxed by this fix. Hardware validation follows. diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c index 75fee95341180..a381f5a6af43c 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c @@ -100,6 +100,10 @@ int m3_dcpext_mode_select(const struct m3_dcpext_mode *m, * Bound every container and skip, including unknown fields. Keep this parser * independent of DRM so captured and malformed publications use the same code * in host sanitizer tests and in the driver. */ +/* Every value consumes at least one four-byte tag. Bound traversal by the + * publication size rather than an unrelated 65536-node ceiling: HDMI sinks + * with many timing/color combinations can legitimately exceed that ceiling. + * The byte, depth and per-container limits remain independently enforced. */ struct native_value { u32 type, count, begin, end; }; @@ -171,7 +175,7 @@ static int native_member(const struct native_blob *b, struct native_value *out) { struct native_value key, value; - u32 pos = dict->begin, budget = 65536; + u32 pos = dict->begin, budget = b->bytes / 4; bool found = false; int ret; @@ -231,7 +235,7 @@ static int native_color_excluded(const struct native_blob *b, const char *name, u32 id) { struct native_value array, value; - u32 pos, budget = 65536; + u32 pos, budget = b->bytes / 4; int ret = native_member(b, mode, name, &array); if (ret || array.type != 2) @@ -251,7 +255,7 @@ static int native_color(const struct native_blob *b, const struct native_value *mode, u32 *id) { struct native_value array, color; - u32 pos, budget = 65536, best_score = 0; + u32 pos, budget = b->bytes / 4, best_score = 0; bool found = false; int ret = native_member(b, mode, "ColorModes", &array); @@ -382,7 +386,7 @@ int m3_dcpext_native_modes_parse(const void *data, u32 bytes, struct native_blob blob = { .data = data, .bytes = bytes }; struct native_value root, value; struct m3_dcpext_native_mode mode; - u32 pos = 4, budget = 65536, used = 0; + u32 pos = 4, budget = bytes / 4, used = 0; int ret; if (!count) @@ -396,7 +400,7 @@ int m3_dcpext_native_modes_parse(const void *data, u32 bytes, ((pos + 3) & ~3U) != bytes) return -EINVAL; pos = root.begin; - budget = 65536; + budget = bytes / 4; for (u32 i = 0; i < root.count; i++) { ret = native_value_read(&blob, &pos, 0, &budget, &value); if (!ret) @@ -467,7 +471,7 @@ int m3_dcpext_native_mode_select(const struct m3_dcpext_native_mode *requested, int m3_dcpext_native_dimensions(const void *data,u32 bytes,u32 *width_mm,u32 *height_mm) { struct native_blob b={.data=data,.bytes=bytes};struct native_value root; - u32 pos=4,budget=65536,w,h;int ret; + u32 pos=4,budget=bytes/4,w,h;int ret; if(!data || bytes<8 || bytes>0x100000 || !width_mm || !height_mm || get_unaligned_le32(data)!=0xd3)return -EINVAL; ret=native_value_read(&b,&pos,0,&budget,&root); if(ret || root.type!=1 || ((pos+3)&~3U)!=bytes)return -EINVAL; From 25beace880d568faf7142e93ac9ad077075b70bb Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 29 Sep 2026 09:17:18 -0700 Subject: [PATCH 1558/2680] drm/apple: support native external scanout geometry through 4K (cherry picked from commit 87486622cb1a52a24ca537b07a6c4638fc6625de on Eryk Wieliczko's eryk_m3_gpu branch; m3-dcp modules only, m3_lab scripts omitted) (cherry picked from commit 83704468095b9300c104966d86c72acf147abbde) --- .../gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.c | 16 ++++++----- .../drm/apple/tools/m3-dcp/m3_dcpext_frame.h | 27 ++++++++++++------- .../tools/m3-dcp/m3_dcpext_kms_adapter.h | 21 ++++++++++----- .../drm/apple/tools/m3-dcp/m3_dcpext_native.c | 3 ++- .../tools/m3-dcp/m3_dcpext_native_adapter.h | 16 +++++------ 5 files changed, 53 insertions(+), 30 deletions(-) diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.c index c4df618902473..2510823221882 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.c +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -405,13 +406,16 @@ static const struct drm_mode_config_helper_funcs m3_mode_helpers = { static int m3_dumb_create(struct drm_file *file, struct drm_device *drm, struct drm_mode_create_dumb *args) { - /* llvmpipe pads its storage to 64-pixel tiles. The scanout framebuffer - * and plane still have to match the exact native panel extent. */ - if (args->bpp != 32 || !args->width || args->width > ALIGN(3024, 64) || - !args->height || args->height > ALIGN(1964, 64)) + int ret; + + /* The compositor also allocates external-output buffers on its primary + * DRM device. Limit scanout geometry in the plane check, not allocations. + * The DRM helper checks pitch/size overflow before applying alignment. */ + if (args->bpp != 32) return -EINVAL; - args->pitch = ALIGN(args->width * 4, 64); - args->size = PAGE_ALIGN((u64)args->pitch * args->height); + ret = drm_mode_size_dumb(drm, args, 64, 0); + if (ret) + return ret; return drm_gem_dma_dumb_create_internal(file, drm, args); } diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_frame.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_frame.h index 03c97a40518c1..5e92ff3276b76 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_frame.h +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_frame.h @@ -4,13 +4,17 @@ * until reboot, including on partial publication: the caller is already pinned. */ #include +#include "m3_dcpext_modes.h" #define FRAME_DVA 0x30000000ULL #define FRAME_SLOT (FRAME_DVA >> 25) -#define FRAME_BYTES ALIGN(1920 * 1080 * 4 + 24 * LEASE_PAGE, LEASE_PAGE) +#define FRAME_BYTES ALIGN(M3_DCPEXT_MAX_PITCH * M3_DCPEXT_MAX_HEIGHT + 24 * LEASE_PAGE, LEASE_PAGE) +#define FRAME_LEAVES DIV_ROUND_UP(2 * FRAME_BYTES / LEASE_PAGE, LEASE_PAGE / 8) +static_assert(FRAME_SLOT + FRAME_LEAVES <= LEASE_PAGE / 8); +static_assert(LEASE_PAGE == PAGE_SIZE); static bool iboot_image; module_param(iboot_image,bool,0400); static void *frame_pixels,*frame_ram; -static u64 *frame_leaf,*frame_root; +static u64 *frame_leaf[FRAME_LEAVES],*frame_root; static void __iomem *frame_regs; static dma_addr_t frame_dma[2*FRAME_BYTES/LEASE_PAGE]; static u64 encode_address(phys_addr_t address); @@ -57,10 +61,14 @@ static int frame_prepare_publish(struct device_node *dcp,int count) * borrowing a live scanout aperture. SID4 PIODMA is not modified. */ ret=-EBUSY; for(unsigned int i=0;i=BIT_ULL(42)){ret=-ERANGE;goto ram_out;} + for(unsigned int i=0;i=BIT_ULL(42)){ret=-ERANGE;goto ram_out;} + } for(unsigned int y=0;y<1080;y++)for(unsigned int x=0;x<1920;x++){ static const u32 bars[]={0xffff0000,0xff00ff00,0xff0000ff,0xffffff00,0xffff00ff,0xff00ffff,0xffffffff,0xff202020}; ((u32 *)frame_pixels)[y*1920+x]=x<8 || y<8 || x>=1912 || y>=1072?0xffffffff:bars[x*8/1920]; @@ -71,13 +79,14 @@ static int frame_prepare_publish(struct device_node *dcp,int count) if(frame_dma[i]>=BIT_ULL(42) || frame_dma[i]&(LEASE_PAGE-1) || dma_to_phys(dma_dev,frame_dma[i])!=frame_dma[i]){ ret=-ERANGE;goto ram_out; } - frame_leaf[i]=encode_address(frame_dma[i])|PTE_RW; + frame_leaf[i/(LEASE_PAGE/8)][i%(LEASE_PAGE/8)]=encode_address(frame_dma[i])|PTE_RW; } ret=-EAGAIN; if(readl(frame_regs+0x1400)!=ttbr || readl(frame_regs+0x1000)!=tcr || readl(frame_regs+0x200)!=protect)goto ram_out; dma_wmb(); - if(cmpxchg64_relaxed(&root[FRAME_SLOT],0,encode_address(virt_to_phys(frame_leaf))|1) || - cmpxchg64_relaxed(&frame_root[FRAME_SLOT],0,encode_address(virt_to_phys(frame_leaf))|1))goto ram_out; + for(unsigned int i=0;igeometry.width!=1920 || mode->geometry.height!=1080 || (pixels && source_pitch<7680))return -EINVAL; + u32 width=mode->geometry.width,height=mode->geometry.height,pitch=ALIGN(width*4,64); + ret=m3_dcpext_surface_validate(width,height,pitch,FRAME_DVA,FRAME_BYTES); + if(ret || (pixels && source_pitchtiming_id!=native_kms_timing || mode->color_id!=native_kms_color){ + if(native_kms_powered){ret=m3_dcpext_native_power(native_client,false);if(ret)return ret;native_kms_powered=false;} + ret=m3_dcpext_native_mode(native_client,mode->color_id,mode->timing_id);if(ret)return ret; + native_kms_timing=mode->timing_id;native_kms_color=mode->color_id; + } if(!native_kms_powered){ret=m3_dcpext_native_power(native_client,true);if(ret)return ret;native_kms_powered=true;} if(!pixels)return m3_dcpext_native_background(native_client,0xff000000); /* Initial color bars occupy A. Copy to B first; alternate only on D589. */ @@ -32,9 +40,9 @@ static int native_kms_present(void *cookie,const struct m3_dcpext_native_mode *m destination=frame_pixels+slot*FRAME_BYTES; dva=(FRAME_DVA+slot*FRAME_BYTES)|BIT_ULL(40); frame_sync(slot,true); - for(u32 y=0;y<1080;y++)iosys_map_memcpy_from(destination+y*7680,pixels,(size_t)y*source_pitch,7680); - frame_sync(slot,false);dma_wmb();native_surface(surface,opaque); - ret=m3_dcpext_native_swap(native_client,surface,dva,1920,1080); + for(u32 y=0;y #include "m3_dcpext_rpc.h" #include "m3_dcpext_native.h" +#include "m3_dcpext_modes.h" #define A(n) M3_DCP_TAG('A', n) #define D(n) M3_DCP_TAG('D', n) @@ -640,7 +641,7 @@ static int native_swap(struct m3_dcpext_native *dcp, const void *surface, u64 dv u32 id; int ret; - if (surface && (!dva || !width || width > 3024 || !height || height > 1964)) + if (surface && (!dva || !width || width > M3_DCPEXT_MAX_WIDTH || !height || height > M3_DCPEXT_MAX_HEIGHT)) return -EINVAL; swap = kzalloc(0x1b58, GFP_KERNEL); if (!swap) diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_adapter.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_adapter.h index 8ee3cf4826a20..56276b17d37aa 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_adapter.h +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_adapter.h @@ -67,19 +67,19 @@ static int native_record(void *cookie,u32 kind,u64 msg,const void *data,u32 size } static const struct m3_dcpext_rpc_ops native_ops={.send=session_send,.poll=native_poll, .alloc=native_alloc,.map=native_map,.retire=native_retire,.dart_power=native_power,.record=native_record}; -static void native_surface(u8 *s,bool opaque) +static void native_surface(u8 *s,bool opaque,u32 width,u32 height,u32 pitch) { memset(s,0,0x22c);s[2]=opaque; /* Qualified M3 linear BGRA s, plane 0. */ put_unaligned_le32(1,s+3);put_unaligned_le32(1,s+7); put_unaligned_le32(0x42475241,s+0xb);s[0x13]=13;s[0x14]=12; - put_unaligned_le32(7680,s+0x15);put_unaligned_le16(1,s+0x19); + put_unaligned_le32(pitch,s+0x15);put_unaligned_le16(1,s+0x19); s[0x1b]=s[0x1c]=1; - put_unaligned_le32(1920,s+0x21);put_unaligned_le32(1080,s+0x25); - put_unaligned_le32(1920*1080*4,s+0x29);put_unaligned_le32(1,s+0x35); - put_unaligned_le64(1,s+0x51);put_unaligned_le32(1920,s+0x59); - put_unaligned_le32(1080,s+0x5d);put_unaligned_le32(7680,s+0x69); - put_unaligned_le32(1920*1080*4,s+0x6d);put_unaligned_le16(4,s+0x71); + put_unaligned_le32(width,s+0x21);put_unaligned_le32(height,s+0x25); + put_unaligned_le32(pitch*height,s+0x29);put_unaligned_le32(1,s+0x35); + put_unaligned_le64(1,s+0x51);put_unaligned_le32(width,s+0x59); + put_unaligned_le32(height,s+0x5d);put_unaligned_le32(pitch,s+0x69); + put_unaligned_le32(pitch*height,s+0x6d);put_unaligned_le16(4,s+0x71); s[0x73]=s[0x74]=1;put_unaligned_le64(1,s+0x149); } static int native_kms_publish(void); @@ -114,7 +114,7 @@ static int native_frame_progress(void) } kfree(modes);kfree(blob);if (ret) return ret; ret=m3_dcpext_native_power(native_client,true);if (ret) return ret; - native_surface(surface,false); + native_surface(surface,false,1920,1080,7680); ret=m3_dcpext_native_swap(native_client,surface,FRAME_DVA|BIT_ULL(40),1920,1080); if (!ret) { native_frame_submitted=true;av_read_ready=true;native_frame_until=session_now(NULL)+5000; From 9290a1a418e161b62b15a8d66d0e7ddff67da96b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 29 Sep 2026 09:17:18 -0700 Subject: [PATCH 1559/2680] drm/apple: grow M3 HDMI metadata without failing the RPC transport (cherry picked from commit 05e5da4ada37816d2c7cd063c97802230c5d149d on Eryk Wieliczko's eryk_m3_gpu branch; m3-dcp modules only, m3_lab scripts omitted) (cherry picked from commit a219f68edcb888f8e4367a21e8010f2b1e1e11f1) --- .../apple/tools/m3-dcp/m3_dcpext_connector.c | 2 +- .../tools/m3-dcp/m3_dcpext_kms_adapter.h | 39 +++++++++- .../drm/apple/tools/m3-dcp/m3_dcpext_modes.c | 42 ++++++---- .../drm/apple/tools/m3-dcp/m3_dcpext_modes.h | 9 +++ .../drm/apple/tools/m3-dcp/m3_dcpext_native.c | 76 ++++++++++--------- .../drm/apple/tools/m3-dcp/m3_dcpext_native.h | 4 +- .../tools/m3-dcp/m3_dcpext_native_adapter.h | 24 +++++- .../apple/tools/m3-dcp/m3_dcpext_property.h | 60 +++++++++++++++ .../apple/tools/m3-dcp/m3_dcpext_session.h | 13 +++- 9 files changed, 206 insertions(+), 63 deletions(-) create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_property.h diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.c index b797107b1b64d..a587ba2b98109 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.c +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.c @@ -295,7 +295,7 @@ static int publish_snapshot(struct m3_dcpext_connector *c, u64 generation, if (native) { for (u32 i = 0; i < count; i++) { u32 rate; - if (m3_dcpext_mode_validate(&native[i].geometry, &rate) || + if (m3_dcpext_mode_validate_link(&native[i].geometry, M3_DCPEXT_HBR2_4LANE_KBPS, &rate) || !native[i].hsync || !native[i].vsync || (u64)native[i].geometry.width + native[i].hfront + native[i].hsync > native[i].geometry.htotal || (u64)native[i].geometry.height + native[i].vfront + native[i].vsync > native[i].geometry.vtotal) diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms_adapter.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms_adapter.h index 4d4d02c20b9fd..bfe741de7a33c 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms_adapter.h +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms_adapter.h @@ -47,6 +47,35 @@ static int native_kms_present(void *cookie,const struct m3_dcpext_native_mode *m return ret; } static const struct m3_dcpext_kms_ops native_kms_ops={.present=native_kms_present}; +/* A rejected catalog is not a failed firmware transport. Keep servicing RPCs + * and expose an unavailable connector until a new sink publication arrives. */ +static int native_kms_unavailable(int error) +{ + int ret; + if(!native_desktop)return error; + if(native_modes_error==error && native_kms_ready)return 0; + native_modes_error=error; + native_modes_error_generation=m3_dcpext_native_generation(native_client); + pr_warn("m3_dcpext: external mode metadata rejected (%d); internal desktop retained\n",error); + if(!native_kms){ + native_kms=m3_dcpext_kms_create(dma_dev,&native_kms_ops,NULL,&native_route_lock, + usb_c?DRM_MODE_CONNECTOR_DisplayPort:DRM_MODE_CONNECTOR_HDMIA); + if(IS_ERR(native_kms)){ret=PTR_ERR(native_kms);native_kms=NULL;return ret;} + } + native_kms_connected=false; + mutex_unlock(&native_route_lock); + m3_dcpext_connector_invalidate(m3_dcpext_kms_connector(native_kms),false); + if(native_kms_ready)m3_dcpext_connector_hotplug(m3_dcpext_kms_connector(native_kms)); + mutex_lock(&native_route_lock); + if(!native_kms_ready){ + ret=dcpext_session_enter_runtime(&session);if(ret)return ret; + afk.streaming=true;aux_afk.streaming=true; + ret=m3_dcpext_kms_register(native_kms);if(ret)return ret; + native_kms_ready=true;native_frame_submitted=true; + desktop_notify(0); + } + return 0; +} static int native_kms_publish(void) { struct m3_dcpext_native_mode *modes; @@ -57,15 +86,17 @@ static int native_kms_publish(void) void *blob;u32 bytes=0,count,index,w,h;u64 generation;int ret; if(native_kms_connected || !av_edid_done)return 0; blob=m3_dcpext_native_property(native_client,"DisplayAttributes",&bytes); + if(IS_ERR(blob))return native_kms_unavailable(PTR_ERR(blob)); if(!blob)return 0; - ret=m3_dcpext_native_dimensions(blob,bytes,&w,&h);kfree(blob);if(ret)return ret; + ret=m3_dcpext_native_dimensions(blob,bytes,&w,&h);kvfree(blob);if(ret)return native_kms_unavailable(ret); blob=m3_dcpext_native_property(native_client,"TimingElements",&bytes); + if(IS_ERR(blob))return native_kms_unavailable(PTR_ERR(blob)); if(!blob)return 0; modes=kcalloc(M3_DCPEXT_NATIVE_MAX_MODES,sizeof(*modes),GFP_KERNEL); - if(!modes){kfree(blob);return -ENOMEM;} - ret=m3_dcpext_native_modes_parse(blob,bytes,modes,M3_DCPEXT_NATIVE_MAX_MODES,&count);kfree(blob); + if(!modes){kvfree(blob);return native_kms_unavailable(-ENOMEM);} + ret=m3_dcpext_native_modes_parse_link(blob,bytes,modes,M3_DCPEXT_NATIVE_MAX_MODES,&count,link_payload_kbps());kvfree(blob); if(!ret)ret=m3_dcpext_native_mode_select(&wanted,modes,count,&index); - if(ret)goto out; + if(ret){kfree(modes);return native_kms_unavailable(ret);} native_kms_generation=m3_dcpext_native_generation(native_client); if(!native_kms){ native_kms=m3_dcpext_kms_create(dma_dev,&native_kms_ops,NULL,&native_route_lock, diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c index a381f5a6af43c..4acd6b64dc99f 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c @@ -27,19 +27,21 @@ int m3_dcpext_surface_validate(u32 width, u32 height, u32 pitch, return 0; } -int m3_dcpext_mode_validate(const struct m3_dcpext_mode *m, u32 *fps_16_16) +int m3_dcpext_mode_validate_link(const struct m3_dcpext_mode *m, + u32 payload_kbps, u32 *fps_16_16) { u64 fps, total; - if (!m || !fps_16_16 || !m->width || + if (!payload_kbps || payload_kbps > M3_DCPEXT_HBR2_4LANE_KBPS || + !m || !fps_16_16 || !m->width || m->width > M3_DCPEXT_MAX_WIDTH || !m->height || m->height > M3_DCPEXT_MAX_HEIGHT || !m->clock_khz || m->htotal <= m->width || m->vtotal <= m->height || m->htotal > 65535 || m->vtotal > 65535) return -EINVAL; if (m->interlaced || m->doublescan) return -EOPNOTSUPP; - /* 4 * 2.7 Gbit/s * 8/10, divided by 24 wire bits per RGB pixel. */ - if ((u64)m->clock_khz * 24 > M3_DCPEXT_MAX_PAYLOAD_KBPS) + /* Link payload after 8b/10b encoding, 24 wire bits per RGB pixel. */ + if ((u64)m->clock_khz * 24 > payload_kbps) return -ERANGE; total = (u64)m->htotal * m->vtotal; fps = ((u64)m->clock_khz * 1000 * 65536 + total / 2) / total; @@ -49,6 +51,11 @@ int m3_dcpext_mode_validate(const struct m3_dcpext_mode *m, u32 *fps_16_16) return 0; } +int m3_dcpext_mode_validate(const struct m3_dcpext_mode *m, u32 *fps_16_16) +{ + return m3_dcpext_mode_validate_link(m, M3_DCPEXT_MAX_PAYLOAD_KBPS, fps_16_16); +} + int m3_dcpext_mode_select(const struct m3_dcpext_mode *m, const u8 *timings, u32 count, u8 selected[24]) { @@ -337,7 +344,7 @@ static int native_pipe_count(const struct native_blob *b, } static int native_mode(const struct native_blob *b, const struct native_value *v, - struct m3_dcpext_native_mode *out) + struct m3_dcpext_native_mode *out, u32 payload_kbps) { struct m3_dcpext_mode *m = &out->geometry; u32 horizontal_rate, rate, pipes_h, pipes_v, ignored; @@ -373,15 +380,15 @@ static int native_mode(const struct native_blob *b, const struct native_value *v return -EOPNOTSUPP; clock = (u64)m->htotal * m->vtotal * rate; m->clock_khz = (clock + 32768000) / 65536000; - ret = m3_dcpext_mode_validate(m, &ignored); + ret = m3_dcpext_mode_validate_link(m, payload_kbps, &ignored); if (ret) return -EOPNOTSUPP; return native_color(b, v, &out->color_id); } -int m3_dcpext_native_modes_parse(const void *data, u32 bytes, +int m3_dcpext_native_modes_parse_link(const void *data, u32 bytes, struct m3_dcpext_native_mode *modes, - u32 capacity, u32 *count) + u32 capacity, u32 *count, u32 payload_kbps) { struct native_blob blob = { .data = data, .bytes = bytes }; struct native_value root, value; @@ -392,8 +399,9 @@ int m3_dcpext_native_modes_parse(const void *data, u32 bytes, if (!count) return -EINVAL; *count = 0; - if (!data || !modes || !capacity || capacity > M3_DCPEXT_NATIVE_MAX_MODES || - bytes < 8 || bytes > 0x100000 || get_unaligned_le32(data) != 0xd3) + if (!payload_kbps || payload_kbps > M3_DCPEXT_HBR2_4LANE_KBPS || + !data || !modes || !capacity || capacity > M3_DCPEXT_NATIVE_MAX_MODES || + bytes < 8 || bytes > M3_DCPEXT_MAX_PROPERTY_BYTES || get_unaligned_le32(data) != 0xd3) return -EINVAL; ret = native_value_read(&blob, &pos, 0, &budget, &root); if (ret || root.type != 2 || root.count > M3_DCPEXT_NATIVE_MAX_MODES || @@ -404,7 +412,7 @@ int m3_dcpext_native_modes_parse(const void *data, u32 bytes, for (u32 i = 0; i < root.count; i++) { ret = native_value_read(&blob, &pos, 0, &budget, &value); if (!ret) - ret = native_mode(&blob, &value, &mode); + ret = native_mode(&blob, &value, &mode, payload_kbps); if (ret == -EOPNOTSUPP) continue; if (ret) @@ -420,6 +428,14 @@ int m3_dcpext_native_modes_parse(const void *data, u32 bytes, return 0; } +int m3_dcpext_native_modes_parse(const void *data, u32 bytes, + struct m3_dcpext_native_mode *modes, + u32 capacity, u32 *count) +{ + return m3_dcpext_native_modes_parse_link(data, bytes, modes, capacity, count, + M3_DCPEXT_MAX_PAYLOAD_KBPS); +} + int m3_dcpext_native_mode_select(const struct m3_dcpext_native_mode *requested, const struct m3_dcpext_native_mode *modes, u32 count, u32 *selected) @@ -432,14 +448,14 @@ int m3_dcpext_native_mode_select(const struct m3_dcpext_native_mode *requested, if (!requested || !modes || !selected || count > M3_DCPEXT_NATIVE_MAX_MODES) return -EINVAL; g = &requested->geometry; - ret = m3_dcpext_mode_validate(g, &rate); + ret = m3_dcpext_mode_validate_link(g, M3_DCPEXT_HBR2_4LANE_KBPS, &rate); if (ret) return ret; for (u32 i = 0; i < count; i++) { const struct m3_dcpext_native_mode *m = &modes[i]; u32 delta, candidate_rate; - if (m3_dcpext_mode_validate(&m->geometry, &candidate_rate) || + if (m3_dcpext_mode_validate_link(&m->geometry, M3_DCPEXT_HBR2_4LANE_KBPS, &candidate_rate) || g->width != m->geometry.width || g->height != m->geometry.height || g->htotal != m->geometry.htotal || g->vtotal != m->geometry.vtotal || requested->hfront != m->hfront || requested->hsync != m->hsync || diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.h index fc725270df3fb..f64555dbbffda 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.h +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.h @@ -14,6 +14,7 @@ typedef uint8_t u8;typedef uint32_t u32;typedef uint64_t u64; #define M3_DCPEXT_MAX_HEIGHT 2160 #define M3_DCPEXT_MAX_PITCH (M3_DCPEXT_MAX_WIDTH * 4) #define M3_DCPEXT_MAX_PAYLOAD_KBPS 8640000 +#define M3_DCPEXT_HBR2_4LANE_KBPS 17280000 /* Mode data is supplied by DRM's EDID parser, never invented here. */ struct m3_dcpext_mode { @@ -22,6 +23,8 @@ struct m3_dcpext_mode { }; #define M3_DCPEXT_NATIVE_MAX_MODES 128 +/* Total retained-property budget; assembly grows as chunks arrive. */ +#define M3_DCPEXT_MAX_PROPERTY_BYTES 0x800000 struct m3_dcpext_native_mode { struct m3_dcpext_mode geometry; u32 hfront, hsync, vfront, vsync; @@ -35,6 +38,12 @@ struct m3_dcpext_native_mode { int m3_dcpext_native_modes_parse(const void *blob, u32 bytes, struct m3_dcpext_native_mode *modes, u32 capacity, u32 *count); +/* Explicit route bandwidth; USB-C remains two-lane HBR2. */ +int m3_dcpext_native_modes_parse_link(const void *blob, u32 bytes, + struct m3_dcpext_native_mode *modes, + u32 capacity, u32 *count, u32 payload_kbps); +int m3_dcpext_mode_validate_link(const struct m3_dcpext_mode *mode, + u32 payload_kbps, u32 *fps_16_16); /* Match full EDID timing geometry to a uniquely closest native timing. */ int m3_dcpext_native_mode_select(const struct m3_dcpext_native_mode *requested, const struct m3_dcpext_native_mode *modes, diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.c index 59473d29bd8c5..965033acfae2b 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.c +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.c @@ -13,6 +13,7 @@ #include "m3_dcpext_rpc.h" #include "m3_dcpext_native.h" #include "m3_dcpext_modes.h" +#include "m3_dcpext_property.h" #define A(n) M3_DCP_TAG('A', n) #define D(n) M3_DCP_TAG('D', n) @@ -39,8 +40,8 @@ struct m3_dcpext_native { u32 property_count; struct { u32 service; char key[64]; void *data; u32 size; } raw[MAX_RAW_PROPERTIES]; u32 raw_count, raw_bytes; - void *chunk; - u32 chunk_size, chunk_offset; + struct m3_property_transfer chunk; + int metadata_error; u64 completion_count; u8 completion_last[0x6f0]; u64 analytics_count; @@ -108,7 +109,7 @@ static int raw_property(struct m3_dcpext_native *dcp, u32 service, const u8 *key void *copy; u32 i, old_size = 0; - if (!memchr(key, 0, 64) || size > SZ_1M) + if (!memchr(key, 0, 64) || size > M3_DCPEXT_MAX_PROPERTY_BYTES) return -EINVAL; for (i = 0; i < dcp->raw_count; i++) if (dcp->raw[i].service == service && !strcmp(dcp->raw[i].key, key)) @@ -119,7 +120,7 @@ static int raw_property(struct m3_dcpext_native *dcp, u32 service, const u8 *key old_size = dcp->raw[i].size; if (dcp->raw_bytes - old_size + size > SZ_8M) return -ENOSPC; - copy = kvmemdup(data, size ?: 1, GFP_KERNEL); + copy = kvmemdup(size ? data : "", size ?: 1, GFP_KERNEL); if (!copy) return -ENOMEM; if (i == dcp->raw_count) @@ -130,7 +131,7 @@ static int raw_property(struct m3_dcpext_native *dcp, u32 service, const u8 *key dcp->raw[i].data = copy; dcp->raw[i].size = size; dcp->raw_bytes += size - old_size; - if(!service && !strcmp(key,"TimingElements"))dcp->mode_generation++; + if(!service && !strcmp(key,"TimingElements")){dcp->mode_generation++;dcp->metadata_error=0;} return 0; } @@ -233,40 +234,25 @@ static int callback(struct m3_dcpext_rpc *b, void *cookie, u32 tag, if ((tag == D(577) && SHAPE(4, 0)) || (tag == D(300) && SHAPE(16, 0))) return 0; if (tag == D(127) && SHAPE(4, 4)) { - count = get_unaligned_le32(in); - if (dcp->chunk || !count || count > 0x100001) - return -EINVAL; - dcp->chunk_size = count - 1; - dcp->chunk_offset = 0; - dcp->chunk = kvzalloc(max_t(u32, 1, dcp->chunk_size), GFP_KERNEL); - if (!dcp->chunk) - return -ENOMEM; - out[0] = 1; - return 0; + ret = m3_property_begin(&dcp->chunk, get_unaligned_le32(in), + M3_DCPEXT_MAX_PROPERTY_BYTES - dcp->raw_bytes); + goto property_reply; } if (tag == D(128) && SHAPE(0x1008, 4)) { - offset = get_unaligned_le32(in + 0x1000); - count = get_unaligned_le32(in + 0x1004); - if (!dcp->chunk || offset != dcp->chunk_offset || count > 4096 || - offset > dcp->chunk_size || count > dcp->chunk_size - offset) - return -EINVAL; - memcpy(dcp->chunk + offset, in, count); - dcp->chunk_offset += count; - out[0] = 1; - return 0; + ret = m3_property_append(&dcp->chunk, get_unaligned_le32(in + 0x1000), + in, get_unaligned_le32(in + 0x1004)); + goto property_reply; } if (tag == D(129) && SHAPE(64, 4)) { - if (!dcp->chunk || dcp->chunk_offset != dcp->chunk_size || !memchr(in, 0, 64) || - dcp->raw_count == MAX_RAW_PROPERTIES || dcp->raw_bytes + dcp->chunk_size > SZ_8M) - return -EINVAL; - ret = raw_property(dcp, 0, in, dcp->chunk, dcp->chunk_size); - if (ret) - return ret; - kvfree(dcp->chunk); - dcp->chunk = NULL; - dev_info(dcp->dev, "M3 property %.64s: %u bytes\n", in, dcp->chunk_size); - out[0] = 1; - return 0; + ret = m3_property_complete(&dcp->chunk); + if (!ret && !memchr(in, 0, 64)) + ret = -EINVAL; + if (!ret) + ret = raw_property(dcp, 0, in, dcp->chunk.data, dcp->chunk.size); + if (!ret) + dev_info(dcp->dev, "M3 property %.64s: %u bytes\n", in, dcp->chunk.size); + m3_property_reset(&dcp->chunk); + goto property_reply; } if (tag == D(408) && SHAPE(8, 8)) { count = get_unaligned_le32(in + 4); @@ -481,6 +467,16 @@ static int callback(struct m3_dcpext_rpc *b, void *cookie, u32 tag, dev_err(dcp->dev, "M3 unhandled kernel callback %#x %u/%u\n", tag, in_size, out_size); return -EOPNOTSUPP; #undef SHAPE +property_reply: + if (ret) { + dev_warn_ratelimited(dcp->dev, "M3 property rejected: %d; keeping RPC alive\n", ret); + dcp->metadata_error = ret; + m3_property_reset(&dcp->chunk); + } + /* D127/D128/D129 return a bool. False rejects data, not the transport. */ + out[0] = !ret; + return 0; + } int m3_dcpext_native_open(struct m3_dcpext_native *dcp) @@ -719,7 +715,8 @@ void *m3_dcpext_native_property(struct m3_dcpext_native *dcp, const char *key, u for (i = 0; i < dcp->raw_count; i++) { if (!dcp->raw[i].service && !strcmp(dcp->raw[i].key, key)) { *size = dcp->raw[i].size; - copy = kmemdup(dcp->raw[i].data, *size, GFP_KERNEL); + copy = kvmemdup(*size ? dcp->raw[i].data : "", *size ?: 1, GFP_KERNEL); + if (!copy) copy = ERR_PTR(-ENOMEM); break; } } @@ -764,6 +761,8 @@ void m3_dcpext_native_invalidate_sink(struct m3_dcpext_native *dcp) { mutex_lock(&dcp->lock); dcp->mode_generation++; + dcp->metadata_error=0; + m3_property_reset(&dcp->chunk); for(u32 i=0;iraw_count;) { if(dcp->raw[i].service || (strcmp(dcp->raw[i].key,"TimingElements") && @@ -779,3 +778,8 @@ u64 m3_dcpext_native_generation(struct m3_dcpext_native *dcp) { u64 generation;mutex_lock(&dcp->lock);generation=dcp->mode_generation;mutex_unlock(&dcp->lock);return generation; } + +int m3_dcpext_native_metadata_error(struct m3_dcpext_native *dcp) +{ + return READ_ONCE(dcp->metadata_error); +} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.h index 6b26dd32371c0..b3b9ed50e8e4f 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.h +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.h @@ -17,8 +17,10 @@ int m3_dcpext_native_swap(struct m3_dcpext_native *dcp, const void *surface, u64 u32 width, u32 height); int m3_dcpext_native_background(struct m3_dcpext_native *dcp, u32 color); int m3_dcpext_native_panel(struct m3_dcpext_native *dcp, unsigned int action); -/* Caller owns the returned snapshot and must kfree it. */ +/* Caller owns the returned snapshot and must kvfree it. NULL means absent; + * ERR_PTR reports allocation failure. */ void *m3_dcpext_native_property(struct m3_dcpext_native *dcp, const char *key, u32 *size); +int m3_dcpext_native_metadata_error(struct m3_dcpext_native *); int m3_dcpext_native_pump(struct m3_dcpext_native *, unsigned long timeout); int m3_dcpext_native_mode(struct m3_dcpext_native *, u32 color, u32 timing); int m3_dcpext_native_power(struct m3_dcpext_native *, bool on); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_adapter.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_adapter.h index 56276b17d37aa..3a4002f406c72 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_adapter.h +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_adapter.h @@ -61,7 +61,11 @@ static int native_record(void *cookie,u32 kind,u64 msg,const void *data,u32 size { u8 *p=native_capture+native_capture_used; if(session.runtime)return 0; - if(size>sizeof(native_capture)-native_capture_used-16)return -ENOSPC; + if(size>sizeof(native_capture)-native_capture_used-16){ + /* Diagnostics must never break the protocol they observe. */ + pr_warn_once("m3_dcpext: native capture full; retaining prefix, RPC continues\n"); + return 0; + } put_unaligned_le32(kind,p);put_unaligned_le32(size,p+4);put_unaligned_le64(msg,p+8); memcpy(p+16,data,size);native_capture_used+=size+16;return 0; } @@ -83,6 +87,10 @@ static void native_surface(u8 *s,bool opaque,u32 width,u32 height,u32 pitch) s[0x73]=s[0x74]=1;put_unaligned_le64(1,s+0x149); } static int native_kms_publish(void); +static int native_kms_unavailable(int error); +static int native_modes_error; +static u64 native_modes_error_generation; +module_param(native_modes_error,int,0400); static int native_frame_progress(void) { struct m3_dcpext_native_mode *modes; @@ -94,6 +102,12 @@ static int native_frame_progress(void) u8 surface[0x22c] = {}; if (!native_frame || native_frame_done) return 0; if(desktop_disconnected || desktop_hpd_dirty || desktop_hpd_rpc || desktop_link_stage)return 0; + ret=m3_dcpext_native_metadata_error(native_client); + if(ret)return native_kms_unavailable(ret); + if(native_modes_error){ + if(native_modes_error_generation==m3_dcpext_native_generation(native_client))return 0; + native_modes_error=0;native_frame_submitted=false; + } if (native_frame_submitted) { if(native_kms_trial)return native_kms_publish(); if (session_now(NULL) +#include +#include + +/* Allocate on receipt, not on the firmware's announced length. The budget is + * shared with retained properties; a rejected transfer never becomes an RPC + * transport error. All sizes and offsets are validated before addition. */ +struct m3_property_transfer { + void *data; + u32 size, used, capacity, budget; + bool active; +}; +static void m3_property_reset(struct m3_property_transfer *p) +{ + kvfree(p->data); + *p = (struct m3_property_transfer){}; +} +static int m3_property_begin(struct m3_property_transfer *p, u32 count, u32 budget) +{ + bool interrupted = p->active; + m3_property_reset(p); + if (interrupted || !count) return -EINVAL; + if (count - 1 > budget) return -ENOSPC; + p->size = count - 1; + p->budget = budget; + p->active = true; + return 0; +} +static int m3_property_append(struct m3_property_transfer *p, u32 offset, + const void *data, u32 count) +{ + u32 need, capacity; + void *grown; + if (!p->active || offset != p->used || count > 4096 || !count || + offset > p->size || count > p->size - offset) return -EINVAL; + need = offset + count; + if (need > p->capacity) { + capacity = p->capacity ? p->capacity : 4096; + while (capacity < need) { + if (capacity > p->budget / 2) { capacity = p->budget; break; } + capacity *= 2; + } + if (capacity > p->budget) capacity = p->budget; + grown = kvrealloc(p->data, capacity, GFP_KERNEL); + if (!grown) return -ENOMEM; + p->data = grown; + p->capacity = capacity; + } + memcpy(p->data + offset, data, count); + p->used = need; + return 0; +} +static int m3_property_complete(const struct m3_property_transfer *p) +{ + return p->active && p->used == p->size ? 0 : -EINVAL; +} +#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h index 338808ba96bc8..c5fb17bff205d 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h @@ -18,12 +18,16 @@ static bool usb_c=DCPEXT_INSTANCE!=0; module_param(usb_c,bool,0400); /* Opt-in compatibility experiment for eventual shared-stream routing. - * Keep the normal HDMI four-lane/HBR policy until separately qualified. + * The two-lane experiment retains the USB-C payload limit. */ static bool hdmi_test_two_lane_hbr2; module_param(hdmi_test_two_lane_hbr2,bool,0400); static unsigned int link_max_lanes(void) {return usb_c || hdmi_test_two_lane_hbr2 ? 2 : 4;} +static unsigned int link_max_rate(void) +{return 20;} +static unsigned int link_payload_kbps(void) +{return link_max_lanes() * link_max_rate() * 270000U * 8 / 10;} static const struct m3_usbc_route_ops *usbc_ops; #include "dcpext_session.c" #include "dcpext_syslog.h" @@ -473,7 +477,7 @@ static int dptx_apcall(u32 channel,const u8 *payload,u32 size) ret=link_get_rate();if(ret<0)return ret; if(ret){ret=link_xbar_up();if(ret)return ret;}break; case 7: - if(len!=32)return -EINVAL;put_unaligned_le32(usb_c || hdmi_test_two_lane_hbr2 ? 20 : 10,data+16);break; + if(len!=32)return -EINVAL;put_unaligned_le32(link_max_rate(),data+16);break; case 8: if(len!=32)return -EINVAL; ret=link_get_rate();if(ret<0)return ret; @@ -717,7 +721,9 @@ static int session_run(void) pr_info("m3_dcpext_session: CPU RUN set; firmware vector unchanged\n"); session.defer_iop_quiesce = quiesce_first; if (afk_endpoint) dir = debugfs_create_dir(port ? "m3_dcpext_session1" : "m3_dcpext_session", NULL); - if(dir)audio_debugfs(dir); + if(dir){audio_debugfs(dir); + if(native_startup)debugfs_create_file("native-capture",0400,dir,NULL,&native_capture_fops); + } early_at = session_now(NULL) + 2000; if(iboot_modes || native_startup){session.duration_ms=16000;session.message_limit=512;} ret = dcpext_session_begin(&session, &ops, NULL); @@ -872,7 +878,6 @@ static int session_run(void) if (afk_endpoint) { syslog_blob.data=syslog_capture.capture;syslog_blob.size=syslog_capture.used; debugfs_create_blob("syslog-capture",0400,dir,&syslog_blob); - if(native_startup)debugfs_create_file("native-capture",0400,dir,NULL,&native_capture_fops); if(aux_endpoint)debugfs_create_file("aux-capture",0400,dir,NULL,&aux_fops); debugfs_create_file("session-pool",0400,dir,NULL,&pool_fops); if(hotplug)snapshot_ram(dir, 1); From 52d70db1642aab60f99661ef3b485fe988495fb5 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 29 Sep 2026 09:17:18 -0700 Subject: [PATCH 1560/2680] drm/apple: enable M3 HDMI HBR3 and SDR YCbCr422 modes (cherry picked from commit 6b628f4d7d0336cc277fec98eab6165811f13f8a on Eryk Wieliczko's eryk_m3_gpu branch; m3-dcp modules only, m3_lab scripts omitted) (cherry picked from commit f9d62ce5a1188ea501f7dbbb4f72e41eac41a918) --- .../apple/tools/m3-dcp/m3_dcpext_connector.c | 2 +- .../drm/apple/tools/m3-dcp/m3_dcpext_modes.c | 24 ++++++++++++------- .../drm/apple/tools/m3-dcp/m3_dcpext_modes.h | 4 +++- .../apple/tools/m3-dcp/m3_dcpext_session.h | 2 +- .../gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy.c | 2 +- .../apple/tools/m3-dcp/m3_hdmi_phy_dp-main.h | 2 +- 6 files changed, 22 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.c index a587ba2b98109..50e761e062eba 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.c +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.c @@ -295,7 +295,7 @@ static int publish_snapshot(struct m3_dcpext_connector *c, u64 generation, if (native) { for (u32 i = 0; i < count; i++) { u32 rate; - if (m3_dcpext_mode_validate_link(&native[i].geometry, M3_DCPEXT_HBR2_4LANE_KBPS, &rate) || + if (m3_dcpext_mode_validate_link(&native[i].geometry, M3_DCPEXT_HBR3_4LANE_KBPS, &rate) || !native[i].hsync || !native[i].vsync || (u64)native[i].geometry.width + native[i].hfront + native[i].hsync > native[i].geometry.htotal || (u64)native[i].geometry.height + native[i].vfront + native[i].vsync > native[i].geometry.vtotal) diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c index 4acd6b64dc99f..5683ead3193f2 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c @@ -32,7 +32,7 @@ int m3_dcpext_mode_validate_link(const struct m3_dcpext_mode *m, { u64 fps, total; - if (!payload_kbps || payload_kbps > M3_DCPEXT_HBR2_4LANE_KBPS || + if (!payload_kbps || payload_kbps > M3_DCPEXT_HBR3_4LANE_KBPS || !m || !fps_16_16 || !m->width || m->width > M3_DCPEXT_MAX_WIDTH || !m->height || m->height > M3_DCPEXT_MAX_HEIGHT || !m->clock_khz || m->htotal <= m->width || m->vtotal <= m->height || @@ -263,7 +263,7 @@ static int native_color(const struct native_blob *b, { struct native_value array, color; u32 pos, budget = b->bytes / 4, best_score = 0; - bool found = false; + bool found = false, best_rgb = false; int ret = native_member(b, mode, "ColorModes", &array); if (ret || array.type != 2 || array.count > 64) @@ -271,7 +271,7 @@ static int native_color(const struct native_blob *b, pos = array.begin; for (u32 i = 0; i < array.count; i++) { u32 candidate, score, depth, encoding, eotf, range, colorimetry; - bool virtual; + bool virtual, rgb, yuv; ret = native_value_read(b, &pos, 0, &budget, &color); if (ret || native_bool(b, &color, "IsVirtual", &virtual)) @@ -286,8 +286,12 @@ static int native_color(const struct native_blob *b, native_uint(b, &color, "Colorimetry", &colorimetry) || native_uint(b, &color, "DynamicRange", &range)) return -EINVAL; - if (depth != 8 || encoding || eotf || range || - (colorimetry != 10 && colorimetry != 16)) + /* DCP performs RGB framebuffer -> wire format conversion for A411. + * Enum values match the shared Asahi parser.h. Prefer RGB to retain + * text chroma; use BT.709 limited-range 4:2:2 only when needed. */ + rgb = !encoding && !range && (colorimetry == 10 || colorimetry == 16); + yuv = encoding == 3 && range == 1 && colorimetry == 1; + if (depth != 8 || eotf || (!rgb && !yuv)) continue; ret = native_color_excluded(b, mode, "UnsafeColorElementIDs", candidate); if (!ret) @@ -296,9 +300,11 @@ static int native_color(const struct native_blob *b, return ret; if (ret) continue; - if (!found || score > best_score || (score == best_score && candidate < *id)) { + if (!found || (rgb && !best_rgb) || + (rgb == best_rgb && (score > best_score || (score == best_score && candidate < *id)))) { *id = candidate; best_score = score; + best_rgb = rgb; found = true; } } @@ -399,7 +405,7 @@ int m3_dcpext_native_modes_parse_link(const void *data, u32 bytes, if (!count) return -EINVAL; *count = 0; - if (!payload_kbps || payload_kbps > M3_DCPEXT_HBR2_4LANE_KBPS || + if (!payload_kbps || payload_kbps > M3_DCPEXT_HBR3_4LANE_KBPS || !data || !modes || !capacity || capacity > M3_DCPEXT_NATIVE_MAX_MODES || bytes < 8 || bytes > M3_DCPEXT_MAX_PROPERTY_BYTES || get_unaligned_le32(data) != 0xd3) return -EINVAL; @@ -448,14 +454,14 @@ int m3_dcpext_native_mode_select(const struct m3_dcpext_native_mode *requested, if (!requested || !modes || !selected || count > M3_DCPEXT_NATIVE_MAX_MODES) return -EINVAL; g = &requested->geometry; - ret = m3_dcpext_mode_validate_link(g, M3_DCPEXT_HBR2_4LANE_KBPS, &rate); + ret = m3_dcpext_mode_validate_link(g, M3_DCPEXT_HBR3_4LANE_KBPS, &rate); if (ret) return ret; for (u32 i = 0; i < count; i++) { const struct m3_dcpext_native_mode *m = &modes[i]; u32 delta, candidate_rate; - if (m3_dcpext_mode_validate_link(&m->geometry, M3_DCPEXT_HBR2_4LANE_KBPS, &candidate_rate) || + if (m3_dcpext_mode_validate_link(&m->geometry, M3_DCPEXT_HBR3_4LANE_KBPS, &candidate_rate) || g->width != m->geometry.width || g->height != m->geometry.height || g->htotal != m->geometry.htotal || g->vtotal != m->geometry.vtotal || requested->hfront != m->hfront || requested->hsync != m->hsync || diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.h index f64555dbbffda..2091e04b59c54 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.h +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.h @@ -15,6 +15,7 @@ typedef uint8_t u8;typedef uint32_t u32;typedef uint64_t u64; #define M3_DCPEXT_MAX_PITCH (M3_DCPEXT_MAX_WIDTH * 4) #define M3_DCPEXT_MAX_PAYLOAD_KBPS 8640000 #define M3_DCPEXT_HBR2_4LANE_KBPS 17280000 +#define M3_DCPEXT_HBR3_4LANE_KBPS 25920000 /* Mode data is supplied by DRM's EDID parser, never invented here. */ struct m3_dcpext_mode { @@ -33,7 +34,8 @@ struct m3_dcpext_native_mode { }; /* Decode the current native TimingElements publication. Only real, uncompressed - * 8bpc full-range RGB/SDR modes within the transport policy are admitted. + * 8bpc SDR RGB or BT.709 limited-range YCbCr422 modes are admitted. RGB is + * preferred; the bandwidth check conservatively budgets 24 wire bits/pixel. * No allocation; count is zero on error. IDs belong to this publication only. */ int m3_dcpext_native_modes_parse(const void *blob, u32 bytes, struct m3_dcpext_native_mode *modes, diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h index c5fb17bff205d..fecd266f0e7ee 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h @@ -25,7 +25,7 @@ module_param(hdmi_test_two_lane_hbr2,bool,0400); static unsigned int link_max_lanes(void) {return usb_c || hdmi_test_two_lane_hbr2 ? 2 : 4;} static unsigned int link_max_rate(void) -{return 20;} +{return usb_c || hdmi_test_two_lane_hbr2 ? 20 : 30;} static unsigned int link_payload_kbps(void) {return link_max_lanes() * link_max_rate() * 270000U * 8 / 10;} static const struct m3_usbc_route_ops *usbc_ops; diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy.c index 53d13e56a92a0..9a9568bb111d1 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy.c +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy.c @@ -63,7 +63,7 @@ EXPORT_SYMBOL_GPL(m3_hdmi_phy_get_rate); int m3_hdmi_phy_set_rate(unsigned int rate) { int ret; - if(rate!=0 && rate!=6 && rate!=10 && rate!=20)return -EINVAL; + if(rate!=0 && rate!=6 && rate!=10 && rate!=20 && rate!=30)return -EINVAL; if(!m3_hdmi_phy_ready())return -EHOSTDOWN; if(rate==main_rate)return 0; ret=main_rate?m3_hdmi_atc_dp_main_stop(core,3):0; diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-main.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-main.h index 0b9c66d47c621..cf3ea1720ba66 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-main.h +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-main.h @@ -102,7 +102,7 @@ static int m3_hdmi_atc_dp_main_prepare(void __iomem *regs, unsigned int pair_mas { int ret; if ((!pair_mask || pair_mask > 3) || - (rate != 6 && rate != 10 && rate != 20) || ssc) + (rate != 6 && rate != 10 && rate != 20 && rate != 30) || ssc) return -EOPNOTSUPP; ret = m3_hdmi_dp_pll_prepare(regs, pair_mask, rate, ssc); if (ret) From 6200c5105c5f400538fee7f1764ed084de4203ce Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 29 Sep 2026 09:17:19 -0700 Subject: [PATCH 1561/2680] drm/apple: map pinned M3 external framebuffers for direct scanout (cherry picked from commit 905c123b7f2ade7b3415f2bac879aff48e52e941 on Eryk Wieliczko's eryk_m3_gpu branch; m3-dcp modules only, m3_lab scripts omitted) (cherry picked from commit 47226ad6433dea6d9f2fb1e6f8536b7b82ee7768) --- .../drm/apple/tools/m3-dcp/m3_dcpext_frame.h | 37 ++++++++++++ .../drm/apple/tools/m3-dcp/m3_dcpext_kms.c | 34 +++++++++-- .../drm/apple/tools/m3-dcp/m3_dcpext_kms.h | 11 ++-- .../tools/m3-dcp/m3_dcpext_kms_adapter.h | 60 +++++++++++++++---- .../tools/m3-dcp/m3_dcpext_native_adapter.h | 2 + 5 files changed, 125 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_frame.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_frame.h index 5e92ff3276b76..e9bc28c19956d 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_frame.h +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_frame.h @@ -4,6 +4,7 @@ * until reboot, including on partial publication: the caller is already pinned. */ #include +#include #include "m3_dcpext_modes.h" #define FRAME_DVA 0x30000000ULL #define FRAME_SLOT (FRAME_DVA >> 25) @@ -148,3 +149,39 @@ static void frame_sync(unsigned int slot,bool for_cpu) else dma_sync_single_for_device(dma_dev,frame_dma[i],LEASE_PAGE,DMA_TO_DEVICE); } } + +/* Replace only the inactive slot, after its last scanout has retired. The + * caller owns the backing framebuffer until these PTEs are replaced again. */ +static int frame_map_pages(unsigned int slot, struct sg_table *sgt, u32 offset, u64 bytes) +{ + struct sg_page_iter it; + unsigned int first=slot*(FRAME_BYTES/LEASE_PAGE), pages=DIV_ROUND_UP_ULL(bytes,LEASE_PAGE), n=0; + u32 status; + int ret; + if(slot>1 || (offset & (LEASE_PAGE-1)) || bytes>FRAME_BYTES)return -EINVAL; + if(sgt){ + for_each_sgtable_page(sgt,&it,offset/LEASE_PAGE){ + phys_addr_t pa=page_to_phys(sg_page_iter_page(&it)); + if(pa>=BIT_ULL(42))return -ERANGE; + if(++n==pages)break; + } + if(n!=pages)return -EINVAL; + } + n=0; + if(sgt)for_each_sgtable_page(sgt,&it,offset/LEASE_PAGE){ + unsigned int i=first+n; + WRITE_ONCE(frame_leaf[i/(LEASE_PAGE/8)][i%(LEASE_PAGE/8)], + encode_address(page_to_phys(sg_page_iter_page(&it)))|PTE_RW); + if(++n==pages)break; + } + /* Keep unused/guard pages backed by owned scratch memory. */ + for(;n #include #include +#include #include #include #include @@ -99,6 +100,8 @@ static int present(struct m3_dcpext_kms *kms, struct drm_atomic_state *state, struct drm_crtc_state *c = drm_atomic_get_new_crtc_state(state, &kms->crtc); struct drm_connector_state *connector; struct drm_framebuffer *fb = NULL; + struct sg_table *sgt = NULL; + bool copied = false; struct m3_dcpext_native_mode mode; struct iosys_map map[DRM_FORMAT_MAX_PLANES], data[DRM_FORMAT_MAX_PLANES]; u64 generation = 0; @@ -111,13 +114,36 @@ static int present(struct m3_dcpext_kms *kms, struct drm_atomic_state *state, fb = p->fb; } if (fb) { + struct drm_gem_object *obj = drm_gem_fb_get_obj(fb, 0); + struct scatterlist *sg; + unsigned int i; + u64 bytes = (u64)fb->pitches[0] * fb->height; + bool direct = !(fb->offsets[0] & (PAGE_SIZE - 1)) && + bytes <= (u64)M3_DCPEXT_MAX_PITCH * M3_DCPEXT_MAX_HEIGHT && + fb->offsets[0] <= obj->size && bytes <= obj->size - fb->offsets[0]; + if (direct) { + sgt = drm_gem_shmem_get_pages_sgt(to_drm_gem_shmem_obj(obj)); + if (IS_ERR(sgt)) return PTR_ERR(sgt); + for_each_sgtable_sg(sgt, sg, i) + if ((sg_phys(sg) & (PAGE_SIZE - 1)) || + sg_phys(sg) >= BIT_ULL(42) || + sg->length > BIT_ULL(42) - sg_phys(sg)) direct = false; + if (!direct) sgt = NULL; + } + if (sgt) { + /* The shmem/PRIME helper mapped this attachment BIDIRECTIONAL. */ + dma_sync_sgtable_for_device(drm_dev_dma_dev(fb->dev), sgt, DMA_BIDIRECTIONAL); + goto mapped; + } ret = drm_gem_fb_begin_cpu_access(fb, DMA_FROM_DEVICE); if (ret) return ret; ret = drm_gem_fb_vmap(fb, map, data); if (ret) goto end_access; + copied = true; } +mapped: mutex_lock(&kms->owner_lock); if (kms->detached) { ret = -ENODEV; @@ -132,8 +158,8 @@ static int present(struct m3_dcpext_kms *kms, struct drm_atomic_state *state, } if (!ret) ret = kms->ops->present(kms->cookie, enable ? &mode : NULL, generation, - fb ? &data[0] : NULL, fb ? fb->pitches[0] : 0, - fb && fb->format->format == DRM_FORMAT_XRGB8888); + copied ? &data[0] : NULL, fb ? fb->pitches[0] : 0, + fb && fb->format->format == DRM_FORMAT_XRGB8888, fb, sgt); if (!ret && enable) kms->completed++; if (ret && ret != -ESTALE && ret != -ENODEV) { @@ -143,10 +169,10 @@ static int present(struct m3_dcpext_kms *kms, struct drm_atomic_state *state, mutex_unlock(kms->route_lock); unlock_owner: mutex_unlock(&kms->owner_lock); - if (fb) + if (copied) drm_gem_fb_vunmap(fb, map); end_access: - if (fb) + if (fb && !sgt) drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE); return ret; } diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.h index 628c8fd5f021d..6f0578c5223f7 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.h +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.h @@ -4,17 +4,20 @@ #include "m3_dcpext_connector.h" struct device; struct iosys_map; +struct drm_framebuffer; +struct sg_table; struct m3_dcpext_kms; -/* The route mutex serializes callbacks/hotplug and hardware commits. Pixels - * are CPU-readable until present returns; the owner copies them into its - * retained scanout slots. Success requires matching firmware completion. +/* The route mutex serializes callbacks/hotplug and hardware commits. For direct scanout, fb/sgt describe pinned GEM pages; the owner retains + * its own framebuffer reference while a DART slot maps them. Otherwise pixels + * are CPU-readable until present returns and copied into retained slots. Success requires matching firmware completion. * Disable has mode/pixels NULL; an active mode with pixels NULL presents only * the background. Neither operation may free uncertain DMA resources. */ struct m3_dcpext_kms_ops { int (*present)(void *cookie, const struct m3_dcpext_native_mode *mode, u64 generation, const struct iosys_map *pixels, - u32 source_pitch, bool opaque); + u32 source_pitch, bool opaque, + struct drm_framebuffer *fb, struct sg_table *sgt); }; struct m3_dcpext_kms *m3_dcpext_kms_create(struct device *dev, const struct m3_dcpext_kms_ops *ops, void *cookie, diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms_adapter.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms_adapter.h index bfe741de7a33c..283c96181c677 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms_adapter.h +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms_adapter.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only OR MIT */ /* M4 native KMS ownership pattern adapted to J514S old RPC protocol. */ #include +#include #include "m3_dcpext_kms.h" static DEFINE_MUTEX(native_route_lock); static struct m3_dcpext_kms *native_kms; @@ -8,13 +9,22 @@ static bool native_kms_ready,native_kms_finished,native_kms_powered=true; static bool native_kms_connected; static unsigned int native_kms_frames,native_kms_slot; static u32 native_kms_timing,native_kms_color; +static struct drm_framebuffer *native_slot_fb[2], *native_failed_fb; +static unsigned int native_direct_frames; +module_param(native_direct_frames,uint,0400); +/* Completed-frame aggregate costs; read deltas around a bounded animation. */ +static unsigned long long native_copy_ns,native_sync_ns,native_swap_ns; +module_param(native_copy_ns,ullong,0400); +module_param(native_sync_ns,ullong,0400); +module_param(native_swap_ns,ullong,0400); static u64 native_kms_until,native_kms_generation; module_param(native_kms_ready,bool,0400); module_param(native_kms_frames,uint,0400); static int native_kms_present(void *cookie,const struct m3_dcpext_native_mode *mode, - u64 generation,const struct iosys_map *pixels,u32 source_pitch,bool opaque) + u64 generation,const struct iosys_map *pixels,u32 source_pitch,bool opaque, + struct drm_framebuffer *fb,struct sg_table *sgt) { - u8 surface[0x22c];void *destination;u64 dva;int ret; + u8 surface[0x22c];void *destination;u64 dva,t0,t1,t2,t3;int ret; lockdep_assert_held(&native_route_lock); /* Hotplug is an ordinary stale atomic commit, not a poisoned device. */ if(desktop_disconnected || !native_kms_connected)return mode?-ESTALE:0; @@ -25,7 +35,7 @@ static int native_kms_present(void *cookie,const struct m3_dcpext_native_mode *m } if(native_kms_finished || !native_kms_ready || generation!=native_kms_generation || generation!=m3_dcpext_native_generation(native_client))return -ESTALE; - u32 width=mode->geometry.width,height=mode->geometry.height,pitch=ALIGN(width*4,64); + u32 width=mode->geometry.width,height=mode->geometry.height,pitch=sgt?source_pitch:ALIGN(width*4,64); ret=m3_dcpext_surface_validate(width,height,pitch,FRAME_DVA,FRAME_BYTES); if(ret || (pixels && source_pitchtiming_id!=native_kms_timing || mode->color_id!=native_kms_color){ @@ -34,16 +44,35 @@ static int native_kms_present(void *cookie,const struct m3_dcpext_native_mode *m native_kms_timing=mode->timing_id;native_kms_color=mode->color_id; } if(!native_kms_powered){ret=m3_dcpext_native_power(native_client,true);if(ret)return ret;native_kms_powered=true;} - if(!pixels)return m3_dcpext_native_background(native_client,0xff000000); - /* Initial color bars occupy A. Copy to B first; alternate only on D589. */ + if(!fb)return m3_dcpext_native_background(native_client,0xff000000); + /* Initial color bars occupy A. Use B first; alternate only on D589. */ unsigned int slot=native_kms_slot?0:1; destination=frame_pixels+slot*FRAME_BYTES; dva=(FRAME_DVA+slot*FRAME_BYTES)|BIT_ULL(40); - frame_sync(slot,true); - for(u32 y=0;yoffsets[0],(u64)pitch*height); + if(ret){native_failed_fb=old;return ret;} + if(old)drm_framebuffer_put(old); + } + t3=ktime_get_ns();native_sync_ns+=t3-t0; + }else{ + if(native_slot_fb[slot]){ + ret=frame_map_pages(slot,NULL,0,0);if(ret)return ret; + drm_framebuffer_put(native_slot_fb[slot]);native_slot_fb[slot]=NULL; + } + frame_sync(slot,true);t1=ktime_get_ns(); + for(u32 y=0;y Date: Tue, 29 Sep 2026 09:17:19 -0700 Subject: [PATCH 1562/2680] drm/apple: import mapped SG tables for M3 direct scanout (cherry picked from commit be7d74b7a14180703d69565bda4a686b6f3260b8 on Eryk Wieliczko's eryk_m3_gpu branch; m3-dcp modules only, m3_lab scripts omitted) (cherry picked from commit f18bec4fdf200cbaf3a87e990df63b41dda79be4) --- drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_dma.c | 4 ++-- drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.c | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_dma.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_dma.c index 536389856308a..4fa7fb609c670 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_dma.c +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_dma.c @@ -79,8 +79,8 @@ static int flush_sid(void) u32 status; /* DART table walks are coherent, as in apple_dart_finalize_domain. */ - pr_info("m3_dcpext_dma: flushing SID5\n"); - msleep(20); + /* Completion polling orders reuse; a diagnostic sleep here would + * impose a 20 ms floor on every runtime scanout remapping. */ dma_wmb(); writel(0x100 | 5, regs + 0x80); return readl_poll_timeout(regs + 0x80, status, !(status & BIT(31)), 1, 10000); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.c index 6cb2c8057b65a..f0561419f12cc 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.c +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.c @@ -1,7 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -/* External native DRM device. PRIME/shmem sources are copied into retained - * hardware scanout by the route owner. Firmware never accesses GEM storage; - * importing a GPU buffer therefore does not require contiguous physical RAM. +/* External native DRM device. The route owner maps pinned PRIME/shmem pages + * into its retained scanout aperture, with a copy fallback for unaligned data. * Native completion handling follows the internal M3 KMS implementation. */ #include #include @@ -285,7 +284,10 @@ static const struct drm_driver kms_driver = { * without requiring a render engine on the display device. */ .driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC | DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE, - DRM_GEM_SHMEM_DRIVER_OPS, + /* Direct scanout needs the pinned SG table. The default shmem ops + * import without mapping and are only suitable for CPU-copy scanout. */ + .gem_prime_import_sg_table = drm_gem_shmem_prime_import_sg_table, + .dumb_create = drm_gem_shmem_dumb_create, .fops = &kms_fops, .name = "m3-dcpext", .desc = "J514S native HDMI", From 41c0033b6513766bef4d4ca49fd1c82e45a0a340 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 29 Sep 2026 09:17:19 -0700 Subject: [PATCH 1563/2680] drm/apple: qualify M3 HDMI completion latency and flip timestamps (cherry picked from commit cb50e69060ddc2c47097f2f343b7235fd04505f0 on Eryk Wieliczko's eryk_m3_gpu branch; m3-dcp modules only, m3_lab scripts omitted) (cherry picked from commit 18bd205f3b7ab70133c827cdb648046a57880744) --- .../drm/apple/tools/m3-dcp/m3_dcpext_kms.c | 36 +++++++++++++++++-- .../drm/apple/tools/m3-dcp/m3_dcpext_kms.h | 12 ++++--- .../drm/apple/tools/m3-dcp/m3_dcpext_native.c | 12 +++++++ 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.c index f0561419f12cc..3216e7b6cdeca 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.c +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.c @@ -24,6 +24,10 @@ #include #include "m3_dcpext_kms.h" +/* Bounded diagnostic control for qualifying callback delivery latency. */ +static unsigned int flip_slack_us = 2000; +module_param(flip_slack_us, uint, 0600); + struct m3_dcpext_kms { struct drm_device drm; struct drm_plane plane; @@ -38,6 +42,7 @@ struct m3_dcpext_kms { bool detached; bool failed; u64 completed; + ktime_t submit_time, completion_time; }; static struct m3_dcpext_kms *to_kms(struct drm_device *drm) @@ -155,10 +160,13 @@ static int present(struct m3_dcpext_kms *kms, struct drm_atomic_state *state, ret = connector ? m3_dcpext_connector_commit_native(&kms->connector, connector, &c->mode, &mode, &generation) : -EINVAL; } - if (!ret) + if (!ret) { + kms->submit_time = ktime_get(); ret = kms->ops->present(kms->cookie, enable ? &mode : NULL, generation, copied ? &data[0] : NULL, fb ? fb->pitches[0] : 0, fb && fb->format->format == DRM_FORMAT_XRGB8888, fb, sgt); + kms->completion_time = ktime_get(); + } if (!ret && enable) kms->completed++; if (ret && ret != -ESTALE && ret != -ENODEV) { @@ -203,7 +211,31 @@ static void finish_event(struct drm_crtc *crtc, struct drm_atomic_state *state, return; } spin_lock_irqsave(&crtc->dev->event_lock, flags); - drm_crtc_send_vblank_event(crtc, event); + if (c->active && event->event.base.type == DRM_EVENT_FLIP_COMPLETE) { + struct m3_dcpext_kms *kms = to_kms(crtc->dev); + ktime_t flip = kms->completion_time; + s64 delta = ktime_us_delta(flip, kms->submit_time); + struct timespec64 tv; + + /* Based on Asahi dcp_crtc_send_page_flip_event(): DCP reports + * completion after the physical flip. Using callback receipt as + * the flip time makes KWin submit too late for the next refresh, + * halving HDMI cadence. Preserve at least 500 us after submit; + * M3 measurements need a 2 ms cap rather than Asahi's 1 ms. + * This is a bounded latency estimate, not + * a hardware timestamp. Never signal before real completion. */ + if (delta > 500) + flip = ktime_sub_us(flip, min_t(s64, delta - 500, + min(READ_ONCE(flip_slack_us), 4000U))); + tv = ktime_to_timespec64(flip); + event->pipe = drm_crtc_index(crtc); + event->event.vbl.sequence = drm_crtc_vblank_count(crtc); + event->event.vbl.tv_sec = tv.tv_sec; + event->event.vbl.tv_usec = tv.tv_nsec / 1000; + drm_send_event_timestamp_locked(crtc->dev, &event->base, flip); + } else { + drm_crtc_send_vblank_event(crtc, event); + } spin_unlock_irqrestore(&crtc->dev->event_lock, flags); } diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.h index 6f0578c5223f7..88152d0250957 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.h +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.h @@ -8,11 +8,13 @@ struct drm_framebuffer; struct sg_table; struct m3_dcpext_kms; -/* The route mutex serializes callbacks/hotplug and hardware commits. For direct scanout, fb/sgt describe pinned GEM pages; the owner retains - * its own framebuffer reference while a DART slot maps them. Otherwise pixels - * are CPU-readable until present returns and copied into retained slots. Success requires matching firmware completion. - * Disable has mode/pixels NULL; an active mode with pixels NULL presents only - * the background. Neither operation may free uncertain DMA resources. */ +/* The route mutex serializes callbacks/hotplug and hardware commits. + * For direct scanout, fb/sgt describe pinned GEM pages; the owner retains its + * own framebuffer reference while a DART slot maps them. Otherwise pixels + * are CPU-readable until present returns and copied into retained slots. + * Success requires matching firmware completion. Disable has mode NULL; + * an active mode with fb NULL presents only the background. Neither operation + * may free uncertain DMA resources. */ struct m3_dcpext_kms_ops { int (*present)(void *cookie, const struct m3_dcpext_native_mode *mode, u64 generation, const struct iosys_map *pixels, diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.c index 965033acfae2b..1c5d08a5048a8 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.c +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -629,12 +630,18 @@ int m3_dcpext_native_power(struct m3_dcpext_native *dcp, bool on) return ret; } +/* Last 32 serialized swaps: ID, begin/A406-ack/completion monotonic us. + * Fixed-size overwrite telemetry; no allocation or logging in the frame loop. */ +static unsigned long long swap_timings[32 * 4]; +static unsigned int swap_timing_index; +module_param_array(swap_timings, ullong, NULL, 0400); static int native_swap(struct m3_dcpext_native *dcp, const void *surface, u64 dva, u32 width, u32 height, u32 background) { __le32 start[4] = {}, started[2], result[3]; u8 *swap; u32 id; + unsigned int ti; int ret; if (surface && (!dva || !width || width > M3_DCPEXT_MAX_WIDTH || !height || height > M3_DCPEXT_MAX_HEIGHT)) @@ -643,12 +650,16 @@ static int native_swap(struct m3_dcpext_native *dcp, const void *surface, u64 dv if (!swap) return -ENOMEM; mutex_lock(&dcp->lock); + ti = (swap_timing_index++ % 32) * 4; + swap_timings[ti + 1] = ktime_to_us(ktime_get()); ret = call(dcp, A(406), start, sizeof(start), started, sizeof(started), 0); if (ret || le32_to_cpu(started[1])) { ret = ret ?: -EIO; goto out; } id = le32_to_cpu(started[0]); + swap_timings[ti] = id; + swap_timings[ti + 2] = ktime_to_us(ktime_get()); if (!id) { ret = -EPROTO; goto out; @@ -683,6 +694,7 @@ static int native_swap(struct m3_dcpext_native *dcp, const void *surface, u64 dv swap[0x1b4b + plane] = 0; } ret = call(dcp, A(407), swap, 0x1b58, result, sizeof(result), id); + swap_timings[ti + 3] = ktime_to_us(ktime_get()); if (!ret && get_unaligned_le32((u8 *)result + 5)) ret = -EIO; if (!ret && !surface) From 96e3f3bf0a672a512b370adf4308c8718b410ec8 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 29 Sep 2026 09:17:19 -0700 Subject: [PATCH 1564/2680] drm/apple: snapshot live external firmware logs and qualify FRL state (cherry picked from commit 90c483c8dc1d2df8d905aa0073bfc701e07746b1 on Eryk Wieliczko's eryk_m3_gpu branch; m3-dcp modules only, m3_lab scripts omitted) (cherry picked from commit 811865654598aa632fd99ae3ef8ea989506a0fe1) --- .../apple/tools/m3-dcp/m3_dcpext_session.h | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h index fecd266f0e7ee..a2997517505cb 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h @@ -662,6 +662,37 @@ static ssize_t afk_read(struct file *file, char __user *buf, size_t count, loff_ static const struct file_operations afk_fops = { .owner = THIS_MODULE, .read = afk_read, .llseek = default_llseek, }; +/* Snapshot the retained RTKit log while the desktop remains active. Allocate + * outside the route lock and copy under it; userspace reads never block RPCs. + * Each open owns an immutable, bounded snapshot, including during wraparound. + */ +struct syslog_snapshot { size_t size; u8 data[]; }; +static int syslog_live_open(struct inode *inode, struct file *file) +{ + struct syslog_snapshot *snapshot; + snapshot=kvmalloc(struct_size(snapshot,data,sizeof(syslog_capture.capture)),GFP_KERNEL); + if(!snapshot)return -ENOMEM; + mutex_lock(&native_route_lock); + snapshot->size=syslog_capture.used; + if(snapshot->size>sizeof(syslog_capture.capture)){ + mutex_unlock(&native_route_lock);kvfree(snapshot);return -EIO; + } + memcpy(snapshot->data,syslog_capture.capture,snapshot->size); + mutex_unlock(&native_route_lock); + file->private_data=snapshot; + return 0; +} +static ssize_t syslog_live_read(struct file *file,char __user *buf,size_t count,loff_t *offset) +{ + struct syslog_snapshot *snapshot=file->private_data; + return simple_read_from_buffer(buf,count,offset,snapshot->data,snapshot->size); +} +static int syslog_live_release(struct inode *inode,struct file *file) +{kvfree(file->private_data);return 0;} +static const struct file_operations syslog_live_fops={ + .owner=THIS_MODULE,.open=syslog_live_open,.read=syslog_live_read, + .release=syslog_live_release,.llseek=default_llseek, +}; static int session_run(void) { static const struct dcpext_session_ops ops = { @@ -722,6 +753,7 @@ static int session_run(void) session.defer_iop_quiesce = quiesce_first; if (afk_endpoint) dir = debugfs_create_dir(port ? "m3_dcpext_session1" : "m3_dcpext_session", NULL); if(dir){audio_debugfs(dir); + debugfs_create_file("syslog-live",0400,dir,NULL,&syslog_live_fops); if(native_startup)debugfs_create_file("native-capture",0400,dir,NULL,&native_capture_fops); } early_at = session_now(NULL) + 2000; From a9e89c7142381f9f75e80c1ddf8a67fb61d0e19d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 29 Sep 2026 09:17:20 -0700 Subject: [PATCH 1565/2680] drm/apple: admit bounded FRL color catalogs larger than 64 entries (cherry picked from commit d69f10fb528492afe2dcb2865cba675935580a15 on Eryk Wieliczko's eryk_m3_gpu branch; m3-dcp modules only, m3_lab scripts omitted) (cherry picked from commit 965bb84a0af78ea053f185420d20d618b2424a84) --- drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c index 5683ead3193f2..8ee674f7987e2 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c @@ -266,7 +266,7 @@ static int native_color(const struct native_blob *b, bool found = false, best_rgb = false; int ret = native_member(b, mode, "ColorModes", &array); - if (ret || array.type != 2 || array.count > 64) + if (ret || array.type != 2) return -EINVAL; pos = array.begin; for (u32 i = 0; i < array.count; i++) { From 589cf3a187e6bca38d9806ec7d3af95d94e6f3b8 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 29 Sep 2026 09:17:20 -0700 Subject: [PATCH 1566/2680] drm/apple: service M3 HDMI converter interrupts during FRL training (cherry picked from commit 9de669da4b1fe74e0a97abfd1dceb2b6a961ced9 on Eryk Wieliczko's eryk_m3_gpu branch; m3-dcp modules only, m3_lab scripts omitted) (cherry picked from commit 8463d8c5aee0bb303c51fcffcda9734d79a35def) --- .../apple/tools/m3-dcp/m3_dcpext_hpd_irq.h | 72 +++++++++++++++++++ .../tools/m3-dcp/m3_dcpext_native_adapter.h | 1 + .../apple/tools/m3-dcp/m3_dcpext_session.h | 11 ++- .../drm/apple/tools/m3-dcp/m3_hdmi_bridge.c | 35 +++++++++ .../drm/apple/tools/m3-dcp/m3_hdmi_control.h | 1 + 5 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_hpd_irq.h diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_hpd_irq.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_hpd_irq.h new file mode 100644 index 0000000000000..20925b6258680 --- /dev/null +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_hpd_irq.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* 14.6 DCPDPTXRemotePortEPClient table at DATA 0x603e48: command 9, + * 16-byte payload, InterruptRequestOccurred -> TEXT 0x37bd0 -> virtual 360. + * Same group 8 as HPD change. This is a DPCD service interrupt, not a cable + * disconnect. Give it separate pinned buffers: firmware may be waiting for + * this IRQ while the ordinary HPD or native modeset RPC is still pending. + */ +static u32 (*hdmi_irq_count)(void); +static u32 hdmi_irq_seen,hdmi_irq_sent,hdmi_irq_acked; +module_param(hdmi_irq_sent,uint,0400); +module_param(hdmi_irq_acked,uint,0400); +static u16 hdmi_irq_tag=0x8000; +static u8 *hdmi_irq_tx,*hdmi_irq_rx; +static u64 hdmi_irq_txd,hdmi_irq_rxd,hdmi_irq_deadline; +static bool hdmi_irq_pending; +/* GPIO47 did not signal the converter's FRL-ready/active service events on + * J514S. Poll the standard firmware service handler while connected as a + * fallback. 100 ms is below both the 500 ms ready and 450 ms training waits; + * only one request may be outstanding, with a bounded response deadline. + * No reads of converter registers or modifications of training policy. */ +static u64 hdmi_service_next; + +static int hdmi_irq_progress(void) +{ + u8 wire[70]={};u32 count;int ret; + if(!hdmi_irq_count)return 0; + if(hdmi_irq_pending) + return session_now(NULL)>=hdmi_irq_deadline ? -ETIMEDOUT : 0; + count=hdmi_irq_count(); + if(!port0_announced || rpc_stage<2){hdmi_irq_seen=count;return 0;} + if(session.phase!=DCPEXT_RUNNING)return 0; + if(READ_ONCE(desktop_stop) || desktop_disconnected)return 0; + if(count==hdmi_irq_seen && session_now(NULL)= 76 && payload[0] == 2 && payload[20] == 4 && !payload[21] && get_unaligned_le16(payload + 22) == 0x30 && !memcmp(payload + 40, port?"dispext1:dcpdptx-port-epic:0":"dispext0:dcpdptx-port-epic:0", sizeof("dispext0:dcpdptx-port-epic:0")) && @@ -538,7 +541,7 @@ static int afk_record(void *cookie, const u8 *payload, u32 size) (get_unaligned_le32(rpc_rx+64)!=(rpc_command==12?0x100:0) || get_unaligned_le32(rpc_rx+68)!=rpc_target))return -EPROTO; pr_info("m3_dcpext_session: DPTX command=%u reply accepted\n",rpc_command); - rpc_pending=false;rpc_tag++; + rpc_pending=false;rpc_tag=(rpc_tag+1)&0x7fff; if(desktop_hpd_rpc){ desktop_hpd_rpc=false; pr_info("m3_dcpext_session: runtime link stage=%u HPD=%u acknowledged\n",desktop_link_stage,desktop_hpd_value); @@ -732,6 +735,11 @@ static int session_run(void) * Diagnostics without a desktop do not require the bridge module. */ desktop_read_hpd=usb_c?link_hpd:symbol_get(m3_hdmi_bridge_hpd); if(!desktop_read_hpd)return -ENODEV; + if(!usb_c){ + hdmi_irq_count=symbol_get(m3_hdmi_bridge_irq_count); + if(!hdmi_irq_count)return -ENODEV; + hdmi_irq_seen=hdmi_irq_count(); + } ret=desktop_read_hpd(); if(ret<0 || (!usb_c && ret!=1))return ret<0?ret:-ENOLINK; } @@ -841,6 +849,7 @@ static int session_run(void) pr_warn("m3_dcpext_session: diagnostic ASC NMI fallback; expect firmware crash/reboot recovery\n"); writel(0x10,session_cpu+0x1004);writel(1,session_cpu+0x1014); } + ret = hdmi_irq_progress();if(ret)break; ret = dcpext_session_poll(&session); if (ret) break; if (session.phase == DCPEXT_RUNNING) { diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_bridge.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_bridge.c index 0912b258189ee..455fa42f04e47 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_bridge.c +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_bridge.c @@ -5,6 +5,8 @@ * exist. Do not toggle the reset, CEC or force-DFU lines. Retain ownership. */ #include +#include +#include #include #include #include @@ -18,6 +20,33 @@ static int hpd = -1; module_param(hpd, int, 0400); static struct device *owner; static struct gpio_desc *port, *bridge, *detect; +static unsigned int hpd_edges,hpd_irqs,hpd_last_low_us; +module_param(hpd_edges,uint,0400); +module_param(hpd_irqs,uint,0400); +module_param(hpd_last_low_us,uint,0400); +static u64 hpd_falling_ns; +static int hpd_irq; +/* Apple GPIO is MMIO-backed and cannot sleep. Classify short HPD pulses in + * hard IRQ context so scheduler latency cannot turn an IRQ into cable loss. + * Firmware RPCs run in the serialized session context, never from here. + */ +static irqreturn_t bridge_hpd_edge(int irq,void *cookie) +{ + u64 now=ktime_get_ns(),low; + int level=gpiod_get_value(detect); + if(level<0)return IRQ_HANDLED; + hpd_edges++; + if(!level){hpd_falling_ns=now;return IRQ_HANDLED;} + if(!hpd_falling_ns)return IRQ_HANDLED; + low=now-hpd_falling_ns;hpd_falling_ns=0; + WRITE_ONCE(hpd_last_low_us,min_t(u64,low/1000,U32_MAX)); + if(low>=100000 && low<=2000000)WRITE_ONCE(hpd_irqs,hpd_irqs+1); + return IRQ_HANDLED; +} +u32 m3_hdmi_bridge_irq_count(void); +u32 m3_hdmi_bridge_irq_count(void) +{return READ_ONCE(hpd_irqs);} +EXPORT_SYMBOL_GPL(m3_hdmi_bridge_irq_count); /* The session owns firmware notifications; this module owns the GPIO. */ int m3_hdmi_bridge_hpd(void); int m3_hdmi_bridge_hpd(void) @@ -52,6 +81,12 @@ static int __init probe(void) if (gpiod_get_direction(port) != 0 || gpiod_get_direction(bridge) != 0) { ret=-EINVAL;goto done; } + if(gpiod_cansleep(detect)){ret=-EOPNOTSUPP;goto done;} + hpd_irq=gpiod_to_irq(detect); + if(hpd_irq<0){ret=hpd_irq;goto done;} + ret=devm_request_irq(owner,hpd_irq,bridge_hpd_edge, + IRQF_TRIGGER_RISING|IRQF_TRIGGER_FALLING,"m3-hdmi-hpd",owner); + if(ret)goto done; p=gpiod_get_value_cansleep(port);b=gpiod_get_value_cansleep(bridge); hpd=gpiod_get_value_cansleep(detect); pr_info("m3_hdmi_bridge: before port=%d bridge=%d hpd=%d\n",p,b,hpd); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_control.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_control.h index 93033bed047ae..1771deaf80e2b 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_control.h +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_control.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only OR MIT */ #ifndef M3_HDMI_CONTROL_H #define M3_HDMI_CONTROL_H +u32 m3_hdmi_bridge_irq_count(void); bool m3_hdmi_phy_ready(void); int m3_hdmi_phy_deactivate(void); int m3_hdmi_phy_activate(void); From 76a5ca9e1de269b753522f8ccf233444cbad0da8 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 29 Sep 2026 09:17:20 -0700 Subject: [PATCH 1567/2680] drm/apple: enable firmware-qualified DSC modes on M3 HDMI (cherry picked from commit c6c119022edf2eae7ff7c58de97818ab54fe87a8 on Eryk Wieliczko's eryk_m3_gpu branch; m3-dcp modules only, m3_lab scripts omitted) (cherry picked from commit 68b58b8882e14c50172b7a4c851f10092356132e) --- .../apple/tools/m3-dcp/m3_dcpext_connector.c | 2 +- .../tools/m3-dcp/m3_dcpext_kms_adapter.h | 2 +- .../drm/apple/tools/m3-dcp/m3_dcpext_modes.c | 72 ++++++++++++++----- .../drm/apple/tools/m3-dcp/m3_dcpext_modes.h | 7 +- .../drm/apple/tools/m3-dcp/m3_dcpext_native.c | 15 ++++ .../drm/apple/tools/m3-dcp/m3_dcpext_native.h | 1 + .../tools/m3-dcp/m3_dcpext_native_adapter.h | 12 +++- 7 files changed, 87 insertions(+), 24 deletions(-) diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.c index 50e761e062eba..3ee61e850d615 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.c +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.c @@ -295,7 +295,7 @@ static int publish_snapshot(struct m3_dcpext_connector *c, u64 generation, if (native) { for (u32 i = 0; i < count; i++) { u32 rate; - if (m3_dcpext_mode_validate_link(&native[i].geometry, M3_DCPEXT_HBR3_4LANE_KBPS, &rate) || + if (m3_dcpext_native_mode_validate(&native[i], M3_DCPEXT_HBR3_4LANE_KBPS, &rate) || !native[i].hsync || !native[i].vsync || (u64)native[i].geometry.width + native[i].hfront + native[i].hsync > native[i].geometry.htotal || (u64)native[i].geometry.height + native[i].vfront + native[i].vsync > native[i].geometry.vtotal) diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms_adapter.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms_adapter.h index 283c96181c677..73606a3714868 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms_adapter.h +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms_adapter.h @@ -125,7 +125,7 @@ static int native_kms_publish(void) if(!blob)return 0; modes=kcalloc(M3_DCPEXT_NATIVE_MAX_MODES,sizeof(*modes),GFP_KERNEL); if(!modes){kvfree(blob);return native_kms_unavailable(-ENOMEM);} - ret=m3_dcpext_native_modes_parse_link(blob,bytes,modes,M3_DCPEXT_NATIVE_MAX_MODES,&count,link_payload_kbps());kvfree(blob); + ret=m3_dcpext_native_modes_parse_transport(blob,bytes,modes,M3_DCPEXT_NATIVE_MAX_MODES,&count,link_payload_kbps(),!usb_c && !hdmi_test_two_lane_hbr2);kvfree(blob); if(!ret)ret=m3_dcpext_native_mode_select(&wanted,modes,count,&index); if(ret){kfree(modes);return native_kms_unavailable(ret);} native_kms_generation=m3_dcpext_native_generation(native_client); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c index 8ee674f7987e2..b5e5c31fdedd5 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c @@ -27,22 +27,17 @@ int m3_dcpext_surface_validate(u32 width, u32 height, u32 pitch, return 0; } -int m3_dcpext_mode_validate_link(const struct m3_dcpext_mode *m, - u32 payload_kbps, u32 *fps_16_16) +static int mode_geometry_validate(const struct m3_dcpext_mode *m, u32 *fps_16_16) { u64 fps, total; - if (!payload_kbps || payload_kbps > M3_DCPEXT_HBR3_4LANE_KBPS || - !m || !fps_16_16 || !m->width || + if (!m || !fps_16_16 || !m->width || m->width > M3_DCPEXT_MAX_WIDTH || !m->height || m->height > M3_DCPEXT_MAX_HEIGHT || !m->clock_khz || m->htotal <= m->width || m->vtotal <= m->height || m->htotal > 65535 || m->vtotal > 65535) return -EINVAL; if (m->interlaced || m->doublescan) return -EOPNOTSUPP; - /* Link payload after 8b/10b encoding, 24 wire bits per RGB pixel. */ - if ((u64)m->clock_khz * 24 > payload_kbps) - return -ERANGE; total = (u64)m->htotal * m->vtotal; fps = ((u64)m->clock_khz * 1000 * 65536 + total / 2) / total; if (fps < 24 * 65536 || fps > 120 * 65536) @@ -51,6 +46,34 @@ int m3_dcpext_mode_validate_link(const struct m3_dcpext_mode *m, return 0; } +int m3_dcpext_mode_validate_link(const struct m3_dcpext_mode *m, + u32 payload_kbps, u32 *fps_16_16) +{ + int ret; + + if (!payload_kbps || payload_kbps > M3_DCPEXT_HBR3_4LANE_KBPS) + return -EINVAL; + ret = mode_geometry_validate(m, fps_16_16); + if (ret) + return ret; + return (u64)m->clock_khz * 24 > payload_kbps ? -ERANGE : 0; +} + +int m3_dcpext_native_mode_validate(const struct m3_dcpext_native_mode *m, + u32 payload_kbps, u32 *fps_16_16) +{ + if (!m) + return -EINVAL; + if (!m->dsc) + return m3_dcpext_mode_validate_link(&m->geometry, payload_kbps, fps_16_16); + /* DSC admission is limited to the HDMI HBR3 route. The current firmware + * publication must explicitly require DSC and must NOT mark the color + * unsafe. DCP owns its encoder/PPS and the converter's DSC decompressor. */ + if (payload_kbps != M3_DCPEXT_HBR3_4LANE_KBPS) + return -EOPNOTSUPP; + return mode_geometry_validate(&m->geometry, fps_16_16); +} + int m3_dcpext_mode_validate(const struct m3_dcpext_mode *m, u32 *fps_16_16) { return m3_dcpext_mode_validate_link(m, M3_DCPEXT_MAX_PAYLOAD_KBPS, fps_16_16); @@ -259,7 +282,7 @@ static int native_color_excluded(const struct native_blob *b, } static int native_color(const struct native_blob *b, - const struct native_value *mode, u32 *id) + const struct native_value *mode, u32 *id, bool allow_dsc, bool *dsc) { struct native_value array, color; u32 pos, budget = b->bytes / 4, best_score = 0; @@ -272,6 +295,7 @@ static int native_color(const struct native_blob *b, for (u32 i = 0; i < array.count; i++) { u32 candidate, score, depth, encoding, eotf, range, colorimetry; bool virtual, rgb, yuv; + int compressed; ret = native_value_read(b, &pos, 0, &budget, &color); if (ret || native_bool(b, &color, "IsVirtual", &virtual)) @@ -294,15 +318,19 @@ static int native_color(const struct native_blob *b, if (depth != 8 || eotf || (!rgb && !yuv)) continue; ret = native_color_excluded(b, mode, "UnsafeColorElementIDs", candidate); - if (!ret) - ret = native_color_excluded(b, mode, "DSCRequiredColorElementIDs", candidate); if (ret < 0) return ret; if (ret) continue; + compressed = native_color_excluded(b, mode, "DSCRequiredColorElementIDs", candidate); + if (compressed < 0) + return compressed; + if (compressed && !allow_dsc) + continue; if (!found || (rgb && !best_rgb) || (rgb == best_rgb && (score > best_score || (score == best_score && candidate < *id)))) { *id = candidate; + *dsc = !!compressed; best_score = score; best_rgb = rgb; found = true; @@ -350,7 +378,7 @@ static int native_pipe_count(const struct native_blob *b, } static int native_mode(const struct native_blob *b, const struct native_value *v, - struct m3_dcpext_native_mode *out, u32 payload_kbps) + struct m3_dcpext_native_mode *out, u32 payload_kbps, bool allow_dsc) { struct m3_dcpext_mode *m = &out->geometry; u32 horizontal_rate, rate, pipes_h, pipes_v, ignored; @@ -386,15 +414,16 @@ static int native_mode(const struct native_blob *b, const struct native_value *v return -EOPNOTSUPP; clock = (u64)m->htotal * m->vtotal * rate; m->clock_khz = (clock + 32768000) / 65536000; - ret = m3_dcpext_mode_validate_link(m, payload_kbps, &ignored); + ret = native_color(b, v, &out->color_id, allow_dsc, &out->dsc); if (ret) - return -EOPNOTSUPP; - return native_color(b, v, &out->color_id); + return ret; + ret = m3_dcpext_native_mode_validate(out, payload_kbps, &ignored); + return ret ? -EOPNOTSUPP : 0; } -int m3_dcpext_native_modes_parse_link(const void *data, u32 bytes, +int m3_dcpext_native_modes_parse_transport(const void *data, u32 bytes, struct m3_dcpext_native_mode *modes, - u32 capacity, u32 *count, u32 payload_kbps) + u32 capacity, u32 *count, u32 payload_kbps, bool allow_dsc) { struct native_blob blob = { .data = data, .bytes = bytes }; struct native_value root, value; @@ -406,6 +435,7 @@ int m3_dcpext_native_modes_parse_link(const void *data, u32 bytes, return -EINVAL; *count = 0; if (!payload_kbps || payload_kbps > M3_DCPEXT_HBR3_4LANE_KBPS || + (allow_dsc && payload_kbps != M3_DCPEXT_HBR3_4LANE_KBPS) || !data || !modes || !capacity || capacity > M3_DCPEXT_NATIVE_MAX_MODES || bytes < 8 || bytes > M3_DCPEXT_MAX_PROPERTY_BYTES || get_unaligned_le32(data) != 0xd3) return -EINVAL; @@ -418,7 +448,7 @@ int m3_dcpext_native_modes_parse_link(const void *data, u32 bytes, for (u32 i = 0; i < root.count; i++) { ret = native_value_read(&blob, &pos, 0, &budget, &value); if (!ret) - ret = native_mode(&blob, &value, &mode, payload_kbps); + ret = native_mode(&blob, &value, &mode, payload_kbps, allow_dsc); if (ret == -EOPNOTSUPP) continue; if (ret) @@ -434,6 +464,10 @@ int m3_dcpext_native_modes_parse_link(const void *data, u32 bytes, return 0; } +int m3_dcpext_native_modes_parse_link(const void *data,u32 bytes, + struct m3_dcpext_native_mode *modes,u32 capacity,u32 *count,u32 payload_kbps) +{return m3_dcpext_native_modes_parse_transport(data,bytes,modes,capacity,count,payload_kbps,false);} + int m3_dcpext_native_modes_parse(const void *data, u32 bytes, struct m3_dcpext_native_mode *modes, u32 capacity, u32 *count) @@ -454,14 +488,14 @@ int m3_dcpext_native_mode_select(const struct m3_dcpext_native_mode *requested, if (!requested || !modes || !selected || count > M3_DCPEXT_NATIVE_MAX_MODES) return -EINVAL; g = &requested->geometry; - ret = m3_dcpext_mode_validate_link(g, M3_DCPEXT_HBR3_4LANE_KBPS, &rate); + ret = mode_geometry_validate(g, &rate); if (ret) return ret; for (u32 i = 0; i < count; i++) { const struct m3_dcpext_native_mode *m = &modes[i]; u32 delta, candidate_rate; - if (m3_dcpext_mode_validate_link(&m->geometry, M3_DCPEXT_HBR3_4LANE_KBPS, &candidate_rate) || + if (m3_dcpext_native_mode_validate(m, M3_DCPEXT_HBR3_4LANE_KBPS, &candidate_rate) || g->width != m->geometry.width || g->height != m->geometry.height || g->htotal != m->geometry.htotal || g->vtotal != m->geometry.vtotal || requested->hfront != m->hfront || requested->hsync != m->hsync || diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.h index 2091e04b59c54..5266a40dc5243 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.h +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.h @@ -30,7 +30,7 @@ struct m3_dcpext_native_mode { struct m3_dcpext_mode geometry; u32 hfront, hsync, vfront, vsync; u32 color_id, timing_id; - bool hpositive, vpositive, preferred; + bool hpositive, vpositive, preferred, dsc; }; /* Decode the current native TimingElements publication. Only real, uncompressed @@ -44,6 +44,11 @@ int m3_dcpext_native_modes_parse(const void *blob, u32 bytes, int m3_dcpext_native_modes_parse_link(const void *blob, u32 bytes, struct m3_dcpext_native_mode *modes, u32 capacity, u32 *count, u32 payload_kbps); +/* Opt-in DSC transport: current firmware-safe DSC-required color IDs only. */ +int m3_dcpext_native_modes_parse_transport(const void *blob,u32 bytes, + struct m3_dcpext_native_mode *modes,u32 capacity,u32 *count,u32 payload_kbps,bool allow_dsc); +int m3_dcpext_native_mode_validate(const struct m3_dcpext_native_mode *mode, + u32 payload_kbps,u32 *fps_16_16); int m3_dcpext_mode_validate_link(const struct m3_dcpext_mode *mode, u32 payload_kbps, u32 *fps_16_16); /* Match full EDID timing geometry to a uniquely closest native timing. */ diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.c index 1c5d08a5048a8..a06108fbab3b7 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.c +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.c @@ -736,6 +736,21 @@ void *m3_dcpext_native_property(struct m3_dcpext_native *dcp, const char *key, u return copy; } +bool m3_dcpext_native_property_ready(struct m3_dcpext_native *dcp,const char *key) +{ + bool ready = false; + + mutex_lock(&dcp->lock); + for (u32 i = 0; i < dcp->raw_count; i++) { + if (!dcp->raw[i].service && !strcmp(dcp->raw[i].key, key)) { + ready = dcp->raw[i].size != 0; + break; + } + } + mutex_unlock(&dcp->lock); + return ready; +} + void m3_dcpext_native_analytics_show(struct m3_dcpext_native *dcp, struct seq_file *seq) { mutex_lock(&dcp->lock); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.h index b3b9ed50e8e4f..0431c2c421eda 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.h +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.h @@ -26,4 +26,5 @@ int m3_dcpext_native_mode(struct m3_dcpext_native *, u32 color, u32 timing); int m3_dcpext_native_power(struct m3_dcpext_native *, bool on); u64 m3_dcpext_native_generation(struct m3_dcpext_native *); void m3_dcpext_native_invalidate_sink(struct m3_dcpext_native *); +bool m3_dcpext_native_property_ready(struct m3_dcpext_native *,const char *key); #endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_adapter.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_adapter.h index b0624d0edff0c..8035f4b0d64c2 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_adapter.h +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_adapter.h @@ -118,13 +118,21 @@ static int native_frame_progress(void) if (!ret) {native_frame_done=true;pr_info("m3_dcpext_native: frame hold and power-off completed\n");} return ret; } - if(native_kms_reconnecting())return native_kms_publish(); + if(native_kms_reconnecting()){ + /* Reconnect has no bootstrap frame. Fresh metadata proves discovery has + * completed, so permit EDID acquisition before publishing the connector. + * Invalidation clears these properties; never reuse the previous sink. */ + if(m3_dcpext_native_property_ready(native_client,"TimingElements") && + m3_dcpext_native_property_ready(native_client,"DisplayAttributes")) + av_read_ready=true; + return native_kms_publish(); + } blob=m3_dcpext_native_property(native_client,"TimingElements",&bytes); if (IS_ERR(blob)) return native_kms_unavailable(PTR_ERR(blob)); if (!blob) return 0; modes=kcalloc(M3_DCPEXT_NATIVE_MAX_MODES,sizeof(*modes),GFP_KERNEL); if (!modes) {kvfree(blob);return native_kms_unavailable(-ENOMEM);} - ret=m3_dcpext_native_modes_parse_link(blob,bytes,modes,M3_DCPEXT_NATIVE_MAX_MODES,&count,link_payload_kbps()); + ret=m3_dcpext_native_modes_parse_transport(blob,bytes,modes,M3_DCPEXT_NATIVE_MAX_MODES,&count,link_payload_kbps(),!usb_c && !hdmi_test_two_lane_hbr2); if (!ret) ret=m3_dcpext_native_mode_select(&wanted,modes,count,&selected); if(ret){kfree(modes);kvfree(blob);return native_kms_unavailable(ret);} if (!ret) { From 635e69fa3e132bca38186b96ab1503997c0d7896 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 29 Sep 2026 09:17:20 -0700 Subject: [PATCH 1568/2680] drm/apple: count bounded DCP syslog wraps during initialization (cherry picked from commit 2e076b0a3459d9473e1511df1c6f48de9e5602c1 on Eryk Wieliczko's eryk_m3_gpu branch; m3-dcp modules only, m3_lab scripts omitted) (cherry picked from commit ce774d11bb76b27d08212eb0610e565fa54f6f31) --- drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h index b3f621b23a164..19e36e50de767 100644 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h +++ b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h @@ -33,6 +33,8 @@ static const struct m3_usbc_route_ops *usbc_ops; #include "dcpext_syslog.h" static struct dcpext_syslog syslog_capture; static struct debugfs_blob_wrapper syslog_blob; +static unsigned int syslog_wraps; +module_param(syslog_wraps,uint,0400); static struct dcpext_session session; static DECLARE_COMPLETION(desktop_started); static DECLARE_COMPLETION(desktop_finished); @@ -612,7 +614,7 @@ static int session_dispatch(u64 endpoint,u64 message) struct dcpext_buffer *b=&session.buffers[2]; u64 offset=b->dva & ~BIT_ULL(40); int ret; if (b->ready && (offsetsession_used || b->size>session_used-(offset-LEASE_DVA))) return -ERANGE; - if(session.runtime && syslog_capture.used>sizeof(syslog_capture.capture)-512)syslog_capture.used=0; + if(syslog_capture.used>sizeof(syslog_capture.capture)-512){syslog_capture.used=0;syslog_wraps++;} ret=dcpext_syslog_observe(&syslog_capture,message,b->ready?pool+offset-LEASE_DVA:NULL,b->size); if(ret)return ret; } From 10030e5131585280126c9bd5b7701c29cd5aa821 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 00:21:43 -0700 Subject: [PATCH 1569/2680] Remove the M3 lab tooling that no build uses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The M3 tree carried its bring-up tooling in the kernel tree: the standalone DCP/HDMI module package under drivers/gpu/drm/apple/tools/, the tools/asahi-m3 and tools/testing/selftests/drm/apple scripts, generated replay schemas and other drm/asahi sources no module declares, and three bring-up notes outside the documentation toctree. No Makefile, Kconfig, mod declaration or include refers to any of them (101 files, 15,893 lines). [ryan: adapted to 7.2.10 — only the drivers/gpu/drm/apple/tools/m3-dcp package and tools/testing/selftests/drm/apple are removed here; the Documentation/gpu/asahi-m3-*.rst notes, tools/asahi-m3 and the drm/asahi sources it deletes are not in this tree (other buckets).] Co-Authored-By: Claude Opus 5.5 (cherry picked from commit eb71a4516a33b704027fc8e5df2796921433780a) --- drivers/gpu/drm/apple/tools/m3-dcp/Makefile | 25 - drivers/gpu/drm/apple/tools/m3-dcp/README.md | 192 ---- .../apple/tools/m3-dcp/dcpext_afk_channel.h | 212 ---- .../drm/apple/tools/m3-dcp/dcpext_session.c | 343 ------- .../drm/apple/tools/m3-dcp/dcpext_session.h | 96 -- .../drm/apple/tools/m3-dcp/dcpext_syslog.h | 49 - .../drm/apple/tools/m3-dcp/m3_dcp_bridge.c | 842 --------------- .../drm/apple/tools/m3-dcp/m3_dcp_bridge.h | 34 - .../apple/tools/m3-dcp/m3_dcp_brightness.h | 33 - .../apple/tools/m3-dcp/m3_dcp_clock_restore.c | 39 - .../gpu/drm/apple/tools/m3-dcp/m3_dcp_hold.c | 57 -- .../gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.c | 734 ------------- .../gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.h | 9 - .../drm/apple/tools/m3-dcp/m3_dcp_native.c | 739 -------------- .../drm/apple/tools/m3-dcp/m3_dcp_native.h | 23 - .../drm/apple/tools/m3-dcp/m3_dcp_parser.c | 39 - .../drm/apple/tools/m3-dcp/m3_dcp_ptd_vote.c | 114 --- .../gpu/drm/apple/tools/m3-dcp/m3_dcp_query.c | 65 -- .../gpu/drm/apple/tools/m3-dcp/m3_dcp_regs.c | 62 -- .../apple/tools/m3-dcp/m3_dcp_rtkit_main.c | 434 -------- .../gpu/drm/apple/tools/m3-dcp/m3_dcp_state.h | 15 - .../gpu/drm/apple/tools/m3-dcp/m3_dcp_vote.c | 37 - .../drm/apple/tools/m3-dcp/m3_dcpext_audio.h | 193 ---- .../apple/tools/m3-dcp/m3_dcpext_audio_api.h | 15 - .../tools/m3-dcp/m3_dcpext_audio_format.h | 190 ---- .../apple/tools/m3-dcp/m3_dcpext_audio_pcm.c | 343 ------- .../gpu/drm/apple/tools/m3-dcp/m3_dcpext_av.h | 69 -- .../apple/tools/m3-dcp/m3_dcpext_connector.c | 422 -------- .../apple/tools/m3-dcp/m3_dcpext_connector.h | 68 -- .../drm/apple/tools/m3-dcp/m3_dcpext_dma.c | 330 ------ .../drm/apple/tools/m3-dcp/m3_dcpext_frame.h | 187 ---- .../apple/tools/m3-dcp/m3_dcpext_hpd_irq.h | 72 -- .../drm/apple/tools/m3-dcp/m3_dcpext_iboot.h | 114 --- .../drm/apple/tools/m3-dcp/m3_dcpext_kms.c | 401 -------- .../drm/apple/tools/m3-dcp/m3_dcpext_kms.h | 30 - .../tools/m3-dcp/m3_dcpext_kms_adapter.h | 192 ---- .../drm/apple/tools/m3-dcp/m3_dcpext_modes.c | 537 ---------- .../drm/apple/tools/m3-dcp/m3_dcpext_modes.h | 70 -- .../drm/apple/tools/m3-dcp/m3_dcpext_native.c | 812 --------------- .../drm/apple/tools/m3-dcp/m3_dcpext_native.h | 30 - .../tools/m3-dcp/m3_dcpext_native_adapter.h | 175 ---- .../tools/m3-dcp/m3_dcpext_native_query.h | 37 - .../drm/apple/tools/m3-dcp/m3_dcpext_power.c | 66 -- .../apple/tools/m3-dcp/m3_dcpext_property.h | 60 -- .../drm/apple/tools/m3-dcp/m3_dcpext_regs.c | 123 --- .../drm/apple/tools/m3-dcp/m3_dcpext_rpc.c | 604 ----------- .../drm/apple/tools/m3-dcp/m3_dcpext_rpc.h | 54 - .../drm/apple/tools/m3-dcp/m3_dcpext_rtkit.c | 3 - .../apple/tools/m3-dcp/m3_dcpext_rtkit_main.c | 2 - .../tools/m3-dcp/m3_dcpext_secondary_main.c | 3 - .../tools/m3-dcp/m3_dcpext_secondary_power.c | 3 - .../apple/tools/m3-dcp/m3_dcpext_session.h | 961 ------------------ .../gpu/drm/apple/tools/m3-dcp/m3_epic_tx.h | 54 - .../drm/apple/tools/m3-dcp/m3_hdmi_bridge.c | 120 --- .../drm/apple/tools/m3-dcp/m3_hdmi_control.h | 26 - .../gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy.c | 192 ---- .../apple/tools/m3-dcp/m3_hdmi_phy_common.h | 65 -- .../apple/tools/m3-dcp/m3_hdmi_phy_dp-drive.h | 32 - .../apple/tools/m3-dcp/m3_hdmi_phy_dp-main.h | 159 --- .../apple/tools/m3-dcp/m3_hdmi_phy_dp-pll.h | 95 -- .../tools/m3-dcp/m3_hdmi_phy_lane-power.h | 75 -- .../apple/tools/m3-dcp/m3_hdmi_phy_power.h | 81 -- .../gpu/drm/apple/tools/m3-dcp/m3_hdmi_xbar.c | 68 -- .../gpu/drm/apple/tools/m3-dcp/m3_hdmi_xbar.h | 145 --- .../drm/apple/tools/m3-dcp/m3_typec_probe.c | 92 -- .../gpu/drm/apple/tools/m3-dcp/m3_usbc_phy.c | 283 ------ .../drm/apple/tools/m3-dcp/m3_usbc_ptd_vote.c | 3 - .../drm/apple/tools/m3-dcp/syncobj-probe.py | 84 -- .../apple/tools/m3-dcp/validation/.gitignore | 11 - .../apple/tools/m3-dcp/validation/README.md | 18 - .../m3-dcp/validation/dcpext_session_test.c | 478 --------- .../validation/panel-dpi-20260921/README.md | 27 - .../validation/panel-dpi-20260921/inspect.py | 19 - .../panel-dpi-20260921/screenshot.sha256 | 1 - tools/testing/selftests/drm/apple/README | 36 - .../selftests/drm/apple/test-dcpext-copy.py | 194 ---- .../drm/apple/test-dcpext-present.py | 239 ----- .../selftests/drm/apple/test-iboot-wire.py | 185 ---- 78 files changed, 13141 deletions(-) delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/Makefile delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/README.md delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/dcpext_afk_channel.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/dcpext_session.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/dcpext_session.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/dcpext_syslog.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_bridge.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_bridge.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_brightness.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_clock_restore.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_hold.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_native.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_native.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_parser.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_ptd_vote.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_query.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_regs.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_rtkit_main.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_state.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_vote.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_api.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_format.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_pcm.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_av.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_dma.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_frame.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_hpd_irq.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_iboot.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms_adapter.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_adapter.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_query.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_power.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_property.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_regs.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rpc.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rpc.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rtkit.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rtkit_main.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_secondary_main.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_secondary_power.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_epic_tx.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_bridge.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_control.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_common.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-drive.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-main.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-pll.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_lane-power.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_power.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_xbar.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_xbar.h delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_typec_probe.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_usbc_phy.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/m3_usbc_ptd_vote.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/syncobj-probe.py delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/validation/.gitignore delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/validation/README.md delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/validation/dcpext_session_test.c delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/README.md delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/inspect.py delete mode 100644 drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/screenshot.sha256 delete mode 100644 tools/testing/selftests/drm/apple/README delete mode 100644 tools/testing/selftests/drm/apple/test-dcpext-copy.py delete mode 100644 tools/testing/selftests/drm/apple/test-dcpext-present.py delete mode 100644 tools/testing/selftests/drm/apple/test-iboot-wire.py diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/Makefile b/drivers/gpu/drm/apple/tools/m3-dcp/Makefile deleted file mode 100644 index 817e86bbdc95a..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only OR MIT -obj-m += m3_dcp_rtkit.o -m3_dcp_rtkit-y := m3_dcp_rtkit_main.o m3_dcp_bridge.o m3_dcp_native.o m3_dcp_kms.o m3_dcp_parser.o -obj-m += m3_dcp_regs.o -obj-m += m3_dcp_hold.o -obj-m += m3_dcp_vote.o -obj-m += m3_dcp_ptd_vote.o -obj-m += m3_dcp_clock_restore.o -obj-m += m3_dcp_query.o -obj-m += m3_dcpext_regs.o -obj-m += m3_dcpext_power.o -obj-m += m3_dcpext_dma.o -obj-m += m3_dcpext_rtkit.o -m3_dcpext_rtkit-y := m3_dcpext_rtkit_main.o m3_dcpext_rpc.o m3_dcpext_native.o m3_dcpext_modes.o m3_dcpext_kms.o m3_dcpext_connector.o -obj-m += m3_hdmi_phy.o -obj-m += m3_hdmi_bridge.o -obj-m += m3_hdmi_xbar.o -obj-m += m3_typec_probe.o -obj-m += m3_usbc_ptd_vote.o -obj-m += m3_dcpext_secondary_power.o -obj-m += m3_dcpext_secondary.o -m3_dcpext_secondary-y := m3_dcpext_secondary_main.o m3_dcpext_rpc.o m3_dcpext_native.o m3_dcpext_modes.o m3_dcpext_kms.o m3_dcpext_connector.o -obj-m += m3_usbc_phy.o - -obj-m += m3_dcpext_audio_pcm.o diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/README.md b/drivers/gpu/drm/apple/tools/m3-dcp/README.md deleted file mode 100644 index 9f88d810ecb2a..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/README.md +++ /dev/null @@ -1,192 +0,0 @@ -# J514S native DCP checkpoint - -Explicit-load native display prototype for the M3 Pro J514S, using the inherited -DCP firmware UUID DDF38191-93B3-324A-BC8F-643006F5AC82. The corresponding captured -TEXT SHA256 is a0af36d10a8abf2435918ba360e697529ddfaed56b89f9466e76d566b296e876. -No firmware image is included here. Do not generalize to another M3 or firmware. - -The isolated m1n1 handoff reserves every DCP firmware segment and publishes the -locked display SID0 and PIODMA SID4 mappings. The power-hold helper uses PMGR's -existing syscon to retain DISP_SYS/FE/CPU before PMP startup; forcing CPU_RUN -on a suspended DCP is not a supported recovery method. Reset by reboot. - -Build against the configured UNSTABLE kernel with ARCH=arm64 LLVM=1 and this -directory as M=. Requires DRM_GEM_DMA_HELPER, BACKLIGHT_CLASS_DEVICE and the normal Apple RTKit, DART, -mailbox and PMP providers. A partial kernel build may leave Module.symvers -without the already-built drm_dma_helper exports; restore that dependency's -symbol metadata before modpost. Do not bypass missing-symbol checks. - -Load m3_dcp_hold before the M3 PMP overlay/thermal/bridge modules. Under the M3 -watchdog/thermal guard, load drm_dma_helper and backlight, then m3_dcp_rtkit kernel_client=1. -After successful startup, root may write background or kms to the control -parameter. KMS takeover removes simpledrm and exposes one native3024x1964 eDP -mode. Stop the display manager before a manual takeover. No automatic probe or -module unload: firmware-visible allocations remain pinned until reboot. - -Hardware evidence on2026-09-18 includes successful service initialization, -solid-background completion, three real KMS flips over three prefilled buffers, -all5,939,136 expected snapshot pixels, and the normal Plasma greeter on native -DCP after allowing llvmpipe's3072x1984 padded allocation. The visible framebuffer -and plane must remain3024x1964. Preferred timing metadata says120Hz; the bounded -flip test measured60FPS. A host snapshot is not optical panel readback. - -This is a fixed-mode display checkpoint, not a full accelerated-desktop claim. -GPU rendering qualification is separate. Linear XRGB/ARGB only; display domains -are held active, runtime display power management remains unqualified, and the -initial firmware panel charge-parity diagnostic is retained in evidence. A -plane disable blanks to black; panel power-off has not been qualified. - -Workspace tools/m3-dcp contains the guarded bootstrap, persistent recovery latch, -offline firmware decoders, lifecycle capture and userspace flip/login checks. -Evidence: _provenance/m3-dcp-20260918 in the source-collection workspace. - -## Visible panel fix and reboot validation (2026-09-18) - -The earlier checkpoint established software completions and host-buffer pixels, -but the user reported a dark physical panel. PMP startup alone caused the -blackout. The legacy device-power mailbox only returns a constant ACK on this -PMP v2 firmware; it does not establish the requested DISP power state. - -Captured macOS PTD state and driver analysis identified request index280 and -ACK index284. The loader now optionally seeds DISP bit7 and ANS bit16 through -an ioremap_np mapping before starting PMP. DISP requires ACK; ANS does not. -The native client verifies status1 and the DISP request/ACK before takeover. -Use the recorded lifecycle --hold --ptd-seed sequence, not the old mailbox. - -The user confirmed visible native3024x1964 login output both after takeover -and after a normal UNSTABLE reboot. Guarded login checks observed new frames -and zero reported failures; all five PMP channels remained below47C. This is -visible internal-display support, not accelerated-desktop qualification. - -The exact bootstrap/diagnostic source snapshot is in -../../asahi/m3_lab/tools/m3-dcp; the PMP loader is in its sibling -m3-boot-export. Snapshot hashes and validation summaries are under m3_lab. -Runtime deployment remains /root/m3-gpu-tools: lifecycle.py is installed as -m3-dcp-lifecycle.py and start.py as m3-dcp-start.py; boot.py, boot-client.py, -control.py and m3-dcp.service retain their names. The existing guard and -watchdog helpers are required. Preserve the failure latch, normal display -manager, original Asahi fallback and five-second GRUB menu. - -## Panel dimensions (2026-09-21) - -The J514S firmware's serialized DisplayAttributes contains product identity but -no image dimensions. The board's DCP panel child therefore supplies width-mm=302 -and height-mm=196, following the existing M1/M2 panel binding. These are the -full 3024x1964 panel dimensions at Apple's specified 254 ppi, rounded to whole -millimetres (https://support.apple.com/en-ie/117736). Do not use the notch-hidden -189 mm height with full-height native scanout. - -KMS copies these dimensions into its preferred mode and DRM connector display -information on each mode probe. An old DT without panel data retains display -operation and emits a warning rather than fabricating dimensions or an EDID. -There is no desktop scale in the driver. Deploy both the module and updated -J514S DT in the m1n1 bundle. tools/m3-dcp/prepare-boot.py preserves the other -40 boards and U-Boot; verify the installed bundle before replacement. - -KWin preserves previously saved output configurations. Removing just the scale -key from an existing setup did not trigger automatic scale selection: KWin -retained 1x. To validate automatic selection on this single-panel installation, -stop the display manager, terminate the user session, wait for KWin to exit, -then archive kwinoutputconfig.json before restarting. Retain the archive for -restoring other output preferences; do not discard multi-monitor layouts. - -After a reboot into the corrected driver, DRM and KWin both reported 302x196 mm. -With the old setup archived, unmodified KWin 6.7.4 selected 1.7x automatically -(logical desktop 1779x1155) and saved that choice. No replacement scale was -supplied. KWin's current heuristic uses a 150-DPI target for internal outputs -without a detected laptop lid switch; its laptop-with-lid target is 125 DPI. -This distinction is userspace/input policy, not a reason to alter panel size. -The guarded real Plasma check, GPU rendering and screenshot passed. Evidence: -validation/panel-dpi-20260921 (raw screenshot retained in workspace build). - -## DRM timeline synchronization (2026-09-22) - -Internal and external prototype DRM devices now expose the core SYNCOBJ and -SYNCOBJ_TIMELINE features, as the main apple_drv.c already does. A display-only -node can import fences from the separate render node; it needs no local shader -engine for DRM core to implement timelines. Without these flags, KWin on J514S -did not advertise wp_linux_drm_syncobj_manager_v1. After the guarded normal -boot, it does, and the Vulkan game confirms explicit=1 instead of implicit sync. - -`syncobj-probe.py DISPLAY_NODE RENDER_NODE` is CPU-only: it checks capability -queries, exports/imports an opaque timeline in both directions, verifies timeout -before signaling point 7, then successful wait and eventfd notification. It -does not submit GPU work or count as GPU synchronization qualification. The -J514S card1/renderD128 probe passes both directions. Native Vulkan 0 A.D. runs -with real explicit-sync presentation under the normal M3 guard, at 98.102 FPS -in the short fixed-scene sample. The previous implicit diagnostic was 94.901; -this is not a matched sustained performance claim and remains below 120 FPS. - -Boot: cd0c4c47-1206-435b-a4b5-b33f8f3a1e73. Private evidence: -`build/m3-0ad-120/dcp-syncobj/` and `0ad-120-vk-explicit-sync/`. External module -variants build, but no external monitor is attached in this qualification. - -## Zero-nit internal-panel recovery (2026-09-24) - -The recurrent internal-dark/external-working state was retained on boot -`ab566889-62ce-4df2-b5d1-63ab03423a80`. KMS was active and completing frames, -while the qualified firmware snapshot reported backlight enabled, correction -1.0, and `last_backlight_nits = 0`. Direct writes to cached brightness fields -were ineffective; they are not a supported interface and were restored. - -Firmware analysis identified the M3 A407 brightness-update byte at 0x354 and -unaligned binary64 nits at 0x35c. M4 uses 0x35e for the latter. The M3 firmware -copies these to transaction+0x76a/+0x778 and converts nits into runtime property -19 through its normal backlight pipeline. A serialized A406/A407 transaction -requesting 140 nits completed successfully (swap 33762) and a subsequent RAM -snapshot reported 140 nits. There was no reboot, DCP reload or desktop restart. -The user subsequently confirmed both screens visible: the internal panel -recovered and the external monitor remained working. This is optical -confirmation of the live recovery, not validation of the unloaded replacement -driver or of repeated wake/hotplug cycles. - -The normal driver now exposes `apple-panel-bl` through the backlight class, -with an SDR range of 0..500 nits and an initial request of 140 nits. It sends -zero on CRTC disable and the retained request on enable. Explicit writes to -the backlight device re-send the request even if its value is unchanged, and -work with an idle compositor by retaining and re-submitting the active FB. -Power-on/CRTC-enable invalidate the last acknowledged brightness; only a -completed, successful swap updates that cache. Background-only diagnostic -commands retain the existing brightness. No raw RAM writes are part of this -implementation. - -Validation so far: the live 0-to-140 command above, an arm64 module build for -a 7.1.9 Asahi development kernel, and the actual production encoder tested against the -host IEEE754 implementation for all 501 levels, including unchanged bytes -outside the two wire fields, under ASan/UBSan. Run the offline test with: - -``` -python3 drivers/gpu/drm/asahi/m3_lab/tools/m3-dcp/test-brightness.py \ - drivers/gpu/drm/apple/tools/m3-dcp -``` - -The complete replacement driver has not been loaded in the preserved boot. -Backlight-class integration, repeated DPMS cycles and hotplug still need a -subsequent hardware validation; the initiating firmware power event has not -yet been identified. Do not describe this checkpoint as a fully qualified -wake/hotplug regression fix. - -One temporary diagnostic had an unrelated host-kernel BTI oops before its -first RPC: a private static driver function was called indirectly without a -BTI landing pad. Its abandoned mutex blocked internal submissions for about -422 seconds. The dead owner's address was verified from the register dump; -a later idle worker had reused that task allocation. A same-boot, pointer- -and birth-time-qualified recovery released that abandoned lock; ordinary KMS -submissions resumed. The successful diagnostic used a range-checked direct -branch. Neither diagnostic nor manual mutex recovery belongs in the driver. -The failed module remains pinned and the kernel is tainted until a future -reboot; no forced unload or reboot was attempted. All raw captures, temporary -helper sources/binaries and logs remain local under `build/m3-panel-wake` and -`tools/m3-dcp/live-brightness` in the source-collection workspace. -## Large HDMI timing publications (2026-09-29) - -A connected HDMI monitor published 57 timing records in 1,013,116 bytes. The -OSSerialize traversal exhausted the old fixed 65,536-value budget despite all -byte/container/depth limits being satisfied. That returned EINVAL before the -first frame; the automatic late-display guard retained the watchdog and the -machine reset. Budgets now derive from the publication size (one tag consumes -at least four bytes). The 1 MiB publication, nesting and container bounds are -unchanged. Offline parsing now admits 29 modes under the existing HBR/SDR -policy. The host sanitizer regression includes an oversized value-count fixture -and the real capture, alongside malformed/truncated inputs. No mode bandwidth -or color restrictions are relaxed by this fix. Hardware validation follows. diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_afk_channel.h b/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_afk_channel.h deleted file mode 100644 index 309fe5320d3d5..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_afk_channel.h +++ /dev/null @@ -1,212 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* Bounded additional AFK channel, independent state for concurrent services. - * Adapted from this probe's Asahi-derived afk_probe ring/handshake code. - * Authors/references: Sven Peter, Asahi Linux Contributors; dynamic ring - * block layout from Eileen Yoon / m1n1 rbep.py. No hardware ownership here: - * caller retains every allocation until ALL endpoints and RTKit quiesce. - * Single serialized caller; callbacks must not reenter this engine. - */ -#ifndef DCPEXT_AFK_CHANNEL_H -#define DCPEXT_AFK_CHANNEL_H -/* Announcement-only M3 copy: the unused M4 ring writer is omitted. */ -struct afkc; -struct afkc_ops { - int (*send)(void *cookie, u8 endpoint, u64 message); - int (*allocate)(void *cookie, u32 size, u8 **buffer, u64 *dva); - int (*record)(void *cookie, const u8 *payload, u32 size); -}; -struct afkc_ring { - u8 *header, *data; - u32 size, block, offset, extent; -}; -struct afkc { - const struct afkc_ops *ops; - void *cookie; - u8 endpoint; - bool started, running, stopping, stopped; - bool streaming; /* Desktop owner accounts traffic across endpoints. */ - int error; - u32 messages, buffer_size, tag, capture_size; - u32 max_payload; /* Zero preserves the 256-byte default; explicit maximum4096. */ - u8 *buffer; - struct afkc_ring rings[2]; - u8 capture[32768]; -}; - -static int afkc_fail(struct afkc *c, int error) -{ - if (!c->error) - c->error = error; - return c->error; -} - -static int afkc_send(struct afkc *c, u32 type, u64 payload) -{ - int ret = c->ops->send(c->cookie, c->endpoint, (u64)type << 48 | payload); - return ret ? afkc_fail(c, ret) : 0; -} - -static int afkc_start(struct afkc *c, const struct afkc_ops *ops, void *cookie, u8 ep) -{ - int ret; - if (c->started || !ops || !ops->send || !ops->allocate || !ops->record || - ep < 0x20 || ep > 0x3f) - return -EINVAL; - c->ops = ops; - c->cookie = cookie; - c->endpoint = ep; - c->started = true; /* Published start may take effect even on send error. */ - ret = ops->send(cookie, 0, (5ULL << 52) | ((u64)ep << 32) | 2); - return ret ? afkc_fail(c, ret) : afkc_send(c, 0x80, 0); -} - -static bool afkc_tx_empty(struct afkc *c) -{ - struct afkc_ring *r = &c->rings[0]; - u32 rd, wr; - if (!r->header) - return false; - rd = READ_ONCE(*(u32 *)(r->header + r->block)); - wr = READ_ONCE(*(u32 *)(r->header + 2 * r->block)); - dma_rmb(); - return rd < r->size && !(rd & (r->block - 1)) && rd == wr; -} - -static int afkc_drain(struct afkc *c) -{ - struct afkc_ring *r = &c->rings[1]; - u32 rd, wr, bytes, total; - u8 *qe; - int ret; - if (!c->running || !r->header) - return -EPROTO; - for (u32 n = 0; n < 64; n++) { - rd = READ_ONCE(*(u32 *)(r->header + r->block)); - wr = READ_ONCE(*(u32 *)(r->header + 2 * r->block)); - dma_rmb(); - if (rd >= r->size || wr >= r->size || (rd | wr) & (r->block - 1)) - return -EPROTO; - if (rd == wr) - return 0; - qe = r->data + rd; - if (get_unaligned_le32(qe) != 0x20504f49) - return -EPROTO; - bytes = get_unaligned_le32(qe + 4); - if (bytes > r->size - 16) - return -EPROTO; - if (bytes + 16 > r->size - rd) { - if (wr >= rd || !wr) - return -EPROTO; - rd = 0; - qe = r->data; - if (get_unaligned_le32(qe) != 0x20504f49) - return -EPROTO; - bytes = get_unaligned_le32(qe + 4); - } - if (bytes < 8 || bytes > r->size - rd - 16) - return -EPROTO; - total = ALIGN(bytes + 16, r->block); - if (c->streaming && bytes + 12 > sizeof(c->capture) - c->capture_size) - c->capture_size = 0; - if (total > r->size - rd || (rd < wr && total > wr - rd) || - bytes + 12 > sizeof(c->capture) - c->capture_size) - return -EOVERFLOW; - /* Preserve the existing offline capture framing. */ - memcpy(c->capture + c->capture_size, qe + 8, 8); - put_unaligned_le32(bytes, c->capture + c->capture_size + 8); - memcpy(c->capture + c->capture_size + 12, qe + 16, bytes); - c->capture_size += bytes + 12; - ret = c->ops->record(c->cookie, qe + 16, bytes); - rd += total; - if (rd == r->size) - rd = 0; - dma_mb(); - WRITE_ONCE(*(u32 *)(r->header + r->block), rd); - if (ret) - return ret; - } - return -E2BIG; -} - -static int afkc_receive(struct afkc *c, u64 message) -{ - u32 type = message >> 48, size, offset, index, data_size, block; - u64 dva; - struct afkc_ring *r, *other; - int ret = 0; - if (c->error) - return c->error; - if (!c->started || c->stopped || (!c->streaming && ++c->messages > 128)) - return afkc_fail(c, -EPROTO); - switch (type) { - case 0xa0: - case 0x8c: - break; - case 0x89: - size = ((message >> 16) & 0xffff) * 64; - if (c->buffer || c->stopping || !size || size > 65536) - return afkc_fail(c, -EINVAL); - ret = c->ops->allocate(c->cookie, size, &c->buffer, &dva); - if (ret) - break; - c->buffer_size = size; - c->tag = message & 0xffff; - if (!c->buffer || dva < (1ULL << 40) || - dva - (1ULL << 40) >= (1ULL << 36) || dva & 0x3fff) - return afkc_fail(c, -EINVAL); - ret = afkc_send(c, 0xa1, dva); - break; - case 0x8a: - case 0x8b: - index = type == 0x8a ? 0 : 1; - r = &c->rings[index]; - other = &c->rings[!index]; - offset = ((message >> 32) & 0xffff) * 64; - size = ((message >> 16) & 0xffff) * 64; - if (!c->buffer || r->header || c->stopping || size < 256 || - (message & 0xffff) != c->tag || offset >= c->buffer_size || - size > c->buffer_size - offset || (other->header && - offset < other->offset + other->extent && other->offset < offset + size)) - return afkc_fail(c, -EINVAL); - data_size = READ_ONCE(*(u32 *)(c->buffer + offset)); - if (data_size >= size || (size - data_size) % 3) - return afkc_fail(c, -EPROTO); - block = (size - data_size) / 3; - if (block < 64 || block > 256 || !is_power_of_2(block) || - data_size < block || data_size % block) - return afkc_fail(c, -EPROTO); - *r = (struct afkc_ring){.header = c->buffer + offset, - .data = c->buffer + offset + 3 * block, .size = data_size, - .block = block, .offset = offset, .extent = size}; - if (other->header) - ret = afkc_send(c, 0xa3, 0); - break; - case 0x86: - if (!c->rings[0].header || !c->rings[1].header || c->running || c->stopping) - return afkc_fail(c, -EPROTO); - c->running = true; - break; - case 0x85: - ret = afkc_drain(c); - break; - case 0xc1: - if (!c->stopping || !afkc_tx_empty(c)) - return afkc_fail(c, -EPROTO); - c->stopped = true; - break; - default: - ret = -EOPNOTSUPP; - } - return ret ? afkc_fail(c, ret) : 0; -} - -static int afkc_stop(struct afkc *c) -{ - if (c->error) - return c->error; - if (!c->running || c->stopping || !afkc_tx_empty(c)) - return -EAGAIN; - c->stopping = true; - return afkc_send(c, 0xc0, 0); -} -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_session.c b/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_session.c deleted file mode 100644 index a3c5a468bc3ab..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_session.c +++ /dev/null @@ -1,343 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* See dcpext_session.h for attribution and the hardware adapter contract. */ -#include "dcpext_session.h" -#ifdef __KERNEL__ -#include -#else -#include -#endif - -#define MSG(type) ((u64)(type) << 52) -#define SYSTEM_PROFILE 0x11fU /* J514S 14.6 session-1 capture, 2026-09-21 */ -#define SESSION_MS 8000 -#define MAX_MESSAGES 256 -#define DVA_SIZE (1ULL << 36) /* Two 11-bit levels plus 14-bit page offset */ -#define DVA_BASE (1ULL << 40) /* M4 firmware wire address, not a table index */ - -static bool valid_buffer_address(unsigned int ep, bool inherited, u64 addr, u32 size) -{ - u64 base = DVA_BASE, limit = DVA_SIZE; - - if (ep == 8 && inherited) { - /* OSLog can report a host physical address; the callback must - * qualify the exact OS-log reservation before accepting it. - */ - base = 0; - limit = 1ULL << 42; - } - return addr && !(addr & 0xfff) && addr >= base && - addr - base < limit && size <= limit - (addr - base); -} - -static int fail(struct dcpext_session *s, int error) -{ - if (!s->error) - s->error = error < 0 ? error : -EIO; - s->phase = DCPEXT_FAILED; - return s->error; -} - -int dcpext_session_poll(struct dcpext_session *s) -{ - u64 now; - - if (s->phase == DCPEXT_FAILED) - return s->error; - if (!s->ops || s->phase == DCPEXT_NEW) - return -EINVAL; - if (s->phase == DCPEXT_QUIESCED) - return 0; - now = s->ops->now_ms(s->cookie); - if (now < s->last_time) - return fail(s, -ERANGE); - s->last_time = now; - if (s->runtime) { - if (s->phase != DCPEXT_RUNNING) - return fail(s, -EPROTO); - if (now - s->runtime_window >= 1000) { - s->runtime_window = now; - s->runtime_messages = 0; - } - } else if (now >= s->deadline) - return fail(s, -ETIMEDOUT); - return 0; -} - -int dcpext_session_enter_runtime(struct dcpext_session *s) -{ - int ret = dcpext_session_poll(s); - - if (ret) - return ret; - if (s->phase != DCPEXT_RUNNING || s->runtime) - return -EINVAL; - s->runtime = true; - s->runtime_window = s->last_time; - s->runtime_messages = 0; - return 0; -} - -int dcpext_session_runtime_message(struct dcpext_session *s) -{ - int ret = dcpext_session_poll(s); - - if (ret) - return ret; - if (s->runtime && ++s->runtime_messages > 8192) - return fail(s, -EOVERFLOW); - return 0; -} - -int dcpext_session_leave_runtime(struct dcpext_session *s) -{ - int ret = dcpext_session_poll(s); - - if (ret) - return ret; - if (!s->runtime || s->last_time > ~(u64)0 - SESSION_MS) - return -EINVAL; - s->runtime = false; - s->deadline = s->last_time + SESSION_MS; - s->received = 0; - return 0; -} - -static int send(struct dcpext_session *s, u8 ep, u64 msg) -{ - int ret = dcpext_session_poll(s); - - if (ret) - return ret; - /* A failed transport return does not establish that delivery failed. */ - s->touched = true; - ret = s->ops->send(s->cookie, ep, msg); - if (ret) - return fail(s, ret); - return dcpext_session_poll(s); -} - -int dcpext_session_begin(struct dcpext_session *s, - const struct dcpext_session_ops *ops, void *cookie) -{ - u64 now, duration = s->duration_ms ? s->duration_ms : SESSION_MS; - - if (duration > 60000 || duration < SESSION_MS) - return -EINVAL; - if (s->message_limit && - !((s->message_limit == 512 && (duration == 16000 || duration == 60000)) || - (s->message_limit == 8192 && duration == 16000))) - return -EINVAL; - if (s->phase != DCPEXT_NEW || s->touched || s->ops) - return -EBUSY; - if (!ops || !ops->send || !ops->buffer || !ops->now_ms) - return -EINVAL; - now = ops->now_ms(cookie); - if (now > ~(u64)0 - duration) - return -ERANGE; - s->ops = ops; - s->cookie = cookie; - s->last_time = now; - s->deadline = now + duration; - s->phase = DCPEXT_HELLO; - /* Linux apple_rtkit_wake and m1n1 rtkit_boot request INIT (0x220), - * including the reinitialization bit, and wait for ON (0x20). - * ON alone wakes an inherited session but does not request this reset - * of protocol state. Never issue CPU reset/run register writes here. - */ - return send(s, 0, MSG(6) | 0x220); -} - -static int ap_on_if_ready(struct dcpext_session *s) -{ - if (!s->epmap_done || !s->iop_on) - return 0; - s->phase = DCPEXT_AP_ON; - return send(s, 0, MSG(0xb) | 0x20); -} - -static int management(struct dcpext_session *s, u64 msg) -{ - unsigned int type = (msg >> 52) & 0xff; - unsigned int base, ep, state = msg & 0xffff; - u32 bitmap; - bool last; - int ret; - - switch (type) { - case 1: - if (s->phase != DCPEXT_HELLO || s->hello || - (msg & 0xffff) > 12 || ((msg >> 16) & 0xffff) < 12) - return fail(s, -EPROTO); - s->hello = true; - s->phase = DCPEXT_EPMAP; - return send(s, 0, MSG(2) | (12 << 16) | 12); - case 8: - base = (msg >> 32) & 7; - bitmap = (u32)msg; - last = (msg >> 51) & 1; - if (s->phase != DCPEXT_EPMAP || !s->hello || - (s->bases & (1U << base))) - return fail(s, -EPROTO); - s->bases |= 1U << base; - s->endpoints[base] = bitmap; - if ((!base && bitmap != SYSTEM_PROFILE) || - (last && !(s->bases & 1))) - return fail(s, -EOPNOTSUPP); - ret = send(s, 0, MSG(8) | ((u64)base << 32) | - (last ? 1ULL << 51 : 1)); - if (ret || !last) - return ret; - s->epmap_done = true; - s->phase = DCPEXT_IOP_ON; - for (ep = 1; ep < 32; ep++) { - if (!(SYSTEM_PROFILE & (1U << ep))) - continue; - ret = send(s, 0, MSG(5) | ((u64)ep << 32) | 2); - if (ret) - return ret; - s->started |= 1U << ep; - } - return ap_on_if_ready(s); - case 7: - if (state == 0x20 && s->hello && !s->iop_on && - (s->phase == DCPEXT_EPMAP || s->phase == DCPEXT_IOP_ON)) { - s->iop_on = true; - return ap_on_if_ready(s); - } - if (state == (s->sleep_iop ? 1 : 0x10) && s->phase == DCPEXT_IOP_QUIESCE) { - s->phase = DCPEXT_QUIESCED; - return 0; - } - return fail(s, -EPROTO); - case 0xb: - if (state == 0x20 && s->phase == DCPEXT_AP_ON) { - s->phase = DCPEXT_RUNNING; - return 0; - } - if (state == 0x10 && s->phase == DCPEXT_AP_QUIESCE && !s->ap_quiesced) { - s->ap_quiesced = true; - return s->defer_iop_quiesce ? 0 : dcpext_session_finish_quiesce(s); - } - return fail(s, -EPROTO); - default: - return fail(s, -EOPNOTSUPP); - } -} - -static int get_buffer(struct dcpext_session *s, unsigned int ep, u64 msg) -{ - struct dcpext_buffer *b = &s->buffers[ep]; - u64 requested, dva = 0, reply; - u32 size; - int ret; - - if (b->ready) - return fail(s, ep == 1 ? -EIO : -EPROTO); /* EP1 repeat = crash */ - if (ep == 8) { - size = (msg >> 36) & 0xfffff; - requested = (msg & ((1ULL << 36) - 1)) << 12; - } else { - size = ((msg >> 44) & 0xff) << 12; - requested = msg & ((1ULL << 44) - 1); - } - if (!size || size > 0x100000 || - (requested && !valid_buffer_address(ep, true, requested, size))) - return fail(s, -EINVAL); - ret = s->ops->buffer(s->cookie, ep, requested, size, &dva); - if (ret) - return fail(s, ret); - /* Record ownership before any further operation can fail. */ - b->dva = dva; - b->size = size; - b->inherited = requested != 0; - b->ready = true; - ret = dcpext_session_poll(s); - if (ret) - return ret; - if (!valid_buffer_address(ep, requested != 0, dva, size) || - (requested && dva != requested)) - return fail(s, -EINVAL); - /* M4 firmware-owned buffers are validated without a host reply. */ - if (requested) - return 0; - if (ep == 8) - reply = (1ULL << 56) | ((u64)size << 36) | (dva >> 12); - else - reply = MSG(1) | ((u64)(size >> 12) << 44) | dva; - return send(s, ep, reply); -} - -int dcpext_session_receive(struct dcpext_session *s, unsigned int ep, u64 msg) -{ - unsigned int type; - int ret = dcpext_session_poll(s); - - if (ret) - return ret; - if (s->phase == DCPEXT_QUIESCED || (!s->runtime && - ++s->received > (s->message_limit ? s->message_limit : MAX_MESSAGES))) - return fail(s, -EPROTO); - if (!ep) - return management(s, msg); - /* This diagnostic never starts any application endpoint. */ - if (ep >= 32 || !(s->started & (1U << ep))) - return fail(s, -EPROTO); - type = ep == 8 ? msg >> 56 : (msg >> 52) & 0xff; - if ((ep == 1 || ep == 2 || ep == 4 || ep == 8) && type == 1) - return get_buffer(s, ep, msg); - /* AppleDCP 1041.120.7 transport at image 0x34c94 consumes a type-2 - * reply as the log ring's read counter and clears its outstanding bit. - * Without this reply the AP power task waits forever in 0x34708. - * Discard the log contents, acknowledging precisely the notified counter; - * it is not an address, byte extent, or proof of display/DMA retirement. - * Only channel zero has a qualified buffer in this session engine. - */ - if (ep == 8) { - s->oslog_notices++; - if (type == 2) { - if (!s->buffers[8].ready || (msg & 0x00ffffff00000000ULL)) - return fail(s, -EPROTO); - return send(s, ep, msg); - } - /* Types 3/4/5 describe log metadata and require no reply. */ - return 0; - } - /* System messages are serviced during both boot and shutdown. Syslog - * payload is deliberately not dereferenced by this diagnostic. - */ - if (ep == 2 && type == 8) - return 0; - if ((ep == 2 && type == 5) || (ep == 4 && (type == 8 || type == 0xc))) - return send(s, ep, msg); - return fail(s, -EOPNOTSUPP); -} - -int dcpext_session_quiesce(struct dcpext_session *s) -{ - int ret = dcpext_session_poll(s); - - if (ret) - return ret; - /* Reject the old endpoint-only attempt without sending IOP quiesce. */ - if (s->phase != DCPEXT_RUNNING) - return -EAGAIN; - if (s->runtime) { - ret = dcpext_session_leave_runtime(s); - if (ret) - return ret; - } - s->phase = DCPEXT_AP_QUIESCE; - return send(s, 0, MSG(0xb) | 0x10); -} - -int dcpext_session_finish_quiesce(struct dcpext_session *s) -{ - int ret = dcpext_session_poll(s); - - if (ret) - return ret; - if (s->phase != DCPEXT_AP_QUIESCE || !s->ap_quiesced) - return -EAGAIN; - s->phase = DCPEXT_IOP_QUIESCE; - return send(s, 0, MSG(6) | (s->sleep_iop ? 1 : 0x10)); -} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_session.h b/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_session.h deleted file mode 100644 index c263bbd6a16c5..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_session.h +++ /dev/null @@ -1,96 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* Transport-independent RTKit system-session diagnostic. No MMIO or DMA API. - * Protocol reference: Asahi Linux Contributors, drivers/soc/apple/rtkit.c - * in linux/linux-displayport at base fba91e5d9577 (see provenance manifest). - * This component starts no application endpoints and cannot read EDID itself. - */ -#ifndef DCPEXT_SESSION_H -#define DCPEXT_SESSION_H - -#ifdef __KERNEL__ -#include -#else -#include -#include -typedef uint8_t u8; -typedef uint32_t u32; -typedef uint64_t u64; -#endif - -enum dcpext_phase { - DCPEXT_NEW, DCPEXT_HELLO, DCPEXT_EPMAP, DCPEXT_IOP_ON, - DCPEXT_AP_ON, DCPEXT_RUNNING, DCPEXT_AP_QUIESCE, - DCPEXT_IOP_QUIESCE, DCPEXT_QUIESCED, DCPEXT_FAILED, -}; - -struct dcpext_buffer { - u64 dva; - u32 size; - bool ready, inherited; -}; - -struct dcpext_session_ops { - /* Single serialized caller. All callbacks must be bounded, non-reentrant. - * send must supply the DMA write barrier before mailbox publication. - * recv's caller supplies the DMA read barrier after mailbox receipt. - */ - u64 (*now_ms)(void *cookie); - int (*send)(void *cookie, u8 ep, u64 msg); - /* Nonzero requested address: validate the ENTIRE inherited mapping - * against live reserved RAM; do not clear/rewrite firmware-owned memory. - * M4 DART wire addresses retain their 1-TiB bias. EP8 may instead name - * the physical OS-log carveout: accept only an exact live reservation, - * never a generic physical-address fallback or an invented DART map. - * Zero DVA: obtain a zeroed, mapped host buffer of at least size bytes. - * Retain ownership even on errors: the engine never frees anything. - * This callback neither sends messages nor starts firmware execution. - */ - int (*buffer)(void *cookie, u8 ep, u64 requested, u32 size, u64 *dva); -}; - -struct dcpext_session { - const struct dcpext_session_ops *ops; - void *cookie; - enum dcpext_phase phase; - int error; - u64 deadline, last_time; - u32 duration_ms; /* 0=8s; explicit bounded image observation, at most60s. */ - u32 message_limit; /* 0=256; native+AV 512; bounded 16s KMS 8192. */ - u32 endpoints[8]; - u32 started; - u8 bases; - unsigned int received; - bool runtime; /* Explicit post-startup desktop state; RPCs keep their deadlines. */ - u64 runtime_window; - u32 runtime_messages; - bool hello, epmap_done, iop_on, touched; - bool sleep_iop; /* Linux shutdown: AP quiesced, then restartable IOP sleep. */ - bool defer_iop_quiesce, ap_quiesced; - unsigned int oslog_notices; - struct dcpext_buffer buffers[9]; -}; - -/* Zero-initialize a new instance before begin; never reinitialize a live one. - * begin admits only the system endpoint profile seen on J514S 14.6 DCPEXT. - * The adapter must separately qualify the firmware UUID and DART handoff. - */ -int dcpext_session_begin(struct dcpext_session *s, - const struct dcpext_session_ops *ops, void *cookie); -int dcpext_session_poll(struct dcpext_session *s); -int dcpext_session_enter_runtime(struct dcpext_session *s); -int dcpext_session_leave_runtime(struct dcpext_session *s); -/* Adapter accounts every mailbox endpoint, including application traffic. */ -int dcpext_session_runtime_message(struct dcpext_session *s); -int dcpext_session_receive(struct dcpext_session *s, unsigned int ep, u64 msg); -int dcpext_session_quiesce(struct dcpext_session *s); -/* Optional ordering experiment: only after AP ACK, let the owner stop its - * application channels while IOP remains awake, then request IOP quiescence. - */ -int dcpext_session_finish_quiesce(struct dcpext_session *s); - -/* QUIESCED means both protocol ACKs arrived. It is NOT a cleanup function or - * proof of real hardware quiescence. Before freeing, the hardware adapter must - * also stop/drain its transport and complete DART removal/invalidation. - * FAILED is sticky; late ACKs cannot permit freeing possibly reachable RAM. - */ -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_syslog.h b/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_syslog.h deleted file mode 100644 index bae0e9ad1d0a5..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/dcpext_syslog.h +++ /dev/null @@ -1,49 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* Capture RTKit syslog slots before acknowledgement allows firmware reuse. - * Format/offsets: Asahi Linux Contributors, drivers/soc/apple/rtkit.c, - * apple_rtkit_syslog_rx_init/log at kernel fba91e5d9577. - * This helper only reads a caller-qualified host buffer. No address mapping, - * firmware commands or string interpretation. Output records are private: - * little-endian u64 mailbox message, u32 slot size, then the complete slot. - */ -#ifndef DCPEXT_SYSLOG_H -#define DCPEXT_SYSLOG_H -struct dcpext_syslog { - u32 last_index, message_size, used; - bool initialized; - u8 capture[1048576]; -}; - -static int dcpext_syslog_observe(struct dcpext_syslog *s, u64 message, - const u8 *buffer, u32 size) -{ - u32 type = (message >> 52) & 0xff; - u32 index = message & 0xff, stride, offset; - if (type == 8) { - if (s->initialized) - return -EPROTO; - s->last_index = index; /* Native/Linux admission uses inclusive max. */ - s->message_size = (message >> 24) & 0xff; - if (!s->message_size) - return -EINVAL; - s->initialized = true; - return 0; - } - if (type != 5) - return 0; - if (!s->initialized || !buffer || index > s->last_index) - return -EINVAL; - stride = 0x20 + s->message_size; - offset = index * stride; - if (offset > size || stride > size - offset) - return -ERANGE; - if (s->used > sizeof(s->capture) || - stride + 12 > sizeof(s->capture) - s->used) - return -ENOSPC; - put_unaligned_le64(message, s->capture + s->used); - put_unaligned_le32(stride, s->capture + s->used + 8); - memcpy(s->capture + s->used + 12, buffer + offset, stride); - s->used += stride + 12; - return 0; -} -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_bridge.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_bridge.c deleted file mode 100644 index d2de75da97583..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_bridge.c +++ /dev/null @@ -1,842 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Root-only RPC transport for bounded firmware diagnostics, not a display ABI. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../../dcp-link.h" -#include "m3_dcp_bridge.h" - -#define MAX_EVENTS 32 -#define MAX_CALLBACKS 8 -#define MAX_BUFFERS 64 - -/* Diagnostic ABI: allocations are retained until reboot, including releases. */ -#define BRIDGE_ALLOC_BUFFER _IOWR('M', 0x40, struct m3_dcp_buffer) -#define BRIDGE_RETIRE_BUFFER _IOW('M', 0x41, __u32) -#define BRIDGE_MAP_PIODMA _IOWR('M', 0x42, struct m3_dcp_buffer) - -struct scanout_request { - __u32 width, height, stride, format; - __u64 size, dva; -}; -#define BRIDGE_ALLOC_SCANOUT _IOWR('M', 0x43, struct scanout_request) - -/* Write op 1: submit, message=0 (normal) or 1 (nested), data=RPC packet. - * Write op 2: callback reply, message=original message, data=output bytes. - * Read kind 1: command reply, kind 2: callback, kind 3: unexpected message. - * All headers are little endian. One complete record per read/write. - */ -struct record_header { - __le32 kind; - __le32 size; - __le64 message; -}; - -struct bridge_event { - struct list_head list; - struct record_header header; - u8 data[]; -}; - -struct m3_dcp_bridge { - struct miscdevice misc; - struct device *dev; - struct apple_rtkit *rtkit; - void *rpc; - struct mutex lock; - wait_queue_head_t wait; - struct list_head events; - unsigned int event_count, callback_count; - bool failed; - bool enabled; - bool kernel_client; - u32 kernel_callback_depth; - struct work_struct *kernel_work; - u32 buffer_count; - u64 buffer_bytes; - struct platform_device *piodma; - struct iommu_domain *piodma_domain; - struct platform_device *scanout; - void *scanout_cpu; - struct scanout_request scanout_info; - struct { - void *cpu; - struct m3_dcp_buffer info; - bool retired; - bool piodma_mapped; - } buffers[MAX_BUFFERS]; - struct { - bool active; - bool remote; - u32 offset; - u32 size; - struct apple_dcp_link_rpc_header header; - } calls[2]; - struct { - u64 message; - void *output; - u32 size; - u32 call; - u64 allocation_size; - u64 mapping_id; - } callbacks[MAX_CALLBACKS]; -}; - -static void fail(struct m3_dcp_bridge *b) -{ - b->failed = true; - dev_err(b->dev, "DCP bridge stopped; retain firmware buffers until reboot\n"); - wake_up_interruptible(&b->wait); -} - -static void event(struct m3_dcp_bridge *b, u32 kind, u64 message, - const void *data, u32 size) -{ - struct bridge_event *e; - - if (b->event_count >= MAX_EVENTS || size > APPLE_DCP_LINK_STREAM_BUFFER_SIZE) { - fail(b); - return; - } - e = kmalloc(sizeof(*e) + size, GFP_KERNEL); - if (!e) { - fail(b); - return; - } - e->header = (struct record_header) { - .kind = cpu_to_le32(kind), .size = cpu_to_le32(size), - .message = cpu_to_le64(message), - }; - if (size) - memcpy(e->data, data, size); - list_add_tail(&e->list, &b->events); - b->event_count++; - wake_up_interruptible(&b->wait); - if (b->kernel_work) - queue_work(system_unbound_wq, b->kernel_work); -} - -void m3_dcp_bridge_crashed(struct m3_dcp_bridge *b) -{ - mutex_lock(&b->lock); - fail(b); - mutex_unlock(&b->lock); -} - -void m3_dcp_bridge_receive(struct m3_dcp_bridge *b, u64 message) -{ - struct apple_dcp_link_stream_layout layout; - struct apple_dcp_link_rpc_view view; - u8 stream = apple_dcp_link_message_stream(message); - bool remote = apple_dcp_link_message_is_remote(message); - bool nested; - int ret; - - mutex_lock(&b->lock); - if (b->failed) - goto out; - if ((message & 3) != APPLE_DCP_LINK_MSG_RPC) { - event(b, 3, message, NULL, 0); - goto out; - } - dma_rmb(); - if (message & BIT_ULL(6)) { - nested = b->calls[1].active; - if (stream || !b->calls[nested].active || b->calls[nested].remote != remote || - apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, 0, remote, &layout)) - goto error; - if (memcmp(b->rpc + layout.local_offset + b->calls[nested].offset, - &b->calls[nested].header, sizeof(b->calls[nested].header))) - goto error; - event(b, 1, message, b->rpc + layout.local_offset + b->calls[nested].offset, - b->calls[nested].size); - b->calls[nested].active = false; - goto out; - } - if (b->callback_count >= MAX_CALLBACKS || - apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, stream, remote, &layout)) - goto error; - ret = apple_dcp_link_rpc_decode(message, b->rpc + layout.remote_offset, - layout.capacity, &view); - if (ret) - goto error; - if ((view.call >> 24) != 'D') { - u32 off; - - /* Preserve stream-placement evidence without acknowledging bad data. */ - for (off = 0; off < APPLE_DCP_LINK_HEAP_OFFSET; - off += APPLE_DCP_LINK_STREAM_BUFFER_SIZE) { - const __le32 *h = b->rpc + off; - - dev_err(b->dev, "M3 RPC header offset=%#x words=%#x/%#x/%#x\n", - off, le32_to_cpu(h[0]), le32_to_cpu(h[1]), le32_to_cpu(h[2])); - } - goto error; - } - b->callbacks[b->callback_count].message = message; - b->callbacks[b->callback_count].output = view.output; - b->callbacks[b->callback_count].size = view.output_size; - b->callbacks[b->callback_count].call = view.call; - b->callbacks[b->callback_count].allocation_size = 0; - b->callbacks[b->callback_count].mapping_id = 0; - if (view.call == 0x44343531 && view.input_size == 20 && view.output_size == 28) { - __le64 allocation_size; - - memcpy(&allocation_size, view.input + 4, sizeof(allocation_size)); - b->callbacks[b->callback_count].allocation_size = le64_to_cpu(allocation_size); - } - if (view.call == 0x44323031 && view.input_size == 12 && view.output_size == 20) { - __le64 mapping_id; - - memcpy(&mapping_id, view.input, sizeof(mapping_id)); - b->callbacks[b->callback_count].mapping_id = le64_to_cpu(mapping_id); - } - b->callback_count++; - dev_dbg(b->dev, "M3 DCP callback tag=%#x input=%u output=%u\n", - view.call, view.input_size, view.output_size); - event(b, 2, message, b->rpc + layout.remote_offset + view.offset, view.total_size); - goto out; -error: - fail(b); -out: - mutex_unlock(&b->lock); -} - -static ssize_t bridge_read(struct file *file, char __user *user, size_t count, loff_t *pos) -{ - struct m3_dcp_bridge *b = container_of(file->private_data, struct m3_dcp_bridge, misc); - struct bridge_event *e; - size_t size; - int ret; - -again: - mutex_lock(&b->lock); - if (list_empty(&b->events)) { - ret = b->failed ? -EIO : -EAGAIN; - mutex_unlock(&b->lock); - if (ret == -EIO || file->f_flags & O_NONBLOCK) - return ret; - ret = wait_event_interruptible(b->wait, READ_ONCE(b->event_count) || READ_ONCE(b->failed)); - if (ret) - return ret; - goto again; - } - e = list_first_entry(&b->events, struct bridge_event, list); - size = sizeof(e->header) + le32_to_cpu(e->header.size); - ret = -EMSGSIZE; - if (count < size) - goto out; - ret = -EFAULT; - if (copy_to_user(user, &e->header, size)) - goto out; - list_del(&e->list); - b->event_count--; - kfree(e); - ret = size; -out: - mutex_unlock(&b->lock); - return ret; -} - -static ssize_t bridge_request(struct m3_dcp_bridge *b, struct record_header *record, size_t count) -{ - struct apple_dcp_link_stream_layout layout; - struct apple_dcp_link_rpc_header *rpc; - u64 message, control; - u32 kind, size, total, offset = 0; - bool nested, remote = false; - int ret; - - kind = le32_to_cpu(record->kind); - size = le32_to_cpu(record->size); - control = le64_to_cpu(record->message); - ret = -EINVAL; - if (size != count - sizeof(*record)) - goto free_record; - mutex_lock(&b->lock); - if (!b->enabled) { - ret = -EAGAIN; - goto out; - } - if (b->failed) { - ret = -EIO; - goto out; - } - if (kind == 2) { - unsigned int top; - - if (!b->callback_count) - goto out; - top = b->callback_count - 1; - if (control != b->callbacks[top].message || size != b->callbacks[top].size) - goto out; - memcpy(b->callbacks[top].output, record + 1, size); - dma_wmb(); - ret = apple_rtkit_send_message(b->rtkit, APPLE_DCP_LINK_ENDPOINT, - apple_dcp_link_rpc_reply(control), NULL, false); - if (ret) - fail(b); - else - b->callback_count--; - goto out; - } - if (kind != 1 || control > 1 || size < sizeof(*rpc)) - goto out; - nested = control; - if (b->calls[nested].active || (nested && !b->callback_count) || - (!nested && b->callback_count)) { - ret = -EBUSY; - goto out; - } - if (nested) { - u64 callback = b->callbacks[b->callback_count - 1].message; - - if (apple_dcp_link_message_stream(callback)) { - ret = -EOPNOTSUPP; - goto out; - } - /* Re-entry uses the callback's stream without the new-stream IRQ flag. */ - remote = apple_dcp_link_message_is_remote(callback); - if (b->calls[0].active && b->calls[0].remote == remote) - offset = ALIGN(b->calls[0].size, 64); - } - rpc = (void *)(record + 1); - ret = apple_dcp_link_rpc_payload_size(le32_to_cpu(rpc->input_size), - le32_to_cpu(rpc->output_size), &total); - if (ret || total != size) { - ret = -EMSGSIZE; - goto out; - } - ret = apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, 0, remote, &layout); - if (ret) - goto out; - if (offset + size > layout.capacity) { - ret = -EMSGSIZE; - goto out; - } - memcpy(b->rpc + layout.local_offset + offset, rpc, size); - memset(b->rpc + layout.local_offset + offset + sizeof(*rpc) + le32_to_cpu(rpc->input_size), - 0, le32_to_cpu(rpc->output_size)); - b->calls[nested].header = *rpc; - b->calls[nested].size = size; - b->calls[nested].offset = offset; - b->calls[nested].remote = remote; - b->calls[nested].active = true; - /* M3 bit 9 requires a free stream; it is not a recursive-call marker. */ - apple_dcp_link_rpc_message(offset, size, false, &message); - apple_dcp_link_stream_message(message, 0, remote, &message); - dma_wmb(); - ret = apple_rtkit_send_message(b->rtkit, APPLE_DCP_LINK_ENDPOINT, message, NULL, false); - if (ret) - fail(b); -out: - mutex_unlock(&b->lock); -free_record: - kfree(record); - return ret ? ret : count; -} - -static ssize_t bridge_write(struct file *file, const char __user *user, size_t count, loff_t *pos) -{ - struct m3_dcp_bridge *b = container_of(file->private_data, struct m3_dcp_bridge, misc); - struct record_header *record; - - if (count < sizeof(*record) || count > sizeof(*record) + APPLE_DCP_LINK_STREAM_BUFFER_SIZE) - return -EMSGSIZE; - record = memdup_user(user, count); - if (IS_ERR(record)) - return PTR_ERR(record); - return bridge_request(b, record, count); -} - -static __poll_t bridge_poll(struct file *file, poll_table *wait) -{ - struct m3_dcp_bridge *b = container_of(file->private_data, struct m3_dcp_bridge, misc); - __poll_t mask = 0; - - poll_wait(file, &b->wait, wait); - if (READ_ONCE(b->event_count)) - mask |= EPOLLIN | EPOLLRDNORM; - if (READ_ONCE(b->failed)) - mask |= EPOLLERR; - return mask; -} - -static int map_piodma_buffer(struct m3_dcp_bridge *b, u32 id) -{ - struct m3_dcp_buffer *info = &b->buffers[id - 1].info; - struct device_node *node; - u64 offset; - int ret; - - if (b->buffers[id - 1].piodma_mapped) - return 0; - if (!b->piodma) { - node = of_get_child_by_name(b->dev->of_node, "piodma"); - if (!node || !of_device_is_available(node) || - !of_property_read_bool(node, "apple,j514s-inherited-mappings")) { - of_node_put(node); - return -ENODEV; - } - b->piodma = of_platform_device_create(node, NULL, b->dev); - if (!b->piodma) { - of_node_put(node); - return -ENOMEM; - } - ret = dma_set_mask_and_coherent(&b->piodma->dev, DMA_BIT_MASK(42)); - if (!ret) - ret = of_dma_configure(&b->piodma->dev, node, true); - of_node_put(node); - /* Never destroy an attached locked DART stream on an error path. */ - if (ret) - return ret; - b->piodma_domain = iommu_get_domain_for_dev(&b->piodma->dev); - } - if (IS_ERR_OR_NULL(b->piodma_domain)) - return -EIO; - ret = iommu_map(b->piodma_domain, info->dva, info->physical, info->size, - IOMMU_READ | IOMMU_WRITE, GFP_KERNEL); - if (ret) - return ret; - for (offset = 0; offset < info->size; offset += SZ_16K) { - if (iommu_iova_to_phys(b->piodma_domain, info->dva + offset) != info->physical + offset) { - fail(b); - return -EIO; - } - } - b->buffers[id - 1].piodma_mapped = true; - dev_info(b->dev, "M3 PIODMA mapped buffer %u size=%llu dva=%#llx\n", - id, info->size, info->dva); - return 0; -} - -static int allocate_scanout(struct m3_dcp_bridge *b, struct scanout_request *request) -{ - static const u32 colors[] = { - 0xffffffff, 0xffffff00, 0xff00ffff, 0xff00ff00, - 0xffff00ff, 0xffff0000, 0xff0000ff, 0xff202020, - }; - struct scanout_request info = *request; - struct device_node *node; - dma_addr_t dva; - u32 *pixels; - u32 x, y; - int ret; - - /* One diagnostic allocation per boot; native J514S panel dimensions. */ - if (b->scanout_cpu || b->calls[0].active || b->calls[1].active || b->callback_count || - info.width != 3024 || info.height != 1964 || info.stride || info.format || info.size || info.dva) - return -EINVAL; - if (!b->scanout) { - node = of_find_node_by_path("disp0"); - if (!node || !of_device_is_available(node) || - !of_device_is_compatible(node, "apple,t6030-display-diagnostics") || - !of_property_read_bool(node, "apple,j514s-inherited-mappings")) { - of_node_put(node); - return -ENODEV; - } - b->scanout = of_find_device_by_node(node); - if (!b->scanout) { - of_node_put(node); - return -ENODEV; - } - ret = dma_set_mask_and_coherent(&b->scanout->dev, DMA_BIT_MASK(42)); - if (!ret) - ret = of_dma_configure(&b->scanout->dev, node, true); - of_node_put(node); - /* Retain the device reference and inherited mappings until reboot. */ - if (ret) - return ret; - } - info.stride = info.width * 4; - info.size = ALIGN((u64)info.stride * info.height, SZ_16K); - info.format = 0x42475241; /* DCP BGRA8: wire FourCC ARGB */ - pixels = dma_alloc_coherent(&b->scanout->dev, info.size, &dva, GFP_KERNEL); - if (!pixels) - return -ENOMEM; - info.dva = dva; - /* Fill once; there is no per-frame CPU framebuffer copy. */ - for (y = 0; y < info.height; y++) { - for (x = 0; x < info.width; x++) { - u32 color = colors[x * ARRAY_SIZE(colors) / info.width]; - - if (x < 4 || y < 4 || x >= info.width - 4 || y >= info.height - 4) - color = 0xffffffff; - pixels[y * info.width + x] = cpu_to_le32(color); - } - } - dma_wmb(); - b->scanout_cpu = pixels; - b->scanout_info = info; - dev_info(b->dev, "M3 native scanout allocated %ux%u stride=%u size=%llu dva=%#llx\n", - info.width, info.height, info.stride, info.size, info.dva); - *request = info; - return 0; -} - -static int bridge_buffer_request(struct m3_dcp_bridge *b, unsigned int command, - struct m3_dcp_buffer *info) -{ - struct m3_dcp_buffer request = *info; - struct sg_table table; - dma_addr_t dva; - void *cpu; - u32 id; - size_t size; - int ret = -EINVAL; - - if (command != BRIDGE_ALLOC_BUFFER && command != BRIDGE_RETIRE_BUFFER && - command != BRIDGE_MAP_PIODMA && command != BRIDGE_ALLOC_SCANOUT) - return -ENOTTY; - mutex_lock(&b->lock); - if (b->failed || !b->enabled) { - ret = -EIO; - goto out; - } - if (command == BRIDGE_ALLOC_SCANOUT) { - ret = allocate_scanout(b, (struct scanout_request *)info); - goto out; - } - if (command == BRIDGE_RETIRE_BUFFER) { - id = request.id; - ret = -EINVAL; - if (!id || id > b->buffer_count || b->buffers[id - 1].retired) - goto out; - /* Keep mappings and memory pinned: this diagnostic has no DMA fence. */ - b->buffers[id - 1].retired = true; - ret = 0; - goto out; - } - ret = -EINVAL; - if (command == BRIDGE_MAP_PIODMA) { - id = request.id; - if (!b->callback_count || !id || id > b->buffer_count || - b->buffers[id - 1].retired || request.size || request.physical || request.dva || request.flags || - b->callbacks[b->callback_count - 1].call != 0x44323031 || - b->callbacks[b->callback_count - 1].mapping_id != id) - goto out; - ret = map_piodma_buffer(b, id); - if (!ret) - *info = b->buffers[id - 1].info; - goto out; - } - if (!b->callback_count || request.flags || request.id || request.dva || request.physical || - !request.size || request.size > SZ_16M || - b->callbacks[b->callback_count - 1].call != 0x44343531 || - b->callbacks[b->callback_count - 1].allocation_size != request.size) - goto out; - size = ALIGN(request.size, SZ_16K); - ret = -ENOSPC; - if (b->buffer_count == MAX_BUFFERS || b->buffer_bytes + size > SZ_64M) - goto out; - /* D451 requests both a physical address and a DCP address. Require a - * physically contiguous backing allocation before advertising either. - */ - cpu = dma_alloc_attrs(b->dev, size, &dva, GFP_KERNEL, DMA_ATTR_FORCE_CONTIGUOUS); - ret = -ENOMEM; - if (!cpu) - goto out; - ret = dma_get_sgtable_attrs(b->dev, &table, cpu, dva, size, DMA_ATTR_FORCE_CONTIGUOUS); - if (ret) - goto free_dma; - ret = -ERANGE; - if (table.orig_nents != 1 || table.sgl->length < size) { - sg_free_table(&table); - goto free_dma; - } - request.physical = sg_phys(table.sgl); - sg_free_table(&table); - request.size = size; - request.dva = dva; - request.id = b->buffer_count + 1; /* firmware treats zero as failure */ - memset(cpu, 0, size); - dma_wmb(); - b->buffers[b->buffer_count].cpu = cpu; - b->buffers[b->buffer_count].info = request; - b->buffer_count++; - b->buffer_bytes += size; - /* One allocation per callback. Retain even on copy_to_user failure. */ - b->callbacks[b->callback_count - 1].allocation_size = 0; - dev_info(b->dev, "M3 DCP buffer id=%u size=%zu dma=%pad\n", request.id, size, &dva); - *info = request; - ret = 0; - goto out; -free_dma: - dma_free_attrs(b->dev, size, cpu, dva, DMA_ATTR_FORCE_CONTIGUOUS); -out: - mutex_unlock(&b->lock); - return ret; -} - -static long bridge_ioctl(struct file *file, unsigned int command, unsigned long argument) -{ - struct m3_dcp_bridge *b = container_of(file->private_data, struct m3_dcp_bridge, misc); - void __user *user = (void __user *)argument; - struct m3_dcp_buffer request = {}; - int ret; - - if (command == BRIDGE_RETIRE_BUFFER) { - if (copy_from_user(&request.id, user, sizeof(request.id))) - return -EFAULT; - } else if (command == BRIDGE_ALLOC_BUFFER || command == BRIDGE_MAP_PIODMA || - command == BRIDGE_ALLOC_SCANOUT) { - if (copy_from_user(&request, user, sizeof(request))) - return -EFAULT; - } else { - return -ENOTTY; - } - ret = bridge_buffer_request(b, command, &request); - if (!ret && command != BRIDGE_RETIRE_BUFFER && copy_to_user(user, &request, sizeof(request))) - ret = -EFAULT; - return ret; -} - -static const struct file_operations bridge_fops = { - .owner = THIS_MODULE, - .read = bridge_read, - .write = bridge_write, - .poll = bridge_poll, - .unlocked_ioctl = bridge_ioctl, -}; - -struct m3_dcp_bridge *m3_dcp_bridge_create(struct device *dev, - struct apple_rtkit *rtkit, void *rpc, bool kernel_client) -{ - struct m3_dcp_bridge *b = devm_kzalloc(dev, sizeof(*b), GFP_KERNEL); - int ret; - - if (!b) - return ERR_PTR(-ENOMEM); - b->dev = dev; - b->kernel_client = kernel_client; - b->rtkit = rtkit; - b->rpc = rpc; - mutex_init(&b->lock); - init_waitqueue_head(&b->wait); - INIT_LIST_HEAD(&b->events); - b->misc = (struct miscdevice) { - .minor = MISC_DYNAMIC_MINOR, .name = "m3-dcp-rpc", .mode = 0600, - .fops = &bridge_fops, .parent = dev, - }; - ret = kernel_client ? 0 : misc_register(&b->misc); - if (ret) - return ERR_PTR(ret); - return b; -} - -void m3_dcp_bridge_activate(struct m3_dcp_bridge *b) -{ - mutex_lock(&b->lock); - b->enabled = true; - mutex_unlock(&b->lock); - dev_info(b->dev, "M3 DCP %s ready\n", b->kernel_client ? "kernel RPC client" : "RPC bridge /dev/m3-dcp-rpc"); -} - -/* One kernel worker owns this client, including recursive callbacks. */ -static struct bridge_event *kernel_event(struct m3_dcp_bridge *b, unsigned long timeout) -{ - struct bridge_event *e; - long ret; - - ret = wait_event_interruptible_timeout(b->wait, - READ_ONCE(b->event_count) || READ_ONCE(b->failed), timeout); - if (ret <= 0) - return ERR_PTR(ret ?: -ETIMEDOUT); - mutex_lock(&b->lock); - if (b->failed || list_empty(&b->events)) { - mutex_unlock(&b->lock); - return ERR_PTR(-EIO); - } - e = list_first_entry(&b->events, struct bridge_event, list); - list_del(&e->list); - b->event_count--; - mutex_unlock(&b->lock); - return e; -} - -static int kernel_send(struct m3_dcp_bridge *b, u32 kind, u64 message, - const void *data, u32 size) -{ - struct record_header *record; - ssize_t ret; - - if (size > APPLE_DCP_LINK_STREAM_BUFFER_SIZE) - return -EMSGSIZE; - record = kmalloc(sizeof(*record) + size, GFP_KERNEL); - if (!record) - return -ENOMEM; - *record = (struct record_header) { - .kind = cpu_to_le32(kind), .size = cpu_to_le32(size), - .message = cpu_to_le64(message), - }; - if (size) - memcpy(record + 1, data, size); - ret = bridge_request(b, record, sizeof(*record) + size); - return ret < 0 ? ret : 0; -} - -static int kernel_callback(struct m3_dcp_bridge *b, struct bridge_event *e, - m3_dcp_callback_fn callback, void *cookie, u32 *completed) -{ - struct apple_dcp_link_rpc_header *h = (void *)e->data; - u32 size = le32_to_cpu(e->header.size), in, out, tag; - int ret; - - if (le32_to_cpu(e->header.kind) != 2 || size < sizeof(*h)) - return -EPROTO; - in = le32_to_cpu(h->input_size); - out = le32_to_cpu(h->output_size); - tag = le32_to_cpu(h->call); - if (in > size - sizeof(*h) || out != size - sizeof(*h) - in || !callback) - return -EPROTO; - memset(e->data + sizeof(*h) + in, 0, out); - b->kernel_callback_depth++; - ret = callback(b, cookie, tag, h + 1, in, e->data + sizeof(*h) + in, out); - b->kernel_callback_depth--; - if (ret) - return ret; - if (tag == 0x44353839 && in == 0x6f0 && completed) - *completed = le32_to_cpup((__le32 *)(h + 1)); - return kernel_send(b, 2, le64_to_cpu(e->header.message), e->data + sizeof(*h) + in, out); -} - -int m3_dcp_bridge_pump(struct m3_dcp_bridge *b, unsigned long timeout, - m3_dcp_callback_fn callback, void *cookie) -{ - struct bridge_event *e; - int ret; - - if (!b->kernel_client) - return -EBUSY; - e = kernel_event(b, timeout); - if (IS_ERR(e)) - return PTR_ERR(e); - ret = kernel_callback(b, e, callback, cookie, NULL); - kfree(e); - return ret; -} - -int m3_dcp_bridge_call(struct m3_dcp_bridge *b, u32 tag, - const void *input, u32 input_size, void *output, u32 output_size, - u32 completion_id, m3_dcp_callback_fn callback, void *cookie) -{ - struct apple_dcp_link_rpc_header *packet; - unsigned long deadline = jiffies + 5 * HZ; - u32 size, completed = 0; - u64 expected = 0x42; - bool nested, replied = false; - int ret; - - if (!b->kernel_client) - return -EBUSY; - ret = apple_dcp_link_rpc_payload_size(input_size, output_size, &size); - if (ret || size > APPLE_DCP_LINK_STREAM_BUFFER_SIZE) - return -EMSGSIZE; - packet = kzalloc(size, GFP_KERNEL); - if (!packet) - return -ENOMEM; - apple_dcp_link_rpc_header_encode(packet, tag, input_size, output_size); - if (input_size) - memcpy(packet + 1, input, input_size); -retry: - /* Drain asynchronous notifications before starting an outer request. - * A queued callback is not recursive entry until its handler is running. - */ - if (!b->kernel_callback_depth) { - while (READ_ONCE(b->event_count)) { - ret = m3_dcp_bridge_pump(b, 0, callback, cookie); - if (ret) { - kfree(packet); - return ret; - } - } - } - mutex_lock(&b->lock); - nested = b->kernel_callback_depth != 0; - if (nested && !b->callback_count) { - mutex_unlock(&b->lock); - kfree(packet); - return -EPROTO; - } - if (nested) - expected |= b->callbacks[b->callback_count - 1].message & BIT_ULL(8); - mutex_unlock(&b->lock); - ret = kernel_send(b, 1, nested, packet, size); - if (ret == -EBUSY && !nested && time_before(jiffies, deadline)) - goto retry; /* A callback arrived between draining and submitting. */ - kfree(packet); - if (ret) - return ret; - while (time_before(jiffies, deadline)) { - long remaining = deadline - jiffies; - struct bridge_event *e = kernel_event(b, max(remaining, 0L)); - struct apple_dcp_link_rpc_header *h; - - if (IS_ERR(e)) - return PTR_ERR(e); - h = (void *)e->data; - if (le32_to_cpu(e->header.kind) == 1) { - if (replied || le32_to_cpu(e->header.size) != size || - le64_to_cpu(e->header.message) != expected || - le32_to_cpu(h->call) != tag || le32_to_cpu(h->input_size) != input_size || - le32_to_cpu(h->output_size) != output_size) { - ret = -EPROTO; - } else { - if (output_size) - memcpy(output, e->data + sizeof(*h) + input_size, output_size); - replied = true; - } - } else { - ret = kernel_callback(b, e, callback, cookie, &completed); - if (completion_id && completed && completed != completion_id) - ret = -EPROTO; - } - kfree(e); - if (ret) - return ret; - if (replied && (!completion_id || completed == completion_id)) - return 0; - } - return -ETIMEDOUT; -} - -int m3_dcp_bridge_alloc(struct m3_dcp_bridge *b, struct m3_dcp_buffer *info) -{ - return bridge_buffer_request(b, BRIDGE_ALLOC_BUFFER, info); -} - -int m3_dcp_bridge_map(struct m3_dcp_bridge *b, struct m3_dcp_buffer *info) -{ - return bridge_buffer_request(b, BRIDGE_MAP_PIODMA, info); -} - -int m3_dcp_bridge_retire(struct m3_dcp_bridge *b, u32 id) -{ - struct m3_dcp_buffer info = { .id = id }; - - return bridge_buffer_request(b, BRIDGE_RETIRE_BUFFER, &info); -} - -void m3_dcp_bridge_set_work(struct m3_dcp_bridge *b, struct work_struct *work) -{ - mutex_lock(&b->lock); - b->kernel_work = work; - if (b->event_count) - queue_work(system_unbound_wq, work); - mutex_unlock(&b->lock); -} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_bridge.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_bridge.h deleted file mode 100644 index 3e47f3f727dce..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_bridge.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -#ifndef M3_DCP_BRIDGE_H -#define M3_DCP_BRIDGE_H - -struct device; -struct work_struct; -struct apple_rtkit; -struct m3_dcp_bridge; - -struct m3_dcp_buffer { - u64 size, physical, dva; - u32 id, flags; -}; - -typedef int (*m3_dcp_callback_fn)(struct m3_dcp_bridge *bridge, void *cookie, - u32 tag, const void *input, u32 input_size, - void *output, u32 output_size); - -struct m3_dcp_bridge *m3_dcp_bridge_create(struct device *dev, - struct apple_rtkit *rtkit, void *rpc, bool kernel_client); -void m3_dcp_bridge_receive(struct m3_dcp_bridge *bridge, u64 message); -void m3_dcp_bridge_activate(struct m3_dcp_bridge *bridge); -void m3_dcp_bridge_crashed(struct m3_dcp_bridge *bridge); -int m3_dcp_bridge_call(struct m3_dcp_bridge *bridge, u32 tag, - const void *input, u32 input_size, void *output, u32 output_size, - u32 completion_id, m3_dcp_callback_fn callback, void *cookie); -int m3_dcp_bridge_pump(struct m3_dcp_bridge *bridge, unsigned long timeout, - m3_dcp_callback_fn callback, void *cookie); -int m3_dcp_bridge_alloc(struct m3_dcp_bridge *bridge, struct m3_dcp_buffer *info); -int m3_dcp_bridge_map(struct m3_dcp_bridge *bridge, struct m3_dcp_buffer *info); -int m3_dcp_bridge_retire(struct m3_dcp_bridge *bridge, u32 id); - -void m3_dcp_bridge_set_work(struct m3_dcp_bridge *bridge, struct work_struct *work); -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_brightness.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_brightness.h deleted file mode 100644 index b7b391ed30deb..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_brightness.h +++ /dev/null @@ -1,33 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -#ifndef M3_DCP_BRIGHTNESS_H -#define M3_DCP_BRIGHTNESS_H - -#include -#include - -#define M3_PANEL_MAX_NITS 500 -#define M3_PANEL_DEFAULT_NITS 140 - -/* J514S TEXT SHA a0af36d10a8abf2435918ba360e697529ddfaed56b89f9466e76d566b296e876: - * 133880..13389c copies wire[354] and wire[35c] to transaction[76a]/[778]. - * 17cd68..17cdc0 converts binary64 nits to runtime property 19 (16.16). - * The M4 double is at 35e: that layout must not be reused on M3. - * Use firmware panel calibration; this is an SDR brightness range. - */ -static inline void m3_dcp_encode_brightness(u8 *swap, unsigned int nits) -{ - u64 bits = 0; - - /* Encode an exactly representable integer as IEEE754 binary64 without - * using floating point in the kernel. Caller bounds nits to 0..500. - */ - if (nits) { - unsigned int exponent = fls(nits) - 1; - - bits = ((u64)(1023 + exponent) << 52) | - (((u64)nits << (52 - exponent)) & ((1ULL << 52) - 1)); - } - swap[0x354] = 1; - put_unaligned_le64(bits, swap + 0x35c); -} -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_clock_restore.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_clock_restore.c deleted file mode 100644 index 8c4318c59af54..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_clock_restore.c +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* J514S diagnostic, not a clock driver. Restore only the single observed - * before/after PMP difference at the ADT-qualified display clock register. - * Do not infer this bit's semantics or use on another boot state/platform. - * Run under the initialization watchdog, with native DCP unattached. - */ -#include -#include -#include -#include - -static int __init restore(void) -{ - void __iomem *reg; - u32 before, after; - - if (!of_machine_is_compatible("apple,j514s") || - !of_machine_is_compatible("apple,t6030")) - return -ENODEV; - reg = ioremap_np(0x350040000ULL, 0x4000); - if (!reg) - return -ENOMEM; - before = readl(reg + 0x64); - if (before != 0x87100000) { - pr_err("M3 DCP clock restore refuses unexpected %#x\n", before); - iounmap(reg); - return -EINVAL; - } - writel(0x85100000, reg + 0x64); - after = readl(reg + 0x64); - pr_info("M3 DCP clock restore before=%#x immediate=%#x\n", before, after); - msleep(100); - pr_info("M3 DCP clock restore after100ms=%#x\n", readl(reg + 0x64)); - iounmap(reg); - return 0; -} -module_init(restore); -MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("J514S restore observed pre-PMP display clock bit; reboot to remove"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_hold.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_hold.c deleted file mode 100644 index 42fd6729b68eb..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_hold.c +++ /dev/null @@ -1,57 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Explicit J514S experiment: retain inherited display domains across PMP boot. - * No unload: keep the power floor until reboot, including on probe failure. - */ -#include -#include -#include -#include -#include - -static int __init hold(void) -{ - const u32 offsets[] = {0x1c0, 0x258, 0x10000}; - struct regmap *pmgr; - struct device_node *node; - int ret; - u32 before[ARRAY_SIZE(offsets)]; - - if (!of_machine_is_compatible("apple,j514s")) - return -ENODEV; - node = of_find_node_by_path("/soc/power-management@350700000"); - if (!node) - return -ENODEV; - pmgr = syscon_node_to_regmap(node); - of_node_put(node); - if (IS_ERR(pmgr)) - return PTR_ERR(pmgr); - for (unsigned int i = 0; i < ARRAY_SIZE(offsets); i++) { - pr_info("M3 DCP hold reading PMGR +%#x through syscon\n", offsets[i]); - msleep(100); - ret = regmap_read(pmgr, offsets[i], &before[i]); - if (ret) - return ret; - pr_info("M3 DCP hold before PMGR +%#x = %#x\n", offsets[i], before[i]); - if ((before[i] & 0xff) != 0xff || (before[i] & BIT(31))) { - return -EBUSY; - } - } - for (unsigned int i = 0; i < ARRAY_SIZE(offsets); i++) { - /* Same PS_MIN field used by apple,min-state in pmgr-pwrstate.c. - * Preserve automatic control and clear write-one-to-clear history. - */ - pr_info("M3 DCP hold raising PMGR +%#x minimum to active\n", offsets[i]); - msleep(100); - ret = regmap_update_bits(pmgr, offsets[i], GENMASK(19, 16) | GENMASK(9, 8), - GENMASK(19, 16)); - if (ret) - return ret; - ret = regmap_read(pmgr, offsets[i], &before[i]); - if (ret) - return ret; - pr_info("M3 DCP hold after PMGR +%#x = %#x\n", offsets[i], before[i]); - } - return 0; -} -module_init(hold); -MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.c deleted file mode 100644 index 2510823221882..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.c +++ /dev/null @@ -1,734 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* J514S native SID0 scanout. Explicit-load, fixed inherited panel mode only. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "m3_dcp_native.h" -#include "m3_dcp_kms.h" -#include "m3_dcp_brightness.h" - -/* Diagnostic history capacity, unrelated to submission or buffering limits. */ -#define M3_PRESENT_HISTORY 256 -struct m3_present_record { - u64 sequence, submitted_ns, completed_ns; - u32 framebuffer; - s32 status; -}; - -struct m3_kms { - struct drm_device drm; - struct drm_plane plane; - struct drm_crtc crtc; - struct drm_encoder encoder; - struct drm_connector connector; - struct drm_display_mode mode; - struct m3_dcp_native *native; - u32 brightness_nits; - /* Keep scanout pinned independently of atomic state until real completion. */ - struct drm_framebuffer *active_fb, *failed_fb; - struct mutex present_lock; - bool failed; - u64 completed; - u64 swap_total_ns, swap_max_ns; - spinlock_t history_lock; - u64 history_count; - struct m3_present_record history[M3_PRESENT_HISTORY]; -}; - -static struct m3_kms *to_m3(struct drm_device *drm) -{ - return container_of(drm, struct m3_kms, drm); -} - -static enum drm_mode_status m3_mode_valid(struct drm_crtc *crtc, - const struct drm_display_mode *mode) -{ - return drm_mode_equal(mode, &to_m3(crtc->dev)->mode) ? MODE_OK : MODE_BAD; -} - -static int m3_plane_check(struct drm_plane *plane, struct drm_atomic_state *state) -{ - struct drm_plane_state *p = drm_atomic_get_new_plane_state(state, plane); - struct drm_crtc_state *c; - struct drm_gem_dma_object *obj; - u64 bytes; - int ret; - - if (READ_ONCE(to_m3(plane->dev)->failed)) - return -EIO; - if (!p->crtc) - return 0; - c = drm_atomic_get_crtc_state(state, p->crtc); - if (IS_ERR(c)) - return PTR_ERR(c); - ret = drm_atomic_helper_check_plane_state(p, c, DRM_PLANE_NO_SCALING, - DRM_PLANE_NO_SCALING, false, true); - if (ret || !p->visible) - return ret; - /* Only the qualified full-panel, uncompressed, single-plane layout. */ - if (p->src_x || p->src_y || p->crtc_x || p->crtc_y || - p->src_w != 3024U << 16 || p->src_h != 1964U << 16 || - p->crtc_w != 3024 || p->crtc_h != 1964 || - p->fb->width != 3024 || p->fb->height != 1964 || - p->fb->modifier != DRM_FORMAT_MOD_LINEAR || p->fb->offsets[0] || - p->fb->pitches[0] < 12096 || (p->fb->pitches[0] & 63)) - return -EINVAL; - obj = drm_fb_dma_get_gem_obj(p->fb, 0); - bytes = (u64)p->fb->pitches[0] * p->fb->height; - if (!obj || bytes > obj->base.size || bytes > U32_MAX || !obj->dma_addr) - return -EINVAL; - return 0; -} - -static void m3_plane_update(struct drm_plane *plane, struct drm_atomic_state *state) -{ - /* One RPC contains the complete atomic state, submitted by CRTC flush. */ -} - -static const struct drm_plane_helper_funcs m3_plane_helpers = { - .prepare_fb = drm_gem_plane_helper_prepare_fb, - .atomic_check = m3_plane_check, - .atomic_update = m3_plane_update, - .atomic_disable = m3_plane_update, -}; - -static const struct drm_plane_funcs m3_plane_funcs = { - .update_plane = drm_atomic_helper_update_plane, - .disable_plane = drm_atomic_helper_disable_plane, - .destroy = drm_plane_cleanup, - .reset = drm_atomic_helper_plane_reset, - .atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state, - .atomic_destroy_state = drm_atomic_helper_plane_destroy_state, -}; - -static void m3_surface(u8 *s, struct drm_framebuffer *fb) -{ - u32 stride = fb->pitches[0], bytes = stride * fb->height; - - memset(s, 0, 0x22c); - /* DCP has no XRGB8. Premultiplied RGB over black preserves XRGB pixels. */ - s[2] = fb->format->format == DRM_FORMAT_XRGB8888; - put_unaligned_le32(1, s + 3); - put_unaligned_le32(1, s + 7); - put_unaligned_le32(0x42475241, s + 0xb); - s[0x13] = 13; - s[0x14] = 12; - put_unaligned_le32(stride, s + 0x15); - put_unaligned_le16(1, s + 0x19); - s[0x1b] = s[0x1c] = 1; - put_unaligned_le32(fb->width, s + 0x21); - put_unaligned_le32(fb->height, s + 0x25); - put_unaligned_le32(bytes, s + 0x29); - put_unaligned_le32(1, s + 0x35); - put_unaligned_le64(1, s + 0x51); - put_unaligned_le32(fb->width, s + 0x59); - put_unaligned_le32(fb->height, s + 0x5d); - put_unaligned_le32(stride, s + 0x69); - put_unaligned_le32(bytes, s + 0x6d); - put_unaligned_le16(4, s + 0x71); - s[0x73] = s[0x74] = 1; - put_unaligned_le64(1, s + 0x149); -} - -static void m3_finish_event(struct drm_crtc *crtc, struct drm_atomic_state *state, bool success) -{ - struct drm_crtc_state *c = drm_atomic_get_new_crtc_state(state, crtc); - struct drm_pending_vblank_event *event = c->event; - unsigned long flags; - - c->event = NULL; - if (!event) - return; - if (!success) { - /* Release helper waiters without reporting a successful page flip. */ - if (event->base.fence) { - dma_fence_set_error(event->base.fence, -EIO); - dma_fence_signal(event->base.fence); - } - if (event->base.completion) { - complete_all(event->base.completion); - event->base.completion_release(event->base.completion); - } - drm_event_cancel_free(crtc->dev, &event->base); - return; - } - spin_lock_irqsave(&crtc->dev->event_lock, flags); - drm_crtc_send_vblank_event(crtc, event); - spin_unlock_irqrestore(&crtc->dev->event_lock, flags); -} - -static bool m3_present_locked(struct m3_kms *m3, struct drm_framebuffer *fb) -{ - u8 surface[0x22c]; - dma_addr_t dva = 0; - u64 start, elapsed; - int ret; - - lockdep_assert_held(&m3->present_lock); - if (m3->failed) - return false; - if (fb) { - drm_framebuffer_get(fb); - m3_surface(surface, fb); - dva = drm_fb_dma_get_gem_obj(fb, 0)->dma_addr; - } - start = ktime_get_ns(); - ret = m3_dcp_native_swap(m3->native, fb ? surface : NULL, dva, - fb ? fb->width : 0, fb ? fb->height : 0, - fb ? m3->brightness_nits : 0); - elapsed = ktime_get_ns() - start; - /* Never make a timing reader wait on present_lock or copy scanout pixels. */ - spin_lock(&m3->history_lock); - m3->history[m3->history_count % M3_PRESENT_HISTORY] = - (struct m3_present_record) { - .sequence = m3->history_count + 1, - .submitted_ns = start, - .completed_ns = start + elapsed, - .framebuffer = fb ? fb->base.id : 0, - .status = ret, - }; - m3->history_count++; - spin_unlock(&m3->history_lock); - if (ret) { - /* Either buffer may still be scanned out. Never unmap them on error. */ - m3->failed_fb = fb; - WRITE_ONCE(m3->failed, true); - drm_err(&m3->drm, "native DCP flip failed %d; buffers pinned until reboot\n", ret); - return false; - } - if (m3->active_fb) - drm_framebuffer_put(m3->active_fb); - m3->active_fb = fb; - m3->completed++; - m3->swap_total_ns += elapsed; - m3->swap_max_ns = max(m3->swap_max_ns, elapsed); - if (m3->completed <= 3 || !(m3->completed % 600)) - drm_info(&m3->drm, "native KMS completed=%llu surface=%d imported=%d\n", - m3->completed, !!fb, fb && !!fb->obj[0]->import_attach); - return true; -} - -static bool m3_present(struct m3_kms *m3, struct drm_framebuffer *fb) -{ - bool success; - - mutex_lock(&m3->present_lock); - success = m3_present_locked(m3, fb); - mutex_unlock(&m3->present_lock); - return success; -} - -static int m3_backlight_update(struct backlight_device *backlight) -{ - struct m3_kms *m3 = bl_get_data(backlight); - int ret = 0; - - /* Serialize with KMS and retain the active framebuffer through D589. - * Brightness must also work when an idle compositor submits no frames. - * An inactive CRTC retains the request until its next enabling swap. - */ - mutex_lock(&m3->present_lock); - if (m3->failed) { - ret = -EIO; - goto out; - } - m3->brightness_nits = backlight_get_brightness(backlight); - /* A repeated userspace request must repair firmware state as well. */ - m3_dcp_native_brightness_invalidate(m3->native); - if (m3->active_fb && !m3_present_locked(m3, m3->active_fb)) - ret = -EIO; -out: - mutex_unlock(&m3->present_lock); - return ret; -} - -static const struct backlight_ops m3_backlight_ops = { - .update_status = m3_backlight_update, -}; - -static void m3_crtc_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) -{ - struct m3_kms *m3 = to_m3(crtc->dev); - bool success; - - if (!drm_atomic_get_new_crtc_state(state, crtc)->active) - return; - success = m3_present(m3, drm_atomic_get_new_plane_state(state, &m3->plane)->fb); - if (success) - drm_crtc_handle_vblank(crtc); - m3_finish_event(crtc, state, success); -} - -static void m3_crtc_enable(struct drm_crtc *crtc, struct drm_atomic_state *state) -{ - const struct drm_vblank_crtc_config config = { .offdelay_ms = 0 }; - - /* Firmware may reset brightness while the panel is powered down. */ - m3_dcp_native_brightness_invalidate(to_m3(crtc->dev)->native); - drm_crtc_vblank_on_config(crtc, &config); -} - -static void m3_crtc_disable(struct drm_crtc *crtc, struct drm_atomic_state *state) -{ - bool success = m3_present(to_m3(crtc->dev), NULL); - - if (success) - drm_crtc_handle_vblank(crtc); - m3_finish_event(crtc, state, success); - drm_crtc_vblank_off(crtc); -} - -static int m3_enable_vblank(struct drm_crtc *crtc) { return 0; } -static void m3_disable_vblank(struct drm_crtc *crtc) { } - -static const struct drm_crtc_helper_funcs m3_crtc_helpers = { - .mode_valid = m3_mode_valid, - .atomic_enable = m3_crtc_enable, - .atomic_disable = m3_crtc_disable, - .atomic_flush = m3_crtc_flush, -}; - -static const struct drm_crtc_funcs m3_crtc_funcs = { - .set_config = drm_atomic_helper_set_config, - .page_flip = drm_atomic_helper_page_flip, - .destroy = drm_crtc_cleanup, - .reset = drm_atomic_helper_crtc_reset, - .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, - .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state, - .enable_vblank = m3_enable_vblank, - .disable_vblank = m3_disable_vblank, -}; - -static int m3_get_modes(struct drm_connector *connector) -{ - struct drm_display_mode *mode = drm_mode_duplicate(connector->dev, - &to_m3(connector->dev)->mode); - if (!mode) - return 0; - connector->display_info.width_mm = mode->width_mm; - connector->display_info.height_mm = mode->height_mm; - mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED; - drm_mode_probed_add(connector, mode); - return 1; -} - -static enum drm_connector_status m3_detect(struct drm_connector *connector, bool force) -{ - return connector_status_connected; -} - -static const struct drm_connector_helper_funcs m3_connector_helpers = { - .get_modes = m3_get_modes, -}; - -static const struct drm_connector_funcs m3_connector_funcs = { - .detect = m3_detect, - .fill_modes = drm_helper_probe_single_connector_modes, - .destroy = drm_connector_cleanup, - .reset = drm_atomic_helper_connector_reset, - .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, - .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, -}; - -static const struct drm_encoder_funcs m3_encoder_funcs = { - .destroy = drm_encoder_cleanup, -}; - -static int m3_atomic_check(struct drm_device *drm, struct drm_atomic_state *state) -{ - struct m3_kms *m3 = to_m3(drm); - struct drm_crtc_state *c; - int ret; - - if (READ_ONCE(m3->failed)) - return -EIO; - c = drm_atomic_get_crtc_state(state, &m3->crtc); - if (IS_ERR(c)) - return PTR_ERR(c); - ret = drm_atomic_add_affected_planes(state, &m3->crtc); - return ret ?: drm_atomic_helper_check(drm, state); -} - -static void m3_commit_tail(struct drm_atomic_state *state) -{ - /* Each flush waits for its matching D589 before releasing the previous FB. - * Waiting for an additional vblank here would stall an idle display. - */ - drm_atomic_helper_commit_modeset_disables(state->dev, state); - drm_atomic_helper_commit_modeset_enables(state->dev, state); - drm_atomic_helper_commit_planes(state->dev, state, DRM_PLANE_COMMIT_ACTIVE_ONLY); - drm_atomic_helper_commit_hw_done(state); - drm_atomic_helper_cleanup_planes(state->dev, state); -} - -static const struct drm_mode_config_funcs m3_mode_funcs = { - .fb_create = drm_gem_fb_create, - .atomic_check = m3_atomic_check, - .atomic_commit = drm_atomic_helper_commit, -}; -static const struct drm_mode_config_helper_funcs m3_mode_helpers = { - .atomic_commit_tail = m3_commit_tail, -}; - -static int m3_dumb_create(struct drm_file *file, struct drm_device *drm, - struct drm_mode_create_dumb *args) -{ - int ret; - - /* The compositor also allocates external-output buffers on its primary - * DRM device. Limit scanout geometry in the plane check, not allocations. - * The DRM helper checks pitch/size overflow before applying alignment. */ - if (args->bpp != 32) - return -EINVAL; - ret = drm_mode_size_dumb(drm, args, 64, 0); - if (ret) - return ret; - return drm_gem_dma_dumb_create_internal(file, drm, args); -} - -DEFINE_DRM_GEM_DMA_FOPS(m3_fops); -static const struct drm_driver m3_driver = { - /* DRM core owns syncobj timelines, including fences imported from the - * separate render device. Compositors query this display node too. */ - .driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC | - DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE, - DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(m3_dumb_create), - .fops = &m3_fops, - .name = "m3-dcp", - .desc = "J514S native DCP scanout", - .major = 0, - .minor = 1, -}; - - -static int m3_status(struct seq_file *seq, void *unused) -{ - struct drm_debugfs_entry *entry = seq->private; - struct m3_kms *m3 = to_m3(entry->dev); - struct drm_framebuffer *fb; - - mutex_lock(&m3->present_lock); - fb = m3->active_fb; - seq_printf(seq, "completed %llu\nfailed %u\nactive_fb %u\nimported %u\n", - m3->completed, m3->failed, fb ? fb->base.id : 0, - fb && !!fb->obj[0]->import_attach); - seq_printf(seq, "swap_total_us %llu\nswap_max_us %llu\n", - m3->swap_total_ns / 1000, m3->swap_max_ns / 1000); - mutex_unlock(&m3->present_lock); - return 0; -} - -static int m3_analytics(struct seq_file *seq, void *unused) -{ - struct drm_debugfs_entry *entry = seq->private; - struct m3_kms *m3 = to_m3(entry->dev); - - m3_dcp_native_analytics_show(m3->native, seq); - return 0; -} - -static int m3_completion(struct seq_file *seq, void *unused) -{ - struct drm_debugfs_entry *entry = seq->private; - struct m3_kms *m3 = to_m3(entry->dev); - - m3_dcp_native_completion_show(m3->native, seq); - return 0; -} - -static int m3_present_history(struct seq_file *seq, void *unused) -{ - struct drm_debugfs_entry *entry = seq->private; - struct m3_kms *m3 = to_m3(entry->dev); - struct m3_present_record *records; - u64 count, first, i; - - records = kmalloc(sizeof(m3->history), GFP_KERNEL); - if (!records) - return -ENOMEM; - spin_lock(&m3->history_lock); - count = m3->history_count; - memcpy(records, m3->history, sizeof(m3->history)); - spin_unlock(&m3->history_lock); - seq_puts(seq, "sequence submitted_ns completed_ns framebuffer status\n"); - first = count > M3_PRESENT_HISTORY ? count - M3_PRESENT_HISTORY : 0; - for (i = first; i < count; i++) { - struct m3_present_record *r = &records[i % M3_PRESENT_HISTORY]; - - seq_printf(seq, "%llu %llu %llu %u %d\n", r->sequence, - r->submitted_ns, r->completed_ns, r->framebuffer, r->status); - } - kfree(records); - return 0; -} - -struct m3_snapshot { - size_t size; - u8 bytes[]; -}; - -static int m3_snapshot_open(struct inode *inode, struct file *file) -{ - struct m3_kms *m3 = inode->i_private; - struct m3_snapshot *snapshot; - struct drm_framebuffer *fb; - struct dma_buf *dmabuf = NULL; - struct iosys_map map = {}; - u32 bytes; - int ret; - - ret = mutex_lock_interruptible(&m3->present_lock); - if (ret) - return ret; - fb = m3->active_fb; - if (m3->failed || !fb) { - ret = -ENODATA; - goto unlock; - } - bytes = fb->pitches[0] * fb->height; - if (bytes > SZ_32M) { - ret = -E2BIG; - goto unlock; - } - snapshot = kvmalloc(sizeof(*snapshot) + 32 + bytes, GFP_KERNEL); - if (!snapshot) { - ret = -ENOMEM; - goto unlock; - } - /* Only this root-only diagnostic maps or reads imported GPU pixels. */ - if (fb->obj[0]->import_attach) { - dmabuf = fb->obj[0]->import_attach->dmabuf; - ret = dma_buf_begin_cpu_access(dmabuf, DMA_FROM_DEVICE); - if (ret) - goto free; - ret = dma_buf_vmap_unlocked(dmabuf, &map); - if (ret) { - dma_buf_end_cpu_access(dmabuf, DMA_FROM_DEVICE); - goto free; - } - } else { - ret = drm_gem_dma_vmap(drm_fb_dma_get_gem_obj(fb, 0), &map); - if (ret) - goto free; - } - snapshot->size = 32 + bytes; - put_unaligned_le32(0x4246334d, snapshot->bytes); /* M3FB */ - put_unaligned_le32(fb->width, snapshot->bytes + 4); - put_unaligned_le32(fb->height, snapshot->bytes + 8); - put_unaligned_le32(fb->pitches[0], snapshot->bytes + 12); - put_unaligned_le32(fb->format->format, snapshot->bytes + 16); - put_unaligned_le32(fb->base.id, snapshot->bytes + 20); - put_unaligned_le32(!!dmabuf, snapshot->bytes + 24); - put_unaligned_le32(bytes, snapshot->bytes + 28); - iosys_map_memcpy_from(snapshot->bytes + 32, &map, 0, bytes); - if (dmabuf) { - dma_buf_vunmap_unlocked(dmabuf, &map); - ret = dma_buf_end_cpu_access(dmabuf, DMA_FROM_DEVICE); - if (ret) - goto free; - } - file->private_data = snapshot; - mutex_unlock(&m3->present_lock); - return 0; -free: - kvfree(snapshot); -unlock: - mutex_unlock(&m3->present_lock); - return ret; -} - -static ssize_t m3_snapshot_read(struct file *file, char __user *data, size_t size, loff_t *pos) -{ - struct m3_snapshot *snapshot = file->private_data; - - return simple_read_from_buffer(data, size, pos, snapshot->bytes, snapshot->size); -} - -static int m3_snapshot_release(struct inode *inode, struct file *file) -{ - kvfree(file->private_data); - return 0; -} - -static const struct file_operations m3_snapshot_fops = { - .owner = THIS_MODULE, - .open = m3_snapshot_open, - .read = m3_snapshot_read, - .release = m3_snapshot_release, - .llseek = default_llseek, -}; - -static void m3_panel_size(struct m3_kms *m3) -{ - struct device_node *dcp, *panel; - u32 width = 0, height = 0; - - /* This firmware's DisplayAttributes omits image dimensions. As with the - * M1/M2 internal panels, use board data rather than inventing an EDID or - * encoding a desktop scale. Native scanout includes the notch region. - */ - dcp = of_find_node_by_path("dcp"); - panel = of_get_compatible_child(dcp, "apple,panel"); - of_node_put(dcp); - if (panel) { - of_property_read_u32(panel, "width-mm", &width); - of_property_read_u32(panel, "height-mm", &height); - of_node_put(panel); - } - if (!width || !height || width > 2000 || height > 2000) { - drm_warn(&m3->drm, "panel physical size missing or invalid; update J514S device tree\n"); - return; - } - m3->mode.width_mm = width; - m3->mode.height_mm = height; - drm_info(&m3->drm, "panel physical size %ux%u mm from device tree\n", width, height); -} - -int m3_dcp_kms_register(struct m3_dcp_native *native, bool takeover) -{ - static const u32 formats[] = { DRM_FORMAT_XRGB8888, DRM_FORMAT_ARGB8888 }; - static const u64 modifiers[] = { DRM_FORMAT_MOD_LINEAR, DRM_FORMAT_MOD_INVALID }; - struct backlight_properties backlight_props = { - .type = BACKLIGHT_RAW, - .max_brightness = M3_PANEL_MAX_NITS, - .brightness = M3_PANEL_DEFAULT_NITS, - }; - struct backlight_device *backlight; - struct device_node *node; - struct platform_device *pdev; - struct m3_kms *m3; - struct drm_device *drm; - const void *timing; - u32 size; - int ret; - - node = of_find_node_by_path("disp0"); - if (!node || !of_device_is_available(node) || - !of_device_is_compatible(node, "apple,t6030-display-diagnostics") || - !of_property_read_bool(node, "apple,j514s-inherited-mappings")) { - of_node_put(node); - return -ENODEV; - } - pdev = of_find_device_by_node(node); - if (!pdev) { - of_node_put(node); - return -ENODEV; - } - /* Native client and its device mappings remain pinned until reboot. */ - ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(42)); - if (!ret) - ret = of_dma_configure(&pdev->dev, node, true); - of_node_put(node); - if (ret) - return ret; - m3 = devm_drm_dev_alloc(&pdev->dev, &m3_driver, struct m3_kms, drm); - if (IS_ERR(m3)) - return PTR_ERR(m3); - drm = &m3->drm; - m3->native = native; - m3->brightness_nits = M3_PANEL_DEFAULT_NITS; - mutex_init(&m3->present_lock); - spin_lock_init(&m3->history_lock); - timing = m3_dcp_native_property(native, "PreferredTimingElements", &size); - if (!timing) - return -ENODATA; - ret = m3_dcp_preferred_mode(timing, size, &m3->mode); - kfree(timing); - if (ret) - return ret; - m3_panel_size(m3); - ret = drmm_mode_config_init(drm); - if (ret) - return ret; - drm->mode_config.min_width = drm->mode_config.max_width = 3024; - drm->mode_config.min_height = drm->mode_config.max_height = 1964; - drm->mode_config.funcs = &m3_mode_funcs; - drm->mode_config.helper_private = &m3_mode_helpers; - ret = drm_universal_plane_init(drm, &m3->plane, 1, &m3_plane_funcs, formats, - ARRAY_SIZE(formats), modifiers, DRM_PLANE_TYPE_PRIMARY, NULL); - if (ret) - return ret; - drm_plane_helper_add(&m3->plane, &m3_plane_helpers); - ret = drm_crtc_init_with_planes(drm, &m3->crtc, &m3->plane, NULL, &m3_crtc_funcs, NULL); - if (ret) - return ret; - drm_crtc_helper_add(&m3->crtc, &m3_crtc_helpers); - ret = drm_encoder_init(drm, &m3->encoder, &m3_encoder_funcs, DRM_MODE_ENCODER_TMDS, NULL); - if (ret) - return ret; - m3->encoder.possible_crtcs = 1; - ret = drm_connector_init(drm, &m3->connector, &m3_connector_funcs, DRM_MODE_CONNECTOR_eDP); - if (ret) - return ret; - drm_connector_helper_add(&m3->connector, &m3_connector_helpers); - ret = drm_connector_attach_encoder(&m3->connector, &m3->encoder); - if (ret) - return ret; - ret = drm_vblank_init(drm, 1); - if (ret) - return ret; - drm_mode_config_reset(drm); - if (takeover) { - struct resource fb; - - node = of_find_compatible_node(NULL, NULL, "simple-framebuffer"); - ret = node ? of_address_to_resource(node, 0, &fb) : -ENODEV; - of_node_put(node); - if (ret) - return ret; - ret = aperture_remove_conflicting_devices(fb.start, resource_size(&fb), m3_driver.name); - if (ret) - return ret; - } - drm_debugfs_add_file(drm, "m3_status", m3_status, NULL); - drm_debugfs_add_file(drm, "m3_analytics", m3_analytics, NULL); - drm_debugfs_add_file(drm, "m3_completion", m3_completion, NULL); - drm_debugfs_add_file(drm, "m3_present_history", m3_present_history, NULL); - backlight = devm_backlight_device_register(&pdev->dev, "apple-panel-bl", - &pdev->dev, m3, &m3_backlight_ops, - &backlight_props); - if (IS_ERR(backlight)) - return PTR_ERR(backlight); - ret = drm_dev_register(drm, 0); - if (!ret) - debugfs_create_file("m3_scanout", 0400, drm->debugfs_root, m3, &m3_snapshot_fops); - if (!ret) - drm_info(drm, "M3 native KMS ready: %ux%u at %d Hz, SID0 DMA scanout\n", - m3->mode.hdisplay, m3->mode.vdisplay, drm_mode_vrefresh(&m3->mode)); - return ret; -} - -MODULE_IMPORT_NS("DMA_BUF"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.h deleted file mode 100644 index ae1b38a12a553..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_kms.h +++ /dev/null @@ -1,9 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -#ifndef M3_DCP_KMS_H -#define M3_DCP_KMS_H -#include -struct m3_dcp_native; -struct drm_display_mode; -int m3_dcp_preferred_mode(const void *blob, u32 size, struct drm_display_mode *mode); -int m3_dcp_kms_register(struct m3_dcp_native *native, bool takeover); -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_native.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_native.c deleted file mode 100644 index a48d4272f42e1..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_native.c +++ /dev/null @@ -1,739 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Hash-qualified J514S DCP protocol; a single kernel client owns all RPCs. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "m3_dcp_bridge.h" -#include "m3_dcp_native.h" -#include "m3_dcp_brightness.h" - -#define A(n) M3_DCP_TAG('A', n) -#define D(n) M3_DCP_TAG('D', n) -#define MAX_PROPERTIES 256 -#define MAX_RAW_PROPERTIES 32 - -struct m3_dcp_property { - u32 service; - char key[64]; - u64 value; -}; - -struct m3_dcp_native { - struct device *dev; - struct m3_dcp_bridge *bridge; - struct mutex lock; - struct work_struct idle_work; - bool failed; - bool opened; - bool brightness_valid; - u32 brightness_nits; - u32 stride; - u32 diagnostic_plane; - bool diagnostic_flat_surface; - struct m3_dcp_property properties[MAX_PROPERTIES]; - u32 property_count; - struct { u32 service; char key[64]; void *data; u32 size; } raw[MAX_RAW_PROPERTIES]; - u32 raw_count, raw_bytes; - void *chunk; - u32 chunk_size, chunk_offset; - u64 completion_count; - u8 completion_last[0x6f0]; - u64 analytics_count; - u8 analytics_last[4164]; -}; - -static void idle_callbacks(struct work_struct *work); - -static int callback(struct m3_dcp_bridge *b, void *cookie, u32 tag, - const void *input, u32 in_size, void *output, u32 out_size); - -static int call(struct m3_dcp_native *dcp, u32 tag, const void *in, u32 ins, - void *out, u32 outs, u32 completion) -{ - int ret; - - if (dcp->failed) - return -EIO; - dev_dbg(dcp->dev, "M3 kernel RPC %#x input=%u output=%u\n", tag, ins, outs); - ret = m3_dcp_bridge_call(dcp->bridge, tag, in, ins, out, outs, - completion, callback, dcp); - if (ret) { - dcp->failed = true; - dev_err(dcp->dev, "M3 kernel RPC %#x failed: %d; recovery requires reboot\n", tag, ret); - } - return ret; -} - -static int simple_call(struct m3_dcp_native *dcp, u32 tag, bool input, bool reply, bool expected) -{ - __le32 one = cpu_to_le32(1), result = 0; - int ret = call(dcp, tag, input ? &one : NULL, input ? 4 : 0, - reply ? &result : NULL, reply ? 4 : 0, 0); - - if (!ret && reply && le32_to_cpu(result) != expected) - return -EPROTO; - return ret; -} - -static struct m3_dcp_property *property(struct m3_dcp_native *dcp, - u32 service, const u8 *key, bool create) -{ - struct m3_dcp_property *p; - u32 i; - - if (!memchr(key, 0, 64)) - return ERR_PTR(-EINVAL); - for (i = 0; i < dcp->property_count; i++) { - p = &dcp->properties[i]; - if (p->service == service && !strcmp(p->key, key)) - return p; - } - if (!create) - return NULL; - if (dcp->property_count == MAX_PROPERTIES) - return ERR_PTR(-ENOSPC); - p = &dcp->properties[dcp->property_count++]; - p->service = service; - strscpy(p->key, key, sizeof(p->key)); - return p; -} - -static int raw_property(struct m3_dcp_native *dcp, u32 service, const u8 *key, - const void *data, u32 size) -{ - void *copy; - u32 i, old_size = 0; - - if (!memchr(key, 0, 64) || size > SZ_1M) - return -EINVAL; - for (i = 0; i < dcp->raw_count; i++) - if (dcp->raw[i].service == service && !strcmp(dcp->raw[i].key, key)) - break; - if (i == MAX_RAW_PROPERTIES) - return -ENOSPC; - if (i < dcp->raw_count) - old_size = dcp->raw[i].size; - if (dcp->raw_bytes - old_size + size > SZ_8M) - return -ENOSPC; - copy = kvmemdup(data, size ?: 1, GFP_KERNEL); - if (!copy) - return -ENOMEM; - if (i == dcp->raw_count) - dcp->raw_count++; - kvfree(dcp->raw[i].data); - dcp->raw[i].service = service; - strscpy(dcp->raw[i].key, key, 64); - dcp->raw[i].data = copy; - dcp->raw[i].size = size; - dcp->raw_bytes += size - old_size; - return 0; -} - -static int callback(struct m3_dcp_bridge *b, void *cookie, u32 tag, - const void *input, u32 in_size, void *output, u32 out_size) -{ - struct m3_dcp_native *dcp = cookie; - const u8 *in = input; - u8 *out = output; - struct m3_dcp_property *p; - struct m3_dcp_buffer buffer = {}; - u32 count, offset, service, key_offset; - int ret; - - dev_dbg(dcp->dev, "M3 kernel callback %#x input=%u output=%u\n", tag, in_size, out_size); -#define SHAPE(i, o) (in_size == (i) && out_size == (o)) - if (tag == D(209) && SHAPE(0, 8)) { - put_unaligned_le64(ktime_to_ms(ktime_get_real()), out); - return 0; - } - if (tag == D(596) && SHAPE(0, 4)) - return 0; /* No default framebuffer was allocated by this host. */ - /* J514S 1e5e14/1e5e9c: event, parameter, u32 value, nullable byte. - * The getter copies output[0] back and returns output[4] as a boolean. - * Match the M2 host: no display tiling, and no setter implementation. - */ - if ((tag == D(115) && SHAPE(16, 8)) || - (tag == D(116) && SHAPE(16, 4))) { - if (in[12] > 1) - return -EINVAL; - dev_info(dcp->dev, "M3 tiling callback %#x event=%u parameter=%u value=%u null=%u\n", - tag, get_unaligned_le32(in), get_unaligned_le32(in + 4), - get_unaligned_le32(in + 8), in[12]); - if (tag == D(115)) - out[4] = 1; - return 0; - } - /* The transport zeroes output, so absent optional metadata stays absent. */ - if (tag == D(114) && SHAPE(4164, 4100)) { - /* J514S CoreAnalyticsSendEvent: name[64], serialized dictionary[4096], - * nullable flag. Retain the latest event locally. The dictionary is - * in/out: firmware parses it even on failure, so a zero-filled reply - * is invalid. Return its own serialized dictionary without changes. - * The final u32 is the method status (zero = accepted). - */ - if (!memchr(in, 0, 64) || in[4160] > 1 || - (!in[4160] && in[64] != 'd')) - return -EINVAL; - memcpy(dcp->analytics_last, in, sizeof(dcp->analytics_last)); - dcp->analytics_count++; - if (!in[4160]) - memcpy(out, in + 64, 4096); - put_unaligned_le32(0, out + 4096); - dev_info(dcp->dev, "M3 analytics event %llu: %.64s (retained locally)\n", - dcp->analytics_count, in); - return 0; - } - if (tag == D(589) && SHAPE(0x6f0, 0)) { - memcpy(dcp->completion_last, in, sizeof(dcp->completion_last)); - dcp->completion_count++; - return 0; - } - if ((tag == D(588) && SHAPE(8, 0)) || - (tag == D(598) && SHAPE(0, 0))) - return 0; - if (tag == D(599) && SHAPE(0, 0)) - return simple_call(dcp, A(410), false, true, true); - if ((tag == D(108) || tag == D(109) || tag == D(110) || tag == D(111) || - tag == D(112) || tag == D(113) || tag == D(0) || tag == D(1)) && SHAPE(0, 4)) { - out[0] = 1; - return 0; - } - if (tag == D(576) && SHAPE(88, 76)) { - if (!(in[84] & 1)) - memcpy(out, in + 8, 76); - return 0; - } - if ((tag == D(577) && SHAPE(4, 0)) || (tag == D(300) && SHAPE(16, 0))) - return 0; - if (tag == D(127) && SHAPE(4, 4)) { - count = get_unaligned_le32(in); - if (dcp->chunk || !count || count > 0x100001) - return -EINVAL; - dcp->chunk_size = count - 1; - dcp->chunk_offset = 0; - dcp->chunk = kvzalloc(max_t(u32, 1, dcp->chunk_size), GFP_KERNEL); - if (!dcp->chunk) - return -ENOMEM; - out[0] = 1; - return 0; - } - if (tag == D(128) && SHAPE(0x1008, 4)) { - offset = get_unaligned_le32(in + 0x1000); - count = get_unaligned_le32(in + 0x1004); - if (!dcp->chunk || offset != dcp->chunk_offset || count > 4096 || - offset > dcp->chunk_size || count > dcp->chunk_size - offset) - return -EINVAL; - memcpy(dcp->chunk + offset, in, count); - dcp->chunk_offset += count; - out[0] = 1; - return 0; - } - if (tag == D(129) && SHAPE(64, 4)) { - if (!dcp->chunk || dcp->chunk_offset != dcp->chunk_size || !memchr(in, 0, 64) || - dcp->raw_count == MAX_RAW_PROPERTIES || dcp->raw_bytes + dcp->chunk_size > SZ_8M) - return -EINVAL; - ret = raw_property(dcp, 0, in, dcp->chunk, dcp->chunk_size); - if (ret) - return ret; - kvfree(dcp->chunk); - dcp->chunk = NULL; - dev_info(dcp->dev, "M3 property %.64s: %u bytes\n", in, dcp->chunk_size); - out[0] = 1; - return 0; - } - if (tag == D(408) && SHAPE(8, 8)) { - count = get_unaligned_le32(in + 4); - if (memcmp(in, "VORP", 4) || count > 1) - return -EINVAL; - /* J514S boot ADT: disp0 clock-ids 348/412 index arm-io's - * clock-frequencies at 92/156 (IDs minus 256): 712 MHz / 0. - * The 48 MHz reference-clock fallback makes updateFrequencies - * reject its input because it is below the 402.28 MHz video clock. - * Report the inherited metadata; do not program clock registers. - */ - put_unaligned_le64(count ? 0 : 712000000, out); - dev_info(dcp->dev, "M3 display clock %u: %llu Hz (boot ADT)\n", - count, get_unaligned_le64(out)); - return 0; - } - if (tag == D(125) && SHAPE(100, 36)) { - count = get_unaligned_le32(in + 64); - if (count > 8) - return -EINVAL; - memcpy(out, in + 68, count * 4); - return 0; - } - if (tag == D(6) && SHAPE(60, 56)) { - if (!(in[56] & 1)) - memcpy(out, in, 56); - return 0; - } - if ((tag == D(122) || tag == D(123)) && SHAPE(0, 4)) - return 0; - if (tag == D(121) && SHAPE(0, 4)) { - static const struct { u32 tag; bool in, reply, expected; } sequence[] = { - {A(444), false, true, true}, {A(29)}, - {A(466), true}, {A(0), true, true, true}, {A(463), false, true, true}, - }; - for (count = 0; count < ARRAY_SIZE(sequence); count++) { - ret = simple_call(dcp, sequence[count].tag, sequence[count].in, - sequence[count].reply, sequence[count].expected); - if (ret) - return ret; - } - out[0] = 1; - return 0; - } - if (tag == D(101) && SHAPE(0, 4)) { - put_unaligned_le32(dcp->stride, out); - return 0; - } - if (tag == D(3) && SHAPE(4, 60)) { - put_unaligned_le32(1, out + 56); /* Explicitly decline bandwidth service. */ - return 0; - } - if (tag == D(451) && SHAPE(20, 28)) { - u32 alignment = get_unaligned_le32(in + 12); - - dev_info(dcp->dev, "M3 allocation flags=%#x size=%llu alignment=%u null=%u/%u/%u\n", - get_unaligned_le32(in), get_unaligned_le64(in + 4), alignment, - in[16], in[17], in[18]); - if (get_unaligned_le32(in) != 0x703 || !is_power_of_2(alignment) || alignment > SZ_16K || - in[16] || in[17] || in[18]) - return -EINVAL; - buffer.size = get_unaligned_le64(in + 4); - ret = m3_dcp_bridge_alloc(b, &buffer); - if (ret) - return ret; - if ((buffer.physical | buffer.dva) & (alignment - 1)) - return -EINVAL; - put_unaligned_le64(buffer.physical, out); - put_unaligned_le64(buffer.dva, out + 8); - put_unaligned_le64(buffer.size, out + 16); - put_unaligned_le32(buffer.id, out + 24); - return 0; - } - if (tag == D(201) && SHAPE(12, 20)) { - u64 id = get_unaligned_le64(in); - - if (id > U32_MAX || get_unaligned_le32(in + 8) != 1) - return -EINVAL; - buffer.id = id; - ret = m3_dcp_bridge_map(b, &buffer); - if (ret) - return ret; - /* Same vaddr/dva/status layout as M2, verified at TEXT 0x18b330. - * The AP virtual-address output is unused by this host. - */ - put_unaligned_le64(0, out); - put_unaligned_le64(buffer.dva, out + 8); - put_unaligned_le32(0, out + 16); - return 0; - } - if (tag == D(454) && SHAPE(4, 4)) { - ret = m3_dcp_bridge_retire(b, get_unaligned_le32(in)); - if (!ret) - out[0] = 1; - return ret; - } - if (tag == D(582) && SHAPE(8, 4)) - return 0; /* Optional default surface is not allocated by this host. */ - if (tag == D(400) && SHAPE(76, 0xc04)) { - count = get_unaligned_le32(in + 68); - service = get_unaligned_le32(in); - if (count > 0xc00 || (in[72] & 1) || !memchr(in + 4, 0, 64)) - return -EINVAL; - for (offset = 0; offset < dcp->raw_count; offset++) { - if (dcp->raw[offset].service != service || strcmp(dcp->raw[offset].key, in + 4)) - continue; - if (dcp->raw[offset].size > count) - return -ENOSPC; - memcpy(out, dcp->raw[offset].data, dcp->raw[offset].size); - put_unaligned_le32(dcp->raw[offset].size, out + 0xc00); - break; - } - return 0; - } - if (tag == D(401) && SHAPE(80, 12)) { - /* Match the existing M2 mini-LED host's PMUS compatibility - * response. This is a fixed calibration placeholder, NOT a - * measured M3 temperature and never a thermal-guard input. - * Firmware 19a4c4..19a4e8 converts centidegrees to 16.16. - */ - if (!memcmp(in, "SUMP", 4) && - !strncmp(in + 4, "Temperature", 64)) { - put_unaligned_le64(3029, out); - out[8] = 1; - dev_info_ratelimited(dcp->dev, - "M3 PMUS Temperature: M2 compatibility placeholder 3029\n"); - return 0; - } - p = property(dcp, get_unaligned_le32(in), in + 4, false); - if (IS_ERR(p)) - return PTR_ERR(p); - if (p) { - put_unaligned_le64(p->value, out); - out[8] = 1; - } else - dev_info_ratelimited(dcp->dev, - "M3 missing host property service=%#x key=%.64s\n", - get_unaligned_le32(in), in + 4); - return 0; - } - if ((tag == D(413) && SHAPE(4168, 4)) || - ((tag == D(552) || tag == D(561)) && SHAPE(4164, 4)) || - (tag == D(567) && SHAPE(128, 4))) { - key_offset = tag == D(413) ? 4 : 0; - service = key_offset ? get_unaligned_le32(in) : 0; - if (tag != D(567) && (in[key_offset + 4160] & 1)) { - out[0] = 1; - return 0; - } - count = tag == D(567) ? strnlen(in + 64, 64) : 4096; - ret = raw_property(dcp, service, in + key_offset, in + key_offset + 64, count); - if (!ret) - out[0] = 1; - return ret; - } - if ((tag == D(563) && SHAPE(76, 4)) || (tag == D(565) && SHAPE(72, 4)) || - (tag == D(414) && SHAPE(80, 4)) || (tag == D(415) && SHAPE(76, 4)) || - ((tag == D(102) || tag == D(104)) && SHAPE(68, 0))) { - key_offset = (tag == D(414) || tag == D(415)) ? 4 : 0; - service = key_offset ? get_unaligned_le32(in) : 0; - count = (tag == D(563) || tag == D(414)) ? 8 : 4; - if (out_size && (in[key_offset + 64 + count] & 1)) { - out[0] = 1; - return 0; - } - p = property(dcp, service, in + key_offset, true); - if (IS_ERR(p)) - return PTR_ERR(p); - p->value = count == 8 ? get_unaligned_le64(in + key_offset + 64) : - get_unaligned_le32(in + key_offset + 64); - if (tag == D(104)) - p->value = in[64]; - if (out_size) - out[0] = 1; - return 0; - } - if (tag == D(406) && SHAPE(72, 0)) - return 0; - if (tag == D(107) && SHAPE(64, 0)) { - p = property(dcp, 0, in, false); - if (IS_ERR(p)) - return PTR_ERR(p); - if (p) { - *p = dcp->properties[--dcp->property_count]; - memset(&dcp->properties[dcp->property_count], 0, sizeof(*p)); - } - return 0; - } - if ((tag == D(206) || tag == D(207)) && SHAPE(0, 4)) { - ret = simple_call(dcp, tag == D(206) ? A(131) : A(132), false, true, false); - if (!ret) - out[0] = 1; - return ret; - } - if (tag == D(100) && SHAPE(0, 0)) - { - __le32 result; - return call(dcp, A(374), NULL, 0, &result, 4, 0); - } - dev_err(dcp->dev, "M3 unhandled kernel callback %#x %u/%u\n", tag, in_size, out_size); - return -EOPNOTSUPP; -#undef SHAPE -} - -int m3_dcp_native_open(struct m3_dcp_native *dcp) -{ - int ret; - - mutex_lock(&dcp->lock); - if (dcp->opened) { - ret = -EALREADY; - } else { - ret = simple_call(dcp, A(455), false, false, false); - if (!ret) { - dcp->opened = true; - dev_info(dcp->dev, "M3 first-client-open complete\n"); - } - } - mutex_unlock(&dcp->lock); - return ret; -} - -struct m3_dcp_native *m3_dcp_native_start(struct device *dev, struct m3_dcp_bridge *bridge, - bool defer_open) -{ - struct m3_dcp_native *dcp; - struct device_node *fb; - int ret; - - dcp = devm_kzalloc(dev, sizeof(*dcp), GFP_KERNEL); - if (!dcp) - return ERR_PTR(-ENOMEM); - dcp->dev = dev; - dcp->bridge = bridge; - mutex_init(&dcp->lock); - INIT_WORK(&dcp->idle_work, idle_callbacks); - fb = of_find_compatible_node(NULL, NULL, "simple-framebuffer"); - ret = fb ? of_property_read_u32(fb, "stride", &dcp->stride) : -ENODEV; - of_node_put(fb); - if (ret || dcp->stride != 12096) - return ERR_PTR(-EINVAL); - ret = simple_call(dcp, A(401), false, true, true); - if (!ret && !defer_open) - ret = m3_dcp_native_open(dcp); - if (ret) - return ERR_PTR(ret); - m3_dcp_bridge_set_work(bridge, &dcp->idle_work); - if (defer_open) - dev_info(dev, "M3 DCP kernel startup complete; first-client-open deferred\n"); - else - dev_info(dev, "M3 DCP kernel startup and first-client-open complete\n"); - return dcp; -} - -/* J514S wrappers: A409@13f7d4 (4/4), A472@1421f4 (12/8), - * A411@13f8a8 (8/4). A472's output status is at byte 4; bytes 9/10 - * are additional boolean inputs and byte 11 is the output-pointer null flag. - * Follow M2's display-handle 0, then power-state 1 sequence explicitly. - */ -int m3_dcp_native_panel(struct m3_dcp_native *dcp, unsigned int action) -{ - u8 input[80] = {}, output[8] = {}; - int ret; - - mutex_lock(&dcp->lock); - switch (action) { - case 1: - dcp->brightness_valid = false; - ret = call(dcp, A(409), input, 4, output, 4, 0); - /* Existing internal-display handle 0 returns 2. */ - if (!ret && get_unaligned_le32(output) != 2) - ret = -EIO; - if (!ret) { - put_unaligned_le64(1, input); - ret = call(dcp, A(472), input, 12, output, 8, 0); - if (!ret && get_unaligned_le32(output + 4)) - ret = -EIO; - } - break; - case 2: - /* PreferredTimingElements: color 1, timing 2, 3024x1964. */ - put_unaligned_le32(1, input); - put_unaligned_le32(2, input + 4); - ret = call(dcp, A(411), input, 8, output, 4, 0); - if (!ret && get_unaligned_le32(output)) - ret = -EIO; - break; - case 3: - ret = call(dcp, A(479), input, 4, output, 8, 0); - break; - case 4: - /* Native stub kernel ab5a53c; firmware wrapper 140fe4. - * Same initialization used by the M2 host before client open. - */ - ret = call(dcp, A(448), input, 4, output, 4, 0); - if (!ret && get_unaligned_le32(output)) - ret = -EIO; - break; - case 5: - /* Native stub kernel ab57948; firmware wrapper 13fe2c. - * Location 9 is the identity CTM used by the M2 KMS path. - */ - put_unaligned_le32(9, input); - put_unaligned_le64(1ULL << 32, input + 4); - put_unaligned_le64(1ULL << 32, input + 4 + 4 * 8); - put_unaligned_le64(1ULL << 32, input + 4 + 8 * 8); - ret = call(dcp, A(421), input, 80, output, 4, 0); - if (!ret && get_unaligned_le32(output)) - ret = -EIO; - break; - case 6: - case 7: - /* Reversible diagnostic: M2's linear primary uses IOMFB plane 2. - * The next swap clears the other two primary slots atomically. - */ - dcp->diagnostic_plane = action == 6 ? 2 : 0; - put_unaligned_le32(dcp->diagnostic_plane, output); - ret = 0; - break; - case 8: - case 9: - /* Captured 26A428 linear BGRA uses the non-planar IOSurface - * representation. SurfaceContents fields match the stub driver. - */ - dcp->diagnostic_flat_surface = action == 8; - ret = 0; - break; - default: - ret = -EINVAL; - } - dev_info(dcp->dev, "M3 panel action %u ret=%d output=%#x/%#x\n", - action, ret, get_unaligned_le32(output), get_unaligned_le32(output + 4)); - if (ret) - dcp->failed = true; - mutex_unlock(&dcp->lock); - return ret; -} - -static int native_swap(struct m3_dcp_native *dcp, const void *surface, u64 dva, - u32 width, u32 height, u32 background, int brightness_nits) -{ - __le32 start[4] = {}, started[2], result[3]; - u8 *swap; - u32 id; - int ret; - - if (brightness_nits < -1 || brightness_nits > M3_PANEL_MAX_NITS) - return -EINVAL; - if (surface && (!dva || !width || width > 3024 || !height || height > 1964)) - return -EINVAL; - swap = kzalloc(0x1b58, GFP_KERNEL); - if (!swap) - return -ENOMEM; - mutex_lock(&dcp->lock); - ret = call(dcp, A(406), start, sizeof(start), started, sizeof(started), 0); - if (ret || le32_to_cpu(started[1])) { - ret = ret ?: -EIO; - goto out; - } - id = le32_to_cpu(started[0]); - if (!id) { - ret = -EPROTO; - goto out; - } - put_unaligned_le32(id, swap + 0x98); - put_unaligned_le32(0x80000007, swap + 0x14c); - put_unaligned_le32(0x80000007, swap + 0x150); - /* A disabled plane must blank to black, including compositor DPMS. */ - put_unaligned_le32(background, swap + 0x154); - memset(swap + 0x1b4b, 1, 10); - swap[0x1b56] = swap[0x1b57] = 1; - if (brightness_nits >= 0 && (!dcp->brightness_valid || - dcp->brightness_nits != brightness_nits)) - m3_dcp_encode_brightness(swap, brightness_nits); - if (surface) { - u32 plane = dcp->diagnostic_plane; - u8 *s = swap + 0x508 + plane * 0x22c; - - memcpy(s, surface, 0x22c); - if (dcp->diagnostic_flat_surface) { - put_unaligned_le32(0, s + 3); - put_unaligned_le32(0, s + 7); - s[0x14] = 1; /* Captured sRGB BGRA surface. */ - put_unaligned_le16(4, s + 0x19); - memset(s + 0x39, 0, 0x1ed - 0x39); - put_unaligned_le64(1, s + 0x51); - put_unaligned_le64(1, s + 0x149); - } - put_unaligned_le64(dva, swap + 0xdb8 + plane * 8); - put_unaligned_le32(1, swap + 0x9c + plane * 4); - put_unaligned_le32(width, swap + 0xb4 + plane * 16); - put_unaligned_le32(height, swap + 0xb8 + plane * 16); - put_unaligned_le32(width, swap + 0x114 + plane * 16); - put_unaligned_le32(height, swap + 0x118 + plane * 16); - swap[0x1b4b + plane] = 0; - } - ret = call(dcp, A(407), swap, 0x1b58, result, sizeof(result), id); - if (!ret && get_unaligned_le32((u8 *)result + 5)) - ret = -EIO; - /* Cache only after the matching frame completion and successful reply. */ - if (!ret && brightness_nits >= 0) { - dcp->brightness_nits = brightness_nits; - dcp->brightness_valid = true; - } - if (!ret && !surface) - dev_info(dcp->dev, "M3 kernel swap %u completed, surface=%d\n", id, !!surface); -out: - if (ret) - dcp->failed = true; - mutex_unlock(&dcp->lock); - kfree(swap); - return ret; -} - -int m3_dcp_native_swap(struct m3_dcp_native *dcp, const void *surface, u64 dva, - u32 width, u32 height, int brightness_nits) -{ - return native_swap(dcp, surface, dva, width, height, 0xff000000, brightness_nits); -} - -int m3_dcp_native_background(struct m3_dcp_native *dcp, u32 color) -{ - return native_swap(dcp, NULL, 0, 0, 0, color, -1); -} - -void m3_dcp_native_brightness_invalidate(struct m3_dcp_native *dcp) -{ - mutex_lock(&dcp->lock); - dcp->brightness_valid = false; - mutex_unlock(&dcp->lock); -} - -void *m3_dcp_native_property(struct m3_dcp_native *dcp, const char *key, u32 *size) -{ - void *copy = NULL; - u32 i; - - mutex_lock(&dcp->lock); - for (i = 0; i < dcp->raw_count; i++) { - if (!dcp->raw[i].service && !strcmp(dcp->raw[i].key, key)) { - *size = dcp->raw[i].size; - copy = kmemdup(dcp->raw[i].data, *size, GFP_KERNEL); - break; - } - } - mutex_unlock(&dcp->lock); - return copy; -} - -void m3_dcp_native_analytics_show(struct m3_dcp_native *dcp, struct seq_file *seq) -{ - mutex_lock(&dcp->lock); - seq_printf(seq, "events %llu\n", dcp->analytics_count); - if (dcp->analytics_count) { - seq_printf(seq, "last_event %.64s\ndictionary_present %u\n", - dcp->analytics_last, !dcp->analytics_last[4160]); - if (!dcp->analytics_last[4160]) - seq_hex_dump(seq, "", DUMP_PREFIX_OFFSET, 16, 1, - dcp->analytics_last + 64, 4096, false); - } - mutex_unlock(&dcp->lock); -} - -void m3_dcp_native_completion_show(struct m3_dcp_native *dcp, struct seq_file *seq) -{ - mutex_lock(&dcp->lock); - seq_printf(seq, "callbacks %llu\n", dcp->completion_count); - if (dcp->completion_count) - seq_hex_dump(seq, "", DUMP_PREFIX_OFFSET, 16, 1, - dcp->completion_last, sizeof(dcp->completion_last), false); - mutex_unlock(&dcp->lock); -} - -static void idle_callbacks(struct work_struct *work) -{ - struct m3_dcp_native *dcp = container_of(work, struct m3_dcp_native, idle_work); - int ret = 0; - - /* Share RPC ownership with atomic flips; this worker never steals an - * in-flight request's reply. Firmware notifications also need service - * when the desktop is static or the CRTC is disabled. */ - mutex_lock(&dcp->lock); - while (!dcp->failed && !ret) - ret = m3_dcp_bridge_pump(dcp->bridge, 0, callback, dcp); - if (ret && ret != -ETIMEDOUT && ret != -EAGAIN) { - dcp->failed = true; - dev_err(dcp->dev, "M3 idle callback failed %d; recovery requires reboot\n", ret); - } - mutex_unlock(&dcp->lock); -} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_native.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_native.h deleted file mode 100644 index 7516ed8a1df5f..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_native.h +++ /dev/null @@ -1,23 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -#ifndef M3_DCP_NATIVE_H -#define M3_DCP_NATIVE_H - -#define M3_DCP_TAG(c, n) (((u32)(c) << 24) | ((u32)('0' + (n) / 100) << 16) | \ - ((u32)('0' + (n) / 10 % 10) << 8) | (u32)('0' + (n) % 10)) -struct m3_dcp_native; -struct m3_dcp_bridge; -struct device; -struct seq_file; -void m3_dcp_native_completion_show(struct m3_dcp_native *dcp, struct seq_file *seq); -void m3_dcp_native_analytics_show(struct m3_dcp_native *dcp, struct seq_file *seq); -struct m3_dcp_native *m3_dcp_native_start(struct device *dev, struct m3_dcp_bridge *bridge, - bool defer_open); -int m3_dcp_native_open(struct m3_dcp_native *dcp); -int m3_dcp_native_swap(struct m3_dcp_native *dcp, const void *surface, u64 dva, - u32 width, u32 height, int brightness_nits); -void m3_dcp_native_brightness_invalidate(struct m3_dcp_native *dcp); -int m3_dcp_native_background(struct m3_dcp_native *dcp, u32 color); -int m3_dcp_native_panel(struct m3_dcp_native *dcp, unsigned int action); -/* Caller owns the returned snapshot and must kfree it. */ -void *m3_dcp_native_property(struct m3_dcp_native *dcp, const char *key, u32 *size); -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_parser.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_parser.c deleted file mode 100644 index 676c129209d9a..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_parser.c +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Reuse the existing serialized-property decoder without an apple_dcp object. */ -#include "m3_dcp_kms.h" -#define parse m3_property_parse -#define enumerate_modes m3_property_enumerate_modes -#define parse_display_attributes m3_property_display_attributes -#define parse_epic_service_init m3_property_epic_service_init -#define parse_sound_constraints m3_property_sound_constraints -#define parse_sound_mode m3_property_sound_mode -#define parse_system_log_mnits m3_property_system_log_mnits -#include "../../trace.h" -#define trace_iomfb_color_mode(...) do { } while (0) -#define trace_iomfb_timing_mode(...) do { } while (0) -#define trace_avep_sound_mode(...) do { } while (0) -#include "../../parser.c" - -int m3_dcp_preferred_mode(const void *blob, u32 size, struct drm_display_mode *mode) -{ - struct dcp_parse_ctx ctx = {}; - struct dcp_display_mode *modes; - unsigned int count; - int ret; - - ret = m3_property_parse(blob, size, &ctx); - if (ret) - return ret; - modes = m3_property_enumerate_modes(&ctx, &count, 0, 0, 0); - if (IS_ERR(modes)) - return PTR_ERR(modes); - ret = -EINVAL; - /* The qualified J514S session has one preferred physical panel timing. */ - if (count == 1 && modes[0].mode.hdisplay == 3024 && modes[0].mode.vdisplay == 1964 && - (drm_mode_vrefresh(&modes[0].mode) == 60 || drm_mode_vrefresh(&modes[0].mode) == 120)) { - *mode = modes[0].mode; - ret = 0; - } - kfree(modes); - return ret; -} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_ptd_vote.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_ptd_vote.c deleted file mode 100644 index 3edb157331f8c..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_ptd_vote.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* J514S PMP v2 display request, qualified by macOS pmgr-76 and ADT. - * Initialization diagnostic only, under the lifecycle watchdog. Reboot to - * discard the request. Never submit display or GPU work from this module. - */ -#include -#include -#include -#include -#include - -#ifndef M3_PTD_INSTANCE -#define M3_PTD_INSTANCE 0 -#endif -static bool external = M3_PTD_INSTANCE != 0; -module_param(external, bool, 0400); -MODULE_PARM_DESC(external, "Also request qualified J514S DISPEXT0; retain until reboot"); - -static int __init vote(void) -{ - struct device_node *node; - const u8 *table; - void __iomem *ptd; - u64 request, ack, mask, ack_mask = BIT_ULL(7); - int length, i, found = 0; - - pr_info("M3 DCP PTD diagnostic entered; no register access yet\n"); - if (!of_machine_is_compatible("apple,j514s") || - !of_machine_is_compatible("apple,t6030")) - return -ENODEV; - node = of_find_node_by_path("/soc/pmp@350500000"); - if (!node) - return -ENODEV; - table = of_get_property(node, "apple,tunable-soc-device", &length); - if (!table || length != 21 * 0x7c || - get_unaligned_le32(table + 7 * 0x7c) != 8 || - memcmp(table + 7 * 0x7c + 0x74, "DISP\0\0\0\0", 8)) - goto invalid; - /* macOS also votes for the in-use ANS controller (bit16). Without - * that dependency, firmware DVFS may stop storage before logs flush. - */ - if (get_unaligned_le32(table + 16 * 0x7c) != 17 || - memcmp(table + 16 * 0x7c + 0x74, "ANS\0\0\0\0\0", 8)) - goto invalid; - if (external) { - /* J514S retained boot ADT, distinct from DISP at index7. The - * power protocol is shared; do not copy macOS26's device indices. */ - unsigned int index = 8 + M3_PTD_INSTANCE; - if (get_unaligned_le32(table + index * 0x7c) != index + 1 || - memcmp(table + index * 0x7c + 0x74, M3_PTD_INSTANCE ? "DISPEXT1" : "DISPEXT0", 8)) - goto invalid; - ack_mask |= BIT_ULL(index); - } - table = of_get_property(node, "apple,tunable-ptd-range", &length); - if (!table || length % 32) - goto invalid; - for (i = 0; i < length; i += 32) { - u32 id = get_unaligned_le32(table + i); - if (id != 10 && id != 11) - continue; - if (get_unaligned_le32(table + i + 4) != (id == 10 ? 280 : 284) || - get_unaligned_le32(table + i + 8) != 4) - goto invalid; - found |= id == 10 ? 1 : 2; - } - if (found != 3) - goto invalid; - of_node_put(node); - pr_info("M3 DCP PTD display/ANS and range tables validated\n"); - ptd = ioremap_np(0x3503c0000ULL, 0x14000); - if (!ptd) - return -ENOMEM; - pr_info("M3 DCP PTD mapping ready; reading status\n"); - msleep(100); - for (i = 0; i < 100 && readq(ptd + 16) != 1; i++) - usleep_range(1000, 2000); - if (readq(ptd + 16) != 1) { - iounmap(ptd); - return -EAGAIN; - } - pr_info("M3 DCP PTD status ready; reading request/ack\n"); - msleep(100); - request = readq(ptd + 280 * 16); - ack = readq(ptd + 284 * 16); - pr_info("M3 DCP PTD display before request=%#llx ack=%#llx\n", request, ack); - /* Preserve every other client bit. No other diagnostic writer may run. */ - /* DISP flags0xb require acknowledgement; ANS flags0 do not. Match - * ApplePMGR's flags-bit1 check, rather than requiring ANS in the ACK. - */ - if ((request ^ ack) & ack_mask) { - iounmap(ptd); - return -EBUSY; - } - mask = ack_mask | BIT_ULL(16); - pr_info("M3 DCP PTD writing display/ANS request through non-posted mapping\n"); - msleep(100); - writeq(request | mask, ptd + 0x10000 + 280 * 8); - for (i = 0; i < 100; i++) { - ack = readq(ptd + 284 * 16); - if ((ack & ack_mask) == ack_mask) - break; - usleep_range(1000, 2000); - } - pr_info("M3 DCP PTD display after request=%#llx ack=%#llx polls=%d\n", - readq(ptd + 280 * 16), ack, i); - iounmap(ptd); - return (ack & ack_mask) == ack_mask ? 0 : -ETIMEDOUT; -invalid: - of_node_put(node); - return -EINVAL; -} -module_init(vote); -MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("J514S isolated PMP v2 PTD display request; reboot to remove"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_query.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_query.c deleted file mode 100644 index 18476a9659958..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_query.c +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Repeat the already-qualified A410 query under an external watchdog. */ -#include -#include -#include -#include -#include "../../dcp-link.h" -#include "m3_dcp_state.h" -static bool dump; -module_param(dump, bool, 0400); - -static int __init query(void) -{ - struct device_node *node; - struct platform_device *pdev; - struct m3_dcp_rtkit *dcp; - int ret; - - if (!of_machine_is_compatible("apple,j514s")) - return -ENODEV; - node = of_find_node_by_path("/soc/dcp@28ec00000"); - pdev = of_find_device_by_node(node); - of_node_put(node); - if (!pdev) - return -ENODEV; - if (!pdev->dev.driver || strcmp(pdev->dev.driver->name, "m3-dcp-rtkit")) { - ret = -ENODEV; - goto out; - } - dcp = platform_get_drvdata(pdev); - if (dump && dcp && dcp->rpc) { - for (unsigned int offset = 0; offset < 0x80000; offset += 0x20000) { - pr_info("M3 DCP RPC snapshot offset=%#x\n", offset); - print_hex_dump(KERN_INFO, "M3 RPC ", DUMP_PREFIX_OFFSET, 16, 1, - dcp->rpc + offset, 256, false); - } - ret = 0; - goto out; - } - if (!dcp || dcp->result || dcp->awaiting_rpc || !dcp->rpc) { - ret = -EBUSY; - goto out; - } - reinit_completion(&dcp->ready); - apple_dcp_link_rpc_header_encode(dcp->rpc, 0x41343130, 0, 4); - *(__le32 *)(dcp->rpc + 12) = cpu_to_le32(0xa5); - dcp->result = -ETIMEDOUT; - dcp->awaiting_rpc = true; - dma_wmb(); - ret = apple_rtkit_send_message(dcp->rtkit, APPLE_DCP_LINK_ENDPOINT, - (16ULL << 32) | APPLE_DCP_LINK_MSG_RPC, NULL, false); - if (!ret) - ret = wait_for_completion_timeout(&dcp->ready, msecs_to_jiffies(3000)) ? - dcp->result : -ETIMEDOUT; - pr_info("M3 repeated A410 query ret=%d value=%u\n", ret, - le32_to_cpup((__le32 *)(dcp->rpc + 12))); -out: - put_device(&pdev->dev); - return ret; -} -static void __exit done(void) {} -module_init(query); -module_exit(done); -MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("J514S one-shot query of an existing DCP diagnostic session"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_regs.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_regs.c deleted file mode 100644 index 37a37cfaa5658..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_regs.c +++ /dev/null @@ -1,62 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* J514S readl-only diagnostic: never use vector loads on DART registers. */ -#include -#include -#include -#include -static int __init probe(void) -{ - const phys_addr_t bases[] = {0x28d30c000ULL, 0x28d304000ULL}; - const unsigned int sid[] = {5, 0, 4}; - if (!of_machine_is_compatible("apple,j514s")) return -ENODEV; - for (unsigned int i=0; i<3; i++) { - phys_addr_t base=bases[i != 0]; - const unsigned int offsets[] = {0x200,0x1000+4*sid[i],0x1400+4*sid[i]}; - void __iomem *regs=ioremap(base,0x4000); - if (!regs) return -ENOMEM; - for (unsigned int n=0; n<3; n++) { - pr_info("M3 DCP reading %pap + %#x using readl\n", &base, offsets[n]); - msleep(100); - pr_info("M3 DCP register %pap + %#x = %#x\n", &base, offsets[n], readl(regs+offsets[n])); - } - iounmap(regs); - } - { - /* Display clock registers from J514S ADT clock-frequencies-regs entries - * 92/156, plus the three real display power-state registers. Parent - * LW00/LS0/IOA0/AFI and display leaf entries have no_ps: do not map them. - */ - const phys_addr_t addresses[] = {0x28ec00044ULL,0x28ec00048ULL, - 0x3507001c0ULL,0x350700258ULL,0x350710000ULL, - 0x350040064ULL,0x350040164ULL,0x28e850000ULL,0x28e850004ULL}; - for (unsigned int i=0; i -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../../dcp-link.h" - -#include "m3_dcp_state.h" -#include "m3_dcp_bridge.h" -#include "m3_dcp_native.h" -#include "m3_dcp_kms.h" -#include -#include - -static bool kernel_client; -module_param(kernel_client, bool, 0400); -static bool defer_client_open; -module_param(defer_client_open, bool, 0400); -MODULE_PARM_DESC(defer_client_open, "Pause native startup before A455 for staged takeover diagnosis"); - -static struct m3_dcp_rtkit *active; -static DEFINE_MUTEX(control_lock); -static bool kms_registered; -static bool client_opened; -static void save_properties(struct m3_dcp_rtkit *dcp); -static int control_set(const char *value, const struct kernel_param *param) -{ - int ret; - - guard(mutex)(&control_lock); - if (!active || IS_ERR_OR_NULL(active->native) || active->result) - return -EBUSY; - if (sysfs_streq(value, "client-open")) { - if (client_opened) - return -EALREADY; - ret = m3_dcp_native_open(active->native); - if (!ret) { - client_opened = true; - save_properties(active); - } - } else if (!client_opened) { - return -EAGAIN; - } else if (sysfs_streq(value, "background")) { - ret = m3_dcp_native_swap(active->native, NULL, 0, 0, 0, -1); - } else if (sysfs_streq(value, "background-green")) { - ret = m3_dcp_native_background(active->native, 0xff00ff00); - } else if (sysfs_streq(value, "panel-on")) { - ret = m3_dcp_native_panel(active->native, 1); - } else if (sysfs_streq(value, "panel-mode")) { - ret = m3_dcp_native_panel(active->native, 2); - } else if (sysfs_streq(value, "panel-state")) { - ret = m3_dcp_native_panel(active->native, 3); - } else if (sysfs_streq(value, "panel-no-idle")) { - ret = m3_dcp_native_panel(active->native, 4); - } else if (sysfs_streq(value, "panel-identity")) { - ret = m3_dcp_native_panel(active->native, 5); - } else if (sysfs_streq(value, "panel-plane2")) { - ret = m3_dcp_native_panel(active->native, 6); - } else if (sysfs_streq(value, "panel-plane0")) { - ret = m3_dcp_native_panel(active->native, 7); - } else if (sysfs_streq(value, "panel-flat")) { - ret = m3_dcp_native_panel(active->native, 8); - } else if (sysfs_streq(value, "panel-planar")) { - ret = m3_dcp_native_panel(active->native, 9); - } else if (sysfs_streq(value, "kms")) { - if (kms_registered) - return -EALREADY; - ret = m3_dcp_kms_register(active->native, true); - if (!ret) - kms_registered = true; - } else { - return -EINVAL; - } - active->result = ret; - return ret; -} -static const struct kernel_param_ops control_ops = { .set = control_set }; -module_param_cb(control, &control_ops, NULL, 0200); -MODULE_PARM_DESC(control, "Explicit guarded diagnostics after native startup: background or kms"); - -static void save_properties(struct m3_dcp_rtkit *dcp) -{ - const char *names[] = { "PreferredTimingElements", "DisplayAttributes" }; - struct dentry *dir = debugfs_create_dir("m3-dcp-properties", NULL); - - for (unsigned int i = 0; i < ARRAY_SIZE(names); i++) { - struct debugfs_blob_wrapper *blob = devm_kzalloc(dcp->dev, sizeof(*blob), GFP_KERNEL); - u32 size; - void *data; - - if (!blob) - continue; - data = m3_dcp_native_property(dcp->native, names[i], &size); - if (!data) - continue; - /* Immutable startup snapshots; this diagnostic remains pinned. */ - blob->data = data; - blob->size = size; - debugfs_create_blob(names[i], 0400, dir, blob); - } -} - -static bool start_link; -module_param(start_link, bool, 0400); -MODULE_PARM_DESC(start_link, "Also negotiate DCPLink, without issuing display RPCs"); -static bool query_display; -module_param(query_display, bool, 0400); -MODULE_PARM_DESC(query_display, "Issue the M3 A410 read-only main-display query after READY"); -static bool power_vote = true; -module_param(power_vote, bool, 0400); -MODULE_PARM_DESC(power_vote, "Require PMP DISP vote; false permits only the watchdog-bounded startup/query diagnostic before PMP"); -static void crashed(void *cookie, const void *log, size_t size) -{ - struct m3_dcp_rtkit *dcp = cookie; - - dcp->result = -EIO; - dev_err(dcp->dev, "DCP firmware crashed; retain all DMA allocations until reboot\n"); - if (dcp->bridge) - m3_dcp_bridge_crashed(dcp->bridge); - complete(&dcp->ready); -} - -static void receive(void *cookie, u8 ep, u64 message) -{ - struct m3_dcp_rtkit *dcp = cookie; - u32 alignment; - - dev_dbg(dcp->dev, "DCP app rx ep=%#x message=%#llx\n", ep, message); - if (ep != APPLE_DCP_LINK_ENDPOINT) - return; - if (dcp->bridge) { - m3_dcp_bridge_receive(dcp->bridge, message); - return; - } - if (dcp->awaiting_rpc) { - u32 output; - - dma_rmb(); - output = le32_to_cpup((__le32 *)(dcp->rpc + 12)); - if (message != APPLE_DCP_LINK_MSG_RPC_REPLY || output > 1) { - dcp->result = -EPROTO; - } else { - dcp->result = 0; - dev_info(dcp->dev, "M3 A410 reply: is_main_display=%u\n", output); - } - dcp->awaiting_rpc = false; - complete(&dcp->ready); - return; - } - if ((message & 3) == APPLE_DCP_LINK_MSG_FIRMWARE_INIT) - return; - /* J514S TEXT 0x1178b8: READY encodes alignment at bit 16, - * with no firmware hash. 0x1161c0 supplies the firmware side bit. - */ - alignment = message >> 16; - if (message != ((64ULL << 16) | 0x101)) { - dcp->result = -EPROTO; - } else { - dcp->result = 0; - dev_info(dcp->dev, "M3 DCPLink READY alignment=%u\n", alignment); - } - complete(&dcp->ready); -} - -static int shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) -{ - struct m3_dcp_rtkit *dcp = cookie; - struct iommu_domain *domain = iommu_get_domain_for_dev(dcp->dev); - phys_addr_t phys; - - if (!bfr->size || bfr->size > SZ_16M || !domain) - return -EINVAL; - if (!bfr->iova) { - bfr->buffer = dma_alloc_coherent(dcp->dev, bfr->size, &bfr->iova, GFP_KERNEL); - if (!bfr->buffer) - return -ENOMEM; - memset(bfr->buffer, 0, bfr->size); - dma_wmb(); - dev_info(dcp->dev, "DCP shmem allocated size=%#zx dva=%pad\n", - bfr->size, &bfr->iova); - return 0; - } - phys = iommu_iova_to_phys(domain, bfr->iova); - /* Restrict firmware-supplied addresses to a declared reserved region. */ - for (int i = 0; ; i++) { - struct device_node *node = of_parse_phandle(dcp->dev->of_node, "memory-region", i); - struct resource res; - bool oslog, valid; - - if (!node) - return -ERANGE; - oslog = of_property_read_bool(node, "apple,dcp-os-log"); - valid = !of_address_to_resource(node, 0, &res); - of_node_put(node); - if (!valid) - continue; - /* The OS-log carveout is a host physical buffer, not a DART alias. */ - if (oslog && bfr->iova == res.start) - phys = res.start; - if (phys < res.start || phys > res.end || bfr->size - 1 > res.end - phys) - continue; - if (!oslog) { - for (size_t off = 0; off < bfr->size; off += SZ_16K) - if (iommu_iova_to_phys(domain, bfr->iova + off) != phys + off) - return -ERANGE; - if (iommu_iova_to_phys(domain, bfr->iova + bfr->size - 1) != phys + bfr->size - 1) - return -ERANGE; - } - bfr->buffer = memremap(phys, bfr->size, MEMREMAP_WB); - if (!bfr->buffer) - return -ENOMEM; - bfr->is_mapped = true; - dev_info(dcp->dev, "DCP shmem inherited size=%#zx dva=%pad phys=%pap\n", - bfr->size, &bfr->iova, &phys); - return 0; - } -} - -static void shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) -{ - struct m3_dcp_rtkit *dcp = cookie; - - if (bfr->is_mapped) - memunmap(bfr->buffer); - else - dma_free_coherent(dcp->dev, bfr->size, bfr->buffer, bfr->iova); -} - -static const struct apple_rtkit_ops ops = { - .crashed = crashed, - .recv_message = receive, - .shmem_setup = shmem_setup, - .shmem_destroy = shmem_destroy, -}; - -/* PMP v2's legacy device-power mailbox only acknowledges the message on - * this firmware. The lifecycle must seed the PTD request before PMP starts. - * Verify the request and firmware acknowledgement without changing votes. - */ -static int keep_display_power(struct device *dev) -{ - struct device_node *node = of_find_node_by_path("/soc/pmp@350500000"); - const u8 *table; - void __iomem *ptd; - u64 status, request, ack; - int length, i, found = 0; - - if (!node) - return -ENODEV; - table = of_get_property(node, "apple,tunable-soc-device", &length); - if (!table || length != 21 * 0x7c || - get_unaligned_le32(table + 7 * 0x7c) != 8 || - !(get_unaligned_le32(table + 7 * 0x7c + 8) & BIT(1)) || - memcmp(table + 7 * 0x7c + 0x74, "DISP\0\0\0\0", 8)) - goto invalid; - table = of_get_property(node, "apple,tunable-ptd-range", &length); - if (!table || length % 32) - goto invalid; - for (i = 0; i < length; i += 32) { - u32 id = get_unaligned_le32(table + i); - - if (id != 10 && id != 11) - continue; - if (get_unaligned_le32(table + i + 4) != (id == 10 ? 280 : 284) || - get_unaligned_le32(table + i + 8) != 4) - goto invalid; - found |= id == 10 ? 1 : 2; - } - if (found != 3) - goto invalid; - of_node_put(node); - ptd = ioremap_np(0x3503c0000ULL, 0x4000); - if (!ptd) - return -ENOMEM; - status = readq(ptd + 16); - request = readq(ptd + 280 * 16); - ack = readq(ptd + 284 * 16); - iounmap(ptd); - dev_info(dev, "PMP PTD display status=%#llx request=%#llx ack=%#llx\n", - status, request, ack); - return status == 1 && (request & ack & BIT_ULL(7)) ? 0 : -EAGAIN; -invalid: - of_node_put(node); - return -EINVAL; -} - -static int probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct m3_dcp_rtkit *dcp; - void __iomem *cpu; - const char *uuid; - u64 message; - int ret; - - if (kernel_client && (!power_vote || query_display)) - return -EINVAL; - if (defer_client_open && !kernel_client) - return -EINVAL; - if (!of_machine_is_compatible("apple,j514s") || - of_property_read_string(dev->of_node, "apple,firmware-uuid", &uuid) || - strcmp(uuid, "DDF38191-93B3-324A-BC8F-643006F5AC82")) - return -ENODEV; - if (!iommu_get_domain_for_dev(dev)) - return -ENODEV; - ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); - if (ret) - return ret; - cpu = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(cpu)) - return PTR_ERR(cpu); - dev_info(dev, "M3 DCP inherited CPU control=%#x status=%#x\n", readl(cpu + 0x44), readl(cpu + 0x48)); - if (!(readl(cpu + 0x44) & BIT(4))) - return dev_err_probe(dev, -EBUSY, "DCP CPU stopped during handoff; do not resume interrupted firmware\n"); - if (power_vote) { - ret = keep_display_power(dev); - if (ret) - return dev_err_probe(dev, ret, "DISP power vote failed\n"); - } - if (!(readl(cpu + 0x44) & BIT(4))) - return dev_err_probe(dev, -EIO, "DCP CPU stopped after DISP power vote\n"); - dcp = devm_kzalloc(dev, sizeof(*dcp), GFP_KERNEL); - if (!dcp) - return -ENOMEM; - dcp->dev = dev; - init_completion(&dcp->ready); - platform_set_drvdata(pdev, dcp); - dcp->rtkit = apple_rtkit_init(dev, dcp, "mbox", 0, &ops); - if (IS_ERR(dcp->rtkit)) - return dev_err_probe(dev, PTR_ERR(dcp->rtkit), "RTKit init failed\n"); - /* This bring-up client has no qualified hot teardown. Retain its DMA and - * RTKit objects even on startup failure; recovery is a guarded reboot. - */ - __module_get(THIS_MODULE); - ret = apple_rtkit_wake(dcp->rtkit); - for (int n = 0; ret == -ETIME && n < 4; n++) - ret = apple_rtkit_boot(dcp->rtkit); - if (ret) - goto retained; - dev_info(dev, "M3 native RTKit boot complete; running=%d\n", apple_rtkit_is_running(dcp->rtkit)); - if (!start_link && !query_display && !kernel_client) - goto retained; - dcp->rpc = dma_alloc_coherent(dev, APPLE_DCP_LINK_RPC_MEMORY_SIZE, - &dcp->rpc_dva, GFP_KERNEL); - if (!dcp->rpc) { - ret = -ENOMEM; - goto retained; - } - memset(dcp->rpc, 0, APPLE_DCP_LINK_RPC_MEMORY_SIZE); - apple_dcp_link_init_desc_encode(dcp->rpc, 0, false); - /* The imported descriptor names this field version; M3 uses alignment. */ - ((struct apple_dcp_link_init_desc *)dcp->rpc)->version = cpu_to_le32(64); - ret = apple_dcp_link_init_message(dcp->rpc_dva, &message); - if (ret) - goto retained; - dcp->result = -ETIMEDOUT; - ret = apple_rtkit_start_ep(dcp->rtkit, APPLE_DCP_LINK_ENDPOINT); - if (ret) - goto retained; - dma_wmb(); - ret = apple_rtkit_send_message(dcp->rtkit, APPLE_DCP_LINK_ENDPOINT, message, NULL, false); - if (ret) - goto retained; - if (!wait_for_completion_timeout(&dcp->ready, msecs_to_jiffies(3000))) - ret = -ETIMEDOUT; - else - ret = dcp->result; - if (!ret && kernel_client) { - dcp->bridge = m3_dcp_bridge_create(dev, dcp->rtkit, dcp->rpc, true); - if (IS_ERR(dcp->bridge)) { - ret = PTR_ERR(dcp->bridge); - dcp->bridge = NULL; - goto retained; - } - m3_dcp_bridge_activate(dcp->bridge); - dcp->native = m3_dcp_native_start(dev, dcp->bridge, defer_client_open); - ret = PTR_ERR_OR_ZERO(dcp->native); - if (!ret) { - client_opened = !defer_client_open; - if (client_opened) - save_properties(dcp); - active = dcp; - } - } - if (!ret && query_display) { - /* __TEXT 0x13f84c: no input; bool result serialized in four bytes. - * One request on AP stream 0 at offset 0, already 64-byte aligned. - */ - reinit_completion(&dcp->ready); - apple_dcp_link_rpc_header_encode(dcp->rpc, 0x41343130, 0, 4); - *(__le32 *)(dcp->rpc + 12) = cpu_to_le32(0xa5); - dcp->result = -ETIMEDOUT; - dcp->awaiting_rpc = true; - dma_wmb(); - ret = apple_rtkit_send_message(dcp->rtkit, APPLE_DCP_LINK_ENDPOINT, - (16ULL << 32) | APPLE_DCP_LINK_MSG_RPC, NULL, false); - if (!ret) { - if (!wait_for_completion_timeout(&dcp->ready, msecs_to_jiffies(3000))) - ret = -ETIMEDOUT; - else - ret = dcp->result; - } - } -retained: - dcp->result = ret; - dev_info(dev, "M3 native DCP diagnostic finished ret=%d; resources retained until reboot\n", ret); - return 0; -} - -static const struct of_device_id matches[] = { - { .compatible = "apple,t6030-dcp" }, - { } -}; -/* Deliberately no module alias: this diagnostic is loaded explicitly. */ -static struct platform_driver m3_dcp_driver = { - .probe = probe, - .driver = { - .name = "m3-dcp-rtkit", - .of_match_table = matches, - .suppress_bind_attrs = true, - }, -}; -module_platform_driver(m3_dcp_driver); -MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("J514S native RTKit/DMA DCPLink startup diagnostic"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_state.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_state.h deleted file mode 100644 index c24c8ca074f57..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_state.h +++ /dev/null @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -#include -#include -/* Private layout shared by the explicit startup and one-shot query probes. */ -struct m3_dcp_rtkit { - struct device *dev; - struct apple_rtkit *rtkit; - struct completion ready; - void *rpc; - dma_addr_t rpc_dva; - int result; - bool awaiting_rpc; - struct m3_dcp_bridge *bridge; - struct m3_dcp_native *native; -}; diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_vote.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_vote.c deleted file mode 100644 index 4fe311d3f2b0c..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcp_vote.c +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Isolate the existing J514S DISP request from native DCP attachment. - * Initialization-only diagnostic: use the bounded lifecycle watchdog. - * Keep the vote until reboot; no unload or display submissions. - */ -#include -#include -#include -#include -#include - -static int __init vote(void) -{ - struct device_node *node; - const u8 *table; - int length, ret = -EINVAL; - - if (!of_machine_is_compatible("apple,j514s") || - !of_machine_is_compatible("apple,t6030")) - return -ENODEV; - node = of_find_node_by_path("/soc/pmp@350500000"); - if (!node) - return -ENODEV; - table = of_get_property(node, "apple,tunable-soc-device", &length); - if (table && length == 21 * 0x7c && - get_unaligned_le32(table + 7 * 0x7c) == 8 && - !memcmp(table + 7 * 0x7c + 0x74, "DISP\0\0\0\0", 8)) - ret = apple_pmp_set_device_power(0x0f, 8, 1); - of_node_put(node); - pr_info("M3 DCP isolated PMP DISP request returned %d\n", ret); - if (!ret) - msleep(20); - return ret; -} -module_init(vote); -MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("J514S isolated PMP display power request; reboot to remove"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio.h deleted file mode 100644 index 2387fb93ed03f..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio.h +++ /dev/null @@ -1,193 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* J514S old-EPIC audio transport. Service framing follows Asahi av.c and - * m1n1 EPICStandardService; serialized lifetime follows the M4 audio port. - * No PCM/DMA playback is initiated by this transport. */ -#define M3_AUDIO_MAX (256 * 1024) -static struct afkc audio_afk; -static u32 audio_channel, audio_command, audio_group, audio_bytes, audio_received; -static u16 audio_tag; -static u32 audio_generation; -static bool audio_owned; -static bool audio_announced, audio_pending; -static int audio_error, audio_result = -ENODATA; -static u8 *audio_tx, *audio_rx; -static u64 audio_tx_dva, audio_rx_dva; -module_param(audio_result, int, 0400); - -static int audio_record(void *cookie, const u8 *p, u32 size) -{ - const u8 *h = audio_afk.capture + audio_afk.capture_size - size - 12; - u32 channel = get_unaligned_le32(h), kind = get_unaligned_le32(h + 4); - if (size < 40 || p[0] != 2 || p[20] != 4) return -EPROTO; - if (!p[21] && get_unaligned_le16(p + 22) == 0x30) { - pr_info("m3_audio: announcement channel=%u name=%.*s bytes=%u\n", channel, min(size-40,32U), p+40, size); - if (size < 76 || memcmp(p+40,"dispext0:dcpav-audio-interface",sizeof("dispext0:dcpav-audio-interface")-1) || get_unaligned_le32(p+72)!=0xd3) - return 0; - if (audio_announced) return -EALREADY; - audio_channel=channel; audio_generation++; audio_announced=true; return 0; - } - if (channel != audio_channel) return 0; - if (!p[21]) { - if (get_unaligned_le16(p+22)==0x32) { - audio_generation++;audio_announced=false; audio_pending=false; audio_error=-ENODEV; - } - return 0; - } - if (!audio_pending || (kind!=0 && kind!=4) || size<70 || p[21]!=0x20 || - get_unaligned_le16(p+22)!=0xc0 || get_unaligned_le16(p+32)!=audio_tag) - return -EPROTO; - if (get_unaligned_le32(p+40)) return -EREMOTEIO; - if (get_unaligned_le64(p+44)!=audio_rx_dva || get_unaligned_le32(p+60)>audio_bytes) - return -EPROTO; - dma_rmb(); - audio_received=get_unaligned_le32(p+60); - audio_pending=false; - return 0; -} -static const struct afkc_ops audio_ops={.send=session_send,.allocate=afk_allocate,.record=audio_record}; -static int audio_receive(u64 message) -{ - int ret=afkc_receive(&audio_afk,message); - if(ret) {audio_error=ret;audio_pending=false;} - return 0; /* Audio faults must not tear down video or free in-flight DMA. */ -} -static int audio_wait(bool announcement) -{ - u64 until=session_now(NULL)+2000; - int ret; - while(announcement ? !audio_announced : audio_pending) { - if(audio_error)return audio_error; - ret=native_poll(NULL,msecs_to_jiffies(5)); - if(ret && ret!=-ETIMEDOUT)return ret; - if(session_now(NULL)>=until)return -ETIMEDOUT; - } - return audio_error; -} -static int audio_call_locked(u16 group,u32 command,const void *body,u32 bytes) -{ - u8 wire[70]={};int ret; - lockdep_assert_held(&native_route_lock); - if(port || usb_c || !native_kms_ready || native_kms_finished || !native_kms_connected || - desktop_disconnected || READ_ONCE(desktop_stop) || !session.runtime)return -ENODEV; - if(audio_error)return audio_error; - if(bytes>M3_AUDIO_MAX+48)return -E2BIG; - if(!audio_afk.started) { - if(!(session.endpoints[1]&BIT(9)))return -ENODEV; - ret=afk_allocate(NULL,M3_AUDIO_MAX+112,&audio_tx,&audio_tx_dva);if(ret)goto fail; - ret=afk_allocate(NULL,M3_AUDIO_MAX+112,&audio_rx,&audio_rx_dva);if(ret)goto fail; - audio_afk.streaming=true; - ret=afkc_start(&audio_afk,&audio_ops,NULL,0x29);if(ret)goto fail; - } - ret=audio_wait(true);if(ret)goto fail; - if(audio_pending){ret=-EBUSY;goto fail;} - audio_bytes=bytes+64;audio_group=group;audio_command=command;audio_tag++; - memset(audio_tx,0,audio_bytes);memset(audio_rx,0,audio_bytes); - put_unaligned_le16(group,audio_tx+2);put_unaligned_le32(command,audio_tx+4); - put_unaligned_le32(bytes,audio_tx+8);put_unaligned_le32(0x69706378,audio_tx+12); - memcpy(audio_tx+64,body,bytes); - wire[0]=2;put_unaligned_le16(epic_seq++,wire+1);put_unaligned_le32(30,wire+16); - wire[20]=4;wire[21]=0x30;put_unaligned_le16(0xc0,wire+22);put_unaligned_le16(audio_tag,wire+32); - put_unaligned_le64(audio_rx_dva,wire+44);put_unaligned_le64(audio_tx_dva,wire+52); - put_unaligned_le32(audio_bytes,wire+60);put_unaligned_le32(audio_bytes,wire+64); - audio_pending=true;audio_received=0;dma_wmb(); - ret=old_epic_write_channel(&audio_afk,audio_channel,3,wire,sizeof(wire));if(ret)goto fail; - ret=audio_wait(false);if(ret)goto fail; - if(audio_received<64 || get_unaligned_le16(audio_rx+2)!=group || - get_unaligned_le32(audio_rx+4)!=command || get_unaligned_le32(audio_rx+12)!=0x69706378 || - get_unaligned_le32(audio_rx+8)>audio_received-64) {ret=-EPROTO;goto fail;} - audio_result=0; - pr_info("m3_audio: group=%u command=%u response=%u body=%u\n",group,command,audio_received,get_unaligned_le32(audio_rx+8)); - return 0; -fail: - audio_result=audio_error=ret;audio_pending=false; - pr_err("m3_audio: error=%d; all firmware DMA retained until reboot\n",ret); - return ret; -} -static ssize_t audio_request_write(struct file *file,const char __user *buf,size_t count,loff_t *offset) -{ - u8 *request;u32 group,command,bytes;int ret; - if(count<8 || count>M3_AUDIO_MAX+56)return -EINVAL; - request=memdup_user(buf,count);if(IS_ERR(request))return PTR_ERR(request); - group=get_unaligned_le32(request);command=get_unaligned_le32(request+4);bytes=count-8; - /* Initial metadata qualification only: no stream/prepare requests. */ - if(!((group==0 && ((command==4 && bytes==32)||(command==5 && bytes==16))) || - (group==1 && (command==16 || command==18) && bytes>=48))) {kfree(request);return -EINVAL;} - mutex_lock(&native_route_lock); - ret=audio_owned ? -EBUSY : audio_call_locked(group,command,request+8,bytes); - mutex_unlock(&native_route_lock);kfree(request); - return ret?ret:count; -} -static ssize_t audio_response_read(struct file *file,char __user *buf,size_t count,loff_t *offset) -{ - ssize_t ret; - mutex_lock(&native_route_lock); - ret=simple_read_from_buffer(buf,count,offset,audio_rx,audio_received); - mutex_unlock(&native_route_lock);return ret; -} -static ssize_t audio_capture_read(struct file *file,char __user *buf,size_t count,loff_t *offset) -{ - ssize_t ret; - mutex_lock(&native_route_lock); - ret=simple_read_from_buffer(buf,count,offset,audio_afk.capture,audio_afk.capture_size); - mutex_unlock(&native_route_lock);return ret; -} -static const struct file_operations audio_request_fops={.owner=THIS_MODULE,.write=audio_request_write}; -static const struct file_operations audio_response_fops={.owner=THIS_MODULE,.read=audio_response_read,.llseek=default_llseek}; -static const struct file_operations audio_capture_fops={.owner=THIS_MODULE,.read=audio_capture_read,.llseek=default_llseek}; -static void audio_debugfs(struct dentry *dir) -{ - if(port)return; - debugfs_create_file("audio-request",0200,dir,NULL,&audio_request_fops); - debugfs_create_file("audio-response",0400,dir,NULL,&audio_response_fops); - debugfs_create_file("audio-capture",0400,dir,NULL,&audio_capture_fops); -} - -#if DCPEXT_INSTANCE == 0 -#include "m3_dcpext_audio_api.h" -static u64 audio_current_generation(void) -{return ((u64)audio_generation<<32)|(u32)native_kms_generation;} -static bool audio_live(void) -{return !port && !usb_c && native_kms_ready && !native_kms_finished && native_kms_connected && - !desktop_disconnected && !READ_ONCE(desktop_stop) && session.runtime;} -static int audio_identify(char *name,size_t size,u64 *generation) -{ - int ret=0; - mutex_lock(&native_route_lock); - if(!audio_live()){ret=-ENODEV;goto out;} - strscpy(name,"M3 HDMI",size); - if(av_edid_bytes>=128)for(u32 off=54;off<=108;off+=18) { - const u8 *d=av_edid+off; - if(d[0] || d[1] || d[2] || d[3]!=0xfc)continue; - char display[14];memcpy(display,d+5,13);display[13]=0; - for(u32 i=0;i<13;i++)if(display[i]=='\n'){display[i]=0;break;} - strscpy(name,display,size);break; - } - *generation=audio_current_generation(); -out:mutex_unlock(&native_route_lock);return ret; -} -static int audio_pcm_call(u16 group,u32 command,void *data,u32 bytes,u64 *generation) -{ - int ret; - if(!data || !generation)return -EINVAL; - mutex_lock(&native_route_lock); - if(!audio_live() || (*generation && *generation!=audio_current_generation())){ret=-ENODEV;goto out;} - /* Lazy endpoint publication may establish the first audio generation. */ - bool first=!audio_afk.started; - ret=audio_call_locked(group,command,data,bytes); - if(ret)goto out; - if(!first && *generation && *generation!=audio_current_generation()){ret=-ENODEV;goto out;} - *generation=audio_current_generation(); - u32 received=get_unaligned_le32(audio_rx+8); - if(received>bytes){ret=-EPROTO;goto out;} - memset(data,0,bytes);memcpy(data,audio_rx+64,received); -out:mutex_unlock(&native_route_lock);return ret; -} -static const struct m3_dcpext_audio_ops audio_pcm_ops={.call=audio_pcm_call,.identify=audio_identify}; -const struct m3_dcpext_audio_ops *m3_dcpext_audio_get(unsigned int controller) -{ - if(controller)return NULL; - mutex_lock(&native_route_lock);audio_owned=true;mutex_unlock(&native_route_lock); - return &audio_pcm_ops; -} -EXPORT_SYMBOL_GPL(m3_dcpext_audio_get); -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_api.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_api.h deleted file mode 100644 index 10082d1845ac6..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_api.h +++ /dev/null @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -#ifndef M3_DCPEXT_AUDIO_H -#define M3_DCPEXT_AUDIO_H -#include -#define M3_AUDIO_ELEMENTS_MAX (256 * 1024) - -/* Calls serialize with scanout/hotplug. Generation binds a PCM stream to one - * connected sink; an unplug/replug must never reuse its opaque audio cookie. - */ -struct m3_dcpext_audio_ops { - int (*call)(u16 group, u32 command, void *data, u32 bytes, u64 *generation); - int (*identify)(char *name, size_t size, u64 *generation); -}; -const struct m3_dcpext_audio_ops *m3_dcpext_audio_get(unsigned int port); -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_format.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_format.h deleted file mode 100644 index f54440fc15a93..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_format.h +++ /dev/null @@ -1,190 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* Bounded OSSerialize format selection. Encoding: Hector Martin / Asahi; - * format fields: Martin Povišer / Asahi parser.c. M3 14.6 cookies are 32 bytes, verified in live metadata-2. - * Initial PCM qualification is stereo S16_LE at 48 kHz only. - */ -#ifndef M3_DCPEXT_AUDIO_FORMAT_H -#define M3_DCPEXT_AUDIO_FORMAT_H -struct audio_object { - const u8 *raw; - u32 size, pos, budget; -}; -static int audio_object_tag(struct audio_object *o, u32 *type, u32 *count) -{ - u32 v; - o->pos = (o->pos + 3) & ~3U; - if (!o->budget-- || o->pos > o->size || o->size - o->pos < 4) - return -EPROTO; - v = get_unaligned_le32(o->raw + o->pos); - o->pos += 4; - if (v & 0x60000000) - return -EPROTO; - *type = (v >> 24) & 31; - *count = v & 0xffffff; - return 0; -} -static int audio_object_bytes(struct audio_object *o, u32 n, const u8 **p) -{ - if (o->pos > o->size || n > o->size - o->pos) - return -EPROTO; - *p = o->raw + o->pos; - o->pos += n; - return 0; -} -static int audio_object_skip(struct audio_object *o, unsigned int depth) -{ - u32 t, n; - const u8 *p; - int ret; - if (depth > 12 || audio_object_tag(o, &t, &n)) - return -EPROTO; - switch (t) { - case 1: - case 2: - if (n > 256) - return -E2BIG; - for (u32 i = 0; i < n * (t == 1 ? 2 : 1); i++) { - ret = audio_object_skip(o, depth + 1); - if (ret) - return ret; - } - return 0; - case 4: return n == 64 ? audio_object_bytes(o, 8, &p) : -EPROTO; - case 9: - case 10: return audio_object_bytes(o, n, &p); - case 11: return n <= 1 ? 0 : -EPROTO; - default: return -EPROTO; - } -} -static int audio_object_string(struct audio_object *o, const u8 **p, u32 *n) -{ - u32 t; - if (audio_object_tag(o, &t, n) || t != 9) - return -EPROTO; - return audio_object_bytes(o, *n, p); -} -static bool audio_key(const u8 *p, u32 n, const char *key) -{ - return n == strlen(key) && !memcmp(p, key, n); -} -static int audio_object_number(struct audio_object *o, u64 *value) -{ - u32 t, n; - const u8 *p; - if (audio_object_tag(o, &t, &n) || t != 4 || n != 64 || - audio_object_bytes(o, 8, &p)) - return -EPROTO; - *value = get_unaligned_le64(p); - return 0; -} -static int audio_stereo_layout(struct audio_object *o) -{ - u32 t, count; - bool found = false; - if (audio_object_tag(o, &t, &count) || t != 2 || count > 256) - return -EPROTO; - for (u32 i = 0; i < count; i++) { - u32 pairs; - u64 channels = 0, active = 0; - bool stereo = false; - if (audio_object_tag(o, &t, &pairs) || t != 1 || pairs > 64) - return -EPROTO; - for (u32 j = 0; j < pairs; j++) { - const u8 *key; - u32 len; - int ret; - if (audio_object_string(o, &key, &len)) - return -EPROTO; - if (audio_key(key, len, "ChannelCount")) - ret = audio_object_number(o, &channels); - else if (audio_key(key, len, "ActiveChannelCount")) - ret = audio_object_number(o, &active); - else if (audio_key(key, len, "ChannelLayout")) { - u32 n; - const u8 *left, *right; - u32 ln, rn; - struct audio_object saved = *o; - ret = audio_object_tag(o, &t, &n); - if (!ret && t == 2 && n == 2 && - !audio_object_string(o, &left, &ln) && - !audio_object_string(o, &right, &rn)) - stereo = audio_key(left, ln, "Front Left") && audio_key(right, rn, "Front Right"); - *o = saved; - ret = audio_object_skip(o, 0); - } else - ret = audio_object_skip(o, 0); - if (ret) - return ret; - } - found |= channels == 2 && active == 2 && stereo; - } - return found ? 1 : 0; -} -static int m3_audio_select_stereo(const u8 *raw, u32 bytes, u8 cookie[32]) -{ - struct audio_object o = { .raw = raw, .size = bytes, .pos = 4, .budget = 32768 }; - u32 t, count; - unsigned int matches = 0; - if (bytes < 8 || bytes > 256 * 1024 || get_unaligned_le32(raw) != 0xd3 || - audio_object_tag(&o, &t, &count) || t != 2 || count > 256) - return -EPROTO; - for (u32 i = 0; i < count; i++) { - const u8 *blob = NULL; - u32 pairs, blob_size = 0, seen = 0; - u64 fields[6] = {0}; - bool is_virtual = true, stereo = false; - static const char * const keys[] = { "ElementType", "Format", "ChannelCount", "SampleSize", "LinkSampleRate", "StreamSampleRate" }; - if (audio_object_tag(&o, &t, &pairs) || t != 1 || pairs > 64) - return -EPROTO; - for (u32 j = 0; j < pairs; j++) { - const u8 *key; - u32 len, k; - int ret; - if (audio_object_string(&o, &key, &len)) - return -EPROTO; - for (k = 0; k < 6; k++) - if (audio_key(key, len, keys[k])) - break; - if (k < 6) { - if (seen & (1U << k)) - return -EPROTO; - seen |= 1U << k; - ret = audio_object_number(&o, &fields[k]); - } else if (audio_key(key, len, "IsVirtual")) { - u32 n; - if (seen & 64) - return -EPROTO; - seen |= 64; - ret = audio_object_tag(&o, &t, &n); - if (ret || t != 11 || n > 1) - return -EPROTO; - is_virtual = n; - } else if (audio_key(key, len, "ElementData")) { - if (blob || audio_object_tag(&o, &t, &blob_size) || t != 10) - return -EPROTO; - ret = audio_object_bytes(&o, blob_size, &blob); - } else if (audio_key(key, len, "AudioChannelLayoutElements")) { - ret = audio_stereo_layout(&o); - stereo = ret == 1; - if (ret > 0) - ret = 0; - } else - ret = audio_object_skip(&o, 0); - if (ret) - return ret; - } - if (seen != 127 || is_virtual || !stereo || fields[0] != 2 || - fields[1] != 1 || fields[2] != 2 || fields[3] != 16 || - fields[4] != 48000 || fields[5] != 48000) - continue; - if (!blob || blob_size != 32 || get_unaligned_le32(blob) != 1 || - get_unaligned_le32(blob + 4) != 2 || get_unaligned_le32(blob + 8) != 16 || - get_unaligned_le32(blob + 12) != 48000 || - get_unaligned_le32(blob + 16) || get_unaligned_le32(blob + 20)) - return -EPROTO; - memcpy(cookie, blob, 32); - matches++; - } - return matches == 1 ? 0 : -ENODATA; -} -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_pcm.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_pcm.c deleted file mode 100644 index 2ffe80576e45c..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_audio_pcm.c +++ /dev/null @@ -1,343 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* M3 port of M4 display PCM integration. Audio command/DMA ordering follows Martin - * Povišer's Asahi audio.c; the transport/cookie and connected-sink lifetime - * are supplied by the M3 old-EPIC transport. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "m3_dcpext_audio_api.h" -#include "m3_dcpext_audio_format.h" - -static u8 metadata_capture[M3_AUDIO_ELEMENTS_MAX + 48]; -static struct debugfs_blob_wrapper metadata_blob = { - .data = metadata_capture, .size = sizeof(metadata_capture), -}; -static struct dentry *metadata_debug; -static bool metadata_only = true; -module_param(metadata_only, bool, 0400); -MODULE_PARM_DESC(metadata_only, "Discover fresh formats without registering PCM for initial qualification"); -static unsigned int controller; -module_param(controller, uint, 0400); -MODULE_PARM_DESC(controller, "DCPEXT instance whose DPA DMA route is described by this device (initially 0)"); - -struct m3_audio_pcm { - struct device *dev; - const struct m3_dcpext_audio_ops *ops; - struct dma_chan *chan; - struct snd_card *card; - u64 generation; - u8 cookie[32]; - char name[32]; - bool opened, prepared, started, pinned; -}; - -static int m3_audio_simple(struct m3_audio_pcm *a, u32 cmd, bool cookie) -{ - u8 data[64] = {}; - u32 size = cmd == 4 ? 32 : cmd == 5 ? 16 : 64; - int ret; - if (cookie || (cmd >= 6 && cmd <= 11)) - memcpy(data, a->cookie, sizeof(a->cookie)); - ret = a->ops->call(0, cmd, data, size, &a->generation); - /* M4 link methods return IOReturn at byte 48 of the 64-byte body. */ - if (!ret && size == 64 && get_unaligned_le32(data + 48)) { - dev_err(a->dev, "audio command %u service status %#x\n", cmd, - get_unaligned_le32(data + 48)); - ret = -EREMOTEIO; - } - return ret; -} - -static int m3_audio_elements(struct m3_audio_pcm *a) -{ - u8 *data; - u64 used; - int ret; - data = kzalloc(M3_AUDIO_ELEMENTS_MAX + 48, GFP_KERNEL); - if (!data) - return -ENOMEM; - put_unaligned_le64(M3_AUDIO_ELEMENTS_MAX, data); - ret = a->ops->call(1, 16, data, M3_AUDIO_ELEMENTS_MAX + 48, &a->generation); - if (ret) - goto out; - memcpy(metadata_capture, data, sizeof(metadata_capture)); - used = get_unaligned_le64(data + 32); - dev_info(a->dev, "audio metadata capacity=%llu status=%#x used=%llu\n", - get_unaligned_le64(data), get_unaligned_le32(data + 16), used); - if (get_unaligned_le64(data) != M3_AUDIO_ELEMENTS_MAX || get_unaligned_le32(data + 16) || - used < 8 || used > M3_AUDIO_ELEMENTS_MAX) { - ret = -EPROTO; - goto out; - } - ret = m3_audio_select_stereo(data + 48, used, a->cookie); - if (!ret) - dev_info(a->dev, "%s: fresh stereo 48kHz S16 cookie, metadata %llu bytes\n", a->name, used); -out: - kfree(data); - return ret; -} - -static int m3_audio_open_service(struct m3_audio_pcm *a) -{ - int ret; - a->generation = 0; - ret = a->ops->identify(a->name, sizeof(a->name), &a->generation); - if (ret) - return ret; - ret = m3_audio_simple(a, 4, false); - if (ret) - return ret; - a->opened = true; - ret = m3_audio_elements(a); - if (ret) { - m3_audio_simple(a, 5, false); - a->opened = false; - } - return ret; -} - -static const struct snd_pcm_hardware m3_audio_hw = { - .info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | - SNDRV_PCM_INFO_INTERLEAVED | SNDRV_PCM_INFO_BATCH, - .formats = SNDRV_PCM_FMTBIT_S16_LE, - .rates = SNDRV_PCM_RATE_48000, - .rate_min = 48000, .rate_max = 48000, - .channels_min = 2, .channels_max = 2, - .buffer_bytes_max = 256 * 1024, - .period_bytes_min = 256, .period_bytes_max = 64 * 1024, - .periods_min = 4, .periods_max = 32, - .fifo_size = 16, -}; - -static int m3_pcm_open(struct snd_pcm_substream *s) -{ - struct m3_audio_pcm *a = s->pcm->private_data; - int ret; - struct dma_tx_state state; - if (dmaengine_tx_status(a->chan, 0, &state) == DMA_ERROR) - return -EIO; - if (!s->dma_buffer.area || s->dma_buffer.bytes != 256 * 1024) - return -ENOMEM; - ret = m3_audio_open_service(a); - if (ret) - return ret; - s->runtime->hw = m3_audio_hw; - /* The DMA helper applies the integer-period constraint itself. The - * constraint API may return positive success, not just zero. - */ - ret = snd_dmaengine_pcm_open(s, a->chan); - if (ret) { - m3_audio_simple(a, 5, false); - a->opened = false; - } - return ret; -} -static int m3_pcm_hw_params(struct snd_pcm_substream *s, struct snd_pcm_hw_params *p) -{ - struct m3_audio_pcm *a = s->pcm->private_data; - struct dma_slave_config cfg = {}; - int ret = snd_hwparams_to_dma_slave_config(s, p, &cfg); - if (ret) - return ret; - cfg.direction = DMA_MEM_TO_DEV; - /* DPA data entry is always 32 bits, including stereo 16-bit PCM. */ - cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; - return dmaengine_slave_config(a->chan, &cfg); -} -static int m3_pcm_prepare(struct snd_pcm_substream *s) -{ - struct m3_audio_pcm *a = s->pcm->private_data; - int ret; - - /* ALSA permits prepare again after drop/XRUN or before a first start. - * DCP retains the prepared link until unprepare and rejects duplicate - * prepare commands. The PCM format is fixed for this open lifetime. - */ - if (a->prepared) - return 0; - ret = m3_audio_simple(a, 6, true); - if (!ret) - a->prepared = true; - return ret; -} -static int m3_pcm_trigger(struct snd_pcm_substream *s, int cmd) -{ - struct m3_audio_pcm *a = s->pcm->private_data; - int ret; - if (cmd == SNDRV_PCM_TRIGGER_START) { - ret = m3_audio_simple(a, 7, true); - if (ret) - return ret; - a->started = true; - /* Bring-up uses one fixed coherent buffer for the card lifetime. - * Keep it and callback storage alive even if SIO cannot retire DMA. - * TODO: removable PCM after SIO error recovery is qualified. - */ - if (!a->pinned) { - __module_get(THIS_MODULE); - a->pinned = true; - } - ret = snd_dmaengine_pcm_trigger(s, cmd); - if (ret) { - m3_audio_simple(a, 10, false); - a->started = false; - } - return ret; - } - if (cmd == SNDRV_PCM_TRIGGER_STOP) { - ret = snd_dmaengine_pcm_trigger(s, cmd); - if (ret) - return ret; - /* Drain DMA before the link is stopped; sync_stop is sleepable. */ - return 0; - } - return -EINVAL; -} -static int m3_pcm_sync_stop(struct snd_pcm_substream *s) -{ - struct m3_audio_pcm *a = s->pcm->private_data; - int ret = snd_dmaengine_pcm_sync_stop(s); - if (!ret && a->started) { - ret = m3_audio_simple(a, 10, false); - a->started = false; - } - return ret; -} -static int m3_pcm_hw_free(struct snd_pcm_substream *s) -{ - struct m3_audio_pcm *a = s->pcm->private_data; - int ret = 0; - if (a->prepared) { - ret = m3_audio_simple(a, 11, false); - a->prepared = false; - } - return ret; -} -static int m3_pcm_close(struct snd_pcm_substream *s) -{ - struct m3_audio_pcm *a = s->pcm->private_data; - int ret = snd_dmaengine_pcm_close(s); - if (a->opened) { - int err = m3_audio_simple(a, 5, false); - a->opened = false; - if (!ret) - ret = err; - } - return ret; -} -static const struct snd_pcm_ops m3_pcm_ops = { - .open = m3_pcm_open, .close = m3_pcm_close, - .hw_params = m3_pcm_hw_params, .hw_free = m3_pcm_hw_free, - .prepare = m3_pcm_prepare, .trigger = m3_pcm_trigger, - .sync_stop = m3_pcm_sync_stop, .pointer = snd_dmaengine_pcm_pointer, -}; - -static int m3_audio_probe(struct platform_device *pdev) -{ - struct m3_audio_pcm *a; - struct snd_pcm *pcm; - int ret; - /* This first DT node describes DCPEXT0 -> DPA1 -> SIO 0x66. - * Never redirect it to a controller whose DMA mapping is unqualified. - */ - if (controller != 0) - return -EINVAL; - a = devm_kzalloc(&pdev->dev, sizeof(*a), GFP_KERNEL); - if (!a) - return -ENOMEM; - a->dev = &pdev->dev; - a->ops = m3_dcpext_audio_get(controller); - if (!a->ops) - return -EPROBE_DEFER; - a->chan = of_dma_request_slave_channel(pdev->dev.of_node, "tx"); - if (IS_ERR(a->chan)) - return dev_err_probe(a->dev, PTR_ERR(a->chan), "SIO channel unavailable\n"); - platform_set_drvdata(pdev, a); - pm_runtime_enable(a->dev); - ret = pm_runtime_resume_and_get(a->dev); - if (ret < 0) - goto release; - ret = m3_audio_open_service(a); - if (ret) - goto power; - ret = m3_audio_simple(a, 5, false); - a->opened = false; - if (ret) - goto power; - if (metadata_only) { - dev_info(a->dev, "metadata-only qualification completed; no PCM or DMA issued\n"); - return 0; - } - ret = snd_card_new(a->dev, -1, "M3HDMI0", THIS_MODULE, 0, &a->card); - if (ret) - goto power; - strscpy(a->card->driver, "M3HDMI", sizeof(a->card->driver)); - snprintf(a->card->shortname, sizeof(a->card->shortname), "HDMI %s", a->name); - snprintf(a->card->longname, sizeof(a->card->longname), "Apple M3 DCPEXT0 HDMI - %s", a->name); - ret = snd_pcm_new(a->card, a->card->shortname, 0, 1, 0, &pcm); - if (ret) - goto card; - pcm->private_data = a; - pcm->nonatomic = true; - strscpy(pcm->name, a->card->shortname, sizeof(pcm->name)); - snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_PLAYBACK, &m3_pcm_ops); - ret = snd_pcm_set_managed_buffer_all(pcm, SNDRV_DMA_TYPE_DEV, - a->chan->device->dev, 256 * 1024, 0); - if (ret) - goto card; - ret = snd_card_register(a->card); - if (!ret) - return 0; -card: - snd_card_free(a->card); -power: - pm_runtime_put_sync(a->dev); -release: - pm_runtime_disable(a->dev); - dma_release_channel(a->chan); - return ret; -} -static void m3_audio_remove(struct platform_device *pdev) -{ - struct m3_audio_pcm *a = platform_get_drvdata(pdev); - if (a->card) - snd_card_free(a->card); - dma_release_channel(a->chan); - pm_runtime_put_sync(a->dev); - pm_runtime_disable(a->dev); -} -static const struct of_device_id m3_audio_match[] = { - { .compatible = "apple,j514s-hdmi-audio" }, {} -}; -MODULE_DEVICE_TABLE(of, m3_audio_match); -static struct platform_driver m3_audio_driver = { - .probe = m3_audio_probe, .remove = m3_audio_remove, - .driver = { .name = "m3-dcpext-audio", .of_match_table = m3_audio_match, - .suppress_bind_attrs = true }, -}; -static int __init m3_audio_init(void) -{ - int ret; - metadata_debug = debugfs_create_blob("m3-hdmi-elements", 0400, NULL, &metadata_blob); - ret = platform_driver_register(&m3_audio_driver); - if (ret) - debugfs_remove(metadata_debug); - return ret; -} -static void __exit m3_audio_exit(void) -{ - platform_driver_unregister(&m3_audio_driver); - debugfs_remove(metadata_debug); -} -module_init(m3_audio_init); -module_exit(m3_audio_exit); -MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("Apple M3 external display PCM audio"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_av.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_av.h deleted file mode 100644 index f99243bfd3dbd..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_av.h +++ /dev/null @@ -1,69 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* Old EPIC AV service lifecycle: m1n1 EPICStandardService group4 open/close; - * Asahi dpavservep copyEDID group1/command7. Buffers retained until reboot. */ -static bool av_read_ready; -static bool av_announced,av_pending,av_opened,av_closed,av_edid_done; -static u32 av_channel,av_command,av_group,av_size,av_tag; -static u8 *av_rx,*av_tx;static u64 av_rx_dva,av_tx_dva;static u8 av_edid[1024];static u32 av_edid_bytes; -static int av_send(u16 group,u32 command,u32 bytes) -{ - u8 *tx,wire[70]={};u64 txd;int ret; - if(av_pending || !av_announced)return -EINVAL; - if(bytes>1144)return -E2BIG; - if(!av_rx){ret=afk_allocate(NULL,1144,&av_rx,&av_rx_dva);if(ret)return ret;} - if(!av_tx){ret=afk_allocate(NULL,1144,&av_tx,&av_tx_dva);if(ret)return ret;} - tx=av_tx;txd=av_tx_dva;memset(tx,0,1144);memset(av_rx,0,1144); - put_unaligned_le16(group,tx+2);put_unaligned_le32(command,tx+4); - put_unaligned_le32(bytes-64,tx+8);put_unaligned_le32(0x69706378,tx+12); - if(group==1)put_unaligned_le64(1032,tx+64); - wire[0]=2;put_unaligned_le16(epic_seq++,wire+1);put_unaligned_le32(30,wire+16); - wire[20]=4;wire[21]=0x30;put_unaligned_le16(0xc0,wire+22);put_unaligned_le16(av_tag,wire+32); - put_unaligned_le64(av_rx_dva,wire+44);put_unaligned_le64(txd,wire+52); - put_unaligned_le32(bytes,wire+60);put_unaligned_le32(bytes,wire+64); - av_pending=true;av_group=group;av_command=command;av_size=bytes; - pr_info("m3_dcpext_av: sending group=%u command=%u bytes=%u\n",group,command,bytes); - return old_epic_write_channel(&aux_afk,av_channel,3,wire,sizeof(wire)); -} -static int av_record(u32 channel,u32 kind,const u8 *p,u32 size) -{ - if(size>=76 && p[0]==2 && p[20]==4 && !p[21] && get_unaligned_le16(p+22)==0x30 && - !memcmp(p+40,port?"dispext1:dcpav-service-epic:0":"dispext0:dcpav-service-epic:0",sizeof("dispext0:dcpav-service-epic:0")) && get_unaligned_le32(p+72)==0xd3){ - if(av_announced)return -EALREADY; - av_announced=true;av_closed=false;av_channel=channel;pr_info("m3_dcpext_av: service announced channel=%u\n",channel);return 0; - } - if(size>=40 && av_announced && channel==av_channel && p[0]==2 && p[20]==4 && !p[21] && get_unaligned_le16(p+22)==0x32){ - pr_info("m3_dcpext_av: service teardown report; no DMA released\n"); - /* Teardown is not proof that a pending DMA has stopped. */ - if(av_pending)return -EBUSY; - av_announced=av_opened=false;av_closed=true;return 0; - } - if(!av_pending || channel!=av_channel || (kind!=0 && kind!=4))return 0; - if(size<70 || p[21]!=0x20 || get_unaligned_le16(p+22)!=0xc0 || get_unaligned_le16(p+32)!=av_tag || - get_unaligned_le32(p+40) || get_unaligned_le64(p+44)!=av_rx_dva || - get_unaligned_le32(p+60)>av_size)return -EPROTO; - dma_rmb(); - if(get_unaligned_le16(av_rx+2)!=av_group || get_unaligned_le32(av_rx+4)!=av_command || - get_unaligned_le32(av_rx+12)!=0x69706378)return -EPROTO; - av_pending=false;av_tag++; - if(av_group==4){av_opened=av_command==6;av_closed=av_command==7;} - else { - u64 used=get_unaligned_le64(av_rx+96);const u8 *edid=av_rx+120;u32 bytes; - if(get_unaligned_le64(av_rx+64)!=1032 || used<136 || used>1032 || - get_unaligned_le32(av_rx+112)!=0xd3)return -EPROTO; - bytes=used-8; - if(bytes%128 || bytes!=128*((u32)edid[126]+1) || memcmp(edid,"\0\xff\xff\xff\xff\xff\xff\0",8) || - get_unaligned_le32(av_rx+116)!=(0x8a000000|bytes))return -EBADMSG; - for(u32 at=0;at -#include -#include -#include -#include -#include -#include -#include "m3_dcpext_connector.h" -#define M3_DCPEXT_EDID_RAW_MAX 1024 - -struct m3_connector_state { - struct drm_connector_state base; - u64 generation; -}; - -static struct m3_dcpext_connector *to_m3(struct drm_connector *connector) -{ - return container_of(connector, struct m3_dcpext_connector, base); -} - -static void mode_geometry(const struct drm_display_mode *mode, - struct m3_dcpext_mode *geometry) -{ - *geometry = (struct m3_dcpext_mode) { - .width = mode->hdisplay, .height = mode->vdisplay, - .clock_khz = mode->clock, .htotal = mode->htotal, - .vtotal = mode->vtotal, - .interlaced = mode->flags & DRM_MODE_FLAG_INTERLACE, - .doublescan = mode->flags & DRM_MODE_FLAG_DBLSCAN, - }; -} - -static int select_mode(struct m3_dcpext_connector *c, - const struct drm_display_mode *mode, u8 timing[24]) -{ - struct m3_dcpext_mode geometry; - - /* Firmware does not accept porch timings here. Only modes produced by - * DRM's EDID parser are eligible; never accept arbitrary custom timings - * just because their size and rounded refresh match a firmware record. - */ - if (mode->vscan > 1 || mode->flags & ~(DRM_MODE_FLAG_PHSYNC | - DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)) - return -EINVAL; - mode_geometry(mode, &geometry); - return m3_dcpext_mode_select(&geometry, c->timings, - c->timings_count, timing); -} - -static bool mode_known(struct m3_dcpext_connector *c, - const struct drm_display_mode *mode) -{ - for (u32 i = 0; i < c->modes_count; i++) - if (drm_mode_equal(mode, &c->modes[i])) - return true; - return false; -} - -static int select_native_mode(struct m3_dcpext_connector *c, - const struct drm_display_mode *mode, u32 *selected) -{ - struct m3_dcpext_native_mode requested = {}; - int ret; - - if (mode->vscan > 1 || mode->flags & ~(DRM_MODE_FLAG_PHSYNC | - DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC) || - mode->hsync_start < mode->hdisplay || mode->hsync_end < mode->hsync_start || - mode->vsync_start < mode->vdisplay || mode->vsync_end < mode->vsync_start) - return -EINVAL; - mode_geometry(mode, &requested.geometry); - requested.hfront = mode->hsync_start - mode->hdisplay; - requested.hsync = mode->hsync_end - mode->hsync_start; - requested.vfront = mode->vsync_start - mode->vdisplay; - requested.vsync = mode->vsync_end - mode->vsync_start; - requested.hpositive = mode->flags & DRM_MODE_FLAG_PHSYNC; - requested.vpositive = mode->flags & DRM_MODE_FLAG_PVSYNC; - ret = m3_dcpext_native_mode_select(&requested, c->native_modes, - c->native_modes_count, selected); - /* This first HDMI path qualifies exact 1080p60 only. Do not advertise an - * EDID 59.94 mode while silently selecting the firmware's 60-Hz ID. */ - if (!ret && requested.geometry.clock_khz != c->native_modes[*selected].geometry.clock_khz) - ret = -EOPNOTSUPP; - return ret; -} - -static int get_modes(struct drm_connector *connector) -{ - struct m3_dcpext_connector *c = to_m3(connector); - struct drm_display_mode *mode, *next, *modes; - u32 count = 0; - u32 selected; - u8 timing[24]; - int ret; - - mutex_lock(&c->lock); - kfree(c->modes); - c->modes = NULL; - c->modes_count = 0; - ret = drm_edid_connector_update(connector, c->connected ? c->edid : NULL); - if (ret || !c->connected || !c->edid) - goto out; - drm_edid_connector_add_modes(connector); - list_for_each_entry_safe(mode, next, &connector->probed_modes, head) { - if (c->native_modes ? select_native_mode(c, mode, &selected) : - select_mode(c, mode, timing)) { - list_del(&mode->head); - drm_mode_destroy(connector->dev, mode); - } else { - count++; - } - } - modes = kcalloc(count, sizeof(*modes), GFP_KERNEL); - if (!modes) { - /* mode_valid will reject all modes, including core fallbacks. */ - count = 0; - goto out; - } - c->modes = modes; - list_for_each_entry(mode, &connector->probed_modes, head) - drm_mode_copy(&modes[c->modes_count++], mode); - /* EDID reports sink capability; v1 transports only RGB at eight bpc. */ - connector->display_info.bpc = 8; - connector->display_info.color_formats = BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444); - out: - mutex_unlock(&c->lock); - return count; -} - -static enum drm_connector_status detect(struct drm_connector *connector, bool force) -{ - struct m3_dcpext_connector *c = to_m3(connector); - enum drm_connector_status status; - - mutex_lock(&c->lock); - status = c->connected ? connector_status_connected : connector_status_disconnected; - mutex_unlock(&c->lock); - return status; -} - -static enum drm_mode_status mode_valid(struct drm_connector *connector, - const struct drm_display_mode *mode) -{ - struct m3_dcpext_connector *c = to_m3(connector); - bool valid; - - mutex_lock(&c->lock); - valid = c->connected && c->edid && mode_known(c, mode); - mutex_unlock(&c->lock); - return valid ? MODE_OK : MODE_BAD; -} - -static void destroy_state(struct drm_connector *connector, - struct drm_connector_state *state) -{ - struct m3_connector_state *s = container_of(state, struct m3_connector_state, base); - - __drm_atomic_helper_connector_destroy_state(state); - kfree(s); -} - -static void reset(struct drm_connector *connector) -{ - struct m3_connector_state *s; - - if (connector->state) - destroy_state(connector, connector->state); - connector->state = NULL; - s = kzalloc(sizeof(*s), GFP_KERNEL); - if (s) - __drm_atomic_helper_connector_reset(connector, &s->base); -} - -static struct drm_connector_state *duplicate_state(struct drm_connector *connector) -{ - struct m3_connector_state *s; - - if (!connector->state) - return NULL; - s = kmemdup(container_of(connector->state, struct m3_connector_state, base), - sizeof(*s), GFP_KERNEL); - if (!s) - return NULL; - __drm_atomic_helper_connector_duplicate_state(connector, &s->base); - return &s->base; -} - -static int atomic_check(struct drm_connector *connector, struct drm_atomic_state *state) -{ - struct m3_dcpext_connector *c = to_m3(connector); - struct drm_connector_state *cs = drm_atomic_get_new_connector_state(state, connector); - struct drm_crtc_state *crtc; - struct m3_connector_state *s = container_of(cs, struct m3_connector_state, base); - int ret = 0; - - if (!cs->crtc) { - s->generation = 0; - return 0; - } - crtc = drm_atomic_get_new_crtc_state(state, cs->crtc); - if (!crtc || !crtc->enable) - return -EINVAL; - mutex_lock(&c->lock); - if (!c->connected || !c->edid || !mode_known(c, &crtc->mode)) - ret = -ENODEV; - else - s->generation = c->generation; - mutex_unlock(&c->lock); - return ret; -} - -static void destroy(struct drm_connector *connector) -{ - struct m3_dcpext_connector *c = to_m3(connector); - - /* Owner has stopped connection workers and drained atomic commits. */ - drm_connector_cleanup(connector); - drm_edid_free(c->edid); - kfree(c->modes); - kfree(c->native_modes); -} - -static const struct drm_connector_funcs connector_funcs = { - .detect = detect, - .fill_modes = drm_helper_probe_single_connector_modes, - .destroy = destroy, - .reset = reset, - .atomic_duplicate_state = duplicate_state, - .atomic_destroy_state = destroy_state, -}; - -static const struct drm_connector_helper_funcs connector_helpers = { - .get_modes = get_modes, - .mode_valid = mode_valid, - .atomic_check = atomic_check, -}; - -int m3_dcpext_connector_init(struct drm_device *drm, - struct m3_dcpext_connector *c, struct drm_encoder *encoder, int connector_type) -{ - int ret; - - mutex_init(&c->lock); - c->generation = 1; - ret = drm_connector_init(drm, &c->base, &connector_funcs, connector_type); - if (ret) - return ret; - c->base.polled = DRM_CONNECTOR_POLL_HPD; - c->base.interlace_allowed = false; - c->base.doublescan_allowed = false; - drm_connector_helper_add(&c->base, &connector_helpers); - ret = drm_connector_attach_encoder(&c->base, encoder); - if (ret) - drm_connector_cleanup(&c->base); - return ret; -} - -u64 m3_dcpext_connector_invalidate(struct m3_dcpext_connector *c, bool connected) -{ - u64 generation; - - mutex_lock(&c->base.dev->mode_config.mutex); - mutex_lock(&c->lock); - generation = ++c->generation; - c->connected = connected; - drm_edid_free(c->edid); - c->edid = NULL; - kfree(c->modes); - c->modes = NULL; - c->modes_count = 0; - c->timings_count = 0; - kfree(c->native_modes); - c->native_modes = NULL; - c->native_modes_count = 0; - c->native_generation = 0; - drm_edid_connector_update(&c->base, NULL); - mutex_unlock(&c->lock); - mutex_unlock(&c->base.dev->mode_config.mutex); - return generation; -} - -static int publish_snapshot(struct m3_dcpext_connector *c, u64 generation, - const u8 *raw, u32 bytes, const u8 *timings, u32 count, - const struct m3_dcpext_native_mode *native, u64 native_generation) -{ - const struct drm_edid *edid; - struct m3_dcpext_native_mode *native_copy = NULL; - int ret = 0; - - if (!raw || bytes < 128 || bytes > M3_DCPEXT_EDID_RAW_MAX || - bytes % 128 || bytes != ((u32)raw[126] + 1) * 128 || - !count || (!native && (!timings || count > 64)) || - (native && (timings || !native_generation || count > M3_DCPEXT_NATIVE_MAX_MODES))) - return -EINVAL; - if (native) { - for (u32 i = 0; i < count; i++) { - u32 rate; - if (m3_dcpext_native_mode_validate(&native[i], M3_DCPEXT_HBR3_4LANE_KBPS, &rate) || - !native[i].hsync || !native[i].vsync || - (u64)native[i].geometry.width + native[i].hfront + native[i].hsync > native[i].geometry.htotal || - (u64)native[i].geometry.height + native[i].vfront + native[i].vsync > native[i].geometry.vtotal) - return -EINVAL; - } - } - edid = drm_edid_alloc(raw, bytes); - if (!edid) - return -ENOMEM; - if (!drm_edid_valid(edid)) { - drm_edid_free(edid); - return -EINVAL; - } - if (native) { - native_copy = kmemdup(native, count * sizeof(*native), GFP_KERNEL); - if (!native_copy) { - drm_edid_free(edid); - return -ENOMEM; - } - } - mutex_lock(&c->base.dev->mode_config.mutex); - mutex_lock(&c->lock); - if (!c->connected || c->generation != generation) { - ret = -ESTALE; - } else if (c->edid) { - /* A new snapshot requires invalidation, even on the same cable. */ - ret = -EALREADY; - } else { - c->edid = edid; - edid = NULL; - if (native) { - c->native_modes = native_copy; - native_copy = NULL; - c->native_modes_count = count; - c->native_generation = native_generation; - } else { - memcpy(c->timings, timings, count * 24); - c->timings_count = count; - } - } - mutex_unlock(&c->lock); - mutex_unlock(&c->base.dev->mode_config.mutex); - drm_edid_free(edid); - kfree(native_copy); - return ret; -} - -int m3_dcpext_connector_publish(struct m3_dcpext_connector *c, u64 generation, - const u8 *raw, u32 bytes, const u8 *timings, u32 count) -{ - return publish_snapshot(c, generation, raw, bytes, timings, count, NULL, 0); -} - -int m3_dcpext_connector_publish_native(struct m3_dcpext_connector *c, - u64 generation, const u8 *raw, u32 bytes, - const struct m3_dcpext_native_mode *modes, - u32 count, u64 native_generation) -{ - if (!modes) - return -EINVAL; - return publish_snapshot(c, generation, raw, bytes, NULL, count, modes, native_generation); -} - -void m3_dcpext_connector_hotplug(struct m3_dcpext_connector *c) -{ - drm_kms_helper_connector_hotplug_event(&c->base); -} - -int m3_dcpext_connector_commit_mode(struct m3_dcpext_connector *c, - const struct drm_connector_state *state, - const struct drm_display_mode *mode, - struct m3_dcpext_mode *geometry, u8 timing[24]) -{ - const struct m3_connector_state *s; - int ret; - - if (!state || state->connector != &c->base || !state->crtc || !mode || !geometry || !timing) - return -EINVAL; - s = container_of(state, struct m3_connector_state, base); - mutex_lock(&c->lock); - if (!c->connected || !c->edid || s->generation != c->generation) - ret = -ESTALE; - else if (c->native_modes) - ret = -EOPNOTSUPP; - else if (!mode_known(c, mode)) - ret = -EINVAL; - else { - ret = select_mode(c, mode, timing); - if (!ret) - mode_geometry(mode, geometry); - } - mutex_unlock(&c->lock); - return ret; -} - -int m3_dcpext_connector_commit_native(struct m3_dcpext_connector *c, - const struct drm_connector_state *state, - const struct drm_display_mode *mode, - struct m3_dcpext_native_mode *selected, - u64 *native_generation) -{ - const struct m3_connector_state *s; - u32 index; - int ret; - - if (!state || state->connector != &c->base || !state->crtc || - !mode || !selected || !native_generation) - return -EINVAL; - s = container_of(state, struct m3_connector_state, base); - mutex_lock(&c->lock); - if (!c->connected || !c->edid || s->generation != c->generation) - ret = -ESTALE; - else if (!c->native_modes || !mode_known(c, mode)) - ret = -EINVAL; - else { - ret = select_native_mode(c, mode, &index); - if (!ret) { - *selected = c->native_modes[index]; - *native_generation = c->native_generation; - } - } - mutex_unlock(&c->lock); - return ret; -} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.h deleted file mode 100644 index 573a1874ace6a..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_connector.h +++ /dev/null @@ -1,68 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -#ifndef M3_DCPEXT_CONNECTOR_H -#define M3_DCPEXT_CONNECTOR_H - -#include -#include -#include "m3_dcpext_modes.h" - -struct drm_edid; -struct drm_encoder; - -/* One native HDMI port, independent of the assigned DCPEXT controller. - * The route owner serializes invalidate/publish against hardware commits. - * Lock order: route owner -> mode_config.mutex -> lock. DRM callbacks take - * only lock (mode_config.mutex is already held during probing). - * No firmware operations or userspace notifications run with lock held. - */ -struct m3_dcpext_connector { - struct drm_connector base; - struct mutex lock; - u64 generation; - bool connected; - const struct drm_edid *edid; - u8 timings[64 * 24]; - u32 timings_count; - struct m3_dcpext_native_mode *native_modes; - u32 native_modes_count; - u64 native_generation; - struct drm_display_mode *modes; - u32 modes_count; -}; - -int m3_dcpext_connector_init(struct drm_device *drm, - struct m3_dcpext_connector *connector, - struct drm_encoder *encoder, int connector_type); -/* Call for every HPD/route change, before dispatching new firmware queries. - * Returns the generation to attach to all asynchronous query results. - * Callers must not hold DRM locks. Hotplug notification is explicit below. - */ -u64 m3_dcpext_connector_invalidate(struct m3_dcpext_connector *connector, - bool connected); -/* Publish one coherent EDID + firmware timing snapshot. Stale replies cannot - * repopulate a disconnected/reassigned port. Original EDID bytes are retained. - */ -int m3_dcpext_connector_publish(struct m3_dcpext_connector *connector, - u64 generation, const u8 *edid, u32 bytes, - const u8 *timings, u32 count); -int m3_dcpext_connector_publish_native(struct m3_dcpext_connector *connector, - u64 generation, const u8 *edid, u32 bytes, - const struct m3_dcpext_native_mode *modes, - u32 count, u64 native_generation); -/* Owner calls after releasing its route lock, with the DRM device registered. */ -void m3_dcpext_connector_hotplug(struct m3_dcpext_connector *connector); -/* Recheck a checked atomic state's generation immediately before hardware - * submission, under the route owner's lock. Returns a complete firmware record. - * This does not implement scanout or imply presentation/retirement completion. - */ -int m3_dcpext_connector_commit_mode(struct m3_dcpext_connector *connector, - const struct drm_connector_state *state, - const struct drm_display_mode *mode, - struct m3_dcpext_mode *geometry, - u8 timing[24]); -int m3_dcpext_connector_commit_native(struct m3_dcpext_connector *connector, - const struct drm_connector_state *state, - const struct drm_display_mode *mode, - struct m3_dcpext_native_mode *selected, - u64 *native_generation); -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_dma.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_dma.c deleted file mode 100644 index 4fa7fb609c670..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_dma.c +++ /dev/null @@ -1,330 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* J514S external DART additive mapping diagnostic. No firmware messages. - * Adapted from the M4 m4_dcpext_dma_probe mapper; no M4 PHY/session code. - * T6030 SID5 TCR=0xc01 and roots qualified in the guarded 2026-09-21 audit. - * DVA 0x40000000 belongs to M3 firmware TEXT: use empty slot 0x20000000. - * PTE encoding and ordering: Asahi io-pgtable-dart.c, fba91e5d9577. - * T8110 SID flush: Asahi apple-dart.c, same revision. - * Reservations: M3 m1n1 9b9fa13 handoff and same-boot J514S table audit. - * This is not a DMA execution test: the private DVA is never sent to DCP. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifndef DCPEXT_INSTANCE -#define DCPEXT_INSTANCE 0 -#endif - -#define LEASE_PAGE 0x4000 -#ifdef DCPEXT_WITH_SESSION -#define LEASE_SIZE 0x400000 -#else -#define LEASE_SIZE 0x10000 -#endif -#define LEASE_DVA 0x20000000ULL -#define LEASE_SLOT (LEASE_DVA >> 25) -#define PTE_ADDRESS GENMASK_ULL(39, 10) -/* T6030 DCP is not dma-coherent. Match dma-iommu + dart_prot_to_pte: - * shared host buffers need DART2 NO_CACHE, unlike firmware-owned TEXT/DATA. - */ -#define PTE_RW (GENMASK_ULL(51, 40) | BIT_ULL(1) | BIT_ULL(0)) - -static unsigned int port = DCPEXT_INSTANCE; -module_param(port, uint, 0400); -static int result = -EINPROGRESS; -module_param(result, int, 0400); -MODULE_PARM_DESC(result, "0 only after private mapping removal and inherited root comparison"); -static struct device *dma_dev; -static void *pool, *leaf; -static dma_addr_t pool_dma; -static u64 *root, *saved; -static void __iomem *regs; -static phys_addr_t dart_base; -static bool region_owned, pinned; -#ifdef DCPEXT_WITH_SESSION -static bool native_desktop; -module_param(native_desktop,bool,0400); -static bool native_kms_trial; -module_param(native_kms_trial,bool,0400); -static bool native_frame; -module_param(native_frame,bool,0400); -static bool native_startup; -module_param(native_startup,bool,0400); -#include "m3_dcpext_frame.h" -#include "m3_dcpext_session.h" -#endif - -static u64 encode_address(phys_addr_t address) -{ - return (address >> 4) & PTE_ADDRESS; -} - -static int fabric_powered(void); -static int remove_private_mapping(void); -static void cleanup(void); - - -static int flush_sid(void) -{ - u32 status; - - /* DART table walks are coherent, as in apple_dart_finalize_domain. */ - /* Completion polling orders reuse; a diagnostic sleep here would - * impose a 20 ms floor on every runtime scanout remapping. */ - dma_wmb(); - writel(0x100 | 5, regs + 0x80); - return readl_poll_timeout(regs + 0x80, status, !(status & BIT(31)), 1, 10000); -} - -static int fabric_powered(void) -{ - struct device_node *node; - struct regmap *map; - u32 state, i, offset = port ? 0x3c0 : 0x3b8; - int ret; - - node = of_find_node_by_path("/soc/power-management@350700000"); - if (!node || !of_device_is_compatible(node, "apple,t6030-pmgr")) { - of_node_put(node); - return -ENODEV; - } - map = syscon_node_to_regmap(node); - of_node_put(node); - if (IS_ERR(map)) - return PTR_ERR(map); - for (i = 0; i < 2; i++) { - ret = regmap_read(map, offset + (i ? (port ? 0x30 : 0x20) : 0), &state); - if (ret) - return ret; - if ((state & 0xff) != 0xff || - state & (BIT(31) | BIT(28) | BIT(12) | BIT(11) | BIT(10))) - return -EHOSTDOWN; - } - return 0; -} - -static void cleanup(void) -{ -#ifdef DCPEXT_WITH_SESSION - session_cleanup(); -#endif - if (root) - memunmap(root); - if (leaf) - free_page((unsigned long)leaf); - if (pool) - dma_free_coherent(dma_dev, LEASE_SIZE, pool, pool_dma); - if (dma_dev) - root_device_unregister(dma_dev); - if (regs) - iounmap(regs); - if (region_owned) - release_mem_region(dart_base, 0x4000); - kfree(saved); -} - -/* Called only after protocol completion, or by the no-firmware mapper. On - * any mismatch all allocations and the module reference remain owned. */ -static int remove_private_mapping(void) -{ - u64 entry = encode_address(virt_to_phys(leaf)) | 1; - unsigned int i; - int ret; - - /* Verify only our private entries; inherited mappings must be unchanged. */ - for (i = 0; i < LEASE_SIZE / LEASE_PAGE; i++) - if ((((u64 *)leaf)[i] & PTE_ADDRESS) << 4 != pool_dma + i * LEASE_PAGE) { - ret = -EIO; - return ret; - } - ret = -EAGAIN; - if (cmpxchg64_relaxed(&root[LEASE_SLOT], entry, 0) != entry) - return ret; - ret = flush_sid(); - if (ret) - return ret; - if (memcmp(root, saved, LEASE_PAGE) || (readl(regs + 0x100) & BIT(31))) { - ret = -EIO; - return ret; - } - return 0; -} - -static int __init probe_init(void) -{ - struct device_node *dcp = NULL, *dart = NULL, *ram = NULL; - struct resource resource, reservation; - phys_addr_t root_pa, leaf_pa; - const char *uuid; - char alias[16]; - u32 marker, ttbr, tcr, protect, params[2]; - u64 entry; - int count, segment_bytes, i, ret = -ENODEV; - - if (port > 1 || PAGE_SIZE != LEASE_PAGE || - !of_machine_is_compatible("apple,j514s") || - !of_machine_is_compatible("apple,t6030")) - return -ENODEV; - snprintf(alias, sizeof(alias), "dcpext%u", port); - dcp = of_find_node_by_path(alias); - if (!dcp || of_device_is_available(dcp) || - !of_device_is_compatible(dcp, "apple,t6030-dcpext") || - of_property_read_u32(dcp, "apple,j514s-dcpext-reservations", &marker) || marker != 1 || - of_property_read_string(dcp, "apple,firmware-uuid", &uuid) || - strcmp(uuid, "DDF38191-93B3-324A-BC8F-643006F5AC82")) - goto done; - if (!of_get_property(dcp, "apple,j514s-dcpext-segments", &segment_bytes) || - segment_bytes <= 0 || segment_bytes % 32) - goto done; - count = of_property_count_u32_elems(dcp, "memory-region"); - if (count != segment_bytes / 32 + 3 || - of_property_read_u32_array(dcp, "iommus", params, 2) || params[1] != 5) - goto done; - dart = of_find_node_by_phandle(params[0]); - ram = of_parse_phandle(dcp, "memory-region", count - 3); - dart_base = port ? 0x2d530c000ULL : 0x2d130c000ULL; - if (!dart || of_device_is_available(dart) || - !of_device_is_compatible(dart, "apple,t8110-dart") || - of_address_to_resource(dart, 0, &resource) || resource.start != dart_base || - resource_size(&resource) != 0x4000 || !ram || - !of_device_is_compatible(ram, "apple,dart-mem") || - !of_property_read_bool(ram, "no-map") || - of_address_to_resource(ram, 0, &reservation)) - goto done; - ret = fabric_powered(); - if (ret) - goto done; - ret = -EBUSY; - if (!request_mem_region(dart_base, 0x4000, "m3-dcpext-dma-probe")) - goto done; - region_owned = true; - /* Match of_iomap/devm_ioremap_resource: T6030 soc is nonposted-mmio. */ - if (!(resource.flags & IORESOURCE_MEM_NONPOSTED)) { - ret = -EINVAL; - goto done; - } - regs = ioremap_np(dart_base, 0x4000); - ret = -ENOMEM; - if (!regs) - goto done; - protect = readl(regs + 0x200); - tcr = readl(regs + 0x1000 + 5 * 4); - ttbr = readl(regs + 0x1400 + 5 * 4); - root_pa = (u64)((ttbr & GENMASK(29, 2)) >> 2) << 14; - pr_info("m3_dcpext_dma: protect=%#x tcr=%#x ttbr=%#x root=%pap reservation=%pr error=%#x\n", - protect, tcr, ttbr, &root_pa, &reservation, readl(regs + 0x100)); - ret = -EINVAL; - /* T8110 bits 11:8 are ignored while REMAP_EN (bit 7) is clear. - * Firmware also leaves 0x801 here; require all active controls exactly - * as before and recheck the untouched full TCR before publication. */ - if (!(protect & 1) || (tcr & ~GENMASK(11, 8)) != 1 || !(ttbr & 1) || - root_pa < reservation.start || root_pa > reservation.end || - reservation.end - root_pa + 1 < LEASE_PAGE || (readl(regs + 0x100) & BIT(31))) - goto done; - root = memremap(root_pa, LEASE_PAGE, MEMREMAP_WB); - ret = -ENOMEM; - if (!root) - goto done; - saved = kmemdup(root, LEASE_PAGE, GFP_KERNEL); - leaf = (void *)get_zeroed_page(GFP_KERNEL); - if (!saved || !leaf) - goto done; - ret = -EBUSY; - if (READ_ONCE(root[LEASE_SLOT])) - goto done; -#ifdef DCPEXT_WITH_SESSION - ret = session_prepare(dcp); - if (!ret) ret = session_tables(&reservation); - if (!ret) ret = session_audit(); - if (ret) goto done; -#endif - dma_dev = root_device_register(port ? "m3-dcpext1" : "m3-dcpext-dma-probe"); - if (IS_ERR(dma_dev)) { - ret = PTR_ERR(dma_dev); - dma_dev = NULL; - goto done; - } - /* No OF node / IOMMU attachment: obtain a host-owned direct allocation. - * Require identity DMA-to-physical translation for the DART leaf. - */ - ret = dma_coerce_mask_and_coherent(dma_dev, DMA_BIT_MASK(42)); - if (ret) - goto done; - pool = dma_alloc_coherent(dma_dev, LEASE_SIZE, &pool_dma, GFP_KERNEL); - ret = -ENOMEM; - if (!pool) - goto done; - leaf_pa = virt_to_phys(leaf); - pr_info("m3_dcpext_dma: pool=%pad physical=%#llx leaf=%pap\n", - &pool_dma, (u64)dma_to_phys(dma_dev, pool_dma), &leaf_pa); - ret = -EINVAL; - if ((pool_dma | leaf_pa) & (LEASE_PAGE - 1) || - dma_to_phys(dma_dev, pool_dma) != pool_dma || - pool_dma + LEASE_SIZE > BIT_ULL(42) || leaf_pa >= BIT_ULL(42)) - goto done; - for (i = 0; i < LEASE_SIZE / LEASE_PAGE; i++) - ((u64 *)leaf)[i] = encode_address(pool_dma + i * LEASE_PAGE) | PTE_RW; - memset(pool, 0xa5, LEASE_SIZE); - dma_wmb(); - entry = encode_address(leaf_pa) | 1; - ret = -EAGAIN; - if (readl(regs + 0x1400 + 5 * 4) != ttbr || - readl(regs + 0x1000 + 5 * 4) != tcr || - readl(regs + 0x200) != protect || memcmp(root, saved, LEASE_PAGE)) - goto done; - /* A published table must survive failed invalidation/teardown. */ - __module_get(THIS_MODULE); - pinned = true; - if (cmpxchg64_relaxed(&root[LEASE_SLOT], 0, entry)) { - module_put(THIS_MODULE); - pinned = false; - goto done; - } - ret = flush_sid(); - if (ret) - goto done; - pr_info("m3_dcpext_dma_probe: port=%u SID5 private DVA=%#llx bytes=%#x mapped; no firmware notification\n", - port, LEASE_DVA, LEASE_SIZE); -#ifdef DCPEXT_WITH_SESSION - ret = frame_prepare_publish(dcp,count); - if(!ret)ret = pio_prepare_publish(dcp,count); - if(!ret)ret = session_start(); - goto done; /* Keep every DMA allocation and module reference until reboot. */ -#endif - ret = remove_private_mapping(); - if (ret) - goto done; - module_put(THIS_MODULE); - pinned = false; - pr_info("m3_dcpext_dma_probe: private mapping removed; inherited root unchanged\n"); -done: - pr_info("m3_dcpext_dma: result=%d published=%u\n", ret, pinned); - of_node_put(ram); - of_node_put(dart); - of_node_put(dcp); - result = ret; - if (pinned) { - pr_info("m3_dcpext_dma_probe: result=%d; retaining all allocations/resources until reboot\n", ret); - return 0; - } - cleanup(); - return ret; -} - -static void __exit probe_exit(void) -{ -} -module_init(probe_init); -module_exit(probe_exit); -MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("J514S external DART additive mapping/remove diagnostic, no firmware execution"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_frame.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_frame.h deleted file mode 100644 index e9bc28c19956d..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_frame.h +++ /dev/null @@ -1,187 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* Paged static image adapted from M4 dcpext_frame_map.h. M3 SID5/FE0, - * qualified reservations and nonposted accesses. All allocations are retained - * until reboot, including on partial publication: the caller is already pinned. - */ -#include -#include -#include "m3_dcpext_modes.h" -#define FRAME_DVA 0x30000000ULL -#define FRAME_SLOT (FRAME_DVA >> 25) -#define FRAME_BYTES ALIGN(M3_DCPEXT_MAX_PITCH * M3_DCPEXT_MAX_HEIGHT + 24 * LEASE_PAGE, LEASE_PAGE) -#define FRAME_LEAVES DIV_ROUND_UP(2 * FRAME_BYTES / LEASE_PAGE, LEASE_PAGE / 8) -static_assert(FRAME_SLOT + FRAME_LEAVES <= LEASE_PAGE / 8); -static_assert(LEASE_PAGE == PAGE_SIZE); -static bool iboot_image; -module_param(iboot_image,bool,0400); -static void *frame_pixels,*frame_ram; -static u64 *frame_leaf[FRAME_LEAVES],*frame_root; -static void __iomem *frame_regs; -static dma_addr_t frame_dma[2*FRAME_BYTES/LEASE_PAGE]; -static u64 encode_address(phys_addr_t address); -static int flush_sid(void); -static int frame_prepare_publish(struct device_node *dcp,int count) -{ - struct device_node *ram,*disp,*dart; - struct resource reservation,res; - u32 params[2],ttbr,tcr,protect,status; - phys_addr_t pa;int ret=-EINVAL; - if(!iboot_image && !native_startup)return 0; - if(!pinned)return -EINVAL; - ram=of_parse_phandle(dcp,"memory-region",count-2); - if(!ram)return -EINVAL; - if(!of_device_is_compatible(ram,"apple,dart-mem") || !of_property_read_bool(ram,"no-map") || - of_address_to_resource(ram,0,&reservation))goto ram_out; - if(resource_size(&reservation)>0x100000 || resource_size(&reservation)>2)<<14; - ret=-EINVAL; - if((tcr & ~GENMASK(11,8))!=5 || !(ttbr&1) || !(protect&1) || pareservation.end || - reservation.end-pa+1=BIT_ULL(42)){ret=-ERANGE;goto ram_out;} - } - for(unsigned int y=0;y<1080;y++)for(unsigned int x=0;x<1920;x++){ - static const u32 bars[]={0xffff0000,0xff00ff00,0xff0000ff,0xffffff00,0xffff00ff,0xff00ffff,0xffffffff,0xff202020}; - ((u32 *)frame_pixels)[y*1920+x]=x<8 || y<8 || x>=1912 || y>=1072?0xffffffff:bars[x*8/1920]; - } - for(unsigned int i=0;i=BIT_ULL(42) || frame_dma[i]&(LEASE_PAGE-1) || dma_to_phys(dma_dev,frame_dma[i])!=frame_dma[i]){ - ret=-ERANGE;goto ram_out; - } - frame_leaf[i/(LEASE_PAGE/8)][i%(LEASE_PAGE/8)]=encode_address(frame_dma[i])|PTE_RW; - } - ret=-EAGAIN; - if(readl(frame_regs+0x1400)!=ttbr || readl(frame_regs+0x1000)!=tcr || readl(frame_regs+0x200)!=protect)goto ram_out; - dma_wmb(); - for(unsigned int i=0;i>2)<<14; - pr_info("m3_dcpext_native: PIODMA TCR=%#x TTBR=%#x root=%pap reservation=%pR\n",tcr,ttbr,&pa,&res); - /* T8110 TCR bits11:8 are the remap selector; bit7 enables remapping. - * Boot values a05/b05/e05 all have remapping disabled. Qualify every active - * control bit, preserve the ignored selector, and recheck the exact register - * before publication. Definitions: drivers/iommu/apple-dart.c. */ - if((tcr!=0xa05 && (tcr & ~GENMASK(11,8))!=5) || !(ttbr&1) || pares.end || res.end-pa+10x100000 || (res.start|resource_size(&res))&(LEASE_PAGE-1))goto out; - mapping=memremap(res.start,resource_size(&res),MEMREMAP_WB); - if(!mapping){ret=-ENOMEM;goto out;} - pio_root=mapping+pa-res.start; - ret=-EBUSY; - if(READ_ONCE(pio_root[LEASE_SLOT]) || readl(frame_regs+0x1410)!=ttbr || - readl(frame_regs+0x1010)!=tcr)goto out; - dma_wmb(); - if(cmpxchg64_relaxed(&pio_root[LEASE_SLOT],0,encode_address(virt_to_phys(leaf))|1))goto out; - dma_wmb();writel(0x104,frame_regs+0x80); - ret=readl_poll_timeout(frame_regs+0x80,status,!(status&BIT(31)),1,10000); - if(!ret && (readl(frame_regs+0x100)&BIT(31)))ret=-EIO; - if(!ret)pio_published=true; - pr_info("m3_dcpext_native: PIODMA SID4 arena mapped ret=%d; retained until reboot\n",ret); -out: - of_node_put(ram);return ret; -} - -static void frame_sync(unsigned int slot,bool for_cpu) -{ - unsigned int first=slot*(FRAME_BYTES/LEASE_PAGE); - for(unsigned int i=first;i1 || (offset & (LEASE_PAGE-1)) || bytes>FRAME_BYTES)return -EINVAL; - if(sgt){ - for_each_sgtable_page(sgt,&it,offset/LEASE_PAGE){ - phys_addr_t pa=page_to_phys(sg_page_iter_page(&it)); - if(pa>=BIT_ULL(42))return -ERANGE; - if(++n==pages)break; - } - if(n!=pages)return -EINVAL; - } - n=0; - if(sgt)for_each_sgtable_page(sgt,&it,offset/LEASE_PAGE){ - unsigned int i=first+n; - WRITE_ONCE(frame_leaf[i/(LEASE_PAGE/8)][i%(LEASE_PAGE/8)], - encode_address(page_to_phys(sg_page_iter_page(&it)))|PTE_RW); - if(++n==pages)break; - } - /* Keep unused/guard pages backed by owned scratch memory. */ - for(;n TEXT 0x37bd0 -> virtual 360. - * Same group 8 as HPD change. This is a DPCD service interrupt, not a cable - * disconnect. Give it separate pinned buffers: firmware may be waiting for - * this IRQ while the ordinary HPD or native modeset RPC is still pending. - */ -static u32 (*hdmi_irq_count)(void); -static u32 hdmi_irq_seen,hdmi_irq_sent,hdmi_irq_acked; -module_param(hdmi_irq_sent,uint,0400); -module_param(hdmi_irq_acked,uint,0400); -static u16 hdmi_irq_tag=0x8000; -static u8 *hdmi_irq_tx,*hdmi_irq_rx; -static u64 hdmi_irq_txd,hdmi_irq_rxd,hdmi_irq_deadline; -static bool hdmi_irq_pending; -/* GPIO47 did not signal the converter's FRL-ready/active service events on - * J514S. Poll the standard firmware service handler while connected as a - * fallback. 100 ms is below both the 500 ms ready and 450 ms training waits; - * only one request may be outstanding, with a bounded response deadline. - * No reads of converter registers or modifications of training policy. */ -static u64 hdmi_service_next; - -static int hdmi_irq_progress(void) -{ - u8 wire[70]={};u32 count;int ret; - if(!hdmi_irq_count)return 0; - if(hdmi_irq_pending) - return session_now(NULL)>=hdmi_irq_deadline ? -ETIMEDOUT : 0; - count=hdmi_irq_count(); - if(!port0_announced || rpc_stage<2){hdmi_irq_seen=count;return 0;} - if(session.phase!=DCPEXT_RUNNING)return 0; - if(READ_ONCE(desktop_stop) || desktop_disconnected)return 0; - if(count==hdmi_irq_seen && session_now(NULL)=108 && p[0]==2 && p[20]==4 && p[21]==0 && - get_unaligned_le16(p+22)==0x30 && !memcmp(p+40,"disp0-service",14)){ - if(ib_announced)return -EPROTO; - ib_announced=true;ib_channel=channel;return 0; - } - if(size<70 || p[0]!=2 || p[20]!=4 || p[21]!=0x20 || - get_unaligned_le16(p+22)!=0xc0)return 0; - if(!ib_pending || channel!=ib_channel || get_unaligned_le16(p+32)!=ib_tag || - get_unaligned_le64(p+44)!=ib_rx_dva)return -EPROTO; - bytes=get_unaligned_le32(p+60); - pr_info("m3_dcpext_iboot: reply op=%u status=%#x bytes=%u\n",ib_operation,get_unaligned_le32(p+40),bytes); - if(get_unaligned_le32(p+40))return -EREMOTEIO; - if(bytes>4096)return -EOVERFLOW; - dma_rmb(); - if(ib_operation!=2 && ib_operation!=6 && ib_operation!=1){ - if(bytes<8 || get_unaligned_le32(ib_rx)!=ib_operation || - get_unaligned_le32(ib_rx+4)>bytes || get_unaligned_le32(ib_rx+4)<8)return -EPROTO; - if(ib_operation==3){ - if(bytes<20 || ib_rx[8]>1)return -EPROTO; - if(!ib_rx[8] || !get_unaligned_le32(ib_rx+12) || !get_unaligned_le32(ib_rx+16)){ - if(++ib_attempts>=16)return -ENODEV; - ib_next_query=session_now(NULL)+250;ib_pending=false;ib_tag++;return 0; - } - ib_timings=get_unaligned_le32(ib_rx+12);ib_colors=get_unaligned_le32(ib_rx+16); - if(!ib_timings || !ib_colors || ib_timings>128 || ib_colors>128)return -ERANGE; - pr_info("m3_dcpext_iboot: HPD=1 timing_count=%u color_count=%u\n",ib_timings,ib_colors); - }else{ - if(bytes<12)return -EPROTO; - count=get_unaligned_le32(ib_rx+8); - if(count!=(ib_operation==4?ib_timings:ib_colors) || bytes<12+count*24)return -EPROTO; - for(unsigned int i=0;i=ARRAY_SIZE(ops))return -EPROTO; - if(iboot_image && ib_stage==6 && session_now(NULL) -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "m3_dcpext_kms.h" - -/* Bounded diagnostic control for qualifying callback delivery latency. */ -static unsigned int flip_slack_us = 2000; -module_param(flip_slack_us, uint, 0600); - -struct m3_dcpext_kms { - struct drm_device drm; - struct drm_plane plane; - struct drm_crtc crtc; - struct drm_encoder encoder; - struct m3_dcpext_connector connector; - const struct m3_dcpext_kms_ops *ops; - void *cookie; - struct mutex *route_lock; - /* Outlives the hardware owner while userspace retains DRM references. */ - struct mutex owner_lock; - bool detached; - bool failed; - u64 completed; - ktime_t submit_time, completion_time; -}; - -static struct m3_dcpext_kms *to_kms(struct drm_device *drm) -{ - return container_of(drm, struct m3_dcpext_kms, drm); -} - -static int plane_check(struct drm_plane *plane, struct drm_atomic_state *state) -{ - struct drm_plane_state *p = drm_atomic_get_new_plane_state(state, plane); - struct drm_crtc_state *c; - struct drm_gem_object *obj; - u64 extent; - int ret; - - if (!p->crtc) - return 0; - c = drm_atomic_get_crtc_state(state, p->crtc); - if (IS_ERR(c)) - return PTR_ERR(c); - ret = drm_atomic_helper_check_plane_state(p, c, DRM_PLANE_NO_SCALING, - DRM_PLANE_NO_SCALING, false, true); - if (ret || !p->visible) - return ret; - /* Native v1 supports one full-screen linear RGB plane; no cropping. */ - if (p->src_x || p->src_y || p->crtc_x || p->crtc_y || - p->src_w != (u32)c->mode.hdisplay << 16 || - p->src_h != (u32)c->mode.vdisplay << 16 || - p->crtc_w != c->mode.hdisplay || p->crtc_h != c->mode.vdisplay || - p->fb->width != c->mode.hdisplay || p->fb->height != c->mode.vdisplay || - p->fb->modifier != DRM_FORMAT_MOD_LINEAR || - p->fb->pitches[0] < p->fb->width * 4) - return -EINVAL; - obj = drm_gem_fb_get_obj(p->fb, 0); - extent = (u64)p->fb->pitches[0] * (p->fb->height - 1) + - (u64)p->fb->width * 4 + p->fb->offsets[0]; - return obj && extent <= obj->size ? 0 : -EINVAL; -} - -static void plane_update(struct drm_plane *plane, struct drm_atomic_state *state) {} -static const struct drm_plane_helper_funcs plane_helpers = { - .prepare_fb = drm_gem_plane_helper_prepare_fb, - .atomic_check = plane_check, - .atomic_update = plane_update, - .atomic_disable = plane_update, -}; -static const struct drm_plane_funcs plane_funcs = { - .update_plane = drm_atomic_helper_update_plane, - .disable_plane = drm_atomic_helper_disable_plane, - .destroy = drm_plane_cleanup, - .reset = drm_atomic_helper_plane_reset, - .atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state, - .atomic_destroy_state = drm_atomic_helper_plane_destroy_state, -}; - -static int present(struct m3_dcpext_kms *kms, struct drm_atomic_state *state, - bool enable) -{ - struct drm_crtc_state *c = drm_atomic_get_new_crtc_state(state, &kms->crtc); - struct drm_connector_state *connector; - struct drm_framebuffer *fb = NULL; - struct sg_table *sgt = NULL; - bool copied = false; - struct m3_dcpext_native_mode mode; - struct iosys_map map[DRM_FORMAT_MAX_PLANES], data[DRM_FORMAT_MAX_PLANES]; - u64 generation = 0; - int ret; - - if (enable) { - struct drm_plane_state *p = drm_atomic_get_new_plane_state(state, &kms->plane); - if (!p) - return -EINVAL; - fb = p->fb; - } - if (fb) { - struct drm_gem_object *obj = drm_gem_fb_get_obj(fb, 0); - struct scatterlist *sg; - unsigned int i; - u64 bytes = (u64)fb->pitches[0] * fb->height; - bool direct = !(fb->offsets[0] & (PAGE_SIZE - 1)) && - bytes <= (u64)M3_DCPEXT_MAX_PITCH * M3_DCPEXT_MAX_HEIGHT && - fb->offsets[0] <= obj->size && bytes <= obj->size - fb->offsets[0]; - if (direct) { - sgt = drm_gem_shmem_get_pages_sgt(to_drm_gem_shmem_obj(obj)); - if (IS_ERR(sgt)) return PTR_ERR(sgt); - for_each_sgtable_sg(sgt, sg, i) - if ((sg_phys(sg) & (PAGE_SIZE - 1)) || - sg_phys(sg) >= BIT_ULL(42) || - sg->length > BIT_ULL(42) - sg_phys(sg)) direct = false; - if (!direct) sgt = NULL; - } - if (sgt) { - /* The shmem/PRIME helper mapped this attachment BIDIRECTIONAL. */ - dma_sync_sgtable_for_device(drm_dev_dma_dev(fb->dev), sgt, DMA_BIDIRECTIONAL); - goto mapped; - } - ret = drm_gem_fb_begin_cpu_access(fb, DMA_FROM_DEVICE); - if (ret) - return ret; - ret = drm_gem_fb_vmap(fb, map, data); - if (ret) - goto end_access; - copied = true; - } -mapped: - mutex_lock(&kms->owner_lock); - if (kms->detached) { - ret = -ENODEV; - goto unlock_owner; - } - mutex_lock(kms->route_lock); - ret = kms->failed ? -EIO : 0; - if (!ret && enable) { - connector = drm_atomic_get_new_connector_state(state, &kms->connector.base); - ret = connector ? m3_dcpext_connector_commit_native(&kms->connector, - connector, &c->mode, &mode, &generation) : -EINVAL; - } - if (!ret) { - kms->submit_time = ktime_get(); - ret = kms->ops->present(kms->cookie, enable ? &mode : NULL, generation, - copied ? &data[0] : NULL, fb ? fb->pitches[0] : 0, - fb && fb->format->format == DRM_FORMAT_XRGB8888, fb, sgt); - kms->completion_time = ktime_get(); - } - if (!ret && enable) - kms->completed++; - if (ret && ret != -ESTALE && ret != -ENODEV) { - kms->failed = true; - drm_err(&kms->drm, "external commit failed %d; scanout retained by owner\n", ret); - } - mutex_unlock(kms->route_lock); -unlock_owner: - mutex_unlock(&kms->owner_lock); - if (copied) - drm_gem_fb_vunmap(fb, map); -end_access: - if (fb && !sgt) - drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE); - return ret; -} - -static void finish_event(struct drm_crtc *crtc, struct drm_atomic_state *state, int ret) -{ - struct drm_crtc_state *c = drm_atomic_get_new_crtc_state(state, crtc); - struct drm_pending_vblank_event *event = c->event; - unsigned long flags; - - if (!ret && c->active) - drm_crtc_handle_vblank(crtc); - if (!event) - return; - c->event = NULL; - if (ret) { - /* Cancel the userspace event without claiming a presentation, but - * release the helper dependency and signal an error on its fence. - * Otherwise a rejected commit blocks every later modeset/unplug. */ - if (event->base.fence) { - dma_fence_set_error(event->base.fence, ret); - dma_fence_signal(event->base.fence); - } - if (event->base.completion) { - complete_all(event->base.completion); - event->base.completion_release(event->base.completion); - } - drm_event_cancel_free(crtc->dev, &event->base); - return; - } - spin_lock_irqsave(&crtc->dev->event_lock, flags); - if (c->active && event->event.base.type == DRM_EVENT_FLIP_COMPLETE) { - struct m3_dcpext_kms *kms = to_kms(crtc->dev); - ktime_t flip = kms->completion_time; - s64 delta = ktime_us_delta(flip, kms->submit_time); - struct timespec64 tv; - - /* Based on Asahi dcp_crtc_send_page_flip_event(): DCP reports - * completion after the physical flip. Using callback receipt as - * the flip time makes KWin submit too late for the next refresh, - * halving HDMI cadence. Preserve at least 500 us after submit; - * M3 measurements need a 2 ms cap rather than Asahi's 1 ms. - * This is a bounded latency estimate, not - * a hardware timestamp. Never signal before real completion. */ - if (delta > 500) - flip = ktime_sub_us(flip, min_t(s64, delta - 500, - min(READ_ONCE(flip_slack_us), 4000U))); - tv = ktime_to_timespec64(flip); - event->pipe = drm_crtc_index(crtc); - event->event.vbl.sequence = drm_crtc_vblank_count(crtc); - event->event.vbl.tv_sec = tv.tv_sec; - event->event.vbl.tv_usec = tv.tv_nsec / 1000; - drm_send_event_timestamp_locked(crtc->dev, &event->base, flip); - } else { - drm_crtc_send_vblank_event(crtc, event); - } - spin_unlock_irqrestore(&crtc->dev->event_lock, flags); -} - -static void crtc_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) -{ - if (drm_atomic_get_new_crtc_state(state, crtc)->active) - finish_event(crtc, state, present(to_kms(crtc->dev), state, true)); -} -static void crtc_enable(struct drm_crtc *crtc, struct drm_atomic_state *state) -{ - const struct drm_vblank_crtc_config config = { .offdelay_ms = 0 }; - drm_crtc_vblank_on_config(crtc, &config); -} -static void crtc_disable(struct drm_crtc *crtc, struct drm_atomic_state *state) -{ - int ret = present(to_kms(crtc->dev), state, false); - /* An active-to-active modeset completes after its new surface, not - * after temporarily powering off the old timing. */ - if (!drm_atomic_get_new_crtc_state(state, crtc)->active) - finish_event(crtc, state, ret); - drm_crtc_vblank_off(crtc); -} -static int enable_vblank(struct drm_crtc *crtc) { return 0; } -static void disable_vblank(struct drm_crtc *crtc) {} -static const struct drm_crtc_helper_funcs crtc_helpers = { - .atomic_enable = crtc_enable, - .atomic_disable = crtc_disable, - .atomic_flush = crtc_flush, -}; -static const struct drm_crtc_funcs crtc_funcs = { - .set_config = drm_atomic_helper_set_config, - .page_flip = drm_atomic_helper_page_flip, - .destroy = drm_crtc_cleanup, - .reset = drm_atomic_helper_crtc_reset, - .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, - .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state, - .enable_vblank = enable_vblank, - .disable_vblank = disable_vblank, -}; -static const struct drm_encoder_funcs encoder_funcs = { .destroy = drm_encoder_cleanup }; - -static int atomic_check(struct drm_device *drm, struct drm_atomic_state *state) -{ - struct m3_dcpext_kms *kms = to_kms(drm); - struct drm_crtc_state *crtc; - struct drm_connector_state *connector; - int ret; - - crtc = drm_atomic_get_crtc_state(state, &kms->crtc); - if (IS_ERR(crtc)) - return PTR_ERR(crtc); - if (READ_ONCE(kms->failed) && crtc->active) - return -EIO; - connector = drm_atomic_get_connector_state(state, &kms->connector.base); - if (IS_ERR(connector)) - return PTR_ERR(connector); - ret = drm_atomic_add_affected_planes(state, &kms->crtc); - return ret ?: drm_atomic_helper_check(drm, state); -} -static void commit_tail(struct drm_atomic_state *state) -{ - drm_atomic_helper_commit_modeset_disables(state->dev, state); - drm_atomic_helper_commit_modeset_enables(state->dev, state); - drm_atomic_helper_commit_planes(state->dev, state, DRM_PLANE_COMMIT_ACTIVE_ONLY); - /* present already waited for firmware completion; no timer vblank. */ - drm_atomic_helper_commit_hw_done(state); - drm_atomic_helper_cleanup_planes(state->dev, state); -} -static const struct drm_mode_config_funcs mode_funcs = { - .fb_create = drm_gem_fb_create, - .atomic_check = atomic_check, - .atomic_commit = drm_atomic_helper_commit, -}; -static const struct drm_mode_config_helper_funcs mode_helpers = { .atomic_commit_tail = commit_tail }; -DEFINE_DRM_GEM_FOPS(kms_fops); -static const struct drm_driver kms_driver = { - /* The common DRM syncobj implementation supports imported render fences - * without requiring a render engine on the display device. */ - .driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC | - DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE, - /* Direct scanout needs the pinned SG table. The default shmem ops - * import without mapping and are only suitable for CPU-copy scanout. */ - .gem_prime_import_sg_table = drm_gem_shmem_prime_import_sg_table, - .dumb_create = drm_gem_shmem_dumb_create, - .fops = &kms_fops, - .name = "m3-dcpext", - .desc = "J514S native HDMI", - .major = 0, .minor = 1, -}; - -struct m3_dcpext_kms *m3_dcpext_kms_create(struct device *dev, - const struct m3_dcpext_kms_ops *ops, void *cookie, struct mutex *route_lock, int connector_type) -{ - static const u32 formats[] = { DRM_FORMAT_XRGB8888, DRM_FORMAT_ARGB8888 }; - static const u64 modifiers[] = { DRM_FORMAT_MOD_LINEAR, DRM_FORMAT_MOD_INVALID }; - struct m3_dcpext_kms *kms; - struct drm_device *drm; - int ret; - - if (!ops || !ops->present || !route_lock) - return ERR_PTR(-EINVAL); - kms = devm_drm_dev_alloc(dev, &kms_driver, struct m3_dcpext_kms, drm); - if (IS_ERR(kms)) - return kms; - drm = &kms->drm; - kms->ops = ops; - kms->cookie = cookie; - kms->route_lock = route_lock; - mutex_init(&kms->owner_lock); - ret = drmm_mode_config_init(drm); - if (ret) - return ERR_PTR(ret); - drm->mode_config.min_width = drm->mode_config.min_height = 1; - drm->mode_config.max_width = M3_DCPEXT_MAX_WIDTH; - drm->mode_config.max_height = M3_DCPEXT_MAX_HEIGHT; - drm->mode_config.funcs = &mode_funcs; - drm->mode_config.helper_private = &mode_helpers; - ret = drm_universal_plane_init(drm, &kms->plane, 1, &plane_funcs, formats, - ARRAY_SIZE(formats), modifiers, DRM_PLANE_TYPE_PRIMARY, NULL); - if (ret) - return ERR_PTR(ret); - drm_plane_helper_add(&kms->plane, &plane_helpers); - ret = drm_crtc_init_with_planes(drm, &kms->crtc, &kms->plane, NULL, &crtc_funcs, NULL); - if (ret) - return ERR_PTR(ret); - drm_crtc_helper_add(&kms->crtc, &crtc_helpers); - ret = drm_encoder_init(drm, &kms->encoder, &encoder_funcs, DRM_MODE_ENCODER_TMDS, NULL); - if (ret) - return ERR_PTR(ret); - kms->encoder.possible_crtcs = 1; - ret = m3_dcpext_connector_init(drm, &kms->connector, &kms->encoder, connector_type); - if (ret) - return ERR_PTR(ret); - ret = drm_vblank_init(drm, 1); - if (ret) - return ERR_PTR(ret); - drm_mode_config_reset(drm); - return kms; -} -struct m3_dcpext_connector *m3_dcpext_kms_connector(struct m3_dcpext_kms *kms) -{ - return &kms->connector; -} -int m3_dcpext_kms_register(struct m3_dcpext_kms *kms) -{ - return drm_dev_register(&kms->drm, 0); -} -void m3_dcpext_kms_unplug(struct m3_dcpext_kms *kms) -{ - drm_dev_unplug(&kms->drm); - drm_atomic_helper_shutdown(&kms->drm); - /* Shutdown performs the final disable while the owner is still alive. - * Serialize with any late commit before severing callback references. - * Existing DRM/GEM file references may survive physical device removal. - */ - mutex_lock(&kms->owner_lock); - kms->detached = true; - kms->ops = NULL; - kms->cookie = NULL; - kms->route_lock = NULL; - mutex_unlock(&kms->owner_lock); -} -MODULE_IMPORT_NS("DMA_BUF"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.h deleted file mode 100644 index 88152d0250957..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms.h +++ /dev/null @@ -1,30 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -#ifndef M3_DCPEXT_KMS_H -#define M3_DCPEXT_KMS_H -#include "m3_dcpext_connector.h" -struct device; -struct iosys_map; -struct drm_framebuffer; -struct sg_table; -struct m3_dcpext_kms; - -/* The route mutex serializes callbacks/hotplug and hardware commits. - * For direct scanout, fb/sgt describe pinned GEM pages; the owner retains its - * own framebuffer reference while a DART slot maps them. Otherwise pixels - * are CPU-readable until present returns and copied into retained slots. - * Success requires matching firmware completion. Disable has mode NULL; - * an active mode with fb NULL presents only the background. Neither operation - * may free uncertain DMA resources. */ -struct m3_dcpext_kms_ops { - int (*present)(void *cookie, const struct m3_dcpext_native_mode *mode, - u64 generation, const struct iosys_map *pixels, - u32 source_pitch, bool opaque, - struct drm_framebuffer *fb, struct sg_table *sgt); -}; -struct m3_dcpext_kms *m3_dcpext_kms_create(struct device *dev, - const struct m3_dcpext_kms_ops *ops, void *cookie, - struct mutex *route_lock, int connector_type); -struct m3_dcpext_connector *m3_dcpext_kms_connector(struct m3_dcpext_kms *kms); -int m3_dcpext_kms_register(struct m3_dcpext_kms *kms); -void m3_dcpext_kms_unplug(struct m3_dcpext_kms *kms); -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms_adapter.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms_adapter.h deleted file mode 100644 index 73606a3714868..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_kms_adapter.h +++ /dev/null @@ -1,192 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* M4 native KMS ownership pattern adapted to J514S old RPC protocol. */ -#include -#include -#include "m3_dcpext_kms.h" -static DEFINE_MUTEX(native_route_lock); -static struct m3_dcpext_kms *native_kms; -static bool native_kms_ready,native_kms_finished,native_kms_powered=true; -static bool native_kms_connected; -static unsigned int native_kms_frames,native_kms_slot; -static u32 native_kms_timing,native_kms_color; -static struct drm_framebuffer *native_slot_fb[2], *native_failed_fb; -static unsigned int native_direct_frames; -module_param(native_direct_frames,uint,0400); -/* Completed-frame aggregate costs; read deltas around a bounded animation. */ -static unsigned long long native_copy_ns,native_sync_ns,native_swap_ns; -module_param(native_copy_ns,ullong,0400); -module_param(native_sync_ns,ullong,0400); -module_param(native_swap_ns,ullong,0400); -static u64 native_kms_until,native_kms_generation; -module_param(native_kms_ready,bool,0400); -module_param(native_kms_frames,uint,0400); -static int native_kms_present(void *cookie,const struct m3_dcpext_native_mode *mode, - u64 generation,const struct iosys_map *pixels,u32 source_pitch,bool opaque, - struct drm_framebuffer *fb,struct sg_table *sgt) -{ - u8 surface[0x22c];void *destination;u64 dva,t0,t1,t2,t3;int ret; - lockdep_assert_held(&native_route_lock); - /* Hotplug is an ordinary stale atomic commit, not a poisoned device. */ - if(desktop_disconnected || !native_kms_connected)return mode?-ESTALE:0; - if(!mode){ - if(!native_kms_powered)return 0; - ret=m3_dcpext_native_power(native_client,false); - if(!ret)native_kms_powered=false;return ret; - } - if(native_kms_finished || !native_kms_ready || generation!=native_kms_generation || - generation!=m3_dcpext_native_generation(native_client))return -ESTALE; - u32 width=mode->geometry.width,height=mode->geometry.height,pitch=sgt?source_pitch:ALIGN(width*4,64); - ret=m3_dcpext_surface_validate(width,height,pitch,FRAME_DVA,FRAME_BYTES); - if(ret || (pixels && source_pitchtiming_id!=native_kms_timing || mode->color_id!=native_kms_color){ - if(native_kms_powered){ret=m3_dcpext_native_power(native_client,false);if(ret)return ret;native_kms_powered=false;} - ret=m3_dcpext_native_mode(native_client,mode->color_id,mode->timing_id);if(ret)return ret; - native_kms_timing=mode->timing_id;native_kms_color=mode->color_id; - } - if(!native_kms_powered){ret=m3_dcpext_native_power(native_client,true);if(ret)return ret;native_kms_powered=true;} - if(!fb)return m3_dcpext_native_background(native_client,0xff000000); - /* Initial color bars occupy A. Use B first; alternate only on D589. */ - unsigned int slot=native_kms_slot?0:1; - destination=frame_pixels+slot*FRAME_BYTES; - dva=(FRAME_DVA+slot*FRAME_BYTES)|BIT_ULL(40); - t0=ktime_get_ns(); - if(sgt){ - if(native_slot_fb[slot]!=fb){ - struct drm_framebuffer *old=native_slot_fb[slot]; - drm_framebuffer_get(fb);native_slot_fb[slot]=fb; - ret=frame_map_pages(slot,sgt,fb->offsets[0],(u64)pitch*height); - if(ret){native_failed_fb=old;return ret;} - if(old)drm_framebuffer_put(old); - } - t3=ktime_get_ns();native_sync_ns+=t3-t0; - }else{ - if(native_slot_fb[slot]){ - ret=frame_map_pages(slot,NULL,0,0);if(ret)return ret; - drm_framebuffer_put(native_slot_fb[slot]);native_slot_fb[slot]=NULL; - } - frame_sync(slot,true);t1=ktime_get_ns(); - for(u32 y=0;y -#include -#include -#else -#include -#include -static u32 get_unaligned_le32(const void *p) { u32 v; memcpy(&v, p, 4); return v; } -static u64 get_unaligned_le64(const void *p) { u64 v; memcpy(&v, p, 8); return v; } -#endif - -int m3_dcpext_surface_validate(u32 width, u32 height, u32 pitch, - u64 dva, u64 mapped_bytes) -{ - u64 bytes; - - /* Scope is the qualified external controller's linear SDR path. */ - if (!width || width > M3_DCPEXT_MAX_WIDTH || !height || height > M3_DCPEXT_MAX_HEIGHT || - pitch < width * 4 || (pitch & 63) || !dva || (dva & 0x3fff)) - return -EINVAL; - bytes = (u64)pitch * height; - if (bytes > mapped_bytes || dva >= (1ULL << 36) || - bytes > (1ULL << 36) - dva) - return -ERANGE; - return 0; -} - -static int mode_geometry_validate(const struct m3_dcpext_mode *m, u32 *fps_16_16) -{ - u64 fps, total; - - if (!m || !fps_16_16 || !m->width || - m->width > M3_DCPEXT_MAX_WIDTH || !m->height || m->height > M3_DCPEXT_MAX_HEIGHT || !m->clock_khz || - m->htotal <= m->width || m->vtotal <= m->height || - m->htotal > 65535 || m->vtotal > 65535) - return -EINVAL; - if (m->interlaced || m->doublescan) - return -EOPNOTSUPP; - total = (u64)m->htotal * m->vtotal; - fps = ((u64)m->clock_khz * 1000 * 65536 + total / 2) / total; - if (fps < 24 * 65536 || fps > 120 * 65536) - return -ERANGE; - *fps_16_16 = fps; - return 0; -} - -int m3_dcpext_mode_validate_link(const struct m3_dcpext_mode *m, - u32 payload_kbps, u32 *fps_16_16) -{ - int ret; - - if (!payload_kbps || payload_kbps > M3_DCPEXT_HBR3_4LANE_KBPS) - return -EINVAL; - ret = mode_geometry_validate(m, fps_16_16); - if (ret) - return ret; - return (u64)m->clock_khz * 24 > payload_kbps ? -ERANGE : 0; -} - -int m3_dcpext_native_mode_validate(const struct m3_dcpext_native_mode *m, - u32 payload_kbps, u32 *fps_16_16) -{ - if (!m) - return -EINVAL; - if (!m->dsc) - return m3_dcpext_mode_validate_link(&m->geometry, payload_kbps, fps_16_16); - /* DSC admission is limited to the HDMI HBR3 route. The current firmware - * publication must explicitly require DSC and must NOT mark the color - * unsafe. DCP owns its encoder/PPS and the converter's DSC decompressor. */ - if (payload_kbps != M3_DCPEXT_HBR3_4LANE_KBPS) - return -EOPNOTSUPP; - return mode_geometry_validate(&m->geometry, fps_16_16); -} - -int m3_dcpext_mode_validate(const struct m3_dcpext_mode *m, u32 *fps_16_16) -{ - return m3_dcpext_mode_validate_link(m, M3_DCPEXT_MAX_PAYLOAD_KBPS, fps_16_16); -} - -int m3_dcpext_mode_select(const struct m3_dcpext_mode *m, - const u8 *timings, u32 count, u8 selected[24]) -{ - const u8 *best = NULL; - u64 fps, best_delta = ~(u64)0; - - int ret; - u32 rate; - bool ambiguous = false; - - if (!timings || !selected || count > 64) - return -EINVAL; - ret = m3_dcpext_mode_validate(m, &rate); - if (ret) - return ret; - fps = rate; - for (u32 i = 0; i < count; i++) { - const u8 *r = timings + i * 24; - u32 reported = get_unaligned_le32(r + 12); - u64 delta = fps > reported ? fps - reported : reported - fps; - - if (get_unaligned_le32(r) != 1 || - get_unaligned_le32(r + 4) != m->width || - get_unaligned_le32(r + 8) != m->height || - get_unaligned_le32(r + 16) || get_unaligned_le32(r + 20)) - continue; - /* Captured DMT records use nominal integer rates: e.g. 800x600 - * 60.316 Hz -> 60, 640x480 72.809 Hz -> 72. Prefer the closest - * reported rate; fractional 59.94 and 60 remain distinguishable. - */ - if (delta >= 65536 || delta > best_delta) - continue; - if (delta < best_delta) - ambiguous = false; - else if (best && memcmp(best, r, 24)) - ambiguous = true; - best = r; - best_delta = delta; - } - if (!best) - return -ENODATA; - if (ambiguous) - return -ENOTUNIQ; - memcpy(selected, best, 24); - return 0; -} - -/* OSSerialize tags follow the existing Asahi parser and m1n1 fw/common.py. - * Bound every container and skip, including unknown fields. Keep this parser - * independent of DRM so captured and malformed publications use the same code - * in host sanitizer tests and in the driver. */ -/* Every value consumes at least one four-byte tag. Bound traversal by the - * publication size rather than an unrelated 65536-node ceiling: HDMI sinks - * with many timing/color combinations can legitimately exceed that ceiling. - * The byte, depth and per-container limits remain independently enforced. */ -struct native_value { - u32 type, count, begin, end; -}; -struct native_blob { - const u8 *data; - u32 bytes; -}; - -static int native_value_read(const struct native_blob *b, u32 *pos, - u32 depth, u32 *budget, struct native_value *v) -{ - struct native_value child; - u32 tag, children = 0, payload = 0; - int ret; - - if (depth > 16 || !*budget || *pos > b->bytes) - return -EINVAL; - --*budget; - *pos = (*pos + 3) & ~3U; - if (*pos > b->bytes || b->bytes - *pos < 4) - return -EINVAL; - tag = get_unaligned_le32(b->data + *pos); - *pos += 4; - *v = (struct native_value) { - .type = (tag >> 24) & 31, .count = tag & 0xffffff, - .begin = *pos, - }; - if (tag & 0x60000000) - return -EINVAL; - switch (v->type) { - case 1: - case 2: - if (v->count > 4096) - return -E2BIG; - children = v->count * (v->type == 1 ? 2 : 1); - break; - case 4: - if (v->count != 64) - return -EINVAL; - payload = 8; - break; - case 9: - case 10: - payload = v->count; - break; - case 11: - if (v->count > 1) - return -EINVAL; - break; - default: - return -EINVAL; - } - if (payload > b->bytes - *pos) - return -EINVAL; - *pos += payload; - for (u32 i = 0; i < children; i++) { - ret = native_value_read(b, pos, depth + 1, budget, &child); - if (ret) - return ret; - if (v->type == 1 && !(i & 1) && child.type != 9) - return -EINVAL; - } - v->end = *pos; - return 0; -} - -static int native_member(const struct native_blob *b, - const struct native_value *dict, const char *name, - struct native_value *out) -{ - struct native_value key, value; - u32 pos = dict->begin, budget = b->bytes / 4; - bool found = false; - int ret; - - if (dict->type != 1) - return -EINVAL; - for (u32 i = 0; i < dict->count; i++) { - ret = native_value_read(b, &pos, 0, &budget, &key); - if (!ret) - ret = native_value_read(b, &pos, 0, &budget, &value); - if (ret || key.type != 9) - return -EINVAL; - if (key.count != strlen(name) || - memcmp(b->data + key.begin, name, key.count)) - continue; - if (found) - return -EINVAL; - *out = value; - found = true; - } - return found ? 0 : -ENOENT; -} - -static int native_uint(const struct native_blob *b, const struct native_value *d, - const char *name, u32 *out) -{ - struct native_value v; - u64 number; - int ret = native_member(b, d, name, &v); - - if (ret) - return ret; - if (v.type != 4) - return -EINVAL; - number = get_unaligned_le64(b->data + v.begin); - if (number > 0xffffffffULL) - return -ERANGE; - *out = number; - return 0; -} - -static int native_bool(const struct native_blob *b, const struct native_value *d, - const char *name, bool *out) -{ - struct native_value v; - int ret = native_member(b, d, name, &v); - - if (ret) - return ret; - if (v.type != 11) - return -EINVAL; - *out = v.count; - return 0; -} - -static int native_color_excluded(const struct native_blob *b, - const struct native_value *mode, - const char *name, u32 id) -{ - struct native_value array, value; - u32 pos, budget = b->bytes / 4; - int ret = native_member(b, mode, name, &array); - - if (ret || array.type != 2) - return ret ?: -EINVAL; - pos = array.begin; - for (u32 i = 0; i < array.count; i++) { - ret = native_value_read(b, &pos, 0, &budget, &value); - if (ret || value.type != 4) - return -EINVAL; - if (get_unaligned_le64(b->data + value.begin) == id) - return 1; - } - return 0; -} - -static int native_color(const struct native_blob *b, - const struct native_value *mode, u32 *id, bool allow_dsc, bool *dsc) -{ - struct native_value array, color; - u32 pos, budget = b->bytes / 4, best_score = 0; - bool found = false, best_rgb = false; - int ret = native_member(b, mode, "ColorModes", &array); - - if (ret || array.type != 2) - return -EINVAL; - pos = array.begin; - for (u32 i = 0; i < array.count; i++) { - u32 candidate, score, depth, encoding, eotf, range, colorimetry; - bool virtual, rgb, yuv; - int compressed; - - ret = native_value_read(b, &pos, 0, &budget, &color); - if (ret || native_bool(b, &color, "IsVirtual", &virtual)) - return -EINVAL; - if (virtual) - continue; - if (native_uint(b, &color, "ID", &candidate) || - native_uint(b, &color, "Score", &score) || - native_uint(b, &color, "Depth", &depth) || - native_uint(b, &color, "PixelEncoding", &encoding) || - native_uint(b, &color, "EOTF", &eotf) || - native_uint(b, &color, "Colorimetry", &colorimetry) || - native_uint(b, &color, "DynamicRange", &range)) - return -EINVAL; - /* DCP performs RGB framebuffer -> wire format conversion for A411. - * Enum values match the shared Asahi parser.h. Prefer RGB to retain - * text chroma; use BT.709 limited-range 4:2:2 only when needed. */ - rgb = !encoding && !range && (colorimetry == 10 || colorimetry == 16); - yuv = encoding == 3 && range == 1 && colorimetry == 1; - if (depth != 8 || eotf || (!rgb && !yuv)) - continue; - ret = native_color_excluded(b, mode, "UnsafeColorElementIDs", candidate); - if (ret < 0) - return ret; - if (ret) - continue; - compressed = native_color_excluded(b, mode, "DSCRequiredColorElementIDs", candidate); - if (compressed < 0) - return compressed; - if (compressed && !allow_dsc) - continue; - if (!found || (rgb && !best_rgb) || - (rgb == best_rgb && (score > best_score || (score == best_score && candidate < *id)))) { - *id = candidate; - *dsc = !!compressed; - best_score = score; - best_rgb = rgb; - found = true; - } - } - return found ? 0 : -EOPNOTSUPP; -} - -static int native_dimension(const struct native_blob *b, - const struct native_value *mode, const char *name, - u32 *active, u32 *total, u32 *front, u32 *sync, - bool *positive, u32 *rate) -{ - struct native_value dimension; - u32 back, polarity, repetition; - int ret = native_member(b, mode, name, &dimension); - - if (ret || native_uint(b, &dimension, "Active", active) || - native_uint(b, &dimension, "Total", total) || - native_uint(b, &dimension, "FrontPorch", front) || - native_uint(b, &dimension, "SyncWidth", sync) || - native_uint(b, &dimension, "BackPorch", &back) || - native_uint(b, &dimension, "SyncPolarity", &polarity) || - native_uint(b, &dimension, "PixelRepetition", &repetition) || - native_uint(b, &dimension, "PreciseSyncRate", rate)) - return -EINVAL; - if (!*active || !*sync || !*total || *total > 65535 || - (u64)*active + *front + *sync + back != *total || polarity > 1) - return -EINVAL; - if (repetition) - return -EOPNOTSUPP; - *positive = polarity; - return 0; -} - -/* 14.6 publishes IsSplit but omits pipe counts. Missing counts mean the - * single pipe only for a non-split timing; malformed/present counts still fail. */ -static int native_pipe_count(const struct native_blob *b, - const struct native_value *v, const char *key, - u32 *count) -{ - int ret = native_uint(b, v, key, count); - if (ret == -ENOENT) { *count = 1; return 0; } - return ret; -} - -static int native_mode(const struct native_blob *b, const struct native_value *v, - struct m3_dcpext_native_mode *out, u32 payload_kbps, bool allow_dsc) -{ - struct m3_dcpext_mode *m = &out->geometry; - u32 horizontal_rate, rate, pipes_h, pipes_v, ignored; - bool virtual, interlaced, split; - u64 clock; - int ret; - - memset(out, 0, sizeof(*out)); - if (native_bool(b, v, "IsVirtual", &virtual)) - return -EINVAL; - if (virtual) - return -EOPNOTSUPP; - if (native_bool(b, v, "IsInterlaced", &interlaced) || - native_bool(b, v, "IsSplit", &split) || - native_bool(b, v, "IsPreferred", &out->preferred) || - native_pipe_count(b, v, "HorizontalPipeCount", &pipes_h) || - native_pipe_count(b, v, "VerticalPipeCount", &pipes_v) || - native_uint(b, v, "ID", &out->timing_id)) - return -EINVAL; - if (interlaced || split || pipes_h != 1 || pipes_v != 1) - return -EOPNOTSUPP; - ret = native_dimension(b, v, "HorizontalAttributes", &m->width, - &m->htotal, &out->hfront, &out->hsync, - &out->hpositive, &horizontal_rate); - if (!ret) - ret = native_dimension(b, v, "VerticalAttributes", &m->height, - &m->vtotal, &out->vfront, &out->vsync, - &out->vpositive, &rate); - if (ret) - return ret; - if (rate < 24 * 65536 || rate > 120 * 65536 || - m->width > M3_DCPEXT_MAX_WIDTH || m->height > M3_DCPEXT_MAX_HEIGHT) - return -EOPNOTSUPP; - clock = (u64)m->htotal * m->vtotal * rate; - m->clock_khz = (clock + 32768000) / 65536000; - ret = native_color(b, v, &out->color_id, allow_dsc, &out->dsc); - if (ret) - return ret; - ret = m3_dcpext_native_mode_validate(out, payload_kbps, &ignored); - return ret ? -EOPNOTSUPP : 0; -} - -int m3_dcpext_native_modes_parse_transport(const void *data, u32 bytes, - struct m3_dcpext_native_mode *modes, - u32 capacity, u32 *count, u32 payload_kbps, bool allow_dsc) -{ - struct native_blob blob = { .data = data, .bytes = bytes }; - struct native_value root, value; - struct m3_dcpext_native_mode mode; - u32 pos = 4, budget = bytes / 4, used = 0; - int ret; - - if (!count) - return -EINVAL; - *count = 0; - if (!payload_kbps || payload_kbps > M3_DCPEXT_HBR3_4LANE_KBPS || - (allow_dsc && payload_kbps != M3_DCPEXT_HBR3_4LANE_KBPS) || - !data || !modes || !capacity || capacity > M3_DCPEXT_NATIVE_MAX_MODES || - bytes < 8 || bytes > M3_DCPEXT_MAX_PROPERTY_BYTES || get_unaligned_le32(data) != 0xd3) - return -EINVAL; - ret = native_value_read(&blob, &pos, 0, &budget, &root); - if (ret || root.type != 2 || root.count > M3_DCPEXT_NATIVE_MAX_MODES || - ((pos + 3) & ~3U) != bytes) - return -EINVAL; - pos = root.begin; - budget = bytes / 4; - for (u32 i = 0; i < root.count; i++) { - ret = native_value_read(&blob, &pos, 0, &budget, &value); - if (!ret) - ret = native_mode(&blob, &value, &mode, payload_kbps, allow_dsc); - if (ret == -EOPNOTSUPP) - continue; - if (ret) - return ret; - for (u32 j = 0; j < used; j++) - if (modes[j].timing_id == mode.timing_id) - return -EINVAL; - if (used == capacity) - return -ENOSPC; - modes[used++] = mode; - } - *count = used; - return 0; -} - -int m3_dcpext_native_modes_parse_link(const void *data,u32 bytes, - struct m3_dcpext_native_mode *modes,u32 capacity,u32 *count,u32 payload_kbps) -{return m3_dcpext_native_modes_parse_transport(data,bytes,modes,capacity,count,payload_kbps,false);} - -int m3_dcpext_native_modes_parse(const void *data, u32 bytes, - struct m3_dcpext_native_mode *modes, - u32 capacity, u32 *count) -{ - return m3_dcpext_native_modes_parse_link(data, bytes, modes, capacity, count, - M3_DCPEXT_MAX_PAYLOAD_KBPS); -} - -int m3_dcpext_native_mode_select(const struct m3_dcpext_native_mode *requested, - const struct m3_dcpext_native_mode *modes, - u32 count, u32 *selected) -{ - const struct m3_dcpext_mode *g; - u32 rate, best = 0, best_delta = ~0U; - bool found = false, ambiguous = false; - int ret; - - if (!requested || !modes || !selected || count > M3_DCPEXT_NATIVE_MAX_MODES) - return -EINVAL; - g = &requested->geometry; - ret = mode_geometry_validate(g, &rate); - if (ret) - return ret; - for (u32 i = 0; i < count; i++) { - const struct m3_dcpext_native_mode *m = &modes[i]; - u32 delta, candidate_rate; - - if (m3_dcpext_native_mode_validate(m, M3_DCPEXT_HBR3_4LANE_KBPS, &candidate_rate) || - g->width != m->geometry.width || g->height != m->geometry.height || - g->htotal != m->geometry.htotal || g->vtotal != m->geometry.vtotal || - requested->hfront != m->hfront || requested->hsync != m->hsync || - requested->vfront != m->vfront || requested->vsync != m->vsync || - requested->hpositive != m->hpositive || requested->vpositive != m->vpositive) - continue; - delta = rate > candidate_rate ? rate - candidate_rate : candidate_rate - rate; - /* Native DMT publications round rates (800x600 says 60 rather than - * 60.317 Hz). Keep EDID's exact clock, match all porches/polarities, - * and choose the uniquely closest native rate within one Hz. */ - if (delta >= 65536 || delta > best_delta) - continue; - if (found && delta == best_delta) - ambiguous = true; - else - ambiguous = false; - best = i; - best_delta = delta; - found = true; - } - if (!found) - return -ENODATA; - if (ambiguous) - return -ENOTUNIQ; - *selected = best; - return 0; -} - -int m3_dcpext_native_dimensions(const void *data,u32 bytes,u32 *width_mm,u32 *height_mm) -{ - struct native_blob b={.data=data,.bytes=bytes};struct native_value root; - u32 pos=4,budget=bytes/4,w,h;int ret; - if(!data || bytes<8 || bytes>0x100000 || !width_mm || !height_mm || get_unaligned_le32(data)!=0xd3)return -EINVAL; - ret=native_value_read(&b,&pos,0,&budget,&root); - if(ret || root.type!=1 || ((pos+3)&~3U)!=bytes)return -EINVAL; - if(native_uint(&b,&root,"MaxHorizontalImageSize",&w) || - native_uint(&b,&root,"MaxVerticalImageSize",&h) || !w || !h || w>255 || h>255)return -EINVAL; - *width_mm=w*10;*height_mm=h*10;return 0; -} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.h deleted file mode 100644 index 5266a40dc5243..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_modes.h +++ /dev/null @@ -1,70 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -#ifndef M3_DCPEXT_MODES_H -#define M3_DCPEXT_MODES_H -#ifdef __KERNEL__ -#include -#else -#include -#include -typedef uint8_t u8;typedef uint32_t u32;typedef uint64_t u64; -#endif - -/* Linear SDR scanout allocation and four-lane HBR transport ceiling. */ -#define M3_DCPEXT_MAX_WIDTH 3840 -#define M3_DCPEXT_MAX_HEIGHT 2160 -#define M3_DCPEXT_MAX_PITCH (M3_DCPEXT_MAX_WIDTH * 4) -#define M3_DCPEXT_MAX_PAYLOAD_KBPS 8640000 -#define M3_DCPEXT_HBR2_4LANE_KBPS 17280000 -#define M3_DCPEXT_HBR3_4LANE_KBPS 25920000 - -/* Mode data is supplied by DRM's EDID parser, never invented here. */ -struct m3_dcpext_mode { - u32 width, height, clock_khz, htotal, vtotal; - bool interlaced, doublescan; -}; - -#define M3_DCPEXT_NATIVE_MAX_MODES 128 -/* Total retained-property budget; assembly grows as chunks arrive. */ -#define M3_DCPEXT_MAX_PROPERTY_BYTES 0x800000 -struct m3_dcpext_native_mode { - struct m3_dcpext_mode geometry; - u32 hfront, hsync, vfront, vsync; - u32 color_id, timing_id; - bool hpositive, vpositive, preferred, dsc; -}; - -/* Decode the current native TimingElements publication. Only real, uncompressed - * 8bpc SDR RGB or BT.709 limited-range YCbCr422 modes are admitted. RGB is - * preferred; the bandwidth check conservatively budgets 24 wire bits/pixel. - * No allocation; count is zero on error. IDs belong to this publication only. */ -int m3_dcpext_native_modes_parse(const void *blob, u32 bytes, - struct m3_dcpext_native_mode *modes, - u32 capacity, u32 *count); -/* Explicit route bandwidth; USB-C remains two-lane HBR2. */ -int m3_dcpext_native_modes_parse_link(const void *blob, u32 bytes, - struct m3_dcpext_native_mode *modes, - u32 capacity, u32 *count, u32 payload_kbps); -/* Opt-in DSC transport: current firmware-safe DSC-required color IDs only. */ -int m3_dcpext_native_modes_parse_transport(const void *blob,u32 bytes, - struct m3_dcpext_native_mode *modes,u32 capacity,u32 *count,u32 payload_kbps,bool allow_dsc); -int m3_dcpext_native_mode_validate(const struct m3_dcpext_native_mode *mode, - u32 payload_kbps,u32 *fps_16_16); -int m3_dcpext_mode_validate_link(const struct m3_dcpext_mode *mode, - u32 payload_kbps, u32 *fps_16_16); -/* Match full EDID timing geometry to a uniquely closest native timing. */ -int m3_dcpext_native_mode_select(const struct m3_dcpext_native_mode *requested, - const struct m3_dcpext_native_mode *modes, - u32 count, u32 *selected); - -/* The current hardware policy is four HBR lanes, 8bpc RGB. Mode records are - * the complete firmware-provided 24-byte records, including reserved fields. - * The selected record is returned unchanged. Unknown flags are not stripped. - */ -int m3_dcpext_mode_validate(const struct m3_dcpext_mode *mode, u32 *fps_16_16); -int m3_dcpext_mode_select(const struct m3_dcpext_mode *mode, - const u8 *timings, u32 count, u8 selected[24]); -/* One linear BGRA plane, caller-owned mapping in both external DARTs. */ -int m3_dcpext_surface_validate(u32 width, u32 height, u32 pitch, - u64 dva, u64 mapped_bytes); -int m3_dcpext_native_dimensions(const void *,u32 bytes,u32 *width_mm,u32 *height_mm); -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.c deleted file mode 100644 index a06108fbab3b7..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.c +++ /dev/null @@ -1,812 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Hash-qualified J514S DCP protocol; a single kernel client owns all RPCs. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "m3_dcpext_rpc.h" -#include "m3_dcpext_native.h" -#include "m3_dcpext_modes.h" -#include "m3_dcpext_property.h" - -#define A(n) M3_DCP_TAG('A', n) -#define D(n) M3_DCP_TAG('D', n) -#define MAX_PROPERTIES 256 -#define MAX_RAW_PROPERTIES 32 - -struct m3_dcp_property { - u32 service; - char key[64]; - u64 value; -}; - -struct m3_dcpext_native { - struct device *dev; - struct m3_dcpext_rpc *bridge; - struct mutex lock; - bool failed; - bool opened; - u32 stride; - u64 mode_generation; - u32 diagnostic_plane; - bool diagnostic_flat_surface; - struct m3_dcp_property properties[MAX_PROPERTIES]; - u32 property_count; - struct { u32 service; char key[64]; void *data; u32 size; } raw[MAX_RAW_PROPERTIES]; - u32 raw_count, raw_bytes; - struct m3_property_transfer chunk; - int metadata_error; - u64 completion_count; - u8 completion_last[0x6f0]; - u64 analytics_count; - u8 analytics_last[4164]; -}; - - -static int callback(struct m3_dcpext_rpc *b, void *cookie, u32 tag, - const void *input, u32 in_size, void *output, u32 out_size); - -static int call(struct m3_dcpext_native *dcp, u32 tag, const void *in, u32 ins, - void *out, u32 outs, u32 completion) -{ - int ret; - - if (dcp->failed) - return -EIO; - dev_dbg(dcp->dev, "M3 kernel RPC %#x input=%u output=%u\n", tag, ins, outs); - ret = m3_dcpext_rpc_call(dcp->bridge, tag, in, ins, out, outs, - completion, callback, dcp); - if (ret) { - dcp->failed = true; - dev_err(dcp->dev, "M3 kernel RPC %#x failed: %d; recovery requires reboot\n", tag, ret); - } - return ret; -} - -static int simple_call(struct m3_dcpext_native *dcp, u32 tag, bool input, bool reply, bool expected) -{ - __le32 one = cpu_to_le32(1), result = 0; - int ret = call(dcp, tag, input ? &one : NULL, input ? 4 : 0, - reply ? &result : NULL, reply ? 4 : 0, 0); - - if (!ret && reply && le32_to_cpu(result) != expected) - return -EPROTO; - return ret; -} - -static struct m3_dcp_property *property(struct m3_dcpext_native *dcp, - u32 service, const u8 *key, bool create) -{ - struct m3_dcp_property *p; - u32 i; - - if (!memchr(key, 0, 64)) - return ERR_PTR(-EINVAL); - for (i = 0; i < dcp->property_count; i++) { - p = &dcp->properties[i]; - if (p->service == service && !strcmp(p->key, key)) - return p; - } - if (!create) - return NULL; - if (dcp->property_count == MAX_PROPERTIES) - return ERR_PTR(-ENOSPC); - p = &dcp->properties[dcp->property_count++]; - p->service = service; - strscpy(p->key, key, sizeof(p->key)); - return p; -} - -static int raw_property(struct m3_dcpext_native *dcp, u32 service, const u8 *key, - const void *data, u32 size) -{ - void *copy; - u32 i, old_size = 0; - - if (!memchr(key, 0, 64) || size > M3_DCPEXT_MAX_PROPERTY_BYTES) - return -EINVAL; - for (i = 0; i < dcp->raw_count; i++) - if (dcp->raw[i].service == service && !strcmp(dcp->raw[i].key, key)) - break; - if (i == MAX_RAW_PROPERTIES) - return -ENOSPC; - if (i < dcp->raw_count) - old_size = dcp->raw[i].size; - if (dcp->raw_bytes - old_size + size > SZ_8M) - return -ENOSPC; - copy = kvmemdup(size ? data : "", size ?: 1, GFP_KERNEL); - if (!copy) - return -ENOMEM; - if (i == dcp->raw_count) - dcp->raw_count++; - kvfree(dcp->raw[i].data); - dcp->raw[i].service = service; - strscpy(dcp->raw[i].key, key, 64); - dcp->raw[i].data = copy; - dcp->raw[i].size = size; - dcp->raw_bytes += size - old_size; - if(!service && !strcmp(key,"TimingElements")){dcp->mode_generation++;dcp->metadata_error=0;} - return 0; -} - -static int callback(struct m3_dcpext_rpc *b, void *cookie, u32 tag, - const void *input, u32 in_size, void *output, u32 out_size) -{ - struct m3_dcpext_native *dcp = cookie; - const u8 *in = input; - u8 *out = output; - struct m3_dcp_property *p; - struct m3_dcpext_buffer buffer = {}; - u32 count, offset, service, key_offset; - int ret; - - dev_dbg(dcp->dev, "M3 kernel callback %#x input=%u output=%u\n", tag, in_size, out_size); -#define SHAPE(i, o) (in_size == (i) && out_size == (o)) - if (tag == D(591) && SHAPE(20, 0)) { - /* Old DCP protocol: packed id/bool/intent/width/height, then padding. - * Firmware-generated intent swaps do not retire an A407 framebuffer. - * The first captured M3 notification has intent 15 at 1920x1080. */ - u32 intent = get_unaligned_le32(in + 5); - if (in[4] > 1 || (intent != 6 && intent != 7 && intent != 15)) - return -EOPNOTSUPP; - dev_info(dcp->dev, "M3 intent swap id=%u flag=%u intent=%u size=%ux%u; no submitted buffer retired\n", - get_unaligned_le32(in), in[4], intent, - get_unaligned_le32(in + 9), get_unaligned_le32(in + 13)); - return 0; - } - /* D002 is the external will-power-off notification; no DMA ownership release. */ - if (tag == D(2) && SHAPE(0, 0)) - return 0; - if (tag == D(574) && SHAPE(4, 4)) { - /* M3 TEXT13e810 stores one bool byte; the other three are padding. - * m1n1's matching old protocol names this powerUpDART. */ - if (in[0] > 1) - return -EINVAL; - return m3_dcpext_rpc_dart_power(b, in[0]); - } - if (tag == D(209) && SHAPE(0, 8)) { - put_unaligned_le64(ktime_to_ms(ktime_get_real()), out); - return 0; - } - if (tag == D(596) && SHAPE(0, 4)) - return 0; /* No default framebuffer was allocated by this host. */ - /* J514S 1e5e14/1e5e9c: event, parameter, u32 value, nullable byte. - * The getter copies output[0] back and returns output[4] as a boolean. - * Match the M2 host: no display tiling, and no setter implementation. - */ - if ((tag == D(115) && SHAPE(16, 8)) || - (tag == D(116) && SHAPE(16, 4))) { - if (in[12] > 1) - return -EINVAL; - dev_info(dcp->dev, "M3 tiling callback %#x event=%u parameter=%u value=%u null=%u\n", - tag, get_unaligned_le32(in), get_unaligned_le32(in + 4), - get_unaligned_le32(in + 8), in[12]); - if (tag == D(115)) - out[4] = 1; - return 0; - } - /* The transport zeroes output, so absent optional metadata stays absent. */ - if (tag == D(114) && SHAPE(4164, 4100)) { - /* J514S CoreAnalyticsSendEvent: name[64], serialized dictionary[4096], - * nullable flag. Retain the latest event locally. The dictionary is - * in/out: firmware parses it even on failure, so a zero-filled reply - * is invalid. Return its own serialized dictionary without changes. - * The final u32 is the method status (zero = accepted). - */ - if (!memchr(in, 0, 64) || in[4160] > 1 || - (!in[4160] && in[64] != 'd')) - return -EINVAL; - memcpy(dcp->analytics_last, in, sizeof(dcp->analytics_last)); - dcp->analytics_count++; - if (!in[4160]) - memcpy(out, in + 64, 4096); - put_unaligned_le32(0, out + 4096); - dev_info(dcp->dev, "M3 analytics event %llu: %.64s (retained locally)\n", - dcp->analytics_count, in); - return 0; - } - if (tag == D(589) && SHAPE(0x6f0, 0)) { - memcpy(dcp->completion_last, in, sizeof(dcp->completion_last)); - dcp->completion_count++; - return 0; - } - if ((tag == D(588) && SHAPE(8, 0)) || - (tag == D(598) && SHAPE(0, 0))) - return 0; - if (tag == D(599) && SHAPE(0, 0)) - return simple_call(dcp, A(410), false, true, false); - if ((tag == D(108) || tag == D(109) || tag == D(110) || tag == D(111) || - tag == D(112) || tag == D(113) || tag == D(0) || tag == D(1)) && SHAPE(0, 4)) { - out[0] = 1; - return 0; - } - if (tag == D(576) && SHAPE(88, 76)) { - if (!(in[84] & 1)) - memcpy(out, in + 8, 76); - return 0; - } - if ((tag == D(577) && SHAPE(4, 0)) || (tag == D(300) && SHAPE(16, 0))) - return 0; - if (tag == D(127) && SHAPE(4, 4)) { - ret = m3_property_begin(&dcp->chunk, get_unaligned_le32(in), - M3_DCPEXT_MAX_PROPERTY_BYTES - dcp->raw_bytes); - goto property_reply; - } - if (tag == D(128) && SHAPE(0x1008, 4)) { - ret = m3_property_append(&dcp->chunk, get_unaligned_le32(in + 0x1000), - in, get_unaligned_le32(in + 0x1004)); - goto property_reply; - } - if (tag == D(129) && SHAPE(64, 4)) { - ret = m3_property_complete(&dcp->chunk); - if (!ret && !memchr(in, 0, 64)) - ret = -EINVAL; - if (!ret) - ret = raw_property(dcp, 0, in, dcp->chunk.data, dcp->chunk.size); - if (!ret) - dev_info(dcp->dev, "M3 property %.64s: %u bytes\n", in, dcp->chunk.size); - m3_property_reset(&dcp->chunk); - goto property_reply; - } - if (tag == D(408) && SHAPE(8, 8)) { - count = get_unaligned_le32(in + 4); - if (memcmp(in, "VORP", 4) || count > 1) - return -EINVAL; - /* J514S boot ADT: dispext0 IDs353/414 (indices97/158), - * dispext1 IDs356/416 (indices100/160), both 935MHz/0. */ - put_unaligned_le64(count ? 0 : 935000000, out); - dev_info(dcp->dev, "M3 display clock %u: %llu Hz (external boot ADT)\n", - count, get_unaligned_le64(out)); - return 0; - } - if (tag == D(125) && SHAPE(100, 36)) { - count = get_unaligned_le32(in + 64); - if (count > 8) - return -EINVAL; - memcpy(out, in + 68, count * 4); - return 0; - } - if (tag == D(6) && SHAPE(60, 56)) { - if (!(in[56] & 1)) - memcpy(out, in, 56); - return 0; - } - if ((tag == D(122) || tag == D(123)) && SHAPE(0, 4)) - return 0; - if (tag == D(121) && SHAPE(0, 4)) { - static const struct { u32 tag; bool in, reply, expected; } sequence[] = { - {A(444), false, true, false}, {A(29)}, - {A(466), true}, {A(0), true, true, true}, {A(463), false, true, true}, - }; - for (count = 0; count < ARRAY_SIZE(sequence); count++) { - ret = simple_call(dcp, sequence[count].tag, sequence[count].in, - sequence[count].reply, sequence[count].expected); - if (ret) - return ret; - } - out[0] = 1; - return 0; - } - if (tag == D(101) && SHAPE(0, 4)) { - put_unaligned_le32(dcp->stride, out); - return 0; - } - if (tag == D(3) && SHAPE(4, 60)) { - put_unaligned_le32(1, out + 56); /* Explicitly decline bandwidth service. */ - return 0; - } - if (tag == D(451) && SHAPE(20, 28)) { - u32 alignment = get_unaligned_le32(in + 12); - - dev_info(dcp->dev, "M3 allocation flags=%#x size=%llu alignment=%u null=%u/%u/%u\n", - get_unaligned_le32(in), get_unaligned_le64(in + 4), alignment, - in[16], in[17], in[18]); - if (get_unaligned_le32(in) != 0x703 || !is_power_of_2(alignment) || alignment > SZ_16K || - in[16] || in[17] || in[18]) - return -EINVAL; - buffer.size = get_unaligned_le64(in + 4); - ret = m3_dcpext_rpc_alloc(b, &buffer); - if (ret) - return ret; - if ((buffer.physical | buffer.dva) & (alignment - 1)) - return -EINVAL; - put_unaligned_le64(buffer.physical, out); - put_unaligned_le64(buffer.dva, out + 8); - put_unaligned_le64(buffer.size, out + 16); - put_unaligned_le32(buffer.id, out + 24); - return 0; - } - if (tag == D(201) && SHAPE(12, 20)) { - u64 id = get_unaligned_le64(in); - - if (id > U32_MAX || get_unaligned_le32(in + 8) != 1) - return -EINVAL; - buffer.id = id; - ret = m3_dcpext_rpc_map(b, &buffer); - if (ret) - return ret; - /* Same vaddr/dva/status layout as M2, verified at TEXT 0x18b330. - * The AP virtual-address output is unused by this host. - */ - put_unaligned_le64(0, out); - put_unaligned_le64(buffer.dva, out + 8); - put_unaligned_le32(0, out + 16); - return 0; - } - if (tag == D(454) && SHAPE(4, 4)) { - ret = m3_dcpext_rpc_retire(b, get_unaligned_le32(in)); - if (!ret) - out[0] = 1; - return ret; - } - if (tag == D(582) && SHAPE(8, 4)) - return 0; /* Optional default surface is not allocated by this host. */ - if (tag == D(400) && SHAPE(76, 0xc04)) { - count = get_unaligned_le32(in + 68); - service = get_unaligned_le32(in); - if (count > 0xc00 || (in[72] & 1) || !memchr(in + 4, 0, 64)) - return -EINVAL; - for (offset = 0; offset < dcp->raw_count; offset++) { - if (dcp->raw[offset].service != service || strcmp(dcp->raw[offset].key, in + 4)) - continue; - if (dcp->raw[offset].size > count) - return -ENOSPC; - memcpy(out, dcp->raw[offset].data, dcp->raw[offset].size); - put_unaligned_le32(dcp->raw[offset].size, out + 0xc00); - break; - } - return 0; - } - if (tag == D(401) && SHAPE(80, 12)) { - /* Match the existing M2 mini-LED host's PMUS compatibility - * response. This is a fixed calibration placeholder, NOT a - * measured M3 temperature and never a thermal-guard input. - * Firmware 19a4c4..19a4e8 converts centidegrees to 16.16. - */ - if (!memcmp(in, "SUMP", 4) && - !strncmp(in + 4, "Temperature", 64)) { - put_unaligned_le64(3029, out); - out[8] = 1; - dev_info_ratelimited(dcp->dev, - "M3 PMUS Temperature: M2 compatibility placeholder 3029\n"); - return 0; - } - p = property(dcp, get_unaligned_le32(in), in + 4, false); - if (IS_ERR(p)) - return PTR_ERR(p); - if (p) { - put_unaligned_le64(p->value, out); - out[8] = 1; - } else - dev_info_ratelimited(dcp->dev, - "M3 missing host property service=%#x key=%.64s\n", - get_unaligned_le32(in), in + 4); - return 0; - } - if ((tag == D(413) && SHAPE(4168, 4)) || - ((tag == D(552) || tag == D(561)) && SHAPE(4164, 4)) || - (tag == D(567) && SHAPE(128, 4))) { - key_offset = tag == D(413) ? 4 : 0; - service = key_offset ? get_unaligned_le32(in) : 0; - if (tag != D(567) && (in[key_offset + 4160] & 1)) { - out[0] = 1; - return 0; - } - count = tag == D(567) ? strnlen(in + 64, 64) : 4096; - ret = raw_property(dcp, service, in + key_offset, in + key_offset + 64, count); - if (!ret) - out[0] = 1; - return ret; - } - if ((tag == D(563) && SHAPE(76, 4)) || (tag == D(565) && SHAPE(72, 4)) || - (tag == D(414) && SHAPE(80, 4)) || (tag == D(415) && SHAPE(76, 4)) || - ((tag == D(102) || tag == D(104)) && SHAPE(68, 0))) { - key_offset = (tag == D(414) || tag == D(415)) ? 4 : 0; - service = key_offset ? get_unaligned_le32(in) : 0; - count = (tag == D(563) || tag == D(414)) ? 8 : 4; - if (out_size && (in[key_offset + 64 + count] & 1)) { - out[0] = 1; - return 0; - } - p = property(dcp, service, in + key_offset, true); - if (IS_ERR(p)) - return PTR_ERR(p); - p->value = count == 8 ? get_unaligned_le64(in + key_offset + 64) : - get_unaligned_le32(in + key_offset + 64); - if (tag == D(104)) - p->value = in[64]; - if (out_size) - out[0] = 1; - return 0; - } - if (tag == D(406) && SHAPE(72, 0)) - return 0; - if (tag == D(107) && SHAPE(64, 0)) { - if(!memchr(in,0,64))return -EINVAL; - if(!strcmp(in,"TimingElements"))dcp->mode_generation++; - for(u32 i=0;iraw_count;i++) { - if(dcp->raw[i].service || strcmp(dcp->raw[i].key,in))continue; - dcp->raw_bytes-=dcp->raw[i].size;kvfree(dcp->raw[i].data); - dcp->raw[i]=dcp->raw[--dcp->raw_count]; - memset(&dcp->raw[dcp->raw_count],0,sizeof(dcp->raw[0]));break; - } - p = property(dcp, 0, in, false); - if (IS_ERR(p)) - return PTR_ERR(p); - if (p) { - *p = dcp->properties[--dcp->property_count]; - memset(&dcp->properties[dcp->property_count], 0, sizeof(*p)); - } - return 0; - } - if (tag == D(207) && SHAPE(0, 4)) { - /* HDMI has no panel backlight. Match iomfb_template.c's external - * display path: acknowledge the request without A132, which - * registers a panel backlight service with firmware. - */ - out[0] = 1; - return 0; - } - if (tag == D(206) && SHAPE(0, 4)) { - ret = simple_call(dcp, A(131), false, true, true); - if (!ret) - out[0] = 1; - return ret; - } - if (tag == D(100) && SHAPE(0, 0)) - { - __le32 result; - return call(dcp, A(374), NULL, 0, &result, 4, 0); - } - dev_err(dcp->dev, "M3 unhandled kernel callback %#x %u/%u\n", tag, in_size, out_size); - return -EOPNOTSUPP; -#undef SHAPE -property_reply: - if (ret) { - dev_warn_ratelimited(dcp->dev, "M3 property rejected: %d; keeping RPC alive\n", ret); - dcp->metadata_error = ret; - m3_property_reset(&dcp->chunk); - } - /* D127/D128/D129 return a bool. False rejects data, not the transport. */ - out[0] = !ret; - return 0; - -} - -int m3_dcpext_native_open(struct m3_dcpext_native *dcp) -{ - int ret; - - mutex_lock(&dcp->lock); - if (dcp->opened) { - ret = -EALREADY; - } else { - ret = simple_call(dcp, A(455), false, false, false); - if (!ret) { - dcp->opened = true; - dev_info(dcp->dev, "M3 first-client-open complete\n"); - } - } - mutex_unlock(&dcp->lock); - return ret; -} - -struct m3_dcpext_native *m3_dcpext_native_start(struct device *dev, struct m3_dcpext_rpc *bridge, - bool defer_open) -{ - struct m3_dcpext_native *dcp; - int ret; - - dcp = devm_kzalloc(dev, sizeof(*dcp), GFP_KERNEL); - if (!dcp) - return ERR_PTR(-ENOMEM); - dcp->dev = dev; - dcp->bridge = bridge; - mutex_init(&dcp->lock); - dcp->stride = 7680; - ret = simple_call(dcp, A(401), false, true, true); - if (!ret && !defer_open) - ret = m3_dcpext_native_open(dcp); - if (ret) - return ERR_PTR(ret); - if (defer_open) - dev_info(dev, "M3 DCP kernel startup complete; first-client-open deferred\n"); - else - dev_info(dev, "M3 DCP kernel startup and first-client-open complete\n"); - return dcp; -} - -/* J514S wrappers: A409@13f7d4 (4/4), A472@1421f4 (12/8), - * A411@13f8a8 (8/4). A472's output status is at byte 4; bytes 9/10 - * are additional boolean inputs and byte 11 is the output-pointer null flag. - * Follow M2's display-handle 0, then power-state 1 sequence explicitly. - */ -int m3_dcpext_native_panel(struct m3_dcpext_native *dcp, unsigned int action) -{ - u8 input[80] = {}, output[8] = {}; - int ret; - - mutex_lock(&dcp->lock); - switch (action) { - case 1: - ret = call(dcp, A(409), input, 4, output, 4, 0); - /* Existing internal-display handle 0 returns 2. */ - if (!ret && get_unaligned_le32(output) != 2) - ret = -EIO; - if (!ret) { - put_unaligned_le64(1, input); - ret = call(dcp, A(472), input, 12, output, 8, 0); - if (!ret && get_unaligned_le32(output + 4)) - ret = -EIO; - } - break; - case 2: - /* PreferredTimingElements: color 1, timing 2, 3024x1964. */ - put_unaligned_le32(1, input); - put_unaligned_le32(2, input + 4); - ret = call(dcp, A(411), input, 8, output, 4, 0); - if (!ret && get_unaligned_le32(output)) - ret = -EIO; - break; - case 3: - ret = call(dcp, A(479), input, 4, output, 8, 0); - break; - case 4: - /* Native stub kernel ab5a53c; firmware wrapper 140fe4. - * Same initialization used by the M2 host before client open. - */ - ret = call(dcp, A(448), input, 4, output, 4, 0); - if (!ret && get_unaligned_le32(output)) - ret = -EIO; - break; - case 5: - /* Native stub kernel ab57948; firmware wrapper 13fe2c. - * Location 9 is the identity CTM used by the M2 KMS path. - */ - put_unaligned_le32(9, input); - put_unaligned_le64(1ULL << 32, input + 4); - put_unaligned_le64(1ULL << 32, input + 4 + 4 * 8); - put_unaligned_le64(1ULL << 32, input + 4 + 8 * 8); - ret = call(dcp, A(421), input, 80, output, 4, 0); - if (!ret && get_unaligned_le32(output)) - ret = -EIO; - break; - case 6: - case 7: - /* Reversible diagnostic: M2's linear primary uses IOMFB plane 2. - * The next swap clears the other two primary slots atomically. - */ - dcp->diagnostic_plane = action == 6 ? 2 : 0; - put_unaligned_le32(dcp->diagnostic_plane, output); - ret = 0; - break; - case 8: - case 9: - /* Captured 26A428 linear BGRA uses the non-planar IOSurface - * representation. SurfaceContents fields match the stub driver. - */ - dcp->diagnostic_flat_surface = action == 8; - ret = 0; - break; - default: - ret = -EINVAL; - } - dev_info(dcp->dev, "M3 panel action %u ret=%d output=%#x/%#x\n", - action, ret, get_unaligned_le32(output), get_unaligned_le32(output + 4)); - if (ret) - dcp->failed = true; - mutex_unlock(&dcp->lock); - return ret; -} - -/* External mode IDs come from this session's TimingElements publication. */ -int m3_dcpext_native_mode(struct m3_dcpext_native *dcp, u32 color, u32 timing) -{ - u8 input[8], output[4]; int ret; - put_unaligned_le32(color, input); put_unaligned_le32(timing, input + 4); - mutex_lock(&dcp->lock); - ret = call(dcp, A(411), input, sizeof(input), output, sizeof(output), 0); - if (!ret && get_unaligned_le32(output)) ret = -EIO; - if (ret) dcp->failed = true; - mutex_unlock(&dcp->lock); - return ret; -} -int m3_dcpext_native_power(struct m3_dcpext_native *dcp, bool on) -{ - u8 input[12] = {}, output[8] = {}; int ret; - put_unaligned_le64(on, input); - mutex_lock(&dcp->lock); - ret = call(dcp, A(472), input, sizeof(input), output, sizeof(output), 0); - if (!ret && get_unaligned_le32(output + 4)) ret = -EIO; - if (ret) dcp->failed = true; - mutex_unlock(&dcp->lock); - return ret; -} - -/* Last 32 serialized swaps: ID, begin/A406-ack/completion monotonic us. - * Fixed-size overwrite telemetry; no allocation or logging in the frame loop. */ -static unsigned long long swap_timings[32 * 4]; -static unsigned int swap_timing_index; -module_param_array(swap_timings, ullong, NULL, 0400); -static int native_swap(struct m3_dcpext_native *dcp, const void *surface, u64 dva, - u32 width, u32 height, u32 background) -{ - __le32 start[4] = {}, started[2], result[3]; - u8 *swap; - u32 id; - unsigned int ti; - int ret; - - if (surface && (!dva || !width || width > M3_DCPEXT_MAX_WIDTH || !height || height > M3_DCPEXT_MAX_HEIGHT)) - return -EINVAL; - swap = kzalloc(0x1b58, GFP_KERNEL); - if (!swap) - return -ENOMEM; - mutex_lock(&dcp->lock); - ti = (swap_timing_index++ % 32) * 4; - swap_timings[ti + 1] = ktime_to_us(ktime_get()); - ret = call(dcp, A(406), start, sizeof(start), started, sizeof(started), 0); - if (ret || le32_to_cpu(started[1])) { - ret = ret ?: -EIO; - goto out; - } - id = le32_to_cpu(started[0]); - swap_timings[ti] = id; - swap_timings[ti + 2] = ktime_to_us(ktime_get()); - if (!id) { - ret = -EPROTO; - goto out; - } - put_unaligned_le32(id, swap + 0x98); - put_unaligned_le32(0x80000007, swap + 0x14c); - put_unaligned_le32(0x80000007, swap + 0x150); - /* A disabled plane must blank to black, including compositor DPMS. */ - put_unaligned_le32(background, swap + 0x154); - memset(swap + 0x1b4b, 1, 10); - swap[0x1b56] = swap[0x1b57] = 1; - if (surface) { - u32 plane = dcp->diagnostic_plane; - u8 *s = swap + 0x508 + plane * 0x22c; - - memcpy(s, surface, 0x22c); - if (dcp->diagnostic_flat_surface) { - put_unaligned_le32(0, s + 3); - put_unaligned_le32(0, s + 7); - s[0x14] = 1; /* Captured sRGB BGRA surface. */ - put_unaligned_le16(4, s + 0x19); - memset(s + 0x39, 0, 0x1ed - 0x39); - put_unaligned_le64(1, s + 0x51); - put_unaligned_le64(1, s + 0x149); - } - put_unaligned_le64(dva, swap + 0xdb8 + plane * 8); - put_unaligned_le32(1, swap + 0x9c + plane * 4); - put_unaligned_le32(width, swap + 0xb4 + plane * 16); - put_unaligned_le32(height, swap + 0xb8 + plane * 16); - put_unaligned_le32(width, swap + 0x114 + plane * 16); - put_unaligned_le32(height, swap + 0x118 + plane * 16); - swap[0x1b4b + plane] = 0; - } - ret = call(dcp, A(407), swap, 0x1b58, result, sizeof(result), id); - swap_timings[ti + 3] = ktime_to_us(ktime_get()); - if (!ret && get_unaligned_le32((u8 *)result + 5)) - ret = -EIO; - if (!ret && !surface) - dev_info(dcp->dev, "M3 kernel swap %u completed, surface=%d\n", id, !!surface); -out: - if (ret) - dcp->failed = true; - mutex_unlock(&dcp->lock); - kfree(swap); - return ret; -} - -int m3_dcpext_native_swap(struct m3_dcpext_native *dcp, const void *surface, u64 dva, - u32 width, u32 height) -{ - return native_swap(dcp, surface, dva, width, height, 0xff000000); -} - -int m3_dcpext_native_background(struct m3_dcpext_native *dcp, u32 color) -{ - return native_swap(dcp, NULL, 0, 0, 0, color); -} - -void *m3_dcpext_native_property(struct m3_dcpext_native *dcp, const char *key, u32 *size) -{ - void *copy = NULL; - u32 i; - - mutex_lock(&dcp->lock); - for (i = 0; i < dcp->raw_count; i++) { - if (!dcp->raw[i].service && !strcmp(dcp->raw[i].key, key)) { - *size = dcp->raw[i].size; - copy = kvmemdup(*size ? dcp->raw[i].data : "", *size ?: 1, GFP_KERNEL); - if (!copy) copy = ERR_PTR(-ENOMEM); - break; - } - } - mutex_unlock(&dcp->lock); - return copy; -} - -bool m3_dcpext_native_property_ready(struct m3_dcpext_native *dcp,const char *key) -{ - bool ready = false; - - mutex_lock(&dcp->lock); - for (u32 i = 0; i < dcp->raw_count; i++) { - if (!dcp->raw[i].service && !strcmp(dcp->raw[i].key, key)) { - ready = dcp->raw[i].size != 0; - break; - } - } - mutex_unlock(&dcp->lock); - return ready; -} - -void m3_dcpext_native_analytics_show(struct m3_dcpext_native *dcp, struct seq_file *seq) -{ - mutex_lock(&dcp->lock); - seq_printf(seq, "events %llu\n", dcp->analytics_count); - if (dcp->analytics_count) { - seq_printf(seq, "last_event %.64s\ndictionary_present %u\n", - dcp->analytics_last, !dcp->analytics_last[4160]); - if (!dcp->analytics_last[4160]) - seq_hex_dump(seq, "", DUMP_PREFIX_OFFSET, 16, 1, - dcp->analytics_last + 64, 4096, false); - } - mutex_unlock(&dcp->lock); -} - -void m3_dcpext_native_completion_show(struct m3_dcpext_native *dcp, struct seq_file *seq) -{ - mutex_lock(&dcp->lock); - seq_printf(seq, "callbacks %llu\n", dcp->completion_count); - if (dcp->completion_count) - seq_hex_dump(seq, "", DUMP_PREFIX_OFFSET, 16, 1, - dcp->completion_last, sizeof(dcp->completion_last), false); - mutex_unlock(&dcp->lock); -} - -int m3_dcpext_native_pump(struct m3_dcpext_native *dcp, unsigned long timeout) -{ - int ret; - mutex_lock(&dcp->lock); - ret=m3_dcpext_rpc_pump(dcp->bridge, timeout, callback, dcp); - mutex_unlock(&dcp->lock); - return ret; -} - -void m3_dcpext_native_invalidate_sink(struct m3_dcpext_native *dcp) -{ - mutex_lock(&dcp->lock); - dcp->mode_generation++; - dcp->metadata_error=0; - m3_property_reset(&dcp->chunk); - for(u32 i=0;iraw_count;) { - if(dcp->raw[i].service || - (strcmp(dcp->raw[i].key,"TimingElements") && - strcmp(dcp->raw[i].key,"DisplayAttributes"))){i++;continue;} - dcp->raw_bytes-=dcp->raw[i].size;kvfree(dcp->raw[i].data); - dcp->raw[i]=dcp->raw[--dcp->raw_count]; - memset(&dcp->raw[dcp->raw_count],0,sizeof(dcp->raw[0])); - } - mutex_unlock(&dcp->lock); -} - -u64 m3_dcpext_native_generation(struct m3_dcpext_native *dcp) -{ - u64 generation;mutex_lock(&dcp->lock);generation=dcp->mode_generation;mutex_unlock(&dcp->lock);return generation; -} - -int m3_dcpext_native_metadata_error(struct m3_dcpext_native *dcp) -{ - return READ_ONCE(dcp->metadata_error); -} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.h deleted file mode 100644 index 0431c2c421eda..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native.h +++ /dev/null @@ -1,30 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -#ifndef M3_DCPEXT_NATIVE_H -#define M3_DCPEXT_NATIVE_H - -#define M3_DCP_TAG(c, n) (((u32)(c) << 24) | ((u32)('0' + (n) / 100) << 16) | \ - ((u32)('0' + (n) / 10 % 10) << 8) | (u32)('0' + (n) % 10)) -struct m3_dcpext_native; -struct m3_dcpext_rpc; -struct device; -struct seq_file; -void m3_dcpext_native_completion_show(struct m3_dcpext_native *dcp, struct seq_file *seq); -void m3_dcpext_native_analytics_show(struct m3_dcpext_native *dcp, struct seq_file *seq); -struct m3_dcpext_native *m3_dcpext_native_start(struct device *dev, struct m3_dcpext_rpc *bridge, - bool defer_open); -int m3_dcpext_native_open(struct m3_dcpext_native *dcp); -int m3_dcpext_native_swap(struct m3_dcpext_native *dcp, const void *surface, u64 dva, - u32 width, u32 height); -int m3_dcpext_native_background(struct m3_dcpext_native *dcp, u32 color); -int m3_dcpext_native_panel(struct m3_dcpext_native *dcp, unsigned int action); -/* Caller owns the returned snapshot and must kvfree it. NULL means absent; - * ERR_PTR reports allocation failure. */ -void *m3_dcpext_native_property(struct m3_dcpext_native *dcp, const char *key, u32 *size); -int m3_dcpext_native_metadata_error(struct m3_dcpext_native *); -int m3_dcpext_native_pump(struct m3_dcpext_native *, unsigned long timeout); -int m3_dcpext_native_mode(struct m3_dcpext_native *, u32 color, u32 timing); -int m3_dcpext_native_power(struct m3_dcpext_native *, bool on); -u64 m3_dcpext_native_generation(struct m3_dcpext_native *); -void m3_dcpext_native_invalidate_sink(struct m3_dcpext_native *); -bool m3_dcpext_native_property_ready(struct m3_dcpext_native *,const char *key); -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_adapter.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_adapter.h deleted file mode 100644 index 8035f4b0d64c2..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_native_adapter.h +++ /dev/null @@ -1,175 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -#include "m3_dcpext_rpc.h" -#include "m3_dcpext_native.h" -#include "m3_dcpext_modes.h" -static bool native_frame_done, native_frame_submitted; -static u64 native_frame_until; -static struct m3_dcpext_rpc *native_rpc; -static struct m3_dcpext_native *native_client; -static bool native_started; -static unsigned int received; -static int session_dispatch(u64 endpoint,u64 message); -static int native_receive(u64 message) -{ - if(!native_rpc)return native_query_receive(message); - m3_dcpext_rpc_receive(native_rpc,message);return 0; -} -static int native_poll(void *cookie,unsigned long timeout) -{ - u64 deadline=session_now(NULL)+jiffies_to_msecs(timeout); - int ret; - do { - u64 msg,ep; - ret=hdmi_irq_progress();if(ret)return ret; - ret=dcpext_session_poll(&session);if(ret)return ret; - if(!(readl(session_fifo+0x114)&BIT(17))){ - msg=readq(session_fifo+0x830);ep=readq(session_fifo+0x838);dma_rmb(); - return session_dispatch(ep,msg); - } - usleep_range(100,200); - }while(session_now(NULL)size || info->size>0x100000 || native_buffers_used==ARRAY_SIZE(native_buffers))return -E2BIG; - ret=session_buffer(NULL,0x37,0,info->size,&dva);if(ret)return ret; - info->size=ALIGN(info->size,LEASE_PAGE);info->dva=dva;info->physical=pool_dma+offset; - info->id=++native_buffers_used;native_buffers[info->id-1].info=*info; - return 0; -} -static int native_map(void *cookie,struct m3_dcpext_buffer *info) -{ - if(!pio_published || !info->id || info->id>native_buffers_used || native_buffers[info->id-1].retired)return -EINVAL; - *info=native_buffers[info->id-1].info;return 0; -} -static int native_retire(void *cookie,u32 id) -{ - if(!id || id>native_buffers_used || native_buffers[id-1].retired)return -EINVAL; - native_buffers[id-1].retired=true;return 0; -} -static int native_power(void *cookie,bool on) -{ - if(!pio_published)return -EUCLEAN; - pr_info("m3_dcpext_native: firmware DART power=%u; session vote retained\n",on);return 0; -} -static u8 native_capture[0x800000]; -static u32 native_capture_used; -static int native_record(void *cookie,u32 kind,u64 msg,const void *data,u32 size) -{ - u8 *p=native_capture+native_capture_used; - if(session.runtime)return 0; - if(size>sizeof(native_capture)-native_capture_used-16){ - /* Diagnostics must never break the protocol they observe. */ - pr_warn_once("m3_dcpext: native capture full; retaining prefix, RPC continues\n"); - return 0; - } - put_unaligned_le32(kind,p);put_unaligned_le32(size,p+4);put_unaligned_le64(msg,p+8); - memcpy(p+16,data,size);native_capture_used+=size+16;return 0; -} -static const struct m3_dcpext_rpc_ops native_ops={.send=session_send,.poll=native_poll, - .alloc=native_alloc,.map=native_map,.retire=native_retire,.dart_power=native_power,.record=native_record}; -static void native_surface(u8 *s,bool opaque,u32 width,u32 height,u32 pitch) -{ - memset(s,0,0x22c);s[2]=opaque; - /* Qualified M3 linear BGRA s, plane 0. */ - put_unaligned_le32(1,s+3);put_unaligned_le32(1,s+7); - put_unaligned_le32(0x42475241,s+0xb);s[0x13]=13;s[0x14]=12; - put_unaligned_le32(pitch,s+0x15);put_unaligned_le16(1,s+0x19); - s[0x1b]=s[0x1c]=1; - put_unaligned_le32(width,s+0x21);put_unaligned_le32(height,s+0x25); - put_unaligned_le32(pitch*height,s+0x29);put_unaligned_le32(1,s+0x35); - put_unaligned_le64(1,s+0x51);put_unaligned_le32(width,s+0x59); - put_unaligned_le32(height,s+0x5d);put_unaligned_le32(pitch,s+0x69); - put_unaligned_le32(pitch*height,s+0x6d);put_unaligned_le16(4,s+0x71); - s[0x73]=s[0x74]=1;put_unaligned_le64(1,s+0x149); -} -static int native_kms_publish(void); -static bool native_kms_reconnecting(void); -static int native_kms_unavailable(int error); -static int native_modes_error; -static u64 native_modes_error_generation; -module_param(native_modes_error,int,0400); -static int native_frame_progress(void) -{ - struct m3_dcpext_native_mode *modes; - const struct m3_dcpext_native_mode wanted = { - .geometry = { .width=1920, .height=1080, .clock_khz=148500, .htotal=2200, .vtotal=1125 }, - .hfront=88, .hsync=44, .vfront=4, .vsync=5, .hpositive=true, .vpositive=true, - }; - u32 bytes=0, count=0, selected; void *blob; int ret; - u8 surface[0x22c] = {}; - if (!native_frame || native_frame_done) return 0; - if(desktop_disconnected || desktop_hpd_dirty || desktop_hpd_rpc || desktop_link_stage)return 0; - ret=m3_dcpext_native_metadata_error(native_client); - if(ret)return native_kms_unavailable(ret); - if(native_modes_error){ - if(native_modes_error_generation==m3_dcpext_native_generation(native_client))return 0; - native_modes_error=0;native_frame_submitted=false; - } - if (native_frame_submitted) { - if(native_kms_trial)return native_kms_publish(); - if (session_now(NULL) -#include -#include -#include -#include - -#ifndef DCPEXT_INSTANCE -#define DCPEXT_INSTANCE 0 -#endif - -static struct platform_device *owner; -static struct device *domain; -static int result = -EINPROGRESS; -module_param(result, int, 0400); - -static int __init acquire(void) -{ - struct device_node *node; - u32 version; - int ret; - - if (!of_machine_is_compatible("apple,j514s") || - !of_machine_is_compatible("apple,t6030")) - return -ENODEV; - node = of_find_node_by_path(DCPEXT_INSTANCE ? "dcpext1" : "dcpext0"); - if (!node) - return -ENODEV; - if (of_device_is_available(node) || - of_property_read_u32(node, "apple,j514s-dcpext-reservations", &version) || - version != 1) { - of_node_put(node); - return -EINVAL; - } - owner = platform_device_alloc(DCPEXT_INSTANCE ? "m3-dcpext1-power" : "m3-dcpext-power", PLATFORM_DEVID_NONE); - if (!owner) { - of_node_put(node); - return -ENOMEM; - } - device_set_node(&owner->dev, of_fwnode_handle(node)); - ret = platform_device_add(owner); - if (ret) { - platform_device_put(owner); - of_node_put(node); - return ret; - } - domain = dev_pm_domain_attach_by_id(&owner->dev, 0); - if (IS_ERR_OR_NULL(domain)) { - ret = domain ? PTR_ERR(domain) : -ENODEV; - platform_device_unregister(owner); - of_node_put(node); - return ret; - } - pr_info("M3 DCPEXT%u acquiring CPU/FE/SYS domains via genpd\n", DCPEXT_INSTANCE); - result = pm_runtime_resume_and_get(domain); - pr_info("M3 DCPEXT%u power acquisition result=%d; retained until reboot\n", DCPEXT_INSTANCE, result); - return 0; -} - -module_init(acquire); -MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("J514S guarded external display power-domain acquisition"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_property.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_property.h deleted file mode 100644 index 0a78cd018a8fe..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_property.h +++ /dev/null @@ -1,60 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -#ifndef M3_DCPEXT_PROPERTY_H -#define M3_DCPEXT_PROPERTY_H -#include -#include -#include - -/* Allocate on receipt, not on the firmware's announced length. The budget is - * shared with retained properties; a rejected transfer never becomes an RPC - * transport error. All sizes and offsets are validated before addition. */ -struct m3_property_transfer { - void *data; - u32 size, used, capacity, budget; - bool active; -}; -static void m3_property_reset(struct m3_property_transfer *p) -{ - kvfree(p->data); - *p = (struct m3_property_transfer){}; -} -static int m3_property_begin(struct m3_property_transfer *p, u32 count, u32 budget) -{ - bool interrupted = p->active; - m3_property_reset(p); - if (interrupted || !count) return -EINVAL; - if (count - 1 > budget) return -ENOSPC; - p->size = count - 1; - p->budget = budget; - p->active = true; - return 0; -} -static int m3_property_append(struct m3_property_transfer *p, u32 offset, - const void *data, u32 count) -{ - u32 need, capacity; - void *grown; - if (!p->active || offset != p->used || count > 4096 || !count || - offset > p->size || count > p->size - offset) return -EINVAL; - need = offset + count; - if (need > p->capacity) { - capacity = p->capacity ? p->capacity : 4096; - while (capacity < need) { - if (capacity > p->budget / 2) { capacity = p->budget; break; } - capacity *= 2; - } - if (capacity > p->budget) capacity = p->budget; - grown = kvrealloc(p->data, capacity, GFP_KERNEL); - if (!grown) return -ENOMEM; - p->data = grown; - p->capacity = capacity; - } - memcpy(p->data + offset, data, count); - p->used = need; - return 0; -} -static int m3_property_complete(const struct m3_property_transfer *p) -{ - return p->active && p->used == p->size ? 0 : -EINVAL; -} -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_regs.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_regs.c deleted file mode 100644 index 9b1bf39519da0..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_regs.c +++ /dev/null @@ -1,123 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* J514S reservation/power audit. Explicit load under the M3 watchdog guard. - * No power, DART, mailbox or PHY writes. Read DART only if the relevant - * real PMGR domains are on. no_ps ADT entries have no register to inspect. - */ -#include -#include -#include -#include -#include -#include -#include - -static unsigned int port; -module_param(port, uint, 0400); -static bool read_dart; -module_param(read_dart, bool, 0400); -MODULE_PARM_DESC(read_dart, "Inspect powered DART after validating reservations"); -static bool read_cpu; -module_param(read_cpu, bool, 0400); - -static int __init audit(void) -{ - const u32 power[2][3] = {{0x3b8, 0x3d8, 0x3e0}, {0x3c0, 0x3f0, 0x3f8}}; - const phys_addr_t coproc[] = {0x2d2c00000ULL, 0x2d6c00000ULL}; - const phys_addr_t dart[] = {0x2d130c000ULL, 0x2d530c000ULL}; - const phys_addr_t scanout[] = {0x2d1304000ULL, 0x2d5304000ULL}; - struct device_node *node; - struct regmap *pmgr; - struct resource resource; - char alias[16]; - u32 marker, value; - bool powered = true; - int ret; - - if (port > 1 || !of_machine_is_compatible("apple,j514s") || - !of_machine_is_compatible("apple,t6030")) - return -ENODEV; - snprintf(alias, sizeof(alias), "dcpext%u", port); - node = of_find_node_by_path(alias); - if (!node) - return -ENODEV; - ret = of_property_read_u32(node, "apple,j514s-dcpext-reservations", &marker); - if (ret || marker != 1 || of_device_is_available(node) || - !of_device_is_compatible(node, "apple,t6030-dcpext") || - of_address_to_resource(node, 0, &resource) || resource.start != coproc[port] || - of_count_phandle_with_args(node, "memory-region", NULL) != 7) { - of_node_put(node); - return -EINVAL; - } - for (unsigned int i = 0; i < 7; i++) { - struct device_node *memory = of_parse_phandle(node, "memory-region", i); - - if (!memory || !of_property_read_bool(memory, "no-map") || - of_address_to_resource(memory, 0, &resource)) { - of_node_put(memory); - of_node_put(node); - return -EINVAL; - } - pr_info("M3 DCPEXT%u protected region %u: %pr\n", port, i, &resource); - of_node_put(memory); - } - of_node_put(node); - node = of_find_node_by_path("/soc/power-management@350700000"); - if (!node) - return -ENODEV; - pmgr = syscon_node_to_regmap(node); - of_node_put(node); - if (IS_ERR(pmgr)) - return PTR_ERR(pmgr); - for (unsigned int i = 0; i < 3; i++) { - ret = regmap_read(pmgr, power[port][i], &value); - if (ret) - return ret; - pr_info("M3 DCPEXT%u PMGR +%#x = %#x\n", port, power[port][i], value); - /* CPU auto-gating is normal on T6030; require the parent - * SYS/FE fabric to remain active before optional ASC reads. - */ - if ((value & 0xff) != 0xff || - value & (BIT(31) | BIT(12) | BIT(11) | BIT(10)) || - (i < 2 && (value & BIT(28)))) - powered = false; - } - if ((!read_dart && !read_cpu) || !powered) { - pr_info("M3 DCPEXT%u no DART reads: requested=%u powered=%u\n", - port, read_dart, powered); - return 0; - } - if (read_cpu) { - void __iomem *regs = ioremap(coproc[port], 0x4000); - - if (!regs) - return -ENOMEM; - pr_info("M3 DCPEXT%u reading ASC control/status\n", port); - msleep(20); - pr_info("M3 DCPEXT%u ASC control=%#x status=%#x\n", - port, readl(regs + 0x44), readl(regs + 0x48)); - iounmap(regs); - } - for (unsigned int i = 0; read_dart && i < 3; i++) { - unsigned int sid = i == 0 ? 5 : i == 1 ? 0 : 4; - phys_addr_t base = i == 0 ? dart[port] : scanout[port]; - const unsigned int offsets[] = {0x200, 0x1000 + sid * 4, 0x1400 + sid * 4}; - void __iomem *regs = ioremap(base, 0x4000); - - if (!regs) - return -ENOMEM; - for (unsigned int n = 0; n < ARRAY_SIZE(offsets); n++) { - pr_info("M3 DCPEXT%u reading %pap + %#x\n", port, &base, offsets[n]); - msleep(20); - pr_info("M3 DCPEXT%u DART %pap + %#x = %#x\n", - port, &base, offsets[n], readl(regs + offsets[n])); - } - iounmap(regs); - } - return 0; -} - -static void __exit done(void) {} -module_init(audit); -module_exit(done); -MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("J514S external display reservation and scalar register audit"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rpc.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rpc.c deleted file mode 100644 index 87eb859dac115..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rpc.c +++ /dev/null @@ -1,604 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* M4 transport snapshot bce0fd0d685485 adapted for M3 D589 completion - * and D201 20-byte output. M3 external startup remains a bounded probe. */ -#include -#include -#include -#include -#include -#include -#include "../../dcp-link.h" -#include "m3_dcpext_rpc.h" -#define MAX_EVENTS 32 -#define MAX_CALLBACKS 8 -struct record_header { - __le32 kind; - __le32 size; - __le64 message; -}; - -struct bridge_event { - struct list_head list; - struct record_header header; - u8 data[]; -}; - -struct m3_dcpext_rpc { - struct device *dev; - const struct m3_dcpext_rpc_ops *ops; - void *cookie; - void *rpc; - struct mutex lock; - struct list_head events; - unsigned int event_count, callback_count; - bool failed; - bool enabled; - u32 kernel_callback_depth; - struct { - bool active; - bool remote; - u32 offset; - u32 size; - struct apple_dcp_link_rpc_header header; - } calls[2]; - struct { - u64 message; - void *output; - u32 size; - u32 call; - u64 allocation_size; - u64 mapping_id; - } callbacks[MAX_CALLBACKS]; -}; - -static void fail(struct m3_dcpext_rpc *b) -{ - b->failed = true; - dev_err(b->dev, - "DCP bridge stopped; retain firmware buffers until reboot\n"); -} - -static void event(struct m3_dcpext_rpc *b, u32 kind, u64 message, - const void *data, u32 size) -{ - struct bridge_event *e; - - if (b->ops->record && - b->ops->record(b->cookie, kind, message, data, size)) { - fail(b); - return; - } - if (b->event_count >= MAX_EVENTS || - size > APPLE_DCP_LINK_STREAM_BUFFER_SIZE) { - fail(b); - return; - } - e = kmalloc(sizeof(*e) + size, GFP_KERNEL); - if (!e) { - fail(b); - return; - } - e->header = (struct record_header){ - .kind = cpu_to_le32(kind), - .size = cpu_to_le32(size), - .message = cpu_to_le64(message), - }; - if (size) - memcpy(e->data, data, size); - list_add_tail(&e->list, &b->events); - b->event_count++; -} - -void m3_dcpext_rpc_crashed(struct m3_dcpext_rpc *b) -{ - mutex_lock(&b->lock); - fail(b); - mutex_unlock(&b->lock); -} - -void m3_dcpext_rpc_receive(struct m3_dcpext_rpc *b, u64 message) -{ - struct apple_dcp_link_stream_layout layout; - struct apple_dcp_link_rpc_view view; - u8 stream = apple_dcp_link_message_stream(message); - bool remote = apple_dcp_link_message_is_remote(message); - bool nested; - int ret; - - mutex_lock(&b->lock); - if (b->failed) - goto out; - if ((message & 3) != APPLE_DCP_LINK_MSG_RPC) { - event(b, 3, message, NULL, 0); - goto out; - } - dma_rmb(); - if (message & BIT_ULL(6)) { - nested = b->calls[1].active; - if (stream || !b->calls[nested].active || - b->calls[nested].remote != remote || - apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, 0, - remote, &layout)) - goto error; - if (memcmp(b->rpc + layout.local_offset + - b->calls[nested].offset, - &b->calls[nested].header, - sizeof(b->calls[nested].header))) - goto error; - event(b, 1, message, - b->rpc + layout.local_offset + b->calls[nested].offset, - b->calls[nested].size); - b->calls[nested].active = false; - goto out; - } - if (b->callback_count >= MAX_CALLBACKS || - apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, stream, remote, - &layout)) - goto error; - ret = apple_dcp_link_rpc_decode(message, b->rpc + layout.remote_offset, - layout.capacity, &view); - if (ret) - goto error; - if ((view.call >> 24) != 'D') { - u32 off; - - /* Preserve stream-placement evidence without acknowledging bad data. */ - for (off = 0; off < APPLE_DCP_LINK_HEAP_OFFSET; - off += APPLE_DCP_LINK_STREAM_BUFFER_SIZE) { - const __le32 *h = b->rpc + off; - - dev_err(b->dev, - "M3 RPC header offset=%#x words=%#x/%#x/%#x\n", - off, le32_to_cpu(h[0]), le32_to_cpu(h[1]), - le32_to_cpu(h[2])); - } - goto error; - } - b->callbacks[b->callback_count].message = message; - b->callbacks[b->callback_count].output = view.output; - b->callbacks[b->callback_count].size = view.output_size; - b->callbacks[b->callback_count].call = view.call; - b->callbacks[b->callback_count].allocation_size = 0; - b->callbacks[b->callback_count].mapping_id = 0; - if (view.call == 0x44343531 && view.input_size == 20 && - view.output_size == 28) { - __le64 allocation_size; - - memcpy(&allocation_size, view.input + 4, - sizeof(allocation_size)); - b->callbacks[b->callback_count].allocation_size = - le64_to_cpu(allocation_size); - } - if (view.call == 0x44323031 && view.input_size == 12 && - view.output_size == 20) { - __le64 mapping_id; - - memcpy(&mapping_id, view.input, sizeof(mapping_id)); - b->callbacks[b->callback_count].mapping_id = - le64_to_cpu(mapping_id); - } - b->callback_count++; - dev_dbg(b->dev, "M3 DCP callback tag=%#x input=%u output=%u\n", - view.call, view.input_size, view.output_size); - event(b, 2, message, b->rpc + layout.remote_offset + view.offset, - view.total_size); - goto out; -error: - fail(b); -out: - mutex_unlock(&b->lock); -} - -static ssize_t bridge_request(struct m3_dcpext_rpc *b, - struct record_header *record, size_t count) -{ - struct apple_dcp_link_stream_layout layout; - struct apple_dcp_link_rpc_header *rpc; - u64 message, control; - u32 kind, size, total, offset = 0; - bool nested, remote = false; - int ret; - - kind = le32_to_cpu(record->kind); - size = le32_to_cpu(record->size); - control = le64_to_cpu(record->message); - ret = -EINVAL; - if (size != count - sizeof(*record)) - goto free_record; - mutex_lock(&b->lock); - if (!b->enabled) { - ret = -EAGAIN; - goto out; - } - if (b->failed) { - ret = -EIO; - goto out; - } - if (kind == 2) { - unsigned int top; - - if (!b->callback_count) - goto out; - top = b->callback_count - 1; - if (control != b->callbacks[top].message || - size != b->callbacks[top].size) - goto out; - if (b->ops->record && - b->ops->record(b->cookie, 4, control, record + 1, size)) { - ret = -ENOSPC; - fail(b); - goto out; - } - memcpy(b->callbacks[top].output, record + 1, size); - dma_wmb(); - ret = b->ops->send(b->cookie, APPLE_DCP_LINK_ENDPOINT, - apple_dcp_link_rpc_reply(control)); - if (ret) - fail(b); - else - b->callback_count--; - goto out; - } - if (kind != 1 || control > 1 || size < sizeof(*rpc)) - goto out; - nested = control; - if (b->calls[nested].active || (nested && !b->callback_count) || - (!nested && b->callback_count)) { - ret = -EBUSY; - goto out; - } - if (nested) { - u64 callback = b->callbacks[b->callback_count - 1].message; - - if (apple_dcp_link_message_stream(callback)) { - ret = -EOPNOTSUPP; - goto out; - } - /* Re-entry uses the callback's stream without the new-stream IRQ flag. */ - remote = apple_dcp_link_message_is_remote(callback); - if (b->calls[0].active && b->calls[0].remote == remote) - offset = ALIGN(b->calls[0].size, 64); - } - rpc = (void *)(record + 1); - ret = apple_dcp_link_rpc_payload_size(le32_to_cpu(rpc->input_size), - le32_to_cpu(rpc->output_size), - &total); - if (ret || total != size) { - ret = -EMSGSIZE; - goto out; - } - ret = apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, 0, remote, - &layout); - if (ret) - goto out; - if (offset + size > layout.capacity) { - ret = -EMSGSIZE; - goto out; - } - if (b->ops->record && - b->ops->record(b->cookie, 3, control, rpc, size)) { - ret = -ENOSPC; - fail(b); - goto out; - } - memcpy(b->rpc + layout.local_offset + offset, rpc, size); - memset(b->rpc + layout.local_offset + offset + sizeof(*rpc) + - le32_to_cpu(rpc->input_size), - 0, le32_to_cpu(rpc->output_size)); - b->calls[nested].header = *rpc; - b->calls[nested].size = size; - b->calls[nested].offset = offset; - b->calls[nested].remote = remote; - b->calls[nested].active = true; - /* M3 bit 9 requires a free stream; it is not a recursive-call marker. */ - apple_dcp_link_rpc_message(offset, size, false, &message); - apple_dcp_link_stream_message(message, 0, remote, &message); - dma_wmb(); - ret = b->ops->send(b->cookie, APPLE_DCP_LINK_ENDPOINT, message); - if (ret) - fail(b); -out: - mutex_unlock(&b->lock); -free_record: - kfree(record); - return ret ? ret : count; -} - -struct m3_dcpext_rpc *m3_dcpext_rpc_create(struct device *dev, void *rpc, - const struct m3_dcpext_rpc_ops *ops, - void *cookie) -{ - struct m3_dcpext_rpc *b; - if (!dev || !rpc || !ops || !ops->send || !ops->poll) - return ERR_PTR(-EINVAL); - b = kzalloc(sizeof(*b), GFP_KERNEL); - if (!b) - return ERR_PTR(-ENOMEM); - b->dev = dev; - b->rpc = rpc; - b->ops = ops; - b->cookie = cookie; - b->enabled = true; - mutex_init(&b->lock); - INIT_LIST_HEAD(&b->events); - return b; -} -void m3_dcpext_rpc_destroy(struct m3_dcpext_rpc *b) -{ - struct bridge_event *e, *next; - list_for_each_entry_safe(e, next, &b->events, list) { - list_del(&e->list); - kfree(e); - } - kfree(b); -} -static struct bridge_event *kernel_event(struct m3_dcpext_rpc *b, - unsigned long timeout) -{ - struct bridge_event *e; - long ret; - - unsigned long deadline = jiffies + timeout; - - while (!READ_ONCE(b->event_count) && !READ_ONCE(b->failed)) { - long remaining = deadline - jiffies; - - if (!timeout || remaining <= 0) - return ERR_PTR(-ETIMEDOUT); - ret = b->ops->poll(b->cookie, remaining); - if (ret) - return ERR_PTR(ret); - } - mutex_lock(&b->lock); - if (b->failed || list_empty(&b->events)) { - mutex_unlock(&b->lock); - return ERR_PTR(-EIO); - } - e = list_first_entry(&b->events, struct bridge_event, list); - list_del(&e->list); - b->event_count--; - mutex_unlock(&b->lock); - return e; -} - -static int kernel_send(struct m3_dcpext_rpc *b, u32 kind, u64 message, - const void *data, u32 size) -{ - struct record_header *record; - ssize_t ret; - - if (size > APPLE_DCP_LINK_STREAM_BUFFER_SIZE) - return -EMSGSIZE; - record = kmalloc(sizeof(*record) + size, GFP_KERNEL); - if (!record) - return -ENOMEM; - *record = (struct record_header){ - .kind = cpu_to_le32(kind), - .size = cpu_to_le32(size), - .message = cpu_to_le64(message), - }; - if (size) - memcpy(record + 1, data, size); - ret = bridge_request(b, record, sizeof(*record) + size); - return ret < 0 ? ret : 0; -} - -static int kernel_callback(struct m3_dcpext_rpc *b, struct bridge_event *e, - m3_dcpext_callback_fn callback, void *cookie, - u32 *completed) -{ - struct apple_dcp_link_rpc_header *h = (void *)e->data; - u32 size = le32_to_cpu(e->header.size), in, out, tag; - int ret; - - if (le32_to_cpu(e->header.kind) != 2 || size < sizeof(*h)) - return -EPROTO; - in = le32_to_cpu(h->input_size); - out = le32_to_cpu(h->output_size); - tag = le32_to_cpu(h->call); - if (in > size - sizeof(*h) || out != size - sizeof(*h) - in || - !callback) - return -EPROTO; - memset(e->data + sizeof(*h) + in, 0, out); - b->kernel_callback_depth++; - ret = callback(b, cookie, tag, h + 1, in, e->data + sizeof(*h) + in, - out); - b->kernel_callback_depth--; - if (ret) - return ret; - if (tag == 0x44353839 && in == 0x6f0 && completed) - *completed = le32_to_cpup((__le32 *)(h + 1)); - return kernel_send(b, 2, le64_to_cpu(e->header.message), - e->data + sizeof(*h) + in, out); -} - -int m3_dcpext_rpc_pump(struct m3_dcpext_rpc *b, unsigned long timeout, - m3_dcpext_callback_fn callback, void *cookie) -{ - struct bridge_event *e; - int ret; - - e = kernel_event(b, timeout); - if (IS_ERR(e)) - return PTR_ERR(e); - ret = kernel_callback(b, e, callback, cookie, NULL); - kfree(e); - return ret; -} - -static int rpc_call(struct m3_dcpext_rpc *b, u32 tag, const void *input, - u32 input_size, void *output, u32 output_size, - u32 completion_id, m3_dcpext_callback_fn callback, - void *cookie, bool (*valid)(void *), bool *rejected) -{ - struct apple_dcp_link_rpc_header *packet; - unsigned long deadline = jiffies + 5 * HZ; - u32 size, completed = 0; - u64 expected = 0x42; - bool nested, replied = false; - int ret; - - ret = apple_dcp_link_rpc_payload_size(input_size, output_size, &size); - if (ret || size > APPLE_DCP_LINK_STREAM_BUFFER_SIZE) - return -EMSGSIZE; - packet = kzalloc(size, GFP_KERNEL); - if (!packet) - return -ENOMEM; - apple_dcp_link_rpc_header_encode(packet, tag, input_size, output_size); - if (input_size) - memcpy(packet + 1, input, input_size); -retry: - /* Drain asynchronous notifications before starting an outer request. - * A queued callback is not recursive entry until its handler is running. - */ - if (!b->kernel_callback_depth) { - while (READ_ONCE(b->event_count)) { - if (time_after_eq(jiffies, deadline)) { - kfree(packet); - return -ETIMEDOUT; - } - ret = m3_dcpext_rpc_pump(b, 0, callback, cookie); - if (ret) { - kfree(packet); - return ret; - } - } - } - mutex_lock(&b->lock); - nested = b->kernel_callback_depth != 0; - if (nested && !b->callback_count) { - mutex_unlock(&b->lock); - kfree(packet); - return -EPROTO; - } - if (nested) - expected |= b->callbacks[b->callback_count - 1].message & - BIT_ULL(8); - mutex_unlock(&b->lock); - if (valid && !valid(cookie)) { - *rejected = true; - kfree(packet); - return -ESTALE; - } - ret = kernel_send(b, 1, nested, packet, size); - if (ret == -EBUSY && !nested && time_before(jiffies, deadline)) - goto retry; /* A callback arrived between draining and submitting. */ - kfree(packet); - if (ret) - return ret; - while (time_before(jiffies, deadline)) { - long remaining = deadline - jiffies; - struct bridge_event *e = kernel_event(b, max(remaining, 0L)); - struct apple_dcp_link_rpc_header *h; - - if (IS_ERR(e)) - return PTR_ERR(e); - h = (void *)e->data; - if (le32_to_cpu(e->header.kind) == 1) { - if (replied || le32_to_cpu(e->header.size) != size || - le64_to_cpu(e->header.message) != expected || - le32_to_cpu(h->call) != tag || - le32_to_cpu(h->input_size) != input_size || - le32_to_cpu(h->output_size) != output_size) { - ret = -EPROTO; - } else { - if (output_size) - memcpy(output, - e->data + sizeof(*h) + - input_size, - output_size); - replied = true; - } - } else { - ret = kernel_callback(b, e, callback, cookie, - &completed); - if (completion_id && completed && - completed != completion_id) - ret = -EPROTO; - } - kfree(e); - if (ret) - return ret; - if (replied && (!completion_id || completed == completion_id)) - return 0; - } - return -ETIMEDOUT; -} - -int m3_dcpext_rpc_alloc(struct m3_dcpext_rpc *b, struct m3_dcpext_buffer *info) -{ - u32 top; - int ret = -EINVAL; - mutex_lock(&b->lock); - if (!b->failed && b->callback_count && b->ops->alloc) { - top = b->callback_count - 1; - if (b->callbacks[top].call == 0x44343531 && info->size && - b->callbacks[top].allocation_size == info->size && - !info->id && !info->flags && !info->dva && - !info->physical) { - ret = b->ops->alloc(b->cookie, info); - if (!ret) - b->callbacks[top].allocation_size = 0; - } - } - mutex_unlock(&b->lock); - return ret; -} -int m3_dcpext_rpc_map(struct m3_dcpext_rpc *b, struct m3_dcpext_buffer *info) -{ - u32 top; - int ret = -EINVAL; - mutex_lock(&b->lock); - if (!b->failed && b->callback_count && b->ops->map) { - top = b->callback_count - 1; - if (b->callbacks[top].call == 0x44323031 && info->id && - b->callbacks[top].mapping_id == info->id && !info->flags && - !info->size && !info->dva && !info->physical) - ret = b->ops->map(b->cookie, info); - } - mutex_unlock(&b->lock); - return ret; -} -int m3_dcpext_rpc_dart_power(struct m3_dcpext_rpc *b, bool on) -{ - int ret = -EOPNOTSUPP; - mutex_lock(&b->lock); - if (!b->failed && b->callback_count && b->ops->dart_power && - b->callbacks[b->callback_count - 1].call == 0x44353734) - ret = b->ops->dart_power(b->cookie, on); - mutex_unlock(&b->lock); - return ret; -} - -int m3_dcpext_rpc_retire(struct m3_dcpext_rpc *b, u32 id) -{ - if (!id || !b->ops->retire) - return -EINVAL; - return b->ops->retire(b->cookie, id); -} - -int m3_dcpext_rpc_call(struct m3_dcpext_rpc *b, u32 tag, const void *input, - u32 input_size, void *output, u32 output_size, - u32 completion_id, m3_dcpext_callback_fn callback, - void *cookie) -{ - return m3_dcpext_rpc_call_checked(b, tag, input, input_size, output, - output_size, completion_id, callback, cookie, NULL); -} - -int m3_dcpext_rpc_call_checked(struct m3_dcpext_rpc *b, u32 tag, const void *input, - u32 input_size, void *output, u32 output_size, - u32 completion_id, m3_dcpext_callback_fn callback, - void *cookie, bool (*valid)(void *)) -{ - bool rejected = false; - int ret; - - if ((input_size && !input) || (output_size && !output)) - return -EINVAL; - ret = rpc_call(b, tag, input, input_size, output, output_size, - completion_id, callback, cookie, valid, &rejected); - if (ret && !rejected) - m3_dcpext_rpc_crashed(b); - return ret; -} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rpc.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rpc.h deleted file mode 100644 index 6fd2064fd2b01..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rpc.h +++ /dev/null @@ -1,54 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -#ifndef M3_DCPEXT_RPC_H -#define M3_DCPEXT_RPC_H -#include -struct device; -struct m3_dcpext_rpc; -struct m3_dcpext_buffer { - u64 size, physical, dva; - u32 id, flags; -}; -/* Owner retains RPC storage and every DMA allocation until qualified firmware - * quiescence, including failures. poll dispatches mailbox messages back into - * receive; it must honor the owner's absolute deadline/watchdog/thermal gates. - * Calls/pump are serialized by the owner; receive may run on another worker. - * send must not synchronously reenter receive (the RPC mutex is held). - */ -struct m3_dcpext_rpc_ops { - int (*send)(void *cookie, u8 endpoint, u64 message); - int (*poll)(void *cookie, unsigned long timeout); - int (*alloc)(void *cookie, struct m3_dcpext_buffer *buffer); - int (*map)(void *cookie, struct m3_dcpext_buffer *buffer); - int (*retire)(void *cookie, u32 id); - /* Firmware power vote; owner may retain an independent session vote. */ - int (*dart_power)(void *cookie, bool on); - /* Optional bounded diagnostic sink: kind 1 reply, 2 callback, - * 3 host request, 4 callback reply. Failure stops transport. */ - int (*record)(void *cookie, u32 kind, u64 message, const void *data, - u32 size); -}; -typedef int (*m3_dcpext_callback_fn)(struct m3_dcpext_rpc *, void *, u32, - const void *, u32, void *, u32); -struct m3_dcpext_rpc *m3_dcpext_rpc_create(struct device *, void *memory, - const struct m3_dcpext_rpc_ops *, - void *cookie); -void m3_dcpext_rpc_receive(struct m3_dcpext_rpc *, u64 message); -void m3_dcpext_rpc_crashed(struct m3_dcpext_rpc *); -/* Only after all callbacks/workers stop and firmware quiescence is proven. */ -void m3_dcpext_rpc_destroy(struct m3_dcpext_rpc *); -int m3_dcpext_rpc_call(struct m3_dcpext_rpc *, u32 tag, const void *, u32, - void *, u32, u32 completion_id, m3_dcpext_callback_fn, - void *); -/* Check owner state after draining callbacks and before each send attempt. - * The owner serializes state changes with calls/pump. A false check returns - * -ESTALE without sending this request or poisoning the transport. */ -int m3_dcpext_rpc_call_checked(struct m3_dcpext_rpc *, u32 tag, const void *, u32, - void *, u32, u32 completion_id, - m3_dcpext_callback_fn, void *, bool (*valid)(void *)); -int m3_dcpext_rpc_pump(struct m3_dcpext_rpc *, unsigned long timeout, - m3_dcpext_callback_fn, void *); -int m3_dcpext_rpc_alloc(struct m3_dcpext_rpc *, struct m3_dcpext_buffer *); -int m3_dcpext_rpc_map(struct m3_dcpext_rpc *, struct m3_dcpext_buffer *); -int m3_dcpext_rpc_retire(struct m3_dcpext_rpc *, u32 id); -int m3_dcpext_rpc_dart_power(struct m3_dcpext_rpc *, bool on); -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rtkit.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rtkit.c deleted file mode 100644 index f678591eb96ef..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rtkit.c +++ /dev/null @@ -1,3 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -#define DCPEXT_WITH_SESSION -#include "m3_dcpext_dma.c" diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rtkit_main.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rtkit_main.c deleted file mode 100644 index 65c408cb40bed..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_rtkit_main.c +++ /dev/null @@ -1,2 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -#include "m3_dcpext_rtkit.c" diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_secondary_main.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_secondary_main.c deleted file mode 100644 index 0d0f0e9f08b51..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_secondary_main.c +++ /dev/null @@ -1,3 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -#define DCPEXT_INSTANCE 1 -#include "m3_dcpext_rtkit.c" diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_secondary_power.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_secondary_power.c deleted file mode 100644 index 6689776e93f9d..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_secondary_power.c +++ /dev/null @@ -1,3 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -#define DCPEXT_INSTANCE 1 -#include "m3_dcpext_power.c" diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h deleted file mode 100644 index 19e36e50de767..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_dcpext_session.h +++ /dev/null @@ -1,961 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* J514S 14.6 RTKit system-only handshake. Based on the M4 session adapter - * and Asahi mailbox/RTKit transport. No application endpoint or PHY access. - * All allocations and power references are retained until reboot, even after - * successful protocol quiescence. That ACK alone is not proof of DMA stop. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "m3_hdmi_control.h" -/* Controller identity selects firmware/memory; connector routing is separate. */ -static bool usb_c=DCPEXT_INSTANCE!=0; -module_param(usb_c,bool,0400); -/* Opt-in compatibility experiment for eventual shared-stream routing. - * The two-lane experiment retains the USB-C payload limit. - */ -static bool hdmi_test_two_lane_hbr2; -module_param(hdmi_test_two_lane_hbr2,bool,0400); -static unsigned int link_max_lanes(void) -{return usb_c || hdmi_test_two_lane_hbr2 ? 2 : 4;} -static unsigned int link_max_rate(void) -{return usb_c || hdmi_test_two_lane_hbr2 ? 20 : 30;} -static unsigned int link_payload_kbps(void) -{return link_max_lanes() * link_max_rate() * 270000U * 8 / 10;} -static const struct m3_usbc_route_ops *usbc_ops; -#include "dcpext_session.c" -#include "dcpext_syslog.h" -static struct dcpext_syslog syslog_capture; -static struct debugfs_blob_wrapper syslog_blob; -static unsigned int syslog_wraps; -module_param(syslog_wraps,uint,0400); -static struct dcpext_session session; -static DECLARE_COMPLETION(desktop_started); -static DECLARE_COMPLETION(desktop_finished); -static bool desktop_stop,desktop_done; -static int desktop_result=-EINPROGRESS; -extern int m3_hdmi_bridge_hpd(void); -static int (*desktop_read_hpd)(void); -static u64 desktop_hpd_next; -static unsigned int desktop_hpd_low_samples; -static unsigned int desktop_hpd_high_samples; -static bool desktop_disconnected; -static bool desktop_hpd_dirty, desktop_hpd_rpc, desktop_hpd_value; -static u64 desktop_hpd_deadline; -static unsigned int desktop_link_stage; -/* Bounded diagnostic HPD cycle; actual GPIO is sampled again after 3 s. */ -static bool desktop_retrain; -static u64 desktop_retrain_until; -module_param(desktop_retrain,bool,0600); -module_param(desktop_disconnected,bool,0400); -module_param(desktop_stop,bool,0600); -module_param(desktop_done,bool,0400); -module_param(desktop_result,int,0400); -static void desktop_notify(int ret) -{WRITE_ONCE(desktop_result,ret);complete_all(&desktop_started);} - -static struct regmap *session_pmgr; -static bool cpu_no_auto; -module_param(cpu_no_auto, bool, 0400); -static bool probe_nmi,nmi_sent; -module_param(probe_nmi,bool,0400); -static bool probe_ping; -module_param(probe_ping, bool, 0400); -static bool ping_sent, ping_replied; -static void __iomem *session_cpu, *session_fifo; -static bool session_cpu_owned, session_fifo_owned; -static phys_addr_t session_base; -static struct resource session_table_ram; -static void *session_table_map, *session_table_saved; -static size_t session_used; -static struct { - struct resource ram; - u64 remap; - u32 size, flags; -} session_segments[32]; -static unsigned int session_nsegments; -static void session_cleanup(void) -{ - if (session_table_map) memunmap(session_table_map); - kfree(session_table_saved); - if (session_fifo) iounmap(session_fifo); - if (session_cpu) iounmap(session_cpu); - if (session_fifo_owned) release_mem_region(session_base + 0x8000, 0x4000); - if (session_cpu_owned) release_mem_region(session_base, 0x4000); -} -static int session_prepare(struct device_node *dcp) -{ - struct device_node *node; - struct resource res; - const u8 *metadata; - struct regmap *map; - u32 power; - int bytes, ret; - - session_base = port ? 0x2d6c00000ULL : 0x2d2c00000ULL; - if (of_address_to_resource(dcp, 0, &res) || res.start != session_base || - resource_size(&res) != 0x4000 || !(res.flags & IORESOURCE_MEM_NONPOSTED)) - return -ENODEV; - metadata = of_get_property(dcp, "apple,j514s-dcpext-segments", &bytes); - if (!metadata || bytes <= 0 || bytes % 32 || bytes / 32 > 32) - return -EINVAL; - session_nsegments = bytes / 32; - for (unsigned int i = 0; i < session_nsegments; i++) { - node = of_parse_phandle(dcp, "memory-region", i); - ret = -EINVAL; - if (node && of_device_is_available(node) && - of_property_read_bool(node, "no-map") && - of_device_is_compatible(node, "apple,asc-mem")) - ret = of_address_to_resource(node, 0, &session_segments[i].ram); - of_node_put(node); - if (ret) - return ret; - session_segments[i].remap = get_unaligned_be64(metadata + i * 32 + 16); - session_segments[i].size = get_unaligned_be32(metadata + i * 32 + 24); - session_segments[i].flags = get_unaligned_be32(metadata + i * 32 + 28); - if (!session_segments[i].size || - session_segments[i].ram.start != get_unaligned_be64(metadata + i * 32) || - resource_size(&session_segments[i].ram) != ALIGN((u64)session_segments[i].size, LEASE_PAGE)) - return -EINVAL; - } - /* No CPU control writes. SYS/FE checks are in the shared mapper. */ - node = of_find_node_by_path("/soc/power-management@350700000"); - if (!node) - return -ENODEV; - map = syscon_node_to_regmap(node); - of_node_put(node); - if (IS_ERR(map)) - return PTR_ERR(map); - session_pmgr = map; - ret = regmap_read(map, port ? 0x3f8 : 0x3e0, &power); - if (ret) - return ret; - if ((power & 15) != 15 || power & (BIT(31) | BIT(12) | BIT(11) | BIT(10))) - return -EHOSTDOWN; - if (!request_mem_region(session_base, 0x4000, "m3-dcpext-session")) - return -EBUSY; - session_cpu_owned = true; - if (!request_mem_region(session_base + 0x8000, 0x4000, "m3-dcpext-session")) - return -EBUSY; - session_fifo_owned = true; - /* Mapping here does not touch the registers; fabric is checked before - * session_run accesses these windows. - */ - session_cpu = ioremap_np(session_base, 0x4000); - session_fifo = ioremap_np(session_base + 0x8000, 0x4000); - return session_cpu && session_fifo ? 0 : -ENOMEM; -} - -static int session_tables(const struct resource *reservation) -{ - session_table_ram = *reservation; - if (!resource_size(reservation) || resource_size(reservation) > 0x100000) - return -EINVAL; - session_table_map = memremap(reservation->start, resource_size(reservation), MEMREMAP_WB); - if (!session_table_map) - return -ENOMEM; - session_table_saved = kmemdup(session_table_map, resource_size(reservation), GFP_KERNEL); - return session_table_saved ? 0 : -ENOMEM; -} - -static int session_translate(u64 wire, phys_addr_t *physical) -{ - u64 dva, entry, leaf_pa, pte; - u64 *table; - - if ((wire >> 36) != 0x10) - return -ERANGE; - dva = wire & GENMASK_ULL(35, 0); - entry = READ_ONCE(root[dva >> 25]); - if (!(entry & 1)) - return -ENOENT; - leaf_pa = (entry & PTE_ADDRESS) << 4; - if (leaf_pa < session_table_ram.start || leaf_pa > session_table_ram.end || - session_table_ram.end - leaf_pa + 1 < LEASE_PAGE) - return -ERANGE; - table = session_table_map + leaf_pa - session_table_ram.start; - pte = READ_ONCE(table[(dva >> 14) & 2047]); - if (!(pte & 1)) - return -ENOENT; - *physical = ((pte & PTE_ADDRESS) << 4) | (dva & (LEASE_PAGE - 1)); - return 0; -} - -static int session_buffer(void *cookie, u8 ep, u64 requested, u32 size, u64 *dva) -{ - phys_addr_t pa, checked; - size_t rounded = ALIGN(size, LEASE_PAGE); - int ret; - - if (!requested) { - if (rounded > LEASE_SIZE - session_used) - return -ENOSPC; - memset(pool + session_used, 0, rounded); - *dva = BIT_ULL(40) | (LEASE_DVA + session_used); - session_used += rounded; - pr_info("m3_dcpext_session: ep=%u host buffer size=%#x dva=%#llx\n", ep, size, *dva); - return 0; - } - for (unsigned int i = 0; i < session_nsegments; i++) { - /* ADT flags 0xa OS-log region is a host physical carveout, not - * a DART mapping. Exact base and declared size are mandatory. - */ - if (ep == 8 && session_segments[i].flags == 0xa && - requested == session_segments[i].ram.start && - requested == session_segments[i].remap && size <= session_segments[i].size) { - *dva = requested; - pr_info("m3_dcpext_session: ep=8 reserved physical OSLog size=%#x address=%#llx\n", size, requested); - return 0; - } - } - ret = session_translate(requested, &pa); - if (ret) - return ret; - for (unsigned int i = 0; i < session_nsegments; i++) { - struct resource *r = &session_segments[i].ram; - if (session_segments[i].flags == 0xa || pa < r->start || pa > r->end || - size - 1 > r->end - pa) - continue; - /* Verify every page boundary, including an unaligned first page. */ - for (u64 off = 0; off < size; ) { - ret = session_translate(requested + off, &checked); - if (ret || checked != pa + off) - return -ERANGE; - off += LEASE_PAGE - ((requested + off) & (LEASE_PAGE - 1)); - } - ret = session_translate(requested + size - 1, &checked); - if (ret || checked != pa + size - 1) - return -ERANGE; - *dva = requested; - pr_info("m3_dcpext_session: ep=%u inherited size=%#x dva=%#llx physical=%#llx\n", ep, size, requested, (u64)pa); - return 0; - } - return -ERANGE; -} - -static u64 session_now(void *cookie) -{ - return ktime_get_boottime_ns() / NSEC_PER_MSEC; -} - - -/* Validate ALL inherited mappings against the bootloader-owned segments, - * not just addresses requested by the firmware later. Never follow a table - * pointer outside the reserved page-table storage. */ -static int session_audit(void) -{ - for (unsigned int i = 0; i < LEASE_PAGE / 8; i++) { - u64 entry = READ_ONCE(root[i]), pa; - u64 *table; - if (!(entry & 1)) continue; - pa = (entry & PTE_ADDRESS) << 4; - if (pa < session_table_ram.start || pa > session_table_ram.end || - session_table_ram.end - pa + 1 < LEASE_PAGE) return -ERANGE; - table = session_table_map + pa - session_table_ram.start; - for (unsigned int j = 0; j < LEASE_PAGE / 8; j++) { - u64 pte = READ_ONCE(table[j]), va = ((u64)i << 25) | ((u64)j << 14); - bool found = false; - if (!(pte & 1)) continue; - pa = (pte & PTE_ADDRESS) << 4; - for (unsigned int n = 0; n < session_nsegments; n++) { - u64 start = session_segments[n].remap & GENMASK_ULL(35, 0); - if (session_segments[n].flags == 0xa) continue; - if (va >= start && va - start < ALIGN((u64)session_segments[n].size, LEASE_PAGE) && - pa == session_segments[n].ram.start + va - start) found = true; - } - if (!found) return -ERANGE; - } - } - return 0; -} -static int session_send(void *cookie, u8 ep, u64 message) -{ - u32 status; - int ret = readl_poll_timeout(session_fifo + 0x110, status, !(status & BIT(16)), 100, 100000); - if (ret) return ret; - dma_wmb(); - writeq(message, session_fifo + 0x800); - writeq(ep, session_fifo + 0x808); - if(!session.runtime)pr_info("m3_dcpext_session: TX ep=%#x msg=%#llx\n", ep, message); - return 0; -} -#include "dcpext_afk_channel.h" -static bool quiesce_first; -module_param(quiesce_first, bool, 0400); -static unsigned int afk_endpoint; -module_param(afk_endpoint, uint, 0400); -MODULE_PARM_DESC(afk_endpoint, "Optional announcement-only AFK endpoint, 0x20..0x2b; zero for system-only"); -static struct afkc afk, aux_afk; -static unsigned int aux_endpoint; -module_param(aux_endpoint, uint, 0400); -static bool validate_connection; -module_param(validate_connection, bool, 0400); -static bool connect_port; -module_param(connect_port, bool, 0400); -static bool validation_sent, validation_reply, port0_announced; -static u32 port0_channel; - -static u64 afk_deadline; -static int afk_allocate(void *cookie, u32 size, u8 **buffer, u64 *dva) -{ - size_t offset = session_used; - int ret = session_buffer(NULL, afk_endpoint, 0, size, dva); - if (!ret) *buffer = pool + offset; - return ret; -} -#include "m3_dcpext_native_query.h" -static bool request_display; -module_param(request_display, bool, 0400); -static bool hpd_low_only; -module_param(hpd_low_only,bool,0400); -static bool hotplug; -module_param(hotplug, bool, 0400); -static u64 hpd_until; -static unsigned int rpc_stage, rpc_command, rpc_bytes, rpc_tag, rpc_target; -static u8 *rpc_rx, *rpc_tx; -static u64 rpc_rx_dva, rpc_tx_dva; -static bool rpc_pending; -static u16 epic_seq; -static unsigned int link_lanes; -module_param(link_lanes,uint,0400); -#include "m3_epic_tx.h" -static int old_epic_write_channel(struct afkc *c,u32 channel,u32 kind,u8 *payload,u32 bytes) -{ - u32 next;int ret; - if(!c->running || c->stopping || c->error)return -EINVAL; - ret=m3_epic_ring_write(&c->rings[0],channel,kind,payload,bytes,&next); - return ret ?: afkc_send(c,0xa2,next); -} -static int old_epic_write(u32 channel,u32 kind,u8 *payload,u32 bytes) -{return old_epic_write_channel(&afk,channel,kind,payload,bytes);} -#include "m3_dcpext_iboot.h" -static int link_activate(void) -{ - int ret;int (*fn)(void); - if(usb_c)return usbc_ops->activate(port); - fn=symbol_get(m3_hdmi_phy_activate);if(!fn)return -ENODEV; - ret=fn();symbol_put(m3_hdmi_phy_activate);return ret; -} -static int link_deactivate(void) -{ - int ret;int (*fn)(void); - if(usb_c)return usbc_ops->deactivate(port); - fn=symbol_get(m3_hdmi_phy_deactivate);if(!fn)return -ENODEV; - ret=fn();symbol_put(m3_hdmi_phy_deactivate);return ret; -} -static int link_xbar_down(void) -{ - int ret;int (*fn)(void); - if(usb_c)return usbc_ops->xbar_down(port); - fn=symbol_get(m3_hdmi_xbar_deactivate);if(!fn)return -ENODEV; - ret=fn();symbol_put(m3_hdmi_xbar_deactivate);return ret; -} -static int link_xbar_up(void) -{ - int ret;int (*fn)(void); - if(usb_c)return usbc_ops->xbar_up(port); - fn=symbol_get(m3_hdmi_xbar_activate);if(!fn)return -ENODEV; - ret=fn();symbol_put(m3_hdmi_xbar_activate);return ret; -} -static int link_get_rate(void) -{ - int ret;int (*fn)(void); - if(usb_c)return usbc_ops->get_rate(port); - fn=symbol_get(m3_hdmi_phy_get_rate);if(!fn)return -ENODEV; - ret=fn();symbol_put(m3_hdmi_phy_get_rate);return ret; -} -static int link_set_rate(unsigned int rate) -{ - int ret;int (*fn)(unsigned int); - if(usb_c)return usbc_ops->set_rate(port,rate); - fn=symbol_get(m3_hdmi_phy_set_rate);if(!fn)return -ENODEV; - ret=fn(rate);symbol_put(m3_hdmi_phy_set_rate);return ret; -} -static int link_set_drive(const unsigned int *drive) -{ - u32 hdmi_drive[12]; - int ret;int (*fn)(const unsigned int *); - if(usb_c)return usbc_ops->set_drive(port,drive); - /* The HDMI PHY retains its qualified four-lane physical orientation. - * DPTX encodes only the two negotiated lanes in the diagnostic mode. - * Program equal pair settings, as required by the existing PHY helper; - * this does not claim a new physical lane order or power down either pair. - */ - if(hdmi_test_two_lane_hbr2){ - memcpy(hdmi_drive,drive,6*sizeof(*drive)); - memcpy(hdmi_drive+6,drive,6*sizeof(*drive)); - drive=hdmi_drive; - } - fn=symbol_get(m3_hdmi_phy_set_drive);if(!fn)return -ENODEV; - ret=fn(drive);symbol_put(m3_hdmi_phy_set_drive);return ret; -} -static int link_hpd(void) -{return usbc_ops->hpd(port);} -static int dptx_rpc_send(u32 command) -{ - u8 *tx,wire[70]={};u64 tx_dva;u32 bytes; - int ret; - if (rpc_pending) return -EBUSY; - bytes=command==12?112:(command==11||command==8)?96:80; - /* One outstanding RPC: reuse its pinned buffers only after its reply. */ - if(!rpc_rx){ret=afk_allocate(NULL,112,&rpc_rx,&rpc_rx_dva);if(ret)return ret;} - if(!rpc_tx){ret=afk_allocate(NULL,112,&rpc_tx,&rpc_tx_dva);if(ret)return ret;} - tx=rpc_tx;tx_dva=rpc_tx_dva;memset(tx,0,112);memset(rpc_rx,0,112); - put_unaligned_le32(command,tx+4);put_unaligned_le32(bytes-64,tx+8); - put_unaligned_le32(0x69706378,tx+12); - if(command==8){put_unaligned_le16(8,tx+2);put_unaligned_le32(desktop_hpd_rpc?desktop_hpd_value:rpc_stage==2 && !hpd_low_only,tx+80);} - if(command==11||command==12){ - put_unaligned_le32(command==12?0x100:0,tx+64); - rpc_target=0x8030; - if(usb_c){ - ret=usbc_ops->target(port);if(ret<0)return ret; - rpc_target=ret; - } - put_unaligned_le32(rpc_target,tx+68); - } - wire[0]=2;put_unaligned_le16(epic_seq++,wire+1); - put_unaligned_le32(30,wire+16);wire[20]=4;wire[21]=0x30; - put_unaligned_le16(0xc0,wire+22);put_unaligned_le16(rpc_tag,wire+32); - put_unaligned_le64(rpc_rx_dva,wire+44);put_unaligned_le64(tx_dva,wire+52); - put_unaligned_le32(bytes,wire+60);put_unaligned_le32(bytes,wire+64); - rpc_command=command;rpc_bytes=bytes;rpc_pending=true; - pr_info("m3_dcpext_session: DPTX command=%u tag=%u bytes=%u\n",command,rpc_tag,bytes); - return old_epic_write(port0_channel,3,wire,sizeof(wire)); -} -#include "m3_dcpext_av.h" -static u32 link_drive[8]; -static int dptx_apcall(u32 channel,const u8 *payload,u32 size) -{ - u8 reply[512];u32 idx,len;u8 *data; - int ret; - if(size<108 || size>sizeof(reply) || get_unaligned_le32(payload+56)!=0x69706378) return -EPROTO; - idx=get_unaligned_le32(payload+48);len=get_unaligned_le32(payload+52); - if(len<4||len>size-108)return -EPROTO; - pr_info("m3_dcpext_session: DPTX callback=%u size=%u\n",idx,len); - memcpy(reply,payload,size);data=reply+108; - put_unaligned_le32(0,data); - switch(idx){ - case 0: - ret=link_activate();if(ret)return ret; - break; - case 1: - ret=link_xbar_down();if(ret)return ret; - ret=link_deactivate();if(ret)return ret; - break; - case 2: /* 14.6 ABI: two words, voltage and preemphasis (no kind). */ - if(len!=32)return -EINVAL; - put_unaligned_le32(3,data+16);put_unaligned_le32(3,data+20);break; - case 3: { - u32 drive[12]={}; - if(len!=32+8*link_max_lanes() || get_unaligned_le32(data+16)!=link_max_lanes() || link_lanes!=link_max_lanes())return -EOPNOTSUPP; - for(unsigned int i=0;i3 || drive[3*i+2]>3 || drive[3*i+1]+drive[3*i+2]>3)return -EINVAL; - } - ret=link_set_drive(drive);if(ret)return ret; - memset(link_drive,0,sizeof(link_drive)); - for(unsigned int i=0;i<2*link_max_lanes();i++)link_drive[i]=get_unaligned_le32(data+32+i*4); - pr_info("m3_dcpext_session: drive %u/%u %u/%u %u/%u %u/%u applied\n", - link_drive[0],link_drive[1],link_drive[2],link_drive[3],link_drive[4],link_drive[5],link_drive[6],link_drive[7]); - break; - } - case 4: - if(len!=32+8*link_max_lanes() || get_unaligned_le32(payload+108)!=link_max_lanes() || link_lanes!=link_max_lanes())return -EOPNOTSUPP; - put_unaligned_le32(link_max_lanes(),data);put_unaligned_le32(0,data+16); - for(unsigned int i=0;i<2*link_max_lanes();i++)put_unaligned_le32(link_drive[i],data+32+i*4); - break; - case 5: - if(len!=16)return -EINVAL; - ret=link_xbar_down();if(ret)return ret;break; - case 6: - if(len!=16)return -EINVAL; - ret=link_get_rate();if(ret<0)return ret; - if(ret){ret=link_xbar_up();if(ret)return ret;}break; - case 7: - if(len!=32)return -EINVAL;put_unaligned_le32(link_max_rate(),data+16);break; - case 8: - if(len!=32)return -EINVAL; - ret=link_get_rate();if(ret<0)return ret; - put_unaligned_le32(ret,data+16);break; - case 9: - if(len!=32 || get_unaligned_le32(data+20))return -EINVAL; - ret=link_xbar_down();if(ret)return ret; - ret=link_set_rate(get_unaligned_le32(data+16));if(ret)return ret;break; - case 10:case 11: - if(len!=32)return -EINVAL;put_unaligned_le64(idx==10?link_max_lanes():link_lanes,data+16);break; - case 12: - if(len!=32 || get_unaligned_le64(data+16)>link_max_lanes() || - (get_unaligned_le64(data+16)!=link_max_lanes() && get_unaligned_le64(data+16)!=0))return -EOPNOTSUPP; - link_lanes=get_unaligned_le32(data+16);break; - case 15: - if(len!=32 || get_unaligned_le64(data+16))return -EOPNOTSUPP;break; - case 20: /* DPTX_APCALL_INACTIVE_SINK_DETECTED: notification only. */ - if(len!=16)return -EINVAL;break; - case 21: /* DPTX_APCALL_SET_TILED_DISPLAY_HINTS: retain/report hints. */ - if(len!=144)return -EINVAL; - pr_info("m3_dcpext_session: display hints width=%u height=%u pixel_clock=%u\n", - get_unaligned_le32(data+56),get_unaligned_le32(data+60),get_unaligned_le32(data+64)); - break; - case 18: /* Host owns the HDMI HPD line and sends the hotplug event. */ - if(len<32)return -EINVAL;put_unaligned_le32(hotplug,data+16);break; - case 13:case 14:case 16: - if(len<32)return -EINVAL;put_unaligned_le32(0,data+16);break; - default: - /* Never ACK an unimplemented hardware transition. */ - print_hex_dump(KERN_INFO,"m3_dcpext_unhandled: ",DUMP_PREFIX_OFFSET,16,1,data,len,true); - return -EOPNOTSUPP; - } - reply[0]=2;put_unaligned_le16(epic_seq++,reply+1);reply[21]=0x20; - reply[35]=8;put_unaligned_le32(size-44,reply+36); - return old_epic_write(channel,8,reply,size); -} -#include "m3_dcpext_hpd_irq.h" -static int afk_record(void *cookie, const u8 *payload, u32 size) -{ - u32 channel = get_unaligned_le32(afk.capture + afk.capture_size - size - 12); - u32 kind = get_unaligned_le32(afk.capture + afk.capture_size - size - 8); - int irq_ret=hdmi_irq_record(channel,kind,payload,size); - if(irq_ret)return irq_ret<0?irq_ret:0; - if (size >= 76 && payload[0] == 2 && payload[20] == 4 && !payload[21] && - get_unaligned_le16(payload + 22) == 0x30 && - !memcmp(payload + 40, port?"dispext1:dcpdptx-port-epic:0":"dispext0:dcpdptx-port-epic:0", sizeof("dispext0:dcpdptx-port-epic:0")) && - get_unaligned_le32(payload + 72) == 0xd3) { - port0_announced = true; - port0_channel = channel; - } - if (rpc_pending && (kind==0 || kind==4) && channel==port0_channel && - size>=70 && payload[0]==2 && payload[20]==4 && payload[21]==0x20 && - get_unaligned_le16(payload+22)==0xc0 && get_unaligned_le16(payload+32)==rpc_tag) { - pr_info("m3_dcpext_session: RPC response status=%#x bytes=%u\n",get_unaligned_le32(payload+40),get_unaligned_le32(payload+60)); - if(get_unaligned_le32(payload+40) || get_unaligned_le64(payload+44)!=rpc_rx_dva || - get_unaligned_le32(payload+60)!=rpc_bytes)return -EPROTO; - dma_rmb(); - if(get_unaligned_le32(rpc_rx+12)!=0x69706378 || - get_unaligned_le32(rpc_rx+4)!=rpc_command)return -EPROTO; - if((rpc_command==11||rpc_command==12) && - (get_unaligned_le32(rpc_rx+64)!=(rpc_command==12?0x100:0) || - get_unaligned_le32(rpc_rx+68)!=rpc_target))return -EPROTO; - pr_info("m3_dcpext_session: DPTX command=%u reply accepted\n",rpc_command); - rpc_pending=false;rpc_tag=(rpc_tag+1)&0x7fff; - if(desktop_hpd_rpc){ - desktop_hpd_rpc=false; - pr_info("m3_dcpext_session: runtime link stage=%u HPD=%u acknowledged\n",desktop_link_stage,desktop_hpd_value); - desktop_link_stage=(desktop_link_stage==3 || desktop_link_stage==5)?0:desktop_link_stage+1; - return 0; - } - rpc_stage++; - if(hotplug && rpc_stage==3)hpd_until=session_now(NULL)+2000; - validation_reply = !request_display || rpc_stage==(hotplug?5:3); - } - if(kind==0 && channel==port0_channel && size>=40 && payload[0]==2 && - payload[20]==4 && payload[21]==0x10 && get_unaligned_le16(payload+22)==0xc0) - return dptx_apcall(channel,payload,size); - pr_info("m3_dcpext_session: AFK record ep=%#x bytes=%u\n", afk_endpoint, size); - print_hex_dump(KERN_INFO, "m3_dcpext_afk: ", DUMP_PREFIX_OFFSET, 16, 1, - payload, min(size, 256U), true); - return 0; -} -static const struct afkc_ops afk_ops = { - .send = session_send, .allocate = afk_allocate, .record = afk_record, -}; -static bool log_sent, log_replied; -static int aux_record(void *cookie, const u8 *payload, u32 size) -{ - pr_info("m3_dcpext_session: auxiliary AFK ep=%#x bytes=%u\n",aux_endpoint,size); - if(aux_endpoint==0x28){ - const u8 *h=aux_afk.capture+aux_afk.capture_size-size-12; - int ret=av_record(get_unaligned_le32(h),get_unaligned_le32(h+4),payload,size);if(ret)return ret; - } - if(iboot_modes && aux_endpoint==0x23){ - u32 channel=get_unaligned_le32(aux_afk.capture+aux_afk.capture_size-size-12); - int ret=ib_record(channel,payload,size);if(ret)return ret; - } - if(aux_endpoint==0x20 && size>=72 && payload[21]==0 && !memcmp(payload+40,"system",7)){ - u8 *tx,*rx,wire[70]={};u64 txd,rxd;int ret; - static const u8 property[]={0x14,0,0,0,'g','A','F','K','C','o','n','f','i','g','L','o','g','M','a','s','k',0,0,0,0xd3,0,0,0,0x40,0,0,0x84,0xff,0xff,0,0,0,0,0,0}; - u32 channel=get_unaligned_le32(aux_afk.capture+aux_afk.capture_size-size-12); - if(log_sent)return -EPROTO; - ret=afk_allocate(NULL,sizeof(property),&tx,&txd);if(ret)return ret; - ret=afk_allocate(NULL,sizeof(property),&rx,&rxd);if(ret)return ret; - memcpy(tx,property,sizeof(property));wire[0]=2;put_unaligned_le32(30,wire+16); - wire[20]=4;wire[21]=0x30;put_unaligned_le16(0x43,wire+22); - put_unaligned_le64(rxd,wire+44);put_unaligned_le64(txd,wire+52); - put_unaligned_le32(sizeof(property),wire+60);put_unaligned_le32(sizeof(property),wire+64); - log_sent=true;return old_epic_write_channel(&aux_afk,channel,3,wire,sizeof(wire)); - } - if(aux_endpoint==0x20 && log_sent && size>=70 && payload[21]==0x20 && get_unaligned_le16(payload+22)==0x43){ - if(get_unaligned_le32(payload+40))return -EREMOTEIO; - log_replied=true;pr_info("m3_dcpext_session: firmware verbose logging enabled\n"); - } - print_hex_dump(KERN_INFO,"m3_dcpext_aux: ",DUMP_PREFIX_OFFSET,16,1,payload,min(size,256U),true); - return 0; -} -static const struct afkc_ops aux_ops={.send=session_send,.allocate=afk_allocate,.record=aux_record}; -#include "m3_dcpext_native_adapter.h" -#include "m3_dcpext_audio.h" -static int session_dispatch(u64 endpoint,u64 message) -{ - if(!session.runtime)pr_info("m3_dcpext_session: RX ep=%#llx msg=%#llx\n",endpoint,message); - if(session.runtime){int ret=dcpext_session_runtime_message(&session);if(ret)return ret;} - else if(++received>(native_kms_trial?8192:native_startup?2048:512))return -E2BIG; - if(!(endpoint&0xff) && (message>>52)==4 && ping_sent && !ping_replied){ping_replied=true;return 0;} - if(native_query && (endpoint&0xff)==0x37)return native_receive(message); - if(audio_afk.started && (endpoint&0xff)==0x29)return audio_receive(message); - if(aux_endpoint && (endpoint&0xff)==aux_endpoint)return afkc_receive(&aux_afk,message); - if(afk_endpoint && (endpoint&0xff)==afk_endpoint)return afkc_receive(&afk,message); - if ((endpoint&0xff)==2) { - struct dcpext_buffer *b=&session.buffers[2]; - u64 offset=b->dva & ~BIT_ULL(40); int ret; - if (b->ready && (offsetsession_used || b->size>session_used-(offset-LEASE_DVA))) return -ERANGE; - if(syslog_capture.used>sizeof(syslog_capture.capture)-512){syslog_capture.used=0;syslog_wraps++;} - ret=dcpext_syslog_observe(&syslog_capture,message,b->ready?pool+offset-LEASE_DVA:NULL,b->size); - if(ret)return ret; - } - return dcpext_session_receive(&session,endpoint&0xff,message); -} - -static struct debugfs_blob_wrapper ram_snapshot[2][32]; -static void snapshot_ram(struct dentry *dir, unsigned int sample) -{ - u32 pmgr_state; - regmap_read(session_pmgr, port?0x3f8:0x3e0, &pmgr_state); - pr_info("m3_dcpext_session: CPU PMGR=%#x\n", pmgr_state); - pr_info("m3_dcpext_session: ASC status=%#x timer_mask_set=%#x timer_mask_clear=%#x\n", readl(session_cpu+0x48), readl(session_cpu+0x1010), readl(session_cpu+0x1018)); - pr_info("m3_dcpext_session: RAM sample=%u host cntpct=%#llx boottime_ms=%llu\n", sample, read_sysreg(cntpct_el0), session_now(NULL)); - for(unsigned int i=0;i0x1000000)continue; - map=memremap(session_segments[i].ram.start,size,MEMREMAP_WB); - if(!map)continue; - ram_snapshot[sample][i].data=vmalloc(size);ram_snapshot[sample][i].size=size; - if(ram_snapshot[sample][i].data){ - /* Reserved no-map RAM is written only by DCP. This host alias is - * read-only: discard clean stale lines before every asynchronous sample. - * All segment boundaries are page aligned; never discard adjacent data. */ - for (unsigned long at=(unsigned long)map; at<(unsigned long)map+size; at+=64) - asm volatile("dc ivac, %0" :: "r"(at) : "memory"); - dsb(sy); - memcpy(ram_snapshot[sample][i].data,map,size); - snprintf(name,sizeof(name),"firmware-ram-%u%s",i,sample?"":"-early"); - debugfs_create_blob(name,0400,dir,&ram_snapshot[sample][i]); - pr_info("m3_dcpext_session: snapshot %s flags=%#x pa=%#llx remap=%#llx size=%#x\n",name,session_segments[i].flags,(u64)session_segments[i].ram.start,session_segments[i].remap,size); - } - memunmap(map); - } -} -static ssize_t pool_read(struct file *file,char __user *buf,size_t count,loff_t *offset) -{ - dma_rmb();return simple_read_from_buffer(buf,count,offset,pool,session_used); -} -static const struct file_operations pool_fops={.owner=THIS_MODULE,.read=pool_read,.llseek=default_llseek}; -static ssize_t aux_read(struct file *file,char __user *buf,size_t count,loff_t *offset) -{ - return simple_read_from_buffer(buf,count,offset,aux_afk.capture,aux_afk.capture_size); -} -static const struct file_operations aux_fops={.owner=THIS_MODULE,.read=aux_read,.llseek=default_llseek}; -static ssize_t afk_read(struct file *file, char __user *buf, size_t count, loff_t *offset) -{ - return simple_read_from_buffer(buf, count, offset, afk.capture, afk.capture_size); -} -static const struct file_operations afk_fops = { - .owner = THIS_MODULE, .read = afk_read, .llseek = default_llseek, -}; -/* Snapshot the retained RTKit log while the desktop remains active. Allocate - * outside the route lock and copy under it; userspace reads never block RPCs. - * Each open owns an immutable, bounded snapshot, including during wraparound. - */ -struct syslog_snapshot { size_t size; u8 data[]; }; -static int syslog_live_open(struct inode *inode, struct file *file) -{ - struct syslog_snapshot *snapshot; - snapshot=kvmalloc(struct_size(snapshot,data,sizeof(syslog_capture.capture)),GFP_KERNEL); - if(!snapshot)return -ENOMEM; - mutex_lock(&native_route_lock); - snapshot->size=syslog_capture.used; - if(snapshot->size>sizeof(syslog_capture.capture)){ - mutex_unlock(&native_route_lock);kvfree(snapshot);return -EIO; - } - memcpy(snapshot->data,syslog_capture.capture,snapshot->size); - mutex_unlock(&native_route_lock); - file->private_data=snapshot; - return 0; -} -static ssize_t syslog_live_read(struct file *file,char __user *buf,size_t count,loff_t *offset) -{ - struct syslog_snapshot *snapshot=file->private_data; - return simple_read_from_buffer(buf,count,offset,snapshot->data,snapshot->size); -} -static int syslog_live_release(struct inode *inode,struct file *file) -{kvfree(file->private_data);return 0;} -static const struct file_operations syslog_live_fops={ - .owner=THIS_MODULE,.open=syslog_live_open,.read=syslog_live_read, - .release=syslog_live_release,.llseek=default_llseek, -}; -static int session_run(void) -{ - static const struct dcpext_session_ops ops = { - .now_ms = session_now, .send = session_send, .buffer = session_buffer, - }; - u32 state; - struct dentry *dir = NULL; - u64 early_at; - bool early_saved = false; - int ret; - if(native_desktop && !native_kms_trial)return -EINVAL; - if(native_kms_trial && aux_endpoint!=0x28)return -EINVAL; - if(native_kms_trial && !native_frame)return -EINVAL; - if(native_frame && !native_startup)return -EINVAL; - if(probe_nmi && !iboot_image)return -EINVAL; - if(native_startup && (!native_query || !hotplug || iboot_modes))return -EINVAL; - if(iboot_image && !iboot_modes)return -EINVAL; - if(iboot_modes && (aux_endpoint!=0x23 || !hotplug || !cpu_no_auto))return -EINVAL; - if(aux_endpoint && ((aux_endpoint!=0x23 && aux_endpoint!=0x24 && aux_endpoint!=0x20 && aux_endpoint!=0x28) || !hotplug))return -EINVAL; - if (hotplug && !request_display) return -EINVAL; - if (request_display && !connect_port) return -EINVAL; - if (connect_port && !validate_connection) return -EINVAL; - if (validate_connection && afk_endpoint != 0x2a) return -EINVAL; - if (afk_endpoint && (afk_endpoint < 0x20 || afk_endpoint > 0x2b)) return -EINVAL; - if(port && !usb_c)return -EINVAL; - if(usb_c && hdmi_test_two_lane_hbr2)return -EINVAL; - link_lanes=link_max_lanes(); - if(usb_c){ - const struct m3_usbc_route_ops *(*get_ops)(void)=symbol_get(m3_usbc_get_ops); - if(!get_ops)return -ENODEV; - /* Retain the module reference with the session, including on failure. */ - usbc_ops=get_ops(); - ret=usbc_ops->prepare(port);if(ret)return ret; - } - if(native_desktop){ - /* Retain the GPIO owner's module reference with this session's resources. - * Diagnostics without a desktop do not require the bridge module. */ - desktop_read_hpd=usb_c?link_hpd:symbol_get(m3_hdmi_bridge_hpd); - if(!desktop_read_hpd)return -ENODEV; - if(!usb_c){ - hdmi_irq_count=symbol_get(m3_hdmi_bridge_irq_count); - if(!hdmi_irq_count)return -ENODEV; - hdmi_irq_seen=hdmi_irq_count(); - } - ret=desktop_read_hpd(); - if(ret<0 || (!usb_c && ret!=1))return ret<0?ret:-ENOLINK; - } - state = readl(session_cpu + 0x44); - /* Fresh boot only. CPU is stopped and no protocol session is inherited. - * Standard Asahi dcp.c start: RUN only, preserve vector/reset state. */ - if (state != 0 || !(readl(session_fifo + 0x114) & BIT(17))) return -EBUSY; - if (cpu_no_auto) { - ret=regmap_update_bits(session_pmgr,port?0x3f8:0x3e0,BIT(28),0); - if(ret)return ret; - ret=regmap_read_poll_timeout(session_pmgr,port?0x3f8:0x3e0,state, - !(state&BIT(28)) && (state&0xff)==0xff,1,1000); - if(ret)return ret; - pr_info("m3_dcpext_session: diagnostic CPU auto-gating disabled PMGR=%#x\n",state); - } - writel(BIT(4), session_cpu + 0x44); - if (readl(session_cpu + 0x44) != BIT(4)) return -EIO; - pr_info("m3_dcpext_session: CPU RUN set; firmware vector unchanged\n"); - session.defer_iop_quiesce = quiesce_first; - if (afk_endpoint) dir = debugfs_create_dir(port ? "m3_dcpext_session1" : "m3_dcpext_session", NULL); - if(dir){audio_debugfs(dir); - debugfs_create_file("syslog-live",0400,dir,NULL,&syslog_live_fops); - if(native_startup)debugfs_create_file("native-capture",0400,dir,NULL,&native_capture_fops); - } - early_at = session_now(NULL) + 2000; - if(iboot_modes || native_startup){session.duration_ms=16000;session.message_limit=512;} - ret = dcpext_session_begin(&session, &ops, NULL); - mutex_lock(&native_route_lock); - while (!ret && session.phase != DCPEXT_QUIESCED) { - if(native_desktop && native_kms_ready && !native_kms_finished && - !READ_ONCE(desktop_stop) && session_now(NULL)>=desktop_hpd_next){ - int level=desktop_read_hpd(); - desktop_hpd_next=session_now(NULL)+200; - if(level<0){ret=level;break;} - if(READ_ONCE(desktop_retrain) && native_kms_connected && !desktop_link_stage){ - WRITE_ONCE(desktop_retrain,false); - desktop_retrain_until=session_now(NULL)+3000; - pr_info("m3_dcpext_session: diagnostic three-second HPD-low cycle\n"); - } - if(session_now(NULL)=11){ - desktop_disconnected=true;desktop_hpd_dirty=true; - native_kms_disconnect(); - pr_info("m3_dcpext_session: HDMI disconnected; DRM retained for reconnect\n"); - } - }else{ - desktop_hpd_low_samples=0; - if(desktop_disconnected && ++desktop_hpd_high_samples>=3){ - desktop_disconnected=false;desktop_hpd_dirty=true; - pr_info("m3_dcpext_session: HDMI cable reconnected; retraining\n"); - } - } - } - if(desktop_hpd_rpc && session_now(NULL)>=desktop_hpd_deadline){ret=-ETIMEDOUT;break;} - if((desktop_hpd_dirty || desktop_link_stage) && !rpc_pending && !av_pending && - (!READ_ONCE(desktop_stop) || desktop_link_stage)){ - if(!desktop_link_stage){ - desktop_hpd_dirty=false;desktop_hpd_value=!desktop_disconnected; - /* Match the native DPTX lifecycle: low/release, then connect/request/high. - * HPD-high alone reopens AV but leaves the sink inactive after removal. */ - desktop_link_stage=desktop_hpd_value?1:4; - m3_dcpext_native_invalidate_sink(native_client); - av_edid_done=false;av_read_ready=false;native_frame_submitted=false; - } - /* USB-C discovery needs its AUX route before request/HPD, including - * reconnect at main-link rate zero (same ordering as M4). */ - if(usb_c && desktop_link_stage==1){ - ret=link_xbar_up(); - if(ret==-EAGAIN){ - /* No crossbar write occurred: mux setup has not caught up with HPD. - * Return to offline polling, without losing the live firmware session. */ - desktop_disconnected=true;desktop_hpd_dirty=false;desktop_link_stage=0; - desktop_hpd_high_samples=0;ret=0; - pr_info("m3_dcpext_session: waiting for Type-C PHY before reconnect\n"); - goto runtime_link_deferred; - } - if(ret)break; - } - desktop_hpd_rpc=true;desktop_hpd_deadline=session_now(NULL)+5000; - ret=dptx_rpc_send(desktop_link_stage==1?11:desktop_link_stage==2?6:desktop_link_stage==5?7:8); - if(ret)break; - } -runtime_link_deferred: - if(native_kms_ready && !native_kms_finished && !desktop_link_stage && (READ_ONCE(desktop_stop) || (!native_desktop && session_now(NULL)>=native_kms_until))){ - mutex_unlock(&native_route_lock);native_kms_unplug();mutex_lock(&native_route_lock); - if(session.runtime){ret=dcpext_session_leave_runtime(&session);received=0;if(ret)break;} - } - u64 message, endpoint; - if (hotplug && !early_saved && session_now(NULL) >= early_at) { - snapshot_ram(dir, 0); - early_saved = true; - if (probe_ping) { - ping_sent = true; - ret = session_send(NULL, 0, 3ULL << 52); - if (ret) break; - } - } - if (probe_nmi && !nmi_sent && ib_pending && ib_operation==6 && session_now(NULL)>=early_at+4000) { - /* AppleASCWrapV6::_generateNMI legacy fallback, 26A428 +0x1004/1014. - * Intentionally request a diagnostic FIQ after the mode command stalls. - * This trial always requires reboot; no firmware code is patched. */ - nmi_sent=true; - pr_warn("m3_dcpext_session: diagnostic ASC NMI fallback; expect firmware crash/reboot recovery\n"); - writel(0x10,session_cpu+0x1004);writel(1,session_cpu+0x1014); - } - ret = hdmi_irq_progress();if(ret)break; - ret = dcpext_session_poll(&session); - if (ret) break; - if (session.phase == DCPEXT_RUNNING) { - if(native_query && !native_phase){ret=native_query_start();if(ret)break;} - if(aux_endpoint && !aux_afk.started){ - ret=afkc_start(&aux_afk,&aux_ops,NULL,aux_endpoint);if(ret)break; - } - if (afk_endpoint && !afk.started && (!aux_endpoint || (aux_afk.running && (aux_endpoint!=0x20 || log_replied)))) { - if (!(session.endpoints[1] & BIT(afk_endpoint - 32))) { ret = -ENODEV; break; } - ret = afkc_start(&afk, &afk_ops, NULL, afk_endpoint); - afk_deadline = session_now(NULL) + 1500; - if (ret) break; - } - if (validate_connection && port0_announced && !validation_sent) { - validation_sent=true; - ret = dptx_rpc_send(connect_port?11:12); - if (ret) break; - } - ret=av_progress();if(ret)break; - ret=native_progress();if(ret)break; - if(native_kms_connected && !native_kms_finished && native_kms_generation!=m3_dcpext_native_generation(native_client)){ - pr_info("m3_dcpext_native: timing publication changed; refreshing HDMI connector\n"); - native_kms_disconnect(); - } - ret=ib_poll();if(ret)break; - if (request_display && validation_sent && !rpc_pending && rpc_stage>0 && rpc_stage<(hotplug?5:3) && - (!hotplug || rpc_stage!=3 || (session_now(NULL)>=hpd_until && (!iboot_modes || ib_done) && (!native_frame || native_frame_done)))) { - /* Type-C negotiation can be stable while a waking HDMI adapter pulses - * HPD low. Prepare AUX first, then announce the actual high level. */ - int level=port && desktop_read_hpd && rpc_stage==2?desktop_read_hpd():1; - if(level<0){ret=level;break;} - if(level)ret=dptx_rpc_send(rpc_stage==1?6:hotplug && rpc_stage<4?8:7); - if(ret)break; - } - if (afk_endpoint && afk.running && !afk.stopping && session_now(NULL) >= afk_deadline && (!validate_connection || validation_reply) && (!native_query || native_phase==3)) { - ret = quiesce_first ? dcpext_session_quiesce(&session) : afkc_stop(&afk); - if (ret) break; - } - if (afk.stopped && aux_endpoint && aux_afk.running && !aux_afk.stopping && (aux_endpoint!=0x28 || !av_announced || av_closed)) { - ret=afkc_stop(&aux_afk);if(ret)break; - } - if ((!afk_endpoint || afk.stopped) && (!aux_endpoint || aux_afk.stopped)) { - pr_info("m3_dcpext_session: application endpoints stopped; requesting AP/IOP quiescence\n"); - ret = dcpext_session_quiesce(&session); - if (ret) break; - } - } - if (quiesce_first && session.phase == DCPEXT_AP_QUIESCE && session.ap_quiesced) { - if (!afk.stopping) ret = afkc_stop(&afk); - else if (afk.stopped) ret = dcpext_session_finish_quiesce(&session); - if (ret) break; - } - if (readl(session_fifo + 0x114) & BIT(17)) { - mutex_unlock(&native_route_lock); - if(session.runtime)usleep_range(1000,1500);else usleep_range(100,200); - mutex_lock(&native_route_lock); - continue; - } - message = readq(session_fifo + 0x830); - endpoint = readq(session_fifo + 0x838); - dma_rmb(); - ret=session_dispatch(endpoint,message); - mutex_unlock(&native_route_lock);cond_resched();mutex_lock(&native_route_lock); - } - mutex_unlock(&native_route_lock); - if(native_kms_ready && !native_kms_finished)native_kms_unplug(); - if (afk_endpoint) { - syslog_blob.data=syslog_capture.capture;syslog_blob.size=syslog_capture.used; - debugfs_create_blob("syslog-capture",0400,dir,&syslog_blob); - if(aux_endpoint)debugfs_create_file("aux-capture",0400,dir,NULL,&aux_fops); - debugfs_create_file("session-pool",0400,dir,NULL,&pool_fops); - if(hotplug)snapshot_ram(dir, 1); - debugfs_create_file("afk-capture", 0400, dir, NULL, &afk_fops); - pr_info("m3_dcpext_session: AFK running=%u stopped=%u error=%d captured=%u\n", - afk.running, afk.stopped, afk.error, afk.capture_size); - } - pr_info("m3_dcpext_session: result=%d phase=%u messages=%u; retaining all resources until reboot\n", - ret, session.phase, session.received); - return ret; -} - -static int session_thread(void *unused) -{ - int ret=session_run(); - desktop_notify(ret);WRITE_ONCE(desktop_done,true);complete_all(&desktop_finished); - return ret; -} -static int desktop_reboot(struct notifier_block *nb,unsigned long event,void *data) -{ - WRITE_ONCE(desktop_stop,true); - if(!wait_for_completion_timeout(&desktop_finished,msecs_to_jiffies(20000))) - pr_err("m3_dcpext: shutdown timed out; all DMA retained\n"); - return NOTIFY_DONE; -} -static struct notifier_block desktop_reboot_nb={.notifier_call=desktop_reboot}; -static int session_start(void) -{ - struct task_struct *task;int ret; - if(!native_desktop)return session_run(); - ret=register_reboot_notifier(&desktop_reboot_nb);if(ret)return ret; - task=kthread_run(session_thread,NULL,"m3-hdmi"); - if(IS_ERR(task)){unregister_reboot_notifier(&desktop_reboot_nb);return PTR_ERR(task);} - if(!wait_for_completion_timeout(&desktop_started,msecs_to_jiffies(20000))){ - WRITE_ONCE(desktop_stop,true);return -ETIMEDOUT; - } - return READ_ONCE(desktop_result); -} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_epic_tx.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_epic_tx.h deleted file mode 100644 index 4879e0db722ff..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_epic_tx.h +++ /dev/null @@ -1,54 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* M4 AFK ring admission with old EPIC channel/kind queue fields. */ -/* Wrap using duplicate queue headers, as in Sven Peter's Asahi afk_send_epic: - * one at the tail and one before the complete payload at offset zero. Never - * split a payload or overwrite unread blocks. Keep one block empty so equal - * pointers mean empty. All admission checks precede shared-memory writes; - * published packets stay mapped even if the mailbox notification fails. - */ -static int m3_epic_ring_write(struct afkc_ring *r, u32 channel, u32 kind, - const u8 *data, u32 bytes, u32 *published) -{ - u32 rd, wr, total, next; - u8 *qe; - bool wrap; - - if (!r || !r->header || !r->data || !data || !published || - r->block < 64 || r->block > 256 || (r->block & (r->block - 1)) || - r->size < r->block || (r->size & (r->block - 1)) || - bytes < 24 || bytes > 1024) - return -EINVAL; - total = ALIGN(bytes + 16, r->block); - rd = READ_ONCE(*(u32 *)(r->header + r->block)); - wr = READ_ONCE(*(u32 *)(r->header + 2 * r->block)); - dma_rmb(); - if (rd >= r->size || wr >= r->size || (rd | wr) & (r->block - 1)) - return -EPROTO; - wrap = total > r->size - wr; - if (total >= r->size || (wrap && (wr < rd || total >= rd)) || - (wr < rd && total >= rd - wr)) - return -ENOSPC; - next = wrap ? total : wr + total; - if (next == r->size) - next = 0; - if (next == rd) - return -ENOSPC; - qe = r->data + wr; - if (wrap) { - memset(qe, 0, 16); - put_unaligned_le32(0x20504f49, qe); - put_unaligned_le32(bytes, qe + 4); - put_unaligned_le32(channel,qe+8);put_unaligned_le32(kind,qe+12); - qe = r->data; - } - memset(qe, 0, total); - put_unaligned_le32(0x20504f49, qe); - put_unaligned_le32(bytes, qe + 4); - put_unaligned_le32(channel,qe+8);put_unaligned_le32(kind,qe+12); - memcpy(qe + 16, data, bytes); - dma_wmb(); - WRITE_ONCE(*(u32 *)(r->header + 2 * r->block), next); - dma_wmb(); - *published = next; - return 0; -} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_bridge.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_bridge.c deleted file mode 100644 index 455fa42f04e47..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_bridge.c +++ /dev/null @@ -1,120 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* J514S native HDMI bridge power/HPD diagnostic. GPIO API only. - * ADT dp2hdmi-gpio0: gP0f HDMI power, gP1a converter power; AOP GPIO47 HPD. - * Native AppleHDMIPortController prefers converter power over reset when both - * exist. Do not toggle the reset, CEC or force-DFU lines. Retain ownership. - */ -#include -#include -#include -#include -#include -#include -#include -#include -static bool power; -module_param(power, bool, 0400); -static int result = -EINPROGRESS; -module_param(result, int, 0400); -static int hpd = -1; -module_param(hpd, int, 0400); -static struct device *owner; -static struct gpio_desc *port, *bridge, *detect; -static unsigned int hpd_edges,hpd_irqs,hpd_last_low_us; -module_param(hpd_edges,uint,0400); -module_param(hpd_irqs,uint,0400); -module_param(hpd_last_low_us,uint,0400); -static u64 hpd_falling_ns; -static int hpd_irq; -/* Apple GPIO is MMIO-backed and cannot sleep. Classify short HPD pulses in - * hard IRQ context so scheduler latency cannot turn an IRQ into cable loss. - * Firmware RPCs run in the serialized session context, never from here. - */ -static irqreturn_t bridge_hpd_edge(int irq,void *cookie) -{ - u64 now=ktime_get_ns(),low; - int level=gpiod_get_value(detect); - if(level<0)return IRQ_HANDLED; - hpd_edges++; - if(!level){hpd_falling_ns=now;return IRQ_HANDLED;} - if(!hpd_falling_ns)return IRQ_HANDLED; - low=now-hpd_falling_ns;hpd_falling_ns=0; - WRITE_ONCE(hpd_last_low_us,min_t(u64,low/1000,U32_MAX)); - if(low>=100000 && low<=2000000)WRITE_ONCE(hpd_irqs,hpd_irqs+1); - return IRQ_HANDLED; -} -u32 m3_hdmi_bridge_irq_count(void); -u32 m3_hdmi_bridge_irq_count(void) -{return READ_ONCE(hpd_irqs);} -EXPORT_SYMBOL_GPL(m3_hdmi_bridge_irq_count); -/* The session owns firmware notifications; this module owns the GPIO. */ -int m3_hdmi_bridge_hpd(void); -int m3_hdmi_bridge_hpd(void) -{ - if (result || IS_ERR_OR_NULL(detect)) return -ENODEV; - return gpiod_get_value_cansleep(detect); -} -EXPORT_SYMBOL_GPL(m3_hdmi_bridge_hpd); -static struct gpiod_lookup_table lines = { - .dev_id = "m3-hdmi-bridge", - .table = { - GPIO_LOOKUP("macsmc-pmu-gpio", 15, "port-power", GPIO_ACTIVE_HIGH), - GPIO_LOOKUP("macsmc-pmu-gpio", 26, "bridge-power", GPIO_ACTIVE_HIGH), - GPIO_LOOKUP("374824000.pinctrl", 47, "hpd", GPIO_ACTIVE_HIGH), - { } - } -}; -static int __init probe(void) -{ - int ret, p, b; - if (!of_machine_is_compatible("apple,j514s") || - !of_machine_is_compatible("apple,t6030")) return -ENODEV; - owner = root_device_register("m3-hdmi-bridge"); - if (IS_ERR(owner)) return PTR_ERR(owner); - gpiod_add_lookup_table(&lines); - port = gpiod_get(owner, "port-power", GPIOD_ASIS); - if (IS_ERR(port)) {ret=PTR_ERR(port);goto done;} - bridge = gpiod_get(owner, "bridge-power", GPIOD_ASIS); - if (IS_ERR(bridge)) {ret=PTR_ERR(bridge);goto done;} - detect = gpiod_get(owner, "hpd", GPIOD_IN); - if (IS_ERR(detect)) {ret=PTR_ERR(detect);goto done;} - if (gpiod_get_direction(port) != 0 || gpiod_get_direction(bridge) != 0) { - ret=-EINVAL;goto done; - } - if(gpiod_cansleep(detect)){ret=-EOPNOTSUPP;goto done;} - hpd_irq=gpiod_to_irq(detect); - if(hpd_irq<0){ret=hpd_irq;goto done;} - ret=devm_request_irq(owner,hpd_irq,bridge_hpd_edge, - IRQF_TRIGGER_RISING|IRQF_TRIGGER_FALLING,"m3-hdmi-hpd",owner); - if(ret)goto done; - p=gpiod_get_value_cansleep(port);b=gpiod_get_value_cansleep(bridge); - hpd=gpiod_get_value_cansleep(detect); - pr_info("m3_hdmi_bridge: before port=%d bridge=%d hpd=%d\n",p,b,hpd); - if (p<0 || b<0 || hpd<0) {ret=-EIO;goto done;} - if (power) { - ret=gpiod_set_value_cansleep(port, 1);if(ret)goto done; - ret=gpiod_set_value_cansleep(bridge, 1);if(ret)goto done; - msleep(500); - /* The converter can pulse HPD low while starting. Wait for the real - * connector signal, rather than injecting HPD high into an absent sink. */ - for(unsigned int sample=0,stable=0;sample<100;sample++){ - hpd=gpiod_get_value_cansleep(detect); - if(hpd<0){ret=hpd;goto done;} - stable=hpd?stable+1:0; - if(stable==5)break; - msleep(50); - if(sample==99){ret=-ENODEV;goto done;} - } - p=gpiod_get_value_cansleep(port);b=gpiod_get_value_cansleep(bridge); - hpd=gpiod_get_value_cansleep(detect); - pr_info("m3_hdmi_bridge: powered port=%d bridge=%d hpd=%d\n",p,b,hpd); - } - ret=(p<0 || b<0 || hpd<0 || (power && (p!=1 || b!=1))) ? -EIO : 0; -done: - result=ret; - pr_info("m3_hdmi_bridge: result=%d power=%u; GPIO ownership retained until reboot\n",ret,power); - return 0; -} -module_init(probe); -MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("J514S native HDMI converter power and HPD diagnostic"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_control.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_control.h deleted file mode 100644 index 1771deaf80e2b..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_control.h +++ /dev/null @@ -1,26 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -#ifndef M3_HDMI_CONTROL_H -#define M3_HDMI_CONTROL_H -u32 m3_hdmi_bridge_irq_count(void); -bool m3_hdmi_phy_ready(void); -int m3_hdmi_phy_deactivate(void); -int m3_hdmi_phy_activate(void); -int m3_hdmi_phy_set_rate(unsigned int rate); -int m3_hdmi_phy_get_rate(void); -int m3_hdmi_phy_set_drive(const unsigned int drive[12]); -int m3_hdmi_xbar_activate(void); -int m3_hdmi_xbar_deactivate(void); -struct m3_usbc_route_ops { - int (*prepare)(unsigned int slot); - int (*hpd)(unsigned int slot); - int (*target)(unsigned int slot); - int (*activate)(unsigned int slot); - int (*deactivate)(unsigned int slot); - int (*xbar_up)(unsigned int slot); - int (*xbar_down)(unsigned int slot); - int (*set_rate)(unsigned int slot, unsigned int rate); - int (*get_rate)(unsigned int slot); - int (*set_drive)(unsigned int slot, const unsigned int drive[12]); -}; -const struct m3_usbc_route_ops *m3_usbc_get_ops(void); -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy.c deleted file mode 100644 index 9a9568bb111d1..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy.c +++ /dev/null @@ -1,192 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* J514S HDMI ATC3 power, common/AUX and standalone main-link diagnostic. - * Addresses: J514S ADT atc-phy3, native 26A428 descriptor capture and T6030 DT. - * Retain genpd/device/MMIO ownership until reboot, including on failure. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "m3_hdmi_control.h" -#include "m3_hdmi_phy_power.h" -#include "m3_hdmi_phy_common.h" -#include "m3_hdmi_phy_dp-main.h" -#include "m3_hdmi_phy_dp-drive.h" -static int result = -EINPROGRESS; -module_param(result, int, 0400); -static bool roundtrip; -module_param(roundtrip, bool, 0400); -static unsigned int main_rate; -module_param(main_rate, uint, 0400); -static bool common; -module_param(common, bool, 0400); -static unsigned int calibration[12]; -static unsigned int calibration_count; -module_param_array(calibration, uint, &calibration_count, 0400); -static const u32 offsets[] = {0x800, 0x2224, 0x2230, 0x2240, 0x22b0, 0x22e4, - 0x2334, 0x2248, 0x16008, 0x1600c, 0xa04, 0x260c}; -static const u32 masks[] = {0x3f0000, 0xc00, 0x1f, 0x18007f, 0x8700000, 0x60001, - 0xf800000, 0x1ff00, 0x38, 0x1f, 0x7c, 0x3ff}; -static struct device *owner; -static void __iomem *core; -static bool phy_active; -bool m3_hdmi_phy_ready(void) -{ - return result==0 && core && phy_active && (readl(core+0x804)&1) && - (!!(readl(core+0x7034)&8)==!!main_rate) && readl(core+0x60)==0x492; -} -EXPORT_SYMBOL_GPL(m3_hdmi_phy_ready); -int m3_hdmi_phy_deactivate(void) -{ - int ret; - if(!m3_hdmi_phy_ready())return -EHOSTDOWN; - ret=main_rate?m3_hdmi_atc_dp_main_stop(core,3):0; - if(ret)return ret; - m3_hdmi_atc_aux_stop(core); - m3_hdmi_atc_common_stop(core); - ret=m3_hdmi_atc_power_down(core); - pr_info("m3_hdmi_phy: deactivation result=%d\n",ret); - if(!ret){main_rate=0;phy_active=false;} - return ret; -} -EXPORT_SYMBOL_GPL(m3_hdmi_phy_deactivate); -int m3_hdmi_phy_get_rate(void) -{ - return m3_hdmi_phy_ready()?main_rate:-EHOSTDOWN; -} -EXPORT_SYMBOL_GPL(m3_hdmi_phy_get_rate); -int m3_hdmi_phy_set_rate(unsigned int rate) -{ - int ret; - if(rate!=0 && rate!=6 && rate!=10 && rate!=20 && rate!=30)return -EINVAL; - if(!m3_hdmi_phy_ready())return -EHOSTDOWN; - if(rate==main_rate)return 0; - ret=main_rate?m3_hdmi_atc_dp_main_stop(core,3):0; - if(!ret && rate)ret=m3_hdmi_atc_dp_main_prepare(core,3,rate,false); - if(ret)result=ret;else main_rate=rate; - pr_info("m3_hdmi_phy: set rate=%u result=%d PLL=%#x\n",rate,ret,readl(core+0x7034)); - return ret; -} -EXPORT_SYMBOL_GPL(m3_hdmi_phy_set_rate); -int m3_hdmi_phy_set_drive(const unsigned int drive[12]) -{ - /* 26A428 AppleT8122TypeCPhy fallback at VA0xfffffe000783cac0. - * Captured J514S ATC3 ADT has no training-table overrides. */ - static const u32 table[16]={0x3f024,0x237013,0x37200b,0x46e004, - 0x3f01c,0x23700b,0x372004,0x3f000,0x3f013,0x237004,0x3f000,0x3f000, - 0x3f004,0x3f000,0x3f000,0x3f000}; - if(!m3_hdmi_phy_ready() || !main_rate)return -EHOSTDOWN; - for(unsigned int i=0;i<12;i+=3) - if(drive[i] || drive[i+1]>3 || drive[i+2]>3 || drive[i+1]+drive[i+2]>3)return -EINVAL; - /* Until native four-lane orientation order is qualified, require equal - * settings on each pair. This makes both possible pair orders identical. */ - if(memcmp(drive,drive+6,6*sizeof(*drive)))return -EOPNOTSUPP; - for(unsigned int i=0;i<4;i++){ - u32 preset=table[drive[3*i+1]*4+drive[3*i+2]]; - int ret=m3_hdmi_atc_dp_drive_preset(core,i/2,i&1,preset); - if(ret){result=ret;return ret;} - pr_info("m3_hdmi_phy: drive lane=%u voltage=%u emphasis=%u preset=%#x\n",i,drive[3*i+1],drive[3*i+2],preset); - } - return 0; -} -EXPORT_SYMBOL_GPL(m3_hdmi_phy_set_drive); -static int phy_start(bool lanes) -{ - int ret=m3_hdmi_atc_power_up_reset(core); - if(ret)return ret; - for (unsigned int i = 0; i < ARRAY_SIZE(offsets); i++) { - m3_hdmi_atc_component_update(core, offsets[i], masks[i], calibration[i]); - if ((readl(core + offsets[i]) & masks[i]) != calibration[i]) { ret = -EIO; return ret; } - } - if (lanes) { - /* Native mode3 is four-lane DP, fixed HDMI orientation. */ - m3_hdmi_atc_component_update(core, 0x60, 0xfff, 0x492); - m3_hdmi_atc_component_update(core, 0x64, 0x1f, 0x14); - m3_hdmi_atc_component_update(core, 0x64, 0x1ffe0, 0x22000); - } - /* Native phy_init_partC releases bit4, waits100us, then cmn_init. */ - udelay(10); - m3_hdmi_atc_component_update(core, 0x20000, 0, BIT(4)); - udelay(100); - ret = m3_hdmi_atc_common_start(core); - if (ret) return ret; - m3_hdmi_atc_aux_start(core); - if ((readl(core + 0x16400) & 0x30f) != 0x10f || (readl(core + 0x16000) & 1)) ret = -EIO; - pr_info("m3_hdmi_phy: common ready=%#x AUX ctrl=%#x cfg=%#x result=%d\n", - readl(core + 0x804), readl(core + 0x16400), readl(core + 0x16000), ret); - if (!ret && main_rate) { - ret = m3_hdmi_atc_dp_main_prepare(core, 3, main_rate, false); - pr_info("m3_hdmi_phy: main rate=%u result=%d request=%#x PLL=%#x lanes=%#x xbar=%#x\n", - main_rate, ret, readl(core + 0x2000), readl(core + 0x7034), - readl(core + 0x60), readl(core + 0x64)); - } - return ret; -} -int m3_hdmi_phy_activate(void) -{ - int ret; - if(result || !core || !common)return -EHOSTDOWN; - if(phy_active)return m3_hdmi_phy_ready()?0:-EIO; - /* DPTX will choose the link rate; restore common/AUX and four-lane mode. */ - main_rate=0; - ret=phy_start(true); - if(!ret)phy_active=true;else result=ret; - pr_info("m3_hdmi_phy: reactivation result=%d\n",ret); - return ret; -} -EXPORT_SYMBOL_GPL(m3_hdmi_phy_activate); -static int __init probe(void) -{ - struct of_phandle_args args = {}; - const char *label; - int ret; - if (main_rate && (!common || main_rate != 10)) return -EINVAL; - if (common) { - if (roundtrip || calibration_count != ARRAY_SIZE(offsets)) return -EINVAL; - for (unsigned int i = 0; i < ARRAY_SIZE(offsets); i++) - if (calibration[i] & ~masks[i]) return -EINVAL; - } - if (!of_machine_is_compatible("apple,j514s") || - !of_machine_is_compatible("apple,t6030")) return -ENODEV; - args.np = of_find_node_by_path("/soc/power-management@350700000/power-controller@4f8"); - if (!args.np) return -ENODEV; - if (of_property_read_string(args.np, "label", &label) || strcmp(label, "atc3_common")) { - of_node_put(args.np); return -EINVAL; - } - owner = root_device_register("m3-hdmi-phy"); - if (IS_ERR(owner)) { of_node_put(args.np); return PTR_ERR(owner); } - ret = of_genpd_add_device(&args, owner); - of_node_put(args.np); - if (ret) { root_device_unregister(owner); return ret; } - pm_runtime_set_suspended(owner); - pm_runtime_enable(owner); - ret = pm_runtime_resume_and_get(owner); - if (ret < 0) goto done; - if (!request_mem_region(0x1303000000ULL, 0x30000, "m3-hdmi-phy")) { - ret = -EBUSY; goto done; - } - core = ioremap_np(0x1303000000ULL, 0x30000); - if (!core) { ret = -ENOMEM; goto done; } - pr_info("m3_hdmi_phy: ATC3 ctrl=%#x status=%#x misc=%#x\n", - readl(core + 0x20000), readl(core + 0x20004), readl(core + 0x20008)); - ret = m3_hdmi_atc_power_idle(core); - if (ret || (!roundtrip && !common)) goto done; - if(roundtrip)ret = m3_hdmi_atc_power_up_reset(core); - if (!ret && roundtrip) ret = m3_hdmi_atc_power_down(core); - if (!ret && common) { - ret=phy_start(!!main_rate); - } -done: - phy_active=!ret && common; - result = ret; - pr_info("m3_hdmi_phy: result=%d roundtrip=%u; resources retained until reboot\n", ret, roundtrip); - return 0; -} -module_init(probe); -MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("J514S HDMI ATC3 power and common/AUX diagnostic"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_common.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_common.h deleted file mode 100644 index 6b2eb453e9c50..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_common.h +++ /dev/null @@ -1,65 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* Basic common/AUX sequence agrees with exact M3 26A428 Unicorn traces. - * Shared M4 implementation copied after comparison, not its calibrations. - * Main-link and lane routing are deliberately absent at this stage. - */ -static void m3_hdmi_atc_component_update(void __iomem *regs, u32 offset, u32 clear, u32 set) -{ - writel((readl(regs + offset) & ~clear) | set, regs + offset); -} -static int m3_hdmi_atc_common_start(void __iomem *regs) -{ - u32 status; - m3_hdmi_atc_component_update(regs, 8, 0, 4); - m3_hdmi_atc_component_update(regs, 8, 0, 8); - udelay(1); - m3_hdmi_atc_component_update(regs, 8, 0, 1); - m3_hdmi_atc_component_update(regs, 8, 0, 2); - udelay(1); - m3_hdmi_atc_component_update(regs, 8, 0x10, 0); - m3_hdmi_atc_component_update(regs, 8, 0, 0x20); - udelay(1); - m3_hdmi_atc_component_update(regs, 0xa00, 0, 2); - udelay(15); - return readl_poll_timeout(regs + 0x804, status, status & 1, 1, 100); -} - -#define ATC_AUX_CTRL 0x16400 -#define ATC_AUX_CFG 0x16000 - -static void m3_hdmi_atc_aux_start(void __iomem *regs) -{ - m3_hdmi_atc_component_update(regs, ATC_AUX_CTRL, 0, 3); - udelay(1); - m3_hdmi_atc_component_update(regs, ATC_AUX_CTRL, 0, 0xc); - udelay(1); - m3_hdmi_atc_component_update(regs, ATC_AUX_CTRL, 0x300, 0x100); - udelay(1); - m3_hdmi_atc_component_update(regs, ATC_AUX_CFG, 1, 0); - udelay(1); -} - - -static void m3_hdmi_atc_common_stop(void __iomem *regs) -{ - m3_hdmi_atc_component_update(regs, 0xa00, 2, 0); - udelay(1); - m3_hdmi_atc_component_update(regs, 8, 0, 0x10); - udelay(1); - m3_hdmi_atc_component_update(regs, 8, 1, 0); - udelay(1); - m3_hdmi_atc_component_update(regs, 8, 4, 0); - udelay(1); -} - -static void m3_hdmi_atc_aux_stop(void __iomem *regs) -{ - m3_hdmi_atc_component_update(regs, ATC_AUX_CFG, 0, 1); - udelay(1); - m3_hdmi_atc_component_update(regs, ATC_AUX_CTRL, 0, 0x200); - udelay(1); - m3_hdmi_atc_component_update(regs, ATC_AUX_CTRL, 8, 0); - udelay(1); - m3_hdmi_atc_component_update(regs, ATC_AUX_CTRL, 2, 0); - udelay(1); -} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-drive.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-drive.h deleted file mode 100644 index 9eb48d09c1330..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-drive.h +++ /dev/null @@ -1,32 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* Shared per-device sequencer: caller owns MMIO, power and device lock. - * Independent packed DP drive-preset register operations. Native behavioral - * reference: AppleM3TypeCPhy::aciophy_dptx_set_{txeq,rxtxeq}_preset, - * 26A428 image; boolean argument false enables the packed preset. Offline qualified only; - * the standalone HBR test used the fixed initial preset, not sink training. - * Caller supplies a qualified 24-bit preset and owns the selected PHY pair. - */ -#ifndef APPLE_ATCPHY_M3_DP_DRIVE_H -#define APPLE_ATCPHY_M3_DP_DRIVE_H - -static int m3_hdmi_atc_dp_drive_preset(void __iomem *regs, unsigned int lane, bool tx, u32 packed) -{ - u32 select, amplitude, base; - if (lane > 1 || packed & 0xff000000) - return -EINVAL; - base = lane * 0x7000; - select = ((packed & 3) << 14) | ((packed >> 2) & 0xf); - amplitude = (((packed >> 6) & 0x3ffff) << 1) | 1; - if (tx) { - m3_hdmi_dp_pll_update(regs, base + 0xc0d0, 0x100000, 0x100000); - m3_hdmi_dp_pll_update(regs, base + 0xc050, 0xc00f, select); - m3_hdmi_dp_pll_update(regs, base + 0xc0d4, 0x10000000, 0x10000000); - m3_hdmi_dp_pll_update(regs, base + 0xc0d0, 0x7ffff, amplitude); - } else { - m3_hdmi_dp_pll_update(regs, base + 0x9210, 0xc00f, select); - m3_hdmi_dp_pll_update(regs, base + 0x9248, 0x10000000, 0x10000000); - m3_hdmi_dp_pll_update(regs, base + 0x9244, 0x7ffff, amplitude); - } - return 0; -} -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-main.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-main.h deleted file mode 100644 index cf3ea1720ba66..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_dp-main.h +++ /dev/null @@ -1,159 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* Adapted from the M4 sequencer, checked against J514S 26A428 native - * main_init and its callees. Experimental; no trained link claim. - */ -#ifndef APPLE_ATCPHY_M3_DP_MAIN_H -#define APPLE_ATCPHY_M3_DP_MAIN_H -#include "m3_hdmi_phy_dp-pll.h" -#include "m3_hdmi_phy_lane-power.h" - -static void m3_hdmi_dp_lane_update(void __iomem *regs, unsigned int lane, u32 offset, u32 mask, u32 value) -{ - m3_hdmi_dp_pll_update(regs, offset + lane * 0x7000, mask, value); -} - -static void m3_hdmi_dp_lane_tx_initial_eq(void __iomem *regs, unsigned int lane) -{ - m3_hdmi_dp_lane_update(regs, lane, 0xc050, 0xc00f, 0); - m3_hdmi_dp_lane_update(regs, lane, 0xc0d4, 0x10000000, 0x10000000); - m3_hdmi_dp_lane_update(regs, lane, 0xc0d0, 0x7ffff, 1); -} - -static void m3_hdmi_dp_lane_rx_initial_eq(void __iomem *regs, unsigned int lane) -{ - m3_hdmi_dp_lane_update(regs, lane, 0x9210, 0xc00f, 0); - m3_hdmi_dp_lane_update(regs, lane, 0x9248, 0x10000000, 0x10000000); - m3_hdmi_dp_lane_update(regs, lane, 0x9244, 0x7ffff, 0x1f81); -} - -static void m3_hdmi_dp_lane_tx_prepare(void __iomem *regs, unsigned int lane, unsigned int rate) -{ - m3_hdmi_atc_lane_tx_power(regs, lane, true); - writel(0x0aafb800, regs + 0xd044 + lane * 0x7000); - ndelay(500); - m3_hdmi_dp_lane_update(regs, lane, 0xd044, 0x1000000, 0x1000000); - ndelay(250); - m3_hdmi_dp_lane_update(regs, lane, 0xd044, 0x4000000, 0x4000000); - ndelay(250); - m3_hdmi_dp_lane_update(regs, lane, 0xd044, 0x7c0, 0x300); - ndelay(250); - m3_hdmi_dp_lane_update(regs, lane, 0xd044, 0x6c000, 0); - ndelay(250); - m3_hdmi_dp_lane_update(regs, lane, 0xc0d0, 0x100000, 0x100000); - m3_hdmi_dp_lane_tx_initial_eq(regs, lane); - m3_hdmi_dp_lane_update(regs, lane, 0xd044, 0x100000, 0x100000); - ndelay(250); - m3_hdmi_dp_lane_update(regs, lane, 0xd044, 0x400000, 0x400000); - udelay(1); - m3_hdmi_dp_lane_update(regs, lane, 0xd004, 0x3000000, 0x2000000); - m3_hdmi_dp_lane_update(regs, lane, 0xd004, 0x30000000, 0x30000000); - m3_hdmi_dp_lane_update(regs, lane, 0xd00c, 0x7000000, rate == 6 ? 0x5000000 : 0x4000000); - m3_hdmi_dp_lane_update(regs, lane, 0xd010, 0x600, 0x600); - m3_hdmi_dp_lane_update(regs, lane, 0xd01c, 0xc00000, 0x800000); -} - -static void m3_hdmi_dp_lane_rx_prepare(void __iomem *regs, unsigned int lane, unsigned int rate) -{ - m3_hdmi_dp_lane_update(regs, lane, 0x9160, 1, 1); - m3_hdmi_dp_lane_update(regs, lane, 0x91dc, 0x18000, 0x18000); - m3_hdmi_dp_lane_update(regs, lane, 0xb0a4, 3, 2); - m3_hdmi_dp_lane_update(regs, lane, 0xb0a8, 0x18000, 0x10000); - m3_hdmi_dp_lane_update(regs, lane, 0xb06c, 0x18000, 0x10000); - m3_hdmi_dp_lane_update(regs, lane, 0xb0d4, 0x1f1f80, 0x121180); - m3_hdmi_dp_lane_update(regs, lane, 0xb064, 0xfc00, 0xb800); - m3_hdmi_dp_lane_update(regs, lane, 0xb0d8, 7, 4); - m3_hdmi_dp_lane_update(regs, lane, 0xb0dc, 0x18, 0); - m3_hdmi_dp_lane_update(regs, lane, 0xb0d4, 0xe000, 0xc000); - ndelay(100); - m3_hdmi_atc_lane_rx_power(regs, lane, true); - m3_hdmi_dp_lane_update(regs, lane, 0xb06c, 0x800, 0x800); - m3_hdmi_dp_lane_update(regs, lane, 0x91d4, 3, 1); - ndelay(10); - m3_hdmi_dp_lane_update(regs, lane, 0x91d4, 2, 2); - m3_hdmi_dp_lane_update(regs, lane, 0xb06c, 0x600, 0xc00); - ndelay(100); - m3_hdmi_dp_lane_update(regs, lane, 0xb0d4, 0x50000, 0x50000); - m3_hdmi_dp_lane_update(regs, lane, 0xb0d8, 1, 1); - ndelay(500); - m3_hdmi_dp_lane_update(regs, lane, 0xb0d4, 0x40000, 0); - ndelay(200); - m3_hdmi_dp_lane_update(regs, lane, 0xb068, 0x180, 0x180); - ndelay(200); - m3_hdmi_dp_lane_update(regs, lane, 0xb068, 0x60, 0x60); - ndelay(200); - m3_hdmi_dp_lane_update(regs, lane, 0xb0d8, 3, 3); - udelay(1); - m3_hdmi_dp_lane_update(regs, lane, 0xb0d8, 3, 2); - udelay(1); - m3_hdmi_dp_lane_update(regs, lane, 0xb0d8, 3, 0); - ndelay(750); - m3_hdmi_dp_lane_update(regs, lane, 0xb0a8, 0x60000, 0x60000); - udelay(1); - m3_hdmi_dp_lane_update(regs, lane, 0xb06c, 0x6000, 0x4000); - m3_hdmi_dp_lane_rx_initial_eq(regs, lane); - m3_hdmi_dp_lane_update(regs, lane, 0xb0a8, 0x380000, rate == 6 ? 0x280000 : 0x200000); - m3_hdmi_dp_lane_update(regs, lane, 0xb0a8, 0xc00000, 0x800000); - ndelay(50); - m3_hdmi_dp_lane_update(regs, lane, 0xb030, 0xc00, 0xc00); - ndelay(100); -} - -static int m3_hdmi_atc_dp_main_prepare(void __iomem *regs, unsigned int pair_mask, unsigned int rate, bool ssc) -{ - int ret; - if ((!pair_mask || pair_mask > 3) || - (rate != 6 && rate != 10 && rate != 20 && rate != 30) || ssc) - return -EOPNOTSUPP; - ret = m3_hdmi_dp_pll_prepare(regs, pair_mask, rate, ssc); - if (ret) - return ret; - for (unsigned int lane = 0; lane < 2; lane++) { - if (!(pair_mask & (1u << lane))) continue; - m3_hdmi_dp_lane_tx_prepare(regs, lane, rate); - m3_hdmi_dp_lane_rx_prepare(regs, lane, rate); - } - m3_hdmi_dp_pll_update(regs, 0x7000, 1, 0); - return 0; -} - -static int m3_hdmi_atc_dp_main_stop(void __iomem *regs, unsigned int pair_mask) -{ - unsigned int lane; - if (!pair_mask || pair_mask > 3) - return -EOPNOTSUPP; - m3_hdmi_dp_pll_update(regs, 0x7000, 1, 1); - for(lane=0;lane<2;lane++){ - if(!(pair_mask & (1u<> 1) & 1) == expected, 1, timeout_us); -} - -/* Capture the original ACK for both phases, not the current request bit. */ -static u32 m3_hdmi_dp_pll_request_begin(void __iomem *regs) -{ - u32 old = readl(regs + DP_PLL_REQUEST); - u32 ack = (old >> 1) & 1; - writel((old & 0xe0000006) | 0x10000000 | (ack ^ 1), - regs + DP_PLL_REQUEST); - return ack; -} - -static int m3_hdmi_dp_pll_prepare(void __iomem *regs, unsigned int pair_mask, unsigned int rate, bool ssc) -{ - u32 freq_a, freq_b, freq_c, ack, status; - int ret; - /* Spread spectrum is outside this diagnostic. */ - if ((!pair_mask || pair_mask > 3) || ssc) - return -EOPNOTSUPP; - switch (rate) { - case 0x06: freq_a = 0x1e0e021c; freq_b = 0; freq_c = 0x156600; break; - case 0x0a: freq_a = 0x1e0e01c2; freq_b = 0x07fffffe; freq_c = 0x155200; break; - case 0x14: freq_a = 0x1e0e01c2; freq_b = 0x07fffffe; freq_c = 0x554800; break; - case 0x1e: freq_a = 0x1e0e02a3; freq_b = 0x0bff7ffc; freq_c = 0x564800; break; - default: return -EOPNOTSUPP; - } - m3_hdmi_dp_pll_update(regs, 0x2234, 3, 0); - m3_hdmi_dp_pll_update(regs, 0x7000, 0x1ff0, 0x490); - writel(freq_a, regs + 0x2080); - writel(freq_b, regs + 0x2084); - writel(freq_c, regs + 0x2088); - m3_hdmi_dp_pll_update(regs, 0x2208, 0x1f0000, 0x70000); - m3_hdmi_dp_pll_update(regs, 0x2218, 1, 1); - m3_hdmi_dp_pll_update(regs, 0x2200, 8, 8); - ack = m3_hdmi_dp_pll_request_begin(regs); - ret = m3_hdmi_dp_pll_ack(regs, ack ^ 1, 30); - if (ret) - return ret; - ret = readl_poll_timeout(regs + DP_PLL_STATUS, status, - status & 8, 1, 20); - if (ret) - return ret; - m3_hdmi_dp_pll_update(regs, DP_PLL_REQUEST, 0x1ffffff9, 0x10014000 | ack); - ret = m3_hdmi_dp_pll_ack(regs, ack, 5); - if (ret) - return ret; - for (unsigned int lane = 0; lane < 2; lane++) - if (pair_mask & (1u << lane)) - m3_hdmi_dp_pll_update(regs, 0xb064 + lane * 0x7000, 0xc, 8); - m3_hdmi_dp_pll_update(regs, 0x7000, 0xc, 0xc); - udelay(1); - return 0; -} - -static int m3_hdmi_dp_pll_stop_after_lanes(void __iomem *regs) -{ - u32 ack, status; - int ret; - m3_hdmi_dp_pll_update(regs, 0x7000, 0xc, 8); - ack = m3_hdmi_dp_pll_request_begin(regs); - ret = m3_hdmi_dp_pll_ack(regs, ack ^ 1, 500); - if (ret) - return ret; - m3_hdmi_dp_pll_update(regs, DP_PLL_REQUEST, 0x1ffffff9, 0x10000018 | ack); - ret = m3_hdmi_dp_pll_ack(regs, ack, 500); - if (ret) - return ret; - ret = readl_poll_timeout(regs + DP_PLL_STATUS, status, - !(status & 8), 1, 5); - if (ret) - return ret; - m3_hdmi_dp_pll_update(regs, 0x2200, 8, 0); - return 0; -} -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_lane-power.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_lane-power.h deleted file mode 100644 index c58be84d3a9a6..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_lane-power.h +++ /dev/null @@ -1,75 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* Adapted from the M4 sequencer, checked against J514S 26A428 native - * main_init and its callees. Experimental; no trained link claim. - */ -#ifndef APPLE_ATCPHY_M3_LANE_POWER_H -#define APPLE_ATCPHY_M3_LANE_POWER_H - -/* M3 uses a single read per selected-lane bit update. */ -static void m3_hdmi_atc_lane_power_field(void __iomem *regs, u32 offset, - unsigned int shift, unsigned int lane, bool enable) -{ - u32 value = readl(regs + offset), bit = 1u << (shift + lane); - writel(enable ? value | bit : value & ~bit, regs + offset); -} - -static int m3_hdmi_atc_lane_tx_power(void __iomem *regs, unsigned int lane, bool enable) -{ - if (lane > 1) - return -EINVAL; - if (enable) { - m3_hdmi_atc_lane_power_field(regs, 0x10c, 4, lane, true); - m3_hdmi_atc_lane_power_field(regs, 0x10c, 6, lane, true); - udelay(1); - m3_hdmi_atc_lane_power_field(regs, 0x10c, 0, lane, true); - m3_hdmi_atc_lane_power_field(regs, 0x10c, 2, lane, true); - udelay(1); - m3_hdmi_atc_lane_power_field(regs, 0x10c, 8, lane, false); - m3_hdmi_atc_lane_power_field(regs, 0x10c, 10, lane, true); - udelay(1); - } else { - m3_hdmi_atc_lane_power_field(regs, 0x10c, 8, lane, true); - udelay(1); - m3_hdmi_atc_lane_power_field(regs, 0x10c, 0, lane, false); - udelay(1); - m3_hdmi_atc_lane_power_field(regs, 0x10c, 4, lane, false); - udelay(1); - } - return 0; -} - -static int m3_hdmi_atc_lane_rx_power(void __iomem *regs, unsigned int lane, bool enable) -{ - if (lane > 1) - return -EINVAL; - if (enable) { - m3_hdmi_atc_lane_power_field(regs, 0xc, 4, lane, true); - m3_hdmi_atc_lane_power_field(regs, 0xc, 6, lane, true); - udelay(1); - m3_hdmi_atc_lane_power_field(regs, 0xc, 0, lane, true); - m3_hdmi_atc_lane_power_field(regs, 0xc, 2, lane, true); - udelay(1); - m3_hdmi_atc_lane_power_field(regs, 8, 10, lane, true); - m3_hdmi_atc_lane_power_field(regs, 8, 12, lane, true); - udelay(1); - m3_hdmi_atc_lane_power_field(regs, 8, 6, lane, true); - m3_hdmi_atc_lane_power_field(regs, 8, 8, lane, true); - udelay(1); - m3_hdmi_atc_lane_power_field(regs, 8, 14, lane, false); - m3_hdmi_atc_lane_power_field(regs, 8, 16, lane, true); - udelay(1); - } else { - m3_hdmi_atc_lane_power_field(regs, 8, 14, lane, true); - udelay(1); - m3_hdmi_atc_lane_power_field(regs, 8, 6, lane, false); - udelay(1); - m3_hdmi_atc_lane_power_field(regs, 8, 10, lane, false); - udelay(1); - m3_hdmi_atc_lane_power_field(regs, 0xc, 0, lane, false); - udelay(1); - m3_hdmi_atc_lane_power_field(regs, 0xc, 4, lane, false); - udelay(1); - } - return 0; -} -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_power.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_power.h deleted file mode 100644 index 6bef5442d7247..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_phy_power.h +++ /dev/null @@ -1,81 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -#ifndef M3_HDMI_PHY_POWER_H -#define M3_HDMI_PHY_POWER_H -/* Copied from M4 power component after matching the M3 26A428 native - * powerup trace (typec-power, 2026-09-21). Register masks/order agree. - * Power/reset only; the PHY reset remains asserted throughout. - * Startup follows native AppleT8122TypeCPhy::aciophy_phy_powerup_rst. - * Reverse transition follows Sven Peter / Asahi atcphy_power_off, using - * the power/misc bits independently confirmed in the J514S ATC3 native routines. - * No USB2, tunables, lane routing, common initialization or AUX changes. - */ -static int m3_hdmi_atc_power_idle(void __iomem *regs) -{ - return readl(regs + 0x20000) == 4 && - !readl(regs + 0x20004) && !readl(regs + 0x20008) ? 0 : -EBUSY; -} - -static void m3_hdmi_atc_power_update(void __iomem *regs, u32 offset, u32 clear, u32 set) -{ - u32 value = readl(regs + offset); - writel((value & ~clear) | set, regs + offset); - udelay(1); -} - -static int m3_hdmi_atc_power_wait(void __iomem *regs, u32 mask, u32 value) -{ - u32 status; - return readl_poll_timeout(regs + 0x20004, status, - (status & mask) == value, 1, 1000); -} - -static int m3_hdmi_atc_power_up_reset(void __iomem *regs) -{ - int ret; - /* Caller has checked idle and pinned its powered device before entry. */ - ndelay(500); - m3_hdmi_atc_power_update(regs, 0x20008, 0, 1); - m3_hdmi_atc_power_update(regs, 0x20000, 0, 1); - ret = m3_hdmi_atc_power_wait(regs, 1, 1); - if (ret) - return ret; - m3_hdmi_atc_power_update(regs, 0x20000, 0, 2); - ret = m3_hdmi_atc_power_wait(regs, 2, 2); - if (ret) - return ret; - m3_hdmi_atc_power_update(regs, 0x20000, 4, 0); - m3_hdmi_atc_power_update(regs, 0x20000, 0, 8); - if (readl(regs + 0x20000) != 0xb || readl(regs + 0x20008) != 1) - return -EIO; - pr_info("m3_hdmi_phy: reset-held PHY powerup reached control=0xb status=%#x misc=1\n", - readl(regs + 0x20004)); - return 0; -} - -static int m3_hdmi_atc_power_down(void __iomem *regs) -{ - int ret; - /* Keep PHY reset asserted, restore clamp, then remove APB reset release - * and the big/small power requests in that order. No USB2 reset call. - */ - m3_hdmi_atc_power_update(regs, 0x20000, 0x10, 0); - m3_hdmi_atc_power_update(regs, 0x20000, 0, 4); - m3_hdmi_atc_power_update(regs, 0x20008, 5, 0); - m3_hdmi_atc_power_update(regs, 0x20000, 8, 0); - m3_hdmi_atc_power_update(regs, 0x20000, 2, 0); - ret = m3_hdmi_atc_power_wait(regs, 2, 0); - if (ret) - return ret; - m3_hdmi_atc_power_update(regs, 0x20000, 1, 0); - ret = m3_hdmi_atc_power_wait(regs, 1, 0); - if (ret) - return ret; - if (readl(regs + 0x20000) != 4 || readl(regs + 0x20004) || - readl(regs + 0x20008)) - return -EIO; - pr_info("m3_hdmi_phy: power roundtrip restored control=4 status=0 misc=0\n"); - return 0; /* Caller verifies PMGR before dropping the pin. */ -} - - -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_xbar.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_xbar.c deleted file mode 100644 index c84ac3aa7ae20..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_xbar.c +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* J514S DCPEXT0->ATC3 crossbar. Acquire ATC3_COMMON; the caller also holds - * DCPEXT0 power. Retain all ownership until reboot, including failed writes. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "m3_hdmi_control.h" -#include "m3_hdmi_xbar.h" -static int result = -EINPROGRESS; -module_param(result, int, 0400); -static bool roundtrip; -module_param(roundtrip, bool, 0400); -static struct device *owner; -static struct m3_hdmi_dpxbar xbar; -int m3_hdmi_xbar_deactivate(void) -{ - if(result || !xbar.regs)return -EHOSTDOWN; - return m3_hdmi_dpxbar_down(&xbar); -} -EXPORT_SYMBOL_GPL(m3_hdmi_xbar_deactivate); -int m3_hdmi_xbar_activate(void) -{ - if(result || !xbar.regs)return -EHOSTDOWN; - return xbar.selected?0:m3_hdmi_dpxbar_up(&xbar); -} -EXPORT_SYMBOL_GPL(m3_hdmi_xbar_activate); -static int __init probe(void) -{ - struct of_phandle_args args = {}; - const char *label; - int ret; - if (!of_machine_is_compatible("apple,j514s") || - !of_machine_is_compatible("apple,t6030")) return -ENODEV; - args.np = of_find_node_by_path("/soc/power-management@350700000/power-controller@4f8"); - if (!args.np) return -ENODEV; - if (of_property_read_string(args.np, "label", &label) || strcmp(label, "atc3_common")) { - of_node_put(args.np); return -EINVAL; - } - owner = root_device_register("m3-hdmi-xbar"); - if (IS_ERR(owner)) { of_node_put(args.np); return PTR_ERR(owner); } - ret = of_genpd_add_device(&args, owner); - of_node_put(args.np); - if (ret) { root_device_unregister(owner); return ret; } - pm_runtime_set_suspended(owner); - pm_runtime_enable(owner); - ret = pm_runtime_resume_and_get(owner); - if (ret < 0) goto done; - if (!request_mem_region(0x130304c000ULL, 0x4000, "m3-hdmi-xbar")) { - ret = -EBUSY; goto done; - } - xbar.regs = ioremap_np(0x130304c000ULL, 0x4000); - if (!xbar.regs) { ret = -ENOMEM; goto done; } - ret = m3_hdmi_dpxbar_up(&xbar); - if (!ret && roundtrip) ret = m3_hdmi_dpxbar_down(&xbar); -done: - result = ret; - pr_info("m3_hdmi_xbar: result=%d; resources retained until reboot\n", ret); - return 0; -} -module_init(probe); -MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("J514S HDMI ATC3 display crossbar diagnostic"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_xbar.h b/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_xbar.h deleted file mode 100644 index f6d26824d5b50..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_hdmi_xbar.h +++ /dev/null @@ -1,145 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* M4 T602x sequencer adapted to captured J514S ATC3 routing. Native - * 26A428 connect/up/down model confirms source8000 -> destination8030. - * Status mirrors in that offline model do not prove real clock readiness. - */ -#ifndef M3_HDMI_DPXBAR_H -#define M3_HDMI_DPXBAR_H -/* Caller holds its device lock and a runtime PM reference throughout each - * operation. selected records an attempted route until a verified idle state; - * it is deliberately retained after uncertain writes or failed shutdown. */ -struct m3_hdmi_dpxbar { - void __iomem *regs; - bool selected; - /* J514S DCPEXT0 UFP endpoint0/port0 maps to selector0. */ - u32 selector; -}; -static const u32 dpxbar_offsets[] = { 0, 4, 8, 0xc, 0x10, 0x14, 0x18, - 0x1c, 0x20, 0x24, 0x28, 0x2c, 0x30, 0x34, - 0x800, 0x804, 0x808, 0x80c, 0x810, 0x814, 0x818, 0x81c }; -static const u32 dpxbar_idle[] = { 0, 0x1ff, 0, 0, 0, 0x1ff, 0, - 0, 0, 0x111, 0, 0, 0, 0, 0, 0x1ff, 0, 0, 0x1ff, 0, 0, 0x111 }; - -static int m3_hdmi_dpxbar_idle(struct m3_hdmi_dpxbar *xbar) -{ - for (unsigned int i = 0; i < ARRAY_SIZE(dpxbar_offsets); i++) { - u32 value = readl(xbar->regs + dpxbar_offsets[i]); - if (dpxbar_offsets[i] == 0x34 && value == 0x100) - continue; - if (value != dpxbar_idle[i]) { - pr_err("m3_hdmi_dpxbar: idle mismatch offset=%#x value=%#x expected=%#x\n", - dpxbar_offsets[i], value, dpxbar_idle[i]); - return -EBUSY; - } - } - return 0; -} - -static void m3_hdmi_dpxbar_update(struct m3_hdmi_dpxbar *xbar, u32 offset, u32 clear, u32 set) -{ - writel((readl(xbar->regs + offset) & ~clear) | set, - xbar->regs + offset); -} - -static int m3_hdmi_dpxbar_up(struct m3_hdmi_dpxbar *xbar) -{ - u32 selector = xbar->selector; - u32 bit, shift; - int ret; - - if (selector != 0 && selector != 2) - return -EINVAL; - bit = 1U << selector; - shift = 2 * selector; - ret = m3_hdmi_dpxbar_idle(xbar); - if (ret) - return ret; - /* Caller already pins the powered PHY and published DART tables. */ - xbar->selected = true; - m3_hdmi_dpxbar_update(xbar, 0x30, 0xf00000, selector << 20); - m3_hdmi_dpxbar_update(xbar, 0x30, 0xf00, selector << 8); - if (readl(xbar->regs + 0x30) != (selector << 20 | selector << 8)) - return -EIO; - /* Native connection-up uses source bit 1 and physical-output bit 0x100. - * Clock class 1 follows native pclkForConnection for attributes 0x100. - */ - m3_hdmi_dpxbar_update(xbar, 4, bit, 0); - m3_hdmi_dpxbar_update(xbar, 0x14, bit, 0); - m3_hdmi_dpxbar_update(xbar, 0x24, 0x100, 0); - udelay(1); - if (readl(xbar->regs + 0x804) & bit || - readl(xbar->regs + 0x810) & bit || - readl(xbar->regs + 0x81c) & 0x100) - return -ETIMEDOUT; - m3_hdmi_dpxbar_update(xbar, 8, 0, bit); - m3_hdmi_dpxbar_update(xbar, 0x18, 3U << shift, 1U << shift); - m3_hdmi_dpxbar_update(xbar, 0x28, 0x300, 0x100); - m3_hdmi_dpxbar_update(xbar, 0, 0, bit); - m3_hdmi_dpxbar_update(xbar, 0xc, 0, bit); - m3_hdmi_dpxbar_update(xbar, 0x1c, 0, 0x100); - m3_hdmi_dpxbar_update(xbar, 0x34, 0, 0x100); - m3_hdmi_dpxbar_update(xbar, 0x2c, 0, bit); - pr_info("m3_hdmi_dpxbar: selector%u up mux=%#x source_clock=%#x output_clock=%#x\n", - selector, readl(xbar->regs + 0x30), readl(xbar->regs), - readl(xbar->regs + 0x1c)); - if (readl(xbar->regs) != bit || readl(xbar->regs + 4) != (0x1ff & ~bit) || - readl(xbar->regs + 8) != bit || readl(xbar->regs + 0xc) != bit || - readl(xbar->regs + 0x14) != (0x1ff & ~bit) || - readl(xbar->regs + 0x18) != (1U << shift) || - readl(xbar->regs + 0x1c) != 0x100 || - readl(xbar->regs + 0x24) != 0x11 || - readl(xbar->regs + 0x28) != 0x100 || - readl(xbar->regs + 0x2c) != bit || - readl(xbar->regs + 0x34) != 0x100) - return -EIO; - return 0; -} - -static int m3_hdmi_dpxbar_down(struct m3_hdmi_dpxbar *xbar) -{ - u32 source, read_clock, output; - u32 selector = xbar->selector; - u32 bit, shift; - if (!xbar->selected) - return 0; - if (selector != 0 && selector != 2) - return -EINVAL; - bit = 1U << selector; - shift = 2 * selector; - m3_hdmi_dpxbar_update(xbar, 0x2c, bit, 0); - m3_hdmi_dpxbar_update(xbar, 0, bit, 0); - m3_hdmi_dpxbar_update(xbar, 0xc, bit, 0); - m3_hdmi_dpxbar_update(xbar, 0x1c, 0x100, 0); - udelay(1); - source = readl(xbar->regs + 0x800); - read_clock = readl(xbar->regs + 0x808); - output = readl(xbar->regs + 0x814); - /* Native takeConnectionDown logs these intermediate samples and - * continues with inverse-clock gates below (9135918..9135a08). - * They are not its completion condition. Require the full idle - * snapshot after those writes; never release on a stale final status. - */ - pr_info("m3_hdmi_dpxbar: intermediate stop status source=%#x read=%#x output=%#x\n", - source, read_clock, output); - m3_hdmi_dpxbar_update(xbar, 8, bit, 0); - m3_hdmi_dpxbar_update(xbar, 0x18, 3U << shift, 0); - m3_hdmi_dpxbar_update(xbar, 0x28, 0x300, 0); - m3_hdmi_dpxbar_update(xbar, 4, 0, bit); - m3_hdmi_dpxbar_update(xbar, 0x14, 0, bit); - m3_hdmi_dpxbar_update(xbar, 0x24, 0, 0x100); - /* Native takeConnectionDown leaves bit8 at0x34 set. Hardware in the - * retained trial ignores a zero write; do not require an unnecessary - * recovery for this native parked value. Clocks/status must be stopped, - * and subsequent RTKit AP+IOP quiescence still gates DART teardown. - * Restore the reversible selector fields after clocks stop. - */ - m3_hdmi_dpxbar_update(xbar, 0x30, 0xf00f00, 0); - if (m3_hdmi_dpxbar_idle(xbar)) - return -EIO; - xbar->selected = false; - pr_info("m3_hdmi_dpxbar: clocks stopped and selector restored; native parked register0x34=%#x\n", - readl(xbar->regs + 0x34)); - return 0; -} - -#endif diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_typec_probe.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_typec_probe.c deleted file mode 100644 index bebd56e2d6be1..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_typec_probe.c +++ /dev/null @@ -1,92 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Read-only J514S Type-C owner snapshot through the exported cache API. - * No private TIPD layout dependency, second bus owner, or PHY writes. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include "../../../../../usb/typec/tipd/tps6598x.h" - -static bool right_dp_ready, right_dp_partner, any_dp_ready, any_dp_partner; -static const char *controller_names[]={"1-0c","1-0a","2-08"}; -static struct spmi_device *controllers[3]; -static unsigned int dp_partner_mask, dp_ready_mask; -static int state_get(char *buffer, const struct kernel_param *parameter) -{ - struct cd321x_dp_snapshot state; - bool partner,ready; - int ret; - int (*snapshot)(struct device *, struct cd321x_dp_snapshot *); - snapshot = symbol_get(cd321x_get_dp_state); - if (!snapshot) return -EOPNOTSUPP; - right_dp_ready=right_dp_partner=any_dp_ready=any_dp_partner=false; - dp_partner_mask=dp_ready_mask=0; - for(unsigned int i=0;idev,&state); - if(ret){symbol_put(cd321x_get_dp_state);return ret;} - partner=(state.status & TPS_STATUS_PLUG_PRESENT) && - (state.data_status & TPS_DATA_STATUS_DP_CONNECTION) && - TPS_DATA_STATUS_DP_SPEC_PIN_ASSIGNMENT(state.data_status)==TPS_DATA_STATUS_DP_SPEC_PIN_ASSIGNMENT_D; - ready=partner && (state.dp_status & DP_STATUS_HPD_STATE); - any_dp_partner|=partner;any_dp_ready|=ready; - if(partner)dp_partner_mask|=BIT(i); - if(ready)dp_ready_mask|=BIT(i); - if(i==2){right_dp_partner=partner;right_dp_ready=ready;} - } - symbol_put(cd321x_get_dp_state); - if(parameter->arg==&dp_partner_mask || parameter->arg==&dp_ready_mask) - return param_get_uint(buffer, parameter); - return param_get_bool(buffer, parameter); -} -static const struct kernel_param_ops state_ops = { .get = state_get }; -module_param_cb(right_dp_ready, &state_ops, &right_dp_ready, 0400); -module_param_cb(right_dp_partner, &state_ops, &right_dp_partner, 0400); -module_param_cb(any_dp_ready, &state_ops, &any_dp_ready, 0400); -module_param_cb(any_dp_partner, &state_ops, &any_dp_partner, 0400); -module_param_cb(dp_partner_mask, &state_ops, &dp_partner_mask, 0400); -module_param_cb(dp_ready_mask, &state_ops, &dp_ready_mask, 0400); - -static int __init m3_typec_probe(void) -{ - struct device_node *node; - int ret, count = 0; - int (*snapshot)(struct device *, struct cd321x_dp_snapshot *); - - if (!of_machine_is_compatible("apple,j514s")) return -ENODEV; - snapshot = symbol_get(cd321x_get_dp_state); - if (!snapshot) return -EOPNOTSUPP; - for_each_compatible_node(node, NULL, "apple,sn201202x") { - struct spmi_device *sdev = spmi_find_device_by_of_node(node); - struct cd321x_dp_snapshot state; - if (!sdev) continue; - ret = snapshot(&sdev->dev, &state); - if (!ret) { - pr_info("m3_typec_probe: %s status=%#x data=%#x dp_status=%#x IRQs=%llu\n", - dev_name(&sdev->dev), state.status, state.data_status, - state.dp_status, (unsigned long long)state.hpd_irqs); - for(unsigned int i=0;idev),controller_names[i])){ - get_device(&sdev->dev);controllers[i]=sdev;break; - } - count++; - } - spmi_device_put(sdev); - } - symbol_put(cd321x_get_dp_state); - return count ? 0 : -ENODEV; -} -static void __exit m3_typec_exit(void) -{ - for(unsigned int i=0;i -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "../../../../../usb/typec/tipd/tps6598x.h" -#include "m3_hdmi_control.h" -#include "m3_hdmi_xbar.h" - -struct m3_usbc_port { - const char *controller_name, *phy_name; - resource_size_t address; - struct spmi_device *controller; - struct phy *phy; - struct m3_hdmi_dpxbar xbar; - int owner; -}; -static struct m3_usbc_port ports[] = { - { .controller_name="1-0c", .phy_name="dp0", .address=0x70304c000ULL, .owner=-1 }, - { .controller_name="1-0a", .phy_name="dp1", .address=0xb0304c000ULL, .owner=-1 }, - { .controller_name="2-08", .phy_name="dp", .address=0xf0304c000ULL, .owner=-1 }, -}; -struct m3_usbc_route { - unsigned int active_port, rate; - bool flipped, powered, prepared; - int result; -}; -static struct m3_usbc_route routes[2] = { - { .active_port=0 }, { .active_port=2 }, -}; -static DEFINE_MUTEX(route_lock); -static int result=-EINPROGRESS; -module_param(result,int,0400); -module_param_named(active_port,routes[1].active_port,uint,0400); -module_param_named(rate,routes[1].rate,uint,0400); -module_param_named(powered,routes[1].powered,bool,0400); -module_param_named(primary_active_port,routes[0].active_port,uint,0400); -module_param_named(primary_rate,routes[0].rate,uint,0400); -module_param_named(primary_powered,routes[0].powered,bool,0400); -module_param_named(primary_result,routes[0].result,int,0400); -module_param_named(secondary_result,routes[1].result,int,0400); - -static int partner(unsigned int index, bool *orientation, bool require_hpd) -{ - struct cd321x_dp_snapshot snapshot;u32 status,data,hpd;int ret; - int (*get_snapshot)(struct device *,struct cd321x_dp_snapshot *); - if(!ports[index].controller)return -ENODEV; - get_snapshot=symbol_get(cd321x_get_dp_state); - if(!get_snapshot)return -EOPNOTSUPP; - ret=get_snapshot(&ports[index].controller->dev,&snapshot); - symbol_put(cd321x_get_dp_state);if(ret)return ret; - status=snapshot.status;data=snapshot.data_status;hpd=snapshot.dp_status; - *orientation=!!(status&TPS_STATUS_PLUG_UPSIDE_DOWN); - return !!((status&TPS_STATUS_PLUG_PRESENT) && (data&TPS_DATA_STATUS_DP_CONNECTION) && - TPS_DATA_STATUS_DP_SPEC_PIN_ASSIGNMENT(data)==TPS_DATA_STATUS_DP_SPEC_PIN_ASSIGNMENT_D && - (!require_hpd || (hpd&DP_STATUS_HPD_STATE))); -} -/* Read only cached negotiation and software mode while an inactive port's - * clocks may be off. Never steal a PHY held by the other DCP session. */ -static int candidate(unsigned int slot, bool require_hpd) -{ - unsigned int index;bool orientation;int ret; - for(unsigned int n=0;n=0 && ports[index].owner!=slot)continue; - ret=partner(index,&orientation,require_hpd); - if(ret<0)return ret; - if(!ret)continue; - ret=phy_validate(ports[index].phy,PHY_MODE_DP,0,&opts); - if(ret==-ENOLINK || ret==-EBUSY)continue; - if(ret)return ret; - if(opts.dp.lanes==2)return index; - } - return -ENOLINK; -} -static int aux(struct m3_usbc_port *p, bool enable) -{ - int ret;int (*fn)(struct phy *,bool)=symbol_get(apple_atc_m3_dp_aux); - if(!fn)return -EOPNOTSUPP; - ret=fn(p->phy,enable);symbol_put(apple_atc_m3_dp_aux);return ret; -} -static int acquire(unsigned int slot, bool require_hpd) -{ - struct m3_usbc_route *r=&routes[slot];struct m3_usbc_port *p; - bool orientation;int ret,index; - if(r->powered)return 0; - if(r->rate)return -EBUSY; - index=candidate(slot,require_hpd);if(index<0)return index; - p=&ports[index]; - ret=partner(index,&orientation,false);if(ret!=1)return ret<0?ret:-ENOLINK; - ret=phy_power_on(p->phy);if(ret)return ret; - /* All later failures retain the exclusive claim and power for recovery. */ - p->owner=slot;r->active_port=index;r->flipped=orientation;r->powered=true; - p->xbar.selector=2*slot; - if(!p->xbar.regs){ - if(!request_mem_region(p->address,0x4000,"m3-usbc-dpxbar"))return -EBUSY; - p->xbar.regs=ioremap_np(p->address,0x4000); - if(!p->xbar.regs)return -ENOMEM; - } - ret=m3_hdmi_dpxbar_idle(&p->xbar); - pr_info("m3_usbc_phy: DCPEXT%u selected ATC%u target=%#x selector=%u flipped=%u result=%d\n", - slot,r->active_port,0x8000|(r->active_port<<4),p->xbar.selector,r->flipped,ret); - return ret; -} -static int activate(unsigned int slot) -{ - struct m3_usbc_route *r=&routes[slot];union phy_configure_opts opts={}; - struct m3_usbc_port *p;int ret;bool orientation; - if(!r->powered)return -EHOSTDOWN; - p=&ports[r->active_port]; - ret=partner(r->active_port,&orientation,false); - if(ret!=1 || orientation!=r->flipped)return ret<0?ret:-ENOLINK; - ret=phy_validate(p->phy,PHY_MODE_DP,0,&opts); - if(ret || opts.dp.lanes!=2)return ret ?: -ENOLINK; - return aux(p,true); -} -static int set_rate(unsigned int slot, unsigned int next) -{ - struct m3_usbc_route *r=&routes[slot];struct m3_usbc_port *p=&ports[r->active_port]; - union phy_configure_opts opts={};int ret; - if(next!=0 && next!=6 && next!=10 && next!=20)return -EOPNOTSUPP; - if(next==r->rate)return 0; - if(!r->powered)return -EHOSTDOWN; - if(p->xbar.selected)return -EBUSY; - if(next){ret=activate(slot);if(ret)return ret;} - opts.dp.set_rate=1;opts.dp.link_rate=next*270;opts.dp.lanes=2; - ret=phy_configure(p->phy,&opts); - if(ret)r->result=ret;else r->rate=next; - pr_info("m3_usbc_phy: DCPEXT%u DP rate=%u result=%d\n",slot,next,ret); - return ret; -} -static int check(unsigned int slot) -{ - if(slot>=ARRAY_SIZE(routes))return -EINVAL; - return result ?: routes[slot].result; -} -static int route_prepare(unsigned int slot) -{ - int ret;guard(mutex)(&route_lock); - ret=check(slot);if(ret)return ret; - if(routes[slot].prepared)return -EBUSY; - ret=acquire(slot,false); - if(!ret)ret=m3_hdmi_dpxbar_up(&ports[routes[slot].active_port].xbar); - routes[slot].result=ret;routes[slot].prepared=!ret; - return ret; -} -static int route_hpd(unsigned int slot) -{ - bool orientation;int ret;guard(mutex)(&route_lock); - ret=check(slot);if(ret)return ret; - if(routes[slot].powered){ - ret=partner(routes[slot].active_port,&orientation,true); - return ret<=0?ret:orientation==routes[slot].flipped; - } - ret=candidate(slot,true); - return ret==-ENOLINK?0:ret<0?ret:1; -} -static int route_target(unsigned int slot) -{ - int ret;guard(mutex)(&route_lock); - ret=check(slot);if(ret)return ret; - /* Keep the released destination until the next explicit acquisition. */ - return 0x8000 | (routes[slot].active_port<<4); -} -static int route_activate(unsigned int slot) -{ - int ret;guard(mutex)(&route_lock); - ret=check(slot);return ret ?: activate(slot); -} -static int route_xbar_down(unsigned int slot) -{ - int ret;guard(mutex)(&route_lock); - ret=check(slot);if(ret)return ret; - /* Once released, the old physical port may belong to the other session. */ - if(!routes[slot].powered)return 0; - return m3_hdmi_dpxbar_down(&ports[routes[slot].active_port].xbar); -} -static int route_xbar_up(unsigned int slot) -{ - struct m3_hdmi_dpxbar *xbar;int ret;guard(mutex)(&route_lock); - ret=check(slot);if(ret)return ret; - ret=acquire(slot,true); - if(!routes[slot].powered && (ret==-ENOLINK || ret==-EBUSY))return -EAGAIN; - if(ret)return ret; - xbar=&ports[routes[slot].active_port].xbar; - return xbar->selected?0:m3_hdmi_dpxbar_up(xbar); -} -static int route_set_rate(unsigned int slot, unsigned int next) -{ - int ret;guard(mutex)(&route_lock); - ret=check(slot);return ret ?: set_rate(slot,next); -} -static int route_deactivate(unsigned int slot) -{ - struct m3_usbc_route *r;struct m3_usbc_port *p;int ret; - guard(mutex)(&route_lock); - ret=check(slot);if(ret)return ret; - r=&routes[slot];if(!r->powered)return 0; - p=&ports[r->active_port]; - ret=m3_hdmi_dpxbar_down(&p->xbar); - if(!ret)ret=set_rate(slot,0); - if(!ret)ret=aux(p,false); - if(!ret)ret=phy_power_off(p->phy); - if(!ret){r->powered=false;p->owner=-1;} - return ret; -} -static int route_get_rate(unsigned int slot) -{ - int ret;guard(mutex)(&route_lock); - ret=check(slot);return ret ?: routes[slot].rate; -} -static int route_set_drive(unsigned int slot, const unsigned int drive[12]) -{ - union phy_configure_opts opts={};int ret;guard(mutex)(&route_lock); - ret=check(slot);if(ret)return ret; - ret=activate(slot);if(ret)return ret; - if(!routes[slot].rate)return -EHOSTDOWN; - for(unsigned int i=0;i<2;i++){ - if(drive[i*3])return -EINVAL; - opts.dp.voltage[i]=drive[i*3+1];opts.dp.pre[i]=drive[i*3+2]; - } - opts.dp.set_voltages=1;opts.dp.lanes=2; - return phy_configure(ports[routes[slot].active_port].phy,&opts); -} -static const struct m3_usbc_route_ops route_ops={ - .prepare=route_prepare,.hpd=route_hpd,.target=route_target, - .activate=route_activate,.deactivate=route_deactivate, - .xbar_up=route_xbar_up,.xbar_down=route_xbar_down, - .set_rate=route_set_rate,.get_rate=route_get_rate,.set_drive=route_set_drive, -}; -const struct m3_usbc_route_ops *m3_usbc_get_ops(void) -{return &route_ops;} -EXPORT_SYMBOL_GPL(m3_usbc_get_ops); -static int __init probe(void) -{ - struct device_node *node;int ret=-ENODEV;int (*supported)(struct phy *); - if(!of_machine_is_compatible("apple,j514s"))return -ENODEV; - for_each_compatible_node(node,NULL,"apple,sn201202x"){ - struct spmi_device *sdev=spmi_find_device_by_of_node(node); - if(!sdev)continue; - for(unsigned int i=0;idev),ports[i].controller_name)){ - get_device(&sdev->dev);ports[i].controller=sdev;break; - } - spmi_device_put(sdev); - } - node=of_find_node_by_path("dcpext1");if(!node)goto done; - for(unsigned int i=0;iops->power_on || !phy->ops->power_off){ret=-EOPNOTSUPP;break;} - ret=phy_init(phy);if(ret)break; - ret=phy_set_mode(phy,PHY_MODE_DP);if(ret)break; - } - of_node_put(node); - done: - result=ret; - pr_info("m3_usbc_phy: two-session three-port consumer ready result=%d\n",ret); - return 0; -} -module_init(probe); -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("J514S exclusive USB-C DP routing for two external DCP engines"); diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/m3_usbc_ptd_vote.c b/drivers/gpu/drm/apple/tools/m3-dcp/m3_usbc_ptd_vote.c deleted file mode 100644 index 6f45f41d8c116..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/m3_usbc_ptd_vote.c +++ /dev/null @@ -1,3 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -#define M3_PTD_INSTANCE 1 -#include "m3_dcp_ptd_vote.c" diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/syncobj-probe.py b/drivers/gpu/drm/apple/tools/m3-dcp/syncobj-probe.py deleted file mode 100644 index 99667062edb81..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/syncobj-probe.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 -"""CPU-only DRM timeline/import/eventfd check; does not submit GPU work. - -Usage: syncobj-probe.py DISPLAY_NODE RENDER_NODE -The JSON result identifies both nodes. Exercise imported timelines in both -directions, requiring an unsignaled point to time out before CPU signaling it. -""" -import ctypes as C -import errno -import json -import os -import select -import sys - -drm = C.CDLL('libdrm.so.2', use_errno=True) -U32, U64, INT = C.c_uint32, C.c_uint64, C.c_int -P32, P64 = C.POINTER(U32), C.POINTER(U64) -signatures = { - 'drmGetCap': [INT, U64, P64], - 'drmSyncobjCreate': [INT, U32, P32], - 'drmSyncobjDestroy': [INT, U32], - 'drmSyncobjHandleToFD': [INT, U32, C.POINTER(INT)], - 'drmSyncobjFDToHandle': [INT, INT, P32], - 'drmSyncobjTimelineSignal': [INT, P32, P64, U32], - 'drmSyncobjTimelineWait': [INT, P32, P64, U32, C.c_int64, U32, P32], - 'drmSyncobjEventfd': [INT, U32, U64, INT, U32], -} -for name, args in signatures.items(): - fn = getattr(drm, name) - fn.argtypes, fn.restype = args, INT - - -def check(result): - if result: - raise OSError(C.get_errno(), f'libdrm returned {result}') - - -assert len(sys.argv) == 3, __doc__ -fds, owned, handles = [], [], [] -report = {'scope': 'CPU-signaled synchronization; no GPU execution', 'nodes': [], - 'directions': []} -try: - for path in sys.argv[1:]: - fd = os.open(path, os.O_RDWR | os.O_CLOEXEC) - fds.append(fd) - caps = {} - for name, cap in [('syncobj', 0x13), ('timeline', 0x14)]: - value = U64() - check(drm.drmGetCap(fd, cap, C.byref(value))) - caps[name] = value.value - assert value.value == 1, (path, name, value.value) - report['nodes'].append({'path': os.path.realpath(path), **caps}) - for src, dst in [(fds[0], fds[1]), (fds[1], fds[0])]: - original, imported, object_fd = U32(), U32(), INT(-1) - check(drm.drmSyncobjCreate(src, 0, C.byref(original))) - handles.append((src, original.value)) - check(drm.drmSyncobjHandleToFD(src, original, C.byref(object_fd))) - owned.append(object_fd.value) - check(drm.drmSyncobjFDToHandle(dst, object_fd, C.byref(imported))) - handles.append((dst, imported.value)) - point = U64(7) - C.set_errno(0) - result = drm.drmSyncobjTimelineWait(dst, C.byref(imported), - C.byref(point), 1, 0, 2, None) # WAIT_FOR_SUBMIT, zero deadline - assert result == -errno.ETIME, (result, C.get_errno()) - event = os.eventfd(0, os.EFD_CLOEXEC | os.EFD_NONBLOCK) - owned.append(event) - check(drm.drmSyncobjEventfd(dst, imported, point, event, 0)) - assert not select.select([event], [], [], 0)[0] - check(drm.drmSyncobjTimelineSignal(src, C.byref(original), C.byref(point), 1)) - check(drm.drmSyncobjTimelineWait(dst, C.byref(imported), - C.byref(point), 1, 0, 2, None)) - assert select.select([event], [], [], 1)[0] == [event] - assert os.eventfd_read(event) == 1 - report['directions'].append({'source': fds.index(src), - 'destination': fds.index(dst), 'point': point.value, - 'pre_signal_timeout': True, 'post_signal_wait': True, 'eventfd': True}) - report['status'] = 'Pass' - print(json.dumps(report, indent=2)) -finally: - for fd, handle in reversed(handles): - drm.drmSyncobjDestroy(fd, handle) - for fd in owned + fds: - os.close(fd) diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/validation/.gitignore b/drivers/gpu/drm/apple/tools/m3-dcp/validation/.gitignore deleted file mode 100644 index 7422560f59a23..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/validation/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -# Raw evidence stays local; commit summaries, hashes and helper source only. -* -!*/ -!.gitignore -!*.md -!*.sha256 -!*.py -!*.c -!*.h -!*.rs -!*.sh diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/validation/README.md b/drivers/gpu/drm/apple/tools/m3-dcp/validation/README.md deleted file mode 100644 index e6c91c9632e47..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/validation/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Validation evidence policy - -Keep concise validation summaries, checksums and helper source in Git. Raw -screenshots, CTS result files, generated case catalogs, kernel logs and telemetry -remain local and are ignored. Existing published evidence is retained unchanged; -this policy removes generated data only from the previously unpublished work. - -On 2026-09-21, 52 unpublished M3 commits were rebuilt to omit generated evidence. -Source code was verified unchanged. Older commit IDs in summaries can be mapped -through `history-map.md` in the M3 lab validation directory. Raw artifact paths in -those summaries refer to local retained evidence, not files shipped in this tree. - -The original history is preserved locally in -`refs/backup/m3-before-artifact-cleanup-20260921` and in the verified incremental -bundle `build/m3-push-audit/unpublished-with-raw-evidence.bundle` in the source -collection workspace. Its prerequisite is the published commit -`4ddd13c6bf5941dc23a298a054b390850cb2294d`. No raw evidence was discarded, and no -published history was rewritten. diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/validation/dcpext_session_test.c b/drivers/gpu/drm/apple/tools/m3-dcp/validation/dcpext_session_test.c deleted file mode 100644 index bb46e59a56ba2..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/validation/dcpext_session_test.c +++ /dev/null @@ -1,478 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Host-only protocol tests. Synthetic requests after the captured EPMAP; - * no claim that external firmware has completed this conversation. - */ -#include "dcpext_session.h" -#include -#include -#include -#include - -#define MSG(t) ((u64)(t) << 52) -struct message { u8 ep; u64 word; }; -struct harness { - struct dcpext_session s; - u64 now; - struct message tx[1024]; - unsigned int n, buffers, fail_at, buffer_fault; -}; - -static u64 now_ms(void *cookie) -{ - return ((struct harness *)cookie)->now; -} - -static int send_message(void *cookie, u8 ep, u64 msg) -{ - struct harness *h = cookie; - assert(h->n < 1024); - h->tx[h->n++] = (struct message){ep, msg}; - return h->fail_at && h->n == h->fail_at ? -EIO : 0; -} - -static int prepare_buffer(void *cookie, u8 ep, u64 requested, u32 size, u64 *dva) -{ - struct harness *h = cookie; - assert(ep == 1 || ep == 2 || ep == 4 || ep == 8); - assert(size && size <= 0x100000); - h->buffers++; - if (h->buffer_fault == 1) - return -ENOMEM; - *dva = requested ? requested : 0x10040000000ULL + (u64)ep * 0x100000; - if (h->buffer_fault == 2) - *dva += 1; - if (h->buffer_fault == 3) - *dva = 0x11000000000ULL - 0x1000; - if (h->buffer_fault == 4) - h->now = h->s.deadline; - if (h->buffer_fault == 5) - *dva = 0x10041000000ULL; /* changed inherited address */ - if (h->buffer_fault == 6) - *dva = 0x40000000; /* table address without the DCP wire bias */ - return 0; -} - -static const struct dcpext_session_ops ops = { - .now_ms = now_ms, .send = send_message, .buffer = prepare_buffer, -}; - -static void begin(struct harness *h) -{ - memset(h, 0, sizeof(*h)); - h->now = 100; - assert(dcpext_session_begin(&h->s, &ops, h) == 0); - assert(h->s.touched && h->s.phase == DCPEXT_HELLO); - assert(h->n == 1 && h->tx[0].word == 0x60000000000220ULL); -} - -static void rx(struct harness *h, u8 ep, u64 msg) -{ - assert(dcpext_session_receive(&h->s, ep, msg) == 0); -} - -static void epmap(struct harness *h, bool early_iop) -{ - /* Actual J514S external hello and system map from session-1. */ - rx(h, 0, 0x100000000c000cULL); - assert(h->tx[1].word == 0x200000000c000cULL); - rx(h, 0, 0x8000000000011fULL); - assert(h->tx[2].word == 0x80000000000001ULL); - if (early_iop) - rx(h, 0, MSG(7) | 0x20); - assert(dcpext_session_quiesce(&h->s) == -EAGAIN); - assert(h->n == 3); /* specifically no premature IOP-quiesce message */ - rx(h, 0, 0x88000100800fffULL); - assert(h->tx[3].word == 0x88000100000000ULL); - const unsigned int endpoints[] = {1, 2, 3, 4, 8}; - for (unsigned int i = 0; i < sizeof(endpoints) / sizeof(*endpoints); i++) - assert(h->tx[4 + i].word == (MSG(5) | (u64)endpoints[i] << 32 | 2)); - assert(h->s.started == 0x11e); - assert(h->s.endpoints[1] == 0x00800fff); - assert(h->s.phase == (early_iop ? DCPEXT_AP_ON : DCPEXT_IOP_ON)); -} - -static void system_buffers(struct harness *h) -{ - unsigned int n = h->n; - /* Synthetic conversation based on Linux's system endpoint definitions. - * The firmware-owned DVA values below are test inputs, not hardware reads. - */ - rx(h, 1, MSG(1) | (16ULL << 44) | 0x10006631000ULL); - assert(h->n == n && h->s.buffers[1].inherited); - rx(h, 2, MSG(1) | (4ULL << 44)); - assert(h->tx[n].ep == 2); - assert(h->tx[n++].word == (MSG(1) | (4ULL << 44) | 0x10040200000ULL)); - rx(h, 4, MSG(1) | (3ULL << 44)); - assert(h->tx[n].ep == 4); - assert(h->tx[n++].word == (MSG(1) | (3ULL << 44) | 0x10040400000ULL)); - rx(h, 8, (1ULL << 56) | (0x24000ULL << 36) | (0x103e1b32000ULL >> 12)); - assert(h->n == n && h->s.buffers[8].size == 0x24000); - assert(h->s.buffers[8].dva == 0x103e1b32000ULL && h->s.buffers[8].inherited); - rx(h, 2, MSG(8) | (128ULL << 24) | 16); - rx(h, 2, MSG(5) | 3); - assert(h->tx[n].ep == 2 && h->tx[n++].word == (MSG(5) | 3)); - rx(h, 4, MSG(8) | 123); - assert(h->tx[n].ep == 4 && h->tx[n++].word == (MSG(8) | 123)); - rx(h, 4, MSG(0xc) | 321); - assert(h->tx[n].ep == 4 && h->tx[n++].word == (MSG(0xc) | 321)); - assert(h->n == n && h->buffers == 4); -} - -static void running(struct harness *h) -{ - begin(h); - epmap(h, false); - system_buffers(h); - rx(h, 0, MSG(7) | 0x20); - assert(h->s.phase == DCPEXT_AP_ON); - assert(h->tx[h->n - 1].word == (MSG(0xb) | 0x20)); - rx(h, 0, MSG(0xb) | 0x20); - assert(h->s.phase == DCPEXT_RUNNING); -} - -static void quiesce(struct harness *h) -{ - unsigned int n = h->n; - assert(dcpext_session_quiesce(&h->s) == 0); - assert(h->s.phase == DCPEXT_AP_QUIESCE); - assert(h->tx[n++].word == (MSG(0xb) | 0x10)); - /* Logging can still arrive while AP quiescence is pending. */ - rx(h, 4, MSG(0xc) | 0x55); - assert(h->tx[n++].word == (MSG(0xc) | 0x55)); - rx(h, 0, MSG(0xb) | 0x10); - assert(h->s.phase == DCPEXT_IOP_QUIESCE); - assert(h->tx[n++].word == (MSG(6) | 0x10)); - rx(h, 0, MSG(7) | 0x10); - assert(h->s.phase == DCPEXT_QUIESCED && h->n == n); - assert(h->buffers == 4); /* engine never discards buffer ownership */ -} - -static void sticky_failure(struct harness *h, int error) -{ - unsigned int n = h->n; - assert(h->s.phase == DCPEXT_FAILED && h->s.error == error); - assert(dcpext_session_receive(&h->s, 0, MSG(7) | 0x10) == error); - assert(dcpext_session_quiesce(&h->s) == error); - assert(dcpext_session_poll(&h->s) == error); - assert(h->n == n && h->s.phase == DCPEXT_FAILED); -} - -int main(void) -{ - struct harness h; - unsigned int n; - - /* AP acknowledgement permits channel shutdown, never DMA release. */ - running(&h); - h.s.defer_iop_quiesce = true; - assert(dcpext_session_finish_quiesce(&h.s) == -EAGAIN); - assert(!dcpext_session_quiesce(&h.s)); - n = h.n; - assert(dcpext_session_finish_quiesce(&h.s) == -EAGAIN); - rx(&h, 0, MSG(0xb) | 0x10); - assert(h.s.ap_quiesced && h.s.phase == DCPEXT_AP_QUIESCE && h.n == n); - rx(&h, 4, MSG(0xc) | 0x55); - assert(!dcpext_session_finish_quiesce(&h.s)); - assert(h.s.phase == DCPEXT_IOP_QUIESCE); - rx(&h, 0, MSG(7) | 0x10); - assert(h.s.phase == DCPEXT_QUIESCED); - running(&h); - h.s.defer_iop_quiesce = true; - assert(!dcpext_session_quiesce(&h.s)); - rx(&h, 0, MSG(0xb) | 0x10); - assert(dcpext_session_receive(&h.s, 0, MSG(0xb) | 0x10) == -EPROTO); - sticky_failure(&h, -EPROTO); - assert(dcpext_session_finish_quiesce(&h.s) == -EPROTO); - puts("PASS: deferred IOP waits for AP ACK; duplicate ACK fails with retained ownership"); - - running(&h); - h.s.sleep_iop = true; - assert(!dcpext_session_quiesce(&h.s)); - rx(&h, 0, MSG(0xb) | 0x10); - assert(h.tx[h.n - 1].word == (MSG(6) | 1)); - rx(&h, 0, MSG(7) | 1); - assert(h.s.phase == DCPEXT_QUIESCED); - running(&h); - h.s.sleep_iop = true; - assert(!dcpext_session_quiesce(&h.s)); - rx(&h, 0, MSG(0xb) | 0x10); - assert(dcpext_session_receive(&h.s, 0, MSG(7) | 0x10) == -EPROTO); - sticky_failure(&h, -EPROTO); - puts("PASS: restartable sleep requires AP quiescence then exact IOP sleep ACK; wrong ACK retains buffers"); - - running(&h); - for (unsigned int type = 0; type < 256; type++) - if (type != 1) { - running(&h); - n = h.n; - rx(&h, 8, (u64)type << 56 | 0x3060); - assert(h.n == n + (type == 2) && h.buffers == 4 && h.s.oslog_notices == 1); - if (type == 2) - assert(h.tx[n].ep == 8 && h.tx[n].word == 0x0200000000003060ULL); - } - /* The absolute message bound must still terminate an OSLog flood. */ - running(&h); - int flood_result = 0; - for (unsigned int i = 0; i < 300 && !flood_result; i++) - flood_result = dcpext_session_receive(&h.s, 8, 0x0200000000003060ULL); - assert(flood_result == -EPROTO); - sticky_failure(&h, -EPROTO); - /* Exact captured notification must advance the firmware's read counter. */ - running(&h); - n = h.n; - rx(&h, 8, 0x0200000000003060ULL); - assert(h.n == n + 1 && h.buffers == 4 && h.s.oslog_notices == 1); - assert(h.tx[n].ep == 8 && h.tx[n].word == 0x0200000000003060ULL); - puts("PASS: captured OSLog type 2 acknowledged; metadata opaque, no dereference, flood bound preserved"); - quiesce(&h); - puts("PASS: captured hello/map, system starts, buffer layouts/ACKs, AP then IOP quiescence"); - assert(dcpext_session_receive(&h.s, 2, MSG(5)) == -EPROTO); - sticky_failure(&h, -EPROTO); /* unexpected traffic invalidates completion */ - - /* Read counters wrap and are cumulative, not offsets into the buffer. */ - running(&h); - assert(!dcpext_session_quiesce(&h.s)); - const u64 counters[] = {0x31e1, 0x379b, 0xffffffff, 0}; - for (unsigned int i = 0; i < sizeof(counters) / sizeof(*counters); i++) { - u64 notice = (2ULL << 56) | counters[i]; - n = h.n; - rx(&h, 8, notice); - assert(h.n == n + 1 && h.tx[n].ep == 8 && h.tx[n].word == notice); - assert(h.s.phase == DCPEXT_AP_QUIESCE && !h.s.ap_quiesced); - } - running(&h); - h.fail_at = h.n + 1; - assert(dcpext_session_receive(&h.s, 8, 0x02000000000031e1ULL) == -EIO); - assert(h.s.buffers[8].ready); - sticky_failure(&h, -EIO); - begin(&h); - epmap(&h, false); - n = h.n; - assert(dcpext_session_receive(&h.s, 8, 0x02000000000031e1ULL) == -EPROTO); - assert(h.n == n); - for (unsigned int bit = 32; bit < 56; bit++) { - running(&h); - n = h.n; - assert(dcpext_session_receive(&h.s, 8, (2ULL << 56) | (1ULL << bit)) == -EPROTO); - assert(h.n == n); - sticky_failure(&h, -EPROTO); - } - puts("PASS: OSLog drain during AP shutdown, counter wrap, missing buffer, malformed fields and uncertain ACK failure"); - - begin(&h); - epmap(&h, true); - system_buffers(&h); - rx(&h, 0, MSG(0xb) | 0x20); - quiesce(&h); - puts("PASS: IOP acknowledgement before last map, with AP start deferred until system starts"); - - begin(&h); - epmap(&h, false); - rx(&h, 8, (1ULL << 56) | (0x23456ULL << 36)); - assert(h.s.buffers[8].size == 0x23456 && !h.s.buffers[8].inherited); - assert(h.tx[h.n - 1].word == ((1ULL << 56) | (0x23456ULL << 36) | (0x10040800000ULL >> 12))); - puts("PASS: OSLog byte size and page-shifted address, including non-page-multiple size"); - begin(&h); - epmap(&h, false); - /* Buffer size/address and notification words from the retained INTERNAL - * J713 kernel-client log. External firmware has not supplied these yet. - */ - n = h.n; - rx(&h, 8, (1ULL << 56) | (0x5ff8ULL << 36) | (0x103e1b32000ULL >> 12)); - rx(&h, 8, 0x3010042100000a1ULL); - rx(&h, 8, 0x4000020d51e4021ULL); - rx(&h, 8, 0x4000040d28078a1ULL); - rx(&h, 8, 0x4000060d51e4001ULL); - assert(h.n == n && h.s.oslog_notices == 4); - assert(h.s.buffers[8].dva == 0x103e1b32000ULL && h.s.buffers[8].size == 0x5ff8); - puts("PASS: saved internal-DCP OSLog buffer fields/notices; no dereference, reply or false protocol failure"); - - /* Fault every TX in the complete conversation. Delivery is uncertain; - * the engine must never report quiescence after any failed send. - */ - running(&h); - quiesce(&h); - n = h.n; - for (unsigned int failed = 1; failed <= n; failed++) { - struct harness recorded = h, f = {0}; - f.fail_at = failed; - int ret = dcpext_session_begin(&f.s, &ops, &f); - const struct message input[] = { - {0, 0x100000000c000cULL}, {0, 0x8000000000011fULL}, - {0, 0x88000100800fffULL}, - {1, MSG(1) | (16ULL << 44) | 0x10006631000ULL}, - {2, MSG(1) | (4ULL << 44)}, {4, MSG(1) | (3ULL << 44)}, - {8, (1ULL << 56) | (0x24000ULL << 36) | (0x103e1b32000ULL >> 12)}, - {2, MSG(8) | (128ULL << 24) | 16}, {2, MSG(5) | 3}, - {4, MSG(8) | 123}, {4, MSG(0xc) | 321}, - {0, MSG(7) | 0x20}, {0, MSG(0xb) | 0x20}, - {4, MSG(0xc) | 0x55}, {0, MSG(0xb) | 0x10}, {0, MSG(7) | 0x10}, - }; - for (unsigned int i = 0; !ret && i < sizeof(input) / sizeof(*input); i++) { - if (i == 13) - ret = dcpext_session_quiesce(&f.s); - if (!ret) - ret = dcpext_session_receive(&f.s, input[i].ep, input[i].word); - } - assert(ret == -EIO && f.n == failed && f.s.touched); - for (unsigned int i = 0; i < f.n; i++) - assert(f.tx[i].ep == recorded.tx[i].ep && f.tx[i].word == recorded.tx[i].word); - sticky_failure(&f, -EIO); - } - printf("PASS: delivery failure injected at all %u sends; failures remain sticky\n", n); - - for (unsigned int fault = 1; fault <= 6; fault++) { - begin(&h); - epmap(&h, false); - h.buffer_fault = fault; - n = h.n; - int ret = dcpext_session_receive(&h.s, 1, MSG(1) | (16ULL << 44) | 0x10006631000ULL); - assert(ret == (fault == 1 ? -ENOMEM : fault == 4 ? -ETIMEDOUT : -EINVAL)); - assert(h.n == n && h.buffers == 1); - if (fault != 1) - assert(h.s.buffers[1].ready); - sticky_failure(&h, ret); - } - puts("PASS: buffer allocation/validation/timeouts retain failure state and suppress replies"); - - const u64 bad_requests[] = {MSG(1), MSG(1) | (1ULL << 44) | (1ULL << 36), - MSG(1) | (1ULL << 44) | 0x40000001, MSG(1) | (2ULL << 44) | 0x10ffffff000ULL}; - for (unsigned int i = 0; i < sizeof(bad_requests) / sizeof(*bad_requests); i++) { - begin(&h); - epmap(&h, false); - assert(dcpext_session_receive(&h.s, 2, bad_requests[i]) == -EINVAL); - assert(!h.buffers); - sticky_failure(&h, -EINVAL); - } - for (unsigned int ep = 1; ep <= 8; ep++) { - if (ep != 1 && ep != 2 && ep != 4 && ep != 8) - continue; - running(&h); - int ret = dcpext_session_receive(&h.s, ep, ep == 8 ? 1ULL << 56 : MSG(1)); - assert(ret == (ep == 1 ? -EIO : -EPROTO) && h.buffers == 4); - sticky_failure(&h, ret); - } - puts("PASS: zero/out-of-aperture/unaligned/cross-boundary requests; crash and duplicate buffers"); - - /* Expire every phase where firmware may still own shared memory. */ - for (unsigned int phase = DCPEXT_HELLO; phase <= DCPEXT_IOP_QUIESCE; phase++) { - begin(&h); - if (phase >= DCPEXT_EPMAP) - rx(&h, 0, 0x100000000c000cULL); - if (phase >= DCPEXT_IOP_ON) - rx(&h, 0, 0x8800000000011fULL); - if (phase >= DCPEXT_AP_ON) - rx(&h, 0, MSG(7) | 0x20); - if (phase >= DCPEXT_RUNNING) - rx(&h, 0, MSG(0xb) | 0x20); - if (phase >= DCPEXT_AP_QUIESCE) - assert(dcpext_session_quiesce(&h.s) == 0); - if (phase >= DCPEXT_IOP_QUIESCE) - rx(&h, 0, MSG(0xb) | 0x10); - assert(h.s.phase == phase); - h.now = h.s.deadline; - assert(dcpext_session_poll(&h.s) == -ETIMEDOUT); - sticky_failure(&h, -ETIMEDOUT); - } - puts("PASS: fixed deadline covers every live phase; late quiesce ACK never permits cleanup"); - /* Explicit observation budget stays finite and cannot be renewed. */ - begin(&h); - h.s = (struct dcpext_session){.duration_ms = 60001}; - assert(dcpext_session_begin(&h.s, &ops, &h) == -EINVAL && !h.s.touched); - h.s = (struct dcpext_session){.duration_ms = 60000}; - assert(!dcpext_session_begin(&h.s, &ops, &h)); - assert(h.s.deadline == h.now + 60000); - h.now = h.s.deadline; - assert(dcpext_session_poll(&h.s) == -ETIMEDOUT); - sticky_failure(&h, -ETIMEDOUT); - - begin(&h); - h.now--; - assert(dcpext_session_poll(&h.s) == -ERANGE); - sticky_failure(&h, -ERANGE); - begin(&h); - assert(dcpext_session_receive(&h.s, 0, 0x100000000d000dULL) == -EPROTO); - begin(&h); - rx(&h, 0, 0x100000000c000cULL); - assert(dcpext_session_receive(&h.s, 0, 0x8800000000059fULL) == -EOPNOTSUPP); - begin(&h); - rx(&h, 0, 0x100000000c000cULL); - rx(&h, 0, 0x8000000000011fULL); - assert(dcpext_session_receive(&h.s, 0, 0x8800000000011fULL) == -EPROTO); - running(&h); - assert(dcpext_session_quiesce(&h.s) == 0); - assert(dcpext_session_receive(&h.s, 0, MSG(7) | 0x10) == -EPROTO); - sticky_failure(&h, -EPROTO); /* IOP quiescence before AP is not sufficient */ - running(&h); - assert(dcpext_session_receive(&h.s, 0x28, 0) == -EPROTO); - assert(dcpext_session_begin(&h.s, &ops, &h) == -EBUSY); - running(&h); - while (h.s.received < 256) - rx(&h, 2, MSG(5)); - assert(dcpext_session_receive(&h.s, 2, MSG(5)) == -EPROTO); - sticky_failure(&h, -EPROTO); - puts("PASS: backwards clock, version/profile/map mismatch, wrong ACK order, unsolicited app EP, reuse and message flood"); - /* The retained 40-surface run reached message 257 during power-off. - * Its explicit larger budget still terminates a flood and cannot extend - * the absolute deadline. Native display plus AV startup can also use - * this budget, without extending its 16-second deadline. */ - begin(&h); - h.s = (struct dcpext_session){.message_limit=512}; - assert(dcpext_session_begin(&h.s,&ops,&h)==-EINVAL && !h.s.touched); - h.s = (struct dcpext_session){.duration_ms=16000,.message_limit=512}; - assert(!dcpext_session_begin(&h.s,&ops,&h)); - assert(h.s.deadline == h.now + 16000); - h.now += 16000; - assert(dcpext_session_poll(&h.s) == -ETIMEDOUT); - h.s = (struct dcpext_session){.duration_ms=16000,.message_limit=8192}; - assert(!dcpext_session_begin(&h.s,&ops,&h)); - assert(h.s.deadline == h.now + 16000); - h.s = (struct dcpext_session){.duration_ms=60000,.message_limit=8192}; - assert(dcpext_session_begin(&h.s,&ops,&h)==-EINVAL && !h.s.touched); - h.s = (struct dcpext_session){.duration_ms=15000,.message_limit=512}; - assert(dcpext_session_begin(&h.s,&ops,&h)==-EINVAL && !h.s.touched); - h.s = (struct dcpext_session){.duration_ms=16000}; - assert(!dcpext_session_begin(&h.s,&ops,&h)); - h.s = (struct dcpext_session){.duration_ms=60000,.message_limit=513}; - assert(dcpext_session_begin(&h.s,&ops,&h)==-EINVAL && !h.s.touched); - h.n = 0; - h.s = (struct dcpext_session){.duration_ms=60000,.message_limit=512}; - assert(!dcpext_session_begin(&h.s,&ops,&h)); - epmap(&h,false); - system_buffers(&h); - rx(&h,0,MSG(7)|0x20);rx(&h,0,MSG(0xb)|0x20); - while(h.s.received<512) rx(&h,8,0x0200000000003060ULL); - assert(dcpext_session_receive(&h.s,8,0x0200000000003060ULL)==-EPROTO); - sticky_failure(&h,-EPROTO); - puts("PASS: explicit 60s surface-trial message budget accepts message 257, rejects 513 and preserves sticky failure"); - /* A desktop becomes persistent only after qualified startup; leaving it - * reestablishes one bounded teardown, and floods still fail permanently. */ - running(&h); - assert(!dcpext_session_enter_runtime(&h.s)); - assert(dcpext_session_enter_runtime(&h.s) == -EINVAL); - for (unsigned int second = 0; second < 120; second++) { - h.now += 1000; - assert(!dcpext_session_poll(&h.s)); - for (unsigned int m = 0; m < 512; m++) - assert(!dcpext_session_runtime_message(&h.s)); - } - assert(!dcpext_session_leave_runtime(&h.s)); - assert(!h.s.runtime && h.s.deadline == h.now + 8000); - assert(dcpext_session_leave_runtime(&h.s) == -EINVAL); - h.now += 8000; - assert(dcpext_session_poll(&h.s) == -ETIMEDOUT); - assert(dcpext_session_enter_runtime(&h.s) == -ETIMEDOUT); - running(&h);assert(!dcpext_session_enter_runtime(&h.s)); - for (unsigned int m = 0; m < 8192; m++) - assert(!dcpext_session_runtime_message(&h.s)); - assert(dcpext_session_runtime_message(&h.s) == -EOVERFLOW); - h.now += 2000;assert(dcpext_session_poll(&h.s) == -EOVERFLOW); - running(&h);assert(!dcpext_session_enter_runtime(&h.s)); - assert(!dcpext_session_quiesce(&h.s)); - assert(!h.s.runtime && h.s.phase == DCPEXT_AP_QUIESCE); - h.now += 8000;assert(dcpext_session_poll(&h.s) == -ETIMEDOUT); - begin(&h);assert(dcpext_session_enter_runtime(&h.s) == -EINVAL); - puts("PASS: explicit desktop admission, sustained runtime, all-endpoint rate bound and bounded teardown"); - puts("OFFLINE ONLY: no mailbox, DMA, external firmware completion, EDID or image was tested."); - return 0; -} diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/README.md b/drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/README.md deleted file mode 100644 index 703e0e618283a..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# J514S physical panel dimensions validation - -Source fix: cc1765d1fe7e. Installed module and m1n1 bundle hashes are in deployment.json. -The bootloader and every byte outside the J514S DT remained unchanged; device-tree.diff -contains only the panel node with 302x196 mm dimensions. - -Reboot 366814ff-4281-477d-babf-c3506c8d08d7 completed guarded native DCP startup. DRM GETCONNECTOR -and KWin independently reported 302x196 mm at 3024x1964, approximately 254 DPI. -Initially removing only the saved scale key retained 1x through KWin's existing-setup -path. After stopping the display manager, terminating eryk's session and waiting for -KWin to exit, the single-panel configuration was archived. No scale was supplied. -KWin 6.7.4 then selected 1.7x automatically (1779x1155 logical desktop). Only keyboard -and touchpad input devices were present, consistent with its 150-DPI internal-output -heuristic without a lid switch. Upstream source consulted: -https://invent.kde.org/plasma/kwin/-/blob/Plasma/6.7/src/outputconfigurationstore.cpp -Panel specification: https://support.apple.com/en-ie/117736 - -Guarded real Plasma/Konsole/screenshot validation completed, peak five-channel PMP -sample 69.50 C, with the existing 60-second watchdog and M3 thermal protections. -The screenshot was visually checked: normally rendered desktop, readable controls. -The temporary validation Konsole closed; normal persistent autologin remains active. -A subsequent native KMS status showed completed=257, failed=0. - -Target backups: /root/m3-dcp-dpi-backup-20260921 (old module, boot bundle, config), -and /home/eryk/.config/kwinoutputconfig.json.before-panel-dpi-autodetect. -Raw boot evidence and screenshot are retained under build/m3-dcp-dpi in the workspace. -This validates dimensions and automatic scaling, not new refresh-rate or recording support. diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/inspect.py b/drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/inspect.py deleted file mode 100644 index 4177b49dd7587..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/inspect.py +++ /dev/null @@ -1,19 +0,0 @@ -import ctypes as C,json,os,subprocess -from pathlib import Path -class Connector(C.Structure): - _fields_=[(n,C.c_uint32) for n in ('connector_id','encoder_id','connector_type','connector_type_id','connection','mmWidth','mmHeight')] -lib=C.CDLL('libdrm.so.2');lib.drmModeGetConnector.argtypes=[C.c_int,C.c_uint32];lib.drmModeGetConnector.restype=C.POINTER(Connector);lib.drmModeFreeConnector.argtypes=[C.POINTER(Connector)] -connectors=[] -for path in Path('/sys/class/drm').glob('card*-eDP-*'): - fd=os.open('/dev/dri/'+path.name.split('-')[0],os.O_RDONLY) - c=lib.drmModeGetConnector(fd,int((path/'connector_id').read_text())) - assert c - connectors.append({'name':path.name,'physical_mm':[c.contents.mmWidth,c.contents.mmHeight],'modes':(path/'modes').read_text().splitlines()}) - lib.drmModeFreeConnector(c);os.close(fd) -node=Path('/sys/firmware/devicetree/base') -dcp=node/(node/'aliases/dcp').read_bytes().rstrip(b'\0').decode().lstrip('/') -physical=[int.from_bytes((dcp/'panel'/k).read_bytes(),'big') for k in ('width-mm','height-mm')] -env=dict(os.environ,XDG_RUNTIME_DIR='/run/user/1000',DBUS_SESSION_BUS_ADDRESS='unix:path=/run/user/1000/bus') -q=subprocess.check_output(['runuser','-u','eryk','--','busctl','--user','--json=short','call','org.kde.KWin','/KWin','org.kde.KWin','supportInformation'],env=env,text=True,timeout=5) -info=json.loads(q)['data'][0] -print(json.dumps({'boot_id':Path('/proc/sys/kernel/random/boot_id').read_text().strip(),'device_tree_mm':physical,'drm_connectors':connectors,'kwin_support':info,'kwin_config':json.loads(Path('/home/eryk/.config/kwinoutputconfig.json').read_text())},indent=2)) diff --git a/drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/screenshot.sha256 b/drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/screenshot.sha256 deleted file mode 100644 index 762488d9d8389..0000000000000 --- a/drivers/gpu/drm/apple/tools/m3-dcp/validation/panel-dpi-20260921/screenshot.sha256 +++ /dev/null @@ -1 +0,0 @@ -3eca5616349eb5983c328c59c1d404fcc29cc6d6cff6866f20daeee80b32532c build/m3-dcp-dpi/dcp-dpi-autoscale/desktop.png diff --git a/tools/testing/selftests/drm/apple/README b/tools/testing/selftests/drm/apple/README deleted file mode 100644 index 210a00b4c80d8..0000000000000 --- a/tools/testing/selftests/drm/apple/README +++ /dev/null @@ -1,36 +0,0 @@ -Apple external-display host checks -================================= - -Run with Python 3 and a C compiler with address/undefined sanitizers: - - python3 tools/testing/selftests/drm/apple/test-iboot-wire.py - python3 tools/testing/selftests/drm/apple/test-dcpext-present.py - python3 tools/testing/selftests/drm/apple/test-dcpext-copy.py - -These compile the driver's actual functions with host transport/DMA stubs. -They exercise received-length validation and late-ACK cleanup, packet bounds, -presentation ordering, failure latching and buffer ownership. They do not -establish firmware behavior or visual correctness; those need a hardware boot. - -The candidate path is opt-in with appledrm.dcpext_pageflip=1 and limited to the -14.7 firmware UUID checked in dcpext_scanout_register(). It retains two mapped -buffers, prepares a complete opaque frame in the idle one, then sends -SwapBegin/SetLayer/SwapEnd/SwapWait. The exact Begin ID is passed to Wait; -successful Wait is required before the old front buffer is reused. The host -wait is bounded by AFK. Failure retains both buffers and stops presentation. -Link loss is terminal in this candidate mode, including hotplug bounces after -startup: replug recovery must first account for an in-flight transaction. - -After initializing both buffers, the copy includes the current and previous -frame's damage: the idle buffer contains frame N-2. Missing damage or invalid -history requires full updates; disabling the plane invalidates history. History -advances only after a completed swap. The copy test compares 48 frames with a -complete reference, covering disjoint/overlapping/empty/clipped damage, reset, -opaque alpha, padded pitch, failed swaps and untouched front-buffer pixels. -It also checks that small updates read less than one percent of a full frame. - -The read-only dcpext_present sysfs attribute reports mode, retained-buffer -count, completed desktop swaps, front-buffer index and maximum command/wait -duration in microseconds. Counters are diagnostics, not hardware timestamps. -DRM still timestamps completion in software after Wait; this is neither a -vblank interrupt implementation nor zero-copy scanout. diff --git a/tools/testing/selftests/drm/apple/test-dcpext-copy.py b/tools/testing/selftests/drm/apple/test-dcpext-copy.py deleted file mode 100644 index b917413c10724..0000000000000 --- a/tools/testing/selftests/drm/apple/test-dcpext-copy.py +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: GPL-2.0-only OR MIT -"""Exercise actual two-buffer copy/history code against complete reference frames.""" -from pathlib import Path -import os -import subprocess -import tempfile - -base = Path(__file__).resolve().parents[5] / "drivers/gpu/drm/apple" -source = (base / "dcpext_drm.c").read_text() - - -def function(start, end): - return source[source.index(start):source.index(end)] - - -preamble = r''' -#include -#include -#include -#include -#include -#include -#include -#include "dcpext_mode.h" -typedef uint8_t u8; -typedef uint32_t u32; -typedef uint64_t u64; -#define EXT_WIDTH DCPEXT_WIDTH -#define EXT_HEIGHT DCPEXT_HEIGHT -#define EXT_STRIDE DCPEXT_STRIDE -#define DIV_ROUND_UP(n,d) (((n)+(d)-1)/(d)) -#define BITS_PER_LONG (8*sizeof(unsigned long)) -#define DECLARE_BITMAP(name,n) unsigned long name[DIV_ROUND_UP(n,BITS_PER_LONG)] -/* Small host equivalents of the kernel's generic bitmap primitives. */ -static void bitmap_zero(unsigned long *b,unsigned n) { memset(b,0,DIV_ROUND_UP(n,BITS_PER_LONG)*sizeof(*b)); } -static void bitmap_fill(unsigned long *b,unsigned n) { memset(b,255,DIV_ROUND_UP(n,BITS_PER_LONG)*sizeof(*b)); } -static void bitmap_copy(unsigned long *d,const unsigned long *s,unsigned n) { memcpy(d,s,DIV_ROUND_UP(n,BITS_PER_LONG)*sizeof(*d)); } -static void bitmap_or(unsigned long *d,const unsigned long *a,const unsigned long *b,unsigned n) { - for(unsigned i=0;i(b)?(a):(b)) -struct drm_rect { int x1,y1,x2,y2; }; -#define DRM_RECT_INIT(x,y,w,h) {x,y,(x)+(w),(y)+(h)} -static bool drm_rect_visible(const struct drm_rect *r) { return r->x1x2 && r->y1y2; } -static int drm_rect_width(const struct drm_rect *r) { return r->x2-r->x1; } -static bool drm_rect_intersect(struct drm_rect *a,const struct drm_rect *b) { - a->x1=max(a->x1,b->x1);a->y1=max(a->y1,b->y1); - a->x2=min(a->x2,b->x2);a->y2=min(a->y2,b->y2);return drm_rect_visible(a); -} -struct iosys_map { void *vaddr; }; -struct drm_framebuffer { u32 pitches[1]; }; -struct drm_plane_state { struct drm_framebuffer *fb; struct drm_rect clips[300]; unsigned count; bool full; }; -struct drm_shadow_plane_state { struct drm_plane_state base; struct iosys_map data[1]; }; -#define to_drm_shadow_plane_state(p) ((struct drm_shadow_plane_state *)(p)) -/* The driver clips each core-supplied rectangle to the fixed plane bounds. */ -struct drm_atomic_helper_damage_iter { const struct drm_plane_state *ps; unsigned next; }; -static void drm_atomic_helper_damage_iter_init(struct drm_atomic_helper_damage_iter *iter, - const struct drm_plane_state *old,const struct drm_plane_state *ps) { - (void)old;iter->ps=ps;iter->next=0; -} -static bool iter_next(struct drm_atomic_helper_damage_iter *iter,struct drm_rect *out) { - if(iter->ps->full) { - if(iter->next++)return false; - *out=(struct drm_rect)DRM_RECT_INIT(0,0,EXT_WIDTH,EXT_HEIGHT);return true; - } - if(iter->next>=iter->ps->count)return false; - *out=iter->ps->clips[iter->next++];return true; -} -#define drm_atomic_for_each_plane_damage(iter,rect) while(iter_next(iter,rect)) -/* TILE_DEFINES */ -struct dcpext_drm { - void *dcp; u8 *row; u32 stride; bool damage_valid; - DECLARE_BITMAP(previous_damage,EXT_TILE_COUNT); - DECLARE_BITMAP(current_damage,EXT_TILE_COUNT); - DECLARE_BITMAP(copy_damage,EXT_TILE_COUNT); -}; -struct dcpext_frame { void *pixels; size_t size; }; -static struct dcpext_frame back; -static bool begin_fails, end_fails, held; -static unsigned presented, discarded; -static size_t copied; -static int dcpext_scanout_begin_frame(void *dcp, struct dcpext_frame *frame) { - (void)dcp; - if(begin_fails) return -ENOLINK; - assert(!held); held=true; *frame=back; return 0; -} -static int dcpext_scanout_end_frame(void *dcp, bool present) { - (void)dcp; assert(held); held=false; - if(end_fails)return -EIO; - if(present) presented++; else discarded++; - return 0; -} -static void iosys_map_memcpy_from(void *dst, const struct iosys_map *src, - size_t offset, size_t size) { - memcpy(dst,(const u8 *)src->vaddr+offset,size);copied+=size; -} -''' -tests = r''' -static void paint(u8 *source,u8 *reference,u32 pitch,struct drm_rect r,unsigned seed) { - struct drm_rect bounds=DRM_RECT_INIT(0,0,EXT_WIDTH,EXT_HEIGHT); - if(!drm_rect_intersect(&r,&bounds))return; - for(int y=r.y1;y=28) { - shadow.base.count=step==47?300:8; - for(unsigned i=0;i=2 && step<12) || step>=28)assert(copied -#include -#include -#include -#include -#include -#include -#include -#include "dcpext_mode.h" -typedef uint8_t u8; -typedef uint32_t u32; -typedef uint64_t u64; -typedef u32 __le32; -typedef u64 __le64; -#define __packed __attribute__((packed)) -#define static_assert _Static_assert -#define cpu_to_le32(x) (x) -#define cpu_to_le64(x) (x) -#define READ_ONCE(x) (x) -#define smp_load_acquire(p) (*(p)) -#define ARRAY_SIZE(x) (sizeof(x)/sizeof((x)[0])) -#define IBOOT_QUERY_RX_SIZE 0x4000 -#define IBOOT_MODE_SIZE 24 -#define IBOOT_PATTERN_WIDTH DCPEXT_WIDTH -#define IBOOT_PATTERN_HEIGHT DCPEXT_HEIGHT -#define IBOOT_PATTERN_STRIDE DCPEXT_STRIDE -#define SCANOUT_STRIDE DCPEXT_STRIDE -#define IBOOT_GET_HPD 3 -#define IBOOT_GET_TIMING_MODES 4 -#define IBOOT_GET_COLOR_MODES 5 -#define EPIC_SUBTYPE_STD_SERVICE 0x20 -#define DCP_FIRMWARE_V_14_7 147 -#define GFP_KERNEL 0 -#define IS_ERR(p) ((intptr_t)(p) < 0) -#define PTR_ERR(p) ((intptr_t)(p)) -#define might_sleep() do {} while(0) -#define dev_err(...) do {} while(0) -#define dma_wmb() do {} while(0) -#define max(a,b) ((a)>(b)?(a):(b)) -static bool alloc_fail; -static unsigned live; -static void *kzalloc(size_t n, int flags) { - (void)flags; - if (alloc_fail) return NULL; - void *p = calloc(1,n); assert(p); live++; return p; -} -static void kfree(void *p) { if(p) live--; free(p); } -static int atomic_cmpxchg(int *p, int old, int new) { - int value=*p; if(value==old) *p=new; return value; -} -#define atomic_set_release(p,v) (*(p)=(v)) -static u32 get_unaligned_le32(const void *p) { u32 v; memcpy(&v,p,4); return v; } -static void put_unaligned_le32(u32 v, void *p) { memcpy(p,&v,4); } -struct dcpext_scanout; -struct apple_dcp { int fw_compat; struct dcpext_scanout *dcpext_scanout; }; -struct apple_epic_service { bool torndown; }; -struct iboot_query { - struct apple_epic_service *service; - int busy; - bool stopping, pattern_requested, presentation_failed; - u32 last_frame_swap; -}; -static struct apple_epic_service service; -static struct iboot_query query; -static struct iboot_query *iboot_find_query(struct apple_dcp *dcp) { - (void)dcp; return &query; -} -static unsigned calls, fail_step, duplicate_id, short_reply, remote_error; -static u32 returned_id; -static u64 submitted_iova; -static int afk_send_command_with_reply_len(struct apple_epic_service *s, u8 type, - const void *request, size_t size, void *output, size_t capacity, - u32 *retcode, size_t *received) { - const unsigned sequence[]={15,16,18,19}; - const u8 *wire=request; - unsigned op=get_unaligned_le32(wire); - assert(s==&service && type==EPIC_SUBTYPE_STD_SERVICE); - assert(calls<4 && op==sequence[calls++]); - assert(get_unaligned_le32(wire+4)==size && capacity==IBOOT_QUERY_RX_SIZE); - if(op==fail_step) return -ETIMEDOUT; - *retcode=remote_error && op==19 ? 1 : 0; - *received=0; - memset(output,0,capacity); - if(op==15) { - returned_id=duplicate_id ? query.last_frame_swap : query.last_frame_swap+1; - put_unaligned_le32(15,output); - /* The audited 14.7 Begin wrapper says 284; EPIC returns 28 bytes. */ - put_unaligned_le32(284,(u8 *)output+4); - put_unaligned_le32(returned_id,(u8 *)output+20); - *received=short_reply ? 27 : 28; - } else if(op==16) { - assert(size==232); - memcpy(&submitted_iova,wire+28,8); - } else if(op==18) { - assert(size==28); - for(unsigned i=16;i<28;i++) assert(wire[i]==0); - } else { - assert(size==32 && get_unaligned_le32(wire+16)==0); - assert(get_unaligned_le32(wire+20)==returned_id); - assert(get_unaligned_le32(wire+24)==1 && get_unaligned_le32(wire+28)==0); - } - return 0; -} -struct dcpext_frame { void *pixels; size_t size; }; -struct dcpext_scanout { - bool pageflips, stopping, pattern_ready, link; - int present_lock, terminal_error; - void *pixels; - size_t frame_size; - unsigned front; - u64 iova, frames_completed, present_ns_max; -}; -static void mutex_lock(int *p) { assert(!*p); *p=1; } -static void mutex_unlock(int *p) { assert(*p); *p=0; } -#define lockdep_assert_held(p) assert(*(p)) -static bool scanout_link_ready(struct dcpext_scanout *s) { return s->link && !s->terminal_error; } -static void scanout_fail(struct dcpext_scanout *s, int error) { s->terminal_error=error; } -static u64 ktime_get_ns(void) { static u64 clock; return ++clock; } -''' - -tests = r''' -static void reset(void) { - assert(!live); - query=(struct iboot_query){.service=&service,.pattern_requested=true}; - calls=fail_step=duplicate_id=short_reply=remote_error=0; - alloc_fail=false; - service.torndown=false; -} -int main(void) { - struct apple_dcp dcp={.fw_compat=DCP_FIRMWARE_V_14_7}; - const u64 address=0x10000000000ULL; - const size_t bytes=(size_t)DCPEXT_STRIDE*DCPEXT_HEIGHT; - reset(); - assert(!ibootep_present_frame(&dcp,address,bytes,DCPEXT_STRIDE)); - assert(calls==4 && query.last_frame_swap==1 && !query.busy && !live); - assert(submitted_iova==address); - calls=0; - assert(!ibootep_present_frame(&dcp,address+bytes,bytes,DCPEXT_STRIDE)); - assert(calls==4 && query.last_frame_swap==2 && submitted_iova==address+bytes); - unsigned steps[]={15,16,18,19}; - for(unsigned i=0;i<4;i++) { - reset(); fail_step=steps[i]; - assert(ibootep_present_frame(&dcp,address,bytes,DCPEXT_STRIDE)==-ETIMEDOUT); - assert(calls==i+1 && query.presentation_failed && !query.last_frame_swap); - assert(!query.busy && !live); - assert(ibootep_present_frame(&dcp,address,bytes,DCPEXT_STRIDE)==-EIO); - assert(calls==i+1); - } - reset(); short_reply=1; - assert(ibootep_present_frame(&dcp,address,bytes,DCPEXT_STRIDE)==-EPROTO); - assert(calls==1 && query.presentation_failed); - reset(); duplicate_id=1; query.last_frame_swap=10; - assert(ibootep_present_frame(&dcp,address,bytes,DCPEXT_STRIDE)==-EPROTO); - assert(calls==1 && query.last_frame_swap==10); - reset(); remote_error=1; - assert(ibootep_present_frame(&dcp,address,bytes,DCPEXT_STRIDE)==-EIO); - assert(calls==4 && query.presentation_failed && !query.last_frame_swap); - reset(); query.busy=1; - assert(ibootep_present_frame(&dcp,address,bytes,DCPEXT_STRIDE)==-EBUSY); - assert(!calls && query.busy); - reset(); alloc_fail=true; - assert(ibootep_present_frame(&dcp,address,bytes,DCPEXT_STRIDE)==-ENOMEM); - assert(!calls && !query.busy && !query.presentation_failed); - reset(); query.stopping=true; - assert(ibootep_present_frame(&dcp,address,bytes,DCPEXT_STRIDE)==-ENODEV); - assert(!calls && query.presentation_failed); - reset(); - struct dcpext_scanout scan={.pageflips=true,.pattern_ready=true,.link=true, - .pixels=calloc(2,bytes),.frame_size=bytes,.iova=address}; - assert(scan.pixels); - dcp.dcpext_scanout=&scan; - struct dcpext_frame frame; - assert(!dcpext_scanout_begin_frame(&dcp,&frame)); - assert(frame.pixels==(u8 *)scan.pixels+bytes && frame.size==bytes && scan.present_lock); - assert(!dcpext_scanout_end_frame(&dcp,false)); - assert(!calls && !scan.front && !scan.present_lock); - for(unsigned i=0;i<6;i++) { - calls=0; - assert(!dcpext_scanout_begin_frame(&dcp,&frame)); - assert(frame.pixels==(u8 *)scan.pixels+(i%2 ? 0 : bytes)); - assert(!dcpext_scanout_end_frame(&dcp,true)); - assert(calls==4 && scan.front==(i%2 ? 0 : 1)); - assert(submitted_iova==address+scan.front*bytes && scan.frames_completed==i+1); - } - calls=0; fail_step=19; - assert(!dcpext_scanout_begin_frame(&dcp,&frame)); - assert(dcpext_scanout_end_frame(&dcp,true)==-ETIMEDOUT); - assert(scan.front==0 && scan.frames_completed==6 && scan.terminal_error==-EIO); - assert(!scan.present_lock && !live); - assert(dcpext_scanout_begin_frame(&dcp,&frame)==-ENOLINK); - assert(calls==4 && !scan.present_lock); - scan.terminal_error=0; scan.link=false; - assert(dcpext_scanout_begin_frame(&dcp,&frame)==-ENOLINK); - scan.link=true; scan.stopping=true; - assert(dcpext_scanout_begin_frame(&dcp,&frame)==-ENOLINK); - free(scan.pixels); - puts("PASS: actual frame sequence, exact swap wait, malformed/failed replies, no retry, buffer reuse only after completion"); -} -''' - -layouts = iboot[iboot.index("struct iboot_plane {"):iboot.index("struct iboot_query {")] -helpers = iboot[iboot.index("static int iboot_build_query("):iboot.index("static int iboot_read_query(")] -body = preamble + layouts + helpers -body += function(iboot, "iboot_frame_command") + function(iboot, "ibootep_present_frame") -body += function(scanout, "dcpext_scanout_begin_frame") + function(scanout, "dcpext_scanout_end_frame") -with tempfile.TemporaryDirectory(prefix="dcpext-present-test-") as tmp: - path = Path(tmp) - (path / "test.c").write_text(body + tests) - subprocess.run([os.environ.get("CC", "cc"), "-Wall", "-Wextra", "-Werror", - "-Wno-unused-function", "-fsanitize=address,undefined", - "-fno-omit-frame-pointer", "-I", str(base), - "-o", str(path / "test"), str(path / "test.c")], check=True) - subprocess.run([str(path / "test")], check=True) diff --git a/tools/testing/selftests/drm/apple/test-iboot-wire.py b/tools/testing/selftests/drm/apple/test-iboot-wire.py deleted file mode 100644 index bfaef16a0503a..0000000000000 --- a/tools/testing/selftests/drm/apple/test-iboot-wire.py +++ /dev/null @@ -1,185 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: GPL-2.0-only OR MIT -"""Compile and exercise the actual ibootep.c wire helpers, no hardware access.""" -from pathlib import Path -import os -import subprocess -import tempfile - -base = Path(__file__).resolve().parents[5] / 'drivers/gpu/drm/apple' -src = (base / 'ibootep.c').read_text() -helpers = src[src.index('static int iboot_build_query('):src.index('static int iboot_read_query(')] -layouts = src[src.index('struct iboot_plane {'):src.index('struct iboot_query {')] -prefix = r''' -#include -#include "dcpext_mode.h" -#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) -#include -#include -#include -#include -#include -#include -typedef uint8_t u8; -typedef uint32_t u32; -typedef uint64_t u64; -typedef uint32_t __le32; -typedef uint64_t __le64; -#define cpu_to_le32(x) (x) -#define cpu_to_le64(x) (x) -#define __packed __attribute__((packed)) -#define static_assert _Static_assert -#define IBOOT_QUERY_RX_SIZE 0x4000 -#define IBOOT_MODE_SIZE 24 -#define IBOOT_PATTERN_WIDTH 3840 -#define IBOOT_PATTERN_HEIGHT 2160 -#define IBOOT_PATTERN_STRIDE 15360 -enum { IBOOT_GET_HPD=3, IBOOT_GET_TIMING_MODES=4, IBOOT_GET_COLOR_MODES=5 }; -static u32 get_unaligned_le32(const void *p_) { - const u8 *p = p_; - return p[0] | ((u32)p[1]<<8) | ((u32)p[2]<<16) | ((u32)p[3]<<24); -} -static void put32(u8 *p, u32 x) { - for (unsigned i=0; i<4; i++) p[i] = x >> (i*8); -} -#define put_unaligned_le32(v,p) put32(p,v) -''' -tests = r''' -int main(void) { - __le32 req[4]; - for (u32 op=0; op<20; op++) { - memset(req, 0xaa, sizeof(req)); - int ret=iboot_build_query(op, req); - if (op<3 || op>5) { assert(ret==-EINVAL); continue; } - assert(ret==0 && req[0]==op && req[1]==16 && req[2]==0 && req[3]==0); - } - u8 reply[IBOOT_QUERY_RX_SIZE] = {0}; - const u8 *p; - size_t n; - put32(reply, 3); put32(reply+4, 20); - assert(!iboot_reply_payload(reply, sizeof(reply), 3, &p, &n)); - assert(p==reply+8 && n==12); - assert(iboot_reply_payload(reply, 7, 3, &p, &n)==-EPROTO); - assert(iboot_reply_payload(reply, sizeof(reply), 4, &p, &n)==-EPROTO); - put32(reply+4, 7); - assert(iboot_reply_payload(reply, sizeof(reply), 3, &p, &n)==-EPROTO); - put32(reply+4, sizeof(reply)+1); - assert(iboot_reply_payload(reply, sizeof(reply), 3, &p, &n)==-EPROTO); - put32(reply+4, 20); - assert(!iboot_reply_payload(reply, sizeof(reply), 3, &p, &n)); - bool hpd; u32 nt, nc; - assert(!iboot_parse_hpd(p,n,&hpd,&nt,&nc) && !hpd && !nt && !nc); - reply[8]=1; put32(reply+12, 4); put32(reply+16, 7); - assert(!iboot_parse_hpd(p,n,&hpd,&nt,&nc) && hpd && nt==4 && nc==7); - assert(iboot_parse_hpd(p,11,&hpd,&nt,&nc)==-EPROTO); - reply[8]=2; - assert(iboot_parse_hpd(p,n,&hpd,&nt,&nc)==-EPROTO); - reply[8]=1; put32(reply+12, UINT32_MAX); - assert(iboot_parse_hpd(p,n,&hpd,&nt,&nc)==-EOVERFLOW); - memset(reply, 0, sizeof(reply)); - put32(reply, 4); put32(reply+4, 8+4+2*24); put32(reply+8, 2); - put32(reply+12, 1); put32(reply+16, 3840); put32(reply+20, 2160); - put32(reply+24, 60U<<16); - assert(!iboot_reply_payload(reply, sizeof(reply), 4, &p, &n)); - u32 count; - assert(!iboot_parse_modes(p,n,&count) && count==2); - assert(iboot_parse_modes(p,3,&count)==-EPROTO); - assert(iboot_parse_modes(p,n-1,&count)==-EPROTO); - put32(reply+12, 2); - assert(iboot_parse_modes(p,n,&count)==-EPROTO); - put32(reply+12, 1); put32(reply+8, UINT32_MAX); - assert(iboot_parse_modes(p,n,&count)==-EPROTO); - put32(reply+8, 0); - assert(!iboot_parse_modes(p,4,&count) && count==0); - /* Every bounded length/count pair: no helper may read past a declared list. */ - for (size_t size=0; size<200; size++) { - for (u32 c=0; c<16; c++) { - memset(reply,0,sizeof(reply)); put32(reply,c); - int r=iboot_parse_modes(reply,size,&count); - assert((r==0)==(size>=4 && c<=(size-4)/24)); - } - } - u8 mode[24]={0}; - put32(mode,1); put32(mode+4,3840); put32(mode+8,2160); put32(mode+12,60U<<16); - assert(iboot_pattern_mode(4,mode)); - put32(mode+4,1920);put32(mode+8,1080);assert(!iboot_pattern_mode(4,mode)); - put32(mode+4,3840);put32(mode+8,2160); - put32(mode+12,(60U<<16)-1); - assert(!iboot_pattern_mode(4,mode)); - put32(mode+4,1); put32(mode+8,1); put32(mode+12,1); put32(mode+16,32); - assert(iboot_pattern_mode(5,mode)); - put32(mode+16,30); assert(!iboot_pattern_mode(5,mode)); - put32(mode+16,32); put32(mode,0); assert(!iboot_pattern_mode(5,mode)); - assert(!iboot_pattern_mode(6,mode)); - u64 iova=0x10100100000ULL; - size_t fb_size=15360*2160; - assert(!iboot_pattern_params(iova,fb_size,15360)); - assert(iboot_pattern_params(0,fb_size,15360)==-EINVAL); - assert(iboot_pattern_params(iova+1,fb_size,15360)==-EINVAL); - assert(iboot_pattern_params(iova,fb_size-1,15360)==-EINVAL); - assert(iboot_pattern_params(iova,fb_size,15364)==-EINVAL); - assert(iboot_pattern_params(iova,65U*1024*1024,15360)==-EINVAL); - assert(iboot_pattern_params((1ULL<<42)-0x4000,fb_size,15360)==-EINVAL); - assert(iboot_pattern_params(UINT64_MAX,fb_size,15360)==-EINVAL); - struct iboot_swap_layer_v13_3 layer; - iboot_build_pattern_layer(&layer,iova,15360); - assert(sizeof(layer)==216); - assert(offsetof(struct iboot_swap_layer_v13_3,src)==180); - assert(offsetof(struct iboot_swap_layer_v13_3,dst)==196); - u8 expected[216]={0}; - /* Independent byte fixture: upstream m1n1 >=13.3, opaque linear BGRA. */ - put32(expected+12,(u32)iova); put32(expected+16,(u32)(iova>>32)); - put32(expected+24,15360); put32(expected+44,1); - put32(expected+144,1); put32(expected+148,3840); put32(expected+152,2160); - put32(expected+156,1); put32(expected+160,1); put32(expected+164,1); - put32(expected+180,3840); put32(expected+184,2160); - put32(expected+196,3840); put32(expected+200,2160); - assert(!memcmp(&layer,expected,sizeof(expected))); - u8 request[232], power=1, modes[48]={0}, end[12]={0}; - size_t request_size; - assert(!iboot_build_pattern_request(2,&power,1,request,sizeof(request),&request_size)); - assert(request_size==17 && request[0]==2 && request[4]==17 && request[16]==1); - assert(!iboot_build_pattern_request(6,modes,48,request,sizeof(request),&request_size)); - assert(request_size==64); - assert(!iboot_build_pattern_request(15,NULL,0,request,sizeof(request),&request_size)); - assert(request_size==16); - assert(!iboot_build_pattern_request(16,&layer,sizeof(layer),request,sizeof(request),&request_size)); - assert(request_size==232 && request[0]==16 && request[4]==232); - assert(!memcmp(request+16,expected,216)); - for (size_t i=8;i<16;i++) assert(request[i]==0); - assert(!iboot_build_pattern_request(18,end,12,request,sizeof(request),&request_size)); - assert(request_size==28); - __le32 wait[4] = {0, 123, 1, 0}; - assert(!iboot_build_pattern_request(19,wait,sizeof(wait),request,sizeof(request),&request_size)); - assert(request_size==32 && request[0]==19 && request[4]==32); - assert(get_unaligned_le32(request+16)==0 && get_unaligned_le32(request+20)==123); - assert(get_unaligned_le32(request+24)==1 && get_unaligned_le32(request+28)==0); - assert(iboot_build_pattern_request(19,wait,12,request,sizeof(request),&request_size)==-EINVAL); - assert(iboot_build_pattern_request(1,&layer,sizeof(layer),request,sizeof(request),&request_size)==-EINVAL); - assert(iboot_build_pattern_request(16,&layer,sizeof(layer)-8,request,sizeof(request),&request_size)==-EINVAL); - assert(iboot_build_pattern_request(16,&layer,sizeof(layer),request,sizeof(request)-1,&request_size)==-EINVAL); - assert(iboot_build_pattern_request(2,NULL,1,request,sizeof(request),&request_size)==-EINVAL); - /* 14.7 Begin has a 284-byte wrapper claim but returns a 28-byte record. */ - u32 swap_id; - memset(reply,0,sizeof(reply)); - put32(reply,15); put32(reply+4,284); put32(reply+20,123); - assert(!iboot_parse_swap_begin(reply,28,&swap_id) && swap_id==123); - for (size_t received=0;received<28;received++) - assert(iboot_parse_swap_begin(reply,received,&swap_id)==-EPROTO); - assert(iboot_parse_swap_begin(reply,29,&swap_id)==-EPROTO); - put32(reply+4,283); - assert(iboot_parse_swap_begin(reply,28,&swap_id)==-EPROTO); - put32(reply+4,284); put32(reply,14); - assert(iboot_parse_swap_begin(reply,28,&swap_id)==-EPROTO); - put32(reply,15); put32(reply+4,28); - assert(!iboot_parse_swap_begin(reply,28,&swap_id) && swap_id==123); - puts("PASS: actual iBoot query/pattern helpers, >=13.3 layer bytes, bounds and malformed replies"); -} -''' -with tempfile.TemporaryDirectory(prefix='iboot-query-test-') as path: - path=Path(path) - (path/'test.c').write_text(prefix+layouts+helpers+tests) - subprocess.run([os.environ.get('CC','cc'), '-Wall', '-Wextra', '-Werror', '-I', str(base), - '-fsanitize=address,undefined', '-fno-omit-frame-pointer', - '-o', str(path/'test'), str(path/'test.c')],check=True) - subprocess.run([str(path/'test')],check=True) From c3a05b1d419c66061a65dec87e06a2e1b64e819d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 01:20:08 -0700 Subject: [PATCH 1570/2680] drm/apple: afk: deliver a 14.7 command result that has no reply payload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DCP firmware 14.7 answers a command to an unpublished interface with a REPLY notification that carries only an IOReturn. The handler completed the command but left its reply length unset, so the strict reply-length path (iBoot frame commands) reported -EPROTO instead of the firmware's result. Record a zero-length reply, and only for a command that is still pending. The handler runs only on 14.7 firmware now; the 12.3 and 13.5 firmware of M1 and M2 Macs keep reporting such a message as unhandled, as before. [ryan: adapted to 7.2.10 — the DCP_FIRMWARE_V_14_7 gate on this handler was already added when 107036c was ported; this carries the cmd_map check and the zero-length reply.] Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 3bcbe123e5a56cb02cb9ed2acf0b0c84479cb6cb) --- drivers/gpu/drm/apple/afk.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index af19176f97c79..bac05dda255fe 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -824,8 +824,12 @@ static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, ep->endpoint, channel, type, tag, rc); if (idx < MAX_PENDING_CMDS) { spin_lock_irqsave(&service->lock, flags); - if (!service->cmds[idx].done && + if (test_bit(idx, service->cmd_map) && + !service->cmds[idx].done && service->cmds[idx].tag == tag) { + /* A result with no reply payload, not a short reply. */ + service->cmds[idx].reply_len = 0; + service->cmds[idx].reply_len_valid = true; service->cmds[idx].done = true; service->cmds[idx].retcode = rc; if (service->cmds[idx].completion) From 958b4391fb80d987c13be1e2b404e547aaedcee4 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 01:20:39 -0700 Subject: [PATCH 1571/2680] drm/apple: dptxep: drop the unused DPTX interface preparation helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dptx_prepare_interfaces() and its AV service table came with the M3 display work but nothing calls them; the 14.7 start path opens the DPAV controller through dpav_ctrl_init() only. [ryan: adapted to 7.2.10 — the rtkit.h include that 107036c added here sat after afk.h in this tree; removed there.] Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 2d15846e0b75075c5975f45e78501861a40efb18) --- drivers/gpu/drm/apple/dptxep.c | 42 ---------------------------------- drivers/gpu/drm/apple/dptxep.h | 1 - 2 files changed, 43 deletions(-) diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index e9a6db2b60b3f..967e065d6d4ce 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -14,7 +14,6 @@ #include #include "afk.h" -#include #include "dcp.h" #include "dptxep.h" #include "dcp-180hz.h" @@ -1112,47 +1111,6 @@ static const struct apple_epic_service_ops dpav_ctrl_ops[] = { {} }; -static void dptx_av_bind(struct apple_epic_service *service, const char *name, - const char *class, s64 unit) -{ - dev_info(service->ep->dcp->dev, - "AV service %s class %s unit %lld channel %u\n", - name ? name : "", class ? class : "", unit, service->channel); -} - -static const struct apple_epic_service_ops dptx_av_ops[] = { - { .name = "DCPAVSimpleVideoInterface", .init = dptx_av_bind }, - { .name = "dcpav-video-interface-epic", .init = dptx_av_bind }, - { .name = "IOAVVideoInterface", .init = dptx_av_bind }, - { .name = "IOAVController", .init = dptx_av_bind }, - {} -}; - -int dptx_prepare_interfaces(struct apple_dcp *dcp) -{ - int ret; - - if (apple_rtkit_has_endpoint(dcp->rtk, AV_ENDPOINT) && !dcp->avep) { - dcp->avep = afk_init(dcp, AV_ENDPOINT, dptx_av_ops); - if (IS_ERR(dcp->avep)) { - dev_warn(dcp->dev, "AV endpoint init failed: %ld\n", - PTR_ERR(dcp->avep)); - dcp->avep = NULL; - } else { - ret = afk_start(dcp->avep); - dev_info(dcp->dev, "AV endpoint start: %d\n", ret); - } - } - - if (apple_rtkit_has_endpoint(dcp->rtk, DPAVSERV_ENDPOINT) && - !dcp->dcpavservep) { - ret = dpavservep_init(dcp); - dev_info(dcp->dev, "DPAV service endpoint: %d\n", ret); - } - - return 0; -} - int dpav_ctrl_init(struct apple_dcp *dcp) { int ret; diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index 0ff6152732645..74f6be07528a2 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -77,5 +77,4 @@ int dptxport_set_hpd_timeout(struct apple_epic_service *service, bool hpd, struct apple_dcp; int dpav_ctrl_init(struct apple_dcp *dcp); -int dptx_prepare_interfaces(struct apple_dcp *dcp); #endif From f1fc2847eb8880c9dca4ba64fb1898a1521af077 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 13:43:21 -0700 Subject: [PATCH 1572/2680] drm/apple: Reclaim timed-out commands on short AFK replies A late 14.7 short command result marks the command done without checking free_on_ack. The sender has already returned, so its coherent buffers and pending-command slot remain allocated. A following full result is rejected as already handled, and repeated failures exhaust the slots. Represent the short result as an epic_cmd with an empty DMA reply and pass it through the existing full-result handler. This releases the slot under the service lock and frees DMA outside it using the original allocation lengths. Handle the short result before the torn-down-service gate so an outstanding command can also be reclaimed after teardown. The short-result branch remains restricted to firmware compatibility 14.7. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit b12111f4d48f979f57bbf44f2c02106793fa4c91) --- drivers/gpu/drm/apple/afk.c | 54 +++++++++++++------------------------ 1 file changed, 18 insertions(+), 36 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index bac05dda255fe..4c9db64069d73 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -724,6 +724,24 @@ static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, ep->endpoint, channel); return; } + /* + * 14.7 can return only an IOReturn, including after a host timeout or + * service teardown. Use the normal result path to release late-ack DMA + * and command slots with their original allocation lengths. + */ + if (ep->dcp->fw_compat == DCP_FIRMWARE_V_14_7 && + eshdr->category == EPIC_CAT_REPLY && payload_size >= 4 && + payload_size < sizeof(struct epic_cmd)) { + struct epic_cmd cmd = {}; + u16 tag = le16_to_cpu(eshdr->tag); + + memcpy(&cmd.retcode, payload, sizeof(cmd.retcode)); + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: channel %d short reply type %#x tag %#x ret %#x\n", + ep->endpoint, channel, type, tag, le32_to_cpu(cmd.retcode)); + return afk_recv_handle_reply(ep, channel, tag, &cmd, sizeof(cmd)); + } + if (service->torndown) { dev_warn(ep->dcp->dev, "AFK[ep:%02x]: std service notify on torn down service " @@ -804,42 +822,6 @@ static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, return; } - /* - * 14.7 answers a command to an unpublished interface with a NOTIFY - * whose category is REPLY and whose payload is one IOReturn - * (kIOReturnNoDevice is 0xe00002c0). That is the command result. - */ - if (ep->dcp->fw_compat == DCP_FIRMWARE_V_14_7 && - eshdr->category == EPIC_CAT_REPLY && payload_size >= 4 && - payload_size < sizeof(struct epic_cmd)) { - u32 rc; - u16 tag = le16_to_cpu(eshdr->tag); - u8 idx = tag & 0xff; - unsigned long flags; - - memcpy(&rc, payload, sizeof(rc)); - rc = le32_to_cpu(rc); - dev_err(ep->dcp->dev, - "AFK[ep:%02x]: channel %d short reply type %#x tag %#x ret %#x\n", - ep->endpoint, channel, type, tag, rc); - if (idx < MAX_PENDING_CMDS) { - spin_lock_irqsave(&service->lock, flags); - if (test_bit(idx, service->cmd_map) && - !service->cmds[idx].done && - service->cmds[idx].tag == tag) { - /* A result with no reply payload, not a short reply. */ - service->cmds[idx].reply_len = 0; - service->cmds[idx].reply_len_valid = true; - service->cmds[idx].done = true; - service->cmds[idx].retcode = rc; - if (service->cmds[idx].completion) - complete(service->cmds[idx].completion); - } - spin_unlock_irqrestore(&service->lock, flags); - } - return; - } - dev_err(ep->dcp->dev, "AFK[ep:%02x]: channel %d received unhandled standard service message: %x / %x\n", ep->endpoint, channel, type, eshdr->category); From 73e06a4dda24f3a3d038878aca9c3ccb6dd4f86f Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 13:41:59 -0700 Subject: [PATCH 1573/2680] drm/apple: wait for T6030 RTKit callbacks before a DCP is freed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On T6030 the DCP firmware session is kept until reboot: the 14.7 code never stops RTKit or frees struct apple_dcp_v14, and that v14 is the RTKit cookie. The receive and crash callbacks reach struct apple_dcp through v14->dcp, but the apple_dcp is devm-allocated and is freed as soon as platform remove returns. The external processor's remove returned without touching v14 at all. Once dcpext_start had run, unbinding apple,t6030-dcpext left v14->dcp pointing at freed memory, and the next mailbox message or crashlog dereferenced it. The internal DCP clears the pointer in component unbind, but a callback that had already loaded it could still be running when devres freed the apple_dcp, dcp_v14_recv() reloaded the pointer for every check, and unbind does not run at all when bind fails after RTKit was started. Add iomfb_v14_7_remove() and call it from platform remove on 14.7 devices: after component_del() for the internal DCP, and in place of the bare return for the external one. The internal DCP's probe also calls it when component_add() fails, because a bind run from there may already have started RTKit before the probe error frees the apple_dcp. iomfb_v14_7_remove() - disables the external bring-up work first, so a running bring-up finishes and a late dcpext_start write cannot start a session for an apple_dcp that is going away; - stores NULL to v14->dcp; - waits for the RTKit receive worker, so a callback that loaded the old pointer finishes while the apple_dcp is still allocated, and every later callback sees NULL; - then flushes the workqueues of the AFK endpoints that dcp_v14_recv() forwards messages to (iBoot, DPTX and DPAV on the external processor). Those endpoints are devm allocations of the same device, and once v14->dcp is NULL and RTKit has been flushed nothing can queue more work on them; - cancels v14->idle_work synchronously. It does not use the apple_dcp, but no idle run started before removal is left running. dcp_v14_recv() now reads v14->dcp once with READ_ONCE() and forwards nothing when it is NULL, as dcp_v14_crashed() already did. DCPLink messages do not use the apple_dcp and are still handled: the session stays. RTKit runs every recv_message, crashed and shmem_setup callback on its private ordered workqueue, and offers no way to wait for it other than apple_rtkit_free(), apple_rtkit_free_retaining_buffers() and apple_rtkit_reinit(). All three detach the mailbox receiver, which would end the session this driver keeps. Add apple_rtkit_flush_rx(), which only flushes that workqueue: work queued before the flush is waited for, and work queued after it observes the NULL store. As before, the coprocessor is not shut down, and v14, its buffers and its DMA are not freed. M1, M2 and Neo are unaffected: iomfb_v14_7_remove() runs only for DCP_FIRMWARE_V_14_7, which only apple,t6030-dcp and apple,t6030-dcpext select, and nothing else calls apple_rtkit_flush_rx(). [ryan: adapted to 7.2.10 — rtkit hunk carried (apple_rtkit_flush_rx() in drivers/soc/apple/rtkit.c and include/linux/soc/apple/rtkit.h, next to our hold_rx/resume_rx); 010-soc owns the file, and iomfb_v14_7_remove() needs it to compile. Probe keeps our H17P external pre-bind and its failure handling before the 14.7 remove on component_add() failure; the external remove also disables the Type-C reconnect work that this tree enables for the T6030 external start.] Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 65707cd86cb9f041e8fc7120886c4bf26a0e6edc) --- drivers/gpu/drm/apple/dcp.c | 12 +++++- drivers/gpu/drm/apple/iomfb_v14_7.c | 63 +++++++++++++++++++++++------ drivers/gpu/drm/apple/iomfb_v14_7.h | 2 + drivers/soc/apple/rtkit.c | 6 +++ include/linux/soc/apple/rtkit.h | 9 +++++ 5 files changed, 79 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8afb3522f8b49..0aa6ca21cdd2b 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -5100,6 +5100,9 @@ static int dcp_platform_probe(struct platform_device *pdev) if (ret && dcp->external_prepared) { dcp_disable_typec_work(dcp, true); } + /* A failed bind run from here may already have started RTKit. */ + if (ret && dcp->fw_compat == DCP_FIRMWARE_V_14_7) + iomfb_v14_7_remove(dcp); return ret; } @@ -5107,9 +5110,16 @@ static void dcp_platform_remove(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); - if (dcp && dcp->external) + if (dcp && dcp->external) { + /* Enabled for the external start at probe. */ + disable_delayed_work_sync(&dcp->typec_reconnect_wq); + iomfb_v14_7_remove(dcp); return; + } component_del(&pdev->dev, &dcp_comp_ops); + /* Unbind does not wait for RTKit callbacks, or run if bind failed late. */ + if (dcp && dcp->fw_compat == DCP_FIRMWARE_V_14_7) + iomfb_v14_7_remove(dcp); } VISIBLE_IF_KUNIT void dcp_platform_shutdown(struct platform_device *pdev) diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index e0a9a82d3452e..2cfaf67b19b93 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -104,7 +104,7 @@ struct dcp_v14_buffer { struct apple_dcp_v14 { struct device *dev; - /* Cleared when KMS unbinds; the firmware session outlives it. */ + /* Cleared on KMS unbind and on removal; the firmware session outlives it. */ struct apple_dcp *dcp; struct apple_rtkit *rtk; struct dcp_v14_link link; @@ -692,34 +692,36 @@ static void dcp_v14_crashed(void *cookie, const void *crashlog, size_t crashlog_ static void dcp_v14_recv(void *cookie, u8 endpoint, u64 message) { struct apple_dcp_v14 *v14 = cookie; + /* NULL once removal has started; removal waits for this callback. */ + struct apple_dcp *dcp = READ_ONCE(v14->dcp); - if (endpoint == DISP0_ENDPOINT && v14->dcp && v14->dcp->external) { - if (v14->dcp->ibootep) - afk_receive_message(v14->dcp->ibootep, message); + if (endpoint == DISP0_ENDPOINT && dcp && dcp->external) { + if (dcp->ibootep) + afk_receive_message(dcp->ibootep, message); return; } if (endpoint == DPTX_ENDPOINT) { dev_info(v14->dev, "DPTX message %#llx\n", message); - if (v14->dcp && v14->dcp->dptxep) - afk_receive_message(v14->dcp->dptxep, message); + if (dcp && dcp->dptxep) + afk_receive_message(dcp->dptxep, message); return; } if (endpoint == DPAV_CTRL_ENDPOINT) { dev_info(v14->dev, "DPAV message %#llx\n", message); - if (v14->dcp && v14->dcp->dpavctrlep) - afk_receive_message(v14->dcp->dpavctrlep, message); + if (dcp && dcp->dpavctrlep) + afk_receive_message(dcp->dpavctrlep, message); return; } if (endpoint == AV_ENDPOINT) { dev_info(v14->dev, "AV message %#llx\n", message); - if (v14->dcp && v14->dcp->avep) - afk_receive_message(v14->dcp->avep, message); + if (dcp && dcp->avep) + afk_receive_message(dcp->avep, message); return; } if (endpoint == DPAVSERV_ENDPOINT) { dev_info(v14->dev, "DPAVSERV message %#llx\n", message); - if (v14->dcp && v14->dcp->dcpavservep) - afk_receive_message(v14->dcp->dcpavservep, message); + if (dcp && dcp->dcpavservep) + afk_receive_message(dcp->dcpavservep, message); return; } if (endpoint != APPLE_DCP_LINK_ENDPOINT) { @@ -1216,6 +1218,43 @@ void iomfb_v14_7_unbind(struct apple_dcp *dcp) dev_info(dcp->dev, "display unbound; the DCP session and its buffers are kept until reboot\n"); } +/* Waits for the messages dcp_v14_recv() has already queued for its endpoints. */ +static void dcp_v14_flush_endpoints(struct apple_dcp *dcp) +{ + struct apple_dcp_afkep *eps[] = { + dcp->ibootep, dcp->dptxep, dcp->dpavctrlep, dcp->avep, dcp->dcpavservep, + }; + int i; + + /* A failed afk_init() can leave an error pointer behind. */ + for (i = 0; i < ARRAY_SIZE(eps); i++) + if (!IS_ERR_OR_NULL(eps[i])) + flush_workqueue(eps[i]->wq); +} + +void iomfb_v14_7_remove(struct apple_dcp *dcp) +{ + struct apple_dcp_v14 *v14; + + /* Bring-up creates the external session; a late dcpext_start must not. */ + if (dcp->external) + disable_work_sync(&dcp->external_work); + v14 = dcp->v14; + if (!v14) + return; + /* + * devres frees the apple_dcp next, but RTKit keeps calling into v14. + * Callbacks that already loaded the old pointer finish before the + * flush returns; later ones see NULL. + */ + WRITE_ONCE(v14->dcp, NULL); + if (v14->rtk) + apple_rtkit_flush_rx(v14->rtk); + /* Their endpoints are freed with the apple_dcp too; nothing queues more now. */ + dcp_v14_flush_endpoints(dcp); + cancel_work_sync(&v14->idle_work); +} + static int dcp_v14_status_show(struct seq_file *m, void *unused) { struct apple_dcp_v14 *v14 = m->private; diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.h b/drivers/gpu/drm/apple/iomfb_v14_7.h index b3f1efd380fdf..a262ccc053854 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7.h @@ -18,6 +18,8 @@ int iomfb_v14_7_bind(struct apple_dcp *dcp); int iomfb_v14_7_external_start(struct apple_dcp *dcp); /* Component unbind: KMS goes away; the firmware session is kept until reboot. */ void iomfb_v14_7_unbind(struct apple_dcp *dcp); +/* Platform remove: the apple_dcp is freed next; the firmware session is kept. */ +void iomfb_v14_7_remove(struct apple_dcp *dcp); /* DCPLink, start signal, first client open and the panel mode. */ int iomfb_v14_7_start(struct apple_dcp *dcp); diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 10f1bd0b39c50..2dada33c5ca04 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -755,6 +755,12 @@ void apple_rtkit_resume_rx(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_resume_rx); +void apple_rtkit_flush_rx(struct apple_rtkit *rtk) +{ + flush_workqueue(rtk->wq); +} +EXPORT_SYMBOL_GPL(apple_rtkit_flush_rx); + bool apple_rtkit_has_endpoint(struct apple_rtkit *rtk, u8 ep) { return test_bit(ep, rtk->endpoints); diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 00b9a21e936d7..3adeddeada13d 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -246,6 +246,15 @@ int apple_rtkit_poll(struct apple_rtkit *rtk); void apple_rtkit_hold_rx(struct apple_rtkit *rtk); void apple_rtkit_resume_rx(struct apple_rtkit *rtk); +/* + * Wait until the messages already received have been handled, including the + * recv_message, crashed and shmem_setup callbacks the worker thread ran for + * them. Must not be called from one of those callbacks. + * + * @rtk: RTKit reference + */ +void apple_rtkit_flush_rx(struct apple_rtkit *rtk); + /* * Checks if an endpoint with a given index exists * From 1133ce2c7b9b66257d9fee13f784be30e54bd156 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 13:44:38 -0700 Subject: [PATCH 1574/2680] drm/apple: stop the T6030 DCPLink when a call is abandoned dcp_v14_link_call() gives up on a call after five seconds, but it returned -ETIMEDOUT with the link still running and calls[nested].active still set. The caller marks the session failed and never pumps the link again, while dcp_v14_link_receive() kept accepting the late reply and any later callbacks, and queued a kmalloc'd copy of each, up to the 32-event limit. Nothing freed them. A call that failed its reply check or a callback handler left the link running in the same way. And link_fail(), where the event limit, a malformed message and a firmware crash all end up, woke the waiters but left the queued copies on a list that nothing would dequeue again. Stop the link on every return that abandons a submitted call, timeout included, and clear that call's active flag, so the receive path drops whatever arrives for it. Make link_fail() free the queued events under the lock: nothing dequeues from a failed link, and an event that a caller has already dequeued is no longer on the list. The session is still kept until reboot, and the firmware buffers and the RPC memory are not freed. This code runs only with DCP firmware 14.7, which only apple,t6030-dcp and apple,t6030-dcpext select. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 52ee956fff3fce0598dbdb9d6f1327cc02692ef1) --- drivers/gpu/drm/apple/iomfb_v14_7_link.c | 26 ++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_link.c b/drivers/gpu/drm/apple/iomfb_v14_7_link.c index 5eeb86e96f282..e01ec8ee319d5 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7_link.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7_link.c @@ -34,9 +34,17 @@ struct dcp_v14_event { /* Called with the lock held. */ static void link_fail(struct dcp_v14_link *link) { + struct dcp_v14_event *e, *next; + if (!link->failed) dev_err(link->dev, "DCP link stopped; firmware buffers are kept until reboot\n"); link->failed = true; + /* Nothing dequeues events from a failed link. */ + list_for_each_entry_safe(e, next, &link->events, list) { + list_del(&e->list); + kfree(e); + } + link->event_count = 0; wake_up(&link->wait); } @@ -372,6 +380,7 @@ int dcp_v14_link_pump(struct dcp_v14_link *link, unsigned long timeout, /* * Calls @tag and handles the callbacks that arrive until its reply, and, for * a non-zero @completion_id, until the swap-complete callback for that swap. + * A call that fails once submitted, including by timing out, stops the link. * Must not be called concurrently with itself or dcp_v14_link_pump(), except * from inside a callback handler. */ @@ -434,8 +443,10 @@ int dcp_v14_link_call(struct dcp_v14_link *link, u32 tag, const void *input, struct dcp_v14_event *e = link_next_event(link, max_t(long, deadline - jiffies, 0)); struct apple_dcp_link_rpc_header *h; - if (IS_ERR(e)) - return PTR_ERR(e); + if (IS_ERR(e)) { + ret = PTR_ERR(e); + goto abandon; + } h = (void *)e->data; if (e->kind == DCP_V14_EVENT_REPLY) { if (replied || e->size != size || e->message != expected || @@ -456,9 +467,16 @@ int dcp_v14_link_call(struct dcp_v14_link *link, u32 tag, const void *input, } kfree(e); if (ret) - return ret; + goto abandon; if (replied && (!completion_id || completed == completion_id)) return 0; } - return -ETIMEDOUT; + ret = -ETIMEDOUT; +abandon: + /* The firmware may still answer: queue nothing for a call nobody waits for. */ + mutex_lock(&link->lock); + link->calls[nested].active = false; + link_fail(link); + mutex_unlock(&link->lock); + return ret; } From 3a282a0e34c83e8ef31f088767e1b459441bbb61 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 14:19:17 -0700 Subject: [PATCH 1575/2680] drm/apple: stop the T6030 DCPLink when a call fails before submit dcp_v14_link_call() now fails the link on every return that abandons a submitted call, but two earlier returns still left it running: a callback error while draining queued notifications before an outer call, and a nested call that finds no callback to answer. The caller marks the session failed and stops pumping either way, so the link kept copying late firmware messages into its event queue, up to the queue limit, with nothing left to dequeue them. Fail the link on those two returns as well, which also frees anything still queued. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 5f860fa92f7a8b07ec3607854cf900c4b9dbb839) --- drivers/gpu/drm/apple/iomfb_v14_7_link.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_link.c b/drivers/gpu/drm/apple/iomfb_v14_7_link.c index e01ec8ee319d5..38bc2acd11859 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7_link.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7_link.c @@ -415,6 +415,10 @@ int dcp_v14_link_call(struct dcp_v14_link *link, u32 tag, const void *input, ret = dcp_v14_link_pump(link, 0, callback, cookie); if (ret) { kfree(packet); + /* The caller gives up on the session: stop queueing. */ + mutex_lock(&link->lock); + link_fail(link); + mutex_unlock(&link->lock); return ret; } } @@ -422,6 +426,7 @@ int dcp_v14_link_call(struct dcp_v14_link *link, u32 tag, const void *input, mutex_lock(&link->lock); nested = link->callback_depth != 0; if (nested && !link->callback_count) { + link_fail(link); mutex_unlock(&link->lock); kfree(packet); return -EPROTO; From ab1b0f4e34dca01dcaf2b7870bb1daf7f9067b80 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 17:52:57 -0700 Subject: [PATCH 1576/2680] drm/apple: lend the hybrid pipeline last without dual-stream docks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A hybrid pipeline (dcpext0) keeps its fixed HDMI output and can be lent to a Type-C port. Machines without dual-stream docks ranked it like any other pipeline, by CRTC index, and dcpext0 sorts first, so a lone Type-C or Thunderbolt display took the HDMI pipeline. An HDMI display plugged in afterwards was then ignored: its hotplug handler returns while the hybrid is lent out, with nothing logged. On those machines: - Rank Type-C-only pipelines before the hybrid, on DP entry, for waiting ports and for Thunderbolt tunnels, so the hybrid is lent only when no other pipeline is free. A port's last pipeline still comes first unless it was the hybrid and another is free. The port's encoder is narrowed to the routed pipeline before its connector reports connected, so the compositor pairs the connector with it. - When a pipeline is freed, route the direct DP-alt ports left waiting for one. Until now only dual-stream machines did, and elsewhere a display that lost the race stayed dark until it was replugged. - When an HDMI display arrives while the hybrid drives a direct DP-alt display, move that display to a free Type-C-only pipeline (an unplug and replug of it) and give the hybrid back to HDMI. Do the same when a pipeline frees up later with HDMI still waiting. A display that cannot move is put back where it was. A Thunderbolt tunnel never moves: the HDMI display waits and gets the hybrid when the tunnel lets it go, as before. Log which case applies. Dual-stream machines (M2 Pro/Max) keep their allocation, pairing plan and HDMI handling unchanged. [ryan: adapted to 7.2.10 — this tree's port has no dp_release_deadline (USB4 fallback suppression is handled elsewhere), so the reclaim does not reset one] Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 4665509f95673c1734ada0136ddf3c153187c1d6) --- drivers/gpu/drm/apple/dcp.c | 156 +++++++++++++++++++++++++++++++----- 1 file changed, 137 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 0aa6ca21cdd2b..7b350be0f06ba 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -358,11 +358,23 @@ static int dcp_dpxbar_tunnel_select_source(struct mux_control *mux, int state) static unsigned int dcp_typec_route_score(struct apple_dcp_typec_route *route) { struct apple_dcp *dcp = route->dcp; + unsigned int score; if (!dcp->crtc) return UINT_MAX - 1; - return drm_crtc_index(&dcp->crtc->base); + score = drm_crtc_index(&dcp->crtc->base); + /* + * Without dual-stream docks no stream is tied to the hybrid, so it goes + * to a Type-C port only when no Type-C-only pipeline is free, and an HDMI + * display plugged in later finds its pipeline idle. The port's encoder is + * narrowed to the routed pipeline before its connector reports connected, + * so the compositor pairs the connector with that pipeline. + */ + if (!dcp_typec_dual_stream() && dcp->fixed_phy) + score += 100; + + return score; } static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, @@ -821,6 +833,27 @@ dcp_typec_lowest_free(struct apple_dcp_typec_port *port) return best; } +/* + * The pipeline a port without one takes: the one it last had if that is + * free, as a compositor keeps a reconnected connector's CRTC, otherwise the + * best-ranked free one. Without dual-stream docks a free Type-C-only + * pipeline comes first even when the port last had the hybrid, which an HDMI + * display needs. + */ +static struct apple_dcp_typec_route * +dcp_typec_free_route(struct apple_dcp_typec_port *port) +{ + struct apple_dcp_typec_route *last = port->preferred_route; + struct apple_dcp_typec_route *best = dcp_typec_lowest_free(port); + + if (!last || !dcp_typec_route_available(last)) + return best; + if (!dcp_typec_dual_stream() && best && + dcp_typec_route_score(best) < dcp_typec_route_score(last)) + return best; + return last; +} + /* * Route the direct DP-alt ports left waiting for a pipeline. Their DP * state is not reported again while it stays the same, so nothing else @@ -837,9 +870,7 @@ static void dcp_typec_route_waiting(void) if (port->owner || !port->dp_wanted || !port->dp_hpd) continue; - route = port->preferred_route; - if (!route || !dcp_typec_route_available(route)) - route = dcp_typec_lowest_free(port); + route = dcp_typec_free_route(port); if (!route || dcp_typec_route_activate(route, route->xbar)) continue; port->owner = route; @@ -949,19 +980,106 @@ dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, } /* - * A route has let its pipeline go. Until a compositor owns the display - * the plan places everything anew; after that the pipeline goes to a port - * left waiting for one. + * A display is plugged into the HDMI port while its pipeline, the hybrid, + * drives a Type-C port. Without dual-stream docks, move a direct DP-alt + * display to a free Type-C-only pipeline -- an unplug and replug of that + * display, as in dcp_typec_rebalance_locked() -- and give the hybrid back to + * HDMI. A Thunderbolt tunnel never moves. Otherwise the HDMI display waits, + * and gets the hybrid when the Type-C display lets it go. A display that + * cannot move is put back where it was rather than left dark. */ -static void dcp_typec_pipeline_freed(void) +static void dcp_typec_reclaim_hybrid(struct apple_dcp *dcp) { - if (!dcp_typec_dual_stream()) + struct apple_dcp_typec_route *owner = READ_ONCE(dcp->active_typec_route); + struct apple_dcp_typec_route *dest = NULL, *candidate; + struct apple_dcp_typec_port *port; + + lockdep_assert_held(&dcp_typec_fabric_lock); + + if (!owner || !dcp->hdmi_hpd || !gpiod_get_value_cansleep(dcp->hdmi_hpd)) + return; + port = owner->port; + if (owner->tunnel || port->owner != owner) { + dev_info(dcp->dev, + "HDMI display waits: its pipeline drives a Thunderbolt display on %pOF\n", + port->connector_np); return; + } + list_for_each_entry(candidate, &port->routes, port_link) { + if (candidate != owner && !candidate->dcp->fixed_phy && + dcp_typec_route_available(candidate)) { + dest = candidate; + break; + } + } + if (!dest) { + dev_info(dcp->dev, + "HDMI display waits: its pipeline drives %pOF and no other pipeline is free\n", + port->connector_np); + return; + } + + dev_info(dcp->dev, "moving %pOF to %s for the HDMI display\n", + port->connector_np, dev_name(dest->dcp->dev)); + if (port->hpd || dcp->typec_cable_connected || + (dcp->typec_connector && dcp->typec_connector->connected)) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + port->hpd = false; + if (dcp_typec_route_deactivate(owner) && owner->selected) { + /* still routed: leave the display where it is */ + dcp_typec_port_attach(port); + return; + } + port->owner = NULL; + + if (dcp_typec_route_activate(dest, dest->xbar)) { + dev_warn(dcp->dev, "could not move %pOF to %s; the HDMI display waits\n", + port->connector_np, dev_name(dest->dcp->dev)); + if (!dcp_typec_route_activate(owner, owner->xbar)) { + port->owner = owner; + dcp_typec_port_attach(port); + } + return; + } + port->owner = dest; + port->preferred_route = dest; + dcp_typec_port_attach(port); + + if (dcp->active) + dcp_dptx_connect(dcp, 0); +} + +/* A pipeline came free: an HDMI display waiting for its hybrid may get it now. */ +static void dcp_typec_reclaim_waiting_hdmi(void) +{ + struct apple_dcp_typec_port *port; + struct apple_dcp_typec_route *route; + + if (dcp_typec_dual_stream()) + return; + + list_for_each_entry(port, &dcp_typec_ports, link) + list_for_each_entry(route, &port->routes, port_link) + if (route->dcp->fixed_phy && + READ_ONCE(route->dcp->active_typec_route)) + dcp_typec_reclaim_hybrid(route->dcp); +} + +/* + * A route has let its pipeline go. Until a compositor owns the display of a + * dual-stream machine the plan places everything anew; otherwise the + * pipeline goes to a port left waiting for one. The caller has already + * handed a hybrid back to a live HDMI display. + */ +static void dcp_typec_pipeline_freed(void) +{ if (dcp_typec_keep_order()) dcp_typec_rebalance_locked(NULL, 0); else dcp_typec_route_waiting(); + /* a waiting port took the pipeline first: it moves no lit display */ + dcp_typec_reclaim_waiting_hdmi(); } /* @@ -1185,13 +1303,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, } if (!port->owner) { - if (port->preferred_route && - dcp_typec_route_available(port->preferred_route)) - best = port->preferred_route; - - if (!best) - best = dcp_typec_lowest_free(port); - + best = dcp_typec_free_route(port); /* Every pipeline is driving another port's display. */ if (!best) return -ENOSPC; @@ -3371,13 +3483,19 @@ static irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) /* * Until a compositor owns the display, a live HDMI output * takes its pipeline back from a direct DP-alt route: the - * compositor pairs the HDMI connector with it first. + * compositor pairs the HDMI connector with it first. Without + * dual-stream docks the Type-C display moves to a free + * Type-C-only pipeline instead, at any time. */ - if (dcp_typec_keep_order() && + if ((dcp_typec_keep_order() || !dcp_typec_dual_stream()) && gpiod_get_value_cansleep(dcp->hdmi_hpd)) { msleep(500); - if (gpiod_get_value_cansleep(dcp->hdmi_hpd)) + if (!gpiod_get_value_cansleep(dcp->hdmi_hpd)) + return IRQ_HANDLED; + if (dcp_typec_keep_order()) dcp_typec_rebalance_locked(NULL, 0); + else + dcp_typec_reclaim_hybrid(dcp); } return IRQ_HANDLED; } From ba6a21e5e5c86686767fe3fee830757e6c768ac9 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:25:11 -0700 Subject: [PATCH 1577/2680] drm/apple: move no lit display under a compositor for the HDMI port MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 113c3a56e32c moved a direct DP-alt display off the hybrid pipeline when an HDMI display arrived, at any time. Under a compositor that breaks the rule behind dcp_typec_frozen(): the compositor keeps its connector-to-CRTC pairing, and if it misses the brief unplug, it keeps driving the old CRTC, which then shows that display's frames on the HDMI monitor. So: - Move a lit Type-C display only while no compositor or boot splash holds the device. Otherwise the HDMI display waits, says so in the log, and is handed the hybrid when the Type-C display lets it go, as already happens. When the last compositor closes, a waiting HDMI display gets the hybrid. - Restore the dual-stream (M2 Pro/Max) HDMI hotplug path to what it was: a compositor starting during the 500 ms debounce could send it into the move, which has no dual-stream guard of its own. The move now has one. - Debounce only when a move is possible, so the fabric lock is not held for 500 ms just to log that HDMI waits. - If the display can be neither moved nor put back, log it, drop the cached Type-C state so the next report is applied, and leave the port to be served as waiting. Report a crossbar that could not switch to HDMI instead of ignoring it, and check that HDMI is still there before connecting it; if it left, a waiting port may take the pipeline. Fixes: 113c3a56e32c ("drm/apple: lend the hybrid pipeline last without dual-stream docks") [ryan: adapted to 7.2.10 — this tree's port has no dp_release_deadline (USB4 fallback suppression is handled elsewhere), so the move does not reset one] Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 9ad2be15a410755b483913a77001b431a504e0bf) --- drivers/gpu/drm/apple/dcp.c | 76 +++++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 7b350be0f06ba..15383f4ad613d 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -674,6 +674,14 @@ static bool dcp_typec_frozen(struct drm_device *drm) return false; } +/* Can a lit Type-C display move to another pipeline without a compositor noticing? */ +static bool dcp_typec_thawed(void) +{ + struct drm_device *drm = dcp_typec_drm(); + + return drm && READ_ONCE(drm->registered) && !dcp_typec_frozen(drm); +} + /* Are the routes kept in compositor pairing order right now? */ static bool dcp_typec_keep_order(void) { @@ -981,10 +989,12 @@ dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, /* * A display is plugged into the HDMI port while its pipeline, the hybrid, - * drives a Type-C port. Without dual-stream docks, move a direct DP-alt + * drives a Type-C port. Without dual-stream docks, and only while no + * compositor owns the display (see dcp_typec_frozen()), move a direct DP-alt * display to a free Type-C-only pipeline -- an unplug and replug of that * display, as in dcp_typec_rebalance_locked() -- and give the hybrid back to - * HDMI. A Thunderbolt tunnel never moves. Otherwise the HDMI display waits, + * HDMI. A compositor keeps its pairing and could miss the brief unplug, + * and a Thunderbolt tunnel never moves. Otherwise the HDMI display waits, * and gets the hybrid when the Type-C display lets it go. A display that * cannot move is put back where it was rather than left dark. */ @@ -993,10 +1003,13 @@ static void dcp_typec_reclaim_hybrid(struct apple_dcp *dcp) struct apple_dcp_typec_route *owner = READ_ONCE(dcp->active_typec_route); struct apple_dcp_typec_route *dest = NULL, *candidate; struct apple_dcp_typec_port *port; + bool hdmi_ok = true; + int ret; lockdep_assert_held(&dcp_typec_fabric_lock); - if (!owner || !dcp->hdmi_hpd || !gpiod_get_value_cansleep(dcp->hdmi_hpd)) + if (dcp_typec_dual_stream() || !owner || !dcp->hdmi_hpd || + !gpiod_get_value_cansleep(dcp->hdmi_hpd)) return; port = owner->port; if (owner->tunnel || port->owner != owner) { @@ -1005,6 +1018,12 @@ static void dcp_typec_reclaim_hybrid(struct apple_dcp *dcp) port->connector_np); return; } + if (!dcp_typec_thawed()) { + dev_info(dcp->dev, + "HDMI display waits: its pipeline drives the display on %pOF\n", + port->connector_np); + return; + } list_for_each_entry(candidate, &port->routes, port_link) { if (candidate != owner && !candidate->dcp->fixed_phy && @@ -1026,11 +1045,17 @@ static void dcp_typec_reclaim_hybrid(struct apple_dcp *dcp) (dcp->typec_connector && dcp->typec_connector->connected)) dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); port->hpd = false; - if (dcp_typec_route_deactivate(owner) && owner->selected) { + ret = dcp_typec_route_deactivate(owner); + if (ret && owner->selected) { /* still routed: leave the display where it is */ dcp_typec_port_attach(port); return; } + if (ret) { + /* released, but the crossbar did not switch to HDMI */ + dev_warn(dcp->dev, "could not switch to the HDMI display: %d\n", ret); + hdmi_ok = false; + } port->owner = NULL; if (dcp_typec_route_activate(dest, dest->xbar)) { @@ -1039,15 +1064,26 @@ static void dcp_typec_reclaim_hybrid(struct apple_dcp *dcp) if (!dcp_typec_route_activate(owner, owner->xbar)) { port->owner = owner; dcp_typec_port_attach(port); + return; } - return; + /* + * Left without a pipeline: it is served as a waiting port when + * one frees, and its next DP state report is applied again. + */ + dev_err(dcp->dev, "could not put %pOF back; replug that display\n", + port->connector_np); + port->applied_valid = false; + } else { + port->owner = dest; + port->preferred_route = dest; + dcp_typec_port_attach(port); } - port->owner = dest; - port->preferred_route = dest; - dcp_typec_port_attach(port); - if (dcp->active) + /* the HDMI display may have gone while the Type-C display moved */ + if (hdmi_ok && dcp->active && gpiod_get_value_cansleep(dcp->hdmi_hpd)) dcp_dptx_connect(dcp, 0); + else if (hdmi_ok) + dcp_typec_route_waiting(); } /* A pipeline came free: an HDMI display waiting for its hybrid may get it now. */ @@ -1090,13 +1126,12 @@ static void dcp_typec_pipeline_freed(void) */ void dcp_typec_reorder(void) { - if (!dcp_typec_dual_stream()) - return; - guard(mutex)(&dcp_typec_fabric_lock); if (dcp_typec_keep_order()) dcp_typec_rebalance_locked(NULL, 0); + else if (!dcp_typec_dual_stream() && dcp_typec_thawed()) + dcp_typec_reclaim_waiting_hdmi(); } static int dcp_typec_route_prepare(struct typec_mux_dev *mux, @@ -3485,17 +3520,20 @@ static irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) * takes its pipeline back from a direct DP-alt route: the * compositor pairs the HDMI connector with it first. Without * dual-stream docks the Type-C display moves to a free - * Type-C-only pipeline instead, at any time. + * Type-C-only pipeline instead, if it can (see + * dcp_typec_reclaim_hybrid()). */ - if ((dcp_typec_keep_order() || !dcp_typec_dual_stream()) && + if (dcp_typec_keep_order() && gpiod_get_value_cansleep(dcp->hdmi_hpd)) { msleep(500); - if (!gpiod_get_value_cansleep(dcp->hdmi_hpd)) - return IRQ_HANDLED; - if (dcp_typec_keep_order()) + if (gpiod_get_value_cansleep(dcp->hdmi_hpd)) dcp_typec_rebalance_locked(NULL, 0); - else - dcp_typec_reclaim_hybrid(dcp); + } else if (!dcp_typec_dual_stream() && + gpiod_get_value_cansleep(dcp->hdmi_hpd)) { + /* debounce only what may move a display */ + if (dcp_typec_thawed()) + msleep(500); + dcp_typec_reclaim_hybrid(dcp); } return IRQ_HANDLED; } From b98ccce6e5bf2f58f100afe787d572ff18029f70 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:43:43 -0700 Subject: [PATCH 1578/2680] drm/apple: point a parked hybrid back at HDMI when a display arrives there When a Type-C display or Thunderbolt tunnel lets the hybrid pipeline go while its HDMI port is empty, dcp_typec_route_deactivate() parks the pipeline on a Type-C PHY, on purpose. Nothing pointed it back at HDMI when a display was plugged in there later: the HDMI hotplug handler connected with the parked PHY, and every attempt timed out ("timed out waiting for port 0 link configuration", target 0:0 instead of 0:3) until a reboot or another borrow handed the pipeline back with HDMI live. Seen on an M1 Pro 14" with the t600x hybrid: TS4 dock display on dcpext0, dock unplugged, HDMI plugged in. The M2 Pro/Max laptops have shipped the same hybrid and hotplug path, so they hit it after a dock display used dcpext0. Select the HDMI PHY and crossbar route again in the hotplug handler, under the fabric lock, before connecting. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit aff9325a582fbeb3dad05911f182a68d5a5c5a05) --- drivers/gpu/drm/apple/dcp.c | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 15383f4ad613d..4bf78d8e75908 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3507,6 +3507,30 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) return dcp_dptx_disconnect(dcp, port); } +/* + * A hybrid let go while its HDMI port was empty was parked on a Type-C PHY + * (see dcp_typec_route_deactivate()). Point it at the HDMI output again + * before connecting a display that has arrived there. + */ +static int dcp_fixed_output_select(struct apple_dcp *dcp) +{ + int ret; + + lockdep_assert_held(&dcp_typec_fabric_lock); + + if (!dcp->fixed_phy || dcp->active_typec_route) + return 0; + dcp->phy = dcp->fixed_phy; + dcp->dptx_phy = dcp->fixed_dptx_phy; + if (dcp->xbar && !dcp->fixed_route_selected) { + ret = mux_control_select(dcp->xbar, dcp->fixed_mux_index); + if (ret) + return ret; + dcp->fixed_route_selected = true; + } + return 0; +} + static irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) { struct apple_dcp *dcp = data; @@ -3553,8 +3577,14 @@ static irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) dev_info(dcp->dev, "DP2HDMI HPD irq, 500ms debounce: connected:%d\n", connected); } - if (connected) - dcp_dptx_connect(dcp, 0); + if (connected) { + int ret = dcp_fixed_output_select(dcp); + + if (ret) + dev_err(dcp->dev, "could not select the HDMI output: %d\n", ret); + else + dcp_dptx_connect(dcp, 0); + } return IRQ_HANDLED; } From a882e5b23ec46b9b371c60fe0276d8fd5fb5e6e2 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:10:54 -0700 Subject: [PATCH 1579/2680] drm/apple: keep the hybrid from a tunnel while HDMI blinks Seen on an M1 Pro 14" with a dock display waiting for a pipeline (HDMI on dcpext0, a USB-C display on dcpext1): after s2idle the HDMI monitor dropped HPD for about a second, a few seconds after waking. The waiting tunnel's next retry found the hybrid free and took it, and as a tunnel never moves, the HDMI display stayed dark when HPD came back. Without the tunnel retry the dock display would have given up long before, so this is new. Without dual-stream docks, refresh a 10 s hold on every HDMI HPD edge and let no tunnel take the hybrid while it runs. The hybrid is still lent once HDMI has been quiet that long; a waiting tunnel then gets it on a retry. Direct DP-alt routes and the dual-stream machines are unchanged. Fixes: 08c8f10e0ef4 ("thunderbolt: apple: retry a DP tunnel left without a display pipeline") Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 69742f23bcd05aa99e7743078d464b8d56a833e7) --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/dcp.c | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 3c957afe53113..4f7687ea40846 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -554,6 +554,8 @@ struct apple_dcp { */ struct list_head dropped; } iomfb; + /* jiffies until which a Thunderbolt tunnel may not borrow this pipeline */ + unsigned long hdmi_hold_until; }; void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, bool connected); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 4bf78d8e75908..362aed1edfbf1 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -694,6 +694,21 @@ static bool dcp_typec_keep_order(void) return drm && READ_ONCE(drm->registered) && !dcp_typec_frozen(drm); } +/* + * HDMI hotplug often blinks: a monitor waking up or switching inputs drops + * HPD for a second or so. A Thunderbolt tunnel never moves once it has a + * pipeline, so without dual-stream docks one may not take the hybrid until + * HDMI has been quiet for this long, or a tunnel waiting for a pipeline + * would take it in the blink and keep the returning HDMI display dark. + */ +#define DCP_HDMI_HOLD_MS 10000 + +static bool dcp_hdmi_held(struct apple_dcp *dcp) +{ + return dcp->hdmi_hpd && dcp->hdmi_hold_until && + time_before(jiffies, dcp->hdmi_hold_until); +} + /* A Thunderbolt tunnel holds @dcp's pipeline: it never moves. */ static bool dcp_typec_tunnel_held(struct apple_dcp *dcp) { @@ -1670,6 +1685,8 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, continue; if (!dcp_typec_route_available(candidate)) continue; + if (!dcp_typec_dual_stream() && dcp_hdmi_held(candidate->dcp)) + continue; if (!dcp_typec_route_fits(candidate, dpin ? port->secondary_connector : port->connector)) @@ -3538,6 +3555,9 @@ static irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) guard(mutex)(&dcp_typec_fabric_lock); + /* any edge: a display is there or just was (see dcp_hdmi_held()) */ + dcp->hdmi_hold_until = jiffies + msecs_to_jiffies(DCP_HDMI_HOLD_MS); + if (READ_ONCE(dcp->active_typec_route)) { /* * Until a compositor owns the display, a live HDMI output From b4a0104fd5002d9a568ef991eab9e58faedd8008 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:13:32 -0700 Subject: [PATCH 1580/2680] drm/apple: drop the HDMI reclaim move; HDMI waits for the hybrid MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of 0348a5b05dfa: the move could not run on any machine. dcp_typec_thawed() goes through dcp_typec_drm(), which needs the secondary connectors that only dual-stream machines have, and those are excluded from the move. Making it reachable would still race a compositor taking master while the move runs, since the frozen check is a snapshot and not a handoff barrier. So keep the simple rule that the hardware tests already cover: without dual-stream docks, a display plugged into HDMI while the hybrid drives a Type-C port waits, says so, and is handed the hybrid when the Type-C display lets it go. Remove the move, its thaw check, the reclaim on a freed pipeline and the reorder hook change; dcp_typec_reorder() is as it was. Fixes: 113c3a56e32c ("drm/apple: lend the hybrid pipeline last without dual-stream docks") [ryan: adapted to 7.2.10 — this tree's port has no dp_release_deadline (USB4 fallback suppression is handled elsewhere), so the move does not reset one] Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 687d01702b95c0b30e4538157402782bcc144808) --- drivers/gpu/drm/apple/dcp.c | 139 ++++-------------------------------- 1 file changed, 15 insertions(+), 124 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 362aed1edfbf1..159a65af93863 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -674,14 +674,6 @@ static bool dcp_typec_frozen(struct drm_device *drm) return false; } -/* Can a lit Type-C display move to another pipeline without a compositor noticing? */ -static bool dcp_typec_thawed(void) -{ - struct drm_device *drm = dcp_typec_drm(); - - return drm && READ_ONCE(drm->registered) && !dcp_typec_frozen(drm); -} - /* Are the routes kept in compositor pairing order right now? */ static bool dcp_typec_keep_order(void) { @@ -1004,117 +996,22 @@ dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, /* * A display is plugged into the HDMI port while its pipeline, the hybrid, - * drives a Type-C port. Without dual-stream docks, and only while no - * compositor owns the display (see dcp_typec_frozen()), move a direct DP-alt - * display to a free Type-C-only pipeline -- an unplug and replug of that - * display, as in dcp_typec_rebalance_locked() -- and give the hybrid back to - * HDMI. A compositor keeps its pairing and could miss the brief unplug, - * and a Thunderbolt tunnel never moves. Otherwise the HDMI display waits, - * and gets the hybrid when the Type-C display lets it go. A display that - * cannot move is put back where it was rather than left dark. + * drives a Type-C port. Without dual-stream docks nothing moves for it: a + * compositor keeps its connector-to-CRTC pairing and could miss a moved + * display's brief unplug, and a Thunderbolt tunnel never moves. The HDMI + * display waits and is handed the hybrid when the Type-C display lets it go + * (see dcp_typec_route_deactivate()). */ -static void dcp_typec_reclaim_hybrid(struct apple_dcp *dcp) +static void dcp_typec_hdmi_waits(struct apple_dcp *dcp) { struct apple_dcp_typec_route *owner = READ_ONCE(dcp->active_typec_route); - struct apple_dcp_typec_route *dest = NULL, *candidate; - struct apple_dcp_typec_port *port; - bool hdmi_ok = true; - int ret; lockdep_assert_held(&dcp_typec_fabric_lock); - if (dcp_typec_dual_stream() || !owner || !dcp->hdmi_hpd || - !gpiod_get_value_cansleep(dcp->hdmi_hpd)) - return; - port = owner->port; - if (owner->tunnel || port->owner != owner) { - dev_info(dcp->dev, - "HDMI display waits: its pipeline drives a Thunderbolt display on %pOF\n", - port->connector_np); - return; - } - if (!dcp_typec_thawed()) { - dev_info(dcp->dev, - "HDMI display waits: its pipeline drives the display on %pOF\n", - port->connector_np); - return; - } - - list_for_each_entry(candidate, &port->routes, port_link) { - if (candidate != owner && !candidate->dcp->fixed_phy && - dcp_typec_route_available(candidate)) { - dest = candidate; - break; - } - } - if (!dest) { - dev_info(dcp->dev, - "HDMI display waits: its pipeline drives %pOF and no other pipeline is free\n", - port->connector_np); - return; - } - - dev_info(dcp->dev, "moving %pOF to %s for the HDMI display\n", - port->connector_np, dev_name(dest->dcp->dev)); - if (port->hpd || dcp->typec_cable_connected || - (dcp->typec_connector && dcp->typec_connector->connected)) - dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - port->hpd = false; - ret = dcp_typec_route_deactivate(owner); - if (ret && owner->selected) { - /* still routed: leave the display where it is */ - dcp_typec_port_attach(port); - return; - } - if (ret) { - /* released, but the crossbar did not switch to HDMI */ - dev_warn(dcp->dev, "could not switch to the HDMI display: %d\n", ret); - hdmi_ok = false; - } - port->owner = NULL; - - if (dcp_typec_route_activate(dest, dest->xbar)) { - dev_warn(dcp->dev, "could not move %pOF to %s; the HDMI display waits\n", - port->connector_np, dev_name(dest->dcp->dev)); - if (!dcp_typec_route_activate(owner, owner->xbar)) { - port->owner = owner; - dcp_typec_port_attach(port); - return; - } - /* - * Left without a pipeline: it is served as a waiting port when - * one frees, and its next DP state report is applied again. - */ - dev_err(dcp->dev, "could not put %pOF back; replug that display\n", - port->connector_np); - port->applied_valid = false; - } else { - port->owner = dest; - port->preferred_route = dest; - dcp_typec_port_attach(port); - } - - /* the HDMI display may have gone while the Type-C display moved */ - if (hdmi_ok && dcp->active && gpiod_get_value_cansleep(dcp->hdmi_hpd)) - dcp_dptx_connect(dcp, 0); - else if (hdmi_ok) - dcp_typec_route_waiting(); -} - -/* A pipeline came free: an HDMI display waiting for its hybrid may get it now. */ -static void dcp_typec_reclaim_waiting_hdmi(void) -{ - struct apple_dcp_typec_port *port; - struct apple_dcp_typec_route *route; - - if (dcp_typec_dual_stream()) - return; - - list_for_each_entry(port, &dcp_typec_ports, link) - list_for_each_entry(route, &port->routes, port_link) - if (route->dcp->fixed_phy && - READ_ONCE(route->dcp->active_typec_route)) - dcp_typec_reclaim_hybrid(route->dcp); + if (owner) + dev_info(dcp->dev, "HDMI display waits: its pipeline drives the %s on %pOF\n", + owner->tunnel ? "Thunderbolt display" : "display", + owner->port->connector_np); } /* @@ -1129,8 +1026,6 @@ static void dcp_typec_pipeline_freed(void) dcp_typec_rebalance_locked(NULL, 0); else dcp_typec_route_waiting(); - /* a waiting port took the pipeline first: it moves no lit display */ - dcp_typec_reclaim_waiting_hdmi(); } /* @@ -1141,12 +1036,13 @@ static void dcp_typec_pipeline_freed(void) */ void dcp_typec_reorder(void) { + if (!dcp_typec_dual_stream()) + return; + guard(mutex)(&dcp_typec_fabric_lock); if (dcp_typec_keep_order()) dcp_typec_rebalance_locked(NULL, 0); - else if (!dcp_typec_dual_stream() && dcp_typec_thawed()) - dcp_typec_reclaim_waiting_hdmi(); } static int dcp_typec_route_prepare(struct typec_mux_dev *mux, @@ -3563,9 +3459,7 @@ static irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) * Until a compositor owns the display, a live HDMI output * takes its pipeline back from a direct DP-alt route: the * compositor pairs the HDMI connector with it first. Without - * dual-stream docks the Type-C display moves to a free - * Type-C-only pipeline instead, if it can (see - * dcp_typec_reclaim_hybrid()). + * dual-stream docks it waits (see dcp_typec_hdmi_waits()). */ if (dcp_typec_keep_order() && gpiod_get_value_cansleep(dcp->hdmi_hpd)) { @@ -3574,10 +3468,7 @@ static irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) dcp_typec_rebalance_locked(NULL, 0); } else if (!dcp_typec_dual_stream() && gpiod_get_value_cansleep(dcp->hdmi_hpd)) { - /* debounce only what may move a display */ - if (dcp_typec_thawed()) - msleep(500); - dcp_typec_reclaim_hybrid(dcp); + dcp_typec_hdmi_waits(dcp); } return IRQ_HANDLED; } From b8c02b2392d041fa277fe574857ff71693556f86 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:55:38 -0700 Subject: [PATCH 1581/2680] drm/apple: start the HDMI hold at the edge, and on resume Review of 5fde3db6ec08: the hold was set by the threaded HPD handler after it took the fabric lock, so a tunnel retry that got the lock first could still find the hybrid free in the blink. That needs the fixed connector to read disconnected already, which it does while resuming: suspend reports it disconnected until dcp_poweron() connects it again. Set the hold from a primary handler at the edge itself, keep refreshing it from the thread, and set it when the HPD interrupt is enabled again on resume, as edges in sleep are not seen and monitors often blink on waking. The hold is still read only without dual-stream docks. Fixes: 5fde3db6ec08 ("drm/apple: keep the hybrid from a tunnel while HDMI blinks") Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit a3828b73e0ef958f23ead2acf3401b8a1057cfc6) --- drivers/gpu/drm/apple/dcp.c | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 159a65af93863..ac36dbffaaf45 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -695,10 +695,17 @@ static bool dcp_typec_keep_order(void) */ #define DCP_HDMI_HOLD_MS 10000 +static void dcp_hdmi_hold(struct apple_dcp *dcp) +{ + WRITE_ONCE(dcp->hdmi_hold_until, + jiffies + msecs_to_jiffies(DCP_HDMI_HOLD_MS)); +} + static bool dcp_hdmi_held(struct apple_dcp *dcp) { - return dcp->hdmi_hpd && dcp->hdmi_hold_until && - time_before(jiffies, dcp->hdmi_hold_until); + unsigned long until = READ_ONCE(dcp->hdmi_hold_until); + + return dcp->hdmi_hpd && until && time_before(jiffies, until); } /* A Thunderbolt tunnel holds @dcp's pipeline: it never moves. */ @@ -3444,6 +3451,18 @@ static int dcp_fixed_output_select(struct apple_dcp *dcp) return 0; } +/* + * Any HDMI HPD edge: a display is there or just was. Start the hold at the + * edge itself, before a tunnel retry can take the fabric lock ahead of the + * thread (see dcp_hdmi_held()). + */ +static irqreturn_t dcp_dp2hdmi_hpd_edge(int irq, void *data) +{ + dcp_hdmi_hold(data); + + return IRQ_WAKE_THREAD; +} + static irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) { struct apple_dcp *dcp = data; @@ -3451,8 +3470,8 @@ static irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) guard(mutex)(&dcp_typec_fabric_lock); - /* any edge: a display is there or just was (see dcp_hdmi_held()) */ - dcp->hdmi_hold_until = jiffies + msecs_to_jiffies(DCP_HDMI_HOLD_MS); + /* again from here, should the edge handler not have run */ + dcp_hdmi_hold(dcp); if (READ_ONCE(dcp->active_typec_route)) { /* @@ -5094,7 +5113,7 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp->hdmi_hpd_irq = irq; ret = devm_request_threaded_irq(dev, dcp->hdmi_hpd_irq, - NULL, dcp_dp2hdmi_hpd, + dcp_dp2hdmi_hpd_edge, dcp_dp2hdmi_hpd, IRQF_ONESHOT | IRQF_NO_AUTOEN | IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING, "dp2hdmi-hpd-irq", dcp); @@ -5439,8 +5458,11 @@ static int dcp_platform_resume(struct device *dev) dcp_release_after_sleep(dcp); dcp_enable_typec_work(dcp); - if (dcp->hdmi_hpd_irq) + if (dcp->hdmi_hpd_irq) { + /* edges were not seen in sleep, and monitors blink on waking */ + dcp_hdmi_hold(dcp); enable_irq(dcp->hdmi_hpd_irq); + } if (dcp->avep) av_service_connect(dcp); From 470cac290c5ba860a710d66b8a86291f4c2b76c1 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 21:58:09 -0700 Subject: [PATCH 1582/2680] drm/apple: move the Type-C display fabric into its own file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Type-C allocator, Thunderbolt display bridge and HDMI routing policy share ownership and locking rules but are buried in the DCP firmware and device lifecycle implementation. Give them a dedicated unit so the allocation decisions can be isolated and tested in subsequent changes. Move the port list, route handling, pairing plan and fixed-output hotplug policy to dcp-fabric.c, with their structures and internal API in dcp-fabric.h. Keep the fabric mutex private by moving the guarded DPTX endpoint shutdown block into a helper. Preserve the existing selection, locking, firmware calls, timeouts, retry policy and device-tree behavior. The new unit remains part of appledrm with no new configuration dependency. Unified arm64 LLVM W=1 directory builds pass with only baseline diagnostics; strict checkpatch and diff checks pass. All 113 Apple DTBs are unchanged. Source and decision-function object comparisons confirm equivalent logic. [ryan: adapted to 7.2.10 — the move is redone on this tree's dcp.c, so dcp-fabric.c carries our fabric code verbatim (SAFE prepare/finish/commit mux phases, H17P external checks, free-route ranking, deactivate error tracking, the T6030 paths) rather than the source's older copy; dcp_is_external_h17p() and the DPTX connect/disconnect helpers become shared; the apple-dcp host selftests read dcp.c and dcp-fabric.c as one unit] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 3a37ff8ce5b3584eb1b8d23441172fa1274174fc) --- MAINTAINERS | 1 + drivers/gpu/drm/apple/Kconfig | 3 +- drivers/gpu/drm/apple/Makefile | 2 +- drivers/gpu/drm/apple/dcp-fabric.c | 1980 +++++++++++++++++ drivers/gpu/drm/apple/dcp-fabric.h | 86 + drivers/gpu/drm/apple/dcp-internal.h | 26 +- drivers/gpu/drm/apple/dcp.c | 1974 +--------------- drivers/gpu/drm/apple/dcp.h | 27 +- .../selftests/drivers/gpu/apple-dcp/common.py | 15 + 9 files changed, 2096 insertions(+), 2018 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp-fabric.c create mode 100644 drivers/gpu/drm/apple/dcp-fabric.h diff --git a/MAINTAINERS b/MAINTAINERS index a80297d5450a7..f3dad1855d4b1 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2010,6 +2010,7 @@ L: dri-devel@lists.freedesktop.org S: Maintained T: git git://anongit.freedesktop.org/drm/drm-misc F: drivers/gpu/drm/apple/ +F: drivers/gpu/drm/apple/dcp-fabric.[ch] APPLE PCIE CONTROLLER DRIVER M: Marc Zyngier diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 621beb052be14..542ec8c9707f1 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -15,7 +15,8 @@ config DRM_APPLE depends on BACKLIGHT_CLASS_DEVICE select MULTIPLEXER help - Say Y if you have an Apple Silicon chipset. + Say Y if you have an Apple Silicon chipset. This includes the + Type-C display fabric and Thunderbolt display routing. config DRM_APPLE_AUDIO bool "DisplayPort/HDMI Audio support" diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 11500d7e54a7a..0e6d991332113 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -3,7 +3,7 @@ CFLAGS_trace.o = -I$(src) appledrm-y := apple_drv.o apple_resume.o -appledrm-y += afk.o dcp.o dcp_backlight.o dptxep.o iomfb.o parser.o systemep.o +appledrm-y += afk.o dcp.o dcp-fabric.o dcp_backlight.o dptxep.o iomfb.o parser.o systemep.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += audio.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += av.o appledrm-y += connector.o dcp-display-metadata.o dcp-180hz.o diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c new file mode 100644 index 0000000000000..50431d6c3f25b --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -0,0 +1,1980 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "afk.h" +#include "dcp.h" +#include "dcp-fabric.h" +#include "dcpext_scanout.h" +#include "ibootep.h" +#include "dcp-internal.h" +struct apple_dcp_typec_port { + struct list_head link; + struct list_head routes; + struct device_node *connector_np; + struct apple_dcp_typec_route *owner; + struct apple_dcp_typec_route *secondary_owner; + /* Keep a port on its last DCP while that pipeline remains free. */ + struct apple_dcp_typec_route *preferred_route; + /* DRM connector for this physical port, driven by whichever DCP owns it */ + struct apple_connector *connector; + /* A second logical stream through this port's USB4 dock. */ + struct apple_connector *secondary_connector; + /* last mux state acted on, to collapse the per-candidate notifications */ + struct typec_altmode *applied_alt; + unsigned long applied_mode; + u32 applied_status; + u32 applied_conf; + bool applied_valid; + /* SAFE preparation retains this owner until the set phase completes. */ + bool safe_pending; + bool safe_finish_ready; + bool hpd; + /* Direct DP-alt: the port is in DP mode, routed or not, and its HPD. */ + bool dp_wanted; + bool dp_hpd; + /* dcp_typec_rebalance_locked()'s plan for the port's two streams */ + struct apple_dcp_typec_route *target; + struct apple_dcp_typec_route *secondary_target; + /* left out of the plan: its planned pipeline is held by a tunnel */ + bool plan_dark; +}; + +/* + * HDMI hotplug often blinks: a monitor waking up or switching inputs drops + * HPD for a second or so. A Thunderbolt tunnel never moves once it has a + * pipeline, so without dual-stream docks one may not take the hybrid until + * HDMI has been quiet for this long, or a tunnel waiting for a pipeline + * would take it in the blink and keep the returning HDMI display dark. + */ +#define DCP_HDMI_HOLD_MS 10000 + +static DEFINE_MUTEX(dcp_typec_fabric_lock); + +static LIST_HEAD(dcp_typec_ports); + +bool dcp_is_typec_output(struct apple_dcp *dcp) +{ + return dcp->active_typec_route || + dcp->fixed_connector_type == DRM_MODE_CONNECTOR_USB; +} + +bool dcp_is_usb4_output(struct apple_dcp *dcp) +{ + return dcp->active_typec_route && dcp->active_typec_route->tunnel; +} + +static bool dcp_typec_route_is_dp(const struct typec_mux_state *state) +{ + return state->alt && state->alt->svid == USB_TYPEC_DP_SID && + state->mode >= TYPEC_DP_STATE_A && + state->mode <= TYPEC_DP_STATE_F; +} + +/* Keep a live fixed output on its own pipeline. */ +static bool dcp_typec_route_fixed_output_busy(struct apple_dcp_typec_route *route) +{ + struct apple_dcp *dcp = route->dcp; + + if (dcp->fixed_connector_type == DRM_MODE_CONNECTOR_USB) + return false; + /* + * The 14.7 firmware drives the internal panel on the IOMFB path. + * The DPTX service is a separate output, so the live panel does not + * occupy the Type-C route. + */ + if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) + return false; + if (dcp->fixed_connector && dcp->fixed_connector->connected) + return true; + if (dcp->hdmi_hpd && gpiod_get_value_cansleep(dcp->hdmi_hpd)) + return true; + + return false; +} + +static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) +{ + struct apple_dcp_typec_port *port; + + list_for_each_entry(port, &dcp_typec_ports, link) { + if (port->safe_pending && port->owner && + port->owner->dcp == route->dcp) + return false; + } + return !route->dcp->active_typec_route && + !dcp_typec_route_fixed_output_busy(route); +} + +/* + * Machines whose USB4 docks carry two independent DP streams through one + * port: DPIN0 drives the hybrid dcpext0, DPIN1 the Type-C-only dcpext1, and + * each port has a second connector for the DPIN1 stream. + */ +bool dcp_typec_dual_stream(void) +{ + return apple_dp_tunnel_t602x(); +} + +bool dcp_is_typec_only(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return !dcp->fixed_phy; +} + +/* + * Can this route feed the given connector without changing its + * possible_crtcs? On dual-stream machines those are fixed at probe, since + * compositors read them once when the connector appears and pair the + * connector with a CRTC before this fabric has routed it. + */ +static bool dcp_typec_route_fits(struct apple_dcp_typec_route *route, + struct apple_connector *connector) +{ + struct apple_dcp *dcp = route->dcp; + + if (!dcp_typec_dual_stream() || !connector || !dcp->crtc) + return true; + return connector->candidate_crtcs & drm_crtc_mask(&dcp->crtc->base); +} + +static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, + struct mux_control *xbar); +static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); +static int dcp_dpxbar_preselect(struct mux_control *mux, int state); + +static int dcp_dpxbar_tunnel_select_source(struct mux_control *mux, int state) +{ + typeof(&apple_dpxbar_tunnel_select_source) select = + symbol_get(apple_dpxbar_tunnel_select_source); + int ret; + + if (!select) + return -ENOENT; + ret = select(mux, state); + symbol_put(apple_dpxbar_tunnel_select_source); + return ret; +} + +/* + * For a port without a prior owner, rank pipelines by CRTC index. A pipeline + * whose fixed output is live is not a candidate at all, so a hybrid is only + * ever ranked here when it is genuinely free. + */ +static unsigned int dcp_typec_route_score(struct apple_dcp_typec_route *route) +{ + struct apple_dcp *dcp = route->dcp; + unsigned int score; + + if (!dcp->crtc) + return UINT_MAX - 1; + + score = drm_crtc_index(&dcp->crtc->base); + /* + * Without dual-stream docks no stream is tied to the hybrid, so it goes + * to a Type-C port only when no Type-C-only pipeline is free, and an HDMI + * display plugged in later finds its pipeline idle. The port's encoder is + * narrowed to the routed pipeline before its connector reports connected, + * so the compositor pairs the connector with that pipeline. + */ + if (!dcp_typec_dual_stream() && dcp->fixed_phy) + score += 100; + + return score; +} + +static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, + struct mux_control *xbar) +{ + struct apple_dcp *dcp = route->dcp; + struct apple_connector *connector = + xbar != route->xbar && route->tunnel_dpin == 1 ? + route->port->secondary_connector : route->port->connector; + int ret; + + dcp_firmware_metadata_connection(dcp, READ_ONCE(dcp->typec_generation), false); + + /* + * The fixed output's HPD handler leaves disconnects to DCP, so the port + * can still be marked connected to a display that is gone. Release it + * (a no-op otherwise), or connecting the borrowed route returns early. + */ + ret = dcp_dptx_disconnect(dcp, 0); + if (ret) + return ret; + + if (dcp->fixed_route_selected) { + ret = mux_control_deselect(dcp->xbar); + dcp->fixed_route_selected = false; + if (ret) + return ret; + } + + /* + * Thunderbolt DP IN: the crossbar connection may only be brought up + * once DCP has configured the link and the tunnel pixel clock runs, + * see dcp_tunnel_crossbar_up(). Just remember the output here. + */ + if (xbar != route->xbar && route->mux_index) { + /* + * The T602X crossbar must point a DP IN at its pipeline before + * DCP probes AUX; other crossbars select it at link-up and + * report -EOPNOTSUPP here. + */ + ret = dcp_dpxbar_tunnel_select_source(xbar, route->mux_index); + if (ret == -EOPNOTSUPP) + ret = 0; + } else { + ret = xbar == route->xbar ? + mux_control_select(xbar, route->mux_index) : 0; + } + if (ret) { + if (dcp->xbar) { + int restore_ret; + + restore_ret = mux_control_select(dcp->xbar, + dcp->fixed_mux_index); + if (!restore_ret) + dcp->fixed_route_selected = true; + else + dev_err(dcp->dev, + "failed to restore fixed display route: %d\n", + restore_ret); + } + return ret; + } + + dcp->phy = route->phy; + dcp->dptx_phy = route->dptx_phy; + dcp->connector_type = DRM_MODE_CONNECTOR_USB; + WRITE_ONCE(dcp->ext_backlight, false); + if (connector) { + WRITE_ONCE(connector->dcp, to_platform_device(dcp->dev)); + dcp->typec_connector = connector; + dcp->connector = connector; + + /* + * Narrow the port to the pipeline now driving it. The encoder + * spans every pipeline that could, which is what lets userspace + * see the port as usable at all -- but only one of them is + * routed to the display, and userspace has no way to tell which. + * Offering it the choice makes it pair the port with a pipeline + * holding a different monitor's mode list, and the modeset is + * rejected with no way for it to recover. The hotplug that + * follows publishes the actual binding on dual-stream machines + * too. Clients must re-read encoder topology after a hotplug. + */ + if (connector->port_encoder && dcp->crtc) + connector->port_encoder->possible_crtcs = + drm_crtc_mask(&dcp->crtc->base); + } + dcp->active_typec_route = route; + scoped_guard(mutex, &dcp->tb_lock) { + route->deactivate_xbar_done = false; + route->deactivate_error = 0; + route->active_xbar = xbar; + route->tunnel = xbar != route->xbar; + route->xbar_up = !route->tunnel; + dcp->dptx_tunnel = route->tunnel; + /* crossbar controls are dpphy, dpin0, dpin1: same order as the DFP port */ + dcp->dptx_dfp_port = route->tunnel ? xbar - &route->xbar->chip->mux[0] : 0; + dcp->tb_clock_ok = false; + } + route->selected = true; + + dev_info(dcp->dev, "allocated Type-C DPTX PHY %u\n", route->dptx_phy); + return 0; +} + +static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) +{ + struct apple_dcp *dcp = route->dcp; + struct apple_connector *connector = dcp->typec_connector; + struct mux_control *active_xbar = route->active_xbar; + bool was_tunnel = route->tunnel; + int ret = 0; + + guard(mutex)(&dcp->external_lock); + if (dcp_is_external_h17p(dcp) && + (dcp->external_error || (dcp->external_power_attempted && !dcp->external_stopped))) + return dcp->external_error ?: -EAGAIN; + + dcp_firmware_metadata_connection(dcp, READ_ONCE(dcp->typec_generation), false); + + /* + * Under tb_lock so a DCP link (re)configuration can neither select the + * crossbar nor restart the tunnel pixel clock behind our back. + */ + scoped_guard(mutex, &dcp->tb_lock) { + if (route->deactivate_error) + return route->deactivate_error; + if (!route->deactivate_xbar_done) { + if (!route->tunnel || route->xbar_up) + ret = mux_control_deselect(route->active_xbar ?: route->xbar); + else + ret = dcp_dpxbar_preselect(route->active_xbar, MUX_IDLE_DISCONNECT); + /* Deselect always unlocks: never consume that selection twice. */ + route->deactivate_xbar_done = true; + route->xbar_up = false; + if (ret) { + /* Idle is unproven; retain the owner and the real error. */ + route->deactivate_error = ret; + return ret; + } + } + + if (route->tunnel && dcp->phy) { + /* the tunnel pixel clock must not outlive the tunnel */ + typeof(&apple_atc_dp_tunnel_rate) stop = + symbol_get(apple_atc_dp_tunnel_rate); + + if (stop) { + stop(dcp->phy, route->tunnel_dpin, 0); + symbol_put(apple_atc_dp_tunnel_rate); + } + } + route->active_xbar = NULL; + route->tunnel = false; + dcp->dptx_tunnel = false; + dcp->dptx_dfp_port = 0; + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + dcp->tb_clock_ok = false; + } + if (was_tunnel && route->mux_index) { + int sel = dcp_dpxbar_tunnel_select_source(active_xbar, -1); + + if (sel && sel != -EOPNOTSUPP) + dev_warn(dcp->dev, "DP tunnel source reset failed: %d\n", sel); + } + if (dcp_is_external_h17p(dcp)) + dpavservep_detach(dcp); + route->selected = false; + if (dcp->active_typec_route == route) + dcp->active_typec_route = NULL; + + if (connector && connector->dcp == to_platform_device(dcp->dev)) { + /* + * Until the port is activated again it has no pipeline behind + * it, and nothing can read modes or EDID from it. Report it + * disconnected for that window: leaving a connected connector + * whose ->dcp is NULL lets anything probing it in between -- + * a compositor starting up while the fabric is still settling + * -- see an output it cannot get a mode for, and give up on + * it. The new pipeline marks it connected again once the + * display has come back up on it. + */ + WRITE_ONCE(connector->connected, false); + /* hotplug work queued before this checks for it */ + WRITE_ONCE(connector->dcp, NULL); + /* no pipeline, so no backlight */ + schedule_work(&connector->bl_sync_wq); + + /* Unrouted: the port could go to any of its pipelines again. */ + if (connector->port_encoder) + connector->port_encoder->possible_crtcs = + connector->candidate_crtcs; + } + dcp->typec_connector = NULL; + dcp->connector = dcp->fixed_connector; + WRITE_ONCE(dcp->ext_backlight, false); + + if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_USB) { + dcp->connector_type = dcp->fixed_connector_type; + + /* + * Only hand the pipeline back to its fixed output if that output + * is live. Re-targeting the DPTX endpoint at the fixed PHY while + * nothing is attached there leaves DCP unable to train a link on a + * later Type-C target: it answers DEVICE_NOT_STARTED and every + * following DPTX call times out. Park on a Type-C PHY instead, + * for the same reason the USB-C-only case does below. + */ + if (dcp->hdmi_hpd && gpiod_get_value_cansleep(dcp->hdmi_hpd)) { + dcp->phy = dcp->fixed_phy; + dcp->dptx_phy = dcp->fixed_dptx_phy; + + if (dcp->xbar && !dcp->fixed_route_selected) { + ret = mux_control_select(dcp->xbar, + dcp->fixed_mux_index); + if (ret) + return ret; + dcp->fixed_route_selected = true; + } + } else { + dcp->phy = dcp->typec_routes[0].phy; + dcp->dptx_phy = dcp->typec_routes[0].dptx_phy; + } + } else { + /* Keep DPTX endpoint discovery working before a cable is attached. */ + dcp->phy = dcp->typec_routes[0].phy; + dcp->dptx_phy = dcp->typec_routes[0].dptx_phy; + dcp->connector_type = DRM_MODE_CONNECTOR_USB; + } + + return 0; +} + +static struct apple_dcp_typec_route * +dcp_typec_port_route(struct apple_dcp_typec_port *port, struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + + list_for_each_entry(route, &port->routes, port_link) + if (route->dcp == dcp) + return route; + + return NULL; +} + +/* + * The DRM device once it has bound, that is once every pipeline has its + * CRTC and every port its connectors; NULL before. + */ +static struct drm_device *dcp_typec_drm(void) +{ + struct apple_dcp_typec_port *port; + struct apple_dcp_typec_route *route; + struct drm_device *drm = NULL; + + list_for_each_entry(port, &dcp_typec_ports, link) { + if (!port->connector || !port->secondary_connector) + return NULL; + list_for_each_entry(route, &port->routes, port_link) { + if (!route->dcp->crtc) + return NULL; + drm = route->dcp->crtc->base.dev; + } + } + + return drm; +} + +/* + * Has a compositor (or boot splash) taken the display? It paired its + * connectors with CRTCs when it started and keeps that pairing, also + * across a VT switch, where it drops DRM master but keeps the device + * open; moving routes under it would hand connectors to pipelines driving + * other displays. So the routes are frozen while any open file has ever + * been master, and thaw when the last such file is closed. + * + * Taken under dcp_typec_fabric_lock, next to the routing it decides, so + * the order is the fabric lock, then filelist_mutex. Nothing nests them + * the other way: filelist_mutex only covers list edits and walks, and + * drm_release() drops it before drm_file_free() calls postclose, which is + * where a close takes the fabric lock. drm_open() makes a file master + * before adding it here, but its owner cannot have paired anything before + * open() returns, and it re-probes on the hotplugs the moves send. + */ +static bool dcp_typec_frozen(struct drm_device *drm) +{ + struct drm_file *file; + + guard(mutex)(&drm->filelist_mutex); + list_for_each_entry(file, &drm->filelist, lhead) { + /* set under master_mutex, and only ever from false to true */ + if (READ_ONCE(file->was_master)) + return true; + } + + return false; +} + +/* Are the routes kept in compositor pairing order right now? */ +static bool dcp_typec_keep_order(void) +{ + struct drm_device *drm; + + if (!dcp_typec_dual_stream()) + return false; + drm = dcp_typec_drm(); + + return drm && READ_ONCE(drm->registered) && !dcp_typec_frozen(drm); +} + +void dcp_hdmi_hold(struct apple_dcp *dcp) +{ + WRITE_ONCE(dcp->hdmi_hold_until, + jiffies + msecs_to_jiffies(DCP_HDMI_HOLD_MS)); +} + +static bool dcp_hdmi_held(struct apple_dcp *dcp) +{ + unsigned long until = READ_ONCE(dcp->hdmi_hold_until); + + return dcp->hdmi_hpd && until && time_before(jiffies, until); +} + +/* A Thunderbolt tunnel holds @dcp's pipeline: it never moves. */ +static bool dcp_typec_tunnel_held(struct apple_dcp *dcp) +{ + return dcp->active_typec_route && dcp->active_typec_route->tunnel; +} + +/* + * Give @dcp's pipeline to the first stream, in connector order, that + * wants one, has none yet and has it in its possible_crtcs. A tunnel + * stream wants one once it is set up, or as @arriving/@dpin. A direct + * DP-alt stream wants one once its sink asserts HPD: only then can its + * connector read connected, and a compositor pairs only those. + */ +static void dcp_typec_plan_pipeline(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct apple_dcp_typec_port *arriving, + unsigned int dpin) +{ + struct apple_dcp_typec_port *port; + bool secondary = false; + + /* primary connectors in port order, then the DPIN1 ones */ + do { + list_for_each_entry(port, &dcp_typec_ports, link) { + struct apple_dcp_typec_route **slot = secondary ? + &port->secondary_target : &port->target; + struct apple_connector *connector = secondary ? + port->secondary_connector : port->connector; + struct apple_dcp_typec_route *route = + dcp_typec_port_route(port, dcp); + bool wants; + + if (secondary) + wants = port->secondary_owner || + (port == arriving && dpin == 1); + else + wants = (port->owner && port->owner->tunnel) || + (port->dp_wanted && port->dp_hpd && + !port->plan_dark) || + (port == arriving && dpin == 0); + + if (!wants || *slot || !route || + !(connector->candidate_crtcs & drm_crtc_mask(crtc))) + continue; + + *slot = route; + return; + } + secondary = !secondary; + } while (secondary); +} + +/* + * The pairing a compositor starting now would make, wherever the streams + * sit at the moment: pipelines in CRTC index order, each to the first + * stream that wants it. A pipeline whose fixed output is live is left to + * that output, unless a tunnel holds it: the output cannot have it back + * then, and its connector reads disconnected. + * + * Tunnels are planned in connector order like any other stream, not + * seated on the pipelines they hold: the compositor pairs their connectors + * by order too, and seating them would plan the direct streams around + * pairings it never makes. A direct stream planned onto a pipeline a + * tunnel holds cannot have it and stays dark, so it is left out and the + * pass re-run, or every stream after it would be planned one pipeline + * off. Streams only ever drop out, so that settles within one pass per + * port. It also brings a tunnel's plan onto the pipeline it holds + * wherever direct streams ahead of it were in the way. A tunnel still + * planned elsewhere cannot be brought there by any direct stream dropping + * out, and the compositor pairs its connector wrongly whatever they do. + */ +static void dcp_typec_plan(struct drm_device *drm, + struct apple_dcp_typec_port *arriving, unsigned int dpin) +{ + struct apple_dcp_typec_port *port; + struct drm_crtc *crtc; + bool again; + + list_for_each_entry(port, &dcp_typec_ports, link) + port->plan_dark = false; + + do { + list_for_each_entry(port, &dcp_typec_ports, link) { + port->target = NULL; + port->secondary_target = NULL; + } + + /* CRTCs are listed in index order */ + drm_for_each_crtc(crtc, drm) { + struct apple_dcp *dcp = + platform_get_drvdata(to_apple_crtc(crtc)->dcp); + + if (dcp->nr_typec_routes && + (dcp_typec_tunnel_held(dcp) || + !dcp_typec_route_fixed_output_busy(&dcp->typec_routes[0]))) + dcp_typec_plan_pipeline(dcp, crtc, arriving, dpin); + } + + again = false; + list_for_each_entry(port, &dcp_typec_ports, link) { + if (port == arriving || !port->target || + (port->owner && port->owner->tunnel) || + !dcp_typec_tunnel_held(port->target->dcp)) + continue; + port->plan_dark = true; + again = true; + } + } while (again); +} + +/* Replay HPD to the pipeline a direct DP-alt port has just been given. */ +static void dcp_typec_port_attach(struct apple_dcp_typec_port *port) +{ + struct apple_dcp *dcp = port->owner->dcp; + + port->hpd = port->dp_hpd; + if (!port->hpd) + return; + + WRITE_ONCE(dcp->typec_cable_connected, true); + if (dcp->typec_connector) + dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); +} + +static struct apple_dcp_typec_route * +dcp_typec_lowest_free(struct apple_dcp_typec_port *port) +{ + struct apple_dcp_typec_route *candidate, *best = NULL; + unsigned int best_score = UINT_MAX; + + list_for_each_entry(candidate, &port->routes, port_link) { + unsigned int score; + + if (!dcp_typec_route_available(candidate)) + continue; + /* + * Prefer the lowest free CRTC index for a stable allocation order. + */ + score = dcp_typec_route_score(candidate); + if (score < best_score) { + best = candidate; + best_score = score; + } + } + + return best; +} + +/* + * The pipeline a port without one takes: the one it last had if that is + * free, as a compositor keeps a reconnected connector's CRTC, otherwise the + * best-ranked free one. Without dual-stream docks a free Type-C-only + * pipeline comes first even when the port last had the hybrid, which an HDMI + * display needs. + */ +static struct apple_dcp_typec_route * +dcp_typec_free_route(struct apple_dcp_typec_port *port) +{ + struct apple_dcp_typec_route *last = port->preferred_route; + struct apple_dcp_typec_route *best = dcp_typec_lowest_free(port); + + if (!last || !dcp_typec_route_available(last)) + return best; + if (!dcp_typec_dual_stream() && best && + dcp_typec_route_score(best) < dcp_typec_route_score(last)) + return best; + return last; +} + +/* + * Route the direct DP-alt ports left waiting for a pipeline. Their DP + * state is not reported again while it stays the same, so nothing else + * would. Each goes back to the pipeline it last had if that is free, as + * a compositor keeps a reconnected connector's CRTC, and otherwise takes + * the lowest free one, the CRTC a compositor gives a new connector. + */ +static void dcp_typec_route_waiting(void) +{ + struct apple_dcp_typec_port *port; + struct apple_dcp_typec_route *route; + + list_for_each_entry(port, &dcp_typec_ports, link) { + if (port->owner || !port->dp_wanted || !port->dp_hpd) + continue; + + route = dcp_typec_free_route(port); + if (!route || dcp_typec_route_activate(route, route->xbar)) + continue; + port->owner = route; + + dcp_typec_port_attach(port); + } +} + +/* + * Prefer connector order when placing Type-C streams before a DRM master + * starts. This is an allocation policy, not a promise about the CRTC that + * userspace must choose: each connected encoder advertises only its actual + * route, and atomic encoder selection checks that ownership. Route changes + * publish the new mask through hotplug; the plan uses candidate_crtcs so + * narrowing the advertised mask does not remove hardware routing choices. + * + * Only direct DP-alt routes move: each goes to exactly its planned + * pipeline. A Thunderbolt tunnel never moves once set up, so where one + * sits on a pipeline planned for a direct stream, or the plan has nothing + * for it, that stream stays unrouted and its connector disconnected. + * Placing it on some other pipeline would have the compositor cross both + * displays; dark is the better failure. @arriving/@dpin is a tunnel + * stream asking for a pipeline, planned like any other; its route is + * returned for apple_dcp_tb_dp_tunnel() to set up, or NULL if the plan + * has none for it or another tunnel holds that one. + * + * A move is an unplug and replug. Every route that moves is taken down + * first, so that two never share a pipeline midway, then each goes up on + * its new pipeline with its HPD replayed; the hotplugs this sends make + * fbdev and userspace re-probe. + */ +static struct apple_dcp_typec_route * +dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, + unsigned int dpin) +{ + struct drm_device *drm = dcp_typec_drm(); + struct apple_dcp_typec_route *planned = NULL; + struct apple_dcp_typec_port *port; + + lockdep_assert_held(&dcp_typec_fabric_lock); + + dcp_typec_plan(drm, arriving, dpin); + if (arriving) { + planned = dpin ? arriving->secondary_target : arriving->target; + /* it will be refused: plan as if it had not asked */ + if (planned && dcp_typec_tunnel_held(planned->dcp)) { + planned = NULL; + dcp_typec_plan(drm, NULL, 0); + } + } + + list_for_each_entry(port, &dcp_typec_ports, link) { + struct apple_dcp_typec_route *owner = port->owner; + struct apple_dcp *dcp; + + if (port->safe_pending) + continue; + if (!owner || owner->tunnel || owner == port->target) + continue; + + dcp = owner->dcp; + dev_info(dcp->dev, "re-routing %pOF from %s to %s for Type-C connector order\n", + port->connector_np, dev_name(dcp->dev), + port->target ? dev_name(port->target->dcp->dev) : "none"); + if (port->hpd || dcp->typec_cable_connected || + (dcp->typec_connector && dcp->typec_connector->connected)) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + port->hpd = false; + if (dcp_typec_route_deactivate(owner) && owner->selected) { + /* still routed: leave the display where it is */ + dcp_typec_port_attach(port); + continue; + } + port->owner = NULL; + + /* as after a DP exit, hand the hybrid back to a live HDMI */ + if (dcp->hdmi_hpd && dcp->active && + gpiod_get_value_cansleep(dcp->hdmi_hpd)) + dcp_dptx_connect(dcp, 0); + } + + list_for_each_entry(port, &dcp_typec_ports, link) { + struct apple_dcp_typec_route *route = port->target; + + if (port->owner || !port->dp_wanted || !port->dp_hpd || + port == arriving) + continue; + + if (!route || !dcp_typec_route_available(route) || + dcp_typec_route_activate(route, route->xbar)) { + /* + * Dark for now. Its unchanged DP state is not reported + * again, so it is placed when the plan next runs, or + * once a compositor owns the display, when a pipeline + * is freed (dcp_typec_route_waiting()). + */ + port->applied_valid = false; + continue; + } + port->owner = route; + port->preferred_route = route; + + dcp_typec_port_attach(port); + } + + return planned; +} + +/* + * A display is plugged into the HDMI port while its pipeline, the hybrid, + * drives a Type-C port. Without dual-stream docks nothing moves for it: a + * compositor keeps its connector-to-CRTC pairing and could miss a moved + * display's brief unplug, and a Thunderbolt tunnel never moves. The HDMI + * display waits and is handed the hybrid when the Type-C display lets it go + * (see dcp_typec_route_deactivate()). + */ +static void dcp_typec_hdmi_waits(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *owner = READ_ONCE(dcp->active_typec_route); + + lockdep_assert_held(&dcp_typec_fabric_lock); + + if (owner) + dev_info(dcp->dev, "HDMI display waits: its pipeline drives the %s on %pOF\n", + owner->tunnel ? "Thunderbolt display" : "display", + owner->port->connector_np); +} + +/* + * A route has let its pipeline go. Until a compositor owns the display of a + * dual-stream machine the plan places everything anew; otherwise the + * pipeline goes to a port left waiting for one. The caller has already + * handed a hybrid back to a live HDMI display. + */ +static void dcp_typec_pipeline_freed(void) +{ + if (dcp_typec_keep_order()) + dcp_typec_rebalance_locked(NULL, 0); + else + dcp_typec_route_waiting(); +} + +/* + * Re-run the pairing pass from outside the fabric: once DRM is registered, + * as ports routed before that could not follow it, and when the last + * compositor or boot splash has closed the device, as the next one pairs + * the connectors from scratch. + */ +void dcp_typec_reorder(void) +{ + if (!dcp_typec_dual_stream()) + return; + + guard(mutex)(&dcp_typec_fabric_lock); + + if (dcp_typec_keep_order()) + dcp_typec_rebalance_locked(NULL, 0); +} + +static int dcp_typec_route_prepare(struct typec_mux_dev *mux, + struct typec_mux_state *state) +{ + struct apple_dcp_typec_route *route = typec_mux_get_drvdata(mux); + struct apple_dcp_typec_port *port = route->port; + struct apple_dcp *dcp; + int ret; + + guard(mutex)(&dcp_typec_fabric_lock); + if (state->mode != TYPEC_STATE_SAFE) + return port->safe_pending ? -EAGAIN : 0; + if (!port->owner || port->owner->tunnel) + return 0; + dcp = port->owner->dcp; + /* Stop new admission, but retain the owner until the set phase succeeds. */ + if (!port->safe_pending) + port->safe_finish_ready = false; + port->safe_pending = true; + port->applied_valid = false; + port->dp_wanted = false; + port->dp_hpd = false; + /* HPD absence alone is not proof of a completed firmware shutdown. */ + if (dcp_is_external_h17p(dcp)) { + scoped_guard(mutex, &dcp->external_lock) { + if (dcp->external_error) + return dcp->external_error; + if (!dcp->external_cable && dcp->external_stopped && + !READ_ONCE(dcp->dptxport[0].connected)) { + port->hpd = false; + return 0; + } + } + } + ret = dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + if (ret) + return ret; + port->hpd = false; + return 0; +} + +static int dcp_typec_route_finish(struct typec_mux_dev *mux, + struct typec_mux_state *state) +{ + struct apple_dcp_typec_port *port = + ((struct apple_dcp_typec_route *)typec_mux_get_drvdata(mux))->port; + int ret; + + guard(mutex)(&dcp_typec_fabric_lock); + if (state->mode != TYPEC_STATE_SAFE || !port->safe_pending || + port->safe_finish_ready) + return 0; + ret = port->owner ? dcp_typec_route_deactivate(port->owner) : 0; + if (ret) + return ret; + port->safe_finish_ready = true; + return 0; +} + +static void dcp_typec_route_commit(struct typec_mux_dev *mux, + struct typec_mux_state *state) +{ + struct apple_dcp_typec_port *port = + ((struct apple_dcp_typec_route *)typec_mux_get_drvdata(mux))->port; + + guard(mutex)(&dcp_typec_fabric_lock); + if (state->mode != TYPEC_STATE_SAFE || !port->safe_pending) + return; + if (port->owner) + port->preferred_route = port->owner; + port->owner = NULL; + port->safe_pending = false; + port->safe_finish_ready = false; + port->applied_alt = state->alt; + port->applied_mode = state->mode; + port->applied_status = 0; + port->applied_conf = 0; + port->applied_valid = true; + dcp_typec_pipeline_freed(); +} + +static int dcp_typec_route_set(struct typec_mux_dev *mux, + struct typec_mux_state *state) +{ + struct apple_dcp_typec_route *route = typec_mux_get_drvdata(mux); + struct apple_dcp_typec_port *port = route->port; + struct apple_dcp_typec_route *best = NULL; + bool is_dp = dcp_typec_route_is_dp(state); + struct typec_displayport_data *dp_data = is_dp ? state->data : NULL; + u32 dp_status = dp_data ? dp_data->status : 0; + u32 dp_conf = dp_data ? dp_data->conf : 0; + bool hpd, was_counted; + int ret = 0; + + guard(mutex)(&dcp_typec_fabric_lock); + + if (state->mode == TYPEC_STATE_SAFE && port->safe_pending) + return 0; + /* The previous SAFE transition has not finished; retry. */ + if (port->safe_pending) + return -EAGAIN; + + /* + * Every candidate route for this port is notified with the same state, + * so only the first to arrive does the work; the others return early. + * + * Deliberately not a nominated coordinator: fwnode_typec_mux_get() + * caps the providers one connector may have and drops the remainder + * without a word, so a nominated route might never be called at all -- + * and the port would then never be routed. + */ + if (port->applied_valid && port->applied_alt == state->alt && + port->applied_mode == state->mode && + port->applied_status == dp_status && port->applied_conf == dp_conf) + return 0; + + port->applied_alt = state->alt; + port->applied_mode = state->mode; + port->applied_status = dp_status; + port->applied_conf = dp_conf; + /* Failed route acquisition must remain retryable on the next update. */ + port->applied_valid = false; + + /* did the pairing pass count this port's direct stream so far? */ + was_counted = port->dp_wanted && port->dp_hpd; + + if (!is_dp) { + port->dp_wanted = false; + port->dp_hpd = false; + + /* a Thunderbolt/USB4 DP tunnel is torn down by its own path */ + if (port->owner && port->owner->tunnel) { + port->applied_valid = true; + return 0; + } + if (port->owner) { + struct apple_dcp *dcp = port->owner->dcp; + + port->preferred_route = port->owner; + if (port->hpd || dcp->typec_cable_connected || + (dcp->typec_connector && + dcp->typec_connector->connected)) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + port->hpd = false; + ret = dcp_typec_route_deactivate(port->owner); + if (ret) + return ret; + port->owner = NULL; + + if (dcp->hdmi_hpd && dcp->active && + gpiod_get_value_cansleep(dcp->hdmi_hpd)) + dcp_dptx_connect(dcp, 0); + dcp_typec_pipeline_freed(); + } else if (was_counted && dcp_typec_keep_order()) { + /* a stream the plan left dark is gone: plan the rest */ + dcp_typec_rebalance_locked(NULL, 0); + } + + port->safe_pending = false; + /* Data-only USB4 changes do not invalidate other display routes. */ + port->applied_valid = true; + return 0; + } + + /* + * A Thunderbolt DP tunnel still owns the port (its teardown is on the + * way): don't act on or remember this state; the tunnel teardown drops + * what was recorded so the next update is applied. + */ + if (port->owner && port->owner->tunnel) { + port->applied_valid = false; + return 0; + } + + hpd = dp_data && (dp_data->status & DP_STATUS_HPD_STATE); + port->dp_wanted = true; + port->dp_hpd = hpd; + + /* + * Until a compositor owns the display, a direct DP-alt stream has a + * pipeline only while its sink asserts HPD, and the pairing pass + * places it: HPD coming or going is the stream connecting or + * disconnecting as far as a compositor can tell. Otherwise the port + * takes the lowest free pipeline on DP entry, as a compositor does. + */ + if (dcp_typec_keep_order()) { + struct apple_dcp_typec_route *owner = port->owner; + + /* it connects or disconnects, or is not routed as it should be */ + if (hpd != was_counted || !owner != !hpd) + dcp_typec_rebalance_locked(NULL, 0); + if (!port->owner) { + if (hpd) + return -ENOSPC; + port->applied_valid = true; + return 0; + } + /* just attached, its HPD replayed: nothing left to apply */ + if (port->owner != owner) { + port->applied_valid = true; + return 0; + } + } + + if (!port->owner) { + best = dcp_typec_free_route(port); + /* Every pipeline is driving another port's display. */ + if (!best) + return -ENOSPC; + + ret = dcp_typec_route_activate(best, best->xbar); + if (ret) + return ret; + port->owner = best; + + } + + + if (!hpd && port->hpd) { + dcp_dptx_disconnect_oob(to_platform_device(port->owner->dcp->dev), 0); + } else if (hpd && !port->hpd) { + struct apple_dcp *dcp = port->owner->dcp; + + WRITE_ONCE(dcp->typec_cable_connected, true); + if (dcp->typec_connector) + dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); + } else if (hpd && dp_data && (dp_data->status & DP_STATUS_IRQ_HPD)) { + struct apple_dcp *dcp = port->owner->dcp; + + if (dcp->typec_connector) + dcp_retrain_oob(dcp->typec_connector); + } + port->hpd = hpd; + port->applied_valid = true; + + return 0; +} + +/* Limited to the T6030 14.7 pipeline; other T602X tunnels keep their order. */ +static bool dcp_t602x_tunnel_route(struct apple_dcp_typec_route *route) +{ + return route && route->tunnel && route->active_xbar && + route->dcp->fw_compat == DCP_FIRMWARE_V_14_7 && + of_device_is_compatible(route->active_xbar->chip->dev.parent->of_node, + "apple,t6020-display-crossbar"); +} + +/* + * Crossbar connection up/down, looked up at runtime so appledrm does not + * require the crossbar driver to be built. + */ +static int dcp_dpxbar_link(struct mux_control *mux, bool up) +{ + typeof(&apple_dpxbar_link_up) fn; + int ret; + + fn = up ? symbol_get(apple_dpxbar_link_up) : + symbol_get(apple_dpxbar_link_down); + if (!fn) + return -ENOENT; + ret = fn(mux); + if (up) + symbol_put(apple_dpxbar_link_up); + else + symbol_put(apple_dpxbar_link_down); + return ret; +} + +static int dcp_dpxbar_preselect(struct mux_control *mux, int state) +{ + typeof(&apple_dpxbar_preselect) fn = symbol_get(apple_dpxbar_preselect); + int ret; + + if (!fn) + return -ENOENT; + ret = fn(mux, state); + symbol_put(apple_dpxbar_preselect); + return ret; +} + +/* + * Thunderbolt DP IN, before DCP hears about the display: point the DP IN + * output at this pipeline and wake the ATC's DP clock path. The crossbar + * connection itself still only comes up in dcp_tunnel_crossbar_up(), but an + * output left at its idle source (dispext0) only suits the pipeline that is + * dispext0: any other one activates the link, gets no answer from the sink + * and gives up with DEVICE_NOT_RESPONDING before it ever sets a link rate. + */ +static void dcp_tunnel_prepare(struct apple_dcp_typec_route *route, + struct mux_control *xbar) +{ + typeof(&apple_atc_dp_tunnel_open) open; + struct apple_dcp *dcp = route->dcp; + int ret; + + ret = dcp_dpxbar_preselect(xbar, route->mux_index); + if (ret && ret != -EOPNOTSUPP) + dev_warn(dcp->dev, "DP tunnel crossbar preselect failed: %d\n", ret); + + open = symbol_get(apple_atc_dp_tunnel_open); + if (!open) + return; + ret = open(dcp->phy); + symbol_put(apple_atc_dp_tunnel_open); + if (ret && ret != -EOPNOTSUPP) + dev_warn(dcp->dev, "DP tunnel PHY open failed: %d\n", ret); +} + +/* DP IN adapter handshake through the thunderbolt glue; tb_lock held */ +static int dcp_tunnel_dpin_locked(struct apple_dcp *dcp, bool active) +{ + lockdep_assert_held(&dcp->tb_lock); + if (!dcp->tb_dpin_set_active) + return -ENODEV; + return dcp->tb_dpin_set_active(dcp->tb_dpin_ctx, active); +} + +/* + * Thunderbolt DP IN, from DCP's DidChangeLinkConfiguration once a link rate + * is set: bring the crossbar connection up (FIFO/PCLK/ATC enables) now that + * the tunnel pixel clock runs, and re-assert the DP IN adapter's + * DPTX_INACTIVE=0 afterwards. The first time this is the mux selection; after + * a re-link only the clocks are brought back (the mux selection and the ATC + * output enable are kept, see dcp_tunnel_crossbar_down()). + * Runs inside a DCP apcall: only tb_lock, which nobody holds across a DCP call. + */ +int dcp_tunnel_crossbar_up(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp->tb_lock); + route = dcp->active_typec_route; + if (!route || !route->tunnel || !route->active_xbar || + route->deactivate_xbar_done) + return -ENODEV; + if (!dcp->tb_clock_ok) { + dev_warn(dcp->dev, "no DP tunnel pixel clock, crossbar left down\n"); + return -EIO; + } + if (!route->xbar_up) { + /* never block a DCP call on the mux semaphore */ + ret = mux_control_try_select(route->active_xbar, route->mux_index); + if (!ret) + route->xbar_up = true; + /* T602X DP IN selection reserves and routes the mux only. */ + if (!ret && dcp_t602x_tunnel_route(route)) + ret = dcp_dpxbar_link(route->active_xbar, true); + } else { + ret = dcp_dpxbar_link(route->active_xbar, true); + } + if (ret) { + dev_warn(dcp->dev, "DP tunnel crossbar up failed: %d\n", ret); + return ret; + } + return dcp_tunnel_dpin_locked(dcp, true); +} + +/* + * Thunderbolt DP IN, from DCP's WillChangeLinkConfiguration on an established + * link: DP IN inactive, crossbar clocks down (mux selection and ATC output + * enable kept). DP IN goes active again in dcp_tunnel_crossbar_up(). + */ +int dcp_tunnel_crossbar_down(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + + guard(mutex)(&dcp->tb_lock); + route = dcp->active_typec_route; + if (!route || !route->tunnel || !route->xbar_up) + return 0; + dcp_tunnel_dpin_locked(dcp, false); + return dcp_dpxbar_link(route->active_xbar, false); +} + +/* + * Thunderbolt DP IN, from DCP's SetLinkRate: start (rate != 0) or stop the + * tunnel pixel clock. A stopped clock also takes the crossbar connection down. + */ +int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate) +{ + typeof(&apple_atc_dp_tunnel_rate) fn; + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp->tb_lock); + if (!dcp->dptx_tunnel) + return -ENODEV; + route = dcp->active_typec_route; + /* The initial T6030 clock port supports core0 -> DP IN0 (PCLK1). + * Other routes require explicit PCLK slot selection and accounting. + */ + if (link_rate && of_machine_is_compatible("apple,t6030") && + (!route || route->mux_index != 0 || dcp->dptx_dfp_port != 1)) + return -EOPNOTSUPP; + fn = symbol_get(apple_atc_dp_tunnel_rate); + if (!fn) { + dev_err(dcp->dev, "phy-apple-atc not loaded, no DP tunnel clock\n"); + return -ENOENT; + } + route = dcp->active_typec_route; + if (!route) { + symbol_put(apple_atc_dp_tunnel_rate); + return -ENODEV; + } + if (!link_rate && route->xbar_up) + dcp_dpxbar_link(route->active_xbar, false); + ret = fn(phy, route->tunnel_dpin, link_rate); + symbol_put(apple_atc_dp_tunnel_rate); + dcp->tb_clock_ok = !ret && link_rate; + if (ret) + dev_warn(dcp->dev, "DP tunnel pixel clock (rate 0x%x) failed: %d\n", + link_rate, ret); + return ret; +} + +/* Thunderbolt DP IN: DCP Activate/Deactivate */ +int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active) +{ + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp->tb_lock); + if (!dcp->dptx_tunnel) + return 0; + /* Route the upstream engine before firmware begins AUX negotiation. + * T602X DP IN .set does not enable clocks; DidChange does that later. + */ + route = dcp->active_typec_route; + if (active && dcp_t602x_tunnel_route(route)) { + if (!route->xbar_up) { + ret = mux_control_try_select(route->active_xbar, route->mux_index); + if (ret) + return ret; + route->xbar_up = true; + } + } + return dcp_tunnel_dpin_locked(dcp, active); +} + +/* + * Thunderbolt DP tunnels: the host router's DP IN adapters sit behind the + * crossbar's dpin0/dpin1 outputs of the port's ATC. When the Thunderbolt + * connection manager has set up a tunnel from one of them, route a free + * display pipeline there and tell DCP a display is attached, so it trains + * the link (and completes DPRX) through the tunnel. + */ +static bool dcp_tb_services_ready(struct apple_dcp *dcp) +{ + if (!dcp->external || dcp->fw_compat != DCP_FIRMWARE_V_14_7) + return true; + return smp_load_acquire(&dcp->dptxport[0].enabled) && ibootep_is_ready(dcp); +} + +int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, + bool active, int (*set_active)(void *ctx, bool active), + void *ctx) +{ + struct apple_dcp_typec_port *port = NULL, *pos; + struct apple_dcp_typec_route *candidate, *best = NULL, *planned = NULL; + struct apple_dcp_typec_route **slot; + unsigned int best_score = UINT_MAX; + bool waiting_for_external = false; + struct mux_control *ctl; + struct apple_dcp *dcp; + bool ordered; + int ret; + + if (!connector_np || dpin > 1) + return -EINVAL; + + guard(mutex)(&dcp_typec_fabric_lock); + + list_for_each_entry(pos, &dcp_typec_ports, link) { + if (pos->connector_np == connector_np) { + port = pos; + break; + } + } + if (!port) + return -ENODEV; + slot = dpin ? &port->secondary_owner : &port->owner; + + if (!active) { + if (!*slot || !(*slot)->tunnel || + (*slot)->tunnel_dpin != dpin) + return 0; + dcp = (*slot)->dcp; + if (port->hpd || dcp->typec_cable_connected) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + ret = dcp_typec_route_deactivate(*slot); + if (ret) { + /* the caller's context is going away regardless */ + scoped_guard(mutex, &dcp->tb_lock) { + dcp->dptx_tunnel = false; + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + } + return ret; + } + *slot = NULL; + port->hpd = !!(port->owner || port->secondary_owner); + /* re-apply the next Type-C mux state in full */ + port->applied_valid = false; + if (dcp->hdmi_hpd && dcp->active && + gpiod_get_value_cansleep(dcp->hdmi_hpd)) + dcp_dptx_connect(dcp, 0); + dcp_typec_pipeline_freed(); + return 0; + } + + if (*slot) { + if ((*slot)->tunnel && (*slot)->tunnel_dpin == dpin) + return 0; + dev_warn((*slot)->dcp->dev, + "port already routed, not taking DP tunnel dpin%u\n", dpin); + return -EBUSY; + } + if (port->owner && !port->owner->tunnel) + return -EBUSY; + + /* + * Until a compositor owns the display, the pairing pass decides where + * the stream goes, moving direct DP-alt routes out of its way, and + * whether it gets a pipeline at all. + */ + ordered = dcp_typec_keep_order(); + if (ordered) + planned = dcp_typec_rebalance_locked(port, dpin); + + list_for_each_entry(candidate, &port->routes, port_link) { + unsigned int score; + + if (ordered && candidate != planned) + continue; + if (!dcp_typec_route_available(candidate)) + continue; + if (!dcp_typec_dual_stream() && dcp_hdmi_held(candidate->dcp)) + continue; + if (!dcp_typec_route_fits(candidate, dpin ? + port->secondary_connector : + port->connector)) + continue; + /* A boot-present tunnel may precede the explicit firmware start. + * Do not claim its route or spend firmware reconnect attempts + * until both command services have been published. Acquire + * pairs with RemotePort publication; iBoot checks its cookie + * with acquire ordering and refuses a stopping service. + */ + /* Retained scanout after link loss must never be reactivated. */ + if (dcpext_scanout_terminal(candidate->dcp)) { + ret = -ESHUTDOWN; + goto err_reorder; + } + if (!dcp_tb_services_ready(candidate->dcp)) { + waiting_for_external = true; + continue; + } + score = dcp_typec_route_score(candidate); + /* + * DPIN0 prefers the hybrid dcpext0, which completes tunneled + * link training. On dual-stream machines DPIN1 is confined to + * dcpext1 by its connector's possible_crtcs, so both pipelines + * drive independent streams through one dock. + */ + if (dcp_typec_dual_stream() && dpin == 0 && + !candidate->dcp->fixed_phy && score < UINT_MAX - 100) + score += 100; + if (score < best_score) { + best = candidate; + best_score = score; + } + } + if (!best) { + ret = waiting_for_external ? -EAGAIN : -EBUSY; + goto err_reorder; + } + + /* The route's crossbar control is dpphy (0); dpin0/dpin1 are 1/2. */ + if (best->xbar != &best->xbar->chip->mux[0] || + best->xbar->chip->controllers < 3) { + ret = -EOPNOTSUPP; + goto err_reorder; + } + ctl = &best->xbar->chip->mux[1 + dpin]; + + dcp = best->dcp; + scoped_guard(mutex, &dcp->tb_lock) { + dcp->tb_dpin_set_active = set_active; + dcp->tb_dpin_ctx = ctx; + } + best->tunnel_dpin = dpin; + ret = dcp_typec_route_activate(best, ctl); + if (ret) { + scoped_guard(mutex, &dcp->tb_lock) { + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + } + goto err_reorder; + } + *slot = best; + /* the port is in USB4 mode, not DP-alt */ + port->dp_wanted = false; + dcp_tunnel_prepare(best, ctl); + + dev_info(dcp->dev, "display routed to Thunderbolt DP tunnel dpin%u\n", dpin); + + if (dcp->fw_compat == DCP_FIRMWARE_V_14_7 && dcp->dptxep && + !dcp->dptxport[0].enabled) + dev_warn(dcp->dev, "DPTX port not announced, not opening the controller\n"); + + /* + * The DP IN adapter may only be woken (DPTX_INACTIVE=0) while DCP + * drives the DPTX, i.e. from DCP's Activate call; waking it earlier + * hangs the machine. dptxep calls set_active back from Activate and + * Deactivate (set above, before the route became a tunnel). + */ + if (!dcp->typec_connector && !dcp->external) + dev_warn(dcp->dev, "no Type-C connector for the DP tunnel\n"); + WRITE_ONCE(dcp->typec_cable_connected, true); + port->hpd = true; + /* External firmware link bring-up precedes its DRM connector. Run it + * after returning to the tunnel manager, outside the fabric lock. + */ + if (dcp->external) { + dcp->typec_reconnect_tries = 0; + mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); + } else if (dcp->typec_connector) + dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); + + return 0; + +err_reorder: + /* the pass kept a pipeline for this stream: give it to the others */ + if (planned) + dcp_typec_rebalance_locked(NULL, 0); + return ret; +} +EXPORT_SYMBOL_GPL(apple_dcp_tb_dp_tunnel); + +static struct apple_dcp_typec_port * +dcp_typec_port_get(struct device_node *connector_np) +{ + struct apple_dcp_typec_port *port, *pos; + + lockdep_assert_held(&dcp_typec_fabric_lock); + + list_for_each_entry(port, &dcp_typec_ports, link) { + if (port->connector_np == connector_np) { + of_node_put(connector_np); + return port; + } + } + + port = kzalloc_obj(*port); + if (!port) { + of_node_put(connector_np); + return NULL; + } + + INIT_LIST_HEAD(&port->routes); + port->connector_np = connector_np; + + /* + * Insert in device-tree order rather than DCP probe order. The list + * index becomes the DRM connector index, and userspace keys its + * per-monitor configuration (scale, rotation, layout) on the connector + * name -- so it has to mean the same physical port on every boot. + */ + list_for_each_entry(pos, &dcp_typec_ports, link) + if (strcmp(of_node_full_name(connector_np), + of_node_full_name(pos->connector_np)) < 0) + break; + list_add_tail(&port->link, &pos->link); + return port; +} + +static struct apple_dcp_typec_port *dcp_typec_port_by_index(unsigned int idx) +{ + struct apple_dcp_typec_port *port; + unsigned int i = 0; + + lockdep_assert_held(&dcp_typec_fabric_lock); + + list_for_each_entry(port, &dcp_typec_ports, link) + if (i++ == idx) + return port; + + return NULL; +} + +unsigned int dcp_typec_nr_ports(void) +{ + struct apple_dcp_typec_port *port; + unsigned int n = 0; + + guard(mutex)(&dcp_typec_fabric_lock); + + list_for_each_entry(port, &dcp_typec_ports, link) + n++; + + return n; +} + +struct device_node *dcp_typec_port_of_node(unsigned int idx) +{ + struct apple_dcp_typec_port *port; + + guard(mutex)(&dcp_typec_fabric_lock); + + port = dcp_typec_port_by_index(idx); + + return port ? port->connector_np : NULL; +} + +bool dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev) +{ + struct apple_dcp_typec_port *port; + struct apple_dcp_typec_route *route; + + guard(mutex)(&dcp_typec_fabric_lock); + + port = dcp_typec_port_by_index(idx); + if (!port) + return false; + + list_for_each_entry(route, &port->routes, port_link) + if (route->dcp->dev == &pdev->dev) + return true; + + return false; +} + +void dcp_typec_port_set_connector(unsigned int idx, bool secondary, + struct apple_connector *connector) +{ + struct apple_dcp_typec_port *port; + struct apple_dcp_typec_route *owner; + + guard(mutex)(&dcp_typec_fabric_lock); + + port = dcp_typec_port_by_index(idx); + if (!port) + return; + + if (secondary) { + port->secondary_connector = connector; + owner = port->secondary_owner; + } else { + port->connector = connector; + owner = port->owner; + } + + /* + * The port may already have been routed, either before DRM bound or + * while these connectors were being created. Adopt that owner now, + * otherwise its display would be reported on the pipeline's fixed + * connector instead of the port it is actually plugged into. + */ + if (owner) { + struct apple_dcp *dcp = owner->dcp; + + if (dcp->crtc && connector->port_encoder) + connector->port_encoder->possible_crtcs = + drm_crtc_mask(&dcp->crtc->base); + + connector->dcp = to_platform_device(dcp->dev); + dcp->typec_connector = connector; + dcp->connector = connector; + dcp->connector_type = DRM_MODE_CONNECTOR_USB; + } +} + +static void dcp_typec_route_unregister(void *data) +{ + struct apple_dcp_typec_route *route = data; + struct apple_dcp_typec_port *port = route->port; + + typec_mux_unregister(route->typec_mux); + + guard(mutex)(&dcp_typec_fabric_lock); + if (port->preferred_route == route) + port->preferred_route = NULL; + if (port->owner == route) { + struct apple_dcp *dcp = route->dcp; + + if (port->hpd || dcp->typec_cable_connected) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + port->hpd = false; + if (dcp_typec_route_deactivate(route) && dcp_is_external_h17p(dcp)) + return; + port->owner = NULL; + } + if (port->secondary_owner == route) { + struct apple_dcp *dcp = route->dcp; + + if (port->hpd || dcp->typec_cable_connected) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + if (dcp_typec_route_deactivate(route) && dcp_is_external_h17p(dcp)) + return; + port->secondary_owner = NULL; + } + port->hpd = !!(port->owner || port->secondary_owner); + list_del(&route->port_link); + if (list_empty(&port->routes)) { + list_del(&port->link); + of_node_put(port->connector_np); + kfree(port); + } +} + +int dcp_register_typec_routes(struct apple_dcp *dcp) +{ + struct device_node *routes __free(device_node) = + of_get_child_by_name(dcp->dev->of_node, "typec-routes"); + struct device *dev = dcp->dev; + u32 route_index; + int ret; + + if (!routes) + return 0; + + for_each_available_child_of_node_scoped(routes, route_np) { + struct apple_dcp_typec_route *route; + struct device_node *endpoint __free(device_node) = NULL; + struct device_node *connector_np; + struct typec_mux_desc desc = {}; + const char *name, *mux_name; + + if (dcp->nr_typec_routes == DCP_MAX_TYPEC_ROUTES) + return dev_err_probe(dev, -E2BIG, "Too many Type-C display routes\n"); + + ret = of_property_read_u32(route_np, "reg", &route_index); + if (ret) + return dev_err_probe(dev, ret, "%pOF: missing route index\n", route_np); + if (route_index >= DCP_MAX_TYPEC_ROUTES) + return dev_err_probe(dev, -EINVAL, "%pOF: invalid route index %u\n", + route_np, route_index); + + name = devm_kasprintf(dev, GFP_KERNEL, "typec%u", route_index); + if (!name) + return -ENOMEM; + + /* + * The DT lookups above are per-DCP, but the typec_mux class is + * global. Several DCPs can offer a route to the same Type-C port, + * so the registered mux needs a name unique across all of them. + */ + mux_name = devm_kasprintf(dev, GFP_KERNEL, "%s-typec%u", + dev_name(dev), route_index); + if (!mux_name) + return -ENOMEM; + + route = &dcp->typec_routes[dcp->nr_typec_routes]; + route->dcp = dcp; + INIT_LIST_HEAD(&route->port_link); + route->phy = devm_phy_get(dev, name); + if (IS_ERR(route->phy)) + return dev_err_probe(dev, PTR_ERR(route->phy), + "%pOF: failed to get DP PHY\n", route_np); + + route->xbar = devm_mux_control_get(dev, name); + if (IS_ERR(route->xbar)) + return dev_err_probe(dev, PTR_ERR(route->xbar), + "%pOF: failed to get display crossbar\n", route_np); + + ret = of_property_read_u32_index(dev->of_node, "apple,typec-mux-indices", + route_index, &route->mux_index); + if (ret) + return dev_err_probe(dev, ret, "%pOF: missing crossbar state\n", + route_np); + + ret = of_property_read_u32_index(dev->of_node, "apple,typec-dptx-phys", + route_index, &route->dptx_phy); + if (ret) + return dev_err_probe(dev, ret, "%pOF: missing DPTX PHY index\n", + route_np); + + endpoint = of_graph_get_next_endpoint(route_np, NULL); + if (!endpoint) + return dev_err_probe(dev, -EINVAL, + "%pOF: missing Type-C graph endpoint\n", + route_np); + connector_np = of_graph_get_remote_port_parent(endpoint); + if (!connector_np) + return dev_err_probe(dev, -EINVAL, + "%pOF: missing Type-C connector\n", + route_np); + + mutex_lock(&dcp_typec_fabric_lock); + route->port = dcp_typec_port_get(connector_np); + if (route->port) + list_add_tail(&route->port_link, &route->port->routes); + mutex_unlock(&dcp_typec_fabric_lock); + if (!route->port) + return -ENOMEM; + + desc.fwnode = of_fwnode_handle(route_np); + desc.set = dcp_typec_route_set; + desc.prepare = dcp_typec_route_prepare; + desc.finish = dcp_typec_route_finish; + desc.commit = dcp_typec_route_commit; + desc.name = mux_name; + desc.drvdata = route; + route->typec_mux = typec_mux_register(dev, &desc); + if (IS_ERR(route->typec_mux)) { + mutex_lock(&dcp_typec_fabric_lock); + list_del(&route->port_link); + if (list_empty(&route->port->routes)) { + list_del(&route->port->link); + of_node_put(route->port->connector_np); + kfree(route->port); + } + mutex_unlock(&dcp_typec_fabric_lock); + return dev_err_probe(dev, PTR_ERR(route->typec_mux), + "%pOF: failed to register Type-C route\n", route_np); + } + + ret = devm_add_action_or_reset(dev, dcp_typec_route_unregister, route); + if (ret) + return ret; + + if (!dcp->phy) + dcp->phy = route->phy; + dcp->nr_typec_routes++; + } + + if (!dcp->nr_typec_routes) + return dev_err_probe(dev, -EINVAL, "Type-C route container is empty\n"); + + dcp->phy_managed_by_typec = true; + return 0; +} + +bool dcp_has_typec_routes(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return dcp->nr_typec_routes; +} + +/* + * A hybrid let go while its HDMI port was empty was parked on a Type-C PHY + * (see dcp_typec_route_deactivate()). Point it at the HDMI output again + * before connecting a display that has arrived there. + */ +static int dcp_fixed_output_select(struct apple_dcp *dcp) +{ + int ret; + + lockdep_assert_held(&dcp_typec_fabric_lock); + + if (!dcp->fixed_phy || dcp->active_typec_route) + return 0; + dcp->phy = dcp->fixed_phy; + dcp->dptx_phy = dcp->fixed_dptx_phy; + if (dcp->xbar && !dcp->fixed_route_selected) { + ret = mux_control_select(dcp->xbar, dcp->fixed_mux_index); + if (ret) + return ret; + dcp->fixed_route_selected = true; + } + return 0; +} + +/* + * Any HDMI HPD edge: a display is there or just was. Start the hold at the + * edge itself, before a tunnel retry can take the fabric lock ahead of the + * thread (see dcp_hdmi_held()). + */ +irqreturn_t dcp_dp2hdmi_hpd_edge(int irq, void *data) +{ + dcp_hdmi_hold(data); + + return IRQ_WAKE_THREAD; +} + +irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) +{ + struct apple_dcp *dcp = data; + bool connected; + + guard(mutex)(&dcp_typec_fabric_lock); + + /* again from here, should the edge handler not have run */ + dcp_hdmi_hold(dcp); + + if (READ_ONCE(dcp->active_typec_route)) { + /* + * Until a compositor owns the display, a live HDMI output + * takes its pipeline back from a direct DP-alt route: the + * compositor pairs the HDMI connector with it first. Without + * dual-stream docks it waits (see dcp_typec_hdmi_waits()). + */ + if (dcp_typec_keep_order() && + gpiod_get_value_cansleep(dcp->hdmi_hpd)) { + msleep(500); + if (gpiod_get_value_cansleep(dcp->hdmi_hpd)) + dcp_typec_rebalance_locked(NULL, 0); + } else if (!dcp_typec_dual_stream() && + gpiod_get_value_cansleep(dcp->hdmi_hpd)) { + dcp_typec_hdmi_waits(dcp); + } + return IRQ_HANDLED; + } + connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); + + /* do nothing on disconnect and trust that dcp detects it itself. + * Parallel disconnect HPDs result drm disabling the CRTC even when it + * should not. + * The interrupt should be changed to rising but for now the disconnect + * IRQs might be helpful for debugging. + */ + dev_info(dcp->dev, "DP2HDMI HPD irq, connected:%d\n", connected); + + if (connected) { + msleep(500); + connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); + dev_info(dcp->dev, "DP2HDMI HPD irq, 500ms debounce: connected:%d\n", connected); + } + + if (connected) { + int ret = dcp_fixed_output_select(dcp); + + if (ret) + dev_err(dcp->dev, "could not select the HDMI output: %d\n", ret); + else + dcp_dptx_connect(dcp, 0); + } + + return IRQ_HANDLED; +} + +void dcp_link(struct platform_device *pdev, struct apple_crtc *crtc, + struct apple_connector *connector) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + WRITE_ONCE(dcp->crtc, crtc); + + /* + * Type-C connectors belong to physical ports and are bound by the + * display fabric when a pipeline takes a route, so a pipeline with no + * fixed output simply has no connector until then. + */ + if (!connector) + return; + + dcp->fixed_connector = connector; + if (!dcp->active_typec_route) { + dcp->connector = connector; + dcp->connector_type = dcp->fixed_connector_type; + } +} + +/* Called after component unbind has drained all firmware callbacks. The + * platform devices and their Type-C muxes can outlive this DRM instance. + */ +void dcp_unlink(struct drm_device *drm) +{ + struct apple_dcp_typec_port *port; + struct drm_crtc *crtc; + + guard(mutex)(&dcp_typec_fabric_lock); + + list_for_each_entry(port, &dcp_typec_ports, link) { + if (port->connector && port->connector->base.dev == drm) + port->connector = NULL; + if (port->secondary_connector && + port->secondary_connector->base.dev == drm) + port->secondary_connector = NULL; + } + + drm_for_each_crtc(crtc, drm) { + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + struct apple_dcp *dcp; + + if (!apple_crtc->dcp) + continue; + dcp = platform_get_drvdata(apple_crtc->dcp); + if (dcp->crtc != apple_crtc) + continue; + /* + * A DCP that kept its DMA pinned this DRM device for its late + * firmware callbacks; its references stay valid, keep them. + */ + if (READ_ONCE(dcp->retain_dma)) + continue; + dcp->crtc = NULL; + dcp->connector = NULL; + dcp->fixed_connector = NULL; + dcp->typec_connector = NULL; + WRITE_ONCE(dcp->ext_backlight, false); + } +} + +void dcp_fabric_shutdown_dptx(struct apple_dcp *dcp) +{ + if (dcp->dptxep) { + /* Mux/tunnel callbacks must stop using the service before its + * endpoint is released. Firmware callbacks are drained by AFK. + */ + guard(mutex)(&dcp_typec_fabric_lock); + + afk_shutdown(dcp->dptxep); + dcp->dptxep = NULL; + scoped_guard(mutex, &dcp->hpd_mutex) { + for (int i = 0; i < ARRAY_SIZE(dcp->dptxport); i++) { + dcp->dptxport[i].enabled = false; + dcp->dptxport[i].connected = false; + dcp->dptxport[i].service = NULL; + } + } + } +} diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h new file mode 100644 index 0000000000000..35535eccb1354 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __APPLE_DCP_FABRIC_H__ +#define __APPLE_DCP_FABRIC_H__ + +#include +#include +#include + +struct apple_connector; +struct apple_crtc; +struct device_node; +struct drm_device; +struct mux_control; +struct platform_device; +struct phy; +struct typec_mux_dev; +struct work_struct; + +#define DCP_MAX_TYPEC_ROUTES 4 + +struct apple_dcp; +struct apple_dcp_typec_port; + +struct apple_dcp_typec_route { + struct apple_dcp *dcp; + struct apple_dcp_typec_port *port; + struct list_head port_link; + struct phy *phy; + struct mux_control *xbar; + struct typec_mux_dev *typec_mux; + u32 dptx_phy; + u32 mux_index; + bool selected; + /* crossbar output actually selected: xbar (dpphy) or a Thunderbolt dpin */ + unsigned int tunnel_dpin; + struct mux_control *active_xbar; + bool tunnel; + /* tunnel: crossbar brought up (at DidChangeLinkConfiguration) */ + bool xbar_up; + /* Lower mux release is consumed even when idle programming fails. */ + bool deactivate_xbar_done; + int deactivate_error; +}; + +bool dcp_is_typec_output(struct apple_dcp *dcp); +bool dcp_is_usb4_output(struct apple_dcp *dcp); + +bool dcp_has_typec_routes(struct platform_device *pdev); + +/* + * The Type-C display fabric. Ports are enumerated in device-tree order, not + * DCP probe order, so a given physical port keeps the same DRM connector index + * across boots -- userspace keys its per-monitor configuration on that name. + */ +unsigned int dcp_typec_nr_ports(void); +struct device_node *dcp_typec_port_of_node(unsigned int idx); +bool dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev); +void dcp_typec_port_set_connector(unsigned int idx, bool secondary, + struct apple_connector *connector); +bool dcp_typec_dual_stream(void); +void dcp_typec_reorder(void); +bool dcp_is_typec_only(struct platform_device *pdev); +void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, + struct apple_connector *connector); +void dcp_unlink(struct drm_device *drm); + +/* Thunderbolt DP tunnels, called from the DPTX endpoint */ +int dcp_tunnel_crossbar_up(struct apple_dcp *dcp); +int dcp_tunnel_crossbar_down(struct apple_dcp *dcp); +int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate); +int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active); + +/* DCP probe, resume and endpoint teardown hooks. */ +int dcp_register_typec_routes(struct apple_dcp *dcp); +void dcp_typec_retrain_work(struct work_struct *work); +void dcp_hdmi_hold(struct apple_dcp *dcp); +irqreturn_t dcp_dp2hdmi_hpd_edge(int irq, void *data); +irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data); +void dcp_fabric_shutdown_dptx(struct apple_dcp *dcp); + +/* DPTX operations supplied by the DCP firmware layer. */ +int dcp_dptx_connect(struct apple_dcp *dcp, u32 port); +int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port); +bool dcp_is_external_h17p(struct apple_dcp *dcp); + +#endif /* __APPLE_DCP_FABRIC_H__ */ diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 4f7687ea40846..e88c97e026642 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -16,6 +16,7 @@ #include #include +#include "dcp-fabric.h" #include "dptxep.h" #include "dcp_backlight.h" #include "dcp-announce.h" @@ -33,9 +34,7 @@ #include "version_utils.h" #define DCP_MAX_PLANES 4 -#define DCP_MAX_TYPEC_ROUTES 4 -struct apple_dcp; struct apple_dcp_afkep; struct dcp_method_entry; @@ -61,31 +60,8 @@ struct iomfb_transaction { }; struct apple_dcp_typec_port; -struct apple_dcp_typec_route { - struct apple_dcp *dcp; - struct apple_dcp_typec_port *port; - struct list_head port_link; - struct phy *phy; - struct mux_control *xbar; - struct typec_mux_dev *typec_mux; - u32 dptx_phy; - u32 mux_index; - bool selected; - /* crossbar output actually selected: xbar (dpphy) or a Thunderbolt dpin */ - unsigned int tunnel_dpin; - struct mux_control *active_xbar; - bool tunnel; - /* tunnel: crossbar brought up (at DidChangeLinkConfiguration) */ - bool xbar_up; - /* Lower mux release is consumed even when idle programming fails. */ - bool deactivate_xbar_done; - int deactivate_error; -}; - -bool dcp_is_typec_output(struct apple_dcp *dcp); void dcp_swap_watchdog_arm(struct apple_dcp *dcp); void dcp_swap_watchdog_complete(struct apple_dcp *dcp); -bool dcp_is_usb4_output(struct apple_dcp *dcp); void dcp_retry_placeholder_edid(struct apple_dcp *dcp, const struct drm_edid *drm_edid); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index ac36dbffaaf45..90aaf9fe36c70 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -45,6 +45,7 @@ #include "afk.h" #include "av.h" #include "dcp.h" +#include "dcp-fabric.h" #include "dcpext_scanout.h" #include "dcp-internal.h" #include "dcp-external-bw.h" @@ -199,1792 +200,15 @@ static bool unstable_edid = true; module_param(unstable_edid, bool, 0644); MODULE_PARM_DESC(unstable_edid, "Enable unstable EDID retrival support"); -struct apple_dcp_typec_port { - struct list_head link; - struct list_head routes; - struct device_node *connector_np; - struct apple_dcp_typec_route *owner; - struct apple_dcp_typec_route *secondary_owner; - /* Keep a port on its last DCP while that pipeline remains free. */ - struct apple_dcp_typec_route *preferred_route; - /* DRM connector for this physical port, driven by whichever DCP owns it */ - struct apple_connector *connector; - /* A second logical stream through this port's USB4 dock. */ - struct apple_connector *secondary_connector; - /* last mux state acted on, to collapse the per-candidate notifications */ - struct typec_altmode *applied_alt; - unsigned long applied_mode; - u32 applied_status; - u32 applied_conf; - bool applied_valid; - /* SAFE preparation retains this owner until the set phase completes. */ - bool safe_pending; - bool safe_finish_ready; - bool hpd; - /* Direct DP-alt: the port is in DP mode, routed or not, and its HPD. */ - bool dp_wanted; - bool dp_hpd; - /* dcp_typec_rebalance_locked()'s plan for the port's two streams */ - struct apple_dcp_typec_route *target; - struct apple_dcp_typec_route *secondary_target; - /* left out of the plan: its planned pipeline is held by a tunnel */ - bool plan_dark; -}; - -static DEFINE_MUTEX(dcp_typec_fabric_lock); -static LIST_HEAD(dcp_typec_ports); - -static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port); -static void disconnected_hpd_event(struct apple_connector *connector); - -bool dcp_is_typec_output(struct apple_dcp *dcp) -{ - return dcp->active_typec_route || - dcp->fixed_connector_type == DRM_MODE_CONNECTOR_USB; -} - -bool dcp_is_usb4_output(struct apple_dcp *dcp) -{ - return dcp->active_typec_route && dcp->active_typec_route->tunnel; -} - -static bool dcp_typec_route_is_dp(const struct typec_mux_state *state) -{ - return state->alt && state->alt->svid == USB_TYPEC_DP_SID && - state->mode >= TYPEC_DP_STATE_A && - state->mode <= TYPEC_DP_STATE_F; -} - -/* Keep a live fixed output on its own pipeline. */ -static bool dcp_typec_route_fixed_output_busy(struct apple_dcp_typec_route *route) -{ - struct apple_dcp *dcp = route->dcp; - - if (dcp->fixed_connector_type == DRM_MODE_CONNECTOR_USB) - return false; - /* - * The 14.7 firmware drives the internal panel on the IOMFB path. - * The DPTX service is a separate output, so the live panel does not - * occupy the Type-C route. - */ - if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) - return false; - if (dcp->fixed_connector && dcp->fixed_connector->connected) - return true; - if (dcp->hdmi_hpd && gpiod_get_value_cansleep(dcp->hdmi_hpd)) - return true; - - return false; -} - -static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) -{ - struct apple_dcp_typec_port *port; - - list_for_each_entry(port, &dcp_typec_ports, link) { - if (port->safe_pending && port->owner && - port->owner->dcp == route->dcp) - return false; - } - return !route->dcp->active_typec_route && - !dcp_typec_route_fixed_output_busy(route); -} - -/* - * Machines whose USB4 docks carry two independent DP streams through one - * port: DPIN0 drives the hybrid dcpext0, DPIN1 the Type-C-only dcpext1, and - * each port has a second connector for the DPIN1 stream. - */ -bool dcp_typec_dual_stream(void) -{ - return apple_dp_tunnel_t602x(); -} - -bool dcp_is_typec_only(struct platform_device *pdev) -{ - struct apple_dcp *dcp = platform_get_drvdata(pdev); - - return !dcp->fixed_phy; -} - -/* - * Can this route feed the given connector without changing its - * possible_crtcs? On dual-stream machines those are fixed at probe, since - * compositors read them once when the connector appears and pair the - * connector with a CRTC before this fabric has routed it. - */ -static bool dcp_typec_route_fits(struct apple_dcp_typec_route *route, - struct apple_connector *connector) -{ - struct apple_dcp *dcp = route->dcp; - - if (!dcp_typec_dual_stream() || !connector || !dcp->crtc) - return true; - return connector->candidate_crtcs & drm_crtc_mask(&dcp->crtc->base); -} - -static bool dcp_is_external_h17p(struct apple_dcp *dcp) -{ - return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL; -} - -static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation); -static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical); - -static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, - struct mux_control *xbar); -static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); -static int dcp_dpxbar_preselect(struct mux_control *mux, int state); -static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port); - -static int dcp_dpxbar_tunnel_select_source(struct mux_control *mux, int state) -{ - typeof(&apple_dpxbar_tunnel_select_source) select = - symbol_get(apple_dpxbar_tunnel_select_source); - int ret; - - if (!select) - return -ENOENT; - ret = select(mux, state); - symbol_put(apple_dpxbar_tunnel_select_source); - return ret; -} - -/* - * For a port without a prior owner, rank pipelines by CRTC index. A pipeline - * whose fixed output is live is not a candidate at all, so a hybrid is only - * ever ranked here when it is genuinely free. - */ -static unsigned int dcp_typec_route_score(struct apple_dcp_typec_route *route) -{ - struct apple_dcp *dcp = route->dcp; - unsigned int score; - - if (!dcp->crtc) - return UINT_MAX - 1; - - score = drm_crtc_index(&dcp->crtc->base); - /* - * Without dual-stream docks no stream is tied to the hybrid, so it goes - * to a Type-C port only when no Type-C-only pipeline is free, and an HDMI - * display plugged in later finds its pipeline idle. The port's encoder is - * narrowed to the routed pipeline before its connector reports connected, - * so the compositor pairs the connector with that pipeline. - */ - if (!dcp_typec_dual_stream() && dcp->fixed_phy) - score += 100; - - return score; -} - -static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, - struct mux_control *xbar) -{ - struct apple_dcp *dcp = route->dcp; - struct apple_connector *connector = - xbar != route->xbar && route->tunnel_dpin == 1 ? - route->port->secondary_connector : route->port->connector; - int ret; - - dcp_firmware_metadata_connection(dcp, READ_ONCE(dcp->typec_generation), false); - - /* - * The fixed output's HPD handler leaves disconnects to DCP, so the port - * can still be marked connected to a display that is gone. Release it - * (a no-op otherwise), or connecting the borrowed route returns early. - */ - ret = dcp_dptx_disconnect(dcp, 0); - if (ret) - return ret; - - if (dcp->fixed_route_selected) { - ret = mux_control_deselect(dcp->xbar); - dcp->fixed_route_selected = false; - if (ret) - return ret; - } - - /* - * Thunderbolt DP IN: the crossbar connection may only be brought up - * once DCP has configured the link and the tunnel pixel clock runs, - * see dcp_tunnel_crossbar_up(). Just remember the output here. - */ - if (xbar != route->xbar && route->mux_index) { - /* - * The T602X crossbar must point a DP IN at its pipeline before - * DCP probes AUX; other crossbars select it at link-up and - * report -EOPNOTSUPP here. - */ - ret = dcp_dpxbar_tunnel_select_source(xbar, route->mux_index); - if (ret == -EOPNOTSUPP) - ret = 0; - } else { - ret = xbar == route->xbar ? - mux_control_select(xbar, route->mux_index) : 0; - } - if (ret) { - if (dcp->xbar) { - int restore_ret; - - restore_ret = mux_control_select(dcp->xbar, - dcp->fixed_mux_index); - if (!restore_ret) - dcp->fixed_route_selected = true; - else - dev_err(dcp->dev, - "failed to restore fixed display route: %d\n", - restore_ret); - } - return ret; - } - - dcp->phy = route->phy; - dcp->dptx_phy = route->dptx_phy; - dcp->connector_type = DRM_MODE_CONNECTOR_USB; - WRITE_ONCE(dcp->ext_backlight, false); - if (connector) { - WRITE_ONCE(connector->dcp, to_platform_device(dcp->dev)); - dcp->typec_connector = connector; - dcp->connector = connector; - - /* - * Narrow the port to the pipeline now driving it. The encoder - * spans every pipeline that could, which is what lets userspace - * see the port as usable at all -- but only one of them is - * routed to the display, and userspace has no way to tell which. - * Offering it the choice makes it pair the port with a pipeline - * holding a different monitor's mode list, and the modeset is - * rejected with no way for it to recover. The hotplug that - * follows publishes the actual binding on dual-stream machines - * too. Clients must re-read encoder topology after a hotplug. - */ - if (connector->port_encoder && dcp->crtc) - connector->port_encoder->possible_crtcs = - drm_crtc_mask(&dcp->crtc->base); - } - dcp->active_typec_route = route; - scoped_guard(mutex, &dcp->tb_lock) { - route->deactivate_xbar_done = false; - route->deactivate_error = 0; - route->active_xbar = xbar; - route->tunnel = xbar != route->xbar; - route->xbar_up = !route->tunnel; - dcp->dptx_tunnel = route->tunnel; - /* crossbar controls are dpphy, dpin0, dpin1: same order as the DFP port */ - dcp->dptx_dfp_port = route->tunnel ? xbar - &route->xbar->chip->mux[0] : 0; - dcp->tb_clock_ok = false; - } - route->selected = true; - - dev_info(dcp->dev, "allocated Type-C DPTX PHY %u\n", route->dptx_phy); - return 0; -} - -static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) -{ - struct apple_dcp *dcp = route->dcp; - struct apple_connector *connector = dcp->typec_connector; - struct mux_control *active_xbar = route->active_xbar; - bool was_tunnel = route->tunnel; - int ret = 0; - - guard(mutex)(&dcp->external_lock); - if (dcp_is_external_h17p(dcp) && - (dcp->external_error || (dcp->external_power_attempted && !dcp->external_stopped))) - return dcp->external_error ?: -EAGAIN; - - dcp_firmware_metadata_connection(dcp, READ_ONCE(dcp->typec_generation), false); - - /* - * Under tb_lock so a DCP link (re)configuration can neither select the - * crossbar nor restart the tunnel pixel clock behind our back. - */ - scoped_guard(mutex, &dcp->tb_lock) { - if (route->deactivate_error) - return route->deactivate_error; - if (!route->deactivate_xbar_done) { - if (!route->tunnel || route->xbar_up) - ret = mux_control_deselect(route->active_xbar ?: route->xbar); - else - ret = dcp_dpxbar_preselect(route->active_xbar, MUX_IDLE_DISCONNECT); - /* Deselect always unlocks: never consume that selection twice. */ - route->deactivate_xbar_done = true; - route->xbar_up = false; - if (ret) { - /* Idle is unproven; retain the owner and the real error. */ - route->deactivate_error = ret; - return ret; - } - } - - if (route->tunnel && dcp->phy) { - /* the tunnel pixel clock must not outlive the tunnel */ - typeof(&apple_atc_dp_tunnel_rate) stop = - symbol_get(apple_atc_dp_tunnel_rate); - - if (stop) { - stop(dcp->phy, route->tunnel_dpin, 0); - symbol_put(apple_atc_dp_tunnel_rate); - } - } - route->active_xbar = NULL; - route->tunnel = false; - dcp->dptx_tunnel = false; - dcp->dptx_dfp_port = 0; - dcp->tb_dpin_set_active = NULL; - dcp->tb_dpin_ctx = NULL; - dcp->tb_clock_ok = false; - } - if (was_tunnel && route->mux_index) { - int sel = dcp_dpxbar_tunnel_select_source(active_xbar, -1); - - if (sel && sel != -EOPNOTSUPP) - dev_warn(dcp->dev, "DP tunnel source reset failed: %d\n", sel); - } - if (dcp_is_external_h17p(dcp)) - dpavservep_detach(dcp); - route->selected = false; - if (dcp->active_typec_route == route) - dcp->active_typec_route = NULL; - - if (connector && connector->dcp == to_platform_device(dcp->dev)) { - /* - * Until the port is activated again it has no pipeline behind - * it, and nothing can read modes or EDID from it. Report it - * disconnected for that window: leaving a connected connector - * whose ->dcp is NULL lets anything probing it in between -- - * a compositor starting up while the fabric is still settling - * -- see an output it cannot get a mode for, and give up on - * it. The new pipeline marks it connected again once the - * display has come back up on it. - */ - WRITE_ONCE(connector->connected, false); - /* hotplug work queued before this checks for it */ - WRITE_ONCE(connector->dcp, NULL); - /* no pipeline, so no backlight */ - schedule_work(&connector->bl_sync_wq); - - /* Unrouted: the port could go to any of its pipelines again. */ - if (connector->port_encoder) - connector->port_encoder->possible_crtcs = - connector->candidate_crtcs; - } - dcp->typec_connector = NULL; - dcp->connector = dcp->fixed_connector; - WRITE_ONCE(dcp->ext_backlight, false); - - if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_USB) { - dcp->connector_type = dcp->fixed_connector_type; - - /* - * Only hand the pipeline back to its fixed output if that output - * is live. Re-targeting the DPTX endpoint at the fixed PHY while - * nothing is attached there leaves DCP unable to train a link on a - * later Type-C target: it answers DEVICE_NOT_STARTED and every - * following DPTX call times out. Park on a Type-C PHY instead, - * for the same reason the USB-C-only case does below. - */ - if (dcp->hdmi_hpd && gpiod_get_value_cansleep(dcp->hdmi_hpd)) { - dcp->phy = dcp->fixed_phy; - dcp->dptx_phy = dcp->fixed_dptx_phy; - - if (dcp->xbar && !dcp->fixed_route_selected) { - ret = mux_control_select(dcp->xbar, - dcp->fixed_mux_index); - if (ret) - return ret; - dcp->fixed_route_selected = true; - } - } else { - dcp->phy = dcp->typec_routes[0].phy; - dcp->dptx_phy = dcp->typec_routes[0].dptx_phy; - } - } else { - /* Keep DPTX endpoint discovery working before a cable is attached. */ - dcp->phy = dcp->typec_routes[0].phy; - dcp->dptx_phy = dcp->typec_routes[0].dptx_phy; - dcp->connector_type = DRM_MODE_CONNECTOR_USB; - } - - return 0; -} - -static struct apple_dcp_typec_route * -dcp_typec_port_route(struct apple_dcp_typec_port *port, struct apple_dcp *dcp) -{ - struct apple_dcp_typec_route *route; - - list_for_each_entry(route, &port->routes, port_link) - if (route->dcp == dcp) - return route; - - return NULL; -} - -/* - * The DRM device once it has bound, that is once every pipeline has its - * CRTC and every port its connectors; NULL before. - */ -static struct drm_device *dcp_typec_drm(void) -{ - struct apple_dcp_typec_port *port; - struct apple_dcp_typec_route *route; - struct drm_device *drm = NULL; - - list_for_each_entry(port, &dcp_typec_ports, link) { - if (!port->connector || !port->secondary_connector) - return NULL; - list_for_each_entry(route, &port->routes, port_link) { - if (!route->dcp->crtc) - return NULL; - drm = route->dcp->crtc->base.dev; - } - } - - return drm; -} - -/* - * Has a compositor (or boot splash) taken the display? It paired its - * connectors with CRTCs when it started and keeps that pairing, also - * across a VT switch, where it drops DRM master but keeps the device - * open; moving routes under it would hand connectors to pipelines driving - * other displays. So the routes are frozen while any open file has ever - * been master, and thaw when the last such file is closed. - * - * Taken under dcp_typec_fabric_lock, next to the routing it decides, so - * the order is the fabric lock, then filelist_mutex. Nothing nests them - * the other way: filelist_mutex only covers list edits and walks, and - * drm_release() drops it before drm_file_free() calls postclose, which is - * where a close takes the fabric lock. drm_open() makes a file master - * before adding it here, but its owner cannot have paired anything before - * open() returns, and it re-probes on the hotplugs the moves send. - */ -static bool dcp_typec_frozen(struct drm_device *drm) -{ - struct drm_file *file; - - guard(mutex)(&drm->filelist_mutex); - list_for_each_entry(file, &drm->filelist, lhead) { - /* set under master_mutex, and only ever from false to true */ - if (READ_ONCE(file->was_master)) - return true; - } - - return false; -} - -/* Are the routes kept in compositor pairing order right now? */ -static bool dcp_typec_keep_order(void) -{ - struct drm_device *drm; - - if (!dcp_typec_dual_stream()) - return false; - drm = dcp_typec_drm(); - - return drm && READ_ONCE(drm->registered) && !dcp_typec_frozen(drm); -} - -/* - * HDMI hotplug often blinks: a monitor waking up or switching inputs drops - * HPD for a second or so. A Thunderbolt tunnel never moves once it has a - * pipeline, so without dual-stream docks one may not take the hybrid until - * HDMI has been quiet for this long, or a tunnel waiting for a pipeline - * would take it in the blink and keep the returning HDMI display dark. - */ -#define DCP_HDMI_HOLD_MS 10000 - -static void dcp_hdmi_hold(struct apple_dcp *dcp) -{ - WRITE_ONCE(dcp->hdmi_hold_until, - jiffies + msecs_to_jiffies(DCP_HDMI_HOLD_MS)); -} - -static bool dcp_hdmi_held(struct apple_dcp *dcp) -{ - unsigned long until = READ_ONCE(dcp->hdmi_hold_until); - - return dcp->hdmi_hpd && until && time_before(jiffies, until); -} - -/* A Thunderbolt tunnel holds @dcp's pipeline: it never moves. */ -static bool dcp_typec_tunnel_held(struct apple_dcp *dcp) -{ - return dcp->active_typec_route && dcp->active_typec_route->tunnel; -} - -/* - * Give @dcp's pipeline to the first stream, in connector order, that - * wants one, has none yet and has it in its possible_crtcs. A tunnel - * stream wants one once it is set up, or as @arriving/@dpin. A direct - * DP-alt stream wants one once its sink asserts HPD: only then can its - * connector read connected, and a compositor pairs only those. - */ -static void dcp_typec_plan_pipeline(struct apple_dcp *dcp, struct drm_crtc *crtc, - struct apple_dcp_typec_port *arriving, - unsigned int dpin) -{ - struct apple_dcp_typec_port *port; - bool secondary = false; - - /* primary connectors in port order, then the DPIN1 ones */ - do { - list_for_each_entry(port, &dcp_typec_ports, link) { - struct apple_dcp_typec_route **slot = secondary ? - &port->secondary_target : &port->target; - struct apple_connector *connector = secondary ? - port->secondary_connector : port->connector; - struct apple_dcp_typec_route *route = - dcp_typec_port_route(port, dcp); - bool wants; - - if (secondary) - wants = port->secondary_owner || - (port == arriving && dpin == 1); - else - wants = (port->owner && port->owner->tunnel) || - (port->dp_wanted && port->dp_hpd && - !port->plan_dark) || - (port == arriving && dpin == 0); - - if (!wants || *slot || !route || - !(connector->candidate_crtcs & drm_crtc_mask(crtc))) - continue; - - *slot = route; - return; - } - secondary = !secondary; - } while (secondary); -} - -/* - * The pairing a compositor starting now would make, wherever the streams - * sit at the moment: pipelines in CRTC index order, each to the first - * stream that wants it. A pipeline whose fixed output is live is left to - * that output, unless a tunnel holds it: the output cannot have it back - * then, and its connector reads disconnected. - * - * Tunnels are planned in connector order like any other stream, not - * seated on the pipelines they hold: the compositor pairs their connectors - * by order too, and seating them would plan the direct streams around - * pairings it never makes. A direct stream planned onto a pipeline a - * tunnel holds cannot have it and stays dark, so it is left out and the - * pass re-run, or every stream after it would be planned one pipeline - * off. Streams only ever drop out, so that settles within one pass per - * port. It also brings a tunnel's plan onto the pipeline it holds - * wherever direct streams ahead of it were in the way. A tunnel still - * planned elsewhere cannot be brought there by any direct stream dropping - * out, and the compositor pairs its connector wrongly whatever they do. - */ -static void dcp_typec_plan(struct drm_device *drm, - struct apple_dcp_typec_port *arriving, unsigned int dpin) -{ - struct apple_dcp_typec_port *port; - struct drm_crtc *crtc; - bool again; - - list_for_each_entry(port, &dcp_typec_ports, link) - port->plan_dark = false; - - do { - list_for_each_entry(port, &dcp_typec_ports, link) { - port->target = NULL; - port->secondary_target = NULL; - } - - /* CRTCs are listed in index order */ - drm_for_each_crtc(crtc, drm) { - struct apple_dcp *dcp = - platform_get_drvdata(to_apple_crtc(crtc)->dcp); - - if (dcp->nr_typec_routes && - (dcp_typec_tunnel_held(dcp) || - !dcp_typec_route_fixed_output_busy(&dcp->typec_routes[0]))) - dcp_typec_plan_pipeline(dcp, crtc, arriving, dpin); - } - - again = false; - list_for_each_entry(port, &dcp_typec_ports, link) { - if (port == arriving || !port->target || - (port->owner && port->owner->tunnel) || - !dcp_typec_tunnel_held(port->target->dcp)) - continue; - port->plan_dark = true; - again = true; - } - } while (again); -} - -/* Replay HPD to the pipeline a direct DP-alt port has just been given. */ -static void dcp_typec_port_attach(struct apple_dcp_typec_port *port) -{ - struct apple_dcp *dcp = port->owner->dcp; - - port->hpd = port->dp_hpd; - if (!port->hpd) - return; - - WRITE_ONCE(dcp->typec_cable_connected, true); - if (dcp->typec_connector) - dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); -} - -static struct apple_dcp_typec_route * -dcp_typec_lowest_free(struct apple_dcp_typec_port *port) -{ - struct apple_dcp_typec_route *candidate, *best = NULL; - unsigned int best_score = UINT_MAX; - - list_for_each_entry(candidate, &port->routes, port_link) { - unsigned int score; - - if (!dcp_typec_route_available(candidate)) - continue; - /* - * Prefer the lowest free CRTC index for a stable allocation order. - */ - score = dcp_typec_route_score(candidate); - if (score < best_score) { - best = candidate; - best_score = score; - } - } - - return best; -} - -/* - * The pipeline a port without one takes: the one it last had if that is - * free, as a compositor keeps a reconnected connector's CRTC, otherwise the - * best-ranked free one. Without dual-stream docks a free Type-C-only - * pipeline comes first even when the port last had the hybrid, which an HDMI - * display needs. - */ -static struct apple_dcp_typec_route * -dcp_typec_free_route(struct apple_dcp_typec_port *port) -{ - struct apple_dcp_typec_route *last = port->preferred_route; - struct apple_dcp_typec_route *best = dcp_typec_lowest_free(port); - - if (!last || !dcp_typec_route_available(last)) - return best; - if (!dcp_typec_dual_stream() && best && - dcp_typec_route_score(best) < dcp_typec_route_score(last)) - return best; - return last; -} - -/* - * Route the direct DP-alt ports left waiting for a pipeline. Their DP - * state is not reported again while it stays the same, so nothing else - * would. Each goes back to the pipeline it last had if that is free, as - * a compositor keeps a reconnected connector's CRTC, and otherwise takes - * the lowest free one, the CRTC a compositor gives a new connector. - */ -static void dcp_typec_route_waiting(void) -{ - struct apple_dcp_typec_port *port; - struct apple_dcp_typec_route *route; - - list_for_each_entry(port, &dcp_typec_ports, link) { - if (port->owner || !port->dp_wanted || !port->dp_hpd) - continue; - - route = dcp_typec_free_route(port); - if (!route || dcp_typec_route_activate(route, route->xbar)) - continue; - port->owner = route; - - dcp_typec_port_attach(port); - } -} - -/* - * Prefer connector order when placing Type-C streams before a DRM master - * starts. This is an allocation policy, not a promise about the CRTC that - * userspace must choose: each connected encoder advertises only its actual - * route, and atomic encoder selection checks that ownership. Route changes - * publish the new mask through hotplug; the plan uses candidate_crtcs so - * narrowing the advertised mask does not remove hardware routing choices. - * - * Only direct DP-alt routes move: each goes to exactly its planned - * pipeline. A Thunderbolt tunnel never moves once set up, so where one - * sits on a pipeline planned for a direct stream, or the plan has nothing - * for it, that stream stays unrouted and its connector disconnected. - * Placing it on some other pipeline would have the compositor cross both - * displays; dark is the better failure. @arriving/@dpin is a tunnel - * stream asking for a pipeline, planned like any other; its route is - * returned for apple_dcp_tb_dp_tunnel() to set up, or NULL if the plan - * has none for it or another tunnel holds that one. - * - * A move is an unplug and replug. Every route that moves is taken down - * first, so that two never share a pipeline midway, then each goes up on - * its new pipeline with its HPD replayed; the hotplugs this sends make - * fbdev and userspace re-probe. - */ -static struct apple_dcp_typec_route * -dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, - unsigned int dpin) -{ - struct drm_device *drm = dcp_typec_drm(); - struct apple_dcp_typec_route *planned = NULL; - struct apple_dcp_typec_port *port; - - lockdep_assert_held(&dcp_typec_fabric_lock); - - dcp_typec_plan(drm, arriving, dpin); - if (arriving) { - planned = dpin ? arriving->secondary_target : arriving->target; - /* it will be refused: plan as if it had not asked */ - if (planned && dcp_typec_tunnel_held(planned->dcp)) { - planned = NULL; - dcp_typec_plan(drm, NULL, 0); - } - } - - list_for_each_entry(port, &dcp_typec_ports, link) { - struct apple_dcp_typec_route *owner = port->owner; - struct apple_dcp *dcp; - - if (port->safe_pending) - continue; - if (!owner || owner->tunnel || owner == port->target) - continue; - - dcp = owner->dcp; - dev_info(dcp->dev, "re-routing %pOF from %s to %s for Type-C connector order\n", - port->connector_np, dev_name(dcp->dev), - port->target ? dev_name(port->target->dcp->dev) : "none"); - if (port->hpd || dcp->typec_cable_connected || - (dcp->typec_connector && dcp->typec_connector->connected)) - dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - port->hpd = false; - if (dcp_typec_route_deactivate(owner) && owner->selected) { - /* still routed: leave the display where it is */ - dcp_typec_port_attach(port); - continue; - } - port->owner = NULL; - - /* as after a DP exit, hand the hybrid back to a live HDMI */ - if (dcp->hdmi_hpd && dcp->active && - gpiod_get_value_cansleep(dcp->hdmi_hpd)) - dcp_dptx_connect(dcp, 0); - } - - list_for_each_entry(port, &dcp_typec_ports, link) { - struct apple_dcp_typec_route *route = port->target; - - if (port->owner || !port->dp_wanted || !port->dp_hpd || - port == arriving) - continue; - - if (!route || !dcp_typec_route_available(route) || - dcp_typec_route_activate(route, route->xbar)) { - /* - * Dark for now. Its unchanged DP state is not reported - * again, so it is placed when the plan next runs, or - * once a compositor owns the display, when a pipeline - * is freed (dcp_typec_route_waiting()). - */ - port->applied_valid = false; - continue; - } - port->owner = route; - port->preferred_route = route; - - dcp_typec_port_attach(port); - } - - return planned; -} - -/* - * A display is plugged into the HDMI port while its pipeline, the hybrid, - * drives a Type-C port. Without dual-stream docks nothing moves for it: a - * compositor keeps its connector-to-CRTC pairing and could miss a moved - * display's brief unplug, and a Thunderbolt tunnel never moves. The HDMI - * display waits and is handed the hybrid when the Type-C display lets it go - * (see dcp_typec_route_deactivate()). - */ -static void dcp_typec_hdmi_waits(struct apple_dcp *dcp) -{ - struct apple_dcp_typec_route *owner = READ_ONCE(dcp->active_typec_route); - - lockdep_assert_held(&dcp_typec_fabric_lock); - - if (owner) - dev_info(dcp->dev, "HDMI display waits: its pipeline drives the %s on %pOF\n", - owner->tunnel ? "Thunderbolt display" : "display", - owner->port->connector_np); -} - -/* - * A route has let its pipeline go. Until a compositor owns the display of a - * dual-stream machine the plan places everything anew; otherwise the - * pipeline goes to a port left waiting for one. The caller has already - * handed a hybrid back to a live HDMI display. - */ -static void dcp_typec_pipeline_freed(void) -{ - if (dcp_typec_keep_order()) - dcp_typec_rebalance_locked(NULL, 0); - else - dcp_typec_route_waiting(); -} - -/* - * Re-run the pairing pass from outside the fabric: once DRM is registered, - * as ports routed before that could not follow it, and when the last - * compositor or boot splash has closed the device, as the next one pairs - * the connectors from scratch. - */ -void dcp_typec_reorder(void) -{ - if (!dcp_typec_dual_stream()) - return; - - guard(mutex)(&dcp_typec_fabric_lock); - - if (dcp_typec_keep_order()) - dcp_typec_rebalance_locked(NULL, 0); -} - -static int dcp_typec_route_prepare(struct typec_mux_dev *mux, - struct typec_mux_state *state) -{ - struct apple_dcp_typec_route *route = typec_mux_get_drvdata(mux); - struct apple_dcp_typec_port *port = route->port; - struct apple_dcp *dcp; - int ret; - - guard(mutex)(&dcp_typec_fabric_lock); - if (state->mode != TYPEC_STATE_SAFE) - return port->safe_pending ? -EAGAIN : 0; - if (!port->owner || port->owner->tunnel) - return 0; - dcp = port->owner->dcp; - /* Stop new admission, but retain the owner until the set phase succeeds. */ - if (!port->safe_pending) - port->safe_finish_ready = false; - port->safe_pending = true; - port->applied_valid = false; - port->dp_wanted = false; - port->dp_hpd = false; - /* HPD absence alone is not proof of a completed firmware shutdown. */ - if (dcp_is_external_h17p(dcp)) { - scoped_guard(mutex, &dcp->external_lock) { - if (dcp->external_error) - return dcp->external_error; - if (!dcp->external_cable && dcp->external_stopped && - !READ_ONCE(dcp->dptxport[0].connected)) { - port->hpd = false; - return 0; - } - } - } - ret = dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - if (ret) - return ret; - port->hpd = false; - return 0; -} - -static int dcp_typec_route_finish(struct typec_mux_dev *mux, - struct typec_mux_state *state) -{ - struct apple_dcp_typec_port *port = - ((struct apple_dcp_typec_route *)typec_mux_get_drvdata(mux))->port; - int ret; - - guard(mutex)(&dcp_typec_fabric_lock); - if (state->mode != TYPEC_STATE_SAFE || !port->safe_pending || - port->safe_finish_ready) - return 0; - ret = port->owner ? dcp_typec_route_deactivate(port->owner) : 0; - if (ret) - return ret; - port->safe_finish_ready = true; - return 0; -} - -static void dcp_typec_route_commit(struct typec_mux_dev *mux, - struct typec_mux_state *state) -{ - struct apple_dcp_typec_port *port = - ((struct apple_dcp_typec_route *)typec_mux_get_drvdata(mux))->port; - - guard(mutex)(&dcp_typec_fabric_lock); - if (state->mode != TYPEC_STATE_SAFE || !port->safe_pending) - return; - if (port->owner) - port->preferred_route = port->owner; - port->owner = NULL; - port->safe_pending = false; - port->safe_finish_ready = false; - port->applied_alt = state->alt; - port->applied_mode = state->mode; - port->applied_status = 0; - port->applied_conf = 0; - port->applied_valid = true; - dcp_typec_pipeline_freed(); -} - -static int dcp_typec_route_set(struct typec_mux_dev *mux, - struct typec_mux_state *state) -{ - struct apple_dcp_typec_route *route = typec_mux_get_drvdata(mux); - struct apple_dcp_typec_port *port = route->port; - struct apple_dcp_typec_route *best = NULL; - bool is_dp = dcp_typec_route_is_dp(state); - struct typec_displayport_data *dp_data = is_dp ? state->data : NULL; - u32 dp_status = dp_data ? dp_data->status : 0; - u32 dp_conf = dp_data ? dp_data->conf : 0; - bool hpd, was_counted; - int ret = 0; - - guard(mutex)(&dcp_typec_fabric_lock); - - if (state->mode == TYPEC_STATE_SAFE && port->safe_pending) - return 0; - /* The previous SAFE transition has not finished; retry. */ - if (port->safe_pending) - return -EAGAIN; - - /* - * Every candidate route for this port is notified with the same state, - * so only the first to arrive does the work; the others return early. - * - * Deliberately not a nominated coordinator: fwnode_typec_mux_get() - * caps the providers one connector may have and drops the remainder - * without a word, so a nominated route might never be called at all -- - * and the port would then never be routed. - */ - if (port->applied_valid && port->applied_alt == state->alt && - port->applied_mode == state->mode && - port->applied_status == dp_status && port->applied_conf == dp_conf) - return 0; - - port->applied_alt = state->alt; - port->applied_mode = state->mode; - port->applied_status = dp_status; - port->applied_conf = dp_conf; - /* Failed route acquisition must remain retryable on the next update. */ - port->applied_valid = false; - - /* did the pairing pass count this port's direct stream so far? */ - was_counted = port->dp_wanted && port->dp_hpd; - - if (!is_dp) { - port->dp_wanted = false; - port->dp_hpd = false; - - /* a Thunderbolt/USB4 DP tunnel is torn down by its own path */ - if (port->owner && port->owner->tunnel) { - port->applied_valid = true; - return 0; - } - if (port->owner) { - struct apple_dcp *dcp = port->owner->dcp; - - port->preferred_route = port->owner; - if (port->hpd || dcp->typec_cable_connected || - (dcp->typec_connector && - dcp->typec_connector->connected)) - dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - port->hpd = false; - ret = dcp_typec_route_deactivate(port->owner); - if (ret) - return ret; - port->owner = NULL; - - if (dcp->hdmi_hpd && dcp->active && - gpiod_get_value_cansleep(dcp->hdmi_hpd)) - dcp_dptx_connect(dcp, 0); - dcp_typec_pipeline_freed(); - } else if (was_counted && dcp_typec_keep_order()) { - /* a stream the plan left dark is gone: plan the rest */ - dcp_typec_rebalance_locked(NULL, 0); - } - - port->safe_pending = false; - /* Data-only USB4 changes do not invalidate other display routes. */ - port->applied_valid = true; - return 0; - } - - /* - * A Thunderbolt DP tunnel still owns the port (its teardown is on the - * way): don't act on or remember this state; the tunnel teardown drops - * what was recorded so the next update is applied. - */ - if (port->owner && port->owner->tunnel) { - port->applied_valid = false; - return 0; - } - - hpd = dp_data && (dp_data->status & DP_STATUS_HPD_STATE); - port->dp_wanted = true; - port->dp_hpd = hpd; - - /* - * Until a compositor owns the display, a direct DP-alt stream has a - * pipeline only while its sink asserts HPD, and the pairing pass - * places it: HPD coming or going is the stream connecting or - * disconnecting as far as a compositor can tell. Otherwise the port - * takes the lowest free pipeline on DP entry, as a compositor does. - */ - if (dcp_typec_keep_order()) { - struct apple_dcp_typec_route *owner = port->owner; - - /* it connects or disconnects, or is not routed as it should be */ - if (hpd != was_counted || !owner != !hpd) - dcp_typec_rebalance_locked(NULL, 0); - if (!port->owner) { - if (hpd) - return -ENOSPC; - port->applied_valid = true; - return 0; - } - /* just attached, its HPD replayed: nothing left to apply */ - if (port->owner != owner) { - port->applied_valid = true; - return 0; - } - } - - if (!port->owner) { - best = dcp_typec_free_route(port); - /* Every pipeline is driving another port's display. */ - if (!best) - return -ENOSPC; - - ret = dcp_typec_route_activate(best, best->xbar); - if (ret) - return ret; - port->owner = best; - - } - - - if (!hpd && port->hpd) { - dcp_dptx_disconnect_oob(to_platform_device(port->owner->dcp->dev), 0); - } else if (hpd && !port->hpd) { - struct apple_dcp *dcp = port->owner->dcp; - - WRITE_ONCE(dcp->typec_cable_connected, true); - if (dcp->typec_connector) - dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); - } else if (hpd && dp_data && (dp_data->status & DP_STATUS_IRQ_HPD)) { - struct apple_dcp *dcp = port->owner->dcp; - - if (dcp->typec_connector) - dcp_retrain_oob(dcp->typec_connector); - } - port->hpd = hpd; - port->applied_valid = true; - - return 0; -} - -/* - * Crossbar connection up/down, looked up at runtime so appledrm does not - * require the crossbar driver to be built. - */ -static int dcp_dpxbar_link(struct mux_control *mux, bool up) -{ - typeof(&apple_dpxbar_link_up) fn; - int ret; - - fn = up ? symbol_get(apple_dpxbar_link_up) : - symbol_get(apple_dpxbar_link_down); - if (!fn) - return -ENOENT; - ret = fn(mux); - if (up) - symbol_put(apple_dpxbar_link_up); - else - symbol_put(apple_dpxbar_link_down); - return ret; -} - -static int dcp_dpxbar_preselect(struct mux_control *mux, int state) -{ - typeof(&apple_dpxbar_preselect) fn = symbol_get(apple_dpxbar_preselect); - int ret; - - if (!fn) - return -ENOENT; - ret = fn(mux, state); - symbol_put(apple_dpxbar_preselect); - return ret; -} - -/* - * Thunderbolt DP IN, before DCP hears about the display: point the DP IN - * output at this pipeline and wake the ATC's DP clock path. The crossbar - * connection itself still only comes up in dcp_tunnel_crossbar_up(), but an - * output left at its idle source (dispext0) only suits the pipeline that is - * dispext0: any other one activates the link, gets no answer from the sink - * and gives up with DEVICE_NOT_RESPONDING before it ever sets a link rate. - */ -static void dcp_tunnel_prepare(struct apple_dcp_typec_route *route, - struct mux_control *xbar) -{ - typeof(&apple_atc_dp_tunnel_open) open; - struct apple_dcp *dcp = route->dcp; - int ret; - - ret = dcp_dpxbar_preselect(xbar, route->mux_index); - if (ret && ret != -EOPNOTSUPP) - dev_warn(dcp->dev, "DP tunnel crossbar preselect failed: %d\n", ret); - - open = symbol_get(apple_atc_dp_tunnel_open); - if (!open) - return; - ret = open(dcp->phy); - symbol_put(apple_atc_dp_tunnel_open); - if (ret && ret != -EOPNOTSUPP) - dev_warn(dcp->dev, "DP tunnel PHY open failed: %d\n", ret); -} - -/* DP IN adapter handshake through the thunderbolt glue; tb_lock held */ -static int dcp_tunnel_dpin_locked(struct apple_dcp *dcp, bool active) -{ - lockdep_assert_held(&dcp->tb_lock); - if (!dcp->tb_dpin_set_active) - return -ENODEV; - return dcp->tb_dpin_set_active(dcp->tb_dpin_ctx, active); -} - -/* Limited to the T6030 14.7 pipeline; other T602X tunnels keep their order. */ -static bool dcp_t602x_tunnel_route(struct apple_dcp_typec_route *route) -{ - return route && route->tunnel && route->active_xbar && - route->dcp->fw_compat == DCP_FIRMWARE_V_14_7 && - of_device_is_compatible(route->active_xbar->chip->dev.parent->of_node, - "apple,t6020-display-crossbar"); -} - -/* - * Thunderbolt DP IN, from DCP's DidChangeLinkConfiguration once a link rate - * is set: bring the crossbar connection up (FIFO/PCLK/ATC enables) now that - * the tunnel pixel clock runs, and re-assert the DP IN adapter's - * DPTX_INACTIVE=0 afterwards. The first time this is the mux selection; after - * a re-link only the clocks are brought back (the mux selection and the ATC - * output enable are kept, see dcp_tunnel_crossbar_down()). - * Runs inside a DCP apcall: only tb_lock, which nobody holds across a DCP call. - */ -int dcp_tunnel_crossbar_up(struct apple_dcp *dcp) -{ - struct apple_dcp_typec_route *route; - int ret; - - guard(mutex)(&dcp->tb_lock); - route = dcp->active_typec_route; - if (!route || !route->tunnel || !route->active_xbar || - route->deactivate_xbar_done) - return -ENODEV; - if (!dcp->tb_clock_ok) { - dev_warn(dcp->dev, "no DP tunnel pixel clock, crossbar left down\n"); - return -EIO; - } - if (!route->xbar_up) { - /* never block a DCP call on the mux semaphore */ - ret = mux_control_try_select(route->active_xbar, route->mux_index); - if (!ret) - route->xbar_up = true; - /* T602X DP IN selection reserves and routes the mux only. */ - if (!ret && dcp_t602x_tunnel_route(route)) - ret = dcp_dpxbar_link(route->active_xbar, true); - } else { - ret = dcp_dpxbar_link(route->active_xbar, true); - } - if (ret) { - dev_warn(dcp->dev, "DP tunnel crossbar up failed: %d\n", ret); - return ret; - } - return dcp_tunnel_dpin_locked(dcp, true); -} - -/* - * Thunderbolt DP IN, from DCP's WillChangeLinkConfiguration on an established - * link: DP IN inactive, crossbar clocks down (mux selection and ATC output - * enable kept). DP IN goes active again in dcp_tunnel_crossbar_up(). - */ -int dcp_tunnel_crossbar_down(struct apple_dcp *dcp) -{ - struct apple_dcp_typec_route *route; - - guard(mutex)(&dcp->tb_lock); - route = dcp->active_typec_route; - if (!route || !route->tunnel || !route->xbar_up) - return 0; - dcp_tunnel_dpin_locked(dcp, false); - return dcp_dpxbar_link(route->active_xbar, false); -} - -/* - * Thunderbolt DP IN, from DCP's SetLinkRate: start (rate != 0) or stop the - * tunnel pixel clock. A stopped clock also takes the crossbar connection down. - */ -int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate) -{ - typeof(&apple_atc_dp_tunnel_rate) fn; - struct apple_dcp_typec_route *route; - int ret; - - guard(mutex)(&dcp->tb_lock); - if (!dcp->dptx_tunnel) - return -ENODEV; - route = dcp->active_typec_route; - /* The initial T6030 clock port supports core0 -> DP IN0 (PCLK1). - * Other routes require explicit PCLK slot selection and accounting. - */ - if (link_rate && of_machine_is_compatible("apple,t6030") && - (!route || route->mux_index != 0 || dcp->dptx_dfp_port != 1)) - return -EOPNOTSUPP; - fn = symbol_get(apple_atc_dp_tunnel_rate); - if (!fn) { - dev_err(dcp->dev, "phy-apple-atc not loaded, no DP tunnel clock\n"); - return -ENOENT; - } - route = dcp->active_typec_route; - if (!route) { - symbol_put(apple_atc_dp_tunnel_rate); - return -ENODEV; - } - if (!link_rate && route->xbar_up) - dcp_dpxbar_link(route->active_xbar, false); - ret = fn(phy, route->tunnel_dpin, link_rate); - symbol_put(apple_atc_dp_tunnel_rate); - dcp->tb_clock_ok = !ret && link_rate; - if (ret) - dev_warn(dcp->dev, "DP tunnel pixel clock (rate 0x%x) failed: %d\n", - link_rate, ret); - return ret; -} - -/* Thunderbolt DP IN: DCP Activate/Deactivate */ -int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active) -{ - struct apple_dcp_typec_route *route; - int ret; - - guard(mutex)(&dcp->tb_lock); - if (!dcp->dptx_tunnel) - return 0; - /* Route the upstream engine before firmware begins AUX negotiation. - * T602X DP IN .set does not enable clocks; DidChange does that later. - */ - route = dcp->active_typec_route; - if (active && dcp_t602x_tunnel_route(route)) { - if (!route->xbar_up) { - ret = mux_control_try_select(route->active_xbar, route->mux_index); - if (ret) - return ret; - route->xbar_up = true; - } - } - return dcp_tunnel_dpin_locked(dcp, active); -} - -/* - * Thunderbolt DP tunnels: the host router's DP IN adapters sit behind the - * crossbar's dpin0/dpin1 outputs of the port's ATC. When the Thunderbolt - * connection manager has set up a tunnel from one of them, route a free - * display pipeline there and tell DCP a display is attached, so it trains - * the link (and completes DPRX) through the tunnel. - */ -static bool dcp_tb_services_ready(struct apple_dcp *dcp) -{ - if (!dcp->external || dcp->fw_compat != DCP_FIRMWARE_V_14_7) - return true; - return smp_load_acquire(&dcp->dptxport[0].enabled) && ibootep_is_ready(dcp); -} - -int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, - bool active, int (*set_active)(void *ctx, bool active), - void *ctx) -{ - struct apple_dcp_typec_port *port = NULL, *pos; - struct apple_dcp_typec_route *candidate, *best = NULL, *planned = NULL; - struct apple_dcp_typec_route **slot; - unsigned int best_score = UINT_MAX; - bool waiting_for_external = false; - struct mux_control *ctl; - struct apple_dcp *dcp; - bool ordered; - int ret; - - if (!connector_np || dpin > 1) - return -EINVAL; - - guard(mutex)(&dcp_typec_fabric_lock); - - list_for_each_entry(pos, &dcp_typec_ports, link) { - if (pos->connector_np == connector_np) { - port = pos; - break; - } - } - if (!port) - return -ENODEV; - slot = dpin ? &port->secondary_owner : &port->owner; - - if (!active) { - if (!*slot || !(*slot)->tunnel || - (*slot)->tunnel_dpin != dpin) - return 0; - dcp = (*slot)->dcp; - if (port->hpd || dcp->typec_cable_connected) - dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - ret = dcp_typec_route_deactivate(*slot); - if (ret) { - /* the caller's context is going away regardless */ - scoped_guard(mutex, &dcp->tb_lock) { - dcp->dptx_tunnel = false; - dcp->tb_dpin_set_active = NULL; - dcp->tb_dpin_ctx = NULL; - } - return ret; - } - *slot = NULL; - port->hpd = !!(port->owner || port->secondary_owner); - /* re-apply the next Type-C mux state in full */ - port->applied_valid = false; - if (dcp->hdmi_hpd && dcp->active && - gpiod_get_value_cansleep(dcp->hdmi_hpd)) - dcp_dptx_connect(dcp, 0); - dcp_typec_pipeline_freed(); - return 0; - } - - if (*slot) { - if ((*slot)->tunnel && (*slot)->tunnel_dpin == dpin) - return 0; - dev_warn((*slot)->dcp->dev, - "port already routed, not taking DP tunnel dpin%u\n", dpin); - return -EBUSY; - } - if (port->owner && !port->owner->tunnel) - return -EBUSY; - - /* - * Until a compositor owns the display, the pairing pass decides where - * the stream goes, moving direct DP-alt routes out of its way, and - * whether it gets a pipeline at all. - */ - ordered = dcp_typec_keep_order(); - if (ordered) - planned = dcp_typec_rebalance_locked(port, dpin); - - list_for_each_entry(candidate, &port->routes, port_link) { - unsigned int score; - - if (ordered && candidate != planned) - continue; - if (!dcp_typec_route_available(candidate)) - continue; - if (!dcp_typec_dual_stream() && dcp_hdmi_held(candidate->dcp)) - continue; - if (!dcp_typec_route_fits(candidate, dpin ? - port->secondary_connector : - port->connector)) - continue; - /* A boot-present tunnel may precede the explicit firmware start. - * Do not claim its route or spend firmware reconnect attempts - * until both command services have been published. Acquire - * pairs with RemotePort publication; iBoot checks its cookie - * with acquire ordering and refuses a stopping service. - */ - /* Retained scanout after link loss must never be reactivated. */ - if (dcpext_scanout_terminal(candidate->dcp)) { - ret = -ESHUTDOWN; - goto err_reorder; - } - if (!dcp_tb_services_ready(candidate->dcp)) { - waiting_for_external = true; - continue; - } - score = dcp_typec_route_score(candidate); - /* - * DPIN0 prefers the hybrid dcpext0, which completes tunneled - * link training. On dual-stream machines DPIN1 is confined to - * dcpext1 by its connector's possible_crtcs, so both pipelines - * drive independent streams through one dock. - */ - if (dcp_typec_dual_stream() && dpin == 0 && - !candidate->dcp->fixed_phy && score < UINT_MAX - 100) - score += 100; - if (score < best_score) { - best = candidate; - best_score = score; - } - } - if (!best) { - ret = waiting_for_external ? -EAGAIN : -EBUSY; - goto err_reorder; - } - - /* The route's crossbar control is dpphy (0); dpin0/dpin1 are 1/2. */ - if (best->xbar != &best->xbar->chip->mux[0] || - best->xbar->chip->controllers < 3) { - ret = -EOPNOTSUPP; - goto err_reorder; - } - ctl = &best->xbar->chip->mux[1 + dpin]; - - dcp = best->dcp; - scoped_guard(mutex, &dcp->tb_lock) { - dcp->tb_dpin_set_active = set_active; - dcp->tb_dpin_ctx = ctx; - } - best->tunnel_dpin = dpin; - ret = dcp_typec_route_activate(best, ctl); - if (ret) { - scoped_guard(mutex, &dcp->tb_lock) { - dcp->tb_dpin_set_active = NULL; - dcp->tb_dpin_ctx = NULL; - } - goto err_reorder; - } - *slot = best; - /* the port is in USB4 mode, not DP-alt */ - port->dp_wanted = false; - dcp_tunnel_prepare(best, ctl); - - dev_info(dcp->dev, "display routed to Thunderbolt DP tunnel dpin%u\n", dpin); - - if (dcp->fw_compat == DCP_FIRMWARE_V_14_7 && dcp->dptxep && - !dcp->dptxport[0].enabled) - dev_warn(dcp->dev, "DPTX port not announced, not opening the controller\n"); - - /* - * The DP IN adapter may only be woken (DPTX_INACTIVE=0) while DCP - * drives the DPTX, i.e. from DCP's Activate call; waking it earlier - * hangs the machine. dptxep calls set_active back from Activate and - * Deactivate (set above, before the route became a tunnel). - */ - if (!dcp->typec_connector && !dcp->external) - dev_warn(dcp->dev, "no Type-C connector for the DP tunnel\n"); - WRITE_ONCE(dcp->typec_cable_connected, true); - port->hpd = true; - /* External firmware link bring-up precedes its DRM connector. Run it - * after returning to the tunnel manager, outside the fabric lock. - */ - if (dcp->external) { - dcp->typec_reconnect_tries = 0; - mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); - } else if (dcp->typec_connector) - dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); - - return 0; - -err_reorder: - /* the pass kept a pipeline for this stream: give it to the others */ - if (planned) - dcp_typec_rebalance_locked(NULL, 0); - return ret; -} -EXPORT_SYMBOL_GPL(apple_dcp_tb_dp_tunnel); - -static struct apple_dcp_typec_port * -dcp_typec_port_get(struct device_node *connector_np) -{ - struct apple_dcp_typec_port *port, *pos; - - lockdep_assert_held(&dcp_typec_fabric_lock); - - list_for_each_entry(port, &dcp_typec_ports, link) { - if (port->connector_np == connector_np) { - of_node_put(connector_np); - return port; - } - } - - port = kzalloc_obj(*port); - if (!port) { - of_node_put(connector_np); - return NULL; - } - - INIT_LIST_HEAD(&port->routes); - port->connector_np = connector_np; - - /* - * Insert in device-tree order rather than DCP probe order. The list - * index becomes the DRM connector index, and userspace keys its - * per-monitor configuration (scale, rotation, layout) on the connector - * name -- so it has to mean the same physical port on every boot. - */ - list_for_each_entry(pos, &dcp_typec_ports, link) - if (strcmp(of_node_full_name(connector_np), - of_node_full_name(pos->connector_np)) < 0) - break; - list_add_tail(&port->link, &pos->link); - return port; -} - -static struct apple_dcp_typec_port *dcp_typec_port_by_index(unsigned int idx) -{ - struct apple_dcp_typec_port *port; - unsigned int i = 0; - - lockdep_assert_held(&dcp_typec_fabric_lock); - - list_for_each_entry(port, &dcp_typec_ports, link) - if (i++ == idx) - return port; - - return NULL; -} - -unsigned int dcp_typec_nr_ports(void) -{ - struct apple_dcp_typec_port *port; - unsigned int n = 0; - - guard(mutex)(&dcp_typec_fabric_lock); - - list_for_each_entry(port, &dcp_typec_ports, link) - n++; - - return n; -} - -struct device_node *dcp_typec_port_of_node(unsigned int idx) -{ - struct apple_dcp_typec_port *port; - - guard(mutex)(&dcp_typec_fabric_lock); - - port = dcp_typec_port_by_index(idx); - - return port ? port->connector_np : NULL; -} - -bool dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev) -{ - struct apple_dcp_typec_port *port; - struct apple_dcp_typec_route *route; - - guard(mutex)(&dcp_typec_fabric_lock); - - port = dcp_typec_port_by_index(idx); - if (!port) - return false; - - list_for_each_entry(route, &port->routes, port_link) - if (route->dcp->dev == &pdev->dev) - return true; - - return false; -} - -void dcp_typec_port_set_connector(unsigned int idx, bool secondary, - struct apple_connector *connector) -{ - struct apple_dcp_typec_port *port; - struct apple_dcp_typec_route *owner; - - guard(mutex)(&dcp_typec_fabric_lock); - - port = dcp_typec_port_by_index(idx); - if (!port) - return; - - if (secondary) { - port->secondary_connector = connector; - owner = port->secondary_owner; - } else { - port->connector = connector; - owner = port->owner; - } - - /* - * The port may already have been routed, either before DRM bound or - * while these connectors were being created. Adopt that owner now, - * otherwise its display would be reported on the pipeline's fixed - * connector instead of the port it is actually plugged into. - */ - if (owner) { - struct apple_dcp *dcp = owner->dcp; - - if (dcp->crtc && connector->port_encoder) - connector->port_encoder->possible_crtcs = - drm_crtc_mask(&dcp->crtc->base); - - connector->dcp = to_platform_device(dcp->dev); - dcp->typec_connector = connector; - dcp->connector = connector; - dcp->connector_type = DRM_MODE_CONNECTOR_USB; - } -} - -static void dcp_typec_route_unregister(void *data) -{ - struct apple_dcp_typec_route *route = data; - struct apple_dcp_typec_port *port = route->port; - - typec_mux_unregister(route->typec_mux); - - guard(mutex)(&dcp_typec_fabric_lock); - if (port->preferred_route == route) - port->preferred_route = NULL; - if (port->owner == route) { - struct apple_dcp *dcp = route->dcp; - - if (port->hpd || dcp->typec_cable_connected) - dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - port->hpd = false; - if (dcp_typec_route_deactivate(route) && dcp_is_external_h17p(dcp)) - return; - port->owner = NULL; - } - if (port->secondary_owner == route) { - struct apple_dcp *dcp = route->dcp; - - if (port->hpd || dcp->typec_cable_connected) - dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - if (dcp_typec_route_deactivate(route) && dcp_is_external_h17p(dcp)) - return; - port->secondary_owner = NULL; - } - port->hpd = !!(port->owner || port->secondary_owner); - list_del(&route->port_link); - if (list_empty(&port->routes)) { - list_del(&port->link); - of_node_put(port->connector_np); - kfree(port); - } -} +static void disconnected_hpd_event(struct apple_connector *connector); -static int dcp_register_typec_routes(struct apple_dcp *dcp) +bool dcp_is_external_h17p(struct apple_dcp *dcp) { - struct device_node *routes __free(device_node) = - of_get_child_by_name(dcp->dev->of_node, "typec-routes"); - struct device *dev = dcp->dev; - u32 route_index; - int ret; - - if (!routes) - return 0; - - for_each_available_child_of_node_scoped(routes, route_np) { - struct apple_dcp_typec_route *route; - struct device_node *endpoint __free(device_node) = NULL; - struct device_node *connector_np; - struct typec_mux_desc desc = {}; - const char *name, *mux_name; - - if (dcp->nr_typec_routes == DCP_MAX_TYPEC_ROUTES) - return dev_err_probe(dev, -E2BIG, "Too many Type-C display routes\n"); - - ret = of_property_read_u32(route_np, "reg", &route_index); - if (ret) - return dev_err_probe(dev, ret, "%pOF: missing route index\n", route_np); - if (route_index >= DCP_MAX_TYPEC_ROUTES) - return dev_err_probe(dev, -EINVAL, "%pOF: invalid route index %u\n", - route_np, route_index); - - name = devm_kasprintf(dev, GFP_KERNEL, "typec%u", route_index); - if (!name) - return -ENOMEM; - - /* - * The DT lookups above are per-DCP, but the typec_mux class is - * global. Several DCPs can offer a route to the same Type-C port, - * so the registered mux needs a name unique across all of them. - */ - mux_name = devm_kasprintf(dev, GFP_KERNEL, "%s-typec%u", - dev_name(dev), route_index); - if (!mux_name) - return -ENOMEM; - - route = &dcp->typec_routes[dcp->nr_typec_routes]; - route->dcp = dcp; - INIT_LIST_HEAD(&route->port_link); - route->phy = devm_phy_get(dev, name); - if (IS_ERR(route->phy)) - return dev_err_probe(dev, PTR_ERR(route->phy), - "%pOF: failed to get DP PHY\n", route_np); - - route->xbar = devm_mux_control_get(dev, name); - if (IS_ERR(route->xbar)) - return dev_err_probe(dev, PTR_ERR(route->xbar), - "%pOF: failed to get display crossbar\n", route_np); - - ret = of_property_read_u32_index(dev->of_node, "apple,typec-mux-indices", - route_index, &route->mux_index); - if (ret) - return dev_err_probe(dev, ret, "%pOF: missing crossbar state\n", - route_np); - - ret = of_property_read_u32_index(dev->of_node, "apple,typec-dptx-phys", - route_index, &route->dptx_phy); - if (ret) - return dev_err_probe(dev, ret, "%pOF: missing DPTX PHY index\n", - route_np); - - endpoint = of_graph_get_next_endpoint(route_np, NULL); - if (!endpoint) - return dev_err_probe(dev, -EINVAL, - "%pOF: missing Type-C graph endpoint\n", - route_np); - connector_np = of_graph_get_remote_port_parent(endpoint); - if (!connector_np) - return dev_err_probe(dev, -EINVAL, - "%pOF: missing Type-C connector\n", - route_np); - - mutex_lock(&dcp_typec_fabric_lock); - route->port = dcp_typec_port_get(connector_np); - if (route->port) - list_add_tail(&route->port_link, &route->port->routes); - mutex_unlock(&dcp_typec_fabric_lock); - if (!route->port) - return -ENOMEM; - - desc.fwnode = of_fwnode_handle(route_np); - desc.set = dcp_typec_route_set; - desc.prepare = dcp_typec_route_prepare; - desc.finish = dcp_typec_route_finish; - desc.commit = dcp_typec_route_commit; - desc.name = mux_name; - desc.drvdata = route; - route->typec_mux = typec_mux_register(dev, &desc); - if (IS_ERR(route->typec_mux)) { - mutex_lock(&dcp_typec_fabric_lock); - list_del(&route->port_link); - if (list_empty(&route->port->routes)) { - list_del(&route->port->link); - of_node_put(route->port->connector_np); - kfree(route->port); - } - mutex_unlock(&dcp_typec_fabric_lock); - return dev_err_probe(dev, PTR_ERR(route->typec_mux), - "%pOF: failed to register Type-C route\n", route_np); - } - - ret = devm_add_action_or_reset(dev, dcp_typec_route_unregister, route); - if (ret) - return ret; - - if (!dcp->phy) - dcp->phy = route->phy; - dcp->nr_typec_routes++; - } - - if (!dcp->nr_typec_routes) - return dev_err_probe(dev, -EINVAL, "Type-C route container is empty\n"); - - dcp->phy_managed_by_typec = true; - return 0; + return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL; } +static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation); +static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical); /* copied and simplified from drm_vblank.c */ static void send_vblank_event(struct drm_device *dev, @@ -2709,13 +933,6 @@ int dcp_get_connector_type(struct platform_device *pdev) return dcp->fixed_connector_type; } -bool dcp_has_typec_routes(struct platform_device *pdev) -{ - struct apple_dcp *dcp = platform_get_drvdata(pdev); - - return dcp->nr_typec_routes; -} - #define DPTX_CONNECT_TIMEOUT msecs_to_jiffies(2000) #define DPTX_TUNNEL_CONNECT_TIMEOUT msecs_to_jiffies(8000) #define DPTX_RECONNECT_DELAY msecs_to_jiffies(1000) @@ -2973,7 +1190,7 @@ static int dcp_external_start_locked(struct apple_dcp *dcp) return 0; } -static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) +int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) { if (!dcp_is_external_h17p(dcp)) return dcp_dptx_connect_raw(dcp, port, NULL); @@ -3325,7 +1542,7 @@ static void disconnected_hpd_event(struct apple_connector *con) schedule_work(&con->bl_sync_wq); } -static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) +int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) { if (dcp_is_external_h17p(dcp)) return dcp_external_disconnect(dcp, false); @@ -3427,162 +1644,6 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) return dcp_dptx_disconnect(dcp, port); } -/* - * A hybrid let go while its HDMI port was empty was parked on a Type-C PHY - * (see dcp_typec_route_deactivate()). Point it at the HDMI output again - * before connecting a display that has arrived there. - */ -static int dcp_fixed_output_select(struct apple_dcp *dcp) -{ - int ret; - - lockdep_assert_held(&dcp_typec_fabric_lock); - - if (!dcp->fixed_phy || dcp->active_typec_route) - return 0; - dcp->phy = dcp->fixed_phy; - dcp->dptx_phy = dcp->fixed_dptx_phy; - if (dcp->xbar && !dcp->fixed_route_selected) { - ret = mux_control_select(dcp->xbar, dcp->fixed_mux_index); - if (ret) - return ret; - dcp->fixed_route_selected = true; - } - return 0; -} - -/* - * Any HDMI HPD edge: a display is there or just was. Start the hold at the - * edge itself, before a tunnel retry can take the fabric lock ahead of the - * thread (see dcp_hdmi_held()). - */ -static irqreturn_t dcp_dp2hdmi_hpd_edge(int irq, void *data) -{ - dcp_hdmi_hold(data); - - return IRQ_WAKE_THREAD; -} - -static irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) -{ - struct apple_dcp *dcp = data; - bool connected; - - guard(mutex)(&dcp_typec_fabric_lock); - - /* again from here, should the edge handler not have run */ - dcp_hdmi_hold(dcp); - - if (READ_ONCE(dcp->active_typec_route)) { - /* - * Until a compositor owns the display, a live HDMI output - * takes its pipeline back from a direct DP-alt route: the - * compositor pairs the HDMI connector with it first. Without - * dual-stream docks it waits (see dcp_typec_hdmi_waits()). - */ - if (dcp_typec_keep_order() && - gpiod_get_value_cansleep(dcp->hdmi_hpd)) { - msleep(500); - if (gpiod_get_value_cansleep(dcp->hdmi_hpd)) - dcp_typec_rebalance_locked(NULL, 0); - } else if (!dcp_typec_dual_stream() && - gpiod_get_value_cansleep(dcp->hdmi_hpd)) { - dcp_typec_hdmi_waits(dcp); - } - return IRQ_HANDLED; - } - connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); - - /* do nothing on disconnect and trust that dcp detects it itself. - * Parallel disconnect HPDs result drm disabling the CRTC even when it - * should not. - * The interrupt should be changed to rising but for now the disconnect - * IRQs might be helpful for debugging. - */ - dev_info(dcp->dev, "DP2HDMI HPD irq, connected:%d\n", connected); - - if (connected) { - msleep(500); - connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); - dev_info(dcp->dev, "DP2HDMI HPD irq, 500ms debounce: connected:%d\n", connected); - } - - if (connected) { - int ret = dcp_fixed_output_select(dcp); - - if (ret) - dev_err(dcp->dev, "could not select the HDMI output: %d\n", ret); - else - dcp_dptx_connect(dcp, 0); - } - - return IRQ_HANDLED; -} - -void dcp_link(struct platform_device *pdev, struct apple_crtc *crtc, - struct apple_connector *connector) -{ - struct apple_dcp *dcp = platform_get_drvdata(pdev); - - WRITE_ONCE(dcp->crtc, crtc); - - /* - * Type-C connectors belong to physical ports and are bound by the - * display fabric when a pipeline takes a route, so a pipeline with no - * fixed output simply has no connector until then. - */ - if (!connector) - return; - - dcp->fixed_connector = connector; - if (!dcp->active_typec_route) { - dcp->connector = connector; - dcp->connector_type = dcp->fixed_connector_type; - } -} - -/* Called after component unbind has drained all firmware callbacks. The - * platform devices and their Type-C muxes can outlive this DRM instance. - */ -void dcp_unlink(struct drm_device *drm) -{ - struct apple_dcp_typec_port *port; - struct drm_crtc *crtc; - - guard(mutex)(&dcp_typec_fabric_lock); - - list_for_each_entry(port, &dcp_typec_ports, link) { - if (port->connector && port->connector->base.dev == drm) - port->connector = NULL; - if (port->secondary_connector && - port->secondary_connector->base.dev == drm) - port->secondary_connector = NULL; - } - - drm_for_each_crtc(crtc, drm) { - struct apple_crtc *apple_crtc = to_apple_crtc(crtc); - struct apple_dcp *dcp; - - if (!apple_crtc->dcp) - continue; - dcp = platform_get_drvdata(apple_crtc->dcp); - if (dcp->crtc != apple_crtc) - continue; - /* - * A DCP that kept its DMA pinned this DRM device for its late - * firmware callbacks; its references stay valid, keep them. - */ - if (READ_ONCE(dcp->retain_dma)) - continue; - dcp->crtc = NULL; - dcp->connector = NULL; - dcp->fixed_connector = NULL; - dcp->typec_connector = NULL; - WRITE_ONCE(dcp->ext_backlight, false); - } -} - - bool dcp_fw_compat_is_12_x(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); @@ -4873,22 +2934,7 @@ VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, v dcp->avep = NULL; } - if (dcp->dptxep) { - /* Mux/tunnel callbacks must stop using the service before its - * endpoint is released. Firmware callbacks are drained by AFK. - */ - guard(mutex)(&dcp_typec_fabric_lock); - - afk_shutdown(dcp->dptxep); - dcp->dptxep = NULL; - scoped_guard(mutex, &dcp->hpd_mutex) { - for (int i = 0; i < ARRAY_SIZE(dcp->dptxport); i++) { - dcp->dptxport[i].enabled = false; - dcp->dptxport[i].connected = false; - dcp->dptxport[i].service = NULL; - } - } - } + dcp_fabric_shutdown_dptx(dcp); if (!IS_ERR_OR_NULL(dcp->dptx_controller_ep)) { afk_shutdown(dcp->dptx_controller_ep); @@ -5009,7 +3055,6 @@ static int dcp_platform_probe(struct platform_device *pdev) iomfb_pause_presentation(dcp, true); } - /* The domain use must survive driver-core DMA cleanup for a live session. */ ret = iommu_device_use_default_domain(dev); if (ret) @@ -5476,7 +3521,6 @@ static const struct dev_pm_ops dcp_platform_pm_ops = { SYSTEM_SLEEP_PM_OPS(dcp_platform_suspend, dcp_platform_resume) }; - static const struct apple_dcp_hw_data apple_dcp_hw_t6020 = { .num_dptx_ports = 1, }; diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index a1ace94a0404f..be130a4ca27ab 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -10,6 +10,7 @@ #include "connector.h" #include "dcp-internal.h" +#include "dcp-fabric.h" #include "parser.h" bool dcp_h17p_external_supported(struct device_node *np); @@ -38,27 +39,9 @@ void dcp_external_recover(struct platform_device *pdev); int dcp_set_crc(struct drm_crtc *crtc, bool enabled); int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commit); int dcp_get_connector_type(struct platform_device *pdev); -bool dcp_has_typec_routes(struct platform_device *pdev); - -/* - * The Type-C display fabric. Ports are enumerated in device-tree order, not - * DCP probe order, so a given physical port keeps the same DRM connector index - * across boots -- userspace keys its per-monitor configuration on that name. - */ -unsigned int dcp_typec_nr_ports(void); -struct device_node *dcp_typec_port_of_node(unsigned int idx); -bool dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev); -void dcp_typec_port_set_connector(unsigned int idx, bool secondary, - struct apple_connector *connector); -bool dcp_typec_dual_stream(void); -void dcp_typec_reorder(void); -bool dcp_is_typec_only(struct platform_device *pdev); bool dcp_fw_compat_is_12_x(struct platform_device *pdev); bool dcp_fw_compat_is_14_7(struct platform_device *pdev); unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev); -void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, - struct apple_connector *connector); -void dcp_unlink(struct drm_device *drm); int dcp_start(struct platform_device *pdev); int dcp_wait_ready(struct platform_device *pdev, u64 timeout); void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit); @@ -88,14 +71,6 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port); int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); bool dcp_hold_resume_event(struct platform_device *pdev); -struct apple_dcp; -struct phy; -/* Thunderbolt DP tunnels, called from the DPTX endpoint */ -int dcp_tunnel_crossbar_up(struct apple_dcp *dcp); -int dcp_tunnel_crossbar_down(struct apple_dcp *dcp); -int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate); -int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active); - int iomfb_start_rtkit(struct apple_dcp *dcp); void iomfb_shutdown(struct apple_dcp *dcp); /* rtkit message handler for IOMFB messages */ diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py index 90c6188f56646..96d127be946ce 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py @@ -4,6 +4,21 @@ from pathlib import Path import hashlib, json, os, re, shlex, subprocess +# The Type-C display fabric moved from dcp.c to dcp-fabric.c; extract from +# them as one unit so a function is found whichever file holds it. +_read_text = Path.read_text + + +def _read_dcp_unit(self, *args, **kwargs): + text = _read_text(self, *args, **kwargs) + fabric = self.with_name('dcp-fabric.c') + if self.name == 'dcp.c' and self.parent.name == 'apple' and fabric.exists(): + text += '\n' + _read_text(fabric, *args, **kwargs) + return text + + +Path.read_text = _read_dcp_unit + def block(source, marker): start = source.index(marker) if isinstance(marker, str) else marker opening = source.index('{', start) From e353c435b06780dfce903cfd7932ce87bf9ac822 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 18 Oct 2026 02:19:09 +1100 Subject: [PATCH 1583/2680] selftests: apple-dcp: Follow the external backlight and hybrid ranking The external display backlight now clears its capability and queues bl_sync_wq when a Type-C route is torn down or DisplayAttributes change, and the hybrid-pipeline ranking added dcp_typec_free_route() beside dcp_typec_lowest_free(). Give the host units the members and stubs that the extracted production code now uses: apple_dcp::ext_backlight, the connector's bl_sync_wq, dcp_has_panel(), schedule_work() and the two route pickers. The checks themselves are unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py | 3 ++- tools/testing/selftests/drivers/gpu/apple-dcp/rate.py | 4 +++- tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py | 5 +++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py index 88e07fb5bbfa5..5d2fc1c32d458 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py @@ -23,6 +23,7 @@ def make_unit(source, parent=False, reference=None): #define guard(k) (void) #define scoped_guard(k,l) for(int once=((void)(l),1);once;once=0) #define READ_ONCE(x) (x) +#define schedule_work(w) ((void)(w)) #define WARN_ON_ONCE(x) (x) #define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) #define to_platform_device(p) (p) @@ -55,7 +56,7 @@ def make_unit(source, parent=False, reference=None): static void mutex_unlock(struct mutex *m) { assert(!pthread_mutex_unlock(&m->value)); } struct typec_mux { struct typec_mux_context *context; struct typec_mux_state transaction; struct task_struct *owner; int srcu_index; unsigned prepared; bool applied; unsigned num_mux_devs; struct typec_mux_dev *mux_devs[3]; bool timeout_transient; }; struct connector { bool connected; }; -struct apple_dcp { int external_lock,external_error; bool external_cable,external_stopped; struct { bool connected; } dptxport[1]; void *hdmi_hpd; bool active; bool external,typec_cable_connected; void *dev; struct connector *typec_connector; }; +struct apple_dcp { int external_lock,external_error; bool external_cable,external_stopped; struct { bool connected; } dptxport[1]; void *hdmi_hpd; bool active; bool external,typec_cable_connected; void *dev; struct connector *typec_connector; bool ext_backlight; }; struct apple_dcp_typec_route; struct apple_dcp_typec_port { int link; void *connector_np; struct apple_dcp_typec_route *owner,*target,*secondary_target,*preferred_route; void *applied_alt; unsigned long applied_mode; unsigned applied_status,applied_conf; bool applied_valid,safe_pending,safe_finish_ready,dp_wanted,dp_hpd,hpd; }; struct apple_dcp_typec_route { struct apple_dcp_typec_port *port; struct apple_dcp *dcp; void *xbar; bool deactivate_xbar_done; int deactivate_error; bool selected,tunnel; }; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py index 06d82d45aae10..19a34888d593e 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py @@ -380,7 +380,7 @@ def case(value, automatic=True, internal=False, external=True, admitted=True): u64 drm_edid_generation, drm_edid_service_epoch; const struct drm_edid *drm_edid; struct { int registration_state; } base; - struct work_struct hotplug_wq; + struct work_struct hotplug_wq, bl_sync_wq; }; struct apple_epic { struct apple_dcp *dcp; unsigned endpoint; bool wire_v2; }; struct apple_epic_service { struct apple_epic *ep; void *cookie; bool enabled; }; @@ -398,6 +398,7 @@ def case(value, automatic=True, internal=False, external=True, admitted=True): u64 typec_generation; unsigned int nr_modes; struct dcp_display_mode *modes; struct { bool holding, stopped; } announce; + bool ext_backlight, has_panel; }; struct rate_guard { bool *lock; }; static struct rate_guard rate_lock(bool *p) { assert(!*p);*p=true;return (struct rate_guard){p}; } @@ -405,6 +406,7 @@ def case(value, automatic=True, internal=False, external=True, admitted=True): #define guard(kind) struct rate_guard __attribute__((cleanup(rate_unlock))) _guard = rate_lock static bool dcp_is_usb4_output(struct apple_dcp *d) { return d->usb4; } static void schedule_work(struct work_struct *w) { w->scheduled++; } +static bool __attribute__((unused)) dcp_has_panel(struct apple_dcp *d) { return d->has_panel; } static const struct edid *drm_edid_raw(const struct drm_edid *e) { return (const void *)e->data; } static bool dcpavserv_edid_current(struct apple_dcp *d, u64 generation) { assert(d->hpd_mutex);return d->typec_cable_connected&&generation==d->typec_generation&&!d->quiescing; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py index 77e3eb7782341..fc75d771f61bc 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py @@ -35,7 +35,7 @@ def check(name, unit, cases=((),), negative=False, flags=()): dcp=(ROOT/'drivers/gpu/drm/apple/dcp.c').read_text() actual=base.replace(block(base,'int main('),block(base,'int main(').replace('int main(','int __attribute__((unused)) prior_main(',1),1) actual=actual.replace('struct typec_mux_state { void *alt;', 'struct typec_altmode { unsigned svid; };\nstruct typec_mux_state { struct typec_altmode *alt;',1) - actual=actual.replace('struct connector { bool connected; };','struct encoder { unsigned possible_crtcs; };\nstruct connector { bool connected; void *dcp; struct encoder *port_encoder; unsigned candidate_crtcs; };\n#define apple_connector connector\nstruct mux_control { int unused; };',1) + actual=actual.replace('struct connector { bool connected; };','struct encoder { unsigned possible_crtcs; };\nstruct connector { bool connected; void *dcp; struct encoder *port_encoder; unsigned candidate_crtcs; int bl_sync_wq; };\n#define apple_connector connector\nstruct mux_control { int unused; };',1) actual=actual.replace('struct apple_dcp { int external_lock,external_error;', '''struct apple_dcp { int external_lock,external_error; bool external_power_attempted,dptx_tunnel,tb_clock_ok,fixed_route_selected; int tb_lock; unsigned typec_generation,dptx_dfp_port,dptx_phy,fixed_dptx_phy,fixed_mux_index; @@ -68,7 +68,8 @@ def check(name, unit, cases=((),), negative=False, flags=()): static void dpavservep_detach(struct apple_dcp *d) { (void)d; } static void dcp_firmware_metadata_connection(struct apple_dcp *d,unsigned n,bool c) { (void)d;(void)n;(void)c; } static bool dcp_typec_keep_order(void) { return false; } - static struct apple_dcp_typec_route *dcp_typec_lowest_free(struct apple_dcp_typec_port *p) { (void)p;return &route; } + static struct apple_dcp_typec_route * __attribute__((unused)) dcp_typec_lowest_free(struct apple_dcp_typec_port *p) { (void)p;return &route; } + static struct apple_dcp_typec_route * __attribute__((unused)) dcp_typec_free_route(struct apple_dcp_typec_port *p) { (void)p;return &route; } static int connect_calls; static int dcp_dptx_connect_oob(void *dev,unsigned n) { (void)dev;(void)n;connect_calls++;display.external_cable=true;display.dptxport[0].connected=true;display.typec_cable_connected=true;connector.connected=true;return 0; } static void dcp_retrain_oob(struct connector *c) { (void)c; } From ee98d3931120b47657f597bebee6ab44be42584d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 22:23:27 -0700 Subject: [PATCH 1584/2680] drm/apple: isolate display fabric decisions and add KUnit coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pipeline allocation and connector ordering depend on mutable device state, which made the hardware-tested display scenarios hard to replay. Represent that state as plain caller-owned snapshots and share a pure decision library between appledrm and the apple-dcp-fabric KUnit suite. Keep GPIO sampling, locks, mux operations and firmware calls in the fabric adapter, with no per-event allocation or new failure path. Cover route scores, preferred owners, waiting ports, dual-stream plans, tunnel errors, fixed-output reselect ordering and the six M3 scenarios. Select one shared core for driver and tests so built-in and modular configurations avoid duplicate linkage; tests require no DRM device. After an effect fails, discard the remaining batch plan and re-enter the core with fresh ownership. Bound immediate effect passes to two, then return a final revalidated plan if the hardware still fails. Unified arm64 LLVM W=1 directory builds pass with baseline diagnostics. KUnit passes 35 cases; S12 awaits W4a and S35 remains TB policy coverage. Five independent mutations fail their intended regression scenarios, and the restored suite passes. Strict checkpatch and diff checks pass. [ryan: adapted to 7.2.10 — keep the SAFE-pending gate: a pipeline whose port is in SAFE counts as owned in the core snapshot and in dcp_typec_route_available(), and such a port is never replanned; tunnel candidate failures (-ESHUTDOWN, -EAGAIN) still undo the reorder; this tree has no fabric retrain work, and its T6030 DP IN handling stays in the native 14.7 path, so those helpers are not added] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 45ff571ace3463cb163be3977e1d5fa36d0d20cb) --- MAINTAINERS | 2 +- drivers/gpu/drm/Kconfig | 2 + drivers/gpu/drm/Makefile | 2 + drivers/gpu/drm/apple/Kconfig | 1 + drivers/gpu/drm/apple/Kconfig.tests | 15 + drivers/gpu/drm/apple/dcp-fabric-core.c | 289 ++++++++++++++ drivers/gpu/drm/apple/dcp-fabric-core.h | 103 +++++ drivers/gpu/drm/apple/dcp-fabric-test.c | 486 +++++++++++++++++++++++ drivers/gpu/drm/apple/dcp-fabric.c | 480 +++++++++++----------- drivers/gpu/drm/apple/dcp-fabric.h | 3 + drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/tests/.kunitconfig | 2 + 12 files changed, 1129 insertions(+), 257 deletions(-) create mode 100644 drivers/gpu/drm/apple/Kconfig.tests create mode 100644 drivers/gpu/drm/apple/dcp-fabric-core.c create mode 100644 drivers/gpu/drm/apple/dcp-fabric-core.h create mode 100644 drivers/gpu/drm/apple/dcp-fabric-test.c create mode 100644 drivers/gpu/drm/apple/tests/.kunitconfig diff --git a/MAINTAINERS b/MAINTAINERS index f3dad1855d4b1..950665486a5e8 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2010,7 +2010,7 @@ L: dri-devel@lists.freedesktop.org S: Maintained T: git git://anongit.freedesktop.org/drm/drm-misc F: drivers/gpu/drm/apple/ -F: drivers/gpu/drm/apple/dcp-fabric.[ch] +F: drivers/gpu/drm/apple/dcp-fabric* APPLE PCIE CONTROLLER DRIVER M: Marc Zyngier diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index 5bc55c8360451..89c899e836797 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -369,6 +369,8 @@ source "drivers/gpu/drm/xlnx/Kconfig" endif +source "drivers/gpu/drm/apple/Kconfig.tests" + # Separate option because drm_panel_orientation_quirks.c is shared with fbdev config DRM_PANEL_ORIENTATION_QUIRKS tristate diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index 8bc9a05484f21..cd966b1b5b47e 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -227,6 +227,8 @@ obj-$(CONFIG_DRM_LIMA) += lima/ obj-$(CONFIG_DRM_PANFROST) += panfrost/ obj-$(CONFIG_DRM_PANTHOR) += panthor/ obj-$(CONFIG_DRM_TYR) += tyr/ +obj-$(CONFIG_DRM_APPLE_FABRIC_CORE) += apple/dcp-fabric-core.o +obj-$(CONFIG_DRM_APPLE_FABRIC_KUNIT_TEST) += apple/dcp-fabric-test.o obj-$(CONFIG_DRM_APPLE) += apple/ obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple/iomfb-state-test.o obj-$(CONFIG_DRM_ASPEED_GFX) += aspeed/ diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 542ec8c9707f1..3822c12500205 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -14,6 +14,7 @@ config DRM_APPLE select VIDEOMODE_HELPERS depends on BACKLIGHT_CLASS_DEVICE select MULTIPLEXER + select DRM_APPLE_FABRIC_CORE help Say Y if you have an Apple Silicon chipset. This includes the Type-C display fabric and Thunderbolt display routing. diff --git a/drivers/gpu/drm/apple/Kconfig.tests b/drivers/gpu/drm/apple/Kconfig.tests new file mode 100644 index 0000000000000..818ffa6b7da1e --- /dev/null +++ b/drivers/gpu/drm/apple/Kconfig.tests @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT + +config DRM_APPLE_FABRIC_CORE + tristate + +config DRM_APPLE_FABRIC_KUNIT_TEST + tristate "KUnit tests for the Apple display fabric" if !KUNIT_ALL_TESTS + depends on KUNIT + select DRM_APPLE_FABRIC_CORE + default KUNIT_ALL_TESTS + help + Test Apple display pipeline allocation, connector ordering and fixed + output priority without DRM devices or Apple display hardware. + The suite covers direct DisplayPort and USB4 tunnel combinations, + immutable tunnel ownership and external display service readiness. diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.c b/drivers/gpu/drm/apple/dcp-fabric-core.c new file mode 100644 index 0000000000000..6ff83c509452e --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-fabric-core.c @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include + +#include "dcp-fabric-core.h" + +int dcp_fabric_tunnel_request(const struct dcp_fabric_port *ports, + unsigned long key, unsigned int dpin, + const struct dcp_fabric_port **found) +{ + const struct dcp_fabric_port *port; + + *found = NULL; + if (!key || dpin > 1) + return -EINVAL; + for (port = ports; port; port = port->next) { + if (port->key != key) + continue; + *found = port; + return 0; + } + return -ENODEV; +} +EXPORT_SYMBOL_GPL(dcp_fabric_tunnel_request); + +enum dcp_fabric_capacity_action dcp_fabric_capacity_action(bool keep_order) +{ + return keep_order ? DCP_FABRIC_REBALANCE : DCP_FABRIC_PROMOTE; +} +EXPORT_SYMBOL_GPL(dcp_fabric_capacity_action); + +bool dcp_fabric_fixed_busy(bool typec_only, bool independent, bool connected, + bool hpd) +{ + return !typec_only && !independent && (connected || hpd); +} +EXPORT_SYMBOL_GPL(dcp_fabric_fixed_busy); + +bool dcp_fabric_available(const struct dcp_fabric_pipeline *pipeline) +{ + return !pipeline->owned && !pipeline->fixed_busy; +} +EXPORT_SYMBOL_GPL(dcp_fabric_available); + +unsigned int dcp_fabric_score(const struct dcp_fabric_pipeline *pipeline, + const struct dcp_fabric_policy *policy) +{ + if (!pipeline->bound) + return UINT_MAX - 1; + return pipeline->crtc_index + + (!policy->dual_stream && pipeline->has_fixed ? 100 : 0); +} +EXPORT_SYMBOL_GPL(dcp_fabric_score); + +bool dcp_fabric_fits(const struct dcp_fabric_pipeline *pipeline, + const struct dcp_fabric_policy *policy, u32 mask, + bool connector_present) +{ + return !policy->dual_stream || !connector_present || !pipeline->bound || + (mask & BIT(pipeline->crtc_index)); +} +EXPORT_SYMBOL_GPL(dcp_fabric_fits); + +bool dcp_fabric_keep_order(bool dual_stream, bool ready, bool frozen) +{ + return dual_stream && ready && !frozen; +} +EXPORT_SYMBOL_GPL(dcp_fabric_keep_order); + +bool dcp_fabric_waiting(const struct dcp_fabric_port *port) +{ + return !port->owner[0] && port->wanted && port->hpd; +} +EXPORT_SYMBOL_GPL(dcp_fabric_waiting); + +bool dcp_fabric_movable(const struct dcp_fabric_route *owner, + const struct dcp_fabric_route *target) +{ + return owner && !owner->tunnel && owner != target; +} +EXPORT_SYMBOL_GPL(dcp_fabric_movable); + +struct dcp_fabric_route * +dcp_fabric_free_route(const struct dcp_fabric_port *port, + const struct dcp_fabric_policy *policy) +{ + struct dcp_fabric_route *route, *best = NULL, *last = port->preferred; + unsigned int best_score = UINT_MAX; + + for (route = port->routes; route; route = route->next) { + unsigned int score; + + if (!dcp_fabric_available(route->pipeline)) + continue; + score = dcp_fabric_score(route->pipeline, policy); + if (score < best_score) { + best = route; + best_score = score; + } + } + if (!last || !dcp_fabric_available(last->pipeline)) + return best; + if (!policy->dual_stream && best && + best_score < dcp_fabric_score(last->pipeline, policy)) + return best; + return last; +} +EXPORT_SYMBOL_GPL(dcp_fabric_free_route); + +static struct dcp_fabric_route * +dcp_fabric_port_route(const struct dcp_fabric_port *port, + const struct dcp_fabric_pipeline *pipeline) +{ + struct dcp_fabric_route *route; + + for (route = port->routes; route; route = route->next) + if (route->pipeline == pipeline) + return route; + return NULL; +} + +static void dcp_fabric_plan_pipeline(struct dcp_fabric_pipeline *pipeline, + const struct dcp_fabric_port *ports, + const struct dcp_fabric_port *arriving, + unsigned int dpin) +{ + const struct dcp_fabric_port *port; + unsigned int stream; + + for (stream = 0; stream < 2; stream++) { + for (port = ports; port; port = port->next) { + struct dcp_fabric_route *route; + bool wants; + + if (stream) + wants = port->owner[1] || + (port == arriving && dpin == 1); + else + wants = (port->owner[0] && + port->owner[0]->tunnel) || + (port->wanted && port->hpd && + !port->plan->dark) || + (port == arriving && dpin == 0); + if (!wants || port->plan->target[stream] || + !(port->candidate_crtcs[stream] & + BIT(pipeline->crtc_index))) + continue; + route = dcp_fabric_port_route(port, pipeline); + if (!route) + continue; + port->plan->target[stream] = route; + return; + } + } +} + +void dcp_fabric_plan(struct dcp_fabric_pipeline *pipelines, + const struct dcp_fabric_port *ports, + const struct dcp_fabric_port *arriving, unsigned int dpin) +{ + const struct dcp_fabric_port *port; + struct dcp_fabric_pipeline *pipeline; + bool again; + + for (port = ports; port; port = port->next) + port->plan->dark = false; + do { + for (port = ports; port; port = port->next) { + port->plan->target[0] = NULL; + port->plan->target[1] = NULL; + } + /* The adapter supplies pipelines in CRTC index order. */ + for (pipeline = pipelines; pipeline; pipeline = pipeline->next) + if (pipeline->tunnel_held || !pipeline->fixed_busy) + dcp_fabric_plan_pipeline(pipeline, ports, + arriving, dpin); + again = false; + for (port = ports; port; port = port->next) { + struct dcp_fabric_route *target = port->plan->target[0]; + + if (port == arriving || !target || + (port->owner[0] && port->owner[0]->tunnel) || + !target->pipeline->tunnel_held) + continue; + port->plan->dark = true; + again = true; + } + } while (again); +} +EXPORT_SYMBOL_GPL(dcp_fabric_plan); + +/* 1 means an already handed, matching stream; zero admits allocation. */ +int dcp_fabric_tunnel_slot(const struct dcp_fabric_port *port, + unsigned int dpin) +{ + const struct dcp_fabric_route *slot = port->owner[dpin]; + + if (slot) + return slot->tunnel && slot->dpin == dpin ? 1 : -EADDRINUSE; + if (port->owner[0] && !port->owner[0]->tunnel) + return -EBUSY; + return 0; +} +EXPORT_SYMBOL_GPL(dcp_fabric_tunnel_slot); + +struct dcp_fabric_route * +dcp_fabric_tunnel_candidate(const struct dcp_fabric_port *port, + const struct dcp_fabric_policy *policy, + const struct dcp_fabric_route *planned, + bool ordered, unsigned int dpin, + bool connector_present, int *error) +{ + struct dcp_fabric_route *route, *best = NULL; + unsigned int best_score = UINT_MAX; + bool waiting = false; + + for (route = port->routes; route; route = route->next) { + const struct dcp_fabric_pipeline *pipeline = route->pipeline; + unsigned int score; + + if ((ordered && route != planned) || + !dcp_fabric_available(pipeline)) + continue; + if (!policy->dual_stream && pipeline->hdmi_held) + continue; + if (!dcp_fabric_fits(pipeline, policy, + port->candidate_crtcs[dpin], + connector_present)) + continue; + if (pipeline->terminal) { + *error = -ESHUTDOWN; + return NULL; + } + if (!pipeline->services_ready) { + waiting = true; + continue; + } + score = dcp_fabric_score(pipeline, policy); + if (policy->dual_stream && !dpin && !pipeline->has_fixed && + score < UINT_MAX - 100) + score += 100; + if (score < best_score) { + best = route; + best_score = score; + } + } + *error = best ? 0 : waiting ? -EAGAIN : -EBUSY; + return best; +} +EXPORT_SYMBOL_GPL(dcp_fabric_tunnel_candidate); + +unsigned int dcp_fabric_fixed_steps(bool has_fixed, bool owned, bool needs_mux, + enum dcp_fabric_fixed_step steps[2]) +{ + if (!has_fixed || owned) + return 0; + steps[0] = DCP_FABRIC_RESTORE_PHY; + if (needs_mux) { + steps[1] = DCP_FABRIC_SELECT_MUX; + return 2; + } + return 1; +} +EXPORT_SYMBOL_GPL(dcp_fabric_fixed_steps); + +u32 dcp_fabric_connector_mask(bool dual_stream, bool bound, bool routed, + unsigned int index, u32 candidates) +{ + return !dual_stream && bound && routed ? BIT(index) : candidates; +} +EXPORT_SYMBOL_GPL(dcp_fabric_connector_mask); + +enum dcp_fabric_attach_action dcp_fabric_attach_action(bool external) +{ + return external ? DCP_FABRIC_ATTACH_WORK : DCP_FABRIC_ATTACH_OOB; +} +EXPORT_SYMBOL_GPL(dcp_fabric_attach_action); + +bool dcp_fabric_t6030_link(bool tunnel, bool has_xbar, bool t6030_dpin) +{ + return tunnel && has_xbar && t6030_dpin; +} +EXPORT_SYMBOL_GPL(dcp_fabric_t6030_link); + +MODULE_DESCRIPTION("Apple display fabric decisions"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.h b/drivers/gpu/drm/apple/dcp-fabric-core.h new file mode 100644 index 0000000000000..dfa6ac24847f0 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-fabric-core.h @@ -0,0 +1,103 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __APPLE_DCP_FABRIC_CORE_H__ +#define __APPLE_DCP_FABRIC_CORE_H__ + +#include + +struct dcp_fabric_pipeline { + struct dcp_fabric_pipeline *next; + unsigned int crtc_index; + bool bound; + bool has_fixed; + bool fixed_busy; + bool owned; + bool tunnel_held; + bool hdmi_held; + bool terminal; + bool services_ready; + bool external; + bool t6030_dpin; +}; + +struct dcp_fabric_route { + struct dcp_fabric_route *next; + struct dcp_fabric_pipeline *pipeline; + bool tunnel; + unsigned int dpin; +}; + +struct dcp_fabric_plan { + struct dcp_fabric_route *target[2]; + bool dark; +}; + +/* Input topology is immutable; only the separate plan records are written. */ +struct dcp_fabric_port { + unsigned long key; + struct dcp_fabric_port *next; + struct dcp_fabric_route *routes; + struct dcp_fabric_route *owner[2]; + struct dcp_fabric_route *preferred; + u32 candidate_crtcs[2]; + bool wanted; + bool hpd; + struct dcp_fabric_plan *plan; +}; + +struct dcp_fabric_policy { + bool dual_stream; +}; + +enum dcp_fabric_capacity_action { + DCP_FABRIC_PROMOTE, + DCP_FABRIC_REBALANCE, +}; + +enum dcp_fabric_fixed_step { + DCP_FABRIC_RESTORE_PHY, + DCP_FABRIC_SELECT_MUX, +}; + +enum dcp_fabric_attach_action { + DCP_FABRIC_ATTACH_OOB, + DCP_FABRIC_ATTACH_WORK, +}; + +int dcp_fabric_tunnel_request(const struct dcp_fabric_port *ports, + unsigned long key, unsigned int dpin, + const struct dcp_fabric_port **found); +enum dcp_fabric_capacity_action dcp_fabric_capacity_action(bool keep_order); +bool dcp_fabric_fixed_busy(bool typec_only, bool independent, bool connected, + bool hpd); +bool dcp_fabric_available(const struct dcp_fabric_pipeline *pipeline); +unsigned int dcp_fabric_score(const struct dcp_fabric_pipeline *pipeline, + const struct dcp_fabric_policy *policy); +bool dcp_fabric_fits(const struct dcp_fabric_pipeline *pipeline, + const struct dcp_fabric_policy *policy, u32 mask, + bool connector_present); +bool dcp_fabric_keep_order(bool dual_stream, bool ready, bool frozen); +bool dcp_fabric_waiting(const struct dcp_fabric_port *port); +bool dcp_fabric_movable(const struct dcp_fabric_route *owner, + const struct dcp_fabric_route *target); +struct dcp_fabric_route * +dcp_fabric_free_route(const struct dcp_fabric_port *port, + const struct dcp_fabric_policy *policy); +void dcp_fabric_plan(struct dcp_fabric_pipeline *pipelines, + const struct dcp_fabric_port *ports, + const struct dcp_fabric_port *arriving, unsigned int dpin); +int dcp_fabric_tunnel_slot(const struct dcp_fabric_port *port, + unsigned int dpin); +struct dcp_fabric_route * +dcp_fabric_tunnel_candidate(const struct dcp_fabric_port *port, + const struct dcp_fabric_policy *policy, + const struct dcp_fabric_route *planned, + bool ordered, unsigned int dpin, + bool connector_present, int *error); +unsigned int dcp_fabric_fixed_steps(bool has_fixed, bool owned, bool needs_mux, + enum dcp_fabric_fixed_step steps[2]); +u32 dcp_fabric_connector_mask(bool dual_stream, bool bound, bool routed, + unsigned int index, u32 candidates); +enum dcp_fabric_attach_action dcp_fabric_attach_action(bool external); +bool dcp_fabric_t6030_link(bool tunnel, bool has_xbar, bool t6030_dpin); + +#endif /* __APPLE_DCP_FABRIC_CORE_H__ */ diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c new file mode 100644 index 0000000000000..d78c1af66d007 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -0,0 +1,486 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include + +#include "dcp-fabric-core.h" + +struct fabric_fixture { + struct dcp_fabric_pipeline pipeline[2]; + struct dcp_fabric_route route[3][2]; + struct dcp_fabric_port port[3]; + struct dcp_fabric_plan plan[3]; + struct dcp_fabric_policy policy; +}; + +static void fabric_init(struct fabric_fixture *f, bool dual) +{ + unsigned int p, r; + + memset(f, 0, sizeof(*f)); + f->policy.dual_stream = dual; + f->pipeline[0].next = &f->pipeline[1]; + for (r = 0; r < 2; r++) { + f->pipeline[r].bound = true; + f->pipeline[r].crtc_index = r + 1; + f->pipeline[r].services_ready = true; + } + f->pipeline[0].has_fixed = true; + for (p = 0; p < 3; p++) { + f->port[p].key = p + 1; + f->port[p].routes = &f->route[p][0]; + f->port[p].plan = &f->plan[p]; + f->port[p].candidate_crtcs[0] = BIT(1) | BIT(2); + f->port[p].candidate_crtcs[1] = BIT(2); + if (p < 2) + f->port[p].next = &f->port[p + 1]; + for (r = 0; r < 2; r++) { + f->route[p][r].pipeline = &f->pipeline[r]; + if (!r) + f->route[p][r].next = &f->route[p][r + 1]; + } + } +} + +/* Fake effects only commit ownership after the real decision succeeds. */ +static int fabric_direct(struct fabric_fixture *f, unsigned int p) +{ + struct dcp_fabric_route *route; + + f->port[p].wanted = true; + f->port[p].hpd = true; + route = dcp_fabric_free_route(&f->port[p], &f->policy); + if (!route) + return -EBUSY; + route->pipeline->owned = true; + f->port[p].owner[0] = route; + return 0; +} + +static int fabric_tunnel(struct fabric_fixture *f, unsigned int p, + unsigned int dpin) +{ + struct dcp_fabric_route *route; + int error; + + error = dcp_fabric_tunnel_slot(&f->port[p], dpin); + if (error) + return error > 0 ? 0 : error; + route = dcp_fabric_tunnel_candidate(&f->port[p], &f->policy, NULL, + false, dpin, true, &error); + if (!route) + return error; + route->pipeline->owned = true; + route->pipeline->tunnel_held = true; + route->tunnel = true; + route->dpin = dpin; + f->port[p].owner[dpin] = route; + return 0; +} + +static void fabric_release(struct fabric_fixture *f, unsigned int p, + unsigned int dpin) +{ + struct dcp_fabric_route *route = f->port[p].owner[dpin]; + + if (!route) + return; + route->pipeline->owned = false; + route->pipeline->tunnel_held = false; + route->tunnel = false; + f->port[p].preferred = route; + f->port[p].owner[dpin] = NULL; + f->port[p].wanted = false; + f->port[p].hpd = false; +} + +static void fabric_promote(struct fabric_fixture *f) +{ + unsigned int p; + + if (dcp_fabric_capacity_action(false) != DCP_FABRIC_PROMOTE) + return; + for (p = 0; p < 3; p++) + if (dcp_fabric_waiting(&f->port[p])) + fabric_direct(f, p); +} + +struct fabric_scenario { + const char *name; + unsigned int id; +}; + +static const struct fabric_scenario scenarios[] = { + { "S1_lone_direct", 1 }, + { "S2_direct_then_hdmi", 2 }, + { "S3_lone_tunnel", 3 }, + { "S4_both_typec_orders", 4 }, + { "S5_full_hdmi_direct_tunnel", 5 }, + { "S6_direct_release_then_tunnel", 6 }, + { "S7_tunnel_release_promotes_direct", 7 }, + { "S8_frozen_hdmi_waits", 8 }, + { "S9_parked_fixed_reselect", 9 }, + { "S10_hdmi_blink", 10 }, + { "S11_hdmi_absence", 11 }, + { "S12_direct_settling", 12 }, + { "S13_boot_three_outputs", 13 }, + { "S14_boot_two_typec", 14 }, + { "S15_boot_hdmi_tunnel", 15 }, + { "S16_resume_protection", 16 }, + { "S17_occupied_slot_errors", 17 }, + { "S18_connector_mask", 18 }, + { "S20_base_capacity", 20 }, + { "S21_base_lone", 21 }, + { "S30_dual_pairing_plan", 30 }, + { "S31_dual_dpin_candidates", 31 }, + { "S32_dual_hdmi_reclaim", 32 }, + { "S33_dual_scores", 33 }, + { "S34_dual_park_reselect", 34 }, + { "S35_dual_retry_off", 35 }, + { "S40_desktop_promotion", 40 }, + { "S41_dedicated_hdmi", 41 }, + { "S50a_panel_has_no_routes", 50 }, + { "S50b_external_service_readiness", 51 }, + { "S50c_terminal_scanout", 52 }, + { "S50d_external_attach_dispatch", 53 }, + { "S50e_t6030_crossbar", 54 }, + { "S50f_unwired_m3_ports", 55 }, +}; + +static void fabric_scenario_desc(const struct fabric_scenario *scenario, + char *desc) +{ + strscpy(desc, scenario->name, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(fabric_scenario, scenarios, fabric_scenario_desc); + +static void fabric_scenario_test(struct kunit *test) +{ + const struct fabric_scenario *scenario = test->param_value; + struct fabric_fixture f; + const struct dcp_fabric_port *found; + enum dcp_fabric_fixed_step steps[2]; + unsigned int count, order; + + fabric_init(&f, scenario->id >= 30 && scenario->id <= 35); + switch (scenario->id) { + case 1: + f.port[0].preferred = &f.route[0][0]; + KUNIT_ASSERT_EQ(test, fabric_direct(&f, 0), 0); + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], &f.route[0][1]); + break; + case 2: + KUNIT_ASSERT_EQ(test, fabric_direct(&f, 0), 0); + f.pipeline[0].fixed_busy = true; + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], &f.route[0][1]); + KUNIT_EXPECT_FALSE(test, dcp_fabric_available(&f.pipeline[0])); + break; + case 3: + KUNIT_ASSERT_EQ(test, fabric_tunnel(&f, 0, 0), 0); + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], &f.route[0][1]); + break; + case 4: + case 14: + for (order = 0; order < 2; order++) { + fabric_init(&f, false); + KUNIT_ASSERT_EQ(test, + order ? fabric_direct(&f, 0) : + fabric_tunnel(&f, 0, 0), + 0); + KUNIT_ASSERT_EQ(test, + order ? fabric_tunnel(&f, 1, 0) : + fabric_direct(&f, 1), + 0); + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], + &f.route[0][1]); + KUNIT_EXPECT_PTR_EQ(test, f.port[1].owner[0], + &f.route[1][0]); + } + break; + case 5: + case 6: + case 13: + f.pipeline[0].fixed_busy = true; + KUNIT_ASSERT_EQ(test, fabric_direct(&f, 0), 0); + KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 1, 0), -EBUSY); + KUNIT_EXPECT_PTR_EQ(test, f.port[1].owner[0], NULL); + if (scenario->id == 6) { + fabric_release(&f, 0, 0); + KUNIT_ASSERT_EQ(test, fabric_tunnel(&f, 1, 0), 0); + KUNIT_EXPECT_PTR_EQ(test, f.port[1].owner[0], + &f.route[1][1]); + } + break; + case 7: + case 20: + case 40: + if (scenario->id != 7) { + for (order = 0; order < 3; order++) + f.port[order].routes = &f.route[order][1]; + } + f.pipeline[0].fixed_busy = true; + KUNIT_ASSERT_EQ(test, fabric_tunnel(&f, 0, 0), 0); + KUNIT_EXPECT_EQ(test, fabric_direct(&f, 1), -EBUSY); + fabric_release(&f, 0, 0); + fabric_promote(&f); + KUNIT_EXPECT_PTR_EQ(test, f.port[1].owner[0], &f.route[1][1]); + break; + case 8: + KUNIT_ASSERT_EQ(test, fabric_direct(&f, 0), 0); + KUNIT_ASSERT_EQ(test, fabric_direct(&f, 1), 0); + fabric_release(&f, 0, 0); + f.pipeline[0].fixed_busy = true; + KUNIT_EXPECT_FALSE(test, + dcp_fabric_keep_order(false, true, true)); + KUNIT_EXPECT_PTR_EQ(test, f.port[1].owner[0], &f.route[1][0]); + KUNIT_EXPECT_EQ(test, + dcp_fabric_fixed_steps(true, true, true, steps), + 0U); + fabric_release(&f, 1, 0); + KUNIT_EXPECT_EQ(test, + dcp_fabric_fixed_steps(true, false, true, steps), + 2U); + break; + case 9: + case 34: + count = dcp_fabric_fixed_steps(true, false, true, steps); + KUNIT_ASSERT_EQ(test, count, 2U); + /* Fake PHY/mux trace consumes the production decision in order. */ + KUNIT_EXPECT_EQ(test, steps[0], DCP_FABRIC_RESTORE_PHY); + KUNIT_EXPECT_EQ(test, steps[1], DCP_FABRIC_SELECT_MUX); + break; + case 10: + case 16: + f.pipeline[1].owned = true; + f.pipeline[0].hdmi_held = true; + KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 0), -EBUSY); + f.pipeline[0].fixed_busy = true; + f.pipeline[0].hdmi_held = false; + KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 0), -EBUSY); + break; + case 11: + f.pipeline[1].owned = true; + f.pipeline[0].hdmi_held = true; + KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 0), -EBUSY); + f.pipeline[0].hdmi_held = false; + KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 0), 0); + break; + case 12: + kunit_skip(test, + "W4a adds direct borrower settling protection"); + break; + case 15: + f.pipeline[0].fixed_busy = true; + KUNIT_ASSERT_EQ(test, fabric_tunnel(&f, 0, 0), 0); + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], &f.route[0][1]); + break; + case 17: + KUNIT_ASSERT_EQ(test, fabric_tunnel(&f, 0, 0), 0); + f.route[0][1].dpin = 1; + KUNIT_EXPECT_EQ(test, dcp_fabric_tunnel_slot(&f.port[0], 0), + -EADDRINUSE); + fabric_init(&f, false); + KUNIT_ASSERT_EQ(test, fabric_direct(&f, 0), 0); + KUNIT_EXPECT_EQ(test, dcp_fabric_tunnel_slot(&f.port[0], 1), + -EBUSY); + /* Preserve the original occupied-primary errno in this extraction. */ + KUNIT_EXPECT_EQ(test, dcp_fabric_tunnel_slot(&f.port[0], 0), + -EADDRINUSE); + break; + case 18: + KUNIT_EXPECT_EQ(test, + dcp_fabric_connector_mask(false, true, true, 2, + BIT(1) | BIT(2)), + (u32)BIT(2)); + KUNIT_EXPECT_EQ(test, + dcp_fabric_connector_mask(false, true, false, 2, + BIT(1) | BIT(2)), + (u32)(BIT(1) | BIT(2))); + break; + case 21: + case 41: + /* A dedicated fixed pipeline has no edge in this port topology. */ + f.port[0].routes = &f.route[0][1]; + KUNIT_ASSERT_EQ(test, fabric_direct(&f, 0), 0); + KUNIT_EXPECT_FALSE(test, f.pipeline[0].owned); + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], &f.route[0][1]); + break; + case 30: + f.port[0].wanted = true; + f.port[0].hpd = true; + f.port[1].wanted = true; + f.port[1].hpd = true; + dcp_fabric_plan(f.pipeline, f.port, NULL, 0); + KUNIT_EXPECT_PTR_EQ(test, f.plan[0].target[0], &f.route[0][0]); + KUNIT_EXPECT_PTR_EQ(test, f.plan[1].target[0], &f.route[1][1]); + KUNIT_EXPECT_TRUE(test, + dcp_fabric_keep_order(true, true, false)); + KUNIT_EXPECT_FALSE(test, + dcp_fabric_keep_order(true, true, true)); + break; + case 31: + KUNIT_ASSERT_EQ(test, fabric_tunnel(&f, 0, 0), 0); + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], &f.route[0][0]); + KUNIT_ASSERT_EQ(test, fabric_tunnel(&f, 0, 1), 0); + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[1], &f.route[0][1]); + break; + case 32: + f.port[0].wanted = true; + f.port[0].hpd = true; + f.pipeline[0].fixed_busy = true; + dcp_fabric_plan(f.pipeline, f.port, NULL, 0); + KUNIT_EXPECT_PTR_EQ(test, f.plan[0].target[0], &f.route[0][1]); + KUNIT_EXPECT_FALSE(test, + dcp_fabric_keep_order(true, true, true)); + break; + case 33: + KUNIT_EXPECT_EQ(test, + dcp_fabric_score(&f.pipeline[0], &f.policy), 1U); + KUNIT_EXPECT_EQ(test, + dcp_fabric_score(&f.pipeline[1], &f.policy), 2U); + f.port[0].preferred = &f.route[0][1]; + KUNIT_EXPECT_PTR_EQ(test, + dcp_fabric_free_route(&f.port[0], &f.policy), + &f.route[0][1]); + f.pipeline[0].hdmi_held = true; + KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 1, 0), 0); + break; + case 35: + kunit_skip(test, "TB match-data retry policy is tested by W3"); + break; + case 50: + f.port[0].routes = NULL; + KUNIT_EXPECT_PTR_EQ(test, + dcp_fabric_free_route(&f.port[0], &f.policy), + NULL); + KUNIT_EXPECT_FALSE(test, f.pipeline[0].owned); + break; + case 51: + case 52: + f.port[0].routes = &f.route[0][1]; + f.pipeline[1].external = true; + f.pipeline[1].services_ready = false; + f.pipeline[1].terminal = scenario->id == 52; + KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 0), + scenario->id == 52 ? -ESHUTDOWN : -EAGAIN); + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], NULL); + f.pipeline[1].services_ready = true; + if (scenario->id == 51) + KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 0), 0); + break; + case 53: + KUNIT_EXPECT_EQ(test, dcp_fabric_attach_action(true), + DCP_FABRIC_ATTACH_WORK); + KUNIT_EXPECT_EQ(test, dcp_fabric_attach_action(false), + DCP_FABRIC_ATTACH_OOB); + break; + case 54: + KUNIT_EXPECT_TRUE(test, dcp_fabric_t6030_link(true, true, true)); + KUNIT_EXPECT_FALSE(test, dcp_fabric_t6030_link(true, true, false)); + KUNIT_EXPECT_FALSE(test, dcp_fabric_t6030_link(true, false, true)); + break; + case 55: + f.port[0].next = NULL; + KUNIT_EXPECT_EQ(test, dcp_fabric_tunnel_request(f.port, 2, 0, &found), -ENODEV); + KUNIT_EXPECT_PTR_EQ(test, found, NULL); + KUNIT_EXPECT_EQ(test, dcp_fabric_tunnel_request(f.port, 3, 0, &found), -ENODEV); + KUNIT_EXPECT_EQ(test, dcp_fabric_tunnel_request(f.port, 0, 0, &found), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_fabric_tunnel_request(f.port, 1, 2, &found), -EINVAL); + break; + } +} + +static void fabric_dark_tunnel_test(struct kunit *test) +{ + struct fabric_fixture f; + + fabric_init(&f, true); + /* A direct stream precedes an immutable tunnel on its planned pipeline. */ + f.port[0].wanted = true; + f.port[0].hpd = true; + f.port[1].owner[0] = &f.route[1][0]; + f.route[1][0].tunnel = true; + f.pipeline[0].tunnel_held = true; + f.pipeline[0].owned = true; + dcp_fabric_plan(f.pipeline, f.port, NULL, 0); + KUNIT_EXPECT_TRUE(test, f.plan[0].dark); + KUNIT_EXPECT_PTR_EQ(test, f.plan[0].target[0], NULL); + KUNIT_EXPECT_PTR_EQ(test, f.plan[1].target[0], &f.route[1][0]); + KUNIT_EXPECT_FALSE(test, + dcp_fabric_movable(&f.route[1][0], &f.route[1][1])); +} + +static void fabric_effect_failure_test(struct kunit *test) +{ + struct fabric_fixture f; + + fabric_init(&f, true); + f.port[0].wanted = true; + f.port[0].hpd = true; + f.port[1].wanted = true; + f.port[1].hpd = true; + dcp_fabric_plan(f.pipeline, f.port, NULL, 0); + KUNIT_EXPECT_PTR_EQ(test, f.plan[0].target[0], &f.route[0][0]); + /* Failed activation rolled back to a live fixed output: fresh sample. */ + f.pipeline[0].fixed_busy = true; + dcp_fabric_plan(f.pipeline, f.port, NULL, 0); + KUNIT_EXPECT_PTR_EQ(test, f.plan[0].target[0], &f.route[0][1]); + KUNIT_EXPECT_PTR_EQ(test, f.plan[1].target[0], NULL); + KUNIT_EXPECT_FALSE(test, f.pipeline[0].owned); + + fabric_init(&f, false); + f.port[0].wanted = true; + f.port[0].hpd = true; + f.port[1].wanted = true; + f.port[1].hpd = true; + /* Port zero's failed mux acquisition committed no ownership. */ + KUNIT_EXPECT_PTR_EQ(test, dcp_fabric_free_route(&f.port[0], &f.policy), + &f.route[0][1]); + KUNIT_EXPECT_EQ(test, fabric_direct(&f, 1), 0); + KUNIT_EXPECT_PTR_EQ(test, f.port[1].owner[0], &f.route[1][1]); +} + +static void fabric_unbound_and_mask_test(struct kunit *test) +{ + struct fabric_fixture f; + + fabric_init(&f, false); + f.pipeline[0].bound = false; + KUNIT_EXPECT_EQ(test, dcp_fabric_score(&f.pipeline[0], &f.policy), + UINT_MAX - 1); + KUNIT_EXPECT_TRUE(test, + dcp_fabric_fixed_busy(false, false, true, false)); + KUNIT_EXPECT_FALSE(test, + dcp_fabric_fixed_busy(false, true, true, true)); + KUNIT_EXPECT_FALSE(test, + dcp_fabric_fixed_busy(true, false, true, true)); + f.policy.dual_stream = true; + KUNIT_EXPECT_TRUE(test, + dcp_fabric_fits(&f.pipeline[0], &f.policy, 0, true)); + KUNIT_EXPECT_FALSE(test, dcp_fabric_fits(&f.pipeline[1], &f.policy, + BIT(1), true)); + KUNIT_EXPECT_EQ(test, + dcp_fabric_connector_mask(true, true, true, 2, + BIT(1) | BIT(2)), + (u32)(BIT(1) | BIT(2))); +} + +static struct kunit_case fabric_tests[] = { + KUNIT_CASE_PARAM(fabric_scenario_test, fabric_scenario_gen_params), + KUNIT_CASE(fabric_dark_tunnel_test), + KUNIT_CASE(fabric_effect_failure_test), + KUNIT_CASE(fabric_unbound_and_mask_test), + {} +}; + +static struct kunit_suite fabric_suite = { + .name = "apple-dcp-fabric", + .test_cases = fabric_tests, +}; + +kunit_test_suite(fabric_suite); + +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 50431d6c3f25b..96700b8f5c399 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -23,6 +23,8 @@ #include "ibootep.h" #include "dcp-internal.h" struct apple_dcp_typec_port { + struct dcp_fabric_port core; + struct dcp_fabric_plan plan; struct list_head link; struct list_head routes; struct device_node *connector_np; @@ -99,25 +101,34 @@ static bool dcp_typec_route_fixed_output_busy(struct apple_dcp_typec_route *rout */ if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) return false; - if (dcp->fixed_connector && dcp->fixed_connector->connected) - return true; - if (dcp->hdmi_hpd && gpiod_get_value_cansleep(dcp->hdmi_hpd)) - return true; + return dcp_fabric_fixed_busy(false, + false, + dcp->fixed_connector && dcp->fixed_connector->connected, + !(dcp->fixed_connector && dcp->fixed_connector->connected) && + dcp->hdmi_hpd && + gpiod_get_value_cansleep(dcp->hdmi_hpd)); +} +/* A port in SAFE retains this pipeline until the mux set phase completes. */ +static bool dcp_typec_safe_held(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_port *port; + + list_for_each_entry(port, &dcp_typec_ports, link) + if (port->safe_pending && port->owner && port->owner->dcp == dcp) + return true; return false; } static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) { - struct apple_dcp_typec_port *port; + struct dcp_fabric_pipeline pipeline = { + .owned = !!route->dcp->active_typec_route || + dcp_typec_safe_held(route->dcp), + .fixed_busy = dcp_typec_route_fixed_output_busy(route), + }; - list_for_each_entry(port, &dcp_typec_ports, link) { - if (port->safe_pending && port->owner && - port->owner->dcp == route->dcp) - return false; - } - return !route->dcp->active_typec_route && - !dcp_typec_route_fixed_output_busy(route); + return dcp_fabric_available(&pipeline); } /* @@ -137,22 +148,6 @@ bool dcp_is_typec_only(struct platform_device *pdev) return !dcp->fixed_phy; } -/* - * Can this route feed the given connector without changing its - * possible_crtcs? On dual-stream machines those are fixed at probe, since - * compositors read them once when the connector appears and pair the - * connector with a CRTC before this fabric has routed it. - */ -static bool dcp_typec_route_fits(struct apple_dcp_typec_route *route, - struct apple_connector *connector) -{ - struct apple_dcp *dcp = route->dcp; - - if (!dcp_typec_dual_stream() || !connector || !dcp->crtc) - return true; - return connector->candidate_crtcs & drm_crtc_mask(&dcp->crtc->base); -} - static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, struct mux_control *xbar); static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); @@ -171,33 +166,6 @@ static int dcp_dpxbar_tunnel_select_source(struct mux_control *mux, int state) return ret; } -/* - * For a port without a prior owner, rank pipelines by CRTC index. A pipeline - * whose fixed output is live is not a candidate at all, so a hybrid is only - * ever ranked here when it is genuinely free. - */ -static unsigned int dcp_typec_route_score(struct apple_dcp_typec_route *route) -{ - struct apple_dcp *dcp = route->dcp; - unsigned int score; - - if (!dcp->crtc) - return UINT_MAX - 1; - - score = drm_crtc_index(&dcp->crtc->base); - /* - * Without dual-stream docks no stream is tied to the hybrid, so it goes - * to a Type-C port only when no Type-C-only pipeline is free, and an HDMI - * display plugged in later finds its pipeline idle. The port's encoder is - * narrowed to the routed pipeline before its connector reports connected, - * so the compositor pairs the connector with that pipeline. - */ - if (!dcp_typec_dual_stream() && dcp->fixed_phy) - score += 100; - - return score; -} - static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, struct mux_control *xbar) { @@ -281,7 +249,10 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, */ if (connector->port_encoder && dcp->crtc) connector->port_encoder->possible_crtcs = - drm_crtc_mask(&dcp->crtc->base); + dcp_fabric_connector_mask(false, + true, true, + drm_crtc_index(&dcp->crtc->base), + connector->candidate_crtcs); } dcp->active_typec_route = route; scoped_guard(mutex, &dcp->tb_lock) { @@ -430,18 +401,6 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) return 0; } -static struct apple_dcp_typec_route * -dcp_typec_port_route(struct apple_dcp_typec_port *port, struct apple_dcp *dcp) -{ - struct apple_dcp_typec_route *route; - - list_for_each_entry(route, &port->routes, port_link) - if (route->dcp == dcp) - return route; - - return NULL; -} - /* * The DRM device once it has bound, that is once every pipeline has its * CRTC and every port its connectors; NULL before. @@ -503,8 +462,9 @@ static bool dcp_typec_keep_order(void) if (!dcp_typec_dual_stream()) return false; drm = dcp_typec_drm(); - - return drm && READ_ONCE(drm->registered) && !dcp_typec_frozen(drm); + if (!drm || !READ_ONCE(drm->registered)) + return false; + return dcp_fabric_keep_order(true, true, dcp_typec_frozen(drm)); } void dcp_hdmi_hold(struct apple_dcp *dcp) @@ -526,49 +486,60 @@ static bool dcp_typec_tunnel_held(struct apple_dcp *dcp) return dcp->active_typec_route && dcp->active_typec_route->tunnel; } -/* - * Give @dcp's pipeline to the first stream, in connector order, that - * wants one, has none yet and has it in its possible_crtcs. A tunnel - * stream wants one once it is set up, or as @arriving/@dpin. A direct - * DP-alt stream wants one once its sink asserts HPD: only then can its - * connector read connected, and a compositor pairs only those. - */ -static void dcp_typec_plan_pipeline(struct apple_dcp *dcp, struct drm_crtc *crtc, - struct apple_dcp_typec_port *arriving, - unsigned int dpin) +static bool dcp_tb_services_ready(struct apple_dcp *dcp); + +static struct apple_dcp_typec_route * +dcp_fabric_real_route(struct dcp_fabric_route *route) { - struct apple_dcp_typec_port *port; - bool secondary = false; - - /* primary connectors in port order, then the DPIN1 ones */ - do { - list_for_each_entry(port, &dcp_typec_ports, link) { - struct apple_dcp_typec_route **slot = secondary ? - &port->secondary_target : &port->target; - struct apple_connector *connector = secondary ? - port->secondary_connector : port->connector; - struct apple_dcp_typec_route *route = - dcp_typec_port_route(port, dcp); - bool wants; - - if (secondary) - wants = port->secondary_owner || - (port == arriving && dpin == 1); - else - wants = (port->owner && port->owner->tunnel) || - (port->dp_wanted && port->dp_hpd && - !port->plan_dark) || - (port == arriving && dpin == 0); + return route ? container_of(route, struct apple_dcp_typec_route, core) : + NULL; +} - if (!wants || *slot || !route || - !(connector->candidate_crtcs & drm_crtc_mask(crtc))) - continue; +static void dcp_fabric_snapshot_port(struct apple_dcp_typec_port *port, + bool tunnel) +{ + struct apple_dcp_typec_route *route; + struct dcp_fabric_route **tail = &port->core.routes; - *slot = route; - return; - } - secondary = !secondary; - } while (secondary); + lockdep_assert_held(&dcp_typec_fabric_lock); + port->core.owner[0] = port->owner ? &port->owner->core : NULL; + port->core.owner[1] = + port->secondary_owner ? &port->secondary_owner->core : NULL; + port->core.preferred = + port->preferred_route ? &port->preferred_route->core : NULL; + port->core.wanted = port->dp_wanted; + port->core.hpd = port->dp_hpd; + port->core.candidate_crtcs[0] = + port->connector ? port->connector->candidate_crtcs : 0; + port->core.candidate_crtcs[1] = + port->secondary_connector ? + port->secondary_connector->candidate_crtcs : + 0; + port->core.plan = &port->plan; + list_for_each_entry(route, &port->routes, port_link) { + struct apple_dcp *dcp = route->dcp; + struct dcp_fabric_pipeline *pipeline = &dcp->fabric; + + pipeline->bound = !!dcp->crtc; + pipeline->crtc_index = + dcp->crtc ? drm_crtc_index(&dcp->crtc->base) : 0; + pipeline->has_fixed = !!dcp->fixed_phy; + pipeline->fixed_busy = dcp_typec_route_fixed_output_busy(route); + pipeline->owned = !!dcp->active_typec_route || + dcp_typec_safe_held(dcp); + pipeline->tunnel_held = dcp_typec_tunnel_held(dcp); + pipeline->hdmi_held = tunnel && dcp_hdmi_held(dcp); + pipeline->terminal = tunnel && dcpext_scanout_terminal(dcp); + pipeline->services_ready = !tunnel || + dcp_tb_services_ready(dcp); + pipeline->external = dcp->external; + route->core.pipeline = pipeline; + route->core.tunnel = route->tunnel; + route->core.dpin = route->tunnel_dpin; + *tail = &route->core; + tail = &route->core.next; + } + *tail = NULL; } /* @@ -594,39 +565,35 @@ static void dcp_typec_plan(struct drm_device *drm, struct apple_dcp_typec_port *arriving, unsigned int dpin) { struct apple_dcp_typec_port *port; + struct dcp_fabric_port *ports = NULL, **port_tail = &ports; + struct dcp_fabric_pipeline *pipelines = NULL, + **pipeline_tail = &pipelines; struct drm_crtc *crtc; - bool again; - - list_for_each_entry(port, &dcp_typec_ports, link) - port->plan_dark = false; - do { - list_for_each_entry(port, &dcp_typec_ports, link) { - port->target = NULL; - port->secondary_target = NULL; - } - - /* CRTCs are listed in index order */ - drm_for_each_crtc(crtc, drm) { - struct apple_dcp *dcp = - platform_get_drvdata(to_apple_crtc(crtc)->dcp); - - if (dcp->nr_typec_routes && - (dcp_typec_tunnel_held(dcp) || - !dcp_typec_route_fixed_output_busy(&dcp->typec_routes[0]))) - dcp_typec_plan_pipeline(dcp, crtc, arriving, dpin); - } + list_for_each_entry(port, &dcp_typec_ports, link) { + dcp_fabric_snapshot_port(port, false); + *port_tail = &port->core; + port_tail = &port->core.next; + } + *port_tail = NULL; + drm_for_each_crtc(crtc, drm) { + struct apple_dcp *dcp = + platform_get_drvdata(to_apple_crtc(crtc)->dcp); - again = false; - list_for_each_entry(port, &dcp_typec_ports, link) { - if (port == arriving || !port->target || - (port->owner && port->owner->tunnel) || - !dcp_typec_tunnel_held(port->target->dcp)) - continue; - port->plan_dark = true; - again = true; - } - } while (again); + if (!dcp->nr_typec_routes) + continue; + *pipeline_tail = &dcp->fabric; + pipeline_tail = &dcp->fabric.next; + } + *pipeline_tail = NULL; + dcp_fabric_plan(pipelines, ports, arriving ? &arriving->core : NULL, + dpin); + list_for_each_entry(port, &dcp_typec_ports, link) { + port->target = dcp_fabric_real_route(port->plan.target[0]); + port->secondary_target = + dcp_fabric_real_route(port->plan.target[1]); + port->plan_dark = port->plan.dark; + } } /* Replay HPD to the pipeline a direct DP-alt port has just been given. */ @@ -643,30 +610,6 @@ static void dcp_typec_port_attach(struct apple_dcp_typec_port *port) dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); } -static struct apple_dcp_typec_route * -dcp_typec_lowest_free(struct apple_dcp_typec_port *port) -{ - struct apple_dcp_typec_route *candidate, *best = NULL; - unsigned int best_score = UINT_MAX; - - list_for_each_entry(candidate, &port->routes, port_link) { - unsigned int score; - - if (!dcp_typec_route_available(candidate)) - continue; - /* - * Prefer the lowest free CRTC index for a stable allocation order. - */ - score = dcp_typec_route_score(candidate); - if (score < best_score) { - best = candidate; - best_score = score; - } - } - - return best; -} - /* * The pipeline a port without one takes: the one it last had if that is * free, as a compositor keeps a reconnected connector's CRTC, otherwise the @@ -677,15 +620,12 @@ dcp_typec_lowest_free(struct apple_dcp_typec_port *port) static struct apple_dcp_typec_route * dcp_typec_free_route(struct apple_dcp_typec_port *port) { - struct apple_dcp_typec_route *last = port->preferred_route; - struct apple_dcp_typec_route *best = dcp_typec_lowest_free(port); - - if (!last || !dcp_typec_route_available(last)) - return best; - if (!dcp_typec_dual_stream() && best && - dcp_typec_route_score(best) < dcp_typec_route_score(last)) - return best; - return last; + struct dcp_fabric_policy policy = { .dual_stream = + dcp_typec_dual_stream() }; + + dcp_fabric_snapshot_port(port, false); + return dcp_fabric_real_route(dcp_fabric_free_route(&port->core, + &policy)); } /* @@ -701,7 +641,13 @@ static void dcp_typec_route_waiting(void) struct apple_dcp_typec_route *route; list_for_each_entry(port, &dcp_typec_ports, link) { - if (port->owner || !port->dp_wanted || !port->dp_hpd) + struct dcp_fabric_port state = { + .owner[0] = port->owner ? &port->owner->core : NULL, + .wanted = port->dp_wanted, + .hpd = port->dp_hpd, + }; + + if (!dcp_fabric_waiting(&state)) continue; route = dcp_typec_free_route(port); @@ -743,9 +689,11 @@ dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, struct drm_device *drm = dcp_typec_drm(); struct apple_dcp_typec_route *planned = NULL; struct apple_dcp_typec_port *port; + unsigned int attempts = 0; lockdep_assert_held(&dcp_typec_fabric_lock); +replan: dcp_typec_plan(drm, arriving, dpin); if (arriving) { planned = dpin ? arriving->secondary_target : arriving->target; @@ -756,13 +704,20 @@ dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, } } + /* Even on persistent failure, return only a fresh, revalidated plan. */ + if (attempts++ == 2) + return planned; + list_for_each_entry(port, &dcp_typec_ports, link) { struct apple_dcp_typec_route *owner = port->owner; struct apple_dcp *dcp; + /* A port in SAFE keeps its owner until the set phase completes. */ if (port->safe_pending) continue; - if (!owner || owner->tunnel || owner == port->target) + if (!dcp_fabric_movable(owner ? &owner->core : NULL, + port->target ? &port->target->core : + NULL)) continue; dcp = owner->dcp; @@ -773,10 +728,12 @@ dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, (dcp->typec_connector && dcp->typec_connector->connected)) dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); port->hpd = false; - if (dcp_typec_route_deactivate(owner) && owner->selected) { - /* still routed: leave the display where it is */ - dcp_typec_port_attach(port); - continue; + if (dcp_typec_route_deactivate(owner)) { + if (owner->selected) + dcp_typec_port_attach(port); + else + port->owner = NULL; + goto replan; } port->owner = NULL; @@ -793,8 +750,11 @@ dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, port == arriving) continue; - if (!route || !dcp_typec_route_available(route) || - dcp_typec_route_activate(route, route->xbar)) { + if (!route || !dcp_typec_route_available(route)) { + port->applied_valid = false; + continue; + } + if (dcp_typec_route_activate(route, route->xbar)) { /* * Dark for now. Its unchanged DP state is not reported * again, so it is placed when the plan next runs, or @@ -802,7 +762,7 @@ dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, * is freed (dcp_typec_route_waiting()). */ port->applied_valid = false; - continue; + goto replan; } port->owner = route; port->preferred_route = route; @@ -841,7 +801,7 @@ static void dcp_typec_hdmi_waits(struct apple_dcp *dcp) */ static void dcp_typec_pipeline_freed(void) { - if (dcp_typec_keep_order()) + if (dcp_fabric_capacity_action(dcp_typec_keep_order()) == DCP_FABRIC_REBALANCE) dcp_typec_rebalance_locked(NULL, 0); else dcp_typec_route_waiting(); @@ -1321,11 +1281,11 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, bool active, int (*set_active)(void *ctx, bool active), void *ctx) { - struct apple_dcp_typec_port *port = NULL, *pos; - struct apple_dcp_typec_route *candidate, *best = NULL, *planned = NULL; + struct apple_dcp_typec_port *port, *pos; + struct dcp_fabric_port *ports = NULL, **tail = &ports; + const struct dcp_fabric_port *found; + struct apple_dcp_typec_route *best = NULL, *planned = NULL; struct apple_dcp_typec_route **slot; - unsigned int best_score = UINT_MAX; - bool waiting_for_external = false; struct mux_control *ctl; struct apple_dcp *dcp; bool ordered; @@ -1337,13 +1297,15 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, guard(mutex)(&dcp_typec_fabric_lock); list_for_each_entry(pos, &dcp_typec_ports, link) { - if (pos->connector_np == connector_np) { - port = pos; - break; - } + pos->core.key = (unsigned long)pos->connector_np; + *tail = &pos->core; + tail = &pos->core.next; } - if (!port) - return -ENODEV; + *tail = NULL; + ret = dcp_fabric_tunnel_request(ports, (unsigned long)connector_np, dpin, &found); + if (ret) + return ret; + port = container_of(found, struct apple_dcp_typec_port, core); slot = dpin ? &port->secondary_owner : &port->owner; if (!active) { @@ -1374,15 +1336,35 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, return 0; } - if (*slot) { - if ((*slot)->tunnel && (*slot)->tunnel_dpin == dpin) + { + struct dcp_fabric_port state = { + .owner[0] = port->owner ? &port->owner->core : NULL, + .owner[1] = port->secondary_owner ? + &port->secondary_owner->core : + NULL, + }; + + if (port->owner) { + port->owner->core.tunnel = port->owner->tunnel; + port->owner->core.dpin = port->owner->tunnel_dpin; + } + if (port->secondary_owner) { + port->secondary_owner->core.tunnel = + port->secondary_owner->tunnel; + port->secondary_owner->core.dpin = + port->secondary_owner->tunnel_dpin; + } + ret = dcp_fabric_tunnel_slot(&state, dpin); + if (ret > 0) return 0; - dev_warn((*slot)->dcp->dev, - "port already routed, not taking DP tunnel dpin%u\n", dpin); - return -EBUSY; + if (ret) { + if (ret == -EADDRINUSE) + dev_warn((*slot)->dcp->dev, + "port already routed, not taking DP tunnel dpin%u\n", + dpin); + return ret; + } } - if (port->owner && !port->owner->tunnel) - return -EBUSY; /* * Until a compositor owns the display, the pairing pass decides where @@ -1393,52 +1375,25 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, if (ordered) planned = dcp_typec_rebalance_locked(port, dpin); - list_for_each_entry(candidate, &port->routes, port_link) { - unsigned int score; - - if (ordered && candidate != planned) - continue; - if (!dcp_typec_route_available(candidate)) - continue; - if (!dcp_typec_dual_stream() && dcp_hdmi_held(candidate->dcp)) - continue; - if (!dcp_typec_route_fits(candidate, dpin ? - port->secondary_connector : - port->connector)) - continue; - /* A boot-present tunnel may precede the explicit firmware start. - * Do not claim its route or spend firmware reconnect attempts - * until both command services have been published. Acquire - * pairs with RemotePort publication; iBoot checks its cookie - * with acquire ordering and refuses a stopping service. + { + struct dcp_fabric_policy policy = { + .dual_stream = dcp_typec_dual_stream() + }; + struct dcp_fabric_route *chosen; + bool connector_present = dpin ? !!port->secondary_connector : !!port->connector; + + dcp_fabric_snapshot_port(port, true); + chosen = dcp_fabric_tunnel_candidate(&port->core, + &policy, planned ? &planned->core : NULL, + ordered, dpin, + connector_present, + &ret); + best = dcp_fabric_real_route(chosen); + /* -ESHUTDOWN (retained terminal scanout) and -EAGAIN (services + * not yet published) also undo the reorder. */ - /* Retained scanout after link loss must never be reactivated. */ - if (dcpext_scanout_terminal(candidate->dcp)) { - ret = -ESHUTDOWN; + if (!best) goto err_reorder; - } - if (!dcp_tb_services_ready(candidate->dcp)) { - waiting_for_external = true; - continue; - } - score = dcp_typec_route_score(candidate); - /* - * DPIN0 prefers the hybrid dcpext0, which completes tunneled - * link training. On dual-stream machines DPIN1 is confined to - * dcpext1 by its connector's possible_crtcs, so both pipelines - * drive independent streams through one dock. - */ - if (dcp_typec_dual_stream() && dpin == 0 && - !candidate->dcp->fixed_phy && score < UINT_MAX - 100) - score += 100; - if (score < best_score) { - best = candidate; - best_score = score; - } - } - if (!best) { - ret = waiting_for_external ? -EAGAIN : -EBUSY; - goto err_reorder; } /* The route's crossbar control is dpphy (0); dpin0/dpin1 are 1/2. */ @@ -1487,7 +1442,7 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, /* External firmware link bring-up precedes its DRM connector. Run it * after returning to the tunnel manager, outside the fabric lock. */ - if (dcp->external) { + if (dcp_fabric_attach_action(dcp->external) == DCP_FABRIC_ATTACH_WORK) { dcp->typec_reconnect_tries = 0; mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); } else if (dcp->typec_connector) @@ -1627,7 +1582,10 @@ void dcp_typec_port_set_connector(unsigned int idx, bool secondary, if (dcp->crtc && connector->port_encoder) connector->port_encoder->possible_crtcs = - drm_crtc_mask(&dcp->crtc->base); + dcp_fabric_connector_mask(false, + true, true, + drm_crtc_index(&dcp->crtc->base), + connector->candidate_crtcs); connector->dcp = to_platform_device(dcp->dev); dcp->typec_connector = connector; @@ -1811,19 +1769,29 @@ bool dcp_has_typec_routes(struct platform_device *pdev) */ static int dcp_fixed_output_select(struct apple_dcp *dcp) { + enum dcp_fabric_fixed_step steps[2]; + unsigned int count, i; int ret; lockdep_assert_held(&dcp_typec_fabric_lock); - - if (!dcp->fixed_phy || dcp->active_typec_route) - return 0; - dcp->phy = dcp->fixed_phy; - dcp->dptx_phy = dcp->fixed_dptx_phy; - if (dcp->xbar && !dcp->fixed_route_selected) { - ret = mux_control_select(dcp->xbar, dcp->fixed_mux_index); - if (ret) - return ret; - dcp->fixed_route_selected = true; + count = dcp_fabric_fixed_steps(!!dcp->fixed_phy, + !!dcp->active_typec_route, + dcp->xbar && !dcp->fixed_route_selected, + steps); + for (i = 0; i < count; i++) { + switch (steps[i]) { + case DCP_FABRIC_RESTORE_PHY: + dcp->phy = dcp->fixed_phy; + dcp->dptx_phy = dcp->fixed_dptx_phy; + break; + case DCP_FABRIC_SELECT_MUX: + ret = mux_control_select(dcp->xbar, + dcp->fixed_mux_index); + if (ret) + return ret; + dcp->fixed_route_selected = true; + break; + } } return 0; } diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index 35535eccb1354..4f55a21ecd5d8 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -6,6 +6,8 @@ #include #include +#include "dcp-fabric-core.h" + struct apple_connector; struct apple_crtc; struct device_node; @@ -22,6 +24,7 @@ struct apple_dcp; struct apple_dcp_typec_port; struct apple_dcp_typec_route { + struct dcp_fabric_route core; struct apple_dcp *dcp; struct apple_dcp_typec_port *port; struct list_head port_link; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index e88c97e026642..8854f81242c83 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -208,6 +208,7 @@ struct apple_dcp { bool drm_retained; /* vblank_wq and swap_watchdog_wq disabled by the last unbind */ bool vblank_works_disabled; + struct dcp_fabric_pipeline fabric; struct device *dev; struct platform_device *piodma; bool piodma_created; diff --git a/drivers/gpu/drm/apple/tests/.kunitconfig b/drivers/gpu/drm/apple/tests/.kunitconfig new file mode 100644 index 0000000000000..2708065668a99 --- /dev/null +++ b/drivers/gpu/drm/apple/tests/.kunitconfig @@ -0,0 +1,2 @@ +CONFIG_KUNIT=y +CONFIG_DRM_APPLE_FABRIC_KUNIT_TEST=y From bed7288783cae06fc2764d742d535a20f30f59c5 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 22:34:19 -0700 Subject: [PATCH 1585/2680] drm/apple: model HDMI presence and serve borrowers after settling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A low HPD blink must not lend a hybrid pipeline to either a tunnel or a DP-alt display. Record SETTLING and an edge generation in hardirq, then sample in the thread. High becomes PRESENT and clears the deadline; low retains the ten-second window. Reject stale samples and expiries. Emit one capacity event when settling confirms ABSENT, promoting waiting DP-alt ports and optionally queueing an eligible Thunderbolt retry. The optional provider callback only performs an RCU lookup and queues work, without taking a mutex or waiting under the fabric lock. Drain the new freezable expiry work with the other endpoint users on suspend/unbind. Enable the HPD IRQ before sampling on resume. Low starts a full window; high clears the hold and reselects the fixed output before connecting. This preserves allocation relative to the old unconditional resume hold: a high sample already makes the fixed output busy, low remains guarded, and later falling edges re-arm in hardirq. Leave first-boot ordering and dual-stream allocation unchanged. Cover both borrowers, one-shot absence, resume low/high, stale expiry and sample, later-edge precedence, dual-stream bypass and jiffies wrap in apple-dcp-fabric KUnit. Four mutations fail the expected scenario rows. [ryan: adapted to 7.2.10 — the capacity-wakeup declaration goes in dcp-fabric.h, not the 170-usb owned dp-tunnel.h (no provider exists; a later commit removes it); the settle work joins this tree's typec work disable/enable, which has no fabric retrain work] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit b0596b223a54f9cfd1aaa8f1bbbaf3ad51c71552) --- drivers/gpu/drm/apple/dcp-fabric-core.c | 50 ++++++-- drivers/gpu/drm/apple/dcp-fabric-core.h | 24 +++- drivers/gpu/drm/apple/dcp-fabric-test.c | 127 +++++++++++++++++--- drivers/gpu/drm/apple/dcp-fabric.c | 151 +++++++++++++++++++++--- drivers/gpu/drm/apple/dcp-fabric.h | 9 +- drivers/gpu/drm/apple/dcp-internal.h | 6 +- drivers/gpu/drm/apple/dcp.c | 7 +- 7 files changed, 327 insertions(+), 47 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.c b/drivers/gpu/drm/apple/dcp-fabric-core.c index 6ff83c509452e..64234e88f4ed3 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.c +++ b/drivers/gpu/drm/apple/dcp-fabric-core.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include "dcp-fabric-core.h" @@ -39,9 +40,46 @@ bool dcp_fabric_fixed_busy(bool typec_only, bool independent, bool connected, } EXPORT_SYMBOL_GPL(dcp_fabric_fixed_busy); -bool dcp_fabric_available(const struct dcp_fabric_pipeline *pipeline) +u64 dcp_fabric_presence_edge(struct dcp_fabric_presence *presence, + unsigned long now, unsigned long window) { - return !pipeline->owned && !pipeline->fixed_busy; + presence->generation++; + presence->state = DCP_FABRIC_SETTLING; + presence->deadline = now + window; + return presence->generation; +} +EXPORT_SYMBOL_GPL(dcp_fabric_presence_edge); + +bool dcp_fabric_presence_sample(struct dcp_fabric_presence *presence, + u64 generation, bool high, + unsigned long now, unsigned long window) +{ + if (generation != presence->generation) + return false; + presence->state = high ? DCP_FABRIC_PRESENT : DCP_FABRIC_SETTLING; + presence->deadline = high ? 0 : now + window; + return true; +} +EXPORT_SYMBOL_GPL(dcp_fabric_presence_sample); + +bool dcp_fabric_presence_expire(struct dcp_fabric_presence *presence, + u64 generation, bool high, unsigned long now) +{ + if (generation != presence->generation || + presence->state != DCP_FABRIC_SETTLING || + time_before(now, presence->deadline)) + return false; + presence->state = high ? DCP_FABRIC_PRESENT : DCP_FABRIC_ABSENT; + presence->deadline = 0; + return !high; +} +EXPORT_SYMBOL_GPL(dcp_fabric_presence_expire); + +bool dcp_fabric_available(const struct dcp_fabric_pipeline *pipeline, + const struct dcp_fabric_policy *policy) +{ + return !pipeline->owned && !pipeline->fixed_busy && + (policy->dual_stream || pipeline->presence == DCP_FABRIC_ABSENT); } EXPORT_SYMBOL_GPL(dcp_fabric_available); @@ -93,7 +131,7 @@ dcp_fabric_free_route(const struct dcp_fabric_port *port, for (route = port->routes; route; route = route->next) { unsigned int score; - if (!dcp_fabric_available(route->pipeline)) + if (!dcp_fabric_available(route->pipeline, policy)) continue; score = dcp_fabric_score(route->pipeline, policy); if (score < best_score) { @@ -101,7 +139,7 @@ dcp_fabric_free_route(const struct dcp_fabric_port *port, best_score = score; } } - if (!last || !dcp_fabric_available(last->pipeline)) + if (!last || !dcp_fabric_available(last->pipeline, policy)) return best; if (!policy->dual_stream && best && best_score < dcp_fabric_score(last->pipeline, policy)) @@ -222,9 +260,7 @@ dcp_fabric_tunnel_candidate(const struct dcp_fabric_port *port, unsigned int score; if ((ordered && route != planned) || - !dcp_fabric_available(pipeline)) - continue; - if (!policy->dual_stream && pipeline->hdmi_held) + !dcp_fabric_available(pipeline, policy)) continue; if (!dcp_fabric_fits(pipeline, policy, port->candidate_crtcs[dpin], diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.h b/drivers/gpu/drm/apple/dcp-fabric-core.h index dfa6ac24847f0..754d481a047c9 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.h +++ b/drivers/gpu/drm/apple/dcp-fabric-core.h @@ -4,6 +4,18 @@ #include +enum dcp_fabric_presence_state { + DCP_FABRIC_ABSENT, + DCP_FABRIC_SETTLING, + DCP_FABRIC_PRESENT, +}; + +struct dcp_fabric_presence { + enum dcp_fabric_presence_state state; + u64 generation; + unsigned long deadline; +}; + struct dcp_fabric_pipeline { struct dcp_fabric_pipeline *next; unsigned int crtc_index; @@ -12,7 +24,7 @@ struct dcp_fabric_pipeline { bool fixed_busy; bool owned; bool tunnel_held; - bool hdmi_held; + enum dcp_fabric_presence_state presence; bool terminal; bool services_ready; bool external; @@ -69,7 +81,15 @@ int dcp_fabric_tunnel_request(const struct dcp_fabric_port *ports, enum dcp_fabric_capacity_action dcp_fabric_capacity_action(bool keep_order); bool dcp_fabric_fixed_busy(bool typec_only, bool independent, bool connected, bool hpd); -bool dcp_fabric_available(const struct dcp_fabric_pipeline *pipeline); +bool dcp_fabric_available(const struct dcp_fabric_pipeline *pipeline, + const struct dcp_fabric_policy *policy); +u64 dcp_fabric_presence_edge(struct dcp_fabric_presence *presence, + unsigned long now, unsigned long window); +bool dcp_fabric_presence_sample(struct dcp_fabric_presence *presence, + u64 generation, bool high, + unsigned long now, unsigned long window); +bool dcp_fabric_presence_expire(struct dcp_fabric_presence *presence, + u64 generation, bool high, unsigned long now); unsigned int dcp_fabric_score(const struct dcp_fabric_pipeline *pipeline, const struct dcp_fabric_policy *policy); bool dcp_fabric_fits(const struct dcp_fabric_pipeline *pipeline, diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c index d78c1af66d007..46147701310bd 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-test.c +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -107,6 +107,88 @@ static void fabric_promote(struct fabric_fixture *f) fabric_direct(f, p); } +/* Both borrowers consume the same hardirq/sample/expiry decisions. */ +static void fabric_presence_scenario(struct kunit *test, + struct fabric_fixture *f, unsigned int id) +{ + struct dcp_fabric_presence presence = {}; + const unsigned long window = 10000; + u64 edge, newer; + unsigned int events = 0; + + f->pipeline[1].owned = true; + edge = dcp_fabric_presence_edge(&presence, 100, window); + KUNIT_EXPECT_EQ(test, presence.deadline, 10100UL); + f->pipeline[0].presence = presence.state; + KUNIT_EXPECT_EQ(test, fabric_tunnel(f, 0, 0), -EBUSY); + KUNIT_EXPECT_EQ(test, fabric_direct(f, 1), -EBUSY); + KUNIT_EXPECT_FALSE(test, + dcp_fabric_presence_expire(&presence, edge, false, 10100 - 1)); + + if (id == 10) { + /* A waking monitor returns before the window ends. */ + newer = dcp_fabric_presence_edge(&presence, 1100, window); + KUNIT_ASSERT_TRUE(test, + dcp_fabric_presence_sample(&presence, newer, true, 1100, window)); + KUNIT_EXPECT_EQ(test, presence.state, DCP_FABRIC_PRESENT); + KUNIT_EXPECT_EQ(test, presence.deadline, 0UL); + KUNIT_EXPECT_FALSE(test, + dcp_fabric_presence_expire(&presence, edge, false, 10100)); + KUNIT_EXPECT_FALSE(test, + dcp_fabric_presence_expire(&presence, newer, false, 11100)); + f->pipeline[0].presence = presence.state; + f->pipeline[0].fixed_busy = true; + KUNIT_EXPECT_EQ(test, fabric_tunnel(f, 0, 0), -EBUSY); + return; + } + if (id == 16) { + /* Resume starts after IRQ enable: low gets the full window. */ + newer = dcp_fabric_presence_edge(&presence, 20100, window); + KUNIT_ASSERT_TRUE(test, + dcp_fabric_presence_sample(&presence, newer, false, + 20200, window)); + KUNIT_EXPECT_EQ(test, presence.deadline, 30200UL); + KUNIT_EXPECT_FALSE(test, + dcp_fabric_presence_expire(&presence, edge, false, 30200)); + /* A high sample clears the edge hold and invalidates its expiry. */ + KUNIT_ASSERT_TRUE(test, + dcp_fabric_presence_sample(&presence, newer, true, + 20300, window)); + KUNIT_EXPECT_EQ(test, presence.state, DCP_FABRIC_PRESENT); + KUNIT_EXPECT_EQ(test, presence.deadline, 0UL); + KUNIT_EXPECT_FALSE(test, + dcp_fabric_presence_expire(&presence, newer, false, 30200)); + /* A falling edge after the sample wins; stale high cannot undo it. */ + edge = dcp_fabric_presence_edge(&presence, 20400, window); + KUNIT_EXPECT_FALSE(test, + dcp_fabric_presence_sample(&presence, newer, true, + 20500, window)); + KUNIT_EXPECT_EQ(test, presence.state, DCP_FABRIC_SETTLING); + KUNIT_EXPECT_EQ(test, presence.deadline, 30400UL); + f->pipeline[0].presence = presence.state; + KUNIT_EXPECT_EQ(test, fabric_tunnel(f, 0, 0), -EBUSY); + KUNIT_EXPECT_EQ(test, fabric_direct(f, 1), -EBUSY); + return; + } + + KUNIT_ASSERT_TRUE(test, + dcp_fabric_presence_expire(&presence, edge, false, 10100)); + events++; + f->pipeline[0].presence = presence.state; + KUNIT_EXPECT_EQ(test, presence.state, DCP_FABRIC_ABSENT); + /* The event is consumed once, even if the expiry callback is repeated. */ + if (dcp_fabric_presence_expire(&presence, edge, false, 10101)) + events++; + KUNIT_EXPECT_EQ(test, events, 1U); + if (id == 11) { + KUNIT_EXPECT_EQ(test, fabric_tunnel(f, 0, 0), 0); + KUNIT_EXPECT_PTR_EQ(test, f->port[0].owner[0], &f->route[0][0]); + } else { + fabric_promote(f); + KUNIT_EXPECT_PTR_EQ(test, f->port[1].owner[0], &f->route[1][0]); + } +} + struct fabric_scenario { const char *name; unsigned int id; @@ -128,7 +210,7 @@ static const struct fabric_scenario scenarios[] = { { "S13_boot_three_outputs", 13 }, { "S14_boot_two_typec", 14 }, { "S15_boot_hdmi_tunnel", 15 }, - { "S16_resume_protection", 16 }, + { "S16_resume_arms_settle", 16 }, { "S17_occupied_slot_errors", 17 }, { "S18_connector_mask", 18 }, { "S20_base_capacity", 20 }, @@ -176,7 +258,7 @@ static void fabric_scenario_test(struct kunit *test) KUNIT_ASSERT_EQ(test, fabric_direct(&f, 0), 0); f.pipeline[0].fixed_busy = true; KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], &f.route[0][1]); - KUNIT_EXPECT_FALSE(test, dcp_fabric_available(&f.pipeline[0])); + KUNIT_EXPECT_FALSE(test, dcp_fabric_available(&f.pipeline[0], &f.policy)); break; case 3: KUNIT_ASSERT_EQ(test, fabric_tunnel(&f, 0, 0), 0); @@ -253,24 +335,10 @@ static void fabric_scenario_test(struct kunit *test) KUNIT_EXPECT_EQ(test, steps[1], DCP_FABRIC_SELECT_MUX); break; case 10: - case 16: - f.pipeline[1].owned = true; - f.pipeline[0].hdmi_held = true; - KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 0), -EBUSY); - f.pipeline[0].fixed_busy = true; - f.pipeline[0].hdmi_held = false; - KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 0), -EBUSY); - break; case 11: - f.pipeline[1].owned = true; - f.pipeline[0].hdmi_held = true; - KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 0), -EBUSY); - f.pipeline[0].hdmi_held = false; - KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 0), 0); - break; case 12: - kunit_skip(test, - "W4a adds direct borrower settling protection"); + case 16: + fabric_presence_scenario(test, &f, scenario->id); break; case 15: f.pipeline[0].fixed_busy = true; @@ -345,7 +413,7 @@ static void fabric_scenario_test(struct kunit *test) KUNIT_EXPECT_PTR_EQ(test, dcp_fabric_free_route(&f.port[0], &f.policy), &f.route[0][1]); - f.pipeline[0].hdmi_held = true; + f.pipeline[0].presence = DCP_FABRIC_SETTLING; KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 1, 0), 0); break; case 35: @@ -468,11 +536,32 @@ static void fabric_unbound_and_mask_test(struct kunit *test) (u32)(BIT(1) | BIT(2))); } +static void fabric_presence_wrap_test(struct kunit *test) +{ + struct dcp_fabric_presence presence = {}; + u64 generation; + + generation = dcp_fabric_presence_edge(&presence, ULONG_MAX - 50, 100); + KUNIT_EXPECT_FALSE(test, + dcp_fabric_presence_expire(&presence, generation, false, ULONG_MAX - 1)); + KUNIT_EXPECT_FALSE(test, + dcp_fabric_presence_expire(&presence, generation, false, 48)); + KUNIT_EXPECT_TRUE(test, + dcp_fabric_presence_expire(&presence, generation, false, 49)); + generation = dcp_fabric_presence_edge(&presence, 100, 100); + /* An expiry which samples high cannot advertise capacity. */ + KUNIT_EXPECT_FALSE(test, + dcp_fabric_presence_expire(&presence, generation, true, 200)); + KUNIT_EXPECT_EQ(test, presence.state, DCP_FABRIC_PRESENT); + KUNIT_EXPECT_EQ(test, presence.deadline, 0UL); +} + static struct kunit_case fabric_tests[] = { KUNIT_CASE_PARAM(fabric_scenario_test, fabric_scenario_gen_params), KUNIT_CASE(fabric_dark_tunnel_test), KUNIT_CASE(fabric_effect_failure_test), KUNIT_CASE(fabric_unbound_and_mask_test), + KUNIT_CASE(fabric_presence_wrap_test), {} }; diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 96700b8f5c399..55fff0b9a64b1 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -69,6 +69,8 @@ static DEFINE_MUTEX(dcp_typec_fabric_lock); static LIST_HEAD(dcp_typec_ports); +static enum dcp_fabric_presence_state dcp_hdmi_presence(struct apple_dcp *dcp); + bool dcp_is_typec_output(struct apple_dcp *dcp) { return dcp->active_typec_route || @@ -128,7 +130,10 @@ static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) .fixed_busy = dcp_typec_route_fixed_output_busy(route), }; - return dcp_fabric_available(&pipeline); + struct dcp_fabric_policy policy = { .dual_stream = dcp_typec_dual_stream() }; + + pipeline.presence = dcp_hdmi_presence(route->dcp); + return dcp_fabric_available(&pipeline, &policy); } /* @@ -467,17 +472,130 @@ static bool dcp_typec_keep_order(void) return dcp_fabric_keep_order(true, true, dcp_typec_frozen(drm)); } -void dcp_hdmi_hold(struct apple_dcp *dcp) +static void dcp_typec_pipeline_freed(void); +static int dcp_fixed_output_select(struct apple_dcp *dcp); + +static enum dcp_fabric_presence_state dcp_hdmi_presence(struct apple_dcp *dcp) { - WRITE_ONCE(dcp->hdmi_hold_until, - jiffies + msecs_to_jiffies(DCP_HDMI_HOLD_MS)); + enum dcp_fabric_presence_state state; + unsigned long flags; + + spin_lock_irqsave(&dcp->hdmi_presence_lock, flags); + state = dcp->hdmi_presence.state; + spin_unlock_irqrestore(&dcp->hdmi_presence_lock, flags); + return state; } -static bool dcp_hdmi_held(struct apple_dcp *dcp) +/* The hardirq records an edge before any borrower can observe HPD low. */ +static u64 dcp_hdmi_edge(struct apple_dcp *dcp) { - unsigned long until = READ_ONCE(dcp->hdmi_hold_until); + unsigned long flags; + u64 generation; + + spin_lock_irqsave(&dcp->hdmi_presence_lock, flags); + generation = dcp_fabric_presence_edge(&dcp->hdmi_presence, jiffies, + msecs_to_jiffies(DCP_HDMI_HOLD_MS)); + spin_unlock_irqrestore(&dcp->hdmi_presence_lock, flags); + return generation; +} + +static void dcp_hdmi_schedule(struct apple_dcp *dcp) +{ + unsigned long flags, delay = 0; + bool settling; + + if (dcp_typec_dual_stream()) + return; + spin_lock_irqsave(&dcp->hdmi_presence_lock, flags); + settling = dcp->hdmi_presence.state == DCP_FABRIC_SETTLING; + if (settling && time_after(dcp->hdmi_presence.deadline, jiffies)) + delay = dcp->hdmi_presence.deadline - jiffies; + spin_unlock_irqrestore(&dcp->hdmi_presence_lock, flags); + if (settling) + mod_delayed_work(system_freezable_wq, &dcp->hdmi_settle_wq, delay); + else + cancel_delayed_work(&dcp->hdmi_settle_wq); +} - return dcp->hdmi_hpd && until && time_before(jiffies, until); +static bool dcp_hdmi_sample(struct apple_dcp *dcp, u64 generation, int level) +{ + unsigned long flags; + bool accepted; + + /* A failed GPIO read cannot establish absence. Keep a full guard. */ + spin_lock_irqsave(&dcp->hdmi_presence_lock, flags); + accepted = dcp_fabric_presence_sample(&dcp->hdmi_presence, generation, + level > 0, jiffies, + msecs_to_jiffies(DCP_HDMI_HOLD_MS)); + spin_unlock_irqrestore(&dcp->hdmi_presence_lock, flags); + dcp_hdmi_schedule(dcp); + return accepted; +} + +static void dcp_hdmi_settle_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), + struct apple_dcp, hdmi_settle_wq); + typeof(apple_tb_dp_capacity_available) *notify; + unsigned long flags; + u64 generation; + bool available; + int level; + + guard(mutex)(&dcp_typec_fabric_lock); + if (!dcp->hdmi_hpd || dcp_typec_dual_stream()) + return; + spin_lock_irqsave(&dcp->hdmi_presence_lock, flags); + generation = dcp->hdmi_presence.generation; + spin_unlock_irqrestore(&dcp->hdmi_presence_lock, flags); + level = gpiod_get_value_cansleep(dcp->hdmi_hpd); + if (level < 0) { + dcp_hdmi_sample(dcp, generation, level); + return; + } + spin_lock_irqsave(&dcp->hdmi_presence_lock, flags); + available = dcp_fabric_presence_expire(&dcp->hdmi_presence, generation, + level > 0, jiffies); + spin_unlock_irqrestore(&dcp->hdmi_presence_lock, flags); + dcp_hdmi_schedule(dcp); + if (!available) + return; + + /* One capacity event, fanned out to direct ports and eligible tunnels. */ + dcp_typec_pipeline_freed(); + notify = symbol_get(apple_tb_dp_capacity_available); + if (!notify) + return; + for (unsigned int i = 0; i < dcp->nr_typec_routes; i++) + notify(dcp->typec_routes[i].port->connector_np); + symbol_put(apple_tb_dp_capacity_available); +} + +void dcp_fabric_init(struct apple_dcp *dcp) +{ + spin_lock_init(&dcp->hdmi_presence_lock); + dcp->hdmi_presence.state = DCP_FABRIC_ABSENT; + INIT_DELAYED_WORK(&dcp->hdmi_settle_wq, dcp_hdmi_settle_work); + disable_delayed_work(&dcp->hdmi_settle_wq); +} + +/* Caller enables the HPD IRQ first; a later edge invalidates this sample. */ +void dcp_fabric_hdmi_resume(struct apple_dcp *dcp) +{ + u64 generation; + int level, ret; + + guard(mutex)(&dcp_typec_fabric_lock); + generation = dcp_hdmi_edge(dcp); + level = gpiod_get_value_cansleep(dcp->hdmi_hpd); + if (!dcp_hdmi_sample(dcp, generation, level) || level <= 0 || + dcp->active_typec_route) + return; + ret = dcp_fixed_output_select(dcp); + if (!ret) + dcp_dptx_connect(dcp, 0); + else + dev_err(dcp->dev, "could not select the HDMI output on resume: %d\n", ret); } /* A Thunderbolt tunnel holds @dcp's pipeline: it never moves. */ @@ -528,7 +646,7 @@ static void dcp_fabric_snapshot_port(struct apple_dcp_typec_port *port, pipeline->owned = !!dcp->active_typec_route || dcp_typec_safe_held(dcp); pipeline->tunnel_held = dcp_typec_tunnel_held(dcp); - pipeline->hdmi_held = tunnel && dcp_hdmi_held(dcp); + pipeline->presence = dcp_hdmi_presence(dcp); pipeline->terminal = tunnel && dcpext_scanout_terminal(dcp); pipeline->services_ready = !tunnel || dcp_tb_services_ready(dcp); @@ -1798,12 +1916,12 @@ static int dcp_fixed_output_select(struct apple_dcp *dcp) /* * Any HDMI HPD edge: a display is there or just was. Start the hold at the - * edge itself, before a tunnel retry can take the fabric lock ahead of the - * thread (see dcp_hdmi_held()). + * edge itself, before a borrower can take the fabric lock ahead of the + * thread. */ irqreturn_t dcp_dp2hdmi_hpd_edge(int irq, void *data) { - dcp_hdmi_hold(data); + dcp_hdmi_edge(data); return IRQ_WAKE_THREAD; } @@ -1812,11 +1930,16 @@ irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) { struct apple_dcp *dcp = data; bool connected; + unsigned long flags; + u64 generation; + int level; guard(mutex)(&dcp_typec_fabric_lock); - - /* again from here, should the edge handler not have run */ - dcp_hdmi_hold(dcp); + spin_lock_irqsave(&dcp->hdmi_presence_lock, flags); + generation = dcp->hdmi_presence.generation; + spin_unlock_irqrestore(&dcp->hdmi_presence_lock, flags); + level = gpiod_get_value_cansleep(dcp->hdmi_hpd); + dcp_hdmi_sample(dcp, generation, level); if (READ_ONCE(dcp->active_typec_route)) { /* diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index 4f55a21ecd5d8..6f051242423eb 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -76,7 +76,8 @@ int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active); /* DCP probe, resume and endpoint teardown hooks. */ int dcp_register_typec_routes(struct apple_dcp *dcp); void dcp_typec_retrain_work(struct work_struct *work); -void dcp_hdmi_hold(struct apple_dcp *dcp); +void dcp_fabric_init(struct apple_dcp *dcp); +void dcp_fabric_hdmi_resume(struct apple_dcp *dcp); irqreturn_t dcp_dp2hdmi_hpd_edge(int irq, void *data); irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data); void dcp_fabric_shutdown_dptx(struct apple_dcp *dcp); @@ -86,4 +87,10 @@ int dcp_dptx_connect(struct apple_dcp *dcp, u32 port); int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port); bool dcp_is_external_h17p(struct apple_dcp *dcp); +/* + * Optional TB capacity wakeup: RCU lookup and queue only, never a mutex/wait. + * Declared here rather than in the 170-usb owned dp-tunnel.h; no provider. + */ +void apple_tb_dp_capacity_available(struct device_node *connector_np); + #endif /* __APPLE_DCP_FABRIC_H__ */ diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 8854f81242c83..8d8c2c79a3ca1 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -531,8 +531,10 @@ struct apple_dcp { */ struct list_head dropped; } iomfb; - /* jiffies until which a Thunderbolt tunnel may not borrow this pipeline */ - unsigned long hdmi_hold_until; + /* Hardirq, resume and expiry share one generation under this lock. */ + spinlock_t hdmi_presence_lock; + struct dcp_fabric_presence hdmi_presence; + struct delayed_work hdmi_settle_wq; }; void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, bool connected); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 90aaf9fe36c70..0763d8c1837ea 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2592,6 +2592,7 @@ VISIBLE_IF_KUNIT void dcp_disable_typec_work(struct apple_dcp *dcp, bool release disable_delayed_work_sync(&dcp->announce_wq); cancel_work_sync(&dcp->external_stop_work); cancel_delayed_work_sync(&dcp->external_stop_retry_wq); + disable_delayed_work_sync(&dcp->hdmi_settle_wq); } VISIBLE_IF_KUNIT void dcp_enable_typec_work(struct apple_dcp *dcp) @@ -2599,6 +2600,7 @@ VISIBLE_IF_KUNIT void dcp_enable_typec_work(struct apple_dcp *dcp) enable_delayed_work(&dcp->typec_reconnect_wq); enable_delayed_work(&dcp->placeholder_edid_wq); enable_delayed_work(&dcp->announce_wq); + enable_delayed_work(&dcp->hdmi_settle_wq); /* A cable can be routed before the DRM component binds. */ if (READ_ONCE(dcp->typec_cable_connected)) mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); @@ -3040,6 +3042,7 @@ static int dcp_platform_probe(struct platform_device *pdev) INIT_WORK(&dcp->external_stop_work, dcp_external_stop_work); INIT_DELAYED_WORK(&dcp->external_stop_retry_wq, dcp_external_stop_retry); mutex_init(&dcp->tb_lock); + dcp_fabric_init(dcp); spin_lock_init(&dcp->mode_state.lock); spin_lock_init(&dcp->metadata.lock); mutex_init(&dcp->dcpavserv.lock); @@ -3504,9 +3507,9 @@ static int dcp_platform_resume(struct device *dev) dcp_release_after_sleep(dcp); dcp_enable_typec_work(dcp); if (dcp->hdmi_hpd_irq) { - /* edges were not seen in sleep, and monitors blink on waking */ - dcp_hdmi_hold(dcp); + /* Observe future edges before sampling any edges lost in sleep. */ enable_irq(dcp->hdmi_hpd_irq); + dcp_fabric_hdmi_resume(dcp); } if (dcp->avep) From 07e5ae8f1c572ce26e5acd6ba644af1380e37656 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 22:42:27 -0700 Subject: [PATCH 1586/2680] drm/apple: preserve fixed-output effects before replanning Deactivate can fail selecting the fixed mux after releasing the direct route and restoring the fixed PHY. The parent rebalance still cleared that owner and attempted the live HDMI connection; the new error replan skipped the connection and could leave the fixed output dark. Return an ordered effect decision that clears the released owner and connects the fixed output before discarding the failed plan. Keep the fresh snapshot and bounded re-entry introduced by the core extraction. Cover the released-route mux failure and the parent success sequence with a fake-effect KUnit test consuming the production decision. Represent resume IRQ enable and HPD sampling as an ordered decision too, so S16 pins IRQ-first sampling as well as low settling, high clearing the hold, stale expiry and later-edge precedence. Leave boot ordering alone. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 042a99c784cd8befe4f4a91de339b7d042d88eac) --- drivers/gpu/drm/apple/dcp-fabric-core.c | 26 +++++++++ drivers/gpu/drm/apple/dcp-fabric-core.h | 16 ++++++ drivers/gpu/drm/apple/dcp-fabric-test.c | 70 +++++++++++++++++++++++++ drivers/gpu/drm/apple/dcp-fabric.c | 32 +++++++---- drivers/gpu/drm/apple/dcp.c | 16 +++++- 5 files changed, 148 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.c b/drivers/gpu/drm/apple/dcp-fabric-core.c index 64234e88f4ed3..e7738ce655ca2 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.c +++ b/drivers/gpu/drm/apple/dcp-fabric-core.c @@ -288,6 +288,32 @@ dcp_fabric_tunnel_candidate(const struct dcp_fabric_port *port, } EXPORT_SYMBOL_GPL(dcp_fabric_tunnel_candidate); +unsigned int dcp_fabric_deactivate_steps(int error, bool selected, bool fixed_live, + enum dcp_fabric_deactivate_step steps[3]) +{ + unsigned int count = 0; + + if (error && selected) { + steps[count++] = DCP_FABRIC_KEEP_OWNER; + } else { + steps[count++] = DCP_FABRIC_CLEAR_OWNER; + if (fixed_live) + steps[count++] = DCP_FABRIC_CONNECT_FIXED; + } + /* Preserve fixed-output effects before discarding the failed plan. */ + if (error) + steps[count++] = DCP_FABRIC_REPLAN; + return count; +} +EXPORT_SYMBOL_GPL(dcp_fabric_deactivate_steps); + +void dcp_fabric_resume_steps(enum dcp_fabric_resume_step steps[2]) +{ + steps[0] = DCP_FABRIC_ENABLE_HPD_IRQ; + steps[1] = DCP_FABRIC_SAMPLE_HPD; +} +EXPORT_SYMBOL_GPL(dcp_fabric_resume_steps); + unsigned int dcp_fabric_fixed_steps(bool has_fixed, bool owned, bool needs_mux, enum dcp_fabric_fixed_step steps[2]) { diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.h b/drivers/gpu/drm/apple/dcp-fabric-core.h index 754d481a047c9..d41aa29561cf6 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.h +++ b/drivers/gpu/drm/apple/dcp-fabric-core.h @@ -70,6 +70,22 @@ enum dcp_fabric_fixed_step { DCP_FABRIC_SELECT_MUX, }; +enum dcp_fabric_deactivate_step { + DCP_FABRIC_KEEP_OWNER, + DCP_FABRIC_CLEAR_OWNER, + DCP_FABRIC_CONNECT_FIXED, + DCP_FABRIC_REPLAN, +}; + +enum dcp_fabric_resume_step { + DCP_FABRIC_ENABLE_HPD_IRQ, + DCP_FABRIC_SAMPLE_HPD, +}; + +unsigned int dcp_fabric_deactivate_steps(int error, bool selected, bool fixed_live, + enum dcp_fabric_deactivate_step steps[3]); +void dcp_fabric_resume_steps(enum dcp_fabric_resume_step steps[2]); + enum dcp_fabric_attach_action { DCP_FABRIC_ATTACH_OOB, DCP_FABRIC_ATTACH_WORK, diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c index 46147701310bd..18b312f6fc97c 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-test.c +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -142,6 +142,24 @@ static void fabric_presence_scenario(struct kunit *test, return; } if (id == 16) { + enum dcp_fabric_resume_step steps[2]; + bool irq_enabled = false, sampled = false; + unsigned int i; + + /* Fake IRQ/GPIO effects consume the production ordering decision. */ + dcp_fabric_resume_steps(steps); + for (i = 0; i < ARRAY_SIZE(steps); i++) { + switch (steps[i]) { + case DCP_FABRIC_ENABLE_HPD_IRQ: + irq_enabled = true; + break; + case DCP_FABRIC_SAMPLE_HPD: + KUNIT_EXPECT_TRUE(test, irq_enabled); + sampled = true; + break; + } + } + KUNIT_EXPECT_TRUE(test, sampled); /* Resume starts after IRQ enable: low gets the full window. */ newer = dcp_fabric_presence_edge(&presence, 20100, window); KUNIT_ASSERT_TRUE(test, @@ -511,6 +529,57 @@ static void fabric_effect_failure_test(struct kunit *test) KUNIT_EXPECT_PTR_EQ(test, f.port[1].owner[0], &f.route[1][1]); } +static void fabric_deactivate_failure_test(struct kunit *test) +{ + struct fabric_fixture f; + enum dcp_fabric_deactivate_step steps[3]; + unsigned int count, i, connects = 0, replans = 0; + bool selected = true; + + fabric_init(&f, true); + f.port[0].wanted = true; + f.port[0].hpd = true; + f.port[0].owner[0] = &f.route[0][0]; + f.pipeline[0].owned = true; + dcp_fabric_plan(f.pipeline, f.port, NULL, 0); + /* Fake deactivate releases the route, restores PHY, then mux fails. */ + selected = false; + f.pipeline[0].owned = false; + f.pipeline[0].fixed_busy = true; + count = dcp_fabric_deactivate_steps(-EIO, selected, true, steps); + KUNIT_ASSERT_EQ(test, count, 3U); + KUNIT_EXPECT_EQ(test, steps[0], DCP_FABRIC_CLEAR_OWNER); + KUNIT_EXPECT_EQ(test, steps[1], DCP_FABRIC_CONNECT_FIXED); + KUNIT_EXPECT_EQ(test, steps[2], DCP_FABRIC_REPLAN); + for (i = 0; i < count; i++) { + switch (steps[i]) { + case DCP_FABRIC_KEEP_OWNER: + KUNIT_FAIL(test, "released route cannot retain ownership"); + break; + case DCP_FABRIC_CLEAR_OWNER: + f.port[0].owner[0] = NULL; + break; + case DCP_FABRIC_CONNECT_FIXED: + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], NULL); + connects++; + break; + case DCP_FABRIC_REPLAN: + KUNIT_EXPECT_EQ(test, connects, 1U); + dcp_fabric_plan(f.pipeline, f.port, NULL, 0); + replans++; + break; + } + } + KUNIT_EXPECT_EQ(test, connects, 1U); + KUNIT_EXPECT_EQ(test, replans, 1U); + KUNIT_EXPECT_PTR_EQ(test, f.plan[0].target[0], &f.route[0][1]); + /* Success uses the same parent clear/connect sequence, without replan. */ + count = dcp_fabric_deactivate_steps(0, false, true, steps); + KUNIT_EXPECT_EQ(test, count, 2U); + KUNIT_EXPECT_EQ(test, steps[0], DCP_FABRIC_CLEAR_OWNER); + KUNIT_EXPECT_EQ(test, steps[1], DCP_FABRIC_CONNECT_FIXED); +} + static void fabric_unbound_and_mask_test(struct kunit *test) { struct fabric_fixture f; @@ -560,6 +629,7 @@ static struct kunit_case fabric_tests[] = { KUNIT_CASE_PARAM(fabric_scenario_test, fabric_scenario_gen_params), KUNIT_CASE(fabric_dark_tunnel_test), KUNIT_CASE(fabric_effect_failure_test), + KUNIT_CASE(fabric_deactivate_failure_test), KUNIT_CASE(fabric_unbound_and_mask_test), KUNIT_CASE(fabric_presence_wrap_test), {} diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 55fff0b9a64b1..ebd573513c6df 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -828,7 +828,11 @@ dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, list_for_each_entry(port, &dcp_typec_ports, link) { struct apple_dcp_typec_route *owner = port->owner; + enum dcp_fabric_deactivate_step steps[3]; struct apple_dcp *dcp; + unsigned int count, i; + bool fixed_live; + int ret; /* A port in SAFE keeps its owner until the set phase completes. */ if (port->safe_pending) @@ -846,19 +850,27 @@ dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, (dcp->typec_connector && dcp->typec_connector->connected)) dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); port->hpd = false; - if (dcp_typec_route_deactivate(owner)) { - if (owner->selected) + ret = dcp_typec_route_deactivate(owner); + /* Deactivate's fixed-mux error has already released the route. */ + fixed_live = !(ret && owner->selected) && dcp->hdmi_hpd && + dcp->active && gpiod_get_value_cansleep(dcp->hdmi_hpd); + count = dcp_fabric_deactivate_steps(ret, owner->selected, + fixed_live, steps); + for (i = 0; i < count; i++) { + switch (steps[i]) { + case DCP_FABRIC_KEEP_OWNER: dcp_typec_port_attach(port); - else + break; + case DCP_FABRIC_CLEAR_OWNER: port->owner = NULL; - goto replan; + break; + case DCP_FABRIC_CONNECT_FIXED: + dcp_dptx_connect(dcp, 0); + break; + case DCP_FABRIC_REPLAN: + goto replan; + } } - port->owner = NULL; - - /* as after a DP exit, hand the hybrid back to a live HDMI */ - if (dcp->hdmi_hpd && dcp->active && - gpiod_get_value_cansleep(dcp->hdmi_hpd)) - dcp_dptx_connect(dcp, 0); } list_for_each_entry(port, &dcp_typec_ports, link) { diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 0763d8c1837ea..aa1e93ea198bf 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3507,9 +3507,21 @@ static int dcp_platform_resume(struct device *dev) dcp_release_after_sleep(dcp); dcp_enable_typec_work(dcp); if (dcp->hdmi_hpd_irq) { + enum dcp_fabric_resume_step steps[2]; + unsigned int i; + /* Observe future edges before sampling any edges lost in sleep. */ - enable_irq(dcp->hdmi_hpd_irq); - dcp_fabric_hdmi_resume(dcp); + dcp_fabric_resume_steps(steps); + for (i = 0; i < ARRAY_SIZE(steps); i++) { + switch (steps[i]) { + case DCP_FABRIC_ENABLE_HPD_IRQ: + enable_irq(dcp->hdmi_hpd_irq); + break; + case DCP_FABRIC_SAMPLE_HPD: + dcp_fabric_hdmi_resume(dcp); + break; + } + } } if (dcp->avep) From 37134d1600b08e1ecb52263b7f0e0479eaeddbf3 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 22:53:19 -0700 Subject: [PATCH 1587/2680] drm/apple: recheck HDMI after the oneshot handler unmasks HPD The Apple GPIO IRQ mask turns off edge detection. A fall during the thread's debounce or firmware waits can be lost after an earlier high sample committed PRESENT, leaving the hybrid unavailable indefinitely. Queue a freezable recheck on both thread return paths. Synchronize the IRQ before taking the fabric lock, so oneshot finalization has unmasked the line before sampling. Synthesize a new edge when the observed level differs from presence, rejecting stale generations. A lost fall starts a full settling window, while a lost rise clears it. Matching low does not extend the deadline. Recheck again after any resulting HDMI update. Drain and disable the work with the other endpoint users. Do not wait for IRQ completion under the fabric lock or change dual-stream policy. Cover a fall lost while masked, stale high rejection, repeated low, absence promotion and a lost rise with fake IRQ/GPIO effects in KUnit. An ignore-low-recheck mutation fails that test. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 8974284a5d01db80414653ba4d261511b0794a6b) --- drivers/gpu/drm/apple/dcp-fabric-core.c | 15 +++++++ drivers/gpu/drm/apple/dcp-fabric-core.h | 3 ++ drivers/gpu/drm/apple/dcp-fabric-test.c | 49 ++++++++++++++++++++++ drivers/gpu/drm/apple/dcp-fabric.c | 55 +++++++++++++++++++++++-- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 2 + 6 files changed, 121 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.c b/drivers/gpu/drm/apple/dcp-fabric-core.c index e7738ce655ca2..b5d4690acdbad 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.c +++ b/drivers/gpu/drm/apple/dcp-fabric-core.c @@ -62,6 +62,21 @@ bool dcp_fabric_presence_sample(struct dcp_fabric_presence *presence, } EXPORT_SYMBOL_GPL(dcp_fabric_presence_sample); +bool dcp_fabric_presence_recheck(struct dcp_fabric_presence *presence, + u64 generation, bool high, + unsigned long now, unsigned long window) +{ + bool present = presence->state == DCP_FABRIC_PRESENT; + + if (generation != presence->generation || high == present) + return false; + /* Synthesize an edge which the masked GPIO detector could not record. */ + generation = dcp_fabric_presence_edge(presence, now, window); + dcp_fabric_presence_sample(presence, generation, high, now, window); + return true; +} +EXPORT_SYMBOL_GPL(dcp_fabric_presence_recheck); + bool dcp_fabric_presence_expire(struct dcp_fabric_presence *presence, u64 generation, bool high, unsigned long now) { diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.h b/drivers/gpu/drm/apple/dcp-fabric-core.h index d41aa29561cf6..2ec248d872853 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.h +++ b/drivers/gpu/drm/apple/dcp-fabric-core.h @@ -104,6 +104,9 @@ u64 dcp_fabric_presence_edge(struct dcp_fabric_presence *presence, bool dcp_fabric_presence_sample(struct dcp_fabric_presence *presence, u64 generation, bool high, unsigned long now, unsigned long window); +bool dcp_fabric_presence_recheck(struct dcp_fabric_presence *presence, + u64 generation, bool high, + unsigned long now, unsigned long window); bool dcp_fabric_presence_expire(struct dcp_fabric_presence *presence, u64 generation, bool high, unsigned long now); unsigned int dcp_fabric_score(const struct dcp_fabric_pipeline *pipeline, diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c index 18b312f6fc97c..7c09c2d7647a7 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-test.c +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -625,6 +625,54 @@ static void fabric_presence_wrap_test(struct kunit *test) KUNIT_EXPECT_EQ(test, presence.deadline, 0UL); } +static void fabric_masked_edge_test(struct kunit *test) +{ + struct fabric_fixture f; + struct dcp_fabric_presence presence = {}; + u64 generation; + bool irq_masked = true, gpio_high = true; + + fabric_init(&f, false); + f.pipeline[1].owned = true; + generation = dcp_fabric_presence_edge(&presence, 100, 10000); + KUNIT_ASSERT_TRUE(test, + dcp_fabric_presence_sample(&presence, generation, gpio_high, 100, 10000)); + KUNIT_ASSERT_EQ(test, presence.state, DCP_FABRIC_PRESENT); + /* Fake oneshot debounce: fall with detector off, no hardirq generation. */ + gpio_high = false; + KUNIT_EXPECT_TRUE(test, irq_masked); + KUNIT_EXPECT_EQ(test, presence.generation, generation); + /* IRQ completion unmasks before the queued synchronized recheck. */ + irq_masked = false; + KUNIT_EXPECT_FALSE(test, irq_masked); + KUNIT_ASSERT_TRUE(test, + dcp_fabric_presence_recheck(&presence, generation, gpio_high, + 600, 10000)); + KUNIT_EXPECT_EQ(test, presence.state, DCP_FABRIC_SETTLING); + KUNIT_EXPECT_EQ(test, presence.deadline, 10600UL); + KUNIT_EXPECT_NE(test, presence.generation, generation); + f.pipeline[0].presence = presence.state; + KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 0), -EBUSY); + KUNIT_EXPECT_EQ(test, fabric_direct(&f, 1), -EBUSY); + KUNIT_EXPECT_FALSE(test, + dcp_fabric_presence_recheck(&presence, generation, true, 601, 10000)); + generation = presence.generation; + KUNIT_EXPECT_FALSE(test, + dcp_fabric_presence_recheck(&presence, generation, false, 602, 10000)); + KUNIT_EXPECT_EQ(test, presence.deadline, 10600UL); + KUNIT_EXPECT_TRUE(test, + dcp_fabric_presence_expire(&presence, generation, false, 10600)); + f.pipeline[0].presence = presence.state; + fabric_promote(&f); + KUNIT_EXPECT_PTR_EQ(test, f.port[1].owner[0], &f.route[1][0]); + /* A rise lost during a low handler clears the hold on post-unmask read. */ + generation = dcp_fabric_presence_edge(&presence, 20000, 10000); + KUNIT_EXPECT_TRUE(test, + dcp_fabric_presence_recheck(&presence, generation, true, 20500, 10000)); + KUNIT_EXPECT_EQ(test, presence.state, DCP_FABRIC_PRESENT); + KUNIT_EXPECT_EQ(test, presence.deadline, 0UL); +} + static struct kunit_case fabric_tests[] = { KUNIT_CASE_PARAM(fabric_scenario_test, fabric_scenario_gen_params), KUNIT_CASE(fabric_dark_tunnel_test), @@ -632,6 +680,7 @@ static struct kunit_case fabric_tests[] = { KUNIT_CASE(fabric_deactivate_failure_test), KUNIT_CASE(fabric_unbound_and_mask_test), KUNIT_CASE(fabric_presence_wrap_test), + KUNIT_CASE(fabric_masked_edge_test), {} }; diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index ebd573513c6df..7528a66e88940 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -70,6 +70,7 @@ static DEFINE_MUTEX(dcp_typec_fabric_lock); static LIST_HEAD(dcp_typec_ports); static enum dcp_fabric_presence_state dcp_hdmi_presence(struct apple_dcp *dcp); +static void dcp_hdmi_update_locked(struct apple_dcp *dcp); bool dcp_is_typec_output(struct apple_dcp *dcp) { @@ -571,12 +572,49 @@ static void dcp_hdmi_settle_work(struct work_struct *work) symbol_put(apple_tb_dp_capacity_available); } +static void dcp_hdmi_recheck_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), + struct apple_dcp, hdmi_recheck_wq); + unsigned long flags; + u64 generation; + bool changed; + int level; + + /* Oneshot unmask precedes thread completion. Never wait under fabric. */ + synchronize_irq(dcp->hdmi_hpd_irq); + guard(mutex)(&dcp_typec_fabric_lock); + if (dcp_typec_dual_stream()) + return; + spin_lock_irqsave(&dcp->hdmi_presence_lock, flags); + generation = dcp->hdmi_presence.generation; + spin_unlock_irqrestore(&dcp->hdmi_presence_lock, flags); + level = gpiod_get_value_cansleep(dcp->hdmi_hpd); + if (level < 0) { + dcp_hdmi_sample(dcp, generation, level); + return; + } + spin_lock_irqsave(&dcp->hdmi_presence_lock, flags); + changed = dcp_fabric_presence_recheck(&dcp->hdmi_presence, generation, + level > 0, jiffies, + msecs_to_jiffies(DCP_HDMI_HOLD_MS)); + spin_unlock_irqrestore(&dcp->hdmi_presence_lock, flags); + if (!changed) + return; + dcp_hdmi_schedule(dcp); + dcp_hdmi_update_locked(dcp); + /* Catch a level change during this update's firmware/debounce waits. */ + mod_delayed_work(system_freezable_wq, &dcp->hdmi_recheck_wq, 0); +} + void dcp_fabric_init(struct apple_dcp *dcp) { spin_lock_init(&dcp->hdmi_presence_lock); dcp->hdmi_presence.state = DCP_FABRIC_ABSENT; INIT_DELAYED_WORK(&dcp->hdmi_settle_wq, dcp_hdmi_settle_work); + INIT_DELAYED_WORK(&dcp->hdmi_recheck_wq, dcp_hdmi_recheck_work); disable_delayed_work(&dcp->hdmi_settle_wq); + disable_delayed_work(&dcp->hdmi_recheck_wq); } /* Caller enables the HPD IRQ first; a later edge invalidates this sample. */ @@ -1938,15 +1976,14 @@ irqreturn_t dcp_dp2hdmi_hpd_edge(int irq, void *data) return IRQ_WAKE_THREAD; } -irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) +static void dcp_hdmi_update_locked(struct apple_dcp *dcp) { - struct apple_dcp *dcp = data; bool connected; unsigned long flags; u64 generation; int level; - guard(mutex)(&dcp_typec_fabric_lock); + lockdep_assert_held(&dcp_typec_fabric_lock); spin_lock_irqsave(&dcp->hdmi_presence_lock, flags); generation = dcp->hdmi_presence.generation; spin_unlock_irqrestore(&dcp->hdmi_presence_lock, flags); @@ -1969,7 +2006,7 @@ irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) gpiod_get_value_cansleep(dcp->hdmi_hpd)) { dcp_typec_hdmi_waits(dcp); } - return IRQ_HANDLED; + return; } connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); @@ -1995,7 +2032,17 @@ irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) else dcp_dptx_connect(dcp, 0); } +} +irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) +{ + struct apple_dcp *dcp = data; + + guard(mutex)(&dcp_typec_fabric_lock); + dcp_hdmi_update_locked(dcp); + /* The GPIO edge detector is off throughout an oneshot handler. */ + if (!dcp_typec_dual_stream()) + mod_delayed_work(system_freezable_wq, &dcp->hdmi_recheck_wq, 0); return IRQ_HANDLED; } diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 8d8c2c79a3ca1..1a2024148d2c6 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -535,6 +535,7 @@ struct apple_dcp { spinlock_t hdmi_presence_lock; struct dcp_fabric_presence hdmi_presence; struct delayed_work hdmi_settle_wq; + struct delayed_work hdmi_recheck_wq; }; void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, bool connected); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index aa1e93ea198bf..77047618123b6 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2593,6 +2593,7 @@ VISIBLE_IF_KUNIT void dcp_disable_typec_work(struct apple_dcp *dcp, bool release cancel_work_sync(&dcp->external_stop_work); cancel_delayed_work_sync(&dcp->external_stop_retry_wq); disable_delayed_work_sync(&dcp->hdmi_settle_wq); + disable_delayed_work_sync(&dcp->hdmi_recheck_wq); } VISIBLE_IF_KUNIT void dcp_enable_typec_work(struct apple_dcp *dcp) @@ -2601,6 +2602,7 @@ VISIBLE_IF_KUNIT void dcp_enable_typec_work(struct apple_dcp *dcp) enable_delayed_work(&dcp->placeholder_edid_wq); enable_delayed_work(&dcp->announce_wq); enable_delayed_work(&dcp->hdmi_settle_wq); + enable_delayed_work(&dcp->hdmi_recheck_wq); /* A cable can be routed before the DRM component binds. */ if (READ_ONCE(dcp->typec_cable_connected)) mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); From 4651ab4212b3728d133cc1e45c1aa1a7bbb68137 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 22:56:44 -0700 Subject: [PATCH 1588/2680] dt-bindings: display: describe Apple DCP Type-C routing Describe the DCP route graph, its per-route PHY and crossbar target arrays, and the named DP Alt Mode and DP IN mux controls. Two graph candidates alone do not describe dual-stream tunnel wiring; both DP IN outputs must be wired too. Keep legacy typecN-usb4 names documented for old j414/j416 boot DTBs, and name both outputs in new device trees. Validate routing as a common schema independently of the coprocessor's other interfaces. Close the route container and route nodes while leaving mailbox, panel and firmware properties to their own schemas. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 451d8062338295ebbf2c09077384143e30be79e2) --- .../display/apple,dcp-typec-routes.yaml | 171 ++++++++++++++++++ MAINTAINERS | 1 + 2 files changed, 172 insertions(+) create mode 100644 Documentation/devicetree/bindings/display/apple,dcp-typec-routes.yaml diff --git a/Documentation/devicetree/bindings/display/apple,dcp-typec-routes.yaml b/Documentation/devicetree/bindings/display/apple,dcp-typec-routes.yaml new file mode 100644 index 0000000000000..7b2e28ac3dca7 --- /dev/null +++ b/Documentation/devicetree/bindings/display/apple,dcp-typec-routes.yaml @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/apple,dcp-typec-routes.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple DCP Type-C display routing properties + +maintainers: + - Janne Grunau + +description: | + Common routing properties of an Apple display coprocessor (DCP). This schema + describes the connections to ATC PHYs and their display crossbars, rather + than the coprocessor's mailbox, memory, panel or firmware interfaces. + + Each route index N names a typecN PHY and a typecN DP Alt Mode crossbar output + on the DCP node. The route's graph endpoint connects to the corresponding + Type-C connector's display port (port@3). Two route endpoints on that port + mean two candidate pipelines; that alone does not imply two tunnel streams. + + For a port wired to both Thunderbolt DP IN adapters, name typecN-dpin0 and + typecN-dpin1 crossbar controls on the pipelines that can drive them. Crossbar + mux cells 0, 1 and 2 describe the DP PHY, DP IN 0 and DP IN 1 outputs, + respectively. The per-port prefix distinguishes multiple ATCs on one DCP. + Dual-stream wiring requires these two named controls and two port@3 + endpoints. It does not describe SoC handshake or retry policy. + + Legacy typecN-usb4 names describe cell 1 on old j414/j416 DTBs. With two + port@3 endpoints they retain the existing dual-stream wiring; cell 2 is + implicit in that legacy ABI. New DTs must name both DP IN controls instead. + +select: + properties: + compatible: + contains: + enum: + - apple,dcp + - apple,dcpext + typec-routes: true + required: + - compatible + - typec-routes + +properties: + phys: + minItems: 1 + maxItems: 5 + description: Fixed-output DP PHY followed by the wired Type-C DP PHYs. + + phy-names: + minItems: 1 + maxItems: 5 + items: + enum: [dp-phy, typec0, typec1, typec2, typec3] + + mux-controls: + minItems: 1 + maxItems: 13 + description: Fixed, DP Alt Mode and wired DP IN crossbar output controls. + + mux-control-names: + minItems: 1 + maxItems: 13 + items: + anyOf: + - enum: + - dp-xbar + - typec0 + - typec1 + - typec2 + - typec3 + - typec0-dpin0 + - typec0-dpin1 + - typec1-dpin0 + - typec1-dpin1 + - typec2-dpin0 + - typec2-dpin1 + - typec3-dpin0 + - typec3-dpin1 + - enum: [typec0-usb4, typec1-usb4, typec2-usb4, typec3-usb4] + deprecated: true + + apple,typec-mux-indices: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 4 + description: | + Crossbar source state for each route index, indexed by route reg. + This selects the pipeline input, not the DP IN adapter number. + + apple,typec-dptx-phys: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 4 + description: DPTX PHY target for each route index, indexed by route reg. + + typec-routes: + type: object + additionalProperties: false + properties: + '#address-cells': + const: 1 + '#size-cells': + const: 0 + patternProperties: + '^route@[0-3]$': + type: object + additionalProperties: false + properties: + reg: + items: + - maximum: 3 + mode-switch: + type: boolean + port: + $ref: /schemas/graph.yaml#/properties/port + required: + - reg + - mode-switch + - port + required: + - '#address-cells' + - '#size-cells' + anyOf: + - required: [route@0] + - required: [route@1] + - required: [route@2] + - required: [route@3] + +required: + - phys + - phy-names + - mux-controls + - mux-control-names + - apple,typec-mux-indices + - apple,typec-dptx-phys + - typec-routes + +# This common schema validates routing independently of other DCP interfaces. +additionalProperties: true + +examples: + - | + #include + + dcp { + compatible = "apple,t6020-dcpext", "apple,dcpext"; + phys = <&atcphy0 PHY_TYPE_DP>; + phy-names = "typec0"; + mux-controls = <&atcphy0_xbar 0>, <&atcphy0_xbar 1>, + <&atcphy0_xbar 2>; + mux-control-names = "typec0", "typec0-dpin0", "typec0-dpin1"; + apple,typec-mux-indices = <2>; + apple,typec-dptx-phys = <0>; + + typec-routes { + #address-cells = <1>; + #size-cells = <0>; + + route@0 { + reg = <0>; + mode-switch; + port { + endpoint { + remote-endpoint = <&typec0_display>; + }; + }; + }; + }; + }; diff --git a/MAINTAINERS b/MAINTAINERS index 950665486a5e8..ccc25f5d50576 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2009,6 +2009,7 @@ M: Janne Grunau L: dri-devel@lists.freedesktop.org S: Maintained T: git git://anongit.freedesktop.org/drm/drm-misc +F: Documentation/devicetree/bindings/display/apple,dcp-typec-routes.yaml F: drivers/gpu/drm/apple/ F: drivers/gpu/drm/apple/dcp-fabric* From 1ade25c5a1c303a4916887ac992a6c77368fa57d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 23:00:39 -0700 Subject: [PATCH 1589/2680] drm/apple: derive tunnel stream wiring from the device tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two graph candidates on j314 describe alternative pipelines, not independent dock streams. Derive dual-stream capacity from two display endpoints plus named per-port DPIN0/DPIN1 controls. Add those controls only to the four j414/j416 device trees and use them at tunnel activation. Probe obtains and validates mux references without selecting them. Preserve the old ESP DTB ABI: typecN-usb4 with two display endpoints retains dual-stream and the implicit cell-2 control, with one warning. Use no board-compatible fallback. Track registered dual-capable routes for the existing global fabric policy, including probe failure cleanup. Split DCP tunnel flow qualification from wiring: T6020 match data plus a wired tunnel route enables its existing timeouts/reconnect/retrain rules, with the USB4-output guard retained. Desktop and T6030 paths do not gain that flow. Drop the DCP board-helper calls; leave the shared helper for the other modules until their route gates are coordinated. Cover old/new laptop DTs, j314, base chips, desktops, M3, malformed pairs and flow guards in KUnit. Legacy, endpoints-only and SoC-gate mutations fail the expected rows. All 113 DTBs validate; 109 are byte-identical, and the four intended differences are only DCP mux controls and names. [ryan: adapted to 7.2.10 — the t602x-j414-j416.dtsi hunk (DP IN wiring for the M2 Pro/Max laptops) belongs to 000-devicetree and is left out here (patch handed over); the T6030 DP IN helper stays out, as in the native 14.7 path; the T6020 tunnel-flow flag sits in apple_dcp_hw_data] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 5751776ca8416a0e6e65ca974cd9f909a773fcb6) --- drivers/gpu/drm/apple/dcp-fabric-core.c | 19 +++++ drivers/gpu/drm/apple/dcp-fabric-core.h | 11 +++ drivers/gpu/drm/apple/dcp-fabric-test.c | 56 ++++++++++++ drivers/gpu/drm/apple/dcp-fabric.c | 108 ++++++++++++++++++++++-- drivers/gpu/drm/apple/dcp-fabric.h | 3 + drivers/gpu/drm/apple/dcp-internal.h | 2 + drivers/gpu/drm/apple/dcp.c | 13 ++- drivers/gpu/drm/apple/iomfb.c | 2 +- 8 files changed, 202 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.c b/drivers/gpu/drm/apple/dcp-fabric-core.c index b5d4690acdbad..e2bbf333db973 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.c +++ b/drivers/gpu/drm/apple/dcp-fabric-core.c @@ -8,6 +8,25 @@ #include "dcp-fabric-core.h" +enum dcp_fabric_wiring dcp_fabric_wiring(bool dpin0, bool dpin1, bool legacy, + unsigned int endpoints) +{ + if (dpin0 != dpin1) + return DCP_FABRIC_INVALID_WIRING; + if (endpoints < 2) + return DCP_FABRIC_SINGLE_STREAM; + if (dpin0 && dpin1) + return DCP_FABRIC_DUAL_NAMED; + return legacy ? DCP_FABRIC_DUAL_LEGACY : DCP_FABRIC_SINGLE_STREAM; +} +EXPORT_SYMBOL_GPL(dcp_fabric_wiring); + +bool dcp_fabric_t6020_flow(bool usb4, bool soc_support, bool route_wired) +{ + return usb4 && soc_support && route_wired; +} +EXPORT_SYMBOL_GPL(dcp_fabric_t6020_flow); + int dcp_fabric_tunnel_request(const struct dcp_fabric_port *ports, unsigned long key, unsigned int dpin, const struct dcp_fabric_port **found) diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.h b/drivers/gpu/drm/apple/dcp-fabric-core.h index 2ec248d872853..04609936964e6 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.h +++ b/drivers/gpu/drm/apple/dcp-fabric-core.h @@ -60,6 +60,17 @@ struct dcp_fabric_policy { bool dual_stream; }; +enum dcp_fabric_wiring { + DCP_FABRIC_SINGLE_STREAM, + DCP_FABRIC_DUAL_NAMED, + DCP_FABRIC_DUAL_LEGACY, + DCP_FABRIC_INVALID_WIRING, +}; + +enum dcp_fabric_wiring dcp_fabric_wiring(bool dpin0, bool dpin1, bool legacy, + unsigned int endpoints); +bool dcp_fabric_t6020_flow(bool usb4, bool soc_support, bool route_wired); + enum dcp_fabric_capacity_action { DCP_FABRIC_PROMOTE, DCP_FABRIC_REBALANCE, diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c index 7c09c2d7647a7..664b3495ea9b3 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-test.c +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -673,6 +673,61 @@ static void fabric_masked_edge_test(struct kunit *test) KUNIT_EXPECT_EQ(test, presence.deadline, 0UL); } +struct fabric_wiring_case { + const char *name; + bool dpin0; + bool dpin1; + bool legacy; + unsigned int endpoints; + bool soc_support; + enum dcp_fabric_wiring expected; + bool flow; +}; + +static const struct fabric_wiring_case wiring_cases[] = { + { "M1_base", false, false, false, 1, false, DCP_FABRIC_SINGLE_STREAM, false }, + { "M1_pro_j314_two_candidates", false, false, false, 2, false, + DCP_FABRIC_SINGLE_STREAM, false }, + { "M1_ultra", false, false, false, 1, false, DCP_FABRIC_SINGLE_STREAM, false }, + { "M2_base", false, false, false, 1, false, DCP_FABRIC_SINGLE_STREAM, false }, + { "M2_desktop", false, false, false, 1, true, DCP_FABRIC_SINGLE_STREAM, false }, + { "M2_j414_new_dtb", true, true, false, 2, true, DCP_FABRIC_DUAL_NAMED, true }, + { "M2_j416_new_dtb", true, true, false, 2, true, DCP_FABRIC_DUAL_NAMED, true }, + { "M2_j414_old_esp_dtb", false, false, true, 2, true, DCP_FABRIC_DUAL_LEGACY, true }, + { "M2_j416_old_esp_dtb", false, false, true, 2, true, DCP_FABRIC_DUAL_LEGACY, true }, + { "M3_shared_crossbar_single", false, false, false, 1, false, + DCP_FABRIC_SINGLE_STREAM, false }, + { "named_wins_over_legacy", true, true, true, 2, true, DCP_FABRIC_DUAL_NAMED, true }, + { "dpin0_only_invalid", true, false, false, 2, true, DCP_FABRIC_INVALID_WIRING, false }, + { "dpin1_only_invalid", false, true, true, 2, true, DCP_FABRIC_INVALID_WIRING, false }, + { "no_graph", true, true, false, 0, true, DCP_FABRIC_SINGLE_STREAM, false }, + { "one_candidate_named", true, true, false, 1, true, DCP_FABRIC_SINGLE_STREAM, false }, + { "one_candidate_legacy", false, false, true, 1, true, DCP_FABRIC_SINGLE_STREAM, false }, + { "future_wiring_cannot_enable_soc_flow", true, true, false, 2, false, + DCP_FABRIC_DUAL_NAMED, false }, +}; + +static void fabric_wiring_desc(const struct fabric_wiring_case *row, char *desc) +{ + strscpy(desc, row->name, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(fabric_wiring, wiring_cases, fabric_wiring_desc); + +static void fabric_wiring_test(struct kunit *test) +{ + const struct fabric_wiring_case *row = test->param_value; + enum dcp_fabric_wiring wiring; + bool dual; + + wiring = dcp_fabric_wiring(row->dpin0, row->dpin1, row->legacy, row->endpoints); + KUNIT_EXPECT_EQ(test, wiring, row->expected); + dual = wiring == DCP_FABRIC_DUAL_NAMED || wiring == DCP_FABRIC_DUAL_LEGACY; + KUNIT_EXPECT_EQ(test, dcp_fabric_t6020_flow(true, row->soc_support, dual), row->flow); + /* SoC and wiring qualification must never change HDMI or DP-alt flow. */ + KUNIT_EXPECT_FALSE(test, dcp_fabric_t6020_flow(false, row->soc_support, dual)); +} + static struct kunit_case fabric_tests[] = { KUNIT_CASE_PARAM(fabric_scenario_test, fabric_scenario_gen_params), KUNIT_CASE(fabric_dark_tunnel_test), @@ -681,6 +736,7 @@ static struct kunit_case fabric_tests[] = { KUNIT_CASE(fabric_unbound_and_mask_test), KUNIT_CASE(fabric_presence_wrap_test), KUNIT_CASE(fabric_masked_edge_test), + KUNIT_CASE_PARAM(fabric_wiring_test, fabric_wiring_gen_params), {} }; diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 7528a66e88940..a09cb272ed76f 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright 2021 Alyssa Rosenzweig */ +#include #include #include #include @@ -68,6 +69,8 @@ struct apple_dcp_typec_port { static DEFINE_MUTEX(dcp_typec_fabric_lock); static LIST_HEAD(dcp_typec_ports); +/* Current DTs wire the same stream capacity on every physical port. */ +static atomic_t dcp_dual_stream_routes = ATOMIC_INIT(0); static enum dcp_fabric_presence_state dcp_hdmi_presence(struct apple_dcp *dcp); static void dcp_hdmi_update_locked(struct apple_dcp *dcp); @@ -83,6 +86,16 @@ bool dcp_is_usb4_output(struct apple_dcp *dcp) return dcp->active_typec_route && dcp->active_typec_route->tunnel; } +/* SoC flow support and board wiring are separate qualifications. */ +bool dcp_uses_t6020_tunnel_flow(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route = READ_ONCE(dcp->active_typec_route); + + return dcp_fabric_t6020_flow(dcp_is_usb4_output(dcp), + dcp->hw.t6020_tunnel_flow, + route && route->dual_stream); +} + static bool dcp_typec_route_is_dp(const struct typec_mux_state *state) { return state->alt && state->alt->svid == USB_TYPEC_DP_SID && @@ -144,7 +157,7 @@ static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) */ bool dcp_typec_dual_stream(void) { - return apple_dp_tunnel_t602x(); + return atomic_read(&dcp_dual_stream_routes) > 0; } bool dcp_is_typec_only(struct platform_device *pdev) @@ -1564,13 +1577,16 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, goto err_reorder; } - /* The route's crossbar control is dpphy (0); dpin0/dpin1 are 1/2. */ - if (best->xbar != &best->xbar->chip->mux[0] || - best->xbar->chip->controllers < 3) { - ret = -EOPNOTSUPP; - goto err_reorder; + ctl = best->dpin[dpin]; + if (!ctl) { + /* Legacy DT ABI: unnamed DPIN controls share the DP-alt chip. */ + if (best->xbar != &best->xbar->chip->mux[0] || + best->xbar->chip->controllers < 3) { + ret = -EOPNOTSUPP; + goto err_reorder; + } + ctl = &best->xbar->chip->mux[1 + dpin]; } - ctl = &best->xbar->chip->mux[1 + dpin]; dcp = best->dcp; scoped_guard(mutex, &dcp->tb_lock) { @@ -1793,6 +1809,8 @@ static void dcp_typec_route_unregister(void *data) } port->hpd = !!(port->owner || port->secondary_owner); list_del(&route->port_link); + if (route->dual_stream) + atomic_dec(&dcp_dual_stream_routes); if (list_empty(&port->routes)) { list_del(&port->link); of_node_put(port->connector_np); @@ -1800,6 +1818,69 @@ static void dcp_typec_route_unregister(void *data) } } +static unsigned int dcp_typec_route_endpoints(struct device_node *connector_np) +{ + struct device_node *port __free(device_node) = + of_graph_get_port_by_id(connector_np, 3); + unsigned int count = 0; + + if (!port) + return 0; + for_each_of_graph_port_endpoint(port, endpoint) + count++; + return count; +} + +/* Look up capability controls without selecting or changing any mux. */ +static int dcp_typec_route_wiring(struct apple_dcp_typec_route *route, + struct device_node *connector_np, unsigned int index) +{ + struct device *dev = route->dcp->dev; + struct mux_control *legacy = NULL; + enum dcp_fabric_wiring wiring; + char name[24]; + unsigned int i; + bool named[2]; + + for (i = 0; i < 2; i++) { + snprintf(name, sizeof(name), "typec%u-dpin%u", index, i); + named[i] = of_property_match_string(dev->of_node, "mux-control-names", name) >= 0; + if (!named[i]) + continue; + route->dpin[i] = devm_mux_control_get(dev, name); + if (IS_ERR(route->dpin[i])) + return PTR_ERR(route->dpin[i]); + if (route->xbar != &route->xbar->chip->mux[0] || + route->xbar->chip->controllers < 3 || + route->dpin[i] != &route->xbar->chip->mux[1 + i]) + return -EINVAL; + } + if (!named[0] && !named[1]) { + snprintf(name, sizeof(name), "typec%u-usb4", index); + if (of_property_match_string(dev->of_node, "mux-control-names", name) >= 0) { + legacy = devm_mux_control_get(dev, name); + if (IS_ERR(legacy)) + return PTR_ERR(legacy); + if (route->xbar != &route->xbar->chip->mux[0] || + route->xbar->chip->controllers < 3 || + legacy != &route->xbar->chip->mux[1]) + return -EINVAL; + } + } + wiring = dcp_fabric_wiring(named[0], named[1], !!legacy, + dcp_typec_route_endpoints(connector_np)); + if (wiring == DCP_FABRIC_INVALID_WIRING) + return -EINVAL; + route->dual_stream = wiring != DCP_FABRIC_SINGLE_STREAM; + if (wiring == DCP_FABRIC_DUAL_LEGACY) { + /* Old j414/j416 ESP DTBs name cell 1 but never name cell 2. */ + route->dpin[0] = legacy; + route->dpin[1] = &route->xbar->chip->mux[2]; + dev_warn_once(dev, "legacy typecN-usb4 DPIN wiring; update the device tree\n"); + } + return 0; +} + int dcp_register_typec_routes(struct apple_dcp *dcp) { struct device_node *routes __free(device_node) = @@ -1878,10 +1959,19 @@ int dcp_register_typec_routes(struct apple_dcp *dcp) "%pOF: missing Type-C connector\n", route_np); + ret = dcp_typec_route_wiring(route, connector_np, route_index); + if (ret) { + of_node_put(connector_np); + return dev_err_probe(dev, ret, "%pOF: invalid DPIN wiring\n", route_np); + } + mutex_lock(&dcp_typec_fabric_lock); route->port = dcp_typec_port_get(connector_np); - if (route->port) + if (route->port) { list_add_tail(&route->port_link, &route->port->routes); + if (route->dual_stream) + atomic_inc(&dcp_dual_stream_routes); + } mutex_unlock(&dcp_typec_fabric_lock); if (!route->port) return -ENOMEM; @@ -1897,6 +1987,8 @@ int dcp_register_typec_routes(struct apple_dcp *dcp) if (IS_ERR(route->typec_mux)) { mutex_lock(&dcp_typec_fabric_lock); list_del(&route->port_link); + if (route->dual_stream) + atomic_dec(&dcp_dual_stream_routes); if (list_empty(&route->port->routes)) { list_del(&route->port->link); of_node_put(route->port->connector_np); diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index 6f051242423eb..d3f487abc603c 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -30,6 +30,8 @@ struct apple_dcp_typec_route { struct list_head port_link; struct phy *phy; struct mux_control *xbar; + struct mux_control *dpin[2]; + bool dual_stream; struct typec_mux_dev *typec_mux; u32 dptx_phy; u32 mux_index; @@ -47,6 +49,7 @@ struct apple_dcp_typec_route { bool dcp_is_typec_output(struct apple_dcp *dcp); bool dcp_is_usb4_output(struct apple_dcp *dcp); +bool dcp_uses_t6020_tunnel_flow(struct apple_dcp *dcp); bool dcp_has_typec_routes(struct platform_device *pdev); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 1a2024148d2c6..789b70f16cc62 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -190,6 +190,8 @@ struct apple_dcp_hw_data { u32 firmware_display_windows; u32 firmware_scratch; u32 firmware_request; + /* M2 Pro/Max laptop T6020 Thunderbolt flow, from the match data */ + bool t6020_tunnel_flow; }; enum dcp_test_point { diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 77047618123b6..97d0c30053768 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1018,7 +1018,7 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable } dcp->dptxport[port].connected = true; if (dcp_is_typec_output(dcp)) { - if (dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x()) + if (dcp_uses_t6020_tunnel_flow(dcp)) ret = dptxport_set_hpd_timeout(dcp->dptxport[port].service, true, 8000); else @@ -1033,7 +1033,7 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable } mutex_unlock(&dcp->hpd_mutex); - timeout = dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x() ? + timeout = dcp_uses_t6020_tunnel_flow(dcp) ? DPTX_TUNNEL_CONNECT_TIMEOUT : DPTX_CONNECT_TIMEOUT; ret = wait_for_completion_timeout(&dcp->dptxport[port].linkcfg_completion, timeout); @@ -1472,7 +1472,7 @@ static void dcp_typec_reconnect_work(struct work_struct *work) dcp_announce_kick(dcp); if (++dcp->typec_reconnect_tries < - (dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x() ? + (dcp_uses_t6020_tunnel_flow(dcp) ? 1 : DPTX_RECONNECT_RETRIES)) { mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, DPTX_RECONNECT_DELAY); @@ -3540,6 +3540,12 @@ static const struct dev_pm_ops dcp_platform_pm_ops = { static const struct apple_dcp_hw_data apple_dcp_hw_t6020 = { .num_dptx_ports = 1, + .t6020_tunnel_flow = true, +}; + +static const struct apple_dcp_hw_data apple_dcp_hw_t6020_dcpext = { + .num_dptx_ports = 2, + .t6020_tunnel_flow = true, }; static const struct apple_dcp_hw_data apple_dcp_hw_t8112 = { @@ -3602,6 +3608,7 @@ static const struct apple_dcp_hw_data apple_dcp_hw_dcpext = { static const struct of_device_id of_match[] = { { .compatible = "apple,t6020-dcp", .data = &apple_dcp_hw_t6020, }, + { .compatible = "apple,t6020-dcpext", .data = &apple_dcp_hw_t6020_dcpext, }, { .compatible = "apple,t8112-dcp", .data = &apple_dcp_hw_t8112, }, { .compatible = "apple,t8140-dcp", .data = &apple_dcp_hw_t8140, }, { .compatible = "apple,t8140-dcpext", .data = &apple_dcp_hw_t8140_external, }, diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 1f9cbcb8ed72f..8bf602a48c8ab 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1148,7 +1148,7 @@ void dcp_hotplug(struct work_struct *work) if (connector->base.state && !READ_ONCE(dcp->mode_state.valid) && connected) dcp_external_relink(dcp, connector); } else if (connector->base.state && !READ_ONCE(dcp->mode_state.valid) && connected && - !(dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x())) { + !dcp_uses_t6020_tunnel_flow(dcp)) { drm_connector_set_link_status_property(&connector->base, DRM_MODE_LINK_STATUS_BAD); From c5e7d48221830dc2481a2b3b6df4af40fd224352 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 23:49:18 -0700 Subject: [PATCH 1590/2680] drm/apple: share the DT query for dual-stream tunnel wiring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expose a per-connector wiring query for the Thunderbolt and ATC PHY consumers. Follow connector port 3 to its DCP routes, read the route index, and require the DP-alt and DPIN mux cells to share a crossbar. Accept either named DPIN0/1 controls or the legacy typecN-usb4 control with two endpoints; two endpoints alone remain single-stream. The query reads physical DT wiring without depending on node status, DCP registration or firmware readiness. Keep SoC flow qualification at each consumer. Share its pure classifier with the fabric core and exercise the public classifier with the existing wiring rows. A host harness compiling the production query with libfdt-backed OF operations matches the old board predicate on all 113 current Apple DTBs and four old ESP DTBs (116 connector rows). The consumer changes and removal of the remaining board helper follow in the W2c TB/PHY patch, with separate per-consumer equivalence checks. Validated with W=1 DRM, Thunderbolt and Apple PHY builds, arm64 QEMU KUnit, strict checkpatch, and query/classifier negative mutations. [ryan: adapted to 7.2.10 — the dp-tunnel.h query helpers are carried by 170-usb (it owns that header and folded this hunk into its consumer adaptation); only the DCP fabric core and its KUnit rows are taken here] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit ce7b33b160d0e099ca79f25dc75987e949c3b5a2) --- drivers/gpu/drm/apple/dcp-fabric-core.c | 7 +++---- drivers/gpu/drm/apple/dcp-fabric-test.c | 13 +++++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.c b/drivers/gpu/drm/apple/dcp-fabric-core.c index e2bbf333db973..50aa3d91c1e35 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.c +++ b/drivers/gpu/drm/apple/dcp-fabric-core.c @@ -5,6 +5,7 @@ #include #include #include +#include #include "dcp-fabric-core.h" @@ -13,11 +14,9 @@ enum dcp_fabric_wiring dcp_fabric_wiring(bool dpin0, bool dpin1, bool legacy, { if (dpin0 != dpin1) return DCP_FABRIC_INVALID_WIRING; - if (endpoints < 2) + if (!apple_dp_tunnel_wiring_dual(endpoints, dpin0, dpin1, legacy)) return DCP_FABRIC_SINGLE_STREAM; - if (dpin0 && dpin1) - return DCP_FABRIC_DUAL_NAMED; - return legacy ? DCP_FABRIC_DUAL_LEGACY : DCP_FABRIC_SINGLE_STREAM; + return dpin0 ? DCP_FABRIC_DUAL_NAMED : DCP_FABRIC_DUAL_LEGACY; } EXPORT_SYMBOL_GPL(dcp_fabric_wiring); diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c index 664b3495ea9b3..c97b434846cf5 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-test.c +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -4,6 +4,7 @@ #include #include #include +#include #include "dcp-fabric-core.h" @@ -728,7 +729,19 @@ static void fabric_wiring_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, dcp_fabric_t6020_flow(false, row->soc_support, dual)); } +static void fabric_shared_wiring_test(struct kunit *test) +{ + const struct fabric_wiring_case *row = test->param_value; + bool dual = row->expected == DCP_FABRIC_DUAL_NAMED || + row->expected == DCP_FABRIC_DUAL_LEGACY; + + KUNIT_EXPECT_EQ(test, apple_dp_tunnel_wiring_dual(row->endpoints, row->dpin0, + row->dpin1, row->legacy), dual); + KUNIT_EXPECT_FALSE(test, apple_dp_tunnel_dual_stream(NULL)); +} + static struct kunit_case fabric_tests[] = { + KUNIT_CASE_PARAM(fabric_shared_wiring_test, fabric_wiring_gen_params), KUNIT_CASE_PARAM(fabric_scenario_test, fabric_scenario_gen_params), KUNIT_CASE(fabric_dark_tunnel_test), KUNIT_CASE(fabric_effect_failure_test), From fc4d734271561ba3a868c3f98245e774bcddeb4e Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 4 Oct 2026 23:59:07 -0700 Subject: [PATCH 1591/2680] drm/apple: exercise fabric effect ordering with fake hardware MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scenario fixtures assert core decisions but do not prove that the driver applies those decisions in the expected order. Label the S8, S16, S32 and S50 rows as core tests and add separate fake-effect cases. Move HDMI handling, IRQ-first resume sampling, route registration admission, tunnel attach dispatch and failure replanning into inline executors shared by the driver and KUnit. Keep the production GPIO, mux, PHY and link operations in adapters, preserving their call order. The rebalance executor owns both port loops and discards a failed plan's remainder before taking a fresh snapshot. Cover frozen HDMI waiting, dual-stream debounce, high/low resume and stale samples, readiness/terminal attach rejection, external queuing, M3 crossbar qualification and missing-port errors. Pin fixed-output reconnection before replanning after a deactivate error, and bound persistent effect failures to two passes plus a fresh final plan. Validated with arm64 QEMU KUnit, W=1 DRM, Thunderbolt and Apple PHY builds, strict checkpatch, and eleven effect-order/admission mutants. [ryan: adapted to 7.2.10 — the effect runners wrap this tree's fabric: the rebalance callback still skips a port in SAFE, the route-waiting effect does not reset a dp_release_deadline (this tree has none), and the T6030 DP IN link runner is not wired up because the T6030 tunnel stays on the native 14.7 path (dcp_t602x_tunnel_route)] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 497ef108a22a510d9a0d505f0b0266788d5013e5) --- drivers/gpu/drm/apple/dcp-fabric-effects.h | 254 +++++++ drivers/gpu/drm/apple/dcp-fabric-test.c | 701 ++++++++++++++++++- drivers/gpu/drm/apple/dcp-fabric.c | 767 +++++++++++++-------- drivers/gpu/drm/apple/dcp.c | 38 +- 4 files changed, 1447 insertions(+), 313 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp-fabric-effects.h diff --git a/drivers/gpu/drm/apple/dcp-fabric-effects.h b/drivers/gpu/drm/apple/dcp-fabric-effects.h new file mode 100644 index 0000000000000..666c42a225df7 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-fabric-effects.h @@ -0,0 +1,254 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __APPLE_DCP_FABRIC_EFFECTS_H__ +#define __APPLE_DCP_FABRIC_EFFECTS_H__ + +#include "dcp-fabric-core.h" + +/* Effect executors shared by the driver and KUnit's fake hardware. */ +struct dcp_fabric_resume_ops { + void (*enable_irq)(void *ctx); + void (*sample)(void *ctx); +}; + +static inline void dcp_fabric_run_resume(const struct dcp_fabric_resume_ops *ops, + void *ctx) +{ + enum dcp_fabric_resume_step steps[2]; + unsigned int i; + + dcp_fabric_resume_steps(steps); + for (i = 0; i < 2; i++) { + switch (steps[i]) { + case DCP_FABRIC_ENABLE_HPD_IRQ: + ops->enable_irq(ctx); + break; + case DCP_FABRIC_SAMPLE_HPD: + ops->sample(ctx); + break; + } + } +} + +struct dcp_fabric_resume_sample_ops { + u64 (*edge)(void *ctx); + int (*read_hpd)(void *ctx); + bool (*sample)(void *ctx, u64 generation, int level); + bool (*borrowed)(void *ctx); + void (*connect_fixed)(void *ctx); +}; + +static inline void +dcp_fabric_run_resume_sample(const struct dcp_fabric_resume_sample_ops *ops, void *ctx) +{ + u64 generation = ops->edge(ctx); + int level = ops->read_hpd(ctx); + + if (ops->sample(ctx, generation, level) && level > 0 && !ops->borrowed(ctx)) + ops->connect_fixed(ctx); +} + +struct dcp_fabric_fixed_ops { + void (*restore_phy)(void *ctx); + int (*select_mux)(void *ctx); +}; + +static inline int dcp_fabric_run_fixed(bool has_fixed, bool owned, bool needs_mux, + const struct dcp_fabric_fixed_ops *ops, void *ctx) +{ + enum dcp_fabric_fixed_step steps[2]; + unsigned int count, i; + int ret; + + count = dcp_fabric_fixed_steps(has_fixed, owned, needs_mux, steps); + for (i = 0; i < count; i++) { + switch (steps[i]) { + case DCP_FABRIC_RESTORE_PHY: + ops->restore_phy(ctx); + break; + case DCP_FABRIC_SELECT_MUX: + ret = ops->select_mux(ctx); + if (ret) + return ret; + break; + } + } + return 0; +} + +struct dcp_fabric_hdmi_ops { + bool (*borrowed)(void *ctx); + bool (*keep_order)(void *ctx); + bool (*dual_stream)(void *ctx); + int (*read_hpd)(void *ctx); + void (*wait)(void *ctx, unsigned int ms); + void (*rebalance)(void *ctx); + void (*waiting)(void *ctx); + void (*observed)(void *ctx, bool high, bool debounced); + void (*connect_fixed)(void *ctx); +}; + +static inline void dcp_fabric_run_hdmi(const struct dcp_fabric_hdmi_ops *ops, + void *ctx) +{ + bool connected; + + if (ops->borrowed(ctx)) { + if (ops->keep_order(ctx) && ops->read_hpd(ctx)) { + ops->wait(ctx, 500); + if (ops->read_hpd(ctx)) + ops->rebalance(ctx); + } else if (!ops->dual_stream(ctx) && ops->read_hpd(ctx)) { + ops->waiting(ctx); + } + return; + } + connected = ops->read_hpd(ctx); + ops->observed(ctx, connected, false); + if (connected) { + ops->wait(ctx, 500); + connected = ops->read_hpd(ctx); + ops->observed(ctx, connected, true); + } + if (connected) + ops->connect_fixed(ctx); +} + +struct dcp_fabric_deactivate_ops { + int (*deactivate)(void *ctx); + bool (*selected)(void *ctx); + bool (*fixed_live)(void *ctx); + void (*keep_owner)(void *ctx); + void (*clear_owner)(void *ctx); + void (*connect_fixed)(void *ctx); +}; + +/* True discards the cached remainder and re-enters with a new snapshot. */ +static inline bool +dcp_fabric_run_deactivate(const struct dcp_fabric_deactivate_ops *ops, void *ctx) +{ + enum dcp_fabric_deactivate_step steps[3]; + unsigned int count, i; + int ret = ops->deactivate(ctx); + bool selected = ops->selected(ctx); + bool fixed_live = !(ret && selected) && ops->fixed_live(ctx); + + count = dcp_fabric_deactivate_steps(ret, selected, fixed_live, steps); + for (i = 0; i < count; i++) { + switch (steps[i]) { + case DCP_FABRIC_KEEP_OWNER: + ops->keep_owner(ctx); + break; + case DCP_FABRIC_CLEAR_OWNER: + ops->clear_owner(ctx); + break; + case DCP_FABRIC_CONNECT_FIXED: + ops->connect_fixed(ctx); + break; + case DCP_FABRIC_REPLAN: + return true; + } + } + return false; +} + +struct dcp_fabric_rebalance_ops { + void (*plan)(void *ctx); + void *(*first)(void *ctx); + void *(*next)(void *ctx, void *port); + bool (*deactivate)(void *ctx, void *port); + bool (*activate)(void *ctx, void *port); +}; + +static inline void +dcp_fabric_run_rebalance(const struct dcp_fabric_rebalance_ops *ops, void *ctx) +{ + unsigned int attempts; + void *port; + + for (attempts = 0; ; attempts++) { + ops->plan(ctx); + if (attempts == 2) + return; + for (port = ops->first(ctx); port; port = ops->next(ctx, port)) + if (ops->deactivate(ctx, port)) + break; + if (port) + continue; + for (port = ops->first(ctx); port; port = ops->next(ctx, port)) + if (ops->activate(ctx, port)) + break; + if (!port) + return; + } +} + +static inline int dcp_fabric_run_probe(bool has_routes, + int (*register_routes)(void *ctx), void *ctx) +{ + return has_routes ? register_routes(ctx) : 0; +} + +struct dcp_fabric_attach_ops { + int (*candidate)(void *ctx); + int (*activate)(void *ctx); + void (*rollback)(void *ctx); + void (*claim)(void *ctx); + bool (*external)(void *ctx); + bool (*has_connector)(void *ctx); + void (*queue_reconnect)(void *ctx); + void (*connect_oob)(void *ctx); +}; + +static inline int dcp_fabric_run_attach(const struct dcp_fabric_attach_ops *ops, + void *ctx) +{ + int ret = ops->candidate(ctx); + + if (ret) + return ret; + ret = ops->activate(ctx); + if (ret) { + ops->rollback(ctx); + return ret; + } + ops->claim(ctx); + if (dcp_fabric_attach_action(ops->external(ctx)) == DCP_FABRIC_ATTACH_WORK) + ops->queue_reconnect(ctx); + else if (ops->has_connector(ctx)) + ops->connect_oob(ctx); + return 0; +} + +static inline int +dcp_fabric_run_request(const struct dcp_fabric_port *ports, unsigned long key, + unsigned int dpin, + int (*dispatch)(void *ctx, const struct dcp_fabric_port *port), + void *ctx) +{ + const struct dcp_fabric_port *port; + int ret = dcp_fabric_tunnel_request(ports, key, dpin, &port); + + return ret ? ret : dispatch(ctx, port); +} + +struct dcp_fabric_link_ops { + bool (*has_xbar)(void *ctx); + bool (*tunnel)(void *ctx); + bool (*t6020_xbar)(void *ctx); + bool (*t6030_soc)(void *ctx); + void (*record_soc)(void *ctx, bool t6030); +}; + +static inline bool dcp_fabric_run_t6030_link(const struct dcp_fabric_link_ops *ops, + void *ctx) +{ + bool t6030; + + if (!ops->has_xbar(ctx)) + return false; + t6030 = ops->t6030_soc(ctx) && ops->t6020_xbar(ctx); + ops->record_soc(ctx, t6030); + return dcp_fabric_t6030_link(ops->tunnel(ctx), true, t6030); +} + +#endif /* __APPLE_DCP_FABRIC_EFFECTS_H__ */ diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c index c97b434846cf5..97f225facf5c7 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-test.c +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -7,6 +7,7 @@ #include #include "dcp-fabric-core.h" +#include "dcp-fabric-effects.h" struct fabric_fixture { struct dcp_fabric_pipeline pipeline[2]; @@ -221,7 +222,7 @@ static const struct fabric_scenario scenarios[] = { { "S5_full_hdmi_direct_tunnel", 5 }, { "S6_direct_release_then_tunnel", 6 }, { "S7_tunnel_release_promotes_direct", 7 }, - { "S8_frozen_hdmi_waits", 8 }, + { "S8_frozen_hdmi_waits_core", 8 }, { "S9_parked_fixed_reselect", 9 }, { "S10_hdmi_blink", 10 }, { "S11_hdmi_absence", 11 }, @@ -229,25 +230,25 @@ static const struct fabric_scenario scenarios[] = { { "S13_boot_three_outputs", 13 }, { "S14_boot_two_typec", 14 }, { "S15_boot_hdmi_tunnel", 15 }, - { "S16_resume_arms_settle", 16 }, + { "S16_resume_arms_settle_core", 16 }, { "S17_occupied_slot_errors", 17 }, { "S18_connector_mask", 18 }, { "S20_base_capacity", 20 }, { "S21_base_lone", 21 }, { "S30_dual_pairing_plan", 30 }, { "S31_dual_dpin_candidates", 31 }, - { "S32_dual_hdmi_reclaim", 32 }, + { "S32_dual_hdmi_reclaim_core", 32 }, { "S33_dual_scores", 33 }, { "S34_dual_park_reselect", 34 }, { "S35_dual_retry_off", 35 }, { "S40_desktop_promotion", 40 }, { "S41_dedicated_hdmi", 41 }, - { "S50a_panel_has_no_routes", 50 }, - { "S50b_external_service_readiness", 51 }, - { "S50c_terminal_scanout", 52 }, - { "S50d_external_attach_dispatch", 53 }, - { "S50e_t6030_crossbar", 54 }, - { "S50f_unwired_m3_ports", 55 }, + { "S50a_panel_has_no_routes_core", 50 }, + { "S50b_external_service_readiness_core", 51 }, + { "S50c_terminal_scanout_core", 52 }, + { "S50d_external_attach_dispatch_core", 53 }, + { "S50e_t6030_crossbar_core", 54 }, + { "S50f_unwired_m3_ports_core", 55 }, }; static void fabric_scenario_desc(const struct fabric_scenario *scenario, @@ -500,7 +501,7 @@ static void fabric_dark_tunnel_test(struct kunit *test) dcp_fabric_movable(&f.route[1][0], &f.route[1][1])); } -static void fabric_effect_failure_test(struct kunit *test) +static void fabric_effect_failure_core_test(struct kunit *test) { struct fabric_fixture f; @@ -740,16 +741,694 @@ static void fabric_shared_wiring_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, apple_dp_tunnel_dual_stream(NULL)); } +enum fabric_fake_effect { + FX_IRQ_ENABLE, + FX_SAMPLE, + FX_RESTORE_PHY, + FX_SELECT_MUX, + FX_MUX_FAIL, + FX_CONNECT_FIXED, + FX_WAIT, + FX_WAITING, + FX_REBALANCE, + FX_REGISTER, + FX_CANDIDATE, + FX_BIND, + FX_ACTIVATE, + FX_ROLLBACK, + FX_CLAIM, + FX_QUEUE, + FX_OOB, + FX_SOC, + FX_XBAR, + FX_RECORD, + FX_PLAN, + FX_DEACTIVATE, + FX_CLEAR, + FX_KEEP, +}; + +struct fabric_fake { + struct kunit *test; + struct fabric_fixture f; + struct dcp_fabric_presence presence; + struct dcp_fabric_route *best; + enum fabric_fake_effect trace[64]; + unsigned int count, reads, wait_ms, plans, attempts[2], releasing; + unsigned int failed_activations; + int levels[3], activate_error, deactivate_error; + bool borrowed, frozen, connector, irq_enabled, inject_edge; + bool selected, fixed_live, has_xbar, t6030, t6020_xbar, persistent; +}; + +static void fabric_fake_init(struct fabric_fake *f, struct kunit *test, bool dual) +{ + memset(f, 0, sizeof(*f)); + f->test = test; + f->connector = true; + f->has_xbar = true; + fabric_init(&f->f, dual); +} + +static void fabric_fake_trace(struct fabric_fake *f, enum fabric_fake_effect effect) +{ + KUNIT_ASSERT_LT(f->test, f->count, (unsigned int)ARRAY_SIZE(f->trace)); + f->trace[f->count++] = effect; +} + +static bool fabric_fake_borrowed(void *ctx) +{ + return ((struct fabric_fake *)ctx)->borrowed; +} + +static bool fabric_fake_keep_order(void *ctx) +{ + struct fabric_fake *f = ctx; + + return dcp_fabric_keep_order(f->f.policy.dual_stream, true, f->frozen); +} + +static bool fabric_fake_dual(void *ctx) +{ + return ((struct fabric_fake *)ctx)->f.policy.dual_stream; +} + +static int fabric_fake_hpd(void *ctx) +{ + struct fabric_fake *f = ctx; + unsigned int index = min(f->reads++, 2U); + + if (f->inject_edge) { + f->inject_edge = false; + dcp_fabric_presence_edge(&f->presence, 1001, 10000); + } + return f->levels[index]; +} + +static void fabric_fake_wait(void *ctx, unsigned int ms) +{ + struct fabric_fake *f = ctx; + + f->wait_ms += ms; + fabric_fake_trace(f, FX_WAIT); +} + +static void fabric_fake_waiting(void *ctx) +{ + fabric_fake_trace(ctx, FX_WAITING); +} + +static void fabric_fake_rebalance(void *ctx) +{ + fabric_fake_trace(ctx, FX_REBALANCE); +} + +static void fabric_fake_observed(void *ctx, bool high, bool debounced) +{ +} + +static void fabric_fake_restore_phy(void *ctx) +{ + fabric_fake_trace(ctx, FX_RESTORE_PHY); +} + +static int fabric_fake_select_mux(void *ctx) +{ + fabric_fake_trace(ctx, FX_SELECT_MUX); + return 0; +} + +static const struct dcp_fabric_fixed_ops fabric_fake_fixed_ops = { + .restore_phy = fabric_fake_restore_phy, + .select_mux = fabric_fake_select_mux, +}; + +static void fabric_fake_connect(void *ctx) +{ + fabric_fake_trace(ctx, FX_CONNECT_FIXED); +} + +static void fabric_fake_select_connect(void *ctx) +{ + struct fabric_fake *f = ctx; + + if (!dcp_fabric_run_fixed(true, f->borrowed, true, &fabric_fake_fixed_ops, f)) + fabric_fake_connect(f); +} + +static const struct dcp_fabric_hdmi_ops fabric_fake_hdmi_ops = { + .borrowed = fabric_fake_borrowed, + .keep_order = fabric_fake_keep_order, + .dual_stream = fabric_fake_dual, + .read_hpd = fabric_fake_hpd, + .wait = fabric_fake_wait, + .rebalance = fabric_fake_rebalance, + .waiting = fabric_fake_waiting, + .observed = fabric_fake_observed, + .connect_fixed = fabric_fake_select_connect, +}; + +static void fabric_fake_enable_irq(void *ctx) +{ + struct fabric_fake *f = ctx; + + f->irq_enabled = true; + fabric_fake_trace(f, FX_IRQ_ENABLE); +} + +static u64 fabric_fake_edge(void *ctx) +{ + struct fabric_fake *f = ctx; + + return dcp_fabric_presence_edge(&f->presence, 1000, 10000); +} + +static bool fabric_fake_sample(void *ctx, u64 generation, int level) +{ + struct fabric_fake *f = ctx; + + KUNIT_EXPECT_TRUE(f->test, f->irq_enabled); + fabric_fake_trace(f, FX_SAMPLE); + return dcp_fabric_presence_sample(&f->presence, generation, level > 0, 1000, 10000); +} + +static const struct dcp_fabric_resume_sample_ops fabric_fake_resume_sample_ops = { + .edge = fabric_fake_edge, + .read_hpd = fabric_fake_hpd, + .sample = fabric_fake_sample, + .borrowed = fabric_fake_borrowed, + .connect_fixed = fabric_fake_select_connect, +}; + +static void fabric_fake_resume_sample(void *ctx) +{ + dcp_fabric_run_resume_sample(&fabric_fake_resume_sample_ops, ctx); +} + +static const struct dcp_fabric_resume_ops fabric_fake_resume_ops = { + .enable_irq = fabric_fake_enable_irq, + .sample = fabric_fake_resume_sample, +}; + +static int fabric_fake_register(void *ctx) +{ + fabric_fake_trace(ctx, FX_REGISTER); + return 0; +} + +static int fabric_fake_candidate(void *ctx) +{ + struct fabric_fake *f = ctx; + int error; + + fabric_fake_trace(f, FX_CANDIDATE); + f->best = dcp_fabric_tunnel_candidate(&f->f.port[0], &f->f.policy, NULL, + false, 0, f->connector, &error); + return f->best ? 0 : error; +} + +static int fabric_fake_activate(void *ctx) +{ + struct fabric_fake *f = ctx; + + fabric_fake_trace(f, FX_BIND); + fabric_fake_trace(f, FX_ACTIVATE); + /* Fault injection may bypass admission: reject a missing target safely. */ + return f->best ? f->activate_error : -EINVAL; +} + +static void fabric_fake_rollback(void *ctx) +{ + fabric_fake_trace(ctx, FX_ROLLBACK); +} + +static void fabric_fake_claim(void *ctx) +{ + struct fabric_fake *f = ctx; + + fabric_fake_trace(f, FX_CLAIM); + f->f.port[0].owner[0] = f->best; + f->best->pipeline->owned = true; +} + +static bool fabric_fake_external(void *ctx) +{ + struct fabric_fake *f = ctx; + + return f->best->pipeline->external; +} + +static bool fabric_fake_connector(void *ctx) +{ + return ((struct fabric_fake *)ctx)->connector; +} + +static void fabric_fake_queue(void *ctx) +{ + fabric_fake_trace(ctx, FX_QUEUE); +} + +static void fabric_fake_oob(void *ctx) +{ + fabric_fake_trace(ctx, FX_OOB); +} + +static const struct dcp_fabric_attach_ops fabric_fake_attach_ops = { + .candidate = fabric_fake_candidate, + .activate = fabric_fake_activate, + .rollback = fabric_fake_rollback, + .claim = fabric_fake_claim, + .external = fabric_fake_external, + .has_connector = fabric_fake_connector, + .queue_reconnect = fabric_fake_queue, + .connect_oob = fabric_fake_oob, +}; + +static int fabric_fake_dispatch(void *ctx, const struct dcp_fabric_port *port) +{ + struct fabric_fake *f = ctx; + + KUNIT_EXPECT_PTR_EQ(f->test, port, &f->f.port[0]); + return dcp_fabric_run_attach(&fabric_fake_attach_ops, f); +} + +static bool fabric_fake_has_xbar(void *ctx) +{ + return ((struct fabric_fake *)ctx)->has_xbar; +} + +static bool fabric_fake_tunnel(void *ctx) +{ + return true; +} + +static bool fabric_fake_xbar(void *ctx) +{ + struct fabric_fake *f = ctx; + + fabric_fake_trace(f, FX_XBAR); + return f->t6020_xbar; +} + +static bool fabric_fake_soc(void *ctx) +{ + struct fabric_fake *f = ctx; + + fabric_fake_trace(f, FX_SOC); + return f->t6030; +} + +static void fabric_fake_record_soc(void *ctx, bool t6030) +{ + struct fabric_fake *f = ctx; + + f->f.pipeline[0].t6030_dpin = t6030; + fabric_fake_trace(f, FX_RECORD); +} + +static const struct dcp_fabric_link_ops fabric_fake_link_ops = { + .has_xbar = fabric_fake_has_xbar, + .tunnel = fabric_fake_tunnel, + .t6020_xbar = fabric_fake_xbar, + .t6030_soc = fabric_fake_soc, + .record_soc = fabric_fake_record_soc, +}; + +static int fabric_fake_deactivate(void *ctx) +{ + struct fabric_fake *f = ctx; + struct dcp_fabric_route *owner = f->f.port[f->releasing].owner[0]; + int ret = f->deactivate_error; + + fabric_fake_trace(f, FX_DEACTIVATE); + /* Model the real failure: release ownership, restore PHY, fail mux. */ + owner->pipeline->owned = false; + f->selected = false; + fabric_fake_trace(f, FX_RESTORE_PHY); + if (ret) + fabric_fake_trace(f, FX_MUX_FAIL); + f->deactivate_error = 0; + return ret; +} + +static bool fabric_fake_selected(void *ctx) +{ + return ((struct fabric_fake *)ctx)->selected; +} + +static bool fabric_fake_fixed_live(void *ctx) +{ + return ((struct fabric_fake *)ctx)->fixed_live; +} + +static void fabric_fake_keep(void *ctx) +{ + fabric_fake_trace(ctx, FX_KEEP); +} + +static void fabric_fake_clear(void *ctx) +{ + struct fabric_fake *f = ctx; + + f->f.port[f->releasing].owner[0] = NULL; + fabric_fake_trace(f, FX_CLEAR); +} + +static const struct dcp_fabric_deactivate_ops fabric_fake_deactivate_ops = { + .deactivate = fabric_fake_deactivate, + .selected = fabric_fake_selected, + .fixed_live = fabric_fake_fixed_live, + .keep_owner = fabric_fake_keep, + .clear_owner = fabric_fake_clear, + .connect_fixed = fabric_fake_connect, +}; + +static void fabric_fake_plan(void *ctx) +{ + struct fabric_fake *f = ctx; + + f->plans++; + fabric_fake_trace(f, FX_PLAN); + dcp_fabric_plan(f->f.pipeline, f->f.port, NULL, 0); +} + +static void *fabric_fake_first(void *ctx) +{ + struct fabric_fake *f = ctx; + + return &f->f.port[0]; +} + +static void *fabric_fake_next(void *ctx, void *entry) +{ + struct fabric_fake *f = ctx; + struct dcp_fabric_port *port = entry; + unsigned int p = port - f->f.port; + + return p < 2 ? &f->f.port[p + 1] : NULL; +} + +static bool fabric_fake_deactivate_one(void *ctx, void *entry) +{ + struct fabric_fake *f = ctx; + struct dcp_fabric_port *port = entry; + unsigned int p = port - f->f.port; + + if (!dcp_fabric_movable(port->owner[0], f->f.plan[p].target[0])) + return false; + f->releasing = p; + return dcp_fabric_run_deactivate(&fabric_fake_deactivate_ops, f); +} + +static bool fabric_fake_activate_one(void *ctx, void *entry) +{ + struct fabric_fake *f = ctx; + struct dcp_fabric_port *port = entry; + unsigned int p = port - f->f.port; + struct dcp_fabric_route *target = f->f.plan[p].target[0]; + unsigned int index; + + if (port->owner[0] || !target || !port->wanted || !port->hpd) + return false; + index = target->pipeline == &f->f.pipeline[0] ? 0 : 1; + f->attempts[index]++; + fabric_fake_trace(f, FX_ACTIVATE); + if (f->failed_activations || (f->persistent && f->plans < 4)) { + if (f->failed_activations) + f->failed_activations--; + f->f.pipeline[0].fixed_busy = true; + return true; + } + target->pipeline->owned = true; + port->owner[0] = target; + return false; +} + +static const struct dcp_fabric_rebalance_ops fabric_fake_rebalance_ops = { + .plan = fabric_fake_plan, + .first = fabric_fake_first, + .next = fabric_fake_next, + .deactivate = fabric_fake_deactivate_one, + .activate = fabric_fake_activate_one, +}; + +static void fabric_effect_s8_test(struct kunit *test) +{ + struct fabric_fake f; + + fabric_fake_init(&f, test, false); + f.borrowed = true; + f.frozen = true; + f.levels[0] = 1; + dcp_fabric_run_hdmi(&fabric_fake_hdmi_ops, &f); + KUNIT_ASSERT_EQ(test, f.count, 1U); + KUNIT_EXPECT_EQ(test, f.trace[0], FX_WAITING); + /* When the direct route leaves, the same handler restores and connects. */ + f.borrowed = false; + f.reads = 0; + f.levels[1] = 1; + dcp_fabric_run_hdmi(&fabric_fake_hdmi_ops, &f); + KUNIT_ASSERT_EQ(test, f.count, 5U); + KUNIT_EXPECT_EQ(test, f.trace[1], FX_WAIT); + KUNIT_EXPECT_EQ(test, f.trace[2], FX_RESTORE_PHY); + KUNIT_EXPECT_EQ(test, f.trace[3], FX_SELECT_MUX); + KUNIT_EXPECT_EQ(test, f.trace[4], FX_CONNECT_FIXED); +} + +static void fabric_effect_s32_test(struct kunit *test) +{ + struct fabric_fake f; + + fabric_fake_init(&f, test, true); + f.borrowed = true; + f.levels[0] = 1; + f.levels[1] = 1; + dcp_fabric_run_hdmi(&fabric_fake_hdmi_ops, &f); + KUNIT_ASSERT_EQ(test, f.count, 2U); + KUNIT_EXPECT_EQ(test, f.trace[0], FX_WAIT); + KUNIT_EXPECT_EQ(test, f.trace[1], FX_REBALANCE); + KUNIT_EXPECT_EQ(test, f.wait_ms, 500U); + /* The post-debounce level and compositor gate both suppress reclaim. */ + f.count = 0; + f.reads = 0; + f.wait_ms = 0; + f.levels[1] = 0; + dcp_fabric_run_hdmi(&fabric_fake_hdmi_ops, &f); + KUNIT_ASSERT_EQ(test, f.count, 1U); + KUNIT_EXPECT_EQ(test, f.trace[0], FX_WAIT); + f.count = 0; + f.reads = 0; + f.frozen = true; + dcp_fabric_run_hdmi(&fabric_fake_hdmi_ops, &f); + KUNIT_EXPECT_EQ(test, f.count, 0U); +} + +static void fabric_effect_s16_test(struct kunit *test) +{ + struct fabric_fake f; + u64 generation; + + fabric_fake_init(&f, test, false); + dcp_fabric_run_resume(&fabric_fake_resume_ops, &f); + KUNIT_ASSERT_EQ(test, f.count, 2U); + KUNIT_EXPECT_EQ(test, f.trace[0], FX_IRQ_ENABLE); + KUNIT_EXPECT_EQ(test, f.trace[1], FX_SAMPLE); + KUNIT_EXPECT_EQ(test, f.presence.state, DCP_FABRIC_SETTLING); + KUNIT_EXPECT_EQ(test, f.presence.deadline, 11000UL); + generation = f.presence.generation; + f.count = 0; + f.reads = 0; + f.levels[0] = 1; + dcp_fabric_run_resume(&fabric_fake_resume_ops, &f); + KUNIT_ASSERT_EQ(test, f.count, 5U); + KUNIT_EXPECT_EQ(test, f.trace[2], FX_RESTORE_PHY); + KUNIT_EXPECT_EQ(test, f.trace[3], FX_SELECT_MUX); + KUNIT_EXPECT_EQ(test, f.trace[4], FX_CONNECT_FIXED); + KUNIT_EXPECT_EQ(test, f.presence.state, DCP_FABRIC_PRESENT); + KUNIT_EXPECT_EQ(test, f.presence.deadline, 0UL); + KUNIT_EXPECT_FALSE(test, + dcp_fabric_presence_expire(&f.presence, generation, false, 11000)); + /* A falling hardirq during GPIO sampling invalidates the high sample. */ + f.count = 0; + f.reads = 0; + f.inject_edge = true; + dcp_fabric_run_resume(&fabric_fake_resume_ops, &f); + KUNIT_EXPECT_EQ(test, f.count, 2U); + KUNIT_EXPECT_EQ(test, f.presence.state, DCP_FABRIC_SETTLING); +} + +static void fabric_effect_s50a_test(struct kunit *test) +{ + struct fabric_fake f; + + fabric_fake_init(&f, test, false); + KUNIT_EXPECT_EQ(test, dcp_fabric_run_probe(false, fabric_fake_register, &f), 0); + KUNIT_EXPECT_EQ(test, f.count, 0U); + KUNIT_EXPECT_FALSE(test, f.f.pipeline[0].owned); + KUNIT_EXPECT_EQ(test, dcp_fabric_run_probe(true, fabric_fake_register, &f), 0); + KUNIT_ASSERT_EQ(test, f.count, 1U); + KUNIT_EXPECT_EQ(test, f.trace[0], FX_REGISTER); +} + +static void fabric_effect_s50bc_test(struct kunit *test) +{ + struct fabric_fake f; + + fabric_fake_init(&f, test, false); + f.f.port[0].routes = &f.f.route[0][1]; + f.f.pipeline[1].external = true; + f.f.pipeline[1].services_ready = false; + KUNIT_EXPECT_EQ(test, dcp_fabric_run_attach(&fabric_fake_attach_ops, &f), -EAGAIN); + KUNIT_ASSERT_EQ(test, f.count, 1U); + KUNIT_EXPECT_EQ(test, f.trace[0], FX_CANDIDATE); + KUNIT_EXPECT_PTR_EQ(test, f.f.port[0].owner[0], NULL); + f.count = 0; + f.f.pipeline[1].terminal = true; + KUNIT_EXPECT_EQ(test, dcp_fabric_run_attach(&fabric_fake_attach_ops, &f), -ESHUTDOWN); + KUNIT_EXPECT_EQ(test, f.count, 1U); + f.count = 0; + f.f.pipeline[1].terminal = false; + f.f.pipeline[1].services_ready = true; + KUNIT_EXPECT_EQ(test, dcp_fabric_run_attach(&fabric_fake_attach_ops, &f), 0); + KUNIT_ASSERT_EQ(test, f.count, 5U); + KUNIT_EXPECT_EQ(test, f.trace[4], FX_QUEUE); + KUNIT_EXPECT_PTR_EQ(test, f.f.port[0].owner[0], &f.f.route[0][1]); +} + +static void fabric_effect_s50d_test(struct kunit *test) +{ + struct fabric_fake f; + + fabric_fake_init(&f, test, false); + KUNIT_ASSERT_EQ(test, dcp_fabric_run_attach(&fabric_fake_attach_ops, &f), 0); + KUNIT_ASSERT_EQ(test, f.count, 5U); + KUNIT_EXPECT_EQ(test, f.trace[4], FX_OOB); + fabric_fake_init(&f, test, false); + f.f.pipeline[1].external = true; + KUNIT_ASSERT_EQ(test, dcp_fabric_run_attach(&fabric_fake_attach_ops, &f), 0); + KUNIT_ASSERT_EQ(test, f.count, 5U); + KUNIT_EXPECT_EQ(test, f.trace[4], FX_QUEUE); + fabric_fake_init(&f, test, false); + f.activate_error = -EIO; + KUNIT_EXPECT_EQ(test, dcp_fabric_run_attach(&fabric_fake_attach_ops, &f), -EIO); + KUNIT_ASSERT_EQ(test, f.count, 4U); + KUNIT_EXPECT_EQ(test, f.trace[3], FX_ROLLBACK); + KUNIT_EXPECT_PTR_EQ(test, f.f.port[0].owner[0], NULL); +} + +static void fabric_effect_s50e_test(struct kunit *test) +{ + struct fabric_fake f; + + fabric_fake_init(&f, test, false); + f.t6030 = true; + f.t6020_xbar = true; + KUNIT_EXPECT_TRUE(test, dcp_fabric_run_t6030_link(&fabric_fake_link_ops, &f)); + KUNIT_EXPECT_TRUE(test, f.f.pipeline[0].t6030_dpin); + f.t6030 = false; + KUNIT_EXPECT_FALSE(test, dcp_fabric_run_t6030_link(&fabric_fake_link_ops, &f)); + KUNIT_EXPECT_FALSE(test, f.f.pipeline[0].t6030_dpin); + f.t6030 = true; + f.t6020_xbar = false; + KUNIT_EXPECT_FALSE(test, dcp_fabric_run_t6030_link(&fabric_fake_link_ops, &f)); + f.has_xbar = false; + f.count = 0; + KUNIT_EXPECT_FALSE(test, dcp_fabric_run_t6030_link(&fabric_fake_link_ops, &f)); + KUNIT_EXPECT_EQ(test, f.count, 0U); +} + +static void fabric_effect_s50f_test(struct kunit *test) +{ + struct fabric_fake f; + + fabric_fake_init(&f, test, false); + f.f.port[0].next = NULL; + KUNIT_EXPECT_EQ(test, + dcp_fabric_run_request(f.f.port, 2, 0, fabric_fake_dispatch, &f), -ENODEV); + KUNIT_EXPECT_EQ(test, + dcp_fabric_run_request(f.f.port, 3, 0, fabric_fake_dispatch, &f), -ENODEV); + KUNIT_EXPECT_EQ(test, f.count, 0U); + KUNIT_EXPECT_EQ(test, + dcp_fabric_run_request(f.f.port, 1, 0, fabric_fake_dispatch, &f), 0); + KUNIT_EXPECT_EQ(test, f.count, 5U); +} + +static void fabric_effect_failure_test(struct kunit *test) +{ + struct fabric_fake f; + + fabric_fake_init(&f, test, true); + f.f.port[0].wanted = true; + f.f.port[0].hpd = true; + f.f.port[1].wanted = true; + f.f.port[1].hpd = true; + f.failed_activations = 1; + dcp_fabric_run_rebalance(&fabric_fake_rebalance_ops, &f); + KUNIT_EXPECT_EQ(test, f.plans, 2U); + KUNIT_EXPECT_EQ(test, f.attempts[0], 1U); + KUNIT_EXPECT_EQ(test, f.attempts[1], 1U); + KUNIT_EXPECT_PTR_EQ(test, f.f.port[0].owner[0], &f.f.route[0][1]); + KUNIT_EXPECT_PTR_EQ(test, f.f.port[1].owner[0], NULL); + /* Persistent failure still takes the third fresh snapshot and stops. */ + fabric_fake_init(&f, test, true); + f.f.port[0].wanted = true; + f.f.port[0].hpd = true; + f.f.port[1].wanted = true; + f.f.port[1].hpd = true; + f.persistent = true; + dcp_fabric_run_rebalance(&fabric_fake_rebalance_ops, &f); + KUNIT_EXPECT_EQ(test, f.plans, 3U); + KUNIT_EXPECT_EQ(test, f.attempts[0], 1U); + KUNIT_EXPECT_EQ(test, f.attempts[1], 1U); + KUNIT_EXPECT_PTR_EQ(test, f.f.port[0].owner[0], NULL); + KUNIT_EXPECT_PTR_EQ(test, f.f.plan[0].target[0], &f.f.route[0][1]); +} + +static void fabric_effect_deactivate_failure_test(struct kunit *test) +{ + struct fabric_fake f; + + fabric_fake_init(&f, test, true); + f.f.port[0].wanted = true; + f.f.port[0].hpd = true; + f.f.port[0].owner[0] = &f.f.route[0][0]; + f.f.pipeline[0].owned = true; + f.f.pipeline[0].fixed_busy = true; + f.deactivate_error = -EIO; + f.fixed_live = true; + dcp_fabric_run_rebalance(&fabric_fake_rebalance_ops, &f); + KUNIT_ASSERT_GE(test, f.count, 8U); + KUNIT_EXPECT_EQ(test, f.trace[0], FX_PLAN); + KUNIT_EXPECT_EQ(test, f.trace[1], FX_DEACTIVATE); + KUNIT_EXPECT_EQ(test, f.trace[2], FX_RESTORE_PHY); + KUNIT_EXPECT_EQ(test, f.trace[3], FX_MUX_FAIL); + KUNIT_EXPECT_EQ(test, f.trace[4], FX_CLEAR); + KUNIT_EXPECT_EQ(test, f.trace[5], FX_CONNECT_FIXED); + KUNIT_EXPECT_EQ(test, f.trace[6], FX_PLAN); + KUNIT_EXPECT_EQ(test, f.plans, 2U); + KUNIT_EXPECT_PTR_EQ(test, f.f.port[0].owner[0], &f.f.route[0][1]); +} + static struct kunit_case fabric_tests[] = { KUNIT_CASE_PARAM(fabric_shared_wiring_test, fabric_wiring_gen_params), KUNIT_CASE_PARAM(fabric_scenario_test, fabric_scenario_gen_params), KUNIT_CASE(fabric_dark_tunnel_test), - KUNIT_CASE(fabric_effect_failure_test), + KUNIT_CASE(fabric_effect_failure_core_test), KUNIT_CASE(fabric_deactivate_failure_test), KUNIT_CASE(fabric_unbound_and_mask_test), KUNIT_CASE(fabric_presence_wrap_test), KUNIT_CASE(fabric_masked_edge_test), KUNIT_CASE_PARAM(fabric_wiring_test, fabric_wiring_gen_params), + KUNIT_CASE(fabric_effect_s8_test), + KUNIT_CASE(fabric_effect_s16_test), + KUNIT_CASE(fabric_effect_s32_test), + KUNIT_CASE(fabric_effect_s50a_test), + KUNIT_CASE(fabric_effect_s50bc_test), + KUNIT_CASE(fabric_effect_s50d_test), + KUNIT_CASE(fabric_effect_s50e_test), + KUNIT_CASE(fabric_effect_s50f_test), + KUNIT_CASE(fabric_effect_failure_test), + KUNIT_CASE(fabric_effect_deactivate_failure_test), {} }; diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index a09cb272ed76f..0e98a4575c53e 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -20,6 +20,7 @@ #include "afk.h" #include "dcp.h" #include "dcp-fabric.h" +#include "dcp-fabric-effects.h" #include "dcpext_scanout.h" #include "ibootep.h" #include "dcp-internal.h" @@ -74,6 +75,8 @@ static atomic_t dcp_dual_stream_routes = ATOMIC_INIT(0); static enum dcp_fabric_presence_state dcp_hdmi_presence(struct apple_dcp *dcp); static void dcp_hdmi_update_locked(struct apple_dcp *dcp); +static int dcp_hdmi_read_hpd(void *ctx); +static bool dcp_hdmi_borrowed(void *ctx); bool dcp_is_typec_output(struct apple_dcp *dcp) { @@ -631,24 +634,41 @@ void dcp_fabric_init(struct apple_dcp *dcp) } /* Caller enables the HPD IRQ first; a later edge invalidates this sample. */ -void dcp_fabric_hdmi_resume(struct apple_dcp *dcp) +static u64 dcp_resume_presence_edge(void *ctx) { - u64 generation; - int level, ret; + return dcp_hdmi_edge(ctx); +} + +static bool dcp_resume_presence_sample(void *ctx, u64 generation, int level) +{ + return dcp_hdmi_sample(ctx, generation, level); +} + +static void dcp_resume_connect_fixed(void *ctx) +{ + struct apple_dcp *dcp = ctx; + int ret = dcp_fixed_output_select(dcp); - guard(mutex)(&dcp_typec_fabric_lock); - generation = dcp_hdmi_edge(dcp); - level = gpiod_get_value_cansleep(dcp->hdmi_hpd); - if (!dcp_hdmi_sample(dcp, generation, level) || level <= 0 || - dcp->active_typec_route) - return; - ret = dcp_fixed_output_select(dcp); if (!ret) dcp_dptx_connect(dcp, 0); else dev_err(dcp->dev, "could not select the HDMI output on resume: %d\n", ret); } +static const struct dcp_fabric_resume_sample_ops dcp_resume_sample_ops = { + .edge = dcp_resume_presence_edge, + .read_hpd = dcp_hdmi_read_hpd, + .sample = dcp_resume_presence_sample, + .borrowed = dcp_hdmi_borrowed, + .connect_fixed = dcp_resume_connect_fixed, +}; + +void dcp_fabric_hdmi_resume(struct apple_dcp *dcp) +{ + guard(mutex)(&dcp_typec_fabric_lock); + dcp_fabric_run_resume_sample(&dcp_resume_sample_ops, dcp); +} + /* A Thunderbolt tunnel holds @dcp's pipeline: it never moves. */ static bool dcp_typec_tunnel_held(struct apple_dcp *dcp) { @@ -851,109 +871,167 @@ static void dcp_typec_route_waiting(void) * its new pipeline with its HPD replayed; the hotplugs this sends make * fbdev and userspace re-probe. */ -static struct apple_dcp_typec_route * -dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, - unsigned int dpin) -{ - struct drm_device *drm = dcp_typec_drm(); - struct apple_dcp_typec_route *planned = NULL; +struct dcp_rebalance_context { + struct drm_device *drm; + struct apple_dcp_typec_port *arriving; + unsigned int dpin; + struct apple_dcp_typec_route *planned; +}; + +struct dcp_deactivate_context { struct apple_dcp_typec_port *port; - unsigned int attempts = 0; + struct apple_dcp_typec_route *owner; + struct apple_dcp *dcp; +}; - lockdep_assert_held(&dcp_typec_fabric_lock); +static int dcp_rebalance_route_deactivate(void *data) +{ + struct dcp_deactivate_context *ctx = data; -replan: - dcp_typec_plan(drm, arriving, dpin); - if (arriving) { - planned = dpin ? arriving->secondary_target : arriving->target; - /* it will be refused: plan as if it had not asked */ - if (planned && dcp_typec_tunnel_held(planned->dcp)) { - planned = NULL; - dcp_typec_plan(drm, NULL, 0); - } - } + return dcp_typec_route_deactivate(ctx->owner); +} - /* Even on persistent failure, return only a fresh, revalidated plan. */ - if (attempts++ == 2) - return planned; +static bool dcp_rebalance_route_selected(void *data) +{ + struct dcp_deactivate_context *ctx = data; - list_for_each_entry(port, &dcp_typec_ports, link) { - struct apple_dcp_typec_route *owner = port->owner; - enum dcp_fabric_deactivate_step steps[3]; - struct apple_dcp *dcp; - unsigned int count, i; - bool fixed_live; - int ret; + return ctx->owner->selected; +} - /* A port in SAFE keeps its owner until the set phase completes. */ - if (port->safe_pending) - continue; - if (!dcp_fabric_movable(owner ? &owner->core : NULL, - port->target ? &port->target->core : - NULL)) - continue; +static bool dcp_rebalance_fixed_live(void *data) +{ + struct dcp_deactivate_context *ctx = data; + struct apple_dcp *dcp = ctx->dcp; - dcp = owner->dcp; - dev_info(dcp->dev, "re-routing %pOF from %s to %s for Type-C connector order\n", - port->connector_np, dev_name(dcp->dev), - port->target ? dev_name(port->target->dcp->dev) : "none"); - if (port->hpd || dcp->typec_cable_connected || - (dcp->typec_connector && dcp->typec_connector->connected)) - dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - port->hpd = false; - ret = dcp_typec_route_deactivate(owner); - /* Deactivate's fixed-mux error has already released the route. */ - fixed_live = !(ret && owner->selected) && dcp->hdmi_hpd && - dcp->active && gpiod_get_value_cansleep(dcp->hdmi_hpd); - count = dcp_fabric_deactivate_steps(ret, owner->selected, - fixed_live, steps); - for (i = 0; i < count; i++) { - switch (steps[i]) { - case DCP_FABRIC_KEEP_OWNER: - dcp_typec_port_attach(port); - break; - case DCP_FABRIC_CLEAR_OWNER: - port->owner = NULL; - break; - case DCP_FABRIC_CONNECT_FIXED: - dcp_dptx_connect(dcp, 0); - break; - case DCP_FABRIC_REPLAN: - goto replan; - } + return dcp->hdmi_hpd && dcp->active && gpiod_get_value_cansleep(dcp->hdmi_hpd); +} + +static void dcp_rebalance_keep_owner(void *data) +{ + struct dcp_deactivate_context *ctx = data; + + dcp_typec_port_attach(ctx->port); +} + +static void dcp_rebalance_clear_owner(void *data) +{ + struct dcp_deactivate_context *ctx = data; + + ctx->port->owner = NULL; +} + +static void dcp_rebalance_connect_fixed(void *data) +{ + struct dcp_deactivate_context *ctx = data; + + dcp_dptx_connect(ctx->dcp, 0); +} + +static const struct dcp_fabric_deactivate_ops dcp_rebalance_deactivate_ops = { + .deactivate = dcp_rebalance_route_deactivate, + .selected = dcp_rebalance_route_selected, + .fixed_live = dcp_rebalance_fixed_live, + .keep_owner = dcp_rebalance_keep_owner, + .clear_owner = dcp_rebalance_clear_owner, + .connect_fixed = dcp_rebalance_connect_fixed, +}; + +static void dcp_rebalance_plan(void *data) +{ + struct dcp_rebalance_context *ctx = data; + + dcp_typec_plan(ctx->drm, ctx->arriving, ctx->dpin); + if (ctx->arriving) { + ctx->planned = ctx->dpin ? ctx->arriving->secondary_target : ctx->arriving->target; + if (ctx->planned && dcp_typec_tunnel_held(ctx->planned->dcp)) { + ctx->planned = NULL; + dcp_typec_plan(ctx->drm, NULL, 0); } } +} - list_for_each_entry(port, &dcp_typec_ports, link) { - struct apple_dcp_typec_route *route = port->target; +static void *dcp_rebalance_first(void *data) +{ + return list_first_entry_or_null(&dcp_typec_ports, struct apple_dcp_typec_port, link); +} - if (port->owner || !port->dp_wanted || !port->dp_hpd || - port == arriving) - continue; +static void *dcp_rebalance_next(void *data, void *entry) +{ + struct apple_dcp_typec_port *port = entry; - if (!route || !dcp_typec_route_available(route)) { - port->applied_valid = false; - continue; - } - if (dcp_typec_route_activate(route, route->xbar)) { - /* - * Dark for now. Its unchanged DP state is not reported - * again, so it is placed when the plan next runs, or - * once a compositor owns the display, when a pipeline - * is freed (dcp_typec_route_waiting()). - */ - port->applied_valid = false; - goto replan; - } - port->owner = route; - port->preferred_route = route; + return list_is_last(&port->link, &dcp_typec_ports) ? NULL : list_next_entry(port, link); +} - dcp_typec_port_attach(port); - } +static bool dcp_rebalance_deactivate(void *data, void *entry) +{ + struct apple_dcp_typec_port *port = entry; + struct apple_dcp_typec_route *owner = port->owner; + struct apple_dcp *dcp; + struct dcp_deactivate_context ctx; + + /* A port in SAFE keeps its owner until the set phase completes. */ + if (port->safe_pending) + return false; + if (!dcp_fabric_movable(owner ? &owner->core : NULL, + port->target ? &port->target->core : NULL)) + return false; + dcp = owner->dcp; + dev_info(dcp->dev, "re-routing %pOF from %s to %s for Type-C connector order\n", + port->connector_np, dev_name(dcp->dev), + port->target ? dev_name(port->target->dcp->dev) : "none"); + if (port->hpd || dcp->typec_cable_connected || + (dcp->typec_connector && dcp->typec_connector->connected)) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + port->hpd = false; + ctx.port = port; + ctx.owner = owner; + ctx.dcp = dcp; + return dcp_fabric_run_deactivate(&dcp_rebalance_deactivate_ops, &ctx); +} - return planned; +static bool dcp_rebalance_activate(void *data, void *entry) +{ + struct dcp_rebalance_context *ctx = data; + struct apple_dcp_typec_port *port = entry; + struct apple_dcp_typec_route *route = port->target; + + if (port->owner || !port->dp_wanted || !port->dp_hpd || port == ctx->arriving) + return false; + if (!route || !dcp_typec_route_available(route)) { + port->applied_valid = false; + return false; + } + if (dcp_typec_route_activate(route, route->xbar)) { + port->applied_valid = false; + return true; + } + port->owner = route; + port->preferred_route = route; + dcp_typec_port_attach(port); + return false; } +static const struct dcp_fabric_rebalance_ops dcp_rebalance_ops = { + .plan = dcp_rebalance_plan, + .first = dcp_rebalance_first, + .next = dcp_rebalance_next, + .deactivate = dcp_rebalance_deactivate, + .activate = dcp_rebalance_activate, +}; + +static struct apple_dcp_typec_route * +dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, unsigned int dpin) +{ + struct dcp_rebalance_context ctx = { + .drm = dcp_typec_drm(), + .arriving = arriving, + .dpin = dpin, + }; + + lockdep_assert_held(&dcp_typec_fabric_lock); + dcp_fabric_run_rebalance(&dcp_rebalance_ops, &ctx); + return ctx.planned; +} /* * A display is plugged into the HDMI port while its pipeline, the hybrid, * drives a Type-C port. Without dual-stream docks nothing moves for it: a @@ -1458,38 +1536,180 @@ static bool dcp_tb_services_ready(struct apple_dcp *dcp) return smp_load_acquire(&dcp->dptxport[0].enabled) && ibootep_is_ready(dcp); } -int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, - bool active, int (*set_active)(void *ctx, bool active), - void *ctx) -{ - struct apple_dcp_typec_port *port, *pos; - struct dcp_fabric_port *ports = NULL, **tail = &ports; - const struct dcp_fabric_port *found; - struct apple_dcp_typec_route *best = NULL, *planned = NULL; +struct dcp_tb_attach_context { + struct apple_dcp_typec_port *port; struct apple_dcp_typec_route **slot; + struct apple_dcp_typec_route *best; + struct apple_dcp_typec_route *planned; struct mux_control *ctl; - struct apple_dcp *dcp; + unsigned int dpin; + bool active; + int (*set_active)(void *binding, bool active); + void *binding; +}; + +static int dcp_tb_candidate(void *data) +{ + struct dcp_tb_attach_context *ctx = data; + struct apple_dcp_typec_port *port = ctx->port; + struct apple_dcp_typec_route *best, *planned = NULL; + struct mux_control *ctl; + unsigned int dpin = ctx->dpin; bool ordered; int ret; - if (!connector_np || dpin > 1) - return -EINVAL; + /* + * Until a compositor owns the display, the pairing pass decides where + * the stream goes, moving direct DP-alt routes out of its way, and + * whether it gets a pipeline at all. + */ + ordered = dcp_typec_keep_order(); + if (ordered) + planned = dcp_typec_rebalance_locked(port, dpin); + ctx->planned = planned; - guard(mutex)(&dcp_typec_fabric_lock); + { + struct dcp_fabric_policy policy = { + .dual_stream = dcp_typec_dual_stream() + }; + struct dcp_fabric_route *chosen; + bool connector_present = dpin ? !!port->secondary_connector : !!port->connector; - list_for_each_entry(pos, &dcp_typec_ports, link) { - pos->core.key = (unsigned long)pos->connector_np; - *tail = &pos->core; - tail = &pos->core.next; + dcp_fabric_snapshot_port(port, true); + chosen = dcp_fabric_tunnel_candidate(&port->core, + &policy, planned ? &planned->core : NULL, + ordered, dpin, + connector_present, + &ret); + best = dcp_fabric_real_route(chosen); + if (!best) + return ret; } - *tail = NULL; - ret = dcp_fabric_tunnel_request(ports, (unsigned long)connector_np, dpin, &found); - if (ret) - return ret; - port = container_of(found, struct apple_dcp_typec_port, core); + + ctl = best->dpin[dpin]; + if (!ctl) { + /* Legacy DT ABI: unnamed DPIN controls share the DP-alt chip. */ + if (best->xbar != &best->xbar->chip->mux[0] || + best->xbar->chip->controllers < 3) { + ret = -EOPNOTSUPP; + return ret; + } + ctl = &best->xbar->chip->mux[1 + dpin]; + } + + ctx->best = best; + ctx->ctl = ctl; + return 0; +} + +static int dcp_tb_activate(void *data) +{ + struct dcp_tb_attach_context *ctx = data; + struct apple_dcp *dcp = ctx->best->dcp; + + scoped_guard(mutex, &dcp->tb_lock) { + dcp->tb_dpin_set_active = ctx->set_active; + dcp->tb_dpin_ctx = ctx->binding; + } + ctx->best->tunnel_dpin = ctx->dpin; + return dcp_typec_route_activate(ctx->best, ctx->ctl); +} + +static void dcp_tb_rollback(void *data) +{ + struct dcp_tb_attach_context *ctx = data; + struct apple_dcp *dcp = ctx->best->dcp; + + guard(mutex)(&dcp->tb_lock); + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; +} + +static void dcp_tb_claim(void *data) +{ + struct dcp_tb_attach_context *ctx = data; + struct apple_dcp_typec_route *best = ctx->best; + struct apple_dcp_typec_port *port = ctx->port; + struct apple_dcp *dcp = best->dcp; + unsigned int dpin = ctx->dpin; + + *ctx->slot = best; + /* the port is in USB4 mode, not DP-alt */ + port->dp_wanted = false; + dcp_tunnel_prepare(best, ctx->ctl); + + dev_info(dcp->dev, "display routed to Thunderbolt DP tunnel dpin%u\n", dpin); + + if (dcp->fw_compat == DCP_FIRMWARE_V_14_7 && dcp->dptxep && + !dcp->dptxport[0].enabled) + dev_warn(dcp->dev, "DPTX port not announced, not opening the controller\n"); + + /* + * The DP IN adapter may only be woken (DPTX_INACTIVE=0) while DCP + * drives the DPTX, i.e. from DCP's Activate call; waking it earlier + * hangs the machine. dptxep calls set_active back from Activate and + * Deactivate (set above, before the route became a tunnel). + */ + if (!dcp->typec_connector && !dcp->external) + dev_warn(dcp->dev, "no Type-C connector for the DP tunnel\n"); + WRITE_ONCE(dcp->typec_cable_connected, true); + port->hpd = true; +} + +static bool dcp_tb_external(void *data) +{ + struct dcp_tb_attach_context *ctx = data; + + return ctx->best->dcp->external; +} + +static bool dcp_tb_has_connector(void *data) +{ + struct dcp_tb_attach_context *ctx = data; + + return !!ctx->best->dcp->typec_connector; +} + +static void dcp_tb_queue_reconnect(void *data) +{ + struct dcp_tb_attach_context *ctx = data; + struct apple_dcp *dcp = ctx->best->dcp; + + dcp->typec_reconnect_tries = 0; + mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); +} + +static void dcp_tb_connect_oob(void *data) +{ + struct dcp_tb_attach_context *ctx = data; + + dcp_dptx_connect_oob(to_platform_device(ctx->best->dcp->dev), 0); +} + +static const struct dcp_fabric_attach_ops dcp_tb_attach_ops = { + .candidate = dcp_tb_candidate, + .activate = dcp_tb_activate, + .rollback = dcp_tb_rollback, + .claim = dcp_tb_claim, + .external = dcp_tb_external, + .has_connector = dcp_tb_has_connector, + .queue_reconnect = dcp_tb_queue_reconnect, + .connect_oob = dcp_tb_connect_oob, +}; + +static int dcp_tb_dispatch(void *data, const struct dcp_fabric_port *found) +{ + struct dcp_tb_attach_context *ctx = data; + struct apple_dcp_typec_port *port = + container_of(found, struct apple_dcp_typec_port, core); + struct apple_dcp_typec_route **slot; + struct apple_dcp *dcp; + unsigned int dpin = ctx->dpin; + int ret; + slot = dpin ? &port->secondary_owner : &port->owner; - if (!active) { + if (!ctx->active) { if (!*slot || !(*slot)->tunnel || (*slot)->tunnel_dpin != dpin) return 0; @@ -1547,98 +1767,39 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, } } - /* - * Until a compositor owns the display, the pairing pass decides where - * the stream goes, moving direct DP-alt routes out of its way, and - * whether it gets a pipeline at all. - */ - ordered = dcp_typec_keep_order(); - if (ordered) - planned = dcp_typec_rebalance_locked(port, dpin); - - { - struct dcp_fabric_policy policy = { - .dual_stream = dcp_typec_dual_stream() - }; - struct dcp_fabric_route *chosen; - bool connector_present = dpin ? !!port->secondary_connector : !!port->connector; - - dcp_fabric_snapshot_port(port, true); - chosen = dcp_fabric_tunnel_candidate(&port->core, - &policy, planned ? &planned->core : NULL, - ordered, dpin, - connector_present, - &ret); - best = dcp_fabric_real_route(chosen); - /* -ESHUTDOWN (retained terminal scanout) and -EAGAIN (services - * not yet published) also undo the reorder. - */ - if (!best) - goto err_reorder; - } + ctx->port = port; + ctx->slot = slot; + ret = dcp_fabric_run_attach(&dcp_tb_attach_ops, ctx); + /* A failed arriving stream releases its reserved plan to the others. */ + if (ret && ret != -ESHUTDOWN && ctx->planned) + dcp_typec_rebalance_locked(NULL, 0); + return ret; +} - ctl = best->dpin[dpin]; - if (!ctl) { - /* Legacy DT ABI: unnamed DPIN controls share the DP-alt chip. */ - if (best->xbar != &best->xbar->chip->mux[0] || - best->xbar->chip->controllers < 3) { - ret = -EOPNOTSUPP; - goto err_reorder; - } - ctl = &best->xbar->chip->mux[1 + dpin]; - } +int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, + bool active, int (*set_active)(void *ctx, bool active), + void *ctx) +{ + struct apple_dcp_typec_port *pos; + struct dcp_fabric_port *ports = NULL, **tail = &ports; + struct dcp_tb_attach_context request = { + .dpin = dpin, + .active = active, + .set_active = set_active, + .binding = ctx, + }; - dcp = best->dcp; - scoped_guard(mutex, &dcp->tb_lock) { - dcp->tb_dpin_set_active = set_active; - dcp->tb_dpin_ctx = ctx; - } - best->tunnel_dpin = dpin; - ret = dcp_typec_route_activate(best, ctl); - if (ret) { - scoped_guard(mutex, &dcp->tb_lock) { - dcp->tb_dpin_set_active = NULL; - dcp->tb_dpin_ctx = NULL; - } - goto err_reorder; + if (!connector_np || dpin > 1) + return -EINVAL; + guard(mutex)(&dcp_typec_fabric_lock); + list_for_each_entry(pos, &dcp_typec_ports, link) { + pos->core.key = (unsigned long)pos->connector_np; + *tail = &pos->core; + tail = &pos->core.next; } - *slot = best; - /* the port is in USB4 mode, not DP-alt */ - port->dp_wanted = false; - dcp_tunnel_prepare(best, ctl); - - dev_info(dcp->dev, "display routed to Thunderbolt DP tunnel dpin%u\n", dpin); - - if (dcp->fw_compat == DCP_FIRMWARE_V_14_7 && dcp->dptxep && - !dcp->dptxport[0].enabled) - dev_warn(dcp->dev, "DPTX port not announced, not opening the controller\n"); - - /* - * The DP IN adapter may only be woken (DPTX_INACTIVE=0) while DCP - * drives the DPTX, i.e. from DCP's Activate call; waking it earlier - * hangs the machine. dptxep calls set_active back from Activate and - * Deactivate (set above, before the route became a tunnel). - */ - if (!dcp->typec_connector && !dcp->external) - dev_warn(dcp->dev, "no Type-C connector for the DP tunnel\n"); - WRITE_ONCE(dcp->typec_cable_connected, true); - port->hpd = true; - /* External firmware link bring-up precedes its DRM connector. Run it - * after returning to the tunnel manager, outside the fabric lock. - */ - if (dcp_fabric_attach_action(dcp->external) == DCP_FABRIC_ATTACH_WORK) { - dcp->typec_reconnect_tries = 0; - mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); - } else if (dcp->typec_connector) - dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); - - return 0; - -err_reorder: - /* the pass kept a pipeline for this stream: give it to the others */ - if (planned) - dcp_typec_rebalance_locked(NULL, 0); - return ret; + *tail = NULL; + return dcp_fabric_run_request(ports, (unsigned long)connector_np, dpin, + dcp_tb_dispatch, &request); } EXPORT_SYMBOL_GPL(apple_dcp_tb_dp_tunnel); @@ -1881,17 +2042,20 @@ static int dcp_typec_route_wiring(struct apple_dcp_typec_route *route, return 0; } -int dcp_register_typec_routes(struct apple_dcp *dcp) +struct dcp_route_probe_context { + struct apple_dcp *dcp; + struct device_node *routes; +}; + +static int dcp_register_typec_routes_present(void *data) { - struct device_node *routes __free(device_node) = - of_get_child_by_name(dcp->dev->of_node, "typec-routes"); + struct dcp_route_probe_context *ctx = data; + struct apple_dcp *dcp = ctx->dcp; + struct device_node *routes = ctx->routes; struct device *dev = dcp->dev; u32 route_index; int ret; - if (!routes) - return 0; - for_each_available_child_of_node_scoped(routes, route_np) { struct apple_dcp_typec_route *route; struct device_node *endpoint __free(device_node) = NULL; @@ -2015,11 +2179,13 @@ int dcp_register_typec_routes(struct apple_dcp *dcp) return 0; } -bool dcp_has_typec_routes(struct platform_device *pdev) +int dcp_register_typec_routes(struct apple_dcp *dcp) { - struct apple_dcp *dcp = platform_get_drvdata(pdev); + struct device_node *routes __free(device_node) = + of_get_child_by_name(dcp->dev->of_node, "typec-routes"); + struct dcp_route_probe_context ctx = { .dcp = dcp, .routes = routes }; - return dcp->nr_typec_routes; + return dcp_fabric_run_probe(!!routes, dcp_register_typec_routes_present, &ctx); } /* @@ -2027,33 +2193,35 @@ bool dcp_has_typec_routes(struct platform_device *pdev) * (see dcp_typec_route_deactivate()). Point it at the HDMI output again * before connecting a display that has arrived there. */ -static int dcp_fixed_output_select(struct apple_dcp *dcp) +static void dcp_fixed_restore_phy(void *ctx) { - enum dcp_fabric_fixed_step steps[2]; - unsigned int count, i; - int ret; + struct apple_dcp *dcp = ctx; + + dcp->phy = dcp->fixed_phy; + dcp->dptx_phy = dcp->fixed_dptx_phy; +} + +static int dcp_fixed_select_mux(void *ctx) +{ + struct apple_dcp *dcp = ctx; + int ret = mux_control_select(dcp->xbar, dcp->fixed_mux_index); + + if (!ret) + dcp->fixed_route_selected = true; + return ret; +} + +static const struct dcp_fabric_fixed_ops dcp_fixed_ops = { + .restore_phy = dcp_fixed_restore_phy, + .select_mux = dcp_fixed_select_mux, +}; +static int dcp_fixed_output_select(struct apple_dcp *dcp) +{ lockdep_assert_held(&dcp_typec_fabric_lock); - count = dcp_fabric_fixed_steps(!!dcp->fixed_phy, - !!dcp->active_typec_route, - dcp->xbar && !dcp->fixed_route_selected, - steps); - for (i = 0; i < count; i++) { - switch (steps[i]) { - case DCP_FABRIC_RESTORE_PHY: - dcp->phy = dcp->fixed_phy; - dcp->dptx_phy = dcp->fixed_dptx_phy; - break; - case DCP_FABRIC_SELECT_MUX: - ret = mux_control_select(dcp->xbar, - dcp->fixed_mux_index); - if (ret) - return ret; - dcp->fixed_route_selected = true; - break; - } - } - return 0; + return dcp_fabric_run_fixed(!!dcp->fixed_phy, !!dcp->active_typec_route, + dcp->xbar && !dcp->fixed_route_selected, + &dcp_fixed_ops, dcp); } /* @@ -2068,9 +2236,80 @@ irqreturn_t dcp_dp2hdmi_hpd_edge(int irq, void *data) return IRQ_WAKE_THREAD; } +static bool dcp_hdmi_borrowed(void *ctx) +{ + struct apple_dcp *dcp = ctx; + + return !!READ_ONCE(dcp->active_typec_route); +} + +static bool dcp_hdmi_keep_order(void *ctx) +{ + return dcp_typec_keep_order(); +} + +static bool dcp_hdmi_dual_stream(void *ctx) +{ + return dcp_typec_dual_stream(); +} + +static int dcp_hdmi_read_hpd(void *ctx) +{ + struct apple_dcp *dcp = ctx; + + return gpiod_get_value_cansleep(dcp->hdmi_hpd); +} + +static void dcp_hdmi_wait(void *ctx, unsigned int ms) +{ + msleep(ms); +} + +static void dcp_hdmi_rebalance(void *ctx) +{ + dcp_typec_rebalance_locked(NULL, 0); +} + +static void dcp_hdmi_waiting(void *ctx) +{ + dcp_typec_hdmi_waits(ctx); +} + +static void dcp_hdmi_observed(void *ctx, bool high, bool debounced) +{ + struct apple_dcp *dcp = ctx; + + if (debounced) + dev_info(dcp->dev, "DP2HDMI HPD irq, 500ms debounce: connected:%d\n", high); + else + dev_info(dcp->dev, "DP2HDMI HPD irq, connected:%d\n", high); +} + +static void dcp_hdmi_connect_fixed(void *ctx) +{ + struct apple_dcp *dcp = ctx; + int ret = dcp_fixed_output_select(dcp); + + if (ret) + dev_err(dcp->dev, "could not select the HDMI output: %d\n", ret); + else + dcp_dptx_connect(dcp, 0); +} + +static const struct dcp_fabric_hdmi_ops dcp_hdmi_ops = { + .borrowed = dcp_hdmi_borrowed, + .keep_order = dcp_hdmi_keep_order, + .dual_stream = dcp_hdmi_dual_stream, + .read_hpd = dcp_hdmi_read_hpd, + .wait = dcp_hdmi_wait, + .rebalance = dcp_hdmi_rebalance, + .waiting = dcp_hdmi_waiting, + .observed = dcp_hdmi_observed, + .connect_fixed = dcp_hdmi_connect_fixed, +}; + static void dcp_hdmi_update_locked(struct apple_dcp *dcp) { - bool connected; unsigned long flags; u64 generation; int level; @@ -2081,49 +2320,7 @@ static void dcp_hdmi_update_locked(struct apple_dcp *dcp) spin_unlock_irqrestore(&dcp->hdmi_presence_lock, flags); level = gpiod_get_value_cansleep(dcp->hdmi_hpd); dcp_hdmi_sample(dcp, generation, level); - - if (READ_ONCE(dcp->active_typec_route)) { - /* - * Until a compositor owns the display, a live HDMI output - * takes its pipeline back from a direct DP-alt route: the - * compositor pairs the HDMI connector with it first. Without - * dual-stream docks it waits (see dcp_typec_hdmi_waits()). - */ - if (dcp_typec_keep_order() && - gpiod_get_value_cansleep(dcp->hdmi_hpd)) { - msleep(500); - if (gpiod_get_value_cansleep(dcp->hdmi_hpd)) - dcp_typec_rebalance_locked(NULL, 0); - } else if (!dcp_typec_dual_stream() && - gpiod_get_value_cansleep(dcp->hdmi_hpd)) { - dcp_typec_hdmi_waits(dcp); - } - return; - } - connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); - - /* do nothing on disconnect and trust that dcp detects it itself. - * Parallel disconnect HPDs result drm disabling the CRTC even when it - * should not. - * The interrupt should be changed to rising but for now the disconnect - * IRQs might be helpful for debugging. - */ - dev_info(dcp->dev, "DP2HDMI HPD irq, connected:%d\n", connected); - - if (connected) { - msleep(500); - connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); - dev_info(dcp->dev, "DP2HDMI HPD irq, 500ms debounce: connected:%d\n", connected); - } - - if (connected) { - int ret = dcp_fixed_output_select(dcp); - - if (ret) - dev_err(dcp->dev, "could not select the HDMI output: %d\n", ret); - else - dcp_dptx_connect(dcp, 0); - } + dcp_fabric_run_hdmi(&dcp_hdmi_ops, dcp); } irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 97d0c30053768..880b0ee5b0517 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -45,6 +45,7 @@ #include "afk.h" #include "av.h" #include "dcp.h" +#include "dcp-fabric-effects.h" #include "dcp-fabric.h" #include "dcpext_scanout.h" #include "dcp-internal.h" @@ -3502,29 +3503,32 @@ static int dcp_platform_suspend(struct device *dev) return 0; } +static void dcp_resume_enable_irq(void *ctx) +{ + struct apple_dcp *dcp = ctx; + + enable_irq(dcp->hdmi_hpd_irq); +} + +static void dcp_resume_sample(void *ctx) +{ + dcp_fabric_hdmi_resume(ctx); +} + +static const struct dcp_fabric_resume_ops dcp_resume_ops = { + .enable_irq = dcp_resume_enable_irq, + .sample = dcp_resume_sample, +}; + static int dcp_platform_resume(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); dcp_release_after_sleep(dcp); dcp_enable_typec_work(dcp); - if (dcp->hdmi_hpd_irq) { - enum dcp_fabric_resume_step steps[2]; - unsigned int i; - - /* Observe future edges before sampling any edges lost in sleep. */ - dcp_fabric_resume_steps(steps); - for (i = 0; i < ARRAY_SIZE(steps); i++) { - switch (steps[i]) { - case DCP_FABRIC_ENABLE_HPD_IRQ: - enable_irq(dcp->hdmi_hpd_irq); - break; - case DCP_FABRIC_SAMPLE_HPD: - dcp_fabric_hdmi_resume(dcp); - break; - } - } - } + /* Observe future edges before sampling any edges lost in sleep. */ + if (dcp->hdmi_hpd_irq) + dcp_fabric_run_resume(&dcp_resume_ops, dcp); if (dcp->avep) av_service_connect(dcp); From 340a076b48c7e7c8d17c83401b0f8b995d82f4db Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 01:43:30 -0700 Subject: [PATCH 1592/2680] drm/apple: Remove unused Thunderbolt capacity notification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The capacity notification has no provider, so its symbol lookup always fails. Remove the declaration and lookup while keeping the direct-port capacity notification and the existing Thunderbolt retry interval. [ryan: adapted to 7.2.10 — the declaration it removes lives in dcp-fabric.h here; the KUnit case is placed among this tree's cases (the binding/session cases it sat beside arrive with the 170-usb generation API)] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 273b6e0705ebf5b3807fea8bcf345e6268f94640) --- drivers/gpu/drm/apple/dcp-fabric-effects.h | 7 +++++++ drivers/gpu/drm/apple/dcp-fabric-test.c | 18 ++++++++++++++++++ drivers/gpu/drm/apple/dcp-fabric.c | 19 +++++++------------ drivers/gpu/drm/apple/dcp-fabric.h | 6 ------ 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric-effects.h b/drivers/gpu/drm/apple/dcp-fabric-effects.h index 666c42a225df7..21bf78c035ba3 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-effects.h +++ b/drivers/gpu/drm/apple/dcp-fabric-effects.h @@ -5,6 +5,13 @@ #include "dcp-fabric-core.h" /* Effect executors shared by the driver and KUnit's fake hardware. */ +static inline void dcp_fabric_run_capacity(bool available, void (*notify)(void *ctx), + void *ctx) +{ + if (available) + notify(ctx); +} + struct dcp_fabric_resume_ops { void (*enable_irq)(void *ctx); void (*sample)(void *ctx); diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c index 97f225facf5c7..6010a6d46d005 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-test.c +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -1409,8 +1409,26 @@ static void fabric_effect_deactivate_failure_test(struct kunit *test) KUNIT_EXPECT_PTR_EQ(test, f.f.port[0].owner[0], &f.f.route[0][1]); } +static void fabric_capacity_notification_test(struct kunit *test) +{ + struct fabric_fake f; + u64 generation; + bool available; + + fabric_fake_init(&f, test, false); + generation = dcp_fabric_presence_edge(&f.presence, 1000, 10000); + for (unsigned int i = 0; i < 3; i++) { + available = dcp_fabric_presence_expire(&f.presence, generation, + false, i ? 11000 : 10999); + dcp_fabric_run_capacity(available, fabric_fake_rebalance, &f); + KUNIT_EXPECT_EQ(test, f.count, i ? 1U : 0U); + } + KUNIT_EXPECT_EQ(test, f.trace[0], FX_REBALANCE); +} + static struct kunit_case fabric_tests[] = { KUNIT_CASE_PARAM(fabric_shared_wiring_test, fabric_wiring_gen_params), + KUNIT_CASE(fabric_capacity_notification_test), KUNIT_CASE_PARAM(fabric_scenario_test, fabric_scenario_gen_params), KUNIT_CASE(fabric_dark_tunnel_test), KUNIT_CASE(fabric_effect_failure_core_test), diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 0e98a4575c53e..84ed1da86ecb4 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -549,11 +549,15 @@ static bool dcp_hdmi_sample(struct apple_dcp *dcp, u64 generation, int level) return accepted; } +static void dcp_hdmi_capacity_available(void *ctx) +{ + dcp_typec_pipeline_freed(); +} + static void dcp_hdmi_settle_work(struct work_struct *work) { struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, hdmi_settle_wq); - typeof(apple_tb_dp_capacity_available) *notify; unsigned long flags; u64 generation; bool available; @@ -575,17 +579,8 @@ static void dcp_hdmi_settle_work(struct work_struct *work) level > 0, jiffies); spin_unlock_irqrestore(&dcp->hdmi_presence_lock, flags); dcp_hdmi_schedule(dcp); - if (!available) - return; - - /* One capacity event, fanned out to direct ports and eligible tunnels. */ - dcp_typec_pipeline_freed(); - notify = symbol_get(apple_tb_dp_capacity_available); - if (!notify) - return; - for (unsigned int i = 0; i < dcp->nr_typec_routes; i++) - notify(dcp->typec_routes[i].port->connector_np); - symbol_put(apple_tb_dp_capacity_available); + /* Notify direct ports once when HDMI capacity becomes available. */ + dcp_fabric_run_capacity(available, dcp_hdmi_capacity_available, dcp); } static void dcp_hdmi_recheck_work(struct work_struct *work) diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index d3f487abc603c..44837410cc60e 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -90,10 +90,4 @@ int dcp_dptx_connect(struct apple_dcp *dcp, u32 port); int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port); bool dcp_is_external_h17p(struct apple_dcp *dcp); -/* - * Optional TB capacity wakeup: RCU lookup and queue only, never a mutex/wait. - * Declared here rather than in the 170-usb owned dp-tunnel.h; no provider. - */ -void apple_tb_dp_capacity_available(struct device_node *connector_np); - #endif /* __APPLE_DCP_FABRIC_H__ */ From 8aded7e1f6af622c0b2f538c5c2da356f6d08db4 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 01:45:56 -0700 Subject: [PATCH 1593/2680] drm/apple: Qualify tunnel flow by connector wiring Use the physical connector wiring for the T602X USB4 flow policy. This also covers plain dcpext2 and dcpext3 routes on M2 Max laptops, which share dual-stream connectors with routes that name DPIN controls. Keep the USB4-output and SoC support guards. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit e05b4e6cbec0e45d78f4372da4cea013079e8d2b) --- drivers/gpu/drm/apple/dcp-fabric-core.c | 4 ++-- drivers/gpu/drm/apple/dcp-fabric-core.h | 2 +- drivers/gpu/drm/apple/dcp-fabric-test.c | 20 ++++++++++++++++++++ drivers/gpu/drm/apple/dcp-fabric.c | 4 ++-- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.c b/drivers/gpu/drm/apple/dcp-fabric-core.c index 50aa3d91c1e35..d0b1a9e7de8ca 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.c +++ b/drivers/gpu/drm/apple/dcp-fabric-core.c @@ -20,9 +20,9 @@ enum dcp_fabric_wiring dcp_fabric_wiring(bool dpin0, bool dpin1, bool legacy, } EXPORT_SYMBOL_GPL(dcp_fabric_wiring); -bool dcp_fabric_t6020_flow(bool usb4, bool soc_support, bool route_wired) +bool dcp_fabric_t6020_flow(bool usb4, bool soc_support, bool connector_wired) { - return usb4 && soc_support && route_wired; + return usb4 && soc_support && connector_wired; } EXPORT_SYMBOL_GPL(dcp_fabric_t6020_flow); diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.h b/drivers/gpu/drm/apple/dcp-fabric-core.h index 04609936964e6..71c1cf75ae999 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.h +++ b/drivers/gpu/drm/apple/dcp-fabric-core.h @@ -69,7 +69,7 @@ enum dcp_fabric_wiring { enum dcp_fabric_wiring dcp_fabric_wiring(bool dpin0, bool dpin1, bool legacy, unsigned int endpoints); -bool dcp_fabric_t6020_flow(bool usb4, bool soc_support, bool route_wired); +bool dcp_fabric_t6020_flow(bool usb4, bool soc_support, bool connector_wired); enum dcp_fabric_capacity_action { DCP_FABRIC_PROMOTE, diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c index 6010a6d46d005..505042144bf5f 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-test.c +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -1426,8 +1426,28 @@ static void fabric_capacity_notification_test(struct kunit *test) KUNIT_EXPECT_EQ(test, f.trace[0], FX_REBALANCE); } +static void fabric_connector_flow_core_test(struct kunit *test) +{ + bool single_stream = apple_dp_tunnel_wiring_dual(2, false, false, false); + + /* Plain dcpext2/3 routes inherit the physical connector's capacity. */ + for (unsigned int dcp = 0; dcp < 4; dcp++) { + bool route_has_dpin = dcp < 2; + bool connector_wired = apple_dp_tunnel_wiring_dual(2, true, true, false); + + KUNIT_EXPECT_TRUE(test, dcp_fabric_t6020_flow(true, true, connector_wired)); + if (!route_has_dpin) + KUNIT_EXPECT_FALSE(test, dcp_fabric_t6020_flow(true, true, route_has_dpin)); + KUNIT_EXPECT_FALSE(test, dcp_fabric_t6020_flow(false, true, connector_wired)); + KUNIT_EXPECT_FALSE(test, dcp_fabric_t6020_flow(true, false, connector_wired)); + } + /* Two endpoints alone do not enable the flow on a single-stream port. */ + KUNIT_EXPECT_FALSE(test, dcp_fabric_t6020_flow(true, true, single_stream)); +} + static struct kunit_case fabric_tests[] = { KUNIT_CASE_PARAM(fabric_shared_wiring_test, fabric_wiring_gen_params), + KUNIT_CASE(fabric_connector_flow_core_test), KUNIT_CASE(fabric_capacity_notification_test), KUNIT_CASE_PARAM(fabric_scenario_test, fabric_scenario_gen_params), KUNIT_CASE(fabric_dark_tunnel_test), diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 84ed1da86ecb4..9f9ad37f615ac 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -93,10 +93,10 @@ bool dcp_is_usb4_output(struct apple_dcp *dcp) bool dcp_uses_t6020_tunnel_flow(struct apple_dcp *dcp) { struct apple_dcp_typec_route *route = READ_ONCE(dcp->active_typec_route); + bool connector_wired = route && apple_dp_tunnel_dual_stream(route->port->connector_np); return dcp_fabric_t6020_flow(dcp_is_usb4_output(dcp), - dcp->hw.t6020_tunnel_flow, - route && route->dual_stream); + dcp->hw.t6020_tunnel_flow, connector_wired); } static bool dcp_typec_route_is_dp(const struct typec_mux_state *state) From fa6b33febed333917ad1d4669f7261cf6e00f054 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 01:46:43 -0700 Subject: [PATCH 1594/2680] drm/apple: Limit HDMI resume to presence sampling Enable the HPD IRQ before sampling presence on resume. Leave DPTX connection and fixed-output selection to the existing poweron path, so the platform resume callback performs no synchronous reconnect. Retain the high-sample hold clearing and low-sample settling window. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 3c431aa17186750387b49807b5374a05acc5e7d7) --- drivers/gpu/drm/apple/dcp-fabric-effects.h | 5 +--- drivers/gpu/drm/apple/dcp-fabric-test.c | 27 +++++++++++++++++----- drivers/gpu/drm/apple/dcp-fabric.c | 13 ----------- 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric-effects.h b/drivers/gpu/drm/apple/dcp-fabric-effects.h index 21bf78c035ba3..5ec2cc96504e1 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-effects.h +++ b/drivers/gpu/drm/apple/dcp-fabric-effects.h @@ -40,8 +40,6 @@ struct dcp_fabric_resume_sample_ops { u64 (*edge)(void *ctx); int (*read_hpd)(void *ctx); bool (*sample)(void *ctx, u64 generation, int level); - bool (*borrowed)(void *ctx); - void (*connect_fixed)(void *ctx); }; static inline void @@ -50,8 +48,7 @@ dcp_fabric_run_resume_sample(const struct dcp_fabric_resume_sample_ops *ops, voi u64 generation = ops->edge(ctx); int level = ops->read_hpd(ctx); - if (ops->sample(ctx, generation, level) && level > 0 && !ops->borrowed(ctx)) - ops->connect_fixed(ctx); + ops->sample(ctx, generation, level); } struct dcp_fabric_fixed_ops { diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c index 505042144bf5f..a914d911bade7 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-test.c +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -916,8 +916,6 @@ static const struct dcp_fabric_resume_sample_ops fabric_fake_resume_sample_ops = .edge = fabric_fake_edge, .read_hpd = fabric_fake_hpd, .sample = fabric_fake_sample, - .borrowed = fabric_fake_borrowed, - .connect_fixed = fabric_fake_select_connect, }; static void fabric_fake_resume_sample(void *ctx) @@ -1240,10 +1238,9 @@ static void fabric_effect_s16_test(struct kunit *test) f.reads = 0; f.levels[0] = 1; dcp_fabric_run_resume(&fabric_fake_resume_ops, &f); - KUNIT_ASSERT_EQ(test, f.count, 5U); - KUNIT_EXPECT_EQ(test, f.trace[2], FX_RESTORE_PHY); - KUNIT_EXPECT_EQ(test, f.trace[3], FX_SELECT_MUX); - KUNIT_EXPECT_EQ(test, f.trace[4], FX_CONNECT_FIXED); + KUNIT_ASSERT_EQ(test, f.count, 2U); + KUNIT_EXPECT_EQ(test, f.trace[0], FX_IRQ_ENABLE); + KUNIT_EXPECT_EQ(test, f.trace[1], FX_SAMPLE); KUNIT_EXPECT_EQ(test, f.presence.state, DCP_FABRIC_PRESENT); KUNIT_EXPECT_EQ(test, f.presence.deadline, 0UL); KUNIT_EXPECT_FALSE(test, @@ -1445,8 +1442,26 @@ static void fabric_connector_flow_core_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, dcp_fabric_t6020_flow(true, true, single_stream)); } +static void fabric_resume_no_connect_test(struct kunit *test) +{ + for (unsigned int row = 0; row < 8; row++) { + struct fabric_fake f; + + fabric_fake_init(&f, test, !!(row & 4)); + f.borrowed = !!(row & 2); + f.levels[0] = !!(row & 1); + dcp_fabric_run_resume(&fabric_fake_resume_ops, &f); + KUNIT_ASSERT_EQ(test, f.count, 2U); + KUNIT_EXPECT_EQ(test, f.trace[0], FX_IRQ_ENABLE); + KUNIT_EXPECT_EQ(test, f.trace[1], FX_SAMPLE); + KUNIT_EXPECT_EQ(test, f.presence.state, + (row & 1) ? DCP_FABRIC_PRESENT : DCP_FABRIC_SETTLING); + } +} + static struct kunit_case fabric_tests[] = { KUNIT_CASE_PARAM(fabric_shared_wiring_test, fabric_wiring_gen_params), + KUNIT_CASE(fabric_resume_no_connect_test), KUNIT_CASE(fabric_connector_flow_core_test), KUNIT_CASE(fabric_capacity_notification_test), KUNIT_CASE_PARAM(fabric_scenario_test, fabric_scenario_gen_params), diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 9f9ad37f615ac..3141bdb62caeb 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -639,23 +639,10 @@ static bool dcp_resume_presence_sample(void *ctx, u64 generation, int level) return dcp_hdmi_sample(ctx, generation, level); } -static void dcp_resume_connect_fixed(void *ctx) -{ - struct apple_dcp *dcp = ctx; - int ret = dcp_fixed_output_select(dcp); - - if (!ret) - dcp_dptx_connect(dcp, 0); - else - dev_err(dcp->dev, "could not select the HDMI output on resume: %d\n", ret); -} - static const struct dcp_fabric_resume_sample_ops dcp_resume_sample_ops = { .edge = dcp_resume_presence_edge, .read_hpd = dcp_hdmi_read_hpd, .sample = dcp_resume_presence_sample, - .borrowed = dcp_hdmi_borrowed, - .connect_fixed = dcp_resume_connect_fixed, }; void dcp_fabric_hdmi_resume(struct apple_dcp *dcp) From eb43eb713a12d7c12342a9a1bdb10b014606db03 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 01:48:08 -0700 Subject: [PATCH 1595/2680] drm/apple: Restrict HDMI settling to hybrid pipelines Arm the HDMI presence hold only for a fixed-output pipeline with Type-C routes on a single-stream fabric. HDMI-only and Type-C-only pipelines, and dual-stream fabrics, skip presence sampling and delayed rechecks. Keep the existing HDMI hotplug and poweron connection paths. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 687b36ba1b5e3c778fef063512c82c6e60211630) --- drivers/gpu/drm/apple/dcp-fabric-effects.h | 17 +++++++-- drivers/gpu/drm/apple/dcp-fabric-test.c | 42 ++++++++++++++++++---- drivers/gpu/drm/apple/dcp-fabric.c | 37 +++++++++++++------ 3 files changed, 77 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric-effects.h b/drivers/gpu/drm/apple/dcp-fabric-effects.h index 5ec2cc96504e1..5c6db057b430b 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-effects.h +++ b/drivers/gpu/drm/apple/dcp-fabric-effects.h @@ -12,6 +12,11 @@ static inline void dcp_fabric_run_capacity(bool available, void (*notify)(void * notify(ctx); } +static inline bool dcp_fabric_hdmi_settle_enabled(bool fixed, unsigned int routes, bool dual) +{ + return fixed && routes && !dual; +} + struct dcp_fabric_resume_ops { void (*enable_irq)(void *ctx); void (*sample)(void *ctx); @@ -43,10 +48,16 @@ struct dcp_fabric_resume_sample_ops { }; static inline void -dcp_fabric_run_resume_sample(const struct dcp_fabric_resume_sample_ops *ops, void *ctx) +dcp_fabric_run_resume_sample(bool enabled, + const struct dcp_fabric_resume_sample_ops *ops, void *ctx) { - u64 generation = ops->edge(ctx); - int level = ops->read_hpd(ctx); + u64 generation; + int level; + + if (!enabled) + return; + generation = ops->edge(ctx); + level = ops->read_hpd(ctx); ops->sample(ctx, generation, level); } diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c index a914d911bade7..40c8341c9b443 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-test.c +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -775,7 +775,7 @@ struct fabric_fake { struct dcp_fabric_route *best; enum fabric_fake_effect trace[64]; unsigned int count, reads, wait_ms, plans, attempts[2], releasing; - unsigned int failed_activations; + unsigned int failed_activations, nr_typec_routes; int levels[3], activate_error, deactivate_error; bool borrowed, frozen, connector, irq_enabled, inject_edge; bool selected, fixed_live, has_xbar, t6030, t6020_xbar, persistent; @@ -787,6 +787,7 @@ static void fabric_fake_init(struct fabric_fake *f, struct kunit *test, bool dua f->test = test; f->connector = true; f->has_xbar = true; + f->nr_typec_routes = 3; fabric_init(&f->f, dual); } @@ -920,7 +921,11 @@ static const struct dcp_fabric_resume_sample_ops fabric_fake_resume_sample_ops = static void fabric_fake_resume_sample(void *ctx) { - dcp_fabric_run_resume_sample(&fabric_fake_resume_sample_ops, ctx); + struct fabric_fake *f = ctx; + bool enabled = dcp_fabric_hdmi_settle_enabled(f->f.pipeline[0].has_fixed, + f->nr_typec_routes, f->f.policy.dual_stream); + + dcp_fabric_run_resume_sample(enabled, &fabric_fake_resume_sample_ops, ctx); } static const struct dcp_fabric_resume_ops fabric_fake_resume_ops = { @@ -1451,16 +1456,41 @@ static void fabric_resume_no_connect_test(struct kunit *test) f.borrowed = !!(row & 2); f.levels[0] = !!(row & 1); dcp_fabric_run_resume(&fabric_fake_resume_ops, &f); - KUNIT_ASSERT_EQ(test, f.count, 2U); + KUNIT_ASSERT_EQ(test, f.count, (row & 4) ? 1U : 2U); + KUNIT_EXPECT_EQ(test, f.trace[0], FX_IRQ_ENABLE); + if (!(row & 4)) { + KUNIT_EXPECT_EQ(test, f.trace[1], FX_SAMPLE); + KUNIT_EXPECT_EQ(test, f.presence.state, + (row & 1) ? DCP_FABRIC_PRESENT : DCP_FABRIC_SETTLING); + } else { + KUNIT_EXPECT_EQ(test, f.presence.state, DCP_FABRIC_ABSENT); + } + } +} + +static void fabric_nonhybrid_hdmi_resume_test(struct kunit *test) +{ + /* HDMI-only pipelines and Type-C-only pipelines never arm the hold. */ + for (unsigned int row = 0; row < 6; row++) { + struct fabric_fake f; + + fabric_fake_init(&f, test, row == 5); + f.nr_typec_routes = row < 4 ? 0 : 3; + f.f.pipeline[0].has_fixed = row != 4; + f.levels[0] = 1; + dcp_fabric_run_resume(&fabric_fake_resume_ops, &f); + KUNIT_ASSERT_EQ(test, f.count, 1U); KUNIT_EXPECT_EQ(test, f.trace[0], FX_IRQ_ENABLE); - KUNIT_EXPECT_EQ(test, f.trace[1], FX_SAMPLE); - KUNIT_EXPECT_EQ(test, f.presence.state, - (row & 1) ? DCP_FABRIC_PRESENT : DCP_FABRIC_SETTLING); + KUNIT_EXPECT_EQ(test, f.reads, 0U); + KUNIT_EXPECT_EQ(test, f.presence.state, DCP_FABRIC_ABSENT); + KUNIT_EXPECT_EQ(test, f.presence.generation, 0ULL); + KUNIT_EXPECT_EQ(test, f.presence.deadline, 0UL); } } static struct kunit_case fabric_tests[] = { KUNIT_CASE_PARAM(fabric_shared_wiring_test, fabric_wiring_gen_params), + KUNIT_CASE(fabric_nonhybrid_hdmi_resume_test), KUNIT_CASE(fabric_resume_no_connect_test), KUNIT_CASE(fabric_connector_flow_core_test), KUNIT_CASE(fabric_capacity_notification_test), diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 3141bdb62caeb..ae698be8167c7 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -492,11 +492,20 @@ static bool dcp_typec_keep_order(void) static void dcp_typec_pipeline_freed(void); static int dcp_fixed_output_select(struct apple_dcp *dcp); +static bool dcp_hdmi_settle_enabled(struct apple_dcp *dcp) +{ + return dcp->hdmi_hpd && + dcp_fabric_hdmi_settle_enabled(!!dcp->fixed_phy, dcp->nr_typec_routes, + dcp_typec_dual_stream()); +} + static enum dcp_fabric_presence_state dcp_hdmi_presence(struct apple_dcp *dcp) { enum dcp_fabric_presence_state state; unsigned long flags; + if (!dcp_hdmi_settle_enabled(dcp)) + return DCP_FABRIC_ABSENT; spin_lock_irqsave(&dcp->hdmi_presence_lock, flags); state = dcp->hdmi_presence.state; spin_unlock_irqrestore(&dcp->hdmi_presence_lock, flags); @@ -509,6 +518,8 @@ static u64 dcp_hdmi_edge(struct apple_dcp *dcp) unsigned long flags; u64 generation; + if (!dcp_hdmi_settle_enabled(dcp)) + return 0; spin_lock_irqsave(&dcp->hdmi_presence_lock, flags); generation = dcp_fabric_presence_edge(&dcp->hdmi_presence, jiffies, msecs_to_jiffies(DCP_HDMI_HOLD_MS)); @@ -521,7 +532,7 @@ static void dcp_hdmi_schedule(struct apple_dcp *dcp) unsigned long flags, delay = 0; bool settling; - if (dcp_typec_dual_stream()) + if (!dcp_hdmi_settle_enabled(dcp)) return; spin_lock_irqsave(&dcp->hdmi_presence_lock, flags); settling = dcp->hdmi_presence.state == DCP_FABRIC_SETTLING; @@ -539,6 +550,8 @@ static bool dcp_hdmi_sample(struct apple_dcp *dcp, u64 generation, int level) unsigned long flags; bool accepted; + if (!dcp_hdmi_settle_enabled(dcp)) + return false; /* A failed GPIO read cannot establish absence. Keep a full guard. */ spin_lock_irqsave(&dcp->hdmi_presence_lock, flags); accepted = dcp_fabric_presence_sample(&dcp->hdmi_presence, generation, @@ -564,7 +577,7 @@ static void dcp_hdmi_settle_work(struct work_struct *work) int level; guard(mutex)(&dcp_typec_fabric_lock); - if (!dcp->hdmi_hpd || dcp_typec_dual_stream()) + if (!dcp_hdmi_settle_enabled(dcp)) return; spin_lock_irqsave(&dcp->hdmi_presence_lock, flags); generation = dcp->hdmi_presence.generation; @@ -595,7 +608,7 @@ static void dcp_hdmi_recheck_work(struct work_struct *work) /* Oneshot unmask precedes thread completion. Never wait under fabric. */ synchronize_irq(dcp->hdmi_hpd_irq); guard(mutex)(&dcp_typec_fabric_lock); - if (dcp_typec_dual_stream()) + if (!dcp_hdmi_settle_enabled(dcp)) return; spin_lock_irqsave(&dcp->hdmi_presence_lock, flags); generation = dcp->hdmi_presence.generation; @@ -647,8 +660,10 @@ static const struct dcp_fabric_resume_sample_ops dcp_resume_sample_ops = { void dcp_fabric_hdmi_resume(struct apple_dcp *dcp) { + if (!dcp_hdmi_settle_enabled(dcp)) + return; guard(mutex)(&dcp_typec_fabric_lock); - dcp_fabric_run_resume_sample(&dcp_resume_sample_ops, dcp); + dcp_fabric_run_resume_sample(true, &dcp_resume_sample_ops, dcp); } /* A Thunderbolt tunnel holds @dcp's pipeline: it never moves. */ @@ -2297,11 +2312,13 @@ static void dcp_hdmi_update_locked(struct apple_dcp *dcp) int level; lockdep_assert_held(&dcp_typec_fabric_lock); - spin_lock_irqsave(&dcp->hdmi_presence_lock, flags); - generation = dcp->hdmi_presence.generation; - spin_unlock_irqrestore(&dcp->hdmi_presence_lock, flags); - level = gpiod_get_value_cansleep(dcp->hdmi_hpd); - dcp_hdmi_sample(dcp, generation, level); + if (dcp_hdmi_settle_enabled(dcp)) { + spin_lock_irqsave(&dcp->hdmi_presence_lock, flags); + generation = dcp->hdmi_presence.generation; + spin_unlock_irqrestore(&dcp->hdmi_presence_lock, flags); + level = gpiod_get_value_cansleep(dcp->hdmi_hpd); + dcp_hdmi_sample(dcp, generation, level); + } dcp_fabric_run_hdmi(&dcp_hdmi_ops, dcp); } @@ -2312,7 +2329,7 @@ irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) guard(mutex)(&dcp_typec_fabric_lock); dcp_hdmi_update_locked(dcp); /* The GPIO edge detector is off throughout an oneshot handler. */ - if (!dcp_typec_dual_stream()) + if (dcp_hdmi_settle_enabled(dcp)) mod_delayed_work(system_freezable_wq, &dcp->hdmi_recheck_wq, 0); return IRQ_HANDLED; } From 344c2fcb4c61bac39f51eb4b4cae0ab990573a18 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 01:52:43 -0700 Subject: [PATCH 1596/2680] drm/apple: Refresh HDMI presence after component bind Reuse the resume sampling sequence when enabling HDMI HPD after bind. Enable the IRQ first, then refresh the hybrid presence state so work drained during unbind cannot leave an expired settling hold behind. A high sample clears the hold; a low sample starts a full window. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 253f6eb68f645732e44ab2555e212ea68d51dcc5) --- drivers/gpu/drm/apple/dcp-fabric-test.c | 29 ++++++++++++++++++++ drivers/gpu/drm/apple/dcp.c | 36 ++++++++++++------------- 2 files changed, 47 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c index 40c8341c9b443..3652a404fbcac 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-test.c +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -1488,8 +1488,37 @@ static void fabric_nonhybrid_hdmi_resume_test(struct kunit *test) } } +static void fabric_rebind_presence_test(struct kunit *test) +{ + for (unsigned int high = 0; high < 2; high++) { + struct fabric_fake f; + u64 previous; + bool expired; + + fabric_fake_init(&f, test, false); + previous = dcp_fabric_presence_edge(&f.presence, 0, 10000); + /* Unbind drained the expiry; bind re-enables work before this sample. */ + f.levels[0] = high; + dcp_fabric_run_resume(&fabric_fake_resume_ops, &f); + KUNIT_ASSERT_EQ(test, f.count, 2U); + KUNIT_EXPECT_EQ(test, f.trace[0], FX_IRQ_ENABLE); + KUNIT_EXPECT_EQ(test, f.trace[1], FX_SAMPLE); + KUNIT_EXPECT_EQ(test, f.presence.state, + high ? DCP_FABRIC_PRESENT : DCP_FABRIC_SETTLING); + KUNIT_EXPECT_EQ(test, f.presence.deadline, high ? 0UL : 11000UL); + expired = dcp_fabric_presence_expire(&f.presence, previous, false, 11000); + KUNIT_EXPECT_FALSE(test, expired); + if (!high) { + expired = dcp_fabric_presence_expire(&f.presence, + f.presence.generation, false, 11000); + KUNIT_EXPECT_TRUE(test, expired); + } + } +} + static struct kunit_case fabric_tests[] = { KUNIT_CASE_PARAM(fabric_shared_wiring_test, fabric_wiring_gen_params), + KUNIT_CASE(fabric_rebind_presence_test), KUNIT_CASE(fabric_nonhybrid_hdmi_resume_test), KUNIT_CASE(fabric_resume_no_connect_test), KUNIT_CASE(fabric_connector_flow_core_test), diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 880b0ee5b0517..960387b5f972b 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1796,6 +1796,23 @@ static void _dcp_poweroff(struct apple_dcp *dcp) } } +static void dcp_resume_enable_irq(void *ctx) +{ + struct apple_dcp *dcp = ctx; + + enable_irq(dcp->hdmi_hpd_irq); +} + +static void dcp_resume_sample(void *ctx) +{ + dcp_fabric_hdmi_resume(ctx); +} + +static const struct dcp_fabric_resume_ops dcp_resume_ops = { + .enable_irq = dcp_resume_enable_irq, + .sample = dcp_resume_sample, +}; + static int dcp_enable_dp2hdmi_hpd(struct apple_dcp *dcp) { if (dcp_is_typec_output(dcp)) { @@ -1813,7 +1830,7 @@ static int dcp_enable_dp2hdmi_hpd(struct apple_dcp *dcp) } if (dcp->hdmi_hpd_irq) - enable_irq(dcp->hdmi_hpd_irq); + dcp_fabric_run_resume(&dcp_resume_ops, dcp); return 0; } @@ -3503,23 +3520,6 @@ static int dcp_platform_suspend(struct device *dev) return 0; } -static void dcp_resume_enable_irq(void *ctx) -{ - struct apple_dcp *dcp = ctx; - - enable_irq(dcp->hdmi_hpd_irq); -} - -static void dcp_resume_sample(void *ctx) -{ - dcp_fabric_hdmi_resume(ctx); -} - -static const struct dcp_fabric_resume_ops dcp_resume_ops = { - .enable_irq = dcp_resume_enable_irq, - .sample = dcp_resume_sample, -}; - static int dcp_platform_resume(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); From 4bca107b664daeff1ff65c34942ed152d5a7b69b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:49:20 -0700 Subject: [PATCH 1597/2680] drm/apple: validate display catalogs and bound sink data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reject incomplete timing and color records before deriving DRM modes, while preserving strict-highest score selection and the first record on ties. Validate both axes, require the consumed vertical rate, and bound timing arithmetic and serialized property extents. Serialize mode replacement and copy selected modes before waiting. Admit a catalog only after successful parsing for the current attachment; preserve an admitted catalog across failed updates and link recovery. Keep the v14 panel mode and fixed external scanout policies unchanged. Invalidate cached EDID on sink and route changes, reject stale in-flight results, and copy only complete blocks within the received reply. Add serialized parser, catalog admission, property bound, and EDID lifetime tests using the production helpers. [ryan: adapted to 7.2.10 — parser.c keeps our bounded parser (iterator_begin walks, depth-limited skip_value, extent checks) and takes only the u64 clock arithmetic, the colour mode id bound, and a horizontal PreciseSyncRate of 0 (parse_mode still requires the vertical one); modes_lock/modes_admitted are layered on our retained-array catalogue: dcp_modes_replace() carries our ticketed H17P publication (H17P external still publishes an empty catalogue on a failed parse; other paths keep the admitted one), dcp_modes_begin_attachment() is a no-op on H17P external, lookup_mode() copies the mode out after our 180 Hz eligibility check; dcp_get_modes() does not unlock modes_lock, which dcp_modes_snapshot() takes and releases with guard(); get_modes keeps our generation-fenced EDID and only honours admission; the connector EDID helpers become invalidate/set_live/set_pipeline wrappers over our edid_invalidation_pending (no edid_lock), set_live runs after the H17P external hotplug hand-off; dpavservep keeps our dpavserv-edid.h, the legacy EDID read gains the reply-length bound; dpavservep-edid.h, its test and connector-edid-test.c (which tests the replaced helpers) are dropped; prop_start keeps our 26.0 1 MiB bound beside the new 16 MiB one] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit ac0d314eda0cae009c9bdb37a44e7c0422c1cd26) --- drivers/gpu/drm/apple/Kconfig | 10 + drivers/gpu/drm/apple/Makefile | 2 + drivers/gpu/drm/apple/apple_drv.c | 4 +- drivers/gpu/drm/apple/connector.c | 33 ++ drivers/gpu/drm/apple/connector.h | 8 + drivers/gpu/drm/apple/dcp-fabric.c | 8 +- drivers/gpu/drm/apple/dcp-internal.h | 3 + drivers/gpu/drm/apple/dcp-property.h | 26 ++ drivers/gpu/drm/apple/dcp.c | 6 +- drivers/gpu/drm/apple/epic/dpavservep.c | 15 +- drivers/gpu/drm/apple/iomfb.c | 104 ++++- drivers/gpu/drm/apple/iomfb_internal.h | 4 +- drivers/gpu/drm/apple/iomfb_template.c | 79 +--- drivers/gpu/drm/apple/iomfb_v14_7.c | 20 +- drivers/gpu/drm/apple/parser-test.c | 557 ++++++++++++++++++++++++ drivers/gpu/drm/apple/parser.c | 29 +- drivers/gpu/drm/apple/parser.h | 8 + 17 files changed, 826 insertions(+), 90 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp-property.h create mode 100644 drivers/gpu/drm/apple/parser-test.c diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 3822c12500205..5cc314ef94406 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -57,3 +57,13 @@ config DRM_APPLE_PARSER_KUNIT_TEST legacy EPIC replies, and H17P IOMFB callback envelopes and sizes. The tests use allocated buffers and fake devices and do not require an Apple display controller or any firmware. + +config DRM_APPLE_DISPLAY_KUNIT_TEST + bool "KUnit tests for Apple display mode probing" if !KUNIT_ALL_TESTS + depends on KUNIT && DRM_APPLE + default KUNIT_ALL_TESTS + help + Test the production timing parser and display reply bounds without + Apple display hardware, including incomplete records and stable + preferred mode selection. Exercise mode list replacement and + catalog admission for each display attachment. diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 0e6d991332113..f09d27d6ab2d6 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -37,3 +37,5 @@ obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp-announce-test.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk-service-test.o obj-$(CONFIG_DRM_APPLE_PARSER_KUNIT_TEST) += afkv2-test.o + +appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += parser-test.o diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 04234d75f4334..147dae4f14825 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -400,6 +400,7 @@ static int apple_connector_create(struct drm_device *drm, return -ENOMEM; mutex_init(&connector->chunk_lock); + apple_connector_edid_init(connector); apple_connector_backlight_init(connector); /* before anything can fail: cleanup cancels it */ INIT_WORK(&connector->hotplug_wq, dcp_hotplug); @@ -418,7 +419,7 @@ static int apple_connector_create(struct drm_device *drm, connector->base.polled = DRM_CONNECTOR_POLL_HPD; connector->connected = false; - connector->dcp = dcp; + apple_connector_set_pipeline(connector, dcp); ret = drm_connector_attach_encoder(&connector->base, &encoder->base); if (ret) @@ -581,6 +582,7 @@ static int apple_probe_typec_ports(struct drm_device *drm, return -ENOMEM; mutex_init(&connector->chunk_lock); + apple_connector_edid_init(connector); apple_connector_backlight_init(connector); /* before anything can fail: cleanup cancels it */ INIT_WORK(&connector->hotplug_wq, dcp_hotplug); diff --git a/drivers/gpu/drm/apple/connector.c b/drivers/gpu/drm/apple/connector.c index 27082dc21038e..8b4e225fbe8c9 100644 --- a/drivers/gpu/drm/apple/connector.c +++ b/drivers/gpu/drm/apple/connector.c @@ -16,6 +16,39 @@ #include "dcp-internal.h" +void apple_connector_edid_init(struct apple_connector *connector) +{ + connector->edid_live = false; + connector->edid_invalidation_pending = false; +} + +/* + * The EDID cache is generation fenced and owned under mode_config.mutex: + * mark it stale here, and the hotplug work or the next get_modes() clears it. + */ +void apple_connector_invalidate_edid(struct apple_connector *connector) +{ + if (connector) + WRITE_ONCE(connector->edid_invalidation_pending, true); +} + +void apple_connector_edid_set_live(struct apple_connector *connector, bool live) +{ + if (!connector) + return; + if (!live || READ_ONCE(connector->edid_live) != live) + apple_connector_invalidate_edid(connector); + WRITE_ONCE(connector->edid_live, live); +} + +void apple_connector_set_pipeline(struct apple_connector *connector, + struct platform_device *pdev) +{ + apple_connector_invalidate_edid(connector); + WRITE_ONCE(connector->edid_live, !!pdev); + WRITE_ONCE(connector->dcp, pdev); +} + enum dcp_chunk_type { DCP_CHUNK_COLOR_ELEMENTS, DCP_CHUNK_TIMING_ELELMENTS, diff --git a/drivers/gpu/drm/apple/connector.h b/drivers/gpu/drm/apple/connector.h index 5b908fb85520d..581aa54efe68d 100644 --- a/drivers/gpu/drm/apple/connector.h +++ b/drivers/gpu/drm/apple/connector.h @@ -51,6 +51,8 @@ struct apple_connector { const struct drm_edid *drm_edid; bool drm_edid_firmware_derived; bool edid_invalidation_pending; + /* The sink is live on this pipeline; a change invalidates its EDID. */ + bool edid_live; u64 drm_edid_generation; u64 drm_edid_service_epoch; @@ -83,6 +85,12 @@ struct apple_connector { #define to_apple_connector(x) container_of(x, struct apple_connector, base) +void apple_connector_edid_init(struct apple_connector *connector); +void apple_connector_invalidate_edid(struct apple_connector *connector); +void apple_connector_edid_set_live(struct apple_connector *connector, bool live); +void apple_connector_set_pipeline(struct apple_connector *connector, + struct platform_device *pdev); + void apple_connector_debugfs_init(struct drm_connector *connector, struct dentry *root); void dcp_connector_update_dict(struct apple_connector *connector, const char *key, diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index ae698be8167c7..f06b5961c9121 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -24,6 +24,7 @@ #include "dcpext_scanout.h" #include "ibootep.h" #include "dcp-internal.h" +#include "parser.h" struct apple_dcp_typec_port { struct dcp_fabric_port core; struct dcp_fabric_plan plan; @@ -249,12 +250,13 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, return ret; } + dcp_modes_begin_attachment(dcp); dcp->phy = route->phy; dcp->dptx_phy = route->dptx_phy; dcp->connector_type = DRM_MODE_CONNECTOR_USB; WRITE_ONCE(dcp->ext_backlight, false); if (connector) { - WRITE_ONCE(connector->dcp, to_platform_device(dcp->dev)); + apple_connector_set_pipeline(connector, to_platform_device(dcp->dev)); dcp->typec_connector = connector; dcp->connector = connector; @@ -374,7 +376,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) */ WRITE_ONCE(connector->connected, false); /* hotplug work queued before this checks for it */ - WRITE_ONCE(connector->dcp, NULL); + apple_connector_set_pipeline(connector, NULL); /* no pipeline, so no backlight */ schedule_work(&connector->bl_sync_wq); @@ -1929,7 +1931,7 @@ void dcp_typec_port_set_connector(unsigned int idx, bool secondary, drm_crtc_index(&dcp->crtc->base), connector->candidate_crtcs); - connector->dcp = to_platform_device(dcp->dev); + apple_connector_set_pipeline(connector, to_platform_device(dcp->dev)); dcp->typec_connector = connector; dcp->connector = connector; dcp->connector_type = DRM_MODE_CONNECTOR_USB; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 789b70f16cc62..71944ba0e97f7 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -368,8 +368,11 @@ struct apple_dcp { bool crc_enabled; /* Modes valid for the connected display */ + /* Readers copy a mode before dropping this lock. */ + struct mutex modes_lock; struct dcp_display_mode *modes; unsigned int nr_modes; + bool modes_admitted; /* Attributes of the connector */ int connector_type; diff --git a/drivers/gpu/drm/apple/dcp-property.h b/drivers/gpu/drm/apple/dcp-property.h new file mode 100644 index 0000000000000..08774131c11ed --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-property.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef _APPLE_DCP_PROPERTY_H +#define _APPLE_DCP_PROPERTY_H + +#include +#include +#include + +/* Bound the allocation for one property dictionary. */ +static inline bool dcp_property_size_valid(size_t size) +{ + return size && size <= SZ_16M; +} + +static inline bool dcp_property_chunk_valid(size_t size, u32 offset, + u32 length, size_t source_size) +{ + return length <= source_size && offset <= size && length <= size - offset; +} + +static inline bool dcp_property_key_valid(const char *key, size_t size) +{ + return memchr(key, '\0', size); +} + +#endif diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 960387b5f972b..c1b3df1a7e145 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -567,10 +567,12 @@ void dcp_set_dimensions(struct apple_dcp *dcp) * TimingElements, dimensions are calculated when parsing * DisplayAttributes, and TimingElements may be sent first */ + mutex_lock(&dcp->modes_lock); for (i = 0; i < dcp->nr_modes; ++i) { dcp->modes[i].mode.width_mm = width_mm; dcp->modes[i].mode.height_mm = height_mm; } + mutex_unlock(&dcp->modes_lock); } void dcp_update_connector_dimensions(struct apple_dcp *dcp, @@ -764,8 +766,7 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ } if (dcp->connector) { dcp->connector->connected = 0; - drm_edid_free(dcp->connector->drm_edid); - dcp->connector->drm_edid = NULL; + apple_connector_edid_set_live(dcp->connector, false); schedule_work(&dcp->connector->hotplug_wq); } complete(&dcp->start_done); @@ -3057,6 +3058,7 @@ static int dcp_platform_probe(struct platform_device *pdev) * Type-C and Thunderbolt routes can be activated as soon as they are * registered below, before the DRM device binds. */ + mutex_init(&dcp->modes_lock); mutex_init(&dcp->hpd_mutex); mutex_init(&dcp->external_lock); INIT_WORK(&dcp->external_stop_work, dcp_external_stop_work); diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 39b808306f624..1a324f8fed277 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -313,6 +313,7 @@ static const struct drm_edid *dcpavserv_read_edid_legacy(struct apple_epic_servi struct dpavserv_copy_edid_cmd cmd; struct dpavserv_copy_edid_resp *resp __free(kfree) = NULL; size_t resp_size = sizeof(*resp) + EDID_BUF_SIZE; + size_t reply_size; int num_blocks; u64 data_size; int ret; @@ -323,20 +324,20 @@ static const struct drm_edid *dcpavserv_read_edid_legacy(struct apple_epic_servi if (!resp) return ERR_PTR(-ENOMEM); - ret = afk_service_call(service, 1, 7, &cmd, sizeof(cmd), EDID_BUF_SIZE, resp, - resp_size, 0); + ret = afk_service_call_with_reply_len(service, 1, 7, &cmd, sizeof(cmd), + EDID_BUF_SIZE, resp, resp_size, 0, + &reply_size); if (ret < 0) return ERR_PTR(ret); - if (le64_to_cpu(resp->max_size) != EDID_BUF_SIZE) + if (reply_size < sizeof(*resp) || + le64_to_cpu(resp->max_size) != EDID_BUF_SIZE) return ERR_PTR(-EIO); - // print_hex_dump(KERN_DEBUG, "dpavserv EDID cmd: ", DUMP_PREFIX_NONE, - // 16, 1, resp, 192, true); - + /* Only complete EDID blocks within the received reply may be copied. */ data_size = le64_to_cpu(resp->used_size); if (data_size < EDID_LEADING_DATA_SIZE + EDID_BLOCK_SIZE || - data_size > EDID_BUF_SIZE) + data_size > EDID_BUF_SIZE || data_size > reply_size - sizeof(*resp)) return ERR_PTR(-EIO); /* diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 8bf602a48c8ab..13c64d7a722c8 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1371,6 +1371,12 @@ static struct dcp_display_mode *dcp_modes_snapshot(struct apple_dcp *dcp, struct dcp_display_mode *modes; unsigned long flags; + /* An attachment sees no catalog until one is admitted for it. */ + guard(mutex)(&dcp->modes_lock); + if (!dcp->modes_admitted) { + *count = 0; + return NULL; + } spin_lock_irqsave(&dcp->metadata.lock, flags); modes = dcp->modes; *count = dcp->nr_modes; @@ -1378,6 +1384,82 @@ static struct dcp_display_mode *dcp_modes_snapshot(struct apple_dcp *dcp, return modes; } +void dcp_modes_begin_attachment(struct apple_dcp *dcp) +{ + /* H17P external catalogs are fenced by their connection receipt. */ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) + return; + guard(mutex)(&dcp->modes_lock); + dcp->modes_admitted = false; +} + +/* + * Publish a TimingElements catalog. Replaced arrays stay allocated, as + * readers may still hold a snapshot. A failed parse keeps an admitted + * catalog, except on H17P external, which publishes an empty one. + */ +int dcp_modes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *ctx) +{ + bool h17p_external = dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL; + struct dcp_display_mode *modes; + struct dcp_metadata_ticket rate_ticket = {}; + u64 rate_service_epoch = 0; + unsigned int nr_modes = 0, i; + unsigned long flags; + int ret = 0; + + if (h17p_external && dcp->chunks.payload_complete && + dcp->chunks.mode_ticket.associated) { + ctx->automatic_1080p180_rgb8 = true; + rate_ticket = dcp->chunks.mode_ticket; + rate_service_epoch = dcp->chunks.av_service_epoch; + } + modes = enumerate_modes(ctx, &nr_modes, dcp->width_mm, dcp->height_mm, + dcp->notch_height, + dcp->fixed_connector_type == DRM_MODE_CONNECTOR_eDP, + h17p_external); + if (IS_ERR(modes)) { + ret = PTR_ERR(modes); + if (!h17p_external) + return ret; + modes = NULL; + nr_modes = 0; + } else { + for (i = 0; i < nr_modes; i++) + if (modes[i].automatic_rgb8_quirk) { + modes[i].unsafe_rgb8_ticket = rate_ticket; + modes[i].unsafe_rgb8_service_epoch = rate_service_epoch; + modes[i].automatic_connection_sequence = dcp->chunks.automatic_connection_sequence; + modes[i].automatic_service_pending = dcp->chunks.automatic_service_pending; + } + } + + mutex_lock(&dcp->modes_lock); + /* Publish one catalog revision with its immutable receipt tickets. */ + spin_lock_irqsave(&dcp->metadata.lock, flags); + dcp->rate_proof.catalog_sequence++; + for (i = 0; i < nr_modes; i++) + if (modes[i].automatic_rgb8_quirk) + modes[i].automatic_catalog_sequence = dcp->rate_proof.catalog_sequence; + dcp->modes = modes; + dcp->nr_modes = nr_modes; + /* Only a complete receipt for a current cable is that cable's catalogue. */ + dcp->metadata.catalog_ticket = rate_ticket; + dcp->metadata.catalog_ticket.associated &= nr_modes > 0; + dcp->metadata.catalog_connection_sequence = + dcp->chunks.automatic_connection_sequence; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + dcp->modes_admitted = !ret; + mutex_unlock(&dcp->modes_lock); + + if (!ret && !h17p_external) + apple_connector_invalidate_edid(dcp->connector); + /* Notify removals as well as newly eligible modes, outside the lock. */ + dcp_180hz_catalog_changed(dcp); + return ret; +} + /* Caller holds hpd_mutex -> dcpavserv.lock -> metadata.lock. */ static bool dcp_exception_mode_current_locked(struct apple_dcp *dcp, const struct dcp_display_mode *mode) @@ -1600,8 +1682,8 @@ int dcp_get_modes(struct drm_connector *connector) } /* The user may own drm_display_mode, so we need to search for our copy */ -struct dcp_display_mode *lookup_mode(struct apple_dcp *dcp, - const struct drm_display_mode *mode) +bool lookup_mode(struct apple_dcp *dcp, const struct drm_display_mode *mode, + struct dcp_display_mode *out) { struct dcp_display_mode *modes; unsigned int count, i; @@ -1614,7 +1696,7 @@ struct dcp_display_mode *lookup_mode(struct apple_dcp *dcp, DRM_MODE_MATCH_TIMINGS | DRM_MODE_MATCH_CLOCK)) { if (!modes[i].automatic_rgb8_quirk) - return &modes[i]; + goto found; mutex_lock(&dcp->hpd_mutex); mutex_lock(&dcp->dcpavserv.lock); spin_lock_irqsave(&dcp->metadata.lock, flags); @@ -1623,11 +1705,17 @@ struct dcp_display_mode *lookup_mode(struct apple_dcp *dcp, mutex_unlock(&dcp->dcpavserv.lock); mutex_unlock(&dcp->hpd_mutex); if (eligible) - return &modes[i]; + goto found; } } - return NULL; + return false; + +found: + /* Callers keep a copy, never a pointer into the catalog. */ + if (out) + *out = modes[i]; + return true; } /* @@ -1640,15 +1728,15 @@ struct dcp_display_mode *lookup_mode(struct apple_dcp *dcp, static bool dcp_mode_settable(struct apple_dcp *dcp, const struct drm_display_mode *mode) { - struct dcp_display_mode *dcp_mode = lookup_mode(dcp, mode); + struct dcp_display_mode dcp_mode; - if (!dcp_mode) + if (!lookup_mode(dcp, mode, &dcp_mode)) return false; if (dcp->fw_compat == DCP_FIRMWARE_H17P && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL) - return dcp_mode->mode.type & DRM_MODE_TYPE_PREFERRED; + return dcp_mode.mode.type & DRM_MODE_TYPE_PREFERRED; return true; } diff --git a/drivers/gpu/drm/apple/iomfb_internal.h b/drivers/gpu/drm/apple/iomfb_internal.h index 75e9d7b0e8cc8..bd55bfd475f3e 100644 --- a/drivers/gpu/drm/apple/iomfb_internal.h +++ b/drivers/gpu/drm/apple/iomfb_internal.h @@ -110,5 +110,5 @@ int dcp_parse_tag(char tag[4]); void dcp_ack(struct apple_dcp *dcp, enum dcp_context_id context); /* The user may own drm_display_mode, so we need to search for our copy */ -struct dcp_display_mode *lookup_mode(struct apple_dcp *dcp, - const struct drm_display_mode *mode); +bool lookup_mode(struct apple_dcp *dcp, const struct drm_display_mode *mode, + struct dcp_display_mode *out); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index f1561b0fe7884..77c9442ecaf36 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -31,6 +31,7 @@ #include "iomfb.h" #include "iomfb_internal.h" #include "parser.h" +#include "dcp-property.h" #include "trace.h" #include "version_utils.h" @@ -946,6 +947,9 @@ static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) } #endif + if (!dcp_property_size_valid(*length)) + return false; + dcp->chunks.length = *length; dcp->chunks.av_service_epoch = 0; dcp->chunks.automatic_service_pending = false; @@ -1021,9 +1025,6 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, dcp_property_complete(dcp->chunks.data, dcp->chunks.received, dcp->chunks.length); - /* used just as opaque pointer for tracing */ - ctx.dcp = dcp; - ret = parse(dcp->chunks.data, dcp->chunks.length, &ctx); if (ret) { @@ -1042,61 +1043,17 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, return false; } - if (!strcmp(req->key, "TimingElements")) { - struct dcp_display_mode *modes; - struct dcp_metadata_ticket rate_ticket = {}; - u64 rate_service_epoch = 0; - unsigned int nr_modes = 0, i; - unsigned long flags; + /* parse() resets the context: set the tracing pointer afterwards. */ + ctx.dcp = dcp; - if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && - dcp->chunks.payload_complete && dcp->chunks.mode_ticket.associated) { - ctx.automatic_1080p180_rgb8 = true; - rate_ticket = dcp->chunks.mode_ticket; - rate_service_epoch = dcp->chunks.av_service_epoch; - } - modes = enumerate_modes(&ctx, &nr_modes, - dcp->width_mm, dcp->height_mm, - dcp->notch_height, - dcp->fixed_connector_type == - DRM_MODE_CONNECTOR_eDP, - dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL); - - if (IS_ERR(modes)) { + if (!strcmp(req->key, "TimingElements")) { + ret = dcp_modes_replace(dcp, &ctx); + if (ret) { dev_warn(dcp->dev, "failed to parse modes\n"); - modes = NULL; - nr_modes = 0; - } else { - for (i = 0; i < nr_modes; i++) - if (modes[i].automatic_rgb8_quirk) { - modes[i].unsafe_rgb8_ticket = rate_ticket; - modes[i].unsafe_rgb8_service_epoch = rate_service_epoch; - modes[i].automatic_connection_sequence = dcp->chunks.automatic_connection_sequence; - modes[i].automatic_service_pending = dcp->chunks.automatic_service_pending; - } - } - /* Publish one catalog revision with its immutable receipt tickets. */ - spin_lock_irqsave(&dcp->metadata.lock, flags); - dcp->rate_proof.catalog_sequence++; - for (i = 0; i < nr_modes; i++) - if (modes[i].automatic_rgb8_quirk) - modes[i].automatic_catalog_sequence = dcp->rate_proof.catalog_sequence; - dcp->modes = modes; - dcp->nr_modes = nr_modes; - /* Only a complete receipt for a current cable is that cable's catalogue. */ - dcp->metadata.catalog_ticket = rate_ticket; - dcp->metadata.catalog_ticket.associated &= nr_modes > 0; - dcp->metadata.catalog_connection_sequence = - dcp->chunks.automatic_connection_sequence; - spin_unlock_irqrestore(&dcp->metadata.lock, flags); - /* Notify removals as well as newly eligible modes, outside the lock. */ - dcp_180hz_catalog_changed(dcp); - if (!modes) { dcp_firmware_metadata_timing(dcp); return false; } - if (nr_modes == 0) + if (!READ_ONCE(dcp->nr_modes)) dev_warn(dcp->dev, "TimingElements without valid modes!\n"); dcp_180hz_notify(dcp); dcp_firmware_metadata_timing(dcp); @@ -1140,7 +1097,15 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, static u8 dcpep_cb_prop_end(struct apple_dcp *dcp, struct dcp_set_dcpav_prop_end_req *req) { - u8 resp = dcpep_process_chunks(dcp, req); + u8 resp; + + if (!dcp_property_key_valid(req->key, sizeof(req->key))) { + kfree(dcp->chunks.data); + dcp->chunks.data = NULL; + dcp->chunks.length = 0; + return false; + } + resp = dcpep_process_chunks(dcp, req); if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_eDP) dev_dbg(dcp->dev, @@ -2007,6 +1972,7 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) return; dcp_firmware_metadata_hotplug(dcp, !!(*connected)); + apple_connector_edid_set_live(connector, !!(*connected)); action = dcp_mode_hotplug(&dcp->mode_state, !!(*connected), connector ? &connector->connected : NULL); /* @@ -2317,13 +2283,12 @@ static void dcp_set_adaptive_sync(struct apple_dcp *dcp, u32 min_vrr, int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, struct drm_crtc_state *crtc_state) { - struct dcp_display_mode *mode; + struct dcp_display_mode selected, *mode = &selected; struct dcp_wait_cookie *cookie; struct dcp_color_mode *cmode = NULL; int ret; - mode = lookup_mode(dcp, &crtc_state->mode); - if (!mode) { + if (!lookup_mode(dcp, &crtc_state->mode, mode)) { dev_err(dcp->dev, "no match for " DRM_MODE_FMT "\n", DRM_MODE_ARG(&crtc_state->mode)); return -EIO; diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 2cfaf67b19b93..e68565e5eb4e5 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -1283,7 +1283,7 @@ DEFINE_SHOW_ATTRIBUTE(dcp_v14_status); /* One mode: the native timing of the boot panel, less the hidden notch rows. */ static int dcp_v14_mode(struct apple_dcp *dcp, struct apple_dcp_v14 *v14) { - struct dcp_display_mode *modes, *best = NULL; + struct dcp_display_mode *modes, *best = NULL, *selected; struct dcp_parse_ctx ctx; unsigned int count, i; void *blob = NULL; @@ -1327,18 +1327,24 @@ static int dcp_v14_mode(struct apple_dcp *dcp, struct apple_dcp_v14 *v14) return -EINVAL; } best->mode.type |= DRM_MODE_TYPE_PREFERRED; - dcp->modes = kmemdup(best, sizeof(*best), GFP_KERNEL); + selected = kmemdup(best, sizeof(*best), GFP_KERNEL); kfree(modes); - if (!dcp->modes) + if (!selected) return -ENOMEM; + mutex_lock(&dcp->modes_lock); + kfree(dcp->modes); + dcp->modes = selected; dcp->nr_modes = 1; + dcp->modes_admitted = true; + mutex_unlock(&dcp->modes_lock); return 0; } int iomfb_v14_7_start(struct apple_dcp *dcp) { struct apple_dcp_v14 *v14 = dcp->v14; - const struct drm_display_mode *mode; + struct drm_display_mode selected; + const struct drm_display_mode *mode = &selected; const char *step; int ret; @@ -1379,7 +1385,9 @@ int iomfb_v14_7_start(struct apple_dcp *dcp) dcp->active = true; complete(&dcp->start_done); - mode = &dcp->modes[0].mode; + mutex_lock(&dcp->modes_lock); + selected = dcp->modes[0].mode; + mutex_unlock(&dcp->modes_lock); dev_info(dcp->dev, "T6030 display started: %ux%u@%d, %u notch rows %s, %ux%u mm\n", mode->hdisplay, mode->vdisplay, drm_mode_vrefresh(mode), dcp->notch_height ?: v14->panel_height - v14->fb_height, @@ -1537,7 +1545,7 @@ int iomfb_v14_7_atomic_check(struct apple_dcp *dcp, struct drm_crtc *crtc, int iomfb_v14_7_modeset(struct apple_dcp *dcp, struct drm_crtc_state *crtc_state) { /* The firmware keeps the timing it booted with, which is the only mode. */ - if (!lookup_mode(dcp, &crtc_state->mode)) + if (!lookup_mode(dcp, &crtc_state->mode, NULL)) return -EINVAL; scoped_guard(spinlock_irqsave, &dcp->mode_state.lock) dcp->mode_state.valid = true; diff --git a/drivers/gpu/drm/apple/parser-test.c b/drivers/gpu/drm/apple/parser-test.c new file mode 100644 index 0000000000000..5b9501fc8d7b6 --- /dev/null +++ b/drivers/gpu/drm/apple/parser-test.c @@ -0,0 +1,557 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include + +#include "dcp-internal.h" +#include "dcp-property.h" +#include "iomfb_internal.h" +#include "parser.h" + +static struct device parser_device = { + .init_name = "apple-display-parser-test", +}; + +static struct apple_dcp parser_dcp = { + .dev = &parser_device, +}; + +struct parser_blob { + u8 data[8192]; + u32 size; +}; + +struct parser_record { + s64 id, score, color_score, color_id; + s64 active, total, porch, sync, rate, horizontal_rate; + u32 missing, horizontal_missing, color_missing, second_color_missing; + s64 first_format; + bool bad_score_type; + bool vrr; +}; + +enum parser_missing { + MISSING_SCORE = BIT(0), + MISSING_ACTIVE = BIT(1), + MISSING_TOTAL = BIT(2), + MISSING_PORCH = BIT(3), + MISSING_SYNC = BIT(4), + MISSING_RATE = BIT(5), + MISSING_COLORS = BIT(6), + MISSING_COLOR_SCORE = BIT(0), + MISSING_COLOR_DEPTH = BIT(1), +}; + +static void blob_tag(struct parser_blob *blob, u32 type, u32 count) +{ + blob->size = ALIGN(blob->size, 4); + put_unaligned_le32(count | type << 24, blob->data + blob->size); + blob->size += 4; +} + +static void blob_key(struct parser_blob *blob, const char *key) +{ + size_t length = strlen(key); + + blob_tag(blob, 9, length); + memcpy(blob->data + blob->size, key, length); + blob->size += length; +} + +static void blob_int(struct parser_blob *blob, const char *key, s64 value) +{ + blob_key(blob, key); + blob_tag(blob, 4, 0); + put_unaligned_le64(value, blob->data + blob->size); + blob->size += 8; +} + +static void blob_bool(struct parser_blob *blob, const char *key, bool value) +{ + blob_key(blob, key); + blob_tag(blob, 11, value); +} + +static void blob_dimension(struct parser_blob *blob, const char *key, + const struct parser_record *record, bool vertical) +{ + u32 missing = vertical ? record->missing >> 1 & 0x1f : record->horizontal_missing; + + blob_key(blob, key); + blob_tag(blob, 1, 5 - hweight32(missing)); + if (!(missing & BIT(0))) + blob_int(blob, "Active", vertical ? record->active : 1920); + if (!(missing & BIT(1))) + blob_int(blob, "Total", vertical ? record->total : 2200); + if (!(missing & BIT(2))) + blob_int(blob, "FrontPorch", vertical ? record->porch : 88); + if (!(missing & BIT(3))) + blob_int(blob, "SyncWidth", vertical ? record->sync : 44); + if (!(missing & BIT(4))) + blob_int(blob, "PreciseSyncRate", + vertical ? record->rate : record->horizontal_rate); +} + +static void blob_color(struct parser_blob *blob, s64 id, s64 score, + u32 missing, s64 format) +{ + blob_tag(blob, 1, 8 - hweight32(missing)); + blob_bool(blob, "IsVirtual", false); + blob_int(blob, "ID", id); + if (!(missing & MISSING_COLOR_SCORE)) + blob_int(blob, "Score", score); + if (!(missing & MISSING_COLOR_DEPTH)) + blob_int(blob, "Depth", 8); + blob_int(blob, "Colorimetry", DCP_COLORIMETRY_RGB); + blob_int(blob, "DynamicRange", DCP_COLOR_YCBCR_RANGE_FULL); + blob_int(blob, "EOTF", DCP_EOTF_SDR_GAMMA); + blob_int(blob, "PixelEncoding", format); +} + +static void blob_record(struct parser_blob *blob, const struct parser_record *record) +{ + unsigned int count = 6 + (record->vrr ? 2 : 0); + + count -= !!(record->missing & MISSING_SCORE); + count -= !!(record->missing & MISSING_COLORS); + blob_tag(blob, 1, count); + blob_bool(blob, "IsVirtual", false); + blob_int(blob, "ID", record->id); + if (!(record->missing & MISSING_SCORE)) { + if (record->bad_score_type) { + blob_key(blob, "Score"); + blob_tag(blob, 9, 1); + blob->data[blob->size++] = 'x'; + } else { + blob_int(blob, "Score", record->score); + } + } + blob_dimension(blob, "HorizontalAttributes", record, false); + blob_dimension(blob, "VerticalAttributes", record, true); + if (!(record->missing & MISSING_COLORS)) { + blob_key(blob, "ColorModes"); + blob_tag(blob, 2, 2); + blob_color(blob, record->color_id, record->color_score, + record->color_missing, record->first_format); + blob_color(blob, 101, record->color_score, + record->second_color_missing, DCP_COLOR_FORMAT_RGB); + } + if (record->vrr) { + blob_int(blob, "MinimumVariableRefreshRate", 48 << 16); + blob_int(blob, "MaximumVariableRefreshRate", 120 << 16); + } +} + +static struct parser_record valid_record(s64 id, s64 score) +{ + return (struct parser_record) { + .id = id, .score = score, .color_score = 10, .color_id = 100, + .active = 1080, .total = 1125, .porch = 4, .sync = 5, + .rate = 60 << 16, .horizontal_rate = (675U << 16) / 10, + }; +} + +static struct parser_blob *make_blob(struct kunit *test, + const struct parser_record *records, unsigned int count) +{ + struct parser_blob *blob = kunit_kzalloc(test, sizeof(*blob), GFP_KERNEL); + unsigned int i; + + if (!blob) + return NULL; + put_unaligned_le32(0xd3, blob->data); + blob->size = 4; + blob_tag(blob, 2, count); + for (i = 0; i < count; i++) + blob_record(blob, &records[i]); + return blob; +} + +static struct dcp_display_mode *parse_blob(struct parser_blob *blob, + unsigned int *count, unsigned int notch, bool internal) +{ + struct dcp_parse_ctx ctx; + int ret = parse(blob->data, blob->size, &ctx); + + if (ret) + return ERR_PTR(ret); + ctx.dcp = &parser_dcp; + return enumerate_modes(&ctx, count, 500, 300, notch, internal, false); +} + +static void parser_score_presence(struct kunit *test) +{ + struct parser_record records[] = { valid_record(7, 100), valid_record(8, 0) }; + struct dcp_display_mode *modes; + struct parser_blob *blob; + unsigned int count; + + records[1].missing = MISSING_SCORE; + blob = make_blob(test, records, ARRAY_SIZE(records)); + KUNIT_ASSERT_NOT_NULL(test, blob); + modes = parse_blob(blob, &count, 0, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, modes); + KUNIT_EXPECT_EQ(test, count, 1U); + KUNIT_EXPECT_EQ(test, modes[0].timing_mode_id, 7U); + KUNIT_EXPECT_TRUE(test, modes[0].mode.type & DRM_MODE_TYPE_PREFERRED); + kfree(modes); + + records[0].missing = MISSING_SCORE; + records[1] = valid_record(8, 0); + blob = make_blob(test, records, ARRAY_SIZE(records)); + KUNIT_ASSERT_NOT_NULL(test, blob); + modes = parse_blob(blob, &count, 0, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, modes); + KUNIT_EXPECT_EQ(test, count, 1U); + KUNIT_EXPECT_EQ(test, modes[0].timing_mode_id, 8U); + kfree(modes); +} + +static void parser_legacy_ties(struct kunit *test) +{ + struct parser_record records[] = { valid_record(7, 50), valid_record(8, 50) }; + struct dcp_display_mode *modes; + struct parser_blob *blob; + unsigned int count, internal; + + for (internal = 0; internal < 2; internal++) { + blob = make_blob(test, records, ARRAY_SIZE(records)); + KUNIT_ASSERT_NOT_NULL(test, blob); + modes = parse_blob(blob, &count, 0, internal); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, modes); + KUNIT_EXPECT_EQ(test, count, 2U); + KUNIT_EXPECT_TRUE(test, modes[0].mode.type & DRM_MODE_TYPE_PREFERRED); + KUNIT_EXPECT_FALSE(test, modes[1].mode.type & DRM_MODE_TYPE_PREFERRED); + KUNIT_EXPECT_EQ(test, modes[0].color_mode_id, 100U); + kfree(modes); + } + records[1].score = 51; + blob = make_blob(test, records, ARRAY_SIZE(records)); + KUNIT_ASSERT_NOT_NULL(test, blob); + modes = parse_blob(blob, &count, 0, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, modes); + KUNIT_EXPECT_FALSE(test, modes[0].mode.type & DRM_MODE_TYPE_PREFERRED); + KUNIT_EXPECT_TRUE(test, modes[1].mode.type & DRM_MODE_TYPE_PREFERRED); + kfree(modes); +} + +static void parser_color_presence(struct kunit *test) +{ + struct parser_record record = valid_record(7, 10); + struct dcp_display_mode *modes; + struct parser_blob *blob; + unsigned int count, missing; + + for (missing = MISSING_COLOR_SCORE; missing <= MISSING_COLOR_DEPTH; missing <<= 1) { + record.color_missing = missing; + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_NOT_NULL(test, blob); + modes = parse_blob(blob, &count, 0, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, modes); + KUNIT_EXPECT_EQ(test, count, 1U); + KUNIT_EXPECT_EQ(test, modes[0].color_mode_id, 101U); + kfree(modes); + } + record.color_missing = 0; + record.first_format = DCP_COLOR_FORMAT_YCBCR420; + record.second_color_missing = MISSING_COLOR_SCORE; + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_NOT_NULL(test, blob); + modes = parse_blob(blob, &count, 0, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, modes); + KUNIT_EXPECT_EQ(test, count, 1U); + KUNIT_EXPECT_EQ(test, modes[0].color_mode_id, 100U); + KUNIT_EXPECT_EQ(test, modes[0].sdr_rgb.score, -1LL); + kfree(modes); + record.missing = MISSING_COLORS; + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_NOT_NULL(test, blob); + modes = parse_blob(blob, &count, 0, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, modes); + KUNIT_EXPECT_EQ(test, count, 0U); + kfree(modes); +} + +static void parser_geometry(struct kunit *test) +{ + struct parser_record record; + struct dcp_display_mode *modes; + struct parser_blob *blob; + unsigned int count, bit; + + for (bit = MISSING_ACTIVE; bit <= MISSING_RATE; bit <<= 1) { + record = valid_record(7, 10); + record.missing = bit; + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_NOT_NULL(test, blob); + modes = parse_blob(blob, &count, 0, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, modes); + KUNIT_EXPECT_EQ(test, count, 0U); + kfree(modes); + } +} + +static void parser_horizontal_rate(struct kunit *test) +{ + struct parser_record record = valid_record(7, 10); + struct dcp_display_mode *modes; + struct parser_blob *blob; + unsigned int count, bit; + + record.horizontal_missing = BIT(4); + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_NOT_NULL(test, blob); + modes = parse_blob(blob, &count, 0, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, modes); + KUNIT_EXPECT_EQ(test, count, 1U); + KUNIT_EXPECT_EQ(test, modes[0].mode.clock, 148500); + kfree(modes); + record.horizontal_missing = 0; + record.horizontal_rate = 0; + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_NOT_NULL(test, blob); + modes = parse_blob(blob, &count, 0, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, modes); + KUNIT_EXPECT_EQ(test, count, 1U); + KUNIT_EXPECT_EQ(test, modes[0].mode.clock, 148500); + kfree(modes); + record.horizontal_rate = -1; + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_NOT_NULL(test, blob); + modes = parse_blob(blob, &count, 0, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, modes); + KUNIT_EXPECT_EQ(test, count, 0U); + kfree(modes); + record.horizontal_rate = (675U << 16) / 10; + for (bit = BIT(0); bit <= BIT(3); bit <<= 1) { + record.horizontal_missing = bit; + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_NOT_NULL(test, blob); + modes = parse_blob(blob, &count, 0, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, modes); + KUNIT_EXPECT_EQ(test, count, 0U); + kfree(modes); + } +} + +static void parser_bad_score(struct kunit *test) +{ + struct parser_record records[] = { valid_record(7, 10), valid_record(8, 20) }; + struct dcp_display_mode *modes; + struct parser_blob *blob; + unsigned int count; + + records[0].bad_score_type = true; + blob = make_blob(test, records, ARRAY_SIZE(records)); + KUNIT_ASSERT_NOT_NULL(test, blob); + modes = parse_blob(blob, &count, 0, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, modes); + KUNIT_EXPECT_EQ(test, count, 1U); + KUNIT_EXPECT_EQ(test, modes[0].timing_mode_id, 8U); + kfree(modes); + records[0].bad_score_type = false; + records[0].score = -1; + blob = make_blob(test, records, ARRAY_SIZE(records)); + KUNIT_ASSERT_NOT_NULL(test, blob); + modes = parse_blob(blob, &count, 0, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, modes); + KUNIT_EXPECT_EQ(test, count, 1U); + KUNIT_EXPECT_EQ(test, modes[0].timing_mode_id, 8U); + kfree(modes); +} + +static void parser_bad_geometry(struct kunit *test) +{ + static const s64 values[][5] = { + { 0, 1125, 4, 5, 60 << 16 }, + { -1, 1125, 4, 5, 60 << 16 }, + { 1200, 1125, 4, 5, 60 << 16 }, + { 1080, 1125, -1, 5, 60 << 16 }, + { 1080, 1125, 4, -1, 60 << 16 }, + { 1080, 1125, 4, 5, 0 }, + { 1080, 1125, 4, 5, 1 }, + { 65536, 65536, 4, 5, 60 << 16 }, + { 1080, 65535, 4, 5, U32_MAX }, + }; + struct parser_record record; + struct dcp_display_mode *modes; + struct parser_blob *blob; + unsigned int count, i; + + for (i = 0; i < ARRAY_SIZE(values); i++) { + record = valid_record(7, 10); + record.active = values[i][0]; + record.total = values[i][1]; + record.porch = values[i][2]; + record.sync = values[i][3]; + record.rate = values[i][4]; + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_NOT_NULL(test, blob); + modes = parse_blob(blob, &count, 0, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, modes); + KUNIT_EXPECT_EQ(test, count, 0U); + kfree(modes); + } +} + +static void parser_notch_vrr(struct kunit *test) +{ + struct parser_record record = valid_record(7, 10); + struct dcp_display_mode *modes; + struct parser_blob *blob; + unsigned int count; + + record.rate = 120 << 16; + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_NOT_NULL(test, blob); + modes = parse_blob(blob, &count, 1080, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, modes); + KUNIT_EXPECT_EQ(test, count, 0U); + kfree(modes); + modes = parse_blob(blob, &count, 32, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, modes); + KUNIT_EXPECT_EQ(test, count, 1U); + KUNIT_EXPECT_EQ(test, modes[0].mode.vdisplay, 1048U); + KUNIT_EXPECT_TRUE(test, modes[0].vrr); + KUNIT_EXPECT_EQ(test, modes[0].min_vrr, 24U << 16); + kfree(modes); + modes = parse_blob(blob, &count, 0, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, modes); + KUNIT_EXPECT_FALSE(test, modes[0].vrr); + kfree(modes); + record.vrr = true; + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_NOT_NULL(test, blob); + modes = parse_blob(blob, &count, 0, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, modes); + KUNIT_EXPECT_EQ(test, modes[0].min_vrr, 48U << 16); + KUNIT_EXPECT_EQ(test, modes[0].max_vrr, 120U << 16); + kfree(modes); +} + +static void parser_replacement(struct kunit *test) +{ + struct parser_record record = valid_record(7, 10); + struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + struct dcp_display_mode selected, *previous; + struct dcp_parse_ctx ctx; + struct parser_blob *blob; + struct drm_display_mode requested; + + KUNIT_ASSERT_NOT_NULL(test, dcp); + mutex_init(&dcp->modes_lock); + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_NOT_NULL(test, blob); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + ctx.dcp = &parser_dcp; + KUNIT_ASSERT_EQ(test, dcp_modes_replace(dcp, &ctx), 0); + requested = dcp->modes[0].mode; + KUNIT_EXPECT_TRUE(test, lookup_mode(dcp, &requested, &selected)); + KUNIT_EXPECT_EQ(test, selected.timing_mode_id, 7U); + previous = dcp->modes; + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size - 1, &ctx), 0); + ctx.dcp = &parser_dcp; + KUNIT_EXPECT_LT(test, dcp_modes_replace(dcp, &ctx), 0); + KUNIT_EXPECT_PTR_EQ(test, dcp->modes, previous); + KUNIT_EXPECT_EQ(test, dcp->nr_modes, 1U); + record.id = 8; + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_NOT_NULL(test, blob); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + ctx.dcp = &parser_dcp; + KUNIT_ASSERT_EQ(test, dcp_modes_replace(dcp, &ctx), 0); + KUNIT_EXPECT_EQ(test, dcp->modes[0].timing_mode_id, 8U); + KUNIT_EXPECT_EQ(test, selected.timing_mode_id, 7U); + KUNIT_EXPECT_TRUE(test, lookup_mode(dcp, &requested, &selected)); + KUNIT_EXPECT_EQ(test, selected.timing_mode_id, 8U); + requested.clock++; + KUNIT_EXPECT_FALSE(test, lookup_mode(dcp, &requested, NULL)); + kfree(dcp->modes); +} + +static void parser_attachment_admission(struct kunit *test) +{ + struct parser_record record = valid_record(7, 10); + struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + struct dcp_display_mode selected, *previous; + struct dcp_parse_ctx ctx; + struct parser_blob *blob; + struct drm_display_mode requested; + + KUNIT_ASSERT_NOT_NULL(test, dcp); + mutex_init(&dcp->modes_lock); + spin_lock_init(&dcp->mode_state.lock); + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_NOT_NULL(test, blob); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + ctx.dcp = &parser_dcp; + KUNIT_ASSERT_EQ(test, dcp_modes_replace(dcp, &ctx), 0); + requested = dcp->modes[0].mode; + KUNIT_EXPECT_TRUE(test, lookup_mode(dcp, &requested, &selected)); + previous = dcp->modes; + dcp_modes_begin_attachment(dcp); + KUNIT_EXPECT_FALSE(test, lookup_mode(dcp, &requested, NULL)); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size - 1, &ctx), 0); + ctx.dcp = &parser_dcp; + KUNIT_EXPECT_LT(test, dcp_modes_replace(dcp, &ctx), 0); + KUNIT_EXPECT_PTR_EQ(test, dcp->modes, previous); + KUNIT_EXPECT_FALSE(test, lookup_mode(dcp, &requested, NULL)); + record.id = 8; + record.active = 720; + record.total = 750; + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_NOT_NULL(test, blob); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + ctx.dcp = &parser_dcp; + KUNIT_ASSERT_EQ(test, dcp_modes_replace(dcp, &ctx), 0); + KUNIT_EXPECT_FALSE(test, lookup_mode(dcp, &requested, NULL)); + requested = dcp->modes[0].mode; + KUNIT_EXPECT_TRUE(test, lookup_mode(dcp, &requested, &selected)); + KUNIT_EXPECT_EQ(test, selected.timing_mode_id, 8U); + /* Link recovery changes modeset state, never catalog admission. */ + dcp_mode_invalidate(&dcp->mode_state); + dcp_mode_hotplug(&dcp->mode_state, false, NULL); + KUNIT_EXPECT_TRUE(test, lookup_mode(dcp, &requested, &selected)); + KUNIT_EXPECT_EQ(test, selected.timing_mode_id, 8U); + kfree(dcp->modes); +} + +static void parser_property_bounds(struct kunit *test) +{ + char key[64]; + + memset(key, 'x', sizeof(key)); + KUNIT_EXPECT_FALSE(test, dcp_property_key_valid(key, sizeof(key))); + key[63] = '\0'; + KUNIT_EXPECT_TRUE(test, dcp_property_key_valid(key, sizeof(key))); + + KUNIT_EXPECT_FALSE(test, dcp_property_size_valid(0)); + KUNIT_EXPECT_TRUE(test, dcp_property_size_valid(SZ_16M)); + KUNIT_EXPECT_FALSE(test, dcp_property_size_valid(SZ_16M + 1)); + KUNIT_EXPECT_TRUE(test, dcp_property_chunk_valid(8192, 4096, 4096, 4096)); + KUNIT_EXPECT_FALSE(test, dcp_property_chunk_valid(8192, 0, 4097, 4096)); + KUNIT_EXPECT_FALSE(test, dcp_property_chunk_valid(8192, U32_MAX - 2, 4, 4096)); + KUNIT_EXPECT_FALSE(test, dcp_property_chunk_valid(8192, 8190, 4, 4096)); +} + +static struct kunit_case parser_cases[] = { + KUNIT_CASE(parser_score_presence), + KUNIT_CASE(parser_legacy_ties), + KUNIT_CASE(parser_color_presence), + KUNIT_CASE(parser_geometry), + KUNIT_CASE(parser_bad_geometry), + KUNIT_CASE(parser_horizontal_rate), + KUNIT_CASE(parser_bad_score), + KUNIT_CASE(parser_notch_vrr), + KUNIT_CASE(parser_replacement), + KUNIT_CASE(parser_attachment_admission), + KUNIT_CASE(parser_property_bounds), + {} +}; + +static struct kunit_suite parser_suite = { + .name = "apple-display-parser", + .test_cases = parser_cases, +}; + +kunit_test_suite(parser_suite); diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 30f470c326163..5558bb2b4992a 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -375,7 +375,8 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim, else if (!strcmp(key, "SyncWidth")) ret = parse_int_bound(it.handle, &dim->sync_width, 0, U16_MAX); else if (!strcmp(key, "PreciseSyncRate")) - ret = parse_int_bound(it.handle, &dim->precise_sync_rate, 1, U32_MAX); + /* Horizontal may be 0; parse_mode() requires the vertical rate. */ + ret = parse_int_bound(it.handle, &dim->precise_sync_rate, 0, U32_MAX); else if (polarity && !strcmp(key, "SyncPolarity")) { if (dim->sync_polarity >= 0) ret = -EINVAL; @@ -627,7 +628,8 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, } /* Skip virtual or partial entries */ - if (is_virtual || cmode.score < 0 || cmode.id < 0) + if (is_virtual || cmode.score < 0 || cmode.id < 0 || + cmode.id > U32_MAX) continue; trace_iomfb_color_mode(handle->dcp, cmode.id, cmode.score, @@ -690,8 +692,8 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, */ static u64 calculate_clock(struct dimension *horiz, struct dimension *vert) { - u32 pixels = horiz->total * vert->total; - u64 clock = mul_u32_u32(pixels, vert->precise_sync_rate); + u64 pixels = (u64)horiz->total * vert->total; + u64 clock = pixels * vert->precise_sync_rate; return DIV_ROUND_CLOSEST_ULL(clock >> 16, 1000); } @@ -1119,6 +1121,25 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, return modes; } +int replace_modes(struct dcp_parse_ctx *handle, + struct dcp_display_mode **modes, unsigned int *count, + int width_mm, int height_mm, unsigned int notch_height, + bool internal) +{ + struct dcp_display_mode *next; + unsigned int next_count; + + next = enumerate_modes(handle, &next_count, width_mm, height_mm, + notch_height, internal, false); + if (IS_ERR(next)) + return PTR_ERR(next); + + kfree(*modes); + *modes = next; + *count = next_count; + return 0; +} + int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, int *height_mm, bool *backlight_control) { diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 1f5612e672887..cb6b861cf46e8 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -139,6 +139,14 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, int height_mm, unsigned notch_height, bool internal, bool external_sdr); int parse_display_metadata(struct dcp_parse_ctx *handle, struct dcp_display_metadata *out); +void dcp_modes_begin_attachment(struct apple_dcp *dcp); +int dcp_modes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *handle); + +/* Caller serializes readers against replacement. */ +int replace_modes(struct dcp_parse_ctx *handle, + struct dcp_display_mode **modes, unsigned int *count, + int width_mm, int height_mm, unsigned int notch_height, + bool internal); int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, int *height_mm, bool *backlight_control); int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, From e6ef75cc0b79f154a9e188be11085159c669400a Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:53:29 -0700 Subject: [PATCH 1598/2680] drm/apple: bind timing property transfers to their attachment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A property transfer can start before a pipeline changes routes and finish after the new attachment clears catalog admission. Remember the attachment epoch at transfer start and require it to match under modes_lock before replacing or admitting TimingElements. Free stale transfer storage at End. Advance the epoch when returning a borrowed Type-C pipeline to its fixed connector as well as when activating a Type-C route. Link recovery keeps the current epoch and admitted catalog. Add production helper tests for late old-transfer rejection, admission of a current transfer, and preservation of the new catalog against late data. [ryan: adapted to 7.2.10 — the attachment check sits at publication in our dcp_modes_replace() (the parse runs outside modes_lock; a stale result is freed unpublished); H17P external never advances the attachment generation, so its ticketed catalogue path is unchanged] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 9e07156f05bdfb75ceded32950f6e2a85aaea47e) --- drivers/gpu/drm/apple/dcp-fabric.c | 2 + drivers/gpu/drm/apple/dcp-internal.h | 2 + drivers/gpu/drm/apple/iomfb.c | 18 +++++++- drivers/gpu/drm/apple/iomfb_template.c | 10 ++++- drivers/gpu/drm/apple/parser-test.c | 59 +++++++++++++++++++++++--- drivers/gpu/drm/apple/parser.h | 4 +- 6 files changed, 85 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index f06b5961c9121..005b22a419520 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -385,6 +385,8 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) connector->port_encoder->possible_crtcs = connector->candidate_crtcs; } + if (dcp->fixed_connector && dcp->connector != dcp->fixed_connector) + dcp_modes_begin_attachment(dcp); dcp->typec_connector = NULL; dcp->connector = dcp->fixed_connector; WRITE_ONCE(dcp->ext_backlight, false); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 71944ba0e97f7..1571f1244bf99 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -90,6 +90,7 @@ enum { /* Temporary backing for a chunked transfer via setDCPAVPropStart/Chunk/End */ struct dcp_chunks { + u64 modes_generation; size_t length; void *data; size_t received; @@ -372,6 +373,7 @@ struct apple_dcp { struct mutex modes_lock; struct dcp_display_mode *modes; unsigned int nr_modes; + u64 modes_generation; bool modes_admitted; /* Attributes of the connector */ diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 13c64d7a722c8..798d46491ce7a 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1390,15 +1390,24 @@ void dcp_modes_begin_attachment(struct apple_dcp *dcp) if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) return; guard(mutex)(&dcp->modes_lock); + dcp->modes_generation++; dcp->modes_admitted = false; } +u64 dcp_modes_transfer_begin(struct apple_dcp *dcp) +{ + guard(mutex)(&dcp->modes_lock); + return dcp->modes_generation; +} + /* * Publish a TimingElements catalog. Replaced arrays stay allocated, as * readers may still hold a snapshot. A failed parse keeps an admitted - * catalog, except on H17P external, which publishes an empty one. + * catalog, except on H17P external, which publishes an empty one. A + * transfer started before the current attachment is never published. */ -int dcp_modes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *ctx) +int dcp_modes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *ctx, + u64 generation) { bool h17p_external = dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL; @@ -1436,6 +1445,11 @@ int dcp_modes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *ctx) } mutex_lock(&dcp->modes_lock); + if (generation != dcp->modes_generation) { + mutex_unlock(&dcp->modes_lock); + kfree(modes); + return -ESTALE; + } /* Publish one catalog revision with its immutable receipt tickets. */ spin_lock_irqsave(&dcp->metadata.lock, flags); dcp->rate_proof.catalog_sequence++; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 77c9442ecaf36..058d19d74cf1e 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -950,6 +950,7 @@ static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) if (!dcp_property_size_valid(*length)) return false; + dcp->chunks.modes_generation = dcp_modes_transfer_begin(dcp); dcp->chunks.length = *length; dcp->chunks.av_service_epoch = 0; dcp->chunks.automatic_service_pending = false; @@ -1047,8 +1048,13 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, ctx.dcp = dcp; if (!strcmp(req->key, "TimingElements")) { - ret = dcp_modes_replace(dcp, &ctx); + ret = dcp_modes_replace(dcp, &ctx, dcp->chunks.modes_generation); if (ret) { + if (ret == -ESTALE) { + kfree(dcp->chunks.data); + dcp->chunks.data = NULL; + dcp->chunks.length = 0; + } dev_warn(dcp->dev, "failed to parse modes\n"); dcp_firmware_metadata_timing(dcp); return false; @@ -1106,6 +1112,8 @@ static u8 dcpep_cb_prop_end(struct apple_dcp *dcp, return false; } resp = dcpep_process_chunks(dcp, req); + if (!dcp->chunks.data) + return resp; if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_eDP) dev_dbg(dcp->dev, diff --git a/drivers/gpu/drm/apple/parser-test.c b/drivers/gpu/drm/apple/parser-test.c index 5b9501fc8d7b6..51946a8eabcd2 100644 --- a/drivers/gpu/drm/apple/parser-test.c +++ b/drivers/gpu/drm/apple/parser-test.c @@ -444,14 +444,14 @@ static void parser_replacement(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, blob); KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); ctx.dcp = &parser_dcp; - KUNIT_ASSERT_EQ(test, dcp_modes_replace(dcp, &ctx), 0); + KUNIT_ASSERT_EQ(test, dcp_modes_replace(dcp, &ctx, dcp_modes_transfer_begin(dcp)), 0); requested = dcp->modes[0].mode; KUNIT_EXPECT_TRUE(test, lookup_mode(dcp, &requested, &selected)); KUNIT_EXPECT_EQ(test, selected.timing_mode_id, 7U); previous = dcp->modes; KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size - 1, &ctx), 0); ctx.dcp = &parser_dcp; - KUNIT_EXPECT_LT(test, dcp_modes_replace(dcp, &ctx), 0); + KUNIT_EXPECT_LT(test, dcp_modes_replace(dcp, &ctx, dcp_modes_transfer_begin(dcp)), 0); KUNIT_EXPECT_PTR_EQ(test, dcp->modes, previous); KUNIT_EXPECT_EQ(test, dcp->nr_modes, 1U); record.id = 8; @@ -459,7 +459,7 @@ static void parser_replacement(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, blob); KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); ctx.dcp = &parser_dcp; - KUNIT_ASSERT_EQ(test, dcp_modes_replace(dcp, &ctx), 0); + KUNIT_ASSERT_EQ(test, dcp_modes_replace(dcp, &ctx, dcp_modes_transfer_begin(dcp)), 0); KUNIT_EXPECT_EQ(test, dcp->modes[0].timing_mode_id, 8U); KUNIT_EXPECT_EQ(test, selected.timing_mode_id, 7U); KUNIT_EXPECT_TRUE(test, lookup_mode(dcp, &requested, &selected)); @@ -485,7 +485,7 @@ static void parser_attachment_admission(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, blob); KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); ctx.dcp = &parser_dcp; - KUNIT_ASSERT_EQ(test, dcp_modes_replace(dcp, &ctx), 0); + KUNIT_ASSERT_EQ(test, dcp_modes_replace(dcp, &ctx, dcp_modes_transfer_begin(dcp)), 0); requested = dcp->modes[0].mode; KUNIT_EXPECT_TRUE(test, lookup_mode(dcp, &requested, &selected)); previous = dcp->modes; @@ -493,7 +493,7 @@ static void parser_attachment_admission(struct kunit *test) KUNIT_EXPECT_FALSE(test, lookup_mode(dcp, &requested, NULL)); KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size - 1, &ctx), 0); ctx.dcp = &parser_dcp; - KUNIT_EXPECT_LT(test, dcp_modes_replace(dcp, &ctx), 0); + KUNIT_EXPECT_LT(test, dcp_modes_replace(dcp, &ctx, dcp_modes_transfer_begin(dcp)), 0); KUNIT_EXPECT_PTR_EQ(test, dcp->modes, previous); KUNIT_EXPECT_FALSE(test, lookup_mode(dcp, &requested, NULL)); record.id = 8; @@ -503,7 +503,7 @@ static void parser_attachment_admission(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, blob); KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); ctx.dcp = &parser_dcp; - KUNIT_ASSERT_EQ(test, dcp_modes_replace(dcp, &ctx), 0); + KUNIT_ASSERT_EQ(test, dcp_modes_replace(dcp, &ctx, dcp_modes_transfer_begin(dcp)), 0); KUNIT_EXPECT_FALSE(test, lookup_mode(dcp, &requested, NULL)); requested = dcp->modes[0].mode; KUNIT_EXPECT_TRUE(test, lookup_mode(dcp, &requested, &selected)); @@ -516,6 +516,52 @@ static void parser_attachment_admission(struct kunit *test) kfree(dcp->modes); } +static void parser_stale_transfer(struct kunit *test) +{ + struct parser_record record = valid_record(7, 10); + struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + struct dcp_parse_ctx ctx; + struct parser_blob *blob; + struct drm_display_mode requested; + u64 old_generation, current_generation; + + KUNIT_ASSERT_NOT_NULL(test, dcp); + mutex_init(&dcp->modes_lock); + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_NOT_NULL(test, blob); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + ctx.dcp = &parser_dcp; + old_generation = dcp_modes_transfer_begin(dcp); + KUNIT_ASSERT_EQ(test, dcp_modes_replace(dcp, &ctx, old_generation), 0); + requested = dcp->modes[0].mode; + + /* An old transfer cannot admit a catalog on a new attachment. */ + dcp_modes_begin_attachment(dcp); + current_generation = dcp_modes_transfer_begin(dcp); + KUNIT_ASSERT_NE(test, old_generation, current_generation); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + ctx.dcp = &parser_dcp; + KUNIT_EXPECT_EQ(test, dcp_modes_replace(dcp, &ctx, old_generation), -ESTALE); + KUNIT_EXPECT_FALSE(test, lookup_mode(dcp, &requested, NULL)); + + record.id = 8; + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_NOT_NULL(test, blob); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + ctx.dcp = &parser_dcp; + KUNIT_ASSERT_EQ(test, dcp_modes_replace(dcp, &ctx, current_generation), 0); + KUNIT_EXPECT_TRUE(test, lookup_mode(dcp, &requested, NULL)); + KUNIT_EXPECT_EQ(test, dcp->modes[0].timing_mode_id, 8U); + + /* A late old transfer cannot overwrite an already admitted new one. */ + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + ctx.dcp = &parser_dcp; + KUNIT_EXPECT_EQ(test, dcp_modes_replace(dcp, &ctx, old_generation), -ESTALE); + KUNIT_EXPECT_EQ(test, dcp->modes[0].timing_mode_id, 8U); + KUNIT_EXPECT_TRUE(test, lookup_mode(dcp, &requested, NULL)); + kfree(dcp->modes); +} + static void parser_property_bounds(struct kunit *test) { char key[64]; @@ -545,6 +591,7 @@ static struct kunit_case parser_cases[] = { KUNIT_CASE(parser_notch_vrr), KUNIT_CASE(parser_replacement), KUNIT_CASE(parser_attachment_admission), + KUNIT_CASE(parser_stale_transfer), KUNIT_CASE(parser_property_bounds), {} }; diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index cb6b861cf46e8..64d96f7e32da5 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -140,7 +140,9 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, bool internal, bool external_sdr); int parse_display_metadata(struct dcp_parse_ctx *handle, struct dcp_display_metadata *out); void dcp_modes_begin_attachment(struct apple_dcp *dcp); -int dcp_modes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *handle); +u64 dcp_modes_transfer_begin(struct apple_dcp *dcp); +int dcp_modes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *handle, + u64 generation); /* Caller serializes readers against replacement. */ int replace_modes(struct dcp_parse_ctx *handle, From 4c8b4e6a4102479bd3e1d50761814ee0f4a7b4c0 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:10:37 -0700 Subject: [PATCH 1599/2680] drm/apple: close display catalog ownership and teardown gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fixed connector can be forced on while its pipeline is borrowed by a Type-C route. Require the queried connector to own the admitted catalog when probing or validating modes, and revoke the old connector's EDID access when ownership changes. Reject a nonempty TimingElements catalog with no valid records without retiring an admitted catalog. Preserve the explicit empty-array behavior. Apply DisplayAttributes only when its transfer epoch matches, and recheck that epoch when publishing connector dimensions from a worker. Enable the worker only after successful bind and drain it before connector teardown. Release the final catalog after callback resources drain. Add production helper regression tests for connector ownership, rejected replacement, attribute epochs and repeat-safe cleanup. [ryan: adapted to 7.2.10 — connector ownership is checked in our catalogue snapshot (get_modes) and before dcp_mode_settable() in mode_valid; lookup_mode() stays a lock-free snapshot walk (no lookup_mode_locked) because the 180 Hz eligibility check takes hpd_mutex; the rejection of a nonempty catalogue without valid records (new parse_array_count() helper) and dcp_modes_end_typec()'s generation bump are skipped on H17P external, whose catalogue keeps its ticketed publication; dcp_attributes_replace() keeps our ext_backlight update on parse failure and our hotplug_wq kick, and the H17P metadata parse stays after it in the template; the dimensions work is also enabled on our RTKit bind and the native 14.7 internal bind] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 79fe48173bb27f48e77bfa20eb793f2343968b91) --- drivers/gpu/drm/apple/dcp-fabric.c | 9 +- drivers/gpu/drm/apple/dcp-internal.h | 2 + drivers/gpu/drm/apple/dcp.c | 53 +++-- drivers/gpu/drm/apple/dcp.h | 2 +- drivers/gpu/drm/apple/iomfb.c | 130 +++++++++++- drivers/gpu/drm/apple/iomfb_template.c | 29 +-- drivers/gpu/drm/apple/iomfb_v14_7.c | 2 +- drivers/gpu/drm/apple/parser-test.c | 272 ++++++++++++++++++++++++- drivers/gpu/drm/apple/parser.c | 29 ++- drivers/gpu/drm/apple/parser.h | 12 ++ 10 files changed, 483 insertions(+), 57 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 005b22a419520..899acc15646c5 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -251,6 +251,7 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, } dcp_modes_begin_attachment(dcp); + apple_connector_edid_set_live(dcp->connector, false); dcp->phy = route->phy; dcp->dptx_phy = route->dptx_phy; dcp->connector_type = DRM_MODE_CONNECTOR_USB; @@ -302,6 +303,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) struct apple_connector *connector = dcp->typec_connector; struct mux_control *active_xbar = route->active_xbar; bool was_tunnel = route->tunnel; + bool ended_attachment; int ret = 0; guard(mutex)(&dcp->external_lock); @@ -360,6 +362,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) if (dcp_is_external_h17p(dcp)) dpavservep_detach(dcp); route->selected = false; + ended_attachment = dcp_modes_end_typec(dcp, route); if (dcp->active_typec_route == route) dcp->active_typec_route = NULL; @@ -385,8 +388,10 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) connector->port_encoder->possible_crtcs = connector->candidate_crtcs; } - if (dcp->fixed_connector && dcp->connector != dcp->fixed_connector) - dcp_modes_begin_attachment(dcp); + if (ended_attachment && dcp->fixed_connector) { + apple_connector_set_pipeline(dcp->fixed_connector, + to_platform_device(dcp->dev)); + } dcp->typec_connector = NULL; dcp->connector = dcp->fixed_connector; WRITE_ONCE(dcp->ext_backlight, false); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 1571f1244bf99..dec6da7c74706 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -374,6 +374,8 @@ struct apple_dcp { struct dcp_display_mode *modes; unsigned int nr_modes; u64 modes_generation; + u64 dimensions_generation; + struct work_struct dimensions_wq; bool modes_admitted; /* Attributes of the connector */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index c1b3df1a7e145..2e9d1873bc7f8 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -543,36 +543,36 @@ const struct drm_edid *dcp_firmware_edid(struct apple_dcp *dcp, u64 *generation) return edid; } -void dcp_set_dimensions(struct apple_dcp *dcp) +void dcp_set_dimensions(struct apple_dcp *dcp, u64 generation) { - int i; - int width_mm = dcp->width_mm; - int height_mm = dcp->height_mm; + struct apple_connector *apple_connector = READ_ONCE(dcp->connector); + struct drm_connector *connector = apple_connector ? &apple_connector->base : NULL; + int width_mm, height_mm, i; - if (width_mm == 0 || height_mm == 0) { + if (connector) + mutex_lock(&connector->dev->mode_config.mutex); + mutex_lock(&dcp->modes_lock); + if (generation != dcp->modes_generation || + apple_connector != READ_ONCE(dcp->connector)) + goto out_unlock; + width_mm = dcp->width_mm; + height_mm = dcp->height_mm; + if (!width_mm || !height_mm) { width_mm = dcp->panel.width_mm; height_mm = dcp->panel.height_mm; } - - /* - * This callback runs on the ordered RTKit RX worker. A mode probe - * can hold mode_config.mutex while waiting for an AFK reply which - * that same worker still has to dispatch. Defer the DRM update. - */ - if (dcp->connector && !READ_ONCE(dcp->quiescing)) - schedule_work(&dcp->connector->hotplug_wq); - - /* - * Fix up any probed modes. Modes are created when parsing - * TimingElements, dimensions are calculated when parsing - * DisplayAttributes, and TimingElements may be sent first - */ - mutex_lock(&dcp->modes_lock); + if (connector) { + connector->display_info.width_mm = width_mm; + connector->display_info.height_mm = height_mm; + } for (i = 0; i < dcp->nr_modes; ++i) { dcp->modes[i].mode.width_mm = width_mm; dcp->modes[i].mode.height_mm = height_mm; } +out_unlock: mutex_unlock(&dcp->modes_lock); + if (connector) + mutex_unlock(&connector->dev->mode_config.mutex); } void dcp_update_connector_dimensions(struct apple_dcp *dcp, @@ -2753,8 +2753,10 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) return 0; ret = iomfb_v14_7_bind(dcp); /* Balances the disable at probe, as below. */ - if (!ret) + if (!ret) { + enable_work(&dcp->dimensions_wq); dcp_enable_typec_work(dcp); + } return ret; } @@ -2853,6 +2855,8 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) ret = dev_err_probe(dev, ret, "Failed to boot RTKit: %d\n", ret); goto err_piodma; } + enable_work(&dcp->dimensions_wq); + schedule_work(&dcp->dimensions_wq); dcp_enable_typec_work(dcp); return 0; @@ -2872,6 +2876,7 @@ VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, v if (!dcp) return; + disable_work_sync(&dcp->dimensions_wq); if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { if (!dcp->external) dcp_disable_typec_work(dcp, false); @@ -3058,7 +3063,11 @@ static int dcp_platform_probe(struct platform_device *pdev) * Type-C and Thunderbolt routes can be activated as soon as they are * registered below, before the DRM device binds. */ - mutex_init(&dcp->modes_lock); + dcp_modes_init(dcp); + /* Registered before callback resources, so they drain before the catalog. */ + ret = devm_add_action_or_reset(dev, dcp_modes_release, dcp); + if (ret) + return ret; mutex_init(&dcp->hpd_mutex); mutex_init(&dcp->external_lock); INIT_WORK(&dcp->external_stop_work, dcp_external_stop_work); diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index be130a4ca27ab..ecfb32beec35d 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -62,7 +62,7 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, const struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode); -void dcp_set_dimensions(struct apple_dcp *dcp); +void dcp_set_dimensions(struct apple_dcp *dcp, u64 generation); void dcp_update_connector_dimensions(struct apple_dcp *dcp, struct drm_connector *connector); void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 798d46491ce7a..9812af882a30e 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1364,8 +1364,36 @@ static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) dcpep_handle_cb(dcp, ctx_id, data, length, offset); } -/* Retained arrays are not freed while mode readers can still refer to them. */ +static void dcp_modes_dimensions_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(work, struct apple_dcp, dimensions_wq); + u64 generation; + + scoped_guard(mutex, &dcp->modes_lock) + generation = dcp->dimensions_generation; + dcp_set_dimensions(dcp, generation); +} + +void dcp_modes_init(struct apple_dcp *dcp) +{ + mutex_init(&dcp->modes_lock); + INIT_WORK(&dcp->dimensions_wq, dcp_modes_dimensions_work); + disable_work(&dcp->dimensions_wq); +} + +bool dcp_modes_for_connector(struct apple_dcp *dcp, + struct apple_connector *connector) +{ + lockdep_assert_held(&dcp->modes_lock); + return dcp->modes_admitted && READ_ONCE(dcp->connector) == connector; +} + +/* + * Retained arrays are not freed while mode readers can still refer to them. + * With @connector, only the connector that owns the admitted catalog sees it. + */ static struct dcp_display_mode *dcp_modes_snapshot(struct apple_dcp *dcp, + struct apple_connector *connector, unsigned int *count) { struct dcp_display_mode *modes; @@ -1373,7 +1401,8 @@ static struct dcp_display_mode *dcp_modes_snapshot(struct apple_dcp *dcp, /* An attachment sees no catalog until one is admitted for it. */ guard(mutex)(&dcp->modes_lock); - if (!dcp->modes_admitted) { + if (!dcp->modes_admitted || + (connector && !dcp_modes_for_connector(dcp, connector))) { *count = 0; return NULL; } @@ -1383,7 +1412,6 @@ static struct dcp_display_mode *dcp_modes_snapshot(struct apple_dcp *dcp, spin_unlock_irqrestore(&dcp->metadata.lock, flags); return modes; } - void dcp_modes_begin_attachment(struct apple_dcp *dcp) { /* H17P external catalogs are fenced by their connection receipt. */ @@ -1394,6 +1422,19 @@ void dcp_modes_begin_attachment(struct apple_dcp *dcp) dcp->modes_admitted = false; } +bool dcp_modes_end_typec(struct apple_dcp *dcp, struct apple_dcp_typec_route *route) +{ + guard(mutex)(&dcp->modes_lock); + if (READ_ONCE(dcp->active_typec_route) != route) + return false; + /* H17P external catalogs are fenced by their connection receipt. */ + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL) { + dcp->modes_generation++; + dcp->modes_admitted = false; + } + return true; +} + u64 dcp_modes_transfer_begin(struct apple_dcp *dcp) { guard(mutex)(&dcp->modes_lock); @@ -1416,6 +1457,7 @@ int dcp_modes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *ctx, u64 rate_service_epoch = 0; unsigned int nr_modes = 0, i; unsigned long flags; + u32 entries = 0; int ret = 0; if (h17p_external && dcp->chunks.payload_complete && @@ -1424,6 +1466,11 @@ int dcp_modes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *ctx, rate_ticket = dcp->chunks.mode_ticket; rate_service_epoch = dcp->chunks.av_service_epoch; } + if (!h17p_external) { + ret = parse_array_count(ctx, &entries); + if (ret) + return ret; + } modes = enumerate_modes(ctx, &nr_modes, dcp->width_mm, dcp->height_mm, dcp->notch_height, dcp->fixed_connector_type == DRM_MODE_CONNECTOR_eDP, @@ -1434,6 +1481,10 @@ int dcp_modes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *ctx, return ret; modes = NULL; nr_modes = 0; + } else if (!h17p_external && entries && !nr_modes) { + /* No valid record: keep an admitted catalog, as for a parse error. */ + kfree(modes); + return -EINVAL; } else { for (i = 0; i < nr_modes; i++) if (modes[i].automatic_rgb8_quirk) { @@ -1509,6 +1560,72 @@ static bool dcp_publish_mode(struct apple_dcp *dcp, struct drm_connector *connec return eligible; } +/* Apply DisplayAttributes only for the attachment its transfer started on. */ +int dcp_attributes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *handle, + u64 generation) +{ + int width_mm, height_mm, ret, i; + bool backlight_control = false, ext; + + scoped_guard(mutex, &dcp->modes_lock) { + if (generation != dcp->modes_generation) + return -ESTALE; + ret = parse_display_attributes(handle, &width_mm, &height_mm, + &backlight_control); + if (!ret) { + dcp->width_mm = width_mm; + dcp->height_mm = height_mm; + if (!width_mm || !height_mm) { + width_mm = dcp->panel.width_mm; + height_mm = dcp->panel.height_mm; + } + for (i = 0; i < dcp->nr_modes; ++i) { + dcp->modes[i].mode.width_mm = width_mm; + dcp->modes[i].mode.height_mm = height_mm; + } + dcp->dimensions_generation = generation; + schedule_work(&dcp->dimensions_wq); + } + } + + /* + * Written once, so the hotplug work never sees a working display + * without its backlight; a display whose attributes do not parse + * gets none. The attributes can arrive after the hotplug work has + * run, so sync the backlight when the answer changes. + */ + ext = !ret && backlight_control && !dcp_has_panel(dcp); + if (ext != READ_ONCE(dcp->ext_backlight)) { + WRITE_ONCE(dcp->ext_backlight, ext); + if (dcp->connector) + schedule_work(&dcp->connector->bl_sync_wq); + } + if (ret) + return ret; + + /* + * This callback runs on the ordered RTKit RX worker. A mode probe + * can hold mode_config.mutex while waiting for an AFK reply which + * that same worker still has to dispatch. The hotplug work + * publishes the connector dimensions. + */ + if (dcp->connector && !READ_ONCE(dcp->quiescing)) + schedule_work(&dcp->connector->hotplug_wq); + return 0; +} + +void dcp_modes_release(void *data) +{ + struct apple_dcp *dcp = data; + + disable_work_sync(&dcp->dimensions_wq); + guard(mutex)(&dcp->modes_lock); + dcp->modes_generation++; + dcp->modes_admitted = false; + kfree(dcp->modes); + dcp->modes = NULL; + dcp->nr_modes = 0; +} int dcp_get_modes(struct drm_connector *connector) { struct apple_connector *apple_connector = to_apple_connector(connector); @@ -1538,7 +1655,7 @@ int dcp_get_modes(struct drm_connector *connector) return 0; } dcp = platform_get_drvdata(pdev); - modes = dcp_modes_snapshot(dcp, &nr_modes); + modes = dcp_modes_snapshot(dcp, apple_connector, &nr_modes); physical_edid = dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL; @@ -1704,7 +1821,7 @@ bool lookup_mode(struct apple_dcp *dcp, const struct drm_display_mode *mode, unsigned long flags; bool eligible; - modes = dcp_modes_snapshot(dcp, &count); + modes = dcp_modes_snapshot(dcp, NULL, &count); for (i = 0; i < count; ++i) { if (drm_mode_match(mode, &modes[i].mode, DRM_MODE_MATCH_TIMINGS | @@ -1766,6 +1883,9 @@ enum drm_mode_status dcp_mode_valid(struct drm_connector *connector, return MODE_ERROR; dcp = platform_get_drvdata(pdev); + scoped_guard(mutex, &dcp->modes_lock) + if (!dcp_modes_for_connector(dcp, apple_connector)) + return MODE_BAD; return dcp_mode_settable(dcp, mode) ? MODE_OK : MODE_BAD; } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 058d19d74cf1e..00a00e89087b1 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -916,9 +916,8 @@ static void iomfbep_cb_enable_backlight_message_ap_gated(struct apple_dcp *dcp, } #endif /* - * update backlight brightness on next swap, on non mini-LED displays - * DCP seems to set an invalid iDAC value after coming out of DPMS. - * syslog: "[BrightnessLCD.cpp:743][AFK]nitsToDBV: iDAC out of range" + * Reapply brightness on the next swap after DPMS wake on panels + * without mini-LED backlights. */ dcp->brightness.update = true; /* Backlight handling is only set up for panels described in the DT. */ @@ -1065,30 +1064,18 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, dcp_firmware_metadata_timing(dcp); } else if (!strcmp(req->key, "DisplayAttributes")) { struct dcp_parse_ctx metadata_ctx = ctx; - bool backlight_control = false, ext; - - ret = parse_display_attributes(&ctx, &dcp->width_mm, - &dcp->height_mm, &backlight_control); - - /* - * Written once, so the hotplug work never sees a working display - * without its backlight; a display whose attributes do not parse - * gets none. The attributes can arrive after the hotplug work has - * run, so sync the backlight when the answer changes. - */ - ext = !ret && backlight_control && !dcp_has_panel(dcp); - if (ext != READ_ONCE(dcp->ext_backlight)) { - WRITE_ONCE(dcp->ext_backlight, ext); - if (dcp->connector) - schedule_work(&dcp->connector->bl_sync_wq); - } + ret = dcp_attributes_replace(dcp, &ctx, dcp->chunks.modes_generation); if (ret) { + if (ret == -ESTALE) { + kfree(dcp->chunks.data); + dcp->chunks.data = NULL; + dcp->chunks.length = 0; + } dev_warn(dcp->dev, "failed to parse display attribs\n"); return false; } - dcp_set_dimensions(dcp); if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { struct dcp_display_metadata metadata; diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index e68565e5eb4e5..4ad5e717c2cbb 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -1379,7 +1379,7 @@ int iomfb_v14_7_start(struct apple_dcp *dcp) /* Never removed, like the session it describes. */ debugfs_create_file("dcp-t6030", 0400, NULL, v14, &dcp_v14_status_fops); dcp->connector->connected = true; - dcp_set_dimensions(dcp); + dcp_set_dimensions(dcp, dcp_modes_transfer_begin(dcp)); scoped_guard(spinlock_irqsave, &dcp->mode_state.lock) dcp->mode_state.valid = true; dcp->active = true; diff --git a/drivers/gpu/drm/apple/parser-test.c b/drivers/gpu/drm/apple/parser-test.c index 51946a8eabcd2..d4590e34a9136 100644 --- a/drivers/gpu/drm/apple/parser-test.c +++ b/drivers/gpu/drm/apple/parser-test.c @@ -3,6 +3,7 @@ #include #include +#include "connector.h" #include "dcp-internal.h" #include "dcp-property.h" #include "iomfb_internal.h" @@ -439,7 +440,7 @@ static void parser_replacement(struct kunit *test) struct drm_display_mode requested; KUNIT_ASSERT_NOT_NULL(test, dcp); - mutex_init(&dcp->modes_lock); + dcp_modes_init(dcp); blob = make_blob(test, &record, 1); KUNIT_ASSERT_NOT_NULL(test, blob); KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); @@ -479,7 +480,7 @@ static void parser_attachment_admission(struct kunit *test) struct drm_display_mode requested; KUNIT_ASSERT_NOT_NULL(test, dcp); - mutex_init(&dcp->modes_lock); + dcp_modes_init(dcp); spin_lock_init(&dcp->mode_state.lock); blob = make_blob(test, &record, 1); KUNIT_ASSERT_NOT_NULL(test, blob); @@ -526,7 +527,7 @@ static void parser_stale_transfer(struct kunit *test) u64 old_generation, current_generation; KUNIT_ASSERT_NOT_NULL(test, dcp); - mutex_init(&dcp->modes_lock); + dcp_modes_init(dcp); blob = make_blob(test, &record, 1); KUNIT_ASSERT_NOT_NULL(test, blob); KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); @@ -562,6 +563,263 @@ static void parser_stale_transfer(struct kunit *test) kfree(dcp->modes); } +static void parser_connector_ownership(struct kunit *test) +{ + struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + struct apple_connector *fixed = kunit_kzalloc(test, sizeof(*fixed), GFP_KERNEL); + struct apple_connector *borrowed = kunit_kzalloc(test, sizeof(*borrowed), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, dcp); + KUNIT_ASSERT_NOT_NULL(test, fixed); + KUNIT_ASSERT_NOT_NULL(test, borrowed); + dcp_modes_init(dcp); + mutex_lock(&dcp->modes_lock); + dcp->connector = borrowed; + dcp->modes_admitted = true; + /* Fixed outputs retain their pipeline even when probing is forced. */ + fixed->base.force = DRM_FORCE_ON; + KUNIT_EXPECT_FALSE(test, dcp_modes_for_connector(dcp, fixed)); + KUNIT_EXPECT_TRUE(test, dcp_modes_for_connector(dcp, borrowed)); + dcp->connector = fixed; + dcp->modes_admitted = false; + KUNIT_EXPECT_FALSE(test, dcp_modes_for_connector(dcp, fixed)); + dcp->modes_admitted = true; + KUNIT_EXPECT_TRUE(test, dcp_modes_for_connector(dcp, fixed)); + KUNIT_EXPECT_FALSE(test, dcp_modes_for_connector(dcp, borrowed)); + mutex_unlock(&dcp->modes_lock); +} + +static void parser_route_retirement(struct kunit *test) +{ + struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + struct apple_connector *fixed = kunit_kzalloc(test, sizeof(*fixed), GFP_KERNEL); + struct apple_dcp_typec_route *route = kunit_kzalloc(test, 1, GFP_KERNEL); + struct apple_dcp_typec_route *other = kunit_kzalloc(test, 1, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, dcp); + KUNIT_ASSERT_NOT_NULL(test, fixed); + KUNIT_ASSERT_NOT_NULL(test, route); + KUNIT_ASSERT_NOT_NULL(test, other); + dcp_modes_init(dcp); + dcp->connector = fixed; + dcp->active_typec_route = route; + dcp->modes_admitted = true; + KUNIT_EXPECT_FALSE(test, dcp_modes_end_typec(dcp, other)); + KUNIT_EXPECT_EQ(test, dcp_modes_transfer_begin(dcp), 0ULL); + KUNIT_EXPECT_TRUE(test, dcp_modes_end_typec(dcp, route)); + KUNIT_EXPECT_EQ(test, dcp_modes_transfer_begin(dcp), 1ULL); + KUNIT_EXPECT_FALSE(test, dcp->modes_admitted); + dcp->active_typec_route = NULL; + KUNIT_EXPECT_FALSE(test, dcp_modes_end_typec(dcp, route)); + KUNIT_EXPECT_EQ(test, dcp_modes_transfer_begin(dcp), 1ULL); + dcp_modes_release(dcp); +} + +static void parser_rejected_replacement(struct kunit *test) +{ + struct parser_record record = valid_record(7, 10); + struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + struct dcp_display_mode *previous; + struct dcp_parse_ctx ctx; + struct parser_blob *blob; + + KUNIT_ASSERT_NOT_NULL(test, dcp); + dcp_modes_init(dcp); + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_NOT_NULL(test, blob); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + ctx.dcp = &parser_dcp; + KUNIT_ASSERT_EQ(test, dcp_modes_replace(dcp, &ctx, 0), 0); + previous = dcp->modes; + record.missing = MISSING_SCORE; + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_NOT_NULL(test, blob); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + ctx.dcp = &parser_dcp; + KUNIT_EXPECT_EQ(test, dcp_modes_replace(dcp, &ctx, 0), -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, dcp->modes, previous); + KUNIT_EXPECT_EQ(test, dcp->nr_modes, 1U); + KUNIT_EXPECT_TRUE(test, dcp->modes_admitted); + dcp_modes_begin_attachment(dcp); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + ctx.dcp = &parser_dcp; + KUNIT_EXPECT_EQ(test, dcp_modes_replace(dcp, &ctx, 1), -EINVAL); + KUNIT_EXPECT_FALSE(test, dcp->modes_admitted); + blob = make_blob(test, &record, 0); + KUNIT_ASSERT_NOT_NULL(test, blob); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + ctx.dcp = &parser_dcp; + KUNIT_ASSERT_EQ(test, dcp_modes_replace(dcp, &ctx, 1), 0); + KUNIT_EXPECT_EQ(test, dcp->nr_modes, 0U); + KUNIT_EXPECT_TRUE(test, dcp->modes_admitted); + dcp_modes_release(dcp); + KUNIT_EXPECT_PTR_EQ(test, dcp->modes, NULL); + KUNIT_EXPECT_FALSE(test, dcp->modes_admitted); +} + +static void parser_attribute_epoch(struct kunit *test) +{ + struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + struct parser_blob *blob = kunit_kzalloc(test, sizeof(*blob), GFP_KERNEL); + struct dcp_parse_ctx ctx; + u64 generation; + + KUNIT_ASSERT_NOT_NULL(test, dcp); + KUNIT_ASSERT_NOT_NULL(test, blob); + dcp_modes_init(dcp); + put_unaligned_le32(0xd3, blob->data); + blob->size = 4; + blob_tag(blob, 1, 3); + blob_int(blob, "MaxHorizontalImageSize", 52); + blob_int(blob, "MaxVerticalImageSize", 29); + blob_bool(blob, "SupportsBacklightControl", true); + generation = dcp_modes_transfer_begin(dcp); + dcp_modes_begin_attachment(dcp); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + KUNIT_EXPECT_EQ(test, dcp_attributes_replace(dcp, &ctx, generation), -ESTALE); + KUNIT_EXPECT_EQ(test, dcp->width_mm, 0); + KUNIT_EXPECT_FALSE(test, dcp->ext_backlight); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + KUNIT_ASSERT_EQ(test, dcp_attributes_replace(dcp, &ctx, 1), 0); + KUNIT_EXPECT_EQ(test, dcp->width_mm, 520); + KUNIT_EXPECT_EQ(test, dcp->height_mm, 290); + KUNIT_EXPECT_TRUE(test, dcp->ext_backlight); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + KUNIT_EXPECT_EQ(test, dcp_attributes_replace(dcp, &ctx, generation), -ESTALE); + KUNIT_EXPECT_EQ(test, dcp->width_mm, 520); + dcp_modes_release(dcp); +} + +static void parser_attributes_malformed(struct kunit *test) +{ + struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + struct parser_blob *blob = kunit_kzalloc(test, sizeof(*blob), GFP_KERNEL); + struct dcp_parse_ctx ctx; + const s64 invalid[] = { -1, INT_MAX / 10 + 1, S64_MAX }; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, dcp); + KUNIT_ASSERT_NOT_NULL(test, blob); + dcp_modes_init(dcp); + dcp->width_mm = 520; + dcp->height_mm = 290; + dcp->ext_backlight = true; + put_unaligned_le32(0xd3, blob->data); + blob->size = 4; + blob_tag(blob, 2, 0); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + KUNIT_EXPECT_LT(test, dcp_attributes_replace(dcp, &ctx, 0), 0); + KUNIT_ASSERT_EQ(test, parse(blob->data, 4, &ctx), 0); + KUNIT_EXPECT_LT(test, dcp_attributes_replace(dcp, &ctx, 0), 0); + blob->size = 4; + blob_tag(blob, 1, 1); + blob_int(blob, "Unknown", 1); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size - 1, &ctx), 0); + KUNIT_EXPECT_LT(test, dcp_attributes_replace(dcp, &ctx, 0), 0); + for (i = 0; i < ARRAY_SIZE(invalid); ++i) { + blob->size = 4; + blob_tag(blob, 1, 1); + blob_int(blob, "MaxHorizontalImageSize", invalid[i]); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + KUNIT_EXPECT_LT(test, dcp_attributes_replace(dcp, &ctx, 0), 0); + } + KUNIT_EXPECT_EQ(test, dcp->width_mm, 520); + KUNIT_EXPECT_EQ(test, dcp->height_mm, 290); + KUNIT_EXPECT_TRUE(test, dcp->ext_backlight); + dcp_modes_release(dcp); +} + +static void parser_attribute_deferred(struct kunit *test) +{ + struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + struct apple_connector *connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL); + struct drm_device *dev = kunit_kzalloc(test, sizeof(*dev), GFP_KERNEL); + struct parser_blob *blob = kunit_kzalloc(test, sizeof(*blob), GFP_KERNEL); + struct dcp_parse_ctx ctx; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, dcp); + KUNIT_ASSERT_NOT_NULL(test, connector); + KUNIT_ASSERT_NOT_NULL(test, dev); + KUNIT_ASSERT_NOT_NULL(test, blob); + dcp_modes_init(dcp); + mutex_init(&dev->mode_config.mutex); + connector->base.dev = dev; + dcp->connector = connector; + dcp->panel.width_mm = 600; + dcp->modes = kmalloc_obj(*dcp->modes); + KUNIT_ASSERT_NOT_NULL(test, dcp->modes); + dcp->nr_modes = 1; + put_unaligned_le32(0xd3, blob->data); + blob->size = 4; + blob_tag(blob, 1, 2); + blob_int(blob, "MaxHorizontalImageSize", 52); + blob_int(blob, "MaxVerticalImageSize", 29); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + enable_work(&dcp->dimensions_wq); + /* RX must return while a probe holds the DRM lock awaiting a reply. */ + mutex_lock(&dev->mode_config.mutex); + ret = dcp_attributes_replace(dcp, &ctx, 0); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, dcp->modes[0].mode.width_mm, 520); + mutex_unlock(&dev->mode_config.mutex); + flush_work(&dcp->dimensions_wq); + KUNIT_EXPECT_EQ(test, connector->base.display_info.width_mm, 520U); + KUNIT_EXPECT_EQ(test, connector->base.display_info.height_mm, 290U); + dcp_modes_release(dcp); +} + +static void parser_dimensions_lifetime(struct kunit *test) +{ + struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + struct apple_connector *connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL); + struct drm_device *dev = kunit_kzalloc(test, sizeof(*dev), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, dcp); + KUNIT_ASSERT_NOT_NULL(test, connector); + KUNIT_ASSERT_NOT_NULL(test, dev); + dcp_modes_init(dcp); + mutex_init(&dev->mode_config.mutex); + connector->base.dev = dev; + dcp->connector = connector; + dcp->width_mm = 520; + dcp->height_mm = 290; + KUNIT_EXPECT_FALSE(test, schedule_work(&dcp->dimensions_wq)); + KUNIT_EXPECT_EQ(test, connector->base.display_info.width_mm, 0U); + enable_work(&dcp->dimensions_wq); + schedule_work(&dcp->dimensions_wq); + flush_work(&dcp->dimensions_wq); + KUNIT_EXPECT_EQ(test, connector->base.display_info.width_mm, 520U); + disable_work_sync(&dcp->dimensions_wq); + dcp->width_mm = 600; + KUNIT_EXPECT_FALSE(test, schedule_work(&dcp->dimensions_wq)); + KUNIT_EXPECT_EQ(test, connector->base.display_info.width_mm, 520U); + enable_work(&dcp->dimensions_wq); + schedule_work(&dcp->dimensions_wq); + flush_work(&dcp->dimensions_wq); + KUNIT_EXPECT_EQ(test, connector->base.display_info.width_mm, 600U); + dcp_modes_release(dcp); + KUNIT_EXPECT_FALSE(test, schedule_work(&dcp->dimensions_wq)); +} + +static void parser_catalog_release(struct kunit *test) +{ + struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, dcp); + dcp_modes_init(dcp); + dcp->modes = kmalloc_obj(*dcp->modes); + KUNIT_ASSERT_NOT_NULL(test, dcp->modes); + dcp->nr_modes = 1; + dcp->modes_admitted = true; + dcp_modes_release(dcp); + KUNIT_EXPECT_PTR_EQ(test, dcp->modes, NULL); + KUNIT_EXPECT_EQ(test, dcp->nr_modes, 0U); + KUNIT_EXPECT_FALSE(test, dcp->modes_admitted); + dcp_modes_release(dcp); + KUNIT_EXPECT_PTR_EQ(test, dcp->modes, NULL); +} + static void parser_property_bounds(struct kunit *test) { char key[64]; @@ -592,6 +850,14 @@ static struct kunit_case parser_cases[] = { KUNIT_CASE(parser_replacement), KUNIT_CASE(parser_attachment_admission), KUNIT_CASE(parser_stale_transfer), + KUNIT_CASE(parser_connector_ownership), + KUNIT_CASE(parser_route_retirement), + KUNIT_CASE(parser_rejected_replacement), + KUNIT_CASE(parser_attribute_epoch), + KUNIT_CASE(parser_attributes_malformed), + KUNIT_CASE(parser_attribute_deferred), + KUNIT_CASE(parser_dimensions_lifetime), + KUNIT_CASE(parser_catalog_release), KUNIT_CASE(parser_property_bounds), {} }; diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 5558bb2b4992a..0de67aa8fe710 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -1121,18 +1121,43 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, return modes; } +/* Entries of the array at the cursor, which is not consumed. */ +int parse_array_count(const struct dcp_parse_ctx *handle, u32 *count) +{ + struct dcp_parse_ctx extent = *handle; + struct iterator it; + int ret; + + ret = iterator_begin(&extent, &it, false); + if (ret) + return ret; + *count = it.len; + return 0; +} + int replace_modes(struct dcp_parse_ctx *handle, struct dcp_display_mode **modes, unsigned int *count, int width_mm, int height_mm, unsigned int notch_height, bool internal) { struct dcp_display_mode *next; + struct dcp_parse_ctx extent = *handle; + struct iterator it; unsigned int next_count; + int ret; + + ret = iterator_begin(&extent, &it, false); + if (ret) + return ret; next = enumerate_modes(handle, &next_count, width_mm, height_mm, notch_height, internal, false); if (IS_ERR(next)) return PTR_ERR(next); + if (it.len && !next_count) { + kfree(next); + return -EINVAL; + } kfree(*modes); *modes = next; @@ -1159,9 +1184,9 @@ int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, if (IS_ERR(key)) ret = PTR_ERR(key); else if (!strcmp(key, "MaxHorizontalImageSize")) - ret = parse_int(it.handle, &width_cm); + ret = parse_int_bound(it.handle, &width_cm, 0, INT_MAX / 10); else if (!strcmp(key, "MaxVerticalImageSize")) - ret = parse_int(it.handle, &height_cm); + ret = parse_int_bound(it.handle, &height_cm, 0, INT_MAX / 10); else if (!strcmp(key, "SupportsBacklightControl")) ret = parse_bool_or_skip(it.handle, backlight_control); else diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 64d96f7e32da5..50eb5f6db4de7 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -11,6 +11,8 @@ #include "dcp-metadata-epoch.h" struct apple_dcp; +struct apple_connector; +struct apple_dcp_typec_route; struct dcp_parse_ctx { struct apple_dcp *dcp; @@ -139,11 +141,21 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, int height_mm, unsigned notch_height, bool internal, bool external_sdr); int parse_display_metadata(struct dcp_parse_ctx *handle, struct dcp_display_metadata *out); +/* Caller holds modes_lock while checking connector ownership. */ +bool dcp_modes_for_connector(struct apple_dcp *dcp, + struct apple_connector *connector); void dcp_modes_begin_attachment(struct apple_dcp *dcp); +bool dcp_modes_end_typec(struct apple_dcp *dcp, struct apple_dcp_typec_route *route); +void dcp_modes_init(struct apple_dcp *dcp); +void dcp_modes_release(void *data); +int dcp_attributes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *handle, + u64 generation); u64 dcp_modes_transfer_begin(struct apple_dcp *dcp); int dcp_modes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *handle, u64 generation); +int parse_array_count(const struct dcp_parse_ctx *handle, u32 *count); + /* Caller serializes readers against replacement. */ int replace_modes(struct dcp_parse_ctx *handle, struct dcp_display_mode **modes, unsigned int *count, From ed5ac75ee6aa441deb63f49afae355fd208d1c30 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:16:03 -0700 Subject: [PATCH 1600/2680] drm/apple: propagate optional property extent errors Keep the false fallback for a complete optional flag value of another type, but reject a value whose serialized extent cannot be skipped. Otherwise truncated SupportsBacklightControl data can replace working display attributes with incomplete values. Add a serialized replacement test that preserves previous dimensions and backlight on truncation and accepts the complete wrong-type fallback. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit f762a6593c0810f6e56bc4945ccdbe82757c59af) --- drivers/gpu/drm/apple/parser-test.c | 43 +++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/drivers/gpu/drm/apple/parser-test.c b/drivers/gpu/drm/apple/parser-test.c index d4590e34a9136..d2199656ce962 100644 --- a/drivers/gpu/drm/apple/parser-test.c +++ b/drivers/gpu/drm/apple/parser-test.c @@ -729,6 +729,48 @@ static void parser_attributes_malformed(struct kunit *test) dcp_modes_release(dcp); } +static void parser_optional_flag_extent(struct kunit *test) +{ + struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + struct parser_blob *blob = kunit_kzalloc(test, sizeof(*blob), GFP_KERNEL); + struct dcp_parse_ctx ctx; + + KUNIT_ASSERT_NOT_NULL(test, dcp); + KUNIT_ASSERT_NOT_NULL(test, blob); + dcp_modes_init(dcp); + put_unaligned_le32(0xd3, blob->data); + blob->size = 4; + blob_tag(blob, 1, 3); + blob_int(blob, "MaxHorizontalImageSize", 52); + blob_int(blob, "MaxVerticalImageSize", 29); + blob_bool(blob, "SupportsBacklightControl", true); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + KUNIT_ASSERT_EQ(test, dcp_attributes_replace(dcp, &ctx, 0), 0); + KUNIT_EXPECT_TRUE(test, dcp->ext_backlight); + + blob->size = 4; + blob_tag(blob, 1, 3); + blob_int(blob, "MaxHorizontalImageSize", 60); + blob_int(blob, "MaxVerticalImageSize", 40); + blob_key(blob, "SupportsBacklightControl"); + blob_tag(blob, 9, 2); + blob->data[blob->size++] = 'x'; + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + KUNIT_EXPECT_LT(test, dcp_attributes_replace(dcp, &ctx, 0), 0); + KUNIT_EXPECT_EQ(test, dcp->width_mm, 520); + KUNIT_EXPECT_EQ(test, dcp->height_mm, 290); + KUNIT_EXPECT_TRUE(test, dcp->ext_backlight); + + /* A complete value of another type keeps the optional false fallback. */ + blob->data[blob->size++] = 'y'; + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + KUNIT_ASSERT_EQ(test, dcp_attributes_replace(dcp, &ctx, 0), 0); + KUNIT_EXPECT_EQ(test, dcp->width_mm, 600); + KUNIT_EXPECT_EQ(test, dcp->height_mm, 400); + KUNIT_EXPECT_FALSE(test, dcp->ext_backlight); + dcp_modes_release(dcp); +} + static void parser_attribute_deferred(struct kunit *test) { struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); @@ -855,6 +897,7 @@ static struct kunit_case parser_cases[] = { KUNIT_CASE(parser_rejected_replacement), KUNIT_CASE(parser_attribute_epoch), KUNIT_CASE(parser_attributes_malformed), + KUNIT_CASE(parser_optional_flag_extent), KUNIT_CASE(parser_attribute_deferred), KUNIT_CASE(parser_dimensions_lifetime), KUNIT_CASE(parser_catalog_release), From ed4f5941532f6f96517b622e47665f391a814723 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 00:01:33 -0700 Subject: [PATCH 1601/2680] drm/apple: refresh connector status before hotplug notifications MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A connector can retain disconnected DRM status after DCP reports a live sink when no client probes it in response to hotplug. Refresh HPD-capable connectors through the DRM helper in the hotplug worker before notifying clients. Keep one notification for unchanged status so link retraining continues to reach clients, and retain direct notification for connectors without HPD polling. Honor forced connector states in the detection callback under the DRM helper locks. Exercise the production worker, detection callback and DRM client dispatch with connected, disconnected, retrain, unrouted, forced and non-HPD connector tests. [ryan: adapted to 7.2.10 — detect keeps our announcement-aware dcp_connector_connected() under the new forced-state handling; the hotplug worker refreshes status through drm_connector_helper_hpd_irq_event() except on an announce-gated H17P external connector, which keeps its plain event because the announcement already decides what is reported; hotplug-test.c drops the mutex_destroy() of the edid_lock this tree does not have] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit e2f682e966f3af2ea2abd46ce2b9fe913d781f44) (cherry picked from commit 4e34c5ac0bab17e77eb9fd84e96f88c8d10cdcd1) --- drivers/gpu/drm/apple/Kconfig | 5 +- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/apple_drv.c | 6 +- drivers/gpu/drm/apple/connector.h | 1 + drivers/gpu/drm/apple/hotplug-test.c | 254 +++++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb.c | 16 +- 6 files changed, 279 insertions(+), 4 deletions(-) create mode 100644 drivers/gpu/drm/apple/hotplug-test.c diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 5cc314ef94406..efad685cb042c 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -61,9 +61,12 @@ config DRM_APPLE_PARSER_KUNIT_TEST config DRM_APPLE_DISPLAY_KUNIT_TEST bool "KUnit tests for Apple display mode probing" if !KUNIT_ALL_TESTS depends on KUNIT && DRM_APPLE + select DRM_KUNIT_TEST_HELPERS + select DRM_CLIENT default KUNIT_ALL_TESTS help Test the production timing parser and display reply bounds without Apple display hardware, including incomplete records and stable preferred mode selection. Exercise mode list replacement and - catalog admission for each display attachment. + catalog admission for each display attachment. Check connector + status updates before hotplug notifications and same-state retrains. diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index f09d27d6ab2d6..28b38ec7ae974 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -39,3 +39,4 @@ obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk-service-test.o obj-$(CONFIG_DRM_APPLE_PARSER_KUNIT_TEST) += afkv2-test.o appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += parser-test.o +appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += hotplug-test.o diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 147dae4f14825..e1b46c72b76e4 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -117,11 +117,15 @@ static const struct drm_driver apple_drm_driver = { .fops = &apple_fops, }; -static enum drm_connector_status +enum drm_connector_status apple_connector_detect(struct drm_connector *connector, bool force) { struct apple_connector *apple_connector = to_apple_connector(connector); + if (connector->force == DRM_FORCE_ON || connector->force == DRM_FORCE_ON_DIGITAL) + return connector_status_connected; + if (connector->force == DRM_FORCE_OFF) + return connector_status_disconnected; return dcp_connector_connected(apple_connector) ? connector_status_connected : connector_status_disconnected; } diff --git a/drivers/gpu/drm/apple/connector.h b/drivers/gpu/drm/apple/connector.h index 581aa54efe68d..03a5ac620c429 100644 --- a/drivers/gpu/drm/apple/connector.h +++ b/drivers/gpu/drm/apple/connector.h @@ -17,6 +17,7 @@ struct apple_connector; void dcp_hotplug(struct work_struct *work); bool dcp_connector_connected(struct apple_connector *connector); void dcp_retrain_oob(struct apple_connector *connector); +enum drm_connector_status apple_connector_detect(struct drm_connector *connector, bool force); void apple_connector_backlight_init(struct apple_connector *connector); void apple_connector_backlight_sync(struct apple_connector *connector); diff --git a/drivers/gpu/drm/apple/hotplug-test.c b/drivers/gpu/drm/apple/hotplug-test.c new file mode 100644 index 0000000000000..8439dc788f60a --- /dev/null +++ b/drivers/gpu/drm/apple/hotplug-test.c @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include +#include +#include +#include +#include + +#include "connector.h" + +struct hotplug_fixture { + struct drm_device *drm; + struct apple_connector connector; + struct apple_dcp dcp; + struct platform_device pipeline; + struct drm_client_dev observer; + enum drm_connector_status observed_status; + u64 observed_link_status; + unsigned int notifications; + unsigned int detects; + unsigned int mode_probes; + bool notification_locked; +}; + +static enum drm_connector_status hotplug_detect(struct drm_connector *connector, bool force) +{ + struct hotplug_fixture *fixture = container_of(to_apple_connector(connector), + struct hotplug_fixture, connector); + + fixture->detects++; + return apple_connector_detect(connector, force); +} + +static int hotplug_get_modes(struct drm_connector *connector) +{ + struct hotplug_fixture *fixture = container_of(to_apple_connector(connector), + struct hotplug_fixture, connector); + + fixture->mode_probes++; + return 0; +} + +static const struct drm_connector_funcs hotplug_connector_funcs = { + .detect = hotplug_detect, + .reset = drm_atomic_helper_connector_reset, + .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, +}; + +static const struct drm_connector_helper_funcs hotplug_connector_helper_funcs = { + .get_modes = hotplug_get_modes, +}; + +static int hotplug_observe(struct drm_client_dev *client) +{ + struct hotplug_fixture *fixture = container_of(client, struct hotplug_fixture, observer); + + fixture->notifications++; + fixture->observed_status = fixture->connector.base.status; + fixture->observed_link_status = fixture->connector.base.state->link_status; + fixture->notification_locked = mutex_is_locked(&client->dev->mode_config.mutex) || + drm_modeset_is_locked(&client->dev->mode_config.connection_mutex); + return 0; +} + +static const struct drm_client_funcs hotplug_observer_funcs = { + .hotplug = hotplug_observe, +}; + +static void hotplug_cleanup(void *data) +{ + struct hotplug_fixture *fixture = data; + + mutex_lock(&fixture->drm->clientlist_mutex); + list_del_init(&fixture->observer.list); + mutex_unlock(&fixture->drm->clientlist_mutex); + cancel_work_sync(&fixture->connector.hotplug_wq); + cancel_work_sync(&fixture->connector.bl_sync_wq); + cancel_work_sync(&fixture->connector.bl_commit_wq); + apple_connector_invalidate_edid(&fixture->connector); + mutex_destroy(&fixture->connector.bl_lock); +} + +static int hotplug_test_init(struct kunit *test) +{ + struct hotplug_fixture *fixture; + struct drm_connector *connector; + struct device *dev; + int ret; + + fixture = kunit_kzalloc(test, sizeof(*fixture), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fixture); + test->priv = fixture; + dev = drm_kunit_helper_alloc_device(test); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + fixture->drm = __drm_kunit_helper_alloc_drm_device(test, dev, + sizeof(*fixture->drm), 0, + DRIVER_MODESET | DRIVER_ATOMIC); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fixture->drm); + connector = &fixture->connector.base; + ret = drmm_connector_init(fixture->drm, connector, &hotplug_connector_funcs, + DRM_MODE_CONNECTOR_DisplayPort, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + drm_connector_helper_add(connector, &hotplug_connector_helper_funcs); + connector->polled = DRM_CONNECTOR_POLL_HPD; + drm_atomic_helper_connector_reset(connector); + KUNIT_ASSERT_NOT_NULL(test, connector->state); + apple_connector_edid_init(&fixture->connector); + apple_connector_backlight_init(&fixture->connector); + INIT_WORK(&fixture->connector.hotplug_wq, dcp_hotplug); + fixture->dcp.dev = dev; + fixture->dcp.connector = &fixture->connector; + fixture->dcp.nr_modes = 1; + spin_lock_init(&fixture->dcp.mode_state.lock); + fixture->dcp.mode_state.valid = true; + platform_set_drvdata(&fixture->pipeline, &fixture->dcp); + apple_connector_set_pipeline(&fixture->connector, &fixture->pipeline); + fixture->observer.dev = fixture->drm; + fixture->observer.name = "apple-hotplug-observer"; + fixture->observer.funcs = &hotplug_observer_funcs; + mutex_lock(&fixture->drm->clientlist_mutex); + list_add(&fixture->observer.list, &fixture->drm->clientlist); + mutex_unlock(&fixture->drm->clientlist_mutex); + return kunit_add_action_or_reset(test, hotplug_cleanup, fixture); +} + +static void hotplug_expect_notification(struct kunit *test, + enum drm_connector_status status) +{ + struct hotplug_fixture *fixture = test->priv; + + KUNIT_EXPECT_EQ(test, fixture->connector.base.status, status); + KUNIT_EXPECT_EQ(test, fixture->observed_status, status); + KUNIT_EXPECT_EQ(test, fixture->notifications, 1U); + KUNIT_EXPECT_EQ(test, fixture->detects, 1U); + KUNIT_EXPECT_EQ(test, fixture->mode_probes, 0U); + KUNIT_EXPECT_FALSE(test, fixture->notification_locked); +} + +static void hotplug_connected(struct kunit *test) +{ + struct hotplug_fixture *fixture = test->priv; + + fixture->connector.base.status = connector_status_disconnected; + fixture->connector.connected = true; + dcp_hotplug(&fixture->connector.hotplug_wq); + hotplug_expect_notification(test, connector_status_connected); +} + +static void hotplug_disconnected(struct kunit *test) +{ + struct hotplug_fixture *fixture = test->priv; + + fixture->connector.base.status = connector_status_connected; + fixture->connector.connected = false; + dcp_hotplug(&fixture->connector.hotplug_wq); + hotplug_expect_notification(test, connector_status_disconnected); +} + +static void hotplug_retrain(struct kunit *test) +{ + struct hotplug_fixture *fixture = test->priv; + + fixture->connector.base.status = connector_status_connected; + fixture->connector.connected = true; + fixture->dcp.mode_state.valid = false; + dcp_hotplug(&fixture->connector.hotplug_wq); + hotplug_expect_notification(test, connector_status_connected); + KUNIT_EXPECT_EQ(test, fixture->observed_link_status, (u64)DRM_MODE_LINK_STATUS_BAD); +} + +static void hotplug_unrouted(struct kunit *test) +{ + struct hotplug_fixture *fixture = test->priv; + + fixture->connector.base.status = connector_status_connected; + fixture->connector.connected = false; + apple_connector_set_pipeline(&fixture->connector, NULL); + dcp_hotplug(&fixture->connector.hotplug_wq); + hotplug_expect_notification(test, connector_status_disconnected); +} + +static void hotplug_forced_on(struct kunit *test) +{ + struct hotplug_fixture *fixture = test->priv; + + fixture->connector.base.force = DRM_FORCE_ON; + fixture->connector.base.status = connector_status_connected; + fixture->connector.connected = false; + dcp_hotplug(&fixture->connector.hotplug_wq); + hotplug_expect_notification(test, connector_status_connected); +} + +static void hotplug_forced_digital(struct kunit *test) +{ + struct hotplug_fixture *fixture = test->priv; + + fixture->connector.base.force = DRM_FORCE_ON_DIGITAL; + fixture->connector.base.status = connector_status_connected; + fixture->connector.connected = false; + dcp_hotplug(&fixture->connector.hotplug_wq); + hotplug_expect_notification(test, connector_status_connected); +} + +static void hotplug_forced_off(struct kunit *test) +{ + struct hotplug_fixture *fixture = test->priv; + + fixture->connector.base.force = DRM_FORCE_OFF; + fixture->connector.base.status = connector_status_disconnected; + fixture->connector.connected = true; + dcp_hotplug(&fixture->connector.hotplug_wq); + hotplug_expect_notification(test, connector_status_disconnected); +} + +static void hotplug_without_hpd(struct kunit *test) +{ + struct hotplug_fixture *fixture = test->priv; + + fixture->connector.base.polled = 0; + fixture->connector.base.status = connector_status_disconnected; + fixture->connector.connected = true; + dcp_hotplug(&fixture->connector.hotplug_wq); + KUNIT_EXPECT_EQ(test, fixture->connector.base.status, connector_status_disconnected); + KUNIT_EXPECT_EQ(test, fixture->observed_status, connector_status_disconnected); + KUNIT_EXPECT_EQ(test, fixture->notifications, 1U); + KUNIT_EXPECT_EQ(test, fixture->detects, 0U); + KUNIT_EXPECT_EQ(test, fixture->mode_probes, 0U); + KUNIT_EXPECT_FALSE(test, fixture->notification_locked); +} + +static struct kunit_case hotplug_test_cases[] = { + KUNIT_CASE(hotplug_connected), + KUNIT_CASE(hotplug_disconnected), + KUNIT_CASE(hotplug_retrain), + KUNIT_CASE(hotplug_unrouted), + KUNIT_CASE(hotplug_forced_on), + KUNIT_CASE(hotplug_forced_digital), + KUNIT_CASE(hotplug_forced_off), + KUNIT_CASE(hotplug_without_hpd), + { } +}; + +static struct kunit_suite hotplug_test_suite = { + .name = "apple-display-hotplug", + .init = hotplug_test_init, + .test_cases = hotplug_test_cases, +}; + +kunit_test_suite(hotplug_test_suite); + +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 9812af882a30e..0eb11c53a0446 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1087,6 +1087,14 @@ void dcp_external_crtc_released(struct apple_dcp *dcp) schedule_work(&con->hotplug_wq); } +static void dcp_notify_hotplug(struct apple_connector *connector) +{ + /* Same-state retrains also need a notification after refreshing status. */ + if (!(connector->base.polled & DRM_CONNECTOR_POLL_HPD) || + !drm_connector_helper_hpd_irq_event(&connector->base)) + drm_kms_helper_connector_hotplug_event(&connector->base); +} + void dcp_hotplug(struct work_struct *work) { struct apple_connector *connector; @@ -1107,7 +1115,7 @@ void dcp_hotplug(struct work_struct *work) connector->drm_edid_firmware_derived = false; } } - drm_kms_helper_connector_hotplug_event(&connector->base); + dcp_notify_hotplug(connector); apple_connector_backlight_sync(connector); return; } @@ -1167,7 +1175,11 @@ void dcp_hotplug(struct work_struct *work) } dcp_test_point(dcp, DCP_TEST_HOTPLUG_EVENT); - drm_kms_helper_connector_hotplug_event(&connector->base); + /* The announcement already decided what an H17P external reports. */ + if (gated) + drm_kms_helper_connector_hotplug_event(&connector->base); + else + dcp_notify_hotplug(connector); /* after the event, so registering a backlight cannot delay it */ apple_connector_backlight_sync(connector); } From 930be8f6f9453a4530623a7aabc76274aa94b1df Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 09:56:14 -0700 Subject: [PATCH 1602/2680] drm/apple: t6030: check each external display processor's own handoff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The T6030 has two external display processors, and the boot loader hands each one over separately. The driver's checks before a start assumed the first one: - the memory check used fixed DART addresses for the firmware, data and heap regions; - the power check required the dispext0_cpu domain; - the manual scanout path required the first display DART's address; - one global flag stood for "a DCP session runs", which the panel and an external processor would share. Read each region's DART address from its iommu-addresses entry for this processor, the way the IOMMU core reserves it, and check that the region is mapped linearly there. Accept the processor's own dispextN_cpu domain. In the manual scanout path, require that the display DART shares the processor's power domain instead of a fixed address. Name the session flag for the panel, which is what the manual start waits for. For the first processor nothing changes: its regions resolve to the same addresses as before. [ryan: adapted to 7.2.10 — of_translate_dma_region_checked() comes from 42d7be2c9500 (900-config-tooling owns include/linux/of_address.h), so iomfb_v14_7.h carries a local prototype; this file links only with that bucket] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit a76531ce4e580c8591ac2278f397249834a3fc37) --- drivers/gpu/drm/apple/dcpext_scanout.c | 15 +++-- drivers/gpu/drm/apple/iomfb_v14_7.c | 91 ++++++++++++++++++++++---- drivers/gpu/drm/apple/iomfb_v14_7.h | 9 +++ 3 files changed, 97 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/apple/dcpext_scanout.c b/drivers/gpu/drm/apple/dcpext_scanout.c index 17e74998d321e..86b20d5541f79 100644 --- a/drivers/gpu/drm/apple/dcpext_scanout.c +++ b/drivers/gpu/drm/apple/dcpext_scanout.c @@ -459,8 +459,9 @@ static int scanout_check_domains(struct dcpext_scanout *scanout) * before attaching: neither firmware nor another client may have populated * the inherited roots in the meantime. This function performs no MMIO writes. */ -static int scanout_recheck_dart(struct device_node *dart) +static int scanout_recheck_dart(struct device_node *dart, struct device_node *dcp) { + struct device_node *dart_pd, *dcp_pd; struct resource regs, table, first = {}; struct device_node *region = NULL; struct reserved_mem *rmem; @@ -475,8 +476,14 @@ static int scanout_recheck_dart(struct device_node *dart) of_property_read_u32_array(dart, "apple,inherited-dart-state", state, 6) || state[0] != 0 || state[3] != 4 || of_count_phandle_with_args(dart, "memory-region", NULL) != 2 || - of_address_to_resource(dart, 0, ®s) || - regs.start != 0x2d1304000ULL || resource_size(®s) != SZ_16K) + of_address_to_resource(dart, 0, ®s) || resource_size(®s) != SZ_16K) + return -EINVAL; + /* The display DART of this processor's own pipe shares its CPU domain. */ + dart_pd = of_parse_phandle(dart, "power-domains", 0); + dcp_pd = of_parse_phandle(dcp, "power-domains", 0); + of_node_put(dart_pd); + of_node_put(dcp_pd); + if (!dart_pd || dart_pd != dcp_pd) return -EINVAL; mmio = ioremap(regs.start, resource_size(®s)); if (!mmio) @@ -543,7 +550,7 @@ static int scanout_create_device(struct dcpext_scanout *scanout) of_device_is_compatible(spec.np, "apple,t8110-dart") && of_device_is_available(spec.np) ? 0 : -EINVAL; if (!ret) - ret = scanout_recheck_dart(spec.np); + ret = scanout_recheck_dart(spec.np, scanout->firmware_dev->of_node); of_node_put(spec.np); if (ret) return ret; diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 4ad5e717c2cbb..0468f468deba7 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -141,8 +141,12 @@ struct apple_dcp_v14 { u64 swap_ns_max; }; -/* One DCP session per boot: RTKit is never started twice. */ -static bool dcp_v14_session; +/* + * The panel's DCP session; one per boot, RTKit is never started twice. The + * manual external start waits for it, as the PMP runs once the panel's + * request is acknowledged. + */ +static bool dcp_v14_panel_session; static int dcp_v14_callback(void *cookie, u32 tag, const void *input, u32 in_size, void *output, u32 out_size); @@ -919,11 +923,67 @@ static int dcp_v14_geometry(struct apple_dcp_v14 *v14) static void dcpext_bringup(struct work_struct *work); -/* Verify the mappings installed by the IOMMU core before touching the ASC. */ +/* + * The bounded translator (900-config-tooling) is optional at link time; + * without it, refuse a tuple that does not fit before using the unbounded one. + */ +static const __be32 *dcpext_translate_region(struct device_node *np, + const __be32 *maps, size_t cells, + phys_addr_t *iova, size_t *length) +{ + typeof(&of_translate_dma_region_checked) checked; + const __be32 *next; + + checked = symbol_get(of_translate_dma_region_checked); + if (checked) { + next = checked(np, maps, cells, iova, length); + symbol_put(of_translate_dma_region_checked); + return next; + } + if (cells < (size_t)(of_n_addr_cells(np) + of_n_size_cells(np))) + return NULL; + return of_translate_dma_region(np, maps, iova, length); +} + +/* + * The IOVA at which @mem's iommu-addresses map @size bytes for the device + * @np, as the IOMMU core reads them; 0 if they do not. + */ +static u64 dcpext_region_iova(struct device_node *np, struct device_node *mem, u64 size) +{ + const __be32 *maps, *end; + int len; + + maps = of_get_property(mem, "iommu-addresses", &len); + if (!maps || len <= 0 || len % sizeof(*maps)) + return 0; + end = maps + len / sizeof(*maps); + while (maps < end) { + struct device_node *owner = of_find_node_by_phandle(be32_to_cpup(maps++)); + phys_addr_t iova; + size_t length; + bool mine = owner == np; + + if (!owner) + return 0; + maps = dcpext_translate_region(owner, maps, end - maps, &iova, &length); + of_node_put(owner); + if (!maps) + return 0; + if (mine && length == size) + return iova; + } + return 0; +} + +/* + * Verify the mappings the IOMMU core installed from the boot loader's + * handoff, before touching the processor: each region is mapped linearly at + * the address its iommu-addresses give this processor. + */ static int dcpext_verify_memory(struct apple_dcp *dcp) { static const char *const names[] = {"asc-firmware", "dcp_data", "heap"}; - static const u64 iovas[] = {0x10040000000ULL, 0x10080000000ULL, 0x100c0000000ULL}; struct device *dev = dcp->dev; struct iommu_domain *domain = iommu_get_domain_for_dev(dev); u32 ready; @@ -935,7 +995,7 @@ static int dcpext_verify_memory(struct apple_dcp *dcp) for (i = 0; i < ARRAY_SIZE(names); i++) { struct device_node *mem; struct resource res; - u64 off, size; + u64 off, size, iova; int idx = of_property_match_string(dev->of_node, "memory-region-names", names[i]); if (idx < 0) return idx; @@ -946,20 +1006,21 @@ static int dcpext_verify_memory(struct apple_dcp *dcp) of_node_put(mem); return -EINVAL; } - of_node_put(mem); size = resource_size(&res); + iova = dcpext_region_iova(dev->of_node, mem, size); + of_node_put(mem); if (!size || size > SZ_256M || !IS_ALIGNED(res.start, SZ_16K) || - !IS_ALIGNED(size, SZ_16K)) + !IS_ALIGNED(size, SZ_16K) || !iova || !IS_ALIGNED(iova, SZ_16K)) return -EINVAL; for (off = 0; off < size; off += SZ_16K) { - if (iommu_iova_to_phys(domain, iovas[i] + off) != res.start + off) { + if (iommu_iova_to_phys(domain, iova + off) != res.start + off) { dev_err(dev, "dcpext %s mapping mismatch at %#llx; CPU untouched\n", - names[i], iovas[i] + off); + names[i], iova + off); return -EINVAL; } } dev_info(dev, "dcpext verified %s: %pa + %#llx at %#llx\n", - names[i], &res.start, size, iovas[i]); + names[i], &res.start, size, iova); } return 0; } @@ -977,7 +1038,7 @@ static ssize_t dcpext_start_store(struct device *dev, struct device_attribute *a guard(mutex)(&dcp->hpd_mutex); if (READ_ONCE(dcp->external_suspended)) return -EBUSY; - if (!dcp_v14_session) + if (!dcp_v14_panel_session) return -EAGAIN; if (atomic_cmpxchg(&dcp->external_requested, 0, 1)) return -EBUSY; @@ -1035,8 +1096,10 @@ static int dcpext_check_power(struct apple_dcp *dcp) ps = of_parse_phandle(dev->of_node, "power-domains", 0); if (!ps) return -ENODEV; + /* The CPU domain of this pipe: dispext_cpu. */ ret = of_property_read_string(ps, "label", &label); - if (ret || strcmp(label, "dispext0_cpu")) { + if (ret || !strstarts(label, "dispext") || strlen(label) < 4 || + strcmp(label + strlen(label) - 4, "_cpu")) { of_node_put(ps); return -EINVAL; } @@ -1155,7 +1218,7 @@ int iomfb_v14_7_bind(struct apple_dcp *dcp) if (!v14) return -ENODEV; - if (v14->rtk || dcp_v14_session) + if (v14->rtk || dcp_v14_panel_session) return dev_err_probe(dev, -EBUSY, "T6030 display not started: an earlier DCP session is kept; reboot to restart the display\n"); @@ -1189,7 +1252,7 @@ int iomfb_v14_7_bind(struct apple_dcp *dcp) v14->rtk = rtk; v14->link.rtk = rtk; dcp->rtk = rtk; - dcp_v14_session = true; + dcp_v14_panel_session = true; __module_get(THIS_MODULE); ret = apple_rtkit_wake(rtk); diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.h b/drivers/gpu/drm/apple/iomfb_v14_7.h index a262ccc053854..c035256269647 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7.h @@ -10,6 +10,15 @@ struct apple_dcp_v14; struct drm_atomic_commit; struct drm_crtc; struct drm_crtc_state; +struct device_node; + +/* + * Bounded DMA-region translation from drivers/of/address.c (42d7be2c9500, + * bucket 900-config-tooling, which owns include/linux/of_address.h). + */ +const __be32 *of_translate_dma_region_checked(struct device_node *dev, + const __be32 *addr, size_t cells, + phys_addr_t *start, size_t *length); /* Platform probe: firmware identity, boot loader handoff, PMP acknowledgment. */ int iomfb_v14_7_probe(struct apple_dcp *dcp); From 2207700f8349d3a2ba740a2294778dff407abe1d Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Tue, 6 Oct 2026 09:58:02 -0700 Subject: [PATCH 1603/2680] drm/apple: t6030: answer an external processor's display callbacks An external display processor runs the same 14.7 firmware image as the panel's DCP, and talks to the host over the same DCPLink calls and callbacks, with a few differences for a pipe that has no panel: - it sends will-power-off (D002), display DART power (D574) and firmware-generated swap notices (D591), which retire no framebuffer; - the main-display, A444 and PMU queries answer differently, and no panel backlight service is registered (D207 without A132); - the attached display's description (TimingElements, ColorElements, DisplayAttributes, Transport) arrives as chunked or dictionary properties, and is removed again (D107) when the display goes. Mark a session as external and answer those callbacks for it. Keep every service-0 property raw and drop it again on removal, so the description of the display that is attached now is what the driver reads. A display property that is malformed or too large is refused with a false reply, so a display's metadata cannot end the session, and a callback this driver does not know gets a zeroed reply on an external processor instead of stopping the link. Nothing sets the flag yet; the panel's session is unchanged. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 78826ea54c06b9cf560f2fb01026eb14c2295df1) --- drivers/gpu/drm/apple/iomfb_v14_7.c | 170 ++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 0468f468deba7..85f17689558e9 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -115,6 +115,12 @@ struct apple_dcp_v14 { bool failed; bool started; + /* + * An external processor: no panel. Its modes come from the + * properties the firmware publishes for the attached display. + */ + bool external; + /* Boot framebuffer and native panel timing (notch rows included). */ u32 stride; u32 fb_width, fb_height; @@ -334,6 +340,158 @@ static int dcp_v14_map_piodma(struct apple_dcp_v14 *v14, u32 id) return 0; } +/* Removes a raw property of service 0. */ +static void dcp_v14_raw_remove(struct apple_dcp_v14 *v14, const char *key) +{ + u32 i; + + for (i = 0; i < v14->raw_count; i++) { + if (v14->raw[i].service || strcmp(v14->raw[i].key, key)) + continue; + v14->raw_bytes -= v14->raw[i].size; + kvfree(v14->raw[i].data); + v14->raw[i] = v14->raw[--v14->raw_count]; + memset(&v14->raw[v14->raw_count], 0, sizeof(v14->raw[0])); + return; + } +} + +/* A call whose u32 answer is informational only. */ +static int dcp_v14_query(struct apple_dcp_v14 *v14, u32 tag) +{ + __le32 result = 0; + int ret; + + ret = dcp_v14_call(v14, tag, NULL, 0, &result, sizeof(result), 0); + if (!ret) + dev_dbg(v14->dev, "call %#x answered %u\n", tag, le32_to_cpu(result)); + return ret; +} + +/* + * Callbacks that an external processor answers differently, or sends only + * there. -ENOENT leaves @tag to the shared handler. A display's property + * transfer that is malformed or too large is refused with a false reply: + * it must not end the session. + */ +static int dcp_v14_external_callback(struct apple_dcp_v14 *v14, u32 tag, const u8 *in, + u32 in_size, u8 *out, u32 out_size) +{ + struct dcp_v14_property *p; + u32 count, offset, service, key_offset; + int ret; + +#define SHAPE(i, o) (in_size == (i) && out_size == (o)) + /* Will power off, and a swap the firmware made itself: nothing to retire. */ + if ((tag == D(2) && SHAPE(0, 0)) || (tag == D(591) && SHAPE(20, 0))) + return 0; + /* Display DART power, a bool: the DART is powered with the processor. */ + if (tag == D(574) && SHAPE(4, 4)) + return in[0] > 1 ? -EINVAL : 0; + /* Main-display query: answered by asking the firmware. */ + if (tag == D(599) && SHAPE(0, 0)) + return dcp_v14_query(v14, A(410)); + /* Late boot: the panel's sequence; the first answer differs here. */ + if (tag == D(121) && SHAPE(0, 4)) { + ret = dcp_v14_query(v14, A(444)); + if (!ret) + ret = dcp_v14_simple_call(v14, A(29), false, false, false); + if (!ret) + ret = dcp_v14_simple_call(v14, A(466), true, false, false); + if (!ret) + ret = dcp_v14_simple_call(v14, A(0), true, true, true); + if (!ret) + ret = dcp_v14_simple_call(v14, A(463), false, true, true); + if (!ret) + out[0] = 1; + return ret; + } + /* PMU service matching; an external display has no panel backlight. */ + if (tag == D(206) && SHAPE(0, 4)) { + ret = dcp_v14_query(v14, A(131)); + if (!ret) + out[0] = 1; + return ret; + } + if (tag == D(207) && SHAPE(0, 4)) { + out[0] = 1; + return 0; + } + /* Chunked property transfer: start, chunk, end. */ + if (tag == D(127) && SHAPE(4, 4)) { + count = get_unaligned_le32(in); + ret = -EINVAL; + if (!v14->chunk && count && count <= 0x100001) { + v14->chunk_size = count - 1; + v14->chunk_offset = 0; + v14->chunk = kvzalloc(max_t(u32, 1, v14->chunk_size), GFP_KERNEL); + ret = v14->chunk ? 0 : -ENOMEM; + } + goto property_reply; + } + if (tag == D(128) && SHAPE(0x1008, 4)) { + offset = get_unaligned_le32(in + 0x1000); + count = get_unaligned_le32(in + 0x1004); + ret = -EINVAL; + if (v14->chunk && offset == v14->chunk_offset && count <= 4096 && + offset <= v14->chunk_size && count <= v14->chunk_size - offset) { + memcpy(v14->chunk + offset, in, count); + v14->chunk_offset += count; + ret = 0; + } + goto property_reply; + } + if (tag == D(129) && SHAPE(64, 4)) { + ret = -EINVAL; + if (v14->chunk && v14->chunk_offset == v14->chunk_size && memchr(in, 0, 64)) + ret = dcp_v14_raw_property(v14, 0, in, v14->chunk, v14->chunk_size); + kvfree(v14->chunk); + v14->chunk = NULL; + goto property_reply; + } + /* Dictionary properties, kept raw. */ + if ((tag == D(413) && SHAPE(4168, 4)) || + ((tag == D(552) || tag == D(561)) && SHAPE(4164, 4)) || + (tag == D(567) && SHAPE(128, 4))) { + key_offset = tag == D(413) ? 4 : 0; + service = key_offset ? get_unaligned_le32(in) : 0; + if (tag != D(567) && (in[key_offset + 4160] & 1)) { + out[0] = 1; + return 0; + } + count = tag == D(567) ? strnlen(in + 64, 64) : 4096; + ret = dcp_v14_raw_property(v14, service, in + key_offset, in + key_offset + 64, + count); + goto property_reply; + } + /* Property removal, including the display's description on unplug. */ + if (tag == D(107) && SHAPE(64, 0)) { + if (!memchr(in, 0, 64)) + return -EINVAL; + dcp_v14_raw_remove(v14, in); + p = dcp_v14_property(v14, 0, in, false); + if (IS_ERR(p)) + return PTR_ERR(p); + if (p) { + *p = v14->properties[--v14->property_count]; + memset(&v14->properties[v14->property_count], 0, sizeof(*p)); + } + return 0; + } +#undef SHAPE + return -ENOENT; + +property_reply: + if (ret) { + dev_warn_ratelimited(v14->dev, "display property refused: %d\n", ret); + kvfree(v14->chunk); + v14->chunk = NULL; + } + /* These answer a bool: false refuses the data, not the link. */ + out[0] = !ret; + return 0; +} + /* Runs on the thread that owns the RPC stream; the lock is held. */ static int dcp_v14_callback(void *cookie, u32 tag, const void *input, u32 in_size, void *output, u32 out_size) @@ -345,6 +503,12 @@ static int dcp_v14_callback(void *cookie, u32 tag, const void *input, u32 in_siz u32 count, offset, service, key_offset, id; int ret; + if (v14->external) { + ret = dcp_v14_external_callback(v14, tag, in, in_size, out, out_size); + if (ret != -ENOENT) + return ret; + } + #define SHAPE(i, o) (in_size == (i) && out_size == (o)) /* get_time */ if (tag == D(209) && SHAPE(0, 8)) { @@ -658,6 +822,12 @@ static int dcp_v14_callback(void *cookie, u32 tag, const void *input, u32 in_siz return dcp_v14_call(v14, A(374), NULL, 0, &result, sizeof(result), 0); } #undef SHAPE + if (v14->external) { + /* A zeroed reply reads as false or none; keep the display session. */ + dev_warn_ratelimited(v14->dev, "unhandled DCP callback %#x %u/%u, answered with zeros\n", + tag, in_size, out_size); + return 0; + } dev_err(v14->dev, "unhandled DCP callback %#x %u/%u\n", tag, in_size, out_size); return -EOPNOTSUPP; } From e85571ebe93cdf896033a453582faa8fec410503 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 09:58:37 -0700 Subject: [PATCH 1604/2680] drm/apple: t6030: show an external display at the modes of its EDID MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On an external processor, the firmware describes the attached display with properties derived from its EDID: TimingElements (the timings with their color modes), DisplayAttributes and ColorElements. They are in the dictionary format the M1/M2 firmware uses, so the shared parser reads them. Make an external session a display pipe like the M1/M2 ones: - TimingElements replace the mode catalog, guarded by the catalog generation of the route that was active when the transfer began, and report the display as connected; their removal reports it as gone. This follows the M1/M2 hotplug callback: an unplug caused by powering a Type-C CRTC off with the cable still in only invalidates the mode, and a description that outlives its cable is ignored. - DisplayAttributes give the physical size; each description is also kept for the connector's debugfs. - A modeset sets the display's timing and color mode by their ids (A411) between powering it down and up (A472); disabling the CRTC powers it down and invalidates the mode, so the next enable sets it again. - Swaps go to the pipe only while the display is powered, in a valid mode and connected; otherwise the flip completes at once. A swap the firmware refuses ends nothing on an external pipe, and its framebuffer is released. An external pipe may show the top-left of a larger framebuffer, as fbdev does when it clones the panel's. - A firmware crash reports the display as gone. - The PIODMA child of an external pipe is the one the display gate adds on its display DART. The panel's session is unchanged. [ryan: adapted to 7.2.10 — dcp_mode_set_valid() does not exist here: the mode is marked valid under mode_state.lock as elsewhere in this file] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit d374b79c8d88a5645bc6efade6764034ca52de79) --- drivers/gpu/drm/apple/iomfb_v14_7.c | 263 ++++++++++++++++++++++++++-- 1 file changed, 251 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 85f17689558e9..491ac97e59f33 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -117,9 +117,16 @@ struct apple_dcp_v14 { /* * An external processor: no panel. Its modes come from the - * properties the firmware publishes for the attached display. + * TimingElements the firmware publishes for the attached display, + * and the kernel sets the mode (A411) and the power state (A472). */ bool external; + /* External: start signal and first client open done. */ + bool opened; + /* External: the display is powered (A472) in the mode last set. */ + bool powered; + /* External: catalog generation when a chunked property started. */ + u64 chunk_generation; /* Boot framebuffer and native panel timing (notch rows included). */ u32 stride; @@ -304,9 +311,11 @@ static int dcp_v14_map_piodma(struct apple_dcp_v14 *v14, u32 id) if (buf->piodma_mapped) return 0; if (!v14->piodma) { + /* The display gate adds an external pipe's piodma with its DART. */ node = of_get_child_by_name(v14->dev->of_node, "piodma"); if (!node || !of_device_is_available(node) || - of_property_read_u32(node, "apple,t6030-handoff", &marker) || marker != 1) { + of_property_read_u32(node, v14->external ? "apple,t6030-dispext-handoff" : + "apple,t6030-handoff", &marker) || marker != 1) { of_node_put(node); return -ENODEV; } @@ -368,6 +377,110 @@ static int dcp_v14_query(struct apple_dcp_v14 *v14, u32 tag) return ret; } +/* + * The attached display came or went, as the firmware's display description + * shows it. Mirrors the hotplug callback of the M1/M2 firmware. + */ +static void dcp_v14_external_hotplug(struct apple_dcp *dcp, bool connected) +{ + struct apple_connector *connector = READ_ONCE(dcp->connector); + unsigned int action; + + /* + * Powering a Type-C CRTC off releases the link, and the firmware then + * withdraws the display. The cable is still in: keep the connector + * as it is and let the next power-on bring the link back. + */ + if (!connected && READ_ONCE(dcp->typec_crtc_off) && + READ_ONCE(dcp->typec_cable_connected)) { + dcp_mode_invalidate(&dcp->mode_state); + return; + } + /* A description that outlives its cable is stale. */ + if (connected && dcp_is_typec_output(dcp) && !READ_ONCE(dcp->typec_cable_connected)) + return; + if (connected && dcp_is_typec_output(dcp)) + complete_all(&dcp->typec_iomfb_hpd_ready); + if (!connected) + WRITE_ONCE(dcp->ext_backlight, false); + if (!connector) + return; + apple_connector_edid_set_live(connector, connected); + action = dcp_mode_hotplug(&dcp->mode_state, connected, &connector->connected); + if (connected && !READ_ONCE(dcp->mode_state.valid) && + !READ_ONCE(dcp->mode_state.changing)) + action |= DCP_HOTPLUG_NOTIFY; + if (!dcp->crtc) + action &= ~DCP_HOTPLUG_VBLANK; + dcp_handle_hotplug_actions(dcp, action); +} + +/* + * A display description the firmware published (or removed, @removed) on an + * external processor: the timings and color modes of the attached display + * (from its EDID), its attributes, and the link transport. @generation is + * the mode catalog generation when the transfer started. + */ +static void dcp_v14_external_published(struct apple_dcp_v14 *v14, const char *key, + u64 generation, bool removed) +{ + struct apple_dcp *dcp = READ_ONCE(v14->dcp); + struct apple_connector *connector; + struct dcp_v14_raw *raw = NULL; + struct dcp_parse_ctx ctx; + u32 i; + int ret; + + if (!dcp || (strcmp(key, "TimingElements") && strcmp(key, "DisplayAttributes") && + strcmp(key, "ColorElements") && strcmp(key, "Transport"))) + return; + for (i = 0; i < v14->raw_count && !removed; i++) + if (!v14->raw[i].service && !strcmp(v14->raw[i].key, key)) + raw = &v14->raw[i]; + if (!removed && !raw) + return; + dev_info(dcp->dev, "display %s %s, %u bytes\n", key, removed ? "withdrawn" : "published", + raw ? raw->size : 0); + + /* The connector's debugfs keeps a copy. */ + connector = READ_ONCE(dcp->connector); + if (raw && raw->size && connector) { + struct dcp_chunks chunks = { + .length = raw->size, + .data = kmemdup(raw->data, raw->size, GFP_KERNEL), + }; + + if (chunks.data) + dcp_connector_update_dict(connector, key, &chunks); + } + + if (!strcmp(key, "TimingElements")) { + if (removed) { + dcp_v14_external_hotplug(dcp, false); + return; + } + ret = parse(raw->data, raw->size, &ctx); + if (!ret) { + ctx.dcp = dcp; + ret = dcp_modes_replace(dcp, &ctx, generation); + } + if (ret) { + dev_warn(dcp->dev, "display timings not used: %d\n", ret); + return; + } + dev_info(dcp->dev, "display has %u usable modes\n", READ_ONCE(dcp->nr_modes)); + dcp_v14_external_hotplug(dcp, READ_ONCE(dcp->nr_modes) > 0); + } else if (!strcmp(key, "DisplayAttributes") && !removed) { + ret = parse(raw->data, raw->size, &ctx); + if (!ret) { + ctx.dcp = dcp; + ret = dcp_attributes_replace(dcp, &ctx, generation); + } + if (ret) + dev_warn(dcp->dev, "display attributes not used: %d\n", ret); + } +} + /* * Callbacks that an external processor answers differently, or sends only * there. -ENOENT leaves @tag to the shared handler. A display's property @@ -426,6 +539,8 @@ static int dcp_v14_external_callback(struct apple_dcp_v14 *v14, u32 tag, const u v14->chunk_offset = 0; v14->chunk = kvzalloc(max_t(u32, 1, v14->chunk_size), GFP_KERNEL); ret = v14->chunk ? 0 : -ENOMEM; + if (v14->dcp) + v14->chunk_generation = dcp_modes_transfer_begin(v14->dcp); } goto property_reply; } @@ -447,6 +562,8 @@ static int dcp_v14_external_callback(struct apple_dcp_v14 *v14, u32 tag, const u ret = dcp_v14_raw_property(v14, 0, in, v14->chunk, v14->chunk_size); kvfree(v14->chunk); v14->chunk = NULL; + if (!ret) + dcp_v14_external_published(v14, in, v14->chunk_generation, false); goto property_reply; } /* Dictionary properties, kept raw. */ @@ -462,6 +579,9 @@ static int dcp_v14_external_callback(struct apple_dcp_v14 *v14, u32 tag, const u count = tag == D(567) ? strnlen(in + 64, 64) : 4096; ret = dcp_v14_raw_property(v14, service, in + key_offset, in + key_offset + 64, count); + if (!ret && !service && v14->dcp) + dcp_v14_external_published(v14, in + key_offset, + dcp_modes_transfer_begin(v14->dcp), false); goto property_reply; } /* Property removal, including the display's description on unplug. */ @@ -476,6 +596,7 @@ static int dcp_v14_external_callback(struct apple_dcp_v14 *v14, u32 tag, const u *p = v14->properties[--v14->property_count]; memset(&v14->properties[v14->property_count], 0, sizeof(*p)); } + dcp_v14_external_published(v14, in, 0, true); return 0; } #undef SHAPE @@ -855,9 +976,17 @@ static void dcp_v14_crashed(void *cookie, const void *crashlog, size_t crashlog_ WRITE_ONCE(v14->failed, true); if (dcp) { + struct apple_connector *connector = READ_ONCE(dcp->connector); + WRITE_ONCE(dcp->crashed, true); if (dcp->external) dcpext_scanout_fault(dcp, -EIO); + /* The display it drove is gone until reboot. */ + if (v14->external && connector) { + WRITE_ONCE(connector->connected, false); + apple_connector_edid_set_live(connector, false); + schedule_work(&connector->hotplug_wq); + } } dev_err(v14->dev, "DCP firmware crashed; its buffers are kept until reboot\n"); dcp_v14_link_fail(&v14->link); @@ -1666,7 +1795,12 @@ static int dcp_v14_swap(struct apple_dcp_v14 *v14, const u8 *surface, u64 iova, if (!ret && get_unaligned_le32((u8 *)result + 5)) ret = -EIO; } - if (ret) + /* + * An external display can go away under a swap: the firmware then + * refuses it, which ends nothing. Link failures stop the session in + * dcp_v14_call() either way. + */ + if (ret && !v14->external) v14->failed = true; mutex_unlock(&v14->lock); @@ -1674,8 +1808,12 @@ static int dcp_v14_swap(struct apple_dcp_v14 *v14, const u8 *surface, u64 iova, return ret; } -/* Shows @fb (or only the black background) and keeps it until the next swap. */ -static bool dcp_v14_present(struct apple_dcp_v14 *v14, struct drm_framebuffer *fb, u32 dst_y) +/* + * Shows the top-left @width x @height of @fb (or only the black background) + * and keeps @fb until the next swap. + */ +static bool dcp_v14_present(struct apple_dcp_v14 *v14, struct drm_framebuffer *fb, + u32 width, u32 height, u32 dst_y) { u8 surface[DCP_V14_SURFACE_SIZE]; struct drm_framebuffer *old; @@ -1691,9 +1829,16 @@ static bool dcp_v14_present(struct apple_dcp_v14 *v14, struct drm_framebuffer *f iova = drm_fb_dma_get_gem_obj(fb, 0)->dma_addr; } start = ktime_get_ns(); - ret = dcp_v14_swap(v14, fb ? surface : NULL, iova, fb ? fb->width : 0, - fb ? fb->height : 0, dst_y); + ret = dcp_v14_swap(v14, fb ? surface : NULL, iova, fb ? width : 0, + fb ? height : 0, dst_y); elapsed = ktime_get_ns() - start; + if (ret && v14->external && !READ_ONCE(v14->failed)) { + /* Refused before it was taken: the old framebuffer stays on screen. */ + if (fb) + drm_framebuffer_put(fb); + dev_warn_ratelimited(v14->dev, "display refused a swap: %d\n", ret); + return false; + } if (ret) { /* Either framebuffer may still be scanned out: keep both. */ dev_err(v14->dev, "native DCP flip failed %d; buffers pinned until reboot\n", ret); @@ -1750,8 +1895,15 @@ int iomfb_v14_7_atomic_check(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_framebuffer *fb; u32 width, height; - if (dcp->crashed || !v14 || READ_ONCE(v14->failed)) + if (dcp->external) { + /* An external pipe can always be switched off, started or not. */ + if (!crtc_state || !crtc_state->active) + return 0; + if (dcp->crashed || !v14 || READ_ONCE(v14->failed) || !v14->opened) + return -EIO; + } else if (dcp->crashed || !v14 || READ_ONCE(v14->failed)) { return -EIO; + } if (!crtc_state || !crtc_state->active || !p || p->crtc != crtc || !p->visible) return 0; @@ -1764,10 +1916,12 @@ int iomfb_v14_7_atomic_check(struct apple_dcp *dcp, struct drm_crtc *crtc, if (fb->modifier != DRM_FORMAT_MOD_LINEAR || fb->offsets[0] || fb->pitches[0] < width * 4 || (fb->pitches[0] & 63)) return -EINVAL; + /* An external pipe may show the top-left of a larger framebuffer. */ if (p->src_x || p->src_y || p->crtc_x || p->crtc_y || p->src_w != width << 16 || p->src_h != height << 16 || p->crtc_w != width || p->crtc_h != height || - fb->width != width || fb->height != height) + (dcp->external ? fb->width < width || fb->height < height : + fb->width != width || fb->height != height)) return -EINVAL; obj = drm_fb_dma_get_gem_obj(fb, 0); if (!obj || !obj->dma_addr || (u64)fb->pitches[0] * height > obj->base.size) @@ -1775,8 +1929,70 @@ int iomfb_v14_7_atomic_check(struct apple_dcp *dcp, struct drm_crtc *crtc, return 0; } +/* A411: the timing and color mode of the attached display, by their ids. */ +static int dcp_v14_set_mode(struct apple_dcp_v14 *v14, u32 color, u32 timing) +{ + __le32 in[2] = { cpu_to_le32(color), cpu_to_le32(timing) }, status = 0; + int ret; + + ret = dcp_v14_call(v14, A(411), in, sizeof(in), &status, sizeof(status), 0); + return ret ?: (le32_to_cpu(status) ? -EIO : 0); +} + +/* A472: the display's power state; its status sits at byte 4 of the reply. */ +static int dcp_v14_set_power(struct apple_dcp_v14 *v14, bool on) +{ + u8 in[12] = {}, out[8] = {}; + int ret; + + put_unaligned_le64(on, in); + ret = dcp_v14_call(v14, A(472), in, sizeof(in), out, sizeof(out), 0); + if (!ret && get_unaligned_le32(out + 4)) + ret = -EIO; + /* A refused power-down leaves nothing on: the display is gone already. */ + if (!ret || (!on && !READ_ONCE(v14->failed))) + v14->powered = on; + return ret; +} + +/* An external display: power down, set the new mode, power up. */ +static int dcp_v14_external_modeset(struct apple_dcp *dcp, struct drm_crtc_state *crtc_state) +{ + struct apple_dcp_v14 *v14 = dcp->v14; + struct dcp_display_mode mode; + int ret = 0; + + if (!v14 || READ_ONCE(v14->failed) || !v14->opened) + return -EIO; + if (!lookup_mode(dcp, &crtc_state->mode, &mode)) + return -EINVAL; + mutex_lock(&v14->lock); + if (v14->powered && dcp_v14_set_power(v14, false)) + dev_warn(dcp->dev, "display did not power down before the mode change\n"); + if (READ_ONCE(v14->failed)) + ret = -EIO; + if (!ret) + ret = dcp_v14_set_mode(v14, mode.color_mode_id, mode.timing_mode_id); + if (!ret) + ret = dcp_v14_set_power(v14, true); + if (!ret) { + v14->panel_width = mode.mode.hdisplay; + v14->panel_height = mode.mode.vdisplay; + } + mutex_unlock(&v14->lock); + dev_info(dcp->dev, "display mode " DRM_MODE_FMT " (timing %u, color %u): %d\n", + DRM_MODE_ARG(&crtc_state->mode), mode.timing_mode_id, mode.color_mode_id, ret); + if (ret) + return ret; + scoped_guard(spinlock_irqsave, &dcp->mode_state.lock) + dcp->mode_state.valid = true; + return 0; +} + int iomfb_v14_7_modeset(struct apple_dcp *dcp, struct drm_crtc_state *crtc_state) { + if (dcp->external) + return dcp_v14_external_modeset(dcp, crtc_state); /* The firmware keeps the timing it booted with, which is the only mode. */ if (!lookup_mode(dcp, &crtc_state->mode, NULL)) return -EINVAL; @@ -1794,8 +2010,18 @@ void iomfb_v14_7_flush(struct apple_dcp *dcp, struct drm_crtc *crtc, if (!p) p = crtc->primary->state; fb = p && p->visible ? p->fb : NULL; + /* No swaps while an external display is off, unset or unplugged. */ + if (dcp->external && (!dcp->v14 || !READ_ONCE(dcp->v14->powered) || + !READ_ONCE(dcp->mode_state.valid) || !dcp->connector || + !READ_ONCE(dcp->connector->connected))) { + dcp_drm_crtc_vblank(dcp->crtc); + return; + } dcp->swap_start = ktime_get(); - if (dcp->v14 && dcp_v14_present(dcp->v14, fb, dcp->notch_height)) + if (dcp->v14 && dcp_v14_present(dcp->v14, fb, + fb ? (dcp->external ? p->src_w >> 16 : fb->width) : 0, + fb ? (dcp->external ? p->src_h >> 16 : fb->height) : 0, + dcp->notch_height)) dcp_drm_crtc_page_flip(dcp, ktime_get()); else dcp_v14_cancel_event(dcp); @@ -1808,7 +2034,20 @@ void iomfb_v14_7_poweron(struct apple_dcp *dcp) void iomfb_v14_7_poweroff(struct apple_dcp *dcp) { + struct apple_dcp_v14 *v14 = dcp->v14; + + /* An external display is powered down; the next enable sets its mode again. */ + if (dcp->external) { + if (v14 && v14->opened && !READ_ONCE(v14->failed)) { + mutex_lock(&v14->lock); + if (v14->powered && dcp_v14_set_power(v14, false)) + dev_warn(dcp->dev, "display did not power down\n"); + mutex_unlock(&v14->lock); + } + dcp_mode_invalidate(&dcp->mode_state); + return; + } /* Blank to black; the panel and the DCP stay powered. */ - if (dcp->v14 && dcp->v14->started) - dcp_v14_present(dcp->v14, NULL, 0); + if (v14 && v14->started) + dcp_v14_present(v14, NULL, 0, 0, 0); } From 560ec648de78fea1c9ca5ad7565da64ba9d7262e Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 09:59:19 -0700 Subject: [PATCH 1605/2680] drm/apple: t6030: start an external display processor on first use MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A T6030 external display processor stayed out of the main DRM device and started only on an explicit sysfs request, so a USB-C route on it never connected a display and every external display needed a separate, manually driven device. Make the processor a pipe of the main DRM device, like an M1/M2 dcpext, when the display gate has given it a display DART: a piodma child on that DART's stream 4, and the DART's stream 0 among the display subsystem's iommus, so that framebuffers are mapped for this pipe as for the panel. Without that the manual path stays as it was. - Probe checks the handoff (firmware image, memory mappings) and maps the coprocessor registers once. A failed check leaves the pipe in the DRM device, never started, so the internal panel is not held up; Type-C routes are registered either way. - The firmware starts when a display first needs it: the first DP alt-mode connect, a Thunderbolt DP tunnel asking for the pipe, or a fixed output's HPD. A start work powers the CPU after the PMP has acknowledged the request, wakes RTKit and brings up DCPLink, the DPTX port service and the DP AV service. Connects meanwhile return -EAGAIN and are retried once the processor runs or announces its DPTX port. - After the first DPTX connect has asserted HPD, the display interface is opened (start signal, first client open). From then on the firmware describes each attached display. - System sleep is refused only while the processor starts or drives a display. Before the first start, and idle with no display, it does not stop sleep: its CPU domain stays at the gate's active floor, and the next attach reconnects. - An external session outlives component unbind, as it serves its ports. [ryan: adapted to 7.2.10 — apple_dcp_in_drm() keeps the availability and T8140 dcpext checks and admits a native T6030 dcpext; the DPTX-port retry kick sits in our common dptxport_init() publish; vblank/swap-watchdog works are already initialised at probe here, so a native pipe only disables them until bind; a failed display-interface open withdraws through our out_withdraw; dcp_external_ready() queues our Type-C reconnect work under hpd_mutex (this tree has no dcp_queue_typec_reconnect() session); the manual external probe/remove keep our typec_reconnect_wq enable/disable] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit d29b3ae21e440db2c1bbbf112d76727f35ffebd5) --- drivers/gpu/drm/apple/apple_drv.c | 8 +- drivers/gpu/drm/apple/dcp-fabric.c | 4 + drivers/gpu/drm/apple/dcp-internal.h | 10 ++ drivers/gpu/drm/apple/dcp.c | 159 +++++++++++++++-- drivers/gpu/drm/apple/dcp.h | 2 + drivers/gpu/drm/apple/dptxep.c | 3 + drivers/gpu/drm/apple/iomfb_v14_7.c | 256 ++++++++++++++++++++++++--- drivers/gpu/drm/apple/iomfb_v14_7.h | 10 ++ 8 files changed, 405 insertions(+), 47 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index e1b46c72b76e4..47d1648c25f83 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -702,13 +702,15 @@ static const struct of_device_id apple_dcp_id_tbl[] = { /* * A supported firmware-described external provider participates in component - * matching. The T6030 external processor runs a separate session and never - * joins this DRM device. + * matching. A T6030 external display processor is a pipe of this DRM device + * once the display gate gave it its display DART; on the manual diagnostic + * path it runs a separate session and never joins this DRM device. */ static bool apple_dcp_usable(struct device_node *np) { return of_device_is_available(np) && - !of_device_is_compatible(np, "apple,t6030-dcpext") && + (!of_device_is_compatible(np, "apple,t6030-dcpext") || + dcp_t6030_ext_native(np)) && (!of_device_is_compatible(np, "apple,t8140-dcpext") || dcp_h17p_external_supported(np)); } diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 899acc15646c5..0314e6b5b84ce 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -1539,6 +1539,10 @@ static bool dcp_tb_services_ready(struct apple_dcp *dcp) { if (!dcp->external || dcp->fw_compat != DCP_FIRMWARE_V_14_7) return true; + /* A native processor starts for the tunnel; the tunnel retries meanwhile. */ + if (dcp->external_native) + return iomfb_v14_7_external_ready(dcp); + /* Pairs with DPTX RemotePort publication before tunnel acquisition. */ return smp_load_acquire(&dcp->dptxport[0].enabled) && ibootep_is_ready(dcp); } diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index dec6da7c74706..4c2a7a23538c4 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -251,10 +251,20 @@ struct apple_dcp { /* firmware version and compatible firmware version */ enum dcp_firmware_version fw_compat; bool external; + /* + * A T6030 external processor in the main DRM device: the kernel starts + * its firmware on first use, its modes come from the attached display. + * Clear for the manual diagnostic path (separate scanout, explicit start). + */ + bool external_native; struct dcpext_scanout *dcpext_scanout; bool external_link_ready; /* Full connect/HPD handshake completed. */ bool external_suspended; /* hpd_mutex serializes PM against explicit start */ struct work_struct external_work; + /* Native: retries pending connects once DPTX ports are announced. */ + struct work_struct external_ready_work; + /* Native: the firmware's start state, see iomfb_v14_7.c. */ + unsigned int external_phase; atomic_t external_requested; /* DCP_FIRMWARE_V_14_7 state; outlives this device once RTKit runs. */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 2e9d1873bc7f8..cc127a77c38cf 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -957,6 +957,9 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable dev_warn(dcp->dev, "dcp_dptx_connect: missing phy\n"); return -ENODEV; } + /* A native external processor starts here; its start retries this. */ + if (dcp->external_native && !iomfb_v14_7_external_ready(dcp)) + return -EAGAIN; /* @port selects the upstream RemotePort service/core. dptx_dfp_port * is the downstream address: dpphy=0, dpin0=1, dpin1=2. */ @@ -1035,6 +1038,15 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable } mutex_unlock(&dcp->hpd_mutex); + /* + * The display interface of a native external processor opens once a + * display is attached; the firmware then describes that display. + */ + if (dcp->external_native) { + ret = iomfb_v14_7_external_open(dcp); + if (ret) + goto out_withdraw; + } timeout = dcp_uses_t6020_tunnel_flow(dcp) ? DPTX_TUNNEL_CONNECT_TIMEOUT : DPTX_CONNECT_TIMEOUT; ret = wait_for_completion_timeout(&dcp->dptxport[port].linkcfg_completion, @@ -1192,6 +1204,65 @@ static int dcp_external_start_locked(struct apple_dcp *dcp) return 0; } +/* + * A T6030 external processor runs as a pipe of the main DRM device when the + * display gate handed it its display DART: a piodma child on stream 4 of + * that DART, and stream 0 among the display subsystem's iommus, so that + * framebuffers are mapped for it as for the panel. + */ +bool dcp_t6030_ext_native(const struct device_node *np) +{ + struct of_phandle_args dart, args; + struct device_node *piodma, *display; + bool native = false; + u32 marker; + int i, n; + + if (!of_device_is_compatible(np, "apple,t6030-dcpext")) + return false; + piodma = of_get_child_by_name(np, "piodma"); + if (!piodma || !of_device_is_available(piodma) || + of_property_read_u32(piodma, "apple,t6030-dispext-handoff", &marker) || + marker != 1 || + of_parse_phandle_with_args(piodma, "iommus", "#iommu-cells", 0, &dart)) { + of_node_put(piodma); + return false; + } + of_node_put(piodma); + if (dart.args_count != 1 || dart.args[0] != 4 || !of_device_is_available(dart.np)) + goto out; + display = of_find_compatible_node(NULL, NULL, "apple,t6030-display-subsystem"); + n = display && of_device_is_available(display) ? + of_count_phandle_with_args(display, "iommus", "#iommu-cells") : 0; + for (i = 0; i < n && !native; i++) { + if (of_parse_phandle_with_args(display, "iommus", "#iommu-cells", i, &args)) + break; + native = args.np == dart.np && args.args_count == 1 && !args.args[0]; + of_node_put(args.np); + } + of_node_put(display); +out: + of_node_put(dart.np); + return native; +} + +/* + * A native external processor is running, or has announced a DPTX port: + * connect the display that is already waiting for it. + */ +void dcp_external_ready(struct apple_dcp *dcp) +{ + if (READ_ONCE(dcp->typec_cable_connected)) { + guard(mutex)(&dcp->hpd_mutex); + dcp->typec_reconnect_tries = 0; + if (dcp->typec_cable_connected) + mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); + } else if (dcp->hdmi_hpd && dcp->active && + gpiod_get_value_cansleep(dcp->hdmi_hpd)) { + dcp_dptx_connect(dcp, 0); + } +} + int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) { if (!dcp_is_external_h17p(dcp)) @@ -1681,6 +1752,9 @@ int dcp_start(struct platform_device *pdev) */ if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { if (dcp->external) { + /* A native pipe is usable now; its firmware starts on first use. */ + if (dcp->external_native) + dcp->active = true; complete(&dcp->start_done); return 0; } @@ -2682,9 +2756,13 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (ret) return ret; - dcp->coproc_reg = devm_platform_ioremap_resource_byname(to_platform_device(dev), "coproc"); - if (IS_ERR(dcp->coproc_reg)) - return PTR_ERR(dcp->coproc_reg); + /* A native external processor mapped these at probe, for its start work. */ + if (!dcp->external_native) { + dcp->coproc_reg = devm_platform_ioremap_resource_byname(to_platform_device(dev), + "coproc"); + if (IS_ERR(dcp->coproc_reg)) + return PTR_ERR(dcp->coproc_reg); + } /* * Display coprocessors running H17-generation firmware are left running @@ -2749,6 +2827,13 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) /* The running T6030 firmware is adopted as is. */ if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { + if (dcp->external_native) { + enable_work(&dcp->vblank_wq); + enable_delayed_work(&dcp->swap_watchdog_wq); + enable_work(&dcp->dimensions_wq); + dcp_enable_typec_work(dcp); + return 0; + } if (dcp->external) return 0; ret = iomfb_v14_7_bind(dcp); @@ -2878,8 +2963,13 @@ VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, v disable_work_sync(&dcp->dimensions_wq); if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { - if (!dcp->external) + if (dcp->external_native) { + dcp_disable_typec_work(dcp, true); + disable_delayed_work_sync(&dcp->swap_watchdog_wq); + disable_work_sync(&dcp->vblank_wq); + } else if (!dcp->external) { dcp_disable_typec_work(dcp, false); + } iomfb_v14_7_unbind(dcp); return; } @@ -3058,6 +3148,7 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp->fw_compat = fw_compat; dcp->external = of_device_is_compatible(dev->of_node, "apple,t6030-dcpext"); + dcp->external_native = dcp->external && dcp_t6030_ext_native(dev->of_node); dcp->dev = dev; /* * Type-C and Thunderbolt routes can be activated as soon as they are @@ -3120,6 +3211,11 @@ static int dcp_platform_probe(struct platform_device *pdev) disable_delayed_work(&dcp->placeholder_edid_wq); disable_delayed_work(&dcp->announce_wq); dcp->announce.stopped = true; + if (dcp->external_native) { + /* Bind enables these; firmware callbacks may come before it. */ + disable_work(&dcp->vblank_wq); + disable_delayed_work(&dcp->swap_watchdog_wq); + } platform_set_drvdata(pdev, dcp); @@ -3270,12 +3366,14 @@ static int dcp_platform_probe(struct platform_device *pdev) return ret; /* - * The T6030 external processor is not part of the panel's DRM device. - * Joining that component set would hold the internal screen until - * dcpext binds. Its tunnel bring-up runs the Type-C reconnect work, - * which component bind would otherwise enable. + * The manual T6030 external path is not part of the panel's DRM + * device. Joining that component set would hold the internal screen + * until dcpext binds. Its tunnel bring-up runs the Type-C reconnect + * work, which component bind would otherwise enable. A native external + * processor joins it, and probe completes for it even when its handoff + * is unusable, so that the panel is never held up. */ - if (dcp->external) { + if (dcp->external && !dcp->external_native) { enable_delayed_work(&dcp->typec_reconnect_wq); return iomfb_v14_7_external_start(dcp); } @@ -3290,6 +3388,11 @@ static int dcp_platform_probe(struct platform_device *pdev) } WRITE_ONCE(dcp->external_prepared, true); } + if (dcp->external_native) { + ret = iomfb_v14_7_external_prepare(dcp); + if (ret) + return ret; + } ret = component_add(&pdev->dev, &dcp_comp_ops); if (ret && dcp->external_prepared) { @@ -3305,7 +3408,7 @@ static void dcp_platform_remove(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); - if (dcp && dcp->external) { + if (dcp && dcp->external && !dcp->external_native) { /* Enabled for the external start at probe. */ disable_delayed_work_sync(&dcp->typec_reconnect_wq); iomfb_v14_7_remove(dcp); @@ -3321,7 +3424,7 @@ VISIBLE_IF_KUNIT void dcp_platform_shutdown(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); - if (dcp && dcp->external) + if (dcp && dcp->external && !dcp->external_native) return; if (dcp && dcp->retain_dma && dcp->hw.iomfb_method_profile == @@ -3431,6 +3534,22 @@ static void dcp_release_after_sleep(struct apple_dcp *dcp) iomfb_pause_presentation(dcp, false); } +/* + * Whether an external processor must keep the system awake. The manual path + * refuses sleep once its firmware was ever started. A native processor + * refuses only while it starts or drives a display: idle, its firmware stays + * powered across s2idle like the panel's (the gate holds its CPU domain at + * the active floor) and the next attach reconnects through it. + */ +static bool dcp_external_sleep_vetoed(struct apple_dcp *dcp) +{ + if (dcp->external_native) + return iomfb_v14_7_external_busy(dcp); + return atomic_read(&dcp->external_requested) || + (dcp->rtk && apple_rtkit_is_running(dcp->rtk)) || + dcpext_scanout_requested(dcp); +} + /* dpm_prepare completes for every device before any dpm_suspend callback. * Veto early so Thunderbolt cannot tear down a working tunnel first. The * same HPD lock serializes this gate against explicit firmware startup. @@ -3442,11 +3561,12 @@ static int dcp_platform_prepare(struct device *dev) if (!dcp->external) return 0; mutex_lock(&dcp->hpd_mutex); - if (atomic_read(&dcp->external_requested) || - (dcp->rtk && apple_rtkit_is_running(dcp->rtk)) || - dcpext_scanout_requested(dcp)) { + if (dcp_external_sleep_vetoed(dcp)) { mutex_unlock(&dcp->hpd_mutex); - dev_warn(dev, "external firmware/scanout attempted: refusing PM prepare; reboot required for retained DMA\n"); + if (dcp->external_native) + dev_warn(dev, "external display attached or starting: refusing system sleep\n"); + else + dev_warn(dev, "external firmware/scanout attempted: refusing PM prepare; reboot required for retained DMA\n"); return -EBUSY; } WRITE_ONCE(dcp->external_suspended, true); @@ -3475,11 +3595,12 @@ static int dcp_platform_suspend(struct device *dev) */ if (dcp->external) { mutex_lock(&dcp->hpd_mutex); - if (atomic_read(&dcp->external_requested) || - (dcp->rtk && apple_rtkit_is_running(dcp->rtk)) || - dcpext_scanout_requested(dcp)) { + if (dcp_external_sleep_vetoed(dcp)) { mutex_unlock(&dcp->hpd_mutex); - dev_warn(dev, "external firmware/scanout attempted: suspend refused; retained DMA requires reboot\n"); + if (dcp->external_native) + dev_warn(dev, "external display attached or starting: suspend refused\n"); + else + dev_warn(dev, "external firmware/scanout attempted: suspend refused; retained DMA requires reboot\n"); return -EBUSY; } WRITE_ONCE(dcp->external_suspended, true); diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index ecfb32beec35d..f1675bd35ef9d 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -68,6 +68,8 @@ void dcp_update_connector_dimensions(struct apple_dcp *dcp, void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message); int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port); +bool dcp_t6030_ext_native(const struct device_node *np); +void dcp_external_ready(struct apple_dcp *dcp); int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); bool dcp_hold_resume_event(struct platform_device *pdev); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 967e065d6d4ce..073d6332334cb 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -865,6 +865,9 @@ static void dptxport_init(struct apple_epic_service *service, const char *name, dptx_service_publish(service); complete(&port->enable_completion); + /* A connect that found no port yet is retried from process context. */ + if (service->ep->dcp->external_native) + schedule_work(&service->ep->dcp->external_ready_work); /* Publication may follow the startup wait or the bounded retry window. */ if (service->ep->dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 491ac97e59f33..b62099b8ca1c4 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -349,6 +349,11 @@ static int dcp_v14_map_piodma(struct apple_dcp_v14 *v14, u32 id) return 0; } +/* An external processor's default stride; no framebuffer is allocated for it. */ +#define DCP_V14_EXT_STRIDE (1920 * 4) +/* The display clock of every T6030 external pipe, unless the DT names one. */ +#define DCP_V14_EXT_CLOCK 935000000ULL + /* Removes a raw property of service 0. */ static void dcp_v14_raw_remove(struct apple_dcp_v14 *v14, const char *key) { @@ -1220,6 +1225,14 @@ static int dcp_v14_geometry(struct apple_dcp_v14 *v14) return 0; } +/* An external processor's firmware: once started, it runs until reboot. */ +enum { + DCPEXT_IDLE, + DCPEXT_STARTING, + DCPEXT_RUNNING, + DCPEXT_FAILED, +}; + static void dcpext_bringup(struct work_struct *work); /* @@ -1356,8 +1369,11 @@ static void dcpext_cancel(void *data) struct apple_dcp *dcp = data; cancel_work_sync(&dcp->external_work); + if (dcp->external_native) + cancel_work_sync(&dcp->external_ready_work); } +/* The manual diagnostic path: memory checks, then an explicit start. */ int iomfb_v14_7_external_start(struct apple_dcp *dcp) { int ret; @@ -1382,6 +1398,96 @@ int iomfb_v14_7_external_start(struct apple_dcp *dcp) return ret; } +static void dcpext_ready_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(work, struct apple_dcp, external_ready_work); + + dcp_external_ready(dcp); +} + +/* + * The native path, at probe: the same checks as a manual start, without a + * start. The coprocessor registers are mapped here, once, for the start + * work and component bind alike. A failed check leaves the pipe in the DRM + * device, never started, so that the internal panel is not held up. + */ +int iomfb_v14_7_external_prepare(struct apple_dcp *dcp) +{ + struct device *dev = dcp->dev; + const char *uuid = NULL; + int ret; + + atomic_set(&dcp->external_requested, 0); + WRITE_ONCE(dcp->external_phase, DCPEXT_IDLE); + INIT_WORK(&dcp->external_work, dcpext_bringup); + INIT_WORK(&dcp->external_ready_work, dcpext_ready_work); + ret = devm_add_action_or_reset(dev, dcpext_cancel, dcp); + if (ret) + return ret; + + dcp->coproc_reg = devm_platform_ioremap_resource_byname(to_platform_device(dev), "coproc"); + if (IS_ERR(dcp->coproc_reg)) { + ret = PTR_ERR(dcp->coproc_reg); + dcp->coproc_reg = NULL; + goto refuse; + } + /* The callbacks here are those of one firmware image. */ + if (of_property_read_string(dev->of_node, "apple,firmware-uuid", &uuid) || + strcmp(uuid, DCP_V14_FIRMWARE_UUID)) { + dev_err(dev, "external display processor firmware %s is not supported\n", + uuid ?: "(unknown)"); + ret = -ENODEV; + goto refuse; + } + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); + if (!ret) + ret = dcpext_verify_memory(dcp); + if (ret) + goto refuse; + dev_info(dev, "external display processor ready; its firmware starts when a display is attached\n"); + return 0; +refuse: + WRITE_ONCE(dcp->external_phase, DCPEXT_FAILED); + dev_err(dev, "external display processor unusable: handoff check failed: %d\n", ret); + return 0; +} + +/* True once DPTX can be used; asks for the start if nothing has. */ +bool iomfb_v14_7_external_ready(struct apple_dcp *dcp) +{ + switch (smp_load_acquire(&dcp->external_phase)) { + case DCPEXT_RUNNING: + return smp_load_acquire(&dcp->dptxport[0].enabled); + case DCPEXT_IDLE: + /* Not between sleep's prepare and complete; the next attach starts it. */ + if (READ_ONCE(dcp->external_suspended)) + return false; + if (!atomic_cmpxchg(&dcp->external_requested, 0, 1)) { + WRITE_ONCE(dcp->external_phase, DCPEXT_STARTING); + dev_info(dcp->dev, "display attached: starting the external display processor\n"); + /* Firmware callbacks may outlive any later module removal. */ + __module_get(THIS_MODULE); + queue_work(system_unbound_wq, &dcp->external_work); + } + return false; + default: + return false; + } +} + +/* + * System sleep with this processor: allowed before its first start and + * while it idles with no display; refused while it starts or drives one, + * as the link and the display's buffers are not handed back across sleep. + */ +bool iomfb_v14_7_external_busy(struct apple_dcp *dcp) +{ + if (smp_load_acquire(&dcp->external_phase) == DCPEXT_STARTING) + return true; + return READ_ONCE(dcp->active_typec_route) || READ_ONCE(dcp->dptxport[0].connected) || + READ_ONCE(dcp->external_link_ready) || (dcp->v14 && READ_ONCE(dcp->v14->powered)); +} + /* Refuse startup unless both the hardware floor and PMP vote are in place. */ static int dcpext_check_power(struct apple_dcp *dcp) { @@ -1424,9 +1530,13 @@ static int dcpext_check_power(struct apple_dcp *dcp) return ret; } -static void dcpext_bringup(struct work_struct *work) +/* + * Starts the processor's CPU and RTKit, then its endpoints. The native path + * also brings up DCPLink, for the display interface opened at the first + * attach, and the DP AV service the EDID comes from. + */ +static int dcpext_boot(struct apple_dcp *dcp) { - struct apple_dcp *dcp = container_of(work, struct apple_dcp, external_work); struct apple_dcp_v14 *v14; struct device *dev = dcp->dev; struct resource *res; @@ -1437,35 +1547,49 @@ static void dcpext_bringup(struct work_struct *work) ret = dcpext_check_power(dcp); if (ret) { dev_err(dev, "dcpext startup refused: power prerequisites failed: %d\n", ret); - return; + return ret; } dev_info(dev, "dcpext CPU power floor active and PMP request acknowledged\n"); - res = platform_get_resource_byname(to_platform_device(dev), - IORESOURCE_MEM, "coproc"); - if (!res) { - dev_err(dev, "dcpext has no coproc register\n"); - return; - } - dcp->coproc_reg = devm_ioremap_resource(dev, res); - if (IS_ERR(dcp->coproc_reg)) { - dev_err(dev, "dcpext coproc map failed: %ld\n", - PTR_ERR(dcp->coproc_reg)); - dcp->coproc_reg = NULL; - return; + if (!dcp->coproc_reg) { + res = platform_get_resource_byname(to_platform_device(dev), + IORESOURCE_MEM, "coproc"); + if (!res) { + dev_err(dev, "dcpext has no coproc register\n"); + return -ENODEV; + } + dcp->coproc_reg = devm_ioremap_resource(dev, res); + if (IS_ERR(dcp->coproc_reg)) { + ret = PTR_ERR(dcp->coproc_reg); + dev_err(dev, "dcpext coproc map failed: %d\n", ret); + dcp->coproc_reg = NULL; + return ret; + } } control = readl(dcp->coproc_reg + DCP_V14_CPU_CONTROL); dev_info(dev, "dcpext CPU control %#x\n", control); ret = dcpext_verify_memory(dcp); if (ret) - return; + return ret; v14 = kzalloc_obj(*v14); if (!v14) - return; + return -ENOMEM; v14->dev = dev; v14->dcp = dcp; + v14->external = dcp->external_native; + if (v14->external) { + struct clk *clk = clk_get_optional(dev, NULL); + + if (!IS_ERR_OR_NULL(clk)) { + v14->clock_rate = clk_get_rate(clk); + clk_put(clk); + } + if (!v14->clock_rate) + v14->clock_rate = DCP_V14_EXT_CLOCK; + v14->stride = DCP_V14_EXT_STRIDE; + } mutex_init(&v14->lock); INIT_WORK(&v14->idle_work, dcp_v14_idle); dcp->v14 = v14; @@ -1474,7 +1598,7 @@ static void dcpext_bringup(struct work_struct *work) rtk = apple_rtkit_init(dev, v14, "mbox", 0, &dcp_v14_rtkit_ops); if (IS_ERR(rtk)) { dev_err(dev, "dcpext RTKit init failed: %ld\n", PTR_ERR(rtk)); - return; + return PTR_ERR(rtk); } v14->rtk = rtk; v14->link.rtk = rtk; @@ -1483,27 +1607,105 @@ static void dcpext_bringup(struct work_struct *work) if (!(control & APPLE_DCP_COPROC_CPU_CONTROL_RUN)) { writel(control | APPLE_DCP_COPROC_CPU_CONTROL_RUN, dcp->coproc_reg + DCP_V14_CPU_CONTROL); - dev_info(dev, "dcpext CPU started by explicit request\n"); + dev_info(dev, "dcpext CPU started\n"); } ret = apple_rtkit_wake(rtk); for (n = 0; ret == -ETIME && n < DCP_V14_RTKIT_RETRIES; n++) ret = apple_rtkit_boot(rtk); if (ret) { dev_err(dev, "dcpext RTKit did not wake: %d\n", ret); - return; + return ret; } - dev_info(dev, "dcpext RTKit session running\n"); + + if (v14->external) { + ret = dcp_v14_link_start(&v14->link); + if (ret) { + v14->failed = true; + dev_err(dev, "dcpext DCPLink did not start: %d\n", ret); + return ret; + } + /* Notifications that arrive with no call in progress. */ + dcp_v14_link_set_idle_work(&v14->link, &v14->idle_work); + } if (apple_rtkit_has_endpoint(rtk, DPAV_CTRL_ENDPOINT)) dpav_ctrl_init(dcp); - if (apple_rtkit_has_endpoint(rtk, DPTX_ENDPOINT)) - dptxep_init(dcp); + if (apple_rtkit_has_endpoint(rtk, DPTX_ENDPOINT)) { + ret = dptxep_init(dcp); + if (ret) + dev_err(dev, "dcpext DPTX endpoint failed: %d\n", ret); + } + if (v14->external) { + /* The service the attached display's EDID is read from. */ + if (apple_rtkit_has_endpoint(rtk, DPAVSERV_ENDPOINT)) { + ret = dpavservep_init(dcp); + if (ret) + dev_info(dev, "display EDID service not available: %d\n", ret); + } + return 0; + } /* External mode discovery only; this does not power or modeset a display. */ if (apple_rtkit_has_endpoint(rtk, DISP0_ENDPOINT)) { ret = ibootep_init(dcp); if (ret) dev_err(dev, "dcpext mode-query endpoint failed: %d\n", ret); } + return 0; +} + +static void dcpext_bringup(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(work, struct apple_dcp, external_work); + int ret = dcpext_boot(dcp); + + if (!dcp->external_native) + return; + /* Pairs with external_ready(): the endpoints are published first. */ + smp_store_release(&dcp->external_phase, ret ? DCPEXT_FAILED : DCPEXT_RUNNING); + if (ret) { + dev_err(dcp->dev, "external display processor did not start: %d; no retry until reboot\n", + ret); + return; + } + dev_info(dcp->dev, "external display processor running\n"); + dcp_external_ready(dcp); +} + +/* + * Opens the display interface (start signal, first client open) on a + * running external processor, once, after its first display is attached. + * The firmware then describes that display and every later one. + */ +int iomfb_v14_7_external_open(struct apple_dcp *dcp) +{ + struct apple_dcp_v14 *v14 = dcp->v14; + int ret; + + if (smp_load_acquire(&dcp->external_phase) != DCPEXT_RUNNING || !v14) + return -ENODEV; + mutex_lock(&v14->lock); + if (v14->opened) { + mutex_unlock(&v14->lock); + return 0; + } + ret = dcp_v14_simple_call(v14, A(401), false, true, true); + if (!ret) + ret = dcp_v14_simple_call(v14, A(455), false, false, false); + if (!ret) + v14->opened = true; + else + v14->failed = true; + mutex_unlock(&v14->lock); + if (ret) { + dev_err(dcp->dev, "external display interface did not open: %d; no retry until reboot\n", + ret); + return ret; + } + dev_info(dcp->dev, "external display interface open\n"); + /* A display described before this could not be enabled until now. */ + if (dcp->connector && READ_ONCE(dcp->connector->connected)) + schedule_work(&dcp->connector->hotplug_wq); + return 0; } int iomfb_v14_7_bind(struct apple_dcp *dcp) @@ -1575,7 +1777,9 @@ void iomfb_v14_7_unbind(struct apple_dcp *dcp) dcp_mode_invalidate(&dcp->mode_state); if (!v14) return; - WRITE_ONCE(v14->dcp, NULL); + /* An external session keeps serving its ports; removal detaches it. */ + if (!v14->external) + WRITE_ONCE(v14->dcp, NULL); if (v14->rtk) dev_info(dcp->dev, "display unbound; the DCP session and its buffers are kept until reboot\n"); } @@ -1598,9 +1802,11 @@ void iomfb_v14_7_remove(struct apple_dcp *dcp) { struct apple_dcp_v14 *v14; - /* Bring-up creates the external session; a late dcpext_start must not. */ + /* Bring-up creates the external session; a late start must not. */ if (dcp->external) disable_work_sync(&dcp->external_work); + if (dcp->external_native) + disable_work_sync(&dcp->external_ready_work); v14 = dcp->v14; if (!v14) return; diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.h b/drivers/gpu/drm/apple/iomfb_v14_7.h index c035256269647..50a7d1775389b 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7.h @@ -25,6 +25,16 @@ int iomfb_v14_7_probe(struct apple_dcp *dcp); /* Component bind: the running DCP's RTKit session. */ int iomfb_v14_7_bind(struct apple_dcp *dcp); int iomfb_v14_7_external_start(struct apple_dcp *dcp); +/* + * Native external processor: probe-time checks, then a firmware start on + * first use. _ready() is true once DPTX can be used, and requests the start + * if nothing has yet; _open() opens the display interface after the first + * display is attached; _busy() vetoes system sleep. + */ +int iomfb_v14_7_external_prepare(struct apple_dcp *dcp); +bool iomfb_v14_7_external_ready(struct apple_dcp *dcp); +int iomfb_v14_7_external_open(struct apple_dcp *dcp); +bool iomfb_v14_7_external_busy(struct apple_dcp *dcp); /* Component unbind: KMS goes away; the firmware session is kept until reboot. */ void iomfb_v14_7_unbind(struct apple_dcp *dcp); /* Platform remove: the apple_dcp is freed next; the firmware session is kept. */ From 7e191d63e28fc7395c1c821574e9c26ecdb63f03 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 09:59:20 -0700 Subject: [PATCH 1606/2680] drm/apple: skip T8122-generation ATC PHY graph peers as components The DRM device's component match skips a DCP's graph link to an apple,t8103-atcphy, as the PHY driver is not a display component. T6030 and T8122 ATC PHYs are apple,t8122-atcphy instead. Now that a T6030 external processor is part of the DRM device, a graph link from it to such a PHY (the HDMI port's) would add a component that never binds, and hold up the whole device, internal panel included. Skip apple,t8122-atcphy peers the same way. M1/M2 device trees have no such nodes. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit fd483d620a640f9cc74a62cc77ff8d1662f211c0) --- drivers/gpu/drm/apple/apple_drv.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 47d1648c25f83..7d103ed0927d8 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -965,7 +965,8 @@ static int add_dcp_components(struct device *dev, * collection for the Apple display-subsystem so * ignore it here. */ - if (of_device_is_compatible(port, "apple,t8103-atcphy")) { + if (of_device_is_compatible(port, "apple,t8103-atcphy") || + of_device_is_compatible(port, "apple,t8122-atcphy")) { of_node_put(port); continue; } From 5c686a0b05c2293ee131b0fa41ddf0df1856d773 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 09:59:37 -0700 Subject: [PATCH 1607/2680] drm/apple: t6030: keep a live fixed output on its external pipe A pipeline's live fixed output keeps the Type-C routes off that pipeline, unless the DCP runs the 14.7 firmware: there the panel is driven on the IOMFB path, separately from DPTX, so the T6030 panel DCP never blocks a route. That exception also covered external processors, whose fixed output is a display behind the HDMI port, driven over DPTX as on M1/M2. Limit the exception to the panel's DCP, so that a USB-C display takes the other, free external processor while HDMI is in use. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 92f7d0ffb7f6bb1790ec080c58e40e552918a519) --- drivers/gpu/drm/apple/dcp-fabric.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 0314e6b5b84ce..12078bd25a0d4 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -117,9 +117,10 @@ static bool dcp_typec_route_fixed_output_busy(struct apple_dcp_typec_route *rout /* * The 14.7 firmware drives the internal panel on the IOMFB path. * The DPTX service is a separate output, so the live panel does not - * occupy the Type-C route. + * occupy the Type-C route. An external processor's fixed output (a + * display behind the HDMI port) does, as on M1/M2. */ - if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) + if (dcp->fw_compat == DCP_FIRMWARE_V_14_7 && !dcp->external) return false; return dcp_fabric_fixed_busy(false, false, From bd877ae26fdc032d0584ac2e984f667cde3e2bb0 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 09:59:38 -0700 Subject: [PATCH 1608/2680] drm/apple: dpavservep: open the 14.7 AV service before reading EDID The 14.7 firmware of an external display processor answers the EDID copy request on its DP AV service only once the service has been opened with the standard open command (group 4, command 6). Open each announced service instance once, before its first EDID read, on 14.7 firmware only. Without an open service the display still gets its modes from the firmware's timing list; only the connector's EDID property is missing. Earlier firmware is unchanged. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 4b81c41ddd5a4f55d83cc24246465766c3e278c0) --- drivers/gpu/drm/apple/epic/dpavservep.c | 17 +++++++++++++++++ drivers/gpu/drm/apple/epic/dpavservep.h | 2 ++ 2 files changed, 19 insertions(+) diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 1a324f8fed277..4dc0f1000fe3f 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -168,6 +168,7 @@ static void dcpavserv_teardown(struct apple_epic_service *service) dcp->dcpavserv.enabled = false; dcp->dcpavserv.service = NULL; dcp->dcpavserv.epoch++; + dcp->dcpavserv.opened = NULL; dcpavserv_invalidate_edid(dcp); reinit_completion(&dcp->dcpavserv.enable_completion); owned = true; @@ -187,6 +188,7 @@ void dpavservep_detach(struct apple_dcp *dcp) dcp->dcpavserv.enabled = false; /* Also reject an in-flight prepare/OPEN with no published owner yet. */ dcp->dcpavserv.epoch++; + dcp->dcpavserv.opened = NULL; reinit_completion(&dcp->dcpavserv.enable_completion); if (service) { afk_service_retire(service); @@ -417,6 +419,21 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation if (!service) return ERR_PTR(-ENODEV); + /* + * The 14.7 firmware of an external processor answers EDID requests + * once its service has been opened; open each announced instance once. + */ + if (dcp->fw_compat == DCP_FIRMWARE_V_14_7 && + READ_ONCE(dcp->dcpavserv.opened) != service) { + int ret = afk_service_call(service, 4, 6, NULL, 0, 32, NULL, 0, 32); + + if (ret) { + afk_service_put(service); + return ERR_PTR(ret); + } + WRITE_ONCE(dcp->dcpavserv.opened, service); + } + edid = service->ep->wire_v2 ? dcpavserv_read_edid_inline(service) : dcpavserv_read_edid_legacy(service); mutex_lock(&dcp->dcpavserv.lock); diff --git a/drivers/gpu/drm/apple/epic/dpavservep.h b/drivers/gpu/drm/apple/epic/dpavservep.h index b824a59918ea3..001e211feeb6b 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.h +++ b/drivers/gpu/drm/apple/epic/dpavservep.h @@ -22,6 +22,8 @@ struct dcpavserv { struct completion enable_completion; u32 unit; struct apple_epic_service *service; + /* The service instance last opened; compared only, never used. */ + struct apple_epic_service *opened; }; bool dcpavserv_edid_current(struct apple_dcp *dcp, u64 generation); From 3f2b7aeccffd5c267c548638304fdd4201b49fcf Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:04:11 -0700 Subject: [PATCH 1609/2680] drm/apple: t6030: clock a direct crossbar output only while its link runs On T6030 the display crossbar routes a DP IN output on select and runs its clocks only between link up and link down, and the crossbar driver treats the DP PHY output the same way. Thunderbolt tunnels already bring their DP IN output up from DCP's link configuration calls; a direct route to a DP PHY (a USB-C port in DP alt mode, or a fixed output) did not. For a direct route on T6030, follow DCP's link configuration: - bring the selected crossbar output up at DidChangeLinkConfiguration once a nonzero link rate is set; - take it down at WillChangeLinkConfiguration, before SetLinkRate reprograms the PHY, and at Deactivate. The output is the borrowed Type-C route's, or the fixed route's. Track which output runs, so the calls stay balanced; selecting or releasing a route resets it, as the crossbar idles a deselected output. Other SoCs are unchanged: their crossbar clocks the output at select. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit d3cc3b66bd9364fb1ffad0911f3ab0f3b17840c7) --- drivers/gpu/drm/apple/dcp-fabric.c | 41 ++++++++++++++++++++++++++++ drivers/gpu/drm/apple/dcp-fabric.h | 1 + drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/dptxep.c | 8 ++++++ 4 files changed, 52 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 12078bd25a0d4..ea644e01861f2 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -287,6 +287,8 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, route->active_xbar = xbar; route->tunnel = xbar != route->xbar; route->xbar_up = !route->tunnel; + /* A newly selected output starts with its clocks off. */ + dcp->direct_xbar_up = NULL; dcp->dptx_tunnel = route->tunnel; /* crossbar controls are dpphy, dpin0, dpin1: same order as the DFP port */ dcp->dptx_dfp_port = route->tunnel ? xbar - &route->xbar->chip->mux[0] : 0; @@ -348,6 +350,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) } route->active_xbar = NULL; route->tunnel = false; + dcp->direct_xbar_up = NULL; dcp->dptx_tunnel = false; dcp->dptx_dfp_port = 0; dcp->tb_dpin_set_active = NULL; @@ -1464,6 +1467,44 @@ int dcp_tunnel_crossbar_down(struct apple_dcp *dcp) return dcp_dpxbar_link(route->active_xbar, false); } +/* + * T6030 routes a direct DP PHY output of its crossbar on select and clocks + * it only while a link runs, as it does its DP IN outputs. From DCP's link + * configuration calls: clocks up once a link is configured at a nonzero + * rate, down before the rate changes and when DCP deactivates the port. + * The output is the borrowed Type-C route's, or the fixed route's. Other + * SoCs clock it at select. Runs inside a DCP apcall: only tb_lock. + */ +int dcp_direct_crossbar_link(struct apple_dcp *dcp, bool up) +{ + struct apple_dcp_typec_route *route; + struct mux_control *xbar = NULL; + int ret; + + if (!of_machine_is_compatible("apple,t6030")) + return 0; + guard(mutex)(&dcp->tb_lock); + route = dcp->active_typec_route; + if (route && !route->tunnel && route->selected) + xbar = route->active_xbar ?: route->xbar; + else if (!route && dcp->fixed_route_selected) + xbar = dcp->xbar; + /* A deselected output went idle, and its clocks with it. */ + if (dcp->direct_xbar_up && dcp->direct_xbar_up != xbar) + dcp->direct_xbar_up = NULL; + if (!xbar || !!dcp->direct_xbar_up == up) + return 0; + ret = dcp_dpxbar_link(xbar, up); + if (ret) { + dev_warn(dcp->dev, "DP crossbar output %s failed: %d\n", up ? "up" : "down", ret); + /* Only a running output can be taken down again. */ + if (up) + return ret; + } + dcp->direct_xbar_up = up ? xbar : NULL; + return ret; +} + /* * Thunderbolt DP IN, from DCP's SetLinkRate: start (rate != 0) or stop the * tunnel pixel clock. A stopped clock also takes the crossbar connection down. diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index 44837410cc60e..3a3652d5a1940 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -73,6 +73,7 @@ void dcp_unlink(struct drm_device *drm); /* Thunderbolt DP tunnels, called from the DPTX endpoint */ int dcp_tunnel_crossbar_up(struct apple_dcp *dcp); int dcp_tunnel_crossbar_down(struct apple_dcp *dcp); +int dcp_direct_crossbar_link(struct apple_dcp *dcp, bool up); int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate); int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 4c2a7a23538c4..bff31352b4e78 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -475,6 +475,8 @@ struct apple_dcp { */ struct mutex tb_lock; bool tb_clock_ok; + /* tb_lock: the direct DP PHY crossbar output whose clocks run (T6030). */ + struct mux_control *direct_xbar_up; /* CRTC powered off while the Type-C cable stays attached */ bool typec_crtc_off; /* IOMFB reports its video interface ready after DPTX link training. */ diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 073d6332334cb..0e398d9fa069b 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -543,6 +543,8 @@ static int dptxport_call_set_link_rate(struct apple_epic_service *service, return 0; } } else if (dptx->atcphy) { + /* No clocked crossbar output across a PHY rate change. */ + dcp_direct_crossbar_link(service->ep->dcp, false); dptx->phy_ops.dp.link_rate = phy_link_rate; dptx->phy_ops.dp.set_rate = 1; ret = phy_configure(dptx->atcphy, &dptx->phy_ops); @@ -657,6 +659,8 @@ dptxport_call_deactivate(struct apple_epic_service *service, dev_dbg(dcp->dev, "DPTXPort: DEACTIVATE\n"); if (dcp->dptx_tunnel) dcp_tunnel_dpin_activate(dcp, false); + else + dcp_direct_crossbar_link(dcp, false); if (dptx->atcphy && !dcp->phy_managed_by_typec) phy_set_mode_ext(dptx->atcphy, PHY_MODE_INVALID, 0); @@ -723,6 +727,8 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, /* A re-link takes an established tunnel's crossbar down first. */ if (service->ep->dcp->dptx_tunnel && dptx->link_rate) dcp_tunnel_crossbar_down(service->ep->dcp); + else if (dptx->link_rate) + dcp_direct_crossbar_link(service->ep->dcp, false); return dptxport_call_will_change_link_config(service); case DPTX_APCALL_DID_CHANGE_LINK_CONFIG: { int ret = dptxport_call_did_change_link_config(service); @@ -733,6 +739,8 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, /* T6030 14.7: fail the link config without a crossbar. */ if (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_14_7) ret = xret; + } else if (!ret && dptx->link_rate) { + dcp_direct_crossbar_link(service->ep->dcp, true); } if (ret && service->ep->dcp->fw_compat == DCP_FIRMWARE_V_14_7 && service->ep->dcp->dptx_tunnel && reply_size >= sizeof(__le32)) { From 3267d33a35811fb68d6725c3d907ddaf95a1879b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:04:20 -0700 Subject: [PATCH 1610/2680] drm/apple: t6030: no external backlight on 14.7 pipes A display whose attributes say the DCP can drive its backlight gets a backlight device on its connector, whose level goes out with the next swap. The 14.7 swap the T6030 pipes send carries no backlight level, so on an external T6030 pipe that device would change nothing. Do not offer it on 14.7 firmware. Earlier firmware is unchanged. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 20dc783543bf3db0d7b642bf1dc79fefa9ce29d3) --- drivers/gpu/drm/apple/iomfb.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 0eb11c53a0446..eff8ee27f7aad 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1604,9 +1604,11 @@ int dcp_attributes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *handle, * Written once, so the hotplug work never sees a working display * without its backlight; a display whose attributes do not parse * gets none. The attributes can arrive after the hotplug work has - * run, so sync the backlight when the answer changes. + * run, so sync the backlight when the answer changes. 14.7 swaps + * carry no external backlight level. */ - ext = !ret && backlight_control && !dcp_has_panel(dcp); + ext = !ret && backlight_control && !dcp_has_panel(dcp) && + dcp->fw_compat != DCP_FIRMWARE_V_14_7; if (ext != READ_ONCE(dcp->ext_backlight)) { WRITE_ONCE(dcp->ext_backlight, ext); if (dcp->connector) From 0848602e208e8c1d205ede6e77210b763b269f0d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:20:17 -0700 Subject: [PATCH 1611/2680] drm/apple: t6030: let an external pipe sleep while it holds no link An external processor refused system sleep while a Type-C route was assigned to it. An adapter that enters DP mode with no display behind it gets a route at once, so sleep was refused for an adapter alone, even before the processor's firmware had ever started. Refuse sleep only for what the firmware holds: while the processor starts, and once it runs, while a DPTX link is connected or configured or a display is powered. Before the first start nothing is refused. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit e659230b5641b11aa9a8948beea1ed0c5e33ec09) --- drivers/gpu/drm/apple/iomfb_v14_7.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index b62099b8ca1c4..a197acb8ba568 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -1477,15 +1477,22 @@ bool iomfb_v14_7_external_ready(struct apple_dcp *dcp) /* * System sleep with this processor: allowed before its first start and - * while it idles with no display; refused while it starts or drives one, - * as the link and the display's buffers are not handed back across sleep. + * while it idles with no display link; refused while it starts or holds a + * link or a powered display, as those are not handed back across sleep. A + * route alone (an adapter in DP mode with no display) holds nothing. */ bool iomfb_v14_7_external_busy(struct apple_dcp *dcp) { - if (smp_load_acquire(&dcp->external_phase) == DCPEXT_STARTING) + switch (smp_load_acquire(&dcp->external_phase)) { + case DCPEXT_STARTING: return true; - return READ_ONCE(dcp->active_typec_route) || READ_ONCE(dcp->dptxport[0].connected) || - READ_ONCE(dcp->external_link_ready) || (dcp->v14 && READ_ONCE(dcp->v14->powered)); + case DCPEXT_RUNNING: + return READ_ONCE(dcp->dptxport[0].connected) || + READ_ONCE(dcp->external_link_ready) || + (dcp->v14 && READ_ONCE(dcp->v14->powered)); + default: + return false; + } } /* Refuse startup unless both the hardware floor and PMP vote are in place. */ From faf9f4b454fbcb1d0e49c15baaf455d37975b170 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 09:56:44 -0700 Subject: [PATCH 1612/2680] drm/apple: forward HDMI converter service requests on t6030 The HDMI port of the M3 Pro MacBook Pros is fed by the first external display processor through ATC3 and an on-board DP-to-HDMI converter, with the same hotplug, power and fixed-route handling as the M1/M2 Pro HDMI port. The converter asks the DisplayPort source for service the way every DP sink does, with a short low pulse on HPD (IRQ_HPD, 0.5 to 1 ms). It does so on link status changes and while it trains an HDMI FRL link, which the modes past HDMI 2.0 rates (4K at 120 Hz) need. The DPTX firmware handles such a request only after the host reports it with the remote port's sink IRQ call (group 8, command 9), which the driver never made. Add dcp-hdmi.c, used only by an "apple,t6030-dcpext" with a fixed HDMI connector and an HPD GPIO: - time each HPD pulse in the existing hard IRQ handler and report a pulse of at most 2 ms (or one too short to see the falling edge) while the HDMI link is up; longer lows stay with the HPD presence logic; - after the firmware configures an HBR3 link, also report every 100 ms for 10 s, since not every converter request reaches the GPIO; stop if the firmware refuses the call; - make the call without hpd_mutex, so a request the firmware waits for cannot be stuck behind another DPTX call. The work is enabled and disabled with the other Type-C and HDMI works, so it never runs while the DPTX endpoint is unavailable. Hooks in shared code: the HPD hard IRQ handler, the DPTX DidChangeLinkConfiguration call, probe, and the work enable/disable helpers. Each is a no-op unless the DCP set up this state, so the M1/M2 HDMI ports are unchanged. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 0960c7e9ef117cc8cdb9635606b949420911f3a7) --- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/dcp-fabric.c | 2 + drivers/gpu/drm/apple/dcp-hdmi.c | 237 +++++++++++++++++++++++++++ drivers/gpu/drm/apple/dcp-hdmi.h | 19 +++ drivers/gpu/drm/apple/dcp-internal.h | 3 + drivers/gpu/drm/apple/dcp.c | 6 + drivers/gpu/drm/apple/dptxep.c | 3 + 7 files changed, 271 insertions(+) create mode 100644 drivers/gpu/drm/apple/dcp-hdmi.c create mode 100644 drivers/gpu/drm/apple/dcp-hdmi.h diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 28b38ec7ae974..a9eaf1706a412 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -4,6 +4,7 @@ CFLAGS_trace.o = -I$(src) appledrm-y := apple_drv.o apple_resume.o appledrm-y += afk.o dcp.o dcp-fabric.o dcp_backlight.o dptxep.o iomfb.o parser.o systemep.o +appledrm-y += dcp-hdmi.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += audio.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += av.o appledrm-y += connector.o dcp-display-metadata.o dcp-180hz.o diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index ea644e01861f2..4220f0da427dd 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -21,6 +21,7 @@ #include "dcp.h" #include "dcp-fabric.h" #include "dcp-fabric-effects.h" +#include "dcp-hdmi.h" #include "dcpext_scanout.h" #include "ibootep.h" #include "dcp-internal.h" @@ -2283,6 +2284,7 @@ static int dcp_fixed_output_select(struct apple_dcp *dcp) */ irqreturn_t dcp_dp2hdmi_hpd_edge(int irq, void *data) { + dcp_hdmi_hpd_edge(data); dcp_hdmi_edge(data); return IRQ_WAKE_THREAD; diff --git a/drivers/gpu/drm/apple/dcp-hdmi.c b/drivers/gpu/drm/apple/dcp-hdmi.c new file mode 100644 index 0000000000000..0603de2a0e229 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-hdmi.c @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * HDMI port of the T6030 (M3 Pro) MacBook Pros. + * + * The port hangs off the first external display coprocessor: it drives + * ATC3 as a four-lane DisplayPort PHY into an on-board DP-to-HDMI + * converter. Power, hotplug and the fixed route work as for the M1/M2 Pro + * MacBook Pro HDMI port (hdmi-hpd, hdmi-pwren and dp2hdmi-pwren GPIOs, see + * dcp.c and dcp-fabric.c). + * + * Like any DisplayPort sink, the converter asks the source for service by + * pulsing HPD low for 0.5 to 1 ms (IRQ_HPD). It does that on link status + * changes and while it trains an HDMI FRL link to the display, which the + * modes beyond HDMI 2.0 rates need. The DPTX firmware services a request + * only once it is told that the sink raised one. This file tells it: + * + * - for every short low pulse on the HPD GPIO while the HDMI link is up; + * - every 100 ms for a while after the firmware configured an HBR3 link, + * the rate those modes use, because not every request of the converter + * shows up as a pulse on the GPIO. + * + * Only the T6030 external coprocessor with a fixed HDMI route uses this. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "afk.h" +#include "dcp-hdmi.h" +#include "dcp-internal.h" +#include "dptxep.h" + +/* Longest HPD low time still taken as a service request (DP: at most 1 ms). */ +#define DCP_HDMI_IRQ_HPD_MAX_NS (2 * NSEC_PER_MSEC) +/* Requests are polled this long after an HBR3 link configuration ... */ +#define DCP_HDMI_SERVICE_WINDOW_MS 10000 +/* ... at this period. */ +#define DCP_HDMI_SERVICE_PERIOD_MS 100 + +/* DPTX remote port call: the sink raised an IRQ_HPD service request. */ +#define DPTX_GROUP_HOTPLUG 8 +#define DPTX_CMD_SINK_IRQ 9 +#define DPTX_SINK_IRQ_PAD 16 + +struct dcp_hdmi { + struct apple_dcp *dcp; + struct delayed_work work; + /* Hard IRQ context only: time of the last falling HPD edge, or 0. */ + u64 falling_ns; + /* Short pulses seen and not yet passed on. */ + atomic_t pulses; + /* Poll requests until this time (jiffies); 0 when not polling. */ + unsigned long poll_until; + /* The HPD GPIO can be read from hard IRQ context. */ + bool timed_edges; + /* Work context only. */ + bool announced; +}; + +static bool dcp_hdmi_link_up(struct apple_dcp *dcp) +{ + /* A Type-C display that borrowed this pipeline has its own HPD. */ + return !READ_ONCE(dcp->crashed) && !READ_ONCE(dcp->active_typec_route) && + READ_ONCE(dcp->dptxport[0].enabled) && + READ_ONCE(dcp->dptxport[0].connected); +} + +/* + * No hpd_mutex: the firmware may be waiting for this request while another + * DPTX call made under that lock is still outstanding. AFK runs concurrent + * service calls on one endpoint. + */ +static int dcp_hdmi_sink_irq(struct apple_dcp *dcp) +{ + struct apple_epic_service *service = READ_ONCE(dcp->dptxport[0].service); + + if (!service) + return -ENODEV; + return afk_service_call(service, DPTX_GROUP_HOTPLUG, DPTX_CMD_SINK_IRQ, + NULL, 0, DPTX_SINK_IRQ_PAD, NULL, 0, + DPTX_SINK_IRQ_PAD); +} + +static void dcp_hdmi_work(struct work_struct *work) +{ + struct dcp_hdmi *hdmi = container_of(to_delayed_work(work), + struct dcp_hdmi, work); + struct apple_dcp *dcp = hdmi->dcp; + unsigned long until = READ_ONCE(hdmi->poll_until); + bool pulse = atomic_xchg(&hdmi->pulses, 0) > 0; + bool poll = until && time_before(jiffies, until); + int ret; + + if (!pulse && !poll) + return; + /* The link rate can be set before the connection is marked up. */ + if (!dcp_hdmi_link_up(dcp)) + goto next; + + ret = dcp_hdmi_sink_irq(dcp); + if (ret) { + dev_warn_ratelimited(dcp->dev, + "HDMI: converter service request not delivered: %d\n", + ret); + /* Don't keep polling a firmware that refuses the request. */ + cmpxchg(&hdmi->poll_until, until, 0); + return; + } + if (!hdmi->announced) { + hdmi->announced = true; + dev_info(dcp->dev, "HDMI: forwarding converter service requests (%s)\n", + pulse ? "HPD pulse" : "HBR3 link"); + } else { + dev_dbg(dcp->dev, "HDMI: converter service request (%s)\n", + pulse ? "HPD pulse" : "poll"); + } + +next: + if (poll) + mod_delayed_work(system_freezable_wq, &hdmi->work, + msecs_to_jiffies(DCP_HDMI_SERVICE_PERIOD_MS)); +} + +/* + * Hard IRQ context, from the HPD GPIO's edge handler, on every edge. + * Records the low time of each pulse and queues short ones. + */ +void dcp_hdmi_hpd_edge(struct apple_dcp *dcp) +{ + struct dcp_hdmi *hdmi = dcp->hdmi; + u64 now, low; + int level; + + if (!hdmi || !hdmi->timed_edges) + return; + + now = ktime_get_ns(); + level = gpiod_get_value(dcp->hdmi_hpd); + if (level < 0) + return; + if (!level) { + hdmi->falling_ns = now; + return; + } + + /* A rising edge without a falling one: the pulse ended before we read it. */ + low = hdmi->falling_ns ? now - hdmi->falling_ns : 0; + hdmi->falling_ns = 0; + if (low > DCP_HDMI_IRQ_HPD_MAX_NS) + return; /* A cable change, handled by the HPD presence logic. */ + + atomic_inc(&hdmi->pulses); + mod_delayed_work(system_freezable_wq, &hdmi->work, 0); +} + +/* From DPTX DidChangeLinkConfiguration, with the rate that is now set. */ +void dcp_hdmi_link_configured(struct apple_dcp *dcp, u32 link_rate) +{ + struct dcp_hdmi *hdmi = dcp->hdmi; + unsigned long until; + + if (!hdmi) + return; + if (link_rate != LINK_RATE_HBR3) { + WRITE_ONCE(hdmi->poll_until, 0); + return; + } + + until = jiffies + msecs_to_jiffies(DCP_HDMI_SERVICE_WINDOW_MS); + WRITE_ONCE(hdmi->poll_until, until ?: 1); + mod_delayed_work(system_freezable_wq, &hdmi->work, 0); +} + +void dcp_hdmi_enable(struct apple_dcp *dcp) +{ + if (dcp->hdmi) + enable_delayed_work(&dcp->hdmi->work); +} + +/* Returns with no request in flight; later edges queue nothing. */ +void dcp_hdmi_disable(struct apple_dcp *dcp) +{ + if (dcp->hdmi) + disable_delayed_work_sync(&dcp->hdmi->work); +} + +static void dcp_hdmi_release(void *data) +{ + struct dcp_hdmi *hdmi = data; + + /* The HPD IRQ was requested later, so it is already freed. */ + hdmi->dcp->hdmi = NULL; + disable_delayed_work_sync(&hdmi->work); +} + +/* + * Called from probe once the HPD GPIO is known and before its IRQ is + * requested, so that the IRQ is released before this state on unbind. + */ +int dcp_hdmi_init(struct apple_dcp *dcp) +{ + struct dcp_hdmi *hdmi; + int ret; + + if (!dcp->hdmi_hpd || + dcp->fixed_connector_type != DRM_MODE_CONNECTOR_HDMIA || + !of_device_is_compatible(dcp->dev->of_node, "apple,t6030-dcpext")) + return 0; + + hdmi = devm_kzalloc(dcp->dev, sizeof(*hdmi), GFP_KERNEL); + if (!hdmi) + return -ENOMEM; + + hdmi->dcp = dcp; + atomic_set(&hdmi->pulses, 0); + hdmi->timed_edges = !gpiod_cansleep(dcp->hdmi_hpd); + INIT_DELAYED_WORK(&hdmi->work, dcp_hdmi_work); + /* Balanced by dcp_hdmi_enable() once the DPTX endpoint may be used. */ + disable_delayed_work(&hdmi->work); + + ret = devm_add_action_or_reset(dcp->dev, dcp_hdmi_release, hdmi); + if (ret) + return ret; + + dcp->hdmi = hdmi; + if (!hdmi->timed_edges) + dev_info(dcp->dev, "HDMI: HPD GPIO can sleep, converter requests are polled only\n"); + return 0; +} diff --git a/drivers/gpu/drm/apple/dcp-hdmi.h b/drivers/gpu/drm/apple/dcp-hdmi.h new file mode 100644 index 0000000000000..2138b400d8c98 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-hdmi.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __APPLE_DCP_HDMI_H__ +#define __APPLE_DCP_HDMI_H__ + +#include + +struct apple_dcp; + +/* + * Service requests of the DP-to-HDMI converter behind the T6030 HDMI port. + * Every call is a no-op for any other DCP. + */ +int dcp_hdmi_init(struct apple_dcp *dcp); +void dcp_hdmi_enable(struct apple_dcp *dcp); +void dcp_hdmi_disable(struct apple_dcp *dcp); +void dcp_hdmi_hpd_edge(struct apple_dcp *dcp); +void dcp_hdmi_link_configured(struct apple_dcp *dcp, u32 link_rate); + +#endif /* __APPLE_DCP_HDMI_H__ */ diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index bff31352b4e78..438dd3396d09c 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -203,6 +203,7 @@ enum dcp_test_point { /* TODO: move IOMFB members to its own struct */ struct dcpext_scanout; +struct dcp_hdmi; struct apple_dcp { /* An adopted session has no confirmed firmware DMA stop boundary. */ @@ -557,6 +558,8 @@ struct apple_dcp { struct dcp_fabric_presence hdmi_presence; struct delayed_work hdmi_settle_wq; struct delayed_work hdmi_recheck_wq; + /* T6030 HDMI converter service requests, see dcp-hdmi.c */ + struct dcp_hdmi *hdmi; }; void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, bool connected); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index cc127a77c38cf..2c4d95945b9c4 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -47,6 +47,7 @@ #include "dcp.h" #include "dcp-fabric-effects.h" #include "dcp-fabric.h" +#include "dcp-hdmi.h" #include "dcpext_scanout.h" #include "dcp-internal.h" #include "dcp-external-bw.h" @@ -2687,6 +2688,7 @@ VISIBLE_IF_KUNIT void dcp_disable_typec_work(struct apple_dcp *dcp, bool release cancel_delayed_work_sync(&dcp->external_stop_retry_wq); disable_delayed_work_sync(&dcp->hdmi_settle_wq); disable_delayed_work_sync(&dcp->hdmi_recheck_wq); + dcp_hdmi_disable(dcp); } VISIBLE_IF_KUNIT void dcp_enable_typec_work(struct apple_dcp *dcp) @@ -2696,6 +2698,7 @@ VISIBLE_IF_KUNIT void dcp_enable_typec_work(struct apple_dcp *dcp) enable_delayed_work(&dcp->announce_wq); enable_delayed_work(&dcp->hdmi_settle_wq); enable_delayed_work(&dcp->hdmi_recheck_wq); + dcp_hdmi_enable(dcp); /* A cable can be routed before the DRM component binds. */ if (READ_ONCE(dcp->typec_cable_connected)) mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); @@ -3279,6 +3282,9 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp->hdmi_hpd = devm_gpiod_get_optional(dev, "hdmi-hpd", GPIOD_IN); if (IS_ERR(dcp->hdmi_hpd)) return PTR_ERR(dcp->hdmi_hpd); + ret = dcp_hdmi_init(dcp); + if (ret) + return ret; if (dcp->hdmi_hpd) { int irq = gpiod_to_irq(dcp->hdmi_hpd); if (irq < 0) { diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 0e398d9fa069b..207f10e6400fd 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -15,6 +15,7 @@ #include "afk.h" #include "dcp.h" +#include "dcp-hdmi.h" #include "dptxep.h" #include "dcp-180hz.h" #include "dptx-service.h" @@ -742,6 +743,8 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, } else if (!ret && dptx->link_rate) { dcp_direct_crossbar_link(service->ep->dcp, true); } + if (!ret) + dcp_hdmi_link_configured(service->ep->dcp, dptx->link_rate); if (ret && service->ep->dcp->fw_compat == DCP_FIRMWARE_V_14_7 && service->ep->dcp->dptx_tunnel && reply_size >= sizeof(__le32)) { put_unaligned_le32(1, reply); From 2f9a37634699693554295834549cab1d5d43062d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:19:17 -0700 Subject: [PATCH 1613/2680] drm/apple: t6030: answer DCPLink callbacks in the order they are handled The 14.7 DCPLink client kept the firmware's outstanding callbacks as a stack and only ever answered the newest one. That holds while every callback is nested in the handling of the one before it. An external processor also sends callbacks from separate firmware threads: a display description being published while a mode set runs, for example. Two such callbacks are outstanding at once; the older one is handled first, its answer matched the newest entry, and the link stopped. A stopped link cannot be restarted, so the processor's display was lost until reboot. Answer the callback that was handled, wherever it sits among those outstanding, and send a call made from inside a handler on the stream of the callback that handler answers. With strictly nested callbacks, as the panel's DCP sends them, nothing changes. Log the tag and error of a callback whose handler fails, and an answer that matches no outstanding callback, since either stops the link. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit dd724a3b4894b2a60e740b597b7897f0236411e2) --- drivers/gpu/drm/apple/iomfb_v14_7_link.c | 59 ++++++++++++++++-------- drivers/gpu/drm/apple/iomfb_v14_7_link.h | 9 +++- 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_link.c b/drivers/gpu/drm/apple/iomfb_v14_7_link.c index 38bc2acd11859..27c3bff046812 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7_link.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7_link.c @@ -224,9 +224,12 @@ void dcp_v14_link_receive(struct dcp_v14_link *link, u64 message) mutex_unlock(&link->lock); } -/* Places a call packet (header, input, zeroed output) and sends it. */ +/* + * Places a call packet (header, input, zeroed output) and sends it. A nested + * call goes out on the stream of @parent, the callback its caller handles. + */ static int link_submit(struct dcp_v14_link *link, const struct apple_dcp_link_rpc_header *rpc, - u32 size, bool nested) + u32 size, bool nested, u64 parent) { struct apple_dcp_link_stream_layout layout; u32 total, offset = 0, input_size, output_size; @@ -245,7 +248,7 @@ static int link_submit(struct dcp_v14_link *link, const struct apple_dcp_link_rp goto out; } if (nested) { - u64 callback = link->callbacks[link->callback_count - 1].message; + u64 callback = parent; if (apple_dcp_link_message_stream(callback)) { ret = -EOPNOTSUPP; @@ -289,10 +292,14 @@ static int link_submit(struct dcp_v14_link *link, const struct apple_dcp_link_rp return ret; } -/* Writes the innermost callback's output in place and acknowledges it. */ +/* + * Writes a callback's output in place and acknowledges it. Usually that is + * the innermost callback; one from another firmware thread may still be + * waiting below it, and is answered in its turn. + */ static int link_reply(struct dcp_v14_link *link, u64 message, const void *output, u32 size) { - unsigned int top; + unsigned int i; int ret = -EINVAL; mutex_lock(&link->lock); @@ -300,19 +307,26 @@ static int link_reply(struct dcp_v14_link *link, u64 message, const void *output ret = -EIO; goto out; } - if (!link->callback_count) + for (i = link->callback_count; i > 0; i--) + if (link->callbacks[i - 1].message == message) + break; + if (!i || size != link->callbacks[i - 1].size) { + dev_err(link->dev, "reply to callback %#llx of %u bytes matches none of %u waiting\n", + message, size, link->callback_count); goto out; - top = link->callback_count - 1; - if (message != link->callbacks[top].message || size != link->callbacks[top].size) - goto out; - memcpy(link->callbacks[top].output, output, size); + } + i--; + memcpy(link->callbacks[i].output, output, size); dma_wmb(); ret = apple_rtkit_send_message(link->rtk, APPLE_DCP_LINK_ENDPOINT, apple_dcp_link_rpc_reply(message), NULL, false); - if (ret) + if (ret) { link_fail(link); - else + } else { link->callback_count--; + memmove(&link->callbacks[i], &link->callbacks[i + 1], + (link->callback_count - i) * sizeof(link->callbacks[0])); + } out: mutex_unlock(&link->lock); return ret; @@ -354,11 +368,15 @@ static int link_callback(struct dcp_v14_link *link, struct dcp_v14_event *e, if (in > e->size - sizeof(*h) || out != e->size - sizeof(*h) - in) return -EPROTO; memset(e->data + sizeof(*h) + in, 0, out); - link->callback_depth++; + if (link->callback_depth >= DCP_V14_LINK_MAX_CALLBACKS) + return -EPROTO; + link->handling[link->callback_depth++] = e->message; ret = callback(cookie, tag, h + 1, in, e->data + sizeof(*h) + in, out); link->callback_depth--; - if (ret) + if (ret) { + dev_err(link->dev, "callback %#x (%u/%u bytes) failed: %d\n", tag, in, out, ret); return ret; + } if (tag == DCP_V14_TAG_D589 && in == DCP_V14_D589_IN_SIZE && completed) *completed = get_unaligned_le32(h + 1); return link_reply(link, e->message, e->data + sizeof(*h) + in, out); @@ -390,7 +408,7 @@ int dcp_v14_link_call(struct dcp_v14_link *link, u32 tag, const void *input, { struct apple_dcp_link_rpc_header *packet; unsigned long deadline = jiffies + DCP_V14_CALL_TIMEOUT; - u64 expected = APPLE_DCP_LINK_MSG_RPC_REPLY; + u64 expected = APPLE_DCP_LINK_MSG_RPC_REPLY, parent = 0; u32 size, completed = 0; bool nested, replied = false; int ret; @@ -431,12 +449,13 @@ int dcp_v14_link_call(struct dcp_v14_link *link, u32 tag, const void *input, kfree(packet); return -EPROTO; } - /* The reply to a nested call carries the callback's side. */ - if (nested) - expected |= link->callbacks[link->callback_count - 1].message & - APPLE_DCP_LINK_MSG_REMOTE; + /* The reply to a nested call carries the side of the callback handled. */ + if (nested) { + parent = link->handling[link->callback_depth - 1]; + expected |= parent & APPLE_DCP_LINK_MSG_REMOTE; + } mutex_unlock(&link->lock); - ret = link_submit(link, packet, size, nested); + ret = link_submit(link, packet, size, nested, parent); /* A callback arrived between the drain and the submit. */ if (ret == -EBUSY && !nested && time_before(jiffies, deadline)) goto retry; diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_link.h b/drivers/gpu/drm/apple/iomfb_v14_7_link.h index 95e6ccdbb63d2..cfceedc018328 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7_link.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7_link.h @@ -59,13 +59,20 @@ struct dcp_v14_link { struct apple_dcp_link_rpc_header header; } calls[2]; - /* Firmware callbacks awaiting a reply, innermost last. */ + /* + * Firmware callbacks awaiting a reply, in arrival order. Callbacks + * from separate firmware threads can be outstanding at once and are + * answered in the order they are handled, not necessarily the last + * one first. + */ struct { u64 message; void *output; u32 size; u32 call; } callbacks[DCP_V14_LINK_MAX_CALLBACKS]; + /* The callback each running handler answers, innermost last. */ + u64 handling[DCP_V14_LINK_MAX_CALLBACKS]; }; void dcp_v14_link_init(struct dcp_v14_link *link, struct device *dev, From dbc69d561e063edaf26b9c3b1f6dfc655ef50cc0 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:19:45 -0700 Subject: [PATCH 1614/2680] drm/apple: t6030: set an external mode only on a full display description An external processor describes the attached display in parts: ColorElements, TimingElements, DisplayAttributes, Transport. On an HPD bounce it withdraws them all and publishes them again once the link is back. A mode set that ran in between asked for timing and color ids of the withdrawn description while the new one was still in transfer, and the link stopped (seen on hw: typec1, adapter plugged at boot, flipped). Track which parts are published. A mode set first waits, for up to 3 s, until timings, color modes and attributes are all published and no property is in transfer, with the RPC lock released so that the publication can proceed; it then looks the mode up in the catalog of the new description. If only the attributes are still missing, the mode is set anyway, as they only size the display. Otherwise the mode set fails with -EAGAIN. Swaps are skipped, and their flips completed, while the description is incomplete. Each wait is logged when it starts and when the description is complete again. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 679ef66f246653e13542cacd50bdd535b5516282) --- drivers/gpu/drm/apple/iomfb_v14_7.c | 113 ++++++++++++++++++++++++++-- 1 file changed, 106 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index a197acb8ba568..91d77d092de0f 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -127,6 +127,9 @@ struct apple_dcp_v14 { bool powered; /* External: catalog generation when a chunked property started. */ u64 chunk_generation; + /* External: the parts of the display's description published now. */ + unsigned int described; + wait_queue_head_t described_wait; /* Boot framebuffer and native panel timing (notch rows included). */ u32 stride; @@ -349,6 +352,14 @@ static int dcp_v14_map_piodma(struct apple_dcp_v14 *v14, u32 id) return 0; } +/* Parts of an external display's description, as the firmware publishes it. */ +#define DCP_V14_DESC_TIMING BIT(0) +#define DCP_V14_DESC_COLOR BIT(1) +#define DCP_V14_DESC_ATTRS BIT(2) +#define DCP_V14_DESC_ALL (DCP_V14_DESC_TIMING | DCP_V14_DESC_COLOR | DCP_V14_DESC_ATTRS) +/* How long a mode set waits for a description being (re)published. */ +#define DCP_V14_DESC_TIMEOUT_MS 3000 + /* An external processor's default stride; no framebuffer is allocated for it. */ #define DCP_V14_EXT_STRIDE (1920 * 4) /* The display clock of every T6030 external pipe, unless the DT names one. */ @@ -433,17 +444,32 @@ static void dcp_v14_external_published(struct apple_dcp_v14 *v14, const char *ke struct apple_connector *connector; struct dcp_v14_raw *raw = NULL; struct dcp_parse_ctx ctx; + unsigned int part = 0; u32 i; int ret; - if (!dcp || (strcmp(key, "TimingElements") && strcmp(key, "DisplayAttributes") && - strcmp(key, "ColorElements") && strcmp(key, "Transport"))) + if (!strcmp(key, "TimingElements")) + part = DCP_V14_DESC_TIMING; + else if (!strcmp(key, "ColorElements")) + part = DCP_V14_DESC_COLOR; + else if (!strcmp(key, "DisplayAttributes")) + part = DCP_V14_DESC_ATTRS; + else if (strcmp(key, "Transport")) return; for (i = 0; i < v14->raw_count && !removed; i++) if (!v14->raw[i].service && !strcmp(v14->raw[i].key, key)) raw = &v14->raw[i]; if (!removed && !raw) return; + /* The timings count once they parse; see below. */ + if (removed) + v14->described &= ~part; + else if (part != DCP_V14_DESC_TIMING) + v14->described |= part; + /* A mode set waits for the whole description; see external_settle(). */ + wake_up_all(&v14->described_wait); + if (!dcp) + return; dev_info(dcp->dev, "display %s %s, %u bytes\n", key, removed ? "withdrawn" : "published", raw ? raw->size : 0); @@ -473,6 +499,9 @@ static void dcp_v14_external_published(struct apple_dcp_v14 *v14, const char *ke dev_warn(dcp->dev, "display timings not used: %d\n", ret); return; } + if (READ_ONCE(dcp->nr_modes)) + v14->described |= DCP_V14_DESC_TIMING; + wake_up_all(&v14->described_wait); dev_info(dcp->dev, "display has %u usable modes\n", READ_ONCE(dcp->nr_modes)); dcp_v14_external_hotplug(dcp, READ_ONCE(dcp->nr_modes) > 0); } else if (!strcmp(key, "DisplayAttributes") && !removed) { @@ -1188,6 +1217,7 @@ int iomfb_v14_7_probe(struct apple_dcp *dcp) v14->dev = dev; v14->dcp = dcp; mutex_init(&v14->lock); + init_waitqueue_head(&v14->described_wait); INIT_WORK(&v14->idle_work, dcp_v14_idle); ret = devm_add_action_or_reset(dev, dcp_v14_release, v14); if (ret) @@ -1598,6 +1628,7 @@ static int dcpext_boot(struct apple_dcp *dcp) v14->stride = DCP_V14_EXT_STRIDE; } mutex_init(&v14->lock); + init_waitqueue_head(&v14->described_wait); INIT_WORK(&v14->idle_work, dcp_v14_idle); dcp->v14 = v14; dcp_v14_link_init(&v14->link, dev, NULL); @@ -2168,18 +2199,81 @@ static int dcp_v14_set_power(struct apple_dcp_v14 *v14, bool on) return ret; } +/* The display is fully described, and no description is in transfer. */ +static bool dcp_v14_external_described(struct apple_dcp_v14 *v14) +{ + return READ_ONCE(v14->failed) || + ((READ_ONCE(v14->described) & DCP_V14_DESC_ALL) == DCP_V14_DESC_ALL && + !READ_ONCE(v14->chunk)); +} + +/* + * Waits, for a bounded time, until the firmware has published the whole + * description of the attached display (timings, color modes, attributes) + * and is not in the middle of publishing one. A hotplug bounce withdraws it + * and publishes it again part by part; a mode set in between would use + * timing ids that are gone, or run alongside the publication. Returns with + * the RPC lock held, unless it fails. + */ +static int dcp_v14_external_settle(struct apple_dcp *dcp, struct apple_dcp_v14 *v14) +{ + unsigned long deadline = jiffies + msecs_to_jiffies(DCP_V14_DESC_TIMEOUT_MS); + bool waited = false; + long left; + + mutex_lock(&v14->lock); + while (!dcp_v14_external_described(v14)) { + left = (long)(deadline - jiffies); + if (left <= 0) + break; + if (!waited) + dev_info(dcp->dev, "display description incomplete (%#x%s); waiting before the mode is set\n", + v14->described, v14->chunk ? ", in transfer" : ""); + waited = true; + mutex_unlock(&v14->lock); + wait_event_timeout(v14->described_wait, dcp_v14_external_described(v14), left); + mutex_lock(&v14->lock); + } + if (v14->failed) { + mutex_unlock(&v14->lock); + return -EIO; + } + if (dcp_v14_external_described(v14)) { + if (waited) + dev_info(dcp->dev, "display description complete again\n"); + return 0; + } + /* The attributes only size the display: timings and colors suffice. */ + if ((v14->described & (DCP_V14_DESC_TIMING | DCP_V14_DESC_COLOR)) == + (DCP_V14_DESC_TIMING | DCP_V14_DESC_COLOR) && !v14->chunk) { + dev_warn(dcp->dev, "display description still incomplete (%#x) after %u ms; setting the mode anyway\n", + v14->described, DCP_V14_DESC_TIMEOUT_MS); + return 0; + } + dev_warn(dcp->dev, "display description incomplete (%#x%s) after %u ms; mode not set\n", + v14->described, v14->chunk ? ", in transfer" : "", DCP_V14_DESC_TIMEOUT_MS); + mutex_unlock(&v14->lock); + return -EAGAIN; +} + /* An external display: power down, set the new mode, power up. */ static int dcp_v14_external_modeset(struct apple_dcp *dcp, struct drm_crtc_state *crtc_state) { struct apple_dcp_v14 *v14 = dcp->v14; - struct dcp_display_mode mode; + struct dcp_display_mode mode = {}; int ret = 0; if (!v14 || READ_ONCE(v14->failed) || !v14->opened) return -EIO; - if (!lookup_mode(dcp, &crtc_state->mode, &mode)) - return -EINVAL; - mutex_lock(&v14->lock); + ret = dcp_v14_external_settle(dcp, v14); + if (ret) + goto out; + /* Only after the wait: a new description replaces the catalog. */ + if (!lookup_mode(dcp, &crtc_state->mode, &mode)) { + mutex_unlock(&v14->lock); + ret = -EINVAL; + goto out; + } if (v14->powered && dcp_v14_set_power(v14, false)) dev_warn(dcp->dev, "display did not power down before the mode change\n"); if (READ_ONCE(v14->failed)) @@ -2193,6 +2287,7 @@ static int dcp_v14_external_modeset(struct apple_dcp *dcp, struct drm_crtc_state v14->panel_height = mode.mode.vdisplay; } mutex_unlock(&v14->lock); +out: dev_info(dcp->dev, "display mode " DRM_MODE_FMT " (timing %u, color %u): %d\n", DRM_MODE_ARG(&crtc_state->mode), mode.timing_mode_id, mode.color_mode_id, ret); if (ret) @@ -2223,8 +2318,12 @@ void iomfb_v14_7_flush(struct apple_dcp *dcp, struct drm_crtc *crtc, if (!p) p = crtc->primary->state; fb = p && p->visible ? p->fb : NULL; - /* No swaps while an external display is off, unset or unplugged. */ + /* + * No swaps while an external display is off, unset, unplugged, or + * being described again. + */ if (dcp->external && (!dcp->v14 || !READ_ONCE(dcp->v14->powered) || + !dcp_v14_external_described(dcp->v14) || !READ_ONCE(dcp->mode_state.valid) || !dcp->connector || !READ_ONCE(dcp->connector->connected))) { dcp_drm_crtc_vblank(dcp->crtc); From 3c74af62eef86913f830c82e408162def1ccc4cb Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:20:20 -0700 Subject: [PATCH 1615/2680] drm/apple: t6030: retry an external display link or mode, bounded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When an external display's mode could not be set, or the desktop powered its pipe off after the display had been withdrawn, nothing tried again: the port still had HPD high and a display behind it, but the connector stayed disconnected (seen on hw after an HPD bounce on typec1). Give a native external pipe a bounded recovery, at most 3 tries per attached display, 0.5, 1 and 2 s apart (2, 4 and 8 s after a withdrawn description), each one logged: - a failed mode set (a mode that is gone, or a description that did not settle) retries; - releasing the link with no display described, or with an HPD deassert error, while the port still has HPD, retries; - a description withdrawn while the port has HPD and the pipe is on gets a retry if it is not published again in time. A retry re-applies the mode if the display is described again, and otherwise releases the link if it is still up and connects it again, which makes the firmware describe the display anew. A new HPD on the port resets the count. A firmware session that has stopped is not retried; that is logged as needing a reboot. The connect path keeps its own reconnect attempts, as before. Deliberate power-off (DPMS) is unaffected: the display stays described and its link comes back on the next power-on. [ryan: adapted to 7.2.10 — the retry work requeues our Type-C reconnect work under hpd_mutex (no dcp_queue_typec_reconnect() here) and checks the port with dptx_service_available(); the mode is marked valid under mode_state.lock] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit e1711e684d80e7e5734f40e76a7b1002f54b8296) --- drivers/gpu/drm/apple/dcp-internal.h | 3 + drivers/gpu/drm/apple/dcp.c | 98 ++++++++++++++++++++++++++++ drivers/gpu/drm/apple/dcp.h | 3 + drivers/gpu/drm/apple/iomfb_v14_7.c | 35 +++++++++- drivers/gpu/drm/apple/iomfb_v14_7.h | 2 + 5 files changed, 138 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 438dd3396d09c..315f47bbe7b4b 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -266,6 +266,9 @@ struct apple_dcp { struct work_struct external_ready_work; /* Native: the firmware's start state, see iomfb_v14_7.c. */ unsigned int external_phase; + /* Native: bounded recovery of a display link or mode, see dcp.c. */ + struct delayed_work external_retry_wq; + atomic_t external_retries; atomic_t external_requested; /* DCP_FIRMWARE_V_14_7 state; outlives this device once RTKit runs. */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 2c4d95945b9c4..54d98adaed372 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1264,6 +1264,87 @@ void dcp_external_ready(struct apple_dcp *dcp) } } +#define DCP_EXTERNAL_RETRIES 3 + +/* + * Bounded recovery for a native external pipe: re-apply the display mode, + * or redo the display link, up to DCP_EXTERNAL_RETRIES times per attached + * display, @base_ms, then twice and four times as long apart. A stopped + * firmware session cannot be recovered, and is not retried. + */ +void dcp_external_retry(struct apple_dcp *dcp, const char *why, int error, + unsigned int base_ms) +{ + unsigned int n, delay; + + if (!dcp->external_native) + return; + if (iomfb_v14_7_external_failed(dcp)) { + dev_err(dcp->dev, "%s (%d): the external display processor stopped; no retry until reboot\n", + why, error); + return; + } + if (delayed_work_pending(&dcp->external_retry_wq)) { + dev_info(dcp->dev, "%s (%d): a retry is already queued\n", why, error); + return; + } + n = atomic_inc_return(&dcp->external_retries); + if (n > DCP_EXTERNAL_RETRIES) { + atomic_set(&dcp->external_retries, DCP_EXTERNAL_RETRIES); + dev_warn(dcp->dev, "%s (%d): no retry left after %u; replug the display\n", + why, error, DCP_EXTERNAL_RETRIES); + return; + } + delay = base_ms << (n - 1); + dev_info(dcp->dev, "%s (%d): retry %u of %u in %u ms\n", why, error, n, + DCP_EXTERNAL_RETRIES, delay); + mod_delayed_work(system_freezable_wq, &dcp->external_retry_wq, msecs_to_jiffies(delay)); +} + +void dcp_external_retry_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, + external_retry_wq); + struct apple_connector *connector = READ_ONCE(dcp->connector); + + if (iomfb_v14_7_external_failed(dcp)) { + dev_err(dcp->dev, "display retry: the external display processor stopped; no retry until reboot\n"); + return; + } + /* Described again: re-apply the mode if it is not set. */ + if (connector && READ_ONCE(connector->connected)) { + if (READ_ONCE(dcp->mode_state.valid)) { + dev_info(dcp->dev, "display retry: the display is back and set\n"); + return; + } + dev_info(dcp->dev, "display retry: setting the display mode again\n"); + dcp_mode_invalidate(&dcp->mode_state); + schedule_work(&connector->hotplug_wq); + return; + } + if (!READ_ONCE(dcp->typec_cable_connected)) { + dev_info(dcp->dev, "display retry: the port has no display any more\n"); + return; + } + /* + * The port holds a display that the firmware no longer describes: + * release the link if it is still up, and connect it again, which + * makes the firmware describe the display anew. + */ + dev_info(dcp->dev, "display retry: reconnecting the display link\n"); + if (dptx_service_available(&dcp->dptxport[0]) && READ_ONCE(dcp->dptxport[0].connected)) { + int ret = dptxport_set_hpd(dcp->dptxport[0].service, false); + + if (ret) + dev_warn(dcp->dev, "display retry: HPD deassert failed: %d\n", ret); + dcp_dptx_disconnect(dcp, 0); + } + guard(mutex)(&dcp->hpd_mutex); + dcp->typec_reconnect_tries = 0; + if (dcp->typec_cable_connected) + mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); +} + int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) { if (!dcp_is_external_h17p(dcp)) @@ -1678,6 +1759,9 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) dcp->typec_reconnect_tries = 0; dcp->placeholder_retried = false; cancel_delayed_work(&dcp->typec_reconnect_wq); + /* A newly attached display gets its own retries. */ + if (dcp->external_native) + atomic_set(&dcp->external_retries, 0); } ret = dcp_dptx_connect(dcp, port); @@ -2129,6 +2213,7 @@ void dcp_poweroff(struct platform_device *pdev) _dcp_poweroff(dcp); if (dcp_is_typec_output(dcp)) { + ret = 0; /* DCP owns a synthetic HPD for Type-C. Release it with the CRTC. */ if (dptx_service_available(&dcp->dptxport[0]) && dcp->dptxport[0].connected) { ret = dptxport_set_hpd(dcp->dptxport[0].service, false); @@ -2137,6 +2222,17 @@ void dcp_poweroff(struct platform_device *pdev) "failed to deassert Type-C DPTX HPD: %d\n", ret); dcp_dptx_disconnect(dcp, 0); } + /* + * A native external pipe whose display is no longer described + * (it went away during a hotplug bounce) is powered off by the + * desktop for good: with HPD still high, connect the link again + * so that the firmware describes the display anew. + */ + if (dcp->external_native && READ_ONCE(dcp->typec_cable_connected) && + (ret || !dcp->connector || !READ_ONCE(dcp->connector->connected))) + dcp_external_retry(dcp, ret ? "display link released with an HPD error" : + "display link released with no display described", + ret, 500); } else if (dcp->hdmi_hpd) { bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); if (!connected) { @@ -2833,6 +2929,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (dcp->external_native) { enable_work(&dcp->vblank_wq); enable_delayed_work(&dcp->swap_watchdog_wq); + enable_delayed_work(&dcp->external_retry_wq); enable_work(&dcp->dimensions_wq); dcp_enable_typec_work(dcp); return 0; @@ -2968,6 +3065,7 @@ VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, v if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { if (dcp->external_native) { dcp_disable_typec_work(dcp, true); + disable_delayed_work_sync(&dcp->external_retry_wq); disable_delayed_work_sync(&dcp->swap_watchdog_wq); disable_work_sync(&dcp->vblank_wq); } else if (!dcp->external) { diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index f1675bd35ef9d..2e3832ff0a17c 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -70,6 +70,9 @@ void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message); int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port); bool dcp_t6030_ext_native(const struct device_node *np); void dcp_external_ready(struct apple_dcp *dcp); +void dcp_external_retry(struct apple_dcp *dcp, const char *why, int error, + unsigned int base_ms); +void dcp_external_retry_work(struct work_struct *work); int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); bool dcp_hold_resume_event(struct platform_device *pdev); diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 91d77d092de0f..d6605d232a873 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -359,6 +359,8 @@ static int dcp_v14_map_piodma(struct apple_dcp_v14 *v14, u32 id) #define DCP_V14_DESC_ALL (DCP_V14_DESC_TIMING | DCP_V14_DESC_COLOR | DCP_V14_DESC_ATTRS) /* How long a mode set waits for a description being (re)published. */ #define DCP_V14_DESC_TIMEOUT_MS 3000 +/* A withdrawn description that is not back by then gets its link redone. */ +#define DCP_V14_DESC_RELINK_MS 2000 /* An external processor's default stride; no framebuffer is allocated for it. */ #define DCP_V14_EXT_STRIDE (1920 * 4) @@ -488,6 +490,15 @@ static void dcp_v14_external_published(struct apple_dcp_v14 *v14, const char *ke if (!strcmp(key, "TimingElements")) { if (removed) { dcp_v14_external_hotplug(dcp, false); + /* + * An HPD bounce withdraws the description, and the + * firmware describes the display again once its link is + * back. If that does not happen while the port still + * holds the display, redo the link. + */ + if (READ_ONCE(dcp->typec_cable_connected) && !READ_ONCE(dcp->typec_crtc_off)) + dcp_external_retry(dcp, "display withdrawn while its port has HPD", + 0, DCP_V14_DESC_RELINK_MS); return; } ret = parse(raw->data, raw->size, &ctx); @@ -1399,8 +1410,10 @@ static void dcpext_cancel(void *data) struct apple_dcp *dcp = data; cancel_work_sync(&dcp->external_work); - if (dcp->external_native) + if (dcp->external_native) { cancel_work_sync(&dcp->external_ready_work); + cancel_delayed_work_sync(&dcp->external_retry_wq); + } } /* The manual diagnostic path: memory checks, then an explicit start. */ @@ -1448,9 +1461,13 @@ int iomfb_v14_7_external_prepare(struct apple_dcp *dcp) int ret; atomic_set(&dcp->external_requested, 0); + atomic_set(&dcp->external_retries, 0); WRITE_ONCE(dcp->external_phase, DCPEXT_IDLE); INIT_WORK(&dcp->external_work, dcpext_bringup); INIT_WORK(&dcp->external_ready_work, dcpext_ready_work); + INIT_DELAYED_WORK(&dcp->external_retry_wq, dcp_external_retry_work); + /* Component bind enables it, once a connector can be retrained. */ + disable_delayed_work(&dcp->external_retry_wq); ret = devm_add_action_or_reset(dev, dcpext_cancel, dcp); if (ret) return ret; @@ -1525,6 +1542,12 @@ bool iomfb_v14_7_external_busy(struct apple_dcp *dcp) } } +bool iomfb_v14_7_external_failed(struct apple_dcp *dcp) +{ + return smp_load_acquire(&dcp->external_phase) == DCPEXT_FAILED || + (dcp->v14 && READ_ONCE(dcp->v14->failed)); +} + /* Refuse startup unless both the hardware floor and PMP vote are in place. */ static int dcpext_check_power(struct apple_dcp *dcp) { @@ -1843,8 +1866,10 @@ void iomfb_v14_7_remove(struct apple_dcp *dcp) /* Bring-up creates the external session; a late start must not. */ if (dcp->external) disable_work_sync(&dcp->external_work); - if (dcp->external_native) + if (dcp->external_native) { disable_work_sync(&dcp->external_ready_work); + disable_delayed_work_sync(&dcp->external_retry_wq); + } v14 = dcp->v14; if (!v14) return; @@ -2290,8 +2315,12 @@ static int dcp_v14_external_modeset(struct apple_dcp *dcp, struct drm_crtc_state out: dev_info(dcp->dev, "display mode " DRM_MODE_FMT " (timing %u, color %u): %d\n", DRM_MODE_ARG(&crtc_state->mode), mode.timing_mode_id, mode.color_mode_id, ret); - if (ret) + if (ret) { + /* A mode that is gone, or a description that never settled. */ + dcp_external_retry(dcp, "display mode not set", ret, 500); return ret; + } + atomic_set(&dcp->external_retries, 0); scoped_guard(spinlock_irqsave, &dcp->mode_state.lock) dcp->mode_state.valid = true; return 0; diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.h b/drivers/gpu/drm/apple/iomfb_v14_7.h index 50a7d1775389b..4e85c8a9b61d8 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7.h @@ -35,6 +35,8 @@ int iomfb_v14_7_external_prepare(struct apple_dcp *dcp); bool iomfb_v14_7_external_ready(struct apple_dcp *dcp); int iomfb_v14_7_external_open(struct apple_dcp *dcp); bool iomfb_v14_7_external_busy(struct apple_dcp *dcp); +/* The external session can no longer be used until reboot. */ +bool iomfb_v14_7_external_failed(struct apple_dcp *dcp); /* Component unbind: KMS goes away; the firmware session is kept until reboot. */ void iomfb_v14_7_unbind(struct apple_dcp *dcp); /* Platform remove: the apple_dcp is freed next; the firmware session is kept. */ From e25e802805e63150cafac8b1bedb14a3c2d953f5 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:20:23 -0700 Subject: [PATCH 1616/2680] drm/apple: t6030: don't warn when a DCP buffer misses the CMA area The 14.7 firmware asks the host for physically contiguous buffers, which the driver allocates with DMA_ATTR_FORCE_CONTIGUOUS. The DMA layer tries the CMA area first and falls back to the page allocator. On the M3 Pro the 64 MiB default CMA area is already in use before any display starts, so every external mode set logged a CMA allocation failure, although the buffer was then allocated. Ask for these buffers without the allocation warning, and log only a real failure. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit ceb082df5ff4fae3392b421443986049d883429e) --- drivers/gpu/drm/apple/iomfb_v14_7.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index d6605d232a873..b73cef2227eb7 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -270,9 +270,16 @@ static int dcp_v14_alloc(struct apple_dcp_v14 *v14, u64 request, u32 *id) if (v14->buffer_count == DCP_V14_MAX_BUFFERS || v14->buffer_bytes + size > DCP_V14_MAX_BUFFER_BYTES) return -ENOSPC; - cpu = dma_alloc_attrs(v14->dev, size, &iova, GFP_KERNEL, DMA_ATTR_FORCE_CONTIGUOUS); - if (!cpu) + /* + * Contiguous memory comes from the CMA area first and from the page + * allocator when that is in use: the CMA miss is not a failure. + */ + cpu = dma_alloc_attrs(v14->dev, size, &iova, GFP_KERNEL | __GFP_NOWARN, + DMA_ATTR_FORCE_CONTIGUOUS); + if (!cpu) { + dev_err(v14->dev, "no contiguous memory for a %zu byte DCP buffer\n", size); return -ENOMEM; + } ret = dma_get_sgtable_attrs(v14->dev, &table, cpu, iova, size, DMA_ATTR_FORCE_CONTIGUOUS); if (ret) From 8d428f083ffb2fded42019028275327965585486 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:15:37 -0700 Subject: [PATCH 1617/2680] drm/apple: t6030: pass a sink's IRQ_HPD to an external display processor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On M1/M2, IRQ_HPD from a Type-C display re-applies the active mode. That keeps the display the firmware described: a DP branch device that reports a new display behind it with IRQ_HPD, HPD staying high, keeps its old EDID and modes. On a T6030 external processor, hand the request to the firmware instead, through the DPTX port's sink interrupt call (the call the HDMI converter requests use), so that it services it as a DP source and describes the display again if it changed. If the call fails, release the display link and connect it again, which also makes the firmware read the display. With no link up, the request is dropped: the next connect reads the display anyway. Only native T6030 external pipes take the new branch; others retrain as before. [ryan: adapted to 7.2.10 — the port is checked with dptx_service_available(), as elsewhere in this tree] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit d06596d407500784a8fde5b73912032d10ecc70e) --- drivers/gpu/drm/apple/dcp-fabric.c | 9 ++++++++- drivers/gpu/drm/apple/dcp.c | 28 ++++++++++++++++++++++++++++ drivers/gpu/drm/apple/dcp.h | 1 + drivers/gpu/drm/apple/dptxep.c | 6 ++++++ drivers/gpu/drm/apple/dptxep.h | 1 + 5 files changed, 44 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 4220f0da427dd..017de7b124e6f 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -1322,7 +1322,14 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, } else if (hpd && dp_data && (dp_data->status & DP_STATUS_IRQ_HPD)) { struct apple_dcp *dcp = port->owner->dcp; - if (dcp->typec_connector) + /* + * A T6030 external processor services the request itself, and + * reads the display again if it changed; elsewhere the active + * mode is applied again. + */ + if (dcp->typec_connector && dcp->external_native) + dcp_external_sink_irq(dcp); + else if (dcp->typec_connector) dcp_retrain_oob(dcp->typec_connector); } port->hpd = hpd; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 54d98adaed372..16484b718e016 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1345,6 +1345,34 @@ void dcp_external_retry_work(struct work_struct *work) mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); } +/* + * The sink of a native external pipe raised IRQ_HPD with HPD staying high, + * as a DP branch device does when the display behind it changes. Pass the + * request to the firmware, which services it as a DP source does. If it + * cannot take the request, connect the display link anew, which makes the + * firmware read the display again. + */ +void dcp_external_sink_irq(struct apple_dcp *dcp) +{ + struct platform_device *pdev = to_platform_device(dcp->dev); + struct apple_epic_service *service = READ_ONCE(dcp->dptxport[0].service); + int ret; + + /* With no link up, the next connect reads the display anyway. */ + if (!dcp->external_native || !service || !dptx_service_available(&dcp->dptxport[0]) || + !READ_ONCE(dcp->dptxport[0].connected)) + return; + ret = dptxport_sink_irq(service); + if (!ret) { + dev_info(dcp->dev, "display sink IRQ_HPD passed to the firmware\n"); + return; + } + dev_info(dcp->dev, "display sink IRQ_HPD not passed (%d): connecting the display anew\n", + ret); + dcp_dptx_disconnect_oob(pdev, 0); + dcp_dptx_connect_oob(pdev, 0); +} + int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) { if (!dcp_is_external_h17p(dcp)) diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 2e3832ff0a17c..e4ea03a45611f 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -73,6 +73,7 @@ void dcp_external_ready(struct apple_dcp *dcp); void dcp_external_retry(struct apple_dcp *dcp, const char *why, int error, unsigned int base_ms); void dcp_external_retry_work(struct work_struct *work); +void dcp_external_sink_irq(struct apple_dcp *dcp); int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); bool dcp_hold_resume_event(struct platform_device *pdev); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 207f10e6400fd..8c2eb535d6b5a 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -227,6 +227,12 @@ int dptxport_set_hpd(struct apple_epic_service *service, bool hpd) return dptxport_set_hpd_timeout(service, hpd, MSEC_PER_SEC); } +/* The sink raised IRQ_HPD: the firmware services the request as a DP source. */ +int dptxport_sink_irq(struct apple_epic_service *service) +{ + return afk_service_call(service, 8, 9, NULL, 0, 16, NULL, 0, 16); +} + static int dptxport_call_get_max_drive_settings(struct apple_epic_service *service, void *reply_, size_t reply_size) diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index 74f6be07528a2..449542f2accbd 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -74,6 +74,7 @@ int dptxport_release_display(struct apple_epic_service *service); int dptxport_set_hpd(struct apple_epic_service *service, bool hpd); int dptxport_set_hpd_timeout(struct apple_epic_service *service, bool hpd, unsigned int timeout_ms); +int dptxport_sink_irq(struct apple_epic_service *service); struct apple_dcp; int dpav_ctrl_init(struct apple_dcp *dcp); From bfe9aa9a1bb73d9299923cd84615cd3cdc275295 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:16:20 -0700 Subject: [PATCH 1618/2680] drm/apple: t6030: take a direct crossbar output down with its display A direct (non-tunnel) crossbar output on T6030 is clocked while a link runs, and taken down when the firmware changes or stops the link, or deactivates the port. If the firmware never gets that far, for example when releasing the display times out, the output stays clocked until the route is deselected. Take it down also when the kernel releases the display. It is a no-op when the output is already down, and the next link configuration brings it up again, so a display that is attached again is unaffected. Native T6030 external pipes only. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 48fba70ec317c93cba30968da8a8e0e1ffb2971f) --- drivers/gpu/drm/apple/dcp.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 16484b718e016..a5fe4e34fcf42 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1742,6 +1742,13 @@ int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) dptxport_release_display(dcp->dptxport[port].service); dcp->dptxport[port].connected = false; } + /* + * T6030: the firmware takes a direct crossbar output down when it + * stops the link. Make sure of it for a display released here, also + * when the firmware did not get that far. + */ + if (dcp->external_native) + dcp_direct_crossbar_link(dcp, false); mutex_unlock(&dcp->hpd_mutex); return 0; From a290116d2db76297b23c0e1dd55066e5cf9806fb Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:17:52 -0700 Subject: [PATCH 1619/2680] drm/apple: t6030: don't set an external mode again that is already showing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After a display comes back from an HPD bounce, the bounded retry and the hotplug work each replay the CRTC, and every replay powered the display down, sent the same timing and color ids (A411) and powered it up again. The firmware warns "set_timings() called after timings were started" on each, seven times in one test run (seen on hw). Track the mode the firmware runs: set once A411 and power-up succeed, cleared when the display is powered down, the session fails, or the timings or color modes are published or withdrawn again (the ids refer to the description they came from), and count the swaps with a framebuffer that complete in it, reset by a refused swap. - A mode set for the ids already running, with swaps completing since, is not sent again: it is logged and the mode is marked valid. - The bounded retry checks the same for the active CRTC's mode first, and then only marks the mode valid, with no replay at all. Native T6030 external pipes only. [ryan: adapted to 7.2.10 — iomfb-state.h gains the dcp_mode_set_valid() helper the callers use; this tree had open-coded it under mode_state.lock] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit c3af53a84dfdca4d6d49fcfbf9e2fc022de69dc0) --- drivers/gpu/drm/apple/dcp.c | 20 ++++++++++ drivers/gpu/drm/apple/iomfb-state.h | 10 +++++ drivers/gpu/drm/apple/iomfb_v14_7.c | 61 +++++++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_v14_7.h | 3 ++ 4 files changed, 94 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a5fe4e34fcf42..0d163cd5eb4f0 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1301,6 +1301,21 @@ void dcp_external_retry(struct apple_dcp *dcp, const char *why, int error, mod_delayed_work(system_freezable_wq, &dcp->external_retry_wq, msecs_to_jiffies(delay)); } +/* The active CRTC's mode is the one the firmware runs and shows swaps in. */ +static bool dcp_external_crtc_showing(struct apple_dcp *dcp) +{ + struct drm_crtc *crtc = dcp->crtc ? &dcp->crtc->base : NULL; + bool showing = false; + + if (!crtc) + return false; + drm_modeset_lock(&crtc->mutex, NULL); + if (crtc->state && crtc->state->active) + showing = iomfb_v14_7_external_showing(dcp, &crtc->state->mode); + drm_modeset_unlock(&crtc->mutex); + return showing; +} + void dcp_external_retry_work(struct work_struct *work) { struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, @@ -1317,6 +1332,11 @@ void dcp_external_retry_work(struct work_struct *work) dev_info(dcp->dev, "display retry: the display is back and set\n"); return; } + if (dcp_external_crtc_showing(dcp)) { + dev_info(dcp->dev, "display retry: the display is back in its mode and showing\n"); + dcp_mode_set_valid(&dcp->mode_state, true); + return; + } dev_info(dcp->dev, "display retry: setting the display mode again\n"); dcp_mode_invalidate(&dcp->mode_state); schedule_work(&connector->hotplug_wq); diff --git a/drivers/gpu/drm/apple/iomfb-state.h b/drivers/gpu/drm/apple/iomfb-state.h index 4cf48977ffd74..f6d512f3c4f43 100644 --- a/drivers/gpu/drm/apple/iomfb-state.h +++ b/drivers/gpu/drm/apple/iomfb-state.h @@ -32,6 +32,16 @@ static inline void dcp_mode_begin(struct dcp_mode_state *state) spin_unlock_irqrestore(&state->lock, flags); } +/* For a pipe whose firmware keeps its own mode state (T6030 14.7). */ +static inline void dcp_mode_set_valid(struct dcp_mode_state *state, bool valid) +{ + unsigned long flags; + + spin_lock_irqsave(&state->lock, flags); + state->valid = valid; + spin_unlock_irqrestore(&state->lock, flags); +} + /* * The mode is gone, also for a mode set in flight, which then does not * complete it. Returns whether a set mode was valid until now. diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index b73cef2227eb7..0ef09d38265dd 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -125,6 +125,14 @@ struct apple_dcp_v14 { bool opened; /* External: the display is powered (A472) in the mode last set. */ bool powered; + /* + * External: the mode last set (A411 then A472(1)) runs, from a + * description not changed since, with these ids; and swaps with a + * framebuffer completed since it was set. + */ + bool mode_live; + u32 mode_timing, mode_color; + u64 mode_swaps; /* External: catalog generation when a chunked property started. */ u64 chunk_generation; /* External: the parts of the display's description published now. */ @@ -475,6 +483,9 @@ static void dcp_v14_external_published(struct apple_dcp_v14 *v14, const char *ke v14->described &= ~part; else if (part != DCP_V14_DESC_TIMING) v14->described |= part; + /* Mode ids refer to the timings and colors described when it was set. */ + if (part == DCP_V14_DESC_TIMING || part == DCP_V14_DESC_COLOR) + v14->mode_live = false; /* A mode set waits for the whole description; see external_settle(). */ wake_up_all(&v14->described_wait); if (!dcp) @@ -2112,6 +2123,7 @@ static bool dcp_v14_present(struct apple_dcp_v14 *v14, struct drm_framebuffer *f /* Refused before it was taken: the old framebuffer stays on screen. */ if (fb) drm_framebuffer_put(fb); + WRITE_ONCE(v14->mode_swaps, 0); dev_warn_ratelimited(v14->dev, "display refused a swap: %d\n", ret); return false; } @@ -2124,6 +2136,8 @@ static bool dcp_v14_present(struct apple_dcp_v14 *v14, struct drm_framebuffer *f old = v14->active_fb; v14->active_fb = fb; swaps = ++v14->swaps; + if (fb) + v14->mode_swaps++; v14->swap_ns_max = max(v14->swap_ns_max, elapsed); mutex_unlock(&v14->lock); if (old) @@ -2228,6 +2242,8 @@ static int dcp_v14_set_power(struct apple_dcp_v14 *v14, bool on) /* A refused power-down leaves nothing on: the display is gone already. */ if (!ret || (!on && !READ_ONCE(v14->failed))) v14->powered = on; + if (!on) + v14->mode_live = false; return ret; } @@ -2288,6 +2304,34 @@ static int dcp_v14_external_settle(struct apple_dcp *dcp, struct apple_dcp_v14 * return -EAGAIN; } +/* Called with the lock held. */ +static bool dcp_v14_external_mode_live(struct apple_dcp_v14 *v14, + const struct dcp_display_mode *mode) +{ + return !v14->failed && v14->powered && v14->mode_live && + v14->mode_timing == mode->timing_mode_id && + v14->mode_color == mode->color_mode_id && READ_ONCE(v14->mode_swaps); +} + +/* + * Whether the firmware runs @drm_mode, set last, from the description it + * has now, and shows swaps in it. + */ +bool iomfb_v14_7_external_showing(struct apple_dcp *dcp, + const struct drm_display_mode *drm_mode) +{ + struct apple_dcp_v14 *v14 = dcp->v14; + struct dcp_display_mode mode = {}; + bool live; + + if (!dcp->external_native || !v14 || !lookup_mode(dcp, drm_mode, &mode)) + return false; + mutex_lock(&v14->lock); + live = dcp_v14_external_described(v14) && dcp_v14_external_mode_live(v14, &mode); + mutex_unlock(&v14->lock); + return live; +} + /* An external display: power down, set the new mode, power up. */ static int dcp_v14_external_modeset(struct apple_dcp *dcp, struct drm_crtc_state *crtc_state) { @@ -2306,6 +2350,19 @@ static int dcp_v14_external_modeset(struct apple_dcp *dcp, struct drm_crtc_state ret = -EINVAL; goto out; } + /* + * The firmware already runs this mode and shows swaps in it: setting + * it again only restarts its timings under the display. A retry or a + * replayed CRTC lands here when the display came back as it was. + */ + if (dcp_v14_external_mode_live(v14, &mode)) { + mutex_unlock(&v14->lock); + dev_info(dcp->dev, "display mode " DRM_MODE_FMT " (timing %u, color %u) already active and showing\n", + DRM_MODE_ARG(&crtc_state->mode), mode.timing_mode_id, mode.color_mode_id); + atomic_set(&dcp->external_retries, 0); + dcp_mode_set_valid(&dcp->mode_state, true); + return 0; + } if (v14->powered && dcp_v14_set_power(v14, false)) dev_warn(dcp->dev, "display did not power down before the mode change\n"); if (READ_ONCE(v14->failed)) @@ -2317,6 +2374,10 @@ static int dcp_v14_external_modeset(struct apple_dcp *dcp, struct drm_crtc_state if (!ret) { v14->panel_width = mode.mode.hdisplay; v14->panel_height = mode.mode.vdisplay; + v14->mode_timing = mode.timing_mode_id; + v14->mode_color = mode.color_mode_id; + v14->mode_swaps = 0; + v14->mode_live = true; } mutex_unlock(&v14->lock); out: diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.h b/drivers/gpu/drm/apple/iomfb_v14_7.h index 4e85c8a9b61d8..c0eebf5704027 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7.h @@ -37,6 +37,9 @@ int iomfb_v14_7_external_open(struct apple_dcp *dcp); bool iomfb_v14_7_external_busy(struct apple_dcp *dcp); /* The external session can no longer be used until reboot. */ bool iomfb_v14_7_external_failed(struct apple_dcp *dcp); +/* The firmware runs this mode, set last, and shows swaps in it. */ +bool iomfb_v14_7_external_showing(struct apple_dcp *dcp, + const struct drm_display_mode *mode); /* Component unbind: KMS goes away; the firmware session is kept until reboot. */ void iomfb_v14_7_unbind(struct apple_dcp *dcp); /* Platform remove: the apple_dcp is freed next; the firmware session is kept. */ From 35b38ca36a29e3720e11a36eeac45a54be1d51be Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:19:09 -0700 Subject: [PATCH 1620/2680] drm/apple: t6030: set the HDMI output up again after system sleep After an s2idle cycle with nothing attached, the HDMI port of the M3 Pro stays dark until reboot (seen on hw): the converter's HPD arrives, the connect waits for a link configuration that never comes, the firmware reports link faults on target 0:3, and replugging the cable fails the same way. USB-C displays on the other external pipe work after the same resume. The PHY that runs the HDMI output as four-lane DP sits in a power domain no device keeps on across sleep, and probe is the only place that puts it, and the crossbar output in front of it, into that state. On resume, for a T6030 external pipe with an HDMI output, redo that setup: drive the converter power GPIOs on, take the PHY out of DP and back into four-lane DP through its mux, and select the crossbar output again if the HDMI route holds it (the mux core skips a select of the state it has cached, so deselect first). No HDMI link is up then: suspend releases it. M1/M2 HDMI ports are unchanged. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 2e86369c545c3adf9a35c7ef22a507f2cc6ea49f) --- drivers/gpu/drm/apple/dcp-fabric.c | 51 ++++++++++++++++++++++++++++++ drivers/gpu/drm/apple/dcp-fabric.h | 1 + drivers/gpu/drm/apple/dcp.c | 3 ++ 3 files changed, 55 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 017de7b124e6f..55a003daada3f 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -680,6 +680,57 @@ void dcp_fabric_hdmi_resume(struct apple_dcp *dcp) dcp_fabric_run_resume_sample(true, &dcp_resume_sample_ops, dcp); } +/* + * T6030 HDMI output: the ATC PHY that runs it as four-lane DP, and the + * crossbar it goes through, are in power domains that system sleep may + * switch off, losing their setup, and the converter is powered through + * GPIOs. Set them up again as probe did: converter power on, the PHY taken + * out of DP and back into it, and the crossbar output selected again if the + * HDMI route holds it. No HDMI link is up when this runs. + */ +static void dcp_fixed_hdmi_reinit_locked(struct apple_dcp *dcp, const char *why) +{ + int ret = 0; + + lockdep_assert_held(&dcp_typec_fabric_lock); + if (!dcp->external_native || !dcp->fixed_phy || !dcp->hdmi_hpd) + return; + dev_info(dcp->dev, "HDMI: setting up the output again (%s)\n", why); + if (dcp->hdmi_pwren) + gpiod_set_value_cansleep(dcp->hdmi_pwren, 1); + if (dcp->dp2hdmi_pwren) + gpiod_set_value_cansleep(dcp->dp2hdmi_pwren, 1); + if (dcp->typec_mux && dcp->phy_managed_by_typec) { + struct typec_altmode alt = { .svid = USB_TYPEC_DP_SID }; + struct typec_mux_state state = { .mode = TYPEC_STATE_SAFE }; + + ret = typec_mux_set(dcp->typec_mux, &state); + if (!ret) { + state.alt = &alt; + state.mode = TYPEC_DP_STATE_C; + ret = typec_mux_set(dcp->typec_mux, &state); + } + if (ret) + dev_warn(dcp->dev, "HDMI: PHY not set up again: %d\n", ret); + } + if (dcp->xbar && dcp->fixed_route_selected && !dcp->active_typec_route) { + scoped_guard(mutex, &dcp->tb_lock) + dcp->direct_xbar_up = NULL; + ret = mux_control_deselect(dcp->xbar); + dcp->fixed_route_selected = false; + if (!ret) + ret = dcp_fixed_output_select(dcp); + if (ret) + dev_warn(dcp->dev, "HDMI: crossbar output not selected again: %d\n", ret); + } +} + +void dcp_fabric_hdmi_reinit(struct apple_dcp *dcp, const char *why) +{ + guard(mutex)(&dcp_typec_fabric_lock); + dcp_fixed_hdmi_reinit_locked(dcp, why); +} + /* A Thunderbolt tunnel holds @dcp's pipeline: it never moves. */ static bool dcp_typec_tunnel_held(struct apple_dcp *dcp) { diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index 3a3652d5a1940..d8277ab293f54 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -82,6 +82,7 @@ int dcp_register_typec_routes(struct apple_dcp *dcp); void dcp_typec_retrain_work(struct work_struct *work); void dcp_fabric_init(struct apple_dcp *dcp); void dcp_fabric_hdmi_resume(struct apple_dcp *dcp); +void dcp_fabric_hdmi_reinit(struct apple_dcp *dcp, const char *why); irqreturn_t dcp_dp2hdmi_hpd_edge(int irq, void *data); irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data); void dcp_fabric_shutdown_dptx(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 0d163cd5eb4f0..9e3432e3a6bdc 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3817,6 +3817,9 @@ static int dcp_platform_resume(struct device *dev) dcp_release_after_sleep(dcp); dcp_enable_typec_work(dcp); + /* The HDMI output's PHY and crossbar may have lost their setup in sleep. */ + if (dcp->external_native) + dcp_fabric_hdmi_reinit(dcp, "after system sleep"); /* Observe future edges before sampling any edges lost in sleep. */ if (dcp->hdmi_hpd_irq) dcp_fabric_run_resume(&dcp_resume_ops, dcp); From 98209262f42fe76e727b5bca9f8f317fcd2d635b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:19:45 -0700 Subject: [PATCH 1621/2680] drm/apple: t6030: retry an HDMI display whose link did not come up When the HDMI connect timed out waiting for the link configuration, the port was left as it was: nothing tried again until the next HPD edge, which failed the same way (seen on hw after resume). Give the HDMI output of a native external pipe the bounded retry USB-C displays have: 3 tries per attached display, 1, 2 and 4 s apart, after a failed connect from an HPD edge or from the first start of the processor. A retry checks that HDMI still has a display and that no USB-C display borrowed the pipe, releases a half-made connection, sets the output up again (converter power, PHY, crossbar, as after sleep), and connects again. A new HDMI attach resets the count, as does a mode set that succeeds. M1/M2 HDMI ports are unchanged. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 56c5197e6585d0cee7b831f7f83554589880e1bd) --- drivers/gpu/drm/apple/dcp-fabric.c | 38 +++++++++++++++++++++++++++++- drivers/gpu/drm/apple/dcp-fabric.h | 1 + drivers/gpu/drm/apple/dcp.c | 10 +++++++- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 55a003daada3f..303e7db58eb1a 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -2397,15 +2397,51 @@ static void dcp_hdmi_observed(void *ctx, bool high, bool debounced) dev_info(dcp->dev, "DP2HDMI HPD irq, connected:%d\n", high); } +/* A failed HDMI connect that a retry may fix; see dcp_fabric_hdmi_retry(). */ +static void dcp_hdmi_connect_failed(struct apple_dcp *dcp, int ret) +{ + if (dcp->external_native && ret && ret != -EAGAIN && ret != -ESTALE && + ret != -ESHUTDOWN) + dcp_external_retry(dcp, "HDMI display link not set up", ret, 1000); +} + static void dcp_hdmi_connect_fixed(void *ctx) { struct apple_dcp *dcp = ctx; int ret = dcp_fixed_output_select(dcp); + /* A new HDMI attach gets its own retries. */ + if (dcp->external_native) + atomic_set(&dcp->external_retries, 0); if (ret) dev_err(dcp->dev, "could not select the HDMI output: %d\n", ret); else - dcp_dptx_connect(dcp, 0); + ret = dcp_dptx_connect(dcp, 0); + dcp_hdmi_connect_failed(dcp, ret); +} + +/* + * The bounded retry of a native external pipe, for its HDMI output: the + * display is attached, but its link did not come up. Set the output up + * again, in case the PHY or crossbar lost their setup, and connect again. + */ +void dcp_fabric_hdmi_retry(struct apple_dcp *dcp) +{ + int ret; + + guard(mutex)(&dcp_typec_fabric_lock); + if (dcp->active_typec_route || gpiod_get_value_cansleep(dcp->hdmi_hpd) <= 0) { + dev_info(dcp->dev, "display retry: the HDMI port has no display any more\n"); + return; + } + dev_info(dcp->dev, "display retry: connecting the HDMI display again\n"); + /* Releases a half-made connection; a no-op otherwise. */ + dcp_dptx_disconnect(dcp, 0); + dcp_fixed_hdmi_reinit_locked(dcp, "its display link did not come up"); + ret = dcp_fixed_output_select(dcp); + if (!ret) + ret = dcp_dptx_connect(dcp, 0); + dcp_hdmi_connect_failed(dcp, ret); } static const struct dcp_fabric_hdmi_ops dcp_hdmi_ops = { diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index d8277ab293f54..b12f01e369e10 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -83,6 +83,7 @@ void dcp_typec_retrain_work(struct work_struct *work); void dcp_fabric_init(struct apple_dcp *dcp); void dcp_fabric_hdmi_resume(struct apple_dcp *dcp); void dcp_fabric_hdmi_reinit(struct apple_dcp *dcp, const char *why); +void dcp_fabric_hdmi_retry(struct apple_dcp *dcp); irqreturn_t dcp_dp2hdmi_hpd_edge(int irq, void *data); irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data); void dcp_fabric_shutdown_dptx(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 9e3432e3a6bdc..adebe423f47cb 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1260,7 +1260,10 @@ void dcp_external_ready(struct apple_dcp *dcp) mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); } else if (dcp->hdmi_hpd && dcp->active && gpiod_get_value_cansleep(dcp->hdmi_hpd)) { - dcp_dptx_connect(dcp, 0); + int ret = dcp_dptx_connect(dcp, 0); + + if (ret && ret != -EAGAIN && ret != -ESTALE && ret != -ESHUTDOWN) + dcp_external_retry(dcp, "HDMI display link not set up", ret, 1000); } } @@ -1342,6 +1345,11 @@ void dcp_external_retry_work(struct work_struct *work) schedule_work(&connector->hotplug_wq); return; } + /* An HDMI output connects through the fabric, as on its HPD. */ + if (!dcp_is_typec_output(dcp) && dcp->hdmi_hpd) { + dcp_fabric_hdmi_retry(dcp); + return; + } if (!READ_ONCE(dcp->typec_cable_connected)) { dev_info(dcp->dev, "display retry: the port has no display any more\n"); return; From eb7f0ada257814fac1c260eb51951796a037f226 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:20:56 -0700 Subject: [PATCH 1622/2680] drm/apple: t6030: refuse sleep while an external display is attached A native external processor refused system sleep only while its firmware held something: a start, a display link, or a powered display. Powering the pipe off (DPMS off, which a desktop typically does before it suspends) clears all three, as does a connect that failed, so sleep went ahead with a USB-C display still plugged in, into a resume path that does not bring an attached display back. Refuse sleep also while a display is attached to the processor at all: - a USB-C route is allocated to it, or its port has HPD; - its HDMI port has HPD. Each refusal now logs why ("refusing system sleep: a USB-C display is attached", "...: a display link is up", "...: its firmware is starting"). Sleep stays allowed with nothing attached, as before. The manual external path and the panel are unchanged. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit c66101b012f4be0ac10a736892fb1c617f132bf5) --- drivers/gpu/drm/apple/dcp.c | 43 ++++++++++++++++++++--------- drivers/gpu/drm/apple/iomfb_v14_7.c | 21 +++++++------- drivers/gpu/drm/apple/iomfb_v14_7.h | 4 +-- 3 files changed, 43 insertions(+), 25 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index adebe423f47cb..0173d380c9712 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3702,19 +3702,31 @@ static void dcp_release_after_sleep(struct apple_dcp *dcp) } /* - * Whether an external processor must keep the system awake. The manual path - * refuses sleep once its firmware was ever started. A native processor - * refuses only while it starts or drives a display: idle, its firmware stays + * Why an external processor must keep the system awake, or NULL. The manual + * path refuses sleep once its firmware was ever started. A native processor + * refuses while its firmware starts or holds a display link, and while a + * display is attached to it at all, lit or not: resume does not bring an + * attached display back. Idle with nothing attached, its firmware stays * powered across s2idle like the panel's (the gate holds its CPU domain at * the active floor) and the next attach reconnects through it. */ -static bool dcp_external_sleep_vetoed(struct apple_dcp *dcp) +static const char *dcp_external_sleep_vetoed(struct apple_dcp *dcp) { - if (dcp->external_native) - return iomfb_v14_7_external_busy(dcp); - return atomic_read(&dcp->external_requested) || - (dcp->rtk && apple_rtkit_is_running(dcp->rtk)) || - dcpext_scanout_requested(dcp); + const char *why; + + if (!dcp->external_native) + return (atomic_read(&dcp->external_requested) || + (dcp->rtk && apple_rtkit_is_running(dcp->rtk)) || + dcpext_scanout_requested(dcp)) ? + "external firmware/scanout attempted" : NULL; + why = iomfb_v14_7_external_busy(dcp); + if (why) + return why; + if (READ_ONCE(dcp->active_typec_route) || READ_ONCE(dcp->typec_cable_connected)) + return "a USB-C display is attached"; + if (dcp->hdmi_hpd && gpiod_get_value_cansleep(dcp->hdmi_hpd) > 0) + return "an HDMI display is attached"; + return NULL; } /* dpm_prepare completes for every device before any dpm_suspend callback. @@ -3724,14 +3736,16 @@ static bool dcp_external_sleep_vetoed(struct apple_dcp *dcp) static int dcp_platform_prepare(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); + const char *why; if (!dcp->external) return 0; mutex_lock(&dcp->hpd_mutex); - if (dcp_external_sleep_vetoed(dcp)) { + why = dcp_external_sleep_vetoed(dcp); + if (why) { mutex_unlock(&dcp->hpd_mutex); if (dcp->external_native) - dev_warn(dev, "external display attached or starting: refusing system sleep\n"); + dev_warn(dev, "refusing system sleep: %s\n", why); else dev_warn(dev, "external firmware/scanout attempted: refusing PM prepare; reboot required for retained DMA\n"); return -EBUSY; @@ -3761,11 +3775,14 @@ static int dcp_platform_suspend(struct device *dev) * attempt we cannot prove DMA/firmware quiescence, even on failure. */ if (dcp->external) { + const char *why; + mutex_lock(&dcp->hpd_mutex); - if (dcp_external_sleep_vetoed(dcp)) { + why = dcp_external_sleep_vetoed(dcp); + if (why) { mutex_unlock(&dcp->hpd_mutex); if (dcp->external_native) - dev_warn(dev, "external display attached or starting: suspend refused\n"); + dev_warn(dev, "suspend refused: %s\n", why); else dev_warn(dev, "external firmware/scanout attempted: suspend refused; retained DMA requires reboot\n"); return -EBUSY; diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 0ef09d38265dd..aa99df5b099ce 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -1541,22 +1541,23 @@ bool iomfb_v14_7_external_ready(struct apple_dcp *dcp) } /* - * System sleep with this processor: allowed before its first start and - * while it idles with no display link; refused while it starts or holds a - * link or a powered display, as those are not handed back across sleep. A - * route alone (an adapter in DP mode with no display) holds nothing. + * What this processor's firmware holds that system sleep would not hand + * back, or NULL: a start in progress, or a display link or a powered + * display. The caller also refuses sleep while a display is attached. */ -bool iomfb_v14_7_external_busy(struct apple_dcp *dcp) +const char *iomfb_v14_7_external_busy(struct apple_dcp *dcp) { switch (smp_load_acquire(&dcp->external_phase)) { case DCPEXT_STARTING: - return true; + return "its firmware is starting"; case DCPEXT_RUNNING: - return READ_ONCE(dcp->dptxport[0].connected) || - READ_ONCE(dcp->external_link_ready) || - (dcp->v14 && READ_ONCE(dcp->v14->powered)); + if (READ_ONCE(dcp->dptxport[0].connected) || + READ_ONCE(dcp->external_link_ready) || + (dcp->v14 && READ_ONCE(dcp->v14->powered))) + return "a display link is up"; + return NULL; default: - return false; + return NULL; } } diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.h b/drivers/gpu/drm/apple/iomfb_v14_7.h index c0eebf5704027..109acf42696c8 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7.h @@ -29,12 +29,12 @@ int iomfb_v14_7_external_start(struct apple_dcp *dcp); * Native external processor: probe-time checks, then a firmware start on * first use. _ready() is true once DPTX can be used, and requests the start * if nothing has yet; _open() opens the display interface after the first - * display is attached; _busy() vetoes system sleep. + * display is attached; _busy() says why system sleep must wait, or NULL. */ int iomfb_v14_7_external_prepare(struct apple_dcp *dcp); bool iomfb_v14_7_external_ready(struct apple_dcp *dcp); int iomfb_v14_7_external_open(struct apple_dcp *dcp); -bool iomfb_v14_7_external_busy(struct apple_dcp *dcp); +const char *iomfb_v14_7_external_busy(struct apple_dcp *dcp); /* The external session can no longer be used until reboot. */ bool iomfb_v14_7_external_failed(struct apple_dcp *dcp); /* The firmware runs this mode, set last, and shows swaps in it. */ From 50471a921b1bf67daea94039041a3d8f82bacbb6 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:21:33 -0700 Subject: [PATCH 1623/2680] drm/apple: t6030: one sleep policy for a failed external processor The sleep veto treated failures two ways. A start that failed after the processor's CPU was started allowed sleep. A session that stopped while the display was powered kept refusing sleep until reboot, also after the display was unplugged, as its last power state could never be cleared. A failed processor (failed start, or a stopped session) drives no display until reboot, and its power domains stay on across s2idle as those of a working one do. Let it never refuse sleep, attached display or not, whatever its session held last, and log that sleep goes ahead with it as it is. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit c2ea973baa76f7f010072fd506f8a16e03fdad75) --- drivers/gpu/drm/apple/dcp.c | 9 +++++++++ drivers/gpu/drm/apple/iomfb_v14_7.c | 3 +++ 2 files changed, 12 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 0173d380c9712..cedd780430d4f 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3709,6 +3709,10 @@ static void dcp_release_after_sleep(struct apple_dcp *dcp) * attached display back. Idle with nothing attached, its firmware stays * powered across s2idle like the panel's (the gate holds its CPU domain at * the active floor) and the next attach reconnects through it. + * + * A processor that failed (its start, or its session later) never refuses + * sleep: it drives no display until reboot, attached or not, and its power + * domains stay on across s2idle as they are, like those of a working one. */ static const char *dcp_external_sleep_vetoed(struct apple_dcp *dcp) { @@ -3722,6 +3726,11 @@ static const char *dcp_external_sleep_vetoed(struct apple_dcp *dcp) why = iomfb_v14_7_external_busy(dcp); if (why) return why; + if (iomfb_v14_7_external_failed(dcp)) { + if (dcp->v14) + dev_info(dcp->dev, "external display processor failed earlier: sleeping with it as it is\n"); + return NULL; + } if (READ_ONCE(dcp->active_typec_route) || READ_ONCE(dcp->typec_cable_connected)) return "a USB-C display is attached"; if (dcp->hdmi_hpd && gpiod_get_value_cansleep(dcp->hdmi_hpd) > 0) diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index aa99df5b099ce..e316b1e316596 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -1551,6 +1551,9 @@ const char *iomfb_v14_7_external_busy(struct apple_dcp *dcp) case DCPEXT_STARTING: return "its firmware is starting"; case DCPEXT_RUNNING: + /* A stopped session drives nothing, whatever it held last. */ + if (dcp->v14 && READ_ONCE(dcp->v14->failed)) + return NULL; if (READ_ONCE(dcp->dptxport[0].connected) || READ_ONCE(dcp->external_link_ready) || (dcp->v14 && READ_ONCE(dcp->v14->powered))) From 82a1665c0e9cf203fa0015be80dcf9b33d777ebc Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:22:23 -0700 Subject: [PATCH 1624/2680] drm/apple: t6030: request an external start under the sleep veto's lock The first attach requests the external processor's start by checking that system sleep is not under way, then moving the processor from idle to starting and queuing the start work. That ran outside the lock that sleep's prepare and suspend hold while they check the processor, and the work ran on a workqueue that is not frozen. A connect racing sleep entry could pass the check, let prepare and suspend see an idle processor, and then start its firmware during or after device suspend. Make the check and the transition under the HPD lock prepare and suspend take, so that a start is either requested before they look (and refuses sleep, as a start in progress does) or not before complete. Queue the start work on the freezable workqueue, and have it check again before it starts anything: if sleep began, it returns the processor to idle, and the next attach starts it. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit b60de54b643c36cf365c60c94de45d461c391986) --- drivers/gpu/drm/apple/iomfb_v14_7.c | 44 ++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index e316b1e316596..b24e55a4d5b89 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -1524,15 +1524,23 @@ bool iomfb_v14_7_external_ready(struct apple_dcp *dcp) case DCPEXT_RUNNING: return smp_load_acquire(&dcp->dptxport[0].enabled); case DCPEXT_IDLE: - /* Not between sleep's prepare and complete; the next attach starts it. */ - if (READ_ONCE(dcp->external_suspended)) - return false; - if (!atomic_cmpxchg(&dcp->external_requested, 0, 1)) { - WRITE_ONCE(dcp->external_phase, DCPEXT_STARTING); - dev_info(dcp->dev, "display attached: starting the external display processor\n"); - /* Firmware callbacks may outlive any later module removal. */ - __module_get(THIS_MODULE); - queue_work(system_unbound_wq, &dcp->external_work); + /* + * Under the lock system sleep's prepare and suspend check the + * phase under: a start is requested before they look, and then + * refuses sleep, or not between them and complete, when the + * next attach starts it. The start work is frozen with tasks. + */ + scoped_guard(mutex, &dcp->hpd_mutex) { + if (READ_ONCE(dcp->external_suspended) || + smp_load_acquire(&dcp->external_phase) != DCPEXT_IDLE) + return false; + if (!atomic_cmpxchg(&dcp->external_requested, 0, 1)) { + WRITE_ONCE(dcp->external_phase, DCPEXT_STARTING); + dev_info(dcp->dev, "display attached: starting the external display processor\n"); + /* Firmware callbacks may outlive any later module removal. */ + __module_get(THIS_MODULE); + queue_work(system_freezable_wq, &dcp->external_work); + } } return false; default: @@ -1736,11 +1744,27 @@ static int dcpext_boot(struct apple_dcp *dcp) return 0; } +/* Native: system sleep began after the start was requested; undo the request. */ +static bool dcpext_start_deferred(struct apple_dcp *dcp) +{ + guard(mutex)(&dcp->hpd_mutex); + if (!READ_ONCE(dcp->external_suspended)) + return false; + WRITE_ONCE(dcp->external_phase, DCPEXT_IDLE); + atomic_set(&dcp->external_requested, 0); + dev_info(dcp->dev, "system sleep began: the external display processor starts on the next attach\n"); + module_put(THIS_MODULE); + return true; +} + static void dcpext_bringup(struct work_struct *work) { struct apple_dcp *dcp = container_of(work, struct apple_dcp, external_work); - int ret = dcpext_boot(dcp); + int ret; + if (dcp->external_native && dcpext_start_deferred(dcp)) + return; + ret = dcpext_boot(dcp); if (!dcp->external_native) return; /* Pairs with external_ready(): the endpoints are published first. */ From faf7db935df45a57eded5315503926927c24c8c1 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:23:17 -0700 Subject: [PATCH 1625/2680] drm/apple: t6030: report an external display gone when its session stops When a DCPLink call or notification of an external session fails (not a crash), the session stops for good, but nothing told userspace: the connector stayed connected and the CRTC active, and every commit that included that CRTC failed its check with -EIO. A desktop that commits all its outputs together could then not reconfigure the internal panel either. When an external session stops on a link error, or its display interface does not open, report the display gone as the crash path does: mark the connector disconnected, its EDID stale, the mode invalid, and send a hotplug, from a work item. Let atomic checks of a native external pipe whose session stopped or crashed pass with it still active: nothing is shown on it, and other outputs can be configured until userspace switches it off. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 0d9c6fd72c29d1b84220f41bd22dda81343ceec0) --- drivers/gpu/drm/apple/iomfb_v14_7.c | 51 +++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index b24e55a4d5b89..8aad2c1f9025c 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -112,6 +112,8 @@ struct apple_dcp_v14 { /* Owns the RPC stream: start, swaps and idle callbacks. */ struct mutex lock; struct work_struct idle_work; + /* External: reports the display gone once the session stopped. */ + struct work_struct stopped_work; bool failed; bool started; @@ -175,6 +177,35 @@ static bool dcp_v14_panel_session; static int dcp_v14_callback(void *cookie, u32 tag, const void *input, u32 in_size, void *output, u32 out_size); +/* + * An external session stopped on a link error, not a crash: its display is + * gone until reboot. Report that as the crash path does, from process + * context, so that userspace stops driving the output. + */ +static void dcp_v14_external_stopped(struct apple_dcp_v14 *v14) +{ + if (v14->external) + schedule_work(&v14->stopped_work); +} + +static void dcp_v14_stopped_work(struct work_struct *work) +{ + struct apple_dcp_v14 *v14 = container_of(work, struct apple_dcp_v14, stopped_work); + struct apple_dcp *dcp = READ_ONCE(v14->dcp); + struct apple_connector *connector; + + if (!dcp) + return; + connector = READ_ONCE(dcp->connector); + dev_err(v14->dev, "external display session stopped: its display is reported disconnected\n"); + dcp_mode_invalidate(&dcp->mode_state); + if (connector) { + WRITE_ONCE(connector->connected, false); + apple_connector_edid_set_live(connector, false); + schedule_work(&connector->hotplug_wq); + } +} + /* Called with the lock held, or from a callback. */ static int dcp_v14_call(struct apple_dcp_v14 *v14, u32 tag, const void *in, u32 in_size, void *out, u32 out_size, u32 completion) @@ -190,6 +221,7 @@ static int dcp_v14_call(struct apple_dcp_v14 *v14, u32 tag, const void *in, u32 v14->failed = true; dev_err(v14->dev, "DCP call %#x failed: %d; recovery requires a reboot\n", tag, ret); + dcp_v14_external_stopped(v14); } return ret; } @@ -1028,6 +1060,7 @@ static void dcp_v14_idle(struct work_struct *work) if (ret && ret != -ETIMEDOUT && ret != -EAGAIN) { v14->failed = true; dev_err(v14->dev, "DCP notification failed: %d; recovery requires a reboot\n", ret); + dcp_v14_external_stopped(v14); } mutex_unlock(&v14->lock); } @@ -1683,6 +1716,7 @@ static int dcpext_boot(struct apple_dcp *dcp) mutex_init(&v14->lock); init_waitqueue_head(&v14->described_wait); INIT_WORK(&v14->idle_work, dcp_v14_idle); + INIT_WORK(&v14->stopped_work, dcp_v14_stopped_work); dcp->v14 = v14; dcp_v14_link_init(&v14->link, dev, NULL); @@ -1798,10 +1832,12 @@ int iomfb_v14_7_external_open(struct apple_dcp *dcp) ret = dcp_v14_simple_call(v14, A(401), false, true, true); if (!ret) ret = dcp_v14_simple_call(v14, A(455), false, false, false); - if (!ret) + if (!ret) { v14->opened = true; - else + } else { v14->failed = true; + dcp_v14_external_stopped(v14); + } mutex_unlock(&v14->lock); if (ret) { dev_err(dcp->dev, "external display interface did not open: %d; no retry until reboot\n", @@ -1930,6 +1966,8 @@ void iomfb_v14_7_remove(struct apple_dcp *dcp) /* Their endpoints are freed with the apple_dcp too; nothing queues more now. */ dcp_v14_flush_endpoints(dcp); cancel_work_sync(&v14->idle_work); + if (v14->external) + cancel_work_sync(&v14->stopped_work); } static int dcp_v14_status_show(struct seq_file *m, void *unused) @@ -2217,6 +2255,15 @@ int iomfb_v14_7_atomic_check(struct apple_dcp *dcp, struct drm_crtc *crtc, /* An external pipe can always be switched off, started or not. */ if (!crtc_state || !crtc_state->active) return 0; + /* + * A stopped native session reports its display gone. Until + * userspace switches the pipe off, let commits that include it + * through: nothing is shown on it, and other outputs must not + * fail with it. + */ + if (dcp->external_native && + (dcp->crashed || (v14 && READ_ONCE(v14->failed)))) + return 0; if (dcp->crashed || !v14 || READ_ONCE(v14->failed) || !v14->opened) return -EIO; } else if (dcp->crashed || !v14 || READ_ONCE(v14->failed)) { From f6a26090d2c78bc126afd236162f6881ab294388 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:24:26 -0700 Subject: [PATCH 1626/2680] drm/apple: t6030: drop a pending display retry when a display comes or goes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bounded retry of an external pipe was neither cancelled when its display was unplugged nor when a new one was attached. A retry queued for one display (up to 8 s later) could then fire during the connect of the next one, deassert HPD and release its link half way, and a real new failure only logged "a retry is already queued" behind the stale one. - Cancel a pending retry on a new Type-C HPD or attach, on a detach, and on a new HDMI attach. - Note the Type-C attach generation when a retry is queued; the work drops itself if it changed since. - Do the release in the work under the HPD lock, after checking the generation and the cable again, so that it cannot run into a connect or release of another display. The release itself is shared with dcp_dptx_disconnect(), which is unchanged. Native T6030 external pipes only. [ryan: adapted to 7.2.10 — dcp_dptx_disconnect() keeps the H17P external hand-off before taking hpd_mutex, and the split-out dcp_dptx_release_locked() is the remainder; the retry's final reconnect queues our Type-C reconnect work under hpd_mutex] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 147557e094e300dd9e2ef71f3553d1430e6ce6a9) --- drivers/gpu/drm/apple/dcp-fabric.c | 4 +- drivers/gpu/drm/apple/dcp-internal.h | 2 + drivers/gpu/drm/apple/dcp.c | 59 ++++++++++++++++++++-------- 3 files changed, 48 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 303e7db58eb1a..ae4b376181c5f 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -2411,8 +2411,10 @@ static void dcp_hdmi_connect_fixed(void *ctx) int ret = dcp_fixed_output_select(dcp); /* A new HDMI attach gets its own retries. */ - if (dcp->external_native) + if (dcp->external_native) { atomic_set(&dcp->external_retries, 0); + cancel_delayed_work(&dcp->external_retry_wq); + } if (ret) dev_err(dcp->dev, "could not select the HDMI output: %d\n", ret); else diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 315f47bbe7b4b..826a0d837bac8 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -269,6 +269,8 @@ struct apple_dcp { /* Native: bounded recovery of a display link or mode, see dcp.c. */ struct delayed_work external_retry_wq; atomic_t external_retries; + /* typec_generation when the pending retry was queued */ + u64 external_retry_generation; atomic_t external_requested; /* DCP_FIRMWARE_V_14_7 state; outlives this device once RTKit runs. */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index cedd780430d4f..809db28f7702b 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1269,6 +1269,8 @@ void dcp_external_ready(struct apple_dcp *dcp) #define DCP_EXTERNAL_RETRIES 3 +static void dcp_dptx_release_locked(struct apple_dcp *dcp, u32 port); + /* * Bounded recovery for a native external pipe: re-apply the display mode, * or redo the display link, up to DCP_EXTERNAL_RETRIES times per attached @@ -1301,6 +1303,8 @@ void dcp_external_retry(struct apple_dcp *dcp, const char *why, int error, delay = base_ms << (n - 1); dev_info(dcp->dev, "%s (%d): retry %u of %u in %u ms\n", why, error, n, DCP_EXTERNAL_RETRIES, delay); + /* An attach or detach after this makes it stale; see the work. */ + WRITE_ONCE(dcp->external_retry_generation, READ_ONCE(dcp->typec_generation)); mod_delayed_work(system_freezable_wq, &dcp->external_retry_wq, msecs_to_jiffies(delay)); } @@ -1324,7 +1328,12 @@ void dcp_external_retry_work(struct work_struct *work) struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, external_retry_wq); struct apple_connector *connector = READ_ONCE(dcp->connector); + u64 generation = READ_ONCE(dcp->external_retry_generation); + if (READ_ONCE(dcp->typec_generation) != generation) { + dev_info(dcp->dev, "display retry: the display was attached or detached since; dropped\n"); + return; + } if (iomfb_v14_7_external_failed(dcp)) { dev_err(dcp->dev, "display retry: the external display processor stopped; no retry until reboot\n"); return; @@ -1359,16 +1368,23 @@ void dcp_external_retry_work(struct work_struct *work) * release the link if it is still up, and connect it again, which * makes the firmware describe the display anew. */ - dev_info(dcp->dev, "display retry: reconnecting the display link\n"); - if (dptx_service_available(&dcp->dptxport[0]) && READ_ONCE(dcp->dptxport[0].connected)) { - int ret = dptxport_set_hpd(dcp->dptxport[0].service, false); + scoped_guard(mutex, &dcp->hpd_mutex) { + /* Not under a connect or release of another display. */ + if (dcp->typec_generation != generation || !dcp->typec_cable_connected) { + dev_info(dcp->dev, "display retry: the display was attached or detached since; dropped\n"); + return; + } + dev_info(dcp->dev, "display retry: reconnecting the display link\n"); + if (dptx_service_available(&dcp->dptxport[0]) && dcp->dptxport[0].connected) { + int ret = dptxport_set_hpd(dcp->dptxport[0].service, false); - if (ret) - dev_warn(dcp->dev, "display retry: HPD deassert failed: %d\n", ret); - dcp_dptx_disconnect(dcp, 0); + if (ret) + dev_warn(dcp->dev, "display retry: HPD deassert failed: %d\n", ret); + dcp_dptx_release_locked(dcp, 0); + } + dcp->typec_reconnect_tries = 0; } guard(mutex)(&dcp->hpd_mutex); - dcp->typec_reconnect_tries = 0; if (dcp->typec_cable_connected) mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); } @@ -1753,15 +1769,9 @@ static void disconnected_hpd_event(struct apple_connector *con) schedule_work(&con->bl_sync_wq); } -int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) +static void dcp_dptx_release_locked(struct apple_dcp *dcp, u32 port) { - if (dcp_is_external_h17p(dcp)) - return dcp_external_disconnect(dcp, false); - - /* Release the caller's RemotePort service, not the downstream DFP port. */ - dev_info(dcp->dev, "%s(port=%d)\n", __func__, port); - - mutex_lock(&dcp->hpd_mutex); + lockdep_assert_held(&dcp->hpd_mutex); if (dcp->external) { smp_store_release(&dcp->external_link_ready, false); dcpext_scanout_invalidate(dcp); @@ -1777,6 +1787,18 @@ int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) */ if (dcp->external_native) dcp_direct_crossbar_link(dcp, false); +} + +int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) +{ + if (dcp_is_external_h17p(dcp)) + return dcp_external_disconnect(dcp, false); + + /* Release the caller's RemotePort service, not the downstream DFP port. */ + dev_info(dcp->dev, "%s(port=%d)\n", __func__, port); + + mutex_lock(&dcp->hpd_mutex); + dcp_dptx_release_locked(dcp, port); mutex_unlock(&dcp->hpd_mutex); return 0; @@ -1823,8 +1845,10 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) dcp->placeholder_retried = false; cancel_delayed_work(&dcp->typec_reconnect_wq); /* A newly attached display gets its own retries. */ - if (dcp->external_native) + if (dcp->external_native) { atomic_set(&dcp->external_retries, 0); + cancel_delayed_work(&dcp->external_retry_wq); + } } ret = dcp_dptx_connect(dcp, port); @@ -1847,6 +1871,9 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) WRITE_ONCE(dcp->typec_cable_connected, false); dcp->typec_generation++; dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); + /* Nothing left to retry for the display that went. */ + if (dcp->external_native) + cancel_delayed_work(&dcp->external_retry_wq); } WRITE_ONCE(dcp->typec_crtc_off, false); reinit_completion(&dcp->typec_iomfb_hpd_ready); From 85a1391699497767139801429ef29c9ed53c64ee Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:25:36 -0700 Subject: [PATCH 1627/2680] drm/apple: t6030: keep external firmware callbacks off unbound DRM objects A native external session outlives DRM unbind, which now runs at every shutdown and reboot. Its callbacks kept reading the pipe's connector: the firmware withdraws the display's description when the CRTC is switched off on the way down, and a callback that loaded the connector could use it after DRM freed it. A crash or a stopped session after unbind did the same. Mark the pipe detached at unbind, under the session lock every callback runs under, then drain the crash callback (RTKit receive), the notification work and the stopped-session work. While detached: - a published or withdrawn description is logged and otherwise ignored (no catalog, EDID or hotplug update); - a crash or stopped session leaves the connector alone; - opening the display interface sends no hotplug. Bind clears the mark. The panel's session is unchanged: unbind already detaches it completely. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 1363a25fde5006266c6bb1d7226677df670b15c8) --- drivers/gpu/drm/apple/dcp-internal.h | 5 +++++ drivers/gpu/drm/apple/dcp.c | 1 + drivers/gpu/drm/apple/iomfb_v14_7.c | 33 +++++++++++++++++++++++----- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 826a0d837bac8..0eb761324179a 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -271,6 +271,11 @@ struct apple_dcp { atomic_t external_retries; /* typec_generation when the pending retry was queued */ u64 external_retry_generation; + /* + * Native: DRM unbound this pipe; firmware callbacks leave its + * connector and CRTC alone. Set under the session lock. + */ + bool external_detached; atomic_t external_requested; /* DCP_FIRMWARE_V_14_7 state; outlives this device once RTKit runs. */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 809db28f7702b..28bfe51b40c79 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3017,6 +3017,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) /* The running T6030 firmware is adopted as is. */ if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { if (dcp->external_native) { + WRITE_ONCE(dcp->external_detached, false); enable_work(&dcp->vblank_wq); enable_delayed_work(&dcp->swap_watchdog_wq); enable_delayed_work(&dcp->external_retry_wq); diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 8aad2c1f9025c..ef3f1b8bcde3f 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -194,7 +194,7 @@ static void dcp_v14_stopped_work(struct work_struct *work) struct apple_dcp *dcp = READ_ONCE(v14->dcp); struct apple_connector *connector; - if (!dcp) + if (!dcp || READ_ONCE(dcp->external_detached)) return; connector = READ_ONCE(dcp->connector); dev_err(v14->dev, "external display session stopped: its display is reported disconnected\n"); @@ -451,6 +451,9 @@ static void dcp_v14_external_hotplug(struct apple_dcp *dcp, bool connected) struct apple_connector *connector = READ_ONCE(dcp->connector); unsigned int action; + if (READ_ONCE(dcp->external_detached)) + return; + /* * Powering a Type-C CRTC off releases the link, and the firmware then * withdraws the display. The cable is still in: keep the connector @@ -524,6 +527,9 @@ static void dcp_v14_external_published(struct apple_dcp_v14 *v14, const char *ke return; dev_info(dcp->dev, "display %s %s, %u bytes\n", key, removed ? "withdrawn" : "published", raw ? raw->size : 0); + /* DRM unbound the pipe: its connector and the users of its modes are gone. */ + if (READ_ONCE(dcp->external_detached)) + return; /* The connector's debugfs keeps a copy. */ connector = READ_ONCE(dcp->connector); @@ -1078,7 +1084,7 @@ static void dcp_v14_crashed(void *cookie, const void *crashlog, size_t crashlog_ if (dcp->external) dcpext_scanout_fault(dcp, -EIO); /* The display it drove is gone until reboot. */ - if (v14->external && connector) { + if (v14->external && connector && !READ_ONCE(dcp->external_detached)) { WRITE_ONCE(connector->connected, false); apple_connector_edid_set_live(connector, false); schedule_work(&connector->hotplug_wq); @@ -1846,7 +1852,8 @@ int iomfb_v14_7_external_open(struct apple_dcp *dcp) } dev_info(dcp->dev, "external display interface open\n"); /* A display described before this could not be enabled until now. */ - if (dcp->connector && READ_ONCE(dcp->connector->connected)) + if (!READ_ONCE(dcp->external_detached) && dcp->connector && + READ_ONCE(dcp->connector->connected)) schedule_work(&dcp->connector->hotplug_wq); return 0; } @@ -1918,11 +1925,27 @@ void iomfb_v14_7_unbind(struct apple_dcp *dcp) dcp->active = false; dcp_mode_invalidate(&dcp->mode_state); + if (dcp->external_native && !v14) + WRITE_ONCE(dcp->external_detached, true); if (!v14) return; - /* An external session keeps serving its ports; removal detaches it. */ - if (!v14->external) + /* + * An external session keeps serving its ports; removal detaches it. + * Its callbacks stop using the connector and CRTC DRM frees next: + * the flag is set under the lock they run under, and the crash + * callback and pending work are drained. + */ + if (v14->external) { + mutex_lock(&v14->lock); + WRITE_ONCE(dcp->external_detached, true); + mutex_unlock(&v14->lock); + if (v14->rtk) + apple_rtkit_flush_rx(v14->rtk); + flush_work(&v14->idle_work); + cancel_work_sync(&v14->stopped_work); + } else { WRITE_ONCE(v14->dcp, NULL); + } if (v14->rtk) dev_info(dcp->dev, "display unbound; the DCP session and its buffers are kept until reboot\n"); } From 2341e52971d7e439f72259859027c0383853821c Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:25:56 -0700 Subject: [PATCH 1628/2680] drm/apple: t6030: disable the HDMI HPD interrupt on external unbind The native external branch of component unbind returned before the HDMI HPD interrupt was disabled. HPD edges then kept reaching the fabric after DRM was gone, and a rebind enabled the interrupt a second time, an unbalanced enable. Disable it there too, before the works it queues are disabled. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit ca507df193a417c5ce01f0cf99d21bb504ad1292) --- drivers/gpu/drm/apple/dcp.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 28bfe51b40c79..f127f0cb1980d 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3155,6 +3155,9 @@ VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, v disable_work_sync(&dcp->dimensions_wq); if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { if (dcp->external_native) { + /* Bind's wait_ready enabled it, as for any HDMI port. */ + if (dcp->hdmi_hpd_irq) + disable_irq(dcp->hdmi_hpd_irq); dcp_disable_typec_work(dcp, true); disable_delayed_work_sync(&dcp->external_retry_wq); disable_delayed_work_sync(&dcp->swap_watchdog_wq); From c407a40380aa94fccba569834492ec95552346ec Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:26:40 -0700 Subject: [PATCH 1629/2680] drm/apple: t6030: route USB-C displays past a failed external processor The Type-C fabric did not know about a native external processor that failed: its handoff check at probe, its start, or its session later. It kept preferring it (dcpext0 has the lower CRTC index) for every USB-C display, which then stayed dark until reboot although dcpext1 was fine. Treat a failed native external pipeline like a busy one, both when a port picks a free pipeline and when the routes are planned, so that a working pipeline takes the display. A route it already holds is left as it is until its display goes. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 8d6a0f2d515444698b6ce0f2f51e8b72c8dd304b) --- drivers/gpu/drm/apple/dcp-fabric.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index ae4b376181c5f..30381e33080b1 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -142,12 +142,26 @@ static bool dcp_typec_safe_held(struct apple_dcp *dcp) return false; } +/* + * A pipeline that can take no Type-C display: its fixed output is live, or + * it is a T6030 external processor that failed (handoff check, start, or a + * stopped session), which drives nothing until reboot. Planned around and + * passed over like a busy one, so that a healthy pipeline takes the port. + */ +static bool dcp_typec_route_busy(struct apple_dcp_typec_route *route) +{ + struct apple_dcp *dcp = route->dcp; + + return dcp_typec_route_fixed_output_busy(route) || + (dcp->external_native && iomfb_v14_7_external_failed(dcp)); +} + static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) { struct dcp_fabric_pipeline pipeline = { .owned = !!route->dcp->active_typec_route || dcp_typec_safe_held(route->dcp), - .fixed_busy = dcp_typec_route_fixed_output_busy(route), + .fixed_busy = dcp_typec_route_busy(route), }; struct dcp_fabric_policy policy = { .dual_stream = dcp_typec_dual_stream() }; @@ -775,7 +789,7 @@ static void dcp_fabric_snapshot_port(struct apple_dcp_typec_port *port, pipeline->crtc_index = dcp->crtc ? drm_crtc_index(&dcp->crtc->base) : 0; pipeline->has_fixed = !!dcp->fixed_phy; - pipeline->fixed_busy = dcp_typec_route_fixed_output_busy(route); + pipeline->fixed_busy = dcp_typec_route_busy(route); pipeline->owned = !!dcp->active_typec_route || dcp_typec_safe_held(dcp); pipeline->tunnel_held = dcp_typec_tunnel_held(dcp); From a4ff98606a4a4ea46dd031e8760be746dcbb9b2a Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:30:25 -0700 Subject: [PATCH 1630/2680] drm/apple: t6030: don't let an external processor's probe hold up USB-C Every USB-C port lists the Type-C routes of both external processors, and the panel's DRM device waits for both processors. A probe that failed or deferred for good on one of them (an HDMI part that never probes, such as the ATC PHY with bad tunables, or the converter's GPIOs) therefore left the PD controllers of all ports deferring, so no USB-C anywhere, and the panel on the boot framebuffer. On a native T6030 external processor: - The fixed (HDMI) output is optional. If one of its parts fails, or is still deferred once the deferred probe timeout has passed, log "HDMI output unusable (: ); USB-C displays are not affected", drop what was set up for it, and go on as a Type-C-only pipe. Before the timeout it keeps deferring, as boot ordering needs. - If its Type-C routes fail to register (or stay deferred past the timeout), disable them in the device tree, so that the ports pass over them, and add the processor to the DRM device as a pipe that is never started ("external display processor unusable: its Type-C display routes did not register"). The error paths of every other processor, M1/M2 included, are unchanged. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 4b198d9ce7cfbb430314b41ac4db62c6bce7e581) --- drivers/gpu/drm/apple/dcp-fabric.c | 36 +++++++++ drivers/gpu/drm/apple/dcp-fabric.h | 1 + drivers/gpu/drm/apple/dcp.c | 112 +++++++++++++++++++++++++++- drivers/gpu/drm/apple/iomfb_v14_7.c | 7 ++ drivers/gpu/drm/apple/iomfb_v14_7.h | 1 + 5 files changed, 155 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 30381e33080b1..cd218c53e4579 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -2304,6 +2304,42 @@ static int dcp_register_typec_routes_present(void *data) return 0; } +/* + * Disables every Type-C route of @dcp in the device tree, for good. A + * connector passes over unavailable routes when it looks for its display + * routes, so its port no longer waits for this processor. + */ +void dcp_typec_routes_disable(struct apple_dcp *dcp) +{ + struct device_node *routes __free(device_node) = + of_get_child_by_name(dcp->dev->of_node, "typec-routes"); + struct device_node *route; + struct of_changeset *cs; + int ret = 0; + + if (!routes) + return; + cs = kzalloc(sizeof(*cs), GFP_KERNEL); + if (!cs) + return; + of_changeset_init(cs); + for_each_available_child_of_node(routes, route) { + ret = of_changeset_update_prop_string(cs, route, "status", "disabled"); + if (ret) { + of_node_put(route); + break; + } + } + if (!ret) + ret = of_changeset_apply(cs); + if (ret) { + dev_err(dcp->dev, "could not disable the Type-C display routes: %d\n", ret); + of_changeset_destroy(cs); + kfree(cs); + } + /* Applied, it stays: the live tree refers to it. */ +} + int dcp_register_typec_routes(struct apple_dcp *dcp) { struct device_node *routes __free(device_node) = diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index b12f01e369e10..12bc416ea08c8 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -79,6 +79,7 @@ int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active); /* DCP probe, resume and endpoint teardown hooks. */ int dcp_register_typec_routes(struct apple_dcp *dcp); +void dcp_typec_routes_disable(struct apple_dcp *dcp); void dcp_typec_retrain_work(struct work_struct *work); void dcp_fabric_init(struct apple_dcp *dcp); void dcp_fabric_hdmi_resume(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index f127f0cb1980d..ae487e7d26ee7 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3293,6 +3293,65 @@ static const struct component_ops dcp_comp_ops = { .unbind = dcp_comp_unbind, }; +/* + * A T6030 external processor's fixed (HDMI) output is optional. Its Type-C + * routes must register whatever happens to that output: every USB-C port + * lists them and waits for them, as the panel's DRM device waits for this + * processor. Keep deferring while the HDMI parts may still appear, then go + * on as a Type-C-only pipe. Elsewhere the error ends the probe, as before. + */ +static int dcp_fixed_output_error(struct apple_dcp *dcp, int err, const char *what) +{ + if (!dcp->external_native) + return err; + if (err == -EPROBE_DEFER) { + err = driver_deferred_probe_check_state(dcp->dev); + if (err == -EPROBE_DEFER) + return err; + } + dev_err(dcp->dev, "HDMI output unusable (%s: %d); USB-C displays are not affected\n", + what, err); + if (dcp->fixed_route_selected) + mux_control_deselect(dcp->xbar); + dcp->fixed_route_selected = false; + dcp->xbar = NULL; + if (!IS_ERR_OR_NULL(dcp->typec_mux)) + typec_mux_put(dcp->typec_mux); + dcp->typec_mux = NULL; + dcp->phy_managed_by_typec = false; + /* A requested HPD interrupt stays disabled: nothing enables it now. */ + dcp->hdmi_hpd_irq = 0; + dcp->hdmi_hpd = NULL; + dcp->hdmi_pwren = NULL; + dcp->dp2hdmi_pwren = NULL; + dcp->phy = NULL; + dcp->fixed_phy = NULL; + dcp->fixed_connector_type = DRM_MODE_CONNECTOR_USB; + dcp->connector_type = DRM_MODE_CONNECTOR_USB; + return 0; +} + +/* + * A T6030 external processor whose Type-C routes did not register: disable + * them in the device tree so that the ports stop waiting for them, and add + * the processor to the DRM device as a pipe that never starts, so that the + * panel is not held up either. + */ +static int dcp_native_routes_error(struct apple_dcp *dcp, int err) +{ + if (!dcp->external_native) + return err; + if (err == -EPROBE_DEFER) { + err = driver_deferred_probe_check_state(dcp->dev); + if (err == -EPROBE_DEFER) + return err; + } + dev_err(dcp->dev, "Type-C display routes unusable: %d; USB-C ports go on without this processor\n", + err); + dcp_typec_routes_disable(dcp); + return 0; +} + static int dcp_platform_probe(struct platform_device *pdev) { const struct apple_dcp_hw_data *hw = of_device_get_match_data(&pdev->dev); @@ -3300,6 +3359,7 @@ static int dcp_platform_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct apple_dcp *dcp; int ret, surf, num_surfs; + bool routes_failed = false; u32 surf_en; u32 mux_index; @@ -3423,6 +3483,11 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp->hw.num_dptx_ports = 2; dcp->phy = devm_phy_optional_get(dev, "dp-phy"); + if (IS_ERR(dcp->phy) && dcp->external_native) { + ret = dcp_fixed_output_error(dcp, PTR_ERR(dcp->phy), "dp-phy"); + if (ret) + return ret; + } if (IS_ERR(dcp->phy)) { dev_err(dev, "Failed to get dp-phy: %ld\n", PTR_ERR(dcp->phy)); return PTR_ERR(dcp->phy); @@ -3472,6 +3537,12 @@ static int dcp_platform_probe(struct platform_device *pdev) * template ADT. TODO: check device ADT */ dcp->hdmi_hpd = devm_gpiod_get_optional(dev, "hdmi-hpd", GPIOD_IN); + if (IS_ERR(dcp->hdmi_hpd) && dcp->external_native) { + ret = dcp_fixed_output_error(dcp, PTR_ERR(dcp->hdmi_hpd), "hdmi-hpd"); + if (ret) + return ret; + goto fixed_done; + } if (IS_ERR(dcp->hdmi_hpd)) return PTR_ERR(dcp->hdmi_hpd); ret = dcp_hdmi_init(dcp); @@ -3479,6 +3550,12 @@ static int dcp_platform_probe(struct platform_device *pdev) return ret; if (dcp->hdmi_hpd) { int irq = gpiod_to_irq(dcp->hdmi_hpd); + if (irq < 0 && dcp->external_native) { + ret = dcp_fixed_output_error(dcp, irq, "hdmi-hpd interrupt"); + if (ret) + return ret; + goto fixed_done; + } if (irq < 0) { dev_err(dev, "failed to translate HDMI hpd GPIO to IRQ\n"); return irq; @@ -3490,6 +3567,12 @@ static int dcp_platform_probe(struct platform_device *pdev) IRQF_ONESHOT | IRQF_NO_AUTOEN | IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING, "dp2hdmi-hpd-irq", dcp); + if (ret < 0 && dcp->external_native) { + ret = dcp_fixed_output_error(dcp, ret, "hdmi-hpd interrupt"); + if (ret) + return ret; + goto fixed_done; + } if (ret < 0) { dev_err(dev, "failed to request HDMI hpd irq %d: %d\n", irq, ret); @@ -3503,10 +3586,22 @@ static int dcp_platform_probe(struct platform_device *pdev) * on battery powered Macbooks. */ dcp->hdmi_pwren = devm_gpiod_get_optional(dev, "hdmi-pwren", GPIOD_OUT_HIGH); + if (IS_ERR(dcp->hdmi_pwren) && dcp->external_native) { + ret = dcp_fixed_output_error(dcp, PTR_ERR(dcp->hdmi_pwren), "hdmi-pwren"); + if (ret) + return ret; + goto fixed_done; + } if (IS_ERR(dcp->hdmi_pwren)) return PTR_ERR(dcp->hdmi_pwren); dcp->dp2hdmi_pwren = devm_gpiod_get_optional(dev, "dp2hdmi-pwren", GPIOD_OUT_HIGH); + if (IS_ERR(dcp->dp2hdmi_pwren) && dcp->external_native) { + ret = dcp_fixed_output_error(dcp, PTR_ERR(dcp->dp2hdmi_pwren), "dp2hdmi-pwren"); + if (ret) + return ret; + goto fixed_done; + } if (IS_ERR(dcp->dp2hdmi_pwren)) return PTR_ERR(dcp->dp2hdmi_pwren); @@ -3521,6 +3616,12 @@ static int dcp_platform_probe(struct platform_device *pdev) if (!ret) { dcp->fixed_mux_index = mux_index; dcp->xbar = devm_mux_control_get(dev, "dp-xbar"); + if (IS_ERR(dcp->xbar) && dcp->external_native) { + ret = dcp_fixed_output_error(dcp, PTR_ERR(dcp->xbar), "dp-xbar"); + if (ret) + return ret; + goto fixed_done; + } if (IS_ERR(dcp->xbar)) { dev_err(dev, "Failed to get dp-xbar: %ld\n", PTR_ERR(dcp->xbar)); return PTR_ERR(dcp->xbar); @@ -3559,9 +3660,14 @@ static int dcp_platform_probe(struct platform_device *pdev) } } +fixed_done: ret = dcp_register_typec_routes(dcp); - if (ret) - return ret; + if (ret) { + ret = dcp_native_routes_error(dcp, ret); + if (ret) + return ret; + routes_failed = true; + } /* * The manual T6030 external path is not part of the panel's DRM @@ -3590,6 +3696,8 @@ static int dcp_platform_probe(struct platform_device *pdev) ret = iomfb_v14_7_external_prepare(dcp); if (ret) return ret; + if (routes_failed) + iomfb_v14_7_external_refuse(dcp, "its Type-C display routes did not register"); } ret = component_add(&pdev->dev, &dcp_comp_ops); diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index ef3f1b8bcde3f..902109c534166 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -1556,6 +1556,13 @@ int iomfb_v14_7_external_prepare(struct apple_dcp *dcp) return 0; } +/* Probe: the pipe stays in the DRM device, never started. */ +void iomfb_v14_7_external_refuse(struct apple_dcp *dcp, const char *why) +{ + WRITE_ONCE(dcp->external_phase, DCPEXT_FAILED); + dev_err(dcp->dev, "external display processor unusable: %s\n", why); +} + /* True once DPTX can be used; asks for the start if nothing has. */ bool iomfb_v14_7_external_ready(struct apple_dcp *dcp) { diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.h b/drivers/gpu/drm/apple/iomfb_v14_7.h index 109acf42696c8..2100e0fc2cded 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7.h @@ -33,6 +33,7 @@ int iomfb_v14_7_external_start(struct apple_dcp *dcp); */ int iomfb_v14_7_external_prepare(struct apple_dcp *dcp); bool iomfb_v14_7_external_ready(struct apple_dcp *dcp); +void iomfb_v14_7_external_refuse(struct apple_dcp *dcp, const char *why); int iomfb_v14_7_external_open(struct apple_dcp *dcp); const char *iomfb_v14_7_external_busy(struct apple_dcp *dcp); /* The external session can no longer be used until reboot. */ From 35661b2b2bd4cad9cb4db77d52c0038abdccfbbd Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:42:29 -0700 Subject: [PATCH 1631/2680] arm64: dts: apple: t8122: describe the internal display; drm/apple: admit it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the T8122 (M3 MacBook Air, J613/J615) internal display nodes: the DCP, its mailbox, the two display DARTs and the display subsystem, all status "disabled", in t8122-dcp.dtsi (included from t8122.dtsi). Register, interrupt and DART addresses are the SoC addresses from the J613 boot device tree; the DCP mailbox interrupts use the send-empty/send-not-empty/recv-empty/ recv-not-empty order the SMC mailbox already uses. This mirrors t6030-dcp.dtsi, which the boot loader's internal-display handoff generalises. Teach the DCP driver the T8122 internal DCP: apple,t8122-dcp and apple,t8122-display-subsystem in the apple-drm and dcp of_match tables, a hw data entry, the 14.7 firmware-compat check, and the DCP image uuid and the apple,t8122-handoff marker in the v14.7 probe's allowlist. T6030 behaviour is unchanged. The display enable gate still refuses to enable these nodes on T8122 because no T8122 PMP description is built in, so the DCP driver does not probe from this device tree yet and the Mac keeps the boot framebuffer. The disp0 fixed-clock the DCP driver needs is likewise left for the PMP increment. This is display groundwork only: no native modeset this change. [ryan: adapted to 7.2.10 — the arm64 dts hunks (new t8122-dcp.dtsi and its include from t8122.dtsi) belong to 000-devicetree and are left out; handed over to 000-devicetree. Nothing of held 7b1603a04e87 is taken] Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 6e53447b23e50ce9f0a020041a595c79727b3aa8) --- drivers/gpu/drm/apple/apple_drv.c | 2 ++ drivers/gpu/drm/apple/dcp.c | 13 +++++++++++++ drivers/gpu/drm/apple/iomfb_v14_7.c | 15 +++++++++------ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 7d103ed0927d8..e7372f471a5e7 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -697,6 +697,7 @@ static const struct of_device_id apple_dcp_id_tbl[] = { { .compatible = "apple,dcpext" }, /* T6030: apple_dcp_usable() keeps its dcpext outside the DRM device. */ { .compatible = "apple,t6030-dcp" }, + { .compatible = "apple,t8122-dcp" }, {}, }; @@ -1005,6 +1006,7 @@ static void apple_platform_remove(struct platform_device *pdev) static const struct of_device_id of_match[] = { { .compatible = "apple,display-subsystem" }, { .compatible = "apple,t6030-display-subsystem" }, + { .compatible = "apple,t8122-display-subsystem" }, {} }; MODULE_DEVICE_TABLE(of, of_match); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index ae487e7d26ee7..56df367f4f6ce 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2801,6 +2801,14 @@ static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) return DCP_FIRMWARE_UNKNOWN; } + if (of_device_is_compatible(dev->of_node, "apple,t8122-dcp")) { + if (ret >= 0 && !strcmp(compat_str, "14.7.0")) + return DCP_FIRMWARE_V_14_7; + dev_err(dev, "T8122 display not started: DCP firmware-compat %s is not 14.7.0\n", + compat_str); + return DCP_FIRMWARE_UNKNOWN; + } + if (strncmp(compat_str, "12.3.0", sizeof(compat_str)) == 0) return DCP_FIRMWARE_V_12_3; /* @@ -4069,6 +4077,10 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t6030 = { .num_dptx_ports = 1, }; +static const struct apple_dcp_hw_data apple_dcp_hw_t8122 = { + .num_dptx_ports = 2, +}; + static const struct apple_dcp_hw_data apple_dcp_hw_t6030_dcpext = { .num_dptx_ports = 1, }; @@ -4086,6 +4098,7 @@ static const struct of_device_id of_match[] = { { .compatible = "apple,t8142-dcp", .data = &apple_dcp_hw_t8142, }, { .compatible = "apple,t6030-dcp", .data = &apple_dcp_hw_t6030, }, { .compatible = "apple,t6030-dcpext", .data = &apple_dcp_hw_t6030_dcpext, }, + { .compatible = "apple,t8122-dcp", .data = &apple_dcp_hw_t8122, }, { .compatible = "apple,dcp", .data = &apple_dcp_hw_dcp, }, { .compatible = "apple,dcpext", .data = &apple_dcp_hw_dcpext, }, {} diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 902109c534166..7b0f43dc32d7b 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -45,8 +45,9 @@ #define A(n) DCP_V14_TAG('A', n) #define D(n) DCP_V14_TAG('D', n) -/* The image whose method and callback layouts this file implements. */ -#define DCP_V14_FIRMWARE_UUID "DDF38191-93B3-324A-BC8F-643006F5AC82" +/* The images whose method and callback layouts this file implements. */ +#define DCP_V14_FIRMWARE_UUID "DDF38191-93B3-324A-BC8F-643006F5AC82" /* T6030 */ +#define DCP_V14_FIRMWARE_UUID_T8122 "90F849E1-B422-367E-B389-50246F8DEC47" /* T8122 */ #define DCP_V14_CPU_CONTROL 0x44 #define DCP_V14_CPU_STATUS 0x48 @@ -1253,13 +1254,15 @@ int iomfb_v14_7_probe(struct apple_dcp *dcp) int ret; if (of_property_read_string(np, "apple,firmware-uuid", &uuid) || - strcmp(uuid, DCP_V14_FIRMWARE_UUID)) + (strcmp(uuid, DCP_V14_FIRMWARE_UUID) && + strcmp(uuid, DCP_V14_FIRMWARE_UUID_T8122))) return dev_err_probe(dev, -ENODEV, - "T6030 display not started: DCP firmware %s is not supported\n", + "display not started: DCP firmware %s is not supported\n", uuid ?: "(unknown)"); - if (of_property_read_u32(np, "apple,t6030-handoff", &marker) || marker != 1) + if ((of_property_read_u32(np, "apple,t6030-handoff", &marker) || marker != 1) && + (of_property_read_u32(np, "apple,t8122-handoff", &marker) || marker != 1)) return dev_err_probe(dev, -ENODEV, - "T6030 display not started: no boot loader display handoff\n"); + "display not started: no boot loader display handoff\n"); if (!iommu_get_domain_for_dev(dev)) return dev_err_probe(dev, -ENODEV, "T6030 display not started: the DCP has no DART domain\n"); From 15be5d185e4ae64ba599c6195dc31d314520af14 Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Tue, 6 Oct 2026 23:09:16 -0700 Subject: [PATCH 1632/2680] drm/apple: follow direct Type-C display routes during modesets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When direct Type-C displays are assigned to the other display pipeline, move or swap their routes with their CRTC assignments on the supported 12.3 and 13.5 paths. Keep tunnel movement, arrival ordering and fixed-output reclaim unchanged. Prepare and validate both catalogs before releasing routes, restore the original routes on attach failure, and block power, modeset and flush when the assignment is unavailable. Wait for the destination's own mode catalog before programming it. Keep ordinary paired DPMS behavior and preserve fixed-output reservations during a route move. Distinguish real hotplug readiness from failure notifications so recovery requests survive coalescing without repeatedly resetting a failed route. Validation: W=1 Apple DRM objects, ARM64 KUnit suites, production-path regression tests and focused negative controls. [ryan: adapted to 7.2.10 — 7.2 types (drm_atomic_commit; apple_atomic_state wraps it via drm_atomic_commit_init/_default_clear/_default_release); the follow decision also refuses a port, or a swap holder's port, with safe_pending, and drops the tb_retiring check (no such state here); the follow lock is our single dcp_typec_fabric_lock (no dcp_tb_handoff_lock) and the attach drops dp_release_deadline; possible_crtcs stay narrowed as before (dual-stream included) except for a direct route that follows its CRTC; best_encoder keeps rejecting another pipeline's CRTC unless the connector's pipeline follows its CRTC; our reconnect work stands down while typec_follow_retiring (this tree has no reconnect sessions); dcp_dptx_park() uses dptx_service_available(); dcp_modes_adopt() publishes under metadata.lock and retains the replaced array; the hotplug reasons gate both our announce relink and the link-status reset; mode_fixup keeps dcp_mode_settable(). The feature is gated to 12.3/13.5 pipelines, so H17P, the 14.7 template and native 14.7 are unchanged] Link: https://github.com/iconidentify/aurora-linux/issues/39 Link: https://github.com/iconidentify/aurora-linux/pull/43 Signed-off-by: Christian Bartels Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 277ed64ad6be50221bf3450f7313ea5393bd75c6) --- drivers/gpu/drm/apple/apple_drv.c | 81 +++- drivers/gpu/drm/apple/connector.h | 3 + drivers/gpu/drm/apple/dcp-fabric-core.c | 88 ++++ drivers/gpu/drm/apple/dcp-fabric-core.h | 28 ++ drivers/gpu/drm/apple/dcp-fabric-test.c | 269 +++++++++++ drivers/gpu/drm/apple/dcp-fabric.c | 583 +++++++++++++++++++++++- drivers/gpu/drm/apple/dcp-fabric.h | 10 + drivers/gpu/drm/apple/dcp-internal.h | 6 + drivers/gpu/drm/apple/dcp.c | 72 ++- drivers/gpu/drm/apple/dcp.h | 14 + drivers/gpu/drm/apple/hotplug-test.c | 44 ++ drivers/gpu/drm/apple/iomfb.c | 177 ++++++- drivers/gpu/drm/apple/iomfb_template.c | 2 +- drivers/gpu/drm/apple/iomfb_v14_7.c | 6 +- drivers/gpu/drm/apple/parser-test.c | 193 ++++++++ drivers/gpu/drm/apple/parser.h | 6 + 16 files changed, 1544 insertions(+), 38 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index e7372f471a5e7..87a8af5380330 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -149,21 +149,51 @@ static void apple_connector_oob_hotplug(struct drm_connector *connector, static void apple_crtc_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_commit *commit) { + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + struct apple_dcp *dcp = platform_get_drvdata(apple_crtc->dcp); struct drm_crtc_state *crtc_state; - crtc_state = drm_atomic_get_new_crtc_state(commit, crtc); + bool locked; + int ret; - if (crtc_state->active_changed && crtc_state->active) { - struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + crtc_state = drm_atomic_get_new_crtc_state(commit, crtc); + locked = false; + ret = to_apple_atomic_state(commit)->failed_routes & drm_crtc_mask(crtc) ? + -EIO : dcp_typec_follow_crtc(crtc, commit, &locked); + /* A failed move must never power or program another sink's pipeline. */ + if (ret || !dcp_crtc_route_ready(crtc, commit, false)) + goto failed; + + if (crtc_state->active && dcp_crtc_needs_route_start(dcp)) { + /* Establish the link/catalog before powering the assigned IOMFB. */ + ret = dcp_typec_prepare_route(dcp); + if (ret) + goto failed; + } + if (crtc_state->active && !dcp_crtc_route_ready(crtc, commit, true)) + goto failed; + if (crtc_state->active && + (crtc_state->active_changed || dcp_crtc_needs_route_start(dcp))) { dcp_poweron(apple_crtc->dcp); - /* Force the CTM to be set on first swap */ crtc_state->color_mgmt_changed = true; } else if (crtc_state->active && drm_atomic_crtc_needs_modeset(crtc_state)) { /* A cable generation can change while the CRTC stays active. */ dcp_external_recover(to_apple_crtc(crtc)->dcp); } - if (crtc_state->active) dcp_crtc_atomic_modeset(crtc, commit); + dcp_typec_follow_done(locked); + return; + +failed: + if (!ret) + ret = -EAGAIN; + dcp_mode_invalidate(&dcp->mode_state); + dev_err(dcp->dev, "CRTC route unavailable (%d); modeset deferred\n", ret); + /* Flush completes any pending flip without issuing a firmware swap. */ + schedule_work(&dcp->vblank_wq); + if (dcp->connector) + dcp_route_failure_notify(dcp->connector); + dcp_typec_follow_done(locked); } static void apple_crtc_atomic_disable(struct drm_crtc *crtc, @@ -273,7 +303,35 @@ static const struct drm_crtc_funcs apple_crtc_funcs = { }; +static struct drm_atomic_commit *apple_atomic_state_alloc(struct drm_device *dev) +{ + struct apple_atomic_state *state = kzalloc_obj(*state); + + if (!state) + return NULL; + if (drm_atomic_commit_init(dev, &state->base)) { + kfree(state); + return NULL; + } + return &state->base; +} + +static void apple_atomic_state_clear(struct drm_atomic_commit *state) +{ + drm_atomic_commit_default_clear(state); + to_apple_atomic_state(state)->failed_routes = 0; +} + +static void apple_atomic_state_free(struct drm_atomic_commit *state) +{ + drm_atomic_commit_default_release(state); + kfree(to_apple_atomic_state(state)); +} + static const struct drm_mode_config_funcs apple_mode_config_funcs = { + .atomic_state_alloc = apple_atomic_state_alloc, + .atomic_state_clear = apple_atomic_state_clear, + .atomic_state_free = apple_atomic_state_free, .atomic_check = drm_atomic_helper_check, .atomic_commit = drm_atomic_helper_commit, .fb_create = drm_gem_fb_create, @@ -358,13 +416,18 @@ apple_connector_atomic_best_encoder(struct drm_connector *conn, /* * Type-C tunnels are routed before the modeset and cannot move here. - * Reject stale topology and explicit assignments to another pipeline. + * Reject stale topology and explicit assignments to another pipeline, + * except for a direct route that follows its CRTC (12.3/13.5, see + * dcp_typec_follow_crtc()): dcp_crtc_atomic_check() decides there. */ if (apple_connector->port_encoder) { + struct platform_device *pdev = READ_ONCE(apple_connector->dcp); + conn_state = drm_atomic_get_new_connector_state(state, conn); - if (!conn_state || !conn_state->crtc || - READ_ONCE(apple_connector->dcp) != - to_apple_crtc(conn_state->crtc)->dcp) + if (!conn_state || !conn_state->crtc || !pdev) + return NULL; + if (pdev != to_apple_crtc(conn_state->crtc)->dcp && + !dcp_typec_follows_crtc(platform_get_drvdata(pdev))) return NULL; return apple_connector->port_encoder; } diff --git a/drivers/gpu/drm/apple/connector.h b/drivers/gpu/drm/apple/connector.h index 03a5ac620c429..6e7ec1badc321 100644 --- a/drivers/gpu/drm/apple/connector.h +++ b/drivers/gpu/drm/apple/connector.h @@ -16,6 +16,8 @@ struct apple_connector; void dcp_hotplug(struct work_struct *work); bool dcp_connector_connected(struct apple_connector *connector); +void dcp_queue_hotplug(struct apple_connector *connector); +void dcp_route_failure_notify(struct apple_connector *connector); void dcp_retrain_oob(struct apple_connector *connector); enum drm_connector_status apple_connector_detect(struct drm_connector *connector, bool force); @@ -59,6 +61,7 @@ struct apple_connector { /* Workqueue for sending hotplug events to the associated device */ struct work_struct hotplug_wq; + atomic_t hotplug_reasons; struct mutex chunk_lock; diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.c b/drivers/gpu/drm/apple/dcp-fabric-core.c index d0b1a9e7de8ca..53fd9d48a60f1 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.c +++ b/drivers/gpu/drm/apple/dcp-fabric-core.c @@ -321,6 +321,94 @@ dcp_fabric_tunnel_candidate(const struct dcp_fabric_port *port, } EXPORT_SYMBOL_GPL(dcp_fabric_tunnel_candidate); +/* + * A modeset gives a Type-C stream routed through @from the CRTC of the + * pipeline its port reaches through @to (NULL: none). Stay, move the + * route there, swap it with @holder, the route of another port holding + * that pipeline, or refuse the modeset. A move needs a pipeline that is + * free but for its CRTC: no live fixed output and its services up. A + * fixed-output settling window follows the existing fabric policy. + * A swap needs a holder whose own display is off, or about to be + * shown on @from's pipeline, and that can reach @from's pipeline through + * @holder_back. + */ +enum dcp_fabric_follow +dcp_fabric_follow(const struct dcp_fabric_route *from, + const struct dcp_fabric_route *to, + const struct dcp_fabric_route *holder, + const struct dcp_fabric_route *holder_back, bool holder_off, + const struct dcp_fabric_policy *policy) +{ + const struct dcp_fabric_pipeline *pipeline; + + if (!from || !to || from->tunnel || (holder && holder->tunnel)) + return DCP_FABRIC_FOLLOW_REFUSE; + if (from == to) + return DCP_FABRIC_FOLLOW_STAY; + pipeline = to->pipeline; + if (!pipeline->bound || pipeline->fixed_busy || pipeline->terminal || + !pipeline->services_ready || + (!policy->dual_stream && pipeline->presence != DCP_FABRIC_ABSENT)) + return DCP_FABRIC_FOLLOW_REFUSE; + if (!holder) + return pipeline->owned || pipeline->fixed_recent ? DCP_FABRIC_FOLLOW_REFUSE : + DCP_FABRIC_FOLLOW_MOVE; + if (holder == from || holder->pipeline != pipeline || !holder_off || + !holder_back || holder_back->pipeline != from->pipeline) + return DCP_FABRIC_FOLLOW_REFUSE; + return DCP_FABRIC_FOLLOW_SWAP; +} +EXPORT_SYMBOL_GPL(dcp_fabric_follow); + +int dcp_fabric_follow_execute(const struct dcp_fabric_follow_ops *ops, + void *ctx, unsigned int count) +{ + unsigned int i, detached = 0, attached = 0; + int ret; + + if (!count || count > 2) + return -EINVAL; + for (i = 0; i < count; i++) { + ret = ops->prepare(ctx, i); + if (ret) + return ret; + } + for (i = 0; i < count; i++) { + ret = ops->validate(ctx, i); + if (ret) + return ret; + } + for (i = 0; i < count; i++) { + /* A failed release may already have relinquished ownership. */ + detached++; + ret = ops->detach(ctx, i, false); + if (ret) + goto rollback; + } + for (i = 0; i < count; i++) { + ret = ops->attach(ctx, i, false); + if (ret) + goto rollback; + attached++; + } + for (i = 0; i < count; i++) + ops->publish(ctx, i, false); + return 0; + +rollback: + /* Only a failed original restore makes a connector terminally lost. */ + while (attached) + ops->detach(ctx, --attached, true); + for (i = 0; i < detached; i++) { + if (ops->attach(ctx, i, true)) + ops->lost(ctx, i); + else + ops->publish(ctx, i, true); + } + return ret; +} +EXPORT_SYMBOL_GPL(dcp_fabric_follow_execute); + unsigned int dcp_fabric_deactivate_steps(int error, bool selected, bool fixed_live, enum dcp_fabric_deactivate_step steps[3]) { diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.h b/drivers/gpu/drm/apple/dcp-fabric-core.h index 71c1cf75ae999..0792b25a6b388 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.h +++ b/drivers/gpu/drm/apple/dcp-fabric-core.h @@ -22,6 +22,7 @@ struct dcp_fabric_pipeline { bool bound; bool has_fixed; bool fixed_busy; + bool fixed_recent; bool owned; bool tunnel_held; enum dcp_fabric_presence_state presence; @@ -97,6 +98,33 @@ unsigned int dcp_fabric_deactivate_steps(int error, bool selected, bool fixed_li enum dcp_fabric_deactivate_step steps[3]); void dcp_fabric_resume_steps(enum dcp_fabric_resume_step steps[2]); +enum dcp_fabric_follow { + DCP_FABRIC_FOLLOW_STAY, + DCP_FABRIC_FOLLOW_MOVE, + DCP_FABRIC_FOLLOW_SWAP, + DCP_FABRIC_FOLLOW_REFUSE, +}; + +enum dcp_fabric_follow +dcp_fabric_follow(const struct dcp_fabric_route *from, + const struct dcp_fabric_route *to, + const struct dcp_fabric_route *holder, + const struct dcp_fabric_route *holder_back, bool holder_off, + const struct dcp_fabric_policy *policy); + +/* All preparation completes before destructive effects; failures restore sources. */ +struct dcp_fabric_follow_ops { + int (*prepare)(void *ctx, unsigned int slot); + int (*validate)(void *ctx, unsigned int slot); + int (*detach)(void *ctx, unsigned int slot, bool destination); + int (*attach)(void *ctx, unsigned int slot, bool restore); + void (*publish)(void *ctx, unsigned int slot, bool restore); + void (*lost)(void *ctx, unsigned int slot); +}; + +int dcp_fabric_follow_execute(const struct dcp_fabric_follow_ops *ops, + void *ctx, unsigned int count); + enum dcp_fabric_attach_action { DCP_FABRIC_ATTACH_OOB, DCP_FABRIC_ATTACH_WORK, diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c index 3652a404fbcac..533c834dfe8a2 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-test.c +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -607,6 +607,267 @@ static void fabric_unbound_and_mask_test(struct kunit *test) (u32)(BIT(1) | BIT(2))); } +static void fabric_follow_test(struct kunit *test) +{ + struct fabric_fixture f; + struct dcp_fabric_route *lg, *hybrid; + + /* A direct port on the second pipeline is paired with the first CRTC. */ + fabric_init(&f, false); + lg = &f.route[0][1]; + hybrid = &f.route[0][0]; + f.pipeline[1].owned = true; + KUNIT_EXPECT_EQ(test, dcp_fabric_follow(lg, hybrid, NULL, NULL, false, + &f.policy), + DCP_FABRIC_FOLLOW_MOVE); + KUNIT_EXPECT_EQ(test, dcp_fabric_follow(lg, lg, NULL, NULL, false, + &f.policy), + DCP_FABRIC_FOLLOW_STAY); + KUNIT_EXPECT_EQ(test, dcp_fabric_follow(NULL, hybrid, NULL, NULL, false, + &f.policy), + DCP_FABRIC_FOLLOW_REFUSE); + KUNIT_EXPECT_EQ(test, dcp_fabric_follow(lg, NULL, NULL, NULL, false, + &f.policy), + DCP_FABRIC_FOLLOW_REFUSE); + + /* Existing fixed-output busy/settling policy also applies to a move. */ + f.pipeline[0].fixed_busy = true; + KUNIT_EXPECT_EQ(test, dcp_fabric_follow(lg, hybrid, NULL, NULL, false, + &f.policy), + DCP_FABRIC_FOLLOW_REFUSE); + f.pipeline[0].fixed_busy = false; + f.pipeline[0].presence = DCP_FABRIC_SETTLING; + KUNIT_EXPECT_EQ(test, dcp_fabric_follow(lg, hybrid, NULL, NULL, false, + &f.policy), + DCP_FABRIC_FOLLOW_REFUSE); + f.pipeline[0].presence = DCP_FABRIC_ABSENT; + + /* Owned with no route to swap with (a retiring tunnel), unbound, not up. */ + f.pipeline[0].owned = true; + KUNIT_EXPECT_EQ(test, dcp_fabric_follow(lg, hybrid, NULL, NULL, false, + &f.policy), + DCP_FABRIC_FOLLOW_REFUSE); + f.pipeline[0].owned = false; + f.pipeline[0].bound = false; + KUNIT_EXPECT_EQ(test, dcp_fabric_follow(lg, hybrid, NULL, NULL, false, + &f.policy), + DCP_FABRIC_FOLLOW_REFUSE); + f.pipeline[0].bound = true; + f.pipeline[0].services_ready = false; + KUNIT_EXPECT_EQ(test, dcp_fabric_follow(lg, hybrid, NULL, NULL, false, + &f.policy), + DCP_FABRIC_FOLLOW_REFUSE); + f.pipeline[0].services_ready = true; + + /* #39 on the M2 Max: DPMS on pairs two direct ports the other way round. */ + fabric_init(&f, true); + f.pipeline[0].owned = true; + f.pipeline[1].owned = true; + KUNIT_EXPECT_EQ(test, dcp_fabric_follow(&f.route[0][1], &f.route[0][0], + &f.route[1][0], &f.route[1][1], + true, &f.policy), + DCP_FABRIC_FOLLOW_SWAP); + /* the holder's display is still lit elsewhere */ + KUNIT_EXPECT_EQ(test, dcp_fabric_follow(&f.route[0][1], &f.route[0][0], + &f.route[1][0], &f.route[1][1], + false, &f.policy), + DCP_FABRIC_FOLLOW_REFUSE); + /* the holder cannot reach the pipeline it would be given */ + KUNIT_EXPECT_EQ(test, dcp_fabric_follow(&f.route[0][1], &f.route[0][0], + &f.route[1][0], NULL, true, + &f.policy), + DCP_FABRIC_FOLLOW_REFUSE); + KUNIT_EXPECT_EQ(test, dcp_fabric_follow(&f.route[0][1], &f.route[0][0], + &f.route[1][0], &f.route[1][0], + true, &f.policy), + DCP_FABRIC_FOLLOW_REFUSE); + /* A direct route must never move into or swap a tunnel binding. */ + f.route[1][0].tunnel = true; + KUNIT_EXPECT_EQ(test, + dcp_fabric_follow(&f.route[0][1], &f.route[0][0], + &f.route[1][0], &f.route[1][1], true, &f.policy), + DCP_FABRIC_FOLLOW_REFUSE); + f.route[1][0].tunnel = false; + f.route[0][1].tunnel = true; + KUNIT_EXPECT_EQ(test, + dcp_fabric_follow(&f.route[0][1], &f.route[0][0], + NULL, NULL, false, &f.policy), + DCP_FABRIC_FOLLOW_REFUSE); +} + +static void fabric_follow_recent_fixed(struct kunit *test) +{ + struct fabric_fixture f; + + fabric_init(&f, true); + f.pipeline[0].fixed_recent = true; + KUNIT_EXPECT_EQ(test, + dcp_fabric_follow(&f.route[0][1], &f.route[0][0], + NULL, NULL, false, &f.policy), + DCP_FABRIC_FOLLOW_REFUSE); + f.pipeline[0].fixed_recent = false; + KUNIT_EXPECT_EQ(test, + dcp_fabric_follow(&f.route[0][1], &f.route[0][0], + NULL, NULL, false, &f.policy), + DCP_FABRIC_FOLLOW_MOVE); + f.pipeline[0].fixed_recent = true; + f.pipeline[0].owned = true; + f.pipeline[1].owned = true; + KUNIT_EXPECT_EQ(test, + dcp_fabric_follow(&f.route[0][1], &f.route[0][0], + &f.route[1][0], &f.route[1][1], true, &f.policy), + DCP_FABRIC_FOLLOW_SWAP); +} + +struct follow_effect_fixture { + int owner[2]; + int prepare_error; + int validate_error; + int attach_error; + int restore_error; + int destination_detach_error; + unsigned int prepared; + unsigned int detached; + unsigned int published; + unsigned int lost; +}; + +static int follow_effect_prepare(void *data, unsigned int slot) +{ + struct follow_effect_fixture *f = data; + + f->prepared++; + return f->prepare_error == slot + 1 ? -ENOMEM : 0; +} + +static int follow_effect_validate(void *data, unsigned int slot) +{ + struct follow_effect_fixture *f = data; + + return f->validate_error == slot + 1 ? -ESTALE : 0; +} + +static int follow_effect_detach(void *data, unsigned int slot, bool destination) +{ + struct follow_effect_fixture *f = data; + + f->owner[destination ? 1 - slot : slot] = -1; + f->detached++; + return destination && f->destination_detach_error == slot + 1 ? -EIO : 0; +} + +static int follow_effect_attach(void *data, unsigned int slot, bool restore) +{ + struct follow_effect_fixture *f = data; + + if ((!restore && f->attach_error == slot + 1) || + (restore && f->restore_error == slot + 1)) + return -EIO; + f->owner[restore ? slot : 1 - slot] = slot; + return 0; +} + +static void follow_effect_publish(void *data, unsigned int slot, bool restore) +{ + struct follow_effect_fixture *f = data; + + f->published |= BIT(slot); +} + +static void follow_effect_lost(void *data, unsigned int slot) +{ + struct follow_effect_fixture *f = data; + + f->lost |= BIT(slot); +} + +static const struct dcp_fabric_follow_ops follow_effect_ops = { + .prepare = follow_effect_prepare, + .validate = follow_effect_validate, + .detach = follow_effect_detach, + .attach = follow_effect_attach, + .publish = follow_effect_publish, + .lost = follow_effect_lost, +}; + +static void fabric_follow_effect_swap(struct kunit *test) +{ + struct follow_effect_fixture f = { .owner = { 0, 1 } }; + + KUNIT_ASSERT_EQ(test, dcp_fabric_follow_execute(&follow_effect_ops, &f, 2), 0); + KUNIT_EXPECT_EQ(test, f.owner[0], 1); + KUNIT_EXPECT_EQ(test, f.owner[1], 0); + KUNIT_EXPECT_EQ(test, f.prepared, 2U); + KUNIT_EXPECT_EQ(test, f.published, 3U); +} + +static void fabric_follow_effect_oom(struct kunit *test) +{ + unsigned int slot; + + for (slot = 1; slot <= 2; slot++) { + struct follow_effect_fixture f = { .owner = { 0, 1 }, .prepare_error = slot }; + + KUNIT_EXPECT_EQ(test, + dcp_fabric_follow_execute(&follow_effect_ops, &f, 2), -ENOMEM); + KUNIT_EXPECT_EQ(test, f.detached, 0U); + KUNIT_EXPECT_EQ(test, f.owner[0], 0); + KUNIT_EXPECT_EQ(test, f.owner[1], 1); + KUNIT_EXPECT_EQ(test, f.published, 0U); + } +} + +static void fabric_follow_effect_stale(struct kunit *test) +{ + struct follow_effect_fixture f = { .owner = { 0, 1 }, .validate_error = 2 }; + + KUNIT_EXPECT_EQ(test, dcp_fabric_follow_execute(&follow_effect_ops, &f, 2), -ESTALE); + KUNIT_EXPECT_EQ(test, f.detached, 0U); + KUNIT_EXPECT_EQ(test, f.owner[0], 0); + KUNIT_EXPECT_EQ(test, f.owner[1], 1); +} + +static void fabric_follow_effect_rollback(struct kunit *test) +{ + unsigned int slot; + + for (slot = 1; slot <= 2; slot++) { + struct follow_effect_fixture f = { .owner = { 0, 1 }, .attach_error = slot }; + + KUNIT_EXPECT_EQ(test, dcp_fabric_follow_execute(&follow_effect_ops, &f, 2), -EIO); + KUNIT_EXPECT_EQ(test, f.owner[0], 0); + KUNIT_EXPECT_EQ(test, f.owner[1], 1); + KUNIT_EXPECT_EQ(test, f.published, 3U); + KUNIT_EXPECT_EQ(test, f.lost, 0U); + } +} + +static void fabric_follow_effect_release_restore(struct kunit *test) +{ + struct follow_effect_fixture f = { + .owner = { 0, 1 }, .attach_error = 2, .destination_detach_error = 1, + }; + + KUNIT_EXPECT_EQ(test, dcp_fabric_follow_execute(&follow_effect_ops, &f, 2), -EIO); + KUNIT_EXPECT_EQ(test, f.owner[0], 0); + KUNIT_EXPECT_EQ(test, f.owner[1], 1); + KUNIT_EXPECT_EQ(test, f.lost, 0U); + KUNIT_EXPECT_EQ(test, f.published, 3U); +} + +static void fabric_follow_effect_lost(struct kunit *test) +{ + struct follow_effect_fixture f = { + .owner = { 0, 1 }, .attach_error = 2, .restore_error = 1, + }; + + KUNIT_EXPECT_EQ(test, dcp_fabric_follow_execute(&follow_effect_ops, &f, 2), -EIO); + KUNIT_EXPECT_EQ(test, f.owner[0], -1); + KUNIT_EXPECT_EQ(test, f.owner[1], 1); + KUNIT_EXPECT_EQ(test, f.lost, (unsigned int)BIT(0)); + KUNIT_EXPECT_EQ(test, f.published, (unsigned int)BIT(1)); +} + static void fabric_presence_wrap_test(struct kunit *test) { struct dcp_fabric_presence presence = {}; @@ -1528,6 +1789,14 @@ static struct kunit_case fabric_tests[] = { KUNIT_CASE(fabric_effect_failure_core_test), KUNIT_CASE(fabric_deactivate_failure_test), KUNIT_CASE(fabric_unbound_and_mask_test), + KUNIT_CASE(fabric_follow_test), + KUNIT_CASE(fabric_follow_recent_fixed), + KUNIT_CASE(fabric_follow_effect_swap), + KUNIT_CASE(fabric_follow_effect_oom), + KUNIT_CASE(fabric_follow_effect_stale), + KUNIT_CASE(fabric_follow_effect_rollback), + KUNIT_CASE(fabric_follow_effect_lost), + KUNIT_CASE(fabric_follow_effect_release_restore), KUNIT_CASE(fabric_presence_wrap_test), KUNIT_CASE(fabric_masked_edge_test), KUNIT_CASE_PARAM(fabric_wiring_test, fabric_wiring_gen_params), diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index cd218c53e4579..cefbe27cf93b6 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -26,6 +26,12 @@ #include "ibootep.h" #include "dcp-internal.h" #include "parser.h" + +static bool typec_follow_crtc = true; +module_param(typec_follow_crtc, bool, 0444); +MODULE_PARM_DESC(typec_follow_crtc, + "Let direct DP-alt routes follow their selected CRTC (default: on)"); + struct apple_dcp_typec_port { struct dcp_fabric_port core; struct dcp_fabric_plan plan; @@ -180,6 +186,26 @@ bool dcp_typec_dual_stream(void) return atomic_read(&dcp_dual_stream_routes) > 0; } +/* + * Do the Type-C routes of @dcp follow the CRTC a modeset pairs them with + * (see dcp_typec_follow_crtc())? This capability applies to direct routes + * on 12.3/13.5 only. + */ +bool dcp_typec_follows_crtc(struct apple_dcp *dcp) +{ + return typec_follow_crtc && dcp->nr_typec_routes && + !dcp->external && !dcp->external_native && + (dcp->fw_compat == DCP_FIRMWARE_V_12_3 || + dcp->fw_compat == DCP_FIRMWARE_V_13_5); +} + +/* Is a routed port's possible_crtcs narrowed to the pipeline driving it? */ +static bool dcp_typec_narrows(struct apple_dcp *dcp, bool tunnel) +{ + /* This tree narrows on dual-stream machines too. */ + return tunnel || !dcp_typec_follows_crtc(dcp); +} + bool dcp_is_typec_only(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); @@ -287,8 +313,12 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, * rejected with no way for it to recover. The hotplug that * follows publishes the actual binding on dual-stream machines * too. Clients must re-read encoder topology after a hotplug. + * Pipelines whose direct routes follow the CRTC they are + * paired with keep the full set: the pairing is the + * compositor's to make there. */ - if (connector->port_encoder && dcp->crtc) + if (connector->port_encoder && dcp->crtc && + dcp_typec_narrows(dcp, xbar != route->xbar)) connector->port_encoder->possible_crtcs = dcp_fabric_connector_mask(false, true, true, @@ -745,7 +775,7 @@ void dcp_fabric_hdmi_reinit(struct apple_dcp *dcp, const char *why) dcp_fixed_hdmi_reinit_locked(dcp, why); } -/* A Thunderbolt tunnel holds @dcp's pipeline: it never moves. */ +/* A Thunderbolt tunnel holds its pipeline; direct moves never steal it. */ static bool dcp_typec_tunnel_held(struct apple_dcp *dcp) { return dcp->active_typec_route && dcp->active_typec_route->tunnel; @@ -1673,6 +1703,19 @@ struct dcp_tb_attach_context { void *binding; }; +/* The crossbar control feeding DP IN @dpin of @route's port to its pipeline */ +static struct mux_control * +dcp_typec_tunnel_ctl(struct apple_dcp_typec_route *route, unsigned int dpin) +{ + if (route->dpin[dpin]) + return route->dpin[dpin]; + /* Legacy DT ABI: unnamed DPIN controls share the DP-alt chip. */ + if (route->xbar != &route->xbar->chip->mux[0] || + route->xbar->chip->controllers < 3) + return NULL; + return &route->xbar->chip->mux[1 + dpin]; +} + static int dcp_tb_candidate(void *data) { struct dcp_tb_attach_context *ctx = data; @@ -1711,16 +1754,9 @@ static int dcp_tb_candidate(void *data) return ret; } - ctl = best->dpin[dpin]; - if (!ctl) { - /* Legacy DT ABI: unnamed DPIN controls share the DP-alt chip. */ - if (best->xbar != &best->xbar->chip->mux[0] || - best->xbar->chip->controllers < 3) { - ret = -EOPNOTSUPP; - return ret; - } - ctl = &best->xbar->chip->mux[1 + dpin]; - } + ctl = dcp_typec_tunnel_ctl(best, dpin); + if (!ctl) + return -EOPNOTSUPP; ctx->best = best; ctx->ctl = ctl; @@ -1928,6 +1964,522 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, } EXPORT_SYMBOL_GPL(apple_dcp_tb_dp_tunnel); +/* + * Direct DP-alt routes follow the CRTC selected by a modeset on 12.3/13.5. + * Tunnels keep their existing masks, binding lifetime and routing policy. + * Firmware 14.7 and native external processors keep their existing path. + */ +struct dcp_typec_follow { + struct apple_dcp_typec_port *port; + struct apple_dcp_typec_route *from; + struct apple_dcp_typec_route *to; + struct apple_dcp_typec_route *holder; + struct apple_dcp_typec_route *holder_back; + enum dcp_fabric_follow action; +}; + +/* Catalog snapshots are allocated before either route is disturbed. */ +struct dcp_typec_follow_slot { + struct apple_dcp_typec_route *from; + struct apple_dcp_typec_route *to; + struct apple_dcp_typec_port *port; + struct apple_connector *connector; + struct dcp_display_mode *modes; + unsigned int nr_modes; + u64 generation; + u64 attachment_generation; + bool was_active; + bool restored; +}; + +struct dcp_typec_follow_context { + struct dcp_typec_follow_slot slots[2]; + struct drm_atomic_commit *state; +}; + +/* + * The fabric lock, from a modeset. A holder of it can wait for the + * modeset locks: a display that goes sends a hotplug event, and with no + * compositor running the fbdev client answers that with a commit. Rather + * than deadlock with it, a modeset gives up after a while, and is then + * refused or left on its pipeline as if the route could not follow. + */ +#define DCP_FOLLOW_LOCK_MS 3000 + +/* This tree serializes Thunderbolt handoffs under the fabric lock too. */ +static bool dcp_typec_follow_lock(bool handoff) +{ + unsigned long timeout = jiffies + msecs_to_jiffies(DCP_FOLLOW_LOCK_MS); + + for (;;) { + if (mutex_trylock(&dcp_typec_fabric_lock)) + return true; + if (time_after(jiffies, timeout)) + return false; + msleep(20); + } +} + +static void dcp_typec_follow_unlock(bool handoff) +{ + mutex_unlock(&dcp_typec_fabric_lock); +} + +static struct apple_dcp_typec_port * +dcp_typec_port_of(struct apple_connector *connector, bool *secondary) +{ + struct apple_dcp_typec_port *port; + + list_for_each_entry(port, &dcp_typec_ports, link) { + if (port->connector != connector && + port->secondary_connector != connector) + continue; + *secondary = port->secondary_connector == connector; + return port; + } + return NULL; +} + +static struct apple_dcp_typec_route * +dcp_typec_port_route(struct apple_dcp_typec_port *port, struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + + list_for_each_entry(route, &port->routes, port_link) + if (route->dcp == dcp) + return route; + return NULL; +} + +/* + * Is the display of @connector off in @state, or to be shown on @back's + * pipeline, so that its route may give @crtc's pipeline up for that one? + */ +static bool dcp_typec_follow_holder_off(struct drm_atomic_commit *state, + struct drm_crtc *crtc, + struct apple_connector *connector, + struct apple_dcp *back) +{ + struct drm_connector_state *conn_state = NULL; + struct drm_crtc_state *crtc_state; + struct drm_crtc *other; + + if (connector) { + conn_state = drm_atomic_get_new_connector_state(state, &connector->base); + if (!conn_state) + conn_state = connector->base.state; + } + other = conn_state ? conn_state->crtc : NULL; + if (!other) + return true; + if (other == crtc) + return false; + if (back->crtc && other == &back->crtc->base) + return true; + crtc_state = drm_atomic_get_new_crtc_state(state, other); + if (!crtc_state) + crtc_state = other->state; + return crtc_state && !crtc_state->active; +} + +/* May @connector's route follow @crtc to @dcp's pipeline, and how? */ +static int dcp_typec_follow_decide(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *state, + struct apple_connector *connector, + struct dcp_typec_follow *follow) +{ + struct dcp_fabric_policy policy = { .dual_stream = dcp_typec_dual_stream() }; + struct apple_dcp_typec_route *from, *to, *holder, *back = NULL; + struct apple_dcp_typec_port *port, *other = NULL; + bool secondary, off = false; + + lockdep_assert_held(&dcp_typec_fabric_lock); + memset(follow, 0, sizeof(*follow)); + follow->action = DCP_FABRIC_FOLLOW_REFUSE; + port = dcp_typec_port_of(connector, &secondary); + if (!port) + return -EINVAL; + from = secondary ? port->secondary_owner : port->owner; + to = dcp_typec_port_route(port, dcp); + if (from && from == to) { + follow->action = DCP_FABRIC_FOLLOW_STAY; + return 0; + } + /* + * Not routed (yet): nothing to follow, the modeset is checked as it + * stands, as when resume restores a display whose tunnel is not back. + */ + if (!from || !READ_ONCE(connector->connected)) + return -ENOENT; + /* A dock's second stream has one pipeline, and the plan rules first. */ + /* A port in SAFE keeps its owner until the mux set phase completes. */ + if (secondary || from->tunnel || port->safe_pending || + !dcp_typec_follows_crtc(from->dcp) || dcp_typec_keep_order()) + return -EINVAL; + holder = dcp->active_typec_route; + if (holder) { + other = holder->port; + if (holder != other->owner || holder->tunnel || other->safe_pending) + return -EINVAL; + back = dcp_typec_port_route(other, from->dcp); + off = dcp_typec_follow_holder_off(state, crtc, other->connector, + from->dcp); + /* the port's own snapshot below decides for @dcp's pipeline */ + dcp_fabric_snapshot_port(other, holder->tunnel); + } + dcp_fabric_snapshot_port(port, from->tunnel); + /* Follow-only reservation; arrival allocation retains its existing policy. */ + if (to) + to->core.pipeline->fixed_recent = dcp->hdmi_hpd && dcp->fixed_phy && + READ_ONCE(dcp->hdmi_edge_seen) && + time_before(jiffies, READ_ONCE(dcp->hdmi_edge_jiffies) + + msecs_to_jiffies(DCP_HDMI_HOLD_MS)); + follow->action = dcp_fabric_follow(&from->core, to ? &to->core : NULL, + holder ? &holder->core : NULL, + back ? &back->core : NULL, off, &policy); + if (follow->action == DCP_FABRIC_FOLLOW_REFUSE) + return -EINVAL; + follow->port = port; + follow->from = from; + follow->to = to; + follow->holder = holder; + follow->holder_back = back; + return 0; +} + +static int dcp_follow_prepare(void *data, unsigned int index) +{ + struct dcp_typec_follow_context *ctx = data; + struct dcp_typec_follow_slot *slot = &ctx->slots[index]; + + slot->port = slot->from->port; + slot->connector = slot->from->dcp->typec_connector; + { + struct drm_crtc *crtc = &slot->from->dcp->crtc->base; + struct drm_crtc_state *old = drm_atomic_get_old_crtc_state(ctx->state, crtc); + + slot->was_active = old && old->active; + } + slot->modes = dcp_modes_dup(slot->from->dcp, slot->connector, + &slot->nr_modes, &slot->generation); + if (IS_ERR(slot->modes)) { + int ret = PTR_ERR(slot->modes); + + slot->modes = NULL; + return ret; + } + return 0; +} + +static int dcp_follow_validate(void *data, unsigned int index) +{ + struct dcp_typec_follow_context *ctx = data; + struct dcp_typec_follow_slot *slot = &ctx->slots[index]; + struct apple_dcp *dcp = slot->from->dcp; + + guard(mutex)(&dcp->modes_lock); + if (dcp->modes_generation != slot->generation || + !dcp_modes_for_connector(dcp, slot->connector) || + dcp->active_typec_route != slot->from || + READ_ONCE(slot->connector->dcp) != to_platform_device(dcp->dev)) + return -ESTALE; + return 0; +} + +static int dcp_follow_release(struct apple_dcp_typec_route *route) +{ + struct apple_dcp *dcp = route->dcp; + int ret; + + /* Revoked sessions cannot touch the new route after a reconnect wait. */ + dcp_dptx_park(dcp); + apple_connector_edid_set_live(dcp->typec_connector, false); + dcp_modes_begin_attachment(dcp); + dcp->typec_connector = NULL; + WRITE_ONCE(dcp->connector, dcp->fixed_connector); + scoped_guard(mutex, &dcp->hpd_mutex) + WRITE_ONCE(dcp->typec_cable_connected, false); + ret = dcp_typec_route_deactivate(route); + if (route->port->owner == route) + route->port->owner = NULL; + return ret; +} + +static int dcp_follow_detach(void *data, unsigned int index, bool destination) +{ + struct dcp_typec_follow_context *ctx = data; + struct dcp_typec_follow_slot *slot = &ctx->slots[index]; + + return dcp_follow_release(destination ? slot->to : slot->from); +} + +static int dcp_follow_attach(void *data, unsigned int index, bool restore) +{ + struct dcp_typec_follow_context *ctx = data; + struct dcp_typec_follow_slot *slot = &ctx->slots[index]; + struct apple_dcp_typec_route *route = restore ? slot->from : slot->to; + struct apple_dcp *dcp = route->dcp; + int ret; + + ret = dcp_typec_route_activate(route, route->xbar); + if (ret) + return ret; + slot->attachment_generation = dcp_modes_transfer_begin(dcp); + slot->port->owner = route; + slot->port->preferred_route = route; + scoped_guard(mutex, &dcp->hpd_mutex) { + WRITE_ONCE(dcp->typec_cable_connected, true); + dcp->typec_generation++; + dcp->typec_follow_start = true; + dcp->typec_follow_gen = dcp->typec_generation; + dcp->typec_reconnect_tries = 0; + dcp->placeholder_retried = false; + reinit_completion(&dcp->typec_iomfb_hpd_ready); + WRITE_ONCE(dcp->typec_crtc_off, true); + } + return 0; +} + +static void dcp_follow_publish(void *data, unsigned int index, bool restore) +{ + struct dcp_typec_follow_context *ctx = data; + struct dcp_typec_follow_slot *slot = &ctx->slots[index]; + struct apple_dcp *dcp = (restore ? slot->from : slot->to)->dcp; + + dcp_modes_adopt(dcp, slot->connector, slot->attachment_generation, slot->modes, + slot->nr_modes); + slot->modes = NULL; + slot->restored = restore; +} + +static void dcp_follow_lost(void *data, unsigned int index) +{ + struct dcp_typec_follow_context *ctx = data; + struct apple_connector *connector = ctx->slots[index].connector; + + if (connector) { + WRITE_ONCE(connector->connected, false); + apple_connector_set_pipeline(connector, NULL); + dcp_queue_hotplug(connector); + } +} + +static const struct dcp_fabric_follow_ops dcp_follow_ops = { + .prepare = dcp_follow_prepare, + .validate = dcp_follow_validate, + .detach = dcp_follow_detach, + .attach = dcp_follow_attach, + .publish = dcp_follow_publish, + .lost = dcp_follow_lost, +}; + +static int dcp_typec_follow_move(struct dcp_typec_follow *follow, + struct drm_atomic_commit *state) +{ + struct dcp_typec_follow_context ctx = { + .state = state, + .slots[0] = { .from = follow->from, .to = follow->to }, + .slots[1] = { .from = follow->holder, .to = follow->holder_back }, + }; + unsigned int count = follow->action == DCP_FABRIC_FOLLOW_SWAP ? 2 : 1; + unsigned int i; + int ret; + + /* Keep newly queued and already waiting sessions outside the handoff. */ + for (i = 0; i < count; i++) { + WRITE_ONCE(ctx.slots[i].from->dcp->typec_follow_retiring, true); + WRITE_ONCE(ctx.slots[i].to->dcp->typec_follow_retiring, true); + } + ret = dcp_fabric_follow_execute(&dcp_follow_ops, &ctx, count); + if (ret) + to_apple_atomic_state(state)->failed_routes |= + drm_crtc_mask(&follow->from->dcp->crtc->base) | + drm_crtc_mask(&follow->to->dcp->crtc->base); + for (i = 0; i < count; i++) { + WRITE_ONCE(ctx.slots[i].from->dcp->typec_follow_retiring, false); + WRITE_ONCE(ctx.slots[i].to->dcp->typec_follow_retiring, false); + } + for (i = 0; i < count; i++) { + struct dcp_typec_follow_slot *slot = &ctx.slots[i]; + + if (slot->restored && slot->was_active) { + int error = dcp_dptx_connect(slot->from->dcp, 0); + + if (error) + dev_err(slot->from->dcp->dev, + "restored route link could not reconnect: %d\n", error); + } + } + if (!ret && count == 1) { + struct apple_dcp *freed = ctx.slots[0].from->dcp; + + if (freed->hdmi_hpd && freed->active && + gpiod_get_value_cansleep(freed->hdmi_hpd)) + dcp_dptx_connect(freed, 0); + dcp_typec_route_waiting(); + } + kfree(ctx.slots[0].modes); + kfree(ctx.slots[1].modes); + if (ret) { + for (i = 0; i < count; i++) + if (ctx.slots[i].connector) + dcp_route_failure_notify(ctx.slots[i].connector); + } + return ret; +} + +/* + * atomic_check: may the Type-C displays @state puts on @crtc be driven by + * its pipeline @dcp? Validate the source catalog under fabric ownership; + * commit revalidates the route before making any destructive changes. + */ +int dcp_typec_follow_check(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *state, + const struct drm_display_mode *mode) +{ + struct drm_connector_state *conn_state; + struct dcp_typec_follow follow; + struct drm_connector *conn; + int i, ret; + + for_each_new_connector_in_state(state, conn, conn_state, i) { + struct apple_connector *connector = to_apple_connector(conn); + + if (conn_state->crtc != crtc || !connector->port_encoder || + READ_ONCE(connector->dcp) == to_platform_device(dcp->dev)) + continue; + if (!state->allow_modeset) + return -EINVAL; + /* Include both sides in DRM dependency ordering, including off holders. */ + { + struct drm_crtc *other; + + drm_for_each_crtc(other, crtc->dev) { + struct drm_crtc_state *other_state; + + other_state = drm_atomic_get_crtc_state(state, other); + if (IS_ERR(other_state)) + return PTR_ERR(other_state); + } + } + if (!dcp_typec_follow_lock(false)) + return -EBUSY; + ret = dcp_typec_follow_decide(dcp, crtc, state, connector, &follow); + if (!ret && !dcp_has_mode(follow.action == DCP_FABRIC_FOLLOW_STAY ? + dcp : follow.from->dcp, mode)) + ret = -EINVAL; + dcp_typec_follow_unlock(false); + if (ret == -ENOENT) + continue; + if (ret) { + dev_info_ratelimited(dcp->dev, "%s cannot follow its CRTC here\n", + conn->name); + return ret; + } + return 0; + } + return dcp_has_mode(dcp, mode) ? 0 : -EINVAL; +} + +static bool dcp_typec_direct_commit(struct drm_crtc *crtc, + struct drm_atomic_commit *state) +{ + struct drm_connector_state *conn_state; + struct drm_connector *conn; + int i; + + for_each_new_connector_in_state(state, conn, conn_state, i) { + struct apple_connector *connector = to_apple_connector(conn); + struct platform_device *pdev = READ_ONCE(connector->dcp); + struct apple_dcp_typec_route *route; + struct apple_dcp *source; + + if (conn_state->crtc != crtc || !connector->port_encoder || !pdev) + continue; + source = platform_get_drvdata(pdev); + /* The second half of a swap still needs its handoff identity held. */ + if (pdev == to_apple_crtc(crtc)->dcp) { + if (dcp_crtc_needs_route_start(source)) + return true; + continue; + } + route = READ_ONCE(source->active_typec_route); + if (route && !READ_ONCE(route->tunnel) && + dcp_typec_follows_crtc(source)) + return true; + } + return false; +} + +static void dcp_follow_fail_commit(struct drm_crtc *crtc, struct drm_atomic_commit *state) +{ + struct drm_connector_state *conn_state; + struct drm_connector *conn; + int i; + + to_apple_atomic_state(state)->failed_routes |= drm_crtc_mask(crtc); + for_each_new_connector_in_state(state, conn, conn_state, i) { + struct apple_connector *connector = to_apple_connector(conn); + struct platform_device *pdev = READ_ONCE(connector->dcp); + struct apple_dcp *source; + + if (conn_state->crtc != crtc || !connector->port_encoder || !pdev) + continue; + source = platform_get_drvdata(pdev); + if (source->crtc && dcp_typec_follows_crtc(source)) + to_apple_atomic_state(state)->failed_routes |= + drm_crtc_mask(&source->crtc->base); + } +} + +/* Hold route identity through power-on/modeset; caller releases on every exit. */ +int dcp_typec_follow_crtc(struct drm_crtc *crtc, struct drm_atomic_commit *state, + bool *locked) +{ + struct platform_device *pdev = to_apple_crtc(crtc)->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + struct drm_connector_state *conn_state; + struct dcp_typec_follow follow; + struct drm_connector *conn; + int i, ret; + + *locked = false; + if (!dcp_typec_follows_crtc(dcp) || !dcp_typec_direct_commit(crtc, state)) + return 0; + if (!dcp_typec_follow_lock(true)) { + dcp_follow_fail_commit(crtc, state); + return -EBUSY; + } + *locked = true; + for_each_new_connector_in_state(state, conn, conn_state, i) { + struct apple_connector *connector = to_apple_connector(conn); + + if (conn_state->crtc != crtc || !connector->port_encoder || + READ_ONCE(connector->dcp) == pdev) + continue; + ret = dcp_typec_follow_decide(dcp, crtc, state, connector, &follow); + if (!ret && follow.action != DCP_FABRIC_FOLLOW_STAY) { + struct drm_crtc_state *new = drm_atomic_get_new_crtc_state(state, crtc); + + if (!dcp_has_mode(follow.from->dcp, &new->mode)) + ret = -EINVAL; + else + ret = dcp_typec_follow_move(&follow, state); + } + if (ret && ret != -ENOENT) { + dcp_follow_fail_commit(crtc, state); + return ret; + } + } + return 0; +} + +void dcp_typec_follow_done(bool locked) +{ + if (locked) + dcp_typec_follow_unlock(true); +} + static struct apple_dcp_typec_port * dcp_typec_port_get(struct device_node *connector_np) { @@ -2050,7 +2602,8 @@ void dcp_typec_port_set_connector(unsigned int idx, bool secondary, if (owner) { struct apple_dcp *dcp = owner->dcp; - if (dcp->crtc && connector->port_encoder) + if (dcp->crtc && connector->port_encoder && + dcp_typec_narrows(dcp, owner->tunnel)) connector->port_encoder->possible_crtcs = dcp_fabric_connector_mask(false, true, true, @@ -2392,6 +2945,10 @@ static int dcp_fixed_output_select(struct apple_dcp *dcp) */ irqreturn_t dcp_dp2hdmi_hpd_edge(int irq, void *data) { + struct apple_dcp *dcp = data; + + WRITE_ONCE(dcp->hdmi_edge_jiffies, jiffies); + WRITE_ONCE(dcp->hdmi_edge_seen, true); dcp_hdmi_hpd_edge(data); dcp_hdmi_edge(data); diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index 12bc416ea08c8..82baa9d3014fb 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -10,6 +10,9 @@ struct apple_connector; struct apple_crtc; +struct drm_atomic_commit; +struct drm_crtc; +struct drm_display_mode; struct device_node; struct drm_device; struct mux_control; @@ -65,6 +68,13 @@ void dcp_typec_port_set_connector(unsigned int idx, bool secondary, struct apple_connector *connector); bool dcp_typec_dual_stream(void); void dcp_typec_reorder(void); +bool dcp_typec_follows_crtc(struct apple_dcp *dcp); +int dcp_typec_follow_check(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *state, + const struct drm_display_mode *mode); +int dcp_typec_follow_crtc(struct drm_crtc *crtc, struct drm_atomic_commit *state, + bool *locked); +void dcp_typec_follow_done(bool locked); bool dcp_is_typec_only(struct platform_device *pdev); void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, struct apple_connector *connector); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 0eb761324179a..59a39c82c0dcb 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -398,6 +398,7 @@ struct apple_dcp { u64 dimensions_generation; struct work_struct dimensions_wq; bool modes_admitted; + bool modes_provisional; /* Attributes of the connector */ int connector_type; @@ -490,6 +491,9 @@ struct apple_dcp { struct mux_control *direct_xbar_up; /* CRTC powered off while the Type-C cable stays attached */ bool typec_crtc_off; + bool typec_follow_start; + bool typec_follow_retiring; + u64 typec_follow_gen; /* IOMFB reports its video interface ready after DPTX link training. */ struct completion typec_iomfb_hpd_ready; struct delayed_work typec_reconnect_wq; @@ -566,6 +570,8 @@ struct apple_dcp { /* Hardirq, resume and expiry share one generation under this lock. */ spinlock_t hdmi_presence_lock; struct dcp_fabric_presence hdmi_presence; + unsigned long hdmi_edge_jiffies; + bool hdmi_edge_seen; struct delayed_work hdmi_settle_wq; struct delayed_work hdmi_recheck_wq; /* T6030 HDMI converter service requests, see dcp-hdmi.c */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 56df367f4f6ce..fb3791f6d5a9e 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -673,7 +673,7 @@ VISIBLE_IF_KUNIT void dcp_swap_watchdog(struct work_struct *work) dcp_drm_crtc_vblank(dcp->crtc); if (dcp->connector && dcp->swap_watchdog_retrains++ < DCP_SWAP_WATCHDOG_RETRAINS) - schedule_work(&dcp->connector->hotplug_wq); + dcp_queue_hotplug(dcp->connector); } void dcp_swap_watchdog_arm(struct apple_dcp *dcp) @@ -768,7 +768,7 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ if (dcp->connector) { dcp->connector->connected = 0; apple_connector_edid_set_live(dcp->connector, false); - schedule_work(&dcp->connector->hotplug_wq); + dcp_queue_hotplug(dcp->connector); } complete(&dcp->start_done); } @@ -912,6 +912,20 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commi crtc_state->vrr_enabled) return -EINVAL; + /* + * A Type-C display routed to another pipeline: its route follows + * this CRTC if it can (see dcp_typec_follow_check()), and the mode is + * one of the display's, which only the pipeline driving it knows. + */ + if (crtc_state->enable && drm_atomic_crtc_needs_modeset(crtc_state) && + dcp_typec_follows_crtc(dcp)) { + int ret = dcp_typec_follow_check(dcp, crtc, commit, + &crtc_state->mode); + + if (ret) + return ret; + } + needs_modeset = drm_atomic_crtc_needs_modeset(crtc_state) || !READ_ONCE(dcp->mode_state.valid); if (!needs_modeset && (!dcp->connector || !dcp->connector->connected)) { @@ -1351,7 +1365,7 @@ void dcp_external_retry_work(struct work_struct *work) } dev_info(dcp->dev, "display retry: setting the display mode again\n"); dcp_mode_invalidate(&dcp->mode_state); - schedule_work(&connector->hotplug_wq); + dcp_queue_hotplug(connector); return; } /* An HDMI output connects through the fabric, as on its HPD. */ @@ -1685,8 +1699,10 @@ static void dcp_typec_reconnect_work(struct work_struct *work) typec_reconnect_wq); int ret; + /* A route following its CRTC hands the link over itself. */ if ((dcp->external && dcpext_scanout_terminal(dcp)) || - !READ_ONCE(dcp->typec_cable_connected)) + !READ_ONCE(dcp->typec_cable_connected) || + READ_ONCE(dcp->typec_follow_retiring)) return; ret = dcp_dptx_connect(dcp, 0); @@ -1892,6 +1908,32 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) return dcp_dptx_disconnect(dcp, port); } +/* + * Release the display link of a Type-C output while its display stays + * attached, as a CRTC power-off does (see dcp_poweroff()): the firmware's + * unplug for it is ignored, and dcp_poweron() connects the link again. + */ +void dcp_dptx_park(struct apple_dcp *dcp) +{ + int ret; + + scoped_guard(mutex, &dcp->hpd_mutex) { + WRITE_ONCE(dcp->typec_crtc_off, true); + dcp->typec_generation++; + } + /* Generation checks around reconnect waits reject the revoked one. */ + cancel_delayed_work(&dcp->typec_reconnect_wq); + cancel_delayed_work(&dcp->placeholder_edid_wq); + if (dcp->avep) + av_service_disconnect(dcp); + if (dptx_service_available(&dcp->dptxport[0]) && dcp->dptxport[0].connected) { + ret = dptxport_set_hpd(dcp->dptxport[0].service, false); + if (ret) + dev_warn(dcp->dev, "failed to deassert Type-C DPTX HPD: %d\n", ret); + dcp_dptx_disconnect(dcp, 0); + } +} + bool dcp_fw_compat_is_12_x(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); @@ -2144,6 +2186,27 @@ static bool dcp_uses_soft_dpms(struct apple_dcp *dcp) dcp->connector_type == DRM_MODE_CONNECTOR_eDP); } +int dcp_typec_prepare_route(struct apple_dcp *dcp) +{ + u64 generation; + unsigned long ready; + int ret; + + if (!dcp_crtc_needs_route_start(dcp)) + return 0; + scoped_guard(mutex, &dcp->hpd_mutex) + generation = dcp->typec_generation; + ret = dcp_dptx_connect(dcp, 0); + if (ret) + return ret; + ready = wait_for_completion_timeout(&dcp->typec_iomfb_hpd_ready, + msecs_to_jiffies(3000)); + guard(mutex)(&dcp->hpd_mutex); + if (generation != dcp->typec_generation || !dcp->typec_cable_connected) + return -ESTALE; + return ready ? 0 : -ETIMEDOUT; +} + void dcp_poweron(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); @@ -2191,6 +2254,7 @@ void dcp_poweron(struct platform_device *pdev) wait_for_typec_hpd = READ_ONCE(dcp->typec_crtc_off) && READ_ONCE(dcp->typec_cable_connected); WRITE_ONCE(dcp->typec_crtc_off, false); + WRITE_ONCE(dcp->typec_follow_start, false); /* * A Type-C CRTC disable releases its DPTX session. Re-establish it diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index e4ea03a45611f..97ae0dff26ee7 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -16,6 +16,13 @@ bool dcp_h17p_external_supported(struct device_node *np); bool dcp_h17p_external_ready(struct device_node *np); +struct apple_atomic_state { + struct drm_atomic_commit base; + /* Failed route transactions suppress both CRTC enables in this commit. */ + u32 failed_routes; +}; + +#define to_apple_atomic_state(s) container_of(s, struct apple_atomic_state, base) struct apple_crtc { struct drm_crtc base; struct drm_pending_vblank_event *event; @@ -34,6 +41,7 @@ struct apple_encoder { #define to_apple_encoder(x) container_of(x, struct apple_encoder, base) void dcp_poweroff(struct platform_device *pdev); +int dcp_typec_prepare_route(struct apple_dcp *dcp); void dcp_poweron(struct platform_device *pdev); void dcp_external_recover(struct platform_device *pdev); int dcp_set_crc(struct drm_crtc *crtc, bool enabled); @@ -59,6 +67,11 @@ enum drm_mode_status dcp_mode_valid(struct drm_connector *connector, const struct drm_display_mode *mode); int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, struct drm_atomic_commit *commit); +bool dcp_crtc_can_retrain(struct drm_crtc *crtc, struct apple_connector *connector); +bool dcp_crtc_needs_route_start(struct apple_dcp *dcp); +bool dcp_crtc_route_ready(struct drm_crtc *crtc, struct drm_atomic_commit *commit, + bool fresh); +bool dcp_has_mode(struct apple_dcp *dcp, const struct drm_display_mode *mode); bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, const struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode); @@ -74,6 +87,7 @@ void dcp_external_retry(struct apple_dcp *dcp, const char *why, int error, unsigned int base_ms); void dcp_external_retry_work(struct work_struct *work); void dcp_external_sink_irq(struct apple_dcp *dcp); +void dcp_dptx_park(struct apple_dcp *dcp); int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); bool dcp_hold_resume_event(struct platform_device *pdev); diff --git a/drivers/gpu/drm/apple/hotplug-test.c b/drivers/gpu/drm/apple/hotplug-test.c index 8439dc788f60a..15ca6291bf495 100644 --- a/drivers/gpu/drm/apple/hotplug-test.c +++ b/drivers/gpu/drm/apple/hotplug-test.c @@ -231,7 +231,51 @@ static void hotplug_without_hpd(struct kunit *test) KUNIT_EXPECT_FALSE(test, fixture->notification_locked); } +static void hotplug_route_failure_notify(struct kunit *test) +{ + struct hotplug_fixture *fixture = test->priv; + + fixture->connector.connected = true; + WRITE_ONCE(fixture->dcp.mode_state.valid, false); + dcp_route_failure_notify(&fixture->connector); + flush_work(&fixture->connector.hotplug_wq); + KUNIT_EXPECT_EQ(test, fixture->notifications, 1U); + KUNIT_EXPECT_EQ(test, atomic_read(&fixture->connector.hotplug_reasons), 0); + KUNIT_EXPECT_FALSE(test, work_pending(&fixture->connector.hotplug_wq)); +} + +static void hotplug_coalesced_ready(struct kunit *test) +{ + struct hotplug_fixture *fixture = test->priv; + unsigned int order; + + fixture->connector.connected = true; + WRITE_ONCE(fixture->dcp.mode_state.valid, false); + for (order = 0; order < 2; order++) { + /* Hold the work so both reasons are pending before consumption. */ + disable_work_sync(&fixture->connector.hotplug_wq); + if (order) { + dcp_queue_hotplug(&fixture->connector); + dcp_route_failure_notify(&fixture->connector); + } else { + dcp_route_failure_notify(&fixture->connector); + dcp_queue_hotplug(&fixture->connector); + } + fixture->connector.base.state->link_status = DRM_MODE_LINK_STATUS_GOOD; + enable_work(&fixture->connector.hotplug_wq); + schedule_work(&fixture->connector.hotplug_wq); + flush_work(&fixture->connector.hotplug_wq); + KUNIT_EXPECT_EQ(test, fixture->observed_link_status, + (u64)DRM_MODE_LINK_STATUS_BAD); + KUNIT_EXPECT_EQ(test, atomic_read(&fixture->connector.hotplug_reasons), 0); + KUNIT_EXPECT_FALSE(test, work_pending(&fixture->connector.hotplug_wq)); + } + KUNIT_EXPECT_EQ(test, fixture->notifications, 2U); +} + static struct kunit_case hotplug_test_cases[] = { + KUNIT_CASE(hotplug_route_failure_notify), + KUNIT_CASE(hotplug_coalesced_ready), KUNIT_CASE(hotplug_connected), KUNIT_CASE(hotplug_disconnected), KUNIT_CASE(hotplug_retrain), diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index eff8ee27f7aad..d3b5ef58c5c39 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -740,6 +740,24 @@ void dcp_ack(struct apple_dcp *dcp, enum dcp_context_id context) * waits for vblank (a DCP callback). That means we deadlock if we call from * the RTKit thread! Instead, move the call to another thread via a workqueue. */ +bool dcp_crtc_can_retrain(struct drm_crtc *crtc, struct apple_connector *connector) +{ + struct platform_device *pdev = READ_ONCE(connector->dcp); + struct apple_dcp_typec_route *route; + struct apple_dcp *dcp; + + if (!connector->port_encoder) + return true; + if (!pdev) + return false; + dcp = platform_get_drvdata(pdev); + route = READ_ONCE(dcp->active_typec_route); + if (!dcp_typec_follows_crtc(dcp) || (route && READ_ONCE(route->tunnel))) + return true; + /* Rollback HPD must not retry the same failed crossing from another reset. */ + return to_apple_crtc(crtc)->dcp == pdev; +} + static int dcp_retrain_active_crtc(struct apple_connector *connector) { struct drm_device *dev = connector->base.dev; @@ -750,7 +768,8 @@ static int dcp_retrain_active_crtc(struct apple_connector *connector) DRM_MODESET_LOCK_ALL_BEGIN(dev, ctx, 0, ret); crtc = connector->base.state ? connector->base.state->crtc : NULL; - if (crtc && crtc->state && crtc->state->active) + if (crtc && crtc->state && crtc->state->active && + dcp_crtc_can_retrain(crtc, connector)) ret = drm_atomic_helper_reset_crtc(crtc, &ctx); else ret = 0; @@ -765,7 +784,7 @@ void dcp_handle_hotplug_actions(struct apple_dcp *dcp, unsigned int action) if (action & DCP_HOTPLUG_VBLANK) schedule_work(&dcp->vblank_wq); if ((action & DCP_HOTPLUG_NOTIFY) && dcp->connector) - schedule_work(&dcp->connector->hotplug_wq); + dcp_queue_hotplug(dcp->connector); } void dcp_retrain_oob(struct apple_connector *connector) @@ -785,7 +804,7 @@ void dcp_retrain_oob(struct apple_connector *connector) * context; sending a synthetic disconnect would tear down the connector. */ dcp_mode_invalidate(&dcp->mode_state); - schedule_work(&connector->hotplug_wq); + dcp_queue_hotplug(connector); } static void dcp_clear_connector_edid(struct apple_connector *connector) @@ -1095,15 +1114,38 @@ static void dcp_notify_hotplug(struct apple_connector *connector) drm_kms_helper_connector_hotplug_event(&connector->base); } +enum dcp_hotplug_reason { + DCP_HOTPLUG_READY = BIT(0), + DCP_HOTPLUG_ROUTE_FAILURE = BIT(1), +}; + +void dcp_queue_hotplug(struct apple_connector *connector) +{ + atomic_or(DCP_HOTPLUG_READY, &connector->hotplug_reasons); + schedule_work(&connector->hotplug_wq); +} + +void dcp_route_failure_notify(struct apple_connector *connector) +{ + /* A failed reset queues notification only; real readiness wins coalescing. */ + atomic_or(DCP_HOTPLUG_ROUTE_FAILURE, &connector->hotplug_reasons); + schedule_work(&connector->hotplug_wq); +} + void dcp_hotplug(struct work_struct *work) { struct apple_connector *connector; struct platform_device *pdev; struct apple_dcp *dcp; bool connected, gated; + bool notify_only; + unsigned int reasons; int ret; connector = container_of(work, struct apple_connector, hotplug_wq); + reasons = atomic_xchg(&connector->hotplug_reasons, 0); + notify_only = (reasons & DCP_HOTPLUG_ROUTE_FAILURE) && + !(reasons & DCP_HOTPLUG_READY); scoped_guard(mutex, &connector->base.dev->mode_config.mutex) dcp_consume_edid_invalidation(connector); @@ -1153,9 +1195,11 @@ void dcp_hotplug(struct work_struct *work) * flush, or the CRTC gets no signal. */ if (gated) { - if (connector->base.state && !READ_ONCE(dcp->mode_state.valid) && connected) + if (!notify_only && connector->base.state && + !READ_ONCE(dcp->mode_state.valid) && connected) dcp_external_relink(dcp, connector); - } else if (connector->base.state && !READ_ONCE(dcp->mode_state.valid) && connected && + } else if (!notify_only && connector->base.state && + !READ_ONCE(dcp->mode_state.valid) && connected && !dcp_uses_t6020_tunnel_flow(dcp)) { drm_connector_set_link_status_property(&connector->base, DRM_MODE_LINK_STATUS_BAD); @@ -1432,6 +1476,7 @@ void dcp_modes_begin_attachment(struct apple_dcp *dcp) guard(mutex)(&dcp->modes_lock); dcp->modes_generation++; dcp->modes_admitted = false; + dcp->modes_provisional = false; } bool dcp_modes_end_typec(struct apple_dcp *dcp, struct apple_dcp_typec_route *route) @@ -1443,10 +1488,61 @@ bool dcp_modes_end_typec(struct apple_dcp *dcp, struct apple_dcp_typec_route *ro if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL) { dcp->modes_generation++; dcp->modes_admitted = false; + dcp->modes_provisional = false; } return true; } +/* A copy of the modes @dcp knows for @connector's display, or NULL. */ +struct dcp_display_mode *dcp_modes_dup(struct apple_dcp *dcp, + struct apple_connector *connector, + unsigned int *count, u64 *generation) +{ + struct dcp_display_mode *modes; + + guard(mutex)(&dcp->modes_lock); + *count = 0; + if (!connector || !dcp->nr_modes || !dcp->modes_admitted || + READ_ONCE(dcp->connector) != connector) + return ERR_PTR(-ENODATA); + *generation = dcp->modes_generation; + modes = kmemdup_array(dcp->modes, dcp->nr_modes, sizeof(*dcp->modes), + GFP_KERNEL); + if (!modes) + return ERR_PTR(-ENOMEM); + *count = dcp->nr_modes; + return modes; +} + +/* + * A display came to @dcp from another pipeline without being unplugged: + * offer the modes it had there until the firmware here describes it. + * Takes @modes. + */ +void dcp_modes_adopt(struct apple_dcp *dcp, struct apple_connector *connector, + u64 generation, struct dcp_display_mode *modes, + unsigned int count) +{ + unsigned long flags; + + if (!modes) + return; + guard(mutex)(&dcp->modes_lock); + /* Never overwrite a fresh catalog or lend it to a different attachment. */ + if (generation != dcp->modes_generation || + READ_ONCE(dcp->connector) != connector || dcp->modes_admitted) { + kfree(modes); + return; + } + /* The replaced array is retained, as for dcp_modes_replace(). */ + spin_lock_irqsave(&dcp->metadata.lock, flags); + dcp->modes = modes; + dcp->nr_modes = count; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + dcp->modes_admitted = true; + dcp->modes_provisional = true; +} + u64 dcp_modes_transfer_begin(struct apple_dcp *dcp) { guard(mutex)(&dcp->modes_lock); @@ -1528,6 +1624,7 @@ int dcp_modes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *ctx, dcp->chunks.automatic_connection_sequence; spin_unlock_irqrestore(&dcp->metadata.lock, flags); dcp->modes_admitted = !ret; + dcp->modes_provisional = false; mutex_unlock(&dcp->modes_lock); if (!ret && !h17p_external) @@ -1978,6 +2075,62 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, return ret; } +bool dcp_crtc_needs_route_start(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route = READ_ONCE(dcp->active_typec_route); + + return dcp_typec_follows_crtc(dcp) && route && !READ_ONCE(route->tunnel) && + READ_ONCE(dcp->typec_crtc_off) && READ_ONCE(dcp->typec_follow_start) && + READ_ONCE(dcp->typec_follow_gen) == READ_ONCE(dcp->typec_generation); +} + +bool dcp_crtc_route_ready(struct drm_crtc *crtc, struct drm_atomic_commit *state, + bool fresh) +{ + struct apple_dcp *dcp = platform_get_drvdata(to_apple_crtc(crtc)->dcp); + struct drm_connector_state *conn_state; + struct drm_crtc_state *crtc_state; + struct drm_connector *conn; + int i; + + if (!dcp_typec_follows_crtc(dcp)) + return true; + guard(mutex)(&dcp->modes_lock); + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + if (!crtc_state) + crtc_state = crtc->state; + /* Plane-only updates need the complete effective assignment as well. */ + if (crtc_state && crtc_state->connector_mask && dcp->active_typec_route && + (!dcp->connector || + !(crtc_state->connector_mask & drm_connector_mask(&dcp->connector->base)) || + READ_ONCE(dcp->connector->dcp) != to_apple_crtc(crtc)->dcp)) + return false; + for_each_new_connector_in_state(state, conn, conn_state, i) { + struct apple_connector *connector = to_apple_connector(conn); + struct platform_device *owner = READ_ONCE(connector->dcp); + + if (conn_state->crtc != crtc || !connector->port_encoder) + continue; + /* Unrouted resume retains its existing deferred-flush behaviour. */ + if (!owner && !READ_ONCE(connector->connected)) + continue; + if (owner != to_apple_crtc(crtc)->dcp || + READ_ONCE(dcp->connector) != connector) + return false; + } + if (fresh && dcp->typec_follow_start && + dcp->typec_follow_gen == dcp->typec_generation && + !dcp_modes_for_connector(dcp, dcp->connector)) + return false; + return !fresh || !dcp->modes_provisional; +} + +/* Does the display @dcp drives offer @mode? */ +bool dcp_has_mode(struct apple_dcp *dcp, const struct drm_display_mode *mode) +{ + return lookup_mode(dcp, mode, NULL); +} + bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, const struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode) @@ -1986,8 +2139,14 @@ bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, struct platform_device *pdev = apple_crtc->dcp; struct apple_dcp *dcp = platform_get_drvdata(pdev); - /* TODO: support synthesized modes through scaling */ - return dcp_mode_settable(dcp, mode); + /* + * TODO: support synthesized modes through scaling + * + * A pipeline whose Type-C routes follow their CRTC may be given a + * display that another pipeline still drives: dcp_crtc_atomic_check() + * checks the mode against that one. + */ + return dcp_mode_settable(dcp, mode) || dcp_typec_follows_crtc(dcp); } @@ -2370,7 +2529,9 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit) * re-runs the modeset, which marks the mode valid again, before the * firmware has reported the display back. */ - if (!READ_ONCE(dcp->mode_state.valid) || !dcp->connector || !dcp->connector->connected) { + if ((to_apple_atomic_state(commit)->failed_routes & drm_crtc_mask(crtc)) || + !dcp_crtc_route_ready(crtc, commit, true) || + !READ_ONCE(dcp->mode_state.valid) || !dcp->connector || !dcp->connector->connected) { schedule_work(&dcp->vblank_wq); return; } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 00a00e89087b1..a9f35b4c900ce 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -2581,7 +2581,7 @@ static void res_is_main_display(struct apple_dcp *dcp, void *out, void *cookie) connector = dcp->connector; if (connector) { connector->connected = dcp->nr_modes > 0; - schedule_work(&connector->hotplug_wq); + dcp_queue_hotplug(connector); } dcp->active = true; diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 7b0f43dc32d7b..07c57a8c30143 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -203,7 +203,7 @@ static void dcp_v14_stopped_work(struct work_struct *work) if (connector) { WRITE_ONCE(connector->connected, false); apple_connector_edid_set_live(connector, false); - schedule_work(&connector->hotplug_wq); + dcp_queue_hotplug(connector); } } @@ -1088,7 +1088,7 @@ static void dcp_v14_crashed(void *cookie, const void *crashlog, size_t crashlog_ if (v14->external && connector && !READ_ONCE(dcp->external_detached)) { WRITE_ONCE(connector->connected, false); apple_connector_edid_set_live(connector, false); - schedule_work(&connector->hotplug_wq); + dcp_queue_hotplug(connector); } } dev_err(v14->dev, "DCP firmware crashed; its buffers are kept until reboot\n"); @@ -1864,7 +1864,7 @@ int iomfb_v14_7_external_open(struct apple_dcp *dcp) /* A display described before this could not be enabled until now. */ if (!READ_ONCE(dcp->external_detached) && dcp->connector && READ_ONCE(dcp->connector->connected)) - schedule_work(&dcp->connector->hotplug_wq); + dcp_queue_hotplug(dcp->connector); return 0; } diff --git a/drivers/gpu/drm/apple/parser-test.c b/drivers/gpu/drm/apple/parser-test.c index d2199656ce962..0004e4c7ff308 100644 --- a/drivers/gpu/drm/apple/parser-test.c +++ b/drivers/gpu/drm/apple/parser-test.c @@ -1,6 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT #include #include +#include + +#include "dcp.h" #include #include "connector.h" @@ -563,6 +566,167 @@ static void parser_stale_transfer(struct kunit *test) kfree(dcp->modes); } +static void parser_route_catalog_handoff(struct kunit *test) +{ + struct parser_record record = valid_record(7, 10); + struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + struct apple_connector *connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL); + struct apple_connector *other = kunit_kzalloc(test, sizeof(*other), GFP_KERNEL); + struct dcp_display_mode *copy; + struct dcp_parse_ctx ctx; + struct parser_blob *blob; + unsigned int count; + u64 generation; + + KUNIT_ASSERT_NOT_NULL(test, dcp); + KUNIT_ASSERT_NOT_NULL(test, connector); + KUNIT_ASSERT_NOT_NULL(test, other); + dcp_modes_init(dcp); + apple_connector_edid_init(connector); + apple_connector_edid_init(other); + dcp->connector = connector; + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_NOT_NULL(test, blob); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + ctx.dcp = &parser_dcp; + KUNIT_ASSERT_EQ(test, dcp_modes_replace(dcp, &ctx, dcp_modes_transfer_begin(dcp)), 0); + copy = dcp_modes_dup(dcp, other, &count, &generation); + KUNIT_EXPECT_EQ(test, PTR_ERR(copy), (long)-ENODATA); + copy = dcp_modes_dup(dcp, connector, &count, &generation); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + KUNIT_EXPECT_EQ(test, count, 1U); + dcp_modes_begin_attachment(dcp); + dcp_modes_adopt(dcp, connector, generation, copy, count); + KUNIT_EXPECT_FALSE(test, dcp->modes_admitted); + + copy = kmemdup(dcp->modes, sizeof(*copy), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, copy); + generation = dcp_modes_transfer_begin(dcp); + dcp_modes_adopt(dcp, other, generation, copy, 1); + KUNIT_EXPECT_FALSE(test, dcp->modes_admitted); + copy = kmemdup(dcp->modes, sizeof(*copy), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, copy); + dcp_modes_adopt(dcp, connector, generation, copy, 1); + KUNIT_EXPECT_TRUE(test, dcp->modes_admitted); + KUNIT_EXPECT_TRUE(test, dcp->modes_provisional); + + record.id = 9; + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + ctx.dcp = &parser_dcp; + KUNIT_ASSERT_EQ(test, dcp_modes_replace(dcp, &ctx, generation), 0); + KUNIT_EXPECT_FALSE(test, dcp->modes_provisional); + KUNIT_EXPECT_EQ(test, dcp->modes[0].timing_mode_id, 9U); + /* A copy arriving after the destination's catalog cannot replace its IDs. */ + copy = kmemdup(dcp->modes, sizeof(*copy), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, copy); + copy->timing_mode_id = 7; + dcp_modes_adopt(dcp, connector, generation, copy, 1); + KUNIT_EXPECT_EQ(test, dcp->modes[0].timing_mode_id, 9U); + kfree(dcp->modes); +} + +static void parser_route_programming_guard(struct kunit *test) +{ + struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + struct apple_crtc *crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL); + struct platform_device *pipeline = kunit_kzalloc(test, sizeof(*pipeline), GFP_KERNEL); + struct platform_device *other = kunit_kzalloc(test, sizeof(*other), GFP_KERNEL); + struct apple_connector *connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL); + struct drm_connector_state conn_state = {}; + struct drm_crtc_state crtc_state = {}; + struct __drm_crtcs_state crtcs[1] = { { .new_state = &crtc_state } }; + struct __drm_connnectors_state connections[1] = {}; + struct drm_atomic_commit state = { + .connectors = connections, .num_connector = 1, .crtcs = crtcs, + }; + struct drm_encoder encoder = {}; + + KUNIT_ASSERT_NOT_NULL(test, dcp); + KUNIT_ASSERT_NOT_NULL(test, crtc); + KUNIT_ASSERT_NOT_NULL(test, pipeline); + KUNIT_ASSERT_NOT_NULL(test, other); + KUNIT_ASSERT_NOT_NULL(test, connector); + dcp_modes_init(dcp); + dcp->fw_compat = DCP_FIRMWARE_V_13_5; + dcp->nr_typec_routes = 1; + dcp->connector = connector; + crtc->dcp = pipeline; + platform_set_drvdata(pipeline, dcp); + connector->port_encoder = &encoder; + connector->dcp = other; + connector->connected = true; + conn_state.crtc = &crtc->base; + connections[0].ptr = &connector->base; + connections[0].new_state = &conn_state; + KUNIT_EXPECT_FALSE(test, dcp_crtc_route_ready(&crtc->base, &state, false)); + KUNIT_EXPECT_FALSE(test, dcp_crtc_route_ready(&crtc->base, &state, true)); + connector->dcp = pipeline; + KUNIT_EXPECT_TRUE(test, dcp_crtc_can_retrain(&crtc->base, connector)); + connector->dcp = other; + platform_set_drvdata(other, dcp); + KUNIT_EXPECT_FALSE(test, dcp_crtc_can_retrain(&crtc->base, connector)); + connector->dcp = pipeline; + dcp->modes_provisional = true; + KUNIT_EXPECT_TRUE(test, dcp_crtc_route_ready(&crtc->base, &state, false)); + KUNIT_EXPECT_FALSE(test, dcp_crtc_route_ready(&crtc->base, &state, true)); + dcp->modes_provisional = false; + KUNIT_EXPECT_TRUE(test, dcp_crtc_route_ready(&crtc->base, &state, true)); + /* A plane-only commit omits connector entries but keeps its complete mask. */ + state.num_connector = 0; + crtc_state.connector_mask = BIT_ULL(1); + dcp->active_typec_route = &dcp->typec_routes[0]; + dcp->typec_crtc_off = true; + KUNIT_EXPECT_FALSE(test, dcp_crtc_needs_route_start(dcp)); + dcp->typec_follow_start = true; + dcp->typec_generation = 5; + dcp->typec_follow_gen = 5; + KUNIT_EXPECT_TRUE(test, dcp_crtc_needs_route_start(dcp)); + dcp->typec_follow_gen = 4; + KUNIT_EXPECT_FALSE(test, dcp_crtc_needs_route_start(dcp)); + dcp->typec_follow_gen = 5; + dcp->active_typec_route->tunnel = true; + KUNIT_EXPECT_FALSE(test, dcp_crtc_needs_route_start(dcp)); + dcp->active_typec_route->tunnel = false; + KUNIT_EXPECT_FALSE(test, dcp_crtc_route_ready(&crtc->base, &state, true)); + crtc_state.connector_mask = BIT_ULL(0); + KUNIT_EXPECT_FALSE(test, dcp_crtc_route_ready(&crtc->base, &state, true)); + dcp->modes_admitted = true; + KUNIT_EXPECT_TRUE(test, dcp_crtc_route_ready(&crtc->base, &state, true)); + dcp->active_typec_route = NULL; + state.num_connector = 1; + /* An unrouted disconnected resume keeps its existing deferred path. */ + connector->dcp = NULL; + connector->connected = false; + KUNIT_EXPECT_TRUE(test, dcp_crtc_route_ready(&crtc->base, &state, false)); + /* Native/14.7 continues to use its existing atomic machinery. */ + dcp->fw_compat = DCP_FIRMWARE_V_14_7; + connector->dcp = other; + KUNIT_EXPECT_TRUE(test, dcp_crtc_route_ready(&crtc->base, &state, true)); +} + +static void parser_route_invalid_own_mode(struct kunit *test) +{ + struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + struct apple_crtc *crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL); + struct platform_device *pdev = kunit_kzalloc(test, sizeof(*pdev), GFP_KERNEL); + struct drm_crtc_state crtc_state = { .enable = true, .mode_changed = true }; + struct __drm_crtcs_state crtcs[1] = { { .new_state = &crtc_state } }; + struct drm_atomic_commit state = { .crtcs = crtcs }; + + KUNIT_ASSERT_NOT_NULL(test, dcp); + KUNIT_ASSERT_NOT_NULL(test, crtc); + KUNIT_ASSERT_NOT_NULL(test, pdev); + dcp_modes_init(dcp); + dcp->fw_compat = DCP_FIRMWARE_V_13_5; + dcp->nr_typec_routes = 1; + crtc->dcp = pdev; + platform_set_drvdata(pdev, dcp); + /* No crossing selected: a fixed/paired output still needs its own mode. */ + KUNIT_EXPECT_TRUE(test, dcp_crtc_mode_fixup(&crtc->base, &crtc_state.mode, NULL)); + KUNIT_EXPECT_EQ(test, dcp_crtc_atomic_check(&crtc->base, &state), -EINVAL); +} + static void parser_connector_ownership(struct kunit *test) { struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); @@ -880,6 +1044,31 @@ static void parser_property_bounds(struct kunit *test) KUNIT_EXPECT_FALSE(test, dcp_property_chunk_valid(8192, 8190, 4, 4096)); } +static void parser_route_prepare_scope(struct kunit *test) +{ + struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + struct device *dev = kunit_kzalloc(test, sizeof(*dev), GFP_KERNEL); + struct apple_dcp_typec_route *route = kunit_kzalloc(test, sizeof(*route), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, dcp); + KUNIT_ASSERT_NOT_NULL(test, dev); + KUNIT_ASSERT_NOT_NULL(test, route); + mutex_init(&dcp->modes_lock); + dcp->dev = dev; + dcp->fw_compat = DCP_FIRMWARE_V_13_5; + dcp->nr_typec_routes = 1; + dcp->active_typec_route = route; + dcp->typec_crtc_off = true; + /* No PHY would fail strict preparation; a paired wake must bypass it. */ + KUNIT_EXPECT_EQ(test, dcp_typec_prepare_route(dcp), 0); + dcp->typec_follow_start = true; + dcp->typec_follow_gen = dcp->typec_generation; + mutex_init(&dcp->hpd_mutex); + KUNIT_EXPECT_EQ(test, dcp_typec_prepare_route(dcp), -ENODEV); + mutex_destroy(&dcp->hpd_mutex); + mutex_destroy(&dcp->modes_lock); +} + static struct kunit_case parser_cases[] = { KUNIT_CASE(parser_score_presence), KUNIT_CASE(parser_legacy_ties), @@ -892,6 +1081,10 @@ static struct kunit_case parser_cases[] = { KUNIT_CASE(parser_replacement), KUNIT_CASE(parser_attachment_admission), KUNIT_CASE(parser_stale_transfer), + KUNIT_CASE(parser_route_catalog_handoff), + KUNIT_CASE(parser_route_programming_guard), + KUNIT_CASE(parser_route_prepare_scope), + KUNIT_CASE(parser_route_invalid_own_mode), KUNIT_CASE(parser_connector_ownership), KUNIT_CASE(parser_route_retirement), KUNIT_CASE(parser_rejected_replacement), diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 50eb5f6db4de7..0557cc09e63e4 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -145,6 +145,12 @@ int parse_display_metadata(struct dcp_parse_ctx *handle, struct dcp_display_meta bool dcp_modes_for_connector(struct apple_dcp *dcp, struct apple_connector *connector); void dcp_modes_begin_attachment(struct apple_dcp *dcp); +struct dcp_display_mode *dcp_modes_dup(struct apple_dcp *dcp, + struct apple_connector *connector, + unsigned int *count, u64 *generation); +void dcp_modes_adopt(struct apple_dcp *dcp, struct apple_connector *connector, + u64 generation, struct dcp_display_mode *modes, + unsigned int count); bool dcp_modes_end_typec(struct apple_dcp *dcp, struct apple_dcp_typec_route *route); void dcp_modes_init(struct apple_dcp *dcp); void dcp_modes_release(void *data); From 246d9fb006b72de8b6c10445827a74a1cad4a387 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:21:56 -0700 Subject: [PATCH 1633/2680] drm/apple: reserve handoff state and replay fresh mode catalogs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reserve the effective swap holder connector before deciding a handoff, without holding fabric locks across DRM state acquisition. Validate the full CRTC connector mask during enable and flush, including plane-only updates after a route has moved. Queue readiness after a successful provisional-to-fresh catalog update only for the current attachment and owning connector. Keep rejected, stale and ordinary replacements on their existing notification path. Validation: W=1 Apple DRM objects and tests, ARM64 KUnit suites, serialized publication and assignment regressions, and targeted negative controls. [ryan: adapted to 7.2.10 — the provisional-to-fresh readiness hotplug is taken in our dcp_modes_replace() publication (H17P external never has a provisional catalogue); 7.2 drm_atomic_commit types] Link: https://github.com/iconidentify/aurora-linux/issues/39 Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 3f75f1f7fe1263b8f5c60c7d58945050e29d5bdd) --- drivers/gpu/drm/apple/dcp-fabric.c | 34 ++++++-- drivers/gpu/drm/apple/dcp-fabric.h | 2 + drivers/gpu/drm/apple/iomfb.c | 59 +++++++++----- drivers/gpu/drm/apple/parser-test.c | 122 +++++++++++++++++++++++++++- 4 files changed, 187 insertions(+), 30 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index cefbe27cf93b6..baa71e9c0fe72 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -2055,10 +2055,10 @@ dcp_typec_port_route(struct apple_dcp_typec_port *port, struct apple_dcp *dcp) * Is the display of @connector off in @state, or to be shown on @back's * pipeline, so that its route may give @crtc's pipeline up for that one? */ -static bool dcp_typec_follow_holder_off(struct drm_atomic_commit *state, - struct drm_crtc *crtc, - struct apple_connector *connector, - struct apple_dcp *back) +bool dcp_typec_follow_holder_off(struct drm_atomic_commit *state, + struct drm_crtc *crtc, + struct apple_connector *connector, + struct apple_dcp *back) { struct drm_connector_state *conn_state = NULL; struct drm_crtc_state *crtc_state; @@ -2066,10 +2066,11 @@ static bool dcp_typec_follow_holder_off(struct drm_atomic_commit *state, if (connector) { conn_state = drm_atomic_get_new_connector_state(state, &connector->base); - if (!conn_state) - conn_state = connector->base.state; } - other = conn_state ? conn_state->crtc : NULL; + /* Check reserved this holder; never inspect a later commit's live state. */ + if (!conn_state) + return false; + other = conn_state->crtc; if (!other) return true; if (other == crtc) @@ -2077,8 +2078,6 @@ static bool dcp_typec_follow_holder_off(struct drm_atomic_commit *state, if (back->crtc && other == &back->crtc->base) return true; crtc_state = drm_atomic_get_new_crtc_state(state, other); - if (!crtc_state) - crtc_state = other->state; return crtc_state && !crtc_state->active; } @@ -2362,8 +2361,25 @@ int dcp_typec_follow_check(struct apple_dcp *dcp, struct drm_crtc *crtc, return PTR_ERR(other_state); } } +retry_holder: if (!dcp_typec_follow_lock(false)) return -EBUSY; + if (dcp->active_typec_route && !dcp->active_typec_route->tunnel) { + struct apple_connector *holder = dcp->active_typec_route->port->connector; + + if (holder && !drm_atomic_get_new_connector_state(state, &holder->base)) { + struct drm_connector_state *holder_state; + + drm_connector_get(&holder->base); + dcp_typec_follow_unlock(false); + /* Acquire DRM state without holding either fabric mutex. */ + holder_state = drm_atomic_get_connector_state(state, &holder->base); + drm_connector_put(&holder->base); + if (IS_ERR(holder_state)) + return PTR_ERR(holder_state); + goto retry_holder; + } + } ret = dcp_typec_follow_decide(dcp, crtc, state, connector, &follow); if (!ret && !dcp_has_mode(follow.action == DCP_FABRIC_FOLLOW_STAY ? dcp : follow.from->dcp, mode)) diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index 82baa9d3014fb..bf56deb535d4c 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -69,6 +69,8 @@ void dcp_typec_port_set_connector(unsigned int idx, bool secondary, bool dcp_typec_dual_stream(void); void dcp_typec_reorder(void); bool dcp_typec_follows_crtc(struct apple_dcp *dcp); +bool dcp_typec_follow_holder_off(struct drm_atomic_commit *state, struct drm_crtc *crtc, + struct apple_connector *connector, struct apple_dcp *back); int dcp_typec_follow_check(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_commit *state, const struct drm_display_mode *mode); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index d3b5ef58c5c39..2d4bb386ef323 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1560,11 +1560,13 @@ int dcp_modes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *ctx, { bool h17p_external = dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL; + struct apple_connector *ready = NULL; struct dcp_display_mode *modes; struct dcp_metadata_ticket rate_ticket = {}; u64 rate_service_epoch = 0; unsigned int nr_modes = 0, i; unsigned long flags; + bool provisional; u32 entries = 0; int ret = 0; @@ -1609,6 +1611,7 @@ int dcp_modes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *ctx, kfree(modes); return -ESTALE; } + provisional = dcp->modes_provisional; /* Publish one catalog revision with its immutable receipt tickets. */ spin_lock_irqsave(&dcp->metadata.lock, flags); dcp->rate_proof.catalog_sequence++; @@ -1625,12 +1628,20 @@ int dcp_modes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *ctx, spin_unlock_irqrestore(&dcp->metadata.lock, flags); dcp->modes_admitted = !ret; dcp->modes_provisional = false; + /* A fresh catalog replaced one adopted across a route move. */ + if (!ret && provisional && dcp->connector && dcp->dev && + dcp_modes_for_connector(dcp, dcp->connector) && + READ_ONCE(dcp->connector->dcp) == to_platform_device(dcp->dev)) + ready = dcp->connector; mutex_unlock(&dcp->modes_lock); if (!ret && !h17p_external) apple_connector_invalidate_edid(dcp->connector); /* Notify removals as well as newly eligible modes, outside the lock. */ dcp_180hz_catalog_changed(dcp); + /* RX lifetime is drained before connector cleanup; work rechecks ownership. */ + if (ready) + dcp_queue_hotplug(ready); return ret; } @@ -2088,10 +2099,11 @@ bool dcp_crtc_route_ready(struct drm_crtc *crtc, struct drm_atomic_commit *state bool fresh) { struct apple_dcp *dcp = platform_get_drvdata(to_apple_crtc(crtc)->dcp); - struct drm_connector_state *conn_state; + struct drm_connector_list_iter iter; struct drm_crtc_state *crtc_state; struct drm_connector *conn; - int i; + u64 mask, seen = 0; + bool ready = true, owned = false; if (!dcp_typec_follows_crtc(dcp)) return true; @@ -2099,30 +2111,37 @@ bool dcp_crtc_route_ready(struct drm_crtc *crtc, struct drm_atomic_commit *state crtc_state = drm_atomic_get_new_crtc_state(state, crtc); if (!crtc_state) crtc_state = crtc->state; - /* Plane-only updates need the complete effective assignment as well. */ - if (crtc_state && crtc_state->connector_mask && dcp->active_typec_route && - (!dcp->connector || - !(crtc_state->connector_mask & drm_connector_mask(&dcp->connector->base)) || - READ_ONCE(dcp->connector->dcp) != to_apple_crtc(crtc)->dcp)) - return false; - for_each_new_connector_in_state(state, conn, conn_state, i) { + mask = crtc_state ? crtc_state->connector_mask : 0; + if (!mask) + return !dcp->connector || !dcp->connector->port_encoder; + /* The mask is the complete assignment, even on a plane-only commit. */ + drm_connector_list_iter_begin(crtc->dev, &iter); + drm_for_each_connector_iter(conn, &iter) { struct apple_connector *connector = to_apple_connector(conn); - struct platform_device *owner = READ_ONCE(connector->dcp); + struct platform_device *owner; + struct drm_connector_state *assignment; - if (conn_state->crtc != crtc || !connector->port_encoder) + if (!(mask & drm_connector_mask(conn))) continue; - /* Unrouted resume retains its existing deferred-flush behaviour. */ + seen |= drm_connector_mask(conn); + owner = READ_ONCE(connector->dcp); + /* Preserve the unrouted, disconnected resume/deferred-flush path. */ if (!owner && !READ_ONCE(connector->connected)) continue; - if (owner != to_apple_crtc(crtc)->dcp || - READ_ONCE(dcp->connector) != connector) - return false; - } - if (fresh && dcp->typec_follow_start && - dcp->typec_follow_gen == dcp->typec_generation && - !dcp_modes_for_connector(dcp, dcp->connector)) + owned = true; + assignment = drm_atomic_get_new_connector_state(state, conn); + if (!assignment) + assignment = conn->state; + if (!assignment || assignment->crtc != crtc || + owner != to_apple_crtc(crtc)->dcp || dcp->connector != connector) + ready = false; + } + drm_connector_list_iter_end(&iter); + if (!ready || seen != mask) return false; - return !fresh || !dcp->modes_provisional; + if (fresh && owned) + return dcp_modes_for_connector(dcp, dcp->connector) && !dcp->modes_provisional; + return true; } /* Does the display @dcp drives offer @mode? */ diff --git a/drivers/gpu/drm/apple/parser-test.c b/drivers/gpu/drm/apple/parser-test.c index 0004e4c7ff308..e93233a592aa7 100644 --- a/drivers/gpu/drm/apple/parser-test.c +++ b/drivers/gpu/drm/apple/parser-test.c @@ -2,6 +2,8 @@ #include #include #include +#include +#include #include "dcp.h" #include @@ -566,12 +568,34 @@ static void parser_stale_transfer(struct kunit *test) kfree(dcp->modes); } +struct parser_catalog_observer { + struct apple_connector connector; + unsigned int notifications; + unsigned int reasons; + bool locked; + struct apple_dcp *dcp; +}; + +static void parser_catalog_ready(struct work_struct *work) +{ + struct apple_connector *connector = container_of(work, struct apple_connector, hotplug_wq); + struct parser_catalog_observer *observer = + container_of(connector, struct parser_catalog_observer, connector); + + observer->notifications++; + observer->reasons = atomic_xchg(&connector->hotplug_reasons, 0); + observer->locked = mutex_is_locked(&observer->dcp->modes_lock); +} + static void parser_route_catalog_handoff(struct kunit *test) { struct parser_record record = valid_record(7, 10); struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); - struct apple_connector *connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL); + struct parser_catalog_observer *observer = + kunit_kzalloc(test, sizeof(*observer), GFP_KERNEL); + struct apple_connector *connector = observer ? &observer->connector : NULL; struct apple_connector *other = kunit_kzalloc(test, sizeof(*other), GFP_KERNEL); + struct platform_device *pipeline = kunit_kzalloc(test, sizeof(*pipeline), GFP_KERNEL); struct dcp_display_mode *copy; struct dcp_parse_ctx ctx; struct parser_blob *blob; @@ -581,7 +605,12 @@ static void parser_route_catalog_handoff(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, dcp); KUNIT_ASSERT_NOT_NULL(test, connector); KUNIT_ASSERT_NOT_NULL(test, other); + KUNIT_ASSERT_NOT_NULL(test, pipeline); dcp_modes_init(dcp); + dcp->dev = &pipeline->dev; + connector->dcp = pipeline; + observer->dcp = dcp; + INIT_WORK(&connector->hotplug_wq, parser_catalog_ready); apple_connector_edid_init(connector); apple_connector_edid_init(other); dcp->connector = connector; @@ -610,13 +639,43 @@ static void parser_route_catalog_handoff(struct kunit *test) KUNIT_EXPECT_TRUE(test, dcp->modes_admitted); KUNIT_EXPECT_TRUE(test, dcp->modes_provisional); + /* Stale and rejected replacement cannot announce readiness. */ + KUNIT_EXPECT_EQ(test, dcp_modes_replace(dcp, &ctx, generation - 1), -ESTALE); + flush_work(&connector->hotplug_wq); + KUNIT_EXPECT_EQ(test, observer->notifications, 0U); + record.score = -1; + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + ctx.dcp = &parser_dcp; + KUNIT_EXPECT_EQ(test, dcp_modes_replace(dcp, &ctx, generation), -EINVAL); + flush_work(&connector->hotplug_wq); + KUNIT_EXPECT_EQ(test, observer->notifications, 0U); + record.score = 10; record.id = 9; blob = make_blob(test, &record, 1); KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); ctx.dcp = &parser_dcp; KUNIT_ASSERT_EQ(test, dcp_modes_replace(dcp, &ctx, generation), 0); + flush_work(&connector->hotplug_wq); + KUNIT_EXPECT_EQ(test, observer->notifications, 1U); + KUNIT_EXPECT_NE(test, observer->reasons, 0U); + KUNIT_EXPECT_FALSE(test, observer->locked); KUNIT_EXPECT_FALSE(test, dcp->modes_provisional); KUNIT_EXPECT_EQ(test, dcp->modes[0].timing_mode_id, 9U); + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + ctx.dcp = &parser_dcp; + KUNIT_EXPECT_EQ(test, dcp_modes_replace(dcp, &ctx, generation), 0); + flush_work(&connector->hotplug_wq); + KUNIT_EXPECT_EQ(test, observer->notifications, 1U); + /* A provisional transition for a connector owned elsewhere sends no READY. */ + connector->dcp = NULL; + dcp->modes_provisional = true; + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + ctx.dcp = &parser_dcp; + KUNIT_EXPECT_EQ(test, dcp_modes_replace(dcp, &ctx, generation), 0); + flush_work(&connector->hotplug_wq); + KUNIT_EXPECT_EQ(test, observer->notifications, 1U); + connector->dcp = pipeline; /* A copy arriving after the destination's catalog cannot replace its IDs. */ copy = kmemdup(dcp->modes, sizeof(*copy), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, copy); @@ -626,6 +685,12 @@ static void parser_route_catalog_handoff(struct kunit *test) kfree(dcp->modes); } +static const struct drm_connector_funcs parser_route_connector_funcs = { + .reset = drm_atomic_helper_connector_reset, + .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, +}; + static void parser_route_programming_guard(struct kunit *test) { struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); @@ -641,13 +706,29 @@ static void parser_route_programming_guard(struct kunit *test) .connectors = connections, .num_connector = 1, .crtcs = crtcs, }; struct drm_encoder encoder = {}; + struct device *dev = drm_kunit_helper_alloc_device(test); + struct drm_device *drm; + int ret; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + drm = __drm_kunit_helper_alloc_drm_device(test, dev, sizeof(*drm), 0, + DRIVER_MODESET | DRIVER_ATOMIC); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, drm); KUNIT_ASSERT_NOT_NULL(test, dcp); KUNIT_ASSERT_NOT_NULL(test, crtc); KUNIT_ASSERT_NOT_NULL(test, pipeline); KUNIT_ASSERT_NOT_NULL(test, other); KUNIT_ASSERT_NOT_NULL(test, connector); + ret = drmm_connector_init(drm, &connector->base, &parser_route_connector_funcs, + DRM_MODE_CONNECTOR_DisplayPort, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + crtc->base.dev = drm; + drm_atomic_helper_connector_reset(&connector->base); + KUNIT_ASSERT_NOT_NULL(test, connector->base.state); + connector->base.state->crtc = &crtc->base; + crtc_state.connector_mask = drm_connector_mask(&connector->base); dcp_modes_init(dcp); + dcp->modes_admitted = true; dcp->fw_compat = DCP_FIRMWARE_V_13_5; dcp->nr_typec_routes = 1; dcp->connector = connector; @@ -689,11 +770,19 @@ static void parser_route_programming_guard(struct kunit *test) KUNIT_EXPECT_FALSE(test, dcp_crtc_needs_route_start(dcp)); dcp->active_typec_route->tunnel = false; KUNIT_EXPECT_FALSE(test, dcp_crtc_route_ready(&crtc->base, &state, true)); + crtc_state.connector_mask = 0; + KUNIT_EXPECT_FALSE(test, dcp_crtc_route_ready(&crtc->base, &state, true)); crtc_state.connector_mask = BIT_ULL(0); + dcp->modes_admitted = false; KUNIT_EXPECT_FALSE(test, dcp_crtc_route_ready(&crtc->base, &state, true)); dcp->modes_admitted = true; KUNIT_EXPECT_TRUE(test, dcp_crtc_route_ready(&crtc->base, &state, true)); dcp->active_typec_route = NULL; + dcp->typec_follow_start = false; + connector->dcp = other; + KUNIT_EXPECT_FALSE(test, dcp_crtc_route_ready(&crtc->base, &state, true)); + connector->dcp = pipeline; + KUNIT_EXPECT_TRUE(test, dcp_crtc_route_ready(&crtc->base, &state, true)); state.num_connector = 1; /* An unrouted disconnected resume keeps its existing deferred path. */ connector->dcp = NULL; @@ -703,6 +792,7 @@ static void parser_route_programming_guard(struct kunit *test) dcp->fw_compat = DCP_FIRMWARE_V_14_7; connector->dcp = other; KUNIT_EXPECT_TRUE(test, dcp_crtc_route_ready(&crtc->base, &state, true)); + connector->base.state->crtc = NULL; } static void parser_route_invalid_own_mode(struct kunit *test) @@ -1044,6 +1134,35 @@ static void parser_property_bounds(struct kunit *test) KUNIT_EXPECT_FALSE(test, dcp_property_chunk_valid(8192, 8190, 4, 4096)); } +static void parser_route_holder_snapshot(struct kunit *test) +{ + struct apple_dcp *back = kunit_kzalloc(test, sizeof(*back), GFP_KERNEL); + struct apple_connector *holder = kunit_kzalloc(test, sizeof(*holder), GFP_KERNEL); + struct drm_crtc *target = kunit_kzalloc(test, sizeof(*target), GFP_KERNEL); + struct drm_crtc *other = kunit_kzalloc(test, sizeof(*other), GFP_KERNEL); + struct drm_connector_state live = { .crtc = target }; + struct drm_connector_state reserved = {}; + struct __drm_connnectors_state conns[1] = {}; + struct drm_atomic_commit state = { .connectors = conns, .num_connector = 1 }; + + KUNIT_ASSERT_NOT_NULL(test, back); + KUNIT_ASSERT_NOT_NULL(test, holder); + KUNIT_ASSERT_NOT_NULL(test, target); + KUNIT_ASSERT_NOT_NULL(test, other); + holder->base.state = &live; + /* An absent holder snapshot is refused even if live state looks inactive. */ + KUNIT_EXPECT_FALSE(test, dcp_typec_follow_holder_off(&state, target, holder, back)); + conns[0].new_state = &reserved; + /* The reserved off holder is retained despite a future live assignment. */ + KUNIT_EXPECT_TRUE(test, dcp_typec_follow_holder_off(&state, target, holder, back)); + reserved.crtc = target; + live.crtc = NULL; + KUNIT_EXPECT_FALSE(test, dcp_typec_follow_holder_off(&state, target, holder, back)); + reserved.crtc = other; + back->crtc = to_apple_crtc(other); + KUNIT_EXPECT_TRUE(test, dcp_typec_follow_holder_off(&state, target, holder, back)); +} + static void parser_route_prepare_scope(struct kunit *test) { struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); @@ -1084,6 +1203,7 @@ static struct kunit_case parser_cases[] = { KUNIT_CASE(parser_route_catalog_handoff), KUNIT_CASE(parser_route_programming_guard), KUNIT_CASE(parser_route_prepare_scope), + KUNIT_CASE(parser_route_holder_snapshot), KUNIT_CASE(parser_route_invalid_own_mode), KUNIT_CASE(parser_connector_ownership), KUNIT_CASE(parser_route_retirement), From b198c60770b2c10918cc1fd3195f2c94bd96bfce Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:26:54 -0700 Subject: [PATCH 1634/2680] drm/apple: preserve paired display startup during catalog updates Keep ordinary paired direct and tunnel outputs on their existing startup path while the catalog arrives. Require fresh owned admission only for a current generation-qualified handoff. Retain complete assignment checks and block provisional timing IDs for every route. Validation: W=1 Apple DRM objects and tests, ARM64 KUnit suites, and the production readiness helper with paired and handoff admission cases. Link: https://github.com/iconidentify/aurora-linux/issues/39 Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 76861e9d45a5ee209e51633ee9869ea0d36bd0b9) --- drivers/gpu/drm/apple/iomfb.c | 7 +++++-- drivers/gpu/drm/apple/parser-test.c | 5 +++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 2d4bb386ef323..e4105ec285d04 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -2139,8 +2139,11 @@ bool dcp_crtc_route_ready(struct drm_crtc *crtc, struct drm_atomic_commit *state drm_connector_list_iter_end(&iter); if (!ready || seen != mask) return false; - if (fresh && owned) - return dcp_modes_for_connector(dcp, dcp->connector) && !dcp->modes_provisional; + if (fresh && dcp->modes_provisional) + return false; + if (fresh && owned && READ_ONCE(dcp->typec_follow_start) && + READ_ONCE(dcp->typec_follow_gen) == READ_ONCE(dcp->typec_generation)) + return dcp_modes_for_connector(dcp, dcp->connector); return true; } diff --git a/drivers/gpu/drm/apple/parser-test.c b/drivers/gpu/drm/apple/parser-test.c index e93233a592aa7..58a31cabbaf87 100644 --- a/drivers/gpu/drm/apple/parser-test.c +++ b/drivers/gpu/drm/apple/parser-test.c @@ -774,6 +774,11 @@ static void parser_route_programming_guard(struct kunit *test) KUNIT_EXPECT_FALSE(test, dcp_crtc_route_ready(&crtc->base, &state, true)); crtc_state.connector_mask = BIT_ULL(0); dcp->modes_admitted = false; + dcp->typec_follow_start = false; + /* Paired reattachment keeps baseline power-on while lookup rejects modes. */ + KUNIT_EXPECT_TRUE(test, dcp_crtc_route_ready(&crtc->base, &state, true)); + dcp->typec_follow_start = true; + /* A current handoff must acquire its own admitted destination catalog. */ KUNIT_EXPECT_FALSE(test, dcp_crtc_route_ready(&crtc->base, &state, true)); dcp->modes_admitted = true; KUNIT_EXPECT_TRUE(test, dcp_crtc_route_ready(&crtc->base, &state, true)); From 978ba4131086d8452336896c7dee2f29a413c803 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Fri, 25 Sep 2026 14:16:11 -0400 Subject: [PATCH 1635/2680] drm/apple: drive the J613 (M3 Air 13") internal display from a board record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move what differs between the internal panels of the 14.x firmware IOMFB path into a per-board record, and add J613 (MacBook Air 13" M3, T8122) as its second board. The T6030 record holds the values the path was written and tested with, so J514S/J516S behaviour and log lines stay as they were. A board is chosen by its DCP compatible and, for J613, the machine compatible. The record holds: - the firmware image tested on the board: 90F849E1-B422-367E-B389-50246F8DEC47 on J613; - the boot loader handoff marker, apple,t8122-handoff on J613; - the panel: J613 must be 2560x1664 (the 2560x1600 boot framebuffer plus 64 notch rows) and has no 120 Hz timing, so only a 60 Hz timing is taken; - the name of its debugfs status file (dcp-t6030, dcp-j613). This replaces the earlier T8122 admission, which accepted either firmware image together with either boot loader marker on any T8122 machine. A T8122 DCP on a machine without a record (J615 so far) is not started. External display processors (T6030 dcpext) have no board record. Their v14 instance is set up without the platform probe, their PIODMA lookup takes their own marker, and every panel-only path (bind, geometry, mode selection, start) checks for the board before using it. The J613 record has run on a J613 with 14.x system firmware. [ck: ported onto the v14.7 backend with external processors and mode admission; brightness limits stay with the panel node] [ryan: adapted to 7.2.10 — the firmware-compat check keeps this tree's early return for pinned H17-generation SoCs and the generic 14.7.0 TEMPLATE mapping; only T6030 dcpext and board-record panels map to native 14.7] (cherry picked from commit 7be01faf532787e11ac561a42bd216ba0c6747dd) Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 57204a7ce451bf29dafe7d2311d6b57c13a7bf6b) --- drivers/gpu/drm/apple/dcp.c | 24 ++-- drivers/gpu/drm/apple/iomfb_v14_7.c | 178 ++++++++++++++++++++++------ drivers/gpu/drm/apple/iomfb_v14_7.h | 9 ++ 3 files changed, 169 insertions(+), 42 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index fb3791f6d5a9e..3dec4d365aa45 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2832,6 +2832,7 @@ static int dcp_read_fw_version(struct device *dev, const char *name, static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) { const struct apple_dcp_hw_data *hw = of_device_get_match_data(dev); + const struct dcp_v14_board *v14_board; char compat_str[DCP_FW_VERSION_STR_LEN]; char fw_str[DCP_FW_VERSION_STR_LEN]; int ret; @@ -2853,23 +2854,28 @@ static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) } /* - * T6030 runs the 14.7 firmware session handed over by the boot loader - * natively (iomfb_v14_7.c); other 14.7.0 DCPs keep the template below. + * T6030 and the internal panels with a board record (T6030, J613) run + * the 14.7 firmware session handed over by the boot loader natively + * (iomfb_v14_7.c); other 14.7.0 DCPs keep the template below. + * The T6030 external processors: the 14.x firmware IOMFB only. */ - if (of_device_is_compatible(dev->of_node, "apple,t6030-dcp") || - of_device_is_compatible(dev->of_node, "apple,t6030-dcpext")) { - if (ret >= 0 && !strcmp(compat_str, "14.7.0")) + if (of_device_is_compatible(dev->of_node, "apple,t6030-dcpext")) { + if (!strcmp(compat_str, "14.7.0")) return DCP_FIRMWARE_V_14_7; dev_err(dev, "T6030 display not started: DCP firmware-compat %s is not 14.7.0\n", compat_str); return DCP_FIRMWARE_UNKNOWN; } - if (of_device_is_compatible(dev->of_node, "apple,t8122-dcp")) { - if (ret >= 0 && !strcmp(compat_str, "14.7.0")) + /* The internal panels with a board record (T6030, J613): the same. */ + v14_board = iomfb_v14_7_board(dev); + if (IS_ERR(v14_board)) + return DCP_FIRMWARE_UNKNOWN; + if (v14_board) { + if (!strcmp(compat_str, "14.7.0")) return DCP_FIRMWARE_V_14_7; - dev_err(dev, "T8122 display not started: DCP firmware-compat %s is not 14.7.0\n", - compat_str); + dev_err(dev, "%s display not started: DCP firmware-compat %s is not 14.7.0\n", + iomfb_v14_7_board_name(v14_board), compat_str); return DCP_FIRMWARE_UNKNOWN; } diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 07c57a8c30143..28d303c24320a 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -47,7 +47,7 @@ /* The images whose method and callback layouts this file implements. */ #define DCP_V14_FIRMWARE_UUID "DDF38191-93B3-324A-BC8F-643006F5AC82" /* T6030 */ -#define DCP_V14_FIRMWARE_UUID_T8122 "90F849E1-B422-367E-B389-50246F8DEC47" /* T8122 */ +#define DCP_V14_J613_FIRMWARE_UUID "90F849E1-B422-367E-B389-50246F8DEC47" /* J613 */ #define DCP_V14_CPU_CONTROL 0x44 #define DCP_V14_CPU_STATUS 0x48 @@ -81,6 +81,60 @@ static int dcp_v14_pmp_wait_ready(struct device_node *entry, #define DCP_V14_LAYER 0 #define DCP_V14_BLACK 0xff000000 +/* + * What differs between the internal panels this file drives. The T6030 + * record holds the values the file was written and tested with (J514S, + * J516S); another board's record enables only what was tested on it, and + * anything else fails closed. External processors have no board record. + */ +struct dcp_v14_board { + /* In log messages. */ + const char *name; + /* The debugfs status file. */ + const char *debugfs; + const char *dcp_compatible; + /* The machine compatible; NULL for any machine with this DCP. */ + const char *machine; + /* The firmware image whose layouts were tested on this board. */ + const char *firmware_uuid; + /* Set to <1> by the boot loader on the DCP, display and PIODMA nodes. */ + const char *handoff; + /* Native panel size, notch rows included; 0: any, from the boot framebuffer. */ + u32 panel_width, panel_height; + /* The panel may have a 120 Hz timing besides 60 Hz. */ + bool promotion; +}; + +static const struct dcp_v14_board dcp_v14_board_t6030 = { + .name = "T6030", + .debugfs = "dcp-t6030", + .dcp_compatible = "apple,t6030-dcp", + .firmware_uuid = DCP_V14_FIRMWARE_UUID, + .handoff = "apple,t6030-handoff", + .promotion = true, +}; + +/* + * J613 (MacBook Air 13", M3): a 2560x1664 panel at 60 Hz only, with 64 notch + * rows above the 2560x1600 boot framebuffer. + */ +static const struct dcp_v14_board dcp_v14_board_j613 = { + .name = "J613", + .debugfs = "dcp-j613", + .dcp_compatible = "apple,t8122-dcp", + .machine = "apple,j613", + .firmware_uuid = DCP_V14_J613_FIRMWARE_UUID, + .handoff = "apple,t8122-handoff", + .panel_width = 2560, + .panel_height = 1664, + .promotion = false, +}; + +static const struct dcp_v14_board *const dcp_v14_boards[] = { + &dcp_v14_board_t6030, + &dcp_v14_board_j613, +}; + struct dcp_v14_property { u32 service; char key[64]; @@ -105,6 +159,8 @@ struct dcp_v14_buffer { struct apple_dcp_v14 { struct device *dev; + /* The internal panel's board; NULL on an external processor. */ + const struct dcp_v14_board *board; /* Cleared on KMS unbind and on removal; the firmware session outlives it. */ struct apple_dcp *dcp; struct apple_rtkit *rtk; @@ -365,8 +421,9 @@ static int dcp_v14_map_piodma(struct apple_dcp_v14 *v14, u32 id) /* The display gate adds an external pipe's piodma with its DART. */ node = of_get_child_by_name(v14->dev->of_node, "piodma"); if (!node || !of_device_is_available(node) || + (!v14->external && !v14->board) || of_property_read_u32(node, v14->external ? "apple,t6030-dispext-handoff" : - "apple,t6030-handoff", &marker) || marker != 1) { + v14->board->handoff, &marker) || marker != 1) { of_node_put(node); return -ENODEV; } @@ -1223,7 +1280,7 @@ static void dcp_v14_release(void *data) } /* Read only: bind maps and claims these registers. */ -static int dcp_v14_cpu_running(struct device *dev) +static int dcp_v14_cpu_running(struct device *dev, const char *name) { struct resource *res; void __iomem *coproc; @@ -1239,53 +1296,90 @@ static int dcp_v14_cpu_running(struct device *dev) iounmap(coproc); if (!(control & APPLE_DCP_COPROC_CPU_CONTROL_RUN)) return dev_err_probe(dev, -EBUSY, - "T6030 display not started: the DCP CPU is stopped (%#x); the PMP is not started for it\n", - control); + "%s display not started: the DCP CPU is stopped (%#x); the PMP is not started for it\n", + name, control); return 0; } +/* + * The board of an internal 14.x DCP node: NULL if the node is not one, an + * error if it is but this machine has no board record. + */ +const struct dcp_v14_board *iomfb_v14_7_board(struct device *dev) +{ + const struct dcp_v14_board *board; + bool dcp_known = false; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(dcp_v14_boards); i++) { + board = dcp_v14_boards[i]; + if (!of_device_is_compatible(dev->of_node, board->dcp_compatible)) + continue; + dcp_known = true; + if (board->machine && !of_machine_is_compatible(board->machine)) + continue; + return board; + } + if (dcp_known) { + dev_err(dev, "display not started: no board record for this machine\n"); + return ERR_PTR(-ENODEV); + } + return NULL; +} + +const char *iomfb_v14_7_board_name(const struct dcp_v14_board *board) +{ + return board->name; +} + int iomfb_v14_7_probe(struct apple_dcp *dcp) { struct device *dev = dcp->dev; struct device_node *np = dev->of_node, *entry; + const struct dcp_v14_board *board = iomfb_v14_7_board(dev); struct apple_dcp_v14 *v14; const char *uuid = NULL; u32 marker = 0; int ret; + if (IS_ERR_OR_NULL(board)) + return -ENODEV; if (of_property_read_string(np, "apple,firmware-uuid", &uuid) || - (strcmp(uuid, DCP_V14_FIRMWARE_UUID) && - strcmp(uuid, DCP_V14_FIRMWARE_UUID_T8122))) + strcmp(uuid, board->firmware_uuid)) return dev_err_probe(dev, -ENODEV, - "display not started: DCP firmware %s is not supported\n", - uuid ?: "(unknown)"); - if ((of_property_read_u32(np, "apple,t6030-handoff", &marker) || marker != 1) && - (of_property_read_u32(np, "apple,t8122-handoff", &marker) || marker != 1)) + "%s display not started: DCP firmware %s is not supported\n", + board->name, uuid ?: "(unknown)"); + if (of_property_read_u32(np, board->handoff, &marker) || marker != 1) return dev_err_probe(dev, -ENODEV, - "display not started: no boot loader display handoff\n"); + "%s display not started: no boot loader display handoff\n", + board->name); if (!iommu_get_domain_for_dev(dev)) return dev_err_probe(dev, -ENODEV, - "T6030 display not started: the DCP has no DART domain\n"); - ret = dcp_v14_cpu_running(dev); + "%s display not started: the DCP has no DART domain\n", + board->name); + ret = dcp_v14_cpu_running(dev, board->name); if (ret) return ret; entry = of_parse_phandle(np, "apple,pmp-report", 0); if (!entry) return dev_err_probe(dev, -ENODEV, - "T6030 display not started: no apple,pmp-report, the PMP is not described\n"); + "%s display not started: no apple,pmp-report, the PMP is not described\n", + board->name); ret = dcp_v14_pmp_wait_ready(entry, DCP_V14_PMP_TIMEOUT); of_node_put(entry); if (ret == -EPROBE_DEFER) return dev_err_probe(dev, ret, "waiting for the PMP report\n"); if (ret) return dev_err_probe(dev, ret, - "T6030 display not started: the PMP has not acknowledged the display request; the boot framebuffer stays\n"); + "%s display not started: the PMP has not acknowledged the display request; the boot framebuffer stays\n", + board->name); v14 = kzalloc_obj(*v14); if (!v14) return -ENOMEM; v14->dev = dev; + v14->board = board; v14->dcp = dcp; mutex_init(&v14->lock); init_waitqueue_head(&v14->described_wait); @@ -1323,6 +1417,15 @@ static int dcp_v14_geometry(struct apple_dcp_v14 *v14) return -EINVAL; v14->panel_width = v14->fb_width; v14->panel_height = v14->fb_height + notch; + /* A board with a known panel takes no other. */ + if (v14->board->panel_width && + (v14->panel_width != v14->board->panel_width || + v14->panel_height != v14->board->panel_height)) { + dev_err(v14->dev, "boot framebuffer %ux%u with %u notch rows is not the %ux%u %s panel\n", + v14->fb_width, v14->fb_height, notch, v14->board->panel_width, + v14->board->panel_height, v14->board->name); + return -EINVAL; + } return 0; } @@ -1877,38 +1980,45 @@ int iomfb_v14_7_bind(struct apple_dcp *dcp) struct clk *clk; int ret, n; - if (!v14) + /* Only an internal panel binds here, and it always has a board. */ + if (!v14 || WARN_ON_ONCE(!v14->board)) return -ENODEV; if (v14->rtk || dcp_v14_panel_session) return dev_err_probe(dev, -EBUSY, - "T6030 display not started: an earlier DCP session is kept; reboot to restart the display\n"); + "%s display not started: an earlier DCP session is kept; reboot to restart the display\n", + v14->board->name); control = readl(dcp->coproc_reg + DCP_V14_CPU_CONTROL); status = readl(dcp->coproc_reg + DCP_V14_CPU_STATUS); dev_info(dev, "DCP CPU control %#x status %#x\n", control, status); if (!(control & APPLE_DCP_COPROC_CPU_CONTROL_RUN)) return dev_err_probe(dev, -EBUSY, - "T6030 display not started: the DCP CPU is stopped, and interrupted firmware is never resumed\n"); + "%s display not started: the DCP CPU is stopped, and interrupted firmware is never resumed\n", + v14->board->name); ret = dcp_v14_geometry(v14); if (ret) return dev_err_probe(dev, ret, - "T6030 display not started: no usable boot framebuffer\n"); + "%s display not started: no usable boot framebuffer\n", + v14->board->name); clk = clk_get(dev, NULL); if (IS_ERR(clk)) return dev_err_probe(dev, PTR_ERR(clk), - "T6030 display not started: no display clock\n"); + "%s display not started: no display clock\n", + v14->board->name); v14->clock_rate = clk_get_rate(clk); clk_put(clk); if (!v14->clock_rate) return dev_err_probe(dev, -EINVAL, - "T6030 display not started: the display clock has no rate\n"); + "%s display not started: the display clock has no rate\n", + v14->board->name); dcp_v14_link_init(&v14->link, dev, NULL); rtk = apple_rtkit_init(dev, v14, "mbox", 0, &dcp_v14_rtkit_ops); if (IS_ERR(rtk)) return dev_err_probe(dev, PTR_ERR(rtk), - "T6030 display not started: RTKit init failed\n"); + "%s display not started: RTKit init failed\n", + v14->board->name); /* From here nothing is freed and the module stays loaded. */ v14->rtk = rtk; v14->link.rtk = rtk; @@ -1922,7 +2032,8 @@ int iomfb_v14_7_bind(struct apple_dcp *dcp) if (ret) { v14->failed = true; return dev_err_probe(dev, ret, - "T6030 display not started: the DCP RTKit session did not wake; the boot framebuffer stays\n"); + "%s display not started: the DCP RTKit session did not wake; the boot framebuffer stays\n", + v14->board->name); } dev_info(dev, "DCP RTKit session running\n"); /* The internal panel uses IOMFB only; dock DPTX belongs to dcpext. */ @@ -2063,14 +2174,15 @@ static int dcp_v14_mode(struct apple_dcp *dcp, struct apple_dcp_v14 *v14) DRM_MODE_ARG(m), m->type & DRM_MODE_TYPE_PREFERRED ? " best" : ""); if (m->hdisplay != v14->panel_width || m->vdisplay != v14->panel_height - dcp->notch_height || - (hz != 60 && hz != 120)) + (hz != 60 && (hz != 120 || !v14->board->promotion))) continue; if (!best || (m->type & DRM_MODE_TYPE_PREFERRED)) best = &modes[i]; } if (!best) { - dev_err(dcp->dev, "no %ux%u timing at 60 or 120 Hz among %u\n", v14->panel_width, - v14->panel_height - dcp->notch_height, count); + dev_err(dcp->dev, "no %ux%u timing at 60%s Hz among %u\n", v14->panel_width, + v14->panel_height - dcp->notch_height, + v14->board->promotion ? " or 120" : "", count); kfree(modes); return -EINVAL; } @@ -2096,7 +2208,7 @@ int iomfb_v14_7_start(struct apple_dcp *dcp) const char *step; int ret; - if (!v14 || !v14->rtk || v14->failed || v14->started) + if (!v14 || !v14->board || !v14->rtk || v14->failed || v14->started) return -ENODEV; step = "DCPLink"; @@ -2125,7 +2237,7 @@ int iomfb_v14_7_start(struct apple_dcp *dcp) v14->started = true; /* Never removed, like the session it describes. */ - debugfs_create_file("dcp-t6030", 0400, NULL, v14, &dcp_v14_status_fops); + debugfs_create_file(v14->board->debugfs, 0400, NULL, v14, &dcp_v14_status_fops); dcp->connector->connected = true; dcp_set_dimensions(dcp, dcp_modes_transfer_begin(dcp)); scoped_guard(spinlock_irqsave, &dcp->mode_state.lock) @@ -2136,16 +2248,16 @@ int iomfb_v14_7_start(struct apple_dcp *dcp) mutex_lock(&dcp->modes_lock); selected = dcp->modes[0].mode; mutex_unlock(&dcp->modes_lock); - dev_info(dcp->dev, "T6030 display started: %ux%u@%d, %u notch rows %s, %ux%u mm\n", - mode->hdisplay, mode->vdisplay, drm_mode_vrefresh(mode), + dev_info(dcp->dev, "%s display started: %ux%u@%d, %u notch rows %s, %ux%u mm\n", + v14->board->name, mode->hdisplay, mode->vdisplay, drm_mode_vrefresh(mode), dcp->notch_height ?: v14->panel_height - v14->fb_height, dcp->notch_height ? "hidden" : "shown", mode->width_mm, mode->height_mm); return 0; fail: v14->failed = true; dev_err(dcp->dev, - "T6030 display not started: %s failed: %d; the boot framebuffer stays (reboot to retry)\n", - step, ret); + "%s display not started: %s failed: %d; the boot framebuffer stays (reboot to retry)\n", + v14->board->name, step, ret); return ret; } diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.h b/drivers/gpu/drm/apple/iomfb_v14_7.h index 2100e0fc2cded..0cefc51f52651 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7.h @@ -7,6 +7,8 @@ struct apple_dcp; struct apple_dcp_v14; +struct dcp_v14_board; +struct device; struct drm_atomic_commit; struct drm_crtc; struct drm_crtc_state; @@ -20,6 +22,13 @@ const __be32 *of_translate_dma_region_checked(struct device_node *dev, const __be32 *addr, size_t cells, phys_addr_t *start, size_t *length); +/* + * The board of an internal 14.x DCP node (by compatible and machine): NULL if + * the node is not one, ERR_PTR(-ENODEV) if this machine has no board record. + */ +const struct dcp_v14_board *iomfb_v14_7_board(struct device *dev); +const char *iomfb_v14_7_board_name(const struct dcp_v14_board *board); + /* Platform probe: firmware identity, boot loader handoff, PMP acknowledgment. */ int iomfb_v14_7_probe(struct apple_dcp *dcp); /* Component bind: the running DCP's RTKit session. */ From 795d628a0a1b9996b0f6ffa94805948655495e1b Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Tue, 29 Sep 2026 11:51:59 -0400 Subject: [PATCH 1636/2680] drm: apple: iomfb v14.7: accept a boot framebuffer that keeps the notch rows The boot loader can keep the notch rows in the boot framebuffer (chosen.asahi,show-notch=1 in m1n1), still recording apple,notch-height. The geometry check added the notch height to the framebuffer height and so refused the full 2560x1664 J613 framebuffer as a 2560x1728 panel. On a board with a known panel, a framebuffer that is already the panel height is the whole panel. Report the notch rows from apple,notch-height. Boards without a known panel size (T6030) are unchanged. [ck: ported onto the v14.7 backend] (cherry picked from commit 1f370ff0634790c6b0de21bd5006b2a8ab7a3a7f) Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 75a5bcf9f55d4c4481bc4dbf7fd51cd304fb9b66) --- drivers/gpu/drm/apple/iomfb_v14_7.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 28d303c24320a..663b1731e2b3c 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -202,6 +202,8 @@ struct apple_dcp_v14 { u32 stride; u32 fb_width, fb_height; u32 panel_width, panel_height; + /* The panel's notch rows (apple,notch-height), hidden or not. */ + u32 notch_rows; u64 clock_rate; struct dcp_v14_property properties[DCP_V14_MAX_PROPERTIES]; @@ -1392,7 +1394,13 @@ int iomfb_v14_7_probe(struct apple_dcp *dcp) return 0; } -/* The boot framebuffer gives the stride and, with the notch rows, the panel size. */ +/* + * The boot framebuffer gives the stride and the panel size. The boot loader + * either hides the notch rows from it (the default) or keeps them; + * apple,notch-height is recorded either way. A board with a known panel + * tells the two apart by the height; any other board is taken to have the + * notch hidden. + */ static int dcp_v14_geometry(struct apple_dcp_v14 *v14) { struct device_node *fb = of_find_compatible_node(NULL, NULL, "simple-framebuffer"); @@ -1409,14 +1417,17 @@ static int dcp_v14_geometry(struct apple_dcp_v14 *v14) of_node_put(fb); if (ret) return ret; - /* The boot loader hides the notch rows from the boot framebuffer. */ of_property_read_u32(v14->dev->of_node, "apple,notch-height", ¬ch); /* The firmware takes this stride as its default: 4 bytes per pixel. */ if (!v14->fb_width || !v14->fb_height || v14->stride != v14->fb_width * 4 || notch > MAX_NOTCH_HEIGHT) return -EINVAL; + v14->notch_rows = notch; v14->panel_width = v14->fb_width; v14->panel_height = v14->fb_height + notch; + /* The boot loader kept the notch rows: the framebuffer is the whole panel. */ + if (v14->board->panel_height && notch && v14->fb_height == v14->board->panel_height) + v14->panel_height = v14->fb_height; /* A board with a known panel takes no other. */ if (v14->board->panel_width && (v14->panel_width != v14->board->panel_width || @@ -2250,7 +2261,7 @@ int iomfb_v14_7_start(struct apple_dcp *dcp) mutex_unlock(&dcp->modes_lock); dev_info(dcp->dev, "%s display started: %ux%u@%d, %u notch rows %s, %ux%u mm\n", v14->board->name, mode->hdisplay, mode->vdisplay, drm_mode_vrefresh(mode), - dcp->notch_height ?: v14->panel_height - v14->fb_height, + v14->notch_rows, dcp->notch_height ? "hidden" : "shown", mode->width_mm, mode->height_mm); return 0; fail: From 64a3f17ba5352896faea3511250323095574780b Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 7 Oct 2026 12:41:46 -0400 Subject: [PATCH 1637/2680] drm: apple: iomfb v14.7: send the colour matrix (CTM) to DCP The v14 backend accepted the CRTC CTM property but never sent it to the DCP, so night light (Hyprland/hyprsunset CTM) had no effect. The colour matrix methods are per firmware image, so the board record names them: on the J613 14.x image the setter is A421 and the getter A420. Records without them (T6030 for now, and every external processor, which has no record) send nothing, as before. Setter input: location u32 at 0, nine unaligned u64 coefficients at 4, null flag at 76, 3 bytes pad (80 in / 4 out). Getter: 80 in / 76 out, coefficients at 0, status at 72. Coefficients are signed two's-complement Q32, not DRM's sign-magnitude S31.32, so negative DRM coefficients are converted. Inside the presentation lock, a CTM change issues setter then getter before the swap (A406/A407). A missing CTM sends identity; the first presentation and modeset/active changes force reapplication. debugfs shows the nine getter coefficients and both statuses. Tested on J613 (14.x firmware): night light warms the panel. [ck: ported onto the v14.7 backend; the methods moved into the board record so that only tested firmware images get them] (cherry picked from commit adcb94fdada3b294a5b5d472d2050d3430ad050d) Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 10cb090a1a75fbde229d60698988a2d83653211c) --- drivers/gpu/drm/apple/iomfb_v14_7.c | 108 +++++++++++++++++++++++++--- 1 file changed, 99 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 663b1731e2b3c..9270f1564d6de 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -103,6 +104,12 @@ struct dcp_v14_board { u32 panel_width, panel_height; /* The panel may have a 120 Hz timing besides 60 Hz. */ bool promotion; + /* + * The colour matrix setter and getter of this board's firmware image; + * 0: the matrix is not sent. Method numbers differ between firmware + * releases, so they are only set where tested. + */ + u32 ctm_set, ctm_get; }; static const struct dcp_v14_board dcp_v14_board_t6030 = { @@ -128,6 +135,8 @@ static const struct dcp_v14_board dcp_v14_board_j613 = { .panel_width = 2560, .panel_height = 1664, .promotion = false, + .ctm_set = A(421), + .ctm_get = A(420), }; static const struct dcp_v14_board *const dcp_v14_boards[] = { @@ -168,6 +177,11 @@ struct apple_dcp_v14 { /* Owns the RPC stream: start, swaps and idle callbacks. */ struct mutex lock; + /* The colour matrix the firmware holds matches the CRTC's. */ + bool ctm_valid; + u64 ctm_calls; + u32 ctm_status, ctm_get_status; + u64 ctm_readback[9]; struct work_struct idle_work; /* External: reports the display gone once the session stopped. */ struct work_struct stopped_work; @@ -2129,6 +2143,7 @@ static int dcp_v14_status_show(struct seq_file *m, void *unused) { struct apple_dcp_v14 *v14 = m->private; struct drm_framebuffer *fb; + unsigned int i; int ret; ret = mutex_lock_interruptible(&v14->lock); @@ -2142,6 +2157,10 @@ static int dcp_v14_status_show(struct seq_file *m, void *unused) seq_printf(m, "panel %ux%u\nboot_fb %ux%u stride %u\nclock %llu\n", v14->panel_width, v14->panel_height, v14->fb_width, v14->fb_height, v14->stride, v14->clock_rate); + seq_printf(m, "ctm valid %d calls %llu setter %#x getter %#x\n", v14->ctm_valid, + v14->ctm_calls, v14->ctm_status, v14->ctm_get_status); + for (i = 0; i < 9; i++) + seq_printf(m, "ctm_readback[%u] %#llx\n", i, v14->ctm_readback[i]); seq_printf(m, "buffers %u bytes %llu\nproperties %u raw %u\nanalytics %llu\n", v14->buffer_count, v14->buffer_bytes, v14->property_count, v14->raw_count, v14->analytics); @@ -2272,11 +2291,74 @@ int iomfb_v14_7_start(struct apple_dcp *dcp) return ret; } -/* Swap start, then the swap; returns once the firmware completed that swap. */ +/* + * Sends the CRTC's colour matrix, then reads it back, under the RPC lock and + * before the swap that should show it. Only on a board whose firmware image + * has tested matrix methods; elsewhere the matrix is not sent. + * + * Request: location (9) at 0, nine unaligned 64-bit coefficients at 4, a + * null flag at 76, 3 bytes of padding (80 bytes in, 4 out). Readback: 80 in, + * 76 out, coefficients at 0 and the status at 72. The firmware takes signed + * two's-complement Q32 coefficients; DRM gives sign-magnitude S31.32, so + * negative ones are converted. No matrix means identity. + */ +static int dcp_v14_ctm_locked(struct apple_dcp_v14 *v14, + const struct drm_crtc_state *state) +{ + u8 request[80] = {}, readback[76] = {}; + const struct drm_color_ctm *ctm; + __le32 status = 0; + int i, ret; + + if (!v14->board || !v14->board->ctm_set || !state || + (v14->ctm_valid && !state->color_mgmt_changed && + !state->mode_changed && !state->active_changed)) + return 0; + ctm = state->ctm ? state->ctm->data : NULL; + put_unaligned_le32(9, request); + for (i = 0; i < 9; i++) { + u64 coefficient = ctm ? ctm->matrix[i] : + ((i == 0 || i == 4 || i == 8) ? 1ULL << 32 : 0); + u64 magnitude = coefficient & ~(1ULL << 63); + u64 signed_q32 = coefficient & (1ULL << 63) ? -magnitude : magnitude; + + put_unaligned_le64(signed_q32, request + 4 + 8 * i); + } + ret = dcp_v14_call(v14, v14->board->ctm_set, request, sizeof(request), + &status, sizeof(status), 0); + v14->ctm_calls++; + v14->ctm_status = le32_to_cpu(status); + v14->ctm_valid = false; + if (ret || v14->ctm_status) { + dev_err(v14->dev, "CTM setter transport=%d status=%#x\n", + ret, v14->ctm_status); + return ret ? ret : -EIO; + } + /* Read back before the swap, under the same lock. */ + memset(request + 4, 0, 76); + ret = dcp_v14_call(v14, v14->board->ctm_get, request, sizeof(request), + readback, sizeof(readback), 0); + v14->ctm_get_status = get_unaligned_le32(readback + 72); + for (i = 0; i < 9; i++) + v14->ctm_readback[i] = get_unaligned_le64(readback + i * 8); + dev_info(v14->dev, "CTM call %llu setter=%#x getter_transport=%d getter=%#x diagonal=%#llx,%#llx,%#llx\n", + v14->ctm_calls, v14->ctm_status, ret, v14->ctm_get_status, + v14->ctm_readback[0], v14->ctm_readback[4], v14->ctm_readback[8]); + if (ret || v14->ctm_get_status) + return ret ? ret : -EIO; + v14->ctm_valid = true; + return 0; +} + +/* + * Swap start, then the swap; returns once the firmware completed that swap. + * @ctm_state, if set, carries the colour matrix to send first. + */ static int dcp_v14_swap(struct apple_dcp_v14 *v14, const u8 *surface, u64 iova, - u32 width, u32 height, u32 dst_y) + u32 width, u32 height, u32 dst_y, + const struct drm_crtc_state *ctm_state) { - __le32 start[4] = {}, started[2], result[3]; + __le32 start[4] = {}, started[2] = {}, result[3]; u8 *swap; u32 id; int ret; @@ -2289,7 +2371,10 @@ static int dcp_v14_swap(struct apple_dcp_v14 *v14, const u8 *surface, u64 iova, return -ENOMEM; mutex_lock(&v14->lock); - ret = dcp_v14_call(v14, A(406), start, sizeof(start), started, sizeof(started), 0); + ret = dcp_v14_ctm_locked(v14, ctm_state); + if (!ret) + ret = dcp_v14_call(v14, A(406), start, sizeof(start), started, + sizeof(started), 0); if (!ret && le32_to_cpu(started[1])) ret = -EIO; id = le32_to_cpu(started[0]); @@ -2318,11 +2403,13 @@ static int dcp_v14_swap(struct apple_dcp_v14 *v14, const u8 *surface, u64 iova, } /* - * Shows the top-left @width x @height of @fb (or only the black background) + * Shows the top-left @width x @height of @fb (or only the black background), + * after the colour matrix of @ctm_state if that is set, * and keeps @fb until the next swap. */ static bool dcp_v14_present(struct apple_dcp_v14 *v14, struct drm_framebuffer *fb, - u32 width, u32 height, u32 dst_y) + u32 width, u32 height, u32 dst_y, + const struct drm_crtc_state *ctm_state) { u8 surface[DCP_V14_SURFACE_SIZE]; struct drm_framebuffer *old; @@ -2339,7 +2426,7 @@ static bool dcp_v14_present(struct apple_dcp_v14 *v14, struct drm_framebuffer *f } start = ktime_get_ns(); ret = dcp_v14_swap(v14, fb ? surface : NULL, iova, fb ? width : 0, - fb ? height : 0, dst_y); + fb ? height : 0, dst_y, ctm_state); elapsed = ktime_get_ns() - start; if (ret && v14->external && !READ_ONCE(v14->failed)) { /* Refused before it was taken: the old framebuffer stays on screen. */ @@ -2641,10 +2728,13 @@ void iomfb_v14_7_flush(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_commit *state) { struct drm_plane_state *p = drm_atomic_get_new_plane_state(state, crtc->primary); + struct drm_crtc_state *cs = drm_atomic_get_new_crtc_state(state, crtc); struct drm_framebuffer *fb; if (!p) p = crtc->primary->state; + if (!cs) + cs = crtc->state; fb = p && p->visible ? p->fb : NULL; /* * No swaps while an external display is off, unset, unplugged, or @@ -2661,7 +2751,7 @@ void iomfb_v14_7_flush(struct apple_dcp *dcp, struct drm_crtc *crtc, if (dcp->v14 && dcp_v14_present(dcp->v14, fb, fb ? (dcp->external ? p->src_w >> 16 : fb->width) : 0, fb ? (dcp->external ? p->src_h >> 16 : fb->height) : 0, - dcp->notch_height)) + dcp->notch_height, cs)) dcp_drm_crtc_page_flip(dcp, ktime_get()); else dcp_v14_cancel_event(dcp); @@ -2689,5 +2779,5 @@ void iomfb_v14_7_poweroff(struct apple_dcp *dcp) } /* Blank to black; the panel and the DCP stay powered. */ if (v14 && v14->started) - dcp_v14_present(v14, NULL, 0, 0, 0); + dcp_v14_present(v14, NULL, 0, 0, 0, NULL); } From e1cf0436283fd9ef9effa6dc440a8902b3b640f6 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 7 Oct 2026 12:41:46 -0400 Subject: [PATCH 1638/2680] drm: apple: iomfb v14.7: a refused CTM must not take the display down As written, a setter/getter failure failed the swap and so the frame, and with ctm_valid false it would retry on every frame. Now a failure logs once, sets ctm_disabled (shown in debugfs) and the swap goes ahead. CTM log lines are rate-limited (Hyprland animates CTM changes). [ck: ported onto the v14.7 backend] (cherry picked from commit 928edd987741174cd1cba4d8bde154741251f7d6) Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 8fd4733c7093a28e6c59cb5952cab869e02e99d9) --- drivers/gpu/drm/apple/iomfb_v14_7.c | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 9270f1564d6de..39fce116220f4 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -179,6 +179,8 @@ struct apple_dcp_v14 { struct mutex lock; /* The colour matrix the firmware holds matches the CRTC's. */ bool ctm_valid; + /* The firmware refused a matrix: none is sent until reboot. */ + bool ctm_disabled; u64 ctm_calls; u32 ctm_status, ctm_get_status; u64 ctm_readback[9]; @@ -2157,8 +2159,9 @@ static int dcp_v14_status_show(struct seq_file *m, void *unused) seq_printf(m, "panel %ux%u\nboot_fb %ux%u stride %u\nclock %llu\n", v14->panel_width, v14->panel_height, v14->fb_width, v14->fb_height, v14->stride, v14->clock_rate); - seq_printf(m, "ctm valid %d calls %llu setter %#x getter %#x\n", v14->ctm_valid, - v14->ctm_calls, v14->ctm_status, v14->ctm_get_status); + seq_printf(m, "ctm valid %d disabled %d calls %llu setter %#x getter %#x\n", + v14->ctm_valid, v14->ctm_disabled, v14->ctm_calls, v14->ctm_status, + v14->ctm_get_status); for (i = 0; i < 9; i++) seq_printf(m, "ctm_readback[%u] %#llx\n", i, v14->ctm_readback[i]); seq_printf(m, "buffers %u bytes %llu\nproperties %u raw %u\nanalytics %llu\n", @@ -2310,7 +2313,7 @@ static int dcp_v14_ctm_locked(struct apple_dcp_v14 *v14, __le32 status = 0; int i, ret; - if (!v14->board || !v14->board->ctm_set || !state || + if (!v14->board || !v14->board->ctm_set || !state || v14->ctm_disabled || (v14->ctm_valid && !state->color_mgmt_changed && !state->mode_changed && !state->active_changed)) return 0; @@ -2330,8 +2333,9 @@ static int dcp_v14_ctm_locked(struct apple_dcp_v14 *v14, v14->ctm_status = le32_to_cpu(status); v14->ctm_valid = false; if (ret || v14->ctm_status) { - dev_err(v14->dev, "CTM setter transport=%d status=%#x\n", + dev_err(v14->dev, "CTM setter transport=%d status=%#x; colour matrix off until reboot\n", ret, v14->ctm_status); + v14->ctm_disabled = true; return ret ? ret : -EIO; } /* Read back before the swap, under the same lock. */ @@ -2341,11 +2345,14 @@ static int dcp_v14_ctm_locked(struct apple_dcp_v14 *v14, v14->ctm_get_status = get_unaligned_le32(readback + 72); for (i = 0; i < 9; i++) v14->ctm_readback[i] = get_unaligned_le64(readback + i * 8); - dev_info(v14->dev, "CTM call %llu setter=%#x getter_transport=%d getter=%#x diagonal=%#llx,%#llx,%#llx\n", + dev_info_ratelimited(v14->dev, "CTM call %llu setter=%#x getter_transport=%d getter=%#x diagonal=%#llx,%#llx,%#llx\n", v14->ctm_calls, v14->ctm_status, ret, v14->ctm_get_status, v14->ctm_readback[0], v14->ctm_readback[4], v14->ctm_readback[8]); - if (ret || v14->ctm_get_status) + if (ret || v14->ctm_get_status) { + dev_err(v14->dev, "CTM getter failed; colour matrix off until reboot\n"); + v14->ctm_disabled = true; return ret ? ret : -EIO; + } v14->ctm_valid = true; return 0; } @@ -2371,10 +2378,12 @@ static int dcp_v14_swap(struct apple_dcp_v14 *v14, const u8 *surface, u64 iova, return -ENOMEM; mutex_lock(&v14->lock); - ret = dcp_v14_ctm_locked(v14, ctm_state); - if (!ret) - ret = dcp_v14_call(v14, A(406), start, sizeof(start), started, - sizeof(started), 0); + /* + * A colour matrix the firmware refuses costs night light, not the + * frame: present anyway, and stop sending it (ctm_disabled). + */ + dcp_v14_ctm_locked(v14, ctm_state); + ret = dcp_v14_call(v14, A(406), start, sizeof(start), started, sizeof(started), 0); if (!ret && le32_to_cpu(started[1])) ret = -EIO; id = le32_to_cpu(started[0]); From d482911d886f52bc9a67e4b9ce66eda672002086 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:52:04 -0700 Subject: [PATCH 1639/2680] drm/apple: t6030: keep a Type-C display connected while re-described MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Right after a USB-C display is attached to a T6030 external processor, the firmware can withdraw the display's description and publish it again a fraction of a second later, with the port's HPD held. On the lab M3 Pro this happened in all six logged boots, each time within 1-40 ms of the compositor's first full connector probe. The driver reported it as a disconnect and a reconnect. A compositor that had just read the connector as connected could then read it back, while it was withdrawn, with no modes and no EDID. aquamarine (stock Hyprland) creates its output from that second read, fails its enable commit with "No mode on enable commit", and keeps the empty output when its next scan finds the display connected again. DP-3 then never gets a mode. When the firmware withdraws the timings of a connected display on a direct Type-C route whose port still has HPD, and the existing retry is queued to redo the link if the display is not described again, keep the connector connected with the modes and EDID it has. The mode is invalidated as before, and a mode set in the window already waits for the description to come back. Describing the display again ends the hold and sends at most one hotplug event, as before; it keeps the EDID, unless that is a placeholder being retried, so the probes that follow do not read it again. If the retry finds the display not described again, it reports the disconnect first and then redoes the link, as before. Only the first withdrawal of a connection is held, so this cannot drive a loop: a display that keeps being withdrawn is reported as it was before. HDMI outputs, Thunderbolt and USB4 tunnels, CRTC power-off, the internal panel and M1/M2 pipes take the old path, as does a withdrawal with no retry left. Tests: a KUnit suite drives the production detect, get_modes and mode_valid callbacks through the DRM probe helper, with a DRM client counting hotplug events. It shows that the read in the withdrawn window returns the display's modes and EDID, and that describing the display again sends one event. If the retry finds the display gone, it sends one disconnect event. A second withdrawal in the same connection disconnects. The suite also covers the paths that keep the old behaviour. There are mode-state cases for the withdrawal and a catalog case for the kept EDID. With the hold disabled, the suite reproduces the empty read. [ryan: adapted to 7.2.10 — the held-display EDID rule sits in our dcp_modes_replace() publication (not H17P external); external-hold-test.c is left out because its fixture is built on the edid_lock/edid_begin/edid_install helpers and a spinlock dcpavserv.lock that this tree's generation-fenced EDID model does not have; parser_held_keeps_edid checks edid_invalidation_pending instead of a cached EDID] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit b2e2021dbe1bd50e282cc9efeaed06826c49b949) --- drivers/gpu/drm/apple/dcp-internal.h | 7 ++ drivers/gpu/drm/apple/dcp.c | 5 +- drivers/gpu/drm/apple/iomfb-state-test.c | 43 ++++++++++++ drivers/gpu/drm/apple/iomfb-state.h | 20 ++++++ drivers/gpu/drm/apple/iomfb.c | 8 ++- drivers/gpu/drm/apple/iomfb_v14_7.c | 85 +++++++++++++++++++++--- drivers/gpu/drm/apple/iomfb_v14_7.h | 8 +++ drivers/gpu/drm/apple/parser-test.c | 44 ++++++++++++ 8 files changed, 207 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 59a39c82c0dcb..b8a85e5a6599b 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -271,6 +271,13 @@ struct apple_dcp { atomic_t external_retries; /* typec_generation when the pending retry was queued */ u64 external_retry_generation; + /* + * Native: the connector is kept connected while the display is + * described again, until the pending retry, once per connection + * (typec_generation + 1 of the last hold); see iomfb_v14_7.c. + */ + atomic_t external_held; + u64 external_held_connection; /* * Native: DRM unbound this pipe; firmware callbacks leave its * connector and CRTC alone. Set under the session lock. diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 3dec4d365aa45..02380079ea687 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1343,17 +1343,20 @@ void dcp_external_retry_work(struct work_struct *work) external_retry_wq); struct apple_connector *connector = READ_ONCE(dcp->connector); u64 generation = READ_ONCE(dcp->external_retry_generation); + bool released; if (READ_ONCE(dcp->typec_generation) != generation) { dev_info(dcp->dev, "display retry: the display was attached or detached since; dropped\n"); return; } + /* Kept connected through a withdrawal, and not described again. */ + released = iomfb_v14_7_external_release(dcp); if (iomfb_v14_7_external_failed(dcp)) { dev_err(dcp->dev, "display retry: the external display processor stopped; no retry until reboot\n"); return; } /* Described again: re-apply the mode if it is not set. */ - if (connector && READ_ONCE(connector->connected)) { + if (!released && connector && READ_ONCE(connector->connected)) { if (READ_ONCE(dcp->mode_state.valid)) { dev_info(dcp->dev, "display retry: the display is back and set\n"); return; diff --git a/drivers/gpu/drm/apple/iomfb-state-test.c b/drivers/gpu/drm/apple/iomfb-state-test.c index 97ba9924d454f..2d872395c0b4f 100644 --- a/drivers/gpu/drm/apple/iomfb-state-test.c +++ b/drivers/gpu/drm/apple/iomfb-state-test.c @@ -107,6 +107,47 @@ static void dcp_mode_disconnected_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, state.valid); } +/* A withdrawn mode keeps the connector connected and flushes vblanks once. */ +static void dcp_mode_withdraw_idle_test(struct kunit *test) +{ + struct dcp_mode_state state = { .valid = true }; + bool connected = true; + + spin_lock_init(&state.lock); + KUNIT_EXPECT_EQ(test, dcp_mode_withdraw(&state), (unsigned int)DCP_HOTPLUG_VBLANK); + KUNIT_EXPECT_TRUE(test, connected); + KUNIT_EXPECT_FALSE(test, state.valid); + /* Described again: no state change, so no notification from here. */ + KUNIT_EXPECT_EQ(test, dcp_mode_hotplug(&state, true, &connected), 0U); + KUNIT_EXPECT_TRUE(test, connected); + KUNIT_EXPECT_FALSE(test, state.valid); + /* Not described again: the disconnect notifies once. */ + KUNIT_EXPECT_EQ(test, dcp_mode_hotplug(&state, false, &connected), + (unsigned int)(DCP_HOTPLUG_VBLANK | DCP_HOTPLUG_NOTIFY)); + KUNIT_EXPECT_FALSE(test, connected); + KUNIT_EXPECT_EQ(test, dcp_mode_hotplug(&state, false, &connected), + (unsigned int)DCP_HOTPLUG_VBLANK); +} + +/* During a modeset the withdrawal is deferred: one notification, mode invalid. */ +static void dcp_mode_withdraw_modeset_test(struct kunit *test) +{ + struct dcp_mode_state state = { }; + bool connected = true; + + spin_lock_init(&state.lock); + dcp_mode_begin(&state); + KUNIT_EXPECT_EQ(test, dcp_mode_withdraw(&state), 0U); + KUNIT_EXPECT_EQ(test, dcp_mode_finish(&state, true, &connected), + (unsigned int)DCP_HOTPLUG_NOTIFY); + KUNIT_EXPECT_TRUE(test, connected); + KUNIT_EXPECT_FALSE(test, state.valid); + /* The replay that notification asks for can then succeed. */ + dcp_mode_begin(&state); + KUNIT_EXPECT_EQ(test, dcp_mode_finish(&state, true, &connected), 0U); + KUNIT_EXPECT_TRUE(test, state.valid); +} + static struct kunit_case dcp_mode_cases[] = { KUNIT_CASE(dcp_mode_success_test), KUNIT_CASE(dcp_mode_failure_test), @@ -115,6 +156,8 @@ static struct kunit_case dcp_mode_cases[] = { KUNIT_CASE(dcp_mode_transient_disconnect_test), KUNIT_CASE(dcp_mode_invalidation_order_test), KUNIT_CASE(dcp_mode_disconnected_test), + KUNIT_CASE(dcp_mode_withdraw_idle_test), + KUNIT_CASE(dcp_mode_withdraw_modeset_test), { } }; diff --git a/drivers/gpu/drm/apple/iomfb-state.h b/drivers/gpu/drm/apple/iomfb-state.h index f6d512f3c4f43..84d16ca4a9405 100644 --- a/drivers/gpu/drm/apple/iomfb-state.h +++ b/drivers/gpu/drm/apple/iomfb-state.h @@ -59,6 +59,26 @@ static inline bool dcp_mode_invalidate(struct dcp_mode_state *state) return valid; } +/* + * The display's mode went away while its connector stays connected. A + * modeset in flight finishes invalid and notifies once (dcp_mode_finish()); + * otherwise the vblanks waiting on the mode are flushed. + */ +static inline unsigned int dcp_mode_withdraw(struct dcp_mode_state *state) +{ + unsigned long flags; + unsigned int action = DCP_HOTPLUG_NONE; + + spin_lock_irqsave(&state->lock, flags); + state->valid = false; + if (state->changing) + state->invalidated = true; + else + action = DCP_HOTPLUG_VBLANK; + spin_unlock_irqrestore(&state->lock, flags); + return action; +} + /* The caller holds state->lock. */ static inline unsigned int dcp_mode_apply_hotplug(struct dcp_mode_state *state, bool connected, diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index e4105ec285d04..cdee006c077cb 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1635,7 +1635,13 @@ int dcp_modes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *ctx, ready = dcp->connector; mutex_unlock(&dcp->modes_lock); - if (!ret && !h17p_external) + /* + * Described again after a withdrawal it was kept connected through, + * the same display keeps its EDID, unless that is a placeholder being + * retried. + */ + if (!ret && !h17p_external && + (!atomic_read(&dcp->external_held) || READ_ONCE(dcp->placeholder_retried))) apple_connector_invalidate_edid(dcp->connector); /* Notify removals as well as newly eligible modes, outside the lock. */ dcp_180hz_catalog_changed(dcp); diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 39fce116220f4..579cb6b8ba66e 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -522,7 +522,7 @@ static int dcp_v14_query(struct apple_dcp_v14 *v14, u32 tag) * The attached display came or went, as the firmware's display description * shows it. Mirrors the hotplug callback of the M1/M2 firmware. */ -static void dcp_v14_external_hotplug(struct apple_dcp *dcp, bool connected) +void dcp_v14_external_hotplug(struct apple_dcp *dcp, bool connected) { struct apple_connector *connector = READ_ONCE(dcp->connector); unsigned int action; @@ -543,6 +543,9 @@ static void dcp_v14_external_hotplug(struct apple_dcp *dcp, bool connected) /* A description that outlives its cable is stale. */ if (connected && dcp_is_typec_output(dcp) && !READ_ONCE(dcp->typec_cable_connected)) return; + /* Described again: a connector kept connected meanwhile is current. */ + if (connected) + atomic_set(&dcp->external_held, 0); if (connected && dcp_is_typec_output(dcp)) complete_all(&dcp->typec_iomfb_hpd_ready); if (!connected) @@ -559,6 +562,75 @@ static void dcp_v14_external_hotplug(struct apple_dcp *dcp, bool connected) dcp_handle_hotplug_actions(dcp, action); } +/* + * The firmware withdrew the description of a connected display on a direct + * Type-C route whose port still has HPD, and a retry is queued that redoes + * the link if the description does not come back. Shortly after an attach + * the firmware can do this and describe the display again a fraction of a + * second later. Reporting the connector disconnected for that window lets a + * compositor that found it connected a moment before read it back with no + * modes and no EDID, and keep that empty output once the display is back. + * Keep it connected with the modes and EDID it has, until the description is + * back or the retry finds it is not (iomfb_v14_7_external_release()). A mode + * set meanwhile waits for the description (dcp_v14_external_settle()). Only + * the first withdrawal of a connection is held; later ones disconnect, so a + * display that keeps being withdrawn is reported as it was before. + */ +static bool dcp_v14_external_hold(struct apple_dcp *dcp) +{ + struct apple_connector *connector = READ_ONCE(dcp->connector); + u64 connection = READ_ONCE(dcp->typec_generation) + 1; + unsigned int action; + + if (!dcp->external_native || READ_ONCE(dcp->external_detached) || + !dcp_is_typec_output(dcp) || dcp_is_usb4_output(dcp) || + !READ_ONCE(dcp->typec_cable_connected) || READ_ONCE(dcp->typec_crtc_off) || + !connector || !READ_ONCE(connector->connected) || + READ_ONCE(dcp->external_held_connection) == connection || + !delayed_work_pending(&dcp->external_retry_wq)) + return false; + WRITE_ONCE(dcp->external_held_connection, connection); + atomic_set(&dcp->external_held, 1); + action = dcp_mode_withdraw(&dcp->mode_state); + if (!dcp->crtc) + action &= ~DCP_HOTPLUG_VBLANK; + dcp_handle_hotplug_actions(dcp, action); + dev_info(dcp->dev, "display withdrawn on a Type-C route with HPD: kept connected until it is described again\n"); + return true; +} + +/* + * The firmware withdrew the timings of the attached display. An HPD bounce + * withdraws the description, and the firmware describes the display again + * once its link is back. If that does not happen while the port still holds + * the display, redo the link. + */ +void iomfb_v14_7_external_withdrawn(struct apple_dcp *dcp) +{ + if (READ_ONCE(dcp->typec_cable_connected) && !READ_ONCE(dcp->typec_crtc_off)) + dcp_external_retry(dcp, "display withdrawn while its port has HPD", + 0, DCP_V14_DESC_RELINK_MS); + if (!dcp_v14_external_hold(dcp)) + dcp_v14_external_hotplug(dcp, false); +} + +/* + * The retry found the display of a connector kept connected by + * dcp_v14_external_hold() not described again: report it disconnected. + * Returns whether it did. + */ +bool iomfb_v14_7_external_release(struct apple_dcp *dcp) +{ + struct apple_connector *connector = READ_ONCE(dcp->connector); + + if (!atomic_xchg(&dcp->external_held, 0) || !connector || + !READ_ONCE(connector->connected)) + return false; + dev_info(dcp->dev, "display not described again: reporting it disconnected\n"); + dcp_v14_external_hotplug(dcp, false); + return true; +} + /* * A display description the firmware published (or removed, @removed) on an * external processor: the timings and color modes of the attached display @@ -621,16 +693,7 @@ static void dcp_v14_external_published(struct apple_dcp_v14 *v14, const char *ke if (!strcmp(key, "TimingElements")) { if (removed) { - dcp_v14_external_hotplug(dcp, false); - /* - * An HPD bounce withdraws the description, and the - * firmware describes the display again once its link is - * back. If that does not happen while the port still - * holds the display, redo the link. - */ - if (READ_ONCE(dcp->typec_cable_connected) && !READ_ONCE(dcp->typec_crtc_off)) - dcp_external_retry(dcp, "display withdrawn while its port has HPD", - 0, DCP_V14_DESC_RELINK_MS); + iomfb_v14_7_external_withdrawn(dcp); return; } ret = parse(raw->data, raw->size, &ctx); diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.h b/drivers/gpu/drm/apple/iomfb_v14_7.h index 0cefc51f52651..f6fe9e199b014 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7.h @@ -50,6 +50,14 @@ bool iomfb_v14_7_external_failed(struct apple_dcp *dcp); /* The firmware runs this mode, set last, and shows swaps in it. */ bool iomfb_v14_7_external_showing(struct apple_dcp *dcp, const struct drm_display_mode *mode); +/* + * The attached display came or went (_hotplug), or its timings were + * withdrawn (_withdrawn); _release() reports a display that was kept + * connected through a withdrawal, and is not described again, gone. + */ +void dcp_v14_external_hotplug(struct apple_dcp *dcp, bool connected); +void iomfb_v14_7_external_withdrawn(struct apple_dcp *dcp); +bool iomfb_v14_7_external_release(struct apple_dcp *dcp); /* Component unbind: KMS goes away; the firmware session is kept until reboot. */ void iomfb_v14_7_unbind(struct apple_dcp *dcp); /* Platform remove: the apple_dcp is freed next; the firmware session is kept. */ diff --git a/drivers/gpu/drm/apple/parser-test.c b/drivers/gpu/drm/apple/parser-test.c index 58a31cabbaf87..34b24ddbad282 100644 --- a/drivers/gpu/drm/apple/parser-test.c +++ b/drivers/gpu/drm/apple/parser-test.c @@ -475,6 +475,49 @@ static void parser_replacement(struct kunit *test) kfree(dcp->modes); } +/* + * Described again after a withdrawal it was kept connected through, the same + * display keeps its EDID; any other replacement, or a placeholder being + * retried, drops it. + */ +static void parser_held_keeps_edid(struct kunit *test) +{ + struct parser_record record = valid_record(7, 10); + struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + struct apple_connector *connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL); + struct platform_device *pipeline = kunit_kzalloc(test, sizeof(*pipeline), GFP_KERNEL); + struct dcp_parse_ctx ctx; + struct parser_blob *blob; + unsigned int pass; + + KUNIT_ASSERT_NOT_NULL(test, dcp); + KUNIT_ASSERT_NOT_NULL(test, connector); + KUNIT_ASSERT_NOT_NULL(test, pipeline); + dcp_modes_init(dcp); + spin_lock_init(&dcp->metadata.lock); + apple_connector_edid_init(connector); + apple_connector_set_pipeline(connector, pipeline); + dcp->connector = connector; + blob = make_blob(test, &record, 1); + KUNIT_ASSERT_NOT_NULL(test, blob); + /* Pass 0: not held. Pass 1: held. Pass 2: held, placeholder retried. */ + for (pass = 0; pass < 3; pass++) { + /* The cached EDID is dropped by marking it stale for get_modes(). */ + WRITE_ONCE(connector->edid_invalidation_pending, false); + atomic_set(&dcp->external_held, pass != 0); + dcp->placeholder_retried = pass == 2; + KUNIT_ASSERT_EQ(test, parse(blob->data, blob->size, &ctx), 0); + ctx.dcp = &parser_dcp; + KUNIT_ASSERT_EQ(test, dcp_modes_replace(dcp, &ctx, + dcp_modes_transfer_begin(dcp)), 0); + if (pass == 1) + KUNIT_EXPECT_FALSE(test, connector->edid_invalidation_pending); + else + KUNIT_EXPECT_TRUE(test, connector->edid_invalidation_pending); + } + kfree(dcp->modes); +} + static void parser_attachment_admission(struct kunit *test) { struct parser_record record = valid_record(7, 10); @@ -1203,6 +1246,7 @@ static struct kunit_case parser_cases[] = { KUNIT_CASE(parser_bad_score), KUNIT_CASE(parser_notch_vrr), KUNIT_CASE(parser_replacement), + KUNIT_CASE(parser_held_keeps_edid), KUNIT_CASE(parser_attachment_admission), KUNIT_CASE(parser_stale_transfer), KUNIT_CASE(parser_route_catalog_handoff), From 4c5f266e6f92112dda11d448f60477d80402de11 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 18 Oct 2026 03:05:23 +1100 Subject: [PATCH 1640/2680] selftests: apple-dcp: Follow the fabric core split and the T6030 backend The Type-C fabric now decides through the pure functions in dcp-fabric-core.c and the effect runners in dcp-fabric-effects.h, and the T6030 native 14.7 backend added fw_compat branches, external processor state and the 14.7 DCPLink encoders in dcp-link.h. Compile the real fabric core and effects into the host units instead of modelling the decisions, extract dcp_typec_rebalance_locked() with its rebalance adapter, and give the mocks the new members, enum values and stubs. The new 14.7 stubs assert that the scenarios never reach them. No scenario assertion is removed or relaxed. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/gpu/apple-dcp/common.py | 41 +++++++++++++++++++ .../selftests/drivers/gpu/apple-dcp/edid.py | 5 ++- .../drivers/gpu/apple-dcp/lifecycle.py | 32 ++++++++++++++- .../drivers/gpu/apple-dcp/quiesce.py | 19 ++++++++- .../selftests/drivers/gpu/apple-dcp/rate.py | 7 +++- .../drivers/gpu/apple-dcp/recovery.py | 26 +++++++++++- .../drivers/gpu/apple-dcp/transaction.py | 15 ++++++- 7 files changed, 136 insertions(+), 9 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py index 96d127be946ce..0cd119bfad4b8 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py @@ -122,3 +122,44 @@ def atc_mux_without_pm(source): return ret; } ''' + + +def fabric_core(root): + """The real Type-C fabric decisions and effect runners as host C. + + dcp-fabric-core.c and dcp-fabric-effects.h are host-compilable by design + (KUnit runs them against fake hardware); only the kernel includes and + module boilerplate are dropped. The unit supplies u32/u64. Unused + static inline helpers are marked so, as they would be in a header. + """ + apple = root / 'drivers/gpu/drm/apple' + core = apple / 'dcp-fabric-core.c' + if not core.exists(): + return '' + + def strip(text): + return '\n'.join(line for line in text.splitlines() + if not line.startswith(('#include', 'EXPORT_SYMBOL', 'MODULE_'))) + '\n' + tunnel = (root / 'include/linux/soc/apple/dp-tunnel.h').read_text() + return ('\n#include \n#include \n#include \n' + '#ifndef BIT\n#define BIT(n) (1UL << (n))\n#endif\n' + '#ifndef time_before\n#define time_before(a, b) ((long)((a) - (b)) < 0)\n#endif\n' + + block(tunnel, 'static inline bool apple_dp_tunnel_wiring_dual(') + '\n' + + strip((apple / 'dcp-fabric-core.h').read_text()) + + strip(core.read_text()) + + strip((apple / 'dcp-fabric-effects.h').read_text())).replace( + 'static inline ', 'static inline __attribute__((unused)) ') + + +def fabric_rebalance(dcp): + """dcp_typec_rebalance_locked() with the adapter callbacks it runs through. + + The fabric core split turned the loop into dcp_fabric_run_rebalance() over + struct dcp_rebalance_context and dcp_rebalance_ops; older sources keep the + loop inline. + """ + marker = 'static struct apple_dcp_typec_route *\ndcp_typec_rebalance_locked(' + end = dcp.index(marker) + end += len(block(dcp, end)) + start = dcp.find('struct dcp_rebalance_context {') + return dcp[start if 0 <= start < end else dcp.index(marker):end] + '\n' diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py index fc86c7f8be7df..a3fa8db014b05 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py @@ -50,6 +50,9 @@ def function(file,signature): static unsigned diagnostic_errors,diagnostic_information,legacy_publishes; #define DCP_FIRMWARE_H17P 17 +/* The T6030 14.7 backend never matches these H17P scenarios. */ +#define DCP_FIRMWARE_V_14_7 (DCP_FIRMWARE_H17P + 1) +#define DCP_FIRMWARE_V_14_7_TEMPLATE (DCP_FIRMWARE_H17P + 2) #define EPIC_TYPE_NOTIFY 0 #define EPIC_CAT_REPORT 0 #define max(a,b) ((a)>(b)?(a):(b)) @@ -76,7 +79,7 @@ def function(file,signature): struct apple_dcp;struct apple_dcp_afkep;struct apple_epic_service; void afk_service_retry(struct apple_dcp_afkep *ep); struct apple_epic_service_ops { char name[32];bool reusable,optional;void (*init)(struct apple_epic_service *,const char *,const char *,s64);int (*prepare)(struct apple_epic_service *,const char *,const char *,s64);int (*revalidate)(struct apple_epic_service *,const char *,const char *,s64);void (*teardown)(struct apple_epic_service *); }; -struct epic_cmd_info { u16 tag;u8 type;size_t reply_length;void *rxbuf,*txbuf;dma_addr_t rxbuf_dma,txbuf_dma;size_t rxlen,txlen;u32 retcode;bool done,free_on_ack;int *completion; }; +struct epic_cmd_info { u16 tag;u8 type;size_t reply_length,reply_len;bool reply_len_valid;void *rxbuf,*txbuf;dma_addr_t rxbuf_dma,txbuf_dma;size_t rxlen,txlen;u32 retcode;bool done,free_on_ack;int *completion; }; struct apple_epic_service { struct apple_dcp_afkep *ep;const struct apple_epic_service_ops *ops; bool lock,enabled,torndown;unsigned int users;u32 channel;u8 cmd_tag; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 9069b8f73d243..279718cbe2ebe 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -430,7 +430,8 @@ def swap(old, new): swap(' struct { bool connected; void *service; } dptxport[1];', ' struct { bool connected, closed; void *service, *atcphy; int linkcfg_completion; } dptxport[1];\n' ' void *phy, *dev, *active_typec_route;\n' - ' int dptx_die, dptx_phy, dptx_dfp_port, connector_type;') + ' int dptx_die, dptx_phy, dptx_dfp_port, connector_type;\n' + ' bool external, external_link_ready, crashed;') old_raw = unit[unit.index('static int dcp_dptx_connect_raw('):] old_raw = old_raw[:old_raw.index('return raw_error; }') + len('return raw_error; }')] swap(old_raw, 'static int dcp_dptx_connect_raw(struct apple_dcp *, u32, bool *);') @@ -473,6 +474,12 @@ def swap(old, new): static int dptxport_request_display(void *s) { (void)s; fw_display |= request_error != -EINVAL; return request_error; } static int dptxport_set_hpd_timeout(void *s, bool on, int t) { (void)t; return dptxport_set_hpd(s, on); } +/* The pre-split condition: neither a USB4 output nor a T602X tunnel machine here. */ +static bool dcp_uses_t6020_tunnel_flow(struct apple_dcp *d) { return dcp_is_usb4_output(d) && apple_dp_tunnel_t602x(); } +/* An H17P display is not a T6030 dcpext (dcp->external): no scanout to fault or restore. */ +#define smp_store_release(p, v) (*(p) = (v)) +static bool dcpext_scanout_terminal(struct apple_dcp *d) { (void)d; return false; } +static void dcpext_scanout_link_restored(struct apple_dcp *d) { (void)d; assert(!"T6030 dcpext path in an H17P scenario"); } ''' tests = r''' static struct platform_device device = { .data = &display }; @@ -875,6 +882,8 @@ def block(marker): headers = "\n".join(line for name in ("dcp-external-bw.h", "dcp-link.h") for line in (base / name).read_text().splitlines() if not line.startswith("#include")) + # Header inline helpers this unit does not call are unused, as in a header. + headers = headers.replace("static inline ", "static inline __attribute__((unused)) ") prefix = r''' #include #include @@ -884,6 +893,16 @@ def block(marker): #include typedef uint32_t u32; typedef uint64_t u64; typedef uint32_t __le32; typedef uint64_t __le64; +/* dcp-link.h also carries the T6030 14.7 DCPLink transport encoders. */ +typedef uint8_t u8; typedef uint16_t u16; +#define le64_to_cpu(v) (v) +#define BIT_ULL(n) (1ULL << (n)) +#define GENMASK_ULL(h, l) ((~0ULL >> (63 - (h))) & (~0ULL << (l))) +#define FIELD_GET(m, v) ((__typeof__(m))(((v) & (m)) >> __builtin_ctzll(m))) +#define FIELD_PREP(m, v) (((__typeof__(m))(v) << __builtin_ctzll(m)) & (m)) +#define check_add_overflow(a, b, d) __builtin_add_overflow(a, b, d) +#define is_power_of_2(n) ((n) != 0 && (((n) & ((n) - 1)) == 0)) +#define U16_MAX UINT16_MAX #define __packed __attribute__((packed)) #undef static_assert #define static_assert(condition, ...) _Static_assert(condition, "wire layout") @@ -1141,10 +1160,15 @@ def function(source, marker): struct device_node { bool available, external, phys; unsigned int index; u32 firmware[3], compat[3]; int firmware_cells, compat_cells; }; struct device { struct device_node *of_node; void *data; }; struct platform_device { struct device dev; bool bound, registered; }; +/* The native T6030 14.7 backend is never selected in these scenarios. */ +enum dcp_firmware_version { DCP_FIRMWARE_UNKNOWN, DCP_FIRMWARE_V_14_7, DCP_FIRMWARE_V_14_7_TEMPLATE }; struct apple_dcp { bool external_prepared, quiescing, crashed; + enum dcp_firmware_version fw_compat; struct { int iomfb_method_profile; } hw; }; +static void __attribute__((unused)) iomfb_v14_7_remove(struct apple_dcp *d) +{ (void)d; assert(!"14.7 backend on a non-14.7 DCP"); } struct drm_device { int unused; }; struct drm_encoder { int unused; }; struct apple_drm_private { struct drm_device drm; }; @@ -1373,8 +1397,11 @@ def vrr(root): #define READ_ONCE(x) (x) #define dev_dbg(...) ((void)0) struct apple_connector { bool connected; }; +/* The native T6030 14.7 backend is never selected in these scenarios. */ +enum dcp_firmware_version { DCP_FIRMWARE_UNKNOWN, DCP_FIRMWARE_V_14_7, DCP_FIRMWARE_V_14_7_TEMPLATE }; struct apple_dcp { bool crashed; + enum dcp_firmware_version fw_compat; struct { int iomfb_method_profile; } hw; struct { bool valid; } mode_state; struct apple_connector *connector; @@ -1392,6 +1419,9 @@ def vrr(root): { (void)c; return &s->crtc; } static bool drm_atomic_crtc_needs_modeset(struct drm_crtc_state *s) { return s->modeset; } +static int __attribute__((unused)) iomfb_v14_7_atomic_check(struct apple_dcp *d, + struct drm_crtc *c, struct drm_atomic_commit *s) +{ (void)d; (void)c; (void)s; assert(!"14.7 backend on a non-14.7 DCP"); return -EINVAL; } """ suffix = r""" int main(void) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py index 5d2fc1c32d458..700bf2266ace1 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only # Copyright 2026 Ryan Murray """Actual mux prepare/set and TPS removal ordering; firmware/PHY leaves are mocks.""" -from common import block, atc_function +from common import block, atc_function, fabric_core, fabric_rebalance def make_unit(source, parent=False, reference=None): reference=source if reference is None else reference @@ -115,7 +115,7 @@ def make_unit(source, parent=False, reference=None): policy=block(mux,'bool typec_mux_safe_after_role(')+'\n' if 'bool typec_mux_safe_after_role(' in mux else '' functions=policy+'\n'.join(block(mux, marker) for marker in ('static void typec_mux_unlock(', 'void typec_mux_abort(', 'int typec_mux_begin(', 'int typec_mux_apply(', 'int typec_mux_end(', 'int typec_mux_prepare(')) if not parent else block((reference/'drivers/usb/typec/mux.c').read_text(),'int typec_mux_prepare(')+'\n' functions+=block(mux,'int typec_mux_set(')+'\n'+block(dcp,'static int dcp_typec_route_prepare(')+'\n'+block(dcp,'static int dcp_typec_route_finish(')+'\n'+block(dcp,'static void dcp_typec_route_commit(')+'\n' - functions+=block(dcp,'static struct apple_dcp_typec_route *\ndcp_typec_rebalance_locked(')+'\n' + functions+=fabric_rebalance(dcp) if 'static bool tps6598x_safe_after_role(' in core: functions+=block(core,'static bool tps6598x_safe_after_role(')+'\n' if not parent:functions+=block(core,'static void tps6598x_drop_partner(')+'\n'+block(core,'static int tps6598x_set_data_role(')+'\n' @@ -279,4 +279,19 @@ def make_unit(source, parent=False, reference=None): prefix=prefix.replace('struct tipd_data {','struct tipd_data { bool role_switch_controls_phy;') controls=controls.replace('data=(struct tipd_data){','data=(struct tipd_data){.role_switch_controls_phy=true,') controls=controls.replace('c.tps.role_sw=&c;\n assert(tps6598x_disconnect', 'c.tps.role_sw=&c;data.role_switch_controls_phy=false;\n assert(tps6598x_disconnect') + fabric=fabric_core(reference) + if fabric: + # The fabric core split: rebalancing runs the real dcp_fabric_run_rebalance() + # and dcp_fabric_run_deactivate() over the real decisions, and the adapter + # hands the core each route's struct dcp_fabric_route. + prefix=prefix.replace('typedef unsigned u32;\n','typedef unsigned u32;\ntypedef unsigned long long u64;\n'+fabric,1) + prefix=prefix.replace('struct apple_dcp_typec_route { ','struct apple_dcp_typec_route { struct dcp_fabric_route core; ',1) + prefix=prefix.replace('#define lockdep_assert_held(p) ((void)(p))\n','''#define lockdep_assert_held(p) ((void)(p)) +#define list_first_entry_or_null(h,t,m) ((void)(h),sole_port()) +#define list_is_last(p,h) ((void)(p),(void)(h),true) +#define list_next_entry(p,m) ((void)(p),(struct apple_dcp_typec_port *)NULL) +''',1) + # The real plan snapshots each route into its core before deciding. + prefix=prefix.replace('static void dcp_typec_plan(struct drm_device *d,struct apple_dcp_typec_port *p,unsigned n) { (void)d;(void)p;(void)n; }', + 'static void dcp_typec_plan(struct drm_device *d,struct apple_dcp_typec_port *p,unsigned n) { (void)d;(void)p;(void)n;route.core.tunnel=route.tunnel; }',1) return prefix+functions+ordinary_phy+controls diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py index 19a34888d593e..f2cfdd6886870 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py @@ -924,9 +924,12 @@ def hotplug(root): if(o->dp.set_rate)hardware_rate=o->dp.link_rate; return 0; } -static void dcp_tunnel_set_rate(struct apple_dcp *d,void *phy,unsigned rate) { - (void)d;(void)phy;(void)rate;assert(false); +static int dcp_tunnel_set_rate(struct apple_dcp *d,void *phy,unsigned rate) { + (void)d;(void)phy;(void)rate;assert(false);return 0; } +/* The T6030 14.7 backend never matches this H17P display. */ +#define DCP_FIRMWARE_V_14_7 (DCP_FIRMWARE_H17P + 1) +#define DCP_FIRMWARE_V_14_7_TEMPLATE (DCP_FIRMWARE_H17P + 2) ''' real_handlers += block(training, 'static int dptx_j700_validate_lanes(') + '\n' real_handlers += block(dptx, 'static int dptxport_call_set_active_lane_count(') + '\n' diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py b/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py index 6c1ef39f86bfc..cc4f72dfb2cba 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py @@ -26,8 +26,12 @@ def extract(text,signature): 'assert(afk_service_get(old)==old);old->cmd_tag=1;old->cmd_map[0]=1;') corrected=corrected.replace(' old->cmd_tag=1;\n','') - sender=extract((root/'drivers/gpu/drm/apple/afk.c').read_text(), - 'static int afk_send_command_timeout_len(') + afk=(root/'drivers/gpu/drm/apple/afk.c').read_text() + sender=extract(afk,'static int afk_send_command_timeout_len(') + # The legacy reply-length variant moved the sender body into + # afk_send_command_common(); afk_send_command_timeout_len() wraps it. + if 'static int afk_send_command_common(' in afk: + sender=extract(afk,'static int afk_send_command_common(')+'\n'+sender unit=unit[:unit.index('int main(void)')] unit=unit.replace('static unsigned int dma_frees,opens;',r''' static unsigned int dma_frees,opens; @@ -239,6 +243,21 @@ def extract(text,signature): ''' unit=unit.replace('bool hpd_mutex,quiescing,typec_cable_connected;', 'bool external_physically_absent,hpd_mutex,quiescing,typec_cable_connected;') + if 'afk_reply_size_valid(' in sender: + # The legacy reply-length path checks its size with the real helper; the + # T6030 dcpext (dcp->external) timeout diagnostic never runs for H17P. + unit=unit.replace('struct apple_dcp_afkep { u8 *command_tags;', + 'struct apple_dcp_afkep { struct { bool ready;u32 *rptr,*wptr; } txbfr,rxbfr;u8 *command_tags;',1) + unit=unit.replace('bool external_physically_absent,hpd_mutex,quiescing,typec_cable_connected;', + 'bool external_physically_absent,hpd_mutex,quiescing,typec_cable_connected;bool external;void *rtk;',1) + sender='''#include "afk_reply.h" + #define U32_MAX UINT32_MAX + #ifndef EOVERFLOW + #define EOVERFLOW 75 + #endif + static bool apple_rtkit_is_running(void *r) { (void)r;assert(!"T6030 dcpext diagnostic on H17P");return false; } + static bool apple_rtkit_is_crashed(void *r) { (void)r;assert(!"T6030 dcpext diagnostic on H17P");return false; } + '''+sender unit+=sender+'\n'+tests out.mkdir(parents=True,exist_ok=True) @@ -294,6 +313,9 @@ def run(name,text,cases): #define complete_all(p) complete(p) ''') afh=(root/'drivers/gpu/drm/apple/afk.h').read_text() + # The actual RX loop uses the real ring buffers instead of the diagnostic mock. + queued=queued.replace('struct apple_dcp_afkep { struct { bool ready;u32 *rptr,*wptr; } txbfr,rxbfr;u8 *command_tags;', + 'struct apple_dcp_afkep { u8 *command_tags;',1) queued=queued.replace('struct apple_dcp_afkep { u8 *command_tags;bool lock,stopping;', 'typedef u32 __le32;\n'+extract(afh,'struct afk_ringbuffer {')+';\n' 'struct apple_dcp_afkep { u8 *command_tags;int started,stopped;struct afk_ringbuffer txbfr,rxbfr;void *debugfs_entry;bool lock,stopping;') diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py index fc75d771f61bc..767e0a1b20ff3 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py @@ -76,7 +76,20 @@ def check(name, unit, cases=((),), negative=False, flags=()): ''' pos=actual.index('static int dcp_typec_route_deactivate(') actual=actual[:pos]+extra+actual[pos:] - actual=actual.replace(block(actual,'static bool dcp_typec_route_available('), 'static bool dcp_typec_route_fixed_output_busy(struct apple_dcp_typec_route *r) { (void)r;return false; }\n'+block(dcp,'static bool dcp_typec_route_available('),1) + # The fabric core split: availability is the real dcp_fabric_available() over + # the real ownership (SAFE lease included) and HDMI presence of the pipeline. + available=''.join(block(dcp,marker)+'\n' for marker in ( + 'static bool dcp_typec_safe_held(struct apple_dcp *dcp)\n{', 'bool dcp_typec_dual_stream(void)\n{', + 'static bool dcp_hdmi_settle_enabled(struct apple_dcp *dcp)\n{', + 'static enum dcp_fabric_presence_state dcp_hdmi_presence(struct apple_dcp *dcp)\n{') if marker in dcp) + if available: + actual=actual.replace('struct apple_dcp { int external_lock,external_error;','struct apple_dcp { int external_lock,external_error;\n struct dcp_fabric_presence hdmi_presence; int hdmi_presence_lock; unsigned nr_typec_routes;',1) + available='''#define atomic_read(p) (*(p)) + #define spin_lock_irqsave(l,f) ((void)(l),(f)=0) + #define spin_unlock_irqrestore(l,f) ((void)(l),(void)(f)) + static int dcp_dual_stream_routes; + '''+available + actual=actual.replace(block(actual,'static bool dcp_typec_route_available('), 'static bool dcp_typec_route_fixed_output_busy(struct apple_dcp_typec_route *r) { (void)r;return false; }\n'+available+block(dcp,'static bool dcp_typec_route_available('),1) actual=actual.replace(block(actual,'static int dcp_typec_route_deactivate('),block(dcp,dcp.rindex('static int dcp_typec_route_deactivate(')),1) pos=actual.index('static struct typec_mux_dev phy_provider') actual=actual[:pos]+block(dcp,'static bool dcp_typec_route_is_dp(')+'\n'+block(dcp,'static int dcp_typec_route_set(')+'\n'+actual[pos:] From b02ca1e15293a70cec90a370507017959f617730 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 7 Oct 2026 19:29:20 -0700 Subject: [PATCH 1641/2680] drm: apple: add the J613 v26 display backend and guarded coldplug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the v26 IOMFB backend alongside v14. Require the exact J613 firmware identity, all three mapping handoffs, the same-boot ADT clock, active inherited power domains, and this Mac's validated PMP image and tunables before activating the disabled display providers. Order the DISP vote after the PMP supplier and preserve the running DCP session and buffers. Use the existing mode catalog locks, generation and validity helpers. A refused handoff keeps the framebuffer available, and registration failures unwind previously registered display services. [ryan: adapted to 7.2.10 — the pmgr-pwrstate.c retained_25g83 hunk is left to the pmdomain owner (handed over to 010-soc); DCP_FIRMWARE_V_26_6 is added to version_utils.h, where this tree keeps the enum; the v26 files use the 7.2 drm_atomic_commit API and enumerate_modes(..., external_sdr=false); the PMP device-power API of linux/soc/apple/pmp.h (010-soc) is declared locally in iomfb_v26_6.h and called through symbol_get() with an -ENODEV fallback; the two PMP table checks of drivers/pmdomain/apple/pmp-report-validation.h are copied into j613-25g83.h; the bw-scratch probe check keeps hw->firmware_clock; vblank/swap-watchdog stay disabled for V_26_6 beside this tree's external_native pipes (no typec_fabric_retrain_wq here); this carries the shared registration error propagation and unwind of ed85a0eb8e] Co-authored-by: Eryk Wieliczko Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 4804ca4d9f3cb7ae4539054a614ecddd732e06a9) --- drivers/gpu/drm/apple/Kconfig | 1 + drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/apple_drv.c | 26 +- drivers/gpu/drm/apple/audio.c | 4 +- drivers/gpu/drm/apple/dcp-internal.h | 2 + drivers/gpu/drm/apple/dcp.c | 65 +- drivers/gpu/drm/apple/dcp.h | 7 +- drivers/gpu/drm/apple/iomfb.c | 7 + drivers/gpu/drm/apple/iomfb_v26_6.c | 1907 +++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_v26_6.h | 55 + drivers/gpu/drm/apple/iomfb_v26_6_link.c | 502 ++++++ drivers/gpu/drm/apple/iomfb_v26_6_link.h | 100 ++ drivers/gpu/drm/apple/iomfb_v26_6_swap.h | 175 ++ drivers/gpu/drm/apple/j613-25g83.h | 99 ++ drivers/gpu/drm/apple/j613_dcp_activate.c | 284 +++ drivers/gpu/drm/apple/version_utils.h | 2 + 16 files changed, 3222 insertions(+), 15 deletions(-) create mode 100644 drivers/gpu/drm/apple/iomfb_v26_6.c create mode 100644 drivers/gpu/drm/apple/iomfb_v26_6.h create mode 100644 drivers/gpu/drm/apple/iomfb_v26_6_link.c create mode 100644 drivers/gpu/drm/apple/iomfb_v26_6_link.h create mode 100644 drivers/gpu/drm/apple/iomfb_v26_6_swap.h create mode 100644 drivers/gpu/drm/apple/j613-25g83.h create mode 100644 drivers/gpu/drm/apple/j613_dcp_activate.c diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index efad685cb042c..0eadf0e602250 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -6,6 +6,7 @@ config DRM_APPLE depends on APPLE_RTKIT depends on OF_ADDRESS select APPLE_PMP_REPORT + select OF_DYNAMIC select DRM_CLIENT_SELECTION select DRM_KMS_HELPER select DRM_KMS_DMA_HELPER diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index a9eaf1706a412..a038caae1e79f 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -24,6 +24,7 @@ appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_ready_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_announce_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_relink_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_dropped_swap_test.o +appledrm-y += iomfb_v26_6.o iomfb_v26_6_link.o j613_dcp_activate.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 87a8af5380330..c14039416e717 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -1142,16 +1142,32 @@ static struct platform_driver apple_platform_driver = { static int __init appledrm_register(void) { + int ret; if (drm_firmware_drivers_only()) return -ENODEV; + /* A refused experimental profile keeps the boot framebuffer available. */ + ret = apple_j613_25g83_coldplug(); + if (ret) + pr_info("J613/25G83 display handoff refused: %d; keeping boot framebuffer\n", ret); + #if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) - dcp_audio_register(); + ret = dcp_audio_register(); + if (ret) + return ret; #endif - dcp_register(); - platform_driver_register(&apple_platform_driver); - - return 0; + ret = dcp_register(); + if (ret) + goto unregister_audio; + ret = platform_driver_register(&apple_platform_driver); + if (!ret) + return 0; + dcp_unregister(); +unregister_audio: +#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) + dcp_audio_unregister(); +#endif + return ret; } static void __exit appledrm_unregister(void) diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index ac55f1f30f6ac..9917d660ded55 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -804,9 +804,9 @@ static struct platform_driver dcpaud_driver = { .shutdown = dcpaud_shutdown, }; -void __init dcp_audio_register(void) +int __init dcp_audio_register(void) { - platform_driver_register(&dcpaud_driver); + return platform_driver_register(&dcpaud_driver); } void __exit dcp_audio_unregister(void) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index b8a85e5a6599b..f6ac5168ecd71 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -30,6 +30,7 @@ #include "iomfb_v13_3.h" #include "iomfb_v14_7.h" #include "iomfb_v14_7_tpl.h" +#include "iomfb_v26_6.h" #include "epic/dpavservep.h" #include "version_utils.h" @@ -287,6 +288,7 @@ struct apple_dcp { /* DCP_FIRMWARE_V_14_7 state; outlives this device once RTKit runs. */ struct apple_dcp_v14 *v14; + struct apple_dcp_v26 *v26; /* Coprocessor control register */ void __iomem *coproc_reg; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 02380079ea687..d4b6f8cd2c643 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -898,6 +898,8 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commi struct drm_crtc_state *crtc_state; bool needs_modeset; + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6) + return iomfb_v26_6_atomic_check(dcp, crtc, commit); if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) return iomfb_v14_7_atomic_check(dcp, crtc, commit); @@ -1948,7 +1950,8 @@ bool dcp_fw_compat_is_14_7(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); - return dcp->fw_compat == DCP_FIRMWARE_V_14_7; + return dcp->fw_compat == DCP_FIRMWARE_V_14_7 || + dcp->fw_compat == DCP_FIRMWARE_V_26_6; } unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev) @@ -1965,6 +1968,8 @@ int dcp_start(struct platform_device *pdev) /* Even if nothing starts, dcp_wait_ready() must find it initialised. */ init_completion(&dcp->start_done); + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6) + return iomfb_v26_6_start(dcp); /* * The T6030 firmware session is adopted for the internal panel. @@ -2079,6 +2084,9 @@ static void _dcp_poweroff(struct apple_dcp *dcp) case DCP_FIRMWARE_V_14_7_TEMPLATE: iomfb_poweroff_v14_7_0(dcp); break; + case DCP_FIRMWARE_V_26_6: + iomfb_v26_6_poweroff(dcp); + break; case DCP_FIRMWARE_V_14_7: iomfb_v14_7_poweroff(dcp); break; @@ -2167,6 +2175,9 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) case DCP_FIRMWARE_V_14_7_TEMPLATE: iomfb_sleep_v14_7_0(dcp); break; + case DCP_FIRMWARE_V_26_6: + iomfb_v26_6_poweroff(dcp); + break; case DCP_FIRMWARE_V_14_7: iomfb_v14_7_poweroff(dcp); break; @@ -2298,6 +2309,9 @@ void dcp_poweron(struct platform_device *pdev) case DCP_FIRMWARE_V_14_7_TEMPLATE: iomfb_poweron_v14_7_0(dcp); break; + case DCP_FIRMWARE_V_26_6: + iomfb_v26_6_poweron(dcp); + break; case DCP_FIRMWARE_V_14_7: iomfb_v14_7_poweron(dcp); break; @@ -2856,6 +2870,21 @@ static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) return DCP_FIRMWARE_UNKNOWN; } + /* J613 25G83 has a distinct callback table, never a v14 fallback. */ + if (of_device_is_compatible(dev->of_node, "apple,t8122-dcp") && + of_property_present(dev->of_node, "apple,j613-25g83-profile")) { + const char *uuid; + u32 profile; + + if (of_machine_is_compatible("apple,j613") && + !of_property_read_u32(dev->of_node, "apple,j613-25g83-profile", &profile) && + profile == 1 && !strcmp(compat_str, "26.6.2") && + !of_property_read_string(dev->of_node, "apple,firmware-uuid", &uuid) && + !strcmp(uuid, "C042E95C-B9D8-3F0E-94B3-582A08AA6FDD")) + return DCP_FIRMWARE_V_26_6; + return DCP_FIRMWARE_UNKNOWN; + } + /* * T6030 and the internal panels with a board record (T6030, J613) run * the 14.7 firmware session handed over by the boot loader natively @@ -3095,6 +3124,13 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) dcp->connector_type = DRM_MODE_CONNECTOR_eDP; } + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6) { + ret = iomfb_v26_6_bind(dcp); + if (!ret) + enable_work(&dcp->dimensions_wq); + return ret; + } + /* The running T6030 firmware is adopted as is. */ if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { if (dcp->external_native) { @@ -3234,6 +3270,10 @@ VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, v return; disable_work_sync(&dcp->dimensions_wq); + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6) { + iomfb_v26_6_unbind(dcp); + return; + } if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { if (dcp->external_native) { /* Bind's wait_ready enabled it, as for any HDMI port. */ @@ -3463,6 +3503,7 @@ static int dcp_platform_probe(struct platform_device *pdev) * display. */ if (fw_compat != DCP_FIRMWARE_V_14_7 && + fw_compat != DCP_FIRMWARE_V_26_6 && !of_property_present(dev->of_node, "apple,bw-scratch") && !hw->firmware_clock) return dev_err_probe(dev, -ENODEV, "Incompatible devicetree! " @@ -3547,14 +3588,22 @@ static int dcp_platform_probe(struct platform_device *pdev) disable_delayed_work(&dcp->placeholder_edid_wq); disable_delayed_work(&dcp->announce_wq); dcp->announce.stopped = true; - if (dcp->external_native) { - /* Bind enables these; firmware callbacks may come before it. */ + /* + * A native external pipe's bind enables these; firmware callbacks may + * come before it. The 26.6 panel never uses them. + */ + if (dcp->external_native || fw_compat == DCP_FIRMWARE_V_26_6) { disable_work(&dcp->vblank_wq); disable_delayed_work(&dcp->swap_watchdog_wq); } platform_set_drvdata(pdev, dcp); + if (fw_compat == DCP_FIRMWARE_V_26_6) { + ret = iomfb_v26_6_probe(dcp); + if (ret) + return ret; + } if (fw_compat == DCP_FIRMWARE_V_14_7 && !dcp->external) { ret = iomfb_v14_7_probe(dcp); if (ret) @@ -3583,7 +3632,7 @@ static int dcp_platform_probe(struct platform_device *pdev) "apple,iomfb-surfaces", sizeof(u32)); - if (fw_compat == DCP_FIRMWARE_V_14_7) { + if (fw_compat == DCP_FIRMWARE_V_14_7 || fw_compat == DCP_FIRMWARE_V_26_6) { set_bit(0, dcp->iomfb_surfaces); } else if (num_surfs == 0 || num_surfs == -ENODATA) { set_bit(0, dcp->iomfb_surfaces); @@ -3785,6 +3834,8 @@ static int dcp_platform_probe(struct platform_device *pdev) if (ret && dcp->external_prepared) { dcp_disable_typec_work(dcp, true); } + if (ret && dcp->fw_compat == DCP_FIRMWARE_V_26_6) + iomfb_v26_6_unbind(dcp); /* A failed bind run from here may already have started RTKit. */ if (ret && dcp->fw_compat == DCP_FIRMWARE_V_14_7) iomfb_v14_7_remove(dcp); @@ -3802,6 +3853,8 @@ static void dcp_platform_remove(struct platform_device *pdev) return; } component_del(&pdev->dev, &dcp_comp_ops); + if (dcp && dcp->fw_compat == DCP_FIRMWARE_V_26_6) + iomfb_v26_6_unbind(dcp); /* Unbind does not wait for RTKit callbacks, or run if bind failed late. */ if (dcp && dcp->fw_compat == DCP_FIRMWARE_V_14_7) iomfb_v14_7_remove(dcp); @@ -4190,9 +4243,9 @@ static struct platform_driver apple_platform_driver = { }, }; -void __init dcp_register(void) +int __init dcp_register(void) { - platform_driver_register(&apple_platform_driver); + return platform_driver_register(&apple_platform_driver); } void __exit dcp_unregister(void) diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 97ae0dff26ee7..9380cd5e0f638 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -104,10 +104,13 @@ int dpavservep_init(struct apple_dcp *dcp); int avep_init(struct apple_dcp *dcp); -void __init dcp_register(void); +int __init dcp_register(void); void __exit dcp_unregister(void); -void __init dcp_audio_register(void); +int __init dcp_audio_register(void); void __exit dcp_audio_unregister(void); +int apple_j613_25g83_coldplug(void); +unsigned int apple_j613_25g83_clock_hz(void); + #endif diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index cdee006c077cb..51e26bd23bea0 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -2070,6 +2070,9 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, case DCP_FIRMWARE_V_14_7_TEMPLATE: ret = iomfb_modeset_v14_7_0(dcp, crtc_state); break; + case DCP_FIRMWARE_V_26_6: + ret = iomfb_v26_6_modeset(dcp, crtc_state); + break; case DCP_FIRMWARE_V_14_7: ret = iomfb_v14_7_modeset(dcp, crtc_state); break; @@ -2538,6 +2541,10 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit) struct platform_device *pdev = to_apple_crtc(crtc)->dcp; struct apple_dcp *dcp = platform_get_drvdata(pdev); + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6) { + iomfb_v26_6_flush(dcp, crtc, commit); + return; + } if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { iomfb_v14_7_flush(dcp, crtc, commit); return; diff --git a/drivers/gpu/drm/apple/iomfb_v26_6.c b/drivers/gpu/drm/apple/iomfb_v26_6.c new file mode 100644 index 0000000000000..b922c74817dec --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v26_6.c @@ -0,0 +1,1907 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * J613 (M3 Air 13 inch) experimental internal display: macOS 26.6.2 (25G83) IOMFB over DCPLink + * + * Copyright The Asahi Linux Contributors + * Based on the M3 Pro DCP lab client by Eryk Wieliczko. + * + * iBoot leaves the DCP running, scanning out the boot framebuffer. This + * variant never boots, stops or powers the coprocessor down. In order: + * + * probe the firmware must be the board's qualified image, the boot loader + * must have handed the display over (apple,j613-25g83-mapping-handoff, or the + * board's own marker), and the PMP must be running and have + * acknowledged the display request; + * bind the DCP CPU must still be running; its RTKit session is woken; + * start DCPLink is brought up, then the start signal (A401) and the first + * client open (A457) run with every callback answered in the + * kernel; the panel mode is taken from PreferredTimingElements; + * KMS only after all of that does drm/apple remove the boot framebuffer + * driver and register the DRM device. + * + * Any failure before KMS leaves the boot framebuffer in place. Everything the + * firmware has seen (the RTKit session, the RPC memory, the buffers it + * allocated, the scanout buffers of failed swaps) is kept until reboot, and + * the module cannot be unloaded once RTKit is up. + * + * Scanout is one plane, XRGB8888 or ARGB8888, linear, exactly the mode size. + * Each flush is one swap, completed when the firmware reports that swap id, + * and then the page-flip event is sent. The notch rows (apple,notch-height, + * unless appledrm.t6030_show_notch=1) are outside the mode and stay black. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dcp.h" +#include "dcp-internal.h" +#include "dcp-lifecycle.h" +#include "iomfb_internal.h" +#include "iomfb_v26_6.h" +#include "j613-25g83.h" +#include "iomfb_v26_6_link.h" +#include "iomfb_v26_6_swap.h" +#include "parser.h" + +#define A(n) DCP_V26_TAG('A', n) +#define D(n) DCP_V26_TAG('D', n) + +/* Exact image from the same-boot proxy observation. No 14.x ABI alias. */ +#define DCP_V26_J613_FIRMWARE_UUID "C042E95C-B9D8-3F0E-94B3-582A08AA6FDD" + +#define DCP_V26_CPU_CONTROL 0x44 +#define DCP_V26_CPU_STATUS 0x48 +#define DCP_V26_MAX_PROPERTIES 256 +#define DCP_V26_MAX_RAW 32 +#define DCP_V26_MAX_RAW_BYTES SZ_8M +#define DCP_V26_MAX_BUFFERS 64 +#define DCP_V26_MAX_BUFFER SZ_16M +#define DCP_V26_MAX_BUFFER_BYTES SZ_64M +#define DCP_V26_MAX_PUBLISHED 128 +/* IOMFB layer of the primary plane. */ +#define DCP_V26_LAYER 0 +#define DCP_V26_BLACK 0xff000000 +/* Swaps kept for the present history, and the swap-complete payload size. */ +#define DCP_V26_HISTORY 256 +#define DCP_V26_D590_SIZE 0x730 + +/* SDR user maximum of the J613/25G83 laptop panels; the DT gives the panel's own. */ +/* Reported until the first request; the firmware keeps its own until then. */ +#define DCP_V26_DEFAULT_NITS 140 +/* How long a brightness request waits for a flip to carry it. */ +#define DCP_V26_BL_FLIP_WAIT msecs_to_jiffies(25) + +/* Fail closed outside the single qualified board and firmware identity. */ +struct dcp_v26_board { + /* In log messages. */ + const char *name; + const char *dcp_compatible; + /* The machine compatible; NULL for any machine with this DCP. */ + const char *machine; + /* The firmware image whose layouts were checked for this board. */ + const char *firmware_uuid; + /* Set to 1 by the boot loader on the DCP, display and PIODMA nodes. */ + const char *handoff; + /* Backlight maximum in nits; the DT maximum is capped to it. */ + u32 max_nits; + /* Native panel size, notch rows included; 0: any, from the boot framebuffer. */ + u32 panel_width, panel_height; + /* The panel may have a 120 Hz variable-refresh (ProMotion) timing. */ + bool promotion; + /* If set and true, start without apple,pmp-report (not measured). */ + const bool *without_pmp; +}; + +static const struct dcp_v26_board dcp_v26_board_j613 = { + .name = "J613/25G83", + .dcp_compatible = "apple,t8122-dcp", + .machine = "apple,j613", + .firmware_uuid = DCP_V26_J613_FIRMWARE_UUID, + .handoff = "apple,j613-25g83-mapping-handoff", + .max_nits = 525, + .panel_width = 2560, + .panel_height = 1664, +}; +static const struct dcp_v26_board *const dcp_v26_boards[] = { + &dcp_v26_board_j613, +}; + +struct dcp_v26_property { + u32 service; + char key[64]; + u64 value; +}; + +struct dcp_v26_raw { + u32 service; + char key[64]; + void *data; + u32 size; +}; + +/* One swap's timeline, CLOCK_MONOTONIC; 0 where it did not get that far. */ +struct dcp_v26_present_record { + ktime_t start; /* swap start sent */ + ktime_t submitted; /* swap start answered, swap sent */ + ktime_t replied; /* swap reply received */ + ktime_t completed; /* swap-complete message received */ + u32 id; + u8 surface; + u8 timed; +}; + +/* The last publication of a runtime property: its id, then 12 bytes. */ +struct dcp_v26_published { + u32 id; + u8 data[12]; + u64 count; +}; + +struct dcp_v26_buffer { + void *cpu; + dma_addr_t iova; + phys_addr_t phys; + size_t size; + bool retired; + bool piodma_mapped; +}; + +struct apple_dcp_v26 { + struct device *dev; + const struct dcp_v26_board *board; + /* Cleared when KMS unbinds; the firmware session outlives it. */ + struct apple_dcp *dcp; + struct apple_rtkit *rtk; + struct dcp_v26_link link; + + /* Owns the RPC stream: start, swaps and idle callbacks. */ + struct mutex lock; + bool ctm_valid; + u64 ctm_calls; + u32 ctm_status, ctm_get_status; + u64 ctm_readback[9]; + struct work_struct idle_work; + bool failed; + bool started; + + /* Boot framebuffer and native panel timing (notch rows included). */ + u32 stride; + u32 fb_width, fb_height; + u32 panel_width, panel_height; + /* The panel's notch rows (apple,notch-height), hidden or not. */ + u32 notch_rows; + u64 clock_rate; + /* The mode is a variable-refresh (ProMotion) timing. */ + bool vrr; + /* + * Set by the modeset: the CRTC shows the 120 Hz ProMotion mode, whose + * swaps carry timing words, and its refresh (for debugfs). + */ + bool timed; + u32 refresh; + + struct dcp_v26_property properties[DCP_V26_MAX_PROPERTIES]; + u32 property_count; + struct dcp_v26_raw raw[DCP_V26_MAX_RAW]; + u32 raw_count, raw_bytes; + void *chunk; + u32 chunk_size, chunk_offset; + u64 analytics; + u8 analytics_last[4164]; + struct dcp_v26_published published[DCP_V26_MAX_PUBLISHED]; + u32 published_count; + u64 publications; + + struct dcp_v26_buffer buffers[DCP_V26_MAX_BUFFERS]; + u32 buffer_count; + u64 buffer_bytes; + struct platform_device *piodma; + struct iommu_domain *piodma_domain; + + /* Scanned out until the next swap completes. */ + struct drm_framebuffer *active_fb; + u64 swaps; + u64 swap_ns_max; + + /* The last swaps and the last two swap-complete payloads, for debugfs. */ + struct dcp_v26_present_record history[DCP_V26_HISTORY]; + u64 history_count; + u8 d590[2][DCP_V26_D590_SIZE]; + ktime_t d590_time[2]; + u64 d590_count; + + /* Panel brightness, carried by swaps. Under the lock. */ + u32 bl_nits; /* requested */ + u32 bl_sent; /* carried by the last brightness swap */ + u64 bl_seq; /* bumped by every change of the target */ + u64 bl_swaps; + bool bl_owned; /* requested at least once: the driver sets it */ + bool bl_off; /* the CRTC is off: the target is 0 */ + bool bl_pending; /* the target has not been carried yet */ + bool bl_queued; /* the backlight device is being registered */ + wait_queue_head_t bl_wait; + + /* The backlight device, registered after the first frame. */ + struct mutex bl_lock; + struct work_struct bl_register_work; + struct backlight_device *bl_dev; + bool bl_detached; + u32 bl_max; +}; + +/* One DCP session per boot: RTKit is never started twice. */ +static bool dcp_v26_session; + +static bool j613_25g83_swap_timestamps = true; +module_param(j613_25g83_swap_timestamps, bool, 0644); +MODULE_PARM_DESC(j613_25g83_swap_timestamps, + "J613/25G83: time-stamp swaps on a 120 Hz variable-refresh panel so they are shown at 120 Hz, not 60 Hz (default: on)"); + +static unsigned int j613_25g83_refresh = 120; +module_param(j613_25g83_refresh, uint, 0444); +MODULE_PARM_DESC(j613_25g83_refresh, + "J613/25G83: preferred refresh of the built-in display: 120 (variable refresh, ProMotion) or 60 (locked); both modes stay available (default: 120)"); + +static bool j613_25g83_flip_timestamp = true; +module_param(j613_25g83_flip_timestamp, bool, 0644); +MODULE_PARM_DESC(j613_25g83_flip_timestamp, + "J613/25G83: stamp page-flip events with the arrival of the firmware's swap-complete message; off: when the flush ends, less 1 ms (default: on)"); + +static int dcp_v26_callback(void *cookie, u32 tag, const void *input, u32 in_size, + void *output, u32 out_size); +static void dcp_v26_backlight_register(struct work_struct *work); +static void dcp_v26_backlight_detach(struct apple_dcp_v26 *v26); + +/* Called with the lock held, or from a callback. */ +static int dcp_v26_call(struct apple_dcp_v26 *v26, u32 tag, const void *in, u32 in_size, + void *out, u32 out_size, u32 completion) +{ + int ret; + + if (v26->failed) + return -EIO; + dev_dbg(v26->dev, "call %#x in %u out %u\n", tag, in_size, out_size); + ret = dcp_v26_link_call(&v26->link, tag, in, in_size, out, out_size, completion, + dcp_v26_callback, v26); + if (ret) { + v26->failed = true; + dev_err(v26->dev, "DCP call %#x failed: %d; recovery requires a reboot\n", + tag, ret); + } + return ret; +} + +/* A call with an optional u32 input of 1 and an optional u32 result to check. */ +static int dcp_v26_simple_call(struct apple_dcp_v26 *v26, u32 tag, bool input, bool reply, + bool expected) +{ + __le32 one = cpu_to_le32(1), result = 0; + int ret; + + ret = dcp_v26_call(v26, tag, input ? &one : NULL, input ? 4 : 0, + reply ? &result : NULL, reply ? 4 : 0, 0); + if (!ret && reply && le32_to_cpu(result) != expected) + return -EPROTO; + return ret; +} + +static struct dcp_v26_property *dcp_v26_property(struct apple_dcp_v26 *v26, u32 service, + const u8 *key, bool create) +{ + struct dcp_v26_property *p; + u32 i; + + if (!memchr(key, 0, 64)) + return ERR_PTR(-EINVAL); + for (i = 0; i < v26->property_count; i++) { + p = &v26->properties[i]; + if (p->service == service && !strcmp(p->key, key)) + return p; + } + if (!create) + return NULL; + if (v26->property_count == DCP_V26_MAX_PROPERTIES) + return ERR_PTR(-ENOSPC); + p = &v26->properties[v26->property_count++]; + p->service = service; + strscpy(p->key, key, sizeof(p->key)); + return p; +} + +static int dcp_v26_raw_property(struct apple_dcp_v26 *v26, u32 service, const u8 *key, + const void *data, u32 size) +{ + u32 i, old_size = 0; + void *copy; + + if (!memchr(key, 0, 64) || size > SZ_1M) + return -EINVAL; + for (i = 0; i < v26->raw_count; i++) + if (v26->raw[i].service == service && !strcmp(v26->raw[i].key, key)) + break; + if (i == DCP_V26_MAX_RAW) + return -ENOSPC; + if (i < v26->raw_count) + old_size = v26->raw[i].size; + if (v26->raw_bytes - old_size + size > DCP_V26_MAX_RAW_BYTES) + return -ENOSPC; + copy = kvmemdup(data, size ?: 1, GFP_KERNEL); + if (!copy) + return -ENOMEM; + if (i == v26->raw_count) + v26->raw_count++; + kvfree(v26->raw[i].data); + v26->raw[i].service = service; + strscpy(v26->raw[i].key, key, sizeof(v26->raw[i].key)); + v26->raw[i].data = copy; + v26->raw[i].size = size; + v26->raw_bytes += size - old_size; + return 0; +} + +/* The firmware's allocate-buffer callback wants a physically contiguous buffer. */ +static int dcp_v26_alloc(struct apple_dcp_v26 *v26, u64 request, u32 *id) +{ + struct dcp_v26_buffer *buf; + struct sg_table table; + dma_addr_t iova; + size_t size; + void *cpu; + int ret; + + if (!request || request > DCP_V26_MAX_BUFFER) + return -EINVAL; + size = ALIGN(request, SZ_16K); + if (v26->buffer_count == DCP_V26_MAX_BUFFERS || + v26->buffer_bytes + size > DCP_V26_MAX_BUFFER_BYTES) + return -ENOSPC; + cpu = dma_alloc_attrs(v26->dev, size, &iova, GFP_KERNEL, DMA_ATTR_FORCE_CONTIGUOUS); + if (!cpu) + return -ENOMEM; + ret = dma_get_sgtable_attrs(v26->dev, &table, cpu, iova, size, + DMA_ATTR_FORCE_CONTIGUOUS); + if (ret) + goto free; + if (table.orig_nents != 1 || table.sgl->length < size) { + sg_free_table(&table); + ret = -ERANGE; + goto free; + } + buf = &v26->buffers[v26->buffer_count]; + buf->phys = sg_phys(table.sgl); + sg_free_table(&table); + memset(cpu, 0, size); + dma_wmb(); + buf->cpu = cpu; + buf->iova = iova; + buf->size = size; + v26->buffer_bytes += size; + /* The firmware takes id 0 as a failure. */ + *id = ++v26->buffer_count; + dev_info(v26->dev, "DCP buffer %u: %zu bytes at %pad\n", *id, size, &iova); + return 0; +free: + dma_free_attrs(v26->dev, size, cpu, iova, DMA_ATTR_FORCE_CONTIGUOUS); + return ret; +} + +/* Maps a firmware buffer at the same address for the PIODMA stream. */ +static int dcp_v26_map_piodma(struct apple_dcp_v26 *v26, u32 id) +{ + struct dcp_v26_buffer *buf = &v26->buffers[id - 1]; + struct device_node *node; + size_t offset; + u32 marker; + int ret; + + if (buf->retired) + return -EINVAL; + if (buf->piodma_mapped) + return 0; + if (!v26->piodma) { + node = of_get_child_by_name(v26->dev->of_node, "piodma"); + if (!node || !of_device_is_available(node) || + of_property_read_u32(node, v26->board->handoff, &marker) || marker != 1) { + of_node_put(node); + return -ENODEV; + } + v26->piodma = of_platform_device_create(node, NULL, v26->dev); + if (!v26->piodma) { + of_node_put(node); + return -ENOMEM; + } + ret = dma_set_mask_and_coherent(&v26->piodma->dev, DMA_BIT_MASK(42)); + if (!ret) + ret = of_dma_configure(&v26->piodma->dev, node, true); + of_node_put(node); + /* Never destroy a device attached to a locked DART stream. */ + if (ret) + return ret; + v26->piodma_domain = iommu_get_domain_for_dev(&v26->piodma->dev); + } + if (IS_ERR_OR_NULL(v26->piodma_domain)) + return -EIO; + ret = iommu_map(v26->piodma_domain, buf->iova, buf->phys, buf->size, + IOMMU_READ | IOMMU_WRITE, GFP_KERNEL); + if (ret) + return ret; + for (offset = 0; offset < buf->size; offset += SZ_16K) + if (iommu_iova_to_phys(v26->piodma_domain, buf->iova + offset) != + buf->phys + offset) + return -EIO; + buf->piodma_mapped = true; + dev_info(v26->dev, "PIODMA mapped DCP buffer %u: %zu bytes at %pad\n", + id, buf->size, &buf->iova); + return 0; +} + +/* + * Keeps the last publication of each runtime property, keyed by its first + * word (the property id). The M1/M2 firmware reports the brightness it + * applies this way; kept here, it can be compared with brightness requests + * on a running system before anything relies on its layout. + */ +static void dcp_v26_publish(struct apple_dcp_v26 *v26, const u8 *in) +{ + u32 id = get_unaligned_le32(in), i; + + v26->publications++; + for (i = 0; i < v26->published_count; i++) + if (v26->published[i].id == id) + break; + if (i == DCP_V26_MAX_PUBLISHED) + return; + if (i == v26->published_count) + v26->published_count++; + v26->published[i].id = id; + memcpy(v26->published[i].data, in + 4, sizeof(v26->published[i].data)); + v26->published[i].count++; + dev_dbg(v26->dev, "published %u: %*ph\n", id, 12, in + 4); +} + +/* Runs on the thread that owns the RPC stream; the lock is held. */ +static int dcp_v26_callback(void *cookie, u32 tag, + const void *input, u32 in_size, void *output, u32 out_size) +{ + struct apple_dcp_v26 *v26 = cookie; + const u8 *in = input; + u8 *out = output; + struct dcp_v26_property *p; + u32 id; + u32 count, offset, service, key_offset; + int ret; + +#define SHAPE(i, o) (in_size == (i) && out_size == (o)) + /* The transport zeroes output, so absent optional metadata stays absent. */ + if (tag == D(114) && SHAPE(4164, 4100)) { + /* J613 CoreAnalyticsSendEvent: name[64], serialized dictionary[4096], + * nullable flag. Retain the latest event locally. The dictionary is + * in/out: firmware parses it even on failure, so a zero-filled reply + * is invalid. Return its own serialized dictionary without changes. + * The final u32 is the method status (zero = accepted). + */ + if (!memchr(in, 0, 64) || in[4160] > 1 || + (!in[4160] && in[64] != 'd')) + return -EINVAL; + memcpy(v26->analytics_last, in, sizeof(v26->analytics_last)); + v26->analytics++; + if (!in[4160]) + memcpy(out, in + 64, 4096); + put_unaligned_le32(0, out + 4096); + dev_info(v26->dev, "J613 analytics event %llu: %.64s (retained locally)\n", + v26->analytics, in); + return 0; + } + if (tag == D(590) && SHAPE(0x730, 0)) { + id = v26->d590_count++ % ARRAY_SIZE(v26->d590); + memcpy(v26->d590[id], in, DCP_V26_D590_SIZE); + v26->d590_time[id] = v26->link.callback_time; + return 0; + } + if (tag == D(588) && SHAPE(0, 0)) + return 0; + if (tag == D(599) && SHAPE(0, 0)) + return dcp_v26_simple_call(v26, A(412), false, true, true); + if ((tag == D(108) || tag == D(109) || tag == D(110) || tag == D(111) || + tag == D(112) || tag == D(113) || tag == D(0) || tag == D(1)) && SHAPE(0, 4)) { + out[0] = 1; + return 0; + } + if (tag == D(575) && SHAPE(88, 76)) { + if (!(in[84] & 1)) + memcpy(out, in + 8, 76); + return 0; + } + if (tag == D(576) && SHAPE(4, 0)) + return 0; + if (tag == D(300) && SHAPE(16, 0)) { + dcp_v26_publish(v26, in); + return 0; + } + if (tag == D(127) && SHAPE(4, 4)) { + count = get_unaligned_le32(in); + if (v26->chunk || !count || count > 0x100001) + return -EINVAL; + v26->chunk_size = count - 1; + v26->chunk_offset = 0; + v26->chunk = kvzalloc(max_t(u32, 1, v26->chunk_size), GFP_KERNEL); + if (!v26->chunk) + return -ENOMEM; + out[0] = 1; + return 0; + } + if (tag == D(128) && SHAPE(0x1008, 4)) { + offset = get_unaligned_le32(in + 0x1000); + count = get_unaligned_le32(in + 0x1004); + if (!v26->chunk || offset != v26->chunk_offset || count > 4096 || + offset > v26->chunk_size || count > v26->chunk_size - offset) + return -EINVAL; + memcpy(v26->chunk + offset, in, count); + v26->chunk_offset += count; + out[0] = 1; + return 0; + } + if (tag == D(129) && SHAPE(64, 4)) { + if (!v26->chunk || v26->chunk_offset != v26->chunk_size || !memchr(in, 0, 64) || + v26->raw_count == DCP_V26_MAX_RAW || v26->raw_bytes + v26->chunk_size > SZ_8M) + return -EINVAL; + ret = dcp_v26_raw_property(v26, 0, in, v26->chunk, v26->chunk_size); + if (ret) + return ret; + kvfree(v26->chunk); + v26->chunk = NULL; + dev_info(v26->dev, "J613 property %.64s: %u bytes\n", in, v26->chunk_size); + out[0] = 1; + return 0; + } + if (tag == D(408) && SHAPE(8, 8)) { + count = get_unaligned_le32(in + 4); + if (memcmp(in, "VORP", 4) || count > 1) + return -EINVAL; + /* Same-boot ADT clock observation, supplied by the guarded runner. */ + put_unaligned_le64(count ? 0 : v26->clock_rate, out); + return 0; + } + if (tag == D(125) && SHAPE(100, 36)) { + count = get_unaligned_le32(in + 64); + if (count > 8) + return -EINVAL; + memcpy(out, in + 68, count * 4); + return 0; + } + if (tag == D(6) && SHAPE(84, 80)) { + if (!(in[80] & 1)) + memcpy(out, in, 80); + return 0; + } + if ((tag == D(122) || tag == D(123)) && SHAPE(0, 4)) + return 0; + if (tag == D(121) && SHAPE(0, 4)) { + static const struct { u32 tag; bool in, reply, expected; } sequence[] = { + {A(389)}, {A(446), false, true, true}, {A(29)}, + {A(468), true}, {A(0), true, true, true}, {A(465), false, true, true}, + }; + for (count = 0; count < ARRAY_SIZE(sequence); count++) { + ret = dcp_v26_simple_call(v26, sequence[count].tag, sequence[count].in, + sequence[count].reply, sequence[count].expected); + if (ret) + return ret; + } + out[0] = 1; + return 0; + } + if (tag == D(101) && SHAPE(0, 4)) { + put_unaligned_le32(v26->stride, out); + return 0; + } + if (tag == D(3) && SHAPE(4, 60)) { + put_unaligned_le32(1, out + 56); /* Explicitly decline bandwidth service. */ + return 0; + } + if (tag == D(451) && SHAPE(20, 28)) { + u32 alignment = get_unaligned_le32(in + 12); + u64 size = get_unaligned_le64(in + 4); + struct dcp_v26_buffer *buf; + + dev_dbg(v26->dev, "allocation flags %#x size %llu alignment %u\n", + get_unaligned_le32(in), size, alignment); + if (get_unaligned_le32(in) != 0x703 || !is_power_of_2(alignment) || + alignment > SZ_16K || in[16] || in[17] || in[18]) + return -EINVAL; + ret = dcp_v26_alloc(v26, size, &id); + if (ret) + return ret; + buf = &v26->buffers[id - 1]; + if ((buf->phys | buf->iova) & (alignment - 1)) + return -EINVAL; + put_unaligned_le64(buf->phys, out); + put_unaligned_le64(buf->iova, out + 8); + put_unaligned_le64(buf->size, out + 16); + put_unaligned_le32(id, out + 24); + return 0; + } + /* Map a buffer for PIODMA; the AP virtual address is not used. */ + if (tag == D(201) && SHAPE(12, 16)) { + u64 buffer = get_unaligned_le64(in); + + if (!buffer || buffer > v26->buffer_count || get_unaligned_le32(in + 8) != 1) + return -EINVAL; + ret = dcp_v26_map_piodma(v26, buffer); + if (ret) + return ret; + put_unaligned_le32(buffer, out); + put_unaligned_le64(v26->buffers[buffer - 1].iova, out + 4); + return 0; + } + /* Release: the buffer stays allocated and mapped until reboot. */ + if (tag == D(454) && SHAPE(4, 4)) { + id = get_unaligned_le32(in); + if (!id || id > v26->buffer_count || v26->buffers[id - 1].retired) + return -EINVAL; + v26->buffers[id - 1].retired = true; + out[0] = 1; + return 0; + } + if (tag == D(582) && SHAPE(8, 4)) + return 0; /* Optional default surface is not allocated by this host. */ + if (tag == D(400) && SHAPE(76, 0xc04)) { + count = get_unaligned_le32(in + 68); + service = get_unaligned_le32(in); + if (count > 0xc00 || (in[72] & 1) || !memchr(in + 4, 0, 64)) + return -EINVAL; + for (offset = 0; offset < v26->raw_count; offset++) { + if (v26->raw[offset].service != service || strcmp(v26->raw[offset].key, in + 4)) + continue; + if (v26->raw[offset].size > count) + return -ENOSPC; + memcpy(out, v26->raw[offset].data, v26->raw[offset].size); + put_unaligned_le32(v26->raw[offset].size, out + 0xc00); + break; + } + return 0; + } + if (tag == D(401) && SHAPE(80, 12)) { + p = dcp_v26_property(v26, get_unaligned_le32(in), in + 4, false); + if (IS_ERR(p)) + return PTR_ERR(p); + if (p) { + put_unaligned_le64(p->value, out); + out[8] = 1; + } + return 0; + } + if ((tag == D(413) && SHAPE(4168, 4)) || + ((tag == D(552) || tag == D(561)) && SHAPE(4164, 4)) || + (tag == D(567) && SHAPE(128, 4))) { + key_offset = tag == D(413) ? 4 : 0; + service = key_offset ? get_unaligned_le32(in) : 0; + if (tag != D(567) && (in[key_offset + 4160] & 1)) { + out[0] = 1; + return 0; + } + count = tag == D(567) ? strnlen(in + 64, 64) : 4096; + ret = dcp_v26_raw_property(v26, service, in + key_offset, in + key_offset + 64, count); + if (!ret) + out[0] = 1; + return ret; + } + if ((tag == D(563) && SHAPE(76, 4)) || (tag == D(565) && SHAPE(72, 4)) || + (tag == D(414) && SHAPE(80, 4)) || (tag == D(415) && SHAPE(76, 4)) || + ((tag == D(102) || tag == D(104)) && SHAPE(68, 0))) { + key_offset = (tag == D(414) || tag == D(415)) ? 4 : 0; + service = key_offset ? get_unaligned_le32(in) : 0; + count = (tag == D(563) || tag == D(414)) ? 8 : 4; + if (out_size && (in[key_offset + 64 + count] & 1)) { + out[0] = 1; + return 0; + } + p = dcp_v26_property(v26, service, in + key_offset, true); + if (IS_ERR(p)) + return PTR_ERR(p); + p->value = count == 8 ? get_unaligned_le64(in + key_offset + 64) : + get_unaligned_le32(in + key_offset + 64); + if (tag == D(104)) + p->value = in[64]; + if (out_size) + out[0] = 1; + return 0; + } + if (tag == D(406) && SHAPE(72, 0)) + return 0; + if (tag == D(107) && SHAPE(64, 0)) { + p = dcp_v26_property(v26, 0, in, false); + if (IS_ERR(p)) + return PTR_ERR(p); + if (p) { + *p = v26->properties[--v26->property_count]; + memset(&v26->properties[v26->property_count], 0, sizeof(*p)); + } + return 0; + } + if ((tag == D(206) || tag == D(207)) && SHAPE(0, 4)) { + ret = dcp_v26_simple_call(v26, tag == D(206) ? A(131) : A(132), false, true, false); + if (!ret) + out[0] = 1; + return ret; + } + if (tag == D(100) && SHAPE(0, 0)) + return dcp_v26_simple_call(v26, A(390), false, true, false); + dev_err(v26->dev, "J613 unhandled kernel callback %#x %u/%u\n", tag, in_size, out_size); + return -EOPNOTSUPP; +#undef SHAPE +} + +static void dcp_v26_idle(struct work_struct *work) +{ + struct apple_dcp_v26 *v26 = container_of(work, struct apple_dcp_v26, idle_work); + int ret = 0; + + mutex_lock(&v26->lock); + while (!v26->failed && !ret) + ret = dcp_v26_link_pump(&v26->link, 0, dcp_v26_callback, v26); + if (ret && ret != -ETIMEDOUT && ret != -EAGAIN) { + v26->failed = true; + dev_err(v26->dev, "DCP notification failed: %d; recovery requires a reboot\n", ret); + } + mutex_unlock(&v26->lock); +} + +static void dcp_v26_crashed(void *cookie, const void *crashlog, size_t crashlog_size) +{ + struct apple_dcp_v26 *v26 = cookie; + /* failed is session-owned; no dereference of a detachable KMS device. */ + WRITE_ONCE(v26->failed, true); + dev_err(v26->dev, "DCP firmware crashed; its buffers are kept until reboot\n"); + dcp_v26_link_fail(&v26->link); +} + +static void dcp_v26_recv(void *cookie, u8 endpoint, u64 message) +{ + struct apple_dcp_v26 *v26 = cookie; + + if (endpoint != APPLE_DCP_LINK_ENDPOINT) { + dev_dbg(v26->dev, "ignored endpoint %#x message %#llx\n", endpoint, message); + return; + } + dcp_v26_link_receive(&v26->link, message); +} + +/* + * Buffers the firmware already owns must lie in a reserved region the boot + * loader described, and be mapped linearly there. The OS log is a physical + * address, not a DART address. + */ +static int dcp_v26_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct apple_dcp_v26 *v26 = cookie; + struct device *dev = v26->dev; + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + phys_addr_t phys; + int i; + + if (!READ_ONCE(v26->dcp) || !bfr->size || bfr->size > SZ_16M || !domain) + return -EINVAL; + if (!bfr->iova) { + bfr->buffer = dma_alloc_coherent(dev, bfr->size, &bfr->iova, GFP_KERNEL); + if (!bfr->buffer) + return -ENOMEM; + memset(bfr->buffer, 0, bfr->size); + dma_wmb(); + return 0; + } + phys = iommu_iova_to_phys(domain, bfr->iova); + for (i = 0; ; i++) { + struct device_node *node = of_parse_phandle(dev->of_node, "memory-region", i); + struct resource res; + size_t off; + bool oslog, valid; + + if (!node) + return -ERANGE; + oslog = of_property_read_bool(node, "apple,dcp-os-log"); + valid = !of_address_to_resource(node, 0, &res); + of_node_put(node); + if (!valid) + continue; + if (oslog && bfr->iova == res.start) + phys = res.start; + if (phys < res.start || phys > res.end || bfr->size - 1 > res.end - phys) + continue; + if (!oslog) { + for (off = 0; off < bfr->size; off += SZ_16K) + if (iommu_iova_to_phys(domain, bfr->iova + off) != phys + off) + return -ERANGE; + if (iommu_iova_to_phys(domain, bfr->iova + bfr->size - 1) != + phys + bfr->size - 1) + return -ERANGE; + } + bfr->buffer = memremap(phys, bfr->size, MEMREMAP_WB); + if (!bfr->buffer) + return -ENOMEM; + bfr->is_mapped = true; + dev_dbg(dev, "RTKit buffer at %pad: %pa, %zu bytes\n", &bfr->iova, &phys, + bfr->size); + return 0; + } +} + +static void dcp_v26_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct apple_dcp_v26 *v26 = cookie; + + if (bfr->is_mapped) + memunmap(bfr->buffer); + else + dma_free_coherent(v26->dev, bfr->size, bfr->buffer, bfr->iova); +} + +static const struct apple_rtkit_ops dcp_v26_rtkit_ops = { + .crashed = dcp_v26_crashed, + .recv_message = dcp_v26_recv, + .shmem_setup = dcp_v26_shmem_setup, + .shmem_destroy = dcp_v26_shmem_destroy, +}; + +/* Freed only if RTKit never started: the firmware may call into it until reboot. */ +static void dcp_v26_release(void *data) +{ + struct apple_dcp_v26 *v26 = data; + + if (!v26->rtk) { + put_device(v26->dev); + kfree(v26); + } +} + +/* Read only: bind maps and claims these registers. */ +static int dcp_v26_cpu_running(struct device *dev, const char *name) +{ + struct resource *res; + void __iomem *coproc; + u32 control; + + res = platform_get_resource_byname(to_platform_device(dev), IORESOURCE_MEM, "coproc"); + if (!res) + return -ENODEV; + coproc = ioremap_np(res->start, resource_size(res)); + if (!coproc) + return -ENOMEM; + control = readl(coproc + DCP_V26_CPU_CONTROL); + iounmap(coproc); + if (!(control & APPLE_DCP_COPROC_CPU_CONTROL_RUN)) + return dev_err_probe(dev, -EBUSY, + "%s display not started: the DCP CPU is stopped (%#x); the PMP is not started for it\n", + name, control); + return 0; +} + +/* + * The board of a 14.x IOMFB DCP node: NULL if the node is not one, an error + * if it is but the machine is not a board this file knows. + */ +const struct dcp_v26_board *iomfb_v26_6_board(struct device *dev) +{ + const struct dcp_v26_board *board; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(dcp_v26_boards); i++) { + board = dcp_v26_boards[i]; + if (!of_device_is_compatible(dev->of_node, board->dcp_compatible)) + continue; + if (board->machine && !of_machine_is_compatible(board->machine)) { + dev_err(dev, "%s display not started: this machine is not a %s\n", + board->name, board->machine); + return ERR_PTR(-ENODEV); + } + return board; + } + return NULL; +} + +const char *iomfb_v26_6_board_name(const struct dcp_v26_board *board) +{ + return board->name; +} + +/* + * The PMP device-power entry points are optional at link time: until a + * provider is built, behave as their !CONFIG_APPLE_PMP stubs do. + */ +static int dcp_v26_pmp_link_device(struct device *dev) +{ + typeof(&apple_pmp_link_device) link; + int ret; + + link = symbol_get(apple_pmp_link_device); + if (!link) + return -ENODEV; + ret = link(dev); + symbol_put(apple_pmp_link_device); + return ret; +} + +static int dcp_v26_pmp_set_device_power(u8 command, u16 device_id, u32 enabled) +{ + typeof(&apple_pmp_set_device_power) set; + int ret; + + set = symbol_get(apple_pmp_set_device_power); + if (!set) + return -ENODEV; + ret = set(command, device_id, enabled); + symbol_put(apple_pmp_set_device_power); + return ret; +} + +static int keep_display_power(struct device *dev) +{ + struct device_node *node; + const u8 *table; + int length, ret = -EINVAL; + + /* The power API is exported before the firmware supplier has probed. + * Link to the DT supplier first so coldplug defers and removal is ordered. */ + ret = dcp_v26_pmp_link_device(dev); + if (ret) + return ret; + node = of_parse_phandle(dev->of_node, "apple,pmp", 0); + if (!node) + return -ENODEV; + table = of_get_property(node, "apple,tunable-soc-device", &length); + /* J613/25G83: logical ID 8, record/bitmap index 11, name DISP. */ + if (table && length == 20 * 0x7c && + get_unaligned_le32(table + 11 * 0x7c) == 8 && + !memcmp(table + 11 * 0x7c + 0x74, "DISP\0\0\0\0", 8)) + ret = dcp_v26_pmp_set_device_power(0x0f, 8, 1); + of_node_put(node); + if (ret) + return ret; + /* G52's synchronous PMP API checks the command acknowledgement. That + * does not qualify domain stability; retain the vote until reboot. */ + dev_info(dev, "J613 native DISP power vote acknowledged: logical-id=8 record=11\n"); + return 0; +} + +int iomfb_v26_6_probe(struct apple_dcp *dcp) +{ + struct device *dev = dcp->dev; + struct device_node *np = dev->of_node; + const struct dcp_v26_board *board = iomfb_v26_6_board(dev); + struct apple_dcp_v26 *v26; + const char *uuid = NULL; + u32 profile, markers[3]; + struct device_node *display, *piodma; + int ret; + u32 version[3]; + + if (of_property_count_u32_elems(np, "apple,j613-25g83-profile") != 1 || + of_property_read_u32(np, "apple,j613-25g83-profile", &profile) || + of_property_count_u32_elems(np, "apple,firmware-compat") != 3 || + of_property_read_u32_array(np, "apple,firmware-compat", version, 3) || + of_property_read_string(np, "apple,firmware-uuid", &uuid) || + !j613_25g83_identity(of_machine_is_compatible("apple,j613"), profile, version, 3, uuid)) + return -ENODEV; + if (IS_ERR_OR_NULL(board)) + return -ENODEV; + if (of_property_read_string(np, "apple,firmware-uuid", &uuid) || + strcmp(uuid, board->firmware_uuid)) + return dev_err_probe(dev, -ENODEV, + "%s display not started: DCP firmware %s is not supported\n", + board->name, uuid ?: "(unknown)"); + display = of_find_node_by_path("/soc/display-subsystem"); + piodma = of_get_child_by_name(np, "piodma"); + ret = of_property_count_u32_elems(np, board->handoff) == 1 && + !of_property_read_u32(np, board->handoff, &markers[0]) && + display && of_property_count_u32_elems(display, board->handoff) == 1 && + !of_property_read_u32(display, board->handoff, &markers[1]) && + piodma && of_property_count_u32_elems(piodma, board->handoff) == 1 && + !of_property_read_u32(piodma, board->handoff, &markers[2]) && + j613_25g83_mappings(markers, 3) && apple_j613_25g83_clock_hz() ? 0 : -ENODEV; + of_node_put(display); + of_node_put(piodma); + if (ret) + return dev_err_probe(dev, ret, "J613/25G83 display mapping or clock handoff missing\n"); + if (!iommu_get_domain_for_dev(dev)) + return dev_err_probe(dev, -ENODEV, + "%s display not started: the DCP has no DART domain\n", + board->name); + ret = dcp_v26_cpu_running(dev, board->name); + if (ret) + return ret; + + ret = keep_display_power(dev); + if (ret) + return dev_err_probe(dev, ret, "J613/25G83 DISP power vote failed\n"); + ret = dcp_v26_cpu_running(dev, board->name); + if (ret) + return ret; + + v26 = kzalloc_obj(*v26); + if (!v26) + return -ENOMEM; + v26->dev = get_device(dev); + v26->board = board; + v26->dcp = dcp; + mutex_init(&v26->lock); + INIT_WORK(&v26->idle_work, dcp_v26_idle); + init_waitqueue_head(&v26->bl_wait); + mutex_init(&v26->bl_lock); + INIT_WORK(&v26->bl_register_work, dcp_v26_backlight_register); + ret = devm_add_action_or_reset(dev, dcp_v26_release, v26); + if (ret) + return ret; + dcp->v26 = v26; + dev_info(dev, "PMP running with the display request acknowledged\n"); + return 0; +} + +/* + * The boot framebuffer gives the stride and the panel size. The boot loader either hides + * the notch rows from it (the default) or, with chosen.asahi,show-notch=1 in m1n1, keeps + * them; apple,notch-height is recorded either way. A board with a known panel tells the + * two apart by the height; any other board is taken to have the notch hidden. + */ +static int dcp_v26_geometry(struct apple_dcp_v26 *v26) +{ + struct device_node *fb = of_find_compatible_node(NULL, NULL, "simple-framebuffer"); + u32 notch = 0; + int ret; + + if (!fb) + return -ENODEV; + ret = of_property_read_u32(fb, "width", &v26->fb_width); + if (!ret) + ret = of_property_read_u32(fb, "height", &v26->fb_height); + if (!ret) + ret = of_property_read_u32(fb, "stride", &v26->stride); + of_node_put(fb); + if (ret) + return ret; + of_property_read_u32(v26->dev->of_node, "apple,notch-height", ¬ch); + /* The firmware takes this stride as its default: 4 bytes per pixel. */ + if (!v26->fb_width || !v26->fb_height || v26->stride != v26->fb_width * 4 || + notch > MAX_NOTCH_HEIGHT) + return -EINVAL; + v26->notch_rows = notch; + v26->panel_width = v26->fb_width; + v26->panel_height = v26->fb_height + notch; + /* The boot loader kept the notch rows: the framebuffer is the whole panel. */ + if (v26->board->panel_height && notch && v26->fb_height == v26->board->panel_height) + v26->panel_height = v26->fb_height; + /* A board with a known panel takes no other. */ + if (v26->board->panel_width && + (v26->panel_width != v26->board->panel_width || + v26->panel_height != v26->board->panel_height)) { + dev_err(v26->dev, "boot framebuffer %ux%u with %u notch rows is not the %ux%u %s panel\n", + v26->fb_width, v26->fb_height, notch, v26->board->panel_width, + v26->board->panel_height, v26->board->name); + return -EINVAL; + } + return 0; +} + +int iomfb_v26_6_bind(struct apple_dcp *dcp) +{ + struct apple_dcp_v26 *v26 = dcp->v26; + struct device *dev = dcp->dev; + struct apple_rtkit *rtk; + u32 control, status; + int ret; + + if (!v26) + return -ENODEV; + if (v26->rtk || dcp_v26_session) + return dev_err_probe(dev, -EBUSY, + "%s display not started: an earlier DCP session is kept; reboot to restart the display\n", + v26->board->name); + + control = readl(dcp->coproc_reg + DCP_V26_CPU_CONTROL); + status = readl(dcp->coproc_reg + DCP_V26_CPU_STATUS); + dev_info(dev, "DCP CPU control %#x status %#x\n", control, status); + if (!(control & APPLE_DCP_COPROC_CPU_CONTROL_RUN)) + return dev_err_probe(dev, -EBUSY, + "%s display not started: the DCP CPU is stopped, and interrupted firmware is never resumed\n", + v26->board->name); + + ret = dcp_v26_geometry(v26); + if (ret) + return dev_err_probe(dev, ret, + "%s display not started: no usable boot framebuffer\n", + v26->board->name); + v26->clock_rate = apple_j613_25g83_clock_hz(); + if (!v26->clock_rate) + return dev_err_probe(dev, -EINVAL, "J613/25G83 same-boot clock observation required\n"); + + dcp_v26_link_init(&v26->link, dev, NULL); + rtk = apple_rtkit_init(dev, v26, "mbox", 0, &dcp_v26_rtkit_ops); + if (IS_ERR(rtk)) + return dev_err_probe(dev, PTR_ERR(rtk), + "%s display not started: RTKit init failed\n", + v26->board->name); + /* From here nothing is freed and the module stays loaded. */ + v26->rtk = rtk; + v26->link.rtk = rtk; + dcp_v26_session = true; + __module_get(THIS_MODULE); + + ret = apple_rtkit_wake(rtk); + if (ret) { + v26->failed = true; + return dev_err_probe(dev, ret, + "%s display not started: the DCP RTKit session did not wake; the boot framebuffer stays\n", + v26->board->name); + } + dev_info(dev, "DCP RTKit session running\n"); + return 0; +} + +void iomfb_v26_6_unbind(struct apple_dcp *dcp) +{ + struct apple_dcp_v26 *v26 = dcp->v26; + + dcp->active = false; + dcp_mode_invalidate(&dcp->mode_state); + if (!v26) + return; + dcp_v26_backlight_detach(v26); + mutex_lock(&v26->lock); + WRITE_ONCE(v26->dcp, NULL); + mutex_unlock(&v26->lock); + if (v26->rtk) + dev_info(dcp->dev, "display unbound; the DCP session and its buffers are kept until reboot\n"); +} + +static int dcp_v26_status_show(struct seq_file *m, void *unused) +{ + struct apple_dcp_v26 *v26 = m->private; + struct drm_framebuffer *fb; + u32 i; + int ret; + + ret = mutex_lock_interruptible(&v26->lock); + if (ret) + return ret; + fb = v26->active_fb; + seq_printf(m, "started %d\nfailed %d\nswaps %llu\nswap_max_us %llu\n", + v26->started, v26->failed, v26->swaps, v26->swap_ns_max / NSEC_PER_USEC); + seq_printf(m, "active_fb %u\nimported %d\n", fb ? fb->base.id : 0, + fb && fb->obj[0]->import_attach); + seq_printf(m, "panel %ux%u\nboot_fb %ux%u stride %u\nclock %llu\n", + v26->panel_width, v26->panel_height, v26->fb_width, v26->fb_height, + v26->stride, v26->clock_rate); + seq_printf(m, "buffers %u bytes %llu\nproperties %u raw %u\nanalytics %llu\n", + v26->buffer_count, v26->buffer_bytes, v26->property_count, v26->raw_count, + v26->analytics); + seq_printf(m, "publications %llu\n", v26->publications); + for (i = 0; i < v26->published_count; i++) + seq_printf(m, "published %u %*phN x%llu\n", v26->published[i].id, + (int)sizeof(v26->published[i].data), v26->published[i].data, + v26->published[i].count); + seq_printf(m, "backlight %d max %u\nbrightness requested %u sent %u owned %d off %d pending %d swaps %llu\n", + !!READ_ONCE(v26->bl_dev), v26->bl_max, v26->bl_nits, v26->bl_sent, + v26->bl_owned, v26->bl_off, v26->bl_pending, v26->bl_swaps); + seq_printf(m, "ctm valid %d calls %llu setter %#x getter %#x\n", v26->ctm_valid, v26->ctm_calls, v26->ctm_status, v26->ctm_get_status); + for (i = 0; i < 9; i++) + seq_printf(m, "ctm_readback[%u] %#llx\n", i, v26->ctm_readback[i]); + mutex_unlock(&v26->lock); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(dcp_v26_status); + +struct dcp_v26_present_snapshot { + struct dcp_v26_present_record history[DCP_V26_HISTORY]; + u8 d590[2][DCP_V26_D590_SIZE]; + ktime_t d590_time[2]; + u64 history_count, d590_count; + bool vrr; +}; + +static s64 dcp_v26_since(ktime_t t, ktime_t start) +{ + return t ? ktime_to_ns(ktime_sub(t, start)) : -1; +} + +/* + * The swap timeline, oldest first: times are ns after the swap start, -1 + * where the swap did not get that far. Then the last swap-complete payloads. + */ +static int dcp_v26_present_show(struct seq_file *m, void *unused) +{ + struct apple_dcp_v26 *v26 = m->private; + struct dcp_v26_present_snapshot *snap; + u64 first, i; + int ret; + + snap = kvmalloc_obj(*snap); + if (!snap) + return -ENOMEM; + ret = mutex_lock_interruptible(&v26->lock); + if (ret) { + kvfree(snap); + return ret; + } + memcpy(snap->history, v26->history, sizeof(snap->history)); + memcpy(snap->d590, v26->d590, sizeof(snap->d590)); + memcpy(snap->d590_time, v26->d590_time, sizeof(snap->d590_time)); + snap->history_count = v26->history_count; + snap->d590_count = v26->d590_count; + snap->vrr = v26->vrr; + mutex_unlock(&v26->lock); + + seq_printf(m, "vrr %d\nrefresh %u\ntimed %d\ntimestamps %d\nflip_timestamp %d\nswaps %llu\n", + snap->vrr, READ_ONCE(v26->refresh), READ_ONCE(v26->timed), + READ_ONCE(j613_25g83_swap_timestamps), READ_ONCE(j613_25g83_flip_timestamp), + snap->history_count); + seq_puts(m, "# id surface timed start_ns submitted_ns replied_ns completed_ns\n"); + first = snap->history_count > DCP_V26_HISTORY ? snap->history_count - DCP_V26_HISTORY : 0; + for (i = first; i < snap->history_count; i++) { + struct dcp_v26_present_record *r = &snap->history[i % DCP_V26_HISTORY]; + + seq_printf(m, "%u %u %u %lld %lld %lld %lld\n", r->id, r->surface, r->timed, + ktime_to_ns(r->start), dcp_v26_since(r->submitted, r->start), + dcp_v26_since(r->replied, r->start), + dcp_v26_since(r->completed, r->start)); + } + first = snap->d590_count > 2 ? snap->d590_count - 2 : 0; + for (i = first; i < snap->d590_count; i++) { + seq_printf(m, "# swap-complete %llu received_ns %lld\n", i + 1, + ktime_to_ns(snap->d590_time[i % 2])); + seq_hex_dump(m, "", DUMP_PREFIX_OFFSET, 16, 1, snap->d590[i % 2], + DCP_V26_D590_SIZE, false); + } + kvfree(snap); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(dcp_v26_present); + +/* + * The modes: the native timing of the boot panel, less the hidden notch rows, + * and on the 120 Hz ProMotion panel also a locked 60 Hz mode. + * + * The locked mode is the same firmware timing with half the pixel clock, so + * it reports 60 Hz. The firmware is never asked for a timing (see + * iomfb_v26_6_modeset()): what differs is only that its swaps carry no + * timing words, and the firmware shows untimed swaps at 60 Hz. Switching + * between the two is a modeset in DRM terms and nothing more for the panel. + * appledrm.j613_25g83_refresh picks which one is preferred. + */ +static int dcp_v26_mode(struct apple_dcp *dcp, struct apple_dcp_v26 *v26) +{ + struct dcp_display_mode *modes, *best = NULL, *preferred, *selected; + struct dcp_parse_ctx ctx; + unsigned int count, i; + void *blob = NULL; + u32 size = 0; + int ret; + + for (i = 0; i < v26->raw_count; i++) { + if (!v26->raw[i].service && !strcmp(v26->raw[i].key, "PreferredTimingElements")) { + size = v26->raw[i].size; + blob = v26->raw[i].data; + break; + } + } + if (!blob) + return -ENODATA; + ret = parse(blob, size, &ctx); + if (ret) + return ret; + ctx.dcp = dcp; + modes = enumerate_modes(&ctx, &count, dcp->panel.width_mm, dcp->panel.height_mm, + dcp->notch_height, true, false); + if (IS_ERR(modes)) + return PTR_ERR(modes); + for (i = 0; i < count; i++) { + struct drm_display_mode *m = &modes[i].mode; + int hz = drm_mode_vrefresh(m); + + dev_dbg(dcp->dev, "timing %u/%u: " DRM_MODE_FMT "%s\n", i + 1, count, + DRM_MODE_ARG(m), m->type & DRM_MODE_TYPE_PREFERRED ? " best" : ""); + if (m->hdisplay != v26->panel_width || + m->vdisplay != v26->panel_height - dcp->notch_height || + (hz != 60 && (hz != 120 || !v26->board->promotion))) + continue; + if (!best || (m->type & DRM_MODE_TYPE_PREFERRED)) + best = &modes[i]; + } + if (!best) { + dev_err(dcp->dev, "no %ux%u timing at 60%s Hz among %u\n", v26->panel_width, + v26->panel_height - dcp->notch_height, + v26->board->promotion ? " or 120" : "", count); + kfree(modes); + return -EINVAL; + } + /* + * The parser marks a ProMotion mode by its size and a sync rate of + * exactly 120 Hz; any 120 Hz mode of this built-in panel is one. + */ + v26->vrr = best->vrr || drm_mode_vrefresh(&best->mode) == 120; + best->vrr = v26->vrr; + count = v26->vrr ? 2 : 1; + selected = kcalloc(count, sizeof(*best), GFP_KERNEL); + if (!selected) { + kfree(modes); + return -ENOMEM; + } + selected[0] = *best; + kfree(modes); + preferred = &selected[0]; + if (v26->vrr) { + struct dcp_display_mode *locked = &selected[1]; + + *locked = selected[0]; + locked->vrr = false; + locked->mode.clock = DIV_ROUND_CLOSEST(locked->mode.clock, 2); + drm_mode_set_name(&locked->mode); + if (j613_25g83_refresh == 60) + preferred = locked; + else if (j613_25g83_refresh != 120) + dev_warn(dcp->dev, "appledrm.j613_25g83_refresh=%u is neither 60 nor 120; preferring 120 Hz\n", + j613_25g83_refresh); + } else if (j613_25g83_refresh == 120 && v26->board->promotion) { + dev_warn(dcp->dev, "the panel has no 120 Hz timing; only %d Hz is available\n", + drm_mode_vrefresh(&preferred->mode)); + } + for (i = 0; i < count; i++) + selected[i].mode.type &= ~DRM_MODE_TYPE_PREFERRED; + preferred->mode.type |= DRM_MODE_TYPE_PREFERRED; + mutex_lock(&dcp->modes_lock); + kfree(dcp->modes); + dcp->modes = selected; + dcp->nr_modes = count; + dcp->modes_admitted = true; + mutex_unlock(&dcp->modes_lock); + dev_info(dcp->dev, "modes: %s; preferred %d Hz (appledrm.j613_25g83_refresh)\n", + v26->vrr ? "120 Hz variable refresh and 60 Hz locked" : "one fixed timing", + drm_mode_vrefresh(&preferred->mode)); + return 0; +} + +int iomfb_v26_6_start(struct apple_dcp *dcp) +{ + struct apple_dcp_v26 *v26 = dcp->v26; + const struct drm_display_mode *mode; + const char *step; + int ret; + + if (!v26 || !v26->rtk || v26->failed || v26->started) + return -ENODEV; + + step = "DCPLink"; + ret = dcp_v26_link_start(&v26->link); + if (ret) + goto fail; + + mutex_lock(&v26->lock); + step = "start signal"; + ret = dcp_v26_simple_call(v26, A(401), false, true, true); + if (!ret) { + step = "first client open"; + ret = dcp_v26_simple_call(v26, A(457), false, false, false); + } + mutex_unlock(&v26->lock); + if (ret) + goto fail; + dcp_v26_link_set_idle_work(&v26->link, &v26->idle_work); + + step = "panel mode"; + mutex_lock(&v26->lock); + ret = dcp_v26_mode(dcp, v26); + mutex_unlock(&v26->lock); + if (ret) + goto fail; + + /* Nits; the backlight device appears with the first frame. */ + v26->bl_max = min_t(u32, dcp->brightness.maximum, v26->board->max_nits); + v26->bl_nits = min_t(u32, DCP_V26_DEFAULT_NITS, v26->bl_max); + v26->started = true; + /* Never removed, like the session it describes. */ + debugfs_create_file("dcp-j613_25g83", 0400, NULL, v26, &dcp_v26_status_fops); + debugfs_create_file("dcp-j613_25g83-present", 0400, NULL, v26, &dcp_v26_present_fops); + dcp->connector->connected = true; + dcp_set_dimensions(dcp, dcp_modes_transfer_begin(dcp)); + dcp_mode_set_valid(&dcp->mode_state, true); + dcp->active = true; + complete(&dcp->start_done); + + mode = &dcp->modes[0].mode; + dev_info(dcp->dev, "%s display started: %ux%u@%d, %u notch rows %s, %ux%u mm\n", + v26->board->name, mode->hdisplay, mode->vdisplay, drm_mode_vrefresh(mode), + v26->notch_rows, + dcp->notch_height ? "hidden" : "shown", mode->width_mm, mode->height_mm); + return 0; +fail: + v26->failed = true; + dev_err(dcp->dev, + "%s display not started: %s failed: %d; the boot framebuffer stays (reboot to retry)\n", + v26->board->name, step, ret); + return ret; +} + +/* + * Panel brightness travels in the swap record, in nits. + * + * Until the first request the firmware keeps the brightness iBoot set, and + * no swap touches it. From then on the driver sets it: a request, the CRTC + * going off (0 nits) and the CRTC coming back (the requested nits) each make + * the target pending, and the next swap carries it. A request waits a frame + * for a flip to carry it, then makes a swap of its own (see + * dcp_v26_brightness_commit()). + */ + +/* Called with the lock held. */ +static void dcp_v26_brightness_changed(struct apple_dcp_v26 *v26) +{ + lockdep_assert_held(&v26->lock); + if (!v26->bl_owned) + return; + WRITE_ONCE(v26->bl_pending, true); + v26->bl_seq++; +} + +/* Adds a pending target to @swap; returns the token dcp_v26_brightness_done() takes. */ +static u64 dcp_v26_brightness_encode(struct apple_dcp_v26 *v26, u8 *swap) +{ + lockdep_assert_held(&v26->lock); + if (!v26->bl_pending) + return 0; + dcp_v26_encode_brightness(swap, v26->bl_off ? 0 : v26->bl_nits); + return v26->bl_seq; +} + +/* A swap completed; @token is what it carried. Called with the lock held. */ +static void dcp_v26_brightness_done(struct apple_dcp_v26 *v26, u64 token) +{ + lockdep_assert_held(&v26->lock); + /* The first frame is on the panel: KMS is up, so is the backlight now. */ + if (!v26->bl_queued && v26->bl_max) { + v26->bl_queued = true; + schedule_work(&v26->bl_register_work); + } + if (!token) + return; + v26->bl_swaps++; + /* A change made since then is still pending. */ + if (token != v26->bl_seq) + return; + v26->bl_sent = v26->bl_off ? 0 : v26->bl_nits; + WRITE_ONCE(v26->bl_pending, false); + wake_up_all(&v26->bl_wait); +} + +/* + * Swap start, then the swap; returns once the firmware completed that swap, + * with @completed set to when its completion came in. + */ +/* Matched UUID ABI: location 9, packed 80-byte request, signed Q32. */ +static int dcp_v26_ctm_locked(struct apple_dcp_v26 *v26, + const struct drm_crtc_state *state) +{ + u8 request[80] = {}, readback[76] = {}; + __le32 status = 0; + const struct drm_color_ctm *ctm; + int i, ret; + + if (!state || (v26->ctm_valid && !state->color_mgmt_changed && + !state->mode_changed && !state->active_changed)) + return 0; + ctm = state->ctm ? state->ctm->data : NULL; + put_unaligned_le32(9, request); + for (i = 0; i < 9; i++) { + u64 coefficient = ctm ? ctm->matrix[i] : + ((i == 0 || i == 4 || i == 8) ? 1ULL << 32 : 0); + u64 magnitude = coefficient & ~(1ULL << 63); + u64 signed_q32 = coefficient & (1ULL << 63) ? -magnitude : magnitude; + + put_unaligned_le64(signed_q32, request + 4 + 8 * i); + } + ret = dcp_v26_call(v26, A(423), request, sizeof(request), + &status, sizeof(status), 0); + v26->ctm_calls++; + v26->ctm_status = le32_to_cpu(status); + v26->ctm_valid = false; + if (ret || v26->ctm_status) { + dev_err(v26->dev, "CTM setter transport=%d status=%#x\n", + ret, v26->ctm_status); + return ret ? ret : -EIO; + } + /* Independent firmware getter, before the swap and on the same lock. */ + memset(request + 4, 0, 76); + ret = dcp_v26_call(v26, A(422), request, sizeof(request), + readback, sizeof(readback), 0); + v26->ctm_get_status = get_unaligned_le32(readback + 72); + for (i = 0; i < 9; i++) + v26->ctm_readback[i] = get_unaligned_le64(readback + i * 8); + dev_info(v26->dev, "CTM call %llu setter=%#x getter_transport=%d getter=%#x diagonal=%#llx,%#llx,%#llx\n", + v26->ctm_calls, v26->ctm_status, ret, v26->ctm_get_status, + v26->ctm_readback[0], v26->ctm_readback[4], v26->ctm_readback[8]); + if (ret || v26->ctm_get_status) + return ret ? ret : -EIO; + v26->ctm_valid = true; + return 0; +} + +static int dcp_v26_swap(struct apple_dcp_v26 *v26, const u8 *surface, u64 iova, + u32 width, u32 height, u32 dst_y, ktime_t *completed, + const struct drm_crtc_state *ctm_state) +{ + struct dcp_v26_present_record rec = {}; + __le32 start[4] = {}, started[2] = {}, result[3]; + u64 brightness = 0; + u8 *swap; + u32 id; + int ret; + + if (surface && (!iova || !width || width > v26->panel_width || !height || + dst_y > v26->panel_height || height > v26->panel_height - dst_y)) + return -EINVAL; + swap = kmalloc(DCP_V26_SWAP_SIZE, GFP_KERNEL); + if (!swap) + return -ENOMEM; + + mutex_lock(&v26->lock); + rec.surface = !!surface; + rec.start = ktime_get(); + ret = dcp_v26_ctm_locked(v26, ctm_state); + if (!ret) + ret = dcp_v26_call(v26, A(406), start, sizeof(start), started, sizeof(started), 0); + if (!ret && le32_to_cpu(started[1])) + ret = -EIO; + id = le32_to_cpu(started[0]); + if (!ret && !id) + ret = -EPROTO; + if (!ret) { + dcp_v26_encode_swap(swap, id, DCP_V26_BLACK, surface, iova, width, height, + dst_y, DCP_V26_LAYER); + /* A target long past: show it at the next refresh, as on M1/M2. */ + if (surface && READ_ONCE(v26->timed) && READ_ONCE(j613_25g83_swap_timestamps)) { + dcp_v26_swap_set_timing(swap, 120); + rec.timed = 1; + } + rec.id = id; + rec.submitted = ktime_get(); + brightness = dcp_v26_brightness_encode(v26, swap); + ret = dcp_v26_call(v26, A(407), swap, DCP_V26_SWAP_SIZE, result, sizeof(result), + id); + /* The swap status sits at byte 5 of the reply. */ + if (!ret && get_unaligned_le32((u8 *)result + 5)) + ret = -EIO; + } + if (ret) { + v26->failed = true; + } else { + dcp_v26_brightness_done(v26, brightness); + rec.replied = v26->link.reply_time; + rec.completed = v26->link.completion_time; + *completed = rec.completed; + } + v26->history[v26->history_count++ % DCP_V26_HISTORY] = rec; + mutex_unlock(&v26->lock); + + kfree(swap); + return ret; +} + +/* + * Shows @fb (or only the black background) and keeps it until the next swap. + * @presented, if set, gets the time the firmware reported the swap complete. + */ +static bool dcp_v26_present(struct apple_dcp_v26 *v26, struct drm_framebuffer *fb, u32 dst_y, + ktime_t *presented, const struct drm_crtc_state *ctm_state) +{ + u8 surface[DCP_V26_SURFACE_SIZE]; + struct drm_framebuffer *old; + u64 iova = 0, start, elapsed, swaps; + ktime_t completed; + int ret; + + if (READ_ONCE(v26->failed)) + return false; + if (fb) { + drm_framebuffer_get(fb); + dcp_v26_encode_surface(surface, fb->pitches[0], fb->width, fb->height, + fb->format->format == DRM_FORMAT_XRGB8888); + iova = drm_fb_dma_get_gem_obj(fb, 0)->dma_addr; + } + start = ktime_get_ns(); + ret = dcp_v26_swap(v26, fb ? surface : NULL, iova, fb ? fb->width : 0, + fb ? fb->height : 0, dst_y, &completed, ctm_state); + elapsed = ktime_get_ns() - start; + if (ret) { + /* Either framebuffer may still be scanned out: keep both. */ + dev_err(v26->dev, "native DCP flip failed %d; buffers pinned until reboot\n", ret); + return false; + } + mutex_lock(&v26->lock); + old = v26->active_fb; + v26->active_fb = fb; + swaps = ++v26->swaps; + v26->swap_ns_max = max(v26->swap_ns_max, elapsed); + mutex_unlock(&v26->lock); + if (old) + drm_framebuffer_put(old); + if (swaps <= 3) + dev_info(v26->dev, "swap %llu complete: surface %d, imported %d, %llu us\n", + swaps, !!fb, fb && fb->obj[0]->import_attach, elapsed / NSEC_PER_USEC); + if (presented) + *presented = completed; + return true; +} + +/* Releases helper waiters without reporting a completed flip. */ +static void dcp_v26_cancel_event(struct apple_dcp *dcp) +{ + struct apple_crtc *crtc = dcp->crtc; + struct drm_pending_vblank_event *event; + unsigned long flags; + + spin_lock_irqsave(&crtc->base.dev->event_lock, flags); + event = crtc->event; + crtc->event = NULL; + spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); + if (!event) + return; + if (event->base.fence) { + dma_fence_set_error(event->base.fence, -EIO); + dma_fence_signal(event->base.fence); + } + if (event->base.completion) { + complete_all(event->base.completion); + if (event->base.completion_release) + event->base.completion_release(event->base.completion); + event->base.completion = NULL; + } + drm_event_cancel_free(crtc->base.dev, &event->base); +} + +/* Only the layout the firmware was qualified with: one full-mode linear plane. */ +int iomfb_v26_6_atomic_check(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *state) +{ + struct apple_dcp_v26 *v26 = dcp->v26; + struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + struct drm_plane_state *p = drm_atomic_get_new_plane_state(state, crtc->primary); + struct drm_gem_dma_object *obj; + struct drm_framebuffer *fb; + u32 width, height; + + if (dcp->crashed || !v26 || READ_ONCE(v26->failed)) + return -EIO; + if (!crtc_state || !crtc_state->active || !p || p->crtc != crtc || !p->visible) + return 0; + + fb = p->fb; + width = crtc_state->mode.hdisplay; + height = crtc_state->mode.vdisplay; + if (fb->format->format != DRM_FORMAT_XRGB8888 && + fb->format->format != DRM_FORMAT_ARGB8888) + return -EINVAL; + if (fb->modifier != DRM_FORMAT_MOD_LINEAR || fb->offsets[0] || + fb->pitches[0] < width * 4 || (fb->pitches[0] & 63)) + return -EINVAL; + if (p->src_x || p->src_y || p->crtc_x || p->crtc_y || + p->src_w != width << 16 || p->src_h != height << 16 || + p->crtc_w != width || p->crtc_h != height || + fb->width != width || fb->height != height) + return -EINVAL; + obj = drm_fb_dma_get_gem_obj(fb, 0); + if (!obj || !obj->dma_addr || (u64)fb->pitches[0] * height > obj->base.size) + return -EINVAL; + return 0; +} + +int iomfb_v26_6_modeset(struct apple_dcp *dcp, struct drm_crtc_state *crtc_state) +{ + struct dcp_display_mode mode; + + if (!lookup_mode(dcp, &crtc_state->mode, &mode)) + return -EINVAL; + /* + * The firmware keeps the timing it booted with, which every mode + * shares: nothing is sent to it. The mode only decides whether the + * swaps that follow carry timing words (120 Hz ProMotion) or not + * (locked 60 Hz). + */ + if (dcp->v26) { + WRITE_ONCE(dcp->v26->timed, mode.vrr); + WRITE_ONCE(dcp->v26->refresh, drm_mode_vrefresh(&mode.mode)); + } + dcp_mode_set_valid(&dcp->mode_state, true); + return 0; +} + +void iomfb_v26_6_flush(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *state) +{ + struct drm_plane_state *p = drm_atomic_get_new_plane_state(state, crtc->primary); + struct drm_framebuffer *fb; + ktime_t presented; + struct drm_crtc_state *cs = drm_atomic_get_new_crtc_state(state, crtc); + + if (!cs) + cs = crtc->state; + if (!p) + p = crtc->primary->state; + fb = p && p->visible ? p->fb : NULL; + dcp->swap_start = ktime_get(); + if (!dcp->v26 || !dcp_v26_present(dcp->v26, fb, dcp->notch_height, &presented, cs)) { + dcp_v26_cancel_event(dcp); + return; + } + if (READ_ONCE(j613_25g83_flip_timestamp)) { + /* + * The flip completed when the firmware's swap-complete message + * came in, not when this worker got to run: stamp the event with + * that time as it is (no swap_start correction). + */ + dcp->swap_start = KTIME_MIN; + dcp_drm_crtc_page_flip(dcp, presented); + } else { + dcp_drm_crtc_page_flip(dcp, ktime_get()); + } +} + +/* + * Makes a swap for a pending brightness: a commit of the CRTC alone, which + * shows the plane as it is. Nothing is done while the CRTC is off (enabling + * it sends the brightness) or before its first modeset (the first frame + * sends it), nor when a flip carried it in the meantime. + */ +static int dcp_v26_brightness_commit(struct apple_dcp_v26 *v26) +{ + struct apple_dcp *dcp = READ_ONCE(v26->dcp); + struct drm_modeset_acquire_ctx ctx; + struct drm_crtc_state *crtc_state; + struct drm_atomic_commit *state; + struct drm_crtc *crtc; + int ret; + + if (!dcp || !dcp->crtc) + return 0; + crtc = &dcp->crtc->base; + state = drm_atomic_commit_alloc(crtc->dev); + if (!state) + return -ENOMEM; + drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE); + state->acquire_ctx = &ctx; +retry: + ret = drm_modeset_lock(&crtc->mutex, &ctx); + if (ret) + goto out; + if (!crtc->state || !crtc->state->active || !READ_ONCE(v26->bl_pending)) + goto out; + crtc_state = drm_atomic_get_crtc_state(state, crtc); + if (IS_ERR(crtc_state)) { + ret = PTR_ERR(crtc_state); + goto out; + } + ret = drm_atomic_commit(state); +out: + if (ret == -EDEADLK) { + drm_atomic_commit_clear(state); + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry; + } + drm_atomic_commit_put(state); + drm_modeset_drop_locks(&ctx); + drm_modeset_acquire_fini(&ctx); + return ret; +} + +static int dcp_v26_backlight_update(struct backlight_device *bd) +{ + struct apple_dcp_v26 *v26 = bl_get_data(bd); + int ret = 0; + + mutex_lock(&v26->lock); + if (v26->failed) { + ret = -EIO; + } else { + /* Sent even when unchanged: it also repairs the firmware's value. */ + v26->bl_nits = backlight_get_brightness(bd); + v26->bl_owned = true; + dcp_v26_brightness_changed(v26); + } + mutex_unlock(&v26->lock); + if (ret) + return ret; + + if (wait_event_timeout(v26->bl_wait, !READ_ONCE(v26->bl_pending), DCP_V26_BL_FLIP_WAIT)) + return 0; + return dcp_v26_brightness_commit(v26); +} + +static const struct backlight_ops dcp_v26_backlight_ops = { + .update_status = dcp_v26_backlight_update, +}; + +/* Nits, like the M1/M2 panels: 0 to the DT maximum, linear in luminance. */ +static void dcp_v26_backlight_register(struct work_struct *work) +{ + struct apple_dcp_v26 *v26 = container_of(work, struct apple_dcp_v26, bl_register_work); + struct backlight_properties props = { + .type = BACKLIGHT_PLATFORM, + .scale = BACKLIGHT_SCALE_LINEAR, + .max_brightness = v26->bl_max, + .brightness = min_t(u32, DCP_V26_DEFAULT_NITS, v26->bl_max), + }; + struct backlight_device *bd; + + mutex_lock(&v26->bl_lock); + if (!v26->bl_detached && !v26->bl_dev) { + bd = backlight_device_register("apple-panel-bl", v26->dev, v26, + &dcp_v26_backlight_ops, &props); + if (IS_ERR(bd)) + dev_err(v26->dev, "no panel backlight device: %pe\n", bd); + else + WRITE_ONCE(v26->bl_dev, bd); + } + mutex_unlock(&v26->bl_lock); +} + +/* KMS goes away: so does the backlight device, before the CRTC it commits to. */ +static void dcp_v26_backlight_detach(struct apple_dcp_v26 *v26) +{ + struct backlight_device *bd; + + mutex_lock(&v26->bl_lock); + v26->bl_detached = true; + bd = v26->bl_dev; + WRITE_ONCE(v26->bl_dev, NULL); + mutex_unlock(&v26->bl_lock); + cancel_work_sync(&v26->bl_register_work); + backlight_device_unregister(bd); +} + +/* 0 nits while the CRTC is off, the requested nits once it is on again. */ +static void dcp_v26_brightness_power(struct apple_dcp_v26 *v26, bool on) +{ + mutex_lock(&v26->lock); + v26->bl_off = !on; + dcp_v26_brightness_changed(v26); + mutex_unlock(&v26->lock); +} + +void iomfb_v26_6_poweron(struct apple_dcp *dcp) +{ + /* The panel stays on; the next swap shows the plane again. */ + if (dcp->v26 && dcp->v26->started) + dcp_v26_brightness_power(dcp->v26, true); +} + +void iomfb_v26_6_poweroff(struct apple_dcp *dcp) +{ + /* Blank to black; the panel and the DCP stay powered. */ + if (dcp->v26 && dcp->v26->started) { + dcp_v26_brightness_power(dcp->v26, false); + dcp_v26_present(dcp->v26, NULL, 0, NULL, NULL); + } +} diff --git a/drivers/gpu/drm/apple/iomfb_v26_6.h b/drivers/gpu/drm/apple/iomfb_v26_6.h new file mode 100644 index 0000000000000..2831ee2f79345 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v26_6.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * J613 (M3 Air 13 inch) internal display: macOS 26.6.2 (25G83) IOMFB over DCPLink + * + * Copyright The Asahi Linux Contributors + */ + +#ifndef __APPLE_IOMFB_V26_6_H__ +#define __APPLE_IOMFB_V26_6_H__ + +#include + +struct apple_dcp; +struct apple_dcp_v26; +struct dcp_v26_board; +struct device; +struct drm_atomic_commit; +struct drm_crtc; +struct drm_crtc_state; + +/* + * The board of a 25G83 IOMFB DCP node (by compatible and machine): NULL if the + * node is not one, ERR_PTR(-ENODEV) if the machine is not a known board. + */ +const struct dcp_v26_board *iomfb_v26_6_board(struct device *dev); +const char *iomfb_v26_6_board_name(const struct dcp_v26_board *board); + +/* Platform probe: firmware identity, boot loader handoff, PMP acknowledgment. */ +int iomfb_v26_6_probe(struct apple_dcp *dcp); +/* Component bind: the running DCP's RTKit session. */ +int iomfb_v26_6_bind(struct apple_dcp *dcp); +/* Component unbind: KMS goes away; the firmware session is kept until reboot. */ +void iomfb_v26_6_unbind(struct apple_dcp *dcp); +/* DCPLink, start signal, first client open and the panel mode. */ +int iomfb_v26_6_start(struct apple_dcp *dcp); + +/* + * The PMP device-power API (include/linux/soc/apple/pmp.h, exported by the + * PMP driver). Declared here until that header is in this tree; the backend + * resolves both with symbol_get() and fails closed without a provider. + */ +#ifndef _LINUX_SOC_APPLE_PMP_H +int apple_pmp_link_device(struct device *consumer); +int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled); +#endif + +int iomfb_v26_6_atomic_check(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *state); +int iomfb_v26_6_modeset(struct apple_dcp *dcp, struct drm_crtc_state *crtc_state); +void iomfb_v26_6_flush(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *state); +void iomfb_v26_6_poweron(struct apple_dcp *dcp); +void iomfb_v26_6_poweroff(struct apple_dcp *dcp); + +#endif /* __APPLE_IOMFB_V26_6_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_v26_6_link.c b/drivers/gpu/drm/apple/iomfb_v26_6_link.c new file mode 100644 index 0000000000000..154e814f421ca --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v26_6_link.c @@ -0,0 +1,502 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * DCPLink RPC transport for the J613/25G83 DCP firmware (macOS 26.6.2 (25G83) IOMFB) + * + * Copyright The Asahi Linux Contributors + * Based on the M3 Pro DCP lab client by Eryk Wieliczko. + * + * This firmware speaks DCPLink with a few differences from the H17P + * variant in iomfb.c: + * - the INIT descriptor's version field carries the RPC alignment (64), + * and READY is exactly 0x004045bc9ef40101 (alignment64, hash45bc9ef4); + * - the AP sends every call on its stream 0. A call made while handling a + * callback reuses that callback's side (the remote bit of its message), + * is placed after an outer call on the same side, 64-byte aligned, and + * never sets the NESTED bit, which this firmware reserves for a free + * stream; + * - callbacks may arrive on either side, and are decoded where their + * message says. + * + * One kernel client owns the link. Its caller processes the callbacks that + * arrive while it waits for a reply, including calls made from inside them, + * so replies, callbacks and swap completions are handled in firmware order. + * Messages arriving while no call is in progress are queued and handed to + * an idle worker. The RTKit receive worker only queues. + * + * Nothing here is ever torn down: once the firmware has seen the RPC memory + * it keeps it until reboot, and a failed link stays failed. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iomfb_v26_6_link.h" + +#define DCP_V26_LINK_ALIGNMENT 64 +#define DCP_V26_LINK_READY_MSG 0x004045bc9ef40101ULL +#define DCP_V26_READY_TIMEOUT msecs_to_jiffies(3000) +#define DCP_V26_CALL_TIMEOUT (5 * HZ) +#define DCP_V26_TAG_D590 DCP_V26_TAG('D', 590) +#define DCP_V26_D590_IN_SIZE 0x730 + +enum dcp_v26_event_kind { + DCP_V26_EVENT_REPLY = 1, + DCP_V26_EVENT_CALLBACK = 2, +}; + +struct dcp_v26_event { + struct list_head list; + u32 kind; + u32 size; + u64 message; + /* When the message came in, before any queueing. */ + ktime_t time; + u8 data[]; +}; + +/* Called with the lock held. */ +static void link_fail(struct dcp_v26_link *link) +{ + if (!link->failed) + dev_err(link->dev, "DCP link stopped; firmware buffers are kept until reboot\n"); + link->failed = true; + wake_up(&link->wait); +} + +/* Called with the lock held. */ +static void link_event(struct dcp_v26_link *link, u32 kind, u64 message, ktime_t time, + const void *data, u32 size) +{ + struct dcp_v26_event *e; + + if (link->event_count >= DCP_V26_LINK_MAX_EVENTS || + size > APPLE_DCP_LINK_STREAM_BUFFER_SIZE) { + link_fail(link); + return; + } + e = kmalloc(struct_size(e, data, size), GFP_KERNEL); + if (!e) { + link_fail(link); + return; + } + e->kind = kind; + e->size = size; + e->message = message; + e->time = time; + if (size) + memcpy(e->data, data, size); + list_add_tail(&e->list, &link->events); + link->event_count++; + wake_up(&link->wait); + if (link->idle_work) + queue_work(system_unbound_wq, link->idle_work); +} + +void dcp_v26_link_init(struct dcp_v26_link *link, struct device *dev, + struct apple_rtkit *rtk) +{ + link->dev = dev; + link->rtk = rtk; + mutex_init(&link->lock); + init_waitqueue_head(&link->wait); + INIT_LIST_HEAD(&link->events); + init_completion(&link->ready); +} + +void dcp_v26_link_fail(struct dcp_v26_link *link) +{ + mutex_lock(&link->lock); + link_fail(link); + mutex_unlock(&link->lock); + complete(&link->ready); +} + +bool dcp_v26_link_failed(struct dcp_v26_link *link) +{ + return READ_ONCE(link->failed); +} + +void dcp_v26_link_set_idle_work(struct dcp_v26_link *link, struct work_struct *work) +{ + mutex_lock(&link->lock); + link->idle_work = work; + if (link->event_count) + queue_work(system_unbound_wq, work); + mutex_unlock(&link->lock); +} + +/* Host-driven start: endpoint, INIT with the RPC memory, then READY. */ +int dcp_v26_link_start(struct dcp_v26_link *link) +{ + struct apple_dcp_link_init_desc *desc; + u64 message; + int ret; + + link->rpc = dma_alloc_coherent(link->dev, APPLE_DCP_LINK_RPC_MEMORY_SIZE, + &link->rpc_dva, GFP_KERNEL); + if (!link->rpc) + return -ENOMEM; + memset(link->rpc, 0, APPLE_DCP_LINK_RPC_MEMORY_SIZE); + desc = link->rpc; + apple_dcp_link_init_desc_encode(desc, 0, false); + /* This firmware reads the version field as the RPC alignment. */ + desc->version = cpu_to_le32(DCP_V26_LINK_ALIGNMENT); + ret = apple_dcp_link_init_message(link->rpc_dva, &message); + if (ret) + return ret; + + link->ready_result = -ETIMEDOUT; + ret = apple_rtkit_start_ep(link->rtk, APPLE_DCP_LINK_ENDPOINT); + if (ret) + return ret; + dma_wmb(); + ret = apple_rtkit_send_message(link->rtk, APPLE_DCP_LINK_ENDPOINT, message, + NULL, false); + if (ret) + return ret; + if (!wait_for_completion_timeout(&link->ready, DCP_V26_READY_TIMEOUT)) + return -ETIMEDOUT; + + mutex_lock(&link->lock); + ret = link->failed ? -EIO : link->ready_result; + if (!ret) + link->enabled = true; + mutex_unlock(&link->lock); + if (!ret) + dev_info(link->dev, "DCPLink ready, RPC alignment %d\n", DCP_V26_LINK_ALIGNMENT); + return ret; +} + +/* RTKit receive worker: check, record and queue; never blocks on a call. */ +void dcp_v26_link_receive(struct dcp_v26_link *link, u64 message) +{ + struct apple_dcp_link_stream_layout layout; + struct apple_dcp_link_rpc_view view; + u8 stream = apple_dcp_link_message_stream(message); + bool remote = apple_dcp_link_message_is_remote(message); + ktime_t now = ktime_get(); + bool nested; + + mutex_lock(&link->lock); + if (link->failed) + goto out; + + if (!link->enabled) { + /* Firmware-side INIT needs no answer; READY ends the handshake. */ + if ((message & 3) == APPLE_DCP_LINK_MSG_FIRMWARE_INIT) + goto out; + link->ready_result = message == DCP_V26_LINK_READY_MSG ? 0 : -EPROTO; + if (link->ready_result) + dev_err(link->dev, "unexpected DCPLink ready message %#llx\n", message); + mutex_unlock(&link->lock); + complete(&link->ready); + return; + } + + if ((message & 3) != APPLE_DCP_LINK_MSG_RPC) { + dev_err(link->dev, "unexpected DCPLink message %#llx\n", message); + goto error; + } + dma_rmb(); + + if (message & BIT_ULL(6)) { + /* The reply to the innermost call, echoed in place. */ + nested = link->calls[1].active; + if (stream || !link->calls[nested].active || + link->calls[nested].remote != remote || + apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, 0, remote, &layout)) + goto error; + if (memcmp(link->rpc + layout.local_offset + link->calls[nested].offset, + &link->calls[nested].header, sizeof(link->calls[nested].header))) + goto error; + link_event(link, DCP_V26_EVENT_REPLY, message, now, + link->rpc + layout.local_offset + link->calls[nested].offset, + link->calls[nested].size); + link->calls[nested].active = false; + goto out; + } + + if (link->callback_count >= DCP_V26_LINK_MAX_CALLBACKS || + apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, stream, remote, &layout) || + apple_dcp_link_rpc_decode(message, link->rpc + layout.remote_offset, + layout.capacity, &view)) + goto error; + if ((view.call >> 24) != 'D') { + dev_err(link->dev, "DCPLink callback with tag %#x\n", view.call); + goto error; + } + link->callbacks[link->callback_count].message = message; + link->callbacks[link->callback_count].output = view.output; + link->callbacks[link->callback_count].size = view.output_size; + link->callbacks[link->callback_count].call = view.call; + link->callback_count++; + dev_dbg(link->dev, "callback %#x in %u out %u\n", + view.call, view.input_size, view.output_size); + link_event(link, DCP_V26_EVENT_CALLBACK, message, now, + link->rpc + layout.remote_offset + view.offset, view.total_size); + goto out; +error: + link_fail(link); +out: + mutex_unlock(&link->lock); +} + +/* Places a call packet (header, input, zeroed output) and sends it. */ +static int link_submit(struct dcp_v26_link *link, const struct apple_dcp_link_rpc_header *rpc, + u32 size, bool nested) +{ + struct apple_dcp_link_stream_layout layout; + u32 total, offset = 0, input_size, output_size; + bool remote = false; + u64 message; + int ret; + + mutex_lock(&link->lock); + if (!link->enabled || link->failed) { + ret = -EIO; + goto out; + } + if (link->calls[nested].active || (nested && !link->callback_count) || + (!nested && link->callback_count)) { + ret = -EBUSY; + goto out; + } + if (nested) { + u64 callback = link->callbacks[link->callback_count - 1].message; + + if (apple_dcp_link_message_stream(callback)) { + ret = -EOPNOTSUPP; + goto out; + } + /* A nested call goes to the callback's side of stream 0, after the outer call. */ + remote = apple_dcp_link_message_is_remote(callback); + if (link->calls[0].active && link->calls[0].remote == remote) + offset = ALIGN(link->calls[0].size, DCP_V26_LINK_ALIGNMENT); + } + input_size = le32_to_cpu(rpc->input_size); + output_size = le32_to_cpu(rpc->output_size); + ret = apple_dcp_link_rpc_payload_size(input_size, output_size, &total); + if (ret || total != size) { + ret = -EMSGSIZE; + goto out; + } + ret = apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, 0, remote, &layout); + if (ret) + goto out; + if (offset + size > layout.capacity) { + ret = -EMSGSIZE; + goto out; + } + memcpy(link->rpc + layout.local_offset + offset, rpc, size); + memset(link->rpc + layout.local_offset + offset + sizeof(*rpc) + input_size, 0, + output_size); + link->calls[nested].header = *rpc; + link->calls[nested].size = size; + link->calls[nested].offset = offset; + link->calls[nested].remote = remote; + link->calls[nested].active = true; + apple_dcp_link_rpc_message(offset, size, false, &message); + apple_dcp_link_stream_message(message, 0, remote, &message); + dma_wmb(); + ret = apple_rtkit_send_message(link->rtk, APPLE_DCP_LINK_ENDPOINT, message, NULL, false); + if (ret) + link_fail(link); +out: + mutex_unlock(&link->lock); + return ret; +} + +/* Writes the innermost callback's output in place and acknowledges it. */ +static int link_reply(struct dcp_v26_link *link, u64 message, const void *output, u32 size) +{ + unsigned int top; + int ret = -EINVAL; + + mutex_lock(&link->lock); + if (!link->enabled || link->failed) { + ret = -EIO; + goto out; + } + if (!link->callback_count) + goto out; + top = link->callback_count - 1; + if (message != link->callbacks[top].message || size != link->callbacks[top].size) + goto out; + memcpy(link->callbacks[top].output, output, size); + dma_wmb(); + ret = apple_rtkit_send_message(link->rtk, APPLE_DCP_LINK_ENDPOINT, + apple_dcp_link_rpc_reply(message), NULL, false); + if (ret) + link_fail(link); + else + link->callback_count--; +out: + mutex_unlock(&link->lock); + return ret; +} + +/* Uninterruptible: an abandoned call would leave the firmware mid-call. */ +static struct dcp_v26_event *link_next_event(struct dcp_v26_link *link, unsigned long timeout) +{ + struct dcp_v26_event *e; + + if (!wait_event_timeout(link->wait, + READ_ONCE(link->event_count) || READ_ONCE(link->failed), + timeout)) + return ERR_PTR(-ETIMEDOUT); + mutex_lock(&link->lock); + if (link->failed || list_empty(&link->events)) { + mutex_unlock(&link->lock); + return ERR_PTR(-EIO); + } + e = list_first_entry(&link->events, struct dcp_v26_event, list); + list_del(&e->list); + link->event_count--; + mutex_unlock(&link->lock); + return e; +} + +static int link_callback(struct dcp_v26_link *link, struct dcp_v26_event *e, + dcp_v26_callback_fn callback, void *cookie, u32 *completed) +{ + struct apple_dcp_link_rpc_header *h = (void *)e->data; + u32 in, out, tag; + int ret; + + if (e->kind != DCP_V26_EVENT_CALLBACK || e->size < sizeof(*h) || !callback) + return -EPROTO; + in = le32_to_cpu(h->input_size); + out = le32_to_cpu(h->output_size); + tag = le32_to_cpu(h->call); + if (in > e->size - sizeof(*h) || out != e->size - sizeof(*h) - in) + return -EPROTO; + memset(e->data + sizeof(*h) + in, 0, out); + link->callback_time = e->time; + link->callback_depth++; + ret = callback(cookie, tag, h + 1, in, e->data + sizeof(*h) + in, out); + link->callback_depth--; + if (ret) { + dev_err(link->dev, "J613/25G83 callback %#x %u/%u depth=%u failed=%d\n", tag, in, out, link->callback_depth + 1, ret); + return ret; + } + if (tag == DCP_V26_TAG_D590 && in == DCP_V26_D590_IN_SIZE) { + link->completion_time = e->time; + if (completed) + *completed = get_unaligned_le32(h + 1); + } + return link_reply(link, e->message, e->data + sizeof(*h) + in, out); +} + +int dcp_v26_link_pump(struct dcp_v26_link *link, unsigned long timeout, + dcp_v26_callback_fn callback, void *cookie) +{ + struct dcp_v26_event *e = link_next_event(link, timeout); + int ret; + + if (IS_ERR(e)) + return PTR_ERR(e); + ret = link_callback(link, e, callback, cookie, NULL); + kfree(e); + return ret; +} + +/* + * Calls @tag and handles the callbacks that arrive until its reply, and, for + * a non-zero @completion_id, until the swap-complete callback for that swap. + * Must not be called concurrently with itself or dcp_v26_link_pump(), except + * from inside a callback handler. + */ +int dcp_v26_link_call(struct dcp_v26_link *link, u32 tag, const void *input, + u32 input_size, void *output, u32 output_size, + u32 completion_id, dcp_v26_callback_fn callback, void *cookie) +{ + struct apple_dcp_link_rpc_header *packet; + unsigned long deadline = jiffies + DCP_V26_CALL_TIMEOUT; + u64 expected = APPLE_DCP_LINK_MSG_RPC_REPLY; + u32 size, completed = 0; + bool nested, replied = false; + int ret; + + ret = apple_dcp_link_rpc_payload_size(input_size, output_size, &size); + if (ret || size > APPLE_DCP_LINK_STREAM_BUFFER_SIZE) + return -EMSGSIZE; + packet = kzalloc(size, GFP_KERNEL); + if (!packet) + return -ENOMEM; + apple_dcp_link_rpc_header_encode(packet, tag, input_size, output_size); + if (input_size) + memcpy(packet + 1, input, input_size); + +retry: + /* + * Handle queued notifications before an outer call. A queued + * callback is not an enclosing call until its handler runs. + */ + if (!link->callback_depth) { + while (READ_ONCE(link->event_count)) { + ret = dcp_v26_link_pump(link, 0, callback, cookie); + if (ret) { + kfree(packet); + return ret; + } + } + } + mutex_lock(&link->lock); + nested = link->callback_depth != 0; + if (nested && !link->callback_count) { + mutex_unlock(&link->lock); + kfree(packet); + return -EPROTO; + } + /* The reply to a nested call carries the callback's side. */ + if (nested) + expected |= link->callbacks[link->callback_count - 1].message & + APPLE_DCP_LINK_MSG_REMOTE; + mutex_unlock(&link->lock); + ret = link_submit(link, packet, size, nested); + /* A callback arrived between the drain and the submit. */ + if (ret == -EBUSY && !nested && time_before(jiffies, deadline)) + goto retry; + kfree(packet); + if (ret) + return ret; + + while (time_before(jiffies, deadline)) { + struct dcp_v26_event *e = link_next_event(link, max_t(long, deadline - jiffies, 0)); + struct apple_dcp_link_rpc_header *h; + + if (IS_ERR(e)) + return PTR_ERR(e); + h = (void *)e->data; + if (e->kind == DCP_V26_EVENT_REPLY) { + if (replied || e->size != size || e->message != expected || + le32_to_cpu(h->call) != tag || + le32_to_cpu(h->input_size) != input_size || + le32_to_cpu(h->output_size) != output_size) { + ret = -EPROTO; + } else { + if (output_size) + memcpy(output, e->data + sizeof(*h) + input_size, + output_size); + link->reply_time = e->time; + replied = true; + } + } else { + ret = link_callback(link, e, callback, cookie, &completed); + if (!ret && completion_id && completed && completed != completion_id) + ret = -EPROTO; + } + kfree(e); + if (ret) + return ret; + if (replied && (!completion_id || completed == completion_id)) + return 0; + } + return -ETIMEDOUT; +} diff --git a/drivers/gpu/drm/apple/iomfb_v26_6_link.h b/drivers/gpu/drm/apple/iomfb_v26_6_link.h new file mode 100644 index 0000000000000..12e14f6be020f --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v26_6_link.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * DCPLink RPC transport for the J613/25G83 DCP firmware (macOS 26.6.2 (25G83) IOMFB) + * + * Copyright The Asahi Linux Contributors + * Based on the M3 Pro DCP lab client by Eryk Wieliczko. + */ + +#ifndef __APPLE_IOMFB_V26_6_LINK_H__ +#define __APPLE_IOMFB_V26_6_LINK_H__ + +#include +#include +#include +#include +#include +#include +#include + +#include "dcp-link.h" + +#define DCP_V26_LINK_MAX_EVENTS 32 +#define DCP_V26_LINK_MAX_CALLBACKS 8 + +/* Method and callback tags: 'A' or 'D' followed by three decimal digits. */ +#define DCP_V26_TAG(c, n) (((u32)(c) << 24) | ((u32)('0' + (n) / 100) << 16) | \ + ((u32)('0' + (n) / 10 % 10) << 8) | (u32)('0' + (n) % 10)) + +struct apple_rtkit; +struct work_struct; + +/* + * Handles one callback from the firmware. @out is zeroed and is sent back + * as the reply when the handler returns 0; any error stops the link. + */ +typedef int (*dcp_v26_callback_fn)(void *cookie, u32 tag, const void *in, + u32 in_size, void *out, u32 out_size); + +struct dcp_v26_link { + struct device *dev; + struct apple_rtkit *rtk; + void *rpc; + dma_addr_t rpc_dva; + + /* Protects everything below; never held across a callback handler. */ + struct mutex lock; + wait_queue_head_t wait; + struct list_head events; + unsigned int event_count; + unsigned int callback_count; + /* Callback handlers running on the caller's stack. */ + unsigned int callback_depth; + bool failed; + bool enabled; + struct work_struct *idle_work; + + struct completion ready; + int ready_result; + + /* + * Receive times (CLOCK_MONOTONIC) of the message being handled: of the + * callback whose handler runs, of the last reply, and of the last swap + * completion. Only the thread that owns the RPC stream uses them. + */ + ktime_t callback_time; + ktime_t reply_time; + ktime_t completion_time; + + /* Our calls: [0] outer, [1] made from inside a callback. */ + struct { + bool active; + bool remote; + u32 offset; + u32 size; + struct apple_dcp_link_rpc_header header; + } calls[2]; + + /* Firmware callbacks awaiting a reply, innermost last. */ + struct { + u64 message; + void *output; + u32 size; + u32 call; + } callbacks[DCP_V26_LINK_MAX_CALLBACKS]; +}; + +void dcp_v26_link_init(struct dcp_v26_link *link, struct device *dev, + struct apple_rtkit *rtk); +int dcp_v26_link_start(struct dcp_v26_link *link); +void dcp_v26_link_receive(struct dcp_v26_link *link, u64 message); +void dcp_v26_link_fail(struct dcp_v26_link *link); +bool dcp_v26_link_failed(struct dcp_v26_link *link); +void dcp_v26_link_set_idle_work(struct dcp_v26_link *link, struct work_struct *work); +int dcp_v26_link_call(struct dcp_v26_link *link, u32 tag, const void *input, + u32 input_size, void *output, u32 output_size, + u32 completion_id, dcp_v26_callback_fn callback, void *cookie); +int dcp_v26_link_pump(struct dcp_v26_link *link, unsigned long timeout, + dcp_v26_callback_fn callback, void *cookie); + +#endif /* __APPLE_IOMFB_V26_6_LINK_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_v26_6_swap.h b/drivers/gpu/drm/apple/iomfb_v26_6_swap.h new file mode 100644 index 0000000000000..0807a5f9a6dd0 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v26_6_swap.h @@ -0,0 +1,175 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Swap (A407) and surface records of the J613/25G83 DCP firmware (macOS 26.6.2 (25G83)) + * + * Copyright The Asahi Linux Contributors + * Based on the M3 Pro DCP lab client by Eryk Wieliczko. + */ + +#ifndef __APPLE_IOMFB_V26_6_SWAP_H__ +#define __APPLE_IOMFB_V26_6_SWAP_H__ + +#include +#include +#include +#include +#include + +#define DCP_V26_SWAP_SIZE 0x1bd8 +#define DCP_V26_SURFACE_SIZE 0x22c + +/* Offsets in the swap record. Rectangles are x, y, w, h (u32 each). */ +#define DCP_V26_SWAP_TS1 0x00 /* u64 presentation timing */ +#define DCP_V26_SWAP_TS2 0x08 +#define DCP_V26_SWAP_TS3 0x30 +#define DCP_V26_SWAP_ID 0x98 +#define DCP_V26_SWAP_SURF_ID(p) (0x9c + (p) * 4) +#define DCP_V26_SWAP_SRC(p) (0xac + (p) * 16) /* in the surface */ +#define DCP_V26_SWAP_DST(p) (0x10c + (p) * 16) /* on the panel */ +#define DCP_V26_SWAP_ENABLED 0x14c +#define DCP_V26_SWAP_COMPLETED 0x150 +#define DCP_V26_SWAP_BACKGROUND 0x154 +/* 25G83: shuffle at TEXT1c20ac preserves update at wire354; the nits + * double at wire35e is copied to the internal record at7a8, consumed by + * TEXT22c7b4..22c7ec as brightnessLevel (0x13), in 16.16 nits. */ +#define DCP_V26_SWAP_BRIGHTNESS_UPDATE 0x354 +#define DCP_V26_SWAP_BRIGHTNESS_NITS 0x35e +#define DCP_V26_SWAP_SURFACE(p) (0x588 + (p) * DCP_V26_SURFACE_SIZE) +#define DCP_V26_SWAP_IOVA(p) (0xe38 + (p) * 8) +#define DCP_V26_SWAP_SURF_NULL(p) (0x1bcb + (p)) +#define DCP_V26_SWAP_SURF_NULL_COUNT 10 + +#define DCP_V26_SWAP_LAYERS 0x7 +#define DCP_V26_SWAP_SET_BACKGROUND BIT(31) + +/* Offsets in a surface record. */ +#define DCP_V26_SURF_OPAQUE 0x02 +#define DCP_V26_SURF_FORMAT 0x0b +#define DCP_V26_SURF_STRIDE 0x15 +#define DCP_V26_SURF_WIDTH 0x21 +#define DCP_V26_SURF_HEIGHT 0x25 +#define DCP_V26_SURF_SIZE 0x29 +#define DCP_V26_SURF_PLANE_WIDTH 0x59 +#define DCP_V26_SURF_PLANE_HEIGHT 0x5d +#define DCP_V26_SURF_PLANE_STRIDE 0x69 +#define DCP_V26_SURF_PLANE_SIZE 0x6d + +/* 'BGRA': 8-bit linear, ARGB8888 in DRM terms. */ +#define DCP_V26_FORMAT_BGRA 0x42475241 + +/* + * A single-plane linear BGRA surface covering the whole framebuffer. The + * firmware has no XRGB format: an opaque surface blends as premultiplied + * alpha over the black background, which shows XRGB pixels unchanged. + */ +static inline void dcp_v26_encode_surface(u8 *s, u32 stride, u32 width, u32 height, + bool opaque) +{ + u32 bytes = stride * height; + + memset(s, 0, DCP_V26_SURFACE_SIZE); + s[DCP_V26_SURF_OPAQUE] = opaque; + put_unaligned_le32(1, s + 0x03); + put_unaligned_le32(1, s + 0x07); + put_unaligned_le32(DCP_V26_FORMAT_BGRA, s + DCP_V26_SURF_FORMAT); + s[0x13] = 13; + s[0x14] = 12; + put_unaligned_le32(stride, s + DCP_V26_SURF_STRIDE); + put_unaligned_le16(1, s + 0x19); + s[0x1b] = 1; + s[0x1c] = 1; + put_unaligned_le32(width, s + DCP_V26_SURF_WIDTH); + put_unaligned_le32(height, s + DCP_V26_SURF_HEIGHT); + put_unaligned_le32(bytes, s + DCP_V26_SURF_SIZE); + put_unaligned_le32(1, s + 0x35); + put_unaligned_le64(1, s + 0x51); + put_unaligned_le32(width, s + DCP_V26_SURF_PLANE_WIDTH); + put_unaligned_le32(height, s + DCP_V26_SURF_PLANE_HEIGHT); + put_unaligned_le32(stride, s + DCP_V26_SURF_PLANE_STRIDE); + put_unaligned_le32(bytes, s + DCP_V26_SURF_PLANE_SIZE); + put_unaligned_le16(4, s + 0x71); + s[0x73] = 1; + s[0x74] = 1; + put_unaligned_le64(1, s + 0x149); +} + +static inline void dcp_v26_put_rect(u8 *r, u32 x, u32 y, u32 w, u32 h) +{ + put_unaligned_le32(x, r); + put_unaligned_le32(y, r + 4); + put_unaligned_le32(w, r + 8); + put_unaligned_le32(h, r + 12); +} + +/* + * The swap record for swap @id. @background fills every panel pixel that no + * surface covers: the whole panel when @surface is NULL. With a surface on + * @layer, the whole @width x @height surface is read (source 0,0) and shown + * unscaled at 0,@dst_y on the panel: 0 for the full panel, the notch height + * to keep the rows above it black. + */ +static inline void dcp_v26_encode_swap(u8 *swap, u32 id, u32 background, const u8 *surface, + u64 iova, u32 width, u32 height, u32 dst_y, u32 layer) +{ + memset(swap, 0, DCP_V26_SWAP_SIZE); + put_unaligned_le32(id, swap + DCP_V26_SWAP_ID); + put_unaligned_le32(DCP_V26_SWAP_SET_BACKGROUND | DCP_V26_SWAP_LAYERS, + swap + DCP_V26_SWAP_ENABLED); + put_unaligned_le32(DCP_V26_SWAP_SET_BACKGROUND | DCP_V26_SWAP_LAYERS, + swap + DCP_V26_SWAP_COMPLETED); + put_unaligned_le32(background, swap + DCP_V26_SWAP_BACKGROUND); + memset(swap + DCP_V26_SWAP_SURF_NULL(0), 1, DCP_V26_SWAP_SURF_NULL_COUNT); + swap[0x1bd6] = 1; + swap[0x1bd7] = 1; + if (surface) { + memcpy(swap + DCP_V26_SWAP_SURFACE(layer), surface, DCP_V26_SURFACE_SIZE); + put_unaligned_le64(iova, swap + DCP_V26_SWAP_IOVA(layer)); + put_unaligned_le32(1, swap + DCP_V26_SWAP_SURF_ID(layer)); + dcp_v26_put_rect(swap + DCP_V26_SWAP_SRC(layer), 0, 0, width, height); + dcp_v26_put_rect(swap + DCP_V26_SWAP_DST(layer), 0, dst_y, width, height); + swap[DCP_V26_SWAP_SURF_NULL(layer)] = 0; + } +} + +/* + * The swap record opens with the same block of presentation timing words + * as the 12.x/13.x firmware. The first is when the swap should be shown, + * in the firmware's clock, or zero for no target; a flag byte at 0x338 + * makes it relative to the swap's arrival instead. The firmware shows a + * swap whose target has passed at the next refresh (it may drop it for a + * newer swap queued behind it), and holds one whose target is more than + * about a frame ahead. It carries the other words along for its traces. + * None of them sets a refresh rate. + * + * With all of them zero, back-to-back swaps on the 120 Hz variable-refresh + * (ProMotion) panel complete about 16.7 ms apart, i.e. at 60 Hz. The M1/M2 + * hosts put a non-zero value in them on such panels, which makes their + * firmware present at 120 Hz: a target long past, so "as soon as possible". + */ +static inline void dcp_v26_swap_set_timing(u8 *swap, u64 value) +{ + put_unaligned_le64(value, swap + DCP_V26_SWAP_TS1); + put_unaligned_le64(value, swap + DCP_V26_SWAP_TS2); + put_unaligned_le64(value, swap + DCP_V26_SWAP_TS3); +} + +/* @nits as an IEEE 754 double, exact for every u32, without kernel FP. */ +static inline u64 dcp_v26_nits_to_double(u32 nits) +{ + unsigned int exponent; + + if (!nits) + return 0; + exponent = fls(nits) - 1; + return ((u64)(1023 + exponent) << 52) | + (((u64)nits << (52 - exponent)) & GENMASK_ULL(51, 0)); +} + +/* Asks for @nits in a swap record built by dcp_v26_encode_swap(). */ +static inline void dcp_v26_encode_brightness(u8 *swap, u32 nits) +{ + swap[DCP_V26_SWAP_BRIGHTNESS_UPDATE] = 1; + put_unaligned_le64(dcp_v26_nits_to_double(nits), swap + DCP_V26_SWAP_BRIGHTNESS_NITS); +} + +#endif /* __APPLE_IOMFB_V26_6_SWAP_H__ */ diff --git a/drivers/gpu/drm/apple/j613-25g83.h b/drivers/gpu/drm/apple/j613-25g83.h new file mode 100644 index 0000000000000..636471b786016 --- /dev/null +++ b/drivers/gpu/drm/apple/j613-25g83.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_J613_25G83_H +#define APPLE_J613_25G83_H + +#include +#include +#include +#include + +#define J613_25G83_DCP_UUID "C042E95C-B9D8-3F0E-94B3-582A08AA6FDD" +#define J613_25G83_PMP_UUID "EFD60284-7C58-33A9-8522-CB3366AF6040" +#define J613_25G83_HANDOFF "apple,j613-25g83-mapping-handoff" + +static inline bool j613_25g83_identity(bool j613, u32 profile, + const u32 *version, size_t cells, + const char *uuid) +{ + return j613 && profile == 1 && version && cells == 3 && + version[0] == 26 && version[1] == 6 && version[2] == 2 && + uuid && !strcmp(uuid, J613_25G83_DCP_UUID); +} + +static inline bool j613_25g83_clock(const u32 *witness, size_t cells) +{ + return witness && cells == 5 && witness[0] == 1 && witness[1] == 345 && + witness[2] == 416 && witness[3] && !witness[4]; +} + +static inline bool j613_25g83_mappings(const u32 *markers, size_t cells) +{ + return markers && cells == 3 && markers[0] == 1 && markers[1] == 1 && + markers[2] == 1; +} + +/* + * Local copies of the PMP table checks of + * drivers/pmdomain/apple/pmp-report-validation.h, which is not in this tree + * yet; the real header wins when it is included first. + */ +#ifndef _APPLE_PMP_REPORT_VALIDATION_H +/* The PTD request read, request update and acknowledgment apertures. */ +static inline bool apple_pmp_ranges_valid(const u8 *table, size_t len, + u32 request_read, u32 request_write, + u32 actual) +{ + unsigned int found = 0; + size_t pos; + + if (!table || !len || len % 32) + return false; + for (pos = 0; pos < len; pos += 32) { + u32 id = get_unaligned_le32(table + pos); + u64 base = get_unaligned_le32(table + pos + 4); + u32 count = get_unaligned_le32(table + pos + 8); + + if (id == 10) { + if ((found & BIT(0)) || !count || base * 16 != request_read || + 0x10000 + base * 8 != request_write) + return false; + found |= BIT(0); + } else if (id == 11) { + if ((found & BIT(1)) || !count || base * 16 != actual) + return false; + found |= BIT(1); + } + } + return found == (BIT(0) | BIT(1)); +} + +/* Packed integer arguments are validated before the PMP driver patches them. */ +static inline bool apple_pmp_bootargs_valid(const u8 *args, size_t len) +{ + unsigned int found = 0; + size_t pos = 0; + + if (!args || !len) + return false; + while (pos < len) { + u32 key, size; + + if (len - pos < 8) + return false; + key = get_unaligned_le32(args + pos); + size = get_unaligned_le32(args + pos + 4); + pos += 8; + if (size > len - pos) + return false; + if (key == 0x42444944 || key == 0x44564944) { /* BDID or DVID */ + if (!size || size > 8) + return false; + found |= key == 0x42444944 ? BIT(0) : BIT(1); + } + pos += size; + } + return found == (BIT(0) | BIT(1)); +} +#endif + +#endif diff --git a/drivers/gpu/drm/apple/j613_dcp_activate.c b/drivers/gpu/drm/apple/j613_dcp_activate.c new file mode 100644 index 0000000000000..8fa196f5f7813 --- /dev/null +++ b/drivers/gpu/drm/apple/j613_dcp_activate.c @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Coldplug of the exact J613 25G83 display and its same-boot PMP supplier. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dcp.h" +#include "j613-25g83.h" + +static unsigned int display_clock_hz; +static struct of_changeset gate; + +unsigned int apple_j613_25g83_clock_hz(void) +{ + return display_clock_hz; +} + +/* These are DT domain nodes, carrying the PMGR offsets, not copied registers. */ +static int j613_power_ready(void) +{ + static const char *const labels[] = { + "disp_sys", "disp_fe", "disp_cpu", "pmp", "pms_sram", + }; + static const u32 offsets[] = { 0x1d8, 0x208, 0x10000, 0x450, 0x458 }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(labels); i++) { + struct device_node *np, *found = NULL; + const char *label; + void __iomem *pmgr; + struct resource res; + u32 offset, state; + + for_each_compatible_node(np, NULL, "apple,t8122-pmgr-pwrstate") { + if (of_property_read_string(np, "label", &label) || + strcmp(label, labels[i])) + continue; + if (found) { + of_node_put(found); + of_node_put(np); + return -EINVAL; + } + found = of_node_get(np); + } + if (!found) + return -ENODEV; + if (of_property_read_u32(found, "reg", &offset) || offset != offsets[i] || + of_address_to_resource(found->parent, 0, &res) || + resource_size(&res) < 4 || offset > resource_size(&res) - 4) { + of_node_put(found); + return -EINVAL; + } + /* The register range is described by the parent syscon. */ + pmgr = of_iomap(found->parent, 0); + of_node_put(found); + if (!pmgr) + return -ENOMEM; + state = readl(pmgr + offset); + iounmap(pmgr); + if (FIELD_GET(GENMASK(7, 4), state) != 15) + return -EBUSY; + } + return 0; +} + +/* The SRAM image and integer bootargs must match this boot's nub. */ +static int j613_pmp_image(struct device_node *pmp) +{ + void __iomem *sram, *asc, *ptd; + struct resource res; + uuid_t image, expected; + u32 offset, size; + u8 *args; + unsigned int pos; + int ret = -EINVAL; + + if (uuid_parse(J613_25G83_PMP_UUID, &expected) || + of_address_to_resource(pmp, 0, &res)) + return -EINVAL; + sram = of_iomap(pmp, 0); + if (!sram) + return -ENOMEM; + if (resource_size(&res) < 0x234) + goto out; + for (pos = 0; pos < sizeof(image.b); pos += 4) + put_unaligned_le32(readl(sram + 0x214 + pos), image.b + pos); + offset = readl(sram + 0x22c); + size = readl(sram + 0x230); + if (!uuid_equal(&image, &expected) || !size || size > SZ_4K || + size > resource_size(&res) || offset > resource_size(&res) - size || offset & 3) + goto out; + args = kmalloc(size, GFP_KERNEL); + if (!args) { + ret = -ENOMEM; + goto out; + } + memcpy_fromio(args, sram + offset, size); + ret = apple_pmp_bootargs_valid(args, size) ? 0 : -EINVAL; + kfree(args); +out: + iounmap(sram); + if (ret) + return ret; + asc = of_iomap(pmp, 1); + if (!asc) + return -ENOMEM; + /* A new RTKit session starts only the halted, matching PMP image. */ + ret = readl(asc + 0x44) & BIT(4) ? -EBUSY : 0; + iounmap(asc); + if (ret) + return ret; + ptd = ioremap_np(0x2d03c0000ULL, 0x14000); + if (!ptd) + return -ENOMEM; + ret = readq(ptd + 0x10) || readq(ptd + 0x1000) ? -EBUSY : 0; + iounmap(ptd); + return ret; +} + +static int j613_pmp_values(struct device_node *pmp) +{ + struct device_node *values = of_find_node_by_path("/chosen/asahi,t8122-pmp"); + struct property *prop, *copy; + const char *uuid; + const u8 *table; + u32 value; + int length, ret = -EINVAL; + + if (!values) + return -ENODEV; + if (of_property_read_u32(values, "apple,board-id", &value) || + of_property_read_u32(values, "apple,dram-vendor-id", &value) || + of_property_read_string(values, "apple,tunable-uuid", &uuid) || + strcmp(uuid, J613_25G83_PMP_UUID)) + goto out; + table = of_get_property(values, "apple,tunable-ptd-range", &length); + if (!apple_pmp_ranges_valid(table, length, 0x1000, 0x10800, 0x1080)) + goto out; + table = of_get_property(values, "apple,tunable-soc-device", &length); + if (!table || length != 20 * 0x7c || + get_unaligned_le32(table + 11 * 0x7c) != 8 || + memcmp(table + 11 * 0x7c + 0x74, "DISP\0\0\0\0", 8)) + goto out; + ret = j613_pmp_image(pmp); + if (ret) + goto out; + for_each_property_of_node(values, prop) { + if (strcmp(prop->name, "apple,board-id") && + strcmp(prop->name, "apple,dram-vendor-id") && + strcmp(prop->name, "apple,dram-capacity") && + !strstarts(prop->name, "apple,tunable-")) + continue; + if (!prop->length || prop->length > SZ_64K || + of_property_present(pmp, prop->name)) { + ret = -EINVAL; + break; + } + copy = kzalloc_obj(*copy); + if (!copy) { + ret = -ENOMEM; + break; + } + copy->name = kstrdup(prop->name, GFP_KERNEL); + copy->value = kmemdup(prop->value, prop->length, GFP_KERNEL); + copy->length = prop->length; + if (!copy->name || !copy->value) { + kfree(copy->name); + kfree(copy->value); + kfree(copy); + ret = -ENOMEM; + break; + } + ret = of_changeset_add_property(&gate, pmp, copy); + if (ret) { + kfree(copy->name); + kfree(copy->value); + kfree(copy); + } + if (ret) + break; + } +out: + of_node_put(values); + return ret; +} + +int __init apple_j613_25g83_coldplug(void) +{ + static const char *const paths[] = { + "/soc/pmp-report@2d03c0000", "/soc/iommu@2d0300000", + "/soc/mailbox@2d0c08000", "/soc/pmp@2d0500000", + "/soc/iommu@28d30c000", "/soc/iommu@28d304000", + "/soc/mailbox@28ec08000", "/soc/display-subsystem", + "/soc/dcp@28ec00000/piodma", "/soc/dcp@28ec00000", + }; + struct device_node *nodes[ARRAY_SIZE(paths)] = {}, *chosen; + const char *uuid, *status; + void __iomem *cpu; + u32 profile, version[3], witness[5], markers[3]; + unsigned int i; + int ret = -EINVAL; + + if (!of_machine_is_compatible("apple,j613")) + return 0; + nodes[9] = of_find_node_by_path(paths[9]); + /* Ordinary 14.x J613 boot keeps the existing gate and report driver. */ + if (!nodes[9] || !of_property_present(nodes[9], "apple,j613-25g83-profile")) { + ret = 0; + goto out; + } + if (of_property_count_u32_elems(nodes[9], "apple,j613-25g83-profile") != 1 || + of_property_read_u32(nodes[9], "apple,j613-25g83-profile", &profile) || + of_property_count_u32_elems(nodes[9], "apple,firmware-compat") != 3 || + of_property_read_u32_array(nodes[9], "apple,firmware-compat", version, 3) || + of_property_read_string(nodes[9], "apple,firmware-uuid", &uuid) || + !j613_25g83_identity(true, profile, version, 3, uuid)) + goto out; + chosen = of_find_node_by_path("/chosen"); + ret = chosen && of_property_count_u32_elems(chosen, + "apple,j613-25g83-display-clock-adt") == 5 ? + of_property_read_u32_array(chosen, "apple,j613-25g83-display-clock-adt", witness, 5) : -EINVAL; + of_node_put(chosen); + if (ret || !j613_25g83_clock(witness, 5)) { + ret = -EINVAL; + goto out; + } + for (i = 0; i < ARRAY_SIZE(nodes); i++) { + if (!nodes[i]) + nodes[i] = of_find_node_by_path(paths[i]); + if (!nodes[i] || of_property_read_string(nodes[i], "status", &status) || + strcmp(status, "disabled")) { + ret = -EBUSY; + goto out; + } + } + for (i = 0; i < 3; i++) { + if (of_property_count_u32_elems(nodes[7 + i], J613_25G83_HANDOFF) != 1 || + of_property_read_u32(nodes[7 + i], J613_25G83_HANDOFF, &markers[i])) { + ret = -EINVAL; + goto out; + } + } + if (!j613_25g83_mappings(markers, 3)) { + ret = -EINVAL; + goto out; + } + ret = j613_power_ready(); + if (ret) + goto out; + cpu = of_iomap(nodes[9], 0); + if (!cpu) { + ret = -ENOMEM; + goto out; + } + ret = readl(cpu + 0x44) & BIT(4) ? 0 : -EBUSY; + iounmap(cpu); + if (ret) + goto out; + of_changeset_init(&gate); + ret = j613_pmp_values(nodes[3]); + for (i = 0; i < ARRAY_SIZE(nodes) && !ret; i++) + ret = of_changeset_update_prop_string(&gate, nodes[i], "status", "okay"); + if (!ret) { + display_clock_hz = witness[3]; + ret = of_changeset_apply(&gate); + } + if (ret) { + display_clock_hz = 0; + of_changeset_destroy(&gate); + } else { + display_clock_hz = witness[3]; + pr_info("J613/25G83 display handoff accepted; ADT clock %u Hz\n", display_clock_hz); + } +out: + for (i = 0; i < ARRAY_SIZE(nodes); i++) + of_node_put(nodes[i]); + return ret; +} diff --git a/drivers/gpu/drm/apple/version_utils.h b/drivers/gpu/drm/apple/version_utils.h index c2684bdb121d4..4627fd148e15c 100644 --- a/drivers/gpu/drm/apple/version_utils.h +++ b/drivers/gpu/drm/apple/version_utils.h @@ -36,6 +36,8 @@ enum dcp_firmware_version { */ DCP_FIRMWARE_V_14_7_TEMPLATE, DCP_FIRMWARE_H17P, + /* J613 macOS 26.6.2 (25G83) IOMFB over DCPLink (iomfb_v26_6.c). */ + DCP_FIRMWARE_V_26_6, }; #endif /*__APPLE_VERSION_UTILS_H__*/ From bed1cbe5469a8ec40de858d92c71fe48667299d1 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:43:18 -0700 Subject: [PATCH 1642/2680] drm: apple: use tabs for display C indentation Convert leading indentation in the v26 CTM code, status output and audio registration to tabs. No code or control flow changes. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 01f6785158535f5254a418503fea42a18ca9ac1f) --- drivers/gpu/drm/apple/audio.c | 2 +- drivers/gpu/drm/apple/iomfb_v26_6.c | 92 ++++++++++++++--------------- 2 files changed, 47 insertions(+), 47 deletions(-) diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index 9917d660ded55..4b8c4d8fdd784 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -806,7 +806,7 @@ static struct platform_driver dcpaud_driver = { int __init dcp_audio_register(void) { - return platform_driver_register(&dcpaud_driver); + return platform_driver_register(&dcpaud_driver); } void __exit dcp_audio_unregister(void) diff --git a/drivers/gpu/drm/apple/iomfb_v26_6.c b/drivers/gpu/drm/apple/iomfb_v26_6.c index b922c74817dec..fe7f0fd9460e8 100644 --- a/drivers/gpu/drm/apple/iomfb_v26_6.c +++ b/drivers/gpu/drm/apple/iomfb_v26_6.c @@ -1208,8 +1208,8 @@ static int dcp_v26_status_show(struct seq_file *m, void *unused) !!READ_ONCE(v26->bl_dev), v26->bl_max, v26->bl_nits, v26->bl_sent, v26->bl_owned, v26->bl_off, v26->bl_pending, v26->bl_swaps); seq_printf(m, "ctm valid %d calls %llu setter %#x getter %#x\n", v26->ctm_valid, v26->ctm_calls, v26->ctm_status, v26->ctm_get_status); - for (i = 0; i < 9; i++) - seq_printf(m, "ctm_readback[%u] %#llx\n", i, v26->ctm_readback[i]); + for (i = 0; i < 9; i++) + seq_printf(m, "ctm_readback[%u] %#llx\n", i, v26->ctm_readback[i]); mutex_unlock(&v26->lock); return 0; } @@ -1502,55 +1502,55 @@ static void dcp_v26_brightness_done(struct apple_dcp_v26 *v26, u64 token) */ /* Matched UUID ABI: location 9, packed 80-byte request, signed Q32. */ static int dcp_v26_ctm_locked(struct apple_dcp_v26 *v26, - const struct drm_crtc_state *state) + const struct drm_crtc_state *state) { - u8 request[80] = {}, readback[76] = {}; - __le32 status = 0; - const struct drm_color_ctm *ctm; - int i, ret; - - if (!state || (v26->ctm_valid && !state->color_mgmt_changed && - !state->mode_changed && !state->active_changed)) - return 0; - ctm = state->ctm ? state->ctm->data : NULL; - put_unaligned_le32(9, request); - for (i = 0; i < 9; i++) { - u64 coefficient = ctm ? ctm->matrix[i] : - ((i == 0 || i == 4 || i == 8) ? 1ULL << 32 : 0); - u64 magnitude = coefficient & ~(1ULL << 63); - u64 signed_q32 = coefficient & (1ULL << 63) ? -magnitude : magnitude; - - put_unaligned_le64(signed_q32, request + 4 + 8 * i); - } - ret = dcp_v26_call(v26, A(423), request, sizeof(request), - &status, sizeof(status), 0); - v26->ctm_calls++; - v26->ctm_status = le32_to_cpu(status); - v26->ctm_valid = false; - if (ret || v26->ctm_status) { - dev_err(v26->dev, "CTM setter transport=%d status=%#x\n", - ret, v26->ctm_status); - return ret ? ret : -EIO; - } - /* Independent firmware getter, before the swap and on the same lock. */ - memset(request + 4, 0, 76); - ret = dcp_v26_call(v26, A(422), request, sizeof(request), - readback, sizeof(readback), 0); - v26->ctm_get_status = get_unaligned_le32(readback + 72); - for (i = 0; i < 9; i++) - v26->ctm_readback[i] = get_unaligned_le64(readback + i * 8); - dev_info(v26->dev, "CTM call %llu setter=%#x getter_transport=%d getter=%#x diagonal=%#llx,%#llx,%#llx\n", - v26->ctm_calls, v26->ctm_status, ret, v26->ctm_get_status, - v26->ctm_readback[0], v26->ctm_readback[4], v26->ctm_readback[8]); - if (ret || v26->ctm_get_status) - return ret ? ret : -EIO; - v26->ctm_valid = true; - return 0; + u8 request[80] = {}, readback[76] = {}; + __le32 status = 0; + const struct drm_color_ctm *ctm; + int i, ret; + + if (!state || (v26->ctm_valid && !state->color_mgmt_changed && + !state->mode_changed && !state->active_changed)) + return 0; + ctm = state->ctm ? state->ctm->data : NULL; + put_unaligned_le32(9, request); + for (i = 0; i < 9; i++) { + u64 coefficient = ctm ? ctm->matrix[i] : + ((i == 0 || i == 4 || i == 8) ? 1ULL << 32 : 0); + u64 magnitude = coefficient & ~(1ULL << 63); + u64 signed_q32 = coefficient & (1ULL << 63) ? -magnitude : magnitude; + + put_unaligned_le64(signed_q32, request + 4 + 8 * i); + } + ret = dcp_v26_call(v26, A(423), request, sizeof(request), + &status, sizeof(status), 0); + v26->ctm_calls++; + v26->ctm_status = le32_to_cpu(status); + v26->ctm_valid = false; + if (ret || v26->ctm_status) { + dev_err(v26->dev, "CTM setter transport=%d status=%#x\n", + ret, v26->ctm_status); + return ret ? ret : -EIO; + } + /* Independent firmware getter, before the swap and on the same lock. */ + memset(request + 4, 0, 76); + ret = dcp_v26_call(v26, A(422), request, sizeof(request), + readback, sizeof(readback), 0); + v26->ctm_get_status = get_unaligned_le32(readback + 72); + for (i = 0; i < 9; i++) + v26->ctm_readback[i] = get_unaligned_le64(readback + i * 8); + dev_info(v26->dev, "CTM call %llu setter=%#x getter_transport=%d getter=%#x diagonal=%#llx,%#llx,%#llx\n", + v26->ctm_calls, v26->ctm_status, ret, v26->ctm_get_status, + v26->ctm_readback[0], v26->ctm_readback[4], v26->ctm_readback[8]); + if (ret || v26->ctm_get_status) + return ret ? ret : -EIO; + v26->ctm_valid = true; + return 0; } static int dcp_v26_swap(struct apple_dcp_v26 *v26, const u8 *surface, u64 iova, u32 width, u32 height, u32 dst_y, ktime_t *completed, - const struct drm_crtc_state *ctm_state) + const struct drm_crtc_state *ctm_state) { struct dcp_v26_present_record rec = {}; __le32 start[4] = {}, started[2] = {}, result[3]; From 3191382008efce44287be5628550e15032f34a80 Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Wed, 7 Oct 2026 06:13:03 -0600 Subject: [PATCH 1643/2680] drm/apple: let a Thunderbolt route follow the CRTC a modeset pairs it with MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 12.1 lets a direct DP-alt route follow the CRTC a modeset pairs it with, but keeps Thunderbolt tunnels on the pipeline they arrived on, with possible_crtcs narrowed to it. That leaves #39's original case open: on an M1 Pro (j314s), an LG UltraFine 5K alone on a Thunderbolt port is routed to dcpext1, aquamarine has read the port's possible_crtcs before the narrowing and pairs it with dcpext0's CRTC, the lowest free one, and every modeset is refused. On 12.2 the LG stays dark at login, with and without the tiled split of #22. Let tunnels follow as direct routes do: - Keep a tunnel port's possible_crtcs at every pipeline that can drive it, where its routes follow their CRTC. - Accept a tunnel in the follow decision, as the moving route or as the holder of a swap, when its DP IN has a crossbar control to the new pipeline. - Carry the tunnel's DP IN and Thunderbolt binding through the handoff: record them before anything is released, revoke them on the pipeline the route leaves, and install them on the pipeline it joins, or back on the original one on rollback. Only the crossbar output of the DP IN changes pipeline; the Thunderbolt side keeps its tunnel. - Start the link of a moved tunnel before its CRTC powers on, as for a moved direct route. A pipeline whose HDMI port only had its presence sampled at probe or resume, with no display there, no longer refuses a move. That sample leaves the port settling for 10 s, which covers the login on a fast boot: aquamarine pairs the LG about 3 s after the sample, and refusing leaves it dark for good. A recent HDMI edge still refuses the move, as it may be a display coming back. Tested on a MacBook Pro 14" M1 Pro (j314s), 12.2 with #22, Hyprland 0.56.2 / aquamarine 0.15.1, an LG UltraFine 5K over Thunderbolt and a Dell U2715H on HDMI or on a USB-C to HDMI adapter: - LG alone at login: 5120x2880 on dcpext0 after one move, with both tiles (dark on plain 12.2). DPMS off/on and s2idle passed. - LG and HDMI, HDMI unplugged, then DPMS: the tunnel and its second tile follow aquamarine from dcpext1 to dcpext0. - LG on dcpext1 and the adapter display on dcpext0: DPMS on swaps the tunnel and the direct route. - The fabric KUnit suite passes (36 cases). [ryan: adapted to 7.2.10 — the follow decision keeps refusing a port, or a swap holder's port, with safe_pending; possible_crtcs narrowing keeps this tree's dual-stream behaviour (narrow unless the route follows its CRTC); there is no tb_retiring here, so the handoff only clears the binding under tb_lock; the Thunderbolt binding generation (tb_generation/tunnel_generation, 170-usb API) is not in this tree yet, so the carried binding is the set_active callback and its context only] Signed-off-by: Christian Bartels (cherry picked from commit afbcf8083bd587a53aa36f4d509cb4ff09b0d053) --- drivers/gpu/drm/apple/dcp-fabric-core.c | 18 +++--- drivers/gpu/drm/apple/dcp-fabric-test.c | 22 +++++-- drivers/gpu/drm/apple/dcp-fabric.c | 81 ++++++++++++++++++++----- drivers/gpu/drm/apple/iomfb.c | 2 +- 4 files changed, 92 insertions(+), 31 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.c b/drivers/gpu/drm/apple/dcp-fabric-core.c index 53fd9d48a60f1..ba8ea0d7320db 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.c +++ b/drivers/gpu/drm/apple/dcp-fabric-core.c @@ -326,11 +326,14 @@ EXPORT_SYMBOL_GPL(dcp_fabric_tunnel_candidate); * pipeline its port reaches through @to (NULL: none). Stay, move the * route there, swap it with @holder, the route of another port holding * that pipeline, or refuse the modeset. A move needs a pipeline that is - * free but for its CRTC: no live fixed output and its services up. A - * fixed-output settling window follows the existing fabric policy. - * A swap needs a holder whose own display is off, or about to be - * shown on @from's pipeline, and that can reach @from's pipeline through - * @holder_back. + * free but for its CRTC: no live fixed output and its services up, and + * no recent HDMI edge, which may be a display coming back. The HDMI + * presence sample taken at probe or resume is no edge: the compositor has + * paired the display already, and refusing would leave it dark. Direct + * routes and Thunderbolt tunnels follow alike; the caller checks that a + * tunnel's DP IN can reach the pipeline. A swap needs a holder whose own + * display is off, or about to be shown on @from's pipeline, and that can + * reach @from's pipeline through @holder_back. */ enum dcp_fabric_follow dcp_fabric_follow(const struct dcp_fabric_route *from, @@ -341,14 +344,13 @@ dcp_fabric_follow(const struct dcp_fabric_route *from, { const struct dcp_fabric_pipeline *pipeline; - if (!from || !to || from->tunnel || (holder && holder->tunnel)) + if (!from || !to) return DCP_FABRIC_FOLLOW_REFUSE; if (from == to) return DCP_FABRIC_FOLLOW_STAY; pipeline = to->pipeline; if (!pipeline->bound || pipeline->fixed_busy || pipeline->terminal || - !pipeline->services_ready || - (!policy->dual_stream && pipeline->presence != DCP_FABRIC_ABSENT)) + !pipeline->services_ready) return DCP_FABRIC_FOLLOW_REFUSE; if (!holder) return pipeline->owned || pipeline->fixed_recent ? DCP_FABRIC_FOLLOW_REFUSE : diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c index 533c834dfe8a2..891f3956cefd8 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-test.c +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -630,7 +630,10 @@ static void fabric_follow_test(struct kunit *test) &f.policy), DCP_FABRIC_FOLLOW_REFUSE); - /* Existing fixed-output busy/settling policy also applies to a move. */ + /* + * A live HDMI display keeps the hybrid. The settling sample taken at + * probe does not; a recent HDMI edge does (fabric_follow_recent_fixed). + */ f.pipeline[0].fixed_busy = true; KUNIT_EXPECT_EQ(test, dcp_fabric_follow(lg, hybrid, NULL, NULL, false, &f.policy), @@ -639,9 +642,16 @@ static void fabric_follow_test(struct kunit *test) f.pipeline[0].presence = DCP_FABRIC_SETTLING; KUNIT_EXPECT_EQ(test, dcp_fabric_follow(lg, hybrid, NULL, NULL, false, &f.policy), - DCP_FABRIC_FOLLOW_REFUSE); + DCP_FABRIC_FOLLOW_MOVE); f.pipeline[0].presence = DCP_FABRIC_ABSENT; + /* #39 on the M1 Pro: a Thunderbolt display alone follows to the hybrid. */ + lg->tunnel = true; + KUNIT_EXPECT_EQ(test, dcp_fabric_follow(lg, hybrid, NULL, NULL, false, + &f.policy), + DCP_FABRIC_FOLLOW_MOVE); + lg->tunnel = false; + /* Owned with no route to swap with (a retiring tunnel), unbound, not up. */ f.pipeline[0].owned = true; KUNIT_EXPECT_EQ(test, dcp_fabric_follow(lg, hybrid, NULL, NULL, false, @@ -681,18 +691,18 @@ static void fabric_follow_test(struct kunit *test) &f.route[1][0], &f.route[1][0], true, &f.policy), DCP_FABRIC_FOLLOW_REFUSE); - /* A direct route must never move into or swap a tunnel binding. */ + /* Tunnels follow and swap like direct routes, keeping their binding. */ f.route[1][0].tunnel = true; KUNIT_EXPECT_EQ(test, dcp_fabric_follow(&f.route[0][1], &f.route[0][0], &f.route[1][0], &f.route[1][1], true, &f.policy), - DCP_FABRIC_FOLLOW_REFUSE); + DCP_FABRIC_FOLLOW_SWAP); f.route[1][0].tunnel = false; f.route[0][1].tunnel = true; KUNIT_EXPECT_EQ(test, dcp_fabric_follow(&f.route[0][1], &f.route[0][0], - NULL, NULL, false, &f.policy), - DCP_FABRIC_FOLLOW_REFUSE); + &f.route[1][0], &f.route[1][1], true, &f.policy), + DCP_FABRIC_FOLLOW_SWAP); } static void fabric_follow_recent_fixed(struct kunit *test) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index baa71e9c0fe72..57fa386391b95 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -188,7 +188,7 @@ bool dcp_typec_dual_stream(void) /* * Do the Type-C routes of @dcp follow the CRTC a modeset pairs them with - * (see dcp_typec_follow_crtc())? This capability applies to direct routes + * (see dcp_typec_follow_crtc())? Direct routes and Thunderbolt tunnels, * on 12.3/13.5 only. */ bool dcp_typec_follows_crtc(struct apple_dcp *dcp) @@ -200,10 +200,10 @@ bool dcp_typec_follows_crtc(struct apple_dcp *dcp) } /* Is a routed port's possible_crtcs narrowed to the pipeline driving it? */ -static bool dcp_typec_narrows(struct apple_dcp *dcp, bool tunnel) +static bool dcp_typec_narrows(struct apple_dcp *dcp) { /* This tree narrows on dual-stream machines too. */ - return tunnel || !dcp_typec_follows_crtc(dcp); + return !dcp_typec_follows_crtc(dcp); } bool dcp_is_typec_only(struct platform_device *pdev) @@ -318,7 +318,7 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, * compositor's to make there. */ if (connector->port_encoder && dcp->crtc && - dcp_typec_narrows(dcp, xbar != route->xbar)) + dcp_typec_narrows(dcp)) connector->port_encoder->possible_crtcs = dcp_fabric_connector_mask(false, true, true, @@ -1965,8 +1965,10 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, EXPORT_SYMBOL_GPL(apple_dcp_tb_dp_tunnel); /* - * Direct DP-alt routes follow the CRTC selected by a modeset on 12.3/13.5. - * Tunnels keep their existing masks, binding lifetime and routing policy. + * Direct DP-alt routes and Thunderbolt tunnels follow the CRTC selected by + * a modeset on 12.3/13.5. A tunnel keeps its Thunderbolt binding: only the + * crossbar output of its DP IN changes pipeline, and the binding is revoked + * on the pipeline it leaves before it is installed on the one it joins. * Firmware 14.7 and native external processors keep their existing path. */ struct dcp_typec_follow { @@ -1990,6 +1992,11 @@ struct dcp_typec_follow_slot { u64 attachment_generation; bool was_active; bool restored; + /* a tunnel's DP IN and Thunderbolt binding, carried along */ + bool tunnel; + unsigned int dpin; + int (*set_active)(void *binding, bool active); + void *binding; }; struct dcp_typec_follow_context { @@ -2112,13 +2119,13 @@ static int dcp_typec_follow_decide(struct apple_dcp *dcp, struct drm_crtc *crtc, return -ENOENT; /* A dock's second stream has one pipeline, and the plan rules first. */ /* A port in SAFE keeps its owner until the mux set phase completes. */ - if (secondary || from->tunnel || port->safe_pending || + if (secondary || port->safe_pending || !dcp_typec_follows_crtc(from->dcp) || dcp_typec_keep_order()) return -EINVAL; holder = dcp->active_typec_route; if (holder) { other = holder->port; - if (holder != other->owner || holder->tunnel || other->safe_pending) + if (holder != other->owner || other->safe_pending) return -EINVAL; back = dcp_typec_port_route(other, from->dcp); off = dcp_typec_follow_holder_off(state, crtc, other->connector, @@ -2136,7 +2143,11 @@ static int dcp_typec_follow_decide(struct apple_dcp *dcp, struct drm_crtc *crtc, follow->action = dcp_fabric_follow(&from->core, to ? &to->core : NULL, holder ? &holder->core : NULL, back ? &back->core : NULL, off, &policy); - if (follow->action == DCP_FABRIC_FOLLOW_REFUSE) + /* A tunnel's DP IN must reach the pipeline it is given. */ + if (follow->action == DCP_FABRIC_FOLLOW_REFUSE || + (from->tunnel && !dcp_typec_tunnel_ctl(to, from->tunnel_dpin)) || + (follow->action == DCP_FABRIC_FOLLOW_SWAP && holder->tunnel && + !dcp_typec_tunnel_ctl(back, holder->tunnel_dpin))) return -EINVAL; follow->port = port; follow->from = from; @@ -2153,6 +2164,18 @@ static int dcp_follow_prepare(void *data, unsigned int index) slot->port = slot->from->port; slot->connector = slot->from->dcp->typec_connector; + slot->tunnel = slot->from->tunnel; + slot->dpin = slot->from->tunnel_dpin; + if (slot->tunnel) { + struct apple_dcp *dcp = slot->from->dcp; + + scoped_guard(mutex, &dcp->tb_lock) { + slot->set_active = dcp->tb_dpin_set_active; + slot->binding = dcp->tb_dpin_ctx; + } + if (!slot->set_active) + return -ESTALE; + } { struct drm_crtc *crtc = &slot->from->dcp->crtc->base; struct drm_crtc_state *old = drm_atomic_get_old_crtc_state(ctx->state, crtc); @@ -2188,6 +2211,7 @@ static int dcp_follow_validate(void *data, unsigned int index) static int dcp_follow_release(struct apple_dcp_typec_route *route) { struct apple_dcp *dcp = route->dcp; + bool tunnel = route->tunnel; int ret; /* Revoked sessions cannot touch the new route after a reconnect wait. */ @@ -2196,6 +2220,13 @@ static int dcp_follow_release(struct apple_dcp_typec_route *route) dcp_modes_begin_attachment(dcp); dcp->typec_connector = NULL; WRITE_ONCE(dcp->connector, dcp->fixed_connector); + /* The binding moves on: no callback may reach it through here. */ + if (tunnel) { + scoped_guard(mutex, &dcp->tb_lock) { + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + } + } scoped_guard(mutex, &dcp->hpd_mutex) WRITE_ONCE(dcp->typec_cable_connected, false); ret = dcp_typec_route_deactivate(route); @@ -2218,11 +2249,31 @@ static int dcp_follow_attach(void *data, unsigned int index, bool restore) struct dcp_typec_follow_slot *slot = &ctx->slots[index]; struct apple_dcp_typec_route *route = restore ? slot->from : slot->to; struct apple_dcp *dcp = route->dcp; + struct mux_control *ctl = route->xbar; int ret; - ret = dcp_typec_route_activate(route, route->xbar); - if (ret) + if (slot->tunnel) { + ctl = dcp_typec_tunnel_ctl(route, slot->dpin); + if (!ctl) + return -EOPNOTSUPP; + scoped_guard(mutex, &dcp->tb_lock) { + dcp->tb_dpin_set_active = slot->set_active; + dcp->tb_dpin_ctx = slot->binding; + } + route->tunnel_dpin = slot->dpin; + } + ret = dcp_typec_route_activate(route, ctl); + if (ret) { + if (slot->tunnel) { + scoped_guard(mutex, &dcp->tb_lock) { + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + } + } return ret; + } + if (slot->tunnel) + dcp_tunnel_prepare(route, ctl); slot->attachment_generation = dcp_modes_transfer_begin(dcp); slot->port->owner = route; slot->port->preferred_route = route; @@ -2364,7 +2415,7 @@ int dcp_typec_follow_check(struct apple_dcp *dcp, struct drm_crtc *crtc, retry_holder: if (!dcp_typec_follow_lock(false)) return -EBUSY; - if (dcp->active_typec_route && !dcp->active_typec_route->tunnel) { + if (dcp->active_typec_route) { struct apple_connector *holder = dcp->active_typec_route->port->connector; if (holder && !drm_atomic_get_new_connector_state(state, &holder->base)) { @@ -2420,8 +2471,7 @@ static bool dcp_typec_direct_commit(struct drm_crtc *crtc, continue; } route = READ_ONCE(source->active_typec_route); - if (route && !READ_ONCE(route->tunnel) && - dcp_typec_follows_crtc(source)) + if (route && dcp_typec_follows_crtc(source)) return true; } return false; @@ -2618,8 +2668,7 @@ void dcp_typec_port_set_connector(unsigned int idx, bool secondary, if (owner) { struct apple_dcp *dcp = owner->dcp; - if (dcp->crtc && connector->port_encoder && - dcp_typec_narrows(dcp, owner->tunnel)) + if (dcp->crtc && connector->port_encoder && dcp_typec_narrows(dcp)) connector->port_encoder->possible_crtcs = dcp_fabric_connector_mask(false, true, true, diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 51e26bd23bea0..cbe3901f6e438 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -2099,7 +2099,7 @@ bool dcp_crtc_needs_route_start(struct apple_dcp *dcp) { struct apple_dcp_typec_route *route = READ_ONCE(dcp->active_typec_route); - return dcp_typec_follows_crtc(dcp) && route && !READ_ONCE(route->tunnel) && + return dcp_typec_follows_crtc(dcp) && route && READ_ONCE(dcp->typec_crtc_off) && READ_ONCE(dcp->typec_follow_start) && READ_ONCE(dcp->typec_follow_gen) == READ_ONCE(dcp->typec_generation); } From 465f0c86bfe5eacacff25b92106880336f8973d0 Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Wed, 7 Oct 2026 06:13:06 -0600 Subject: [PATCH 1644/2680] drm/apple: route an arriving Type-C display where a compositor pairs it Without dual-stream docks, a display arriving on a Type-C port takes a free Type-C-only pipeline before the hybrid, which is kept for an HDMI display that may arrive later. Where routes follow their CRTC, the compositor gives the new display the lowest free CRTC, the hybrid's while HDMI is empty, and the first modeset moves the route there: one more link bring-up, and a dark display for a second or two longer. Rank free pipelines by CRTC index alone on those machines, so that the route is where the compositor pairs it from the start. An HDMI display arriving later takes the hybrid back (next commit) or waits, as it already did whenever both Type-C pipelines were taken. Tested on the j314s: an LG UltraFine 5K replugged with HDMI empty lands on dcpext0 directly, where aquamarine pairs it, with no move. Signed-off-by: Christian Bartels (cherry picked from commit b3a79e04f7699f44be2d88bf7845e4d484adbcf6) --- drivers/gpu/drm/apple/dcp-fabric-core.c | 9 ++++++++- drivers/gpu/drm/apple/dcp-fabric-core.h | 2 ++ drivers/gpu/drm/apple/dcp-fabric-test.c | 24 ++++++++++++++++++++++++ drivers/gpu/drm/apple/dcp-fabric.c | 23 +++++++++++++++++++---- 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.c b/drivers/gpu/drm/apple/dcp-fabric-core.c index ba8ea0d7320db..f9862a0594a77 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.c +++ b/drivers/gpu/drm/apple/dcp-fabric-core.c @@ -121,8 +121,15 @@ unsigned int dcp_fabric_score(const struct dcp_fabric_pipeline *pipeline, { if (!pipeline->bound) return UINT_MAX - 1; + /* + * Without dual-stream docks a hybrid comes last, kept for an HDMI + * display arriving later. Not where routes follow their CRTC: a + * compositor pairs a new display with the lowest free CRTC, and its + * route would only follow it there. + */ return pipeline->crtc_index + - (!policy->dual_stream && pipeline->has_fixed ? 100 : 0); + (!policy->dual_stream && !policy->follow && pipeline->has_fixed ? + 100 : 0); } EXPORT_SYMBOL_GPL(dcp_fabric_score); diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.h b/drivers/gpu/drm/apple/dcp-fabric-core.h index 0792b25a6b388..d78cb361a37c5 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.h +++ b/drivers/gpu/drm/apple/dcp-fabric-core.h @@ -59,6 +59,8 @@ struct dcp_fabric_port { struct dcp_fabric_policy { bool dual_stream; + /* routes follow the CRTC a modeset pairs them with */ + bool follow; }; enum dcp_fabric_wiring { diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c index 891f3956cefd8..d3e041750782a 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-test.c +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -705,6 +705,29 @@ static void fabric_follow_test(struct kunit *test) DCP_FABRIC_FOLLOW_SWAP); } +static void fabric_follow_arrival_test(struct kunit *test) +{ + struct fabric_fixture f; + + /* Where routes follow their CRTC, streams arrive on the lowest CRTC. */ + fabric_init(&f, false); + KUNIT_EXPECT_PTR_EQ(test, dcp_fabric_free_route(&f.port[0], &f.policy), + &f.route[0][1]); + f.policy.follow = true; + KUNIT_EXPECT_PTR_EQ(test, dcp_fabric_free_route(&f.port[0], &f.policy), + &f.route[0][0]); + /* still not on a hybrid whose HDMI display is live */ + f.pipeline[0].fixed_busy = true; + KUNIT_EXPECT_PTR_EQ(test, dcp_fabric_free_route(&f.port[0], &f.policy), + &f.route[0][1]); + /* dual-stream ranking is unchanged */ + fabric_init(&f, true); + f.policy.follow = true; + KUNIT_EXPECT_EQ(test, dcp_fabric_score(&f.pipeline[0], &f.policy), + dcp_fabric_score(&f.pipeline[0], &(struct dcp_fabric_policy){ + .dual_stream = true })); +} + static void fabric_follow_recent_fixed(struct kunit *test) { struct fabric_fixture f; @@ -1800,6 +1823,7 @@ static struct kunit_case fabric_tests[] = { KUNIT_CASE(fabric_deactivate_failure_test), KUNIT_CASE(fabric_unbound_and_mask_test), KUNIT_CASE(fabric_follow_test), + KUNIT_CASE(fabric_follow_arrival_test), KUNIT_CASE(fabric_follow_recent_fixed), KUNIT_CASE(fabric_follow_effect_swap), KUNIT_CASE(fabric_follow_effect_oom), diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 57fa386391b95..e23acc21f4ccf 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -199,6 +199,17 @@ bool dcp_typec_follows_crtc(struct apple_dcp *dcp) dcp->fw_compat == DCP_FIRMWARE_V_13_5); } +/* Do all pipelines that can drive @port let its routes follow their CRTC? */ +static bool dcp_typec_port_follows(struct apple_dcp_typec_port *port) +{ + struct apple_dcp_typec_route *route; + + list_for_each_entry(route, &port->routes, port_link) + if (!dcp_typec_follows_crtc(route->dcp)) + return false; + return !list_empty(&port->routes); +} + /* Is a routed port's possible_crtcs narrowed to the pipeline driving it? */ static bool dcp_typec_narrows(struct apple_dcp *dcp) { @@ -910,13 +921,16 @@ static void dcp_typec_port_attach(struct apple_dcp_typec_port *port) * free, as a compositor keeps a reconnected connector's CRTC, otherwise the * best-ranked free one. Without dual-stream docks a free Type-C-only * pipeline comes first even when the port last had the hybrid, which an HDMI - * display needs. + * display needs, unless the route would only follow its CRTC to the hybrid + * (see dcp_fabric_score()). */ static struct apple_dcp_typec_route * dcp_typec_free_route(struct apple_dcp_typec_port *port) { - struct dcp_fabric_policy policy = { .dual_stream = - dcp_typec_dual_stream() }; + struct dcp_fabric_policy policy = { + .dual_stream = dcp_typec_dual_stream(), + .follow = dcp_typec_port_follows(port), + }; dcp_fabric_snapshot_port(port, false); return dcp_fabric_real_route(dcp_fabric_free_route(&port->core, @@ -1738,7 +1752,8 @@ static int dcp_tb_candidate(void *data) { struct dcp_fabric_policy policy = { - .dual_stream = dcp_typec_dual_stream() + .dual_stream = dcp_typec_dual_stream(), + .follow = dcp_typec_port_follows(port), }; struct dcp_fabric_route *chosen; bool connector_present = dpin ? !!port->secondary_connector : !!port->connector; From 9fc49bb9b36b15206aa6697b92ef66b77a054b4a Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Wed, 7 Oct 2026 06:13:10 -0600 Subject: [PATCH 1645/2680] drm/apple: hand the hybrid back to an HDMI display when a Type-C display can move MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without dual-stream docks, an HDMI display plugged in while its hybrid pipeline drives a Type-C display waits until that display goes. On the M1 Pro with an LG UltraFine 5K alone, the LG holds dcpext0, as the compositor pairs it with dcpext0's CRTC; a monitor plugged into HDMI afterwards then stays dark while dcpext1 sits free. Where routes follow their CRTC and the Type-C display's port can reach another free pipeline, move it out of the way once HDMI has stayed up for 500 ms: report the Type-C display gone, connect the HDMI display on the hybrid, and bring the Type-C display back on the other pipeline. The compositor drops the CRTC it had given the Type-C display, finds the HDMI display there, and pairs the returning display with the other CRTC. The HDMI display goes first, as the returning display would otherwise be paired with the hybrid's CRTC again. A Thunderbolt tunnel keeps its binding, as when it follows a CRTC. Without a free pipeline, and on dual-stream machines, the HDMI display waits as before. Tested on the j314s with an LG UltraFine 5K alone on dcpext0 and a Dell U2715H plugged into HDMI: the LG blinks for about a second and returns at 5120x2880 on dcpext1, and the Dell lights on dcpext0. DPMS afterwards keeps the pairing. [ryan: adapted to 7.2.10 — the hand-back refuses a port with safe_pending (a port in SAFE keeps its owner); the Thunderbolt binding generation (tunnel_generation, 170-usb API) is not in this tree yet, so a tunnel carries only its set_active callback and context] Signed-off-by: Christian Bartels (cherry picked from commit a6285508f39dcb4ce5e6ecaf732b8aef099992ea) --- drivers/gpu/drm/apple/dcp-fabric.c | 133 +++++++++++++++++++++++++++-- 1 file changed, 125 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index e23acc21f4ccf..4af0d6f0390ec 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -1152,13 +1152,17 @@ dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, unsigned int d dcp_fabric_run_rebalance(&dcp_rebalance_ops, &ctx); return ctx.planned; } + +static bool dcp_typec_hdmi_reclaim(struct apple_dcp *dcp); + /* * A display is plugged into the HDMI port while its pipeline, the hybrid, * drives a Type-C port. Without dual-stream docks nothing moves for it: a * compositor keeps its connector-to-CRTC pairing and could miss a moved - * display's brief unplug, and a Thunderbolt tunnel never moves. The HDMI - * display waits and is handed the hybrid when the Type-C display lets it go - * (see dcp_typec_route_deactivate()). + * display's brief unplug. The HDMI display waits and is handed the hybrid + * when the Type-C display lets it go (see dcp_typec_route_deactivate()), + * unless the Type-C display can be moved out of its way (see + * dcp_typec_hdmi_reclaim()). */ static void dcp_typec_hdmi_waits(struct apple_dcp *dcp) { @@ -1166,6 +1170,8 @@ static void dcp_typec_hdmi_waits(struct apple_dcp *dcp) lockdep_assert_held(&dcp_typec_fabric_lock); + if (owner && dcp_typec_hdmi_reclaim(dcp)) + return; if (owner) dev_info(dcp->dev, "HDMI display waits: its pipeline drives the %s on %pOF\n", owner->tunnel ? "Thunderbolt display" : "display", @@ -2258,11 +2264,10 @@ static int dcp_follow_detach(void *data, unsigned int index, bool destination) return dcp_follow_release(destination ? slot->to : slot->from); } -static int dcp_follow_attach(void *data, unsigned int index, bool restore) +/* Route @slot's display through @route, a tunnel with its binding. */ +static int dcp_follow_activate(struct apple_dcp_typec_route *route, + struct dcp_typec_follow_slot *slot) { - struct dcp_typec_follow_context *ctx = data; - struct dcp_typec_follow_slot *slot = &ctx->slots[index]; - struct apple_dcp_typec_route *route = restore ? slot->from : slot->to; struct apple_dcp *dcp = route->dcp; struct mux_control *ctl = route->xbar; int ret; @@ -2289,9 +2294,23 @@ static int dcp_follow_attach(void *data, unsigned int index, bool restore) } if (slot->tunnel) dcp_tunnel_prepare(route, ctl); - slot->attachment_generation = dcp_modes_transfer_begin(dcp); slot->port->owner = route; slot->port->preferred_route = route; +return 0; +} + +static int dcp_follow_attach(void *data, unsigned int index, bool restore) +{ + struct dcp_typec_follow_context *ctx = data; + struct dcp_typec_follow_slot *slot = &ctx->slots[index]; + struct apple_dcp_typec_route *route = restore ? slot->from : slot->to; + struct apple_dcp *dcp = route->dcp; + int ret; + + ret = dcp_follow_activate(route, slot); + if (ret) + return ret; + slot->attachment_generation = dcp_modes_transfer_begin(dcp); scoped_guard(mutex, &dcp->hpd_mutex) { WRITE_ONCE(dcp->typec_cable_connected, true); dcp->typec_generation++; @@ -3109,6 +3128,104 @@ static void dcp_hdmi_connect_fixed(void *ctx) dcp_hdmi_connect_failed(dcp, ret); } +/* + * An HDMI display has arrived, and stayed, while its hybrid drives a + * Type-C display that another free pipeline can drive too. Where routes + * follow their CRTC, hand the hybrid back: the Type-C display goes, the + * HDMI display connects, and the Type-C display returns on the other + * pipeline. The compositor sees an unplug and a replug, drops the CRTC it + * gave the Type-C display, and finds the HDMI display there before the + * Type-C display is back to be paired with the other CRTC. In the other + * order it would pair the returning display with the hybrid's CRTC again. + * Without such a pipeline, or on dual-stream machines, the HDMI display + * waits as before. + */ +#define DCP_RECLAIM_HDMI_MS 3000 + +static bool dcp_typec_hdmi_reclaim(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *owner = dcp->active_typec_route; + struct dcp_typec_follow_slot slot = {}; + struct apple_dcp_typec_route *route, *target = NULL; + struct apple_connector *connector, *hdmi; + struct apple_dcp_typec_port *port; + unsigned long timeout; + struct apple_dcp *to; + int ret; + + lockdep_assert_held(&dcp_typec_fabric_lock); + if (!owner || dcp_typec_dual_stream()) + return false; + port = owner->port; + /* A port in SAFE keeps its owner until the mux set phase completes. */ + if (port->owner != owner || port->safe_pending || + !dcp_typec_port_follows(port)) + return false; + list_for_each_entry(route, &port->routes, port_link) { + if (route == owner || !route->dcp->crtc || + !dcp_typec_route_available(route) || + (owner->tunnel && !dcp_typec_tunnel_ctl(route, owner->tunnel_dpin))) + continue; + target = route; + break; + } + if (!target) + return false; + + /* not a blink */ + msleep(500); + if (gpiod_get_value_cansleep(dcp->hdmi_hpd) <= 0) + return false; + + connector = dcp->typec_connector; + slot.port = port; + slot.connector = connector; + slot.tunnel = owner->tunnel; + slot.dpin = owner->tunnel_dpin; + if (slot.tunnel) { + scoped_guard(mutex, &dcp->tb_lock) { + slot.set_active = dcp->tb_dpin_set_active; + slot.binding = dcp->tb_dpin_ctx; + } + if (!slot.set_active) + return false; + } + to = target->dcp; + dev_info(dcp->dev, "HDMI display takes its pipeline back, %s on %pOF moves to %s\n", + connector ? connector->base.name : "the display", + port->connector_np, dev_name(to->dev)); + ret = dcp_follow_release(owner); + if (ret) + dev_warn(dcp->dev, "route not released cleanly: %d\n", ret); + if (connector) { + WRITE_ONCE(connector->connected, false); + apple_connector_set_pipeline(connector, NULL); + dcp_queue_hotplug(connector); + } + + dcp_hdmi_connect_fixed(dcp); + hdmi = dcp->fixed_connector; + timeout = jiffies + msecs_to_jiffies(DCP_RECLAIM_HDMI_MS); + while (hdmi && !READ_ONCE(hdmi->connected) && time_before(jiffies, timeout)) + msleep(20); + + ret = dcp_follow_activate(target, &slot); + if (ret) { + dev_err(to->dev, "could not take the display on %pOF: %d\n", + port->connector_np, ret); + return true; + } + /* A new display on its pipeline, connected now: no CRTC is on yet. */ + scoped_guard(mutex, &to->hpd_mutex) { + WRITE_ONCE(to->typec_cable_connected, true); + to->typec_generation++; + to->typec_follow_start = false; + WRITE_ONCE(to->typec_crtc_off, false); + } + dcp_dptx_connect_oob(to_platform_device(to->dev), 0); + return true; +} + /* * The bounded retry of a native external pipe, for its HDMI output: the * display is attached, but its link did not come up. Set the output up From 6a4167326a23fa572d22a7c9d9f981805a676475 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Thu, 8 Oct 2026 00:55:08 -0700 Subject: [PATCH 1646/2680] drm/apple: restore Type-C display after a failed HDMI handoff A destination crossbar failure must not leave the previously connected Type-C output without a route. Restore its source route, reconnect its link, and notify userspace if both move and restoration fail. Exercise the same effect executor for success, release failure and activation failures. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 06e937b6ad46ab99d4859e0d04383c2ee690a07a) --- drivers/gpu/drm/apple/dcp-fabric-effects.h | 31 +++++ drivers/gpu/drm/apple/dcp-fabric-test.c | 95 +++++++++++++++ drivers/gpu/drm/apple/dcp-fabric.c | 131 +++++++++++++++------ 3 files changed, 224 insertions(+), 33 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric-effects.h b/drivers/gpu/drm/apple/dcp-fabric-effects.h index 5c6db057b430b..d59b4888b3129 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-effects.h +++ b/drivers/gpu/drm/apple/dcp-fabric-effects.h @@ -137,6 +137,37 @@ struct dcp_fabric_deactivate_ops { void (*connect_fixed)(void *ctx); }; +struct dcp_fabric_reclaim_ops { + int (*release)(void *ctx); + void (*unplug)(void *ctx); + void (*connect_hdmi)(void *ctx); + int (*activate)(void *ctx, bool restore); + void (*publish)(void *ctx, bool restore); + void (*lost)(void *ctx); +}; + +/* A failed handoff must restore the old display or report its loss. */ +static inline int +dcp_fabric_reclaim_execute(const struct dcp_fabric_reclaim_ops *ops, void *ctx) +{ + int ret = ops->release(ctx); + + ops->unplug(ctx); + if (!ret) { + ops->connect_hdmi(ctx); + ret = ops->activate(ctx, false); + if (!ret) { + ops->publish(ctx, false); + return 0; + } + } + if (!ops->activate(ctx, true)) + ops->publish(ctx, true); + else + ops->lost(ctx); + return ret; +} + /* True discards the cached remainder and re-enters with a new snapshot. */ static inline bool dcp_fabric_run_deactivate(const struct dcp_fabric_deactivate_ops *ops, void *ctx) diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c index d3e041750782a..16fada29559a5 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-test.c +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -1810,7 +1810,102 @@ static void fabric_rebind_presence_test(struct kunit *test) } } +struct reclaim_fixture { + int release_error, move_error, restore_error; + unsigned int events[8], count; + int owner; + bool reported_lost; +}; + +static int reclaim_release(void *data) +{ + struct reclaim_fixture *f = data; + + f->events[f->count++] = 1; + f->owner = -1; + return f->release_error; +} + +static void reclaim_unplug(void *data) +{ + struct reclaim_fixture *f = data; + + f->events[f->count++] = 2; +} + +static void reclaim_hdmi(void *data) +{ + struct reclaim_fixture *f = data; + + f->events[f->count++] = 3; +} + +static int reclaim_activate(void *data, bool restore) +{ + struct reclaim_fixture *f = data; + int ret = restore ? f->restore_error : f->move_error; + + f->events[f->count++] = restore ? 5 : 4; + if (!ret) + f->owner = restore ? 0 : 1; + return ret; +} + +static void reclaim_publish(void *data, bool restore) +{ + struct reclaim_fixture *f = data; + + f->events[f->count++] = restore ? 7 : 6; +} + +static void reclaim_lost(void *data) +{ + struct reclaim_fixture *f = data; + + f->events[f->count++] = 8; + f->reported_lost = true; +} + +static const struct dcp_fabric_reclaim_ops reclaim_ops = { + .release = reclaim_release, + .unplug = reclaim_unplug, + .connect_hdmi = reclaim_hdmi, + .activate = reclaim_activate, + .publish = reclaim_publish, + .lost = reclaim_lost, +}; + +static void fabric_reclaim_transaction_test(struct kunit *test) +{ + struct reclaim_fixture success = {}, move = { .move_error = -EIO }; + struct reclaim_fixture release = { .release_error = -EIO }; + struct reclaim_fixture lost = { .move_error = -EIO, .restore_error = -EIO }; + unsigned int i; + const unsigned int expected[] = { 1, 2, 3, 4, 6 }; + + KUNIT_EXPECT_EQ(test, dcp_fabric_reclaim_execute(&reclaim_ops, &success), 0); + KUNIT_ASSERT_EQ(test, success.count, (unsigned int)ARRAY_SIZE(expected)); + for (i = 0; i < ARRAY_SIZE(expected); i++) + KUNIT_EXPECT_EQ(test, success.events[i], expected[i]); + KUNIT_EXPECT_EQ(test, success.owner, 1); + KUNIT_EXPECT_EQ(test, dcp_fabric_reclaim_execute(&reclaim_ops, &move), -EIO); + KUNIT_EXPECT_EQ(test, move.owner, 0); + KUNIT_EXPECT_EQ(test, move.events[4], 5U); + KUNIT_EXPECT_EQ(test, move.events[5], 7U); + KUNIT_EXPECT_FALSE(test, move.reported_lost); + KUNIT_EXPECT_EQ(test, dcp_fabric_reclaim_execute(&reclaim_ops, &release), -EIO); + KUNIT_EXPECT_EQ(test, release.owner, 0); + KUNIT_ASSERT_EQ(test, release.count, 4U); + KUNIT_EXPECT_EQ(test, release.events[2], 5U); + KUNIT_EXPECT_EQ(test, release.events[3], 7U); + KUNIT_EXPECT_EQ(test, dcp_fabric_reclaim_execute(&reclaim_ops, &lost), -EIO); + KUNIT_EXPECT_EQ(test, lost.owner, -1); + KUNIT_EXPECT_TRUE(test, lost.reported_lost); + KUNIT_EXPECT_EQ(test, lost.events[5], 8U); +} + static struct kunit_case fabric_tests[] = { + KUNIT_CASE(fabric_reclaim_transaction_test), KUNIT_CASE_PARAM(fabric_shared_wiring_test, fabric_wiring_gen_params), KUNIT_CASE(fabric_rebind_presence_test), KUNIT_CASE(fabric_nonhybrid_hdmi_resume_test), diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 4af0d6f0390ec..4db4812b6752a 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -2296,7 +2296,7 @@ static int dcp_follow_activate(struct apple_dcp_typec_route *route, dcp_tunnel_prepare(route, ctl); slot->port->owner = route; slot->port->preferred_route = route; -return 0; + return 0; } static int dcp_follow_attach(void *data, unsigned int index, bool restore) @@ -3142,16 +3142,102 @@ static void dcp_hdmi_connect_fixed(void *ctx) */ #define DCP_RECLAIM_HDMI_MS 3000 +struct dcp_hdmi_reclaim_context { + struct apple_dcp *from; + struct apple_dcp_typec_route *owner; + struct apple_dcp_typec_route *target; + struct dcp_typec_follow_slot slot; +}; + +static int dcp_reclaim_release(void *data) +{ + struct dcp_hdmi_reclaim_context *ctx = data; + + return dcp_follow_release(ctx->owner); +} + +static void dcp_reclaim_unplug(void *data) +{ + struct dcp_hdmi_reclaim_context *ctx = data; + struct apple_connector *connector = ctx->slot.connector; + + if (connector) { + WRITE_ONCE(connector->connected, false); + apple_connector_set_pipeline(connector, NULL); + dcp_queue_hotplug(connector); + } +} + +static void dcp_reclaim_connect_hdmi(void *data) +{ + struct dcp_hdmi_reclaim_context *ctx = data; + struct apple_dcp *dcp = ctx->from; + struct apple_connector *hdmi = dcp->fixed_connector; + unsigned long timeout; + + /* Park revoked the old attachment before this fixed output starts. */ + WRITE_ONCE(dcp->typec_follow_retiring, false); + dcp_hdmi_connect_fixed(dcp); + timeout = jiffies + msecs_to_jiffies(DCP_RECLAIM_HDMI_MS); + while (hdmi && !READ_ONCE(hdmi->connected) && time_before(jiffies, timeout)) + msleep(20); +} + +static int dcp_reclaim_activate(void *data, bool restore) +{ + struct dcp_hdmi_reclaim_context *ctx = data; + struct apple_dcp_typec_route *route = restore ? ctx->owner : ctx->target; + int ret = dcp_follow_activate(route, &ctx->slot); + + if (ret) + dev_err(route->dcp->dev, "could not %s display route on %pOF: %d\n", + restore ? "restore" : "move", ctx->slot.port->connector_np, ret); + return ret; +} + +static void dcp_reclaim_publish(void *data, bool restore) +{ + struct dcp_hdmi_reclaim_context *ctx = data; + struct apple_dcp *to = (restore ? ctx->owner : ctx->target)->dcp; + + /* Connected on a fresh attachment; no CRTC is on yet. */ + scoped_guard(mutex, &to->hpd_mutex) { + WRITE_ONCE(to->typec_cable_connected, true); + to->typec_generation++; + to->typec_follow_start = false; + WRITE_ONCE(to->typec_crtc_off, false); + } + WRITE_ONCE(ctx->owner->dcp->typec_follow_retiring, false); + WRITE_ONCE(ctx->target->dcp->typec_follow_retiring, false); + dcp_dptx_connect_oob(to_platform_device(to->dev), 0); +} + +static void dcp_reclaim_lost(void *data) +{ + struct dcp_hdmi_reclaim_context *ctx = data; + + if (ctx->slot.connector) + dcp_route_failure_notify(ctx->slot.connector); +} + +static const struct dcp_fabric_reclaim_ops dcp_reclaim_ops = { + .release = dcp_reclaim_release, + .unplug = dcp_reclaim_unplug, + .connect_hdmi = dcp_reclaim_connect_hdmi, + .activate = dcp_reclaim_activate, + .publish = dcp_reclaim_publish, + .lost = dcp_reclaim_lost, +}; + static bool dcp_typec_hdmi_reclaim(struct apple_dcp *dcp) { struct apple_dcp_typec_route *owner = dcp->active_typec_route; struct dcp_typec_follow_slot slot = {}; struct apple_dcp_typec_route *route, *target = NULL; - struct apple_connector *connector, *hdmi; + struct apple_connector *connector; struct apple_dcp_typec_port *port; - unsigned long timeout; struct apple_dcp *to; - int ret; + struct dcp_hdmi_reclaim_context ctx; lockdep_assert_held(&dcp_typec_fabric_lock); if (!owner || dcp_typec_dual_stream()) @@ -3194,35 +3280,14 @@ static bool dcp_typec_hdmi_reclaim(struct apple_dcp *dcp) dev_info(dcp->dev, "HDMI display takes its pipeline back, %s on %pOF moves to %s\n", connector ? connector->base.name : "the display", port->connector_np, dev_name(to->dev)); - ret = dcp_follow_release(owner); - if (ret) - dev_warn(dcp->dev, "route not released cleanly: %d\n", ret); - if (connector) { - WRITE_ONCE(connector->connected, false); - apple_connector_set_pipeline(connector, NULL); - dcp_queue_hotplug(connector); - } - - dcp_hdmi_connect_fixed(dcp); - hdmi = dcp->fixed_connector; - timeout = jiffies + msecs_to_jiffies(DCP_RECLAIM_HDMI_MS); - while (hdmi && !READ_ONCE(hdmi->connected) && time_before(jiffies, timeout)) - msleep(20); - - ret = dcp_follow_activate(target, &slot); - if (ret) { - dev_err(to->dev, "could not take the display on %pOF: %d\n", - port->connector_np, ret); - return true; - } - /* A new display on its pipeline, connected now: no CRTC is on yet. */ - scoped_guard(mutex, &to->hpd_mutex) { - WRITE_ONCE(to->typec_cable_connected, true); - to->typec_generation++; - to->typec_follow_start = false; - WRITE_ONCE(to->typec_crtc_off, false); - } - dcp_dptx_connect_oob(to_platform_device(to->dev), 0); + ctx = (struct dcp_hdmi_reclaim_context) { + .from = dcp, .owner = owner, .target = target, .slot = slot, + }; + WRITE_ONCE(dcp->typec_follow_retiring, true); + WRITE_ONCE(to->typec_follow_retiring, true); + dcp_fabric_reclaim_execute(&dcp_reclaim_ops, &ctx); + WRITE_ONCE(dcp->typec_follow_retiring, false); + WRITE_ONCE(to->typec_follow_retiring, false); return true; } From bee1f4b4422baa30423aee1fc02a3b0d3765505c Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Thu, 8 Oct 2026 01:58:07 -0700 Subject: [PATCH 1647/2680] drm: apple: return IOMFB startup failures before optional audio Keep the required display endpoint error when HDMI audio is enabled. Optional audio startup cannot admit a display controller whose IOMFB endpoint did not start. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit a200636bce5bcfed9960270a66924c2e40817a0e) --- drivers/gpu/drm/apple/dcp.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index d4b6f8cd2c643..66dec813a5aa8 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2057,8 +2057,10 @@ int dcp_start(struct platform_device *pdev) dev_warn(dcp->dev, "OS firmware incompatible with dptxport EP\n"); } ret = iomfb_start_rtkit(dcp); - if (ret) + if (ret) { dev_err(dcp->dev, "Failed to start IOMFB endpoint: %d\n", ret); + return ret; + } #if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) if (hdmi_audio) { From 4f4e87ce6b56c8ac690b62c284d9faa776538bd0 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Thu, 8 Oct 2026 02:33:43 -0700 Subject: [PATCH 1648/2680] drm/apple: recover a disconnected direct DP sink on delivered IRQ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An IRQ_HPD with physical HPD high previously did nothing when firmware had already disconnected the DRM connector. Confirm synthetic HPD deassertion and RemotePort release before reconnecting that sink once, without changing cable state or adding retries. Refuse recovery while display work is disabled, the CRTC is off, or its route session has been revoked. Keep connected-sink retraining, native external IRQ handling and ordinary parked-route startup unchanged. [ryan: adapted to 7.2.10 — this tree has no fabric session (dcp_session_locked/tb_generation/tb_retiring, part of the 170-usb Thunderbolt binding API): dcp_dptx_recover_irq() revokes older reconnect waits by a typec_generation bump, refuses recovery (-EAGAIN) unless typec_work_enabled, the cable is routed, no follow handoff is retiring and the DPTX service is available, then reconnects once through dcp_dptx_connect_raw(); H17P external pipes are excluded; the per-step session rechecks inside the connect path are not ported] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 65361daa5f7816e28cd3406d35882c975db86bf6) --- drivers/gpu/drm/apple/dcp-fabric.c | 13 +++++-- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 55 ++++++++++++++++++++++++++++ drivers/gpu/drm/apple/dcp.h | 1 + 4 files changed, 67 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 4db4812b6752a..f07ecbe04b177 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -1440,11 +1440,18 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, /* * A T6030 external processor services the request itself, and * reads the display again if it changed; elsewhere the active - * mode is applied again. + * mode is applied again. H17P always applies it again. */ - if (dcp->typec_connector && dcp->external_native) + if (dcp->typec_connector && dcp->fw_compat == DCP_FIRMWARE_H17P) + dcp_retrain_oob(dcp->typec_connector); + else if (dcp->typec_connector && dcp->external_native) dcp_external_sink_irq(dcp); - else if (dcp->typec_connector) + else if (dcp->typec_connector && + !READ_ONCE(dcp->typec_connector->connected)) { + ret = dcp_dptx_recover_irq(dcp); + if (ret) + return ret; + } else if (dcp->typec_connector) dcp_retrain_oob(dcp->typec_connector); } port->hpd = hpd; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index f6ac5168ecd71..5a6fa2c934556 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -510,6 +510,7 @@ struct apple_dcp { /* hpd_mutex: what DRM reports for the external display, and when */ struct dcp_announce announce; struct delayed_work announce_wq; + bool typec_work_enabled; /* hpd_mutex; IRQ recovery admission */ #if IS_ENABLED(CONFIG_DRM_APPLE_KUNIT_TEST) /* Lets a test stop work where it races, see dcp_test_point(). */ void (*test_point)(struct apple_dcp *dcp, enum dcp_test_point point); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 66dec813a5aa8..29437049c63a8 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1913,6 +1913,58 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) return dcp_dptx_disconnect(dcp, port); } +/* + * A delivered sink IRQ can follow a firmware unplug with cable HPD still high. + * This tree has no fabric session: the route is revoked by a typec_generation + * bump, and recovery is refused unless the Type-C work is enabled, the cable + * is routed and no follow handoff is retiring the attachment. + */ +int dcp_dptx_recover_irq(struct apple_dcp *dcp) +{ + int ret; + + mutex_lock(&dcp->hpd_mutex); + if (dcp->external_native || dcp_is_external_h17p(dcp) || + !dcp_is_typec_output(dcp) || dcp_is_usb4_output(dcp) || + !dcp->typec_connector || + READ_ONCE(dcp->typec_connector->connected) || + READ_ONCE(dcp->typec_crtc_off)) { + ret = 0; + goto out_unlock; + } + if (!dcp->typec_work_enabled || READ_ONCE(dcp->crashed) || + !READ_ONCE(dcp->typec_cable_connected) || + READ_ONCE(dcp->typec_follow_retiring) || + !dcp->dptxport[0].enabled || + !dptx_service_available(&dcp->dptxport[0])) { + ret = -EAGAIN; + goto out_unlock; + } + + /* Revoke older reconnect waits without changing physical cable state. */ + dcp->typec_generation++; + cancel_delayed_work(&dcp->typec_reconnect_wq); + cancel_delayed_work(&dcp->placeholder_edid_wq); + if (dcp->avep) + av_service_disconnect(dcp); + ret = dptxport_set_hpd(dcp->dptxport[0].service, false); + if (ret) + goto out_unlock; + /* A failed earlier connect may also have left an unconfirmed release. */ + ret = dptxport_release_display(dcp->dptxport[0].service); + if (ret) + goto out_unlock; + dcp->dptxport[0].connected = false; + mutex_unlock(&dcp->hpd_mutex); + + /* Another delivered IRQ must confirm release before a failed attempt retries. */ + return dcp_dptx_connect_raw(dcp, 0, NULL); + +out_unlock: + mutex_unlock(&dcp->hpd_mutex); + return ret; +} + /* * Release the display link of a Type-C output while its display stays * attached, as a CRTC power-off does (see dcp_poweroff()): the firmware's @@ -2969,6 +3021,7 @@ static void dcp_release_fixed_route(void *data) VISIBLE_IF_KUNIT void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) { scoped_guard(mutex, &dcp->hpd_mutex) { + dcp->typec_work_enabled = false; if (release_cable) WRITE_ONCE(dcp->typec_cable_connected, false); dcp->typec_generation++; @@ -2997,6 +3050,8 @@ VISIBLE_IF_KUNIT void dcp_enable_typec_work(struct apple_dcp *dcp) enable_delayed_work(&dcp->hdmi_settle_wq); enable_delayed_work(&dcp->hdmi_recheck_wq); dcp_hdmi_enable(dcp); + scoped_guard(mutex, &dcp->hpd_mutex) + dcp->typec_work_enabled = true; /* A cable can be routed before the DRM component binds. */ if (READ_ONCE(dcp->typec_cable_connected)) mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 9380cd5e0f638..0cb225cb4eecc 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -87,6 +87,7 @@ void dcp_external_retry(struct apple_dcp *dcp, const char *why, int error, unsigned int base_ms); void dcp_external_retry_work(struct work_struct *work); void dcp_external_sink_irq(struct apple_dcp *dcp); +int dcp_dptx_recover_irq(struct apple_dcp *dcp); void dcp_dptx_park(struct apple_dcp *dcp); int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); bool dcp_hold_resume_event(struct platform_device *pdev); From e4449e9a3e4dcb920d34917a2a1e77bb0fa84b5b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 9 Oct 2026 08:29:56 -0700 Subject: [PATCH 1649/2680] drm/apple: handle backlight factor notifications on 14.x DCP The J613 firmware sends D208 after a colour matrix changes. Accept its four-byte backlight correction factor and empty reply, keeping the display RPC stream active. Record the last factor and notification count in panel status. Other callback layouts remain unchanged. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 08bba63dfb41ed6e1ee0ebc9beb0e477c31cfdd2) --- drivers/gpu/drm/apple/iomfb_v14_7.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 579cb6b8ba66e..297f13699944e 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -182,6 +182,9 @@ struct apple_dcp_v14 { /* The firmware refused a matrix: none is sent until reboot. */ bool ctm_disabled; u64 ctm_calls; + /* The last firmware backlight correction notification; not a nits target. */ + u32 backlight_factor; + u64 backlight_factor_updates; u32 ctm_status, ctm_get_status; u64 ctm_readback[9]; struct work_struct idle_work; @@ -871,6 +874,16 @@ static int dcp_v14_callback(void *cookie, u32 tag, const void *input, u32 in_siz } #define SHAPE(i, o) (in_size == (i) && out_size == (o)) + /* + * Backlight correction notification: one signed 32-bit factor, no + * reply payload. The firmware applies it when the colour matrix + * changes; it is not a host request to set the panel brightness. + */ + if (tag == D(208) && SHAPE(4, 0)) { + v14->backlight_factor = get_unaligned_le32(in); + v14->backlight_factor_updates++; + return 0; + } /* get_time */ if (tag == D(209) && SHAPE(0, 8)) { put_unaligned_le64(ktime_to_ms(ktime_get_real()), out); @@ -2222,6 +2235,8 @@ static int dcp_v14_status_show(struct seq_file *m, void *unused) seq_printf(m, "panel %ux%u\nboot_fb %ux%u stride %u\nclock %llu\n", v14->panel_width, v14->panel_height, v14->fb_width, v14->fb_height, v14->stride, v14->clock_rate); + seq_printf(m, "backlight factor %#x updates %llu\n", + v14->backlight_factor, v14->backlight_factor_updates); seq_printf(m, "ctm valid %d disabled %d calls %llu setter %#x getter %#x\n", v14->ctm_valid, v14->ctm_disabled, v14->ctm_calls, v14->ctm_status, v14->ctm_get_status); From a851c2343061c5c9cf086edb097a7fad7f3092d5 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 9 Oct 2026 08:59:53 -0700 Subject: [PATCH 1650/2680] drm/apple: support the J615 internal panel with current14 DCP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Select the existing T8122 14.x DCP image on the 15-inch M3 Air, retaining the firmware UUID, loader handoff and PMP checks. Validate its own 2880x1864 panel against the boot framebuffer, including hidden or visible notch rows. Add nominal panel size and brightness limits to its device tree. The J613 25G83 profile remains separate. [ryan: adapted to 7.2.10 — the t8122-j615.dts panel node is dropped here and handed over to 000-devicetree] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 7d5a612eceaa5299131c12f79873db13d969daa9) --- drivers/gpu/drm/apple/iomfb_v14_7.c | 16 +++++ tools/asahi/dcp14-panel-tests.py | 101 ++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100755 tools/asahi/dcp14-panel-tests.py diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 297f13699944e..ca8b6f18d6bb7 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -139,9 +139,25 @@ static const struct dcp_v14_board dcp_v14_board_j613 = { .ctm_get = A(420), }; +/* J615 (MacBook Air 15", M3): the same DCP image, a 2880x1864 60 Hz panel. */ +static const struct dcp_v14_board dcp_v14_board_j615 = { + .name = "J615", + .debugfs = "dcp-j615", + .dcp_compatible = "apple,t8122-dcp", + .machine = "apple,j615", + .firmware_uuid = DCP_V14_J613_FIRMWARE_UUID, + .handoff = "apple,t8122-handoff", + .panel_width = 2880, + .panel_height = 1864, + .promotion = false, + .ctm_set = A(421), + .ctm_get = A(420), +}; + static const struct dcp_v14_board *const dcp_v14_boards[] = { &dcp_v14_board_t6030, &dcp_v14_board_j613, + &dcp_v14_board_j615, }; struct dcp_v14_property { diff --git a/tools/asahi/dcp14-panel-tests.py b/tools/asahi/dcp14-panel-tests.py new file mode 100755 index 0000000000000..4f504a79e5f44 --- /dev/null +++ b/tools/asahi/dcp14-panel-tests.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Host checks for current14 internal-panel admission and boot geometry.""" +from pathlib import Path +import argparse +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--old-source', type=Path) +args = parser.parse_args() +source = (root/'drivers/gpu/drm/apple/iomfb_v14_7.c').read_text() +cases = [('current', source)] +if args.old_source: + cases.append(('old', args.old_source.read_text())) +with tempfile.TemporaryDirectory(prefix='dcp14-panels-') as directory: + out = Path(directory) + prefix=r'''#include + #include + #include + #include + #include + #include + #include + typedef uint32_t u32; + #define A(n) ((u32)'A'<<24|('0'+(n)/100)<<16|('0'+(n)/10%10)<<8|('0'+(n)%10)) + #define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0])) + #define ERR_PTR(e) ((void *)(intptr_t)(e)) + #define IS_ERR(p) ((uintptr_t)(p) >= (uintptr_t)-4095) + #define dev_err(...) ((void)0) + #define MAX_NOTCH_HEIGHT 100 + struct device_node {const char *compatible;}; + struct device {struct device_node *of_node;}; + static const char *machine; + static bool of_device_is_compatible(struct device_node *n,const char *c){return !strcmp(n->compatible,c);} + static bool of_machine_is_compatible(const char*c){return !strcmp(machine,c);} + static struct device_node fb={"simple-framebuffer"},dcp={"apple,t8122-dcp"}; + static u32 width,height,stride,notch,marker_value=1; + static const char *image_uuid; + static int of_property_read_string(struct device_node*n,const char*k,const char**v){*v=image_uuid;return 0;} + #define dev_err_probe(d,e,...) (e) + static struct device_node *of_find_compatible_node(void*a,void*b,const char*c){return &fb;} + static int of_property_read_u32(struct device_node*n,const char*k,u32*v){ + if(!strcmp(k,"apple,t8122-handoff"))*v=marker_value; + else if(!strcmp(k,"width"))*v=width; + else if(!strcmp(k,"height"))*v=height; + else if(!strcmp(k,"stride"))*v=stride; + else if(!strcmp(k,"apple,notch-height"))*v=notch; + else return -EINVAL; + return 0; + } + static void of_node_put(struct device_node*n){} + ''' + for name,src in cases: + start=src.index('struct dcp_v14_board {');end=src.index('struct dcp_v14_property {') + records=src[start:end] + defines='\n'.join(l for l in src.splitlines() if l.startswith('#define DCP_V14_') and 'FIRMWARE_UUID' in l)+'\n' + sel=src[src.index('const struct dcp_v14_board *iomfb_v14_7_board('):src.index('const char *iomfb_v14_7_board_name(')] + geo=src[src.index('static int dcp_v14_geometry('):src.index('/* An external processor\'s firmware:')] + driver='''struct apple_dcp_v14 {struct device *dev;const struct dcp_v14_board *board;u32 fb_width,fb_height,stride,notch_rows,panel_width,panel_height;};\n''' + probe=src[src.index('int iomfb_v14_7_probe('):] + uuid_gate=probe[probe.index('\tif (of_property_read_string(np, "apple,firmware-uuid"'):probe.index('\tif (!iommu_get_domain_for_dev(dev))')] + gate='static int probe_gate(struct device *dev,const struct dcp_v14_board *board){struct device_node *np=dev->of_node;const char *uuid=NULL;u32 marker=0;'+uuid_gate+'return 0;}\n' + tests=r'''int main(void){ + struct device dev={&dcp};machine="apple,j615"; + const struct dcp_v14_board *b=iomfb_v14_7_board(&dev); + #ifdef OLD + assert(IS_ERR(b));puts("old J615 board selection rejects: PASS");return 0; + #else + assert(!IS_ERR(b) && b && b->panel_width==2880 && b->panel_height==1864); + assert(!strcmp(b->firmware_uuid,"90F849E1-B422-367E-B389-50246F8DEC47")); + assert(!strcmp(b->handoff,"apple,t8122-handoff") && !b->promotion); + image_uuid=b->firmware_uuid;assert(probe_gate(&dev,b)==0); + image_uuid="C042E95C-B9D8-3F0E-94B3-582A08AA6FDD";assert(probe_gate(&dev,b)==-ENODEV); + image_uuid="unknown";assert(probe_gate(&dev,b)==-ENODEV); + image_uuid=b->firmware_uuid;marker_value=0;assert(probe_gate(&dev,b)==-ENODEV);marker_value=2;assert(probe_gate(&dev,b)==-ENODEV);marker_value=1; + assert(b->ctm_set==A(421) && b->ctm_get==A(420)); + machine="apple,j613";const struct dcp_v14_board*j=iomfb_v14_7_board(&dev); + assert(!IS_ERR(j)&&j->panel_width==2560 && j->panel_height==1664); + machine="apple,j700";assert(IS_ERR(iomfb_v14_7_board(&dev))); + machine="apple,j504";assert(IS_ERR(iomfb_v14_7_board(&dev))); + dcp.compatible="apple,t6030-dcp";machine="apple,j516s";assert(iomfb_v14_7_board(&dev)==&dcp_v14_board_t6030); + dcp.compatible="apple,t8122-dcp"; + struct apple_dcp_v14 v={.dev=&dev,.board=b};width=2880;height=1800;stride=width*4;notch=64; + assert(dcp_v14_geometry(&v)==0 && v.panel_height==1864); + height=1864;assert(dcp_v14_geometry(&v)==0 && v.panel_height==1864); + notch=0;assert(dcp_v14_geometry(&v)==0 && v.panel_height==1864); + height=1800;assert(dcp_v14_geometry(&v)==-EINVAL); + notch=64;height=1800;stride++;assert(dcp_v14_geometry(&v)==-EINVAL);stride--; + width=2560;stride=width*4;assert(dcp_v14_geometry(&v)==-EINVAL); + width=2880;stride=width*4;notch=101;assert(dcp_v14_geometry(&v)==-EINVAL); + puts("current J615/J613/T6030 admission + hidden/full notch, wrong geometry controls: PASS"); + #endif + } + ''' + p=out/(name+'-panel-control.c');p.write_text(prefix+defines+records+driver+sel+geo+gate+tests) + cmd=['cc','-std=gnu11','-Wall','-Werror','-Wno-unused-function'] + if name=='old':cmd.append('-DOLD') + subprocess.run(cmd+[str(p),'-o',str(out/(name+'-panel-control'))],check=True) + subprocess.run([str(out/(name+'-panel-control'))],check=True) From 10dfd212d54e42c3d5097ed08c725f32fa33997a Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Fri, 9 Oct 2026 15:23:40 -0400 Subject: [PATCH 1651/2680] drm/apple: accept the M3 full-height notch parameter Accept appledrm.t6030_show_notch for the 14.x and 26.6 internal-display backends while preserving the existing show_notch option and default geometry. Co-authored-by: Claude Opus 5.5 Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 1219e74caeb317aed75696e2b80f3b67eebea6b0) --- drivers/gpu/drm/apple/dcp.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 29437049c63a8..864b1cf83a317 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -194,6 +194,11 @@ static bool dcpext_probe; module_param(dcpext_probe, bool, 0444); MODULE_PARM_DESC(dcpext_probe, "Opt into experimental T6030 external DCP probe (may hang)"); +static bool t6030_show_notch; +module_param(t6030_show_notch, bool, 0444); +MODULE_PARM_DESC(t6030_show_notch, + "M3 internal display (14.x and 26.6 IOMFB): use the full height and show the notch"); + bool hdmi_audio; module_param(hdmi_audio, bool, 0644); MODULE_PARM_DESC(hdmi_audio, "Enable unstable HDMI audio support"); @@ -3080,6 +3085,20 @@ static bool dcp_external_is_idle(struct apple_dcp *dcp) (status & APPLE_DCP_COPROC_CPU_STATUS_IDLE); } +/* + * The M3 internal displays (14.x and 26.6 IOMFB) also accept + * appledrm.t6030_show_notch=1, the switch existing J613/T6030 installs pass, + * so the panel comes up at its full height (2560x1664 on the J613) instead + * of the module refusing an unknown parameter. + */ +static bool dcp_show_notch(struct apple_dcp *dcp) +{ + if (dcp->fw_compat != DCP_FIRMWARE_V_14_7 && + dcp->fw_compat != DCP_FIRMWARE_V_26_6) + return show_notch; + return show_notch || t6030_show_notch; +} + static int dcp_comp_bind(struct device *dev, struct device *main, void *data) { struct device_node *panel_np; @@ -3144,7 +3163,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) dev_info(dev, "DCP index:%u dptx target phy: %u dptx die: %u\n", dcp->index, dcp->dptx_phy, dcp->dptx_die); - if (!show_notch) + if (!dcp_show_notch(dcp)) ret = of_property_read_u32(dev->of_node, "apple,notch-height", &dcp->notch_height); From 5570edd507bf8565ab5f3e1b09fed5526c607f03 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:32:59 -0700 Subject: [PATCH 1652/2680] drm/apple-neo: require reachable Type-C mux helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ryan: adapted to 7.2.10 — applied to the shared DRM_APPLE: the apple-neo copy no longer exists; TYPEC=m with DRM_APPLE=y would not link] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 0e5fa8933f8216843b6f23fb1876ea02fa269de5) --- drivers/gpu/drm/apple/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 0eadf0e602250..51785525b74b4 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -4,6 +4,7 @@ config DRM_APPLE depends on DRM && OF && ARM64 depends on ARCH_APPLE || COMPILE_TEST depends on APPLE_RTKIT + depends on TYPEC || !TYPEC depends on OF_ADDRESS select APPLE_PMP_REPORT select OF_DYNAMIC From eccef333eab07d6aa80f4420dd5c56bfbad6d833 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 10 Oct 2026 08:29:10 -0700 Subject: [PATCH 1653/2680] drm/apple-neo: retain registration rollback helpers in built-in kernels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Registration failure cleanup is reachable from init on built-in Neo drivers. Keep the unregister helpers outside discarded exit sections so those error paths remain callable. [ryan: adapted to 7.2.10 — applied to the shared driver: __exit dropped from dcp_unregister() and dcp_audio_unregister() in dcp.c, audio.c and the dcp.h prototypes; the apple-neo copy does not exist here] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 77030de18bbbbf077777da5f821e9c15dd1e4c47) --- drivers/gpu/drm/apple/audio.c | 2 +- drivers/gpu/drm/apple/dcp.c | 2 +- drivers/gpu/drm/apple/dcp.h | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index 4b8c4d8fdd784..cc64168e1d8b3 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -809,7 +809,7 @@ int __init dcp_audio_register(void) return platform_driver_register(&dcpaud_driver); } -void __exit dcp_audio_unregister(void) +void dcp_audio_unregister(void) { platform_driver_unregister(&dcpaud_driver); } diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 864b1cf83a317..0e0fed7c7758b 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -4324,7 +4324,7 @@ int __init dcp_register(void) return platform_driver_register(&apple_platform_driver); } -void __exit dcp_unregister(void) +void dcp_unregister(void) { platform_driver_unregister(&apple_platform_driver); } diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 0cb225cb4eecc..58df330d55afa 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -106,10 +106,10 @@ int avep_init(struct apple_dcp *dcp); int __init dcp_register(void); -void __exit dcp_unregister(void); +void dcp_unregister(void); int __init dcp_audio_register(void); -void __exit dcp_audio_unregister(void); +void dcp_audio_unregister(void); int apple_j613_25g83_coldplug(void); unsigned int apple_j613_25g83_clock_hz(void); From f24631f02ed551353eb0838f1af95161891d337f Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Sat, 10 Oct 2026 09:39:35 -0700 Subject: [PATCH 1654/2680] drm/apple: add explicit experimental J615 25G83 profile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Require exactly one Air board identity, exact firmware compatibility and the explicit J615 experimental switch. Use this Mac’s own calibration and reservations. Keep the setup-record override disabled by default and require the exact firmware file after carveout validation. Add the J615 panel and six-speaker topology, keep NVRAM read-only and use the T8122-specific MTP compatible to leave firmware startup to Linux. [ryan: adapted to 7.2.10 — DCP part only. The J615 coldplug changes, written against drivers/soc/apple/j613-display-gate.c (78412c5ca5ae, not in this tree), are applied to this tree's drivers/gpu/drm/apple/j613_dcp_activate.c; t8122_25g83_board() and J615_25G83_OPT_IN go into j613-25g83.h and apple_t8122_25g83_board() is defined there unexported and declared in dcp.h (no linux/soc/apple/j613-display.h here). Dropped: the dts (handed over to 000-devicetree); drivers/gpu/drm/asahi (handed over to 210-gpu; nothing taken from held 8b20dcee4cbf); pmgr-pwrstate.c, t6030-display-gate.c and j613-display.h (handed over to 010-soc); the hwmon driver and its test (handed over to 060-power-thermal); tools/asahi/display26-tests.py, which this tree does not have] Co-Authored-By: Claude Opus 5.5 Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 628d47ea4c04860bcaf5e21b2ea664c7b3943f19) --- drivers/gpu/drm/apple/dcp.c | 4 +-- drivers/gpu/drm/apple/dcp.h | 2 ++ drivers/gpu/drm/apple/iomfb_v26_6.c | 28 ++++++++++++++---- drivers/gpu/drm/apple/j613-25g83.h | 15 ++++++++-- drivers/gpu/drm/apple/j613_dcp_activate.c | 36 ++++++++++++++++++++--- 5 files changed, 71 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 0e0fed7c7758b..bae3330f5df3d 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2929,13 +2929,13 @@ static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) return DCP_FIRMWARE_UNKNOWN; } - /* J613 25G83 has a distinct callback table, never a v14 fallback. */ + /* 25G83 (J613; J615 experimental) has a distinct callback table, never a v14 fallback. */ if (of_device_is_compatible(dev->of_node, "apple,t8122-dcp") && of_property_present(dev->of_node, "apple,j613-25g83-profile")) { const char *uuid; u32 profile; - if (of_machine_is_compatible("apple,j613") && + if (apple_t8122_25g83_board() && !of_property_read_u32(dev->of_node, "apple,j613-25g83-profile", &profile) && profile == 1 && !strcmp(compat_str, "26.6.2") && !of_property_read_string(dev->of_node, "apple,firmware-uuid", &uuid) && diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 58df330d55afa..aae0e427d7ec5 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -113,5 +113,7 @@ void dcp_audio_unregister(void); int apple_j613_25g83_coldplug(void); unsigned int apple_j613_25g83_clock_hz(void); +/* A J613, or a J615 whose boot loader carries the experimental opt-in. */ +bool apple_t8122_25g83_board(void); #endif diff --git a/drivers/gpu/drm/apple/iomfb_v26_6.c b/drivers/gpu/drm/apple/iomfb_v26_6.c index fe7f0fd9460e8..7203143d5b01a 100644 --- a/drivers/gpu/drm/apple/iomfb_v26_6.c +++ b/drivers/gpu/drm/apple/iomfb_v26_6.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* - * J613 (M3 Air 13 inch) experimental internal display: macOS 26.6.2 (25G83) IOMFB over DCPLink + * J613 (M3 Air 13 inch) experimental internal display: macOS 26.6.2 (25G83) IOMFB over DCPLink. + * J615 uses its own panel size and requires the experimental chosen switch. * * Copyright The Asahi Linux Contributors * Based on the M3 Pro DCP lab client by Eryk Wieliczko. @@ -129,8 +130,20 @@ static const struct dcp_v26_board dcp_v26_board_j613 = { .panel_width = 2560, .panel_height = 1664, }; +/* Experimental J615 panel: own 2880x1864 geometry and DT brightness cap. */ +static const struct dcp_v26_board dcp_v26_board_j615 = { + .name = "J615/25G83 (experimental)", + .dcp_compatible = "apple,t8122-dcp", + .machine = "apple,j615", + .firmware_uuid = DCP_V26_J613_FIRMWARE_UUID, + .handoff = "apple,j613-25g83-mapping-handoff", + .max_nits = 525, + .panel_width = 2880, + .panel_height = 1864, +}; static const struct dcp_v26_board *const dcp_v26_boards[] = { &dcp_v26_board_j613, + &dcp_v26_board_j615, }; struct dcp_v26_property { @@ -908,7 +921,7 @@ static int dcp_v26_cpu_running(struct device *dev, const char *name) */ const struct dcp_v26_board *iomfb_v26_6_board(struct device *dev) { - const struct dcp_v26_board *board; + const struct dcp_v26_board *board, *known = NULL; unsigned int i; for (i = 0; i < ARRAY_SIZE(dcp_v26_boards); i++) { @@ -916,12 +929,15 @@ const struct dcp_v26_board *iomfb_v26_6_board(struct device *dev) if (!of_device_is_compatible(dev->of_node, board->dcp_compatible)) continue; if (board->machine && !of_machine_is_compatible(board->machine)) { - dev_err(dev, "%s display not started: this machine is not a %s\n", - board->name, board->machine); - return ERR_PTR(-ENODEV); + known = board; + continue; } return board; } + if (known) { + dev_err(dev, "25G83 display not started: this machine is none of the 25G83 boards\n"); + return ERR_PTR(-ENODEV); + } return NULL; } @@ -1006,7 +1022,7 @@ int iomfb_v26_6_probe(struct apple_dcp *dcp) of_property_count_u32_elems(np, "apple,firmware-compat") != 3 || of_property_read_u32_array(np, "apple,firmware-compat", version, 3) || of_property_read_string(np, "apple,firmware-uuid", &uuid) || - !j613_25g83_identity(of_machine_is_compatible("apple,j613"), profile, version, 3, uuid)) + !j613_25g83_identity(apple_t8122_25g83_board(), profile, version, 3, uuid)) return -ENODEV; if (IS_ERR_OR_NULL(board)) return -ENODEV; diff --git a/drivers/gpu/drm/apple/j613-25g83.h b/drivers/gpu/drm/apple/j613-25g83.h index 636471b786016..2ac282ce95456 100644 --- a/drivers/gpu/drm/apple/j613-25g83.h +++ b/drivers/gpu/drm/apple/j613-25g83.h @@ -10,12 +10,23 @@ #define J613_25G83_DCP_UUID "C042E95C-B9D8-3F0E-94B3-582A08AA6FDD" #define J613_25G83_PMP_UUID "EFD60284-7C58-33A9-8522-CB3366AF6040" #define J613_25G83_HANDOFF "apple,j613-25g83-mapping-handoff" +/* J615 requires an explicit chosen switch in addition to the 25G83 profile. */ +#define J615_25G83_OPT_IN "asahi,j615-25g83-experimental" -static inline bool j613_25g83_identity(bool j613, u32 profile, +/* The board half of the identity: a J613, or a J615 with the opt-in. */ +static inline bool t8122_25g83_board(bool j613, bool j615, const char *optin, + int optin_len) +{ + return (j613 && !j615) || (!j613 && j615 && optin && optin_len == 2 && + optin[0] == '1' && !optin[1]); +} + +/* `board`: t8122_25g83_board() for the running machine. */ +static inline bool j613_25g83_identity(bool board, u32 profile, const u32 *version, size_t cells, const char *uuid) { - return j613 && profile == 1 && version && cells == 3 && + return board && profile == 1 && version && cells == 3 && version[0] == 26 && version[1] == 6 && version[2] == 2 && uuid && !strcmp(uuid, J613_25G83_DCP_UUID); } diff --git a/drivers/gpu/drm/apple/j613_dcp_activate.c b/drivers/gpu/drm/apple/j613_dcp_activate.c index 8fa196f5f7813..ecf90ece055e5 100644 --- a/drivers/gpu/drm/apple/j613_dcp_activate.c +++ b/drivers/gpu/drm/apple/j613_dcp_activate.c @@ -1,5 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Coldplug of the exact J613 25G83 display and its same-boot PMP supplier. */ +/* + * Coldplug of the exact 25G83 display and its same-boot PMP supplier: a J613, + * or (experimental) a J615 whose boot loader carries the J615 opt-in. + */ #include #include #include @@ -21,6 +24,22 @@ unsigned int apple_j613_25g83_clock_hz(void) return display_clock_hz; } +/* Require one Air board identity and the explicit J615 experimental switch. */ +bool apple_t8122_25g83_board(void) +{ + struct device_node *chosen = of_find_node_by_path("/chosen"); + const char *optin = NULL; + int len = 0; + bool ok; + + if (chosen) + optin = of_get_property(chosen, J615_25G83_OPT_IN, &len); + ok = t8122_25g83_board(of_machine_is_compatible("apple,j613"), + of_machine_is_compatible("apple,j615"), optin, len); + of_node_put(chosen); + return ok; +} + /* These are DT domain nodes, carrying the PMGR offsets, not copied registers. */ static int j613_power_ready(void) { @@ -206,14 +225,21 @@ int __init apple_j613_25g83_coldplug(void) unsigned int i; int ret = -EINVAL; - if (!of_machine_is_compatible("apple,j613")) + if (!of_machine_is_compatible("apple,j613") && !of_machine_is_compatible("apple,j615")) return 0; nodes[9] = of_find_node_by_path(paths[9]); - /* Ordinary 14.x J613 boot keeps the existing gate and report driver. */ + /* Ordinary 14.x J613/J615 boot keeps the existing gate and report driver. */ if (!nodes[9] || !of_property_present(nodes[9], "apple,j613-25g83-profile")) { ret = 0; goto out; } + /* A J615 25G83 device tree alone is not consent: the owner's opt-in is. */ + if (!apple_t8122_25g83_board()) { + pr_info("J615/25G83 display handoff not admitted: no /chosen/%s = \"1\"\n", + J615_25G83_OPT_IN); + ret = -EPERM; + goto out; + } if (of_property_count_u32_elems(nodes[9], "apple,j613-25g83-profile") != 1 || of_property_read_u32(nodes[9], "apple,j613-25g83-profile", &profile) || of_property_count_u32_elems(nodes[9], "apple,firmware-compat") != 3 || @@ -275,7 +301,9 @@ int __init apple_j613_25g83_coldplug(void) of_changeset_destroy(&gate); } else { display_clock_hz = witness[3]; - pr_info("J613/25G83 display handoff accepted; ADT clock %u Hz\n", display_clock_hz); + pr_info("%s/25G83 display handoff accepted; ADT clock %u Hz\n", + of_machine_is_compatible("apple,j615") ? "J615 (experimental)" : "J613", + display_clock_hz); } out: for (i = 0; i < ARRAY_SIZE(nodes); i++) From 79daaadf51ed3c98aea594924882fa0ec4079114 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:47:01 -0700 Subject: [PATCH 1655/2680] drm/apple: retain Type-C routes when display release fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Return HPD and RemotePort release failures before changing a live Type-C route or its PHY mode. Reattempt release during bounded cable revalidation, and keep retained routes out of handoff rollback while final removal detaches their owners. [ryan: adapted to 7.2.10 — the tipd core.c half is left to 170-usb (handed over to 170-usb), so cd321x does not retry a refused release until that lands; this tree has no dcp_dptx_disconnect_drained(): the HPD deassert failure is returned from dcp_dptx_disconnect_oob() instead; dcp_dptx_release_locked() keeps dptx_service_available(); dcp_dptx_disconnect() keeps the H17P external path; the two new release-error returns (rebalance deactivation, route_set on HPD loss) do not apply to H17P external owners, which keep their own stop recovery as before; tools/asahi/dp-release-tests.py and its fixture are left out: they extract upstream-shaped functions (dcp_dptx_disconnect_drained, tipd core.c) that this tree does not have] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 25b138b77409fcb49c2e4fbebee57d81bdea9bb3) --- drivers/gpu/drm/apple/dcp-fabric-core.c | 17 ++++-- drivers/gpu/drm/apple/dcp-fabric-core.h | 1 + drivers/gpu/drm/apple/dcp-fabric-effects.h | 3 + drivers/gpu/drm/apple/dcp-fabric.c | 69 +++++++++++++++++----- drivers/gpu/drm/apple/dcp.c | 47 ++++++++++----- drivers/gpu/drm/apple/dcp.h | 2 +- 6 files changed, 106 insertions(+), 33 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.c b/drivers/gpu/drm/apple/dcp-fabric-core.c index f9862a0594a77..1ba04abb6131b 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.c +++ b/drivers/gpu/drm/apple/dcp-fabric-core.c @@ -372,7 +372,7 @@ EXPORT_SYMBOL_GPL(dcp_fabric_follow); int dcp_fabric_follow_execute(const struct dcp_fabric_follow_ops *ops, void *ctx, unsigned int count) { - unsigned int i, detached = 0, attached = 0; + unsigned int i, n, detached = 0, attached = 0; int ret; if (!count || count > 2) @@ -406,9 +406,18 @@ int dcp_fabric_follow_execute(const struct dcp_fabric_follow_ops *ops, rollback: /* Only a failed original restore makes a connector terminally lost. */ - while (attached) - ops->detach(ctx, --attached, true); - for (i = 0; i < detached; i++) { + while (attached) { + i = --attached; + if (ops->detach(ctx, i, true) && ops->retained && + ops->retained(ctx, i, true)) + ops->publish(ctx, i, false); + } + for (n = 0; n < detached; n++) { + i = ops->retained ? detached - n - 1 : n; + /* A rejected release leaves the selected route where it was. */ + if (ops->retained && + (ops->retained(ctx, i, true) || ops->retained(ctx, i, false))) + continue; if (ops->attach(ctx, i, true)) ops->lost(ctx, i); else diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.h b/drivers/gpu/drm/apple/dcp-fabric-core.h index d78cb361a37c5..2216c3f8e4888 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.h +++ b/drivers/gpu/drm/apple/dcp-fabric-core.h @@ -119,6 +119,7 @@ struct dcp_fabric_follow_ops { int (*prepare)(void *ctx, unsigned int slot); int (*validate)(void *ctx, unsigned int slot); int (*detach)(void *ctx, unsigned int slot, bool destination); + bool (*retained)(void *ctx, unsigned int slot, bool destination); int (*attach)(void *ctx, unsigned int slot, bool restore); void (*publish)(void *ctx, unsigned int slot, bool restore); void (*lost)(void *ctx, unsigned int slot); diff --git a/drivers/gpu/drm/apple/dcp-fabric-effects.h b/drivers/gpu/drm/apple/dcp-fabric-effects.h index d59b4888b3129..3f4ad7c7eb52d 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-effects.h +++ b/drivers/gpu/drm/apple/dcp-fabric-effects.h @@ -139,6 +139,7 @@ struct dcp_fabric_deactivate_ops { struct dcp_fabric_reclaim_ops { int (*release)(void *ctx); + bool (*retained)(void *ctx); void (*unplug)(void *ctx); void (*connect_hdmi)(void *ctx); int (*activate)(void *ctx, bool restore); @@ -152,6 +153,8 @@ dcp_fabric_reclaim_execute(const struct dcp_fabric_reclaim_ops *ops, void *ctx) { int ret = ops->release(ctx); + if (ret && ops->retained && ops->retained(ctx)) + return ret; ops->unplug(ctx); if (!ret) { ops->connect_hdmi(ctx); diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index f07ecbe04b177..6adebba5cc930 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -1088,6 +1088,7 @@ static bool dcp_rebalance_deactivate(void *data, void *entry) struct apple_dcp_typec_route *owner = port->owner; struct apple_dcp *dcp; struct dcp_deactivate_context ctx; + int ret; /* A port in SAFE keeps its owner until the set phase completes. */ if (port->safe_pending) @@ -1100,8 +1101,12 @@ static bool dcp_rebalance_deactivate(void *data, void *entry) port->connector_np, dev_name(dcp->dev), port->target ? dev_name(port->target->dcp->dev) : "none"); if (port->hpd || dcp->typec_cable_connected || - (dcp->typec_connector && dcp->typec_connector->connected)) - dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + (dcp->typec_connector && dcp->typec_connector->connected)) { + ret = dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + /* H17P external pipes keep their own stop recovery, as before. */ + if (ret && !dcp_is_external_h17p(dcp)) + return true; + } port->hpd = false; ctx.port = port; ctx.owner = owner; @@ -1299,7 +1304,8 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, struct typec_displayport_data *dp_data = is_dp ? state->data : NULL; u32 dp_status = dp_data ? dp_data->status : 0; u32 dp_conf = dp_data ? dp_data->conf : 0; - bool hpd, was_counted; + bool hpd = dp_data && (dp_data->status & DP_STATUS_HPD_STATE); + bool was_counted; int ret = 0; guard(mutex)(&dcp_typec_fabric_lock); @@ -1330,6 +1336,18 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, port->applied_conf = dp_conf; /* Failed route acquisition must remain retryable on the next update. */ port->applied_valid = false; + /* Keep a direct route until DCP acknowledges release of its display. */ + if (!hpd && port->owner && !port->owner->tunnel) { + struct apple_dcp *dcp = port->owner->dcp; + + if (port->hpd || dcp->typec_cable_connected || + (dcp->typec_connector && dcp->typec_connector->connected)) { + ret = dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + /* H17P external pipes keep their own stop recovery, as before. */ + if (ret && !dcp_is_external_h17p(dcp)) + return ret; + } + } /* did the pairing pass count this port's direct stream so far? */ was_counted = port->dp_wanted && port->dp_hpd; @@ -1347,10 +1365,6 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, struct apple_dcp *dcp = port->owner->dcp; port->preferred_route = port->owner; - if (port->hpd || dcp->typec_cable_connected || - (dcp->typec_connector && - dcp->typec_connector->connected)) - dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); port->hpd = false; ret = dcp_typec_route_deactivate(port->owner); if (ret) @@ -1382,7 +1396,6 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, return 0; } - hpd = dp_data && (dp_data->status & DP_STATUS_HPD_STATE); port->dp_wanted = true; port->dp_hpd = hpd; @@ -1425,10 +1438,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, } - - if (!hpd && port->hpd) { - dcp_dptx_disconnect_oob(to_platform_device(port->owner->dcp->dev), 0); - } else if (hpd && !port->hpd) { + if (hpd && !port->hpd) { struct apple_dcp *dcp = port->owner->dcp; WRITE_ONCE(dcp->typec_cable_connected, true); @@ -2243,7 +2253,9 @@ static int dcp_follow_release(struct apple_dcp_typec_route *route) int ret; /* Revoked sessions cannot touch the new route after a reconnect wait. */ - dcp_dptx_park(dcp); + ret = dcp_dptx_park(dcp); + if (ret) + return ret; apple_connector_edid_set_live(dcp->typec_connector, false); dcp_modes_begin_attachment(dcp); dcp->typec_connector = NULL; @@ -2271,6 +2283,20 @@ static int dcp_follow_detach(void *data, unsigned int index, bool destination) return dcp_follow_release(destination ? slot->to : slot->from); } +static bool dcp_follow_owned(struct apple_dcp_typec_route *route) +{ + return route->port->owner == route && route->selected && + route->dcp->active_typec_route == route; +} + +static bool dcp_follow_retained(void *data, unsigned int index, bool destination) +{ + struct dcp_typec_follow_context *ctx = data; + struct dcp_typec_follow_slot *slot = &ctx->slots[index]; + + return dcp_follow_owned(destination ? slot->to : slot->from); +} + /* Route @slot's display through @route, a tunnel with its binding. */ static int dcp_follow_activate(struct apple_dcp_typec_route *route, struct dcp_typec_follow_slot *slot) @@ -2314,6 +2340,8 @@ static int dcp_follow_attach(void *data, unsigned int index, bool restore) struct apple_dcp *dcp = route->dcp; int ret; + if (restore && dcp->active_typec_route && dcp->active_typec_route != route) + return -EBUSY; ret = dcp_follow_activate(route, slot); if (ret) return ret; @@ -2359,6 +2387,7 @@ static const struct dcp_fabric_follow_ops dcp_follow_ops = { .prepare = dcp_follow_prepare, .validate = dcp_follow_validate, .detach = dcp_follow_detach, + .retained = dcp_follow_retained, .attach = dcp_follow_attach, .publish = dcp_follow_publish, .lost = dcp_follow_lost, @@ -3163,6 +3192,13 @@ static int dcp_reclaim_release(void *data) return dcp_follow_release(ctx->owner); } +static bool dcp_reclaim_retained(void *data) +{ + struct dcp_hdmi_reclaim_context *ctx = data; + + return dcp_follow_owned(ctx->owner); +} + static void dcp_reclaim_unplug(void *data) { struct dcp_hdmi_reclaim_context *ctx = data; @@ -3229,6 +3265,7 @@ static void dcp_reclaim_lost(void *data) static const struct dcp_fabric_reclaim_ops dcp_reclaim_ops = { .release = dcp_reclaim_release, + .retained = dcp_reclaim_retained, .unplug = dcp_reclaim_unplug, .connect_hdmi = dcp_reclaim_connect_hdmi, .activate = dcp_reclaim_activate, @@ -3314,7 +3351,11 @@ void dcp_fabric_hdmi_retry(struct apple_dcp *dcp) } dev_info(dcp->dev, "display retry: connecting the HDMI display again\n"); /* Releases a half-made connection; a no-op otherwise. */ - dcp_dptx_disconnect(dcp, 0); + ret = dcp_dptx_disconnect(dcp, 0); + if (ret) { + dcp_hdmi_connect_failed(dcp, ret); + return; + } dcp_fixed_hdmi_reinit_locked(dcp, "its display link did not come up"); ret = dcp_fixed_output_select(dcp); if (!ret) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index bae3330f5df3d..feba2cbde9af9 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1290,7 +1290,7 @@ void dcp_external_ready(struct apple_dcp *dcp) #define DCP_EXTERNAL_RETRIES 3 -static void dcp_dptx_release_locked(struct apple_dcp *dcp, u32 port); +static int dcp_dptx_release_locked(struct apple_dcp *dcp, u32 port); /* * Bounded recovery for a native external pipe: re-apply the display mode, @@ -1402,9 +1402,13 @@ void dcp_external_retry_work(struct work_struct *work) if (dptx_service_available(&dcp->dptxport[0]) && dcp->dptxport[0].connected) { int ret = dptxport_set_hpd(dcp->dptxport[0].service, false); - if (ret) + if (ret) { dev_warn(dcp->dev, "display retry: HPD deassert failed: %d\n", ret); - dcp_dptx_release_locked(dcp, 0); + return; + } + ret = dcp_dptx_release_locked(dcp, 0); + if (ret) + return; } dcp->typec_reconnect_tries = 0; } @@ -1437,7 +1441,9 @@ void dcp_external_sink_irq(struct apple_dcp *dcp) } dev_info(dcp->dev, "display sink IRQ_HPD not passed (%d): connecting the display anew\n", ret); - dcp_dptx_disconnect_oob(pdev, 0); + ret = dcp_dptx_disconnect_oob(pdev, 0); + if (ret) + return; dcp_dptx_connect_oob(pdev, 0); } @@ -1795,15 +1801,19 @@ static void disconnected_hpd_event(struct apple_connector *con) schedule_work(&con->bl_sync_wq); } -static void dcp_dptx_release_locked(struct apple_dcp *dcp, u32 port) +static int dcp_dptx_release_locked(struct apple_dcp *dcp, u32 port) { + int ret; + lockdep_assert_held(&dcp->hpd_mutex); if (dcp->external) { smp_store_release(&dcp->external_link_ready, false); dcpext_scanout_invalidate(dcp); } if (dptx_service_available(&dcp->dptxport[port]) && dcp->dptxport[port].connected) { - dptxport_release_display(dcp->dptxport[port].service); + ret = dptxport_release_display(dcp->dptxport[port].service); + if (ret) + return ret; dcp->dptxport[port].connected = false; } /* @@ -1813,21 +1823,23 @@ static void dcp_dptx_release_locked(struct apple_dcp *dcp, u32 port) */ if (dcp->external_native) dcp_direct_crossbar_link(dcp, false); + return 0; } int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) { + int ret; + if (dcp_is_external_h17p(dcp)) return dcp_external_disconnect(dcp, false); - /* Release the caller's RemotePort service, not the downstream DFP port. */ dev_info(dcp->dev, "%s(port=%d)\n", __func__, port); mutex_lock(&dcp->hpd_mutex); - dcp_dptx_release_locked(dcp, port); + ret = dcp_dptx_release_locked(dcp, port); mutex_unlock(&dcp->hpd_mutex); - return 0; + return ret; } int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) @@ -1912,8 +1924,12 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) if (dcp->avep) av_service_disconnect(dcp); - if (dptx_service_available(&dcp->dptxport[port])) - dptxport_set_hpd(dcp->dptxport[port].service, false); + if (dptx_service_available(&dcp->dptxport[port])) { + int ret = dptxport_set_hpd(dcp->dptxport[port].service, false); + + if (ret) + return ret; + } return dcp_dptx_disconnect(dcp, port); } @@ -1975,7 +1991,7 @@ int dcp_dptx_recover_irq(struct apple_dcp *dcp) * attached, as a CRTC power-off does (see dcp_poweroff()): the firmware's * unplug for it is ignored, and dcp_poweron() connects the link again. */ -void dcp_dptx_park(struct apple_dcp *dcp) +int dcp_dptx_park(struct apple_dcp *dcp) { int ret; @@ -1990,10 +2006,13 @@ void dcp_dptx_park(struct apple_dcp *dcp) av_service_disconnect(dcp); if (dptx_service_available(&dcp->dptxport[0]) && dcp->dptxport[0].connected) { ret = dptxport_set_hpd(dcp->dptxport[0].service, false); - if (ret) + if (ret) { dev_warn(dcp->dev, "failed to deassert Type-C DPTX HPD: %d\n", ret); - dcp_dptx_disconnect(dcp, 0); + return ret; + } + return dcp_dptx_disconnect(dcp, 0); } + return 0; } bool dcp_fw_compat_is_12_x(struct platform_device *pdev) diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index aae0e427d7ec5..687ca5c2c413e 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -88,7 +88,7 @@ void dcp_external_retry(struct apple_dcp *dcp, const char *why, int error, void dcp_external_retry_work(struct work_struct *work); void dcp_external_sink_irq(struct apple_dcp *dcp); int dcp_dptx_recover_irq(struct apple_dcp *dcp); -void dcp_dptx_park(struct apple_dcp *dcp); +int dcp_dptx_park(struct apple_dcp *dcp); int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); bool dcp_hold_resume_event(struct platform_device *pdev); From 3d88afc9bd89324bccdcc1c94069a2b7d3f865b1 Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Mon, 5 Oct 2026 09:38:12 -0600 Subject: [PATCH 1656/2680] drm/apple: drive both tiles of a tiled Thunderbolt display on one pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The LG UltraFine 5K is a tiled display: over Thunderbolt 3 it takes two DP 1.2 streams, one per 2560x2880 half, through the port's two DP IN adapters. A host shows it at 5120x2880 only if one DCP drives both halves: given its own pipeline, the second half is refused by DCP ("PAVSplitDisplay: no single tile support: tileLocation (1,0)") and the display stays at the 3840x2160 single-stream mode. An external DCP has two DPTX ports, and the crossbar can feed DP IN 1 into the second one (state dispextN,1). Route the second DP IN of a port there, onto the pipeline that already drives its first one, with the per-port tunnel state the first one keeps on the route: - dcp->split holds the DP IN 1 crossbar control and state, its link state and pixel clock flag, and the Thunderbolt binding (generation, callback) the way the route keeps DP IN 0's. - The tunnel helpers called from the DPTX endpoint take the DPTX unit, and unit 1 uses dcp->split. On t600x one tunnel pixel clock serves both DP INs, so unit 1 never stops it; t602x stops its own. - dcp_dptx_connect() puts DP IN 1's DFP port into the DPTX target of unit 1, and brings DPTX port 1 back whenever port 0 (re)connects: DCP pairs the tiles only while both ports are connected. A CRTC power-off releases port 1 together with port 0, so dcp_poweron() after DPMS or s2idle restores both. Without that, port 1 stays stale across s2idle, DCP reports a link fault and refuses the mode (0x80000104), and the display stays dark until it is replugged. - The Thunderbolt dispatch takes DP IN 1 onto DPTX port 1 once DP IN 0 is routed on the port (the Thunderbolt side retries on -ENODEV until then), and tears it down before DP IN 0 and when the route goes away. A repeated or stale request for DP IN 1 is answered like one for a route of its own. The split goes live only once the crossbar carries DP IN 1 to the pipeline; a crossbar or DPTX port 1 failure is undone and returned, and DP IN 0 keeps its single-stream route. DCP then pairs the two tiles itself and offers 5120x2880@60 on the one connector, as macOS does. The second pipeline stays free: on the M1 Pro the HDMI port keeps dcpext0 next to the 5K display. tiled_split=0 restores the old routing. t6030 external DCPs are left out, as their DP IN 1 has no tunnel clock yet. [ryan: adapted to 7.2.10 — the Thunderbolt binding generation (dcp_fabric_binding_request, tb_generation; 170-usb API) is not in this tree: dcp->split carries no generation, a repeated dpin1 request on a split port is answered as done and a release tears the split down; dcp_tb_dispatch() keeps this tree's tunnel slot logic with the split hooks added; route unregister tears the split down without dcp_tb_binding_drain()/dcp_dptx_disconnect_drained() (absent here); dcp_dptx_connect() reconnects DPTX port 1 after a non-H17P port-0 connect; the T6030 14.7 failure replies in dptxep.c are kept and take the DPTX unit] Signed-off-by: Christian Bartels Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit fcc9c340027c28ff33032afd2bb7cd0b6935d242) --- drivers/gpu/drm/apple/dcp-fabric.c | 205 ++++++++++++++++++++++++++- drivers/gpu/drm/apple/dcp-fabric.h | 9 +- drivers/gpu/drm/apple/dcp-internal.h | 14 ++ drivers/gpu/drm/apple/dcp.c | 42 +++++- drivers/gpu/drm/apple/dptxep.c | 10 +- 5 files changed, 263 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 6adebba5cc930..d3441b1b98c1a 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -27,6 +27,11 @@ #include "dcp-internal.h" #include "parser.h" +static bool tiled_split = true; +module_param(tiled_split, bool, 0644); +MODULE_PARM_DESC(tiled_split, + "Drive both halves of a tiled Thunderbolt display (LG UltraFine 5K) on one pipeline, through its two DPTX ports (default: on)"); + static bool typec_follow_crtc = true; module_param(typec_follow_crtc, bool, 0444); MODULE_PARM_DESC(typec_follow_crtc, @@ -46,6 +51,8 @@ struct apple_dcp_typec_port { struct apple_connector *connector; /* A second logical stream through this port's USB4 dock. */ struct apple_connector *secondary_connector; + /* tiled_split: the pipeline whose DPTX port 1 carries dpin1 */ + struct apple_dcp *split_dcp; /* last mux state acted on, to collapse the per-candidate notifications */ struct typec_altmode *applied_alt; unsigned long applied_mode; @@ -1558,12 +1565,42 @@ static int dcp_tunnel_dpin_locked(struct apple_dcp *dcp, bool active) * output enable are kept, see dcp_tunnel_crossbar_down()). * Runs inside a DCP apcall: only tb_lock, which nobody holds across a DCP call. */ -int dcp_tunnel_crossbar_up(struct apple_dcp *dcp) +static int dcp_tunnel_split_up_locked(struct apple_dcp *dcp) +{ + int ret; + + lockdep_assert_held(&dcp->tb_lock); + if (!dcp->split.active || !dcp->split.xbar) + return -ENODEV; + if (!dcp->split.clock_ok) { + dev_warn(dcp->dev, "tiled: no DP tunnel pixel clock, dpin1 crossbar left down\n"); + return -EIO; + } + if (!dcp->split.xbar_up) { + /* never block a DCP call on the mux semaphore */ + ret = mux_control_try_select(dcp->split.xbar, dcp->split.mux_state); + if (!ret) + dcp->split.xbar_up = true; + } else { + ret = dcp_dpxbar_link(dcp->split.xbar, true); + } + if (ret) { + dev_warn(dcp->dev, "tiled: dpin1 crossbar up failed: %d\n", ret); + return ret; + } + if (!dcp->split.set_active) + return -ENODEV; + return dcp->split.set_active(dcp->split.binding, true); +} + +int dcp_tunnel_crossbar_up(struct apple_dcp *dcp, u32 unit) { struct apple_dcp_typec_route *route; int ret; guard(mutex)(&dcp->tb_lock); + if (unit) + return dcp_tunnel_split_up_locked(dcp); route = dcp->active_typec_route; if (!route || !route->tunnel || !route->active_xbar || route->deactivate_xbar_done) @@ -1595,11 +1632,18 @@ int dcp_tunnel_crossbar_up(struct apple_dcp *dcp) * link: DP IN inactive, crossbar clocks down (mux selection and ATC output * enable kept). DP IN goes active again in dcp_tunnel_crossbar_up(). */ -int dcp_tunnel_crossbar_down(struct apple_dcp *dcp) +int dcp_tunnel_crossbar_down(struct apple_dcp *dcp, u32 unit) { struct apple_dcp_typec_route *route; guard(mutex)(&dcp->tb_lock); + if (unit) { + if (!dcp->split.active || !dcp->split.xbar_up) + return 0; + if (dcp->split.set_active) + dcp->split.set_active(dcp->split.binding, false); + return dcp_dpxbar_link(dcp->split.xbar, false); + } route = dcp->active_typec_route; if (!route || !route->tunnel || !route->xbar_up) return 0; @@ -1649,7 +1693,8 @@ int dcp_direct_crossbar_link(struct apple_dcp *dcp, bool up) * Thunderbolt DP IN, from DCP's SetLinkRate: start (rate != 0) or stop the * tunnel pixel clock. A stopped clock also takes the crossbar connection down. */ -int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate) +int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 unit, + u32 link_rate) { typeof(&apple_atc_dp_tunnel_rate) fn; struct apple_dcp_typec_route *route; @@ -1670,6 +1715,27 @@ int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate) dev_err(dcp->dev, "phy-apple-atc not loaded, no DP tunnel clock\n"); return -ENOENT; } + if (unit) { + /* + * t602x has a pixel clock per DP IN; t600x runs one for both, + * which the second tile never stops on its own: dpin0 does. + */ + ret = 0; + if (!dcp->split.active) { + ret = -ENODEV; + } else { + if (!link_rate && dcp->split.xbar_up) + dcp_dpxbar_link(dcp->split.xbar, false); + if (link_rate || dcp->hw.t6020_tunnel_flow) + ret = fn(phy, 1, link_rate); + } + symbol_put(apple_atc_dp_tunnel_rate); + dcp->split.clock_ok = !ret && link_rate; + if (ret) + dev_warn(dcp->dev, "tiled: dpin1 pixel clock (rate 0x%x) failed: %d\n", + link_rate, ret); + return ret; + } route = dcp->active_typec_route; if (!route) { symbol_put(apple_atc_dp_tunnel_rate); @@ -1687,7 +1753,7 @@ int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate) } /* Thunderbolt DP IN: DCP Activate/Deactivate */ -int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active) +int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, u32 unit, bool active) { struct apple_dcp_typec_route *route; int ret; @@ -1695,6 +1761,11 @@ int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active) guard(mutex)(&dcp->tb_lock); if (!dcp->dptx_tunnel) return 0; + if (unit) { + if (!dcp->split.active || !dcp->split.set_active) + return 0; + return dcp->split.set_active(dcp->split.binding, active); + } /* Route the upstream engine before firmware begins AUX negotiation. * T602X DP IN .set does not enable clocks; DidChange does that later. */ @@ -1896,6 +1967,122 @@ static const struct dcp_fabric_attach_ops dcp_tb_attach_ops = { .connect_oob = dcp_tb_connect_oob, }; +/* + * tiled_split: take the second half of a tiled display (dpin1) onto DPTX + * port 1 of the pipeline that already drives dpin0 on this port. DCP pairs + * the two tiles itself (PAVSplitDisplay) and presents one display; on its + * own pipeline the second tile is refused ("no single tile support"). + * Fabric lock held. + */ +static void dcp_tb_split_teardown_locked(struct apple_dcp_typec_port *port) +{ + struct apple_dcp *dcp = port->split_dcp; + + lockdep_assert_held(&dcp_typec_fabric_lock); + if (!dcp) + return; + if (dcp->dptxport[1].enabled) + dptxport_set_hpd(dcp->dptxport[1].service, false); + dcp_dptx_disconnect(dcp, 1); + scoped_guard(mutex, &dcp->tb_lock) { + if (dcp->split.xbar) { + if (dcp->split.xbar_up) + mux_control_deselect(dcp->split.xbar); + else + dcp_dpxbar_preselect(dcp->split.xbar, + MUX_IDLE_DISCONNECT); + dcp_dpxbar_tunnel_select_source(dcp->split.xbar, -1); + } + dcp->split.active = false; + dcp->split.xbar = NULL; + dcp->split.xbar_up = false; + dcp->split.clock_ok = false; + dcp->split.set_active = NULL; + dcp->split.binding = NULL; + } + port->split_dcp = NULL; + dev_info(dcp->dev, "tiled: dpin1 released from DPTX port 1\n"); +} + +/* -EOPNOTSUPP: route dpin1 as a display of its own. */ +static int dcp_tb_split_locked(struct apple_dcp_typec_port *port, + const struct dcp_tb_attach_context *req) +{ + struct apple_dcp_typec_route *route = port->owner; + struct mux_control *xbar; + struct apple_dcp *dcp; + int ret, state; + + lockdep_assert_held(&dcp_typec_fabric_lock); + /* + * No Thunderbolt binding generation in this tree: a repeated request + * for a split dpin1 is answered as done, a release tears it down. + */ + if (port->split_dcp) { + if (req->active) + return 0; + dcp_tb_split_teardown_locked(port); + return 0; + } + if (!req->active) + return -EOPNOTSUPP; + /* dpin0 first: the Thunderbolt side retries on -ENODEV */ + if (!route || !route->tunnel || route->tunnel_dpin != 0) + return -ENODEV; + dcp = route->dcp; + if (dcp->external || !dcp->dptxport[1].enabled) + return -EOPNOTSUPP; + xbar = route->dpin[1]; + if (!xbar) { + /* Legacy DT ABI: unnamed DPIN controls share the DP-alt chip. */ + if (route->xbar != &route->xbar->chip->mux[0] || + route->xbar->chip->controllers < 3) + return -EOPNOTSUPP; + xbar = &route->xbar->chip->mux[2]; + } + state = route->mux_index | 1; + + /* T602X points the DP IN at its pipeline before DCP probes AUX */ + ret = dcp_dpxbar_tunnel_select_source(xbar, state); + if (ret && ret != -EOPNOTSUPP) { + dev_warn(dcp->dev, "tiled: dpin1 source select failed: %d\n", ret); + goto unwind; + } + ret = dcp_dpxbar_preselect(xbar, state); + if (ret && ret != -EOPNOTSUPP) { + dev_warn(dcp->dev, "tiled: dpin1 crossbar preselect failed: %d\n", ret); + goto unwind; + } + + /* Only a crossbar that carries dpin1 to the pipeline makes the split live. */ + scoped_guard(mutex, &dcp->tb_lock) { + dcp->split.xbar = xbar; + dcp->split.mux_state = state; + dcp->split.xbar_up = false; + dcp->split.clock_ok = false; + dcp->split.set_active = req->set_active; + dcp->split.binding = req->binding; + dcp->split.active = true; + } + port->split_dcp = dcp; + dev_info(dcp->dev, + "tiled: dpin1 joins dpin0 on DPTX port 1 (crossbar state %d)\n", + state); + + ret = dcp_dptx_connect(dcp, 1); + if (ret) { + dev_warn(dcp->dev, "tiled: DPTX port 1 connect failed: %d\n", ret); + /* dpin0 keeps its route and stays a single-stream display */ + dcp_tb_split_teardown_locked(port); + } + return ret; + +unwind: + dcp_dpxbar_preselect(xbar, MUX_IDLE_DISCONNECT); + dcp_dpxbar_tunnel_select_source(xbar, -1); + return ret; +} + static int dcp_tb_dispatch(void *data, const struct dcp_fabric_port *found) { struct dcp_tb_attach_context *ctx = data; @@ -1908,10 +2095,18 @@ static int dcp_tb_dispatch(void *data, const struct dcp_fabric_port *found) slot = dpin ? &port->secondary_owner : &port->owner; + if (dpin == 1 && (tiled_split || port->split_dcp)) { + ret = dcp_tb_split_locked(port, ctx); + if (ret != -EOPNOTSUPP) + return ret; + } if (!ctx->active) { if (!*slot || !(*slot)->tunnel || (*slot)->tunnel_dpin != dpin) return 0; + /* the second tile goes before the route it shares */ + if (dpin == 0) + dcp_tb_split_teardown_locked(port); dcp = (*slot)->dcp; if (port->hpd || dcp->typec_cable_connected) dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); @@ -2762,6 +2957,8 @@ static void dcp_typec_route_unregister(void *data) guard(mutex)(&dcp_typec_fabric_lock); if (port->preferred_route == route) port->preferred_route = NULL; + if (port->split_dcp == route->dcp) + dcp_tb_split_teardown_locked(port); if (port->owner == route) { struct apple_dcp *dcp = route->dcp; diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index bf56deb535d4c..46578a7abaeee 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -83,11 +83,12 @@ void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, void dcp_unlink(struct drm_device *drm); /* Thunderbolt DP tunnels, called from the DPTX endpoint */ -int dcp_tunnel_crossbar_up(struct apple_dcp *dcp); -int dcp_tunnel_crossbar_down(struct apple_dcp *dcp); +int dcp_tunnel_crossbar_up(struct apple_dcp *dcp, u32 unit); +int dcp_tunnel_crossbar_down(struct apple_dcp *dcp, u32 unit); int dcp_direct_crossbar_link(struct apple_dcp *dcp, bool up); -int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate); -int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active); +int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 unit, + u32 link_rate); +int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, u32 unit, bool active); /* DCP probe, resume and endpoint teardown hooks. */ int dcp_register_typec_routes(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 5a6fa2c934556..08cd43744a19b 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -498,6 +498,20 @@ struct apple_dcp { bool tb_clock_ok; /* tb_lock: the direct DP PHY crossbar output whose clocks run (T6030). */ struct mux_control *direct_xbar_up; + /* + * Tiled display (LG UltraFine 5K): the second Thunderbolt DP IN of the + * same port drives DPTX port 1 of this pipeline, so DCP sees both tiles + * and presents one display. Under tb_lock like the fields above. + */ + struct { + bool active; + struct mux_control *xbar; /* the port's dpin1 crossbar control */ + int mux_state; /* dispextN, DPTX port 1 */ + bool xbar_up; + bool clock_ok; + int (*set_active)(void *binding, bool active); + void *binding; + } split; /* CRTC powered off while the Type-C cable stays attached */ bool typec_crtc_off; bool typec_follow_start; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index feba2cbde9af9..90406d2bc3bce 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -971,6 +971,7 @@ int dcp_get_connector_type(struct platform_device *pdev) static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable) { unsigned long timeout; + u8 dfp_port; int ret = 0, err; if (retryable) @@ -1015,8 +1016,10 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable reinit_completion(&dcp->dptxport[port].linkcfg_completion); dcp->dptxport[port].atcphy = dcp->phy; + /* a tiled display's second half comes in on the port's dpin1 */ + dfp_port = port && dcp->split.active ? 2 : dcp->dptx_dfp_port; ret = dptxport_validate_connection(dcp->dptxport[port].service, - dcp->dptx_dfp_port, + dfp_port, dcp->dptx_phy, dcp->dptx_die); if (ret) { dev_err(dcp->dev, @@ -1026,7 +1029,7 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable } ret = dptxport_connect(dcp->dptxport[port].service, - dcp->dptx_dfp_port, + dfp_port, dcp->dptx_phy, dcp->dptx_die, dcp_is_typec_output(dcp)); if (ret) { @@ -1447,10 +1450,35 @@ void dcp_external_sink_irq(struct apple_dcp *dcp) dcp_dptx_connect_oob(pdev, 0); } +/* + * tiled_split: DPTX port 1 carries the second half of a tiled display on + * this pipeline. Bring it back whenever port 0 (re)connects, as DCP pairs + * the tiles only while both ports are connected. + */ +static void dcp_dptx_connect_tile(struct apple_dcp *dcp) +{ + bool split; + int ret; + + scoped_guard(mutex, &dcp->tb_lock) + split = dcp->split.active; + if (!split) + return; + ret = dcp_dptx_connect(dcp, 1); + if (ret) + dev_warn(dcp->dev, "tiled: DPTX port 1 reconnect failed: %d\n", ret); +} + int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) { - if (!dcp_is_external_h17p(dcp)) - return dcp_dptx_connect_raw(dcp, port, NULL); + int ret; + + if (!dcp_is_external_h17p(dcp)) { + ret = dcp_dptx_connect_raw(dcp, port, NULL); + if (!ret && !port) + dcp_dptx_connect_tile(dcp); + return ret; + } guard(mutex)(&dcp->external_lock); return dcp_external_link_locked(dcp); } @@ -2482,6 +2510,12 @@ void dcp_poweroff(struct platform_device *pdev) dcp_external_retry(dcp, ret ? "display link released with an HPD error" : "display link released with no display described", ret, 500); + /* and the second tile's, which dcp_poweron() brings back too */ + if (READ_ONCE(dcp->split.active) && dcp->dptxport[1].enabled && + dcp->dptxport[1].connected) { + dptxport_set_hpd(dcp->dptxport[1].service, false); + dcp_dptx_disconnect(dcp, 1); + } } else if (dcp->hdmi_hpd) { bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); if (!connected) { diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 8c2eb535d6b5a..8138ad5552900 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -542,7 +542,7 @@ static int dptxport_call_set_link_rate(struct apple_epic_service *service, */ if (dptx->atcphy && service->ep->dcp->dptx_tunnel) { ret = dcp_tunnel_set_rate(service->ep->dcp, dptx->atcphy, - link_rate); + dptx->unit, link_rate); /* T6030 14.7: refuse a rate whose tunnel clock failed. */ if (ret && service->ep->dcp->fw_compat == DCP_FIRMWARE_V_14_7) { reply->retcode = cpu_to_le32(1); @@ -635,7 +635,7 @@ dptxport_call_activate(struct apple_epic_service *service, if (dptx->atcphy && !dcp->phy_managed_by_typec) phy_set_mode_ext(dptx->atcphy, PHY_MODE_DP, dcp->index); if (dcp->dptx_tunnel) { - ret = dcp_tunnel_dpin_activate(dcp, true); + ret = dcp_tunnel_dpin_activate(dcp, dptx->unit, true); /* T6030 14.7: report a failed tunnel activation to DCP. */ if (ret && dcp->fw_compat == DCP_FIRMWARE_V_14_7) { dev_err(dcp->dev, "DPTX tunnel activation failed: %d\n", ret); @@ -665,7 +665,7 @@ dptxport_call_deactivate(struct apple_epic_service *service, dcp_180hz_invalidate(dcp); dev_dbg(dcp->dev, "DPTXPort: DEACTIVATE\n"); if (dcp->dptx_tunnel) - dcp_tunnel_dpin_activate(dcp, false); + dcp_tunnel_dpin_activate(dcp, dptx->unit, false); else dcp_direct_crossbar_link(dcp, false); if (dptx->atcphy && !dcp->phy_managed_by_typec) @@ -733,7 +733,7 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, case DPTX_APCALL_WILL_CHANGE_LINKG_CONFIG: /* A re-link takes an established tunnel's crossbar down first. */ if (service->ep->dcp->dptx_tunnel && dptx->link_rate) - dcp_tunnel_crossbar_down(service->ep->dcp); + dcp_tunnel_crossbar_down(service->ep->dcp, dptx->unit); else if (dptx->link_rate) dcp_direct_crossbar_link(service->ep->dcp, false); return dptxport_call_will_change_link_config(service); @@ -741,7 +741,7 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, int ret = dptxport_call_did_change_link_config(service); if (!ret && service->ep->dcp->dptx_tunnel && dptx->link_rate) { - int xret = dcp_tunnel_crossbar_up(service->ep->dcp); + int xret = dcp_tunnel_crossbar_up(service->ep->dcp, dptx->unit); /* T6030 14.7: fail the link config without a crossbar. */ if (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_14_7) From 5b60e03ed320af45bcb10b415f05b23e0b0a0e02 Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Mon, 5 Oct 2026 09:38:12 -0600 Subject: [PATCH 1657/2680] drm/apple: take a second DP IN as a tile only when DCP reports a tiled sink The previous change puts every second DP IN of a port on DPTX port 1 of the first one's pipeline. Two separate monitors behind a dock must keep their own pipelines instead. DCP tells the host the sink's tile topology through the DPTX SetTiledDisplayHints call, which the driver acknowledged without reading. For the LG UltraFine 5K it carries the tile location at 0x08/ 0x0c, the tile counts at 0x30/0x34 (2x1) and the whole display's size at 0x38/0x3c (5120x2880). Keep the counts and location per DPTX port, clear them whenever the port connects or disconnects, and take DP IN 1 as a tile only once DPTX port 0 reports a display of more than one tile. DCP sends the hint about 100 ms after DP IN 0's link comes up, so DP IN 1 waits for it for up to 2 s (the Thunderbolt side retries on -ENODEV) before it is routed the usual way. Tested on a MacBook Pro 14" M1 Pro (j314s) on 11.38 with its j314s device tree, an LG UltraFine 5K on the left Thunderbolt port and an HDMI monitor on dcpext0: at boot, after unplug/replug, across DPMS off/on and across two s2idle cycles (one of about 40 minutes): DPTXPort: port 0 carries tile (0,0) of a 2x1 tiled 5120x2880 display tiled: dpin1 joins dpin0 on DPTX port 1 (crossbar state 3) DPTXPort: port 1 carries tile (1,0) of a 2x1 tiled 5120x2880 display Rebased onto 12.2 and tested again on the j314s with 12.2's device tree, together with the Thunderbolt route-follows-CRTC series (#44) and the tile-follow commit on top: the LG alone at login, LG and HDMI, DPMS off/on, s2idle and replug as above. 12.3 and 12.4 change only M3 display code, and the two commits apply to 12.4 as they are. Signed-off-by: Christian Bartels Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit ae84041c57c9a300e4fc80f48584c6b68f86ee83) --- drivers/gpu/drm/apple/dcp-fabric.c | 19 ++++++++++++- drivers/gpu/drm/apple/dcp.c | 3 ++ drivers/gpu/drm/apple/dptxep.c | 44 ++++++++++++++++++++++++++++-- drivers/gpu/drm/apple/dptxep.h | 7 +++++ 4 files changed, 69 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index d3441b1b98c1a..60472776bba10 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -53,6 +53,8 @@ struct apple_dcp_typec_port { struct apple_connector *secondary_connector; /* tiled_split: the pipeline whose DPTX port 1 carries dpin1 */ struct apple_dcp *split_dcp; + /* how long dpin1 waits for dpin0's tile hint before routing normally */ + unsigned long tile_hint_deadline; /* last mux state acted on, to collapse the per-candidate notifications */ struct typec_altmode *applied_alt; unsigned long applied_mode; @@ -1904,6 +1906,8 @@ static void dcp_tb_claim(void *data) unsigned int dpin = ctx->dpin; *ctx->slot = best; + if (dpin == 0) + port->tile_hint_deadline = jiffies + msecs_to_jiffies(2000); /* the port is in USB4 mode, not DP-alt */ port->dp_wanted = false; dcp_tunnel_prepare(best, ctx->ctl); @@ -2004,7 +2008,7 @@ static void dcp_tb_split_teardown_locked(struct apple_dcp_typec_port *port) dev_info(dcp->dev, "tiled: dpin1 released from DPTX port 1\n"); } -/* -EOPNOTSUPP: route dpin1 as a display of its own. */ +/* -EOPNOTSUPP: not a tile, route dpin1 as a display of its own. */ static int dcp_tb_split_locked(struct apple_dcp_typec_port *port, const struct dcp_tb_attach_context *req) { @@ -2040,6 +2044,19 @@ static int dcp_tb_split_locked(struct apple_dcp_typec_port *port, return -EOPNOTSUPP; xbar = &route->xbar->chip->mux[2]; } + /* + * Only the second half of a tiled display goes here; two separate + * monitors behind a dock keep their own pipelines. DCP reports the + * topology for dpin0 shortly after its link comes up: wait for that, + * but not for ever, as a sink may never send one. + */ + if (!READ_ONCE(dcp->dptxport[0].tile_hint)) { + if (time_before(jiffies, port->tile_hint_deadline)) + return -ENODEV; + return -EOPNOTSUPP; + } + if (dcp->dptxport[0].tiles_h * dcp->dptxport[0].tiles_v < 2) + return -EOPNOTSUPP; state = route->mux_index | 1; /* T602X points the DP IN at its pipeline before DCP probes AUX */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 90406d2bc3bce..83af14a5c47a1 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1015,6 +1015,7 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable smp_store_release(&dcp->external_link_ready, false); reinit_completion(&dcp->dptxport[port].linkcfg_completion); + WRITE_ONCE(dcp->dptxport[port].tile_hint, false); dcp->dptxport[port].atcphy = dcp->phy; /* a tiled display's second half comes in on the port's dpin1 */ dfp_port = port && dcp->split.active ? 2 : dcp->dptx_dfp_port; @@ -1865,6 +1866,8 @@ int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) mutex_lock(&dcp->hpd_mutex); ret = dcp_dptx_release_locked(dcp, port); + if (!ret) + WRITE_ONCE(dcp->dptxport[port].tile_hint, false); mutex_unlock(&dcp->hpd_mutex); return ret; diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 8138ad5552900..bc39064144597 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright 2022 Sven Peter */ +#include #include #include #include @@ -85,6 +86,19 @@ struct dptxport_apcall_set_tiled { __le32 retcode; }; +/* + * SetTiledDisplayHints payload, as DCP sends it for an LG UltraFine 5K + * (offsets in bytes): tile location (column, row) at 0x08/0x0c, tiles + * across and down at 0x30/0x34, the whole display's size at 0x38/0x3c. + */ +#define DPTX_TILE_HINT_LOC_X 0x08 +#define DPTX_TILE_HINT_LOC_Y 0x0c +#define DPTX_TILE_HINT_TILES_H 0x30 +#define DPTX_TILE_HINT_TILES_V 0x34 +#define DPTX_TILE_HINT_WIDTH 0x38 +#define DPTX_TILE_HINT_HEIGHT 0x3c +#define DPTX_TILE_HINT_MIN_SIZE 0x40 + /* * A Thunderbolt DP tunnel uses the same CORE|ATC|DIE|CONNECTED target as a * direct alt-mode PHY: CORE is the DFP port (0 = dpphy, 1/2 = dpin0/dpin1) @@ -605,14 +619,36 @@ dptxport_call_get_supports_downspread(struct apple_epic_service *service, return 0; } -static int dptxport_call_set_tiled_display_hint(void *reply_, - size_t reply_size) +static int dptxport_call_set_tiled_display_hint(struct apple_epic_service *service, + const void *data, size_t data_size, + void *reply_, size_t reply_size) { struct dptxport_apcall_set_tiled *reply = reply_; + struct dptx_port *dptx = service->cookie; + const u8 *hint = data; if (reply_size < sizeof(*reply)) return -EINVAL; + if (dptx && data_size >= DPTX_TILE_HINT_MIN_SIZE) { + u32 h = get_unaligned_le32(hint + DPTX_TILE_HINT_TILES_H); + u32 v = get_unaligned_le32(hint + DPTX_TILE_HINT_TILES_V); + u32 x = get_unaligned_le32(hint + DPTX_TILE_HINT_LOC_X); + u32 y = get_unaligned_le32(hint + DPTX_TILE_HINT_LOC_Y); + + dptx->tiles_h = min(h, 255U); + dptx->tiles_v = min(v, 255U); + dptx->tile_x = min(x, 255U); + dptx->tile_y = min(y, 255U); + WRITE_ONCE(dptx->tile_hint, true); + if (h * v > 1) + dev_info(service->ep->dcp->dev, + "DPTXPort: port %u carries tile (%u,%u) of a %ux%u tiled %ux%u display\n", + dptx->unit, x, y, h, v, + get_unaligned_le32(hint + DPTX_TILE_HINT_WIDTH), + get_unaligned_le32(hint + DPTX_TILE_HINT_HEIGHT)); + } + reply->retcode = cpu_to_le32(1); return 0; } @@ -801,7 +837,9 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, return 0; case DPTX_APCALL_SET_TILED_DISPLAY_HINTS: memcpy(reply, data, min(reply_size, data_size)); - return dptxport_call_set_tiled_display_hint(reply, reply_size); + return dptxport_call_set_tiled_display_hint(service, data, + data_size, reply, + reply_size); case DPTX_APCALL_GET_DRIVE_SETTINGS: return dptxport_call_get_drive_settings(service, data, data_size, reply, reply_size); diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index 449542f2accbd..211e72d9855ea 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -63,6 +63,13 @@ struct dptx_port { u32 link_rate, pending_link_rate; u32 drive_settings[2]; u32 trained_voltage[4], trained_pre[4]; + /* + * The sink's tile topology as DCP reports it in SetTiledDisplayHints: + * tiles across and down, and this stream's tile. Valid once + * tile_hint is set; cleared whenever the port (re)connects. + */ + bool tile_hint; + u8 tiles_h, tiles_v, tile_x, tile_y; }; int dptxport_validate_connection(struct apple_epic_service *service, u8 core, From 889b61473e3911c6c6a13bac402fe9c09d7acd54 Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Wed, 7 Oct 2026 05:39:19 -0600 Subject: [PATCH 1658/2680] drm/apple: take the second tile along when a tiled display's route follows its CRTC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A tiled Thunderbolt display (LG UltraFine 5K) drives its second half through DP IN 1 on DPTX port 1 of the pipeline that drives its first. When the route of DP IN 0 follows its CRTC to another pipeline, the second tile has to go with it, or DCP sees half a display. Release DPTX port 1 with port 0 when the link is parked, take the split down with the route, keeping its Thunderbolt binding, and join DP IN 1 to DPTX port 1 of the new pipeline. dcp_poweron() then connects port 1 after port 0, as after DPMS. A pipeline without a second DPTX port, or without a crossbar control for DP IN 1, refuses the modeset, and is not taken when an HDMI display takes its hybrid back. If the crossbar does not take DP IN 1 on the new pipeline, the display moves single-stream. Tested on 12.2 on the j314s with an LG UltraFine 5K, together with the Thunderbolt CRTC-following series: its route and both tiles moved between dcpext1 and dcpext0 when it followed its CRTC at login, on DPMS on after HDMI was unplugged, in a swap with a USB-C adapter display, and when an HDMI display took the hybrid back; 5120x2880 each time. [ryan: adapted to 7.2.10 — no Thunderbolt binding generation here (170-usb API): dcp_tb_split_join_locked() and the follow slot carry only the second tile's set_active callback and context; dcp_dptx_park() keeps dptx_service_available() for both DPTX ports] Signed-off-by: Christian Bartels Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 5eeea775031de79309949267cf52f9d2e7564e3f) --- drivers/gpu/drm/apple/dcp-fabric.c | 129 +++++++++++++++++++++-------- drivers/gpu/drm/apple/dcp.c | 10 +++ 2 files changed, 104 insertions(+), 35 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 60472776bba10..65e24ed4fd5e7 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -2008,6 +2008,54 @@ static void dcp_tb_split_teardown_locked(struct apple_dcp_typec_port *port) dev_info(dcp->dev, "tiled: dpin1 released from DPTX port 1\n"); } +/* + * Feed dpin1 into DPTX port 1 of @route's pipeline, with its binding. The + * split goes live only once the crossbar carries dpin1 there; a failure is + * undone and returned. + */ +static int dcp_tb_split_join_locked(struct apple_dcp_typec_port *port, + struct apple_dcp_typec_route *route, + struct mux_control *xbar, + int (*set_active)(void *binding, bool active), + void *binding) +{ + struct apple_dcp *dcp = route->dcp; + int ret, state = route->mux_index | 1; + + lockdep_assert_held(&dcp_typec_fabric_lock); + /* T602X points the DP IN at its pipeline before DCP probes AUX */ + ret = dcp_dpxbar_tunnel_select_source(xbar, state); + if (ret && ret != -EOPNOTSUPP) { + dev_warn(dcp->dev, "tiled: dpin1 source select failed: %d\n", ret); + goto unwind; + } + ret = dcp_dpxbar_preselect(xbar, state); + if (ret && ret != -EOPNOTSUPP) { + dev_warn(dcp->dev, "tiled: dpin1 crossbar preselect failed: %d\n", ret); + goto unwind; + } + + scoped_guard(mutex, &dcp->tb_lock) { + dcp->split.xbar = xbar; + dcp->split.mux_state = state; + dcp->split.xbar_up = false; + dcp->split.clock_ok = false; + dcp->split.set_active = set_active; + dcp->split.binding = binding; + dcp->split.active = true; + } + port->split_dcp = dcp; + dev_info(dcp->dev, + "tiled: dpin1 joins dpin0 on DPTX port 1 (crossbar state %d)\n", + state); + return 0; + +unwind: + dcp_dpxbar_preselect(xbar, MUX_IDLE_DISCONNECT); + dcp_dpxbar_tunnel_select_source(xbar, -1); + return ret; +} + /* -EOPNOTSUPP: not a tile, route dpin1 as a display of its own. */ static int dcp_tb_split_locked(struct apple_dcp_typec_port *port, const struct dcp_tb_attach_context *req) @@ -2015,7 +2063,7 @@ static int dcp_tb_split_locked(struct apple_dcp_typec_port *port, struct apple_dcp_typec_route *route = port->owner; struct mux_control *xbar; struct apple_dcp *dcp; - int ret, state; + int ret; lockdep_assert_held(&dcp_typec_fabric_lock); /* @@ -2057,34 +2105,10 @@ static int dcp_tb_split_locked(struct apple_dcp_typec_port *port, } if (dcp->dptxport[0].tiles_h * dcp->dptxport[0].tiles_v < 2) return -EOPNOTSUPP; - state = route->mux_index | 1; - - /* T602X points the DP IN at its pipeline before DCP probes AUX */ - ret = dcp_dpxbar_tunnel_select_source(xbar, state); - if (ret && ret != -EOPNOTSUPP) { - dev_warn(dcp->dev, "tiled: dpin1 source select failed: %d\n", ret); - goto unwind; - } - ret = dcp_dpxbar_preselect(xbar, state); - if (ret && ret != -EOPNOTSUPP) { - dev_warn(dcp->dev, "tiled: dpin1 crossbar preselect failed: %d\n", ret); - goto unwind; - } - - /* Only a crossbar that carries dpin1 to the pipeline makes the split live. */ - scoped_guard(mutex, &dcp->tb_lock) { - dcp->split.xbar = xbar; - dcp->split.mux_state = state; - dcp->split.xbar_up = false; - dcp->split.clock_ok = false; - dcp->split.set_active = req->set_active; - dcp->split.binding = req->binding; - dcp->split.active = true; - } - port->split_dcp = dcp; - dev_info(dcp->dev, - "tiled: dpin1 joins dpin0 on DPTX port 1 (crossbar state %d)\n", - state); + ret = dcp_tb_split_join_locked(port, route, xbar, req->set_active, + req->binding); + if (ret) + return ret; ret = dcp_dptx_connect(dcp, 1); if (ret) { @@ -2093,11 +2117,6 @@ static int dcp_tb_split_locked(struct apple_dcp_typec_port *port, dcp_tb_split_teardown_locked(port); } return ret; - -unwind: - dcp_dpxbar_preselect(xbar, MUX_IDLE_DISCONNECT); - dcp_dpxbar_tunnel_select_source(xbar, -1); - return ret; } static int dcp_tb_dispatch(void *data, const struct dcp_fabric_port *found) @@ -2247,8 +2266,34 @@ struct dcp_typec_follow_slot { unsigned int dpin; int (*set_active)(void *binding, bool active); void *binding; + /* tiled_split: the second tile, on DPTX port 1 of the same pipeline */ + bool split; + int (*split_set_active)(void *binding, bool active); + void *split_binding; }; +/* Record @route's second tile, if it has one, for the pipeline it joins. */ +static void dcp_follow_save_split(struct apple_dcp_typec_route *route, + struct dcp_typec_follow_slot *slot) +{ + struct apple_dcp *dcp = route->dcp; + + if (route->port->split_dcp != dcp) + return; + slot->split = true; + scoped_guard(mutex, &dcp->tb_lock) { + slot->split_set_active = dcp->split.set_active; + slot->split_binding = dcp->split.binding; + } +} + +/* Can @route's pipeline take a second tile? */ +static bool dcp_follow_split_fits(struct apple_dcp_typec_route *route) +{ + return dcp_typec_tunnel_ctl(route, 1) && !route->dcp->external && + route->dcp->dptxport[1].enabled; +} + struct dcp_typec_follow_context { struct dcp_typec_follow_slot slots[2]; struct drm_atomic_commit *state; @@ -2396,6 +2441,7 @@ static int dcp_typec_follow_decide(struct apple_dcp *dcp, struct drm_crtc *crtc, /* A tunnel's DP IN must reach the pipeline it is given. */ if (follow->action == DCP_FABRIC_FOLLOW_REFUSE || (from->tunnel && !dcp_typec_tunnel_ctl(to, from->tunnel_dpin)) || + (port->split_dcp == from->dcp && !dcp_follow_split_fits(to)) || (follow->action == DCP_FABRIC_FOLLOW_SWAP && holder->tunnel && !dcp_typec_tunnel_ctl(back, holder->tunnel_dpin))) return -EINVAL; @@ -2426,6 +2472,7 @@ static int dcp_follow_prepare(void *data, unsigned int index) if (!slot->set_active) return -ESTALE; } + dcp_follow_save_split(slot->from, slot); { struct drm_crtc *crtc = &slot->from->dcp->crtc->base; struct drm_crtc_state *old = drm_atomic_get_old_crtc_state(ctx->state, crtc); @@ -2468,6 +2515,8 @@ static int dcp_follow_release(struct apple_dcp_typec_route *route) ret = dcp_dptx_park(dcp); if (ret) return ret; + if (route->port->split_dcp == dcp) + dcp_tb_split_teardown_locked(route->port); apple_connector_edid_set_live(dcp->typec_connector, false); dcp_modes_begin_attachment(dcp); dcp->typec_connector = NULL; @@ -2539,6 +2588,14 @@ static int dcp_follow_activate(struct apple_dcp_typec_route *route, } if (slot->tunnel) dcp_tunnel_prepare(route, ctl); + /* + * DPTX port 1 connects with port 0, see dcp_dptx_connect_tile(). If + * the second tile cannot join, the display moves single-stream. + */ + if (slot->split && + dcp_tb_split_join_locked(slot->port, route, dcp_typec_tunnel_ctl(route, 1), + slot->split_set_active, slot->split_binding)) + dev_warn(route->dcp->dev, "tiled: second tile left behind on the move\n"); slot->port->owner = route; slot->port->preferred_route = route; return 0; @@ -3508,7 +3565,8 @@ static bool dcp_typec_hdmi_reclaim(struct apple_dcp *dcp) list_for_each_entry(route, &port->routes, port_link) { if (route == owner || !route->dcp->crtc || !dcp_typec_route_available(route) || - (owner->tunnel && !dcp_typec_tunnel_ctl(route, owner->tunnel_dpin))) + (owner->tunnel && !dcp_typec_tunnel_ctl(route, owner->tunnel_dpin)) || + (port->split_dcp == dcp && !dcp_follow_split_fits(route))) continue; target = route; break; @@ -3534,6 +3592,7 @@ static bool dcp_typec_hdmi_reclaim(struct apple_dcp *dcp) if (!slot.set_active) return false; } + dcp_follow_save_split(owner, &slot); to = target->dcp; dev_info(dcp->dev, "HDMI display takes its pipeline back, %s on %pOF moves to %s\n", connector ? connector->base.name : "the display", diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 83af14a5c47a1..4619d2d0bc952 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2035,6 +2035,16 @@ int dcp_dptx_park(struct apple_dcp *dcp) cancel_delayed_work(&dcp->placeholder_edid_wq); if (dcp->avep) av_service_disconnect(dcp); + if (READ_ONCE(dcp->split.active) && + dptx_service_available(&dcp->dptxport[1]) && + dcp->dptxport[1].connected) { + ret = dptxport_set_hpd(dcp->dptxport[1].service, false); + if (ret) + return ret; + ret = dcp_dptx_disconnect(dcp, 1); + if (ret) + return ret; + } if (dptx_service_available(&dcp->dptxport[0]) && dcp->dptxport[0].connected) { ret = dptxport_set_hpd(dcp->dptxport[0].service, false); if (ret) { From bf747320109ee2c24038f244d05a13845a4b5ecf Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:40:43 -0700 Subject: [PATCH 1659/2680] drm/apple: preserve tiled tunnel ownership across setup and release failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Balance both DP IN clock references, reject unsupported tile geometry, and prepare the second output before moving a tiled route. Keep failed runtime releases owned until teardown succeeds; unregister still detaches all callback pointers. [ryan: adapted to 7.2.10 — no Thunderbolt binding generation here (170-usb API): the split carries no generation and an existing split answers a repeated request with its ready state; the new dcp_tb_release_locked() keeps this tree's release order: dcp_dptx_disconnect_oob() (no dcp_dptx_disconnect_drained/dcp_tb_binding_drain/tb_retiring here), whose failure now returns with the route and binding still owned, and a failed route deactivation still clears the binding as before; the tile-geometry check moves into the SetTiledDisplayHints handler as in the source; dcp_dptx_connect() returns the tile reconnect result for non-H17P pipes; tools/asahi/tunnel-clock-tests.py and the dp-release test updates are left out: they extract phy/atc.c and upstream-shaped dcp functions not in this tree] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 89881421474a76a1724e4bebff88df68ec1a5dc9) --- drivers/gpu/drm/apple/dcp-fabric.c | 175 ++++++++++++++++++--------- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 9 +- drivers/gpu/drm/apple/dptxep.c | 10 +- 4 files changed, 129 insertions(+), 66 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 65e24ed4fd5e7..ef17c8de9c213 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -1718,18 +1718,14 @@ int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 unit, return -ENOENT; } if (unit) { - /* - * t602x has a pixel clock per DP IN; t600x runs one for both, - * which the second tile never stops on its own: dpin0 does. - */ + /* Each DP IN owns a reference, including on a shared clock. */ ret = 0; if (!dcp->split.active) { ret = -ENODEV; } else { if (!link_rate && dcp->split.xbar_up) dcp_dpxbar_link(dcp->split.xbar, false); - if (link_rate || dcp->hw.t6020_tunnel_flow) - ret = fn(phy, 1, link_rate); + ret = fn(phy, 1, link_rate); } symbol_put(apple_atc_dp_tunnel_rate); dcp->split.clock_ok = !ret && link_rate; @@ -1978,17 +1974,35 @@ static const struct dcp_fabric_attach_ops dcp_tb_attach_ops = { * own pipeline the second tile is refused ("no single tile support"). * Fabric lock held. */ -static void dcp_tb_split_teardown_locked(struct apple_dcp_typec_port *port) +static int dcp_tb_split_teardown_locked(struct apple_dcp_typec_port *port, + bool force) { struct apple_dcp *dcp = port->split_dcp; + int ret; lockdep_assert_held(&dcp_typec_fabric_lock); if (!dcp) - return; - if (dcp->dptxport[1].enabled) - dptxport_set_hpd(dcp->dptxport[1].service, false); - dcp_dptx_disconnect(dcp, 1); + return 0; + if (dcp->dptxport[1].enabled && dcp->dptxport[1].connected) { + ret = dptxport_set_hpd(dcp->dptxport[1].service, false); + if (ret && !force) + return ret; + } + ret = dcp_dptx_disconnect(dcp, 1); + if (ret && !force) + return ret; scoped_guard(mutex, &dcp->tb_lock) { + /* Retire the callback before dropping its clock reference. */ + dcp->split.active = false; + if (dcp->phy) { + typeof(apple_atc_dp_tunnel_rate) *stop = + symbol_get(apple_atc_dp_tunnel_rate); + + if (stop) { + stop(dcp->phy, 1, 0); + symbol_put(apple_atc_dp_tunnel_rate); + } + } if (dcp->split.xbar) { if (dcp->split.xbar_up) mux_control_deselect(dcp->split.xbar); @@ -1997,15 +2011,16 @@ static void dcp_tb_split_teardown_locked(struct apple_dcp_typec_port *port) MUX_IDLE_DISCONNECT); dcp_dpxbar_tunnel_select_source(dcp->split.xbar, -1); } - dcp->split.active = false; dcp->split.xbar = NULL; dcp->split.xbar_up = false; dcp->split.clock_ok = false; + dcp->split.ready = false; dcp->split.set_active = NULL; dcp->split.binding = NULL; } port->split_dcp = NULL; dev_info(dcp->dev, "tiled: dpin1 released from DPTX port 1\n"); + return 0; } /* @@ -2022,6 +2037,9 @@ static int dcp_tb_split_join_locked(struct apple_dcp_typec_port *port, struct apple_dcp *dcp = route->dcp; int ret, state = route->mux_index | 1; + if (!xbar || port->split_dcp || dcp->split.active) + return -EBUSY; + lockdep_assert_held(&dcp_typec_fabric_lock); /* T602X points the DP IN at its pipeline before DCP probes AUX */ ret = dcp_dpxbar_tunnel_select_source(xbar, state); @@ -2042,6 +2060,7 @@ static int dcp_tb_split_join_locked(struct apple_dcp_typec_port *port, dcp->split.clock_ok = false; dcp->split.set_active = set_active; dcp->split.binding = binding; + dcp->split.ready = false; dcp->split.active = true; } port->split_dcp = dcp; @@ -2063,7 +2082,7 @@ static int dcp_tb_split_locked(struct apple_dcp_typec_port *port, struct apple_dcp_typec_route *route = port->owner; struct mux_control *xbar; struct apple_dcp *dcp; - int ret; + int ret, release; lockdep_assert_held(&dcp_typec_fabric_lock); /* @@ -2071,10 +2090,10 @@ static int dcp_tb_split_locked(struct apple_dcp_typec_port *port, * for a split dpin1 is answered as done, a release tears it down. */ if (port->split_dcp) { + dcp = port->split_dcp; if (req->active) - return 0; - dcp_tb_split_teardown_locked(port); - return 0; + return dcp->split.ready ? 0 : -EAGAIN; + return dcp_tb_split_teardown_locked(port, false); } if (!req->active) return -EOPNOTSUPP; @@ -2098,13 +2117,11 @@ static int dcp_tb_split_locked(struct apple_dcp_typec_port *port, * topology for dpin0 shortly after its link comes up: wait for that, * but not for ever, as a sink may never send one. */ - if (!READ_ONCE(dcp->dptxport[0].tile_hint)) { + if (!smp_load_acquire(&dcp->dptxport[0].tile_hint)) { if (time_before(jiffies, port->tile_hint_deadline)) return -ENODEV; return -EOPNOTSUPP; } - if (dcp->dptxport[0].tiles_h * dcp->dptxport[0].tiles_v < 2) - return -EOPNOTSUPP; ret = dcp_tb_split_join_locked(port, route, xbar, req->set_active, req->binding); if (ret) @@ -2114,9 +2131,57 @@ static int dcp_tb_split_locked(struct apple_dcp_typec_port *port, if (ret) { dev_warn(dcp->dev, "tiled: DPTX port 1 connect failed: %d\n", ret); /* dpin0 keeps its route and stays a single-stream display */ - dcp_tb_split_teardown_locked(port); + release = dcp_tb_split_teardown_locked(port, false); + + if (release) + return release; + /* This is a failed tile setup, not an independent display. */ + return ret == -EOPNOTSUPP ? -EIO : ret; } - return ret; + scoped_guard(mutex, &dcp->tb_lock) + dcp->split.ready = true; + return 0; +} + +static int dcp_tb_release_locked(struct apple_dcp_typec_port *port, + unsigned int dpin) +{ + struct apple_dcp_typec_route **slot = dpin ? &port->secondary_owner : &port->owner; + struct apple_dcp *dcp; + int ret; + + /* the second tile goes before the route it shares */ + if (dpin == 0) { + ret = dcp_tb_split_teardown_locked(port, false); + if (ret) + return ret; + } + dcp = (*slot)->dcp; + /* A refused display release keeps the route and its binding owned. */ + if (port->hpd || dcp->typec_cable_connected) { + ret = dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + if (ret) + return ret; + } + ret = dcp_typec_route_deactivate(*slot); + if (ret) { + /* the caller's context is going away regardless */ + scoped_guard(mutex, &dcp->tb_lock) { + dcp->dptx_tunnel = false; + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + } + return ret; + } + *slot = NULL; + port->hpd = !!(port->owner || port->secondary_owner); + /* re-apply the next Type-C mux state in full */ + port->applied_valid = false; + if (dcp->hdmi_hpd && dcp->active && + gpiod_get_value_cansleep(dcp->hdmi_hpd)) + dcp_dptx_connect(dcp, 0); + dcp_typec_pipeline_freed(); + return 0; } static int dcp_tb_dispatch(void *data, const struct dcp_fabric_port *found) @@ -2125,7 +2190,6 @@ static int dcp_tb_dispatch(void *data, const struct dcp_fabric_port *found) struct apple_dcp_typec_port *port = container_of(found, struct apple_dcp_typec_port, core); struct apple_dcp_typec_route **slot; - struct apple_dcp *dcp; unsigned int dpin = ctx->dpin; int ret; @@ -2140,31 +2204,7 @@ static int dcp_tb_dispatch(void *data, const struct dcp_fabric_port *found) if (!*slot || !(*slot)->tunnel || (*slot)->tunnel_dpin != dpin) return 0; - /* the second tile goes before the route it shares */ - if (dpin == 0) - dcp_tb_split_teardown_locked(port); - dcp = (*slot)->dcp; - if (port->hpd || dcp->typec_cable_connected) - dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - ret = dcp_typec_route_deactivate(*slot); - if (ret) { - /* the caller's context is going away regardless */ - scoped_guard(mutex, &dcp->tb_lock) { - dcp->dptx_tunnel = false; - dcp->tb_dpin_set_active = NULL; - dcp->tb_dpin_ctx = NULL; - } - return ret; - } - *slot = NULL; - port->hpd = !!(port->owner || port->secondary_owner); - /* re-apply the next Type-C mux state in full */ - port->applied_valid = false; - if (dcp->hdmi_hpd && dcp->active && - gpiod_get_value_cansleep(dcp->hdmi_hpd)) - dcp_dptx_connect(dcp, 0); - dcp_typec_pipeline_freed(); - return 0; + return dcp_tb_release_locked(port, dpin); } { @@ -2443,7 +2483,9 @@ static int dcp_typec_follow_decide(struct apple_dcp *dcp, struct drm_crtc *crtc, (from->tunnel && !dcp_typec_tunnel_ctl(to, from->tunnel_dpin)) || (port->split_dcp == from->dcp && !dcp_follow_split_fits(to)) || (follow->action == DCP_FABRIC_FOLLOW_SWAP && holder->tunnel && - !dcp_typec_tunnel_ctl(back, holder->tunnel_dpin))) + !dcp_typec_tunnel_ctl(back, holder->tunnel_dpin)) || + (follow->action == DCP_FABRIC_FOLLOW_SWAP && + holder->port->split_dcp == holder->dcp && !dcp_follow_split_fits(back))) return -EINVAL; follow->port = port; follow->from = from; @@ -2515,8 +2557,11 @@ static int dcp_follow_release(struct apple_dcp_typec_route *route) ret = dcp_dptx_park(dcp); if (ret) return ret; - if (route->port->split_dcp == dcp) - dcp_tb_split_teardown_locked(route->port); + if (route->port->split_dcp == dcp) { + ret = dcp_tb_split_teardown_locked(route->port, false); + if (ret) + return ret; + } apple_connector_edid_set_live(dcp->typec_connector, false); dcp_modes_begin_attachment(dcp); dcp->typec_connector = NULL; @@ -2566,10 +2611,22 @@ static int dcp_follow_activate(struct apple_dcp_typec_route *route, struct mux_control *ctl = route->xbar; int ret; + /* Reserve both stream outputs before publishing the new main route. */ + if (slot->split) { + ret = dcp_tb_split_join_locked(slot->port, route, + dcp_typec_tunnel_ctl(route, 1), + slot->split_set_active, + slot->split_binding); + if (ret) + return ret; + } if (slot->tunnel) { ctl = dcp_typec_tunnel_ctl(route, slot->dpin); - if (!ctl) + if (!ctl) { + if (slot->split) + dcp_tb_split_teardown_locked(slot->port, false); return -EOPNOTSUPP; + } scoped_guard(mutex, &dcp->tb_lock) { dcp->tb_dpin_set_active = slot->set_active; dcp->tb_dpin_ctx = slot->binding; @@ -2578,6 +2635,8 @@ static int dcp_follow_activate(struct apple_dcp_typec_route *route, } ret = dcp_typec_route_activate(route, ctl); if (ret) { + if (slot->split) + dcp_tb_split_teardown_locked(slot->port, false); if (slot->tunnel) { scoped_guard(mutex, &dcp->tb_lock) { dcp->tb_dpin_set_active = NULL; @@ -2588,14 +2647,10 @@ static int dcp_follow_activate(struct apple_dcp_typec_route *route, } if (slot->tunnel) dcp_tunnel_prepare(route, ctl); - /* - * DPTX port 1 connects with port 0, see dcp_dptx_connect_tile(). If - * the second tile cannot join, the display moves single-stream. - */ - if (slot->split && - dcp_tb_split_join_locked(slot->port, route, dcp_typec_tunnel_ctl(route, 1), - slot->split_set_active, slot->split_binding)) - dev_warn(route->dcp->dev, "tiled: second tile left behind on the move\n"); + if (slot->split) { + scoped_guard(mutex, &dcp->tb_lock) + dcp->split.ready = true; + } slot->port->owner = route; slot->port->preferred_route = route; return 0; @@ -3032,7 +3087,7 @@ static void dcp_typec_route_unregister(void *data) if (port->preferred_route == route) port->preferred_route = NULL; if (port->split_dcp == route->dcp) - dcp_tb_split_teardown_locked(port); + dcp_tb_split_teardown_locked(port, true); if (port->owner == route) { struct apple_dcp *dcp = route->dcp; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 08cd43744a19b..d785a06a4e880 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -505,6 +505,7 @@ struct apple_dcp { */ struct { bool active; + bool ready; /* admitted second-stream setup */ struct mux_control *xbar; /* the port's dpin1 crossbar control */ int mux_state; /* dispextN, DPTX port 1 */ bool xbar_up; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 4619d2d0bc952..00d04546a5009 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1018,7 +1018,7 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable WRITE_ONCE(dcp->dptxport[port].tile_hint, false); dcp->dptxport[port].atcphy = dcp->phy; /* a tiled display's second half comes in on the port's dpin1 */ - dfp_port = port && dcp->split.active ? 2 : dcp->dptx_dfp_port; + dfp_port = port && READ_ONCE(dcp->split.active) ? 2 : dcp->dptx_dfp_port; ret = dptxport_validate_connection(dcp->dptxport[port].service, dfp_port, dcp->dptx_phy, dcp->dptx_die); @@ -1456,7 +1456,7 @@ void dcp_external_sink_irq(struct apple_dcp *dcp) * this pipeline. Bring it back whenever port 0 (re)connects, as DCP pairs * the tiles only while both ports are connected. */ -static void dcp_dptx_connect_tile(struct apple_dcp *dcp) +static int dcp_dptx_connect_tile(struct apple_dcp *dcp) { bool split; int ret; @@ -1464,10 +1464,11 @@ static void dcp_dptx_connect_tile(struct apple_dcp *dcp) scoped_guard(mutex, &dcp->tb_lock) split = dcp->split.active; if (!split) - return; + return 0; ret = dcp_dptx_connect(dcp, 1); if (ret) dev_warn(dcp->dev, "tiled: DPTX port 1 reconnect failed: %d\n", ret); + return ret; } int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) @@ -1477,7 +1478,7 @@ int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) if (!dcp_is_external_h17p(dcp)) { ret = dcp_dptx_connect_raw(dcp, port, NULL); if (!ret && !port) - dcp_dptx_connect_tile(dcp); + ret = dcp_dptx_connect_tile(dcp); return ret; } guard(mutex)(&dcp->external_lock); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index bc39064144597..3da358642cba2 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -630,6 +630,8 @@ static int dptxport_call_set_tiled_display_hint(struct apple_epic_service *servi if (reply_size < sizeof(*reply)) return -EINVAL; + if (dptx) + smp_store_release(&dptx->tile_hint, false); if (dptx && data_size >= DPTX_TILE_HINT_MIN_SIZE) { u32 h = get_unaligned_le32(hint + DPTX_TILE_HINT_TILES_H); u32 v = get_unaligned_le32(hint + DPTX_TILE_HINT_TILES_V); @@ -640,8 +642,12 @@ static int dptxport_call_set_tiled_display_hint(struct apple_epic_service *servi dptx->tiles_v = min(v, 255U); dptx->tile_x = min(x, 255U); dptx->tile_y = min(y, 255U); - WRITE_ONCE(dptx->tile_hint, true); - if (h * v > 1) + /* Only the two horizontal tiles carried by these two ports fit. */ + smp_store_release(&dptx->tile_hint, + h == 2 && v == 1 && x == dptx->unit && y == 0 && + get_unaligned_le32(hint + DPTX_TILE_HINT_WIDTH) && + get_unaligned_le32(hint + DPTX_TILE_HINT_HEIGHT)); + if (h > 1 || v > 1) dev_info(service->ep->dcp->dev, "DPTXPort: port %u carries tile (%u,%u) of a %ux%u tiled %ux%u display\n", dptx->unit, x, y, h, v, From 68b10e5e6ee65ccea5fbdac0f4e51f83fb271a4f Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:47:15 -0700 Subject: [PATCH 1660/2680] drm/apple: drain terminal Thunderbolt bindings before provider return MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Failed attach and detach release the provider reference. Clear their exact callbacks before return, retain failed hardware releases without callback pointers, and retry checked teardown before replacement. A retired second stream cannot be copied into a live CRTC move. [ryan: adapted to 7.2.10 — this tree has no Thunderbolt binding generation, dcp_tb_binding_drain() or tb_retiring (all part of the 170-usb binding API added later), so the intent is ported on the callback pointers: dcp_tb_finish_request() clears the callbacks of a failed attach or of any detach when they belong to that request (matched by context and set_active), a tunnel or split whose callbacks are gone counts as retired and its hardware release is retried before a new request (dcp_tb_binding_live()), and the split clock/crossbar/tile-reconnect/follow paths test split.set_active instead of split.generation; the dp-release host test updates are left out] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 479d414d8e6ad910b1ea24521794a18519cf3050) --- drivers/gpu/drm/apple/dcp-fabric.c | 93 ++++++++++++++++++++++++++++-- drivers/gpu/drm/apple/dcp.c | 3 +- 2 files changed, 89 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index ef17c8de9c213..7a866c85b1e12 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -1572,7 +1572,7 @@ static int dcp_tunnel_split_up_locked(struct apple_dcp *dcp) int ret; lockdep_assert_held(&dcp->tb_lock); - if (!dcp->split.active || !dcp->split.xbar) + if (!dcp->split.active || !dcp->split.set_active || !dcp->split.xbar) return -ENODEV; if (!dcp->split.clock_ok) { dev_warn(dcp->dev, "tiled: no DP tunnel pixel clock, dpin1 crossbar left down\n"); @@ -1720,7 +1720,7 @@ int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 unit, if (unit) { /* Each DP IN owns a reference, including on a shared clock. */ ret = 0; - if (!dcp->split.active) { + if (!dcp->split.active || !dcp->split.set_active) { ret = -ENODEV; } else { if (!link_rate && dcp->split.xbar_up) @@ -1967,6 +1967,13 @@ static const struct dcp_fabric_attach_ops dcp_tb_attach_ops = { .connect_oob = dcp_tb_connect_oob, }; +/* Does @dcp's tunnel still hold its Thunderbolt binding's callbacks? */ +static bool dcp_tb_binding_live(struct apple_dcp *dcp) +{ + guard(mutex)(&dcp->tb_lock); + return !!dcp->tb_dpin_set_active; +} + /* * tiled_split: take the second half of a tiled display (dpin1) onto DPTX * port 1 of the pipeline that already drives dpin0 on this port. DCP pairs @@ -2086,9 +2093,16 @@ static int dcp_tb_split_locked(struct apple_dcp_typec_port *port, lockdep_assert_held(&dcp_typec_fabric_lock); /* - * No Thunderbolt binding generation in this tree: a repeated request - * for a split dpin1 is answered as done, a release tears it down. + * A retired provider leaves only hardware ownership to retry. This + * tree has no Thunderbolt binding generation: a split whose callbacks + * were cleared is the retired one. */ + if (port->split_dcp && !port->split_dcp->split.set_active) { + ret = dcp_tb_split_teardown_locked(port, false); + if (ret || !req->active) + return ret; + } + /* A repeated request for a split dpin1 is answered by its state. */ if (port->split_dcp) { dcp = port->split_dcp; if (req->active) @@ -2184,7 +2198,7 @@ static int dcp_tb_release_locked(struct apple_dcp_typec_port *port, return 0; } -static int dcp_tb_dispatch(void *data, const struct dcp_fabric_port *found) +static int dcp_tb_dispatch_request(void *data, const struct dcp_fabric_port *found) { struct dcp_tb_attach_context *ctx = data; struct apple_dcp_typec_port *port = @@ -2200,6 +2214,13 @@ static int dcp_tb_dispatch(void *data, const struct dcp_fabric_port *found) if (ret != -EOPNOTSUPP) return ret; } + /* A retired binding leaves only hardware ownership to retry. */ + if (*slot && (*slot)->tunnel && (*slot)->tunnel_dpin == dpin && + !dcp_tb_binding_live((*slot)->dcp)) { + ret = dcp_tb_release_locked(port, dpin); + if (ret || !ctx->active) + return ret; + } if (!ctx->active) { if (!*slot || !(*slot)->tunnel || (*slot)->tunnel_dpin != dpin) @@ -2246,6 +2267,63 @@ static int dcp_tb_dispatch(void *data, const struct dcp_fabric_port *found) return ret; } +/* + * Failed attach and every detach return before the provider frees its binding. + * This tree has no binding generation or drain: a request's callbacks are + * matched by their context and cleared, and the hardware ownership they leave + * is retried by the next request (see dcp_tb_binding_live()). + */ +static bool dcp_tb_request_matches(int (*set_active)(void *binding, bool active), + void *binding, + const struct dcp_tb_attach_context *request) +{ + if (!request->binding) + return !request->active; + return binding == request->binding && set_active == request->set_active; +} + +static void dcp_tb_finish_request(struct apple_dcp_typec_port *port, + const struct dcp_tb_attach_context *request) +{ + struct apple_dcp_typec_route *route = request->dpin ? + port->secondary_owner : port->owner; + struct apple_dcp *dcp; + + if (request->dpin == 1 && port->split_dcp) { + dcp = port->split_dcp; + scoped_guard(mutex, &dcp->tb_lock) { + if (dcp_tb_request_matches(dcp->split.set_active, + dcp->split.binding, request)) { + dcp->split.set_active = NULL; + dcp->split.binding = NULL; + dcp->split.ready = false; + } + } + } + if (!route || !route->tunnel) + return; + dcp = route->dcp; + scoped_guard(mutex, &dcp->tb_lock) { + if (dcp_tb_request_matches(dcp->tb_dpin_set_active, + dcp->tb_dpin_ctx, request)) { + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + } + } +} + +static int dcp_tb_dispatch(void *data, const struct dcp_fabric_port *found) +{ + struct dcp_tb_attach_context *request = data; + struct apple_dcp_typec_port *port = + container_of(found, struct apple_dcp_typec_port, core); + int ret = dcp_tb_dispatch_request(data, found); + + if (ret || !request->active) + dcp_tb_finish_request(port, request); + return ret; +} + int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, bool active, int (*set_active)(void *ctx, bool active), void *ctx) @@ -2320,8 +2398,11 @@ static void dcp_follow_save_split(struct apple_dcp_typec_route *route, if (route->port->split_dcp != dcp) return; - slot->split = true; scoped_guard(mutex, &dcp->tb_lock) { + /* A retired second stream cannot be carried into the move. */ + if (!dcp->split.set_active) + return; + slot->split = true; slot->split_set_active = dcp->split.set_active; slot->split_binding = dcp->split.binding; } diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 00d04546a5009..1bf8c5a93bc0b 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1462,7 +1462,8 @@ static int dcp_dptx_connect_tile(struct apple_dcp *dcp) int ret; scoped_guard(mutex, &dcp->tb_lock) - split = dcp->split.active; + split = dcp->split.active && dcp->split.ready && + dcp->split.set_active; if (!split) return 0; ret = dcp_dptx_connect(dcp, 1); From 8804710e24cb51a1f1e30a697c96f23f798ca7cf Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 10 Oct 2026 22:06:57 -0700 Subject: [PATCH 1661/2680] drm/apple: select clockable T6030 Thunderbolt display routes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Select source 0 for T6030 DP IN0 before claiming a display engine. Apply clock eligibility to ordered pairing and route moves while retaining direct DisplayPort selection. Enable the existing capacity retry and suspend admission gates for T6030 handoffs. Unsupported DP INs fail instead of waiting indefinitely; occupied supported engines can recover when freed. [ryan: adapted to 7.2.10 — DCP side only: the drivers/thunderbolt half (T6030 source-0 select, capacity retry and suspend admission) is left to 170-usb and handed over to 170-usb; the clock eligibility composes with this tree's dcp_t602x_tunnel_route() native path and safe_held ownership; tools/asahi/tunnel-route fixture gains a dcp_typec_safe_held() stub for this tree's snapshot (passes)] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit a29fbe8f9244d3ab42fae7aa188c62e980d40a82) --- drivers/gpu/drm/apple/dcp-fabric-core.c | 35 +++++++- drivers/gpu/drm/apple/dcp-fabric-core.h | 3 + drivers/gpu/drm/apple/dcp-fabric-test.c | 107 ++++++++++++++++++++++++ drivers/gpu/drm/apple/dcp-fabric.c | 6 ++ tools/asahi/tunnel-route-tests.py | 67 +++++++++++++++ tools/asahi/tunnel-route/fixture.c | 93 ++++++++++++++++++++ 6 files changed, 307 insertions(+), 4 deletions(-) create mode 100644 tools/asahi/tunnel-route-tests.py create mode 100644 tools/asahi/tunnel-route/fixture.c diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.c b/drivers/gpu/drm/apple/dcp-fabric-core.c index 1ba04abb6131b..6a857ba15e7f1 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.c +++ b/drivers/gpu/drm/apple/dcp-fabric-core.c @@ -26,6 +26,13 @@ bool dcp_fabric_t6020_flow(bool usb4, bool soc_support, bool connector_wired) } EXPORT_SYMBOL_GPL(dcp_fabric_t6020_flow); +u8 dcp_fabric_tunnel_clock_blocked(bool t6030, unsigned int source) +{ + /* T6030 currently clocks only source 0 feeding DP IN0. */ + return t6030 ? BIT(1) | (source ? BIT(0) : 0) : 0; +} +EXPORT_SYMBOL_GPL(dcp_fabric_tunnel_clock_blocked); + int dcp_fabric_tunnel_request(const struct dcp_fabric_port *ports, unsigned long key, unsigned int dpin, const struct dcp_fabric_port **found) @@ -211,7 +218,7 @@ static void dcp_fabric_plan_pipeline(struct dcp_fabric_pipeline *pipeline, for (stream = 0; stream < 2; stream++) { for (port = ports; port; port = port->next) { struct dcp_fabric_route *route; - bool wants; + bool wants, tunnel; if (stream) wants = port->owner[1] || @@ -227,7 +234,10 @@ static void dcp_fabric_plan_pipeline(struct dcp_fabric_pipeline *pipeline, BIT(pipeline->crtc_index))) continue; route = dcp_fabric_port_route(port, pipeline); - if (!route) + tunnel = stream || (port->owner[0] && port->owner[0]->tunnel) || + (port == arriving && dpin == 0); + if (!route || (tunnel && + (route->tunnel_clock_blocked & BIT(stream)))) continue; port->plan->target[stream] = route; return; @@ -293,12 +303,22 @@ dcp_fabric_tunnel_candidate(const struct dcp_fabric_port *port, { struct dcp_fabric_route *route, *best = NULL; unsigned int best_score = UINT_MAX; - bool waiting = false; + bool waiting = false, clock_blocked = false, clock_capable = false; + + if (dpin > 1) { + *error = -EINVAL; + return NULL; + } for (route = port->routes; route; route = route->next) { const struct dcp_fabric_pipeline *pipeline = route->pipeline; unsigned int score; + if (route->tunnel_clock_blocked & BIT(dpin)) { + clock_blocked = true; + continue; + } + clock_capable = true; if ((ordered && route != planned) || !dcp_fabric_available(pipeline, policy)) continue; @@ -323,7 +343,8 @@ dcp_fabric_tunnel_candidate(const struct dcp_fabric_port *port, best_score = score; } } - *error = best ? 0 : waiting ? -EAGAIN : -EBUSY; + *error = best ? 0 : waiting ? -EAGAIN : + clock_blocked && !clock_capable ? -EOPNOTSUPP : -EBUSY; return best; } EXPORT_SYMBOL_GPL(dcp_fabric_tunnel_candidate); @@ -355,6 +376,12 @@ dcp_fabric_follow(const struct dcp_fabric_route *from, return DCP_FABRIC_FOLLOW_REFUSE; if (from == to) return DCP_FABRIC_FOLLOW_STAY; + if ((from->tunnel && (from->dpin > 1 || + (to->tunnel_clock_blocked & BIT(from->dpin)))) || + (holder && holder->tunnel && holder_back && + (holder->dpin > 1 || + (holder_back->tunnel_clock_blocked & BIT(holder->dpin))))) + return DCP_FABRIC_FOLLOW_REFUSE; pipeline = to->pipeline; if (!pipeline->bound || pipeline->fixed_busy || pipeline->terminal || !pipeline->services_ready) diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.h b/drivers/gpu/drm/apple/dcp-fabric-core.h index 2216c3f8e4888..cd211826a273a 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.h +++ b/drivers/gpu/drm/apple/dcp-fabric-core.h @@ -37,6 +37,8 @@ struct dcp_fabric_route { struct dcp_fabric_pipeline *pipeline; bool tunnel; unsigned int dpin; + /* DP IN adapters this source cannot clock; direct DP is unaffected. */ + u8 tunnel_clock_blocked; }; struct dcp_fabric_plan { @@ -73,6 +75,7 @@ enum dcp_fabric_wiring { enum dcp_fabric_wiring dcp_fabric_wiring(bool dpin0, bool dpin1, bool legacy, unsigned int endpoints); bool dcp_fabric_t6020_flow(bool usb4, bool soc_support, bool connector_wired); +u8 dcp_fabric_tunnel_clock_blocked(bool t6030, unsigned int source); enum dcp_fabric_capacity_action { DCP_FABRIC_PROMOTE, diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c index 16fada29559a5..0bb37b13a1639 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-test.c +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -1904,7 +1904,114 @@ static void fabric_reclaim_transaction_test(struct kunit *test) KUNIT_EXPECT_EQ(test, lost.events[5], 8U); } +static void fabric_t6030_init(struct fabric_fixture *f) +{ + unsigned int p; + + fabric_init(f, false); + for (p = 0; p < 3; p++) { + f->route[p][0].tunnel_clock_blocked = + dcp_fabric_tunnel_clock_blocked(true, 0); + f->route[p][1].tunnel_clock_blocked = + dcp_fabric_tunnel_clock_blocked(true, 2); + } +} + +static void fabric_t6030_clock_routes_test(struct kunit *test) +{ + struct fabric_fixture f; + struct dcp_fabric_route *route; + int error; + + fabric_t6030_init(&f); + KUNIT_ASSERT_EQ(test, fabric_tunnel(&f, 0, 0), 0); + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], &f.route[0][0]); + fabric_release(&f, 0, 0); + /* Unusable engines cannot override readiness of the supported engine. */ + f.pipeline[1].terminal = true; + f.pipeline[1].services_ready = false; + KUNIT_ASSERT_EQ(test, fabric_tunnel(&f, 0, 0), 0); + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], &f.route[0][0]); + fabric_release(&f, 0, 0); + f.pipeline[1].terminal = false; + f.pipeline[1].services_ready = true; + /* The Type-C-only engine wins direct DP, but cannot clock a tunnel. */ + KUNIT_ASSERT_EQ(test, fabric_direct(&f, 1), 0); + KUNIT_EXPECT_PTR_EQ(test, f.port[1].owner[0], &f.route[1][1]); + KUNIT_ASSERT_EQ(test, fabric_tunnel(&f, 0, 0), 0); + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], &f.route[0][0]); + fabric_release(&f, 0, 0); + fabric_release(&f, 1, 0); + /* HDMI owns source 0: do not hand off the unclockable source 2. */ + f.pipeline[0].fixed_busy = true; + KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 0), -EBUSY); + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], NULL); + KUNIT_EXPECT_FALSE(test, f.pipeline[1].owned); + f.pipeline[0].fixed_busy = false; + f.pipeline[0].services_ready = false; + KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 0), -EAGAIN); + f.pipeline[0].services_ready = true; + KUNIT_ASSERT_EQ(test, fabric_tunnel(&f, 0, 0), 0); + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], &f.route[0][0]); + fabric_release(&f, 0, 0); + KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 1), -EOPNOTSUPP); + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[1], NULL); + f.port[0].routes = &f.route[0][1]; + KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 0), -EOPNOTSUPP); + f.port[0].routes = NULL; + KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 0), -EBUSY); + route = dcp_fabric_tunnel_candidate(&f.port[0], &f.policy, NULL, + false, 2, true, &error); + KUNIT_EXPECT_PTR_EQ(test, route, NULL); + KUNIT_EXPECT_EQ(test, error, -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_fabric_tunnel_clock_blocked(false, 0), (u8)0); + KUNIT_EXPECT_EQ(test, dcp_fabric_tunnel_clock_blocked(false, 2), (u8)0); +} + +static void fabric_t6030_plan_test(struct kunit *test) +{ + struct fabric_fixture f; + + fabric_t6030_init(&f); + dcp_fabric_plan(f.pipeline, f.port, &f.port[0], 0); + KUNIT_EXPECT_PTR_EQ(test, f.plan[0].target[0], &f.route[0][0]); + f.pipeline[0].fixed_busy = true; + dcp_fabric_plan(f.pipeline, f.port, &f.port[0], 0); + KUNIT_EXPECT_PTR_EQ(test, f.plan[0].target[0], NULL); + /* The same available source remains usable by a direct USB-C display. */ + f.port[1].wanted = true; + f.port[1].hpd = true; + dcp_fabric_plan(f.pipeline, f.port, &f.port[0], 0); + KUNIT_EXPECT_PTR_EQ(test, f.plan[1].target[0], &f.route[1][1]); + dcp_fabric_plan(f.pipeline, f.port, &f.port[0], 1); + KUNIT_EXPECT_PTR_EQ(test, f.plan[0].target[1], NULL); +} + +static void fabric_t6030_follow_test(struct kunit *test) +{ + struct fabric_fixture f; + + fabric_t6030_init(&f); + f.route[0][0].tunnel = true; + f.route[0][0].dpin = 0; + KUNIT_EXPECT_EQ(test, dcp_fabric_follow(&f.route[0][0], &f.route[0][1], + NULL, NULL, false, &f.policy), DCP_FABRIC_FOLLOW_REFUSE); + /* Direct streams do not acquire a tunnel clock restriction. */ + f.route[0][0].tunnel = false; + KUNIT_EXPECT_EQ(test, dcp_fabric_follow(&f.route[0][0], &f.route[0][1], + NULL, NULL, false, &f.policy), DCP_FABRIC_FOLLOW_MOVE); + /* A swap must also preserve the holder's clockable source. */ + f.route[1][0].tunnel = true; + f.route[1][0].dpin = 0; + KUNIT_EXPECT_EQ(test, dcp_fabric_follow(&f.route[0][1], &f.route[0][0], + &f.route[1][0], &f.route[1][1], true, &f.policy), + DCP_FABRIC_FOLLOW_REFUSE); +} + static struct kunit_case fabric_tests[] = { + KUNIT_CASE(fabric_t6030_clock_routes_test), + KUNIT_CASE(fabric_t6030_plan_test), + KUNIT_CASE(fabric_t6030_follow_test), KUNIT_CASE(fabric_reclaim_transaction_test), KUNIT_CASE_PARAM(fabric_shared_wiring_test, fabric_wiring_gen_params), KUNIT_CASE(fabric_rebind_presence_test), diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 7a866c85b1e12..a99561e565fc9 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -851,6 +851,8 @@ static void dcp_fabric_snapshot_port(struct apple_dcp_typec_port *port, route->core.pipeline = pipeline; route->core.tunnel = route->tunnel; route->core.dpin = route->tunnel_dpin; + route->core.tunnel_clock_blocked = dcp_fabric_tunnel_clock_blocked( + of_machine_is_compatible("apple,t6030"), route->mux_index); *tail = &route->core; tail = &route->core.next; } @@ -1813,6 +1815,10 @@ struct dcp_tb_attach_context { static struct mux_control * dcp_typec_tunnel_ctl(struct apple_dcp_typec_route *route, unsigned int dpin) { + if (dpin > 1 || + (dcp_fabric_tunnel_clock_blocked(of_machine_is_compatible("apple,t6030"), + route->mux_index) & BIT(dpin))) + return NULL; if (route->dpin[dpin]) return route->dpin[dpin]; /* Legacy DT ABI: unnamed DPIN controls share the DP-alt chip. */ diff --git a/tools/asahi/tunnel-route-tests.py b/tools/asahi/tunnel-route-tests.py new file mode 100644 index 0000000000000..b1d5574ea4b50 --- /dev/null +++ b/tools/asahi/tunnel-route-tests.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Host controls for display-engine clock eligibility at the DCP adapter.""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +FUNCTIONS = { + 'drivers/gpu/drm/apple/dcp-fabric-core.c': ['dcp_fabric_tunnel_clock_blocked'], + 'drivers/gpu/drm/apple/dcp-fabric.c': [ + 'dcp_fabric_snapshot_port', 'dcp_typec_tunnel_ctl'], +} + + +def function(source, name): + match = re.search(r'^(?:static\s+)?(?:void|u8|struct mux_control \*)\s*' + + name + r'\([^;]*?\)\n\{', source, re.M | re.S) + if not match: + raise ValueError(name) + pos = source.index('{', match.start()) + 1 + depth = 1 + while depth: + depth += (source[pos] == '{') - (source[pos] == '}') + pos += 1 + return source[match.start():pos] + '\n' + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--out', type=Path) + args = parser.parse_args() + bodies = [function((ROOT / path).read_text(), name) + for path, names in FUNCTIONS.items() for name in names] + header = (ROOT / 'drivers/gpu/drm/apple/dcp-fabric-core.h').read_text() + header = re.sub(r'^#include.*\n', '', header, flags=re.M) + with tempfile.TemporaryDirectory(prefix='tunnel-routes-') as temporary: + out = args.out or Path(temporary) + out.mkdir(parents=True, exist_ok=True) + code = (ROOT / 'tools/asahi/tunnel-route/fixture.c').read_text().replace( + '/* CORE_HEADER */', header).replace( + '/* PRODUCTION_FUNCTIONS */', '\n'.join(bodies)) + source = out / 'production.c' + source.write_text(code) + binary = out / 'production' + subprocess.run(['cc', '-std=gnu11', '-Wall', '-Werror', + str(source), '-o', str(binary)], check=True) + run = subprocess.run([str(binary)], capture_output=True, text=True) + receipt = { + 'bodies': {name: hashlib.sha256(body.encode()).hexdigest() + for name, body in zip( + [n for names in FUNCTIONS.values() for n in names], bodies)}, + 'translation_sha256': hashlib.sha256(code.encode()).hexdigest(), + 'exit': run.returncode, 'stdout': run.stdout, 'stderr': run.stderr, + } + (out / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + print(run.stdout, end='') + if run.returncode: + raise SystemExit(run.returncode) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/tunnel-route/fixture.c b/tools/asahi/tunnel-route/fixture.c new file mode 100644 index 0000000000000..63b43a4e3ffb4 --- /dev/null +++ b/tools/asahi/tunnel-route/fixture.c @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint32_t u32; +typedef uint64_t u64; +#define BIT(n) (1U << (n)) +#define lockdep_assert_held(x) ((void)0) +#define list_for_each_entry(r, head, link) \ + for ((r) = *(head); (r); (r) = (r)->next) +/* CORE_HEADER */ +struct mux_chip; +struct mux_control { struct mux_chip *chip; }; +struct mux_chip { struct mux_control mux[3]; unsigned int controllers; }; +struct apple_connector { u32 candidate_crtcs; }; +struct apple_crtc { struct { unsigned int index; } base; }; +struct apple_dcp { + struct dcp_fabric_pipeline fabric; + struct apple_crtc *crtc; + bool tb_retiring, external; + void *fixed_phy, *active_typec_route; +}; +struct apple_dcp_typec_route { + struct apple_dcp_typec_route *next; + struct apple_dcp *dcp; + struct dcp_fabric_route core; + bool tunnel; + unsigned int tunnel_dpin, mux_index; + struct mux_control *xbar, *dpin[2]; +}; +struct apple_dcp_typec_port { + struct dcp_fabric_port core; + struct dcp_fabric_plan plan; + struct apple_dcp_typec_route *routes, *owner, *secondary_owner, *preferred_route; + struct apple_connector *connector, *secondary_connector; + bool dp_wanted, dp_hpd; +}; +static bool t6030; +static bool of_machine_is_compatible(const char *compatible) +{ + assert(!strcmp(compatible, "apple,t6030")); + return t6030; +} +#define drm_crtc_index(crtc) ((crtc)->index) +static bool dcp_typec_route_busy(struct apple_dcp_typec_route *r) { return false; } +static bool dcp_typec_tunnel_held(struct apple_dcp *d) { return false; } +static bool dcp_typec_safe_held(struct apple_dcp *d) { return false; } +static enum dcp_fabric_presence_state dcp_hdmi_presence(struct apple_dcp *d) +{ + return DCP_FABRIC_ABSENT; +} +static bool dcpext_scanout_terminal(struct apple_dcp *d) { return false; } +static bool dcp_tb_services_ready(struct apple_dcp *d) { return true; } +/* PRODUCTION_FUNCTIONS */ +int main(void) +{ + struct mux_chip chip = { .controllers = 3 }; + struct apple_crtc crtc = { .base.index = 1 }; + struct apple_dcp dcp = { .crtc = &crtc }; + struct apple_dcp_typec_route source0 = { .dcp = &dcp, .xbar = &chip.mux[0] }; + struct apple_dcp_typec_route source2 = { + .dcp = &dcp, .xbar = &chip.mux[0], .mux_index = 2, + }; + struct apple_dcp_typec_port port = { .routes = &source0 }; + source0.next = &source2; + for (unsigned int i = 0; i < 3; i++) chip.mux[i].chip = &chip; + for (unsigned int soc = 0; soc < 2; soc++) { + t6030 = soc; + for (unsigned int named = 0; named < 2; named++) { + source0.dpin[0] = source2.dpin[0] = named ? &chip.mux[1] : NULL; + source0.dpin[1] = source2.dpin[1] = named ? &chip.mux[2] : NULL; + dcp_fabric_snapshot_port(&port, true); + assert(port.core.routes == &source0.core); + assert(source0.core.next == &source2.core); + assert(!source2.core.next); + assert(source0.core.tunnel_clock_blocked == (soc ? BIT(1) : 0)); + assert(source2.core.tunnel_clock_blocked == (soc ? BIT(0) | BIT(1) : 0)); + assert(dcp_typec_tunnel_ctl(&source0, 0) == &chip.mux[1]); + assert(dcp_typec_tunnel_ctl(&source0, 1) == (soc ? NULL : &chip.mux[2])); + assert(dcp_typec_tunnel_ctl(&source2, 0) == (soc ? NULL : &chip.mux[1])); + assert(dcp_typec_tunnel_ctl(&source2, 1) == (soc ? NULL : &chip.mux[2])); + assert(!dcp_typec_tunnel_ctl(&source0, 2)); + /* Late migration checks the real route, even with a stale snapshot. */ + source2.core.tunnel_clock_blocked = 0; + assert(dcp_typec_tunnel_ctl(&source2, 0) == (soc ? NULL : &chip.mux[1])); + } + } + puts("PASS: DCP snapshot, named/legacy controls, invalid DP IN, stale snapshot"); + return 0; +} From a3fd032c8e3125cecbbbf4bc6cb785752140ed3c Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:09:06 -0700 Subject: [PATCH 1662/2680] drm/apple: recheck external display readiness before submitting swaps Check the current description generation, mode and connection immediately before each external swap RPC is submitted. Retry admission after pending callbacks are drained. An unsubmitted readiness veto leaves the session usable; submitted transport failures retain session failure and framebuffer protection. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit a85f8eee2fd54ed9e7237f25d1f085e9d7848111) --- drivers/gpu/drm/apple/iomfb_v14_7.c | 55 ++++++++++++++++++++---- drivers/gpu/drm/apple/iomfb_v14_7_link.c | 27 ++++++++++-- drivers/gpu/drm/apple/iomfb_v14_7_link.h | 7 +++ 3 files changed, 77 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index ca8b6f18d6bb7..4cbe089b453e5 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -231,6 +231,7 @@ struct apple_dcp_v14 { u64 chunk_generation; /* External: the parts of the display's description published now. */ unsigned int described; + u64 description_generation; wait_queue_head_t described_wait; /* Boot framebuffer and native panel timing (notch rows included). */ @@ -301,17 +302,19 @@ static void dcp_v14_stopped_work(struct work_struct *work) } /* Called with the lock held, or from a callback. */ -static int dcp_v14_call(struct apple_dcp_v14 *v14, u32 tag, const void *in, u32 in_size, - void *out, u32 out_size, u32 completion) +static int dcp_v14_call_guarded(struct apple_dcp_v14 *v14, u32 tag, const void *in, u32 in_size, + void *out, u32 out_size, u32 completion, + dcp_v14_admit_fn admit, void *admit_cookie) { + bool vetoed; int ret; if (v14->failed) return -EIO; dev_dbg(v14->dev, "call %#x in %u out %u\n", tag, in_size, out_size); - ret = dcp_v14_link_call(&v14->link, tag, in, in_size, out, out_size, completion, - dcp_v14_callback, v14); - if (ret) { + ret = dcp_v14_link_call_guarded(&v14->link, tag, in, in_size, out, out_size, completion, + dcp_v14_callback, v14, admit, admit_cookie, &vetoed); + if (ret && !vetoed) { v14->failed = true; dev_err(v14->dev, "DCP call %#x failed: %d; recovery requires a reboot\n", tag, ret); @@ -320,6 +323,13 @@ static int dcp_v14_call(struct apple_dcp_v14 *v14, u32 tag, const void *in, u32 return ret; } +static int dcp_v14_call(struct apple_dcp_v14 *v14, u32 tag, const void *in, u32 in_size, + void *out, u32 out_size, u32 completion) +{ + return dcp_v14_call_guarded(v14, tag, in, in_size, out, out_size, + completion, NULL, NULL); +} + /* A call with an optional u32 input of 1 and an optional u32 result to check. */ static int dcp_v14_simple_call(struct apple_dcp_v14 *v14, u32 tag, bool input, bool reply, bool expected) @@ -680,6 +690,7 @@ static void dcp_v14_external_published(struct apple_dcp_v14 *v14, const char *ke raw = &v14->raw[i]; if (!removed && !raw) return; + v14->description_generation++; /* The timings count once they parse; see below. */ if (removed) v14->described &= ~part; @@ -2451,6 +2462,31 @@ static int dcp_v14_ctm_locked(struct apple_dcp_v14 *v14, return 0; } +static bool dcp_v14_external_described(struct apple_dcp_v14 *v14); + +struct dcp_v14_swap_admission { + struct apple_dcp_v14 *v14; + u64 generation; +}; + +static bool dcp_v14_swap_admit(void *cookie) +{ + struct dcp_v14_swap_admission *admission = cookie; + struct apple_dcp_v14 *v14 = admission->v14; + struct apple_dcp *dcp = READ_ONCE(v14->dcp); + struct apple_connector *connector; + + if (!v14->external) + return true; + if (!dcp || READ_ONCE(v14->failed) || !READ_ONCE(v14->powered) || + !READ_ONCE(v14->mode_live) || + admission->generation != v14->description_generation || + !dcp_v14_external_described(v14) || !READ_ONCE(dcp->mode_state.valid)) + return false; + connector = READ_ONCE(dcp->connector); + return connector && READ_ONCE(connector->connected); +} + /* * Swap start, then the swap; returns once the firmware completed that swap. * @ctm_state, if set, carries the colour matrix to send first. @@ -2459,6 +2495,7 @@ static int dcp_v14_swap(struct apple_dcp_v14 *v14, const u8 *surface, u64 iova, u32 width, u32 height, u32 dst_y, const struct drm_crtc_state *ctm_state) { + struct dcp_v14_swap_admission admission = { .v14 = v14 }; __le32 start[4] = {}, started[2] = {}, result[3]; u8 *swap; u32 id; @@ -2472,12 +2509,14 @@ static int dcp_v14_swap(struct apple_dcp_v14 *v14, const u8 *surface, u64 iova, return -ENOMEM; mutex_lock(&v14->lock); + admission.generation = v14->description_generation; /* * A colour matrix the firmware refuses costs night light, not the * frame: present anyway, and stop sending it (ctm_disabled). */ dcp_v14_ctm_locked(v14, ctm_state); - ret = dcp_v14_call(v14, A(406), start, sizeof(start), started, sizeof(started), 0); + ret = dcp_v14_call_guarded(v14, A(406), start, sizeof(start), started, sizeof(started), + 0, dcp_v14_swap_admit, &admission); if (!ret && le32_to_cpu(started[1])) ret = -EIO; id = le32_to_cpu(started[0]); @@ -2486,8 +2525,8 @@ static int dcp_v14_swap(struct apple_dcp_v14 *v14, const u8 *surface, u64 iova, if (!ret) { dcp_v14_encode_swap(swap, id, DCP_V14_BLACK, surface, iova, width, height, dst_y, DCP_V14_LAYER); - ret = dcp_v14_call(v14, A(407), swap, DCP_V14_SWAP_SIZE, result, sizeof(result), - id); + ret = dcp_v14_call_guarded(v14, A(407), swap, DCP_V14_SWAP_SIZE, result, sizeof(result), + id, dcp_v14_swap_admit, &admission); /* The swap status sits at byte 5 of the reply. */ if (!ret && get_unaligned_le32((u8 *)result + 5)) ret = -EIO; diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_link.c b/drivers/gpu/drm/apple/iomfb_v14_7_link.c index 27c3bff046812..e4c2c3fc3fd25 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7_link.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7_link.c @@ -229,7 +229,8 @@ void dcp_v14_link_receive(struct dcp_v14_link *link, u64 message) * call goes out on the stream of @parent, the callback its caller handles. */ static int link_submit(struct dcp_v14_link *link, const struct apple_dcp_link_rpc_header *rpc, - u32 size, bool nested, u64 parent) + u32 size, bool nested, u64 parent, + dcp_v14_admit_fn admit, void *admit_cookie, bool *vetoed) { struct apple_dcp_link_stream_layout layout; u32 total, offset = 0, input_size, output_size; @@ -273,6 +274,12 @@ static int link_submit(struct dcp_v14_link *link, const struct apple_dcp_link_rp ret = -EMSGSIZE; goto out; } + /* No callback can run or enter the queue between this check and send. */ + if (admit && !admit(admit_cookie)) { + *vetoed = true; + ret = -EAGAIN; + goto out; + } memcpy(link->rpc + layout.local_offset + offset, rpc, size); memset(link->rpc + layout.local_offset + offset + sizeof(*rpc) + input_size, 0, output_size); @@ -402,9 +409,10 @@ int dcp_v14_link_pump(struct dcp_v14_link *link, unsigned long timeout, * Must not be called concurrently with itself or dcp_v14_link_pump(), except * from inside a callback handler. */ -int dcp_v14_link_call(struct dcp_v14_link *link, u32 tag, const void *input, +int dcp_v14_link_call_guarded(struct dcp_v14_link *link, u32 tag, const void *input, u32 input_size, void *output, u32 output_size, - u32 completion_id, dcp_v14_callback_fn callback, void *cookie) + u32 completion_id, dcp_v14_callback_fn callback, void *cookie, + dcp_v14_admit_fn admit, void *admit_cookie, bool *vetoed) { struct apple_dcp_link_rpc_header *packet; unsigned long deadline = jiffies + DCP_V14_CALL_TIMEOUT; @@ -413,6 +421,7 @@ int dcp_v14_link_call(struct dcp_v14_link *link, u32 tag, const void *input, bool nested, replied = false; int ret; + *vetoed = false; ret = apple_dcp_link_rpc_payload_size(input_size, output_size, &size); if (ret || size > APPLE_DCP_LINK_STREAM_BUFFER_SIZE) return -EMSGSIZE; @@ -455,7 +464,7 @@ int dcp_v14_link_call(struct dcp_v14_link *link, u32 tag, const void *input, expected |= parent & APPLE_DCP_LINK_MSG_REMOTE; } mutex_unlock(&link->lock); - ret = link_submit(link, packet, size, nested, parent); + ret = link_submit(link, packet, size, nested, parent, admit, admit_cookie, vetoed); /* A callback arrived between the drain and the submit. */ if (ret == -EBUSY && !nested && time_before(jiffies, deadline)) goto retry; @@ -504,3 +513,13 @@ int dcp_v14_link_call(struct dcp_v14_link *link, u32 tag, const void *input, mutex_unlock(&link->lock); return ret; } + +int dcp_v14_link_call(struct dcp_v14_link *link, u32 tag, const void *input, + u32 input_size, void *output, u32 output_size, + u32 completion_id, dcp_v14_callback_fn callback, void *cookie) +{ + bool vetoed; + + return dcp_v14_link_call_guarded(link, tag, input, input_size, output, + output_size, completion_id, callback, cookie, NULL, NULL, &vetoed); +} diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_link.h b/drivers/gpu/drm/apple/iomfb_v14_7_link.h index cfceedc018328..9d98217a44d15 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7_link.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7_link.h @@ -29,6 +29,9 @@ struct work_struct; typedef int (*dcp_v14_callback_fn)(void *cookie, u32 tag, const void *in, u32 in_size, void *out, u32 out_size); +/* Called under the link lock immediately before a packet is submitted. */ +typedef bool (*dcp_v14_admit_fn)(void *cookie); + struct dcp_v14_link { struct device *dev; struct apple_rtkit *rtk; @@ -85,6 +88,10 @@ void dcp_v14_link_set_idle_work(struct dcp_v14_link *link, struct work_struct *w int dcp_v14_link_call(struct dcp_v14_link *link, u32 tag, const void *input, u32 input_size, void *output, u32 output_size, u32 completion_id, dcp_v14_callback_fn callback, void *cookie); +int dcp_v14_link_call_guarded(struct dcp_v14_link *link, u32 tag, const void *input, + u32 input_size, void *output, u32 output_size, + u32 completion_id, dcp_v14_callback_fn callback, void *cookie, + dcp_v14_admit_fn admit, void *admit_cookie, bool *vetoed); int dcp_v14_link_pump(struct dcp_v14_link *link, unsigned long timeout, dcp_v14_callback_fn callback, void *cookie); From a8da816d457e63fb154b2a674912c0bf46c8c025 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 12 Oct 2026 20:07:04 -0700 Subject: [PATCH 1663/2680] drm/apple: reserve the T6030 HDMI processor with source-aware tunnel clocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow external sources 0 and 2 to feed DP IN0 through PCLK1. Track the source owning the ATC pixel clock so a stale source cannot retune or stop it, and use that identity during route teardown. The allocator can then prefer the USB-C-only processor while leaving the fixed HDMI processor available. Keep DP IN1 unsupported on T6030 and retain the existing clock sequences on other SoCs. Cover both HDMI attachment orders, source ownership, tunnel teardown and legacy clock behavior. [ryan: adapted to 7.2.10 — DCP side only: the phy/apple/atc.c source-owned clock (apple_atc_dp_tunnel_rate_source) and its dp-tunnel.h additions are left to 170-usb (handed over to 170-usb); DCP gets a local dcp_fabric_t6030_pclk() copy of apple_dp_tunnel_t6030_pclk() in dcp-fabric-core.h and an apple_atc_dp_tunnel_rate_source() prototype in dcp-fabric.h, called through symbol_get(); without that provider T6030 source 0 falls back to apple_atc_dp_tunnel_rate() as before and source 2 gets -ENOENT; dcp_tunnel_set_rate() has no tb_generation callback check here; the tunnel-route host test reads the local helper (passes); dp-release/tunnel-clock test updates are left out] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 13d2affbc4d176b528d0c5ef5b17d639c6daab19) --- drivers/gpu/drm/apple/dcp-fabric-core.c | 4 +- drivers/gpu/drm/apple/dcp-fabric-core.h | 10 ++++ drivers/gpu/drm/apple/dcp-fabric-test.c | 60 ++++++++++------------ drivers/gpu/drm/apple/dcp-fabric.c | 67 +++++++++++++++---------- drivers/gpu/drm/apple/dcp-fabric.h | 7 +++ tools/asahi/tunnel-route-tests.py | 2 +- tools/asahi/tunnel-route/fixture.c | 7 +-- 7 files changed, 92 insertions(+), 65 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.c b/drivers/gpu/drm/apple/dcp-fabric-core.c index 6a857ba15e7f1..dccb400a9ff0f 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.c +++ b/drivers/gpu/drm/apple/dcp-fabric-core.c @@ -28,8 +28,8 @@ EXPORT_SYMBOL_GPL(dcp_fabric_t6020_flow); u8 dcp_fabric_tunnel_clock_blocked(bool t6030, unsigned int source) { - /* T6030 currently clocks only source 0 feeding DP IN0. */ - return t6030 ? BIT(1) | (source ? BIT(0) : 0) : 0; + return t6030 ? BIT(1) | + (dcp_fabric_t6030_pclk(source, 0) < 0 ? BIT(0) : 0) : 0; } EXPORT_SYMBOL_GPL(dcp_fabric_tunnel_clock_blocked); diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.h b/drivers/gpu/drm/apple/dcp-fabric-core.h index cd211826a273a..e4eae1f785efa 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.h +++ b/drivers/gpu/drm/apple/dcp-fabric-core.h @@ -2,6 +2,7 @@ #ifndef __APPLE_DCP_FABRIC_CORE_H__ #define __APPLE_DCP_FABRIC_CORE_H__ +#include #include enum dcp_fabric_presence_state { @@ -75,6 +76,15 @@ enum dcp_fabric_wiring { enum dcp_fabric_wiring dcp_fabric_wiring(bool dpin0, bool dpin1, bool legacy, unsigned int endpoints); bool dcp_fabric_t6020_flow(bool usb4, bool soc_support, bool connector_wired); +/* + * T6030's even external sources feed DP IN0 through DPTX PCLK1. A local copy + * of apple_dp_tunnel_t6030_pclk() (linux/soc/apple/dp-tunnel.h, 170-usb), + * which the dp-tunnel.h this tree builds against does not have yet. + */ +static inline int dcp_fabric_t6030_pclk(unsigned int source, unsigned int dpin) +{ + return !dpin && (source == 0 || source == 2) ? 0 : -EOPNOTSUPP; +} u8 dcp_fabric_tunnel_clock_blocked(bool t6030, unsigned int source); enum dcp_fabric_capacity_action { diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c index 0bb37b13a1639..c4fd58b9d7988 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-test.c +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -1920,50 +1920,48 @@ static void fabric_t6030_init(struct fabric_fixture *f) static void fabric_t6030_clock_routes_test(struct kunit *test) { struct fabric_fixture f; - struct dcp_fabric_route *route; - int error; fabric_t6030_init(&f); + /* Reserve the hybrid processor for HDMI, even before HDMI arrives. */ KUNIT_ASSERT_EQ(test, fabric_tunnel(&f, 0, 0), 0); - KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], &f.route[0][0]); + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], &f.route[0][1]); + KUNIT_EXPECT_FALSE(test, f.pipeline[0].owned); + f.pipeline[0].fixed_busy = true; + KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 0), 0); fabric_release(&f, 0, 0); - /* Unusable engines cannot override readiness of the supported engine. */ - f.pipeline[1].terminal = true; - f.pipeline[1].services_ready = false; + /* HDMI first: the USB-C-only processor still drives the dock. */ KUNIT_ASSERT_EQ(test, fabric_tunnel(&f, 0, 0), 0); - KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], &f.route[0][0]); + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], &f.route[0][1]); fabric_release(&f, 0, 0); - f.pipeline[1].terminal = false; - f.pipeline[1].services_ready = true; - /* The Type-C-only engine wins direct DP, but cannot clock a tunnel. */ + f.pipeline[0].fixed_busy = false; + /* A direct USB-C display can hold source 2; source 0 remains a fallback. */ KUNIT_ASSERT_EQ(test, fabric_direct(&f, 1), 0); KUNIT_EXPECT_PTR_EQ(test, f.port[1].owner[0], &f.route[1][1]); KUNIT_ASSERT_EQ(test, fabric_tunnel(&f, 0, 0), 0); KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], &f.route[0][0]); fabric_release(&f, 0, 0); - fabric_release(&f, 1, 0); - /* HDMI owns source 0: do not hand off the unclockable source 2. */ f.pipeline[0].fixed_busy = true; KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 0), -EBUSY); KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], NULL); - KUNIT_EXPECT_FALSE(test, f.pipeline[1].owned); + fabric_release(&f, 1, 0); f.pipeline[0].fixed_busy = false; + /* A processor still starting does not displace the ready fallback. */ + f.pipeline[1].services_ready = false; + KUNIT_ASSERT_EQ(test, fabric_tunnel(&f, 0, 0), 0); + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], &f.route[0][0]); + fabric_release(&f, 0, 0); f.pipeline[0].services_ready = false; + f.pipeline[1].services_ready = false; KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 0), -EAGAIN); - f.pipeline[0].services_ready = true; + f.pipeline[1].services_ready = true; KUNIT_ASSERT_EQ(test, fabric_tunnel(&f, 0, 0), 0); - KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], &f.route[0][0]); + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], &f.route[0][1]); fabric_release(&f, 0, 0); + /* A second stream on the same dock is still unsupported on T6030. */ KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 1), -EOPNOTSUPP); KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[1], NULL); - f.port[0].routes = &f.route[0][1]; - KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 0), -EOPNOTSUPP); - f.port[0].routes = NULL; - KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 0), -EBUSY); - route = dcp_fabric_tunnel_candidate(&f.port[0], &f.policy, NULL, - false, 2, true, &error); - KUNIT_EXPECT_PTR_EQ(test, route, NULL); - KUNIT_EXPECT_EQ(test, error, -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_fabric_tunnel_clock_blocked(true, 1), (u8)3); + KUNIT_EXPECT_EQ(test, dcp_fabric_tunnel_clock_blocked(true, 4), (u8)3); KUNIT_EXPECT_EQ(test, dcp_fabric_tunnel_clock_blocked(false, 0), (u8)0); KUNIT_EXPECT_EQ(test, dcp_fabric_tunnel_clock_blocked(false, 2), (u8)0); } @@ -1977,12 +1975,12 @@ static void fabric_t6030_plan_test(struct kunit *test) KUNIT_EXPECT_PTR_EQ(test, f.plan[0].target[0], &f.route[0][0]); f.pipeline[0].fixed_busy = true; dcp_fabric_plan(f.pipeline, f.port, &f.port[0], 0); - KUNIT_EXPECT_PTR_EQ(test, f.plan[0].target[0], NULL); - /* The same available source remains usable by a direct USB-C display. */ + KUNIT_EXPECT_PTR_EQ(test, f.plan[0].target[0], &f.route[0][1]); f.port[1].wanted = true; f.port[1].hpd = true; dcp_fabric_plan(f.pipeline, f.port, &f.port[0], 0); - KUNIT_EXPECT_PTR_EQ(test, f.plan[1].target[0], &f.route[1][1]); + /* HDMI and the dock exhaust both external processors. */ + KUNIT_EXPECT_PTR_EQ(test, f.plan[1].target[0], NULL); dcp_fabric_plan(f.pipeline, f.port, &f.port[0], 1); KUNIT_EXPECT_PTR_EQ(test, f.plan[0].target[1], NULL); } @@ -1994,18 +1992,14 @@ static void fabric_t6030_follow_test(struct kunit *test) fabric_t6030_init(&f); f.route[0][0].tunnel = true; f.route[0][0].dpin = 0; + KUNIT_EXPECT_EQ(test, dcp_fabric_follow(&f.route[0][0], &f.route[0][1], + NULL, NULL, false, &f.policy), DCP_FABRIC_FOLLOW_MOVE); + f.route[0][0].dpin = 1; KUNIT_EXPECT_EQ(test, dcp_fabric_follow(&f.route[0][0], &f.route[0][1], NULL, NULL, false, &f.policy), DCP_FABRIC_FOLLOW_REFUSE); - /* Direct streams do not acquire a tunnel clock restriction. */ f.route[0][0].tunnel = false; KUNIT_EXPECT_EQ(test, dcp_fabric_follow(&f.route[0][0], &f.route[0][1], NULL, NULL, false, &f.policy), DCP_FABRIC_FOLLOW_MOVE); - /* A swap must also preserve the holder's clockable source. */ - f.route[1][0].tunnel = true; - f.route[1][0].dpin = 0; - KUNIT_EXPECT_EQ(test, dcp_fabric_follow(&f.route[0][1], &f.route[0][0], - &f.route[1][0], &f.route[1][1], true, &f.policy), - DCP_FABRIC_FOLLOW_REFUSE); } static struct kunit_case fabric_tests[] = { diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index a99561e565fc9..2ef6dc46ea17b 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -233,6 +233,38 @@ bool dcp_is_typec_only(struct platform_device *pdev) return !dcp->fixed_phy; } +static int dcp_route_clock_rate(struct apple_dcp_typec_route *route, + struct phy *phy, unsigned int dpin, u8 rate) +{ + int ret; + + if (of_machine_is_compatible("apple,t6030")) { + typeof(apple_atc_dp_tunnel_rate_source) *fn; + + if (dcp_fabric_t6030_pclk(route->mux_index, dpin) < 0) + return -EOPNOTSUPP; + fn = symbol_get(apple_atc_dp_tunnel_rate_source); + if (fn) { + ret = fn(phy, route->mux_index, dpin, rate); + symbol_put(apple_atc_dp_tunnel_rate_source); + return ret; + } + /* An older ATC provider clocks source 0 only, as before. */ + if (route->mux_index != 0) + return -ENOENT; + } + { + typeof(apple_atc_dp_tunnel_rate) *fn; + + fn = symbol_get(apple_atc_dp_tunnel_rate); + if (!fn) + return -ENOENT; + ret = fn(phy, dpin, rate); + symbol_put(apple_atc_dp_tunnel_rate); + } + return ret; +} + static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, struct mux_control *xbar); static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); @@ -405,13 +437,10 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) if (route->tunnel && dcp->phy) { /* the tunnel pixel clock must not outlive the tunnel */ - typeof(&apple_atc_dp_tunnel_rate) stop = - symbol_get(apple_atc_dp_tunnel_rate); + int err = dcp_route_clock_rate(route, dcp->phy, route->tunnel_dpin, 0); - if (stop) { - stop(dcp->phy, route->tunnel_dpin, 0); - symbol_put(apple_atc_dp_tunnel_rate); - } + if (err) + dev_warn(dcp->dev, "DP tunnel clock stop failed: %d\n", err); } route->active_xbar = NULL; route->tunnel = false; @@ -1700,7 +1729,6 @@ int dcp_direct_crossbar_link(struct apple_dcp *dcp, bool up) int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 unit, u32 link_rate) { - typeof(&apple_atc_dp_tunnel_rate) fn; struct apple_dcp_typec_route *route; int ret; @@ -1708,17 +1736,11 @@ int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 unit, if (!dcp->dptx_tunnel) return -ENODEV; route = dcp->active_typec_route; - /* The initial T6030 clock port supports core0 -> DP IN0 (PCLK1). - * Other routes require explicit PCLK slot selection and accounting. - */ - if (link_rate && of_machine_is_compatible("apple,t6030") && - (!route || route->mux_index != 0 || dcp->dptx_dfp_port != 1)) + if (!route) + return -ENODEV; + if (of_machine_is_compatible("apple,t6030") && + (unit || dcp->dptx_dfp_port != 1)) return -EOPNOTSUPP; - fn = symbol_get(apple_atc_dp_tunnel_rate); - if (!fn) { - dev_err(dcp->dev, "phy-apple-atc not loaded, no DP tunnel clock\n"); - return -ENOENT; - } if (unit) { /* Each DP IN owns a reference, including on a shared clock. */ ret = 0; @@ -1727,24 +1749,17 @@ int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 unit, } else { if (!link_rate && dcp->split.xbar_up) dcp_dpxbar_link(dcp->split.xbar, false); - ret = fn(phy, 1, link_rate); + ret = dcp_route_clock_rate(route, phy, 1, link_rate); } - symbol_put(apple_atc_dp_tunnel_rate); dcp->split.clock_ok = !ret && link_rate; if (ret) dev_warn(dcp->dev, "tiled: dpin1 pixel clock (rate 0x%x) failed: %d\n", link_rate, ret); return ret; } - route = dcp->active_typec_route; - if (!route) { - symbol_put(apple_atc_dp_tunnel_rate); - return -ENODEV; - } if (!link_rate && route->xbar_up) dcp_dpxbar_link(route->active_xbar, false); - ret = fn(phy, route->tunnel_dpin, link_rate); - symbol_put(apple_atc_dp_tunnel_rate); + ret = dcp_route_clock_rate(route, phy, route->tunnel_dpin, link_rate); dcp->tb_clock_ok = !ret && link_rate; if (ret) dev_warn(dcp->dev, "DP tunnel pixel clock (rate 0x%x) failed: %d\n", diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index 46578a7abaeee..228ba9f1df4cf 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -86,6 +86,13 @@ void dcp_unlink(struct drm_device *drm); int dcp_tunnel_crossbar_up(struct apple_dcp *dcp, u32 unit); int dcp_tunnel_crossbar_down(struct apple_dcp *dcp, u32 unit); int dcp_direct_crossbar_link(struct apple_dcp *dcp, bool up); +/* + * T6030 ATC PHY: keep the source owning the pixel clock across rate changes + * and stop (linux/soc/apple/dp-tunnel.h, 170-usb). Declared here until that + * header has it; called through symbol_get() only. + */ +int apple_atc_dp_tunnel_rate_source(struct phy *phy, unsigned int source, + unsigned int dpin, u8 rate); int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 unit, u32 link_rate); int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, u32 unit, bool active); diff --git a/tools/asahi/tunnel-route-tests.py b/tools/asahi/tunnel-route-tests.py index b1d5574ea4b50..05d0cbc018802 100644 --- a/tools/asahi/tunnel-route-tests.py +++ b/tools/asahi/tunnel-route-tests.py @@ -18,7 +18,7 @@ def function(source, name): - match = re.search(r'^(?:static\s+)?(?:void|u8|struct mux_control \*)\s*' + + match = re.search(r'^(?:static\s+)?(?:inline\s+)?(?:int|void|u8|struct mux_control \*)\s*' + name + r'\([^;]*?\)\n\{', source, re.M | re.S) if not match: raise ValueError(name) diff --git a/tools/asahi/tunnel-route/fixture.c b/tools/asahi/tunnel-route/fixture.c index 63b43a4e3ffb4..da8e6c70a212c 100644 --- a/tools/asahi/tunnel-route/fixture.c +++ b/tools/asahi/tunnel-route/fixture.c @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only OR MIT */ #include +#include #include #include #include @@ -77,15 +78,15 @@ int main(void) assert(source0.core.next == &source2.core); assert(!source2.core.next); assert(source0.core.tunnel_clock_blocked == (soc ? BIT(1) : 0)); - assert(source2.core.tunnel_clock_blocked == (soc ? BIT(0) | BIT(1) : 0)); + assert(source2.core.tunnel_clock_blocked == (soc ? BIT(1) : 0)); assert(dcp_typec_tunnel_ctl(&source0, 0) == &chip.mux[1]); assert(dcp_typec_tunnel_ctl(&source0, 1) == (soc ? NULL : &chip.mux[2])); - assert(dcp_typec_tunnel_ctl(&source2, 0) == (soc ? NULL : &chip.mux[1])); + assert(dcp_typec_tunnel_ctl(&source2, 0) == &chip.mux[1]); assert(dcp_typec_tunnel_ctl(&source2, 1) == (soc ? NULL : &chip.mux[2])); assert(!dcp_typec_tunnel_ctl(&source0, 2)); /* Late migration checks the real route, even with a stale snapshot. */ source2.core.tunnel_clock_blocked = 0; - assert(dcp_typec_tunnel_ctl(&source2, 0) == (soc ? NULL : &chip.mux[1])); + assert(dcp_typec_tunnel_ctl(&source2, 0) == &chip.mux[1]); } } puts("PASS: DCP snapshot, named/legacy controls, invalid DP IN, stale snapshot"); From 82da9a232a120aafc70d121cf94957deea306d89 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 11 Oct 2026 15:59:47 -0700 Subject: [PATCH 1664/2680] drm/apple: add the T6031 DCP and the J516C panel record Bind apple,t6031-dcp (one DPTX port, as T6030) and apple,t6031-display-subsystem, and add a 14.x board record for the J516C (MacBook Pro 16", M3 Max): the J516S panel, 3456x2234 with 74 notch rows above the 3456x2160 boot framebuffer, at 60 or 120 Hz, marked by apple,t6031-handoff, no colour matrix. Only the T6030 DCP image (DDF38191-93B3-324A-BC8F-643006F5AC82) has validated IOMFB layouts, so the record admits that image only. A J516C DCP with any other image is refused at probe, before the PMP is started or the DCP touched, with one line naming the image it has; the boot framebuffer stays. A T6031 DCP on any other board has no record and is refused the same way. The first boots of this board are judged from the kernel log, so on J516C every panel timing the firmware offers and the display clock it is told are logged at info level, not only with debugging on. The record type moves to iomfb_v14_7_board.h, and record selection and image admission become dcp_v14_board_select() and dcp_v14_board_admits(), unchanged for T6030, J613 and J615, so that the new apple-dcp-v14-board KUnit suite can check them: the J516C record, no record on J514C or T6034, the image admission, and the existing records as they were. tools/asahi/dcp14-panel-tests.py reads the record type from the new header and also checks the J516C admission, marker and boot geometry (notch hidden or kept, wrong panel refused). Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 8b964861dacea9de6c7f827678bb20788962af0d) --- drivers/gpu/drm/apple/Kconfig | 2 + drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/apple_drv.c | 2 + drivers/gpu/drm/apple/dcp.c | 6 + drivers/gpu/drm/apple/iomfb_v14_7.c | 110 +++++++++------ drivers/gpu/drm/apple/iomfb_v14_7_board.h | 62 +++++++++ drivers/gpu/drm/apple/v14-board-test.c | 161 ++++++++++++++++++++++ tools/asahi/dcp14-panel-tests.py | 38 ++++- 8 files changed, 337 insertions(+), 45 deletions(-) create mode 100644 drivers/gpu/drm/apple/iomfb_v14_7_board.h create mode 100644 drivers/gpu/drm/apple/v14-board-test.c diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 51785525b74b4..6eb42409c1d0d 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -72,3 +72,5 @@ config DRM_APPLE_DISPLAY_KUNIT_TEST preferred mode selection. Exercise mode list replacement and catalog admission for each display attachment. Check connector status updates before hotplug notifications and same-state retrains. + Check which 14.x board record an internal display gets on which + machine, and which DCP firmware image each record admits. diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index a038caae1e79f..26b4c8e4bbdab 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -42,3 +42,4 @@ obj-$(CONFIG_DRM_APPLE_PARSER_KUNIT_TEST) += afkv2-test.o appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += parser-test.o appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += hotplug-test.o +appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += v14-board-test.o diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index c14039416e717..db515d81e933a 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -761,6 +761,7 @@ static const struct of_device_id apple_dcp_id_tbl[] = { /* T6030: apple_dcp_usable() keeps its dcpext outside the DRM device. */ { .compatible = "apple,t6030-dcp" }, { .compatible = "apple,t8122-dcp" }, + { .compatible = "apple,t6031-dcp" }, {}, }; @@ -1070,6 +1071,7 @@ static const struct of_device_id of_match[] = { { .compatible = "apple,display-subsystem" }, { .compatible = "apple,t6030-display-subsystem" }, { .compatible = "apple,t8122-display-subsystem" }, + { .compatible = "apple,t6031-display-subsystem" }, {} }; MODULE_DEVICE_TABLE(of, of_match); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 1bf8c5a93bc0b..1b0a1556a4dd0 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -4351,6 +4351,11 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8122 = { .num_dptx_ports = 2, }; +/* The internal panel only, through the 14.x IOMFB, as on T6030. */ +static const struct apple_dcp_hw_data apple_dcp_hw_t6031 = { + .num_dptx_ports = 1, +}; + static const struct apple_dcp_hw_data apple_dcp_hw_t6030_dcpext = { .num_dptx_ports = 1, }; @@ -4369,6 +4374,7 @@ static const struct of_device_id of_match[] = { { .compatible = "apple,t6030-dcp", .data = &apple_dcp_hw_t6030, }, { .compatible = "apple,t6030-dcpext", .data = &apple_dcp_hw_t6030_dcpext, }, { .compatible = "apple,t8122-dcp", .data = &apple_dcp_hw_t8122, }, + { .compatible = "apple,t6031-dcp", .data = &apple_dcp_hw_t6031, }, { .compatible = "apple,dcp", .data = &apple_dcp_hw_dcp, }, { .compatible = "apple,dcpext", .data = &apple_dcp_hw_dcpext, }, {} diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 4cbe089b453e5..1134c3c7fd39a 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -39,6 +39,7 @@ #include "dcp-lifecycle.h" #include "iomfb_internal.h" #include "iomfb_v14_7.h" +#include "iomfb_v14_7_board.h" #include "iomfb_v14_7_link.h" #include "iomfb_v14_7_swap.h" #include "parser.h" @@ -82,36 +83,6 @@ static int dcp_v14_pmp_wait_ready(struct device_node *entry, #define DCP_V14_LAYER 0 #define DCP_V14_BLACK 0xff000000 -/* - * What differs between the internal panels this file drives. The T6030 - * record holds the values the file was written and tested with (J514S, - * J516S); another board's record enables only what was tested on it, and - * anything else fails closed. External processors have no board record. - */ -struct dcp_v14_board { - /* In log messages. */ - const char *name; - /* The debugfs status file. */ - const char *debugfs; - const char *dcp_compatible; - /* The machine compatible; NULL for any machine with this DCP. */ - const char *machine; - /* The firmware image whose layouts were tested on this board. */ - const char *firmware_uuid; - /* Set to <1> by the boot loader on the DCP, display and PIODMA nodes. */ - const char *handoff; - /* Native panel size, notch rows included; 0: any, from the boot framebuffer. */ - u32 panel_width, panel_height; - /* The panel may have a 120 Hz timing besides 60 Hz. */ - bool promotion; - /* - * The colour matrix setter and getter of this board's firmware image; - * 0: the matrix is not sent. Method numbers differ between firmware - * releases, so they are only set where tested. - */ - u32 ctm_set, ctm_get; -}; - static const struct dcp_v14_board dcp_v14_board_t6030 = { .name = "T6030", .debugfs = "dcp-t6030", @@ -154,10 +125,33 @@ static const struct dcp_v14_board dcp_v14_board_j615 = { .ctm_get = A(420), }; +/* + * J516C (MacBook Pro 16", M3 Max): the J516S panel, 3456x2234 with 74 notch + * rows above the 3456x2160 boot framebuffer, at 60 or 120 Hz, on the T6031 + * DCP. Only the T6030 DCP image has validated IOMFB layouts, so a J516C DCP + * running any other image is refused and the boot framebuffer stays. Every + * panel timing and the display clock the firmware is told are logged: the + * first boots on this board are read from the kernel log. The clock is the + * T6030 rate (clk_disp0), unconfirmed on T6031. + */ +static const struct dcp_v14_board dcp_v14_board_j516c = { + .name = "J516C", + .debugfs = "dcp-j516c", + .dcp_compatible = "apple,t6031-dcp", + .machine = "apple,j516c", + .firmware_uuid = DCP_V14_FIRMWARE_UUID, + .handoff = "apple,t6031-handoff", + .panel_width = 3456, + .panel_height = 2234, + .promotion = true, + .log_bringup = true, +}; + static const struct dcp_v14_board *const dcp_v14_boards[] = { &dcp_v14_board_t6030, &dcp_v14_board_j613, &dcp_v14_board_j615, + &dcp_v14_board_j516c, }; struct dcp_v14_property { @@ -1012,7 +1006,12 @@ static int dcp_v14_callback(void *cookie, u32 tag, const void *input, u32 in_siz if (memcmp(in, "VORP", 4) || count > 1) return -EINVAL; put_unaligned_le64(count ? 0 : v14->clock_rate, out); - dev_dbg(v14->dev, "display clock %u: %llu Hz\n", count, get_unaligned_le64(out)); + if (v14->board && v14->board->log_bringup) + dev_info(v14->dev, "display clock %u: %llu Hz\n", count, + get_unaligned_le64(out)); + else + dev_dbg(v14->dev, "display clock %u: %llu Hz\n", count, + get_unaligned_le64(out)); return 0; } if (tag == D(125) && SHAPE(100, 36)) { @@ -1426,21 +1425,50 @@ static int dcp_v14_cpu_running(struct device *dev, const char *name) * The board of an internal 14.x DCP node: NULL if the node is not one, an * error if it is but this machine has no board record. */ -const struct dcp_v14_board *iomfb_v14_7_board(struct device *dev) +const struct dcp_v14_board *dcp_v14_board_select(dcp_v14_compat_fn dcp_is, + dcp_v14_compat_fn machine_is, + const void *ctx, bool *dcp_known) { const struct dcp_v14_board *board; - bool dcp_known = false; unsigned int i; + *dcp_known = false; for (i = 0; i < ARRAY_SIZE(dcp_v14_boards); i++) { board = dcp_v14_boards[i]; - if (!of_device_is_compatible(dev->of_node, board->dcp_compatible)) + if (!dcp_is(ctx, board->dcp_compatible)) continue; - dcp_known = true; - if (board->machine && !of_machine_is_compatible(board->machine)) + *dcp_known = true; + if (board->machine && !machine_is(ctx, board->machine)) continue; return board; } + return NULL; +} + +bool dcp_v14_board_admits(const struct dcp_v14_board *board, const char *uuid) +{ + return uuid && !strcmp(uuid, board->firmware_uuid); +} + +static bool dcp_v14_node_is(const void *np, const char *compat) +{ + return of_device_is_compatible(np, compat); +} + +static bool dcp_v14_machine_is(const void *np, const char *compat) +{ + return of_machine_is_compatible(compat); +} + +const struct dcp_v14_board *iomfb_v14_7_board(struct device *dev) +{ + const struct dcp_v14_board *board; + bool dcp_known; + + board = dcp_v14_board_select(dcp_v14_node_is, dcp_v14_machine_is, dev->of_node, + &dcp_known); + if (board) + return board; if (dcp_known) { dev_err(dev, "display not started: no board record for this machine\n"); return ERR_PTR(-ENODEV); @@ -1466,7 +1494,7 @@ int iomfb_v14_7_probe(struct apple_dcp *dcp) if (IS_ERR_OR_NULL(board)) return -ENODEV; if (of_property_read_string(np, "apple,firmware-uuid", &uuid) || - strcmp(uuid, board->firmware_uuid)) + !dcp_v14_board_admits(board, uuid)) return dev_err_probe(dev, -ENODEV, "%s display not started: DCP firmware %s is not supported\n", board->name, uuid ?: "(unknown)"); @@ -2308,8 +2336,12 @@ static int dcp_v14_mode(struct apple_dcp *dcp, struct apple_dcp_v14 *v14) struct drm_display_mode *m = &modes[i].mode; int hz = drm_mode_vrefresh(m); - dev_dbg(dcp->dev, "timing %u/%u: " DRM_MODE_FMT "%s\n", i + 1, count, - DRM_MODE_ARG(m), m->type & DRM_MODE_TYPE_PREFERRED ? " best" : ""); + if (v14->board->log_bringup) + dev_info(dcp->dev, "timing %u/%u: " DRM_MODE_FMT "%s\n", i + 1, count, + DRM_MODE_ARG(m), m->type & DRM_MODE_TYPE_PREFERRED ? " best" : ""); + else + dev_dbg(dcp->dev, "timing %u/%u: " DRM_MODE_FMT "%s\n", i + 1, count, + DRM_MODE_ARG(m), m->type & DRM_MODE_TYPE_PREFERRED ? " best" : ""); if (m->hdisplay != v14->panel_width || m->vdisplay != v14->panel_height - dcp->notch_height || (hz != 60 && (hz != 120 || !v14->board->promotion))) diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_board.h b/drivers/gpu/drm/apple/iomfb_v14_7_board.h new file mode 100644 index 0000000000000..368dab49c2944 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v14_7_board.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ + +#ifndef __APPLE_IOMFB_V14_7_BOARD_H__ +#define __APPLE_IOMFB_V14_7_BOARD_H__ + +#include + +/* + * What differs between the internal panels the 14.x IOMFB drives. The T6030 + * record holds the values the IOMFB code was written and tested with (J514S, + * J516S); another board's record enables only what was tested on it, and + * anything else fails closed. External processors have no board record. + */ +struct dcp_v14_board { + /* In log messages. */ + const char *name; + /* The debugfs status file. */ + const char *debugfs; + const char *dcp_compatible; + /* The machine compatible; NULL for any machine with this DCP. */ + const char *machine; + /* The firmware image whose layouts were tested on this board. */ + const char *firmware_uuid; + /* Set to <1> by the boot loader on the DCP, display and PIODMA nodes. */ + const char *handoff; + /* Native panel size, notch rows included; 0: any, from the boot framebuffer. */ + u32 panel_width, panel_height; + /* The panel may have a 120 Hz timing besides 60 Hz. */ + bool promotion; + /* + * Log what the first boots of a new board are judged by, not only with + * debugging on: every panel timing the firmware offers, and the display + * clock it is told. + */ + bool log_bringup; + /* + * The colour matrix setter and getter of this board's firmware image; + * 0: the matrix is not sent. Method numbers differ between firmware + * releases, so they are only set where tested. + */ + u32 ctm_set, ctm_get; +}; + +/* Whether the DCP node or the machine (as @ctx gives them) has @compat. */ +typedef bool (*dcp_v14_compat_fn)(const void *ctx, const char *compat); + +/* + * The board record of a DCP. NULL if there is none: *@dcp_known then says + * whether a record names the DCP's compatible (but no record is for this + * machine) or none does (not an internal 14.x DCP). + */ +const struct dcp_v14_board *dcp_v14_board_select(dcp_v14_compat_fn dcp_is, + dcp_v14_compat_fn machine_is, + const void *ctx, bool *dcp_known); + +/* + * Whether @board runs a DCP with the firmware image @uuid (NULL: none given): + * only the image whose IOMFB layouts were validated for it. + */ +bool dcp_v14_board_admits(const struct dcp_v14_board *board, const char *uuid); + +#endif /* __APPLE_IOMFB_V14_7_BOARD_H__ */ diff --git a/drivers/gpu/drm/apple/v14-board-test.c b/drivers/gpu/drm/apple/v14-board-test.c new file mode 100644 index 0000000000000..3e292664c386e --- /dev/null +++ b/drivers/gpu/drm/apple/v14-board-test.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * The 14.x IOMFB board records: which record a DCP gets on which machine, + * and which DCP firmware image a record admits. The J516C (T6031) record + * admits only the T6030 image, whose IOMFB layouts were validated, and + * refuses every other; the existing records are unchanged. + */ +#include +#include + +#include "iomfb_v14_7_board.h" + +#define T6030_DCP_IMAGE "DDF38191-93B3-324A-BC8F-643006F5AC82" +#define J613_DCP_IMAGE "90F849E1-B422-367E-B389-50246F8DEC47" + +struct board_ctx { + const char *dcp; + const char *const *machine; /* NULL-terminated root compatibles */ +}; + +static bool board_ctx_dcp_is(const void *ctx, const char *compat) +{ + const struct board_ctx *c = ctx; + + return !strcmp(c->dcp, compat); +} + +static bool board_ctx_machine_is(const void *ctx, const char *compat) +{ + const struct board_ctx *c = ctx; + const char *const *m; + + for (m = c->machine; *m; m++) + if (!strcmp(*m, compat)) + return true; + return false; +} + +static const struct dcp_v14_board *board_for(const char *dcp, const char *const *machine, + bool *known) +{ + const struct board_ctx ctx = { .dcp = dcp, .machine = machine }; + + return dcp_v14_board_select(board_ctx_dcp_is, board_ctx_machine_is, &ctx, known); +} + +static const char *const j516c[] = { "apple,j516c", "apple,t6031", "apple,arm-platform", NULL }; +static const char *const j514c[] = { "apple,j514c", "apple,t6031", "apple,arm-platform", NULL }; +static const char *const j516m[] = { "apple,j516m", "apple,t6034", "apple,arm-platform", NULL }; +static const char *const j516s[] = { "apple,j516s", "apple,t6030", "apple,arm-platform", NULL }; +static const char *const j613[] = { "apple,j613", "apple,t8122", "apple,arm-platform", NULL }; + +static void v14_board_j516c(struct kunit *test) +{ + const struct dcp_v14_board *b; + bool known; + + b = board_for("apple,t6031-dcp", j516c, &known); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_EXPECT_TRUE(test, known); + KUNIT_EXPECT_STREQ(test, b->name, "J516C"); + KUNIT_EXPECT_STREQ(test, b->debugfs, "dcp-j516c"); + KUNIT_EXPECT_STREQ(test, b->machine, "apple,j516c"); + KUNIT_EXPECT_STREQ(test, b->handoff, "apple,t6031-handoff"); + KUNIT_EXPECT_STREQ(test, b->firmware_uuid, T6030_DCP_IMAGE); + KUNIT_EXPECT_EQ(test, b->panel_width, 3456); + KUNIT_EXPECT_EQ(test, b->panel_height, 2234); + KUNIT_EXPECT_TRUE(test, b->promotion); + KUNIT_EXPECT_TRUE(test, b->log_bringup); + /* No colour matrix method is known for this board. */ + KUNIT_EXPECT_EQ(test, b->ctm_set, 0); + KUNIT_EXPECT_EQ(test, b->ctm_get, 0); +} + +/* A T6031 DCP on another board, or on a T6034, has no record: refused. */ +static void v14_board_t6031_other_boards(struct kunit *test) +{ + bool known; + + KUNIT_EXPECT_NULL(test, board_for("apple,t6031-dcp", j514c, &known)); + KUNIT_EXPECT_TRUE(test, known); + KUNIT_EXPECT_NULL(test, board_for("apple,t6031-dcp", j516m, &known)); + KUNIT_EXPECT_TRUE(test, known); + /* Not a 14.x internal DCP at all. */ + KUNIT_EXPECT_NULL(test, board_for("apple,t8112-dcp", j516c, &known)); + KUNIT_EXPECT_FALSE(test, known); +} + +/* The DCP image the record admits: the validated one, exactly. */ +static void v14_board_j516c_image(struct kunit *test) +{ + const struct dcp_v14_board *b; + bool known; + + b = board_for("apple,t6031-dcp", j516c, &known); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_EXPECT_TRUE(test, dcp_v14_board_admits(b, T6030_DCP_IMAGE)); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, NULL)); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, "")); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, J613_DCP_IMAGE)); + /* The T6030 PMP image is not a DCP image. */ + KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, "2F4EB4C4-001B-3ACF-A9A0-68D8E42FC3A7")); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, "ddf38191-93b3-324a-bc8f-643006f5ac82")); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, T6030_DCP_IMAGE " ")); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, "DDF38191-93B3-324A-BC8F-643006F5AC8")); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, "00000000-0000-0000-0000-000000000000")); +} + +/* The records that were there before are what they were. */ +static void v14_board_unchanged(struct kunit *test) +{ + const struct dcp_v14_board *b; + bool known; + + b = board_for("apple,t6030-dcp", j516s, &known); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_EXPECT_STREQ(test, b->name, "T6030"); + KUNIT_EXPECT_STREQ(test, b->debugfs, "dcp-t6030"); + KUNIT_EXPECT_NULL(test, b->machine); + KUNIT_EXPECT_STREQ(test, b->handoff, "apple,t6030-handoff"); + KUNIT_EXPECT_STREQ(test, b->firmware_uuid, T6030_DCP_IMAGE); + KUNIT_EXPECT_EQ(test, b->panel_width, 0); + KUNIT_EXPECT_EQ(test, b->panel_height, 0); + KUNIT_EXPECT_TRUE(test, b->promotion); + KUNIT_EXPECT_FALSE(test, b->log_bringup); + KUNIT_EXPECT_EQ(test, b->ctm_set, 0); + KUNIT_EXPECT_TRUE(test, dcp_v14_board_admits(b, T6030_DCP_IMAGE)); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, J613_DCP_IMAGE)); + /* The T6030 record serves a T6030 DCP on any machine, as before. */ + KUNIT_EXPECT_PTR_EQ(test, board_for("apple,t6030-dcp", j516c, &known), b); + + b = board_for("apple,t8122-dcp", j613, &known); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_EXPECT_STREQ(test, b->name, "J613"); + KUNIT_EXPECT_STREQ(test, b->handoff, "apple,t8122-handoff"); + KUNIT_EXPECT_STREQ(test, b->firmware_uuid, J613_DCP_IMAGE); + KUNIT_EXPECT_EQ(test, b->panel_width, 2560); + KUNIT_EXPECT_EQ(test, b->panel_height, 1664); + KUNIT_EXPECT_FALSE(test, b->promotion); + KUNIT_EXPECT_FALSE(test, b->log_bringup); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, T6030_DCP_IMAGE)); + + /* A T8122 DCP on a T6031 machine has no record. */ + KUNIT_EXPECT_NULL(test, board_for("apple,t8122-dcp", j516c, &known)); + KUNIT_EXPECT_TRUE(test, known); +} + +static struct kunit_case v14_board_cases[] = { + KUNIT_CASE(v14_board_j516c), + KUNIT_CASE(v14_board_t6031_other_boards), + KUNIT_CASE(v14_board_j516c_image), + KUNIT_CASE(v14_board_unchanged), + {} +}; + +static struct kunit_suite v14_board_suite = { + .name = "apple-dcp-v14-board", + .test_cases = v14_board_cases, +}; + +kunit_test_suite(v14_board_suite); diff --git a/tools/asahi/dcp14-panel-tests.py b/tools/asahi/dcp14-panel-tests.py index 4f504a79e5f44..a22df09c6959b 100755 --- a/tools/asahi/dcp14-panel-tests.py +++ b/tools/asahi/dcp14-panel-tests.py @@ -11,6 +11,8 @@ parser.add_argument('--old-source', type=Path) args = parser.parse_args() source = (root/'drivers/gpu/drm/apple/iomfb_v14_7.c').read_text() +board_header = root/'drivers/gpu/drm/apple/iomfb_v14_7_board.h' +board_types = board_header.read_text() if board_header.exists() else '' cases = [('current', source)] if args.old_source: cases.append(('old', args.old_source.read_text())) @@ -33,7 +35,7 @@ struct device_node {const char *compatible;}; struct device {struct device_node *of_node;}; static const char *machine; - static bool of_device_is_compatible(struct device_node *n,const char *c){return !strcmp(n->compatible,c);} + static bool of_device_is_compatible(const struct device_node *n,const char *c){return !strcmp(n->compatible,c);} static bool of_machine_is_compatible(const char*c){return !strcmp(machine,c);} static struct device_node fb={"simple-framebuffer"},dcp={"apple,t8122-dcp"}; static u32 width,height,stride,notch,marker_value=1; @@ -42,7 +44,7 @@ #define dev_err_probe(d,e,...) (e) static struct device_node *of_find_compatible_node(void*a,void*b,const char*c){return &fb;} static int of_property_read_u32(struct device_node*n,const char*k,u32*v){ - if(!strcmp(k,"apple,t8122-handoff"))*v=marker_value; + if(!strcmp(k,"apple,t8122-handoff")||!strcmp(k,"apple,t6031-handoff"))*v=marker_value; else if(!strcmp(k,"width"))*v=width; else if(!strcmp(k,"height"))*v=height; else if(!strcmp(k,"stride"))*v=stride; @@ -53,10 +55,16 @@ static void of_node_put(struct device_node*n){} ''' for name,src in cases: - start=src.index('struct dcp_v14_board {');end=src.index('struct dcp_v14_property {') - records=src[start:end] + end=src.index('struct dcp_v14_property {') + if 'struct dcp_v14_board {' in src: + records=src[src.index('struct dcp_v14_board {'):end] + else: + # The record type is in iomfb_v14_7_board.h; the records stay in the driver. + records=board_types[board_types.index('struct dcp_v14_board {'):board_types.index('#endif')] + records+=src[src.index('static const struct dcp_v14_board dcp_v14_board_t6030 = {'):end] defines='\n'.join(l for l in src.splitlines() if l.startswith('#define DCP_V14_') and 'FIRMWARE_UUID' in l)+'\n' - sel=src[src.index('const struct dcp_v14_board *iomfb_v14_7_board('):src.index('const char *iomfb_v14_7_board_name(')] + sel_start='const struct dcp_v14_board *dcp_v14_board_select(' if 'const struct dcp_v14_board *dcp_v14_board_select(' in src else 'const struct dcp_v14_board *iomfb_v14_7_board(' + sel=src[src.index(sel_start):src.index('const char *iomfb_v14_7_board_name(')] geo=src[src.index('static int dcp_v14_geometry('):src.index('/* An external processor\'s firmware:')] driver='''struct apple_dcp_v14 {struct device *dev;const struct dcp_v14_board *board;u32 fb_width,fb_height,stride,notch_rows,panel_width,panel_height;};\n''' probe=src[src.index('int iomfb_v14_7_probe('):] @@ -90,7 +98,25 @@ notch=64;height=1800;stride++;assert(dcp_v14_geometry(&v)==-EINVAL);stride--; width=2560;stride=width*4;assert(dcp_v14_geometry(&v)==-EINVAL); width=2880;stride=width*4;notch=101;assert(dcp_v14_geometry(&v)==-EINVAL); - puts("current J615/J613/T6030 admission + hidden/full notch, wrong geometry controls: PASS"); + /* J516C (T6031): only the T6030 DCP image, the J516S panel. */ + dcp.compatible="apple,t6031-dcp";machine="apple,j516c"; + const struct dcp_v14_board*c=iomfb_v14_7_board(&dev); + assert(!IS_ERR(c)&&c&&!strcmp(c->name,"J516C")&&c->panel_width==3456&&c->panel_height==2234&&c->promotion); + assert(!strcmp(c->handoff,"apple,t6031-handoff")&&!c->ctm_set&&!c->ctm_get); + image_uuid="DDF38191-93B3-324A-BC8F-643006F5AC82";assert(probe_gate(&dev,c)==0); + image_uuid="90F849E1-B422-367E-B389-50246F8DEC47";assert(probe_gate(&dev,c)==-ENODEV); + image_uuid="DDF38191-93B3-324A-BC8F-643006F5AC8";assert(probe_gate(&dev,c)==-ENODEV); + image_uuid=NULL;assert(probe_gate(&dev,c)==-ENODEV); + image_uuid="DDF38191-93B3-324A-BC8F-643006F5AC82";marker_value=0;assert(probe_gate(&dev,c)==-ENODEV);marker_value=1; + struct apple_dcp_v14 m={.dev=&dev,.board=c};width=3456;height=2160;stride=width*4;notch=74; + assert(dcp_v14_geometry(&m)==0&&m.panel_height==2234&&m.notch_rows==74); + height=2234;assert(dcp_v14_geometry(&m)==0&&m.panel_height==2234); + height=2160;notch=0;assert(dcp_v14_geometry(&m)==-EINVAL); + notch=74;width=3024;stride=width*4;height=1890;assert(dcp_v14_geometry(&m)==-EINVAL); + machine="apple,j514c";assert(IS_ERR(iomfb_v14_7_board(&dev))); + machine="apple,j516m";assert(IS_ERR(iomfb_v14_7_board(&dev))); + dcp.compatible="apple,t6030-dcp";machine="apple,j516s";assert(iomfb_v14_7_board(&dev)==&dcp_v14_board_t6030); + puts("current J615/J613/T6030/J516C admission + hidden/full notch, wrong geometry controls: PASS"); #endif } ''' From 5d575522507e7b6361799f322cf20c64fa3a6be3 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 13 Oct 2026 11:29:30 -0700 Subject: [PATCH 1665/2680] drm/apple: t6030: don't fail an HDMI connect whose link came up On a T6030 external pipe, dcp_dptx_connect() checks at the end that the port still has its display before it marks the link ready. The check reads the Type-C cable flag, which the fixed HDMI output never sets, so every HDMI connect returned -ENOLINK after the link had come up and the firmware had started to describe the display. Since the HDMI output got the bounded retry, each such connect has also queued one: HDMI display link not set up (-67): retry 1 of 3 in 1000 ms on every HDMI attach. It used up one of the three tries of the attach, and when it fired while the display was being described again, with the connector reported disconnected, it released the HDMI link and set it up again, which withdraws the description once more. For the fixed HDMI output, check its HPD instead. A connect that brings the link up now returns 0 and marks it ready; one whose display went away while it waited (HPD low) still fails with -ENOLINK and is retried as before. With appledrm.hdmi_audio=1 the T6030 HDMI port now also connects its AV service at that point, as the other outputs do. Type-C and Thunderbolt routes still pass on the cable flag. dcp->external is only set on T6030 external pipes, so M1/M2 are unchanged. Link: https://github.com/omacom/linux-aurora/issues/87 Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 1edaec85fb8a1708b8fb281f1c2802ce1b112a84) --- drivers/gpu/drm/apple/dcp-fabric.c | 6 ++++++ drivers/gpu/drm/apple/dcp-fabric.h | 2 ++ drivers/gpu/drm/apple/dcp.c | 8 ++++++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 2ef6dc46ea17b..72635b8218140 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -116,6 +116,12 @@ bool dcp_uses_t6020_tunnel_flow(struct apple_dcp *dcp) dcp->hw.t6020_tunnel_flow, connector_wired); } +bool dcp_hdmi_port_attached(struct apple_dcp *dcp) +{ + return dcp->hdmi_hpd && !READ_ONCE(dcp->active_typec_route) && + gpiod_get_value_cansleep(dcp->hdmi_hpd) > 0; +} + static bool dcp_typec_route_is_dp(const struct typec_mux_state *state) { return state->alt && state->alt->svid == USB_TYPEC_DP_SID && diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index 228ba9f1df4cf..ce185187610c4 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -53,6 +53,8 @@ struct apple_dcp_typec_route { bool dcp_is_typec_output(struct apple_dcp *dcp); bool dcp_is_usb4_output(struct apple_dcp *dcp); bool dcp_uses_t6020_tunnel_flow(struct apple_dcp *dcp); +/* The fixed HDMI output, not lent to a Type-C display, has HPD high. */ +bool dcp_hdmi_port_attached(struct apple_dcp *dcp); bool dcp_has_typec_routes(struct platform_device *pdev); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 1b0a1556a4dd0..3965f5a927014 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1097,10 +1097,14 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable goto out_withdraw; } if (dcp->external) { + /* + * The port must still have its display: a Type-C cable, or HPD + * on the fixed HDMI output, which has no cable flag. + */ mutex_lock(&dcp->hpd_mutex); if (!dcp->dptxport[port].connected || - !READ_ONCE(dcp->typec_cable_connected) || READ_ONCE(dcp->crashed) || - dcpext_scanout_terminal(dcp)) { + !(READ_ONCE(dcp->typec_cable_connected) || dcp_hdmi_port_attached(dcp)) || + READ_ONCE(dcp->crashed) || dcpext_scanout_terminal(dcp)) { mutex_unlock(&dcp->hpd_mutex); return -ENOLINK; } From 8624ec3e7252cb1fec1669d8bf5dda023089ec9c Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 13 Oct 2026 11:32:39 -0700 Subject: [PATCH 1666/2680] drm/apple: t6030: keep an HDMI display connected while it is described again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the T6030 HDMI port (J514S, J516S), the firmware can withdraw the description of the attached display (timings, color modes, attributes) after a connect and after a mode set, with HPD high, and publish it again a fraction of a second later. A J516S with a 1080p HDMI display shows it on every connect and mode set: display TimingElements withdrawn, 0 bytes dcp_hotplug() connected:0 valid_mode:0 nr_modes:24 display description complete again dcp_hotplug() connected:1 valid_mode:0 nr_modes:24 The driver reported each withdrawal as a disconnect and the return as a connect, about half a second apart. Compositors dropped the output and added it again. aquamarine could read the connector in that window as connected with no modes and keep a 0x0 output, and Hyprland removed and re-added its wl_output global while Xwayland was binding it. Direct Type-C routes already keep the connector connected through such a withdrawal; the fixed HDMI output was left out. Hold it the same way while its HPD is high: - A withdrawal with HPD high queues the bounded retry (2 s, then 4 and 8 s), as on Type-C. If the display is not described again by then, the retry reports it disconnected and sets the HDMI link up again (dcp_fabric_hdmi_retry()). - While that retry is queued, the connector stays connected with the modes and EDID it has. The mode is invalidated as before, and a mode set in the window waits for the description. - The description coming back ends the hold with at most one hotplug event, the connector still connected, which re-applies the mode as before. A display that does not come back gets one disconnect event. - Every HDMI withdrawal with HPD high is held, not only the first of a connection as on Type-C, since this port withdraws on mode sets too. The queued retry bounds it: with no retry left (3 per attach, given back by a mode set that succeeds), a withdrawal disconnects as before. - An HDMI HPD edge while a display is held keeps the retry that bounds the hold instead of cancelling it; the converter's HPD pulses also run the attach path. HPD low at the withdrawal, a pipe lent to a Type-C display, and a withdrawal with no retry left take the old path. Type-C, Thunderbolt and M1/M2 pipes are unchanged. Tests: the apple-display-external-hold KUnit suite gets five HDMI cases, with the HPD level supplied through a KUnit static stub of the new dcp_hdmi_port_attached(). A probe in the withdrawn window reads the display with its mode and EDID, the return sends one event, and the retry then sends none. Four withdrawals in a row, more than there are retries, are each held with one event per return. A display that is not described again and was unplugged gets one disconnect event. HPD low at the withdrawal, or no retry left, disconnects at once. These replace the case that expected the HDMI output to disconnect at once. [ryan: adapted to 7.2.10 — the external-hold-test.c KUnit hunk is left out: that test is not in this tree (dropped with b2e2021dbe, its fixture assumes the edid_lock helpers)] Link: https://github.com/omacom/linux-aurora/issues/87 Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 (cherry picked from commit d4a128739e3d4ad0cc940010b1621c30d1517140) --- drivers/gpu/drm/apple/dcp-fabric.c | 12 +++++- drivers/gpu/drm/apple/iomfb_v14_7.c | 62 ++++++++++++++++++----------- 2 files changed, 49 insertions(+), 25 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 72635b8218140..cd784df522a4b 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -17,6 +17,8 @@ #include +#include + #include "afk.h" #include "dcp.h" #include "dcp-fabric.h" @@ -118,6 +120,8 @@ bool dcp_uses_t6020_tunnel_flow(struct apple_dcp *dcp) bool dcp_hdmi_port_attached(struct apple_dcp *dcp) { + KUNIT_STATIC_STUB_REDIRECT(dcp_hdmi_port_attached, dcp); + return dcp->hdmi_hpd && !READ_ONCE(dcp->active_typec_route) && gpiod_get_value_cansleep(dcp->hdmi_hpd) > 0; } @@ -3586,8 +3590,12 @@ static void dcp_hdmi_connect_fixed(void *ctx) struct apple_dcp *dcp = ctx; int ret = dcp_fixed_output_select(dcp); - /* A new HDMI attach gets its own retries. */ - if (dcp->external_native) { + /* + * A new HDMI attach gets its own retries. The converter's HPD pulses + * also land here: while a display is kept connected through a + * withdrawal, keep the retry that bounds that. + */ + if (dcp->external_native && !atomic_read(&dcp->external_held)) { atomic_set(&dcp->external_retries, 0); cancel_delayed_work(&dcp->external_retry_wq); } diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 1134c3c7fd39a..9050909b561dc 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -586,54 +586,70 @@ void dcp_v14_external_hotplug(struct apple_dcp *dcp, bool connected) } /* - * The firmware withdrew the description of a connected display on a direct - * Type-C route whose port still has HPD, and a retry is queued that redoes - * the link if the description does not come back. Shortly after an attach - * the firmware can do this and describe the display again a fraction of a - * second later. Reporting the connector disconnected for that window lets a - * compositor that found it connected a moment before read it back with no - * modes and no EDID, and keep that empty output once the display is back. - * Keep it connected with the modes and EDID it has, until the description is - * back or the retry finds it is not (iomfb_v14_7_external_release()). A mode - * set meanwhile waits for the description (dcp_v14_external_settle()). Only - * the first withdrawal of a connection is held; later ones disconnect, so a - * display that keeps being withdrawn is reported as it was before. + * The firmware withdrew the description of a connected display whose port + * still has HPD (@hdmi_hpd for the fixed HDMI output), and a retry is queued + * that redoes the link if the description does not come back. Reporting the + * connector disconnected until it is back lets a compositor that found it + * connected a moment before read it back with no modes and no EDID, keep + * that empty output once the display is back, or drop the output and add it + * again. Keep it connected with the modes and EDID it has, until the + * description is back or the retry finds it is not + * (iomfb_v14_7_external_release()). A mode set meanwhile waits for the + * description (dcp_v14_external_settle()). + * + * On a direct Type-C route the firmware can do this shortly after an attach. + * Only the first withdrawal of a Type-C connection is held; later ones + * disconnect, so a display that keeps being withdrawn is reported as it was + * before. On the fixed HDMI output it can do this after every connect and + * every mode set, so each withdrawal with HPD high is held. The queued retry + * bounds that: with no retry left, a withdrawal disconnects as before. */ -static bool dcp_v14_external_hold(struct apple_dcp *dcp) +static bool dcp_v14_external_hold(struct apple_dcp *dcp, bool hdmi_hpd) { struct apple_connector *connector = READ_ONCE(dcp->connector); u64 connection = READ_ONCE(dcp->typec_generation) + 1; + bool typec = dcp_is_typec_output(dcp); unsigned int action; if (!dcp->external_native || READ_ONCE(dcp->external_detached) || - !dcp_is_typec_output(dcp) || dcp_is_usb4_output(dcp) || - !READ_ONCE(dcp->typec_cable_connected) || READ_ONCE(dcp->typec_crtc_off) || !connector || !READ_ONCE(connector->connected) || - READ_ONCE(dcp->external_held_connection) == connection || !delayed_work_pending(&dcp->external_retry_wq)) return false; - WRITE_ONCE(dcp->external_held_connection, connection); + if (typec) { + if (dcp_is_usb4_output(dcp) || !READ_ONCE(dcp->typec_cable_connected) || + READ_ONCE(dcp->typec_crtc_off) || + READ_ONCE(dcp->external_held_connection) == connection) + return false; + WRITE_ONCE(dcp->external_held_connection, connection); + } else if (!hdmi_hpd) { + return false; + } atomic_set(&dcp->external_held, 1); action = dcp_mode_withdraw(&dcp->mode_state); if (!dcp->crtc) action &= ~DCP_HOTPLUG_VBLANK; dcp_handle_hotplug_actions(dcp, action); - dev_info(dcp->dev, "display withdrawn on a Type-C route with HPD: kept connected until it is described again\n"); + dev_info(dcp->dev, "display withdrawn on %s with HPD: kept connected until it is described again\n", + typec ? "a Type-C route" : "the HDMI port"); return true; } /* * The firmware withdrew the timings of the attached display. An HPD bounce - * withdraws the description, and the firmware describes the display again - * once its link is back. If that does not happen while the port still holds - * the display, redo the link. + * withdraws the description, as can a connect or a mode set on the HDMI + * port, and the firmware describes the display again once its link is back. + * If that does not happen while the port still holds the display, redo the + * link. */ void iomfb_v14_7_external_withdrawn(struct apple_dcp *dcp) { - if (READ_ONCE(dcp->typec_cable_connected) && !READ_ONCE(dcp->typec_crtc_off)) + bool hdmi_hpd = dcp_hdmi_port_attached(dcp); + + if ((READ_ONCE(dcp->typec_cable_connected) && !READ_ONCE(dcp->typec_crtc_off)) || + hdmi_hpd) dcp_external_retry(dcp, "display withdrawn while its port has HPD", 0, DCP_V14_DESC_RELINK_MS); - if (!dcp_v14_external_hold(dcp)) + if (!dcp_v14_external_hold(dcp, hdmi_hpd)) dcp_v14_external_hotplug(dcp, false); } From 6e3a44fd21626a6196fa1ea75244fe189f394de3 Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Tue, 13 Oct 2026 23:39:53 +0530 Subject: [PATCH 1667/2680] drm/apple: dcp: admit the J516C's own 14.7 DCP image The J516C board record admitted only the T6030 DCP image, the one the 14.7 IOMFB support was validated on. An M3 Max carries another 14.7 image, F493F03E-CC40-3D89-9A08-5F259BA2B246 (its ADT, and the boot loader's apple,firmware-uuid), so with the T6031 display enabled and the PMP admitted the DCP refused: "J516C display not started: DCP firmware F493F03E-... is not supported". Admit this Mac's image. The 14.7 interface is chosen by firmware version, not by image, and the same iomfb_v14_7 code has run unchanged on the T6030 and the T8122 images; this image's layouts are shown by its first boots, which log every timing and the display clock. The T6030 and J613/J615 records are unchanged. The v14 board KUnit cases and the panel-test script now expect the J516C image and refuse the T6030 one. Assisted-by: Claude:claude-fable-5-1 Signed-off-by: Naeem Malik (cherry picked from commit ae5417235cf16024f7d1c93e65ce89422f71d204) --- drivers/gpu/drm/apple/iomfb_v14_7.c | 9 ++++++--- drivers/gpu/drm/apple/v14-board-test.c | 19 +++++++++++-------- tools/asahi/dcp14-panel-tests.py | 9 +++++---- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 9050909b561dc..743c14cbbf63b 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -50,6 +50,7 @@ /* The images whose method and callback layouts this file implements. */ #define DCP_V14_FIRMWARE_UUID "DDF38191-93B3-324A-BC8F-643006F5AC82" /* T6030 */ #define DCP_V14_J613_FIRMWARE_UUID "90F849E1-B422-367E-B389-50246F8DEC47" /* J613 */ +#define DCP_V14_J516C_FIRMWARE_UUID "F493F03E-CC40-3D89-9A08-5F259BA2B246" /* J516C */ #define DCP_V14_CPU_CONTROL 0x44 #define DCP_V14_CPU_STATUS 0x48 @@ -128,8 +129,10 @@ static const struct dcp_v14_board dcp_v14_board_j615 = { /* * J516C (MacBook Pro 16", M3 Max): the J516S panel, 3456x2234 with 74 notch * rows above the 3456x2160 boot framebuffer, at 60 or 120 Hz, on the T6031 - * DCP. Only the T6030 DCP image has validated IOMFB layouts, so a J516C DCP - * running any other image is refused and the boot framebuffer stays. Every + * DCP, running this Mac's own 14.7 DCP image (the J516C ADT names it). The + * 14.7 IOMFB interface follows the firmware version and has run unchanged on + * the T6030 and T8122 images; this image's layouts are shown by its first + * boots. Any other image is refused and the boot framebuffer stays. Every * panel timing and the display clock the firmware is told are logged: the * first boots on this board are read from the kernel log. The clock is the * T6030 rate (clk_disp0), unconfirmed on T6031. @@ -139,7 +142,7 @@ static const struct dcp_v14_board dcp_v14_board_j516c = { .debugfs = "dcp-j516c", .dcp_compatible = "apple,t6031-dcp", .machine = "apple,j516c", - .firmware_uuid = DCP_V14_FIRMWARE_UUID, + .firmware_uuid = DCP_V14_J516C_FIRMWARE_UUID, .handoff = "apple,t6031-handoff", .panel_width = 3456, .panel_height = 2234, diff --git a/drivers/gpu/drm/apple/v14-board-test.c b/drivers/gpu/drm/apple/v14-board-test.c index 3e292664c386e..aa95cd2580857 100644 --- a/drivers/gpu/drm/apple/v14-board-test.c +++ b/drivers/gpu/drm/apple/v14-board-test.c @@ -12,6 +12,7 @@ #define T6030_DCP_IMAGE "DDF38191-93B3-324A-BC8F-643006F5AC82" #define J613_DCP_IMAGE "90F849E1-B422-367E-B389-50246F8DEC47" +#define J516C_DCP_IMAGE "F493F03E-CC40-3D89-9A08-5F259BA2B246" struct board_ctx { const char *dcp; @@ -62,7 +63,7 @@ static void v14_board_j516c(struct kunit *test) KUNIT_EXPECT_STREQ(test, b->debugfs, "dcp-j516c"); KUNIT_EXPECT_STREQ(test, b->machine, "apple,j516c"); KUNIT_EXPECT_STREQ(test, b->handoff, "apple,t6031-handoff"); - KUNIT_EXPECT_STREQ(test, b->firmware_uuid, T6030_DCP_IMAGE); + KUNIT_EXPECT_STREQ(test, b->firmware_uuid, J516C_DCP_IMAGE); KUNIT_EXPECT_EQ(test, b->panel_width, 3456); KUNIT_EXPECT_EQ(test, b->panel_height, 2234); KUNIT_EXPECT_TRUE(test, b->promotion); @@ -86,7 +87,7 @@ static void v14_board_t6031_other_boards(struct kunit *test) KUNIT_EXPECT_FALSE(test, known); } -/* The DCP image the record admits: the validated one, exactly. */ +/* The DCP image the record admits: this Mac's own, exactly. */ static void v14_board_j516c_image(struct kunit *test) { const struct dcp_v14_board *b; @@ -94,15 +95,17 @@ static void v14_board_j516c_image(struct kunit *test) b = board_for("apple,t6031-dcp", j516c, &known); KUNIT_ASSERT_NOT_NULL(test, b); - KUNIT_EXPECT_TRUE(test, dcp_v14_board_admits(b, T6030_DCP_IMAGE)); + KUNIT_EXPECT_TRUE(test, dcp_v14_board_admits(b, J516C_DCP_IMAGE)); KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, NULL)); KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, "")); + /* The other 14.7 DCP images are not this Mac's. */ + KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, T6030_DCP_IMAGE)); KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, J613_DCP_IMAGE)); - /* The T6030 PMP image is not a DCP image. */ - KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, "2F4EB4C4-001B-3ACF-A9A0-68D8E42FC3A7")); - KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, "ddf38191-93b3-324a-bc8f-643006f5ac82")); - KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, T6030_DCP_IMAGE " ")); - KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, "DDF38191-93B3-324A-BC8F-643006F5AC8")); + /* This Mac's PMP image is not a DCP image. */ + KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, "07FF4C5B-CB2E-3DE6-95FF-3B107D19B7AB")); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, "f493f03e-cc40-3d89-9a08-5f259ba2b246")); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, J516C_DCP_IMAGE " ")); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, "F493F03E-CC40-3D89-9A08-5F259BA2B24")); KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, "00000000-0000-0000-0000-000000000000")); } diff --git a/tools/asahi/dcp14-panel-tests.py b/tools/asahi/dcp14-panel-tests.py index a22df09c6959b..3d0d98a887566 100755 --- a/tools/asahi/dcp14-panel-tests.py +++ b/tools/asahi/dcp14-panel-tests.py @@ -98,16 +98,17 @@ notch=64;height=1800;stride++;assert(dcp_v14_geometry(&v)==-EINVAL);stride--; width=2560;stride=width*4;assert(dcp_v14_geometry(&v)==-EINVAL); width=2880;stride=width*4;notch=101;assert(dcp_v14_geometry(&v)==-EINVAL); - /* J516C (T6031): only the T6030 DCP image, the J516S panel. */ + /* J516C (T6031): its own DCP image, the J516S panel. */ dcp.compatible="apple,t6031-dcp";machine="apple,j516c"; const struct dcp_v14_board*c=iomfb_v14_7_board(&dev); assert(!IS_ERR(c)&&c&&!strcmp(c->name,"J516C")&&c->panel_width==3456&&c->panel_height==2234&&c->promotion); assert(!strcmp(c->handoff,"apple,t6031-handoff")&&!c->ctm_set&&!c->ctm_get); - image_uuid="DDF38191-93B3-324A-BC8F-643006F5AC82";assert(probe_gate(&dev,c)==0); + image_uuid="F493F03E-CC40-3D89-9A08-5F259BA2B246";assert(probe_gate(&dev,c)==0); + image_uuid="DDF38191-93B3-324A-BC8F-643006F5AC82";assert(probe_gate(&dev,c)==-ENODEV); image_uuid="90F849E1-B422-367E-B389-50246F8DEC47";assert(probe_gate(&dev,c)==-ENODEV); - image_uuid="DDF38191-93B3-324A-BC8F-643006F5AC8";assert(probe_gate(&dev,c)==-ENODEV); + image_uuid="F493F03E-CC40-3D89-9A08-5F259BA2B24";assert(probe_gate(&dev,c)==-ENODEV); image_uuid=NULL;assert(probe_gate(&dev,c)==-ENODEV); - image_uuid="DDF38191-93B3-324A-BC8F-643006F5AC82";marker_value=0;assert(probe_gate(&dev,c)==-ENODEV);marker_value=1; + image_uuid="F493F03E-CC40-3D89-9A08-5F259BA2B246";marker_value=0;assert(probe_gate(&dev,c)==-ENODEV);marker_value=1; struct apple_dcp_v14 m={.dev=&dev,.board=c};width=3456;height=2160;stride=width*4;notch=74; assert(dcp_v14_geometry(&m)==0&&m.panel_height==2234&&m.notch_rows==74); height=2234;assert(dcp_v14_geometry(&m)==0&&m.panel_height==2234); From e6d003866120cadbd2106b505546cb8d53ceadc4 Mon Sep 17 00:00:00 2001 From: Marius Gehrmann Date: Mon, 12 Oct 2026 21:12:44 +0200 Subject: [PATCH 1668/2680] drm: apple: Use a RAM page for the T8122 dcpext's bw-scratch mapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The T8122 dcpext has apple,bw-scratch and no apple,bw-doorbell, and asks to map the PMGR bandwidth page through map_reg. With that MMIO page mapped, the DART reports a write fault when the display powers off, the DCP crashes, and the display does not come back after the first blank. For the T8122 dcpext only (the 14.7 template), hand it a RAM page instead: a device-managed coherent page of the range's size. The T8122 dcpext keeps apple,bw-scratch = <&pmgr_dcp 0 5 0x910>. The RAM page is compiled into the 14.7 template instance only, which only apple,t8122-dcpext uses. The 12.3 and 13.x instances keep mapping the MMIO page through dma_map_resource(), so the M2 and the M1/M2 Pro/Max/Ultra DCPs, which also have bw-scratch without bw-doorbell, are unchanged. Assisted-by: Claude:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marius Gehrmann [Chris Kearney: rebased onto integration/aurora 1bf7fffcd67b; the RAM page limited to the 14.7 template instance (DCP_FW_VER >= 14.7.0) in a helper, so the 13.3 instance used by M2 and M1/M2 Pro/Max is unchanged; subject, message and comment reworded; Claude-Session trailer dropped] [ryan: adapted to 7.2.10 — in this tree the 14.7 template instance also drives generic M1/M2 DCPs on 14.7.0 (DCP_FIRMWARE_V_14_7_TEMPLATE) and the template is also built for H17P: the RAM page is limited to DCP_FW_LAYOUT_14_7() and to an apple,t8122-dcpext node at run time, and the H17P instance keeps its physical dva] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 18e48afd8ee2b1f6a7575fe13d7da678abad44d9) --- drivers/gpu/drm/apple/iomfb_template.c | 31 ++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index a9f35b4c900ce..78f6eec0652a6 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -853,6 +853,34 @@ static u64 dcpep_cb_get_frequency(struct apple_dcp *dcp) return clk_get_rate(dcp->clk); } +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) && DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) +/* The DVA that map_reg hands out for display register range @index. */ +static dma_addr_t dcpep_map_reg_dva(struct apple_dcp *dcp, u32 index, + struct resource *rsrc) +{ +#if DCP_FW_LAYOUT_14_7(DCP_FW_VER) + /* + * T8122 dcpext: with the PMGR bandwidth page mapped, the DART faults + * at display power-off and the DCP crashes. Give it a RAM page + * instead. Generic M1/M2 DCPs on 14.7.0 also use this template + * instance and keep the MMIO page. + */ + if (of_device_is_compatible(dcp->dev->of_node, "apple,t8122-dcpext") && + dcp->disp_bw_scratch_index && !dcp->disp_bw_doorbell_index && + index == dcp->disp_bw_scratch_index) { + dma_addr_t dva; + + if (!dmam_alloc_coherent(dcp->dev, resource_size(rsrc), &dva, + GFP_KERNEL)) + return DMA_MAPPING_ERROR; + return dva; + } +#endif + return dma_map_resource(dcp->dev, rsrc->start, resource_size(rsrc), + DMA_BIDIRECTIONAL, 0); +} +#endif + static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *dcp, struct DCP_FW_NAME(dcp_map_reg_req) *req) { @@ -872,8 +900,7 @@ static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *d */ dma_addr_t dva = rsrc->start; #elif DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - dma_addr_t dva = dma_map_resource(dcp->dev, rsrc->start, resource_size(rsrc), - DMA_BIDIRECTIONAL, 0); + dma_addr_t dva = dcpep_map_reg_dva(dcp, req->index, rsrc); WARN_ON(dva == DMA_MAPPING_ERROR); #endif From f842b5c539763cfad80e5ad3c284f78aee8709cd Mon Sep 17 00:00:00 2001 From: Marius Gehrmann Date: Tue, 13 Oct 2026 12:13:26 +0200 Subject: [PATCH 1669/2680] drm: apple: Drive the T8122 dcpext through the 14.7 IOMFB template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aurora accepts 14.7 firmware only for processors that the boot loader hands over (T6030, the J613 panel) and drives them through its DCPLink session. The T8122 dcpext is not handed over: Linux starts it, as on M1/M2. asahi-wip-7.2 with AsahiLinux/linux#607 drives this dcpext with 14.7 firmware through the classic IOMFB template. Add that path, for apple,t8122-dcpext only: - iomfb_v14_7_tpl.c: the asahi-wip-7.2 14.7 method and callback tables around the shared template (symbols *_v14_7_0, no clash with Aurora's iomfb_v14_7_*); - DCP_FIRMWARE_V_14_7_TEMPLATE, selected by dcp_check_firmware_version() for apple,t8122-dcpext with firmware-compat 14.7.0, and dispatched like 12.3/13.5 (component bind, RTKit boot, dcp_start endpoints, template modeset/flush/power), so that every path that tests DCP_FIRMWARE_V_14_7 (the DCPLink session, T6030) is left alone; - dcp_fw_is_14_7() for the firmware protocol of the AFK, DPTX and DPAV endpoints, which both paths share; external backlight levels stay off on 14.7 either way; - the plane id as surface id in swaps, as asahi-wip-7.2 does for 14.7; - an apple,t8122-dcpext match entry (2 DPTX ports, as generic dcpext); - AFK: after START_ENDPOINT give the firmware 10 ms to set the endpoint up, and poll the mailbox while waiting for the init acknowledgment, because the mailbox of a freshly booted processor can leave messages queued without another interrupt. The internal panel (apple,t8122-dcp, DCP_FIRMWARE_V_14_7) and every other processor are unchanged. Link: https://github.com/AsahiLinux/linux/pull/607 Assisted-by: Claude:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 Co-developed-by: Ramzi Abdiche Signed-off-by: Ramzi Abdiche Signed-off-by: Marius Gehrmann [Chris Kearney: rebased onto integration/aurora 1bf7fffcd67b; Co-developed-by and Signed-off-by added for Ramzi Abdiche, whose AsahiLinux/linux#607 commits this ports (AFK start polling, the 14.7 tiling callback slots); one comment and one message line reworded; Claude-Session trailer dropped] [ryan: adapted to 7.2.10 — this tree already has the 14.7 template instance (iomfb_v14_7_tpl.*, built and dispatched for DCP_FIRMWARE_V_14_7_TEMPLATE since the 107036c port, enum in version_utils.h), its swap union member and plane-id surf_ids, so those hunks are dropped; taken: the apple,t8122-dcpext match entry and firmware check (compat 14.7.0 -> TEMPLATE), dcp_fw_is_14_7() and the polled AFK start; because generic M1/M2 14.7.0 DCPs also run on TEMPLATE here, dcp_fw_is_14_7() and the polled start apply on TEMPLATE only where the match data sets the new apple_dcp_hw_data.fw_14_7_endpoints (T8122 dcpext); dcp_fw_is_14_7() replaces the V_14_7 tests of this tree's equivalent sites: the AFK short reply, NOTIFY reply and START_ACK drain, dptxport_prepare() name/class admission (this tree has one dptxep ops table), the not-announced DPTX port, the DPAV open-once and the external backlight] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 869a3351d0828668c7b56b63d4406bf0e379eca7) --- drivers/gpu/drm/apple/afk.c | 48 ++++++++++++++++++++++--- drivers/gpu/drm/apple/dcp-internal.h | 20 +++++++++++ drivers/gpu/drm/apple/dcp.c | 18 ++++++++++ drivers/gpu/drm/apple/dptxep.c | 4 +-- drivers/gpu/drm/apple/epic/dpavservep.c | 2 +- drivers/gpu/drm/apple/iomfb.c | 2 +- 6 files changed, 85 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 4c9db64069d73..89eef9a80b292 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -3,8 +3,10 @@ #include #include +#include #include #include +#include #include #include #include @@ -168,6 +170,39 @@ void afk_shutdown(struct apple_dcp_afkep *afkep) } } +/* + * A processor that Linux starts itself with 14.7 firmware (the T8122 + * dcpext): as in AsahiLinux/linux#607, give the firmware time to set up + * the endpoint after START_ENDPOINT, and poll the mailbox while waiting, + * because the mailbox can leave messages queued without another interrupt + * after RTKit bootstrap. + */ +static int afk_start_polled(struct apple_dcp_afkep *ep) +{ + unsigned long timeout; + int ret; + + usleep_range(10000, 11000); + afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_INIT)); + + timeout = jiffies + msecs_to_jiffies(1000); + do { + ret = apple_rtkit_poll(ep->dcp->rtk); + if (ret < 0) + return ret; + + ret = wait_for_completion_timeout(&ep->started, + msecs_to_jiffies(10)); + if (ret > 0) + return 0; + } while (time_before(jiffies, timeout)); + + dev_err(ep->dcp->dev, + "RTKit AFK endpoint %02x did not acknowledge init\n", + ep->endpoint); + return -ETIMEDOUT; +} + int afk_start(struct apple_dcp_afkep *ep) { int ret; @@ -184,6 +219,9 @@ int afk_start(struct apple_dcp_afkep *ep) ret = apple_rtkit_start_ep(ep->dcp->rtk, ep->endpoint); if (ret) return ret; + if (ep->dcp->fw_compat == DCP_FIRMWARE_V_14_7_TEMPLATE && + ep->dcp->hw.fw_14_7_endpoints) + return afk_start_polled(ep); afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_INIT)); ret = wait_for_completion_timeout(&ep->started, msecs_to_jiffies(1000)); @@ -729,7 +767,7 @@ static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, * service teardown. Use the normal result path to release late-ack DMA * and command slots with their original allocation lengths. */ - if (ep->dcp->fw_compat == DCP_FIRMWARE_V_14_7 && + if (dcp_fw_is_14_7(ep->dcp) && eshdr->category == EPIC_CAT_REPLY && payload_size >= 4 && payload_size < sizeof(struct epic_cmd)) { struct epic_cmd cmd = {}; @@ -1205,7 +1243,7 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, * REPLY and whose body is a normal epic_cmd. The tag check inside the * reply handler ignores anything that is not one of our commands. */ - if (ep->dcp->fw_compat == DCP_FIRMWARE_V_14_7 && + if (dcp_fw_is_14_7(ep->dcp) && (type == EPIC_TYPE_REPLY || type == EPIC_TYPE_NOTIFY) && eshdr->category == EPIC_CAT_REPLY && payload_size >= sizeof(struct epic_cmd)) @@ -1213,7 +1251,7 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, le16_to_cpu(eshdr->tag), payload, payload_size); - if (ep->dcp->fw_compat != DCP_FIRMWARE_V_14_7 && + if (!dcp_fw_is_14_7(ep->dcp) && type == EPIC_TYPE_REPLY && eshdr->category == EPIC_CAT_REPLY) return afk_recv_handle_reply(ep, channel, le16_to_cpu(eshdr->tag), payload, @@ -1355,8 +1393,8 @@ static void afk_receive_message_worker(struct work_struct *work_) break; case RBEP_START_ACK: - /* T6030 14.7 may queue messages before it acks the start. */ - if (work->ep->dcp->fw_compat == DCP_FIRMWARE_V_14_7 && + /* 14.7 may queue messages before it acks the start. */ + if (dcp_fw_is_14_7(work->ep->dcp) && work->ep->rxbfr.ready) afk_recv_drain(work->ep); complete_all(&work->ep->started); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index d785a06a4e880..be6479050d6f1 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -194,6 +194,12 @@ struct apple_dcp_hw_data { u32 firmware_request; /* M2 Pro/Max laptop T6020 Thunderbolt flow, from the match data */ bool t6020_tunnel_flow; + /* + * A processor Linux starts itself on 14.7 firmware through the IOMFB + * template whose AFK, DPTX and DPAV endpoints speak the 14.7 protocol + * (the T8122 dcpext), see dcp_fw_is_14_7(). + */ + bool fw_14_7_endpoints; }; enum dcp_test_point { @@ -674,6 +680,20 @@ int dcp_backlight_update(struct apple_dcp *dcp); s32 dcp_ext_backlight_value(struct apple_dcp *dcp); bool dcp_has_panel(struct apple_dcp *dcp); +/* + * Does @dcp's firmware speak the 14.7 AFK, DPTX and DPAV endpoint protocol, + * whichever IOMFB path drives it? DCP_FIRMWARE_V_14_7 is the native DCPLink + * session; on DCP_FIRMWARE_V_14_7_TEMPLATE only the processors marked by + * their match data (the T8122 dcpext): generic M1/M2 DCPs on 14.7.0 keep the + * endpoint handling they had. + */ +static inline bool dcp_fw_is_14_7(const struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_V_14_7 || + (dcp->fw_compat == DCP_FIRMWARE_V_14_7_TEMPLATE && + dcp->hw.fw_14_7_endpoints); +} + #define DCP_AUDIO_MAX_CHANS 15 #endif /* __APPLE_DCP_INTERNAL_H__ */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 3965f5a927014..fbdb7e0d87fb3 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3030,6 +3030,18 @@ static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) return DCP_FIRMWARE_UNKNOWN; } + /* + * The T8122 external processor: Linux starts it and drives 14.7 + * firmware through the classic IOMFB template, as on M1/M2. + */ + if (of_device_is_compatible(dev->of_node, "apple,t8122-dcpext")) { + if (!strcmp(compat_str, "14.7.0")) + return DCP_FIRMWARE_V_14_7_TEMPLATE; + dev_err(dev, "T8122 external display not started: DCP firmware-compat %s is not 14.7.0\n", + compat_str); + return DCP_FIRMWARE_UNKNOWN; + } + /* The internal panels with a board record (T6030, J613): the same. */ v14_board = iomfb_v14_7_board(dev); if (IS_ERR(v14_board)) @@ -4364,6 +4376,11 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t6030_dcpext = { .num_dptx_ports = 1, }; +static const struct apple_dcp_hw_data apple_dcp_hw_t8122_dcpext = { + .num_dptx_ports = 2, + .fw_14_7_endpoints = true, +}; + static const struct apple_dcp_hw_data apple_dcp_hw_dcpext = { .num_dptx_ports = 2, }; @@ -4379,6 +4396,7 @@ static const struct of_device_id of_match[] = { { .compatible = "apple,t6030-dcpext", .data = &apple_dcp_hw_t6030_dcpext, }, { .compatible = "apple,t8122-dcp", .data = &apple_dcp_hw_t8122, }, { .compatible = "apple,t6031-dcp", .data = &apple_dcp_hw_t6031, }, + { .compatible = "apple,t8122-dcpext", .data = &apple_dcp_hw_t8122_dcpext, }, { .compatible = "apple,dcp", .data = &apple_dcp_hw_dcp, }, { .compatible = "apple,dcpext", .data = &apple_dcp_hw_dcpext, }, {} diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 3da358642cba2..c090518662a5d 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -904,7 +904,7 @@ static int dptxport_prepare(struct apple_epic_service *service, const char *name * T6030 14.7 also announces the low-power port service, and may leave * its class empty. */ - if (dcp->fw_compat == DCP_FIRMWARE_V_14_7 && name && + if (dcp_fw_is_14_7(dcp) && name && (!strcmp(name, "dcpdptx-port-epic") || !strcmp(name, "dcp-lpdptx-port-epic")) && (!class || !class[0] || !strcmp(class, "AppleDCPDPTXRemotePort"))) @@ -1128,7 +1128,7 @@ int dptxep_init(struct apple_dcp *dcp) * kIOReturnNoDevice (0xe00002c0). Leave the endpoint * up so a later announce can still bind. */ - if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { + if (dcp_fw_is_14_7(dcp)) { dev_info(dcp->dev, "DPTX port %u not announced yet\n", port); return 0; diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 4dc0f1000fe3f..5b17f4b9aa8a5 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -423,7 +423,7 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation * The 14.7 firmware of an external processor answers EDID requests * once its service has been opened; open each announced instance once. */ - if (dcp->fw_compat == DCP_FIRMWARE_V_14_7 && + if (dcp_fw_is_14_7(dcp) && READ_ONCE(dcp->dcpavserv.opened) != service) { int ret = afk_service_call(service, 4, 6, NULL, 0, 32, NULL, 0, 32); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index cbe3901f6e438..b1362faeff036 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1722,7 +1722,7 @@ int dcp_attributes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *handle, * carry no external backlight level. */ ext = !ret && backlight_control && !dcp_has_panel(dcp) && - dcp->fw_compat != DCP_FIRMWARE_V_14_7; + !dcp_fw_is_14_7(dcp); if (ext != READ_ONCE(dcp->ext_backlight)) { WRITE_ONCE(dcp->ext_backlight, ext); if (dcp->connector) From e61f4fe55955b2476c214cab2e6f40fc159d1f1d Mon Sep 17 00:00:00 2001 From: Marius Gehrmann Date: Mon, 12 Oct 2026 21:12:48 +0200 Subject: [PATCH 1670/2680] drm: apple: Never let a failed T8122 dcpext hold up the internal panel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit apple_drv adds every available apple,dcpext node as a component of the panel's DRM device. If the dcpext does not register its component (probe failure) or fails its bind (RTKit boot), the device never binds or fails as a whole, and the internal panel stays on the boot framebuffer. The dcpext's Type-C route is also a mode switch that the USB-C port controller waits for. For apple,t8122-dcpext only: - probe runs in a devres group. -EPROBE_DEFER is passed on while driver_deferred_probe_check_state() still defers (bounded by the deferred probe timeout). Any other failure releases the group, and the device joins as an unusable pipe (ext_dead): a minimal component whose bind does nothing, with its Type-C routes disabled in the live tree so that the port controller stops waiting for them. A failure after a route was registered keeps the pipe in place instead (a consumer may already hold the mux) and marks it unusable. - a bind failure marks the pipe unusable and returns 0. - an unusable pipe gets no CRTC (apple_drm_init_dcp() skips it), its port gets no connector instead of failing apple_probe_typec_ports(), its route ignores mode changes, its RTKit messages are dropped, and unbind/suspend/resume skip it. Other DCPs, the internal panel's included, are unchanged. Assisted-by: Claude:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marius Gehrmann [Chris Kearney: rebased onto integration/aurora 1bf7fffcd67b; Claude-Session trailer dropped] [ryan: adapted to 7.2.10 — the ext_dead early return in dcp_recv_msg() comes before this tree's H17P unhandled-endpoint logging, and resume skips a dead pipe before dcp_release_after_sleep(); the probe devres group also releases this tree's kzalloc'd context through dcp_release_context() (a T8122 dcpext never retains DMA); prepare/complete/shutdown/remove need no guard here, as they only act on external, H17P or adopted-session DCPs] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit f07166b2b82e4ad4f0f0c44f25516bdb3a1d5411) --- drivers/gpu/drm/apple/apple_drv.c | 14 ++++ drivers/gpu/drm/apple/dcp-fabric.c | 27 +++++++ drivers/gpu/drm/apple/dcp-fabric.h | 1 + drivers/gpu/drm/apple/dcp-internal.h | 8 ++ drivers/gpu/drm/apple/dcp.c | 111 ++++++++++++++++++++++++++- drivers/gpu/drm/apple/dcp.h | 1 + 6 files changed, 159 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index db515d81e933a..c5db84d31e5a8 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -644,6 +644,13 @@ static int apple_probe_typec_ports(struct drm_device *drm, bool secondary = idx >= nr_ports; u32 mask = 0; + /* Only a failed T8122 dcpext could drive it: no connector. */ + if (dcp_typec_port_unusable(port_idx)) { + drm_warn(drm, "Type-C port %u: its display pipeline is unusable\n", + port_idx); + continue; + } + connector = kzalloc_obj(*connector); if (!connector) return -ENOMEM; @@ -828,6 +835,13 @@ static int apple_drm_init_dcp(struct device *dev) dcp[num_dcp] = of_find_device_by_node(np); if (!dcp[num_dcp]) continue; + /* A failed T8122 dcpext: bound, but no pipeline. */ + if (dcp_ext_unusable(dcp[num_dcp])) { + dev_warn(dev, "%s: external display processor unusable, skipped\n", + dev_name(&dcp[num_dcp]->dev)); + put_device(&dcp[num_dcp]->dev); + continue; + } device_link_add(dev, &dcp[num_dcp]->dev, DL_FLAG_AUTOREMOVE_SUPPLIER); diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index cd784df522a4b..531b69920800a 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -1358,6 +1358,10 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, bool was_counted; int ret = 0; + /* A failed T8122 dcpext drives nothing; USB on the port goes on. */ + if (READ_ONCE(route->dcp->ext_dead)) + return 0; + guard(mutex)(&dcp_typec_fabric_lock); if (state->mode == TYPEC_STATE_SAFE && port->safe_pending) @@ -3145,6 +3149,29 @@ bool dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev return false; } +/* + * Is every pipeline that can drive port @idx unusable (a failed T8122 + * dcpext)? Such a port gets no connector, rather than failing the DRM + * device of the internal panel. + */ +bool dcp_typec_port_unusable(unsigned int idx) +{ + struct apple_dcp_typec_port *port; + struct apple_dcp_typec_route *route; + + guard(mutex)(&dcp_typec_fabric_lock); + + port = dcp_typec_port_by_index(idx); + if (!port || list_empty(&port->routes)) + return false; + + list_for_each_entry(route, &port->routes, port_link) + if (!READ_ONCE(route->dcp->ext_dead)) + return false; + + return true; +} + void dcp_typec_port_set_connector(unsigned int idx, bool secondary, struct apple_connector *connector) { diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index ce185187610c4..04d6dbb3449cc 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -66,6 +66,7 @@ bool dcp_has_typec_routes(struct platform_device *pdev); unsigned int dcp_typec_nr_ports(void); struct device_node *dcp_typec_port_of_node(unsigned int idx); bool dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev); +bool dcp_typec_port_unusable(unsigned int idx); void dcp_typec_port_set_connector(unsigned int idx, bool secondary, struct apple_connector *connector); bool dcp_typec_dual_stream(void); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index be6479050d6f1..5acf9f48ececb 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -259,6 +259,14 @@ struct apple_dcp { /* firmware version and compatible firmware version */ enum dcp_firmware_version fw_compat; bool external; + /* + * T8122 dcpext: the panel's DRM device never waits on this processor. + * A probe or bind failure leaves it in the device as an unusable pipe + * (ext_dead): no CRTC, Type-C routes disabled or ignored, RTKit + * messages dropped. See dcp_platform_probe(). + */ + bool ext_failsafe; + bool ext_dead; /* * A T6030 external processor in the main DRM device: the kernel starts * its firmware on first use, its modes come from the attached display. diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index fbdb7e0d87fb3..681193089655a 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -723,6 +723,10 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) trace_dcp_recv_msg(dcp, endpoint, message); + /* A failed T8122 dcpext: its firmware may still talk, nobody listens. */ + if (READ_ONCE(dcp->ext_dead)) + return; + /* * H17P starts endpoints that have no AFK instance behind them (the * remote allocator endpoint, for one). Log their messages rather than @@ -2069,6 +2073,14 @@ bool dcp_fw_compat_is_12_x(struct platform_device *pdev) return dcp->fw_compat == DCP_FIRMWARE_V_12_3; } +/* A failed T8122 dcpext: no pipeline of the DRM device. */ +bool dcp_ext_unusable(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return !dcp || READ_ONCE(dcp->ext_dead); +} + bool dcp_fw_compat_is_14_7(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); @@ -3183,7 +3195,7 @@ static bool dcp_show_notch(struct apple_dcp *dcp) return show_notch || t6030_show_notch; } -static int dcp_comp_bind(struct device *dev, struct device *main, void *data) +static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *data) { struct device_node *panel_np; struct apple_dcp *dcp = dev_get_drvdata(dev); @@ -3418,6 +3430,27 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) return ret; } +static void dcp_ext_give_up(struct apple_dcp *dcp, int err, const char *when); + +/* + * The T8122 dcpext boots its firmware here. If that fails, it must not take + * the internal panel's DRM device down with it: keep it as an unusable pipe. + */ +static int dcp_comp_bind(struct device *dev, struct device *main, void *data) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + int ret; + + if (READ_ONCE(dcp->ext_dead)) + return 0; + ret = dcp_comp_bind_pipe(dev, main, data); + if (ret && dcp->ext_failsafe) { + dcp_ext_give_up(dcp, ret, "bind"); + return 0; + } + return ret; +} + /* * We need to shutdown DCP before tearing down the display subsystem. Otherwise * the DCP will crash and briefly flash a green screen of death. @@ -3426,7 +3459,7 @@ VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, v { struct apple_dcp *dcp = dev_get_drvdata(dev); - if (!dcp) + if (!dcp || READ_ONCE(dcp->ext_dead)) return; disable_work_sync(&dcp->dimensions_wq); @@ -3633,7 +3666,7 @@ static int dcp_native_routes_error(struct apple_dcp *dcp, int err) return 0; } -static int dcp_platform_probe(struct platform_device *pdev) +static int dcp_platform_probe_pipe(struct platform_device *pdev) { const struct apple_dcp_hw_data *hw = of_device_get_match_data(&pdev->dev); enum dcp_firmware_version fw_compat; @@ -3684,6 +3717,7 @@ static int dcp_platform_probe(struct platform_device *pdev) iomfb_queue_init(dcp); dcp->fw_compat = fw_compat; + dcp->ext_failsafe = of_device_is_compatible(dev->of_node, "apple,t8122-dcpext"); dcp->external = of_device_is_compatible(dev->of_node, "apple,t6030-dcpext"); dcp->external_native = dcp->external && dcp_t6030_ext_native(dev->of_node); dcp->dev = dev; @@ -4002,6 +4036,71 @@ static int dcp_platform_probe(struct platform_device *pdev) return ret; } +/* + * Marks a T8122 dcpext unusable, for good: its Type-C routes are disabled in + * the device tree, so that a USB-C port controller probing later does not + * wait for them, and a route already registered ignores mode changes (see + * dcp_typec_route_set()), so that USB on the port goes on. + */ +static void dcp_ext_give_up(struct apple_dcp *dcp, int err, const char *when) +{ + WRITE_ONCE(dcp->ext_dead, true); + dev_err(dcp->dev, "external display processor unusable (%s failed: %d); the internal display goes on without it\n", + when, err); + dcp_typec_routes_disable(dcp); +} + +/* + * The T8122 dcpext is a component of the panel's DRM device, so it must + * always register one, or the panel stays on the boot framebuffer. Keep + * deferring while its suppliers may still appear (bounded by the deferred + * probe timeout); on any other failure join as an unusable pipe. + */ +static int dcp_platform_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_dcp *dcp; + void *group; + int ret; + + if (!of_device_is_compatible(dev->of_node, "apple,t8122-dcpext")) + return dcp_platform_probe_pipe(pdev); + + group = devres_open_group(dev, NULL, GFP_KERNEL); + if (!group) + return -ENOMEM; + ret = dcp_platform_probe_pipe(pdev); + if (!ret) { + devres_remove_group(dev, group); + return 0; + } + + dcp = platform_get_drvdata(pdev); + if (dcp && dcp->nr_typec_routes) { + /* Routes may be in use already: keep them, and the pipe, as they are. */ + devres_remove_group(dev, group); + dcp_ext_give_up(dcp, ret, "probe"); + return component_add(dev, &dcp_comp_ops); + } + + devres_release_group(dev, group); + platform_set_drvdata(pdev, NULL); + if (ret == -EPROBE_DEFER) { + ret = driver_deferred_probe_check_state(dev); + if (ret == -EPROBE_DEFER) + return ret; + } + + dcp = devm_kzalloc(dev, sizeof(*dcp), GFP_KERNEL); + if (!dcp) + return -ENOMEM; + dcp->dev = dev; + dcp->ext_failsafe = true; + platform_set_drvdata(pdev, dcp); + dcp_ext_give_up(dcp, ret, "probe"); + return component_add(dev, &dcp_comp_ops); +} + static void dcp_platform_remove(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); @@ -4212,6 +4311,9 @@ static int dcp_platform_suspend(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); + if (READ_ONCE(dcp->ext_dead)) + return 0; + /* Serialize PM against explicit bring-up, including queued startup work. * Before a startup attempt the device remains suspendable. After any * attempt we cannot prove DMA/firmware quiescence, even on failure. @@ -4282,6 +4384,9 @@ static int dcp_platform_resume(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); + if (READ_ONCE(dcp->ext_dead)) + return 0; + dcp_release_after_sleep(dcp); dcp_enable_typec_work(dcp); /* The HDMI output's PHY and crossbar may have lost their setup in sleep. */ diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 687ca5c2c413e..8f302a54beb4d 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -49,6 +49,7 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commi int dcp_get_connector_type(struct platform_device *pdev); bool dcp_fw_compat_is_12_x(struct platform_device *pdev); bool dcp_fw_compat_is_14_7(struct platform_device *pdev); +bool dcp_ext_unusable(struct platform_device *pdev); unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev); int dcp_start(struct platform_device *pdev); int dcp_wait_ready(struct platform_device *pdev, u64 timeout); From daf166ca7c40f26d825623079534ad11fefc564b Mon Sep 17 00:00:00 2001 From: Marius Gehrmann Date: Tue, 13 Oct 2026 12:13:46 +0200 Subject: [PATCH 1671/2680] drm: apple: Start and connect the T8122 dcpext as its firmware expects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three things the T8122 dcpext firmware (14.7, classic IOMFB template) needs and the shared paths do differently; each applies to DCP_FIRMWARE_V_14_7_TEMPLATE only. PMP: the display gate gives the dcpext a PMP DISPEXT request (apple,pmp-report) when the PMP runs. As T6030 does for its dcpext (dcpext_check_power()), run the processor only once the PMP has acknowledged that request, in the bind before the CPU RUN bit is set and RTKit is woken. Without it the first modeset hangs. The internal display has started the PMP long before, so the wait is short; apple_pmp_report_wait_ready() reads the acknowledgment once after the PMP start, so poll it (20 ms) for up to 3 s while it returns -EIO. On any failure, or without a PMP, the processor is started anyway, with one log line saying so. EDID: dcpavserv_copy_edid() sends Aurora's T6030 "open" call (DPAV service group 4, command 6) before the first EDID read. The dcpext never answers it: copy_edid times out, the firmware drops and retrains the link, and the connector flaps every ~4 s. AsahiLinux/linux#607 reads the EDID with group 1, command 7 directly; send the open call only on Aurora's DCPLink session (DCP_FIRMWARE_V_14_7), as before. DPTX connect: the fabric validates the target (group 0, cmd 12), connects with the supports-HPD attribute, answers GET_SUPPORTS_HPD with 1, asserts HPD right after request_display and acknowledges SetTiledDisplayHints with 1. #607 does none of that: it connects with 0, answers 0, asserts HPD only after the link-configuration wait (that HPD starts link training) and acknowledges the hints with 0. Do it the #607 way; a link-configuration timeout before the late HPD is expected and not an error. Tested on J613 with the rest of the series: rear and front USB-C port, hotplug, DPMS, 1920x1080@60 and 3440x1440@100. Link: https://github.com/AsahiLinux/linux/pull/607 Assisted-by: Claude:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marius Gehrmann [Chris Kearney: rebased onto integration/aurora 1bf7fffcd67b; Claude-Session trailer dropped] [ryan: adapted to 7.2.10 — dcp_ext_dptx_607() and the PMP wait apply on TEMPLATE only to the T8122 dcpext (apple_dcp_hw_data.fw_14_7_endpoints), not to the generic M1/M2 DCPs this tree also runs on the template; the DISPEXT PMP wait runs before this tree's non-adopted CPU RUN write and calls apple_pmp_report_wait_ready() (170-usb) through symbol_get(), with an -ENODEV fallback, as iomfb_v14_7.c does; the late-HPD connect uses this tree's out_withdraw path; GET_SUPPORTS_HPD keeps this tree's dev_dbg; no linux/soc/apple/j613-display.h include] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit abe4a822c3f080d681a1bbd2f6b709e0b2acb06b) --- drivers/gpu/drm/apple/dcp-internal.h | 15 ++++ drivers/gpu/drm/apple/dcp.c | 109 ++++++++++++++++++++---- drivers/gpu/drm/apple/dptxep.c | 18 ++-- drivers/gpu/drm/apple/epic/dpavservep.c | 8 +- 4 files changed, 128 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 5acf9f48ececb..3001438796070 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -702,6 +702,21 @@ static inline bool dcp_fw_is_14_7(const struct apple_dcp *dcp) dcp->hw.fw_14_7_endpoints); } +/* + * The T8122 dcpext connects its DPTX port as AsahiLinux/linux#607 does, + * the kernel this firmware was first driven with: no validation of the + * target, a connect without the supports-HPD attribute, GET_SUPPORTS_HPD + * answered with 0, HPD asserted only after the link-configuration wait, + * and tiled display hints acknowledged with 0. Every other DCP keeps the + * Type-C model of dcp_dptx_connect_session(). + */ +static inline bool dcp_ext_dptx_607(const struct apple_dcp *dcp) +{ + /* Not the generic M1/M2 DCPs this tree also runs on the template. */ + return dcp->fw_compat == DCP_FIRMWARE_V_14_7_TEMPLATE && + dcp->hw.fw_14_7_endpoints; +} + #define DCP_AUDIO_MAX_CHANS 15 #endif /* __APPLE_DCP_INTERNAL_H__ */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 681193089655a..d42d14cacc76f 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -974,6 +975,8 @@ int dcp_get_connector_type(struct platform_device *pdev) */ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable) { + /* T8122 dcpext: HPD only after the link wait, see dcp_ext_dptx_607() */ + bool late_hpd = dcp_ext_dptx_607(dcp); unsigned long timeout; u8 dfp_port; int ret = 0, err; @@ -1023,20 +1026,22 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable dcp->dptxport[port].atcphy = dcp->phy; /* a tiled display's second half comes in on the port's dpin1 */ dfp_port = port && READ_ONCE(dcp->split.active) ? 2 : dcp->dptx_dfp_port; - ret = dptxport_validate_connection(dcp->dptxport[port].service, - dfp_port, - dcp->dptx_phy, dcp->dptx_die); - if (ret) { - dev_err(dcp->dev, - "dcp_dptx_connect: failed to validate DPTX target %u:%u: %d\n", - dcp->dptx_die, dcp->dptx_phy, ret); - goto out_unlock; + if (!dcp_ext_dptx_607(dcp)) { + ret = dptxport_validate_connection(dcp->dptxport[port].service, + dfp_port, + dcp->dptx_phy, dcp->dptx_die); + if (ret) { + dev_err(dcp->dev, + "dcp_dptx_connect: failed to validate DPTX target %u:%u: %d\n", + dcp->dptx_die, dcp->dptx_phy, ret); + goto out_unlock; + } } ret = dptxport_connect(dcp->dptxport[port].service, dfp_port, dcp->dptx_phy, dcp->dptx_die, - dcp_is_typec_output(dcp)); + dcp_is_typec_output(dcp) && !dcp_ext_dptx_607(dcp)); if (ret) { dev_err(dcp->dev, "dcp_dptx_connect: failed to connect DPTX target %u:%u: %d\n", @@ -1052,7 +1057,7 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable goto out_release; } dcp->dptxport[port].connected = true; - if (dcp_is_typec_output(dcp)) { + if (dcp_is_typec_output(dcp) && !late_hpd) { if (dcp_uses_t6020_tunnel_flow(dcp)) ret = dptxport_set_hpd_timeout(dcp->dptxport[port].service, true, 8000); @@ -1081,23 +1086,32 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable DPTX_TUNNEL_CONNECT_TIMEOUT : DPTX_CONNECT_TIMEOUT; ret = wait_for_completion_timeout(&dcp->dptxport[port].linkcfg_completion, timeout); - if (!ret) { + if (!ret && late_hpd) { + /* Without HPD this firmware does not train: assert it now. */ + dev_dbg(dcp->dev, + "dcp_dptx_connect: no link configuration on port %u yet, asserting HPD\n", + port); + } else if (!ret) { dev_err(dcp->dev, "dcp_dptx_connect: timed out waiting for port %u link configuration\n", port); ret = -ETIMEDOUT; goto out_withdraw; + } else { + dev_dbg(dcp->dev, "dcp_dptx_connect: waited %d ms for link\n", + jiffies_to_msecs(timeout - ret)); } - dev_dbg(dcp->dev, "dcp_dptx_connect: waited %d ms for link\n", - jiffies_to_msecs(timeout - ret)); - usleep_range(5, 10); - if (dcp->connector_type == DRM_MODE_CONNECTOR_DisplayPort) { + if (dcp->connector_type == DRM_MODE_CONNECTOR_DisplayPort || late_hpd) { ret = dptxport_set_hpd(dcp->dptxport[port].service, true); + if (ret && late_hpd) + dev_err(dcp->dev, + "dcp_dptx_connect: failed to assert HPD after the link wait: %d\n", + ret); /* T6030 external: out_withdraw drops the request as out_disconnect did. */ - if (ret && dcp->external) + if (ret && (dcp->external || late_hpd)) goto out_withdraw; } if (dcp->external) { @@ -3195,6 +3209,67 @@ static bool dcp_show_notch(struct apple_dcp *dcp) return show_notch || t6030_show_notch; } +/* + * T8122 dcpext (DCP_FIRMWARE_V_14_7_TEMPLATE): if the PMP runs, the display + * gate added its DISPEXT request and pointed the dcpext at it + * (apple,pmp-report). As on T6030, run the processor only once the PMP has + * acknowledged that request: the running PMP owns the SoC power and + * performance states and treats a block whose request it has not taken as + * idle. The internal display started the PMP long before this, so the wait + * is short; it is bounded, and without an acknowledgement the processor is + * started anyway (as before the request existed), with a warning. + */ +#define DCP_EXT_PMP_TIMEOUT_MS 3000 + +/* + * The PMP report's wait-ready entry point is optional at link time: until a + * provider implements it, behave as its !CONFIG_APPLE_PMP_REPORT stub does. + */ +static int dcp_ext_pmp_wait_ready(struct device_node *entry, + unsigned long timeout) +{ + typeof(&apple_pmp_report_wait_ready) wait; + int ret; + + wait = symbol_get(apple_pmp_report_wait_ready); + if (!wait) + return -ENODEV; + ret = wait(entry, timeout); + symbol_put(apple_pmp_report_wait_ready); + return ret; +} + +static void dcp_ext_wait_pmp(struct apple_dcp *dcp) +{ + struct device_node *entry; + unsigned long deadline; + int ret; + + if (!dcp_ext_dptx_607(dcp)) + return; + entry = of_parse_phandle(dcp->dev->of_node, "apple,pmp-report", 0); + if (!entry) { + dev_info(dcp->dev, "no PMP DISPEXT request; starting without one\n"); + return; + } + deadline = jiffies + msecs_to_jiffies(DCP_EXT_PMP_TIMEOUT_MS); + for (;;) { + ret = dcp_ext_pmp_wait_ready(entry, + msecs_to_jiffies(DCP_EXT_PMP_TIMEOUT_MS)); + /* -EIO: the PMP runs, but has not acknowledged the request yet. */ + if (ret != -EIO || time_after(jiffies, deadline)) + break; + msleep(20); + } + of_node_put(entry); + if (ret) + dev_warn(dcp->dev, + "the PMP has not acknowledged the DISPEXT request (%d); starting anyway\n", + ret); + else + dev_info(dcp->dev, "PMP acknowledged the DISPEXT request\n"); +} + static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *data) { struct device_node *panel_np; @@ -3380,6 +3455,8 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat } } + dcp_ext_wait_pmp(dcp); + if (!dcp->hw.adopt_live_session) { cpu_ctrl = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index c090518662a5d..c3ef3c742375b 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -599,7 +599,8 @@ static int dptxport_call_get_supports_hpd(struct apple_epic_service *service, * DEVICE_NOT_STARTED (~5.5s). Advertise HPD so firmware uses * that path instead of waiting for a PHY start. */ - reply->supported = cpu_to_le32(dcp_is_typec_output(dcp) ? 1 : 0); + reply->supported = cpu_to_le32(dcp_is_typec_output(dcp) && + !dcp_ext_dptx_607(dcp) ? 1 : 0); dev_dbg(dcp->dev, "DPTXPort: GET_SUPPORTS_HPD %u usb4=%d\n", le32_to_cpu(reply->supported), dcp_is_usb4_output(dcp)); return 0; @@ -841,11 +842,18 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, service->ep->dcp->dptx_phy); memcpy(reply, data, min(reply_size, data_size)); return 0; - case DPTX_APCALL_SET_TILED_DISPLAY_HINTS: + case DPTX_APCALL_SET_TILED_DISPLAY_HINTS: { + int ret; + memcpy(reply, data, min(reply_size, data_size)); - return dptxport_call_set_tiled_display_hint(service, data, - data_size, reply, - reply_size); + ret = dptxport_call_set_tiled_display_hint(service, data, + data_size, reply, + reply_size); + /* The T8122 dcpext gets 0, as from AsahiLinux/linux#607. */ + if (!ret && reply_size >= 4 && dcp_ext_dptx_607(service->ep->dcp)) + memset(reply, 0, 4); + return ret; + } case DPTX_APCALL_GET_DRIVE_SETTINGS: return dptxport_call_get_drive_settings(service, data, data_size, reply, reply_size); diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 5b17f4b9aa8a5..bec8f711fbd14 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -422,8 +422,14 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation /* * The 14.7 firmware of an external processor answers EDID requests * once its service has been opened; open each announced instance once. + * + * Only on Aurora's DCPLink session (T6030). The T8122 dcpext, which + * Linux starts and drives through the IOMFB template, reads the EDID + * directly, as in AsahiLinux/linux#607. Its firmware never answers the + * open call and drops the display link instead, about 50 ms after it, + * which brings the display back 4 s later: hotplug in a loop. */ - if (dcp_fw_is_14_7(dcp) && + if (dcp->fw_compat == DCP_FIRMWARE_V_14_7 && READ_ONCE(dcp->dcpavserv.opened) != service) { int ret = afk_service_call(service, 4, 6, NULL, 0, 32, NULL, 0, 32); From 34d59d5610b2dea0aaf4354db52fac033bd4f2e3 Mon Sep 17 00:00:00 2001 From: Marius Gehrmann Date: Tue, 13 Oct 2026 10:01:28 +0200 Subject: [PATCH 1672/2680] drm: apple: Leave the next Type-C display alone when the old CRTC powers off When a Type-C display is unplugged while another port waits for the pipeline (the T8122 dcpext drives one display, a second one waits with -EBUSY), dcp_typec_pipeline_freed() hands the pipeline to the waiting port at once, and its connection starts before the desktop has turned the old display's CRTC off. That power-off then drops HPD and disconnects the DPTX port, which by now belongs to the new display: the firmware reports the new sink inactive and the display stays dark. Whether it works depends on who is faster, the desktop or the fabric; on J613 (display on the front port unplugged, a display on the rear port waiting) both orders were seen. Count the Type-C unplugs (dcp_dptx_disconnect_oob(), which already releases the link of the display that went) and, on the T8122 dcpext, remember the count when the CRTC is powered on. If a display was unplugged since, the power-off only powers IOMFB off and leaves the current connection, its reconnect work and its HPD completion alone. DPMS off of a display that is still attached is unchanged. Only the T8122 dcpext (ext_failsafe) takes the new path; it is the only pipeline this was run on. Every other Type-C output releases its connection at CRTC power-off exactly as before, including the M1/M2 dcpext routes and the M3 Pro's split (tiled) second port and external retry. Assisted-by: Claude:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marius Gehrmann [Chris Kearney: rebased onto integration/aurora 1bf7fffcd67b; the new power-on snapshot and power-off path limited to the T8122 dcpext (dcp->ext_failsafe), so other Type-C outputs keep the old release; Claude-Session trailer dropped] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 6d08793a774b69787abeea6f1711b4c3ab16d016) --- drivers/gpu/drm/apple/dcp-internal.h | 4 ++++ drivers/gpu/drm/apple/dcp.c | 32 ++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 3001438796070..4b73e2dfb2cae 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -387,6 +387,10 @@ struct apple_dcp { /* One HPD pulse after a placeholder EDID, per Type-C connection. */ bool placeholder_retried; u64 typec_generation; /* hpd_mutex: identifies the current connection */ + /* hpd_mutex: Type-C displays unplugged so far; see dcp_poweroff() */ + u64 typec_unplugs; + /* typec_unplugs when the CRTC was last powered on */ + u64 crtc_unplugs; u64 placeholder_generation; struct delayed_work placeholder_edid_wq; bool use_timestamps; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index d42d14cacc76f..f6f6e177fe977 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1964,6 +1964,7 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) WRITE_ONCE(dcp->typec_cable_connected, false); dcp->typec_generation++; dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); + dcp->typec_unplugs++; /* Nothing left to retry for the display that went. */ if (dcp->external_native) cancel_delayed_work(&dcp->external_retry_wq); @@ -2416,6 +2417,11 @@ void dcp_poweron(struct platform_device *pdev) } if (dcp_is_typec_output(dcp)) { + /* T8122 dcpext only, see dcp_poweroff() */ + if (dcp->ext_failsafe) { + scoped_guard(mutex, &dcp->hpd_mutex) + dcp->crtc_unplugs = dcp->typec_unplugs; + } wait_for_typec_hpd = READ_ONCE(dcp->typec_crtc_off) && READ_ONCE(dcp->typec_cable_connected); WRITE_ONCE(dcp->typec_crtc_off, false); @@ -2489,6 +2495,7 @@ void dcp_external_recover(struct platform_device *pdev) void dcp_poweroff(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); + bool display_gone = false; int ret; if (dcp_is_external_h17p(dcp)) { @@ -2515,6 +2522,31 @@ void dcp_poweroff(struct platform_device *pdev) if (dcp->avep) av_service_disconnect(dcp); + /* + * The display this CRTC was powered on for has been unplugged since: + * its link went down with the cable (dcp_dptx_disconnect_oob()), and + * the desktop only now turns the CRTC off. By then the pipeline may + * already drive the next display, on this port or on another one that + * was waiting for it, and that connection is not the CRTC's to + * release: dropping its HPD makes the firmware report the new display + * inactive, and it stays dark. + * + * Only the T8122 dcpext (ext_failsafe) takes this path, the one + * pipeline where it was run. Every other Type-C output releases its + * connection below as before: the M1/M2 dcpext routes, and the T6030 + * split second port and external retry. + */ + if (dcp->ext_failsafe && dcp_is_typec_output(dcp)) { + scoped_guard(mutex, &dcp->hpd_mutex) + display_gone = dcp->typec_unplugs != dcp->crtc_unplugs; + if (display_gone) { + dev_dbg(dcp->dev, + "CRTC off after its display was unplugged: leaving the current connection alone\n"); + _dcp_poweroff(dcp); + return; + } + } + /* * Powering a Type-C CRTC off drops DCP's synthetic HPD, and the firmware * reports that as an unplug. The display is still attached: keep the From 30c3839f56ed2fffffe07db5177f0ca55c9d79e4 Mon Sep 17 00:00:00 2001 From: Marius Gehrmann Date: Tue, 13 Oct 2026 11:32:21 +0200 Subject: [PATCH 1673/2680] drm: apple: Say when a Type-C display waits for a pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When every pipeline that could drive a DP-alt display drives another one (the T8122 dcpext drives a single display), the route returns -EBUSY and the display stays dark. The only trace is the port controller's "failed to configure Type-C connection: -16", which says nothing a user could act on. Log one line per plug, with the connector's label, that says what is going on and what to do: display on USB-C Left-front waits for an external display pipeline: it drives another display; to switch, unplug that one, then plug this one in again Desktop helpers can turn it into a notification. Plugging in again is what reliably works on J613: when the other display goes, a waiting display whose port runs four-lane DP is not brought up by the fabric alone. Assisted-by: Claude:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marius Gehrmann [Chris Kearney: rebased onto integration/aurora 1bf7fffcd67b; Claude-Session trailer dropped] [ryan: adapted to 7.2.10 — this tree returns -ENOSPC, not -EBUSY, when no pipeline can take a DP-alt display: dcp_typec_port_busy() logs once per plug and keeps returning -ENOSPC] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit a372a728eaf9cc8bedf97a42c97d9afc15f27797) --- drivers/gpu/drm/apple/dcp-fabric.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 531b69920800a..5e1620d5f4251 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -70,6 +70,8 @@ struct apple_dcp_typec_port { /* Direct DP-alt: the port is in DP mode, routed or not, and its HPD. */ bool dp_wanted; bool dp_hpd; + /* the wait for a pipeline was logged since the port entered DP mode */ + bool busy_noted; /* dcp_typec_rebalance_locked()'s plan for the port's two streams */ struct apple_dcp_typec_route *target; struct apple_dcp_typec_route *secondary_target; @@ -1344,6 +1346,27 @@ static void dcp_typec_route_commit(struct typec_mux_dev *mux, dcp_typec_pipeline_freed(); } +/* + * A DP display on @port gets no pipeline: every one that could drive it + * drives another display. Say so once per plug, in words a user can act on; + * the port controller only logs the -EBUSY. + */ +static int dcp_typec_port_busy(struct apple_dcp_typec_port *port, + struct apple_dcp *dcp) +{ + const char *label = NULL; + + /* This tree reports a port without a pipeline as -ENOSPC. */ + if (port->busy_noted) + return -ENOSPC; + port->busy_noted = true; + of_property_read_string(port->connector_np, "label", &label); + dev_info(dcp->dev, + "display on %s waits for an external display pipeline: it drives another display; to switch, unplug that one, then plug this one in again\n", + label ?: "a USB-C port"); + return -ENOSPC; +} + static int dcp_typec_route_set(struct typec_mux_dev *mux, struct typec_mux_state *state) { @@ -1409,6 +1432,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (!is_dp) { port->dp_wanted = false; port->dp_hpd = false; + port->busy_noted = false; /* a Thunderbolt/USB4 DP tunnel is torn down by its own path */ if (port->owner && port->owner->tunnel) { @@ -1468,7 +1492,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, dcp_typec_rebalance_locked(NULL, 0); if (!port->owner) { if (hpd) - return -ENOSPC; + return dcp_typec_port_busy(port, route->dcp); port->applied_valid = true; return 0; } @@ -1483,7 +1507,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, best = dcp_typec_free_route(port); /* Every pipeline is driving another port's display. */ if (!best) - return -ENOSPC; + return dcp_typec_port_busy(port, route->dcp); ret = dcp_typec_route_activate(best, best->xbar); if (ret) From 537a84c5fb14cf43760742639f463f495c8d90c7 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 14 Oct 2026 23:51:31 -0700 Subject: [PATCH 1674/2680] drm: apple: T8122 dcpext: wait for all route suppliers, quiet a dead pipe Three fixes on the T8122 dcpext paths only: - Probe looks up each route's DP PHY and display crossbar while it registers the routes, one at a time. If the second route's supplier was not there yet, the first route was already registered and could not be taken back, so the dcpext was given up for the boot instead of deferring. Probe now checks that every route's suppliers are there before it registers any, and defers until they are (bounded by the deferred probe timeout, as before). - A dcpext given up after its firmware was started can still report a crash. dcp_rtk_crashed() then completed start_done, which is set up only in dcp_start(), which a given-up pipe never reaches. It now marks the pipe crashed and returns. - The polled AFK start wait reported a timeout when the endpoint acknowledged after the last 10 ms wait but before the deadline check. It takes that acknowledgement. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 (cherry picked from commit c05c92b38d1108bc423628d771686b04b23b2019) --- drivers/gpu/drm/apple/afk.c | 3 +++ drivers/gpu/drm/apple/dcp.c | 52 +++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 89eef9a80b292..b97ec275e25ea 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -197,6 +197,9 @@ static int afk_start_polled(struct apple_dcp_afkep *ep) return 0; } while (time_before(jiffies, timeout)); + /* Completed after the last wait, before the deadline check. */ + if (try_wait_for_completion(&ep->started)) + return 0; dev_err(ep->dcp->dev, "RTKit AFK endpoint %02x did not acknowledge init\n", ep->endpoint); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index f6f6e177fe977..4ef7fb1ac9cd1 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -762,6 +762,15 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ if (READ_ONCE(dcp->quiescing)) return; + /* + * A failed T8122 dcpext: its start never ran (start_done is not set + * up) and it has no connector. Its firmware may still report a crash. + */ + if (READ_ONCE(dcp->ext_dead)) { + dcp->crashed = true; + dev_err(dcp->dev, "unusable external display processor has crashed\n"); + return; + } dcp->crashed = true; dev_err(dcp->dev, "DCP has crashed\n"); if (dcp->fw_compat == DCP_FIRMWARE_H17P && @@ -4159,6 +4168,44 @@ static void dcp_ext_give_up(struct apple_dcp *dcp, int err, const char *when) dcp_typec_routes_disable(dcp); } +/* + * Whether the DP PHY and display crossbar of every Type-C route are there. + * Routes register one at a time, and one registered cannot be taken back, + * since a port controller may hold it: a later route waiting for its + * supplier would leave the dcpext unusable for the boot. Returns + * -EPROBE_DEFER until all of them are; other errors are left to the route + * probe, which reports them. + */ +static int dcp_ext_route_suppliers(struct device *dev) +{ + struct device_node *routes __free(device_node) = + of_get_child_by_name(dev->of_node, "typec-routes"); + char name[16]; + u32 index; + + if (!routes) + return 0; + for_each_available_child_of_node_scoped(routes, route_np) { + struct mux_control *xbar; + struct phy *phy; + + if (of_property_read_u32(route_np, "reg", &index)) + continue; + snprintf(name, sizeof(name), "typec%u", index); + phy = of_phy_get(dev->of_node, name); + if (PTR_ERR_OR_ZERO(phy) == -EPROBE_DEFER) + return -EPROBE_DEFER; + if (!IS_ERR(phy)) + of_phy_put(phy); + xbar = mux_control_get(dev, name); + if (PTR_ERR_OR_ZERO(xbar) == -EPROBE_DEFER) + return -EPROBE_DEFER; + if (!IS_ERR(xbar)) + mux_control_put(xbar); + } + return 0; +} + /* * The T8122 dcpext is a component of the panel's DRM device, so it must * always register one, or the panel stays on the boot framebuffer. Keep @@ -4175,6 +4222,10 @@ static int dcp_platform_probe(struct platform_device *pdev) if (!of_device_is_compatible(dev->of_node, "apple,t8122-dcpext")) return dcp_platform_probe_pipe(pdev); + ret = dcp_ext_route_suppliers(dev); + if (ret) + goto unusable; + group = devres_open_group(dev, NULL, GFP_KERNEL); if (!group) return -ENOMEM; @@ -4194,6 +4245,7 @@ static int dcp_platform_probe(struct platform_device *pdev) devres_release_group(dev, group); platform_set_drvdata(pdev, NULL); +unusable: if (ret == -EPROBE_DEFER) { ret = driver_deferred_probe_check_state(dev); if (ret == -EPROBE_DEFER) From 02b3808037aaadefcdb0b45649c54765d95f8b64 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 14 Oct 2026 23:51:31 -0700 Subject: [PATCH 1675/2680] drm: apple: say a waiting Type-C display lights up on its own MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The log line for a DP display that gets no pipeline told the user to unplug the other display and then plug this one in again. No replug is needed: when a display goes, dcp_typec_pipeline_freed() hands the freed pipeline to a waiting display, and the J613 tester saw the waiting display come up a few seconds after the unplug, in both directions. - The line now says the display lights up on its own when one of the other displays is unplugged. - On the path that takes the lowest free pipeline (the J613, M1 and M2), it is logged only with the display's HPD up, not for an adapter with no display. - A port that got a pipeline reports a later wait again. Only the log changes; the route still returns -EBUSY as before. [ryan: adapted to 7.2.10 — this tree's -ENOSPC is kept for a port without a pipeline, also when its HPD is down] Reported-by: Marius Gehrmann Link: https://github.com/omacom/linux-aurora/issues/103 Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 (cherry picked from commit d5224e9f10ba11d6b0cb99e7c9d9cea486849510) --- drivers/gpu/drm/apple/dcp-fabric.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 5e1620d5f4251..7b168f9cb7d07 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -1348,8 +1348,10 @@ static void dcp_typec_route_commit(struct typec_mux_dev *mux, /* * A DP display on @port gets no pipeline: every one that could drive it - * drives another display. Say so once per plug, in words a user can act on; - * the port controller only logs the -EBUSY. + * drives another display. Say so once per plug, with the display's HPD up; + * the port controller only logs the -ENOSPC. When one of those displays goes, + * the freed pipeline takes a waiting display without a replug (see + * dcp_typec_pipeline_freed()). */ static int dcp_typec_port_busy(struct apple_dcp_typec_port *port, struct apple_dcp *dcp) @@ -1362,7 +1364,7 @@ static int dcp_typec_port_busy(struct apple_dcp_typec_port *port, port->busy_noted = true; of_property_read_string(port->connector_np, "label", &label); dev_info(dcp->dev, - "display on %s waits for an external display pipeline: it drives another display; to switch, unplug that one, then plug this one in again\n", + "display on %s waits for a free external display pipeline (each one drives another display); it lights up on its own when one of those displays is unplugged\n", label ?: "a USB-C port"); return -ENOSPC; } @@ -1507,13 +1509,14 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, best = dcp_typec_free_route(port); /* Every pipeline is driving another port's display. */ if (!best) - return dcp_typec_port_busy(port, route->dcp); + return hpd ? dcp_typec_port_busy(port, route->dcp) : -ENOSPC; ret = dcp_typec_route_activate(best, best->xbar); if (ret) return ret; port->owner = best; - + /* a later wait of this plug is reported again */ + port->busy_noted = false; } if (hpd && !port->hpd) { From d6fdbd0a34f461ceb592e4f416e2731a28539f93 Mon Sep 17 00:00:00 2001 From: Marcelo Alcantara Date: Mon, 12 Oct 2026 20:17:36 +1000 Subject: [PATCH 1676/2680] drm: apple: hold the fabric up while an external display scans out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit External display domains stay powered even when idle, so the pmgr-misc idle fabric governor cannot tell from them whether an external display is active. Take a fabric hold before an external CRTC powers up and release it when the CRTC turns off, so scanout never starts or runs on a lowered fabric. The internal display was qualified at the low fabric p-state and does not hold it. [ryan: adapted to 7.2.10 — linux/soc/apple/pmgr-misc.h and its provider (apple-pmgr-misc.c) are in 010-soc, not in this tree: dcp.h declares apple_pmgr_fabric_hold()/apple_pmgr_fabric_release() and apple_drv.c calls them through symbol_get(), behaving as the !CONFIG_APPLE_PMGR_MISC stubs (hold returns 0, release does nothing) without a provider; the hold/release sits in this tree's drm_atomic_commit CRTC callbacks] Signed-off-by: Marcelo Alcantara (cherry picked from commit 4622789751a3a09d7194f268e973689b395b2870) --- drivers/gpu/drm/apple/apple_drv.c | 34 +++++++++++++++++++++++++++++++ drivers/gpu/drm/apple/dcp.h | 14 +++++++++++++ 2 files changed, 48 insertions(+) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index c5db84d31e5a8..1c23338034312 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -146,6 +146,30 @@ static void apple_connector_oob_hotplug(struct drm_connector *connector, ":0x%x in oob_hotplug event\n", (u32)status); } +/* The fabric hold is optional at link time, see dcp.h. */ +static int apple_drm_fabric_hold(void) +{ + typeof(&apple_pmgr_fabric_hold) hold = symbol_get(apple_pmgr_fabric_hold); + int ret; + + if (!hold) + return 0; + ret = hold(); + symbol_put(apple_pmgr_fabric_hold); + return ret; +} + +static void apple_drm_fabric_release(void) +{ + typeof(&apple_pmgr_fabric_release) release = + symbol_get(apple_pmgr_fabric_release); + + if (!release) + return; + release(); + symbol_put(apple_pmgr_fabric_release); +} + static void apple_crtc_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_commit *commit) { @@ -171,6 +195,12 @@ static void apple_crtc_atomic_enable(struct drm_crtc *crtc, } if (crtc_state->active && !dcp_crtc_route_ready(crtc, commit, true)) goto failed; + if (crtc_state->active && !dcp->main_display && !apple_crtc->fabric_held) { + /* Raise the fabric before external scanout starts. */ + if (apple_drm_fabric_hold()) + dev_warn(dcp->dev, "fabric raise not confirmed before scanout\n"); + apple_crtc->fabric_held = true; + } if (crtc_state->active && (crtc_state->active_changed || dcp_crtc_needs_route_start(dcp))) { dcp_poweron(apple_crtc->dcp); @@ -206,6 +236,10 @@ static void apple_crtc_atomic_disable(struct drm_crtc *crtc, if (crtc_state->active_changed && !crtc_state->active) { dcp_poweroff(apple_crtc->dcp); + if (apple_crtc->fabric_held) { + apple_crtc->fabric_held = false; + apple_drm_fabric_release(); + } } if (crtc->state->event && !crtc->state->active) { diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 8f302a54beb4d..bb87662893c19 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -30,10 +30,24 @@ struct apple_crtc { /* Reference to the DCP device owning this CRTC */ struct platform_device *dcp; + + /* External scanout keeps the fabric up while the CRTC is active. */ + bool fabric_held; }; #define to_apple_crtc(x) container_of(x, struct apple_crtc, base) +/* + * The pmgr-misc fabric hold (linux/soc/apple/pmgr-misc.h, 010-soc). Declared + * here until that header is in this tree; apple_drv.c resolves both with + * symbol_get() and behaves as their !CONFIG_APPLE_PMGR_MISC stubs without a + * provider. + */ +#ifndef __LINUX_SOC_APPLE_PMGR_MISC_H +int apple_pmgr_fabric_hold(void); +void apple_pmgr_fabric_release(void); +#endif + struct apple_encoder { struct drm_encoder base; }; From 5f93510036abf460c3fba49ac918815393effd60 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 11 Oct 2026 16:23:33 +0200 Subject: [PATCH 1677/2680] drm: apple: relink a Type-C sink that lost its link to a hub's USB3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A USB-C monitor with a built-in USB hub (EIZO EV2780) intermittently stays dark after a cold boot or a hot-plug on the M1 MacBook Air and the M1 Mac mini. DCP trains the link for the first modeset at about the time the monitor's hub brings its SuperSpeed link up. When the two overlap, the sink loses the link, but DCP still reports the modeset as linked: UnifiedPipeline::mode_set_gated(): 2560x1440@59 Hz link: 1 Then one of two things happens. DCP gives up on its own 2.008 s after asserting HPD: IOMFB: IOAVVideoInterface terminated VideoInterfaceIOAV::unplug_gated(): display HPD removed or it keeps driving the dead link without noticing, while the monitor pulses HPD about 2-4 s later to be detected again. In both cases the CD321x still reports DP alt mode with HPD high, and nothing re-detects the sink. With dwc3 kept off the port, 5 out of 5 boots brought the display up; with it, 3 out of 6 failed. Replay an unplug and replug of the sink when DCP drops a sink that still asserts HPD, or when the sink asks for attention (IRQ_HPD): the link is then retrained after the hub has settled, as a cable replug does. A request from the sink within 4 s after a relink is the sink's late reaction to the loss the relink already handled and is ignored; another relink would only interrupt the new link. A sink request must not be dropped as an unchanged route state either. A connection gets at most three relinks; the replay does not reset that count. Pulsing only DCP's HPD instead, with the connector kept connected, raced with DCP re-detecting the sink on its own and left the display dark on the Mac mini. H17P and native external pipes keep their own recovery. Once the relink budget is spent, a sink request falls back to dcp_dptx_recover_irq() or dcp_retrain_oob() as before. With this, 8 out of 8 cold boots of the MacBook Air and 8 out of 8 of the Mac mini (EIZO on the port that failed) brought the display up. [ryan: adapted to 7.2.10 — the relink work joins this tree's typec work disable/enable and probe setup by hand (the surrounding probe and teardown code differ); dcp_typec_relink_start() keeps every H17P pipe out] Signed-off-by: Thomas Glanzmann (cherry picked from commit b609a9e03cbe63fb6d2c00a4c11c6a4e046887d5) --- drivers/gpu/drm/apple/dcp-fabric.c | 19 +-- drivers/gpu/drm/apple/dcp-internal.h | 8 ++ drivers/gpu/drm/apple/dcp.c | 108 ++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 6 + .../selftests/drivers/gpu/apple-dcp/rate.py | 1 + .../drivers/gpu/apple-dcp/transaction.py | 1 + 6 files changed, 135 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 7b168f9cb7d07..fc2ee7e5e30ea 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -1404,7 +1404,8 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, * without a word, so a nominated route might never be called at all -- * and the port would then never be routed. */ - if (port->applied_valid && port->applied_alt == state->alt && + if (port->applied_valid && !(dp_status & DP_STATUS_IRQ_HPD) && + port->applied_alt == state->alt && port->applied_mode == state->mode && port->applied_status == dp_status && port->applied_conf == dp_conf) return 0; @@ -1537,13 +1538,15 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, dcp_retrain_oob(dcp->typec_connector); else if (dcp->typec_connector && dcp->external_native) dcp_external_sink_irq(dcp); - else if (dcp->typec_connector && - !READ_ONCE(dcp->typec_connector->connected)) { - ret = dcp_dptx_recover_irq(dcp); - if (ret) - return ret; - } else if (dcp->typec_connector) - dcp_retrain_oob(dcp->typec_connector); + else if (dcp->typec_connector && !dcp_typec_sink_relink(dcp)) { + if (!READ_ONCE(dcp->typec_connector->connected)) { + ret = dcp_dptx_recover_irq(dcp); + if (ret) + return ret; + } else { + dcp_retrain_oob(dcp->typec_connector); + } + } } port->hpd = hpd; port->applied_valid = true; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 4b73e2dfb2cae..026f8070b1a6d 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -65,6 +65,8 @@ void dcp_swap_watchdog_arm(struct apple_dcp *dcp); void dcp_swap_watchdog_complete(struct apple_dcp *dcp); void dcp_retry_placeholder_edid(struct apple_dcp *dcp, const struct drm_edid *drm_edid); +bool dcp_typec_relink(struct apple_dcp *dcp); +bool dcp_typec_sink_relink(struct apple_dcp *dcp); struct dcpav_service_epic; @@ -393,6 +395,12 @@ struct apple_dcp { u64 crtc_unplugs; u64 placeholder_generation; struct delayed_work placeholder_edid_wq; + /* replugs replayed for a Type-C sink that lost its link */ + u32 relink_tries; + bool relink_active; + unsigned long relink_done; + u64 relink_generation; + struct delayed_work typec_relink_wq; bool use_timestamps; bool vrr_enabled; struct dcp_set_digital_out_mode_req mode; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 4ef7fb1ac9cd1..788156f9d2dcf 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1841,6 +1841,110 @@ bool dcp_hold_resume_event(struct platform_device *pdev) return true; } +#define DPTX_RELINK_RETRIES 3 +#define DPTX_RELINK_UNPLUG_MS 1000 +/* a monitor pulses HPD up to about 4 s after it lost the link */ +#define DPTX_RELINK_SETTLE_MS 4000 + +/* + * A USB-C monitor with a built-in hub can lose the DP link when the hub + * brings its SuperSpeed link up right around DCP's link training for the + * first modeset. DCP may then report the sink as unplugged two seconds + * later although it still asserts HPD, or keep driving the dead link + * without noticing while the monitor pulses HPD to be detected again. + * Neither is recovered by anything else: replay an unplug and replug of + * the sink, which retrains the link once the hub has settled, as a cable + * replug does. + * + * The retries cover one connection: the replay itself does not reset them. + */ +static bool dcp_typec_relink_start(struct apple_dcp *dcp, const char *why) +{ + u64 generation = READ_ONCE(dcp->typec_generation); + + /* H17P and native external pipes recover through their own lifecycle */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->external_native || + !dcp_is_typec_output(dcp) || dcp_is_usb4_output(dcp) || + dcp_is_external_h17p(dcp) || + !READ_ONCE(dcp->typec_cable_connected) || + READ_ONCE(dcp->typec_crtc_off) || !dcp->typec_connector) + return false; + + if (dcp->relink_generation != generation) { + dcp->relink_generation = generation; + dcp->relink_tries = 0; + WRITE_ONCE(dcp->relink_active, false); + } + /* the unplug a running relink replays makes DCP report one too */ + if (READ_ONCE(dcp->relink_active)) + return true; + if (dcp->relink_tries >= DPTX_RELINK_RETRIES) + return false; + + dcp->relink_tries++; + WRITE_ONCE(dcp->relink_active, true); + dev_info(dcp->dev, "%s, relinking the Type-C sink (%u/%u)\n", why, + dcp->relink_tries, DPTX_RELINK_RETRIES); + mod_delayed_work(system_freezable_wq, &dcp->typec_relink_wq, 0); + + return true; +} + +/* DCP reported an unplug of a Type-C sink that still asserts HPD */ +bool dcp_typec_relink(struct apple_dcp *dcp) +{ + return dcp_typec_relink_start(dcp, "DCP dropped a sink that asserts HPD"); +} + +/* + * The sink asked for attention: IRQ_HPD or a coalesced HPD pulse. Right + * after a relink that is the sink's late reaction to the loss the relink + * already handled; another relink would only interrupt the new link. + */ +bool dcp_typec_sink_relink(struct apple_dcp *dcp) +{ + unsigned long done = READ_ONCE(dcp->relink_done); + + if (done && time_before(jiffies, done + msecs_to_jiffies(DPTX_RELINK_SETTLE_MS)) && + READ_ONCE(dcp->relink_generation) == READ_ONCE(dcp->typec_generation)) { + dev_info(dcp->dev, "Type-C sink requested attention right after a relink, ignored\n"); + return true; + } + + return dcp_typec_relink_start(dcp, "Type-C sink requested attention"); +} + +static void dcp_typec_relink_work(struct work_struct *work) +{ + struct apple_dcp *dcp = + container_of(to_delayed_work(work), struct apple_dcp, + typec_relink_wq); + struct platform_device *pdev = to_platform_device(dcp->dev); + u64 generation; + + scoped_guard(mutex, &dcp->hpd_mutex) + generation = dcp->typec_generation; + if (generation != dcp->relink_generation || + !READ_ONCE(dcp->typec_cable_connected)) + goto out; + + dcp_dptx_disconnect_oob(pdev, 0); + msleep(DPTX_RELINK_UNPLUG_MS); + + /* a real unplug or replug meanwhile takes over */ + scoped_guard(mutex, &dcp->hpd_mutex) { + if (dcp->typec_generation != generation + 1) + goto out; + } + dcp_dptx_connect_oob(pdev, 0); + + scoped_guard(mutex, &dcp->hpd_mutex) + dcp->relink_generation = dcp->typec_generation; + WRITE_ONCE(dcp->relink_done, jiffies); +out: + WRITE_ONCE(dcp->relink_active, false); +} + static void disconnected_hpd_event(struct apple_connector *con) { struct platform_device *pdev; @@ -3190,6 +3294,7 @@ VISIBLE_IF_KUNIT void dcp_disable_typec_work(struct apple_dcp *dcp, bool release /* Block new enqueues as well as draining users of the AFK endpoints. */ disable_delayed_work_sync(&dcp->typec_reconnect_wq); disable_delayed_work_sync(&dcp->placeholder_edid_wq); + disable_delayed_work_sync(&dcp->typec_relink_wq); disable_delayed_work_sync(&dcp->announce_wq); cancel_work_sync(&dcp->external_stop_work); cancel_delayed_work_sync(&dcp->external_stop_retry_wq); @@ -3202,6 +3307,7 @@ VISIBLE_IF_KUNIT void dcp_enable_typec_work(struct apple_dcp *dcp) { enable_delayed_work(&dcp->typec_reconnect_wq); enable_delayed_work(&dcp->placeholder_edid_wq); + enable_delayed_work(&dcp->typec_relink_wq); enable_delayed_work(&dcp->announce_wq); enable_delayed_work(&dcp->hdmi_settle_wq); enable_delayed_work(&dcp->hdmi_recheck_wq); @@ -3887,6 +3993,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) dcp_typec_reconnect_work); INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, dcp_placeholder_edid_work); + INIT_DELAYED_WORK(&dcp->typec_relink_wq, dcp_typec_relink_work); INIT_DELAYED_WORK(&dcp->announce_wq, dcp_announce_work); INIT_DELAYED_WORK(&dcp->bl_fallback_wq, dcp_work_backlight_fallback); /* System sleep and shutdown cancel these even if no bind succeeded. */ @@ -3898,6 +4005,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) /* Balanced by enable at successful component bind. */ disable_delayed_work(&dcp->typec_reconnect_wq); disable_delayed_work(&dcp->placeholder_edid_wq); + disable_delayed_work(&dcp->typec_relink_wq); disable_delayed_work(&dcp->announce_wq); dcp->announce.stopped = true; /* diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 78f6eec0652a6..ce10656765af2 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1980,6 +1980,12 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) schedule_work(&dcp->vblank_wq); return; } + /* DCP lost a sink which is still attached, see dcp_typec_relink() */ + if (!(*connected) && dcp_typec_relink(dcp)) { + dcp_mode_invalidate(&dcp->mode_state); + schedule_work(&dcp->vblank_wq); + return; + } if (dcp_is_typec_output(dcp) && *connected && dcp->nr_modes) complete_all(&dcp->typec_iomfb_hpd_ready); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py index f2cfdd6886870..a2a6bf320047c 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py @@ -955,6 +955,7 @@ def hotplug(root): #define dcp_mode_invalidate(s) ((s)->valid=false) #define dcp_mode_hotplug(s,c,p) __extension__ ({ (void)(s);if(p)*(p)=(c);1u; }) static void dcp_handle_hotplug_actions(struct apple_dcp *d,unsigned action) { (void)d;(void)action; } +static bool dcp_typec_relink(struct apple_dcp *d) { (void)d;return false; } ''' dcp = (root / 'drivers/gpu/drm/apple/dcp.c').read_text() template = (root / 'drivers/gpu/drm/apple/iomfb_template.c').read_text() diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py index 767e0a1b20ff3..0b19ebdc4c14b 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py @@ -73,6 +73,7 @@ def check(name, unit, cases=((),), negative=False, flags=()): static int connect_calls; static int dcp_dptx_connect_oob(void *dev,unsigned n) { (void)dev;(void)n;connect_calls++;display.external_cable=true;display.dptxport[0].connected=true;display.typec_cable_connected=true;connector.connected=true;return 0; } static void dcp_retrain_oob(struct connector *c) { (void)c; } + static bool dcp_typec_sink_relink(struct apple_dcp *d) { (void)d;return false; } ''' pos=actual.index('static int dcp_typec_route_deactivate(') actual=actual[:pos]+extra+actual[pos:] From 6eaedd8da2ad35bffa86146a73523b1592b74fd2 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 12 Oct 2026 13:04:11 +0200 Subject: [PATCH 1678/2680] drm: apple: do not report a bad link for the HPD a Type-C wake awaits Waking a DP alt mode display from DPMS off with sway ("output * power on") on the M1 Mac mini (Fujitsu E24T-7 Pro on USB-C, which keeps HPD asserted while blanked) power cycles the display about every 750 ms until reboot: dcp_hotplug() connected:1 valid_mode:0 nr_modes:1 dcp_poweron() starting set_digital_out_mode finished dcp_poweroff() done ... With the cable attached, dcp_poweron() reconnects DPTX and waits for IOMFB to report the sink before the commit does its modeset. At that point no mode is valid yet, so dcpep_cb_hotplug() takes the HPD for a sink that asserted HPD only after a failed modeset and sends DCP_HOTPLUG_NOTIFY. dcp_hotplug() then sets the connector's link-status to BAD. wlroots treats a bad link as a disconnect and recreates the output, which disables the CRTC and enables it again. That wake does the same, so the loop never ends. Each round also leaks AFK service slots ("AFK[ep:29]: no reusable service slots") and the system becomes sluggish. Do not take the HPD that dcp_poweron() waits for as a late one: the modeset of the same commit follows. A sink that asserts HPD only after dcp_poweron() stopped waiting is still handled as before. Signed-off-by: Thomas Glanzmann (cherry picked from commit 1739b38bc583dee273a7d9331fa818f297563b3b) --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/dcp.c | 3 +++ drivers/gpu/drm/apple/iomfb_template.c | 9 ++++++++- tools/testing/selftests/drivers/gpu/apple-dcp/rate.py | 2 +- 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 026f8070b1a6d..9330f6cb95e10 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -544,6 +544,8 @@ struct apple_dcp { bool typec_follow_start; bool typec_follow_retiring; u64 typec_follow_gen; + /* dcp_poweron() reconnects DPTX; its own modeset follows the HPD */ + bool typec_poweron_hpd; /* IOMFB reports its video interface ready after DPTX link training. */ struct completion typec_iomfb_hpd_ready; struct delayed_work typec_reconnect_wq; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 788156f9d2dcf..e14052f0d07cb 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2547,6 +2547,8 @@ void dcp_poweron(struct platform_device *pdev) if (READ_ONCE(dcp->typec_cable_connected)) { cancel_delayed_work(&dcp->typec_reconnect_wq); dcp->typec_reconnect_tries = 0; + /* the modeset of this commit follows, see dcpep_cb_hotplug() */ + WRITE_ONCE(dcp->typec_poweron_hpd, true); ret = dcp_dptx_connect(dcp, 0); if (ret) mod_delayed_work(system_freezable_wq, @@ -2560,6 +2562,7 @@ void dcp_poweron(struct platform_device *pdev) dev_warn(dcp->dev, "Type-C IOMFB hotplug not ready on wake\n"); } + WRITE_ONCE(dcp->typec_poweron_hpd, false); } } else if (dcp->hdmi_hpd) { bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index ce10656765af2..086b4f1b79c62 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1950,6 +1950,8 @@ void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp) static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) { struct apple_connector *connector = dcp->connector; + /* sampled before complete_all() lets dcp_poweron() clear it */ + bool poweron_hpd = READ_ONCE(dcp->typec_poweron_hpd); unsigned int action; /* DCP issues hotplug_gated callbacks after SetPowerState() calls on @@ -2007,8 +2009,13 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) * A Type-C sink can assert HPD only after a modeset has already failed, * as a TV behind a DP-to-HDMI converter does when it wakes from standby. * The connector state does not change then; re-apply the mode anyway. + * + * Not for the HPD dcp_poweron() waits for: the modeset of that commit + * has not started yet, rather than failed. Reporting the link as bad + * then makes wlroots replace the output, which powers the CRTC off and + * on again, and the next wake does the same, forever. */ - if (*connected && dcp_is_typec_output(dcp) && + if (*connected && dcp_is_typec_output(dcp) && !poweron_hpd && !READ_ONCE(dcp->mode_state.valid) && !READ_ONCE(dcp->mode_state.changing)) { dcp->swap_watchdog_retrains = 0; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py index a2a6bf320047c..bda5b2bbda60a 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py @@ -944,7 +944,7 @@ def hotplug(root): struct { bool valid,changing; } mode_state; struct work_struct external_stop_work,vblank_wq; bool external_physically_absent,main_display,typec_crtc_off,external_cable,external_error,external_power_attempted,external_stopped; - bool external_hotplug_pending,external_hotplug_connected; + bool external_hotplug_pending,external_hotplug_connected,typec_poweron_hpd; u64 external_stop_generation,external_rx_generation,external_crtc_generation; unsigned swap_watchdog_retrains;int typec_iomfb_hpd_ready;''') stubs = r''' From 6e2f8b89e68a276cd8acf227fe5ffc08da43d800 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 12 Oct 2026 14:01:54 +0200 Subject: [PATCH 1679/2680] drm: apple: audio: play into the void without a sink On the M1 Mac mini with two Fujitsu E24T-7 Pro monitors, which have no audio (single-block EDID 1.3), PipeWire logs spa.alsa: hw:0p: snd_pcm_start: No such device or address about 130000 times a second, for hours, as soon as something plays to a DCP audio output: 4.1 million log lines in one boot, with journald dropping most of them, and a core kept busy. Without a sink, opening, configuring and preparing the PCM succeed, as userspace drops an output that fails its probe for good, but START fails with -ENXIO. That does not stop userspace from trying: PipeWire starts playback after its first write and ignores the error, and a stream that never starts never makes it wait, so the next wakeup is immediate and tries again. Any other error ends up in its recovery, which prepares and starts again. Play into the void instead, as HDMI PCMs on other platforms do: without a sink, or once the sink the stream was opened for is gone, START succeeds and the stream runs on the clock, without DMA and without a DCP audio link. The position follows the time since START and a work item reports the periods. The samples are dropped. A stream playing into the void is not stopped as disconnected when a sink goes away; a sink attached later is used by the next stream opened. Signed-off-by: Thomas Glanzmann (cherry picked from commit cf7ddf4de16c678739540f02c62071a221cfe271) --- drivers/gpu/drm/apple/audio.c | 113 ++++++++++++++++++++++++++++++---- 1 file changed, 100 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index cc64168e1d8b3..16abff44ffcbc 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -52,6 +52,11 @@ struct dcp_audio { struct snd_pcm_chmap_elem selected_chmap; struct dcp_sound_cookie selected_cookie; + /* stream runs without a sink, see dcpaud_void_start() */ + bool void_running; + ktime_t void_start; + u64 void_periods; + struct delayed_work void_work; void *elements; void *productattrs; @@ -81,7 +86,7 @@ static const struct snd_pcm_hardware dcp_pcm_hw = { * configured, as HDMI PCMs on other platforms can. Userspace (PipeWire, * PulseAudio) probes the PCM once when the card appears and drops the * output for good if that fails, so a monitor attached later would never - * show up. Starting the stream still fails until a sink is connected. + * show up. The stream then plays into the void, see dcpaud_void_start(). */ static const struct snd_pcm_hardware dcp_pcm_hw_unplugged = { .info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | @@ -352,12 +357,10 @@ static int dcp_pcm_hw_params(struct snd_pcm_substream *substream, struct dma_chan *chan = snd_dmaengine_pcm_get_chan(substream); int ret; - if (dcpaud->open_unplugged) + /* no sink: the stream runs without DMA, see dcpaud_void_start() */ + if (dcpaud->open_unplugged || !dcpaud_connection_up(dcpaud)) return 0; - if (!dcpaud_connection_up(dcpaud)) - return -ENXIO; - ret = dcpaud_select_cookie(dcpaud, params); if (ret < 0) return ret; @@ -397,16 +400,79 @@ static int dcp_pcm_prepare(struct snd_pcm_substream *substream) struct dcp_audio *dcpaud = substream->pcm->private_data; /* alsa-lib prepares right after hw_params; see dcp_pcm_hw_unplugged */ - if (dcpaud->open_unplugged) + if (dcpaud->open_unplugged || !dcpaud_connection_up(dcpaud)) return 0; - if (!dcpaud_connection_up(dcpaud)) - return -ENXIO; - return dcp_audiosrv_prepare(dcpaud->dcp_dev, &dcpaud->selected_cookie); } +static u64 dcpaud_void_period_ns(struct snd_pcm_runtime *runtime) +{ + return div_u64((u64)runtime->period_size * NSEC_PER_SEC, runtime->rate); +} + +static void dcpaud_void_work(struct work_struct *work) +{ + struct dcp_audio *dcpaud = container_of(to_delayed_work(work), + struct dcp_audio, void_work); + struct snd_pcm_substream *substream = dcpaud->substream; + ktime_t next; + s64 delay; + + snd_pcm_period_elapsed(substream); + + snd_pcm_stream_lock_irq(substream); + if (dcpaud->void_running) { + /* the next period boundary, so that periods do not drift */ + next = ktime_add_ns(dcpaud->void_start, + ++dcpaud->void_periods * + dcpaud_void_period_ns(substream->runtime)); + delay = ktime_to_ns(ktime_sub(next, ktime_get())); + schedule_delayed_work(&dcpaud->void_work, + delay > 0 ? nsecs_to_jiffies(delay) : 0); + } + snd_pcm_stream_unlock_irq(substream); +} + +/* + * Without a sink, DCP has no audio link to start. Failing START does not + * stop userspace from trying: PipeWire retries it on every wakeup, and a + * stream that never starts never makes it wait, so it retries at about + * 130000 times a second for as long as something plays (a monitor + * without audio, or none attached). Play into the void instead, as HDMI + * PCMs elsewhere do: the stream runs on the clock, without DMA, and its + * samples are dropped. A sink attached later gets the next stream opened. + */ +static int dcpaud_void_start(struct snd_pcm_substream *substream) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + u64 period_ns = dcpaud_void_period_ns(substream->runtime); + + dev_dbg(dcpaud->dev, "no sink, playing into the void\n"); + dcpaud->void_running = true; + dcpaud->void_start = ktime_get(); + dcpaud->void_periods = 0; + schedule_delayed_work(&dcpaud->void_work, nsecs_to_jiffies(period_ns)); + + return 0; +} + +static snd_pcm_uframes_t dcp_pcm_pointer(struct snd_pcm_substream *substream) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + struct snd_pcm_runtime *runtime = substream->runtime; + u64 frames; + + if (!dcpaud->void_running) + return snd_dmaengine_pcm_pointer(substream); + + frames = mul_u64_u32_div(ktime_to_ns(ktime_sub(ktime_get(), + dcpaud->void_start)), + runtime->rate, NSEC_PER_SEC); + return do_div(frames, runtime->buffer_size); +} + static int dcp_pcm_trigger(struct snd_pcm_substream *substream, int cmd) { struct dcp_audio *dcpaud = substream->pcm->private_data; @@ -415,8 +481,8 @@ static int dcp_pcm_trigger(struct snd_pcm_substream *substream, int cmd) switch (cmd) { case SNDRV_PCM_TRIGGER_START: case SNDRV_PCM_TRIGGER_RESUME: - if (!dcpaud_connection_up(dcpaud)) - return -ENXIO; + if (dcpaud->open_unplugged || !dcpaud_connection_up(dcpaud)) + return dcpaud_void_start(substream); WARN_ON(pm_runtime_get_sync(dcpaud->dev) < 0); ret = dcp_audiosrv_startlink(dcpaud->dcp_dev, @@ -427,6 +493,11 @@ static int dcp_pcm_trigger(struct snd_pcm_substream *substream, int cmd) case SNDRV_PCM_TRIGGER_STOP: case SNDRV_PCM_TRIGGER_SUSPEND: + if (dcpaud->void_running) { + /* sync_stop waits for the work */ + dcpaud->void_running = false; + return 0; + } break; default: @@ -455,6 +526,19 @@ static int dcp_pcm_trigger(struct snd_pcm_substream *substream, int cmd) return 0; } +/* + * ALSA calls this once a stream has stopped, outside the stream lock, before + * the next hw_params, prepare or hw_free and at close. + */ +static int dcp_pcm_sync_stop(struct snd_pcm_substream *substream) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + + cancel_delayed_work_sync(&dcpaud->void_work); + + return 0; +} + struct snd_pcm_ops dcp_playback_ops = { .open = dcp_pcm_open, .close = dcp_pcm_close, @@ -462,7 +546,8 @@ struct snd_pcm_ops dcp_playback_ops = { .hw_free = dcp_pcm_hw_free, .prepare = dcp_pcm_prepare, .trigger = dcp_pcm_trigger, - .pointer = snd_dmaengine_pcm_pointer, + .sync_stop = dcp_pcm_sync_stop, + .pointer = dcp_pcm_pointer, }; // Transitional workaround: for the chmap control TLV, advertise options @@ -544,7 +629,8 @@ static void dcpaud_report_hotplug(struct dcp_audio *dcpaud, bool connected) if (!connected) { snd_pcm_stream_lock(substream); - if (substream->runtime) + /* a stream without a sink plays on, see dcpaud_void_start() */ + if (substream->runtime && !dcpaud->void_running) snd_pcm_stop(substream, SNDRV_PCM_STATE_DISCONNECTED); snd_pcm_stream_unlock(substream); } @@ -756,6 +842,7 @@ static int dcpaud_probe(struct platform_device *pdev) dcpaud->dev = &pdev->dev; mutex_init(&dcpaud->data_lock); + INIT_DELAYED_WORK(&dcpaud->void_work, dcpaud_void_work); platform_set_drvdata(pdev, dcpaud); return component_add(&pdev->dev, &dcpaud_comp_ops); From bd0ad3c05dd0910ab15208aa914bee2b266441f4 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Wed, 14 Oct 2026 20:02:50 +0200 Subject: [PATCH 1680/2680] drm: apple: reset the Type-C relink budget on every DPMS wake A USB-C monitor with a built-in hub drops the DP link right after the first modeset of every DPMS wake, which dcp_typec_relink() recovers by replaying an unplug and replug. Its retry counter is only reset when the Type-C generation changes, i.e. on a real cable replug, so all wakes of one connection share DPTX_RELINK_RETRIES tries: after three wakes the fourth drop is handled as an unplug and the display stays dark until the cable is replugged. The limit exists to stop relink loops within one link bring-up. Reset it in dcp_poweron() when the CRTC comes back from DPMS off. No relink can run while the CRTC is off, dcp_typec_relink_start() refuses then. Signed-off-by: Thomas Glanzmann (cherry picked from commit 99415c89b3b217d6f95c70fb7cd95e8bb9d2c74a) --- drivers/gpu/drm/apple/dcp.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index e14052f0d07cb..0ba54fe6fb989 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1856,7 +1856,8 @@ bool dcp_hold_resume_event(struct platform_device *pdev) * the sink, which retrains the link once the hub has settled, as a cable * replug does. * - * The retries cover one connection: the replay itself does not reset them. + * The retries cover one connection or one DPMS wake: the replay itself + * does not reset them, dcp_poweron() does. */ static bool dcp_typec_relink_start(struct apple_dcp *dcp, const char *why) { @@ -2539,6 +2540,12 @@ void dcp_poweron(struct platform_device *pdev) READ_ONCE(dcp->typec_cable_connected); WRITE_ONCE(dcp->typec_crtc_off, false); WRITE_ONCE(dcp->typec_follow_start, false); + /* + * A DPMS wake retrains the link like a replug: give it a fresh + * relink budget. No relink runs while the CRTC was off. + */ + if (wait_for_typec_hpd) + dcp->relink_tries = 0; /* * A Type-C CRTC disable releases its DPTX session. Re-establish it From 1672385fb05729cfa89c653ef93adbf093cca8ae Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Wed, 14 Oct 2026 21:41:25 +0200 Subject: [PATCH 1681/2680] drm: apple: do not report a bad link for a powered-off Type-C CRTC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit 57f9f3623299 ("drm/apple: fence EDID publication by service lifetime and unblock RX"), the teardown of DCP's AV service schedules the connector hotplug worker to invalidate the EDID. DCP tears that service down whenever a Type-C CRTC is powered off with the display attached (DPMS off). dcp_hotplug() then finds the connector connected without a valid mode and reports the link as bad. wlroots treats a bad link as a disconnect and replaces the output on the next wake, which powers the CRTC off and on again, and that power off does the same: the display never comes back from the first DPMS off (Mac mini, EIZO EV2780 on USB-C). There is no link while the CRTC is off, and dcp_poweron() retrains one on wake. Leave the link status and the CRTC alone then; the EDID is still invalidated. [ryan: adapted to 7.2.10 — applied to the non-gated branch of this tree's dcp_hotplug(); the announce-gated H17P external branch keeps its own relink handling] Signed-off-by: Thomas Glanzmann (cherry picked from commit ce9084c4138f96e8e68ad15ef623d3f48549321e) --- drivers/gpu/drm/apple/iomfb.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index b1362faeff036..30310006cc70f 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1200,6 +1200,14 @@ void dcp_hotplug(struct work_struct *work) dcp_external_relink(dcp, connector); } else if (!notify_only && connector->base.state && !READ_ONCE(dcp->mode_state.valid) && connected && + /* + * Not for a Type-C CRTC powered off with the display + * attached: it has no link, and dcp_poweron() retrains one + * on wake. A link reported bad then makes wlroots replace + * the output on wake, which powers the CRTC off and on + * again, and that wake does the same, forever. + */ + !READ_ONCE(dcp->typec_crtc_off) && !dcp_uses_t6020_tunnel_flow(dcp)) { drm_connector_set_link_status_property(&connector->base, DRM_MODE_LINK_STATUS_BAD); From 88545d18df50d8c3440fc9d37132bc155e3df592 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 11 Oct 2026 21:00:37 +0200 Subject: [PATCH 1682/2680] drm: apple: audio: limit the number of periods to what SIO can serve SIO has 64 descriptor slots, shared by all its channels, and its cyclic prep takes one slot per period. The PCM allowed any number of periods, so PipeWire's default of 256-frame periods in a 32768-frame buffer asked for 128 of them. sio_prep_dma_cyclic() then ran out of slots, snd_dmaengine_pcm_trigger() returned -ENOMEM, ALSA undid the START with a STOP (a stop_link round trip to DCP), and PipeWire retried about 100 times a second with no sound. Up to three DP audio outputs share one SIO on t600x and t602x, so allow at most 16 periods per stream, 48 slots for all three. Fixes: cb24343242a7 ("gpu: drm: apple: Add DCP audio driver") Signed-off-by: Thomas Glanzmann (cherry picked from commit ffadcdbe431ddc6c753a573d5aa291ee7d039e4a) --- drivers/gpu/drm/apple/audio.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index 16abff44ffcbc..cc54ae2a9b587 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -33,6 +33,13 @@ #define DCPAUD_ELEMENTS_MAXSIZE 16384 #define DCPAUD_PRODUCTATTRS_MAXSIZE 1024 +/* + * SIO takes one of its 64 descriptor slots per period, shared by all its + * channels, and up to three DP audio outputs use one SIO. A stream with + * more periods fails START with -ENOMEM. + */ +#define DCPAUD_PERIODS_MAX 16 + struct dcp_audio { struct device *dev; struct device *dcp_dev; @@ -77,7 +84,7 @@ static const struct snd_pcm_hardware dcp_pcm_hw = { .period_bytes_min = 4096, /* TODO */ .period_bytes_max = SIZE_MAX, .periods_min = 2, - .periods_max = UINT_MAX, + .periods_max = DCPAUD_PERIODS_MAX, }; /* @@ -103,7 +110,7 @@ static const struct snd_pcm_hardware dcp_pcm_hw_unplugged = { .period_bytes_min = 4096, .period_bytes_max = SIZE_MAX, .periods_min = 2, - .periods_max = UINT_MAX, + .periods_max = DCPAUD_PERIODS_MAX, }; static int dcpaud_read_remote_info(struct dcp_audio *dcpaud) @@ -316,7 +323,7 @@ static int dcp_pcm_open(struct snd_pcm_substream *substream) hw.info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | SNDRV_PCM_INFO_INTERLEAVED; hw.periods_min = 2; - hw.periods_max = UINT_MAX; + hw.periods_max = DCPAUD_PERIODS_MAX; hw.period_bytes_min = 256; hw.period_bytes_max = SIZE_MAX; // TODO dma_get_max_seg_size(dma_dev); hw.buffer_bytes_max = SIZE_MAX; From fb3036a34f33cef7b1be0bd278bdebef324580e3 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Thu, 15 Oct 2026 08:49:52 +0200 Subject: [PATCH 1683/2680] drm/apple: av: recycle AV endpoint service slots The AV endpoint (0x29) announces a fresh DCPAVSimpleVideoInterface and DCPAVAudioInterface on new channels for every DPTX connection, so every display replug or DPMS wake that drops HPD uses two more slots. Since "drm/apple: pin DP service owners before recycling retired AFK slots" a torn-down slot is only reclaimed if its ops opt in with .reusable, which the AV services never did. After eight connections the 16-slot table is full, every later announcement fails with AFK[ep:29]: no reusable service slots and the audio service of that port is not re-registered until reboot. Convert the AV owners and opt in to reuse: - DCPAVSimpleVideoInterface keeps no pointer to its service. - asrv->srv now holds an owner pin taken in init and dropped in teardown. A teardown of an older generation no longer clears the pointer or unplugs the jack of its successor. - av_service_connect() and av_service_disconnect() call into the service after dropping srv_rwsem, so they pin it for the duration. Add afk_service_pin() for this; unlike afk_service_get() it also succeeds after teardown, so the close is still sent. Add a KUnit case for a close pinned across teardown. Signed-off-by: Thomas Glanzmann (cherry picked from commit d0153933646a4199821eec1364a162389cc76dfc) --- drivers/gpu/drm/apple/afk-service-test.c | 18 ++++++++++ drivers/gpu/drm/apple/afk.h | 19 ++++++++++ drivers/gpu/drm/apple/av.c | 46 ++++++++++++++++++------ 3 files changed, 72 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/apple/afk-service-test.c b/drivers/gpu/drm/apple/afk-service-test.c index 358d0edf34519..06db01a8f84d3 100644 --- a/drivers/gpu/drm/apple/afk-service-test.c +++ b/drivers/gpu/drm/apple/afk-service-test.c @@ -101,6 +101,23 @@ static void afk_service_completed_reader_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); } +static void afk_service_close_pin_test(struct kunit *test) +{ + struct apple_epic_service *service = test->priv; + + KUNIT_EXPECT_PTR_EQ(test, afk_service_pin(NULL), NULL); + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 1)); + KUNIT_ASSERT_PTR_EQ(test, afk_service_get(service), service); /* owner */ + afk_test_retire(service); + /* A close issued after teardown still pins the owner's service. */ + KUNIT_ASSERT_PTR_EQ(test, afk_service_pin(service), service); + afk_service_put(service); /* owner detaches */ + KUNIT_EXPECT_FALSE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); + KUNIT_EXPECT_EQ(test, service->channel, 1U); + afk_service_put(service); + KUNIT_EXPECT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); +} + static void afk_service_tag_sequence_test(struct kunit *test) { struct apple_epic_service *service = test->priv; @@ -257,6 +274,7 @@ static struct kunit_case afk_service_cases[] = { KUNIT_CASE(afk_service_reader_pin_test), KUNIT_CASE(afk_service_pending_reply_test), KUNIT_CASE(afk_service_completed_reader_test), + KUNIT_CASE(afk_service_close_pin_test), KUNIT_CASE(afk_service_tag_sequence_test), KUNIT_CASE(afk_service_unconverted_owner_test), KUNIT_CASE(afk_service_retired_channel_test), diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 716b9fe595c64..2b6609abe361a 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -159,6 +159,25 @@ static inline void afk_service_put(struct apple_epic_service *service) afk_service_retry(service->ep); } +/* + * Take another pin on a service the caller already keeps pinned, e.g. an + * owner pointer read under the owner's lock. Unlike afk_service_get() this + * also succeeds after teardown, so a close can still be issued. + */ +static inline struct apple_epic_service *afk_service_pin(struct apple_epic_service *service) +{ + unsigned long flags; + + if (!service) + return NULL; + + spin_lock_irqsave(&service->lock, flags); + service->users++; + spin_unlock_irqrestore(&service->lock, flags); + + return service; +} + static inline void afk_service_disable(struct apple_epic_service *service) { unsigned long flags; diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c index 052950d7a4ff7..8a19f2a295bbc 100644 --- a/drivers/gpu/drm/apple/av.c +++ b/drivers/gpu/drm/apple/av.c @@ -96,12 +96,21 @@ static void av_audiosrv_init(struct apple_epic_service *service, const char *nam { struct apple_dcp *dcp = service->ep->dcp; struct audiosrv_data *asrv = dcp->audiosrv; + struct apple_epic_service *old; + + /* The owner pin keeps the slot from being recycled while published. */ + service = afk_service_get(service); + if (!service) + return; mutex_lock(&asrv->plug_lock); down_write(&asrv->srv_rwsem); + old = asrv->srv; asrv->srv = service; up_write(&asrv->srv_rwsem); + if (old) + afk_service_put(old); asrv->plugged = true; mutex_unlock(&asrv->plug_lock); @@ -112,18 +121,27 @@ static void av_audiosrv_teardown(struct apple_epic_service *service) { struct apple_dcp *dcp = service->ep->dcp; struct audiosrv_data *asrv = dcp->audiosrv; + bool owned; mutex_lock(&asrv->plug_lock); down_write(&asrv->srv_rwsem); - asrv->srv = NULL; + owned = asrv->srv == service; + if (owned) + asrv->srv = NULL; up_write(&asrv->srv_rwsem); - asrv->plugged = false; - if (asrv->audio_dev) - dcpaud_disconnect(asrv->audio_dev); + /* A stale generation must not unplug its successor. */ + if (owned) { + asrv->plugged = false; + if (asrv->audio_dev) + dcpaud_disconnect(asrv->audio_dev); + } mutex_unlock(&asrv->plug_lock); + + if (owned) + afk_service_put(service); } int dcp_audiosrv_prepare(struct device *dev, struct dcp_sound_cookie *cookie) @@ -279,11 +297,13 @@ static int av_audiosrv_report(struct apple_epic_service *service, u32 idx, static const struct apple_epic_service_ops avep_ops[] = { { .name = "DCPAVSimpleVideoInterface", + .reusable = true, .init = av_interface_init, .teardown = av_interface_teardown, }, { .name = "DCPAVAudioInterface", + .reusable = true, .init = av_audiosrv_init, .report = av_audiosrv_report, .teardown = av_audiosrv_teardown, @@ -298,21 +318,21 @@ void av_service_connect(struct apple_dcp *dcp) int ret; scoped_guard(rwsem_write, &asrv->srv_rwsem) { - if (!asrv->srv) + service = afk_service_pin(asrv->srv); + if (!service) return; - service = asrv->srv; } /* open AV audio service */ dev_info(dcp->dev, "%s: starting audio service, plugged:%d\n", __func__, asrv->plugged); if (asrv->is_open) - return; + goto out; ret = afk_service_call(service, 0, asrv->cmds.open, NULL, 0, 32, NULL, 0, 32); if (ret) { dev_err(dcp->dev, "error opening audio service: %d\n", ret); - return; + goto out; } mutex_lock(&asrv->plug_lock); asrv->is_open = true; @@ -320,6 +340,8 @@ void av_service_connect(struct apple_dcp *dcp) if (asrv->audio_dev) dcpaud_connect(asrv->audio_dev, asrv->plugged); mutex_unlock(&asrv->plug_lock); +out: + afk_service_put(service); } void av_service_disconnect(struct apple_dcp *dcp) @@ -329,15 +351,15 @@ void av_service_disconnect(struct apple_dcp *dcp) int ret; scoped_guard(rwsem_write, &asrv->srv_rwsem) { - if (!asrv->srv) + service = afk_service_pin(asrv->srv); + if (!service) return; - service = asrv->srv; } /* close AV audio service */ dev_info(dcp->dev, "%s: stopping audio service\n", __func__); if (!asrv->is_open) - return; + goto out; mutex_lock(&asrv->plug_lock); @@ -354,6 +376,8 @@ void av_service_disconnect(struct apple_dcp *dcp) if (service->torndown) afk_service_disable(service); asrv->is_open = false; +out: + afk_service_put(service); } static void av_work_service_start(struct work_struct *work) From f883647baac5868ed2b4f46523ea75fe8a5f9907 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 01:08:13 -0700 Subject: [PATCH 1684/2680] thunderbolt: apple: Bind display handoffs to immutable generations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue nonzero cookies per handoff and separate requested, admitted and returned display ownership. Validate immutable bindings before DP-IN MMIO, including synchronous Activate before attach returns. Reject superseded completion and force replacement when tunnel events coalesce. Change the shared interface, TB caller and DCP provider together. Revoke callbacks before draining reconnect outside the fabric and DCP locks on both detach and failed attach. Idle the powered adapter synchronously on replacement before the old binding drains and its registers are unmapped. Publish cookie acceptance and lifecycle state in the same locked operation, so a running worker cannot activate before a delayed replacement event. Preserve SoC qualification, readiness waits and retry policy. Exercise cookie admission, supersession, same-cookie idempotence, counter saturation and powered request ordering with KUnit and fake effects. [ryan: adapted to 7.2.10 — DCP side only: include/linux/soc/apple/dp-tunnel.h and the drivers/thunderbolt caller are carried by 170-usb (MAINTAINERS hunk not taken). Placed last so every earlier commit builds against aurora/bits/170-usb's interface; from here the topic requires unified/bits/170-usb's dp-tunnel.h (apple_dcp_tb_dp_tunnel() with a u64 generation). This tree has no tb_retiring, dcp_tb_handoff_lock or dcp_dptx_disconnect_drained(): dcp_tb_binding_retire() clears the cookie and callbacks under tb_lock and cancels the reconnect work synchronously under the fabric lock, as dcp_dptx_disconnect_oob() already does there; dcp_fabric_drain_binding() is kept for its KUnit contract only. Reconnect sessions ({typec_generation, tb_generation}) are captured at enqueue by dcp_queue_typec_reconnect() and rechecked at each step of dcp_dptx_connect_raw(); typec_follow_retiring stands in for tb_retiring. H17P (J700/T8140) connects and reconnects without a session, as before. dcp_tb_claim() does not bump typec_generation (the cookie separates bindings). Upgraded from the no-generation adaptations: afbcf8083b/a6285508f3 (route follow and HDMI hand-back carry tunnel_generation; a slot without one is stale), 65361daa5f (per-step session checks in dcp_dptx_recover_irq()), fcc9c34002/5eeea77503 (split.generation in the split, its callbacks, tile reconnect and follow carry), 8988142147 (a failed deactivate also retires the cookie), 479d414d8e (generation-keyed dcp_tb_finish_request() and tunnel_generation retirement replace the callback-keyed dcp_tb_binding_live()), 13d2affbc4 (callback_valid check in dcp_tunnel_set_rate()). -ENOSPC is kept for the DP-alt mux; Thunderbolt still gets -EBUSY from dcp_fabric_tunnel_candidate(), which its capacity retry waits on.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 3519a4bc491fa3a2d44d4bddaecca7e50daa915a) --- drivers/gpu/drm/apple/dcp-fabric-session.h | 63 +++++++ drivers/gpu/drm/apple/dcp-fabric-test.c | 139 ++++++++++++++++ drivers/gpu/drm/apple/dcp-fabric.c | 183 +++++++++++++++------ drivers/gpu/drm/apple/dcp-fabric.h | 1 + drivers/gpu/drm/apple/dcp-internal.h | 4 + drivers/gpu/drm/apple/dcp.c | 180 +++++++++++++++++--- drivers/gpu/drm/apple/dcp.h | 1 + 7 files changed, 498 insertions(+), 73 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp-fabric-session.h diff --git a/drivers/gpu/drm/apple/dcp-fabric-session.h b/drivers/gpu/drm/apple/dcp-fabric-session.h new file mode 100644 index 0000000000000..be850a2897562 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-fabric-session.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __APPLE_DCP_FABRIC_SESSION_H__ +#define __APPLE_DCP_FABRIC_SESSION_H__ + +#include +#include + +struct dcp_fabric_session { + u64 generation; + u64 cookie; +}; + +/* The live binding is a lease, not a tombstone across TB module reloads. */ +static inline int dcp_fabric_binding_request(u64 live, u64 cookie, bool active, + bool same_binding) +{ + if (!cookie) + return -EINVAL; + if (!live) + return active ? 0 : -ESTALE; + if (cookie != live || (active && !same_binding)) + return -ESTALE; + return active ? 1 : 0; +} + +static inline bool dcp_fabric_callback_valid(u64 route, u64 admitted) +{ + return admitted && admitted == route; +} + +struct dcp_fabric_drain_ops { + void (*reserve_revoke)(void *ctx); + void (*invalidate)(void *ctx); + void (*unlock)(void *ctx); + void (*drain)(void *ctx); + void (*lock)(void *ctx); +}; + +/* + * The source's production ordering; drain waits with neither fabric nor tb + * held. This tree has no tb_retiring to keep the pipeline reserved while the + * fabric lock is dropped, so its fabric retires under the lock instead (see + * dcp_tb_binding_retire()); the KUnit cases keep the ordering contract. + */ +static inline void dcp_fabric_drain_binding(const struct dcp_fabric_drain_ops *ops, + void *ctx) +{ + ops->reserve_revoke(ctx); + ops->invalidate(ctx); + ops->unlock(ctx); + ops->drain(ctx); + ops->lock(ctx); +} + +static inline bool dcp_fabric_session_valid(struct dcp_fabric_session expected, + struct dcp_fabric_session live_session, + bool cable, bool retiring) +{ + return !retiring && cable && expected.generation == live_session.generation && + expected.cookie == live_session.cookie; +} + +#endif /* __APPLE_DCP_FABRIC_SESSION_H__ */ diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c index c4fd58b9d7988..115df27de0c69 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-test.c +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -8,6 +8,7 @@ #include "dcp-fabric-core.h" #include "dcp-fabric-effects.h" +#include "dcp-fabric-session.h" struct fabric_fixture { struct dcp_fabric_pipeline pipeline[2]; @@ -2002,12 +2003,150 @@ static void fabric_t6030_follow_test(struct kunit *test) NULL, NULL, false, &f.policy), DCP_FABRIC_FOLLOW_MOVE); } +static void fabric_binding_cookie_test(struct kunit *test) +{ + /* First admission may callback before attach returns/owner is published. */ + KUNIT_EXPECT_EQ(test, dcp_fabric_binding_request(0, 11, true, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_fabric_callback_valid(11, 11)); + KUNIT_EXPECT_EQ(test, dcp_fabric_binding_request(11, 11, true, true), 1); + KUNIT_EXPECT_EQ(test, dcp_fabric_binding_request(11, 11, true, false), -ESTALE); + KUNIT_EXPECT_EQ(test, dcp_fabric_binding_request(11, 12, true, true), -ESTALE); + KUNIT_EXPECT_EQ(test, dcp_fabric_binding_request(11, 10, false, true), -ESTALE); + KUNIT_EXPECT_EQ(test, dcp_fabric_binding_request(11, 11, false, false), 0); + KUNIT_EXPECT_EQ(test, dcp_fabric_binding_request(0, 11, false, true), -ESTALE); + KUNIT_EXPECT_EQ(test, dcp_fabric_binding_request(0, 0, true, true), -EINVAL); + KUNIT_EXPECT_FALSE(test, dcp_fabric_callback_valid(11, 0)); + KUNIT_EXPECT_FALSE(test, dcp_fabric_callback_valid(12, 11)); + /* Complete drain removes the lease; a reloaded TB allocator may start at 1. */ + KUNIT_EXPECT_EQ(test, dcp_fabric_binding_request(0, 1, true, false), 0); +} + +static void fabric_reconnect_session_test(struct kunit *test) +{ + struct dcp_fabric_session queued = { .generation = 7, .cookie = 11 }; + struct dcp_fabric_session live_session = queued; + + KUNIT_EXPECT_TRUE(test, dcp_fabric_session_valid(queued, live_session, true, false)); + KUNIT_EXPECT_FALSE(test, dcp_fabric_session_valid(queued, live_session, false, false)); + KUNIT_EXPECT_FALSE(test, dcp_fabric_session_valid(queued, live_session, true, true)); + live_session.generation++; + KUNIT_EXPECT_FALSE(test, dcp_fabric_session_valid(queued, live_session, true, false)); + live_session = queued; + live_session.cookie++; + KUNIT_EXPECT_FALSE(test, dcp_fabric_session_valid(queued, live_session, true, false)); + live_session.cookie = 0; + KUNIT_EXPECT_FALSE(test, dcp_fabric_session_valid(queued, live_session, true, false)); + /* Retained sleep keeps the TB cookie but invalidates the DCP connection. */ + live_session = queued; + live_session.generation++; + queued = live_session; + KUNIT_EXPECT_TRUE(test, dcp_fabric_session_valid(queued, live_session, true, false)); + /* Direct DP-alt uses cookie zero; local generation still guards queued work. */ + queued.cookie = 0; + live_session.cookie = 0; + KUNIT_EXPECT_TRUE(test, dcp_fabric_session_valid(queued, live_session, true, false)); + live_session.generation++; + KUNIT_EXPECT_FALSE(test, dcp_fabric_session_valid(queued, live_session, true, false)); +} + +struct fabric_drain_fake { + struct kunit *test; + unsigned int step; + u64 lease; + u64 admitted; + bool fabric_locked; + bool reserved; + bool cable; + bool work_running; +}; + +static void fabric_drain_reserve(void *data) +{ + struct fabric_drain_fake *f = data; + + KUNIT_EXPECT_EQ(f->test, f->step++, 0U); + KUNIT_EXPECT_TRUE(f->test, f->fabric_locked); + f->reserved = true; + f->admitted = 0; +} + +static void fabric_drain_invalidate(void *data) +{ + struct fabric_drain_fake *f = data; + + KUNIT_EXPECT_EQ(f->test, f->step++, 1U); + KUNIT_EXPECT_FALSE(f->test, dcp_fabric_callback_valid(f->lease, f->admitted)); + f->cable = false; +} + +static void fabric_drain_unlock(void *data) +{ + struct fabric_drain_fake *f = data; + + KUNIT_EXPECT_EQ(f->test, f->step++, 2U); + f->fabric_locked = false; +} + +static void fabric_drain_wait(void *data) +{ + struct fabric_drain_fake *f = data; + + KUNIT_EXPECT_EQ(f->test, f->step++, 3U); + KUNIT_EXPECT_FALSE(f->test, f->fabric_locked); + KUNIT_EXPECT_TRUE(f->test, f->reserved); + KUNIT_EXPECT_EQ(f->test, f->lease, 11ULL); + KUNIT_EXPECT_FALSE(f->test, f->cable); + KUNIT_EXPECT_TRUE(f->test, f->work_running); + f->work_running = false; +} + +static void fabric_drain_lock(void *data) +{ + struct fabric_drain_fake *f = data; + + KUNIT_EXPECT_EQ(f->test, f->step++, 4U); + KUNIT_EXPECT_FALSE(f->test, f->work_running); + f->fabric_locked = true; +} + +static void fabric_binding_drain_test(struct kunit *test) +{ + static const struct dcp_fabric_drain_ops ops = { + .reserve_revoke = fabric_drain_reserve, + .invalidate = fabric_drain_invalidate, + .unlock = fabric_drain_unlock, + .drain = fabric_drain_wait, + .lock = fabric_drain_lock, + }; + unsigned int i; + + /* The shared executor drives both failed-attach and detach drain paths. */ + for (i = 0; i < 2; i++) { + struct fabric_drain_fake f = { + .test = test, .lease = 11, .admitted = 11, + .fabric_locked = true, .cable = true, .work_running = true, + }; + + dcp_fabric_drain_binding(&ops, &f); + KUNIT_EXPECT_EQ(test, f.step, 5U); + KUNIT_EXPECT_FALSE(test, f.work_running); + KUNIT_EXPECT_TRUE(test, f.fabric_locked); + /* Only now may the caller release its lease/reservation and return. */ + f.lease = 0; + f.reserved = false; + KUNIT_EXPECT_EQ(test, dcp_fabric_binding_request(f.lease, 1, true, false), 0); + } +} + static struct kunit_case fabric_tests[] = { KUNIT_CASE(fabric_t6030_clock_routes_test), KUNIT_CASE(fabric_t6030_plan_test), KUNIT_CASE(fabric_t6030_follow_test), KUNIT_CASE(fabric_reclaim_transaction_test), KUNIT_CASE_PARAM(fabric_shared_wiring_test, fabric_wiring_gen_params), + KUNIT_CASE(fabric_binding_cookie_test), + KUNIT_CASE(fabric_reconnect_session_test), + KUNIT_CASE(fabric_binding_drain_test), KUNIT_CASE(fabric_rebind_presence_test), KUNIT_CASE(fabric_nonhybrid_hdmi_resume_test), KUNIT_CASE(fabric_resume_no_connect_test), diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index fc2ee7e5e30ea..3e8a34cc4c09d 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -461,6 +461,8 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) dcp->dptx_dfp_port = 0; dcp->tb_dpin_set_active = NULL; dcp->tb_dpin_ctx = NULL; + dcp->tb_generation = 0; + route->tunnel_generation = 0; dcp->tb_clock_ok = false; } if (was_tunnel && route->mux_index) { @@ -1628,6 +1630,11 @@ static void dcp_tunnel_prepare(struct apple_dcp_typec_route *route, static int dcp_tunnel_dpin_locked(struct apple_dcp *dcp, bool active) { lockdep_assert_held(&dcp->tb_lock); + /* Only the binding admitted for the route DCP drives is called. */ + if (!dcp->active_typec_route || + !dcp_fabric_callback_valid(dcp->active_typec_route->tunnel_generation, + dcp->tb_generation)) + return -ESTALE; if (!dcp->tb_dpin_set_active) return -ENODEV; return dcp->tb_dpin_set_active(dcp->tb_dpin_ctx, active); @@ -1647,7 +1654,7 @@ static int dcp_tunnel_split_up_locked(struct apple_dcp *dcp) int ret; lockdep_assert_held(&dcp->tb_lock); - if (!dcp->split.active || !dcp->split.set_active || !dcp->split.xbar) + if (!dcp->split.active || !dcp->split.generation || !dcp->split.xbar) return -ENODEV; if (!dcp->split.clock_ok) { dev_warn(dcp->dev, "tiled: no DP tunnel pixel clock, dpin1 crossbar left down\n"); @@ -1682,6 +1689,8 @@ int dcp_tunnel_crossbar_up(struct apple_dcp *dcp, u32 unit) if (!route || !route->tunnel || !route->active_xbar || route->deactivate_xbar_done) return -ENODEV; + if (!dcp_fabric_callback_valid(route->tunnel_generation, dcp->tb_generation)) + return -ESTALE; if (!dcp->tb_clock_ok) { dev_warn(dcp->dev, "no DP tunnel pixel clock, crossbar left down\n"); return -EIO; @@ -1724,6 +1733,8 @@ int dcp_tunnel_crossbar_down(struct apple_dcp *dcp, u32 unit) route = dcp->active_typec_route; if (!route || !route->tunnel || !route->xbar_up) return 0; + if (!dcp_fabric_callback_valid(route->tunnel_generation, dcp->tb_generation)) + return -ESTALE; dcp_tunnel_dpin_locked(dcp, false); return dcp_dpxbar_link(route->active_xbar, false); } @@ -1782,13 +1793,15 @@ int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 unit, route = dcp->active_typec_route; if (!route) return -ENODEV; + if (!dcp_fabric_callback_valid(route->tunnel_generation, dcp->tb_generation)) + return -ESTALE; if (of_machine_is_compatible("apple,t6030") && (unit || dcp->dptx_dfp_port != 1)) return -EOPNOTSUPP; if (unit) { /* Each DP IN owns a reference, including on a shared clock. */ ret = 0; - if (!dcp->split.active || !dcp->split.set_active) { + if (!dcp->split.active || !dcp->split.generation) { ret = -ENODEV; } else { if (!link_rate && dcp->split.xbar_up) @@ -1821,7 +1834,7 @@ int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, u32 unit, bool active) if (!dcp->dptx_tunnel) return 0; if (unit) { - if (!dcp->split.active || !dcp->split.set_active) + if (!dcp->split.active || !dcp->split.generation || !dcp->split.set_active) return 0; return dcp->split.set_active(dcp->split.binding, active); } @@ -1829,6 +1842,8 @@ int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, u32 unit, bool active) * T602X DP IN .set does not enable clocks; DidChange does that later. */ route = dcp->active_typec_route; + if (!route || !dcp_fabric_callback_valid(route->tunnel_generation, dcp->tb_generation)) + return -ESTALE; if (active && dcp_t602x_tunnel_route(route)) { if (!route->xbar_up) { ret = mux_control_try_select(route->active_xbar, route->mux_index); @@ -1859,6 +1874,7 @@ static bool dcp_tb_services_ready(struct apple_dcp *dcp) } struct dcp_tb_attach_context { + u64 generation; struct apple_dcp_typec_port *port; struct apple_dcp_typec_route **slot; struct apple_dcp_typec_route *best; @@ -1940,9 +1956,12 @@ static int dcp_tb_activate(void *data) struct dcp_tb_attach_context *ctx = data; struct apple_dcp *dcp = ctx->best->dcp; + /* Admitted before the route goes live: Activate may run before we return. */ scoped_guard(mutex, &dcp->tb_lock) { dcp->tb_dpin_set_active = ctx->set_active; dcp->tb_dpin_ctx = ctx->binding; + dcp->tb_generation = ctx->generation; + ctx->best->tunnel_generation = ctx->generation; } ctx->best->tunnel_dpin = ctx->dpin; return dcp_typec_route_activate(ctx->best, ctx->ctl); @@ -1953,9 +1972,12 @@ static void dcp_tb_rollback(void *data) struct dcp_tb_attach_context *ctx = data; struct apple_dcp *dcp = ctx->best->dcp; + /* The failed attempt's binding is retired before the provider hears of it. */ guard(mutex)(&dcp->tb_lock); dcp->tb_dpin_set_active = NULL; dcp->tb_dpin_ctx = NULL; + dcp->tb_generation = 0; + ctx->best->tunnel_generation = 0; } static void dcp_tb_claim(void *data) @@ -2011,7 +2033,7 @@ static void dcp_tb_queue_reconnect(void *data) struct apple_dcp *dcp = ctx->best->dcp; dcp->typec_reconnect_tries = 0; - mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); + dcp_queue_typec_reconnect(dcp, 0); } static void dcp_tb_connect_oob(void *data) @@ -2032,11 +2054,26 @@ static const struct dcp_fabric_attach_ops dcp_tb_attach_ops = { .connect_oob = dcp_tb_connect_oob, }; -/* Does @dcp's tunnel still hold its Thunderbolt binding's callbacks? */ -static bool dcp_tb_binding_live(struct apple_dcp *dcp) +/* + * Retire @dcp's Thunderbolt binding (and @route's, if given): no DCP callback, + * crossbar or clock change reaches it once this returns, and reconnect work + * queued or running for its session finds that session revoked. The source + * drains with the fabric lock dropped and the pipeline reserved by + * tb_retiring; this tree has no tb_retiring and keeps the fabric lock across + * the wait, as dcp_dptx_disconnect_oob() always has. Neither worker takes it. + */ +static void dcp_tb_binding_retire(struct apple_dcp *dcp, + struct apple_dcp_typec_route *route) { - guard(mutex)(&dcp->tb_lock); - return !!dcp->tb_dpin_set_active; + lockdep_assert_held(&dcp_typec_fabric_lock); + scoped_guard(mutex, &dcp->tb_lock) { + dcp->tb_generation = 0; + if (route) + route->tunnel_generation = 0; + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + } + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); } /* @@ -2087,6 +2124,7 @@ static int dcp_tb_split_teardown_locked(struct apple_dcp_typec_port *port, dcp->split.xbar_up = false; dcp->split.clock_ok = false; dcp->split.ready = false; + dcp->split.generation = 0; dcp->split.set_active = NULL; dcp->split.binding = NULL; } @@ -2102,7 +2140,7 @@ static int dcp_tb_split_teardown_locked(struct apple_dcp_typec_port *port, */ static int dcp_tb_split_join_locked(struct apple_dcp_typec_port *port, struct apple_dcp_typec_route *route, - struct mux_control *xbar, + struct mux_control *xbar, u64 generation, int (*set_active)(void *binding, bool active), void *binding) { @@ -2130,6 +2168,7 @@ static int dcp_tb_split_join_locked(struct apple_dcp_typec_port *port, dcp->split.mux_state = state; dcp->split.xbar_up = false; dcp->split.clock_ok = false; + dcp->split.generation = generation; dcp->split.set_active = set_active; dcp->split.binding = binding; dcp->split.ready = false; @@ -2157,19 +2196,22 @@ static int dcp_tb_split_locked(struct apple_dcp_typec_port *port, int ret, release; lockdep_assert_held(&dcp_typec_fabric_lock); - /* - * A retired provider leaves only hardware ownership to retry. This - * tree has no Thunderbolt binding generation: a split whose callbacks - * were cleared is the retired one. - */ - if (port->split_dcp && !port->split_dcp->split.set_active) { + /* A retired provider leaves only hardware ownership to retry. */ + if (port->split_dcp && !port->split_dcp->split.generation) { ret = dcp_tb_split_teardown_locked(port, false); if (ret || !req->active) return ret; } - /* A repeated request for a split dpin1 is answered by its state. */ + /* Only the live cookie and binding may repeat or end a split dpin1. */ if (port->split_dcp) { dcp = port->split_dcp; + scoped_guard(mutex, &dcp->tb_lock) + ret = dcp_fabric_binding_request(dcp->split.generation, + req->generation, req->active, + dcp->split.set_active == req->set_active && + dcp->split.binding == req->binding); + if (ret < 0) + return ret; if (req->active) return dcp->split.ready ? 0 : -EAGAIN; return dcp_tb_split_teardown_locked(port, false); @@ -2201,8 +2243,8 @@ static int dcp_tb_split_locked(struct apple_dcp_typec_port *port, return -ENODEV; return -EOPNOTSUPP; } - ret = dcp_tb_split_join_locked(port, route, xbar, req->set_active, - req->binding); + ret = dcp_tb_split_join_locked(port, route, xbar, req->generation, + req->set_active, req->binding); if (ret) return ret; @@ -2244,11 +2286,16 @@ static int dcp_tb_release_locked(struct apple_dcp_typec_port *port, } ret = dcp_typec_route_deactivate(*slot); if (ret) { - /* the caller's context is going away regardless */ + /* + * The caller's context is going away regardless: retire the + * binding, the route stays owned for the next request to release. + */ scoped_guard(mutex, &dcp->tb_lock) { dcp->dptx_tunnel = false; dcp->tb_dpin_set_active = NULL; dcp->tb_dpin_ctx = NULL; + dcp->tb_generation = 0; + (*slot)->tunnel_generation = 0; } return ret; } @@ -2269,6 +2316,7 @@ static int dcp_tb_dispatch_request(void *data, const struct dcp_fabric_port *fou struct apple_dcp_typec_port *port = container_of(found, struct apple_dcp_typec_port, core); struct apple_dcp_typec_route **slot; + struct apple_dcp *dcp; unsigned int dpin = ctx->dpin; int ret; @@ -2281,18 +2329,33 @@ static int dcp_tb_dispatch_request(void *data, const struct dcp_fabric_port *fou } /* A retired binding leaves only hardware ownership to retry. */ if (*slot && (*slot)->tunnel && (*slot)->tunnel_dpin == dpin && - !dcp_tb_binding_live((*slot)->dcp)) { + !(*slot)->tunnel_generation) { ret = dcp_tb_release_locked(port, dpin); if (ret || !ctx->active) return ret; } - if (!ctx->active) { - if (!*slot || !(*slot)->tunnel || - (*slot)->tunnel_dpin != dpin) - return 0; - return dcp_tb_release_locked(port, dpin); + /* + * Only the live cookie and binding may repeat or end a handed stream; a + * mismatch is -ESTALE before anything changes. + */ + if (*slot && (*slot)->tunnel && (*slot)->tunnel_dpin == dpin) { + dcp = (*slot)->dcp; + scoped_guard(mutex, &dcp->tb_lock) { + bool same = dcp->tb_dpin_set_active == ctx->set_active && + dcp->tb_dpin_ctx == ctx->binding; + + ret = dcp_fabric_binding_request((*slot)->tunnel_generation, + ctx->generation, ctx->active, same); + } + if (ret) + return ret > 0 ? 0 : ret; + } else if (!ctx->active) { + return -ESTALE; } + if (!ctx->active) + return dcp_tb_release_locked(port, dpin); + { struct dcp_fabric_port state = { .owner[0] = port->owner ? &port->owner->core : NULL, @@ -2333,34 +2396,29 @@ static int dcp_tb_dispatch_request(void *data, const struct dcp_fabric_port *fou } /* - * Failed attach and every detach return before the provider frees its binding. - * This tree has no binding generation or drain: a request's callbacks are - * matched by their context and cleared, and the hardware ownership they leave - * is retried by the next request (see dcp_tb_binding_live()). + * Failed attach and every detach return before the provider frees its + * binding: the request's own cookie (and, for an attach, its own binding) is + * retired. A retired route keeps its hardware ownership for the next request + * to release (see dcp_tb_dispatch_request()). */ -static bool dcp_tb_request_matches(int (*set_active)(void *binding, bool active), - void *binding, - const struct dcp_tb_attach_context *request) -{ - if (!request->binding) - return !request->active; - return binding == request->binding && set_active == request->set_active; -} - static void dcp_tb_finish_request(struct apple_dcp_typec_port *port, const struct dcp_tb_attach_context *request) { struct apple_dcp_typec_route *route = request->dpin ? port->secondary_owner : port->owner; struct apple_dcp *dcp; + bool matching; if (request->dpin == 1 && port->split_dcp) { dcp = port->split_dcp; scoped_guard(mutex, &dcp->tb_lock) { - if (dcp_tb_request_matches(dcp->split.set_active, - dcp->split.binding, request)) { + if (dcp->split.generation == request->generation && + (!request->active || + (dcp->split.binding == request->binding && + dcp->split.set_active == request->set_active))) { dcp->split.set_active = NULL; dcp->split.binding = NULL; + dcp->split.generation = 0; dcp->split.ready = false; } } @@ -2368,13 +2426,13 @@ static void dcp_tb_finish_request(struct apple_dcp_typec_port *port, if (!route || !route->tunnel) return; dcp = route->dcp; - scoped_guard(mutex, &dcp->tb_lock) { - if (dcp_tb_request_matches(dcp->tb_dpin_set_active, - dcp->tb_dpin_ctx, request)) { - dcp->tb_dpin_set_active = NULL; - dcp->tb_dpin_ctx = NULL; - } - } + scoped_guard(mutex, &dcp->tb_lock) + matching = dcp->tb_generation == request->generation && + (!request->active || + (dcp->tb_dpin_ctx == request->binding && + dcp->tb_dpin_set_active == request->set_active)); + if (matching) + dcp_tb_binding_retire(dcp, route); } static int dcp_tb_dispatch(void *data, const struct dcp_fabric_port *found) @@ -2390,19 +2448,20 @@ static int dcp_tb_dispatch(void *data, const struct dcp_fabric_port *found) } int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, - bool active, int (*set_active)(void *ctx, bool active), - void *ctx) + u64 generation, bool active, + int (*set_active)(void *binding, bool active), void *binding) { struct apple_dcp_typec_port *pos; struct dcp_fabric_port *ports = NULL, **tail = &ports; struct dcp_tb_attach_context request = { + .generation = generation, .dpin = dpin, .active = active, .set_active = set_active, - .binding = ctx, + .binding = binding, }; - if (!connector_np || dpin > 1) + if (!connector_np || dpin > 1 || !generation || (active && !set_active)) return -EINVAL; guard(mutex)(&dcp_typec_fabric_lock); list_for_each_entry(pos, &dcp_typec_ports, link) { @@ -2447,10 +2506,12 @@ struct dcp_typec_follow_slot { /* a tunnel's DP IN and Thunderbolt binding, carried along */ bool tunnel; unsigned int dpin; + u64 tunnel_generation; int (*set_active)(void *binding, bool active); void *binding; /* tiled_split: the second tile, on DPTX port 1 of the same pipeline */ bool split; + u64 split_generation; int (*split_set_active)(void *binding, bool active); void *split_binding; }; @@ -2465,9 +2526,10 @@ static void dcp_follow_save_split(struct apple_dcp_typec_route *route, return; scoped_guard(mutex, &dcp->tb_lock) { /* A retired second stream cannot be carried into the move. */ - if (!dcp->split.set_active) + if (!dcp->split.generation || !dcp->split.set_active) return; slot->split = true; + slot->split_generation = dcp->split.generation; slot->split_set_active = dcp->split.set_active; slot->split_binding = dcp->split.binding; } @@ -2654,10 +2716,11 @@ static int dcp_follow_prepare(void *data, unsigned int index) struct apple_dcp *dcp = slot->from->dcp; scoped_guard(mutex, &dcp->tb_lock) { + slot->tunnel_generation = slot->from->tunnel_generation; slot->set_active = dcp->tb_dpin_set_active; slot->binding = dcp->tb_dpin_ctx; } - if (!slot->set_active) + if (!slot->tunnel_generation || !slot->set_active) return -ESTALE; } dcp_follow_save_split(slot->from, slot); @@ -2715,6 +2778,8 @@ static int dcp_follow_release(struct apple_dcp_typec_route *route) /* The binding moves on: no callback may reach it through here. */ if (tunnel) { scoped_guard(mutex, &dcp->tb_lock) { + dcp->tb_generation = 0; + route->tunnel_generation = 0; dcp->tb_dpin_set_active = NULL; dcp->tb_dpin_ctx = NULL; } @@ -2761,6 +2826,7 @@ static int dcp_follow_activate(struct apple_dcp_typec_route *route, if (slot->split) { ret = dcp_tb_split_join_locked(slot->port, route, dcp_typec_tunnel_ctl(route, 1), + slot->split_generation, slot->split_set_active, slot->split_binding); if (ret) @@ -2776,6 +2842,8 @@ static int dcp_follow_activate(struct apple_dcp_typec_route *route, scoped_guard(mutex, &dcp->tb_lock) { dcp->tb_dpin_set_active = slot->set_active; dcp->tb_dpin_ctx = slot->binding; + dcp->tb_generation = slot->tunnel_generation; + route->tunnel_generation = slot->tunnel_generation; } route->tunnel_dpin = slot->dpin; } @@ -2787,6 +2855,8 @@ static int dcp_follow_activate(struct apple_dcp_typec_route *route, scoped_guard(mutex, &dcp->tb_lock) { dcp->tb_dpin_set_active = NULL; dcp->tb_dpin_ctx = NULL; + dcp->tb_generation = 0; + route->tunnel_generation = 0; } } return ret; @@ -3260,6 +3330,8 @@ static void dcp_typec_route_unregister(void *data) if (port->owner == route) { struct apple_dcp *dcp = route->dcp; + if (route->tunnel) + dcp_tb_binding_retire(dcp, route); if (port->hpd || dcp->typec_cable_connected) dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); port->hpd = false; @@ -3270,6 +3342,8 @@ static void dcp_typec_route_unregister(void *data) if (port->secondary_owner == route) { struct apple_dcp *dcp = route->dcp; + if (route->tunnel) + dcp_tb_binding_retire(dcp, route); if (port->hpd || dcp->typec_cable_connected) dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); if (dcp_typec_route_deactivate(route) && dcp_is_external_h17p(dcp)) @@ -3814,10 +3888,11 @@ static bool dcp_typec_hdmi_reclaim(struct apple_dcp *dcp) slot.dpin = owner->tunnel_dpin; if (slot.tunnel) { scoped_guard(mutex, &dcp->tb_lock) { + slot.tunnel_generation = owner->tunnel_generation; slot.set_active = dcp->tb_dpin_set_active; slot.binding = dcp->tb_dpin_ctx; } - if (!slot.set_active) + if (!slot.tunnel_generation || !slot.set_active) return false; } dcp_follow_save_split(owner, &slot); diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index 04d6dbb3449cc..62b1a72d5946a 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -41,6 +41,7 @@ struct apple_dcp_typec_route { bool selected; /* crossbar output actually selected: xbar (dpphy) or a Thunderbolt dpin */ unsigned int tunnel_dpin; + u64 tunnel_generation; struct mux_control *active_xbar; bool tunnel; /* tunnel: crossbar brought up (at DidChangeLinkConfiguration) */ diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 9330f6cb95e10..b79a27f8c605f 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -17,6 +17,7 @@ #include #include "dcp-fabric.h" +#include "dcp-fabric-session.h" #include "dptxep.h" #include "dcp_backlight.h" #include "dcp-announce.h" @@ -515,6 +516,7 @@ struct apple_dcp { /* wakes/sleeps the Thunderbolt DP IN adapter from DCP Activate/Deactivate */ int (*tb_dpin_set_active)(void *ctx, bool active); void *tb_dpin_ctx; + u64 tb_generation; /* tb_lock: admitted callback cookie, including attach */ /* * Serializes the Thunderbolt DP IN callback and tunnel crossbar state * between DCP apcalls and tunnel teardown; never held while waiting @@ -536,6 +538,7 @@ struct apple_dcp { int mux_state; /* dispextN, DPTX port 1 */ bool xbar_up; bool clock_ok; + u64 generation; /* the Thunderbolt binding's */ int (*set_active)(void *binding, bool active); void *binding; } split; @@ -550,6 +553,7 @@ struct apple_dcp { struct completion typec_iomfb_hpd_ready; struct delayed_work typec_reconnect_wq; u32 typec_reconnect_tries; + struct dcp_fabric_session typec_reconnect_session; /* hpd_mutex */ /* hpd_mutex: what DRM reports for the external display, and when */ struct dcp_announce announce; struct delayed_work announce_wq; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 0ba54fe6fb989..bd6b26367ba6f 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -976,13 +976,94 @@ int dcp_get_connector_type(struct platform_device *pdev) #define DPTX_RECONNECT_DELAY msecs_to_jiffies(1000) #define DPTX_RECONNECT_RETRIES 5 +/* + * The connection a DPTX connect or queued reconnect belongs to: the Type-C + * attachment generation and, for a Thunderbolt tunnel, the cookie of the + * binding DCP admitted (zero for direct DP-alt and HDMI). + */ +static struct dcp_fabric_session dcp_session_locked(struct apple_dcp *dcp) +{ + struct dcp_fabric_session session = { .generation = dcp->typec_generation }; + + lockdep_assert_held(&dcp->hpd_mutex); + scoped_guard(mutex, &dcp->tb_lock) + session.cookie = dcp->tb_generation; + return session; +} + +/* This tree has no tb_retiring: a route follow is the retiring handoff. */ +static bool dcp_session_valid_locked(struct apple_dcp *dcp, + struct dcp_fabric_session session, bool cable) +{ + return dcp_fabric_session_valid(session, dcp_session_locked(dcp), + !cable || dcp->typec_cable_connected, + READ_ONCE(dcp->typec_follow_retiring)); +} + +/* + * Recovery must not wake a deliberately parked or disabled output. Without + * a @session (H17P) nothing is checked. + */ +static bool dcp_connect_session_valid_locked(struct apple_dcp *dcp, + const struct dcp_fabric_session *session, + bool cable, bool recovery) +{ + lockdep_assert_held(&dcp->hpd_mutex); + if (!session) + return true; + return (!recovery || (dcp->typec_work_enabled && + !READ_ONCE(dcp->typec_crtc_off) && + !READ_ONCE(dcp->crashed))) && + dcp_session_valid_locked(dcp, *session, cable); +} + +static bool dcp_connect_session_valid(struct apple_dcp *dcp, + const struct dcp_fabric_session *session, + bool cable, bool recovery) +{ + if (!session) + return true; + guard(mutex)(&dcp->hpd_mutex); + return dcp_connect_session_valid_locked(dcp, session, cable, recovery); +} + +/* H17P (J700, T8140, T8142) connects and reconnects as before, without one. */ +static bool dcp_typec_sessionless(struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_H17P || dcp_is_external_h17p(dcp); +} + +/* Capture the session at enqueue, not when a stale worker eventually runs. */ +static void dcp_queue_typec_reconnect_locked(struct apple_dcp *dcp, + unsigned long delay) +{ + lockdep_assert_held(&dcp->hpd_mutex); + /* H17P keeps its cable-checked reconnect. */ + if (!dcp_typec_sessionless(dcp)) { + if (!dcp->typec_cable_connected) + return; + dcp->typec_reconnect_session = dcp_session_locked(dcp); + } + mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, delay); +} + +void dcp_queue_typec_reconnect(struct apple_dcp *dcp, unsigned long delay) +{ + guard(mutex)(&dcp->hpd_mutex); + dcp_queue_typec_reconnect_locked(dcp, delay); +} + /* * Connect DPTX port @port. On failure, @retryable (if given) tells whether * the attempt left nothing outstanding in the firmware, so that a later * connection may simply try again: the service was not published yet, or a * link that never came up was withdrawn and the firmware acknowledged that. + * With a @session, every step checks that it is still the live one, and a + * revoked session returns -ESTALE without touching the new one's state. */ -static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable) +static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable, + const struct dcp_fabric_session *session, bool cable, + bool recovery) { /* T8122 dcpext: HPD only after the link wait, see dcp_ext_dptx_607() */ bool late_hpd = dcp_ext_dptx_607(dcp); @@ -1010,6 +1091,10 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable dcp->connector_type, dcp->dptxport[port].connected); mutex_lock(&dcp->hpd_mutex); + if (!dcp_connect_session_valid_locked(dcp, session, cable, recovery)) { + ret = -ESTALE; + goto out_unlock; + } if (dcp->external && dcpext_scanout_terminal(dcp)) { mutex_unlock(&dcp->hpd_mutex); return -ESHUTDOWN; @@ -1047,6 +1132,10 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable } } + if (!dcp_connect_session_valid_locked(dcp, session, cable, recovery)) { + ret = -ESTALE; + goto out_unlock; + } ret = dptxport_connect(dcp->dptxport[port].service, dfp_port, dcp->dptx_phy, dcp->dptx_die, @@ -1058,6 +1147,10 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable goto out_unlock; } + if (!dcp_connect_session_valid_locked(dcp, session, cable, recovery)) { + ret = -ESTALE; + goto out_unlock; + } ret = dptxport_request_display(dcp->dptxport[port].service); if (ret) { dev_err(dcp->dev, @@ -1065,6 +1158,10 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable ret); goto out_release; } + if (!dcp_connect_session_valid_locked(dcp, session, cable, recovery)) { + ret = -ESTALE; + goto out_release; + } dcp->dptxport[port].connected = true; if (dcp_is_typec_output(dcp) && !late_hpd) { if (dcp_uses_t6020_tunnel_flow(dcp)) @@ -1095,6 +1192,9 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable DPTX_TUNNEL_CONNECT_TIMEOUT : DPTX_CONNECT_TIMEOUT; ret = wait_for_completion_timeout(&dcp->dptxport[port].linkcfg_completion, timeout); + /* A revoked wait must not disconnect or publish readiness for a new session. */ + if (!dcp_connect_session_valid(dcp, session, cable, recovery)) + return -ESTALE; if (!ret && late_hpd) { /* Without HPD this firmware does not train: assert it now. */ dev_dbg(dcp->dev, @@ -1123,6 +1223,8 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable if (ret && (dcp->external || late_hpd)) goto out_withdraw; } + if (!dcp_connect_session_valid(dcp, session, cable, recovery)) + return -ESTALE; if (dcp->external) { /* * The port must still have its display: a Type-C cable, or HPD @@ -1223,7 +1325,7 @@ static int dcp_external_link_locked(struct apple_dcp *dcp) return dcp->external_error; if (!dcp->external_cable) return -ENODEV; - ret = dcp_dptx_connect_raw(dcp, 0, &retryable); + ret = dcp_dptx_connect_raw(dcp, 0, &retryable, NULL, false, false); if (ret && !retryable) dcp->external_error = ret; return ret; @@ -1312,8 +1414,7 @@ void dcp_external_ready(struct apple_dcp *dcp) if (READ_ONCE(dcp->typec_cable_connected)) { guard(mutex)(&dcp->hpd_mutex); dcp->typec_reconnect_tries = 0; - if (dcp->typec_cable_connected) - mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); + dcp_queue_typec_reconnect_locked(dcp, 0); } else if (dcp->hdmi_hpd && dcp->active && gpiod_get_value_cansleep(dcp->hdmi_hpd)) { int ret = dcp_dptx_connect(dcp, 0); @@ -1448,8 +1549,7 @@ void dcp_external_retry_work(struct work_struct *work) dcp->typec_reconnect_tries = 0; } guard(mutex)(&dcp->hpd_mutex); - if (dcp->typec_cable_connected) - mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); + dcp_queue_typec_reconnect_locked(dcp, 0); } /* @@ -1494,7 +1594,7 @@ static int dcp_dptx_connect_tile(struct apple_dcp *dcp) scoped_guard(mutex, &dcp->tb_lock) split = dcp->split.active && dcp->split.ready && - dcp->split.set_active; + dcp->split.generation; if (!split) return 0; ret = dcp_dptx_connect(dcp, 1); @@ -1505,10 +1605,19 @@ static int dcp_dptx_connect_tile(struct apple_dcp *dcp) int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) { + struct dcp_fabric_session session = {}; int ret; if (!dcp_is_external_h17p(dcp)) { - ret = dcp_dptx_connect_raw(dcp, port, NULL); + if (dcp_typec_sessionless(dcp)) { + ret = dcp_dptx_connect_raw(dcp, port, NULL, NULL, false, false); + } else { + scoped_guard(mutex, &dcp->hpd_mutex) + session = dcp_session_locked(dcp); + /* A tunnel's connection also needs its cable; DP-alt/HDMI do not. */ + ret = dcp_dptx_connect_raw(dcp, port, NULL, &session, + !!session.cookie, false); + } if (!ret && !port) ret = dcp_dptx_connect_tile(dcp); return ret; @@ -1775,6 +1884,7 @@ static void dcp_typec_reconnect_work(struct work_struct *work) struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, typec_reconnect_wq); + struct dcp_fabric_session session; int ret; /* A route following its CRTC hands the link over itself. */ @@ -1783,7 +1893,23 @@ static void dcp_typec_reconnect_work(struct work_struct *work) READ_ONCE(dcp->typec_follow_retiring)) return; - ret = dcp_dptx_connect(dcp, 0); + if (dcp_typec_sessionless(dcp)) { + ret = dcp_dptx_connect(dcp, 0); + } else { + /* Only the session this work was queued for may connect. */ + scoped_guard(mutex, &dcp->hpd_mutex) { + session = dcp->typec_reconnect_session; + if (!dcp_session_valid_locked(dcp, session, true)) + return; + } + ret = dcp_dptx_connect_raw(dcp, 0, NULL, &session, true, false); + if (!ret) + ret = dcp_dptx_connect_tile(dcp); + scoped_guard(mutex, &dcp->hpd_mutex) { + if (!dcp_session_valid_locked(dcp, session, true)) + return; + } + } if (!ret) { dcp->typec_reconnect_tries = 0; return; @@ -2060,8 +2186,7 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) ret = dcp_dptx_connect(dcp, port); if (ret && ret != -ESHUTDOWN && dcp_is_typec_output(dcp)) - mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, - DPTX_RECONNECT_DELAY); + dcp_queue_typec_reconnect(dcp, DPTX_RECONNECT_DELAY); return ret; } @@ -2106,12 +2231,15 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) /* * A delivered sink IRQ can follow a firmware unplug with cable HPD still high. - * This tree has no fabric session: the route is revoked by a typec_generation - * bump, and recovery is refused unless the Type-C work is enabled, the cable - * is routed and no follow handoff is retiring the attachment. + * Recovery is refused unless the Type-C work is enabled, the cable is routed + * and no follow handoff is retiring the attachment; the session it revokes + * older waits with is rechecked at every step. */ int dcp_dptx_recover_irq(struct apple_dcp *dcp) { + struct dcp_fabric_session session; + const struct dcp_fabric_session *checked = + dcp_typec_sessionless(dcp) ? NULL : &session; int ret; mutex_lock(&dcp->hpd_mutex); @@ -2131,9 +2259,17 @@ int dcp_dptx_recover_irq(struct apple_dcp *dcp) ret = -EAGAIN; goto out_unlock; } + if (checked) + session = dcp_session_locked(dcp); + if (!dcp_connect_session_valid_locked(dcp, checked, true, true)) { + ret = -ESTALE; + goto out_unlock; + } /* Revoke older reconnect waits without changing physical cable state. */ dcp->typec_generation++; + if (checked) + session = dcp_session_locked(dcp); cancel_delayed_work(&dcp->typec_reconnect_wq); cancel_delayed_work(&dcp->placeholder_edid_wq); if (dcp->avep) @@ -2141,15 +2277,23 @@ int dcp_dptx_recover_irq(struct apple_dcp *dcp) ret = dptxport_set_hpd(dcp->dptxport[0].service, false); if (ret) goto out_unlock; + if (!dcp_connect_session_valid_locked(dcp, checked, true, true)) { + ret = -ESTALE; + goto out_unlock; + } /* A failed earlier connect may also have left an unconfirmed release. */ ret = dptxport_release_display(dcp->dptxport[0].service); if (ret) goto out_unlock; + if (!dcp_connect_session_valid_locked(dcp, checked, true, true)) { + ret = -ESTALE; + goto out_unlock; + } dcp->dptxport[0].connected = false; mutex_unlock(&dcp->hpd_mutex); /* Another delivered IRQ must confirm release before a failed attempt retries. */ - return dcp_dptx_connect_raw(dcp, 0, NULL); + return dcp_dptx_connect_raw(dcp, 0, NULL, checked, true, true); out_unlock: mutex_unlock(&dcp->hpd_mutex); @@ -2558,9 +2702,7 @@ void dcp_poweron(struct platform_device *pdev) WRITE_ONCE(dcp->typec_poweron_hpd, true); ret = dcp_dptx_connect(dcp, 0); if (ret) - mod_delayed_work(system_freezable_wq, - &dcp->typec_reconnect_wq, - DPTX_RECONNECT_DELAY); + dcp_queue_typec_reconnect(dcp, DPTX_RECONNECT_DELAY); else if (wait_for_typec_hpd) { remaining = wait_for_completion_timeout( &dcp->typec_iomfb_hpd_ready, @@ -3326,7 +3468,7 @@ VISIBLE_IF_KUNIT void dcp_enable_typec_work(struct apple_dcp *dcp) dcp->typec_work_enabled = true; /* A cable can be routed before the DRM component binds. */ if (READ_ONCE(dcp->typec_cable_connected)) - mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); + dcp_queue_typec_reconnect(dcp, 0); if (dcp_is_external_h17p(dcp)) { struct apple_connector *con = READ_ONCE(dcp->connector); diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index bb87662893c19..baff04ee201e7 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -102,6 +102,7 @@ void dcp_external_retry(struct apple_dcp *dcp, const char *why, int error, unsigned int base_ms); void dcp_external_retry_work(struct work_struct *work); void dcp_external_sink_irq(struct apple_dcp *dcp); +void dcp_queue_typec_reconnect(struct apple_dcp *dcp, unsigned long delay); int dcp_dptx_recover_irq(struct apple_dcp *dcp); int dcp_dptx_park(struct apple_dcp *dcp); int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); From 25be3dedaed1b7d1d97182f4369d545ab5d61572 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 18 Oct 2026 03:58:04 +1100 Subject: [PATCH 1685/2680] selftests: apple-dcp: Follow catalogue ownership and Type-C route follow The ported catalogue and attachment model (modes_lock, modes_admitted, dcp_modes_replace(), dcp_attributes_replace()), the connector EDID helpers, the Type-C route-follow step, the T6030/T8122 external processor paths, the J613 26.6 backend and the 14.7 endpoint rules (dcp_fw_is_14_7()) and the Thunderbolt reconnect sessions (dcp_dptx_connect_raw() with a session, dcp_queue_typec_reconnect()) changed the production code that the host units extract. Find extracted functions in whichever drivers/gpu/drm/apple file holds them, compile the real catalogue, connector and 14.7 helpers into the units, pass the new lookup_mode() output argument and DPTX unit, give the mocks the new members, and stub the T6030, T8122 and 26.6 paths so that reaching one fails the unit. No scenario assertion is removed or relaxed. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/gpu/apple-dcp/common.py | 29 ++++++ .../selftests/drivers/gpu/apple-dcp/edid.py | 72 +++++++++++++-- .../drivers/gpu/apple-dcp/lifecycle.py | 92 ++++++++++++++++--- .../drivers/gpu/apple-dcp/quiesce.py | 4 +- .../selftests/drivers/gpu/apple-dcp/rate.py | 62 ++++++++++--- .../drivers/gpu/apple-dcp/recovery.py | 4 +- .../drivers/gpu/apple-dcp/transaction.py | 40 ++++++-- 7 files changed, 261 insertions(+), 42 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py index 0cd119bfad4b8..ed3fdb8b4eaa1 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py @@ -163,3 +163,32 @@ def fabric_rebalance(dcp): end += len(block(dcp, end)) start = dcp.find('struct dcp_rebalance_context {') return dcp[start if 0 <= start < end else dcp.index(marker):end] + '\n' + + +def apple_function(root, name): + """The definition of @name in whichever drivers/gpu/drm/apple source has it. + + Robust to functions moving between files; '' when no source defines it. + """ + pattern = re.compile(r'^(?:[A-Za-z_][\w \t*]*\n)?(?:[A-Za-z_][\w \t*]*?\b)?' + re.escape(name) + + r'\([^;{]*?\)\s*\{', re.M) + for path in sorted((root / 'drivers/gpu/drm/apple').glob('*.c')): + if 'test' in path.name: + continue + text = _read_text(path) + match = pattern.search(text) + if match: + return block(text, match.start()) + '\n' + return '' + + +def internal_inline(root, name): + """A static inline helper from dcp-internal.h, marked unused; '' if absent. + + dcp_fw_is_14_7() reads fw_compat and hw.fw_14_7_endpoints. + """ + text = (root / 'drivers/gpu/drm/apple/dcp-internal.h').read_text() + match = re.search(r'^static inline [^\n;]*\b' + re.escape(name) + r'\(', text, re.M) + if not match: + return '' + return block(text, match.start()).replace('static inline ', 'static inline __attribute__((unused)) ', 1) + '\n' diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py index a3fa8db014b05..163850a1899ce 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py @@ -2,7 +2,7 @@ # Copyright 2026 Ryan Murray """Actual native EDID admission, service epochs, stale replies and RX progress.""" from pathlib import Path -from common import block, source_defines +from common import block, source_defines, apple_function, internal_inline def rebind(root): base = root / 'drivers/gpu/drm/apple' @@ -92,10 +92,10 @@ def function(file,signature): struct apple_dcp { void *dev;int fw_compat;struct apple_dcp_afkep *dcpavservep;struct apple_connector *connector; bool external_physically_absent,hpd_mutex,quiescing,typec_cable_connected;u64 typec_generation;unsigned int nr_modes; - struct { int iomfb_method_profile; } hw; - struct { bool lock,enabled;u64 epoch;int enable_completion;struct apple_epic_service *service; } dcpavserv; + struct { int iomfb_method_profile; bool fw_14_7_endpoints; } hw; bool ext_dead; + struct { bool lock,enabled;u64 epoch;int enable_completion;struct apple_epic_service *service,*opened; } dcpavserv; }; -struct epic_cmd { u32 retcode;u64 rxbuf,txbuf;u32 rxlen,txlen; }; +'''+internal_inline(root,'dcp_fw_is_14_7')+r'''struct epic_cmd { u32 retcode;u64 rxbuf,txbuf;u32 rxlen,txlen; }; static unsigned int dma_frees,opens; static int open_error,open_action; @@ -593,6 +593,8 @@ def function(file, signature): #define trace_dcp_recv_msg(...) ((void)0) #define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 #define DCP_FIRMWARE_H17P 2 +/* The 14.7 open-before-EDID handshake is for T6030 external processors only. */ +#define DCP_FIRMWARE_V_14_7 (DCP_FIRMWARE_H17P + 1) #define AV_ENDPOINT 0x29 #define SYSTEM_ENDPOINT 0x20 #define DISP0_ENDPOINT 0x23 @@ -608,9 +610,9 @@ def function(file, signature): struct apple_epic_service { struct apple_dcp_afkep *ep; unsigned int users; bool enabled,torndown; struct { u64 generation,epoch; } admission; void *cookie; unsigned int channel; }; struct apple_dcp { struct apple_dcp_afkep *dcpavservep, *dptxep, *avep, *systemep, *ibootep, *dptx_controller_ep; - struct { bool lock, enabled; u64 epoch; int enable_completion; struct apple_epic_service *service; } dcpavserv; + struct { bool lock, enabled; u64 epoch; int enable_completion; struct apple_epic_service *service, *opened; } dcpavserv; struct apple_connector *connector; unsigned int nr_modes; - struct { int iomfb_method_profile; } hw; + struct { int iomfb_method_profile; bool fw_14_7_endpoints; } hw; bool ext_dead; bool external_physically_absent,hpd_mutex, typec_cable_connected, quiescing; u64 typec_generation; int fw_compat; @@ -639,6 +641,8 @@ def function(file, signature): } static const struct drm_edid *dcpavserv_read_edid_legacy(struct apple_epic_service *s) { legacy_reads++; return dcpavserv_read_edid_inline(s); } +static int __attribute__((unused)) afk_service_call(struct apple_epic_service *s,unsigned short group,unsigned command,const void *data,size_t data_len,size_t data_pad,void *output,size_t output_len,size_t output_pad) +{ (void)s;(void)group;(void)command;(void)data;(void)data_len;(void)data_pad;(void)output;(void)output_len;(void)output_pad;assert(!"14.7 open-before-EDID on H17P");return -ENODEV; } static void iomfb_recv_msg(struct apple_dcp *d, u64 message) { (void)d; (void)message; } static void afk_receive_message(struct apple_dcp_afkep *ep, u64 message) { (void)message; received_ep=ep; receives++; } @@ -753,7 +757,11 @@ def function(text, signature): template=source('drivers/gpu/drm/apple/iomfb_template.c') assert 'alloc_ordered_workqueue("rtkit-%s"' in rtkit assert '.recv_message = dcp_recv_msg' in dcp - assert 'dcp_set_dimensions(dcp);' in template[template.index('"DisplayAttributes"'):] + attributes=template[template.index('"DisplayAttributes"'):] + # Catalogue ownership: DisplayAttributes records the dimensions for the + # transfer's attachment and defers the DRM update to dimensions_wq. + owned='dcp_attributes_replace(' in attributes[:attributes.index('}')] + assert ('dcp_attributes_replace(dcp, &ctx,' if owned else 'dcp_set_dimensions(dcp);') in attributes assert 'queue_work(ep->wq, &work->work)' in af prefix=r''' #undef NDEBUG @@ -815,7 +823,7 @@ def function(text, signature): struct test_mode_ticket { u64 generation; bool associated; }; struct apple_dcp { bool quiescing; int fw_compat; void *dev; - struct { int iomfb_method_profile; } hw; + struct { int iomfb_method_profile; bool fw_14_7_endpoints; } hw; bool ext_dead; struct { struct mutex lock; u64 epoch; bool enabled; void *service; } dcpavserv; struct test_mode_ticket dummy_ticket; struct { u64 connection_sequence; } rate_proof; @@ -938,6 +946,52 @@ def function(text, signature): puts("PASS: actual ordered RTKit RX -> synchronous IOMFB dimensions/property-start callbacks -> AFK enqueue makes reply progress while DRM mutex held; deferred dimensions update completes after release"); } ''' + if owned: + prefix=prefix.replace('struct apple_connector { struct drm_connector base; struct work_struct hotplug_wq; };', + 'struct apple_connector { struct drm_connector base; struct work_struct hotplug_wq,bl_sync_wq; };',1) + prefix=prefix.replace(' int width_mm,height_mm; struct { int width_mm,height_mm; } panel;', + ' int width_mm,height_mm; struct { int width_mm,height_mm; } panel;\n' + ' struct mutex modes_lock; u64 modes_generation,dimensions_generation; struct work_struct dimensions_wq; bool ext_backlight;',1) + prefix=prefix.replace('struct { size_t length; void *data;','struct { size_t length; void *data; u64 modes_generation;',1) + prefix=prefix.replace('static void schedule_work(struct work_struct *w) { queue_work(&dimensions_queue,w); }',''' +static struct workqueue_struct hotplug_queue={.ready=PTHREAD_COND_INITIALIZER,.lock=PTHREAD_MUTEX_INITIALIZER}; +static void (*hotplug_fn)(struct work_struct *); +/* dimensions_wq and the connector's hotplug work run on separate workers. */ +static void schedule_work(struct work_struct *w) { queue_work(w->fn==hotplug_fn?&hotplug_queue:&dimensions_queue,w); } +static struct mutex *guard_lock(struct mutex *m) { mutex_lock(m); return m; } +static void guard_unlock(struct mutex **m) { if(*m) mutex_unlock(*m); } +#define guard(kind) struct mutex * __attribute__((cleanup(guard_unlock))) _guard = guard_lock +#define scoped_guard(kind,l) for(struct mutex *_scoped __attribute__((cleanup(guard_unlock)))=guard_lock(l);_scoped;mutex_unlock(_scoped),_scoped=NULL) +#define DCP_FIRMWARE_V_14_7 (DCP_FIRMWARE_H17P + 1) +#define DCP_FIRMWARE_V_14_7_TEMPLATE (DCP_FIRMWARE_H17P + 2) +static bool dcp_has_panel(struct apple_dcp *d) { (void)d; return false; } +/* The parser is a leaf here: the attributes report the display's 60x34 cm. */ +struct dcp_parse_ctx { int unused; }; +static int parse_display_attributes(struct dcp_parse_ctx *h,int *w,int *ht,bool *backlight) +{ (void)h; *w=600; *ht=340; *backlight=false; return 0; } +static int dcp_attributes_replace(struct apple_dcp *d,struct dcp_parse_ctx *h,u64 generation); +static u64 dcp_modes_transfer_begin(struct apple_dcp *d);''',1) + old_rx=''' assert(message==1 && held(&d->iomfb.lock)); dcp_set_dimensions(d);''' + assert old_rx in prefix + prefix=prefix.replace(old_rx,''' struct dcp_parse_ctx attributes={0}; + assert(message==1 && held(&d->iomfb.lock)); + assert(!dcp_attributes_replace(d,&attributes,dcp_modes_transfer_begin(d)));''',1) + prefix=prefix.replace('static void dcp_set_dimensions(struct apple_dcp *d);','static void dcp_set_dimensions(struct apple_dcp *d,u64 generation);',1) + prefix=prefix.replace('static u64 dcp_modes_transfer_begin(struct apple_dcp *d);','static u64 dcp_modes_transfer_begin(struct apple_dcp *d);\n'+internal_inline(root,'dcp_fw_is_14_7'),1) + property_header=(root/'drivers/gpu/drm/apple/dcp-property.h').read_text() + parts=['#include \n'+''.join(line+'\n' for line in property_header.splitlines() if not line.startswith('#include')) + .replace('static inline ','static inline __attribute__((unused)) ').replace('SZ_16M','(16*SZ_1M)')]+parts + parts+=[apple_function(root,name) for name in ('dcp_modes_transfer_begin','dcp_modes_dimensions_work','dcp_attributes_replace')] + old_main=''' d.dcpavservep=&av; connector.hotplug_wq.fn=dimension_work;''' + assert old_main in suffix + suffix=suffix.replace(old_main,''' pthread_t hotplug; + d.dcpavservep=&av; connector.hotplug_wq.fn=dimension_work; hotplug_fn=dimension_work; + d.dimensions_wq.fn=dcp_modes_dimensions_work; + assert(!pthread_mutex_init(&d.modes_lock.native,NULL)); + assert(!pthread_create(&hotplug,NULL,worker,&hotplug_queue));''',1) + old_join=''' assert(!pthread_join(rx,NULL) && !pthread_join(afk,NULL) && !pthread_join(dimensions,NULL));''' + assert old_join in suffix + suffix=suffix.replace(old_join,old_join+'\n assert(!pthread_join(hotplug,NULL));',1) return prefix + '\n'.join(parts) + suffix @@ -965,7 +1019,7 @@ def startup(root): }; struct apple_dcp { void *dev; - struct { int iomfb_method_profile; } hw; + struct { int iomfb_method_profile; bool fw_14_7_endpoints; } hw; bool ext_dead; struct { unsigned enable_completion; bool enabled; } dcpavserv; struct apple_dcp_afkep *dcpavservep; }; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 279718cbe2ebe..ba3c32e54be57 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -3,7 +3,7 @@ """Actual disconnect authority, DWC3 teardown and firmware resource admission.""" from pathlib import Path import re -from common import block, atc_mux_without_pm, source_defines, ATC_POST_TUNABLE, tipd_pm_header +from common import block, apple_function, internal_inline, atc_mux_without_pm, source_defines, ATC_POST_TUNABLE, tipd_pm_header def disconnect(root): # Copyright 2026 Ryan Murray @@ -132,7 +132,10 @@ def function(file, name): u64 external_stop_generation, typec_generation; int external_error, mode_state; void *avep; struct dcp_announce announce; + /* T6030 external processors only; an H17P display is never one. */ + bool external_native; int external_retries; struct delayed_work external_retry_wq; }; +#define atomic_set(p, v) (*(p) = (v)) struct platform_device { struct apple_dcp *data; }; static struct apple_dcp *platform_get_drvdata(struct platform_device *p) { return p->data; } static int raw_error, raw_links; @@ -406,9 +409,27 @@ def function(file, name): shim = shim.replace(old, 'static int cd321x_typec_update_mode(struct tps6598x *t, struct cd321x_status *s, bool *busy,\n' ' bool *held)\n' '{ (void)t; (void)s; *busy = mode_error == -EBUSY; *held = false; return mode_error; }') + # Reconnect sessions (TB binding cookies/generations): the raw connect + # checks a session, and the Type-C retry goes through + # dcp_queue_typec_reconnect(). An H17P external link is sessionless. + if SESSION_RAW.search((r / dcp).read_text()): + old = ('static int dcp_dptx_connect_raw(struct apple_dcp *d, u32 port, bool *waiting)\n' + '{ assert(port == 0); *waiting = false; raw_links++;') + assert prefix.count(old) == 1 + session = (r / 'drivers/gpu/drm/apple/dcp-fabric-session.h').read_text() + prefix = prefix.replace(old, block(session, 'struct dcp_fabric_session {') + ';\n' + 'static int dcp_dptx_connect_raw(struct apple_dcp *d, u32 port, bool *waiting,\n' + ' const struct dcp_fabric_session *session, bool cable, bool recovery)\n' + '{ assert(port == 0); assert(!session && !cable && !recovery); *waiting = false; raw_links++;') + prefix += ('/* Only a Type-C (non-H17P) output queues the session reconnect. */\n' + 'static void dcp_queue_typec_reconnect(struct apple_dcp *d, unsigned long delay)\n' + '{ (void)d; (void)delay; assert(!"Type-C session reconnect for H17P"); }\n') source = prefix + pm + shim + dp + funcs + tests return source + +SESSION_RAW = re.compile(r'dcp_dptx_connect_raw\([^)]*const struct dcp_fabric_session \*') + def bounce(root): """Fast dock replug: the sink vanishes while the DPTX link trains. @@ -428,13 +449,17 @@ def swap(old, new): unit = unit.replace(old, new) swap(' struct { bool connected; void *service; } dptxport[1];', - ' struct { bool connected, closed; void *service, *atcphy; int linkcfg_completion; } dptxport[1];\n' + ' struct { bool connected, closed, tile_hint; void *service, *atcphy; int linkcfg_completion; } dptxport[1];\n' ' void *phy, *dev, *active_typec_route;\n' ' int dptx_die, dptx_phy, dptx_dfp_port, connector_type;\n' - ' bool external, external_link_ready, crashed;') + ' bool external, external_link_ready, crashed;\n' + ' bool typec_work_enabled; int fw_compat; struct { bool fw_14_7_endpoints; } hw;\n' + ' struct { bool active; } split;') old_raw = unit[unit.index('static int dcp_dptx_connect_raw('):] old_raw = old_raw[:old_raw.index('return raw_error; }') + len('return raw_error; }')] - swap(old_raw, 'static int dcp_dptx_connect_raw(struct apple_dcp *, u32, bool *);') + sessions = bool(SESSION_RAW.search(dcp)) + swap(old_raw, 'static int dcp_dptx_connect_raw(struct apple_dcp *, u32, bool *' + + (', const struct dcp_fabric_session *, bool, bool' if sessions else '') + ');') swap('static int dptxport_set_hpd(void *p, bool on)\n{ (void)p; assert(!on); return hpd_error; }', 'static bool fw_hpd, fw_display, link_trains;\n' 'static int hpd_drops, connects, release_error_once, assert_error;\n' @@ -480,6 +505,8 @@ def swap(old, new): #define smp_store_release(p, v) (*(p) = (v)) static bool dcpext_scanout_terminal(struct apple_dcp *d) { (void)d; return false; } static void dcpext_scanout_link_restored(struct apple_dcp *d) { (void)d; assert(!"T6030 dcpext path in an H17P scenario"); } +static bool iomfb_v14_7_external_ready(struct apple_dcp *d) { (void)d; assert(!"T6030 external processor in an H17P scenario"); return false; } +static int iomfb_v14_7_external_open(struct apple_dcp *d) { (void)d; assert(!"T6030 external processor in an H17P scenario"); return -ENODEV; } ''' tests = r''' static struct platform_device device = { .data = &display }; @@ -583,6 +610,18 @@ def swap(old, new): printf("%d assertions, %d failures\n", checks, failures); return !!failures; } ''' + if sessions: + # Not a T8122 dcpext (fw_compat UNKNOWN); the session checks are real. + firmware += ('#define DCP_FIRMWARE_V_14_7_TEMPLATE 4\n' + internal_inline(root, 'dcp_ext_dptx_607') + + 'struct dcp_fabric_session;\n' + 'static bool dcp_session_valid_locked(struct apple_dcp *d, struct dcp_fabric_session s, bool cable);\n' + 'static bool __attribute__((unused)) dcp_hdmi_port_attached(struct apple_dcp *d)\n' + '{ (void)d; assert(!"T6030 external HDMI port in an H17P scenario"); return false; }\n' + + apple_function(root, 'dcp_connect_session_valid_locked') + + apple_function(root, 'dcp_connect_session_valid') + + '/* Never reached: the H17P external link passes no session. */\n' + 'static bool dcp_session_valid_locked(struct apple_dcp *d, struct dcp_fabric_session s, bool cable)\n' + '{ (void)d; (void)s; (void)cable; assert(!"session check for a sessionless link"); return false; }\n') return unit + firmware + raw + '\n' + tests def dropped(root): @@ -1126,16 +1165,22 @@ def function(source, marker): # This slice is the production probe's publication boundary, including its # failure exits and prepared-before-component_add ordering, not a copied model. - probe = function(dcp, 'static int dcp_platform_probe(') + # The T8122 dcpext wrapper moved the pipe's probe into dcp_platform_probe_pipe(). + probe = function(dcp, 'static int dcp_platform_probe_pipe(' if 'static int dcp_platform_probe_pipe(' in dcp + else 'static int dcp_platform_probe(') publish = probe[probe.rindex('\tif (dcp->hw.iomfb_method_profile ==\n\t DCP_IOMFB_METHODS_H17P_EXTERNAL) {'):] production = '\n'.join(( function(dcp, 'bool dcp_h17p_external_supported('), function(dcp, 'bool dcp_h17p_external_ready('), + # A failed T8122 dcpext stays bound without a pipeline; none fails here. + apple_function(root, 'dcp_ext_unusable'), function(drm, 'static bool apple_dcp_usable('), function(drm, 'static int add_dcp_components('), function(drm, 'static int apple_drm_init_dcp('), 'static int publish_provider(struct platform_device *pdev, struct apple_dcp *dcp)\n' - '{\n\tstruct device *dev = &pdev->dev;\n\tint ret;\n' + publish, + '{\n\tstruct device *dev = &pdev->dev;\n\tint ret;\n' + # Type-C route registration, earlier in probe, succeeded here. + + ('\tbool routes_failed = false;\n' if 'routes_failed' in publish else '') + publish, )) prefix = r''' #include @@ -1161,14 +1206,25 @@ def function(source, marker): struct device { struct device_node *of_node; void *data; }; struct platform_device { struct device dev; bool bound, registered; }; /* The native T6030 14.7 backend is never selected in these scenarios. */ -enum dcp_firmware_version { DCP_FIRMWARE_UNKNOWN, DCP_FIRMWARE_V_14_7, DCP_FIRMWARE_V_14_7_TEMPLATE }; +enum dcp_firmware_version { DCP_FIRMWARE_UNKNOWN, DCP_FIRMWARE_V_14_7, DCP_FIRMWARE_V_14_7_TEMPLATE, DCP_FIRMWARE_V_26_6 }; struct apple_dcp { - bool external_prepared, quiescing, crashed; + bool external_prepared, quiescing, crashed, external_native, ext_dead; enum dcp_firmware_version fw_compat; struct { int iomfb_method_profile; } hw; }; static void __attribute__((unused)) iomfb_v14_7_remove(struct apple_dcp *d) { (void)d; assert(!"14.7 backend on a non-14.7 DCP"); } +/* Nor the J613 26.6 backend. */ +static void __attribute__((unused)) iomfb_v26_6_unbind(struct apple_dcp *d) +{ (void)d; assert(!"26.6 backend on a non-26.6 DCP"); } +/* No node here is a T6030 dcpext, so no native external processor. */ +struct device_node; +static bool __attribute__((unused)) dcp_t6030_ext_native(const struct device_node *n) +{ (void)n; assert(!"T6030 dcpext in a J700 scenario"); return false; } +static int __attribute__((unused)) iomfb_v14_7_external_prepare(struct apple_dcp *d) +{ (void)d; assert(!"T6030 external processor in a J700 scenario"); return -ENODEV; } +static void __attribute__((unused)) iomfb_v14_7_external_refuse(struct apple_dcp *d, const char *why) +{ (void)d; (void)why; assert(!"T6030 external processor in a J700 scenario"); } struct drm_device { int unused; }; struct drm_encoder { int unused; }; struct apple_drm_private { struct drm_device drm; }; @@ -1398,9 +1454,11 @@ def vrr(root): #define dev_dbg(...) ((void)0) struct apple_connector { bool connected; }; /* The native T6030 14.7 backend is never selected in these scenarios. */ -enum dcp_firmware_version { DCP_FIRMWARE_UNKNOWN, DCP_FIRMWARE_V_14_7, DCP_FIRMWARE_V_14_7_TEMPLATE }; +enum dcp_firmware_version { DCP_FIRMWARE_UNKNOWN, DCP_FIRMWARE_V_12_3, DCP_FIRMWARE_V_13_5, + DCP_FIRMWARE_V_14_7, DCP_FIRMWARE_V_14_7_TEMPLATE, DCP_FIRMWARE_V_26_6 }; struct apple_dcp { - bool crashed; + bool crashed, external, external_native; + unsigned int nr_typec_routes; enum dcp_firmware_version fw_compat; struct { int iomfb_method_profile; } hw; struct { bool valid; } mode_state; @@ -1409,7 +1467,8 @@ def vrr(root): }; struct platform_device { struct apple_dcp *data; }; struct drm_crtc { struct platform_device *dcp; }; -struct drm_crtc_state { bool vrr_enabled, modeset; }; +struct drm_display_mode { int unused; }; +struct drm_crtc_state { bool enable, vrr_enabled, modeset; struct drm_display_mode mode; }; /* mainline 7.2 renamed struct drm_atomic_state to struct drm_atomic_commit */ struct drm_atomic_commit { struct drm_crtc_state crtc; }; #define to_apple_crtc(x) (x) @@ -1422,7 +1481,15 @@ def vrr(root): static int __attribute__((unused)) iomfb_v14_7_atomic_check(struct apple_dcp *d, struct drm_crtc *c, struct drm_atomic_commit *s) { (void)d; (void)c; (void)s; assert(!"14.7 backend on a non-14.7 DCP"); return -EINVAL; } -""" +static int __attribute__((unused)) iomfb_v26_6_atomic_check(struct apple_dcp *d, + struct drm_crtc *c, struct drm_atomic_commit *s) +{ (void)d; (void)c; (void)s; assert(!"26.6 backend on a non-26.6 DCP"); return -EINVAL; } +/* Route-follow is for Type-C displays; this DCP has no Type-C routes. */ +static bool __attribute__((unused)) typec_follow_crtc = true; +static int __attribute__((unused)) dcp_typec_follow_check(struct apple_dcp *d, struct drm_crtc *c, + struct drm_atomic_commit *s, const struct drm_display_mode *m) +{ (void)d; (void)c; (void)s; (void)m; assert(!"route-follow without Type-C routes"); return -EINVAL; } +""" + apple_function(root, 'dcp_typec_follows_crtc') suffix = r""" int main(void) { @@ -1441,6 +1508,7 @@ def vrr(root): dcp.hw.iomfb_method_profile = profile; dcp.mode_state.valid = valid; state.crtc.modeset = modeset; + state.crtc.enable = true; state.crtc.vrr_enabled = vrr; dcp.connector = link == 2 ? NULL : &connector; connector.connected = link == 1; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py index 700bf2266ace1..87323e84e8b1a 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py @@ -58,7 +58,7 @@ def make_unit(source, parent=False, reference=None): struct connector { bool connected; }; struct apple_dcp { int external_lock,external_error; bool external_cable,external_stopped; struct { bool connected; } dptxport[1]; void *hdmi_hpd; bool active; bool external,typec_cable_connected; void *dev; struct connector *typec_connector; bool ext_backlight; }; struct apple_dcp_typec_route; -struct apple_dcp_typec_port { int link; void *connector_np; struct apple_dcp_typec_route *owner,*target,*secondary_target,*preferred_route; void *applied_alt; unsigned long applied_mode; unsigned applied_status,applied_conf; bool applied_valid,safe_pending,safe_finish_ready,dp_wanted,dp_hpd,hpd; }; +struct apple_dcp_typec_port { int link; void *connector_np; struct apple_dcp_typec_route *owner,*target,*secondary_target,*preferred_route; void *applied_alt; unsigned long applied_mode; unsigned applied_status,applied_conf; bool applied_valid,safe_pending,safe_finish_ready,dp_wanted,dp_hpd,hpd,busy_noted; }; struct apple_dcp_typec_route { struct apple_dcp_typec_port *port; struct apple_dcp *dcp; void *xbar; bool deactivate_xbar_done; int deactivate_error; bool selected,tunnel; }; static int dcp_typec_fabric_lock,dcp_typec_ports,deactivate_calls; struct drm_device { int unused; }; @@ -284,7 +284,7 @@ def make_unit(source, parent=False, reference=None): # The fabric core split: rebalancing runs the real dcp_fabric_run_rebalance() # and dcp_fabric_run_deactivate() over the real decisions, and the adapter # hands the core each route's struct dcp_fabric_route. - prefix=prefix.replace('typedef unsigned u32;\n','typedef unsigned u32;\ntypedef unsigned long long u64;\n'+fabric,1) + prefix=prefix.replace('typedef unsigned u32;\n','typedef unsigned u32;\ntypedef unsigned long long u64;\ntypedef unsigned char u8;\n'+fabric,1) prefix=prefix.replace('struct apple_dcp_typec_route { ','struct apple_dcp_typec_route { struct dcp_fabric_route core; ',1) prefix=prefix.replace('#define lockdep_assert_held(p) ((void)(p))\n','''#define lockdep_assert_held(p) ((void)(p)) #define list_first_entry_or_null(h,t,m) ((void)(h),sole_port()) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py index bda5b2bbda60a..b69a0bd01a4bd 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py @@ -2,8 +2,8 @@ # Copyright 2026 Ryan Murray """Actual parser, bounded rate admission, catalog replacement and HPD.""" from pathlib import Path -import copy, struct -from common import block, source_defines +import copy, re, struct +from common import block, source_defines, apple_function, internal_inline def parser_fixture(root): @@ -386,7 +386,7 @@ def case(value, automatic=True, internal=False, external=True, admitted=True): struct apple_epic_service { struct apple_epic *ep; void *cookie; bool enabled; }; struct dptx_port { u32 lane_count, link_rate; bool enabled, closed, connected; void *atcphy; struct apple_epic_service *service; }; struct apple_dcp { - struct { int iomfb_method_profile; } hw; + struct { int iomfb_method_profile; bool fw_14_7_endpoints; } hw; struct { bool lock, attributes_valid, timing_valid, hpd_seen; struct dcp_metadata_epoch epoch; struct dcp_metadata_ticket catalog_ticket; u64 catalog_connection_sequence; } metadata; struct { bool lock, enabled; void *service; u64 epoch; } dcpavserv; @@ -559,11 +559,16 @@ def publication(root): state = state.replace('struct apple_epic_service { struct apple_epic *ep; void *cookie; bool enabled; };', '''struct apple_epic_service { struct apple_epic *ep; void *cookie; bool enabled,torndown; unsigned users,channel; struct { u64 generation,epoch; } admission; };''') - state = state.replace('struct { bool lock, enabled; void *service; u64 epoch; } dcpavserv;', 'struct { bool lock, enabled; void *service; u64 epoch; int enable_completion; } dcpavserv;') + state = state.replace('struct { bool lock, enabled; void *service; u64 epoch; } dcpavserv;', 'struct { bool lock, enabled; void *service, *opened; u64 epoch; int enable_completion; } dcpavserv;') + # Catalogue/EDID ownership: modes are admitted per attachment under modes_lock. + state = state.replace(' struct work_struct hotplug_wq, bl_sync_wq;\n};', ' struct work_struct hotplug_wq, bl_sync_wq;\n bool edid_live; unsigned hotplug_reasons;\n};', 1) state = state.replace('u64 typec_generation; unsigned int nr_modes;', '''u64 typec_generation; unsigned int nr_modes; struct apple_epic *dcpavservep; int fw_compat,width_mm,height_mm,fixed_connector_type; unsigned notch_height; - struct { size_t length,received; void *data; bool contiguous,payload_complete; + bool modes_lock,modes_admitted,modes_provisional,placeholder_retried; int external_held; + u64 modes_generation,dimensions_generation; struct work_struct dimensions_wq; + struct { int width_mm,height_mm; } panel; + struct { size_t length,received; void *data; bool contiguous,payload_complete; u64 modes_generation; u64 av_service_epoch,automatic_connection_sequence; bool automatic_service_pending; struct dcp_metadata_ticket metadata_ticket,mode_ticket; } chunks;''') @@ -645,8 +650,25 @@ def publication(root): } static void dcp_firmware_metadata_timing(struct apple_dcp *d) { (void)d; } static void dcp_firmware_metadata_attributes(struct apple_dcp *d,const struct dcp_display_metadata *m) { (void)d; (void)m; } -static void dcp_set_dimensions(struct apple_dcp *d) { (void)d; } +/* Deferred to dimensions_wq since the catalogue ownership change. */ +static void __attribute__((unused)) dcp_set_dimensions(struct apple_dcp *d) { (void)d; } struct dcp_set_dcpav_prop_end_req { char key[64]; }; +static void rate_scoped_unlock(struct rate_guard *g) { if(g->lock) rate_unlock(g); } +#define scoped_guard(kind,l) for(struct rate_guard _scoped __attribute__((cleanup(rate_scoped_unlock)))=rate_lock(l);_scoped.lock;rate_unlock(&_scoped),_scoped.lock=NULL) +#define atomic_read(p) (*(p)) +#define atomic_or(v,p) (*(p)|=(v)) +#define SZ_16M (16*1024*1024) +#ifndef BIT +#define BIT(n) (1UL << (n)) +#endif +static void *kmemdup_array(const void *src,size_t n,size_t size,int gfp) { (void)gfp;void *p=malloc(n*size);if(p)memcpy(p,src,n*size);return p; } +static void disable_work_sync(struct work_struct *w) { (void)w; } +#ifndef DCP_FIRMWARE_V_14_7 +#define DCP_FIRMWARE_V_14_7 (DCP_FIRMWARE_H17P + 1) +#define DCP_FIRMWARE_V_14_7_TEMPLATE (DCP_FIRMWARE_H17P + 2) +#endif +'''+internal_inline(root,'dcp_fw_is_14_7')+r'''/* Off T6030 machines the direct crossbar output is not clocked separately. */ +static int __attribute__((unused)) dcp_direct_crossbar_link(struct apple_dcp *d,bool up) { (void)d;(void)up;return 0; } ''' transport = '\n'.join(block(av,marker) for marker in [ 'static void dcpavserv_notify_edid(', 'static bool dcpavserv_identity(', @@ -656,7 +678,17 @@ def publication(root): 'bool dcpavserv_edid_current(', 'bool dcpavserv_edid_service_current(', ]) mode_helpers = iomfb[iomfb.index('static struct dcp_display_mode *dcp_modes_snapshot('):iomfb.index('int dcp_get_modes(')] - publication = block(iomfb,'static void dcp_clear_connector_edid(')+'\n'+block(iomfb,'static void dcp_consume_edid_invalidation(')+mode_helpers+block(iomfb,'int dcp_get_modes(')+block(iomfb,'struct dcp_display_mode *lookup_mode(') + # The catalogue owner check, hotplug queueing, connector EDID helpers and + # property bounds the catalogue helpers use, all actual. + property_header = base/'dcp-property.h' + reasons = re.search(r'^enum dcp_hotplug_reason \{', iomfb, re.M) + mode_helpers = (''.join(line+'\n' for line in (property_header.read_text().splitlines() if property_header.exists() else ()) + if not line.startswith('#include')).replace('static inline ', 'static inline __attribute__((unused)) ') + + (block(iomfb, reasons.start())+';\n' if reasons else '') + + ''.join(apple_function(root, name) for name in ('dcp_queue_hotplug', 'dcp_modes_for_connector', + 'apple_connector_invalidate_edid', 'apple_connector_edid_set_live')) + + mode_helpers) + publication = block(iomfb,'static void dcp_clear_connector_edid(')+'\n'+block(iomfb,'static void dcp_consume_edid_invalidation(')+mode_helpers+block(iomfb,'int dcp_get_modes(')+block(iomfb,re.search(r'^[^\n;]*\blookup_mode\(',iomfb,re.M).start()) # Drive malformed replacements through the same chunk/end ingress as firmware. # The dictionary cache is a boundary substitute and takes ownership of the blob. receipt = r''' @@ -679,7 +711,8 @@ def publication(root): assert(!raw_allocations); d=(struct apple_dcp){.dev=&d,.phy=&transport_edid,.active_typec_route=&con,.typec_cable_connected=true,.typec_generation=7,.fw_compat=2,.width_mm=600,.height_mm=330}; drm=(struct drm_device){}; - con=(struct apple_connector){.dcp=&d,.connected=true,.base={.dev=&drm,.registration_state=1}}; + /* Bound to its pipeline, so its EDID is live (apple_connector_set_pipeline()). */ + con=(struct apple_connector){.dcp=&d,.connected=true,.edid_live=true,.base={.dev=&drm,.registration_state=1}}; ep=(struct apple_epic){.dcp=&d,.endpoint=0x2c,.wire_v2=true}; av_ep=(struct apple_epic){.dcp=&d,.endpoint=0x2a,.wire_v2=true}; s=(struct apple_epic_service){.ep=&ep,.cookie=&d.dptxport[0],.enabled=true}; @@ -888,6 +921,10 @@ def publication(root): ''' unit=unit.replace('int main(int argc,char **argv) {',failed_receipts+'int main(int argc,char **argv) {') unit=unit.replace(' permutations();conflicting_receipts();',' permutations();conflicting_receipts();failed_replacements();') + # Catalogue ownership: lookup_mode() copies the matched mode out and only + # reports whether one matched; the fixtures only ask whether one did. + if re.search(r'^bool lookup_mode\(', iomfb, re.M): + unit=re.sub(r'\blookup_mode\((&d,&[\w.\[\]]+)\)', r'lookup_mode(\1,NULL)', unit) return {**locals(), "block": block} def hotplug(root): @@ -899,7 +936,7 @@ def hotplug(root): training = (base / 'dptx-j700-training.h').read_text() unit = unit.replace('struct dptx_port { u32 lane_count, link_rate;', ''' union phy_configure_opts { struct { unsigned lanes,link_rate; bool set_lanes,set_rate,set_voltages; } dp; }; -struct dptx_port { union phy_configure_opts phy_ops; int linkcfg_completion; u32 pending_link_rate; +struct dptx_port { union phy_configure_opts phy_ops; int linkcfg_completion; u32 pending_link_rate, unit; u32 lane_count, link_rate;''') unit = unit.replace('int fw_compat,width_mm,height_mm,fixed_connector_type;', 'unsigned dptx_phy; int fw_compat,width_mm,height_mm,fixed_connector_type;') real_handlers = r''' @@ -924,8 +961,9 @@ def hotplug(root): if(o->dp.set_rate)hardware_rate=o->dp.link_rate; return 0; } -static int dcp_tunnel_set_rate(struct apple_dcp *d,void *phy,unsigned rate) { - (void)d;(void)phy;(void)rate;assert(false);return 0; +/* Varargs: the DPTX unit argument came with the T6030 tunnel clocks. */ +static int dcp_tunnel_set_rate(struct apple_dcp *d,void *phy,...) { + (void)d;(void)phy;assert(false);return 0; } /* The T6030 14.7 backend never matches this H17P display. */ #define DCP_FIRMWARE_V_14_7 (DCP_FIRMWARE_H17P + 1) @@ -1006,6 +1044,8 @@ def hotplug(root): puts("PASS: actual RX HPD loss immediately denies getter/lookup before deferred work; 24 fresh-proof recovery orders need only one rate/lane callback; bootstrap/power/paused suppression preserved"); }""".replace('MAIN_PREFIX',n['main_prefix']) unit=unit.replace(block(unit,'int main(int argc,char **argv) {'),functions+'\n'+main) + if re.search(r'^bool lookup_mode\(', (base / 'iomfb.c').read_text(), re.M): + unit=re.sub(r'\blookup_mode\((&d,&[\w.\[\]]+)\)', r'lookup_mode(\1,NULL)', unit) return unit diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py b/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py index cc4f72dfb2cba..5fa3a4900ce18 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py @@ -405,7 +405,9 @@ def run(name,text,cases): if(read_action==4) { dpavservep_detach(&d);d.quiescing=true; } struct drm_edid *edid=malloc(sizeof(*edid));assert(edid);edid->marker=1;edid_allocations++;return edid; } - static const struct drm_edid *dcpavserv_read_edid_legacy(struct apple_epic_service *service) { return dcpavserv_read_edid_inline(service); } + static const struct drm_edid *dcpavserv_read_edid_legacy(struct apple_epic_service *service) { return dcpavserv_read_edid_inline(service); } /* The 14.7 open-before-EDID handshake is for T6030 external processors only. */ + static int __attribute__((unused)) afk_service_call(struct apple_epic_service *s,u16 group,u32 command,const void *data,size_t data_len,size_t data_pad,void *output,size_t output_len,size_t output_pad) +{ (void)s;(void)group;(void)command;(void)data;(void)data_len;(void)data_pad;(void)output;(void)output_len;(void)output_pad;assert(!"14.7 open-before-EDID on H17P");return -ENODEV; } ''' dpav=(root/'drivers/gpu/drm/apple/epic/dpavservep.c').read_text() for signature in ['bool dcpavserv_edid_current(', 'bool dcpavserv_edid_service_current(', 'const struct drm_edid *dcpavserv_copy_edid(']: diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py index 0b19ebdc4c14b..9ba585c7a825a 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py @@ -6,7 +6,7 @@ case executes production transactions with real pthread connector/fabric locks. """ import quiesce -from common import block +from common import block, apple_function def controls(ROOT): @@ -35,7 +35,7 @@ def check(name, unit, cases=((),), negative=False, flags=()): dcp=(ROOT/'drivers/gpu/drm/apple/dcp.c').read_text() actual=base.replace(block(base,'int main('),block(base,'int main(').replace('int main(','int __attribute__((unused)) prior_main(',1),1) actual=actual.replace('struct typec_mux_state { void *alt;', 'struct typec_altmode { unsigned svid; };\nstruct typec_mux_state { struct typec_altmode *alt;',1) - actual=actual.replace('struct connector { bool connected; };','struct encoder { unsigned possible_crtcs; };\nstruct connector { bool connected; void *dcp; struct encoder *port_encoder; unsigned candidate_crtcs; int bl_sync_wq; };\n#define apple_connector connector\nstruct mux_control { int unused; };',1) + actual=actual.replace('struct connector { bool connected; };','struct encoder { unsigned possible_crtcs; };\nstruct connector { bool connected,edid_live,edid_invalidation_pending; void *dcp; struct encoder *port_encoder; unsigned candidate_crtcs; int bl_sync_wq; };\n#define apple_connector connector\nstruct mux_control { int unused; };',1) actual=actual.replace('struct apple_dcp { int external_lock,external_error;', '''struct apple_dcp { int external_lock,external_error; bool external_power_attempted,dptx_tunnel,tb_clock_ok,fixed_route_selected; int tb_lock; unsigned typec_generation,dptx_dfp_port,dptx_phy,fixed_dptx_phy,fixed_mux_index; @@ -43,10 +43,14 @@ def check(name, unit, cases=((),), negative=False, flags=()): struct mux_control *xbar; struct apple_dcp_typec_route *active_typec_route,*typec_routes; struct connector *connector,*fixed_connector;int connector_type,fixed_connector_type; + struct mux_control *direct_xbar_up; bool external_native, ext_dead; int fw_compat; unsigned long long tb_generation; + int modes_lock; unsigned long long modes_generation; bool modes_admitted,modes_provisional; + struct { int iomfb_method_profile; } hw; ''',1) actual=actual.replace('bool applied_valid,safe_pending,dp_wanted,dp_hpd,hpd;', 'struct typec_altmode *applied_alt; unsigned long applied_mode; unsigned applied_status,applied_conf; bool applied_valid,safe_pending,dp_wanted,dp_hpd,hpd;',1) - actual=actual.replace('void *xbar; bool deactivate_xbar_done; int deactivate_error; bool selected,tunnel;', 'struct mux_control *xbar,*active_xbar; void *phy; unsigned dptx_phy,tunnel_dpin,mux_index;bool deactivate_xbar_done; int deactivate_error;bool xbar_up,selected,tunnel;',1) + actual=actual.replace('void *xbar; bool deactivate_xbar_done; int deactivate_error; bool selected,tunnel;', 'struct mux_control *xbar,*active_xbar; void *phy; unsigned dptx_phy,tunnel_dpin,mux_index;unsigned long long tunnel_generation;bool deactivate_xbar_done; int deactivate_error;bool xbar_up,selected,tunnel;',1) extra=r''' + /* actual DCP leaves */ #define WRITE_ONCE(x,v) ((x)=(v)) #define USB_TYPEC_DP_SID 0xff01 #define TYPEC_DP_STATE_A 10 @@ -61,7 +65,7 @@ def check(name, unit, cases=((),), negative=False, flags=()): static int gpiod_get_value_cansleep(void *p); static int dcp_dpxbar_preselect(struct mux_control *m,int s) { (void)m;(void)s;return 0; } static int dcp_dpxbar_tunnel_select_source(struct mux_control *m,int s) { (void)m;(void)s;return 0; } - static int apple_atc_dp_tunnel_rate(void *p,unsigned n,unsigned rate) { (void)p;(void)n;(void)rate;return 0; } + static int __attribute__((unused)) apple_atc_dp_tunnel_rate(void *p,unsigned n,unsigned rate) { (void)p;(void)n;(void)rate;return 0; } static int xbar_error,xbar_calls; static int mux_control_deselect(struct mux_control *m) { (void)m;event('C');xbar_calls++;return xbar_error; } static int mux_control_select(struct mux_control *m,unsigned n) { (void)m;(void)n;return 0; } @@ -75,6 +79,25 @@ def check(name, unit, cases=((),), negative=False, flags=()): static void dcp_retrain_oob(struct connector *c) { (void)c; } static bool dcp_typec_sink_relink(struct apple_dcp *d) { (void)d;return false; } ''' + # Catalogue/EDID ownership: deactivation ends the Type-C attachment of the + # pipeline's modes and moves the connector's pipeline through the real helpers. + extra+=''' + struct platform_device; + #ifndef DCP_IOMFB_METHODS_H17P_EXTERNAL + #define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 + #endif + /* T6030 external processors (dcp->external_native) are not in these scenarios. */ + static void __attribute__((unused)) dcp_external_sink_irq(struct apple_dcp *d) { (void)d;assert(!"T6030 external sink IRQ"); } + static bool __attribute__((unused)) iomfb_v14_7_external_failed(struct apple_dcp *d) { (void)d;assert(!"T6030 external state");return false; } + /* fw_compat is UNKNOWN here: never H17P, 14.7 or 26.6. */ + #ifndef DCP_FIRMWARE_H17P + #define DCP_FIRMWARE_H17P 5 + #endif + /* No route here is a tunnel, and no IRQ_HPD arrives for a disconnected sink. */ + static int __attribute__((unused)) dcp_route_clock_rate(struct apple_dcp_typec_route *r,void *phy,unsigned dpin,u8 rate) { (void)r;(void)phy;(void)dpin;(void)rate;assert(!"tunnel pixel clock");return -EINVAL; } + static int __attribute__((unused)) dcp_dptx_recover_irq(struct apple_dcp *d) { (void)d;assert(!"IRQ_HPD recovery");return -EINVAL; } + static int __attribute__((unused)) of_property_read_string(void *np,const char *name,const char **out) { (void)np;(void)name;(void)out;return -EINVAL; } + '''+''.join(apple_function(ROOT,name) for name in ('apple_connector_invalidate_edid','apple_connector_set_pipeline','dcp_modes_end_typec','dcp_typec_port_busy')) pos=actual.index('static int dcp_typec_route_deactivate(') actual=actual[:pos]+extra+actual[pos:] # The fabric core split: availability is the real dcp_fabric_available() over @@ -83,6 +106,8 @@ def check(name, unit, cases=((),), negative=False, flags=()): 'static bool dcp_typec_safe_held(struct apple_dcp *dcp)\n{', 'bool dcp_typec_dual_stream(void)\n{', 'static bool dcp_hdmi_settle_enabled(struct apple_dcp *dcp)\n{', 'static enum dcp_fabric_presence_state dcp_hdmi_presence(struct apple_dcp *dcp)\n{') if marker in dcp) + # Route-follow: the fixed output, or a failed T6030 external processor, keeps the route busy. + available+=apple_function(ROOT,'dcp_typec_route_busy') if available: actual=actual.replace('struct apple_dcp { int external_lock,external_error;','struct apple_dcp { int external_lock,external_error;\n struct dcp_fabric_presence hdmi_presence; int hdmi_presence_lock; unsigned nr_typec_routes;',1) available='''#define atomic_read(p) (*(p)) @@ -163,10 +188,10 @@ def check(name, unit, cases=((),), negative=False, flags=()): concurrent=concurrent.replace('#define scoped_guard(k,l) for(int once=((void)(l),1);once;once=0)','') locks=r''' #include - static pthread_mutex_t fabric_mutex=PTHREAD_MUTEX_INITIALIZER,external_mutex=PTHREAD_MUTEX_INITIALIZER,tb_mutex=PTHREAD_MUTEX_INITIALIZER; + static pthread_mutex_t fabric_mutex=PTHREAD_MUTEX_INITIALIZER,external_mutex=PTHREAD_MUTEX_INITIALIZER,tb_mutex=PTHREAD_MUTEX_INITIALIZER,modes_mutex=PTHREAD_MUTEX_INITIALIZER; struct real_guard { pthread_mutex_t *mutex; bool held; }; static struct real_guard take_lock(int *key) { - pthread_mutex_t *m=key==&dcp_typec_fabric_lock?&fabric_mutex:key==&display.external_lock?&external_mutex:&tb_mutex; + pthread_mutex_t *m=key==&dcp_typec_fabric_lock?&fabric_mutex:key==&display.external_lock?&external_mutex:key==&display.modes_lock?&modes_mutex:&tb_mutex; assert(pthread_mutex_lock(m)==0);return (struct real_guard){m,true}; } static void drop_lock(struct real_guard *g) { if(g->held){g->held=false;assert(pthread_mutex_unlock(g->mutex)==0);} } @@ -174,7 +199,8 @@ def check(name, unit, cases=((),), negative=False, flags=()): #define scoped_guard(k,l) for(struct real_guard __attribute__((cleanup(drop_lock))) _scope=take_lock(l);_scope.held;drop_lock(&_scope)) static bool observed_unquiesced_safe; ''' - pos=concurrent.index('static int dcp_typec_route_deactivate(') + # Before the DCP leaves: the extracted helpers take real locks too. + pos=concurrent.index('/* actual DCP leaves */') concurrent=concurrent[:pos]+locks+concurrent[pos:] concurrent=concurrent.replace("(void)m;(void)s;event('L');phy_calls++;", "(void)m;if(s->mode==TYPEC_STATE_SAFE&&display.dptxport[0].connected)observed_unquiesced_safe=true;event('L');phy_calls++;",1) concurrent+=r''' From 8593bf338d9da01c6c523fc121b116763a13afdf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 18 Oct 2026 05:03:39 +1100 Subject: [PATCH 1686/2680] drm/apple: Keep display attributes and refuse stale catalogs early Two catalogue rules differ from the ported source. A DisplayAttributes dictionary that does not parse cleared the external backlight capability while it kept the display's previous size. The source keeps the whole previous description, as it keeps an admitted mode catalogue after a failed parse; do the same and leave ext_backlight alone on a parse error. A stale timing transfer was parsed before its attachment generation was checked, so a superseded transfer could fail with -EINVAL from the parser instead of -ESTALE. Check the generation first, as the source does, and keep the check at publication for a transfer that goes stale while it is parsed. Both are caught by the apple-display-parser KUnit cases parser_attributes_malformed, parser_optional_flag_extent and parser_route_catalog_handoff. Fixes: 8d47586b76bc ("drm/apple: close display catalog ownership and teardown gaps") Fixes: 4e81c66ce66d ("drm/apple: bind timing property transfers to their attachment") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 30310006cc70f..5cb82883a05d2 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1578,6 +1578,12 @@ int dcp_modes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *ctx, u32 entries = 0; int ret = 0; + /* A stale transfer is refused before it is parsed, as it is published. */ + scoped_guard(mutex, &dcp->modes_lock) { + if (generation != dcp->modes_generation) + return -ESTALE; + } + if (h17p_external && dcp->chunks.payload_complete && dcp->chunks.mode_ticket.associated) { ctx->automatic_1080p180_rgb8 = true; @@ -1722,22 +1728,22 @@ int dcp_attributes_replace(struct apple_dcp *dcp, struct dcp_parse_ctx *handle, } } + /* Malformed attributes keep the display's previous description. */ + if (ret) + return ret; + /* * Written once, so the hotplug work never sees a working display - * without its backlight; a display whose attributes do not parse - * gets none. The attributes can arrive after the hotplug work has - * run, so sync the backlight when the answer changes. 14.7 swaps - * carry no external backlight level. + * without its backlight. The attributes can arrive after the + * hotplug work has run, so sync the backlight when the answer + * changes. 14.7 swaps carry no external backlight level. */ - ext = !ret && backlight_control && !dcp_has_panel(dcp) && - !dcp_fw_is_14_7(dcp); + ext = backlight_control && !dcp_has_panel(dcp) && !dcp_fw_is_14_7(dcp); if (ext != READ_ONCE(dcp->ext_backlight)) { WRITE_ONCE(dcp->ext_backlight, ext); if (dcp->connector) schedule_work(&dcp->connector->bl_sync_wq); } - if (ret) - return ret; /* * This callback runs on the ordered RTKit RX worker. A mode probe From 069cb4ef4074cd29ac6d6fa5a05e44c38ec357a8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 18 Oct 2026 05:03:39 +1100 Subject: [PATCH 1687/2680] drm/apple: KUnit: Follow the catalogue admission and Thunderbolt follow The external display relink cases set up their catalogue by hand. Since the catalogue gained attachment ownership, lookup_mode() only serves a catalogue that was admitted for the connector, as the firmware publication path does, so the real mode set found no mode. Mark the fixture's catalogue admitted and initialise its modes lock. parser_route_programming_guard still expected a Thunderbolt route not to need a route start. Since commit 4b0768365a61 ("drm/apple: let a Thunderbolt route follow the CRTC a modeset pairs it with"), a Thunderbolt route follows its CRTC as well and dcp_crtc_needs_route_start() no longer excludes tunnels; expect the start. Fixes: 5db87eb90031 ("drm/apple: follow direct Type-C display routes during modesets") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp_external_relink_test.c | 3 +++ drivers/gpu/drm/apple/parser-test.c | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp_external_relink_test.c b/drivers/gpu/drm/apple/dcp_external_relink_test.c index dc80ca636c753..c078781221021 100644 --- a/drivers/gpu/drm/apple/dcp_external_relink_test.c +++ b/drivers/gpu/drm/apple/dcp_external_relink_test.c @@ -402,6 +402,7 @@ static struct relink_test *relink_test_init(struct kunit *test) mutex_init(&dcp->external_lock); mutex_init(&dcp->hpd_mutex); mutex_init(&dcp->dcpavserv.lock); + dcp_modes_init(dcp); spin_lock_init(&dcp->metadata.lock); spin_lock_init(&dcp->mode_state.lock); INIT_DELAYED_WORK(&dcp->announce_wq, dcp_announce_work); @@ -740,6 +741,8 @@ static void relink_test_real(struct relink_test *t) t->catalog.mode = *t->mode; t->dcp.modes = &t->catalog; t->dcp.nr_modes = 1; + /* The catalog the firmware published for this connection. */ + t->dcp.modes_admitted = true; t->dcp.test_push = relink_test_push; } diff --git a/drivers/gpu/drm/apple/parser-test.c b/drivers/gpu/drm/apple/parser-test.c index 34b24ddbad282..1b8136ae17a5e 100644 --- a/drivers/gpu/drm/apple/parser-test.c +++ b/drivers/gpu/drm/apple/parser-test.c @@ -809,8 +809,9 @@ static void parser_route_programming_guard(struct kunit *test) dcp->typec_follow_gen = 4; KUNIT_EXPECT_FALSE(test, dcp_crtc_needs_route_start(dcp)); dcp->typec_follow_gen = 5; + /* A Thunderbolt route follows its CRTC too since afbcf8083b. */ dcp->active_typec_route->tunnel = true; - KUNIT_EXPECT_FALSE(test, dcp_crtc_needs_route_start(dcp)); + KUNIT_EXPECT_TRUE(test, dcp_crtc_needs_route_start(dcp)); dcp->active_typec_route->tunnel = false; KUNIT_EXPECT_FALSE(test, dcp_crtc_route_ready(&crtc->base, &state, true)); crtc_state.connector_mask = 0; From de41b42f8c5e802bb04328ab1d4c9d530cc08427 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 22:40:45 +1100 Subject: [PATCH 1688/2680] usb: typec: tipd: Limit the retries of a held mux by time, not count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A mux that holds a change up is retried until it goes through, and a PHY that stays stuck is given up after 16 retries, said to be some two minutes. That holds only while nothing else happens. A controller event during the backoff brings the retry forward to the usual debounce, and that retry counted against the 16 too, without making the hold-up any shorter. With an event in every backoff, as a partner that keeps renegotiating raises, the retries come about a second apart and the port gave up after some 16.5 s. If the PHY then let go after the last event, nothing applied the state again until a later event. Limit a series by time instead: note when its first failure was, and give up at the first failure two minutes or more after it. The start is noted whenever a failure finds no series under way, so a series that ended in any way, each of which clears its backoff, never lends its start to the next. The state machine keeps to milliseconds, as for its delays, and the driver passes jiffies_to_msecs(jiffies); the difference is taken so that the clock may wrap. Keep the count only as a bound on the warnings of a series. Ordinary controller events bring retries about a second apart, debouncing both the read and the update. Tunnel notifications skip the read debounce, but the 500 ms update debounce alone keeps the limit of 240 from ending a series before two minutes. Counting only the retries that the timer runs would bound nothing that the time limit does not. The warning now gives the retries and the time. Correct the texts that promised about two minutes for a count. Also make typec_mux_timeout_transient()'s kernel-doc say that it describes the last request that failed with -ETIMEDOUT: one that fails with -EBUSY before it owns the providers leaves the answer as it was. A consumer asks only right after -ETIMEDOUT, so say so rather than change the core. The query keeps its form, a flag on the handle; upstream would rather have typec_mux_set() return the answer. The new tests inject an event into every backoff while the PHY holds the change up, and expect the port to give up only two minutes after the first refusal, after some 120 retries rather than 16. With an event every 1.5 s for well over a minute, after which the PHY lets go, the next retry of the backoff must apply the safe and USB states with no further event. The fixture runs works before their delay, so it makes a series as old on the driver's clock as it is in the time of the tests. The state machine tests cover the limit, a clock that wraps, retries a second apart, the count and the fresh start after an event, and the backoff test reads the backoff rather than the time left on its timer. The apple-dcp host selftests pass the time of each failure and check that retries a second apart do not end a series sooner. Repair two existing test races exposed while qualifying this change on ed289aabb019. The IRQ test joins the immediate resume read and checks that retries attempt acknowledgment, rather than requiring work to stay pending. The host teardown fixture ends its observation at the lock that fences the selected removal: direct xHCI teardown may proceed once the host mutex is dropped, while the controller mutex is still held. Force that interleaving and check that the PHY and root accesses have finished before removal can proceed. [ryan: rebased onto #355 — kept HPD v2 and #349 test changes; combined the ceiling retry race checks, clock shims and new cases.] [ryan: split by topic: only the apple-dcp selftests part; the tipd, Type-C mux and ATC PHY part is in 170-usb] [ryan: adapted for the unified 200-dcp — lifecycle.py keeps the unified Type-C session-reconnect shims and adds this change's driver-clock shim after them; replayed on top of the unified DCP series] Fixes: 70f7c21e7d32 ("phy: apple: atc: Route the PIPE from the Type-C mux on a fixed-hub port") (cherry picked from commit 6f715d4dc5ac7fb4e165720f65f43a97d0a57d43 ("usb: typec: tipd: Limit the retries of a held mux by time, not count")) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../drivers/gpu/apple-dcp/lifecycle.py | 8 +++++ .../selftests/drivers/gpu/apple-dcp/pm.py | 31 ++++++++++++++----- .../drivers/gpu/apple-dcp/teardown_pm.py | 24 ++++++++++++-- 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index ba3c32e54be57..6ee0b716b01de 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -424,6 +424,10 @@ def function(file, name): prefix += ('/* Only a Type-C (non-H17P) output queues the session reconnect. */\n' 'static void dcp_queue_typec_reconnect(struct apple_dcp *d, unsigned long delay)\n' '{ (void)d; (void)delay; assert(!"Type-C session reconnect for H17P"); }\n') + # How long a held mux has been retried is measured on the driver's clock. + if 'jiffies_to_msecs(jiffies)' in funcs: + shim += ('\nstatic unsigned long jiffies;\n' + 'static unsigned jiffies_to_msecs(unsigned long t) { return t; }\n') source = prefix + pm + shim + dp + funcs + tests return source @@ -622,6 +626,10 @@ def swap(old, new): '/* Never reached: the H17P external link passes no session. */\n' 'static bool dcp_session_valid_locked(struct apple_dcp *d, struct dcp_fabric_session s, bool cable)\n' '{ (void)d; (void)s; (void)cable; assert(!"session check for a sessionless link"); return false; }\n') + # unless the update work already needed the clock + clock = 'static unsigned jiffies_to_msecs(unsigned long t) { return t; }\n' + if clock in unit: + firmware = firmware.replace(clock, '') return unit + firmware + raw + '\n' + tests def dropped(root): diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index 557162b627907..f704af0a4cc28 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -615,11 +615,17 @@ def function(text, name): if 'cd321x_pm_mux_held(' in pm: lost = ' puts("PASS actual lost events reopen a stale port; unacknowledged events keep their budget with capped backoff");\n' assert controls.count(lost) == 1 - controls = controls.replace(lost, lost + r''' struct cd321x_pm_state held={0}; + controls = controls.replace(lost, lost + r'''/* The time of the failure, once the limit of the retries is on time */ +#ifdef CD321X_MUX_HELD_MS +#define MUX_HELD(pm,ms) cd321x_pm_mux_held(pm,ms) +#else +#define MUX_HELD(pm,ms) ((void)(ms),cd321x_pm_mux_held(pm)) +#endif + struct cd321x_pm_state held={0}; held.phase=CD321X_PM_RUNNING; - for(unsigned int i=0,delay=0;i<4*CD321X_RESUME_ATTEMPTS;i++){ + for(unsigned int i=0,delay=0,now=0;i<4*CD321X_RESUME_ATTEMPTS;i++,now+=delay){ unsigned int expected=i?(2*delaylock&&!pm_refs&&!link_pm_refs); + atomic_store(&handoff_active,false); + } + host_locked=false;pthread_mutex_unlock(&host_mutex); + if(!remover&&remove_target){ + /* Force teardown into the gap before the controller lock is released. */ + pthread_mutex_lock(&schedule_mutex); + while(!removal_started)pthread_cond_wait(&schedule_cond,&schedule_mutex); + pthread_mutex_unlock(&schedule_mutex); + } + }else{ + assert(d==&controller.dev&&controller_locked); + if(!remover&&!remove_target){ + assert(!root_locked&&!link_locked&&!pm_phy->lock&&!pm_refs&&!link_pm_refs); + atomic_store(&handoff_active,false); + } + controller_locked=false;pthread_mutex_unlock(&controller_mutex); + } }''' unit=unit.replace(old,new) # The scheduler injection is a child-lookup interleaving, as in the review, or From 5e3e744084d60c85bfc9d246096e0fb555426fde Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 05:51:41 +1100 Subject: [PATCH 1689/2680] usb: hub: Add helpers to power a port off and on for a lane change A Type-C PHY that switches the PIPE interface of a SuperSpeed root port between its USB3 lanes and a dummy backend needs the controller to have stopped using that interface first. A dual-role port gets that from a role change, which takes the whole controller down. On Apple's T8140 (J700) the controller's USB2 port serves a fixed hub on the board, behind which sit both USB-C connectors and whatever keyboard is plugged in, so the controller has to stay up and only its SuperSpeed root port can be taken down. The port's "disable" attribute already does that for userspace: it disconnects the device on the port and switches the port power off, recorded in power_bits. Give a driver that holds the hub the same, and a counterpart that powers the port on again and has hub_wq look at it. hub_wq warm resets a SuperSpeed port before that, as toggling the port power can leave the two ends of a link out of sync, see usb_port_runtime_resume(). The driver relies on the port staying off until it powers it on, but the hub lock it holds does not keep everyone else away from the port power: - Port runtime PM switches it without the hub lock. Once userspace lets a port power off, its runtime suspend and resume can run at any time, and the resume of its USB2 peer resumes it too. Power-off therefore resumes the port and keeps a runtime PM reference until power-on. Resuming first also tells a port that runtime PM had switched off, which is then held off for the caller, from one that someone switched off on purpose, which is reported and left alone. Taking the hub lock in the runtime PM callbacks instead would deadlock, as hub_event() waits for port runtime PM with that lock held. A port that was switched off through "disable" is recorded as such, so that runtime PM suspending it later does not make it the driver's, and a port whose runtime PM failed earlier is held as it is. - A connect change queued for the port, say by a failed reset of the device just disconnected, made hub_port_connect() power it on again. Power-off drops the connect change and warm reset requests still queued for the port once it is off, and hub_port_connect() no longer switches on a port whose power_bits bit is clear, as hub_power_on() already does not. That changes hub_port_connect() for every hub: a port switched off through the "disable" attribute now stays off through such a change as well. - Writing 0 to the "disable" attribute powers the port on. For a port that a driver holds off, that write now fails with -EBUSY. Writing 1 takes the port over from the driver: the hold ends, and power-on then reports that the port is no longer held and leaves it off. The ABI description of the attribute says so. Only hubs that switch the power of each port on its own are supported, so that nothing but the given port is touched. On an xHCI root hub with port power control that is the port's PORTSC power bit, which the hub driver already toggles for its "attempt power cycle" recovery. (cherry picked from commit 17224f4570a4d4f2dd2d5ae34f584455efd217e6 ("usb: hub: Add helpers to power a port off and on for a lane change")) Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/ABI/testing/sysfs-bus-usb | 5 + drivers/usb/core/hub.c | 174 +++++++++++++++++++++++- drivers/usb/core/hub.h | 5 + drivers/usb/core/port.c | 21 ++- include/linux/usb.h | 4 + 5 files changed, 204 insertions(+), 5 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-bus-usb b/Documentation/ABI/testing/sysfs-bus-usb index af9b653422f16..a556843b7eb76 100644 --- a/Documentation/ABI/testing/sysfs-bus-usb +++ b/Documentation/ABI/testing/sysfs-bus-usb @@ -247,6 +247,11 @@ Description: attached to the port will not be detected, initialized, or enumerated. + A driver that moves the data lanes behind a port can hold + the port off for a while, see usb_hub_port_power_off(). + Writing 0 then fails with -EBUSY. Writing 1 takes the port + over from the driver, which leaves it off from then on. + What: /sys/bus/usb/devices/...//port/early_stop Date: Sep 2022 Contact: Ray Chi diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index df313698b0918..47b4e4ba4054a 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -2130,6 +2130,170 @@ bool usb_device_is_owned(struct usb_device *udev) return !!hub->ports[udev->portnum - 1]->port_owner; } +/** + * usb_hub_port_power_off - disconnect a hub port and hold its power off + * @hdev: USB device belonging to the usb hub + * @port1: port number + * + * For a driver that has to move the data lanes behind a port while the hub + * keeps running, such as a Type-C PHY that switches the PIPE interface of a + * SuperSpeed root port to another backend. The device on the port is + * disconnected and the port is powered off, which takes its link down, as + * the port's "disable" attribute does. Connect changes still queued for the + * port are dropped. Only hubs that switch the power of each port on its own + * are supported. + * + * The port is then held off until usb_hub_port_power_on(): the hub driver + * leaves it off, and the port is kept runtime-active, so that neither its + * own runtime resume nor that of its USB2 peer switches it on in between. + * Writing 0 to its "disable" attribute fails with -EBUSY; writing 1 takes the + * port over, after which usb_hub_port_power_on() leaves it off. + * + * A port that is off for someone else, say through its "disable" attribute, + * is left as it is, and the caller can tell from the return value that it is + * not held for it. A port that runtime PM had switched off is not such a + * port: it is resumed, then held off for the caller. + * + * Context: process context; the caller holds @hdev's device lock. + * + * Return: 0 if the port is held off for the caller, also if it was already, + * 1 if it is off for someone else, -EOPNOTSUPP if the hub cannot switch the + * power of a single port, or another negative error code. + */ +int usb_hub_port_power_off(struct usb_device *hdev, int port1) +{ + struct usb_hub *hub = usb_hub_to_struct_hub(hdev); + struct usb_interface *intf; + struct usb_port *port_dev; + int ret; + + device_lock_assert(&hdev->dev); + if (!hub || hub->disconnected) + return -ENODEV; + if (port1 < 1 || port1 > hdev->maxchild) + return -EINVAL; + if ((le16_to_cpu(hub->descriptor->wHubCharacteristics) & HUB_CHAR_LPSM) != + HUB_CHAR_INDV_PORT_LPSM) + return -EOPNOTSUPP; + + port_dev = hub->ports[port1 - 1]; + if (port_dev->held_off) + return 0; + /* Runtime PM may have suspended it since, which does not make it ours. */ + if (port_dev->disabled) + return 1; + + intf = to_usb_interface(hub->intfdev); + ret = usb_autopm_get_interface(intf); + if (ret < 0) + return ret; + + /* + * The port's runtime resume switches its power on, and does not take + * the hub lock, as hub_event() waits for it under that lock. Keep port + * runtime PM out instead, after waiting for any of it in progress. + */ + ret = pm_runtime_resume_and_get(&port_dev->dev); + if (ret < 0) { + /* After a runtime PM error, nothing resumes the port any more. */ + if (!pm_runtime_active(&port_dev->dev)) + goto out; + pm_runtime_get_noresume(&port_dev->dev); + } + + if (!port_dev->child && !test_bit(port1, hub->power_bits)) { + ret = 1; + goto put_port; + } + + if (port_dev->child) + usb_disconnect(&port_dev->child); + + ret = usb_hub_set_port_power(hdev, hub, port1, false); + if (ret) { + /* The link may still be up: have hub_wq look for its device. */ + set_bit(port1, hub->change_bits); + kick_hub_wq(hub); + goto put_port; + } + + /* The runtime PM reference is dropped once the hold ends. */ + port_dev->held_off = true; + /* Nothing hub_wq still has queued for the port applies to it off. */ + clear_bit(port1, hub->change_bits); + clear_bit(port1, hub->warm_reset_bits); + msleep(2 * hub_power_on_good_delay(hub)); + /* Losing the connection with the power is not an event. */ + usb_clear_port_feature(hdev, port1, USB_PORT_FEAT_C_CONNECTION); + if (!port_dev->is_superspeed) + usb_clear_port_feature(hdev, port1, USB_PORT_FEAT_C_ENABLE); + usb_autopm_put_interface(intf); + return 0; + +put_port: + pm_runtime_put(&port_dev->dev); +out: + usb_autopm_put_interface(intf); + return ret; +} +EXPORT_SYMBOL_GPL(usb_hub_port_power_off); + +/** + * usb_hub_port_power_on - switch on a port held off by usb_hub_port_power_off() + * @hdev: USB device belonging to the usb hub + * @port1: port number + * + * hub_wq then checks the port for a connection. It warm resets a SuperSpeed + * port first: toggling the port power can leave the two ends of the link + * out of sync (see usb_port_runtime_resume()), and the warm reset starts + * both from a reset. The port is under its usual runtime PM policy again. + * + * A port that is not held off, because its "disable" attribute took it over + * for example, is left alone. + * + * Context: process context; the caller holds @hdev's device lock. + * + * Return: 0 if the port was switched on, 1 if it was not held off, or a + * negative error code, in which case the port stays held off. + */ +int usb_hub_port_power_on(struct usb_device *hdev, int port1) +{ + struct usb_hub *hub = usb_hub_to_struct_hub(hdev); + struct usb_interface *intf; + struct usb_port *port_dev; + int ret; + + device_lock_assert(&hdev->dev); + if (!hub || hub->disconnected) + return -ENODEV; + if (port1 < 1 || port1 > hdev->maxchild) + return -EINVAL; + + port_dev = hub->ports[port1 - 1]; + if (!port_dev->held_off) + return 1; + + intf = to_usb_interface(hub->intfdev); + ret = usb_autopm_get_interface(intf); + if (ret < 0) + return ret; + + ret = usb_hub_set_port_power(hdev, hub, port1, true); + if (!ret) { + msleep(hub_power_on_good_delay(hub)); + if (hub_is_superspeed(hdev)) + set_bit(port1, hub->warm_reset_bits); + set_bit(port1, hub->change_bits); + kick_hub_wq(hub); + port_dev->held_off = false; + pm_runtime_put(&port_dev->dev); + } + + usb_autopm_put_interface(intf); + return ret; +} +EXPORT_SYMBOL_GPL(usb_hub_port_power_on); + static void update_port_device_state(struct usb_device *udev) { struct usb_hub *hub; @@ -5493,11 +5657,13 @@ static void hub_port_connect(struct usb_hub *hub, int port1, u16 portstatus, /* * maybe switch power back on (e.g. root hub was reset) - * but only if the port isn't owned by someone else. + * but only if the port isn't owned by someone else, and + * was not switched off on purpose, see hub_power_on(). */ - if (hub_is_port_power_switchable(hub) - && !usb_port_is_power_on(hub, portstatus) - && !port_dev->port_owner) + if (hub_is_port_power_switchable(hub) && + !usb_port_is_power_on(hub, portstatus) && + !port_dev->port_owner && + test_bit(port1, hub->power_bits)) set_port_feature(hdev, port1, USB_PORT_FEAT_POWER); if (portstatus & USB_PORT_STAT_ENABLE) diff --git a/drivers/usb/core/hub.h b/drivers/usb/core/hub.h index da3235930b0c8..3c4465e378ef4 100644 --- a/drivers/usb/core/hub.h +++ b/drivers/usb/core/hub.h @@ -102,6 +102,9 @@ struct usb_hub { * @usb3_lpm_u2_permit: whether USB3 U2 LPM is permitted. * @early_stop: whether port initialization will be stopped earlier. * @ignore_event: whether events of the port are ignored. + * @disabled: switched off through the "disable" attribute + * @held_off: switched off by usb_hub_port_power_off(), which holds the port + * runtime-active until usb_hub_port_power_on() */ struct usb_port { struct usb_device *child; @@ -123,6 +126,8 @@ struct usb_port { unsigned int is_superspeed:1; unsigned int usb3_lpm_u1_permit:1; unsigned int usb3_lpm_u2_permit:1; + bool disabled; + bool held_off; }; #define to_usb_port(_dev) \ diff --git a/drivers/usb/core/port.c b/drivers/usb/core/port.c index b4452b665f591..89066ef6ffdd0 100644 --- a/drivers/usb/core/port.c +++ b/drivers/usb/core/port.c @@ -151,6 +151,21 @@ static ssize_t disable_store(struct device *dev, struct device_attribute *attr, goto out_hdev_lock; } + /* + * A driver holds the port off while it moves the lanes behind it, see + * usb_hub_port_power_off(). Powering it on would run its link on lanes + * in transition. Keeping it off takes the port over from the driver, + * which then leaves it off. + */ + if (port_dev->held_off) { + if (!disabled) { + rc = -EBUSY; + goto out_hdev_lock; + } + port_dev->held_off = false; + pm_runtime_put(&port_dev->dev); + } + if (disabled && port_dev->child) usb_disconnect(&port_dev->child); @@ -163,8 +178,10 @@ static ssize_t disable_store(struct device *dev, struct device_attribute *attr, usb_clear_port_feature(hdev, port1, USB_PORT_FEAT_C_ENABLE); } - if (!rc) + if (!rc) { + port_dev->disabled = disabled; rc = count; + } out_hdev_lock: usb_unlock_device(hdev); @@ -860,6 +877,8 @@ void usb_hub_remove_port_device(struct usb_hub *hub, int port1) peer = port_dev->peer; if (peer) unlink_peers(port_dev, peer); + if (port_dev->held_off) + pm_runtime_put_noidle(&port_dev->dev); component_del(&port_dev->dev, &connector_ops); sysfs_put(port_dev->state_kn); device_unregister(&port_dev->dev); diff --git a/include/linux/usb.h b/include/linux/usb.h index 482d37aff4a72..8c14a6bd3e08b 100644 --- a/include/linux/usb.h +++ b/include/linux/usb.h @@ -955,6 +955,10 @@ int usb_hub_claim_port(struct usb_device *hdev, unsigned port1, int usb_hub_release_port(struct usb_device *hdev, unsigned port1, struct usb_dev_state *owner); +/* port power switching for a driver that moves the lanes behind a port */ +int usb_hub_port_power_off(struct usb_device *hdev, int port1); +int usb_hub_port_power_on(struct usb_device *hdev, int port1); + /** * usb_make_path - returns stable device path in the usb tree * @dev: the device whose path is being constructed From f390ab6eb497f0a938e109a0cc837cd016134792 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 07:27:28 +1100 Subject: [PATCH 1690/2680] usb: core: Add KUnit tests for the port power helpers usb_hub_port_power_off() promises that a port stays off until usb_hub_port_power_on(), whatever its runtime PM, the runtime PM of its USB2 peer and its "disable" attribute do in between. A mocked hub cannot show that. Run the real hub driver instead, on the two root hubs of a host controller made of software: a USB2 and a SuperSpeed root hub with one port each, peered as on an xHCI. The controller keeps the port status and nothing else, and no device ever connects. The tests write port attributes through sysfs, as userspace does, and check that: - the port is held off, and only the given port is switched; - with port power-off allowed, neither the port's runtime suspend and resume nor the resume of its USB2 peer power it on while it is held; - a port that runtime PM had switched off is held for the caller, not reported as someone else's; - the "disable" attribute cannot power a held port on, and can take it over, after which power-on leaves it off. (cherry picked from commit 24b1c5eb3cb6e0677845c2ad3e203b7d3bf25a72 ("usb: core: Add KUnit tests for the port power helpers")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/core/Kconfig | 13 + drivers/usb/core/Makefile | 1 + drivers/usb/core/port-power-test.c | 558 +++++++++++++++++++++++++++++ 3 files changed, 572 insertions(+) create mode 100644 drivers/usb/core/port-power-test.c diff --git a/drivers/usb/core/Kconfig b/drivers/usb/core/Kconfig index 1251c543f6cb0..869acdf0c3a36 100644 --- a/drivers/usb/core/Kconfig +++ b/drivers/usb/core/Kconfig @@ -165,3 +165,16 @@ config USB_DEFAULT_AUTHORIZATION_MODE ACPI selecting value 2 is analogous to selecting value 0. If unsure, keep the default value. + +config USB_PORT_POWER_KUNIT_TEST + tristate "KUnit tests for switching hub port power" if !KUNIT_ALL_TESTS + depends on USB && KUNIT + default KUNIT_ALL_TESTS + help + Test that a port powered off with usb_hub_port_power_off() stays + off until usb_hub_port_power_on(), whatever port runtime PM and + the port's "disable" attribute do in between. The tests run the + hub driver on the root hubs of a software host controller and + need no USB hardware. + + If unsure, say N. diff --git a/drivers/usb/core/Makefile b/drivers/usb/core/Makefile index 64468ad4a6d51..68101ef9880d8 100644 --- a/drivers/usb/core/Makefile +++ b/drivers/usb/core/Makefile @@ -25,3 +25,4 @@ obj-$(CONFIG_USB_QUIRKS_KUNIT_TEST) += usb-quirks-test.o obj-$(CONFIG_USB_WAKEUP_KUNIT_TEST) += usb-wakeup-test.o obj-$(CONFIG_USB_LEDS_TRIGGER_USBPORT) += ledtrig-usbport.o +obj-$(CONFIG_USB_PORT_POWER_KUNIT_TEST) += port-power-test.o diff --git a/drivers/usb/core/port-power-test.c b/drivers/usb/core/port-power-test.c new file mode 100644 index 0000000000000..2a5fee7fe6854 --- /dev/null +++ b/drivers/usb/core/port-power-test.c @@ -0,0 +1,558 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit tests for usb_hub_port_power_off() and usb_hub_port_power_on(). + * + * The tests run on a host controller made of software: a USB2 and a + * SuperSpeed root hub with one port each, peered as on an xHCI, so that the + * hub driver, the port devices and their runtime PM are the real ones. The + * controller only keeps the status of its ports, and no device ever connects. + * Port attributes are written through sysfs, as userspace writes them. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct port_power_test_hc { + spinlock_t lock; /* protects the port status, as hub_wq reads it too */ + u16 status; + u16 change; + unsigned int power_ons; +}; + +struct port_power_test { + struct usb_hcd *hcd; + struct usb_hcd *shared_hcd; + struct usb_device *hs_root; + struct usb_device *ss_root; + struct device *hs_port; + struct device *ss_port; + struct vfsmount *sysfs; +}; + +static const struct { + struct usb_bos_descriptor bos; + struct usb_ss_cap_descriptor ss_cap; +} __packed port_power_test_bos = { + .bos = { + .bLength = USB_DT_BOS_SIZE, + .bDescriptorType = USB_DT_BOS, + .wTotalLength = cpu_to_le16(sizeof(port_power_test_bos)), + .bNumDeviceCaps = 1, + }, + .ss_cap = { + .bLength = USB_DT_USB_SS_CAP_SIZE, + .bDescriptorType = USB_DT_DEVICE_CAPABILITY, + .bDevCapabilityType = USB_SS_CAP_TYPE, + .wSpeedSupported = cpu_to_le16(USB_5GBPS_OPERATION), + .bFunctionalitySupport = USB_LOW_SPEED_OPERATION, + }, +}; + +static struct port_power_test_hc *port_power_test_hc(struct usb_hcd *hcd) +{ + return (struct port_power_test_hc *)hcd->hcd_priv; +} + +static u16 port_power_test_power_bit(struct usb_hcd *hcd) +{ + return hcd->speed >= HCD_USB3 ? USB_SS_PORT_STAT_POWER : USB_PORT_STAT_POWER; +} + +static void port_power_test_set_power(struct usb_hcd *hcd, bool on) +{ + struct port_power_test_hc *hc = port_power_test_hc(hcd); + + if (on) { + hc->status |= port_power_test_power_bit(hcd); + hc->power_ons++; + } else { + hc->status &= ~port_power_test_power_bit(hcd); + } + if (hcd->speed >= HCD_USB3) { + hc->status &= ~USB_PORT_STAT_LINK_STATE; + hc->status |= on ? USB_SS_PORT_LS_RX_DETECT : USB_SS_PORT_LS_SS_DISABLED; + } +} + +static bool port_power_test_powered(struct usb_hcd *hcd) +{ + struct port_power_test_hc *hc = port_power_test_hc(hcd); + + guard(spinlock_irqsave)(&hc->lock); + return hc->status & port_power_test_power_bit(hcd); +} + +static unsigned int port_power_test_power_ons(struct usb_hcd *hcd) +{ + struct port_power_test_hc *hc = port_power_test_hc(hcd); + + guard(spinlock_irqsave)(&hc->lock); + return hc->power_ons; +} + +static void port_power_test_hub_descriptor(struct usb_hcd *hcd, struct usb_hub_descriptor *desc) +{ + memset(desc, 0, sizeof(*desc)); + desc->bNbrPorts = 1; + desc->wHubCharacteristics = cpu_to_le16(HUB_CHAR_INDV_PORT_LPSM | HUB_CHAR_NO_OCPM); + desc->bPwrOn2PwrGood = 1; + if (hcd->speed >= HCD_USB3) { + desc->bDescriptorType = USB_DT_SS_HUB; + desc->bDescLength = USB_DT_SS_HUB_SIZE; + } else { + desc->bDescriptorType = USB_DT_HUB; + desc->bDescLength = USB_DT_HUB_NONVAR_SIZE + 2; + desc->u.hs.DeviceRemovable[1] = 0xff; + } +} + +static u16 port_power_test_change_bit(u16 feature) +{ + switch (feature) { + case USB_PORT_FEAT_C_CONNECTION: + return USB_PORT_STAT_C_CONNECTION; + case USB_PORT_FEAT_C_ENABLE: + return USB_PORT_STAT_C_ENABLE; + case USB_PORT_FEAT_C_RESET: + return USB_PORT_STAT_C_RESET; + case USB_PORT_FEAT_C_BH_PORT_RESET: + return USB_PORT_STAT_C_BH_RESET; + case USB_PORT_FEAT_C_PORT_LINK_STATE: + return USB_PORT_STAT_C_LINK_STATE; + } + return 0; +} + +static int port_power_test_hub_control(struct usb_hcd *hcd, u16 type_req, u16 value, u16 index, + char *buf, u16 length) +{ + struct port_power_test_hc *hc = port_power_test_hc(hcd); + + switch (type_req) { + case DeviceRequest | USB_REQ_GET_DESCRIPTOR: + if (hcd->speed < HCD_USB3 || value >> 8 != USB_DT_BOS) + return -EPIPE; + length = min_t(u16, length, sizeof(port_power_test_bos)); + memcpy(buf, &port_power_test_bos, length); + return length; + case GetHubDescriptor: + port_power_test_hub_descriptor(hcd, (struct usb_hub_descriptor *)buf); + return 0; + case GetHubStatus: + memset(buf, 0, 4); + return 0; + } + + if ((index & 0xff) != 1) + return -EPIPE; + + guard(spinlock_irqsave)(&hc->lock); + switch (type_req) { + case GetPortStatus: + put_unaligned_le16(hc->status, buf); + put_unaligned_le16(hc->change, buf + 2); + return 0; + case SetPortFeature: + if (value == USB_PORT_FEAT_POWER) + port_power_test_set_power(hcd, true); + else if (value == USB_PORT_FEAT_RESET) + hc->change |= USB_PORT_STAT_C_RESET; + else if (value == USB_PORT_FEAT_BH_PORT_RESET) + hc->change |= USB_PORT_STAT_C_RESET | USB_PORT_STAT_C_BH_RESET; + return 0; + case ClearPortFeature: + if (value == USB_PORT_FEAT_POWER) + port_power_test_set_power(hcd, false); + else + hc->change &= ~port_power_test_change_bit(value); + return 0; + } + return -EPIPE; +} + +static int port_power_test_reset(struct usb_hcd *hcd) +{ + struct port_power_test_hc *hc = port_power_test_hc(hcd); + + /* As on an xHCI, the primary HCD has the USB2 root hub */ + if (usb_hcd_is_primary_hcd(hcd)) { + hcd->speed = HCD_USB2; + hcd->self.root_hub->speed = USB_SPEED_HIGH; + } + spin_lock_init(&hc->lock); + port_power_test_set_power(hcd, false); + hc->power_ons = 0; + + return 0; +} + +static int port_power_test_start(struct usb_hcd *hcd) +{ + hcd->uses_new_polling = 1; + return 0; +} + +static void port_power_test_stop(struct usb_hcd *hcd) +{ +} + +static int port_power_test_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags) +{ + return -ENODEV; +} + +static int port_power_test_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status) +{ + return -EINVAL; +} + +static int port_power_test_get_frame_number(struct usb_hcd *hcd) +{ + return 0; +} + +static int port_power_test_hub_status_data(struct usb_hcd *hcd, char *buf) +{ + return 0; +} + +static const struct hc_driver port_power_test_hc_driver = { + .description = "usb-port-power-test", + .product_desc = "USB port power test controller", + .hcd_priv_size = sizeof(struct port_power_test_hc), + .flags = HCD_USB3 | HCD_SHARED, + .reset = port_power_test_reset, + .start = port_power_test_start, + .stop = port_power_test_stop, + .urb_enqueue = port_power_test_urb_enqueue, + .urb_dequeue = port_power_test_urb_dequeue, + .get_frame_number = port_power_test_get_frame_number, + .hub_status_data = port_power_test_hub_status_data, + .hub_control = port_power_test_hub_control, +}; + +static void port_power_test_put_hcds(void *data) +{ + struct port_power_test *t = data; + + usb_put_hcd(t->shared_hcd); + usb_put_hcd(t->hcd); +} + +static void port_power_test_remove_hcds(void *data) +{ + struct port_power_test *t = data; + + put_device(t->ss_port); + put_device(t->hs_port); + usb_remove_hcd(t->shared_hcd); + usb_remove_hcd(t->hcd); +} + +static void port_power_test_unmount(void *data) +{ + kern_unmount(data); +} + +/* The hub driver has set up a root hub once it waits for status changes. */ +static void port_power_test_wait_hub(struct kunit *test, struct usb_hcd *hcd) +{ + unsigned int ms; + + for (ms = 0; ms < 2000 && !READ_ONCE(hcd->status_urb); ms += 20) + msleep(20); + KUNIT_ASSERT_NOT_NULL(test, READ_ONCE(hcd->status_urb)); +} + +static struct device *port_power_test_port(struct kunit *test, struct usb_device *root) +{ + char *name = kasprintf(GFP_KERNEL, "%s-port1", dev_name(&root->dev)); + struct device *port; + + KUNIT_ASSERT_NOT_NULL(test, name); + port = device_find_child_by_name(&root->actconfig->interface[0]->dev, name); + kfree(name); + KUNIT_ASSERT_NOT_NULL(test, port); + + return port; +} + +static int port_power_test_init(struct kunit *test) +{ + struct file_system_type *sysfs_type; + struct port_power_test *t; + struct device *dev; + int ret; + + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t); + test->priv = t; + + dev = kunit_device_register(test, "usb-port-power-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + t->hcd = usb_create_hcd(&port_power_test_hc_driver, dev, dev_name(dev)); + KUNIT_ASSERT_NOT_NULL(test, t->hcd); + t->hcd->skip_phy_initialization = 1; + t->shared_hcd = usb_create_shared_hcd(&port_power_test_hc_driver, dev, dev_name(dev), + t->hcd); + if (!t->shared_hcd) + usb_put_hcd(t->hcd); + KUNIT_ASSERT_NOT_NULL(test, t->shared_hcd); + ret = kunit_add_action_or_reset(test, port_power_test_put_hcds, t); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = usb_add_hcd(t->hcd, 0, 0); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = usb_add_hcd(t->shared_hcd, 0, 0); + if (ret) + usb_remove_hcd(t->hcd); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = kunit_add_action_or_reset(test, port_power_test_remove_hcds, t); + KUNIT_ASSERT_EQ(test, ret, 0); + + t->hs_root = t->hcd->self.root_hub; + t->ss_root = t->shared_hcd->self.root_hub; + port_power_test_wait_hub(test, t->hcd); + port_power_test_wait_hub(test, t->shared_hcd); + t->hs_port = port_power_test_port(test, t->hs_root); + t->ss_port = port_power_test_port(test, t->ss_root); + KUNIT_ASSERT_TRUE(test, port_power_test_powered(t->hcd)); + KUNIT_ASSERT_TRUE(test, port_power_test_powered(t->shared_hcd)); + + /* Root hub runtime PM is not what these tests are about. */ + usb_disable_autosuspend(t->hs_root); + usb_disable_autosuspend(t->ss_root); + + /* sysfs is built in, so its type holds no module reference. */ + sysfs_type = get_fs_type("sysfs"); + KUNIT_ASSERT_NOT_NULL(test, sysfs_type); + t->sysfs = kern_mount(sysfs_type); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->sysfs); + ret = kunit_add_action_or_reset(test, port_power_test_unmount, t->sysfs); + KUNIT_ASSERT_EQ(test, ret, 0); + + return 0; +} + +/* Write @val to the attribute @attr of @dev, as userspace does. */ +static int port_power_test_write(struct kunit *test, struct device *dev, const char *attr, + const char *val) +{ + struct port_power_test *t = test->priv; + struct path root = { .mnt = t->sysfs, .dentry = t->sysfs->mnt_root }; + char *kpath = kobject_get_path(&dev->kobj, GFP_KERNEL); + struct file *file; + loff_t pos = 0; + ssize_t ret; + char *path; + + KUNIT_ASSERT_NOT_NULL(test, kpath); + path = kasprintf(GFP_KERNEL, "%s/%s", kpath + 1, attr); + kfree(kpath); + KUNIT_ASSERT_NOT_NULL(test, path); + + file = file_open_root(&root, path, O_WRONLY, 0); + kfree(path); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = kernel_write(file, val, strlen(val), &pos); + fput(file); + + return ret < 0 ? ret : 0; +} + +/* Let userspace allow runtime PM to switch off both ports. */ +static void port_power_test_allow_power_off(struct kunit *test) +{ + struct port_power_test *t = test->priv; + + if (!IS_ENABLED(CONFIG_PM)) + kunit_skip(test, "needs CONFIG_PM"); + + KUNIT_ASSERT_EQ(test, port_power_test_write(test, t->hs_port, "power/pm_qos_no_power_off", + "0"), 0); + KUNIT_ASSERT_EQ(test, port_power_test_write(test, t->ss_port, "power/pm_qos_no_power_off", + "0"), 0); +} + +static int port_power_test_usage(struct device *dev) +{ + return atomic_read(&dev->power.usage_count); +} + +/* The port stays off until powered on, and only the given port is switched. */ +static void port_power_test_hold(struct kunit *test) +{ + struct port_power_test *t = test->priv; + unsigned int hs_power_ons = port_power_test_power_ons(t->hcd); + int usage = port_power_test_usage(t->ss_port); + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + + /* A second change of the lanes finds the port held for it */ + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 0); + KUNIT_EXPECT_TRUE(test, port_power_test_powered(t->shared_hcd)); + KUNIT_EXPECT_EQ(test, port_power_test_usage(t->ss_port), usage); + + /* Nothing is held any more */ + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 1); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 2), -EINVAL); + usb_unlock_device(t->ss_root); + + /* The USB2 root port was never switched */ + KUNIT_EXPECT_TRUE(test, port_power_test_powered(t->hcd)); + KUNIT_EXPECT_EQ(test, port_power_test_power_ons(t->hcd), hs_power_ons); +} + +/* + * With port power-off allowed, port runtime PM runs while the caller holds + * the hub: it can neither suspend the held port nor resume it, which would + * switch its power on, and neither can the resume of its USB2 peer. + */ +static void port_power_test_runtime_pm(struct kunit *test) +{ + struct port_power_test *t = test->priv; + + port_power_test_allow_power_off(test); + /* The USB2 port, while active, keeps its SuperSpeed peer active */ + KUNIT_ASSERT_EQ(test, pm_runtime_resume_and_get(t->hs_port), 0); + KUNIT_ASSERT_TRUE(test, pm_runtime_active(t->ss_port)); + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + + /* The USB2 port suspends, which lets its peer suspend */ + pm_runtime_put_sync(t->hs_port); + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(t->hs_port)); + KUNIT_EXPECT_LT(test, pm_runtime_suspend(t->ss_port), 0); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(t->ss_port)); + + /* The USB2 port resumes, and resumes its peer first */ + KUNIT_ASSERT_EQ(test, pm_runtime_resume_and_get(t->hs_port), 0); + KUNIT_EXPECT_GE(test, pm_runtime_resume(t->ss_port), 0); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 0); + KUNIT_EXPECT_TRUE(test, port_power_test_powered(t->shared_hcd)); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(t->ss_port)); + usb_unlock_device(t->ss_root); + pm_runtime_put(t->hs_port); +} + +/* A port that runtime PM switched off is the caller's to hold, not someone else's */ +static void port_power_test_runtime_suspended(struct kunit *test) +{ + struct port_power_test *t = test->priv; + + port_power_test_allow_power_off(test); + KUNIT_ASSERT_GE(test, pm_runtime_suspend(t->hs_port), 0); + KUNIT_ASSERT_GE(test, pm_runtime_suspend(t->ss_port), 0); + KUNIT_ASSERT_TRUE(test, pm_runtime_suspended(t->ss_port)); + KUNIT_ASSERT_FALSE(test, port_power_test_powered(t->shared_hcd)); + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(t->ss_port)); + + /* The resume of the USB2 peer leaves it off */ + KUNIT_ASSERT_EQ(test, pm_runtime_resume_and_get(t->hs_port), 0); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + + /* and only the caller switches it on */ + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 0); + KUNIT_EXPECT_TRUE(test, port_power_test_powered(t->shared_hcd)); + usb_unlock_device(t->ss_root); + pm_runtime_put(t->hs_port); +} + +/* The "disable" attribute can keep a held port off, and then owns it */ +static void port_power_test_disable(struct kunit *test) +{ + struct port_power_test *t = test->priv; + int usage = port_power_test_usage(t->ss_port); + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + usb_unlock_device(t->ss_root); + + /* Not powered on behind the lanes' back */ + KUNIT_EXPECT_EQ(test, port_power_test_write(test, t->ss_port, "disable", "0"), -EBUSY); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + + /* Kept off: from now on it is userspace's port */ + KUNIT_EXPECT_EQ(test, port_power_test_write(test, t->ss_port, "disable", "1"), 0); + KUNIT_EXPECT_EQ(test, port_power_test_usage(t->ss_port), usage); + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 1); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 1); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + usb_unlock_device(t->ss_root); + + KUNIT_EXPECT_EQ(test, port_power_test_write(test, t->ss_port, "disable", "0"), 0); + KUNIT_EXPECT_TRUE(test, port_power_test_powered(t->shared_hcd)); +} + +/* A disabled port stays userspace's after runtime PM has suspended it */ +static void port_power_test_disabled_suspended(struct kunit *test) +{ + struct port_power_test *t = test->priv; + + KUNIT_ASSERT_EQ(test, port_power_test_write(test, t->ss_port, "disable", "1"), 0); + port_power_test_allow_power_off(test); + KUNIT_ASSERT_GE(test, pm_runtime_suspend(t->hs_port), 0); + KUNIT_ASSERT_GE(test, pm_runtime_suspend(t->ss_port), 0); + KUNIT_ASSERT_TRUE(test, pm_runtime_suspended(t->ss_port)); + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 1); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 1); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + usb_unlock_device(t->ss_root); +} + +/* The host controller can go away while a port is held off */ +static void port_power_test_removed_held(struct kunit *test) +{ + struct port_power_test *t = test->priv; + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + usb_unlock_device(t->ss_root); +} + +static struct kunit_case port_power_test_cases[] = { + KUNIT_CASE(port_power_test_hold), + KUNIT_CASE(port_power_test_runtime_pm), + KUNIT_CASE(port_power_test_runtime_suspended), + KUNIT_CASE(port_power_test_disable), + KUNIT_CASE(port_power_test_disabled_suspended), + KUNIT_CASE(port_power_test_removed_held), + {} +}; + +static struct kunit_suite port_power_test_suite = { + .name = "usb-hub-port-power", + .init = port_power_test_init, + .test_cases = port_power_test_cases, +}; + +kunit_test_suite(port_power_test_suite); + +MODULE_DESCRIPTION("KUnit tests for USB hub port power switching"); +MODULE_LICENSE("GPL"); From e3456ba562bb8a7b5ef0e8301647e20b23d8cca1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 05:51:41 +1100 Subject: [PATCH 1691/2680] phy: apple: atc: Move a fixed-hub PIPE with the SuperSpeed port off MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a port behind a fixed hub the controller stays in host mode, so the Type-C mux callback parks the PIPE, rebuilds the lanes and routes the PIPE back to the USB3 PHY underneath a running controller. dwc3-apple.c documents the order the hardware expects instead: the controller stops using the PIPE, then the PHY switches it to the dummy backend, then the lanes change, and in reverse on the way up. Get it wrong and the port "is just broken until a phy+dwc3 reset". On a J700 the PIPE lock is not acknowledged while the controller's SuperSpeed port still has a device, so unplugging a USB-C dock with DisplayPort and USB3 failed to park the PIPE until hub_wq had processed the SuperSpeed disconnect, seconds later: 22:05:35 usb 1-1.2: USB disconnect, device number 17 22:05:36 phy-apple-atc 40b000000.phy: Pipehandler lock not acked. 22:05:36 sn201202x 1-0c: failed to place Type-C mux in safe mode: -110 ... 22:05:39 sn201202x 1-0c: cable revalidation failed after 3 attempts 22:05:48 usb 2-1: USB disconnect, device number 9 After a run of these, the dock's SuperSpeed hub trained at 10 Gb/s on every following plug but never enumerated, until a reboot. What kept it there is not established, but it followed PIPE moves under a live link. Stopping the controller is no option here: its USB2 port serves the hub on the board, and with it both USB-C connectors and the keyboard. Follow the order for the SuperSpeed side only. Before a mux update touches the PIPE or the lanes, have the USB core disconnect the device on the controller's SuperSpeed root port and power the port off. That is the port power the hub driver toggles for its "attempt power cycle" recovery, which the same logs show leaving the USB2 devices alone. Power the port on again only once the PIPE is on the USB3 backend of the running host with no change left unfinished, and let hub_wq warm reset it before enumerating. After a failed change, or in a mode without USB3, the port stays off, so that the controller does not train a link against a half-configured PHY. A mode that the PHY cannot run, such as USB4 on a T8140, is turned down before the port goes down for it, so that a working USB3 link is not disconnected for nothing. When the port cannot be restored after a failed change, that error is returned rather than -EOPNOTSUPP, which the Type-C core takes for success. A port that someone else switched off, through its "disable" attribute, stays theirs: the USB core reports it when the PHY takes the port down or brings it up, and the PHY then owes it nothing. A mux update that keeps the mode still skips all of this, unless an earlier update left the port off; then it only powers the port on, and a USB failure on the way does not fail what may be a DisplayPort update. A host that starts up powers its ports itself, so starting one clears the record of a port left off. The device on the port used to be resumed and held across the change instead. It is now disconnected, as it would be anyway once its lanes change. Ports without a fixed hub, which includes every M1 and M2 port, take no root hub and are unchanged. [ryan: adapted to 7.2.10 — context only: atcphy_mux_switch() asserts the lock without the mux debug message, which 7.2.10 already removed in commit 7156d4c0a7b6 ("phy: apple: atc: Look up the Type-C target mode before the mux update")] Fixes: 70f7c21e7d32 ("phy: apple: atc: Route the PIPE from the Type-C mux on a fixed-hub port") (cherry picked from commit cbb9188fef4aca19c88d7e47b81eb29acd4d315b ("phy: apple: atc: Move a fixed-hub PIPE with the SuperSpeed port off")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/Kconfig | 7 +- drivers/phy/apple/atc-test.c | 384 ++++++++++++++++++++++++++++++++++- drivers/phy/apple/atc.c | 309 ++++++++++++++++++---------- 3 files changed, 593 insertions(+), 107 deletions(-) diff --git a/drivers/phy/apple/Kconfig b/drivers/phy/apple/Kconfig index e67f51241c29b..1fb7d1368fa21 100644 --- a/drivers/phy/apple/Kconfig +++ b/drivers/phy/apple/Kconfig @@ -21,9 +21,10 @@ config PHY_APPLE_ATC_KUNIT_TEST help Build the KUnit tests for the Type-C mux decisions of the Apple Type-C PHY driver into the driver. They check how Type-C states - map to PHY modes and which mux updates may skip the lane and - PIPE work. The tests need no hardware and run when the driver - is loaded. + map to PHY modes, which mux updates may skip the lane and PIPE + work, and that a port behind a fixed hub has the controller's + SuperSpeed port off while its PIPE moves. The tests need no + hardware and run when the driver is loaded. If unsure, say N. diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index d3a760a7d62fd..6cc291bc052ce 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -5,6 +5,7 @@ */ #include +#include #include static struct apple_atcphy *atcphy_test_alloc(struct kunit *test, bool fixed_usb2) @@ -91,6 +92,13 @@ static void atcphy_test_noop_same_mode(struct kunit *test) atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; atcphy->pipe_pending = true; KUNIT_EXPECT_TRUE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); + + /* A mode without USB3 leaves the SuperSpeed port off */ + atcphy = atcphy_test_alloc(test, true); + atcphy->mode = APPLE_ATCPHY_MODE_DP; + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + atcphy->usb3_port_off = true; + KUNIT_EXPECT_TRUE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_DP)); } static void atcphy_test_noop_needs_work(struct kunit *test) @@ -114,6 +122,39 @@ static void atcphy_test_noop_needs_work(struct kunit *test) atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_USB3; atcphy->pipe_pending = true; KUNIT_EXPECT_FALSE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); + atcphy->pipe_pending = false; + + /* or its SuperSpeed port, still off since the last handoff */ + atcphy->usb3_port_off = true; + KUNIT_EXPECT_FALSE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); +} + +/* The controller's SuperSpeed port only runs on a completed USB3 route */ +static void atcphy_test_usb3_port_ready(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc(test, true); + + KUNIT_EXPECT_TRUE(test, atcphy_usb3_port_ready(atcphy)); + + atcphy->mode_dirty = true; + KUNIT_EXPECT_FALSE(test, atcphy_usb3_port_ready(atcphy)); + atcphy->mode_dirty = false; + + atcphy->pipe_pending = true; + KUNIT_EXPECT_FALSE(test, atcphy_usb3_port_ready(atcphy)); + atcphy->pipe_pending = false; + + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + KUNIT_EXPECT_FALSE(test, atcphy_usb3_port_ready(atcphy)); + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_USB3; + + atcphy->host_active = false; + KUNIT_EXPECT_FALSE(test, atcphy_usb3_port_ready(atcphy)); + + /* A Type-C port's controller is never handled here */ + atcphy = atcphy_test_alloc(test, false); + atcphy->host_active = true; + KUNIT_EXPECT_FALSE(test, atcphy_usb3_port_ready(atcphy)); } /* @@ -127,6 +168,9 @@ static struct apple_atcphy *atcphy_test_alloc_fixed_hub(struct kunit *test) struct apple_atcphy *atcphy = atcphy_test_alloc_dev(test, true); atcphy->hw = &atcphy_hw_t8140; + /* The lane and DisplayPort tunables a J700 has */ + atcphy->ss_tunables = true; + atcphy->dp_tunables = true; mutex_init(&atcphy->lock); return atcphy; @@ -182,6 +226,328 @@ static void atcphy_test_update_takes_usb(struct kunit *test) KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), -ENODEV); } +/* + * The steps of a mux update on a fixed-hub port, recorded by static stubs + * for the USB side and for the locked mode change: g(et the root hub), + * 0 (SuperSpeed port off), m(ode change) and 1 (SuperSpeed port on). + */ +struct atcphy_test_steps { + char log[16]; + int pm_get_ret; + int port_off_ret; + int port_on_ret; + int mode_ret; + bool mode_port_on; +}; + +static struct atcphy_test_steps *atcphy_test_steps(void) +{ + return kunit_get_current_test()->priv; +} + +static void atcphy_test_step(char step) +{ + struct atcphy_test_steps *steps = atcphy_test_steps(); + size_t len = strlen(steps->log); + + if (len < sizeof(steps->log) - 1) + steps->log[len] = step; +} + +static int atcphy_test_pm_get(struct apple_atcphy *atcphy, struct atcphy_usb3_pm *pm) +{ + atcphy_test_step('g'); + return atcphy_test_steps()->pm_get_ret; +} + +static int atcphy_test_port_off(struct apple_atcphy *atcphy, struct atcphy_usb3_pm *pm) +{ + atcphy_test_step('0'); + return atcphy_test_steps()->port_off_ret; +} + +static int atcphy_test_port_on(struct apple_atcphy *atcphy, struct atcphy_usb3_pm *pm) +{ + atcphy_test_step('1'); + return atcphy_test_steps()->port_on_ret; +} + +static int atcphy_test_mode(struct apple_atcphy *atcphy, enum atcphy_mode target_mode, + bool *port_on) +{ + atcphy_test_step('m'); + *port_on = atcphy_test_steps()->mode_port_on; + return atcphy_test_steps()->mode_ret; +} + +static struct atcphy_test_steps *atcphy_test_stub_steps(struct kunit *test) +{ + struct atcphy_test_steps *steps = kunit_kzalloc(test, sizeof(*steps), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, steps); + test->priv = steps; + kunit_activate_static_stub(test, atcphy_usb3_pm_get, atcphy_test_pm_get); + kunit_activate_static_stub(test, atcphy_usb3_port_off, atcphy_test_port_off); + kunit_activate_static_stub(test, atcphy_usb3_port_on, atcphy_test_port_on); + kunit_activate_static_stub(test, atcphy_mux_set_locked, atcphy_test_mode); + + return steps; +} + +/* The SuperSpeed port goes down before the PHY changes and comes back after it */ +static void atcphy_test_update_port_order(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct atcphy_test_steps *steps = atcphy_test_stub_steps(test); + + /* A new pin assignment with USB3 routed again */ + steps->mode_port_on = true; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), 0); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m1"); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); + + /* An update keeping the mode touches neither */ + memset(steps->log, 0, sizeof(steps->log)); + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + KUNIT_EXPECT_STREQ(test, steps->log, ""); + + /* A mode without USB3: the port stays down */ + steps->mode_port_on = false; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), 0); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m"); + KUNIT_EXPECT_TRUE(test, atcphy->usb3_port_off); +} + +/* No PIPE or lane work unless the port went down, and no port up after a failure */ +static void atcphy_test_update_port_failure(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct atcphy_test_steps *steps = atcphy_test_stub_steps(test); + + steps->port_off_ret = -EPIPE; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), -EPIPE); + KUNIT_EXPECT_STREQ(test, steps->log, "g0"); + + memset(steps->log, 0, sizeof(steps->log)); + steps->port_off_ret = 0; + steps->mode_ret = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), -ETIMEDOUT); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m"); + + /* A change refused before it touched the USB3 route brings the port back */ + memset(steps->log, 0, sizeof(steps->log)); + steps->mode_ret = -EOPNOTSUPP; + steps->mode_port_on = true; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), -EOPNOTSUPP); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m1"); + + /* and if that fails, the Type-C core must not take the refusal for success */ + memset(steps->log, 0, sizeof(steps->log)); + steps->port_on_ret = -ENOMEM; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), -ENOMEM); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m1"); + steps->port_on_ret = 0; + + /* A hub that cannot switch the port is no success the Type-C core skips */ + memset(steps->log, 0, sizeof(steps->log)); + steps->port_off_ret = -EOPNOTSUPP; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), -ENODEV); + KUNIT_EXPECT_STREQ(test, steps->log, "g0"); +} + +/* A port owed only its power is brought up, and USB alone never fails the display */ +static void atcphy_test_update_port_owed(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct atcphy_test_steps *steps = atcphy_test_stub_steps(test); + + atcphy->usb3_port_off = true; + + /* An update keeping the pin assignment */ + steps->mode_port_on = true; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m1"); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); + atcphy->usb3_port_off = true; + + /* The same, with the root hub gone */ + memset(steps->log, 0, sizeof(steps->log)); + steps->pm_get_ret = -ENODEV; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + KUNIT_EXPECT_STREQ(test, steps->log, "g"); + + /* or failing to take the port down */ + memset(steps->log, 0, sizeof(steps->log)); + steps->pm_get_ret = 0; + steps->port_off_ret = -EPIPE; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + KUNIT_EXPECT_STREQ(test, steps->log, "g0"); + + /* A new pin assignment still needs the USB side */ + memset(steps->log, 0, sizeof(steps->log)); + steps->pm_get_ret = -ENODEV; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), -ENODEV); + KUNIT_EXPECT_STREQ(test, steps->log, "g"); + + /* A failed mode change is no USB failure */ + memset(steps->log, 0, sizeof(steps->log)); + steps->pm_get_ret = 0; + steps->port_off_ret = 0; + steps->mode_ret = -EIO; + steps->mode_port_on = false; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), -EIO); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m"); +} + +/* A mode the PHY cannot run is turned down before USB3 goes down for it */ +static void atcphy_test_update_unsupported(struct kunit *test) +{ + struct typec_altmode tbt = { .svid = USB_TYPEC_TBT_SID }; + struct typec_mux_state state = { .alt = &tbt, .mode = TYPEC_STATE_MODAL }; + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct atcphy_test_steps *steps = atcphy_test_stub_steps(test); + + /* Thunderbolt, which this PHY does not have */ + KUNIT_EXPECT_EQ(test, atcphy_mux_update(atcphy, &state), -EOPNOTSUPP); + KUNIT_EXPECT_STREQ(test, steps->log, ""); + + /* DisplayPort without its tunables */ + atcphy->dp_tunables = false; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), -EOPNOTSUPP); + KUNIT_EXPECT_STREQ(test, steps->log, ""); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); +} + +/* The locked mode change as it reports the port: up once the PHY took it down */ +static int atcphy_test_mode_ready(struct apple_atcphy *atcphy, enum atcphy_mode target_mode, + bool *port_on) +{ + atcphy_test_step('m'); + *port_on = atcphy->usb3_port_off && atcphy_test_steps()->mode_port_on; + return atcphy_test_steps()->mode_ret; +} + +/* A port that userspace switched off while the PHY had it down stays off */ +static void atcphy_test_update_port_taken(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct atcphy_test_steps *steps = atcphy_test_stub_steps(test); + + kunit_activate_static_stub(test, atcphy_mux_set_locked, atcphy_test_mode_ready); + steps->mode_port_on = true; + + /* Unplugged, with the port down for the PHY, then disabled through sysfs */ + atcphy->mode = APPLE_ATCPHY_MODE_USB2; + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + atcphy->usb3_port_off = true; + + /* The replug finds the port off for someone else, and leaves it to them */ + steps->port_off_ret = 1; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m"); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); + + /* Disabled while the PHY moved the lanes: nothing is owed afterwards */ + memset(steps->log, 0, sizeof(steps->log)); + steps->port_off_ret = 0; + steps->port_on_ret = 1; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), 0); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m1"); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); +} + +/* + * Zeroed memory in place of the core, PIPE handler and USB2 PHY registers: + * enough to park the PIPE and power the PHY off, with a PIPE lock request + * that is never acknowledged. + */ +static void atcphy_test_fake_regs(struct kunit *test, struct apple_atcphy *atcphy) +{ + void *core = kunit_kzalloc(test, ATCPHY_MISC + 4, GFP_KERNEL); + void *pipehandler = kunit_kzalloc(test, PIPEHANDLER_NONSELECTED_OVERRIDE + 4, GFP_KERNEL); + void *usb2phy = kunit_kzalloc(test, USB2PHY_MISCTUNE + 4, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, core); + KUNIT_ASSERT_NOT_NULL(test, pipehandler); + KUNIT_ASSERT_NOT_NULL(test, usb2phy); + atcphy->regs.core = (__force void __iomem *)core; + atcphy->regs.pipehandler = (__force void __iomem *)pipehandler; + atcphy->regs.usb2phy = (__force void __iomem *)usb2phy; +} + +/* The locked mode change reports a port that can come up again */ +static void atcphy_test_set_port_on(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + bool port_on = true; + + /* The port was not taken down */ + KUNIT_EXPECT_EQ(test, atcphy_mux_set_locked(atcphy, APPLE_ATCPHY_MODE_USB3_DP, &port_on), + 0); + KUNIT_EXPECT_FALSE(test, port_on); + + /* The port is down and the PIPE already on the USB3 backend */ + atcphy->usb3_port_off = true; + KUNIT_EXPECT_EQ(test, atcphy_mux_set_locked(atcphy, APPLE_ATCPHY_MODE_USB3_DP, &port_on), + 0); + KUNIT_EXPECT_TRUE(test, port_on); + + /* A change that failed, here an unplug whose PIPE lock was refused */ + atcphy_test_fake_regs(test, atcphy); + KUNIT_EXPECT_EQ(test, atcphy_mux_set_locked(atcphy, APPLE_ATCPHY_MODE_USB2, &port_on), + -ETIMEDOUT); + KUNIT_EXPECT_TRUE(test, atcphy->mode_dirty); + KUNIT_EXPECT_FALSE(test, port_on); +} + +/* A host that starts up powers its SuperSpeed port on itself */ +static void atcphy_test_host_start(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct phy *phy = kunit_kzalloc(test, sizeof(*phy), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, phy); + phy_set_drvdata(phy, atcphy); + atcphy->usb3_port_off = true; + + KUNIT_EXPECT_EQ(test, atcphy_usb3_set_mode(phy, PHY_MODE_USB_HOST, 0), 0); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); + KUNIT_EXPECT_TRUE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); +} + +#if IS_REACHABLE(CONFIG_USB) +/* Only the controller's SuperSpeed root hub is ever handed the port changes */ +static void atcphy_test_usb3_root(struct kunit *test) +{ + struct device *controller = kunit_kzalloc(test, 2 * sizeof(*controller), GFP_KERNEL); + struct usb_bus *bus = kunit_kzalloc(test, 2 * sizeof(*bus), GFP_KERNEL); + struct usb_device *udev = kunit_kzalloc(test, 2 * sizeof(*udev), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, controller); + KUNIT_ASSERT_NOT_NULL(test, bus); + KUNIT_ASSERT_NOT_NULL(test, udev); + bus[0].sysdev = &controller[0]; + bus[1].sysdev = &controller[1]; + + udev[0].bus = &bus[0]; + udev[0].speed = USB_SPEED_SUPER_PLUS; + KUNIT_EXPECT_TRUE(test, atcphy_usb3_is_root(&udev[0], &controller[0])); + + /* The USB2 root hub, with the fixed hub on it */ + udev[1].bus = &bus[0]; + udev[1].speed = USB_SPEED_HIGH; + KUNIT_EXPECT_FALSE(test, atcphy_usb3_is_root(&udev[1], &controller[0])); + + /* A device on the SuperSpeed root port */ + udev[1].speed = USB_SPEED_SUPER; + udev[1].parent = &udev[0]; + KUNIT_EXPECT_FALSE(test, atcphy_usb3_is_root(&udev[1], &controller[0])); + + /* The SuperSpeed root hub of another controller */ + KUNIT_EXPECT_FALSE(test, atcphy_usb3_is_root(&udev[0], &controller[1])); +} +#endif + /* USB shutdown can turn the lanes off before the SAFE mux request arrives. */ static struct apple_atcphy *atcphy_test_alloc_powered_off(struct kunit *test) { @@ -210,8 +576,9 @@ static void atcphy_test_safe_after_poweroff(struct kunit *test) static void atcphy_test_locked_safe_after_poweroff(struct kunit *test) { struct apple_atcphy *atcphy = atcphy_test_alloc_powered_off(test); + bool port_on; - KUNIT_EXPECT_EQ(test, atcphy_mux_set_locked(atcphy, APPLE_ATCPHY_MODE_OFF), 0); + KUNIT_EXPECT_EQ(test, atcphy_mux_set_locked(atcphy, APPLE_ATCPHY_MODE_OFF, &port_on), 0); KUNIT_EXPECT_EQ(test, atcphy->typec_mode, APPLE_ATCPHY_MODE_OFF); } @@ -240,13 +607,15 @@ static void atcphy_test_safe_live_pipe(struct kunit *test) { struct apple_atcphy *atcphy = atcphy_test_alloc_dev(test, false); struct atcphy_test_configure config = {}; + bool port_on; test->priv = &config; kunit_activate_static_stub(test, atcphy_configure, atcphy_test_configure_stub); atcphy->hw = &atcphy_hw_t8103; atcphy->typec_mode = APPLE_ATCPHY_MODE_USB3; mutex_init(&atcphy->lock); - KUNIT_EXPECT_EQ(test, atcphy_mux_set_locked(atcphy, APPLE_ATCPHY_MODE_OFF), -EBUSY); + KUNIT_EXPECT_EQ(test, atcphy_mux_set_locked(atcphy, APPLE_ATCPHY_MODE_OFF, &port_on), + -EBUSY); KUNIT_EXPECT_EQ(test, config.calls, 0U); KUNIT_EXPECT_EQ(test, atcphy->mode, APPLE_ATCPHY_MODE_USB3_DP); KUNIT_EXPECT_EQ(test, atcphy->typec_mode, APPLE_ATCPHY_MODE_USB3); @@ -278,8 +647,19 @@ static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_target_mode_fixed_hub), KUNIT_CASE(atcphy_test_noop_same_mode), KUNIT_CASE(atcphy_test_noop_needs_work), + KUNIT_CASE(atcphy_test_usb3_port_ready), KUNIT_CASE(atcphy_test_update_skips_usb), KUNIT_CASE(atcphy_test_update_takes_usb), + KUNIT_CASE(atcphy_test_update_port_order), + KUNIT_CASE(atcphy_test_update_port_failure), + KUNIT_CASE(atcphy_test_update_port_owed), + KUNIT_CASE(atcphy_test_update_unsupported), + KUNIT_CASE(atcphy_test_update_port_taken), + KUNIT_CASE(atcphy_test_set_port_on), + KUNIT_CASE(atcphy_test_host_start), +#if IS_REACHABLE(CONFIG_USB) + KUNIT_CASE(atcphy_test_usb3_root), +#endif KUNIT_CASE(atcphy_test_safe_after_poweroff), KUNIT_CASE(atcphy_test_locked_safe_after_poweroff), KUNIT_CASE(atcphy_test_safe_live_pipe), diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index e721533fa8eae..cad63ca465ef8 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -22,9 +22,8 @@ * Author: Sven Peter */ -#include - #include +#include #include #include #include @@ -682,6 +681,8 @@ struct atcphy_hw { * it when the PHY is brought back up after a power-off * @host_active: dwc3 has selected host mode on the USB3 PHY; a fixed-hub port * then routes the PIPE itself on later Type-C mode changes + * @usb3_port_off: The SuperSpeed root port of a fixed-hub controller was powered + * off for a PIPE or lane change and is not powered on again yet * @mode: Current PHY operating mode * @swap_lanes: True if lanes must be swapped due to cable orientation * @dp_link_rate: DisplayPort link rate @@ -747,6 +748,7 @@ struct apple_atcphy { bool fixed_usb2; enum atcphy_mode typec_mode; bool host_active; + bool usb3_port_off; enum atcphy_mode mode; int dp_link_rate; bool swap_lanes; @@ -3074,6 +3076,8 @@ static int atcphy_usb3_power_off(struct phy *phy) return ret; } atcphy->host_active = false; + /* The next host powers its SuperSpeed port on as it starts. */ + atcphy->usb3_port_off = false; return 0; } @@ -3090,6 +3094,8 @@ static int atcphy_usb3_set_mode(struct phy *phy, enum phy_mode mode, int submode switch (mode) { case PHY_MODE_USB_HOST: atcphy->host_active = true; + /* A host that starts up powers the ports of its root hubs itself. */ + atcphy->usb3_port_off = false; break; case PHY_MODE_USB_DEVICE: atcphy->host_active = false; @@ -3345,6 +3351,7 @@ static int atcphy_dwc3_reset_assert(struct reset_controller_dev *rcdev, unsigned _atcphy_dwc3_reset_assert(atcphy); atcphy->host_active = false; + atcphy->usb3_port_off = false; atcphy_usb2_power_off(atcphy); return 0; @@ -3443,71 +3450,31 @@ static int atcphy_probe_switch(struct apple_atcphy *atcphy) } /* - * The fixed USB2 hub keeps the host controller active, but its independent - * SuperSpeed root hub may autosuspend while PIPE still needs it awake. - * Own a bounded root-hub resume reference across a Type-C handoff. Resume - * precedes the ATC lock: USB PM can call back into the PHY on recovery. + * The controller of a fixed-hub port stays up across Type-C mode changes, so + * the PHY moves the PIPE while its SuperSpeed root port is powered off, see + * atcphy_mux_update(). Hold the controller, its xHCI device and the awake + * SuperSpeed root hub across the change: hub_wq then cannot use the port in + * between, and none of them can go away. They are taken before the ATC lock, + * as USB PM can call back into the PHY on recovery. */ struct atcphy_usb3_pm { struct platform_device *controller; struct device *host; struct usb_device *root; - struct usb_device *link; }; #if IS_REACHABLE(CONFIG_USB) -/* The root's runtime state does not cover selective suspend of its port. */ -static int atcphy_usb3_link_get(struct atcphy_usb3_pm *pm) -{ - struct usb_device *link; - int ret; - - /* The graph identifies the controller, not a port in a multiport root. */ - if (pm->root->maxchild != 1) - return pm->root->maxchild ? -ENODEV : -EAGAIN; - link = usb_hub_find_child(pm->root, 1); - if (!link) - return 0; - pm->link = usb_get_dev(link); - usb_lock_device(pm->link); - if (READ_ONCE(pm->link->state) == USB_STATE_NOTATTACHED) { - ret = 0; - goto unlock_link; - } - ret = pm_runtime_resume_and_get(&pm->link->dev); - if (ret < 0) - goto unlock_link; - if (READ_ONCE(pm->link->state) == USB_STATE_NOTATTACHED) { - pm_runtime_put_noidle(&pm->link->dev); - ret = -ENODEV; - goto unlock_link; - } - return 0; - -unlock_link: - usb_unlock_device(pm->link); - usb_put_dev(pm->link); - pm->link = NULL; - return ret; -} - -static void atcphy_usb3_link_put(struct atcphy_usb3_pm *pm) +/* The SuperSpeed root hub of the controller; its USB2 root hub never matches */ +static bool atcphy_usb3_is_root(const struct usb_device *udev, const struct device *controller) { - if (!pm->link) - return; - usb_mark_last_busy(pm->link); - pm_runtime_put_autosuspend(&pm->link->dev); - usb_unlock_device(pm->link); - usb_put_dev(pm->link); - pm->link = NULL; + return !udev->parent && udev->speed >= USB_SPEED_SUPER && udev->bus->sysdev == controller; } static int atcphy_usb3_pm_find(struct usb_device *udev, void *data) { struct atcphy_usb3_pm *pm = data; - if (udev->parent || udev->speed < USB_SPEED_SUPER || - udev->bus->sysdev != &pm->controller->dev) + if (!atcphy_usb3_is_root(udev, &pm->controller->dev)) return 0; pm->root = usb_get_dev(udev); @@ -3524,6 +3491,8 @@ static int atcphy_usb3_pm_get(struct apple_atcphy *atcphy, struct atcphy_usb3_pm int ret; #endif + KUNIT_STATIC_STUB_REDIRECT(atcphy_usb3_pm_get, atcphy, pm); + if (!atcphy->fixed_usb2 || !atcphy->hw->has_t8140_dp) return 0; @@ -3574,26 +3543,17 @@ static int atcphy_usb3_pm_get(struct apple_atcphy *atcphy, struct atcphy_usb3_pm ret = -ENODEV; goto unlock_root; } - ret = pm_runtime_resume_and_get(&pm->root->dev); - if (ret < 0) - goto unlock_root; - if (READ_ONCE(pm->root->state) == USB_STATE_NOTATTACHED || - HCD_DEAD(hcd) || !HCD_HW_ACCESSIBLE(hcd)) { - pm_runtime_put_noidle(&pm->root->dev); - ret = -ENODEV; + /* The graph identifies the controller, not a port in a multiport root. */ + if (pm->root->maxchild != 1) { + ret = pm->root->maxchild ? -ENODEV : -EAGAIN; goto unlock_root; } - /* Resume the attached upstream link before taking the ATC mutex. */ - ret = atcphy_usb3_link_get(pm); - if (ret) { - usb_mark_last_busy(pm->root); - pm_runtime_put_autosuspend(&pm->root->dev); + ret = pm_runtime_resume_and_get(&pm->root->dev); + if (ret < 0) goto unlock_root; - } if (READ_ONCE(pm->root->state) == USB_STATE_NOTATTACHED || READ_ONCE(hcd->state) != HC_STATE_RUNNING || HCD_DEAD(hcd) || !HCD_HW_ACCESSIBLE(hcd)) { - atcphy_usb3_link_put(pm); pm_runtime_put_noidle(&pm->root->dev); ret = -ENODEV; goto unlock_root; @@ -3625,9 +3585,6 @@ static void atcphy_usb3_pm_put(struct atcphy_usb3_pm *pm) if (!pm->root) return; - /* Release child ownership before the root: keep the lock order hierarchical. */ - atcphy_usb3_link_put(pm); - /* Restore normal runtime policy after both success and failed parking. */ usb_mark_last_busy(pm->root); pm_runtime_put_autosuspend(&pm->root->dev); @@ -3643,6 +3600,64 @@ static void atcphy_usb3_pm_put(struct atcphy_usb3_pm *pm) #endif } +/* + * Whether the controller of a fixed-hub port may run its SuperSpeed port: the + * PIPE is on the USB3 backend for its host, and no lane or PIPE change is left + * unfinished. + */ +static bool atcphy_usb3_port_ready(const struct apple_atcphy *atcphy) +{ + return atcphy->fixed_usb2 && atcphy->host_active && !atcphy->mode_dirty && + !atcphy->pipe_pending && atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3; +} + +/* + * Take the controller's SuperSpeed port down before the PIPE moves: the device + * on it is disconnected and the port held powered off, so that nothing drives + * the PIPE when it is handed over. Nothing else of the controller stops, its + * USB2 root port and the fixed hub on it included. Returns 1 when there is no + * such port to take down, and for a port that someone else switched off: that + * stays theirs and is not powered on again. + */ +static int atcphy_usb3_port_off(struct apple_atcphy *atcphy, struct atcphy_usb3_pm *pm) +{ + int ret = 1; + + KUNIT_STATIC_STUB_REDIRECT(atcphy_usb3_port_off, atcphy, pm); + +#if IS_REACHABLE(CONFIG_USB) + if (!pm->root) + return 1; + + ret = usb_hub_port_power_off(pm->root, 1); + if (ret < 0) + dev_warn(atcphy->dev, "Failed to power the SuperSpeed port off: %d\n", ret); +#endif + return ret; +} + +/* + * Bring the SuperSpeed port up again once the PIPE is back on the USB3 backend. + * Returns 1 for a port that someone else has taken over since it went down, + * through its "disable" attribute: it stays off. + */ +static int atcphy_usb3_port_on(struct apple_atcphy *atcphy, struct atcphy_usb3_pm *pm) +{ + int ret = 1; + + KUNIT_STATIC_STUB_REDIRECT(atcphy_usb3_port_on, atcphy, pm); + +#if IS_REACHABLE(CONFIG_USB) + if (!pm->root) + return 1; + + ret = usb_hub_port_power_on(pm->root, 1); + if (ret < 0) + dev_warn(atcphy->dev, "Failed to power the SuperSpeed port on: %d\n", ret); +#endif + return ret; +} + /* The PHY mode a Type-C mux state asks for; unsupported states turn the PHY off. */ static enum atcphy_mode atcphy_typec_target_mode(struct apple_atcphy *atcphy, const struct typec_mux_state *state) @@ -3712,12 +3727,11 @@ static enum atcphy_mode atcphy_typec_target_mode(struct apple_atcphy *atcphy, } /* - * Whether a mux update leaves both the lanes and the PIPE as they are. That is - * the case when the mode does not change and no earlier PIPE handoff to a - * running controller behind a fixed hub is still outstanding. A DisplayPort - * HPD or status update with an unchanged pin assignment is one of these. + * Whether the lanes and the PIPE are as @target_mode needs them: the mode does + * not change, and no earlier PIPE handoff to a running controller behind a + * fixed hub is still outstanding. */ -static bool atcphy_mux_is_noop(const struct apple_atcphy *atcphy, enum atcphy_mode target_mode) +static bool atcphy_mux_in_mode(const struct apple_atcphy *atcphy, enum atcphy_mode target_mode) { if (atcphy->mode != target_mode || atcphy->mode_dirty) return false; @@ -3727,11 +3741,49 @@ static bool atcphy_mux_is_noop(const struct apple_atcphy *atcphy, enum atcphy_mo atcphy->pipe_pending)); } -static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode target_mode) +/* + * Whether a mux update leaves both the lanes and the PIPE as they are, and + * owes no SuperSpeed port its power either. A DisplayPort HPD or status + * update with an unchanged pin assignment is one of these. + */ +static bool atcphy_mux_is_noop(const struct apple_atcphy *atcphy, enum atcphy_mode target_mode) +{ + return atcphy_mux_in_mode(atcphy, target_mode) && + !(atcphy->usb3_port_off && atcphy_usb3_port_ready(atcphy)); +} + +/* Whether the PHY can run @mode at all, which needs no lane or PIPE work to tell */ +static int atcphy_mux_check(const struct apple_atcphy *atcphy, enum atcphy_mode mode) +{ + switch (mode) { + case APPLE_ATCPHY_MODE_TBT: + case APPLE_ATCPHY_MODE_USB4: + if (!atcphy->hw->has_usb4) + return -EOPNOTSUPP; + fallthrough; + case APPLE_ATCPHY_MODE_USB3: + case APPLE_ATCPHY_MODE_USB3_DP: + case APPLE_ATCPHY_MODE_DP: + if (atcphy->hw->has_t8140_dp && atcphy_modes[mode].enable_dp_aux && + !atcphy->dp_tunables) + return -EOPNOTSUPP; + /* Without the lane tunables the SuperSpeed lanes are not calibrated */ + if (!atcphy->ss_tunables) + return -EOPNOTSUPP; + break; + case APPLE_ATCPHY_MODE_OFF: + case APPLE_ATCPHY_MODE_USB2: + break; + } + + return 0; +} + +static int atcphy_mux_switch(struct apple_atcphy *atcphy, enum atcphy_mode target_mode) { int ret; - guard(mutex)(&atcphy->lock); + lockdep_assert_held(&atcphy->lock); if (atcphy->mode == target_mode && !atcphy->mode_dirty) { atcphy->typec_mode = target_mode; /* Retry a failed PIPE setup without disturbing the configured lanes. */ @@ -3753,26 +3805,9 @@ static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode t atcphy->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY) return -EBUSY; - switch (target_mode) { - case APPLE_ATCPHY_MODE_TBT: - case APPLE_ATCPHY_MODE_USB4: - if (!atcphy->hw->has_usb4) - return -EOPNOTSUPP; - fallthrough; - case APPLE_ATCPHY_MODE_USB3: - case APPLE_ATCPHY_MODE_USB3_DP: - case APPLE_ATCPHY_MODE_DP: - if (atcphy->hw->has_t8140_dp && atcphy_modes[target_mode].enable_dp_aux && - !atcphy->dp_tunables) - return -EOPNOTSUPP; - /* Without the lane tunables the SuperSpeed lanes are not calibrated */ - if (!atcphy->ss_tunables) - return -EOPNOTSUPP; - break; - case APPLE_ATCPHY_MODE_OFF: - case APPLE_ATCPHY_MODE_USB2: - break; - } + ret = atcphy_mux_check(atcphy, target_mode); + if (ret) + return ret; atcphy->typec_mode = target_mode; if (atcphy->fixed_usb2) { @@ -3785,6 +3820,8 @@ static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode t * from a later set_mode call. Even another USB3 mode can move the * backend to the other PMA when orientation changes. Park before * rebuilding the lanes, then reconnect while dwc3 is in host mode. + * The controller's SuperSpeed port is off by now, see + * atcphy_mux_update(). */ ret = atcphy_park_pipehandler(atcphy); if (ret) @@ -3817,32 +3854,100 @@ static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode t return atcphy_configure(atcphy, target_mode); } +/* + * Change the mode under the PHY lock. @port_on tells whether the SuperSpeed + * port of a fixed-hub controller, taken down for the change, can come up again. + */ +static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode target_mode, + bool *port_on) +{ + int ret; + + KUNIT_STATIC_STUB_REDIRECT(atcphy_mux_set_locked, atcphy, target_mode, port_on); + + guard(mutex)(&atcphy->lock); + ret = atcphy_mux_switch(atcphy, target_mode); + *port_on = atcphy->usb3_port_off && atcphy_usb3_port_ready(atcphy); + return ret; +} + static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux_state *state) { struct atcphy_usb3_pm pm = {}; enum atcphy_mode target_mode; + bool in_mode, port_on, usb_err = true; int ret; scoped_guard(mutex, &atcphy->lock) { target_mode = atcphy_typec_target_mode(atcphy, state); /* - * Only a lane or PIPE change needs the USB3 root hub below. - * Taking it means waiting for the hub's ongoing work, which - * includes a whole enumeration with its timeouts, and would - * hold up a DisplayPort update that changes neither. + * Only a lane or PIPE change, or a SuperSpeed port still owed + * its power, needs the USB3 root hub below. Taking it means + * waiting for the hub's ongoing work, which includes a whole + * enumeration with its timeouts, and would hold up a + * DisplayPort update that changes neither. */ if (atcphy_mux_is_noop(atcphy, target_mode)) { /* Record matching requests for the next PHY init to restore. */ atcphy->typec_mode = target_mode; return 0; } + /* Turn down a mode the PHY cannot run before USB3 goes down for it. */ + ret = atcphy_mux_check(atcphy, target_mode); + if (ret) + return ret; + in_mode = atcphy_mux_in_mode(atcphy, target_mode); } ret = atcphy_usb3_pm_get(atcphy, &pm); if (ret) - return ret; - ret = atcphy_mux_set_locked(atcphy, target_mode); + goto out; + + /* + * The order in which dwc3 is taken down and brought up around a PHY + * change (see dwc3-apple.c), for the SuperSpeed side only: the + * controller's SuperSpeed port goes down before the PIPE is parked and + * the lanes are rebuilt, and comes up only once the PIPE is on the USB3 + * backend again. It stays down after a failed change, and in a mode + * without USB3. + */ + ret = atcphy_usb3_port_off(atcphy, &pm); + if (ret < 0) { + /* typec_mux_set() skips EOPNOTSUPP, which would report a false success. */ + if (ret == -EOPNOTSUPP) + ret = -ENODEV; + goto put; + } + /* A port that someone else switched off is no longer the PHY's to restore. */ + scoped_guard(mutex, &atcphy->lock) + atcphy->usb3_port_off = !ret; + + ret = atcphy_mux_set_locked(atcphy, target_mode, &port_on); + usb_err = false; + if (port_on) { + int err = atcphy_usb3_port_on(atcphy, &pm); + + if (err >= 0) { + scoped_guard(mutex, &atcphy->lock) + atcphy->usb3_port_off = false; + } else if (!ret || ret == -EOPNOTSUPP) { + /* A port left off must not hide behind a code typec_mux_set() skips */ + ret = err; + usb_err = true; + } + } +put: atcphy_usb3_pm_put(&pm); +out: + /* + * With the lanes and the PIPE already as the mode needs them, only + * USB3 is missing, and that must not fail what may be a DisplayPort + * update. The port stays off until the next update. + */ + if (ret && usb_err && in_mode) { + dev_dbg(atcphy->dev, "SuperSpeed port left off: %d\n", ret); + return 0; + } return ret; } From 239a6717ef262f1eaf7f77d6d01e7488e41fd16a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 05:51:41 +1100 Subject: [PATCH 1692/2680] phy: apple: atc: Park the PIPE of a stopped fixed-hub controller A PIPE handoff that did not complete leaves pipe_pending set, and the dummy handoff then insists on the PIPE lock being acknowledged before it moves the PIPE, as it must while a host is running. dwc3 powers the USB3 PHY off and asserts its reset only after its xHCI has gone, though. On a port behind a fixed hub a missing acknowledgment then fails both; the glue keeps the power-off and the reset pending, and its next start retries the same park before anything else. Unbinding and binding dwc3-apple, the way to restart a fixed-hub controller by hand, could then leave the board without USB until a reboot. Clear host_active as the USB3 PHY is powered off or the reset asserted, before the PIPE is parked, and treat the PIPE of a fixed-hub controller as driven only while that controller is in host mode, pending handoff or not. A stopped controller's park then warns about the missing acknowledgment and goes on, as the park does on other ports once no handoff is pending. A running host still has to acknowledge. Ports without a fixed hub keep their check. The new tests run the power-off and the reset on registers that never acknowledge; both failed with -ETIMEDOUT before. Fixes: f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate") (cherry picked from commit 50ff54154f2cfe87131d22b9c7444cea91bc389a ("phy: apple: atc: Park the PIPE of a stopped fixed-hub controller")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-test.c | 53 ++++++++++++++++++++++++++++++++++++ drivers/phy/apple/atc.c | 37 +++++++++++++++++++------ 2 files changed, 82 insertions(+), 8 deletions(-) diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index 6cc291bc052ce..994f758da0680 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -515,6 +515,57 @@ static void atcphy_test_host_start(struct kunit *test) KUNIT_EXPECT_TRUE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); } +/* A fixed-hub PHY whose USB3 handoff failed while the host was up */ +static struct apple_atcphy *atcphy_test_alloc_failed_handoff(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + + atcphy_test_fake_regs(test, atcphy); + atcphy->pipe_pending = true; + atcphy->usb3_port_off = true; + + /* The running host has to acknowledge the PIPE lock */ + scoped_guard(mutex, &atcphy->lock) + KUNIT_EXPECT_EQ(test, atcphy_park_pipehandler(atcphy), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_USB3); + KUNIT_EXPECT_TRUE(test, atcphy->pipe_pending); + + return atcphy; +} + +/* Once dwc3 has stopped, its USB3 PHY power-off parks the PIPE without one */ +static void atcphy_test_stopped_power_off(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_failed_handoff(test); + struct phy *phy = kunit_kzalloc(test, sizeof(*phy), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, phy); + phy_set_drvdata(phy, atcphy); + + KUNIT_EXPECT_EQ(test, atcphy_usb3_power_off(phy), 0); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); + KUNIT_EXPECT_FALSE(test, atcphy->pipe_pending); + KUNIT_EXPECT_EQ(test, atcphy->mode, APPLE_ATCPHY_MODE_OFF); + KUNIT_EXPECT_FALSE(test, atcphy->host_active); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); +} + +/* and so does the reset that dwc3 asserts once it has stopped */ +static void atcphy_test_stopped_reset(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_failed_handoff(test); + u32 aon; + + KUNIT_EXPECT_EQ(test, atcphy_dwc3_reset_assert(&atcphy->rcdev, 0), 0); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); + KUNIT_EXPECT_FALSE(test, atcphy->pipe_pending); + KUNIT_EXPECT_FALSE(test, atcphy->host_active); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); + aon = readl(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN); + KUNIT_EXPECT_TRUE(test, aon & PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); + KUNIT_EXPECT_FALSE(test, aon & PIPEHANDLER_AON_GEN_DWC3_RESET_N); +} + #if IS_REACHABLE(CONFIG_USB) /* Only the controller's SuperSpeed root hub is ever handed the port changes */ static void atcphy_test_usb3_root(struct kunit *test) @@ -657,6 +708,8 @@ static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_update_port_taken), KUNIT_CASE(atcphy_test_set_port_on), KUNIT_CASE(atcphy_test_host_start), + KUNIT_CASE(atcphy_test_stopped_power_off), + KUNIT_CASE(atcphy_test_stopped_reset), #if IS_REACHABLE(CONFIG_USB) KUNIT_CASE(atcphy_test_usb3_root), #endif diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index cad63ca465ef8..b1833bef10990 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1466,11 +1466,26 @@ static void atcphy_enable_dummy_phy(struct apple_atcphy *atcphy) PIPEHANDLER_DUMMY_PHY_EN); } +/* + * Whether a controller may still be driving the PIPE, which then has to + * acknowledge the lock before the PIPE moves. A handoff that did not complete + * leaves the PIPE in an unknown state. A controller behind a fixed hub only + * drives it in host mode, which it has left by the time it powers the USB3 PHY + * off or asserts its reset. + */ +static bool atcphy_pipe_live(const struct apple_atcphy *atcphy) +{ + if (atcphy->hw->has_t8140_dp && atcphy->fixed_usb2) + return atcphy->host_active && + (atcphy->pipe_pending || + atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3); + + return atcphy->pipe_pending; +} + static int atcphy_configure_pipehandler_dummy(struct apple_atcphy *atcphy) { - bool live_usb3 = atcphy->pipe_pending || - (atcphy->hw->has_t8140_dp && atcphy->fixed_usb2 && atcphy->host_active && - atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3); + bool live_usb3 = atcphy_pipe_live(atcphy); int ret; ret = atcphy_pipehandler_check(atcphy); @@ -3067,6 +3082,13 @@ static int atcphy_usb3_power_off(struct phy *phy) guard(mutex)(&atcphy->lock); + /* + * dwc3 has left host mode: nothing drives the PIPE any more, and the + * next host powers its SuperSpeed port on as it starts. + */ + atcphy->host_active = false; + atcphy->usb3_port_off = false; + ret = atcphy_park_pipehandler(atcphy); if (ret) return ret; @@ -3075,9 +3097,6 @@ static int atcphy_usb3_power_off(struct phy *phy) if (ret) return ret; } - atcphy->host_active = false; - /* The next host powers its SuperSpeed port on as it starts. */ - atcphy->usb3_port_off = false; return 0; } @@ -3344,14 +3363,16 @@ static int atcphy_dwc3_reset_assert(struct reset_controller_dev *rcdev, unsigned guard(mutex)(&atcphy->lock); + /* The controller is only reset once it has stopped. */ + atcphy->host_active = false; + atcphy->usb3_port_off = false; + /* Do not reset the controller or USB2 PHY if PIPE parking fails. */ ret = atcphy_park_pipehandler(atcphy); if (ret) return ret; _atcphy_dwc3_reset_assert(atcphy); - atcphy->host_active = false; - atcphy->usb3_port_off = false; atcphy_usb2_power_off(atcphy); return 0; From 8e3116caf0364ca87175676245810b4ebed94541 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 05:51:41 +1100 Subject: [PATCH 1693/2680] phy: apple: atc: Don't let USB3 PHY trouble stop a fixed-hub host dwc3 calls phy_set_mode(PHY_MODE_USB_HOST) on the USB3 PHY as it brings its host up, and usb_add_hcd() calls it again for the SuperSpeed root hub of the xHCI it registers. The PHY routes the PIPE to the USB3 backend there. It fails with -EBUSY while a lane change is unfinished, or with the error of a PIPE lock or BIST that did not complete. A failure in usb_add_hcd() fails the whole xHCI registration: xhci-plat removes its USB2 HCD again as well. On a port behind a fixed hub that takes down the USB2 root hub, the hub on it and everything behind both USB-C connectors. A cable flip that arrives while the host starts, which only marks the lanes for a rebuild, is enough. For a fixed-hub host, a PIPE that cannot be routed no longer fails the call: park it if that works, and return 0. The Type-C mux update that follows a flip, or retries a failed change, restores the lanes and routes the PIPE with the controller's SuperSpeed port off. If the PIPE cannot even be parked, say so: the SuperSpeed port then faces a PIPE in an unknown state until that update. Other ports keep the error: their controller only starts after the mux has configured the lanes. Fixes: f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate") (cherry picked from commit 29dbc02ce89c73e39faebd969c3eca585226e795 ("phy: apple: atc: Don't let USB3 PHY trouble stop a fixed-hub host")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-test.c | 35 +++++++++++++++++++++ drivers/phy/apple/atc.c | 61 +++++++++++++++++++++++++----------- 2 files changed, 78 insertions(+), 18 deletions(-) diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index 994f758da0680..aef8ab6009e81 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -566,6 +566,40 @@ static void atcphy_test_stopped_reset(struct kunit *test) KUNIT_EXPECT_FALSE(test, aon & PIPEHANDLER_AON_GEN_DWC3_RESET_N); } +/* A SuperSpeed problem does not keep the host behind a fixed hub from starting */ +static void atcphy_test_host_start_without_usb3(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct phy *phy = kunit_kzalloc(test, sizeof(*phy), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, phy); + phy_set_drvdata(phy, atcphy); + atcphy_test_fake_regs(test, atcphy); + atcphy->host_active = false; + + /* An unfinished lane change: the PIPE stays parked for the next mux update */ + atcphy->mode_dirty = true; + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + KUNIT_EXPECT_EQ(test, atcphy_usb3_set_mode(phy, PHY_MODE_USB_HOST, 0), 0); + KUNIT_EXPECT_TRUE(test, atcphy->host_active); + KUNIT_EXPECT_TRUE(test, atcphy->mode_dirty); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); + + /* A handoff that cannot finish, as the PIPE lock is refused */ + atcphy->mode_dirty = false; + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_USB3; + atcphy->pipe_pending = true; + KUNIT_EXPECT_EQ(test, atcphy_usb3_set_mode(phy, PHY_MODE_USB_HOST, 0), 0); + KUNIT_EXPECT_TRUE(test, atcphy->pipe_pending); + + /* A Type-C port's controller still waits for the lanes */ + atcphy = atcphy_test_alloc(test, false); + mutex_init(&atcphy->lock); + phy_set_drvdata(phy, atcphy); + atcphy->mode_dirty = true; + KUNIT_EXPECT_EQ(test, atcphy_usb3_set_mode(phy, PHY_MODE_USB_HOST, 0), -EBUSY); +} + #if IS_REACHABLE(CONFIG_USB) /* Only the controller's SuperSpeed root hub is ever handed the port changes */ static void atcphy_test_usb3_root(struct kunit *test) @@ -710,6 +744,7 @@ static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_host_start), KUNIT_CASE(atcphy_test_stopped_power_off), KUNIT_CASE(atcphy_test_stopped_reset), + KUNIT_CASE(atcphy_test_host_start_without_usb3), #if IS_REACHABLE(CONFIG_USB) KUNIT_CASE(atcphy_test_usb3_root), #endif diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index b1833bef10990..76faa80e1bfbd 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -3101,9 +3101,38 @@ static int atcphy_usb3_power_off(struct phy *phy) return 0; } +/* Route the PIPE to the backend of the current mode for the role dwc3 selected. */ +static int atcphy_usb3_route(struct apple_atcphy *atcphy, bool host) +{ + int ret; + + lockdep_assert_held(&atcphy->lock); + + /* + * We may get multiple calls to set_mode (for host mode e.g. at least one from the dwc3 glue + * driver and then another one from the generic xhci code) but must only configure the + * PIPE handler once. Nothing needs to be done either when the PIPE is already routed to + * the backend the current mode uses, which includes the dummy backend of the USB2 modes. + */ + /* A failed lane/domain transition must be restored before routing PIPE. */ + if (atcphy->mode_dirty) + return -EBUSY; + if (atcphy->pipe_pending) { + ret = atcphy_park_pipehandler(atcphy); + if (ret) + return ret; + } + if (atcphy->pipe_state == atcphy_modes[atcphy->mode].pipehandler_state && + !atcphy->pipe_pending) + return 0; + + return atcphy_configure_pipehandler(atcphy, host); +} + static int atcphy_usb3_set_mode(struct phy *phy, enum phy_mode mode, int submode) { struct apple_atcphy *atcphy = phy_get_drvdata(phy); + int ret; guard(mutex)(&atcphy->lock); @@ -3123,26 +3152,22 @@ static int atcphy_usb3_set_mode(struct phy *phy, enum phy_mode mode, int submode return -EINVAL; } + ret = atcphy_usb3_route(atcphy, mode == PHY_MODE_USB_HOST); + if (!ret || !atcphy->fixed_usb2 || mode != PHY_MODE_USB_HOST) + return ret; + /* - * We may get multiple calls to set_mode (for host mode e.g. at least one from the dwc3 glue - * driver and then another one from the generic xhci code) but must only configure the - * PIPE handler once. Nothing needs to be done either when the PIPE is already routed to - * the backend the current mode uses, which includes the dummy backend of the USB2 modes. + * The host behind a fixed hub serves that hub as well, and when this + * fails, xHCI registers neither of its root hubs. Let it start, with + * the PIPE parked if that works: the Type-C mux update that restores + * the lanes or retries the handoff routes the PIPE, with the + * SuperSpeed port off. */ - /* A failed lane/domain transition must be restored before routing PIPE. */ - if (atcphy->mode_dirty) - return -EBUSY; - if (atcphy->pipe_pending) { - int ret = atcphy_park_pipehandler(atcphy); - - if (ret) - return ret; - } - if (atcphy->pipe_state == atcphy_modes[atcphy->mode].pipehandler_state && - !atcphy->pipe_pending) - return 0; - - return atcphy_configure_pipehandler(atcphy, mode == PHY_MODE_USB_HOST); + if (atcphy_park_pipehandler(atcphy)) + dev_warn(atcphy->dev, "Host starts with its PIPE not parked\n"); + else + dev_dbg(atcphy->dev, "Host starts without USB3: %d\n", ret); + return 0; } static const struct phy_ops apple_atc_usb3_phy_ops = { From eb55ad3e24893adc41d559be57b87c6044b53738 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 07:27:28 +1100 Subject: [PATCH 1694/2680] phy: apple: atc: Retry a SuperSpeed port power-on that a mux update owes A mux update that keeps the mode, but still owes the SuperSpeed root port of a fixed-hub controller its power, does not fail on the USB side, so that the display it may carry stays up. The Type-C driver then records the mode as applied, and nothing brings the port up later: with a USB3 dock that sends no further event, USB3 stays off although the host and the PHY are ready for it. The request to the root hub can fail for want of memory, which leaves the controller running. Retry the power-on from delayed work, five times over about three seconds. Each try takes the locks of a mux update, looks up the current root hub, and powers the port on only while it is still owed and the PIPE is on the USB3 backend of a running host; a mux update, a host stop or a new host in between ends the retries. A mode change that fails only in that last power-on still returns the error, and its power-on is retried the same way. The work is freezable, so it waits across system sleep, and it is cancelled when the PHY goes away. (cherry picked from commit 77ebd433838e09c662238c18ca51ff7fc5a0ab02 ("phy: apple: atc: Retry a SuperSpeed port power-on that a mux update owes")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-test.c | 80 ++++++++++++++++++++++++++++++++ drivers/phy/apple/atc.c | 88 ++++++++++++++++++++++++++++++++++-- 2 files changed, 164 insertions(+), 4 deletions(-) diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index aef8ab6009e81..65e82a6d6edf0 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -157,6 +157,13 @@ static void atcphy_test_usb3_port_ready(struct kunit *test) KUNIT_EXPECT_FALSE(test, atcphy_usb3_port_ready(atcphy)); } +static void atcphy_test_cancel_port_work(void *data) +{ + struct apple_atcphy *atcphy = data; + + cancel_delayed_work_sync(&atcphy->usb3_port_work); +} + /* * A J700-style PHY behind a fixed hub, in DisplayPort + USB3 with the PIPE on * USB3 for a running host, but without a device tree node. A mux update that @@ -166,6 +173,11 @@ static void atcphy_test_usb3_port_ready(struct kunit *test) static struct apple_atcphy *atcphy_test_alloc_fixed_hub(struct kunit *test) { struct apple_atcphy *atcphy = atcphy_test_alloc_dev(test, true); + int ret; + + INIT_DELAYED_WORK(&atcphy->usb3_port_work, atcphy_usb3_port_work); + ret = kunit_add_action_or_reset(test, atcphy_test_cancel_port_work, atcphy); + KUNIT_ASSERT_EQ(test, ret, 0); atcphy->hw = &atcphy_hw_t8140; /* The lane and DisplayPort tunables a J700 has */ @@ -280,6 +292,20 @@ static int atcphy_test_mode(struct apple_atcphy *atcphy, enum atcphy_mode target return atcphy_test_steps()->mode_ret; } +/* Retries stay queued until a test runs them, in its own context. */ +static unsigned long atcphy_test_retry_delay(unsigned int retry) +{ + return MAX_JIFFY_OFFSET; +} + +static bool atcphy_test_run_retry(struct apple_atcphy *atcphy) +{ + if (!cancel_delayed_work_sync(&atcphy->usb3_port_work)) + return false; + atcphy_usb3_port_work(&atcphy->usb3_port_work.work); + return true; +} + static struct atcphy_test_steps *atcphy_test_stub_steps(struct kunit *test) { struct atcphy_test_steps *steps = kunit_kzalloc(test, sizeof(*steps), GFP_KERNEL); @@ -290,6 +316,7 @@ static struct atcphy_test_steps *atcphy_test_stub_steps(struct kunit *test) kunit_activate_static_stub(test, atcphy_usb3_port_off, atcphy_test_port_off); kunit_activate_static_stub(test, atcphy_usb3_port_on, atcphy_test_port_on); kunit_activate_static_stub(test, atcphy_mux_set_locked, atcphy_test_mode); + kunit_activate_static_stub(test, atcphy_usb3_port_retry_delay, atcphy_test_retry_delay); return steps; } @@ -399,6 +426,58 @@ static void atcphy_test_update_port_owed(struct kunit *test) KUNIT_EXPECT_STREQ(test, steps->log, "g0m"); } +/* A power-on owed after a USB failure is retried, while it is owed, a few times */ +static void atcphy_test_update_port_retry(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct atcphy_test_steps *steps = atcphy_test_stub_steps(test); + unsigned int i; + + atcphy->usb3_port_off = true; + steps->mode_port_on = true; + + /* Out of memory for the hub request: the update keeps the display */ + steps->port_on_ret = -ENOMEM; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m1"); + + /* and with no further Type-C event, the retry brings the port up */ + memset(steps->log, 0, sizeof(steps->log)); + steps->port_on_ret = 0; + KUNIT_EXPECT_TRUE(test, atcphy_test_run_retry(atcphy)); + KUNIT_EXPECT_STREQ(test, steps->log, "g1"); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); + KUNIT_EXPECT_FALSE(test, atcphy_test_run_retry(atcphy)); + + /* A retry finding the host gone leaves the port to the next host */ + atcphy->usb3_port_off = true; + steps->port_on_ret = -ENOMEM; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + atcphy->host_active = false; + memset(steps->log, 0, sizeof(steps->log)); + KUNIT_EXPECT_TRUE(test, atcphy_test_run_retry(atcphy)); + KUNIT_EXPECT_STREQ(test, steps->log, "g"); + KUNIT_EXPECT_FALSE(test, atcphy_test_run_retry(atcphy)); + atcphy->host_active = true; + + /* A port that keeps failing is given up after a few retries */ + memset(steps->log, 0, sizeof(steps->log)); + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + for (i = 0; i < ATCPHY_USB3_PORT_RETRIES; i++) + KUNIT_EXPECT_TRUE(test, atcphy_test_run_retry(atcphy)); + KUNIT_EXPECT_FALSE(test, atcphy_test_run_retry(atcphy)); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m1g1g1g1g1g1"); + KUNIT_EXPECT_TRUE(test, atcphy->usb3_port_off); + + /* A mode change fails on the last step, and is retried from there too */ + memset(steps->log, 0, sizeof(steps->log)); + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), -ENOMEM); + steps->port_on_ret = 0; + KUNIT_EXPECT_TRUE(test, atcphy_test_run_retry(atcphy)); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m1g1"); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); +} + /* A mode the PHY cannot run is turned down before USB3 goes down for it */ static void atcphy_test_update_unsupported(struct kunit *test) { @@ -738,6 +817,7 @@ static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_update_port_order), KUNIT_CASE(atcphy_test_update_port_failure), KUNIT_CASE(atcphy_test_update_port_owed), + KUNIT_CASE(atcphy_test_update_port_retry), KUNIT_CASE(atcphy_test_update_unsupported), KUNIT_CASE(atcphy_test_update_port_taken), KUNIT_CASE(atcphy_test_set_port_on), diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 76faa80e1bfbd..5e73be16e0788 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -683,6 +684,8 @@ struct atcphy_hw { * then routes the PIPE itself on later Type-C mode changes * @usb3_port_off: The SuperSpeed root port of a fixed-hub controller was powered * off for a PIPE or lane change and is not powered on again yet + * @usb3_port_work: Retries the power-on of @usb3_port_off after a USB failure + * @usb3_port_retries: Retries left for @usb3_port_work * @mode: Current PHY operating mode * @swap_lanes: True if lanes must be swapped due to cable orientation * @dp_link_rate: DisplayPort link rate @@ -749,6 +752,8 @@ struct apple_atcphy { enum atcphy_mode typec_mode; bool host_active; bool usb3_port_off; + struct delayed_work usb3_port_work; + unsigned int usb3_port_retries; enum atcphy_mode mode; int dp_link_rate; bool swap_lanes; @@ -3704,6 +3709,77 @@ static int atcphy_usb3_port_on(struct apple_atcphy *atcphy, struct atcphy_usb3_p return ret; } +/* + * A mux update that keeps the mode does not fail on the USB side, so that the + * display it may carry stays up, and no Type-C event need follow any update. + * A SuperSpeed port power-on that an update still owes after a USB failure is + * retried from here, a few times, with the locks of a mux update, the current + * root hub, and only while the PIPE is on the USB3 backend of a running host. + * The work is freezable, so it waits across system sleep, and is cancelled on + * removal. + */ +#define ATCPHY_USB3_PORT_RETRIES 5 +#define ATCPHY_USB3_PORT_RETRY_MS 100 + +static unsigned long atcphy_usb3_port_retry_delay(unsigned int retry) +{ + KUNIT_STATIC_STUB_REDIRECT(atcphy_usb3_port_retry_delay, retry); + + return msecs_to_jiffies(ATCPHY_USB3_PORT_RETRY_MS << retry); +} + +static void atcphy_usb3_port_retry(struct apple_atcphy *atcphy, bool restart, int err) +{ + unsigned int retry = 0; + + scoped_guard(mutex, &atcphy->lock) { + if (restart) + atcphy->usb3_port_retries = ATCPHY_USB3_PORT_RETRIES; + if (!atcphy->usb3_port_retries) { + dev_warn(atcphy->dev, "SuperSpeed port left off: %d\n", err); + return; + } + retry = ATCPHY_USB3_PORT_RETRIES - atcphy->usb3_port_retries--; + } + dev_dbg(atcphy->dev, "SuperSpeed port power-on deferred: %d\n", err); + mod_delayed_work(system_freezable_wq, &atcphy->usb3_port_work, + atcphy_usb3_port_retry_delay(retry)); +} + +/* Power the SuperSpeed port on if it is still owed and the route still ready */ +static int atcphy_usb3_port_restore(struct apple_atcphy *atcphy, struct atcphy_usb3_pm *pm) +{ + int ret; + + scoped_guard(mutex, &atcphy->lock) { + if (!atcphy->usb3_port_off || !atcphy_usb3_port_ready(atcphy)) + return 0; + } + ret = atcphy_usb3_port_on(atcphy, pm); + if (ret < 0) + return ret; + scoped_guard(mutex, &atcphy->lock) + atcphy->usb3_port_off = false; + + return 0; +} + +static void atcphy_usb3_port_work(struct work_struct *work) +{ + struct apple_atcphy *atcphy = container_of(to_delayed_work(work), struct apple_atcphy, + usb3_port_work); + struct atcphy_usb3_pm pm = {}; + int ret; + + ret = atcphy_usb3_pm_get(atcphy, &pm); + if (!ret) { + ret = atcphy_usb3_port_restore(atcphy, &pm); + atcphy_usb3_pm_put(&pm); + } + if (ret) + atcphy_usb3_port_retry(atcphy, false, ret); +} + /* The PHY mode a Type-C mux state asks for; unsupported states turn the PHY off. */ static enum atcphy_mode atcphy_typec_target_mode(struct apple_atcphy *atcphy, const struct typec_mux_state *state) @@ -3988,11 +4064,12 @@ static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux /* * With the lanes and the PIPE already as the mode needs them, only * USB3 is missing, and that must not fail what may be a DisplayPort - * update. The port stays off until the next update. + * update. Either way, retry the port from here. */ - if (ret && usb_err && in_mode) { - dev_dbg(atcphy->dev, "SuperSpeed port left off: %d\n", ret); - return 0; + if (ret && usb_err) { + atcphy_usb3_port_retry(atcphy, true, ret); + if (in_mode) + return 0; } return ret; } @@ -4289,6 +4366,9 @@ static int atcphy_probe(struct platform_device *pdev) atcphy->np = dev->of_node; mutex_init(&atcphy->lock); platform_set_drvdata(pdev, atcphy); + ret = devm_delayed_work_autocancel(dev, &atcphy->usb3_port_work, atcphy_usb3_port_work); + if (ret) + return ret; ret = atcphy_map_resources(pdev, atcphy); if (ret) From 34a3441ba3f2e517a30da86ff836747eeabc0592 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 05:51:41 +1100 Subject: [PATCH 1695/2680] usb: dwc3: apple: Document the fixed-hub SuperSpeed handover The comment at the top of the driver lays down the order in which dwc3 and its USB3 PHY have to be brought down and up around a PHY change, and that a controller behind a fixed hub never sees a role change. It does not say how such a controller keeps to that order when the SuperSpeed lanes change underneath it. Add that the USB core holds the controller's SuperSpeed root port powered off for the PHY, port runtime PM included, from before the PIPE moves until it is back on the USB3 PHY, while the USB2 side stays up. (cherry picked from commit 29431eb20dbd81c256b921f70d6f31015d49198d ("usb: dwc3: apple: Document the fixed-hub SuperSpeed handover")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/dwc3-apple.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index 49953a518eec9..464e5d9e691fe 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -76,7 +76,11 @@ * the board, whose downstream ports serve the Type-C connectors (Apple J700). Such a controller is * a host whatever happens at the connectors and never sees a role change: it is brought up in * DWC3_APPLE_HOST at probe and stays there. The Type-C port controllers only drive the PHY's - * Type-C mux for the SuperSpeed lanes, and the PHY follows those mode changes on its own. + * Type-C mux for the SuperSpeed lanes, and the PHY follows those mode changes on its own. It keeps + * to the order above for the SuperSpeed side: the USB core holds the controller's SuperSpeed root + * port powered off for the PHY, port runtime PM included, from before the PIPE goes to the dummy + * backend until the PIPE is back on the USB3 PHY. The USB2 root port and the hub on it stay up + * throughout. * * Whenever the core is up, for either kind of port, it is kept running across system sleep, see * dwc3_apple_init(). In host mode xHCI then keeps running in suspend-to-idle when a USB device From d84da3661e58105ae690ea98c27df723e9f8e1be Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 23:16:00 +1100 Subject: [PATCH 1696/2680] usb: core: Preserve the port hold when administrative disable fails A disable=1 write can take over a port held off for a PHY lane change. Dropping the hold before ClearPortFeature(POWER) succeeds loses both the runtime PM exclusion and the obligation to restore power if that request fails. A later PHY update then treats the unpowered port as someone else's. Publish administrative ownership and release the hold only after the power request succeeds. Keep the hold and its PM reference on failure, and retain the rejection of disable=0 while the PHY owns the port. Add a software-HCD regression test with an injected failed power request. Fixes: 8ccb06392f04 ("usb: hub: Add helpers to power a port off and on for a lane change") (cherry picked from commit 5b70b099a3003e104471dfa240515dac59b171a6 ("usb: core: Preserve the port hold when administrative disable fails")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/core/port-power-test.c | 28 ++++++++++++++++++++++++++++ drivers/usb/core/port.c | 14 +++++++------- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/drivers/usb/core/port-power-test.c b/drivers/usb/core/port-power-test.c index 2a5fee7fe6854..d4a0460c7a08b 100644 --- a/drivers/usb/core/port-power-test.c +++ b/drivers/usb/core/port-power-test.c @@ -26,6 +26,7 @@ struct port_power_test_hc { u16 status; u16 change; unsigned int power_ons; + bool fail_power_off; }; struct port_power_test { @@ -170,6 +171,10 @@ static int port_power_test_hub_control(struct usb_hcd *hcd, u16 type_req, u16 va hc->change |= USB_PORT_STAT_C_RESET | USB_PORT_STAT_C_BH_RESET; return 0; case ClearPortFeature: + if (value == USB_PORT_FEAT_POWER && hc->fail_power_off) { + hc->fail_power_off = false; + return -ENOMEM; + } if (value == USB_PORT_FEAT_POWER) port_power_test_set_power(hcd, false); else @@ -536,6 +541,28 @@ static void port_power_test_removed_held(struct kunit *test) usb_unlock_device(t->ss_root); } +/* Review: a failed takeover request must not consume the driver's PM hold. */ +static void port_power_review_failed_takeover(struct kunit *test) +{ + struct port_power_test *t = test->priv; + struct port_power_test_hc *hc = port_power_test_hc(t->shared_hcd); + int usage = port_power_test_usage(t->ss_port); + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + usb_unlock_device(t->ss_root); + scoped_guard(spinlock_irqsave, &hc->lock) + hc->fail_power_off = true; + KUNIT_EXPECT_EQ(test, port_power_test_write(test, t->ss_port, "disable", "1"), -ENOMEM); + KUNIT_EXPECT_EQ(test, port_power_test_usage(t->ss_port), usage + 1); + usb_lock_device(t->ss_root); + /* The next PHY update must still own its restoration. */ + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 0); + KUNIT_EXPECT_TRUE(test, port_power_test_powered(t->shared_hcd)); + usb_unlock_device(t->ss_root); +} + static struct kunit_case port_power_test_cases[] = { KUNIT_CASE(port_power_test_hold), KUNIT_CASE(port_power_test_runtime_pm), @@ -543,6 +570,7 @@ static struct kunit_case port_power_test_cases[] = { KUNIT_CASE(port_power_test_disable), KUNIT_CASE(port_power_test_disabled_suspended), KUNIT_CASE(port_power_test_removed_held), + KUNIT_CASE(port_power_review_failed_takeover), {} }; diff --git a/drivers/usb/core/port.c b/drivers/usb/core/port.c index 89066ef6ffdd0..721757e6242dc 100644 --- a/drivers/usb/core/port.c +++ b/drivers/usb/core/port.c @@ -157,13 +157,9 @@ static ssize_t disable_store(struct device *dev, struct device_attribute *attr, * in transition. Keeping it off takes the port over from the driver, * which then leaves it off. */ - if (port_dev->held_off) { - if (!disabled) { - rc = -EBUSY; - goto out_hdev_lock; - } - port_dev->held_off = false; - pm_runtime_put(&port_dev->dev); + if (port_dev->held_off && !disabled) { + rc = -EBUSY; + goto out_hdev_lock; } if (disabled && port_dev->child) @@ -180,6 +176,10 @@ static ssize_t disable_store(struct device *dev, struct device_attribute *attr, if (!rc) { port_dev->disabled = disabled; + if (port_dev->held_off) { + port_dev->held_off = false; + pm_runtime_put(&port_dev->dev); + } rc = count; } From 89c7701ee2295aa93eb4dbaa3ab421825939e81e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 23:17:23 +1100 Subject: [PATCH 1697/2680] usb: core: Keep administratively disabled ports off during runtime resume A PHY may move lanes after usb_hub_port_power_off() returns a nonnegative result. An administrative disable used to return early without excluding the port runtime-resume writer. Resuming its USB2 peer could then power the SuperSpeed port while the PHY changed lanes, even under the root-hub lock. Serialize disable and the runtime-resume power writer with a per-port mutex. Once disable succeeds, runtime resume leaves the port off, including the hub-reset path that records the desired power state. Take this mutex after resuming the peer and hub interface, never the hub device lock: the hub worker waits for port PM while holding that lock. Reassert power-off for administrative ports before allowing the lane change, since a disabled flag alone does not establish the physical state. Leave their ownership with sysfs and propagate any failed power request. Every other port now acquires a PHY hold, even if already unpowered, so there is no other successful path without runtime-PM exclusion. Retain the peer-resume counterexample and test a previously repowered administrative port, failed reassertion, retry, and explicit enable. Fixes: 8ccb06392f04 ("usb: hub: Add helpers to power a port off and on for a lane change") (cherry picked from commit fff435f15b8a259ece48395518cc5cf6d5545e26 ("usb: core: Keep administratively disabled ports off during runtime resume")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/core/hub.c | 51 +++++++++-------- drivers/usb/core/hub.h | 2 + drivers/usb/core/port-power-test.c | 89 ++++++++++++++++++++++++++++++ drivers/usb/core/port.c | 24 ++++++-- 4 files changed, 136 insertions(+), 30 deletions(-) diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index 47b4e4ba4054a..f2e6718c3dfc4 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -2149,15 +2149,17 @@ bool usb_device_is_owned(struct usb_device *udev) * Writing 0 to its "disable" attribute fails with -EBUSY; writing 1 takes the * port over, after which usb_hub_port_power_on() leaves it off. * - * A port that is off for someone else, say through its "disable" attribute, - * is left as it is, and the caller can tell from the return value that it is - * not held for it. A port that runtime PM had switched off is not such a - * port: it is resumed, then held off for the caller. + * An administratively disabled port is powered off again, in case it was + * repowered before the handover. It stays under administrative ownership: + * runtime resume leaves it off, and the caller's hub lock prevents a write + * to "disable" from enabling it during the lane change. The return value + * distinguishes this from a port held for the caller. Other ports, including + * those switched off by runtime PM, are resumed and held off for the caller. * * Context: process context; the caller holds @hdev's device lock. * * Return: 0 if the port is held off for the caller, also if it was already, - * 1 if it is off for someone else, -EOPNOTSUPP if the hub cannot switch the + * 1 if it is administratively disabled, -EOPNOTSUPP if the hub cannot switch the * power of a single port, or another negative error code. */ int usb_hub_port_power_off(struct usb_device *hdev, int port1) @@ -2165,6 +2167,7 @@ int usb_hub_port_power_off(struct usb_device *hdev, int port1) struct usb_hub *hub = usb_hub_to_struct_hub(hdev); struct usb_interface *intf; struct usb_port *port_dev; + bool disabled; int ret; device_lock_assert(&hdev->dev); @@ -2179,9 +2182,7 @@ int usb_hub_port_power_off(struct usb_device *hdev, int port1) port_dev = hub->ports[port1 - 1]; if (port_dev->held_off) return 0; - /* Runtime PM may have suspended it since, which does not make it ours. */ - if (port_dev->disabled) - return 1; + disabled = port_dev->disabled; intf = to_usb_interface(hub->intfdev); ret = usb_autopm_get_interface(intf); @@ -2189,21 +2190,18 @@ int usb_hub_port_power_off(struct usb_device *hdev, int port1) return ret; /* - * The port's runtime resume switches its power on, and does not take - * the hub lock, as hub_event() waits for it under that lock. Keep port - * runtime PM out instead, after waiting for any of it in progress. + * Administrative disable excludes the runtime-resume power writer under + * power_lock. For other ports, wait for any port PM and keep it out + * until power-on. Neither path takes the hub lock in the PM callback. */ - ret = pm_runtime_resume_and_get(&port_dev->dev); - if (ret < 0) { - /* After a runtime PM error, nothing resumes the port any more. */ - if (!pm_runtime_active(&port_dev->dev)) - goto out; - pm_runtime_get_noresume(&port_dev->dev); - } - - if (!port_dev->child && !test_bit(port1, hub->power_bits)) { - ret = 1; - goto put_port; + if (!disabled) { + ret = pm_runtime_resume_and_get(&port_dev->dev); + if (ret < 0) { + /* After a runtime PM error, nothing resumes the port any more. */ + if (!pm_runtime_active(&port_dev->dev)) + goto out; + pm_runtime_get_noresume(&port_dev->dev); + } } if (port_dev->child) @@ -2218,7 +2216,7 @@ int usb_hub_port_power_off(struct usb_device *hdev, int port1) } /* The runtime PM reference is dropped once the hold ends. */ - port_dev->held_off = true; + port_dev->held_off = !disabled; /* Nothing hub_wq still has queued for the port applies to it off. */ clear_bit(port1, hub->change_bits); clear_bit(port1, hub->warm_reset_bits); @@ -2227,11 +2225,12 @@ int usb_hub_port_power_off(struct usb_device *hdev, int port1) usb_clear_port_feature(hdev, port1, USB_PORT_FEAT_C_CONNECTION); if (!port_dev->is_superspeed) usb_clear_port_feature(hdev, port1, USB_PORT_FEAT_C_ENABLE); - usb_autopm_put_interface(intf); - return 0; + ret = disabled; + goto out; put_port: - pm_runtime_put(&port_dev->dev); + if (!disabled) + pm_runtime_put(&port_dev->dev); out: usb_autopm_put_interface(intf); return ret; diff --git a/drivers/usb/core/hub.h b/drivers/usb/core/hub.h index 3c4465e378ef4..aabaf445b012e 100644 --- a/drivers/usb/core/hub.h +++ b/drivers/usb/core/hub.h @@ -96,6 +96,7 @@ struct usb_hub { * @state_kn: kernfs_node of the sysfs attribute that accesses @state * @location: opaque representation of platform connector location * @status_lock: synchronize port_event() vs usb_port_{suspend|resume} + * @power_lock: serialize administrative disable with runtime resume * @portnum: port index num based one * @is_superspeed cache super-speed status * @usb3_lpm_u1_permit: whether USB3 U1 LPM is permitted. @@ -118,6 +119,7 @@ struct usb_port { struct kernfs_node *state_kn; usb_port_location_t location; struct mutex status_lock; + struct mutex power_lock; /* administrative disable vs runtime resume */ u32 over_current_count; u8 portnum; u32 quirks; diff --git a/drivers/usb/core/port-power-test.c b/drivers/usb/core/port-power-test.c index d4a0460c7a08b..9fe6ef0eb6d83 100644 --- a/drivers/usb/core/port-power-test.c +++ b/drivers/usb/core/port-power-test.c @@ -541,6 +541,29 @@ static void port_power_test_removed_held(struct kunit *test) usb_unlock_device(t->ss_root); } +/* Review: admin-disabled SS port must stay off while the mux changes lanes. */ +static void port_power_review_disabled_peer_resume(struct kunit *test) +{ + struct port_power_test *t = test->priv; + + KUNIT_ASSERT_EQ(test, port_power_test_write(test, t->ss_port, "disable", "1"), 0); + port_power_test_allow_power_off(test); + KUNIT_ASSERT_GE(test, pm_runtime_suspend(t->hs_port), 0); + KUNIT_ASSERT_GE(test, pm_runtime_suspend(t->ss_port), 0); + KUNIT_ASSERT_TRUE(test, pm_runtime_suspended(t->ss_port)); + KUNIT_ASSERT_FALSE(test, port_power_test_powered(t->shared_hcd)); + + /* The same root-hub lock the mux holds over the PIPE/lane change. */ + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 1); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + /* A peer resume is not excluded by that root-hub lock. */ + KUNIT_ASSERT_GE(test, pm_runtime_resume_and_get(t->hs_port), 0); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + usb_unlock_device(t->ss_root); + pm_runtime_put(t->hs_port); +} + /* Review: a failed takeover request must not consume the driver's PM hold. */ static void port_power_review_failed_takeover(struct kunit *test) { @@ -563,6 +586,69 @@ static void port_power_review_failed_takeover(struct kunit *test) usb_unlock_device(t->ss_root); } +/* A previously repowered administrative port must be off before moving lanes. */ +static void port_power_test_disabled_repowered(struct kunit *test) +{ + struct port_power_test *t = test->priv; + struct port_power_test_hc *hc = port_power_test_hc(t->shared_hcd); + int usage = port_power_test_usage(t->ss_port); + + KUNIT_ASSERT_EQ(test, port_power_test_write(test, t->ss_port, "disable", "1"), 0); + /* Model power restored behind usbcore's back, with disable still set. */ + scoped_guard(spinlock_irqsave, &hc->lock) + port_power_test_set_power(t->shared_hcd, true); + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 1); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 1); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + + /* A failed reassertion must not permit the lane change. */ + scoped_guard(spinlock_irqsave, &hc->lock) { + port_power_test_set_power(t->shared_hcd, true); + hc->fail_power_off = true; + } + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), -ENOMEM); + KUNIT_EXPECT_TRUE(test, port_power_test_powered(t->shared_hcd)); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 1); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 1); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + usb_unlock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, port_power_test_usage(t->ss_port), usage); + + /* Ending the handover leaves ownership with sysfs. */ + KUNIT_EXPECT_EQ(test, port_power_test_write(test, t->ss_port, "disable", "0"), 0); + KUNIT_EXPECT_TRUE(test, port_power_test_powered(t->shared_hcd)); +} + +/* An unpowered port without an administrative owner still needs a PM hold. */ +static void port_power_test_unowned_off(struct kunit *test) +{ + struct port_power_test *t = test->priv; + int usage; + + port_power_test_allow_power_off(test); + KUNIT_ASSERT_GE(test, pm_runtime_suspend(t->hs_port), 0); + KUNIT_ASSERT_GE(test, pm_runtime_suspend(t->ss_port), 0); + KUNIT_ASSERT_FALSE(test, port_power_test_powered(t->shared_hcd)); + /* Model an active port whose PM is unavailable and PP is still off. */ + pm_runtime_disable(t->ss_port); + KUNIT_EXPECT_EQ(test, pm_runtime_set_active(t->ss_port), 0); + usage = port_power_test_usage(t->ss_port); + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + KUNIT_EXPECT_EQ(test, port_power_test_usage(t->ss_port), usage + 1); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 0); + KUNIT_EXPECT_TRUE(test, port_power_test_powered(t->shared_hcd)); + KUNIT_EXPECT_EQ(test, port_power_test_usage(t->ss_port), usage); + usb_unlock_device(t->ss_root); + pm_runtime_enable(t->ss_port); +} + static struct kunit_case port_power_test_cases[] = { KUNIT_CASE(port_power_test_hold), KUNIT_CASE(port_power_test_runtime_pm), @@ -570,6 +656,9 @@ static struct kunit_case port_power_test_cases[] = { KUNIT_CASE(port_power_test_disable), KUNIT_CASE(port_power_test_disabled_suspended), KUNIT_CASE(port_power_test_removed_held), + KUNIT_CASE(port_power_review_disabled_peer_resume), + KUNIT_CASE(port_power_test_disabled_repowered), + KUNIT_CASE(port_power_test_unowned_off), KUNIT_CASE(port_power_review_failed_takeover), {} }; diff --git a/drivers/usb/core/port.c b/drivers/usb/core/port.c index 721757e6242dc..7f0d8960a2e2a 100644 --- a/drivers/usb/core/port.c +++ b/drivers/usb/core/port.c @@ -165,7 +165,12 @@ static ssize_t disable_store(struct device *dev, struct device_attribute *attr, if (disabled && port_dev->child) usb_disconnect(&port_dev->child); - rc = usb_hub_set_port_power(hdev, hub, port1, !disabled); + /* Runtime resume must not repower the port after a successful disable. */ + scoped_guard(mutex, &port_dev->power_lock) { + rc = usb_hub_set_port_power(hdev, hub, port1, !disabled); + if (!rc) + port_dev->disabled = disabled; + } msleep(2 * hub_power_on_good_delay(hub)); if (disabled) { @@ -175,7 +180,6 @@ static ssize_t disable_store(struct device *dev, struct device_attribute *attr, } if (!rc) { - port_dev->disabled = disabled; if (port_dev->held_off) { port_dev->held_off = false; pm_runtime_put(&port_dev->dev); @@ -394,7 +398,10 @@ static int usb_port_runtime_resume(struct device *dev) if (!hub) return -EINVAL; if (hub->in_reset) { - set_bit(port1, hub->power_bits); + guard(mutex)(&port_dev->power_lock); + + if (!port_dev->disabled) + set_bit(port1, hub->power_bits); return 0; } @@ -409,7 +416,14 @@ static int usb_port_runtime_resume(struct device *dev) if (retval < 0) return retval; - retval = usb_hub_set_port_power(hdev, hub, port1, true); + /* hub_event() waits for port PM under the hub lock; do not take it here. */ + scoped_guard(mutex, &port_dev->power_lock) { + if (port_dev->disabled) { + retval = 0; + goto out; + } + retval = usb_hub_set_port_power(hdev, hub, port1, true); + } msleep(hub_power_on_good_delay(hub)); if (udev && !retval) { /* @@ -434,6 +448,7 @@ static int usb_port_runtime_resume(struct device *dev) } } +out: usb_autopm_put_interface(intf); return retval; @@ -800,6 +815,7 @@ int usb_hub_create_port_device(struct usb_hub *hub, int port1) dev_set_name(&port_dev->dev, "%s-port%d", dev_name(&hub->hdev->dev), port1); mutex_init(&port_dev->status_lock); + mutex_init(&port_dev->power_lock); retval = device_register(&port_dev->dev); if (retval) { put_device(&port_dev->dev); From c974caf06a33afee99ffdc4454ae58d7d90737b1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 19:34:47 +1100 Subject: [PATCH 1698/2680] usb: typec: mux: Let a provider declare its timeouts transient A mux provider can time out on a change that its hardware holds up for a while, and that goes through later unchanged. On a J700 the ATC PHY cannot park the PIPE of the controller behind the fixed hub while that controller still drives it, which a partner renegotiating power can keep up for several seconds. Such a timeout says nothing wrong about the state asked for, and a Type-C driver that gives the state up after a few quick retries leaves the port in the wrong mode until the next cable event. Let a provider declare that -ETIMEDOUT from its set callback is such a hold-up. That is a property of the provider, not of a mux handle: one handle can hold several providers, and on the rear connector of a J700 it holds the PHY and a DCP display route, whose own timeouts can be permanent. So note, for each request, whether the provider that failed it timed out having declared its timeouts transient, and let the consumer ask with typec_mux_timeout_transient() after a request that timed out. Nothing declares it yet, so no consumer changes. The apple-dcp host selftests extract the transaction code; their mux shims gain the two new fields. [ryan: split by topic: only the Type-C mux part; the apple-dcp selftests part is in 200-dcp] (cherry picked from commit 373c3271edb7da0489b5ab4ec63e1e199ade74f8 ("usb: typec: mux: Let a provider declare its timeouts transient")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/mux.c | 29 +++++++++++++++++++++++++++++ drivers/usb/typec/mux.h | 1 + include/linux/usb/typec_mux.h | 13 +++++++++++++ 3 files changed, 43 insertions(+) diff --git a/drivers/usb/typec/mux.c b/drivers/usb/typec/mux.c index 549f68585cf42..d79453bdfdcfd 100644 --- a/drivers/usb/typec/mux.c +++ b/drivers/usb/typec/mux.c @@ -284,6 +284,8 @@ struct typec_mux { int srcu_index; unsigned int prepared; bool applied; + /* The last request timed out in a provider with transient timeouts. */ + bool timeout_transient; }; static struct typec_mux_context *typec_mux_context_get(struct fwnode_handle *fwnode) @@ -334,6 +336,28 @@ bool typec_mux_safe_after_role(struct typec_mux *mux) } EXPORT_SYMBOL_GPL(typec_mux_safe_after_role); +/** + * typec_mux_timeout_transient - Check whether a mux request timed out for now + * @mux: Mux handle + * + * For use after a request on @mux, by typec_mux_set() or by typec_mux_begin() + * and the calls that follow it, failed with -ETIMEDOUT. A handle can hold more + * than one provider, and only the one that timed out is asked: whether it + * declared its timeouts transient (&typec_mux_desc.transient_timeouts). Its + * hardware then only held the change up, and the same state can be applied + * again later. A timeout of any other provider can be permanent. + * + * The answer is about the last request on @mux: no other request on the same + * handle may run in between. + * + * Return: true if the provider that timed out declared its timeouts transient. + */ +bool typec_mux_timeout_transient(struct typec_mux *mux) +{ + return !IS_ERR_OR_NULL(mux) && mux->timeout_transient; +} +EXPORT_SYMBOL_GPL(typec_mux_timeout_transient); + static int mux_fwnode_match(struct device *dev, const void *fwnode) { if (!is_typec_mux_dev(dev)) @@ -523,6 +547,7 @@ int typec_mux_begin(struct typec_mux *mux, struct typec_mux_state *state) mux->srcu_index = index; mux->owner = current; mux->transaction = *state; + mux->timeout_transient = false; for (i = 0; i < mux->num_mux_devs; i++) { struct typec_mux_dev *dev = mux->mux_devs[i]; @@ -564,6 +589,9 @@ int typec_mux_apply(struct typec_mux *mux) ret = dev->set(dev, &mux->transaction); if (ret && ret != -EOPNOTSUPP) { + /* Only the provider that failed knows what its timeout means. */ + mux->timeout_transient = ret == -ETIMEDOUT && + dev->transient_timeouts; typec_mux_abort(mux); return ret; } @@ -669,6 +697,7 @@ typec_mux_register(struct device *parent, const struct typec_mux_desc *desc) mux_dev->commit = desc->commit; mux_dev->abort = desc->abort; mux_dev->safe_after_role = desc->safe_after_role; + mux_dev->transient_timeouts = desc->transient_timeouts; device_initialize(&mux_dev->dev); mux_dev->dev.parent = parent; diff --git a/drivers/usb/typec/mux.h b/drivers/usb/typec/mux.h index 823c5529ca675..89c6061e0863f 100644 --- a/drivers/usb/typec/mux.h +++ b/drivers/usb/typec/mux.h @@ -21,6 +21,7 @@ struct typec_mux_dev { typec_mux_notify_fn_t abort; bool removed; bool safe_after_role; + bool transient_timeouts; }; #define to_typec_switch_dev(_dev_) container_of(_dev_, struct typec_switch_dev, dev) diff --git a/include/linux/usb/typec_mux.h b/include/linux/usb/typec_mux.h index 185da86c43d25..f853143048ded 100644 --- a/include/linux/usb/typec_mux.h +++ b/include/linux/usb/typec_mux.h @@ -105,6 +105,13 @@ struct typec_mux_desc { typec_mux_notify_fn_t abort; /* SAFE needs the USB role consumer to stop the shared PHY first. */ bool safe_after_role; + /* + * -ETIMEDOUT from @set only means that the hardware held the change up + * for now: the same state can be applied again later, and is worth + * applying again for longer than after other failures. Consumers learn + * it from typec_mux_timeout_transient(). + */ + bool transient_timeouts; }; #if IS_ENABLED(CONFIG_TYPEC) @@ -117,6 +124,7 @@ int typec_mux_begin(struct typec_mux *mux, struct typec_mux_state *state); /* Apply the providers while retaining the transaction and ownership. */ int typec_mux_apply(struct typec_mux *mux); bool typec_mux_safe_after_role(struct typec_mux *mux); +bool typec_mux_timeout_transient(struct typec_mux *mux); int typec_mux_end(struct typec_mux *mux); void typec_mux_abort(struct typec_mux *mux); int typec_mux_prepare(struct typec_mux *mux, struct typec_mux_state *state); @@ -150,6 +158,11 @@ static inline bool typec_mux_safe_after_role(struct typec_mux *mux) return false; } +static inline bool typec_mux_timeout_transient(struct typec_mux *mux) +{ + return false; +} + static inline int typec_mux_end(struct typec_mux *mux) { return 0; } static inline void typec_mux_abort(struct typec_mux *mux) {} From 63c3a7ece83b7c3c7b324ced2af3fbbc5a236eb4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 19:45:51 +1100 Subject: [PATCH 1699/2680] phy: apple: atc: Declare the J700 fixed-hub mux timeouts transient On a J700 the rear port's controller keeps running across Type-C mode changes, so the PHY parks its PIPE itself, after taking the controller's SuperSpeed root port down. The PIPE lock is only acknowledged once the controller lets go of the PIPE. With a Mac on the rear port that did not happen within the lock timeout twice in one day, both times in a burst of SMC power source elections, 11 and 18 "Elec Cause" lines over a few seconds, through which the Mac took the device role: phy-apple-atc 40b000000.phy: Pipehandler lock not acked. phy-apple-atc 40b000000.phy: Failed to switch PIPE to dummy: -110 sn201202x 1-0c: failed to place Type-C mux in safe mode: -110 The Mac then stayed on USB2 behind the fixed hub for the rest of the boot. A refused change leaves the lanes dirty and the SuperSpeed port down, and the next mux update that gets through puts both right: the safe state parks the PIPE, and the USB state routes it to USB3 and powers the port on. But the Type-C driver tries its whole state only a few times, about a second apart, which such a burst outlasts, and then waits for the next cable event, which need not come. Declare the mux timeouts of a J700 fixed-hub port transient, so that the Type-C driver keeps applying its whole state until it goes through. Ports without a fixed hub, every M1 and M2 port among them, and fixed-hub ports on other PHYs do not declare it. Only the PHY's own timeouts count: the DCP display route on the same connector declares nothing, and its timeouts keep the bounded retries. The declaration covers the PHY's other timeouts too, those of its power and PLL polls, which retrying does not clear. Telling them apart from a refused lock would mean changing what the shared poll helpers return to every other caller, and the Type-C driver stops retrying after about two minutes, which bounds them as well. The new tests run the safe state of a data role swap, and the USB state after it, against a PIPE lock that is refused until the elections settle, with the real mode change between stubbed USB steps. However often the lock was refused, a refusal leaves the port down with nothing queued, and the safe state and then the USB state applied again bring SuperSpeed back. They also check which PHYs declare the timeouts transient. (cherry picked from commit 81903767d050f2d39d2baf402539ce0fb831e2cd ("phy: apple: atc: Declare the J700 fixed-hub mux timeouts transient")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-test.c | 173 +++++++++++++++++++++++++++++++++++ drivers/phy/apple/atc.c | 38 +++++++- 2 files changed, 210 insertions(+), 1 deletion(-) diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index 65e82a6d6edf0..6efddf059e18e 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -679,6 +679,176 @@ static void atcphy_test_host_start_without_usb3(struct kunit *test) KUNIT_EXPECT_EQ(test, atcphy_usb3_set_mode(phy, PHY_MODE_USB_HOST, 0), -EBUSY); } +/* + * The controller as the PIPE lock sees it: while @held, as through the PD + * elections of a Mac taking another data role, it does not acknowledge the + * lock. The USB side stubs share @steps. + */ +struct atcphy_test_election { + struct atcphy_test_steps steps; + bool held; +}; + +static int atcphy_test_lock(struct apple_atcphy *atcphy) +{ + struct atcphy_test_election *election = kunit_get_current_test()->priv; + + return election->held ? -ETIMEDOUT : 0; +} + +/* The host BIST of a USB3 handoff, which zeroed registers cannot run */ +static int atcphy_test_bist(struct apple_atcphy *atcphy) +{ + return 0; +} + +/* The lane change, as far as the PIPE handling depends on it */ +static int atcphy_test_lanes(struct apple_atcphy *atcphy, enum atcphy_mode mode) +{ + lockdep_assert_held(&atcphy->lock); + atcphy->mode = mode; + atcphy->mode_dirty = false; + return 0; +} + +/* + * The rear port of a J700 with a Mac on it: USB3, with the PIPE on the USB3 + * backend of the running host. The real mode change runs between the recorded + * USB side steps of atcphy_test_stub_steps(). + */ +static struct apple_atcphy *atcphy_test_alloc_election(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct atcphy_test_election *election; + + election = kunit_kzalloc(test, sizeof(*election), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, election); + test->priv = election; + kunit_activate_static_stub(test, atcphy_pipehandler_lock, atcphy_test_lock); + kunit_activate_static_stub(test, atcphy_configure_pipehandler_usb3_host_t8122, + atcphy_test_bist); + kunit_activate_static_stub(test, atcphy_configure, atcphy_test_lanes); + kunit_activate_static_stub(test, atcphy_usb3_pm_get, atcphy_test_pm_get); + kunit_activate_static_stub(test, atcphy_usb3_port_off, atcphy_test_port_off); + kunit_activate_static_stub(test, atcphy_usb3_port_on, atcphy_test_port_on); + kunit_activate_static_stub(test, atcphy_usb3_port_retry_delay, atcphy_test_retry_delay); + atcphy_test_fake_regs(test, atcphy); + atcphy->mode = APPLE_ATCPHY_MODE_USB3; + atcphy->typec_mode = APPLE_ATCPHY_MODE_USB3; + + return atcphy; +} + +static const struct typec_mux_state atcphy_test_safe = { .mode = TYPEC_STATE_SAFE }; +static const struct typec_mux_state atcphy_test_usb = { .mode = TYPEC_STATE_USB }; + +/* A refused PIPE change leaves the port down, and nothing queued */ +static void atcphy_test_election_refused(struct kunit *test, struct apple_atcphy *atcphy, + const struct typec_mux_state *state) +{ + struct atcphy_test_election *election = test->priv; + + memset(election->steps.log, 0, sizeof(election->steps.log)); + KUNIT_EXPECT_EQ(test, atcphy_mux_update(atcphy, state), -ETIMEDOUT); + KUNIT_EXPECT_STREQ(test, election->steps.log, "g0"); + KUNIT_EXPECT_TRUE(test, atcphy->usb3_port_off); + KUNIT_EXPECT_FALSE(test, atcphy_usb3_port_ready(atcphy)); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&atcphy->usb3_port_work)); +} + +/* The safe state and then the USB state, as the Type-C driver applies them again */ +static void atcphy_test_election_redrive(struct kunit *test, struct apple_atcphy *atcphy) +{ + struct atcphy_test_election *election = test->priv; + char *log = election->steps.log; + + memset(log, 0, sizeof(election->steps.log)); + KUNIT_EXPECT_EQ(test, atcphy_mux_update(atcphy, &atcphy_test_safe), 0); + KUNIT_EXPECT_STREQ(test, log, "g0"); + KUNIT_EXPECT_EQ(test, atcphy->mode, APPLE_ATCPHY_MODE_USB2); + KUNIT_EXPECT_FALSE(test, atcphy->mode_dirty); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); + KUNIT_EXPECT_TRUE(test, atcphy->usb3_port_off); + + memset(log, 0, sizeof(election->steps.log)); + KUNIT_EXPECT_EQ(test, atcphy_mux_update(atcphy, &atcphy_test_usb), 0); + KUNIT_EXPECT_STREQ(test, log, "g01"); + KUNIT_EXPECT_EQ(test, atcphy->mode, APPLE_ATCPHY_MODE_USB3); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_USB3); + KUNIT_EXPECT_FALSE(test, atcphy->pipe_pending); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); + KUNIT_EXPECT_TRUE(test, atcphy_usb3_port_ready(atcphy)); +} + +/* + * SuperSpeed comes back after a data role swap whose safe state found the PIPE + * held, however many times it was refused, once the Type-C driver applies its + * whole state again after the elections. + */ +static void atcphy_test_election_held_safe(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_election(test); + struct atcphy_test_election *election = test->priv; + unsigned int i; + + election->held = true; + /* Many more times than the Type-C driver used to retry */ + for (i = 0; i < 8; i++) + atcphy_test_election_refused(test, atcphy, &atcphy_test_safe); + /* The PIPE could not be parked, so the lanes were left alone */ + KUNIT_EXPECT_TRUE(test, atcphy->mode_dirty); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_USB3); + + election->held = false; + atcphy_test_election_redrive(test, atcphy); +} + +/* The same if the elections go on into the USB state that follows the safe one */ +static void atcphy_test_election_held_usb(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_election(test); + struct atcphy_test_election *election = test->priv; + + election->held = true; + atcphy_test_election_refused(test, atcphy, &atcphy_test_safe); + election->held = false; + KUNIT_EXPECT_EQ(test, atcphy_mux_update(atcphy, &atcphy_test_safe), 0); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); + + /* The lanes are rebuilt for USB3, but the PIPE cannot follow */ + election->held = true; + atcphy_test_election_refused(test, atcphy, &atcphy_test_usb); + KUNIT_EXPECT_EQ(test, atcphy->mode, APPLE_ATCPHY_MODE_USB3); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); + + election->held = false; + atcphy_test_election_redrive(test, atcphy); +} + +/* + * Only the fixed-hub ports of a J700 have the Type-C driver retry a timeout + * until it passes; every M1 and M2 port, and fixed-hub ports on other PHYs, + * keep its bounded retries. + */ +static void atcphy_test_transient_timeouts(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc(test, true); + + atcphy->hw = &atcphy_hw_t8140; + KUNIT_EXPECT_TRUE(test, atcphy_mux_transient_timeouts(atcphy)); + atcphy->fixed_usb2 = false; + KUNIT_EXPECT_FALSE(test, atcphy_mux_transient_timeouts(atcphy)); + + atcphy->hw = &atcphy_hw_t8103; + KUNIT_EXPECT_FALSE(test, atcphy_mux_transient_timeouts(atcphy)); + atcphy->hw = &atcphy_hw_t8122; + KUNIT_EXPECT_FALSE(test, atcphy_mux_transient_timeouts(atcphy)); + atcphy->fixed_usb2 = true; + KUNIT_EXPECT_FALSE(test, atcphy_mux_transient_timeouts(atcphy)); + atcphy->hw = &atcphy_hw_t8142; + KUNIT_EXPECT_FALSE(test, atcphy_mux_transient_timeouts(atcphy)); +} + #if IS_REACHABLE(CONFIG_USB) /* Only the controller's SuperSpeed root hub is ever handed the port changes */ static void atcphy_test_usb3_root(struct kunit *test) @@ -825,6 +995,9 @@ static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_stopped_power_off), KUNIT_CASE(atcphy_test_stopped_reset), KUNIT_CASE(atcphy_test_host_start_without_usb3), + KUNIT_CASE(atcphy_test_election_held_safe), + KUNIT_CASE(atcphy_test_election_held_usb), + KUNIT_CASE(atcphy_test_transient_timeouts), #if IS_REACHABLE(CONFIG_USB) KUNIT_CASE(atcphy_test_usb3_root), #endif diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 5e73be16e0788..9d5fa415beac1 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1116,8 +1116,13 @@ static int atcphy_pipehandler_lock(struct apple_atcphy *atcphy) { int ret; u32 reg; - u32 requested = readl(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ); + u32 requested; + /* The tests stand in for the controller that acknowledges the lock */ + if (IS_ENABLED(CONFIG_PHY_APPLE_ATC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(atcphy_pipehandler_lock, atcphy); + + requested = readl(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ); dev_dbg(atcphy->dev, "PIPE request: req=%#x mode=%u pipe=%u host=%u dirty=%u\n", requested, atcphy->mode, atcphy->pipe_state, atcphy->host_active, atcphy->mode_dirty); @@ -1189,6 +1194,9 @@ static int atcphy_configure_pipehandler_usb3_host_t8122(struct apple_atcphy *atc int ret, error = 0; u32 reg; + if (IS_ENABLED(CONFIG_PHY_APPLE_ATC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(atcphy_configure_pipehandler_usb3_host_t8122, atcphy); + core_set32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_CLOCK_EN); /* This should be a mask, but unsure what bits should be cleared */ writel(ACIOPHY_BIST_MAC_USBPHY_CFG_OV1_LN0_PIPE_RESET_N_OV | FIELD_PREP(ACIOPHY_BIST_MAC_USBPHY_CFG_OV1_LN0_PWR_DOWN_OV, 2), @@ -4084,6 +4092,33 @@ static void atcphy_typec_mux_unregister(void *data) typec_mux_unregister(data); } +/* + * A controller behind a fixed hub keeps running across mode changes, so the PHY + * parks its PIPE with the controller's SuperSpeed port taken down. The PIPE + * lock is still only acknowledged once the controller lets go of the PIPE, and + * on a J700 that need not happen within the lock timeout while the partner + * renegotiates. A Mac on the port did so through a burst of power elections + * lasting several seconds, and the safe state of its data role swap then found + * the PIPE held: + * + * phy-apple-atc 40b000000.phy: Pipehandler lock not acked. + * phy-apple-atc 40b000000.phy: Failed to switch PIPE to dummy: -110 + * sn201202x 1-0c: failed to place Type-C mux in safe mode: -110 + * + * The failed change leaves the lanes dirty and the SuperSpeed port down, and + * the next mux update that gets through puts both right. Such a timeout says + * nothing about the state asked for, so tell the Type-C driver to keep applying + * its whole state, the safe state and then the USB one, until it goes through. + * Only a timeout of this PHY counts, not one of the display route that shares + * the connector. The PHY's other timeouts, of its power and PLL polls, are + * faults that retrying does not clear, but the Type-C driver stops retrying + * after about two minutes, which bounds those as well. + */ +static bool atcphy_mux_transient_timeouts(const struct apple_atcphy *atcphy) +{ + return atcphy->fixed_usb2 && atcphy->hw->has_t8140_dp; +} + static int atcphy_probe_mux(struct apple_atcphy *atcphy) { struct typec_mux_dev *mux; @@ -4092,6 +4127,7 @@ static int atcphy_probe_mux(struct apple_atcphy *atcphy) .fwnode = atcphy->dev->fwnode, .set = atcphy_mux_set, .safe_after_role = !atcphy->fixed_usb2, + .transient_timeouts = atcphy_mux_transient_timeouts(atcphy), }; mux = typec_mux_register(atcphy->dev, &mux_desc); From 7e27733cb5b8d0056d164545ecc4ef03cefaa6a9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 19:45:51 +1100 Subject: [PATCH 1700/2680] usb: typec: tipd: Keep applying a Type-C state that a busy mux holds up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a mux update fails, the CD321x update work hands the port to the cable-state recovery: a fresh read and a new attempt at the whole state, three times, then nothing until the next controller interrupt. Each attempt takes about a second, the resume read and the update being debounced by half a second each. On the rear port of a J700 that is not enough. The PHY cannot park its PIPE while the controller behind the fixed hub still drives it, and a Mac renegotiating power, in bursts of up to about 5.5 s, kept that up past the last attempt. Its safe state failed with -ETIMEDOUT each time, so the USB state of the new data role was never asked for, and with no further interrupt the Mac stayed on USB2 for the rest of the boot. Such a failure says nothing about the cable state, which was read fine. When the mux provider that failed a change timed out having declared its timeouts transient, do as for events that could not be acknowledged: do not charge the attempt to the cable-state budget. Apply the whole state again from a fresh read, starting with the safe state, after a delay that doubles from 500 ms up to 10 s. A controller event during the backoff brings the retry forward to the usual debounce, so that its state is applied promptly and the retry that was due later finds nothing left to do. Sleep cancels the retry and the resume read takes it over; removal, or any failure that is not such a hold-up, ends it, the latter going back to the bounded retries with no backoff left over. Only the first failure of such a series is reported, and the change that finally goes through is logged. The provider cannot tell a held-up lock from its other timeouts, so a PHY that stays stuck is not retried for good: after 16 retries, some two minutes and over twenty times the longest burst seen, the port gives up with one warning, as the budget does, and the next controller event starts over. The rear connector's mux handle holds the DCP display route as well as the PHY, and the route declares nothing, so whether a change was held up is asked of the provider that failed it, right after each mux request. The safe state of the update work now goes through the driver's own handle too, the one it asks, rather than the port's: both are the connector's, with the same providers, and the state is the same. Only the PHY of a J700 fixed-hub port declares its timeouts transient. For every other provider, any M1 or M2 port's among them and the DCP display routes, and for any other error, the driver behaves as before. The new tests drive the interrupt, resume read and update work of a port whose mux is refused for longer than the budget lasts, and expect the safe state and then the USB state once it lets go, with no further event, also on a J700 rear connector, where the display route follows the PHY on the handle. They also cover the backoff and its limit, an event that takes over from the backoff, sleep and removal during the retries, giving up on a PHY that stays stuck until the next event, a series that another error ends, and that other muxes, the display route on the J700 connector, and other errors keep the bounded retries. The apple-dcp host selftests follow: the units that extract the update work extract the new helpers with it, against a mux that does not declare its timeouts transient, and the probe unit runs the new state transitions. [ryan: split by topic: only the tipd part; the apple-dcp selftests part is in 200-dcp. Adapted to 7.2.10 — the mux test takes the CD321x data from tipd_hw_variants[TIPD_VARIANT_CD321X]: 7.2.10 keeps the variant data static behind that exported table instead of exporting tipd_cd321x_data] Fixes: 70f7c21e7d32 ("phy: apple: atc: Route the PIPE from the Type-C mux on a fixed-hub port") (cherry picked from commit ed289aabb019c8dfb1ee8575ba55430d8f43089c ("usb: typec: tipd: Keep applying a Type-C state that a busy mux holds up")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/Kconfig | 5 + drivers/usb/typec/tipd/Makefile | 2 +- drivers/usb/typec/tipd/cd321x-mux-test.c | 812 +++++++++++++++++++++++ drivers/usb/typec/tipd/cd321x-pm-test.c | 137 ++++ drivers/usb/typec/tipd/cd321x-pm.h | 74 +++ drivers/usb/typec/tipd/core.c | 90 ++- 6 files changed, 1105 insertions(+), 15 deletions(-) create mode 100644 drivers/usb/typec/tipd/cd321x-mux-test.c diff --git a/drivers/usb/typec/tipd/Kconfig b/drivers/usb/typec/tipd/Kconfig index b8a904446109a..e0ee8fa1f55b6 100644 --- a/drivers/usb/typec/tipd/Kconfig +++ b/drivers/usb/typec/tipd/Kconfig @@ -42,6 +42,11 @@ config TYPEC_TIPD_KUNIT_TEST controller keeps raising its interrupt. Also test that a USB4/Thunderbolt router which never registers its switch does not keep the port from probing. + Also test that a Type-C state which a mux provider with transient + timeouts holds up is applied again until it goes through, for up to + about two minutes, also next to a display route that declares + nothing, and that other providers and errors keep the bounded + retries. These tests run without accessing a physical Type-C controller. config TYPEC_SN201202X_KUNIT_TEST diff --git a/drivers/usb/typec/tipd/Makefile b/drivers/usb/typec/tipd/Makefile index fcb10573a05b1..0138a734b7a2e 100644 --- a/drivers/usb/typec/tipd/Makefile +++ b/drivers/usb/typec/tipd/Makefile @@ -13,5 +13,5 @@ obj-$(CONFIG_TYPEC_SN201202X) += sn201202x.o sn201202x-y := spmi.o obj-$(CONFIG_TYPEC_TIPD_KUNIT_TEST) += cd321x-pm-test.o cd321x-tbt-test.o \ - cd321x-irq-test.o + cd321x-irq-test.o cd321x-mux-test.o obj-$(CONFIG_TYPEC_SN201202X_KUNIT_TEST) += sn201202x-pm-test.o diff --git a/drivers/usb/typec/tipd/cd321x-mux-test.c b/drivers/usb/typec/tipd/cd321x-mux-test.c new file mode 100644 index 0000000000000..c4f32145b0023 --- /dev/null +++ b/drivers/usb/typec/tipd/cd321x-mux-test.c @@ -0,0 +1,812 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for CD321x Type-C states that the mux holds up + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tps6598x.h" + +#define TEST_REG_INT_EVENT1 0x14 +#define TEST_REG_INT_CLEAR1 0x18 +#define TEST_REG_STATUS 0x1a +#define TEST_REG_DATA_STATUS 0x5f + +/* Longer than the bursts of power elections seen so far, up to ~5.5 s */ +#define TEST_ELECTIONS_MS 8000 + +/* A connector whose mode switch is the PHY, as on Apple machines */ +static const struct property_entry cd321x_mux_test_switch_props[] = { + PROPERTY_ENTRY_BOOL("mode-switch"), + { } +}; + +static const struct software_node cd321x_mux_test_phy_node = { + .name = "cd321x-mux-test-phy", + .properties = cd321x_mux_test_switch_props, +}; + +/* The DCP display route, the second mode switch of a J700's rear connector */ +static const struct software_node cd321x_mux_test_display_node = { + .name = "cd321x-mux-test-display", + .properties = cd321x_mux_test_switch_props, +}; + +static const struct software_node_ref_args cd321x_mux_test_refs[] = { + SOFTWARE_NODE_REFERENCE(&cd321x_mux_test_phy_node), +}; + +static const struct property_entry cd321x_mux_test_connector_props[] = { + PROPERTY_ENTRY_REF_ARRAY("mode-switch", cd321x_mux_test_refs), + { } +}; + +static const struct software_node cd321x_mux_test_connector = { + .name = "cd321x-mux-test-connector", + .properties = cd321x_mux_test_connector_props, +}; + +/* The rear connector of a J700, as in t8140-j700.dts */ +static const struct software_node_ref_args cd321x_mux_test_j700_refs[] = { + SOFTWARE_NODE_REFERENCE(&cd321x_mux_test_phy_node), + SOFTWARE_NODE_REFERENCE(&cd321x_mux_test_display_node), +}; + +static const struct property_entry cd321x_mux_test_j700_connector_props[] = { + PROPERTY_ENTRY_REF_ARRAY("mode-switch", cd321x_mux_test_j700_refs), + { } +}; + +static const struct software_node cd321x_mux_test_j700_connector = { + .name = "cd321x-mux-test-j700-connector", + .properties = cd321x_mux_test_j700_connector_props, +}; + +static const struct software_node * const cd321x_mux_test_nodes[] = { + &cd321x_mux_test_phy_node, + &cd321x_mux_test_display_node, + &cd321x_mux_test_connector, + &cd321x_mux_test_j700_connector, + NULL +}; + +struct cd321x_mux_test; + +/* + * A mode switch on the connector. While @held, as the rear port PHY of a J700 + * is through the power elections of a Mac taking another data role, when it + * cannot park the PIPE, it fails every change with @held_ret. @log records the + * states asked for: s(afe), u(sb) or o(ther), in capitals when refused. The + * display route also fails to quiesce, before any provider changes, while + * @prepare_held. + */ +struct cd321x_mux_test_phy { + struct cd321x_mux_test *test; + bool held; + bool prepare_held; + int held_ret; + char log[64]; + unsigned int last_refused_ms; +}; + +struct cd321x_mux_test { + struct cd321x cd; + u64 latched; + u32 status; + u32 data_status; + struct cd321x_mux_test_phy phy; + /* Only on a J700's rear connector, where it does not declare anything */ + struct cd321x_mux_test_phy display; + /* The time the scheduled work has been run at */ + unsigned int now_ms; +}; + +static int cd321x_mux_test_read(void *context, const void *reg, size_t reg_size, + void *val, size_t val_size) +{ + struct cd321x_mux_test *t = context; + + memset(val, 0, val_size); + switch (*(const u8 *)reg) { + case TEST_REG_INT_EVENT1: + memcpy(val, &t->latched, min(val_size, sizeof(t->latched))); + break; + case TEST_REG_STATUS: + memcpy(val, &t->status, min(val_size, sizeof(t->status))); + break; + case TEST_REG_DATA_STATUS: + memcpy(val, &t->data_status, min(val_size, sizeof(t->data_status))); + break; + } + + return 0; +} + +static int cd321x_mux_test_write(void *context, const void *data, size_t count) +{ + struct cd321x_mux_test *t = context; + const u8 *buf = data; + u64 clear = 0; + + if (buf[0] == TEST_REG_INT_CLEAR1) { + memcpy(&clear, buf + 1, min(count - 1, sizeof(clear))); + t->latched &= ~clear; + } + + return 0; +} + +static const struct regmap_bus cd321x_mux_test_bus = { + .read = cd321x_mux_test_read, + .write = cd321x_mux_test_write, + .reg_format_endian_default = REGMAP_ENDIAN_NATIVE, + .val_format_endian_default = REGMAP_ENDIAN_NATIVE, +}; + +static int cd321x_mux_test_set(struct typec_mux_dev *mux, struct typec_mux_state *state) +{ + struct cd321x_mux_test_phy *phy = typec_mux_get_drvdata(mux); + size_t len = strlen(phy->log); + char step; + + if (state->mode == TYPEC_STATE_SAFE) + step = 's'; + else if (state->mode == TYPEC_STATE_USB && !state->alt) + step = 'u'; + else + step = 'o'; + if (phy->held) { + step = toupper(step); + phy->last_refused_ms = phy->test->now_ms; + } + if (len < sizeof(phy->log) - 1) + phy->log[len] = step; + + return phy->held ? phy->held_ret : 0; +} + +static int cd321x_mux_test_prepare(struct typec_mux_dev *mux, struct typec_mux_state *state) +{ + struct cd321x_mux_test_phy *phy = typec_mux_get_drvdata(mux); + + return phy->prepare_held ? phy->held_ret : 0; +} + +static int cd321x_mux_test_psy_get(struct power_supply *psy, + enum power_supply_property psp, + union power_supply_propval *val) +{ + val->intval = 0; + return 0; +} + +static const enum power_supply_property cd321x_mux_test_psy_props[] = { + POWER_SUPPLY_PROP_ONLINE, +}; + +static const struct power_supply_desc cd321x_mux_test_psy_desc = { + .name = "cd321x-mux-test", + .type = POWER_SUPPLY_TYPE_USB, + .properties = cd321x_mux_test_psy_props, + .num_properties = ARRAY_SIZE(cd321x_mux_test_psy_props), + .get_property = cd321x_mux_test_psy_get, +}; + +static void cd321x_mux_test_unregister_nodes(void *unused) +{ + software_node_unregister_node_group(cd321x_mux_test_nodes); +} + +static void cd321x_mux_test_unregister_mux(void *mux) +{ + typec_mux_unregister(mux); +} + +static void cd321x_mux_test_unregister_port(void *port) +{ + typec_unregister_port(port); +} + +static void cd321x_mux_test_put_mux(void *mux) +{ + typec_mux_put(mux); +} + +static void cd321x_mux_test_unregister_psy(void *psy) +{ + power_supply_unregister(psy); +} + +static void cd321x_mux_test_stop(void *data) +{ + struct cd321x *cd = data; + + scoped_guard(mutex, &cd->tps.lock) + cd321x_pm_remove(&cd->pm); + cancel_delayed_work_sync(&cd->resume_work); + cancel_delayed_work_sync(&cd->update_work); + if (!IS_ERR_OR_NULL(cd->tps.partner)) + typec_unregister_partner(cd->tps.partner); + cd->tps.partner = NULL; +} + +/* + * A running port with nothing attached, whose mux is the test PHY, and on a + * @j700 rear connector the display route after it. Only a PHY with @transient + * timeouts, the rear port PHY of a J700, has the driver retry its timeouts + * until they pass. + */ +static struct cd321x_mux_test *cd321x_mux_test_setup(struct kunit *test, bool transient, + bool j700) +{ + struct typec_mux_desc mux_desc = { + .set = cd321x_mux_test_set, + .transient_timeouts = transient, + }; + struct typec_mux_desc display_desc = { + .set = cd321x_mux_test_set, + .prepare = cd321x_mux_test_prepare, + .name = "cd321x-mux-test-display", + }; + struct typec_capability cap = { + .type = TYPEC_PORT_DRP, + .data = TYPEC_PORT_DRD, + .revision = USB_TYPEC_REV_1_2, + .prefer_role = TYPEC_NO_PREFERRED_ROLE, + }; + struct power_supply_config psy_cfg = {}; + struct fwnode_handle *connector; + struct typec_mux_dev *mux_dev; + struct cd321x_mux_test *t; + struct device *dev; + int ret; + + ret = software_node_register_node_group(cd321x_mux_test_nodes); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = kunit_add_action_or_reset(test, cd321x_mux_test_unregister_nodes, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + connector = software_node_fwnode(j700 ? &cd321x_mux_test_j700_connector : + &cd321x_mux_test_connector); + mux_desc.fwnode = software_node_fwnode(&cd321x_mux_test_phy_node); + display_desc.fwnode = software_node_fwnode(&cd321x_mux_test_display_node); + + dev = kunit_device_register(test, "cd321x-mux-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t); + t->phy.test = t; + t->phy.held_ret = -ETIMEDOUT; + t->display.test = t; + t->display.held_ret = -ETIMEDOUT; + + mux_desc.drvdata = &t->phy; + mux_dev = typec_mux_register(dev, &mux_desc); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, mux_dev); + ret = kunit_add_action_or_reset(test, cd321x_mux_test_unregister_mux, mux_dev); + KUNIT_ASSERT_EQ(test, ret, 0); + if (j700) { + display_desc.drvdata = &t->display; + mux_dev = typec_mux_register(dev, &display_desc); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, mux_dev); + ret = kunit_add_action_or_reset(test, cd321x_mux_test_unregister_mux, mux_dev); + KUNIT_ASSERT_EQ(test, ret, 0); + } + + mutex_init(&t->cd.tps.lock); + t->cd.tps.dev = dev; + t->cd.tps.data = tipd_hw_variants[TIPD_VARIANT_CD321X]; + t->cd.tps.regmap = devm_regmap_init(dev, &cd321x_mux_test_bus, t, + &tps6598x_regmap_config); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->cd.tps.regmap); + dev_set_drvdata(dev, &t->cd.tps); + + cap.fwnode = connector; + cap.driver_data = &t->cd.tps; + t->cd.tps.port = typec_register_port(dev, &cap); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->cd.tps.port); + ret = kunit_add_action_or_reset(test, cd321x_mux_test_unregister_port, t->cd.tps.port); + KUNIT_ASSERT_EQ(test, ret, 0); + + t->cd.mux = fwnode_typec_mux_get(connector); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->cd.mux); + ret = kunit_add_action_or_reset(test, cd321x_mux_test_put_mux, t->cd.mux); + KUNIT_ASSERT_EQ(test, ret, 0); + + psy_cfg.drv_data = &t->cd.tps; + t->cd.tps.psy = power_supply_register(dev, &cd321x_mux_test_psy_desc, &psy_cfg); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->cd.tps.psy); + ret = kunit_add_action_or_reset(test, cd321x_mux_test_unregister_psy, t->cd.tps.psy); + KUNIT_ASSERT_EQ(test, ret, 0); + + cd321x_init_work(&t->cd); + t->cd.pm.phase = CD321X_PM_RUNNING; + ret = kunit_add_action_or_reset(test, cd321x_mux_test_stop, &t->cd); + KUNIT_ASSERT_EQ(test, ret, 0); + + return t; +} + +static struct cd321x_mux_test *cd321x_mux_test_port(struct kunit *test, bool transient) +{ + return cd321x_mux_test_setup(test, transient, false); +} + +/* The rear port of a J700: its PHY declares its timeouts transient. */ +static struct cd321x_mux_test *cd321x_mux_test_j700_port(struct kunit *test) +{ + return cd321x_mux_test_setup(test, true, true); +} + +/* A controller event, delivered as the interrupt handler would get it */ +static void cd321x_mux_test_event(struct cd321x_mux_test *t, u64 event) +{ + t->latched |= event; + t->cd.tps.data->irq_handler(0, &t->cd.tps); +} + +/* A Mac attaches, the port being the USB host */ +static void cd321x_mux_test_attach(struct cd321x_mux_test *t) +{ + t->status = TPS_STATUS_PLUG_PRESENT; + t->data_status = TPS_DATA_STATUS_DATA_CONNECTION | TPS_DATA_STATUS_USB2_CONNECTION | + TPS_DATA_STATUS_USB3_CONNECTION; + cd321x_mux_test_event(t, APPLE_CD_REG_INT_PLUG_EVENT | + APPLE_CD_REG_INT_DATA_STATUS_UPDATE); +} + +/* The delay the work was scheduled with, from now */ +static unsigned int cd321x_mux_test_due_ms(struct delayed_work *dwork) +{ + long left = (long)(dwork->timer.expires - jiffies); + + return left > 0 ? jiffies_to_msecs(left) : 0; +} + +/* + * Run the next work of the port, the one due first, as if its delay had passed. + * Returns false if none is scheduled. + */ +static bool cd321x_mux_test_next(struct cd321x_mux_test *t) +{ + struct delayed_work *resume = &t->cd.resume_work, *update = &t->cd.update_work; + struct delayed_work *next; + + /* A work whose delay ran out meanwhile may be running already. */ + if (!delayed_work_pending(resume) && !delayed_work_pending(update)) { + flush_delayed_work(resume); + flush_delayed_work(update); + } + if (delayed_work_pending(resume) && + (!delayed_work_pending(update) || + time_before_eq(resume->timer.expires, update->timer.expires))) + next = resume; + else if (delayed_work_pending(update)) + next = update; + else + return false; + + t->now_ms += cd321x_mux_test_due_ms(next); + flush_delayed_work(next); + return true; +} + +/* Run the work of the port until none is left, which must be soon. */ +static void cd321x_mux_test_settle(struct kunit *test, struct cd321x_mux_test *t) +{ + unsigned int i; + + for (i = 0; i < 16 && cd321x_mux_test_next(t); i++) + ; + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.update_work)); +} + +static enum cd321x_pm_phase cd321x_mux_test_phase(struct cd321x_mux_test *t) +{ + guard(mutex)(&t->cd.tps.lock); + return t->cd.pm.phase; +} + +/* + * The delay of the retry of a held mux. Unlike what is left of it, this does + * not shrink as real time passes while the tests run. + */ +static unsigned int cd321x_mux_test_backoff_ms(struct cd321x_mux_test *t) +{ + guard(mutex)(&t->cd.tps.lock); + return t->cd.pm.mux_retry_ms; +} + +/* + * The resume read is queued without delay, so with no timer, rather than after + * a backoff. A worker may run it already. + */ +static void cd321x_mux_test_expect_read_now(struct kunit *test, struct cd321x_mux_test *t) +{ + struct delayed_work *resume = &t->cd.resume_work; + + KUNIT_EXPECT_FALSE(test, timer_pending(&resume->timer)); + flush_delayed_work(resume); +} + +/* The state is applied: the mux set to USB, and nothing retried any more */ +static void cd321x_mux_test_expect_usb(struct kunit *test, struct cd321x_mux_test *t) +{ + KUNIT_EXPECT_EQ(test, cd321x_mux_test_phase(t), CD321X_PM_RUNNING); + KUNIT_EXPECT_TRUE(test, t->cd.state_valid); + KUNIT_EXPECT_EQ(test, t->cd.state.mode, TYPEC_STATE_USB); + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0); + KUNIT_EXPECT_NOT_NULL(test, t->cd.tps.partner); +} + +/* + * Through the elections the PHY refuses the safe state of the new connection + * for longer than the three cable-state attempts last. A mux request for USB + * must still follow once they settle, with no further controller event. + */ +static void cd321x_mux_held_past_budget_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_port(test, true); + + t->phy.held = true; + cd321x_mux_test_attach(t); + while (t->now_ms < TEST_ELECTIONS_MS) { + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + KUNIT_ASSERT_NE(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + } + /* Refused well past the third attempt, never asked for USB */ + KUNIT_EXPECT_GT(test, t->phy.last_refused_ms, 3000U); + KUNIT_EXPECT_NULL(test, strpbrk(t->phy.log, "uU")); + KUNIT_EXPECT_GE(test, strlen(t->phy.log), (size_t)CD321X_RESUME_ATTEMPTS + 1); + + t->phy.held = false; + memset(t->phy.log, 0, sizeof(t->phy.log)); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "su"); + cd321x_mux_test_expect_usb(test, t); +} + +/* Retried more and more slowly while held, but at least every ten seconds */ +static void cd321x_mux_held_backs_off_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_port(test, true); + unsigned int i, first = 0, delay = 0; + + t->phy.held = true; + cd321x_mux_test_attach(t); + for (i = 0; i < 8 * CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + if (!delayed_work_pending(&t->cd.resume_work)) + continue; + delay = cd321x_mux_test_due_ms(&t->cd.resume_work); + if (!first) + first = delay; + KUNIT_EXPECT_LE(test, delay, (unsigned int)CD321X_MUX_RETRY_MAX_MS); + KUNIT_EXPECT_NE(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + KUNIT_EXPECT_GT(test, t->cd.pm.attempts_left, 0); + } + KUNIT_EXPECT_GE(test, delay, 8 * first); + KUNIT_EXPECT_GT(test, t->now_ms, 30000U); + + t->phy.held = false; + memset(t->phy.log, 0, sizeof(t->phy.log)); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "su"); + cd321x_mux_test_expect_usb(test, t); +} + +/* + * A controller event during the backoff takes over from the retry: its state + * is applied after the usual debounce, and the retry that was due later steps + * aside rather than repeating the change. + */ +static void cd321x_mux_held_event_takes_over_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_port(test, true); + + t->phy.held = true; + cd321x_mux_test_attach(t); + while (!delayed_work_pending(&t->cd.resume_work) || + cd321x_mux_test_backoff_ms(t) < 4000) + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + + /* The elections end with the Mac taking the device role */ + t->phy.held = false; + memset(t->phy.log, 0, sizeof(t->phy.log)); + t->data_status |= TPS_DATA_STATUS_USB_DATA_ROLE; + cd321x_mux_test_event(t, APPLE_CD_REG_INT_DATA_STATUS_UPDATE); + KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); + KUNIT_EXPECT_LE(test, cd321x_mux_test_due_ms(&t->cd.resume_work), 500U); + + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "su"); + cd321x_mux_test_expect_usb(test, t); + KUNIT_EXPECT_LT(test, t->now_ms, TEST_ELECTIONS_MS); +} + +/* Sleep cancels the retry, and the resume read takes it over */ +static void cd321x_mux_held_across_sleep_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_port(test, true); + struct device *dev = t->cd.tps.dev; + unsigned int i; + + t->phy.held = true; + cd321x_mux_test_attach(t); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + + KUNIT_ASSERT_EQ(test, tipd_prepare(dev), 0); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.update_work)); + tipd_complete(dev); + /* The resume read is queued without delay: a worker may run it already. */ + flush_delayed_work(&t->cd.resume_work); + + /* Still held after resume: the state is retried, not given up */ + memset(t->phy.log, 0, sizeof(t->phy.log)); + for (i = 0; i < 6 * CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + KUNIT_EXPECT_NE(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + } + KUNIT_EXPECT_GT(test, strlen(t->phy.log), (size_t)CD321X_RESUME_ATTEMPTS + 1); + + t->phy.held = false; + memset(t->phy.log, 0, sizeof(t->phy.log)); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "su"); + cd321x_mux_test_expect_usb(test, t); +} + +/* Removal ends the retry */ +static void cd321x_mux_held_ends_on_removal_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_port(test, true); + + t->phy.held = true; + cd321x_mux_test_attach(t); + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); + + scoped_guard(mutex, &t->cd.tps.lock) + cd321x_pm_remove(&t->cd.pm); + memset(t->phy.log, 0, sizeof(t->phy.log)); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, ""); +} + +/* + * Every other mux, such as the PHY of any M1 or M2 port, keeps the bounded + * cable-state retries: the driver gives up after them, and the next event + * starts over. + */ +static void cd321x_mux_timeout_bounded_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_port(test, false); + + t->phy.held = true; + cd321x_mux_test_attach(t); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "SSSS"); + KUNIT_EXPECT_EQ(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0); + + t->phy.held = false; + memset(t->phy.log, 0, sizeof(t->phy.log)); + cd321x_mux_test_event(t, APPLE_CD_REG_INT_DATA_STATUS_UPDATE); + flush_delayed_work(&t->cd.resume_work); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "su"); + cd321x_mux_test_expect_usb(test, t); +} + +/* and on the J700, any failure but a timeout does too */ +static void cd321x_mux_other_error_bounded_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_port(test, true); + + t->phy.held = true; + t->phy.held_ret = -EBUSY; + cd321x_mux_test_attach(t); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "SSSS"); + KUNIT_EXPECT_EQ(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0); +} + +/* + * The rear connector of a J700 has a second mode switch after the PHY, the DCP + * display route, which does not declare its timeouts transient. A refusal of + * the PHY is still retried past the budget, and once the elections settle + * both providers get the safe state and then the USB one. + */ +static void cd321x_mux_j700_held_past_budget_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_j700_port(test); + + t->phy.held = true; + cd321x_mux_test_attach(t); + while (t->now_ms < TEST_ELECTIONS_MS) { + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + KUNIT_ASSERT_NE(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + } + KUNIT_EXPECT_GT(test, t->phy.last_refused_ms, 3000U); + KUNIT_EXPECT_NULL(test, strpbrk(t->phy.log, "uU")); + KUNIT_EXPECT_GE(test, strlen(t->phy.log), (size_t)CD321X_RESUME_ATTEMPTS + 1); + + t->phy.held = false; + memset(t->phy.log, 0, sizeof(t->phy.log)); + memset(t->display.log, 0, sizeof(t->display.log)); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "su"); + KUNIT_EXPECT_STREQ(test, t->display.log, "su"); + cd321x_mux_test_expect_usb(test, t); +} + +/* There a timeout of the display route keeps the bounded retries. */ +static void cd321x_mux_j700_display_timeout_bounded_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_j700_port(test); + + t->display.held = true; + cd321x_mux_test_attach(t); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->display.log, "SSSS"); + KUNIT_EXPECT_EQ(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0); + + t->display.held = false; + memset(t->phy.log, 0, sizeof(t->phy.log)); + memset(t->display.log, 0, sizeof(t->display.log)); + cd321x_mux_test_event(t, APPLE_CD_REG_INT_DATA_STATUS_UPDATE); + flush_delayed_work(&t->cd.resume_work); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "su"); + KUNIT_EXPECT_STREQ(test, t->display.log, "su"); + cd321x_mux_test_expect_usb(test, t); +} + +/* + * So does a timeout of the display route while it quiesces the display, before + * any provider changes, even right after the PHY held changes up. + */ +static void cd321x_mux_j700_display_prepare_timeout_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_j700_port(test); + + t->phy.held = true; + cd321x_mux_test_attach(t); + while (!delayed_work_pending(&t->cd.resume_work) || + cd321x_mux_test_backoff_ms(t) < 2000) + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + + t->phy.held = false; + t->display.prepare_held = true; + memset(t->phy.log, 0, sizeof(t->phy.log)); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, ""); + KUNIT_EXPECT_EQ(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0); + + t->display.prepare_held = false; + memset(t->display.log, 0, sizeof(t->display.log)); + cd321x_mux_test_event(t, APPLE_CD_REG_INT_DATA_STATUS_UPDATE); + cd321x_mux_test_expect_read_now(test, t); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "su"); + KUNIT_EXPECT_STREQ(test, t->display.log, "su"); + cd321x_mux_test_expect_usb(test, t); +} + +/* + * A PHY that stays stuck is taken for a fault after some two minutes of + * retries. The port then waits for the next controller event, which starts + * the retries over. + */ +static void cd321x_mux_held_gives_up_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_j700_port(test); + unsigned int i; + + t->phy.held = true; + cd321x_mux_test_attach(t); + for (i = 0; i < 4 * CD321X_MUX_RETRIES && cd321x_mux_test_next(t); i++) + ; + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.update_work)); + KUNIT_EXPECT_EQ(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + KUNIT_EXPECT_EQ(test, strlen(t->phy.log), (size_t)CD321X_MUX_RETRIES + 1); + KUNIT_EXPECT_NULL(test, strpbrk(t->phy.log, "suU")); + KUNIT_EXPECT_GE(test, t->phy.last_refused_ms, 120000U); + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0); + KUNIT_EXPECT_FALSE(test, t->cd.state_valid); + + /* Still stuck at the next event: the retries start over */ + memset(t->phy.log, 0, sizeof(t->phy.log)); + cd321x_mux_test_event(t, APPLE_CD_REG_INT_DATA_STATUS_UPDATE); + cd321x_mux_test_expect_read_now(test, t); + for (i = 0; i < 8 && !strchr(t->phy.log, 'S'); i++) + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + KUNIT_ASSERT_STREQ(test, t->phy.log, "S"); + KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); + KUNIT_EXPECT_LE(test, cd321x_mux_test_due_ms(&t->cd.resume_work), + (unsigned int)CD321X_MUX_RETRY_MS); + + t->phy.held = false; + memset(t->phy.log, 0, sizeof(t->phy.log)); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "su"); + cd321x_mux_test_expect_usb(test, t); +} + +/* + * A series of retries that ends in another failure leaves no backoff behind: + * the next event is read without delay, and the next hold-up starts over at + * the first delay. + */ +static void cd321x_mux_held_then_other_error_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_port(test, true); + unsigned int i; + + t->phy.held = true; + cd321x_mux_test_attach(t); + while (!delayed_work_pending(&t->cd.resume_work) || + cd321x_mux_test_backoff_ms(t) < 4000) + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + + t->phy.held_ret = -EBUSY; + cd321x_mux_test_settle(test, t); + KUNIT_ASSERT_EQ(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0); + + t->phy.held_ret = -ETIMEDOUT; + memset(t->phy.log, 0, sizeof(t->phy.log)); + t->data_status |= TPS_DATA_STATUS_USB_DATA_ROLE; + cd321x_mux_test_event(t, APPLE_CD_REG_INT_DATA_STATUS_UPDATE); + cd321x_mux_test_expect_read_now(test, t); + for (i = 0; i < 8 && !strchr(t->phy.log, 'S'); i++) + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + KUNIT_ASSERT_NOT_NULL(test, strchr(t->phy.log, 'S')); + KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); + KUNIT_EXPECT_LE(test, cd321x_mux_test_due_ms(&t->cd.resume_work), + (unsigned int)CD321X_MUX_RETRY_MS); + + t->phy.held = false; + cd321x_mux_test_settle(test, t); + cd321x_mux_test_expect_usb(test, t); +} + +static struct kunit_case cd321x_mux_cases[] = { + KUNIT_CASE(cd321x_mux_held_past_budget_test), + KUNIT_CASE(cd321x_mux_held_backs_off_test), + KUNIT_CASE(cd321x_mux_held_event_takes_over_test), + KUNIT_CASE(cd321x_mux_held_across_sleep_test), + KUNIT_CASE(cd321x_mux_held_ends_on_removal_test), + KUNIT_CASE(cd321x_mux_timeout_bounded_test), + KUNIT_CASE(cd321x_mux_other_error_bounded_test), + KUNIT_CASE(cd321x_mux_j700_held_past_budget_test), + KUNIT_CASE(cd321x_mux_j700_display_timeout_bounded_test), + KUNIT_CASE(cd321x_mux_j700_display_prepare_timeout_test), + KUNIT_CASE(cd321x_mux_held_gives_up_test), + KUNIT_CASE(cd321x_mux_held_then_other_error_test), + {} +}; + +static struct kunit_suite cd321x_mux_suite = { + .name = "cd321x-mux", + .test_cases = cd321x_mux_cases, +}; + +kunit_test_suite(cd321x_mux_suite); +MODULE_DESCRIPTION("KUnit tests for CD321x Type-C states that the mux holds up"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/drivers/usb/typec/tipd/cd321x-pm-test.c b/drivers/usb/typec/tipd/cd321x-pm-test.c index 08adc096cbb7c..2196ec4642721 100644 --- a/drivers/usb/typec/tipd/cd321x-pm-test.c +++ b/drivers/usb/typec/tipd/cd321x-pm-test.c @@ -520,6 +520,139 @@ static void cd321x_unacked_events_across_sleep_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); } +static void cd321x_held_mux_outlasts_budget_test(struct kunit *test) +{ + struct cd321x_pm_state pm = { .phase = CD321X_PM_RUNNING }; + unsigned int i, delay = 0; + + for (i = 0; i < 4 * CD321X_RESUME_ATTEMPTS; i++) { + unsigned int next = cd321x_pm_mux_held(&pm); + + /* Never given up, and not charged to the cable-state budget... */ + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_REVALIDATE); + KUNIT_EXPECT_EQ(test, pm.attempts_left, (unsigned int)CD321X_RESUME_ATTEMPTS); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + /* ...but applied afresh from SAFE, less and less often, down to a limit. */ + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); + KUNIT_EXPECT_EQ(test, next, i ? min(2 * delay, CD321X_MUX_RETRY_MAX_MS) : + CD321X_MUX_RETRY_MS); + delay = next; + /* An event brings the retry forward, but not a read already under way. */ + KUNIT_EXPECT_TRUE(test, cd321x_pm_mux_event(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_mux_event(&pm)); + } + KUNIT_EXPECT_EQ(test, delay, CD321X_MUX_RETRY_MAX_MS); + + /* Once through, the next failure has the bounded budget again. */ + KUNIT_EXPECT_TRUE(test, cd321x_pm_mux_done(&pm)); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_mux_done(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_mux_event(&pm)); + for (i = 0; i <= CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_EXPECT_EQ(test, cd321x_pm_retry(&pm, true), i < CD321X_RESUME_ATTEMPTS); + if (i == CD321X_RESUME_ATTEMPTS) + break; + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + } + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); + KUNIT_EXPECT_FALSE(test, cd321x_pm_mux_event(&pm)); +} + +static void cd321x_held_mux_across_sleep_test(struct kunit *test) +{ + struct cd321x_pm_state pm = { .phase = CD321X_PM_RUNNING }; + + cd321x_pm_mux_held(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm), 2 * CD321X_MUX_RETRY_MS); + + /* The resume read takes the retry over, and a new hold-up starts afresh. */ + cd321x_pm_prepare(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_mux_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); + KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm), CD321X_MUX_RETRY_MS); + + /* Removal ends it. */ + cd321x_pm_remove(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_mux_event(&pm)); +} + +static void cd321x_held_mux_gives_up_test(struct kunit *test) +{ + struct cd321x_pm_state pm = { .phase = CD321X_PM_RUNNING }; + unsigned int i, total = 0; + + for (i = 0; i < CD321X_MUX_RETRIES; i++) { + unsigned int delay = cd321x_pm_mux_held(&pm); + + KUNIT_EXPECT_GT(test, delay, 0); + total += delay; + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + } + /* Some two minutes on, a hold-up is taken for a fault... */ + KUNIT_EXPECT_GE(test, total, 120000U); + KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm), 0); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); + KUNIT_EXPECT_EQ(test, pm.mux_retry_ms, 0); + KUNIT_EXPECT_FALSE(test, cd321x_pm_mux_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + + /* ...until a later event applies the whole state again, afresh. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_event(&pm)); + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm), CD321X_MUX_RETRY_MS); + KUNIT_EXPECT_EQ(test, pm.mux_retries, 1); +} + +static void cd321x_held_mux_other_failure_test(struct kunit *test) +{ + struct cd321x_pm_state pm = { .phase = CD321X_PM_RUNNING }; + unsigned int i; + + for (i = 0; i < 4; i++) { + cd321x_pm_mux_held(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + } + KUNIT_ASSERT_EQ(test, pm.mux_retry_ms, 8 * CD321X_MUX_RETRY_MS); + + /* + * A budgeted failure ends the hold-up, and what is left of the budget, + * all but the read under way, runs out as usual. + */ + for (i = 0; cd321x_pm_retry(&pm, true); i++) { + KUNIT_EXPECT_EQ(test, pm.mux_retry_ms, 0); + KUNIT_EXPECT_EQ(test, pm.mux_retries, 0); + KUNIT_EXPECT_FALSE(test, cd321x_pm_mux_event(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + } + KUNIT_EXPECT_EQ(test, i, CD321X_RESUME_ATTEMPTS - 1); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); + KUNIT_EXPECT_EQ(test, pm.mux_retry_ms, 0); + + /* The next hold-up starts over, and removal ends it. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_event(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm), CD321X_MUX_RETRY_MS); + cd321x_pm_remove(&pm); + KUNIT_EXPECT_EQ(test, pm.mux_retry_ms, 0); + KUNIT_EXPECT_EQ(test, pm.mux_retries, 0); +} + /* Only a pipeline owned by another port skips the retry. */ static void cd321x_display_taken_test(struct kunit *test) { @@ -559,6 +692,10 @@ static struct kunit_case cd321x_pm_cases[] = { KUNIT_CASE(cd321x_lost_event_keeps_reconnect_test), KUNIT_CASE(cd321x_unacked_events_outlast_budget_test), KUNIT_CASE(cd321x_unacked_events_across_sleep_test), + KUNIT_CASE(cd321x_held_mux_outlasts_budget_test), + KUNIT_CASE(cd321x_held_mux_across_sleep_test), + KUNIT_CASE(cd321x_held_mux_gives_up_test), + KUNIT_CASE(cd321x_held_mux_other_failure_test), KUNIT_CASE(cd321x_display_taken_test), {} }; diff --git a/drivers/usb/typec/tipd/cd321x-pm.h b/drivers/usb/typec/tipd/cd321x-pm.h index 3879c8a660dcc..1a3b5a4b659e3 100644 --- a/drivers/usb/typec/tipd/cd321x-pm.h +++ b/drivers/usb/typec/tipd/cd321x-pm.h @@ -11,6 +11,10 @@ #define CD321X_LINK_EVENTS 3 #define CD321X_ACK_RETRY_MS 500 #define CD321X_ACK_RETRY_MAX_MS 30000 +#define CD321X_MUX_RETRY_MS 500 +#define CD321X_MUX_RETRY_MAX_MS 10000 +/* About two minutes of backoff, twenty times the longest hold-up seen */ +#define CD321X_MUX_RETRIES 16 enum cd321x_pm_phase { CD321X_PM_RUNNING, @@ -29,6 +33,10 @@ struct cd321x_pm_state { unsigned int link_events_left; /* Non-zero while events may be left unacknowledged: the retry delay. */ unsigned int ack_retry_ms; + /* Non-zero while a mux that held a change up is retried: the retry delay. */ + unsigned int mux_retry_ms; + /* The retries of that change so far, up to CD321X_MUX_RETRIES. */ + unsigned int mux_retries; bool force_reconnect; }; @@ -57,11 +65,26 @@ static inline bool cd321x_pm_can_update(const struct cd321x_pm_state *pm) return pm->phase == CD321X_PM_RUNNING || pm->phase == CD321X_PM_APPLY; } +/* + * Ends the retries of a change that a mux held up, see cd321x_pm_mux_held(). + * Returns true if there were any. + */ +static inline bool cd321x_pm_mux_done(struct cd321x_pm_state *pm) +{ + bool retried = pm->mux_retry_ms; + + pm->mux_retry_ms = 0; + pm->mux_retries = 0; + return retried; +} + static inline void cd321x_pm_prepare(struct cd321x_pm_state *pm) { if (pm->phase != CD321X_PM_REMOVED) pm->phase = CD321X_PM_PREPARED; pm->attempts_left = 0; + /* The resume read applies the state afresh. */ + cd321x_pm_mux_done(pm); } static inline bool cd321x_pm_resume(struct cd321x_pm_state *pm) @@ -104,6 +127,8 @@ static inline void cd321x_pm_snapshot_ready(struct cd321x_pm_state *pm) static inline bool cd321x_pm_retry(struct cd321x_pm_state *pm, bool reconnect) { + /* Any other failure ends a hold-up: this one is budgeted. */ + cd321x_pm_mux_done(pm); /* Runtime cable updates need the same bounded fresh-read recovery. */ if (pm->phase == CD321X_PM_RUNNING) { pm->force_reconnect |= reconnect; @@ -174,6 +199,54 @@ static inline bool cd321x_pm_acked(struct cd321x_pm_state *pm) return retried; } +/* + * The mux provider that failed a change timed out because its hardware held + * the change up (typec_mux_timeout_transient()). On a J700 the PHY cannot park + * the PIPE while the controller behind the fixed hub still drives it, and a + * partner renegotiating power can keep that up for longer than the cable-state + * budget lasts, after which nothing would apply the state again. The cable + * state itself was read fine, so such a failure does not count against the + * budget: it gives back the attempt taken by cd321x_pm_begin_read(), and the + * whole state is applied again from a fresh read, starting with SAFE, after a + * delay that doubles up to CD321X_MUX_RETRY_MAX_MS. A later event brings the + * retry forward, see cd321x_pm_mux_event(); sleep cancels it and the resume + * read takes it over; removal or any other failure ends it. + * + * A hold-up that outlasts CD321X_MUX_RETRIES retries is taken for a fault, as + * the provider cannot tell its timeouts apart: the retries end as the budget + * does, and only a later event starts over. Returns the delay in milliseconds, + * or 0 once the retries have ended. + */ +static inline unsigned int cd321x_pm_mux_held(struct cd321x_pm_state *pm) +{ + pm->force_reconnect = true; + if (pm->mux_retries >= CD321X_MUX_RETRIES) { + cd321x_pm_mux_done(pm); + pm->phase = CD321X_PM_STALE; + pm->attempts_left = 0; + return 0; + } + if (pm->phase == CD321X_PM_RUNNING) + pm->attempts_left = CD321X_RESUME_ATTEMPTS; + else + pm->attempts_left++; + pm->phase = CD321X_PM_REVALIDATE; + pm->mux_retries++; + pm->mux_retry_ms = clamp(2 * pm->mux_retry_ms, CD321X_MUX_RETRY_MS, + CD321X_MUX_RETRY_MAX_MS); + return pm->mux_retry_ms; +} + +/* + * Whether a controller event should bring forward the retry of a held mux: the + * event's state is then read and applied after the usual debounce, instead of + * waiting out the backoff. + */ +static inline bool cd321x_pm_mux_event(const struct cd321x_pm_state *pm) +{ + return pm->mux_retry_ms && pm->phase == CD321X_PM_REVALIDATE; +} + static inline void cd321x_pm_complete(struct cd321x_pm_state *pm) { if (pm->phase == CD321X_PM_APPLY) { @@ -187,6 +260,7 @@ static inline void cd321x_pm_remove(struct cd321x_pm_state *pm) { pm->phase = CD321X_PM_REMOVED; pm->attempts_left = 0; + cd321x_pm_mux_done(pm); } /* diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 9799500cbabfd..f62d0c081ab0b 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -578,8 +578,30 @@ static void tps6598x_handle_plug_event(struct tps6598x *tps, u32 status) } } +/* The mux held the change up, see cd321x_pm_mux_held(). */ +static bool cd321x_mux_held(struct cd321x *cd321x, int ret) +{ + return ret == -ETIMEDOUT && typec_mux_timeout_transient(cd321x->mux); +} + +/* Set the mux to the mode state, telling whether a provider held it up */ +static int cd321x_mux_set(struct cd321x *cd321x, bool *held) +{ + int ret = typec_mux_set(cd321x->mux, &cd321x->state); + + *held = cd321x_mux_held(cd321x, ret); + return ret; +} + +/* A held mux is retried: report only the first failure of the series. */ +static bool cd321x_mux_still_held(struct cd321x *cd321x, bool held) +{ + return held && cd321x->pm.mux_retry_ms; +} + static int cd321x_typec_update_mode(struct tps6598x *tps, - struct cd321x_status *st, bool *display_busy) + struct cd321x_status *st, bool *display_busy, + bool *held) { struct cd321x *cd321x = container_of(tps, struct cd321x, tps); struct typec_mux_state old_state = cd321x->state; @@ -589,6 +611,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, int ret = 0; *display_busy = false; + *held = false; if (!(st->data_status & TPS_DATA_STATUS_DATA_CONNECTION)) { if (cd321x->state_valid && cd321x->state.mode == TYPEC_STATE_SAFE) return 0; @@ -598,7 +621,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); if (ret) goto out; - ret = typec_mux_set(cd321x->mux, &cd321x->state); + ret = cd321x_mux_set(cd321x, held); } else if (st->data_status & CD321X_DATA_STATUS_USB4_CONNECTION) { struct enter_usb_data eusb_data; @@ -613,7 +636,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, cd321x->state.alt = NULL; cd321x->state.data = &eusb_data; cd321x->state.mode = TYPEC_MODE_USB4; - ret = typec_mux_set(cd321x->mux, &cd321x->state); + ret = cd321x_mux_set(cd321x, held); if (ret) goto out; @@ -646,7 +669,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, cd321x->state.alt = cd321x->port_altmode_tbt; cd321x->state.mode = TYPEC_TBT_MODE; cd321x->state.data = &tbt_data; - ret = typec_mux_set(cd321x->mux, &cd321x->state); + ret = cd321x_mux_set(cd321x, held); if (ret) goto out; @@ -703,7 +726,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); if (ret) goto out; - ret = typec_mux_set(cd321x->mux, &cd321x->state); + ret = cd321x_mux_set(cd321x, held); *display_busy = cd321x_display_taken(ret); if (!ret) { cd321x->dp_status = dp_data.status; @@ -719,7 +742,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); if (ret) goto out; - ret = typec_mux_set(cd321x->mux, &cd321x->state); + ret = cd321x_mux_set(cd321x, held); } out: @@ -732,6 +755,8 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, /* A provider that is not ready yet asks for a retry. */ if (ret == -EAGAIN) dev_dbg(tps->dev, "Type-C connection deferred\n"); + else if (cd321x_mux_still_held(cd321x, *held)) + dev_dbg(tps->dev, "Type-C connection still held up: %d\n", ret); else dev_warn(tps->dev, "failed to configure Type-C connection: %d\n", ret); } else if (cd321x->state.alt != cd321x->port_altmode_dp) { @@ -760,6 +785,31 @@ static void cd321x_retry_revalidation(struct cd321x *cd321x, bool reconnect) CD321X_RESUME_ATTEMPTS); } +static void cd321x_retry_held_mux(struct cd321x *cd321x) +{ + unsigned int delay; + + lockdep_assert_held(&cd321x->tps.lock); + cd321x->state_valid = false; + if (!cd321x->pm.mux_retry_ms) + dev_info(cd321x->tps.dev, "Type-C mux held up, retrying\n"); + delay = cd321x_pm_mux_held(&cd321x->pm); + if (delay) + mod_delayed_work(system_freezable_wq, &cd321x->resume_work, + msecs_to_jiffies(delay)); + else + dev_warn(cd321x->tps.dev, + "Type-C mux still held up after %u retries, waiting for the next event\n", + CD321X_MUX_RETRIES); +} + +static void cd321x_update_done(struct cd321x *cd321x) +{ + if (cd321x_pm_mux_done(&cd321x->pm)) + dev_info(cd321x->tps.dev, "Type-C mux change went through after retrying\n"); + cd321x_pm_complete(&cd321x->pm); +} + static void cd321x_update_work(struct work_struct *work) { struct cd321x *cd321x = container_of(to_delayed_work(work), @@ -769,7 +819,7 @@ static void cd321x_update_work(struct work_struct *work) struct typec_thunderbolt_switch_data tbt_switch_data = { .state = TYPEC_THUNDERBOLT_SWITCH_OFF, }; - bool display_busy; + bool display_busy, held = false; int ret; guard(mutex)(&tps->lock); @@ -877,12 +927,16 @@ static void cd321x_update_work(struct work_struct *work) cd321x->state.alt = NULL; cd321x->state.mode = TYPEC_STATE_SAFE; cd321x->state.data = NULL; - ret = typec_set_mode(tps->port, TYPEC_STATE_SAFE); + /* Through cd321x->mux, which can then tell if SAFE was held up */ + ret = cd321x_mux_set(cd321x, &held); cd321x->state_valid = !ret; if (ret) { - dev_warn(tps->dev, - "failed to place Type-C mux in safe mode: %d\n", - ret); + if (cd321x_mux_still_held(cd321x, held)) + dev_dbg(tps->dev, "Type-C mux still held up: %d\n", ret); + else + dev_warn(tps->dev, + "failed to place Type-C mux in safe mode: %d\n", + ret); cd321x->state = old_state; goto retry; } @@ -899,7 +953,7 @@ static void cd321x_update_work(struct work_struct *work) /* If the plug is disconnected, we are done */ if (!new_connected) { - cd321x_pm_complete(&cd321x->pm); + cd321x_update_done(cd321x); return; } @@ -934,7 +988,7 @@ static void cd321x_update_work(struct work_struct *work) } /* Update the TypeC MUX/PHY state */ - ret = cd321x_typec_update_mode(tps, &st, &display_busy); + ret = cd321x_typec_update_mode(tps, &st, &display_busy, &held); if (ret && !display_busy) goto retry; /* A shared display route can be busy while the dock's USB path is usable. */ @@ -951,10 +1005,14 @@ static void cd321x_update_work(struct work_struct *work) connector_status_connected); power_supply_changed(tps->psy); - cd321x_pm_complete(&cd321x->pm); + cd321x_update_done(cd321x); return; retry: + if (held) { + cd321x_retry_held_mux(cd321x); + return; + } /* A deferred PCIe activation is not evidence of a failed cable. */ cd321x_retry_revalidation(cd321x, ret != -EAGAIN); } @@ -986,6 +1044,10 @@ static int cd321x_connect(struct tps6598x *tps, u32 status) /* A plug or power IRQ alone does not refresh the cached cable metadata. */ if (cd321x_pm_event(&cd321x->pm)) mod_delayed_work(system_freezable_wq, &cd321x->resume_work, 0); + /* A held mux waits out its backoff only while nothing else happens. */ + if (cd321x_pm_mux_event(&cd321x->pm)) + mod_delayed_work(system_freezable_wq, &cd321x->resume_work, + msecs_to_jiffies(CD321X_DEBOUNCE_DELAY_MS)); if (!cd321x_pm_can_update(&cd321x->pm)) return 0; From e31582bff0cf7000f5db8e734413dab961bc3659 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Tue, 6 Oct 2026 07:58:02 -0500 Subject: [PATCH 1701/2680] accel/ane: t6021: fail the probe before power and clean up on failure The probe staged and validated the firmware after it raised the power domains and read the engine. A missing or invalid firmware file then left the extra genpd attachments in place with runtime PM enabled, and a second probe repeated the whole setup. Issue 9 reports the hang on that path. Validate the firmware first with a request-only preflight. Split the probe into an inner function that reports whether it touched the hardware, unwind the genpd attachments and the runtime PM state on every failure path, and refuse a re-probe after a hardware-touching failure with -EBUSY until reboot. Measured on an M2 Max with the out-of-tree build of the same change: with the firmware file removed, the probe fails clean with -ENOENT and no genpd domain stays on; a re-probe after a failure returns -EBUSY; with the firmware present the module binds at boot and passes 9 of 9 fixture gates. Link: https://github.com/iconidentify/aurora-linux/issues/9 Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit e0bf1fab4a4911ab363a918dbebdb59777a289b0) --- drivers/accel/ane/ane_t6021.h | 1 + drivers/accel/ane/ane_t6021_fwload.c | 31 ++++++++ drivers/accel/ane/ane_t6021_rtclient_main.c | 79 ++++++++++++++++++++- 3 files changed, 108 insertions(+), 3 deletions(-) diff --git a/drivers/accel/ane/ane_t6021.h b/drivers/accel/ane/ane_t6021.h index d50ed9e6873ec..cf04c6699710e 100644 --- a/drivers/accel/ane/ane_t6021.h +++ b/drivers/accel/ane/ane_t6021.h @@ -735,6 +735,7 @@ void ane_t6021_csne_ping_attempt(struct ane_t6021 *ane); * the staging half of the boot sequence: it provides the fw DVA that * the RVBAR fold uses; ane_t6021_boot.c consumes the staged surface. */ +int ane_t6021_fwload_check(struct device *dev); int ane_t6021_fwload_probe(struct ane_t6021 *ane); void ane_t6021_fwload_remove(struct ane_t6021 *ane); bool ane_t6021_fwload_options_ok(void); diff --git a/drivers/accel/ane/ane_t6021_fwload.c b/drivers/accel/ane/ane_t6021_fwload.c index 68d775a38dcfe..4dc4ce27b1674 100644 --- a/drivers/accel/ane/ane_t6021_fwload.c +++ b/drivers/accel/ane/ane_t6021_fwload.c @@ -520,6 +520,37 @@ static int ane_t6021_fw_patch(struct ane_t6021 *ane, u8 *img) return 0; } +int ane_t6021_fwload_check(struct device *dev) +{ + const struct ane_t602x_soc *soc = of_device_get_match_data(dev); + const struct ane_fw_image *img = soc->fw; + const struct firmware *fw = NULL; + struct ane_fw_seg segs[ANE_FW_NSEGS]; + u64 entry = 0; + u8 actual_sha[32]; + const char *reason = NULL; + int ret; + + if (!fw_load) + return 0; + ret = request_firmware(&fw, img->name, dev); + if (ret) { + dev_err(dev, + "cannot load firmware %s: %d; install it with omarchy-ane-firmware-fetch (or equivalent for your distribution)\n", + img->name, ret); + return ret; + } + + sha256(fw->data, fw->size, actual_sha); + ret = ane_fw_validate_blob(fw->data, fw->size, img, actual_sha, + segs, &entry, &reason); + if (ret) + dev_err(dev, "fwload: validation failed: %s\n", + reason ? reason : "?"); + release_firmware(fw); + return ret; +} + int ane_t6021_fwload_probe(struct ane_t6021 *ane) { const struct ane_t602x_soc *soc = of_device_get_match_data(ane->dev); diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index 8cf2d3cea39bc..ceb0875e3372e 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -1907,6 +1907,8 @@ struct ane_rtclient_pd { static LIST_HEAD(ane_rtclient_pd_list); static DEFINE_MUTEX(ane_rtclient_pd_lock); static bool ane_rtclient_pinned; +static bool ane_rtclient_probe_failed_dirty; +static DEFINE_MUTEX(ane_rtclient_probe_guard_lock); static void ane_rtclient_pd_free(struct ane_rtclient_pd *pd) { @@ -1991,7 +1993,38 @@ static int ane_rtclient_attach_genpd(struct ane_rtclient *ane) return err; } -static int ane_rtclient_probe(struct platform_device *pdev) +static void ane_rtclient_detach_genpd(struct device *dev) +{ + struct ane_rtclient_pd *pd, *tmp; + int count, i; + + count = of_count_phandle_with_args(dev->of_node, "power-domains", + "#power-domain-cells"); + if (count <= 1) + return; + + mutex_lock(&ane_rtclient_pd_lock); + list_for_each_entry_safe(pd, tmp, &ane_rtclient_pd_list, list) { + if (pd->dev != dev) + continue; + for (i = count - 1; i >= 0; i--) { + if (pd->pd_link[i]) + device_link_del(pd->pd_link[i]); + if (pd->pd_dev[i]) + dev_pm_domain_detach(pd->pd_dev[i], true); + } + ane_rtclient_pd_free(pd); + if (list_empty(&ane_rtclient_pd_list) && ane_rtclient_pinned) { + module_put(THIS_MODULE); + ane_rtclient_pinned = false; + } + break; + } + mutex_unlock(&ane_rtclient_pd_lock); +} + +static int ane_rtclient_probe_inner(struct platform_device *pdev, + bool *hardware_touched) { struct device *dev = &pdev->dev; struct resource *res; @@ -2026,6 +2059,11 @@ static int ane_rtclient_probe(struct platform_device *pdev) dev_err(dev, "legacy_query requires legacy_only=1\n"); return -EINVAL; } + if (fw_start && ane_t6021_fwload_requested()) { + ret = ane_t6021_fwload_check(dev); + if (ret) + return dev_err_probe(dev, ret, "firmware preflight failed\n"); + } ane = devm_kzalloc(dev, sizeof(*ane), GFP_KERNEL); if (!ane) @@ -2057,13 +2095,16 @@ static int ane_rtclient_probe(struct platform_device *pdev) if (ret) return ret; + *hardware_touched = true; ret = ane_rtclient_attach_genpd(ane); if (ret) return dev_err_probe(dev, ret, "extra genpd attach\n"); pm_runtime_enable(dev); ret = pm_runtime_resume_and_get(dev); - if (ret) + if (ret) { + pm_runtime_disable(dev); return dev_err_probe(dev, ret, "genpd raise failed\n"); + } ane->pmgr = devm_of_iomap(dev, dev->of_node, 1, NULL); if (IS_ERR(ane->pmgr)) { @@ -2421,6 +2462,35 @@ static int ane_rtclient_probe(struct platform_device *pdev) return ret; } +static int ane_rtclient_probe(struct platform_device *pdev) +{ + bool hardware_touched = false; + int ret; + + mutex_lock(&ane_rtclient_probe_guard_lock); + if (ane_rtclient_probe_failed_dirty) { + mutex_unlock(&ane_rtclient_probe_guard_lock); + dev_err(&pdev->dev, + "previous probe failed after touching the hardware: reboot required\n"); + return -EBUSY; + } + + ret = ane_rtclient_probe_inner(pdev, &hardware_touched); + if (ret) { + struct ane_rtclient *ane = platform_get_drvdata(pdev); + + if (hardware_touched) { + ane_rtclient_probe_failed_dirty = true; + dev_err(&pdev->dev, + "probe failed after hardware access; reboot required before retry\n"); + } + if (!ane || !ane->held) + ane_rtclient_detach_genpd(&pdev->dev); + } + mutex_unlock(&ane_rtclient_probe_guard_lock); + return ret; +} + static void ane_rtclient_remove(struct platform_device *pdev) { struct ane_rtclient *ane = platform_get_drvdata(pdev); @@ -2497,7 +2567,10 @@ static struct platform_driver ane_rtclient_driver = { static int __init ane_rtclient_init(void) { - int ret = platform_driver_register(&ane_rtclient_driver); + int ret; + + WRITE_ONCE(ane_rtclient_probe_failed_dirty, false); + ret = platform_driver_register(&ane_rtclient_driver); if (ret) ane_t6021_trace_free(); From 3bf66270fe22109b79e5e9c2cee1916b811dc2a0 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:10:50 -0700 Subject: [PATCH 1702/2680] HID: magicmouse: no pressure pad or force feedback on the C1FE trackpad MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The J700 trackpad (C1FE report layout) carries positions and the button only. It reports no finger force, so: - it is not a pressure pad: INPUT_PROP_PRESSUREPAD and the summed pressure of INPUT_MT_TOTAL_FORCE would describe an axis it does not have; - with the actuator in host-controlled mode, the click is derived from the summed force, which is always zero here, so an uploaded effect would leave the pad without a button. Set neither on C1FE, and do not create the force feedback device there. Every other MTP and SPI trackpad is unchanged. [ryan: adapted to 7.2.10 — Pressure and total-force guards already exist; add the missing C1FE haptics exclusion.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 5daf8ba9d8a3539f5da294388944a82be0e64e91) --- drivers/hid/hid-magicmouse.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index e417a65acd7e7..eda706e19aa76 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -1268,6 +1268,7 @@ static int magicmouse_setup_input_mtp(struct input_dev *input, struct magicmouse_sc *msc = hid_get_drvdata(hdev); __set_bit(INPUT_PROP_BUTTONPAD, input->propbit); + /* The C1FE controller reports no force: it is not a pressure pad. */ if (!msc->mtp_c1fe) __set_bit(INPUT_PROP_PRESSUREPAD, input->propbit); __clear_bit(BTN_0, input->keybit); @@ -2205,6 +2206,13 @@ static int magicmouse_init_haptics(struct magicmouse_sc *msc, struct hid_device trackpad_hdev->type != HID_TYPE_SPI_MOUSE) return 0; + /* + * The C1FE trackpad reports no force, so the button cannot be derived + * while the host holds an actuator: offer no force feedback there. + */ + if (msc->mtp_c1fe) + return 0; + haptics = devm_kzalloc(&trackpad_hdev->dev, sizeof(*haptics), GFP_KERNEL); if (!haptics) return -ENOMEM; From a46f77877e3360fecdf2b9625c4a32d8c2b59895 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 23:35:33 +1100 Subject: [PATCH 1703/2680] rust: sync: barrier: Add dma_wmb() and dma_rmb() A driver that fills a coherent DMA buffer and then rings the device with a relaxed MMIO write has to order the two itself, as C drivers do with dma_wmb(). On receive it is the other way round, with dma_rmb(). The barrier module has only the smp_*() barriers. Those order accesses between CPUs, not against a device, and compile away on !SMP. Add both, through helpers as for the others. They do not depend on SMP. (cherry picked from commit 9e04a4f5f16cf71ee7a554ec830732368161a3d4 ("rust: sync: barrier: Add dma_wmb() and dma_rmb()")) Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/helpers/barrier.c | 10 ++++++++++ rust/kernel/sync/barrier.rs | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/rust/helpers/barrier.c b/rust/helpers/barrier.c index fed8853745c83..7d681b1c4ab60 100644 --- a/rust/helpers/barrier.c +++ b/rust/helpers/barrier.c @@ -16,3 +16,13 @@ __rust_helper void rust_helper_smp_rmb(void) { smp_rmb(); } + +__rust_helper void rust_helper_dma_wmb(void) +{ + dma_wmb(); +} + +__rust_helper void rust_helper_dma_rmb(void) +{ + dma_rmb(); +} diff --git a/rust/kernel/sync/barrier.rs b/rust/kernel/sync/barrier.rs index 8f2d435fcd944..b1c9b8526a5fc 100644 --- a/rust/kernel/sync/barrier.rs +++ b/rust/kernel/sync/barrier.rs @@ -59,3 +59,23 @@ pub fn smp_rmb() { barrier(); } } + +/// A write-write memory barrier for memory shared with a device. +/// +/// Orders the CPU's writes to memory a device reads, such as a coherent DMA buffer, before its +/// later writes, including a relaxed MMIO write that tells the device to read that memory. +#[inline(always)] +pub fn dma_wmb() { + // SAFETY: `dma_wmb()` is safe to call. + unsafe { bindings::dma_wmb() }; +} + +/// A read-read memory barrier for memory shared with a device. +/// +/// Orders the CPU's reads of memory a device writes, such as a coherent DMA buffer, after its +/// earlier reads, including a relaxed MMIO read that told it the device had written that memory. +#[inline(always)] +pub fn dma_rmb() { + // SAFETY: `dma_rmb()` is safe to call. + unsafe { bindings::dma_rmb() }; +} From 4cbf11b980f042be406aa778e495ffd27640b43a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 23:36:29 +1100 Subject: [PATCH 1704/2680] soc: apple: sep: Order the shared buffers against the mailbox The host writes a buffer and then sends the message that points the enclave at it: a request payload, the poison that refills an SBIO reply header, the shared memory a registration hands over. The buffers are coherent allocations. The SEP node is not dma-coherent, so on arm64 they are mapped Normal-NC, and the mailbox write is writeq_relaxed() to Device memory. Nothing orders the two, so the enclave can be told to look before the bytes are there. It would then read a stale request, or write its reply's header and have the poison refill land over it. Receiving is the same the other way round: the message comes in through readq_relaxed(), and nothing keeps the reads of the buffer it points to from going ahead of it. RTKit has the same contract with the same mailbox. It puts dma_wmb() before apple_mbox_send() and dma_rmb() after a message is read. The SEP driver calls the mailbox directly and has neither. Add them: dma_wmb() before every send, in the one place sends reach the mailbox, and dma_rmb() in the receive callback, before the message is queued for the work item that reads the buffers. dma_wmb() is the right barrier for this mapping. A coherent buffer has no dma_sync_*_for_device() call, which would order it as part of the cache maintenance a streaming mapping needs. Here the barrier has to be explicit. Normal-NC needs no cache maintenance either, only the ordering of the CPU's stores to the buffer before its store to the mailbox, as the enclave sees them. On arm64 dma_wmb() is dmb(oshst), the barrier writel() itself puts before the MMIO store (__io_bw()). So dma_wmb() followed by the relaxed doorbell is what writeq() would do, with no wmb()'s full dsb. dma_rmb() is dmb(oshld), as in readq(). It also orders the MMIO read before the stores that queue the message, so the work item's later reads are ordered after it. Fixes: bac0846c9087 ("soc: apple: add Apple SEP driver (Touch ID and key sealing)") (cherry picked from commit 44710408a8cf73b76901de1cb53a865d09c7955d ("soc: apple: sep: Order the shared buffers against the mailbox")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sep.rs | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 69d3d52ffa5b4..763985d642f93 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -61,6 +61,10 @@ use kernel::{ Atomic, Relaxed, // }, + barrier::{ + dma_rmb, + dma_wmb, // + }, new_condvar, Arc, CondVar, @@ -956,7 +960,7 @@ impl SepData { } self.registered.store(true, Relaxed); - self.mbox.lock().as_ref().ok_or(EINVAL)?.send(msg, false)?; + Self::ring(self.mbox.lock().as_ref().ok_or(EINVAL)?, msg)?; // no ack on 0xFE; success is the discovery burst on 0xFD Ok(()) @@ -1109,7 +1113,20 @@ impl SepData { if self.shutting_down.load(Relaxed) { return Err(ENODEV); } - self.mbox.lock().as_ref().ok_or(ENODEV)?.send(msg, false) + Self::ring(self.mbox.lock().as_ref().ok_or(ENODEV)?, msg) + } + + /// Puts `msg` in the mailbox once the enclave can see what the CPU wrote + /// to the buffers before it: a request payload, an SBIO header's poison, + /// the shared memory a registration hands over. Those are coherent + /// allocations, which have no sync call to order them, and the mailbox + /// write is relaxed MMIO, which is not ordered after them either. + /// Without the barrier the enclave could read old bytes, or have its + /// reply's header overwritten by a refill that lands late. dma_wmb() is + /// what writel() would put there, and what RTKit puts before its sends. + fn ring(mbox: &Mailbox, msg: Message) -> Result<()> { + dma_wmb(); + mbox.send(msg, false) } @@ -2360,6 +2377,9 @@ impl MailCallback for SepData { if data.shutting_down.load(Relaxed) { return; } + // The message was read from the mailbox with relaxed MMIO. Read the + // buffers the enclave wrote before it only after that, as RTKit does. + dma_rmb(); if !data.rx.push(msg) && data.rx.dropped() == 1 { dev_err!( data.dev, From 353bb695abdde3b798e471d65b284e599aeb4dd6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 23:38:32 +1100 Subject: [PATCH 1705/2680] soc: apple: sep: Test the hwrng and sleep glue KUnit covers ControlState, the entropy exchange's bookkeeping. The code that drives it from the hwrng core and the PM notifier has no test: - the drain on the way to sleep, and its limit; - the order of the sleep holds, and their release when Touch ID refuses the suspend; - the mapping of a mailbox timeout to a full mailbox, which holds the next request back; - the hwrng read's result when requests are held back, which is not logged as a failure; - the ready-phase survey, which does not count an attempt that asked the enclave nothing. Move each decision into a small function the driver calls, with the locking, sleeping and logging left where they are. Test them there: the drain, the read's result, the survey and the mapping in control.rs, and the sleep event in pm.rs, through a trait that SepData implements and the test records calls on. Nothing changes in what the driver does. (cherry picked from commit 101931a15c0ea701d7c9f0d1ceb9118110baf0be ("soc: apple: sep: Test the hwrng and sleep glue")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/control.rs | 194 +++++++++++++++++++++++++++++++++++ drivers/soc/apple/pm.rs | 149 ++++++++++++++++++++++++--- drivers/soc/apple/sep.rs | 74 ++++++------- 3 files changed, 369 insertions(+), 48 deletions(-) diff --git a/drivers/soc/apple/control.rs b/drivers/soc/apple/control.rs index 5eb9d51c9f421..3c6bb59149824 100644 --- a/drivers/soc/apple/control.rs +++ b/drivers/soc/apple/control.rs @@ -56,6 +56,92 @@ pub(crate) enum EntropyEnd { Abandoned, } +impl EntropyEnd { + /// How a request ended that the mailbox refused with `e`. ETIMEDOUT is + /// its wait for room running out. + pub(crate) fn not_sent(e: Error) -> Self { + EntropyEnd::NotSent { + full: e == ETIMEDOUT, + } + } +} + +/// How a hwrng read ends. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ReadEnd { + /// It took this many bytes. + Bytes(usize), + /// None, as requests are being held back: nothing failed, and the core + /// tries again later. + Held(Error), + /// None, as the request failed. + Failed(Error), +} + +/// How a hwrng read ends that took `written` bytes and then stopped for +/// `failure`, if anything stopped it. +pub(crate) fn read_end(written: usize, failure: Option) -> ReadEnd { + if written > 0 { + return ReadEnd::Bytes(written); + } + // must return a negative errno, not 0: a blocking reader would spin on 0 + match failure.unwrap_or(EIO) { + e if e == EAGAIN => ReadEnd::Held(e), + e => ReadEnd::Failed(e), + } +} + +/// What came of an attempt at the ready-phase hwrng survey. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Survey { + Registered, + /// Requests were held back, so the enclave was not asked: try again, + /// without counting the attempt. + Held, + /// It failed, and attempts are left. + Retry(Error), + /// It failed, and that was the last attempt. + GiveUp(Error), +} + +/// Runs `survey` once if fewer than `max` attempts have been `used`. +/// Returns the attempts used after it, and what came of it. +pub(crate) fn survey_step( + used: u32, + max: u32, + survey: impl FnOnce() -> Result<()>, +) -> Option<(u32, Survey)> { + let attempt = used + 1; + if attempt > max { + return None; + } + Some(match survey() { + Ok(()) => (attempt, Survey::Registered), + Err(e) if e == EAGAIN => (used, Survey::Held), + Err(e) if attempt < max => (attempt, Survey::Retry(e)), + Err(e) => (attempt, Survey::GiveUp(e)), + }) +} + +/// Waits for `busy()` to clear, sleeping `step_ms` at a time through +/// `sleep()`, for at most `limit_ms`. Returns whether it cleared. +pub(crate) fn wait_idle( + limit_ms: u32, + step_ms: u32, + mut busy: impl FnMut() -> bool, + mut sleep: impl FnMut(u32), +) -> bool { + let mut waited: u32 = 0; + while busy() { + if waited >= limit_ms { + return false; + } + sleep(step_ms); + waited = waited.saturating_add(step_ms); + } + true +} + pub(crate) enum Delivery { Entropy, /// The answer to an entropy request that had timed out. `recovered` if @@ -280,6 +366,7 @@ impl ControlState { #[kernel::macros::kunit_tests(apple_sep_control)] mod tests { use super::*; + use core::cell::Cell; fn entropy(value: u32) -> proto::ControlReply { proto::ControlReply { @@ -446,4 +533,111 @@ mod tests { assert!(matches!(state.deliver(entropy(1)), Delivery::Unmatched)); assert_eq!(draw(&mut state, 0, 2), Some(2)); } + + #[test] + fn a_full_mailbox_is_told_from_other_send_failures() { + assert_eq!( + EntropyEnd::not_sent(ETIMEDOUT), + EntropyEnd::NotSent { full: true } + ); + for e in [ENODEV, EINTR, EIO] { + assert_eq!(EntropyEnd::not_sent(e), EntropyEnd::NotSent { full: false }); + } + // Only a full mailbox holds the next request back. + let mut state = ControlState::new(); + assert!(state.entropy_begin(0).is_ok()); + state.entropy_end(EntropyEnd::not_sent(ENODEV), 0); + assert_eq!(draw(&mut state, 0, 1), Some(1)); + assert!(state.entropy_begin(0).is_ok()); + state.entropy_end(EntropyEnd::not_sent(ETIMEDOUT), 0); + assert_eq!(state.entropy_begin(0), Err(EAGAIN)); + } + + #[test] + fn a_read_returns_what_it_took_or_why_not() { + assert_eq!(read_end(4, Some(ETIMEDOUT)), ReadEnd::Bytes(4)); + assert_eq!(read_end(64, None), ReadEnd::Bytes(64)); + assert_eq!(read_end(0, Some(ETIMEDOUT)), ReadEnd::Failed(ETIMEDOUT)); + // Never 0, which a blocking reader would spin on. + assert_eq!(read_end(0, None), ReadEnd::Failed(EIO)); + } + + #[test] + fn a_held_read_is_not_a_failure() { + assert_eq!(read_end(0, Some(EAGAIN)), ReadEnd::Held(EAGAIN)); + assert_eq!(read_end(8, Some(EAGAIN)), ReadEnd::Bytes(8)); + } + + #[test] + fn a_held_survey_attempt_is_not_counted() { + let max = 5; + assert_eq!(survey_step(0, max, || Err(EAGAIN)), Some((0, Survey::Held))); + assert_eq!(survey_step(3, max, || Err(EAGAIN)), Some((3, Survey::Held))); + // Not even when it would have been the last. + assert_eq!( + survey_step(max - 1, max, || Err(EAGAIN)), + Some((max - 1, Survey::Held)) + ); + } + + #[test] + fn the_survey_gives_up_after_its_attempts() { + let max = 5; + let mut used = 0; + for _ in 1..max { + assert_eq!( + survey_step(used, max, || Err(EIO)), + Some((used + 1, Survey::Retry(EIO))) + ); + used += 1; + } + assert_eq!( + survey_step(used, max, || Err(EIO)), + Some((max, Survey::GiveUp(EIO))) + ); + let ran = Cell::new(false); + let run = || { + ran.set(true); + Ok(()) + }; + assert_eq!(survey_step(max, max, run), None); + assert!(!ran.get()); + assert_eq!( + survey_step(2, max, || Ok(())), + Some((3, Survey::Registered)) + ); + } + + #[test] + fn the_drain_waits_for_the_request_in_flight() { + let slept = Cell::new(0u32); + let polls = Cell::new(0u32); + let busy = || { + polls.set(polls.get() + 1); + polls.get() <= 3 + }; + assert!(wait_idle(3000, 10, busy, |ms| slept.set(slept.get() + ms))); + assert_eq!(slept.get(), 30); + // Nothing in flight: no wait at all. + slept.set(0); + assert!(wait_idle( + 3000, + 10, + || false, + |ms| slept.set(slept.get() + ms) + )); + assert_eq!(slept.get(), 0); + } + + #[test] + fn the_drain_gives_up_at_its_limit() { + let slept = Cell::new(0u32); + assert!(!wait_idle( + 3000, + 10, + || true, + |ms| slept.set(slept.get() + ms) + )); + assert_eq!(slept.get(), 3000); + } } diff --git a/drivers/soc/apple/pm.rs b/drivers/soc/apple/pm.rs index b276890c27c78..2d918ec1fcf2a 100644 --- a/drivers/soc/apple/pm.rs +++ b/drivers/soc/apple/pm.rs @@ -41,6 +41,49 @@ pub(crate) fn keep_domains(dev: &kernel::device::Device) -> Result<()> { Ok(()) } +/// The holds a sleep event takes and releases, on [`SepData`]. +trait SleepHolds { + /// Holds entropy requests, once any in flight is answered. + fn hold_entropy(&self); + /// Ends any Touch ID capture and holds new ones. + fn hold_captures(&self) -> Result<()>; + fn release_captures(&self); + fn release_entropy(&self); +} + +impl SleepHolds for SepData { + fn hold_entropy(&self) { + self.rng_sleep_prepare(); + } + fn hold_captures(&self) -> Result<()> { + self.sleep_prepare() + } + fn release_captures(&self) { + self.sleep_finished(); + } + fn release_entropy(&self) { + self.rng_sleep_finished(); + } +} + +/// `entering`: the system is about to sleep; otherwise it has resumed, or +/// the suspend was aborted. +fn sleep_event(sep: &impl SleepHolds, entering: bool) -> Result<()> { + if entering { + sep.hold_entropy(); + if let Err(error) = sep.hold_captures() { + // A failing notifier is not included in the PM chain's rollback. + sep.release_captures(); + sep.release_entropy(); + return Err(error); + } + } else { + sep.release_captures(); + sep.release_entropy(); + } + Ok(()) +} + /// # Safety /// Called by `pm_shim.c` from the PM notifier chain, in process context. unsafe extern "C" fn pm_event(entering: bool) -> c_int { @@ -52,19 +95,10 @@ unsafe extern "C" fn pm_event(entering: bool) -> c_int { // `unregister` only after `sep_pm_unregister` has waited out any running // notifier call, so it is live for this borrow. let sep = unsafe { as ForeignOwnable>::borrow(ptr) }; - if entering { - sep.rng_sleep_prepare(); - if let Err(error) = sep.sleep_prepare() { - // A failing notifier is not included in the PM chain's rollback. - sep.sleep_finished(); - sep.rng_sleep_finished(); - return error.to_errno(); - } - } else { - sep.sleep_finished(); - sep.rng_sleep_finished(); + match sleep_event(&*sep, entering) { + Ok(()) => 0, + Err(error) => error.to_errno(), } - 0 } pub(crate) fn register(sep: Arc) -> Result<()> { @@ -114,3 +148,94 @@ pub(crate) fn unregister() { drop(unsafe { as ForeignOwnable>::from_foreign(ptr) }); } } + +#[kernel::macros::kunit_tests(apple_sep_pm)] +mod tests { + use super::*; + use core::cell::{Cell, RefCell}; + + const HOLD_ENTROPY: u8 = 1; + const HOLD_CAPTURES: u8 = 2; + const RELEASE_CAPTURES: u8 = 3; + const RELEASE_ENTROPY: u8 = 4; + + struct Calls { + made: RefCell<[u8; 8]>, + count: Cell, + refuse: Option, + } + + impl Calls { + fn new(refuse: Option) -> Self { + Calls { + made: RefCell::new([0; 8]), + count: Cell::new(0), + refuse, + } + } + fn call(&self, which: u8) { + let n = self.count.get(); + self.made.borrow_mut()[n] = which; + self.count.set(n + 1); + } + fn made(&self) -> [u8; 8] { + *self.made.borrow() + } + } + + impl SleepHolds for Calls { + fn hold_entropy(&self) { + self.call(HOLD_ENTROPY); + } + fn hold_captures(&self) -> Result<()> { + self.call(HOLD_CAPTURES); + self.refuse.map_or(Ok(()), Err) + } + fn release_captures(&self) { + self.call(RELEASE_CAPTURES); + } + fn release_entropy(&self) { + self.call(RELEASE_ENTROPY); + } + } + + #[test] + fn going_to_sleep_holds_entropy_then_captures() { + let calls = Calls::new(None); + assert_eq!(sleep_event(&calls, true), Ok(())); + assert_eq!( + calls.made(), + [HOLD_ENTROPY, HOLD_CAPTURES, 0, 0, 0, 0, 0, 0] + ); + } + + #[test] + fn a_refused_sleep_releases_what_it_held() { + // The chain sends no POST event to the notifier that failed. + let calls = Calls::new(Some(EBUSY)); + assert_eq!(sleep_event(&calls, true), Err(EBUSY)); + assert_eq!( + calls.made(), + [ + HOLD_ENTROPY, + HOLD_CAPTURES, + RELEASE_CAPTURES, + RELEASE_ENTROPY, + 0, + 0, + 0, + 0 + ] + ); + } + + #[test] + fn waking_releases_captures_then_entropy() { + let calls = Calls::new(None); + assert_eq!(sleep_event(&calls, false), Ok(())); + assert_eq!( + calls.made(), + [RELEASE_CAPTURES, RELEASE_ENTROPY, 0, 0, 0, 0, 0, 0] + ); + } +} diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 763985d642f93..d55a0ad55e772 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -1193,10 +1193,9 @@ impl SepData { let msg = proto::encode_control(&op, proto::TAG_ENTROPY); if let Err(e) = self.send(msg) { - let full = e == ETIMEDOUT; self.control .lock() - .entropy_end(control::EntropyEnd::NotSent { full }, boottime_ms()); + .entropy_end(control::EntropyEnd::not_sent(e), boottime_ms()); return Err(e); } @@ -1234,18 +1233,18 @@ impl SepData { if !self.control.lock().entropy_sleep() { return; } - let mut waited: u32 = 0; - while self.control.lock().entropy_busy() { - if waited >= RNG_SLEEP_DRAIN_MS { - dev_warn!( - self.dev, - "hwrng: a request was still in flight after {} ms; suspending anyway\n", - RNG_SLEEP_DRAIN_MS - ); - return; - } - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(10)); - waited += 10; + let idle = control::wait_idle( + RNG_SLEEP_DRAIN_MS, + 10, + || self.control.lock().entropy_busy(), + |ms| kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(ms))), + ); + if !idle { + dev_warn!( + self.dev, + "hwrng: a request was still in flight after {} ms; suspending anyway\n", + RNG_SLEEP_DRAIN_MS + ); } } @@ -1282,24 +1281,32 @@ impl SepData { if this.rng.lock().is_some() { return; } - let attempt = this.rng_survey_attempts.load(Relaxed) + 1; - if attempt > RNG_SURVEY_ATTEMPTS { + let used = this.rng_survey_attempts.load(Relaxed); + let Some((attempt, next)) = + control::survey_step(used, RNG_SURVEY_ATTEMPTS, || this.survey_hwrng()) + else { return; - } + }; this.rng_survey_attempts.store(attempt, Relaxed); - match this.survey_hwrng() { - Ok(()) => dev_info!(this.dev, "hwrng: registered after ready-phase survey (attempt {})\n", attempt), + match next { + control::Survey::Registered => dev_info!( + this.dev, + "hwrng: registered after ready-phase survey (attempt {})\n", + attempt + ), // Asleep, or holding back after a request that went unanswered: // this attempt did not ask the enclave anything. - Err(e) if e == EAGAIN => { - this.rng_survey_attempts.store(attempt - 1, Relaxed); - Self::arm_settle(this); - } - Err(e) if attempt < RNG_SURVEY_ATTEMPTS => { + control::Survey::Held => Self::arm_settle(this), + control::Survey::Retry(e) => { dev_warn!(this.dev, "hwrng: survey attempt {} failed ({:?}); retrying\n", attempt, e); Self::arm_settle(this); } - Err(e) => dev_err!(this.dev, "hwrng: survey failed after {} attempts ({:?})\n", attempt, e), + control::Survey::GiveUp(e) => dev_err!( + this.dev, + "hwrng: survey failed after {} attempts ({:?})\n", + attempt, + e + ), } } @@ -1929,17 +1936,12 @@ impl SepData { } } - if written > 0 { - return written as c_int; - } - - // must return a negative errno, not 0: a blocking reader would spin on 0 - let e = failure.unwrap_or(EIO); - // Asleep, or holding back after a request that went unanswered. - // Nothing failed in this read; the core tries again later. - if e == EAGAIN { - return e.to_errno(); - } + let e = match control::read_end(written, failure) { + control::ReadEnd::Bytes(n) => return n as c_int, + // Asleep, or holding back after a request that went unanswered. + control::ReadEnd::Held(e) => return e.to_errno(), + control::ReadEnd::Failed(e) => e, + }; let n = self.rng_failures.load(Relaxed).wrapping_add(1); self.rng_failures.store(n, Relaxed); if n == 1 { From c6caf5f14682c464c30628494f91e460c21c461d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 23:39:47 +1100 Subject: [PATCH 1706/2680] soc: apple: sep: Write no entropy request once the system starts to sleep Going to sleep sets the hold and then waits for a request in flight, for up to 3 s. That was meant to cover the mailbox's 500 ms wait for room and then the 2 s wait for the reply. But the hold is checked when the exchange is claimed, before the sender takes the mailbox lock. A sender claimed in time can then queue for the lock behind other senders, each of which may wait 500 ms for room. One of them can be a Touch ID capture, which is ended only after the entropy drain. So the 3 s bound did not hold. Once it ran out, the drain warned and the suspend went on, and the sender could write its request later in the window. That is what the hold exists to stop: on a T6000 a request sent while devices were suspended was never answered. Check the hold again with the mailbox lock held, just before the request is written. A request claimed before the sleep began but not yet written ends there as not sent, owing nothing and holding nothing back after the wake. Only a request already being written can still go out. Its 500 ms wait for room and 2 s wait for the reply are within the drain's 3 s, so the bound now holds. The drain still only warns when it runs out, as there is then nothing left that it could stop. Add a KUnit case for a request that sleep overtakes before it is written. It fails when the new check always allows the send. Fixes: 1b0451249175 ("soc: apple: sep: Hold entropy requests while the system sleeps") (cherry picked from commit d6bdc9d4ba38487b9fd41101b3f175015655c45b ("soc: apple: sep: Write no entropy request once the system starts to sleep")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/control.rs | 28 ++++++++++++++++++++++++++++ drivers/soc/apple/sep.rs | 24 ++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/control.rs b/drivers/soc/apple/control.rs index 3c6bb59149824..9140989d37305 100644 --- a/drivers/soc/apple/control.rs +++ b/drivers/soc/apple/control.rs @@ -358,6 +358,13 @@ impl ControlState { self.entropy.busy } + /// Whether a request claimed by [`Self::entropy_begin`] may still be + /// written: not once the system has begun going to sleep. Such a + /// request ends as [`EntropyEnd::NotSent`], not full. + pub(crate) fn entropy_may_send(&self) -> bool { + !self.entropy.asleep + } + pub(crate) fn entropy_wake(&mut self) { self.entropy.asleep = false; } @@ -428,6 +435,27 @@ mod tests { assert_eq!(draw(&mut state, 0, 6), Some(6)); } + #[test] + fn a_request_not_yet_written_when_sleep_begins_is_not_sent() { + let mut state = ControlState::new(); + assert!(state.entropy_begin(0).is_ok()); + assert!(state.entropy_may_send()); + // The sleep begins while the request waits for the mailbox. + assert!(state.entropy_sleep()); + assert!(!state.entropy_may_send()); + state.entropy_end(EntropyEnd::not_sent(EAGAIN), 0); + // That ends the drain, and the request is not held against the + // enclave: nothing is owed, and nothing is held back after the wake. + assert!(!state.entropy_busy()); + assert_eq!(state.entropy_begin(0), Err(EAGAIN)); + state.entropy_wake(); + assert!(state.entropy_begin(0).is_ok()); + assert!(state.entropy_may_send()); + assert!(matches!(state.deliver(entropy(3)), Delivery::Entropy)); + assert_eq!(state.entropy_take(), Some(3)); + state.entropy_end(EntropyEnd::Answered, 0); + } + #[test] fn a_late_reply_is_not_the_next_requests_answer() { let mut state = ControlState::new(); diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index d55a0ad55e772..5c5d07f19abbc 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -182,7 +182,9 @@ const PROTECTED_DATA_AVAILABLE: bool = true; const RNG_MAX_WORDS_PER_READ: usize = 16; // How long going to sleep waits for an entropy request in flight: the -// mailbox's 500 ms wait for room, then the wait for the reply. +// mailbox's 500 ms wait for room, then the wait for the reply. Only a request +// already being written can still go out once the sleep has begun; see +// `send_entropy_request()`. const RNG_SLEEP_DRAIN_MS: u32 = proto::CONTROL_TIMEOUT_MS + 1000; extern "C" { @@ -1192,7 +1194,7 @@ impl SepData { self.control.lock().entropy_begin(boottime_ms())?; let msg = proto::encode_control(&op, proto::TAG_ENTROPY); - if let Err(e) = self.send(msg) { + if let Err(e) = self.send_entropy_request(msg) { self.control .lock() .entropy_end(control::EntropyEnd::not_sent(e), boottime_ms()); @@ -1224,6 +1226,24 @@ impl SepData { result } + /// Sends an entropy request claimed by `entropy_begin()`, or EAGAIN if + /// the system has begun going to sleep since. The check is made with the + /// mailbox lock held. A sender can wait for that lock behind others, such + /// as a Touch ID capture still running, for longer than the drain on the + /// way to sleep waits for it. It must not then write its request into + /// the suspend. Lock order: mailbox, then control. + fn send_entropy_request(&self, msg: Message) -> Result<()> { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } + let mbox = self.mbox.lock(); + let mbox = mbox.as_ref().ok_or(ENODEV)?; + if !self.control.lock().entropy_may_send() { + return Err(EAGAIN); + } + Self::ring(mbox, msg) + } + /// The system is about to suspend: send no entropy request until it has /// resumed, and let one already sent be answered while the enclave can /// still answer it. On a T6000 a request sent while devices were From 52e41786573dc450ef30ae43ade89a82edea991b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 23:40:39 +1100 Subject: [PATCH 1707/2680] soc: apple: sep: Let a blocking hwrng read wait while requests are held While entropy requests are held back, after one went unanswered or while the system goes to sleep, the read refuses at once with EAGAIN. The hwrng core passes that straight to a reader of /dev/hwrng, even one that blocks. Before the hold, such a reader blocked for the 2 s request timeout and then got ETIMEDOUT. Now it gets EAGAIN from a blocking file descriptor, which is unusual. A reader that tries again on EAGAIN spins for the whole backoff, up to 60 s. The fill thread is not affected: it sleeps 10 s after any read that returns nothing. hwrng_fill() cannot tell the fill thread from a user reader, but both read with wait set. So when a blocking read has nothing yet, let it wait for the hold to end, for at most the time a request waits for its reply. That is until the retry time, the wake, or the enclave answering again, each of which now wakes it. If the hold ends, the read asks then. If it does not, the read fails with ETIMEDOUT, as a request that went unanswered does, and that is not logged as a failure. A signal ends the wait with ERESTARTSYS, so a reader being frozen for the suspend restarts its read after the resume. The enclave is still not asked while requests are held. A read that already has bytes still returns them at once. The wait holds the hwrng core's reading_mutex for up to 2 s. That is no longer than a request that timed out held it before the hold existed. Add KUnit cases for how long a held read waits and for what it returns when the hold outlasts it. The second fails with the old EAGAIN mapping. Fixes: f771659aba5e ("soc: apple: sep: Recover the entropy exchange after a request times out") Fixes: 1b0451249175 ("soc: apple: sep: Hold entropy requests while the system sleeps") (cherry picked from commit 4bba4c5c965970d0c752e245e703163f107d5dac ("soc: apple: sep: Let a blocking hwrng read wait while requests are held")) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/control.rs | 66 +++++++++++++++++++++++++++++++++--- drivers/soc/apple/sep.rs | 64 ++++++++++++++++++++++++++++++---- 2 files changed, 120 insertions(+), 10 deletions(-) diff --git a/drivers/soc/apple/control.rs b/drivers/soc/apple/control.rs index 9140989d37305..946c95d7a46d3 100644 --- a/drivers/soc/apple/control.rs +++ b/drivers/soc/apple/control.rs @@ -71,8 +71,8 @@ impl EntropyEnd { pub(crate) enum ReadEnd { /// It took this many bytes. Bytes(usize), - /// None, as requests are being held back: nothing failed, and the core - /// tries again later. + /// None, but nothing failed: requests are being held back, or a signal + /// ended the wait for that to end. The core tries again later. Held(Error), /// None, as the request failed. Failed(Error), @@ -86,7 +86,12 @@ pub(crate) fn read_end(written: usize, failure: Option) -> ReadEnd { } // must return a negative errno, not 0: a blocking reader would spin on 0 match failure.unwrap_or(EIO) { - e if e == EAGAIN => ReadEnd::Held(e), + // The reader blocked while requests were held back, as long as a + // request waits for its reply. Tell it that it timed out, as an + // unanswered request would. EAGAIN would invite it to read again + // at once. + e if e == EAGAIN => ReadEnd::Held(ETIMEDOUT), + e if e == ERESTARTSYS => ReadEnd::Held(e), e => ReadEnd::Failed(e), } } @@ -358,6 +363,18 @@ impl ControlState { self.entropy.busy } + /// How long from `now_ms` requests are held back, up to `limit_ms`, or + /// `None` if [`Self::entropy_begin`] would not refuse one for that. + pub(crate) fn entropy_held_ms(&self, now_ms: u64, limit_ms: u64) -> Option { + if self.entropy.asleep { + return Some(limit_ms); + } + match self.entropy.retry_at { + Some(at) if now_ms < at => Some(core::cmp::min(at - now_ms, limit_ms)), + _ => None, + } + } + /// Whether a request claimed by [`Self::entropy_begin`] may still be /// written: not once the system has begun going to sleep. Such a /// request ends as [`EntropyEnd::NotSent`], not full. @@ -592,10 +609,51 @@ mod tests { #[test] fn a_held_read_is_not_a_failure() { - assert_eq!(read_end(0, Some(EAGAIN)), ReadEnd::Held(EAGAIN)); + assert_eq!(read_end(0, Some(ERESTARTSYS)), ReadEnd::Held(ERESTARTSYS)); assert_eq!(read_end(8, Some(EAGAIN)), ReadEnd::Bytes(8)); } + #[test] + fn a_read_held_until_it_gives_up_is_told_it_timed_out() { + // Not EAGAIN: a blocking reader that retries on it would spin. + assert_eq!(read_end(0, Some(EAGAIN)), ReadEnd::Held(ETIMEDOUT)); + } + + #[test] + fn a_held_read_knows_how_long_to_wait() { + let limit = u64::from(proto::CONTROL_TIMEOUT_MS); + let mut state = ControlState::new(); + assert_eq!(state.entropy_held_ms(0, limit), None); + // Held back after a request went unanswered: until the retry time, + // or the reader's limit if that comes first. + assert!(state.entropy_begin(0).is_ok()); + state.entropy_end(EntropyEnd::TimedOut, 0); + let at = ENTROPY_RETRY_MIN_MS; + assert_eq!( + state.entropy_held_ms(0, limit), + Some(core::cmp::min(at, limit)) + ); + assert_eq!(state.entropy_held_ms(at - 10, limit), Some(10)); + assert_eq!(state.entropy_held_ms(at - 10, 0), Some(0)); + assert_eq!(state.entropy_held_ms(at, limit), None); + // The hold ends early when the enclave answers again. + assert!(matches!( + state.deliver(entropy(1)), + Delivery::EntropyLate { recovered: true } + )); + assert_eq!(state.entropy_held_ms(0, limit), None); + // Asleep: for as long as the reader may wait. + assert!(!state.entropy_sleep()); + assert_eq!(state.entropy_held_ms(0, limit), Some(limit)); + state.entropy_wake(); + assert_eq!(state.entropy_held_ms(0, limit), None); + // Whenever begin refuses with EAGAIN, the reader has a hold to wait on. + assert!(state.entropy_begin(0).is_ok()); + state.entropy_end(EntropyEnd::TimedOut, 0); + assert_eq!(state.entropy_begin(1), Err(EAGAIN)); + assert!(state.entropy_held_ms(1, limit).is_some()); + } + #[test] fn a_held_survey_attempt_is_not_counted() { let max = 5; diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 5c5d07f19abbc..e94f6dc94caf0 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -1226,6 +1226,46 @@ impl SepData { result } + /// As [`Self::get_entropy_word`], for a reader that blocks. While requests + /// are held back, wait for that to end, for as long as a request waits + /// for its reply, rather than fail at once. EAGAIN if it does not end + /// in that time, ERESTARTSYS if a signal comes. + fn get_entropy_word_blocking(&self) -> Result { + let limit = u64::from(proto::CONTROL_TIMEOUT_MS); + let deadline = boottime_ms().saturating_add(limit); + loop { + // As `hwrng_fill()` checks before each word: a removal wakes the + // wait below, and no request may go out after it. + if self.rng_shutdown.load(Relaxed) { + return Err(ENODEV); + } + match self.get_entropy_word() { + Err(e) if e == EAGAIN => {} + done => return done, + } + let mut guard = self.control.lock(); + let now = boottime_ms(); + let left = deadline.saturating_sub(now); + match guard.entropy_held_ms(now, left) { + // It ended since; ask now. + None if left > 0 => continue, + None | Some(0) => return Err(EAGAIN), + Some(ms) => { + if self.rng_shutdown.load(Relaxed) { + return Err(ENODEV); + } + let ms = u32::try_from(ms).unwrap_or(proto::CONTROL_TIMEOUT_MS); + if let CondVarTimeoutResult::Signal { .. } = self + .control_wq + .wait_interruptible_timeout(&mut guard, time::msecs_to_jiffies(ms)) + { + return Err(ERESTARTSYS); + } + } + } + } + } + /// Sends an entropy request claimed by `entropy_begin()`, or EAGAIN if /// the system has begun going to sleep since. The check is made with the /// mailbox lock held. A sender can wait for that lock behind others, such @@ -1271,6 +1311,8 @@ impl SepData { /// The system has resumed, or the suspend was aborted. pub(crate) fn rng_sleep_finished(&self) { self.control.lock().entropy_wake(); + // A blocking hwrng reader may be waiting for this. + self.control_wq.notify_all(); } fn control_survey(&self) { @@ -1942,7 +1984,13 @@ impl SepData { failure = Some(ENODEV); break; } - match self.get_entropy_word() { + // Only a read that has nothing yet waits while requests are held. + let word = if written == 0 { + self.get_entropy_word_blocking() + } else { + self.get_entropy_word() + }; + match word { Ok(value) => { // SAFETY: the core guarantees `buf` is valid for `max` // bytes and aligned for any type, and `i < max / 4`. @@ -1958,7 +2006,7 @@ impl SepData { let e = match control::read_end(written, failure) { control::ReadEnd::Bytes(n) => return n as c_int, - // Asleep, or holding back after a request that went unanswered. + // Held back for as long as the read could wait, or interrupted. control::ReadEnd::Held(e) => return e.to_errno(), control::ReadEnd::Failed(e) => e, }; @@ -2209,10 +2257,14 @@ impl SepData { control::Delivery::Entropy => { self.control_wq.notify_all(); } - control::Delivery::EntropyLate { recovered: true } => dev_info!( - self.dev, - "hwrng: the enclave answered a request that had timed out; reads resume\n" - ), + control::Delivery::EntropyLate { recovered: true } => { + dev_info!( + self.dev, + "hwrng: the enclave answered a request that had timed out; reads resume\n" + ); + // A blocking hwrng reader may be waiting for this. + self.control_wq.notify_all(); + } control::Delivery::EntropyLate { recovered: false } => {} control::Delivery::Matched => self.control_wq.notify_all(), control::Delivery::Unmatched => {} From 6a0c5377b4dd3a6d9a6030b0c444b52afb8ae2dc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 23:41:54 +1100 Subject: [PATCH 1708/2680] KEYS: trusted: Skip after_core_init for the reason it cannot run trusted_source_after_core_init expects the core's init to succeed and then skips when the built-in fake was not tried, blaming another source for holding the slot. With trusted.rng=trusted_source_kunit the init fails with -EINVAL, as it should, since that RNG is no built-in source's. The expectation fails, and the skip that follows overwrites the failure, so the case is reported as skipped for a slot nobody holds. With trusted.source= naming another source, the fake is not tried either, and the skip blames another source too. Look at the result first, and skip with the reason the fake was not tried: trusted.rng= rules it out, trusted.source= rules it out, or a source holds the slot. Which of the last two applies is found by trying to register the runtime fake, which trusted.source= refuses with -ENODEV and a held slot with -EBUSY. Any other result of the init is still a failure. Let the built-in fake's init() result be set by a test, for the next change. It stays -ENODEV here. Fixes: 4f15946bdd60 ("KEYS: trusted: Keep a source that registers before init_trusted()") (cherry picked from commit 39b1fdcee0d32127f3be04c578353da9a2783cd6 ("KEYS: trusted: Skip after_core_init for the reason it cannot run")) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../trusted-keys/tests/trusted_source_kunit.c | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/security/keys/trusted-keys/tests/trusted_source_kunit.c b/security/keys/trusted-keys/tests/trusted_source_kunit.c index 6a5eb3e79d756..3892df41078ff 100644 --- a/security/keys/trusted-keys/tests/trusted_source_kunit.c +++ b/security/keys/trusted-keys/tests/trusted_source_kunit.c @@ -33,6 +33,7 @@ struct trusted_source_fake { /* A built-in source for trusted_key_core_init(). */ struct trusted_key_ops builtin_ops; struct trusted_key_source builtin; + int builtin_ret; int builtin_inits; }; @@ -54,7 +55,7 @@ static void fake_exit(void) static int fake_builtin_init(void) { fake->builtin_inits++; - return -ENODEV; + return fake->builtin_ret; } /* Not a cipher: an invertible stand-in, so that unseal can be checked. */ @@ -107,6 +108,32 @@ static void fake_register(struct kunit *test) KUNIT_ASSERT_EQ(test, ret, 0); } +/* + * Run the core's init with the built-in fake, or skip, giving the reason, if + * the fake is not tried: a source holds the slot, or trusted.source= or + * trusted.rng= rules the fake out. + */ +static int fake_core_init(struct kunit *test) +{ + int ret = trusted_key_core_init(&fake->builtin, 1); + + if (fake->builtin_inits) + return ret; + + /* The only error before a source's init(). */ + if (ret == -EINVAL) + kunit_skip(test, "trusted.rng= rules out the built-in source"); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = register_trusted_key_source(&fake->src); + if (!ret) + unregister_trusted_key_source(&fake->src); + if (ret == -EBUSY) + kunit_skip(test, "another trusted-key source holds the slot"); + kunit_skip(test, "trusted.source= rules out the built-in source"); + return 0; +} + /* Instantiate a trusted key from @cmd; it is put when the test ends. */ static struct key *fake_key(struct kunit *test, const char *cmd) { @@ -195,9 +222,7 @@ static void trusted_source_before_core_init(struct kunit *test) */ static void trusted_source_after_core_init(struct kunit *test) { - KUNIT_EXPECT_EQ(test, trusted_key_core_init(&fake->builtin, 1), 0); - if (!fake->builtin_inits) - kunit_skip(test, "another trusted-key source holds the slot"); + KUNIT_EXPECT_EQ(test, fake_core_init(test), 0); /* With no source, the absent stubs answer. */ KUNIT_EXPECT_EQ(test, PTR_ERR(fake_key(test, "new 32")), -ENODEV); @@ -380,6 +405,7 @@ static int trusted_source_init(struct kunit *test) fake->builtin_ops = fake->ops; fake->builtin_ops.init = fake_builtin_init; + fake->builtin_ret = -ENODEV; fake->builtin.name = FAKE_BUILTIN_NAME; fake->builtin.ops = &fake->builtin_ops; From eb9d78de7f9e4ea0749ad982ecc1f5bdf7c5b843 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 23:42:37 +1100 Subject: [PATCH 1709/2680] KEYS: trusted: Give the slot to a built-in source only from init_trusted() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit trusted_key_core_init() is exported for KUnit so that the tests can run the core's init again, after init_trusted(), with synthetic built-in sources. When such a source's init() succeeds, the core gives it the slot as init_trusted() would. But a source that takes the slot this way is never withdrawn: unregister_trusted_key_source() withdraws only a registered source, and this one is not. A test's source is freed when the test ends, so the dispatch would be left on it, and every later seal would call into freed memory. Today's tests pass only a built-in source that fails with -ENODEV, so nothing has hit this. Let only the first call, init_trusted()'s, give the slot to a built-in source. A later call whose source initialises calls the source's exit() and returns -EBUSY, leaving the slot as it was. init_trusted() runs once, at late_initcall, and built-in KUnit runs after the initcalls, so no real source is affected. Nothing changes without KUnit. Add a KUnit case for a built-in source that initialises after the core's init. It fails without the check. [ryan: adapted to 7.2.10 — context only: trusted_key_core_done goes next to trusted_key_source_migratable, which 7.2.10 adds in commit af02166939e7 ("KEYS: trusted: Keep the state of a runtime source out of read-only data")] (cherry picked from commit 75890a6821030364000ad84e49b58fdb2e2332e7 ("KEYS: trusted: Give the slot to a built-in source only from init_trusted()")) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../trusted-keys/tests/trusted_source_kunit.c | 24 +++++++++++++++++++ security/keys/trusted-keys/trusted_core.c | 17 +++++++++++-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/security/keys/trusted-keys/tests/trusted_source_kunit.c b/security/keys/trusted-keys/tests/trusted_source_kunit.c index 3892df41078ff..3ef2e348f17a0 100644 --- a/security/keys/trusted-keys/tests/trusted_source_kunit.c +++ b/security/keys/trusted-keys/tests/trusted_source_kunit.c @@ -237,6 +237,28 @@ static void trusted_source_after_core_init(struct kunit *test) KUNIT_EXPECT_EQ(test, fake->builtin_inits, 1); } +/* + * After the core's init, a built-in source is not given the slot, even one + * that initialises: nothing could withdraw it, and this one is gone once the + * test ends. The slot stays free for a source module. + */ +static void trusted_source_builtin_after_core_init(struct kunit *test) +{ + fake->builtin_ret = 0; + KUNIT_EXPECT_EQ(test, fake_core_init(test), -EBUSY); + KUNIT_EXPECT_EQ(test, fake->builtin_inits, 1); + KUNIT_EXPECT_EQ(test, fake->exits, 1); + KUNIT_EXPECT_EQ(test, PTR_ERR(fake_key(test, "new 32")), -ENODEV); + KUNIT_EXPECT_EQ(test, fake->seals, 0); + + /* The fake was tried, so no source held the slot: no skip here. */ + KUNIT_ASSERT_EQ(test, register_trusted_key_source(&fake->src), 0); + KUNIT_EXPECT_FALSE(test, IS_ERR(fake_key(test, "new 32"))); + KUNIT_EXPECT_EQ(test, fake->seals, 1); + unregister_trusted_key_source(&fake->src); + KUNIT_EXPECT_EQ(test, fake->exits, 2); +} + /* Keys are sealed and unsealed by the source until it withdraws. */ static void trusted_source_dispatch(struct kunit *test) { @@ -423,6 +445,8 @@ static struct kunit_case trusted_source_cases[] = { KUNIT_CASE(trusted_source_exclusive), KUNIT_CASE(trusted_source_init_failure), KUNIT_CASE(trusted_source_invalid), + /* Last, as a core without the check would leave the fake in the slot. */ + KUNIT_CASE(trusted_source_builtin_after_core_init), {} }; diff --git a/security/keys/trusted-keys/trusted_core.c b/security/keys/trusted-keys/trusted_core.c index 5387f66316ff1..e5acabbad7333 100644 --- a/security/keys/trusted-keys/trusted_core.c +++ b/security/keys/trusted-keys/trusted_core.c @@ -77,6 +77,8 @@ static struct trusted_key_source *trusted_key_source_registered; * registered later keeps its migration policy here, -1 while there is none. */ static int trusted_key_source_migratable = -1; +/* trusted_key_core_init() has run once, from init_trusted(). */ +static bool trusted_key_core_done; /* Defined below; registration needs it before its definition. */ static int kernel_get_random(unsigned char *key, size_t key_len); @@ -505,9 +507,13 @@ static int kernel_get_random(unsigned char *key, size_t key_len) * the absent stubs, so that a source module can register later. * * init_trusted() calls this once, with trusted_key_sources. The KUnit tests - * call it with synthetic sources of their own. + * call it later, with synthetic sources of their own. Only the first call + * gives the slot to a built-in source: one that takes it is never withdrawn, + * as unregister_trusted_key_source() withdraws only a registered source, + * and a test's source is gone once the test ends. * - * Return: 0, or the error of trusted.rng or of a built-in source's init(). + * Return: 0, the error of trusted.rng or of a built-in source's init(), or + * -EBUSY for a built-in source that initialises after the first call. */ VISIBLE_IF_KUNIT int trusted_key_core_init(const struct trusted_key_source *sources, size_t n) @@ -557,6 +563,12 @@ trusted_key_core_init(const struct trusted_key_source *sources, size_t n) get_random = kernel_get_random; ret = sources[i].ops->init(); + if (!ret && trusted_key_core_done) { + if (sources[i].ops->exit) + sources[i].ops->exit(); + ret = -EBUSY; + goto out; + } if (!ret) { static_call_update(trusted_key_seal, sources[i].ops->seal); static_call_update(trusted_key_unseal, sources[i].ops->unseal); @@ -591,6 +603,7 @@ trusted_key_core_init(const struct trusted_key_source *sources, size_t n) if (!ret) trusted_key_source_active = true; out: + trusted_key_core_done = true; mutex_unlock(&trusted_key_source_lock); return ret; } From 594d2b2ab35bf2d531a0baaa8116c654023eb92a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 18 Oct 2026 01:41:01 +1100 Subject: [PATCH 1710/2680] rust: workqueue: expose the default system workqueue Linux 7.2.10 provides system_dfl_wq for work without CPU affinity, but its Rust workqueue API only exposes the deprecated system_wq and the older system queues. Add a wrapper using the existing Queue::from_raw pattern and the upstream system_dfl_wq declaration. Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/kernel/workqueue.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs index 7e253b6f299ce..597f24f2d804c 100644 --- a/rust/kernel/workqueue.rs +++ b/rust/kernel/workqueue.rs @@ -1024,6 +1024,18 @@ where { } +/// Returns the default system work queue (`system_dfl_wq`). +/// +/// Workers are not bound to any specific CPU, not concurrency managed, and all queued work items +/// are executed immediately as long as the `max_active` limit is not reached and resources are +/// available. +/// +/// Use this queue for work items which do not require a specific CPU. +pub fn system_dfl() -> &'static Queue { + // SAFETY: `system_dfl_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_dfl_wq) } +} + /// Returns the system work queue (`system_wq`). /// /// It is the one used by `schedule[_delayed]_work[_on]()`. Multi-CPU multi-threaded. There are From 43bbdd06302fe9417bcc1452b1409d8c0398aafe Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 18 Oct 2026 01:41:01 +1100 Subject: [PATCH 1711/2680] soc: apple: sep: queue all work on system_dfl_wq Move mailbox receive, delayed settle, enrolment and verification work from deprecated system_wq to system_dfl_wq. These workers operate on shared SepData state and do not depend on the submitting CPU. Unbound workers also allow capture waits without occupying a per-CPU queue. Keep the work IDs, delays, cancellation and session locking unchanged. This removes the deprecated-workqueue message at boot. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 4 ++-- drivers/soc/apple/sep.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index e3f6b613cc605..be2563c050cfd 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -3071,7 +3071,7 @@ impl SepData { return; }; this.capture_epoch.store(epoch, Relaxed); - if workqueue::system() + if workqueue::system_dfl() .enqueue::, ENROL_WORK_ID>(this.clone()) .is_err() { @@ -3085,7 +3085,7 @@ impl SepData { return; }; this.capture_epoch.store(epoch, Relaxed); - if workqueue::system() + if workqueue::system_dfl() .enqueue::, VERIFY_WORK_ID>(this.clone()) .is_err() { diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index e94f6dc94caf0..f04f67d424531 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -2063,7 +2063,7 @@ impl SepData { return; } let delay = time::msecs_to_jiffies(SETTLE_MS); - let _ = workqueue::system() + let _ = workqueue::system_dfl() .enqueue_delayed::, SETTLE_WORK_ID>(this.clone(), delay); } @@ -2463,7 +2463,7 @@ impl MailCallback for SepData { } let this: Arc = data.into(); - let _ = workqueue::system().enqueue::, 0>(this); + let _ = workqueue::system_dfl().enqueue::, 0>(this); } } From 31b72e2168c42be681d35ebdc49f31d242338036 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 18 Oct 2026 01:41:02 +1100 Subject: [PATCH 1712/2680] drm/asahi: free completed commands on system_dfl_wq Completed commands are queued for destruction outside the queue lock. Their cleanup has no dependence on the submitting CPU. Use the default unbound system queue to avoid enqueueing on deprecated system_wq. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/workqueue.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs index a88fe6d4a6a94..3c31d383d5ffd 100644 --- a/drivers/gpu/drm/asahi/workqueue.rs +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -919,7 +919,7 @@ impl WorkQueue for WorkQueue::ver { cmd.inner.gpu_va() ); cmd.as_mut().inner_mut().complete(); - workqueue::system().enqueue(cmd); + workqueue::system_dfl().enqueue(cmd); } mod_pr_debug!( From a07558257ef1d7f1f91f22a00a01741c400ca34d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 18 Oct 2026 01:41:02 +1100 Subject: [PATCH 1713/2680] rust_binder: defer process cleanup on system_dfl_wq Deferred release and flush use the refcounted Process and its locks, without CPU-local state. Queue them on the default unbound system queue instead of deprecated system_wq, retaining the existing coalescing. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/android/binder/process.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/android/binder/process.rs b/drivers/android/binder/process.rs index 5b8f73ec19312..f59b4ba204b2c 100644 --- a/drivers/android/binder/process.rs +++ b/drivers/android/binder/process.rs @@ -1650,7 +1650,7 @@ impl Process { if should_schedule { // Ignore failures to schedule to the workqueue. Those just mean that we're already // scheduled for execution. - let _ = workqueue::system().enqueue(this); + let _ = workqueue::system_dfl().enqueue(this); } drop(binderfs_file); @@ -1667,7 +1667,7 @@ impl Process { if should_schedule { // Ignore failures to schedule to the workqueue. Those just mean that we're already // scheduled for execution. - let _ = workqueue::system().enqueue(Arc::from(this)); + let _ = workqueue::system_dfl().enqueue(Arc::from(this)); } Ok(()) } From cf5241eb84e0a4e14ff567210ab511cfe10be269 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 09:37:21 -0700 Subject: [PATCH 1714/2680] tools: aurora-sep: release 11.31 linux-aurora 7.1.12.aurora2-11.31, built from fd04a98e74da: - PCIe over Thunderbolt on M1 Pro/Max and M2 Pro/Max by default (#10) with the T600x DART window and dock readiness fixes; - igb and igc for Thunderbolt 3 and 4 dock Ethernet; - the MacBook Neo radios in the shared kernel (#11); - Neural Engine runtime PM (aurora-silicon/linux#155) and the ANE enabled on the M1, M1 Pro and M2 Pro. The installer checks a MacBook Neo for its own Wi-Fi firmware, calibration and country files and prints the Neo's sleep, Wi-Fi and Bluetooth limits. The test plan covers dock PCIe on the Pro/Max chips with exact log lines, the Neural Engine on the newly enabled chips, and a MacBook Neo step. libfprint, aurora-touchid and m1n1-aurora are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 87e165cc0aa6e6587f15032f94006e92d529aff2) --- tools/aurora-sep/install-aurora-sep.sh | 132 +++++++++++++++++++++---- 1 file changed, 112 insertions(+), 20 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index ced4100c78b5a..b8041c9505068 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -5,7 +5,7 @@ # ... | bash -s -- --read-only install, but never let the driver write to the enclave # ... | bash -s -- --uninstall go back to the kernel this Mac had before # -# Kernel: iconidentify/aurora-linux custom/sep (0cde04307749), aurora-silicon/linux aurora-wip plus the +# Kernel: iconidentify/aurora-linux custom/sep (fd04a98e74da), aurora-silicon/linux aurora-wip plus the # Secure Enclave (Touch ID) driver, Thunderbolt (#8), the Apple video # decoder (#45), the M2 Max (t6021) profile and the consolidated Touch ID # series (aurora-silicon/linux#69: matching after a reboot on every profile, @@ -42,6 +42,18 @@ # 11.25.1 (installer only, same packages): rebuilding m1n1's stage 2 now keeps # the rest of /etc/default/update-m1n1, so a MacBook Neo keeps its own M1N1= # and U_BOOT= instead of being rebuilt from an m1n1 that cannot boot it. +# 11.31 brings up PCIe over Thunderbolt on the M1 Pro/Max and M2 Pro/Max by +# default (iconidentify/aurora-linux#10, Wesley Grimes, with follow-up fixes; +# pcie_apple.tunnel_kernel_init=0 turns it off): USB, Ethernet and audio +# behind Thunderbolt docks and displays now work there as they do on M1. It +# builds the drivers for the Intel Ethernet in Thunderbolt 3 and 4 docks (igb +# and igc), and carries the MacBook Neo's Wi-Fi in the shared kernel +# (iconidentify/aurora-linux#11). On a Neo the radios need that Neo's own +# firmware, calibration and country files, which the script checks for; sleep +# is refused while they are active, and Bluetooth is off by default. +# The Neural Engine now powers off when idle (aurora-silicon/linux#155, Joshua +# Warren), and is switched on for the M1, M1 Pro and M2 Pro as well as the M1 +# Max and M2 Max. # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -70,8 +82,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.25 -TAG=sep-7.1.12.aurora2-11.25.1 +VERSION=7.1.12.aurora2-11.31 +TAG=sep-7.1.12.aurora2-11.31 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -79,8 +91,8 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst f8b78b21f4031113a06c7f7fc7cbe1d72a23ba803adfcfc838ee3e927d3d0c85" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst a1a1524ccc35b4b4c82e4a01fda90ae13c43719ecb8dfc8108fa9a008d9500e6" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst 7e7e04db48e502325f9f287ffc01543140beb64feb63719f69f937851d7cd345" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst a04ae0735998f409976fb0dd815f2de53eee2740c7326aef2f6ea403f3077494" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20261002-1-any.pkg.tar.zst a9dda6e0526874e4ac760629f3aa5bd37421379a1d8af34000f3dc7b73a21b17" "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" @@ -502,6 +514,50 @@ ane_dkms_notice() { Pro/Max." } +# The MacBook Neo's Wi-Fi and Bluetooth need this Neo's own firmware, +# calibration and country files from its macOS; none ship with Linux, and +# another unit's files can't be substituted. Report what is there without +# failing the install: the kernel runs without them, the radios stay off. +# File names follow Documentation/networking/device_drivers/wifi/mt7932-neo.rst. +neo_radio_notice() { + local fw=/usr/lib/firmware/mediatek f missing="" mac cc="" + for f in IZUBA_WIFI_MT7932_patch_mcu_1_2_hdr.bin IZUBA_W7932_2.bin ppr.bin \ + config-original.bin wcal.bin oca2.bin; do + [[ -s $fw/mt7932/$f ]] || missing+=" mt7932/$f" + done + # The policy is per country with no fallback to the world file once a + # country is set; world-XZ.bin only covers the unset ("00") case. + if command -v iw >/dev/null; then + cc=$(iw reg get 2>/dev/null | awk '$1=="global"{g=1; next} g && $1=="country"{sub(":","",$2); print $2; exit}') + fi + if [[ -z $cc || $cc == 00 ]]; then + [[ -s $fw/mt7932/policy/world-XZ.bin ]] || missing+=" mt7932/policy/world-XZ.bin" + else + [[ -s $fw/mt7932/policy/$cc.bin ]] || missing+=" mt7932/policy/$cc.bin (country $cc)" + fi + mac=$(find /proc/device-tree -path '*wifi*' -name local-mac-address 2>/dev/null | head -1) + if [[ -n $mac ]] && od -An -tx1 "$mac" | grep -q '[1-9a-f]'; then + say "Wi-Fi MAC address: provided by this Neo's m1n1" + else + warn "m1n1 did not provide the Wi-Fi MAC address (wifi0 local-mac-address); Wi-Fi will refuse to start" + fi + if [[ -z $missing ]]; then + say "Wi-Fi firmware, calibration and country files: all present under $fw/mt7932" + else + warn "Wi-Fi needs this Neo's own files under $fw, and these are missing:$missing + They come from this Neo's own macOS and can't be shared between machines. + See Documentation/networking/device_drivers/wifi/mt7932-neo.rst in + iconidentify/aurora-linux. Wi-Fi stays off until they are all there." + fi + say "MacBook Neo, read before you reboot: + - Sleep isn't supported on the Neo yet. While Wi-Fi/Bluetooth support is + active (the default) the kernel refuses suspend, so closing the lid does + nothing: shut down instead of putting it in a bag. + - Wi-Fi works on 2.4 GHz and on 5 GHz channels 36-48 only, with WPA2 (AES) + or open networks. 5 GHz networks on channels 149-165 won't be listed. + - Bluetooth is off by default in this release." +} + install_all() { local entry file sha kernel chain version_notice @@ -575,6 +631,7 @@ install_all() { calibration $sudo systemctl daemon-reload sep_policy + if is_neo; then neo_radio_notice; fi pacman -Q linux-aurora libfprint aurora-touchid echo if [[ -f $MODPROBE_CONF ]]; then @@ -772,12 +829,16 @@ SAFETY, NON-NEGOTIABLE An EMPTY /sys/bus/thunderbolt/devices/ with nothing plugged in is normal: routers only appear when a device is attached. Do not report that as a failure. - Also normal, on every Mac except the M1 Air and 13" M1 MacBook Pro: - PCIe-C tunnel disabled: m1n1 handoff is not initialized - Anything behind a dock's PCIe controller -- ethernet, storage, and on - Thunderbolt 3 docks such as the CalDigit TS3 the USB ports too -- only - works on t8103 so far. Displays, and USB on USB4 docks, do not need the - PCIe tunnel. Report the line only if you see it on a t8103. + Anything behind a Thunderbolt 3 dock's PCIe controller -- Ethernet, + storage, and on docks such as the CalDigit TS3 Plus the USB ports too -- + now comes up on M1, M1 Pro/Max and M2 Pro/Max. On the Pro/Max chips the + kernel starts the tunnel itself when a dock is plugged in, and logs + port ... cold init done, status 0x3 ... + (pcie_apple.tunnel_kernel_init=0 turns that off). Displays, and USB on + USB4 docks, do not need the PCIe tunnel. On the M2 MacBook Air/Pro 13" + and on M3 the tunnel isn't supported yet, so this line is normal there: + PCIe-C tunnel disabled: not initialized by m1n1 or the kernel + Report it if you see it on an M1, M1 Pro/Max or M2 Pro/Max. 6b. With a dock or a DisplayPort monitor, in this order: - attached at boot: does the display come up, and at what resolution @@ -793,8 +854,15 @@ SAFETY, NON-NEGOTIABLE (no dock), attached at boot. Does each come up at its own native resolution? Quote "hyprctl monitors" (name, mode, and the port each is on). Then log out and back in, and repeat. - - PCIe behind the dock: ethernet, USB storage, card readers - do they - enumerate and still work after a replug + - PCIe behind the dock: Ethernet, USB storage, card readers, and a + keyboard and mouse on the dock's USB ports. Do they all work, and + still work after a replug? Quote: + lspci -nn + sudo dmesg | grep -E 'cold init done|link up after|translation fault|HC died' + Any "translation fault" or "HC died" line is a failure to report. + For a full report, after plugging the dock in: + curl -fsSLO https://raw.githubusercontent.com/iconidentify/aurora-linux/refs/tags/sep-7.1.12.aurora2-11.31/tools/aurora-tb/tb-pcie-report + sudo sh tb-pcie-report --no-wait 6c. Across suspend: systemctl suspend @@ -817,24 +885,48 @@ SAFETY, NON-NEGOTIABLE 9. NEURAL ENGINE This build carries the in-tree Apple Neural Engine driver - (aurora-silicon/linux#155), switched on only for the M1 Max and the M2 Max. - On every other Mac no Neural Engine lines are expected, and that is a pass. + (aurora-silicon/linux#155), switched on for the M1, M1 Pro, M1 Max, M2 Pro + and M2 Max. On every other Mac no Neural Engine lines are expected, and that + is a pass. The ANE now powers off about 1.5 s after its last use: + cat /sys/bus/platform/drivers/ane*/*.ane/power/runtime_status + should read "suspended" while nothing uses it (M1 family). pacman -Q omarchy-ane-dkms 2>/dev/null # if installed, say so: its modules replace this kernel's modinfo -F filename ane ane_t6021 ls -l /dev/accel/ 2>/dev/null sudo dmesg | grep -iE '\bane\b|ane_t6021|neural' | head -40 - M1 Max PASS: the ane module is bound, /dev/accel/accel0 exists, and the - module path is under kernel/drivers/accel/ane (not updates/dkms). - M2 Max: without the Neural Engine firmware, expect a firmware load error and - nothing else broken. With it (fetched by Joshua Warren's + M1, M1 Pro, M1 Max PASS: the ane module is bound, /dev/accel/accel0 exists, + and the module path is under kernel/drivers/accel/ane (not updates/dkms). + M2 Pro, M2 Max: without the Neural Engine firmware, expect a firmware load + error and nothing else broken. With it (fetched by Joshua Warren's omarchy-ane-firmware-fetch), expect ane_t6021 bound and /dev/accel/accel0. + The M2 Pro has not run it before this build: report it either way. If omarchy-ane's tools are installed, also run omarchy-ane-check, and on an - M2 Max omarchy-ane-check --smoke; quote their result lines. + M2 Pro/Max omarchy-ane-check --smoke; quote their result lines. Report any kernel log line at emergency or alert level, and whether idle battery drain changed against the previous build. If the Mac does not finish booting, add module_blacklist=ane,ane_t6021 to the kernel command line from the boot menu, and report that. +10. MACBOOK NEO (J700) ONLY + The Neo's Wi-Fi is in the regular kernel now. It needs this Neo's own + firmware, calibration and country files; the installer listed any that + were missing. Never install another Neo's files. + sudo dmesg | grep -iE 'mt7932|piodma|REGULATORY|CALIBRATION|admission' + iw reg get | head -3 + nmcli device wifi list | head + - Without the country file, expect + REGULATORY_BLOCKED: generation=N error=-2 recovery-required=0 + and nothing else broken; quote it. After installing that file, retry a + scan (nmcli device wifi rescan): it should come up without a reboot. + - With every file present: does it scan, connect to a WPA2 network and + pass traffic? Try a 2.4 GHz network and a 5 GHz one on channels 36-48. + Networks on channels 149-165 are not listed, by design. + - systemctl suspend is refused by design while the radios are active. + Expect "sleep refused: Neo radio bootstrap retains DMA memory until full + hardware reset" and quote it. + - Bluetooth is off by default. Only test it if the owner asks (see + Documentation/networking/device_drivers/wifi/mt7932-neo.rst). + Report "not tested" honestly rather than guessing, for any of the above. HOW TO REPORT From d81283301a74d79ae023b6ffcd3b4af5c625fad9 Mon Sep 17 00:00:00 2001 From: RyanTheTide Date: Fri, 9 Oct 2026 21:42:56 +1100 Subject: [PATCH 1715/2680] media: apple: isp: reject overlapping and unaligned IPC rings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retain the boot argument and command span, then reject channel rings that overlap it, the channel table or another ring. Require aligned u64 publication addresses and complete 64-byte message slots. Keep full 64-bit addresses and use subtraction for extent bounds and overlap checks to reject wrapping firmware-selected ranges. [ryan: adapted to 7.2.10 — Use VIDEO_APPLE_ISP_IPC_LAYOUT_KUNIT_TEST for the standalone layout suite and preserve the existing vb2 suite.] Signed-off-by: RyanTheTide Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit dab73ebe59d70d8bec4249848e71afe08c69dbc2) --- drivers/media/platform/apple/isp/Kconfig | 10 ++ drivers/media/platform/apple/isp/Makefile | 1 + drivers/media/platform/apple/isp/isp-drv.h | 2 + drivers/media/platform/apple/isp/isp-fw.c | 27 ++++ .../platform/apple/isp/isp-ipc-layout-test.c | 120 ++++++++++++++++++ .../media/platform/apple/isp/isp-ipc-layout.h | 49 +++++++ 6 files changed, 209 insertions(+) create mode 100644 drivers/media/platform/apple/isp/isp-ipc-layout-test.c create mode 100644 drivers/media/platform/apple/isp/isp-ipc-layout.h diff --git a/drivers/media/platform/apple/isp/Kconfig b/drivers/media/platform/apple/isp/Kconfig index cc2c59b51a441..6876f1fadb6ca 100644 --- a/drivers/media/platform/apple/isp/Kconfig +++ b/drivers/media/platform/apple/isp/Kconfig @@ -29,3 +29,13 @@ config VIDEO_APPLE_ISP_KUNIT_TEST kernel developers running KUnit. If unsure, say N. + +config VIDEO_APPLE_ISP_IPC_LAYOUT_KUNIT_TEST + tristate "Apple ISP IPC layout tests" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Test firmware-selected IPC ring extents, including 64-bit addresses, + overflow, alignment and overlap with the channel table or other rings. + These tests run without starting or accessing the ISP and do not + require a supported Apple device or the ISP driver itself. diff --git a/drivers/media/platform/apple/isp/Makefile b/drivers/media/platform/apple/isp/Makefile index 4649f32987f02..c07e54a522131 100644 --- a/drivers/media/platform/apple/isp/Makefile +++ b/drivers/media/platform/apple/isp/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only apple-isp-y := isp-cam.o isp-cmd.o isp-drv.o isp-fw.o isp-iommu.o isp-ipc.o isp-v4l2.o obj-$(CONFIG_VIDEO_APPLE_ISP) += apple-isp.o +obj-$(CONFIG_VIDEO_APPLE_ISP_IPC_LAYOUT_KUNIT_TEST) += isp-ipc-layout-test.o diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index ac93aba409840..bb7725cbd02b0 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -334,6 +334,8 @@ struct apple_isp { wait_queue_head_t wait; dma_addr_t cmd_iova; void *cmd_virt; + dma_addr_t ipc_boot_iova; + size_t ipc_boot_size; unsigned long state; spinlock_t buf_lock; diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 5c3026502e0e5..6d8f4b91416b4 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -2,6 +2,7 @@ /* Copyright 2023 Eileen Yoon */ #include "isp-fw.h" +#include "isp-ipc-layout.h" #include #include @@ -560,6 +561,8 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) } isp->cmd_iova = cmd_iova; + isp->ipc_boot_iova = args_iova; + isp->ipc_boot_size = args_size + 0x40 + cmd_size; isp->cmd_virt = cmd_virt; if (h17) @@ -621,6 +624,11 @@ static void isp_free_channel_info(struct apple_isp *isp) static int isp_fill_channel_info(struct apple_isp *isp) { + struct isp_ipc_extent reserved[ISP_IPC_MAX_CHANNELS + 2]; + struct isp_ipc_extent ipc = { + .iova = isp->ipc_surf->iova, + .size = isp->ipc_surf->size, + }; u64 table_iova = isp_gpio_read32(isp, ISP_GPIO_0) | ((u64)isp_gpio_read32(isp, ISP_GPIO_1)) << 32; void *table_virt = apple_isp_ipc_translate( @@ -632,6 +640,18 @@ static int isp_fill_channel_info(struct apple_isp *isp) dev_err(isp->dev, "Failed to find channel table\n"); return -EIO; } + reserved[0].iova = isp_fw_iova(isp, table_iova); + reserved[0].size = array_size(sizeof(struct isp_chan_desc), + isp->num_ipc_chans); + /* Use the argument/command span already validated by stage 2. */ + reserved[1].iova = isp->ipc_boot_iova; + reserved[1].size = isp->ipc_boot_size; + if (!isp_ipc_extent_contains(ipc, reserved[0]) || + !isp_ipc_extent_contains(ipc, reserved[1]) || + isp_ipc_extents_overlap(reserved[0], reserved[1])) { + isp_err(isp, "invalid ipc channel table or boot storage\n"); + return -EIO; + } isp->ipc_chans = kcalloc(isp->num_ipc_chans, sizeof(struct isp_channel *), GFP_KERNEL); @@ -679,6 +699,13 @@ static int isp_fill_channel_info(struct apple_isp *isp) goto out; } chan->doorbell = BIT(chan->src); + reserved[i + 2].iova = chan->iova; + reserved[i + 2].size = chan->size; + if (!isp_ipc_ring_valid(ipc, reserved[i + 2], reserved, i + 2)) { + isp_err(isp, "invalid ipc chan %s layout\n", + chan->name); + goto out; + } chan->virt = apple_isp_ipc_translate(isp, chan->iova, chan->size); diff --git a/drivers/media/platform/apple/isp/isp-ipc-layout-test.c b/drivers/media/platform/apple/isp/isp-ipc-layout-test.c new file mode 100644 index 0000000000000..8ca5d5b580d23 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-ipc-layout-test.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include + +#include "isp-ipc-layout.h" + +static void isp_ipc_high_address_test(struct kunit *test) +{ + struct isp_ipc_extent ipc = { 0x10002000000ULL, 0x1c000 }; + struct isp_ipc_extent table = { ipc.iova + 0x1000, 0x700 }; + struct isp_ipc_extent ring = { ipc.iova + 0x4000, 0x400 }; + + KUNIT_EXPECT_TRUE(test, isp_ipc_ring_valid(ipc, ring, &table, 1)); + /* Truncating the ring to 32 bits must not identify owned memory. */ + ring.iova = (u32)ring.iova; + KUNIT_EXPECT_FALSE(test, isp_ipc_ring_valid(ipc, ring, &table, 1)); +} + +static void isp_ipc_extent_test(struct kunit *test) +{ + struct isp_ipc_extent ipc = { 0x10000, 0x4000 }; + struct isp_ipc_extent ring = { 0x13fc0, 0x40 }; + + KUNIT_EXPECT_TRUE(test, isp_ipc_ring_valid(ipc, ring, NULL, 0)); + ring.size += 0x40; + KUNIT_EXPECT_FALSE(test, isp_ipc_ring_valid(ipc, ring, NULL, 0)); + ring = (struct isp_ipc_extent){ 0xffc0, 0x40 }; + KUNIT_EXPECT_FALSE(test, isp_ipc_ring_valid(ipc, ring, NULL, 0)); + ring = (struct isp_ipc_extent){ 0x10000, 0 }; + KUNIT_EXPECT_FALSE(test, isp_ipc_ring_valid(ipc, ring, NULL, 0)); +} + +static void isp_ipc_alignment_test(struct kunit *test) +{ + struct isp_ipc_extent ipc = { 0x10000, 0x4000 }; + struct isp_ipc_extent ring = { 0x10001, 0x40 }; + + KUNIT_EXPECT_FALSE(test, isp_ipc_ring_valid(ipc, ring, NULL, 0)); + ring = (struct isp_ipc_extent){ 0x10000, 0x41 }; + KUNIT_EXPECT_FALSE(test, isp_ipc_ring_valid(ipc, ring, NULL, 0)); + /* Only the u64 publication alignment is required for older firmware. */ + ring = (struct isp_ipc_extent){ 0x10008, 0x40 }; + KUNIT_EXPECT_TRUE(test, isp_ipc_ring_valid(ipc, ring, NULL, 0)); +} + +static void isp_ipc_reserved_test(struct kunit *test) +{ + struct isp_ipc_extent ipc = { 0x10000, 0x4000 }; + struct isp_ipc_extent reserved[] = { { 0x10100, 0x700 }, + { 0x11000, 0x400 } }; + struct isp_ipc_extent ring = { 0x100c0, 0x80 }; + + KUNIT_EXPECT_FALSE(test, isp_ipc_ring_valid(ipc, ring, reserved, 2)); + ring = (struct isp_ipc_extent){ 0x10800, 0x40 }; + KUNIT_EXPECT_TRUE(test, isp_ipc_ring_valid(ipc, ring, reserved, 2)); + ring = (struct isp_ipc_extent){ 0x10fc0, 0x40 }; + KUNIT_EXPECT_TRUE(test, isp_ipc_ring_valid(ipc, ring, reserved, 2)); + ring = reserved[1]; + KUNIT_EXPECT_FALSE(test, isp_ipc_ring_valid(ipc, ring, reserved, 2)); + ring = (struct isp_ipc_extent){ 0x11100, 0x40 }; + KUNIT_EXPECT_FALSE(test, isp_ipc_ring_valid(ipc, ring, reserved, 2)); + ring = (struct isp_ipc_extent){ 0x10fc0, 0x80 }; + KUNIT_EXPECT_FALSE(test, isp_ipc_ring_valid(ipc, ring, reserved, 2)); +} + +static void isp_ipc_overflow_test(struct kunit *test) +{ + struct isp_ipc_extent ipc = { (u64)-1 - 0x3fff, 0x4000 }; + struct isp_ipc_extent ring = { ipc.iova, 0x40 }; + + KUNIT_EXPECT_FALSE(test, isp_ipc_ring_valid(ipc, ring, NULL, 0)); + ipc = (struct isp_ipc_extent){ 0, (u64)-1 }; + ring = (struct isp_ipc_extent){ (u64)-1 - 7, 0x40 }; + KUNIT_EXPECT_FALSE(test, isp_ipc_ring_valid(ipc, ring, NULL, 0)); +} + +static void isp_ipc_captured_layout_test(struct kunit *test) +{ + /* J616s 25G76 channel rings occupy addresses above 1 TB. */ + struct isp_ipc_extent ipc = { 0x10001ff0000ULL, 0x1c000 }; + struct isp_ipc_extent reserved[9] = { + { 0x10001ff0000ULL, 0x700 }, + { 0x10001ffef80ULL, 0x290 + 0x40 + 0x400 }, + { 0x10001ff0700ULL, 768 * 64 }, + { 0x10001ffc700ULL, 8 * 64 }, + { 0x10001ffc900ULL, 8 * 64 }, + { 0x10001ffcb00ULL, 64 * 64 }, + { 0x10001ffdb00ULL, 64 * 64 }, + { 0x10001ffeb00ULL, 8 * 64 }, + { 0x10001ffed00ULL, 8 * 64 }, + }; + + for (unsigned int i = 2; i < ARRAY_SIZE(reserved); i++) + KUNIT_EXPECT_TRUE(test, + isp_ipc_ring_valid(ipc, reserved[i], reserved, i)); + + /* A ring over the command storage must not be admitted. */ + reserved[8] = (struct isp_ipc_extent){ 0x10001fff280ULL, 0x40 }; + KUNIT_EXPECT_FALSE(test, + isp_ipc_ring_valid(ipc, reserved[8], reserved, 8)); +} + +static struct kunit_case isp_ipc_layout_cases[] = { + KUNIT_CASE(isp_ipc_high_address_test), + KUNIT_CASE(isp_ipc_extent_test), + KUNIT_CASE(isp_ipc_alignment_test), + KUNIT_CASE(isp_ipc_reserved_test), + KUNIT_CASE(isp_ipc_overflow_test), + KUNIT_CASE(isp_ipc_captured_layout_test), + {} +}; + +static struct kunit_suite isp_ipc_layout_suite = { + .name = "apple-isp-ipc-layout", + .test_cases = isp_ipc_layout_cases, +}; +kunit_test_suite(isp_ipc_layout_suite); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Apple ISP IPC layout KUnit tests"); diff --git a/drivers/media/platform/apple/isp/isp-ipc-layout.h b/drivers/media/platform/apple/isp/isp-ipc-layout.h new file mode 100644 index 0000000000000..44f06f94c0cee --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-ipc-layout.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ISP_IPC_LAYOUT_H__ +#define __ISP_IPC_LAYOUT_H__ + +#include + +struct isp_ipc_extent { + u64 iova; + u64 size; +}; + +/* Subtraction keeps firmware-selected extents from wrapping at U64_MAX. */ +static inline bool isp_ipc_extent_contains(struct isp_ipc_extent outer, + struct isp_ipc_extent inner) +{ + return outer.size <= (u64)-1 - outer.iova && + inner.size && inner.iova >= outer.iova && + inner.size <= outer.size && + inner.iova - outer.iova <= outer.size - inner.size; +} + +static inline bool isp_ipc_extents_overlap(struct isp_ipc_extent a, + struct isp_ipc_extent b) +{ + if (!a.size || !b.size) + return false; + if (a.iova <= b.iova) + return b.iova - a.iova < a.size; + return a.iova - b.iova < b.size; +} + +static inline bool isp_ipc_ring_valid(struct isp_ipc_extent ipc, + struct isp_ipc_extent ring, + const struct isp_ipc_extent *reserved, + unsigned int count) +{ + /* Messages are 64 bytes and arg0 is published with a u64 store. */ + if ((ring.iova & 7) || (ring.size & 63) || + !isp_ipc_extent_contains(ipc, ring)) + return false; + + for (unsigned int i = 0; i < count; i++) + if (isp_ipc_extents_overlap(ring, reserved[i])) + return false; + + return true; +} + +#endif /* __ISP_IPC_LAYOUT_H__ */ From 8493abe030220e8afd6def31f0c8f236e7003ecb Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:11:00 -0700 Subject: [PATCH 1716/2680] HID: magicmouse: document the host-driven click members Describe the work items, the force click input device and the click state that the host-driven click added to struct magicmouse_sc, so the kernel-doc of the structure is complete again. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 2ab122916d816ba479be7b6840de9480f3c10aee) --- drivers/hid/hid-magicmouse.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index eda706e19aa76..970a10d2b9509 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -272,6 +272,13 @@ struct effect_job { * @haptics: Haptic device state of an MTP trackpad, NULL otherwise. * @taptic_hdev: Actuator HID device of an MTP trackpad, if resolved. * @haptic_effects: Uploaded haptic effects of an MTP trackpad. + * @haptic_press_work: Plays the press waveform of a host-driven click. + * @haptic_release_work: Plays the release waveform, or hands the actuator back. + * @haptic_deep_work: Plays the waveform of a deep click. + * @haptic_deep_release_work: Plays the release waveform on leaving a deep click. + * @force_input: Input device for the deep-click key, NULL if none. + * @haptic_button_down: The host-derived button is pressed. + * @haptic_deep_down: The host-derived click is in its deep stage. * @hdev: Pointer to the underlying HID device. * @work: Workqueue to handle initialization retry for quirky devices. * @battery_timer: Timer for obtaining battery level information. From 17077238628bc5fe59728acdf6b60888e545e13c Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 10 Oct 2026 20:19:29 -0700 Subject: [PATCH 1717/2680] media: apple: avd: align AV1 tails in imported capture planes AV1 places color metadata and CDF state at the end of the capture plane. The negotiated minimum is aligned, but an oversized imported DMA buffer can have an unaligned length. This leaves the CPU seed and reference-copy addresses displaced from the aligned firmware writes. Round the usable plane end down to AVD_ALIGN before placing both tails. The existing aligned minimum reservation keeps them beyond compressed storage; aligned capture planes retain their offsets. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 6d8f5745560a83d7b822b6edceb09373c49d604c) --- drivers/media/platform/apple/avd/avd-av1.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/avd/avd-av1.c b/drivers/media/platform/apple/avd/avd-av1.c index cd8e876b4e10a..113edc267fd33 100644 --- a/drivers/media/platform/apple/avd/avd-av1.c +++ b/drivers/media/platform/apple/avd/avd-av1.c @@ -197,8 +197,9 @@ #define AVD_CDFS_SIZE sizeof(struct avd_av1_cdfs) +/* Imported planes can have an unaligned length; AVD aligns both tail addresses. */ #define AVD_AV1_COLOR_OFFSET(dst, cl) \ - ((dst) - ALIGN(cl, AVD_ALIGN)) + (ALIGN_DOWN((dst), AVD_ALIGN) - ALIGN(cl, AVD_ALIGN)) #define AVD_AV1_CDFS_OFFSET(dst, cl) \ (AVD_AV1_COLOR_OFFSET(dst, cl) - ALIGN(AVD_CDFS_SIZE, AVD_ALIGN)) From 124ab5b789b76b0778a18b8a489970d21779f34d Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 12 Oct 2026 12:45:13 +0200 Subject: [PATCH 1718/2680] media: apple: avd: allow more than 32 capture buffers per context The capture queue never sets max_num_buffers, so vb2 limits every context to VB2_MAX_FRAME (32) buffers. Nothing in the driver needs that limit: reference frames are looked up by timestamp (vb2_find_buffer()) and handed to the hardware by DMA address, never by buffer index. 32 buffers leave too little room for userspace to decode ahead. With a 16-frame DPB, four frame threads and the frames held for display, mpv's decoder queue (--vd-queue) gets only a few frames before VIDIOC_CREATE_BUFS fails with ENOBUFS, and the FFmpeg V4L2 request hwaccel then falls back to software decoding. Decoding ahead is needed because the t8103 AVD decodes 4K59.94 H.264 at only ~41 fps during a 125-156 Mbit/s burst, so playback drops frames unless the decoder has built up a lead of 60-90 frames beforehand. Add a max_capture_buffers module parameter, default 128. Buffers are still allocated on demand by userspace, so this only raises the limit. Signed-off-by: Thomas Glanzmann (cherry picked from commit 2f645406d54107b4de63ba68fe48e3ea6e55e88e) --- drivers/media/platform/apple/avd/avd-drv.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 9751b8d292ffe..b5df2a58b0823 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -431,6 +431,17 @@ static void avd_device_run(void *priv) avd_job_finish(ctx, result); } +/* + * Capture buffers are found by timestamp and handed to the hardware by DMA + * address, so nothing limits them to VB2_MAX_FRAME. A larger pool lets + * userspace decode ahead to ride out bitrate spikes the hardware can't + * decode in real time (e.g. 150+ Mbit/s 4K60 H.264). + */ +static unsigned int max_capture_buffers = 128; +module_param(max_capture_buffers, uint, 0444); +MODULE_PARM_DESC(max_capture_buffers, + "Maximum number of capture buffers per context (default 128)"); + static int avd_queue_init(void *priv, struct vb2_queue *src_vq, struct vb2_queue *dst_vq) { @@ -468,6 +479,7 @@ static int avd_queue_init(void *priv, struct vb2_queue *src_vq, dst_vq->lock = &ctx->dev->vdev_lock; dst_vq->dev = ctx->dev->v4l2_dev.dev; dst_vq->allow_cache_hints = true; + dst_vq->max_num_buffers = max(max_capture_buffers, VB2_MAX_FRAME); return vb2_queue_init(dst_vq); } From 1225455aa595173ceba9cbdd44d90154b1b48c91 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 12:10:49 -0700 Subject: [PATCH 1719/2680] soc: apple: sep: pick the M2 Pro/Max key store by system firmware MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The T6020 profile always uses the variant-5 keybag create. That works on J414c (M2 Max) on system firmware 26.6.2 and on J416s on 27.0. On older firmware the enclave refuses it with mailbox status -13 and an empty reply, so the identity keybag is never created and Touch ID cannot be set up: J414s on 26.0, J416s on 26.2. On J414s at 26.0 the 13.5 key store that T8103 uses (create 2/0x400000/-1, cpx-encryption-mode 2) works end to end: enrolment, verify, reference-key signing, and restore after a reboot. Choose at probe time from the machine's iBoot build, which m1n1 copies into /chosen/asahi,iboot1-version: a build before 26.3's (iBoot-13822.80) gets a T6020 profile with the 13.5 key store, and everything else keeps the current one. That covers 26.0 to 26.2, where variant 5 is known to fail, and earlier firmware, which predates it. 26.3 to 26.5 are untested and keep variant 5, as does anything that is not an iBoot- build (26.4 onwards report mBoot-) or a missing property, so a machine that works today keeps the encoding it works with. If 26.3 to 26.5 turn out to refuse variant 5 too, the boundary moves up. The raw iBoot build is used rather than asahi,system-fw-version, because that is a lookup in m1n1's own table: m1n1 releases before 2026-09-30 stop at 26.4 and report later firmware as "unknown", so the key store would depend on which m1n1 is installed. The log line shows both, and the OS firmware version. Selecting at probe time, rather than retrying the other encoding after a -13, means the enclave never gets a create it refuses. A keybag made by one key store will not load under the other, so a macOS update from 26.2 or earlier to 26.3 or later leaves Touch ID needing a new identity keybag and new enrolments. When recovery fails that way, say so: only the 13.5 key store records a generated UUID. Also say so in the other direction, a bag with a read-back UUID on an M2 Pro/Max now given the 13.5 key store, since that would mean a board that worked on variant 5 was moved off it. The selected profile is named "T6020/J414s (13.5 key store)" so that the probe log and diag/profile show it, and comments that used "T6020" to mean the variant-5 key store now say so. [ryan: adapted to 7.2.10 — resolve profile and codec comment context; retain authenticated-sensor profile fields and current persistence.] Reported-by: Justin Pfister Suggested-by: Justin Pfister Link: https://github.com/aurora-silicon/linux/pull/69#issuecomment-5970016911 Link: https://github.com/omacom/linux/pull/7#issuecomment-5946814550 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 682f310497a05380b5fecb06b0c4d9eb145a07f2) --- drivers/soc/apple/profile.rs | 120 +++++++++++++++++++++++++++++++---- drivers/soc/apple/refkey.rs | 2 +- drivers/soc/apple/sbio.rs | 22 +++++++ drivers/soc/apple/sks.rs | 25 ++++---- 4 files changed, 142 insertions(+), 27 deletions(-) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 85ed9c4b03cec..cc5e1fcd819e7 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -16,7 +16,8 @@ //! * `T6000` / J316s (MacBook Pro 16", M1 Pro): the T8103 boot handshake and //! key store, with the sensor on the T6020 SPI2 address. //! * `T6020` / J414s (MacBook Pro 14", M2 Pro): the driver does the warm -//! single-message registration over a 0x40000 window. +//! single-message registration over a 0x40000 window. Before system +//! firmware 26.3 the enclave speaks the T8103 key store. //! * `T8140` / J700 (MacBook Neo, A18 Pro): iBoot boots sepOS before the AP //! OS, so the driver uses the warm registration path. //! * `T8112` / J413, J415 (MacBook Air, M2): iBoot hands over a running SEP and @@ -27,6 +28,7 @@ #![allow(dead_code)] use kernel::of; +use kernel::pr_info_once; use kernel::prelude::*; /// How the driver brings the shared-memory table to the SEP. @@ -95,23 +97,24 @@ pub(crate) struct SensorProfile { /// Identity keybag `CREATE_KEYBAG` field encoding. The first word is the /// codec's struct version: the 13.5 enclave (T8103) takes versions 0-2 only -/// and macOS creates an identity with version 2, while the T6020 enclave takes -/// variant 5. Kept per-SoC so the T8103 path is correct without disturbing the -/// proven T6020 encoding (hardware-verified for enrol, match, and reboot). +/// and an identity is created with version 2, while the T6020 enclave on +/// newer system firmware takes variant 5. Kept per profile so the T8103 path +/// is correct without disturbing the proven T6020 encoding (hardware-verified +/// for enrol, match, and reboot). pub(crate) struct KeybagCreate { /// First word: struct version (request variant), echoed back in the reply. pub(crate) variant: u32, - /// Third word: bag type. Identity is `0x400000` on 13.5; `0` on T6020, - /// which distinguishes the bag by the variant word instead. + /// Third word: bag type. Identity is `0x400000` on 13.5; `0` on variant + /// 5, which distinguishes the bag by the variant word instead. pub(crate) bag_type: u32, /// Fourth word: create argument / parent handle. pub(crate) arg: i32, } /// Which key-store protocol the enclave speaks. It follows the sepOS the -/// firmware hands the SEP, not the SoC alone: the T8103 stub boots macOS 13.5 -/// (22G74)'s sepOS, whose request shapes are those of the 13.5 -/// key store; the T6020 sepOS is newer. +/// firmware hands the SEP, not the SoC alone: the T8103 stub boots the 13.5 +/// (22G74) sepOS, whose request shapes are the 13.5 ones; the T6020 sepOS +/// speaks them too before system firmware 26.3 (see [`t6020_profile`]). #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum KeyStore { /// macOS 13.5's key store. Before any other request the endpoint is @@ -251,6 +254,24 @@ const T6020: PlatformProfile = PlatformProfile { nonce_session: false, }; +/// M2 Pro/Max on system firmware before 26.3. Their enclave refuses the +/// variant-5 create (status -13, empty reply: J414s on 26.0, J416s on 26.2) +/// and takes the 13.5 key store, as T8103 does; J414s on 26.0 enrols, +/// matches, signs with the reference key and restores after a reboot this way. +/// Everything else is [`T6020`]. +const T6020_SEPOS13: PlatformProfile = PlatformProfile { + name: "T6020/J414s (13.5 key store)", + keybag_create: KeybagCreate { + variant: T8103.keybag_create.variant, + bag_type: T8103.keybag_create.bag_type, + arg: T8103.keybag_create.arg, + }, + key_store: KeyStore::Sepos13 { + cpx_encryption_mode: 2, + }, + ..T6020 +}; + /// MacBook Neo. The J700 ADT records a pre-booted SEP, a spi2 Mesa sensor /// (0x3356), and a SEP DART aperture above 4 GiB. These values describe that /// board; device-tree nodes remain authoritative for addresses and resources. @@ -294,7 +315,7 @@ const T8140: PlatformProfile = PlatformProfile { /// carries `cpx-encryption-mode = 2`. /// /// The shared-memory geometry and key-store dialect follow the other -/// warm-registration platforms (T6020, T8140). +/// warm-registration platforms (T6020 on current firmware, T8140). const T8112: PlatformProfile = PlatformProfile { name: "T8112/J415", shmem_capacity: 0x4_0000, @@ -316,7 +337,7 @@ const T8112: PlatformProfile = PlatformProfile { firmware_region: c"sepfw", // J415 accepted the 13.5 GET_CAPABILITIES/SET_ENV init but refused the // strict CREATE_KEYBAG (mailbox status -1, empty reply), as J700 did; - // use the T6020 form like the other warm-registration platforms. + // use the variant-5 form like the other warm-registration platforms. keybag_create: KeybagCreate { variant: 5, bag_type: 0, @@ -367,6 +388,79 @@ pub(crate) fn machine_has(compatible: &[u8]) -> bool { false } +/// iBoot builds before 26.3's (betas 13822.80, release 13822.81) get the 13.5 +/// key store. The 26.0 and 26.2 enclaves refuse the variant-5 create, while +/// 26.6.2 and 27.0 take it; 26.3 to 26.5 are untested and keep variant 5, as +/// before. Firmware from 26.4 on has an `mBoot-` build and never gets here. +const T6020_SEPOS13_BEFORE: (u32, u32) = (13822, 80); + +/// The leading `major.minor` of an `iBoot-` build string, as m1n1 copies the +/// machine's own iBoot into `asahi,iboot1-version` ("iBoot-13822.61.10"). +/// Anything else, including an `mBoot-` build, yields `None`. Unlike +/// `asahi,system-fw-version`, this does not depend on m1n1's version table, +/// so every m1n1 gives the same answer. +fn parse_iboot_build(raw: &[u8]) -> Option<(u32, u32)> { + fn number(text: &[u8]) -> Option<(u32, &[u8])> { + let digits = text.iter().take_while(|byte| byte.is_ascii_digit()).count(); + if digits == 0 || digits > 6 { + return None; + } + let value = text[..digits] + .iter() + .fold(0u32, |value, &byte| value * 10 + u32::from(byte - b'0')); + Some((value, &text[digits..])) + } + + let text = match raw.iter().position(|&byte| byte == 0) { + Some(end) => &raw[..end], + None => raw, + }; + let (major, rest) = number(text.strip_prefix(b"iBoot-")?)?; + let (minor, rest) = number(rest.strip_prefix(b".")?)?; + match rest.first() { + None | Some(b'.') => Some((major, minor)), + Some(_) => None, + } +} + +fn chosen_string(name: &CStr) -> Option> { + of::chosen()?.get_property::>(name).ok() +} + +/// A `/chosen` string for the log, without its NUL. +fn shown(raw: &Option>) -> &kernel::str::BStr { + let raw = raw.as_deref().unwrap_or(&b"none"[..]); + let end = raw.iter().position(|&byte| byte == 0).unwrap_or(raw.len()); + kernel::str::BStr::from_bytes(&raw[..end]) +} + +/// The M2 Pro/Max profile for the machine's system firmware. Only an iBoot +/// build before 26.3's selects the 13.5 key store; anything else, including +/// a missing property, keeps [`T6020`], so a machine that works today keeps +/// the encoding it works with. +fn t6020_profile() -> &'static PlatformProfile { + let iboot = chosen_string(c"asahi,iboot1-version"); + let firmware = chosen_string(c"asahi,system-fw-version"); + let os_firmware = chosen_string(c"asahi,os-fw-version"); + let sepos13 = iboot + .as_deref() + .and_then(parse_iboot_build) + .is_some_and(|build| build < T6020_SEPOS13_BEFORE); + let profile = if sepos13 { &T6020_SEPOS13 } else { &T6020 }; + pr_info_once!( + "M2 Pro/Max on system firmware {} ({}), OS firmware {}: using {}\n", + shown(&firmware), + shown(&iboot), + shown(&os_firmware), + if sepos13 { + "the 13.5 key store" + } else { + "the variant-5 key store" + } + ); + profile +} + /// Select the profile for the running machine. An unsupported SoC is refused /// rather than guessed at, so the driver cannot run a handshake with the wrong /// geometry. @@ -375,7 +469,7 @@ pub(crate) fn detect() -> Result<&'static PlatformProfile> { return Ok(&T8103); } if machine_has(b"apple,t6020") { - return Ok(&T6020); + return Ok(t6020_profile()); } // t6020.dtsi is defined as a cut-down t6021: it includes t6021.dtsi and // disables the parts the smaller die lacks. Both pull in t602x-die0.dtsi, @@ -383,7 +477,7 @@ pub(crate) fn detect() -> Result<&'static PlatformProfile> { // with the same interrupts on either part, and the T6020 constants apply // to the M2 Max unchanged. if machine_has(b"apple,t6021") { - return Ok(&T6020); + return Ok(t6020_profile()); } if machine_has(b"apple,t6000") { return Ok(&T6000); diff --git a/drivers/soc/apple/refkey.rs b/drivers/soc/apple/refkey.rs index d8e6a5c9a67c8..c408968d0540f 100644 --- a/drivers/soc/apple/refkey.rs +++ b/drivers/soc/apple/refkey.rs @@ -99,7 +99,7 @@ impl SepData { }; if self.profile.key_store == profile::KeyStore::Variant5 { - // T6020's ref-key is bound to the unlocked identity bag. + // On variant 5 the ref-key is bound to the unlocked identity bag. if let Some(healthy) = self.sks_health_check(c"ref-key create") { let _ = self.sks_send(self.sks_req_change_lock_state( handle, diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index be2563c050cfd..6a4a47fdd52a2 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1301,6 +1301,28 @@ impl SepData { self.dev, "Touch ID: the persisted identity keybag did not recover\n" ); + // Only the 13.5 key store records the UUID it generated. An M2 + // Pro/Max uses it before system firmware 26.3 and variant 5 after. + if self.profile.key_store == profile::KeyStore::Variant5 + && stored.uuid_provenance() == crate::keybag::UuidProvenance::AsGenerated + { + dev_err!( + self.dev, + "Touch ID: the keybag was made by the 13.5 key store, but this enclave speaks variant 5; if macOS was updated past 26.2, Touch ID needs a new identity keybag and new enrolments\n" + ); + } + // Only the variant-5 key store records the UUID read back from + // the bag on a warm-registration Mac. Firmware that selects the + // 13.5 key store for a bag variant 5 made is worth a report. + if matches!(self.profile.key_store, profile::KeyStore::Sepos13 { .. }) + && self.profile.bootstrap == profile::Bootstrap::WarmRegister + && stored.uuid_provenance() == crate::keybag::UuidProvenance::ReadBackFromBag + { + dev_err!( + self.dev, + "Touch ID: the keybag was made by the variant-5 key store, but this firmware selects the 13.5 one; please report this with the \"M2 Pro/Max on system firmware\" line\n" + ); + } diag::keybag(diag::Keybag::Failed); EIO })?; diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index d7a203f9e0b15..fd5286795dffb 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -331,11 +331,11 @@ impl SepData { self.sks_seal(&op, &body) } - /// Endpoint init for the 13.5 key store, which must precede any other - /// request: `0x4d`, whose capability word sets the IPC header version to - /// `min(word, 2)`, then `set_env`. - /// Returns false if the key store must stay closed this boot. - /// The T6020 path sends nothing here. + /// Endpoint init on the 13.5 key store, before any other request: `0x4d`, + /// whose capability word sets the IPC header version to `min(word, 2)`, + /// then `set_env`. Returns false if the key store must stay closed this + /// boot. + /// The variant-5 key store sends nothing here. pub(crate) fn sks_init_endpoint(&self) -> bool { let profile::KeyStore::Sepos13 { cpx_encryption_mode, @@ -579,12 +579,11 @@ impl SepData { if proof.slot() != keybag::Slot::Identity { return Err(EINVAL); } - // The 13.5 (T8103) and T6020 enclaves encode the identity bag - // differently; the per-SoC profile carries the field values (see - // profile::KeybagCreate). On 13.5 the identity-bag create carries: - // secret, an empty second - // blob and the 16-byte identity UUID, nothing after it, so the body - // has exactly this length. + // The 13.5 and variant-5 key stores encode the identity bag + // differently; the profile carries the field values (see + // profile::KeybagCreate). On 13.5 the request is: secret, an empty + // second blob and the 16-byte identity UUID, nothing after it, decoded + // as an exact-length message. let enc = &self.profile.keybag_create; let mut body = image::Body::new(); body.put_u32(enc.variant)?; @@ -689,8 +688,8 @@ impl SepData { self.sks_mark_create_refused(slot, out.reply.status); return false; }; - // 13.5 replies with the struct version and the handle alone; - // the T6020 reply adds a blob. + // 13.5 replies with the struct version and the handle alone; the + // variant-5 reply adds a blob. let reply_min = match self.profile.key_store { profile::KeyStore::Sepos13 { .. } => 8, profile::KeyStore::Variant5 => 12, From ef7e5d948c44ea5bad8bcfda530bef648c667794 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Tue, 6 Oct 2026 07:58:23 -0500 Subject: [PATCH 1720/2680] accel/ane: t6021: default call_settle_us to 0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A CALL already waits for the firmware's state-1 IO_T2H finish event. It then slept call_settle_us (1,000 us), a margin kept from an older wait that returned at dispatch. The finish event already marks the output in DRAM: on an M2 Max with the sleep at 0, 6000 add calls with new inputs per call each read their own result right after the ioctl, and program 20 and the Parakeet encoder stayed bit-identical. The sleep was the whole fixed per-call cost: H14 add went from 1.478 to about 0.31 ms per call. Default the parameter to 0 and keep it runtime-writable for bisection. A completion timeout now logs at error level and names the timeout. [ryan: adapted to 7.2.10 — Retain the finish-event wait; describe output visibility without test-system history.] Link: https://github.com/iconidentify/aurora-linux/issues/9 Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit da5b09512eebf02fecaa6b7aecb9ca3cb421ef08) --- drivers/accel/ane/ane_t6021_rtclient_main.c | 35 ++++++++++----------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index ceb0875e3372e..664100049081d 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -589,17 +589,16 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, return result; } -/* Time to let the output writes land after the completion signal. - * On some boots the fw ack, the TD counter and the TQ words all report - * done ~0.13 ms before the output reaches DRAM (the output then reads - * as zeros). The finish event arrives at the same time for short - * programs. No signal for "output landed" is known, so the wait is a - * fixed margin of about 8x the lag. +/* + * Extra sleep after a CALL's finish event. 0 by default: the finish event + * already marks the output in DRAM (ane_rtclient_call_wait). The 1 ms + * default it replaces dated from a wait on dispatch signals; set it only + * to test a suspected late output write. */ -static unsigned int call_settle_us = 1000; +static unsigned int call_settle_us; module_param(call_settle_us, uint, 0644); MODULE_PARM_DESC(call_settle_us, - "Microseconds to wait after a CALL completes so its output lands (default 1000, 0 = none)"); + "Microseconds to sleep after a CALL's finish event (default 0: the event marks the output in DRAM)"); /* trace_td: a read-only timeline of each CALL for performance work. Off * by default; switch it at runtime with @@ -862,15 +861,12 @@ static int ane_rtclient_call_wait_traced(struct ane_rtclient *ane, return ret; } -/* Completion wait for one PROCEDURE_CALL: the finish event on IO_T2H - * (channel 6). The ack, the eight TQ status words and the last-committed - * TD word (TM +0x20458: the call's nid in bits 23:16 and the index of the - * last task taken in bits 15:0) all report done once the last task has - * been dispatched, not when it has run: a 20-task program can show its - * last task index 0.22 ms after the ack but post its finish event 3.5 ms - * after it, and a caller that returns at the first signal reads an - * all-zero output. Returns 0 when the event - * arrived, -ETIMEDOUT else. +/* + * Wait for the PROCEDURE_CALL finish event on IO_T2H channel 6. + * Acknowledgment, TQ status and the last-committed TD word indicate + * dispatch rather than execution completion. The finish event marks + * completion and output visibility, so no extra settle delay is needed. + * Return zero on the event, or -ETIMEDOUT. */ static int ane_rtclient_call_wait(struct ane_rtclient *ane, unsigned int timeout_ms) @@ -955,8 +951,9 @@ static int ane_rtclient_command(struct ane_rtclient *ane, stats_ticket, ktime_get_ns(), (u32)ret, 0); - dev_info(ane->dev, "call completion wait failed %d\n", - ret); + dev_err(ane->dev, + "call completion wait failed %d: no finish event in %u ms\n", + ret, timeout_ms); atomic_set(&ane_t6021_quarantined, 1); return ret; } From 2bfed830dd4dfce490c872613b4a82f124403ad6 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Thu, 8 Oct 2026 03:21:41 -0700 Subject: [PATCH 1721/2680] accel/ane: drain polling work when probe fails The RX fallback can remain queued after CONFIG_GET or DRM admission fails. Cancel and drain it before returning failure so driver-core resource cleanup cannot leave a worker using released client data. Publish the performance client only after DRM registration succeeds. Driver-core unbind cleanup already clears driver data, so retain that existing metadata cleanup rather than duplicating it. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 95bf7da7aabef7af21679bf398c5317864d073bc) --- drivers/accel/ane/ane_t6021_rtclient_main.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index 664100049081d..1d1e5afc151e1 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -2430,13 +2430,13 @@ static int ane_rtclient_probe_inner(struct platform_device *pdev, drm_debugfs_add_file(&adrm->drm, "ane_timeline", ane_timeline_show, &ane->stats_ring); - WRITE_ONCE(ane_t6021_perf_ane, ane); drmret = drm_dev_register(&adrm->drm, 0); if (drmret) { dev_err_probe(dev, drmret, "drm_dev_register\n"); ret = drmret; goto err_pm_or_hold; } + WRITE_ONCE(ane_t6021_perf_ane, ane); dev_info(dev, "loaded ane_t6021 (DRM major %d minor %d; ABI 2; legacy_only=%u chman_ok=%u booted=%u; state %s; BO cap %u MiB)\n", DRM_ANE_ABI_V2, 0, @@ -2476,6 +2476,8 @@ static int ane_rtclient_probe(struct platform_device *pdev) if (ret) { struct ane_rtclient *ane = platform_get_drvdata(pdev); + if (ane) + cancel_delayed_work_sync(&ane->poll_work); if (hardware_touched) { ane_rtclient_probe_failed_dirty = true; dev_err(&pdev->dev, From 574c1c576c3d3636f0bf5508bde0d66d40962c07 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:19:20 -0500 Subject: [PATCH 1722/2680] accel/ane: t6021: poll the CALL finish event at 1 us for the first call_poll_us MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ane_rtclient_call_wait() detects the IO_T2H finish event with a uniform 50..100 us sleep. The ring is coherent memory, so the poll cadence is the whole detection lag: every CALL waits 0..100 us longer than the event. Poll the head of the wait at a 1 us cadence for call_poll_us microseconds (default 1000), then fall back to the 50..100 us sleep, so long programs (the Parakeet encoder, about 254 ms) do not spend the wait on hrtimer wakeups. call_poll_us=0 restores the old cadence. The parameter is writable at run time (0644), so both cadences can be compared in one boot. The same change is in omarchy-ane (commit e0c0933, 2026-10-06), where it was measured on an M2 Max on aurora 11.38, one boot, arms switched through sysfs and interleaved (receipts/2026-10-06-t6021-call-poll): add median 0.359 ms to 0.257 ms, 2932.5 to 3348.0 jobs/s (+14.1 percent), 1000-call landing check with 0 mismatched, 0 stale and 0 all-zero results, outputs bit-exact, Parakeet encoder bit-exact. A 300 us busy-spin head added 0.4 percent for about one core and is not included. On the current kernel family, the in-tree module shows the old cadence: its sysfs has no call_poll_us, and the add exec distribution steps in about 51 us (0.205, 0.256, 0.306, 0.357 ms; M2 Max, 7.1.12-2-20261013.18, 20000 calls). That is an observation, not a measurement of this patch. This tree has not run this patch. It builds: drivers/accel/ane/ with the arm64 LLVM and rustc stack of the lab's release builds compiles, and ane_t6021.o carries the new parameter. checkpatch --strict is clean. [ryan: adapted to 7.2.10 — Preserve the current TD timeline documentation and power-state guard while adding the bounded short- sleep head.] Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> (cherry picked from commit d1c7d3f453a46d488df3987df07d10d02d1998e0) --- drivers/accel/ane/ane_t6021_rtclient_main.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index 1d1e5afc151e1..513e34d41c19b 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -600,6 +600,20 @@ module_param(call_settle_us, uint, 0644); MODULE_PARM_DESC(call_settle_us, "Microseconds to sleep after a CALL's finish event (default 0: the event marks the output in DRAM)"); +/* + * The finish-event wait polls the coherent IO_T2H ring, so the poll cadence is + * the whole detection lag: a uniform 50..100 us sleep adds 0..100 us of pure + * wait to every CALL. call_poll_us is how long the head of the wait polls at a + * 1 us cadence before it falls back to the 50..100 us sleep, which keeps long + * programs (the Parakeet encoder, about 254 ms) from spending the wait on + * hrtimer wakeups. 0 restores the uniform 50..100 us cadence. Writable at run + * time (0644), so both cadences can be compared in one boot. + */ +static unsigned int call_poll_us = 1000; +module_param(call_poll_us, uint, 0644); +MODULE_PARM_DESC(call_poll_us, + "Microseconds of 1 us polling at the head of the CALL finish-event wait, then 50..100 us (default 1000; 0 = uniform 50..100 us)"); + /* trace_td: a read-only timeline of each CALL for performance work. Off * by default; switch it at runtime with * /sys/module/ane_t6021/parameters/trace_td (no device needed). Off, the @@ -872,6 +886,8 @@ static int ane_rtclient_call_wait(struct ane_rtclient *ane, unsigned int timeout_ms) { unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms); + ktime_t tight_end = ktime_add_ns(ktime_get(), + (u64)call_poll_us * NSEC_PER_USEC); if (ane_t6021_tracing) return ane_rtclient_call_wait_traced(ane, deadline); @@ -879,7 +895,10 @@ static int ane_rtclient_call_wait(struct ane_rtclient *ane, do { if (ane_rtclient_drain_t2h(ane, 6)) return 0; - usleep_range(50, 100); + if (call_poll_us && ktime_before(ktime_get(), tight_end)) + usleep_range(1, 2); + else + usleep_range(50, 100); } while (time_before(jiffies, deadline)); return ane_rtclient_drain_t2h(ane, 6) ? 0 : -ETIMEDOUT; } From b0dabef3b6fff453a58308aa794e19dad35f3622 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:30:41 -0500 Subject: [PATCH 1723/2680] accel/ane: t6021: describe call_poll_us as short sleeps and read it once per CALL The head of the wait sleeps 1..2 us between looks with usleep_range(); the real period is the hrtimer wake latency, not a 1 us poll. Say so in the comment and in the parameter description, and note that a very large value makes the whole wait 1..2 us sleeps. The parameter is writable at run time and was read twice per CALL. Read it once into a local so the deadline and the loop test use the same value. No change to the default behaviour. drivers/accel/ane/ builds with the arm64 LLVM and rustc stack of the lab's release builds; checkpatch --strict is clean. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> (cherry picked from commit a9968b1cf9963e7e6c73dbbb5715e4a47fcf2e59) --- drivers/accel/ane/ane_t6021_rtclient_main.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index 513e34d41c19b..40e643f58a8ca 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -603,16 +603,18 @@ MODULE_PARM_DESC(call_settle_us, /* * The finish-event wait polls the coherent IO_T2H ring, so the poll cadence is * the whole detection lag: a uniform 50..100 us sleep adds 0..100 us of pure - * wait to every CALL. call_poll_us is how long the head of the wait polls at a - * 1 us cadence before it falls back to the 50..100 us sleep, which keeps long - * programs (the Parakeet encoder, about 254 ms) from spending the wait on - * hrtimer wakeups. 0 restores the uniform 50..100 us cadence. Writable at run - * time (0644), so both cadences can be compared in one boot. + * wait to every CALL. call_poll_us is how long the head of the wait sleeps + * only 1..2 us between looks (the real period is the hrtimer wake latency) + * before it falls back to the 50..100 us sleep, which keeps long programs (the + * Parakeet encoder, about 254 ms) from spending the wait on hrtimer wakeups. + * 0 restores the uniform 50..100 us cadence. A very large value makes the + * whole wait 1..2 us sleeps. Writable at run time (0644), so both cadences can + * be compared in one boot. */ static unsigned int call_poll_us = 1000; module_param(call_poll_us, uint, 0644); MODULE_PARM_DESC(call_poll_us, - "Microseconds of 1 us polling at the head of the CALL finish-event wait, then 50..100 us (default 1000; 0 = uniform 50..100 us)"); + "Microseconds at the head of the CALL finish-event wait that sleep only 1..2 us between looks, then 50..100 us (default 1000; 0 = uniform 50..100 us; a very large value makes the whole wait 1..2 us sleeps)"); /* trace_td: a read-only timeline of each CALL for performance work. Off * by default; switch it at runtime with @@ -886,8 +888,9 @@ static int ane_rtclient_call_wait(struct ane_rtclient *ane, unsigned int timeout_ms) { unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms); + unsigned int poll_us = READ_ONCE(call_poll_us); ktime_t tight_end = ktime_add_ns(ktime_get(), - (u64)call_poll_us * NSEC_PER_USEC); + (u64)poll_us * NSEC_PER_USEC); if (ane_t6021_tracing) return ane_rtclient_call_wait_traced(ane, deadline); @@ -895,7 +898,7 @@ static int ane_rtclient_call_wait(struct ane_rtclient *ane, do { if (ane_rtclient_drain_t2h(ane, 6)) return 0; - if (call_poll_us && ktime_before(ktime_get(), tight_end)) + if (poll_us && ktime_before(ktime_get(), tight_end)) usleep_range(1, 2); else usleep_range(50, 100); From a8035ba34cbe8f53e151638d02ec786bf70fe163 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 12:11:05 -0700 Subject: [PATCH 1724/2680] soc: apple: sep: save the Catacombs when a fingerprint is deleted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DELETE and DELETE_ALL send 0x57, which removes the identity from the enclave's memory, and drop it from the host index, but nothing saves the Catacombs afterwards. The saved user Catacomb still holds the template, so the next attach restores it and the deleted fingerprint matches again. After 0x57 removes at least one identity, save as a match does: only the components the enclave marks save-pending, user before master with the state re-read before each (saving the user can change the master, and a SAVE_CATACOMB of a component that is not pending has been refused), each with its host file present, then the lockout and the identity bag. If 0x57 did not mark the user Catacomb pending, the deletion cannot be made durable and the call fails. Only after the save are the host index entries dropped. If the save fails the call returns EIO and the entries stay: userspace still lists the fingerprint, the next boot restores it consistently, and deleting it again retries the save, which accepts a user Catacomb that the failed attempt already saved. DELETE_ALL saves what it removed even when another deletion failed, and still returns that error. 0x57 cannot be undone, so it is not sent unless the deletion can be saved: - With xART writes off the save would be refused, and the driver already refuses captures for that reason. Return EROFS. - Without the owner Catacomb file or the identity-bag binding the save cannot succeed. Return EIO without touching the enclave. The saves must not interleave with other saves, or the user and master Catacombs on disk could come from different moments. A verify that is cancelled can still reach a verdict and run its post-match saves, so a new lock is held across each enrol or verify work item and across a deletion. It is taken before the work registers with the sleep drain, so time spent waiting for it never counts against a suspend. A deletion then registers like a capture: a suspend waits for its saves, and it is waits while the system is going to sleep. DELETE and DELETE_ALL return EBUSY while an enrolment or verify is live, as starting one does. [ryan: adapted to 7.2.10 — serialize deletion and capture saves before the suspend drain while retaining epoch-bound worker completion and worker_gone notification.] Reported-by: Justin Pfister Suggested-by: Justin Pfister Link: https://github.com/aurora-silicon/linux/pull/69#issuecomment-5970016911 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 42a8f5cfe5283fcb9f0fa2bef47459502987c4da) --- drivers/soc/apple/bio.rs | 6 ++ drivers/soc/apple/sbio.rs | 191 ++++++++++++++++++++++++++++++++++---- drivers/soc/apple/sep.rs | 20 +++- 3 files changed, 196 insertions(+), 21 deletions(-) diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index 9ba7722f5cf4e..0dd9418708d20 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -959,6 +959,9 @@ pub(crate) fn starts_capture(cmd: u32) -> bool { fn delete(ctx: &mut Context<'_>, user: UserPtr) -> Result { require_admin()?; + if !ctx.session.op.may_start() { + return Err(EBUSY); + } let request: Delete = UserSlice::new(user, core::mem::size_of::()) .reader() .read()?; @@ -988,6 +991,9 @@ fn delete(ctx: &mut Context<'_>, user: UserPtr) -> Result { fn delete_all(ctx: &mut Context<'_>) -> Result { require_admin()?; + if !ctx.session.op.may_start() { + return Err(EBUSY); + } if ctx.index.total() == 0 { pr_info!("sep_bio: DELETE_ALL on empty index; reporting success (nothing to delete)\n"); diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 6a4a47fdd52a2..c8040b5b5e332 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2973,10 +2973,11 @@ impl SepData { } } - /// Called by the enrol and verify work items before they touch the sensor. - /// Returns false if the system is going to sleep; the session has then - /// been ended and the capture must not start. A true return must be - /// paired with [`Self::capture_end`]. + /// Called by the enrol and verify work items before they touch the sensor, + /// and by a deletion before it touches the enclave. Returns false if the + /// system is going to sleep; the session has then been ended and the + /// capture or deletion must not start. A true return must be paired with + /// [`Self::capture_end`]. pub(crate) fn capture_begin(&self) -> bool { let mut session = self.bio_session.lock(); if self.suspending.load(Relaxed) { @@ -3194,19 +3195,33 @@ impl SepData { bio::ioctl(&mut ctx, cmd, arg)? }; - // 0x57 sent with the session lock released (it waits on a mailbox reply) - if let Some(identity) = handled.delete_identity { - self.delete_identity(&identity)?; - } - - let mut delete_all_err: Option = None; - for identity in handled.delete_identities.iter() { - if let Err(e) = self.delete_identity(identity) { - delete_all_err = delete_all_err.or(Some(e)); - } - } - if let Some(e) = delete_all_err { - return Err(e); + if handled.delete_identity.is_some() || !handled.delete_identities.is_empty() { + // A deletion is made durable by saving the Catacombs, and with + // xART writes off that save would be refused. Refuse before 0x57 + // rather than report a deletion that the next boot undoes. + if !self.xart_writable() { + dev_warn!( + self.dev, + "Touch ID: deleting a fingerprint needs xart_writes=1; with writes off the deletion would not survive a reboot\n" + ); + return Err(EROFS); + } + // 0x57 and the saves are sent with the session lock released + // (they wait on mailbox replies). The system must not sleep + // partway through the saves, and a verify that finishes after a + // cancel saves too. + let deleted = loop { + self.wait_until_awake()?; + let _writer = self.catacomb_writer.lock(); + // A suspend began while this waited for the lock: wait it out. + if !self.capture_begin() { + continue; + } + let deleted = self.delete_and_persist(&handled); + self.capture_end(); + break deleted; + }; + deleted?; } if handled.wake { @@ -3440,10 +3455,77 @@ impl SepData { Some(records.lists_uuid(uuid)) } - fn delete_identity(&self, identity: &crate::sbio::IdentityV1) -> Result<()> { + /// Called with `catacomb_writer` held. A host index entry goes only once + /// its deletion is saved: if the save fails, the entry stays, userspace + /// still lists the fingerprint, and a retried deletion saves again. + fn delete_and_persist(&self, handled: &bio::Handled) -> Result<()> { + let retry = self.deletion_unsaved.load(Relaxed); + let wanted = + Iterator::chain(handled.delete_identity.iter(), handled.delete_identities.iter()); + // Allocated before any 0x57, so recording a removal cannot fail. + let mut gone: KVec<[u8; bio::UUID_LEN]> = + KVec::with_capacity(wanted.clone().count(), GFP_KERNEL)?; + let mut delete_err: Option = None; + for identity in wanted { + match self.delete_identity(identity) { + Ok(true) => { + let _ = gone.push_within_capacity(*identity.uuid()); + } + Ok(false) => {} + Err(e) => delete_err = delete_err.or(Some(e)), + } + } + // Save what was removed even if another deletion failed. + if !gone.is_empty() { + match self.persist_deletion(retry) { + Ok(()) => { + self.deletion_unsaved.store(false, Relaxed); + for uuid in gone.iter() { + self.forget_identity(uuid, c"removed from the enclave"); + } + } + Err(e) => { + dev_err!( + self.dev, + "bio: removed from the enclave but not saved; the fingerprint stays listed, returns at the next boot, and deleting it again retries the save\n" + ); + self.deletion_unsaved.store(true, Relaxed); + delete_err = delete_err.or(Some(e)); + } + } + } + match delete_err { + Some(e) => Err(e), + None => Ok(()), + } + } + + /// Returns whether the enclave no longer holds the identity and the host + /// entry must wait for [`Self::persist_deletion`]. + fn delete_identity(&self, identity: &crate::sbio::IdentityV1) -> Result { if self.enclave_lists(identity.uuid()) == Some(false) { + // Unless an earlier deletion's save failed, which leaves the + // enclave without an identity the saved Catacomb still holds. + if self.deletion_unsaved.load(Relaxed) { + return Ok(true); + } self.forget_identity(identity.uuid(), c"a host entry the enclave never had"); - return Ok(()); + return Ok(false); + } + + // 0x57 cannot be undone. Check first that its deletion can be saved. + if crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_none() + || !self + .enrol_material + .lock() + .as_ref() + .is_some_and(|material| material.snapshot_identity.is_some()) + { + dev_err!( + self.dev, + "bio: not deleting: there is no owner Catacomb file or identity-bag binding to save the deletion with; nothing was changed\n" + ); + return Err(EIO); } let op = crate::sbio::sbio_delete_identity(identity); @@ -3456,7 +3538,76 @@ impl SepData { return Err(EIO); } - self.forget_identity(identity.uuid(), c"removed from the enclave"); + Ok(true) + } + + /// 0x57 removes an identity from the enclave's memory only. Until the user + /// Catacomb is saved, the next attach restores the saved one and the + /// deleted fingerprint with it. Saved like a match: only the components + /// the enclave marks save-pending (a SAVE_CATACOMB of one that is not has + /// been refused), the user before the master because saving the user can + /// change the master, state re-read before each, then the lockout and the + /// identity bag, which must match the Catacombs. Safe to repeat after a + /// failure partway: a `retry` accepts a user Catacomb an earlier attempt + /// already saved. + fn persist_deletion(&self, retry: bool) -> Result<()> { + let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + return Err(EIO); + }; + for (who, kind, what) in [ + ( + crate::sbio::CatacombUser::enrolling(user), + PRIVATE_TYPE_CATACOMB_USER, + c"user Catacomb after deletion", + ), + ( + crate::sbio::CatacombUser::MASTER, + PRIVATE_TYPE_CATACOMB_MASTER, + c"master Catacomb after deletion", + ), + ] { + let Some(reply) = self.read_component_states() else { + dev_err!(self.dev, "bio: cannot read component state before saving the {}\n", what); + return Err(EIO); + }; + let state = crate::sbio::ComponentStates::new(&reply).state_for(who.value()); + dev_info!(self.dev, "bio: {} state {:?}\n", what, state); + let Some(state) = state else { + dev_err!(self.dev, "bio: the {} is absent; the deletion cannot be saved\n", what); + return Err(EIO); + }; + if state & crate::sbio::COMPONENT_STATE_ACTIVE == 0 { + dev_err!(self.dev, "bio: the {} is not active; the deletion cannot be saved\n", what); + return Err(EIO); + } + if state & crate::sbio::COMPONENT_STATE_SAVE_PENDING != 0 { + if !self.save_catacomb(who, kind, what) { + dev_err!(self.dev, "bio: saving the {} failed\n", what); + return Err(EIO); + } + } else if who.value() == user.value() && !retry { + dev_err!( + self.dev, + "bio: the enclave did not mark the user Catacomb for saving after 0x57; the deletion cannot be saved\n" + ); + return Err(EIO); + } + if crate::catacomb::read(kind).is_none() { + dev_err!(self.dev, "bio: the {} has no durable host file\n", what); + return Err(EIO); + } + } + if !self.save_lockout() { + dev_err!(self.dev, "bio: lockout persistence failed after the deletion\n"); + return Err(EIO); + } + if !self.resnapshot_identity_keybag() { + dev_err!( + self.dev, + "bio: Catacombs saved after the deletion but the identity-bag snapshot failed\n" + ); + return Err(EIO); + } Ok(()) } diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index f04f67d424531..16e178dc6fcd9 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -733,13 +733,25 @@ struct SepData { #[pin] sleep_over: CondVar, - // Enrol and verify work items between `capture_begin` and `capture_end`. + // Enrol and verify work items, and deletions, between `capture_begin` + // and `capture_end`. captures_running: Atomic, // Signalled when an enrol or verify work item has stopped. #[pin] worker_gone: CondVar, + // Held by an enrol or verify work item for its whole run and by a + // deletion while it removes identities and saves the Catacombs, so two + // Catacomb save sequences never interleave. + #[pin] + catacomb_writer: Mutex<()>, + + // An identity has left the enclave but the Catacomb save that makes its + // deletion durable failed; the next deletion retries the save. Changed + // only under `catacomb_writer`. + deletion_unsaved: Atomic, + #[pin] rx_work: Work, @@ -919,6 +931,8 @@ impl SepData { sleep_over <- new_condvar!("SepData::sleep_over"), captures_running: Atomic::new(0), worker_gone <- new_condvar!("SepData::worker_gone"), + catacomb_writer <- new_mutex!(()), + deletion_unsaved: Atomic::new(false), rx_work <- new_work!("SepData::rx_work"), enrol_work <- new_work!("SepData::enrol_work"), verify_work <- new_work!("SepData::verify_work"), @@ -2495,6 +2509,8 @@ impl WorkItem for SepData { type Pointer = Arc; fn run(this: Arc) { + // Serialize saves before joining the suspend drain. + let _writer = this.catacomb_writer.lock(); let epoch = this.capture_epoch.load(Relaxed); if !this.shutting_down.load(Relaxed) && this.capture_begin() { this.run_enrolment(); @@ -2508,6 +2524,8 @@ impl WorkItem for SepData { type Pointer = Arc; fn run(this: Arc) { + // Serialize saves before joining the suspend drain. + let _writer = this.catacomb_writer.lock(); let epoch = this.capture_epoch.load(Relaxed); if !this.shutting_down.load(Relaxed) && this.capture_begin() { this.run_verify(); From 47e804d018da7886fd07635620ec0fbcdf60ee86 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 12:41:28 -0700 Subject: [PATCH 1725/2680] tools: aurora-sep: describe the M2 Pro/Max key-store split An M2 Pro/Max on system firmware 26.2 or earlier now gets the 13.5 key store and the profile "T6020/J414s (13.5 key store)". The agent test plan said every M2 Pro/Max reads "T6020/J414s", and told an older Mac that hit -13 to update to 26.6.x. With the split, updating after enrolling loses the enrolments. Describe both profiles, ask for the new firmware log line, say where a -13 is still worth reporting, and warn about the update. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit f62ac02e9cb07ec9f19cc90b564d1dcdde70e1f1) --- tools/aurora-sep/install-aurora-sep.sh | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index b8041c9505068..5ba006bda083a 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -772,13 +772,18 @@ SAFETY, NON-NEGOTIABLE apple_sep lines at all. Capture the whole block either way. If the profile line says a SoC you did not expect, report that verbatim. On every MacBook Pro M2 Pro/Max (J414s, J414c, J416s, J416c) it reads - "T6020/J414s", on the MacBook Pro 14"/16" M1 Max (J314c, J316c) - "T6000/J316s", and on the MacBook Air 13" M2 (J413) "T8112/J415"; those are - expected. - If "CREATE_KEYBAG" fails with status -13, quote your system-fw-version. - Every M2-family Mac that has created its keybag was on 26.6.x; on an older - one, update macOS to 26.6.x, boot back into Linux, re-run the installer and - report again. If it still fails on 26.6.x, that is the report we need. + "T6020/J414s", or "T6020/J414s (13.5 key store)" on system firmware 26.2 + or earlier; on the MacBook Pro 14"/16" M1 Max (J314c, J316c) + "T6000/J316s", and on the MacBook Air 13" M2 (J413) "T8112/J415". Those + are expected. On an M2 Pro/Max, also quote the line that starts + "M2 Pro/Max on system firmware". + If "CREATE_KEYBAG" fails with status -13, quote that line too. Firmware + up to 26.2 uses the 13.5 key store and later firmware variant 5. Variant + 5 works on 26.6.x and 27.0; 26.3 to 26.5 have not been tried, so a -13 + there is the report we need. + Updating macOS from 26.2 or earlier to a later release after enrolling + loses the enrolments: Touch ID then needs a new keybag. Say so if that + is what happened. The sensor itself: "sudo dmesg | grep apple-mesa" should say the power line came "from the device node". Quote it if it says anything else. From 4d63e4524751ba500a0d33e7b83f6156640a322b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:25:42 -0700 Subject: [PATCH 1726/2680] tools: aurora-sep: release 11.32 linux-aurora 7.1.12.aurora2-11.32, built from f62ac02e9cb0, with three Touch ID fixes from Justin Pfister's review of aurora-silicon/linux#69: - a deleted fingerprint is saved as deleted and no longer returns after a reboot; - an M2 Pro/Max on system firmware 26.2 or earlier uses the 13.5 key store, whose keybag that enclave accepts; - the hwrng thread no longer asks the enclave for entropy while the system sleeps. Tested on a J314s (MacBook Pro 14" M1 Pro): the release checks pass, and deleting one finger, then the last, survives a reboot each time, with re-enrolment, sleep and Touch ID unlock afterwards. Device trees and the kernel config are unchanged from 11.31. libfprint, aurora-touchid and m1n1-aurora are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 6c2e974176c18d501f1e354ba1d2f401b98b29a3) --- tools/aurora-sep/install-aurora-sep.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 5ba006bda083a..abdc48ddce1d1 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -54,6 +54,12 @@ # The Neural Engine now powers off when idle (aurora-silicon/linux#155, Joshua # Warren), and is switched on for the M1, M1 Pro and M2 Pro as well as the M1 # Max and M2 Max. +# 11.32 carries three Touch ID fixes from Justin Pfister's review +# (aurora-silicon/linux#69): a deleted fingerprint is saved as deleted and no +# longer returns after a reboot; an M2 Pro/Max on system firmware 26.2 or +# earlier uses the 13.5 key store, whose keybag that enclave accepts; and the +# kernel's random-number thread no longer asks the enclave for data while the +# Mac sleeps, which left Touch ID failing after resume on an M2 Pro. # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -82,8 +88,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.31 -TAG=sep-7.1.12.aurora2-11.31 +VERSION=7.1.12.aurora2-11.32 +TAG=sep-7.1.12.aurora2-11.32 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -91,8 +97,8 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst 7e7e04db48e502325f9f287ffc01543140beb64feb63719f69f937851d7cd345" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst a04ae0735998f409976fb0dd815f2de53eee2740c7326aef2f6ea403f3077494" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst d96dde00dcee4249417eab44aadef68adc9bdbf899b83dd47813c9d7862dda29" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst e9f4e9e22b57b6c10a7db232179e2f0e85962ae0bde9d2629f753baecff445cd" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20261002-1-any.pkg.tar.zst a9dda6e0526874e4ac760629f3aa5bd37421379a1d8af34000f3dc7b73a21b17" "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" @@ -866,7 +872,7 @@ SAFETY, NON-NEGOTIABLE sudo dmesg | grep -E 'cold init done|link up after|translation fault|HC died' Any "translation fault" or "HC died" line is a failure to report. For a full report, after plugging the dock in: - curl -fsSLO https://raw.githubusercontent.com/iconidentify/aurora-linux/refs/tags/sep-7.1.12.aurora2-11.31/tools/aurora-tb/tb-pcie-report + curl -fsSLO https://raw.githubusercontent.com/iconidentify/aurora-linux/refs/tags/sep-7.1.12.aurora2-11.32/tools/aurora-tb/tb-pcie-report sudo sh tb-pcie-report --no-wait 6c. Across suspend: From fa6fc2c9332c79911a446893b2ace4b6ef50c3a4 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 09:47:05 -0700 Subject: [PATCH 1727/2680] arm64: dts: apple: t6030: add GPU, USB4/Thunderbolt and display nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe the T6030 (M3 Pro) AGX G15S GPU and its ASC mailbox, the USB4/Thunderbolt routers, PCIe tunnels and the external display routing used on J514S and J516S. [ryan: adapted to 7.2.10 — Retain Asahi USB4 nodes, five-cell DART apertures and connector graphs; add only missing router windows, power suppliers and PCIe adapter aliases. Fold the same-author GPU comment cleanup from the subsequent display-tree change into this import, omitting its private-document citations.] Co-Authored-By: Claude Opus 5.5 (1M context) (cherry picked from commit 8337f93c3589d36fff7ea5904389ed03079488a9) --- arch/arm64/boot/dts/apple/t6030-gpu.dtsi | 185 ++++++++++++++++++ arch/arm64/boot/dts/apple/t6030-j514s.dts | 1 + arch/arm64/boot/dts/apple/t6030-j516s.dts | 1 + arch/arm64/boot/dts/apple/t6030-usb4.dtsi | 32 +++ arch/arm64/boot/dts/apple/t6030.dtsi | 2 + .../dts/apple/t603x-j514s-j516s-usb4.dtsi | 10 + 6 files changed, 231 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t6030-gpu.dtsi create mode 100644 arch/arm64/boot/dts/apple/t6030-usb4.dtsi create mode 100644 arch/arm64/boot/dts/apple/t603x-j514s-j516s-usb4.dtsi diff --git a/arch/arm64/boot/dts/apple/t6030-gpu.dtsi b/arch/arm64/boot/dts/apple/t6030-gpu.dtsi new file mode 100644 index 0000000000000..6713bb39e45eb --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6030-gpu.dtsi @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT + +/ { + aliases { + gpu = &gpu; /* m1n1 dt_set_gpu() finds the node through this alias */ + }; + + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* + * Placeholder only. m1n1 dt_set_gpu_t6030() deletes these children and + * creates one OPP per runtime pstate, with one opp-microvolt cell per + * MGPU (2). The pstate count and the frequency and voltage values are + * filled by the bootloader; the template pstates are zero. + */ + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <400000 400000>; + opp-microwatt = <0>; + }; + opp01 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <400000 400000>; + opp-microwatt = <0>; + }; + }; + + reserved-memory { + gpu_globals: globals { + status = "disabled"; + }; + + gpu_hw_cal_a: hw-cal-a { + status = "disabled"; + }; + + gpu_hw_cal_b: hw-cal-b { + status = "disabled"; + }; + + /* + * Filled and enabled by m1n1 from the ADT sgx gfx-handoff-*, gfx-shared-region-* and gpu-region-* + * properties. Disabled until then, so that neither the kernel nor m1n1's RAM-map scan (dt_set_memory) + * sees a zero-sized placeholder. + */ + uat_handoff: uat-handoff { + reg = <0 0 0 0>; + status = "disabled"; + }; + + uat_pagetables: uat-pagetables { + reg = <0 0 0 0>; + status = "disabled"; + }; + + uat_ttbs: uat-ttbs { + reg = <0 0 0 0>; + status = "disabled"; + }; + + /* G15: iBoot's second-level table of the firmware TTBR1 tree (ADT gfx-shared-l2-region); + * must be in the linear map like uat-pagetables. m1n1 fills reg and enables it. */ + uat_pagetables_l2: uat-pagetables-l2 { + reg = <0 0 0 0>; + status = "disabled"; + }; + }; +}; + +&soc { + /* + * asc is 0x40000 at 0x2_9240_0000. sgx is 16 MiB at 0x2_9000_0000, which covers + * the ID block at sgx+0xd04000, the core masks at sgx+0xe01500 and the Fender + * MMU block at sgx+0xd08000. The GPU has no DART: drm/asahi owns the UAT, as on t602x. + */ + gpu: gpu@292400000 { + /* m1n1 sets "okay" as its last step, after every other property has been filled in */ + status = "disabled"; + /* apple,agx-t6030 is the compatible drm/asahi and m1n1 match. */ + compatible = "apple,agx-t6030", "apple,agx-g15x", "apple,agx-g15s"; + reg = <0x2 0x92400000 0 0x40000>, + <0x2 0x90000000 0 0x1000000>; + reg-names = "asc", "sgx"; + interrupt-parent = <&aic>; + /* + * Seven level interrupts in source order. The valid mask 0x5f marks entry 5 + * (838) invalid; it is kept so the list stays aligned, as t602x keeps all 7. + */ + interrupts = , + , + , + , + , + , + ; + mboxes = <&agx_mbox>; + /* "gfx" power domain at pmgr offset 0x110 (parents gpx, afr). */ + power-domains = <&ps_gfx>; + /* + * Do not hold back the ps_gfx sync_state() on this device when it never probes (fw_devlink would + * otherwise keep "gfx" in its boot state forever). + */ + post-init-providers = <&ps_gfx>; + memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>, + <&gpu_hw_cal_a>, <&gpu_hw_cal_b>, <&gpu_globals>; + memory-region-names = "ttbs", "pagetables", "handoff", + "hw-cal-a", "hw-cal-b", "globals"; + + apple,firmware-abi = <0 0 0>; + apple,firmware-version = <0 0 0>; + apple,firmware-compat = <0 0 0>; + + /* No apple,cs-opp, apple,afr-opp or CS/AFR leak coefficients: this GPU has no CS or AFR domain. */ + operating-points-v2 = <&gpu_opp>; + + apple,min-sram-microvolt = <810000>; + apple,perf-base-pstate = <1>; + + /* Power-management coefficients. Floats are IEEE-754 bit patterns. */ + apple,avg-power-filter-tc-ms = <40>; + apple,avg-power-ki-only = <45.0>; + apple,avg-power-kp = <1.4>; + apple,avg-power-min-duty-cycle = <40>; + apple,avg-power-target-filter-tc = <1>; + apple,fast-die0-integral-gain = <1552.0>; + apple,fast-die0-proportional-gain = <37.0>; + apple,idleoff-standby-timer = <1500>; + apple,perf-boost-ce-step = <50>; + apple,perf-boost-min-util = <90>; + apple,perf-filter-drop-threshold = <0>; + apple,perf-filter-time-constant = <5>; + apple,perf-filter-time-constant2 = <200>; + apple,perf-integral-gain = <1.755>; + apple,perf-integral-gain2 = <1.755>; + apple,perf-integral-min-clamp = <0>; + apple,perf-proportional-gain = <5.85>; + apple,perf-proportional-gain2 = <5.85>; + apple,perf-tgt-utilization = <80>; + apple,power-sample-period = <8>; + apple,ppm-filter-time-constant-ms = <32>; + apple,ppm-ki = <28.0>; + apple,ppm-kp = <0.1>; + apple,pwr-filter-time-constant = <313>; + apple,pwr-integral-gain = <0.0202129>; + apple,pwr-integral-min-clamp = <0>; + apple,pwr-min-duty-cycle = <40>; + apple,pwr-proportional-gain = <5.2831855>; + apple,pwr-sample-period-aic-clks = <200000>; + apple,se-engagement-criteria = <600>; + apple,se-filter-time-constant = <9>; + apple,se-filter-time-constant-1 = <3>; + apple,se-inactive-threshold = <2500>; + apple,se-ki = <-50.0>; + apple,se-ki-1 = <-100.0>; + apple,se-kp = <-5.0>; + apple,se-kp-1 = <-10.0>; + apple,se-reset-criteria = <50>; + /* + * One value per MGPU (drm/asahi requires two entries). Placeholders: m1n1 + * writes the fused core leakage into both entries and 0 for SRAM. + */ + apple,core-leak-coef = <1.0 1.0>; + apple,sram-leak-coef = <0.0 0.0>; + }; + + /* + * gfx-asc mailbox at asc + 0x8000, as on t602x and for the other t6030 ASC mailboxes. + * Interrupts 832..835 are send-empty, send-not-empty, recv-empty, recv-not-empty. + * Compatible is only apple,t6030-agx-asc-mailbox, with no v4 fallback, so a kernel + * without G15 support leaves IRQs 832-835 unbound. + */ + agx_mbox: mbox@292408000 { + compatible = "apple,t6030-agx-asc-mailbox"; + reg = <0x2 0x92408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6030-j514s.dts b/arch/arm64/boot/dts/apple/t6030-j514s.dts index 724d94dc65575..0b096c8f058c6 100644 --- a/arch/arm64/boot/dts/apple/t6030-j514s.dts +++ b/arch/arm64/boot/dts/apple/t6030-j514s.dts @@ -11,6 +11,7 @@ #include "t6030.dtsi" #include "t603x-j514-j516.dtsi" +#include "t603x-j514s-j516s-usb4.dtsi" / { compatible = "apple,j514s", "apple,t6030", "apple,arm-platform"; diff --git a/arch/arm64/boot/dts/apple/t6030-j516s.dts b/arch/arm64/boot/dts/apple/t6030-j516s.dts index c6fb1c55cecfb..6da56a3db828f 100644 --- a/arch/arm64/boot/dts/apple/t6030-j516s.dts +++ b/arch/arm64/boot/dts/apple/t6030-j516s.dts @@ -11,6 +11,7 @@ #include "t6030.dtsi" #include "t603x-j514-j516.dtsi" +#include "t603x-j514s-j516s-usb4.dtsi" / { compatible = "apple,j516s", "apple,t6030", "apple,arm-platform"; diff --git a/arch/arm64/boot/dts/apple/t6030-usb4.dtsi b/arch/arm64/boot/dts/apple/t6030-usb4.dtsi new file mode 100644 index 0000000000000..bc4c01d8ab189 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6030-usb4.dtsi @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* Extend the existing USB4 routers with their PCIe adapter and ASC windows. */ + +&usb4_0_acio { + reg = <0x7 0x01ac0000 0x0 0xc000>, + <0x7 0x01080000 0x0 0x20000>, + <0x7 0x01e44000 0x0 0x4000>, + <0x7 0x01100000 0x0 0x4000>; + reg-names = "rc", "sram", "pcie-adapter", "cpu"; + power-domains = <&ps_atc0_cio>, <&ps_atc0_cio_pcie>, + <&ps_atc0_cio_usb>, <&ps_atc0_pcie>; +}; + +&usb4_1_acio { + reg = <0xb 0x01ac0000 0x0 0xc000>, + <0xb 0x01080000 0x0 0x20000>, + <0xb 0x01e44000 0x0 0x4000>, + <0xb 0x01100000 0x0 0x4000>; + reg-names = "rc", "sram", "pcie-adapter", "cpu"; + power-domains = <&ps_atc1_cio>, <&ps_atc1_cio_pcie>, + <&ps_atc1_cio_usb>, <&ps_atc1_pcie>; +}; + +&usb4_2_acio { + reg = <0xf 0x01ac0000 0x0 0xc000>, + <0xf 0x01080000 0x0 0x20000>, + <0xf 0x01e44000 0x0 0x4000>, + <0xf 0x01100000 0x0 0x4000>; + reg-names = "rc", "sram", "pcie-adapter", "cpu"; + power-domains = <&ps_atc2_cio>, <&ps_atc2_cio_pcie>, + <&ps_atc2_cio_usb>, <&ps_atc2_pcie>; +}; diff --git a/arch/arm64/boot/dts/apple/t6030.dtsi b/arch/arm64/boot/dts/apple/t6030.dtsi index 8f1b57bc2bb3c..849df0e53bca8 100644 --- a/arch/arm64/boot/dts/apple/t6030.dtsi +++ b/arch/arm64/boot/dts/apple/t6030.dtsi @@ -2071,3 +2071,5 @@ }; #include "t6030-pmgr.dtsi" +#include "t6030-gpu.dtsi" +#include "t6030-usb4.dtsi" diff --git a/arch/arm64/boot/dts/apple/t603x-j514s-j516s-usb4.dtsi b/arch/arm64/boot/dts/apple/t603x-j514s-j516s-usb4.dtsi new file mode 100644 index 0000000000000..d128b1f3b9112 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t603x-j514s-j516s-usb4.dtsi @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* Additional loader aliases for the T6030 PCIe adapter tunables. */ + +/ { + aliases { + usb4_0_pcie_adapter = &usb4_0_acio; + usb4_1_pcie_adapter = &usb4_1_acio; + usb4_2_pcie_adapter = &usb4_2_acio; + }; +}; From a7181062f84ce2fb4fd045ae9bceb36b3c3de74a Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:51:21 -0700 Subject: [PATCH 1728/2680] soc: apple: sep: say why an enrolment capture failed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a capture fails partway through an enrolment, userspace only sees "enrolment failed, driver status 0x1". The driver logged nothing for most of those exits: the capture could not start, the sensor fell to an unexpected state, its status became unreadable, the frame read failed, or there was no memory for it. The same goes for the sensor not coming back online when an enrolment starts, and for the retries after a bad checksum or frame length. On a J314s a fifth-touch failure like this left nothing to go on. Log each of them the way the verify path already does, with the capture number. A cancelled enrolment also abandons its capture, so as in verify only a live one is reported. [ryan: adapted to 7.2.10 — bind the new diagnostic liveness check to our queued capture epoch.] Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 390826240cd757554edf32a600c26293e83047cc) --- drivers/soc/apple/sbio.rs | 56 ++++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index c8040b5b5e332..a049dbaed1569 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1649,6 +1649,7 @@ impl SepData { self.enrol_frames_accepted.store(0, Relaxed); if !self.bring_sensor_online() { + dev_err!(self.dev, "enrol: the sensor did not come back online\n"); self.finish_enrolment(Err(ENROL_STATUS_SENSOR)); let _ = sensor::idle(); return; @@ -2483,36 +2484,73 @@ impl SepData { false } + // Like verify, every exit that ends an enrolment says why: userspace sees + // only "driver status 0x1", which cannot tell these apart. fn enrol_one_image(&self, counter: u32) -> ImageOutcome { - if sensor::start_capture().is_err() { + if let Err(e) = sensor::start_capture() { + dev_err!(self.dev, "enrol: could not start capture {} ({:?})\n", counter, e); return ImageOutcome::Failed(ENROL_STATUS_SENSOR); } let available = match self.await_capture() { CaptureWait::Ready(n) => n, - CaptureWait::Timeout => return ImageOutcome::NoFinger, - CaptureWait::Fault(_) => { + CaptureWait::Timeout => { + dev_warn!(self.dev, "enrol: capture {} ended without a frame\n", counter); + return ImageOutcome::NoFinger; + } + CaptureWait::Fault(state) => { + dev_err!( + self.dev, + "enrol: the sensor fell to state {} during capture {}\n", + state, + counter + ); + return ImageOutcome::Failed(ENROL_STATUS_SENSOR); + } + CaptureWait::Abandon => { + // A cancelled enrolment also abandons; only a live one is a fault. + if bio::enrol_is_live(&self.bio_session.lock(), self.capture_epoch.load(Relaxed)) { + dev_err!( + self.dev, + "enrol: the sensor status became unreadable during capture {}\n", + counter + ); + } return ImageOutcome::Failed(ENROL_STATUS_SENSOR); } - CaptureWait::Abandon => return ImageOutcome::Failed(ENROL_STATUS_SENSOR), }; let capture = match sensor::read_capture(available) { Ok(c) => c, Err(sensor::CaptureError::Checksum { - advertised: _advertised, - computed: _computed, + advertised: sent, + computed, }) => { + dev_warn!( + self.dev, + "enrol: capture {} frame checksum mismatch (sensor sent {:#06x}, computed {:#06x}); retrying\n", + counter, + sent, + computed + ); return ImageOutcome::Retry; } - Err(sensor::CaptureError::Length(_n)) => { + Err(sensor::CaptureError::Length(n)) => { + dev_warn!( + self.dev, + "enrol: capture {} advertised an unusable frame length {}; retrying\n", + counter, + n + ); return ImageOutcome::Retry; } - Err(sensor::CaptureError::Bus(_e)) => { + Err(sensor::CaptureError::Bus(e)) => { + dev_err!(self.dev, "enrol: reading the frame of capture {} failed ({:?})\n", counter, e); return ImageOutcome::Failed(ENROL_STATUS_SENSOR); } Err(sensor::CaptureError::NoMemory) => { - return ImageOutcome::Failed(ENROL_STATUS_SENSOR) + dev_err!(self.dev, "enrol: no memory for the frame of capture {}\n", counter); + return ImageOutcome::Failed(ENROL_STATUS_SENSOR); } }; From 4caccfcc2772f8baa06d0121296f15defcb1321b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:57:19 -0700 Subject: [PATCH 1729/2680] tools: aurora-sep: add --reset-touchid to start Touch ID over An identity keybag that no longer loads leaves Touch ID refusing to start, and nothing short of hand-deleting files fixed it. That happens to an M2 Pro/Max enrolled on system firmware 26.2 or earlier and then updated, since the keybag was made by the other key store. --reset-touchid moves the SEP driver's state (host state, Catacombs, reference key, and the keybag last) and fprintd's prints for the Apple sensor into /var/lib/aurora-sep-touchid-reset-/dev/null || die "no terminal to confirm on; run again with --reset-touchid --yes" + local answer="" + printf 'Type RESET to continue: ' >/dev/tty + read -r answer /dev/null) + for f in "${TOUCHID_STATE_FILES[@]}"; do [[ -e $f ]] && sources+=("$f"); done + for src in "${sources[@]}"; do + case $src in + /var/lib/fprint/*) to=$dest/fprint/$(basename "$(dirname "$src")")/apple-sep ;; + *) to=$dest/$(basename "$src") ;; + esac + if ! { $sudo mkdir -p "$(dirname "$to")" && $sudo mv "$src" "$to"; }; then + for d in "${moved[@]}"; do $sudo mv "${d#*|}" "${d%%|*}" || warn "could not put back ${d%%|*} from ${d#*|}"; done + die "could not move $src; put back everything already moved, so Touch ID is as it was" + fi + moved+=("$src|$to") + done + say "Moved the Touch ID state to $dest" + say "Reboot now. Touch ID creates its new keybag during boot; then enrol your fingers again." +} + # Printed by --agent-prompt, and pointed at from the end of a successful # install. This is written for an agent driving the test on a real Mac: it says # what to establish, what counts as a pass, and how to write it up. @@ -972,13 +1069,15 @@ HOW TO REPORT PROMPT } -preflight_needed() { case ${1:-} in --agent-prompt) return 1 ;; *) return 0 ;; esac; } +# A reset needs none of the kernel and boot checks; it checks for itself. +preflight_needed() { case ${1:-} in --agent-prompt | --reset-touchid) return 1 ;; *) return 0 ;; esac; } if preflight_needed "${1:-}"; then preflight; fi case ${1:-} in "") install_all ;; --read-only) READ_ONLY=1; install_all ;; --uninstall) uninstall_all ;; + --reset-touchid) shift; reset_touchid "$@" ;; --agent-prompt) t=$(newer_release); [[ -n $t ]] && warn "the current release is $t; this plan is for $TAG"; agent_prompt ;; - *) die "unknown option $1 (--read-only, --uninstall or --agent-prompt)" ;; + *) die "unknown option $1 (--read-only, --uninstall, --reset-touchid or --agent-prompt)" ;; esac From 3945ffb74ec75324bff9be40bcae9182e36d9b7d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:57:48 -0700 Subject: [PATCH 1730/2680] tools: aurora-touchid: verify as root in aurora-touchid-setup aurora-touchid-setup enrols with sudo but verified as the user. polkit lets an unprivileged user claim the reader only from the active local session, so run over SSH (as a tester's agent may) the verify step was refused with "Not Authorized: net.reactivated.fprint.device.enroll", and the script stopped before configuring PAM although the enrolment had worked. Verify with sudo too. Bump aurora-touchid to 20261003. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 9b70be708ca1e83df9ce6fb23456844bb51d9d0b) --- tools/aurora-sep/aurora-touchid/PKGBUILD | 4 ++-- tools/aurora-sep/aurora-touchid/aurora-touchid-setup | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/aurora-sep/aurora-touchid/PKGBUILD b/tools/aurora-sep/aurora-touchid/PKGBUILD index 052669d917f5e..a3088d4b375f5 100644 --- a/tools/aurora-sep/aurora-touchid/PKGBUILD +++ b/tools/aurora-sep/aurora-touchid/PKGBUILD @@ -1,6 +1,6 @@ # Maintainer: Chris Kearney <303316+iconidentify@users.noreply.github.com> pkgname=aurora-touchid -pkgver=20261002 +pkgver=20261003 pkgrel=1 pkgdesc="Touch ID for Omarchy on Apple Silicon (aurora kernel SEP driver)" arch=('any') @@ -15,7 +15,7 @@ source=('aurora-touchid-setup' 'aurora-touchid.sleep' 'wait-for-display' 'sddm-wait-display.conf') -sha256sums=('45639372c72c3dc519e815ea7162165c1b3116f1b08102960e67a83c58bd742a' +sha256sums=('406da23baf0127dd5ba116c2f83808fb10d979315a17227a78731c5d3081efe4' '2e1d50b5274223e107e4aa4f73de194f3a49d347be1edc93efd3f81b31b092cc' '4f7b8a97e35e3511afa482ee8aedd019e960b1a67320675bd347f534feab49c3' '2f4b265a4c2180a9e0e4926c53ec77c3dbb074ddc3e083d7925d58db10a576b9' diff --git a/tools/aurora-sep/aurora-touchid/aurora-touchid-setup b/tools/aurora-sep/aurora-touchid/aurora-touchid-setup index 3869fad7fa273..f317ee0d76857 100755 --- a/tools/aurora-sep/aurora-touchid/aurora-touchid-setup +++ b/tools/aurora-sep/aurora-touchid/aurora-touchid-setup @@ -73,7 +73,9 @@ say "Place your right index finger on Touch ID, lift and repeat until enrolment $sudo fprintd-enroll "${SUDO_USER:-$USER}" || die "Enrolment failed. Run aurora-touchid-setup again." say "Enrolled. Now touch the sensor once more to verify." -fprintd-verify "${SUDO_USER:-$USER}" || die "Verification failed; PAM was not changed. Run aurora-touchid-setup again." +# As root, like the enrolment: polkit lets an unprivileged user claim the +# reader only from the active local session, so over SSH it refuses. +$sudo fprintd-verify "${SUDO_USER:-$USER}" || die "Verification failed; PAM was not changed. Run aurora-touchid-setup again." configure_pam say "Touch ID now works for sudo, polkit prompts and the lock screen. Your password still works too." From 2b0c2c3c4c205b9bcbde18d90f2ca03998e64c78 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 15:07:18 -0700 Subject: [PATCH 1731/2680] tools: aurora-sep: --reset-touchid: test the keybag before refusing The reset runs only when the driver reports keybag=failed, and probed the driver only when it reported keybag=unknown. But at boot the driver reports keybag=present for any keybag file it finds, and loads it, or reports failed, only when the sensor is first opened. So on a Mac whose keybag no longer loads, run right after boot (an agent over SSH, or the curl one-liner), the reset said the keybag was not broken and refused the very case it is for. Probe whenever touchid=unknown, which is what the driver reports until the sensor has been opened. Run sudo once before the probe, in the foreground: under timeout a password prompt cannot read the terminal and the probe silently gave up. Use "sudo true", not "sudo -v": -v asks for a password unless every sudoers rule for the user is NOPASSWD, and a wheel member with a NOPASSWD drop-in still has wheel's rule, so an agent with passwordless sudo was refused (seen on a J314s). Say so when the SEP driver is not loaded, instead of calling the keybag "not broken". Suggest a reboot and one more try before resetting, since a passing error also reports failed. After a failed move puts everything back, unmask fprintd as well. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit c1d0a04fe9d7fde277c19fb3729236d0795b75f0) --- tools/aurora-sep/install-aurora-sep.sh | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 58698aa952318..a27a00f74bec0 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -744,11 +744,20 @@ reset_touchid() { for f in "${TOUCHID_STATE_FILES[@]}"; do [[ -e $f ]] && found=1; done ((found)) || die "no Touch ID state on this Mac; nothing to reset" - # Only a keybag the driver could not load is worth replacing. It loads the - # keybag the first time the sensor is opened, so ask fprintd once first. + # Ask for sudo now, in the foreground: a password prompt under timeout + # below could not read the terminal. Not sudo -v, which wants a password + # unless every rule for the user is NOPASSWD (a wheel member's is not). + if [[ -n $sudo ]]; then $sudo true || die "--reset-touchid needs sudo"; fi + + # Only a keybag the driver could not load is worth replacing. At boot the + # driver reports keybag=present for any keybag file it finds; it loads the + # keybag, and reports failed if that does not work, only when the sensor is + # first opened. Until then touchid=unknown, so open it through fprintd once. keybag=$(sep_diag keybag) - if [[ $keybag == unknown ]]; then - timeout 20 $sudo fprintd-list root >/dev/null 2>&1 || true + [[ $keybag == none ]] && + die "the Touch ID driver is not running (no SEP diagnostics). Boot the aurora kernel first." + if [[ $(sep_diag touchid) == unknown ]]; then + timeout --foreground 20 $sudo fprintd-list root >/dev/null 2>&1 || true keybag=$(sep_diag keybag) fi if [[ $keybag != failed ]]; then @@ -759,7 +768,8 @@ reset_touchid() { warn "this removes every enrolled fingerprint on this Mac and creates a new Touch ID keybag at the next boot. Anything sealed with the Secure Enclave (kernel trusted keys) can no longer be unsealed. - The old keybag stays in the Secure Enclave, unused; it cannot be deleted, and this cannot be undone once the Mac has rebooted." + The old keybag stays in the Secure Enclave, unused; it cannot be deleted, and this cannot be undone once the Mac has rebooted. + If Touch ID has failed only this once, reboot and try it again first: a passing error looks the same." if ((!yes)); then # Piped from curl, stdin is the script; ask on the terminal, if there is one. { : /dev/null || die "no terminal to confirm on; run again with --reset-touchid --yes" @@ -790,6 +800,7 @@ reset_touchid() { esac if ! { $sudo mkdir -p "$(dirname "$to")" && $sudo mv "$src" "$to"; }; then for d in "${moved[@]}"; do $sudo mv "${d#*|}" "${d%%|*}" || warn "could not put back ${d%%|*} from ${d#*|}"; done + $sudo systemctl unmask --runtime fprintd.service || true die "could not move $src; put back everything already moved, so Touch ID is as it was" fi moved+=("$src|$to") From b3b54656146282823305d2d5c1e9248af7a2b264 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 15:13:53 -0700 Subject: [PATCH 1732/2680] tools: aurora-sep: release 11.33 linux-aurora 7.1.12.aurora2-11.33, built from 9b70be708ca1, and aurora-touchid 20261003-1: - --reset-touchid starts Touch ID over on a Mac whose identity keybag no longer loads, such as an M2 Pro/Max enrolled on system firmware 26.2 or earlier and then updated; - enrolment failures are logged with their cause; - aurora-touchid-setup verifies as root, so it also completes over SSH. Tested on a J314s (MacBook Pro 14" M1 Pro): - the release checks pass, and the enrolment restores after a reboot; - the reset refuses a working keybag; - with --force, the next boot created a second keybag, and a finger enrolled on it matched and restored after a reboot (on 11.32, before the gate fix); - aurora-touchid-setup's verify claims the reader over SSH. The new enrolment log lines are not yet seen on hardware: with a finger already enrolled, fprintd's duplicate check runs a verify first. Device trees and the kernel config are unchanged from 11.32. libfprint and m1n1-aurora are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 345bcac08c596460304dad000ffd91a36afdf2fd) --- tools/aurora-sep/install-aurora-sep.sh | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index a27a00f74bec0..8693d4ccc8e02 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -6,7 +6,7 @@ # ... | bash -s -- --uninstall go back to the kernel this Mac had before # ... | bash -s -- --reset-touchid start Touch ID over: new keybag, enrol again # -# Kernel: iconidentify/aurora-linux custom/sep (f62ac02e9cb0), aurora-silicon/linux aurora-wip plus the +# Kernel: iconidentify/aurora-linux custom/sep (9b70be708ca1), aurora-silicon/linux aurora-wip plus the # Secure Enclave (Touch ID) driver, Thunderbolt (#8), the Apple video # decoder (#45), the M2 Max (t6021) profile and the consolidated Touch ID # series (aurora-silicon/linux#69: matching after a reboot on every profile, @@ -61,6 +61,11 @@ # earlier uses the 13.5 key store, whose keybag that enclave accepts; and the # kernel's random-number thread no longer asks the enclave for data while the # Mac sleeps, which left Touch ID failing after resume on an M2 Pro. +# 11.33 adds --reset-touchid, which starts Touch ID over on a Mac whose +# keybag no longer loads (such an M2 Pro/Max after a macOS update): it moves +# the Touch ID state aside for a new keybag at the next boot. Enrolment +# failures are now logged with their cause, and aurora-touchid-setup verifies +# as root, so it also completes over SSH. # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -89,8 +94,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.32 -TAG=sep-7.1.12.aurora2-11.32 +VERSION=7.1.12.aurora2-11.33 +TAG=sep-7.1.12.aurora2-11.33 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -98,10 +103,10 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst d96dde00dcee4249417eab44aadef68adc9bdbf899b83dd47813c9d7862dda29" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst e9f4e9e22b57b6c10a7db232179e2f0e85962ae0bde9d2629f753baecff445cd" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst 57969eb176558969a9162751d66b023921f0c4bc3a06ca8e17e19654a71838fd" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 639d1ff305995f8e4acee24b778a8afa7cf5c52855beb55929d52bb137775b81" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" - "aurora-touchid-20261002-1-any.pkg.tar.zst a9dda6e0526874e4ac760629f3aa5bd37421379a1d8af34000f3dc7b73a21b17" + "aurora-touchid-20261003-1-any.pkg.tar.zst 29b0360fac8c257d754e64bd1b9c33c487eb2595dd3c31e9138d7a476afa3d64" "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" ) PINNED="linux-aurora linux-aurora-headers libfprint m1n1-aurora" @@ -897,7 +902,9 @@ SAFETY, NON-NEGOTIABLE there is the report we need. Updating macOS from 26.2 or earlier to a later release after enrolling loses the enrolments: Touch ID then needs a new keybag. Say so if that - is what happened. + is what happened. With the owner's agreement, start Touch ID over: + curl -fsSL https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh | bash -s -- --reset-touchid + then reboot and enrol again. The sensor itself: "sudo dmesg | grep apple-mesa" should say the power line came "from the device node". Quote it if it says anything else. @@ -980,7 +987,7 @@ SAFETY, NON-NEGOTIABLE sudo dmesg | grep -E 'cold init done|link up after|translation fault|HC died' Any "translation fault" or "HC died" line is a failure to report. For a full report, after plugging the dock in: - curl -fsSLO https://raw.githubusercontent.com/iconidentify/aurora-linux/refs/tags/sep-7.1.12.aurora2-11.32/tools/aurora-tb/tb-pcie-report + curl -fsSLO https://raw.githubusercontent.com/iconidentify/aurora-linux/refs/tags/sep-7.1.12.aurora2-11.33/tools/aurora-tb/tb-pcie-report sudo sh tb-pcie-report --no-wait 6c. Across suspend: From 81e6f62ee0c7e6b06bc1b0512a63322835d73e48 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 19:44:33 -0700 Subject: [PATCH 1733/2680] tools: aurora-sep: release 11.34 linux-aurora 7.1.12.aurora2-11.34, built from e508e107a473: - Apple Studio Display brightness through DCP (iconidentify/aurora-linux#15, Wesley Grimes), as dcp--bl, sending nothing until a level is chosen; - the PCIe-C root port's link kept out of ASPM L1 on cold-initialized tunnels (M1/M2 Pro and Max), so unplugging an idle dock no longer leaves the port dead until reboot (#15); - review fixes: one hotplug event per unplug, the bl_update_wq guard, lenient attribute parsing, connector and DCP teardown lifetimes, and a parser allocation-failure fix. Tested on a J293 (MacBook Pro 13" M1) in the lab: a monitor through a USB-C to HDMI adapter (plug, unplug with one hotplug event, replug, DPMS off/on, s2idle), a CalDigit TS4's display and USB, a Kensington dock with a TV, and three reboots with panel brightness writes in flight, each with no oops, warning or hung task at shutdown. The Studio Display itself and the M1 Pro/Max dock path are untested here. Device trees and the kernel config are unchanged from 11.33. libfprint, aurora-touchid and m1n1-aurora are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit aac9f2360b038bb54449854f0d01fa7b513beac2) --- tools/aurora-sep/install-aurora-sep.sh | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 8693d4ccc8e02..55baa1b5600da 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -6,7 +6,7 @@ # ... | bash -s -- --uninstall go back to the kernel this Mac had before # ... | bash -s -- --reset-touchid start Touch ID over: new keybag, enrol again # -# Kernel: iconidentify/aurora-linux custom/sep (9b70be708ca1), aurora-silicon/linux aurora-wip plus the +# Kernel: iconidentify/aurora-linux custom/sep (e508e107a473), aurora-silicon/linux aurora-wip plus the # Secure Enclave (Touch ID) driver, Thunderbolt (#8), the Apple video # decoder (#45), the M2 Max (t6021) profile and the consolidated Touch ID # series (aurora-silicon/linux#69: matching after a reboot on every profile, @@ -66,6 +66,11 @@ # the Touch ID state aside for a new keybag at the next boot. Enrolment # failures are now logged with their cause, and aurora-touchid-setup verifies # as root, so it also completes over SSH. +# 11.34 sets the brightness of an Apple Studio Display through DCP +# (iconidentify/aurora-linux#15, Wesley Grimes): brightnessctl -d 'dcp-DP-*', +# and nothing is sent until a level is chosen. On the M1/M2 Pro and Max it +# keeps the Thunderbolt root port's link out of ASPM L1, so unplugging an idle +# dock no longer leaves the port dead until reboot. # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -94,8 +99,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.33 -TAG=sep-7.1.12.aurora2-11.33 +VERSION=7.1.12.aurora2-11.34 +TAG=sep-7.1.12.aurora2-11.34 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -103,8 +108,8 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst 57969eb176558969a9162751d66b023921f0c4bc3a06ca8e17e19654a71838fd" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 639d1ff305995f8e4acee24b778a8afa7cf5c52855beb55929d52bb137775b81" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst 2d525694a1451c26b5ede7b1e8e63abd2085b03b620b6c54864119fc04c0f0ee" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 2242777b47969d42910a4375872721368eb69015db7a773f3e723d45adf49e36" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20261003-1-any.pkg.tar.zst 29b0360fac8c257d754e64bd1b9c33c487eb2595dd3c31e9138d7a476afa3d64" "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" @@ -987,7 +992,7 @@ SAFETY, NON-NEGOTIABLE sudo dmesg | grep -E 'cold init done|link up after|translation fault|HC died' Any "translation fault" or "HC died" line is a failure to report. For a full report, after plugging the dock in: - curl -fsSLO https://raw.githubusercontent.com/iconidentify/aurora-linux/refs/tags/sep-7.1.12.aurora2-11.33/tools/aurora-tb/tb-pcie-report + curl -fsSLO https://raw.githubusercontent.com/iconidentify/aurora-linux/refs/tags/sep-7.1.12.aurora2-11.34/tools/aurora-tb/tb-pcie-report sudo sh tb-pcie-report --no-wait 6c. Across suspend: From de993060a29103dd97d9b8accb4f751595b8c2e2 Mon Sep 17 00:00:00 2001 From: Justin Pfister Date: Sat, 3 Oct 2026 22:39:59 -0400 Subject: [PATCH 1734/2680] soc: apple: sep: expand enrollment capture budget to 36 On M2 Pro (T6020 / J414s), biometric template generation covers edge, center, and tip stages that require 16 to 22 captures to reach 100% completion. The previous limit of 12 captures prematurely triggered ENROL_STATUS_TOO_MANY (reported in userspace as enroll-unknown-error) after 11 successful touches. Raise ENROL_MAX_CAPTURES from 12 to 36 and add a warning message if the budget is exhausted before completion. Reported-by: Justin Pfister Signed-off-by: Justin Pfister (cherry picked from commit 7ff4eee64c03807d9b2702746e586ffbbf9ed7f9) --- drivers/soc/apple/sbio.rs | 5 +++++ drivers/soc/apple/sep.rs | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index d5be89f0f1d80..fe3c26f147a10 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1672,6 +1672,11 @@ impl SepData { break None; } if counter >= ENROL_MAX_CAPTURES { + dev_warn!( + self.dev, + "enrol: capture budget exhausted ({} captures); aborting\n", + counter + ); break Some(Err(ENROL_STATUS_TOO_MANY)); } diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 16e178dc6fcd9..a966747471b3b 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -406,7 +406,7 @@ static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_TIMEOUT); static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_ENCLAVE); static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_SENSOR); -const ENROL_MAX_CAPTURES: u32 = 12; +const ENROL_MAX_CAPTURES: u32 = 36; const ENROL_POLL_MS: u32 = 2; From 363707ab258bf0e925a3e484d3e338e6fc9d7a18 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:35:32 -0700 Subject: [PATCH 1735/2680] soc: apple: sep: report enough enrolment stages for a J414s Justin Pfister found that a J414s (MacBook Pro 14" M2 Pro) needs 16 to 22 accepted captures before the enclave reports a complete template, and raised the capture budget to 36 in #17. The reported stage count stayed at 16, and the stage is held one short of the total until completion, so from the 16th capture on the stage stopped rising. libfprint reports a touch only when the stage rises, so fprintd showed no progress for the last several touches of an enrolment that was still going. Report 24 stages: up to 23 accepted captures each raise the stage, with the last stage left for the completed template. An enrolment that completes sooner, as on the M1, still jumps to the total. The two constants have drifted apart before: cdacd961bf73 doubled the stages to 16 for the J414s and left the budget at 12, which is the failure #17 fixed. Assert at build time that the budget stays at least half again the stages, the ratio the driver started with (8 stages, 12 captures). Log an exhausted budget as an error, like every other way an enrolment can end in failure, so it shows up with them in dmesg -l err. Thanks to Justin for tracking down the J414s capture counts. Link: https://github.com/iconidentify/aurora-linux/pull/17 Reported-by: Justin Pfister Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 0ec24b1ab7caef3819e7273e9cd99d22b10b4806) --- drivers/soc/apple/bio.rs | 6 ++++-- drivers/soc/apple/sbio.rs | 2 +- drivers/soc/apple/sep.rs | 4 ++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index 0dd9418708d20..5961eef9e8436 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -53,8 +53,10 @@ enum MatchResult { } // A stage is an accepted capture, not a bucket of SEP's coverage percentage. -// Leave the final stage for the completed template. -pub(crate) const ENROL_STAGES: u32 = 16; +// Leave the final stage for the completed template. libfprint reports a touch +// only when the stage rises, so this must cover the longest enrolment (J414s +// takes up to 22 captures); one that completes sooner jumps to the total. +pub(crate) const ENROL_STAGES: u32 = 24; const TOKEN_LIFETIME_NS: u64 = 10 * 1_000_000_000; diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index fe3c26f147a10..14fa80b527978 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1672,7 +1672,7 @@ impl SepData { break None; } if counter >= ENROL_MAX_CAPTURES { - dev_warn!( + dev_err!( self.dev, "enrol: capture budget exhausted ({} captures); aborting\n", counter diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index a966747471b3b..cc04228c681a6 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -407,6 +407,10 @@ static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_ENCLAVE); static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_SENSOR); const ENROL_MAX_CAPTURES: u32 = 36; +// Partial-contact retries count against the budget too. Keep it at least +// half again the stages, as it was when the driver began (8 stages, 12 +// captures), so a change to one cannot leave the other behind. +static_assert!(ENROL_MAX_CAPTURES >= bio::ENROL_STAGES + bio::ENROL_STAGES / 2); const ENROL_POLL_MS: u32 = 2; From 4f3a2ec77e9bc1b9ab34415091145ab104bcb285 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:47:50 -0700 Subject: [PATCH 1736/2680] soc: apple: sep: give every enrolment capture its own stage The previous commit reported 24 stages and asserted that the capture budget stays at least half again the stage count. Review found that the assert guards only one direction: it rejects raising the stages alone, as cdacd961bf73 did, but not raising the budget alone, as #17 did, which is what left the stage count behind. And with 24 stages and a budget of 36, accepted captures 24 to 35 would again show no progress; 24 is one more than the most a J414s has needed so far, on one Mac. Derive the stage count from the budget instead, so every capture the budget allows can raise the stage and the two cannot drift. An enrolment that completes sooner, as on the M1, still jumps to the total, from a smaller fraction. fprintd-enroll, which aurora-touchid-setup and Omarchy use, prints each touch and draws no bar. Log an exhausted budget at warning level again, like the no-finger timeout: both usually come from how the finger is placed. Include how many captures were accepted, how far the template got and how many were retried, so a report shows whether the enclave needed more captures or the contact kept failing. Link: https://github.com/iconidentify/aurora-linux/pull/17 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit b3289e4618f09238f04076415c56d1fd7053a10b) --- drivers/soc/apple/bio.rs | 6 +++--- drivers/soc/apple/sbio.rs | 12 +++++++++--- drivers/soc/apple/sep.rs | 7 +++---- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index 5961eef9e8436..ad470ef771127 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -54,9 +54,9 @@ enum MatchResult { // A stage is an accepted capture, not a bucket of SEP's coverage percentage. // Leave the final stage for the completed template. libfprint reports a touch -// only when the stage rises, so this must cover the longest enrolment (J414s -// takes up to 22 captures); one that completes sooner jumps to the total. -pub(crate) const ENROL_STAGES: u32 = 24; +// only when the stage rises, so every capture the budget allows gets a stage; +// an enrolment that completes sooner jumps to the total. +pub(crate) const ENROL_STAGES: u32 = crate::ENROL_MAX_CAPTURES; const TOKEN_LIFETIME_NS: u64 = 10 * 1_000_000_000; diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 14fa80b527978..a86934656d14f 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1666,16 +1666,21 @@ impl SepData { let mut enrolment_completed = false; let mut enrolment_persisted = false; let mut identity: Option<[u8; bio::UUID_LEN]> = None; + let mut last_percent: u32 = 0; let outcome = loop { if !bio::enrol_is_live(&self.bio_session.lock(), self.capture_epoch.load(Relaxed)) { break None; } if counter >= ENROL_MAX_CAPTURES { - dev_err!( + let accepted = self.enrol_frames_accepted.load(Relaxed); + dev_warn!( self.dev, - "enrol: capture budget exhausted ({} captures); aborting\n", - counter + "enrol: capture budget exhausted ({} captures: {} accepted, reaching {}%, {} retried); aborting\n", + counter, + accepted, + last_percent, + counter.saturating_sub(accepted) ); break Some(Err(ENROL_STATUS_TOO_MANY)); } @@ -1688,6 +1693,7 @@ impl SepData { has_template, } => { counter = counter.saturating_add(1); + last_percent = percent; let woke = bio::enrol_advance( &mut self.bio_session.lock(), self.capture_epoch.load(Relaxed), diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index cc04228c681a6..602036868733d 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -406,11 +406,10 @@ static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_TIMEOUT); static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_ENCLAVE); static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_SENSOR); +// Captures per enrolment, including retries for partial contact and bad +// frames. A J414s needs up to 22 accepted captures. bio::ENROL_STAGES is +// derived from this, so every capture allowed here can raise the stage. const ENROL_MAX_CAPTURES: u32 = 36; -// Partial-contact retries count against the budget too. Keep it at least -// half again the stages, as it was when the driver began (8 stages, 12 -// captures), so a change to one cannot leave the other behind. -static_assert!(ENROL_MAX_CAPTURES >= bio::ENROL_STAGES + bio::ENROL_STAGES / 2); const ENROL_POLL_MS: u32 = 2; From 3d7e8ca9bb7539ced656d4ef0f55813e99148a3e Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:52:06 -0700 Subject: [PATCH 1737/2680] soc: apple: sep: fix the enrolment budget comments and its final pause Review of the previous commit: - Its comments overstated the stage guarantee. A retry never raises the stage, and an accepted 36th capture that does not complete the template gets stage 35, the same as the 35th, before the run fails. Say what holds: every accepted capture short of the last raises it, and the stages cannot run out before the budget does. - "Up to 22 captures" is what one J414s has needed, not a known limit. - Its message got the margin wrong: 24 stages let the stage rise through 23 accepted captures, one more than the most seen, not 24. - The budget log said "reaching" but printed the last capture's percentage. Keep the highest instead. The last capture in the budget was also followed by a reposition pause, 1.8 s and up to about 7 s if the sensor is slow to idle, before the loop noticed the spent budget and reported the failure. Skip it; a completed enrolment already leaves the loop without one. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 343853c3e861b1cf4c0d99ec5531d5beef13481e) --- drivers/soc/apple/bio.rs | 5 +++-- drivers/soc/apple/sbio.rs | 8 ++++++-- drivers/soc/apple/sep.rs | 4 ++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index ad470ef771127..6b4e1edbe2966 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -54,8 +54,9 @@ enum MatchResult { // A stage is an accepted capture, not a bucket of SEP's coverage percentage. // Leave the final stage for the completed template. libfprint reports a touch -// only when the stage rises, so every capture the budget allows gets a stage; -// an enrolment that completes sooner jumps to the total. +// only when the stage rises, so there is a stage for each capture the budget +// allows: every accepted capture short of the last raises it, and an enrolment +// that completes sooner jumps to the total. pub(crate) const ENROL_STAGES: u32 = crate::ENROL_MAX_CAPTURES; const TOKEN_LIFETIME_NS: u64 = 10 * 1_000_000_000; diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index a86934656d14f..3c88b33d23f55 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1693,7 +1693,7 @@ impl SepData { has_template, } => { counter = counter.saturating_add(1); - last_percent = percent; + last_percent = last_percent.max(percent); let woke = bio::enrol_advance( &mut self.bio_session.lock(), self.capture_epoch.load(Relaxed), @@ -1738,7 +1738,11 @@ impl SepData { ImageOutcome::Failed(status) => break Some(Err(status)), } - self.pace_between_captures(); + // A spent budget ends the run at the top of the loop; don't make + // the person wait out a reposition pause first. + if counter < ENROL_MAX_CAPTURES { + self.pace_between_captures(); + } }; if enrolment_completed { diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 602036868733d..af77b3fb8a5f2 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -407,8 +407,8 @@ static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_ENCLAVE); static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_SENSOR); // Captures per enrolment, including retries for partial contact and bad -// frames. A J414s needs up to 22 accepted captures. bio::ENROL_STAGES is -// derived from this, so every capture allowed here can raise the stage. +// frames. One J414s has needed up to 22. bio::ENROL_STAGES is derived from +// this, so the stages cannot run out before the budget does. const ENROL_MAX_CAPTURES: u32 = 36; const ENROL_POLL_MS: u32 = 2; From 68780b1665cdb0390b63ed2240b8110642b98368 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:10:22 -0700 Subject: [PATCH 1738/2680] tools: aurora-sep: release 11.35 linux-aurora 7.1.12.aurora2-11.35, built from 343853c3e861: - a Touch ID enrolment may take up to 36 captures, enough for the 16 to 22 a J414s needs (iconidentify/aurora-linux#17, Justin Pfister); - one stage per capture the budget allows, so every accepted touch shows progress in fprintd-enroll, and the stage count can no longer drift from the budget; - an exhausted budget logs the accepted, retried and percent counts. Tested on a J293 (MacBook Pro 13" M1) in the lab, without the Touch ID stack: boot, Wi-Fi, Bluetooth, an s2idle cycle and the desktop tests, with no regression against the known-good kernel. Enrolment with this build has not run on a Mac yet; Justin's J414s results in #17 cover the budget. Device trees and the kernel config are unchanged from 11.34. libfprint, aurora-touchid and m1n1-aurora are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit 1caa9fdc37457f3b548258b201b353ae153578d6) --- tools/aurora-sep/install-aurora-sep.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 55baa1b5600da..cea898b05aed9 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -6,7 +6,7 @@ # ... | bash -s -- --uninstall go back to the kernel this Mac had before # ... | bash -s -- --reset-touchid start Touch ID over: new keybag, enrol again # -# Kernel: iconidentify/aurora-linux custom/sep (e508e107a473), aurora-silicon/linux aurora-wip plus the +# Kernel: iconidentify/aurora-linux custom/sep (343853c3e861), aurora-silicon/linux aurora-wip plus the # Secure Enclave (Touch ID) driver, Thunderbolt (#8), the Apple video # decoder (#45), the M2 Max (t6021) profile and the consolidated Touch ID # series (aurora-silicon/linux#69: matching after a reboot on every profile, @@ -71,6 +71,9 @@ # and nothing is sent until a level is chosen. On the M1/M2 Pro and Max it # keeps the Thunderbolt root port's link out of ASPM L1, so unplugging an idle # dock no longer leaves the port dead until reboot. +# 11.35 lets a Touch ID enrolment take up to 36 captures, enough for the 16 to +# 22 an M2 Pro needs (iconidentify/aurora-linux#17, Justin Pfister), and +# reports one stage per capture, so every accepted touch shows progress. # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -99,8 +102,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.34 -TAG=sep-7.1.12.aurora2-11.34 +VERSION=7.1.12.aurora2-11.35 +TAG=sep-7.1.12.aurora2-11.35 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -108,8 +111,8 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst 2d525694a1451c26b5ede7b1e8e63abd2085b03b620b6c54864119fc04c0f0ee" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 2242777b47969d42910a4375872721368eb69015db7a773f3e723d45adf49e36" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst 675547a1d03600c27eeaf149996a8316c8a8d9793cab2890723303fb4b00a3d0" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 7135fb88679696b04b59e2d5d1760d4c5a50f8216a7d5de2ca10e25aa89f4d70" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20261003-1-any.pkg.tar.zst 29b0360fac8c257d754e64bd1b9c33c487eb2595dd3c31e9138d7a476afa3d64" "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" @@ -992,7 +995,7 @@ SAFETY, NON-NEGOTIABLE sudo dmesg | grep -E 'cold init done|link up after|translation fault|HC died' Any "translation fault" or "HC died" line is a failure to report. For a full report, after plugging the dock in: - curl -fsSLO https://raw.githubusercontent.com/iconidentify/aurora-linux/refs/tags/sep-7.1.12.aurora2-11.34/tools/aurora-tb/tb-pcie-report + curl -fsSLO https://raw.githubusercontent.com/iconidentify/aurora-linux/refs/tags/sep-7.1.12.aurora2-11.35/tools/aurora-tb/tb-pcie-report sudo sh tb-pcie-report --no-wait 6c. Across suspend: From c28848df62ba7b42ae9795d72c3e955cb78cb011 Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Mon, 5 Oct 2026 00:20:56 -0700 Subject: [PATCH 1739/2680] tools: aurora-sep: kernel-only, experimental M3 path 11.36 brings the M3 into the unified kernel, but not its m1n1: the M3 Pro's display and GPU handoff needs an m1n1 this release doesn't ship. Without an M3 case the script would install m1n1-aurora on an M3 and rebuild its boot.bin, and on the M3 bring-up Mac it would say "Rebuilding m1n1" while update-m1n1's freeze made that a no-op. On an M3 (t8122, t6030, t6031, t6034) the script now: - says M3 support is experimental and kernel-only; - installs linux-aurora and its headers, but not m1n1-aurora; - writes no /etc/m1n1.conf and never runs update-m1n1; - before pacman's update-m1n1 hook can see the kernel's device trees, makes sure update-m1n1 is frozen: a freeze already in place (the bring-up's install-m3gpu.sh) is left exactly as it is, otherwise it adds its own marked one, judged the way update-m1n1 judges it; - checks the EFI partition's boot.bin has the same sha256 after the install, stops with a report request if not, and says plainly that pacman's "Updating m1n1 image" step did nothing. --uninstall on an M3 keeps its m1n1 and boot.bin, never restores or deletes /etc/default/update-m1n1 and never runs update-m1n1, and lifts only the freeze this script added. M1, M2 and the Neo take the 11.35 path. The one change there: m1n1_update now warns that boot.bin was not rebuilt when update-m1n1 is frozen, instead of announcing a rebuild. tests/test_m3_kernel_only.py covers the M3 board match, the freeze check, adding and lifting the freeze around existing settings and the bring-up's, the boot.bin check, and the frozen message. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit db5ccb1f3592d24b0aac4b1a590ee8fc01a619bf) --- tools/aurora-sep/install-aurora-sep.sh | 158 ++++++++++++++++- tools/aurora-sep/tests/test_m3_kernel_only.py | 164 ++++++++++++++++++ 2 files changed, 314 insertions(+), 8 deletions(-) create mode 100644 tools/aurora-sep/tests/test_m3_kernel_only.py diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index cea898b05aed9..b80aafbda9939 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -89,6 +89,9 @@ # the SEP warm-registration guard and preboot-UUID forwarding from # aurora-silicon/m1n1, and the usb4-N-pcie-adapter alias fallback # (aurora-silicon/m1n1#4). +# M3: experimental and kernel-only. linux-aurora goes on, and m1n1's boot.bin +# stays exactly as it is: no m1n1-aurora, no /etc/m1n1.conf, no update-m1n1 +# run, and a freeze on update-m1n1 unless one is in place already. # On M2 and later the platform hands Linux an already-running Secure # Enclave, and the driver attaches to it with one registration that can only be # sent once per boot. Stock m1n1 asks the enclave for randomness on the way up, @@ -125,6 +128,7 @@ FALLBACK_ID=aurora-sep-previous-kernel # Written only by --read-only; keeps the driver from writing to the enclave. MODPROBE_CONF=/etc/modprobe.d/aurora-sep.conf READ_ONLY=0 +DT=/proc/device-tree say() { printf '\033[1;34m==>\033[0m %s\n' "$*"; } warn() { printf '\033[1;33mwarning:\033[0m %s\n' "$*" >&2; } @@ -184,7 +188,112 @@ boot_chain() { # branch. m1n1-aurora has no T8140 support, so on a Neo this script never # installs it or the stock m1n1 over the one the Mac already boots. is_neo() { - tr '\0' '\n' /dev/null | grep -qx 'apple,t8140' + tr '\0' '\n' <"$DT/compatible" 2>/dev/null | grep -qx 'apple,t8140' +} + +# Every M3 chip: M3 (t8122), M3 Pro (t6030), M3 Max (t6031, t6034). +is_m3() { + tr '\0' '\n' <"$DT/compatible" 2>/dev/null | grep -Eqx 'apple,(t8122|t6030|t6031|t6034)' +} + +# M3 support is experimental in this release and kernel-only. An M3 gets +# linux-aurora, but m1n1's stage 2 (boot.bin: m1n1, device trees and U-Boot) +# stays exactly as it is: the M3 Pro's display and GPU handoff needs an m1n1 +# this release doesn't ship. So on an M3 the script installs no m1n1-aurora, +# writes no /etc/m1n1.conf and never runs update-m1n1. pacman's update-m1n1 +# hook would still rebuild boot.bin when the kernel's device trees arrive, so +# unless something has frozen update-m1n1 already (the M3 bring-up's +# install-m3gpu.sh has), the script adds its own freeze; --uninstall lifts it. +UPDATE_M1N1_CONF=/etc/default/update-m1n1 +M3_FREEZE_BEGIN="# >>> aurora-sep: keep this M3's boot.bin as it is (remove with: install-aurora-sep.sh --uninstall)" +M3_FREEZE_END="# <<< aurora-sep: keep this M3's boot.bin as it is" +M3_FROZEN_BY="" +M3_BOOTBIN_SHA="" + +# Whether update-m1n1 would exit without building, judged the way it judges: +# it sources this file and stops when M1N1_UPDATE_DISABLED is non-empty. +update_m1n1_frozen() { + [[ -f $UPDATE_M1N1_CONF ]] || return 1 + # shellcheck disable=SC2016 # expanded by that sh, not here + env -i PATH="$PATH" sh -c 'set -e; . "$1"; [ -n "${M1N1_UPDATE_DISABLED:-}" ]' _ \ + "$UPDATE_M1N1_CONF" >/dev/null 2>&1 +} + +m3_freeze() { + local tmp + if update_m1n1_frozen; then + M3_FROZEN_BY=already + return 0 + fi + tmp=$(mktemp) + if [[ -f $UPDATE_M1N1_CONF ]]; then cat "$UPDATE_M1N1_CONF" >"$tmp"; fi + printf '%s\nM1N1_UPDATE_DISABLED=1\n%s\n' "$M3_FREEZE_BEGIN" "$M3_FREEZE_END" >>"$tmp" + $sudo install -m 644 "$tmp" "$UPDATE_M1N1_CONF" + rm -f "$tmp" + update_m1n1_frozen || die "could not freeze update-m1n1 in $UPDATE_M1N1_CONF; nothing was installed" + M3_FROZEN_BY=aurora-sep +} + +# Lift only this script's freeze; a file that held nothing else goes. +m3_unfreeze() { + local tmp + [[ -f $UPDATE_M1N1_CONF ]] && grep -qxF "$M3_FREEZE_BEGIN" "$UPDATE_M1N1_CONF" || return 0 + tmp=$(mktemp) + awk -v b="$M3_FREEZE_BEGIN" -v e="$M3_FREEZE_END" ' + $0 == b { skip = 1; next } + skip && $0 == e { skip = 0; next } + !skip' "$UPDATE_M1N1_CONF" >"$tmp" + if grep -q '[^[:space:]]' "$tmp"; then + $sudo install -m 644 "$tmp" "$UPDATE_M1N1_CONF" + else + $sudo rm -f "$UPDATE_M1N1_CONF" + fi + rm -f "$tmp" +} + +# m1n1's boot.bin on the EFI partition. Omarchy mounts that partition readable +# by root only, so look through sudo. +m3_bootbin() { + local target + for target in /boot/efi/m1n1/boot.bin /boot/m1n1/boot.bin; do + if $sudo test -f "$target"; then + echo "$target" + return 0 + fi + done + return 1 +} + +m3_bootbin_sha() { + local target + target=$(m3_bootbin) || return 0 + $sudo sha256sum "$target" | cut -d' ' -f1 +} + +m3_notice() { + warn "M3 support is experimental in $VERSION, and kernel-only: this installs + linux-aurora and leaves m1n1's boot.bin (m1n1, device trees and U-Boot) exactly + as it is, so the kernel boots with the device trees already in it. The M3 Pro's + display and GPU handoff needs an m1n1 that a later release brings; an M3 without + it runs on the boot framebuffer." +} + +# After the install: say plainly what happened to boot.bin, and prove it. +m3_bootbin_report() { + local now + now=$(m3_bootbin_sha) + if [[ -n $M3_BOOTBIN_SHA && $now != "$M3_BOOTBIN_SHA" ]]; then + die "m1n1's boot.bin changed during the install, which it must not on an M3. + Please report it at https://github.com/iconidentify/aurora-linux/issues before rebooting." + fi + if [[ $M3_FROZEN_BY == aurora-sep ]]; then + say "M3: m1n1's boot.bin is unchanged. pacman's \"Updating m1n1 image\" step did nothing: + this script froze update-m1n1 in $UPDATE_M1N1_CONF, so kernel and m1n1 + updates keep boot.bin as it is. --uninstall lifts the freeze." + else + say "M3: m1n1's boot.bin is unchanged. pacman's \"Updating m1n1 image\" step did nothing: + update-m1n1 was already frozen in $UPDATE_M1N1_CONF, and stays that way." + fi } # Boards whose Touch ID support nobody has booted yet. Once a board's device @@ -376,6 +485,13 @@ EOF for target in /boot/m1n1/boot.bin /boot/efi/m1n1/boot.bin; do [[ -f $target && ! -f $STATE/boot.bin.saved ]] && $sudo cp "$target" "$STATE/boot.bin.saved" done + # update-m1n1 exits without a word while this is set; say so rather than + # claim a rebuild. + if update_m1n1_frozen; then + warn "$conf sets M1N1_UPDATE_DISABLED, so m1n1's boot.bin was not rebuilt; + it keeps the m1n1 and device trees it already had" + return 0 + fi say "Rebuilding m1n1 with the aurora device trees" $sudo update-m1n1 || die "update-m1n1 failed; the previous boot.bin is saved in $STATE/boot.bin.saved" } @@ -586,6 +702,7 @@ install_all() { kernel=$(current_kernel) chain=$(boot_chain) if [[ $chain == grub ]]; then boot_space "$kernel"; fi + if is_m3; then m3_notice; fi work=$(mktemp -d) trap 'rm -rf "${work:-}"' EXIT for entry in "${PACKAGES[@]}"; do @@ -594,6 +711,10 @@ install_all() { say "Keeping this MacBook Neo's own m1n1 (m1n1-aurora has no T8140 support)" continue fi + if [[ $file == m1n1-aurora-* ]] && is_m3; then + say "Keeping this M3's own m1n1 (M3 support in $VERSION is kernel-only)" + continue + fi say "Downloading $file" curl -fL --retry 3 --progress-bar -o "$work/$file" "$RELEASE_URL/$file" || die "could not download $file from $TAG. @@ -611,6 +732,11 @@ install_all() { keep_grub_fallback "$kernel" fi pacman -Q "$kernel" | $sudo tee "$STATE/previous-package" >/dev/null + # Before pacman's update-m1n1 hook runs on the kernel's device trees. + if is_m3; then + M3_BOOTBIN_SHA=$(m3_bootbin_sha) + m3_freeze + fi # linux-aurora-headers pulls in pahole, and fprintd below comes from the # repositories. On a Mac whose package database has gone stale, pacman @@ -643,7 +769,12 @@ install_all() { # is not this one -- and those DTBs lack the Touch ID sensor node. pacman's # own hook already ran update-m1n1 during the install above, before this # configuration existed, so run it again now that it is in place. - m1n1_update + # An M3 keeps the boot.bin it has (see is_m3). + if is_m3; then + m3_bootbin_report + else + m1n1_update + fi if [[ $chain == grub ]]; then grub_update fi @@ -679,6 +810,9 @@ uninstall_all() { if is_neo; then m1n1= say "Reinstalling $previous and the stock libfprint; this MacBook Neo keeps its own m1n1" + elif is_m3; then + m1n1= + say "Reinstalling $previous and the stock libfprint; this M3 keeps its m1n1 and boot.bin as they are" else say "Reinstalling $previous, the stock m1n1 and the stock libfprint" fi @@ -686,7 +820,9 @@ uninstall_all() { $sudo pacman -Sy --noconfirm --ask 4 "$previous" "$previous-headers" libfprint $m1n1 # Restore the stock update-m1n1 configuration on either chain before the # rebuild below, so boot.bin goes back to the packaged m1n1 and DTBs. - if [[ $(boot_chain) != grub ]]; then + # An M3's boot.bin and update-m1n1 configuration were never changed; only + # this script's freeze comes off, at the end. + if [[ $(boot_chain) != grub ]] && ! is_m3; then if [[ -f $STATE/update-m1n1.default.saved ]]; then $sudo cp "$STATE/update-m1n1.default.saved" /etc/default/update-m1n1 else @@ -699,17 +835,20 @@ uninstall_all() { [[ -f $STATE/linux-asahi.preset.saved && ! -f /etc/mkinitcpio.d/linux-asahi.preset ]] && $sudo mv "$STATE/linux-asahi.preset.saved" /etc/mkinitcpio.d/linux-asahi.preset [[ -f $STATE/grub.default.saved ]] && $sudo cp "$STATE/grub.default.saved" /etc/default/grub - if [[ -f $STATE/update-m1n1.default.saved ]]; then - $sudo cp "$STATE/update-m1n1.default.saved" /etc/default/update-m1n1 - else - $sudo rm -f /etc/default/update-m1n1 + if ! is_m3; then + if [[ -f $STATE/update-m1n1.default.saved ]]; then + $sudo cp "$STATE/update-m1n1.default.saved" /etc/default/update-m1n1 + else + $sudo rm -f /etc/default/update-m1n1 + fi + $sudo update-m1n1 fi - $sudo update-m1n1 $sudo systemctl disable aurora-sep-fallback-modules.service 2>/dev/null || true $sudo rm -f /etc/systemd/system/aurora-sep-fallback-modules.service $sudo rm -f /etc/grub.d/42_aurora_sep_previous /boot/vmlinuz-aurora-sep-previous /boot/initramfs-aurora-sep-previous.img $sudo grub-mkconfig -o /boot/grub/grub.cfg fi + if is_m3; then m3_unfreeze; fi $sudo rm -f "$MODPROBE_CONF" $sudo rm -rf "$STATE" say "Done. Reboot to run $previous." @@ -1098,6 +1237,9 @@ PROMPT # A reset needs none of the kernel and boot checks; it checks for itself. preflight_needed() { case ${1:-} in --agent-prompt | --reset-touchid) return 1 ;; *) return 0 ;; esac; } +# Tests source this file for its functions only. +if [[ ${AURORA_SEP_SOURCE_ONLY:-} == 1 ]]; then return 0; fi + if preflight_needed "${1:-}"; then preflight; fi case ${1:-} in "") install_all ;; diff --git a/tools/aurora-sep/tests/test_m3_kernel_only.py b/tools/aurora-sep/tests/test_m3_kernel_only.py new file mode 100644 index 0000000000000..38c2c02401999 --- /dev/null +++ b/tools/aurora-sep/tests/test_m3_kernel_only.py @@ -0,0 +1,164 @@ +"""The installer's kernel-only M3 path, exercised on fixture files. + +Each case sources install-aurora-sep.sh for its functions only +(AURORA_SEP_SOURCE_ONLY=1), points the paths it touches at a temporary +directory, and replaces sudo and the EFI-partition lookup. +""" +from pathlib import Path +import os +import shutil +import subprocess +import tempfile +import unittest + +INSTALLER = Path(__file__).resolve().parent.parent / "install-aurora-sep.sh" + +BRINGUP_FREEZE = ( + "# Added by m3-gpu-work/scripts/install-m3gpu.sh: keep the M3 GPU candidate boot.bin.\n" + "# Undo with rollback-m3gpu.sh (then run: sudo update-m1n1).\n" + "M1N1_UPDATE_DISABLED=1\n" +) +# What m1n1_update writes on M1/M2 (11.35). +AURORA_DTBS = ( + "# aurora-sep: build m1n1's stage 2 from the device trees the installed\n" + "DTBS=$(pacman -Qlq linux-aurora 2>/dev/null | grep '/dtbs/[^/]*\\.dtb$'; true)\n" +) +COMPATIBLE = { + "j516s": ["apple,j516s", "apple,t6030"], + "j514s": ["apple,j514s", "apple,t6030"], + "j514c": ["apple,j514c", "apple,t6031"], + "j514m": ["apple,j514m", "apple,t6034"], + "j613": ["apple,j613", "apple,t8122"], + "j314s": ["apple,j314s", "apple,t6000"], + "j293": ["apple,j293", "apple,t8103"], + "j700": ["apple,j700", "apple,t8140"], +} + + +class M3KernelOnlyTest(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.tmp) + self.dt = self.tmp / "dt" + self.dt.mkdir() + self.conf = self.tmp / "update-m1n1" + self.boot = self.tmp / "esp" / "m1n1" / "boot.bin" + self.boot.parent.mkdir(parents=True) + self.boot.write_bytes(b"m1n1+dtbs+u-boot\nchosen.asahi,t6030-gpu=1\n") + # A pacman that owns linux-aurora's device trees, as on a Mac after + # the install: the DTBS= line m1n1_update writes needs one, because + # update-m1n1 sources it under sh's set -e. + self.bin = self.tmp / "bin" + self.bin.mkdir() + (self.bin / "pacman").write_text( + "#!/bin/sh\necho /usr/lib/modules/7.1.12-2-11.36-sep-ARCH/dtbs/t6030-j516s.dtb\n") + (self.bin / "pacman").chmod(0o755) + + def mac(self, board): + (self.dt / "compatible").write_bytes( + b"".join(c.encode() + b"\0" for c in COMPATIBLE[board] + ["apple,arm-platform"]) + ) + + def run_sh(self, body, check=True): + script = f""" +set -euo pipefail +AURORA_SEP_SOURCE_ONLY=1 source '{INSTALLER}' +sudo="" +DT='{self.dt}' +UPDATE_M1N1_CONF='{self.conf}' +m3_bootbin() {{ [[ -f '{self.boot}' ]] && echo '{self.boot}'; }} +{body} +""" + proc = subprocess.run(["bash", "-c", script], capture_output=True, text=True, + env={**os.environ, "NO_COLOR": "1", + "PATH": f"{self.bin}:{os.environ['PATH']}"}) + if check: + self.assertEqual(proc.returncode, 0, proc.stderr + proc.stdout) + return proc + + def frozen(self): + return self.run_sh("update_m1n1_frozen && echo yes || echo no").stdout.strip() + + def test_m3_boards(self): + for board, want in [("j516s", "m3"), ("j514s", "m3"), ("j514c", "m3"), ("j514m", "m3"), + ("j613", "m3"), ("j314s", "no"), ("j293", "no"), ("j700", "no")]: + with self.subTest(board=board): + self.mac(board) + self.assertEqual(self.run_sh("is_m3 && echo m3 || echo no").stdout.strip(), want) + + def test_neo_is_not_m3_and_still_neo(self): + self.mac("j700") + self.assertEqual(self.run_sh("is_neo && echo neo").stdout.strip(), "neo") + + def test_frozen_like_update_m1n1(self): + for text, want in [(BRINGUP_FREEZE, "yes"), ("M1N1_UPDATE_DISABLED=\n", "no"), + ("# M1N1_UPDATE_DISABLED=1\n", "no"), (AURORA_DTBS, "no"), + (AURORA_DTBS + "M1N1_UPDATE_DISABLED=1\n", "yes"), + ("export M1N1_UPDATE_DISABLED=y\n", "yes")]: + with self.subTest(text=text): + self.conf.write_text(text) + self.assertEqual(self.frozen(), want) + self.conf.unlink() + self.assertEqual(self.frozen(), "no") + + def test_freeze_creates_the_file(self): + out = self.run_sh("m3_freeze\necho $M3_FROZEN_BY").stdout.strip() + self.assertEqual(out, "aurora-sep") + self.assertEqual(self.frozen(), "yes") + self.run_sh("m3_unfreeze") + self.assertFalse(self.conf.exists()) + + def test_freeze_keeps_existing_settings(self): + self.conf.write_text(AURORA_DTBS) + self.run_sh("m3_freeze") + self.assertTrue(self.conf.read_text().startswith(AURORA_DTBS)) + self.assertEqual(self.frozen(), "yes") + self.run_sh("m3_unfreeze\nm3_unfreeze") + self.assertEqual(self.conf.read_text(), AURORA_DTBS) + + def test_bringup_freeze_left_alone(self): + self.conf.write_text(BRINGUP_FREEZE) + out = self.run_sh("m3_freeze\necho $M3_FROZEN_BY").stdout.strip() + self.assertEqual(out, "already") + self.assertEqual(self.conf.read_text(), BRINGUP_FREEZE) + self.run_sh("m3_unfreeze") + self.assertEqual(self.conf.read_text(), BRINGUP_FREEZE) + + def test_freeze_twice_is_one_block(self): + self.run_sh("m3_freeze\nm3_freeze") + self.assertEqual(self.conf.read_text().count("M1N1_UPDATE_DISABLED=1"), 1) + + def test_report_unchanged(self): + self.conf.write_text(BRINGUP_FREEZE) + out = self.run_sh('M3_BOOTBIN_SHA=$(m3_bootbin_sha)\nm3_freeze\nm3_bootbin_report').stdout + self.assertIn("boot.bin is unchanged", out) + self.assertIn("already frozen", out) + + def test_report_ours(self): + out = self.run_sh('M3_BOOTBIN_SHA=$(m3_bootbin_sha)\nm3_freeze\nm3_bootbin_report').stdout + self.assertIn("this script froze update-m1n1", out) + + def test_report_catches_a_change(self): + proc = self.run_sh( + f"M3_BOOTBIN_SHA=$(m3_bootbin_sha)\nm3_freeze\nprintf x >>'{self.boot}'\nm3_bootbin_report", + check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("boot.bin changed", proc.stderr) + + def test_m1n1_update_says_when_frozen(self): + # m1n1_update on a frozen non-M3 Mac must not claim a rebuild. + self.conf.write_text("M1N1_UPDATE_DISABLED=1\n") + state = self.tmp / "state" + state.mkdir() + proc = self.run_sh( + f"STATE='{state}'\n" + # m1n1_update writes /etc/default/update-m1n1 itself; point it here. + f"eval \"$(declare -f m1n1_update | command sed 's|conf=/etc/default/update-m1n1|conf={self.conf}|')\"\n" + "m1n1_update", check=False) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertIn("was not rebuilt", proc.stderr) + self.assertNotIn("Rebuilding m1n1", proc.stdout) + + +if __name__ == "__main__": + unittest.main() From df4345bed95a9c9b73d49259e10b1a8934fad9bf Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Mon, 5 Oct 2026 00:15:35 -0700 Subject: [PATCH 1740/2680] tools: aurora-sep: see Limine switched on without omarchy-mac-boot boot_chain only reported Limine when omarchy-mac-boot was installed. A Mac switched to Limine by hand, like the lab M3 Pro (Limine 12.9.1 with UKIs, limine-mkinitcpio-hook's Apple gate active, no omarchy-mac-boot), was treated as a GRUB Mac: the script checked /boot for room for a fallback kernel, and would have added a GRUB fallback entry nothing boots. Also accept Limine's activation marker and defaults (/var/lib/omarchy/limine.enabled and /etc/default/limine), the same test omarchy-mac-limine-active and the hook's limine-apple-gate make. Macs without them are unaffected. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 849cf776156ebd5dc10180c3e157ce3a4f6e880a) --- tools/aurora-sep/install-aurora-sep.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index b80aafbda9939..30f5a24e3fe55 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -172,8 +172,12 @@ version_notice() { } boot_chain() { - # /boot/efi can be readable only by root, so test through sudo. - if pacman -Q omarchy-mac-boot >/dev/null 2>&1 && + # /boot/efi can be readable only by root, so test through sudo. Limine is + # active where omarchy-mac-boot is installed, or where its activation marker + # and defaults exist without it (the test omarchy-mac-limine-active and + # limine-mkinitcpio-hook's Apple gate make), as on Macs switched by hand. + if { pacman -Q omarchy-mac-boot >/dev/null 2>&1 || + [[ -f /var/lib/omarchy/limine.enabled && -f /etc/default/limine ]]; } && { $sudo test -e /boot/EFI/BOOT/BOOTAA64.EFI || $sudo test -e /boot/efi/EFI/BOOT/BOOTAA64.EFI; } && command -v limine >/dev/null; then echo limine From ff749c663f19f232635c77069ce050589c37cde1 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 03:09:20 -0700 Subject: [PATCH 1741/2680] tools: aurora-sep: release 11.36 linux-aurora 7.1.12.aurora2-11.36, built from dbbfb92908ba: one kernel for M1, M2 and M3 (custom/sep 11.35 plus the unified M3 tree). - M3 support, experimental and kernel-only: the installer adds the kernel and leaves m1n1's boot.bin as it is, so an M3 keeps the boot loader it has. - M1 Pro/Max laptops: a second USB-C or Thunderbolt display gets the HDMI port's pipeline while HDMI is idle; a dock display left waiting lights once a pipeline frees, also after sleep; HDMI works after its pipeline was lent out, and a monitor's HPD blink after waking can't hand that pipeline to a waiting dock. - M2 Pro/Max laptops: HDMI is pointed back at its own PHY when a display is plugged in after a dock display used that pipeline. - The display routing decisions and the dock display lifecycle are now separate, tested units, and dual-stream dock wiring is read from the device tree. Tested in the lab: J314s (MacBook Pro 14" M1 Pro) with a Thunderbolt dock display and HDMI: boot, Wi-Fi, Bluetooth, Touch ID, kexec and an s2idle cycle with the displays restored; display hot-plug was tested on the 11.98-11.104 test builds of the same display series. J293 (MacBook Pro 13" M1): the same plus the desktop tests, no regression against 11.35. J516S (MacBook Pro 16" M3 Pro) with its existing boot loader: GPU, always-on processor and desktop tests. libfprint, aurora-touchid and m1n1-aurora are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> (cherry picked from commit d4f243f628e422c86277d38ac0deda192b2fd7ab) --- tools/aurora-sep/install-aurora-sep.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 30f5a24e3fe55..830e0fa3165b7 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -6,7 +6,7 @@ # ... | bash -s -- --uninstall go back to the kernel this Mac had before # ... | bash -s -- --reset-touchid start Touch ID over: new keybag, enrol again # -# Kernel: iconidentify/aurora-linux custom/sep (343853c3e861), aurora-silicon/linux aurora-wip plus the +# Kernel: iconidentify/aurora-linux custom/sep (dbbfb92908ba), aurora-silicon/linux aurora-wip plus the # Secure Enclave (Touch ID) driver, Thunderbolt (#8), the Apple video # decoder (#45), the M2 Max (t6021) profile and the consolidated Touch ID # series (aurora-silicon/linux#69: matching after a reboot on every profile, @@ -74,6 +74,10 @@ # 11.35 lets a Touch ID enrolment take up to 36 captures, enough for the 16 to # 22 an M2 Pro needs (iconidentify/aurora-linux#17, Justin Pfister), and # reports one stage per capture, so every accepted touch shows progress. +# 11.36 is one kernel for M1, M2 and M3: on M1 Pro/Max laptops a second +# USB-C or Thunderbolt display gets the HDMI port's pipeline when HDMI is idle, +# and a dock display left waiting lights once a pipeline frees. M3 support is +# experimental and kernel-only (see is_m3). # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -105,8 +109,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.35 -TAG=sep-7.1.12.aurora2-11.35 +VERSION=7.1.12.aurora2-11.36 +TAG=sep-7.1.12.aurora2-11.36 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -114,8 +118,8 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst 675547a1d03600c27eeaf149996a8316c8a8d9793cab2890723303fb4b00a3d0" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 7135fb88679696b04b59e2d5d1760d4c5a50f8216a7d5de2ca10e25aa89f4d70" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst cb5cb0800fafaaecde2dcb350044bc3afbfc7018ef2f54dd4422bc26c293c9ec" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst e21ab2add5491d404980f87bce168cfd05ee298e1fc0f12815d9ff68d12e6398" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20261003-1-any.pkg.tar.zst 29b0360fac8c257d754e64bd1b9c33c487eb2595dd3c31e9138d7a476afa3d64" "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" @@ -1138,7 +1142,7 @@ SAFETY, NON-NEGOTIABLE sudo dmesg | grep -E 'cold init done|link up after|translation fault|HC died' Any "translation fault" or "HC died" line is a failure to report. For a full report, after plugging the dock in: - curl -fsSLO https://raw.githubusercontent.com/iconidentify/aurora-linux/refs/tags/sep-7.1.12.aurora2-11.35/tools/aurora-tb/tb-pcie-report + curl -fsSLO https://raw.githubusercontent.com/iconidentify/aurora-linux/refs/tags/sep-7.1.12.aurora2-11.36/tools/aurora-tb/tb-pcie-report sudo sh tb-pcie-report --no-wait 6c. Across suspend: From b62675d34e40e76fa91671b597cf0b9d9f529a0c Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 06:54:23 -0700 Subject: [PATCH 1742/2680] tools: aurora-sep: M3 Pro display and GPU handoff, release 11.36.1 11.36 installs every M3 kernel-only, so an M3 Pro runs on the boot framebuffer with software rendering. The display and GPU need m1n1 to hand them over, and m1n1-aurora 1.6.1.aurora5 carries that T6030 handoff (from iconidentify/m3-m1n1), switched on by three chosen.asahi,t6030-* lines in /etc/m1n1.conf. m3_plan now picks one of two paths per M3 before anything is downloaded: - kernel: as in 11.36. boot.bin is left as it is, update-m1n1 is frozen, and the install stops if boot.bin changed. - handoff: an M3 Pro model in M3_HANDOFF_BOARDS (j516s for now) on the macOS 14.8.3 system-firmware stub, or any M3 Pro whose owner asks with --m3-handoff. The installer keeps boot.bin.before- with the macOS restore steps, checks that the package carries the switches, writes them, lifts this script's freeze and the M3 bring-up's (keeping the latter for --uninstall), rebuilds boot.bin and checks the switches are in it. --m3-handoff is refused on an M3 or M3 Max (m1n1 has no handoff for them), off the tested stub, and on anything that isn't an M3. Only an M3 on the handoff path gets aurora5 (M3_M1N1_PACKAGE). M1, M2 and the Neo keep aurora3, so their boot loader doesn't change in this release. --uninstall follows the path recorded in $STATE/m3-mode; an 11.36 install, which recorded none, is kernel-only. --agent-prompt gains an M3 section: what each path should look like, and how a tester's agent qualifies a new M3 Pro model with --m3-handoff, with the owner present and able to start macOS. Also from the earlier installer/m3-path draft: the EFI partition is readable by root only on Omarchy, so the unproven-board backup and m1n1_update's boot.bin.saved now look through sudo (esp_bootbin). tests/test_m3_handoff.py covers the board classes, the plan with and without --m3-handoff, the stub check, both freezes, /etc/m1n1.conf, the package and boot.bin checks and which packages each Mac gets. tests/test_m3_flow.py runs install_all and uninstall_all against a fake pacman, update-m1n1 and EFI partition: a fresh M3 Pro, one upgraded from an 11.36 kernel-only install, the bring-up M3 Pro and back, a --m3-handoff trial, the kernel-only M3s and an M1 Pro. Five mutants (an M3 given aurora3, the freeze not lifted, switches not removed or written too late, the bring-up not restored) each fail them. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit 5c7daa753c4c1600da61c8d98e8529928babc741) --- tools/aurora-sep/install-aurora-sep.sh | 511 +++++++++++++++--- tools/aurora-sep/tests/test_m3_flow.py | 315 +++++++++++ tools/aurora-sep/tests/test_m3_handoff.py | 338 ++++++++++++ tools/aurora-sep/tests/test_m3_kernel_only.py | 4 +- 4 files changed, 1085 insertions(+), 83 deletions(-) create mode 100644 tools/aurora-sep/tests/test_m3_flow.py create mode 100644 tools/aurora-sep/tests/test_m3_handoff.py diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 830e0fa3165b7..7246e8109b89e 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -78,6 +78,12 @@ # USB-C or Thunderbolt display gets the HDMI port's pipeline when HDMI is idle, # and a dock display left waiting lights once a pipeline frees. M3 support is # experimental and kernel-only (see is_m3). +# 11.36.1 changes only this script and adds an M3 m1n1: an M3 Pro model whose +# m1n1 display and GPU handoff has been booted (the 16" J516S for now) gets +# m1n1-aurora with it switched on, and the built-in display and GPU work. +# Every other M3 stays kernel-only, and --m3-handoff lets an M3 Pro owner try +# the handoff on a model not yet on the list (see --agent-prompt). M1, M2 and +# the Neo are unchanged. # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -93,9 +99,12 @@ # the SEP warm-registration guard and preboot-UUID forwarding from # aurora-silicon/m1n1, and the usb4-N-pcie-adapter alias fallback # (aurora-silicon/m1n1#4). -# M3: experimental and kernel-only. linux-aurora goes on, and m1n1's boot.bin -# stays exactly as it is: no m1n1-aurora, no /etc/m1n1.conf, no update-m1n1 -# run, and a freeze on update-m1n1 unless one is in place already. +# M3: experimental. linux-aurora goes on. On an M3 Pro model in +# M3_HANDOFF_BOARDS (or with --m3-handoff), M3_M1N1_PACKAGE replaces m1n1 and +# three chosen.asahi,t6030-* lines in /etc/m1n1.conf switch its display and +# GPU handoff on. Every other M3 keeps m1n1's boot.bin exactly as it is: no +# m1n1-aurora, no update-m1n1 run, and a freeze on update-m1n1 unless one is +# in place already (see m3_plan). # On M2 and later the platform hands Linux an already-running Secure # Enclave, and the driver attaches to it with one registration that can only be # sent once per boot. Stock m1n1 asks the enclave for randomness on the way up, @@ -110,7 +119,7 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. VERSION=7.1.12.aurora2-11.36 -TAG=sep-7.1.12.aurora2-11.36 +TAG=sep-7.1.12.aurora2-11.36.1 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -124,6 +133,9 @@ PACKAGES=( "aurora-touchid-20261003-1-any.pkg.tar.zst 29b0360fac8c257d754e64bd1b9c33c487eb2595dd3c31e9138d7a476afa3d64" "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" ) +# Only for an M3 on the handoff path (see m3_plan), in place of the m1n1-aurora +# above: the same m1n1 plus the T6030 display and GPU handoff. +M3_M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora5-1-aarch64.pkg.tar.zst e06dd92a75fb95cba822eb0a414a6f01f6d5bf9ca4f85749b0ef663922c38294" PINNED="linux-aurora linux-aurora-headers libfprint m1n1-aurora" PIN_BEGIN="# >>> aurora-sep pin (remove with: install-aurora-sep.sh --uninstall)" PIN_END="# <<< aurora-sep pin" @@ -204,19 +216,56 @@ is_m3() { tr '\0' '\n' <"$DT/compatible" 2>/dev/null | grep -Eqx 'apple,(t8122|t6030|t6031|t6034)' } -# M3 support is experimental in this release and kernel-only. An M3 gets -# linux-aurora, but m1n1's stage 2 (boot.bin: m1n1, device trees and U-Boot) -# stays exactly as it is: the M3 Pro's display and GPU handoff needs an m1n1 -# this release doesn't ship. So on an M3 the script installs no m1n1-aurora, -# writes no /etc/m1n1.conf and never runs update-m1n1. pacman's update-m1n1 -# hook would still rebuild boot.bin when the kernel's device trees arrive, so -# unless something has frozen update-m1n1 already (the M3 bring-up's -# install-m3gpu.sh has), the script adds its own freeze; --uninstall lifts it. +is_m3_pro() { + tr '\0' '\n' <"$DT/compatible" 2>/dev/null | grep -qx 'apple,t6030' +} + +# M3 support is experimental. Every M3 gets linux-aurora; what happens to m1n1's +# stage 2 (boot.bin: m1n1, device trees and U-Boot) depends on the model, and +# m3_plan decides it before anything is downloaded (M3_MODE): +# kernel boot.bin stays exactly as it is. No m1n1-aurora, no /etc/m1n1.conf +# change, no update-m1n1 run, and a freeze on update-m1n1 unless one +# is in place already (the M3 bring-up's install-m3gpu.sh has one), +# because pacman's hook would otherwise rebuild boot.bin when the +# kernel's device trees arrive. --uninstall lifts it. +# handoff An M3 Pro model in M3_HANDOFF_BOARDS on the tested stub, or any M3 +# Pro whose owner asked for it with --m3-handoff. It gets +# M3_M1N1_PACKAGE, whose T6030 display and GPU handoff (from +# iconidentify/m3-m1n1) stays off unless boot.bin ends with the +# three M3_SWITCHES lines; update-m1n1 copies chosen.* lines from +# /etc/m1n1.conf into every rebuild, so they survive updates. +# M1, M2 and the Neo keep the m1n1-aurora in PACKAGES: only an M3 on the +# handoff path gets the newer m1n1. UPDATE_M1N1_CONF=/etc/default/update-m1n1 M3_FREEZE_BEGIN="# >>> aurora-sep: keep this M3's boot.bin as it is (remove with: install-aurora-sep.sh --uninstall)" M3_FREEZE_END="# <<< aurora-sep: keep this M3's boot.bin as it is" M3_FROZEN_BY="" M3_BOOTBIN_SHA="" +# Models the handoff has been booted on, by us or by a tester's report. +M3_HANDOFF_BOARDS="j516s" +M3_SWITCHES="chosen.asahi,t6030-gpu=1 chosen.asahi,t6030-dcp=1 chosen.asahi,t6030-dcpext=1" +# The handoff is tested with one macOS system-firmware stub only, 14.8.3 (GPU +# firmware 14.8.3, DCP 14.7), which the Omarchy installer gives every M3. m1n1 +# reports the stub's iBoot as asahi,iboot2-version. +M3_STUB_VERSION=14.8.3 +M3_STUB_IBOOT=iBoot-10151.140.19 +M1N1_CONF=/etc/m1n1.conf +M1N1_CONF_BEGIN="# >>> aurora-sep: M3 Pro display and GPU handoff (remove with: install-aurora-sep.sh --uninstall)" +M1N1_CONF_END="# <<< aurora-sep: M3 Pro display and GPU handoff" +# The M3 bring-up's install-m3gpu.sh froze boot.bin with exactly these lines, +# and left this marker when it created the file. +M3GPU_FREEZE=( + "# Added by m3-gpu-work/scripts/install-m3gpu.sh: keep the M3 GPU candidate boot.bin." + "# Undo with rollback-m3gpu.sh (then run: sudo update-m1n1)." + "M1N1_UPDATE_DISABLED=1" +) +M3GPU_MARKER=/etc/default/.update-m1n1.created-by-m3gpu +# none (not an M3), kernel or handoff; set by m3_plan, kept in $STATE/m3-mode +# for --uninstall. +M3_MODE=none +# Set by --m3-handoff: the owner asks for the handoff on an M3 Pro model that +# isn't in M3_HANDOFF_BOARDS yet. +M3_TRY=0 # Whether update-m1n1 would exit without building, judged the way it judges: # it sources this file and stops when M1N1_UPDATE_DISABLED is non-empty. @@ -242,15 +291,20 @@ m3_freeze() { M3_FROZEN_BY=aurora-sep } +# update-m1n1's configuration without this script's freeze, on stdout. +m3_unfrozen() { + awk -v b="$M3_FREEZE_BEGIN" -v e="$M3_FREEZE_END" ' + $0 == b { skip = 1; next } + skip && $0 == e { skip = 0; next } + !skip' "$UPDATE_M1N1_CONF" +} + # Lift only this script's freeze; a file that held nothing else goes. m3_unfreeze() { local tmp [[ -f $UPDATE_M1N1_CONF ]] && grep -qxF "$M3_FREEZE_BEGIN" "$UPDATE_M1N1_CONF" || return 0 tmp=$(mktemp) - awk -v b="$M3_FREEZE_BEGIN" -v e="$M3_FREEZE_END" ' - $0 == b { skip = 1; next } - skip && $0 == e { skip = 0; next } - !skip' "$UPDATE_M1N1_CONF" >"$tmp" + m3_unfrozen >"$tmp" if grep -q '[^[:space:]]' "$tmp"; then $sudo install -m 644 "$tmp" "$UPDATE_M1N1_CONF" else @@ -259,34 +313,26 @@ m3_unfreeze() { rm -f "$tmp" } -# m1n1's boot.bin on the EFI partition. Omarchy mounts that partition readable -# by root only, so look through sudo. -m3_bootbin() { +# m1n1's boot.bin on the EFI partition, where macOS can reach it. Omarchy +# mounts that partition readable by root only, so look through sudo. +esp_bootbin() { local target for target in /boot/efi/m1n1/boot.bin /boot/m1n1/boot.bin; do - if $sudo test -f "$target"; then - echo "$target" - return 0 - fi + $sudo test -f "$target" || continue + [[ $(findmnt -no FSTYPE --target "${target%/m1n1/boot.bin}") == vfat ]] || continue + echo "$target" + return 0 done return 1 } m3_bootbin_sha() { local target - target=$(m3_bootbin) || return 0 + target=$(esp_bootbin) || return 0 $sudo sha256sum "$target" | cut -d' ' -f1 } -m3_notice() { - warn "M3 support is experimental in $VERSION, and kernel-only: this installs - linux-aurora and leaves m1n1's boot.bin (m1n1, device trees and U-Boot) exactly - as it is, so the kernel boots with the device trees already in it. The M3 Pro's - display and GPU handoff needs an m1n1 that a later release brings; an M3 without - it runs on the boot framebuffer." -} - -# After the install: say plainly what happened to boot.bin, and prove it. +# After a kernel-only install: say plainly what happened to boot.bin, and prove it. m3_bootbin_report() { local now now=$(m3_bootbin_sha) @@ -304,6 +350,202 @@ m3_bootbin_report() { fi } +is_m3_handoff_board() { + local board + board=$(this_board) + is_m3_pro && [[ -n $board && " $M3_HANDOFF_BOARDS " == *" $board "* ]] +} + +# Why this Mac's system-firmware stub isn't the one the handoff is tested +# with; nothing when it is. The installer's stub_info.json names the stub's +# macOS version, and m1n1 reports the stub's iBoot. +m3_stub_problem() { + local iboot bootbin info="" version="" + iboot=$(tr -d '\0' <"$DT/chosen/asahi,iboot2-version" 2>/dev/null) || iboot="" + if bootbin=$(esp_bootbin); then + info=${bootbin%/m1n1/boot.bin}/asahi/stub_info.json + version=$($sudo cat "$info" 2>/dev/null | grep -o '"ProductVersion": *"[^"]*"' | head -1 | + sed 's/.*"\([^"]*\)"$/\1/') || version="" + fi + if [[ -n $version && $version != "$M3_STUB_VERSION" ]]; then + echo "its macOS system-firmware stub is $version ($info)" + elif [[ $iboot != "$M3_STUB_IBOOT"* ]]; then + echo "its system-firmware stub's iBoot is ${iboot:-not reported by m1n1}" + fi + return 0 +} + +# True when a downloaded m1n1-aurora package carries all three handoff switches. +m1n1_pkg_has_handoff() { + local bin s rc=0 + bin=$(mktemp) + bsdtar -xOf "$1" usr/lib/asahi-boot/m1n1.bin >"$bin" 2>/dev/null || rc=1 + for s in asahi,t6030-gpu asahi,t6030-dcp asahi,t6030-dcpext; do + ((rc == 0)) && ! grep -qaF "$s" "$bin" && rc=1 + done + rm -f "$bin" + return "$rc" +} + +# update-m1n1's configuration without the M3 bring-up's freeze, on stdout. +m3gpu_unfrozen() { + awk -v a="${M3GPU_FREEZE[0]}" -v b="${M3GPU_FREEZE[1]}" -v c="${M3GPU_FREEZE[2]}" ' + { line[NR] = $0 } + END { + for (i = 1; i <= NR; i++) { + if (line[i] == a && line[i + 1] == b && line[i + 2] == c) { i += 2; continue } + print line[i] + } + }' "$UPDATE_M1N1_CONF" +} + +# True when something other than the M3 bring-up or this script stops update-m1n1. +update_m1n1_frozen_by_others() { + local tmp rc=1 + [[ -f $UPDATE_M1N1_CONF ]] || return 1 + tmp=$(mktemp) + m3gpu_unfrozen | awk -v b="$M3_FREEZE_BEGIN" -v e="$M3_FREEZE_END" ' + $0 == b { skip = 1; next } + skip && $0 == e { skip = 0; next } + !skip' >"$tmp" + grep -Eq '^[^#]*M1N1_UPDATE_DISABLED=' "$tmp" && rc=0 + rm -f "$tmp" + return "$rc" +} + +# Lift the M3 bring-up's freeze: only its own three lines, and its marker. +# The original is kept so --uninstall can put the freeze back. +m3gpu_unfreeze() { + local tmp + [[ -f $UPDATE_M1N1_CONF ]] && grep -qxF "${M3GPU_FREEZE[0]}" "$UPDATE_M1N1_CONF" || return 0 + [[ -f $STATE/update-m1n1.m3gpu.saved ]] || $sudo cp -p "$UPDATE_M1N1_CONF" "$STATE/update-m1n1.m3gpu.saved" + tmp=$(mktemp) + m3gpu_unfrozen >"$tmp" + if [[ -e $M3GPU_MARKER ]]; then + $sudo touch "$STATE/m3gpu-marker.saved" + $sudo rm -f "$M3GPU_MARKER" + fi + if grep -q '[^[:space:]]' "$tmp"; then + $sudo install -m 644 "$tmp" "$UPDATE_M1N1_CONF" + else + $sudo rm -f "$UPDATE_M1N1_CONF" + fi + rm -f "$tmp" + say "Lifted the M3 bring-up's freeze on update-m1n1 (from install-m3gpu.sh)" +} + +# /etc/m1n1.conf without this script's switch block, on stdout. +m1n1_conf_without_switches() { + [[ -f $M1N1_CONF ]] || return 0 + awk -v b="$M1N1_CONF_BEGIN" -v e="$M1N1_CONF_END" ' + $0 == b { skip = 1; next } + skip && $0 == e { skip = 0; next } + !skip' "$M1N1_CONF" +} + +m3_switches_write() { + local tmp s + tmp=$(mktemp) + m1n1_conf_without_switches >"$tmp" + { + echo "$M1N1_CONF_BEGIN" + for s in $M3_SWITCHES; do echo "$s"; done + echo "$M1N1_CONF_END" + } >>"$tmp" + $sudo install -m 644 "$tmp" "$M1N1_CONF" + rm -f "$tmp" +} + +m3_switches_remove() { + local tmp + [[ -f $M1N1_CONF ]] && grep -qxF "$M1N1_CONF_BEGIN" "$M1N1_CONF" || return 0 + tmp=$(mktemp) + m1n1_conf_without_switches >"$tmp" + if grep -q '[^[:space:]]' "$tmp"; then + $sudo install -m 644 "$tmp" "$M1N1_CONF" + else + $sudo rm -f "$M1N1_CONF" + fi + rm -f "$tmp" +} + +# Decide the M3 path before anything is downloaded, so a Mac this release can't +# set up as asked stops with nothing changed. +m3_plan() { + local board problem + M3_MODE=none + if ! is_m3; then + ((M3_TRY == 0)) || die "--m3-handoff is for an M3 Pro, and this Mac isn't an M3. Nothing was installed." + return 0 + fi + board=$(this_board) + M3_MODE=kernel + if ! is_m3_pro; then + ((M3_TRY == 0)) || die "--m3-handoff is for an M3 Pro (t6030); this M3 ($board) has no + display and GPU handoff in m1n1 yet. Nothing was installed." + say "M3 ($board): installing the kernel only. m1n1 has no display and GPU handoff for this chip + yet, so boot.bin stays as it is and the display runs on the boot framebuffer." + return 0 + fi + if ! is_m3_handoff_board && ((M3_TRY == 0)); then + say "M3 Pro ($board): installing the kernel only, and boot.bin stays as it is. The display and + GPU handoff in m1n1 hasn't been booted on this model yet. To try it (it replaces this + Mac's boot loader), see the M3 section of: bash -s -- --agent-prompt" + return 0 + fi + problem=$(m3_stub_problem) + if [[ -n $problem ]]; then + ((M3_TRY == 0)) || die "--m3-handoff: the handoff is only tested with the macOS $M3_STUB_VERSION + system-firmware stub, and this Mac differs: $problem. Nothing was installed." + warn "the M3 Pro display and GPU handoff is only tested with the macOS $M3_STUB_VERSION + system-firmware stub, and this Mac differs: $problem. Installing the kernel only; + boot.bin stays as it is." + return 0 + fi + if update_m1n1_frozen_by_others; then + die "$UPDATE_M1N1_CONF sets M1N1_UPDATE_DISABLED, and not from this script or the M3 + bring-up's install-m3gpu.sh. Switching the M3 Pro handoff on needs m1n1's boot.bin rebuilt. + Remove that line and run this again. Nothing was installed." + fi + M3_MODE=handoff + if is_m3_handoff_board; then + say "M3 Pro ($board, macOS $M3_STUB_VERSION stub): installing m1n1 with the display and GPU handoff" + else + warn "--m3-handoff: trying m1n1's display and GPU handoff on an M3 Pro model ($board) nobody + has booted it on yet. This replaces the Mac's boot loader; the steps to put the old one + back from macOS follow." + fi +} + +# After update-m1n1, boot.bin must end with the switch lines. update-m1n1 +# appends /etc/m1n1.conf's lines straight after the gzipped U-Boot, so the +# first one has no newline in front: look for the block, not whole lines. +m3_verify_bootbin() { + local target tail block + target=$(esp_bootbin) || die "could not find m1n1's boot.bin to check the M3 Pro switches in" + block=$(tr ' ' '\n' <<<"$M3_SWITCHES") + tail=$($sudo tail -c 1024 "$target" | tr -d '\0') + [[ $tail == *"$block"* ]] || + die "the rebuilt $target does not carry the M3 Pro switch lines; the previous one is saved in $STATE/boot.bin.saved" + say "m1n1's boot.bin carries the M3 Pro handoff switches" +} + +# --uninstall on a bring-up Mac: update-m1n1 has just rebuilt boot.bin from the +# stock m1n1, which has no handoff. Put back the boot.bin the bring-up froze, and +# the freeze. +m3_restore_bringup() { + local target + [[ -f $STATE/update-m1n1.m3gpu.saved ]] || return 0 + $sudo cp -p "$STATE/update-m1n1.m3gpu.saved" "$UPDATE_M1N1_CONF" + if [[ -f $STATE/m3gpu-marker.saved ]]; then $sudo touch "$M3GPU_MARKER"; fi + if [[ -f $STATE/boot.bin.saved ]] && target=$(esp_bootbin); then + $sudo cp "$STATE/boot.bin.saved" "$target" + say "Restored the M3 bring-up's boot.bin and its freeze on update-m1n1" + else + warn "restored the M3 bring-up's freeze on update-m1n1, but not its boot.bin (no saved copy)" + fi +} + # Boards whose Touch ID support nobody has booted yet. Once a board's device # tree names the enclave, m1n1 needs two boot manifests from the platform for # it, and a board without them stops in m1n1 before any boot entry. Every @@ -311,24 +553,28 @@ m3_bootbin_report() { UNPROVEN_SEP_BOARDS="j314c j316c j413" this_board() { - tr '\0' '\n' /dev/null | sed -n '1s/^apple,//p' + tr '\0' '\n' <"$DT/compatible" 2>/dev/null | sed -n '1s/^apple,//p' } # Before anything rebuilds boot.bin on such a board, keep the one this boot # came up on next to it on the EFI partition, where macOS can reach it, and # say how to put it back. A later run keeps the first copy. unproven_board_backup() { - local board target keep uuid + local board board=$(this_board) [[ -n $board && " $UNPROVEN_SEP_BOARDS " == *" $board "* ]] || return 0 - for target in /boot/efi/m1n1/boot.bin /boot/m1n1/boot.bin; do - [[ -f $target && $(findmnt -no FSTYPE --target "$target") == vfat ]] || continue + keep_bootbin_on_esp "Touch ID on this Mac model ($board) is new in $VERSION, and nobody has + booted it on this model yet." +} + +keep_bootbin_on_esp() { + local why=$1 target keep uuid + if target=$(esp_bootbin); then keep=$target.before-$VERSION - [[ -f $keep ]] || $sudo cp "$target" "$keep" || + $sudo test -f "$keep" || $sudo cp "$target" "$keep" || die "could not keep a copy of $target; nothing was installed" - uuid=$(findmnt -no PARTUUID --target "$target") - warn "Touch ID on this Mac model ($board) is new in $VERSION, and nobody has - booted it on this model yet. If the Mac stops in m1n1 after this install (m1n1 + uuid=$(findmnt -no PARTUUID --target "${target%/m1n1/boot.bin}") + warn "$why If the Mac stops in m1n1 after this install (m1n1 text on screen, often \"No valid payload found\", and no boot menu), put the boot loader it booted with back from macOS: 1. Hold the power button until the Mac turns off, then press and hold it @@ -338,11 +584,11 @@ unproven_board_backup() { Find the partition whose UUID is $uuid (any case) with: diskutil info diskNsM | grep -i 'partition uuid' 3. sudo diskutil mount diskNsM (it prints the /Volumes path) - 4. cp '/m1n1/boot.bin.before-$VERSION' '/m1n1/boot.bin' + 4. cp -X '/m1n1/boot.bin.before-$VERSION' '/m1n1/boot.bin' The copy is kept at $keep. Either way, please report it at https://github.com/iconidentify/aurora-linux/issues" return 0 - done + fi die "could not find m1n1's boot.bin on the EFI partition to keep a copy of; nothing was installed" } @@ -467,7 +713,7 @@ EOF # version number, kept for the fallback entry) without the Touch ID sensor node. # Point it at whatever directory linux-aurora owns, now and after its updates. m1n1_update() { - local target conf=/etc/default/update-m1n1 tmp + local target conf=$UPDATE_M1N1_CONF tmp [[ -f $conf && ! -f $STATE/update-m1n1.default.saved ]] && $sudo cp "$conf" "$STATE/update-m1n1.default.saved" # Replace only the DTBS setting and keep the rest of the file: a MacBook @@ -490,9 +736,9 @@ DTBS=$(pacman -Qlq linux-aurora 2>/dev/null | grep '/dtbs/[^/]*\.dtb$'; true) EOF $sudo install -m 644 "$tmp" "$conf" rm -f "$tmp" - for target in /boot/m1n1/boot.bin /boot/efi/m1n1/boot.bin; do - [[ -f $target && ! -f $STATE/boot.bin.saved ]] && $sudo cp "$target" "$STATE/boot.bin.saved" - done + if [[ ! -f $STATE/boot.bin.saved ]] && target=$(esp_bootbin); then + $sudo cp "$target" "$STATE/boot.bin.saved" + fi # update-m1n1 exits without a word while this is set; say so rather than # claim a rebuild. if update_m1n1_frozen; then @@ -702,27 +948,36 @@ neo_radio_notice() { - Bluetooth is off by default in this release." } +# The packages this Mac gets, one "file sha256" per line: PACKAGES, except +# that a Neo keeps its own m1n1, and an M3 gets M3_M1N1_PACKAGE on the handoff +# path and no m1n1 otherwise. Needs m3_plan first. +packages_for_this_mac() { + local entry + for entry in "${PACKAGES[@]}"; do + if [[ $entry == m1n1-aurora-* ]] && { is_neo || [[ $M3_MODE != none ]]; }; then continue; fi + echo "$entry" + done + if [[ $M3_MODE == handoff ]]; then echo "$M3_M1N1_PACKAGE"; fi + return 0 +} + install_all() { local entry file sha kernel chain + local -a entries version_notice sep_write_notice ane_dkms_notice kernel=$(current_kernel) chain=$(boot_chain) if [[ $chain == grub ]]; then boot_space "$kernel"; fi - if is_m3; then m3_notice; fi + m3_plan work=$(mktemp -d) trap 'rm -rf "${work:-}"' EXIT - for entry in "${PACKAGES[@]}"; do + if is_neo; then say "Keeping this MacBook Neo's own m1n1 (m1n1-aurora has no T8140 support)"; fi + if [[ $M3_MODE == kernel ]]; then say "Keeping this M3's own m1n1 and boot.bin"; fi + mapfile -t entries < <(packages_for_this_mac) + for entry in "${entries[@]}"; do read -r file sha <<<"$entry" - if [[ $file == m1n1-aurora-* ]] && is_neo; then - say "Keeping this MacBook Neo's own m1n1 (m1n1-aurora has no T8140 support)" - continue - fi - if [[ $file == m1n1-aurora-* ]] && is_m3; then - say "Keeping this M3's own m1n1 (M3 support in $VERSION is kernel-only)" - continue - fi say "Downloading $file" curl -fL --retry 3 --progress-bar -o "$work/$file" "$RELEASE_URL/$file" || die "could not download $file from $TAG. @@ -731,20 +986,33 @@ install_all() { Please report it with the file name above." [[ $(sha256sum "$work/$file" | cut -d' ' -f1) == "$sha" ]] || die "$file does not match its published checksum" done + if [[ $M3_MODE == handoff ]] && ! m1n1_pkg_has_handoff "$work/${M3_M1N1_PACKAGE%% *}"; then + die "this release's M3 m1n1 has no M3 Pro display and GPU handoff, so it can't switch it + on. Nothing was installed. Please report it at https://github.com/iconidentify/aurora-linux/issues" + fi snapshot "aurora-sep $VERSION" # Before pacman's update-m1n1 hook rebuilds boot.bin below. is_neo || unproven_board_backup + if [[ $M3_MODE == handoff ]]; then + keep_bootbin_on_esp "This replaces the boot loader of this M3 Pro ($(this_board)) with + m1n1-aurora and its display and GPU handoff." + fi $sudo install -d "$STATE" if [[ ! -f $STATE/previous && $chain == grub ]]; then keep_grub_fallback "$kernel" fi pacman -Q "$kernel" | $sudo tee "$STATE/previous-package" >/dev/null + if [[ $M3_MODE != none ]]; then echo "$M3_MODE" | $sudo tee "$STATE/m3-mode" >/dev/null; fi # Before pacman's update-m1n1 hook runs on the kernel's device trees. - if is_m3; then - M3_BOOTBIN_SHA=$(m3_bootbin_sha) - m3_freeze - fi + case $M3_MODE in + kernel) + M3_BOOTBIN_SHA=$(m3_bootbin_sha) + m3_freeze + ;; + # So the hook's rebuild, where it runs, already carries the switches. + handoff) m3_switches_write ;; + esac # linux-aurora-headers pulls in pahole, and fprintd below comes from the # repositories. On a Mac whose package database has gone stale, pacman @@ -777,12 +1045,19 @@ install_all() { # is not this one -- and those DTBs lack the Touch ID sensor node. pacman's # own hook already ran update-m1n1 during the install above, before this # configuration existed, so run it again now that it is in place. - # An M3 keeps the boot.bin it has (see is_m3). - if is_m3; then - m3_bootbin_report - else - m1n1_update - fi + # A kernel-only M3 keeps the boot.bin it has. On the handoff path, the + # freezes kept the hook from rebuilding boot.bin before the new m1n1 was in + # place; lift them only now (see m3_plan). + case $M3_MODE in + kernel) m3_bootbin_report ;; + handoff) + m3gpu_unfreeze + m3_unfreeze + m1n1_update + m3_verify_bootbin + ;; + *) m1n1_update ;; + esac if [[ $chain == grub ]]; then grub_update fi @@ -808,8 +1083,14 @@ install_all() { } uninstall_all() { - local previous=linux-asahi + local previous=linux-asahi m3_mode=none [[ -f $STATE/previous-package ]] && read -r previous _ <"$STATE/previous-package" + # 11.36 kept no record: every M3 it installed was kernel-only. + if [[ -f $STATE/m3-mode ]]; then + read -r m3_mode _ <"$STATE/m3-mode" + elif is_m3; then + m3_mode=kernel + fi command -v aurora-touchid-setup >/dev/null && aurora-touchid-setup --remove || true $sudo systemctl disable apple-sep.path apple-sep.service 2>/dev/null || true remove_pin @@ -818,23 +1099,25 @@ uninstall_all() { if is_neo; then m1n1= say "Reinstalling $previous and the stock libfprint; this MacBook Neo keeps its own m1n1" - elif is_m3; then + elif [[ $m3_mode == kernel ]]; then m1n1= say "Reinstalling $previous and the stock libfprint; this M3 keeps its m1n1 and boot.bin as they are" else say "Reinstalling $previous, the stock m1n1 and the stock libfprint" fi $sudo pacman -Rdd --noconfirm aurora-touchid 2>/dev/null || true + # The stock m1n1 has no M3 handoff; drop the switches before its rebuild. + m3_switches_remove $sudo pacman -Sy --noconfirm --ask 4 "$previous" "$previous-headers" libfprint $m1n1 # Restore the stock update-m1n1 configuration on either chain before the # rebuild below, so boot.bin goes back to the packaged m1n1 and DTBs. - # An M3's boot.bin and update-m1n1 configuration were never changed; only - # this script's freeze comes off, at the end. - if [[ $(boot_chain) != grub ]] && ! is_m3; then + # A kernel-only M3's boot.bin and update-m1n1 configuration were never + # changed; only this script's freeze comes off, at the end. + if [[ $(boot_chain) != grub && $m3_mode != kernel ]]; then if [[ -f $STATE/update-m1n1.default.saved ]]; then - $sudo cp "$STATE/update-m1n1.default.saved" /etc/default/update-m1n1 + $sudo cp "$STATE/update-m1n1.default.saved" "$UPDATE_M1N1_CONF" else - $sudo rm -f /etc/default/update-m1n1 + $sudo rm -f "$UPDATE_M1N1_CONF" fi $sudo update-m1n1 || warn "update-m1n1 failed; boot.bin still has the aurora m1n1" fi @@ -843,11 +1126,11 @@ uninstall_all() { [[ -f $STATE/linux-asahi.preset.saved && ! -f /etc/mkinitcpio.d/linux-asahi.preset ]] && $sudo mv "$STATE/linux-asahi.preset.saved" /etc/mkinitcpio.d/linux-asahi.preset [[ -f $STATE/grub.default.saved ]] && $sudo cp "$STATE/grub.default.saved" /etc/default/grub - if ! is_m3; then + if [[ $m3_mode != kernel ]]; then if [[ -f $STATE/update-m1n1.default.saved ]]; then - $sudo cp "$STATE/update-m1n1.default.saved" /etc/default/update-m1n1 + $sudo cp "$STATE/update-m1n1.default.saved" "$UPDATE_M1N1_CONF" else - $sudo rm -f /etc/default/update-m1n1 + $sudo rm -f "$UPDATE_M1N1_CONF" fi $sudo update-m1n1 fi @@ -856,7 +1139,10 @@ uninstall_all() { $sudo rm -f /etc/grub.d/42_aurora_sep_previous /boot/vmlinuz-aurora-sep-previous /boot/initramfs-aurora-sep-previous.img $sudo grub-mkconfig -o /boot/grub/grub.cfg fi - if is_m3; then m3_unfreeze; fi + case $m3_mode in + kernel) m3_unfreeze ;; + handoff) m3_restore_bringup ;; + esac $sudo rm -f "$MODPROBE_CONF" $sudo rm -rf "$STATE" say "Done. Reboot to run $previous." @@ -1033,6 +1319,8 @@ SAFETY, NON-NEGOTIABLE normally. If you see the "couldn't get enough random bytes" line on any machine, the stock m1n1 is still in boot.bin - say so and stop, because every later result is then measuring the wrong thing. + On an M3 installed kernel-only (step 11), the boot loader is the one the Mac + already had, so this step does not apply: say "M3, kernel-only" instead. 3. DRIVER ATTACH The driver is already loaded at boot by apple-sep.service; do not reload @@ -1142,7 +1430,7 @@ SAFETY, NON-NEGOTIABLE sudo dmesg | grep -E 'cold init done|link up after|translation fault|HC died' Any "translation fault" or "HC died" line is a failure to report. For a full report, after plugging the dock in: - curl -fsSLO https://raw.githubusercontent.com/iconidentify/aurora-linux/refs/tags/sep-7.1.12.aurora2-11.36/tools/aurora-tb/tb-pcie-report + curl -fsSLO https://raw.githubusercontent.com/iconidentify/aurora-linux/refs/tags/sep-7.1.12.aurora2-11.36.1/tools/aurora-tb/tb-pcie-report sudo sh tb-pcie-report --no-wait 6c. Across suspend: @@ -1210,6 +1498,58 @@ SAFETY, NON-NEGOTIABLE Report "not tested" honestly rather than guessing, for any of the above. +11. M3 (M3, M3 PRO, M3 MAX) ONLY + Every M3 is experimental. The installer took one of two paths and said + which near the start; the file below records it too: + cat /var/lib/aurora-sep/m3-mode + - kernel: the kernel only. m1n1's boot.bin was left exactly as it was, + and the install ended with "M3: m1n1's boot.bin is unchanged". Expect + the desktop on the boot framebuffer (one fixed resolution, software + rendering). + - handoff: an M3 Pro model whose m1n1 display and GPU handoff has been + booted before. The installer replaced boot.bin, kept the old one as + m1n1/boot.bin.before- on the EFI partition, and printed the + macOS steps to put it back. Expect the built-in display at its native + resolution and the GPU in use. + Check and quote: + tr -d '\0' < /proc/device-tree/compatible; echo + ls /sys/class/drm/ + sudo dmesg | grep -iE 'asahi.*(gpu|firmware)|apple-dcp|t6030' | head -20 + ls /proc/device-tree/soc/usb4-pcie-tunnel-0/pcie@730000000/pci@0,0/apple,tunable 2>&1 + Then: does it reach the desktop, and do Wi-Fi, Bluetooth, keyboard, + trackpad, audio and one suspend/resume work? With a Thunderbolt dock, do + its display and its PCIe devices (Ethernet, storage) come up? Report each + one, including "not tested". + + ADDING AN M3 PRO MODEL TO THE HANDOFF + Only an M3 Pro (apple,t6030: J514S 14", J516S 16") can try the handoff; + m1n1 has none for the M3 or M3 Max yet, and the installer refuses there. + The installer also refuses unless the Mac has the macOS 14.8.3 + system-firmware stub that the Omarchy installer gives every M3. + This REPLACES THE MAC'S BOOT LOADER. If the new one does not start, the + Mac stops before any boot menu, and the only way back is from macOS on + the same Mac. So: + - Do it only with the owner's explicit agreement, and only if the owner + can start macOS on this Mac (hold the power button at startup). + - Keep the restore steps the installer prints: they name this Mac's EFI + partition and the saved boot.bin.before-. + - Do not try it unattended or over SSH alone: someone must be at the Mac + for the first boot. + Then, with the owner watching: + curl -fsSL https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh | bash -s -- --m3-handoff + and reboot. Report one of: + - It boots to the desktop on the built-in display at native resolution: + give the checks above, and again after one suspend/resume. That + report is what adds the model to the installer's list. + - It stops in m1n1 (text on screen, no boot menu) or stays black: the + owner puts the old boot.bin back from macOS with the printed steps. + Report exactly what the screen showed and when. + - It boots, but the display or GPU is wrong: collect + sudo journalctl -b -k | grep -iE 'dcp|asahi|t6030|m1n1' + over SSH if you can, then run the installer with --uninstall, which + puts the stock m1n1 back, and reboot. + Title such a report " (): M3 handoff works" or "... fails". + HOW TO REPORT Open one issue per Mac at https://github.com/iconidentify/aurora-linux/issues (not on omacom/linux#7 any more), with the first line of the report as its @@ -1223,6 +1563,7 @@ HOW TO REPORT **m1n1 guard:** fired / did not fire / stock m1n1 still installed **Touch ID:** enrol , verify , verify after reboot **Thunderbolt:** on port <1 / 2 / both> + **M3 path:** not an M3 / kernel / handoff / handoff with --m3-handoff ## What worked ## What did not @@ -1248,6 +1589,16 @@ preflight_needed() { case ${1:-} in --agent-prompt | --reset-touchid) return 1 ; # Tests source this file for its functions only. if [[ ${AURORA_SEP_SOURCE_ONLY:-} == 1 ]]; then return 0; fi +# --m3-handoff goes with an install, alone or with --read-only. +args=() +for a in "$@"; do + if [[ $a == --m3-handoff ]]; then M3_TRY=1; else args+=("$a"); fi +done +set -- "${args[@]}" +if ((M3_TRY)) && [[ -n ${1:-} && $1 != --read-only ]]; then + die "--m3-handoff goes with an install (alone or with --read-only), not with $1" +fi + if preflight_needed "${1:-}"; then preflight; fi case ${1:-} in "") install_all ;; @@ -1255,5 +1606,5 @@ case ${1:-} in --uninstall) uninstall_all ;; --reset-touchid) shift; reset_touchid "$@" ;; --agent-prompt) t=$(newer_release); [[ -n $t ]] && warn "the current release is $t; this plan is for $TAG"; agent_prompt ;; - *) die "unknown option $1 (--read-only, --uninstall, --reset-touchid or --agent-prompt)" ;; + *) die "unknown option $1 (--read-only, --uninstall, --reset-touchid, --agent-prompt or --m3-handoff)" ;; esac diff --git a/tools/aurora-sep/tests/test_m3_flow.py b/tools/aurora-sep/tests/test_m3_flow.py new file mode 100644 index 0000000000000..fac251c13a620 --- /dev/null +++ b/tools/aurora-sep/tests/test_m3_flow.py @@ -0,0 +1,315 @@ +"""install_all and uninstall_all end to end, against a fake Mac. + +The real functions run; pacman, update-m1n1, curl, findmnt and systemctl are +small stubs that record what happened. The fake update-m1n1 behaves like the +real one where it matters here: it honours M1N1_UPDATE_DISABLED, builds +boot.bin from the installed m1n1 and device trees, and appends /etc/m1n1.conf's +chosen.* lines. pacman -U and the m1n1/kernel installs run it, as the hooks do. +""" +from pathlib import Path +import hashlib +import os +import shutil +import subprocess +import tempfile +import unittest + +INSTALLER = Path(__file__).resolve().parent.parent / "install-aurora-sep.sh" + +BOARDS = { + "j516s": ["apple,j516s", "apple,t6030", "apple,arm-platform"], + "j514s": ["apple,j514s", "apple,t6030", "apple,arm-platform"], + "j613": ["apple,j613", "apple,t8122", "apple,arm-platform"], + "j314s": ["apple,j314s", "apple,t6000", "apple,arm-platform"], +} +SWITCHES = b"chosen.asahi,t6030-gpu=1\nchosen.asahi,t6030-dcp=1\nchosen.asahi,t6030-dcpext=1\n" +BRINGUP_FREEZE = ( + "# Added by m3-gpu-work/scripts/install-m3gpu.sh: keep the M3 GPU candidate boot.bin.\n" + "# Undo with rollback-m3gpu.sh (then run: sudo update-m1n1).\n" + "M1N1_UPDATE_DISABLED=1\n" +) +OUR_FREEZE = ( + "# >>> aurora-sep: keep this M3's boot.bin as it is (remove with: install-aurora-sep.sh --uninstall)\n" + "M1N1_UPDATE_DISABLED=1\n" + "# <<< aurora-sep: keep this M3's boot.bin as it is\n" +) +AURORA3 = "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst" +AURORA5 = "m1n1-aurora-1.6.1.aurora5-1-aarch64.pkg.tar.zst" +OTHERS = [ + "linux-aurora-7.1.12.aurora2-11.36-aarch64.pkg.tar.zst", + "linux-aurora-headers-7.1.12.aurora2-11.36-aarch64.pkg.tar.zst", + "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst", + "aurora-touchid-20261003-1-any.pkg.tar.zst", +] + +PACMAN = r"""#!/bin/bash +echo "pacman $*" >>"$FAKE/log" +op=$1; shift +case $op in + -Q) + for p in "$@"; do + case $p in linux-aurora | linux-asahi) grep -qx "$p" "$FAKE/installed" || exit 1 ;; esac + echo "$p 1-1" + done ;; + -Qq) exit 1 ;; + -Qlq) + [[ $1 == linux-aurora ]] && grep -qx linux-aurora "$FAKE/installed" && + echo /usr/lib/modules/7.1.12-2-11.36-sep-ARCH/dtbs/fake.dtb + exit 0 ;; + -U) + for f in "$@"; do + [[ $f == -* || $f == 4 ]] && continue + b=$(basename "$f") + case $b in + m1n1-aurora-*) echo "${b%-aarch64.pkg.tar.zst}" >"$FAKE/m1n1" ;; + linux-aurora-headers-*) ;; + linux-aurora-*) sed -i '/^linux-asahi$/d' "$FAKE/installed"; echo linux-aurora >>"$FAKE/installed" ;; + esac + done + update-m1n1 hook ;; + -S | -Sy) + hook=0 + for p in "$@"; do + case $p in + m1n1) echo m1n1-stock >"$FAKE/m1n1"; hook=1 ;; + linux-asahi) sed -i '/^linux-aurora$/d' "$FAKE/installed"; echo linux-asahi >>"$FAKE/installed"; hook=1 ;; + esac + done + if ((hook)); then update-m1n1 hook; fi ;; +esac +exit 0 +""" + +UPDATE_M1N1 = r"""#!/bin/bash +echo "update-m1n1 $*" >>"$FAKE/log" +if [[ -f $FAKE_UPDATE_CONF ]] && + bash -c 'set -e; . "$1"; [ -n "${M1N1_UPDATE_DISABLED:-}" ]' _ "$FAKE_UPDATE_CONF" 2>/dev/null; then + echo "update-m1n1 frozen" >>"$FAKE/log" + exit 0 +fi +dtbs=$(bash -c 'set -e; DTBS=; [ -f "$1" ] && . "$1"; echo "$DTBS"' _ "$FAKE_UPDATE_CONF") +{ + printf 'M1N1:%s\n' "$(cat "$FAKE/m1n1")" + printf 'DTBS:%s\n' "${dtbs:-asahi}" + printf 'UBOOT' + grep -E '^chosen\.' "$FAKE_M1N1_CONF" 2>/dev/null || true +} >"$FAKE_ESP/m1n1/boot.bin" +echo "update-m1n1 rebuilt" >>"$FAKE/log" +""" + +CURL = r"""#!/bin/bash +out= url= +while (($#)); do + case $1 in -o) out=$2; shift ;; http*) url=$1 ;; esac + shift +done +echo "curl $url" >>"$FAKE/log" +cp "$FAKE_PKGS/$(basename "$url")" "$out" +""" + + +class M3FlowTest(unittest.TestCase): + def setUp(self): + if not shutil.which("bsdtar") or not shutil.which("zstd"): + self.skipTest("bsdtar and zstd are needed to build the fixture m1n1 packages") + self.tmp = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.tmp) + for d in ("dt/chosen", "esp/m1n1", "esp/asahi", "etc/default", "state", "pkgs", "bin", "fake"): + (self.tmp / d).mkdir(parents=True) + self.boot = self.tmp / "esp/m1n1/boot.bin" + self.update_conf = self.tmp / "etc/default/update-m1n1" + self.m1n1_conf = self.tmp / "etc/m1n1.conf" + self.state = self.tmp / "state" + self.fake = self.tmp / "fake" + for name, body in (("pacman", PACMAN), ("update-m1n1", UPDATE_M1N1), ("curl", CURL), + ("findmnt", "#!/bin/sh\ncase \"$*\" in *PARTUUID*) echo fake-uuid ;; *) echo vfat ;; esac\n"), + ("systemctl", "#!/bin/sh\nexit 0\n")): + (self.tmp / "bin" / name).write_text(body) + (self.tmp / "bin" / name).chmod(0o755) + self.shas = {} + for name in OTHERS: + self.fixture(name, name.encode()) + self.fixture(AURORA3, None, []) + self.fixture(AURORA5, None, ["asahi,t6030-gpu", "asahi,t6030-dcp", "asahi,t6030-dcpext"]) + + def fixture(self, name, data, strings=None): + path = self.tmp / "pkgs" / name + if data is not None: + path.write_bytes(data) + else: + root = self.tmp / ("root-" + name) + (root / "usr/lib/asahi-boot").mkdir(parents=True) + (root / "usr/lib/asahi-boot/m1n1.bin").write_bytes( + b"m1n1\0" + b"\0".join(s.encode() for s in strings) + b"\0end") + subprocess.run(["bsdtar", "--zstd", "-cf", str(path), "-C", str(root), "usr"], check=True) + self.shas[name] = hashlib.sha256(path.read_bytes()).hexdigest() + + def mac(self, board, kernel="linux-asahi", m1n1="m1n1-stock", bootbin=b"M1N1:original\n", + stub="14.8.3", iboot="iBoot-10151.140.19.700.2"): + (self.tmp / "dt/compatible").write_bytes(b"".join(c.encode() + b"\0" for c in BOARDS[board])) + (self.tmp / "dt/chosen/asahi,iboot2-version").write_bytes(iboot.encode() + b"\0") + (self.tmp / "esp/asahi/stub_info.json").write_text( + '{"system_version": {"ProductVersion": "%s"}}' % stub) + (self.fake / "installed").write_text(kernel + "\n") + (self.fake / "m1n1").write_text(m1n1 + "\n") + (self.fake / "log").write_text("") + self.boot.write_bytes(bootbin) + + def run_sh(self, body, check=True): + pkgs = "\n".join(f' "{n} {self.shas[n]}"' for n in OTHERS + [AURORA3]) + script = f""" +set -euo pipefail +AURORA_SEP_SOURCE_ONLY=1 source '{INSTALLER}' +sudo="" +DT='{self.tmp}/dt' +STATE='{self.state}' +M1N1_CONF='{self.m1n1_conf}' +UPDATE_M1N1_CONF='{self.update_conf}' +M3GPU_MARKER='{self.tmp}/etc/default/.update-m1n1.created-by-m3gpu' +MODPROBE_CONF='{self.tmp}/etc/aurora-sep.conf' +PACKAGES=( +{pkgs} +) +M3_M1N1_PACKAGE="{AURORA5} {self.shas[AURORA5]}" +esp_bootbin() {{ echo '{self.boot}'; }} +version_notice() {{ :; }}; sep_write_notice() {{ :; }}; ane_dkms_notice() {{ :; }} +snapshot() {{ :; }}; add_pin() {{ :; }}; remove_pin() {{ :; }} +calibration() {{ :; }}; sep_policy() {{ :; }}; neo_radio_notice() {{ :; }} +boot_chain() {{ echo limine; }} +{body} +""" + env = {**os.environ, "NO_COLOR": "1", "PATH": f"{self.tmp}/bin:{os.environ['PATH']}", + "FAKE": str(self.fake), "FAKE_PKGS": str(self.tmp / "pkgs"), + "FAKE_ESP": str(self.tmp / "esp"), "FAKE_UPDATE_CONF": str(self.update_conf), + "FAKE_M1N1_CONF": str(self.m1n1_conf)} + proc = subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env) + if check: + self.assertEqual(proc.returncode, 0, proc.stderr + proc.stdout) + return proc + + def install(self, try_=0, check=True): + return self.run_sh(f"M3_TRY={try_}\ninstall_all", check=check) + + def uninstall(self): + return self.run_sh("uninstall_all") + + def log(self): + return (self.fake / "log").read_text() + + def downloaded(self): + return [l.rsplit("/", 1)[1] for l in self.log().splitlines() if l.startswith("curl ")] + + def kept_copy(self): + return self.tmp / "esp/m1n1/boot.bin.before-7.1.12.aurora2-11.36" + + # M3 Pro on the handoff path + + def test_fresh_m3_pro_gets_the_handoff(self): + self.mac("j516s") + self.install() + boot = self.boot.read_bytes() + self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora5-1\n"), boot) + self.assertTrue(boot.endswith(SWITCHES), boot) + self.assertIn(AURORA5, self.downloaded()) + self.assertNotIn(AURORA3, self.downloaded()) + self.assertEqual(self.kept_copy().read_bytes(), b"M1N1:original\n") + self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff") + self.assertFalse(self.update_conf.read_text().count("M1N1_UPDATE_DISABLED")) + + self.uninstall() + boot = self.boot.read_bytes() + self.assertTrue(boot.startswith(b"M1N1:m1n1-stock\n"), boot) + self.assertNotIn(b"chosen.", boot) + self.assertFalse(self.m1n1_conf.exists()) + self.assertFalse(self.state.exists()) + + def test_m3_pro_over_an_11_36_kernel_only_install(self): + self.mac("j516s", kernel="linux-aurora", bootbin=b"M1N1:original\n") + self.update_conf.write_text(OUR_FREEZE) + self.install() + boot = self.boot.read_bytes() + self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora5-1\n"), boot) + self.assertTrue(boot.endswith(SWITCHES), boot) + self.assertNotIn("M1N1_UPDATE_DISABLED", self.update_conf.read_text()) + + def test_bringup_m3_pro_and_back(self): + handbuilt = b"M1N1:bringup\nDTBS:hand\nUBOOT" + SWITCHES + self.mac("j516s", kernel="linux-asahi", m1n1="m1n1-bringup", bootbin=handbuilt) + self.update_conf.write_text(BRINGUP_FREEZE) + marker = self.tmp / "etc/default/.update-m1n1.created-by-m3gpu" + marker.touch() + self.install() + boot = self.boot.read_bytes() + self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora5-1\n"), boot) + self.assertTrue(boot.endswith(SWITCHES)) + # The hook ran while the bring-up freeze still held, so the first + # rebuild is the installer's own, with the new m1n1 in place. + log = self.log() + self.assertLess(log.index("update-m1n1 frozen"), log.index("update-m1n1 rebuilt")) + self.assertFalse(marker.exists()) + + self.uninstall() + self.assertEqual(self.boot.read_bytes(), handbuilt) + self.assertEqual(self.update_conf.read_text(), BRINGUP_FREEZE) + self.assertTrue(marker.exists()) + + def test_trial_on_an_unlisted_m3_pro(self): + self.mac("j514s") + self.install(try_=1) + self.assertTrue(self.boot.read_bytes().endswith(SWITCHES)) + self.assertTrue(self.kept_copy().exists()) + + # M3s that stay kernel-only + + def assert_kernel_only(self, board, **kw): + self.mac(board, **kw) + before = self.boot.read_bytes() + out = self.install().stdout + self.assertEqual(self.boot.read_bytes(), before) + self.assertIn("boot.bin is unchanged", out) + self.assertFalse([d for d in self.downloaded() if d.startswith("m1n1-aurora-")]) + self.assertNotIn("update-m1n1 rebuilt", self.log()) + self.assertFalse(self.kept_copy().exists()) + self.assertEqual((self.state / "m3-mode").read_text().strip(), "kernel") + self.assertIn("M1N1_UPDATE_DISABLED=1", self.update_conf.read_text()) + + self.uninstall() + self.assertEqual(self.boot.read_bytes(), before) + self.assertNotIn("pacman -Sy --noconfirm --ask 4 linux-asahi linux-asahi-headers libfprint m1n1", + self.log()) + self.assertFalse(self.update_conf.exists()) + + def test_unlisted_m3_pro_is_kernel_only(self): + self.assert_kernel_only("j514s") + + def test_m3_is_kernel_only(self): + self.assert_kernel_only("j613") + + def test_listed_m3_pro_on_another_stub_is_kernel_only(self): + self.assert_kernel_only("j516s", stub="15.6") + + def test_trial_refused_on_an_m3(self): + self.mac("j613") + before = self.boot.read_bytes() + proc = self.install(try_=1, check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("Nothing was installed", proc.stderr) + self.assertEqual(self.boot.read_bytes(), before) + self.assertNotIn("pacman -U", self.log()) + + # M1/M2 are unchanged: aurora3, no switches, no M3 state + + def test_m1_pro_unchanged(self): + self.mac("j314s") + self.install() + boot = self.boot.read_bytes() + self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora3-1\n"), boot) + self.assertNotIn(b"chosen.", boot) + self.assertIn(AURORA3, self.downloaded()) + self.assertNotIn(AURORA5, self.downloaded()) + self.assertFalse((self.state / "m3-mode").exists()) + self.assertFalse(self.m1n1_conf.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/aurora-sep/tests/test_m3_handoff.py b/tools/aurora-sep/tests/test_m3_handoff.py new file mode 100644 index 0000000000000..692a5189d7790 --- /dev/null +++ b/tools/aurora-sep/tests/test_m3_handoff.py @@ -0,0 +1,338 @@ +"""The installer's M3 handoff path, exercised on fixture files instead of a Mac. + +Each case sources install-aurora-sep.sh for its functions only +(AURORA_SEP_SOURCE_ONLY=1), points the paths it touches at a temporary +directory, and replaces sudo and the EFI-partition lookup. +""" +from pathlib import Path +import gzip +import os +import shutil +import subprocess +import tempfile +import unittest + +INSTALLER = Path(__file__).resolve().parent.parent / "install-aurora-sep.sh" + +OUR_FREEZE = ( + "# >>> aurora-sep: keep this M3's boot.bin as it is (remove with: install-aurora-sep.sh --uninstall)\n" + "M1N1_UPDATE_DISABLED=1\n" + "# <<< aurora-sep: keep this M3's boot.bin as it is\n" +) +FREEZE = ( + "# Added by m3-gpu-work/scripts/install-m3gpu.sh: keep the M3 GPU candidate boot.bin.\n" + "# Undo with rollback-m3gpu.sh (then run: sudo update-m1n1).\n" + "M1N1_UPDATE_DISABLED=1\n" +) +SWITCHES = [ + "chosen.asahi,t6030-gpu=1", + "chosen.asahi,t6030-dcp=1", + "chosen.asahi,t6030-dcpext=1", +] +BOARDS = { + "j516s": ["apple,j516s", "apple,t6030", "apple,arm-platform"], + "j514s": ["apple,j514s", "apple,t6030", "apple,arm-platform"], + "j514c": ["apple,j514c", "apple,t6031", "apple,arm-platform"], + "j293": ["apple,j293", "apple,t8103", "apple,arm-platform"], + "j613": ["apple,j613", "apple,t8122", "apple,arm-platform"], + "j314s": ["apple,j314s", "apple,t6000", "apple,arm-platform"], + "j700": ["apple,j700", "apple,t8140", "apple,arm-platform"], +} +GOOD_IBOOT = "iBoot-10151.140.19.700.2" + + +class M3PathTest(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.tmp) + self.dt = self.tmp / "dt" + (self.dt / "chosen").mkdir(parents=True) + self.etc = self.tmp / "etc" + (self.etc / "default").mkdir(parents=True) + self.state = self.tmp / "state" + self.state.mkdir() + self.esp = self.tmp / "esp" + (self.esp / "m1n1").mkdir(parents=True) + (self.esp / "asahi").mkdir() + (self.esp / "m1n1" / "boot.bin").write_bytes(b"m1n1-old\x00\x01" * 8) + + def mac(self, board, iboot=GOOD_IBOOT, stub="14.8.3"): + (self.dt / "compatible").write_bytes( + b"".join(c.encode() + b"\0" for c in BOARDS[board]) + ) + if iboot is not None: + (self.dt / "chosen" / "asahi,iboot2-version").write_bytes(iboot.encode() + b"\0") + info = self.esp / "asahi" / "stub_info.json" + if stub is None: + info.unlink(missing_ok=True) + else: + info.write_text( + '{"vgid": "x", "system_version": {"ProductUserVisibleVersion": "%s (stub)", ' + '"ProductVersion": "%s", "ProductBuildVersion": "23J220"}}' % (stub, stub) + ) + + def run_sh(self, body, check=True): + script = f""" +set -euo pipefail +AURORA_SEP_SOURCE_ONLY=1 source '{INSTALLER}' +sudo="" +DT='{self.dt}' +STATE='{self.state}' +M1N1_CONF='{self.etc}/m1n1.conf' +UPDATE_M1N1_CONF='{self.etc}/default/update-m1n1' +M3GPU_MARKER='{self.etc}/default/.update-m1n1.created-by-m3gpu' +esp_bootbin() {{ [[ -f '{self.esp}/m1n1/boot.bin' ]] && echo '{self.esp}/m1n1/boot.bin'; }} +{body} +""" + proc = subprocess.run( + ["bash", "-c", script], capture_output=True, text=True, + env={**os.environ, "NO_COLOR": "1"}, + ) + if check: + self.assertEqual(proc.returncode, 0, proc.stderr + proc.stdout) + return proc + + # Boards + + def test_board_classes(self): + expect = {"j516s": "m3 handoff", "j514s": "m3 off", "j613": "m3 off", + "j514c": "m3 off", "j314s": "not-m3", "j700": "not-m3 neo"} + for board, want in expect.items(): + with self.subTest(board=board): + self.mac(board) + out = self.run_sh( + 'if is_m3; then printf m3; else printf not-m3; fi\n' + 'if is_m3; then if is_m3_handoff_board; then printf " handoff"; else printf " off"; fi; fi\n' + 'if is_neo; then printf " neo"; fi\n' + ).stdout + self.assertEqual(out, want) + + def test_board_list_is_one_line(self): + self.mac("j514s") + out = self.run_sh('M3_HANDOFF_BOARDS="j516s j514s"\nis_m3_handoff_board && echo yes').stdout + self.assertEqual(out.strip(), "yes") + + def test_board_list_never_takes_a_non_m3_pro(self): + # The switches are T6030 ones: listing an M3 or M3 Max board must not + # give it the handoff. + self.mac("j514c") + out = self.run_sh('M3_HANDOFF_BOARDS="j516s j514c"\nis_m3_handoff_board && echo yes || echo no').stdout + self.assertEqual(out.strip(), "no") + + # Stub + + def test_stub_ok(self): + self.mac("j516s") + self.assertEqual(self.run_sh("m3_stub_problem").stdout, "") + + def test_stub_other_version(self): + self.mac("j516s", stub="15.6") + self.assertIn("stub is 15.6", self.run_sh("m3_stub_problem").stdout) + + def test_stub_info_missing_falls_back_to_iboot(self): + self.mac("j516s", stub=None) + self.assertEqual(self.run_sh("m3_stub_problem").stdout, "") + self.mac("j516s", stub=None, iboot="iBoot-11881.1.1") + self.assertIn("iBoot-11881.1.1", self.run_sh("m3_stub_problem").stdout) + + def test_stub_iboot_unreported(self): + self.mac("j516s", iboot=None, stub=None) + self.assertIn("not reported", self.run_sh("m3_stub_problem").stdout) + + # Plan + + def plan(self, try_=0): + return self.run_sh(f"M3_TRY={try_}\nm3_plan >/dev/null 2>&1\necho $M3_MODE").stdout.strip() + + def test_plan_modes(self): + for board, stub, want in [("j516s", "14.8.3", "handoff"), ("j516s", "15.6", "kernel"), + ("j514s", "14.8.3", "kernel"), ("j613", "14.8.3", "kernel"), + ("j514c", "14.8.3", "kernel"), + ("j314s", "13.5", "none"), ("j700", "26.4", "none")]: + with self.subTest(board=board, stub=stub): + self.mac(board, stub=stub) + self.assertEqual(self.plan(), want) + + def test_plan_try_on_an_unlisted_m3_pro(self): + self.mac("j514s") + self.assertEqual(self.plan(try_=1), "handoff") + out = self.run_sh("M3_TRY=1\nm3_plan").stderr + self.assertIn("nobody", out) + + def test_plan_try_refused_elsewhere(self): + for board, stub, why in [("j613", "14.8.3", "M3 Pro"), ("j514c", "14.8.3", "M3 Pro"), + ("j314s", "13.5", "isn't an M3"), ("j700", "26.4", "isn't an M3"), + ("j514s", "15.6", "stub is 15.6"), ("j516s", "15.6", "stub is 15.6")]: + with self.subTest(board=board, stub=stub): + self.mac(board, stub=stub) + proc = self.run_sh("M3_TRY=1\nm3_plan", check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn(why, proc.stderr) + self.assertIn("Nothing was installed", proc.stderr) + + def test_plan_handoff_over_an_11_36_kernel_only_install(self): + self.mac("j516s") + (self.etc / "default" / "update-m1n1").write_text(OUR_FREEZE) + self.assertEqual(self.plan(), "handoff") + + def test_plan_handoff_with_bringup_freeze(self): + self.mac("j516s") + (self.etc / "default" / "update-m1n1").write_text(FREEZE) + self.assertEqual(self.plan(), "handoff") + + def test_plan_refuses_someone_elses_freeze(self): + self.mac("j516s") + (self.etc / "default" / "update-m1n1").write_text("M1N1_UPDATE_DISABLED=1\n") + proc = self.run_sh("m3_plan", check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("Nothing was installed", proc.stderr) + + # Bring-up freeze + + def test_unfreeze_file_the_bringup_created(self): + conf = self.etc / "default" / "update-m1n1" + marker = self.etc / "default" / ".update-m1n1.created-by-m3gpu" + conf.write_text(FREEZE) + marker.touch() + self.run_sh("m3gpu_unfreeze") + self.assertFalse(conf.exists()) + self.assertFalse(marker.exists()) + self.assertEqual((self.state / "update-m1n1.m3gpu.saved").read_text(), FREEZE) + self.assertTrue((self.state / "m3gpu-marker.saved").exists()) + + def test_unfreeze_keeps_the_rest_of_the_file(self): + conf = self.etc / "default" / "update-m1n1" + conf.write_text("M1N1=/usr/lib/neo/m1n1.bin\n" + FREEZE + "U_BOOT=/x\n") + self.run_sh("m3gpu_unfreeze") + self.assertEqual(conf.read_text(), "M1N1=/usr/lib/neo/m1n1.bin\nU_BOOT=/x\n") + + def test_unfreeze_leaves_other_freezes(self): + conf = self.etc / "default" / "update-m1n1" + conf.write_text("M1N1_UPDATE_DISABLED=1\n") + self.run_sh("m3gpu_unfreeze") + self.assertEqual(conf.read_text(), "M1N1_UPDATE_DISABLED=1\n") + self.assertFalse((self.state / "update-m1n1.m3gpu.saved").exists()) + + def test_frozen_by_others(self): + conf = self.etc / "default" / "update-m1n1" + for text, want in [(FREEZE, "no"), (OUR_FREEZE, "no"), (FREEZE + OUR_FREEZE, "no"), + ("M1N1_UPDATE_DISABLED=1\n", "yes"), + ("# M1N1_UPDATE_DISABLED=1\n", "no"), (FREEZE + "export M1N1_UPDATE_DISABLED=y\n", "yes")]: + with self.subTest(text=text): + conf.write_text(text) + out = self.run_sh("update_m1n1_frozen_by_others && echo yes || echo no").stdout.strip() + self.assertEqual(out, want) + + # /etc/m1n1.conf + + def test_switches_write_and_remove(self): + conf = self.etc / "m1n1.conf" + conf.write_text("display=2560x1600\n") + self.run_sh("m3_switches_write\nm3_switches_write") + lines = conf.read_text().splitlines() + self.assertEqual(lines[0], "display=2560x1600") + self.assertEqual([l for l in lines if l.startswith("chosen.")], SWITCHES) + self.run_sh("m3_switches_remove") + self.assertEqual(conf.read_text(), "display=2560x1600\n") + + def test_switches_remove_deletes_a_file_of_only_ours(self): + conf = self.etc / "m1n1.conf" + self.run_sh("m3_switches_write") + self.assertTrue(conf.exists()) + self.run_sh("m3_switches_remove") + self.assertFalse(conf.exists()) + + def test_switches_remove_leaves_hand_written_lines(self): + conf = self.etc / "m1n1.conf" + conf.write_text("chosen.asahi,t6030-gpu=1\n") + self.run_sh("m3_switches_remove") + self.assertEqual(conf.read_text(), "chosen.asahi,t6030-gpu=1\n") + + # Package and boot.bin checks + + def package(self, strings): + if not shutil.which("bsdtar") or not shutil.which("zstd"): + self.skipTest("bsdtar and zstd are needed to build a fixture package") + root = self.tmp / "pkgroot" + (root / "usr/lib/asahi-boot").mkdir(parents=True) + (root / "usr/lib/asahi-boot/m1n1.bin").write_bytes( + b"\x7fm1n1\x00" + b"\x00".join(s.encode() for s in strings) + b"\x00tail" + ) + pkg = self.tmp / "m1n1-aurora-test-1-aarch64.pkg.tar.zst" + subprocess.run(["bsdtar", "--zstd", "-cf", str(pkg), "-C", str(root), "usr"], check=True) + return pkg + + def test_package_with_handoff(self): + pkg = self.package(["asahi,t6030-gpu", "asahi,t6030-dcp", "asahi,t6030-dcpext"]) + self.assertEqual(self.run_sh(f"m1n1_pkg_has_handoff '{pkg}' && echo yes").stdout.strip(), "yes") + + def test_package_without_dcpext(self): + pkg = self.package(["asahi,t6030-gpu", "asahi,t6030-dcp"]) + self.assertEqual(self.run_sh(f"m1n1_pkg_has_handoff '{pkg}' && echo yes || echo no").stdout.strip(), "no") + + def test_verify_bootbin(self): + # As update-m1n1 writes it: the first config line follows the gzip + # stream directly, with no newline in front. + boot = self.esp / "m1n1" / "boot.bin" + boot.write_bytes(b"m1n1" + gzip.compress(b"u-boot" * 64) + "\n".join(SWITCHES).encode() + b"\n") + self.run_sh("m3_verify_bootbin") + boot.write_bytes(b"m1n1" + gzip.compress(b"u-boot" * 64) + b"display=2560x1600\n" + + "\n".join(SWITCHES).encode() + b"\n") + self.run_sh("m3_verify_bootbin") + boot.write_bytes(b"m1n1" + gzip.compress(b"u-boot" * 64) + b"chosen.asahi,t6030-gpu=1\n") + proc = self.run_sh("m3_verify_bootbin", check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("switch lines", proc.stderr) + + def test_verify_real_layout(self): + # The m3pro's hand-built boot.bin ends exactly like this (2026-10-03). + boot = self.esp / "m1n1" / "boot.bin" + boot.write_bytes(b"\x1f\x8b" + os.urandom(900) + b"chosen.asahi,t6030-gpu=1\n" + b"chosen.asahi,t6030-dcp=1\nchosen.asahi,t6030-dcpext=1\n") + self.run_sh("m3_verify_bootbin") + + # Which packages each Mac gets + + def packages(self, board, stub="14.8.3", try_=0): + self.mac(board, stub=stub) + out = self.run_sh(f"M3_TRY={try_}\nm3_plan >/dev/null 2>&1\npackages_for_this_mac").stdout + return [line.split()[0] for line in out.splitlines()] + + def test_packages_per_mac(self): + m1n1_m1m2 = "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst" + m1n1_m3 = "m1n1-aurora-1.6.1.aurora5-1-aarch64.pkg.tar.zst" + for board, stub, try_, want in [("j314s", "13.5", 0, m1n1_m1m2), ("j293", "13.5", 0, m1n1_m1m2), + ("j700", "26.4", 0, None), ("j613", "14.8.3", 0, None), + ("j514c", "14.8.3", 0, None), ("j514s", "14.8.3", 0, None), + ("j516s", "15.6", 0, None), ("j516s", "14.8.3", 0, m1n1_m3), + ("j514s", "14.8.3", 1, m1n1_m3)]: + with self.subTest(board=board, stub=stub, try_=try_): + got = self.packages(board, stub, try_) + m1n1 = [p for p in got if p.startswith("m1n1-aurora-")] + self.assertEqual(m1n1, [want] if want else []) + self.assertIn("linux-aurora-7.1.12.aurora2-11.36-aarch64.pkg.tar.zst", got) + self.assertEqual(len(got), 5 if want else 4) + + # Uninstall + + def test_restore_bringup(self): + conf = self.etc / "default" / "update-m1n1" + marker = self.etc / "default" / ".update-m1n1.created-by-m3gpu" + conf.write_text(FREEZE) + marker.touch() + handbuilt = (self.esp / "m1n1" / "boot.bin").read_bytes() + self.run_sh("m3gpu_unfreeze") + (self.state / "boot.bin.saved").write_bytes(handbuilt) + (self.esp / "m1n1" / "boot.bin").write_bytes(b"stock rebuild") + self.run_sh("m3_restore_bringup") + self.assertEqual(conf.read_text(), FREEZE) + self.assertTrue(marker.exists()) + self.assertEqual((self.esp / "m1n1" / "boot.bin").read_bytes(), handbuilt) + + def test_restore_bringup_noop_elsewhere(self): + self.run_sh("m3_restore_bringup") + self.assertFalse((self.etc / "default" / "update-m1n1").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/aurora-sep/tests/test_m3_kernel_only.py b/tools/aurora-sep/tests/test_m3_kernel_only.py index 38c2c02401999..f31c67542f558 100644 --- a/tools/aurora-sep/tests/test_m3_kernel_only.py +++ b/tools/aurora-sep/tests/test_m3_kernel_only.py @@ -66,7 +66,7 @@ def run_sh(self, body, check=True): sudo="" DT='{self.dt}' UPDATE_M1N1_CONF='{self.conf}' -m3_bootbin() {{ [[ -f '{self.boot}' ]] && echo '{self.boot}'; }} +esp_bootbin() {{ [[ -f '{self.boot}' ]] && echo '{self.boot}'; }} {body} """ proc = subprocess.run(["bash", "-c", script], capture_output=True, text=True, @@ -152,8 +152,6 @@ def test_m1n1_update_says_when_frozen(self): state.mkdir() proc = self.run_sh( f"STATE='{state}'\n" - # m1n1_update writes /etc/default/update-m1n1 itself; point it here. - f"eval \"$(declare -f m1n1_update | command sed 's|conf=/etc/default/update-m1n1|conf={self.conf}|')\"\n" "m1n1_update", check=False) self.assertEqual(proc.returncode, 0, proc.stderr) self.assertIn("was not rebuilt", proc.stderr) From 88e215cbdf6223256ae5eb80de0cbe174a1d4b6a Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 07:44:42 -0700 Subject: [PATCH 1743/2680] tools: aurora-sep: harden the M3 handoff path, ship m1n1 aurora6 M3_M1N1_PACKAGE is now m1n1-aurora 1.6.1.aurora6. On the 16" M3 Pro, aurora5 booted but its display handoff refused: the DCP __OS_LOG buffers iBoot places after stage 1 lay inside aurora5's .bss. aurora6 adds one patch, so that stage 2 runs from above the chainloaded image. With it the m3pro booted with the handoff published, 12 CPUs, and 11.36 driving the native display and the GPU. Fixes from an independent review of the previous commit: - A plain run on a Mac that has the handoff (listed, or tried with --m3-handoff) keeps it. It used to drop to kernel mode and leave aurora5 in place for --uninstall. - The first install records, before anything changes, update-m1n1's configuration (without the 11.36 M3 freeze) or that there was none, the bring-up's freeze, and boot.bin. --uninstall after a run that stopped partway puts them back, instead of deleting the configuration and rebuilding the bring-up's boot.bin from stock m1n1. - previous-package is written once. 11.36 rewrote it on every run, so an updated Mac named linux-aurora, which --uninstall can't reinstall. linux-aurora there now means linux-asahi. - --uninstall reinstalls the stock m1n1 on any M3 that has m1n1-aurora, whatever the record says. An empty m3-mode no longer stops it. - The handoff path refuses to start when update-m1n1 points at its own M1N1, SOURCE, U_BOOT, CONFIG or TARGET, judges "frozen by someone else" by sourcing the file as update-m1n1 does, dies if update-m1n1 is still frozen after the unfreeze, and checks that boot.bin starts with the installed m1n1.bin, not only that it ends with the switches. - The package check matches whole strings: asahi,t6030-dcp is a prefix of asahi,t6030-dcpext. - The copy kept on the EFI partition, and the bring-up's restored boot.bin, are written to a temporary name, compared and renamed, so a full partition can't leave a truncated boot loader under the final name. - On a GRUB Mac, the handoff path updates grub.cfg before its checks. - Extra arguments are refused, except after --reset-touchid. - The DTBS= line uses '|| true': update-m1n1 sources it with sh, which on Arch keeps set -e inside $(...), so '; true' did not save it when grep matched nothing. The flow tests now source the configuration with sh, can fail pacman, and cover each of the above. 58 tests pass. Seven of eight mutants fail them; the eighth, the still-frozen check, is also covered by the boot.bin check. shellcheck shows the same 4 notes as 11.36. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit 8885b64a491256e2c92305c64c7b0633e9ecdeef) --- tools/aurora-sep/install-aurora-sep.sh | 149 +++++++++++++++--- tools/aurora-sep/tests/test_m3_flow.py | 132 +++++++++++++--- tools/aurora-sep/tests/test_m3_handoff.py | 29 +++- tools/aurora-sep/tests/test_m3_kernel_only.py | 2 +- 4 files changed, 269 insertions(+), 43 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 7246e8109b89e..1647b182474e8 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -135,7 +135,7 @@ PACKAGES=( ) # Only for an M3 on the handoff path (see m3_plan), in place of the m1n1-aurora # above: the same m1n1 plus the T6030 display and GPU handoff. -M3_M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora5-1-aarch64.pkg.tar.zst e06dd92a75fb95cba822eb0a414a6f01f6d5bf9ca4f85749b0ef663922c38294" +M3_M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora6-1-aarch64.pkg.tar.zst 16747a63b59815ba54f95c59ab97514fb7178aa459953aa4369effce90456af8" PINNED="linux-aurora linux-aurora-headers libfprint m1n1-aurora" PIN_BEGIN="# >>> aurora-sep pin (remove with: install-aurora-sep.sh --uninstall)" PIN_END="# <<< aurora-sep pin" @@ -250,6 +250,8 @@ M3_SWITCHES="chosen.asahi,t6030-gpu=1 chosen.asahi,t6030-dcp=1 chosen.asahi,t603 M3_STUB_VERSION=14.8.3 M3_STUB_IBOOT=iBoot-10151.140.19 M1N1_CONF=/etc/m1n1.conf +# What update-m1n1 puts at the start of boot.bin. +M1N1_BIN=/usr/lib/asahi-boot/m1n1.bin M1N1_CONF_BEGIN="# >>> aurora-sep: M3 Pro display and GPU handoff (remove with: install-aurora-sep.sh --uninstall)" M1N1_CONF_END="# <<< aurora-sep: M3 Pro display and GPU handoff" # The M3 bring-up's install-m3gpu.sh froze boot.bin with exactly these lines, @@ -361,7 +363,7 @@ is_m3_handoff_board() { # macOS version, and m1n1 reports the stub's iBoot. m3_stub_problem() { local iboot bootbin info="" version="" - iboot=$(tr -d '\0' <"$DT/chosen/asahi,iboot2-version" 2>/dev/null) || iboot="" + iboot=$({ tr -d '\0' <"$DT/chosen/asahi,iboot2-version"; } 2>/dev/null) || iboot="" if bootbin=$(esp_bootbin); then info=${bootbin%/m1n1/boot.bin}/asahi/stub_info.json version=$($sudo cat "$info" 2>/dev/null | grep -o '"ProductVersion": *"[^"]*"' | head -1 | @@ -381,7 +383,7 @@ m1n1_pkg_has_handoff() { bin=$(mktemp) bsdtar -xOf "$1" usr/lib/asahi-boot/m1n1.bin >"$bin" 2>/dev/null || rc=1 for s in asahi,t6030-gpu asahi,t6030-dcp asahi,t6030-dcpext; do - ((rc == 0)) && ! grep -qaF "$s" "$bin" && rc=1 + ((rc == 0)) && ! tr '\0' '\n' <"$bin" | grep -qaxF "$s" && rc=1 done rm -f "$bin" return "$rc" @@ -408,11 +410,23 @@ update_m1n1_frozen_by_others() { $0 == b { skip = 1; next } skip && $0 == e { skip = 0; next } !skip' >"$tmp" - grep -Eq '^[^#]*M1N1_UPDATE_DISABLED=' "$tmp" && rc=0 + # shellcheck disable=SC2016 # expanded by that sh, not here + env -i PATH="$PATH" sh -c 'set -e; . "$1"; [ -n "${M1N1_UPDATE_DISABLED:-}" ]' _ "$tmp" \ + >/dev/null 2>&1 && rc=0 rm -f "$tmp" return "$rc" } +# True when update-m1n1's configuration points it at another m1n1, U-Boot or +# config than the packaged ones: the handoff check could then pass on a boot.bin +# this release did not build. +update_m1n1_customised() { + [[ -f $UPDATE_M1N1_CONF ]] || return 1 + # shellcheck disable=SC2016 # expanded by that sh, not here + env -i PATH="$PATH" sh -c '. "$1" >/dev/null 2>&1; [ -n "${M1N1:-}${SOURCE:-}${U_BOOT:-}${CONFIG:-}${TARGET:-}" ]' _ \ + "$UPDATE_M1N1_CONF" >/dev/null 2>&1 +} + # Lift the M3 bring-up's freeze: only its own three lines, and its marker. # The original is kept so --uninstall can put the freeze back. m3gpu_unfreeze() { @@ -422,7 +436,7 @@ m3gpu_unfreeze() { tmp=$(mktemp) m3gpu_unfrozen >"$tmp" if [[ -e $M3GPU_MARKER ]]; then - $sudo touch "$STATE/m3gpu-marker.saved" + [[ -f $STATE/m3gpu-marker.saved ]] || $sudo touch "$STATE/m3gpu-marker.saved" $sudo rm -f "$M3GPU_MARKER" fi if grep -q '[^[:space:]]' "$tmp"; then @@ -469,6 +483,13 @@ m3_switches_remove() { rm -f "$tmp" } +# The M3 path an earlier install took, or nothing. +m3_recorded_mode() { + local mode="" + if [[ -f $STATE/m3-mode ]]; then read -r mode _ <"$STATE/m3-mode" || true; fi + echo "$mode" +} + # Decide the M3 path before anything is downloaded, so a Mac this release can't # set up as asked stops with nothing changed. m3_plan() { @@ -479,6 +500,12 @@ m3_plan() { return 0 fi board=$(this_board) + # An update never takes the handoff away again: once a Mac has it (listed, or + # tried with --m3-handoff), a plain run keeps it, and its checks still apply. + if ((M3_TRY == 0)) && [[ $(m3_recorded_mode) == handoff ]]; then + M3_TRY=1 + say "M3 ($board): this Mac has m1n1's display and GPU handoff from an earlier install; keeping it" + fi M3_MODE=kernel if ! is_m3_pro; then ((M3_TRY == 0)) || die "--m3-handoff is for an M3 Pro (t6030); this M3 ($board) has no @@ -507,6 +534,11 @@ m3_plan() { bring-up's install-m3gpu.sh. Switching the M3 Pro handoff on needs m1n1's boot.bin rebuilt. Remove that line and run this again. Nothing was installed." fi + if update_m1n1_customised; then + die "$UPDATE_M1N1_CONF points update-m1n1 at its own m1n1, U-Boot, config or target + (M1N1=, SOURCE=, U_BOOT=, CONFIG= or TARGET=). The M3 Pro handoff needs boot.bin built + from this release's m1n1. Remove those lines and run this again. Nothing was installed." + fi M3_MODE=handoff if is_m3_handoff_board; then say "M3 Pro ($board, macOS $M3_STUB_VERSION stub): installing m1n1 with the display and GPU handoff" @@ -521,13 +553,20 @@ m3_plan() { # appends /etc/m1n1.conf's lines straight after the gzipped U-Boot, so the # first one has no newline in front: look for the block, not whole lines. m3_verify_bootbin() { - local target tail block + local target tail block size target=$(esp_bootbin) || die "could not find m1n1's boot.bin to check the M3 Pro switches in" + size=$(stat -c %s "$M1N1_BIN") + $sudo cmp -s -n "$size" "$M1N1_BIN" "$target" || + die "$target does not start with this release's m1n1 ($M1N1_BIN), so it was not rebuilt. + The boot loader this Mac booted with is kept as m1n1/boot.bin.before-$VERSION on the EFI + partition and in $STATE/boot.bin.saved. Please report it before rebooting." block=$(tr ' ' '\n' <<<"$M3_SWITCHES") tail=$($sudo tail -c 1024 "$target" | tr -d '\0') [[ $tail == *"$block"* ]] || - die "the rebuilt $target does not carry the M3 Pro switch lines; the previous one is saved in $STATE/boot.bin.saved" - say "m1n1's boot.bin carries the M3 Pro handoff switches" + die "the rebuilt $target does not carry the M3 Pro switch lines. The boot loader this Mac + booted with is kept as m1n1/boot.bin.before-$VERSION on the EFI partition and in + $STATE/boot.bin.saved. Please report it before rebooting." + say "m1n1's boot.bin is this release's m1n1 with the M3 Pro handoff switches" } # --uninstall on a bring-up Mac: update-m1n1 has just rebuilt boot.bin from the @@ -539,7 +578,7 @@ m3_restore_bringup() { $sudo cp -p "$STATE/update-m1n1.m3gpu.saved" "$UPDATE_M1N1_CONF" if [[ -f $STATE/m3gpu-marker.saved ]]; then $sudo touch "$M3GPU_MARKER"; fi if [[ -f $STATE/boot.bin.saved ]] && target=$(esp_bootbin); then - $sudo cp "$STATE/boot.bin.saved" "$target" + replace_on_esp "$STATE/boot.bin.saved" "$target" say "Restored the M3 bring-up's boot.bin and its freeze on update-m1n1" else warn "restored the M3 bring-up's freeze on update-m1n1, but not its boot.bin (no saved copy)" @@ -567,12 +606,28 @@ unproven_board_backup() { booted it on this model yet." } +# Replace a file on the EFI partition without ever leaving a partial one in +# its place: copy next to it, compare, then rename. +replace_on_esp() { + local src=$1 dst=$2 + if ! { $sudo cp "$src" "$dst.new" && $sudo cmp -s "$src" "$dst.new" && sync && + $sudo mv "$dst.new" "$dst" && sync; }; then + die "could not write $dst; the previous one is still there" + fi +} + keep_bootbin_on_esp() { local why=$1 target keep uuid if target=$(esp_bootbin); then keep=$target.before-$VERSION - $sudo test -f "$keep" || $sudo cp "$target" "$keep" || - die "could not keep a copy of $target; nothing was installed" + # Only a complete, compared copy gets the final name, so a later run can + # trust one it finds. + if ! $sudo test -f "$keep"; then + if ! { $sudo cp "$target" "$keep.new" && $sudo cmp -s "$target" "$keep.new" && sync && + $sudo mv "$keep.new" "$keep"; }; then + die "could not keep a copy of $target; nothing was installed" + fi + fi uuid=$(findmnt -no PARTUUID --target "${target%/m1n1/boot.bin}") warn "$why If the Mac stops in m1n1 after this install (m1n1 text on screen, often \"No valid payload found\", and no boot menu), put the @@ -731,8 +786,9 @@ m1n1_update() { # upgrade the running kernel's modules are restored unowned but still carry that # pkgbase, so both kernels' DTBs would be bundled. m1n1 keeps the last matching # DTB, and the glob sorts 11.10 before 11.9, so the stale one can win. -# (update-m1n1 runs under set -e, so this assignment must always succeed.) -DTBS=$(pacman -Qlq linux-aurora 2>/dev/null | grep '/dtbs/[^/]*\.dtb$'; true) +# (update-m1n1 runs under sh's set -e, which on Arch also applies inside +# $(...), so the pipeline must succeed even when grep matches nothing.) +DTBS=$(pacman -Qlq linux-aurora 2>/dev/null | grep '/dtbs/[^/]*\.dtb$' || true) EOF $sudo install -m 644 "$tmp" "$conf" rm -f "$tmp" @@ -948,6 +1004,33 @@ neo_radio_notice() { - Bluetooth is off by default in this release." } +# The first install records what it found, before anything changes, so that +# --uninstall can put it back even after a run that stopped partway: +# update-m1n1's configuration (without this script's own M3 freeze from 11.36), +# or that there was none; the bring-up's freeze; and boot.bin itself. +snapshot_boot_state() { + local tmp target + if [[ ! -e $STATE/update-m1n1.default.saved && ! -e $STATE/update-m1n1.absent ]]; then + if [[ -f $UPDATE_M1N1_CONF ]]; then + tmp=$(mktemp) + m3_unfrozen >"$tmp" + $sudo install -m 644 "$tmp" "$STATE/update-m1n1.default.saved" + rm -f "$tmp" + else + $sudo touch "$STATE/update-m1n1.absent" + fi + fi + if [[ -f $UPDATE_M1N1_CONF && ! -f $STATE/update-m1n1.m3gpu.saved ]] && + grep -qxF "${M3GPU_FREEZE[0]}" "$UPDATE_M1N1_CONF"; then + $sudo cp -p "$UPDATE_M1N1_CONF" "$STATE/update-m1n1.m3gpu.saved" + if [[ -e $M3GPU_MARKER ]]; then $sudo touch "$STATE/m3gpu-marker.saved"; fi + fi + if [[ ! -f $STATE/boot.bin.saved ]] && target=$(esp_bootbin); then + $sudo cp "$target" "$STATE/boot.bin.saved" + fi + return 0 +} + # The packages this Mac gets, one "file sha256" per line: PACKAGES, except # that a Neo keeps its own m1n1, and an M3 gets M3_M1N1_PACKAGE on the handoff # path and no m1n1 otherwise. Needs m3_plan first. @@ -999,10 +1082,14 @@ install_all() { m1n1-aurora and its display and GPU handoff." fi $sudo install -d "$STATE" + snapshot_boot_state if [[ ! -f $STATE/previous && $chain == grub ]]; then keep_grub_fallback "$kernel" fi - pacman -Q "$kernel" | $sudo tee "$STATE/previous-package" >/dev/null + # The kernel this Mac had before the first install; an update keeps it. + if [[ ! -f $STATE/previous-package ]]; then + pacman -Q "$kernel" | $sudo tee "$STATE/previous-package" >/dev/null + fi if [[ $M3_MODE != none ]]; then echo "$M3_MODE" | $sudo tee "$STATE/m3-mode" >/dev/null; fi # Before pacman's update-m1n1 hook runs on the kernel's device trees. case $M3_MODE in @@ -1051,14 +1138,21 @@ install_all() { case $M3_MODE in kernel) m3_bootbin_report ;; handoff) + # grub.cfg first, so a failed check below never leaves it pointing at + # the replaced kernel. + if [[ $chain == grub ]]; then grub_update; fi m3gpu_unfreeze m3_unfreeze + if update_m1n1_frozen; then + die "update-m1n1 is still frozen in $UPDATE_M1N1_CONF, so boot.bin can't be rebuilt with + the handoff. boot.bin was not changed. Please report it with that file." + fi m1n1_update m3_verify_bootbin ;; *) m1n1_update ;; esac - if [[ $chain == grub ]]; then + if [[ $chain == grub && $M3_MODE != handoff ]]; then grub_update fi @@ -1084,12 +1178,21 @@ install_all() { uninstall_all() { local previous=linux-asahi m3_mode=none - [[ -f $STATE/previous-package ]] && read -r previous _ <"$STATE/previous-package" - # 11.36 kept no record: every M3 it installed was kernel-only. - if [[ -f $STATE/m3-mode ]]; then - read -r m3_mode _ <"$STATE/m3-mode" - elif is_m3; then - m3_mode=kernel + if [[ -f $STATE/previous-package ]]; then read -r previous _ <"$STATE/previous-package" || true; fi + # 11.36 rewrote this on every run, so an updated Mac may name linux-aurora + # itself, which the repositories don't carry for these Macs. + if [[ -z $previous || $previous == linux-aurora ]]; then previous="linux-asahi"; fi + # 11.36 kept no record: every M3 it installed was kernel-only. Whatever the + # record says, an M3 with m1n1-aurora installed had its boot loader replaced. + m3_mode=$(m3_recorded_mode) + if is_m3; then + if pacman -Q m1n1-aurora >/dev/null 2>&1; then + m3_mode=handoff + elif [[ $m3_mode != handoff ]]; then + m3_mode=kernel + fi + else + m3_mode=none fi command -v aurora-touchid-setup >/dev/null && aurora-touchid-setup --remove || true $sudo systemctl disable apple-sep.path apple-sep.service 2>/dev/null || true @@ -1598,6 +1701,10 @@ set -- "${args[@]}" if ((M3_TRY)) && [[ -n ${1:-} && $1 != --read-only ]]; then die "--m3-handoff goes with an install (alone or with --read-only), not with $1" fi +# One option at a time; only --reset-touchid takes arguments of its own. +if (($# > 1)) && [[ $1 != --reset-touchid ]]; then + die "unexpected arguments after $1: ${*:2}" +fi if preflight_needed "${1:-}"; then preflight; fi case ${1:-} in diff --git a/tools/aurora-sep/tests/test_m3_flow.py b/tools/aurora-sep/tests/test_m3_flow.py index fac251c13a620..350905653c1b0 100644 --- a/tools/aurora-sep/tests/test_m3_flow.py +++ b/tools/aurora-sep/tests/test_m3_flow.py @@ -34,7 +34,7 @@ "# <<< aurora-sep: keep this M3's boot.bin as it is\n" ) AURORA3 = "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst" -AURORA5 = "m1n1-aurora-1.6.1.aurora5-1-aarch64.pkg.tar.zst" +AURORA6 = "m1n1-aurora-1.6.1.aurora6-1-aarch64.pkg.tar.zst" OTHERS = [ "linux-aurora-7.1.12.aurora2-11.36-aarch64.pkg.tar.zst", "linux-aurora-headers-7.1.12.aurora2-11.36-aarch64.pkg.tar.zst", @@ -48,7 +48,11 @@ case $op in -Q) for p in "$@"; do - case $p in linux-aurora | linux-asahi) grep -qx "$p" "$FAKE/installed" || exit 1 ;; esac + case $p in + linux-aurora | linux-asahi) grep -qx "$p" "$FAKE/installed" || exit 1 ;; + m1n1-aurora) grep -q '^m1n1-aurora' "$FAKE/m1n1" || exit 1 ;; + m1n1) grep -q '^m1n1-stock' "$FAKE/m1n1" || exit 1 ;; + esac echo "$p 1-1" done ;; -Qq) exit 1 ;; @@ -57,11 +61,12 @@ echo /usr/lib/modules/7.1.12-2-11.36-sep-ARCH/dtbs/fake.dtb exit 0 ;; -U) + if [[ -n ${FAKE_FAIL_U:-} ]]; then echo "error: failed to commit transaction" >&2; exit 1; fi for f in "$@"; do [[ $f == -* || $f == 4 ]] && continue b=$(basename "$f") case $b in - m1n1-aurora-*) echo "${b%-aarch64.pkg.tar.zst}" >"$FAKE/m1n1" ;; + m1n1-aurora-*) echo "${b%-aarch64.pkg.tar.zst}" >"$FAKE/m1n1"; printf 'M1N1:%s\n' "${b%-aarch64.pkg.tar.zst}" >"$FAKE/m1n1.bin" ;; linux-aurora-headers-*) ;; linux-aurora-*) sed -i '/^linux-asahi$/d' "$FAKE/installed"; echo linux-aurora >>"$FAKE/installed" ;; esac @@ -71,7 +76,7 @@ hook=0 for p in "$@"; do case $p in - m1n1) echo m1n1-stock >"$FAKE/m1n1"; hook=1 ;; + m1n1) echo m1n1-stock >"$FAKE/m1n1"; printf 'M1N1:m1n1-stock\n' >"$FAKE/m1n1.bin"; hook=1 ;; linux-asahi) sed -i '/^linux-aurora$/d' "$FAKE/installed"; echo linux-asahi >>"$FAKE/installed"; hook=1 ;; esac done @@ -82,14 +87,20 @@ UPDATE_M1N1 = r"""#!/bin/bash echo "update-m1n1 $*" >>"$FAKE/log" -if [[ -f $FAKE_UPDATE_CONF ]] && - bash -c 'set -e; . "$1"; [ -n "${M1N1_UPDATE_DISABLED:-}" ]' _ "$FAKE_UPDATE_CONF" 2>/dev/null; then - echo "update-m1n1 frozen" >>"$FAKE/log" - exit 0 +# As the real one: sourced by sh under set -e (bash in POSIX mode on Arch). +if [[ -f $FAKE_UPDATE_CONF ]]; then + if ! sh -c 'set -e; . "$1"' _ "$FAKE_UPDATE_CONF" 2>/dev/null; then + echo "update-m1n1 failed sourcing" >>"$FAKE/log" + exit 1 + fi + if sh -c 'set -e; . "$1"; [ -n "${M1N1_UPDATE_DISABLED:-}" ]' _ "$FAKE_UPDATE_CONF" 2>/dev/null; then + echo "update-m1n1 frozen" >>"$FAKE/log" + exit 0 + fi fi -dtbs=$(bash -c 'set -e; DTBS=; [ -f "$1" ] && . "$1"; echo "$DTBS"' _ "$FAKE_UPDATE_CONF") +dtbs=$(sh -c 'set -e; DTBS=; [ -f "$1" ] && . "$1"; echo "$DTBS"' _ "$FAKE_UPDATE_CONF") { - printf 'M1N1:%s\n' "$(cat "$FAKE/m1n1")" + cat "$FAKE/m1n1.bin" printf 'DTBS:%s\n' "${dtbs:-asahi}" printf 'UBOOT' grep -E '^chosen\.' "$FAKE_M1N1_CONF" 2>/dev/null || true @@ -126,11 +137,12 @@ def setUp(self): ("systemctl", "#!/bin/sh\nexit 0\n")): (self.tmp / "bin" / name).write_text(body) (self.tmp / "bin" / name).chmod(0o755) + self.extra_env = {} self.shas = {} for name in OTHERS: self.fixture(name, name.encode()) self.fixture(AURORA3, None, []) - self.fixture(AURORA5, None, ["asahi,t6030-gpu", "asahi,t6030-dcp", "asahi,t6030-dcpext"]) + self.fixture(AURORA6, None, ["asahi,t6030-gpu", "asahi,t6030-dcp", "asahi,t6030-dcpext"]) def fixture(self, name, data, strings=None): path = self.tmp / "pkgs" / name @@ -152,6 +164,7 @@ def mac(self, board, kernel="linux-asahi", m1n1="m1n1-stock", bootbin=b"M1N1:ori '{"system_version": {"ProductVersion": "%s"}}' % stub) (self.fake / "installed").write_text(kernel + "\n") (self.fake / "m1n1").write_text(m1n1 + "\n") + (self.fake / "m1n1.bin").write_text("M1N1:" + m1n1 + "\n") (self.fake / "log").write_text("") self.boot.write_bytes(bootbin) @@ -167,10 +180,11 @@ def run_sh(self, body, check=True): UPDATE_M1N1_CONF='{self.update_conf}' M3GPU_MARKER='{self.tmp}/etc/default/.update-m1n1.created-by-m3gpu' MODPROBE_CONF='{self.tmp}/etc/aurora-sep.conf' +M1N1_BIN='{self.fake}/m1n1.bin' PACKAGES=( {pkgs} ) -M3_M1N1_PACKAGE="{AURORA5} {self.shas[AURORA5]}" +M3_M1N1_PACKAGE="{AURORA6} {self.shas[AURORA6]}" esp_bootbin() {{ echo '{self.boot}'; }} version_notice() {{ :; }}; sep_write_notice() {{ :; }}; ane_dkms_notice() {{ :; }} snapshot() {{ :; }}; add_pin() {{ :; }}; remove_pin() {{ :; }} @@ -182,13 +196,14 @@ def run_sh(self, body, check=True): "FAKE": str(self.fake), "FAKE_PKGS": str(self.tmp / "pkgs"), "FAKE_ESP": str(self.tmp / "esp"), "FAKE_UPDATE_CONF": str(self.update_conf), "FAKE_M1N1_CONF": str(self.m1n1_conf)} + env.update(self.extra_env) proc = subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env) if check: self.assertEqual(proc.returncode, 0, proc.stderr + proc.stdout) return proc - def install(self, try_=0, check=True): - return self.run_sh(f"M3_TRY={try_}\ninstall_all", check=check) + def install(self, try_=0, check=True, env=""): + return self.run_sh(f"{env}\nM3_TRY={try_}\ninstall_all", check=check) def uninstall(self): return self.run_sh("uninstall_all") @@ -208,9 +223,9 @@ def test_fresh_m3_pro_gets_the_handoff(self): self.mac("j516s") self.install() boot = self.boot.read_bytes() - self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora5-1\n"), boot) + self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora6-1\n"), boot) self.assertTrue(boot.endswith(SWITCHES), boot) - self.assertIn(AURORA5, self.downloaded()) + self.assertIn(AURORA6, self.downloaded()) self.assertNotIn(AURORA3, self.downloaded()) self.assertEqual(self.kept_copy().read_bytes(), b"M1N1:original\n") self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff") @@ -228,7 +243,7 @@ def test_m3_pro_over_an_11_36_kernel_only_install(self): self.update_conf.write_text(OUR_FREEZE) self.install() boot = self.boot.read_bytes() - self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora5-1\n"), boot) + self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora6-1\n"), boot) self.assertTrue(boot.endswith(SWITCHES), boot) self.assertNotIn("M1N1_UPDATE_DISABLED", self.update_conf.read_text()) @@ -240,7 +255,7 @@ def test_bringup_m3_pro_and_back(self): marker.touch() self.install() boot = self.boot.read_bytes() - self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora5-1\n"), boot) + self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora6-1\n"), boot) self.assertTrue(boot.endswith(SWITCHES)) # The hook ran while the bring-up freeze still held, so the first # rebuild is the installer's own, with the new m1n1 in place. @@ -297,6 +312,85 @@ def test_trial_refused_on_an_m3(self): self.assertEqual(self.boot.read_bytes(), before) self.assertNotIn("pacman -U", self.log()) + # Later runs and failures (review of 5c7daa75) + + def test_rerun_without_the_flag_keeps_a_tried_handoff(self): + self.mac("j514s") + self.install(try_=1) + out = self.install().stdout + self.assertIn("keeping it", out) + self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff") + self.assertTrue(self.boot.read_bytes().endswith(SWITCHES)) + self.assertNotIn("M1N1_UPDATE_DISABLED", self.update_conf.read_text()) + self.uninstall() + self.assertTrue(self.boot.read_bytes().startswith(b"M1N1:m1n1-stock\n")) + self.assertFalse(self.m1n1_conf.exists()) + + def test_failed_install_then_uninstall_keeps_the_bringup(self): + handbuilt = b"M1N1:bringup\nDTBS:hand\nUBOOT" + SWITCHES + self.mac("j516s", m1n1="m1n1-bringup", bootbin=handbuilt) + self.update_conf.write_text(BRINGUP_FREEZE) + marker = self.tmp / "etc/default/.update-m1n1.created-by-m3gpu" + marker.touch() + self.extra_env = {"FAKE_FAIL_U": "1"} + proc = self.install(check=False) + self.assertNotEqual(proc.returncode, 0) + self.extra_env = {} + self.uninstall() + self.assertEqual(self.boot.read_bytes(), handbuilt) + self.assertEqual(self.update_conf.read_text(), BRINGUP_FREEZE) + self.assertTrue(marker.exists()) + + def test_failed_install_then_uninstall_keeps_omarchys_config(self): + own = "export LC_ALL=C\n" + self.mac("j516s") + self.update_conf.write_text(own) + self.extra_env = {"FAKE_FAIL_U": "1"} + self.assertNotEqual(self.install(check=False).returncode, 0) + self.extra_env = {} + self.uninstall() + self.assertEqual(self.update_conf.read_text(), own) + + def test_uninstall_after_an_11_36_kernel_only_install_and_update(self): + # 11.36 recorded linux-asahi, then rewrote it to linux-aurora on a re-run. + self.mac("j516s", kernel="linux-aurora") + self.update_conf.write_text(OUR_FREEZE) + (self.state / "previous-package").write_text("linux-aurora 7.1.12.aurora2-11.36-1\n") + self.install() + self.uninstall() + self.assertIn("linux-asahi linux-asahi-headers libfprint m1n1", self.log()) + self.assertNotIn("-S --noconfirm --ask 4 linux-aurora", self.log()) + + def test_previous_package_is_kept_across_updates(self): + self.mac("j314s") + self.install() + self.install() + self.assertTrue((self.state / "previous-package").read_text().startswith("linux-asahi")) + + def test_handoff_refuses_a_boot_bin_that_was_not_rebuilt(self): + # A freeze update-m1n1 honours but the pattern check would miss, on a + # hand-built boot.bin that already ends with the switches. + handbuilt = b"M1N1:bringup\nDTBS:hand\nUBOOT" + SWITCHES + self.mac("j516s", m1n1="m1n1-bringup", bootbin=handbuilt) + self.update_conf.write_text(BRINGUP_FREEZE + ": ${M1N1_UPDATE_DISABLED:=1}\n") + proc = self.install(check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertEqual(self.boot.read_bytes(), handbuilt) + + def test_handoff_refuses_a_custom_m1n1_path(self): + self.mac("j516s") + self.update_conf.write_text("M1N1=/opt/my-m1n1.bin\n") + proc = self.install(check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("Nothing was installed", proc.stderr) + self.assertNotIn("pacman -U", self.log()) + + def test_empty_mode_file_does_not_break_uninstall(self): + self.mac("j613") + self.install() + (self.state / "m3-mode").write_text("") + self.uninstall() + # M1/M2 are unchanged: aurora3, no switches, no M3 state def test_m1_pro_unchanged(self): @@ -306,7 +400,7 @@ def test_m1_pro_unchanged(self): self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora3-1\n"), boot) self.assertNotIn(b"chosen.", boot) self.assertIn(AURORA3, self.downloaded()) - self.assertNotIn(AURORA5, self.downloaded()) + self.assertNotIn(AURORA6, self.downloaded()) self.assertFalse((self.state / "m3-mode").exists()) self.assertFalse(self.m1n1_conf.exists()) diff --git a/tools/aurora-sep/tests/test_m3_handoff.py b/tools/aurora-sep/tests/test_m3_handoff.py index 692a5189d7790..cad1f437ab809 100644 --- a/tools/aurora-sep/tests/test_m3_handoff.py +++ b/tools/aurora-sep/tests/test_m3_handoff.py @@ -55,6 +55,7 @@ def setUp(self): (self.esp / "m1n1").mkdir(parents=True) (self.esp / "asahi").mkdir() (self.esp / "m1n1" / "boot.bin").write_bytes(b"m1n1-old\x00\x01" * 8) + (self.tmp / "m1n1.bin").write_bytes(b"m1n1") def mac(self, board, iboot=GOOD_IBOOT, stub="14.8.3"): (self.dt / "compatible").write_bytes( @@ -81,6 +82,7 @@ def run_sh(self, body, check=True): M1N1_CONF='{self.etc}/m1n1.conf' UPDATE_M1N1_CONF='{self.etc}/default/update-m1n1' M3GPU_MARKER='{self.etc}/default/.update-m1n1.created-by-m3gpu' +M1N1_BIN='{self.tmp}/m1n1.bin' esp_bootbin() {{ [[ -f '{self.esp}/m1n1/boot.bin' ]] && echo '{self.esp}/m1n1/boot.bin'; }} {body} """ @@ -217,7 +219,8 @@ def test_frozen_by_others(self): conf = self.etc / "default" / "update-m1n1" for text, want in [(FREEZE, "no"), (OUR_FREEZE, "no"), (FREEZE + OUR_FREEZE, "no"), ("M1N1_UPDATE_DISABLED=1\n", "yes"), - ("# M1N1_UPDATE_DISABLED=1\n", "no"), (FREEZE + "export M1N1_UPDATE_DISABLED=y\n", "yes")]: + ("# M1N1_UPDATE_DISABLED=1\n", "no"), (FREEZE + "export M1N1_UPDATE_DISABLED=y\n", "yes"), + (FREEZE + ": ${M1N1_UPDATE_DISABLED:=1}\n", "yes")]: with self.subTest(text=text): conf.write_text(text) out = self.run_sh("update_m1n1_frozen_by_others && echo yes || echo no").stdout.strip() @@ -266,6 +269,20 @@ def test_package_with_handoff(self): pkg = self.package(["asahi,t6030-gpu", "asahi,t6030-dcp", "asahi,t6030-dcpext"]) self.assertEqual(self.run_sh(f"m1n1_pkg_has_handoff '{pkg}' && echo yes").stdout.strip(), "yes") + def test_package_with_only_dcpext(self): + # asahi,t6030-dcp is a prefix of asahi,t6030-dcpext: match whole strings. + pkg = self.package(["asahi,t6030-gpu", "asahi,t6030-dcpext"]) + self.assertEqual(self.run_sh(f"m1n1_pkg_has_handoff '{pkg}' && echo yes || echo no").stdout.strip(), "no") + + def test_customised_update_m1n1(self): + conf = self.etc / "default" / "update-m1n1" + for text, want in [("M1N1=/opt/x.bin\n", "yes"), ("U_BOOT=/opt/u\n", "yes"), ("CONFIG=/x\n", "yes"), + (FREEZE, "no"), (OUR_FREEZE, "no"), ("# M1N1=/x\n", "no")]: + with self.subTest(text=text): + conf.write_text(text) + out = self.run_sh("update_m1n1_customised && echo yes || echo no").stdout.strip() + self.assertEqual(out, want) + def test_package_without_dcpext(self): pkg = self.package(["asahi,t6030-gpu", "asahi,t6030-dcp"]) self.assertEqual(self.run_sh(f"m1n1_pkg_has_handoff '{pkg}' && echo yes || echo no").stdout.strip(), "no") @@ -284,9 +301,17 @@ def test_verify_bootbin(self): self.assertNotEqual(proc.returncode, 0) self.assertIn("switch lines", proc.stderr) + def test_verify_bootbin_wrong_m1n1(self): + boot = self.esp / "m1n1" / "boot.bin" + boot.write_bytes(b"MIMI" + gzip.compress(b"u-boot" * 64) + "\n".join(SWITCHES).encode() + b"\n") + proc = self.run_sh("m3_verify_bootbin", check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("not rebuilt", proc.stderr) + def test_verify_real_layout(self): # The m3pro's hand-built boot.bin ends exactly like this (2026-10-03). boot = self.esp / "m1n1" / "boot.bin" + (self.tmp / "m1n1.bin").write_bytes(b"\x1f\x8b") boot.write_bytes(b"\x1f\x8b" + os.urandom(900) + b"chosen.asahi,t6030-gpu=1\n" b"chosen.asahi,t6030-dcp=1\nchosen.asahi,t6030-dcpext=1\n") self.run_sh("m3_verify_bootbin") @@ -300,7 +325,7 @@ def packages(self, board, stub="14.8.3", try_=0): def test_packages_per_mac(self): m1n1_m1m2 = "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst" - m1n1_m3 = "m1n1-aurora-1.6.1.aurora5-1-aarch64.pkg.tar.zst" + m1n1_m3 = "m1n1-aurora-1.6.1.aurora6-1-aarch64.pkg.tar.zst" for board, stub, try_, want in [("j314s", "13.5", 0, m1n1_m1m2), ("j293", "13.5", 0, m1n1_m1m2), ("j700", "26.4", 0, None), ("j613", "14.8.3", 0, None), ("j514c", "14.8.3", 0, None), ("j514s", "14.8.3", 0, None), diff --git a/tools/aurora-sep/tests/test_m3_kernel_only.py b/tools/aurora-sep/tests/test_m3_kernel_only.py index f31c67542f558..7b0bed3e53e64 100644 --- a/tools/aurora-sep/tests/test_m3_kernel_only.py +++ b/tools/aurora-sep/tests/test_m3_kernel_only.py @@ -21,7 +21,7 @@ # What m1n1_update writes on M1/M2 (11.35). AURORA_DTBS = ( "# aurora-sep: build m1n1's stage 2 from the device trees the installed\n" - "DTBS=$(pacman -Qlq linux-aurora 2>/dev/null | grep '/dtbs/[^/]*\\.dtb$'; true)\n" + "DTBS=$(pacman -Qlq linux-aurora 2>/dev/null | grep '/dtbs/[^/]*\\.dtb$' || true)\n" ) COMPATIBLE = { "j516s": ["apple,j516s", "apple,t6030"], From df3601ee966de10e56b72df335ca694c05898dc8 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 07:47:36 -0700 Subject: [PATCH 1744/2680] tools: aurora-sep: read the m1n1 package's strings from a file On the m3pro the 11.36.1 prerelease refused aurora6: "this release's M3 m1n1 has no M3 Pro display and GPU handoff". The check piped the 3.8 MB m1n1.bin through tr into grep -q, which exits at the first match. tr then dies with SIGPIPE, and under pipefail the check reported the string as missing. Nothing was installed. The test fixture was a few bytes, so tr always finished first. Write the strings to a file once and grep that. The fixtures now carry 1 MB after the strings; the previous code fails 7 tests against them. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit 2af6be1960e7c27dd615864d714bc24953fdb0be) --- tools/aurora-sep/install-aurora-sep.sh | 6 ++++-- tools/aurora-sep/tests/test_m3_flow.py | 2 +- tools/aurora-sep/tests/test_m3_handoff.py | 3 +++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 1647b182474e8..3aaff655b7b94 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -381,9 +381,11 @@ m3_stub_problem() { m1n1_pkg_has_handoff() { local bin s rc=0 bin=$(mktemp) - bsdtar -xOf "$1" usr/lib/asahi-boot/m1n1.bin >"$bin" 2>/dev/null || rc=1 + # One string per line, in a file: grep -q on a pipe would stop reading early + # and fail the 3.8 MB tr with SIGPIPE under pipefail. + { bsdtar -xOf "$1" usr/lib/asahi-boot/m1n1.bin | tr '\0' '\n' >"$bin"; } 2>/dev/null || rc=1 for s in asahi,t6030-gpu asahi,t6030-dcp asahi,t6030-dcpext; do - ((rc == 0)) && ! tr '\0' '\n' <"$bin" | grep -qaxF "$s" && rc=1 + ((rc == 0)) && ! grep -qaxF "$s" "$bin" && rc=1 done rm -f "$bin" return "$rc" diff --git a/tools/aurora-sep/tests/test_m3_flow.py b/tools/aurora-sep/tests/test_m3_flow.py index 350905653c1b0..9619c47d41ea4 100644 --- a/tools/aurora-sep/tests/test_m3_flow.py +++ b/tools/aurora-sep/tests/test_m3_flow.py @@ -152,7 +152,7 @@ def fixture(self, name, data, strings=None): root = self.tmp / ("root-" + name) (root / "usr/lib/asahi-boot").mkdir(parents=True) (root / "usr/lib/asahi-boot/m1n1.bin").write_bytes( - b"m1n1\0" + b"\0".join(s.encode() for s in strings) + b"\0end") + b"m1n1\0" + b"\0".join(s.encode() for s in strings) + b"\0end" + bytes(range(256)) * 4096) subprocess.run(["bsdtar", "--zstd", "-cf", str(path), "-C", str(root), "usr"], check=True) self.shas[name] = hashlib.sha256(path.read_bytes()).hexdigest() diff --git a/tools/aurora-sep/tests/test_m3_handoff.py b/tools/aurora-sep/tests/test_m3_handoff.py index cad1f437ab809..5a01315cb61c5 100644 --- a/tools/aurora-sep/tests/test_m3_handoff.py +++ b/tools/aurora-sep/tests/test_m3_handoff.py @@ -258,8 +258,11 @@ def package(self, strings): self.skipTest("bsdtar and zstd are needed to build a fixture package") root = self.tmp / "pkgroot" (root / "usr/lib/asahi-boot").mkdir(parents=True) + # Strings near the start of a large binary, as in the real 3.8 MB m1n1: + # a reader that stops early must not fail the check under pipefail. (root / "usr/lib/asahi-boot/m1n1.bin").write_bytes( b"\x7fm1n1\x00" + b"\x00".join(s.encode() for s in strings) + b"\x00tail" + + bytes(range(256)) * 4096 ) pkg = self.tmp / "m1n1-aurora-test-1-aarch64.pkg.tar.zst" subprocess.run(["bsdtar", "--zstd", "-cf", str(pkg), "-C", str(root), "usr"], check=True) From 80e6adca571b16c211e8eabbf632d703cabffaab Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 07:54:59 -0700 Subject: [PATCH 1745/2680] tools: aurora-sep: say what should happen on each M3, and how to add one Rewrite step 11 of --agent-prompt as three cases, each with the lines the installer prints and what the screen should show: - A: the 16" M3 Pro (J516S) gets the handoff by default. - B: another M3 Pro model (the 14" J514S) is kernel-only until a --m3-handoff report adds it. Covers the owner's agreement and the macOS way back, and what to report for WORKS, STOPS and WRONG. - C: M3 and M3 Max stay kernel-only, and their reports decide which chip gets the handoff next. The prompt now starts with a pointer for M3 owners (Touch ID isn't supported there yet, so steps 2 to 5 don't apply). The checks in step 11 were run as written on the m3pro. An M3 install no longer ends with "run aurora-touchid-setup". It says what the reboot should show on the handoff and kernel-only paths. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit bc7149dbf4e2de7c7239dd79178cbbfd9ae8b627) --- tools/aurora-sep/install-aurora-sep.sh | 138 ++++++++++++++++--------- tools/aurora-sep/tests/test_m3_flow.py | 9 +- 2 files changed, 94 insertions(+), 53 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 3aaff655b7b94..323863889b59c 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -1164,7 +1164,15 @@ install_all() { if is_neo; then neo_radio_notice; fi pacman -Q linux-aurora libfprint aurora-touchid echo - if [[ -f $MODPROBE_CONF ]]; then + if [[ $M3_MODE == handoff ]]; then + say "Done. Reboot with someone watching: expect the Omarchy logo, the boot menu, then the + desktop on the built-in display at its native resolution. Touch ID is not supported + on M3 yet." + elif [[ $M3_MODE == kernel ]]; then + say "Done. Reboot: expect the desktop on the boot framebuffer. Touch ID is not supported + on M3 yet. How to help bring this M3 further: step 11 of the test plan that the + --agent-prompt command below prints." + elif [[ -f $MODPROBE_CONF ]]; then say "Done, read-only. Reboot; the SEP driver will attach and report without writing." echo " Enrolling a finger needs writes: delete $MODPROBE_CONF, reboot, then run aurora-touchid-setup." else @@ -1386,6 +1394,11 @@ SAFETY, NON-NEGOTIABLE - Do not paste key material, serial numbers, or the contents of mesa_calibration.bin into a report. +ON AN M3 (M3, M3 Pro, M3 Max): Touch ID is not supported there yet. Do steps +0 and 1, then go to step 11, which says what should happen on your model and +how to get it added. Steps 6, 7 and 9 apply as well, and step 8 without the +fingerprint. + 0. CONFIRM YOU HAVE THE CURRENT BUILD Results against a superseded build waste everyone's time, and a saved copy of this script installs its own packages forever. This script checks on @@ -1603,57 +1616,80 @@ SAFETY, NON-NEGOTIABLE Report "not tested" honestly rather than guessing, for any of the above. -11. M3 (M3, M3 PRO, M3 MAX) ONLY - Every M3 is experimental. The installer took one of two paths and said - which near the start; the file below records it too: - cat /var/lib/aurora-sep/m3-mode - - kernel: the kernel only. m1n1's boot.bin was left exactly as it was, - and the install ended with "M3: m1n1's boot.bin is unchanged". Expect - the desktop on the boot framebuffer (one fixed resolution, software - rendering). - - handoff: an M3 Pro model whose m1n1 display and GPU handoff has been - booted before. The installer replaced boot.bin, kept the old one as - m1n1/boot.bin.before- on the EFI partition, and printed the - macOS steps to put it back. Expect the built-in display at its native - resolution and the GPU in use. - Check and quote: - tr -d '\0' < /proc/device-tree/compatible; echo - ls /sys/class/drm/ - sudo dmesg | grep -iE 'asahi.*(gpu|firmware)|apple-dcp|t6030' | head -20 +11. M3 MACS (M3, M3 PRO, M3 MAX): WHAT SHOULD HAPPEN, AND HOW YOURS GETS ADDED + Every M3 is experimental, and every M3 report moves M3 support forward. + Touch ID is not supported on M3 yet: skip steps 2 to 5 and the fingerprint + part of step 8, and expect + "apple_sep: no platform profile for this SoC; the SEP stays disabled". + First find the model and chip, then follow the matching case below: + tr '\0' '\n' < /proc/device-tree/compatible | head -2 + cat /var/lib/aurora-sep/m3-mode # after an install: kernel or handoff + + A. apple,j516s + apple,t6030 (MacBook Pro 16" M3 Pro): SUPPORTED, handoff on. + The plain one-liner installs the kernel and replaces m1n1 with one + that hands the built-in display and the GPU over to Linux. While it + runs it prints: + M3 Pro (j516s, macOS 14.8.3 stub): installing m1n1 with the display and GPU handoff + then the macOS steps to put the old boot loader back (it is kept as + m1n1/boot.bin.before- on the EFI partition), and at the end: + m1n1's boot.bin is this release's m1n1 with the M3 Pro handoff switches + Reboot with the owner watching. Expected within about a minute: the + Omarchy logo, the boot menu, then the desktop on the built-in display + at 3456x2234, GPU accelerated. Then run the checks below and report. + + B. any other apple,t6030 model (MacBook Pro 14" M3 Pro, J514S): NOT ON + THE LIST YET. A REPORT FROM YOU IS HOW IT GETS ADDED. + The plain one-liner installs the kernel only (boot.bin untouched, boot + framebuffer) and says so: + M3 Pro (j514s): installing the kernel only, and boot.bin stays as it is. + Run the checks below on that first; it is a useful report on its own. + Then, to try the display and GPU handoff on this model: + - Only with the owner's explicit agreement, with the owner at the + Mac for the first boot, and only if the owner can start macOS on + this Mac (hold the power button at startup). If the new boot + loader does not start, macOS on the same Mac is the only way back. + - The installer itself refuses unless the Mac has the macOS 14.8.3 + system-firmware stub that the Omarchy installer gives every M3. + - Run: + curl -fsSL https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh | bash -s -- --m3-handoff + Keep the restore steps it prints (they name this Mac's EFI + partition). It must end with "m1n1's boot.bin is this release's + m1n1 with the M3 Pro handoff switches"; if it stops with an error + instead, nothing was changed: report the error. + - Reboot, with the owner watching. Then report exactly one of: + * WORKS: the desktop on the built-in display at native resolution. + Give the checks below, and again after one suspend/resume. Title + it " (): M3 handoff works". This report is what + adds the model to the list in the next release; until then a + plain re-run of the one-liner keeps the handoff on this Mac. + * STOPS: m1n1 text and no boot menu, or a black screen for more than + two minutes. The owner puts the old boot.bin back from macOS with + the printed steps. Report what the screen showed and when. + * WRONG: it boots, but the display or the GPU is wrong. Collect + sudo journalctl -b -k | grep -iE 'dcp|asahi|t6030|m1n1' + over SSH if you can, then run the installer with --uninstall + (it puts the stock m1n1 back) and reboot. + + C. apple,t8122 (MacBook Air 13"/15" M3, MacBook Pro 14" M3, iMac M3) or + apple,t6031 / apple,t6034 (M3 Max): KERNEL ONLY for now. + m1n1 has no display and GPU handoff for these chips yet, and + --m3-handoff refuses here. The one-liner installs the kernel only and + leaves boot.bin as it is. Expected: the desktop on the boot framebuffer + (one fixed resolution, software rendering). Report what works and + what does not: Wi-Fi, Bluetooth, keyboard, trackpad, audio, USB and + Thunderbolt, suspend/resume, battery. These reports decide which chip + gets the handoff next. + + Checks for every M3 (quote the output; on a kernel-only M3 the handoff + lines are expected to be missing, so say so): + tr -d '\0' < /proc/device-tree/chosen/asahi,m1n1-stage2-version; echo + ls /proc/device-tree/soc/dcp@*/apple,t6030-handoff 2>&1 + for c in /sys/class/drm/card*-*; do [ -e "$c/status" ] && echo "$c $(cat "$c/status") $(head -1 "$c/modes")"; done + sudo dmesg | grep -E 't6030-display|\[drm\] Initialized|GPU firmware|aop.*crash|apple_sep' | head -12 + nproc ls /proc/device-tree/soc/usb4-pcie-tunnel-0/pcie@730000000/pci@0,0/apple,tunable 2>&1 - Then: does it reach the desktop, and do Wi-Fi, Bluetooth, keyboard, - trackpad, audio and one suspend/resume work? With a Thunderbolt dock, do - its display and its PCIe devices (Ethernet, storage) come up? Report each - one, including "not tested". - - ADDING AN M3 PRO MODEL TO THE HANDOFF - Only an M3 Pro (apple,t6030: J514S 14", J516S 16") can try the handoff; - m1n1 has none for the M3 or M3 Max yet, and the installer refuses there. - The installer also refuses unless the Mac has the macOS 14.8.3 - system-firmware stub that the Omarchy installer gives every M3. - This REPLACES THE MAC'S BOOT LOADER. If the new one does not start, the - Mac stops before any boot menu, and the only way back is from macOS on - the same Mac. So: - - Do it only with the owner's explicit agreement, and only if the owner - can start macOS on this Mac (hold the power button at startup). - - Keep the restore steps the installer prints: they name this Mac's EFI - partition and the saved boot.bin.before-. - - Do not try it unattended or over SSH alone: someone must be at the Mac - for the first boot. - Then, with the owner watching: - curl -fsSL https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh | bash -s -- --m3-handoff - and reboot. Report one of: - - It boots to the desktop on the built-in display at native resolution: - give the checks above, and again after one suspend/resume. That - report is what adds the model to the installer's list. - - It stops in m1n1 (text on screen, no boot menu) or stays black: the - owner puts the old boot.bin back from macOS with the printed steps. - Report exactly what the screen showed and when. - - It boots, but the display or GPU is wrong: collect - sudo journalctl -b -k | grep -iE 'dcp|asahi|t6030|m1n1' - over SSH if you can, then run the installer with --uninstall, which - puts the stock m1n1 back, and reboot. - Title such a report " (): M3 handoff works" or "... fails". + In the report header, set **M3 path:** to kernel, handoff, or handoff + with --m3-handoff. HOW TO REPORT Open one issue per Mac at https://github.com/iconidentify/aurora-linux/issues diff --git a/tools/aurora-sep/tests/test_m3_flow.py b/tools/aurora-sep/tests/test_m3_flow.py index 9619c47d41ea4..ae7e491c7f579 100644 --- a/tools/aurora-sep/tests/test_m3_flow.py +++ b/tools/aurora-sep/tests/test_m3_flow.py @@ -221,7 +221,9 @@ def kept_copy(self): def test_fresh_m3_pro_gets_the_handoff(self): self.mac("j516s") - self.install() + out = self.install().stdout + self.assertIn("built-in display at its native resolution", out) + self.assertNotIn("run: aurora-touchid-setup", out) boot = self.boot.read_bytes() self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora6-1\n"), boot) self.assertTrue(boot.endswith(SWITCHES), boot) @@ -280,6 +282,8 @@ def assert_kernel_only(self, board, **kw): self.mac(board, **kw) before = self.boot.read_bytes() out = self.install().stdout + self.assertIn("Touch ID is not supported", out) + self.assertNotIn("run: aurora-touchid-setup", out) self.assertEqual(self.boot.read_bytes(), before) self.assertIn("boot.bin is unchanged", out) self.assertFalse([d for d in self.downloaded() if d.startswith("m1n1-aurora-")]) @@ -395,7 +399,8 @@ def test_empty_mode_file_does_not_break_uninstall(self): def test_m1_pro_unchanged(self): self.mac("j314s") - self.install() + out = self.install().stdout + self.assertIn("run: aurora-touchid-setup", out) boot = self.boot.read_bytes() self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora3-1\n"), boot) self.assertNotIn(b"chosen.", boot) From 1dc0036038ed9d799ae9536c5d966dccdc58d315 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 12:52:20 -0700 Subject: [PATCH 1746/2680] tools: aurora-sep: J514S display handoff test build (11.109-test) Not a release. A test installer for the 14" M3 Pro (J514S) display handoff, used with --m3-handoff: - kernel 7.1.12.aurora2-11.109 (custom/sep plus m3/j514s-handoff ce3cd9e45376): the display gate takes each Mac's PMP values from the boot loader, the J514S gets its panel, and the J514S needs both GPU thermal zones; - m1n1-aurora 1.6.1.aurora7: passes this Mac's PMP values in /chosen/asahi,t6030-pmp. M3_HANDOFF_BOARDS is unchanged, so a J514S still opts in with --m3-handoff. The tests now read VERSION and the m1n1 package names from the installer instead of hard-coding the 11.36 ones. They pass against this installer and the released 11.36.1 one. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit 399f1a46eb970bc17064e1a9f61bc785c7096567) --- tools/aurora-sep/install-aurora-sep.sh | 15 +++++++++------ tools/aurora-sep/tests/test_m3_flow.py | 5 ++++- tools/aurora-sep/tests/test_m3_handoff.py | 14 +++++++++++--- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 323863889b59c..b5f121d3e4a86 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -78,6 +78,9 @@ # USB-C or Thunderbolt display gets the HDMI port's pipeline when HDMI is idle, # and a dock display left waiting lights once a pipeline frees. M3 support is # experimental and kernel-only (see is_m3). +# 11.109-test (not a release): the M3 Pro display handoff on the 14" J514S. +# Kernel 11.109 takes each Mac's PMP values from the boot loader, and m1n1 +# aurora7 passes them (/chosen/asahi,t6030-pmp). For --m3-handoff on a J514S. # 11.36.1 changes only this script and adds an M3 m1n1: an M3 Pro model whose # m1n1 display and GPU handoff has been booted (the 16" J516S for now) gets # m1n1-aurora with it switched on, and the built-in display and GPU work. @@ -118,8 +121,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.36 -TAG=sep-7.1.12.aurora2-11.36.1 +VERSION=7.1.12.aurora2-11.109 +TAG=sep-7.1.12.aurora2-11.109-test # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -127,15 +130,15 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst cb5cb0800fafaaecde2dcb350044bc3afbfc7018ef2f54dd4422bc26c293c9ec" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst e21ab2add5491d404980f87bce168cfd05ee298e1fc0f12815d9ff68d12e6398" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst 036f8a20a94363c247443cb5809707b5e884bc80f11531723c7409d91f867815" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 9eb62c16a727d17c809a5eb5f847e4ad4adce211304a12f99ff936cf9c9487b5" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20261003-1-any.pkg.tar.zst 29b0360fac8c257d754e64bd1b9c33c487eb2595dd3c31e9138d7a476afa3d64" "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" ) # Only for an M3 on the handoff path (see m3_plan), in place of the m1n1-aurora # above: the same m1n1 plus the T6030 display and GPU handoff. -M3_M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora6-1-aarch64.pkg.tar.zst 16747a63b59815ba54f95c59ab97514fb7178aa459953aa4369effce90456af8" +M3_M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora7-1-aarch64.pkg.tar.zst 547d0d5f9daa283749d84e5a9c875fee06ce449eeb8252e41c11b3ce63e11184" PINNED="linux-aurora linux-aurora-headers libfprint m1n1-aurora" PIN_BEGIN="# >>> aurora-sep pin (remove with: install-aurora-sep.sh --uninstall)" PIN_END="# <<< aurora-sep pin" @@ -1548,7 +1551,7 @@ fingerprint. sudo dmesg | grep -E 'cold init done|link up after|translation fault|HC died' Any "translation fault" or "HC died" line is a failure to report. For a full report, after plugging the dock in: - curl -fsSLO https://raw.githubusercontent.com/iconidentify/aurora-linux/refs/tags/sep-7.1.12.aurora2-11.36.1/tools/aurora-tb/tb-pcie-report + curl -fsSLO https://raw.githubusercontent.com/iconidentify/aurora-linux/refs/tags/sep-7.1.12.aurora2-11.109-test/tools/aurora-tb/tb-pcie-report sudo sh tb-pcie-report --no-wait 6c. Across suspend: diff --git a/tools/aurora-sep/tests/test_m3_flow.py b/tools/aurora-sep/tests/test_m3_flow.py index ae7e491c7f579..23a1c4cbdc76e 100644 --- a/tools/aurora-sep/tests/test_m3_flow.py +++ b/tools/aurora-sep/tests/test_m3_flow.py @@ -9,12 +9,15 @@ from pathlib import Path import hashlib import os +import re import shutil import subprocess import tempfile import unittest INSTALLER = Path(__file__).resolve().parent.parent / "install-aurora-sep.sh" +# The installer's VERSION names the boot.bin copy it keeps on the EFI partition. +VERSION = re.search(r"^VERSION=(\S+)$", INSTALLER.read_text(), re.M).group(1) BOARDS = { "j516s": ["apple,j516s", "apple,t6030", "apple,arm-platform"], @@ -215,7 +218,7 @@ def downloaded(self): return [l.rsplit("/", 1)[1] for l in self.log().splitlines() if l.startswith("curl ")] def kept_copy(self): - return self.tmp / "esp/m1n1/boot.bin.before-7.1.12.aurora2-11.36" + return self.tmp / f"esp/m1n1/boot.bin.before-{VERSION}" # M3 Pro on the handoff path diff --git a/tools/aurora-sep/tests/test_m3_handoff.py b/tools/aurora-sep/tests/test_m3_handoff.py index 5a01315cb61c5..9e01b3da8b9e9 100644 --- a/tools/aurora-sep/tests/test_m3_handoff.py +++ b/tools/aurora-sep/tests/test_m3_handoff.py @@ -6,6 +6,7 @@ """ from pathlib import Path import gzip +import re import os import shutil import subprocess @@ -14,6 +15,13 @@ INSTALLER = Path(__file__).resolve().parent.parent / "install-aurora-sep.sh" +# Release names, read from the installer so the tests follow each release. +_SRC = INSTALLER.read_text() +VERSION = re.search(r"^VERSION=(\S+)$", _SRC, re.M).group(1) +M1N1_M1M2 = re.search(r'^ "(m1n1-aurora-\S+) ', _SRC, re.M).group(1) +M1N1_M3 = re.search(r'^M3_M1N1_PACKAGE="(\S+) ', _SRC, re.M).group(1) +KERNEL_PKG = f"linux-aurora-{VERSION}-aarch64.pkg.tar.zst" + OUR_FREEZE = ( "# >>> aurora-sep: keep this M3's boot.bin as it is (remove with: install-aurora-sep.sh --uninstall)\n" "M1N1_UPDATE_DISABLED=1\n" @@ -327,8 +335,8 @@ def packages(self, board, stub="14.8.3", try_=0): return [line.split()[0] for line in out.splitlines()] def test_packages_per_mac(self): - m1n1_m1m2 = "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst" - m1n1_m3 = "m1n1-aurora-1.6.1.aurora6-1-aarch64.pkg.tar.zst" + m1n1_m1m2 = M1N1_M1M2 + m1n1_m3 = M1N1_M3 for board, stub, try_, want in [("j314s", "13.5", 0, m1n1_m1m2), ("j293", "13.5", 0, m1n1_m1m2), ("j700", "26.4", 0, None), ("j613", "14.8.3", 0, None), ("j514c", "14.8.3", 0, None), ("j514s", "14.8.3", 0, None), @@ -338,7 +346,7 @@ def test_packages_per_mac(self): got = self.packages(board, stub, try_) m1n1 = [p for p in got if p.startswith("m1n1-aurora-")] self.assertEqual(m1n1, [want] if want else []) - self.assertIn("linux-aurora-7.1.12.aurora2-11.36-aarch64.pkg.tar.zst", got) + self.assertIn(KERNEL_PKG, got) self.assertEqual(len(got), 5 if want else 4) # Uninstall From c78e40b555b25ec94c552591849f680a0e6e259b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:56:15 -0700 Subject: [PATCH 1747/2680] tools: aurora-sep: M3 MacBook Air GPU handoff, opt-in with --m3-handoff The M3 MacBook Air (T8122: 13" J613, 15" J615) can take m1n1's GPU handoff, but only when its owner asks with --m3-handoff and the release carries an m1n1 for the Air: - is_m3_air finds the Airs. The 14" MacBook Pro M3 (J504) and the iMacs stay kernel-only, and --m3-handoff still refuses on them. - M3_AIR_M1N1_PACKAGE names the Air's m1n1 (aurora8, once it is built). It is empty here, so a plain run on an Air stays kernel-only and --m3-handoff stops with nothing installed. - The Air's /etc/m1n1.conf block carries chosen.asahi,t8122-gpu=1 only. chosen.asahi,t8122-dcp=1 joins it when M3_AIR_DCP is set to 1, once the Air's display handoff has been booted. - No Air is in M3_HANDOFF_BOARDS, so none gets the handoff by default. A test keeps it that way until a tester's report adds a board. - The package check, the boot.bin check and the messages follow the Mac's chip. An M3 Pro gets the same switches, package, messages and /etc/m1n1.conf block as before. - --agent-prompt step 11 gets case D for the Air: the kernel-only baseline first, then the opt-in with the owner at the Mac and the maintainer reachable, and what to report for WORKS, STOPS and WRONG. test_m3_air.py runs the Air path against test_m3_flow's fake Mac, whose helpers move to a base class. The refusal on a plain M3 now uses the J504, because --m3-handoff on an Air is the Air's opt-in. All 92 tests pass; the 11.109-test M3 tests pass unchanged except that one J613 row. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit 6e8493e47cfbbcf6810b6b594470423d3f323cb7) --- tools/aurora-sep/install-aurora-sep.sh | 263 +++++++++++++++--- tools/aurora-sep/tests/test_m3_air.py | 317 ++++++++++++++++++++++ tools/aurora-sep/tests/test_m3_flow.py | 11 +- tools/aurora-sep/tests/test_m3_handoff.py | 4 +- 4 files changed, 548 insertions(+), 47 deletions(-) create mode 100644 tools/aurora-sep/tests/test_m3_air.py diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index b5f121d3e4a86..21e3e55811d28 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -87,6 +87,9 @@ # Every other M3 stays kernel-only, and --m3-handoff lets an M3 Pro owner try # the handoff on a model not yet on the list (see --agent-prompt). M1, M2 and # the Neo are unchanged. +# The M3 MacBook Air (T8122, 13" J613 and 15" J615) can opt in to m1n1's GPU +# handoff with --m3-handoff once a release carries M3_AIR_M1N1_PACKAGE: the +# display stays on the boot framebuffer, and a plain run stays kernel-only. # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -139,6 +142,12 @@ PACKAGES=( # Only for an M3 on the handoff path (see m3_plan), in place of the m1n1-aurora # above: the same m1n1 plus the T6030 display and GPU handoff. M3_M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora7-1-aarch64.pkg.tar.zst 547d0d5f9daa283749d84e5a9c875fee06ce449eeb8252e41c11b3ce63e11184" +# Only for an M3 MacBook Air on the handoff path, in place of the m1n1-aurora +# above: m1n1-aurora with the T8122 GPU handoff (aurora8 is the first), as +# "file sha256". It may name the same package as M3_M1N1_PACKAGE once one m1n1 +# carries both. Empty until that package is published: an Air then stays +# kernel-only, and --m3-handoff on an Air stops with nothing installed. +M3_AIR_M1N1_PACKAGE="" PINNED="linux-aurora linux-aurora-headers libfprint m1n1-aurora" PIN_BEGIN="# >>> aurora-sep pin (remove with: install-aurora-sep.sh --uninstall)" PIN_END="# <<< aurora-sep pin" @@ -223,6 +232,16 @@ is_m3_pro() { tr '\0' '\n' <"$DT/compatible" 2>/dev/null | grep -qx 'apple,t6030' } +# The M3 MacBook Airs. The other T8122 Macs (the 14" MacBook Pro M3, J504, and +# the iMacs, J433 and J434) have no handoff path and stay kernel-only. +M3_AIR_BOARDS="j613 j615" +is_m3_air() { + local board + tr '\0' '\n' <"$DT/compatible" 2>/dev/null | grep -qx 'apple,t8122' || return 1 + board=$(this_board) + [[ -n $board && " $M3_AIR_BOARDS " == *" $board "* ]] +} + # M3 support is experimental. Every M3 gets linux-aurora; what happens to m1n1's # stage 2 (boot.bin: m1n1, device trees and U-Boot) depends on the model, and # m3_plan decides it before anything is downloaded (M3_MODE): @@ -237,6 +256,10 @@ is_m3_pro() { # iconidentify/m3-m1n1) stays off unless boot.bin ends with the # three M3_SWITCHES lines; update-m1n1 copies chosen.* lines from # /etc/m1n1.conf into every rebuild, so they survive updates. +# An M3 MacBook Air takes the same path with its own m1n1 +# (M3_AIR_M1N1_PACKAGE) and switches (m3_switches): the GPU +# handoff only, and only with --m3-handoff until its board is in +# M3_HANDOFF_BOARDS. # M1, M2 and the Neo keep the m1n1-aurora in PACKAGES: only an M3 on the # handoff path gets the newer m1n1. UPDATE_M1N1_CONF=/etc/default/update-m1n1 @@ -244,9 +267,17 @@ M3_FREEZE_BEGIN="# >>> aurora-sep: keep this M3's boot.bin as it is (remove with M3_FREEZE_END="# <<< aurora-sep: keep this M3's boot.bin as it is" M3_FROZEN_BY="" M3_BOOTBIN_SHA="" -# Models the handoff has been booted on, by us or by a tester's report. +# Models the handoff has been booted on, by us or by a tester's report. Only +# an M3 Pro (t6030) or an M3 MacBook Air (M3_AIR_BOARDS) listed here gets it +# by default; no Air is listed yet. M3_HANDOFF_BOARDS="j516s" M3_SWITCHES="chosen.asahi,t6030-gpu=1 chosen.asahi,t6030-dcp=1 chosen.asahi,t6030-dcpext=1" +# The M3 MacBook Air's switches. The GPU handoff alone for now: the display +# stays on the boot framebuffer. Set M3_AIR_DCP=1 to add the display handoff +# once m1n1's T8122 DCP handoff has been booted on an Air. +M3_AIR_GPU_SWITCH="chosen.asahi,t8122-gpu=1" +M3_AIR_DCP_SWITCH="chosen.asahi,t8122-dcp=1" +M3_AIR_DCP=0 # The handoff is tested with one macOS system-firmware stub only, 14.8.3 (GPU # firmware 14.8.3, DCP 14.7), which the Omarchy installer gives every M3. m1n1 # reports the stub's iBoot as asahi,iboot2-version. @@ -257,6 +288,8 @@ M1N1_CONF=/etc/m1n1.conf M1N1_BIN=/usr/lib/asahi-boot/m1n1.bin M1N1_CONF_BEGIN="# >>> aurora-sep: M3 Pro display and GPU handoff (remove with: install-aurora-sep.sh --uninstall)" M1N1_CONF_END="# <<< aurora-sep: M3 Pro display and GPU handoff" +M1N1_CONF_AIR_BEGIN="# >>> aurora-sep: M3 Air handoff (remove with: install-aurora-sep.sh --uninstall)" +M1N1_CONF_AIR_END="# <<< aurora-sep: M3 Air handoff" # The M3 bring-up's install-m3gpu.sh froze boot.bin with exactly these lines, # and left this marker when it created the file. M3GPU_FREEZE=( @@ -268,8 +301,8 @@ M3GPU_MARKER=/etc/default/.update-m1n1.created-by-m3gpu # none (not an M3), kernel or handoff; set by m3_plan, kept in $STATE/m3-mode # for --uninstall. M3_MODE=none -# Set by --m3-handoff: the owner asks for the handoff on an M3 Pro model that -# isn't in M3_HANDOFF_BOARDS yet. +# Set by --m3-handoff: the owner asks for the handoff on an M3 Pro or M3 +# MacBook Air model that isn't in M3_HANDOFF_BOARDS yet. M3_TRY=0 # Whether update-m1n1 would exit without building, judged the way it judges: @@ -358,7 +391,41 @@ m3_bootbin_report() { is_m3_handoff_board() { local board board=$(this_board) - is_m3_pro && [[ -n $board && " $M3_HANDOFF_BOARDS " == *" $board "* ]] + { is_m3_pro || is_m3_air; } && [[ -n $board && " $M3_HANDOFF_BOARDS " == *" $board "* ]] +} + +# The switch lines this Mac's handoff needs, separated by spaces: the T6030 +# ones, or the M3 Air's. +m3_switches() { + if ! is_m3_air; then + echo "$M3_SWITCHES" + elif [[ $M3_AIR_DCP == 1 ]]; then + echo "$M3_AIR_GPU_SWITCH $M3_AIR_DCP_SWITCH" + else + echo "$M3_AIR_GPU_SWITCH" + fi +} + +# This Mac's m1n1 package on the handoff path, as "file sha256". +m3_m1n1_package() { + if is_m3_air; then echo "$M3_AIR_M1N1_PACKAGE"; else echo "$M3_M1N1_PACKAGE"; fi +} + +# Whether this release carries an m1n1 for the M3 Air: a whole "file sha256" +# entry, not the empty placeholder. +m3_air_package_ok() { + [[ $M3_AIR_M1N1_PACKAGE =~ ^m1n1-aurora-[^[:space:]]+-aarch64\.pkg\.tar\.zst\ [0-9a-f]{64}$ ]] +} + +# What the handoff does on this Mac, for messages. +m3_handoff_name() { + if ! is_m3_air; then + echo "M3 Pro display and GPU handoff" + elif [[ $M3_AIR_DCP == 1 ]]; then + echo "M3 Air display and GPU handoff" + else + echo "M3 Air GPU handoff" + fi } # Why this Mac's system-firmware stub isn't the one the handoff is tested @@ -380,14 +447,17 @@ m3_stub_problem() { return 0 } -# True when a downloaded m1n1-aurora package carries all three handoff switches. +# True when a downloaded m1n1-aurora package knows every switch this Mac's +# handoff needs (m3_switches): asahi,t6030-gpu for chosen.asahi,t6030-gpu=1. m1n1_pkg_has_handoff() { local bin s rc=0 bin=$(mktemp) # One string per line, in a file: grep -q on a pipe would stop reading early # and fail the 3.8 MB tr with SIGPIPE under pipefail. { bsdtar -xOf "$1" usr/lib/asahi-boot/m1n1.bin | tr '\0' '\n' >"$bin"; } 2>/dev/null || rc=1 - for s in asahi,t6030-gpu asahi,t6030-dcp asahi,t6030-dcpext; do + for s in $(m3_switches); do + s=${s#chosen.} + s=${s%%=*} ((rc == 0)) && ! grep -qaxF "$s" "$bin" && rc=1 done rm -f "$bin" @@ -453,23 +523,29 @@ m3gpu_unfreeze() { say "Lifted the M3 bring-up's freeze on update-m1n1 (from install-m3gpu.sh)" } -# /etc/m1n1.conf without this script's switch block, on stdout. +# /etc/m1n1.conf without this script's switch block (M3 Pro or M3 Air), on +# stdout. m1n1_conf_without_switches() { [[ -f $M1N1_CONF ]] || return 0 - awk -v b="$M1N1_CONF_BEGIN" -v e="$M1N1_CONF_END" ' - $0 == b { skip = 1; next } - skip && $0 == e { skip = 0; next } + awk -v b="$M1N1_CONF_BEGIN" -v e="$M1N1_CONF_END" \ + -v ab="$M1N1_CONF_AIR_BEGIN" -v ae="$M1N1_CONF_AIR_END" ' + $0 == b || $0 == ab { skip = 1; next } + skip && ($0 == e || $0 == ae) { skip = 0; next } !skip' "$M1N1_CONF" } m3_switches_write() { - local tmp s + local tmp s begin=$M1N1_CONF_BEGIN end=$M1N1_CONF_END + if is_m3_air; then + begin=$M1N1_CONF_AIR_BEGIN + end=$M1N1_CONF_AIR_END + fi tmp=$(mktemp) m1n1_conf_without_switches >"$tmp" { - echo "$M1N1_CONF_BEGIN" - for s in $M3_SWITCHES; do echo "$s"; done - echo "$M1N1_CONF_END" + echo "$begin" + for s in $(m3_switches); do echo "$s"; done + echo "$end" } >>"$tmp" $sudo install -m 644 "$tmp" "$M1N1_CONF" rm -f "$tmp" @@ -477,7 +553,7 @@ m3_switches_write() { m3_switches_remove() { local tmp - [[ -f $M1N1_CONF ]] && grep -qxF "$M1N1_CONF_BEGIN" "$M1N1_CONF" || return 0 + [[ -f $M1N1_CONF ]] && grep -qxF -e "$M1N1_CONF_BEGIN" -e "$M1N1_CONF_AIR_BEGIN" "$M1N1_CONF" || return 0 tmp=$(mktemp) m1n1_conf_without_switches >"$tmp" if grep -q '[^[:space:]]' "$tmp"; then @@ -498,28 +574,50 @@ m3_recorded_mode() { # Decide the M3 path before anything is downloaded, so a Mac this release can't # set up as asked stops with nothing changed. m3_plan() { - local board problem + local board problem air=0 M3_MODE=none if ! is_m3; then - ((M3_TRY == 0)) || die "--m3-handoff is for an M3 Pro, and this Mac isn't an M3. Nothing was installed." + ((M3_TRY == 0)) || die "--m3-handoff is for an M3 Pro or an M3 MacBook Air, and this Mac isn't an M3. Nothing was installed." return 0 fi board=$(this_board) + if is_m3_air; then air=1; fi # An update never takes the handoff away again: once a Mac has it (listed, or # tried with --m3-handoff), a plain run keeps it, and its checks still apply. if ((M3_TRY == 0)) && [[ $(m3_recorded_mode) == handoff ]]; then M3_TRY=1 - say "M3 ($board): this Mac has m1n1's display and GPU handoff from an earlier install; keeping it" + if ((air)); then + say "M3 MacBook Air ($board): this Mac has m1n1's GPU handoff from an earlier install; keeping it" + else + say "M3 ($board): this Mac has m1n1's display and GPU handoff from an earlier install; keeping it" + fi fi M3_MODE=kernel - if ! is_m3_pro; then - ((M3_TRY == 0)) || die "--m3-handoff is for an M3 Pro (t6030); this M3 ($board) has no - display and GPU handoff in m1n1 yet. Nothing was installed." + if ((air)); then + # Never by default: a plain run gives an Air the handoff only once its + # board is in M3_HANDOFF_BOARDS, and never without this release's m1n1. + if ! m3_air_package_ok; then + ((M3_TRY == 0)) || die "--m3-handoff: this release has no m1n1 with the M3 MacBook Air GPU + handoff yet, so it can't switch it on. Nothing was installed. (--uninstall puts the stock + m1n1 back on an Air that has the handoff from an earlier test build.)" + say "M3 MacBook Air ($board): installing the kernel only. This release has no m1n1 with the + GPU handoff for the Air, so boot.bin stays as it is and the display runs on the boot + framebuffer." + return 0 + fi + if ! is_m3_handoff_board && ((M3_TRY == 0)); then + say "M3 MacBook Air ($board): installing the kernel only, and boot.bin stays as it is. m1n1's + GPU handoff for the Air is being tested and is not switched on by default. To help test it + (it replaces this Mac's boot loader), see case D in the M3 section of: bash -s -- --agent-prompt" + return 0 + fi + elif ! is_m3_pro; then + ((M3_TRY == 0)) || die "--m3-handoff is for an M3 Pro (t6030) or an M3 MacBook Air (j613, j615); + this M3 ($board) has no display and GPU handoff in m1n1 yet. Nothing was installed." say "M3 ($board): installing the kernel only. m1n1 has no display and GPU handoff for this chip yet, so boot.bin stays as it is and the display runs on the boot framebuffer." return 0 - fi - if ! is_m3_handoff_board && ((M3_TRY == 0)); then + elif ! is_m3_handoff_board && ((M3_TRY == 0)); then say "M3 Pro ($board): installing the kernel only, and boot.bin stays as it is. The display and GPU handoff in m1n1 hasn't been booted on this model yet. To try it (it replaces this Mac's boot loader), see the M3 section of: bash -s -- --agent-prompt" @@ -529,23 +627,32 @@ m3_plan() { if [[ -n $problem ]]; then ((M3_TRY == 0)) || die "--m3-handoff: the handoff is only tested with the macOS $M3_STUB_VERSION system-firmware stub, and this Mac differs: $problem. Nothing was installed." - warn "the M3 Pro display and GPU handoff is only tested with the macOS $M3_STUB_VERSION + warn "the $(m3_handoff_name) is only tested with the macOS $M3_STUB_VERSION system-firmware stub, and this Mac differs: $problem. Installing the kernel only; boot.bin stays as it is." return 0 fi if update_m1n1_frozen_by_others; then die "$UPDATE_M1N1_CONF sets M1N1_UPDATE_DISABLED, and not from this script or the M3 - bring-up's install-m3gpu.sh. Switching the M3 Pro handoff on needs m1n1's boot.bin rebuilt. - Remove that line and run this again. Nothing was installed." + bring-up's install-m3gpu.sh. Switching the $(m3_handoff_name) on needs m1n1's boot.bin + rebuilt. Remove that line and run this again. Nothing was installed." fi if update_m1n1_customised; then die "$UPDATE_M1N1_CONF points update-m1n1 at its own m1n1, U-Boot, config or target - (M1N1=, SOURCE=, U_BOOT=, CONFIG= or TARGET=). The M3 Pro handoff needs boot.bin built - from this release's m1n1. Remove those lines and run this again. Nothing was installed." + (M1N1=, SOURCE=, U_BOOT=, CONFIG= or TARGET=). The $(m3_handoff_name) needs boot.bin + built from this release's m1n1. Remove those lines and run this again. Nothing was installed." fi M3_MODE=handoff - if is_m3_handoff_board; then + if ((air)); then + if is_m3_handoff_board; then + say "M3 MacBook Air ($board, macOS $M3_STUB_VERSION stub): installing m1n1 with the $(m3_handoff_name)" + else + warn "--m3-handoff: trying m1n1's GPU handoff on an M3 MacBook Air ($board). It is in testing + and not on by default. This replaces the Mac's boot loader; the steps to put the old one + back from macOS follow. The display stays on the boot framebuffer, and the desktop keeps + rendering in software: this is for testing the GPU handoff, not a faster desktop." + fi + elif is_m3_handoff_board; then say "M3 Pro ($board, macOS $M3_STUB_VERSION stub): installing m1n1 with the display and GPU handoff" else warn "--m3-handoff: trying m1n1's display and GPU handoff on an M3 Pro model ($board) nobody @@ -558,20 +665,21 @@ m3_plan() { # appends /etc/m1n1.conf's lines straight after the gzipped U-Boot, so the # first one has no newline in front: look for the block, not whole lines. m3_verify_bootbin() { - local target tail block size - target=$(esp_bootbin) || die "could not find m1n1's boot.bin to check the M3 Pro switches in" + local target tail block size kind="M3 Pro" + if is_m3_air; then kind="M3 Air"; fi + target=$(esp_bootbin) || die "could not find m1n1's boot.bin to check the $kind switches in" size=$(stat -c %s "$M1N1_BIN") $sudo cmp -s -n "$size" "$M1N1_BIN" "$target" || die "$target does not start with this release's m1n1 ($M1N1_BIN), so it was not rebuilt. The boot loader this Mac booted with is kept as m1n1/boot.bin.before-$VERSION on the EFI partition and in $STATE/boot.bin.saved. Please report it before rebooting." - block=$(tr ' ' '\n' <<<"$M3_SWITCHES") + block=$(m3_switches | tr ' ' '\n') tail=$($sudo tail -c 1024 "$target" | tr -d '\0') [[ $tail == *"$block"* ]] || - die "the rebuilt $target does not carry the M3 Pro switch lines. The boot loader this Mac + die "the rebuilt $target does not carry the $kind switch lines. The boot loader this Mac booted with is kept as m1n1/boot.bin.before-$VERSION on the EFI partition and in $STATE/boot.bin.saved. Please report it before rebooting." - say "m1n1's boot.bin is this release's m1n1 with the M3 Pro handoff switches" + say "m1n1's boot.bin is this release's m1n1 with the $kind handoff switches" } # --uninstall on a bring-up Mac: update-m1n1 has just rebuilt boot.bin from the @@ -1037,15 +1145,16 @@ snapshot_boot_state() { } # The packages this Mac gets, one "file sha256" per line: PACKAGES, except -# that a Neo keeps its own m1n1, and an M3 gets M3_M1N1_PACKAGE on the handoff -# path and no m1n1 otherwise. Needs m3_plan first. +# that a Neo keeps its own m1n1, and an M3 gets its handoff m1n1 +# (m3_m1n1_package) on the handoff path and no m1n1 otherwise. Needs m3_plan +# first. packages_for_this_mac() { local entry for entry in "${PACKAGES[@]}"; do if [[ $entry == m1n1-aurora-* ]] && { is_neo || [[ $M3_MODE != none ]]; }; then continue; fi echo "$entry" done - if [[ $M3_MODE == handoff ]]; then echo "$M3_M1N1_PACKAGE"; fi + if [[ $M3_MODE == handoff ]]; then m3_m1n1_package; fi return 0 } @@ -1074,15 +1183,20 @@ install_all() { Please report it with the file name above." [[ $(sha256sum "$work/$file" | cut -d' ' -f1) == "$sha" ]] || die "$file does not match its published checksum" done - if [[ $M3_MODE == handoff ]] && ! m1n1_pkg_has_handoff "$work/${M3_M1N1_PACKAGE%% *}"; then - die "this release's M3 m1n1 has no M3 Pro display and GPU handoff, so it can't switch it + if [[ $M3_MODE == handoff ]]; then + entry=$(m3_m1n1_package) + m1n1_pkg_has_handoff "$work/${entry%% *}" || + die "this release's M3 m1n1 has no $(m3_handoff_name), so it can't switch it on. Nothing was installed. Please report it at https://github.com/iconidentify/aurora-linux/issues" fi snapshot "aurora-sep $VERSION" # Before pacman's update-m1n1 hook rebuilds boot.bin below. is_neo || unproven_board_backup - if [[ $M3_MODE == handoff ]]; then + if [[ $M3_MODE == handoff ]] && is_m3_air; then + keep_bootbin_on_esp "This replaces the boot loader of this M3 MacBook Air ($(this_board)) + with m1n1-aurora and its GPU handoff." + elif [[ $M3_MODE == handoff ]]; then keep_bootbin_on_esp "This replaces the boot loader of this M3 Pro ($(this_board)) with m1n1-aurora and its display and GPU handoff." fi @@ -1167,7 +1281,12 @@ install_all() { if is_neo; then neo_radio_notice; fi pacman -Q linux-aurora libfprint aurora-touchid echo - if [[ $M3_MODE == handoff ]]; then + if [[ $M3_MODE == handoff ]] && is_m3_air && [[ $M3_AIR_DCP != 1 ]]; then + say "Done. Reboot with someone watching: expect the Omarchy logo, the boot menu, then the + desktop on the boot framebuffer, as before. The GPU handoff is for testing: the desktop + still renders in software. Touch ID is not supported on M3 yet. What to check and + report: case D of step 11 of the test plan that the --agent-prompt command below prints." + elif [[ $M3_MODE == handoff ]]; then say "Done. Reboot with someone watching: expect the Omarchy logo, the boot menu, then the desktop on the built-in display at its native resolution. Touch ID is not supported on M3 yet." @@ -1627,6 +1746,7 @@ fingerprint. First find the model and chip, then follow the matching case below: tr '\0' '\n' < /proc/device-tree/compatible | head -2 cat /var/lib/aurora-sep/m3-mode # after an install: kernel or handoff + On an M3 MacBook Air, follow case D. A. apple,j516s + apple,t6030 (MacBook Pro 16" M3 Pro): SUPPORTED, handoff on. The plain one-liner installs the kernel and replaces m1n1 with one @@ -1673,9 +1793,10 @@ fingerprint. over SSH if you can, then run the installer with --uninstall (it puts the stock m1n1 back) and reboot. - C. apple,t8122 (MacBook Air 13"/15" M3, MacBook Pro 14" M3, iMac M3) or - apple,t6031 / apple,t6034 (M3 Max): KERNEL ONLY for now. - m1n1 has no display and GPU handoff for these chips yet, and + C. apple,t8122 other than the MacBook Air (MacBook Pro 14" M3, J504; + iMac M3, J433/J434) or apple,t6031 / apple,t6034 (M3 Max): KERNEL + ONLY for now. + m1n1 has no display and GPU handoff for these Macs yet, and --m3-handoff refuses here. The one-liner installs the kernel only and leaves boot.bin as it is. Expected: the desktop on the boot framebuffer (one fixed resolution, software rendering). Report what works and @@ -1683,6 +1804,60 @@ fingerprint. Thunderbolt, suspend/resume, battery. These reports decide which chip gets the handoff next. + D. apple,j613 or apple,j615 + apple,t8122 (MacBook Air 13" or 15" M3): + KERNEL ONLY BY DEFAULT. THE GPU HANDOFF IS AN OPT-IN TEST. + The plain one-liner installs the kernel only, as in case C, and says so: + M3 MacBook Air (j613): installing the kernel only ... + Run the checks below on that first and report it: it is the baseline + every later Air test is compared with. + Then, to test m1n1's GPU handoff on this Air: + - What it does: it replaces this Mac's boot loader (m1n1) with one + that starts the Air's GPU for Linux. The screen does not change: + the display stays on the boot framebuffer (one fixed resolution), + and the desktop still renders in software, because no Mesa drives + this GPU yet. Success means the kernel finds a working GPU, not a + faster desktop. + - Only with the owner's explicit agreement, with the owner at the + Mac for the first boot, with the Aurora maintainer told first and + reachable while it runs, and only if the owner can start macOS on + this Mac (hold the power button at startup). If the new boot + loader does not start, macOS on the same Mac is the only way back. + - If the installer says this release has no m1n1 with the Air's GPU + handoff, the test build is not out yet: stop there. It also + refuses unless the Mac has the macOS 14.8.3 system-firmware stub + that the Omarchy installer gives every M3. + - Run: + curl -fsSL https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh | bash -s -- --m3-handoff + It warns "--m3-handoff: trying m1n1's GPU handoff on an M3 MacBook + Air", then prints the restore steps: keep them, they name this + Mac's EFI partition. It must end with "m1n1's boot.bin is this + release's m1n1 with the M3 Air handoff switches"; if it stops with + an error instead, nothing was changed: report the error. + - Reboot, with the owner watching. Expected within about a minute: + the Omarchy logo, the boot menu, then the same desktop as before. + Then report exactly one of: + * WORKS: the desktop comes up as before. On a build whose kernel + drives the Air's GPU (its release notes say so), the kernel has + also taken the GPU: a render node, renderD128, is listed. + ls -l /dev/dri/ + sudo dmesg | grep -iE 'asahi|gpu|g15|t8122' | head -40 + Give the checks below too, and again after one suspend/resume. + Title it " (): M3 Air GPU handoff works". + * STOPS: m1n1 text and no boot menu, or a black screen for more + than two minutes. The owner puts the old boot.bin back from + macOS with the printed steps. Report what the screen showed and + when. + * WRONG: it boots, but something from the baseline no longer works + (keyboard, Wi-Fi, suspend), a build that should drive the GPU + shows no render node, or the kernel log shows GPU errors. Collect + sudo journalctl -b -k | grep -iE 'asahi|gpu|g15|t8122|m1n1' + over SSH if you can, then run the installer with --uninstall + (it puts the stock m1n1 back) and reboot. + - Do not start games or GPU programs: nothing on the desktop uses + this GPU yet, and a GPU fault can freeze the Mac until a reboot. + - Later plain runs of the one-liner keep the handoff on this Air; + --uninstall takes it off. + Checks for every M3 (quote the output; on a kernel-only M3 the handoff lines are expected to be missing, so say so): tr -d '\0' < /proc/device-tree/chosen/asahi,m1n1-stage2-version; echo diff --git a/tools/aurora-sep/tests/test_m3_air.py b/tools/aurora-sep/tests/test_m3_air.py new file mode 100644 index 0000000000000..2db150bb5726c --- /dev/null +++ b/tools/aurora-sep/tests/test_m3_air.py @@ -0,0 +1,317 @@ +"""The installer's M3 MacBook Air (T8122) path, against the fake Mac of test_m3_flow. + +An Air stays kernel-only unless its owner asks for m1n1's GPU handoff with +--m3-handoff, and then only when the release carries M3_AIR_M1N1_PACKAGE. +The handoff writes chosen.asahi,t8122-gpu=1 (and chosen.asahi,t8122-dcp=1 +only with M3_AIR_DCP=1) to /etc/m1n1.conf. Every case sets +M3_AIR_M1N1_PACKAGE itself, so the tests don't depend on what the installer +ships with. +""" +from pathlib import Path +import re +import unittest + +import test_m3_flow as flow + +INSTALLER = flow.INSTALLER +SRC = INSTALLER.read_text() + +AURORA8 = "m1n1-aurora-1.6.1.aurora8-1-aarch64.pkg.tar.zst" +AIR_GPU = b"chosen.asahi,t8122-gpu=1\n" +AIR_DCP = b"chosen.asahi,t8122-dcp=1\n" + + +def shell_value(name): + return re.search(rf'^{name}="([^"]*)"$', SRC, re.M).group(1) + + +class M3AirTest(flow.M3FlowBase): + def setUp(self): + super().setUp() + # aurora8: the T6030 handoff plus the T8122 GPU and display switches. + self.fixture(AURORA8, None, ["asahi,t6030-gpu", "asahi,t6030-dcp", "asahi,t6030-dcpext", + "asahi,t8122-gpu", "asahi,t8122-dcp"]) + self.air_pkg = f"{AURORA8} {self.shas[AURORA8]}" + + def sh(self, body, pkg=None, dcp=0, boards=None, check=True): + pkg = self.air_pkg if pkg is None else pkg + pre = f'M3_AIR_M1N1_PACKAGE="{pkg}"\nM3_AIR_DCP={dcp}\n' + if boards is not None: + pre += f'M3_HANDOFF_BOARDS="{boards}"\n' + return self.run_sh(pre + body, check=check) + + def plan(self, try_=0, **kw): + out = self.sh(f"M3_TRY={try_}\nm3_plan >/dev/null 2>&1\necho $M3_MODE", **kw).stdout + return out.strip() + + def air_install(self, try_=0, check=True, **kw): + return self.sh(f"M3_TRY={try_}\ninstall_all", check=check, **kw) + + # Which Macs are Airs + + def test_only_the_airs(self): + flow.BOARDS["j433"] = ["apple,j433", "apple,t8122", "apple,arm-platform"] + self.addCleanup(flow.BOARDS.pop, "j433") + for board, want in [("j613", "yes"), ("j615", "yes"), ("j504", "no"), ("j433", "no"), + ("j516s", "no"), ("j514s", "no"), ("j314s", "no")]: + with self.subTest(board=board): + self.mac(board) + self.assertEqual(self.sh("is_m3_air && echo yes || echo no").stdout.strip(), want) + + def test_listing_another_t8122_mac_gives_it_nothing(self): + # Only M3 Pros and Airs have a handoff path; a listed J504 stays out. + self.mac("j504") + out = self.sh("is_m3_handoff_board && echo yes || echo no", boards="j516s j504").stdout + self.assertEqual(out.strip(), "no") + self.assertEqual(self.plan(boards="j516s j504"), "kernel") + + # Switches + + def test_switches(self): + self.mac("j613") + self.assertEqual(self.sh("m3_switches").stdout.strip(), "chosen.asahi,t8122-gpu=1") + self.assertEqual(self.sh("m3_switches", dcp=1).stdout.strip(), + "chosen.asahi,t8122-gpu=1 chosen.asahi,t8122-dcp=1") + # The M3 Pro keeps its three, whatever the Air's settings. + self.mac("j516s") + self.assertEqual(self.sh("m3_switches", dcp=1).stdout.strip(), shell_value("M3_SWITCHES")) + + def test_dcp_stays_off_in_the_installer(self): + self.assertEqual(re.search(r"^M3_AIR_DCP=(\S+)$", SRC, re.M).group(1), "0") + self.assertEqual(shell_value("M3_AIR_GPU_SWITCH"), "chosen.asahi,t8122-gpu=1") + self.assertEqual(shell_value("M3_AIR_DCP_SWITCH"), "chosen.asahi,t8122-dcp=1") + + def test_no_air_on_by_default(self): + # Every Air needs --m3-handoff until a tester's report adds its board. + # Listing one is a release decision: change this test with it. + listed = shell_value("M3_HANDOFF_BOARDS").split() + airs = shell_value("M3_AIR_BOARDS").split() + self.assertEqual(sorted(airs), ["j613", "j615"]) + self.assertFalse(set(listed) & set(airs), listed) + + def test_a_listed_air_needs_its_package(self): + listed = set(shell_value("M3_HANDOFF_BOARDS").split()) + if listed & set(shell_value("M3_AIR_BOARDS").split()): + self.mac("j613") + self.assertEqual(self.run_sh("m3_air_package_ok && echo yes || echo no").stdout.strip(), "yes") + + def test_package_placeholder(self): + self.mac("j613") + good = "m1n1-aurora-1.6.1.aurora8-1-aarch64.pkg.tar.zst " + "a" * 64 + for pkg, want in [("", "no"), ("TODO", "no"), ("m1n1-aurora-1.6.1.aurora8-1-aarch64.pkg.tar.zst", "no"), + ("m1n1-aurora-1.6.1.aurora8-1-aarch64.pkg.tar.zst abc", "no"), + ("linux-aurora-x-aarch64.pkg.tar.zst " + "a" * 64, "no"), + (good + " extra", "no"), (good, "yes")]: + with self.subTest(pkg=pkg): + self.assertEqual(self.sh("m3_air_package_ok && echo yes || echo no", pkg=pkg).stdout.strip(), want) + + # Plan + + def test_plan(self): + for board, stub, try_, pkg, boards, want in [ + ("j613", "14.8.3", 0, None, None, "kernel"), # never by default + ("j615", "14.8.3", 0, None, None, "kernel"), + ("j613", "14.8.3", 0, "", None, "kernel"), # no Air m1n1 in this release + ("j613", "14.8.3", 1, None, None, "handoff"), # the owner asked + ("j615", "14.8.3", 1, None, None, "handoff"), + ("j613", "14.8.3", 0, None, "j516s j613", "handoff"), # listed later + ("j613", "14.8.3", 0, "", "j516s j613", "kernel"), # listed, but no m1n1 + ("j613", "15.6", 0, None, "j516s j613", "kernel"), # listed, other stub + ]: + with self.subTest(board=board, stub=stub, try_=try_, pkg=pkg, boards=boards): + self.mac(board, stub=stub) + self.assertEqual(self.plan(try_=try_, pkg=pkg, boards=boards), want) + + def test_plan_messages(self): + self.mac("j613") + self.assertIn("case D", self.sh("m3_plan").stdout) + self.assertIn("has no m1n1 with the", self.sh("m3_plan", pkg="").stdout) + err = self.sh("M3_TRY=1\nm3_plan").stderr + self.assertIn("in testing", err) + self.assertIn("desktop keeps", err) + + def test_plan_try_refused(self): + for board, stub, pkg, why in [("j613", "14.8.3", "", "no m1n1 with the M3 MacBook Air GPU"), + ("j613", "15.6", None, "stub is 15.6"), + ("j504", "14.8.3", None, "M3 MacBook Air (j613, j615)")]: + with self.subTest(board=board, stub=stub, pkg=pkg): + self.mac(board, stub=stub) + proc = self.sh("M3_TRY=1\nm3_plan", pkg=pkg, check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn(why, proc.stderr) + self.assertIn("Nothing was installed", proc.stderr) + + # /etc/m1n1.conf + + def test_conf_block(self): + self.mac("j613") + self.m1n1_conf.write_text("display=2560x1600\n") + self.sh("m3_switches_write\nm3_switches_write") + lines = self.m1n1_conf.read_text().splitlines() + self.assertEqual(lines[0], "display=2560x1600") + self.assertEqual([l for l in lines if l.startswith("chosen.")], ["chosen.asahi,t8122-gpu=1"]) + self.assertIn("# >>> aurora-sep: M3 Air handoff (remove with: install-aurora-sep.sh --uninstall)", lines) + self.sh("m3_switches_remove") + self.assertEqual(self.m1n1_conf.read_text(), "display=2560x1600\n") + + def test_conf_block_replaces_an_m3_pro_block(self): + # A T6030 block in the file (say, copied from another Mac) never + # survives next to the Air's. + self.mac("j516s") + self.sh("m3_switches_write") + self.mac("j613") + self.sh("m3_switches_write") + chosen = [l for l in self.m1n1_conf.read_text().splitlines() if l.startswith("chosen.")] + self.assertEqual(chosen, ["chosen.asahi,t8122-gpu=1"]) + + def test_m3_pro_block_unchanged(self): + self.mac("j516s") + self.sh("m3_switches_write", dcp=1) + text = self.m1n1_conf.read_text() + self.assertTrue(text.startswith("# >>> aurora-sep: M3 Pro display and GPU handoff"), text) + self.assertNotIn("t8122", text) + + # Package checks + + def test_package_check(self): + pkgs = self.tmp / "pkgs" + self.mac("j613") + self.assertEqual(self.sh(f"m1n1_pkg_has_handoff '{pkgs / AURORA8}' && echo yes").stdout.strip(), "yes") + # aurora6 has only the T6030 switches. + self.assertEqual(self.sh(f"m1n1_pkg_has_handoff '{pkgs / flow.AURORA6}' && echo yes || echo no") + .stdout.strip(), "no") + self.fixture("m1n1-gpu-only.pkg.tar.zst", None, ["asahi,t8122-gpu"]) + only = pkgs / "m1n1-gpu-only.pkg.tar.zst" + self.assertEqual(self.sh(f"m1n1_pkg_has_handoff '{only}' && echo yes || echo no").stdout.strip(), "yes") + self.assertEqual(self.sh(f"m1n1_pkg_has_handoff '{only}' && echo yes || echo no", dcp=1) + .stdout.strip(), "no") + # The M3 Pro still needs its own three. + self.mac("j516s") + self.assertEqual(self.sh(f"m1n1_pkg_has_handoff '{only}' && echo yes || echo no").stdout.strip(), "no") + + def test_verify_bootbin(self): + self.mac("j613") + self.boot.write_bytes(b"M1N1:original\nDTBS:x\nUBOOT" + AIR_GPU) + (self.fake / "m1n1.bin").write_text("M1N1:original\n") + self.assertIn("M3 Air handoff switches", self.sh("m3_verify_bootbin").stdout) + self.boot.write_bytes(b"M1N1:original\nDTBS:x\nUBOOT" + flow.SWITCHES) + proc = self.sh("m3_verify_bootbin", check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("M3 Air switch lines", proc.stderr) + + def test_packages(self): + for board, try_, pkg, want in [("j613", 0, None, None), ("j613", 1, None, AURORA8), + ("j613", 0, "", None), ("j516s", 0, None, flow.AURORA6)]: + with self.subTest(board=board, try_=try_, pkg=pkg): + self.mac(board) + out = self.sh(f"M3_TRY={try_}\nm3_plan >/dev/null 2>&1\npackages_for_this_mac", pkg=pkg).stdout + got = [line.split()[0] for line in out.splitlines()] + self.assertEqual([p for p in got if p.startswith("m1n1-aurora-")], [want] if want else []) + self.assertEqual(len(got), 5 if want else 4) + + # install_all and uninstall_all + + def test_plain_run_is_kernel_only(self): + self.mac("j613") + before = self.boot.read_bytes() + out = self.air_install().stdout + self.assertIn("case D", out) + self.assertIn("boot.bin is unchanged", out) + self.assertEqual(self.boot.read_bytes(), before) + self.assertFalse([d for d in self.downloaded() if d.startswith("m1n1-aurora-")]) + self.assertNotIn("update-m1n1 rebuilt", self.log()) + self.assertEqual((self.state / "m3-mode").read_text().strip(), "kernel") + self.assertFalse(self.m1n1_conf.exists()) + self.sh("uninstall_all") + self.assertEqual(self.boot.read_bytes(), before) + self.assertFalse(self.update_conf.exists()) + + def test_handoff_and_back(self): + self.mac("j613") + proc = self.air_install(try_=1) + self.assertIn("still renders in software", proc.stdout) + self.assertIn("M3 Air handoff switches", proc.stdout) + self.assertIn("M3 MacBook Air (j613)", proc.stderr) + boot = self.boot.read_bytes() + self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora8-1\n"), boot) + self.assertTrue(boot.endswith(b"UBOOT" + AIR_GPU), boot) + self.assertNotIn(b"t6030", boot) + self.assertNotIn(b"t8122-dcp", boot) + self.assertEqual(self.downloaded().count(AURORA8), 1) + self.assertNotIn(flow.AURORA3, self.downloaded()) + self.assertNotIn(flow.AURORA6, self.downloaded()) + self.assertEqual(self.kept_copy().read_bytes(), b"M1N1:original\n") + self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff") + self.assertNotIn("M1N1_UPDATE_DISABLED", self.update_conf.read_text()) + + self.sh("uninstall_all") + boot = self.boot.read_bytes() + self.assertTrue(boot.startswith(b"M1N1:m1n1-stock\n"), boot) + self.assertNotIn(b"chosen.", boot) + self.assertFalse(self.m1n1_conf.exists()) + self.assertFalse(self.state.exists()) + + def test_rerun_keeps_the_handoff(self): + self.mac("j615") + self.air_install(try_=1) + out = self.air_install().stdout + self.assertIn("keeping it", out) + self.assertTrue(self.boot.read_bytes().endswith(AIR_GPU)) + self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff") + + def test_rerun_without_an_air_package_stops(self): + # A later build without the Air's m1n1 never drops a tested Air back + # to kernel-only under a boot.bin it can't rebuild. + self.mac("j613") + self.air_install(try_=1) + before = self.boot.read_bytes() + proc = self.air_install(pkg="", check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("Nothing was installed", proc.stderr) + self.assertIn("--uninstall", proc.stderr) + self.assertEqual(self.boot.read_bytes(), before) + + def test_display_switch(self): + self.mac("j613") + self.air_install(try_=1, dcp=1) + self.assertTrue(self.boot.read_bytes().endswith(b"UBOOT" + AIR_GPU + AIR_DCP)) + + def test_refused_without_an_air_package(self): + self.mac("j613") + before = self.boot.read_bytes() + proc = self.air_install(try_=1, pkg="", check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("Nothing was installed", proc.stderr) + self.assertEqual(self.boot.read_bytes(), before) + self.assertNotIn("pacman -U", self.log()) + self.assertFalse(self.kept_copy().exists()) + + def test_refused_with_an_m1n1_that_lacks_the_switch(self): + self.mac("j613") + before = self.boot.read_bytes() + proc = self.air_install(try_=1, pkg=f"{flow.AURORA6} {self.shas[flow.AURORA6]}", check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("no M3 Air GPU handoff", proc.stderr) + self.assertEqual(self.boot.read_bytes(), before) + self.assertNotIn("pacman -U", self.log()) + + def test_m3_pro_unchanged_by_air_settings(self): + self.mac("j516s") + out = self.air_install(dcp=1).stdout + self.assertIn("built-in display at its native resolution", out) + boot = self.boot.read_bytes() + self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora6-1\n"), boot) + self.assertTrue(boot.endswith(flow.SWITCHES), boot) + self.assertNotIn(AURORA8, self.downloaded()) + + def test_m1_unchanged_by_air_settings(self): + self.mac("j314s") + self.air_install(dcp=1) + boot = self.boot.read_bytes() + self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora3-1\n"), boot) + self.assertNotIn(b"chosen.", boot) + self.assertFalse(self.m1n1_conf.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/aurora-sep/tests/test_m3_flow.py b/tools/aurora-sep/tests/test_m3_flow.py index 23a1c4cbdc76e..7d350b4814c06 100644 --- a/tools/aurora-sep/tests/test_m3_flow.py +++ b/tools/aurora-sep/tests/test_m3_flow.py @@ -23,6 +23,8 @@ "j516s": ["apple,j516s", "apple,t6030", "apple,arm-platform"], "j514s": ["apple,j514s", "apple,t6030", "apple,arm-platform"], "j613": ["apple,j613", "apple,t8122", "apple,arm-platform"], + "j615": ["apple,j615", "apple,t8122", "apple,arm-platform"], + "j504": ["apple,j504", "apple,t8122", "apple,arm-platform"], "j314s": ["apple,j314s", "apple,t6000", "apple,arm-platform"], } SWITCHES = b"chosen.asahi,t6030-gpu=1\nchosen.asahi,t6030-dcp=1\nchosen.asahi,t6030-dcpext=1\n" @@ -122,7 +124,9 @@ """ -class M3FlowTest(unittest.TestCase): +class M3FlowBase(unittest.TestCase): + """The fake Mac and its helpers, without tests (test_m3_air.py uses it too).""" + def setUp(self): if not shutil.which("bsdtar") or not shutil.which("zstd"): self.skipTest("bsdtar and zstd are needed to build the fixture m1n1 packages") @@ -220,6 +224,8 @@ def downloaded(self): def kept_copy(self): return self.tmp / f"esp/m1n1/boot.bin.before-{VERSION}" + +class M3FlowTest(M3FlowBase): # M3 Pro on the handoff path def test_fresh_m3_pro_gets_the_handoff(self): @@ -311,7 +317,8 @@ def test_listed_m3_pro_on_another_stub_is_kernel_only(self): self.assert_kernel_only("j516s", stub="15.6") def test_trial_refused_on_an_m3(self): - self.mac("j613") + # A plain M3 that isn't an Air (the Air's opt-in is in test_m3_air.py). + self.mac("j504") before = self.boot.read_bytes() proc = self.install(try_=1, check=False) self.assertNotEqual(proc.returncode, 0) diff --git a/tools/aurora-sep/tests/test_m3_handoff.py b/tools/aurora-sep/tests/test_m3_handoff.py index 9e01b3da8b9e9..873c9236f3c7f 100644 --- a/tools/aurora-sep/tests/test_m3_handoff.py +++ b/tools/aurora-sep/tests/test_m3_handoff.py @@ -43,6 +43,7 @@ "j514c": ["apple,j514c", "apple,t6031", "apple,arm-platform"], "j293": ["apple,j293", "apple,t8103", "apple,arm-platform"], "j613": ["apple,j613", "apple,t8122", "apple,arm-platform"], + "j504": ["apple,j504", "apple,t8122", "apple,arm-platform"], "j314s": ["apple,j314s", "apple,t6000", "apple,arm-platform"], "j700": ["apple,j700", "apple,t8140", "apple,arm-platform"], } @@ -170,7 +171,8 @@ def test_plan_try_on_an_unlisted_m3_pro(self): self.assertIn("nobody", out) def test_plan_try_refused_elsewhere(self): - for board, stub, why in [("j613", "14.8.3", "M3 Pro"), ("j514c", "14.8.3", "M3 Pro"), + # The M3 MacBook Air's own opt-in is in test_m3_air.py. + for board, stub, why in [("j504", "14.8.3", "M3 Pro"), ("j514c", "14.8.3", "M3 Pro"), ("j314s", "13.5", "isn't an M3"), ("j700", "26.4", "isn't an M3"), ("j514s", "15.6", "stub is 15.6"), ("j516s", "15.6", "stub is 15.6")]: with self.subTest(board=board, stub=stub): From 3928f1ac9d656ba7a18faf0d3a6d94bae41c040d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:43:52 -0700 Subject: [PATCH 1748/2680] tools: aurora-sep: M3 MacBook Air dry run test build (11.110-test) Kernel 11.110 (aurora-linux air/t8122) and, for an Air that opts in with --m3-handoff, m1n1-aurora 1.6.1.aurora8 with a dry run instead of the GPU handoff (M3_AIR_DRY_RUN=1): m1n1 reads the Air's GPU and display details from the boot firmware and reports them on the serial console and under /chosen, powers the GPU only for a bounded identity read, and starts nothing. The Air's switches are M3_AIR_DRY_RUN_SWITCHES; the handoff's stay as they were for a later build. M3 Pros keep aurora7 and their three switches; M1, M2 and the Neo are unchanged. Case D of --agent-prompt now walks an Air owner, or their agent, through the dry run and what to send back. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit 8e9344df1f88ce5d2369ad87acf3fb378a79e647) --- tools/aurora-sep/install-aurora-sep.sh | 105 +++++++++++-------- tools/aurora-sep/tests/test_m3_air.py | 137 ++++++++++++++++++++++++- 2 files changed, 199 insertions(+), 43 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 21e3e55811d28..c84e36c5da0df 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -124,8 +124,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.109 -TAG=sep-7.1.12.aurora2-11.109-test +VERSION=7.1.12.aurora2-11.110 +TAG=sep-7.1.12.aurora2-11.110-test # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -147,7 +147,7 @@ M3_M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora7-1-aarch64.pkg.tar.zst 547d0d5f9daa283 # "file sha256". It may name the same package as M3_M1N1_PACKAGE once one m1n1 # carries both. Empty until that package is published: an Air then stays # kernel-only, and --m3-handoff on an Air stops with nothing installed. -M3_AIR_M1N1_PACKAGE="" +M3_AIR_M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora8-1-aarch64.pkg.tar.zst d443671d819887b01ed712e1ff86429999bec9f857c6d67bf3040647b5c3176e" PINNED="linux-aurora linux-aurora-headers libfprint m1n1-aurora" PIN_BEGIN="# >>> aurora-sep pin (remove with: install-aurora-sep.sh --uninstall)" PIN_END="# <<< aurora-sep pin" @@ -278,6 +278,13 @@ M3_SWITCHES="chosen.asahi,t6030-gpu=1 chosen.asahi,t6030-dcp=1 chosen.asahi,t603 M3_AIR_GPU_SWITCH="chosen.asahi,t8122-gpu=1" M3_AIR_DCP_SWITCH="chosen.asahi,t8122-dcp=1" M3_AIR_DCP=0 +# A dry run in place of the handoff (M3_AIR_DRY_RUN=1): m1n1 reads the Air's +# GPU and display details from the boot firmware and reports them on the +# serial console and under /chosen, powers the GPU only for a bounded +# identity read, and starts nothing. The desktop stays as it was. This build +# ships the dry run: the Air's GPU can't be started until those details are in. +M3_AIR_DRY_RUN=1 +M3_AIR_DRY_RUN_SWITCHES="chosen.asahi,t8122-gpu-diag=1 chosen.asahi,t8122-gpu-handoff-diag=1 chosen.asahi,t8122-gpu-power-diag=1 chosen.asahi,t8122-dcp=1" # The handoff is tested with one macOS system-firmware stub only, 14.8.3 (GPU # firmware 14.8.3, DCP 14.7), which the Omarchy installer gives every M3. m1n1 # reports the stub's iBoot as asahi,iboot2-version. @@ -399,6 +406,8 @@ is_m3_handoff_board() { m3_switches() { if ! is_m3_air; then echo "$M3_SWITCHES" + elif [[ $M3_AIR_DRY_RUN == 1 ]]; then + echo "$M3_AIR_DRY_RUN_SWITCHES" elif [[ $M3_AIR_DCP == 1 ]]; then echo "$M3_AIR_GPU_SWITCH $M3_AIR_DCP_SWITCH" else @@ -421,6 +430,8 @@ m3_air_package_ok() { m3_handoff_name() { if ! is_m3_air; then echo "M3 Pro display and GPU handoff" + elif [[ $M3_AIR_DRY_RUN == 1 ]]; then + echo "M3 Air GPU and display dry run" elif [[ $M3_AIR_DCP == 1 ]]; then echo "M3 Air display and GPU handoff" else @@ -646,6 +657,11 @@ m3_plan() { if ((air)); then if is_m3_handoff_board; then say "M3 MacBook Air ($board, macOS $M3_STUB_VERSION stub): installing m1n1 with the $(m3_handoff_name)" + elif [[ $M3_AIR_DRY_RUN == 1 ]]; then + warn "--m3-handoff: installing m1n1's GPU and display dry run on this M3 MacBook Air ($board). + It reads the Air's GPU and display details and reports them for the bring-up; it switches + nothing on, so the desktop stays as it is, on the boot framebuffer and rendered in software. + This replaces the Mac's boot loader; the steps to put the old one back from macOS follow." else warn "--m3-handoff: trying m1n1's GPU handoff on an M3 MacBook Air ($board). It is in testing and not on by default. This replaces the Mac's boot loader; the steps to put the old one @@ -1195,7 +1211,7 @@ install_all() { is_neo || unproven_board_backup if [[ $M3_MODE == handoff ]] && is_m3_air; then keep_bootbin_on_esp "This replaces the boot loader of this M3 MacBook Air ($(this_board)) - with m1n1-aurora and its GPU handoff." + with m1n1-aurora and its $(m3_handoff_name)." elif [[ $M3_MODE == handoff ]]; then keep_bootbin_on_esp "This replaces the boot loader of this M3 Pro ($(this_board)) with m1n1-aurora and its display and GPU handoff." @@ -1281,7 +1297,13 @@ install_all() { if is_neo; then neo_radio_notice; fi pacman -Q linux-aurora libfprint aurora-touchid echo - if [[ $M3_MODE == handoff ]] && is_m3_air && [[ $M3_AIR_DCP != 1 ]]; then + if [[ $M3_MODE == handoff ]] && is_m3_air && [[ $M3_AIR_DRY_RUN == 1 ]]; then + say "Done. Reboot with the serial recorder running and someone watching: expect the Omarchy + logo, the boot menu, then the desktop on the boot framebuffer, as before. This build only + reads and reports, so nothing on the desktop changes. Touch ID is not supported on M3 yet. + What to send back: case D of step 11 of the test plan that the --agent-prompt command below + prints." + elif [[ $M3_MODE == handoff ]] && is_m3_air && [[ $M3_AIR_DCP != 1 ]]; then say "Done. Reboot with someone watching: expect the Omarchy logo, the boot menu, then the desktop on the boot framebuffer, as before. The GPU handoff is for testing: the desktop still renders in software. Touch ID is not supported on M3 yet. What to check and @@ -1805,57 +1827,60 @@ fingerprint. gets the handoff next. D. apple,j613 or apple,j615 + apple,t8122 (MacBook Air 13" or 15" M3): - KERNEL ONLY BY DEFAULT. THE GPU HANDOFF IS AN OPT-IN TEST. + KERNEL ONLY BY DEFAULT. THE BOOT LOADER TEST IS OPT-IN. The plain one-liner installs the kernel only, as in case C, and says so: M3 MacBook Air (j613): installing the kernel only ... Run the checks below on that first and report it: it is the baseline every later Air test is compared with. - Then, to test m1n1's GPU handoff on this Air: + Then, on a test build that carries the Air's dry run (its message says + so), run the dry run on this Air: - What it does: it replaces this Mac's boot loader (m1n1) with one - that starts the Air's GPU for Linux. The screen does not change: - the display stays on the boot framebuffer (one fixed resolution), - and the desktop still renders in software, because no Mesa drives - this GPU yet. Success means the kernel finds a working GPU, not a - faster desktop. + that reads the Air's GPU and display details from the boot + firmware and reports them: on the serial console, and under + /proc/device-tree/chosen in Linux. It powers the GPU only for a + short identity read and switches nothing on. The screen and the + desktop stay exactly as in the baseline. These reports are what + the next build, the one that starts the Air's GPU, is made from. - Only with the owner's explicit agreement, with the owner at the Mac for the first boot, with the Aurora maintainer told first and reachable while it runs, and only if the owner can start macOS on this Mac (hold the power button at startup). If the new boot loader does not start, macOS on the same Mac is the only way back. - - If the installer says this release has no m1n1 with the Air's GPU - handoff, the test build is not out yet: stop there. It also - refuses unless the Mac has the macOS 14.8.3 system-firmware stub - that the Omarchy installer gives every M3. - - Run: - curl -fsSL https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh | bash -s -- --m3-handoff - It warns "--m3-handoff: trying m1n1's GPU handoff on an M3 MacBook - Air", then prints the restore steps: keep them, they name this - Mac's EFI partition. It must end with "m1n1's boot.bin is this - release's m1n1 with the M3 Air handoff switches"; if it stops with - an error instead, nothing was changed: report the error. - - Reboot, with the owner watching. Expected within about a minute: - the Omarchy logo, the boot menu, then the same desktop as before. - Then report exactly one of: - * WORKS: the desktop comes up as before. On a build whose kernel - drives the Air's GPU (its release notes say so), the kernel has - also taken the GPU: a render node, renderD128, is listed. - ls -l /dev/dri/ - sudo dmesg | grep -iE 'asahi|gpu|g15|t8122' | head -40 - Give the checks below too, and again after one suspend/resume. - Title it " (): M3 Air GPU handoff works". + - Record the boot on the serial console if the owner has a second + Apple Silicon Mac and a USB 3 USB-C cable between the two Macs' + DFU ports: tools/aurora-m3/m3-serial.py in this repository + ("sudo python3 m3-serial.py reboot" on the second Mac). Most of + the display details are only printed there. + - Run the test build's one-liner with --m3-handoff (the release + tag's URL from its message, not releases/latest): + curl -fsSL https://github.com/iconidentify/aurora-linux/releases/download//install-aurora-sep.sh | bash -s -- --m3-handoff + It warns "--m3-handoff: installing m1n1's GPU and display dry run", + then prints the restore steps: keep them, they name this Mac's EFI + partition. It must end with "m1n1's boot.bin is this release's + m1n1 with the M3 Air handoff switches"; if it stops with an error + instead, nothing was changed: report the error. + - Reboot (with the serial recorder running, if there is one), with + the owner watching. Expected within about a minute: the Omarchy + logo, the boot menu, then the same desktop as before. Then report + exactly one of: + * WORKS: the desktop comes up as before. Collect and attach: + sudo dmesg > dmesg.txt + sudo tar czf air-dryrun.tgz -C /proc/device-tree/chosen $(cd /proc/device-tree/chosen && ls -d asahi,t8122-* 2>/dev/null) + ls /proc/device-tree/chosen | grep t8122 + plus the serial log, with the line that shows the Mac's serial + number removed. Give the baseline checks below again too. + Title it " (): M3 Air dry run". * STOPS: m1n1 text and no boot menu, or a black screen for more than two minutes. The owner puts the old boot.bin back from macOS with the printed steps. Report what the screen showed and - when. + when, with the serial log if there is one: it says where m1n1 + stopped. * WRONG: it boots, but something from the baseline no longer works - (keyboard, Wi-Fi, suspend), a build that should drive the GPU - shows no render node, or the kernel log shows GPU errors. Collect - sudo journalctl -b -k | grep -iE 'asahi|gpu|g15|t8122|m1n1' + (keyboard, Wi-Fi, suspend). Collect + sudo journalctl -b -k | grep -iE 'asahi|gpu|g15|t8122|dcp|m1n1' over SSH if you can, then run the installer with --uninstall (it puts the stock m1n1 back) and reboot. - - Do not start games or GPU programs: nothing on the desktop uses - this GPU yet, and a GPU fault can freeze the Mac until a reboot. - - Later plain runs of the one-liner keep the handoff on this Air; + - Later plain runs of the one-liner keep the dry run on this Air; --uninstall takes it off. Checks for every M3 (quote the output; on a kernel-only M3 the handoff diff --git a/tools/aurora-sep/tests/test_m3_air.py b/tools/aurora-sep/tests/test_m3_air.py index 2db150bb5726c..681a06cd9fc56 100644 --- a/tools/aurora-sep/tests/test_m3_air.py +++ b/tools/aurora-sep/tests/test_m3_air.py @@ -4,11 +4,13 @@ --m3-handoff, and then only when the release carries M3_AIR_M1N1_PACKAGE. The handoff writes chosen.asahi,t8122-gpu=1 (and chosen.asahi,t8122-dcp=1 only with M3_AIR_DCP=1) to /etc/m1n1.conf. Every case sets -M3_AIR_M1N1_PACKAGE itself, so the tests don't depend on what the installer -ships with. +M3_AIR_M1N1_PACKAGE and M3_AIR_DRY_RUN itself, so the tests don't depend on +what the installer ships with. M3AirDryRunTest covers the dry run, which +writes M3_AIR_DRY_RUN_SWITCHES instead. """ from pathlib import Path import re +import shutil import unittest import test_m3_flow as flow @@ -33,9 +35,11 @@ def setUp(self): "asahi,t8122-gpu", "asahi,t8122-dcp"]) self.air_pkg = f"{AURORA8} {self.shas[AURORA8]}" + dry_run = 0 + def sh(self, body, pkg=None, dcp=0, boards=None, check=True): pkg = self.air_pkg if pkg is None else pkg - pre = f'M3_AIR_M1N1_PACKAGE="{pkg}"\nM3_AIR_DCP={dcp}\n' + pre = f'M3_AIR_M1N1_PACKAGE="{pkg}"\nM3_AIR_DCP={dcp}\nM3_AIR_DRY_RUN={self.dry_run}\n' if boards is not None: pre += f'M3_HANDOFF_BOARDS="{boards}"\n' return self.run_sh(pre + body, check=check) @@ -313,5 +317,132 @@ def test_m1_unchanged_by_air_settings(self): self.assertFalse(self.m1n1_conf.exists()) + +DRY_RUN = shell_value("M3_AIR_DRY_RUN_SWITCHES").split() +DRY_RUN_LINES = b"".join(l.encode() + b"\n" for l in DRY_RUN) + + +class M3AirDryRunTest(M3AirTest): + """The dry run this build ships: read and report, switch nothing on.""" + dry_run = 1 + + def setUp(self): + super().setUp() + # aurora8 knows every dry-run switch, the handoff's too. + shutil.rmtree(self.tmp / ("root-" + AURORA8)) + names = [l[len("chosen."):].split("=")[0] for l in DRY_RUN] + self.fixture(AURORA8, None, ["asahi,t6030-gpu", "asahi,t6030-dcp", "asahi,t6030-dcpext", + "asahi,t8122-gpu"] + names) + self.air_pkg = f"{AURORA8} {self.shas[AURORA8]}" + + # The inherited handoff tests run with M3_AIR_DRY_RUN=0 in M3AirTest; the + # ones that only make sense for the handoff's switches are skipped here. + def test_switches(self): + self.mac("j613") + for dcp in (0, 1): + self.assertEqual(self.sh("m3_switches", dcp=dcp).stdout.split(), DRY_RUN) + self.mac("j516s") + self.assertEqual(self.sh("m3_switches").stdout.strip(), shell_value("M3_SWITCHES")) + + def test_shipped_settings(self): + # A release decision: change this test with it. + self.assertEqual(re.search(r"^M3_AIR_DRY_RUN=(\S+)$", SRC, re.M).group(1), "1") + self.assertEqual(DRY_RUN, ["chosen.asahi,t8122-gpu-diag=1", + "chosen.asahi,t8122-gpu-handoff-diag=1", + "chosen.asahi,t8122-gpu-power-diag=1", + "chosen.asahi,t8122-dcp=1"]) + # The dry run never asks for the GPU setup itself. + self.assertNotIn("chosen.asahi,t8122-gpu=1", DRY_RUN) + self.assertTrue(re.search(r'^M3_AIR_M1N1_PACKAGE="m1n1-aurora-\S+ [0-9a-f]{64}"$', SRC, re.M)) + + def test_conf_block(self): + self.mac("j613") + self.sh("m3_switches_write\nm3_switches_write") + chosen = [l for l in self.m1n1_conf.read_text().splitlines() if l.startswith("chosen.")] + self.assertEqual(chosen, DRY_RUN) + self.sh("m3_switches_remove") + # Nothing else was in it, so the file goes. + self.assertFalse(self.m1n1_conf.exists()) + + def test_conf_block_replaces_an_m3_pro_block(self): + self.mac("j516s") + self.sh("m3_switches_write") + self.mac("j613") + self.sh("m3_switches_write") + chosen = [l for l in self.m1n1_conf.read_text().splitlines() if l.startswith("chosen.")] + self.assertEqual(chosen, DRY_RUN) + + def test_package_check(self): + pkgs = self.tmp / "pkgs" + self.mac("j613") + self.assertEqual(self.sh(f"m1n1_pkg_has_handoff '{pkgs / AURORA8}' && echo yes").stdout.strip(), "yes") + # An m1n1 with the GPU handoff switch alone can't run the dry run. + self.fixture("m1n1-gpu-only.pkg.tar.zst", None, ["asahi,t8122-gpu"]) + only = pkgs / "m1n1-gpu-only.pkg.tar.zst" + self.assertEqual(self.sh(f"m1n1_pkg_has_handoff '{only}' && echo yes || echo no").stdout.strip(), "no") + + def test_verify_bootbin(self): + self.mac("j613") + self.boot.write_bytes(b"M1N1:original\nDTBS:x\nUBOOT" + DRY_RUN_LINES) + (self.fake / "m1n1.bin").write_text("M1N1:original\n") + self.assertIn("M3 Air handoff switches", self.sh("m3_verify_bootbin").stdout) + self.boot.write_bytes(b"M1N1:original\nDTBS:x\nUBOOT" + AIR_GPU) + proc = self.sh("m3_verify_bootbin", check=False) + self.assertNotEqual(proc.returncode, 0) + + def test_plan_messages(self): + self.mac("j613") + self.assertIn("case D", self.sh("m3_plan").stdout) + err = self.sh("M3_TRY=1\nm3_plan").stderr + self.assertIn("GPU and display dry run", err) + self.assertIn("switches\n nothing on", err) + + def test_handoff_and_back(self): + self.mac("j613") + proc = self.air_install(try_=1) + self.assertIn("only\n reads and reports", proc.stdout) + self.assertIn("serial recorder", proc.stdout) + self.assertIn("M3 Air handoff switches", proc.stdout) + boot = self.boot.read_bytes() + self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora8-1\n"), boot) + self.assertTrue(boot.endswith(b"UBOOT" + DRY_RUN_LINES), boot) + self.assertNotIn(b"t8122-gpu=1", boot) + self.assertNotIn(b"t6030", boot) + self.assertEqual(self.kept_copy().read_bytes(), b"M1N1:original\n") + self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff") + self.sh("uninstall_all") + boot = self.boot.read_bytes() + self.assertTrue(boot.startswith(b"M1N1:m1n1-stock\n"), boot) + self.assertNotIn(b"chosen.", boot) + self.assertFalse(self.m1n1_conf.exists()) + + def test_rerun_keeps_the_handoff(self): + self.mac("j615") + self.air_install(try_=1) + self.assertIn("keeping it", self.air_install().stdout) + self.assertTrue(self.boot.read_bytes().endswith(DRY_RUN_LINES)) + + def test_display_switch(self): + self.skipTest("the dry run always includes the display part") + + def test_refused_with_an_m1n1_that_lacks_the_switch(self): + self.mac("j613") + before = self.boot.read_bytes() + proc = self.air_install(try_=1, pkg=f"{flow.AURORA6} {self.shas[flow.AURORA6]}", check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("no M3 Air GPU and display dry run", proc.stderr) + self.assertEqual(self.boot.read_bytes(), before) + self.assertNotIn("pacman -U", self.log()) + + def test_plan_try_refused(self): + for board, stub, pkg, why in [("j613", "14.8.3", "", "no m1n1 with the M3 MacBook Air GPU"), + ("j613", "15.6", None, "stub is 15.6")]: + with self.subTest(board=board, stub=stub, pkg=pkg): + self.mac(board, stub=stub) + proc = self.sh("M3_TRY=1\nm3_plan", pkg=pkg, check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn(why, proc.stderr) + self.assertIn("Nothing was installed", proc.stderr) + if __name__ == "__main__": unittest.main() From 1610e2ecb2c5f397d55e22d7f1d0e73960779988 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:06:28 -0700 Subject: [PATCH 1749/2680] tools: aurora-sep: 11.110-test kernel, record the Air's handoff variant - Kernel 11.110 checksums. - $STATE/m3-mode records the handoff's variant (t6030, air-dry-run, air-gpu, air-gpu-dcp). A plain re-run keeps an Air's handoff only when this release's variant is the recorded one, so a later build that starts the GPU never reaches an Air that only agreed to the dry run; it stops with nothing installed until --m3-handoff is given again. M3 Pros keep their handoff as before, bare 11.36.1 records included. - sync after the M3 boot.bin rebuild: the test reboot may be a hard reset. - The boot.bin backup message also gives the Linux restore line and the update-m1n1 freeze; case D's STOPS and WRONG paths use them instead of --uninstall, which on an Air would rebuild boot.bin from the stock m1n1. - Case D: only "Nothing was installed" means nothing changed; sync, then m3-serial.py reboot; a filtered kernel log instead of a whole dmesg; the tar only when /chosen has t8122 entries; the new device tree noted. - Messages name the dry run instead of a GPU handoff. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit f5d099e187fa099f63d6586002ed337899aae97a) --- tools/aurora-sep/install-aurora-sep.sh | 126 +++++++++++++++++-------- tools/aurora-sep/tests/test_m3_air.py | 50 +++++++++- tools/aurora-sep/tests/test_m3_flow.py | 4 +- 3 files changed, 136 insertions(+), 44 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index c84e36c5da0df..d926516afcffe 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -87,9 +87,11 @@ # Every other M3 stays kernel-only, and --m3-handoff lets an M3 Pro owner try # the handoff on a model not yet on the list (see --agent-prompt). M1, M2 and # the Neo are unchanged. -# The M3 MacBook Air (T8122, 13" J613 and 15" J615) can opt in to m1n1's GPU -# handoff with --m3-handoff once a release carries M3_AIR_M1N1_PACKAGE: the -# display stays on the boot framebuffer, and a plain run stays kernel-only. +# 11.110-test (not a release): kernel 11.110 (aurora-linux air/t8122) and, for +# an M3 MacBook Air (T8122, 13" J613 and 15" J615) that opts in with +# --m3-handoff, m1n1-aurora aurora8 with a dry run (M3_AIR_DRY_RUN): m1n1 +# reads and reports the Air's GPU and display details and starts nothing. A +# plain run keeps an Air kernel-only. # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -133,8 +135,8 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst 036f8a20a94363c247443cb5809707b5e884bc80f11531723c7409d91f867815" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 9eb62c16a727d17c809a5eb5f847e4ad4adce211304a12f99ff936cf9c9487b5" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst eeab04119026280308987fb7ac2f5f8996504012d2a6d38fe645f5609cc56986" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 35af393eaaa3eb646217283566e769be5aadfe822d08b3aa0a285b1dd7f26722" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20261003-1-any.pkg.tar.zst 29b0360fac8c257d754e64bd1b9c33c487eb2595dd3c31e9138d7a476afa3d64" "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" @@ -143,10 +145,10 @@ PACKAGES=( # above: the same m1n1 plus the T6030 display and GPU handoff. M3_M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora7-1-aarch64.pkg.tar.zst 547d0d5f9daa283749d84e5a9c875fee06ce449eeb8252e41c11b3ce63e11184" # Only for an M3 MacBook Air on the handoff path, in place of the m1n1-aurora -# above: m1n1-aurora with the T8122 GPU handoff (aurora8 is the first), as +# above: m1n1-aurora with the T8122 switches (aurora8 is the first), as # "file sha256". It may name the same package as M3_M1N1_PACKAGE once one m1n1 -# carries both. Empty until that package is published: an Air then stays -# kernel-only, and --m3-handoff on an Air stops with nothing installed. +# carries both. Left empty, an Air stays kernel-only and --m3-handoff on an +# Air stops with nothing installed. M3_AIR_M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora8-1-aarch64.pkg.tar.zst d443671d819887b01ed712e1ff86429999bec9f857c6d67bf3040647b5c3176e" PINNED="linux-aurora linux-aurora-headers libfprint m1n1-aurora" PIN_BEGIN="# >>> aurora-sep pin (remove with: install-aurora-sep.sh --uninstall)" @@ -272,9 +274,9 @@ M3_BOOTBIN_SHA="" # by default; no Air is listed yet. M3_HANDOFF_BOARDS="j516s" M3_SWITCHES="chosen.asahi,t6030-gpu=1 chosen.asahi,t6030-dcp=1 chosen.asahi,t6030-dcpext=1" -# The M3 MacBook Air's switches. The GPU handoff alone for now: the display -# stays on the boot framebuffer. Set M3_AIR_DCP=1 to add the display handoff -# once m1n1's T8122 DCP handoff has been booted on an Air. +# The M3 MacBook Air's handoff switches, for a later build that starts the +# Air's GPU (M3_AIR_DRY_RUN=0): the GPU alone, or with M3_AIR_DCP=1 the +# display too, once m1n1's T8122 DCP handoff has been booted on an Air. M3_AIR_GPU_SWITCH="chosen.asahi,t8122-gpu=1" M3_AIR_DCP_SWITCH="chosen.asahi,t8122-dcp=1" M3_AIR_DCP=0 @@ -283,6 +285,9 @@ M3_AIR_DCP=0 # serial console and under /chosen, powers the GPU only for a bounded # identity read, and starts nothing. The desktop stays as it was. This build # ships the dry run: the Air's GPU can't be started until those details are in. +# chosen.asahi,t8122-dcp=1 is a dry run only while m1n1 pins no T8122 DCP +# firmware image (aurora8 pins none); a release whose m1n1 pins one must give +# the dry run another variant name in m3_variant, so no Air gets it unasked. M3_AIR_DRY_RUN=1 M3_AIR_DRY_RUN_SWITCHES="chosen.asahi,t8122-gpu-diag=1 chosen.asahi,t8122-gpu-handoff-diag=1 chosen.asahi,t8122-gpu-power-diag=1 chosen.asahi,t8122-dcp=1" # The handoff is tested with one macOS system-firmware stub only, 14.8.3 (GPU @@ -582,6 +587,23 @@ m3_recorded_mode() { echo "$mode" } +# What this release's handoff does on this Mac. It is recorded with the mode, +# so a later release never turns one kind of Air test into another (a dry run +# into a GPU start) without a new --m3-handoff. +m3_variant() { + if ! is_m3_air; then echo t6030 + elif [[ $M3_AIR_DRY_RUN == 1 ]]; then echo air-dry-run + elif [[ $M3_AIR_DCP == 1 ]]; then echo air-gpu-dcp + else echo air-gpu + fi +} + +m3_recorded_variant() { + local mode="" variant="" + if [[ -f $STATE/m3-mode ]]; then read -r mode variant _ <"$STATE/m3-mode" || true; fi + echo "$variant" +} + # Decide the M3 path before anything is downloaded, so a Mac this release can't # set up as asked stops with nothing changed. m3_plan() { @@ -596,9 +618,14 @@ m3_plan() { # An update never takes the handoff away again: once a Mac has it (listed, or # tried with --m3-handoff), a plain run keeps it, and its checks still apply. if ((M3_TRY == 0)) && [[ $(m3_recorded_mode) == handoff ]]; then + if ((air)) && [[ $(m3_recorded_variant) != "$(m3_variant)" ]]; then + die "M3 MacBook Air ($board): this Mac has an earlier test build's boot loader + ($(m3_recorded_variant)), and this release's Air boot loader does something else: the + $(m3_handoff_name). Run this again with --m3-handoff to switch to it. Nothing was installed." + fi M3_TRY=1 if ((air)); then - say "M3 MacBook Air ($board): this Mac has m1n1's GPU handoff from an earlier install; keeping it" + say "M3 MacBook Air ($board): this Mac has m1n1's $(m3_handoff_name) from an earlier install; keeping it" else say "M3 ($board): this Mac has m1n1's display and GPU handoff from an earlier install; keeping it" fi @@ -618,7 +645,7 @@ m3_plan() { fi if ! is_m3_handoff_board && ((M3_TRY == 0)); then say "M3 MacBook Air ($board): installing the kernel only, and boot.bin stays as it is. m1n1's - GPU handoff for the Air is being tested and is not switched on by default. To help test it + $(m3_handoff_name) for the Air is being tested and is not on by default. To help test it (it replaces this Mac's boot loader), see case D in the M3 section of: bash -s -- --agent-prompt" return 0 fi @@ -659,8 +686,9 @@ m3_plan() { say "M3 MacBook Air ($board, macOS $M3_STUB_VERSION stub): installing m1n1 with the $(m3_handoff_name)" elif [[ $M3_AIR_DRY_RUN == 1 ]]; then warn "--m3-handoff: installing m1n1's GPU and display dry run on this M3 MacBook Air ($board). - It reads the Air's GPU and display details and reports them for the bring-up; it switches - nothing on, so the desktop stays as it is, on the boot framebuffer and rendered in software. + It reads the Air's GPU and display details and reports them for the bring-up, powering the + GPU only for a short identity read; it switches nothing on, so the desktop stays as it is, + on the boot framebuffer and rendered in software. This replaces the Mac's boot loader; the steps to put the old one back from macOS follow." else warn "--m3-handoff: trying m1n1's GPU handoff on an M3 MacBook Air ($board). It is in testing @@ -769,6 +797,10 @@ keep_bootbin_on_esp() { diskutil info diskNsM | grep -i 'partition uuid' 3. sudo diskutil mount diskNsM (it prints the /Volumes path) 4. cp -X '/m1n1/boot.bin.before-$VERSION' '/m1n1/boot.bin' + If it boots but something is wrong, the same copy goes back from Linux with: + sudo cp '$keep' '$target' && sync + After either, keep updates from rebuilding the test boot loader until it is sorted: + echo M1N1_UPDATE_DISABLED=1 | sudo tee -a $UPDATE_M1N1_CONF The copy is kept at $keep. Either way, please report it at https://github.com/iconidentify/aurora-linux/issues" return 0 @@ -1225,7 +1257,11 @@ install_all() { if [[ ! -f $STATE/previous-package ]]; then pacman -Q "$kernel" | $sudo tee "$STATE/previous-package" >/dev/null fi - if [[ $M3_MODE != none ]]; then echo "$M3_MODE" | $sudo tee "$STATE/m3-mode" >/dev/null; fi + if [[ $M3_MODE == handoff ]]; then + echo "$M3_MODE $(m3_variant)" | $sudo tee "$STATE/m3-mode" >/dev/null + elif [[ $M3_MODE != none ]]; then + echo "$M3_MODE" | $sudo tee "$STATE/m3-mode" >/dev/null + fi # Before pacman's update-m1n1 hook runs on the kernel's device trees. case $M3_MODE in kernel) @@ -1284,6 +1320,9 @@ install_all() { fi m1n1_update m3_verify_bootbin + # The test reboot may be a hard reset (m3-serial.py reboot): get the new + # boot.bin onto the EFI partition first. + sync ;; *) m1n1_update ;; esac @@ -1692,7 +1731,7 @@ fingerprint. sudo dmesg | grep -E 'cold init done|link up after|translation fault|HC died' Any "translation fault" or "HC died" line is a failure to report. For a full report, after plugging the dock in: - curl -fsSLO https://raw.githubusercontent.com/iconidentify/aurora-linux/refs/tags/sep-7.1.12.aurora2-11.109-test/tools/aurora-tb/tb-pcie-report + curl -fsSLO https://raw.githubusercontent.com/iconidentify/aurora-linux/refs/tags/sep-7.1.12.aurora2-11.110-test/tools/aurora-tb/tb-pcie-report sudo sh tb-pcie-report --no-wait 6c. Across suspend: @@ -1855,33 +1894,44 @@ fingerprint. tag's URL from its message, not releases/latest): curl -fsSL https://github.com/iconidentify/aurora-linux/releases/download//install-aurora-sep.sh | bash -s -- --m3-handoff It warns "--m3-handoff: installing m1n1's GPU and display dry run", - then prints the restore steps: keep them, they name this Mac's EFI - partition. It must end with "m1n1's boot.bin is this release's - m1n1 with the M3 Air handoff switches"; if it stops with an error - instead, nothing was changed: report the error. - - Reboot (with the serial recorder running, if there is one), with - the owner watching. Expected within about a minute: the Omarchy - logo, the boot menu, then the same desktop as before. Then report - exactly one of: + then prints the restore steps, from macOS and from Linux: keep + them, they name this Mac's EFI partition and boot.bin. Near the end + it must print "m1n1's boot.bin is this release's m1n1 with the M3 + Air handoff switches". An error ending "Nothing was installed" + changed nothing: report it. For any other error, do not reboot: + report it first. + - Reboot: run sync on the Air, then, if the second Mac records the + serial console, start "sudo python3 m3-serial.py reboot" there; it + reboots the Air itself. Without one, run "sudo reboot" on the Air. + The owner watches the screen. This boot also uses this release's + device tree for the Air for the first time. Expected within about a + minute: the Omarchy logo, the boot menu, then the same desktop as + before. Then report exactly one of: * WORKS: the desktop comes up as before. Collect and attach: - sudo dmesg > dmesg.txt - sudo tar czf air-dryrun.tgz -C /proc/device-tree/chosen $(cd /proc/device-tree/chosen && ls -d asahi,t8122-* 2>/dev/null) - ls /proc/device-tree/chosen | grep t8122 - plus the serial log, with the line that shows the Mac's serial - number removed. Give the baseline checks below again too. - Title it " (): M3 Air dry run". + sudo dmesg | grep -iE 'asahi|agx|gpu|dcp|dart|t8122|reserved|iommu|mailbox|pmp|simpledrm|m1n1' > dmesg-air.txt + ls -d /proc/device-tree/chosen/asahi,t8122-* + and, if that lists anything: + sudo tar czf air-dryrun.tgz -C /proc/device-tree/chosen $(cd /proc/device-tree/chosen && ls -d asahi,t8122-*) + (if it lists nothing, say so instead), plus the serial log with + the line that shows the Mac's serial number removed. Skim + dmesg-air.txt for serial numbers or MAC addresses before posting. + Give the baseline checks below again too. Title it + " (): M3 Air dry run". * STOPS: m1n1 text and no boot menu, or a black screen for more than two minutes. The owner puts the old boot.bin back from - macOS with the printed steps. Report what the screen showed and - when, with the serial log if there is one: it says where m1n1 - stopped. + macOS with the printed steps, starts Linux, and runs the printed + M1N1_UPDATE_DISABLED line so no update rebuilds the test boot + loader. Do not run the one-liner again or --uninstall until the + maintainer says so. Report what the screen showed and when, with + the serial log if there is one: it says where m1n1 stopped. * WRONG: it boots, but something from the baseline no longer works (keyboard, Wi-Fi, suspend). Collect sudo journalctl -b -k | grep -iE 'asahi|gpu|g15|t8122|dcp|m1n1' - over SSH if you can, then run the installer with --uninstall - (it puts the stock m1n1 back) and reboot. - - Later plain runs of the one-liner keep the dry run on this Air; - --uninstall takes it off. + then run the two printed lines from Linux (put the old boot.bin + back, then M1N1_UPDATE_DISABLED), reboot, and report. + - Later plain runs of this test build keep the dry run on this Air. + To go back, use the printed restore lines; ask the maintainer + before using --uninstall on an Air. Checks for every M3 (quote the output; on a kernel-only M3 the handoff lines are expected to be missing, so say so): diff --git a/tools/aurora-sep/tests/test_m3_air.py b/tools/aurora-sep/tests/test_m3_air.py index 681a06cd9fc56..2ef7905360afc 100644 --- a/tools/aurora-sep/tests/test_m3_air.py +++ b/tools/aurora-sep/tests/test_m3_air.py @@ -245,7 +245,7 @@ def test_handoff_and_back(self): self.assertNotIn(flow.AURORA3, self.downloaded()) self.assertNotIn(flow.AURORA6, self.downloaded()) self.assertEqual(self.kept_copy().read_bytes(), b"M1N1:original\n") - self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff") + self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff air-gpu") self.assertNotIn("M1N1_UPDATE_DISABLED", self.update_conf.read_text()) self.sh("uninstall_all") @@ -261,7 +261,7 @@ def test_rerun_keeps_the_handoff(self): out = self.air_install().stdout self.assertIn("keeping it", out) self.assertTrue(self.boot.read_bytes().endswith(AIR_GPU)) - self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff") + self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff air-gpu") def test_rerun_without_an_air_package_stops(self): # A later build without the Air's m1n1 never drops a tested Air back @@ -395,7 +395,8 @@ def test_plan_messages(self): self.assertIn("case D", self.sh("m3_plan").stdout) err = self.sh("M3_TRY=1\nm3_plan").stderr self.assertIn("GPU and display dry run", err) - self.assertIn("switches\n nothing on", err) + self.assertIn("it switches nothing on", err) + self.assertIn("short identity read", err) def test_handoff_and_back(self): self.mac("j613") @@ -409,7 +410,7 @@ def test_handoff_and_back(self): self.assertNotIn(b"t8122-gpu=1", boot) self.assertNotIn(b"t6030", boot) self.assertEqual(self.kept_copy().read_bytes(), b"M1N1:original\n") - self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff") + self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff air-dry-run") self.sh("uninstall_all") boot = self.boot.read_bytes() self.assertTrue(boot.startswith(b"M1N1:m1n1-stock\n"), boot) @@ -444,5 +445,46 @@ def test_plan_try_refused(self): self.assertIn(why, proc.stderr) self.assertIn("Nothing was installed", proc.stderr) + # The recorded variant: a dry run never becomes a GPU start unasked. + + def test_later_gpu_build_needs_a_new_opt_in(self): + self.mac("j613") + self.air_install(try_=1) + self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff air-dry-run") + before = self.boot.read_bytes() + self.dry_run = 0 # a later release that starts the GPU + proc = self.air_install(check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("earlier test build's boot loader", proc.stderr) + self.assertIn("(air-dry-run)", proc.stderr) + self.assertIn("Nothing was installed", proc.stderr) + self.assertEqual(self.boot.read_bytes(), before) + # Asking again switches it. + self.air_install(try_=1) + self.assertTrue(self.boot.read_bytes().endswith(b"UBOOT" + AIR_GPU), self.boot.read_bytes()) + self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff air-gpu") + + def test_unrecorded_variant_needs_a_new_opt_in(self): + # A record without a variant (the never-released 6e8493e4) is not kept. + self.mac("j613") + self.air_install(try_=1) + (self.state / "m3-mode").write_text("handoff\n") + proc = self.air_install(check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("Nothing was installed", proc.stderr) + + def test_m3_pro_old_record_still_kept(self): + # 11.36.1 recorded a bare "handoff" on M3 Pros; that still keeps it. + self.mac("j516s") + self.air_install() + (self.state / "m3-mode").write_text("handoff\n") + out = self.air_install().stdout + self.assertIn("keeping it", out) + self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff t6030") + + def test_install_syncs_after_the_rebuild(self): + src = SRC[SRC.index(" m3_verify_bootbin\n"):] + self.assertTrue(re.match(r" m3_verify_bootbin\n(\s*#.*\n)*\s*sync\n", src), src[:200]) + if __name__ == "__main__": unittest.main() diff --git a/tools/aurora-sep/tests/test_m3_flow.py b/tools/aurora-sep/tests/test_m3_flow.py index 7d350b4814c06..1e664fe92be90 100644 --- a/tools/aurora-sep/tests/test_m3_flow.py +++ b/tools/aurora-sep/tests/test_m3_flow.py @@ -239,7 +239,7 @@ def test_fresh_m3_pro_gets_the_handoff(self): self.assertIn(AURORA6, self.downloaded()) self.assertNotIn(AURORA3, self.downloaded()) self.assertEqual(self.kept_copy().read_bytes(), b"M1N1:original\n") - self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff") + self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff t6030") self.assertFalse(self.update_conf.read_text().count("M1N1_UPDATE_DISABLED")) self.uninstall() @@ -333,7 +333,7 @@ def test_rerun_without_the_flag_keeps_a_tried_handoff(self): self.install(try_=1) out = self.install().stdout self.assertIn("keeping it", out) - self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff") + self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff t6030") self.assertTrue(self.boot.read_bytes().endswith(SWITCHES)) self.assertNotIn("M1N1_UPDATE_DISABLED", self.update_conf.read_text()) self.uninstall() From b66b24167c823f8aa61c0d59ef060e294f2ae163 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:20:10 -0700 Subject: [PATCH 1750/2680] tools: aurora-sep: refuse chips this release doesn't support (#32) The kernel carries device trees for the M3 Ultra (t6032) and M4 and later chips (t8132, t8142, t8152), but neither is_m3 nor is_neo matched them, so the installer took the M1/M2 path there: it installed m1n1-aurora and rebuilt boot.bin with an m1n1 that can't start them, leaving the Mac to be recovered from macOS. install_all and uninstall_all now stop first, with nothing downloaded or changed, unless the device tree names a chip in SUPPORTED_SOCS (M1, M2, the MacBook Neo and the four M3s). --agent-prompt and --reset-touchid, which install nothing, still run anywhere. Fixes: https://github.com/iconidentify/aurora-linux/issues/32 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit 6eac663c2cbff29c0a6fe57ed410a6c32c312a55) --- tools/aurora-sep/install-aurora-sep.sh | 27 +++++ .../aurora-sep/tests/test_unsupported_soc.py | 107 ++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 tools/aurora-sep/tests/test_unsupported_soc.py diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index d926516afcffe..d47fe5ddf9362 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -225,6 +225,31 @@ is_neo() { tr '\0' '\n' <"$DT/compatible" 2>/dev/null | grep -qx 'apple,t8140' } +# The chips this release installs on: M1 (t8103, t6000, t6001, t6002), M2 +# (t8112, t6020, t6021, t6022), the MacBook Neo (t8140) and the M3s that is_m3 +# lists. The kernel also carries device trees for the M3 Ultra (t6032) and +# M4 and later chips (t8132, t8142, t8152), but nobody has booted this kernel +# on them, and the M1/M2 path would replace their boot loader with an m1n1 +# that can't start them. Keep this list in step with is_m3 and is_neo. +SUPPORTED_SOCS="t8103 t6000 t6001 t6002 t8112 t6020 t6021 t6022 t8140 t8122 t6030 t6031 t6034" + +# This Mac's chip from the device tree, as tNNNN; empty when there is none. +this_soc() { + tr '\0' '\n' <"$DT/compatible" 2>/dev/null | sed -n 's/^apple,\(t[0-9][0-9][0-9][0-9]\)$/\1/p' | head -1 +} + +# Stops before anything is downloaded or changed on a chip this release +# doesn't support. --agent-prompt and --reset-touchid don't come here. +require_supported_soc() { + local soc + soc=$(this_soc) + [[ -n $soc && " $SUPPORTED_SOCS " == *" $soc "* ]] && return 0 + die "this Mac (${soc:+apple,$soc, }$(this_board)) is not one $VERSION supports: M1, M2, the MacBook + Neo, and M3, M3 Pro and M3 Max. ${1:-Installing} would replace its boot loader with an m1n1 that + can't start it. Nothing was changed. If you are bringing this Mac up, please open an issue at + https://github.com/iconidentify/aurora-linux/issues" +} + # Every M3 chip: M3 (t8122), M3 Pro (t6030), M3 Max (t6031, t6034). is_m3() { tr '\0' '\n' <"$DT/compatible" 2>/dev/null | grep -Eqx 'apple,(t8122|t6030|t6031|t6034)' @@ -1209,6 +1234,7 @@ packages_for_this_mac() { install_all() { local entry file sha kernel chain local -a entries + require_supported_soc version_notice sep_write_notice ane_dkms_notice @@ -1371,6 +1397,7 @@ install_all() { uninstall_all() { local previous=linux-asahi m3_mode=none + require_supported_soc "Uninstalling, which rebuilds boot.bin with the stock m1n1," if [[ -f $STATE/previous-package ]]; then read -r previous _ <"$STATE/previous-package" || true; fi # 11.36 rewrote this on every run, so an updated Mac may name linux-aurora # itself, which the repositories don't carry for these Macs. diff --git a/tools/aurora-sep/tests/test_unsupported_soc.py b/tools/aurora-sep/tests/test_unsupported_soc.py new file mode 100644 index 0000000000000..3615def9e275e --- /dev/null +++ b/tools/aurora-sep/tests/test_unsupported_soc.py @@ -0,0 +1,107 @@ +"""Chips this release doesn't support stop before anything changes (issue #32). + +The kernel carries device trees for the M3 Ultra (t6032) and M4 and later +chips (t8132, t8142, t8152). On those Macs the M1/M2 path would install +m1n1-aurora and rebuild boot.bin with an m1n1 that can't start them, so +install_all and uninstall_all refuse first. --agent-prompt still works. +Runs against the fake Mac of test_m3_flow. +""" +import re +import unittest + +import test_m3_flow as flow + +SRC = flow.INSTALLER.read_text() + +# The fake Mac's helpers: M3FlowBase, or on older trees the test class that +# carries them, with its own tests switched off here. +Base = getattr(flow, "M3FlowBase", None) +if Base is None: + Base = type("Base", (flow.M3FlowTest,), + {n: None for n in dir(flow.M3FlowTest) if n.startswith("test")}) + +UNSUPPORTED = { + "j575d": "t6032", # Mac Studio M3 Ultra + "j713": "t8132", # M4 + "j813": "t8142", + "j873g": "t8152", +} +SUPPORTED = { + "j293": "t8103", "j314s": "t6000", "j314c": "t6001", "j375d": "t6002", + "j413": "t8112", "j414s": "t6020", "j414c": "t6021", "j180d": "t6022", + "j700": "t8140", "j613": "t8122", "j516s": "t6030", "j514c": "t6031", "j514m": "t6034", +} + + +class UnsupportedSocTest(Base): + def setUp(self): + super().setUp() + for board, soc in {**UNSUPPORTED, **SUPPORTED}.items(): + if board not in flow.BOARDS: + flow.BOARDS[board] = [f"apple,{board}", f"apple,{soc}", "apple,arm-platform"] + self.addCleanup(flow.BOARDS.pop, board) + + def test_supported_list_matches_is_m3_and_is_neo(self): + listed = set(re.search(r'^SUPPORTED_SOCS="([^"]*)"$', SRC, re.M).group(1).split()) + m3 = set(re.search(r"grep -Eqx 'apple,\(([^)]*)\)'", SRC).group(1).split("|")) + self.assertTrue(m3 <= listed, m3 - listed) + self.assertIn("t8140", listed) + self.assertFalse(listed & set(UNSUPPORTED.values())) + self.assertEqual(listed, set(SUPPORTED.values())) + + def test_supported_chips_pass(self): + for board in SUPPORTED: + with self.subTest(board=board): + self.mac(board) + self.assertEqual(self.run_sh("require_supported_soc && echo ok").stdout.strip(), "ok") + + def test_install_refuses_before_anything(self): + for board, soc in UNSUPPORTED.items(): + for try_ in (0, 1): + with self.subTest(board=board, try_=try_): + self.mac(board) + before = self.boot.read_bytes() + proc = self.run_sh(f"M3_TRY={try_}\ninstall_all", check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn(f"apple,{soc}", proc.stderr) + self.assertIn("Nothing was changed", proc.stderr) + self.assertEqual(self.boot.read_bytes(), before) + self.assertEqual(self.downloaded(), []) + self.assertNotIn("pacman", self.log()) + self.assertNotIn("update-m1n1", self.log()) + self.assertFalse(self.m1n1_conf.exists()) + self.assertFalse(self.state.exists() and any(self.state.iterdir())) + + def test_read_only_install_refuses_too(self): + self.mac("j713") + proc = self.run_sh("READ_ONLY=1\ninstall_all", check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("apple,t8132", proc.stderr) + + def test_uninstall_refuses(self): + for board, soc in UNSUPPORTED.items(): + with self.subTest(board=board): + self.mac(board) + before = self.boot.read_bytes() + proc = self.run_sh("uninstall_all", check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn(f"apple,{soc}", proc.stderr) + self.assertIn("Uninstalling", proc.stderr) + self.assertEqual(self.boot.read_bytes(), before) + self.assertNotIn("pacman", self.log()) + + def test_no_chip_in_the_device_tree_refuses(self): + self.mac("j713") + (self.tmp / "dt/compatible").write_bytes(b"apple,j999\0apple,arm-platform\0") + proc = self.run_sh("install_all", check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("Nothing was changed", proc.stderr) + + def test_agent_prompt_still_works(self): + self.mac("j575d") + proc = self.run_sh("agent_prompt >/dev/null && echo printed") + self.assertEqual(proc.stdout.strip(), "printed") + + +if __name__ == "__main__": + unittest.main() From 2b6d9e69c3fd55fd43c6c79e0f787536a23bc687 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:53:19 -0700 Subject: [PATCH 1751/2680] tools: aurora-sep: --m3-report, one file for an M3 test report Writes aurora-m3-report--.tgz in the current directory: the board, chip, firmware and boot loader versions, the M3 mode and switches, the boot loader's /chosen asahi,t8122-* and asahi,t6030-* entries, a kernel log filtered to the M3 bring-up (no USB serial numbers, MAC addresses masked), and USB-C and display state. It reads only and needs no preflight. Case D now asks for this file instead of three commands. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit 8b7f82c24fcd7db9f26a56f54605dd0768083ff7) --- tools/aurora-sep/install-aurora-sep.sh | 64 +++++++++++++++---- tools/aurora-sep/tests/test_m3_report.py | 79 ++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 11 deletions(-) create mode 100644 tools/aurora-sep/tests/test_m3_report.py diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index d47fe5ddf9362..d566722398907 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -1581,6 +1581,50 @@ reset_touchid() { # Printed by --agent-prompt, and pointed at from the end of a successful # install. This is written for an agent driving the test on a real Mac: it says # what to establish, what counts as a pass, and how to write it up. +# --m3-report: one file with what an M3 test report needs, in the current +# directory. It reads only: the boot loader's /chosen entries, a kernel log +# filtered to the M3 bring-up, USB-C and display state. Lines naming a USB +# serial number are dropped and MAC addresses are masked. +M3_REPORT_DMESG='asahi|agx|gpu|g15|dcp|dart|t8122|t6030|reserved|iommu|mailbox|pmp|simpledrm|m1n1|tipd|typec|sn201202|atc|usb|xhci|dwc3|thermal|macsmc' +m3_report() { + local dir out board soc f + board=$(this_board) soc=$(this_soc) + out=$PWD/aurora-m3-report-${board:-mac}-$(date +%Y%m%d-%H%M%S).tgz + dir=$(mktemp -d) + { + echo "board: ${board:-?} soc: ${soc:-?}" + echo "kernel: $(uname -r)" + for f in os-fw-version system-fw-version iboot2-version m1n1-stage1-version m1n1-stage2-version; do + printf '%s: ' "$f"; { tr -d '\0' <"$DT/chosen/asahi,$f"; } 2>/dev/null || printf '-'; echo + done + pacman -Q linux-aurora m1n1-aurora m1n1 2>/dev/null || true + printf 'm3-mode: '; cat "$STATE/m3-mode" 2>/dev/null || echo - + echo "m1n1.conf switches:"; grep '^chosen\.' "$M1N1_CONF" 2>/dev/null || echo - + } >"$dir/system.txt" + { dmesg 2>/dev/null || $sudo dmesg; } | grep -iE "$M3_REPORT_DMESG" | grep -viE 'serialnumber|serial number' | + sed -E 's/([0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}/xx:xx:xx:xx:xx:xx/g' >"$dir/dmesg-m3.txt" || true + mkdir -p "$dir/chosen" + for f in "$DT"/chosen/asahi,t8122-* "$DT"/chosen/asahi,t6030-*; do + [[ -e $f ]] && cp -r "$f" "$dir/chosen/" + done + { + echo "== lsusb -t"; lsusb -t 2>/dev/null || echo "(lsusb not installed)" + echo "== /sys/class/typec" + for f in /sys/class/typec/port*; do + [[ -d $f ]] || continue + echo "$(basename "$f"): data_role=$(cat "$f/data_role" 2>/dev/null) power_role=$(cat "$f/power_role" 2>/dev/null)" + done + echo "== /dev/dri"; ls -l /dev/dri 2>/dev/null || true + echo "== drm connectors" + for f in /sys/class/drm/card*-*/status; do [[ -e $f ]] && echo "$f: $(cat "$f")"; done + } >"$dir/usb-display.txt" 2>&1 + tar czf "$out" -C "$dir" . && rm -rf "$dir" + say "Report written to $out + Attach it to your issue at https://github.com/iconidentify/aurora-linux/issues, together + with the serial log if you recorded one. It has no full kernel log, no USB serial numbers + and no MAC addresses." +} + agent_prompt() { cat <<'PROMPT' You are testing an experimental Secure Enclave / Touch ID kernel on an Apple @@ -1934,15 +1978,12 @@ fingerprint. device tree for the Air for the first time. Expected within about a minute: the Omarchy logo, the boot menu, then the same desktop as before. Then report exactly one of: - * WORKS: the desktop comes up as before. Collect and attach: - sudo dmesg | grep -iE 'asahi|agx|gpu|dcp|dart|t8122|reserved|iommu|mailbox|pmp|simpledrm|m1n1' > dmesg-air.txt - ls -d /proc/device-tree/chosen/asahi,t8122-* - and, if that lists anything: - sudo tar czf air-dryrun.tgz -C /proc/device-tree/chosen $(cd /proc/device-tree/chosen && ls -d asahi,t8122-*) - (if it lists nothing, say so instead), plus the serial log with - the line that shows the Mac's serial number removed. Skim - dmesg-air.txt for serial numbers or MAC addresses before posting. - Give the baseline checks below again too. Title it + * WORKS: the desktop comes up as before. Run the test build's + one-liner with --m3-report (same URL, "bash -s -- --m3-report"): + it writes one file, aurora-m3-report--.tgz, in the + current directory. Attach that and the serial log, with the + line that shows the Mac's serial number removed. Give the + baseline checks below again too. Title it " (): M3 Air dry run". * STOPS: m1n1 text and no boot menu, or a black screen for more than two minutes. The owner puts the old boot.bin back from @@ -2005,7 +2046,7 @@ PROMPT } # A reset needs none of the kernel and boot checks; it checks for itself. -preflight_needed() { case ${1:-} in --agent-prompt | --reset-touchid) return 1 ;; *) return 0 ;; esac; } +preflight_needed() { case ${1:-} in --agent-prompt | --reset-touchid | --m3-report) return 1 ;; *) return 0 ;; esac; } # Tests source this file for its functions only. if [[ ${AURORA_SEP_SOURCE_ONLY:-} == 1 ]]; then return 0; fi @@ -2030,6 +2071,7 @@ case ${1:-} in --read-only) READ_ONLY=1; install_all ;; --uninstall) uninstall_all ;; --reset-touchid) shift; reset_touchid "$@" ;; + --m3-report) m3_report ;; --agent-prompt) t=$(newer_release); [[ -n $t ]] && warn "the current release is $t; this plan is for $TAG"; agent_prompt ;; - *) die "unknown option $1 (--read-only, --uninstall, --reset-touchid, --agent-prompt or --m3-handoff)" ;; + *) die "unknown option $1 (--read-only, --uninstall, --reset-touchid, --agent-prompt, --m3-report or --m3-handoff)" ;; esac diff --git a/tools/aurora-sep/tests/test_m3_report.py b/tools/aurora-sep/tests/test_m3_report.py new file mode 100644 index 0000000000000..90b53e75b1de7 --- /dev/null +++ b/tools/aurora-sep/tests/test_m3_report.py @@ -0,0 +1,79 @@ +"""--m3-report: one file for an M3 test report, read-only, without private data. + +Runs against the fake Mac of test_m3_flow, with a fake dmesg and lsusb. +""" +import tarfile +import unittest + +import test_m3_flow as flow + +Base = getattr(flow, "M3FlowBase", None) +if Base is None: + Base = type("Base", (flow.M3FlowTest,), + {n: None for n in dir(flow.M3FlowTest) if n.startswith("test")}) + +DMESG = """\ +[ 0.000000] OF: reserved mem: 0x00000103fff70000..0x00000103fff73fff (16 KiB) map non-reusable uat-handoff +[ 0.055969] [drm] Initialized simpledrm 1.0.0 for 103e1c94000.framebuffer on minor 0 +[ 2.100000] usb 1-1: SerialNumber: ABC123SECRET +[ 3.000000] brcmfmac: wlan0 address 12:34:56:78:9a:bc +[ 3.100000] usb 2-1: new SuperSpeed USB device, mac 12:34:56:78:9a:bc +[ 4.000000] some unrelated line +""" + + +class M3ReportTest(Base): + def setUp(self): + super().setUp() + (self.tmp / "bin/dmesg").write_text("#!/bin/sh\ncat <<'EOF'\n" + DMESG + "EOF\n") + (self.tmp / "bin/dmesg").chmod(0o755) + (self.tmp / "bin/lsusb").write_text("#!/bin/sh\necho '/: Bus 001.Port 001: Dev 001, Class=root_hub, Driver=xhci-hcd/1p, 5000M'\n") + (self.tmp / "bin/lsusb").chmod(0o755) + self.out = self.tmp / "out" + self.out.mkdir() + + def report(self): + proc = self.run_sh(f"cd '{self.out}'\nm3_report") + files = list(self.out.glob("aurora-m3-report-*.tgz")) + self.assertEqual(len(files), 1, proc.stdout + proc.stderr) + with tarfile.open(files[0]) as t: + return proc, {m.name.lstrip("./"): (t.extractfile(m).read() if m.isfile() else None) + for m in t.getmembers()} + + def test_air_report(self): + self.mac("j613") + node = self.tmp / "dt/chosen/asahi,t8122-gpu-powered-identity" + node.mkdir(parents=True) + (node / "id-version").write_bytes(b"\x07\x02\x20\x00") + (self.tmp / "dt/chosen/asahi,t8122-dcp").write_bytes(b"1\0") + (self.tmp / "dt/chosen/asahi,os-fw-version").write_bytes(b"14.7\0") + proc, files = self.report() + self.assertIn("aurora-m3-report-j613-", proc.stdout) + self.assertIn("board: j613 soc: t8122", files["system.txt"].decode()) + self.assertIn("os-fw-version: 14.7", files["system.txt"].decode()) + self.assertEqual(files["chosen/asahi,t8122-gpu-powered-identity/id-version"], b"\x07\x02\x20\x00") + self.assertIn("chosen/asahi,t8122-dcp", files) + dmesg = files["dmesg-m3.txt"].decode() + self.assertIn("uat-handoff", dmesg) + self.assertIn("simpledrm", dmesg) + self.assertNotIn("ABC123SECRET", dmesg) + self.assertNotIn("12:34:56:78:9a:bc", dmesg) + self.assertIn("xx:xx:xx:xx:xx:xx", dmesg) + self.assertNotIn("unrelated", dmesg) + self.assertIn("5000M", files["usb-display.txt"].decode()) + + def test_report_changes_nothing(self): + self.mac("j613") + before = self.boot.read_bytes() + self.report() + self.assertEqual(self.boot.read_bytes(), before) + self.assertNotIn("pacman -U", self.log()) + self.assertNotIn("update-m1n1", self.log()) + + def test_report_needs_no_preflight(self): + src = flow.INSTALLER.read_text() + self.assertIn("--agent-prompt | --reset-touchid | --m3-report) return 1", src) + + +if __name__ == "__main__": + unittest.main() From 5daa76b6f2f0c4a5e58fcaf9338da6fe3a9db988 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:57:10 -0700 Subject: [PATCH 1752/2680] tools: aurora-sep: test a dry-run Air taking a newer dry-run m1n1 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit d283f67f7a1e46d2e538ad3f17074129fd3fd08a) --- tools/aurora-sep/tests/test_m3_air.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tools/aurora-sep/tests/test_m3_air.py b/tools/aurora-sep/tests/test_m3_air.py index 2ef7905360afc..adb2e0f09dc61 100644 --- a/tools/aurora-sep/tests/test_m3_air.py +++ b/tools/aurora-sep/tests/test_m3_air.py @@ -482,6 +482,23 @@ def test_m3_pro_old_record_still_kept(self): self.assertIn("keeping it", out) self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff t6030") + def test_plain_rerun_updates_the_dry_run_m1n1(self): + # Scott's path: an Air on 11.110-test's dry run takes a newer dry-run + # m1n1 from the next test build's plain one-liner, no flag needed. + self.mac("j613") + self.air_install(try_=1) + newer = "m1n1-aurora-1.6.1.aurora8.1-1-aarch64.pkg.tar.zst" + names = [l[len("chosen."):].split("=")[0] for l in DRY_RUN] + self.fixture(newer, None, ["asahi,t8122-gpu"] + names) + proc = self.air_install(pkg=f"{newer} {self.shas[newer]}") + self.assertIn("keeping it", proc.stdout) + boot = self.boot.read_bytes() + self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora8.1-1\n"), boot) + self.assertTrue(boot.endswith(b"UBOOT" + DRY_RUN_LINES), boot) + self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff air-dry-run") + # The boot loader the Mac had before the first test stays the kept copy. + self.assertEqual(self.kept_copy().read_bytes(), b"M1N1:original\n") + def test_install_syncs_after_the_rebuild(self): src = SRC[SRC.index(" m3_verify_bootbin\n"):] self.assertTrue(re.match(r" m3_verify_bootbin\n(\s*#.*\n)*\s*sync\n", src), src[:200]) From 03364aeaa4f5e3700998e52fad1396b40cdb80ec Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 17:16:07 -0700 Subject: [PATCH 1753/2680] tools: aurora-sep: M3 MacBook Air dry run, round 2 (11.110.1-test) The same kernel as 11.110-test. An Air on the dry run takes m1n1-aurora 1.6.1.aurora8.1: it also reads the display's live state when the Mac's own description of its power domain says it is on, prints the panel and PMP details, and publishes the GPU's core count. A plain run of the test build keeps an Air's dry run and updates it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit 34f4738fed57809cc0a2b7e95a24c4c9005eb786) --- tools/aurora-sep/install-aurora-sep.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index d566722398907..533fec91c1235 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -92,6 +92,9 @@ # --m3-handoff, m1n1-aurora aurora8 with a dry run (M3_AIR_DRY_RUN): m1n1 # reads and reports the Air's GPU and display details and starts nothing. A # plain run keeps an Air kernel-only. +# 11.110.1-test: the same kernel; m1n1 aurora8.1's dry run also reads the +# display's live state when the Mac reports its power domain on, the panel +# and PMP details and the GPU's core count; and --m3-report. # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -127,7 +130,7 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. VERSION=7.1.12.aurora2-11.110 -TAG=sep-7.1.12.aurora2-11.110-test +TAG=sep-7.1.12.aurora2-11.110.1-test # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -149,7 +152,7 @@ M3_M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora7-1-aarch64.pkg.tar.zst 547d0d5f9daa283 # "file sha256". It may name the same package as M3_M1N1_PACKAGE once one m1n1 # carries both. Left empty, an Air stays kernel-only and --m3-handoff on an # Air stops with nothing installed. -M3_AIR_M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora8-1-aarch64.pkg.tar.zst d443671d819887b01ed712e1ff86429999bec9f857c6d67bf3040647b5c3176e" +M3_AIR_M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora8.1-1-aarch64.pkg.tar.zst 100919f635e3358027cceb6055f3076f3d29ea11cf2c71721c17a25608dcee13" PINNED="linux-aurora linux-aurora-headers libfprint m1n1-aurora" PIN_BEGIN="# >>> aurora-sep pin (remove with: install-aurora-sep.sh --uninstall)" PIN_END="# <<< aurora-sep pin" From cf408385d743240d9a41dc57d073ceee732229c1 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:39:25 -0700 Subject: [PATCH 1754/2680] tools: aurora-sep: release 11.37 (unified M1/M2/M3 kernel) Kernel 7.1.12.aurora2-11.37 (aurora-linux air/t8122 881930355): the unified M1/M2/M3 kernel. Only the five M3 (T8122) device trees and the 14" J514S tree change from 11.36; every other Mac's trees are identical. The T8122 GPU and display nodes stay disabled and the driver fails closed, so an M3 Air is kernel-only here (framebuffer), exactly as in 11.36; the Air GPU/display handoff stays in the test channel. Installer: - M3 Pro 16" J516S keeps its display+GPU handoff, now on m1n1-aurora 1.6.1.aurora7 (passes each Mac's PMP values; kernel reads them). - 14" J514S handoff is opt-in with --m3-handoff (not on by default). - M3 Air: no Air m1n1 shipped, so it stays kernel-only; --m3-handoff on an Air refuses cleanly. - New --m3-report writes one sanitised report file on an M3. - Refuses chips this release can't boot (M3 Ultra, M4+), nothing changed (#32). - The m3-serial.py tool-lookup fix carries over. M1, M2 and the MacBook Neo are unchanged. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit 2ea711a7bc5d4385caebc4a8675ac9f2ad7e4600) --- tools/aurora-sep/install-aurora-sep.sh | 28 +++++++++++++------------- tools/aurora-sep/tests/test_m3_air.py | 17 ++++++++++++---- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 533fec91c1235..fc1af420c8e25 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -87,14 +87,14 @@ # Every other M3 stays kernel-only, and --m3-handoff lets an M3 Pro owner try # the handoff on a model not yet on the list (see --agent-prompt). M1, M2 and # the Neo are unchanged. -# 11.110-test (not a release): kernel 11.110 (aurora-linux air/t8122) and, for -# an M3 MacBook Air (T8122, 13" J613 and 15" J615) that opts in with -# --m3-handoff, m1n1-aurora aurora8 with a dry run (M3_AIR_DRY_RUN): m1n1 -# reads and reports the Air's GPU and display details and starts nothing. A -# plain run keeps an Air kernel-only. -# 11.110.1-test: the same kernel; m1n1 aurora8.1's dry run also reads the -# display's live state when the Mac reports its power domain on, the panel -# and PMP details and the GPU's core count; and --m3-report. +# 11.37: the unified M1/M2/M3 kernel, now 11.110 (aurora-linux air/t8122). Its +# only device-tree change from 11.36 is the five M3 (T8122) trees; the T8122 +# GPU and display nodes stay disabled and the driver fails closed, so without +# an opt-in M3 Air m1n1 (shipped only in the -test channel) an Air is kernel- +# only, exactly as in 11.36. The 16" J516S keeps its display + GPU handoff +# (now m1n1 aurora7, which also passes each Mac's PMP values). --m3-handoff is +# available on the 14" J514S (opt-in, not yet on by default). This release adds +# --m3-report and refuses chips it does not support (SUPPORTED_SOCS; #32). # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -129,8 +129,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.110 -TAG=sep-7.1.12.aurora2-11.110.1-test +VERSION=7.1.12.aurora2-11.37 +TAG=sep-7.1.12.aurora2-11.37 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -138,8 +138,8 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst eeab04119026280308987fb7ac2f5f8996504012d2a6d38fe645f5609cc56986" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 35af393eaaa3eb646217283566e769be5aadfe822d08b3aa0a285b1dd7f26722" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst 1a31635c7073d4a90692a88c9af0b87931caaabd0888c31c1cf1b2335f8f30b5" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst c6624e2996e5185ef9a7d24059e4ed639c05697f7e5e2181bd19539e524c9f57" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20261003-1-any.pkg.tar.zst 29b0360fac8c257d754e64bd1b9c33c487eb2595dd3c31e9138d7a476afa3d64" "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" @@ -152,7 +152,7 @@ M3_M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora7-1-aarch64.pkg.tar.zst 547d0d5f9daa283 # "file sha256". It may name the same package as M3_M1N1_PACKAGE once one m1n1 # carries both. Left empty, an Air stays kernel-only and --m3-handoff on an # Air stops with nothing installed. -M3_AIR_M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora8.1-1-aarch64.pkg.tar.zst 100919f635e3358027cceb6055f3076f3d29ea11cf2c71721c17a25608dcee13" +M3_AIR_M1N1_PACKAGE="" PINNED="linux-aurora linux-aurora-headers libfprint m1n1-aurora" PIN_BEGIN="# >>> aurora-sep pin (remove with: install-aurora-sep.sh --uninstall)" PIN_END="# <<< aurora-sep pin" @@ -316,7 +316,7 @@ M3_AIR_DCP=0 # chosen.asahi,t8122-dcp=1 is a dry run only while m1n1 pins no T8122 DCP # firmware image (aurora8 pins none); a release whose m1n1 pins one must give # the dry run another variant name in m3_variant, so no Air gets it unasked. -M3_AIR_DRY_RUN=1 +M3_AIR_DRY_RUN=0 M3_AIR_DRY_RUN_SWITCHES="chosen.asahi,t8122-gpu-diag=1 chosen.asahi,t8122-gpu-handoff-diag=1 chosen.asahi,t8122-gpu-power-diag=1 chosen.asahi,t8122-dcp=1" # The handoff is tested with one macOS system-firmware stub only, 14.8.3 (GPU # firmware 14.8.3, DCP 14.7), which the Omarchy installer gives every M3. m1n1 diff --git a/tools/aurora-sep/tests/test_m3_air.py b/tools/aurora-sep/tests/test_m3_air.py index adb2e0f09dc61..2c749515e193e 100644 --- a/tools/aurora-sep/tests/test_m3_air.py +++ b/tools/aurora-sep/tests/test_m3_air.py @@ -345,15 +345,24 @@ def test_switches(self): self.assertEqual(self.sh("m3_switches").stdout.strip(), shell_value("M3_SWITCHES")) def test_shipped_settings(self): - # A release decision: change this test with it. - self.assertEqual(re.search(r"^M3_AIR_DRY_RUN=(\S+)$", SRC, re.M).group(1), "1") + # Two shipped shapes, both valid, depending on the build: + # - a -test build ships the Air dry run: DRY_RUN=1 and a real Air m1n1. + # - a Latest build keeps the Air kernel-only: an empty Air m1n1, so + # --m3-handoff on an Air refuses and nothing is switched on. + # The dry-run switch list, when used, is always these four and never + # asks for the GPU setup itself. self.assertEqual(DRY_RUN, ["chosen.asahi,t8122-gpu-diag=1", "chosen.asahi,t8122-gpu-handoff-diag=1", "chosen.asahi,t8122-gpu-power-diag=1", "chosen.asahi,t8122-dcp=1"]) - # The dry run never asks for the GPU setup itself. self.assertNotIn("chosen.asahi,t8122-gpu=1", DRY_RUN) - self.assertTrue(re.search(r'^M3_AIR_M1N1_PACKAGE="m1n1-aurora-\S+ [0-9a-f]{64}"$', SRC, re.M)) + dry = re.search(r"^M3_AIR_DRY_RUN=(\S+)$", SRC, re.M).group(1) + air_pkg = shell_value("M3_AIR_M1N1_PACKAGE") + if air_pkg: + self.assertEqual(dry, "1", "a shipped Air m1n1 must ship the dry run") + self.assertRegex(air_pkg, r"^m1n1-aurora-\S+ [0-9a-f]{64}$") + else: + self.assertEqual(dry, "0", "no Air m1n1 shipped, so the dry run is off") def test_conf_block(self): self.mac("j613") From d2e683b6ebba11c013c44ee4e67ba30cd1778819 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 00:39:51 -0700 Subject: [PATCH 1755/2680] tools/aurora-sep: release 11.38 Kernel 11.38, built from 1613116279ab with the unified M1/M2/M3 configuration. The device trees and boot loaders are the same as 11.37's. - The display driver checks each display's mode list and attributes before using them, keeps the display's identification data in bounds, and refreshes a connector's state before reporting a plug or unplug. - The M3 Pro PMP's boot settings and startup tables are checked completely before the display starts it. - The M3 GPU settings come from each chip's own table, and the GPU firmware's layout is checked before use. - Each PCI USB controller reserves only the interrupt it uses, instead of extra ones, leaving interrupts free for a dock's other devices. An M3 MacBook Air stays kernel-only. On an Air running a test build the installer stops without changing anything, so the test build stays. Installer tests pass; shellcheck reports the same as for 11.37. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 (cherry picked from commit d14f756c9a65752ca532b93391f775387ba4f191) --- tools/aurora-sep/install-aurora-sep.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index fc1af420c8e25..5c5342f03e52d 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -95,6 +95,12 @@ # (now m1n1 aurora7, which also passes each Mac's PMP values). --m3-handoff is # available on the 14" J514S (opt-in, not yet on by default). This release adds # --m3-report and refuses chips it does not support (SUPPORTED_SOCS; #32). +# 11.38: the same device trees and boot loaders as 11.37. The display driver +# checks each display's modes and attributes before using them and refreshes a +# connector's state before reporting a plug or unplug; the M3 Pro PMP's boot +# settings and startup tables are checked before it starts; the M3 GPU +# settings are per chip; each PCI USB controller reserves only the interrupt it +# uses, leaving interrupts for a dock's other devices. An M3 Air stays kernel-only. # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -129,8 +135,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.37 -TAG=sep-7.1.12.aurora2-11.37 +VERSION=7.1.12.aurora2-11.38 +TAG=sep-7.1.12.aurora2-11.38 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -138,8 +144,8 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst 1a31635c7073d4a90692a88c9af0b87931caaabd0888c31c1cf1b2335f8f30b5" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst c6624e2996e5185ef9a7d24059e4ed639c05697f7e5e2181bd19539e524c9f57" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst f1cecde567b42b93a03c26b78fc13ae8cf0a93e40a429c509df7b62836f9de1a" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 8c9fd8d30fc0392c8b0d4f690bf5c4ef02daea18529a645935bbe83b5a22e746" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20261003-1-any.pkg.tar.zst 29b0360fac8c257d754e64bd1b9c33c487eb2595dd3c31e9138d7a476afa3d64" "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" From aea3f19912dc62781058d7ed30529f271f43228b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:35:28 -0700 Subject: [PATCH 1756/2680] tools/aurora-sep: release 12.0 12.0 is one release with one kernel for every Mac this script supports. It replaces the 11.1xx test builds, and its version sorts above them for pacman (7.1.12.aurora2-12.0 > -11.111 > -11.110 > -11.38). M3 boot loaders: - The 16" M3 Pro (J516S) default handoff moves to m1n1-aurora 1.6.1.aurora8.5-2, which also hands over the second external display processor. A plain run upgrades a J516S that has the handoff, as the recorded t6030 variant is unchanged. - Other M3 Pros keep the handoff opt-in with --m3-handoff; M3_HANDOFF_BOARDS stays "j516s". - An M3 MacBook Air gets the same package only with --m3-handoff, as the display handoff with GPU diagnostics: the dry run's four switches with an m1n1 that hands the internal display over. Nothing starts the GPU, and the desktop stays on the boot framebuffer until the kernel has a T8122 PMP description. It is recorded as a new variant, air-display-handoff-85, so an Air on an earlier test build's boot loader (air-display-handoff, aurora8.3 and 8.4) stops on a plain run with nothing installed, and moves only with a new --m3-handoff. - M1 and M2 keep m1n1-aurora aurora3, and the Neo keeps its own m1n1. The Air variant logic follows the 11.111 and 11.111.1 test installers, ported by hand onto this script, which already has the M3 report, the unsupported-SoC refusal and the m3-serial fix. The agent prompt's Air case describes the display handoff and the report to send back, and the J516S case asks for each external display port. The two linux-aurora checksums are placeholders until the lab package build of this tree; the installer stops on a checksum mismatch, so the script can't install anything until they are filled in. Installer tests: 137 run, 136 pass, 1 skipped. Four new Air tests cover the display variant, its opt-in, and an Air on the 11.111.1 test build's handoff (refused on a plain run, moved by --m3-handoff with its original restore copy kept). shellcheck: the same four notes as 11.38. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 2a30a748f3036895336dc7de9d98ad5dff25c265) --- tools/aurora-sep/install-aurora-sep.sh | 154 +++++++++++++++++-------- tools/aurora-sep/tests/test_m3_air.py | 110 +++++++++++++++--- 2 files changed, 197 insertions(+), 67 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 5c5342f03e52d..01a234eb664e5 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -101,6 +101,19 @@ # settings and startup tables are checked before it starts; the M3 GPU # settings are per chip; each PCI USB controller reserves only the interrupt it # uses, leaving interrupts for a dock's other devices. An M3 Air stays kernel-only. +# 12.0: one release and one kernel for every Mac this script supports; the +# 11.1xx test builds end here. On the 16" M3 Pro (J516S) the kernel drives +# external displays on the USB-C and HDMI ports, and its m1n1 moves to +# aurora8.5, which also hands over the second external display processor. The +# M3 Type-C PHY changes behind this apply to every M3. Other M3 Pros keep the +# handoff opt-in (--m3-handoff). An M3 MacBook Air stays kernel-only unless its +# owner asks with --m3-handoff for m1n1's display handoff with GPU diagnostics +# (the same aurora8.5; the desktop stays on the boot framebuffer). An Air that +# has an earlier test build's boot loader must ask again with --m3-handoff. +# The kernel also describes the Air's internal display and GPU configuration, +# both inert for now, and the M2 Max Neural Engine cleans up after a failed +# probe (iconidentify/aurora-linux#37, Joshua Warren). M1, M2 and the Neo +# keep their boot loaders. # It replaces linux-asahi (or linux-aurora) as a pacman package, # so mkinitcpio and update-m1n1 run from their own hooks; on a GRUB Mac this # script regenerates grub.cfg and keeps the previous kernel as a fallback entry. @@ -135,8 +148,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-11.38 -TAG=sep-7.1.12.aurora2-11.38 +VERSION=7.1.12.aurora2-12.0 +TAG=sep-7.1.12.aurora2-12.0 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG @@ -144,21 +157,22 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst f1cecde567b42b93a03c26b78fc13ae8cf0a93e40a429c509df7b62836f9de1a" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst 8c9fd8d30fc0392c8b0d4f690bf5c4ef02daea18529a645935bbe83b5a22e746" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst PENDING-12.0-LAB-BUILD" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst PENDING-12.0-LAB-BUILD" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20261003-1-any.pkg.tar.zst 29b0360fac8c257d754e64bd1b9c33c487eb2595dd3c31e9138d7a476afa3d64" "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" ) # Only for an M3 on the handoff path (see m3_plan), in place of the m1n1-aurora -# above: the same m1n1 plus the T6030 display and GPU handoff. -M3_M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora7-1-aarch64.pkg.tar.zst 547d0d5f9daa283749d84e5a9c875fee06ce449eeb8252e41c11b3ce63e11184" +# above: the same m1n1 plus the T6030 display and GPU handoff. aurora8.5 also +# hands over the second external display processor. +M3_M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora8.5-2-aarch64.pkg.tar.zst 7bbf89455b9a96487ee05bfef18ee3e5c8ec57c1af3f97fc15548774d5e92a44" # Only for an M3 MacBook Air on the handoff path, in place of the m1n1-aurora -# above: m1n1-aurora with the T8122 switches (aurora8 is the first), as -# "file sha256". It may name the same package as M3_M1N1_PACKAGE once one m1n1 -# carries both. Left empty, an Air stays kernel-only and --m3-handoff on an -# Air stops with nothing installed. -M3_AIR_M1N1_PACKAGE="" +# above: m1n1-aurora with the T8122 switches, as "file sha256". From aurora8.5 +# one M3 m1n1 carries both, so this names the same package as M3_M1N1_PACKAGE. +# Left empty, an Air stays kernel-only and --m3-handoff on an Air stops with +# nothing installed. +M3_AIR_M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora8.5-2-aarch64.pkg.tar.zst 7bbf89455b9a96487ee05bfef18ee3e5c8ec57c1af3f97fc15548774d5e92a44" PINNED="linux-aurora linux-aurora-headers libfprint m1n1-aurora" PIN_BEGIN="# >>> aurora-sep pin (remove with: install-aurora-sep.sh --uninstall)" PIN_END="# <<< aurora-sep pin" @@ -293,9 +307,9 @@ is_m3_air() { # three M3_SWITCHES lines; update-m1n1 copies chosen.* lines from # /etc/m1n1.conf into every rebuild, so they survive updates. # An M3 MacBook Air takes the same path with its own m1n1 -# (M3_AIR_M1N1_PACKAGE) and switches (m3_switches): the GPU -# handoff only, and only with --m3-handoff until its board is in -# M3_HANDOFF_BOARDS. +# (M3_AIR_M1N1_PACKAGE) and switches (m3_switches): the display +# handoff with GPU diagnostics, and only with --m3-handoff until +# its board is in M3_HANDOFF_BOARDS. # M1, M2 and the Neo keep the m1n1-aurora in PACKAGES: only an M3 on the # handoff path gets the newer m1n1. UPDATE_M1N1_CONF=/etc/default/update-m1n1 @@ -317,12 +331,23 @@ M3_AIR_DCP=0 # A dry run in place of the handoff (M3_AIR_DRY_RUN=1): m1n1 reads the Air's # GPU and display details from the boot firmware and reports them on the # serial console and under /chosen, powers the GPU only for a bounded -# identity read, and starts nothing. The desktop stays as it was. This build -# ships the dry run: the Air's GPU can't be started until those details are in. +# identity read, and starts nothing. The desktop stays as it was. # chosen.asahi,t8122-dcp=1 is a dry run only while m1n1 pins no T8122 DCP # firmware image (aurora8 pins none); a release whose m1n1 pins one must give # the dry run another variant name in m3_variant, so no Air gets it unasked. -M3_AIR_DRY_RUN=0 +# The display handoff (M3_AIR_DISPLAY_HANDOFF=1) is that case: the same +# switches with an m1n1 that pins the image (aurora8.3 and later), so m1n1 +# hands the internal display over and publishes its checked state for Linux. +# The kernel keeps the boot framebuffer until it has a T8122 PMP description, +# and the GPU switches stay diagnostics: nothing starts the GPU. +# M3_AIR_DISPLAY_VARIANT names it in $STATE/m3-mode. A plain run keeps an +# Air's boot loader only while that name matches, so it changes with each m1n1 +# that changes what an Air boots: an Air on an earlier test build's boot loader +# (air-display-handoff, aurora8.3 and 8.4) moves to this one only with a new +# --m3-handoff. +M3_AIR_DISPLAY_HANDOFF=1 +M3_AIR_DISPLAY_VARIANT="air-display-handoff-85" +M3_AIR_DRY_RUN=1 M3_AIR_DRY_RUN_SWITCHES="chosen.asahi,t8122-gpu-diag=1 chosen.asahi,t8122-gpu-handoff-diag=1 chosen.asahi,t8122-gpu-power-diag=1 chosen.asahi,t8122-dcp=1" # The handoff is tested with one macOS system-firmware stub only, 14.8.3 (GPU # firmware 14.8.3, DCP 14.7), which the Omarchy installer gives every M3. m1n1 @@ -469,6 +494,8 @@ m3_air_package_ok() { m3_handoff_name() { if ! is_m3_air; then echo "M3 Pro display and GPU handoff" + elif [[ $M3_AIR_DISPLAY_HANDOFF == 1 ]]; then + echo "M3 Air display handoff with GPU diagnostics" elif [[ $M3_AIR_DRY_RUN == 1 ]]; then echo "M3 Air GPU and display dry run" elif [[ $M3_AIR_DCP == 1 ]]; then @@ -626,6 +653,7 @@ m3_recorded_mode() { # into a GPU start) without a new --m3-handoff. m3_variant() { if ! is_m3_air; then echo t6030 + elif [[ $M3_AIR_DISPLAY_HANDOFF == 1 ]]; then echo "$M3_AIR_DISPLAY_VARIANT" elif [[ $M3_AIR_DRY_RUN == 1 ]]; then echo air-dry-run elif [[ $M3_AIR_DCP == 1 ]]; then echo air-gpu-dcp else echo air-gpu @@ -654,8 +682,9 @@ m3_plan() { if ((M3_TRY == 0)) && [[ $(m3_recorded_mode) == handoff ]]; then if ((air)) && [[ $(m3_recorded_variant) != "$(m3_variant)" ]]; then die "M3 MacBook Air ($board): this Mac has an earlier test build's boot loader - ($(m3_recorded_variant)), and this release's Air boot loader does something else: the - $(m3_handoff_name). Run this again with --m3-handoff to switch to it. Nothing was installed." + ($(m3_recorded_variant)), and this release's Air boot loader is a different one: the + $(m3_handoff_name) ($(m3_variant)). Run this again with --m3-handoff to switch to it. + Nothing was installed." fi M3_TRY=1 if ((air)); then @@ -718,6 +747,12 @@ m3_plan() { if ((air)); then if is_m3_handoff_board; then say "M3 MacBook Air ($board, macOS $M3_STUB_VERSION stub): installing m1n1 with the $(m3_handoff_name)" + elif [[ $M3_AIR_DISPLAY_HANDOFF == 1 ]]; then + warn "--m3-handoff: trying m1n1's display handoff on this M3 MacBook Air ($board). + It publishes the checked display state for Linux and collects GPU diagnostics; + it does not start the GPU. Native display still needs T8122 PMP support, so the + desktop stays on the boot framebuffer and renders in software. + This replaces the Mac's boot loader; the steps to put the old one back from macOS follow." elif [[ $M3_AIR_DRY_RUN == 1 ]]; then warn "--m3-handoff: installing m1n1's GPU and display dry run on this M3 MacBook Air ($board). It reads the Air's GPU and display details and reports them for the bring-up, powering the @@ -1371,7 +1406,14 @@ install_all() { if is_neo; then neo_radio_notice; fi pacman -Q linux-aurora libfprint aurora-touchid echo - if [[ $M3_MODE == handoff ]] && is_m3_air && [[ $M3_AIR_DRY_RUN == 1 ]]; then + if [[ $M3_MODE == handoff ]] && is_m3_air && [[ $M3_AIR_DISPLAY_HANDOFF == 1 ]]; then + say "Done. Reboot with the serial recorder running and someone watching: expect the Omarchy + logo, the boot menu, then the same desktop on the boot framebuffer. Check the serial log + for the T8122 handoff result and the \"PMP: T8122:\" lines. + Native display and GPU acceleration are not enabled. Touch ID is not supported on M3 yet. + What to send back: case D of step 11 of the test plan that the --agent-prompt command below + prints." + elif [[ $M3_MODE == handoff ]] && is_m3_air && [[ $M3_AIR_DRY_RUN == 1 ]]; then say "Done. Reboot with the serial recorder running and someone watching: expect the Omarchy logo, the boot menu, then the desktop on the boot framebuffer, as before. This build only reads and reports, so nothing on the desktop changes. Touch ID is not supported on M3 yet. @@ -1900,6 +1942,10 @@ fingerprint. Reboot with the owner watching. Expected within about a minute: the Omarchy logo, the boot menu, then the desktop on the built-in display at 3456x2234, GPU accelerated. Then run the checks below and report. + External displays: try each USB-C port and the HDMI port, one at a + time and then HDMI with one USB-C display, and say which port, which + display and which mode lit. Sleep has known limits on M3; if you try + it with an external display attached, quote what happened. B. any other apple,t6030 model (MacBook Pro 14" M3 Pro, J514S): NOT ON THE LIST YET. A REPORT FROM YOU IS HOW IT GETS ADDED. @@ -1951,15 +1997,15 @@ fingerprint. M3 MacBook Air (j613): installing the kernel only ... Run the checks below on that first and report it: it is the baseline every later Air test is compared with. - Then, on a test build that carries the Air's dry run (its message says - so), run the dry run on this Air: + Then, to try m1n1's display handoff with GPU diagnostics on this Air: - What it does: it replaces this Mac's boot loader (m1n1) with one - that reads the Air's GPU and display details from the boot - firmware and reports them: on the serial console, and under + that hands the built-in display over and publishes its checked + state for Linux, and reads the Air's GPU, PMP and display clock + details and reports them: on the serial console, and under /proc/device-tree/chosen in Linux. It powers the GPU only for a - short identity read and switches nothing on. The screen and the - desktop stay exactly as in the baseline. These reports are what - the next build, the one that starts the Air's GPU, is made from. + short identity read and does not start it. Native display still + needs T8122 PMP support in the kernel, so the desktop stays on the + boot framebuffer and renders in software, as in the baseline. - Only with the owner's explicit agreement, with the owner at the Mac for the first boot, with the Aurora maintainer told first and reachable while it runs, and only if the owner can start macOS on @@ -1970,34 +2016,40 @@ fingerprint. DFU ports: tools/aurora-m3/m3-serial.py in this repository ("sudo python3 m3-serial.py reboot" on the second Mac). Most of the display details are only printed there. - - Run the test build's one-liner with --m3-handoff (the release - tag's URL from its message, not releases/latest): - curl -fsSL https://github.com/iconidentify/aurora-linux/releases/download//install-aurora-sep.sh | bash -s -- --m3-handoff - It warns "--m3-handoff: installing m1n1's GPU and display dry run", - then prints the restore steps, from macOS and from Linux: keep - them, they name this Mac's EFI partition and boot.bin. Near the end - it must print "m1n1's boot.bin is this release's m1n1 with the M3 - Air handoff switches". An error ending "Nothing was installed" - changed nothing: report it. For any other error, do not reboot: - report it first. + - Run the one-liner with --m3-handoff, also on an Air that has an + earlier test build's boot loader (a plain run there stops with + "Run this again with --m3-handoff" and changes nothing): + curl -fsSL https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh | bash -s -- --m3-handoff + It warns "--m3-handoff: trying m1n1's display handoff", then + prints the restore steps, from macOS and from Linux: keep them, + they name this Mac's EFI partition and boot.bin. Near the end it + must print "m1n1's boot.bin is this release's m1n1 with the M3 Air + handoff switches". An error ending "Nothing was installed" changed + nothing: report it. For any other error, do not reboot: report it + first. - Reboot: run sync on the Air, then, if the second Mac records the serial console, start "sudo python3 m3-serial.py reboot" there; it reboots the Air itself. Without one, run "sudo reboot" on the Air. - The owner watches the screen. This boot also uses this release's - device tree for the Air for the first time. Expected within about a - minute: the Omarchy logo, the boot menu, then the same desktop as - before. Then report exactly one of: - * WORKS: the desktop comes up as before. Run the test build's - one-liner with --m3-report (same URL, "bash -s -- --m3-report"): - it writes one file, aurora-m3-report--.tgz, in the - current directory. Attach that and the serial log, with the - line that shows the Mac's serial number removed. Give the + The owner watches the screen. Expected within about a minute: the + Omarchy logo, the boot menu, then the same desktop as before. Then + report exactly one of: + * WORKS: the desktop comes up as before. Run the one-liner with + --m3-report ("bash -s -- --m3-report"): it writes one file, + aurora-m3-report--.tgz, in the current directory. + Attach that and the serial log, with the line that shows the + Mac's serial number removed. A handoff that went through prints + "DCP: T8122: handoff published" on the serial console, and Linux + keeps the display disabled with "no T8122 PMP description is + built in"; if a handoff check refused instead, quote the refusal + line (the boot framebuffer is then expected too). The serial log + also has "PMP: T8122:" lines, and the report carries + /chosen/asahi,t8122-pmp, -pmp-facts and -clock-facts. Give the baseline checks below again too. Title it - " (): M3 Air dry run". + " (): M3 Air display handoff". * STOPS: m1n1 text and no boot menu, or a black screen for more than two minutes. The owner puts the old boot.bin back from macOS with the printed steps, starts Linux, and runs the printed - M1N1_UPDATE_DISABLED line so no update rebuilds the test boot + M1N1_UPDATE_DISABLED line so no update rebuilds the new boot loader. Do not run the one-liner again or --uninstall until the maintainer says so. Report what the screen showed and when, with the serial log if there is one: it says where m1n1 stopped. @@ -2006,9 +2058,9 @@ fingerprint. sudo journalctl -b -k | grep -iE 'asahi|gpu|g15|t8122|dcp|m1n1' then run the two printed lines from Linux (put the old boot.bin back, then M1N1_UPDATE_DISABLED), reboot, and report. - - Later plain runs of this test build keep the dry run on this Air. - To go back, use the printed restore lines; ask the maintainer - before using --uninstall on an Air. + - Later plain runs keep the display handoff on this Air. To go back, + use the printed restore lines; ask the maintainer before using + --uninstall on an Air. Checks for every M3 (quote the output; on a kernel-only M3 the handoff lines are expected to be missing, so say so): diff --git a/tools/aurora-sep/tests/test_m3_air.py b/tools/aurora-sep/tests/test_m3_air.py index 2c749515e193e..87e29b3658771 100644 --- a/tools/aurora-sep/tests/test_m3_air.py +++ b/tools/aurora-sep/tests/test_m3_air.py @@ -6,7 +6,9 @@ only with M3_AIR_DCP=1) to /etc/m1n1.conf. Every case sets M3_AIR_M1N1_PACKAGE and M3_AIR_DRY_RUN itself, so the tests don't depend on what the installer ships with. M3AirDryRunTest covers the dry run, which -writes M3_AIR_DRY_RUN_SWITCHES instead. +writes M3_AIR_DRY_RUN_SWITCHES instead, and the display handoff +(M3_AIR_DISPLAY_HANDOFF=1), which writes the same switches under its own +variant name. """ from pathlib import Path import re @@ -27,6 +29,13 @@ def shell_value(name): return re.search(rf'^{name}="([^"]*)"$', SRC, re.M).group(1) +def shell_word(name): + return re.search(rf'^{name}=(\S+)$', SRC, re.M).group(1) + + +DISPLAY_VARIANT = shell_value("M3_AIR_DISPLAY_VARIANT") + + class M3AirTest(flow.M3FlowBase): def setUp(self): super().setUp() @@ -37,9 +46,12 @@ def setUp(self): dry_run = 0 - def sh(self, body, pkg=None, dcp=0, boards=None, check=True): + def sh(self, body, pkg=None, dcp=0, boards=None, check=True, display=0, variant=None): pkg = self.air_pkg if pkg is None else pkg - pre = f'M3_AIR_M1N1_PACKAGE="{pkg}"\nM3_AIR_DCP={dcp}\nM3_AIR_DRY_RUN={self.dry_run}\n' + pre = (f'M3_AIR_M1N1_PACKAGE="{pkg}"\nM3_AIR_DCP={dcp}\nM3_AIR_DRY_RUN={self.dry_run}\n' + f'M3_AIR_DISPLAY_HANDOFF={display}\n') + if variant is not None: + pre += f'M3_AIR_DISPLAY_VARIANT={variant}\n' if boards is not None: pre += f'M3_HANDOFF_BOARDS="{boards}"\n' return self.run_sh(pre + body, check=check) @@ -323,7 +335,7 @@ def test_m1_unchanged_by_air_settings(self): class M3AirDryRunTest(M3AirTest): - """The dry run this build ships: read and report, switch nothing on.""" + """The Air dry-run profile, and the display handoff that ships with its switches.""" dry_run = 1 def setUp(self): @@ -345,24 +357,23 @@ def test_switches(self): self.assertEqual(self.sh("m3_switches").stdout.strip(), shell_value("M3_SWITCHES")) def test_shipped_settings(self): - # Two shipped shapes, both valid, depending on the build: - # - a -test build ships the Air dry run: DRY_RUN=1 and a real Air m1n1. - # - a Latest build keeps the Air kernel-only: an empty Air m1n1, so - # --m3-handoff on an Air refuses and nothing is switched on. - # The dry-run switch list, when used, is always these four and never - # asks for the GPU setup itself. + # A release decision: change this test with it. 12.0 ships the Air + # display handoff with the dry run's switches, opt-in only, from the + # same m1n1 as the M3 Pros, under a variant name no test build used. self.assertEqual(DRY_RUN, ["chosen.asahi,t8122-gpu-diag=1", "chosen.asahi,t8122-gpu-handoff-diag=1", "chosen.asahi,t8122-gpu-power-diag=1", "chosen.asahi,t8122-dcp=1"]) + # The dry run never asks for the GPU setup itself. self.assertNotIn("chosen.asahi,t8122-gpu=1", DRY_RUN) - dry = re.search(r"^M3_AIR_DRY_RUN=(\S+)$", SRC, re.M).group(1) + self.assertEqual(shell_word("M3_AIR_DRY_RUN"), "1") + self.assertEqual(shell_word("M3_AIR_DISPLAY_HANDOFF"), "1") air_pkg = shell_value("M3_AIR_M1N1_PACKAGE") - if air_pkg: - self.assertEqual(dry, "1", "a shipped Air m1n1 must ship the dry run") - self.assertRegex(air_pkg, r"^m1n1-aurora-\S+ [0-9a-f]{64}$") - else: - self.assertEqual(dry, "0", "no Air m1n1 shipped, so the dry run is off") + self.assertRegex(air_pkg, r"^m1n1-aurora-\S+ [0-9a-f]{64}$") + self.assertEqual(air_pkg, shell_value("M3_M1N1_PACKAGE")) + # 11.111-test and 11.111.1-test recorded air-display-handoff. + self.assertNotIn(DISPLAY_VARIANT, ("air-display-handoff", "air-dry-run", "air-gpu", "air-gpu-dcp")) + self.assertRegex(DISPLAY_VARIANT, r"^air-display-handoff-\S+$") def test_conf_block(self): self.mac("j613") @@ -508,6 +519,73 @@ def test_plain_rerun_updates_the_dry_run_m1n1(self): # The boot loader the Mac had before the first test stays the kept copy. self.assertEqual(self.kept_copy().read_bytes(), b"M1N1:original\n") + # The display handoff (M3_AIR_DISPLAY_HANDOFF=1) + + def test_display_handoff_needs_fresh_opt_in_and_keeps_restore(self): + self.mac("j613") + self.air_install(try_=1) + before = self.boot.read_bytes() + before_conf = self.m1n1_conf.read_bytes() + before_downloads = self.downloaded() + proc = self.air_install(display=1, check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("Nothing was installed", proc.stderr) + self.assertIn("display handoff with GPU diagnostics", proc.stderr) + self.assertEqual(self.boot.read_bytes(), before) + self.assertEqual(self.m1n1_conf.read_bytes(), before_conf) + self.assertEqual(self.downloaded(), before_downloads) + proc = self.air_install(try_=1, display=1) + self.assertIn("Native display still needs T8122 PMP support", proc.stderr) + self.assertIn("does not start the GPU", proc.stderr) + self.assertIn("Native display and GPU acceleration are not enabled", proc.stdout) + self.assertEqual((self.state / "m3-mode").read_text().strip(), f"handoff {DISPLAY_VARIANT}") + self.assertTrue(self.boot.read_bytes().endswith(b"UBOOT" + DRY_RUN_LINES)) + self.assertNotIn(AIR_GPU, self.boot.read_bytes()) + self.assertEqual(self.kept_copy().read_bytes(), b"M1N1:original\n") + proc = self.air_install(display=1) + self.assertIn("keeping it", proc.stdout) + self.assertEqual(self.kept_copy().read_bytes(), b"M1N1:original\n") + + def test_shipped_display_profile_uses_diagnostics_without_gpu_start(self): + self.mac("j613") + proc = self.run_sh("M3_TRY=1\nm3_plan\nm3_variant\nm3_switches") + self.assertIn(DISPLAY_VARIANT, proc.stdout) + self.assertIn("does not start the GPU", proc.stderr) + self.assertIn("Native display still needs T8122 PMP support", proc.stderr) + switches = [s for s in proc.stdout.split() if s.startswith("chosen.")] + self.assertEqual(switches, DRY_RUN) + self.assertNotIn("chosen.asahi,t8122-gpu=1", switches) + + def test_release_display_variant_requires_opt_in(self): + self.mac("j615") + self.assertEqual(self.plan(display=1), "kernel") + self.assertEqual(self.plan(try_=1, display=1), "handoff") + + def test_test_build_display_handoff_does_not_move_silently(self): + # Scott's path: an Air on 11.111.1-test's display handoff (variant + # air-display-handoff, aurora8.4) runs this release's plain one-liner. + # It must stop before downloading anything; --m3-handoff moves it. + self.mac("j613") + self.air_install(try_=1, display=1, variant="air-display-handoff") + self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff air-display-handoff") + before = self.boot.read_bytes() + before_conf = self.m1n1_conf.read_bytes() + before_downloads = self.downloaded() + proc = self.air_install(display=1, check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("(air-display-handoff)", proc.stderr) + self.assertIn(f"({DISPLAY_VARIANT})", proc.stderr) + self.assertIn("--m3-handoff", proc.stderr) + self.assertIn("Nothing was installed", proc.stderr) + self.assertEqual(self.boot.read_bytes(), before) + self.assertEqual(self.m1n1_conf.read_bytes(), before_conf) + self.assertEqual(self.downloaded(), before_downloads) + proc = self.air_install(try_=1, display=1) + self.assertEqual((self.state / "m3-mode").read_text().strip(), f"handoff {DISPLAY_VARIANT}") + self.assertTrue(self.boot.read_bytes().endswith(b"UBOOT" + DRY_RUN_LINES)) + # The boot loader the Mac had before its first test stays the kept copy. + self.assertEqual(self.kept_copy().read_bytes(), b"M1N1:original\n") + def test_install_syncs_after_the_rebuild(self): src = SRC[SRC.index(" m3_verify_bootbin\n"):] self.assertTrue(re.match(r" m3_verify_bootbin\n(\s*#.*\n)*\s*sync\n", src), src[:200]) From 1a2dd71da0ee90273410db467d42f0f340d58dc4 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:01:39 -0700 Subject: [PATCH 1757/2680] tools/aurora-sep: check the M3 m1n1's stage 1, and report a display log overlap The M3 m1n1 (stage 2) must fit below the display processors' log buffers. Every M3 boot so far placed the panel's buffer 0x120000 bytes above where stage 1 loaded m1n1, and every one of them reported stage 1 v1.6.1-dirty. Nothing guarantees that layout for another stage 1. Check it before the handoff. m3_stage1_problem() reads /chosen/asahi,m1n1-stage1-version and compares it with M3_STAGE1_VERSIONS ("v1.6.1-dirty"); a Mac that reports no version fails too. m3_plan() handles it like the system-firmware stub check: - with --m3-handoff, or on a Mac that already has the handoff, it stops with nothing installed; - a plain run warns, installs the kernel only and leaves boot.bin as it is. It covers the M3 Pro and the M3 MacBook Air path alike. After a boot, m1n1 aurora8.5-2 adds /chosen/asahi,m1n1-oslog-overlap when one of those buffers overlaps where stage 1 loaded it. Each run on an M3, and --m3-report, now warn when it is present and ask for a report. --m3-report records whether it is there, copies it, and lists the reserved display log nodes. The agent prompt's M3 checks ask for both. M1, M2 and the Neo are unchanged. Installer tests: 149 run, 148 pass, 1 skipped. New: the stage 1 check itself, plans on another or a missing stage 1 (kernel only on a plain run, refused with --m3-handoff), a handoff Mac on another stage 1 (stops before any download), the same on an Air, the overlap warning, and an M1 that ignores both. shellcheck: the same four notes as 11.38. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 84aa4e9185c352839cf5cb60fba7d9e3a209586c) --- tools/aurora-sep/install-aurora-sep.sh | 53 +++++++++++++++++++- tools/aurora-sep/tests/test_m3_air.py | 13 +++++ tools/aurora-sep/tests/test_m3_flow.py | 60 ++++++++++++++++++++++- tools/aurora-sep/tests/test_m3_handoff.py | 46 ++++++++++++++++- 4 files changed, 169 insertions(+), 3 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 01a234eb664e5..a0b5e93f0451f 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -354,6 +354,16 @@ M3_AIR_DRY_RUN_SWITCHES="chosen.asahi,t8122-gpu-diag=1 chosen.asahi,t8122-gpu-ha # reports the stub's iBoot as asahi,iboot2-version. M3_STUB_VERSION=14.8.3 M3_STUB_IBOOT=iBoot-10151.140.19 +# The M3 m1n1 (stage 2) must fit below the display processors' log buffers, +# which every M3 boot so far placed 0x120000 bytes above where stage 1 loaded +# m1n1. That has only been seen with these stage 1 versions, as m1n1 reports +# them in /chosen/asahi,m1n1-stage1-version; the handoff is refused with any +# other, and on a Mac that reports none. +M3_STAGE1_VERSIONS="v1.6.1-dirty" +# m1n1 (aurora8.5-2 and later) adds this node when one of those log buffers +# overlaps where stage 1 loaded it. It keeps the buffer reserved and boots on, +# but it was never checked in that layout. +M3_OSLOG_OVERLAP=chosen/asahi,m1n1-oslog-overlap M1N1_CONF=/etc/m1n1.conf # What update-m1n1 puts at the start of boot.bin. M1N1_BIN=/usr/lib/asahi-boot/m1n1.bin @@ -524,6 +534,30 @@ m3_stub_problem() { return 0 } +# Why this Mac's m1n1 stage 1 isn't one the M3 m1n1 is checked with (see +# M3_STAGE1_VERSIONS); nothing when it is. +m3_stage1_problem() { + local stage1 + stage1=$({ tr -d '\0' <"$DT/chosen/asahi,m1n1-stage1-version"; } 2>/dev/null) || stage1="" + if [[ -z $stage1 ]]; then + echo "its m1n1 reports no stage 1 version" + elif [[ " $M3_STAGE1_VERSIONS " != *" $stage1 "* ]]; then + echo "its m1n1 stage 1 is $stage1" + fi + return 0 +} + +# After a boot: warn when this boot's m1n1 found a display log buffer over +# the place stage 1 loaded it (M3_OSLOG_OVERLAP). Only an M3 m1n1 sets it. +m3_oslog_overlap_check() { + [[ -e $DT/$M3_OSLOG_OVERLAP ]] || return 0 + warn "this boot's m1n1 reports that a display log buffer overlaps where its stage 1 loaded + it ($DT/$M3_OSLOG_OVERLAP). This m1n1 was only checked with those buffers clear of it. + Please report it at https://github.com/iconidentify/aurora-linux/issues with the file that + --m3-report writes. The boot loader this Mac had before the handoff is kept on the EFI + partition as m1n1/boot.bin.before-." +} + # True when a downloaded m1n1-aurora package knows every switch this Mac's # handoff needs (m3_switches): asahi,t6030-gpu for chosen.asahi,t6030-gpu=1. m1n1_pkg_has_handoff() { @@ -677,6 +711,7 @@ m3_plan() { fi board=$(this_board) if is_m3_air; then air=1; fi + m3_oslog_overlap_check # An update never takes the handoff away again: once a Mac has it (listed, or # tried with --m3-handoff), a plain run keeps it, and its checks still apply. if ((M3_TRY == 0)) && [[ $(m3_recorded_mode) == handoff ]]; then @@ -733,6 +768,14 @@ m3_plan() { boot.bin stays as it is." return 0 fi + problem=$(m3_stage1_problem) + if [[ -n $problem ]]; then + ((M3_TRY == 0)) || die "--m3-handoff: this release's M3 m1n1 is only checked with m1n1 stage 1 + $M3_STAGE1_VERSIONS, and this Mac differs: $problem. Nothing was installed." + warn "the $(m3_handoff_name) is only checked with m1n1 stage 1 $M3_STAGE1_VERSIONS, and + this Mac differs: $problem. Installing the kernel only; boot.bin stays as it is." + return 0 + fi if update_m1n1_frozen_by_others; then die "$UPDATE_M1N1_CONF sets M1N1_UPDATE_DISABLED, and not from this script or the M3 bring-up's install-m3gpu.sh. Switching the $(m3_handoff_name) on needs m1n1's boot.bin @@ -1651,11 +1694,13 @@ m3_report() { pacman -Q linux-aurora m1n1-aurora m1n1 2>/dev/null || true printf 'm3-mode: '; cat "$STATE/m3-mode" 2>/dev/null || echo - echo "m1n1.conf switches:"; grep '^chosen\.' "$M1N1_CONF" 2>/dev/null || echo - + printf 'm1n1-oslog-overlap: '; if [[ -e $DT/$M3_OSLOG_OVERLAP ]]; then echo present; else echo absent; fi + echo "reserved display logs:"; ls -d "$DT"/reserved-memory/dcp-oslog@* 2>/dev/null || echo - } >"$dir/system.txt" { dmesg 2>/dev/null || $sudo dmesg; } | grep -iE "$M3_REPORT_DMESG" | grep -viE 'serialnumber|serial number' | sed -E 's/([0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}/xx:xx:xx:xx:xx:xx/g' >"$dir/dmesg-m3.txt" || true mkdir -p "$dir/chosen" - for f in "$DT"/chosen/asahi,t8122-* "$DT"/chosen/asahi,t6030-*; do + for f in "$DT"/chosen/asahi,t8122-* "$DT"/chosen/asahi,t6030-* "$DT/$M3_OSLOG_OVERLAP"; do [[ -e $f ]] && cp -r "$f" "$dir/chosen/" done { @@ -1670,6 +1715,7 @@ m3_report() { for f in /sys/class/drm/card*-*/status; do [[ -e $f ]] && echo "$f: $(cat "$f")"; done } >"$dir/usb-display.txt" 2>&1 tar czf "$out" -C "$dir" . && rm -rf "$dir" + m3_oslog_overlap_check say "Report written to $out Attach it to your issue at https://github.com/iconidentify/aurora-linux/issues, together with the serial log if you recorded one. It has no full kernel log, no USB serial numbers @@ -2070,6 +2116,11 @@ fingerprint. sudo dmesg | grep -E 't6030-display|\[drm\] Initialized|GPU firmware|aop.*crash|apple_sep' | head -12 nproc ls /proc/device-tree/soc/usb4-pcie-tunnel-0/pcie@730000000/pci@0,0/apple,tunable 2>&1 + ls /proc/device-tree/chosen/asahi,m1n1-oslog-overlap 2>&1 + ls -d /proc/device-tree/reserved-memory/dcp-oslog@* 2>&1 + With the handoff, the first of those two must say "No such file or + directory": if the node exists, report that before anything else. The + second lists three dcp-oslog nodes on an M3 Pro and one on an Air. In the report header, set **M3 path:** to kernel, handoff, or handoff with --m3-handoff. diff --git a/tools/aurora-sep/tests/test_m3_air.py b/tools/aurora-sep/tests/test_m3_air.py index 87e29b3658771..14686f8fb2ac9 100644 --- a/tools/aurora-sep/tests/test_m3_air.py +++ b/tools/aurora-sep/tests/test_m3_air.py @@ -586,6 +586,19 @@ def test_test_build_display_handoff_does_not_move_silently(self): # The boot loader the Mac had before its first test stays the kept copy. self.assertEqual(self.kept_copy().read_bytes(), b"M1N1:original\n") + def test_display_handoff_needs_the_checked_stage1(self): + # Scott's Air on another m1n1 stage 1: --m3-handoff stops with nothing + # downloaded, and a plain run stays kernel-only. + self.mac("j613", stage1="v1.7.0") + before = self.boot.read_bytes() + proc = self.air_install(try_=1, display=1, check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("stage 1 is v1.7.0", proc.stderr) + self.assertIn("Nothing was installed", proc.stderr) + self.assertEqual(self.boot.read_bytes(), before) + self.assertEqual(self.downloaded(), []) + self.assertEqual(self.plan(display=1), "kernel") + def test_install_syncs_after_the_rebuild(self): src = SRC[SRC.index(" m3_verify_bootbin\n"):] self.assertTrue(re.match(r" m3_verify_bootbin\n(\s*#.*\n)*\s*sync\n", src), src[:200]) diff --git a/tools/aurora-sep/tests/test_m3_flow.py b/tools/aurora-sep/tests/test_m3_flow.py index 1e664fe92be90..2f5ecf8a3a4e2 100644 --- a/tools/aurora-sep/tests/test_m3_flow.py +++ b/tools/aurora-sep/tests/test_m3_flow.py @@ -164,9 +164,14 @@ def fixture(self, name, data, strings=None): self.shas[name] = hashlib.sha256(path.read_bytes()).hexdigest() def mac(self, board, kernel="linux-asahi", m1n1="m1n1-stock", bootbin=b"M1N1:original\n", - stub="14.8.3", iboot="iBoot-10151.140.19.700.2"): + stub="14.8.3", iboot="iBoot-10151.140.19.700.2", stage1="v1.6.1-dirty"): (self.tmp / "dt/compatible").write_bytes(b"".join(c.encode() + b"\0" for c in BOARDS[board])) (self.tmp / "dt/chosen/asahi,iboot2-version").write_bytes(iboot.encode() + b"\0") + stage1_node = self.tmp / "dt/chosen/asahi,m1n1-stage1-version" + if stage1 is None: + stage1_node.unlink(missing_ok=True) + else: + stage1_node.write_bytes(stage1.encode() + b"\0") (self.tmp / "esp/asahi/stub_info.json").write_text( '{"system_version": {"ProductVersion": "%s"}}' % stub) (self.fake / "installed").write_text(kernel + "\n") @@ -316,6 +321,59 @@ def test_m3_is_kernel_only(self): def test_listed_m3_pro_on_another_stub_is_kernel_only(self): self.assert_kernel_only("j516s", stub="15.6") + # Stage 1 (M3_STAGE1_VERSIONS) and the log-buffer overlap marker + + def test_listed_m3_pro_on_another_stage1_is_kernel_only(self): + self.assert_kernel_only("j516s", stage1="v1.7.0") + + def test_listed_m3_pro_without_a_stage1_version_is_kernel_only(self): + self.assert_kernel_only("j516s", stage1=None) + + def test_handoff_mac_on_another_stage1_stops_with_nothing_installed(self): + # A Mac with the handoff from an earlier install (M3_TRY=1 from its + # record) keeps its boot.bin as it is: nothing is downloaded or rebuilt. + self.mac("j516s") + self.install() + self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff t6030") + self.mac("j516s", kernel="linux-aurora", bootbin=self.boot.read_bytes(), stage1="v1.7.0") + before = self.boot.read_bytes() + before_log = self.log() + proc = self.install(check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("stage 1 is v1.7.0", proc.stderr) + self.assertIn("Nothing was installed", proc.stderr) + self.assertEqual(self.boot.read_bytes(), before) + self.assertNotIn("curl ", self.log()[len(before_log):]) + + def test_trial_on_another_stage1_is_refused(self): + self.mac("j514s", stage1="v1.5.2") + before = self.boot.read_bytes() + proc = self.install(try_=1, check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("stage 1 is v1.5.2", proc.stderr) + self.assertIn("Nothing was installed", proc.stderr) + self.assertEqual(self.boot.read_bytes(), before) + self.assertNotIn("pacman -U", self.log()) + + def test_oslog_overlap_is_reported_after_a_boot(self): + self.mac("j516s") + self.install() + (self.tmp / "dt/chosen/asahi,m1n1-oslog-overlap").write_bytes(b"\0" * 16) + proc = self.install() + self.assertIn("display log buffer overlaps", proc.stderr) + self.assertIn("--m3-report", proc.stderr) + (self.tmp / "dt/chosen/asahi,m1n1-oslog-overlap").unlink() + self.assertNotIn("overlaps", self.install().stderr) + + def test_m1_ignores_the_m3_stage1_and_overlap_checks(self): + # M1 and M2 are unchanged: no stage 1 gate, no overlap warning. + self.mac("j314s", stage1=None) + (self.tmp / "dt/chosen/asahi,m1n1-oslog-overlap").write_bytes(b"\0" * 16) + proc = self.install() + self.assertNotIn("stage 1", proc.stderr + proc.stdout) + self.assertNotIn("overlaps", proc.stderr + proc.stdout) + self.assertTrue(self.boot.read_bytes().startswith(b"M1N1:m1n1-aurora-1.6.1.aurora3-1\n")) + def test_trial_refused_on_an_m3(self): # A plain M3 that isn't an Air (the Air's opt-in is in test_m3_air.py). self.mac("j504") diff --git a/tools/aurora-sep/tests/test_m3_handoff.py b/tools/aurora-sep/tests/test_m3_handoff.py index 873c9236f3c7f..9ec677d61348f 100644 --- a/tools/aurora-sep/tests/test_m3_handoff.py +++ b/tools/aurora-sep/tests/test_m3_handoff.py @@ -66,12 +66,17 @@ def setUp(self): (self.esp / "m1n1" / "boot.bin").write_bytes(b"m1n1-old\x00\x01" * 8) (self.tmp / "m1n1.bin").write_bytes(b"m1n1") - def mac(self, board, iboot=GOOD_IBOOT, stub="14.8.3"): + def mac(self, board, iboot=GOOD_IBOOT, stub="14.8.3", stage1="v1.6.1-dirty"): (self.dt / "compatible").write_bytes( b"".join(c.encode() + b"\0" for c in BOARDS[board]) ) if iboot is not None: (self.dt / "chosen" / "asahi,iboot2-version").write_bytes(iboot.encode() + b"\0") + stage1_node = self.dt / "chosen" / "asahi,m1n1-stage1-version" + if stage1 is None: + stage1_node.unlink(missing_ok=True) + else: + stage1_node.write_bytes(stage1.encode() + b"\0") info = self.esp / "asahi" / "stub_info.json" if stub is None: info.unlink(missing_ok=True) @@ -150,6 +155,45 @@ def test_stub_iboot_unreported(self): self.mac("j516s", iboot=None, stub=None) self.assertIn("not reported", self.run_sh("m3_stub_problem").stdout) + # Stage 1 + + def test_stage1_versions_shipped(self): + # A release decision: widen it only after a captured boot on the + # m3pro and on an Air with the new stage 1. + self.assertEqual(re.search(r'^M3_STAGE1_VERSIONS="([^"]*)"$', _SRC, re.M).group(1), "v1.6.1-dirty") + + def test_stage1_ok(self): + self.mac("j516s") + self.assertEqual(self.run_sh("m3_stage1_problem").stdout, "") + + def test_stage1_other_or_missing(self): + for stage1, why in [("v1.6.1", "stage 1 is v1.6.1"), ("v1.7.0-dirty", "stage 1 is v1.7.0-dirty"), + ("", "no stage 1 version"), (None, "no stage 1 version")]: + with self.subTest(stage1=stage1): + self.mac("j516s", stage1=stage1) + self.assertIn(why, self.run_sh("m3_stage1_problem").stdout) + + def test_plan_on_another_stage1(self): + self.mac("j516s", stage1="v1.7.0") + self.assertEqual(self.plan(), "kernel") + self.assertIn("only checked with m1n1 stage 1", self.run_sh("m3_plan").stderr) + proc = self.run_sh("M3_TRY=1\nm3_plan", check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("stage 1 is v1.7.0", proc.stderr) + self.assertIn("Nothing was installed", proc.stderr) + + def test_overlap_marker(self): + self.mac("j516s") + self.assertEqual(self.run_sh("m3_oslog_overlap_check").stderr, "") + (self.dt / "chosen" / "asahi,m1n1-oslog-overlap").write_bytes(b"\0" * 16) + err = self.run_sh("m3_oslog_overlap_check").stderr + self.assertIn("overlaps where its stage 1 loaded", err) + self.assertIn("boot.bin.before-", err) + # m3_plan warns on every M3, and installs as before. + proc = self.run_sh("m3_plan\necho $M3_MODE") + self.assertIn("overlaps", proc.stderr) + self.assertEqual(proc.stdout.split()[-1], "handoff") + # Plan def plan(self, try_=0): From bc4279ed2ec0ae1171113ed8c63eb36e3851e2e6 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:43:27 -0700 Subject: [PATCH 1758/2680] tools/aurora-sep: one m1n1 package for every Mac Every Mac that gets an m1n1 from this script now gets the same one, M1N1_PACKAGE (m1n1-aurora aurora12): M1 and M2 move to it from aurora3, and an M3 on the handoff path takes it in place of M3_M1N1_PACKAGE and M3_AIR_M1N1_PACKAGE, which are gone. Macs differ only in the switches /etc/m1n1.conf arms. The package's checksum is a placeholder until the aurora12 build exists; like the kernel's, it stops the install before anything changes. The MacBook Neo keeps its own m1n1 unless NEO_AURORA_M1N1 is 1. With it set, the Neo gets M1N1_PACKAGE, update-m1n1 builds its boot.bin from that m1n1 and the Neo's own U-Boot (only its M1N1= line is dropped), and --uninstall puts back the stock m1n1 package and its saved configuration. With it unset the Neo runs exactly as in 11.38. Every Mac whose boot.bin this run rebuilds with M1N1_PACKAGE first keeps the boot.bin it booted with as m1n1/boot.bin.before- on the EFI partition and prints the steps to put it back from macOS, as the M3s and the three boards in UNPROVEN_SEP_BOARDS already did. The stage 1 check stays M3-only: the log buffer it guards against is placed by the M3's boot firmware, and M1 and M2 have booted aurora3, whose file part reached about 0x110000, from every stage 1 in the field. Tests: the fake pacman installs each package's own m1n1.bin, and the fake update-m1n1 honours M1N1=; one package for M1, M2 and the M3s; the Neo with and without the switch. The Air cases for a release without an Air m1n1 are gone. Against 11.38's script on the same fake Macs, M1 and M2 differ only by the kept copy and its restore steps, and the Neo not at all. shellcheck: the same notes as 11.38. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 3801d84c5ab179adc506bbb258fe3fa886f1a704) --- tools/aurora-sep/install-aurora-sep.sh | 181 ++++++++++++---------- tools/aurora-sep/tests/test_m3_air.py | 99 ++++-------- tools/aurora-sep/tests/test_m3_flow.py | 131 +++++++++++++--- tools/aurora-sep/tests/test_m3_handoff.py | 12 +- 4 files changed, 243 insertions(+), 180 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index a0b5e93f0451f..cf838455eb575 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -103,13 +103,18 @@ # uses, leaving interrupts for a dock's other devices. An M3 Air stays kernel-only. # 12.0: one release and one kernel for every Mac this script supports; the # 11.1xx test builds end here. On the 16" M3 Pro (J516S) the kernel drives -# external displays on the USB-C and HDMI ports, and its m1n1 moves to -# aurora8.5, which also hands over the second external display processor. The -# M3 Type-C PHY changes behind this apply to every M3. Other M3 Pros keep the -# handoff opt-in (--m3-handoff). An M3 MacBook Air stays kernel-only unless its -# owner asks with --m3-handoff for m1n1's display handoff with GPU diagnostics -# (the same aurora8.5; the desktop stays on the boot framebuffer). An Air that -# has an earlier test build's boot loader must ask again with --m3-handoff. +# external displays on the USB-C and HDMI ports, and its m1n1 also hands over +# the second external display processor. The M3 Type-C PHY changes behind this +# apply to every M3. Other M3 Pros keep the handoff opt-in (--m3-handoff). An +# M3 MacBook Air stays kernel-only unless its owner asks with --m3-handoff for +# m1n1's display handoff with GPU diagnostics (the desktop stays on the boot +# framebuffer). An Air that has an earlier test build's boot loader must ask +# again with --m3-handoff. +# Every Mac that gets an m1n1 from this script now gets the same one, +# m1n1-aurora aurora12: M1 and M2 move to it from aurora3, and the M3s on the +# handoff path from aurora7 and 8.x. Only the switches in /etc/m1n1.conf +# differ between Macs. Each Mac's boot.bin is kept on the EFI partition before +# it is rebuilt, with the steps to put it back. # The kernel also describes the Air's internal display and GPU configuration, # both inert for now, and the M2 Max Neural Engine cleans up after a failed # probe (iconidentify/aurora-linux#37, Joshua Warren). M1, M2 and the Neo @@ -124,13 +129,17 @@ # this Mac's own sensor calibration. After the reboot, run # aurora-touchid-setup to enrol a finger and use it for sudo and the lock screen. # -# m1n1: m1n1-aurora builds AsahiLinux/m1n1 main (3e354a24), which knows the -# macOS 26.5 to 27.0 firmware the MacBook Neo ships with, plus three patches: -# the SEP warm-registration guard and preboot-UUID forwarding from -# aurora-silicon/m1n1, and the usb4-N-pcie-adapter alias fallback -# (aurora-silicon/m1n1#4). +# m1n1: one m1n1-aurora (M1N1_PACKAGE) for every Mac. It builds +# AsahiLinux/m1n1 main (3e354a24), which knows the macOS 26.5 to 27.0 firmware +# the MacBook Neo ships with, plus Omarchy's patches: the SEP +# warm-registration guard and preboot-UUID forwarding from aurora-silicon/m1n1, +# the usb4-N-pcie-adapter alias fallback (aurora-silicon/m1n1#4), and the M3 +# display and GPU handoff, which stays off unless /etc/m1n1.conf arms it. +# Before boot.bin is rebuilt with it, the boot.bin the Mac booted with is kept +# on the EFI partition (keep_bootbin_on_esp). A MacBook Neo keeps its own m1n1 +# unless NEO_AURORA_M1N1 is 1. # M3: experimental. linux-aurora goes on. On an M3 Pro model in -# M3_HANDOFF_BOARDS (or with --m3-handoff), M3_M1N1_PACKAGE replaces m1n1 and +# M3_HANDOFF_BOARDS (or with --m3-handoff), m1n1-aurora replaces m1n1 and # three chosen.asahi,t6030-* lines in /etc/m1n1.conf switch its display and # GPU handoff on. Every other M3 keeps m1n1's boot.bin exactly as it is: no # m1n1-aurora, no update-m1n1 run, and a freeze on update-m1n1 unless one is @@ -161,18 +170,17 @@ PACKAGES=( "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst PENDING-12.0-LAB-BUILD" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20261003-1-any.pkg.tar.zst 29b0360fac8c257d754e64bd1b9c33c487eb2595dd3c31e9138d7a476afa3d64" - "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst bc3451aaa88bc3f4912bc3613f9569aa8f3e05f376fa851fa837b5e2080e8c2f" ) -# Only for an M3 on the handoff path (see m3_plan), in place of the m1n1-aurora -# above: the same m1n1 plus the T6030 display and GPU handoff. aurora8.5 also -# hands over the second external display processor. -M3_M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora8.5-2-aarch64.pkg.tar.zst 7bbf89455b9a96487ee05bfef18ee3e5c8ec57c1af3f97fc15548774d5e92a44" -# Only for an M3 MacBook Air on the handoff path, in place of the m1n1-aurora -# above: m1n1-aurora with the T8122 switches, as "file sha256". From aurora8.5 -# one M3 m1n1 carries both, so this names the same package as M3_M1N1_PACKAGE. -# Left empty, an Air stays kernel-only and --m3-handoff on an Air stops with -# nothing installed. -M3_AIR_M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora8.5-2-aarch64.pkg.tar.zst 7bbf89455b9a96487ee05bfef18ee3e5c8ec57c1af3f97fc15548774d5e92a44" +# The one m1n1 for every Mac, as "file sha256": M1 and M2, an M3 on the handoff +# path (see m3_plan), and the MacBook Neo once NEO_AURORA_M1N1 is 1. Macs differ +# only in the switches /etc/m1n1.conf arms (m3_switches), never in the binary. +M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora12-1-aarch64.pkg.tar.zst PENDING-AURORA12-BUILD" +# 0: a MacBook Neo keeps its own m1n1 (its M1N1= and U_BOOT= in +# /etc/default/update-m1n1), as before 12.0. 1: it gets M1N1_PACKAGE like every +# other Mac, and update-m1n1 builds its boot.bin from that m1n1 and the Neo's +# own U-Boot. Set it to 1 only once M1N1_PACKAGE carries every patch of the +# Neo's own m1n1 (aurora-silicon/m1n1, J700) and has booted on a Neo. +NEO_AURORA_M1N1=0 PINNED="linux-aurora linux-aurora-headers libfprint m1n1-aurora" PIN_BEGIN="# >>> aurora-sep pin (remove with: install-aurora-sep.sh --uninstall)" PIN_END="# <<< aurora-sep pin" @@ -302,16 +310,16 @@ is_m3_air() { # kernel's device trees arrive. --uninstall lifts it. # handoff An M3 Pro model in M3_HANDOFF_BOARDS on the tested stub, or any M3 # Pro whose owner asked for it with --m3-handoff. It gets -# M3_M1N1_PACKAGE, whose T6030 display and GPU handoff (from +# M1N1_PACKAGE, whose T6030 display and GPU handoff (from # iconidentify/m3-m1n1) stays off unless boot.bin ends with the # three M3_SWITCHES lines; update-m1n1 copies chosen.* lines from # /etc/m1n1.conf into every rebuild, so they survive updates. -# An M3 MacBook Air takes the same path with its own m1n1 -# (M3_AIR_M1N1_PACKAGE) and switches (m3_switches): the display -# handoff with GPU diagnostics, and only with --m3-handoff until -# its board is in M3_HANDOFF_BOARDS. -# M1, M2 and the Neo keep the m1n1-aurora in PACKAGES: only an M3 on the -# handoff path gets the newer m1n1. +# An M3 MacBook Air takes the same path with the same m1n1 and its +# own switches (m3_switches): the display handoff with GPU +# diagnostics, and only with --m3-handoff until its board is in +# M3_HANDOFF_BOARDS. +# M1 and M2 get the same M1N1_PACKAGE with no switches; the Neo keeps its own +# m1n1 unless NEO_AURORA_M1N1 is 1. UPDATE_M1N1_CONF=/etc/default/update-m1n1 M3_FREEZE_BEGIN="# >>> aurora-sep: keep this M3's boot.bin as it is (remove with: install-aurora-sep.sh --uninstall)" M3_FREEZE_END="# <<< aurora-sep: keep this M3's boot.bin as it is" @@ -359,6 +367,12 @@ M3_STUB_IBOOT=iBoot-10151.140.19 # m1n1. That has only been seen with these stage 1 versions, as m1n1 reports # them in /chosen/asahi,m1n1-stage1-version; the handoff is refused with any # other, and on a Mac that reports none. +# The check is for M3s only. That log buffer placement is the M3's; on M1 and +# M2 nothing has been seen at that offset, and they have booted aurora3, whose +# file part reached about 0x110000, from every stage 1 in the field since 11.x. +# M1N1_PACKAGE is linked to end its file part below 0x120000 (0xe0000 for the +# aurora8.5 line it comes from), so a stage 1 allowlist there would only +# refuse Macs that boot today. M3_STAGE1_VERSIONS="v1.6.1-dirty" # m1n1 (aurora8.5-2 and later) adds this node when one of those log buffers # overlaps where stage 1 loaded it. It keeps the buffer reserved and boots on, @@ -489,15 +503,22 @@ m3_switches() { fi } -# This Mac's m1n1 package on the handoff path, as "file sha256". -m3_m1n1_package() { - if is_m3_air; then echo "$M3_AIR_M1N1_PACKAGE"; else echo "$M3_M1N1_PACKAGE"; fi +# Whether this run puts M1N1_PACKAGE on this Mac: M1 and M2, an M3 on the +# handoff path, and a MacBook Neo only with NEO_AURORA_M1N1=1. Needs m3_plan +# first. +m1n1_for_this_mac() { + if is_neo; then + [[ $NEO_AURORA_M1N1 == 1 ]] + elif [[ $M3_MODE != none ]]; then + [[ $M3_MODE == handoff ]] + fi } -# Whether this release carries an m1n1 for the M3 Air: a whole "file sha256" -# entry, not the empty placeholder. -m3_air_package_ok() { - [[ $M3_AIR_M1N1_PACKAGE =~ ^m1n1-aurora-[^[:space:]]+-aarch64\.pkg\.tar\.zst\ [0-9a-f]{64}$ ]] +# M1N1_PACKAGE's version, as pacman prints it: 1.6.1.aurora12-1. +m1n1_version() { + local file=${M1N1_PACKAGE%% *} + file=${file#m1n1-aurora-} + echo "${file%-aarch64.pkg.tar.zst}" } # What the handoff does on this Mac, for messages. @@ -731,16 +752,7 @@ m3_plan() { M3_MODE=kernel if ((air)); then # Never by default: a plain run gives an Air the handoff only once its - # board is in M3_HANDOFF_BOARDS, and never without this release's m1n1. - if ! m3_air_package_ok; then - ((M3_TRY == 0)) || die "--m3-handoff: this release has no m1n1 with the M3 MacBook Air GPU - handoff yet, so it can't switch it on. Nothing was installed. (--uninstall puts the stock - m1n1 back on an Air that has the handoff from an earlier test build.)" - say "M3 MacBook Air ($board): installing the kernel only. This release has no m1n1 with the - GPU handoff for the Air, so boot.bin stays as it is and the display runs on the boot - framebuffer." - return 0 - fi + # board is in M3_HANDOFF_BOARDS. if ! is_m3_handoff_board && ((M3_TRY == 0)); then say "M3 MacBook Air ($board): installing the kernel only, and boot.bin stays as it is. m1n1's $(m3_handoff_name) for the Air is being tested and is not on by default. To help test it @@ -864,15 +876,28 @@ this_board() { tr '\0' '\n' <"$DT/compatible" 2>/dev/null | sed -n '1s/^apple,//p' } -# Before anything rebuilds boot.bin on such a board, keep the one this boot -# came up on next to it on the EFI partition, where macOS can reach it, and -# say how to put it back. A later run keeps the first copy. -unproven_board_backup() { +# Before anything rebuilds boot.bin with this release's m1n1 (every Mac that +# m1n1_for_this_mac names), keep the one this boot came up on next to it on the +# EFI partition, where macOS can reach it, and say how to put it back. A later +# run of the same release keeps the first copy. +bootbin_backup() { local board + m1n1_for_this_mac || return 0 board=$(this_board) - [[ -n $board && " $UNPROVEN_SEP_BOARDS " == *" $board "* ]] || return 0 - keep_bootbin_on_esp "Touch ID on this Mac model ($board) is new in $VERSION, and nobody has - booted it on this model yet." + if [[ $M3_MODE == handoff ]] && is_m3_air; then + keep_bootbin_on_esp "This replaces the boot loader of this M3 MacBook Air ($board) + with m1n1-aurora and its $(m3_handoff_name)." + elif [[ $M3_MODE == handoff ]]; then + keep_bootbin_on_esp "This replaces the boot loader of this M3 Pro ($board) with + m1n1-aurora and its display and GPU handoff." + elif [[ -n $board && " $UNPROVEN_SEP_BOARDS " == *" $board "* ]]; then + keep_bootbin_on_esp "Touch ID on this Mac model ($board) is new in $VERSION, and nobody has + booted it on this model yet. This also replaces its boot loader with m1n1-aurora + $(m1n1_version), the one m1n1 every Mac gets from $VERSION on." + else + keep_bootbin_on_esp "This replaces this Mac's boot loader with m1n1-aurora $(m1n1_version), + the one m1n1 every Mac gets from $VERSION on." + fi } # Replace a file on the EFI partition without ever leaving a partial one in @@ -1042,14 +1067,21 @@ EOF # Point it at whatever directory linux-aurora owns, now and after its updates. m1n1_update() { local target conf=$UPDATE_M1N1_CONF tmp + local -a drop=(-e '/^# aurora-sep: build m1n1/,/^DTBS=/d' -e '/^DTBS=/d') [[ -f $conf && ! -f $STATE/update-m1n1.default.saved ]] && $sudo cp "$conf" "$STATE/update-m1n1.default.saved" # Replace only the DTBS setting and keep the rest of the file: a MacBook # Neo's M1N1= and U_BOOT= point at its own J700 builds, and dropping them - # would rebuild boot.bin from an m1n1 that cannot boot it. + # would rebuild boot.bin from an m1n1 that cannot boot it. A Neo on + # M1N1_PACKAGE (NEO_AURORA_M1N1=1) drops only its M1N1=, so update-m1n1 + # takes the packaged m1n1 with the Neo's own U-Boot. --uninstall puts the + # saved file back. + if is_neo && [[ $NEO_AURORA_M1N1 == 1 ]]; then + drop+=(-e '/^[[:space:]]*\(export[[:space:]]\{1,\}\)\{0,1\}M1N1=/d') + fi tmp=$(mktemp) if [[ -f $conf ]]; then - $sudo sed -e '/^# aurora-sep: build m1n1/,/^DTBS=/d' -e '/^DTBS=/d' "$conf" >"$tmp" + $sudo sed "${drop[@]}" "$conf" >"$tmp" fi cat >>"$tmp" <<'EOF' # aurora-sep: build m1n1's stage 2 from the device trees the installed @@ -1304,17 +1336,11 @@ snapshot_boot_state() { return 0 } -# The packages this Mac gets, one "file sha256" per line: PACKAGES, except -# that a Neo keeps its own m1n1, and an M3 gets its handoff m1n1 -# (m3_m1n1_package) on the handoff path and no m1n1 otherwise. Needs m3_plan -# first. +# The packages this Mac gets, one "file sha256" per line: PACKAGES, and +# M1N1_PACKAGE where m1n1_for_this_mac says so. Needs m3_plan first. packages_for_this_mac() { - local entry - for entry in "${PACKAGES[@]}"; do - if [[ $entry == m1n1-aurora-* ]] && { is_neo || [[ $M3_MODE != none ]]; }; then continue; fi - echo "$entry" - done - if [[ $M3_MODE == handoff ]]; then m3_m1n1_package; fi + printf '%s\n' "${PACKAGES[@]}" + if m1n1_for_this_mac; then echo "$M1N1_PACKAGE"; fi return 0 } @@ -1331,7 +1357,7 @@ install_all() { m3_plan work=$(mktemp -d) trap 'rm -rf "${work:-}"' EXIT - if is_neo; then say "Keeping this MacBook Neo's own m1n1 (m1n1-aurora has no T8140 support)"; fi + if is_neo && ! m1n1_for_this_mac; then say "Keeping this MacBook Neo's own m1n1 (m1n1-aurora has no T8140 support)"; fi if [[ $M3_MODE == kernel ]]; then say "Keeping this M3's own m1n1 and boot.bin"; fi mapfile -t entries < <(packages_for_this_mac) for entry in "${entries[@]}"; do @@ -1345,22 +1371,14 @@ install_all() { [[ $(sha256sum "$work/$file" | cut -d' ' -f1) == "$sha" ]] || die "$file does not match its published checksum" done if [[ $M3_MODE == handoff ]]; then - entry=$(m3_m1n1_package) - m1n1_pkg_has_handoff "$work/${entry%% *}" || - die "this release's M3 m1n1 has no $(m3_handoff_name), so it can't switch it + m1n1_pkg_has_handoff "$work/${M1N1_PACKAGE%% *}" || + die "this release's m1n1 has no $(m3_handoff_name), so it can't switch it on. Nothing was installed. Please report it at https://github.com/iconidentify/aurora-linux/issues" fi snapshot "aurora-sep $VERSION" # Before pacman's update-m1n1 hook rebuilds boot.bin below. - is_neo || unproven_board_backup - if [[ $M3_MODE == handoff ]] && is_m3_air; then - keep_bootbin_on_esp "This replaces the boot loader of this M3 MacBook Air ($(this_board)) - with m1n1-aurora and its $(m3_handoff_name)." - elif [[ $M3_MODE == handoff ]]; then - keep_bootbin_on_esp "This replaces the boot loader of this M3 Pro ($(this_board)) with - m1n1-aurora and its display and GPU handoff." - fi + bootbin_backup $sudo install -d "$STATE" snapshot_boot_state if [[ ! -f $STATE/previous && $chain == grub ]]; then @@ -1513,7 +1531,10 @@ uninstall_all() { remove_pin snapshot "removing aurora-sep" local m1n1=m1n1 - if is_neo; then + # A Neo keeps its own m1n1, unless it got m1n1-aurora (NEO_AURORA_M1N1): + # then the stock m1n1 package goes back, and its own M1N1= with the saved + # update-m1n1 configuration below. + if is_neo && { [[ $NEO_AURORA_M1N1 != 1 ]] || ! pacman -Q m1n1-aurora >/dev/null 2>&1; }; then m1n1= say "Reinstalling $previous and the stock libfprint; this MacBook Neo keeps its own m1n1" elif [[ $m3_mode == kernel ]]; then diff --git a/tools/aurora-sep/tests/test_m3_air.py b/tools/aurora-sep/tests/test_m3_air.py index 14686f8fb2ac9..0d77af50aa0e7 100644 --- a/tools/aurora-sep/tests/test_m3_air.py +++ b/tools/aurora-sep/tests/test_m3_air.py @@ -1,11 +1,11 @@ """The installer's M3 MacBook Air (T8122) path, against the fake Mac of test_m3_flow. An Air stays kernel-only unless its owner asks for m1n1's GPU handoff with ---m3-handoff, and then only when the release carries M3_AIR_M1N1_PACKAGE. -The handoff writes chosen.asahi,t8122-gpu=1 (and chosen.asahi,t8122-dcp=1 -only with M3_AIR_DCP=1) to /etc/m1n1.conf. Every case sets -M3_AIR_M1N1_PACKAGE and M3_AIR_DRY_RUN itself, so the tests don't depend on -what the installer ships with. M3AirDryRunTest covers the dry run, which +--m3-handoff. It then gets the one m1n1 every Mac gets (M1N1_PACKAGE), and +the handoff writes chosen.asahi,t8122-gpu=1 (and chosen.asahi,t8122-dcp=1 +only with M3_AIR_DCP=1) to /etc/m1n1.conf. Every case sets the m1n1 package +and M3_AIR_DRY_RUN itself, so the tests don't depend on what the installer +ships with. M3AirDryRunTest covers the dry run, which writes M3_AIR_DRY_RUN_SWITCHES instead, and the display handoff (M3_AIR_DISPLAY_HANDOFF=1), which writes the same switches under its own variant name. @@ -42,14 +42,17 @@ def setUp(self): # aurora8: the T6030 handoff plus the T8122 GPU and display switches. self.fixture(AURORA8, None, ["asahi,t6030-gpu", "asahi,t6030-dcp", "asahi,t6030-dcpext", "asahi,t8122-gpu", "asahi,t8122-dcp"]) - self.air_pkg = f"{AURORA8} {self.shas[AURORA8]}" + # The one m1n1 every Mac of this fake release gets. + self.m1n1_pkg = AURORA8 dry_run = 0 def sh(self, body, pkg=None, dcp=0, boards=None, check=True, display=0, variant=None): - pkg = self.air_pkg if pkg is None else pkg - pre = (f'M3_AIR_M1N1_PACKAGE="{pkg}"\nM3_AIR_DCP={dcp}\nM3_AIR_DRY_RUN={self.dry_run}\n' + # pkg: another fixture as this release's one m1n1 package. + pre = (f'M3_AIR_DCP={dcp}\nM3_AIR_DRY_RUN={self.dry_run}\n' f'M3_AIR_DISPLAY_HANDOFF={display}\n') + if pkg is not None: + pre += f'M1N1_PACKAGE="{pkg} {self.shas[pkg]}"\n' if variant is not None: pre += f'M3_AIR_DISPLAY_VARIANT={variant}\n' if boards is not None: @@ -105,21 +108,7 @@ def test_no_air_on_by_default(self): self.assertEqual(sorted(airs), ["j613", "j615"]) self.assertFalse(set(listed) & set(airs), listed) - def test_a_listed_air_needs_its_package(self): - listed = set(shell_value("M3_HANDOFF_BOARDS").split()) - if listed & set(shell_value("M3_AIR_BOARDS").split()): - self.mac("j613") - self.assertEqual(self.run_sh("m3_air_package_ok && echo yes || echo no").stdout.strip(), "yes") - def test_package_placeholder(self): - self.mac("j613") - good = "m1n1-aurora-1.6.1.aurora8-1-aarch64.pkg.tar.zst " + "a" * 64 - for pkg, want in [("", "no"), ("TODO", "no"), ("m1n1-aurora-1.6.1.aurora8-1-aarch64.pkg.tar.zst", "no"), - ("m1n1-aurora-1.6.1.aurora8-1-aarch64.pkg.tar.zst abc", "no"), - ("linux-aurora-x-aarch64.pkg.tar.zst " + "a" * 64, "no"), - (good + " extra", "no"), (good, "yes")]: - with self.subTest(pkg=pkg): - self.assertEqual(self.sh("m3_air_package_ok && echo yes || echo no", pkg=pkg).stdout.strip(), want) # Plan @@ -127,11 +116,9 @@ def test_plan(self): for board, stub, try_, pkg, boards, want in [ ("j613", "14.8.3", 0, None, None, "kernel"), # never by default ("j615", "14.8.3", 0, None, None, "kernel"), - ("j613", "14.8.3", 0, "", None, "kernel"), # no Air m1n1 in this release ("j613", "14.8.3", 1, None, None, "handoff"), # the owner asked ("j615", "14.8.3", 1, None, None, "handoff"), ("j613", "14.8.3", 0, None, "j516s j613", "handoff"), # listed later - ("j613", "14.8.3", 0, "", "j516s j613", "kernel"), # listed, but no m1n1 ("j613", "15.6", 0, None, "j516s j613", "kernel"), # listed, other stub ]: with self.subTest(board=board, stub=stub, try_=try_, pkg=pkg, boards=boards): @@ -141,14 +128,12 @@ def test_plan(self): def test_plan_messages(self): self.mac("j613") self.assertIn("case D", self.sh("m3_plan").stdout) - self.assertIn("has no m1n1 with the", self.sh("m3_plan", pkg="").stdout) err = self.sh("M3_TRY=1\nm3_plan").stderr self.assertIn("in testing", err) self.assertIn("desktop keeps", err) def test_plan_try_refused(self): - for board, stub, pkg, why in [("j613", "14.8.3", "", "no m1n1 with the M3 MacBook Air GPU"), - ("j613", "15.6", None, "stub is 15.6"), + for board, stub, pkg, why in [("j613", "15.6", None, "stub is 15.6"), ("j504", "14.8.3", None, "M3 MacBook Air (j613, j615)")]: with self.subTest(board=board, stub=stub, pkg=pkg): self.mac(board, stub=stub) @@ -217,7 +202,7 @@ def test_verify_bootbin(self): def test_packages(self): for board, try_, pkg, want in [("j613", 0, None, None), ("j613", 1, None, AURORA8), - ("j613", 0, "", None), ("j516s", 0, None, flow.AURORA6)]: + ("j516s", 0, None, AURORA8), ("j314s", 0, None, AURORA8)]: with self.subTest(board=board, try_=try_, pkg=pkg): self.mac(board) out = self.sh(f"M3_TRY={try_}\nm3_plan >/dev/null 2>&1\npackages_for_this_mac", pkg=pkg).stdout @@ -251,11 +236,9 @@ def test_handoff_and_back(self): boot = self.boot.read_bytes() self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora8-1\n"), boot) self.assertTrue(boot.endswith(b"UBOOT" + AIR_GPU), boot) - self.assertNotIn(b"t6030", boot) - self.assertNotIn(b"t8122-dcp", boot) - self.assertEqual(self.downloaded().count(AURORA8), 1) - self.assertNotIn(flow.AURORA3, self.downloaded()) - self.assertNotIn(flow.AURORA6, self.downloaded()) + self.assertNotIn(b"t6030", boot.split(b"UBOOT")[-1]) + self.assertNotIn(b"t8122-dcp", boot.split(b"UBOOT")[-1]) + self.assertEqual([d for d in self.downloaded() if d.startswith("m1n1-")], [AURORA8]) self.assertEqual(self.kept_copy().read_bytes(), b"M1N1:original\n") self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff air-gpu") self.assertNotIn("M1N1_UPDATE_DISABLED", self.update_conf.read_text()) @@ -275,56 +258,37 @@ def test_rerun_keeps_the_handoff(self): self.assertTrue(self.boot.read_bytes().endswith(AIR_GPU)) self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff air-gpu") - def test_rerun_without_an_air_package_stops(self): - # A later build without the Air's m1n1 never drops a tested Air back - # to kernel-only under a boot.bin it can't rebuild. - self.mac("j613") - self.air_install(try_=1) - before = self.boot.read_bytes() - proc = self.air_install(pkg="", check=False) - self.assertNotEqual(proc.returncode, 0) - self.assertIn("Nothing was installed", proc.stderr) - self.assertIn("--uninstall", proc.stderr) - self.assertEqual(self.boot.read_bytes(), before) def test_display_switch(self): self.mac("j613") self.air_install(try_=1, dcp=1) self.assertTrue(self.boot.read_bytes().endswith(b"UBOOT" + AIR_GPU + AIR_DCP)) - def test_refused_without_an_air_package(self): - self.mac("j613") - before = self.boot.read_bytes() - proc = self.air_install(try_=1, pkg="", check=False) - self.assertNotEqual(proc.returncode, 0) - self.assertIn("Nothing was installed", proc.stderr) - self.assertEqual(self.boot.read_bytes(), before) - self.assertNotIn("pacman -U", self.log()) - self.assertFalse(self.kept_copy().exists()) def test_refused_with_an_m1n1_that_lacks_the_switch(self): self.mac("j613") before = self.boot.read_bytes() - proc = self.air_install(try_=1, pkg=f"{flow.AURORA6} {self.shas[flow.AURORA6]}", check=False) + proc = self.air_install(try_=1, pkg=flow.AURORA6, check=False) self.assertNotEqual(proc.returncode, 0) self.assertIn("no M3 Air GPU handoff", proc.stderr) self.assertEqual(self.boot.read_bytes(), before) self.assertNotIn("pacman -U", self.log()) def test_m3_pro_unchanged_by_air_settings(self): + # The same m1n1 as the Air, with the M3 Pro's own switches. self.mac("j516s") out = self.air_install(dcp=1).stdout self.assertIn("built-in display at its native resolution", out) boot = self.boot.read_bytes() - self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora6-1\n"), boot) - self.assertTrue(boot.endswith(flow.SWITCHES), boot) - self.assertNotIn(AURORA8, self.downloaded()) + self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora8-1\n"), boot[:80]) + self.assertTrue(boot.endswith(flow.SWITCHES), boot[-200:]) + self.assertEqual(self.downloaded().count(AURORA8), 1) def test_m1_unchanged_by_air_settings(self): self.mac("j314s") self.air_install(dcp=1) boot = self.boot.read_bytes() - self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora3-1\n"), boot) + self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora8-1\n"), boot[:80]) self.assertNotIn(b"chosen.", boot) self.assertFalse(self.m1n1_conf.exists()) @@ -345,7 +309,6 @@ def setUp(self): names = [l[len("chosen."):].split("=")[0] for l in DRY_RUN] self.fixture(AURORA8, None, ["asahi,t6030-gpu", "asahi,t6030-dcp", "asahi,t6030-dcpext", "asahi,t8122-gpu"] + names) - self.air_pkg = f"{AURORA8} {self.shas[AURORA8]}" # The inherited handoff tests run with M3_AIR_DRY_RUN=0 in M3AirTest; the # ones that only make sense for the handoff's switches are skipped here. @@ -368,9 +331,10 @@ def test_shipped_settings(self): self.assertNotIn("chosen.asahi,t8122-gpu=1", DRY_RUN) self.assertEqual(shell_word("M3_AIR_DRY_RUN"), "1") self.assertEqual(shell_word("M3_AIR_DISPLAY_HANDOFF"), "1") - air_pkg = shell_value("M3_AIR_M1N1_PACKAGE") - self.assertRegex(air_pkg, r"^m1n1-aurora-\S+ [0-9a-f]{64}$") - self.assertEqual(air_pkg, shell_value("M3_M1N1_PACKAGE")) + # One m1n1 for every Mac: no package of the Air's own. + self.assertNotIn("M3_AIR_M1N1_PACKAGE", SRC) + self.assertNotIn("M3_M1N1_PACKAGE", SRC) + self.assertRegex(shell_value("M1N1_PACKAGE"), r"^m1n1-aurora-\S+-aarch64\.pkg\.tar\.zst \S+$") # 11.111-test and 11.111.1-test recorded air-display-handoff. self.assertNotIn(DISPLAY_VARIANT, ("air-display-handoff", "air-dry-run", "air-gpu", "air-gpu-dcp")) self.assertRegex(DISPLAY_VARIANT, r"^air-display-handoff-\S+$") @@ -427,8 +391,8 @@ def test_handoff_and_back(self): boot = self.boot.read_bytes() self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora8-1\n"), boot) self.assertTrue(boot.endswith(b"UBOOT" + DRY_RUN_LINES), boot) - self.assertNotIn(b"t8122-gpu=1", boot) - self.assertNotIn(b"t6030", boot) + self.assertNotIn(b"t8122-gpu=1", boot.split(b"UBOOT")[-1]) + self.assertNotIn(b"t6030", boot.split(b"UBOOT")[-1]) self.assertEqual(self.kept_copy().read_bytes(), b"M1N1:original\n") self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff air-dry-run") self.sh("uninstall_all") @@ -449,15 +413,14 @@ def test_display_switch(self): def test_refused_with_an_m1n1_that_lacks_the_switch(self): self.mac("j613") before = self.boot.read_bytes() - proc = self.air_install(try_=1, pkg=f"{flow.AURORA6} {self.shas[flow.AURORA6]}", check=False) + proc = self.air_install(try_=1, pkg=flow.AURORA6, check=False) self.assertNotEqual(proc.returncode, 0) self.assertIn("no M3 Air GPU and display dry run", proc.stderr) self.assertEqual(self.boot.read_bytes(), before) self.assertNotIn("pacman -U", self.log()) def test_plan_try_refused(self): - for board, stub, pkg, why in [("j613", "14.8.3", "", "no m1n1 with the M3 MacBook Air GPU"), - ("j613", "15.6", None, "stub is 15.6")]: + for board, stub, pkg, why in [("j613", "15.6", None, "stub is 15.6")]: with self.subTest(board=board, stub=stub, pkg=pkg): self.mac(board, stub=stub) proc = self.sh("M3_TRY=1\nm3_plan", pkg=pkg, check=False) @@ -510,7 +473,7 @@ def test_plain_rerun_updates_the_dry_run_m1n1(self): newer = "m1n1-aurora-1.6.1.aurora8.1-1-aarch64.pkg.tar.zst" names = [l[len("chosen."):].split("=")[0] for l in DRY_RUN] self.fixture(newer, None, ["asahi,t8122-gpu"] + names) - proc = self.air_install(pkg=f"{newer} {self.shas[newer]}") + proc = self.air_install(pkg=newer) self.assertIn("keeping it", proc.stdout) boot = self.boot.read_bytes() self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora8.1-1\n"), boot) diff --git a/tools/aurora-sep/tests/test_m3_flow.py b/tools/aurora-sep/tests/test_m3_flow.py index 2f5ecf8a3a4e2..4be32868bf164 100644 --- a/tools/aurora-sep/tests/test_m3_flow.py +++ b/tools/aurora-sep/tests/test_m3_flow.py @@ -2,9 +2,12 @@ The real functions run; pacman, update-m1n1, curl, findmnt and systemctl are small stubs that record what happened. The fake update-m1n1 behaves like the -real one where it matters here: it honours M1N1_UPDATE_DISABLED, builds -boot.bin from the installed m1n1 and device trees, and appends /etc/m1n1.conf's -chosen.* lines. pacman -U and the m1n1/kernel installs run it, as the hooks do. +real one where it matters here: it honours M1N1_UPDATE_DISABLED and M1N1=, +builds boot.bin from the installed m1n1 and device trees, and appends +/etc/m1n1.conf's chosen.* lines. pacman -U and the m1n1/kernel installs run it, +as the hooks do. pacman -U installs a package's own m1n1.bin, so boot.bin +starts with the bytes of the package the installer downloaded; every fixture +m1n1.bin starts with "M1N1:". """ from pathlib import Path import hashlib @@ -16,8 +19,15 @@ import unittest INSTALLER = Path(__file__).resolve().parent.parent / "install-aurora-sep.sh" +SRC = INSTALLER.read_text() # The installer's VERSION names the boot.bin copy it keeps on the EFI partition. -VERSION = re.search(r"^VERSION=(\S+)$", INSTALLER.read_text(), re.M).group(1) +VERSION = re.search(r"^VERSION=(\S+)$", SRC, re.M).group(1) +# The one m1n1 package every Mac gets, by the name this release ships. +M1N1_PKG = re.search(r'^M1N1_PACKAGE="(\S+) ', SRC, re.M).group(1) +M1N1_BASE = M1N1_PKG[:-len("-aarch64.pkg.tar.zst")] +# Every switch name the shipped m1n1 has to know: the M3 Pro's and the Air's. +SWITCH_NAMES = sorted({w[len("chosen."):].split("=")[0] + for w in re.findall(r"chosen\.asahi,t(?:6030|8122)-[a-z0-9-]+=1", SRC)}) BOARDS = { "j516s": ["apple,j516s", "apple,t6030", "apple,arm-platform"], @@ -26,6 +36,10 @@ "j615": ["apple,j615", "apple,t8122", "apple,arm-platform"], "j504": ["apple,j504", "apple,t8122", "apple,arm-platform"], "j314s": ["apple,j314s", "apple,t6000", "apple,arm-platform"], + "j293": ["apple,j293", "apple,t8103", "apple,arm-platform"], + "j314c": ["apple,j314c", "apple,t6001", "apple,arm-platform"], + "j414s": ["apple,j414s", "apple,t6020", "apple,arm-platform"], + "j700": ["apple,j700", "apple,t8140", "apple,arm-platform"], } SWITCHES = b"chosen.asahi,t6030-gpu=1\nchosen.asahi,t6030-dcp=1\nchosen.asahi,t6030-dcpext=1\n" BRINGUP_FREEZE = ( @@ -38,7 +52,7 @@ "M1N1_UPDATE_DISABLED=1\n" "# <<< aurora-sep: keep this M3's boot.bin as it is\n" ) -AURORA3 = "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst" +# An older M3 m1n1 that knows only the M3 Pro's switches. AURORA6 = "m1n1-aurora-1.6.1.aurora6-1-aarch64.pkg.tar.zst" OTHERS = [ "linux-aurora-7.1.12.aurora2-11.36-aarch64.pkg.tar.zst", @@ -71,7 +85,7 @@ [[ $f == -* || $f == 4 ]] && continue b=$(basename "$f") case $b in - m1n1-aurora-*) echo "${b%-aarch64.pkg.tar.zst}" >"$FAKE/m1n1"; printf 'M1N1:%s\n' "${b%-aarch64.pkg.tar.zst}" >"$FAKE/m1n1.bin" ;; + m1n1-aurora-*) echo "${b%-aarch64.pkg.tar.zst}" >"$FAKE/m1n1"; bsdtar -xOf "$f" usr/lib/asahi-boot/m1n1.bin >"$FAKE/m1n1.bin" ;; linux-aurora-headers-*) ;; linux-aurora-*) sed -i '/^linux-asahi$/d' "$FAKE/installed"; echo linux-aurora >>"$FAKE/installed" ;; esac @@ -104,8 +118,9 @@ fi fi dtbs=$(sh -c 'set -e; DTBS=; [ -f "$1" ] && . "$1"; echo "$DTBS"' _ "$FAKE_UPDATE_CONF") +m1n1=$(sh -c 'set -e; M1N1=; [ -f "$1" ] && . "$1"; echo "$M1N1"' _ "$FAKE_UPDATE_CONF") { - cat "$FAKE/m1n1.bin" + cat "${m1n1:-$FAKE/m1n1.bin}" printf 'DTBS:%s\n' "${dtbs:-asahi}" printf 'UBOOT' grep -E '^chosen\.' "$FAKE_M1N1_CONF" 2>/dev/null || true @@ -146,10 +161,13 @@ def setUp(self): (self.tmp / "bin" / name).chmod(0o755) self.extra_env = {} self.shas = {} + self.bin_shas = {} for name in OTHERS: self.fixture(name, name.encode()) - self.fixture(AURORA3, None, []) + self.fixture(M1N1_PKG, None, SWITCH_NAMES) self.fixture(AURORA6, None, ["asahi,t6030-gpu", "asahi,t6030-dcp", "asahi,t6030-dcpext"]) + # The m1n1 package every Mac gets in this fake release. + self.m1n1_pkg = M1N1_PKG def fixture(self, name, data, strings=None): path = self.tmp / "pkgs" / name @@ -157,10 +175,14 @@ def fixture(self, name, data, strings=None): path.write_bytes(data) else: root = self.tmp / ("root-" + name) + shutil.rmtree(root, ignore_errors=True) (root / "usr/lib/asahi-boot").mkdir(parents=True) - (root / "usr/lib/asahi-boot/m1n1.bin").write_bytes( - b"m1n1\0" + b"\0".join(s.encode() for s in strings) + b"\0end" + bytes(range(256)) * 4096) + base = name.removesuffix("-aarch64.pkg.tar.zst").removesuffix(".pkg.tar.zst") + m1n1 = (b"M1N1:" + base.encode() + b"\n" + b"m1n1\0" + b"\0".join(s.encode() for s in strings) + + b"\0end" + bytes(range(256)) * 4096) + (root / "usr/lib/asahi-boot/m1n1.bin").write_bytes(m1n1) subprocess.run(["bsdtar", "--zstd", "-cf", str(path), "-C", str(root), "usr"], check=True) + self.bin_shas[name] = hashlib.sha256(m1n1).hexdigest() self.shas[name] = hashlib.sha256(path.read_bytes()).hexdigest() def mac(self, board, kernel="linux-asahi", m1n1="m1n1-stock", bootbin=b"M1N1:original\n", @@ -181,7 +203,7 @@ def mac(self, board, kernel="linux-asahi", m1n1="m1n1-stock", bootbin=b"M1N1:ori self.boot.write_bytes(bootbin) def run_sh(self, body, check=True): - pkgs = "\n".join(f' "{n} {self.shas[n]}"' for n in OTHERS + [AURORA3]) + pkgs = "\n".join(f' "{n} {self.shas[n]}"' for n in OTHERS) script = f""" set -euo pipefail AURORA_SEP_SOURCE_ONLY=1 source '{INSTALLER}' @@ -196,7 +218,7 @@ def run_sh(self, body, check=True): PACKAGES=( {pkgs} ) -M3_M1N1_PACKAGE="{AURORA6} {self.shas[AURORA6]}" +M1N1_PACKAGE="{self.m1n1_pkg} {self.shas[self.m1n1_pkg]}" esp_bootbin() {{ echo '{self.boot}'; }} version_notice() {{ :; }}; sep_write_notice() {{ :; }}; ane_dkms_notice() {{ :; }} snapshot() {{ :; }}; add_pin() {{ :; }}; remove_pin() {{ :; }} @@ -239,10 +261,9 @@ def test_fresh_m3_pro_gets_the_handoff(self): self.assertIn("built-in display at its native resolution", out) self.assertNotIn("run: aurora-touchid-setup", out) boot = self.boot.read_bytes() - self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora6-1\n"), boot) - self.assertTrue(boot.endswith(SWITCHES), boot) - self.assertIn(AURORA6, self.downloaded()) - self.assertNotIn(AURORA3, self.downloaded()) + self.assertTrue(boot.startswith(b"M1N1:" + M1N1_BASE.encode() + b"\n"), boot[:80]) + self.assertTrue(boot.endswith(SWITCHES), boot[-200:]) + self.assertEqual([d for d in self.downloaded() if d.startswith("m1n1-")], [M1N1_PKG]) self.assertEqual(self.kept_copy().read_bytes(), b"M1N1:original\n") self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff t6030") self.assertFalse(self.update_conf.read_text().count("M1N1_UPDATE_DISABLED")) @@ -259,7 +280,7 @@ def test_m3_pro_over_an_11_36_kernel_only_install(self): self.update_conf.write_text(OUR_FREEZE) self.install() boot = self.boot.read_bytes() - self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora6-1\n"), boot) + self.assertTrue(boot.startswith(b"M1N1:" + M1N1_BASE.encode() + b"\n"), boot[:80]) self.assertTrue(boot.endswith(SWITCHES), boot) self.assertNotIn("M1N1_UPDATE_DISABLED", self.update_conf.read_text()) @@ -271,7 +292,7 @@ def test_bringup_m3_pro_and_back(self): marker.touch() self.install() boot = self.boot.read_bytes() - self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora6-1\n"), boot) + self.assertTrue(boot.startswith(b"M1N1:" + M1N1_BASE.encode() + b"\n"), boot[:80]) self.assertTrue(boot.endswith(SWITCHES)) # The hook ran while the bring-up freeze still held, so the first # rebuild is the installer's own, with the new m1n1 in place. @@ -372,7 +393,7 @@ def test_m1_ignores_the_m3_stage1_and_overlap_checks(self): proc = self.install() self.assertNotIn("stage 1", proc.stderr + proc.stdout) self.assertNotIn("overlaps", proc.stderr + proc.stdout) - self.assertTrue(self.boot.read_bytes().startswith(b"M1N1:m1n1-aurora-1.6.1.aurora3-1\n")) + self.assertTrue(self.boot.read_bytes().startswith(b"M1N1:" + M1N1_BASE.encode() + b"\n")) def test_trial_refused_on_an_m3(self): # A plain M3 that isn't an Air (the Air's opt-in is in test_m3_air.py). @@ -463,19 +484,79 @@ def test_empty_mode_file_does_not_break_uninstall(self): (self.state / "m3-mode").write_text("") self.uninstall() - # M1/M2 are unchanged: aurora3, no switches, no M3 state + # M1/M2: the same m1n1 as every Mac, no switches, no M3 state, and the + # boot.bin they booted with kept first - def test_m1_pro_unchanged(self): + def test_m1_pro_gets_the_one_m1n1(self): self.mac("j314s") - out = self.install().stdout + proc = self.install() + out = proc.stdout self.assertIn("run: aurora-touchid-setup", out) boot = self.boot.read_bytes() - self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora3-1\n"), boot) + self.assertTrue(boot.startswith(b"M1N1:" + M1N1_BASE.encode() + b"\n"), boot[:80]) self.assertNotIn(b"chosen.", boot) - self.assertIn(AURORA3, self.downloaded()) - self.assertNotIn(AURORA6, self.downloaded()) + self.assertEqual([d for d in self.downloaded() if d.startswith("m1n1-")], [M1N1_PKG]) self.assertFalse((self.state / "m3-mode").exists()) self.assertFalse(self.m1n1_conf.exists()) + # The safety net every Mac whose boot loader changes gets. + self.assertEqual(self.kept_copy().read_bytes(), b"M1N1:original\n") + self.assertIn("put the\n boot loader it booted with back from macOS", proc.stderr) + self.assertIn("m1n1-aurora " + M1N1_BASE[len("m1n1-aurora-"):], proc.stderr) + self.uninstall() + self.assertTrue(self.boot.read_bytes().startswith(b"M1N1:m1n1-stock\n")) + + def test_every_mac_gets_the_same_m1n1(self): + # M1, M2 and the M3s on the handoff path: one package, and only the + # switches differ. + for board, try_, switches in [("j293", 0, b""), ("j314s", 0, b""), ("j414s", 0, b""), + ("j516s", 0, SWITCHES), ("j514s", 1, SWITCHES)]: + with self.subTest(board=board): + shutil.rmtree(self.state) + self.state.mkdir() + self.m1n1_conf.unlink(missing_ok=True) + self.update_conf.unlink(missing_ok=True) + self.mac(board) + self.install(try_=try_) + boot = self.boot.read_bytes() + self.assertTrue(boot.startswith(b"M1N1:" + M1N1_BASE.encode() + b"\n"), boot[:80]) + self.assertTrue(boot.endswith(b"UBOOT" + switches), boot[-200:]) + self.assertEqual([d for d in self.downloaded() if d.startswith("m1n1-")], [M1N1_PKG]) + + # The MacBook Neo keeps its own m1n1 until NEO_AURORA_M1N1 is 1 + + NEO_CONF = "M1N1=__NEO__\nU_BOOT=/usr/lib/neo/u-boot.bin\n" + + def neo(self): + own = self.tmp / "neo-m1n1.bin" + own.write_bytes(b"M1N1:neo-own\n") + self.mac("j700", m1n1="m1n1-stock") + conf = self.NEO_CONF.replace("__NEO__", str(own)) + self.update_conf.write_text(conf) + return conf + + def test_neo_keeps_its_own_m1n1(self): + conf = self.neo() + out = self.install().stdout + self.assertIn("Keeping this MacBook Neo's own m1n1", out) + self.assertEqual([d for d in self.downloaded() if d.startswith("m1n1-")], []) + self.assertTrue(self.boot.read_bytes().startswith(b"M1N1:neo-own\n")) + self.assertIn(conf, self.update_conf.read_text()) + self.assertFalse(self.kept_copy().exists()) + + def test_neo_on_the_one_m1n1_with_the_switch(self): + conf = self.neo() + proc = self.install(env="NEO_AURORA_M1N1=1") + self.assertNotIn("own m1n1", proc.stdout) + self.assertEqual([d for d in self.downloaded() if d.startswith("m1n1-")], [M1N1_PKG]) + self.assertTrue(self.boot.read_bytes().startswith(b"M1N1:" + M1N1_BASE.encode() + b"\n")) + text = self.update_conf.read_text() + self.assertNotIn("M1N1=", text.replace("M1N1_UPDATE", "")) + self.assertIn("U_BOOT=/usr/lib/neo/u-boot.bin", text) + self.assertEqual(self.kept_copy().read_bytes(), b"M1N1:original\n") + self.run_sh("NEO_AURORA_M1N1=1\nuninstall_all") + self.assertEqual(self.update_conf.read_text(), conf) + self.assertTrue(self.boot.read_bytes().startswith(b"M1N1:neo-own\n")) + self.assertIn("libfprint m1n1", self.log()) if __name__ == "__main__": diff --git a/tools/aurora-sep/tests/test_m3_handoff.py b/tools/aurora-sep/tests/test_m3_handoff.py index 9ec677d61348f..9e89a51bf6852 100644 --- a/tools/aurora-sep/tests/test_m3_handoff.py +++ b/tools/aurora-sep/tests/test_m3_handoff.py @@ -18,8 +18,8 @@ # Release names, read from the installer so the tests follow each release. _SRC = INSTALLER.read_text() VERSION = re.search(r"^VERSION=(\S+)$", _SRC, re.M).group(1) -M1N1_M1M2 = re.search(r'^ "(m1n1-aurora-\S+) ', _SRC, re.M).group(1) -M1N1_M3 = re.search(r'^M3_M1N1_PACKAGE="(\S+) ', _SRC, re.M).group(1) +# One m1n1 for every Mac that gets one. +M1N1 = re.search(r'^M1N1_PACKAGE="(\S+) ', _SRC, re.M).group(1) KERNEL_PKG = f"linux-aurora-{VERSION}-aarch64.pkg.tar.zst" OUR_FREEZE = ( @@ -381,13 +381,11 @@ def packages(self, board, stub="14.8.3", try_=0): return [line.split()[0] for line in out.splitlines()] def test_packages_per_mac(self): - m1n1_m1m2 = M1N1_M1M2 - m1n1_m3 = M1N1_M3 - for board, stub, try_, want in [("j314s", "13.5", 0, m1n1_m1m2), ("j293", "13.5", 0, m1n1_m1m2), + for board, stub, try_, want in [("j314s", "13.5", 0, M1N1), ("j293", "13.5", 0, M1N1), ("j700", "26.4", 0, None), ("j613", "14.8.3", 0, None), ("j514c", "14.8.3", 0, None), ("j514s", "14.8.3", 0, None), - ("j516s", "15.6", 0, None), ("j516s", "14.8.3", 0, m1n1_m3), - ("j514s", "14.8.3", 1, m1n1_m3)]: + ("j516s", "15.6", 0, None), ("j516s", "14.8.3", 0, M1N1), + ("j514s", "14.8.3", 1, M1N1)]: with self.subTest(board=board, stub=stub, try_=try_): got = self.packages(board, stub, try_) m1n1 = [p for p in got if p.startswith("m1n1-aurora-")] From d9423631f1a9282fac78901c1b3108a749003048 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:46:21 -0700 Subject: [PATCH 1759/2680] tools/aurora-sep: check the m1n1 in boot.bin by its bytes, and record it Two m1n1 builds can report the same version string: aurora8.5-1 and 8.5-2 both reported v1.6.1-omarchy.aurora8.5, so neither a log nor a version check could tell which one a Mac booted. M1N1_BIN_SHA names the m1n1.bin that M1N1_PACKAGE must hold (a placeholder until the aurora12 build). After the download, the package's m1n1.bin must have that sha256, or the install stops with nothing installed. After update-m1n1 has rebuilt boot.bin with it, the first bytes of boot.bin (as many as m1n1.bin has) must have it too, on every Mac that got the package; the result is printed and recorded in $STATE/m1n1-installed as "sha256 bytes package". A boot.bin that update-m1n1 does not rebuild (frozen) or builds from its own m1n1 (M1N1=, SOURCE= or TARGET=, as on a Neo without NEO_AURORA_M1N1) is not checked, as before. --m3-report gives the installed m1n1.bin's sha256, boot.bin's, the sha256 of boot.bin's first bytes, and the record. Tests: the record on an M1 Pro and a J516S; a package whose m1n1.bin has other bytes is refused before anything changes; a rebuild from other m1n1 bytes stops before the record; a frozen M1 is reported as in 11.38; the report lines. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit eaaeeb094ae60a2e93d26f47ffe6103b77b4f565) --- tools/aurora-sep/install-aurora-sep.sh | 79 +++++++++++++++++++++++- tools/aurora-sep/tests/test_m3_air.py | 4 +- tools/aurora-sep/tests/test_m3_flow.py | 61 ++++++++++++++++++ tools/aurora-sep/tests/test_m3_report.py | 10 +++ 4 files changed, 152 insertions(+), 2 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index cf838455eb575..aa7f316674de5 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -175,6 +175,11 @@ PACKAGES=( # path (see m3_plan), and the MacBook Neo once NEO_AURORA_M1N1 is 1. Macs differ # only in the switches /etc/m1n1.conf arms (m3_switches), never in the binary. M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora12-1-aarch64.pkg.tar.zst PENDING-AURORA12-BUILD" +# The sha256 of the m1n1.bin in M1N1_PACKAGE: the bytes update-m1n1 puts at +# the start of boot.bin. The script tells m1n1 builds apart by these bytes, +# never by the version string they report: aurora8.5-1 and 8.5-2 both reported +# v1.6.1-omarchy.aurora8.5. +M1N1_BIN_SHA=PENDING-AURORA12-BUILD # 0: a MacBook Neo keeps its own m1n1 (its M1N1= and U_BOOT= in # /etc/default/update-m1n1), as before 12.0. 1: it gets M1N1_PACKAGE like every # other Mac, and update-m1n1 builds its boot.bin from that m1n1 and the Neo's @@ -381,6 +386,8 @@ M3_OSLOG_OVERLAP=chosen/asahi,m1n1-oslog-overlap M1N1_CONF=/etc/m1n1.conf # What update-m1n1 puts at the start of boot.bin. M1N1_BIN=/usr/lib/asahi-boot/m1n1.bin +# After a rebuild with M1N1_PACKAGE, $STATE/m1n1-installed records the m1n1 +# that boot.bin was checked to start with: "sha256 bytes package". M1N1_CONF_BEGIN="# >>> aurora-sep: M3 Pro display and GPU handoff (remove with: install-aurora-sep.sh --uninstall)" M1N1_CONF_END="# <<< aurora-sep: M3 Pro display and GPU handoff" M1N1_CONF_AIR_BEGIN="# >>> aurora-sep: M3 Air handoff (remove with: install-aurora-sep.sh --uninstall)" @@ -579,6 +586,42 @@ m3_oslog_overlap_check() { partition as m1n1/boot.bin.before-." } +# The sha256 of a downloaded m1n1-aurora package's m1n1.bin. +m1n1_pkg_sha() { + { bsdtar -xOf "$1" usr/lib/asahi-boot/m1n1.bin | sha256sum | cut -d' ' -f1; } 2>/dev/null +} + +# The sha256 of the first $2 bytes of boot.bin $1: the m1n1 at its start, when +# $2 is that m1n1's size. +bootbin_m1n1_sha() { + $sudo head -c "$2" "$1" | sha256sum | cut -d' ' -f1 +} + +# True when update-m1n1's configuration names its own m1n1 or target: boot.bin +# then never starts with M1N1_PACKAGE's m1n1, which is not checked there. +update_m1n1_own_m1n1() { + [[ -f $UPDATE_M1N1_CONF ]] || return 1 + # shellcheck disable=SC2016 # expanded by that sh, not here + env -i PATH="$PATH" sh -c '. "$1" >/dev/null 2>&1; [ -n "${M1N1:-}${SOURCE:-}${TARGET:-}" ]' _ \ + "$UPDATE_M1N1_CONF" >/dev/null 2>&1 +} + +# After update-m1n1 rebuilt boot.bin with M1N1_PACKAGE: check the m1n1 at its +# start by its bytes, and record it in $STATE/m1n1-installed. +m1n1_check_and_record() { + local target size sha + target=$(esp_bootbin) || die "could not find m1n1's boot.bin to check its m1n1 in" + size=$(stat -c %s "$M1N1_BIN") + sha=$(bootbin_m1n1_sha "$target" "$size") + [[ $sha == "$M1N1_BIN_SHA" ]] || + die "the m1n1 at the start of $target is sha256 $sha, not this release's + ($M1N1_BIN_SHA), so boot.bin was not rebuilt as it should be. The boot loader this Mac booted + with is kept as m1n1/boot.bin.before-$VERSION on the EFI partition. Please report it at + https://github.com/iconidentify/aurora-linux/issues before rebooting." + printf '%s %s %s\n' "$sha" "$size" "${M1N1_PACKAGE%% *}" | $sudo tee "$STATE/m1n1-installed" >/dev/null + say "m1n1's boot.bin starts with this release's m1n1 (sha256 $sha)" +} + # True when a downloaded m1n1-aurora package knows every switch this Mac's # handoff needs (m3_switches): asahi,t6030-gpu for chosen.asahi,t6030-gpu=1. m1n1_pkg_has_handoff() { @@ -1370,6 +1413,13 @@ install_all() { Please report it with the file name above." [[ $(sha256sum "$work/$file" | cut -d' ' -f1) == "$sha" ]] || die "$file does not match its published checksum" done + if m1n1_for_this_mac; then + sha=$(m1n1_pkg_sha "$work/${M1N1_PACKAGE%% *}") + [[ $sha == "$M1N1_BIN_SHA" ]] || + die "${M1N1_PACKAGE%% *} holds an m1n1.bin with sha256 ${sha:-(none)}, not the + $M1N1_BIN_SHA this release names. Nothing was installed. Please report it at + https://github.com/iconidentify/aurora-linux/issues" + fi if [[ $M3_MODE == handoff ]]; then m1n1_pkg_has_handoff "$work/${M1N1_PACKAGE%% *}" || die "this release's m1n1 has no $(m3_handoff_name), so it can't switch it @@ -1451,11 +1501,19 @@ install_all() { fi m1n1_update m3_verify_bootbin + m1n1_check_and_record # The test reboot may be a hard reset (m3-serial.py reboot): get the new # boot.bin onto the EFI partition first. sync ;; - *) m1n1_update ;; + *) + m1n1_update + # Not where update-m1n1 is frozen (m1n1_update said so) or builds from + # an m1n1 of its own, as a Neo without NEO_AURORA_M1N1 does. + if m1n1_for_this_mac && ! update_m1n1_frozen && ! update_m1n1_own_m1n1; then + m1n1_check_and_record + fi + ;; esac if [[ $chain == grub && $M3_MODE != handoff ]]; then grub_update @@ -1700,6 +1758,24 @@ reset_touchid() { # directory. It reads only: the boot loader's /chosen entries, a kernel log # filtered to the M3 bring-up, USB-C and display state. Lines naming a USB # serial number are dropped and MAC addresses are masked. +# The m1n1 in boot.bin by its bytes, for --m3-report: two builds can report the +# same stage 2 version. +m3_report_m1n1() { + local target size="" installed="" + if [[ -f $M1N1_BIN ]]; then + size=$(stat -c %s "$M1N1_BIN") + installed=$(sha256sum "$M1N1_BIN" | cut -d' ' -f1) + fi + echo "installed m1n1.bin: sha256 ${installed:--} (${size:--} bytes)" + if [[ -n $size ]] && target=$(esp_bootbin); then + echo "boot.bin: sha256 $($sudo sha256sum "$target" | cut -d' ' -f1)" + echo "boot.bin's first $size bytes: sha256 $(bootbin_m1n1_sha "$target" "$size")" + else + echo "boot.bin: -" + fi + printf 'm1n1-installed: '; cat "$STATE/m1n1-installed" 2>/dev/null || echo - +} + M3_REPORT_DMESG='asahi|agx|gpu|g15|dcp|dart|t8122|t6030|reserved|iommu|mailbox|pmp|simpledrm|m1n1|tipd|typec|sn201202|atc|usb|xhci|dwc3|thermal|macsmc' m3_report() { local dir out board soc f @@ -1714,6 +1790,7 @@ m3_report() { done pacman -Q linux-aurora m1n1-aurora m1n1 2>/dev/null || true printf 'm3-mode: '; cat "$STATE/m3-mode" 2>/dev/null || echo - + m3_report_m1n1 echo "m1n1.conf switches:"; grep '^chosen\.' "$M1N1_CONF" 2>/dev/null || echo - printf 'm1n1-oslog-overlap: '; if [[ -e $DT/$M3_OSLOG_OVERLAP ]]; then echo present; else echo absent; fi echo "reserved display logs:"; ls -d "$DT"/reserved-memory/dcp-oslog@* 2>/dev/null || echo - diff --git a/tools/aurora-sep/tests/test_m3_air.py b/tools/aurora-sep/tests/test_m3_air.py index 0d77af50aa0e7..ff839b664772e 100644 --- a/tools/aurora-sep/tests/test_m3_air.py +++ b/tools/aurora-sep/tests/test_m3_air.py @@ -53,6 +53,7 @@ def sh(self, body, pkg=None, dcp=0, boards=None, check=True, display=0, variant= f'M3_AIR_DISPLAY_HANDOFF={display}\n') if pkg is not None: pre += f'M1N1_PACKAGE="{pkg} {self.shas[pkg]}"\n' + pre += f'M1N1_BIN_SHA={self.bin_shas[pkg]}\n' if variant is not None: pre += f'M3_AIR_DISPLAY_VARIANT={variant}\n' if boards is not None: @@ -564,7 +565,8 @@ def test_display_handoff_needs_the_checked_stage1(self): def test_install_syncs_after_the_rebuild(self): src = SRC[SRC.index(" m3_verify_bootbin\n"):] - self.assertTrue(re.match(r" m3_verify_bootbin\n(\s*#.*\n)*\s*sync\n", src), src[:200]) + self.assertTrue(re.match(r" m3_verify_bootbin\n m1n1_check_and_record\n(\s*#.*\n)*\s*sync\n", src), + src[:200]) if __name__ == "__main__": unittest.main() diff --git a/tools/aurora-sep/tests/test_m3_flow.py b/tools/aurora-sep/tests/test_m3_flow.py index 4be32868bf164..adf3ef8e8b222 100644 --- a/tools/aurora-sep/tests/test_m3_flow.py +++ b/tools/aurora-sep/tests/test_m3_flow.py @@ -119,6 +119,8 @@ fi dtbs=$(sh -c 'set -e; DTBS=; [ -f "$1" ] && . "$1"; echo "$DTBS"' _ "$FAKE_UPDATE_CONF") m1n1=$(sh -c 'set -e; M1N1=; [ -f "$1" ] && . "$1"; echo "$M1N1"' _ "$FAKE_UPDATE_CONF") +# A rebuild that picks up another m1n1 than the installed one. +m1n1=${FAKE_OTHER_M1N1:-$m1n1} { cat "${m1n1:-$FAKE/m1n1.bin}" printf 'DTBS:%s\n' "${dtbs:-asahi}" @@ -219,6 +221,7 @@ def run_sh(self, body, check=True): {pkgs} ) M1N1_PACKAGE="{self.m1n1_pkg} {self.shas[self.m1n1_pkg]}" +M1N1_BIN_SHA={self.bin_shas[self.m1n1_pkg]} esp_bootbin() {{ echo '{self.boot}'; }} version_notice() {{ :; }}; sep_write_notice() {{ :; }}; ane_dkms_notice() {{ :; }} snapshot() {{ :; }}; add_pin() {{ :; }}; remove_pin() {{ :; }} @@ -522,6 +525,64 @@ def test_every_mac_gets_the_same_m1n1(self): self.assertTrue(boot.endswith(b"UBOOT" + switches), boot[-200:]) self.assertEqual([d for d in self.downloaded() if d.startswith("m1n1-")], [M1N1_PKG]) + # The m1n1 by its bytes (M1N1_BIN_SHA), never by the version it reports + + def fresh_state(self): + shutil.rmtree(self.state) + self.state.mkdir() + self.m1n1_conf.unlink(missing_ok=True) + self.update_conf.unlink(missing_ok=True) + for kept in self.boot.parent.glob("boot.bin.*"): + kept.unlink() + + def test_m1n1_is_checked_and_recorded_by_its_bytes(self): + sha = self.bin_shas[M1N1_PKG] + size = len((self.tmp / ("root-" + M1N1_PKG) / "usr/lib/asahi-boot/m1n1.bin").read_bytes()) + for board in ("j314s", "j516s"): + with self.subTest(board=board): + self.fresh_state() + self.mac(board) + out = self.install().stdout + self.assertIn(f"starts with this release's m1n1 (sha256 {sha})", out) + self.assertEqual((self.state / "m1n1-installed").read_text(), f"{sha} {size} {M1N1_PKG}\n") + + def test_a_package_with_other_m1n1_bytes_is_refused(self): + # Same file name and version, other bytes: refused before anything changes. + for board in ("j314s", "j516s"): + with self.subTest(board=board): + self.fresh_state() + self.mac(board) + before = self.boot.read_bytes() + proc = self.install(env="M1N1_BIN_SHA=" + "0" * 64, check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn(f"holds an m1n1.bin with sha256 {self.bin_shas[M1N1_PKG]}", proc.stderr) + self.assertIn("Nothing was installed", proc.stderr) + self.assertEqual(self.boot.read_bytes(), before) + self.assertNotIn("pacman -U", self.log()) + self.assertFalse(self.kept_copy().exists()) + + def test_a_rebuild_with_other_m1n1_bytes_stops(self): + # update-m1n1 rebuilt boot.bin, but not from this release's m1n1. + other = self.tmp / "other-m1n1.bin" + other.write_bytes(b"M1N1:" + M1N1_BASE.encode() + b"\n" + b"\0" * 64) + self.mac("j314s") + self.extra_env = {"FAKE_OTHER_M1N1": str(other)} + proc = self.install(check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("not this release's", proc.stderr) + self.assertIn(f"boot.bin.before-{VERSION}", proc.stderr) + self.assertFalse((self.state / "m1n1-installed").exists()) + + def test_frozen_m1_keeps_its_boot_bin_unchecked(self): + # As in 11.38: a freeze the owner set is reported, and nothing checks + # a boot.bin that was not rebuilt. + self.mac("j314s") + self.update_conf.write_text("M1N1_UPDATE_DISABLED=1\n") + proc = self.install() + self.assertIn("was not rebuilt", proc.stderr) + self.assertEqual(self.boot.read_bytes(), b"M1N1:original\n") + self.assertFalse((self.state / "m1n1-installed").exists()) + # The MacBook Neo keeps its own m1n1 until NEO_AURORA_M1N1 is 1 NEO_CONF = "M1N1=__NEO__\nU_BOOT=/usr/lib/neo/u-boot.bin\n" diff --git a/tools/aurora-sep/tests/test_m3_report.py b/tools/aurora-sep/tests/test_m3_report.py index 90b53e75b1de7..e36b10042035b 100644 --- a/tools/aurora-sep/tests/test_m3_report.py +++ b/tools/aurora-sep/tests/test_m3_report.py @@ -70,6 +70,16 @@ def test_report_changes_nothing(self): self.assertNotIn("pacman -U", self.log()) self.assertNotIn("update-m1n1", self.log()) + def test_report_names_the_m1n1_by_its_bytes(self): + self.mac("j516s") + self.install() + sha = self.bin_shas[self.m1n1_pkg] + _, files = self.report() + text = files["system.txt"].decode() + self.assertIn(f"installed m1n1.bin: sha256 {sha}", text) + self.assertRegex(text, rf"boot.bin's first \d+ bytes: sha256 {sha}") + self.assertIn(f"m1n1-installed: {sha} ", text) + def test_report_needs_no_preflight(self): src = flow.INSTALLER.read_text() self.assertIn("--agent-prompt | --reset-touchid | --m3-report) return 1", src) From 4d8a0ede8e657fdf9f97950f03414ef42c48465d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:51:10 -0700 Subject: [PATCH 1760/2680] tools/aurora-sep: never put back an m1n1 that failed on this Mac After a failed boot the restore steps had the owner put the old boot.bin back from macOS and freeze update-m1n1. The next plain run on a J516S then refused with "Remove that line and run this again", and doing so rebuilt boot.bin with the same m1n1, which stopped the Mac again. With one m1n1 for every Mac, M1 and M2 could loop the same way. $STATE/m1n1-failed now lists, by sha256, the m1n1 builds that failed on this Mac. A line gets there in two ways: - the restore steps print a second line to run after the freeze, which appends this release's m1n1 sha256; - the installer finds boot.bin put back by hand: it starts neither with the m1n1 recorded in $STATE/m1n1-installed nor with the installed one. A re-run, or a rebuild from a newer installed m1n1, is not taken for one. With a failed m1n1 on record: - an M3 on a plain run installs the kernel only and keeps the boot loader it has, whatever this release's m1n1 is; --m3-handoff refuses the m1n1 that failed, and tries a different one; - an M1, M2 or a Neo on M1N1_PACKAGE keeps its boot.bin when this release's m1n1 is the one that failed, behind the freeze; without the freeze it stops first and prints the line to run; - --uninstall keeps the list. A Mac with the handoff from an earlier release, which recorded no m1n1, is kept on a plain run by a freeze that is not this script's (the restore steps' line), instead of being told to remove it. Every message points at issue #6. The restore steps' Linux copy goes through a .new file and a rename. The agent prompt has one section for a Mac that stops in m1n1, which cases A, B and D follow. The handoff's verification errors name only m1n1/boot.bin.before-, the copy this run kept. Tests: the review's J516S loop, including a lifted freeze, a new --m3-handoff, and --uninstall then a new install; the printed line on its own; a pre-12.0 put-back kept by its freeze; the same on an M1 Pro, with and without the freeze; rebuilds that are not put-backs; a newer m1n1 after a failed one; the printed and the agent-prompt steps. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 59f37a38432be3ae6557df44c0e34b4b4708b070) --- tools/aurora-sep/install-aurora-sep.sh | 190 +++++++++++++++--- tools/aurora-sep/tests/test_m1n1_rollback.py | 198 +++++++++++++++++++ tools/aurora-sep/tests/test_m3_flow.py | 22 +-- 3 files changed, 372 insertions(+), 38 deletions(-) create mode 100644 tools/aurora-sep/tests/test_m1n1_rollback.py diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index aa7f316674de5..e09081bdc7874 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -406,6 +406,12 @@ M3_MODE=none # Set by --m3-handoff: the owner asks for the handoff on an M3 Pro or M3 # MacBook Air model that isn't in M3_HANDOFF_BOARDS yet. M3_TRY=0 +# 1 when this run keeps the boot.bin this Mac has because an m1n1 from this +# script failed on it before ($STATE/m1n1-failed): set by m3_plan and +# m1n1_keep_plan. +M1N1_KEEP=0 +# Where issue reports for a boot loader that failed go. +ISSUE_URL=https://github.com/iconidentify/aurora-linux/issues/6 # Whether update-m1n1 would exit without building, judged the way it judges: # it sources this file and stops when M1N1_UPDATE_DISABLED is non-empty. @@ -514,7 +520,9 @@ m3_switches() { # handoff path, and a MacBook Neo only with NEO_AURORA_M1N1=1. Needs m3_plan # first. m1n1_for_this_mac() { - if is_neo; then + if ((M1N1_KEEP)); then + return 1 + elif is_neo; then [[ $NEO_AURORA_M1N1 == 1 ]] elif [[ $M3_MODE != none ]]; then [[ $M3_MODE == handoff ]] @@ -606,6 +614,64 @@ update_m1n1_own_m1n1() { "$UPDATE_M1N1_CONF" >/dev/null 2>&1 } +# The m1n1 builds that failed on this Mac: the sha256 that starts each line of +# $STATE/m1n1-failed. The restore steps (keep_bootbin_on_esp) have the owner +# add a line; m1n1_rollback_check adds one when it finds the boot.bin put back. +m1n1_failed_shas() { + [[ -f $STATE/m1n1-failed ]] || return 0 + grep -oE '^[0-9a-f]{64}' "$STATE/m1n1-failed" || true +} + +m1n1_failed_add() { + m1n1_failed_shas | grep -qxF "$1" && return 0 + $sudo install -d "$STATE" + echo "$1 $2" | $sudo tee -a "$STATE/m1n1-failed" >/dev/null +} + +# A Mac whose boot.bin was put back by hand after this script rebuilt it, as +# the restore steps say: boot.bin starts neither with the m1n1 recorded in +# $STATE/m1n1-installed nor with the m1n1 that is installed now. That m1n1 is +# recorded as failed on this Mac. A rebuild that only changed the device trees +# or the switches keeps the same m1n1, and is not taken for one. +m1n1_rollback_check() { + local sha size pkg target + [[ -f $STATE/m1n1-installed ]] || return 0 + read -r sha size pkg _ <"$STATE/m1n1-installed" || return 0 + [[ $sha =~ ^[0-9a-f]{64}$ && $size =~ ^[0-9]+$ ]] || return 0 + target=$(esp_bootbin) || return 0 + [[ $(bootbin_m1n1_sha "$target" "$size") == "$sha" ]] && return 0 + if [[ -f $M1N1_BIN ]] && $sudo cmp -s -n "$(stat -c %s "$M1N1_BIN")" "$M1N1_BIN" "$target"; then + return 0 + fi + m1n1_failed_add "$sha" "${pkg:-m1n1}: boot.bin was put back by hand, found $(date +%F)" +} + +# On a Mac that is not an M3 (m3_plan decides for those): when this release's +# m1n1 failed on it before, keep the boot.bin it has (M1N1_KEEP). update-m1n1 +# must stay frozen, or pacman's hook would put that m1n1 back. +m1n1_keep_plan() { + if is_m3 || ! m1n1_for_this_mac; then return 0; fi + m1n1_failed_shas | grep -qxF "$M1N1_BIN_SHA" || return 0 + update_m1n1_frozen || + die "this release's m1n1 (sha256 $M1N1_BIN_SHA) failed on this Mac before (recorded in + $STATE/m1n1-failed), so this script does not put it back, and update-m1n1 must not either. + Keep updates from rebuilding boot.bin, then run this again: + echo M1N1_UPDATE_DISABLED=1 | sudo tee -a $UPDATE_M1N1_CONF + Nothing was installed. If you have not yet, please report what happened at $ISSUE_URL" + M1N1_KEEP=1 + say "This release's m1n1 (sha256 $M1N1_BIN_SHA) failed on this Mac before (recorded in + $STATE/m1n1-failed), so it is not put back: boot.bin stays as it is, and update-m1n1 stays + frozen in $UPDATE_M1N1_CONF. If you have not yet, please report what happened at $ISSUE_URL" +} + +# After an install that kept boot.bin (M1N1_KEEP) on a Mac that is not an M3. +m1n1_keep_report() { + [[ $(m3_bootbin_sha) == "$M3_BOOTBIN_SHA" ]] || + die "m1n1's boot.bin changed during the install, which it must not here. Please report it + at $ISSUE_URL before rebooting." + say "m1n1's boot.bin is unchanged: update-m1n1 stays frozen in $UPDATE_M1N1_CONF." +} + # After update-m1n1 rebuilt boot.bin with M1N1_PACKAGE: check the m1n1 at its # start by its bytes, and record it in $STATE/m1n1-installed. m1n1_check_and_record() { @@ -767,7 +833,7 @@ m3_recorded_variant() { # Decide the M3 path before anything is downloaded, so a Mac this release can't # set up as asked stops with nothing changed. m3_plan() { - local board problem air=0 + local board problem failed again="run this again" air=0 M3_MODE=none if ! is_m3; then ((M3_TRY == 0)) || die "--m3-handoff is for an M3 Pro or an M3 MacBook Air, and this Mac isn't an M3. Nothing was installed." @@ -776,9 +842,36 @@ m3_plan() { board=$(this_board) if is_m3_air; then air=1; fi m3_oslog_overlap_check + # An m1n1 from this script failed on this Mac before: a plain run keeps the + # boot loader it has now, whatever this release's m1n1 is, and --m3-handoff + # never puts back the one that failed. + failed=$(m1n1_failed_shas) + if [[ -n $failed ]] && ((M3_TRY == 0)); then + M3_MODE=kernel M1N1_KEEP=1 + say "M3 ($board): an m1n1 from this script failed on this Mac before (recorded in + $STATE/m1n1-failed). Installing the kernel only: the boot loader this Mac has now stays as + it is. If you have not yet, please report what happened at $ISSUE_URL" + return 0 + fi + if [[ -n $failed ]] && grep -qxF "$M1N1_BIN_SHA" <<<"$failed"; then + die "--m3-handoff: this release's m1n1 (sha256 $M1N1_BIN_SHA) is the one that failed on this + Mac (recorded in $STATE/m1n1-failed), so this script does not put it back. + Nothing was installed. Keep the kernel-only install: run this again without --m3-handoff. + If you have not yet, please report what happened at $ISSUE_URL" + fi # An update never takes the handoff away again: once a Mac has it (listed, or # tried with --m3-handoff), a plain run keeps it, and its checks still apply. if ((M3_TRY == 0)) && [[ $(m3_recorded_mode) == handoff ]]; then + # A freeze that is neither this script's nor the bring-up's: the line the + # restore steps give after a boot loader failed. Keep that boot.bin. + if update_m1n1_frozen_by_others; then + M3_MODE=kernel M1N1_KEEP=1 + say "M3 ($board): this Mac has m1n1's handoff from an earlier install, and update-m1n1 is + frozen in $UPDATE_M1N1_CONF, not by this script (the restore steps add that line). + Installing the kernel only: the boot loader this Mac has now stays as it is. If a boot + loader from this script failed on this Mac, please report it at $ISSUE_URL" + return 0 + fi if ((air)) && [[ $(m3_recorded_variant) != "$(m3_variant)" ]]; then die "M3 MacBook Air ($board): this Mac has an earlier test build's boot loader ($(m3_recorded_variant)), and this release's Air boot loader is a different one: the @@ -831,10 +924,11 @@ m3_plan() { this Mac differs: $problem. Installing the kernel only; boot.bin stays as it is." return 0 fi + if ((M3_TRY)); then again+=" with --m3-handoff"; fi if update_m1n1_frozen_by_others; then die "$UPDATE_M1N1_CONF sets M1N1_UPDATE_DISABLED, and not from this script or the M3 bring-up's install-m3gpu.sh. Switching the $(m3_handoff_name) on needs m1n1's boot.bin - rebuilt. Remove that line and run this again. Nothing was installed." + rebuilt with this release's m1n1. Remove that line and $again. Nothing was installed." fi if update_m1n1_customised; then die "$UPDATE_M1N1_CONF points update-m1n1 at its own m1n1, U-Boot, config or target @@ -883,13 +977,13 @@ m3_verify_bootbin() { $sudo cmp -s -n "$size" "$M1N1_BIN" "$target" || die "$target does not start with this release's m1n1 ($M1N1_BIN), so it was not rebuilt. The boot loader this Mac booted with is kept as m1n1/boot.bin.before-$VERSION on the EFI - partition and in $STATE/boot.bin.saved. Please report it before rebooting." + partition. Please report it before rebooting." block=$(m3_switches | tr ' ' '\n') tail=$($sudo tail -c 1024 "$target" | tr -d '\0') [[ $tail == *"$block"* ]] || die "the rebuilt $target does not carry the $kind switch lines. The boot loader this Mac - booted with is kept as m1n1/boot.bin.before-$VERSION on the EFI partition and in - $STATE/boot.bin.saved. Please report it before rebooting." + booted with is kept as m1n1/boot.bin.before-$VERSION on the EFI partition. Please report + it before rebooting." say "m1n1's boot.bin is this release's m1n1 with the $kind handoff switches" } @@ -978,11 +1072,14 @@ keep_bootbin_on_esp() { 3. sudo diskutil mount diskNsM (it prints the /Volumes path) 4. cp -X '/m1n1/boot.bin.before-$VERSION' '/m1n1/boot.bin' If it boots but something is wrong, the same copy goes back from Linux with: - sudo cp '$keep' '$target' && sync - After either, keep updates from rebuilding the test boot loader until it is sorted: + sudo cp '$keep' '$target.new' && sync && sudo mv '$target.new' '$target' && sync + After either, start Linux and run these two lines. The first keeps updates from + rebuilding the new boot loader; the second records that its m1n1 failed on this + Mac, so no later run of this script puts it back: echo M1N1_UPDATE_DISABLED=1 | sudo tee -a $UPDATE_M1N1_CONF + echo $M1N1_BIN_SHA | sudo tee -a $STATE/m1n1-failed The copy is kept at $keep. Either way, please report it at - https://github.com/iconidentify/aurora-linux/issues" + $ISSUE_URL" return 0 fi die "could not find m1n1's boot.bin on the EFI partition to keep a copy of; nothing was installed" @@ -1397,7 +1494,9 @@ install_all() { kernel=$(current_kernel) chain=$(boot_chain) if [[ $chain == grub ]]; then boot_space "$kernel"; fi + m1n1_rollback_check m3_plan + m1n1_keep_plan work=$(mktemp -d) trap 'rm -rf "${work:-}"' EXIT if is_neo && ! m1n1_for_this_mac; then say "Keeping this MacBook Neo's own m1n1 (m1n1-aurora has no T8140 support)"; fi @@ -1451,6 +1550,8 @@ install_all() { ;; # So the hook's rebuild, where it runs, already carries the switches. handoff) m3_switches_write ;; + # update-m1n1 is frozen (m1n1_keep_plan): prove boot.bin stays as it is. + *) if ((M1N1_KEEP)); then M3_BOOTBIN_SHA=$(m3_bootbin_sha); fi ;; esac # linux-aurora-headers pulls in pahole, and fprintd below comes from the @@ -1507,11 +1608,15 @@ install_all() { sync ;; *) - m1n1_update - # Not where update-m1n1 is frozen (m1n1_update said so) or builds from - # an m1n1 of its own, as a Neo without NEO_AURORA_M1N1 does. - if m1n1_for_this_mac && ! update_m1n1_frozen && ! update_m1n1_own_m1n1; then - m1n1_check_and_record + if ((M1N1_KEEP)); then + m1n1_keep_report + else + m1n1_update + # Not where update-m1n1 is frozen (m1n1_update said so) or builds from + # an m1n1 of its own, as a Neo without NEO_AURORA_M1N1 does. + if m1n1_for_this_mac && ! update_m1n1_frozen && ! update_m1n1_own_m1n1; then + m1n1_check_and_record + fi fi ;; esac @@ -1547,6 +1652,9 @@ install_all() { say "Done. Reboot with someone watching: expect the Omarchy logo, the boot menu, then the desktop on the built-in display at its native resolution. Touch ID is not supported on M3 yet." + elif [[ $M3_MODE == kernel ]] && ((M1N1_KEEP)); then + say "Done. Reboot: this Mac keeps the boot loader it has now, with this release's kernel. + Touch ID is not supported on M3 yet." elif [[ $M3_MODE == kernel ]]; then say "Done. Reboot: expect the desktop on the boot framebuffer. Touch ID is not supported on M3 yet. How to help bring this M3 further: step 11 of the test plan that the @@ -1557,6 +1665,9 @@ install_all() { else say "Done. Reboot, then run: aurora-touchid-setup" fi + if ((M1N1_KEEP)) && ! is_m3; then + echo " This Mac keeps the boot loader it has: the m1n1 that failed on it is not put back." + fi echo " Testing this build? The plan and reporting format:" echo " curl -fsSL $RELEASE_URL/install-aurora-sep.sh | bash -s -- --agent-prompt" if [[ $chain == grub ]]; then @@ -1640,7 +1751,16 @@ uninstall_all() { handoff) m3_restore_bringup ;; esac $sudo rm -f "$MODPROBE_CONF" + # The m1n1 builds that failed on this Mac stay recorded, so a later install + # never puts one of them back. + local failed="" + if [[ -f $STATE/m1n1-failed ]]; then failed=$(cat "$STATE/m1n1-failed"); fi $sudo rm -rf "$STATE" + if [[ -n $failed ]]; then + $sudo install -d "$STATE" + printf '%s\n' "$failed" | $sudo tee "$STATE/m1n1-failed" >/dev/null + say "Kept $STATE/m1n1-failed: a later install never puts back an m1n1 that failed on this Mac" + fi say "Done. Reboot to run $previous." } @@ -1842,6 +1962,23 @@ SAFETY, NON-NEGOTIABLE it in writing: repeated cycles drift a device-wide counter. - Do not paste key material, serial numbers, or the contents of mesa_calibration.bin into a report. + - The install replaces the Mac's boot loader (m1n1), on every Mac but an + M3 that stays kernel-only and a MacBook Neo. Before it does, it keeps + the old one on the EFI partition and prints the steps to put it back: + keep them, they name this Mac's EFI partition. + +IF THE MAC STOPS IN m1n1 AFTER AN INSTALL (any Mac) + m1n1 text on screen and no boot menu, or a black screen for more than two + minutes: the owner puts the old boot.bin back from macOS with the printed + steps, starts Linux, and runs the two printed lines. The first keeps + updates from rebuilding the new boot loader; the second records that its + m1n1 failed on this Mac. Later plain runs of the one-liner then keep the + boot loader the Mac has and install the rest, and never put that m1n1 + back. If it boots but something is wrong, the same copy goes back from + Linux with the printed line, followed by the same two lines. Do not use + --uninstall until the maintainer says so. Report what the screen showed + and when, with the serial log if there is one, at + https://github.com/iconidentify/aurora-linux/issues/6 ON AN M3 (M3, M3 Pro, M3 Max): Touch ID is not supported there yet. Do steps 0 and 1, then go to step 11, which says what should happen on your model and @@ -2090,6 +2227,11 @@ fingerprint. time and then HDMI with one USB-C display, and say which port, which display and which mode lit. Sleep has known limits on M3; if you try it with an external display attached, quote what happened. + If it STOPS (m1n1 text and no boot menu, or a black screen for more + than two minutes), follow "IF THE MAC STOPS IN m1n1" above. If it + boots but the display or the GPU is WRONG, collect + sudo journalctl -b -k | grep -iE 'dcp|asahi|t6030|m1n1' + first, then follow the same section from Linux. B. any other apple,t6030 model (MacBook Pro 14" M3 Pro, J514S): NOT ON THE LIST YET. A REPORT FROM YOU IS HOW IT GETS ADDED. @@ -2117,12 +2259,12 @@ fingerprint. adds the model to the list in the next release; until then a plain re-run of the one-liner keeps the handoff on this Mac. * STOPS: m1n1 text and no boot menu, or a black screen for more than - two minutes. The owner puts the old boot.bin back from macOS with - the printed steps. Report what the screen showed and when. + two minutes. Follow "IF THE MAC STOPS IN m1n1" above, and report + what the screen showed and when. * WRONG: it boots, but the display or the GPU is wrong. Collect sudo journalctl -b -k | grep -iE 'dcp|asahi|t6030|m1n1' - over SSH if you can, then run the installer with --uninstall - (it puts the stock m1n1 back) and reboot. + over SSH if you can, then follow the same section from Linux, + reboot, and report. C. apple,t8122 other than the MacBook Air (MacBook Pro 14" M3, J504; iMac M3, J433/J434) or apple,t6031 / apple,t6034 (M3 Max): KERNEL @@ -2191,17 +2333,13 @@ fingerprint. baseline checks below again too. Title it " (): M3 Air display handoff". * STOPS: m1n1 text and no boot menu, or a black screen for more - than two minutes. The owner puts the old boot.bin back from - macOS with the printed steps, starts Linux, and runs the printed - M1N1_UPDATE_DISABLED line so no update rebuilds the new boot - loader. Do not run the one-liner again or --uninstall until the - maintainer says so. Report what the screen showed and when, with - the serial log if there is one: it says where m1n1 stopped. + than two minutes. Follow "IF THE MAC STOPS IN m1n1" above. + Report what the screen showed and when, with the serial log if + there is one: it says where m1n1 stopped. * WRONG: it boots, but something from the baseline no longer works (keyboard, Wi-Fi, suspend). Collect sudo journalctl -b -k | grep -iE 'asahi|gpu|g15|t8122|dcp|m1n1' - then run the two printed lines from Linux (put the old boot.bin - back, then M1N1_UPDATE_DISABLED), reboot, and report. + then follow the same section from Linux, reboot, and report. - Later plain runs keep the display handoff on this Air. To go back, use the printed restore lines; ask the maintainer before using --uninstall on an Air. diff --git a/tools/aurora-sep/tests/test_m1n1_rollback.py b/tools/aurora-sep/tests/test_m1n1_rollback.py new file mode 100644 index 0000000000000..c94ab2f7303b1 --- /dev/null +++ b/tools/aurora-sep/tests/test_m1n1_rollback.py @@ -0,0 +1,198 @@ +"""A boot loader that failed is never put back (the restore loop). + +After an install, a Mac that stops in m1n1 gets its old boot.bin back from +macOS with the printed steps, then two printed lines: a freeze on +update-m1n1, and a line that records the failed m1n1 in $STATE/m1n1-failed. +The installer also records it when it finds boot.bin put back by hand. Later +runs keep the boot loader the Mac has, and never install that m1n1 again. +Runs against the fake Mac of test_m3_flow. +""" +import unittest + +import test_m3_flow as flow + +FREEZE = "M1N1_UPDATE_DISABLED=1\n" + + +class RollbackTest(flow.M3FlowBase): + def setUp(self): + super().setUp() + self.sha = self.bin_shas[self.m1n1_pkg] + + def m1n1_downloads(self, since=""): + return [l for l in self.log()[len(since):].splitlines() if l.startswith("curl ") and "/m1n1-" in l] + + def failed(self): + path = self.state / "m1n1-failed" + return [l.split()[0] for l in path.read_text().splitlines()] if path.exists() else [] + + def put_back(self, freeze=True): + # The printed macOS steps, then the printed freeze line. + self.boot.write_bytes(self.kept_copy().read_bytes()) + if freeze: + with open(self.update_conf, "a") as f: + f.write(FREEZE) + + def test_restore_steps_name_the_m1n1_and_the_issue(self): + for board, try_ in (("j516s", 0), ("j314s", 0), ("j514s", 1)): + with self.subTest(board=board): + self.fresh_state() + self.mac(board) + err = self.install(try_=try_).stderr + self.assertIn(f"echo {self.sha} | sudo tee -a {self.state}/m1n1-failed", err) + self.assertIn(f"echo M1N1_UPDATE_DISABLED=1 | sudo tee -a {self.update_conf}", err) + self.assertIn(f"sudo cp '{self.kept_copy()}' '{self.boot}.new' && sync && " + f"sudo mv '{self.boot}.new' '{self.boot}' && sync", err) + self.assertIn("https://github.com/iconidentify/aurora-linux/issues/6", err) + + def test_j516s_put_back_and_frozen_keeps_its_boot_loader(self): + # The review's loop: the J516S stops, the owner restores the kept copy + # and freezes updates. No later run puts that m1n1 back. + self.mac("j516s", bootbin=b"M1N1:aurora7\nDTBS:x\nUBOOT" + flow.SWITCHES) + self.install() + self.put_back() + restored = self.boot.read_bytes() + log = self.log() + proc = self.install() + self.assertIn("failed on this Mac before", proc.stdout) + self.assertIn("issues/6", proc.stdout) + self.assertNotIn("Remove that line", proc.stdout + proc.stderr) + self.assertIn("keeps the boot loader it has now", proc.stdout) + self.assertEqual(self.boot.read_bytes(), restored) + self.assertEqual(self.m1n1_downloads(log), []) + self.assertEqual(self.failed(), [self.sha]) + self.assertEqual((self.state / "m3-mode").read_text().strip(), "kernel") + + # The owner lifts the freeze anyway: still kept, and frozen again. + self.update_conf.write_text(self.update_conf.read_text().replace(FREEZE, "")) + log = self.log() + self.install() + self.assertEqual(self.boot.read_bytes(), restored) + self.assertEqual(self.m1n1_downloads(log), []) + self.assertIn("M1N1_UPDATE_DISABLED=1", self.update_conf.read_text()) + + # Asking for the handoff again does not bring that m1n1 back. + log = self.log() + proc = self.install(try_=1, check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("is the one that failed", proc.stderr) + self.assertIn("without --m3-handoff", proc.stderr) + self.assertIn("Nothing was installed", proc.stderr) + self.assertEqual(self.boot.read_bytes(), restored) + self.assertNotIn("curl ", self.log()[len(log):]) + + # --uninstall keeps the record, and a new install keeps the stock boot.bin. + self.uninstall() + self.assertEqual(self.failed(), [self.sha]) + stock = self.boot.read_bytes() + log = self.log() + self.install() + self.assertEqual(self.boot.read_bytes(), stock) + self.assertEqual(self.m1n1_downloads(log), []) + + def test_the_printed_line_alone_records_it(self): + # WRONG from Linux: the owner runs the printed lines; nothing to detect. + self.mac("j516s") + self.install() + self.put_back() + with open(self.state / "m1n1-failed", "a") as f: + f.write(self.sha + "\n") + restored = self.boot.read_bytes() + proc = self.install() + self.assertIn("failed on this Mac before", proc.stdout) + self.assertEqual(self.boot.read_bytes(), restored) + self.assertEqual(self.failed(), [self.sha]) + + def test_put_back_before_12_0_kept_by_its_freeze(self): + # An earlier release recorded no m1n1: the freeze from the restore + # steps alone keeps the boot loader, without "remove that line". + self.mac("j516s") + self.install() + (self.state / "m1n1-installed").unlink() + self.put_back() + restored = self.boot.read_bytes() + proc = self.install() + self.assertIn("not by this script (the restore steps add that line)", proc.stdout) + self.assertNotIn("Remove that line", proc.stdout + proc.stderr) + self.assertEqual(self.boot.read_bytes(), restored) + self.assertEqual(self.failed(), []) + + def test_m1_put_back_keeps_its_boot_loader(self): + # M1 and M2 get the same m1n1, so the same loop is closed there. + self.mac("j314s") + self.install() + self.put_back() + restored = self.boot.read_bytes() + log = self.log() + proc = self.install() + self.assertIn("failed on this Mac before", proc.stdout) + self.assertIn("boot.bin is unchanged", proc.stdout) + self.assertIn("keeps the boot loader it has", proc.stdout) + self.assertIn("run: aurora-touchid-setup", proc.stdout) + self.assertEqual(self.boot.read_bytes(), restored) + self.assertEqual(self.m1n1_downloads(log), []) + self.assertIn("linux-aurora", self.log()[len(log):]) + self.assertEqual(self.failed(), [self.sha]) + + def test_m1_put_back_without_the_freeze_stops_first(self): + self.mac("j314s") + self.install() + self.put_back(freeze=False) + restored = self.boot.read_bytes() + log = self.log() + proc = self.install(check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("Nothing was installed", proc.stderr) + self.assertIn(f"echo M1N1_UPDATE_DISABLED=1 | sudo tee -a {self.update_conf}", proc.stderr) + self.assertEqual(self.boot.read_bytes(), restored) + self.assertNotIn("pacman -U", self.log()[len(log):]) + self.assertEqual(self.failed(), [self.sha]) + + def test_rebuilds_are_not_taken_for_a_put_back(self): + # A re-run, and a newer m1n1 that the hook built before its check + # ran, leave boot.bin starting with an m1n1 this script installed. + self.mac("j314s") + self.install() + self.install() + self.assertEqual(self.failed(), []) + newer = "m1n1-aurora-1.6.1.aurora13-1-aarch64.pkg.tar.zst" + self.fixture(newer, None, flow.SWITCH_NAMES) + self.run_sh(f"pacman -U '{self.tmp / 'pkgs' / newer}'\nm1n1_rollback_check") + self.assertTrue(self.boot.read_bytes().startswith(b"M1N1:m1n1-aurora-1.6.1.aurora13-1\n")) + self.assertEqual(self.failed(), []) + + def test_a_newer_m1n1_after_a_failed_one_needs_the_flag(self): + self.mac("j516s") + self.install() + self.put_back() + self.install() + self.assertEqual(self.failed(), [self.sha]) + # The next release ships another m1n1. + newer = "m1n1-aurora-1.6.1.aurora13-1-aarch64.pkg.tar.zst" + self.fixture(newer, None, flow.SWITCH_NAMES) + self.m1n1_pkg = newer + restored = self.boot.read_bytes() + self.install() + self.assertEqual(self.boot.read_bytes(), restored) + # Asked for, it is installed once the owner lifts their freeze. + proc = self.install(try_=1, check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("Remove that line and run this again with --m3-handoff", proc.stderr) + self.update_conf.write_text(self.update_conf.read_text().replace(FREEZE, "")) + self.install(try_=1) + boot = self.boot.read_bytes() + self.assertTrue(boot.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora13-1\n"), boot[:80]) + self.assertTrue(boot.endswith(flow.SWITCHES)) + + def test_agent_prompt_steps_match_the_printed_ones(self): + self.mac("j516s") + prompt = self.run_sh("agent_prompt").stdout + self.assertIn("IF THE MAC STOPS IN m1n1 AFTER AN INSTALL (any Mac)", prompt) + self.assertIn("runs the two printed lines", prompt) + self.assertEqual(prompt.lower().count('follow "if the mac stops in m1n1" above'), 3) + self.assertNotIn("it puts the stock m1n1 back", prompt) + self.assertNotIn("Do not run the one-liner again", prompt) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/aurora-sep/tests/test_m3_flow.py b/tools/aurora-sep/tests/test_m3_flow.py index adf3ef8e8b222..159ac7c00e2b7 100644 --- a/tools/aurora-sep/tests/test_m3_flow.py +++ b/tools/aurora-sep/tests/test_m3_flow.py @@ -239,6 +239,15 @@ def run_sh(self, body, check=True): self.assertEqual(proc.returncode, 0, proc.stderr + proc.stdout) return proc + def fresh_state(self): + # A Mac this script never ran on, for the next subtest. + shutil.rmtree(self.state) + self.state.mkdir() + self.m1n1_conf.unlink(missing_ok=True) + self.update_conf.unlink(missing_ok=True) + for kept in self.boot.parent.glob("boot.bin.*"): + kept.unlink() + def install(self, try_=0, check=True, env=""): return self.run_sh(f"{env}\nM3_TRY={try_}\ninstall_all", check=check) @@ -514,10 +523,7 @@ def test_every_mac_gets_the_same_m1n1(self): for board, try_, switches in [("j293", 0, b""), ("j314s", 0, b""), ("j414s", 0, b""), ("j516s", 0, SWITCHES), ("j514s", 1, SWITCHES)]: with self.subTest(board=board): - shutil.rmtree(self.state) - self.state.mkdir() - self.m1n1_conf.unlink(missing_ok=True) - self.update_conf.unlink(missing_ok=True) + self.fresh_state() self.mac(board) self.install(try_=try_) boot = self.boot.read_bytes() @@ -527,14 +533,6 @@ def test_every_mac_gets_the_same_m1n1(self): # The m1n1 by its bytes (M1N1_BIN_SHA), never by the version it reports - def fresh_state(self): - shutil.rmtree(self.state) - self.state.mkdir() - self.m1n1_conf.unlink(missing_ok=True) - self.update_conf.unlink(missing_ok=True) - for kept in self.boot.parent.glob("boot.bin.*"): - kept.unlink() - def test_m1n1_is_checked_and_recorded_by_its_bytes(self): sha = self.bin_shas[M1N1_PKG] size = len((self.tmp / ("root-" + M1N1_PKG) / "usr/lib/asahi-boot/m1n1.bin").read_bytes()) From 5a1831452140a3e7f69c796269a15568a731e003 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:53:03 -0700 Subject: [PATCH 1761/2680] tools/aurora-sep: an M3 Pro off the list asks again for a newer m1n1 Every M3 Pro recorded the variant t6030, so a J514S that took --m3-handoff on 11.37, 11.38 or 11.109-test kept the handoff on a plain run and was moved to this release's m1n1 with no question asked. That m1n1 hands over the second external display processor and reads the display facts, neither of which has run on a J514S. M3 Pros now record M3_PRO_VARIANT (t6030-12), named after the m1n1 generation, as the Air's variant already is. A plain run keeps the handoff on a Pro that is not in M3_HANDOFF_BOARDS only while its recorded variant matches. Otherwise it stops before downloading anything, says that this Mac keeps its boot loader and kernel, and gives the --m3-handoff one-liner to switch. A Pro on the list (the J516S) moves with the release, as before. The Air's display variant becomes air-display-handoff-12 for the same m1n1; air-display-handoff-85 was never released. Case B of the agent prompt says a newer m1n1 asks again. Tests: a J514S recorded as t6030 or with no variant stops with nothing downloaded, then moves with --m3-handoff and keeps it on plain runs; a J516S recorded as t6030 moves; the shipped variant's shape. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 7f4f19645599084fa8e6d31a3d01d432476f9890) --- tools/aurora-sep/install-aurora-sep.sh | 25 +++++++++++-- tools/aurora-sep/tests/test_m3_air.py | 2 +- tools/aurora-sep/tests/test_m3_flow.py | 49 ++++++++++++++++++++++++-- 3 files changed, 69 insertions(+), 7 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index e09081bdc7874..26076a72367a6 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -335,6 +335,12 @@ M3_BOOTBIN_SHA="" # by default; no Air is listed yet. M3_HANDOFF_BOARDS="j516s" M3_SWITCHES="chosen.asahi,t6030-gpu=1 chosen.asahi,t6030-dcp=1 chosen.asahi,t6030-dcpext=1" +# The M3 Pro handoff's name in $STATE/m3-mode. A plain run keeps the handoff on +# an M3 Pro that is not in M3_HANDOFF_BOARDS only while this matches, so it +# changes with each m1n1 that changes what a Pro boots: a J514S that opted in +# on an earlier release (t6030: aurora7 or 8.x) moves to this release's m1n1 +# only with a new --m3-handoff. A listed Pro moves with the release. +M3_PRO_VARIANT="t6030-12" # The M3 MacBook Air's handoff switches, for a later build that starts the # Air's GPU (M3_AIR_DRY_RUN=0): the GPU alone, or with M3_AIR_DCP=1 the # display too, once m1n1's T8122 DCP handoff has been booted on an Air. @@ -359,7 +365,7 @@ M3_AIR_DCP=0 # (air-display-handoff, aurora8.3 and 8.4) moves to this one only with a new # --m3-handoff. M3_AIR_DISPLAY_HANDOFF=1 -M3_AIR_DISPLAY_VARIANT="air-display-handoff-85" +M3_AIR_DISPLAY_VARIANT="air-display-handoff-12" M3_AIR_DRY_RUN=1 M3_AIR_DRY_RUN_SWITCHES="chosen.asahi,t8122-gpu-diag=1 chosen.asahi,t8122-gpu-handoff-diag=1 chosen.asahi,t8122-gpu-power-diag=1 chosen.asahi,t8122-dcp=1" # The handoff is tested with one macOS system-firmware stub only, 14.8.3 (GPU @@ -816,7 +822,7 @@ m3_recorded_mode() { # so a later release never turns one kind of Air test into another (a dry run # into a GPU start) without a new --m3-handoff. m3_variant() { - if ! is_m3_air; then echo t6030 + if ! is_m3_air; then echo "$M3_PRO_VARIANT" elif [[ $M3_AIR_DISPLAY_HANDOFF == 1 ]]; then echo "$M3_AIR_DISPLAY_VARIANT" elif [[ $M3_AIR_DRY_RUN == 1 ]]; then echo air-dry-run elif [[ $M3_AIR_DCP == 1 ]]; then echo air-gpu-dcp @@ -833,7 +839,7 @@ m3_recorded_variant() { # Decide the M3 path before anything is downloaded, so a Mac this release can't # set up as asked stops with nothing changed. m3_plan() { - local board problem failed again="run this again" air=0 + local board problem failed variant again="run this again" air=0 M3_MODE=none if ! is_m3; then ((M3_TRY == 0)) || die "--m3-handoff is for an M3 Pro or an M3 MacBook Air, and this Mac isn't an M3. Nothing was installed." @@ -878,6 +884,17 @@ m3_plan() { $(m3_handoff_name) ($(m3_variant)). Run this again with --m3-handoff to switch to it. Nothing was installed." fi + # The same for an M3 Pro that is not on the list: it opted in to an + # earlier m1n1, not to this one. + variant=$(m3_recorded_variant) + if is_m3_pro && ! is_m3_handoff_board && [[ $variant != "$(m3_variant)" ]]; then + die "M3 Pro ($board): this Mac has m1n1's display and GPU handoff from an earlier release + (${variant:-no variant recorded}), and this release's m1n1 is a newer one ($(m3_variant)) that + nobody has booted on this model yet. Nothing was installed; this Mac keeps the boot loader + and kernel it has. To switch to the new m1n1 (it replaces this Mac's boot loader, and the + steps to put the old one back follow), run this again with --m3-handoff: + curl -fsSL $LATEST_URL | bash -s -- --m3-handoff" + fi M3_TRY=1 if ((air)); then say "M3 MacBook Air ($board): this Mac has m1n1's $(m3_handoff_name) from an earlier install; keeping it" @@ -2258,6 +2275,8 @@ fingerprint. it " (): M3 handoff works". This report is what adds the model to the list in the next release; until then a plain re-run of the one-liner keeps the handoff on this Mac. + A release with a newer m1n1 asks again: its plain run stops + with "run this again with --m3-handoff" and changes nothing. * STOPS: m1n1 text and no boot menu, or a black screen for more than two minutes. Follow "IF THE MAC STOPS IN m1n1" above, and report what the screen showed and when. diff --git a/tools/aurora-sep/tests/test_m3_air.py b/tools/aurora-sep/tests/test_m3_air.py index ff839b664772e..5152b01eafc3c 100644 --- a/tools/aurora-sep/tests/test_m3_air.py +++ b/tools/aurora-sep/tests/test_m3_air.py @@ -464,7 +464,7 @@ def test_m3_pro_old_record_still_kept(self): (self.state / "m3-mode").write_text("handoff\n") out = self.air_install().stdout self.assertIn("keeping it", out) - self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff t6030") + self.assertEqual((self.state / "m3-mode").read_text().strip(), f"handoff {flow.PRO_VARIANT}") def test_plain_rerun_updates_the_dry_run_m1n1(self): # Scott's path: an Air on 11.110-test's dry run takes a newer dry-run diff --git a/tools/aurora-sep/tests/test_m3_flow.py b/tools/aurora-sep/tests/test_m3_flow.py index 159ac7c00e2b7..73a5a7cdc30da 100644 --- a/tools/aurora-sep/tests/test_m3_flow.py +++ b/tools/aurora-sep/tests/test_m3_flow.py @@ -25,6 +25,8 @@ # The one m1n1 package every Mac gets, by the name this release ships. M1N1_PKG = re.search(r'^M1N1_PACKAGE="(\S+) ', SRC, re.M).group(1) M1N1_BASE = M1N1_PKG[:-len("-aarch64.pkg.tar.zst")] +# What $STATE/m3-mode records for an M3 Pro on this release's m1n1. +PRO_VARIANT = re.search(r'^M3_PRO_VARIANT="([^"]*)"$', SRC, re.M).group(1) # Every switch name the shipped m1n1 has to know: the M3 Pro's and the Air's. SWITCH_NAMES = sorted({w[len("chosen."):].split("=")[0] for w in re.findall(r"chosen\.asahi,t(?:6030|8122)-[a-z0-9-]+=1", SRC)}) @@ -277,7 +279,7 @@ def test_fresh_m3_pro_gets_the_handoff(self): self.assertTrue(boot.endswith(SWITCHES), boot[-200:]) self.assertEqual([d for d in self.downloaded() if d.startswith("m1n1-")], [M1N1_PKG]) self.assertEqual(self.kept_copy().read_bytes(), b"M1N1:original\n") - self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff t6030") + self.assertEqual((self.state / "m3-mode").read_text().strip(), f"handoff {PRO_VARIANT}") self.assertFalse(self.update_conf.read_text().count("M1N1_UPDATE_DISABLED")) self.uninstall() @@ -367,7 +369,7 @@ def test_handoff_mac_on_another_stage1_stops_with_nothing_installed(self): # record) keeps its boot.bin as it is: nothing is downloaded or rebuilt. self.mac("j516s") self.install() - self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff t6030") + self.assertEqual((self.state / "m3-mode").read_text().strip(), f"handoff {PRO_VARIANT}") self.mac("j516s", kernel="linux-aurora", bootbin=self.boot.read_bytes(), stage1="v1.7.0") before = self.boot.read_bytes() before_log = self.log() @@ -424,13 +426,54 @@ def test_rerun_without_the_flag_keeps_a_tried_handoff(self): self.install(try_=1) out = self.install().stdout self.assertIn("keeping it", out) - self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff t6030") + self.assertEqual((self.state / "m3-mode").read_text().strip(), f"handoff {PRO_VARIANT}") self.assertTrue(self.boot.read_bytes().endswith(SWITCHES)) self.assertNotIn("M1N1_UPDATE_DISABLED", self.update_conf.read_text()) self.uninstall() self.assertTrue(self.boot.read_bytes().startswith(b"M1N1:m1n1-stock\n")) self.assertFalse(self.m1n1_conf.exists()) + def test_m3_pro_variant_shipped(self): + # A release decision: change it with each m1n1 that changes what an + # M3 Pro boots. 11.36.1 to 11.38 recorded t6030 (or nothing). + self.assertRegex(PRO_VARIANT, r"^t6030-\S+$") + + def test_unlisted_pro_on_an_earlier_m1n1_needs_a_new_opt_in(self): + # A J514S that took --m3-handoff on 11.37 or 11.38 runs this + # release's plain one-liner: nothing moves it to the new m1n1. + for recorded in ("handoff t6030", "handoff"): + with self.subTest(recorded=recorded): + self.fresh_state() + self.mac("j514s") + self.install(try_=1) + (self.state / "m3-mode").write_text(recorded + "\n") + (self.state / "m1n1-installed").unlink() + before = self.boot.read_bytes() + before_conf = self.m1n1_conf.read_bytes() + log = self.log() + proc = self.install(check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("nobody has booted on this model yet", " ".join(proc.stderr.split())) + self.assertIn("bash -s -- --m3-handoff", proc.stderr) + self.assertIn("Nothing was installed", proc.stderr) + self.assertNotIn("keeping it", proc.stdout) + self.assertNotIn("--m3-handoff: trying", proc.stderr) + self.assertEqual(self.boot.read_bytes(), before) + self.assertEqual(self.m1n1_conf.read_bytes(), before_conf) + self.assertNotIn("curl ", self.log()[len(log):]) + # Asked for, it moves; later plain runs keep it. + self.install(try_=1) + self.assertEqual((self.state / "m3-mode").read_text().strip(), f"handoff {PRO_VARIANT}") + self.assertIn("keeping it", self.install().stdout) + + def test_listed_pro_on_an_earlier_m1n1_moves_with_the_release(self): + self.mac("j516s") + self.install() + (self.state / "m3-mode").write_text("handoff t6030\n") + self.install() + self.assertEqual((self.state / "m3-mode").read_text().strip(), f"handoff {PRO_VARIANT}") + self.assertTrue(self.boot.read_bytes().endswith(SWITCHES)) + def test_failed_install_then_uninstall_keeps_the_bringup(self): handbuilt = b"M1N1:bringup\nDTBS:hand\nUBOOT" + SWITCHES self.mac("j516s", m1n1="m1n1-bringup", bootbin=handbuilt) From f8b1010910f92c8836f0cf53751f849053d13120 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:55:31 -0700 Subject: [PATCH 1762/2680] tools/aurora-sep: say what a Mac with the handoff keeps when a check fails On a Mac that has the handoff from an earlier install, a plain run printed "...from an earlier install; keeping it", then stopped on the stub or stage 1 check with "error: --m3-handoff: ...", naming a flag the owner never typed, and with no next step. "keeping it" now prints only once every check has passed. When the stub or the stage 1 check fails on such a Mac, the error says that it has the handoff from an earlier install, what this release's m1n1 is checked with and how this Mac differs, that nothing was installed and the Mac keeps the boot loader and kernel it has, and asks for a report on issue #6 with the --m3-report one-liner. A kept handoff also no longer prints the "--m3-handoff: trying" warning of a first opt-in. --m3-handoff typed by the owner keeps its own messages. The custom update-m1n1 refusal says to run again with --m3-handoff when that is how the run started. Tests: a J516S with the handoff whose stage 1 or stub changed, and an Air on this release's display handoff whose stage 1 changed, stop with nothing downloaded and that wording; a kept J514S and a kept Air print "keeping it" without the opt-in warning. All five fail on the previous commit. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit aa7fce74dd0707c19ba4f39391f47b12aeb7f321) --- tools/aurora-sep/install-aurora-sep.sh | 35 +++++++++++++++++++------- tools/aurora-sep/tests/test_m3_air.py | 27 ++++++++++++++++++++ tools/aurora-sep/tests/test_m3_flow.py | 30 +++++++++++++++++++--- 3 files changed, 80 insertions(+), 12 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 26076a72367a6..1d5d0059724f9 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -836,10 +836,20 @@ m3_recorded_variant() { echo "$variant" } +# A Mac that has the handoff from an earlier install, on which this release's +# m1n1 is not checked ($1, and how this Mac differs, $2): it keeps what it has. +m3_kept_refusal() { + die "M3 ($(this_board)): this Mac has m1n1's $(m3_handoff_name) from an earlier install, and + this release's m1n1 is only checked with $1, and this Mac differs: $2. + Nothing was installed; this Mac keeps the boot loader and kernel it has. Please report it + at $ISSUE_URL with the file that this writes: + curl -fsSL $LATEST_URL | bash -s -- --m3-report" +} + # Decide the M3 path before anything is downloaded, so a Mac this release can't # set up as asked stops with nothing changed. m3_plan() { - local board problem failed variant again="run this again" air=0 + local board problem failed variant again="run this again" air=0 kept=0 M3_MODE=none if ! is_m3; then ((M3_TRY == 0)) || die "--m3-handoff is for an M3 Pro or an M3 MacBook Air, and this Mac isn't an M3. Nothing was installed." @@ -895,12 +905,9 @@ m3_plan() { steps to put the old one back follow), run this again with --m3-handoff: curl -fsSL $LATEST_URL | bash -s -- --m3-handoff" fi - M3_TRY=1 - if ((air)); then - say "M3 MacBook Air ($board): this Mac has m1n1's $(m3_handoff_name) from an earlier install; keeping it" - else - say "M3 ($board): this Mac has m1n1's display and GPU handoff from an earlier install; keeping it" - fi + # Kept from the record, not asked for: say so only once every check + # below has passed, and refuse in words that name no flag. + M3_TRY=1 kept=1 fi M3_MODE=kernel if ((air)); then @@ -926,6 +933,7 @@ m3_plan() { fi problem=$(m3_stub_problem) if [[ -n $problem ]]; then + if ((kept)); then m3_kept_refusal "the macOS $M3_STUB_VERSION system-firmware stub" "$problem"; fi ((M3_TRY == 0)) || die "--m3-handoff: the handoff is only tested with the macOS $M3_STUB_VERSION system-firmware stub, and this Mac differs: $problem. Nothing was installed." warn "the $(m3_handoff_name) is only tested with the macOS $M3_STUB_VERSION @@ -935,6 +943,7 @@ m3_plan() { fi problem=$(m3_stage1_problem) if [[ -n $problem ]]; then + if ((kept)); then m3_kept_refusal "m1n1 stage 1 $M3_STAGE1_VERSIONS" "$problem"; fi ((M3_TRY == 0)) || die "--m3-handoff: this release's M3 m1n1 is only checked with m1n1 stage 1 $M3_STAGE1_VERSIONS, and this Mac differs: $problem. Nothing was installed." warn "the $(m3_handoff_name) is only checked with m1n1 stage 1 $M3_STAGE1_VERSIONS, and @@ -950,12 +959,20 @@ m3_plan() { if update_m1n1_customised; then die "$UPDATE_M1N1_CONF points update-m1n1 at its own m1n1, U-Boot, config or target (M1N1=, SOURCE=, U_BOOT=, CONFIG= or TARGET=). The $(m3_handoff_name) needs boot.bin - built from this release's m1n1. Remove those lines and run this again. Nothing was installed." + built from this release's m1n1. Remove those lines and $again. Nothing was installed." fi M3_MODE=handoff + if ((kept && air)); then + say "M3 MacBook Air ($board): this Mac has m1n1's $(m3_handoff_name) from an earlier install; keeping it" + elif ((kept)); then + say "M3 ($board): this Mac has m1n1's display and GPU handoff from an earlier install; keeping it" + fi if ((air)); then if is_m3_handoff_board; then say "M3 MacBook Air ($board, macOS $M3_STUB_VERSION stub): installing m1n1 with the $(m3_handoff_name)" + elif ((kept)); then + # Its owner asked with --m3-handoff when it was installed. + : elif [[ $M3_AIR_DISPLAY_HANDOFF == 1 ]]; then warn "--m3-handoff: trying m1n1's display handoff on this M3 MacBook Air ($board). It publishes the checked display state for Linux and collects GPU diagnostics; @@ -976,7 +993,7 @@ m3_plan() { fi elif is_m3_handoff_board; then say "M3 Pro ($board, macOS $M3_STUB_VERSION stub): installing m1n1 with the display and GPU handoff" - else + elif ((!kept)); then warn "--m3-handoff: trying m1n1's display and GPU handoff on an M3 Pro model ($board) nobody has booted it on yet. This replaces the Mac's boot loader; the steps to put the old one back from macOS follow." diff --git a/tools/aurora-sep/tests/test_m3_air.py b/tools/aurora-sep/tests/test_m3_air.py index 5152b01eafc3c..8c12d16f4ed89 100644 --- a/tools/aurora-sep/tests/test_m3_air.py +++ b/tools/aurora-sep/tests/test_m3_air.py @@ -563,6 +563,33 @@ def test_display_handoff_needs_the_checked_stage1(self): self.assertEqual(self.downloaded(), []) self.assertEqual(self.plan(display=1), "kernel") + def test_display_handoff_air_on_another_stage1_keeps_what_it_has(self): + # Scott's Air with this release's display handoff, after its stage 1 + # changed: a plain run stops before any download and names no flag. + self.mac("j613") + self.air_install(try_=1, display=1) + self.assertEqual((self.state / "m3-mode").read_text().strip(), f"handoff {DISPLAY_VARIANT}") + self.mac("j613", bootbin=self.boot.read_bytes(), stage1="v1.7.0") + before = self.boot.read_bytes() + before_downloads = self.downloaded() + proc = self.air_install(display=1, check=False) + self.assertNotEqual(proc.returncode, 0) + err = " ".join(proc.stderr.split()) + self.assertIn("stage 1 is v1.7.0", err) + self.assertIn("this Mac keeps the boot loader and kernel it has", err) + self.assertIn("issues/6", err) + self.assertNotIn("--m3-handoff", err) + self.assertNotIn("keeping it", proc.stdout) + self.assertEqual(self.boot.read_bytes(), before) + self.assertEqual(self.downloaded(), before_downloads) + + def test_kept_air_gets_no_new_opt_in_warning(self): + self.mac("j613") + self.air_install(try_=1, display=1) + proc = self.air_install(display=1) + self.assertIn("keeping it", proc.stdout) + self.assertNotIn("--m3-handoff: trying", proc.stderr) + def test_install_syncs_after_the_rebuild(self): src = SRC[SRC.index(" m3_verify_bootbin\n"):] self.assertTrue(re.match(r" m3_verify_bootbin\n m1n1_check_and_record\n(\s*#.*\n)*\s*sync\n", src), diff --git a/tools/aurora-sep/tests/test_m3_flow.py b/tools/aurora-sep/tests/test_m3_flow.py index 73a5a7cdc30da..c611ef39cc6ba 100644 --- a/tools/aurora-sep/tests/test_m3_flow.py +++ b/tools/aurora-sep/tests/test_m3_flow.py @@ -375,8 +375,30 @@ def test_handoff_mac_on_another_stage1_stops_with_nothing_installed(self): before_log = self.log() proc = self.install(check=False) self.assertNotEqual(proc.returncode, 0) - self.assertIn("stage 1 is v1.7.0", proc.stderr) - self.assertIn("Nothing was installed", proc.stderr) + self.assert_kept_refusal(proc, "stage 1 is v1.7.0") + self.assertEqual(self.boot.read_bytes(), before) + self.assertNotIn("curl ", self.log()[len(before_log):]) + + def assert_kept_refusal(self, proc, why): + # No "keeping it" before the error, no flag the owner did not type, + # and what to do next. + err = " ".join(proc.stderr.split()) + self.assertIn(why, err) + self.assertIn("Nothing was installed; this Mac keeps the boot loader and kernel it has", err) + self.assertIn("issues/6", err) + self.assertIn("bash -s -- --m3-report", err) + self.assertNotIn("--m3-handoff", err) + self.assertNotIn("keeping it", proc.stdout) + + def test_handoff_mac_on_another_stub_stops_with_nothing_installed(self): + self.mac("j516s") + self.install() + self.mac("j516s", kernel="linux-aurora", bootbin=self.boot.read_bytes(), stub="15.6") + before = self.boot.read_bytes() + before_log = self.log() + proc = self.install(check=False) + self.assertNotEqual(proc.returncode, 0) + self.assert_kept_refusal(proc, "stub is 15.6") self.assertEqual(self.boot.read_bytes(), before) self.assertNotIn("curl ", self.log()[len(before_log):]) @@ -424,8 +446,10 @@ def test_trial_refused_on_an_m3(self): def test_rerun_without_the_flag_keeps_a_tried_handoff(self): self.mac("j514s") self.install(try_=1) - out = self.install().stdout + proc = self.install() + out = proc.stdout self.assertIn("keeping it", out) + self.assertNotIn("--m3-handoff: trying", proc.stderr) self.assertEqual((self.state / "m3-mode").read_text().strip(), f"handoff {PRO_VARIANT}") self.assertTrue(self.boot.read_bytes().endswith(SWITCHES)) self.assertNotIn("M1N1_UPDATE_DISABLED", self.update_conf.read_text()) From e1ceeddc7113238e79752c26246aa3b475448a38 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:58:43 -0700 Subject: [PATCH 1763/2680] tools/aurora-sep: test the release guard, upgrades from 11.38 and --m3-report The review found the suite passing with placeholder checksums, with a fixture in place of the shipped m1n1 package, with one VERSION for every run, and without the --m3-report additions. test_release.py: - a release guard that fails while any package checksum, or M1N1_BIN_SHA, is not a sha256 (it fails now: the kernel packages and the aurora12 m1n1 are placeholders until their lab builds); - M1N1_PACKAGE's name: one m1n1 package, whose version sorts above every m1n1 an earlier release installed (aurora3, aurora7, 8.4, 8.5-2); - the package itself when it is at hand (AURORA_M1N1_PKG, or the release's staging directory): its sha256, its m1n1.bin's sha256 against M1N1_BIN_SHA, the version string it reports, every M3 switch name, and nothing in 0x120000-0x180000. Checked against the lab-built 8.5-2 package: it passes with that build's m1n1 sha256, and fails against a build that reports the same version with other bytes; - 11.38's own script (from git), then this one, on one fake Mac: a J516S moves and keeps 11.38's image as the restore copy next to the original's, and is not put back on it after a failed boot; a J514S that opted in on 11.38 stops; an M1 Pro gets the one m1n1 with 11.38's image kept; a Neo is unchanged. The fake Mac's harness can run another script first (self.installer). test_m3_report.py: the overlap marker present and absent, the dcp-oslog list, the /chosen entries copied, and the warning. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit b4d489edd8a8c07738b72c47f7aa71477dc61c1a) --- tools/aurora-sep/tests/test_m3_flow.py | 4 +- tools/aurora-sep/tests/test_m3_report.py | 29 ++++ tools/aurora-sep/tests/test_release.py | 197 +++++++++++++++++++++++ 3 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 tools/aurora-sep/tests/test_release.py diff --git a/tools/aurora-sep/tests/test_m3_flow.py b/tools/aurora-sep/tests/test_m3_flow.py index c611ef39cc6ba..ff73f0830d558 100644 --- a/tools/aurora-sep/tests/test_m3_flow.py +++ b/tools/aurora-sep/tests/test_m3_flow.py @@ -172,6 +172,8 @@ def setUp(self): self.fixture(AURORA6, None, ["asahi,t6030-gpu", "asahi,t6030-dcp", "asahi,t6030-dcpext"]) # The m1n1 package every Mac gets in this fake release. self.m1n1_pkg = M1N1_PKG + # The script under test; a test can run an earlier release's first. + self.installer = INSTALLER def fixture(self, name, data, strings=None): path = self.tmp / "pkgs" / name @@ -210,7 +212,7 @@ def run_sh(self, body, check=True): pkgs = "\n".join(f' "{n} {self.shas[n]}"' for n in OTHERS) script = f""" set -euo pipefail -AURORA_SEP_SOURCE_ONLY=1 source '{INSTALLER}' +AURORA_SEP_SOURCE_ONLY=1 source '{self.installer}' sudo="" DT='{self.tmp}/dt' STATE='{self.state}' diff --git a/tools/aurora-sep/tests/test_m3_report.py b/tools/aurora-sep/tests/test_m3_report.py index e36b10042035b..c3589c59acd58 100644 --- a/tools/aurora-sep/tests/test_m3_report.py +++ b/tools/aurora-sep/tests/test_m3_report.py @@ -70,6 +70,35 @@ def test_report_changes_nothing(self): self.assertNotIn("pacman -U", self.log()) self.assertNotIn("update-m1n1", self.log()) + def test_overlap_marker_and_display_logs(self): + # 12.0's additions: the marker's presence, the dcp-oslog list, the + # marker copied with the other /chosen entries, and the warning. + self.mac("j516s") + marker = self.tmp / "dt/chosen/asahi,m1n1-oslog-overlap" + marker.write_bytes(bytes(range(16))) + for addr in ("10000000000", "10000100000", "10000200000"): + (self.tmp / f"dt/reserved-memory/dcp-oslog@{addr}").mkdir(parents=True) + (self.tmp / "dt/chosen/asahi,t6030-display-facts").write_bytes(b"facts\0") + proc, files = self.report() + text = files["system.txt"].decode() + self.assertIn("m1n1-oslog-overlap: present", text) + listed = text.split("reserved display logs:\n", 1)[1].splitlines()[:3] + self.assertEqual([l.rsplit("/", 1)[1] for l in listed], + ["dcp-oslog@10000000000", "dcp-oslog@10000100000", "dcp-oslog@10000200000"]) + self.assertEqual(files["chosen/asahi,m1n1-oslog-overlap"], bytes(range(16))) + self.assertEqual(files["chosen/asahi,t6030-display-facts"], b"facts\0") + self.assertIn("display log buffer overlaps", proc.stderr) + self.assertIn("issues", proc.stderr) + + def test_no_overlap_marker(self): + self.mac("j613") + proc, files = self.report() + text = files["system.txt"].decode() + self.assertIn("m1n1-oslog-overlap: absent", text) + self.assertIn("reserved display logs:\n-\n", text) + self.assertNotIn("chosen/asahi,m1n1-oslog-overlap", files) + self.assertNotIn("overlaps", proc.stderr) + def test_report_names_the_m1n1_by_its_bytes(self): self.mac("j516s") self.install() diff --git a/tools/aurora-sep/tests/test_release.py b/tools/aurora-sep/tests/test_release.py new file mode 100644 index 0000000000000..0245c06e00778 --- /dev/null +++ b/tools/aurora-sep/tests/test_release.py @@ -0,0 +1,197 @@ +"""Release guards and upgrades from the previous release. + +ReleaseGuardTest fails while any package checksum is still a placeholder: +a release is cut only when it passes. RealM1n1PackageTest checks the +m1n1 package this release names, by its file name always, and by its +content when the file is at hand (AURORA_M1N1_PKG, or the release staging +directory). UpgradeTest runs 11.38's own script on the fake Mac of +test_m3_flow, then this one, as an owner updating would. +""" +from pathlib import Path +import hashlib +import os +import re +import shutil +import subprocess +import tarfile +import tempfile +import unittest + +import test_m3_flow as flow + +SRC = flow.SRC +VERSION = flow.VERSION +M1N1_PACKAGE = re.search(r'^M1N1_PACKAGE="(\S+) (\S+)"$', SRC, re.M) +M1N1_BIN_SHA = re.search(r"^M1N1_BIN_SHA=(\S+)$", SRC, re.M).group(1) +# Every m1n1 a release put on a Mac before this one. +EARLIER_M1N1 = ["1.6.1.aurora3-1", "1.6.1.aurora7-1", "1.6.1.aurora8.4-1", "1.6.1.aurora8.5-2"] + + +def package_entries(): + """Every "file sha256" this script downloads.""" + block = re.search(r"^PACKAGES=\(\n(.*?)^\)", SRC, re.M | re.S).group(1) + entries = re.findall(r'^\s*"(\S+) (\S+)"$', block, re.M) + entries.append((M1N1_PACKAGE.group(1), M1N1_PACKAGE.group(2))) + return entries + + +def staged_m1n1(): + name = M1N1_PACKAGE.group(1) + candidates = [os.environ.get("AURORA_M1N1_PKG", "")] + stage = Path.home() / "source/aurora-recipes" / f"stage-{VERSION.split('-')[-1]}" + candidates.append(str(stage / name)) + for c in candidates: + if c and Path(c).is_file() and Path(c).name == name: + return Path(c) + return None + + +class ReleaseGuardTest(unittest.TestCase): + def test_no_placeholder_checksums(self): + # PENDING-* stands in for a lab build's sha256 until it exists. The + # script refuses such a download, so nothing installs; this test is + # what stops the release from being cut with one. + pending = [f"{f} {sha}" for f, sha in package_entries() if not re.fullmatch(r"[0-9a-f]{64}", sha)] + if not re.fullmatch(r"[0-9a-f]{64}", M1N1_BIN_SHA): + pending.append(f"M1N1_BIN_SHA={M1N1_BIN_SHA}") + self.assertEqual(pending, [], "placeholders left in install-aurora-sep.sh") + + def test_packages_follow_version(self): + names = [f for f, _ in package_entries()] + self.assertIn(f"linux-aurora-$VERSION-aarch64.pkg.tar.zst", names) + self.assertIn(f"linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst", names) + self.assertEqual(len([n for n in names if n.startswith("m1n1-")]), 1, names) + + +class RealM1n1PackageTest(unittest.TestCase): + def test_name_and_version(self): + name = M1N1_PACKAGE.group(1) + m = re.fullmatch(r"m1n1-aurora-(1\.6\.1\.aurora[0-9.]+)-(\d+)-aarch64\.pkg\.tar\.zst", name) + self.assertTrue(m, name) + if not shutil.which("vercmp"): + self.skipTest("vercmp (pacman) is needed to order the versions") + new = f"{m.group(1)}-{m.group(2)}" + for old in EARLIER_M1N1: + with self.subTest(old=old): + out = subprocess.run(["vercmp", new, old], capture_output=True, text=True).stdout.strip() + self.assertEqual(out, "1", f"{new} must sort above {old}") + + def test_the_package_itself(self): + path = staged_m1n1() + if path is None: + self.skipTest(f"{M1N1_PACKAGE.group(1)} is not at hand (set AURORA_M1N1_PKG)") + self.assertEqual(hashlib.sha256(path.read_bytes()).hexdigest(), M1N1_PACKAGE.group(2)) + with tarfile.open(path) as t: + info = t.extractfile(".PKGINFO").read().decode() + m1n1 = t.extractfile("usr/lib/asahi-boot/m1n1.bin").read() + version = re.search(r"^pkgver = (\S+)$", info, re.M).group(1) + self.assertIn(f"-{version}-aarch64", M1N1_PACKAGE.group(1)) + self.assertIn("conflict = m1n1", info) + self.assertRegex(info, r"provides = m1n1=1\.6\.1") + # The bytes this release names, and the version they report. + self.assertEqual(hashlib.sha256(m1n1).hexdigest(), M1N1_BIN_SHA) + tag = "v1.6.1-omarchy." + version.split("-")[0][len("1.6.1."):] + self.assertIn(tag.encode() + b"\0", m1n1) + # Every switch an M3 Pro or Air arms, as whole strings. + strings = set(m1n1.split(b"\0")) + for name in flow.SWITCH_NAMES: + self.assertIn(name.encode(), strings, name) + # The file part stays clear of the M3 display log buffer. + self.assertFalse(any(m1n1[0x120000:0x180000])) + + +class UpgradeTest(flow.M3FlowBase): + """11.38's own script, then this one, on one fake Mac.""" + + OLD_REV = "d14f756c" + + def setUp(self): + super().setUp() + old = subprocess.run(["git", "show", f"{self.OLD_REV}:tools/aurora-sep/install-aurora-sep.sh"], + cwd=flow.INSTALLER.parent, capture_output=True) + if old.returncode: + self.skipTest(f"git can't show 11.38's script ({self.OLD_REV})") + self.old = self.tmp / "install-11.38.sh" + self.old.write_bytes(old.stdout) + self.old_version = re.search(rb"^VERSION=(\S+)$", old.stdout, re.M).group(1).decode() + # Stand-ins for 11.38's m1n1 packages: aurora3 for M1/M2, aurora7 for M3. + self.aurora3 = "m1n1-aurora-1.6.1.aurora3-1-aarch64.pkg.tar.zst" + self.aurora7 = "m1n1-aurora-1.6.1.aurora7-1-aarch64.pkg.tar.zst" + self.fixture(self.aurora3, None, []) + self.fixture(self.aurora7, None, ["asahi,t6030-gpu", "asahi,t6030-dcp", "asahi,t6030-dcpext"]) + + def install_1138(self, try_=0): + self.installer = self.old + try: + return self.run_sh(f'PACKAGES+=("{self.aurora3} {self.shas[self.aurora3]}")\n' + f'M3_M1N1_PACKAGE="{self.aurora7} {self.shas[self.aurora7]}"\n' + f"M3_TRY={try_}\ninstall_all") + finally: + self.installer = flow.INSTALLER + + def kept(self, version): + return self.boot.parent / f"boot.bin.before-{version}" + + def test_j516s_from_11_38(self): + self.mac("j516s") + self.install_1138() + image_1138 = self.boot.read_bytes() + self.assertTrue(image_1138.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora7-1\n")) + self.assertEqual((self.state / "m3-mode").read_text().strip(), "handoff t6030") + # The plain one-liner of this release moves a listed J516S. + self.install() + boot = self.boot.read_bytes() + self.assertTrue(boot.startswith(b"M1N1:" + flow.M1N1_BASE.encode() + b"\n"), boot[:80]) + self.assertTrue(boot.endswith(flow.SWITCHES)) + # Each release keeps what the Mac booted before it. + self.assertEqual(self.kept(self.old_version).read_bytes(), b"M1N1:original\n") + self.assertEqual(self.kept(VERSION).read_bytes(), image_1138) + self.assertEqual((self.state / "m3-mode").read_text().strip(), f"handoff {flow.PRO_VARIANT}") + # It stops; the owner puts 11.38's image back and freezes updates. + self.boot.write_bytes(self.kept(VERSION).read_bytes()) + with open(self.update_conf, "a") as f: + f.write("M1N1_UPDATE_DISABLED=1\n") + proc = self.install() + self.assertNotIn("Remove that line", proc.stdout + proc.stderr) + self.assertEqual(self.boot.read_bytes(), image_1138) + self.assertEqual((self.state / "m1n1-failed").read_text().split()[0], self.bin_shas[flow.M1N1_PKG]) + + def test_j514s_opted_in_on_11_38(self): + self.mac("j514s") + self.install_1138(try_=1) + before = self.boot.read_bytes() + log = self.log() + proc = self.install(check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("--m3-handoff", proc.stderr) + self.assertIn("Nothing was installed", proc.stderr) + self.assertEqual(self.boot.read_bytes(), before) + self.assertNotIn("curl ", self.log()[len(log):]) + + def test_m1_from_11_38(self): + self.mac("j314s") + self.install_1138() + image_1138 = self.boot.read_bytes() + self.assertTrue(image_1138.startswith(b"M1N1:m1n1-aurora-1.6.1.aurora3-1\n")) + self.assertFalse(self.kept(self.old_version).exists()) + proc = self.install() + self.assertIn("put the\n boot loader it booted with back from macOS", proc.stderr) + self.assertEqual(self.kept(VERSION).read_bytes(), image_1138) + self.assertTrue(self.boot.read_bytes().startswith(b"M1N1:" + flow.M1N1_BASE.encode() + b"\n")) + self.assertNotIn(b"chosen.", self.boot.read_bytes()) + + def test_neo_from_11_38(self): + own = self.tmp / "neo-m1n1.bin" + own.write_bytes(b"M1N1:neo-own\n") + self.mac("j700") + self.update_conf.write_text(f"M1N1={own}\nU_BOOT=/x\n") + self.install_1138() + image_1138 = self.boot.read_bytes() + self.install() + self.assertEqual(self.boot.read_bytes(), image_1138) + self.assertFalse(self.kept(VERSION).exists()) + self.assertNotIn("m1n1-aurora", " ".join(self.downloaded())) + + +if __name__ == "__main__": + unittest.main() From 98efc7b47e509f9081aaf8f45b6818172d1f98b2 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:00:06 -0700 Subject: [PATCH 1764/2680] tools/aurora-sep: keep m3_report_m1n1 apart from --m3-report's comment m3_report_m1n1 sat between the comment that describes --m3-report and m3_report itself; it now comes before it. The lookups in the list of failed m1n1 builds read it through a here-string instead of a pipe into grep -q, which pipefail could fail if grep stopped reading early. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit e124dc69f0bc9d49abddc50d72546bc5131a2ac7) --- tools/aurora-sep/install-aurora-sep.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 1d5d0059724f9..3779b95419dc1 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -629,7 +629,7 @@ m1n1_failed_shas() { } m1n1_failed_add() { - m1n1_failed_shas | grep -qxF "$1" && return 0 + grep -qxF "$1" <<<"$(m1n1_failed_shas)" && return 0 $sudo install -d "$STATE" echo "$1 $2" | $sudo tee -a "$STATE/m1n1-failed" >/dev/null } @@ -657,7 +657,7 @@ m1n1_rollback_check() { # must stay frozen, or pacman's hook would put that m1n1 back. m1n1_keep_plan() { if is_m3 || ! m1n1_for_this_mac; then return 0; fi - m1n1_failed_shas | grep -qxF "$M1N1_BIN_SHA" || return 0 + grep -qxF "$M1N1_BIN_SHA" <<<"$(m1n1_failed_shas)" || return 0 update_m1n1_frozen || die "this release's m1n1 (sha256 $M1N1_BIN_SHA) failed on this Mac before (recorded in $STATE/m1n1-failed), so this script does not put it back, and update-m1n1 must not either. @@ -1905,13 +1905,6 @@ reset_touchid() { say "Reboot now. Touch ID creates its new keybag during boot; then enrol your fingers again." } -# Printed by --agent-prompt, and pointed at from the end of a successful -# install. This is written for an agent driving the test on a real Mac: it says -# what to establish, what counts as a pass, and how to write it up. -# --m3-report: one file with what an M3 test report needs, in the current -# directory. It reads only: the boot loader's /chosen entries, a kernel log -# filtered to the M3 bring-up, USB-C and display state. Lines naming a USB -# serial number are dropped and MAC addresses are masked. # The m1n1 in boot.bin by its bytes, for --m3-report: two builds can report the # same stage 2 version. m3_report_m1n1() { @@ -1930,6 +1923,13 @@ m3_report_m1n1() { printf 'm1n1-installed: '; cat "$STATE/m1n1-installed" 2>/dev/null || echo - } +# Printed by --agent-prompt, and pointed at from the end of a successful +# install. This is written for an agent driving the test on a real Mac: it says +# what to establish, what counts as a pass, and how to write it up. +# --m3-report: one file with what an M3 test report needs, in the current +# directory. It reads only: the boot loader's /chosen entries, a kernel log +# filtered to the M3 bring-up, USB-C and display state. Lines naming a USB +# serial number are dropped and MAC addresses are masked. M3_REPORT_DMESG='asahi|agx|gpu|g15|dcp|dart|t8122|t6030|reserved|iommu|mailbox|pmp|simpledrm|m1n1|tipd|typec|sn201202|atc|usb|xhci|dwc3|thermal|macsmc' m3_report() { local dir out board soc f From a4eeb72aaeef42f43ff1efd15f30ac0a40050a21 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:17:54 -0700 Subject: [PATCH 1765/2680] tools/aurora-sep: fill in the 12.0 package checksums The kernel packages, linux-aurora and linux-aurora-headers 7.1.12.aurora2-12.0, are lab package build b1006-141119-f6a7 from 84aa4e9185c3, whose kernel tree is the same as this branch's. The m1n1, m1n1-aurora 1.6.1.aurora12-1, is lab build b1006-145229-de40. M1N1_BIN_SHA is the sha256 of the m1n1.bin in that package. The release guard test passes now that no checksum is a placeholder. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 4bdd1249a086945e7c7d595baaeae2a02c136b35) --- tools/aurora-sep/install-aurora-sep.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 3779b95419dc1..9356c9581cb73 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -166,20 +166,20 @@ RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst PENDING-12.0-LAB-BUILD" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst PENDING-12.0-LAB-BUILD" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst 7cd8b4b63ae672eaf80fc08fdeca89bf88ea3473fe35d27f1e7a7367c552a151" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst a12330c3871cf34cff2757e37db422feba9ecdeb570486a462ed6953e0f5e1d3" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20261003-1-any.pkg.tar.zst 29b0360fac8c257d754e64bd1b9c33c487eb2595dd3c31e9138d7a476afa3d64" ) # The one m1n1 for every Mac, as "file sha256": M1 and M2, an M3 on the handoff # path (see m3_plan), and the MacBook Neo once NEO_AURORA_M1N1 is 1. Macs differ # only in the switches /etc/m1n1.conf arms (m3_switches), never in the binary. -M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora12-1-aarch64.pkg.tar.zst PENDING-AURORA12-BUILD" +M1N1_PACKAGE="m1n1-aurora-1.6.1.aurora12-1-aarch64.pkg.tar.zst c3df0896ca925feb73ca4d9768596fe2b1cf3b8972f68866a79da8485444f2fd" # The sha256 of the m1n1.bin in M1N1_PACKAGE: the bytes update-m1n1 puts at # the start of boot.bin. The script tells m1n1 builds apart by these bytes, # never by the version string they report: aurora8.5-1 and 8.5-2 both reported # v1.6.1-omarchy.aurora8.5. -M1N1_BIN_SHA=PENDING-AURORA12-BUILD +M1N1_BIN_SHA=f727705e6b7b4906b4cbe016995a4561a6daa5c13a525e09910a6b3df4199820 # 0: a MacBook Neo keeps its own m1n1 (its M1N1= and U_BOOT= in # /etc/default/update-m1n1), as before 12.0. 1: it gets M1N1_PACKAGE like every # other Mac, and update-m1n1 builds its boot.bin from that m1n1 and the Neo's From 202bb47e4cd6d5f7da23062e54242fa73e27877b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:19:44 -0700 Subject: [PATCH 1766/2680] tools/aurora-sep: let a staging or mirror copy stand in for the release URL RELEASE_URL, where the packages are downloaded from, and RELEASES_API, where the newer-release notice looks, can now be set from AURORA_RELEASE_URL and AURORA_RELEASES_API. Unset or empty, they are the release's own tag and the GitHub API as before. Any URL curl takes works, file:// included. The checksums in the script still decide what is installed. test_release.py: the defaults, an empty value, an override of each, the notice read from a file:// copy, and install_all with the real curl from a file:// copy: it installs, a changed file stops it at the checksum, and a missing file stops it before anything is installed. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 4269a6425fcb5c9b53f6f562548fe741610d1cda) --- tools/aurora-sep/install-aurora-sep.sh | 7 +- tools/aurora-sep/tests/test_release.py | 89 +++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 3 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 9356c9581cb73..cc4238e40e311 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -161,8 +161,11 @@ VERSION=7.1.12.aurora2-12.0 TAG=sep-7.1.12.aurora2-12.0 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. -RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG -RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases +# AURORA_RELEASE_URL and AURORA_RELEASES_API are a staging or mirror override +# for these two (any URL curl takes, file:// included); the checksums below +# still decide what is installed. +RELEASE_URL=${AURORA_RELEASE_URL:-https://github.com/iconidentify/aurora-linux/releases/download/$TAG} +RELEASES_API=${AURORA_RELEASES_API:-https://api.github.com/repos/iconidentify/aurora-linux/releases} # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( diff --git a/tools/aurora-sep/tests/test_release.py b/tools/aurora-sep/tests/test_release.py index 0245c06e00778..e62c402d69b12 100644 --- a/tools/aurora-sep/tests/test_release.py +++ b/tools/aurora-sep/tests/test_release.py @@ -4,7 +4,10 @@ a release is cut only when it passes. RealM1n1PackageTest checks the m1n1 package this release names, by its file name always, and by its content when the file is at hand (AURORA_M1N1_PKG, or the release staging -directory). UpgradeTest runs 11.38's own script on the fake Mac of +directory). ReleaseUrlTest and StagedCopyTest cover the staging or mirror +override (AURORA_RELEASE_URL, AURORA_RELEASES_API): the default is the +release's own tag, and a staged copy installs through the same download and +checksum loop. UpgradeTest runs 11.38's own script on the fake Mac of test_m3_flow, then this one, as an owner updating would. """ from pathlib import Path @@ -100,6 +103,90 @@ def test_the_package_itself(self): self.assertFalse(any(m1n1[0x120000:0x180000])) +TAG = re.search(r"^TAG=(\S+)$", SRC, re.M).group(1) +DEFAULT_RELEASE_URL = f"https://github.com/iconidentify/aurora-linux/releases/download/{TAG}" +DEFAULT_RELEASES_API = "https://api.github.com/repos/iconidentify/aurora-linux/releases" +OVERRIDES = ("AURORA_RELEASE_URL", "AURORA_RELEASES_API") + + +def sourced(body, **env_extra): + """Runs body after sourcing the script, with only the overrides given.""" + env = {k: v for k, v in os.environ.items() if k not in OVERRIDES} + env.update(env_extra) + script = f"set -euo pipefail\nAURORA_SEP_SOURCE_ONLY=1 source '{flow.INSTALLER}'\n{body}" + return subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env) + + +class ReleaseUrlTest(unittest.TestCase): + """AURORA_RELEASE_URL and AURORA_RELEASES_API, the staging or mirror override.""" + + def urls(self, **env): + proc = sourced('printf "%s\\n" "$RELEASE_URL" "$RELEASES_API"', **env) + self.assertEqual(proc.returncode, 0, proc.stderr) + return proc.stdout.splitlines() + + def test_default(self): + self.assertEqual(self.urls(), [DEFAULT_RELEASE_URL, DEFAULT_RELEASES_API]) + + def test_empty_is_the_default(self): + self.assertEqual(self.urls(AURORA_RELEASE_URL="", AURORA_RELEASES_API=""), + [DEFAULT_RELEASE_URL, DEFAULT_RELEASES_API]) + + def test_override(self): + self.assertEqual(self.urls(AURORA_RELEASE_URL="file:///srv/stage", AURORA_RELEASES_API="http://127.0.0.1:8000/api"), + ["file:///srv/stage", "http://127.0.0.1:8000/api"]) + + def test_releases_api_override(self): + if not shutil.which("curl"): + self.skipTest("curl is needed") + with tempfile.TemporaryDirectory() as d: + latest = Path(d) / "latest" + api = Path(d).as_uri() + latest.write_text('{"tag_name": "%s"}' % TAG) + proc = sourced("newer_release", AURORA_RELEASES_API=api) + self.assertEqual((proc.returncode, proc.stdout), (0, ""), proc.stderr) + latest.write_text('{"tag_name": "sep-7.1.12.aurora2-99.0"}') + proc = sourced("newer_release", AURORA_RELEASES_API=api) + self.assertEqual(proc.stdout.strip(), "sep-7.1.12.aurora2-99.0", proc.stderr) + + +class StagedCopyTest(flow.M3FlowBase): + """install_all with the real curl against a staged copy (file://).""" + + def setUp(self): + super().setUp() + if not shutil.which("curl"): + self.skipTest("curl is needed") + # The real curl, not the harness's stub. + (self.tmp / "bin/curl").unlink() + self.stage = self.tmp / "pkgs" + self.extra_env["AURORA_RELEASE_URL"] = self.stage.as_uri() + + def test_install_from_a_staged_copy(self): + self.mac("j293") + self.install() + self.assertTrue(self.boot.read_bytes().startswith(b"M1N1:" + flow.M1N1_BASE.encode() + b"\n")) + self.assertIn("linux-aurora", (self.fake / "installed").read_text().split()) + + def test_a_staged_copy_is_still_checked(self): + self.mac("j293") + with open(self.stage / "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst", "ab") as f: + f.write(b"x") + proc = self.install(check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("does not match its published checksum", proc.stderr) + self.assertEqual(self.boot.read_bytes(), b"M1N1:original\n") + self.assertNotIn("pacman -U", self.log()) + + def test_a_file_missing_from_the_staged_copy(self): + self.mac("j293") + (self.stage / "aurora-touchid-20261003-1-any.pkg.tar.zst").unlink() + proc = self.install(check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("could not download aurora-touchid-20261003-1-any.pkg.tar.zst", proc.stderr) + self.assertNotIn("pacman -U", self.log()) + + class UpgradeTest(flow.M3FlowBase): """11.38's own script, then this one, on one fake Mac.""" From b22c03ad399379540f15385f175d538c70d6a5cf Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:40:37 -0700 Subject: [PATCH 1767/2680] tools/aurora-sep: print the public release URL in the closing commands The --agent-prompt and --uninstall commands printed at the end of an install used RELEASE_URL, so with AURORA_RELEASE_URL set they named the staging or mirror copy. The public tag URL is now PUBLIC_RELEASE_URL (and PUBLIC_RELEASES_API for the API), RELEASE_URL defaults to it, and both printed commands use it whatever the override. test_release.py: PUBLIC_RELEASE_URL stays the public tag with both overrides set, no echo in the script names RELEASE_URL, and an install from a staged copy prints every curl command with the public URL. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit bb7a5e6f45598b300b39f194a34275092d899af7) --- tools/aurora-sep/install-aurora-sep.sh | 14 +++++++++----- tools/aurora-sep/tests/test_release.py | 24 +++++++++++++++++++++--- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index cc4238e40e311..6b72819170713 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -161,11 +161,15 @@ VERSION=7.1.12.aurora2-12.0 TAG=sep-7.1.12.aurora2-12.0 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. +PUBLIC_RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG +PUBLIC_RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # AURORA_RELEASE_URL and AURORA_RELEASES_API are a staging or mirror override # for these two (any URL curl takes, file:// included); the checksums below -# still decide what is installed. -RELEASE_URL=${AURORA_RELEASE_URL:-https://github.com/iconidentify/aurora-linux/releases/download/$TAG} -RELEASES_API=${AURORA_RELEASES_API:-https://api.github.com/repos/iconidentify/aurora-linux/releases} +# still decide what is installed. Every command this script prints names +# PUBLIC_RELEASE_URL, never the override: a staged copy can go away, and the +# script itself is not covered by the checksums. +RELEASE_URL=${AURORA_RELEASE_URL:-$PUBLIC_RELEASE_URL} +RELEASES_API=${AURORA_RELEASES_API:-$PUBLIC_RELEASES_API} # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( @@ -1706,11 +1710,11 @@ install_all() { echo " This Mac keeps the boot loader it has: the m1n1 that failed on it is not put back." fi echo " Testing this build? The plan and reporting format:" - echo " curl -fsSL $RELEASE_URL/install-aurora-sep.sh | bash -s -- --agent-prompt" + echo " curl -fsSL $PUBLIC_RELEASE_URL/install-aurora-sep.sh | bash -s -- --agent-prompt" if [[ $chain == grub ]]; then echo " The previous kernel stays in the GRUB menu as 'Previous kernel … before aurora-sep'." fi - echo " To undo everything: curl -fsSL $RELEASE_URL/install-aurora-sep.sh | bash -s -- --uninstall" + echo " To undo everything: curl -fsSL $PUBLIC_RELEASE_URL/install-aurora-sep.sh | bash -s -- --uninstall" } uninstall_all() { diff --git a/tools/aurora-sep/tests/test_release.py b/tools/aurora-sep/tests/test_release.py index e62c402d69b12..d03b4d48ca1c7 100644 --- a/tools/aurora-sep/tests/test_release.py +++ b/tools/aurora-sep/tests/test_release.py @@ -6,8 +6,9 @@ content when the file is at hand (AURORA_M1N1_PKG, or the release staging directory). ReleaseUrlTest and StagedCopyTest cover the staging or mirror override (AURORA_RELEASE_URL, AURORA_RELEASES_API): the default is the -release's own tag, and a staged copy installs through the same download and -checksum loop. UpgradeTest runs 11.38's own script on the fake Mac of +release's own tag, a staged copy installs through the same download and +checksum loop, and the commands an install prints name the public release. +UpgradeTest runs 11.38's own script on the fake Mac of test_m3_flow, then this one, as an owner updating would. """ from pathlib import Path @@ -136,6 +137,13 @@ def test_override(self): self.assertEqual(self.urls(AURORA_RELEASE_URL="file:///srv/stage", AURORA_RELEASES_API="http://127.0.0.1:8000/api"), ["file:///srv/stage", "http://127.0.0.1:8000/api"]) + def test_printed_commands_name_the_public_release(self): + # The commands an install prints stay on the public tag under an override. + proc = sourced('printf "%s\\n" "$PUBLIC_RELEASE_URL"', + AURORA_RELEASE_URL="file:///srv/stage", AURORA_RELEASES_API="http://127.0.0.1:8000/api") + self.assertEqual(proc.stdout.splitlines(), [DEFAULT_RELEASE_URL], proc.stderr) + self.assertNotRegex(SRC, r"echo .*\$RELEASE_URL") + def test_releases_api_override(self): if not shutil.which("curl"): self.skipTest("curl is needed") @@ -164,7 +172,17 @@ def setUp(self): def test_install_from_a_staged_copy(self): self.mac("j293") - self.install() + proc = self.install() + # Every command it prints names the public release, not the stage. + printed = proc.stdout + proc.stderr + script = f"curl -fsSL {DEFAULT_RELEASE_URL}/install-aurora-sep.sh | bash -s -- " + self.assertIn(script + "--agent-prompt", printed) + self.assertIn("To undo everything: " + script + "--uninstall", printed) + commands = [l for l in printed.splitlines() if "curl " in l] + self.assertTrue(commands) + for line in commands: + self.assertIn("https://github.com/iconidentify/aurora-linux/releases/", line) + self.assertNotIn(self.stage.as_uri(), line) self.assertTrue(self.boot.read_bytes().startswith(b"M1N1:" + flow.M1N1_BASE.encode() + b"\n")) self.assertIn("linux-aurora", (self.fake / "installed").read_text().split()) From a61395db16a71215c641624d79ff9ebb295546f7 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:41:44 -0700 Subject: [PATCH 1768/2680] tools/aurora-sep: check the staging override values before use AURORA_RELEASE_URL and AURORA_RELEASES_API are now taken only as a file://, http:// or https:// URL in printable ASCII: a value starting with "-", with another or no scheme, or with spaces or control characters stops the run before the first download or release lookup. The message names the variable, not its value. release_source does the check, first in install_all and in --agent-prompt. newer_release takes a tag only as letters, digits, ".", "_" and "-", so control characters in a release list's answer are never printed. test_release.py: values that are refused and taken, a refused value stopping an install before any download and --agent-prompt before its lookup, and tags with escape sequences or other characters ignored. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit b36cfa1dffcfc3742a77319cde21116014d0009d) --- tools/aurora-sep/install-aurora-sep.sh | 36 +++++++++++++++---- tools/aurora-sep/tests/test_release.py | 50 ++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 6b72819170713..a03497b8bf80a 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -164,8 +164,8 @@ TAG=sep-7.1.12.aurora2-12.0 PUBLIC_RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG PUBLIC_RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # AURORA_RELEASE_URL and AURORA_RELEASES_API are a staging or mirror override -# for these two (any URL curl takes, file:// included); the checksums below -# still decide what is installed. Every command this script prints names +# for these two (a file://, http:// or https:// URL; see release_source); the +# checksums below still decide what is installed. Every command this script prints names # PUBLIC_RELEASE_URL, never the override: a staged copy can go away, and the # script itself is not covered by the checksums. RELEASE_URL=${AURORA_RELEASE_URL:-$PUBLIC_RELEASE_URL} @@ -213,6 +213,27 @@ if (( EUID != 0 )); then sudo=sudo fi +# AURORA_RELEASE_URL and AURORA_RELEASES_API go to curl and into printed +# text, so only a plain file://, http:// or https:// URL is taken: nothing curl +# could read as an option, and no spaces, control characters or other bytes +# outside printable ASCII (percent-encode those). +override_ok() { + local LC_ALL=C + [[ $1 != -* && $1 =~ ^(file|https?)://. && $1 != *[![:graph:]]* ]] +} + +# Checked before the first download or release lookup that uses them. +release_source() { + local name + for name in AURORA_RELEASE_URL AURORA_RELEASES_API; do + [[ -z ${!name:-} ]] || override_ok "${!name}" || + die "$name must be a file://, http:// or https:// URL without spaces or + control characters (percent-encode anything outside printable ASCII). + Unset it to use the public release. Nothing was installed." + done + return 0 +} + # A saved copy of this script keeps installing its own build forever. Tell the # operator - human or agent - when a newer one exists. Never fatal: no network, # rate limit or API change should stop an install that was going to work. @@ -224,10 +245,12 @@ newer_release() { local seen="" # Ask for the release marked Latest. Listing all releases is not ordered by # version: they share a commit, so GitHub falls back to comparing tag names - # as text, and 11.9 sorts above 11.10. + # as text, and 11.9 sorts above 11.10. Only letters, digits, '.', '_' and + # '-' are taken as a tag, so a mirror's answer can't put terminal escapes + # into the notice. seen=$(curl -fsSL --max-time 8 "$RELEASES_API/latest" 2>/dev/null | - grep -o '"tag_name"[[:space:]]*:[[:space:]]*"sep-[^"]*"' | - head -1 | sed 's/.*"\(sep-[^"]*\)"$/\1/') || true + LC_ALL=C grep -o '"tag_name"[[:space:]]*:[[:space:]]*"sep-[A-Za-z0-9._-]*"' | + head -1 | sed 's/.*"\(sep-[A-Za-z0-9._-]*\)"$/\1/') || true [[ -n $seen && $seen != "$TAG" ]] && echo "$seen" return 0 } @@ -1528,6 +1551,7 @@ packages_for_this_mac() { install_all() { local entry file sha kernel chain local -a entries + release_source require_supported_soc version_notice sep_write_notice @@ -2463,6 +2487,6 @@ case ${1:-} in --uninstall) uninstall_all ;; --reset-touchid) shift; reset_touchid "$@" ;; --m3-report) m3_report ;; - --agent-prompt) t=$(newer_release); [[ -n $t ]] && warn "the current release is $t; this plan is for $TAG"; agent_prompt ;; + --agent-prompt) release_source; t=$(newer_release); [[ -n $t ]] && warn "the current release is $t; this plan is for $TAG"; agent_prompt ;; *) die "unknown option $1 (--read-only, --uninstall, --reset-touchid, --agent-prompt, --m3-report or --m3-handoff)" ;; esac diff --git a/tools/aurora-sep/tests/test_release.py b/tools/aurora-sep/tests/test_release.py index d03b4d48ca1c7..125ae4ef4fbab 100644 --- a/tools/aurora-sep/tests/test_release.py +++ b/tools/aurora-sep/tests/test_release.py @@ -108,6 +108,12 @@ def test_the_package_itself(self): DEFAULT_RELEASE_URL = f"https://github.com/iconidentify/aurora-linux/releases/download/{TAG}" DEFAULT_RELEASES_API = "https://api.github.com/repos/iconidentify/aurora-linux/releases" OVERRIDES = ("AURORA_RELEASE_URL", "AURORA_RELEASES_API") +# Values release_source refuses: curl options, no or another scheme, spaces, +# control characters and bytes outside printable ASCII. +BAD_VALUES = ("-K/etc/hostname", "--config=/tmp/x", "-o/tmp/x", "ftp://mirror/x", "stage-12.0", + "/srv/stage", "file:///srv/st age", "https://m/a\tb", "https://m/a\nb", + "https://m/\x1b]0;t\x07", "https://m/\x1b[2J", "https://m/\x7f", "https://m/\u00e9", + "HTTPS://m/x", "file://") def sourced(body, **env_extra): @@ -137,6 +143,24 @@ def test_override(self): self.assertEqual(self.urls(AURORA_RELEASE_URL="file:///srv/stage", AURORA_RELEASES_API="http://127.0.0.1:8000/api"), ["file:///srv/stage", "http://127.0.0.1:8000/api"]) + def test_bad_values_are_refused(self): + for name in OVERRIDES: + for value in BAD_VALUES: + with self.subTest(name=name, value=value): + proc = sourced("release_source", **{name: value}) + self.assertEqual(proc.returncode, 1, proc.stdout) + schemes = "a file://, http:// or https:// URL" + self.assertIn(f"{name} must be {schemes}", proc.stderr) + # The value itself is never echoed. + self.assertNotIn(value, proc.stderr.replace(schemes, "")) + + def test_good_values_are_taken(self): + for value in ("file:///srv/stage", "file:///srv/stage/", "file://localhost/srv/a%20b", + "http://127.0.0.1:18712", "https://mirror.example/aurora/12.0/"): + with self.subTest(value=value): + proc = sourced("release_source", AURORA_RELEASE_URL=value, AURORA_RELEASES_API=value) + self.assertEqual(proc.returncode, 0, proc.stderr) + def test_printed_commands_name_the_public_release(self): # The commands an install prints stay on the public tag under an override. proc = sourced('printf "%s\\n" "$PUBLIC_RELEASE_URL"', @@ -156,6 +180,22 @@ def test_releases_api_override(self): latest.write_text('{"tag_name": "sep-7.1.12.aurora2-99.0"}') proc = sourced("newer_release", AURORA_RELEASES_API=api) self.assertEqual(proc.stdout.strip(), "sep-7.1.12.aurora2-99.0", proc.stderr) + # A tag with control characters or anything but [A-Za-z0-9._-] is not taken. + for tag in (b"sep-7.1.12.aurora2-99.0\x1b]0;x\x07", b"sep-7.1.12.aurora2-99.0\x1b[2J", + b"sep-7.1.12.aurora2-99.0\\u001b[2J", b"sep-99.0$(id)", b"sep-99.0 x"): + with self.subTest(tag=tag): + latest.write_bytes(b'{"tag_name": "' + tag + b'"}') + proc = sourced("newer_release", AURORA_RELEASES_API=api) + self.assertEqual((proc.returncode, proc.stdout), (0, ""), proc.stderr) + + def test_agent_prompt_refuses_a_bad_value(self): + env = {k: v for k, v in os.environ.items() if k not in OVERRIDES} + env["AURORA_RELEASES_API"] = "-K/etc/hostname" + proc = subprocess.run(["bash", str(flow.INSTALLER), "--agent-prompt"], + capture_output=True, text=True, env=env) + self.assertEqual(proc.returncode, 1) + self.assertIn("AURORA_RELEASES_API must be", proc.stderr) + self.assertEqual(proc.stdout, "") class StagedCopyTest(flow.M3FlowBase): @@ -196,6 +236,16 @@ def test_a_staged_copy_is_still_checked(self): self.assertEqual(self.boot.read_bytes(), b"M1N1:original\n") self.assertNotIn("pacman -U", self.log()) + def test_a_bad_value_stops_the_install(self): + self.mac("j293") + self.extra_env["AURORA_RELEASE_URL"] = "-K/etc/hostname" + proc = self.install(check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("AURORA_RELEASE_URL must be", proc.stderr) + self.assertNotIn("Downloading", proc.stdout) + self.assertEqual(self.boot.read_bytes(), b"M1N1:original\n") + self.assertNotIn("pacman -U", self.log()) + def test_a_file_missing_from_the_staged_copy(self): self.mac("j293") (self.stage / "aurora-touchid-20261003-1-any.pkg.tar.zst").unlink() From 88a634598344683b51b548ec92deae28bcc67748 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:42:49 -0700 Subject: [PATCH 1769/2680] tools/aurora-sep: name a staging or mirror copy when one is in use With AURORA_RELEASE_URL or AURORA_RELEASES_API set, release_source now prints one line at the start: "Using a staging/mirror copy: ; checksums are still verified". --agent-prompt prints it on stderr, so the prompt itself is unchanged. A file that can't be downloaded from such a copy is reported as missing from that copy or unreachable, with the variable to check or unset, instead of as a packaging mistake in the release to report. Without an override the message is as before. test_release.py: the line for each combination and none without an override or with the public URL, the line first and the only one naming the stage in an install, --agent-prompt keeping it off stdout, and the two messages for a missing file. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 732a376050a9037ff5eff006bb2ac52e0718c051) --- tools/aurora-sep/install-aurora-sep.sh | 26 ++++++++-- tools/aurora-sep/tests/test_release.py | 69 ++++++++++++++++++++++++-- 2 files changed, 87 insertions(+), 8 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index a03497b8bf80a..653cba2fe033e 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -222,16 +222,26 @@ override_ok() { [[ $1 != -* && $1 =~ ^(file|https?)://. && $1 != *[![:graph:]]* ]] } -# Checked before the first download or release lookup that uses them. +# Checked before the first download or release lookup that uses them. A run +# from a staging or mirror copy says so once, at the start. release_source() { - local name + local name shown for name in AURORA_RELEASE_URL AURORA_RELEASES_API; do [[ -z ${!name:-} ]] || override_ok "${!name}" || die "$name must be a file://, http:// or https:// URL without spaces or control characters (percent-encode anything outside printable ASCII). Unset it to use the public release. Nothing was installed." done - return 0 + if [[ $RELEASE_URL != "$PUBLIC_RELEASE_URL" && $RELEASES_API != "$PUBLIC_RELEASES_API" ]]; then + shown="$RELEASE_URL (release list $RELEASES_API)" + elif [[ $RELEASE_URL != "$PUBLIC_RELEASE_URL" ]]; then + shown=$RELEASE_URL + elif [[ $RELEASES_API != "$PUBLIC_RELEASES_API" ]]; then + shown="release list $RELEASES_API" + else + return 0 + fi + say "Using a staging/mirror copy: $shown; checksums are still verified" } # A saved copy of this script keeps installing its own build forever. Tell the @@ -1570,11 +1580,17 @@ install_all() { for entry in "${entries[@]}"; do read -r file sha <<<"$entry" say "Downloading $file" - curl -fL --retry 3 --progress-bar -o "$work/$file" "$RELEASE_URL/$file" || + if ! curl -fL --retry 3 --progress-bar -o "$work/$file" "$RELEASE_URL/$file"; then + [[ $RELEASE_URL == "$PUBLIC_RELEASE_URL" ]] || + die "could not download $file from the staging/mirror copy that + AURORA_RELEASE_URL names (shown at the start). That copy is missing the file + or can't be reached: check it, or unset AURORA_RELEASE_URL to install from + the public release. Nothing was installed." die "could not download $file from $TAG. The release is missing a file this script expects, which is a packaging mistake rather than anything wrong with this Mac. Nothing was installed. Please report it with the file name above." + fi [[ $(sha256sum "$work/$file" | cut -d' ' -f1) == "$sha" ]] || die "$file does not match its published checksum" done if m1n1_for_this_mac; then @@ -2487,6 +2503,6 @@ case ${1:-} in --uninstall) uninstall_all ;; --reset-touchid) shift; reset_touchid "$@" ;; --m3-report) m3_report ;; - --agent-prompt) release_source; t=$(newer_release); [[ -n $t ]] && warn "the current release is $t; this plan is for $TAG"; agent_prompt ;; + --agent-prompt) release_source >&2; t=$(newer_release); [[ -n $t ]] && warn "the current release is $t; this plan is for $TAG"; agent_prompt ;; *) die "unknown option $1 (--read-only, --uninstall, --reset-touchid, --agent-prompt, --m3-report or --m3-handoff)" ;; esac diff --git a/tools/aurora-sep/tests/test_release.py b/tools/aurora-sep/tests/test_release.py index 125ae4ef4fbab..eb973cfb3343f 100644 --- a/tools/aurora-sep/tests/test_release.py +++ b/tools/aurora-sep/tests/test_release.py @@ -6,9 +6,11 @@ content when the file is at hand (AURORA_M1N1_PKG, or the release staging directory). ReleaseUrlTest and StagedCopyTest cover the staging or mirror override (AURORA_RELEASE_URL, AURORA_RELEASES_API): the default is the -release's own tag, a staged copy installs through the same download and -checksum loop, and the commands an install prints name the public release. -UpgradeTest runs 11.38's own script on the fake Mac of +release's own tag, only a plain file://, http:// or https:// URL is taken, +a staged copy is named once at the start and installs through the same +download and checksum loop, and the commands an install prints name the +public release. PublicDownloadTest keeps the public release's own message +for a missing file. UpgradeTest runs 11.38's own script on the fake Mac of test_m3_flow, then this one, as an owner updating would. """ from pathlib import Path @@ -161,6 +163,40 @@ def test_good_values_are_taken(self): proc = sourced("release_source", AURORA_RELEASE_URL=value, AURORA_RELEASES_API=value) self.assertEqual(proc.returncode, 0, proc.stderr) + def test_the_copy_in_use_is_named_once(self): + notice = "Using a staging/mirror copy: %s; checksums are still verified" + for env, shown in (({}, None), + ({"AURORA_RELEASE_URL": DEFAULT_RELEASE_URL}, None), + ({"AURORA_RELEASE_URL": "file:///srv/stage"}, "file:///srv/stage"), + ({"AURORA_RELEASES_API": "http://127.0.0.1:8000/api"}, + "release list http://127.0.0.1:8000/api"), + ({"AURORA_RELEASE_URL": "file:///srv/stage", "AURORA_RELEASES_API": "http://127.0.0.1:8000/api"}, + "file:///srv/stage (release list http://127.0.0.1:8000/api)")): + with self.subTest(env=env): + proc = sourced("release_source", **env) + self.assertEqual(proc.returncode, 0, proc.stderr) + if shown is None: + self.assertEqual(proc.stdout, "") + else: + self.assertEqual(len(proc.stdout.splitlines()), 1, proc.stdout) + self.assertIn(notice % shown, proc.stdout) + + def test_agent_prompt_names_the_copy_on_stderr(self): + if not shutil.which("curl"): + self.skipTest("curl is needed") + with tempfile.TemporaryDirectory() as d: + (Path(d) / "latest").write_text('{"tag_name": "%s"}' % TAG) + env = {k: v for k, v in os.environ.items() if k not in OVERRIDES} + env.update(AURORA_RELEASE_URL="file:///srv/stage", AURORA_RELEASES_API=Path(d).as_uri()) + proc = subprocess.run(["bash", str(flow.INSTALLER), "--agent-prompt"], + capture_output=True, text=True, env=env) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertIn("Using a staging/mirror copy: file:///srv/stage", proc.stderr) + # The prompt itself is untouched and names only the public release. + self.assertNotIn("staging/mirror", proc.stdout) + self.assertNotIn("file:///srv/stage", proc.stdout) + self.assertNotIn(Path(d).as_uri(), proc.stdout) + def test_printed_commands_name_the_public_release(self): # The commands an install prints stay on the public tag under an override. proc = sourced('printf "%s\\n" "$PUBLIC_RELEASE_URL"', @@ -223,6 +259,11 @@ def test_install_from_a_staged_copy(self): for line in commands: self.assertIn("https://github.com/iconidentify/aurora-linux/releases/", line) self.assertNotIn(self.stage.as_uri(), line) + # The stage is named once, in the first line, and nowhere else. + notice = f"Using a staging/mirror copy: {self.stage.as_uri()}; checksums are still verified" + self.assertIn(notice, proc.stdout.splitlines()[0]) + self.assertEqual([l for l in printed.splitlines() if self.stage.as_uri() in l], + [proc.stdout.splitlines()[0]]) self.assertTrue(self.boot.read_bytes().startswith(b"M1N1:" + flow.M1N1_BASE.encode() + b"\n")) self.assertIn("linux-aurora", (self.fake / "installed").read_text().split()) @@ -252,6 +293,28 @@ def test_a_file_missing_from_the_staged_copy(self): proc = self.install(check=False) self.assertNotEqual(proc.returncode, 0) self.assertIn("could not download aurora-touchid-20261003-1-any.pkg.tar.zst", proc.stderr) + # It blames the staged copy, not the release. + self.assertIn("from the staging/mirror copy that\n AURORA_RELEASE_URL names", proc.stderr) + self.assertNotIn("packaging", proc.stderr) + self.assertNotIn("report", proc.stderr) + self.assertNotIn("pacman -U", self.log()) + + +class PublicDownloadTest(flow.M3FlowBase): + """A file missing from the public release, with no override set.""" + + def setUp(self): + super().setUp() + self.extra_env.update({name: "" for name in OVERRIDES}) + + def test_a_file_missing_from_the_release(self): + self.mac("j293") + (self.tmp / "pkgs/aurora-touchid-20261003-1-any.pkg.tar.zst").unlink() + proc = self.install(check=False) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("could not download aurora-touchid-20261003-1-any.pkg.tar.zst from " + TAG, proc.stderr) + self.assertIn("packaging\n mistake", proc.stderr) + self.assertNotIn("staging/mirror", proc.stdout + proc.stderr) self.assertNotIn("pacman -U", self.log()) From 9f607e7d0523ab3ca979bdfb8662535a67a5b9bc Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:44:46 -0700 Subject: [PATCH 1770/2680] tools/aurora-sep: name a newer release only when its number is higher newer_release named any Latest tag that differed from TAG, so a run of a release staged before it was published pointed at the older public one. It now compares the release numbers, the part after the kernel version, with sort -V, and names the tag only when it is higher. With AURORA_RELEASE_URL or AURORA_RELEASES_API set, version_notice and --agent-prompt name a newer release without any command to run, and the install no longer says its own tag is the current release. test_release.py: newer and older tags (a point release, a higher release on an older kernel, a lower one on a newer kernel, a tag with no number), version_notice with and without an override, and --agent-prompt with a newer and an older release. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 077d5ac0c144c0f6654293567636d57c571da052) --- tools/aurora-sep/install-aurora-sep.sh | 44 ++++++++++++--- tools/aurora-sep/tests/test_release.py | 76 +++++++++++++++++++++++++- 2 files changed, 110 insertions(+), 10 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 653cba2fe033e..aceef79003ca8 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -222,6 +222,11 @@ override_ok() { [[ $1 != -* && $1 =~ ^(file|https?)://. && $1 != *[![:graph:]]* ]] } +# True when either override points away from the public release. +release_overridden() { + [[ $RELEASE_URL != "$PUBLIC_RELEASE_URL" || $RELEASES_API != "$PUBLIC_RELEASES_API" ]] +} + # Checked before the first download or release lookup that uses them. A run # from a staging or mirror copy says so once, at the start. release_source() { @@ -232,14 +237,14 @@ release_source() { control characters (percent-encode anything outside printable ASCII). Unset it to use the public release. Nothing was installed." done - if [[ $RELEASE_URL != "$PUBLIC_RELEASE_URL" && $RELEASES_API != "$PUBLIC_RELEASES_API" ]]; then + if ! release_overridden; then + return 0 + elif [[ $RELEASE_URL != "$PUBLIC_RELEASE_URL" && $RELEASES_API != "$PUBLIC_RELEASES_API" ]]; then shown="$RELEASE_URL (release list $RELEASES_API)" elif [[ $RELEASE_URL != "$PUBLIC_RELEASE_URL" ]]; then shown=$RELEASE_URL - elif [[ $RELEASES_API != "$PUBLIC_RELEASES_API" ]]; then - shown="release list $RELEASES_API" else - return 0 + shown="release list $RELEASES_API" fi say "Using a staging/mirror copy: $shown; checksums are still verified" } @@ -252,7 +257,7 @@ newer_release() { # network, GitHub rate-limits, or the response is not what we expect, and # whether set -e acts on that inside a command substitution is subtle enough # that it should not be left to chance in a script that runs as root. - local seen="" + local seen="" mine theirs # Ask for the release marked Latest. Listing all releases is not ordered by # version: they share a commit, so GitHub falls back to comparing tag names # as text, and 11.9 sorts above 11.10. Only letters, digits, '.', '_' and @@ -261,22 +266,43 @@ newer_release() { seen=$(curl -fsSL --max-time 8 "$RELEASES_API/latest" 2>/dev/null | LC_ALL=C grep -o '"tag_name"[[:space:]]*:[[:space:]]*"sep-[A-Za-z0-9._-]*"' | head -1 | sed 's/.*"\(sep-[A-Za-z0-9._-]*\)"$/\1/') || true - [[ -n $seen && $seen != "$TAG" ]] && echo "$seen" + # Only a higher release number counts, the part after the kernel version + # (sep-7.1.12.aurora2-12.0 is 12.0): a release staged before it is marked + # Latest must not be pointed at the older one. + mine=${TAG#sep-*-} theirs=${seen#sep-*-} + [[ $theirs == [0-9]* && $theirs != "$mine" && + $(printf '%s\n' "$mine" "$theirs" | sort -V | tail -1) == "$theirs" ]] && echo "$seen" return 0 } +# A staging or mirror run names a newer release but points at nothing: the +# public script is not what it is testing. version_notice() { local seen seen=$(newer_release) - if [[ -n $seen ]]; then + if [[ -n $seen ]] && release_overridden; then + warn "$seen is newer than $TAG, which this script installs" + elif [[ -n $seen ]]; then warn "this script installs $TAG, but $seen is published. You are probably running a saved copy. The current one: curl -fsSL $LATEST_URL | bash" - else + elif ! release_overridden; then say "$TAG is the current release" fi } +# The same check for --agent-prompt, on stderr only. +prompt_notice() { + local seen + seen=$(newer_release) + if [[ -n $seen ]] && release_overridden; then + warn "$seen is newer than $TAG, which this plan is for" + elif [[ -n $seen ]]; then + warn "the current release is $seen; this plan is for $TAG" + fi + return 0 +} + boot_chain() { # /boot/efi can be readable only by root, so test through sudo. Limine is # active where omarchy-mac-boot is installed, or where its activation marker @@ -2503,6 +2529,6 @@ case ${1:-} in --uninstall) uninstall_all ;; --reset-touchid) shift; reset_touchid "$@" ;; --m3-report) m3_report ;; - --agent-prompt) release_source >&2; t=$(newer_release); [[ -n $t ]] && warn "the current release is $t; this plan is for $TAG"; agent_prompt ;; + --agent-prompt) release_source >&2; prompt_notice; agent_prompt ;; *) die "unknown option $1 (--read-only, --uninstall, --reset-touchid, --agent-prompt, --m3-report or --m3-handoff)" ;; esac diff --git a/tools/aurora-sep/tests/test_release.py b/tools/aurora-sep/tests/test_release.py index eb973cfb3343f..1be635d0e0396 100644 --- a/tools/aurora-sep/tests/test_release.py +++ b/tools/aurora-sep/tests/test_release.py @@ -9,7 +9,8 @@ release's own tag, only a plain file://, http:// or https:// URL is taken, a staged copy is named once at the start and installs through the same download and checksum loop, and the commands an install prints name the -public release. PublicDownloadTest keeps the public release's own message +public release. Only a newer release number is named, and under an override +with no command to run. PublicDownloadTest keeps the public release's own message for a missing file. UpgradeTest runs 11.38's own script on the fake Mac of test_m3_flow, then this one, as an owner updating would. """ @@ -109,6 +110,15 @@ def test_the_package_itself(self): TAG = re.search(r"^TAG=(\S+)$", SRC, re.M).group(1) DEFAULT_RELEASE_URL = f"https://github.com/iconidentify/aurora-linux/releases/download/{TAG}" DEFAULT_RELEASES_API = "https://api.github.com/repos/iconidentify/aurora-linux/releases" +LATEST_URL = re.search(r"^LATEST_URL=(\S+)$", SRC, re.M).group(1) +# sep--: the release number orders releases, whatever +# the kernel version. +_kernel, _major, _minor = re.match(r"sep-([^-]+)-(\d+)\.(\d+)", TAG).groups() +_major, _minor = int(_major), int(_minor) +NEWER_TAGS = (f"{TAG}.1", f"sep-{_kernel}-{_major}.{_minor + 1}", f"sep-{_kernel}-{_major + 1}.0", + f"sep-1.0.0.aurora1-{_major + 1}.0") +OLDER_TAGS = (TAG, f"sep-{_kernel}-{_major - 1}.38", f"sep-{_kernel}-{_major - 1}.36.1", + f"sep-{_kernel}-{_major - 1}.9", f"sep-99.0.0.aurora9-{_major - 1}.99", "sep-latest") OVERRIDES = ("AURORA_RELEASE_URL", "AURORA_RELEASES_API") # Values release_source refuses: curl options, no or another scheme, spaces, # control characters and bytes outside printable ASCII. @@ -224,6 +234,70 @@ def test_releases_api_override(self): proc = sourced("newer_release", AURORA_RELEASES_API=api) self.assertEqual((proc.returncode, proc.stdout), (0, ""), proc.stderr) + def api(self, d, tag): + (Path(d) / "latest").write_text('{"tag_name": "%s"}' % tag) + return Path(d).as_uri() + + def test_only_a_newer_release_is_named(self): + if not shutil.which("curl"): + self.skipTest("curl is needed") + with tempfile.TemporaryDirectory() as d: + for tag, newer in [(t, True) for t in NEWER_TAGS] + [(t, False) for t in OLDER_TAGS]: + with self.subTest(tag=tag): + proc = sourced("newer_release", AURORA_RELEASES_API=self.api(d, tag)) + self.assertEqual(proc.stdout.strip(), tag if newer else "", proc.stderr) + + def notice(self, d, tag, override): + api = self.api(d, tag) + if override: + return sourced("version_notice", AURORA_RELEASES_API=api) + # The public check, read from the same file. + return sourced(f"RELEASES_API={api}; PUBLIC_RELEASES_API={api}; version_notice") + + def test_version_notice(self): + if not shutil.which("curl"): + self.skipTest("curl is needed") + newer, older = NEWER_TAGS[-2], OLDER_TAGS[1] + with tempfile.TemporaryDirectory() as d: + # The public script: a newer release points at the current script. + proc = self.notice(d, newer, override=False) + self.assertIn(f"this script installs {TAG}, but {newer} is published", proc.stderr) + self.assertIn(f"curl -fsSL {LATEST_URL} | bash", proc.stderr) + # An older or the same one is not named. + for tag in (older, TAG): + proc = self.notice(d, tag, override=False) + self.assertEqual(proc.stderr, "") + self.assertIn(f"{TAG} is the current release", proc.stdout) + # An override: a newer release is named, with no command to run. + proc = self.notice(d, newer, override=True) + self.assertIn(f"{newer} is newer than {TAG}, which this script installs", proc.stderr) + self.assertNotIn("curl", proc.stderr) + self.assertNotIn("saved copy", proc.stderr) + self.assertNotIn("current release", proc.stdout) + # An older one: nothing at all. + proc = self.notice(d, older, override=True) + self.assertEqual((proc.stdout, proc.stderr), ("", "")) + + def agent_prompt(self, **env_extra): + env = {k: v for k, v in os.environ.items() if k not in OVERRIDES} + env.update(env_extra) + return subprocess.run(["bash", str(flow.INSTALLER), "--agent-prompt"], + capture_output=True, text=True, env=env) + + def test_agent_prompt_notice(self): + if not shutil.which("curl"): + self.skipTest("curl is needed") + newer, older = NEWER_TAGS[-2], OLDER_TAGS[1] + with tempfile.TemporaryDirectory() as d: + proc = self.agent_prompt(AURORA_RELEASES_API=self.api(d, newer)) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertIn(f"{newer} is newer than {TAG}, which this plan is for", proc.stderr) + self.assertNotIn("current release", proc.stderr) + proc = self.agent_prompt(AURORA_RELEASE_URL="file:///srv/stage", AURORA_RELEASES_API=self.api(d, older)) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertNotIn(older, proc.stdout + proc.stderr) + self.assertNotIn("newer", proc.stderr) + def test_agent_prompt_refuses_a_bad_value(self): env = {k: v for k, v in os.environ.items() if k not in OVERRIDES} env["AURORA_RELEASES_API"] = "-K/etc/hostname" From 2e8f983df0e3e8f62c2c259a319fef621ae7db1c Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:45:30 -0700 Subject: [PATCH 1771/2680] tools/aurora-sep: reword the comment on the release URL override The comment above RELEASE_URL now says that the printed commands use PUBLIC_RELEASE_URL or LATEST_URL, and is wrapped like the rest. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit df0c433027e9eb283b14a56894efcc817c8a3cfb) --- tools/aurora-sep/install-aurora-sep.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index aceef79003ca8..29c95429f21d8 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -165,9 +165,10 @@ PUBLIC_RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/downloa PUBLIC_RELEASES_API=https://api.github.com/repos/iconidentify/aurora-linux/releases # AURORA_RELEASE_URL and AURORA_RELEASES_API are a staging or mirror override # for these two (a file://, http:// or https:// URL; see release_source); the -# checksums below still decide what is installed. Every command this script prints names -# PUBLIC_RELEASE_URL, never the override: a staged copy can go away, and the -# script itself is not covered by the checksums. +# checksums below still decide what is installed. The commands this script +# prints name the public release (PUBLIC_RELEASE_URL or LATEST_URL), never the +# override: a staged copy can go away, and the checksums don't cover the +# script itself. RELEASE_URL=${AURORA_RELEASE_URL:-$PUBLIC_RELEASE_URL} RELEASES_API=${AURORA_RELEASES_API:-$PUBLIC_RELEASES_API} # Where to always get the current script, whatever this copy turns out to be. From 6c231eee845842e7d767ba2274836743f20cd1b5 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:27:26 -0700 Subject: [PATCH 1772/2680] tools/aurora-sep: release 12.1 Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit a3460526c640ecc9e2f8d326f0a6979500cf3334) --- tools/aurora-sep/install-aurora-sep.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 29c95429f21d8..311977f45b0f3 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -157,8 +157,8 @@ set -euo pipefail # The kernel package version and the release tag move independently: a release # that only changes m1n1 reuses the previous kernel packages unchanged. -VERSION=7.1.12.aurora2-12.0 -TAG=sep-7.1.12.aurora2-12.0 +VERSION=7.1.12.aurora2-12.1 +TAG=sep-7.1.12.aurora2-12.1 # Packages are fetched from this script's own tag, never from "latest": the # checksums below belong to this release and nothing else. PUBLIC_RELEASE_URL=https://github.com/iconidentify/aurora-linux/releases/download/$TAG From 1295f527fc2431d9d9aad208c0545176253cb503 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:39:53 -0700 Subject: [PATCH 1773/2680] tools/aurora-sep: fill in the 12.1 kernel checksums Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 1d41e1b7ff5db2b31de8fcee56959d70ce152c0a) --- tools/aurora-sep/install-aurora-sep.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/aurora-sep/install-aurora-sep.sh b/tools/aurora-sep/install-aurora-sep.sh index 311977f45b0f3..a520e614fe930 100755 --- a/tools/aurora-sep/install-aurora-sep.sh +++ b/tools/aurora-sep/install-aurora-sep.sh @@ -174,8 +174,8 @@ RELEASES_API=${AURORA_RELEASES_API:-$PUBLIC_RELEASES_API} # Where to always get the current script, whatever this copy turns out to be. LATEST_URL=https://github.com/iconidentify/aurora-linux/releases/latest/download/install-aurora-sep.sh PACKAGES=( - "linux-aurora-$VERSION-aarch64.pkg.tar.zst 7cd8b4b63ae672eaf80fc08fdeca89bf88ea3473fe35d27f1e7a7367c552a151" - "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst a12330c3871cf34cff2757e37db422feba9ecdeb570486a462ed6953e0f5e1d3" + "linux-aurora-$VERSION-aarch64.pkg.tar.zst 4755db7fc371257229d80de68417ba6fff49192b383d8c5a28c5115a8a84d4ee" + "linux-aurora-headers-$VERSION-aarch64.pkg.tar.zst ffd923a7420a5e8d664168ed1a74e39ee647998b60afa3782ecc2a94c3570e91" "libfprint-1.94.100-1.1-aarch64.pkg.tar.zst bc7d9762db6644f2cfb58ddb209602c1d513845eb1498c098e01f12600fcbdf9" "aurora-touchid-20261003-1-any.pkg.tar.zst 29b0360fac8c257d754e64bd1b9c33c487eb2595dd3c31e9138d7a476afa3d64" ) From 3c57f1e9e1fcfd99cf8d91121b700f8b2e05890e Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:04:30 -0700 Subject: [PATCH 1774/2680] tools/aurora-sep: add the M3 Air GPU experiment scripts Three scripts for running the T8122 start experiment (asahi.t8122_*) on an M3 MacBook Air, one boot at a time: - air-gpu-oneshot.sh arms one boot. The GPU driver is built into the kernel and starts before the initramfs, so the knobs go on the kernel command line: the script adds one Limine entry, air-gpu-oneshot, that boots the installed linux-aurora UKI (same path and pin) with that entry's command line plus asahi.t8122_start=1 and the given knobs, and sets LoaderEntryOneShot to it. Limine deletes that variable before it boots the entry, so every later boot, after a power cycle too, is the normal default entry. The normal entry, default_entry and the UKI are not changed. It checks the knobs against the kernel's accepted values and against the parameter names in the UKI, and refuses on anything but an Air (j613, j615; an M3 Pro with --allow-test-on-m3pro), with remember_last_entry or config enrollment on, with Secure Boot on, with another one-shot armed, or when the UKI does not match its pin. Edits are made under the Limine tools' locks, read back, and undone if a read-back fails. --check, --status and --disarm. - air-gpu-collect.sh writes one tgz to the user's home: the kernel log of the armed boot (this boot, or the one before from the journal when this one is normal), the GPU and firmware log lines, devcoredumps, asahi parameters, debugfs asahi-m3 state, every SMC T* and P* key, the render node and the job records; it prints one verdict line, from the kernel's "M3 G15G verdict:" lines or, without them, from the order of the existing runtime lines. - air-gpu-job.sh runs one Vulkan compute job through a given Mesa prefix (Python and ctypes; the SPIR-V is embedded), up to 8 s of submissions every 10 ms, each checked on the CPU, and holds the whole run to 10 s. It records the result, exit code and timing with the boot id. Tests run against a fake Mac (device tree, EFI variables, ESP with Limine and U-Boot's variable file) and need no root or hardware. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GWRhUgND1qhew6aEd6vsj8 (cherry picked from commit 4463585e25ad37699e7daad6843120d5f592f5a3) --- tools/aurora-sep/air-gpu/air-gpu-collect.sh | 398 +++++++++ tools/aurora-sep/air-gpu/air-gpu-job.sh | 482 +++++++++++ tools/aurora-sep/air-gpu/air-gpu-oneshot.sh | 535 ++++++++++++ tools/aurora-sep/tests/test_air_gpu.py | 906 ++++++++++++++++++++ 4 files changed, 2321 insertions(+) create mode 100755 tools/aurora-sep/air-gpu/air-gpu-collect.sh create mode 100755 tools/aurora-sep/air-gpu/air-gpu-job.sh create mode 100755 tools/aurora-sep/air-gpu/air-gpu-oneshot.sh create mode 100644 tools/aurora-sep/tests/test_air_gpu.py diff --git a/tools/aurora-sep/air-gpu/air-gpu-collect.sh b/tools/aurora-sep/air-gpu/air-gpu-collect.sh new file mode 100755 index 0000000000000..ff73abace7874 --- /dev/null +++ b/tools/aurora-sep/air-gpu/air-gpu-collect.sh @@ -0,0 +1,398 @@ +#!/bin/bash +# air-gpu-collect.sh: after a boot, collect what the M3 Air GPU experiment needs into one tgz in +# your home directory, and print one verdict line. +# +# sudo air-gpu-collect.sh the armed boot: this one, or the one before it when this +# boot is a normal one (the armed boot hung, say), read back +# from the journal +# sudo air-gpu-collect.sh --boot 0 this boot +# sudo air-gpu-collect.sh --boot -1 the boot before this one +# +# Read-only: apart from the tgz (and a temporary directory) it writes nothing. The kernel log in +# the tgz has lines naming a serial number dropped and MAC addresses masked. +# +# The last line printed is the verdict, one of: +# NOT-ARMED the boot did not have asahi.t8122_start=1 (no one-shot boot) +# ARMED-NOT-STARTED armed, but the driver stopped before it started the GPU firmware +# FW-BOOT-FAILED the firmware was started but did not come up (or crashed, with no job) +# INITDATA-REJECTED the firmware came up but did not accept the InitData +# FW-RUNNING the firmware accepted the InitData and runs; no GPU job ran +# JOB-NOT-DISPATCHED a GPU job was submitted and did not complete correctly +# JOB-COMPLETED air-gpu-job.sh ran, and every submission completed with correct results +set -euo pipefail + +# ---- the verdict table: the kernel log lines each step is judged by -------------------------- +# Keep this in step with the kernel branch (air/t8122-gpu-start: t8122_start.rs). +# +# 1. The kernel's own verdict lines, on an armed T8122 ("M3 G15G verdict: "), and the +# verdict each outcome gives. A job outcome outranks a firmware one; among job outcomes a +# failure outranks job-completed. +K1_VERDICTS=' +firmware-boot-failed FW-BOOT-FAILED +driver-refused ARMED-NOT-STARTED +initdata-rejected INITDATA-REJECTED +firmware-running-check-failed FW-BOOT-FAILED +firmware-running FW-RUNNING +job-accepted-never-dispatched JOB-NOT-DISPATCHED +job-timed-out-powered JOB-NOT-DISPATCHED +job-timed-out JOB-NOT-DISPATCHED +job-faulted JOB-NOT-DISPATCHED +job-retired-without-timestamps JOB-NOT-DISPATCHED +job-completed JOB-COMPLETED +' +RE_K1_VERDICT='M3 G15G verdict: [a-z-]+' +# The start experiment's own lines: the values it armed with, and its refusals. +RE_K1_ARMED='M3 G15G start: armed' +# +# 2. Without those lines (a kernel before them, or the M3 Pro rehearsal), these extended regexes +# decide, in this order: armed? -> firmware started? -> InitData sent? -> accepted? -> job? +# The boot was armed: on its command line. +RE_ARMED='(^| )asahi\.t8122_start=1( |$)' +# The driver refused or stopped before it started the firmware (the last such line is the reason). +RE_REFUSED='M3 G15G start: (refused|not armed)|GPU startup disabled|identity or ABI rejected|startup refused|no GPU backend|invalid for parameter .asahi\.|InitData self-check failed' +# The probe ended in an error (the reason when nothing more specific was logged). +RE_PROBE_FAILED='probe with driver asahi failed' +# The driver probed the GPU at all (without a boot-loader handoff the GPU node is disabled). +RE_PROBED='asahi [0-9a-f]+\.gpu: Probing' +# The GPU coprocessor's firmware was started. +RE_FW_STARTED='asahi [0-9a-f]+\.gpu: RTKit: Initializing|M3: publishing owned initdata' +# The firmware crashed. +RE_FW_CRASH='\.gpu: RTKit: co-processor has crashed|M3 G15[A-Z]*: firmware crashed|firmware crashed while waiting for init|GPU firmware crashed' +# The InitData was handed to the firmware, and the firmware accepted it. +RE_INITDATA_SENT='M3: publishing owned initdata' +RE_INITDATA_OK='M3: firmware accepted owned initdata' +# The firmware's readiness report (the reason when the InitData was not accepted). +RE_READINESS='M3 firmware readiness:|M3 firmware error event' +# The kernel saw a GPU job fail or never run. +RE_JOB_FAILED='M3 scheduler: execution failed|M3 completion events=|GPU is powered down|M3 fault render|M3 bank [0-9]+ fault=0x[0-9a-f]*[1-9a-f]|M3 fault mapping' +# +# The GPU lines kept in gpu-log.txt, and the firmware's own log lines in firmware-log.txt. +RE_GPU_LINES='asahi|agx|G15|M3[ _:]|t8122|gpu|RTKit|devcoredump|macsmc' +RE_FW_LOG='\.gpu: RTKit: (syslog|crashlog|co-processor|Unknown)|firmware crashed|crashlog|M3 firmware|M3 G15G (verdict|start)' +# SMC power keys that read zero at idle on Scott's J613 (2026-10-06): the GPU rail candidates. +GPU_RAIL_CANDIDATES="PP1b PP2l PP3b PP9l PPdb PR6b PRab PRcb PReb PZC0 PZC1 PZCB PZCU PZD1" +# ---------------------------------------------------------------------------------------------- + +PROG=air-gpu-collect +TAG=air_gpu.oneshot +DT=/proc/device-tree +EFIVARS=/sys/firmware/efi/efivars +BLI_GUID=4a67b082-0a4c-41cf-b6c7-440b29bb8c4f +DEBUGFS=/sys/kernel/debug +SYS=/sys +CMDLINE=/proc/cmdline +BOOT_ID=/proc/sys/kernel/random/boot_id +STATE_DIR=/var/lib/air-gpu + +is_root() { ((EUID == 0)); } +# The tgz goes to this user's home (the invoking user's under sudo). +user_home() { + local h + h=$(getent passwd "$1" | cut -d: -f6) + echo "${h:-/root}" +} +say() { printf '%s: %s\n' "$PROG" "$*"; } + +# Kernel log text without serial numbers and MAC addresses. +scrub() { + { grep -aviE 'serialnumber|serial number|serial-number' || true; } | + sed -E 's/([0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}/xx:xx:xx:xx:xx:xx/g' +} + +cmdline_of() { # boot: 0 or -1 + if [[ $1 == 0 ]]; then cat "$CMDLINE"; return; fi + { journalctl -k -b "$1" -o cat --no-pager 2>/dev/null || true; } | sed -n 's/^.*Kernel command line: //p' | head -1 +} +boot_id_of() { + if [[ $1 == 0 ]]; then tr -d '-' <"$BOOT_ID"; return; fi + { journalctl --list-boots --no-pager 2>/dev/null || true; } | awk -v i="$1" '$1 == i { print $2; exit }' +} +oneshot_of() { tr ' ' '\n' <<<"$1" | sed -n "s/^${TAG//./\\.}=//p" | head -1; } +knobs_of() { tr ' ' '\n' <<<"$1" | grep '^asahi\.' | tr '\n' ' ' | sed 's/ $//'; } + +# The kernel log of a boot, oldest first: this boot's from the journal when the journal has its +# start, otherwise dmesg; an earlier boot's from the journal. +kernel_log() { + local j + if [[ $1 == 0 ]]; then + j=$(journalctl -k -b 0 -o short-monotonic --no-pager 2>/dev/null || true) + if grep -qaE 'Kernel command line:|Linux version' <<<"$j"; then printf '%s\n' "$j"; return; fi + dmesg 2>/dev/null || true + return 0 + fi + journalctl -k -b "$1" -o short-monotonic --no-pager 2>/dev/null || true +} + +# The analysed boot's job records (air-gpu-job.sh), oldest first. +job_records() { # boot_id home + local f + [[ -n $1 && -d $2/air-gpu-runs ]] || return 0 + for f in "$2"/air-gpu-runs/job-*.txt; do + [[ -f $f ]] && grep -qx "boot_id=$1" "$f" && echo "$f" + done + return 0 +} +rec_key() { sed -n "s/^$2=//p" "$1" | head -1; } + +# ---- the verdict ----------------------------------------------------------------------------- + +VERDICT="" REASON="" +# classify LOG CMDLINE LAST_JOB_RECORD (the record may be empty) +classify() { + local log=$1 cmd=$2 rec=${3:-} accept_ln crash_ln line job_seen result reached + # Line number of the first match, and the text of the last one (both empty when none). + first_ln() { { grep -naE "$1" "$log" || true; } | head -1 | cut -d: -f1; } + last_line() { { grep -aE "$1" "$log" || true; } | tail -1 | sed -E 's/^\[[^]]*\] *//; s/^[^ ]+ kernel: //'; } + if ! [[ " $cmd " =~ $RE_ARMED ]]; then + VERDICT=NOT-ARMED REASON="asahi.t8122_start=1 is not on the analysed boot's command line" + return 0 + fi + if grep -qaE "$RE_K1_VERDICT" "$log"; then + classify_k1 "$log" "$rec" + return 0 + fi + if ! grep -qaE "$RE_FW_STARTED" "$log"; then + VERDICT=ARMED-NOT-STARTED + if line=$(last_line "$RE_REFUSED") && [[ -n $line ]]; then REASON=$line + elif line=$(last_line "$RE_PROBE_FAILED") && [[ -n $line ]]; then REASON=$line + elif ! grep -qaE "$RE_PROBED" "$log"; then + REASON="the GPU driver never probed: the GPU node was disabled (the boot loader did not hand the GPU over)" + else REASON="the driver stopped before it started the GPU firmware"; fi + return 0 + fi + crash_ln=$(first_ln "$RE_FW_CRASH") + if ! grep -qaE "$RE_INITDATA_SENT" "$log"; then + VERDICT=FW-BOOT-FAILED + if [[ -n $crash_ln ]]; then REASON=$(last_line "$RE_FW_CRASH") + elif line=$(last_line "$RE_PROBE_FAILED") && [[ -n $line ]]; then REASON="the firmware did not come up: $line" + else REASON="the firmware was started but never took the InitData"; fi + return 0 + fi + accept_ln=$(first_ln "$RE_INITDATA_OK") + if [[ -z $accept_ln ]]; then + VERDICT=INITDATA-REJECTED + if [[ -n $crash_ln ]]; then REASON="the firmware crashed after taking the InitData: $(last_line "$RE_FW_CRASH")" + elif line=$(last_line "$RE_READINESS") && [[ -n $line ]]; then REASON=$line + else REASON="the InitData was sent but the firmware never accepted it"; fi + return 0 + fi + # The firmware accepted the InitData. Did it crash afterwards, and did a job run? + crash_ln=$({ grep -naE "$RE_FW_CRASH" "$log" || true; } | awk -F: -v a="$accept_ln" '$1 > a { print $1; exit }') + result="" reached="" + if [[ -n $rec ]]; then result=$(rec_key "$rec" result) reached=$(rec_key "$rec" reached_gpu); fi + job_seen=0 + if grep -qaE "$RE_JOB_FAILED" "$log" || [[ $reached == yes || $result == running ]]; then job_seen=1; fi + if [[ -n $crash_ln ]]; then + if ((job_seen)); then + VERDICT=JOB-NOT-DISPATCHED REASON="the firmware crashed during the job: $(last_line "$RE_FW_CRASH")" + else + VERDICT=FW-BOOT-FAILED REASON="the firmware accepted the InitData, then crashed: $(last_line "$RE_FW_CRASH")" + fi + return 0 + fi + if ((!job_seen)); then + VERDICT=FW-RUNNING + if [[ -n $result ]]; then REASON="the firmware accepted the InitData and runs; the job did not reach the GPU ($result)" + else REASON="the firmware accepted the InitData and runs; no GPU job ran in this boot"; fi + return 0 + fi + if [[ $result == pass ]] && ! grep -qaE "$RE_JOB_FAILED" "$log"; then + VERDICT=JOB-COMPLETED + REASON="air-gpu-job: $(rec_key "$rec" submits) submissions completed with correct results (first in $(rec_key "$rec" first_submit_ms) ms) on $(rec_key "$rec" device)" + return 0 + fi + VERDICT=JOB-NOT-DISPATCHED + if line=$(last_line "$RE_JOB_FAILED") && [[ -n $line ]]; then REASON=$line + elif [[ $result == running ]]; then REASON="the boot ended while air-gpu-job was running (a hang or a power cycle)" + else REASON="air-gpu-job: $result: $(rec_key "$rec" error)"; fi +} + +# The kernel's verdict lines decide, with the job record for a job that did not reach the kernel. +classify_k1() { # LOG RECORD + local log=$1 rec=${2:-} outcomes o v line fail="" done_line="" fw="" result="" + k1_line() { { grep -aE "M3 G15G verdict: $1" "$log" || true; } | tail -1 | sed -E 's/^.*(M3 G15G verdict: )/\1/'; } + outcomes=$({ grep -aoE "$RE_K1_VERDICT" "$log" || true; } | sed 's/^M3 G15G verdict: //') + for o in $outcomes; do + v=$(printf '%s\n' "$K1_VERDICTS" | awk -v o="$o" '$1 == o { print $2 }') + case $v in + JOB-NOT-DISPATCHED) fail=$o ;; + JOB-COMPLETED) done_line=$o ;; + "") ;; + *) fw=$o ;; + esac + done + if [[ -n $rec ]]; then result=$(rec_key "$rec" result); fi + if [[ -n $fail ]]; then + VERDICT=JOB-NOT-DISPATCHED REASON=$(k1_line "$fail") + elif [[ -n $done_line && ( -z $rec || $result == pass ) ]]; then + VERDICT=JOB-COMPLETED REASON=$(k1_line job-completed) + if [[ -n $rec ]]; then REASON+="; air-gpu-job: $(rec_key "$rec" submits) submissions verified"; fi + elif [[ -n $done_line ]]; then + VERDICT=JOB-NOT-DISPATCHED + REASON="air-gpu-job: $result: $(rec_key "$rec" error) (the kernel: $(k1_line job-completed))" + elif [[ -n $fw ]]; then + VERDICT=$(printf '%s\n' "$K1_VERDICTS" | awk -v o="$fw" '$1 == o { print $2 }') + REASON=$(k1_line "$fw") + if [[ $VERDICT == FW-RUNNING && $result == pass ]]; then + # The job checked its results on the CPU, but the kernel logged no job-completed line. + VERDICT=JOB-COMPLETED + REASON="air-gpu-job: $(rec_key "$rec" submits) submissions completed with correct results (no job-completed line from the kernel)" + elif [[ $VERDICT == FW-RUNNING && -n $rec ]]; then + if [[ $(rec_key "$rec" reached_gpu) == yes || $result == running ]]; then + VERDICT=JOB-NOT-DISPATCHED + if [[ $result == running ]]; then REASON="the boot ended while air-gpu-job was running (a hang or a power cycle)" + else REASON="air-gpu-job: $result: $(rec_key "$rec" error)"; fi + else + REASON+="; the job did not reach the GPU ($result)" + fi + fi + else + line=$({ grep -aoE "$RE_K1_VERDICT.*" "$log" || true; } | tail -1) + VERDICT=ARMED-NOT-STARTED REASON="a kernel verdict this script does not know: $line" + fi + return 0 +} + +# ---- collection ------------------------------------------------------------------------------ + +smc_keys_file() { find "$DEBUGFS" -maxdepth 3 -name keys -path '*smc*' 2>/dev/null | head -1; } + +main() { + local boot="" cmd0 b cmd id home user dir out log rec recs f board kf n live n_tg rails + # Removed on any exit (a global, so the EXIT trap still sees it). + WORK="" + trap 'rm -rf "${WORK:-}"' EXIT + while (($#)); do + case $1 in + --boot) boot=${2:-}; shift ;; + -h | --help) sed -n '2,21p' "$0"; exit 0 ;; + *) say "refused: unknown argument $1 (--boot 0 or --boot -1)."; exit 2 ;; + esac + shift + done + [[ -z $boot || $boot == 0 || $boot == -1 ]] || { say "refused: --boot takes 0 or -1."; exit 2; } + is_root || { say "refused: run as root (sudo): the SMC keys and the kernel log need it."; exit 2; } + + cmd0=$(cat "$CMDLINE") + if [[ -z $boot ]]; then + boot=0 + if ! [[ " $cmd0 " =~ $RE_ARMED ]] && [[ " $(cmdline_of -1) " =~ $RE_ARMED ]]; then boot=-1; fi + fi + cmd=$(cmdline_of "$boot") + b=$(boot_id_of "$boot") + id=$(oneshot_of "$cmd") + user=${SUDO_USER:-root} + home=$(user_home "$user") + board=$(tr '\0' '\n' <"$DT/compatible" 2>/dev/null | sed -n 's/^apple,\(j[0-9a-z]*\)$/\1/p' | head -1) + out=$home/air-gpu-collect-${board:-mac}-$(date +%Y%m%d-%H%M%S) + dir=$(mktemp -d) + WORK=$dir + mkdir -p "$dir/c" + local d=$dir/c + + # The analysed boot's kernel log, and this boot's dmesg. + log=$d/kernel-log-boot$boot.txt + kernel_log "$boot" | scrub >"$log" + if [[ $boot != 0 ]]; then dmesg 2>/dev/null | scrub >"$d/dmesg-boot0.txt" || true; fi + grep -aiE "$RE_GPU_LINES" "$log" >"$d/gpu-log.txt" || true + grep -aE "$RE_FW_LOG" "$log" >"$d/firmware-log.txt" || true + + # The analysed boot's job records. + mkdir -p "$d/jobs" + mapfile -t recs < <(job_records "$b" "$home") + rec="" + for f in "${recs[@]}"; do cp "$f" "$d/jobs/"; rec=$f; done + + # Judged without the command-line line, which names the knobs. + grep -av 'Kernel command line:' "$log" >"$dir/judged.txt" || true + classify "$dir/judged.txt" "$cmd" "$rec" + + { + echo "== analysed boot: $boot (boot id ${b:-unknown})" + echo "cmdline: $cmd" + echo "oneshot: ${id:-none}" + echo "knobs: $(knobs_of "$cmd")" + echo "== this boot" + uname -a + echo "cmdline: $cmd0" + echo "board: ${board:-?} compatible: $(tr '\0' ' ' <"$DT/compatible" 2>/dev/null)" + pacman -Q linux-aurora m1n1-aurora 2>/dev/null || true + pacman -Qq 2>/dev/null | grep -i 'mesa' | xargs -r pacman -Q 2>/dev/null || true + printf 'm3-mode: '; cat /var/lib/aurora-sep/m3-mode 2>/dev/null || echo - + for f in "$DT"/soc/gpu@*; do + [[ -e $f/compatible ]] || continue + echo "gpu node ${f##*/}: $(tr '\0' ' ' <"$f/compatible") status=$(tr -d '\0' <"$f/status" 2>/dev/null || echo okay)" + done + echo "chosen:" + for f in "$DT"/chosen/asahi,t8122-* "$DT"/chosen/asahi,t6030-*; do [[ -e $f ]] && echo " ${f##*/}"; done + } >"$d/system.txt" 2>&1 + + # The one-shot's state now: the variable, the armed record, the boot entry. + { + printf 'LoaderEntryOneShot: ' + if [[ -e $EFIVARS/LoaderEntryOneShot-$BLI_GUID ]]; then + tail -c +5 "$EFIVARS/LoaderEntryOneShot-$BLI_GUID" | iconv -f UTF-16LE -t UTF-8 2>/dev/null | tr -d '\0'; echo + else echo "not set"; fi + printf 'LoaderEntrySelected (this boot): ' + tail -c +5 "$EFIVARS/LoaderEntrySelected-$BLI_GUID" 2>/dev/null | iconv -f UTF-16LE -t UTF-8 2>/dev/null | tr -d '\0'; echo + printf 'last armed: '; cat "$STATE_DIR/armed" 2>/dev/null || echo - + command -v air-gpu-oneshot.sh >/dev/null && air-gpu-oneshot.sh --status 2>&1 || true + } >"$d/oneshot.txt" 2>&1 + + # The asahi parameters: the ones sysfs shows, and the ones the analysed boot asked for. + { + echo "== /sys/module/asahi/parameters" + for f in "$SYS"/module/asahi/parameters/*; do [[ -r $f ]] && echo "${f##*/}=$(cat "$f" 2>/dev/null)"; done + echo "== asked for on the command line (boot $boot)" + tr ' ' '\n' <<<"$cmd" | grep '^asahi\.' || echo - + } >"$d/asahi-params.txt" 2>&1 + { + for f in progress activity memory timing; do + echo "== $DEBUGFS/asahi-m3/$f" + head -c 65536 "$DEBUGFS/asahi-m3/$f" 2>&1 || true + done + } >"$d/debugfs-asahi-m3.txt" 2>&1 + for f in "$SYS"/class/devcoredump/devcd*; do + [[ -e $f/data ]] || continue + echo "${f##*/}: failing_device=$(readlink -f "$f/failing_device" 2>/dev/null)" >>"$d/devcoredump.txt" + head -c 25165824 "$f/data" >"$d/${f##*/}.bin" 2>/dev/null || true + done + + # Every SMC T* and P* key, with its value (mC, mW), from this boot. + kf=$(smc_keys_file) + if [[ -n $kf ]]; then + { head -1 "$kf"; awk '$2 ~ /^[TP]/' "$kf"; } >"$d/smc-keys.txt" 2>&1 || true + else + echo "no SMC key list in debugfs" >"$d/smc-keys.txt" + fi + + { + echo "== /dev/dri"; ls -l /dev/dri 2>&1 || true + for f in "$SYS"/class/drm/renderD*; do + [[ -e $f/device/driver ]] && echo "${f##*/}: driver $(basename "$(readlink -f "$f/device/driver")")" + done + } >"$d/render-node.txt" 2>&1 + if grep -q 'driver asahi' "$d/render-node.txt"; then n=present; else n=absent; fi + + n_tg=$(awk '$2 ~ /^Tg/' "$d/smc-keys.txt" | wc -l) + live=$(awk '$2 ~ /^Tg/ && $6 + 0 >= 10000' "$d/smc-keys.txt" | wc -l) + rails=$(for k in $GPU_RAIL_CANDIDATES; do awk -v k="$k" '$2 == k { printf "%s=%s ", k, $6 }' "$d/smc-keys.txt"; done) + { + echo "AIR-GPU VERDICT: $VERDICT | boot $boot | oneshot ${id:-none} | $REASON" + echo "knobs: $(knobs_of "$cmd")" + echo "kernel armed with: $({ grep -aoE "$RE_K1_ARMED.*" "$log" || true; } | tail -1 | sed -n 's/^.*admitted): //p')" + echo "render node: $n" + echo "SMC (this boot): $live of $n_tg Tg* keys read 10 C or more; GPU rail candidates (mW): ${rails:-none found}" + echo "job records: ${#recs[@]}" + } >"$d/summary.txt" + + mv "$d" "$dir/$(basename "$out")" + tar -C "$dir" -czf "$out.tgz" "$(basename "$out")" + chown "$user": "$out.tgz" 2>/dev/null || true + say "wrote $out.tgz" + sed -n '2,6p' "$dir/$(basename "$out")/summary.txt" + head -1 "$dir/$(basename "$out")/summary.txt" +} + +if [[ ${AIR_GPU_SOURCE_ONLY:-} == 1 ]]; then return 0; fi +main "$@" diff --git a/tools/aurora-sep/air-gpu/air-gpu-job.sh b/tools/aurora-sep/air-gpu/air-gpu-job.sh new file mode 100755 index 0000000000000..4136f901a27cd --- /dev/null +++ b/tools/aurora-sep/air-gpu/air-gpu-job.sh @@ -0,0 +1,482 @@ +#!/bin/bash +# air-gpu-job.sh: run ONE short headless GPU compute job through a given Mesa prefix, and record +# its exit code and timing. +# +# air-gpu-job.sh [--seconds N] N: 1 to 8 seconds of submissions (default 5) +# +# The job is a Vulkan compute shader (a 32-bit LCG per invocation, 65536 invocations per +# dispatch), submitted once every 10 ms until N seconds have passed. Each submission is waited +# for with a 2 s fence timeout, and a sample of every result is checked on the CPU. The work per +# dispatch is tiny, so the GPU is idle most of the time: this is a dispatch test, not a load. +# It uses only the prefix's Vulkan driver, opens no window and needs no desktop. +# +# The whole run, start-up included, is held to 10 s: past that the job is killed, and the script +# returns even if the kernel holds the process. Run it as root (sudo) or as the desktop user. +# +# The record goes to ~/air-gpu-runs/job-